diff --git a/.asf.yaml b/.asf.yaml
index 91483dfed336c..e1c11790c8bfb 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -22,4 +22,4 @@ notifications:
   commits:      common-commits@hadoop.apache.org
   issues:       common-issues@hadoop.apache.org
   pullrequests: common-issues@hadoop.apache.org
-  jira_options: link label worklog
\ No newline at end of file
+  jira_options: comment link label
\ No newline at end of file
diff --git a/.yetus/excludes.txt b/.yetus/excludes.txt
new file mode 100644
index 0000000000000..0064dc8a3a4bb
--- /dev/null
+++ b/.yetus/excludes.txt
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dev-support/docker/Dockerfile_windows_10
diff --git a/BUILDING.txt b/BUILDING.txt
index edf47c5f1337a..b872d7e41944f 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -57,7 +57,7 @@ Refer to  dev-support/docker/Dockerfile):
 
 * Open JDK 1.8
   $ sudo apt-get update
-  $ sudo apt-get -y install java-8-openjdk
+  $ sudo apt-get -y install openjdk-8-jdk
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
@@ -492,39 +492,66 @@ Building on CentOS 8
 
 ----------------------------------------------------------------------------------
 
-Building on Windows
+Building on Windows 10
 
 ----------------------------------------------------------------------------------
 Requirements:
 
-* Windows System
+* Windows 10
 * JDK 1.8
-* Maven 3.0 or later
-* Boost 1.72
-* Protocol Buffers 3.7.1
-* CMake 3.19 or newer
-* Visual Studio 2010 Professional or Higher
-* Windows SDK 8.1 (if building CPU rate control for the container executor)
-* zlib headers (if building native code bindings for zlib)
+* Maven 3.0 or later (maven.apache.org)
+* Boost 1.72 (boost.org)
+* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
+* CMake 3.19 or newer (cmake.org)
+* Visual Studio 2019 (visualstudio.com)
+* Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from
+                   http://msdn.microsoft.com/en-us/windows/bg162891.aspx)
+* Zlib (zlib.net, if building native code bindings for zlib)
+* Git (preferably, get this from https://git-scm.com/download/win since the package also contains
+       Unix command-line tools that are needed during packaging).
+* Python (python.org, for generation of docs using 'mvn site')
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
-* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
-  tools must be present on your PATH.
-* Python ( for generation of docs using 'mvn site')
 
-Unix command-line tools are also included with the Windows Git package which
-can be downloaded from http://git-scm.com/downloads
+----------------------------------------------------------------------------------
 
-If using Visual Studio, it must be Professional level or higher.
-Do not use Visual Studio Express.  It does not support compiling for 64-bit,
-which is problematic if running a 64-bit system.
+Building guidelines:
 
-The Windows SDK 8.1 is available to download at:
+Hadoop repository provides the Dockerfile for building Hadoop on Windows 10, located at
+dev-support/docker/Dockerfile_windows_10. It is highly recommended to use this and create the
+Docker image for building Hadoop on Windows 10, since you don't have to install anything else
+other than Docker and no additional steps are required in terms of aligning the environment with
+the necessary paths etc.
 
-http://msdn.microsoft.com/en-us/windows/bg162891.aspx
+However, if you still prefer taking the route of not using Docker, this Dockerfile_windows_10 will
+still be immensely useful as a raw guide for all the steps involved in creating the environment
+needed to build Hadoop on Windows 10.
 
-Cygwin is not required.
+Building using the Docker:
+We first need to build the Docker image for building Hadoop on Windows 10. Run this command from
+the root of the Hadoop repository.
+> docker build -t hadoop-windows-10-builder -f .\dev-support\docker\Dockerfile_windows_10 .\dev-support\docker\
+
+Start the container with the image that we just built.
+> docker run --rm -it hadoop-windows-10-builder
+
+You can now clone the Hadoop repo inside this container and proceed with the build.
+
+NOTE:
+While one may perceive the idea of mounting the locally cloned (on the host filesystem) Hadoop
+repository into the container (using the -v option), we have seen the build to fail owing to some
+files not being able to be located by Maven. Thus, we suggest cloning the Hadoop repository to a
+non-mounted folder inside the container and proceed with the build. When the build is completed,
+you may use the "docker cp" command to copy the built Hadoop tar.gz file from the docker container
+to the host filesystem. If you still would like to mount the Hadoop codebase, a workaround would
+be to copy the mounted Hadoop codebase into another folder (which doesn't point to a mount) in the
+container's filesystem and use this for building.
+
+However, we noticed no build issues when the Maven repository from the host filesystem was mounted
+into the container. One may use this to greatly reduce the build time. Assuming that the Maven
+repository is located at D:\Maven\Repository in the host filesystem, one can use the following
+command to mount the same onto the default Maven repository location while launching the container.
+> docker run --rm -v D:\Maven\Repository:C:\Users\ContainerAdministrator\.m2\repository -it hadoop-windows-10-builder
 
-----------------------------------------------------------------------------------
 Building:
 
 Keep the source code tree in a short path to avoid running into problems related
@@ -540,6 +567,24 @@ configure the bit-ness of the build, and set several optional components.
 Several tests require that the user must have the Create Symbolic Links
 privilege.
 
+To simplify the installation of Boost, Protocol buffers, OpenSSL and Zlib dependencies we can use
+vcpkg (https://github.com/Microsoft/vcpkg.git). Upon cloning the vcpkg repo, checkout the commit
+7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d to get the required versions of the dependencies
+mentioned above.
+> git clone https://github.com/Microsoft/vcpkg.git
+> cd vcpkg
+> git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d
+> .\bootstrap-vcpkg.bat
+> .\vcpkg.exe install boost:x64-windows
+> .\vcpkg.exe install protobuf:x64-windows
+> .\vcpkg.exe install openssl:x64-windows
+> .\vcpkg.exe install zlib:x64-windows
+
+Set the following environment variables -
+(Assuming that vcpkg was checked out at C:\vcpkg)
+> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
+> set MAVEN_OPTS=-Xmx2048M -Xss128M
+
 All Maven goals are the same as described above with the exception that
 native code is built by enabling the 'native-win' Maven profile. -Pnative-win
 is enabled by default when building on Windows since the native components
@@ -557,6 +602,24 @@ the zlib 1.2.7 source tree.
 
 http://www.zlib.net/
 
+
+Build command:
+The following command builds all the modules in the Hadoop project and generates the tar.gz file in
+hadoop-dist/target upon successful build. Run these commands from an
+"x64 Native Tools Command Prompt for VS 2019" which can be found under "Visual Studio 2019" in the
+Windows start menu. If you're using the Docker image from Dockerfile_windows_10, you'll be
+logged into "x64 Native Tools Command Prompt for VS 2019" automatically when you start the
+container.
+
+> set classpath=
+> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
+> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist^
+    -Drequire.openssl -Drequire.test.libhadoop -Pyarn-ui -Dshell-executable=C:\Git\bin\bash.exe^
+    -Dtar -Dopenssl.prefix=C:\vcpkg\installed\x64-windows^
+    -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows^
+    -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo^
+    -Dwindows.build.hdfspp.dll=off -Dwindows.no.sasl=on -Duse.platformToolsetVersion=v142
+
 ----------------------------------------------------------------------------------
 Building distributions:
 
diff --git a/LICENSE-binary b/LICENSE-binary
index 824dbee29f6a9..988e38fa390d4 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -214,24 +214,24 @@ com.aliyun:aliyun-java-sdk-core:3.4.0
 com.aliyun:aliyun-java-sdk-ecs:4.2.0
 com.aliyun:aliyun-java-sdk-ram:3.0.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
-com.aliyun.oss:aliyun-sdk-oss:3.13.0
-com.amazonaws:aws-java-sdk-bundle:1.11.901
+com.aliyun.oss:aliyun-sdk-oss:3.13.2
+com.amazonaws:aws-java-sdk-bundle:1.12.316
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.9.9
-com.fasterxml.jackson.core:jackson-core:2.9.9
-com.fasterxml.jackson.core:jackson-databind:2.9.9.2
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.9.9
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.9
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.9.9
+com.fasterxml.jackson.core:jackson-annotations:2.12.7
+com.fasterxml.jackson.core:jackson-core:2.12.7
+com.fasterxml.jackson.core:jackson-databind:2.12.7.1
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
 com.fasterxml.uuid:java-uuid-generator:3.1.4
-com.fasterxml.woodstox:woodstox-core:5.3.0
+com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
 com.google:guice:4.0
 com.google:guice-servlet:4.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
-com.google.code.gson:2.2.4
+com.google.code.gson:2.9.0
 com.google.errorprone:error_prone_annotations:2.2.0
 com.google.j2objc:j2objc-annotations:1.1
 com.google.json-simple:json-simple:1.1.1
@@ -241,18 +241,17 @@ com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.8.1
-com.squareup.okhttp:okhttp:2.7.5
-com.squareup.okio:okio:1.6.0
+com.squareup.okhttp3:okhttp:4.10.0
+com.squareup.okio:okio:3.2.0
 com.zaxxer:HikariCP:4.0.3
-commons-beanutils:commons-beanutils:1.9.3
+commons-beanutils:commons-beanutils:1.9.4
 commons-cli:commons-cli:1.2
 commons-codec:commons-codec:1.11
 commons-collections:commons-collections:3.2.2
 commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.8.0
-commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.1.3
-commons-net:commons-net:3.6
+commons-net:commons-net:3.8.0
 de.ruedigermoeller:fst:2.50
 io.grpc:grpc-api:1.26.0
 io.grpc:grpc-context:1.26.0
@@ -262,17 +261,36 @@ io.grpc:grpc-protobuf:1.26.0
 io.grpc:grpc-protobuf-lite:1.26.0
 io.grpc:grpc-stub:1.26.0
 io.netty:netty:3.10.6.Final
-io.netty:netty-all:4.1.42.Final
-io.netty:netty-buffer:4.1.27.Final
-io.netty:netty-codec:4.1.27.Final
-io.netty:netty-codec-http:4.1.27.Final
-io.netty:netty-codec-http2:4.1.27.Final
-io.netty:netty-codec-socks:4.1.27.Final
-io.netty:netty-common:4.1.27.Final
-io.netty:netty-handler:4.1.27.Final
-io.netty:netty-handler-proxy:4.1.27.Final
-io.netty:netty-resolver:4.1.27.Final
-io.netty:netty-transport:4.1.27.Final
+io.netty:netty-all:4.1.77.Final
+io.netty:netty-buffer:4.1.77.Final
+io.netty:netty-codec:4.1.77.Final
+io.netty:netty-codec-dns:4.1.77.Final
+io.netty:netty-codec-haproxy:4.1.77.Final
+io.netty:netty-codec-http:4.1.77.Final
+io.netty:netty-codec-http2:4.1.77.Final
+io.netty:netty-codec-memcache:4.1.77.Final
+io.netty:netty-codec-mqtt:4.1.77.Final
+io.netty:netty-codec-redis:4.1.77.Final
+io.netty:netty-codec-smtp:4.1.77.Final
+io.netty:netty-codec-socks:4.1.77.Final
+io.netty:netty-codec-stomp:4.1.77.Final
+io.netty:netty-codec-xml:4.1.77.Final
+io.netty:netty-common:4.1.77.Final
+io.netty:netty-handler:4.1.77.Final
+io.netty:netty-handler-proxy:4.1.77.Final
+io.netty:netty-resolver:4.1.77.Final
+io.netty:netty-resolver-dns:4.1.77.Final
+io.netty:netty-transport:4.1.77.Final
+io.netty:netty-transport-rxtx:4.1.77.Final
+io.netty:netty-transport-sctp:4.1.77.Final
+io.netty:netty-transport-udt:4.1.77.Final
+io.netty:netty-transport-classes-epoll:4.1.77.Final
+io.netty:netty-transport-native-unix-common:4.1.77.Final
+io.netty:netty-transport-classes-kqueue:4.1.77.Final
+io.netty:netty-resolver-dns-classes-macos:4.1.77.Final
+io.netty:netty-transport-native-epoll:4.1.77.Final
+io.netty:netty-transport-native-kqueue:4.1.77.Final
+io.netty:netty-resolver-dns-native-macos:4.1.77.Final
 io.opencensus:opencensus-api:0.12.3
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
 io.reactivex:rxjava:1.3.8
@@ -284,70 +302,67 @@ log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 net.minidev:json-smart:2.4.7
-org.apache.avro:avro:1.7.7
+org.apache.avro:avro:1.9.2
 org.apache.commons:commons-collections4:4.2
 org.apache.commons:commons-compress:1.21
-org.apache.commons:commons-configuration2:2.1.1
-org.apache.commons:commons-csv:1.0
+org.apache.commons:commons-configuration2:2.8.0
+org.apache.commons:commons-csv:1.9.0
 org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.7
-org.apache.commons:commons-math3:3.1.1
-org.apache.commons:commons-text:1.4
+org.apache.commons:commons-lang3:3.12.0
+org.apache.commons:commons-math3:3.6.1
+org.apache.commons:commons-text:1.10.0
 org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
 org.apache.curator:curator-recipes:5.2.0
 org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
-org.apache.hbase:hbase-annotations:1.4.8
-org.apache.hbase:hbase-client:1.4.8
-org.apache.hbase:hbase-common:1.4.8
-org.apache.hbase:hbase-protocol:1.4.8
+org.apache.hbase:hbase-annotations:1.7.1
+org.apache.hbase:hbase-client:1.7.1
+org.apache.hbase:hbase-common:1.7.1
+org.apache.hbase:hbase-protocol:1.7.1
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
 org.apache.kafka:kafka-clients:2.8.1
-org.apache.kerby:kerb-admin:1.0.1
-org.apache.kerby:kerb-client:1.0.1
-org.apache.kerby:kerb-common:1.0.1
-org.apache.kerby:kerb-core:1.0.1
-org.apache.kerby:kerb-crypto:1.0.1
-org.apache.kerby:kerb-identity:1.0.1
-org.apache.kerby:kerb-server:1.0.1
-org.apache.kerby:kerb-simplekdc:1.0.1
-org.apache.kerby:kerb-util:1.0.1
-org.apache.kerby:kerby-asn1:1.0.1
-org.apache.kerby:kerby-config:1.0.1
-org.apache.kerby:kerby-pkix:1.0.1
-org.apache.kerby:kerby-util:1.0.1
-org.apache.kerby:kerby-xdr:1.0.1
-org.apache.kerby:token-provider:1.0.1
+org.apache.kerby:kerb-admin:2.0.2
+org.apache.kerby:kerb-client:2.0.2
+org.apache.kerby:kerb-common:2.0.2
+org.apache.kerby:kerb-core:2.0.2
+org.apache.kerby:kerb-crypto:2.0.2
+org.apache.kerby:kerb-identity:2.0.2
+org.apache.kerby:kerb-server:2.0.2
+org.apache.kerby:kerb-simplekdc:2.0.2
+org.apache.kerby:kerb-util:2.0.2
+org.apache.kerby:kerby-asn1:2.0.2
+org.apache.kerby:kerby-config:2.0.2
+org.apache.kerby:kerby-pkix:2.0.2
+org.apache.kerby:kerby-util:2.0.2
+org.apache.kerby:kerby-xdr:2.0.2
+org.apache.kerby:token-provider:2.0.2
+org.apache.solr:solr-solrj:8.8.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.6.3
-org.codehaus.jackson:jackson-core-asl:1.9.13
-org.codehaus.jackson:jackson-jaxrs:1.9.13
-org.codehaus.jackson:jackson-mapper-asl:1.9.13
-org.codehaus.jackson:jackson-xc:1.9.13
-org.codehaus.jettison:jettison:1.1
-org.eclipse.jetty:jetty-annotations:9.3.27.v20190418
-org.eclipse.jetty:jetty-http:9.3.27.v20190418
-org.eclipse.jetty:jetty-io:9.3.27.v20190418
-org.eclipse.jetty:jetty-jndi:9.3.27.v20190418
-org.eclipse.jetty:jetty-plus:9.3.27.v20190418
-org.eclipse.jetty:jetty-security:9.3.27.v20190418
-org.eclipse.jetty:jetty-server:9.3.27.v20190418
-org.eclipse.jetty:jetty-servlet:9.3.27.v20190418
-org.eclipse.jetty:jetty-util:9.3.27.v20190418
-org.eclipse.jetty:jetty-util-ajax:9.3.27.v20190418
-org.eclipse.jetty:jetty-webapp:9.3.27.v20190418
-org.eclipse.jetty:jetty-xml:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418
+org.codehaus.jettison:jettison:1.5.1
+org.eclipse.jetty:jetty-annotations:9.4.48.v20220622
+org.eclipse.jetty:jetty-http:9.4.48.v20220622
+org.eclipse.jetty:jetty-io:9.4.48.v20220622
+org.eclipse.jetty:jetty-jndi:9.4.48.v20220622
+org.eclipse.jetty:jetty-plus:9.4.48.v20220622
+org.eclipse.jetty:jetty-security:9.4.48.v20220622
+org.eclipse.jetty:jetty-server:9.4.48.v20220622
+org.eclipse.jetty:jetty-servlet:9.4.48.v20220622
+org.eclipse.jetty:jetty-util:9.4.48.v20220622
+org.eclipse.jetty:jetty-util-ajax:9.4.48.v20220622
+org.eclipse.jetty:jetty-webapp:9.4.48.v20220622
+org.eclipse.jetty:jetty-xml:9.4.48.v20220622
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.48.v20220622
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.48.v20220622
 org.ehcache:ehcache:3.3.1
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
-org.yaml:snakeyaml:1.16:
+org.yaml:snakeyaml:1.33
 org.wildfly.openssl:wildfly-openssl:1.0.7.Final
 
 
@@ -405,14 +420,14 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
 hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
 hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/*
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/*
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL
@@ -420,7 +435,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
 bootstrap v3.3.6
 broccoli-asset-rev v2.4.2
 broccoli-funnel v1.0.1
-datatables v1.10.8
+datatables v1.11.5
 em-helpers v0.5.13
 em-table v0.1.6
 ember v2.2.0
@@ -468,8 +483,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.60
-org.bouncycastle:bcprov-jdk15on:1.60
+org.bouncycastle:bcpkix-jdk15on:1.68
+org.bouncycastle:bcprov-jdk15on:1.68
 org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.17
 org.jruby.jcodings:jcodings:1.0.13
@@ -484,12 +499,12 @@ org.slf4j:slf4j-log4j12:1.7.25
 CDDL 1.1 + GPLv2 with classpath exception
 -----------------------------------------
 
-com.sun.jersey:jersey-client:1.19
-com.sun.jersey:jersey-core:1.19
-com.sun.jersey:jersey-guice:1.19
-com.sun.jersey:jersey-json:1.19
-com.sun.jersey:jersey-server:1.19
-com.sun.jersey:jersey-servlet:1.19
+com.github.pjfanning:jersey-json:1.20
+com.sun.jersey:jersey-client:1.19.4
+com.sun.jersey:jersey-core:1.19.4
+com.sun.jersey:jersey-guice:1.19.4
+com.sun.jersey:jersey-server:1.19.4
+com.sun.jersey:jersey-servlet:1.19.4
 com.sun.xml.bind:jaxb-impl:2.2.3-1
 javax.annotation:javax.annotation-api:1.3.2
 javax.servlet:javax.servlet-api:3.1.0
@@ -508,13 +523,13 @@ junit:junit:4.13.2
 HSQL License
 ------------
 
-org.hsqldb:hsqldb:2.3.4
+org.hsqldb:hsqldb:2.7.1
 
 
 JDOM License
 ------------
 
-org.jdom:jdom:1.1
+org.jdom:jdom2:2.0.6.1
 
 
 Public Domain
diff --git a/LICENSE.txt b/LICENSE.txt
index 3c079898b9071..2dfc0b9da47a7 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -245,14 +245,14 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
 hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
 hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/*
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/*
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL
diff --git a/NOTICE-binary b/NOTICE-binary
index 2f8a9241a8d00..b96e052658876 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
 
 
 AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 This product includes software developed by
 Amazon Technologies, Inc (http://www.amazon.com/).
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 7896810d017d5..51225268b653a 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -15,13 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-def getGithubAndJiraCreds() {
+def getGithubCreds() {
     return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
                                 passwordVariable: 'GITHUB_TOKEN',
-                                usernameVariable: 'GITHUB_USER'),
-            usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
-                                passwordVariable: 'JIRA_PASSWORD',
-                                usernameVariable: 'JIRA_USER')]
+                                usernameVariable: 'GITHUB_USER')]
 }
 
 // Publish JUnit results only if there are XML files under surefire-reports
@@ -50,7 +47,7 @@ pipeline {
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '5'))
-        timeout (time: 24, unit: 'HOURS')
+        timeout (time: 48, unit: 'HOURS')
         timestamps()
         checkoutToSubdirectory('src')
     }
@@ -58,7 +55,7 @@ pipeline {
     environment {
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
+        YETUS_VERSION='rel/0.14.0'
     }
 
     parameters {
@@ -130,7 +127,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -176,7 +173,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -222,7 +219,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
@@ -265,7 +262,7 @@ pipeline {
             }
 
             steps {
-                withCredentials(getGithubAndJiraCreds()) {
+                withCredentials(getGithubCreds()) {
                     sh '''#!/usr/bin/env bash
 
                     chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index 31ae6ee1b0659..693b41c4f3910 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -293,6 +293,7 @@ function usage
   echo "--security              Emergency security release"
   echo "--sign                  Use .gnupg dir to sign the artifacts and jars"
   echo "--version=[version]     Use an alternative version string"
+  echo "--mvnargs=[args]        Extra Maven args to be provided when running mvn commands"
 }
 
 function option_parse
@@ -347,6 +348,9 @@ function option_parse
       --version=*)
         HADOOP_VERSION=${i#*=}
       ;;
+      --mvnargs=*)
+        MVNEXTRAARGS=${i#*=}
+      ;;
     esac
   done
 
@@ -413,6 +417,9 @@ function option_parse
       MVN_ARGS=("-Dmaven.repo.local=${MVNCACHE}")
     fi
   fi
+  if [ -n "$MVNEXTRAARGS" ]; then
+    MVN_ARGS+=("$MVNEXTRAARGS")
+  fi
 
   if [[ "${SECURITYRELEASE}" = true ]]; then
     if [[ ! -d "${BASEDIR}/hadoop-common-project/hadoop-common/src/site/markdown/release/${HADOOP_VERSION}" ]]; then
@@ -535,6 +542,10 @@ function makearelease
 
   big_console_header "Cleaning the Source Tree"
 
+  # Since CVE-2022-24765 in April 2022, git refuses to work in directories
+  # whose owner != the current user, unless explicitly told to trust it.
+  git config --global --add safe.directory /build/source
+
   # git clean to clear any remnants from previous build
   run "${GIT}" clean -xdf -e /patchprocess
 
diff --git a/dev-support/bin/yetus-wrapper b/dev-support/bin/yetus-wrapper
index 8532d1749701b..a93833767a201 100755
--- a/dev-support/bin/yetus-wrapper
+++ b/dev-support/bin/yetus-wrapper
@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.13.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8
index 7b82c4997dee6..8f3b008f7ba03 100644
--- a/dev-support/docker/Dockerfile_centos_8
+++ b/dev-support/docker/Dockerfile_centos_8
@@ -30,6 +30,13 @@ COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
     && chmod a+r pkg-resolver/*.json
 
+######
+# Centos 8 has reached its EOL and the packages
+# are no longer available on mirror.centos.org site.
+# Please see https://www.centos.org/centos-linux-eol/
+######
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
+
 ######
 # Install packages from yum
 ######
diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10
index 256f0d5786ab9..ec3de11035cee 100644
--- a/dev-support/docker/Dockerfile_debian_10
+++ b/dev-support/docker/Dockerfile_debian_10
@@ -82,6 +82,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 ####
 # Install packages
 ####
+RUN pkg-resolver/install-cmake.sh debian:10
 RUN pkg-resolver/install-spotbugs.sh debian:10
 RUN pkg-resolver/install-boost.sh debian:10
 RUN pkg-resolver/install-protobuf.sh debian:10
diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10
new file mode 100644
index 0000000000000..7a69a2727ae50
--- /dev/null
+++ b/dev-support/docker/Dockerfile_windows_10
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM mcr.microsoft.com/windows:ltsc2019
+
+# Need to disable the progress bar for speeding up the downloads.
+# hadolint ignore=SC2086
+RUN powershell $Global:ProgressPreference = 'SilentlyContinue'
+
+# Restore the default Windows shell for correct batch processing.
+SHELL ["cmd", "/S", "/C"]
+
+# Install Visual Studio 2019 Build Tools.
+RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe \
+    && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache \
+    --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\BuildTools" \
+    --add Microsoft.VisualStudio.Workload.VCTools \
+    --add Microsoft.VisualStudio.Component.VC.ASAN \
+    --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \
+    --add Microsoft.VisualStudio.Component.Windows10SDK.19041 \
+    || IF "%ERRORLEVEL%"=="3010" EXIT 0) \
+    && del /q vs_buildtools.exe
+
+# Install Chocolatey.
+RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
+RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
+
+# Install git.
+RUN choco install git.install -y
+RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\Git
+
+# Install vcpkg.
+# hadolint ignore=DL3003
+RUN powershell git clone https://github.com/microsoft/vcpkg.git \
+    && cd vcpkg \
+    && git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
+    && .\bootstrap-vcpkg.bat
+RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
+RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
+RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
+RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
+ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
+
+# Install Azul Java 8 JDK.
+RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
+ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
+RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
+
+# Install Apache Maven.
+RUN powershell Invoke-WebRequest -URI https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
+RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
+ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
+
+# Install CMake 3.19.0.
+RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
+RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
+
+# We get strange Javadoc errors without this.
+RUN setx classpath ""
+
+# Define the entry point for the docker container.
+ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]
diff --git a/dev-support/docker/pkg-resolver/install-maven.sh b/dev-support/docker/pkg-resolver/install-maven.sh
index f9ff961a190f9..d1d0dc97fe5e4 100644
--- a/dev-support/docker/pkg-resolver/install-maven.sh
+++ b/dev-support/docker/pkg-resolver/install-maven.sh
@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "3.6.3" ]; then
   mkdir -p /opt/maven /tmp/maven &&
-    curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+    curl -L -s -S https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
       -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
     tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
 else
diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json
index afe8a7a32b107..2225517834601 100644
--- a/dev-support/docker/pkg-resolver/packages.json
+++ b/dev-support/docker/pkg-resolver/packages.json
@@ -62,7 +62,6 @@
     "centos:8": "clang"
   },
   "cmake": {
-    "debian:10": "cmake",
     "ubuntu:focal": "cmake",
     "ubuntu:focal::arch64": "cmake"
   },
diff --git a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
new file mode 100644
index 0000000000000..4be4cd956b15b
--- /dev/null
+++ b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
+  cd /etc/yum.repos.d/ || exit &&
+    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
+    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
+    yum update -y &&
+    cd /root || exit
+else
+  echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments"
+  exit 1
+fi
diff --git a/dev-support/git-jira-validation/README.md b/dev-support/git-jira-validation/README.md
new file mode 100644
index 0000000000000..308c54228d17c
--- /dev/null
+++ b/dev-support/git-jira-validation/README.md
@@ -0,0 +1,134 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Apache Hadoop Git/Jira FixVersion validation
+============================================================
+
+Git commits in Apache Hadoop contains Jira number of the format
+HADOOP-XXXX or HDFS-XXXX or YARN-XXXX or MAPREDUCE-XXXX.
+While creating a release candidate, we also include changelist
+and this changelist can be identified based on Fixed/Closed Jiras
+with the correct fix versions. However, sometimes we face few
+inconsistencies between fixed Jira and Git commit message.
+
+git_jira_fix_version_check.py script takes care of
+identifying all git commits with commit
+messages with any of these issues:
+
+1. commit is reverted as per commit message
+2. commit does not contain Jira number format in message
+3. Jira does not have expected fixVersion
+4. Jira has expected fixVersion, but it is not yet resolved
+
+Moreover, this script also finds any resolved Jira with expected
+fixVersion but without any corresponding commit present.
+
+This should be useful as part of RC preparation.
+
+git_jira_fix_version_check supports python3 and it required
+installation of jira:
+
+```
+$ python3 --version
+Python 3.9.7
+
+$ python3 -m venv ./venv
+
+$ ./venv/bin/pip install -r dev-support/git-jira-validation/requirements.txt
+
+$ ./venv/bin/python dev-support/git-jira-validation/git_jira_fix_version_check.py
+
+```
+
+The script also requires below inputs:
+```
+1. First commit hash to start excluding commits from history:
+   Usually we can provide latest commit hash from last tagged release
+   so that the script will only loop through all commits in git commit
+   history before this commit hash. e.g for 3.3.2 release, we can provide
+   git hash: fa4915fdbbbec434ab41786cb17b82938a613f16
+   because this commit bumps up hadoop pom versions to 3.3.2:
+   https://github.com/apache/hadoop/commit/fa4915fdbbbec434ab41786cb17b82938a613f16
+
+2. Fix Version:
+   Exact fixVersion that we would like to compare all Jira's fixVersions
+   with. e.g for 3.3.2 release, it should be 3.3.2.
+
+3. JIRA Project Name:
+   The exact name of Project as case-sensitive e.g HADOOP / OZONE
+
+4. Path of project's working dir with release branch checked-in:
+   Path of project from where we want to compare git hashes from. Local fork
+   of the project should be up-to date with upstream and expected release
+   branch should be checked-in.
+
+5. Jira server url (default url: https://issues.apache.org/jira):
+   Default value of server points to ASF Jiras but this script can be
+   used outside of ASF Jira too.
+```
+
+
+Example of script execution:
+```
+JIRA Project Name (e.g HADOOP / OZONE etc): HADOOP
+First commit hash to start excluding commits from history: fa4915fdbbbec434ab41786cb17b82938a613f16
+Fix Version: 3.3.2
+Jira server url (default: https://issues.apache.org/jira):
+Path of project's working dir with release branch checked-in: /Users/vjasani/Documents/src/hadoop-3.3/hadoop
+
+Check git status output and verify expected branch
+
+On branch branch-3.3.2
+Your branch is up to date with 'origin/branch-3.3.2'.
+
+nothing to commit, working tree clean
+
+
+Jira/Git commit message diff starting: ##############################################
+Jira not present with version: 3.3.2. 	 Commit: 8cd8e435fb43a251467ca74fadcb14f21a3e8163 HADOOP-17198. Support S3 Access Points  (#3260) (branch-3.3.2) (#3955)
+WARN: Jira not found. 			 Commit: 8af28b7cca5c6020de94e739e5373afc69f399e5 Updated the index as per 3.3.2 release
+WARN: Jira not found. 			 Commit: e42e483d0085aa46543ebcb1196dd155ddb447d0 Make upstream aware of 3.3.1 release
+Commit seems reverted. 			 Commit: 6db1165380cd308fb74c9d17a35c1e57174d1e09 Revert "HDFS-14099. Unknown frame descriptor when decompressing multiple frames (#3836)"
+Commit seems reverted. 			 Commit: 1e3f94fa3c3d4a951d4f7438bc13e6f008f228f4 Revert "HDFS-16333. fix balancer bug when transfer an EC block (#3679)"
+Jira not present with version: 3.3.2. 	 Commit: ce0bc7b473a62a580c1227a4de6b10b64b045d3a HDFS-16344. Improve DirectoryScanner.Stats#toString (#3695)
+Jira not present with version: 3.3.2. 	 Commit: 30f0629d6e6f735c9f4808022f1a1827c5531f75 HDFS-16339. Show the threshold when mover threads quota is exceeded (#3689)
+Jira not present with version: 3.3.2. 	 Commit: e449daccf486219e3050254d667b74f92e8fc476 YARN-11007. Correct words in YARN documents (#3680)
+Commit seems reverted. 			 Commit: 5c189797828e60a3329fd920ecfb99bcbccfd82d Revert "HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686)"
+Jira not present with version: 3.3.2. 	 Commit: 544dffd179ed756bc163e4899e899a05b93d9234 HDFS-16171. De-flake testDecommissionStatus (#3280)
+Jira not present with version: 3.3.2. 	 Commit: c6914b1cb6e4cab8263cd3ae5cc00bc7a8de25de HDFS-16350. Datanode start time should be set after RPC server starts successfully (#3711)
+Jira not present with version: 3.3.2. 	 Commit: 328d3b84dfda9399021ccd1e3b7afd707e98912d HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686)
+Jira not present with version: 3.3.2. 	 Commit: 3ae8d4ccb911c9ababd871824a2fafbb0272c016 HDFS-16336. De-flake TestRollingUpgrade#testRollback (#3686)
+Jira not present with version: 3.3.2. 	 Commit: 15d3448e25c797b7d0d401afdec54683055d4bb5 HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579)
+Jira not present with version: 3.3.2. 	 Commit: dd50261219de71eaa0a1ad28529953e12dfb92e0 YARN-10991. Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method (#3592)
+Jira not present with version: 3.3.2. 	 Commit: ef462b21bf03b10361d2f9ea7b47d0f7360e517f HDFS-16332. Handle invalid token exception in sasl handshake (#3677)
+WARN: Jira not found. 			 Commit: b55edde7071419410ea5bea4ce6462b980e48f5b Also update hadoop.version to 3.3.2
+...
+...
+...
+Found first commit hash after which git history is redundant. commit: fa4915fdbbbec434ab41786cb17b82938a613f16
+Exiting successfully
+Jira/Git commit message diff completed: ##############################################
+
+Any resolved Jira with fixVersion 3.3.2 but corresponding commit not present
+Starting diff: ##############################################
+HADOOP-18066 is marked resolved with fixVersion 3.3.2 but no corresponding commit found
+HADOOP-17936 is marked resolved with fixVersion 3.3.2 but no corresponding commit found
+Completed diff: ##############################################
+
+
+```
+
diff --git a/dev-support/git-jira-validation/git_jira_fix_version_check.py b/dev-support/git-jira-validation/git_jira_fix_version_check.py
new file mode 100644
index 0000000000000..513cc8e25ffdb
--- /dev/null
+++ b/dev-support/git-jira-validation/git_jira_fix_version_check.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+"""An application to assist Release Managers with ensuring that histories in
+Git and fixVersions in JIRA are in agreement. See README.md for a detailed
+explanation.
+"""
+
+
+import os
+import re
+import subprocess
+
+from jira import JIRA
+
+jira_project_name = input("JIRA Project Name (e.g HADOOP / OZONE etc): ") \
+                    or "HADOOP"
+# Define project_jira_keys with - appended. e.g for HADOOP Jiras,
+# project_jira_keys should include HADOOP-, HDFS-, YARN-, MAPREDUCE-
+project_jira_keys = [jira_project_name + '-']
+if jira_project_name == 'HADOOP':
+    project_jira_keys.append('HDFS-')
+    project_jira_keys.append('YARN-')
+    project_jira_keys.append('MAPREDUCE-')
+
+first_exclude_commit_hash = input("First commit hash to start excluding commits from history: ")
+fix_version = input("Fix Version: ")
+
+jira_server_url = input(
+    "Jira server url (default: https://issues.apache.org/jira): ") \
+        or "https://issues.apache.org/jira"
+
+jira = JIRA(server=jira_server_url)
+
+local_project_dir = input("Path of project's working dir with release branch checked-in: ")
+os.chdir(local_project_dir)
+
+GIT_STATUS_MSG = subprocess.check_output(['git', 'status']).decode("utf-8")
+print('\nCheck git status output and verify expected branch\n')
+print(GIT_STATUS_MSG)
+
+print('\nJira/Git commit message diff starting: ##############################################')
+
+issue_set_from_commit_msg = set()
+
+for commit in subprocess.check_output(['git', 'log', '--pretty=oneline']).decode(
+        "utf-8").splitlines():
+    if commit.startswith(first_exclude_commit_hash):
+        print("Found first commit hash after which git history is redundant. commit: "
+              + first_exclude_commit_hash)
+        print("Exiting successfully")
+        break
+    if re.search('revert', commit, re.IGNORECASE):
+        print("Commit seems reverted. \t\t\t Commit: " + commit)
+        continue
+    ACTUAL_PROJECT_JIRA = None
+    matches = re.findall('|'.join(project_jira_keys), commit)
+    if matches:
+        ACTUAL_PROJECT_JIRA = matches[0]
+    if not ACTUAL_PROJECT_JIRA:
+        print("WARN: Jira not found. \t\t\t Commit: " + commit)
+        continue
+    JIRA_NUM = ''
+    for c in commit.split(ACTUAL_PROJECT_JIRA)[1]:
+        if c.isdigit():
+            JIRA_NUM = JIRA_NUM + c
+        else:
+            break
+    issue = jira.issue(ACTUAL_PROJECT_JIRA + JIRA_NUM)
+    EXPECTED_FIX_VERSION = False
+    for version in issue.fields.fixVersions:
+        if version.name == fix_version:
+            EXPECTED_FIX_VERSION = True
+            break
+    if not EXPECTED_FIX_VERSION:
+        print("Jira not present with version: " + fix_version + ". \t Commit: " + commit)
+        continue
+    if issue.fields.status is None or issue.fields.status.name not in ('Resolved', 'Closed'):
+        print("Jira is not resolved yet? \t\t Commit: " + commit)
+    else:
+        # This means Jira corresponding to current commit message is resolved with expected
+        # fixVersion.
+        # This is no-op by default, if needed, convert to print statement.
+        issue_set_from_commit_msg.add(ACTUAL_PROJECT_JIRA + JIRA_NUM)
+
+print('Jira/Git commit message diff completed: ##############################################')
+
+print('\nAny resolved Jira with fixVersion ' + fix_version
+      + ' but corresponding commit not present')
+print('Starting diff: ##############################################')
+all_issues_with_fix_version = jira.search_issues(
+    'project=' + jira_project_name + ' and status in (Resolved,Closed) and fixVersion='
+    + fix_version)
+
+for issue in all_issues_with_fix_version:
+    if issue.key not in issue_set_from_commit_msg:
+        print(issue.key + ' is marked resolved with fixVersion ' + fix_version
+            + ' but no corresponding commit found')
+
+print('Completed diff: ##############################################')
diff --git a/dev-support/git-jira-validation/requirements.txt b/dev-support/git-jira-validation/requirements.txt
new file mode 100644
index 0000000000000..ae7535a119fa9
--- /dev/null
+++ b/dev-support/git-jira-validation/requirements.txt
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+jira==3.1.1
diff --git a/dev-support/hadoop-vote.sh b/dev-support/hadoop-vote.sh
new file mode 100755
index 0000000000000..825e776fabf16
--- /dev/null
+++ b/dev-support/hadoop-vote.sh
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script is useful to perform basic sanity tests for the given
+# Hadoop RC. It checks for the Checksum, Signature, Rat check,
+# Build from source and building tarball from the source.
+
+set -e -o pipefail
+
+usage() {
+  SCRIPT=$(basename "${BASH_SOURCE[@]}")
+
+  cat << __EOF
+hadoop-vote. A script for standard vote which verifies the following items
+1. Checksum of sources and binaries
+2. Signature of sources and binaries
+3. Rat check
+4. Built from source
+5. Built tar from source
+
+Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>] [-D property[=value]] [-P profiles]
+       ${SCRIPT} -h | --help
+
+  -h | --help                   Show this screen.
+  -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
+                                e.g. https://dist.apache.org/repos/dist/dev/hadoop/hadoop-<version>RC0/
+  -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
+  -f | --keys-file-url '<url>'   the URL of the key file, default is
+                                https://downloads.apache.org/hadoop/common/KEYS
+  -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
+  -D |                          list of maven properties to set for the mvn invocations, e.g. <-D hbase.profile=2.0 -D skipTests> Defaults to unset
+  -P |                          list of maven profiles to set for the build from source, e.g. <-P native -P yarn-ui>
+__EOF
+}
+
+MVN_PROPERTIES=()
+MVN_PROFILES=()
+
+while ((${#})); do
+  case "${1}" in
+    -h | --help )
+      usage; exit 0 ;;
+    -s | --source )
+      SOURCE_URL="${2}"; shift 2 ;;
+    -k | --key )
+      SIGNING_KEY="${2}"; shift 2 ;;
+    -f | --keys-file-url )
+      KEY_FILE_URL="${2}"; shift 2 ;;
+    -o | --output-dir )
+      OUTPUT_DIR="${2}"; shift 2 ;;
+    -D )
+      MVN_PROPERTIES+=("-D ${2}"); shift 2 ;;
+    -P )
+      MVN_PROFILES+=("-P ${2}"); shift 2 ;;
+    * )
+      usage >&2; exit 1             ;;
+  esac
+done
+
+# Source url must be provided
+if [ -z "${SOURCE_URL}" ]; then
+  usage;
+  exit 1
+fi
+
+cat << __EOF
+Although This tool helps verifying Hadoop RC build and unit tests,
+operator may still consider verifying the following manually:
+1. Verify the API compatibility report
+2. Integration/performance/benchmark tests
+3. Object store specific Integration tests against an endpoint
+4. Verify overall unit test stability from Jenkins builds or locally
+5. Other concerns if any
+__EOF
+
+[[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
+HADOOP_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
+HADOOP_VERSION=$(echo "${HADOOP_RC_VERSION}" | sed -e 's/-RC[0-9]//g' | sed -e 's/hadoop-//g')
+JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
+OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
+
+if [ ! -d "${OUTPUT_DIR}" ]; then
+    echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
+    exit 1
+fi
+
+OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HADOOP_RC_VERSION}"
+
+# default value for verification targets, 0 = failed
+SIGNATURE_PASSED=0
+CHECKSUM_PASSED=0
+RAT_CHECK_PASSED=0
+BUILD_FROM_SOURCE_PASSED=0
+BUILD_TAR_FROM_SOURCE_PASSED=0
+
+function download_and_import_keys() {
+    KEY_FILE_URL="${KEY_FILE_URL:-https://downloads.apache.org/hadoop/common/KEYS}"
+    echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
+    # download the keys file into file KEYS
+    wget -O KEYS "${KEY_FILE_URL}"
+    gpg --import KEYS
+    if [ -n "${SIGNING_KEY}" ]; then
+        gpg --list-keys "${SIGNING_KEY}"
+    fi
+}
+
+function download_release_candidate () {
+    # get all files from release candidate repo
+    wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}"
+}
+
+function verify_signatures() {
+    rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
+    for file in *.tar.gz; do
+        gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
+    done
+}
+
+function verify_checksums() {
+    rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
+    SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
+    for file in *.tar.gz; do
+        sha512sum --tag "${file}" > "${file}"."${SHA_EXT}".tmp
+        diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
+        rm -f "${file}"."${SHA_EXT}".tmp
+    done
+}
+
+function unzip_from_source() {
+    tar -zxvf hadoop-"${HADOOP_VERSION}"-src.tar.gz
+    cd hadoop-"${HADOOP_VERSION}"-src
+}
+
+function rat_test() {
+    rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
+    mvn clean apache-rat:check "${MVN_PROPERTIES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
+}
+
+function build_from_source() {
+    rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
+    # No unit test run.
+    mvn clean install "${MVN_PROPERTIES[@]}" -DskipTests "${MVN_PROFILES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
+}
+
+function build_tar_from_source() {
+    rm -f "${OUTPUT_PATH_PREFIX}"_build_tar_from_source
+    # No unit test run.
+    mvn clean package "${MVN_PROPERTIES[@]}" -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_tar_from_source && BUILD_TAR_FROM_SOURCE_PASSED=1
+}
+
+function execute() {
+   ${1} || print_when_exit
+}
+
+function print_when_exit() {
+  cat << __EOF
+        * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
+        * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
+        * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
+         - mvn clean apache-rat:check ${MVN_PROPERTIES[@]}
+        * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
+         - mvn clean install ${MVN_PROPERTIES[@]} -DskipTests ${MVN_PROFILES[@]}
+        * Built tar from source (${JAVA_VERSION}): $( ((BUILD_TAR_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
+         - mvn clean package ${MVN_PROPERTIES[@]} -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true
+__EOF
+  if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((BUILD_TAR_FROM_SOURCE_PASSED)) ; then
+    exit 0
+  fi
+  exit 1
+}
+
+pushd "${OUTPUT_DIR}"
+
+download_and_import_keys
+download_release_candidate
+
+pushd "${HADOOP_RC_VERSION}"
+
+execute verify_signatures
+execute verify_checksums
+execute unzip_from_source
+execute rat_test
+execute build_from_source
+execute build_tar_from_source
+
+popd
+popd
+
+print_when_exit
diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh
index 7ead90b5da24d..1bb080d19cabc 100644
--- a/dev-support/jenkins.sh
+++ b/dev-support/jenkins.sh
@@ -149,10 +149,6 @@ function run_ci() {
   # enable writing back to Github
   YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
 
-  # enable writing back to ASF JIRA
-  YETUS_ARGS+=("--jira-password=${JIRA_PASSWORD}")
-  YETUS_ARGS+=("--jira-user=${JIRA_USER}")
-
   # auto-kill any surefire stragglers during unit test runs
   YETUS_ARGS+=("--reapermode=kill")
 
@@ -173,7 +169,7 @@ function run_ci() {
   YETUS_ARGS+=("--build-url-artifacts=artifact/out")
 
   # plugins to enable
-  YETUS_ARGS+=("--plugins=all")
+  YETUS_ARGS+=("--plugins=all,-jira")
 
   # don't let these tests cause -1s because we aren't really paying that
   # much attention to them
diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
index 73012dd47a864..ca8d137dd5e49 100644
--- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
+++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml
@@ -122,9 +122,8 @@
         <!-- Checks for imports                              -->
         <!-- See http://checkstyle.sf.net/config_import.html -->
         <module name="IllegalImport">
-          <property name="regexp" value="true"/>
-          <property name="illegalPkgs" value="sun, com\.google\.common"/>
-          <property name="illegalClasses" value="^org\.apache\.hadoop\.thirdparty\.com\.google\.common\.io\.BaseEncoding, ^org\.apache\.hadoop\.thirdparty\.com\.google\.common\.base\.(Optional|Function|Predicate|Supplier), ^org\.apache\.hadoop\.thirdparty\.com\.google\.common\.collect\.(ImmutableListMultimap)"/>
+            <property name="regexp" value="true"/>
+            <property name="illegalPkgs" value="sun"/>
         </module>
         <module name="RedundantImport"/>
         <module name="UnusedImports"/>
diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml
index d0d62f5277b9b..b4b81011eb517 100644
--- a/hadoop-client-modules/hadoop-client-api/pom.xml
+++ b/hadoop-client-modules/hadoop-client-api/pom.xml
@@ -98,13 +98,6 @@
               <createSourcesJar>true</createSourcesJar>
               <shadeSourcesContent>true</shadeSourcesContent>
             </configuration>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -161,6 +154,9 @@
                         <!-- Exclude snappy-java -->
                         <exclude>org/xerial/snappy/*</exclude>
                         <exclude>org/xerial/snappy/**/*</exclude>
+                        <!-- Exclude org.widlfly.openssl -->
+                        <exclude>org/wildfly/openssl/*</exclude>
+                        <exclude>org/wildfly/openssl/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -179,6 +175,8 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
+                        <exclude>com/ibm/security/*</exclude>
+                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -249,8 +247,7 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resource>NOTICE.txt</resource>
diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
index 9d4bce1fddccd..b1c00678406d7 100644
--- a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml
@@ -56,7 +56,7 @@
           <dependency>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.0-beta-3</version>
+            <version>1.5.1</version>
           </dependency>
         </dependencies>
         <executions>
diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
index 635250ec1ae1f..0e576ac6f0666 100644
--- a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
+++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml
@@ -60,7 +60,7 @@
           <dependency>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.0-beta-3</version>
+            <version>1.5.1</version>
           </dependency>
         </dependencies>
         <executions>
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
index 967782163f7e8..ba593ebd1b42d 100644
--- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
+++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
@@ -184,6 +184,12 @@
           <artifactId>hadoop-hdfs</artifactId>
           <scope>test</scope>
           <type>test-jar</type>
+          <exclusions>
+            <exclusion>
+              <groupId>org.ow2.asm</groupId>
+              <artifactId>asm-commons</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index c9ce6f297d22a..208345d5f5a53 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -332,6 +332,10 @@
           <groupId>org.apache.hadoop.thirdparty</groupId>
           <artifactId>hadoop-shaded-guava</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <!-- Add optional runtime dependency on the in-development timeline server module
@@ -403,8 +407,8 @@
     <!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
     <!-- Skip jetty-util because it's in client -->
     <dependency>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>jersey-core</artifactId>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-core</artifactId>
       <optional>true</optional>
       <exclusions>
         <exclusion>
@@ -419,29 +423,25 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
       <optional>true</optional>
       <exclusions>
         <exclusion>
-          <groupId>javax.xml.bind</groupId>
-          <artifactId>jaxb-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-core-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-mapper-asl</artifactId>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-jaxrs</artifactId>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.codehaus.jackson</groupId>
-          <artifactId>jackson-xc</artifactId>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
@@ -451,9 +451,23 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-          <groupId>com.sun.jersey</groupId>
-          <artifactId>jersey-servlet</artifactId>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-servlet</artifactId>
       <optional>true</optional>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.enterprise</groupId>
+          <artifactId>cdi-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.cal10n</groupId>
+          <artifactId>cal10n-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
     <dependency>
@@ -657,13 +671,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -736,6 +743,18 @@
                         <exclude>testdata/*</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>com.fasterxml.jackson.*:*</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/11/module-info.class</exclude>
+                      </excludes>
+                    </filter>
+                    <filter>
+                      <artifact>com.google.code.gson:gson</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/9/module-info.class</exclude>
+                      </excludes>
+                    </filter>
 
                     <!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
                     <filter>
@@ -919,6 +938,8 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
+                        <exclude>com/ibm/security/*</exclude>
+                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -1024,8 +1045,7 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>
diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml
index b6a71e50c41ab..d5185f0fffc41 100644
--- a/hadoop-client-modules/hadoop-client-runtime/pom.xml
+++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml
@@ -128,13 +128,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-maven-plugins</artifactId>
-                <version>${project.version}</version>
-              </dependency>
-            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -155,6 +148,7 @@
                       <!-- Leave javax APIs that are stable -->
                       <!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
                       <exclude>com.google.code.findbugs:jsr305</exclude>
+                      <exclude>io.netty:*</exclude>
                       <exclude>io.dropwizard.metrics:metrics-core</exclude>
                       <exclude>org.eclipse.jetty:jetty-servlet</exclude>
                       <exclude>org.eclipse.jetty:jetty-security</exclude>
@@ -163,6 +157,8 @@
                       <exclude>org.bouncycastle:*</exclude>
                       <!-- Leave snappy that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <!-- leave out kotlin classes -->
+                      <exclude>org.jetbrains.kotlin:*</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -242,6 +238,19 @@
                         <exclude>google/protobuf/**/*.proto</exclude>
                       </excludes>
                     </filter>
+                    <filter>
+                      <artifact>com.fasterxml.jackson.*:*</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/11/module-info.class</exclude>
+                      </excludes>
+                    </filter>
+                    <filter>
+                      <artifact>com.google.code.gson:gson</artifact>
+                      <excludes>
+                        <exclude>META-INF/versions/9/module-info.class</exclude>
+                      </excludes>
+                    </filter>
+
                   </filters>
                   <relocations>
                     <relocation>
@@ -288,6 +297,8 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
+                        <exclude>com/ibm/security/*</exclude>
+                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -382,8 +393,7 @@
     -->
                   </relocations>
                   <transformers>
-                    <!-- Needed until MSHADE-182 -->
-                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index dced359b286d9..5299c9e8713df 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -66,7 +66,7 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -114,6 +114,18 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-stdlib</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-stdlib-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.squareup.okhttp3</groupId>
+          <artifactId>okhttp</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
@@ -167,7 +179,7 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -218,7 +230,7 @@
           <artifactId>jersey-server</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -275,7 +287,7 @@
           <artifactId>guice-servlet</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index a8f45a7f3a222..6c8a0916802f2 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -101,6 +101,10 @@
           <groupId>org.apache.zookeeper</groupId>
           <artifactId>zookeeper</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -123,11 +127,6 @@
       <artifactId>hadoop-azure-datalake</artifactId>
       <scope>compile</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-openstack</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-cos</artifactId>
diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
index 0e801c4e439ac..ca7c4bf516cad 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
@@ -96,38 +96,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
@@ -141,7 +109,7 @@
     <dependency>
       <groupId>com.qcloud</groupId>
       <artifactId>cos_api-bundle</artifactId>
-      <version>5.6.19</version>
+      <version>5.6.69</version>
       <scope>compile</scope>
     </dependency>
 
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
index d2d2f8c9a7cab..66ef4b1c6fd87 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
@@ -24,7 +24,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.qcloud.cos.auth.AnonymousCOSCredentials;
 import com.qcloud.cos.auth.COSCredentials;
 import com.qcloud.cos.auth.COSCredentialsProvider;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
index 7a9ee61836e1c..b96883b9ac80d 100755
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml
@@ -92,38 +92,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
   <dependencies>
@@ -132,10 +100,14 @@
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
       <exclusions>
-      <exclusion>
-      <groupId>jdk.tools</groupId>
-      <artifactId>jdk.tools</artifactId>
-      </exclusion>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.javassist</groupId>
+          <artifactId>javassist</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -193,6 +165,14 @@
           <artifactId>okio</artifactId>
           <groupId>com.squareup.okio</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>log4j-core</artifactId>
+          <groupId>org.apache.logging.log4j</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>log4j-api</artifactId>
+          <groupId>org.apache.logging.log4j</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -208,4 +188,4 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java
index cefa897927790..22c6cb5c350c9 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSBlockOutputStream.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.obs;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java
index d477cec186b0e..3a06961d3acd9 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSCommonUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.obs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.obs.services.ObsClient;
 import com.obs.services.exception.ObsException;
 import com.obs.services.model.AbortMultipartUploadRequest;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java
index b58eaa00aa697..e347970ee8446 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSDataBlocks.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.obs;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSExceptionMessages;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java
index 29a92c71919a8..3f99fd610efa5 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSIOException.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.obs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.obs.services.exception.ObsException;
 
 import java.io.IOException;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java
index e94565a4d760a..3f7e9888889b5 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSInputStream.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.obs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.obs.services.ObsClient;
 import com.obs.services.exception.ObsException;
 import com.obs.services.model.GetObjectRequest;
diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java
index 5cc3008f1dcfb..2b02f962a0598 100644
--- a/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java
+++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/src/main/java/org/apache/hadoop/fs/obs/OBSWriteOperationHelper.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.obs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.obs.services.ObsClient;
 import com.obs.services.exception.ObsException;
 import com.obs.services.model.AbortMultipartUploadRequest;
diff --git a/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java b/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java
index 80489917e0857..e96e5062919a7 100644
--- a/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java
+++ b/hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java
@@ -116,6 +116,7 @@ private List<String> getHeaderValues(String name, boolean reset) {
     public void addCookie(Cookie cookie) {
       super.addCookie(cookie);
       List<String> cookies = getHeaderValues("Set-Cookie", false);
+      cookies.addAll(getHeaderValues("set-cookie", false));
       cookies.add(cookie.getName() + "=" + cookie.getValue());
     }
 
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index d21d6273fffd4..6eaa4fdfce5b4 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -248,40 +248,6 @@
           <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided implementations rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
-                    <bannedImport>com.google.common.base.Preconditions</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index 32f4edfbc5710..cb7d36368aa39 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -92,6 +92,9 @@ public synchronized Map<String, List<String>> get(URI uri,
     @Override
     public void put(URI uri, Map<String, List<String>> responseHeaders) {
       List<String> headers = responseHeaders.get("Set-Cookie");
+      if (headers == null) {
+        headers = responseHeaders.get("set-cookie");
+      }
       if (headers != null) {
         for (String header : headers) {
           List<HttpCookie> cookies;
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
index 06b63c1b9916c..30e65efe10cba 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
@@ -280,6 +280,9 @@ private boolean isNegotiate(HttpURLConnection conn) throws IOException {
     boolean negotiate = false;
     if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
       String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
+      if (authHeader == null) {
+        authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
+      }
       negotiate = authHeader != null && authHeader.trim().startsWith(NEGOTIATE);
     }
     return negotiate;
@@ -388,6 +391,9 @@ private byte[] readToken(HttpURLConnection conn)
     int status = conn.getResponseCode();
     if (status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_UNAUTHORIZED) {
       String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
+      if (authHeader == null) {
+        authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
+      }
       if (authHeader == null || !authHeader.trim().startsWith(NEGOTIATE)) {
         throw new AuthenticationException("Invalid SPNEGO sequence, '" + WWW_AUTHENTICATE +
                                           "' header incorrect: " + authHeader);
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 3658bd8b8ec01..7cc70c493c0f6 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -616,7 +616,9 @@ && getMaxInactiveInterval() > 0) {
         // present.. reset to 403 if not found..
         if ((errCode == HttpServletResponse.SC_UNAUTHORIZED)
             && (!httpResponse.containsHeader(
-                KerberosAuthenticator.WWW_AUTHENTICATE))) {
+                KerberosAuthenticator.WWW_AUTHENTICATE)
+                && !httpResponse.containsHeader(
+                KerberosAuthenticator.WWW_AUTHENTICATE.toLowerCase()))) {
           errCode = HttpServletResponse.SC_FORBIDDEN;
         }
         // After Jetty 9.4.21, sendError() no longer allows a custom message.
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/CertificateUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/CertificateUtil.java
index cf17aca15ceac..f25602c67d4a3 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/CertificateUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/CertificateUtil.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.security.authentication.util;
 
 import java.io.ByteArrayInputStream;
-import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.security.PublicKey;
 import java.security.cert.CertificateException;
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/JaasConfiguration.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/JaasConfiguration.java
new file mode 100644
index 0000000000000..d03e630cedf7e
--- /dev/null
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/JaasConfiguration.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+
+
+/**
+ * Creates a programmatic version of a jaas.conf file. This can be used
+ * instead of writing a jaas.conf file and setting the system property,
+ * "java.security.auth.login.config", to point to that file. It is meant to be
+ * used for connecting to ZooKeeper.
+ */
+public class JaasConfiguration extends Configuration {
+
+  private final javax.security.auth.login.Configuration baseConfig =
+      javax.security.auth.login.Configuration.getConfiguration();
+  private final AppConfigurationEntry[] entry;
+  private final String entryName;
+
+  /**
+   * Add an entry to the jaas configuration with the passed in name,
+   * principal, and keytab. The other necessary options will be set for you.
+   *
+   * @param entryName The name of the entry (e.g. "Client")
+   * @param principal The principal of the user
+   * @param keytab The location of the keytab
+   */
+  public JaasConfiguration(String entryName, String principal, String keytab) {
+    this.entryName = entryName;
+    Map<String, String> options = new HashMap<>();
+    options.put("keyTab", keytab);
+    options.put("principal", principal);
+    options.put("useKeyTab", "true");
+    options.put("storeKey", "true");
+    options.put("useTicketCache", "false");
+    options.put("refreshKrb5Config", "true");
+    String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
+    if ("true".equalsIgnoreCase(jaasEnvVar)) {
+      options.put("debug", "true");
+    }
+    entry = new AppConfigurationEntry[]{
+        new AppConfigurationEntry(getKrb5LoginModuleName(),
+            AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+            options)};
+  }
+
+  @Override
+  public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+    return (entryName.equals(name)) ? entry : ((baseConfig != null)
+        ? baseConfig.getAppConfigurationEntry(name) : null);
+  }
+
+  private String getKrb5LoginModuleName() {
+    String krb5LoginModuleName;
+    if (System.getProperty("java.vendor").contains("IBM")) {
+      krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
+    } else {
+      krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
+    }
+    return krb5LoginModuleName;
+  }
+}
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
index fc6f957b9622e..5125be078d67b 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
@@ -236,7 +236,7 @@ public static final String getServicePrincipal(String service,
    */
   static final String[] getPrincipalNames(String keytabFileName) throws IOException {
     Keytab keytab = Keytab.loadKeytab(new File(keytabFileName));
-    Set<String> principals = new HashSet<String>();
+    Set<String> principals = new HashSet<>();
     List<PrincipalName> entries = keytab.getPrincipals();
     for (PrincipalName entry : entries) {
       principals.add(entry.getName().replace("\\", "/"));
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 374f4a5665796..53b6f4d239f98 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -17,12 +17,9 @@
 import java.nio.ByteBuffer;
 import java.security.SecureRandom;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
-import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.Configuration;
 import javax.servlet.ServletContext;
 import org.apache.curator.RetryPolicy;
@@ -429,62 +426,4 @@ public List<ACL> getAclForPath(String path) {
       return saslACL;
     }
   }
-
-  /**
-   * Creates a programmatic version of a jaas.conf file. This can be used
-   * instead of writing a jaas.conf file and setting the system property,
-   * "java.security.auth.login.config", to point to that file. It is meant to be
-   * used for connecting to ZooKeeper.
-   */
-  @InterfaceAudience.Private
-  public static class JaasConfiguration extends Configuration {
-
-    private final javax.security.auth.login.Configuration baseConfig =
-        javax.security.auth.login.Configuration.getConfiguration();
-    private static AppConfigurationEntry[] entry;
-    private String entryName;
-
-    /**
-     * Add an entry to the jaas configuration with the passed in name,
-     * principal, and keytab. The other necessary options will be set for you.
-     *
-     * @param entryName The name of the entry (e.g. "Client")
-     * @param principal The principal of the user
-     * @param keytab The location of the keytab
-     */
-    public JaasConfiguration(String entryName, String principal, String keytab) {
-      this.entryName = entryName;
-      Map<String, String> options = new HashMap<String, String>();
-      options.put("keyTab", keytab);
-      options.put("principal", principal);
-      options.put("useKeyTab", "true");
-      options.put("storeKey", "true");
-      options.put("useTicketCache", "false");
-      options.put("refreshKrb5Config", "true");
-      String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
-      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
-        options.put("debug", "true");
-      }
-      entry = new AppConfigurationEntry[]{
-                  new AppConfigurationEntry(getKrb5LoginModuleName(),
-                  AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
-                  options)};
-    }
-
-    @Override
-    public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
-      return (entryName.equals(name)) ? entry : ((baseConfig != null)
-        ? baseConfig.getAppConfigurationEntry(name) : null);
-    }
-
-    private String getKrb5LoginModuleName() {
-      String krb5LoginModuleName;
-      if (System.getProperty("java.vendor").contains("IBM")) {
-        krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
-      } else {
-        krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
-      }
-      return krb5LoginModuleName;
-    }
-  }
 }
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java
index 8fc08e2171f67..293871bcd0620 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java
@@ -108,9 +108,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
   public static <T> T doAs(String principal, final Callable<T> callable) throws Exception {
     LoginContext loginContext = null;
     try {
-      Set<Principal> principals = new HashSet<Principal>();
+      Set<Principal> principals = new HashSet<>();
       principals.add(new KerberosPrincipal(KerberosTestUtils.getClientPrincipal()));
-      Subject subject = new Subject(false, principals, new HashSet<Object>(), new HashSet<Object>());
+      Subject subject = new Subject(false, principals, new HashSet<>(), new HashSet<>());
       loginContext = new LoginContext("", subject, null, new KerberosConfiguration(principal));
       loginContext.login();
       subject = loginContext.getSubject();
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
index b56fc65b25b45..5b3de5208f49a 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
@@ -89,6 +89,44 @@ public void testExtractTokenFail() throws Exception {
     }
   }
 
+  @Test
+  public void testExtractTokenCookieHeader() throws Exception {
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
+
+    String tokenStr = "foo";
+    Map<String, List<String>> headers = new HashMap<>();
+    List<String> cookies = new ArrayList<>();
+    cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
+    headers.put("Set-Cookie", cookies);
+    Mockito.when(conn.getHeaderFields()).thenReturn(headers);
+
+    AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+    AuthenticatedURL.extractToken(conn, token);
+
+    Assert.assertTrue(token.isSet());
+  }
+
+  @Test
+  public void testExtractTokenLowerCaseCookieHeader() throws Exception {
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
+
+    String tokenStr = "foo";
+    Map<String, List<String>> headers = new HashMap<>();
+    List<String> cookies = new ArrayList<>();
+    cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
+    headers.put("set-cookie", cookies);
+    Mockito.when(conn.getHeaderFields()).thenReturn(headers);
+
+    AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+    AuthenticatedURL.extractToken(conn, token);
+
+    Assert.assertTrue(token.isSet());
+  }
+
   @Test
   public void testConnectionConfigurator() throws Exception {
     HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
index 177bcb4547e33..bc316ef8cb8b2 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
@@ -21,8 +21,13 @@
 import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.NAME_RULES;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.nio.charset.CharacterCodingException;
 import javax.security.sasl.AuthenticationException;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
 import org.apache.hadoop.security.authentication.KerberosTestUtils;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@@ -32,10 +37,12 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.File;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.Arrays;
 import java.util.Properties;
 import java.util.concurrent.Callable;
 
@@ -248,4 +255,79 @@ public void testWrapExceptionWithMessage() {
     Assert.assertTrue(ex.equals(ex2));
   }
 
+  @Test(timeout = 60000)
+  public void testNegotiate() throws NoSuchMethodException, InvocationTargetException,
+          IllegalAccessException, IOException {
+    KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
+
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+    Mockito.when(conn.getHeaderField(KerberosAuthenticator.WWW_AUTHENTICATE)).
+            thenReturn(KerberosAuthenticator.NEGOTIATE);
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
+
+    Method method = KerberosAuthenticator.class.getDeclaredMethod("isNegotiate",
+            HttpURLConnection.class);
+    method.setAccessible(true);
+
+    Assert.assertTrue((boolean)method.invoke(kerberosAuthenticator, conn));
+  }
+
+  @Test(timeout = 60000)
+  public void testNegotiateLowerCase() throws NoSuchMethodException, InvocationTargetException,
+          IllegalAccessException, IOException {
+    KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
+
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+    Mockito.when(conn.getHeaderField("www-authenticate"))
+            .thenReturn(KerberosAuthenticator.NEGOTIATE);
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
+
+    Method method = KerberosAuthenticator.class.getDeclaredMethod("isNegotiate",
+            HttpURLConnection.class);
+    method.setAccessible(true);
+
+    Assert.assertTrue((boolean)method.invoke(kerberosAuthenticator, conn));
+  }
+
+  @Test(timeout = 60000)
+  public void testReadToken() throws NoSuchMethodException, IOException, IllegalAccessException,
+          InvocationTargetException {
+    KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
+    FieldUtils.writeField(kerberosAuthenticator, "base64", new Base64(), true);
+
+    Base64 base64 = new Base64();
+
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
+    Mockito.when(conn.getHeaderField(KerberosAuthenticator.WWW_AUTHENTICATE))
+            .thenReturn(KerberosAuthenticator.NEGOTIATE + " " +
+                    Arrays.toString(base64.encode("foobar".getBytes())));
+
+    Method method = KerberosAuthenticator.class.getDeclaredMethod("readToken",
+            HttpURLConnection.class);
+    method.setAccessible(true);
+
+    method.invoke(kerberosAuthenticator, conn); // expecting this not to throw an exception
+  }
+
+  @Test(timeout = 60000)
+  public void testReadTokenLowerCase() throws NoSuchMethodException, IOException,
+          IllegalAccessException, InvocationTargetException {
+    KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
+    FieldUtils.writeField(kerberosAuthenticator, "base64", new Base64(), true);
+
+    Base64 base64 = new Base64();
+
+    HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
+    Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
+    Mockito.when(conn.getHeaderField("www-authenticate"))
+            .thenReturn(KerberosAuthenticator.NEGOTIATE +
+                    Arrays.toString(base64.encode("foobar".getBytes())));
+
+    Method method = KerberosAuthenticator.class.getDeclaredMethod("readToken",
+            HttpURLConnection.class);
+    method.setAccessible(true);
+
+    method.invoke(kerberosAuthenticator, conn); // expecting this not to throw an exception
+  }
 }
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
index 4f4a4521b2f0c..b0066abbdd3cf 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
@@ -574,6 +574,44 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
     }
   }
 
+  @Test
+  public void testDoFilterNotAuthenticatedLowerCase() throws Exception {
+    AuthenticationFilter filter = new AuthenticationFilter();
+    try {
+      FilterConfig config = Mockito.mock(FilterConfig.class);
+      Mockito.when(config.getInitParameter("management.operation.return")).
+              thenReturn("true");
+      Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
+              DummyAuthenticationHandler.class.getName());
+      Mockito.when(config.getInitParameterNames()).thenReturn(
+              new Vector<>(
+                      Arrays.asList(AuthenticationFilter.AUTH_TYPE,
+                              "management.operation.return")).elements());
+      getMockedServletContextWithStringSigner(config);
+      filter.init(config);
+
+      HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
+      Mockito.when(request.getRequestURL()).thenReturn(new StringBuffer("http://foo:8080/bar"));
+
+      HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
+
+      FilterChain chain = Mockito.mock(FilterChain.class);
+
+      Mockito.doAnswer((Answer<Object>) invocation -> {
+        Assert.fail();
+        return null;
+      }).when(chain).doFilter(any(), any());
+
+      Mockito.when(response.containsHeader("www-authenticate")).thenReturn(true);
+      filter.doFilter(request, response, chain);
+
+      Mockito.verify(response).sendError(
+              HttpServletResponse.SC_UNAUTHORIZED, "Authentication required");
+    } finally {
+      filter.destroy();
+    }
+  }
+
   private void _testDoFilterAuthentication(boolean withDomainPath,
                                            boolean invalidToken,
                                            boolean expired) throws Exception {
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthenticationHandler.java
index 5a2db9ba6fd97..7587bca2012d0 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthenticationHandler.java
@@ -25,7 +25,6 @@
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Properties;
-import java.util.Vector;
 import java.util.Date;
 
 import javax.servlet.ServletException;
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
index b52915d9cc4ac..ac6221d642a6f 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
@@ -13,7 +13,6 @@
  */
 package org.apache.hadoop.security.authentication.server;
 
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
 import org.junit.Assert;
 import org.junit.Test;
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
index 5582c923ae0e7..ed6b1aeccc7c2 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
@@ -18,7 +18,6 @@
 import javax.servlet.ServletContext;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestJaasConfiguration.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestJaasConfiguration.java
index 2b70135800be0..5de4122471f50 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestJaasConfiguration.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestJaasConfiguration.java
@@ -32,8 +32,8 @@ public void test() throws Exception {
       krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
     }
 
-    ZKSignerSecretProvider.JaasConfiguration jConf =
-            new ZKSignerSecretProvider.JaasConfiguration("foo", "foo/localhost",
+    JaasConfiguration jConf =
+            new JaasConfiguration("foo", "foo/localhost",
             "/some/location/foo.keytab");
     AppConfigurationEntry[] entries = jConf.getAppConfigurationEntry("bar");
     Assert.assertNull(entries);
diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index 23e39d055ffc5..b885891af73d0 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -379,21 +379,6 @@
        <Bug code="JLM" />
      </Match>
 
-  <!--
-  OpenStack Swift FS module -closes streams in a different method
-  from where they are opened.
-  -->
-    <Match>
-      <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
-      <Method name="uploadFileAttempt"/>
-      <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
-    </Match>
-    <Match>
-      <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
-      <Method name="uploadFilePartAttempt"/>
-      <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
-    </Match>
-
      <!-- code from maven source, null value is checked at callee side. -->
      <Match>
        <Class name="org.apache.hadoop.util.ComparableVersion$ListItem" />
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.10.2.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.10.2.xml
new file mode 100644
index 0000000000000..7ddea60d5f724
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.10.2.xml
@@ -0,0 +1,41055 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue May 24 22:52:15 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop Common 2.10.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-common-project/hadoop-common/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.10.2.jar:/usr/lib/jvm/zulu-7-amd64/lib/tools.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.5/commons-io-2.5.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/org/apache/ant/ant/1.10.11/ant-1.10.11.jar:/maven/org/apache/ant/ant-launcher/1.10.11/ant-launcher-1.10.11.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.10.2.jar:/maven/com/nimbusds/nimbus-jose-jwt/7.9/nimbus-jose-jwt-7.9.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.3/json-smart-1.3.3.jar:/maven/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar -sourcepath /build/source/hadoop-common-project/hadoop-common/src/main/java -apidir /build/source/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 2.10.2 -->
+<package name="org.apache.hadoop">
+  <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+  <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message detailed message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+  <!-- start interface org.apache.hadoop.conf.Configurable -->
+  <interface name="Configurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.conf.Configurable -->
+  <!-- start class org.apache.hadoop.conf.Configuration -->
+  <class name="Configuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Iterable"/>
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration where the behavior of reading from the default 
+ resources can be turned off.
+ 
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files. 
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration with the same settings cloned from another.
+ 
+ @param other the configuration from which to clone settings.]]>
+      </doc>
+    </constructor>
+    <method name="addDeprecations"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+      <doc>
+      <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless.  It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context.  If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas   The deprecations to add.]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+ 
+ @param key
+ @param newKeys
+ @param customMessage
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKey
+ @param customMessage]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+      </doc>
+    </method>
+    <method name="isDeprecated" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[checks whether the given <code>key</code> is deprecated.
+ 
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and 
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setDeprecatedProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+      </doc>
+    </method>
+    <method name="reloadExistingConfigurations"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload existing configuration instances.]]>
+      </doc>
+    </method>
+    <method name="addDefaultResource"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a default resource. Resources are loaded in the order of the resources 
+ added.
+ @param name file name. File should be present in the classpath.]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemPropertiesDefault"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="setRestrictSystemProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param name resource to be added, the classpath is examined for a file 
+             with that name.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param url url of the resource to be added, the local filesystem is 
+            examined directly to find the resource, without referring to 
+            the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param file file-path of resource to be added, the local filesystem is
+             examined directly to find the resource, without referring to 
+             the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ WARNING: The contents of the InputStream will be cached, by this method. 
+ So use this sparingly because it does increase the memory consumption.
+ 
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next.  After it is read the stream will be
+ closed.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+      </doc>
+    </method>
+    <method name="reloadConfiguration"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added 
+ resources, and final parameters. This will make the resources to 
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="setAllowNullValueProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+      <doc>
+      <![CDATA[Set Configuration to allow keys without values during setup.  Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemProps"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="onlyKeyExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>null</code> if no such property exists. 
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>defaultValue</code> if no such property exists. 
+ See @{Configuration#getTrimmed} for more details.
+ 
+ @param name          the property name.
+ @param defaultValue  the property default value.
+ @return              the value of the <code>name</code> or defaultValue
+                      if it is not set.]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is 
+ deprecated, it returns the value of the first key which replaces 
+ the deprecated key and is not null.
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> property or 
+         its replacing property and null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="source" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from 
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+      </doc>
+    </method>
+    <method name="unset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unset a previously set property.]]>
+      </doc>
+    </method>
+    <method name="setIfUnset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+ 
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property 
+         doesn't exist.]]>
+      </doc>
+    </method>
+    <method name="getInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="int"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+   
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInts" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+ 
+ If no such property exists, an empty array is returned.
+ 
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+         <code>int</code> values]]>
+      </doc>
+    </method>
+    <method name="setInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+ 
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getLongBytes" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+ 
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="float"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getBoolean" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="boolean"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.  
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setBoolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+ 
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="setBooleanIfUnset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="setEnum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="T"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="getEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="T"/>
+      <doc>
+      <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided]]>
+      </doc>
+    </method>
+    <method name="setTimeDuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
+ provided, the default unit is applied.
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param defaultUnit Default time unit if no valid suffix is provided.
+ @param returnUnit The unit used for the returned value.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDurations" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="pattern" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+      </doc>
+    </method>
+    <method name="getPropertySources" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Gets information about why a property was set.  Typically this is the 
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise, 
+ returns a list of the sources of the resource.  The older sources are
+ the first ones in the list.  So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+      </doc>
+    </method>
+    <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+      </doc>
+    </method>
+    <method name="getStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s.  
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+ 
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then <code>null</code> is returned.
+ 
+ @param name property name.
+ @return property value as an array of <code>String</code>s, 
+         or <code>null</code>.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+ 
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s, 
+         or empty array.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="setStrings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the array of string values for the <code>name</code> property as 
+ as comma delimited values.  
+ 
+ @param name property name.
+ @param values The values]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromCredentialProviders" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromConfig" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="setSocketAddr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address.  This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="getClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Load a class by name.
+ 
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+      </doc>
+    </method>
+    <method name="getClassByNameOrNull" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+ 
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+      </doc>
+    </method>
+    <method name="getClasses" return="java.lang.Class[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class[]"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the class name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+   
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ An exception is thrown if the returned class does not implement the named
+ interface. 
+ 
+ @param name the class name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+ 
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+ 
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+        <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+      </doc>
+    </method>
+    <method name="setClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the name of a 
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+ 
+ An exception is thrown if <code>theClass</code> does not implement the 
+ interface <code>xface</code>. 
+ 
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+      </doc>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link URL} for the named resource.
+ 
+ @param name resource name.
+ @return the url for the named resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsReader" return="java.io.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getFinalParameters" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+      </doc>
+    </method>
+    <method name="getProps" return="java.util.Properties"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all keys from the configuration.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code> 
+ key-value pairs in the configuration.
+ 
+ @return an iterator over the entries.]]>
+      </doc>
+    </method>
+    <method name="getPropsWithPrefix" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration and includes all properties that
+ start with the specified configuration prefix.  Property names in the
+ mapping are trimmed to remove the configuration prefix.
+
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties with prefix stripped]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+ 
+ @param out the output stream to write to.]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the
+ given {@link Writer}.
+
+ <li>
+ When property name is not empty and the property exists in the
+ configuration, this method writes the property and its attributes
+ to the {@link Writer}.
+ </li>
+ <p>
+
+ <li>
+ When property name is null or empty, this method writes all the
+ configuration properties and their attributes to the {@link Writer}.
+ </li>
+ <p>
+
+ <li>
+ When property name is not empty but the property doesn't exist in
+ the configuration, this method throws an {@link IllegalArgumentException}.
+ </li>
+ <p>
+ @param out the writer to write to.]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes properties and their attributes (final and resource)
+  to the given {@link Writer}.
+
+  <li>
+  When propertyName is not empty, and the property exists
+  in the configuration, the format of the output would be,
+  <pre>
+  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is null or empty, it behaves same as
+  {@link #dumpConfiguration(Configuration, Writer)}, the
+  output would be,
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is not empty, and the property is not
+  found in the configuration, this method will throw an
+  {@link IllegalArgumentException}.
+  </li>
+  <p>
+ @param config the configuration
+ @param propertyName property name
+ @param out the Writer to write to
+ @throws IOException
+ @throws IllegalArgumentException when property name is not
+   empty and the property is not found in configuration]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes out all properties and their attributes (final and resource) to
+  the given {@link Writer}, the format of the output would be,
+
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+
+  It does not output the properties of the configuration object which
+  is loaded from an input stream.
+  <p>
+
+ @param config the configuration
+ @param out the Writer to write to
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClassLoader" return="java.lang.ClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+      </doc>
+    </method>
+    <method name="setClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classLoader" type="java.lang.ClassLoader"/>
+      <doc>
+      <![CDATA[Set the class loader that will be used to load the various objects.
+ 
+ @param classLoader the new class loader.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQuietMode"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quietmode" type="boolean"/>
+      <doc>
+      <![CDATA[Set the quietness-mode. 
+ 
+ In the quiet-mode, error and informational messages might not be logged.
+ 
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+              to turn it off.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[For debugging.  List non-default properties to the terminal and exit.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getValByRegex" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="regex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[get keys matching the the regex 
+ @param regex
+ @return Map<String,String> with matching keys]]>
+      </doc>
+    </method>
+    <method name="dumpDeprecatedKeys"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasWarnedDeprecation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides access to configuration parameters.
+
+ <h4 id="Resources">Resources</h4>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a 
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
+ then the classpath is examined for a file with that name.  If named by a 
+ <code>Path</code>, then the local filesystem is examined directly, without 
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two 
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+ 
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>. 
+ Once a resource declares a value final, no subsequently-loaded 
+ resource can alter that value.  
+ For example, one might define a final parameter with:
+ <tt><pre>
+  &lt;property&gt;
+    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
+    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
+    <b>&lt;final&gt;true&lt;/final&gt;</b>
+  &lt;/property&gt;</pre></tt>
+
+ Administrators typically define parameters as final in 
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions: 
+ <tt><pre>
+  &lt;property&gt;
+    &lt;name&gt;basedir&lt;/name&gt;
+    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
+  &lt;/property&gt;
+  
+  &lt;property&gt;
+    &lt;name&gt;tempdir&lt;/name&gt;
+    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
+  &lt;/property&gt;</pre></tt>
+
+ When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ By default, warnings will be given to any deprecated configuration
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configuration -->
+  <!-- start class org.apache.hadoop.conf.Configured -->
+  <class name="Configured" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="Configured"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configured -->
+  <!-- start class org.apache.hadoop.conf.ReconfigurationTaskStatus -->
+  <class name="ReconfigurationTaskStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReconfigurationTaskStatus" type="long, long, java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasTask" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if
+   - A reconfiguration task has finished or
+   - an active reconfiguration task is running]]>
+      </doc>
+    </method>
+    <method name="stopped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the latest reconfiguration task has finished and there is
+ no another active task running.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.conf.ReconfigurationTaskStatus -->
+  <doc>
+  <![CDATA[Configuration of system parameters.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.crypto">
+</package>
+<package name="org.apache.hadoop.crypto.key">
+  <!-- start class org.apache.hadoop.crypto.key.KeyProvider -->
+  <class name="KeyProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyProvider" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ 
+ @param conf configuration for the provider]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the provider configuration.
+ 
+ @return the provider configuration]]>
+      </doc>
+    </method>
+    <method name="options" return="org.apache.hadoop.crypto.key.KeyProvider.Options"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[A helper function to create an options object.
+ @param conf the configuration to use
+ @return a new options object]]>
+      </doc>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide access to
+ keying material rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getKeyVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for a specific version of the key. This method is used
+ when decrypting data.
+ @param versionName the name of a specific version of the key
+ @return the key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeys" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key names for all keys.
+ @return the list of key names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeysMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get key metadata in bulk.
+ @param names the names of the keys to get
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeyVersions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for all versions of a specific key name.
+ @return the list of key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current version of the key, which should be used for encrypting new
+ data.
+ @param name the base name of the key
+ @return the version name of the current version of the key or null if the
+    key version doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get metadata about the key.
+ @param name the basename of the key
+ @return the key's metadata or null if the key doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key. The given key must not already exist.
+ @param name the base name of the key
+ @param material the key material for the first version of the key.
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <param name="algorithm" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <doc>
+      <![CDATA[Generates a key material.
+
+ @param size length of the key.
+ @param algorithm algorithm to use for generating the key.
+ @return the generated key.
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key generating the material for it.
+ The given key must not already exist.
+ <p/>
+ This implementation generates the key material and calls the
+ {@link #createKey(String, byte[], Options)} method.
+
+ @param name the base name of the key
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="deleteKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given key.
+ @param name the name of the key to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key.
+ @param name the basename of the key
+ @param material the new key material
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to close any resources
+ that require closing]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key generating the material for it.
+ <p/>
+ This implementation generates the key material and calls the
+ {@link #rollNewVersion(String, byte[])} method.
+
+ @param name the basename of the key
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the keys are written to persistent store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getBaseName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Split the versionName in to a base name. Converts "/aaa/bbb/3" to
+ "/aaa/bbb".
+ @param versionName the version name to split
+ @return the base name of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="buildVersionName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Build a version string from a basename and version number. Converts
+ "/aaa/bbb" and 3 to "/aaa/bbb@3".
+ @param name the basename of the key
+ @param version the version of the key
+ @return the versionName of the key.]]>
+      </doc>
+    </method>
+    <method name="findProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerList" type="java.util.List"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the provider with the given key.
+ @param providerList the list of providers
+ @param keyName the key name we are looking for
+ @return the KeyProvider that has the key]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_CIPHER_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CIPHER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIALFILTER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIAL_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of secret key material for Hadoop applications. Provides an
+ abstraction to separate key storage from users of encryption. It
+ is intended to support getting or storing keys in a variety of ways,
+ including third party bindings.
+ <P/>
+ <code>KeyProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProvider -->
+  <!-- start class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+  <class name="KeyProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a KeyProvider based on a provided URI.
+
+ @param uri key provider URI
+ @param conf configuration to initialize the key provider
+ @return the key provider for the specified URI, or <code>NULL</code> if
+         a provider for the specified URI scheme could not be found.
+ @throws IOException thrown if the provider failed to initialize.]]>
+      </doc>
+    </method>
+    <field name="KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of KeyProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ KeyProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+</package>
+<package name="org.apache.hadoop.crypto.key.kms">
+</package>
+<package name="org.apache.hadoop.crypto.random">
+</package>
+<package name="org.apache.hadoop.fs">
+  <!-- start class org.apache.hadoop.fs.AbstractFileSystem -->
+  <class name="AbstractFileSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AbstractFileSystem" type="java.net.URI, java.lang.String, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[Constructor to be called by subclasses.
+ 
+ @param uri for this file system.
+ @param supportedScheme the scheme supported by the implementor
+ @param authorityNeeded if true then theURI must have authority, if false
+          then the URI must have null authority.
+
+ @throws URISyntaxException <code>uri</code> has syntax error]]>
+      </doc>
+    </constructor>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isValidName" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns true if the specified string is considered valid in the path part
+ of a URI by this file system.  The default implementation enforces the rules
+ of HDFS, but subclasses may override this method to implement specific
+ validation rules for specific file systems.
+ 
+ @param src String source filename to check, path part of the URI
+ @return boolean true if the specified string is considered valid]]>
+      </doc>
+    </method>
+    <method name="createFileSystem" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a file system instance for the specified uri using the conf. The
+ conf is used to find the class name that implements the file system. The
+ conf is also passed to the file system for its configuration.
+
+ @param uri URI of the file system
+ @param conf Configuration for the file system
+ 
+ @return Returns the file system for the given URI
+
+ @throws UnsupportedFileSystemException file system for <code>uri</code> is
+           not found]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ 
+ @param uri
+          used as key to lookup STATISTICS_TABLE. Only scheme and authority
+          part of the uri are used.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[The main factory method for creating a file system. Get a file system for
+ the URI's scheme and authority. The scheme of the <code>uri</code>
+ determines a configuration property name,
+ <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
+ AbstractFileSystem class.
+ 
+ The entire URI and conf is passed to the AbstractFileSystem factory method.
+ 
+ @param uri for the file system to be created.
+ @param conf which is passed to the file system impl.
+ 
+ @return file system for the given URI.
+ 
+ @throws UnsupportedFileSystemException if the file system for
+           <code>uri</code> is not supported.]]>
+      </doc>
+    </method>
+    <method name="checkScheme"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="supportedScheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check that the Uri's scheme matches
+ @param uri
+ @param supportedScheme]]>
+      </doc>
+    </method>
+    <method name="getUriDefaultPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default port of this file system.
+ 
+ @return default port of this file system's Uri scheme
+         A uri with a port of -1 => default port;]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.
+ 
+ @return the uri of this file system.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+ 
+ If the path is fully qualified URI, then its scheme and authority
+ matches that of this file system. Otherwise the path must be 
+ slash-relative name.
+ 
+ @throws InvalidPathException if the path is invalid]]>
+      </doc>
+    </method>
+    <method name="getUriPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the path-part of a pathname. Checks that URI matches this file system
+ and that the path-part is a valid name.
+ 
+ @param p path
+ 
+ @return path-part of the Path p]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified to this file system
+ @param path
+ @return the qualified path]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Some file systems like LocalFileSystem have an initial workingDir
+ that is used as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+ 
+ @return the initial workingDir if the file system has such a notion
+         otherwise return a null.]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ 
+ @return current user's home directory.]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ 
+ @return server default configuration values
+ 
+ @throws IOException an I/O error occurred
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param f path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path f resolving the path
+ through any internal symlinks or mount point
+ @param p path to be resolved
+ @return fully qualified path 
+ @throws FileNotFoundException, AccessControlException, IOException
+         UnresolvedLinkException if symbolic link on path cannot be resolved
+          internally]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
+ that the Path f must be fully qualified and the permission is absolute
+ (i.e. umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="createInternal" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
+ have been declared explicitly.]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
+ f must be fully qualified and the permission is absolute (i.e. 
+ umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#delete(Path, boolean)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path, int)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#truncate(Path, long)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setReplication(Path, short)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system and NO OVERWRITE is performed.
+ 
+ File systems that do not have a built in overwrite need implement only this
+ method and can take advantage of the default impl of the other
+ {@link #renameInternal(Path, Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the file system supports symlinks, false otherwise.
+ @return true if filesystem supports symlinks]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of  
+ {@link FileContext#createSymlink(Path, Path, boolean)};]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partially resolves the path. This is used during symlink resolution in
+ {@link FSLinkResolver}, and differs from the similarly named method
+ {@link FileContext#getLinkTarget(Path)}.
+ @throws IOException subclass implementations may throw IOException]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setPermission(Path, FsPermission)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setOwner(Path, String, String)} except that Path f must
+ be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setTimes(Path, long, long)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileChecksum(Path)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileStatus(Path)} 
+ except that an UnresolvedLinkException may be thrown if a symlink is 
+ encountered in the path.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p/>In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileLinkStatus(Path)}
+ except that an UnresolvedLinkException may be thrown if a symlink is  
+ encountered in the path leading up to the final path component.
+ If the file system does not support symlinks then the behavior is
+ equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileBlockLocations(Path, long, long)} except that
+ Path f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listLocatedStatus(Path)} except that Path f 
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext.Util#listStatus(Path)} except that Path f must be 
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical name for this file system.
+ @return a URI string that uniquely identifies this file system]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications, must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator<AclStatus> which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only the xattr names for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#createSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#renameSnapshot(Path, String, String)}.]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="snapshotDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#deleteSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for implementors of a Hadoop file system
+ (analogous to the VFS of Unix). Applications do not access this class;
+ instead they access files across all file systems using {@link FileContext}.
+ 
+ Pathnames passed to AbstractFileSystem can be fully qualified URI that
+ matches the "this" file system (ie same scheme and authority) 
+ or a Slash-relative name that is assumed to be relative
+ to the root of the "this" file system .]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AbstractFileSystem -->
+  <!-- start class org.apache.hadoop.fs.AvroFSInput -->
+  <class name="AvroFSInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.avro.file.SeekableInput"/>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FSDataInputStream, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct given an {@link FSDataInputStream} and its length.]]>
+      </doc>
+    </constructor>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FileContext, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct given a {@link FileContext} and a {@link Path}.]]>
+      </doc>
+    </constructor>
+    <method name="length" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="tell" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AvroFSInput -->
+  <!-- start class org.apache.hadoop.fs.BlockLocation -->
+  <class name="BlockLocation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default Constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="org.apache.hadoop.fs.BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset and length]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset, length and corrupt flag]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset and length]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset, length 
+ and corrupt flag]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], org.apache.hadoop.fs.StorageType[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting this block]]>
+      </doc>
+    </method>
+    <method name="getCachedHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting a cached replica of the block]]>
+      </doc>
+    </method>
+    <method name="getNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of names (IP:xferPort) hosting this block]]>
+      </doc>
+    </method>
+    <method name="getTopologyPaths" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of network topology paths for each of the hosts.
+ The last component of the path is the "name" (IP:xferPort).]]>
+      </doc>
+    </method>
+    <method name="getStorageIds" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storageID of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storage type of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start offset of file associated with this block]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of the block]]>
+      </doc>
+    </method>
+    <method name="isCorrupt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setOffset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="offset" type="long"/>
+      <doc>
+      <![CDATA[Set the start offset of file associated with this block]]>
+      </doc>
+    </method>
+    <method name="setLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[Set the length of block]]>
+      </doc>
+    </method>
+    <method name="setCorrupt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="corrupt" type="boolean"/>
+      <doc>
+      <![CDATA[Set the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hosts" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the hosts hosting this block]]>
+      </doc>
+    </method>
+    <method name="setCachedHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cachedHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the hosts hosting a cached replica of this block]]>
+      </doc>
+    </method>
+    <method name="setNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the names (host:port) hosting this block]]>
+      </doc>
+    </method>
+    <method name="setTopologyPaths"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="topologyPaths" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the network topology paths of the hosts]]>
+      </doc>
+    </method>
+    <method name="setStorageIds"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageIds" type="java.lang.String[]"/>
+    </method>
+    <method name="setStorageTypes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="org.apache.hadoop.fs.StorageType[]"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Represents the network location of a block, information about the hosts
+ that contain block replicas, and other block metadata (E.g. the file
+ offset associated with the block, length, whether it is corrupt, etc).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.BlockLocation -->
+  <!-- start interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <interface name="BlockStoragePolicySpi"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the storage policy. Policies are uniquely
+ identified by name.
+
+ @return the name of the storage policy.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the preferred storage types associated with this policy. These
+ storage types are used sequentially for successive block replicas.
+
+ @return preferred storage types used for placing block replicas.]]>
+      </doc>
+    </method>
+    <method name="getCreationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for creating new block replicas. Fallback
+ storage types are used if the preferred storage types are not available.
+
+ @return fallback storage types for new block replicas..]]>
+      </doc>
+    </method>
+    <method name="getReplicationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for replicating existing block replicas.
+ Fallback storage types are used if the preferred storage types are not
+ available.
+
+ @return fallback storage types for replicating existing block replicas.]]>
+      </doc>
+    </method>
+    <method name="isCopyOnCreateFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the policy is inherit-only and cannot be changed for
+ an existing file.
+
+ @return true if the policy is inherit-only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A storage policy specifies the placement of block replicas on specific
+ storage types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <interface name="ByteBufferPositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to {@code buf.remaining()} bytes into buf from a given position in
+ the file and returns the number of bytes read. Callers should use
+ {@code buf.limit(...)} to control the size of the desired read and
+ {@code buf.position(...)} to control the offset into the buffer the data
+ should be written to.
+ <p>
+ After a successful call, {@code buf.position()} will be advanced by the
+ number of bytes read and {@code buf.limit()} will be unchanged.
+ <p>
+ In the case of an exception, the state of the buffer (the contents of the
+ buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is
+ undefined, and callers should be prepared to recover from this eventuality.
+ <p>
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with
+ {@link StreamCapabilities#PREADBYTEBUFFER} to check if the underlying
+ stream supports this interface, otherwise they might get a
+ {@link UnsupportedOperationException}.
+ <p>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+
+ @param position position within file
+ @param buf the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if reached
+         end-of-stream
+ @throws IOException if there is some error performing the read]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a positioned read API that writes to a
+ {@link ByteBuffer} rather than a {@code byte[]}.
+
+ @see PositionedReadable
+ @see ByteBufferReadable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <interface name="ByteBufferReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to buf.remaining() bytes into buf. Callers should use
+ buf.limit(..) to control the size of the desired read.
+ <p/>
+ After a successful call, buf.position() will be advanced by the number 
+ of bytes read and buf.limit() should be unchanged.
+ <p/>
+ In the case of an exception, the values of buf.position() and buf.limit()
+ are undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p/>
+ Many implementations will throw {@link UnsupportedOperationException}, so
+ callers that are not confident in support for this method from the
+ underlying filesystem should be prepared to handle that exception.
+ <p/>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+
+ @param buf
+          the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if 
+         reach end-of-stream
+ @throws IOException
+           if there is some error performing the read]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a read API that writes to a
+ ByteBuffer, not a byte[].]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <!-- start interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <interface name="CanSetDropBehind"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropCache" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Configure whether the stream should drop the cache.
+
+ @param dropCache     Whether to drop the cache.  null means to use the
+                      default value.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting the drop-behind.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <!-- start interface org.apache.hadoop.fs.CanSetReadahead -->
+  <interface name="CanSetReadahead"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setReadahead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Set the readahead on this stream.
+
+ @param readahead     The readahead to use.  null means to use the default.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting readahead.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetReadahead -->
+  <!-- start interface org.apache.hadoop.fs.CanUnbuffer -->
+  <interface name="CanUnbuffer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reduce the buffering.  This will also free sockets and file descriptors
+ held by the stream, if possible.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSDataInputStreams implement this interface to indicate that they can clear
+ their buffers on request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanUnbuffer -->
+  <!-- start class org.apache.hadoop.fs.ChecksumException -->
+  <class name="ChecksumException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumException" type="java.lang.String, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown for checksum errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumException -->
+  <!-- start class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <class name="ChecksumFileSystem" extends="org.apache.hadoop.fs.FilterFileSystem"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApproxChkSumLength" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to verify checksum.]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the raw file system]]>
+      </doc>
+    </method>
+    <method name="getChecksumFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the name of the checksum file associated with a file.]]>
+      </doc>
+    </method>
+    <method name="isChecksumFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return true iff file is a checksum file name.]]>
+      </doc>
+    </method>
+    <method name="getChecksumFileLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="fileSize" type="long"/>
+      <doc>
+      <![CDATA[Return the length of the checksum file given the size of the 
+ actual file.]]>
+      </doc>
+    </method>
+    <method name="getBytesPerSum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the bytes Per Checksum]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChecksumLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <param name="bytesPerSum" type="int"/>
+      <doc>
+      <![CDATA[Calculated the length of the checksum file in bytes.
+ @param size the length of the data file in bytes
+ @param bytesPerSum the number of bytes in a checksum block
+ @return the number of bytes in the checksum file]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename files/dirs]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Implement the delete(Path, boolean) in checksum
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given path
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given patch
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="copyCrc" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ If src and dst are directories, the copyCrc parameter
+ determines whether to copy CRC files.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Report a checksum error to the file system.
+ @param f the file name containing the error
+ @param in the stream open on the file
+ @param inPos the position of the beginning of the bad data in the file
+ @param sums the stream open on the checksum file
+ @param sumsPos the position of the beginning of the bad data in the checksum file
+ @return if retry is neccessary]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abstract Checksumed FileSystem.
+ It provide a basic implementation of a Checksumed FileSystem,
+ which creates a checksum file for each raw file.
+ It generates & verifies checksums at the client side.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <!-- start class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <class name="CommonConfigurationKeysPublic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommonConfigurationKeysPublic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="IO_NATIVE_LIB_AVAILABLE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_NATIVE_LIB_AVAILABLE_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_NATIVE_LIB_AVAILABLE_KEY]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DEFAULT_NAME_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DF_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DU_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_GETSPACEUSED_JITTER_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_CHECKPOINT_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_PROTECTED_DIRECTORIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Directories that cannot be removed unless empty, even by an
+ administrator.]]>
+      </doc>
+    </field>
+    <field name="FS_LOCAL_BLOCK_SIZE_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Not used anywhere, looks like default value for FS_LOCAL_BLOCK_SIZE]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_AUTOMATIC_CLOSE_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_FILE_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_PORT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_ERROR_RATE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_COMPRESSION_CODEC_LZO_CLASS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Codec class that implements Lzo compression algorithm]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_INTERVAL_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_SKIP_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_FILE_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SKIP_CHECKSUM_ERRORS_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SERIALIZATIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_IO_CHUNK_SIZE_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_INPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_OUTPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable low-latency connections from the client]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of IPC_CLIENT_LOW_LATENCY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_LISTEN_QUEUE_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_KILL_MAX_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_IDLETHRESHOLD_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_MAX_CONNECTIONS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Logs if a RPC is really slow compared to rest of RPCs.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SOCKS_SERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_UTIL_HASH_TYPE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_MAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHENTICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHORIZATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SERVICE_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_INTERFACE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_NAMESERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_AUTHENTICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SSL_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SSL_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[See <a href="{@docRoot}/../core-default.html">core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN]]>
+      </doc>
+    </field>
+    <field name="HTTP_POLICY_HTTP_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HTTP_POLICY_HTTPS_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_RPC_PROTECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Sasl Properties for a connection]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Impersonation provider]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_SIZE]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_NUM_REFILL_THREADS]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is the number of providers specified.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 100 ms.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 2 secs.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_HTTP_LOGS_ENABLED]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option for the shutdown hook manager shutdown time:
+  {@value}.]]>
+      </doc>
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default shutdown hook timeout: {@value} seconds.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class contains constants for configuration keys used
+ in the common code.
+
+ It includes all publicly documented configuration keys. In general
+ this class should not be used directly (use CommonConfigurationKeys
+ instead)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <!-- start class org.apache.hadoop.fs.ContentSummary -->
+  <class name="ContentSummary" extends="org.apache.hadoop.fs.QuotaUsage"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ContentSummary"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder
+  This constructor implicitly set spaceConsumed the same as length.
+  spaceConsumed and length must be set explicitly with
+  ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the length]]>
+      </doc>
+    </method>
+    <method name="getSnapshotLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the directory count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the file count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSnapshotSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the header of the output.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="getHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields from the summary header.
+ 
+ @return names of fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="getQuotaHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields used in the quota summary.
+ 
+ @return names of quota fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ if hOption is false, file sizes are returned in bytes
+ if hOption is true, file sizes are returned in human readable
+ if tOption is true, display the quota by storage types
+ if tOption is false, same logic with #toString(boolean,boolean)
+ if xOption is false, output includes the calculation from snapshots
+ if xOption is true, output excludes the calculation from snapshots
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Store the summary of a content (a directory or a file).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ContentSummary -->
+  <!-- start class org.apache.hadoop.fs.CreateFlag -->
+  <class name="CreateFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.CreateFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.CreateFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag and throw exception if it is invalid
+ @param flag set of CreateFlag
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.Object"/>
+      <param name="pathExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for create operation
+ @param path Object representing the path; usually String or {@link Path}
+ @param pathExists pass true if the path exists in the file system
+ @param flag set of CreateFlag
+ @throws IOException on error
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validateForAppend"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for the append operation. The flag must contain
+ APPEND, and cannot contain OVERWRITE.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[CreateFlag specifies the file create semantic. Users can combine flags like: <br>
+ <code>
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
+ <code>
+ <p>
+ 
+ Use the CreateFlag as follows:
+ <ol>
+ <li> CREATE - to create a file if it does not exist, 
+ else throw FileAlreadyExists.</li>
+ <li> APPEND - to append to a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> OVERWRITE - to truncate a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> CREATE|APPEND - to create a file if it does not exist, 
+ else append to an existing file.</li>
+ <li> CREATE|OVERWRITE - to create a file if it does not exist, 
+ else overwrite an existing file.</li>
+ <li> SYNC_BLOCK - to force closed blocks to the disk device.
+ In addition {@link Syncable#hsync()} should be called after each write,
+ if true synchronous behavior is required.</li>
+ <li> LAZY_PERSIST - Create the block on transient storage (RAM) if
+ available.</li>
+ <li> APPEND_NEWBLOCK - Append data to a new block instead of end of the last
+ partial block.</li>
+ </ol>
+ 
+ Following combinations are not valid and will result in
+ {@link HadoopIllegalArgumentException}:
+ <ol>
+ <li> APPEND|OVERWRITE</li>
+ <li> CREATE|APPEND|OVERWRITE</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CreateFlag -->
+  <!-- start class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.fs.FileChecksum -->
+  <class name="FileChecksum" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileChecksum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAlgorithmName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The checksum algorithm name]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The length of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The value of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if both the algorithms and the values are the same.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An abstract class representing file checksums for files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileChecksum -->
+  <!-- start class org.apache.hadoop.fs.FileContext -->
+  <class name="FileContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getFSofPath" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the file system of supplied path.
+ 
+ @param absOrFqPath - absolute or fully qualified path
+ @return the file system of the path
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>absOrFqPath</code> is not supported.
+ @throws IOException If the file system for <code>absOrFqPath</code> could
+         not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a FileContext with specified FS as default using the specified
+ config.
+ 
+ @param defFS
+ @param aConf
+ @return new FileContext with specifed FS as default.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="defaultFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified file system using the default config.
+ 
+ @param defaultFS
+ @return a FileContext with the specified AbstractFileSystem
+                 as the default FS.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the default config read from the
+ $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
+ from core-defaults.xml in the release jar.
+ 
+ @throws UnsupportedFileSystemException If the file system from the default
+           configuration is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@return a FileContext for the local file system using the default config.
+ @throws UnsupportedFileSystemException If the file system for
+           {@link FsConstants#LOCAL_FS_URI} is not supported.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified URI using the default config.
+ 
+ @param defaultFsUri
+ @return a FileContext with the specified URI as the default FS.
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>defaultFsUri</code> is not supported]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified default URI using the specified config.
+ 
+ @param defaultFsUri
+ @param aConf
+ @return new FileContext for specified uri
+ @throws UnsupportedFileSystemException If the file system with specified is
+           not supported
+ @throws RuntimeException If the file system specified is supported but
+         could not be instantiated, or if login fails.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the passed config. Generally it is better to use
+ {@link #getFileContext(URI, Configuration)} instead of this one.
+ 
+ 
+ @param aConf
+ @return new FileContext
+ @throws UnsupportedFileSystemException If file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@param aConf - from which the FileContext is configured
+ @return a FileContext for the local file system using the specified config.
+ 
+ @throws UnsupportedFileSystemException If default file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newWDir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the working directory for wd-relative names (such a "foo/bar"). Working
+ directory feature is provided by simply prefixing relative names with the
+ working dir. Note this is different from Unix where the wd is actually set
+ to the inode. Hence setWorkingDir does not follow symlinks etc. This works
+ better in a distributed environment that has multiple independent roots.
+ {@link #getWorkingDirectory()} should return what setWorkingDir() set.
+ 
+ @param newWDir new working directory
+ @throws IOException 
+ <br>
+           NewWdir can be one of:
+           <ul>
+           <li>relative path: "foo/bar";</li>
+           <li>absolute without scheme: "/foo/bar"</li>
+           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
+           </ul>
+ <br>
+           Illegal WDs:
+           <ul>
+           <li>relative with scheme: "xx:foo/bar"</li>
+           <li>non existent directory</li>
+           </ul>]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the working directory for wd-relative names (such a "foo/bar").]]>
+      </doc>
+    </method>
+    <method name="getUgi" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the ugi in the file-context
+ @return UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ @return the home directory]]>
+      </doc>
+    </method>
+    <method name="getUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the umask of this FileContext]]>
+      </doc>
+    </method>
+    <method name="setUMask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newUmask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set umask to the supplied parameter.
+ @param newUmask  the new umask]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolve the path following any symlinks or mount points
+ @param f to be resolved
+ @return fully qualified resolved path
+ 
+ @throws FileNotFoundException  If <code>f</code> does not exist
+ @throws AccessControlException if access denied
+ @throws IOException If an IO Error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified if it is isn't. 
+ A Fully-qualified path has scheme and authority specified and an absolute
+ path.
+ Use the default file system and working dir in this FileContext to qualify.
+ @param path
+ @return qualified path]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create or overwrite file on indicated path and returns an output stream for
+ writing into the file.
+ 
+ @param f the file name to open
+ @param createFlag gives the semantics of create; see {@link CreateFlag}
+ @param opts file creation options; see {@link Options.CreateOpts}.
+          <ul>
+          <li>Progress - to report progress on the operation - default null
+          <li>Permission - umask is applied against permisssion: default is
+          FsPermissions:getDefault()
+ 
+          <li>CreateParent - create missing parent path; default is to not
+          to create parents
+          <li>The defaults for the following are SS defaults of the file
+          server implementing the target path. Not all parameters make sense
+          for all kinds of file system - eg. localFS ignores Blocksize,
+          replication, checksum
+          <ul>
+          <li>BufferSize - buffersize used in FSDataOutputStream
+          <li>Blocksize - block size for file blocks
+          <li>ReplicationFactor - replication for blocks
+          <li>ChecksumParam - Checksum parameters. server default is used
+          if not specified.
+          </ul>
+          </ul>
+ 
+ @return {@link FSDataOutputStream} for created file
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>f</code> already exists
+ @throws FileNotFoundException If parent of <code>f</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>f</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make(create) a directory and all the non-existent parents.
+ 
+ @param dir - the dir to make
+ @param permission - permissions is set permission&~umask
+ @param createParent - if true then missing parent dirs are created if false
+          then parent must exist
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If directory <code>dir</code> already
+           exists
+ @throws FileNotFoundException If parent of <code>dir</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>dir</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>dir</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+ @param f the path to delete.
+ @param recursive if path is a directory and set to 
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is invalid]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path using
+ default buffersize.
+ @param f the file name to open
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ 
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+ <li>Fails if path is a directory.
+ <li>Fails if path does not exist.
+ <li>Fails if path is not closed.
+ <li>Fails if new size is greater than current size.
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param f file name
+ @param replication new replication
+
+ @return true if successful
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+ <li
+ <li>Fails if src is a file and dst is a directory.
+ <li>Fails if src is a directory and dst is a file.
+ <li>Fails if the parent of dst does not exist or is a file.
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails if the dst
+ already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites the dst if
+ it is a file or an empty directory. Rename fails if dst is a non-empty
+ directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for details
+ <p>
+ 
+ @param src path to be renamed
+ @param dst new path after rename
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If <code>dst</code> already exists and
+           <code>options</options> has {@link Options.Rename#OVERWRITE} 
+           option false.
+ @throws FileNotFoundException If <code>src</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>src</code>
+           and <code>dst</code> is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param f
+ @param permission - the new absolute permission (umask is not applied)
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory). The parameters username
+ and groupname cannot both be null.
+ 
+ @param f The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws HadoopIllegalArgumentException If <code>username</code> or
+           <code>groupname</code> is invalid.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param f The path
+ @param mtime Set the modification time of this file.
+        The number of milliseconds since epoch (Jan 1, 1970). 
+        A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+        The number of milliseconds since Jan 1, 1970. 
+        A value of -1 means that this call should not set access time.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file.
+
+ @param f file path
+
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag for the  file system denoted by the path.
+ This is only applicable if the 
+ corresponding FileSystem supports checksum. By default doesn't do anything.
+ @param verifyChecksum
+ @param f set the verifyChecksum for the Filesystem containing this path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+
+ @return a FileStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path. If the path 
+ refers to a symlink then the FileStatus of the symlink is returned.
+ The behavior is equivalent to #getFileStatus() if the underlying
+ file system does not support symbolic links.
+ @param  f The path we want information from.
+ @return A FileStatus object
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the target of the given symbolic link as it was specified
+ when the link was created.  Links in the path leading up to the
+ final path component are resolved transparently.
+
+ @param f the path to return the target of
+ @return The un-interpreted target of the symbolic link.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If path <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If the given path does not refer to a symlink
+           or an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ file system denoted by the Parh argument p.
+ If the file system has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ 
+ @param f Path for which status should be obtained. null means the
+ root partition of the default file system. 
+
+ @return a FsStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a symbolic link to an existing file. An exception is thrown if 
+ the symlink exits, the user does not have permission to create symlink,
+ or the underlying file system does not support symlinks.
+ 
+ Symlink permissions are ignored, access to a symlink is determined by
+ the permissions of the symlink target.
+ 
+ Symlinks in paths leading up to the final path component are resolved 
+ transparently. If the final path component refers to a symlink some 
+ functions operate on the symlink itself, these are:
+ - delete(f) and deleteOnExit(f) - Deletes the symlink.
+ - rename(src, dst) - If src refers to a symlink, the symlink is 
+   renamed. If dst refers to a symlink, the symlink is over-written.
+ - getLinkTarget(f) - Returns the target of the symlink. 
+ - getFileLinkStatus(f) - Returns a FileStatus object describing
+   the symlink.
+ Some functions, create() and mkdir(), expect the final path component
+ does not exist. If they are given a path that refers to a symlink that 
+ does exist they behave as if the path referred to an existing file or 
+ directory. All other functions fully resolve, ie follow, the symlink. 
+ These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
+ setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
+ getFsStatus, getFileStatus, exists, and listStatus.
+ 
+ Symlink targets are stored as given to createSymlink, assuming the 
+ underlying file system is capable of storing a fully qualified URI.
+ Dangling symlinks are permitted. FileContext supports four types of 
+ symlink targets, and resolves them as follows
+ <pre>
+ Given a path referring to a symlink of form:
+ 
+   <---X---> 
+   fs://host/A/B/link 
+   <-----Y----->
+ 
+ In this path X is the scheme and authority that identify the file system,
+ and Y is the path leading up to the final path component "link". If Y is
+ a symlink  itself then let Y' be the target of Y and X' be the scheme and
+ authority of Y'. Symlink targets may:
+ 
+ 1. Fully qualified URIs
+ 
+ fs://hostX/A/B/file  Resolved according to the target file system.
+ 
+ 2. Partially qualified URIs (eg scheme but no host)
+ 
+ fs:///A/B/file  Resolved according to the target file system. Eg resolving
+                 a symlink to hdfs:///A results in an exception because
+                 HDFS URIs must be fully qualified, while a symlink to 
+                 file:///A will not since Hadoop's local file systems 
+                 require partially qualified URIs.
+ 
+ 3. Relative paths
+ 
+ path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
+       is "../B/file" then [Y'][path] is hdfs://host/B/file
+ 
+ 4. Absolute paths
+ 
+ path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
+       is "/file" then [X][path] is hdfs://host/file
+ </pre>
+ 
+ @param target the target of the symbolic link
+ @param link the path to be created that points to target
+ @param createParent if true then missing parent dirs are created if 
+                     false then parent must exist
+
+
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>linkcode> already exists
+ @throws FileNotFoundException If <code>target</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>link</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for 
+           <code>target</code> or <code>link</code> is not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory. 
+ Return the file's status and block locations If the path is a file.
+ 
+ If a returned status is a file, it contains the file's block locations.
+ 
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+ If any IO exception (for example the input directory gets deleted while
+ listing is being executed), next() or hasNext() of the returned iterator
+ may throw a RuntimeException with the io exception as the cause.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted on JVM shutdown.
+ 
+ @param f the existing path to delete.
+
+ @return  true if deleteOnExit is successful, otherwise false.
+
+ @throws AccessControlException If access is denied
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="util" return="org.apache.hadoop.fs.FileContext.Util"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolve" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path.
+ Returns the new path object.]]>
+      </doc>
+    </method>
+    <method name="resolveIntermediate" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path leading up 
+ to, but not including the final path component.
+ @param f path to resolve
+ @return the new path object.]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system
+ 
+ @param uri
+          the uri to lookup the statistics. Only scheme and authority part
+          of the uri are used as the key to store and lookup.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all the statistics stored in AbstractFileSystem, for all the file
+ systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints the statistics to standard output. File System is identified by the
+ scheme and authority.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Map of uri and statistics for each filesystem instantiated. The uri
+         consists of scheme and authority for the filesystem.]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications, must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator<AclStatus> which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List<String> of the XAttr names of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+
+ @param path The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param path file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory and symlink
+ In previous versions, this default permission was also used to
+ create files, so files created end up with ugo+x permission.
+ See HADOOP-9155 for detail. 
+ Two new constants are added to solve this, please use 
+ {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ {@link FileContext#FILE_DEFAULT_PERM} for file.
+ This constant is kept for compatibility.]]>
+      </doc>
+    </field>
+    <field name="DIR_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory]]>
+      </doc>
+    </field>
+    <field name="FILE_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for file]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileContext shutdown hook.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The FileContext class provides an interface to the application writer for
+ using the Hadoop file system.
+ It provides a set of methods for the usual operation: create, open, 
+ list, etc 
+ 
+ <p>
+ <b> *** Path Names *** </b>
+ <p>
+ 
+ The Hadoop file system supports a URI name space and URI names.
+ It offers a forest of file systems that can be referenced using fully
+ qualified URIs.
+ Two common Hadoop file systems implementations are
+ <ul>
+ <li> the local file system: file:///path
+ <li> the hdfs file system hdfs://nnAddress:nnPort/path
+ </ul>
+ 
+ While URI names are very flexible, it requires knowing the name or address
+ of the server. For convenience one often wants to access the default system
+ in one's environment without knowing its name/address. This has an
+ additional benefit that it allows one to change one's default fs
+  (e.g. admin moves application from cluster1 to cluster2).
+ <p>
+ 
+ To facilitate this, Hadoop supports a notion of a default file system.
+ The user can set his default file system, although this is
+ typically set up for you in your environment via your default config.
+ A default file system implies a default scheme and authority; slash-relative
+ names (such as /for/bar) are resolved relative to that default FS.
+ Similarly a user can also have working-directory-relative names (i.e. names
+ not starting with a slash). While the working directory is generally in the
+ same default FS, the wd can be in a different FS.
+ <p>
+  Hence Hadoop path names can be one of:
+  <ul>
+  <li> fully qualified URI: scheme://authority/path
+  <li> slash relative names: /path relative to the default file system
+  <li> wd-relative names: path  relative to the working dir
+  </ul>   
+  Relative paths with scheme (scheme:foo/bar) are illegal.
+  
+  <p>
+  <b>****The Role of the FileContext and configuration defaults****</b>
+  <p>
+  The FileContext provides file namespace context for resolving file names;
+  it also contains the umask for permissions, In that sense it is like the
+  per-process file-related state in Unix system.
+  These two properties
+  <ul> 
+  <li> default file system i.e your slash)
+  <li> umask
+  </ul>
+  in general, are obtained from the default configuration file
+  in your environment,  (@see {@link Configuration}).
+  
+  No other configuration parameters are obtained from the default config as 
+  far as the file context layer is concerned. All file system instances
+  (i.e. deployments of file systems) have default properties; we call these
+  server side (SS) defaults. Operation like create allow one to select many 
+  properties: either pass them in as explicit parameters or use
+  the SS properties.
+  <p>
+  The file system related SS defaults are
+  <ul>
+  <li> the home directory (default is "/user/userName")
+  <li> the initial wd (only for local fs)
+  <li> replication factor
+  <li> block size
+  <li> buffer size
+  <li> encryptDataTransfer 
+  <li> checksum option. (checksumType and  bytesPerChecksum)
+  </ul>
+
+ <p>
+ <b> *** Usage Model for the FileContext class *** </b>
+ <p>
+ Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
+   Unspecified values come from core-defaults.xml in the release jar.
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(); // uses the default config
+                                                // which has your default FS 
+  <li>  myFContext.create(path, ...);
+  <li>  myFContext.setWorkingDir(path)
+  <li>  myFContext.open (path, ...);  
+  </ul>  
+ Example 2: Get a FileContext with a specific URI as the default FS
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(URI)
+  <li> myFContext.create(path, ...);
+   ...
+ </ul> 
+ Example 3: FileContext with local file system as the default
+  <ul> 
+  <li> myFContext = FileContext.getLocalFSFileContext()
+  <li> myFContext.create(path, ...);
+  <li> ...
+  </ul> 
+ Example 4: Use a specific config, ignoring $HADOOP_CONFIG
+  Generally you should not need use a config unless you are doing
+   <ul> 
+   <li> configX = someConfigSomeOnePassedToYou.
+   <li> myFContext = getFileContext(configX); // configX is not changed,
+                                              // is passed down 
+   <li> myFContext.create(path, ...);
+   <li>...
+  </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileContext -->
+  <!-- start class org.apache.hadoop.fs.FileStatus -->
+  <class name="FileStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <constructor name="FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for file systems on which symbolic links are not supported]]>
+      </doc>
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="org.apache.hadoop.fs.FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor.
+
+ @param other FileStatus to copy]]>
+      </doc>
+    </constructor>
+    <method name="getLen" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of this file, in bytes.
+ @return the length of this file, in bytes.]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a file?
+ @return true if this is a file]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a directory?
+ @return true if this is a directory]]>
+      </doc>
+    </method>
+    <method name="isDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link FileStatus#isFile()},  
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()} 
+ instead.">
+      <doc>
+      <![CDATA[Old interface, instead use the explicit {@link FileStatus#isFile()}, 
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()} 
+ @return true if this is a directory.
+ @deprecated Use {@link FileStatus#isFile()},  
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()} 
+ instead.]]>
+      </doc>
+    </method>
+    <method name="isSymlink" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a symbolic link?
+ @return true if this is a symbolic link]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the block size of the file.
+ @return the number of bytes]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the replication factor of a file.
+ @return the replication factor of a file.]]>
+      </doc>
+    </method>
+    <method name="getModificationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modification time of the file.
+ @return the modification time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getAccessTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the access time of the file.
+ @return the access time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get FsPermission associated with the file.
+ @return permssion. If a filesystem does not have a notion of permissions
+         or if permissions could not be determined, then default 
+         permissions equivalent of "rwxrwxrwx" is returned.]]>
+      </doc>
+    </method>
+    <method name="isEncrypted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is encrypted or not.
+
+ @return true if the underlying file is encrypted.]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the owner of the file.
+ @return owner of the file. The string could be empty if there is no
+         notion of owner of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getGroup" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group associated with the file.
+ @return group for the file. The string could be empty if there is no
+         notion of group of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Sets permission.
+ @param permission if permission is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets owner.
+ @param owner if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets group.
+ @param group if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return The contents of the symbolic link.]]>
+      </doc>
+    </method>
+    <method name="setSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface that represents the client side information for a file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileStatus -->
+  <!-- start class org.apache.hadoop.fs.FileSystem -->
+  <class name="FileSystem" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="FileSystem"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a FileSystem instance based on the uri, the passed in
+ configuration and the user.
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return the filesystem instance
+ @throws IOException failure to load
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+ somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the configured FileSystem implementation.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="getDefaultUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the default FileSystem URI from a configuration.
+ @param conf the configuration to use
+ @return the uri of the default filesystem]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a FileSystem.
+
+ Called after the new FileSystem instance is constructed, and before it
+ is ready for use.
+
+ FileSystem implementations overriding this method MUST forward it to
+ their superclass, though the order in which it is done, and whether
+ to alter the configuration before the invocation are options of the
+ subclass.
+ @param name a URI whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration
+ @throws IOException on any failure to initialize this instance.
+ @throws IllegalArgumentException if the URI is considered invalid.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for this FileSystem.
+ <p>
+ This implementation throws an <code>UnsupportedOperationException</code>.
+
+ @return the protocol scheme for this FileSystem.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI which identifies this FileSystem.
+
+ @return the URI of this filesystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a canonicalized form of this FileSystem's URI.
+
+ The default implementation simply calls {@link #canonicalizeUri(URI)}
+ on the filesystem's own URI, so subclasses typically only need to
+ implement that method.
+
+ @see #canonicalizeUri(URI)]]>
+      </doc>
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Canonicalize the given URI.
+
+ This is implementation-dependent, and may for example consist of
+ canonicalizing the hostname using DNS and adding the default
+ port if not specified.
+
+ The default implementation simply fills in the default port if
+ not specified and if {@link #getDefaultPort()} returns a
+ default port.
+
+ @return URI
+ @see NetUtils#getCanonicalUri(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FileSystem.
+ @return the default port or 0 if there isn't one]]>
+      </doc>
+    </method>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical service name for this FileSystem.
+ The token cache is the only user of the canonical service name,
+ and uses it to lookup this FileSystem's service tokens.
+ If the file system provides a token of its own then it must have a canonical
+ name, otherwise the canonical name can be null.
+
+ Default implementation: If the FileSystem has child file systems
+ (such as an embedded file system) then it is assumed that the FS has no
+ tokens of its own and hence returns a null name; otherwise a service
+ name is built using Uri and port.
+
+ @return a service string that uniquely identifies this file system, null
+         if the filesystem does not implement tokens
+ @see SecurityUtil#buildDTServiceName(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="call {@link #getUri()} instead.">
+      <doc>
+      <![CDATA[@deprecated call {@link #getUri()} instead.]]>
+      </doc>
+    </method>
+    <method name="getNamed" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="call {@link #get(URI, Configuration)} instead.">
+      <param name="name" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated call {@link #get(URI, Configuration)} instead.]]>
+      </doc>
+    </method>
+    <method name="getLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the local FileSystem.
+ @param conf the configuration to configure the FileSystem with
+ if it is newly instantiated.
+ @return a LocalFileSystem
+ @throws IOException if somehow the local FS cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a FileSystem for this URI's scheme and authority.
+ <ol>
+ <li>
+   If the configuration has the property
+   {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+   a new instance will be created, initialized with the supplied URI and
+   configuration, then returned without being cached.
+ </li>
+ <li>
+   If the there is a cached FS instance matching the same URI, it will
+   be returned.
+ </li>
+ <li>
+   Otherwise: a new FS instance will be created, initialized with the
+   configuration and URI, cached and returned to the caller.
+ </li>
+ </ol>
+ @throws IOException if the FileSystem cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority and the
+ given user. Internally invokes {@link #newInstance(URI, Configuration)}
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return filesystem instance
+ @throws IOException if the FileSystem cannot be instantiated.
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+         somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority.
+ The entire URI is passed to the FileSystem instance's initialize method.
+ This always returns a new FileSystem object.
+ @param uri FS URI
+ @param config configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a unique configured FileSystem implementation for the default
+ filesystem of the supplied configuration.
+ This always returns a new FileSystem object.
+ @param conf the configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstanceLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a unique local FileSystem object.
+ @param conf the configuration to configure the FileSystem with
+ @return a new LocalFileSystem object.
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="closeAll"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances. After this operation, they
+ may not be used in any operations.
+
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="closeAllForUGI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances for a given UGI.
+ Be sure those filesystems are not used anymore.
+ @param ugi user group info to close
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Qualify a path to one which uses this FileSystem and, if relative,
+ made absolute.
+ @param path to qualify.
+ @return this path if it contains a scheme and authority and is absolute, or
+ a new path that includes a path and authority and is fully qualified
+ @see Path#makeQualified(URI, Path)
+ @throws IllegalArgumentException if the path has a schema/URI different
+ from this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="addDelegationTokens" return="org.apache.hadoop.security.token.Token[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <param name="credentials" type="org.apache.hadoop.security.Credentials"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Obtain all delegation tokens used by this FileSystem that are not
+ already present in the given Credentials. Existing tokens will neither
+ be verified as valid nor having the given renewer.  Missing tokens will
+ be acquired and added to the given Credentials.
+
+ Default Impl: works for simple FS with its own token
+ and also for an embedded FS whose tokens are those of its
+ child FileSystems (i.e. the embedded FS has no tokens of its own).
+
+ @param renewer the user allowed to renew the delegation tokens
+ @param credentials cache in which to add new delegation tokens
+ @return list of new delegation tokens
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file with the provided permission.
+
+ The permission of the file is set to be the provided permission as in
+ setPermission, not permission&~umask
+
+ The HDFS implementation is implemented using two RPCs.
+ It is understood that it is inefficient,
+ but the implementation is thread-safe. The other option is to change the
+ value of umask in configuration to be 0, but it is not thread-safe.
+
+ @param fs FileSystem
+ @param file the name of the file to be created
+ @param permission the permission of the file
+ @return an output stream
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a directory with the provided permission.
+ The permission of the directory is set to be the provided permission as in
+ setPermission, not permission&~umask
+
+ @see #create(FileSystem, Path, FsPermission)
+
+ @param fs FileSystem handle
+ @param dir the name of the directory to be created
+ @param permission the permission of the directory
+ @return true if the directory creation succeeds; false otherwise
+ @throws IOException A problem creating the directories.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+
+ The base implementation performs case insensitive equality checks
+ of the URIs' schemes and authorities. Subclasses may implement slightly
+ different checks.
+ @param path to check
+ @throws IllegalArgumentException if the path is not considered to be
+ part of this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For nonexistent
+ file or regions, {@code null} is returned.
+
+ <pre>
+   if f == null :
+     result = null
+   elif f.getLen() <= start:
+     result = []
+   else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ </pre>
+ This call is most helpful with and distributed filesystem
+ where the hostnames of machines that contain blocks of the given file
+ can be determined.
+
+ The default implementation returns an array containing one element:
+ <pre>
+ BlockLocation( { "localhost:50010" },  { "localhost" }, 0, file.getLen())
+ </pre>>
+
+ @param file FilesStatus to get data from
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For a nonexistent
+ file or regions, {@code null} is returned.
+
+ This call is most helpful with location-aware distributed
+ filesystems, where it returns hostnames of machines that
+ contain the given file.
+
+ A FileSystem will normally return the equivalent result
+ of passing the {@code FileStatus} of the path to
+ {@link #getFileBlockLocations(FileStatus, long, long)}
+
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @return server default configuration values
+ @throws IOException IO failure
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @return server default configuration values
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path, resolving the path
+ through any symlinks or mount point.
+ @param p path to be resolved
+ @return fully qualified path
+ @throws FileNotFoundException if the path is not present
+ @throws IOException for any other error]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file to open
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an exception will be thrown.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a path with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an {@link FSDataOutputStream} at the indicated Path
+ with write-progress reporting.
+
+ The frequency of callbacks is implementation-specific; it may be "none".
+ @param f the path of the file to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with a custom
+ checksum option.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @param checksumOpt checksum parameter. If null, the values
+        found in conf will be used.
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This create has been added to support the FileContext that processes
+ the permission with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @param f path
+ @param absolutePermission permissions
+ @return true if the directory was actually created.
+ @throws IOException IO failure
+ @see #mkdirs(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNewFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the given Path as a brand-new zero-length file.  If
+ create fails, or if it already existed, return false.
+ <i>Important: the default implementation is not atomic</i>
+ @param f path to use for create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as
+ {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+     IO_FILE_BUFFER_SIZE_DEFAULT), null)}
+ @param f the existing file to be appended.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as append(f, bufferSize, null).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @param progress for reporting progress if it is not null.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Concat existing files together.
+ @param trg the path to the target destination.
+ @param psrcs the paths to the sources to use for the concatenation.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the replication factor.
+
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @param src file name
+ @return file replication
+ @throws FileNotFoundException if the path does not resolve.
+ @throws IOException an IO failure]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the replication for an existing file.
+ If a filesystem does not support replication, it will always
+ return true: the check for a file existing may be bypassed.
+ This is the default behavior.
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful, or the feature in unsupported;
+         false if replication is supported but the file does not exist,
+         or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws IOException on failure
+ @return true if rename is successful]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+   <li>Fails if src is a file and dst is a directory.</li>
+   <li>Fails if src is a directory and dst is a file.</li>
+   <li>Fails if the parent of dst does not exist or is a file.</li>
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails
+ if the dst already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites
+ the dst if it is a file or an empty directory. Rename fails if dst is
+ a non-empty directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for
+ details. This default implementation is non atomic.
+ <p>
+ This method is deprecated since it is a temporary method added to
+ support the transition from FileSystem to FileContext for user
+ applications.
+
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws FileNotFoundException src path does not exist, or the parent
+ path of dst does not exist.
+ @throws FileAlreadyExistsException dest path exists and is a file
+ @throws ParentNotDirectoryException if the parent path of dest is not
+ a directory
+ @throws IOException on failure]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+   <li>Fails if path is a directory.</li>
+   <li>Fails if path does not exist.</li>
+   <li>Fails if path is not closed.</li>
+   <li>Fails if new size is greater than current size.</li>
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #delete(Path, boolean)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file/directory.
+ @deprecated Use {@link #delete(Path, boolean)} instead.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+
+ @param f the path to delete.
+ @param recursive if path is a directory and set to
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+ @return  true if delete is successful else false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted when its FileSystem is closed.
+ When the JVM shuts down cleanly, all cached FileSystem objects will be
+ closed automatically #these the marked paths will be deleted as a result.
+
+ If a FileSystem instance is not cached, i.e. has been created with
+ {@link #createFileSystem(URI, Configuration)}, then the paths will
+ be deleted in when {@link #close()} is called on that instance.
+
+ The path must exist in the filesystem at the time of the method call;
+ it does not have to exist at the time of JVM shutdown.
+
+ Notes
+ <ol>
+   <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+   <li>The time to shut down a FileSystem will depends on the number of
+   files to delete. For filesystems where the cost of checking
+   for the existence of a file/directory and the actual delete operation
+   (for example: object stores) is high, the time to shutdown the JVM can be
+   significantly extended by over-use of this feature.</li>
+   <li>Connectivity problems with a remote filesystem may delay shutdown
+   further, and may cause the files to not be deleted.</li>
+ </ol>
+ @param f the path to delete.
+ @return  true if deleteOnExit is successful, otherwise false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="cancelDeleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Cancel the scheduled deletion of the path when the FileSystem is closed.
+ @param f the path to cancel deletion
+ @return true if the path was found in the delete-on-exit list.]]>
+      </doc>
+    </method>
+    <method name="processDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delete all paths that were marked as delete-on-exit. This recursively
+ deletes all files and directories in the specified paths.
+
+ The time to process this operation is {@code O(paths)}, with the actual
+ time dependent on the time for existence and deletion operations to
+ complete, successfully or not.]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if a path exists.
+ @param f source path
+ @return true if the path exists
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a directory.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by getFileStatus() or listStatus() methods.
+ @param f path to check
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a regular file.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by {@link #getFileStatus(Path)} or listStatus() methods.
+ @param f path to check
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The number of bytes in a file.
+ @return the number of bytes; 0 for a directory
+ @deprecated Use {@link #getFileStatus(Path)} instead.
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getContentSummary" return="org.apache.hadoop.fs.ContentSummary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link ContentSummary} of a given {@link Path}.
+ @param f path to use
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="org.apache.hadoop.fs.QuotaUsage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link QuotaUsage} of a given {@link Path}.
+ @param f path to use
+ @return the quota usage
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+ @param f given path
+ @return the statuses of the files/directories in the given patch
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List corrupted file blocks.
+ @return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given path using the user-supplied path
+ filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param f
+          a path name
+ @param filter
+          the user-supplied path filter
+ @return an array of FileStatus objects for the files under the given path
+         after applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using default
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @return a list of statuses for the files under the given paths after
+         applying the filter default Path filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using user-supplied
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @param filter
+          the user-supplied path filter
+ @return a list of statuses for the files under the given paths after
+         applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Return all the files that match filePattern and are not checksum
+ files. Results are sorted by their names.
+
+ <p>
+ A filename pattern is composed of <i>regular</i> characters and
+ <i>special pattern matching</i> characters, which are:
+
+ <dl>
+  <dd>
+   <dl>
+    <p>
+    <dt> <tt> ? </tt>
+    <dd> Matches any single character.
+
+    <p>
+    <dt> <tt> * </tt>
+    <dd> Matches zero or more characters.
+
+    <p>
+    <dt> <tt> [<i>abc</i>] </tt>
+    <dd> Matches a single character from character set
+     <tt>{<i>a,b,c</i>}</tt>.
+
+    <p>
+    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
+    <dd> Matches a single character from the character range
+     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
+     lexicographically less than or equal to character <tt><i>b</i></tt>.
+
+    <p>
+    <dt> <tt> [^<i>a</i>] </tt>
+    <dd> Matches a single character that is not from character set or range
+     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
+     immediately to the right of the opening bracket.
+
+    <p>
+    <dt> <tt> \<i>c</i> </tt>
+    <dd> Removes (escapes) any special meaning of character <i>c</i>.
+
+    <p>
+    <dt> <tt> {ab,cd} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
+
+    <p>
+    <dt> <tt> {ab,c{de,fh}} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
+
+   </dl>
+  </dd>
+ </dl>
+
+ @param pathPattern a regular expression specifying a pth pattern
+
+ @return an array of paths that match the path pattern
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of {@link FileStatus} objects whose path names match
+ {@code pathPattern}
+ and is accepted by the user-supplied path filter. Results are sorted by
+ their path names.
+ Return
+
+ @param pathPattern
+          a regular expression specifying the path pattern
+ @param filter
+          a user-supplied path filter
+ @return an array of FileStatus objects.
+         null if pathPattern has no glob and the path does not exist.
+         an empty array if pathPattern has a glob and no path matches it.
+ @throws IOException if any I/O error occurs when fetching file status]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ Return the file's status and block locations If the path is a file.
+
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List a directory.
+ The returned results include its block location if it is a file
+ The results are filtered by the given path filter
+ @param f a path
+ @param filter a path filter
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+ @throws FileNotFoundException if <code>f</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a remote iterator so that followup calls are made on demand
+ while consuming the entries. Each FileSystem implementation should
+ override this method and provide a more efficient implementation, if
+ possible.
+
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ @param p target path
+ @return remote iterator
+ @throws FileNotFoundException if <code>p</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses and block locations of the files in the given path.
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+ <pre>
+ If the path is a directory,
+   if recursive is false, returns files in the directory;
+   if recursive is true, return files in the subtree rooted at the path.
+ If the path is a file, return the file's status and block locations.
+ </pre>
+ @param f is the path
+ @param recursive if the subdirectories need to be traversed recursively
+
+ @return an iterator that traverses statuses of the files
+
+ @throws FileNotFoundException when the path does not exist;
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this FileSystem.
+ The default implementation returns {@code "/user/$USER/"}.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given FileSystem. All relative
+ paths will be resolved relative to it.
+
+ @param new_dir Path of new working directory]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given FileSystem
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Note: with the new FileContext class, getWorkingDirectory()
+ will be removed.
+ The working directory is implemented in FileContext.
+
+ Some FileSystems like LocalFileSystem have an initial workingDir
+ that we use as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+
+ @return if there is built in notion of workingDir then it
+ is returned; else a null is returned.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ @param f path
+ @return true if the directory was created
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make the given file and all non-existent parents into
+ directories. Has roughly the semantics of Unix @{code mkdir -p}.
+ Existence of the directory hierarchy is not an error.
+ @param f path to create
+ @param permission to apply to f
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to filesystem at
+ the given dst name and the source is kept intact afterwards
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files is on the local disk.  Add it to filesystem at
+ the given dst name, removing the source afterwards.
+ @param srcs source paths
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name, removing the source afterwards.
+ @param src local path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param src path
+ @param dst path]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param srcs array of paths which are source
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from the remote filesystem to the local one.
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy a file to the local filesystem, then delete it from the
+ remote filesystem (if successfully copied).
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from a remote filesystem to the local one.
+ delSrc indicates if the src will be removed or not.
+ @param delSrc whether to delete the src
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="useRawLocalFileSystem" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under this filesystem, and the dst is on the local disk.
+ Copy it from the remote filesystem to the local dst name.
+ delSrc indicates if the src will be removed
+ or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
+ as the local file system or not. RawLocalFileSystem is non checksumming,
+ So, It will not create any crc files at local.
+
+ @param delSrc
+          whether to delete the src
+ @param src
+          path
+ @param dst
+          path
+ @param useRawLocalFileSystem
+          whether to use RawLocalFileSystem as local file system or not.
+
+ @throws IOException for any IO error]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local file that the user can write output to.  The caller
+ provides both the eventual target name in this FileSystem
+ and the local working file path.
+ If this FileSystem is local, we write directly into the target.  If
+ the FileSystem is not local, we write into the tmp local area.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path of local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.
+ A local FS will do nothing, because we've written to exactly the
+ right place.
+ A remote FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path to local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this FileSystem instance.
+ Will release any held locks, delete all files queued for deletion
+ through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ from the cache, if cached.
+
+ After this operation, the outcome of any method call on this FileSystem
+ instance, or any input/output stream created by it is <i>undefined</i>.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the block size for a particular file.
+ @param f the filename
+ @return the number of bytes in a block
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @throws FileNotFoundException if the path is not present
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultBlockSize(Path)} instead">
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.
+ @deprecated use {@link #getDefaultBlockSize(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.  The given path will be used to
+ locate the actual filesystem.  The full path does not have to exist.
+ @param f path of file
+ @return the default block size for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultReplication(Path)} instead">
+      <doc>
+      <![CDATA[Get the default replication.
+ @return the replication; the default value is "1".
+ @deprecated use {@link #getDefaultReplication(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the default replication for a path.
+ The given path will be used to locate the actual FileSystem to query.
+ The full path does not have to exist.
+ @param path of the file
+ @return default replication for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+ @return a FileStatus object
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p/>In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="fixRelativePart" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[See {@link FileContext#fixRelativePart}.]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#createSymlink(Path, Path, boolean)}.]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getFileLinkStatus(Path)}.
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#supportsSymlinks()}.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, if the FS supports checksums.
+
+ @param f The file path
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, from the beginning of the file till the
+ specific length.
+ @param f The file path
+ @param length The length of the file range for checksum calculation
+ @return The file checksum or null if checksums are not supported.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums. By default doesn't do anything.
+ @param verifyChecksum Verify checksum flag]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the write checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums. By default doesn't do anything.
+ @param writeChecksum Write checksum flag]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the root partition is reflected.
+
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ @param p Path for which status should be obtained. null means
+ the default partition.
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param p The path
+ @param permission permission
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory).
+ The parameters username and groupname cannot both be null.
+ @param p The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param p The path
+ @param mtime Set the modification time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set access time.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+ @param path  The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List describing entries to remove
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List describing modifications, which must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACL of a file or directory.
+
+ @param path Path to get
+ @return AclStatus describing the ACL of the file or directory
+ @throws IOException if an ACL could not be read
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr name and value for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List<String> of the XAttr names of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the root directory of Trash for current user when the path specified
+ is deleted.
+
+ @param path the trash root of the path to be determined.
+ @return the default implementation returns {@code /user/$USER/.Trash}]]>
+      </doc>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+      <doc>
+      <![CDATA[Get all the trash roots for current user or all users.
+
+ @param allUsers return trash roots for all users if true.
+ @return all the trash root directories.
+         Default FileSystem returns .Trash under users' home directories if
+         {@code /user/$USER/.Trash} exists.]]>
+      </doc>
+    </method>
+    <method name="getFileSystemClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the FileSystem implementation class of a filesystem.
+ This triggers a scan and load of all FileSystem implementations listed as
+ services and discovered via the {@link ServiceLoader}
+ @param scheme URL scheme of FS
+ @param conf configuration: can be null, in which case the check for
+ a filesystem binding declaration in the configuration is skipped.
+ @return the filesystem
+ @throws UnsupportedFileSystemException if there was no known implementation
+         for the scheme.
+ @throws IOException if the filesystem could not be loaded]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Get the Map of Statistics object indexed by URI Scheme.
+ @return a Map having a key as URI scheme and value as Statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Return the FileSystem classes that have Statistics.
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ @param cls the class to lookup
+ @return a statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print all statistics for all file systems to {@code System.out}]]>
+      </doc>
+    </method>
+    <method name="areSymlinksEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="enableSymlinks"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStorageStatistics" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the StorageStatistics for this FileSystem object.  These statistics are
+ per-instance.  They are not shared with any other FileSystem object.
+
+ <p>This is a default method which is intended to be overridden by
+ subclasses. The default implementation returns an empty storage statistics
+ object.</p>
+
+ @return    The StorageStatistics for this FileSystem instance.
+            Will never be null.]]>
+      </doc>
+    </method>
+    <method name="getGlobalStorageStatistics" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a new FSDataOutputStreamBuilder for the file with path.
+ Files are overwritten by default.
+
+ @param path file path
+ @return a FSDataOutputStreamBuilder object to build the file
+
+ HADOOP-14384. Temporarily reduce the visibility of method before the
+ builder interface becomes stable.]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a Builder to append a file.
+ @param path file path.
+ @return a {@link FSDataOutputStreamBuilder} to build file append request.]]>
+      </doc>
+    </method>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG" type="org.apache.commons.logging.Log"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This log is widely used in the org.apache.hadoop.fs code and tests,
+ so must be considered something to only be changed with care.]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileSystem shutdown hook: {@value}.]]>
+      </doc>
+    </field>
+    <field name="TRASH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for trash directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="USER_HOME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An abstract base class for a fairly generic filesystem.  It
+ may be implemented as a distributed filesystem, or as a "local"
+ one that reflects the locally-connected disk.  The local version
+ exists for small Hadoop instances and for testing.
+
+ <p>
+
+ All user code that may potentially use the Hadoop Distributed
+ File System should be written to use a FileSystem object or its
+ successor, {@link FileContext}.
+
+ <p>
+ The local implementation is {@link LocalFileSystem} and distributed
+ implementation is DistributedFileSystem. There are other implementations
+ for object stores and (outside the Apache Hadoop codebase),
+ third party filesystems.
+ <p>
+ Notes
+ <ol>
+ <li>The behaviour of the filesystem is
+ <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ specified in the Hadoop documentation. </a>
+ However, the normative specification of the behavior of this class is
+ actually HDFS: if HDFS does not behave the way these Javadocs or
+ the specification in the Hadoop documentations define, assume that
+ the documentation is incorrect.
+ </li>
+ <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ <li>The term {@code filesystem} refers to the distributed/local filesystem
+ itself, rather than the class used to interact with it.</li>
+ <li>The term "file" refers to a file in the remote filesystem,
+ rather than instances of {@code java.io.File}.</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileSystem -->
+  <!-- start class org.apache.hadoop.fs.FileUtil -->
+  <class name="FileUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileUtil"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path
+ 
+ @param stats
+          an array of FileStatus objects
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path.
+ If stats if null, return path
+ @param stats
+          an array of FileStatus objects
+ @param path
+          default path to return in stats is null
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <doc>
+      <![CDATA[Register all files recursively to be deleted on exit.
+ @param file File/directory to be deleted]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.
+ @param dir the file or directory to be deleted
+ @param tryGrantPermissions true if permissions should be modified to delete a file.
+ @return true on success false on failure.]]>
+      </doc>
+    </method>
+    <method name="readLink" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns the target of the given symlink. Returns the empty string if
+ the given path does not refer to a symlink or there is an error
+ accessing the symlink.
+ @param f File representing the symbolic link.
+ @return The target of the symbolic link, empty string on error or if not
+         a symlink.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.
+ @param tryGrantPermissions if 'true', try grant +rwx permissions to this 
+ and all the underlying directories before trying to delete their contents.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link FileSystem#delete(Path, boolean)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recursively delete a directory.
+ 
+ @param fs {@link FileSystem} on which the path is present
+ @param dir directory to recursively delete 
+ @throws IOException
+ @deprecated Use {@link FileSystem#delete(Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcStatus" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copyMerge" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dstFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="addString" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy local files to a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="java.io.File"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy FileSystem files to local files.]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param filename The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="makeCanonicalPath" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @param makeCanonicalPath 
+          Whether to make canonical path for the file passed
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="getDU" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Takes an input dir and returns the du on that local directory. Very basic
+ implementation.
+ 
+ @param dir
+          The input dir to get the disk space of this local dir
+ @return The total disk space of the input local directory]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="unzipDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a File input it will unzip the file in a the unzip directory
+ passed as the second parameter
+ @param inFile The zip file as input
+ @param unzipDir The unzip directory where to unzip the zip file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="untarDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+ 
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+  
+ @param inFile The tar file as input. 
+ @param untarDir The untar directory where to untar the tar file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="symLink" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="linkname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a soft link between a src and destination
+ only on a local disk. HDFS does not support this.
+ On Windows, when symlink creation fails due to security
+ setting, we will log a warning. The return code in this
+ case is 2.
+
+ @param target the target for symlink 
+ @param linkname the symlink
+ @return 0 on success]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Change the permissions on a filename.
+ @param filename the name of the file to change
+ @param perm the permission string
+ @return the exit code from the command
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Change the permissions on a file / directory, recursively, if
+ needed.
+ @param filename name of the file whose permissions are to change
+ @param perm permission string
+ @param recursive true, if permissions should be changed recursively
+ @return the exit code from the command.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the ownership on a file / directory. User name and group name
+ cannot both be null.
+ @param file the file to change
+ @param username the new user owner name
+ @param groupname the new group owner name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setReadable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="readable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setReadable(boolean)}
+ File#setReadable does not work as expected on Windows.
+ @param f input file
+ @param readable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setWritable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="writable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setWritable(boolean)}
+ File#setWritable does not work as expected on Windows.
+ @param f input file
+ @param writable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setExecutable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="executable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setExecutable(boolean)}
+ File#setExecutable does not work as expected on Windows.
+ Note: revoking execute permission on folders does not have the same
+ behavior on Windows as on Unix platforms. Creating, deleting or renaming
+ a file within that folder will still succeed on Windows.
+ @param f input file
+ @param executable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="canRead" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canRead()}
+ @param f input file
+ @return On Unix, same as {@link File#canRead()}
+         On Windows, true if process has read access on the path]]>
+      </doc>
+    </method>
+    <method name="canWrite" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canWrite()}
+ @param f input file
+ @return On Unix, same as {@link File#canWrite()}
+         On Windows, true if process has write access on the path]]>
+      </doc>
+    </method>
+    <method name="canExecute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canExecute()}
+ @param f input file
+ @return On Unix, same as {@link File#canExecute()}
+         On Windows, true if process has execute access on the path]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permissions to the required value. Uses the java primitives instead
+ of forking if group == other.
+ @param f the file to change
+ @param permission the new permissions
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createLocalTempFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="basefile" type="java.io.File"/>
+      <param name="prefix" type="java.lang.String"/>
+      <param name="isDeleteOnExit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a tmp file for a base file.
+ @param basefile the base file of the tmp
+ @param prefix file name prefix of tmp
+ @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
+ @return a newly created tmp file
+ @exception IOException If a tmp file cannot created
+ @see java.io.File#createTempFile(String, String, File)
+ @see java.io.File#deleteOnExit()]]>
+      </doc>
+    </method>
+    <method name="replaceFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="target" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the src file to the name specified by target.
+ @param src the source file
+ @param target the target file
+ @exception IOException If this operation fails]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="java.io.File[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#listFiles()}. This java.io API returns null 
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#listFiles() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer 
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of files or empty list
+ @exception IOException for invalid directory or for a bad disk.]]>
+      </doc>
+    </method>
+    <method name="list" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#list()}. This java.io API returns null 
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#list() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer 
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of file names or empty string list
+ @exception IOException for invalid directory or for a bad disk.]]>
+      </doc>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="targetDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a jar file at the given path, containing a manifest with a classpath
+ that references all specified entries.
+ 
+ Some platforms may have an upper limit on command line length.  For example,
+ the maximum command line length on Windows is 8191 characters, but the
+ length of the classpath may exceed this.  To work around this limitation,
+ use this method to create a small intermediate jar with a manifest that
+ contains the full classpath.  It returns the absolute path to the new jar,
+ which the caller may set as the classpath for a new process.
+ 
+ Environment variable evaluation is not supported within a jar manifest, so
+ this method expands environment variables before inserting classpath entries
+ to the manifest.  The method parses environment variables according to
+ platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+ environment variables are case-insensitive.  For example, %VAR% and %var%
+ evaluate to the same value.
+ 
+ Specifying the classpath in a jar manifest does not support wildcards, so
+ this method expands wildcards internally.  Any classpath entry that ends
+ with * is translated to all files at that path with extension .jar or .JAR.
+ 
+ @param inputClassPath String input classpath to bundle into the jar manifest
+ @param pwd Path to working directory to save jar
+ @param targetDir path to where the jar execution will have its working dir
+ @param callerEnv Map<String, String> caller's environment variables to use
+   for expansion
+ @return String[] with absolute path to new jar in position 0 and
+   unexpanded wild card entry path in position 1
+ @throws IOException if there is an I/O error while writing the jar file]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+ It operates only on local paths.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist locally]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="useLocal" type="boolean"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist]]>
+      </doc>
+    </method>
+    <method name="compareFs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="destFs" type="org.apache.hadoop.fs.FileSystem"/>
+    </method>
+    <field name="SYMLINK_NO_PRIVILEGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A collection of file-processing util methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileUtil -->
+  <!-- start class org.apache.hadoop.fs.FilterFileSystem -->
+  <class name="FilterFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the raw file system 
+ @return FileSystem being filtered]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called after a new FileSystem instance is constructed.
+ @param name a uri whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make sure that a path specifies a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.  Can take place on local fs
+ or remote DFS.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files in a directory.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files and its block locations in a directory.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a remote iterator for listing in a directory]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given file system. All relative
+ paths will be resolved relative to it.
+ 
+ @param newDir]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given file system
+ 
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ delSrc indicates if the src will be removed or not.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local File that the user can write output to.  The caller
+ provides both the eventual FS target name and the local working
+ file.  If the FS is local, we write directly into the target.  If
+ the FS is remote, we write into the tmp local area.]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.  A local FS will
+ do nothing, because we've written to exactly the right place.  A remote
+ FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get file status.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="abdolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChildFileSystems" return="org.apache.hadoop.fs.FileSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="swapScheme" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A <code>FilterFileSystem</code> contains
+ some other file system, which it uses as
+ its  basic file system, possibly transforming
+ the data along the way or providing  additional
+ functionality. The class <code>FilterFileSystem</code>
+ itself simply overrides all  methods of
+ <code>FileSystem</code> with versions that
+ pass all requests to the contained  file
+ system. Subclasses of <code>FilterFileSystem</code>
+ may further override some of  these methods
+ and may also provide additional methods
+ and fields.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FilterFileSystem -->
+  <!-- start interface org.apache.hadoop.fs.FsConstants -->
+  <interface name="FsConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="LOCAL_FS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FTP_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_PATH_LINKS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ViewFs: viewFs file system (ie the mount file system on client side)]]>
+      </doc>
+    </field>
+    <field name="VIEWFS_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FileSystem related constants.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FsConstants -->
+  <!-- start class org.apache.hadoop.fs.FSDataInputStream -->
+  <class name="FSDataInputStream" extends="java.io.DataInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+    <implements name="org.apache.hadoop.fs.HasFileDescriptor"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.CanSetReadahead"/>
+    <implements name="org.apache.hadoop.fs.HasEnhancedByteBufferAccess"/>
+    <implements name="org.apache.hadoop.fs.CanUnbuffer"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferPositionedReadable"/>
+    <constructor name="FSDataInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desired" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset.
+
+ @param desired offset to seek to]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current position in the input stream.
+
+ @return current position in the input stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    maximum number of bytes to read
+ @return total number of bytes read into the buffer, or <code>-1</code>
+         if there is no more data because the end of the stream has been
+         reached]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+ Continues to read until <code>length</code> bytes have been read.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    the number of bytes to read
+ @throws IOException IO problems
+ @throws EOFException If the end of stream is reached while reading.
+                      If an exception is thrown an undetermined number
+                      of bytes in the buffer may have been written.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link #readFully(long, byte[], int, int)}.]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given position on an alternate copy of the data.
+
+ @param  targetPos  position to seek to
+ @return true if a new source is found, false otherwise]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileDescriptor" return="java.io.FileDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReadahead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <param name="opts" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="releaseBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String value. Includes the string value of the inner stream
+ @return the stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link FSInputStream} in a {@link DataInputStream}
+ and buffers input through a {@link java.io.BufferedInputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataInputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStream -->
+  <class name="FSDataOutputStream" extends="java.io.DataOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Syncable"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current position in the output stream.
+
+ @return the current position in the output stream]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the underlying output stream.]]>
+      </doc>
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="sync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hflush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <class name="FSDataOutputStreamBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FSDataOutputStreamBuilder" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="getThisBuilder" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the concrete implementation of the builder instance.]]>
+      </doc>
+    </method>
+    <method name="getFS" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="permission" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set permission for the file.]]>
+      </doc>
+    </method>
+    <method name="getBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="bufferSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufSize" type="int"/>
+      <doc>
+      <![CDATA[Set the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="replication" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="replica" type="short"/>
+      <doc>
+      <![CDATA[Set replication factor.]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="blockSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blkSize" type="long"/>
+      <doc>
+      <![CDATA[Set block size.]]>
+      </doc>
+    </method>
+    <method name="isRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true to create the parent directories if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="recursive" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the parent directory if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="org.apache.hadoop.util.Progressable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="progress" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prog" type="org.apache.hadoop.util.Progressable"/>
+      <doc>
+      <![CDATA[Set the facility of reporting progress.]]>
+      </doc>
+    </method>
+    <method name="getFlags" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="create" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the specified path.]]>
+      </doc>
+    </method>
+    <method name="overwrite" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="overwrite" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to overwrite the existing file.
+ Set it to false, an exception will be thrown when calling {@link #build()}
+ if the file exists.]]>
+      </doc>
+    </method>
+    <method name="append" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="checksumOpt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="chksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <doc>
+      <![CDATA[Set checksum opt.]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the FSDataOutputStream to write on the file system.
+
+ @throws HadoopIllegalArgumentException if the parameters are not valid.
+ @throws IOException on errors when file system creates or appends the file.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for {@link FSDataOutputStream} and its subclasses.
+
+ It is used to create {@link FSDataOutputStream} when creating a new file or
+ appending an existing file on {@link FileSystem}.
+
+ By default, it does not create parent directory that do not exist.
+ {@link FileSystem#createNonRecursive(Path, boolean, int, short, long,
+ Progressable)}.
+
+ To create missing parent directory, use {@link #recursive()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.FSError -->
+  <class name="FSError" extends="java.lang.Error"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Thrown for unexpected filesystem errors, presumed to reflect disk errors
+ in the native filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSError -->
+  <!-- start class org.apache.hadoop.fs.FSInputStream -->
+  <class name="FSInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <constructor name="FSInputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seeks a different copy of the data.  Returns true if 
+ found a new source, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="validatePositionedReadArgs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="EOFException" type="java.io.EOFException"/>
+      <doc>
+      <![CDATA[Validation code, available for use in subclasses.
+ @param position position: if negative an EOF exception is raised
+ @param buffer destination buffer
+ @param offset offset within the buffer
+ @param length length of bytes to read
+ @throws EOFException if the position is negative
+ @throws IndexOutOfBoundsException if there isn't space for the amount of
+ data requested.
+ @throws IllegalArgumentException other arguments are invalid.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[FSInputStream is a generic old InputStream with a little bit
+ of RAF-style seek ability.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSInputStream -->
+  <!-- start class org.apache.hadoop.fs.FsServerDefaults -->
+  <class name="FsServerDefaults" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsServerDefaults"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBytesPerChecksum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWritePacketSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEncryptDataTransfer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrashInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getChecksumType" return="org.apache.hadoop.util.DataChecksum.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyProviderUri" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultStoragePolicyId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Provides server default configuration values to clients.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsServerDefaults -->
+  <!-- start class org.apache.hadoop.fs.FsStatus -->
+  <class name="FsStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsStatus" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a FsStatus object, using the specified statistics]]>
+      </doc>
+    </constructor>
+    <method name="getCapacity" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the capacity in bytes of the file system]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of bytes used on the file system]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of remaining bytes on the file system]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent the capacity, free and used space on a
+ {@link FileSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsStatus -->
+  <!-- start class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <class name="GlobalStorageStatistics" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.GlobalStorageStatistics[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @return            The StorageStatistics object with the given name, or
+                      null if there is none.]]>
+      </doc>
+    </method>
+    <method name="put" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="provider" type="org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider"/>
+      <doc>
+      <![CDATA[Create or return the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @param provider    An object which can create a new StorageStatistics
+                      object if needed.
+ @return            The StorageStatistics object with the given name.
+ @throws RuntimeException  If the StorageStatisticsProvider provides a null
+                           object or a new StorageStatistics object with the
+                           wrong name.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator that we can use to iterate throw all the global storage
+ statistics objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stores global storage statistics objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.GlobFilter -->
+  <class name="GlobFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="GlobFilter" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern.
+
+ @param filePattern the file pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <constructor name="GlobFilter" type="java.lang.String, org.apache.hadoop.fs.PathFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern and an user filter.
+
+ @param filePattern the file pattern.
+ @param filter user filter in addition to the glob pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <method name="hasPattern" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[A filter for POSIX glob pattern with brace expansions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobFilter -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathException -->
+  <class name="InvalidPathException" extends="org.apache.hadoop.HadoopIllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.]]>
+      </doc>
+    </constructor>
+    <constructor name="InvalidPathException" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.
+ @param reason Reason <code>path</code> is invalid]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Path string is invalid either because it has invalid characters or due to
+ other file system specific reasons.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathException -->
+  <!-- start class org.apache.hadoop.fs.LocalFileSystem -->
+  <class name="LocalFileSystem" extends="org.apache.hadoop.fs.ChecksumFileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocalFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p/>
+
+ @return <code>file</code>]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Moves files to a bad file directory on the same device, so that their
+ storage will not be reused.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the checksumed local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.LocatedFileStatus -->
+  <class name="LocatedFileStatus" extends="org.apache.hadoop.fs.FileStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocatedFileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocatedFileStatus" type="org.apache.hadoop.fs.FileStatus, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor 
+ @param stat a file status
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ 
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <method name="getBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file's block locations
+ @return the file's block locations]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class defines a FileStatus that includes a file's block locations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocatedFileStatus -->
+  <!-- start class org.apache.hadoop.fs.Options -->
+  <class name="Options" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Options"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class contains options related to file system operations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Options -->
+  <!-- start class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <class name="ParentNotDirectoryException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ParentNotDirectoryException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ParentNotDirectoryException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the parent of specified Path is not a directory
+ as expected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <!-- start class org.apache.hadoop.fs.Path -->
+  <class name="Path" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Path" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a path from a String.  Path strings are URIs, but with
+ unescaped elements and some additional normalization.
+
+ @param pathString the path string]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.net.URI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a path from a URI
+
+ @param aUri the source URI]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Path from components.
+
+ @param scheme the scheme
+ @param authority the authority
+ @param path the path]]>
+      </doc>
+    </constructor>
+    <method name="getPathWithoutSchemeAndAuthority" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return a version of the given Path without the scheme information.
+
+ @param path the source Path
+ @return a copy of this Path without the scheme information]]>
+      </doc>
+    </method>
+    <method name="mergePaths" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path1" type="org.apache.hadoop.fs.Path"/>
+      <param name="path2" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Merge 2 paths such that the second path is appended relative to the first.
+ The returned path has the scheme and authority of the first path.  On
+ Windows, the drive specification in the second path is discarded.
+ 
+ @param path1 the first path
+ @param path2 the second path, to be appended relative to path1
+ @return the merged path]]>
+      </doc>
+    </method>
+    <method name="isWindowsAbsolutePath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <param name="slashed" type="boolean"/>
+      <doc>
+      <![CDATA[Determine whether a given path string represents an absolute path on
+ Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+
+ @param pathString the path string to evaluate
+ @param slashed true if the given path is prefixed with "/"
+ @return true if the supplied path looks like an absolute path with a Windows
+ drive-specifier]]>
+      </doc>
+    </method>
+    <method name="toUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert this Path to a URI.
+
+ @return this Path as a URI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the FileSystem that owns this Path.
+
+ @param conf the configuration to use when resolving the FileSystem
+ @return the FileSystem that owns this Path
+ @throws java.io.IOException thrown if there's an issue resolving the
+ FileSystem]]>
+      </doc>
+    </method>
+    <method name="isAbsoluteAndSchemeAuthorityNull" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute <strong>and</strong> the scheme is null, <b>and</b> the authority
+ is null.
+
+ @return whether the path is absolute and the URI has no scheme nor
+ authority parts]]>
+      </doc>
+    </method>
+    <method name="isUriPathAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.  This method is a wrapper for {@link #isUriPathAbsolute()}.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isRoot" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if and only if this path represents the root of a file system.
+
+ @return true if and only if this path represents the root of a file system]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the final component of this path.
+
+ @return the final component of this path]]>
+      </doc>
+    </method>
+    <method name="getParent" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the parent of a path or null if at root.
+ @return the parent of a path or null if at root]]>
+      </doc>
+    </method>
+    <method name="suffix" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds a suffix to the final name in the path.
+
+ @param suffix the suffix to add
+ @return a new path with the suffix added]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="depth" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of elements in this path.
+ @return the number of elements in this path]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #makeQualified(URI, Path)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Returns a qualified path object for the {@link FileSystem}'s working
+ directory.
+  
+ @param fs the target FileSystem
+ @return a qualified path object for the FileSystem's working directory
+ @deprecated use {@link #makeQualified(URI, Path)}]]>
+      </doc>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash.]]>
+      </doc>
+    </field>
+    <field name="SEPARATOR_CHAR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash, as a character.]]>
+      </doc>
+    </field>
+    <field name="CUR_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current directory, ".".]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the current host is a Windows machine.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Names a file or directory in a {@link FileSystem}.
+ Path strings use slash as the directory separator.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Path -->
+  <!-- start interface org.apache.hadoop.fs.PathFilter -->
+  <interface name="PathFilter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Tests whether or not the specified abstract pathname should be
+ included in a pathname list.
+
+ @param  path  The abstract pathname to be tested
+ @return  <code>true</code> if and only if <code>pathname</code>
+          should be included]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathFilter -->
+  <!-- start interface org.apache.hadoop.fs.PositionedReadable -->
+  <interface name="PositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read up to the specified number of bytes, from a given
+ position within a file, and return the number of bytes read. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @return actual number of bytes read; -1 means "none"
+ @throws IOException IO problems.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the specified number of bytes, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read number of bytes equal to the length of the buffer, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits positional reading.
+
+ Implementations are required to implement thread-safe operations; this may
+ be supported by concurrent access to the data, or by using a synchronization
+ mechanism to serialize access.
+
+ Not all implementations meet this requirement. Those that do not cannot
+ be used as a backing store for some applications, such as Apache HBase.
+
+ Independent of whether or not they are thread safe, some implementations
+ may make the intermediate state of the system, specifically the position
+ obtained in {@code Seekable.getPos()} visible.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PositionedReadable -->
+  <!-- start class org.apache.hadoop.fs.QuotaUsage -->
+  <class name="QuotaUsage" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QuotaUsage"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QuotaUsage" type="org.apache.hadoop.fs.QuotaUsage.Builder"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build the instance based on the builder.]]>
+      </doc>
+    </constructor>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="quota" type="long"/>
+    </method>
+    <method name="setSpaceConsumed"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceConsumed" type="long"/>
+    </method>
+    <method name="setSpaceQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceQuota" type="long"/>
+    </method>
+    <method name="getFileAndDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory count.]]>
+      </doc>
+    </method>
+    <method name="getQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory quota.]]>
+      </doc>
+    </method>
+    <method name="getSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space consumed.]]>
+      </doc>
+    </method>
+    <method name="getSpaceQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type consumed.]]>
+      </doc>
+    </method>
+    <method name="isTypeQuotaSet" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type quota has been set.]]>
+      </doc>
+    </method>
+    <method name="isTypeConsumedAvailable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type consumption information is available.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the header of the output.
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if hOption is false file sizes are returned in bytes
+ if hOption is true file sizes are returned in human readable
+
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="getTypesQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+    </method>
+    <method name="getStorageTypeHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="java.util.List"/>
+      <doc>
+      <![CDATA[return the header of with the StorageTypes.
+
+ @param storageTypes
+ @return storage header string]]>
+      </doc>
+    </method>
+    <field name="QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output format:
+ <----12----> <----15----> <----15----> <----15----> <-------18------->
+    QUOTA   REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]>
+      </doc>
+    </field>
+    <field name="SPACE_QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER_FIELDS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Store the quota usage of a directory.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.QuotaUsage -->
+  <!-- start class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <class name="RawLocalFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RawLocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="useStatIfAvailable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStreamWithMode" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="handleEmptyDstDirectoryOnWindows" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="srcFile" type="java.io.File"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFile" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given path to a file or directory.
+ @param p the path to delete
+ @param recursive to delete sub-directories
+ @return true if the file or directory and all its contents were deleted
+ @throws IOException if p is non-empty and recursive is false]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p2f" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDirWithMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="p2f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the specified directory hierarchy. Does not
+ treat existence as an error.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the working directory to the given directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsWorkingFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chown to set owner.]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chmod to set permission.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the {@link Path}'s last modified time and last access time to
+ the given valid times.
+
+ @param mtime the modification time to set (only if no less than zero).
+ @param atime the access time to set (only if no less than zero).
+ @throws IOException if setting the times fails.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a FileStatus representing the given path. If the path refers
+ to a symlink return a FileStatus representing the link rather than
+ the object the link refers to.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the raw local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.ReadOption -->
+  <class name="ReadOption" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.ReadOption[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.ReadOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Options that can be used when reading from a FileSystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ReadOption -->
+  <!-- start interface org.apache.hadoop.fs.Seekable -->
+  <interface name="Seekable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits seeking.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Seekable -->
+  <!-- start class org.apache.hadoop.fs.StorageStatistics -->
+  <class name="StorageStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StorageStatistics" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this StorageStatistics object.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the associated file system scheme if this is scheme specific,
+ else return null.]]>
+      </doc>
+    </method>
+    <method name="getLongStatistics" return="java.util.Iterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over all the currently tracked long statistics.
+
+ The values returned will depend on the type of FileSystem or FileContext
+ object.  The values do not necessarily reflect a snapshot in time.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="java.lang.Long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of a statistic.
+
+ @return         null if the statistic is not being tracked or is not a
+                 long statistic. The value of the statistic, otherwise.]]>
+      </doc>
+    </method>
+    <method name="isTracked" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return true if a statistic is being tracked.
+
+ @return         True only if the statistic is being tracked.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all the statistic data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[StorageStatistics contains statistics data for a FileSystem or FileContext
+ instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.StorageType -->
+  <class name="StorageType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="supportTypeQuota" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMovable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="asList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMovableTypes" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTypesSupportingQuota" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <field name="DEFAULT" type="org.apache.hadoop.fs.StorageType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.fs.StorageType[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines the types of supported storage media. The default storage
+ medium is assumed to be DISK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageType -->
+  <!-- start interface org.apache.hadoop.fs.StreamCapabilities -->
+  <interface name="StreamCapabilities"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Query the stream for a specific capability.
+
+ @param capability string to query the stream support for.
+ @return True if the stream supports capability.]]>
+      </doc>
+    </method>
+    <field name="HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hflush capability implemented by {@link Syncable#hflush()}.]]>
+      </doc>
+    </field>
+    <field name="HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hsync capability implemented by {@link Syncable#hsync()}.]]>
+      </doc>
+    </field>
+    <field name="READAHEAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setReadahead capability implemented by
+ {@link CanSetReadahead#setReadahead(Long)}.]]>
+      </doc>
+    </field>
+    <field name="DROPBEHIND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setDropBehind capability implemented by
+ {@link CanSetDropBehind#setDropBehind(Boolean)}.]]>
+      </doc>
+    </field>
+    <field name="UNBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.]]>
+      </doc>
+    </field>
+    <field name="READBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(ByteBuffer) capability implemented by
+ {@link ByteBufferReadable#read(java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <field name="PREADBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(long, ByteBuffer) capability implemented by
+ {@link ByteBufferPositionedReadable#read(long, java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface to query streams for supported capabilities.
+
+ Capability strings must be in lower case.
+
+ Constant strings are chosen over enums in order to allow other file systems
+ to define their own capabilities.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.StreamCapabilities -->
+  <!-- start class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <class name="StreamCapabilitiesPolicy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StreamCapabilitiesPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Implement the policy for {@link CanUnbuffer#unbuffer()}.
+
+ @param in the input stream]]>
+      </doc>
+    </method>
+    <field name="CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Static methods to implement policies for {@link StreamCapabilities}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <!-- start interface org.apache.hadoop.fs.Syncable -->
+  <interface name="Syncable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="sync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="As of HADOOP 0.21.0, replaced by hflush">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated As of HADOOP 0.21.0, replaced by hflush
+ @see #hflush()]]>
+      </doc>
+    </method>
+    <method name="hflush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Flush out the data in client's user buffer. After the return of
+ this call, new readers will see the data.
+ @throws IOException if any error occurs]]>
+      </doc>
+    </method>
+    <method name="hsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to posix fsync, flush out the data in client's user buffer 
+ all the way to the disk device (but the disk may have it in its cache).
+ @throws IOException if error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface for flush/sync operation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Syncable -->
+  <!-- start class org.apache.hadoop.fs.Trash -->
+  <class name="Trash" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Trash" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor.
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <constructor name="Trash" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor for the FileSystem provided.
+ @param fs the FileSystem
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <method name="moveToAppropriateTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[In case of the symlinks or mount points, one has to move the appropriate
+ trashbin in the actual volume of the path p being deleted.
+
+ Hence we get the file system of the fully-qualified resolved-path and
+ then move the path p to the trashbin in that volume,
+ @param fs - the filesystem of path p
+ @param p - the  path being deleted - to be moved to trasg
+ @param conf - configuration
+ @return false if the item is already in the trash or trash is disabled
+ @throws IOException on error]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the trash is enabled for this filesystem]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="checkpoint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="expunge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Provides a trash facility which supports pluggable Trash policies. 
+
+ See the implementation of the configured TrashPolicy for more
+ details.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Trash -->
+  <!-- start class org.apache.hadoop.fs.TrashPolicy -->
+  <class name="TrashPolicy" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrashPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #initialize(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used
+ @param home the home directory
+ @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations. Different from initialize(conf, fs, home), this one does
+ not assume trash always under /user/$USER due to HDFS encryption zone.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the Trash Policy is enabled for this filesystem.]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="createCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old trash checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory of the Trash Policy
+ This API does not work with files deleted from encryption zone when HDFS
+ data encryption at rest feature is enabled as rename file between
+ encryption zones or encryption zone and non-encryption zone is not allowed.
+
+ The caller is recommend to use the new API
+ TrashPolicy#getCurrentTrashDir(Path path).
+ It returns the trash location correctly for the path specified no matter
+ the path is in encryption zone or not.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current trash directory for path specified based on the Trash
+ Policy
+ @param path path to be deleted
+ @return current trash directory for the path to be deleted
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @param home the home directory
+ @return an instance of TrashPolicy
+ @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @return an instance of TrashPolicy]]>
+      </doc>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trash" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="deletionInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface is used for implementing different Trash policies.
+ Provides factory method to create instances of the configured Trash policy.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.TrashPolicy -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <class name="UnsupportedFileSystemException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedFileSystemException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[File system for a given file system name/scheme is not supported]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <!-- start class org.apache.hadoop.fs.XAttrCodec -->
+  <class name="XAttrCodec" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrCodec[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrCodec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="decodeValue" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode string representation of a value and check whether it's 
+ encoded. If the given string begins with 0x or 0X, it expresses
+ a hexadecimal number. If the given string begins with 0s or 0S,
+ base64 encoding is expected. If the given string is enclosed in 
+ double quotes, the inner string is treated as text. Otherwise 
+ the given string is treated as text. 
+ @param value string representation of the value.
+ @return byte[] the value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="encodeValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte[]"/>
+      <param name="encoding" type="org.apache.hadoop.fs.XAttrCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode byte[] value to string representation with encoding. 
+ Values encoded as text strings are enclosed in double quotes (\"), 
+ while strings encoded as hexadecimal and base64 are prefixed with 
+ 0x and 0s, respectively.
+ @param value byte[] value
+ @param encoding
+ @return String string representation of value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The value of <code>XAttr</code> is byte[], this class is to 
+ covert byte[] to some kind of string representation or convert back.
+ String representation is convenient for display and input. For example
+ display in screen as shell response and json response, input as http
+ or shell parameter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrCodec -->
+  <!-- start class org.apache.hadoop.fs.XAttrSetFlag -->
+  <class name="XAttrSetFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrSetFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrSetFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="xAttrName" type="java.lang.String"/>
+      <param name="xAttrExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrSetFlag -->
+  <doc>
+  <![CDATA[An abstract file system API.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.fs.crypto">
+</package>
+<package name="org.apache.hadoop.fs.ftp">
+  <!-- start class org.apache.hadoop.fs.ftp.FTPException -->
+  <class name="FTPException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A class to wrap a {@link Throwable} into a Runtime Exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPException -->
+  <!-- start class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+  <class name="FTPFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p/>
+
+ @return <code>ftp</code>]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FTPFileSystem.
+
+ @return the default port]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A stream obtained via this call must be closed before using other APIs of
+ this class or else the invocation will block.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This optional operation is not yet supported.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BLOCK_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST_PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_PASSWORD_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_DATA_CONNECTION_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TRANSFER_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="E_SAME_DIRECTORY_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ A {@link FileSystem} backed by an FTP client provided by <a
+ href="http://commons.apache.org/net/">Apache Commons Net</a>.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+</package>
+<package name="org.apache.hadoop.fs.http">
+</package>
+<package name="org.apache.hadoop.fs.permission">
+  <!-- start class org.apache.hadoop.fs.permission.AccessControlException -->
+  <class name="AccessControlException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link org.apache.hadoop.security.AccessControlException} 
+             instead.">
+    <constructor name="AccessControlException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor is needed for unwrapping from 
+ {@link org.apache.hadoop.ipc.RemoteException}.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an {@link AccessControlException}
+ with the specified detail message.
+ @param s the detail message.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[An exception class for access control related issues.
+ @deprecated Use {@link org.apache.hadoop.security.AccessControlException} 
+             instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.AccessControlException -->
+  <!-- start class org.apache.hadoop.fs.permission.AclEntry -->
+  <class name="AclEntry" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.fs.permission.AclEntryType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the ACL entry type.
+
+ @return AclEntryType ACL entry type]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the optional ACL entry name.
+
+ @return String ACL entry name, or null if undefined]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the set of permissions in the ACL entry.
+
+ @return FsAction set of permissions in the ACL entry]]>
+      </doc>
+    </method>
+    <method name="getScope" return="org.apache.hadoop.fs.permission.AclEntryScope"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the scope of the ACL entry.
+
+ @return AclEntryScope scope of the ACL entry]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStringStable" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation guaranteed to be stable across versions to
+ satisfy backward compatibility requirements, such as for shell command
+ output or serialization.  The format of this string representation matches
+ what is expected by the {@link #parseAclSpec(String, boolean)} and
+ {@link #parseAclEntry(String, boolean)} methods.
+
+ @return stable, backward compatible string representation]]>
+      </doc>
+    </method>
+    <method name="parseAclSpec" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aclSpec" type="java.lang.String"/>
+      <param name="includePermission" type="boolean"/>
+      <doc>
+      <![CDATA[Parses a string representation of an ACL spec into a list of AclEntry
+ objects. Example: "user::rwx,user:foo:rw-,group::r--,other::---"
+ The expected format of ACL entries in the string parameter is the same
+ format produced by the {@link #toStringStable()} method.
+ 
+ @param aclSpec
+          String representation of an ACL spec.
+ @param includePermission
+          for setAcl operations this will be true. i.e. AclSpec should
+          include permissions.<br>
+          But for removeAcl operation it will be false. i.e. AclSpec should
+          not contain permissions.<br>
+          Example: "user:foo,group:bar"
+ @return Returns list of {@link AclEntry} parsed]]>
+      </doc>
+    </method>
+    <method name="parseAclEntry" return="org.apache.hadoop.fs.permission.AclEntry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aclStr" type="java.lang.String"/>
+      <param name="includePermission" type="boolean"/>
+      <doc>
+      <![CDATA[Parses a string representation of an ACL into a AclEntry object.<br>
+ The expected format of ACL entries in the string parameter is the same
+ format produced by the {@link #toStringStable()} method.
+ 
+ @param aclStr
+          String representation of an ACL.<br>
+          Example: "user:foo:rw-"
+ @param includePermission
+          for setAcl operations this will be true. i.e. Acl should include
+          permissions.<br>
+          But for removeAcl operation it will be false. i.e. Acl should not
+          contain permissions.<br>
+          Example: "user:foo,group:bar,mask::"
+ @return Returns an {@link AclEntry} object]]>
+      </doc>
+    </method>
+    <method name="aclSpecToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aclSpec" type="java.util.List"/>
+      <doc>
+      <![CDATA[Convert a List of AclEntries into a string - the reverse of parseAclSpec.
+ @param aclSpec List of AclEntries to convert
+ @return String representation of aclSpec]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Defines a single entry in an ACL.  An ACL entry has a type (user, group,
+ mask, or other), an optional name (referring to a specific user or group), a
+ set of permissions (any combination of read, write and execute), and a scope
+ (access or default).  AclEntry instances are immutable.  Use a {@link Builder}
+ to create a new instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.AclEntry -->
+  <!-- start class org.apache.hadoop.fs.permission.AclEntryScope -->
+  <class name="AclEntryScope" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.permission.AclEntryScope[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.permission.AclEntryScope"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Specifies the scope or intended usage of an ACL entry.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.AclEntryScope -->
+  <!-- start class org.apache.hadoop.fs.permission.AclEntryType -->
+  <class name="AclEntryType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.permission.AclEntryType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.permission.AclEntryType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStringStable" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation guaranteed to be stable across versions to
+ satisfy backward compatibility requirements, such as for shell command
+ output or serialization.
+
+ @return stable, backward compatible string representation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specifies the type of an ACL entry.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.AclEntryType -->
+  <!-- start class org.apache.hadoop.fs.permission.AclStatus -->
+  <class name="AclStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the file owner.
+
+ @return String file owner]]>
+      </doc>
+    </method>
+    <method name="getGroup" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the file group.
+
+ @return String file group]]>
+      </doc>
+    </method>
+    <method name="isStickyBit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the sticky bit.
+ 
+ @return boolean sticky bit]]>
+      </doc>
+    </method>
+    <method name="getEntries" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the list of all ACL entries, ordered by their natural ordering.
+
+ @return List<AclEntry> unmodifiable ordered list of all ACL entries]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the permission set for the path
+ @return {@link FsPermission} for the path]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEffectivePermission" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entry" type="org.apache.hadoop.fs.permission.AclEntry"/>
+      <doc>
+      <![CDATA[Get the effective permission for the AclEntry
+ @param entry AclEntry to get the effective action]]>
+      </doc>
+    </method>
+    <method name="getEffectivePermission" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entry" type="org.apache.hadoop.fs.permission.AclEntry"/>
+      <param name="permArg" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Get the effective permission for the AclEntry. <br>
+ Recommended to use this API ONLY if client communicates with the old
+ NameNode, needs to pass the Permission for the path to get effective
+ permission, else use {@link AclStatus#getEffectivePermission(AclEntry)}.
+ @param entry AclEntry to get the effective action
+ @param permArg Permission for the path. However if the client is NOT
+          communicating with old namenode, then this argument will not have
+          any preference.
+ @return Returns the effective permission for the entry.
+ @throws IllegalArgumentException If the client communicating with old
+           namenode and permission is not passed as an argument.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An AclStatus contains the ACL information of a specific file. AclStatus
+ instances are immutable. Use a {@link Builder} to create a new instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.AclStatus -->
+  <!-- start class org.apache.hadoop.fs.permission.FsAction -->
+  <class name="FsAction" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.permission.FsAction[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="implies" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <doc>
+      <![CDATA[Return true if this action implies that action.
+ @param that]]>
+      </doc>
+    </method>
+    <method name="and" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <doc>
+      <![CDATA[AND operation.]]>
+      </doc>
+    </method>
+    <method name="or" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <doc>
+      <![CDATA[OR operation.]]>
+      </doc>
+    </method>
+    <method name="not" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NOT operation.]]>
+      </doc>
+    </method>
+    <method name="getFsAction" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="permission" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the FsAction enum for String representation of permissions
+ 
+ @param permission
+          3-character string representation of permission. ex: rwx
+ @return Returns FsAction enum if the corresponding FsAction exists for permission.
+         Otherwise returns null]]>
+      </doc>
+    </method>
+    <field name="SYMBOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Symbolic representation]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[File system actions, e.g. read, write, etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.FsAction -->
+  <!-- start class org.apache.hadoop.fs.permission.FsPermission -->
+  <class name="FsPermission" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsPermission" type="org.apache.hadoop.fs.permission.FsAction, org.apache.hadoop.fs.permission.FsAction, org.apache.hadoop.fs.permission.FsAction"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct by the given {@link FsAction}.
+ @param u user action
+ @param g group action
+ @param o other action]]>
+      </doc>
+    </constructor>
+    <constructor name="FsPermission" type="org.apache.hadoop.fs.permission.FsAction, org.apache.hadoop.fs.permission.FsAction, org.apache.hadoop.fs.permission.FsAction, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsPermission" type="short"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct by the given mode.
+ @param mode
+ @see #toShort()]]>
+      </doc>
+    </constructor>
+    <constructor name="FsPermission" type="org.apache.hadoop.fs.permission.FsPermission"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor
+ 
+ @param other other permission]]>
+      </doc>
+    </constructor>
+    <constructor name="FsPermission" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct by given mode, either in octal or symbolic format.
+ @param mode mode as a string, either in octal or symbolic format
+ @throws IllegalArgumentException if <code>mode</code> is invalid]]>
+      </doc>
+    </constructor>
+    <method name="createImmutable" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="permission" type="short"/>
+      <doc>
+      <![CDATA[Create an immutable {@link FsPermission} object.]]>
+      </doc>
+    </method>
+    <method name="getUserAction" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return user {@link FsAction}.]]>
+      </doc>
+    </method>
+    <method name="getGroupAction" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return group {@link FsAction}.]]>
+      </doc>
+    </method>
+    <method name="getOtherAction" return="org.apache.hadoop.fs.permission.FsAction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return other {@link FsAction}.]]>
+      </doc>
+    </method>
+    <method name="fromShort"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="short"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create and initialize a {@link FsPermission} from {@link DataInput}.]]>
+      </doc>
+    </method>
+    <method name="toShort" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Encode the object to a short.]]>
+      </doc>
+    </method>
+    <method name="toExtendedShort" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Encodes the object to a short.  Unlike {@link #toShort()}, this method may
+ return values outside the fixed range 00000 - 01777 if extended features
+ are encoded into this permission, such as the ACL bit.
+
+ @return short extended short representation of this permission]]>
+      </doc>
+    </method>
+    <method name="toOctal" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the FsPermission in an octal format.
+
+ @return short Unlike {@link #toShort()} which provides a binary
+ representation, this method returns the standard octal style permission.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="applyUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="umask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Apply a umask to this permission and return a new one.
+
+ The umask is used by create, mkdir, and other Hadoop filesystem operations.
+ The mode argument for these operations is modified by removing the bits
+ which are set in the umask.  Thus, the umask limits the permissions which
+ newly created files and directories get.
+
+ @param umask              The umask to use
+ 
+ @return                   The effective permission]]>
+      </doc>
+    </method>
+    <method name="getUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the user file creation mask (umask)
+ 
+ {@code UMASK_LABEL} config param has umask value that is either symbolic 
+ or octal.
+ 
+ Symbolic umask is applied relative to file mode creation mask; 
+ the permission op characters '+' clears the corresponding bit in the mask, 
+ '-' sets bits in the mask.
+ 
+ Octal umask, the specified bits are set in the file mode creation mask.
+ 
+ {@code DEPRECATED_UMASK_LABEL} config param has umask value set to decimal.]]>
+      </doc>
+    </method>
+    <method name="getStickyBit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAclBit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if there is also an ACL (access control list).
+
+ @return boolean true if there is also an ACL (access control list).]]>
+      </doc>
+    </method>
+    <method name="getEncryptedBit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the file is encrypted or directory is in an encryption zone]]>
+      </doc>
+    </method>
+    <method name="setUMask"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="umask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set the user file creation mask (umask)]]>
+      </doc>
+    </method>
+    <method name="getDefault" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default permission for directory and symlink.
+ In previous versions, this default permission was also used to
+ create files, so files created end up with ugo+x permission.
+ See HADOOP-9155 for detail. 
+ Two new methods are added to solve this, please use 
+ {@link FsPermission#getDirDefault()} for directory, and use
+ {@link FsPermission#getFileDefault()} for file.
+ This method is kept for compatibility.]]>
+      </doc>
+    </method>
+    <method name="getDirDefault" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default permission for directory.]]>
+      </doc>
+    </method>
+    <method name="getFileDefault" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default permission for file.]]>
+      </doc>
+    </method>
+    <method name="getCachePoolDefault" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default permission for cache pools.]]>
+      </doc>
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="unixSymbolicPermission" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a FsPermission from a Unix symbolic permission string
+ @param unixSymbolicPermission e.g. "-rw-rw-rw-"]]>
+      </doc>
+    </method>
+    <field name="MAX_PERMISSION_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum acceptable length of a permission string to parse]]>
+      </doc>
+    </field>
+    <field name="DEPRECATED_UMASK_LABEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[umask property label deprecated key and code in getUMask method
+  to accommodate it may be removed in version .23]]>
+      </doc>
+    </field>
+    <field name="UMASK_LABEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_UMASK" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A class for file/directory permissions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.permission.FsPermission -->
+</package>
+<package name="org.apache.hadoop.fs.sftp">
+</package>
+<package name="org.apache.hadoop.fs.shell.find">
+</package>
+<package name="org.apache.hadoop.fs.viewfs">
+  <!-- start class org.apache.hadoop.fs.viewfs.NotInMountpointException -->
+  <class name="NotInMountpointException" extends="java.lang.UnsupportedOperationException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NotInMountpointException" type="org.apache.hadoop.fs.Path, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NotInMountpointException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[NotInMountpointException extends the UnsupportedOperationException.
+ Exception class used in cases where the given path is not mounted 
+ through viewfs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.viewfs.NotInMountpointException -->
+  <!-- start class org.apache.hadoop.fs.viewfs.ViewFileSystem -->
+  <class name="ViewFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ViewFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is the  constructor with the signature needed by
+ {@link FileSystem#createFileSystem(URI, Configuration)}
+ 
+ After this constructor is called initialize() is called.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="ViewFileSystem" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience Constructor for apps to call directly
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+
+ @return <code>viewfs</code>]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theUri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called after a new FileSystem instance is constructed.
+ @param theUri a uri whose authority section names the host, port, etc. for
+        this FileSystem
+ @param conf the configuration]]>
+      </doc>
+    </method>
+    <method name="getTrashCanLocation" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_dir" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContentSummary" return="org.apache.hadoop.fs.ContentSummary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getQuotaUsage" return="org.apache.hadoop.fs.QuotaUsage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getChildFileSystems" return="org.apache.hadoop.fs.FileSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMountPoints" return="org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the trash root directory for current user when the path
+ specified is deleted.
+
+ If FORCE_INSIDE_MOUNT_POINT flag is not set, return the default trash root
+ from targetFS.
+
+ When FORCE_INSIDE_MOUNT_POINT is set to true,
+ <ol>
+   <li>
+     If the trash root for path p is in the same mount point as path p,
+       and one of:
+       <ol>
+         <li>The mount point isn't at the top of the target fs.</li>
+         <li>The resolved path of path is root (in fallback FS).</li>
+         <li>The trash isn't in user's target fs home directory
+            get the corresponding viewFS path for the trash root and return
+            it.
+         </li>
+       </ol>
+   </li>
+   <li>
+     else, return the trash root under the root of the mount point
+     (/{mntpoint}/.Trash/{user}).
+   </li>
+ </ol>
+
+ These conditions handle several different important cases:
+ <ul>
+   <li>File systems may need to have more local trash roots, such as
+         encryption zones or snapshot roots.</li>
+   <li>The fallback mount should use the user's home directory.</li>
+   <li>Cloud storage systems should not use trash in an implicity defined
+        home directory, per a container, unless it is the fallback fs.</li>
+ </ul>
+
+ @param path the trash root of the path to be determined.
+ @return the trash root path.]]>
+      </doc>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+      <doc>
+      <![CDATA[Get all the trash roots for current user or all users.
+
+ When FORCE_INSIDE_MOUNT_POINT is set to true, we also return trash roots
+ under the root of each mount point, with their viewFS paths.
+
+ @param allUsers return trash roots for all users if true.
+ @return all Trash root directories.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[ViewFileSystem (extends the FileSystem interface) implements a client-side
+ mount table. Its spec and implementation is identical to {@link ViewFs}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.viewfs.ViewFileSystem -->
+  <!-- start class org.apache.hadoop.fs.viewfs.ViewFs -->
+  <class name="ViewFs" extends="org.apache.hadoop.fs.AbstractFileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ViewFs" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+    </constructor>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUriDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInternal" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMountPoints" return="org.apache.hadoop.fs.viewfs.ViewFs.MountPoint[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDelegationTokens" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isValidName" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.lang.String"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[ViewFs (extends the AbstractFileSystem interface) implements a client-side
+ mount table. The viewFs file system is implemented completely in memory on
+ the client side. The client-side mount table allows a client to provide a 
+ customized view of a file system namespace that is composed from 
+ one or more individual file systems (a localFs or Hdfs, S3fs, etc).
+ For example one could have a mount table that provides links such as
+ <ul>
+ <li>  /user          -> hdfs://nnContainingUserDir/user
+ <li>  /project/foo   -> hdfs://nnProject1/projects/foo
+ <li>  /project/bar   -> hdfs://nnProject2/projects/bar
+ <li>  /tmp           -> hdfs://nnTmp/privateTmpForUserXXX
+ </ul> 
+ 
+ ViewFs is specified with the following URI: <b>viewfs:///</b> 
+ <p>
+ To use viewfs one would typically set the default file system in the
+ config  (i.e. fs.defaultFS < = viewfs:///) along with the
+ mount table config variables as described below. 
+ 
+ <p>
+ <b> ** Config variables to specify the mount table entries ** </b>
+ <p>
+ 
+ The file system is initialized from the standard Hadoop config through
+ config variables.
+ See {@link FsConstants} for URI and Scheme constants; 
+ See {@link Constants} for config var constants; 
+ see {@link ConfigUtil} for convenient lib.
+ 
+ <p>
+ All the mount table config entries for view fs are prefixed by 
+ <b>fs.viewfs.mounttable.</b>
+ For example the above example can be specified with the following
+  config variables:
+  <ul>
+  <li> fs.viewfs.mounttable.default.link./user=
+  hdfs://nnContainingUserDir/user
+  <li> fs.viewfs.mounttable.default.link./project/foo=
+  hdfs://nnProject1/projects/foo
+  <li> fs.viewfs.mounttable.default.link./project/bar=
+  hdfs://nnProject2/projects/bar
+  <li> fs.viewfs.mounttable.default.link./tmp=
+  hdfs://nnTmp/privateTmpForUserXXX
+  </ul>
+  
+ The default mount table (when no authority is specified) is 
+ from config variables prefixed by <b>fs.viewFs.mounttable.default </b>
+ The authority component of a URI can be used to specify a different mount
+ table. For example,
+ <ul>
+ <li>  viewfs://sanjayMountable/
+ </ul>
+ is initialized from fs.viewFs.mounttable.sanjayMountable.* config variables.
+ 
+  <p> 
+  <b> **** Merge Mounts **** </b>(NOTE: merge mounts are not implemented yet.)
+  <p>
+  
+   One can also use "MergeMounts" to merge several directories (this is
+   sometimes  called union-mounts or junction-mounts in the literature.
+   For example of the home directories are stored on say two file systems
+   (because they do not fit on one) then one could specify a mount
+   entry such as following merges two dirs:
+   <ul>
+   <li> /user -> hdfs://nnUser1/user,hdfs://nnUser2/user
+   </ul>
+  Such a mergeLink can be specified with the following config var where ","
+  is used as the separator for each of links to be merged:
+  <ul>
+  <li> fs.viewfs.mounttable.default.linkMerge./user=
+  hdfs://nnUser1/user,hdfs://nnUser1/user
+  </ul>
+   A special case of the merge mount is where mount table's root is merged
+   with the root (slash) of another file system:
+   <ul>
+   <li>    fs.viewfs.mounttable.default.linkMergeSlash=hdfs://nn99/
+   </ul>
+   In this cases the root of the mount table is merged with the root of
+            <b>hdfs://nn99/ </b>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.viewfs.ViewFs -->
+</package>
+<package name="org.apache.hadoop.ha">
+  <!-- start class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <class name="BadFencingConfigurationException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BadFencingConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BadFencingConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the operator has specified an invalid configuration
+ for fencing methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <!-- start class org.apache.hadoop.ha.FailoverFailedException -->
+  <class name="FailoverFailedException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailoverFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FailoverFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate service failover has failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.FailoverFailedException -->
+  <!-- start interface org.apache.hadoop.ha.FenceMethod -->
+  <interface name="FenceMethod"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="checkArgs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Verify that the given fencing method's arguments are valid.
+ @param args the arguments provided in the configuration. This may
+        be null if the operator did not configure any arguments.
+ @throws BadFencingConfigurationException if the arguments are invalid]]>
+      </doc>
+    </method>
+    <method name="tryFence" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.ha.HAServiceTarget"/>
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Attempt to fence the target node.
+ @param target the address (host:ipcport) of the service to fence
+ @param args the configured arguments, which were checked at startup by
+             {@link #checkArgs(String)}
+ @return true if fencing was successful, false if unsuccessful or
+              indeterminate
+ @throws BadFencingConfigurationException if the configuration was
+         determined to be invalid only at runtime]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A fencing method is a method by which one node can forcibly prevent
+ another node from making continued progress. This might be implemented
+ by killing a process on the other node, by denying the other node's
+ access to shared storage, or by accessing a PDU to cut the other node's
+ power.
+ <p>
+ Since these methods are often vendor- or device-specific, operators
+ may implement this interface in order to achieve fencing.
+ <p>
+ Fencing is configured by the operator as an ordered list of methods to
+ attempt. Each method will be tried in turn, and the next in the list
+ will only be attempted if the previous one fails. See {@link NodeFencer}
+ for more information.
+ <p>
+ If an implementation also implements {@link Configurable} then its
+ <code>setConf</code> method will be called upon instantiation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.FenceMethod -->
+  <!-- start interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <interface name="HAServiceProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="monitorHealth"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="HealthCheckFailedException" type="org.apache.hadoop.ha.HealthCheckFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Monitor the health of service. This periodically called by the HA
+ frameworks to monitor the health of the service.
+ 
+ Service is expected to perform checks to ensure it is functional.
+ If the service is not healthy due to failure or partial failure,
+ it is expected to throw {@link HealthCheckFailedException}.
+ The definition of service not healthy is left to the service.
+ 
+ Note that when health check of an Active service fails,
+ failover to standby may be done.
+ 
+ @throws HealthCheckFailedException
+           if the health check of a service fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToActive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to active state. No operation, if the
+ service is already in active state.
+ 
+ @throws ServiceFailedException
+           if transition from standby to active fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToStandby"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to standby state. No operation, if the
+ service is already in standby state.
+ 
+ @throws ServiceFailedException
+           if transition from active to standby fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToObserver"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to observer state. No operation, if the
+ service is already in observer state.
+
+ @throws ServiceFailedException
+           if transition from standby to observer fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="getServiceStatus" return="org.apache.hadoop.ha.HAServiceStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current status of the service. The status indicates
+ the current <em>state</em> (e.g ACTIVE/STANDBY) as well as
+ some additional information.
+
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen
+ @see HAServiceStatus]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial version of the protocol]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides High Availability related primitives to
+ monitor and fail-over the service.
+ 
+ This interface could be used by HA frameworks to manage the service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <!-- start class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <class name="HAServiceProtocolHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceProtocolHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="monitorHealth"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToActive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToStandby"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToObserver"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Helper for making {@link HAServiceProtocol} RPC calls. This helper
+ unwraps the {@link RemoteException} to specific exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <!-- start class org.apache.hadoop.ha.HAServiceTarget -->
+  <class name="HAServiceTarget" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceTarget"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the target node.]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an optional separate RPC server address for health checks at the
+ target node.  If defined, then this address is used by the health monitor
+ for the {@link HAServiceProtocol#monitorHealth()} and
+ {@link HAServiceProtocol#getServiceStatus()} calls.  This can be useful for
+ separating out these calls onto separate RPC handlers to protect against
+ resource exhaustion in the main RPC handler pool.  If null (which is the
+ default implementation), then all RPC calls go to the address defined by
+ {@link #getAddress()}.
+
+ @return IPC address of the lifeline RPC server on the target node, or null
+     if no lifeline RPC server is used]]>
+      </doc>
+    </method>
+    <method name="getZKFCAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the ZKFC on the target node]]>
+      </doc>
+    </method>
+    <method name="getFencer" return="org.apache.hadoop.ha.NodeFencer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a Fencer implementation configured for this target node]]>
+      </doc>
+    </method>
+    <method name="checkFencingConfigured"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[@throws BadFencingConfigurationException if the fencing configuration
+ appears to be invalid. This is divorced from the above
+ {@link #getFencer()} method so that the configuration can be checked
+ during the pre-flight phase of failover.]]>
+      </doc>
+    </method>
+    <method name="getProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to connect to the target HA Service.]]>
+      </doc>
+    </method>
+    <method name="setTransitionTargetHAStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"/>
+    </method>
+    <method name="getTransitionTargetHAStatus" return="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a proxy to connect to the target HA service for health monitoring.
+ If {@link #getHealthMonitorAddress()} is implemented to return a non-null
+ address, then this proxy will connect to that address.  Otherwise, the
+ returned proxy defaults to using {@link #getAddress()}, which means this
+ method's behavior is identical to {@link #getProxy(Configuration, int)}.
+
+ @param conf Configuration
+ @param timeoutMs timeout in milliseconds
+ @return a proxy to connect to the target HA service for health monitoring
+ @throws IOException if there is an error]]>
+      </doc>
+    </method>
+    <method name="getZKFCProxy" return="org.apache.hadoop.ha.ZKFCProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to the ZKFC which is associated with this HA service.]]>
+      </doc>
+    </method>
+    <method name="getFencingParameters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addFencingParameters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ret" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Hook to allow subclasses to add any parameters they would like to
+ expose to fencing implementations/scripts. Fencing methods are free
+ to use this map as they see fit -- notably, the shell script
+ implementation takes each entry, prepends 'target_', substitutes
+ '_' for '.', and adds it to the environment of the script.
+
+ Subclass implementations should be sure to delegate to the superclass
+ implementation as well as adding their own keys.
+
+ @param ret map which can be mutated to pass parameters to the fencer]]>
+      </doc>
+    </method>
+    <method name="isAutoFailoverEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if auto failover should be considered enabled]]>
+      </doc>
+    </method>
+    <method name="supportObserver" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this target supports the Observer state, false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents a target of the client side HA administration commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceTarget -->
+  <!-- start class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <class name="HealthCheckFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HealthCheckFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="HealthCheckFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that health check of a service failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <!-- start class org.apache.hadoop.ha.ServiceFailedException -->
+  <class name="ServiceFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ServiceFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that an operation performed
+ to modify the state of a service or application failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.ServiceFailedException -->
+</package>
+<package name="org.apache.hadoop.ha.protocolPB">
+  <!-- start interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <interface name="HAServiceProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <!-- start interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+  <interface name="ZKFCProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.ZKFCProtocolProtos.ZKFCProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+</package>
+<package name="org.apache.hadoop.http.lib">
+  <doc>
+  <![CDATA[This package provides user-selectable (via configuration) classes that add
+functionality to the web UI. They are configured as a list of classes in the
+configuration parameter <b>hadoop.http.filter.initializers</b>.
+
+<ul>
+<li> <b>StaticUserWebFilter</b> - An authorization plugin that makes all
+users a static configured user.
+</ul>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io">
+  <!-- start class org.apache.hadoop.io.AbstractMapWritable -->
+  <class name="AbstractMapWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractMapWritable"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addToMap"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a Class to the maps if it is not already present.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="byte"/>
+      <doc>
+      <![CDATA[@return the Class class for the specified id]]>
+      </doc>
+    </method>
+    <method name="getId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[@return the id for the specified Class]]>
+      </doc>
+    </method>
+    <method name="copy"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Used by child copy constructors.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the conf]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[@param conf the conf to set]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Abstract base class for MapWritable and SortedMapWritable
+ 
+ Unlike org.apache.nutch.crawl.MapWritable, this class allows creation of
+ MapWritable&lt;Writable, MapWritable&gt; so the CLASS_TO_ID and ID_TO_CLASS
+ maps travel with the class instead of being static.
+ 
+ Class ids range from 1 to 127 so there can be at most 127 distinct classes
+ in any specific map instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.AbstractMapWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayFile -->
+  <class name="ArrayFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ArrayFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A dense file-based mapping from integers to values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayFile -->
+  <!-- start class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <class name="ArrayPrimitiveWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayPrimitiveWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an empty instance, for use during Writable read]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance of known type but no value yet
+ for use with type-specific wrapper classes]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Wrap an existing array of primitives
+ @param value - array of primitives]]>
+      </doc>
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original array.  
+ Client must cast it back to type componentType[]
+ (or may use type-specific wrapper classes).
+ @return - original array as Object]]>
+      </doc>
+    </method>
+    <method name="getComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDeclaredComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDeclaredComponentType" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="componentType" type="java.lang.Class"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This is a wrapper class.  It wraps a Writable implementation around
+ an array of primitives (e.g., int[], long[], etc.), with optimized 
+ wire format, and without creating new objects per element.
+ 
+ This is a wrapper class only; it does not make a copy of the 
+ underlying array.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayWritable -->
+  <class name="ArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for arrays containing instances of a class. The elements of this
+ writable must all be instances of the same class. If this writable will be
+ the input for a Reducer, you will need to create a subclass that sets the
+ value to be of the proper type.
+
+ For example:
+ <code>
+ public class IntArrayWritable extends ArrayWritable {
+   public IntArrayWritable() { 
+     super(IntWritable.class); 
+   }	
+ }
+ </code>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayWritable -->
+  <!-- start class org.apache.hadoop.io.BinaryComparable -->
+  <class name="BinaryComparable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="BinaryComparable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return n st bytes 0..n-1 from {#getBytes()} are valid.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return representative byte array for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.BinaryComparable"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#compareBytes(byte[],int,int,byte[],int,int)]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()} to those provided.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if bytes from {#getBytes()} match.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a hash of the bytes returned from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#hashBytes(byte[],int)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface supported by {@link org.apache.hadoop.io.WritableComparable}
+ types supporting ordering/permutation by a representative set of bytes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BinaryComparable -->
+  <!-- start class org.apache.hadoop.io.BloomMapFile -->
+  <class name="BloomMapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomMapFile"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BLOOM_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HASH_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends {@link MapFile} and provides very much the same
+ functionality. However, it uses dynamic Bloom filters to provide
+ quick membership test for keys, and it offers a fast version of 
+ {@link Reader#get(WritableComparable, Writable)} operation, especially in
+ case of sparsely populated MapFile-s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BloomMapFile -->
+  <!-- start class org.apache.hadoop.io.BooleanWritable -->
+  <class name="BooleanWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BooleanWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BooleanWritable" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="get" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.BooleanWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for booleans.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BooleanWritable -->
+  <!-- start interface org.apache.hadoop.io.ByteBufferPool -->
+  <interface name="ByteBufferPool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Get a new direct ByteBuffer.  The pool can provide this from
+ removing a buffer from its internal cache, or by allocating a 
+ new buffer.
+
+ @param direct     Whether the buffer should be direct.
+ @param length     The minimum length the buffer will have.
+ @return           A new ByteBuffer.  This ByteBuffer must be direct.
+                   Its capacity can be less than what was requested, but
+                   must be at least 1 byte.]]>
+      </doc>
+    </method>
+    <method name="putBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Release a buffer back to the pool.
+ The pool may choose to put this buffer into its cache.
+
+ @param buffer    a direct bytebuffer]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.ByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.BytesWritable -->
+  <class name="BytesWritable" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BytesWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a zero-size sequence.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value.
+ @param bytes This array becomes the backing storage for the object.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[], int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value
+ and length as the length. Use this constructor if the array is larger
+ than the value it represents.
+ @param bytes This array becomes the backing storage for the object.
+ @param length The number of bytes to use from array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data backing the BytesWritable. Please use {@link #copyBytes()}
+ if you need the returned array to be precisely the length of the data.
+ @return The data is only valid between 0 and getLength() - 1.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getBytes()} instead.">
+      <doc>
+      <![CDATA[Get the data from the BytesWritable.
+ @deprecated Use {@link #getBytes()} instead.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current size of the buffer.]]>
+      </doc>
+    </method>
+    <method name="getSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getLength()} instead.">
+      <doc>
+      <![CDATA[Get the current size of the buffer.
+ @deprecated Use {@link #getLength()} instead.]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <doc>
+      <![CDATA[Change the size of the buffer. The values in the old range are preserved
+ and any new values are undefined. The capacity is changed if it is 
+ necessary.
+ @param size The new number of bytes]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the capacity, which is the maximum size that could handled without
+ resizing the backing storage.
+ @return The number of bytes]]>
+      </doc>
+    </method>
+    <method name="setCapacity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_cap" type="int"/>
+      <doc>
+      <![CDATA[Change the capacity of the backing storage.
+ The data is preserved.
+ @param new_cap The new capacity in bytes.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="org.apache.hadoop.io.BytesWritable"/>
+      <doc>
+      <![CDATA[Set the BytesWritable to the contents of the given newData.
+ @param newData the value to set this BytesWritable to.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Set the value to a copy of the given byte range
+ @param newData the new values to copy in
+ @param offset the offset in newData to start at
+ @param length the number of bytes to copy]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right_obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Are the two byte sequences equal?]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate the stream of bytes as hex pairs separated by ' '.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A byte sequence that is usable as a key or value.
+ It is resizable and distinguishes between the size of the sequence and
+ the current capacity. The hash function is the front of the md5 of the 
+ buffer. The sort order is the same as memcmp.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BytesWritable -->
+  <!-- start class org.apache.hadoop.io.ByteWritable -->
+  <class name="ByteWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ByteWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ByteWritable" type="byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Set the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ByteWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ByteWritable"/>
+      <doc>
+      <![CDATA[Compares two ByteWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for a single byte.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ByteWritable -->
+  <!-- start interface org.apache.hadoop.io.Closeable -->
+  <interface name="Closeable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="use java.io.Closeable">
+    <implements name="java.io.Closeable"/>
+    <doc>
+    <![CDATA[@deprecated use java.io.Closeable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Closeable -->
+  <!-- start class org.apache.hadoop.io.CompressedWritable -->
+  <class name="CompressedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompressedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="ensureInflated"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Must be called by all methods which access fields to ensure that the data
+ has been uncompressed.]]>
+      </doc>
+    </method>
+    <method name="readFieldsCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #readFields(DataInput)}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #write(DataOutput)}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base-class for Writables which store themselves compressed and lazily
+ inflate on field access.  This is useful for large objects whose fields are
+ not be altered during a map or reduce operation: leaving the field data
+ compressed makes copying the instance from one file to another much
+ faster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.CompressedWritable -->
+  <!-- start class org.apache.hadoop.io.DataOutputOutputStream -->
+  <class name="DataOutputOutputStream" extends="java.io.OutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="constructOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <doc>
+      <![CDATA[Construct an OutputStream from the given DataOutput. If 'out'
+ is already an OutputStream, simply returns it. Otherwise, wraps
+ it in an OutputStream.
+ @param out the DataOutput to wrap
+ @return an OutputStream instance that outputs to 'out']]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[OutputStream implementation that wraps a DataOutput.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DataOutputOutputStream -->
+  <!-- start class org.apache.hadoop.io.DefaultStringifier -->
+  <class name="DefaultStringifier" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Stringifier"/>
+    <constructor name="DefaultStringifier" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="fromString" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="store"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="item" type="K"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the item in the configuration with the given keyName.
+ 
+ @param <K>  the class of the item
+ @param conf the configuration to store
+ @param item the object to be stored
+ @param keyName the name of the key to use
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="load" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="storeArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="items" type="K[]"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the array of items in the configuration with the given keyName.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use 
+ @param items the objects to be stored
+ @param keyName the name of the key to use
+ @throws IndexOutOfBoundsException if the items array is empty
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="loadArray" return="K[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the array of objects from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DefaultStringifier is the default implementation of the {@link Stringifier}
+ interface which stringifies the objects using base64 encoding of the
+ serialized version of the objects. The {@link Serializer} and
+ {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ <br>
+ DefaultStringifier offers convenience methods to store/load objects to/from
+ the configuration.
+ 
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DefaultStringifier -->
+  <!-- start class org.apache.hadoop.io.DoubleWritable -->
+  <class name="DoubleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="DoubleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="DoubleWritable" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="double"/>
+    </method>
+    <method name="get" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a DoubleWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.DoubleWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Writable for Double values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DoubleWritable -->
+  <!-- start class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <class name="ElasticByteBufferPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.ByteBufferPool"/>
+    <constructor name="ElasticByteBufferPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+    </method>
+    <method name="putBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <doc>
+    <![CDATA[This is a simple ByteBufferPool which just creates ByteBuffers as needed.
+ It also caches ByteBuffers after they're released.  It will always return
+ the smallest cached buffer with at least the capacity you request.
+ We don't try to do anything clever here like try to limit the maximum cache
+ size.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.EnumSetWritable -->
+  <class name="EnumSetWritable" extends="java.util.AbstractCollection"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. If the <tt>value</tt> argument is null or
+ its size is zero, the <tt>elementType</tt> argument must not be null. If
+ the argument <tt>value</tt>'s size is bigger than zero, the argument
+ <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </constructor>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. Argument <tt>value</tt> should not be null
+ or empty.
+ 
+ @param value]]>
+      </doc>
+    </constructor>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="E"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.util.EnumSet"/>
+      <param name="elementType" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[reset the EnumSetWritable with specified
+ <tt>value</value> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ is null or its size is zero, the <tt>elementType</tt> argument must not be
+ null. If the argument <tt>value</tt>'s size is bigger than zero, the
+ argument <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </method>
+    <method name="get" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this EnumSetWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true if <code>o</code> is an EnumSetWritable with the same value,
+ or both are null.]]>
+      </doc>
+    </method>
+    <method name="getElementType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class of all the elements of the underlying EnumSetWriable. It
+ may return null.
+ 
+ @return the element class]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable wrapper for EnumSet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.EnumSetWritable -->
+  <!-- start class org.apache.hadoop.io.FloatWritable -->
+  <class name="FloatWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="FloatWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FloatWritable" type="float"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a FloatWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.FloatWritable"/>
+      <doc>
+      <![CDATA[Compares two FloatWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for floats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.FloatWritable -->
+  <!-- start class org.apache.hadoop.io.GenericWritable -->
+  <class name="GenericWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="GenericWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Set the instance that is wrapped.
+ 
+ @param obj]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the wrapped instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTypes" return="java.lang.Class[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all classes that may be wrapped.  Subclasses should implement this
+ to return a constant array of classes.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for Writable instances.
+ <p>
+ When two sequence files, which have same Key type but different Value
+ types, are mapped out to reduce, multiple Value types is not allowed.
+ In this case, this class can help you wrap instances with different types.
+ </p>
+ 
+ <p>
+ Compared with <code>ObjectWritable</code>, this class is much more effective,
+ because <code>ObjectWritable</code> will append the class declaration as a String 
+ into the output file in every Key-Value pair.
+ </p>
+ 
+ <p>
+ Generic Writable implements {@link Configurable} interface, so that it will be 
+ configured by the framework. The configuration is passed to the wrapped objects
+ implementing {@link Configurable} interface <i>before deserialization</i>. 
+ </p>
+ 
+ how to use it: <br>
+ 1. Write your own class, such as GenericObject, which extends GenericWritable.<br> 
+ 2. Implements the abstract method <code>getTypes()</code>, defines 
+    the classes which will be wrapped in GenericObject in application.
+    Attention: this classes defined in <code>getTypes()</code> method, must
+    implement <code>Writable</code> interface.
+ <br><br>
+ 
+ The code looks like this:
+ <blockquote><pre>
+ public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+ </pre></blockquote>
+ 
+ @since Nov 8, 2006]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.GenericWritable -->
+  <!-- start class org.apache.hadoop.io.IntWritable -->
+  <class name="IntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="IntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="IntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a IntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.IntWritable"/>
+      <doc>
+      <![CDATA[Compares two IntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for ints.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IntWritable -->
+  <!-- start class org.apache.hadoop.io.IOUtils -->
+  <class name="IOUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IOUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer 
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+ 
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another. <strong>closes the input and output streams 
+ at the end</strong>.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="count" type="long"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies count bytes from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param count number of bytes to copy
+ @param close whether to close the streams
+ @throws IOException if bytes can not be read or written]]>
+      </doc>
+    </method>
+    <method name="wrappedReadForCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="is" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility wrapper for reading from {@link InputStream}. It catches any errors
+ thrown by the underlying stream (either IO or decompression-related), and
+ re-throws as an IOException.
+ 
+ @param is - InputStream to be read from
+ @param buf - buffer the data is read into
+ @param off - offset within buf
+ @param len - amount of data to be read
+ @return number of bytes read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads len bytes in a loop.
+
+ @param in InputStream to read from
+ @param buf The buffer to fill
+ @param off offset from the buffer
+ @param len the length of bytes to read
+ @throws IOException if it could not read requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to readFully(). Skips bytes in a loop.
+ @param in The InputStream to skip bytes from
+ @param len number of bytes to skip.
+ @throws IOException if it could not skip requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param log the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close
+ @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="cleanupWithLogger"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logger" type="org.slf4j.Logger"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param logger the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close]]>
+      </doc>
+    </method>
+    <method name="closeStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[Closes the stream ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param stream the Stream to close]]>
+      </doc>
+    </method>
+    <method name="closeStreams"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="streams" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Closes the streams ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param streams the Streams to close]]>
+      </doc>
+    </method>
+    <method name="closeSocket"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sock" type="java.net.Socket"/>
+      <doc>
+      <![CDATA[Closes the socket ignoring {@link IOException}
+
+ @param sock the Socket to close]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bc" type="java.nio.channels.WritableByteChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a WritableByteChannel, handling short writes.
+ 
+ @param bc               The WritableByteChannel to write to
+ @param buf              The input buffer
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="java.nio.channels.FileChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <param name="offset" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a FileChannel at a given offset, 
+ handling short writes.
+ 
+ @param fc               The FileChannel to write to
+ @param buf              The input buffer
+ @param offset           The offset in the file to start writing at
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="listDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="filter" type="java.io.FilenameFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the complete list of files in a directory as strings.<p/>
+
+ This is better than File#listDir because it does not ignore IOExceptions.
+
+ @param dir              The directory to list.
+ @param filter           If non-null, the filter to use when listing
+                         this directory.
+ @return                 The list of files in the directory.
+
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileToSync" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.<br>
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param fileToSync the file to fsync]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="channel" type="java.nio.channels.FileChannel"/>
+      <param name="isDir" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param channel Channel to sync
+ @param isDir if true, the given file is a directory (Channel should be
+          opened for read and ignore IOExceptions, because not all file
+          systems and operating systems allow to fsync on a directory)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFullyToByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a DataInput until EOF and returns a byte array.  Make sure not to
+ pass in an infinite DataInput or this will never return.
+
+ @param in A DataInput
+ @return a byte array containing the data from the DataInput
+ @throws IOException on I/O error, other than EOF]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An utility class for I/O related functionality.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IOUtils -->
+  <!-- start class org.apache.hadoop.io.LongWritable -->
+  <class name="LongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="LongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a LongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.LongWritable"/>
+      <doc>
+      <![CDATA[Compares two LongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.LongWritable -->
+  <!-- start class org.apache.hadoop.io.MapFile -->
+  <class name="MapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="oldName" type="java.lang.String"/>
+      <param name="newName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames an existing map directory.]]>
+      </doc>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deletes the named map file.]]>
+      </doc>
+    </method>
+    <method name="fix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <param name="dryrun" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[This method attempts to fix a corrupt MapFile by re-creating its index.
+ @param fs filesystem
+ @param dir directory containing the MapFile data and index
+ @param keyClass key class (has to be a subclass of Writable)
+ @param valueClass value class (has to be a subclass of Writable)
+ @param dryrun do not perform any changes, just report what needs to be done
+ @return number of valid entries in this MapFile, or -1 if no fixing was needed
+ @throws Exception]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="INDEX_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the index file.]]>
+      </doc>
+    </field>
+    <field name="DATA_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the data file.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A file-based map from keys to values.
+ 
+ <p>A map is a directory containing two files, the <code>data</code> file,
+ containing all keys and values in the map, and a smaller <code>index</code>
+ file, containing a fraction of the keys.  The fraction is determined by
+ {@link Writer#getIndexInterval()}.
+
+ <p>The index file is read entirely into memory.  Thus key implementations
+ should try to keep themselves small.
+
+ <p>Map files are created by adding entries in-order.  To maintain a large
+ database, perform updates by copying the previous version of a database and
+ merging in a sorted change list, to create a new version of the database in
+ a new file.  Sorting large change lists can be done with {@link
+ SequenceFile.Sorter}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapFile -->
+  <!-- start class org.apache.hadoop.io.MapWritable -->
+  <class name="MapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Map"/>
+    <constructor name="MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="MapWritable" type="org.apache.hadoop.io.MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Writable"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable Map.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapWritable -->
+  <!-- start class org.apache.hadoop.io.MD5Hash -->
+  <class name="MD5Hash" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="MD5Hash"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash from a hex string.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash with a specified value.]]>
+      </doc>
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs, reads and returns an instance.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Copy the contents of another instance into this instance.]]>
+      </doc>
+    </method>
+    <method name="getDigest" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the digest bytes.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="getDigester" return="java.security.MessageDigest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a thread local MD5 digester]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a hash value for the content from the InputStream.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="org.apache.hadoop.io.UTF8"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="halfDigest" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a half-sized version of this MD5.  Fits in a long]]>
+      </doc>
+    </method>
+    <method name="quarterDigest" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a 32-bit digest of the MD5.
+ @return the first 4 bytes of the md5]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is an MD5Hash whose digest contains the
+ same values.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for this object.
+ Only uses the first 4 bytes, since md5s are evenly distributed.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Compares this object with the specified object for order.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="setDigest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the digest value from a hex string.]]>
+      </doc>
+    </method>
+    <field name="MD5_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Writable for MD5 hash values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MD5Hash -->
+  <!-- start class org.apache.hadoop.io.MultipleIOException -->
+  <class name="MultipleIOException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getExceptions" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the underlying exceptions]]>
+      </doc>
+    </method>
+    <method name="createIOException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="exceptions" type="java.util.List"/>
+      <doc>
+      <![CDATA[A convenient method to create an {@link IOException}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Encapsulate a list of {@link IOException} into an {@link IOException}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MultipleIOException -->
+  <!-- start class org.apache.hadoop.io.NullWritable -->
+  <class name="NullWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <method name="get" return="org.apache.hadoop.io.NullWritable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the single instance of this class.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.NullWritable"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Singleton Writable with no data.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.NullWritable -->
+  <!-- start class org.apache.hadoop.io.ObjectWritable -->
+  <class name="ObjectWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="ObjectWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Class, java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the instance, or null if none.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the class this is meant to be.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="instance" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reset the instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="allowCompactArrays" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.  
+ 
+ @param allowCompactArrays - set true for RPC and internal or intra-cluster
+ usages.  Set false for inter-cluster, File, and other persisted output 
+ usages, to preserve the ability to interchange files with other clusters 
+ that may not be running the same version of software.  Sometime in ~2013 
+ we can consider removing this parameter and always using the compact format.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="objectWritable" type="org.apache.hadoop.io.ObjectWritable"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find and load the class with given name <tt>className</tt> by first finding
+ it in the specified <tt>conf</tt>. If the specified <tt>conf</tt> is null,
+ try load it directly.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A polymorphic Writable that writes an instance with it's class name.
+ Handles arrays, strings and primitive types without a Writable wrapper.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ObjectWritable -->
+  <!-- start interface org.apache.hadoop.io.RawComparator -->
+  <interface name="RawComparator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Compare two objects in binary.
+ b1[s1:l1] is the first object, and b2[s2:l2] is the second object.
+ 
+ @param b1 The first byte array.
+ @param s1 The position index in b1. The object under comparison's starting index.
+ @param l1 The length of the object in b1.
+ @param b2 The second byte array.
+ @param s2 The position index in b2. The object under comparison's starting index.
+ @param l2 The length of the object under comparison in b2.
+ @return An integer result of the comparison.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ A {@link Comparator} that operates directly on byte representations of
+ objects.
+ </p>
+ @param <T>
+ @see DeserializerComparator]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.RawComparator -->
+  <!-- start class org.apache.hadoop.io.SequenceFile -->
+  <class name="SequenceFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDefaultCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the compression type for the reduce outputs
+ @param job the job config to look in
+ @return the kind of compression to use]]>
+      </doc>
+    </method>
+    <method name="setDefaultCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="val" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the default compression type for sequence files.
+ @param job the configuration to modify
+ @param val the new compression type (none, block, record)]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="opts" type="org.apache.hadoop.io.SequenceFile.Writer.Option[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new Writer with the given options.
+ @param conf the configuration to use
+ @param opts the options to create the file with
+ @return a new Writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="createParent" type="boolean"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param createParent create parent directory if non-existent
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fc The context for the specified file.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @param createFlag gives the semantics of create: overwrite, append etc.
+ @param opts file creation options; see {@link CreateOpts}.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <field name="SYNC_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes between sync points.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[<code>SequenceFile</code>s are flat files consisting of binary key/value 
+ pairs.
+ 
+ <p><code>SequenceFile</code> provides {@link SequenceFile.Writer},
+ {@link SequenceFile.Reader} and {@link Sorter} classes for writing,
+ reading and sorting respectively.</p>
+ 
+ There are three <code>SequenceFile</code> <code>Writer</code>s based on the 
+ {@link CompressionType} used to compress key/value pairs:
+ <ol>
+   <li>
+   <code>Writer</code> : Uncompressed records.
+   </li>
+   <li>
+   <code>RecordCompressWriter</code> : Record-compressed files, only compress 
+                                       values.
+   </li>
+   <li>
+   <code>BlockCompressWriter</code> : Block-compressed files, both keys & 
+                                      values are collected in 'blocks' 
+                                      separately and compressed. The size of 
+                                      the 'block' is configurable.
+ </ol>
+ 
+ <p>The actual compression algorithm used to compress key and/or values can be
+ specified by using the appropriate {@link CompressionCodec}.</p>
+ 
+ <p>The recommended way is to use the static <tt>createWriter</tt> methods
+ provided by the <code>SequenceFile</code> to chose the preferred format.</p>
+
+ <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
+ above <code>SequenceFile</code> formats.</p>
+
+ <h4 id="Formats">SequenceFile Formats</h4>
+ 
+ <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
+ depending on the <code>CompressionType</code> specified. All of them share a
+ <a href="#Header">common header</a> described below.
+ 
+ <h5 id="Header">SequenceFile Header</h5>
+ <ul>
+   <li>
+   version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual 
+             version number (e.g. SEQ4 or SEQ6)
+   </li>
+   <li>
+   keyClassName -key class
+   </li>
+   <li>
+   valueClassName - value class
+   </li>
+   <li>
+   compression - A boolean which specifies if compression is turned on for 
+                 keys/values in this file.
+   </li>
+   <li>
+   blockCompression - A boolean which specifies if block-compression is 
+                      turned on for keys/values in this file.
+   </li>
+   <li>
+   compression codec - <code>CompressionCodec</code> class which is used for  
+                       compression of keys and/or values (if compression is 
+                       enabled).
+   </li>
+   <li>
+   metadata - {@link Metadata} for this file.
+   </li>
+   <li>
+   sync - A sync marker to denote end of the header.
+   </li>
+ </ul>
+ 
+ <h5 id="#UncompressedFormat">Uncompressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li>Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> bytes or so.
+ </li>
+ </ul>
+
+ <h5 id="#RecordCompressedFormat">Record-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li><i>Compressed</i> Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> bytes or so.
+ </li>
+ </ul>
+ 
+ <h5 id="#BlockCompressedFormat">Block-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record <i>Block</i>
+   <ul>
+     <li>Uncompressed number of records in the block</li>
+     <li>Compressed key-lengths block-size</li>
+     <li>Compressed key-lengths block</li>
+     <li>Compressed keys block-size</li>
+     <li>Compressed keys block</li>
+     <li>Compressed value-lengths block-size</li>
+     <li>Compressed value-lengths block</li>
+     <li>Compressed values block-size</li>
+     <li>Compressed values block</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every block.
+ </li>
+ </ul>
+ 
+ <p>The compressed blocks of key lengths and value lengths consist of the 
+ actual lengths of individual keys/values encoded in ZeroCompressedInteger 
+ format.</p>
+ 
+ @see CompressionCodec]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SequenceFile -->
+  <!-- start class org.apache.hadoop.io.SetFile -->
+  <class name="SetFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SetFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A file-based set of keys.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SetFile -->
+  <!-- start class org.apache.hadoop.io.ShortWritable -->
+  <class name="ShortWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ShortWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ShortWritable" type="short"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="short"/>
+      <doc>
+      <![CDATA[Set the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[read the short value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[write short value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ShortWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[hash code]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ShortWritable"/>
+      <doc>
+      <![CDATA[Compares two ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Short values in string format]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for shorts.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ShortWritable -->
+  <!-- start class org.apache.hadoop.io.SortedMapWritable -->
+  <class name="SortedMapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.SortedMap"/>
+    <constructor name="SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="SortedMapWritable" type="org.apache.hadoop.io.SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="comparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="firstKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="headMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="toKey" type="org.apache.hadoop.io.WritableComparable"/>
+    </method>
+    <method name="lastKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="subMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="toKey" type="org.apache.hadoop.io.WritableComparable"/>
+    </method>
+    <method name="tailMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="org.apache.hadoop.io.WritableComparable"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable SortedMap.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SortedMapWritable -->
+  <!-- start interface org.apache.hadoop.io.Stringifier -->
+  <interface name="Stringifier"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Converts the object to a string representation
+ @param obj the object to convert
+ @return the string representation of the object
+ @throws IOException if the object cannot be converted]]>
+      </doc>
+    </method>
+    <method name="fromString" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from its string representation.
+ @param str the string representation of the object
+ @return restored object
+ @throws IOException if the object cannot be restored]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes this object. 
+ @throws IOException if an I/O error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stringifier interface offers two methods to convert an object 
+ to a string representation and restore the object given its 
+ string representation.
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Stringifier -->
+  <!-- start class org.apache.hadoop.io.Text -->
+  <class name="Text" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Text" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a string.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another text.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a byte array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the raw bytes; however, only data up to {@link #getLength()} is
+ valid. Please use {@link #copyBytes()} if you
+ need the returned array to be precisely the length of the data.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes in the byte array]]>
+      </doc>
+    </method>
+    <method name="charAt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="int"/>
+      <doc>
+      <![CDATA[Returns the Unicode Scalar Value (32-bit integer value)
+ for the character at <code>position</code>. Note that this
+ method avoids using the converter or doing String instantiation
+ @return the Unicode scalar value at position or -1
+          if the position is invalid or points to a
+          trailing byte]]>
+      </doc>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Finds any occurence of <code>what</code> in the backing
+ buffer, starting as position <code>start</code>. The starting
+ position is measured in bytes and the return value is in
+ terms of byte position in the buffer. The backing buffer is
+ not converted to a string for this operation.
+ @return byte position of the first occurence of the search
+         string in the UTF-8 buffer or -1 if not found]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set to contain the contents of a string.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <doc>
+      <![CDATA[Set to a utf8 byte array]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[copy a text.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Set the Text to range of bytes
+ @param utf8 the data to copy from
+ @param start the first position of the new string
+ @param len the number of bytes of the new string]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Append a range of bytes to the end of the given text
+ @param utf8 the data to copy from
+ @param start the first position to append from utf8
+ @param len the number of bytes to append]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the string to empty.
+
+ <em>Note</em>: For performance reasons, this call does not clear the
+ underlying byte array that is retrievable via {@link #getBytes()}.
+ In order to free the byte-array memory, call {@link #set(byte[])}
+ with an empty byte array (For example, <code>new byte[0]</code>).]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert text back to string
+ @see java.lang.Object#toString()]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[deserialize]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skips over one Text in the input.]]>
+      </doc>
+    </method>
+    <method name="readWithKnownLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a Text object whose length is already known.
+ This allows creating Text from a stream which uses a different serialization
+ format.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[serialize
+ write this object to out
+ length uses zero-compressed encoding
+ @see Writable#write(DataOutput)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a Text with the same contents.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If the input is malformed,
+ replace by a default value.]]>
+      </doc>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If the input is malformed,
+ invalid chars are replaced by a default value.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string from in]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string with a maximum size]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string to out]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string with a maximum size to out]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check if a byte array contains valid utf-8
+ @param utf8 byte array
+ @throws MalformedInputException if the byte array contains invalid utf-8]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check to see if a byte array is valid utf-8
+ @param utf8 the array of bytes
+ @param start the offset of the first byte in the array
+ @param len the length of the byte sequence
+ @throws MalformedInputException if the byte array contains invalid bytes]]>
+      </doc>
+    </method>
+    <method name="bytesToCodePoint" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Returns the next code point at the current position in
+ the buffer. The buffer's position will be incremented.
+ Any mark set on this buffer will be changed by this method!]]>
+      </doc>
+    </method>
+    <method name="utf8Length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[For the given string, returns the number of UTF-8 bytes
+ required to encode the string.
+ @param string text to encode
+ @return number of UTF-8 bytes required to encode]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_MAX_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class stores text using standard UTF8 encoding.  It provides methods
+ to serialize, deserialize, and compare texts at byte level.  The type of
+ length is integer and is serialized using zero-compressed format.  <p>In
+ addition, it provides methods for string traversal without converting the
+ byte array to a string.  <p>Also includes utilities for
+ serializing/deserialing a string, coding/decoding a string, checking if a
+ byte array contains valid UTF8 code, calculating the length of an encoded
+ string.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.Text -->
+  <!-- start class org.apache.hadoop.io.TwoDArrayWritable -->
+  <class name="TwoDArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[][]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[][]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[][]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for 2D arrays containing a matrix of instances of a class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.TwoDArrayWritable -->
+  <!-- start class org.apache.hadoop.io.VersionedWritable -->
+  <class name="VersionedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="VersionedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getVersion" return="byte"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the version number of the current implementation.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A base class for Writables that provides version checking.
+
+ <p>This is useful when a class may evolve, so that instances written by the
+ old version of the class may still be processed by the new version.  To
+ handle this situation, {@link #readFields(DataInput)}
+ implementations should catch {@link VersionMismatchException}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionedWritable -->
+  <!-- start class org.apache.hadoop.io.VersionMismatchException -->
+  <class name="VersionMismatchException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionMismatchException" type="byte, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Thrown by {@link VersionedWritable#readFields(DataInput)} when the
+ version of an object being read does not match the current implementation
+ version as returned by {@link VersionedWritable#getVersion()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionMismatchException -->
+  <!-- start class org.apache.hadoop.io.VIntWritable -->
+  <class name="VIntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VIntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VIntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VIntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VIntWritable"/>
+      <doc>
+      <![CDATA[Compares two VIntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for integer values stored in variable-length format.
+ Such values take between one and five bytes.  Smaller values take fewer bytes.
+ 
+ @see org.apache.hadoop.io.WritableUtils#readVInt(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VIntWritable -->
+  <!-- start class org.apache.hadoop.io.VLongWritable -->
+  <class name="VLongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VLongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VLongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VLongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VLongWritable"/>
+      <doc>
+      <![CDATA[Compares two VLongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs in a variable-length format. Such values take
+  between one and five bytes.  Smaller values take fewer bytes.
+  
+  @see org.apache.hadoop.io.WritableUtils#readVLong(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VLongWritable -->
+  <!-- start interface org.apache.hadoop.io.Writable -->
+  <interface name="Writable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize the fields of this object to <code>out</code>.
+ 
+ @param out <code>DataOuput</code> to serialize this object into.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize the fields of this object from <code>in</code>.  
+ 
+ <p>For efficiency, implementations should attempt to re-use storage in the 
+ existing object where possible.</p>
+ 
+ @param in <code>DataInput</code> to deseriablize this object from.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A serializable object which implements a simple, efficient, serialization 
+ protocol, based on {@link DataInput} and {@link DataOutput}.
+
+ <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
+ framework implements this interface.</p>
+ 
+ <p>Implementations typically implement a static <code>read(DataInput)</code>
+ method which constructs a new instance, calls {@link #readFields(DataInput)} 
+ and returns the instance.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyWritable implements Writable {
+       // Some data     
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+ </pre></blockquote></p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Writable -->
+  <!-- start interface org.apache.hadoop.io.WritableComparable -->
+  <interface name="WritableComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <doc>
+    <![CDATA[A {@link Writable} which is also {@link Comparable}. 
+
+ <p><code>WritableComparable</code>s can be compared to each other, typically 
+ via <code>Comparator</code>s. Any type which is to be used as a 
+ <code>key</code> in the Hadoop Map-Reduce framework should implement this
+ interface.</p>
+
+ <p>Note that <code>hashCode()</code> is frequently used in Hadoop to partition
+ keys. It's important that your implementation of hashCode() returns the same 
+ result across different instances of the JVM. Note also that the default 
+ <code>hashCode()</code> implementation in <code>Object</code> does <b>not</b>
+ satisfy this property.</p>
+  
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyWritableComparable implements WritableComparable<MyWritableComparable> {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue &lt; thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp &gt;&gt;&gt; 32));
+         return result
+       }
+     }
+ </pre></blockquote></p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableComparable -->
+  <!-- start class org.apache.hadoop.io.WritableComparator -->
+  <class name="WritableComparator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.RawComparator"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="WritableComparator"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, org.apache.hadoop.conf.Configuration, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[For backwards compatibility.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a comparator for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="define"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="comparator" type="org.apache.hadoop.io.WritableComparator"/>
+      <doc>
+      <![CDATA[Register an optimized comparator for a {@link WritableComparable}
+ implementation. Comparators registered with this method must be
+ thread-safe.]]>
+      </doc>
+    </method>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the WritableComparable implementation class.]]>
+      </doc>
+    </method>
+    <method name="newKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new {@link WritableComparable} instance.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Optimization hook.  Override this to make SequenceFile.Sorter's scream.
+
+ <p>The default implementation reads the data into two {@link
+ WritableComparable}s (using {@link
+ Writable#readFields(DataInput)}, then calls {@link
+ #compare(WritableComparable,WritableComparable)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="b" type="org.apache.hadoop.io.WritableComparable"/>
+      <doc>
+      <![CDATA[Compare two WritableComparables.
+
+ <p> The default implementation uses the natural ordering, calling {@link
+ Comparable#compareTo(Object)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="java.lang.Object"/>
+      <param name="b" type="java.lang.Object"/>
+    </method>
+    <method name="compareBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Lexicographic order of binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="readUnsignedShort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an unsigned short from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an integer from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a float from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a long from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a double from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a byte array and returns it.
+ @param bytes byte array with decode long
+ @param start starting index
+ @throws java.io.IOException 
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a byte array and returns it.
+ @param bytes byte array with the encoded integer
+ @param start start index
+ @throws java.io.IOException 
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A Comparator for {@link WritableComparable}s.
+
+ <p>This base implemenation uses the natural ordering.  To define alternate
+ orderings, override {@link #compare(WritableComparable,WritableComparable)}.
+
+ <p>One may optimize compare-intensive operations by overriding
+ {@link #compare(byte[],int,int,byte[],int,int)}.  Static utility methods are
+ provided to assist in optimized implementations of this method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableComparator -->
+  <!-- start class org.apache.hadoop.io.WritableFactories -->
+  <class name="WritableFactories" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="factory" type="org.apache.hadoop.io.WritableFactory"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="getFactory" return="org.apache.hadoop.io.WritableFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Factories for non-public writables.  Defining a factory permits {@link
+ ObjectWritable} to be able to construct instances of non-public classes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableFactories -->
+  <!-- start interface org.apache.hadoop.io.WritableFactory -->
+  <interface name="WritableFactory"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a new instance.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A factory for a class of Writable.
+ @see WritableFactories]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableFactory -->
+  <!-- start class org.apache.hadoop.io.WritableUtils -->
+  <class name="WritableUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WritableUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readCompressedByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skipCompressedByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedByteArray" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="displayByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="byte[]"/>
+    </method>
+    <method name="clone" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orig" type="T"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Make a copy of a writable object using serialization to a buffer.
+ @param orig The object to copy
+ @return The copied object]]>
+      </doc>
+    </method>
+    <method name="cloneInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use ReflectionUtils.cloneInto instead.">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialiation to a buffer
+ @param dst the object to copy from
+ @param src the object to copy into, which is destroyed
+ @throws IOException
+ @deprecated use ReflectionUtils.cloneInto instead.]]>
+      </doc>
+    </method>
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes an integer to a binary stream with zero-compressed encoding.
+ For -112 <= i <= 127, only one byte is used with the actual value.
+ For other values of i, the first byte value indicates whether the
+ integer is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -116, the following integer
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -124, the following integer
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+
+ @param stream Binary output stream
+ @param i Integer to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes a long to a binary stream with zero-compressed encoding.
+ For -112 <= i <= 127, only one byte is used with the actual value.
+ For other values of i, the first byte value indicates whether the
+ long is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -120, the following long
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -128, the following long
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+ 
+ @param stream Binary output stream
+ @param i Long to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized long from stream.]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized integer from stream.]]>
+      </doc>
+    </method>
+    <method name="readVIntInRange" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <param name="lower" type="int"/>
+      <param name="upper" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads an integer from the input stream and returns it.
+
+ This function validates that the integer is between [lower, upper],
+ inclusive.
+
+ @param stream Binary input stream
+ @throws java.io.IOException
+ @return deserialized integer from stream]]>
+      </doc>
+    </method>
+    <method name="isNegativeVInt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Given the first byte of a vint/vlong, determine the sign
+ @param value the first byte
+ @return is the value negative]]>
+      </doc>
+    </method>
+    <method name="decodeVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Parse the first byte of a vint/vlong to determine the number of bytes
+ @param value the first byte of the vint/vlong
+ @return the total number of bytes (1 to 9)]]>
+      </doc>
+    </method>
+    <method name="getVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="long"/>
+      <doc>
+      <![CDATA[Get the encoded length if an integer is stored in a variable-length format
+ @return the encoded length]]>
+      </doc>
+    </method>
+    <method name="readEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="enumType" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read an Enum value from DataInput, Enums are read and written 
+ using String values. 
+ @param <T> Enum type
+ @param in DataInput to read from 
+ @param enumType Class type of Enum
+ @return Enum represented by String read from DataInput
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeEnum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="enumVal" type="java.lang.Enum"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[writes String value of enum to DataOutput. 
+ @param out Dataoutput stream
+ @param enumVal enum value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip <i>len</i> number of bytes in input stream<i>in</i>
+ @param in input stream
+ @param len number of bytes to skip
+ @throws IOException when skipped less number of bytes]]>
+      </doc>
+    </method>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writables" type="org.apache.hadoop.io.Writable[]"/>
+      <doc>
+      <![CDATA[Convert writables to a byte array]]>
+      </doc>
+    </method>
+    <method name="readStringSafely" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Read a string, but check it for sanity. The format consists of a vint
+ followed by the given number of bytes.
+ @param in the stream to read from
+ @param maxLength the largest acceptable length of the encoded string
+ @return the bytes as a string
+ @throws IOException if reading from the DataInput fails
+ @throws IllegalArgumentException if the encoded byte size for string 
+             is negative or larger than maxSize. Only the vint is read.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableUtils -->
+  <doc>
+  <![CDATA[Generic i/o code for use when reading and writing data to the network,
+to databases, and to files.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.compress">
+  <!-- start class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <class name="BlockCompressorStream" extends="org.apache.hadoop.io.compress.CompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream}.
+ 
+ @param out stream
+ @param compressor compressor to be used
+ @param bufferSize size of buffer
+ @param compressionOverhead maximum 'overhead' of the compression 
+                            algorithm with given bufferSize]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream} with given output-stream and 
+ compressor.
+ Use default of 512 as bufferSize and compressionOverhead of 
+ (1% of bufferSize + 12 bytes) =  18 bytes (zlib algorithm).
+ 
+ @param out stream
+ @param compressor compressor to be used]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the data provided to the compression codec, compressing no more
+ than the buffer size less the compression overhead as specified during
+ construction for each block.
+
+ Each block contains the uncompressed length for the block, followed by
+ one or more length-prefixed blocks of compressed data.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.CompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.
+
+ It should be noted that this wrapper does not guarantee that blocks will
+ be sized for the compressor. If the
+ {@link org.apache.hadoop.io.compress.Compressor} requires buffering to
+ effect meaningful compression, it is responsible for it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <class name="BlockDecompressorStream" extends="org.apache.hadoop.io.compress.DecompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @param bufferSize size of buffer
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.DecompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BZip2Codec -->
+  <class name="BZip2Codec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.SplittableCompressionCodec"/>
+    <constructor name="BZip2Codec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BZip2Codec.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+
+ @param conf the configuration object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+
+ @return the configuration object used by this objec.]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream}.
+
+ @param out        the location for the final output stream
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream} with the given {@link Compressor}.
+
+ @param out        the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream and return a stream for uncompressed data.
+
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ {@link InputStream} with the given {@link Decompressor}, and return a 
+ stream for uncompressed data.
+
+ @param in           the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates CompressionInputStream to be used to read off uncompressed data
+ in one of the two reading modes. i.e. Continuous or Blocked reading modes
+
+ @param seekableIn The InputStream
+ @param start The start offset into the compressed stream
+ @param end The end offset into the compressed stream
+ @param readMode Controls whether progress is reported continuously or
+                 only at block boundaries.
+
+ @return CompressionInputStream for BZip2 aligned at block boundaries]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[.bz2 is recognized as the default extension for compressed BZip2 files
+
+ @return A String telling the default bzip2 file extension]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class provides output and input streams for bzip2 compression
+ and decompression.  It uses the native bzip2 library on the system
+ if possible, else it uses a pure-Java implementation of the bzip2
+ algorithm.  The configuration parameter
+ io.compression.codec.bzip2.library can be used to control this
+ behavior.
+
+ In the pure-Java mode, the Compressor and Decompressor interfaces
+ are not implemented.  Therefore, in that mode, those methods of
+ CompressionCodec which have a Compressor or Decompressor type
+ argument, throw UnsupportedOperationException.
+
+ Currently, support for splittability is available only in the
+ pure-Java mode; therefore, if a SplitCompressionInputStream is
+ requested, the pure-Java implementation is used, regardless of the
+ setting of the configuration parameter mentioned above.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BZip2Codec -->
+  <!-- start class org.apache.hadoop.io.compress.CodecPool -->
+  <class name="CodecPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CodecPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a {@link Compressor} for the given {@link CompressionCodec} from the 
+ pool or a new one.
+
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Compressor</code>
+ @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
+ @return <code>Compressor</code> for the given 
+         <code>CompressionCodec</code> from the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+    </method>
+    <method name="getDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Get a {@link Decompressor} for the given {@link CompressionCodec} from the
+ pool or a new one.
+  
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Decompressor</code>
+ @return <code>Decompressor</code> for the given 
+         <code>CompressionCodec</code> the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="returnCompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <doc>
+      <![CDATA[Return the {@link Compressor} to the pool.
+ 
+ @param compressor the <code>Compressor</code> to be returned to the pool]]>
+      </doc>
+    </method>
+    <method name="returnDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <doc>
+      <![CDATA[Return the {@link Decompressor} to the pool.
+ 
+ @param decompressor the <code>Decompressor</code> to be returned to the 
+                     pool]]>
+      </doc>
+    </method>
+    <method name="getLeasedCompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Compressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <method name="getLeasedDecompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Decompressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A global compressor/decompressor pool used to save and reuse 
+ (possibly native) compression/decompression codecs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecPool -->
+  <!-- start interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <interface name="CompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream}.
+ 
+ @param out the location for the final output stream
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream} with the given {@link Compressor}.
+ 
+ @param out the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream.
+ 
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given 
+ {@link InputStream} with the given {@link Decompressor}.
+ 
+ @param in the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default filename extension for this kind of compression.
+ @return the extension including the '.']]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a streaming compression/decompression pair.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <class name="CompressionCodecFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionCodecFactory" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Find the codecs specified in the config value io.compression.codecs 
+ and register them. Defaults to gzip and deflate.]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Print the extension map out as a string.]]>
+      </doc>
+    </method>
+    <method name="getCodecClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the list of codecs discovered via a Java ServiceLoader, or
+ listed in the configuration. Codecs specified in configuration come
+ later in the returned list, and are considered to override those
+ from the ServiceLoader.
+ @param conf the configuration to look in
+ @return a list of the {@link CompressionCodec} classes]]>
+      </doc>
+    </method>
+    <method name="setCodecClasses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="classes" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets a list of codec classes in the configuration. In addition to any
+ classes specified using this method, {@link CompressionCodec} classes on
+ the classpath are discovered using a Java ServiceLoader.
+ @param conf the configuration to modify
+ @param classes the list of classes to set]]>
+      </doc>
+    </method>
+    <method name="getCodec" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the given file based on its
+ filename suffix.
+ @param file the filename to check
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByClassName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classname" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name.
+ @param classname the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias.
+ <p/>
+ Codec aliases are case insensitive.
+ <p/>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias and returns its implemetation class.
+ <p/>
+ Codec aliases are case insensitive.
+ <p/>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec class]]>
+      </doc>
+    </method>
+    <method name="removeSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Removes a suffix from a filename, if it has it.
+ @param filename the filename to strip
+ @param suffix the suffix to remove
+ @return the shortened filename]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A little test program.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory that will find the correct codec for a given filename.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <class name="CompressionInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <constructor name="CompressionInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a compression input stream that reads
+ the decompressed bytes from the given stream.
+ 
+ @param in The input stream to be compressed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the decompressor to its initial state and discard any buffered data,
+ as the underlying stream may have been repositioned.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method returns the current position in the stream.
+
+ @return Current position in stream as a long]]>
+      </doc>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <field name="in" type="java.io.InputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The input stream to be compressed.]]>
+      </doc>
+    </field>
+    <field name="maxAvailableData" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A compression input stream.
+
+ <p>Implementations are assumed to be buffered.  This permits clients to
+ reposition the underlying input stream then call {@link #resetState()},
+ without having to also synchronize client buffers.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <class name="CompressionOutputStream" extends="java.io.OutputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionOutputStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a compression output stream that writes
+ the compressed bytes to the given stream.
+ @param out]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write compressed bytes to the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finishes writing compressed data to the output stream 
+ without closing the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the compression to the initial state. 
+ Does not reset the underlying stream.]]>
+      </doc>
+    </method>
+    <field name="out" type="java.io.OutputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The output stream to be compressed.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A compression output stream.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Compressor -->
+  <interface name="Compressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for compression. 
+ This should be called whenever #needsInput() returns 
+ <code>true</code> indicating that more input data is required.
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the input data buffer is empty and 
+ #setInput() should be called to provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ #setInput() should be called in order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary 
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="getBytesRead" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of uncompressed bytes input so far.]]>
+      </doc>
+    </method>
+    <method name="getBytesWritten" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of compressed bytes output so far.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When called, indicates that compression should end
+ with the current contents of the input buffer.]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the end of the compressed 
+ data output stream has been reached.
+ @return <code>true</code> if the end of the compressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="compress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with compressed data. Returns actual number
+ of bytes of compressed data. A return value of 0 indicates that
+ needsInput() should be called in order to determine if more input
+ data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of compressed data.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets compressor so that a new set of input data can be processed.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the compressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <method name="reinit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Prepare the compressor to be used in a new stream with settings defined in
+ the given Configuration
+ 
+ @param conf Configuration from which new setting are fetched]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'compressor' which can be  
+ plugged into a {@link CompressionOutputStream} to compress data.
+ This is modelled after {@link java.util.zip.Deflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Compressor -->
+  <!-- start class org.apache.hadoop.io.compress.CompressorStream -->
+  <class name="CompressorStream" extends="org.apache.hadoop.io.compress.CompressionOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param out Underlying output stream.]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="compressor" type="org.apache.hadoop.io.compress.Compressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressorStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Decompressor -->
+  <interface name="Decompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for decompression. 
+ This should be called if and only if {@link #needsInput()} returns 
+ <code>true</code> indicating that more input data is required.
+ (Both native and non-native versions of various Decompressors require
+ that the data passed in via <code>b[]</code> remain unmodified until
+ the caller is explicitly notified--via {@link #needsInput()}--that the
+ buffer may be safely modified.  With this requirement, an extra
+ buffer-copy can be avoided.)
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called to
+ provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called in
+ order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsDictionary" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if a preset dictionary is needed for decompression.
+ @return <code>true</code> if a preset dictionary is needed for decompression]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the end of the decompressed 
+ data output stream has been reached. Indicates a concatenated data stream
+ when finished() returns <code>true</code> and {@link #getRemaining()}
+ returns a positive value. finished() will be reset with the
+ {@link #reset()} method.
+ @return <code>true</code> if the end of the decompressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with uncompressed data. Returns actual number
+ of bytes of uncompressed data. A return value of 0 indicates that
+ {@link #needsInput()} should be called in order to determine if more
+ input data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of uncompressed data.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes remaining in the compressed data buffer.
+ Indicates a concatenated data stream if {@link #finished()} returns
+ <code>true</code> and getRemaining() returns a positive value. If
+ {@link #finished()} returns <code>true</code> and getRemaining() returns
+ a zero value, indicates that the end of data stream has been reached and
+ is not a concatenated data stream. 
+ @return The number of bytes remaining in the compressed data buffer.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets decompressor and input and output buffers so that a new set of
+ input data can be processed. If {@link #finished()}} returns
+ <code>true</code> and {@link #getRemaining()} returns a positive value,
+ reset() is called before processing of the next data stream in the
+ concatenated data stream. {@link #finished()} will be reset and will
+ return <code>false</code> when reset() is called.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the decompressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'de-compressor' which can be  
+ plugged into a {@link CompressionInputStream} to compress data.
+ This is modelled after {@link java.util.zip.Inflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Decompressor -->
+  <!-- start class org.apache.hadoop.io.compress.DecompressorStream -->
+  <class name="DecompressorStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param in Underlying input stream.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="available" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="markSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="mark"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readlimit" type="int"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="eof" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.DefaultCodec -->
+  <class name="DefaultCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <implements name="org.apache.hadoop.io.compress.DirectDecompressionCodec"/>
+    <constructor name="DefaultCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DefaultCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <interface name="DirectDecompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link DirectDecompressor} for use by this {@link DirectDecompressionCodec}.
+ 
+ @return a new direct decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a codec which can decompress direct bytebuffers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <interface name="DirectDecompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="decompress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.nio.ByteBuffer"/>
+      <param name="dst" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Specification of a direct ByteBuffer 'de-compressor'.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <!-- start class org.apache.hadoop.io.compress.GzipCodec -->
+  <class name="GzipCodec" extends="org.apache.hadoop.io.compress.DefaultCodec"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GzipCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class creates gzip compressors/decompressors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.GzipCodec -->
+  <!-- start class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <class name="SplitCompressionInputStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitCompressionInputStream" type="java.io.InputStream, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="setStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="start" type="long"/>
+    </method>
+    <method name="setEnd"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="end" type="long"/>
+    </method>
+    <method name="getAdjustedStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of start.
+ @return The changed value of start]]>
+      </doc>
+    </method>
+    <method name="getAdjustedEnd" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of end.
+ @return The changed value of end]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputStream covering a range of compressed data. The start and end
+ offsets requested by a client may be modified by the codec to fit block
+ boundaries or other algorithm-dependent requirements.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+  <interface name="SplittableCompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a stream as dictated by the readMode.  This method is used when
+ the codecs wants the ability to work with the underlying stream positions.
+
+ @param seekableIn  The seekable input stream (seeks in compressed data)
+ @param start The start offset into the compressed stream. May be changed
+              by the underlying codec.
+ @param end The end offset into the compressed stream. May be changed by
+            the underlying codec.
+ @param readMode Controls whether stream position is reported continuously
+                 from the compressed stream only only at block boundaries.
+ @return  a stream to read uncompressed bytes from]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface is meant to be implemented by those compression codecs
+ which are capable to compress / de-compress a stream starting at any
+ arbitrary position.
+
+ Especially the process of de-compressing a stream starting at some arbitrary
+ position is challenging.  Most of the codecs are only able to successfully
+ de-compress a stream, if they start from the very beginning till the end.
+ One of the reasons is the stored state at the beginning of the stream which
+ is crucial for de-compression.
+
+ Yet there are few codecs which do not save the whole state at the beginning
+ of the stream and hence can be used to de-compress stream starting at any
+ arbitrary points.  This interface is meant to be used by such codecs.  Such
+ codecs are highly valuable, especially in the context of Hadoop, because
+ an input compressed file can be split and hence can be worked on by multiple
+ machines in parallel.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+</package>
+<package name="org.apache.hadoop.io.file.tfile">
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <class name="MetaBlockAlreadyExists" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - Meta Block with the same name already exists.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <class name="MetaBlockDoesNotExist" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - No such Meta Block with the given name.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <!-- start interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <interface name="RawComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="buffer" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying byte array.
+ 
+ @return The underlying byte array.]]>
+      </doc>
+    </method>
+    <method name="offset" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the offset of the first byte in the byte array.
+ 
+ @return The offset of the first byte in the byte array.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the size of the byte range in the byte array.
+ 
+ @return The size of the byte range in the byte array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for objects that can be compared through {@link RawComparator}.
+ This is useful in places where we need a single object reference to specify a
+ range of bytes in a byte array, such as {@link Comparable} or
+ {@link Collections#binarySearch(java.util.List, Object, Comparator)}
+ 
+ The actual comparison among RawComparable's requires an external
+ RawComparator and it is applications' responsibility to ensure two
+ RawComparable are supposed to be semantically comparable with the same
+ RawComparator.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <!-- start class org.apache.hadoop.io.file.tfile.TFile -->
+  <class name="TFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="makeComparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Make a raw comparator from a string name.
+ 
+ @param name
+          Comparator name
+ @return A RawComparable comparator.]]>
+      </doc>
+    </method>
+    <method name="getSupportedCompressionAlgorithms" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get names of supported compression algorithms. The names are acceptable by
+ TFile.Writer.
+ 
+ @return Array of strings, each represents a supported compression
+         algorithm. Currently, the following compression algorithms are
+         supported.
+         <ul>
+         <li>"none" - No compression.
+         <li>"lzo" - LZO compression.
+         <li>"gz" - GZIP compression.
+         </ul>]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Dumping the TFile information.
+ 
+ @param args
+          A list of TFile paths.]]>
+      </doc>
+    </method>
+    <field name="COMPRESSION_GZ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: gzip]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_LZO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: lzo]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_NONE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: none]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_MEMCMP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator: memcmp]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_JCLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator prefix: java class]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A TFile is a container of key-value pairs. Both keys and values are type-less
+ bytes. Keys are restricted to 64KB, value length is not restricted
+ (practically limited to the available disk storage). TFile further provides
+ the following features:
+ <ul>
+ <li>Block Compression.
+ <li>Named meta data blocks.
+ <li>Sorted or unsorted keys.
+ <li>Seek by key or by file offset.
+ </ul>
+ The memory footprint of a TFile includes the following:
+ <ul>
+ <li>Some constant overhead of reading or writing a compressed block.
+ <ul>
+ <li>Each compressed block requires one compression/decompression codec for
+ I/O.
+ <li>Temporary space to buffer the key.
+ <li>Temporary space to buffer the value (for TFile.Writer only). Values are
+ chunk encoded, so that we buffer at most one chunk of user data. By default,
+ the chunk buffer is 1MB. Reading chunked value does not require additional
+ memory.
+ </ul>
+ <li>TFile index, which is proportional to the total number of Data Blocks.
+ The total amount of memory needed to hold the index can be estimated as
+ (56+AvgKeySize)*NumBlocks.
+ <li>MetaBlock index, which is proportional to the total number of Meta
+ Blocks.The total amount of memory needed to hold the index for Meta Blocks
+ can be estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ </ul>
+ <p>
+ The behavior of TFile can be customized by the following variables through
+ Configuration:
+ <ul>
+ <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default
+ to 1MB. Values of the length less than the chunk size is guaranteed to have
+ known value length in read time (See
+ {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for
+ FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for
+ FSDataInputStream. Integer (in bytes). Default to 256KB.
+ </ul>
+ <p>
+ Suggestions on performance optimization.
+ <ul>
+ <li>Minimum block size. We recommend a setting of minimum block size between
+ 256KB to 1MB for general usage. Larger block size is preferred if files are
+ primarily for sequential access. However, it would lead to inefficient random
+ access (because there are more data to decompress). Smaller blocks are good
+ for random access, but require more memory to hold the block index, and may
+ be slower to create (because we must flush the compressor stream at the
+ conclusion of each data block, which leads to an FS I/O flush). Further, due
+ to the internal caching in Compression codec, the smallest possible block
+ size would be around 20KB-30KB.
+ <li>The current implementation does not offer true multi-threading for
+ reading. The implementation uses FSDataInputStream seek()+read(), which is
+ shown to be much faster than positioned-read call in single thread mode.
+ However, it also means that if multiple threads attempt to access the same
+ TFile (using multiple scanners) simultaneously, the actual I/O is carried out
+ sequentially even if they access different DFS blocks.
+ <li>Compression codec. Use "none" if the data is not very compressable (by
+ compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
+ as the starting point for experimenting. "gz" overs slightly better
+ compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
+ decompress, comparing to "lzo".
+ <li>File system buffering, if the underlying FSDataInputStream and
+ FSDataOutputStream is already adequately buffered; or if applications
+ reads/writes keys and values in large buffers, we can reduce the sizes of
+ input/output buffering in TFile layer by setting the configuration parameters
+ "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ </ul>
+ 
+ Some design rationale behind TFile can be found at <a
+ href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.TFile -->
+  <!-- start class org.apache.hadoop.io.file.tfile.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding an integer into a variable-length encoding format. Synonymous to
+ <code>Utils#writeVLong(out, n)</code>.
+ 
+ @param out
+          output stream
+ @param n
+          The integer to be encoded
+ @throws IOException
+ @see Utils#writeVLong(DataOutput, long)]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding a Long integer into a variable-length encoding format.
+ <ul>
+ <li>if n in [-32, 127): encode in one byte with the actual value.
+ Otherwise,
+ <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
+ byte[1]=n&0xff. Otherwise,
+ <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
+ 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
+ byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
+ (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
+ (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff;
+ byte[4]=(n>>8)&0xff; byte[5]=n&0xff
+ <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
+ (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff;
+ byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
+ (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff;
+ byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff;
+ byte[7]=n&0xff;
+ <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
+ (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
+ byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff;
+ byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ </ul>
+ 
+ @param out
+          output stream
+ @param n
+          the integer number
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Synonymous to
+ <code>(int)Utils#readVLong(in)</code>.
+ 
+ @param in
+          input stream
+ @return the decoded integer
+ @throws IOException
+ 
+ @see Utils#readVLong(DataInput)]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Suppose the value of the first byte
+ is FB, and the following bytes are NB[*].
+ <ul>
+ <li>if (FB >= -32), return (long)FB;
+ <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 +
+ NB[1]&0xff;
+ <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 +
+ (NB[1]&0xff)<<8 + NB[2]&0xff;
+ <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
+ big-endian integer.
+ 
+ @param in
+          input stream
+ @return the decoded long integer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a String as a VInt n, followed by n Bytes as in Text format.
+ 
+ @param out
+ @param s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a String as a VInt n, followed by n Bytes in Text format.
+ 
+ @param in
+          The input stream.
+ @return The string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Supporting Utility classes used by TFile, and shared by users of TFile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.Utils -->
+</package>
+<package name="org.apache.hadoop.io.serializer">
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <class name="JavaSerialization" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="JavaSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ An experimental {@link Serialization} for Java {@link Serializable} classes.
+ </p>
+ @see JavaSerializationComparator]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <class name="JavaSerializationComparator" extends="org.apache.hadoop.io.serializer.DeserializerComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JavaSerializationComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ A {@link RawComparator} that uses a {@link JavaSerialization}
+ {@link Deserializer} to deserialize objects that are then compared via
+ their {@link Comparable} interfaces.
+ </p>
+ @param <T>
+ @see JavaSerialization]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <!-- start class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <class name="WritableSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="WritableSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A {@link Serialization} for {@link Writable}s that delegates to
+ {@link Writable#write(java.io.DataOutput)} and
+ {@link Writable#readFields(java.io.DataInput)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides a mechanism for using different serialization frameworks
+in Hadoop. The property "io.serializations" defines a list of
+{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create
+{@link org.apache.hadoop.io.serializer.Serializer}s and
+{@link org.apache.hadoop.io.serializer.Deserializer}s.
+</p>
+
+<p>
+To add a new serialization framework write an implementation of
+{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the
+"io.serializations" property.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.serializer.avro">
+  <!-- start interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <interface name="AvroReflectSerializable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Tag interface for Avro 'reflect' serializable classes. Classes implementing 
+ this interface can be serialized/deserialized using 
+ {@link AvroReflectSerialization}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <class name="AvroReflectSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroReflectSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_REFLECT_PACKAGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key to configure packages that contain classes to be serialized and 
+ deserialized using this class. Multiple packages can be specified using 
+ comma-separated list.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Serialization for Avro Reflect classes. For a class to be accepted by this 
+ serialization, it must either be in the package list configured via 
+ <code>avro.reflect.pkgs</code> or implement 
+ {@link AvroReflectSerializable} interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <class name="AvroSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="AvroSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_SCHEMA_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class for providing serialization to Avro types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <class name="AvroSpecificSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroSpecificSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Serialization for Avro Specific classes. This serialization is to be used 
+ for classes generated by Avro's 'specific' compiler.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides Avro serialization in Hadoop. This can be used to 
+serialize/deserialize Avro types in Hadoop.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for 
+serialization of classes generated by Avro's 'specific' compiler.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for 
+other classes. 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for  
+any class which is either in the package list configured via 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} 
+or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable}
+interface.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.ipc.protocolPB">
+</package>
+<package name="org.apache.hadoop.log">
+</package>
+<package name="org.apache.hadoop.log.metrics">
+  <!-- start class org.apache.hadoop.log.metrics.EventCounter -->
+  <class name="EventCounter" extends="org.apache.log4j.AppenderSkeleton"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventCounter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="requiresLayout" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A log4J Appender that simply counts logging events in three levels:
+ fatal, error and warn. The class name is used in log4j.properties]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.log.metrics.EventCounter -->
+</package>
+<package name="org.apache.hadoop.metrics">
+  <doc>
+  <![CDATA[This package defines an API for reporting performance metric information.
+<p/>
+The API is abstract so that it can be implemented on top of
+a variety of metrics client libraries.  The choice of 
+client library is a configuration option, and different 
+modules within the same application can use
+different metrics implementation libraries.
+<p/>
+Sub-packages:
+<dl>
+    <dt><code>org.apache.hadoop.metrics.spi</code></dt>
+    <dd>The abstract Server Provider Interface package. Those wishing to
+    integrate the metrics API with a particular metrics client library should 
+    extend this package.</dd>
+    
+    <dt><code>org.apache.hadoop.metrics.file</code></dt>
+    <dd>An implementation package which writes the metric data to 
+    a file, or sends it to the standard output stream.</dd>
+ 
+    <dt> <code>org.apache.hadoop.metrics.ganglia</code></dt>
+    <dd>An implementation package which sends metric data to 
+    <a href="http://ganglia.sourceforge.net/">Ganglia</a>.</dd>
+</dl>
+
+<h3>Introduction to the Metrics API</h3>
+
+Here is a simple example of how to use this package to report a single
+metric value:
+<pre>
+    private ContextFactory contextFactory = ContextFactory.getFactory();
+    
+    void reportMyMetric(float myMetric) {
+        MetricsContext myContext = contextFactory.getContext("myContext");
+        MetricsRecord myRecord = myContext.getRecord("myRecord");
+        myRecord.setMetric("myMetric", myMetric);
+        myRecord.update();
+    }
+</pre>
+  
+In this example there are three names:
+<dl>
+  <dt><i>myContext</i></dt>
+  <dd>The context name will typically identify either the application, or else a
+  module within an application or library.</dd>
+  
+  <dt><i>myRecord</i></dt>
+  <dd>The record name generally identifies some entity for which a set of
+  metrics are to be reported.  For example, you could have a record named 
+  "cacheStats" for reporting a number of statistics relating to the usage of
+  some cache in your application.</dd>
+  
+  <dt><i>myMetric</i></dt>
+  <dd>This identifies a particular metric.  For example, you might have metrics
+  named "cache_hits" and "cache_misses".
+  </dd>
+</dl>
+
+<h3>Tags</h3>
+
+In some cases it is useful to have multiple records with the same name. For 
+example, suppose that you want to report statistics about each disk on a computer. 
+In this case, the record name would be something like "diskStats", but you also
+need to identify the disk which is done by adding a <i>tag</i> to the record.
+The code could look something like this:
+<pre>
+    private MetricsRecord diskStats =
+            contextFactory.getContext("myContext").getRecord("diskStats");
+            
+    void reportDiskMetrics(String diskName, float diskBusy, float diskUsed) {
+        diskStats.setTag("diskName", diskName);
+        diskStats.setMetric("diskBusy", diskBusy);
+        diskStats.setMetric("diskUsed", diskUsed);
+        diskStats.update();
+    }
+</pre>
+
+<h3>Buffering and Callbacks</h3>
+
+Data is not sent immediately to the metrics system when 
+<code>MetricsRecord.update()</code> is called. Instead it is stored in an
+internal table, and the contents of the table are sent periodically.
+This can be important for two reasons:
+<ol>
+    <li>It means that a programmer is free to put calls to this API in an 
+    inner loop, since updates can be very frequent without slowing down
+    the application significantly.</li>
+    <li>Some implementations can gain efficiency by combining many metrics 
+    into a single UDP message.</li>
+</ol>
+
+The API provides a timer-based callback via the 
+<code>registerUpdater()</code> method.  The benefit of this
+versus using <code>java.util.Timer</code> is that the callbacks will be done 
+immediately before sending the data, making the data as current as possible.
+
+<h3>Configuration</h3>
+
+It is possible to programmatically examine and modify configuration data
+before creating a context, like this:
+<pre>
+    ContextFactory factory = ContextFactory.getFactory();
+    ... examine and/or modify factory attributes ...
+    MetricsContext context = factory.getContext("myContext");
+</pre>
+The factory attributes can be examined and modified using the following
+<code>ContextFactory</code>methods:
+<ul>
+    <li><code>Object getAttribute(String attributeName)</code></li>
+    <li><code>String[] getAttributeNames()</code></li>
+    <li><code>void setAttribute(String name, Object value)</code></li>
+    <li><code>void removeAttribute(attributeName)</code></li>
+</ul>
+
+<p/>
+<code>ContextFactory.getFactory()</code> initializes the factory attributes by
+reading the properties file <code>hadoop-metrics.properties</code> if it exists 
+on the class path.
+
+<p/>
+A factory attribute named:
+<pre>
+<i>contextName</i>.class
+</pre>
+should have as its value the fully qualified name of the class to be 
+instantiated by a call of the <code>CodeFactory</code> method
+<code>getContext(<i>contextName</i>)</code>.  If this factory attribute is not 
+specified, the default is to instantiate 
+<code>org.apache.hadoop.metrics.file.FileContext</code>.
+
+<p/>
+Other factory attributes are specific to a particular implementation of this 
+API and are documented elsewhere.  For example, configuration attributes for
+the file and Ganglia implementations can be found in the javadoc for 
+their respective packages.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.metrics.ganglia">
+  <!-- start class org.apache.hadoop.metrics.ganglia.GangliaContext -->
+  <class name="GangliaContext" extends="org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link org.apache.hadoop.metrics2.sink.ganglia.GangliaSink30}
+ instead.">
+    <constructor name="GangliaContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of GangliaContext]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[method to close the datagram socket]]>
+      </doc>
+    </method>
+    <method name="emitMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUnits" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+    </method>
+    <method name="getSlope" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+    </method>
+    <method name="getTmax" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+    </method>
+    <method name="getDmax" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+    </method>
+    <method name="xdr_string"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Puts a string into the buffer by first writing the size of the string
+ as an int, followed by the bytes of the string, padded if necessary to
+ a multiple of 4.]]>
+      </doc>
+    </method>
+    <method name="xdr_int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Puts an integer into the buffer as 4 bytes, big-endian.]]>
+      </doc>
+    </method>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="offset" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="metricsServers" type="java.util.List"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="datagramSocket" type="java.net.DatagramSocket"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Context for sending metrics to Ganglia.
+
+ @deprecated Use {@link org.apache.hadoop.metrics2.sink.ganglia.GangliaSink30}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.ganglia.GangliaContext -->
+  <doc>
+  <![CDATA[<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+-->
+
+Implementation of the metrics package that sends metric data to 
+<a href="http://ganglia.sourceforge.net/">Ganglia</a>.
+Programmers should not normally need to use this package directly. Instead
+they should use org.hadoop.metrics.
+
+<p/>
+These are the implementation specific factory attributes 
+(See ContextFactory.getFactory()):
+
+<dl>
+    <dt><i>contextName</i>.servers</dt>
+    <dd>Space and/or comma separated sequence of servers to which UDP
+    messages should be sent.</dd>
+    
+    <dt><i>contextName</i>.period</dt>
+    <dd>The period in seconds on which the metric data is sent to the
+    server(s).</dd>
+
+    <dt><i>contextName</i>.multicast</dt>
+    <dd>Enable multicast for Ganglia</dd>
+
+    <dt><i>contextName</i>.multicast.ttl</dt>
+    <dd>TTL for multicast packets</dd>
+    
+    <dt><i>contextName</i>.units.<i>recordName</i>.<i>metricName</i></dt>
+    <dd>The units for the specified metric in the specified record.</dd>
+    
+    <dt><i>contextName</i>.slope.<i>recordName</i>.<i>metricName</i></dt>
+    <dd>The slope for the specified metric in the specified record.</dd>
+    
+    <dt><i>contextName</i>.tmax.<i>recordName</i>.<i>metricName</i></dt>
+    <dd>The tmax for the specified metric in the specified record.</dd>
+    
+    <dt><i>contextName</i>.dmax.<i>recordName</i>.<i>metricName</i></dt>
+    <dd>The dmax for the specified metric in the specified record.</dd>
+    
+</dl>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.metrics.spi">
+  <!-- start class org.apache.hadoop.metrics.spi.AbstractMetricsContext -->
+  <class name="AbstractMetricsContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <implements name="org.apache.hadoop.metrics.MetricsContext"/>
+    <constructor name="AbstractMetricsContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of AbstractMetricsContext]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="contextName" type="java.lang.String"/>
+      <param name="factory" type="org.apache.hadoop.metrics.ContextFactory"/>
+      <doc>
+      <![CDATA[Initializes the context.]]>
+      </doc>
+    </method>
+    <method name="getAttribute" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for subclasses to access factory attributes.]]>
+      </doc>
+    </method>
+    <method name="getAttributeTable" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns an attribute-value map derived from the factory attributes
+ by finding all factory attributes that begin with 
+ <i>contextName</i>.<i>tableName</i>.  The returned map consists of
+ those attributes with the contextName and tableName stripped off.]]>
+      </doc>
+    </method>
+    <method name="getContextName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the context name.]]>
+      </doc>
+    </method>
+    <method name="getContextFactory" return="org.apache.hadoop.metrics.ContextFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the factory by which this context was created.]]>
+      </doc>
+    </method>
+    <method name="startMonitoring"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Starts or restarts monitoring, the emitting of metrics records.]]>
+      </doc>
+    </method>
+    <method name="stopMonitoring"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stops monitoring.  This does not free buffered data. 
+ @see #close()]]>
+      </doc>
+    </method>
+    <method name="isMonitoring" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if monitoring is currently in progress.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stops monitoring and frees buffered data, returning this
+ object to its initial state.]]>
+      </doc>
+    </method>
+    <method name="createRecord" return="org.apache.hadoop.metrics.MetricsRecord"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="recordName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Creates a new AbstractMetricsRecord instance with the given <code>recordName</code>.
+ Throws an exception if the metrics implementation is configured with a fixed
+ set of record names and <code>recordName</code> is not in that set.
+ 
+ @param recordName the name of the record
+ @throws MetricsException if recordName conflicts with configuration data]]>
+      </doc>
+    </method>
+    <method name="newRecord" return="org.apache.hadoop.metrics.MetricsRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="recordName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Subclasses should override this if they subclass MetricsRecordImpl.
+ @param recordName the name of the record
+ @return newly created instance of MetricsRecordImpl or subclass]]>
+      </doc>
+    </method>
+    <method name="registerUpdater"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updater" type="org.apache.hadoop.metrics.Updater"/>
+      <doc>
+      <![CDATA[Registers a callback to be called at time intervals determined by
+ the configuration.
+
+ @param updater object to be run periodically; it should update
+ some metrics records]]>
+      </doc>
+    </method>
+    <method name="unregisterUpdater"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updater" type="org.apache.hadoop.metrics.Updater"/>
+      <doc>
+      <![CDATA[Removes a callback, if it exists.
+
+ @param updater object to be removed from the callback list]]>
+      </doc>
+    </method>
+    <method name="getAllRecords" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieves all the records managed by this MetricsContext.
+ Useful for monitoring systems that are polling-based.
+ @return A non-null collection of all monitoring records.]]>
+      </doc>
+    </method>
+    <method name="emitRecord"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="contextName" type="java.lang.String"/>
+      <param name="recordName" type="java.lang.String"/>
+      <param name="outRec" type="org.apache.hadoop.metrics.spi.OutputRecord"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sends a record to the metrics system.]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called each period after all records have been emitted, this method does nothing.
+ Subclasses may override it in order to perform some kind of flush.]]>
+      </doc>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics.spi.MetricsRecordImpl"/>
+      <doc>
+      <![CDATA[Called by MetricsRecordImpl.update().  Creates or updates a row in
+ the internal table of metric data.]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics.spi.MetricsRecordImpl"/>
+      <doc>
+      <![CDATA[Called by MetricsRecordImpl.remove().  Removes all matching rows in
+ the internal table of metric data.  A row matches if it has the same
+ tag names and values as record, but it may also have additional
+ tags.]]>
+      </doc>
+    </method>
+    <method name="getPeriod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the timer period.]]>
+      </doc>
+    </method>
+    <method name="setPeriod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="period" type="int"/>
+      <doc>
+      <![CDATA[Sets the timer period]]>
+      </doc>
+    </method>
+    <method name="parseAndSetPeriod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[If a period is set in the attribute passed in, override
+ the default with it.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The main class of the Service Provider Interface.  This class should be
+ extended in order to integrate the Metrics API with a specific metrics
+ client library. <p/>
+
+ This class implements the internal table of metric data, and the timer
+ on which data is to be sent to the metrics system.  Subclasses must
+ override the abstract <code>emitRecord</code> method in order to transmit
+ the data. <p/>
+
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.AbstractMetricsContext -->
+  <!-- start class org.apache.hadoop.metrics.spi.CompositeContext -->
+  <class name="CompositeContext" extends="org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <constructor name="CompositeContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.CompositeContext -->
+  <!-- start class org.apache.hadoop.metrics.spi.MetricsRecordImpl -->
+  <class name="MetricsRecordImpl" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link org.apache.hadoop.metrics2.impl.MetricsRecordImpl}
+ instead.">
+    <implements name="org.apache.hadoop.metrics.MetricsRecord"/>
+    <constructor name="MetricsRecordImpl" type="java.lang.String, org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of FileRecord]]>
+      </doc>
+    </constructor>
+    <method name="getRecordName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the record name. 
+
+ @return the record name]]>
+      </doc>
+    </method>
+    <method name="setTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <param name="tagValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the named tag to the specified value.
+
+ @param tagName name of the tag
+ @param tagValue new value of the tag
+ @throws MetricsException if the tagName conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <param name="tagValue" type="int"/>
+      <doc>
+      <![CDATA[Sets the named tag to the specified value.
+
+ @param tagName name of the tag
+ @param tagValue new value of the tag
+ @throws MetricsException if the tagName conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <param name="tagValue" type="long"/>
+      <doc>
+      <![CDATA[Sets the named tag to the specified value.
+
+ @param tagName name of the tag
+ @param tagValue new value of the tag
+ @throws MetricsException if the tagName conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <param name="tagValue" type="short"/>
+      <doc>
+      <![CDATA[Sets the named tag to the specified value.
+
+ @param tagName name of the tag
+ @param tagValue new value of the tag
+ @throws MetricsException if the tagName conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <param name="tagValue" type="byte"/>
+      <doc>
+      <![CDATA[Sets the named tag to the specified value.
+
+ @param tagName name of the tag
+ @param tagValue new value of the tag
+ @throws MetricsException if the tagName conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="removeTag"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Removes any tag of the specified name.]]>
+      </doc>
+    </method>
+    <method name="setMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="int"/>
+      <doc>
+      <![CDATA[Sets the named metric to the specified value.
+
+ @param metricName name of the metric
+ @param metricValue new value of the metric
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="long"/>
+      <doc>
+      <![CDATA[Sets the named metric to the specified value.
+
+ @param metricName name of the metric
+ @param metricValue new value of the metric
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="short"/>
+      <doc>
+      <![CDATA[Sets the named metric to the specified value.
+
+ @param metricName name of the metric
+ @param metricValue new value of the metric
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="byte"/>
+      <doc>
+      <![CDATA[Sets the named metric to the specified value.
+
+ @param metricName name of the metric
+ @param metricValue new value of the metric
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="setMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="float"/>
+      <doc>
+      <![CDATA[Sets the named metric to the specified value.
+
+ @param metricName name of the metric
+ @param metricValue new value of the metric
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="incrMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="int"/>
+      <doc>
+      <![CDATA[Increments the named metric by the specified value.
+
+ @param metricName name of the metric
+ @param metricValue incremental value
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="incrMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="long"/>
+      <doc>
+      <![CDATA[Increments the named metric by the specified value.
+
+ @param metricName name of the metric
+ @param metricValue incremental value
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="incrMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="short"/>
+      <doc>
+      <![CDATA[Increments the named metric by the specified value.
+
+ @param metricName name of the metric
+ @param metricValue incremental value
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="incrMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="byte"/>
+      <doc>
+      <![CDATA[Increments the named metric by the specified value.
+
+ @param metricName name of the metric
+ @param metricValue incremental value
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="incrMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricName" type="java.lang.String"/>
+      <param name="metricValue" type="float"/>
+      <doc>
+      <![CDATA[Increments the named metric by the specified value.
+
+ @param metricName name of the metric
+ @param metricValue incremental value
+ @throws MetricsException if the metricName or the type of the metricValue 
+ conflicts with the configuration]]>
+      </doc>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Updates the table of buffered data which is to be sent periodically.
+ If the tag values match an existing row, that row is updated; 
+ otherwise, a new row is added.]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Removes the row, if it exists, in the buffered data table having tags 
+ that equal the tags that have been set on this record.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An implementation of MetricsRecord.  Keeps a back-pointer to the context
+ from which it was created, and delegates back to it on <code>update</code>
+ and <code>remove()</code>.
+
+ @deprecated Use {@link org.apache.hadoop.metrics2.impl.MetricsRecordImpl}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.MetricsRecordImpl -->
+  <!-- start class org.apache.hadoop.metrics.spi.MetricValue -->
+  <class name="MetricValue" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricValue" type="java.lang.Number, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of MetricValue]]>
+      </doc>
+    </constructor>
+    <method name="isIncrement" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNumber" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="ABSOLUTE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INCREMENT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Number that is either an absolute or an incremental amount.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.MetricValue -->
+  <!-- start class org.apache.hadoop.metrics.spi.NoEmitMetricsContext -->
+  <class name="NoEmitMetricsContext" extends="org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <constructor name="NoEmitMetricsContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of NullContextWithUpdateThread]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A MetricsContext that does not emit data, but, unlike NullContextWithUpdate,
+ does save it for retrieval with getAllRecords().
+ 
+ This is useful if you want to support {@link MetricsServlet}, but
+ not emit metrics in any other way.
+
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.NoEmitMetricsContext -->
+  <!-- start class org.apache.hadoop.metrics.spi.NullContext -->
+  <class name="NullContext" extends="org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <constructor name="NullContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of NullContext]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Null metrics context: a metrics context which does nothing.  Used as the
+ default context, so that no performance data is emitted if no configuration
+ data is found.
+ 
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.NullContext -->
+  <!-- start class org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -->
+  <class name="NullContextWithUpdateThread" extends="org.apache.hadoop.metrics.spi.AbstractMetricsContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <constructor name="NullContextWithUpdateThread"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of NullContextWithUpdateThread]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A null context which has a thread calling 
+ periodically when monitoring is started. This keeps the data sampled 
+ correctly.
+ In all other respects, this is like the NULL context: No data is emitted.
+ This is suitable for Monitoring systems like JMX which reads the metrics
+  when someone reads the data from JMX.
+ 
+ The default impl of start and stop monitoring:
+  is the AbstractMetricsContext is good enough.
+ 
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -->
+  <!-- start class org.apache.hadoop.metrics.spi.OutputRecord -->
+  <class name="OutputRecord" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <method name="getTagNames" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the set of tag names]]>
+      </doc>
+    </method>
+    <method name="getTag" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a tag object which is can be a String, Integer, Short or Byte.
+
+ @return the tag value, or null if there is no such tag]]>
+      </doc>
+    </method>
+    <method name="getMetricNames" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the set of metric names.]]>
+      </doc>
+    </method>
+    <method name="getMetric" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the metric object which can be a Float, Integer, Short or Byte.]]>
+      </doc>
+    </method>
+    <method name="getTagsCopy" return="org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a copy of this record's tags.]]>
+      </doc>
+    </method>
+    <method name="getMetricsCopy" return="org.apache.hadoop.metrics.spi.AbstractMetricsContext.MetricMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a copy of this record's metrics.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents a record of metric data to be sent to a metrics system.
+
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.OutputRecord -->
+  <!-- start class org.apache.hadoop.metrics.spi.Util -->
+  <class name="Util" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use org.apache.hadoop.metrics2 package instead.">
+    <method name="parse" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="specs" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Parses a space and/or comma separated sequence of server specifications
+ of the form <i>hostname</i> or <i>hostname:port</i>.  If 
+ the specs string is null, defaults to localhost:defaultPort.
+ 
+ @return a list of InetSocketAddress objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Static utility methods
+
+ @deprecated Use org.apache.hadoop.metrics2 package instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics.spi.Util -->
+  <doc>
+  <![CDATA[The Service Provider Interface for the Metrics API.  This package provides
+an interface allowing a variety of metrics reporting implementations to be
+plugged in to the Metrics API.  Examples of such implementations can be found 
+in the packages <code>org.apache.hadoop.metrics.file</code> and
+<code>org.apache.hadoop.metrics.ganglia</code>.<p/>
+
+Plugging in an implementation involves writing a concrete subclass of 
+<code>AbstractMetricsContext</code>.  The subclass should get its
+ configuration information using the <code>getAttribute(<i>attributeName</i>)</code>
+ method.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.metrics2">
+  <!-- start class org.apache.hadoop.metrics2.AbstractMetric -->
+  <class name="AbstractMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="AbstractMetric" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the metric
+ @param info  about the metric]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="value" return="java.lang.Number"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the metric
+ @return the value of the metric]]>
+      </doc>
+    </method>
+    <method name="type" return="org.apache.hadoop.metrics2.MetricType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the metric
+ @return the type of the metric]]>
+      </doc>
+    </method>
+    <method name="visit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visitor" type="org.apache.hadoop.metrics2.MetricsVisitor"/>
+      <doc>
+      <![CDATA[Accept a visitor interface
+ @param visitor of the metric]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The immutable metric]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.AbstractMetric -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <interface name="MetricsCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param name  of the record
+ @return  a metrics record builder for the record]]>
+      </doc>
+    </method>
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param info  of the record
+ @return  a metrics record builder for the record]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics collector interface]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsException -->
+  <class name="MetricsException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message
+ @param message for the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message and a cause
+ @param message for the exception
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a cause
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A general metrics exception wrapper]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsException -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsFilter -->
+  <class name="MetricsFilter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <constructor name="MetricsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether to accept the name
+ @param name  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Whether to accept the tag
+ @param tag to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Whether to accept the tags
+ @param tags to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Whether to accept the record
+ @param record  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics filter interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsFilter -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <interface name="MetricsInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the name of the metric/tag]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the description of the metric/tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to provide immutable meta info for metrics]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <class name="MetricsJsonBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsJsonBuilder" type="org.apache.hadoop.metrics2.MetricsCollector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method]]>
+      </doc>
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Build a JSON dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <interface name="MetricsPlugin"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration.SubsetConfiguration"/>
+      <doc>
+      <![CDATA[Initialize the plugin
+ @param conf  the configuration object for the plugin]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The plugin interface for the metrics framework]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <interface name="MetricsRecord"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="timestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the metrics
+ @return  the timestamp]]>
+      </doc>
+    </method>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the record name]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the description of the record]]>
+      </doc>
+    </method>
+    <method name="context" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the context name of the record]]>
+      </doc>
+    </method>
+    <method name="tags" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tags of the record
+ Note: returning a collection instead of iterable as we
+ need to use tags as keys (hence Collection#hashCode etc.) in maps
+ @return an unmodifiable collection of tags]]>
+      </doc>
+    </method>
+    <method name="metrics" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the metrics of the record
+ @return an immutable iterable interface for metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An immutable snapshot of metrics with a timestamp]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <class name="MetricsRecordBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRecordBuilder"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics tag
+ @param info  metadata of the tag
+ @param value of the tag
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Add an immutable metrics tag object
+ @param tag a pre-made tag object (potentially save an object construction)
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+      <doc>
+      <![CDATA[Add a pre-made immutable metric object
+ @param metric  the pre-made metric to save an object construction
+ @return self]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the context tag
+ @param value of the context
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add an integer metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add an long metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add a integer gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a long gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Add a float gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Add a double gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <method name="endRecord" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Syntactic sugar to add multiple records in a collector in a one liner.
+ @return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics record builder interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSink -->
+  <interface name="MetricsSink"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Put a metrics record in the sink
+ @param record  the record to put]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flush any buffered metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics sink interface. <p>
+ Implementations of this interface consume the {@link MetricsRecord} generated
+ from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ periodically pushes the {@link MetricsRecord} to the sink using
+ {@link #putMetrics(MetricsRecord)} method.  If the implementing class also
+ implements {@link Closeable}, then the MetricsSystem will close the sink when
+ it is stopped.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSink -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSource -->
+  <interface name="MetricsSource"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collector" type="org.apache.hadoop.metrics2.MetricsCollector"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get metrics from the source
+ @param collector to contain the resulting metrics snapshot
+ @param all if true, return all metrics even if unchanged.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics source interface]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSource -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsSystem -->
+  <class name="MetricsSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSystemMXBean"/>
+    <constructor name="MetricsSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unregisterSource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unregister a metrics source
+ @param name of the source. This is the name you use to call register()]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source (deriving name and description from the object)
+ @param <T>   the actual type of the source object
+ @param source  object to register
+ @return  the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source
+ @param <T>   the actual type of the source object
+ @param source object to register
+ @param name  of the source. Must be unique or null (then extracted from
+              the annotations of the source object.)
+ @param desc  the description of the source (or null. See above.)
+ @return the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callback" type="org.apache.hadoop.metrics2.MetricsSystem.Callback"/>
+      <doc>
+      <![CDATA[Register a callback interface for JMX events
+ @param callback  the callback object implementing the MBean interface.]]>
+      </doc>
+    </method>
+    <method name="publishMetricsNow"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests an immediate publish of all metrics from sources to sinks.
+ 
+ This is a "soft" request: the expectation is that a best effort will be
+ done to synchronously snapshot the metrics from all the sources and put
+ them in all the sinks (including flushing the sinks) before returning to
+ the caller. If this can't be accomplished in reasonable time it's OK to
+ return to the caller before everything is done.]]>
+      </doc>
+    </method>
+    <method name="shutdown" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system completely (usually during server shutdown.)
+ The MetricsSystemMXBean will be unregistered.
+ @return true if shutdown completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics system interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsSystem -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <interface name="MetricsSystemMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="startMetricsMBeans"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start metrics MBeans
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stopMetricsMBeans"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop metrics MBeans.
+ Note, it doesn't stop the metrics system control MBean,
+ i.e this interface.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="currentConfig" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the current config
+ Avoided getConfig, as it'll turn into a "Config" attribute,
+ which doesn't support multiple line values in jconsole.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The JMX interface to the metrics system]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsTag -->
+  <class name="MetricsTag" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="MetricsTag" type="org.apache.hadoop.metrics2.MetricsInfo, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the tag with name, description and value
+ @param info  of the tag
+ @param value of the tag]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the tag]]>
+      </doc>
+    </method>
+    <method name="value" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the tag
+ @return  the value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Immutable tag for metrics (for grouping on host/queue/username etc.)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsTag -->
+  <!-- start class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <class name="MetricStringBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricStringBuilder" type="org.apache.hadoop.metrics2.MetricsCollector, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method
+ @param prefix string before each entry
+ @param separator separator between name and value
+ @param suffix suffix after each entry]]>
+      </doc>
+    </constructor>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="tuple" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add any key,val pair to the string, between the prefix and suffix,
+ separated by the separator.
+ @param key key
+ @param value value
+ @return this instance]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Build a string dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.
+
+ Every entry is formatted as
+ {@code prefix + name + separator + value + suffix}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsVisitor -->
+  <interface name="MetricsVisitor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="gauge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Callback for float value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Callback for double value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A visitor interface for metrics]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsVisitor -->
+</package>
+<package name="org.apache.hadoop.metrics2.annotation">
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metric -->
+  <class name="Metric"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a single metric]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metric -->
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metrics -->
+  <class name="Metrics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a group of metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metrics -->
+</package>
+<package name="org.apache.hadoop.metrics2.filter">
+  <!-- start class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <class name="GlobFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GlobFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A glob pattern filter for metrics.
+
+ The class name is used in metrics config files]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <!-- start class org.apache.hadoop.metrics2.filter.RegexFilter -->
+  <class name="RegexFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A regex pattern filter for metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.RegexFilter -->
+</package>
+<package name="org.apache.hadoop.metrics2.lib">
+  <!-- start class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <class name="DefaultMetricsSystem" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="initialize" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method to initialize the metrics system
+ @param prefix  for the metrics system configuration
+ @return the metrics system instance]]>
+      </doc>
+    </method>
+    <method name="instance" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics system object]]>
+      </doc>
+    </method>
+    <method name="shutdown"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The default metrics system singleton]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <!-- start class org.apache.hadoop.metrics2.lib.Interns -->
+  <class name="Interns" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Interns"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric info object.
+ @param name Name of metric info object
+ @param description Description of metric info object
+ @return an interned metric info object]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param info  of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to create interned metrics info.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.Interns -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <class name="MetricsRegistry" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRegistry" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a record name
+ @param name  of the record of the metrics]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsRegistry" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a metadata object
+ @param info  the info object for the metrics record/group]]>
+      </doc>
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the metrics registry]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.lib.MutableMetric"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric by name
+ @param name  of the metric
+ @return the metric object]]>
+      </doc>
+    </method>
+    <method name="getTag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a tag by name
+ @param name  of the tag
+ @return the tag object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newQuantiles" return="org.apache.hadoop.metrics2.lib.MutableQuantiles"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="interval" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable metric that estimates quantiles of a stream of values
+ @param name of the metric
+ @param desc metric description
+ @param sampleName of the metric (e.g., "Ops")
+ @param valueName of the metric (e.g., "Time" or "Latency")
+ @param interval rollover interval of estimator in seconds
+ @return a new quantile estimator object
+ @throws MetricsException if interval is not a positive integer]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @param extended    produce extended stat (stdev, min/max etc.) if true.
+ @return a new mutable stat metric object]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @param description of the metric
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric (for throughput measurement)
+ @param name  of the metric
+ @param desc  description
+ @param extended  produce extended stat (stdev/min/max etc.) if true
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRatesWithAggregation" return="org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="newMutableRollingAverages" return="org.apache.hadoop.metrics2.lib.MutableRollingAverages"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add sample to a stat metric by name.
+ @param name  of the metric
+ @param value of the snapshot to add]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the metrics context tag
+ @param name of the context
+ @return the registry itself as a convenience]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @param override  existing tag if true
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param info  metadata of the tag
+ @param value of the tag
+ @param override existing tag if true
+ @return the registry (for keep adding tags etc.)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Sample all the mutable metrics and put the snapshot in the builder
+ @param builder to contain the metrics snapshot
+ @param all get all the metrics even if the values are not changed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An optional metrics registry class for creating and maintaining a
+ collection of MetricsMutables, making writing metrics source easier.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <class name="MutableCounter" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounter" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the metric value by 1.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable counter (monotonically increasing) metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <class name="MutableCounterInt" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable int counter for implementing metrics sources]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <class name="MutableCounterLong" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounterLong" type="org.apache.hadoop.metrics2.MetricsInfo, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable long counter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <class name="MutableGauge" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableGauge" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the value of the metric by 1]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Decrement the value of the metric by 1]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable gauge metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <class name="MutableGaugeInt" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable int gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <class name="MutableGaugeLong" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable long gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <class name="MutableMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="snapshot"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get a snapshot of the metric
+ @param builder the metrics record builder
+ @param all if true, snapshot unchanged metrics as well]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <doc>
+      <![CDATA[Get a snapshot of metric if changed
+ @param builder the metrics record builder]]>
+      </doc>
+    </method>
+    <method name="setChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the changed flag in mutable operations]]>
+      </doc>
+    </method>
+    <method name="clearChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the changed flag in the snapshot operations]]>
+      </doc>
+    </method>
+    <method name="changed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  true if metric is changed since last snapshot/snapshot]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <class name="MutableQuantiles" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableQuantiles" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiates a new {@link MutableQuantiles} for a metric that rolls itself
+ over on the specified time interval.
+ 
+ @param name
+          of the metric
+ @param description
+          long-form textual description of the metric
+ @param sampleName
+          type of items in the stream (e.g., "Ops")
+ @param valueName
+          type of the values
+ @param interval
+          rollover interval (in seconds) of the estimator]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="getInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEstimator" return="org.apache.hadoop.metrics2.util.QuantileEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the quantile estimator.
+
+ @return the quantile estimator]]>
+      </doc>
+    </method>
+    <method name="setEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quantileEstimator" type="org.apache.hadoop.metrics2.util.QuantileEstimator"/>
+    </method>
+    <field name="quantiles" type="org.apache.hadoop.metrics2.util.Quantile[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="previousSnapshot" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Watches a stream of long values, maintaining online estimates of specific
+ quantiles with provably low error bounds. This is particularly useful for
+ accurate high-percentile (e.g. 95th, 99th) latency metrics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <class name="MutableRate" extends="org.apache.hadoop.metrics2.lib.MutableStat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[A convenient mutable metric for throughput measurement]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <class name="MutableRates" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics
+
+ This class synchronizes all accesses to the metrics it
+ contains, so it should not be used in situations where
+ there is high contention on the metrics.
+ {@link MutableRatesWithAggregation} is preferable in that
+ situation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <class name="MutableRatesWithAggregation" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableRatesWithAggregation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric.
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics.
+
+ Each thread will maintain a local rate count, and upon snapshot,
+ these values will be aggregated into a global rate. This class
+ should only be used for long running threads, as any metrics
+ produced between the last snapshot and the death of a thread
+ will be lost. This allows for significantly higher concurrency
+ than {@link MutableRates}. See HADOOP-24420.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <class name="MutableRollingAverages" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="MutableRollingAverages" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for {@link MutableRollingAverages}.
+ @param metricValueName]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="collectThreadLocalStates"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Collects states maintained in {@link ThreadLocal}, if any.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[@param name
+          name of metric
+ @param value
+          value of metric]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStats" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="minSamples" type="long"/>
+      <doc>
+      <![CDATA[Retrieve a map of metric name -> (aggregate).
+ Filter out entries that don't have at least minSamples.
+
+ @return a map of peer DataNode Id to the average latency to that
+         node seen over the measurement period.]]>
+      </doc>
+    </method>
+    <method name="setRecordValidityMs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Use for test only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class maintains a group of rolling average metrics. It implements the
+ algorithm of rolling average, i.e. a number of sliding windows are kept to
+ roll over and evict old subsets of samples. Each window has a subset of
+ samples in a stream, where sub-sum and sub-total are collected. All sub-sums
+ and sub-totals in all windows will be aggregated to final-sum and final-total
+ used to compute final average, which is called rolling average.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableStat -->
+  <class name="MutableStat" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a sample statistics metric
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")
+ @param extended    create extended stats (stdev, min/max etc.) by default.]]>
+      </doc>
+    </constructor>
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a snapshot stat metric with extended stat off by default
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")]]>
+      </doc>
+    </constructor>
+    <method name="setExtended"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to display the extended stats (stdev, min/max etc.) or not
+ @param extended enable/disable displaying extended stats]]>
+      </doc>
+    </method>
+    <method name="setUpdateTimeStamp"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateTimeStamp" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to update the snapshot time or not.
+ @param updateTimeStamp enable update stats snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numSamples" type="long"/>
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Add a number of samples and their sum to the running stat
+
+ Note that although use of this method will preserve accurate mean values,
+ large values for numSamples may result in inaccurate variance values due
+ to the use of a single step of the Welford method for variance calculation.
+ @param numSamples  number of samples
+ @param sum of the samples]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a snapshot to the metric.
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="lastStat" return="org.apache.hadoop.metrics2.util.SampleStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a SampleStat object that supports
+ calls like StdDev and Mean.
+ @return SampleStat]]>
+      </doc>
+    </method>
+    <method name="resetMinMax"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset the all time min max of the metric]]>
+      </doc>
+    </method>
+    <method name="getSnapshotTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the SampleStat snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mutable metric with stats.
+
+ Useful for keeping throughput/latency stats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableStat -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink">
+  <!-- start class org.apache.hadoop.metrics2.sink.FileSink -->
+  <class name="FileSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="FileSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a file]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.FileSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <class name="GraphiteSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="GraphiteSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a Graphite server]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <class name="RollingFileSystemSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="RollingFileSystemSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty instance.  Required for reflection.]]>
+      </doc>
+    </constructor>
+    <constructor name="RollingFileSystemSink" type="long, long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance for testing.
+
+ @param flushIntervalMillis the roll interval in millis
+ @param flushOffsetIntervalMillis the roll offset interval in millis]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics2Properties" type="org.apache.commons.configuration.SubsetConfiguration"/>
+    </method>
+    <method name="getRollInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Extract the roll interval from the configuration and return it in
+ milliseconds.
+
+ @return the roll interval in millis]]>
+      </doc>
+    </method>
+    <method name="updateFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Update the {@link #nextFlush} variable to the next flush time. Add
+ an integer number of flush intervals, preserving the initial random offset.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="setInitialFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Set the {@link #nextFlush} variable to the initial flush time. The initial
+ flush will be an integer number of flush intervals past the beginning of
+ the current hour and will have a random offset added, up to
+ {@link #rollOffsetIntervalMillis}. The initial flush will be a time in
+ past that can be used from which to calculate future flush times.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="source" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ignoreError" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allowAppend" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="basePath" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollOffsetIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nextFlush" type="java.util.Calendar"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="forceFlush" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="hasFlushed" type="boolean"
+      transient="false" volatile="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedConf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedFilesystem" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>This class is a metrics sink that uses
+ {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs.  Every
+ roll interval a new directory will be created under the path specified by the
+ <code>basepath</code> property. All metrics will be logged to a file in the
+ current interval's directory in a file named &lt;hostname&gt;.log, where
+ &lt;hostname&gt; is the name of the host on which the metrics logging
+ process is running. The base path is set by the
+ <code>&lt;prefix&gt;.sink.&lt;instance&gt;.basepath</code> property.  The
+ time zone used to create the current interval's directory name is GMT.  If
+ the <code>basepath</code> property isn't specified, it will default to
+ &quot;/tmp&quot;, which is the temp directory on whatever default file
+ system is configured for the cluster.</p>
+
+ <p>The <code>&lt;prefix&gt;.sink.&lt;instance&gt;.ignore-error</code>
+ property controls whether an exception is thrown when an error is encountered
+ writing a log file.  The default value is <code>true</code>.  When set to
+ <code>false</code>, file errors are quietly swallowed.</p>
+
+ <p>The <code>roll-interval</code> property sets the amount of time before
+ rolling the directory. The default value is 1 hour. The roll interval may
+ not be less than 1 minute. The property's value should be given as
+ <i>number unit</i>, where <i>number</i> is an integer value, and
+ <i>unit</i> is a valid unit.  Valid units are <i>minute</i>, <i>hour</i>,
+ and <i>day</i>.  The units are case insensitive and may be abbreviated or
+ plural. If no units are specified, hours are assumed. For example,
+ &quot;2&quot;, &quot;2h&quot;, &quot;2 hour&quot;, and
+ &quot;2 hours&quot; are all valid ways to specify two hours.</p>
+
+ <p>The <code>roll-offset-interval-millis</code> property sets the upper
+ bound on a random time interval (in milliseconds) that is used to delay
+ before the initial roll.  All subsequent rolls will happen an integer
+ number of roll intervals after the initial roll, hence retaining the original
+ offset. The purpose of this property is to insert some variance in the roll
+ times so that large clusters using this sink on every node don't cause a
+ performance impact on HDFS by rolling simultaneously.  The default value is
+ 30000 (30s).  When writing to HDFS, as a rule of thumb, the roll offset in
+ millis should be no less than the number of sink instances times 5.
+
+ <p>The primary use of this class is for logging to HDFS.  As it uses
+ {@link org.apache.hadoop.fs.FileSystem} to access the target file system,
+ however, it can be used to write to the local file system, Amazon S3, or any
+ other supported file system.  The base path for the sink will determine the
+ file system used.  An unqualified path will write to the default file system
+ set by the configuration.</p>
+
+ <p>Not all file systems support the ability to append to files.  In file
+ systems without the ability to append to files, only one writer can write to
+ a file at a time.  To allow for concurrent writes from multiple daemons on a
+ single host, the <code>source</code> property is used to set unique headers
+ for the log files.  The property should be set to the name of
+ the source daemon, e.g. <i>namenode</i>.  The value of the
+ <code>source</code> property should typically be the same as the property's
+ prefix.  If this property is not set, the source is taken to be
+ <i>unknown</i>.</p>
+
+ <p>Instead of appending to an existing file, by default the sink
+ will create a new file with a suffix of &quot;.&lt;n&gt;&quet;, where
+ <i>n</i> is the next lowest integer that isn't already used in a file name,
+ similar to the Hadoop daemon logs.  NOTE: the file with the <b>highest</b>
+ sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
+
+ <p>For file systems that allow append, the sink supports appending to the
+ existing file instead. If the <code>allow-append</code> property is set to
+ true, the sink will instead append to the existing file on file systems that
+ support appends. By default, the <code>allow-append</code> property is
+ false.</p>
+
+ <p>Note that when writing to HDFS with <code>allow-append</code> set to true,
+ there is a minimum acceptable number of data nodes.  If the number of data
+ nodes drops below that minimum, the append will succeed, but reading the
+ data will fail with an IOException in the DataStreamer class.  The minimum
+ number of data nodes required for a successful append is generally 2 or
+ 3.</p>
+
+ <p>Note also that when writing to HDFS, the file size information is not
+ updated until the file is closed (at the end of the interval) even though
+ the data is being written successfully. This is a known HDFS limitation that
+ exists because of the performance cost of updating the metadata.  See
+ <a href="https://issues.apache.org/jira/browse/HDFS-5478">HDFS-5478</a>.</p>
+
+ <p>When using this sink in a secure (Kerberos) environment, two additional
+ properties must be set: <code>keytab-key</code> and
+ <code>principal-key</code>. <code>keytab-key</code> should contain the key by
+ which the keytab file can be found in the configuration, for example,
+ <code>yarn.nodemanager.keytab</code>. <code>principal-key</code> should
+ contain the key by which the principal can be found in the configuration,
+ for example, <code>yarn.nodemanager.principal</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.StatsDSink -->
+  <class name="StatsDSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="StatsDSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="writeMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="line" type="java.lang.String"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes metrics to a StatsD daemon.
+ This sink will produce metrics of the form
+ '[hostname].servicename.context.name.metricname:value|type'
+ where hostname is optional. This is useful when sending to
+ a daemon that is running on the localhost and will add the
+ hostname to the metric (such as the
+ <a href="https://collectd.org/">CollectD</a> StatsD plugin).
+ <br/>
+ To configure this plugin, you will need to add the following
+ entries to your hadoop-metrics2.properties file:
+ <br/>
+ <pre>
+ *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.StatsDSink -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink.ganglia">
+</package>
+<package name="org.apache.hadoop.metrics2.source">
+</package>
+<package name="org.apache.hadoop.metrics2.util">
+  <!-- start class org.apache.hadoop.metrics2.util.MBeans -->
+  <class name="MBeans" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MBeans"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service=<serviceName>,name=<nameName>"
+ Where the <serviceName> and <nameName> are the supplied parameters
+
+ @param serviceName
+ @param nameName
+ @param theMbean - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="getMbeanNameService" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="getMbeanNameName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mbeanName" type="javax.management.ObjectName"/>
+    </method>
+    <doc>
+    <![CDATA[This util class provides a method to register an MBean using
+ our standard naming convention as described in the doc
+  for {link {@link #register(String, String, Object)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MBeans -->
+  <!-- start class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <class name="MetricsCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="MetricsCache" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a metrics cache
+ @param maxRecsPerName  limit of the number records per record name]]>
+      </doc>
+    </constructor>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <param name="includingTags" type="boolean"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cached record
+ @param mr the update record
+ @param includingTags cache tag values (for later lookup by name) if true
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cache record
+ @param mr the update record
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="tags" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Get the cached record
+ @param name of the record
+ @param tags of the record
+ @return the cached record or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A metrics cache for sinks that don't support sparse updates.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <!-- start class org.apache.hadoop.metrics2.util.Servers -->
+  <class name="Servers" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parse" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="specs" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Parses a space and/or comma separated sequence of server specifications
+ of the form <i>hostname</i> or <i>hostname:port</i>.  If
+ the specs string is null, defaults to localhost:defaultPort.
+
+ @param specs   server specs (see description)
+ @param defaultPort the default port if not specified
+ @return a list of InetSocketAddress objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to handle server addresses]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.Servers -->
+</package>
+<package name="org.apache.hadoop.net">
+  <!-- start class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <class name="AbstractDNSToSwitchMapping" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.net.DNSToSwitchMapping"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractDNSToSwitchMapping"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an unconfigured instance]]>
+      </doc>
+    </constructor>
+    <constructor name="AbstractDNSToSwitchMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance, caching the configuration file.
+ This constructor does not call {@link #setConf(Configuration)}; if
+ a subclass extracts information in that method, it must call it explicitly.
+ @param conf the configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate that indicates that the switch mapping is known to be
+ single-switch. The base class returns false: it assumes all mappings are
+ multi-rack. Subclasses may override this with methods that are more aware
+ of their topologies.
+
+ <p/>
+
+ This method is used when parts of Hadoop need know whether to apply
+ single rack vs multi-rack policies, such as during block placement.
+ Such algorithms behave differently if they are on multi-switch systems.
+ </p>
+
+ @return true if the mapping thinks that it is on a single switch]]>
+      </doc>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the map (for diagnostics)
+ @return a clone of the map or null for none known]]>
+      </doc>
+    </method>
+    <method name="dumpTopology" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a string listing the switch mapping implementation,
+ the mapping for every known node and the number of nodes and
+ unique switches known about -each entry to a separate line.
+ @return a string that can be presented to the ops team or used in
+ debug messages.]]>
+      </doc>
+    </method>
+    <method name="isSingleSwitchByScriptPolicy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMappingSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapping" type="org.apache.hadoop.net.DNSToSwitchMapping"/>
+      <doc>
+      <![CDATA[Query for a {@link DNSToSwitchMapping} instance being on a single
+ switch.
+ <p/>
+ This predicate simply assumes that all mappings not derived from
+ this class are multi-switch.
+ @param mapping the mapping to query
+ @return true if the base class says it is single switch, or the mapping
+ is not derived from this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a base class for DNS to Switch mappings. <p/> It is not mandatory to
+ derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
+ recommended, as it makes it easy for the Hadoop developers to add new methods
+ to this base class that are automatically picked up by all implementations.
+ <p/>
+
+ This class does not extend the <code>Configured</code>
+ base class, and should not be changed to do so, as it causes problems
+ for subclasses. The constructor of the <code>Configured</code> calls
+ the  {@link #setConf(Configuration)} method, which will call into the
+ subclasses before they have been fully constructed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <class name="CachedDNSToSwitchMapping" extends="org.apache.hadoop.net.AbstractDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CachedDNSToSwitchMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[cache a raw DNS mapping
+ @param rawMapping the raw mapping to cache]]>
+      </doc>
+    </constructor>
+    <method name="resolve" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the (host x switch) map.
+ @return a copy of the cached map of hosts to rack]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegate the switch topology query to the raw mapping, via
+ {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+ @return true iff the raw mapper is considered single-switch.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <field name="rawMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The uncached mapping]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A cached implementation of DNSToSwitchMapping that takes an
+ raw DNSToSwitchMapping and stores the resolved network location in 
+ a cache. The following calls to a resolved network location
+ will get its location from the cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ConnectTimeoutException -->
+  <class name="ConnectTimeoutException" extends="java.net.SocketTimeoutException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConnectTimeoutException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown by {@link NetUtils#connect(java.net.Socket, java.net.SocketAddress, int)}
+ if it times out while connecting to the remote host.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ConnectTimeoutException -->
+  <!-- start interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <interface name="DNSToSwitchMapping"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="resolve" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Resolves a list of DNS-names/IP-addresses and returns back a list of
+ switch information (network paths). One-to-one correspondence must be 
+ maintained between the elements in the lists. 
+ Consider an element in the argument list - x.y.com. The switch information
+ that is returned must be a network path of the form /foo/rack, 
+ where / is the root, and 'foo' is the switch where 'rack' is connected.
+ Note the hostname/ip-address is not part of the returned path.
+ The network topology of the cluster would determine the number of
+ components in the network path.
+ <p/>
+
+ If a name cannot be resolved to a rack, the implementation
+ should return {@link NetworkTopology#DEFAULT_RACK}. This
+ is what the bundled implementations do, though it is not a formal requirement
+
+ @param names the list of hosts to resolve (can be empty)
+ @return list of resolved network paths.
+ If <i>names</i> is empty, the returned list is also empty]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload all of the cached mappings.
+
+ If there is a cache, this method will clear it, so that future accesses
+ will get a chance to see the new data.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Reload cached mappings on specific nodes.
+
+ If there is a cache on these nodes, this method will clear it, so that 
+ future accesses will see updated data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface that must be implemented to allow pluggable
+ DNS-name/IP-address to RackID resolvers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ScriptBasedMapping -->
+  <class name="ScriptBasedMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ScriptBasedMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with the default configuration.
+ </p>
+ Calling {@link #setConf(Configuration)} will trigger a
+ re-evaluation of the configuration settings and so be used to
+ set up the mapping script.]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given raw mapping
+ @param rawMap raw DNSTOSwithMapping]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given configuration
+ @param conf configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p/>
+ This will get called in the superclass constructor, so a check is needed
+ to ensure that the raw mapping is defined before trying to relaying a null
+ configuration.
+ @param conf]]>
+      </doc>
+    </method>
+    <field name="NO_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Text used in the {@link #toString()} method if there is no string
+ {@value}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class implements the {@link DNSToSwitchMapping} interface using a 
+ script configured via the
+ {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ <p/>
+ It contains a static class <code>RawScriptBasedMapping</code> that performs
+ the work: reading the configuration parameters, executing any defined
+ script, handling errors and such like. The outer
+ class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ queries.
+ <p/>
+ This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ true if and only if a script is defined.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ScriptBasedMapping -->
+  <!-- start class org.apache.hadoop.net.SocksSocketFactory -->
+  <class name="SocksSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="SocksSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <constructor name="SocksSocketFactory" type="java.net.Proxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with a supplied Proxy
+ 
+ @param proxy the proxy to use to create sockets]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.SocksSocketFactory -->
+  <!-- start class org.apache.hadoop.net.StandardSocketFactory -->
+  <class name="StandardSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StandardSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.StandardSocketFactory -->
+  <!-- start class org.apache.hadoop.net.TableMapping -->
+  <class name="TableMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TableMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ file. The columns are separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ </p>
+ <p>
+ This class uses the configuration parameter {@code
+ net.topology.table.file.name} to locate the mapping file.
+ </p>
+ <p>
+ Calls to {@link #resolve(List)} will look up the address as defined in the
+ mapping file. If no entry corresponding to the address is found, the value
+ {@code /default-rack} is returned.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.TableMapping -->
+  <doc>
+  <![CDATA[Network-related classes.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.net.unix">
+</package>
+<package name="org.apache.hadoop.record">
+  <!-- start class org.apache.hadoop.record.BinaryRecordInput -->
+  <class name="BinaryRecordInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordInput"/>
+    <constructor name="BinaryRecordInput" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BinaryRecordInput]]>
+      </doc>
+    </constructor>
+    <constructor name="BinaryRecordInput" type="java.io.DataInput"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BinaryRecordInput]]>
+      </doc>
+    </constructor>
+    <method name="get" return="org.apache.hadoop.record.BinaryRecordInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inp" type="java.io.DataInput"/>
+      <doc>
+      <![CDATA[Get a thread-local record input for the supplied DataInput.
+ @param inp data input stream
+ @return binary record input corresponding to the supplied DataInput.]]>
+      </doc>
+    </method>
+    <method name="readByte" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBool" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBuffer" return="org.apache.hadoop.record.Buffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.BinaryRecordInput -->
+  <!-- start class org.apache.hadoop.record.BinaryRecordOutput -->
+  <class name="BinaryRecordOutput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordOutput"/>
+    <constructor name="BinaryRecordOutput" type="java.io.OutputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BinaryRecordOutput]]>
+      </doc>
+    </constructor>
+    <constructor name="BinaryRecordOutput" type="java.io.DataOutput"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BinaryRecordOutput]]>
+      </doc>
+    </constructor>
+    <method name="get" return="org.apache.hadoop.record.BinaryRecordOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <doc>
+      <![CDATA[Get a thread-local record output for the supplied DataOutput.
+ @param out data output stream
+ @return binary record output corresponding to the supplied DataOutput.]]>
+      </doc>
+    </method>
+    <method name="writeByte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="boolean"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="long"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="float"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="d" type="double"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="org.apache.hadoop.record.Buffer"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.BinaryRecordOutput -->
+  <!-- start class org.apache.hadoop.record.Buffer -->
+  <class name="Buffer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://avro.apache.org/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="Buffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a zero-count sequence.]]>
+      </doc>
+    </constructor>
+    <constructor name="Buffer" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a Buffer using the byte array as the initial value.
+
+ @param bytes This array becomes the backing storage for the object.]]>
+      </doc>
+    </constructor>
+    <constructor name="Buffer" type="byte[], int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a Buffer using the byte range as the initial value.
+
+ @param bytes Copy of this array becomes the backing storage for the object.
+ @param offset offset into byte array
+ @param length length of data]]>
+      </doc>
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <doc>
+      <![CDATA[Use the specified bytes array as underlying sequence.
+
+ @param bytes byte sequence]]>
+      </doc>
+    </method>
+    <method name="copy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Copy the specified byte array to the Buffer. Replaces the current buffer.
+
+ @param bytes byte array to be assigned
+ @param offset offset into byte array
+ @param length length of data]]>
+      </doc>
+    </method>
+    <method name="get" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data from the Buffer.
+ 
+ @return The data is only valid between 0 and getCount() - 1.]]>
+      </doc>
+    </method>
+    <method name="getCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current count of the buffer.]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the capacity, which is the maximum count that could handled without
+ resizing the backing storage.
+ 
+ @return The number of bytes]]>
+      </doc>
+    </method>
+    <method name="setCapacity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newCapacity" type="int"/>
+      <doc>
+      <![CDATA[Change the capacity of the backing storage.
+ The data is preserved if newCapacity {@literal >=} getCount().
+ @param newCapacity The new capacity in bytes.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset the buffer to 0 size]]>
+      </doc>
+    </method>
+    <method name="truncate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Change the capacity of the backing store to be the same as the current 
+ count of buffer.]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Append specified bytes to the buffer.
+
+ @param bytes byte array to be appended
+ @param offset offset into byte array
+ @param length length of data]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <doc>
+      <![CDATA[Append specified bytes to the buffer
+
+ @param bytes byte array to be appended]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Define the sort order of the Buffer.
+ 
+ @param other The other buffer
+ @return Positive if this is bigger than other, 0 if they are equal, and
+         negative if this is smaller than other.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="charsetName" type="java.lang.String"/>
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+      <doc>
+      <![CDATA[Convert the byte buffer to a string an specific character encoding
+
+ @param charsetName Valid Java Character Set Name]]>
+      </doc>
+    </method>
+    <method name="clone" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="CloneNotSupportedException" type="java.lang.CloneNotSupportedException"/>
+    </method>
+    <doc>
+    <![CDATA[A byte sequence that is used as a Java native type for buffer.
+ It is resizable and distinguishes between the count of the sequence and
+ the current capacity.
+ 
+ @deprecated Replaced by <a href="http://avro.apache.org/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.Buffer -->
+  <!-- start class org.apache.hadoop.record.CsvRecordInput -->
+  <class name="CsvRecordInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordInput"/>
+    <constructor name="CsvRecordInput" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of CsvRecordInput]]>
+      </doc>
+    </constructor>
+    <method name="readByte" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBool" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBuffer" return="org.apache.hadoop.record.Buffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.CsvRecordInput -->
+  <!-- start class org.apache.hadoop.record.CsvRecordOutput -->
+  <class name="CsvRecordOutput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordOutput"/>
+    <constructor name="CsvRecordOutput" type="java.io.OutputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of CsvRecordOutput]]>
+      </doc>
+    </constructor>
+    <method name="writeByte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="boolean"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="long"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="float"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="d" type="double"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="org.apache.hadoop.record.Buffer"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.CsvRecordOutput -->
+  <!-- start interface org.apache.hadoop.record.Index -->
+  <interface name="Index"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="done" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface that acts as an iterator for deserializing maps.
+ The deserializer returns an instance that the record uses to
+ read vectors and maps. An example of usage is as follows:
+
+ <code>
+ Index idx = startVector(...);
+ while (!idx.done()) {
+   .... // read element of a vector
+   idx.incr();
+ }
+ </code>
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.record.Index -->
+  <!-- start class org.apache.hadoop.record.Record -->
+  <class name="Record" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="Record"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rout" type="org.apache.hadoop.record.RecordOutput"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a record with tag (ususally field name)
+ @param rout Record output destination
+ @param tag record tag (Used only in tagged serialization e.g. XML)]]>
+      </doc>
+    </method>
+    <method name="deserialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rin" type="org.apache.hadoop.record.RecordInput"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize a record with a tag (usually field name)
+ @param rin Record input source
+ @param tag Record tag (Used only in tagged serialization e.g. XML)]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="peer" type="java.lang.Object"/>
+      <exception name="ClassCastException" type="java.lang.ClassCastException"/>
+    </method>
+    <method name="serialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rout" type="org.apache.hadoop.record.RecordOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a record without a tag
+ @param rout Record output destination]]>
+      </doc>
+    </method>
+    <method name="deserialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rin" type="org.apache.hadoop.record.RecordInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize a record without a tag
+ @param rin Record input source]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="din" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Abstract class that is extended by generated classes.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.Record -->
+  <!-- start class org.apache.hadoop.record.RecordComparator -->
+  <class name="RecordComparator" extends="org.apache.hadoop.io.WritableComparator"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="RecordComparator" type="java.lang.Class"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a raw {@link Record} comparison implementation.]]>
+      </doc>
+    </constructor>
+    <method name="compare" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+    </method>
+    <method name="define"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="comparator" type="org.apache.hadoop.record.RecordComparator"/>
+      <doc>
+      <![CDATA[Register an optimized comparator for a {@link Record} implementation.
+
+ @param c record classs for which a raw comparator is provided
+ @param comparator Raw comparator instance for class c]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A raw record comparator base class
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.RecordComparator -->
+  <!-- start interface org.apache.hadoop.record.RecordInput -->
+  <interface name="RecordInput"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="readByte" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a byte from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readBool" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a boolean from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read an integer from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a long integer from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a single-precision float from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a double-precision number from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF-8 encoded string from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="readBuffer" return="org.apache.hadoop.record.Buffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read byte array from serialized record.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return value read from serialized record.]]>
+      </doc>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for start of the serialized record.
+ @param tag Used by tagged serialization formats (such as XML)]]>
+      </doc>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for end of the serialized record.
+ @param tag Used by tagged serialization formats (such as XML)]]>
+      </doc>
+    </method>
+    <method name="startVector" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for start of the serialized vector.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return Index that is used to count the number of elements.]]>
+      </doc>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for end of the serialized vector.
+ @param tag Used by tagged serialization formats (such as XML)]]>
+      </doc>
+    </method>
+    <method name="startMap" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for start of the serialized map.
+ @param tag Used by tagged serialization formats (such as XML)
+ @return Index that is used to count the number of map entries.]]>
+      </doc>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check the mark for end of the serialized map.
+ @param tag Used by tagged serialization formats (such as XML)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface that all the Deserializers have to implement.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.record.RecordInput -->
+  <!-- start interface org.apache.hadoop.record.RecordOutput -->
+  <interface name="RecordOutput"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="writeByte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a byte to serialized record.
+ @param b Byte to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeBool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="boolean"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a boolean to serialized record.
+ @param b Boolean to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write an integer to serialized record.
+ @param i Integer to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="long"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a long integer to serialized record.
+ @param l Long to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="float"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a single-precision float to serialized record.
+ @param f Float to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="d" type="double"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a double precision floating point number to serialized record.
+ @param d Double to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a unicode string to serialized record.
+ @param s String to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="writeBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="org.apache.hadoop.record.Buffer"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a buffer to serialized record.
+ @param buf Buffer to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the start of a record to be serialized.
+ @param r Record to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the end of a serialized record.
+ @param r Record to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="startVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the start of a vector to be serialized.
+ @param v Vector to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the end of a serialized vector.
+ @param v Vector to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="startMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="m" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the start of a map to be serialized.
+ @param m Map to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="m" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark the end of a serialized map.
+ @param m Map to be serialized
+ @param tag Used by tagged serialization formats (such as XML)
+ @throws IOException Indicates error in serialization]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface that all the serializers have to implement.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.record.RecordOutput -->
+  <!-- start class org.apache.hadoop.record.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://avro.apache.org/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a float from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a double from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a byte array and returns it.
+ @param bytes byte array with decode long
+ @param start starting index
+ @throws java.io.IOException
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a byte array and returns it.
+ @param bytes byte array with the encoded integer
+ @param start start index
+ @throws java.io.IOException
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a stream and return it.
+ @param in input stream
+ @throws java.io.IOException
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a stream and returns it.
+ @param in input stream
+ @throws java.io.IOException
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <method name="getVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="long"/>
+      <doc>
+      <![CDATA[Get the encoded length if an integer is stored in a variable-length format
+ @return the encoded length]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes a long to a binary stream with zero-compressed encoding.
+ For {@literal -112 <= i <= 127}, only one byte is used with the actual
+ value. For other values of i, the first byte value indicates whether the
+ long is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -120, the following long
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -128, the following long
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+
+ @param stream Binary output stream
+ @param i Long to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes an int to a binary stream with zero-compressed encoding.
+
+ @param stream Binary output stream
+ @param i int to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="compareBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Lexicographic order of binary data.]]>
+      </doc>
+    </method>
+    <field name="hexchars" type="char[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Various utility functions for Hadoop record I/O runtime.
+ 
+ @deprecated Replaced by <a href="http://avro.apache.org/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.Utils -->
+  <!-- start class org.apache.hadoop.record.XmlRecordInput -->
+  <class name="XmlRecordInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordInput"/>
+    <constructor name="XmlRecordInput" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of XmlRecordInput]]>
+      </doc>
+    </constructor>
+    <method name="readByte" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBool" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readBuffer" return="org.apache.hadoop.record.Buffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap" return="org.apache.hadoop.record.Index"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[XML Deserializer.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.XmlRecordInput -->
+  <!-- start class org.apache.hadoop.record.XmlRecordOutput -->
+  <class name="XmlRecordOutput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.RecordOutput"/>
+    <constructor name="XmlRecordOutput" type="java.io.OutputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of XmlRecordOutput]]>
+      </doc>
+    </constructor>
+    <method name="writeByte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="boolean"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="long"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="float"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="d" type="double"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="org.apache.hadoop.record.Buffer"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.record.Record"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.ArrayList"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="endMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="v" type="java.util.TreeMap"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[XML Serializer.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.XmlRecordOutput -->
+  <doc>
+  <![CDATA[<p>
+    (DEPRECATED) Hadoop record I/O contains classes and a record description language
+    translator for simplifying serialization and deserialization of records in a
+    language-neutral manner.
+    </p>
+    
+    <p>
+    DEPRECATED: Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
+    </p>
+  
+  <h2>Introduction</h2>
+  
+  Software systems of any significant complexity require mechanisms for data 
+interchange with the outside world. These interchanges typically involve the
+marshaling and unmarshaling of logical units of data to and from data streams
+(files, network connections, memory buffers etc.). Applications usually have
+some code for serializing and deserializing the data types that they manipulate
+embedded in them. The work of serialization has several features that make
+automatic code generation for it worthwhile. Given a particular output encoding
+(binary, XML, etc.), serialization of primitive types and simple compositions
+of primitives (structs, vectors etc.) is a very mechanical task. Manually
+written serialization code can be susceptible to bugs especially when records
+have a large number of fields or a record definition changes between software
+versions. Lastly, it can be very useful for applications written in different
+programming languages to be able to share and interchange data. This can be 
+made a lot easier by describing the data records manipulated by these
+applications in a language agnostic manner and using the descriptions to derive
+implementations of serialization in multiple target languages. 
+
+This document describes Hadoop Record I/O, a mechanism that is aimed 
+at
+<ul> 
+<li> enabling the specification of simple serializable data types (records) 
+<li> enabling the generation of code in multiple target languages for
+marshaling and unmarshaling such types
+<li> providing target language specific support that will enable application 
+programmers to incorporate generated code into their applications
+</ul>
+
+The goals of Hadoop Record I/O are similar to those of mechanisms such as XDR,
+ASN.1, PADS and ICE. While these systems all include a DDL that enables
+the specification of most record types, they differ widely in what else they
+focus on. The focus in Hadoop Record I/O is on data marshaling and
+multi-lingual support.  We take a translator-based approach to serialization.
+Hadoop users have to describe their data in a simple data description
+language. The Hadoop DDL translator rcc generates code that users
+can invoke in order to read/write their data from/to simple stream 
+abstractions. Next we list explicitly some of the goals and non-goals of
+Hadoop Record I/O.
+
+
+<h3>Goals</h3>
+
+<ul>
+<li> Support for commonly used primitive types. Hadoop should include as
+primitives commonly used builtin types from programming languages we intend to
+support.
+
+<li> Support for common data compositions (including recursive compositions).
+Hadoop should support widely used composite types such as structs and
+vectors.
+
+<li> Code generation in multiple target languages. Hadoop should be capable of
+generating serialization code in multiple target languages and should be
+easily extensible to new target languages. The initial target languages are
+C++ and Java.
+
+<li> Support for generated target languages. Hadooop should include support
+in the form of headers, libraries, packages for supported target languages 
+that enable easy inclusion and use of generated code in applications.
+
+<li> Support for multiple output encodings. Candidates include
+packed binary, comma-separated text, XML etc.
+
+<li> Support for specifying record types in a backwards/forwards compatible
+manner. This will probably be in the form of support for optional fields in
+records. This version of the document does not include a description of the
+planned mechanism, we intend to include it in the next iteration.
+
+</ul>
+
+<h3>Non-Goals</h3>
+
+<ul>
+  <li> Serializing existing arbitrary C++ classes.
+  <li> Serializing complex data structures such as trees, linked lists etc.
+  <li> Built-in indexing schemes, compression, or check-sums.
+  <li> Dynamic construction of objects from an XML schema.
+</ul>
+
+The remainder of this document describes the features of Hadoop record I/O
+in more detail. Section 2 describes the data types supported by the system.
+Section 3 lays out the DDL syntax with some examples of simple records. 
+Section 4 describes the process of code generation with rcc. Section 5
+describes target language mappings and support for Hadoop types. We include a
+fairly complete description of C++ mappings with intent to include Java and
+others in upcoming iterations of this document. The last section talks about
+supported output encodings.
+
+
+<h2>Data Types and Streams</h2>
+
+This section describes the primitive and composite types supported by Hadoop.
+We aim to support a set of types that can be used to simply and efficiently
+express a wide range of record types in different programming languages.
+
+<h3>Primitive Types</h3>
+
+For the most part, the primitive types of Hadoop map directly to primitive
+types in high level programming languages. Special cases are the
+ustring (a Unicode string) and buffer types, which we believe
+find wide use and which are usually implemented in library code and not
+available as language built-ins. Hadoop also supplies these via library code
+when a target language built-in is not present and there is no widely
+adopted "standard" implementation. The complete list of primitive types is:
+
+<ul>
+  <li> byte: An 8-bit unsigned integer.
+  <li> boolean: A boolean value.
+  <li> int: A 32-bit signed integer.
+  <li> long: A 64-bit signed integer.
+  <li> float: A single precision floating point number as described by
+    IEEE-754.
+  <li> double: A double precision floating point number as described by
+    IEEE-754.
+  <li> ustring: A string consisting of Unicode characters.
+  <li> buffer: An arbitrary sequence of bytes. 
+</ul>
+
+
+<h3>Composite Types</h3>
+Hadoop supports a small set of composite types that enable the description
+of simple aggregate types and containers. A composite type is serialized
+by sequentially serializing it constituent elements. The supported
+composite types are:
+
+<ul>
+
+  <li> record: An aggregate type like a C-struct. This is a list of
+typed fields that are together considered a single unit of data. A record
+is serialized by sequentially serializing its constituent fields. In addition
+to serialization a record has comparison operations (equality and less-than)
+implemented for it, these are defined as memberwise comparisons.
+
+  <li>vector: A sequence of entries of the same data type, primitive
+or composite.
+
+  <li> map: An associative container mapping instances of a key type to
+instances of a value type. The key and value types may themselves be primitive
+or composite types. 
+
+</ul>
+
+<h3>Streams</h3>
+
+Hadoop generates code for serializing and deserializing record types to
+abstract streams. For each target language Hadoop defines very simple input
+and output stream interfaces. Application writers can usually develop
+concrete implementations of these by putting a one method wrapper around
+an existing stream implementation.
+
+
+<h2>DDL Syntax and Examples</h2>
+
+We now describe the syntax of the Hadoop data description language. This is
+followed by a few examples of DDL usage.
+ 
+<h3>Hadoop DDL Syntax</h3>
+
+<pre><code>
+recfile = *include module *record
+include = "include" path
+path = (relative-path / absolute-path)
+module = "module" module-name
+module-name = name *("." name)
+record := "class" name "{" 1*(field) "}"
+field := type name ";"
+name :=  ALPHA (ALPHA / DIGIT / "_" )*
+type := (ptype / ctype)
+ptype := ("byte" / "boolean" / "int" |
+          "long" / "float" / "double"
+          "ustring" / "buffer")
+ctype := (("vector" "<" type ">") /
+          ("map" "<" type "," type ">" ) ) / name)
+</code></pre>
+
+A DDL file describes one or more record types. It begins with zero or
+more include declarations, a single mandatory module declaration
+followed by zero or more class declarations. The semantics of each of
+these declarations are described below:
+
+<ul>
+
+<li>include: An include declaration specifies a DDL file to be
+referenced when generating code for types in the current DDL file. Record types
+in the current compilation unit may refer to types in all included files.
+File inclusion is recursive. An include does not trigger code
+generation for the referenced file.
+
+<li> module: Every Hadoop DDL file must have a single module
+declaration that follows the list of includes and precedes all record
+declarations. A module declaration identifies a scope within which
+the names of all types in the current file are visible. Module names are
+mapped to C++ namespaces, Java packages etc. in generated code.
+
+<li> class: Records types are specified through class
+declarations. A class declaration is like a Java class declaration.
+It specifies a named record type and a list of fields that constitute records
+of the type. Usage is illustrated in the following examples.
+
+</ul>
+
+<h3>Examples</h3>
+
+<ul>
+<li>A simple DDL file links.jr with just one record declaration. 
+<pre><code>
+module links {
+    class Link {
+        ustring URL;
+        boolean isRelative;
+        ustring anchorText;
+    };
+}
+</code></pre>
+
+<li> A DDL file outlinks.jr which includes another
+<pre><code>
+include "links.jr"
+
+module outlinks {
+    class OutLinks {
+        ustring baseURL;
+        vector<links.Link> outLinks;
+    };
+}
+</code></pre>
+</ul>
+
+<h2>Code Generation</h2>
+
+The Hadoop translator is written in Java. Invocation is done by executing a 
+wrapper shell script named named rcc. It takes a list of
+record description files as a mandatory argument and an
+optional language argument (the default is Java) --language or
+-l. Thus a typical invocation would look like:
+<pre><code>
+$ rcc -l C++ <filename> ...
+</code></pre>
+
+
+<h2>Target Language Mappings and Support</h2>
+
+For all target languages, the unit of code generation is a record type. 
+For each record type, Hadoop generates code for serialization and
+deserialization, record comparison and access to record members.
+
+<h3>C++</h3>
+
+Support for including Hadoop generated C++ code in applications comes in the
+form of a header file recordio.hh which needs to be included in source
+that uses Hadoop types and a library librecordio.a which applications need
+to be linked with. The header declares the Hadoop C++ namespace which defines
+appropriate types for the various primitives, the basic interfaces for
+records and streams and enumerates the supported serialization encodings.
+Declarations of these interfaces and a description of their semantics follow:
+
+<pre><code>
+namespace hadoop {
+
+  enum RecFormat { kBinary, kXML, kCSV };
+
+  class InStream {
+  public:
+    virtual ssize_t read(void *buf, size_t n) = 0;
+  };
+
+  class OutStream {
+  public:
+    virtual ssize_t write(const void *buf, size_t n) = 0;
+  };
+
+  class IOError : public runtime_error {
+  public:
+    explicit IOError(const std::string& msg);
+  };
+
+  class IArchive;
+  class OArchive;
+
+  class RecordReader {
+  public:
+    RecordReader(InStream& in, RecFormat fmt);
+    virtual ~RecordReader(void);
+
+    virtual void read(Record& rec);
+  };
+
+  class RecordWriter {
+  public:
+    RecordWriter(OutStream& out, RecFormat fmt);
+    virtual ~RecordWriter(void);
+
+    virtual void write(Record& rec);
+  };
+
+
+  class Record {
+  public:
+    virtual std::string type(void) const = 0;
+    virtual std::string signature(void) const = 0;
+  protected:
+    virtual bool validate(void) const = 0;
+
+    virtual void
+    serialize(OArchive& oa, const std::string& tag) const = 0;
+
+    virtual void
+    deserialize(IArchive& ia, const std::string& tag) = 0;
+  };
+}
+</code></pre>
+
+<ul>
+
+<li> RecFormat: An enumeration of the serialization encodings supported
+by this implementation of Hadoop.
+
+<li> InStream: A simple abstraction for an input stream. This has a 
+single public read method that reads n bytes from the stream into
+the buffer buf. Has the same semantics as a blocking read system
+call. Returns the number of bytes read or -1 if an error occurs.
+
+<li> OutStream: A simple abstraction for an output stream. This has a 
+single write method that writes n bytes to the stream from the
+buffer buf. Has the same semantics as a blocking write system
+call. Returns the number of bytes written or -1 if an error occurs.
+
+<li> RecordReader: A RecordReader reads records one at a time from
+an underlying stream in a specified record format. The reader is instantiated
+with a stream and a serialization format. It has a read method that
+takes an instance of a record and deserializes the record from the stream.
+
+<li> RecordWriter: A RecordWriter writes records one at a
+time to an underlying stream in a specified record format. The writer is
+instantiated with a stream and a serialization format. It has a
+write method that takes an instance of a record and serializes the
+record to the stream.
+
+<li> Record: The base class for all generated record types. This has two
+public methods type and signature that return the typename and the
+type signature of the record.
+
+</ul>
+
+Two files are generated for each record file (note: not for each record). If a
+record file is named "name.jr", the generated files are 
+"name.jr.cc" and "name.jr.hh" containing serialization 
+implementations and record type declarations respectively.
+
+For each record in the DDL file, the generated header file will contain a
+class definition corresponding to the record type, method definitions for the
+generated type will be present in the '.cc' file.  The generated class will
+inherit from the abstract class hadoop::Record. The DDL files
+module declaration determines the namespace the record belongs to.
+Each '.' delimited token in the module declaration results in the
+creation of a namespace. For instance, the declaration module docs.links
+results in the creation of a docs namespace and a nested 
+docs::links namespace. In the preceding examples, the Link class
+is placed in the links namespace. The header file corresponding to
+the links.jr file will contain:
+
+<pre><code>
+namespace links {
+  class Link : public hadoop::Record {
+    // ....
+  };
+};
+</code></pre>
+
+Each field within the record will cause the generation of a private member
+declaration of the appropriate type in the class declaration, and one or more
+acccessor methods. The generated class will implement the serialize and
+deserialize methods defined in hadoop::Record+. It will also 
+implement the inspection methods type and signature from
+hadoop::Record. A default constructor and virtual destructor will also
+be generated. Serialization code will read/write records into streams that
+implement the hadoop::InStream and the hadoop::OutStream interfaces.
+
+For each member of a record an accessor method is generated that returns 
+either the member or a reference to the member. For members that are returned 
+by value, a setter method is also generated. This is true for primitive 
+data members of the types byte, int, long, boolean, float and 
+double. For example, for a int field called MyField the folowing
+code is generated.
+
+<pre><code>
+...
+private:
+  int32_t mMyField;
+  ...
+public:
+  int32_t getMyField(void) const {
+    return mMyField;
+  };
+
+  void setMyField(int32_t m) {
+    mMyField = m;
+  };
+  ...
+</code></pre>
+
+For a ustring or buffer or composite field. The generated code
+only contains accessors that return a reference to the field. A const
+and a non-const accessor are generated. For example:
+
+<pre><code>
+...
+private:
+  std::string mMyBuf;
+  ...
+public:
+
+  std::string& getMyBuf() {
+    return mMyBuf;
+  };
+
+  const std::string& getMyBuf() const {
+    return mMyBuf;
+  };
+  ...
+</code></pre>
+
+<h4>Examples</h4>
+
+Suppose the inclrec.jr file contains:
+<pre><code>
+module inclrec {
+    class RI {
+        int      I32;
+        double   D;
+        ustring  S;
+    };
+}
+</code></pre>
+
+and the testrec.jr file contains:
+
+<pre><code>
+include "inclrec.jr"
+module testrec {
+    class R {
+        vector<float> VF;
+        RI            Rec;
+        buffer        Buf;
+    };
+}
+</code></pre>
+
+Then the invocation of rcc such as:
+<pre><code>
+$ rcc -l c++ inclrec.jr testrec.jr
+</code></pre>
+will result in generation of four files:
+inclrec.jr.{cc,hh} and testrec.jr.{cc,hh}.
+
+The inclrec.jr.hh will contain:
+
+<pre><code>
+#ifndef _INCLREC_JR_HH_
+#define _INCLREC_JR_HH_
+
+#include "recordio.hh"
+
+namespace inclrec {
+  
+  class RI : public hadoop::Record {
+
+  private:
+
+    int32_t      I32;
+    double       D;
+    std::string  S;
+
+  public:
+
+    RI(void);
+    virtual ~RI(void);
+
+    virtual bool operator==(const RI& peer) const;
+    virtual bool operator<(const RI& peer) const;
+
+    virtual int32_t getI32(void) const { return I32; }
+    virtual void setI32(int32_t v) { I32 = v; }
+
+    virtual double getD(void) const { return D; }
+    virtual void setD(double v) { D = v; }
+
+    virtual std::string& getS(void) const { return S; }
+    virtual const std::string& getS(void) const { return S; }
+
+    virtual std::string type(void) const;
+    virtual std::string signature(void) const;
+
+  protected:
+
+    virtual void serialize(hadoop::OArchive& a) const;
+    virtual void deserialize(hadoop::IArchive& a);
+  };
+} // end namespace inclrec
+
+#endif /* _INCLREC_JR_HH_ */
+
+</code></pre>
+
+The testrec.jr.hh file will contain:
+
+
+<pre><code>
+
+#ifndef _TESTREC_JR_HH_
+#define _TESTREC_JR_HH_
+
+#include "inclrec.jr.hh"
+
+namespace testrec {
+  class R : public hadoop::Record {
+
+  private:
+
+    std::vector<float> VF;
+    inclrec::RI        Rec;
+    std::string        Buf;
+
+  public:
+
+    R(void);
+    virtual ~R(void);
+
+    virtual bool operator==(const R& peer) const;
+    virtual bool operator<(const R& peer) const;
+
+    virtual std::vector<float>& getVF(void) const;
+    virtual const std::vector<float>& getVF(void) const;
+
+    virtual std::string& getBuf(void) const ;
+    virtual const std::string& getBuf(void) const;
+
+    virtual inclrec::RI& getRec(void) const;
+    virtual const inclrec::RI& getRec(void) const;
+    
+    virtual bool serialize(hadoop::OutArchive& a) const;
+    virtual bool deserialize(hadoop::InArchive& a);
+    
+    virtual std::string type(void) const;
+    virtual std::string signature(void) const;
+  };
+}; // end namespace testrec
+#endif /* _TESTREC_JR_HH_ */
+
+</code></pre>
+
+<h3>Java</h3>
+
+Code generation for Java is similar to that for C++. A Java class is generated
+for each record type with private members corresponding to the fields. Getters
+and setters for fields are also generated. Some differences arise in the
+way comparison is expressed and in the mapping of modules to packages and
+classes to files. For equality testing, an equals method is generated
+for each record type. As per Java requirements a hashCode method is also
+generated. For comparison a compareTo method is generated for each
+record type. This has the semantics as defined by the Java Comparable
+interface, that is, the method returns a negative integer, zero, or a positive
+integer as the invoked object is less than, equal to, or greater than the
+comparison parameter.
+
+A .java file is generated per record type as opposed to per DDL
+file as in C++. The module declaration translates to a Java
+package declaration. The module name maps to an identical Java package
+name. In addition to this mapping, the DDL compiler creates the appropriate
+directory hierarchy for the package and places the generated .java
+files in the correct directories.
+
+<h2>Mapping Summary</h2>
+
+<pre><code>
+DDL Type        C++ Type            Java Type 
+
+boolean         bool                boolean
+byte            int8_t              byte
+int             int32_t             int
+long            int64_t             long
+float           float               float
+double          double              double
+ustring         std::string         java.lang.String
+buffer          std::string         org.apache.hadoop.record.Buffer
+class type      class type          class type
+vector<type>    std::vector<type>   java.util.ArrayList<type>
+map<type,type>  std::map<type,type> java.util.TreeMap<type,type>
+</code></pre>
+
+<h2>Data encodings</h2>
+
+This section describes the format of the data encodings supported by Hadoop.
+Currently, three data encodings are supported, namely binary, CSV and XML.
+
+<h3>Binary Serialization Format</h3>
+
+The binary data encoding format is fairly dense. Serialization of composite
+types is simply defined as a concatenation of serializations of the constituent
+elements (lengths are included in vectors and maps).
+
+Composite types are serialized as follows:
+<ul>
+<li> class: Sequence of serialized members.
+<li> vector: The number of elements serialized as an int. Followed by a
+sequence of serialized elements.
+<li> map: The number of key value pairs serialized as an int. Followed
+by a sequence of serialized (key,value) pairs.
+</ul>
+
+Serialization of primitives is more interesting, with a zero compression
+optimization for integral types and normalization to UTF-8 for strings. 
+Primitive types are serialized as follows:
+
+<ul>
+<li> byte: Represented by 1 byte, as is.
+<li> boolean: Represented by 1-byte (0 or 1)
+<li> int/long: Integers and longs are serialized zero compressed.
+Represented as 1-byte if -120 <= value < 128. Otherwise, serialized as a
+sequence of 2-5 bytes for ints, 2-9 bytes for longs. The first byte represents
+the number of trailing bytes, N, as the negative number (-120-N). For example,
+the number 1024 (0x400) is represented by the byte sequence 'x86 x04 x00'.
+This doesn't help much for 4-byte integers but does a reasonably good job with
+longs without bit twiddling.
+<li> float/double: Serialized in IEEE 754 single and double precision
+format in network byte order. This is the format used by Java.
+<li> ustring: Serialized as 4-byte zero compressed length followed by
+data encoded as UTF-8. Strings are normalized to UTF-8 regardless of native
+language representation.
+<li> buffer: Serialized as a 4-byte zero compressed length followed by the
+raw bytes in the buffer.
+</ul>
+
+
+<h3>CSV Serialization Format</h3>
+
+The CSV serialization format has a lot more structure than the "standard"
+Excel CSV format, but we believe the additional structure is useful because
+
+<ul>
+<li> it makes parsing a lot easier without detracting too much from legibility
+<li> the delimiters around composites make it obvious when one is reading a
+sequence of Hadoop records
+</ul>
+
+Serialization formats for the various types are detailed in the grammar that
+follows. The notable feature of the formats is the use of delimiters for 
+indicating the certain field types.
+
+<ul>
+<li> A string field begins with a single quote (').
+<li> A buffer field begins with a sharp (#).
+<li> A class, vector or map begins with 's{', 'v{' or 'm{' respectively and
+ends with '}'.
+</ul>
+
+The CSV format can be described by the following grammar:
+
+<pre><code>
+record = primitive / struct / vector / map
+primitive = boolean / int / long / float / double / ustring / buffer
+
+boolean = "T" / "F"
+int = ["-"] 1*DIGIT
+long = ";" ["-"] 1*DIGIT
+float = ["-"] 1*DIGIT "." 1*DIGIT ["E" / "e" ["-"] 1*DIGIT]
+double = ";" ["-"] 1*DIGIT "." 1*DIGIT ["E" / "e" ["-"] 1*DIGIT]
+
+ustring = "'" *(UTF8 char except NULL, LF, % and , / "%00" / "%0a" / "%25" / "%2c" )
+
+buffer = "#" *(BYTE except NULL, LF, % and , / "%00" / "%0a" / "%25" / "%2c" )
+
+struct = "s{" record *("," record) "}"
+vector = "v{" [record *("," record)] "}"
+map = "m{" [*(record "," record)] "}"
+</code></pre>
+
+<h3>XML Serialization Format</h3>
+
+The XML serialization format is the same used by Apache XML-RPC
+(http://ws.apache.org/xmlrpc/types.html). This is an extension of the original
+XML-RPC format and adds some additional data types. All record I/O types are
+not directly expressible in this format, and access to a DDL is required in
+order to convert these to valid types. All types primitive or composite are
+represented by &lt;value&gt; elements. The particular XML-RPC type is
+indicated by a nested element in the &lt;value&gt; element. The encoding for
+records is always UTF-8. Primitive types are serialized as follows:
+
+<ul>
+<li> byte: XML tag &lt;ex:i1&gt;. Values: 1-byte unsigned 
+integers represented in US-ASCII
+<li> boolean: XML tag &lt;boolean&gt;. Values: "0" or "1"
+<li> int: XML tags &lt;i4&gt; or &lt;int&gt;. Values: 4-byte
+signed integers represented in US-ASCII.
+<li> long: XML tag &lt;ex:i8&gt;. Values: 8-byte signed integers
+represented in US-ASCII.
+<li> float: XML tag &lt;ex:float&gt;. Values: Single precision
+floating point numbers represented in US-ASCII.
+<li> double: XML tag &lt;double&gt;. Values: Double precision
+floating point numbers represented in US-ASCII.
+<li> ustring: XML tag &lt;;string&gt;. Values: String values
+represented as UTF-8. XML does not permit all Unicode characters in literal
+data. In particular, NULLs and control chars are not allowed. Additionally,
+XML processors are required to replace carriage returns with line feeds and to
+replace CRLF sequences with line feeds. Programming languages that we work
+with do not impose these restrictions on string types. To work around these
+restrictions, disallowed characters and CRs are percent escaped in strings.
+The '%' character is also percent escaped.
+<li> buffer: XML tag &lt;string&&gt;. Values: Arbitrary binary
+data. Represented as hexBinary, each byte is replaced by its 2-byte
+hexadecimal representation.
+</ul>
+
+Composite types are serialized as follows:
+
+<ul>
+<li> class: XML tag &lt;struct&gt;. A struct is a sequence of
+&lt;member&gt; elements. Each &lt;member&gt; element has a &lt;name&gt;
+element and a &lt;value&gt; element. The &lt;name&gt; is a string that must
+match /[a-zA-Z][a-zA-Z0-9_]*/. The value of the member is represented
+by a &lt;value&gt; element.
+
+<li> vector: XML tag &lt;array&lt;. An &lt;array&gt; contains a
+single &lt;data&gt; element. The &lt;data&gt; element is a sequence of
+&lt;value&gt; elements each of which represents an element of the vector.
+
+<li> map: XML tag &lt;array&gt;. Same as vector.
+
+</ul>
+
+For example:
+
+<pre><code>
+class {
+  int           MY_INT;            // value 5
+  vector<float> MY_VEC;            // values 0.1, -0.89, 2.45e4
+  buffer        MY_BUF;            // value '\00\n\tabc%'
+}
+</code></pre>
+
+is serialized as
+
+<pre><code class="XML">
+&lt;value&gt;
+  &lt;struct&gt;
+    &lt;member&gt;
+      &lt;name&gt;MY_INT&lt;/name&gt;
+      &lt;value&gt;&lt;i4&gt;5&lt;/i4&gt;&lt;/value&gt;
+    &lt;/member&gt;
+    &lt;member&gt;
+      &lt;name&gt;MY_VEC&lt;/name&gt;
+      &lt;value&gt;
+        &lt;array&gt;
+          &lt;data&gt;
+            &lt;value&gt;&lt;ex:float&gt;0.1&lt;/ex:float&gt;&lt;/value&gt;
+            &lt;value&gt;&lt;ex:float&gt;-0.89&lt;/ex:float&gt;&lt;/value&gt;
+            &lt;value&gt;&lt;ex:float&gt;2.45e4&lt;/ex:float&gt;&lt;/value&gt;
+          &lt;/data&gt;
+        &lt;/array&gt;
+      &lt;/value&gt;
+    &lt;/member&gt;
+    &lt;member&gt;
+      &lt;name&gt;MY_BUF&lt;/name&gt;
+      &lt;value&gt;&lt;string&gt;%00\n\tabc%25&lt;/string&gt;&lt;/value&gt;
+    &lt;/member&gt;
+  &lt;/struct&gt;
+&lt;/value&gt; 
+</code></pre>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.record.compiler">
+  <!-- start class org.apache.hadoop.record.compiler.CodeBuffer -->
+  <class name="CodeBuffer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A wrapper around StringBuffer that automatically does indentation
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.CodeBuffer -->
+  <!-- start class org.apache.hadoop.record.compiler.Consts -->
+  <class name="Consts" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <field name="RIO_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RTI_VAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RTI_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RTI_FILTER_FIELDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECORD_OUTPUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECORD_INPUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TAG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[const definitions for Record I/O compiler
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.Consts -->
+  <!-- start class org.apache.hadoop.record.compiler.JBoolean -->
+  <class name="JBoolean" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JBoolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JBoolean]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JBoolean -->
+  <!-- start class org.apache.hadoop.record.compiler.JBuffer -->
+  <class name="JBuffer" extends="org.apache.hadoop.record.compiler.JCompType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JBuffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JBuffer]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Code generator for "buffer" type.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JBuffer -->
+  <!-- start class org.apache.hadoop.record.compiler.JByte -->
+  <class name="JByte" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JByte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Code generator for "byte" type.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JByte -->
+  <!-- start class org.apache.hadoop.record.compiler.JDouble -->
+  <class name="JDouble" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JDouble"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JDouble]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JDouble -->
+  <!-- start class org.apache.hadoop.record.compiler.JField -->
+  <class name="JField" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JField" type="java.lang.String, T"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JField]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A thin wrappper around record field.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JField -->
+  <!-- start class org.apache.hadoop.record.compiler.JFile -->
+  <class name="JFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JFile" type="java.lang.String, java.util.ArrayList, java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JFile
+
+ @param name possibly full pathname to the file
+ @param inclFiles included files (as JFile)
+ @param recList List of records defined within this file]]>
+      </doc>
+    </constructor>
+    <method name="genCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="language" type="java.lang.String"/>
+      <param name="destDir" type="java.lang.String"/>
+      <param name="options" type="java.util.ArrayList"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Generate record code in given language. Language should be all
+  lowercase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Container for the Hadoop Record DDL.
+ The main components of the file are filename, list of included files,
+ and records defined in that file.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JFile -->
+  <!-- start class org.apache.hadoop.record.compiler.JFloat -->
+  <class name="JFloat" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JFloat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JFloat]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JFloat -->
+  <!-- start class org.apache.hadoop.record.compiler.JInt -->
+  <class name="JInt" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JInt"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JInt]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Code generator for "int" type
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JInt -->
+  <!-- start class org.apache.hadoop.record.compiler.JLong -->
+  <class name="JLong" extends="org.apache.hadoop.record.compiler.JType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JLong"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JLong]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Code generator for "long" type
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JLong -->
+  <!-- start class org.apache.hadoop.record.compiler.JMap -->
+  <class name="JMap" extends="org.apache.hadoop.record.compiler.JCompType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JMap" type="org.apache.hadoop.record.compiler.JType, org.apache.hadoop.record.compiler.JType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JMap]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JMap -->
+  <!-- start class org.apache.hadoop.record.compiler.JRecord -->
+  <class name="JRecord" extends="org.apache.hadoop.record.compiler.JCompType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JRecord" type="java.lang.String, java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JRecord]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JRecord -->
+  <!-- start class org.apache.hadoop.record.compiler.JString -->
+  <class name="JString" extends="org.apache.hadoop.record.compiler.JCompType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JString"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JString]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JString -->
+  <!-- start class org.apache.hadoop.record.compiler.JType -->
+  <class name="JType" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Abstract Base class for all types supported by Hadoop Record I/O.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JType -->
+  <!-- start class org.apache.hadoop.record.compiler.JVector -->
+  <class name="JVector" extends="org.apache.hadoop.record.compiler.JCompType"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="JVector" type="org.apache.hadoop.record.compiler.JType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of JVector]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.JVector -->
+  <doc>
+  <![CDATA[<p>
+    (DEPRECATED) This package contains classes needed for code generation
+    from the hadoop record compiler. CppGenerator and JavaGenerator
+    are the main entry points from the parser. There are classes
+    corrsponding to every primitive type and compound type
+    included in Hadoop record I/O syntax.
+    </p>
+    
+    <p>
+    DEPRECATED: Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
+    </p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.record.compiler.ant">
+  <!-- start class org.apache.hadoop.record.compiler.ant.RccTask -->
+  <class name="RccTask" extends="org.apache.tools.ant.Task"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="RccTask"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of RccTask]]>
+      </doc>
+    </constructor>
+    <method name="setLanguage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="language" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the output language option
+ @param language "java"/"c++"]]>
+      </doc>
+    </method>
+    <method name="setFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <doc>
+      <![CDATA[Sets the record definition file attribute
+ @param file record definition file]]>
+      </doc>
+    </method>
+    <method name="setFailonerror"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Given multiple files (via fileset), set the error handling behavior
+ @param flag true will throw build exception in case of failure (default)]]>
+      </doc>
+    </method>
+    <method name="setDestdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Sets directory where output files will be generated
+ @param dir output directory]]>
+      </doc>
+    </method>
+    <method name="addFileset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="set" type="org.apache.tools.ant.types.FileSet"/>
+      <doc>
+      <![CDATA[Adds a fileset that can consist of one or more files
+ @param set Set of record definition files]]>
+      </doc>
+    </method>
+    <method name="execute"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="BuildException" type="org.apache.tools.ant.BuildException"/>
+      <doc>
+      <![CDATA[Invoke the Hadoop record compiler on each record definition file]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hadoop record compiler ant Task
+<p> This task takes the given record definition files and compiles them into
+ java or c++
+ files. It is then up to the user to compile the generated files.
+
+ <p> The task requires the <code>file</code> or the nested fileset element to be
+ specified. Optional attributes are <code>language</code> (set the output
+ language, default is "java"),
+ <code>destdir</code> (name of the destination directory for generated java/c++
+ code, default is ".") and <code>failonerror</code> (specifies error handling
+ behavior. default is true).
+ <p><h4>Usage</h4>
+ <pre>
+ &lt;recordcc
+       destdir="${basedir}/gensrc"
+       language="java"&gt;
+   &lt;fileset include="**\/*.jr" /&gt;
+ &lt;/recordcc&gt;
+ </pre>
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.ant.RccTask -->
+</package>
+<package name="org.apache.hadoop.record.compiler.generated">
+  <!-- start class org.apache.hadoop.record.compiler.generated.ParseException -->
+  <class name="ParseException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="ParseException" type="org.apache.hadoop.record.compiler.generated.Token, int[][], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constructor is used by the method "generateParseException"
+ in the generated parser.  Calling this constructor generates
+ a new object of this type with the fields "currentToken",
+ "expectedTokenSequences", and "tokenImage" set.  The boolean
+ flag "specialConstructor" is also set to true to indicate that
+ this constructor was used to create this object.
+ This constructor calls its super class with the empty string
+ to force the "toString" method of parent class "Throwable" to
+ print the error message in the form:
+     ParseException: <result of getMessage>]]>
+      </doc>
+    </constructor>
+    <constructor name="ParseException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The following constructors are for use by you for whatever
+ purpose you can think of.  Constructing the exception in this
+ manner makes the exception behave in the normal way - i.e., as
+ documented in the class "Throwable".  The fields "errorToken",
+ "expectedTokenSequences", and "tokenImage" do not contain
+ relevant information.  The JavaCC generated code does not use
+ these constructors.]]>
+      </doc>
+    </constructor>
+    <constructor name="ParseException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method has the standard behavior when this object has been
+ created using the standard constructors.  Otherwise, it uses
+ "currentToken" and "expectedTokenSequences" to generate a parse
+ error message and returns it.  If this object has been created
+ due to a parse error, and you do not catch it (it gets thrown
+ from the parser), then this method is called during the printing
+ of the final stack trace, and hence the correct error message
+ gets displayed.]]>
+      </doc>
+    </method>
+    <method name="add_escapes" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Used to convert raw characters to their escaped version
+ when these raw version cannot be used as part of an ASCII
+ string literal.]]>
+      </doc>
+    </method>
+    <field name="specialConstructor" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This variable determines which constructor was used to create
+ this object and thereby affects the semantics of the
+ "getMessage" method (see below).]]>
+      </doc>
+    </field>
+    <field name="currentToken" type="org.apache.hadoop.record.compiler.generated.Token"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This is the last token that has been consumed successfully.  If
+ this object has been created due to a parse error, the token
+ followng this token will (therefore) be the first error token.]]>
+      </doc>
+    </field>
+    <field name="expectedTokenSequences" type="int[][]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Each entry in this array is an array of integers.  Each array
+ of integers represents a sequence of tokens (by their ordinal
+ values) that is expected at this point of the parse.]]>
+      </doc>
+    </field>
+    <field name="tokenImage" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This is a reference to the "tokenImage" array of the generated
+ parser within which the parse error occurred.  This array is
+ defined in the generated ...Constants interface.]]>
+      </doc>
+    </field>
+    <field name="eol" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The end of line string for this machine.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This exception is thrown when parse errors are encountered.
+ You can explicitly create objects of this exception type by
+ calling the method generateParseException in the generated
+ parser.
+
+ You can modify this class to customize your error reporting
+ mechanisms so long as you retain the public fields.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.ParseException -->
+  <!-- start class org.apache.hadoop.record.compiler.generated.Rcc -->
+  <class name="Rcc" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.compiler.generated.RccConstants"/>
+    <constructor name="Rcc" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Rcc" type="java.io.InputStream, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Rcc" type="java.io.Reader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Rcc" type="org.apache.hadoop.record.compiler.generated.RccTokenManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <method name="usage"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="driver" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <method name="Input" return="org.apache.hadoop.record.compiler.JFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Include" return="org.apache.hadoop.record.compiler.JFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Module" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="ModuleName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="RecordList" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Record" return="org.apache.hadoop.record.compiler.JRecord"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Field" return="org.apache.hadoop.record.compiler.JField"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Type" return="org.apache.hadoop.record.compiler.JType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Map" return="org.apache.hadoop.record.compiler.JMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="Vector" return="org.apache.hadoop.record.compiler.JVector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ParseException" type="org.apache.hadoop.record.compiler.generated.ParseException"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.InputStream"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.InputStream"/>
+      <param name="encoding" type="java.lang.String"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.Reader"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tm" type="org.apache.hadoop.record.compiler.generated.RccTokenManager"/>
+    </method>
+    <method name="getNextToken" return="org.apache.hadoop.record.compiler.generated.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getToken" return="org.apache.hadoop.record.compiler.generated.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="index" type="int"/>
+    </method>
+    <method name="generateParseException" return="org.apache.hadoop.record.compiler.generated.ParseException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="enable_tracing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="disable_tracing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="token_source" type="org.apache.hadoop.record.compiler.generated.RccTokenManager"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="token" type="org.apache.hadoop.record.compiler.generated.Token"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="jj_nt" type="org.apache.hadoop.record.compiler.generated.Token"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.Rcc -->
+  <!-- start interface org.apache.hadoop.record.compiler.generated.RccConstants -->
+  <interface name="RccConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <field name="EOF" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MODULE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECORD_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INCLUDE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BYTE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BOOLEAN_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOAT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USTRING_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BUFFER_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VECTOR_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAP_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LBRACE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RBRACE_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SEMICOLON_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMMA_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CSTRING_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IDENT_TKN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="WithinOneLineComment" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="WithinMultiLineComment" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="tokenImage" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.record.compiler.generated.RccConstants -->
+  <!-- start class org.apache.hadoop.record.compiler.generated.RccTokenManager -->
+  <class name="RccTokenManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <implements name="org.apache.hadoop.record.compiler.generated.RccConstants"/>
+    <constructor name="RccTokenManager" type="org.apache.hadoop.record.compiler.generated.SimpleCharStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="RccTokenManager" type="org.apache.hadoop.record.compiler.generated.SimpleCharStream, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setDebugStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ds" type="java.io.PrintStream"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="org.apache.hadoop.record.compiler.generated.SimpleCharStream"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="org.apache.hadoop.record.compiler.generated.SimpleCharStream"/>
+      <param name="lexState" type="int"/>
+    </method>
+    <method name="SwitchTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="lexState" type="int"/>
+    </method>
+    <method name="jjFillToken" return="org.apache.hadoop.record.compiler.generated.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNextToken" return="org.apache.hadoop.record.compiler.generated.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="debugStream" type="java.io.PrintStream"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="jjstrLiteralImages" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="lexStateNames" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="jjnewLexState" type="int[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="input_stream" type="org.apache.hadoop.record.compiler.generated.SimpleCharStream"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curChar" type="char"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.RccTokenManager -->
+  <!-- start class org.apache.hadoop.record.compiler.generated.SimpleCharStream -->
+  <class name="SimpleCharStream" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="SimpleCharStream" type="java.io.Reader, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.Reader, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.Reader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream, java.lang.String, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream, java.lang.String, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </constructor>
+    <constructor name="SimpleCharStream" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setTabSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="getTabSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="ExpandBuff"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="wrapAround" type="boolean"/>
+    </method>
+    <method name="FillBuff"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="BeginToken" return="char"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="UpdateLineColumn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="c" type="char"/>
+    </method>
+    <method name="readChar" return="char"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getEndColumn" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEndLine" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBeginColumn" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBeginLine" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="backup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amount" type="int"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.Reader"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+      <param name="buffersize" type="int"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.Reader"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.Reader"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+      <param name="encoding" type="java.lang.String"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+      <param name="buffersize" type="int"/>
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+      <param name="buffersize" type="int"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+      <param name="encoding" type="java.lang.String"/>
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+      <param name="encoding" type="java.lang.String"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </method>
+    <method name="ReInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dstream" type="java.io.InputStream"/>
+      <param name="startline" type="int"/>
+      <param name="startcolumn" type="int"/>
+    </method>
+    <method name="GetImage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="GetSuffix" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="len" type="int"/>
+    </method>
+    <method name="Done"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="adjustBeginLineColumn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newLine" type="int"/>
+      <param name="newCol" type="int"/>
+      <doc>
+      <![CDATA[Method to adjust line and column numbers for the start of a token.]]>
+      </doc>
+    </method>
+    <field name="staticFlag" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="bufpos" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="bufline" type="int[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="bufcolumn" type="int[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="column" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="line" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="prevCharIsCR" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="prevCharIsLF" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="inputStream" type="java.io.Reader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="char[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="maxNextCharInd" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="inBuf" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="tabSize" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An implementation of interface CharStream, where the stream is assumed to
+ contain only ASCII characters (without unicode processing).
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.SimpleCharStream -->
+  <!-- start class org.apache.hadoop.record.compiler.generated.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the image.]]>
+      </doc>
+    </method>
+    <method name="newToken" return="org.apache.hadoop.record.compiler.generated.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="ofKind" type="int"/>
+      <doc>
+      <![CDATA[Returns a new Token object, by default. However, if you want, you
+ can create and return subclass objects based on the value of ofKind.
+ Simply add the cases to the switch for all those special cases.
+ For example, if you have a subclass of Token called IDToken that
+ you want to create if ofKind is ID, simlpy add something like :
+
+    case MyParserConstants.ID : return new IDToken();
+
+ to the following switch statement. Then you can cast matchedToken
+ variable to the appropriate type and use it in your lexical actions.]]>
+      </doc>
+    </method>
+    <field name="kind" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An integer that describes the kind of this token.  This numbering
+ system is determined by JavaCCParser, and a table of these numbers is
+ stored in the file ...Constants.java.]]>
+      </doc>
+    </field>
+    <field name="beginLine" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[beginLine and beginColumn describe the position of the first character
+ of this token; endLine and endColumn describe the position of the
+ last character of this token.]]>
+      </doc>
+    </field>
+    <field name="beginColumn" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[beginLine and beginColumn describe the position of the first character
+ of this token; endLine and endColumn describe the position of the
+ last character of this token.]]>
+      </doc>
+    </field>
+    <field name="endLine" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[beginLine and beginColumn describe the position of the first character
+ of this token; endLine and endColumn describe the position of the
+ last character of this token.]]>
+      </doc>
+    </field>
+    <field name="endColumn" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[beginLine and beginColumn describe the position of the first character
+ of this token; endLine and endColumn describe the position of the
+ last character of this token.]]>
+      </doc>
+    </field>
+    <field name="image" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The string image of the token.]]>
+      </doc>
+    </field>
+    <field name="next" type="org.apache.hadoop.record.compiler.generated.Token"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A reference to the next regular (non-special) token from the input
+ stream.  If this is the last token from the input stream, or if the
+ token manager has not read tokens beyond this one, this field is
+ set to null.  This is true only if this token is also a regular
+ token.  Otherwise, see below for a description of the contents of
+ this field.]]>
+      </doc>
+    </field>
+    <field name="specialToken" type="org.apache.hadoop.record.compiler.generated.Token"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This field is used to access special tokens that occur prior to this
+ token, but after the immediately preceding regular (non-special) token.
+ If there are no such special tokens, this field is set to null.
+ When there are more than one such special token, this field refers
+ to the last of these special tokens, which in turn refers to the next
+ previous special token through its specialToken field, and so on
+ until the first special token (whose specialToken field is null).
+ The next fields of special tokens refer to other special tokens that
+ immediately follow it (without an intervening regular token).  If there
+ is no such token, this field is null.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Describes the input token stream.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.Token -->
+  <!-- start class org.apache.hadoop.record.compiler.generated.TokenMgrError -->
+  <class name="TokenMgrError" extends="java.lang.Error"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="TokenMgrError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TokenMgrError" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TokenMgrError" type="boolean, int, int, int, java.lang.String, char, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addEscapes" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Replaces unprintable characters by their espaced (or unicode escaped)
+ equivalents in the given string]]>
+      </doc>
+    </method>
+    <method name="LexicalError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="EOFSeen" type="boolean"/>
+      <param name="lexState" type="int"/>
+      <param name="errorLine" type="int"/>
+      <param name="errorColumn" type="int"/>
+      <param name="errorAfter" type="java.lang.String"/>
+      <param name="curChar" type="char"/>
+      <doc>
+      <![CDATA[Returns a detailed message for the Error when it is thrown by the
+ token manager to indicate a lexical error.
+ Parameters : 
+    EOFSeen     : indicates if EOF caused the lexicl error
+    curLexState : lexical state in which this error occured
+    errorLine   : line number when the error occured
+    errorColumn : column number when the error occured
+    errorAfter  : prefix that was seen before this error occured
+    curchar     : the offending character
+ Note: You can customize the lexical error message by modifying this method.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[You can also modify the body of this method to customize your error messages.
+ For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ of end-users concern, so you can return something like : 
+
+     "Internal Error : Please file a bug report .... "
+
+ from this method for such cases in the release version of your parser.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[@deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.compiler.generated.TokenMgrError -->
+  <doc>
+  <![CDATA[<p>
+    (DEPRECATED) This package contains code generated by JavaCC from the
+    Hadoop record syntax file rcc.jj. For details about the
+    record file syntax please @see org.apache.hadoop.record.
+    </p>
+  
+    <p>
+    DEPRECATED: Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.
+    </p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.record.meta">
+  <!-- start class org.apache.hadoop.record.meta.FieldTypeInfo -->
+  <class name="FieldTypeInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="getTypeID" return="org.apache.hadoop.record.meta.TypeID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the field's TypeID object]]>
+      </doc>
+    </method>
+    <method name="getFieldID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the field's id (name)]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Two FieldTypeInfos are equal if ach of their fields matches]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[We use a basic hashcode implementation, since this class will likely not
+ be used as a hashmap key]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ti" type="org.apache.hadoop.record.meta.FieldTypeInfo"/>
+    </method>
+    <doc>
+    <![CDATA[Represents a type information for a field, which is made up of its 
+ ID (name) and its type (a TypeID object).
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.FieldTypeInfo -->
+  <!-- start class org.apache.hadoop.record.meta.MapTypeID -->
+  <class name="MapTypeID" extends="org.apache.hadoop.record.meta.TypeID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="MapTypeID" type="org.apache.hadoop.record.meta.TypeID, org.apache.hadoop.record.meta.TypeID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKeyTypeID" return="org.apache.hadoop.record.meta.TypeID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the TypeID of the map's key element]]>
+      </doc>
+    </method>
+    <method name="getValueTypeID" return="org.apache.hadoop.record.meta.TypeID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the TypeID of the map's value element]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Two map  typeIDs are equal if their constituent elements have the 
+ same type]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[We use a basic hashcode implementation, since this class will likely not
+ be used as a hashmap key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents typeID for a Map 
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.MapTypeID -->
+  <!-- start class org.apache.hadoop.record.meta.RecordTypeInfo -->
+  <class name="RecordTypeInfo" extends="org.apache.hadoop.record.Record"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="RecordTypeInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty RecordTypeInfo object.]]>
+      </doc>
+    </constructor>
+    <constructor name="RecordTypeInfo" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a RecordTypeInfo object representing a record with the given name
+ @param name Name of the record]]>
+      </doc>
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return the name of the record]]>
+      </doc>
+    </method>
+    <method name="setName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[set the name of the record]]>
+      </doc>
+    </method>
+    <method name="addField"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldName" type="java.lang.String"/>
+      <param name="tid" type="org.apache.hadoop.record.meta.TypeID"/>
+      <doc>
+      <![CDATA[Add a field. 
+ @param fieldName Name of the field
+ @param tid Type ID of the field]]>
+      </doc>
+    </method>
+    <method name="getFieldTypeInfos" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a collection of field type infos]]>
+      </doc>
+    </method>
+    <method name="getNestedStructTypeInfo" return="org.apache.hadoop.record.meta.RecordTypeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return the type info of a nested record. We only consider nesting 
+ to one level. 
+ @param name Name of the nested record]]>
+      </doc>
+    </method>
+    <method name="serialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rout" type="org.apache.hadoop.record.RecordOutput"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize the type information for a record]]>
+      </doc>
+    </method>
+    <method name="deserialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rin" type="org.apache.hadoop.record.RecordInput"/>
+      <param name="tag" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize the type information for a record]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="peer_" type="java.lang.Object"/>
+      <exception name="ClassCastException" type="java.lang.ClassCastException"/>
+      <doc>
+      <![CDATA[This class doesn't implement Comparable as it's not meant to be used 
+ for anything besides de/serializing.
+ So we always throw an exception.
+ Not implemented. Always returns 0 if another RecordTypeInfo is passed in.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A record's Type Information object which can read/write itself. 
+ 
+ Type information for a record comprises metadata about the record, 
+ as well as a collection of type information for each field in the record. 
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.RecordTypeInfo -->
+  <!-- start class org.apache.hadoop.record.meta.StructTypeID -->
+  <class name="StructTypeID" extends="org.apache.hadoop.record.meta.TypeID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="StructTypeID" type="org.apache.hadoop.record.meta.RecordTypeInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a StructTypeID based on the RecordTypeInfo of some record]]>
+      </doc>
+    </constructor>
+    <method name="getFieldTypeInfos" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Represents typeID for a struct 
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.StructTypeID -->
+  <!-- start class org.apache.hadoop.record.meta.TypeID -->
+  <class name="TypeID" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="getTypeVal" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type value. One of the constants in RIOType.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Two base typeIDs are equal if they refer to the same type]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[We use a basic hashcode implementation, since this class will likely not
+ be used as a hashmap key]]>
+      </doc>
+    </method>
+    <field name="BoolTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constant classes for the basic types, so we can share them.]]>
+      </doc>
+    </field>
+    <field name="BufferTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ByteTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DoubleTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FloatTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IntTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LongTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="StringTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="typeVal" type="byte"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Represents typeID for basic types.
+  
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.TypeID -->
+  <!-- start class org.apache.hadoop.record.meta.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rin" type="org.apache.hadoop.record.RecordInput"/>
+      <param name="tag" type="java.lang.String"/>
+      <param name="typeID" type="org.apache.hadoop.record.meta.TypeID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[read/skip bytes from stream based on a type]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Various utility functions for Hadooop record I/O platform.
+ 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.Utils -->
+  <!-- start class org.apache.hadoop.record.meta.VectorTypeID -->
+  <class name="VectorTypeID" extends="org.apache.hadoop.record.meta.TypeID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Replaced by lEsS_tHaNa href=quote_cHaRhttp://hadoop.apache.org/avro/quote_cHaR>AvrolEsS_tHaN/a>.">
+    <constructor name="VectorTypeID" type="org.apache.hadoop.record.meta.TypeID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getElementTypeID" return="org.apache.hadoop.record.meta.TypeID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Two vector typeIDs are equal if their constituent elements have the 
+ same type]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[We use a basic hashcode implementation, since this class will likely not
+ be used as a hashmap key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents typeID for vector. 
+ @deprecated Replaced by <a href="http://hadoop.apache.org/avro/">Avro</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.record.meta.VectorTypeID -->
+</package>
+<package name="org.apache.hadoop.security">
+  <!-- start class org.apache.hadoop.security.AccessControlException -->
+  <class name="AccessControlException" extends="org.apache.hadoop.fs.permission.AccessControlException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessControlException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor is needed for unwrapping from 
+ {@link org.apache.hadoop.ipc.RemoteException}.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an {@link AccessControlException}
+ with the specified detail message.
+ @param s the detail message.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[An exception class for access control related issues.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.AccessControlException -->
+  <!-- start class org.apache.hadoop.security.Credentials -->
+  <class name="Credentials" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty credentials instance]]>
+      </doc>
+    </constructor>
+    <constructor name="Credentials" type="org.apache.hadoop.security.Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of the given credentials
+ @param credentials to copy]]>
+      </doc>
+    </constructor>
+    <method name="getToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the Token object for the alias
+ @param alias the alias for the Token
+ @return token for this alias]]>
+      </doc>
+    </method>
+    <method name="addToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="t" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token in the storage (in memory)
+ @param alias the alias for the key
+ @param t the token object]]>
+      </doc>
+    </method>
+    <method name="getAllTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the tokens in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="numberOfTokens" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of Tokens in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="getSecretKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the key bytes for the alias
+ @param alias the alias for the key
+ @return key for this alias]]>
+      </doc>
+    </method>
+    <method name="numberOfSecretKeys" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of keys in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="addSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Set the key for an alias
+ @param alias the alias for the key
+ @param key the key bytes]]>
+      </doc>
+    </method>
+    <method name="removeSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Remove the key for a given alias.
+ @param alias the alias for the key]]>
+      </doc>
+    </method>
+    <method name="getAllSecretKeys" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the secret key entries in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file, and loading the Tokens
+ therein in the passed UGI
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.io.File"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file, and loading the Tokens
+ therein in the passed UGI
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file directly from a 
+ datainputstream]]>
+      </doc>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores all the keys to DataOutput
+ @param out
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Loads all the keys
+ @param in
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <method name="mergeAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are not overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that provides the facilities of reading and writing
+ secret keys and Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.Credentials -->
+  <!-- start interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <interface name="GroupMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getGroups" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all various group memberships of a given user.
+ Returns EMPTY list in case of non-existing user
+ @param user User's name
+ @return group memberships of user
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsRefresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the cache of groups and user mapping
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsAdd"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groups" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Caches the group user information
+ @param groups list of groups to add to cache
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="GROUP_MAPPING_CONFIG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An interface for the implementation of a user-to-groups mapping service
+ used by {@link Groups}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <!-- start interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <interface name="IdMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getUid" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getGid" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getGroupName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="gid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getUidAllowingUnknown" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getGidAllowingUnknown" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[An interface for the implementation of <userId, userName> mapping
+ and <groupId, groupName> mapping]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <!-- start class org.apache.hadoop.security.KerberosAuthException -->
+  <class name="KerberosAuthException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosAuthException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="u" type="java.lang.String"/>
+    </method>
+    <method name="setPrincipal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+    </method>
+    <method name="setKeytabFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="java.lang.String"/>
+    </method>
+    <method name="setTicketCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.String"/>
+    </method>
+    <method name="getInitialMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The initial message, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getKeytabFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The keytab file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The principal, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getTicketCacheFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The ticket cache file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The user, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown when {@link UserGroupInformation} failed with an unrecoverable error,
+ such as failure in kerberos login/logout, invalid subject etc.
+
+ Caller should not retry when catching this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.KerberosAuthException -->
+  <!-- start class org.apache.hadoop.security.SecurityUtil -->
+  <class name="SecurityUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOriginalTGT" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ticket" type="javax.security.auth.kerberos.KerberosTicket"/>
+      <doc>
+      <![CDATA[Check whether the server principal is the TGS's principal
+ @param ticket the original TGT (the ticket that is obtained when a 
+ kinit is done)
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal
+ names. It replaces hostname pattern with hostname, which should be
+ fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
+ dynamically looked-up fqdn of the current host instead.
+ 
+ @param principalConfig
+          the Kerberos principal name conf value to convert
+ @param hostname
+          the fully-qualified domain name used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal names.
+ This method is similar to {@link #getServerPrincipal(String, String)},
+ except 1) the reverse DNS lookup from addr to hostname is done only when
+ necessary, 2) param addr can't be null (no default behavior of using local
+ hostname when addr is null).
+ 
+ @param principalConfig
+          Kerberos principal name pattern to convert
+ @param addr
+          InetAddress of the host used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in
+ user's Kerberos principal name with a dynamically looked-up fully-qualified
+ domain name of the current host.
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in user's Kerberos principal 
+ name with hostname. If non-secure mode - return. If no keytab available -
+ bail out with an exception
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @param hostname
+          hostname to use for substitution
+ @throws IOException if the config doesn't specify a keytab]]>
+      </doc>
+    </method>
+    <method name="buildDTServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="defPort" type="int"/>
+      <doc>
+      <![CDATA[create the service name for a Delegation token
+ @param uri of the service
+ @param defPort is used if the uri lacks a port
+ @return the token service, or null if no authority
+ @see #buildTokenService(InetSocketAddress)]]>
+      </doc>
+    </method>
+    <method name="getHostFromPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the host name from the principal name of format <service>/host@realm.
+ @param principalName principal name of format as described above
+ @return host name if the the string conforms to the above format, else null]]>
+      </doc>
+    </method>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the KerberosInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return the KerberosInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the TokenInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol The protocol class to get the information for.
+ @param conf Configuration object
+ @return the TokenInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getTokenServiceAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Decode the given token's service field into an InetAddress
+ @param token from which to obtain the service
+ @return InetAddress for the service]]>
+      </doc>
+    </method>
+    <method name="setTokenService"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the given token's service to the format expected by the RPC client 
+ @param token a delegation token
+ @param addr the socket for the rpc connection]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param addr InetSocketAddress of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param uri of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUserOrFatal" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If the login
+ user cannot be determined, this will log a FATAL error and exit
+ the whole JVM.]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="doAsCurrentUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's current user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticationMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isPrivilegedPort" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HOSTNAME_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED_TO_GET_UGI_MSG_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Security Utils.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.SecurityUtil -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation -->
+  <class name="UserGroupInformation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setShouldRenewImmediatelyForTests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="immediate" type="boolean"/>
+      <doc>
+      <![CDATA[For the purposes of unit tests, we want to test login
+ from keytab and don't want to wait until the renew
+ window (controlled by TICKET_RENEW_WINDOW).
+ @param immediate true if we should login without waiting for ticket window]]>
+      </doc>
+    </method>
+    <method name="reattachMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reattach the class's metrics to a new metric system.]]>
+      </doc>
+    </method>
+    <method name="isInitialized" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the static configuration for UGI.
+ In particular, set the security authentication mechanism and the
+ group look up service.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="isSecurityEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Determine if UserGroupInformation is using Kerberos to determine
+ user identities or is relying on simple authentication
+ 
+ @return true if UGI is working in a secure environment]]>
+      </doc>
+    </method>
+    <method name="hasKerberosCredentials" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[checks if logged in using kerberos
+ @return true if the subject logged via keytab or has a Kerberos TGT]]>
+      </doc>
+    </method>
+    <method name="getCurrentUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current user, including any doAs in the current stack.
+ @return the current user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="getBestUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCachePath" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the most appropriate UserGroupInformation to use
+
+ @param ticketCachePath    The Kerberos ticket cache path, or NULL
+                           if none is specfied
+ @param user               The user name, or NULL if none is specified.
+
+ @return                   The most appropriate UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getUGIFromTicketCache" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCache" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Kerberos ticket cache.
+ 
+ @param user                The principal name to load from the ticket
+                            cache
+ @param ticketCache     the path to the ticket cache file
+
+ @throws IOException        if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getUGIFromSubject" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Subject with Kerberos principal.
+
+ @param subject                The KerberosPrincipal to use in UGI
+
+ @throws IOException
+ @throws KerberosAuthException if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getLoginUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the currently logged in user.
+ @return the logged in user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="trimLoginMethod" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[remove the login method that is followed by a space from the username
+ e.g. "jack (auth:SIMPLE)" -> "jack"
+
+ @param userName
+ @return userName without login method]]>
+      </doc>
+    </method>
+    <method name="loginUserFromSubject"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log in a user using the given subject
+ @param subject the subject to use when logging in a user, or null to
+ create a new subject.
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="isFromKeytab" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this user logged in from a keytab file?
+ @return true if the credentials are from a keytab file.]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytab"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="logoutUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log the current user out who previously logged in using keytab.
+ This method assumes that the user logged in by calling
+ {@link #loginUserFromKeytab(String, String)}.
+
+ @throws IOException
+ @throws KerberosAuthException if a failure occurred in logout,
+ or if the user did not log in by invoking loginUserFromKeyTab() before.]]>
+      </doc>
+    </method>
+    <method name="checkTGTAndReloginFromKeytab"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-login a user from keytab if TGT is expired or is close to expiry.
+ 
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="reloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user. This
+ method assumes that {@link #loginUserFromKeytab(String, String)} had
+ happened already. The Subject field of this UserGroupInformation object is
+ updated to have the new credentials.
+
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="forceReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Force re-Login a user in from a keytab file irrespective of the last login
+ time. Loads a user identity from a keytab file and logs them in. They
+ become the currently logged-in user. This method assumes that
+ {@link #loginUserFromKeytab(String, String)} had happened already. The
+ Subject field of this UserGroupInformation object is updated to have the
+ new credentials.
+
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="reloginFromTicketCache"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from the ticket cache.  This
+ method assumes that login had happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytabAndReturnUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and login them in. This new user does not affect the currently
+ logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException if the keytab file can't be read]]>
+      </doc>
+    </method>
+    <method name="isLoginKeytabBased" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via keytab
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="isLoginTicketBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via ticket cache
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createProxyUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create a proxy user using username of the effective user and the ugi of the
+ real user.
+ @param user
+ @param realUser
+ @return proxyUser ugi]]>
+      </doc>
+    </method>
+    <method name="getRealUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get RealUser (vs. EffectiveUser)
+ @return realUser running over proxy user]]>
+      </doc>
+    </method>
+    <method name="getRealUserOrSelf" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[If this is a proxy user, get the real user. Otherwise, return
+ this user.
+ @param user the user to check
+ @return the real user or self]]>
+      </doc>
+    </method>
+    <method name="createUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a UGI for testing HDFS and MapReduce
+ @param user the full user principal name
+ @param userGroups the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="createProxyUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a proxy user UGI for testing HDFS and MapReduce
+ 
+ @param user
+          the full user principal name for effective user
+ @param realUser
+          UGI of the real user
+ @param userGroups
+          the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="getShortUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's login name.
+ @return the user's name up to the first '/' or '@'.]]>
+      </doc>
+    </method>
+    <method name="getPrimaryGroupName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's full principal name.
+ @return the user's full principal name.]]>
+      </doc>
+    </method>
+    <method name="addTokenIdentifier" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokenId" type="org.apache.hadoop.security.token.TokenIdentifier"/>
+      <doc>
+      <![CDATA[Add a TokenIdentifier to this UGI. The TokenIdentifier has typically been
+ authenticated by the RPC layer as belonging to the user represented by this
+ UGI.
+ 
+ @param tokenId
+          tokenIdentifier to be added
+ @return true on successful add of new tokenIdentifier]]>
+      </doc>
+    </method>
+    <method name="getTokenIdentifiers" return="java.util.Set"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of TokenIdentifiers belonging to this UGI
+ 
+ @return the set of TokenIdentifiers belonging to this UGI]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token to this UGI
+ 
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a named token to this UGI
+ 
+ @param alias Name of the token
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="getTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the collection of tokens associated with this user.
+ 
+ @return an unmodifiable collection of tokens associated with user]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the tokens in credentials form associated with this user.
+ 
+ @return Credentials of tokens associated with this user]]>
+      </doc>
+    </method>
+    <method name="addCredentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="credentials" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Add the given Credentials to this user.
+ @param credentials of tokens and secrets]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user. {@link #getGroups()} is less
+ expensive alternative when checking for a contained element.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the username.]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the subject
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the real user's subject.  If there
+ is no real user, return the given user's authentication method.
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Returns the authentication method of a ugi. If the authentication method is
+ PROXY, returns the authentication method of the real user.
+ 
+ @param ugi
+ @return AuthenticationMethod]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare the subjects to see if they are equal to each other.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the hash of the subject.]]>
+      </doc>
+    </method>
+    <method name="getSubject" return="javax.security.auth.Subject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying subject from this ugi.
+ @return the subject that represents this user.]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Run the given action as the user.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the given action as the user, potentially throwing an exception.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method
+ @throws IOException if the action throws an IOException
+ @throws Error if the action throws an Error
+ @throws RuntimeException if the action throws a RuntimeException
+ @throws InterruptedException if the action throws an InterruptedException
+ @throws UndeclaredThrowableException if the action throws something else]]>
+      </doc>
+    </method>
+    <method name="logAllUserInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log all (current, real, login) UGI and token info into UGI debug log.
+ @param ugi - UGI
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A test method to print out the current user's UGI.
+ @param args if there are two arguments, read the user from the keytab
+ and print it out.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <field name="HADOOP_TOKEN_FILE_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the token cache file]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[User and group information for Hadoop.
+ This class wraps around a JAAS Subject and provides methods to determine the
+ user's username and groups. It supports both the Windows, Unix and Kerberos 
+ login modules.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+  <class name="UserGroupInformation.AuthenticationMethod" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getAuthMethod" return="org.apache.hadoop.security.SaslRpcServer.AuthMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+    </method>
+    <doc>
+    <![CDATA[existing types of authentications' methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+</package>
+<package name="org.apache.hadoop.security.alias">
+  <!-- start class org.apache.hadoop.security.alias.CredentialProvider -->
+  <class name="CredentialProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide job access to
+ passwords rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the credentials are written to persistent
+ store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the credential entry for a specific alias.
+ @param alias the name of a specific credential
+ @return the credentialEntry
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAliases" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the aliases for all credentials.
+ @return the list of alias names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="credential" type="char[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new credential. The given alias must not already exist.
+ @param name the alias of the credential
+ @param credential the credential value for the alias.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteCredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given credential.
+ @param name the alias of the credential to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of credentials or password for Hadoop applications. Provides an
+ abstraction to separate credential storage from users of them. It
+ is intended to support getting or storing passwords in a variety of ways,
+ including third party bindings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProvider -->
+  <!-- start class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+  <class name="CredentialProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.security.alias.CredentialProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of CredentialProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ CredentialProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+</package>
+<package name="org.apache.hadoop.security.authorize">
+  <!-- start class org.apache.hadoop.security.authorize.AccessControlList -->
+  <class name="AccessControlList" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="AccessControlList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constructor exists primarily for AccessControlList to be Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from a String representation of the same.
+ 
+ The String is a a comma separated list of users and groups.
+ The user list comes first and is separated by a space followed 
+ by the group list. For e.g. "user1,user2 group1,group2"
+ 
+ @param aclString String representation of the ACL]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from String representation of users and groups
+ 
+ The arguments are comma separated lists
+ 
+ @param users comma separated list of users
+ @param groups comma separated list of groups]]>
+      </doc>
+    </constructor>
+    <method name="isAllAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add user to the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="addGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add group to the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="removeUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove user from the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="removeGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove group from the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="getUsers" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of users allowed for this service.
+ @return the set of user names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of user groups allowed for this service.
+ @return the set of group names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="isUserInList" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Checks if a user represented by the provided {@link UserGroupInformation}
+ is a member of the Access Control List. If user was proxied and
+ USE_REAL_ACLS + the real user name is in the control list, then treat this
+ case as if user were in the ACL list.
+ @param ugi UserGroupInformation to check if contained in the ACL
+ @return true if ugi is member of the list or if USE_REAL_ACLS + real user
+ is in the list]]>
+      </doc>
+    </method>
+    <method name="isUserAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns descriptive way of users and groups that are part of this ACL.
+ Use {@link #getAclString()} to get the exact String that can be given to
+ the constructor of AccessControlList to create a new instance.]]>
+      </doc>
+    </method>
+    <method name="getAclString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the access control list as a String that can be used for building a
+ new instance by sending it to the constructor of {@link AccessControlList}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <field name="WILDCARD_ACL_VALUE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_REAL_ACLS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Class representing a configured access control list.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AccessControlList -->
+  <!-- start class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <class name="AuthorizationException" extends="org.apache.hadoop.security.AccessControlException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuthorizationException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <method name="getStackTrace" return="java.lang.StackTraceElement[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintStream"/>
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintWriter"/>
+    </method>
+    <doc>
+    <![CDATA[An exception class for authorization-related issues.
+ 
+ This class <em>does not</em> provide the stack trace for security purposes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <!-- start class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <class name="DefaultImpersonationProvider" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authorize.ImpersonationProvider"/>
+    <constructor name="DefaultImpersonationProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTestProvider" return="org.apache.hadoop.security.authorize.DefaultImpersonationProvider"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.lang.String"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+    </method>
+    <method name="getProxySuperuserUserConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective usergroups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser usergroups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserGroupConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective groups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser groups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserIpConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return configuration key for superuser ip addresses
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser ip-addresses]]>
+      </doc>
+    </method>
+    <method name="getProxyGroups" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProxyHosts" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <!-- start interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+  <interface name="ImpersonationProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Specifies the configuration prefix for the proxy user properties and
+ initializes the provider.
+
+ @param configurationPrefix the configuration prefix for the proxy user
+ properties]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.lang.String"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs
+ 
+ @param user ugi of the effective or proxy user which contains a real user
+ @param remoteAddress the ip address of client
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+</package>
+<package name="org.apache.hadoop.security.http">
+  <!-- start class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <class name="RestCsrfPreventionFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="RestCsrfPreventionFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filterConfig" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="isBrowser" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userAgent" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This method interrogates the User-Agent String and returns whether it
+ refers to a browser.  If its not a browser, then the requirement for the
+ CSRF header will not be enforced; if it is a browser, the requirement will
+ be enforced.
+ <p>
+ A User-Agent String is considered to be a browser if it matches
+ any of the regex patterns from browser-useragent-regex; the default
+ behavior is to consider everything a browser that matches the following:
+ "^Mozilla.*,^Opera.*".  Subclasses can optionally override
+ this method to use different behavior.
+
+ @param userAgent The User-Agent String, or null if there isn't one
+ @return true if the User-Agent String refers to a browser, false if not]]>
+      </doc>
+    </method>
+    <method name="handleHttpInteraction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="httpInteraction" type="org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+      <doc>
+      <![CDATA[Handles an {@link HttpInteraction} by applying the filtering logic.
+
+ @param httpInteraction caller's HTTP interaction
+ @throws IOException if there is an I/O error
+ @throws ServletException if the implementation relies on the servlet API
+     and a servlet API call has failed]]>
+      </doc>
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="javax.servlet.ServletRequest"/>
+      <param name="response" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="HEADER_USER_AGENT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BROWSER_USER_AGENT_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_METHODS_TO_IGNORE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HEADER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter provides protection against cross site request forgery (CSRF)
+ attacks for REST APIs. Enabling this filter on an endpoint results in the
+ requirement of all client to send a particular (configurable) HTTP header
+ with every request. In the absense of this header the filter will reject the
+ attempt as a bad request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <!-- start class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+  <class name="XFrameOptionsFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="XFrameOptionsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="javax.servlet.ServletRequest"/>
+      <param name="res" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="X_FRAME_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter protects webapps from clickjacking attacks that
+ are possible through use of Frames to embed the resources in another
+ application and intercept clicks to accomplish nefarious things.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+</package>
+<package name="org.apache.hadoop.security.protocolPB">
+</package>
+<package name="org.apache.hadoop.security.ssl">
+</package>
+<package name="org.apache.hadoop.security.token">
+  <!-- start class org.apache.hadoop.security.token.SecretManager -->
+  <class name="SecretManager" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createPassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <doc>
+      <![CDATA[Create the password for the given identifier.
+ identifier may be modified inside this method.
+ @param identifier the identifier to use
+ @return the new password]]>
+      </doc>
+    </method>
+    <method name="retrievePassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <doc>
+      <![CDATA[Retrieve the password for the given token identifier. Should check the date
+ or registry to make sure the token hasn't expired or been revoked. Returns 
+ the relevant password.
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid]]>
+      </doc>
+    </method>
+    <method name="retriableRetrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <exception name="RetriableException" type="org.apache.hadoop.ipc.RetriableException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The same functionality with {@link #retrievePassword}, except that this 
+ method can throw a {@link RetriableException} or a {@link StandbyException}
+ to indicate that client can retry/failover the same operation because of 
+ temporary issue on the server side.
+ 
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid
+ @throws StandbyException the server is in standby state, the client can
+         try other servers
+ @throws RetriableException the token was invalid, and the server thinks 
+         this may be a temporary issue and suggests the client to retry
+ @throws IOException to allow future exceptions to be added without breaking
+         compatibility]]>
+      </doc>
+    </method>
+    <method name="createIdentifier" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty token identifier.
+ @return the newly created empty token identifier]]>
+      </doc>
+    </method>
+    <method name="checkAvailableForRead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <doc>
+      <![CDATA[No-op if the secret manager is available for reading tokens, throw a
+ StandbyException otherwise.
+ 
+ @throws StandbyException if the secret manager is not available to read
+         tokens]]>
+      </doc>
+    </method>
+    <method name="generateSecret" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a new random secret key.
+ @return the new key]]>
+      </doc>
+    </method>
+    <method name="createPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="byte[]"/>
+      <param name="key" type="javax.crypto.SecretKey"/>
+      <doc>
+      <![CDATA[Compute HMAC of the identifier using the secret key and return the 
+ output as password
+ @param identifier the bytes of the identifier
+ @param key the secret key
+ @return the bytes of the generated password]]>
+      </doc>
+    </method>
+    <method name="createSecretKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Convert the byte[] to a secret key
+ @param key the byte[] to create a secret key from
+ @return the secret key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The server-side secret manager for each token type.
+ @param <T> The type of the token identifier]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.SecretManager -->
+  <!-- start class org.apache.hadoop.security.token.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Token" type="T, org.apache.hadoop.security.token.SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token given a token identifier and a secret manager for the
+ type of the token identifier.
+ @param id the token identifier
+ @param mgr the secret manager]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="byte[], byte[], org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token from the components.
+ @param identifier the token identifier
+ @param password the token's password
+ @param kind the kind of token
+ @param service the service for this token]]>
+      </doc>
+    </constructor>
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="org.apache.hadoop.security.token.Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clone a token.
+ @param other the token to clone]]>
+      </doc>
+    </constructor>
+    <method name="getIdentifier" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier's byte representation
+ @return the token identifier's byte representation]]>
+      </doc>
+    </method>
+    <method name="decodeIdentifier" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the token identifier object, or null if it could not be constructed
+ (because the class could not be loaded, for example).
+ @return the token identifier, or null
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password/secret
+ @return the token password/secret]]>
+      </doc>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service on which the token is supposed to be used
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="setService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Set the service on which the token is supposed to be used
+ @param newService the service name]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+    </method>
+    <method name="setPassword"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newPassword" type="byte[]"/>
+    </method>
+    <method name="isPrivate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether this is a private token.
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="isPrivateCloneOf" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thePublicService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Whether this is a private clone of a public token.
+ @param thePublicService the public service name
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="privateClone" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Create a private clone of a public token.
+ @param newService the new service name
+ @return a private token]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="encodeToUrlString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode this token as a url safe string
+ @return the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="decodeFromUrlString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode the given url safe string into this token.
+ @param newValue the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="buildCacheKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is this token managed so that it can be renewed or cancelled?
+ @return true, if it can be renewed and cancelled.]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew this delegation token
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel this delegation token
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The client-side form of the token.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token -->
+  <!-- start class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <class name="Token.TrivialRenewer" extends="org.apache.hadoop.security.token.TokenRenewer"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token.TrivialRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="handleKind" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[A trivial renewer for token kinds that aren't managed. Sub-classes need
+ to implement getKind for their token kind.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <!-- start class org.apache.hadoop.security.token.TokenIdentifier -->
+  <class name="TokenIdentifier" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Ugi with the username encoded in the token identifier
+ 
+ @return the username. null is returned if username in the identifier is
+         empty or null.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes for the token identifier
+ @return the bytes of the identifier]]>
+      </doc>
+    </method>
+    <method name="getTrackingId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a tracking identifier that can be used to associate usages of a
+ token across multiple client sessions.
+
+ Currently, this function just returns an MD5 of {{@link #getBytes()}.
+
+ @return tracking identifier]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An identifier that identifies a token, may contain public information 
+ about a token, including its kind (or type).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenIdentifier -->
+  <!-- start class org.apache.hadoop.security.token.TokenInfo -->
+  <class name="TokenInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Indicates Token related information to be used]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenInfo -->
+  <!-- start class org.apache.hadoop.security.token.TokenRenewer -->
+  <class name="TokenRenewer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="handleKind" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Does this renewer handle this kind of token?
+ @param kind the kind of the token
+ @return true if this renewer can renew it]]>
+      </doc>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is the given token managed? Only managed tokens may be renewed or
+ cancelled.
+ @param token the token being checked
+ @return true if the token may be renewed or cancelled
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew the given token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel the given token
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for plugins that handle tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenRenewer -->
+  <!-- start interface org.apache.hadoop.security.token.TokenSelector -->
+  <interface name="TokenSelector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+    <doc>
+    <![CDATA[Select token of type T from tokens for use with named service
+ 
+ @param <T>
+          T extends TokenIdentifier]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.token.TokenSelector -->
+</package>
+<package name="org.apache.hadoop.security.token.delegation.web">
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <class name="DelegationTokenAuthenticatedURL" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DelegationTokenAuthenticatedURL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+ <p/>
+ An instance of the default {@link DelegationTokenAuthenticator} will be
+ used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code> using the default
+ {@link DelegationTokenAuthenticator} class.
+
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator, org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <method name="setDefaultDelegationTokenAuthenticator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticator" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Sets the default {@link DelegationTokenAuthenticator} class to use when an
+ {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @param authenticator the authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="getDefaultDelegationTokenAuthenticator" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the default {@link DelegationTokenAuthenticator} class to use when
+ an {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+ <p/>
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @return the delegation token authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="setUseQueryStringForDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="useQueryString" type="boolean"/>
+      <doc>
+      <![CDATA[Sets if delegation token should be transmitted in the URL query string.
+ By default it is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ <p/>
+ This method is provided to enable WebHDFS backwards compatibility.
+
+ @param useQueryString  <code>TRUE</code> if the token is transmitted in the
+ URL query string, <code>FALSE</code> if the delegation token is transmitted
+ using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ header.]]>
+      </doc>
+    </method>
+    <method name="useQueryStringForDelegationToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns if delegation token is transmitted as a HTTP header.
+
+ @return <code>TRUE</code> if the token is transmitted in the URL query
+ string, <code>FALSE</code> if the delegation token is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
+ Token only if the given auth token is an instance of {@link Token} and
+ it contains a Delegation Token, otherwise use the configured
+ {@link DelegationTokenAuthenticator} to authenticate the connection.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAs" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>. If the <code>doAs</code> parameter is not NULL,
+ the request will be done on behalf of the specified <code>doAs</code> user.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @param doAs user to do the the request on behalf of, if NULL the request is
+ as self.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The <code>DelegationTokenAuthenticatedURL</code> is a
+ {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
+ functionality.
+ <p/>
+ The authentication mechanisms supported by default are Hadoop Simple
+ authentication (also known as pseudo authentication) and Kerberos SPNEGO
+ authentication.
+ <p/>
+ Additional authentication mechanisms can be supported via {@link
+ DelegationTokenAuthenticator} implementations.
+ <p/>
+ The default {@link DelegationTokenAuthenticator} is the {@link
+ KerberosDelegationTokenAuthenticator} class which supports
+ automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
+ the {@link PseudoDelegationTokenAuthenticator} class.
+ <p/>
+ <code>AuthenticatedURL</code> instances are not thread-safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <class name="DelegationTokenAuthenticatedURL.Token" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DelegationTokenAuthenticatedURL.Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delegationToken" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <doc>
+    <![CDATA[Client side authentication token that handles Delegation Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <class name="DelegationTokenAuthenticator" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authentication.client.Authenticator"/>
+    <constructor name="DelegationTokenAuthenticator" type="org.apache.hadoop.security.authentication.client.Authenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConnectionConfigurator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurator" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"/>
+    </method>
+    <method name="authenticate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <field name="OP_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TOKEN_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEWER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_URL_STRING_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEW_DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@link Authenticator} wrapper that enhances an {@link Authenticator} with
+ Delegation Token support.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <class name="KerberosDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>KerberosDelegationTokenAuthenticator</code> provides support for
+ Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
+ Token operations.
+ <p/>
+ It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
+ endpoint does not trigger a SPNEGO authentication]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+  <class name="PseudoDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PseudoDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>PseudoDelegationTokenAuthenticator</code> provides support for
+ Hadoop's pseudo authentication mechanism that accepts
+ the user name specified as a query string parameter and support for Hadoop
+ Delegation Token operations.
+ <p/>
+ This mimics the model of Hadoop Simple authentication trusting the
+ {@link UserGroupInformation#getCurrentUser()} value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+</package>
+<package name="org.apache.hadoop.service">
+  <!-- start class org.apache.hadoop.service.AbstractService -->
+  <class name="AbstractService" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <constructor name="AbstractService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the service.
+ @param name service name]]>
+      </doc>
+    </constructor>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration for this service.
+ This method is called during {@link #init(Configuration)}
+ and should only be needed if for some reason a service implementation
+ needs to override that initial setting -for example replacing
+ it with a new subclass of {@link Configuration}
+ @param conf new configuration.]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ This invokes {@link #serviceInit}
+ @param conf the configuration of the service. This must not be null
+ @throws ServiceStateException if the configuration was null,
+ the state change not permitted, or something else went wrong]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws ServiceStateException if the current service state does not permit
+ this action]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Relay to {@link #stop()}
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noteFailure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <param name="exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Failure handling: record the exception
+ that triggered it -if there was not one already.
+ Services are free to call this themselves.
+ @param exception the exception]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[All initialization code needed by a service.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #init(Configuration)} prevents re-entrancy.
+
+ The base implementation checks to see if the subclass has created
+ a new configuration instance, and if so, updates the base class value
+ @param conf configuration
+ @throws Exception on a failure -these will be caught,
+ possibly wrapped, and wil; trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the INITED to STARTED transition.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #start()} prevents re-entrancy.
+
+ @throws Exception if needed -these will be caught,
+ wrapped, and trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the transition to the STOPPED state.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #stop()} prevents re-entrancy.
+
+ Implementations MUST write this to be robust against failures, including
+ checks for null references -and for the first failure to not stop other
+ attempts to shut down parts of the service.
+
+ @throws Exception if needed -these will be caught and logged.]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="registerGlobalListener"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a global listener, which receives notifications
+ from the state change events of all services in the JVM
+ @param l listener]]>
+      </doc>
+    </method>
+    <method name="unregisterGlobalListener" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[unregister a global listener.
+ @param l listener to unregister
+ @return true if the listener was found (and then deleted)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="expected" type="org.apache.hadoop.service.Service.STATE"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="putBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="details" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a blocker to the blocker map -replacing any
+ with the same name.
+ @param name blocker name
+ @param details any specifics on the block. This must be non-null.]]>
+      </doc>
+    </method>
+    <method name="removeBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a blocker from the blocker map -
+ this is a no-op if the blocker is not present
+ @param name the name of the blocker]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This is the base implementation class for services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.AbstractService -->
+  <!-- start class org.apache.hadoop.service.CompositeService -->
+  <class name="CompositeService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getServices" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a cloned list of services
+ @return a list of child services at the time of invocation -
+ added services will not be picked up.]]>
+      </doc>
+    </method>
+    <method name="addService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Add the passed {@link Service} to the list of services managed by this
+ {@link CompositeService}
+ @param service the {@link Service} to be added]]>
+      </doc>
+    </method>
+    <method name="addIfService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="object" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[If the passed object is an instance of {@link Service},
+ add it to the list of services managed by this {@link CompositeService}
+ @param object
+ @return true if a service is added, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="removeService" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="STOP_ONLY_STARTED_SERVICES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Policy on shutdown: attempt to close everything (purest) or
+ only try to close started services (which assumes
+ that the service implementations may not handle the stop() operation
+ except when started.
+ Irrespective of this policy, if a child service fails during
+ its init() or start() operations, it will have stop() called on it.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Composition of services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.CompositeService -->
+  <!-- start class org.apache.hadoop.service.LifecycleEvent -->
+  <class name="LifecycleEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="LifecycleEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="time" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Local time in milliseconds when the event occurred]]>
+      </doc>
+    </field>
+    <field name="state" type="org.apache.hadoop.service.Service.STATE"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[new state]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A serializable lifecycle event: the time a state
+ transition occurred, and what state was entered.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LifecycleEvent -->
+  <!-- start class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <class name="LoggingStateChangeListener" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    <constructor name="LoggingStateChangeListener" type="org.slf4j.Logger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the given log
+ @param log destination for events]]>
+      </doc>
+    </constructor>
+    <constructor name="LoggingStateChangeListener"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the static log for this class]]>
+      </doc>
+    </constructor>
+    <method name="stateChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback for a state change event: log it
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a state change listener that logs events at INFO level]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <!-- start interface org.apache.hadoop.service.Service -->
+  <interface name="Service"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Initialize the service.
+
+ The transition MUST be from {@link STATE#NOTINITED} to {@link STATE#INITED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @param config the configuration of the service
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the service.
+
+ The transition MUST be from {@link STATE#INITED} to {@link STATE#STARTED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the service. This MUST be a no-op if the service is already
+ in the {@link STATE#STOPPED} state. It SHOULD be a best-effort attempt
+ to stop all parts of the service.
+
+ The implementation must be designed to complete regardless of the service
+ state, including the initialized/uninitialized state of all its internal
+ fields.
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A version of stop() that is designed to be usable in Java7 closure
+ clauses.
+ Implementation classes MUST relay this directly to {@link #stop()}
+ @throws IOException never
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a listener to the service state change events.
+ If the supplied listener is already listening to this service,
+ this method is a no-op.
+ @param listener a new listener]]>
+      </doc>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Unregister a previously registered listener of the service state
+ change events. No-op if the listener is already unregistered.
+ @param listener the listener to unregister.]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this service.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration of this service.
+ This is normally not a clone and may be manipulated, though there are no
+ guarantees as to what the consequences of such actions may be
+ @return the current configuration, unless a specific implentation chooses
+ otherwise.]]>
+      </doc>
+    </method>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current service state
+ @return the state of the service]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service start time
+ @return the start time of the service. This will be zero if the service
+ has not yet been started.]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query to see if the service is in a specific state.
+ In a multi-threaded system, the state may not hold for very long.
+ @param state the expected state
+ @return true if, at the time of invocation, the service was in that state.]]>
+      </doc>
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the first exception raised during the service failure. If null,
+ no exception was logged
+ @return the failure logged during a transition to the stopped state]]>
+      </doc>
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the state in which the failure in {@link #getFailureCause()} occurred.
+ @return the state or null if there was no failure]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+      <doc>
+      <![CDATA[Block waiting for the service to stop; uses the termination notification
+ object to do so.
+
+ This method will only return after all the service stop actions
+ have been executed (to success or failure), or the timeout elapsed
+ This method can be called before the service is inited or started; this is
+ to eliminate any race condition with the service stopping before
+ this event occurs.
+ @param timeout timeout in milliseconds. A value of zero means "forever"
+ @return true iff the service stopped in the time period]]>
+      </doc>
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a snapshot of the lifecycle history; it is a static list
+ @return a possibly empty but never null list of lifecycle events.]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the blockers on a service -remote dependencies
+ that are stopping the service from being <i>live</i>.
+ @return a (snapshotted) map of blocker name-&gt;description values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Service LifeCycle.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.Service -->
+  <!-- start class org.apache.hadoop.service.ServiceOperations -->
+  <class name="ServiceOperations" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service.
+ <p/>Do nothing if the service is null or not
+ in a state in which it can be/needs to be stopped.
+ <p/>
+ The service state is checked <i>before</i> the operation begins.
+ This process is <i>not</i> thread safe.
+ @param service a service or null]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains a set of methods to work with services, especially
+ to walk them through their lifecycle.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceOperations -->
+  <!-- start interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <interface name="ServiceStateChangeListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="stateChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback to notify of a state change. The service will already
+ have changed state before this callback is invoked.
+
+ This operation is invoked on the thread that initiated the state change,
+ while the service itself in in a sychronized section.
+ <ol>
+   <li>Any long-lived operation here will prevent the service state
+   change from completing in a timely manner.</li>
+   <li>If another thread is somehow invoked from the listener, and
+   that thread invokes the methods of the service (including
+   subclass-specific methods), there is a risk of a deadlock.</li>
+ </ol>
+
+
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to notify state changes of a service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <!-- start class org.apache.hadoop.service.ServiceStateException -->
+  <class name="ServiceStateException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <constructor name="ServiceStateException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate
+ @param message error message]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate with a message and cause; if the cause has an exit code
+ then it is used, otherwise the generic
+ {@link LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION} exit code
+ is used.
+ @param message exception message
+ @param cause optional inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="int, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate, using the specified exit code as the exit code
+ of the exception, irrespetive of any exit code supplied by any inner
+ cause.
+
+ @param exitCode exit code to declare
+ @param message exception message
+ @param cause inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="text" type="java.lang.String"/>
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ If the caught exception is already of that type, it is typecast to a
+ {@link RuntimeException} and returned.
+
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param text text to use if a new exception is created
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Exception that can be raised on state change operations, whose
+ exit code can be explicitly set, determined from that of any nested
+ cause, or a default value of
+ {@link  LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateException -->
+  <!-- start class org.apache.hadoop.service.ServiceStateModel -->
+  <class name="ServiceStateModel" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceStateModel" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the service state model in the {@link Service.STATE#NOTINITED}
+ state.]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateModel" type="java.lang.String, org.apache.hadoop.service.Service.STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a service state model instance in the chosen state
+ @param state the starting state]]>
+      </doc>
+    </constructor>
+    <method name="getState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Query the service state. This is a non-blocking operation.
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query that the state is in a specific state
+ @param proposed proposed new state
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="ensureCurrentState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expectedState" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Verify that that a service is in a given state.
+ @param expectedState the desired state
+ @throws ServiceStateException if the service state is different from
+ the desired state]]>
+      </doc>
+    </method>
+    <method name="enterState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Enter a state -thread safe.
+
+ @param proposed proposed new state
+ @return the original state
+ @throws ServiceStateException if the transition is not permitted]]>
+      </doc>
+    </method>
+    <method name="checkStateTransition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Check that a state tansition is valid and
+ throw an exception if not
+ @param name name of the service (can be null)
+ @param state current state
+ @param proposed proposed new state]]>
+      </doc>
+    </method>
+    <method name="isValidStateTransition" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="current" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Is a state transition valid?
+ There are no checks for current==proposed
+ as that is considered a non-transition.
+
+ using an array kills off all branch misprediction costs, at the expense
+ of cache line misses.
+
+ @param current current state
+ @param proposed proposed new state
+ @return true if the transition to a new state is valid]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return the state text as the toString() value
+ @return the current state's description]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the service state model.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateModel -->
+</package>
+<package name="org.apache.hadoop.service.launcher">
+  <!-- start class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <class name="AbstractLaunchableService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.launcher.LaunchableService"/>
+    <constructor name="AbstractLaunchableService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance with the given name.]]>
+      </doc>
+    </constructor>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The base implementation logs all arguments at the debug level,
+ then returns the passed in config unchanged.]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The action is to signal success by returning the exit code 0.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Subclass of {@link AbstractService} that provides basic implementations
+ of the {@link LaunchableService} methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <!-- start class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <class name="HadoopUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="HadoopUncaughtExceptionHandler" type="java.lang.Thread.UncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance delegating to the supplied handler if
+ the exception is considered "simple".
+ @param delegate a delegate exception handler.]]>
+      </doc>
+    </constructor>
+    <constructor name="HadoopUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Basic exception handler -logs simple exceptions, then continues.]]>
+      </doc>
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thread" type="java.lang.Thread"/>
+      <param name="exception" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Uncaught exception handler.
+ If an error is raised: shutdown
+ The state of the system is unknown at this point -attempting
+ a clean shutdown is dangerous. Instead: exit
+ @param thread thread that failed
+ @param exception the raised exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ in the main entry point. 
+
+ The base class will always attempt to shut down the process if an Error
+ was raised; the behavior on a standard Exception, raised outside 
+ process shutdown, is simply to log it. 
+
+ (Based on the class {@code YarnUncaughtExceptionHandler})]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <!-- start interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <interface name="LaunchableService"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Propagate the command line arguments.
+ <p>
+ This method is called before {@link #init(Configuration)};
+ Any non-null configuration that is returned from this operation
+ becomes the one that is passed on to that {@link #init(Configuration)}
+ operation.
+ <p>
+ This permits implementations to change the configuration before
+ the init operation. As the ServiceLauncher only creates
+ an instance of the base {@link Configuration} class, it is
+ recommended to instantiate any subclass (such as YarnConfiguration)
+ that injects new resources.
+ <p>
+ @param config the initial configuration build up by the
+ service launcher.
+ @param args list of arguments passed to the command line
+ after any launcher-specific commands have been stripped.
+ @return the configuration to init the service with.
+ Recommended: pass down the config parameter with any changes
+ @throws Exception any problem]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Run a service. This method is called after {@link Service#start()}.
+ <p>
+ The return value becomes the exit code of the launched process.
+ <p>
+ If an exception is raised, the policy is:
+ <ol>
+   <li>Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}:
+   the exception is passed up unmodified.
+   </li>
+   <li>Any exception which implements
+   {@link org.apache.hadoop.util.ExitCodeProvider}:
+   A new {@link ServiceLaunchException} is created with the exit code
+   and message of the thrown exception; the thrown exception becomes the
+   cause.</li>
+   <li>Any other exception: a new {@link ServiceLaunchException} is created
+   with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and
+   the message of the original exception (which becomes the cause).</li>
+ </ol>
+ @return the exit code
+ @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed
+  up as the exit code and error text.
+ @throws Exception any exception to report. If it provides an exit code
+ this is used in a wrapping exception.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface which services can implement to have their
+ execution managed by the ServiceLauncher.
+ <p>
+ The command line options will be passed down before the 
+ {@link Service#init(Configuration)} operation is invoked via an
+ invocation of {@link LaunchableService#bindArgs(Configuration, List)}
+ After the service has been successfully started via {@link Service#start()}
+ the {@link LaunchableService#execute()} method is called to execute the 
+ service. When this method returns, the service launcher will exit, using
+ the return code from the method as its exit option.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <!-- start interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <interface name="LauncherExitCodes"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="EXIT_SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Success: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FAIL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generic "false/fail" response: {@value}.
+ The operation worked but the result was not "true" from the viewpoint
+ of the executed code.]]>
+      </doc>
+    </field>
+    <field name="EXIT_CLIENT_INITIATED_SHUTDOWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a client requested service termination: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_TASK_LAUNCH_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when targets could not be launched: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_INTERRUPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a control-C, kill -3, signal was picked up: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_OTHER_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when something happened but we can't be specific: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_COMMAND_ARGUMENT_ERROR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the command line doesn't parse: {@value}, or
+ when it is otherwise invalid.
+ <p>
+ Approximate HTTP equivalent: {@code 400 Bad Request}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNAUTHORIZED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The request requires user authentication: {@value}.
+ <p>
+ approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]>
+      </doc>
+    </field>
+    <field name="EXIT_USAGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a usage message was printed: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FORBIDDEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Forbidden action: {@value}.
+ <p>
+ Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_FOUND" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Something was not found: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 404: Not Found}]]>
+      </doc>
+    </field>
+    <field name="EXIT_OPERATION_NOT_ALLOWED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The operation is not allowed: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 405: Not allowed}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_ACCEPTABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The command is somehow not acceptable: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 406: Not Acceptable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_CONNECTIVITY_PROBLEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code on connectivity problems: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 408: Request Timeout}]]>
+      </doc>
+    </field>
+    <field name="EXIT_BAD_CONFIGURATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the configurations in valid/incomplete: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 409: Conflict}]]>
+      </doc>
+    </field>
+    <field name="EXIT_EXCEPTION_THROWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when an exception was thrown from the service: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 500 Internal Server Error}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNIMPLEMENTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Unimplemented feature: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 501: Not Implemented}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Service Unavailable; it may be available later: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 503 Service Unavailable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNSUPPORTED_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The application does not support, or refuses to support this
+ version: {@value}.
+ <p>
+ If raised, this is expected to be raised server-side and likely due
+ to client/server version incompatibilities.
+ <p>
+ Approximate HTTP equivalent: {@code 505: Version Not Supported}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_CREATION_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_LIFECYCLE_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Common Exit codes.
+ <p>
+ Codes with a YARN prefix are YARN-related.
+ <p>
+ Many of the exit codes are designed to resemble HTTP error codes,
+ squashed into a single byte. e.g 44 , "not found" is the equivalent
+ of 404. The various 2XX HTTP error codes aren't followed;
+ the Unix standard of "0" for success is used.
+ <pre>
+    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <!-- start class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+  <class name="ServiceLaunchException" extends="org.apache.hadoop.util.ExitUtil.ExitException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <implements name="org.apache.hadoop.service.launcher.LauncherExitCodes"/>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code.
+ @param exitCode exit code
+ @param cause cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code and text.
+ @param exitCode exit code
+ @param message message to use in exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ <p>
+ If the last argument is a throwable, it becomes the cause of the exception.
+ It will also be used as a parameter for the format.
+ @param exitCode exit code
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A service launch exception that includes an exit code.
+ <p>
+ When caught by the ServiceLauncher, it will convert that
+ into a process exit code.
+ 
+ The {@link #ServiceLaunchException(int, String, Object...)} constructor
+ generates formatted exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+</package>
+<package name="org.apache.hadoop.tools">
+</package>
+<package name="org.apache.hadoop.tools.protocolPB">
+</package>
+<package name="org.apache.hadoop.tracing">
+  <!-- start class org.apache.hadoop.tracing.SpanReceiverInfo -->
+  <class name="SpanReceiverInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClassName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.tracing.SpanReceiverInfo -->
+  <!-- start class org.apache.hadoop.tracing.SpanReceiverInfoBuilder -->
+  <class name="SpanReceiverInfoBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SpanReceiverInfoBuilder" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addConfigurationPair"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="build" return="org.apache.hadoop.tracing.SpanReceiverInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.tracing.SpanReceiverInfoBuilder -->
+  <!-- start interface org.apache.hadoop.tracing.TraceAdminProtocol -->
+  <interface name="TraceAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="listSpanReceivers" return="org.apache.hadoop.tracing.SpanReceiverInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the currently active trace span receivers.
+ 
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <method name="addSpanReceiver" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desc" type="org.apache.hadoop.tracing.SpanReceiverInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a new trace span receiver.
+ 
+ @param desc                The span receiver description.
+ @return                    The ID of the new trace span receiver.
+
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <method name="removeSpanReceiver"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="spanReceiverId" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove a trace span receiver.
+
+ @param spanReceiverId      The id of the span receiver to remove.
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides tracing.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.tracing.TraceAdminProtocol -->
+  <!-- start interface org.apache.hadoop.tracing.TraceAdminProtocolPB -->
+  <interface name="TraceAdminProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.tracing.TraceAdminPB.TraceAdminService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.tracing.TraceAdminProtocolPB -->
+</package>
+<package name="org.apache.hadoop.util">
+  <!-- start class org.apache.hadoop.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="java.net.URLClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="resolve" type="boolean"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="isSystemClass" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="systemClasses" type="java.util.List"/>
+      <doc>
+      <![CDATA[Checks if a class should be included as a system class.
+
+ A class is a system class if and only if it matches one of the positive
+ patterns and none of the negative ones.
+
+ @param name the class name to check
+ @param systemClasses a list of system class configurations.
+ @return true if the class is a system class]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_CLASSES_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of the system classes if the user did not override them.
+ JDK classes, hadoop classes and resources, and some select third-party
+ classes are considered system classes, and are not loaded by the
+ application classloader.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A {@link URLClassLoader} for application isolation. Classes from the
+ application JARs are loaded in preference to the parent loader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ApplicationClassLoader -->
+  <!-- start interface org.apache.hadoop.util.IPList -->
+  <interface name="IPList"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="isIn" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ipAddress" type="java.lang.String"/>
+      <doc>
+      <![CDATA[returns true if the ipAddress is in the IPList.
+ @param ipAddress
+ @return boolean value indicating whether the ipAddress is in the IPList]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.IPList -->
+  <!-- start interface org.apache.hadoop.util.Progressable -->
+  <interface name="Progressable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="progress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Report progress to the Hadoop framework.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A facility for reporting progress.
+ 
+ <p>Clients and/or applications can use the provided <code>Progressable</code>
+ to explicitly report progress to the Hadoop framework. This is especially
+ important for operations which take significant amount of time since,
+ in-lieu of the reported progress, the framework has to assume that an error
+ has occured and time-out the operation.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Progressable -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32 -->
+  <class name="PureJavaCrc32" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the same polynomial as the built-in native CRC32.
+
+ This is to avoid the JNI overhead for certain uses of Checksumming
+ where many small pieces of data are checksummed in succession.
+
+ The current version is ~10x to 1.8x as fast as Sun's native
+ java.util.zip.CRC32 in Java 1.6
+
+ @see java.util.zip.CRC32]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32 -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32C -->
+  <class name="PureJavaCrc32C" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32C"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the CRC32-C polynomial, the same polynomial used by iSCSI
+ and implemented on many Intel chipsets supporting SSE4.2.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32C -->
+  <!-- start class org.apache.hadoop.util.ReflectionUtils -->
+  <class name="ReflectionUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReflectionUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theObject" type="java.lang.Object"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Check and set 'configuration' if necessary.
+ 
+ @param theObject object for which to set configuration
+ @param conf Configuration]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create an object for the given class and initialize it from conf
+ 
+ @param theClass class of which an object is created
+ @param conf Configuration
+ @return a new object]]>
+      </doc>
+    </method>
+    <method name="setContentionTracing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="printThreadInfo"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.PrintStream"/>
+      <param name="title" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Print all of the thread's information and stack traces.
+ 
+ @param stream the stream to
+ @param title a string title for the stack trace]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="T"/>
+      <doc>
+      <![CDATA[Return the correctly-typed {@link Class} of the given object.
+  
+ @param o object whose correctly-typed <code>Class</code> is to be obtained
+ @return the correctly typed <code>Class</code> of the given object.]]>
+      </doc>
+    </method>
+    <method name="copy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="src" type="T"/>
+      <param name="dst" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param src the object to copy from
+ @param dst the object to copy into, which is destroyed
+ @return dst param (the copy)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cloneWritableInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDeclaredFieldsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared fields of a class including fields declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredMethodsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared methods of a class including methods declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[General reflection utils]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ReflectionUtils -->
+  <!-- start class org.apache.hadoop.util.Shell -->
+  <class name="Shell" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Shell"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with no minimum interval between runs; stderr is
+ not merged with stdout.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with a minimum interval between executions; stderr is
+ not merged with stdout.
+ @param interval interval in milliseconds between command executions.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a shell instance which can be re-executed when the {@link #run()}
+ method is invoked with a given elapsed time between calls.
+
+ @param interval the minimum duration in milliseconds to wait before
+        re-executing the command. If set to 0, there is no minimum.
+ @param redirectErrorStream should the error stream be merged with
+        the normal output stream?]]>
+      </doc>
+    </constructor>
+    <method name="isJava7OrAbove" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="This call isn't needed any more: please remove uses of it.">
+      <doc>
+      <![CDATA[query to see if system is Java 7 or later.
+ Now that Hadoop requires Java 7 or later, this always returns true.
+ @deprecated This call isn't needed any more: please remove uses of it.
+ @return true, always.]]>
+      </doc>
+    </method>
+    <method name="checkWindowsCommandLineLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if a given command (String[]) fits in the Windows maximum command
+ line length Note that the input is expected to already include space
+ delimiters, no extra count will be added for delimiters.
+
+ @param commands command parts, including any space delimiters]]>
+      </doc>
+    </method>
+    <method name="getGroupsCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's groups list.]]>
+      </doc>
+    </method>
+    <method name="getGroupsForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's groups list.
+ If the OS is not WINDOWS, the command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.]]>
+      </doc>
+    </method>
+    <method name="getGroupsIDForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's group id list.
+ The command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.
+ This command does not support Windows and will only return group names.]]>
+      </doc>
+    </method>
+    <method name="getUsersForNetgroupCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="netgroup" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given netgroup's user list.]]>
+      </doc>
+    </method>
+    <method name="getGetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a command to get permission information.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Return a command to set permission.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set permission for specific file.
+
+ @param perm String permission to set
+ @param recursive boolean true to apply to all sub-directories recursively
+ @param file String file to set
+ @return String[] containing command and arguments]]>
+      </doc>
+    </method>
+    <method name="getSetOwnerCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set owner.]]>
+      </doc>
+    </method>
+    <method name="getSymlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to create symbolic links.]]>
+      </doc>
+    </method>
+    <method name="getReadlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to read the target of the a symbolic link.]]>
+      </doc>
+    </method>
+    <method name="getCheckProcessIsAliveCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command for determining if process with specified pid is alive.
+ @param pid process ID
+ @return a <code>kill -0</code> command or equivalent]]>
+      </doc>
+    </method>
+    <method name="getSignalKillCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to send a signal to a given pid.]]>
+      </doc>
+    </method>
+    <method name="getEnvironmentVariableRegex" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a regular expression string that match environment variables.]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.io.File"/>
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a File referencing a script with the given basename, inside the
+ given parent directory.  The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param parent File parent directory
+ @param basename String script file basename
+ @return File referencing the script in the directory]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a script file name with the given basename.
+
+ The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param basename String script file basename
+ @return String script file name]]>
+      </doc>
+    </method>
+    <method name="getRunScriptCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="script" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns a command to run the given script.  The script interpreter is
+ inferred by platform: cmd on Windows or bash otherwise.
+
+ @param script File script to run
+ @return String[] command to run the script]]>
+      </doc>
+    </method>
+    <method name="getHadoopHome" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the Hadoop home directory. Raises an exception if not found
+ @return the home dir
+ @throws IOException if the home directory cannot be located.]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBin" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBinPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist
+ @throws IOException on path canonicalization failures]]>
+      </doc>
+    </method>
+    <method name="hasWinutilsPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate to indicate whether or not the path to winutils is known.
+
+ If true, then {@link #WINUTILS} is non-null, and both
+ {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()}
+ will successfully return this value. Always false on non-windows systems.
+ @return true if there is a valid path to the binary]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Locate the winutils binary, or fail with a meaningful
+ exception and stack trace as an RTE.
+ This method is for use in methods which don't explicitly throw
+ an <code>IOException</code>.
+ @return the path to {@link #WINUTILS_EXE}
+ @throws RuntimeException if the path is not resolvable]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Get a file reference to winutils.
+ Always raises an exception if there isn't one
+ @return the file instance referring to the winutils bin.
+ @throws FileNotFoundException on any failure to locate that file.]]>
+      </doc>
+    </method>
+    <method name="checkIsBashSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+    </method>
+    <method name="setEnvironment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the environment for the command.
+ @param env Mapping of environment variables]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Set the working directory.
+ @param dir The directory where the command will be executed]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check to see if a command needs to be executed and execute if needed.]]>
+      </doc>
+    </method>
+    <method name="getExecString" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return an array containing the command name and its parameters.]]>
+      </doc>
+    </method>
+    <method name="parseExecResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="lines" type="java.io.BufferedReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the execution result]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an environment variable.
+ @param env the environment var
+ @return the value or null if it was unset.]]>
+      </doc>
+    </method>
+    <method name="getProcess" return="java.lang.Process"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the current sub-process executing the given command.
+ @return process executing the command]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the exit code.
+ @return the exit code of the process]]>
+      </doc>
+    </method>
+    <method name="getWaitingThread" return="java.lang.Thread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the thread that is waiting on this instance of <code>Shell</code>.
+ @return the thread that ran runCommand() that spawned this shell
+ or null if no thread is waiting for this shell to complete]]>
+      </doc>
+    </method>
+    <method name="isTimedOut" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[To check if the passed script to shell command executor timed out or
+ not.
+
+ @return if the script timed out.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param cmd shell command to execute.
+ @return the output of the executed command.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <param name="timeout" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @param timeout time in milliseconds after which script should be marked timeout
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="destroyAllShellProcesses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to destroy all running <code>Shell</code> processes.
+ Iterates through a map of all currently running <code>Shell</code>
+ processes and destroys them one by one. This method is thread safe]]>
+      </doc>
+    </method>
+    <method name="getAllShells" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to return a Set of all <code>Shell</code> objects.]]>
+      </doc>
+    </method>
+    <method name="getMemlockLimit" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ulimit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Static method to return the memory lock limit for datanode.
+ @param ulimit max value at which memory locked should be capped.
+ @return long value specifying the memory lock limit.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSPROP_HADOOP_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[System property for the Hadoop home directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ENV_HADOOP_HOME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable for Hadoop's home dir: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum command line length in Windows
+ KB830473 documents this as 8191]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGHT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use the correctly spelled constant.">
+      <doc>
+      <![CDATA[mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
+ @deprecated use the correctly spelled constant.]]>
+      </doc>
+    </field>
+    <field name="USER_NAME_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's name: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WindowsProcessLaunchLock" type="java.lang.Object"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Windows <code>CreateProcess</code> synchronization object.]]>
+      </doc>
+    </field>
+    <field name="osType" type="org.apache.hadoop.util.Shell.OSType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the operating system, as determined from parsing
+ the <code>os.name</code> property.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SOLARIS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FREEBSD" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OTHER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PPC_64" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENV_NAME_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Regular expression for environment variables: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_PERMISSION_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set permission: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_OWNER_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set owner: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_GROUP_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set the change user's groups list: {@value}.]]>
+      </doc>
+    </field>
+    <field name="LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to create a link: {@value}.]]>
+      </doc>
+    </field>
+    <field name="READ_LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get a link target: {@value}.]]>
+      </doc>
+    </field>
+    <field name="timeOutInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time after which the executing script would be timedout.]]>
+      </doc>
+    </field>
+    <field name="inheritParentEnv" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if the parent env vars should be inherited or not]]>
+      </doc>
+    </field>
+    <field name="WINUTILS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}">
+      <doc>
+      <![CDATA[Location of winutils as a string; null if not found.
+ <p>
+ <i>Important: caller must check for this value being null</i>.
+ The lack of such checks has led to many support issues being raised.
+ <p>
+ @deprecated use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]>
+      </doc>
+    </field>
+    <field name="isSetsidAvailable" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag which is true if setsid exists.]]>
+      </doc>
+    </field>
+    <field name="TOKEN_SEPARATOR_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Token separator regex used to parse Shell tool outputs.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for running a Shell command.
+
+ <code>Shell</code> can be used to run shell commands like <code>du</code> or
+ <code>df</code>. It also offers facilities to gate commands by
+ time-intervals.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.Shell -->
+  <!-- start class org.apache.hadoop.util.ShutdownHookManager -->
+  <class name="ShutdownHookManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="get" return="org.apache.hadoop.util.ShutdownHookManager"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return <code>ShutdownHookManager</code> singleton.
+
+ @return <code>ShutdownHookManager</code> singleton.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority, the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority and timeout the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order. The shutdown hook will be terminated if it
+ has not been finished in the specified period of time.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook
+ @param timeout timeout of the shutdownHook
+ @param unit unit of the timeout <code>TimeUnit</code>]]>
+      </doc>
+    </method>
+    <method name="removeShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Removes a shutdownHook.
+
+ @param shutdownHook shutdownHook to remove.
+ @return TRUE if the shutdownHook was registered and removed,
+ FALSE otherwise.]]>
+      </doc>
+    </method>
+    <method name="hasShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Indicates if a shutdownHook is registered or not.
+
+ @param shutdownHook shutdownHook to check if registered.
+ @return TRUE/FALSE depending if the shutdownHook is is registered.]]>
+      </doc>
+    </method>
+    <method name="isShutdownInProgress" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if shutdown is in progress or not.
+ 
+ @return TRUE if the shutdown is in progress, otherwise FALSE.]]>
+      </doc>
+    </method>
+    <method name="clearShutdownHooks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[clear all registered shutdownHooks.]]>
+      </doc>
+    </method>
+    <field name="TIMEOUT_MINIMUM" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum shutdown timeout: {@value} second(s).]]>
+      </doc>
+    </field>
+    <field name="TIME_UNIT_DEFAULT" type="java.util.concurrent.TimeUnit"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default time unit used: seconds.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The <code>ShutdownHookManager</code> enables running shutdownHook
+ in a deterministic order, higher priority first.
+ <p/>
+ The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
+ This class registers a single JVM shutdownHook and run all the
+ shutdownHooks registered to it (to this class) in order based on their
+ priority.
+
+ Unless a hook was registered with a shutdown explicitly set through
+ {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ the shutdown time allocated to it is set by the configuration option
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in
+ {@code core-site.xml}, with a default value of
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ seconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ShutdownHookManager -->
+  <!-- start class org.apache.hadoop.util.StringInterner -->
+  <class name="StringInterner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StringInterner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="strongIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains strong reference to the instance, 
+ thus preventing it from being garbage-collected. 
+ 
+ @param sample string instance to be interned
+ @return strong reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="weakIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains weak reference to the instance, 
+ and so does not prevent it from being garbage-collected.
+ 
+ @param sample string instance to be interned
+ @return weak reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="internStringsInArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="strings" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Interns all the strings in the given array in place,
+ returning the same array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides string interning utility methods. For weak interning,
+ we use the standard String.intern() call, that performs very well
+ (no problems with PermGen overflowing, etc.) starting from JDK 7.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.StringInterner -->
+  <!-- start class org.apache.hadoop.util.SysInfo -->
+  <class name="SysInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SysInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.util.SysInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return default OS instance.
+ @throws UnsupportedOperationException If cannot determine OS.
+ @return Default instance for the detected OS.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the virtual memory present in the system.
+
+ @return virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the physical memory present in the system.
+
+ @return physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailableVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available virtual memory present
+ in the system.
+
+ @return available virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailablePhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available physical memory present
+ in the system.
+
+ @return available physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getNumProcessors" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total number of logical processors present on the system.
+
+ @return number of logical processors]]>
+      </doc>
+    </method>
+    <method name="getNumCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain total number of physical cores present on the system.
+
+ @return number of physical cores]]>
+      </doc>
+    </method>
+    <method name="getCpuFrequency" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU frequency of on the system.
+
+ @return CPU frequency in kHz]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the cumulative CPU time since the system is on.
+
+ @return cumulative CPU time in milliseconds]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU usage % of the machine. Return -1 if it is unavailable
+
+ @return CPU usage as a percentage (from 0 to 100) of available cycles.]]>
+      </doc>
+    </method>
+    <method name="getNumVCoresUsed" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the number of VCores used. Return -1 if it is unavailable
+
+ @return Number of VCores used a percentage (from 0 to #VCores).]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read over the network.
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to the network.
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read from disks.
+
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to disks.
+
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Plugin to calculate resource information on the system.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.SysInfo -->
+  <!-- start interface org.apache.hadoop.util.Tool -->
+  <interface name="Tool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Execute the command with the given arguments.
+ 
+ @param args command specific arguments.
+ @return exit code.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A tool interface that supports handling of generic command-line options.
+ 
+ <p><code>Tool</code>, is the standard for any Map-Reduce tool/application. 
+ The tool/application should delegate the handling of 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ standard command-line options</a> to {@link ToolRunner#run(Tool, String[])} 
+ and only handle its custom arguments.</p>
+ 
+ <p>Here is how a typical <code>Tool</code> is implemented:</p>
+ <p><blockquote><pre>
+     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // <code>Configuration</code> processed by <code>ToolRunner</code>
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed <code>conf</code>
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let <code>ToolRunner</code> handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+ </pre></blockquote></p>
+ 
+ @see GenericOptionsParser
+ @see ToolRunner]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Tool -->
+  <!-- start class org.apache.hadoop.util.ToolRunner -->
+  <class name="ToolRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ToolRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the given <code>Tool</code> by {@link Tool#run(String[])}, after 
+ parsing with the given generic arguments. Uses the given 
+ <code>Configuration</code>, or builds one if null.
+ 
+ Sets the <code>Tool</code>'s configuration with the possibly modified 
+ version of the <code>conf</code>.  
+ 
+ @param conf <code>Configuration</code> for the <code>Tool</code>.
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the <code>Tool</code> with its <code>Configuration</code>.
+ 
+ Equivalent to <code>run(tool.getConf(), tool, args)</code>.
+ 
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="printGenericCommandUsage"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.PrintStream"/>
+      <doc>
+      <![CDATA[Prints generic command-line argurments and usage information.
+ 
+  @param out stream to write usage information to.]]>
+      </doc>
+    </method>
+    <method name="confirmPrompt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prompt" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print out a prompt to the user, and return true if the user
+ responds with "y" or "yes". (case insensitive)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A utility to help run {@link Tool}s.
+ 
+ <p><code>ToolRunner</code> can be used to run classes implementing 
+ <code>Tool</code> interface. It works in conjunction with 
+ {@link GenericOptionsParser} to parse the 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ generic hadoop command line arguments</a> and modifies the 
+ <code>Configuration</code> of the <code>Tool</code>. The 
+ application-specific options are passed along without being modified.
+ </p>
+ 
+ @see Tool
+ @see GenericOptionsParser]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ToolRunner -->
+  <!-- start class org.apache.hadoop.util.VersionInfo -->
+  <class name="VersionInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionInfo" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="_getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Hadoop version.
+ @return the Hadoop version string, eg. "0.6.3-dev"]]>
+      </doc>
+    </method>
+    <method name="getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Git commit hash of the repository when compiled.
+ @return the commit hash, eg. "18f64065d5db6208daf50b02c1b5ed4ee3ce547a"]]>
+      </doc>
+    </method>
+    <method name="getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the branch on which this originated.
+ @return The branch name, e.g. "trunk" or "branches/branch-0.20"]]>
+      </doc>
+    </method>
+    <method name="getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The date that Hadoop was compiled.
+ @return the compilation date in unix date format]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user that compiled Hadoop.
+ @return the username of the user]]>
+      </doc>
+    </method>
+    <method name="getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL for the Hadoop repository.
+ @return the URL of the Hadoop repository]]>
+      </doc>
+    </method>
+    <method name="getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the checksum of the source files from which Hadoop was built.
+ @return the checksum of the source files]]>
+      </doc>
+    </method>
+    <method name="getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the buildVersion which includes version,
+ revision, user and date.
+ @return the buildVersion]]>
+      </doc>
+    </method>
+    <method name="getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the protoc version used for the build.
+ @return the protoc version]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <doc>
+    <![CDATA[This class returns build information about Hadoop components.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.VersionInfo -->
+  <doc>
+  <![CDATA[Common utilities.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.util.bloom">
+  <!-- start class org.apache.hadoop.util.bloom.BloomFilter -->
+  <class name="BloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="BloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVectorSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return size of the the bloomfilter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>Bloom filter</i>, as defined by Bloom in 1970.
+ <p>
+ The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by 
+ the networking research community in the past decade thanks to the bandwidth efficiencies that it
+ offers for the transmission of set membership information between networked hosts.  A sender encodes 
+ the information into a bit vector, the Bloom filter, that is more compact than a conventional 
+ representation. Computation and space costs for construction are linear in the number of elements.  
+ The receiver uses the filter to test whether various elements are members of the set. Though the 
+ filter will occasionally return a false positive, it will never return a false negative. When creating 
+ the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. 
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=362692&dl=ACM&coll=portal">Space/Time Trade-Offs in Hash Coding with Allowable Errors</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.BloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <class name="CountingBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CountingBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="CountingBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Removes a specified key from <i>this</i> counting Bloom filter.
+ <p>
+ <b>Invariant</b>: nothing happens if the specified key does not belong to <i>this</i> counter Bloom filter.
+ @param key The key to remove.]]>
+      </doc>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="approximateCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[This method calculates an approximate count of the key, i.e. how many
+ times the key was added to the filter. This allows the filter to be
+ used as an approximate <code>key -&gt; count</code> map.
+ <p>NOTE: due to the bucket size of this filter, inserting the same
+ key more than 15 times will cause an overflow at all filter positions
+ associated with this key, and it will significantly increase the error
+ rate for this and other keys. For this reason the filter can only be
+ used to store small count values <code>0 &lt;= N &lt;&lt; 15</code>.
+ @param key key to be tested
+ @return 0 if the key is not present. Otherwise, a positive value v will
+ be returned such that <code>v == count</code> with probability equal to the
+ error rate of this filter, and <code>v &gt; count</code> otherwise.
+ Additionally, if the filter experienced an underflow as a result of
+ {@link #delete(Key)} operation, the return value may be lower than the
+ <code>count</code> with the probability of the false negative rate of such
+ filter.]]>
+      </doc>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>counting Bloom filter</i>, as defined by Fan et al. in a ToN
+ 2000 paper.
+ <p>
+ A counting Bloom filter is an improvement to standard a Bloom filter as it
+ allows dynamic additions and deletions of set membership information.  This 
+ is achieved through the use of a counting vector instead of a bit vector.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=343571.343572">Summary cache: a scalable wide-area web cache sharing protocol</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <class name="DynamicBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DynamicBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zero-args constructor for the serialization.]]>
+      </doc>
+    </constructor>
+    <constructor name="DynamicBloomFilter" type="int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds an empty Dynamic Bloom filter.
+ @param vectorSize The number of bits in the vector.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).
+ @param nr The threshold for the maximum number of keys to record in a
+ dynamic Bloom filter row.]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>dynamic Bloom filter</i>, as defined in the INFOCOM 2006 paper.
+ <p>
+ A dynamic Bloom filter (DBF) makes use of a <code>s * m</code> bit matrix but
+ each of the <code>s</code> rows is a standard Bloom filter. The creation 
+ process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
+ bit matrix, i.e., it is composed of a single standard Bloom filter.
+ It assumes that <code>n<sub>r</sub></code> elements are recorded in the 
+ initial bit vector, where <code>n<sub>r</sub> <= n</code> (<code>n</code> is
+ the cardinality of the set <code>A</code> to record in the filter).  
+ <p>
+ As the size of <code>A</code> grows during the execution of the application,
+ several keys must be inserted in the DBF.  When inserting a key into the DBF,
+ one must first get an active Bloom filter in the matrix.  A Bloom filter is
+ active when the number of recorded keys, <code>n<sub>r</sub></code>, is 
+ strictly less than the current cardinality of <code>A</code>, <code>n</code>.
+ If an active Bloom filter is found, the key is inserted and 
+ <code>n<sub>r</sub></code> is incremented by one. On the other hand, if there
+ is no active Bloom filter, a new one is created (i.e., a new row is added to
+ the matrix) according to the current size of <code>A</code> and the element
+ is added in this new Bloom filter and the <code>n<sub>r</sub></code> value of
+ this new Bloom filter is set to one.  A given key is said to belong to the
+ DBF if the <code>k</code> positions are set to one in one of the matrix rows.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ 
+ @see <a href="http://www.cse.fau.edu/~jie/research/publications/Publication_files/infocom2006.pdf">Theory and Network Applications of Dynamic Bloom Filters</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.HashFunction -->
+  <class name="HashFunction" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashFunction" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds a hash function that must obey to a given maximum number of returned values and a highest value.
+ @param maxValue The maximum highest returned value.
+ @param nbHash The number of resulting hashed values.
+ @param hashType type of the hashing function (see {@link Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears <i>this</i> hash function. A NOOP]]>
+      </doc>
+    </method>
+    <method name="hash" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Hashes a specified key into several integers.
+ @param k The specified key.
+ @return The array of hashed values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements a hash object that returns a certain number of hashed values.
+ 
+ @see Key The general behavior of a key being stored in a filter
+ @see Filter The general behavior of a filter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.HashFunction -->
+  <!-- start interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <interface name="RemoveScheme"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="RANDOM" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Random selection.
+ <p>
+ The idea is to randomly select a bit to reset.]]>
+      </doc>
+    </field>
+    <field name="MINIMUM_FN" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MinimumFN Selection.
+ <p>
+ The idea is to select the bit to reset that will generate the minimum
+ number of false negative.]]>
+      </doc>
+    </field>
+    <field name="MAXIMUM_FP" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MaximumFP Selection.
+ <p>
+ The idea is to select the bit to reset that will remove the maximum number
+ of false positive.]]>
+      </doc>
+    </field>
+    <field name="RATIO" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Ratio Selection.
+ <p>
+ The idea is to select the bit to reset that will, at the same time, remove
+ the maximum number of false positve while minimizing the amount of false
+ negative generated.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Defines the different remove scheme for retouched Bloom filters.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <!-- start class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+  <class name="RetouchedBloomFilter" extends="org.apache.hadoop.util.bloom.BloomFilter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.bloom.RemoveScheme"/>
+    <constructor name="RetouchedBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="RetouchedBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Adds a false positive information to <i>this</i> retouched Bloom filter.
+ <p>
+ <b>Invariant</b>: if the false positive is <code>null</code>, nothing happens.
+ @param key The false positive key to add.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="coll" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Adds a collection of false positive information to <i>this</i> retouched Bloom filter.
+ @param coll The collection of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="java.util.List"/>
+      <doc>
+      <![CDATA[Adds a list of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The list of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="org.apache.hadoop.util.bloom.Key[]"/>
+      <doc>
+      <![CDATA[Adds an array of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The array of false positive.]]>
+      </doc>
+    </method>
+    <method name="selectiveClearing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <param name="scheme" type="short"/>
+      <doc>
+      <![CDATA[Performs the selective clearing for a given key.
+ @param k The false positive key to remove from <i>this</i> retouched Bloom filter.
+ @param scheme The selective clearing scheme to apply.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>retouched Bloom filter</i>, as defined in the CoNEXT 2006 paper.
+ <p>
+ It allows the removal of selected false positives at the cost of introducing
+ random false negatives, and with the benefit of eliminating some random false
+ positives at the same time.
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ @see RemoveScheme The different selective clearing algorithms
+ 
+ @see <a href="http://www-rp.lip6.fr/site_npa/site_rp/_publications/740-rbf_cameraready.pdf">Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+</package>
+<package name="org.apache.hadoop.util.curator">
+</package>
+<package name="org.apache.hadoop.util.hash">
+</package>
+
+</api>
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml
new file mode 100644
index 0000000000000..10a4f0d5f16e5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml
@@ -0,0 +1,35426 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:50:33 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop Common 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-common-project/hadoop-common/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/slf4j/slf4j-reload4j/1.7.35/slf4j-reload4j-1.7.35.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-common-project/hadoop-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -apidir /build/source/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 3.2.4 -->
+<package name="org.apache.hadoop">
+  <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+  <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message detailed message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+  <!-- start interface org.apache.hadoop.conf.Configurable -->
+  <interface name="Configurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.conf.Configurable -->
+  <!-- start class org.apache.hadoop.conf.Configuration -->
+  <class name="Configuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Iterable"/>
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration where the behavior of reading from the default 
+ resources can be turned off.
+ 
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files. 
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration with the same settings cloned from another.
+ 
+ @param other the configuration from which to clone settings.]]>
+      </doc>
+    </constructor>
+    <method name="addDeprecations"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+      <doc>
+      <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless.  It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context.  If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas   The deprecations to add.]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+ 
+ @param key
+ @param newKeys
+ @param customMessage
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKey
+ @param customMessage]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+      </doc>
+    </method>
+    <method name="isDeprecated" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[checks whether the given <code>key</code> is deprecated.
+ 
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and 
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setDeprecatedProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+      </doc>
+    </method>
+    <method name="reloadExistingConfigurations"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload existing configuration instances.]]>
+      </doc>
+    </method>
+    <method name="addDefaultResource"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a default resource. Resources are loaded in the order of the resources 
+ added.
+ @param name file name. File should be present in the classpath.]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemPropertiesDefault"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="setRestrictSystemProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param name resource to be added, the classpath is examined for a file 
+             with that name.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param url url of the resource to be added, the local filesystem is 
+            examined directly to find the resource, without referring to 
+            the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param file file-path of resource to be added, the local filesystem is
+             examined directly to find the resource, without referring to 
+             the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ WARNING: The contents of the InputStream will be cached, by this method. 
+ So use this sparingly because it does increase the memory consumption.
+ 
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next.  After it is read the stream will be
+ closed.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+      </doc>
+    </method>
+    <method name="reloadConfiguration"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added 
+ resources, and final parameters. This will make the resources to 
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="setAllowNullValueProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+      <doc>
+      <![CDATA[Set Configuration to allow keys without values during setup.  Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemProps"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="onlyKeyExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>null</code> if no such property exists. 
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>defaultValue</code> if no such property exists. 
+ See @{Configuration#getTrimmed} for more details.
+ 
+ @param name          the property name.
+ @param defaultValue  the property default value.
+ @return              the value of the <code>name</code> or defaultValue
+                      if it is not set.]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is 
+ deprecated, it returns the value of the first key which replaces 
+ the deprecated key and is not null.
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> property or 
+         its replacing property and null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="source" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from 
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+      </doc>
+    </method>
+    <method name="unset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unset a previously set property.]]>
+      </doc>
+    </method>
+    <method name="setIfUnset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+ 
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property 
+         doesn't exist.]]>
+      </doc>
+    </method>
+    <method name="getInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="int"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+   
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInts" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+ 
+ If no such property exists, an empty array is returned.
+ 
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+         <code>int</code> values]]>
+      </doc>
+    </method>
+    <method name="setInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+ 
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getLongBytes" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+ 
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="float"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getBoolean" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="boolean"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.  
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setBoolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+ 
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="setBooleanIfUnset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="setEnum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="T"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="getEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="T"/>
+      <doc>
+      <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided]]>
+      </doc>
+    </method>
+    <method name="setTimeDuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
+ provided, the default unit is applied.
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param defaultUnit Default time unit if no valid suffix is provided.
+ @param returnUnit The unit used for the returned value.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDurationHelper" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="vStr" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param vStr The string value with time unit suffix to be converted.
+ @param unit Unit to convert the stored property, if it exists.]]>
+      </doc>
+    </method>
+    <method name="getTimeDurations" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets the Storage Size from the config, or returns the defaultValue. The
+ unit of return value is specified in target unit.
+
+ @param name - Key Name
+ @param defaultValue - Default Value -- e.g. 100MB
+ @param targetUnit - The units that we want result to be in.
+ @return double -- formatted in target Units]]>
+      </doc>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets storage size from a config file.
+
+ @param name - Key to read.
+ @param defaultValue - The default value to return in case the key is
+ not present.
+ @param targetUnit - The Storage unit that should be used
+ for the return value.
+ @return - double value in the Storage Unit specified.]]>
+      </doc>
+    </method>
+    <method name="setStorageSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <param name="unit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Sets Storage Size for the specified key.
+
+ @param name - Key to set.
+ @param value - The numeric value to set.
+ @param unit - Storage Unit to be used.]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="pattern" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+      </doc>
+    </method>
+    <method name="getPropertySources" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Gets information about why a property was set.  Typically this is the 
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programmatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise, 
+ returns a list of the sources of the resource.  The older sources are
+ the first ones in the list.  So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+      </doc>
+    </method>
+    <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+      </doc>
+    </method>
+    <method name="getStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s.  
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+ 
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then <code>null</code> is returned.
+ 
+ @param name property name.
+ @return property value as an array of <code>String</code>s, 
+         or <code>null</code>.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+ 
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s, 
+         or empty array.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="setStrings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the array of string values for the <code>name</code> property as 
+ as comma delimited values.  
+ 
+ @param name property name.
+ @param values The values]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromCredentialProviders" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromConfig" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="setSocketAddr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address.  This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="getClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Load a class by name.
+ 
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+      </doc>
+    </method>
+    <method name="getClassByNameOrNull" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+ 
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+      </doc>
+    </method>
+    <method name="getClasses" return="java.lang.Class[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class[]"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the class name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+   
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ An exception is thrown if the returned class does not implement the named
+ interface. 
+ 
+ @param name the class name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+ 
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+ 
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+        <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+      </doc>
+    </method>
+    <method name="setClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the name of a 
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+ 
+ An exception is thrown if <code>theClass</code> does not implement the 
+ interface <code>xface</code>. 
+ 
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+      </doc>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link URL} for the named resource.
+ 
+ @param name resource name.
+ @return the url for the named resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsReader" return="java.io.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getFinalParameters" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+      </doc>
+    </method>
+    <method name="getProps" return="java.util.Properties"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all keys from the configuration.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code> 
+ key-value pairs in the configuration.
+ 
+ @return an iterator over the entries.]]>
+      </doc>
+    </method>
+    <method name="getPropsWithPrefix" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration and includes all properties that
+ start with the specified configuration prefix.  Property names in the
+ mapping are trimmed to remove the configuration prefix.
+
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties with prefix stripped]]>
+      </doc>
+    </method>
+    <method name="addTags"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prop" type="java.util.Properties"/>
+      <doc>
+      <![CDATA[Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
+ @param prop]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+ 
+ @param out the output stream to write to.]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the
+ given {@link Writer}.
+
+ <li>
+ When property name is not empty and the property exists in the
+ configuration, this method writes the property and its attributes
+ to the {@link Writer}.
+ </li>
+ <p>
+
+ <li>
+ When property name is null or empty, this method writes all the
+ configuration properties and their attributes to the {@link Writer}.
+ </li>
+ <p>
+
+ <li>
+ When property name is not empty but the property doesn't exist in
+ the configuration, this method throws an {@link IllegalArgumentException}.
+ </li>
+ <p>
+ @param out the writer to write to.]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes properties and their attributes (final and resource)
+  to the given {@link Writer}.
+
+  <li>
+  When propertyName is not empty, and the property exists
+  in the configuration, the format of the output would be,
+  <pre>
+  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is null or empty, it behaves same as
+  {@link #dumpConfiguration(Configuration, Writer)}, the
+  output would be,
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is not empty, and the property is not
+  found in the configuration, this method will throw an
+  {@link IllegalArgumentException}.
+  </li>
+  <p>
+ @param config the configuration
+ @param propertyName property name
+ @param out the Writer to write to
+ @throws IOException
+ @throws IllegalArgumentException when property name is not
+   empty and the property is not found in configuration]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes out all properties and their attributes (final and resource) to
+  the given {@link Writer}, the format of the output would be,
+
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+
+  It does not output the properties of the configuration object which
+  is loaded from an input stream.
+  <p>
+
+ @param config the configuration
+ @param out the Writer to write to
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClassLoader" return="java.lang.ClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+      </doc>
+    </method>
+    <method name="setClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classLoader" type="java.lang.ClassLoader"/>
+      <doc>
+      <![CDATA[Set the class loader that will be used to load the various objects.
+ 
+ @param classLoader the new class loader.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQuietMode"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quietmode" type="boolean"/>
+      <doc>
+      <![CDATA[Set the quietness-mode. 
+ 
+ In the quiet-mode, error and informational messages might not be logged.
+ 
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+              to turn it off.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[For debugging.  List non-default properties to the terminal and exit.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getValByRegex" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="regex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[get keys matching the the regex 
+ @param regex
+ @return Map<String,String> with matching keys]]>
+      </doc>
+    </method>
+    <method name="dumpDeprecatedKeys"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasWarnedDeprecation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTag" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get all properties belonging to tag.
+ @param tag tag
+ @return Properties with matching tag]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTags" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get all properties belonging to list of input tags. Calls
+ getAllPropertiesByTag internally.
+ @param tagList list of input tags
+ @return Properties with matching tags]]>
+      </doc>
+    </method>
+    <method name="isPropertyTag" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagStr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get Property tag Enum corresponding to given source.
+
+ @param tagStr String representation of Enum
+ @return true if tagStr is a valid tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides access to configuration parameters.
+
+ <h4 id="Resources">Resources</h4>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a 
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
+ then the classpath is examined for a file with that name.  If named by a 
+ <code>Path</code>, then the local filesystem is examined directly, without 
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two 
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+ 
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>. 
+ Once a resource declares a value final, no subsequently-loaded 
+ resource can alter that value.  
+ For example, one might define a final parameter with:
+ <tt><pre>
+  &lt;property&gt;
+    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
+    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
+    <b>&lt;final&gt;true&lt;/final&gt;</b>
+  &lt;/property&gt;</pre></tt>
+
+ Administrators typically define parameters as final in 
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Environment variables in {@link System#getenv()} if a name starts with
+ "env.", or</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions: 
+ <tt><pre>
+  &lt;property&gt;
+    &lt;name&gt;basedir&lt;/name&gt;
+    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
+  &lt;/property&gt;
+  
+  &lt;property&gt;
+    &lt;name&gt;tempdir&lt;/name&gt;
+    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;otherdir&lt;/name&gt;
+    &lt;value&gt;${<i>env.BASE_DIR</i>}/other&lt;/value&gt;
+  &lt;/property&gt;
+  </pre></tt>
+
+ <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ <p>When <tt>conf.get("otherdir")</tt> is called, then <tt>${<i>env.BASE_DIR</i>}</tt>
+ will be resolved to the value of the <tt>${<i>BASE_DIR</i>}</tt> environment variable.
+ It supports <tt>${<i>env.NAME:-default</i>}</tt> and <tt>${<i>env.NAME-default</i>}</tt> notations.
+ The former is resolved to "default" if <tt>${<i>NAME</i>}</tt> environment variable is undefined
+ or its value is empty.
+ The latter behaves the same way only if <tt>${<i>NAME</i>}</tt> is undefined.
+ <p>By default, warnings will be given to any deprecated configuration 
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.
+
+ <h4 id="Tags">Tags</h4>
+
+ <p>Optionally we can tag related properties together by using tag
+ attributes. System tags are defined by hadoop.tags.system property. Users
+ can define there own custom tags in  hadoop.tags.custom property.
+
+ <p>For example, we can tag existing property as:
+ <tt><pre>
+  &lt;property&gt;
+    &lt;name&gt;dfs.replication&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,REQUIRED&lt;/tag&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;dfs.data.transfer.protection&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,SECURITY&lt;/tag&gt;
+  &lt;/property&gt;
+ </pre></tt>
+ <p> Properties marked with tags can be retrieved with <tt>conf
+ .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
+ (Arrays.asList("YARN","SECURITY"))</tt>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configuration -->
+  <!-- start class org.apache.hadoop.conf.Configured -->
+  <class name="Configured" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="Configured"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configured -->
+  <doc>
+  <![CDATA[Configuration of system parameters.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.crypto">
+</package>
+<package name="org.apache.hadoop.crypto.key">
+  <!-- start class org.apache.hadoop.crypto.key.KeyProvider -->
+  <class name="KeyProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="KeyProvider" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ 
+ @param conf configuration for the provider]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the provider configuration.
+ 
+ @return the provider configuration]]>
+      </doc>
+    </method>
+    <method name="options" return="org.apache.hadoop.crypto.key.KeyProvider.Options"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[A helper function to create an options object.
+ @param conf the configuration to use
+ @return a new options object]]>
+      </doc>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide access to
+ keying material rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getKeyVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for a specific version of the key. This method is used
+ when decrypting data.
+ @param versionName the name of a specific version of the key
+ @return the key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeys" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key names for all keys.
+ @return the list of key names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeysMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get key metadata in bulk.
+ @param names the names of the keys to get
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeyVersions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for all versions of a specific key name.
+ @return the list of key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current version of the key, which should be used for encrypting new
+ data.
+ @param name the base name of the key
+ @return the version name of the current version of the key or null if the
+    key version doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get metadata about the key.
+ @param name the basename of the key
+ @return the key's metadata or null if the key doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key. The given key must not already exist.
+ @param name the base name of the key
+ @param material the key material for the first version of the key.
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <param name="algorithm" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <doc>
+      <![CDATA[Generates a key material.
+
+ @param size length of the key.
+ @param algorithm algorithm to use for generating the key.
+ @return the generated key.
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key generating the material for it.
+ The given key must not already exist.
+ <p/>
+ This implementation generates the key material and calls the
+ {@link #createKey(String, byte[], Options)} method.
+
+ @param name the base name of the key
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="deleteKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given key.
+ @param name the name of the key to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key.
+ @param name the basename of the key
+ @param material the new key material
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to close any resources
+ that require closing]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key generating the material for it.
+ <p/>
+ This implementation generates the key material and calls the
+ {@link #rollNewVersion(String, byte[])} method.
+
+ @param name the basename of the key
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="invalidateCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to invalidate the caches. This could be
+ used after rollNewVersion to provide a strong guarantee to return the new
+ version of the given key.
+
+ @param name the basename of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the keys are written to persistent store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getBaseName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Split the versionName in to a base name. Converts "/aaa/bbb/3" to
+ "/aaa/bbb".
+ @param versionName the version name to split
+ @return the base name of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="buildVersionName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Build a version string from a basename and version number. Converts
+ "/aaa/bbb" and 3 to "/aaa/bbb@3".
+ @param name the basename of the key
+ @param version the version of the key
+ @return the versionName of the key.]]>
+      </doc>
+    </method>
+    <method name="findProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerList" type="java.util.List"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the provider with the given key.
+ @param providerList the list of providers
+ @param keyName the key name we are looking for
+ @return the KeyProvider that has the key]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_CIPHER_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CIPHER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIALFILTER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIAL_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of secret key material for Hadoop applications. Provides an
+ abstraction to separate key storage from users of encryption. It
+ is intended to support getting or storing keys in a variety of ways,
+ including third party bindings.
+ <P/>
+ <code>KeyProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProvider -->
+  <!-- start class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+  <class name="KeyProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a KeyProvider based on a provided URI.
+
+ @param uri key provider URI
+ @param conf configuration to initialize the key provider
+ @return the key provider for the specified URI, or <code>NULL</code> if
+         a provider for the specified URI scheme could not be found.
+ @throws IOException thrown if the provider failed to initialize.]]>
+      </doc>
+    </method>
+    <field name="KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of KeyProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ KeyProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+</package>
+<package name="org.apache.hadoop.crypto.key.kms">
+</package>
+<package name="org.apache.hadoop.crypto.random">
+</package>
+<package name="org.apache.hadoop.fs">
+  <!-- start class org.apache.hadoop.fs.AbstractFileSystem -->
+  <class name="AbstractFileSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="AbstractFileSystem" type="java.net.URI, java.lang.String, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[Constructor to be called by subclasses.
+ 
+ @param uri for this file system.
+ @param supportedScheme the scheme supported by the implementor
+ @param authorityNeeded if true then theURI must have authority, if false
+          then the URI must have null authority.
+
+ @throws URISyntaxException <code>uri</code> has syntax error]]>
+      </doc>
+    </constructor>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isValidName" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns true if the specified string is considered valid in the path part
+ of a URI by this file system.  The default implementation enforces the rules
+ of HDFS, but subclasses may override this method to implement specific
+ validation rules for specific file systems.
+ 
+ @param src String source filename to check, path part of the URI
+ @return boolean true if the specified string is considered valid]]>
+      </doc>
+    </method>
+    <method name="createFileSystem" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a file system instance for the specified uri using the conf. The
+ conf is used to find the class name that implements the file system. The
+ conf is also passed to the file system for its configuration.
+
+ @param uri URI of the file system
+ @param conf Configuration for the file system
+ 
+ @return Returns the file system for the given URI
+
+ @throws UnsupportedFileSystemException file system for <code>uri</code> is
+           not found]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ 
+ @param uri
+          used as key to lookup STATISTICS_TABLE. Only scheme and authority
+          part of the uri are used.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[The main factory method for creating a file system. Get a file system for
+ the URI's scheme and authority. The scheme of the <code>uri</code>
+ determines a configuration property name,
+ <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
+ AbstractFileSystem class.
+ 
+ The entire URI and conf is passed to the AbstractFileSystem factory method.
+ 
+ @param uri for the file system to be created.
+ @param conf which is passed to the file system impl.
+ 
+ @return file system for the given URI.
+ 
+ @throws UnsupportedFileSystemException if the file system for
+           <code>uri</code> is not supported.]]>
+      </doc>
+    </method>
+    <method name="checkScheme"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="supportedScheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check that the Uri's scheme matches
+ @param uri
+ @param supportedScheme]]>
+      </doc>
+    </method>
+    <method name="getUriDefaultPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default port of this file system.
+ 
+ @return default port of this file system's Uri scheme
+         A uri with a port of -1 => default port;]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.
+ 
+ @return the uri of this file system.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+ 
+ If the path is fully qualified URI, then its scheme and authority
+ matches that of this file system. Otherwise the path must be 
+ slash-relative name.
+ 
+ @throws InvalidPathException if the path is invalid]]>
+      </doc>
+    </method>
+    <method name="getUriPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the path-part of a pathname. Checks that URI matches this file system
+ and that the path-part is a valid name.
+ 
+ @param p path
+ 
+ @return path-part of the Path p]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified to this file system
+ @param path
+ @return the qualified path]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Some file systems like LocalFileSystem have an initial workingDir
+ that is used as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+ 
+ @return the initial workingDir if the file system has such a notion
+         otherwise return a null.]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ 
+ @return current user's home directory.]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ 
+ @return server default configuration values
+ 
+ @throws IOException an I/O error occurred
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param f path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path f resolving the path
+ through any internal symlinks or mount point
+ @param p path to be resolved
+ @return fully qualified path 
+ @throws FileNotFoundException, AccessControlException, IOException
+         UnresolvedLinkException if symbolic link on path cannot be resolved
+          internally]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
+ that the Path f must be fully qualified and the permission is absolute
+ (i.e. umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="createInternal" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
+ have been declared explicitly.]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
+ f must be fully qualified and the permission is absolute (i.e. 
+ umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#delete(Path, boolean)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path, int)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#truncate(Path, long)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setReplication(Path, short)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system and NO OVERWRITE is performed.
+ 
+ File systems that do not have a built in overwrite need implement only this
+ method and can take advantage of the default impl of the other
+ {@link #renameInternal(Path, Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the file system supports symlinks, false otherwise.
+ @return true if filesystem supports symlinks]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of  
+ {@link FileContext#createSymlink(Path, Path, boolean)};]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partially resolves the path. This is used during symlink resolution in
+ {@link FSLinkResolver}, and differs from the similarly named method
+ {@link FileContext#getLinkTarget(Path)}.
+ @throws IOException subclass implementations may throw IOException]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setPermission(Path, FsPermission)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setOwner(Path, String, String)} except that Path f must
+ be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setTimes(Path, long, long)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileChecksum(Path)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileStatus(Path)} 
+ except that an UnresolvedLinkException may be thrown if a symlink is 
+ encountered in the path.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileLinkStatus(Path)}
+ except that an UnresolvedLinkException may be thrown if a symlink is  
+ encountered in the path leading up to the final path component.
+ If the file system does not support symlinks then the behavior is
+ equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileBlockLocations(Path, long, long)} except that
+ Path f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listLocatedStatus(Path)} except that Path f 
+ must be for this file system.
+
+ In HDFS implementation, the BlockLocation of returned LocatedFileStatus
+ will have different formats for replicated and erasure coded file. Please
+ refer to {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext.Util#listStatus(Path)} except that Path f must be 
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical name for this file system.
+ @return a URI string that uniquely identifies this file system]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications, must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator<AclStatus> which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only the xattr names for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#createSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#renameSnapshot(Path, String, String)}.]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="snapshotDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#deleteSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for implementors of a Hadoop file system
+ (analogous to the VFS of Unix). Applications do not access this class;
+ instead they access files across all file systems using {@link FileContext}.
+ 
+ Pathnames passed to AbstractFileSystem can be fully qualified URI that
+ matches the "this" file system (ie same scheme and authority) 
+ or a Slash-relative name that is assumed to be relative
+ to the root of the "this" file system .]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AbstractFileSystem -->
+  <!-- start class org.apache.hadoop.fs.AvroFSInput -->
+  <class name="AvroFSInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.avro.file.SeekableInput"/>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FSDataInputStream, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct given an {@link FSDataInputStream} and its length.]]>
+      </doc>
+    </constructor>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FileContext, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct given a {@link FileContext} and a {@link Path}.]]>
+      </doc>
+    </constructor>
+    <method name="length" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="tell" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AvroFSInput -->
+  <!-- start class org.apache.hadoop.fs.BlockLocation -->
+  <class name="BlockLocation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default Constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="org.apache.hadoop.fs.BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset and length]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset, length and corrupt flag]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset and length]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset, length 
+ and corrupt flag]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], org.apache.hadoop.fs.StorageType[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting this block]]>
+      </doc>
+    </method>
+    <method name="getCachedHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting a cached replica of the block]]>
+      </doc>
+    </method>
+    <method name="getNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of names (IP:xferPort) hosting this block]]>
+      </doc>
+    </method>
+    <method name="getTopologyPaths" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of network topology paths for each of the hosts.
+ The last component of the path is the "name" (IP:xferPort).]]>
+      </doc>
+    </method>
+    <method name="getStorageIds" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storageID of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storage type of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start offset of file associated with this block]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of the block]]>
+      </doc>
+    </method>
+    <method name="isCorrupt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setOffset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="offset" type="long"/>
+      <doc>
+      <![CDATA[Set the start offset of file associated with this block]]>
+      </doc>
+    </method>
+    <method name="setLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[Set the length of block]]>
+      </doc>
+    </method>
+    <method name="setCorrupt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="corrupt" type="boolean"/>
+      <doc>
+      <![CDATA[Set the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hosts" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the hosts hosting this block]]>
+      </doc>
+    </method>
+    <method name="setCachedHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cachedHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the hosts hosting a cached replica of this block]]>
+      </doc>
+    </method>
+    <method name="setNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the names (host:port) hosting this block]]>
+      </doc>
+    </method>
+    <method name="setTopologyPaths"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="topologyPaths" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the network topology paths of the hosts]]>
+      </doc>
+    </method>
+    <method name="setStorageIds"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageIds" type="java.lang.String[]"/>
+    </method>
+    <method name="setStorageTypes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="org.apache.hadoop.fs.StorageType[]"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Represents the network location of a block, information about the hosts
+ that contain block replicas, and other block metadata (E.g. the file
+ offset associated with the block, length, whether it is corrupt, etc).
+
+ For a single BlockLocation, it will have different meanings for replicated
+ and erasure coded files.
+
+ If the file is 3-replicated, offset and length of a BlockLocation represent
+ the absolute value in the file and the hosts are the 3 datanodes that
+ holding the replicas. Here is an example:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ </pre>
+
+ And if the file is erasure-coded, each BlockLocation represents a logical
+ block groups. Value offset is the offset of a block group in the file and
+ value length is the total length of a block group. Hosts of a BlockLocation
+ are the datanodes that holding all the data blocks and parity blocks of a
+ block group.
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ A BlockLocation example will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ </pre>
+
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or
+ {@link FileContext#getFileBlockLocations(Path, long, long)}
+ for more examples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.BlockLocation -->
+  <!-- start interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <interface name="BlockStoragePolicySpi"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the storage policy. Policies are uniquely
+ identified by name.
+
+ @return the name of the storage policy.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the preferred storage types associated with this policy. These
+ storage types are used sequentially for successive block replicas.
+
+ @return preferred storage types used for placing block replicas.]]>
+      </doc>
+    </method>
+    <method name="getCreationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for creating new block replicas. Fallback
+ storage types are used if the preferred storage types are not available.
+
+ @return fallback storage types for new block replicas..]]>
+      </doc>
+    </method>
+    <method name="getReplicationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for replicating existing block replicas.
+ Fallback storage types are used if the preferred storage types are not
+ available.
+
+ @return fallback storage types for replicating existing block replicas.]]>
+      </doc>
+    </method>
+    <method name="isCopyOnCreateFile" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the policy is inherit-only and cannot be changed for
+ an existing file.
+
+ @return true if the policy is inherit-only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A storage policy specifies the placement of block replicas on specific
+ storage types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <interface name="ByteBufferReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to buf.remaining() bytes into buf. Callers should use
+ buf.limit(..) to control the size of the desired read.
+ <p/>
+ After a successful call, buf.position() will be advanced by the number 
+ of bytes read and buf.limit() should be unchanged.
+ <p/>
+ In the case of an exception, the values of buf.position() and buf.limit()
+ are undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p/>
+ Many implementations will throw {@link UnsupportedOperationException}, so
+ callers that are not confident in support for this method from the
+ underlying filesystem should be prepared to handle that exception.
+ <p/>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+
+ @param buf
+          the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if 
+         reach end-of-stream
+ @throws IOException
+           if there is some error performing the read]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a read API that writes to a
+ ByteBuffer, not a byte[].]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <!-- start interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <interface name="CanSetDropBehind"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setDropBehind"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropCache" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Configure whether the stream should drop the cache.
+
+ @param dropCache     Whether to drop the cache.  null means to use the
+                      default value.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting the drop-behind.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <!-- start interface org.apache.hadoop.fs.CanSetReadahead -->
+  <interface name="CanSetReadahead"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setReadahead"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Set the readahead on this stream.
+
+ @param readahead     The readahead to use.  null means to use the default.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting readahead.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetReadahead -->
+  <!-- start interface org.apache.hadoop.fs.CanUnbuffer -->
+  <interface name="CanUnbuffer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="unbuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reduce the buffering.  This will also free sockets and file descriptors
+ held by the stream, if possible.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSDataInputStreams implement this interface to indicate that they can clear
+ their buffers on request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanUnbuffer -->
+  <!-- start class org.apache.hadoop.fs.ChecksumException -->
+  <class name="ChecksumException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumException" type="java.lang.String, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown for checksum errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumException -->
+  <!-- start class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <class name="ChecksumFileSystem" extends="org.apache.hadoop.fs.FilterFileSystem"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApproxChkSumLength" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to verify checksum.]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the raw file system]]>
+      </doc>
+    </method>
+    <method name="getChecksumFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the name of the checksum file associated with a file.]]>
+      </doc>
+    </method>
+    <method name="isChecksumFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return true iff file is a checksum file name.]]>
+      </doc>
+    </method>
+    <method name="getChecksumFileLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="fileSize" type="long"/>
+      <doc>
+      <![CDATA[Return the length of the checksum file given the size of the 
+ actual file.]]>
+      </doc>
+    </method>
+    <method name="getBytesPerSum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the bytes Per Checksum]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChecksumLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <param name="bytesPerSum" type="int"/>
+      <doc>
+      <![CDATA[Calculated the length of the checksum file in bytes.
+ @param size the length of the data file in bytes
+ @param bytesPerSum the number of bytes in a checksum block
+ @return the number of bytes in the checksum file]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename files/dirs]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Implement the delete(Path, boolean) in checksum
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given path
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given patch
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="copyCrc" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ If src and dst are directories, the copyCrc parameter
+ determines whether to copy CRC files.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Report a checksum error to the file system.
+ @param f the file name containing the error
+ @param in the stream open on the file
+ @param inPos the position of the beginning of the bad data in the file
+ @param sums the stream open on the checksum file
+ @param sumsPos the position of the beginning of the bad data in the checksum file
+ @return if retry is necessary]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Disable those operations which the checksummed FS blocks.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abstract Checksumed FileSystem.
+ It provide a basic implementation of a Checksumed FileSystem,
+ which creates a checksum file for each raw file.
+ It generates & verifies checksums at the client side.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <!-- start class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <class name="CommonConfigurationKeysPublic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommonConfigurationKeysPublic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DEFAULT_NAME_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DF_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DU_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_GETSPACEUSED_JITTER_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_CHECKPOINT_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_PROTECTED_DIRECTORIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Directories that cannot be removed unless empty, even by an
+ administrator.]]>
+      </doc>
+    </field>
+    <field name="FS_LOCAL_BLOCK_SIZE_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Not used anywhere, looks like default value for FS_LOCAL_BLOCK_SIZE]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_AUTOMATIC_CLOSE_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_FILE_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_PORT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_ERROR_RATE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_COMPRESSION_CODEC_LZO_CLASS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Codec class that implements Lzo compression algorithm]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_INTERVAL_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_SKIP_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_FILE_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SKIP_CHECKSUM_ERRORS_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SERIALIZATIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_IO_CHUNK_SIZE_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_INPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_OUTPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable low-latency connections from the client]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of IPC_CLIENT_LOW_LATENCY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_LISTEN_QUEUE_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_KILL_MAX_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_IDLETHRESHOLD_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_MAX_CONNECTIONS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Logs if a RPC is really slow compared to rest of RPCs.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SOCKS_SERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_UTIL_HASH_TYPE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_MAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHENTICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHORIZATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SERVICE_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_INTERFACE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_NAMESERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_AUTHENTICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_PROTECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Sasl Properties for a connection]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Impersonation provider]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_SIZE]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_NUM_REFILL_THREADS]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is the number of providers specified.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 100 ms.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 2 secs.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_HTTP_LOGS_ENABLED]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SYSTEM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CUSTOM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TAGS_SYSTEM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_TAGS_CUSTOM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option for the shutdown hook manager shutdown time:
+  {@value}.]]>
+      </doc>
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default shutdown hook timeout: {@value} seconds.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains constants for configuration keys used
+ in the common code.
+
+ It includes all publicly documented configuration keys. In general
+ this class should not be used directly (use CommonConfigurationKeys
+ instead)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <!-- start class org.apache.hadoop.fs.ContentSummary -->
+  <class name="ContentSummary" extends="org.apache.hadoop.fs.QuotaUsage"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ContentSummary"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder
+  This constructor implicitly set spaceConsumed the same as length.
+  spaceConsumed and length must be set explicitly with
+  ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the length]]>
+      </doc>
+    </method>
+    <method name="getSnapshotLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the directory count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the file count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSnapshotSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getErasureCodingPolicy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the header of the output.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="getHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields from the summary header.
+ 
+ @return names of fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="getQuotaHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields used in the quota summary.
+ 
+ @return names of quota fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ if hOption is false, file sizes are returned in bytes
+ if hOption is true, file sizes are returned in human readable
+ if tOption is true, display the quota by storage types
+ if tOption is false, same logic with #toString(boolean,boolean)
+ if xOption is false, output includes the calculation from snapshots
+ if xOption is true, output excludes the calculation from snapshots
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Store the summary of a content (a directory or a file).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ContentSummary -->
+  <!-- start class org.apache.hadoop.fs.CreateFlag -->
+  <class name="CreateFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.CreateFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.CreateFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag and throw exception if it is invalid
+ @param flag set of CreateFlag
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.Object"/>
+      <param name="pathExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for create operation
+ @param path Object representing the path; usually String or {@link Path}
+ @param pathExists pass true if the path exists in the file system
+ @param flag set of CreateFlag
+ @throws IOException on error
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validateForAppend"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for the append operation. The flag must contain
+ APPEND, and cannot contain OVERWRITE.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[CreateFlag specifies the file create semantic. Users can combine flags like: <br>
+ <code>
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
+ <code>
+ <p>
+ 
+ Use the CreateFlag as follows:
+ <ol>
+ <li> CREATE - to create a file if it does not exist, 
+ else throw FileAlreadyExists.</li>
+ <li> APPEND - to append to a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> OVERWRITE - to truncate a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> CREATE|APPEND - to create a file if it does not exist, 
+ else append to an existing file.</li>
+ <li> CREATE|OVERWRITE - to create a file if it does not exist, 
+ else overwrite an existing file.</li>
+ <li> SYNC_BLOCK - to force closed blocks to the disk device.
+ In addition {@link Syncable#hsync()} should be called after each write,
+ if true synchronous behavior is required.</li>
+ <li> LAZY_PERSIST - Create the block on transient storage (RAM) if
+ available.</li>
+ <li> APPEND_NEWBLOCK - Append data to a new block instead of end of the last
+ partial block.</li>
+ </ol>
+ 
+ Following combinations are not valid and will result in
+ {@link HadoopIllegalArgumentException}:
+ <ol>
+ <li> APPEND|OVERWRITE</li>
+ <li> CREATE|APPEND|OVERWRITE</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CreateFlag -->
+  <!-- start class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.fs.FileChecksum -->
+  <class name="FileChecksum" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileChecksum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAlgorithmName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The checksum algorithm name]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The length of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The value of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if both the algorithms and the values are the same.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An abstract class representing file checksums for files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileChecksum -->
+  <!-- start class org.apache.hadoop.fs.FileContext -->
+  <class name="FileContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the file system of supplied path.
+ 
+ @param absOrFqPath - absolute or fully qualified path
+ @return the file system of the path
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>absOrFqPath</code> is not supported.
+ @throws IOException If the file system for <code>absOrFqPath</code> could
+         not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a FileContext with specified FS as default using the specified
+ config.
+ 
+ @param defFS
+ @param aConf
+ @return new FileContext with specified FS as default.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="defaultFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified file system using the default config.
+ 
+ @param defaultFS
+ @return a FileContext with the specified AbstractFileSystem
+                 as the default FS.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the default config read from the
+ $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
+ from core-defaults.xml in the release jar.
+ 
+ @throws UnsupportedFileSystemException If the file system from the default
+           configuration is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@return a FileContext for the local file system using the default config.
+ @throws UnsupportedFileSystemException If the file system for
+           {@link FsConstants#LOCAL_FS_URI} is not supported.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified URI using the default config.
+ 
+ @param defaultFsUri
+ @return a FileContext with the specified URI as the default FS.
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>defaultFsUri</code> is not supported]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified default URI using the specified config.
+ 
+ @param defaultFsUri
+ @param aConf
+ @return new FileContext for specified uri
+ @throws UnsupportedFileSystemException If the file system with specified is
+           not supported
+ @throws RuntimeException If the file system specified is supported but
+         could not be instantiated, or if login fails.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the passed config. Generally it is better to use
+ {@link #getFileContext(URI, Configuration)} instead of this one.
+ 
+ 
+ @param aConf
+ @return new FileContext
+ @throws UnsupportedFileSystemException If file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@param aConf - from which the FileContext is configured
+ @return a FileContext for the local file system using the specified config.
+ 
+ @throws UnsupportedFileSystemException If default file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newWDir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the working directory for wd-relative names (such a "foo/bar"). Working
+ directory feature is provided by simply prefixing relative names with the
+ working dir. Note this is different from Unix where the wd is actually set
+ to the inode. Hence setWorkingDir does not follow symlinks etc. This works
+ better in a distributed environment that has multiple independent roots.
+ {@link #getWorkingDirectory()} should return what setWorkingDir() set.
+ 
+ @param newWDir new working directory
+ @throws IOException 
+ <br>
+           NewWdir can be one of:
+           <ul>
+           <li>relative path: "foo/bar";</li>
+           <li>absolute without scheme: "/foo/bar"</li>
+           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
+           </ul>
+ <br>
+           Illegal WDs:
+           <ul>
+           <li>relative with scheme: "xx:foo/bar"</li>
+           <li>non existent directory</li>
+           </ul>]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the working directory for wd-relative names (such a "foo/bar").]]>
+      </doc>
+    </method>
+    <method name="getUgi" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the ugi in the file-context
+ @return UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ @return the home directory]]>
+      </doc>
+    </method>
+    <method name="getUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the umask of this FileContext]]>
+      </doc>
+    </method>
+    <method name="setUMask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newUmask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set umask to the supplied parameter.
+ @param newUmask  the new umask]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolve the path following any symlinks or mount points
+ @param f to be resolved
+ @return fully qualified resolved path
+ 
+ @throws FileNotFoundException  If <code>f</code> does not exist
+ @throws AccessControlException if access denied
+ @throws IOException If an IO Error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified if it is isn't. 
+ A Fully-qualified path has scheme and authority specified and an absolute
+ path.
+ Use the default file system and working dir in this FileContext to qualify.
+ @param path
+ @return qualified path]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create or overwrite file on indicated path and returns an output stream for
+ writing into the file.
+ 
+ @param f the file name to open
+ @param createFlag gives the semantics of create; see {@link CreateFlag}
+ @param opts file creation options; see {@link Options.CreateOpts}.
+          <ul>
+          <li>Progress - to report progress on the operation - default null
+          <li>Permission - umask is applied against permission: default is
+          FsPermissions:getDefault()
+ 
+          <li>CreateParent - create missing parent path; default is to not
+          to create parents
+          <li>The defaults for the following are SS defaults of the file
+          server implementing the target path. Not all parameters make sense
+          for all kinds of file system - eg. localFS ignores Blocksize,
+          replication, checksum
+          <ul>
+          <li>BufferSize - buffersize used in FSDataOutputStream
+          <li>Blocksize - block size for file blocks
+          <li>ReplicationFactor - replication for blocks
+          <li>ChecksumParam - Checksum parameters. server default is used
+          if not specified.
+          </ul>
+          </ul>
+ 
+ @return {@link FSDataOutputStream} for created file
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>f</code> already exists
+ @throws FileNotFoundException If parent of <code>f</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>f</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ a file on indicated path.
+
+ @param f the file path to create builder for.
+ @return {@link FSDataOutputStreamBuilder} to build a
+         {@link FSDataOutputStream}.
+
+ Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ builder parameters will be verified by {@link FileContext} and
+ {@link AbstractFileSystem#create}. And filesystem states will be modified.
+
+ Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ same exceptions as create(Path, EnumSet, CreateOpts...).]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make(create) a directory and all the non-existent parents.
+ 
+ @param dir - the dir to make
+ @param permission - permissions is set permission&~umask
+ @param createParent - if true then missing parent dirs are created if false
+          then parent must exist
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If directory <code>dir</code> already
+           exists
+ @throws FileNotFoundException If parent of <code>dir</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>dir</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>dir</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+ @param f the path to delete.
+ @param recursive if path is a directory and set to 
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is invalid]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path using
+ default buffersize.
+ @param f the file name to open
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ 
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+ <li>Fails if path is a directory.
+ <li>Fails if path does not exist.
+ <li>Fails if path is not closed.
+ <li>Fails if new size is greater than current size.
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param f file name
+ @param replication new replication
+
+ @return true if successful
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+ <li
+ <li>Fails if src is a file and dst is a directory.
+ <li>Fails if src is a directory and dst is a file.
+ <li>Fails if the parent of dst does not exist or is a file.
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails if the dst
+ already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites the dst if
+ it is a file or an empty directory. Rename fails if dst is a non-empty
+ directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for details
+ <p>
+ 
+ @param src path to be renamed
+ @param dst new path after rename
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If <code>dst</code> already exists and
+           <code>options</options> has {@link Options.Rename#OVERWRITE} 
+           option false.
+ @throws FileNotFoundException If <code>src</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>src</code>
+           and <code>dst</code> is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param f
+ @param permission - the new absolute permission (umask is not applied)
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory). The parameters username
+ and groupname cannot both be null.
+ 
+ @param f The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws HadoopIllegalArgumentException If <code>username</code> or
+           <code>groupname</code> is invalid.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param f The path
+ @param mtime Set the modification time of this file.
+        The number of milliseconds since epoch (Jan 1, 1970). 
+        A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+        The number of milliseconds since Jan 1, 1970. 
+        A value of -1 means that this call should not set access time.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file.
+
+ @param f file path
+
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag for the  file system denoted by the path.
+ This is only applicable if the 
+ corresponding FileSystem supports checksum. By default doesn't do anything.
+ @param verifyChecksum
+ @param f set the verifyChecksum for the Filesystem containing this path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+
+ @return a FileStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path. If the path 
+ refers to a symlink then the FileStatus of the symlink is returned.
+ The behavior is equivalent to #getFileStatus() if the underlying
+ file system does not support symbolic links.
+ @param  f The path we want information from.
+ @return A FileStatus object
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the target of the given symbolic link as it was specified
+ when the link was created.  Links in the path leading up to the
+ final path component are resolved transparently.
+
+ @param f the path to return the target of
+ @return The un-interpreted target of the symbolic link.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If path <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If the given path does not refer to a symlink
+           or an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ file system denoted by the Parh argument p.
+ If the file system has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ 
+ @param f Path for which status should be obtained. null means the
+ root partition of the default file system. 
+
+ @return a FsStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a symbolic link to an existing file. An exception is thrown if 
+ the symlink exits, the user does not have permission to create symlink,
+ or the underlying file system does not support symlinks.
+ 
+ Symlink permissions are ignored, access to a symlink is determined by
+ the permissions of the symlink target.
+ 
+ Symlinks in paths leading up to the final path component are resolved 
+ transparently. If the final path component refers to a symlink some 
+ functions operate on the symlink itself, these are:
+ - delete(f) and deleteOnExit(f) - Deletes the symlink.
+ - rename(src, dst) - If src refers to a symlink, the symlink is 
+   renamed. If dst refers to a symlink, the symlink is over-written.
+ - getLinkTarget(f) - Returns the target of the symlink. 
+ - getFileLinkStatus(f) - Returns a FileStatus object describing
+   the symlink.
+ Some functions, create() and mkdir(), expect the final path component
+ does not exist. If they are given a path that refers to a symlink that 
+ does exist they behave as if the path referred to an existing file or 
+ directory. All other functions fully resolve, ie follow, the symlink. 
+ These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
+ setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
+ getFsStatus, getFileStatus, exists, and listStatus.
+ 
+ Symlink targets are stored as given to createSymlink, assuming the 
+ underlying file system is capable of storing a fully qualified URI.
+ Dangling symlinks are permitted. FileContext supports four types of 
+ symlink targets, and resolves them as follows
+ <pre>
+ Given a path referring to a symlink of form:
+ 
+   <---X---> 
+   fs://host/A/B/link 
+   <-----Y----->
+ 
+ In this path X is the scheme and authority that identify the file system,
+ and Y is the path leading up to the final path component "link". If Y is
+ a symlink  itself then let Y' be the target of Y and X' be the scheme and
+ authority of Y'. Symlink targets may:
+ 
+ 1. Fully qualified URIs
+ 
+ fs://hostX/A/B/file  Resolved according to the target file system.
+ 
+ 2. Partially qualified URIs (eg scheme but no host)
+ 
+ fs:///A/B/file  Resolved according to the target file system. Eg resolving
+                 a symlink to hdfs:///A results in an exception because
+                 HDFS URIs must be fully qualified, while a symlink to 
+                 file:///A will not since Hadoop's local file systems 
+                 require partially qualified URIs.
+ 
+ 3. Relative paths
+ 
+ path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
+       is "../B/file" then [Y'][path] is hdfs://host/B/file
+ 
+ 4. Absolute paths
+ 
+ path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
+       is "/file" then [X][path] is hdfs://host/file
+ </pre>
+ 
+ @param target the target of the symbolic link
+ @param link the path to be created that points to target
+ @param createParent if true then missing parent dirs are created if 
+                     false then parent must exist
+
+
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>linkcode> already exists
+ @throws FileNotFoundException If <code>target</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>link</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for 
+           <code>target</code> or <code>link</code> is not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory. 
+ Return the file's status and block locations If the path is a file.
+ 
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+ If any IO exception (for example the input directory gets deleted while
+ listing is being executed), next() or hasNext() of the returned iterator
+ may throw a RuntimeException with the io exception as the cause.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted on JVM shutdown.
+ 
+ @param f the existing path to delete.
+
+ @return  true if deleteOnExit is successful, otherwise false.
+
+ @throws AccessControlException If access is denied
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="util" return="org.apache.hadoop.fs.FileContext.Util"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolve" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path.
+ Returns the new path object.]]>
+      </doc>
+    </method>
+    <method name="resolveIntermediate" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path leading up 
+ to, but not including the final path component.
+ @param f path to resolve
+ @return the new path object.]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system
+ 
+ @param uri
+          the uri to lookup the statistics. Only scheme and authority part
+          of the uri are used as the key to store and lookup.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all the statistics stored in AbstractFileSystem, for all the file
+ systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints the statistics to standard output. File System is identified by the
+ scheme and authority.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Map of uri and statistics for each filesystem instantiated. The uri
+         consists of scheme and authority for the filesystem.]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications, must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator<AclStatus> which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map<String, byte[]> describing the XAttrs of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p/>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List<String> of the XAttr names of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+
+ @param path The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param path file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path capabilities of the bonded {@code AbstractFileSystem}.
+ @param path path to query the capability of.
+ @param capability string to query the stream support for.
+ @return true iff the capability is supported under that FS.
+ @throws IOException path resolution or other IO failure
+ @throws IllegalArgumentException invalid arguments]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory and symlink
+ In previous versions, this default permission was also used to
+ create files, so files created end up with ugo+x permission.
+ See HADOOP-9155 for detail. 
+ Two new constants are added to solve this, please use 
+ {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ {@link FileContext#FILE_DEFAULT_PERM} for file.
+ This constant is kept for compatibility.]]>
+      </doc>
+    </field>
+    <field name="DIR_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory]]>
+      </doc>
+    </field>
+    <field name="FILE_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for file]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileContext shutdown hook.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The FileContext class provides an interface for users of the Hadoop
+ file system. It exposes a number of file system operations, e.g. create,
+ open, list.
+ 
+ <h2>Path Names</h2>
+ 
+ The Hadoop file system supports a URI namespace and URI names. This enables
+ multiple types of file systems to be referenced using fully-qualified URIs.
+ Two common Hadoop file system implementations are
+ <ul>
+ <li>the local file system: file:///path
+ <li>the HDFS file system: hdfs://nnAddress:nnPort/path
+ </ul>
+ 
+ The Hadoop file system also supports additional naming schemes besides URIs.
+ Hadoop has the concept of a <i>default file system</i>, which implies a
+ default URI scheme and authority. This enables <i>slash-relative names</i>
+ relative to the default FS, which are more convenient for users and
+ application writers. The default FS is typically set by the user's
+ environment, though it can also be manually specified.
+ <p>
+ 
+ Hadoop also supports <i>working-directory-relative</i> names, which are paths
+ relative to the current working directory (similar to Unix). The working
+ directory can be in a different file system than the default FS.
+ <p>
+ Thus, Hadoop path names can be specified as one of the following:
+ <ul>
+ <li>a fully-qualified URI: scheme://authority/path (e.g.
+ hdfs://nnAddress:nnPort/foo/bar)
+ <li>a slash-relative name: path relative to the default file system (e.g.
+ /foo/bar)
+ <li>a working-directory-relative name: path relative to the working dir (e.g.
+ foo/bar)
+ </ul>
+  Relative paths with scheme (scheme:foo/bar) are illegal.
+  
+ <h2>Role of FileContext and Configuration Defaults</h2>
+
+ The FileContext is the analogue of per-process file-related state in Unix. It
+ contains two properties:
+ 
+ <ul>
+ <li>the default file system (for resolving slash-relative names)
+ <li>the umask (for file permissions)
+ </ul>
+ In general, these properties are obtained from the default configuration file
+ in the user's environment (see {@link Configuration}).
+ 
+ Further file system properties are specified on the server-side. File system
+ operations default to using these server-side defaults unless otherwise
+ specified.
+ <p>
+ The file system related server-side defaults are:
+  <ul>
+  <li> the home directory (default is "/user/userName")
+  <li> the initial wd (only for local fs)
+  <li> replication factor
+  <li> block size
+  <li> buffer size
+  <li> encryptDataTransfer 
+  <li> checksum option. (checksumType and  bytesPerChecksum)
+  </ul>
+
+ <h2>Example Usage</h2>
+
+ Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
+   Unspecified values come from core-defaults.xml in the release jar.
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(); // uses the default config
+                                                // which has your default FS 
+  <li>  myFContext.create(path, ...);
+  <li>  myFContext.setWorkingDir(path);
+  <li>  myFContext.open (path, ...);  
+  <li>...
+  </ul>  
+ Example 2: Get a FileContext with a specific URI as the default FS
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(URI);
+  <li> myFContext.create(path, ...);
+  <li>...
+ </ul>
+ Example 3: FileContext with local file system as the default
+  <ul> 
+  <li> myFContext = FileContext.getLocalFSFileContext();
+  <li> myFContext.create(path, ...);
+  <li> ...
+  </ul> 
+ Example 4: Use a specific config, ignoring $HADOOP_CONFIG
+  Generally you should not need use a config unless you are doing
+   <ul> 
+   <li> configX = someConfigSomeOnePassedToYou;
+   <li> myFContext = getFileContext(configX); // configX is not changed,
+                                              // is passed down 
+   <li> myFContext.create(path, ...);
+   <li>...
+  </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileContext -->
+  <!-- start class org.apache.hadoop.fs.FileStatus -->
+  <class name="FileStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for file systems on which symbolic links are not supported]]>
+      </doc>
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="org.apache.hadoop.fs.FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor.
+
+ @param other FileStatus to copy]]>
+      </doc>
+    </constructor>
+    <method name="attributes" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acl" type="boolean"/>
+      <param name="crypt" type="boolean"/>
+      <param name="ec" type="boolean"/>
+      <param name="sn" type="boolean"/>
+      <doc>
+      <![CDATA[Convert boolean attributes to a set of flags.
+ @param acl   See {@link AttrFlags#HAS_ACL}.
+ @param crypt See {@link AttrFlags#HAS_CRYPT}.
+ @param ec    See {@link AttrFlags#HAS_EC}.
+ @param sn    See {@link AttrFlags#SNAPSHOT_ENABLED}.
+ @return converted set of flags.]]>
+      </doc>
+    </method>
+    <method name="getLen" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of this file, in bytes.
+ @return the length of this file, in bytes.]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a file?
+ @return true if this is a file]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a directory?
+ @return true if this is a directory]]>
+      </doc>
+    </method>
+    <method name="isDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.">
+      <doc>
+      <![CDATA[Old interface, instead use the explicit {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ @return true if this is a directory.
+ @deprecated Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.]]>
+      </doc>
+    </method>
+    <method name="isSymlink" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a symbolic link?
+ @return true if this is a symbolic link]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the block size of the file.
+ @return the number of bytes]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the replication factor of a file.
+ @return the replication factor of a file.]]>
+      </doc>
+    </method>
+    <method name="getModificationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modification time of the file.
+ @return the modification time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getAccessTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the access time of the file.
+ @return the access time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get FsPermission associated with the file.
+ @return permission. If a filesystem does not have a notion of permissions
+         or if permissions could not be determined, then default 
+         permissions equivalent of "rwxrwxrwx" is returned.]]>
+      </doc>
+    </method>
+    <method name="hasAcl" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory has ACLs set.
+
+ @return true if the underlying file or directory has ACLs set.]]>
+      </doc>
+    </method>
+    <method name="isEncrypted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is encrypted or not.
+
+ @return true if the underlying file is encrypted.]]>
+      </doc>
+    </method>
+    <method name="isErasureCoded" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is erasure coded or not.
+
+ @return true if the underlying file or directory is erasure coded.]]>
+      </doc>
+    </method>
+    <method name="isSnapshotEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if directory is Snapshot enabled or not.
+
+ @return true if directory is snapshot enabled]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the owner of the file.
+ @return owner of the file. The string could be empty if there is no
+         notion of owner of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getGroup" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group associated with the file.
+ @return group for the file. The string could be empty if there is no
+         notion of group of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Sets permission.
+ @param permission if permission is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets owner.
+ @param owner if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets group.
+ @param group if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return The contents of the symbolic link.]]>
+      </doc>
+    </method>
+    <method name="setSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus.
+ This method was added back by HADOOP-14683 to keep binary compatibility.
+
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.
+ @throws ClassCastException if the specified object is not FileStatus]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read instance encoded as protobuf from stream.
+ @param in Input stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write instance encoded as protobuf to stream.
+ @param out Output stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+    </method>
+    <field name="NONE" type="java.util.Set"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shared, empty set of attributes (a common case for FileStatus).]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface that represents the client side information for a file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileStatus -->
+  <!-- start class org.apache.hadoop.fs.FileSystem -->
+  <class name="FileSystem" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.hadoop.security.token.DelegationTokenIssuer"/>
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="FileSystem"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a FileSystem instance based on the uri, the passed in
+ configuration and the user.
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return the filesystem instance
+ @throws IOException failure to load
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+ somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the configured FileSystem implementation.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="getDefaultUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the default FileSystem URI from a configuration.
+ @param conf the configuration to use
+ @return the uri of the default filesystem]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a FileSystem.
+
+ Called after the new FileSystem instance is constructed, and before it
+ is ready for use.
+
+ FileSystem implementations overriding this method MUST forward it to
+ their superclass, though the order in which it is done, and whether
+ to alter the configuration before the invocation are options of the
+ subclass.
+ @param name a URI whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration
+ @throws IOException on any failure to initialize this instance.
+ @throws IllegalArgumentException if the URI is considered invalid.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for this FileSystem.
+ <p>
+ This implementation throws an <code>UnsupportedOperationException</code>.
+
+ @return the protocol scheme for this FileSystem.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI which identifies this FileSystem.
+
+ @return the URI of this filesystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a canonicalized form of this FileSystem's URI.
+
+ The default implementation simply calls {@link #canonicalizeUri(URI)}
+ on the filesystem's own URI, so subclasses typically only need to
+ implement that method.
+
+ @see #canonicalizeUri(URI)]]>
+      </doc>
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Canonicalize the given URI.
+
+ This is implementation-dependent, and may for example consist of
+ canonicalizing the hostname using DNS and adding the default
+ port if not specified.
+
+ The default implementation simply fills in the default port if
+ not specified and if {@link #getDefaultPort()} returns a
+ default port.
+
+ @return URI
+ @see NetUtils#getCanonicalUri(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FileSystem.
+ @return the default port or 0 if there isn't one]]>
+      </doc>
+    </method>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical service name for this FileSystem.
+ The token cache is the only user of the canonical service name,
+ and uses it to lookup this FileSystem's service tokens.
+ If the file system provides a token of its own then it must have a
+ canonical name, otherwise the canonical name can be null.
+
+ Default implementation: If the FileSystem has child file systems
+ (such as an embedded file system) then it is assumed that the FS has no
+ tokens of its own and hence returns a null name; otherwise a service
+ name is built using Uri and port.
+
+ @return a service string that uniquely identifies this file system, null
+         if the filesystem does not implement tokens
+ @see SecurityUtil#buildDTServiceName(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="call {@link #getUri()} instead.">
+      <doc>
+      <![CDATA[@deprecated call {@link #getUri()} instead.]]>
+      </doc>
+    </method>
+    <method name="getNamed" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="call {@link #get(URI, Configuration)} instead.">
+      <param name="name" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated call {@link #get(URI, Configuration)} instead.]]>
+      </doc>
+    </method>
+    <method name="getLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the local FileSystem.
+ @param conf the configuration to configure the FileSystem with
+ if it is newly instantiated.
+ @return a LocalFileSystem
+ @throws IOException if somehow the local FS cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a FileSystem for this URI's scheme and authority.
+ <ol>
+ <li>
+   If the configuration has the property
+   {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+   a new instance will be created, initialized with the supplied URI and
+   configuration, then returned without being cached.
+ </li>
+ <li>
+   If the there is a cached FS instance matching the same URI, it will
+   be returned.
+ </li>
+ <li>
+   Otherwise: a new FS instance will be created, initialized with the
+   configuration and URI, cached and returned to the caller.
+ </li>
+ </ol>
+ @throws IOException if the FileSystem cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority and the
+ given user. Internally invokes {@link #newInstance(URI, Configuration)}
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return filesystem instance
+ @throws IOException if the FileSystem cannot be instantiated.
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+         somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority.
+ The entire URI is passed to the FileSystem instance's initialize method.
+ This always returns a new FileSystem object.
+ @param uri FS URI
+ @param config configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a unique configured FileSystem implementation for the default
+ filesystem of the supplied configuration.
+ This always returns a new FileSystem object.
+ @param conf the configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstanceLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a unique local FileSystem object.
+ @param conf the configuration to configure the FileSystem with
+ @return a new LocalFileSystem object.
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="closeAll"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances. After this operation, they
+ may not be used in any operations.
+
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="closeAllForUGI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances for a given UGI.
+ Be sure those filesystems are not used anymore.
+ @param ugi user group info to close
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Qualify a path to one which uses this FileSystem and, if relative,
+ made absolute.
+ @param path to qualify.
+ @return this path if it contains a scheme and authority and is absolute, or
+ a new path that includes a path and authority and is fully qualified
+ @see Path#makeQualified(URI, Path)
+ @throws IllegalArgumentException if the path has a schema/URI different
+ from this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file with the provided permission.
+
+ The permission of the file is set to be the provided permission as in
+ setPermission, not permission&~umask
+
+ The HDFS implementation is implemented using two RPCs.
+ It is understood that it is inefficient,
+ but the implementation is thread-safe. The other option is to change the
+ value of umask in configuration to be 0, but it is not thread-safe.
+
+ @param fs FileSystem
+ @param file the name of the file to be created
+ @param permission the permission of the file
+ @return an output stream
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a directory with the provided permission.
+ The permission of the directory is set to be the provided permission as in
+ setPermission, not permission&~umask
+
+ @see #create(FileSystem, Path, FsPermission)
+
+ @param fs FileSystem handle
+ @param dir the name of the directory to be created
+ @param permission the permission of the directory
+ @return true if the directory creation succeeds; false otherwise
+ @throws IOException A problem creating the directories.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+
+ The base implementation performs case insensitive equality checks
+ of the URIs' schemes and authorities. Subclasses may implement slightly
+ different checks.
+ @param path to check
+ @throws IllegalArgumentException if the path is not considered to be
+ part of this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For nonexistent
+ file or regions, {@code null} is returned.
+
+ <pre>
+   if f == null :
+     result = null
+   elif f.getLen() <= start:
+     result = []
+   else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ </pre>
+ This call is most helpful with and distributed filesystem
+ where the hostnames of machines that contain blocks of the given file
+ can be determined.
+
+ The default implementation returns an array containing one element:
+ <pre>
+ BlockLocation( { "localhost:9866" },  { "localhost" }, 0, file.getLen())
+ </pre>
+
+ In HDFS, if file is three-replicated, the returned array contains
+ elements like:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+   hosts: {"host2:9866", "host3:9866, host4:9866"})
+ </pre>
+
+ And if a file is erasure-coded, the returned BlockLocation are logical
+ block groups.
+
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then
+ there will be one BlockLocation returned, with 0 offset, actual file size
+ and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks.
+ 3. If the file size is less than one group size but greater than one
+ stripe size, then there will be one BlockLocation returned, with 0 offset,
+ actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting
+ the actual blocks.
+ 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123
+ for example, then the result will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+   "host4:9866", "host5:9866"})
+ </pre>
+
+ @param file FilesStatus to get data from
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For a nonexistent
+ file or regions, {@code null} is returned.
+
+ This call is most helpful with location-aware distributed
+ filesystems, where it returns hostnames of machines that
+ contain the given file.
+
+ A FileSystem will normally return the equivalent result
+ of passing the {@code FileStatus} of the path to
+ {@link #getFileBlockLocations(FileStatus, long, long)}
+
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @return server default configuration values
+ @throws IOException IO failure
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @return server default configuration values
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path, resolving the path
+ through any symlinks or mount point.
+ @param p path to be resolved
+ @return fully qualified path
+ @throws FileNotFoundException if the path is not present
+ @throws IOException for any other error]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file to open
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If {@link #open(PathHandle, int)}
+                                       not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @param bufferSize the size of the buffer to use
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="getPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Create a durable, serializable handle to the referent of the given
+ entity.
+ @param stat Referent in the target FileSystem
+ @param opt If absent, assume {@link HandleOpt#path()}.
+ @throws IllegalArgumentException If the FileStatus does not belong to
+         this FileSystem
+ @throws UnsupportedOperationException If {@link #createPathHandle}
+         not overridden by subclass.
+ @throws UnsupportedOperationException If this FileSystem cannot enforce
+         the specified constraints.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opt Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an exception will be thrown.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a path with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an {@link FSDataOutputStream} at the indicated Path
+ with write-progress reporting.
+
+ The frequency of callbacks is implementation-specific; it may be "none".
+ @param f the path of the file to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with a custom
+ checksum option.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @param checksumOpt checksum parameter. If null, the values
+        found in conf will be used.
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This create has been added to support the FileContext that processes
+ the permission with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @param f path
+ @param absolutePermission permissions
+ @return true if the directory was actually created.
+ @throws IOException IO failure
+ @see #mkdirs(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNewFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the given Path as a brand-new zero-length file.  If
+ create fails, or if it already existed, return false.
+ <i>Important: the default implementation is not atomic</i>
+ @param f path to use for create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as
+ {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+     IO_FILE_BUFFER_SIZE_DEFAULT), null)}
+ @param f the existing file to be appended.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as append(f, bufferSize, null).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @param progress for reporting progress if it is not null.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Concat existing files together.
+ @param trg the path to the target destination.
+ @param psrcs the paths to the sources to use for the concatenation.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the replication factor.
+
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @param src file name
+ @return file replication
+ @throws FileNotFoundException if the path does not resolve.
+ @throws IOException an IO failure]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the replication for an existing file.
+ If a filesystem does not support replication, it will always
+ return true: the check for a file existing may be bypassed.
+ This is the default behavior.
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful, or the feature in unsupported;
+         false if replication is supported but the file does not exist,
+         or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws IOException on failure
+ @return true if rename is successful]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+   <li>Fails if src is a file and dst is a directory.</li>
+   <li>Fails if src is a directory and dst is a file.</li>
+   <li>Fails if the parent of dst does not exist or is a file.</li>
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails
+ if the dst already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites
+ the dst if it is a file or an empty directory. Rename fails if dst is
+ a non-empty directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for
+ details. This default implementation is non atomic.
+ <p>
+ This method is deprecated since it is a temporary method added to
+ support the transition from FileSystem to FileContext for user
+ applications.
+
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws FileNotFoundException src path does not exist, or the parent
+ path of dst does not exist.
+ @throws FileAlreadyExistsException dest path exists and is a file
+ @throws ParentNotDirectoryException if the parent path of dest is not
+ a directory
+ @throws IOException on failure]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+   <li>Fails if path is a directory.</li>
+   <li>Fails if path does not exist.</li>
+   <li>Fails if path is not closed.</li>
+   <li>Fails if new size is greater than current size.</li>
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #delete(Path, boolean)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file/directory.
+ @deprecated Use {@link #delete(Path, boolean)} instead.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+
+ @param f the path to delete.
+ @param recursive if path is a directory and set to
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+ @return  true if delete is successful else false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted when its FileSystem is closed.
+ When the JVM shuts down cleanly, all cached FileSystem objects will be
+ closed automatically. These the marked paths will be deleted as a result.
+
+ If a FileSystem instance is not cached, i.e. has been created with
+ {@link #createFileSystem(URI, Configuration)}, then the paths will
+ be deleted in when {@link #close()} is called on that instance.
+
+ The path must exist in the filesystem at the time of the method call;
+ it does not have to exist at the time of JVM shutdown.
+
+ Notes
+ <ol>
+   <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+   <li>The time to shut down a FileSystem will depends on the number of
+   files to delete. For filesystems where the cost of checking
+   for the existence of a file/directory and the actual delete operation
+   (for example: object stores) is high, the time to shutdown the JVM can be
+   significantly extended by over-use of this feature.</li>
+   <li>Connectivity problems with a remote filesystem may delay shutdown
+   further, and may cause the files to not be deleted.</li>
+ </ol>
+ @param f the path to delete.
+ @return  true if deleteOnExit is successful, otherwise false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="cancelDeleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Cancel the scheduled deletion of the path when the FileSystem is closed.
+ @param f the path to cancel deletion
+ @return true if the path was found in the delete-on-exit list.]]>
+      </doc>
+    </method>
+    <method name="processDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delete all paths that were marked as delete-on-exit. This recursively
+ deletes all files and directories in the specified paths.
+
+ The time to process this operation is {@code O(paths)}, with the actual
+ time dependent on the time for existence and deletion operations to
+ complete, successfully or not.]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if a path exists.
+
+ It is highly discouraged to call this method back to back with other
+ {@link #getFileStatus(Path)} calls, as this will involve multiple redundant
+ RPC calls in HDFS.
+
+ @param f source path
+ @return true if the path exists
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a directory.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by getFileStatus() or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a regular file.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by {@link #getFileStatus(Path)} or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The number of bytes in a file.
+ @return the number of bytes; 0 for a directory
+ @deprecated Use {@link #getFileStatus(Path)} instead.
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getContentSummary" return="org.apache.hadoop.fs.ContentSummary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link ContentSummary} of a given {@link Path}.
+ @param f path to use
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="org.apache.hadoop.fs.QuotaUsage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link QuotaUsage} of a given {@link Path}.
+ @param f path to use
+ @return the quota usage
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="namespaceQuota" type="long"/>
+      <param name="storagespaceQuota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set quota for the given {@link Path}.
+
+ @param src the target path to set quota for
+ @param namespaceQuota the namespace quota (i.e., # of files/directories)
+                       to set
+ @param storagespaceQuota the storage space quota to set
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuotaByStorageType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <param name="quota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set per storage type quota for the given {@link Path}.
+
+ @param src the target path to set storage type quota for
+ @param type the storage type to set
+ @param quota the quota to set for the given storage type
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+ <p>
+ Will not return null. Expect IOException upon access error.
+ @param f given path
+ @return the statuses of the files/directories in the given patch
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List corrupted file blocks.
+ @return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given path using the user-supplied path
+ filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param f
+          a path name
+ @param filter
+          the user-supplied path filter
+ @return an array of FileStatus objects for the files under the given path
+         after applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using default
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @return a list of statuses for the files under the given paths after
+         applying the filter default Path filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using user-supplied
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @param filter
+          the user-supplied path filter
+ @return a list of statuses for the files under the given paths after
+         applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Return all the files that match filePattern and are not checksum
+ files. Results are sorted by their names.
+
+ <p>
+ A filename pattern is composed of <i>regular</i> characters and
+ <i>special pattern matching</i> characters, which are:
+
+ <dl>
+  <dd>
+   <dl>
+    <p>
+    <dt> <tt> ? </tt>
+    <dd> Matches any single character.
+
+    <p>
+    <dt> <tt> * </tt>
+    <dd> Matches zero or more characters.
+
+    <p>
+    <dt> <tt> [<i>abc</i>] </tt>
+    <dd> Matches a single character from character set
+     <tt>{<i>a,b,c</i>}</tt>.
+
+    <p>
+    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
+    <dd> Matches a single character from the character range
+     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
+     lexicographically less than or equal to character <tt><i>b</i></tt>.
+
+    <p>
+    <dt> <tt> [^<i>a</i>] </tt>
+    <dd> Matches a single character that is not from character set or range
+     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
+     immediately to the right of the opening bracket.
+
+    <p>
+    <dt> <tt> \<i>c</i> </tt>
+    <dd> Removes (escapes) any special meaning of character <i>c</i>.
+
+    <p>
+    <dt> <tt> {ab,cd} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
+
+    <p>
+    <dt> <tt> {ab,c{de,fh}} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
+
+   </dl>
+  </dd>
+ </dl>
+
+ @param pathPattern a glob specifying a path pattern
+
+ @return an array of paths that match the path pattern
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of {@link FileStatus} objects whose path names match
+ {@code pathPattern} and is accepted by the user-supplied path filter.
+ Results are sorted by their path names.
+
+ @param pathPattern a glob specifying the path pattern
+ @param filter a user-supplied path filter
+ @return null if {@code pathPattern} has no glob and the path does not exist
+         an empty array if {@code pathPattern} has a glob and no path
+         matches it else an array of {@link FileStatus} objects matching the
+         pattern
+ @throws IOException if any I/O error occurs when fetching file status]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ Return the file's status and block locations If the path is a file.
+
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List a directory.
+ The returned results include its block location if it is a file
+ The results are filtered by the given path filter
+ @param f a path
+ @param filter a path filter
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+ @throws FileNotFoundException if <code>f</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a remote iterator so that followup calls are made on demand
+ while consuming the entries. Each FileSystem implementation should
+ override this method and provide a more efficient implementation, if
+ possible.
+
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ @param p target path
+ @return remote iterator
+ @throws FileNotFoundException if <code>p</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses and block locations of the files in the given path.
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ <pre>
+ If the path is a directory,
+   if recursive is false, returns files in the directory;
+   if recursive is true, return files in the subtree rooted at the path.
+ If the path is a file, return the file's status and block locations.
+ </pre>
+ @param f is the path
+ @param recursive if the subdirectories need to be traversed recursively
+
+ @return an iterator that traverses statuses of the files
+
+ @throws FileNotFoundException when the path does not exist;
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this FileSystem.
+ The default implementation returns {@code "/user/$USER/"}.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given FileSystem. All relative
+ paths will be resolved relative to it.
+
+ @param new_dir Path of new working directory]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given FileSystem
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Note: with the new FileContext class, getWorkingDirectory()
+ will be removed.
+ The working directory is implemented in FileContext.
+
+ Some FileSystems like LocalFileSystem have an initial workingDir
+ that we use as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+
+ @return if there is built in notion of workingDir then it
+ is returned; else a null is returned.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ @param f path
+ @return true if the directory was created
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make the given file and all non-existent parents into
+ directories. Has roughly the semantics of Unix @{code mkdir -p}.
+ Existence of the directory hierarchy is not an error.
+ @param f path to create
+ @param permission to apply to f
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to filesystem at
+ the given dst name and the source is kept intact afterwards
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files is on the local disk.  Add it to filesystem at
+ the given dst name, removing the source afterwards.
+ @param srcs source paths
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name, removing the source afterwards.
+ @param src local path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param src path
+ @param dst path]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param srcs array of paths which are source
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from the remote filesystem to the local one.
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy a file to the local filesystem, then delete it from the
+ remote filesystem (if successfully copied).
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from a remote filesystem to the local one.
+ delSrc indicates if the src will be removed or not.
+ @param delSrc whether to delete the src
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="useRawLocalFileSystem" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under this filesystem, and the dst is on the local disk.
+ Copy it from the remote filesystem to the local dst name.
+ delSrc indicates if the src will be removed
+ or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
+ as the local file system or not. RawLocalFileSystem is non checksumming,
+ So, It will not create any crc files at local.
+
+ @param delSrc
+          whether to delete the src
+ @param src
+          path
+ @param dst
+          path
+ @param useRawLocalFileSystem
+          whether to use RawLocalFileSystem as local file system or not.
+
+ @throws IOException for any IO error]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local file that the user can write output to.  The caller
+ provides both the eventual target name in this FileSystem
+ and the local working file path.
+ If this FileSystem is local, we write directly into the target.  If
+ the FileSystem is not local, we write into the tmp local area.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path of local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.
+ A local FS will do nothing, because we've written to exactly the
+ right place.
+ A remote FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path to local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this FileSystem instance.
+ Will release any held locks, delete all files queued for deletion
+ through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ from the cache, if cached.
+
+ After this operation, the outcome of any method call on this FileSystem
+ instance, or any input/output stream created by it is <i>undefined</i>.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the block size for a particular file.
+ @param f the filename
+ @return the number of bytes in a block
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @throws FileNotFoundException if the path is not present
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultBlockSize(Path)} instead">
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.
+ @deprecated use {@link #getDefaultBlockSize(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.  The given path will be used to
+ locate the actual filesystem.  The full path does not have to exist.
+ @param f path of file
+ @return the default block size for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultReplication(Path)} instead">
+      <doc>
+      <![CDATA[Get the default replication.
+ @return the replication; the default value is "1".
+ @deprecated use {@link #getDefaultReplication(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the default replication for a path.
+ The given path will be used to locate the actual FileSystem to query.
+ The full path does not have to exist.
+ @param path of the file
+ @return default replication for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+ @return a FileStatus object
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="fixRelativePart" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[See {@link FileContext#fixRelativePart}.]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#createSymlink(Path, Path, boolean)}.]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getFileLinkStatus(Path)}.
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#supportsSymlinks()}.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, if the FS supports checksums.
+
+ @param f The file path
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, from the beginning of the file till the
+ specific length.
+ @param f The file path
+ @param length The length of the file range for checksum calculation
+ @return The file checksum or null if checksums are not supported.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param verifyChecksum Verify checksum flag]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the write checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param writeChecksum Write checksum flag]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the root partition is reflected.
+
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ @param p Path for which status should be obtained. null means
+ the default partition.
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param p The path
+ @param permission permission
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory).
+ The parameters username and groupname cannot both be null.
+ @param p The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param p The path
+ @param mtime Set the modification time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set access time.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+ @param path  The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List<AclEntry> describing modifications
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List describing entries to remove
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List describing modifications, which must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACL of a file or directory.
+
+ @param path Path to get
+ @return AclStatus describing the ACL of the file or directory
+ @throws IOException if an ACL could not be read
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr name and value for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List<String> of the XAttr names of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the root directory of Trash for current user when the path specified
+ is deleted.
+
+ @param path the trash root of the path to be determined.
+ @return the default implementation returns {@code /user/$USER/.Trash}]]>
+      </doc>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+      <doc>
+      <![CDATA[Get all the trash roots for current user or all users.
+
+ @param allUsers return trash roots for all users if true.
+ @return all the trash root directories.
+         Default FileSystem returns .Trash under users' home directories if
+         {@code /user/$USER/.Trash} exists.]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The base FileSystem implementation generally has no knowledge
+ of the capabilities of actual implementations.
+ Unless it has a way to explicitly determine the capabilities,
+ this method returns false.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getFileSystemClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the FileSystem implementation class of a filesystem.
+ This triggers a scan and load of all FileSystem implementations listed as
+ services and discovered via the {@link ServiceLoader}
+ @param scheme URL scheme of FS
+ @param conf configuration: can be null, in which case the check for
+ a filesystem binding declaration in the configuration is skipped.
+ @return the filesystem
+ @throws UnsupportedFileSystemException if there was no known implementation
+         for the scheme.
+ @throws IOException if the filesystem could not be loaded]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Get the Map of Statistics object indexed by URI Scheme.
+ @return a Map having a key as URI scheme and value as Statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Return the FileSystem classes that have Statistics.
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ @param cls the class to lookup
+ @return a statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print all statistics for all file systems to {@code System.out}]]>
+      </doc>
+    </method>
+    <method name="areSymlinksEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="enableSymlinks"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStorageStatistics" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the StorageStatistics for this FileSystem object.  These statistics are
+ per-instance.  They are not shared with any other FileSystem object.
+
+ <p>This is a default method which is intended to be overridden by
+ subclasses. The default implementation returns an empty storage statistics
+ object.</p>
+
+ @return    The StorageStatistics for this FileSystem instance.
+            Will never be null.]]>
+      </doc>
+    </method>
+    <method name="getGlobalStorageStatistics" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a new FSDataOutputStreamBuilder for the file with path.
+ Files are overwritten by default.
+
+ @param path file path
+ @return a FSDataOutputStreamBuilder object to build the file
+
+ HADOOP-14384. Temporarily reduce the visibility of method before the
+ builder interface becomes stable.]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a Builder to append a file.
+ @param path file path.
+ @return a {@link FSDataOutputStreamBuilder} to build file append request.]]>
+      </doc>
+    </method>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG" type="org.apache.commons.logging.Log"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This log is widely used in the org.apache.hadoop.fs code and tests,
+ so must be considered something to only be changed with care.]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileSystem shutdown hook: {@value}.]]>
+      </doc>
+    </field>
+    <field name="TRASH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for trash directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="USER_HOME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An abstract base class for a fairly generic filesystem.  It
+ may be implemented as a distributed filesystem, or as a "local"
+ one that reflects the locally-connected disk.  The local version
+ exists for small Hadoop instances and for testing.
+
+ <p>
+
+ All user code that may potentially use the Hadoop Distributed
+ File System should be written to use a FileSystem object or its
+ successor, {@link FileContext}.
+
+ <p>
+ The local implementation is {@link LocalFileSystem} and distributed
+ implementation is DistributedFileSystem. There are other implementations
+ for object stores and (outside the Apache Hadoop codebase),
+ third party filesystems.
+ <p>
+ Notes
+ <ol>
+ <li>The behaviour of the filesystem is
+ <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ specified in the Hadoop documentation. </a>
+ However, the normative specification of the behavior of this class is
+ actually HDFS: if HDFS does not behave the way these Javadocs or
+ the specification in the Hadoop documentations define, assume that
+ the documentation is incorrect.
+ </li>
+ <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ <li>The term {@code filesystem} refers to the distributed/local filesystem
+ itself, rather than the class used to interact with it.</li>
+ <li>The term "file" refers to a file in the remote filesystem,
+ rather than instances of {@code java.io.File}.</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileSystem -->
+  <!-- start class org.apache.hadoop.fs.FileUtil -->
+  <class name="FileUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileUtil"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path
+
+ @param stats
+          an array of FileStatus objects
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path.
+ If stats if null, return path
+ @param stats
+          an array of FileStatus objects
+ @param path
+          default path to return in stats is null
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <doc>
+      <![CDATA[Register all files recursively to be deleted on exit.
+ @param file File/directory to be deleted]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.
+ @param dir the file or directory to be deleted
+ @param tryGrantPermissions true if permissions should be modified to delete a file.
+ @return true on success false on failure.]]>
+      </doc>
+    </method>
+    <method name="readLink" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns the target of the given symlink. Returns the empty string if
+ the given path does not refer to a symlink or there is an error
+ accessing the symlink.
+ @param f File representing the symbolic link.
+ @return The target of the symbolic link, empty string on error or if not
+         a symlink.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.
+ @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ and all the underlying directories before trying to delete their contents.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link FileSystem#delete(Path, boolean)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recursively delete a directory.
+
+ @param fs {@link FileSystem} on which the path is present
+ @param dir directory to recursively delete
+ @throws IOException
+ @deprecated Use {@link FileSystem#delete(Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcStatus" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy local files to a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="java.io.File"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy FileSystem files to local files.]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param filename The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeSecureShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell
+ and avoids script injection attacks.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="makeCanonicalPath" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @param makeCanonicalPath
+          Whether to make canonical path for the file passed
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="getDU" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Takes an input dir and returns the du on that local directory. Very basic
+ implementation.
+
+ @param dir
+          The input dir to get the disk space of this local dir
+ @return The total disk space of the input local directory]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="toDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a stream input it will unzip the it in the unzip directory.
+ passed as the second parameter
+ @param inputStream The zip file as input
+ @param toDir The unzip directory where to unzip the zip file.
+ @throws IOException an exception occurred]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="unzipDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a File input it will unzip it in the unzip directory.
+ passed as the second parameter
+ @param inFile The zip file as input
+ @param unzipDir The unzip directory where to unzip the zip file.
+ @throws IOException An I/O exception has occurred]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="untarDir" type="java.io.File"/>
+      <param name="gzipped" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ExecutionException" type="java.util.concurrent.ExecutionException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inputStream The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @param gzipped The input stream is gzipped
+                TODO Use magic number and PusbackInputStream to identify
+ @throws IOException an exception occurred
+ @throws InterruptedException command interrupted
+ @throws ExecutionException task submit failed]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="untarDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inFile The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="symLink" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="linkname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a soft link between a src and destination
+ only on a local disk. HDFS does not support this.
+ On Windows, when symlink creation fails due to security
+ setting, we will log a warning. The return code in this
+ case is 2.
+
+ @param target the target for symlink
+ @param linkname the symlink
+ @return 0 on success]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Change the permissions on a filename.
+ @param filename the name of the file to change
+ @param perm the permission string
+ @return the exit code from the command
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Change the permissions on a file / directory, recursively, if
+ needed.
+ @param filename name of the file whose permissions are to change
+ @param perm permission string
+ @param recursive true, if permissions should be changed recursively
+ @return the exit code from the command.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the ownership on a file / directory. User name and group name
+ cannot both be null.
+ @param file the file to change
+ @param username the new user owner name
+ @param groupname the new group owner name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setReadable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="readable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setReadable(boolean)}
+ File#setReadable does not work as expected on Windows.
+ @param f input file
+ @param readable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setWritable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="writable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setWritable(boolean)}
+ File#setWritable does not work as expected on Windows.
+ @param f input file
+ @param writable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setExecutable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="executable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setExecutable(boolean)}
+ File#setExecutable does not work as expected on Windows.
+ Note: revoking execute permission on folders does not have the same
+ behavior on Windows as on Unix platforms. Creating, deleting or renaming
+ a file within that folder will still succeed on Windows.
+ @param f input file
+ @param executable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="canRead" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canRead()}
+ @param f input file
+ @return On Unix, same as {@link File#canRead()}
+         On Windows, true if process has read access on the path]]>
+      </doc>
+    </method>
+    <method name="canWrite" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canWrite()}
+ @param f input file
+ @return On Unix, same as {@link File#canWrite()}
+         On Windows, true if process has write access on the path]]>
+      </doc>
+    </method>
+    <method name="canExecute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canExecute()}
+ @param f input file
+ @return On Unix, same as {@link File#canExecute()}
+         On Windows, true if process has execute access on the path]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permissions to the required value. Uses the java primitives instead
+ of forking if group == other.
+ @param f the file to change
+ @param permission the new permissions
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createLocalTempFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="basefile" type="java.io.File"/>
+      <param name="prefix" type="java.lang.String"/>
+      <param name="isDeleteOnExit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a tmp file for a base file.
+ @param basefile the base file of the tmp
+ @param prefix file name prefix of tmp
+ @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
+ @return a newly created tmp file
+ @exception IOException If a tmp file cannot created
+ @see java.io.File#createTempFile(String, String, File)
+ @see java.io.File#deleteOnExit()]]>
+      </doc>
+    </method>
+    <method name="replaceFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="target" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the src file to the name specified by target.
+ @param src the source file
+ @param target the target file
+ @exception IOException If this operation fails]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="java.io.File[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#listFiles()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#listFiles() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of files or empty list
+ @exception IOException for invalid directory or for a bad disk.]]>
+      </doc>
+    </method>
+    <method name="list" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#list()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#list() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of file names or empty string list
+ @exception AccessDeniedException for unreadable directory
+ @exception IOException for invalid directory or for bad disk]]>
+      </doc>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="targetDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a jar file at the given path, containing a manifest with a classpath
+ that references all specified entries.
+
+ Some platforms may have an upper limit on command line length.  For example,
+ the maximum command line length on Windows is 8191 characters, but the
+ length of the classpath may exceed this.  To work around this limitation,
+ use this method to create a small intermediate jar with a manifest that
+ contains the full classpath.  It returns the absolute path to the new jar,
+ which the caller may set as the classpath for a new process.
+
+ Environment variable evaluation is not supported within a jar manifest, so
+ this method expands environment variables before inserting classpath entries
+ to the manifest.  The method parses environment variables according to
+ platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+ environment variables are case-insensitive.  For example, %VAR% and %var%
+ evaluate to the same value.
+
+ Specifying the classpath in a jar manifest does not support wildcards, so
+ this method expands wildcards internally.  Any classpath entry that ends
+ with * is translated to all files at that path with extension .jar or .JAR.
+
+ @param inputClassPath String input classpath to bundle into the jar manifest
+ @param pwd Path to working directory to save jar
+ @param targetDir path to where the jar execution will have its working dir
+ @param callerEnv Map<String, String> caller's environment variables to use
+   for expansion
+ @return String[] with absolute path to new jar in position 0 and
+   unexpanded wild card entry path in position 1
+ @throws IOException if there is an I/O error while writing the jar file]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+ It operates only on local paths.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist locally]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="useLocal" type="boolean"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist]]>
+      </doc>
+    </method>
+    <method name="compareFs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="destFs" type="org.apache.hadoop.fs.FileSystem"/>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param FileContext the file context with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fs the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fileContext the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <field name="SYMLINK_NO_PRIVILEGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A collection of file-processing util methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileUtil -->
+  <!-- start class org.apache.hadoop.fs.FilterFileSystem -->
+  <class name="FilterFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the raw file system 
+ @return FileSystem being filtered]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called after a new FileSystem instance is constructed.
+ @param name a uri whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make sure that a path specifies a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.  Can take place on local fs
+ or remote DFS.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files in a directory.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files and its block locations in a directory.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a remote iterator for listing in a directory]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given file system. All relative
+ paths will be resolved relative to it.
+ 
+ @param newDir]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given file system
+ 
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ delSrc indicates if the src will be removed or not.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local File that the user can write output to.  The caller
+ provides both the eventual FS target name and the local working
+ file.  If the FS is local, we write directly into the target.  If
+ the FS is remote, we write into the tmp local area.]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.  A local FS will
+ do nothing, because we've written to exactly the right place.  A remote
+ FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get file status.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="abdolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChildFileSystems" return="org.apache.hadoop.fs.FileSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="swapScheme" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A <code>FilterFileSystem</code> contains
+ some other file system, which it uses as
+ its  basic file system, possibly transforming
+ the data along the way or providing  additional
+ functionality. The class <code>FilterFileSystem</code>
+ itself simply overrides all  methods of
+ <code>FileSystem</code> with versions that
+ pass all requests to the contained  file
+ system. Subclasses of <code>FilterFileSystem</code>
+ may further override some of  these methods
+ and may also provide additional methods
+ and fields.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FilterFileSystem -->
+  <!-- start interface org.apache.hadoop.fs.FsConstants -->
+  <interface name="FsConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="LOCAL_FS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FTP_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_PATH_LINKS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ViewFs: viewFs file system (ie the mount file system on client side)]]>
+      </doc>
+    </field>
+    <field name="VIEWFS_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFSOS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FileSystem related constants.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FsConstants -->
+  <!-- start class org.apache.hadoop.fs.FSDataInputStream -->
+  <class name="FSDataInputStream" extends="java.io.DataInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+    <implements name="org.apache.hadoop.fs.HasFileDescriptor"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.CanSetReadahead"/>
+    <implements name="org.apache.hadoop.fs.HasEnhancedByteBufferAccess"/>
+    <implements name="org.apache.hadoop.fs.CanUnbuffer"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <constructor name="FSDataInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desired" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset.
+
+ @param desired offset to seek to]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current position in the input stream.
+
+ @return current position in the input stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    maximum number of bytes to read
+ @return total number of bytes read into the buffer, or <code>-1</code>
+         if there is no more data because the end of the stream has been
+         reached]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+ Continues to read until <code>length</code> bytes have been read.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    the number of bytes to read
+ @throws IOException IO problems
+ @throws EOFException If the end of stream is reached while reading.
+                      If an exception is thrown an undetermined number
+                      of bytes in the buffer may have been written.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link #readFully(long, byte[], int, int)}.]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given position on an alternate copy of the data.
+
+ @param  targetPos  position to seek to
+ @return true if a new source is found, false otherwise]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileDescriptor" return="java.io.FileDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReadahead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <param name="opts" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="releaseBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String value. Includes the string value of the inner stream
+ @return the stream]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link FSInputStream} in a {@link DataInputStream}
+ and buffers input through a {@link java.io.BufferedInputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataInputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStream -->
+  <class name="FSDataOutputStream" extends="java.io.DataOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Syncable"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current position in the output stream.
+
+ @return the current position in the output stream]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the underlying output stream.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="hflush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <class name="FSDataOutputStreamBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FSDataOutputStreamBuilder" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="getThisBuilder" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the concrete implementation of the builder instance.]]>
+      </doc>
+    </method>
+    <method name="getFS" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="permission" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set permission for the file.]]>
+      </doc>
+    </method>
+    <method name="getBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="bufferSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufSize" type="int"/>
+      <doc>
+      <![CDATA[Set the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="replication" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="replica" type="short"/>
+      <doc>
+      <![CDATA[Set replication factor.]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="blockSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blkSize" type="long"/>
+      <doc>
+      <![CDATA[Set block size.]]>
+      </doc>
+    </method>
+    <method name="isRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true to create the parent directories if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="recursive" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the parent directory if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="org.apache.hadoop.util.Progressable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="progress" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prog" type="org.apache.hadoop.util.Progressable"/>
+      <doc>
+      <![CDATA[Set the facility of reporting progress.]]>
+      </doc>
+    </method>
+    <method name="getFlags" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="create" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the specified path.]]>
+      </doc>
+    </method>
+    <method name="overwrite" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="overwrite" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to overwrite the existing file.
+ Set it to false, an exception will be thrown when calling {@link #build()}
+ if the file exists.]]>
+      </doc>
+    </method>
+    <method name="append" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="checksumOpt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="chksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <doc>
+      <![CDATA[Set checksum opt.]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set optional Builder parameter.]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set optional boolean parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set optional int parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set optional float parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set optional double parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set an array of string values as optional parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set mandatory option to the Builder.
+
+ If the option is not supported or unavailable on the {@link FileSystem},
+ the client should expect {@link #build()} throws IllegalArgumentException.]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set mandatory boolean option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set mandatory int option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set mandatory float option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set mandatory double option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set a string array as mandatory option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="getOptions" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMandatoryKeys" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the keys that are set as mandatory keys.]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the FSDataOutputStream to write on the file system.
+
+ @throws IllegalArgumentException if the parameters are not valid.
+ @throws IOException on errors when file system creates or appends the file.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for {@link FSDataOutputStream} and its subclasses.
+
+ It is used to create {@link FSDataOutputStream} when creating a new file or
+ appending an existing file on {@link FileSystem}.
+
+ By default, it does not create parent directory that do not exist.
+ {@link FileSystem#createNonRecursive(Path, boolean, int, short, long,
+ Progressable)}.
+
+ To create missing parent directory, use {@link #recursive()}.
+
+ To be more generic, {@link #opt(String, int)} and {@link #must(String, int)}
+ variants provide implementation-agnostic way to customize the builder.
+ Each FS-specific builder implementation can interpret the FS-specific
+ options accordingly, for example:
+
+ <code>
+
+ // Don't
+ if (fs instanceof FooFileSystem) {
+   FooFileSystem fs = (FooFileSystem) fs;
+   OutputStream out = dfs.createFile(path)
+     .optionA()
+     .optionB("value")
+     .cache()
+   .build()
+ } else if (fs instanceof BarFileSystem) {
+   ...
+ }
+
+ // Do
+ OutputStream out = fs.createFile(path)
+   .permission(perm)
+   .bufferSize(bufSize)
+   .opt("foofs:option.a", true)
+   .opt("foofs:option.b", "value")
+   .opt("barfs:cache", true)
+   .must("foofs:cache", true)
+   .must("barfs:cache-size", 256 * 1024 * 1024)
+   .build();
+ </code>
+
+ If the option is not related to the file system, the option will be ignored.
+ If the option is must, but not supported by the file system, a
+ {@link IllegalArgumentException} will be thrown.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.FSError -->
+  <class name="FSError" extends="java.lang.Error"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Thrown for unexpected filesystem errors, presumed to reflect disk errors
+ in the native filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSError -->
+  <!-- start class org.apache.hadoop.fs.FSInputStream -->
+  <class name="FSInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <constructor name="FSInputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seeks a different copy of the data.  Returns true if 
+ found a new source, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="validatePositionedReadArgs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="EOFException" type="java.io.EOFException"/>
+      <doc>
+      <![CDATA[Validation code, available for use in subclasses.
+ @param position position: if negative an EOF exception is raised
+ @param buffer destination buffer
+ @param offset offset within the buffer
+ @param length length of bytes to read
+ @throws EOFException if the position is negative
+ @throws IndexOutOfBoundsException if there isn't space for the amount of
+ data requested.
+ @throws IllegalArgumentException other arguments are invalid.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[FSInputStream is a generic old InputStream with a little bit
+ of RAF-style seek ability.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSInputStream -->
+  <!-- start class org.apache.hadoop.fs.FsServerDefaults -->
+  <class name="FsServerDefaults" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsServerDefaults"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBytesPerChecksum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWritePacketSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEncryptDataTransfer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrashInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getChecksumType" return="org.apache.hadoop.util.DataChecksum.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyProviderUri" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultStoragePolicyId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Provides server default configuration values to clients.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsServerDefaults -->
+  <!-- start class org.apache.hadoop.fs.FsStatus -->
+  <class name="FsStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsStatus" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a FsStatus object, using the specified statistics]]>
+      </doc>
+    </constructor>
+    <method name="getCapacity" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the capacity in bytes of the file system]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of bytes used on the file system]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of remaining bytes on the file system]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent the capacity, free and used space on a
+ {@link FileSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsStatus -->
+  <!-- start class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <class name="GlobalStorageStatistics" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.GlobalStorageStatistics[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @return            The StorageStatistics object with the given name, or
+                      null if there is none.]]>
+      </doc>
+    </method>
+    <method name="put" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="provider" type="org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider"/>
+      <doc>
+      <![CDATA[Create or return the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @param provider    An object which can create a new StorageStatistics
+                      object if needed.
+ @return            The StorageStatistics object with the given name.
+ @throws RuntimeException  If the StorageStatisticsProvider provides a null
+                           object or a new StorageStatistics object with the
+                           wrong name.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator that we can use to iterate throw all the global storage
+ statistics objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stores global storage statistics objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.GlobFilter -->
+  <class name="GlobFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="GlobFilter" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern.
+
+ @param filePattern the file pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <constructor name="GlobFilter" type="java.lang.String, org.apache.hadoop.fs.PathFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern and an user filter.
+
+ @param filePattern the file pattern.
+ @param filter user filter in addition to the glob pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <method name="hasPattern" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[A filter for POSIX glob pattern with brace expansions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobFilter -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathException -->
+  <class name="InvalidPathException" extends="org.apache.hadoop.HadoopIllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.]]>
+      </doc>
+    </constructor>
+    <constructor name="InvalidPathException" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.
+ @param reason Reason <code>path</code> is invalid]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Path string is invalid either because it has invalid characters or due to
+ other file system specific reasons.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathException -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <class name="InvalidPathHandleException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathHandleException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidPathHandleException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown when the constraints enoded in a {@link PathHandle} do not hold.
+ For example, if a handle were created with the default
+ {@link Options.HandleOpt#path()} constraints, a call to
+ {@link FileSystem#open(PathHandle)} would succeed if the file were
+ modified, but if a different file was at that location then it would throw
+ this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <!-- start class org.apache.hadoop.fs.LocalFileSystem -->
+  <class name="LocalFileSystem" extends="org.apache.hadoop.fs.ChecksumFileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocalFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p/>
+
+ @return <code>file</code>]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Moves files to a bad file directory on the same device, so that their
+ storage will not be reused.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the checksumed local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.LocatedFileStatus -->
+  <class name="LocatedFileStatus" extends="org.apache.hadoop.fs.FileStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocatedFileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocatedFileStatus" type="org.apache.hadoop.fs.FileStatus, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor 
+ @param stat a file status
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ 
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param hasAcl entity has associated ACLs
+ @param isEncrypted entity is encrypted
+ @param isErasureCoded entity is erasure coded
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param attr Attribute flags (See {@link FileStatus.AttrFlags}).
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <method name="getBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file's block locations
+
+ In HDFS, the returned BlockLocation will have different formats for
+ replicated and erasure coded file.
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.
+
+ @return the file's block locations]]>
+      </doc>
+    </method>
+    <method name="setBlockLocations"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="locations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <doc>
+      <![CDATA[Hook for subclasses to lazily set block locations. The {@link #locations}
+ field should be null before this is called.
+ @param locations Block locations for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class defines a FileStatus that includes a file's block locations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocatedFileStatus -->
+  <!-- start class org.apache.hadoop.fs.Options -->
+  <class name="Options" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Options"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class contains options related to file system operations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Options -->
+  <!-- start class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <class name="ParentNotDirectoryException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ParentNotDirectoryException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ParentNotDirectoryException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the parent of specified Path is not a directory
+ as expected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <!-- start interface org.apache.hadoop.fs.PartHandle -->
+  <interface name="PartHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to a part id for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PartHandle -->
+  <!-- start class org.apache.hadoop.fs.Path -->
+  <class name="Path" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="Path" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a path from a String.  Path strings are URIs, but with
+ unescaped elements and some additional normalization.
+
+ @param pathString the path string]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.net.URI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a path from a URI
+
+ @param aUri the source URI]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Path from components.
+
+ @param scheme the scheme
+ @param authority the authority
+ @param path the path]]>
+      </doc>
+    </constructor>
+    <method name="getPathWithoutSchemeAndAuthority" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return a version of the given Path without the scheme information.
+
+ @param path the source Path
+ @return a copy of this Path without the scheme information]]>
+      </doc>
+    </method>
+    <method name="mergePaths" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path1" type="org.apache.hadoop.fs.Path"/>
+      <param name="path2" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Merge 2 paths such that the second path is appended relative to the first.
+ The returned path has the scheme and authority of the first path.  On
+ Windows, the drive specification in the second path is discarded.
+ 
+ @param path1 the first path
+ @param path2 the second path, to be appended relative to path1
+ @return the merged path]]>
+      </doc>
+    </method>
+    <method name="isWindowsAbsolutePath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <param name="slashed" type="boolean"/>
+      <doc>
+      <![CDATA[Determine whether a given path string represents an absolute path on
+ Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+
+ @param pathString the path string to evaluate
+ @param slashed true if the given path is prefixed with "/"
+ @return true if the supplied path looks like an absolute path with a Windows
+ drive-specifier]]>
+      </doc>
+    </method>
+    <method name="toUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert this Path to a URI.
+
+ @return this Path as a URI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the FileSystem that owns this Path.
+
+ @param conf the configuration to use when resolving the FileSystem
+ @return the FileSystem that owns this Path
+ @throws java.io.IOException thrown if there's an issue resolving the
+ FileSystem]]>
+      </doc>
+    </method>
+    <method name="isAbsoluteAndSchemeAuthorityNull" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute <strong>and</strong> the scheme is null, <b>and</b> the authority
+ is null.
+
+ @return whether the path is absolute and the URI has no scheme nor
+ authority parts]]>
+      </doc>
+    </method>
+    <method name="isUriPathAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.  This method is a wrapper for {@link #isUriPathAbsolute()}.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isRoot" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if and only if this path represents the root of a file system.
+
+ @return true if and only if this path represents the root of a file system]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the final component of this path.
+
+ @return the final component of this path]]>
+      </doc>
+    </method>
+    <method name="getParent" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the parent of a path or null if at root.
+ @return the parent of a path or null if at root]]>
+      </doc>
+    </method>
+    <method name="suffix" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds a suffix to the final name in the path.
+
+ @param suffix the suffix to add
+ @return a new path with the suffix added]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="depth" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of elements in this path.
+ @return the number of elements in this path]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #makeQualified(URI, Path)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Returns a qualified path object for the {@link FileSystem}'s working
+ directory.
+  
+ @param fs the target FileSystem
+ @return a qualified path object for the FileSystem's working directory
+ @deprecated use {@link #makeQualified(URI, Path)}]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+      <doc>
+      <![CDATA[Validate the contents of a deserialized Path, so as
+ to defend against malicious object streams.
+ @throws InvalidObjectException if there's no URI]]>
+      </doc>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash.]]>
+      </doc>
+    </field>
+    <field name="SEPARATOR_CHAR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash, as a character.]]>
+      </doc>
+    </field>
+    <field name="CUR_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current directory, ".".]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the current host is a Windows machine.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Names a file or directory in a {@link FileSystem}.
+ Path strings use slash as the directory separator.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Path -->
+  <!-- start interface org.apache.hadoop.fs.PathFilter -->
+  <interface name="PathFilter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="accept" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Tests whether or not the specified abstract pathname should be
+ included in a pathname list.
+
+ @param  path  The abstract pathname to be tested
+ @return  <code>true</code> if and only if <code>pathname</code>
+          should be included]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathFilter -->
+  <!-- start interface org.apache.hadoop.fs.PathHandle -->
+  <interface name="PathHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized form in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes of this path handle.
+ @return the bytes to get to the process completing the upload.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an entity in the FileSystem. May contain
+ metadata sufficient to resolve or verify subsequent accesses independent of
+ other modifications to the FileSystem.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathHandle -->
+  <!-- start interface org.apache.hadoop.fs.PositionedReadable -->
+  <interface name="PositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read up to the specified number of bytes, from a given
+ position within a file, and return the number of bytes read. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @return actual number of bytes read; -1 means "none"
+ @throws IOException IO problems.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the specified number of bytes, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read number of bytes equal to the length of the buffer, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits positional reading.
+
+ Implementations are required to implement thread-safe operations; this may
+ be supported by concurrent access to the data, or by using a synchronization
+ mechanism to serialize access.
+
+ Not all implementations meet this requirement. Those that do not cannot
+ be used as a backing store for some applications, such as Apache HBase.
+
+ Independent of whether or not they are thread safe, some implementations
+ may make the intermediate state of the system, specifically the position
+ obtained in {@code Seekable.getPos()} visible.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PositionedReadable -->
+  <!-- start class org.apache.hadoop.fs.QuotaUsage -->
+  <class name="QuotaUsage" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QuotaUsage"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QuotaUsage" type="org.apache.hadoop.fs.QuotaUsage.Builder"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build the instance based on the builder.]]>
+      </doc>
+    </constructor>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="quota" type="long"/>
+    </method>
+    <method name="setSpaceConsumed"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceConsumed" type="long"/>
+    </method>
+    <method name="setSpaceQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceQuota" type="long"/>
+    </method>
+    <method name="getFileAndDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory count.]]>
+      </doc>
+    </method>
+    <method name="getQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory quota.]]>
+      </doc>
+    </method>
+    <method name="getSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space consumed.]]>
+      </doc>
+    </method>
+    <method name="getSpaceQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type consumed.]]>
+      </doc>
+    </method>
+    <method name="isTypeQuotaSet" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type quota has been set.]]>
+      </doc>
+    </method>
+    <method name="isTypeConsumedAvailable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type consumption information is available.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the header of the output.
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if hOption is false file sizes are returned in bytes
+ if hOption is true file sizes are returned in human readable
+
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="getTypesQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+    </method>
+    <method name="getStorageTypeHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="java.util.List"/>
+      <doc>
+      <![CDATA[return the header of with the StorageTypes.
+
+ @param storageTypes
+ @return storage header string]]>
+      </doc>
+    </method>
+    <field name="QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output format:
+ <----12----> <----15----> <----15----> <----15----> <-------18------->
+    QUOTA   REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]>
+      </doc>
+    </field>
+    <field name="SPACE_QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER_FIELDS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Store the quota usage of a directory.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.QuotaUsage -->
+  <!-- start class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <class name="RawLocalFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RawLocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="useStatIfAvailable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStreamWithMode" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="handleEmptyDstDirectoryOnWindows" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="srcFile" type="java.io.File"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFile" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given path to a file or directory.
+ @param p the path to delete
+ @param recursive to delete sub-directories
+ @return true if the file or directory and all its contents were deleted
+ @throws IOException if p is non-empty and recursive is false]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+
+ (<b>Note</b>: Returned list is not sorted in any given order,
+ due to reliance on Java's {@link File#list()} API.)]]>
+      </doc>
+    </method>
+    <method name="mkOneDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p2f" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDirWithMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="p2f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the specified directory hierarchy. Does not
+ treat existence as an error.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the working directory to the given directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsWorkingFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chown to set owner.]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chmod to set permission.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the {@link Path}'s last modified time and last access time to
+ the given valid times.
+
+ @param mtime the modification time to set (only if no less than zero).
+ @param atime the access time to set (only if no less than zero).
+ @throws IOException if setting the times fails.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opts Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a FileStatus representing the given path. If the path refers
+ to a symlink return a FileStatus representing the link rather than
+ the object the link refers to.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the raw local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.ReadOption -->
+  <class name="ReadOption" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.ReadOption[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.ReadOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Options that can be used when reading from a FileSystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ReadOption -->
+  <!-- start interface org.apache.hadoop.fs.Seekable -->
+  <interface name="Seekable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits seeking.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Seekable -->
+  <!-- start class org.apache.hadoop.fs.StorageStatistics -->
+  <class name="StorageStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StorageStatistics" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this StorageStatistics object.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the associated file system scheme if this is scheme specific,
+ else return null.]]>
+      </doc>
+    </method>
+    <method name="getLongStatistics" return="java.util.Iterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over all the currently tracked long statistics.
+
+ The values returned will depend on the type of FileSystem or FileContext
+ object.  The values do not necessarily reflect a snapshot in time.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="java.lang.Long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of a statistic.
+
+ @return         null if the statistic is not being tracked or is not a
+                 long statistic. The value of the statistic, otherwise.]]>
+      </doc>
+    </method>
+    <method name="isTracked" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return true if a statistic is being tracked.
+
+ @return         True only if the statistic is being tracked.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all the statistic data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[StorageStatistics contains statistics data for a FileSystem or FileContext
+ instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.StorageType -->
+  <class name="StorageType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="supportTypeQuota" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMovable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="asList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMovableTypes" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTypesSupportingQuota" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <field name="DEFAULT" type="org.apache.hadoop.fs.StorageType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.fs.StorageType[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines the types of supported storage media. The default storage
+ medium is assumed to be DISK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageType -->
+  <!-- start interface org.apache.hadoop.fs.StreamCapabilities -->
+  <interface name="StreamCapabilities"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasCapability" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Query the stream for a specific capability.
+
+ @param capability string to query the stream support for.
+ @return True if the stream supports capability.]]>
+      </doc>
+    </method>
+    <field name="HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hflush capability implemented by {@link Syncable#hflush()}.]]>
+      </doc>
+    </field>
+    <field name="HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hsync capability implemented by {@link Syncable#hsync()}.]]>
+      </doc>
+    </field>
+    <field name="READAHEAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setReadahead capability implemented by
+ {@link CanSetReadahead#setReadahead(Long)}.]]>
+      </doc>
+    </field>
+    <field name="DROPBEHIND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setDropBehind capability implemented by
+ {@link CanSetDropBehind#setDropBehind(Boolean)}.]]>
+      </doc>
+    </field>
+    <field name="UNBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface to query streams for supported capabilities.
+
+ Capability strings must be in lower case.
+
+ Constant strings are chosen over enums in order to allow other file systems
+ to define their own capabilities.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.StreamCapabilities -->
+  <!-- start class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <class name="StreamCapabilitiesPolicy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StreamCapabilitiesPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Implement the policy for {@link CanUnbuffer#unbuffer()}.
+
+ @param in the input stream]]>
+      </doc>
+    </method>
+    <field name="CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Static methods to implement policies for {@link StreamCapabilities}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <!-- start interface org.apache.hadoop.fs.Syncable -->
+  <interface name="Syncable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hflush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Flush out the data in client's user buffer. After the return of
+ this call, new readers will see the data.
+ @throws IOException if any error occurs]]>
+      </doc>
+    </method>
+    <method name="hsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to posix fsync, flush out the data in client's user buffer 
+ all the way to the disk device (but the disk may have it in its cache).
+ @throws IOException if error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface for flush/sync operation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Syncable -->
+  <!-- start class org.apache.hadoop.fs.Trash -->
+  <class name="Trash" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Trash" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor.
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <constructor name="Trash" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor for the FileSystem provided.
+ @param fs the FileSystem
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <method name="moveToAppropriateTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[In case of the symlinks or mount points, one has to move the appropriate
+ trashbin in the actual volume of the path p being deleted.
+
+ Hence we get the file system of the fully-qualified resolved-path and
+ then move the path p to the trashbin in that volume,
+ @param fs - the filesystem of path p
+ @param p - the  path being deleted - to be moved to trasg
+ @param conf - configuration
+ @return false if the item is already in the trash or trash is disabled
+ @throws IOException on error]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the trash is enabled for this filesystem]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="checkpoint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="expunge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="expungeImmediately"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all trash immediately.]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Provides a trash facility which supports pluggable Trash policies. 
+
+ See the implementation of the configured TrashPolicy for more
+ details.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Trash -->
+  <!-- start class org.apache.hadoop.fs.TrashPolicy -->
+  <class name="TrashPolicy" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrashPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #initialize(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used
+ @param home the home directory
+ @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations. Different from initialize(conf, fs, home), this one does
+ not assume trash always under /user/$USER due to HDFS encryption zone.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the Trash Policy is enabled for this filesystem.]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="createCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old trash checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpointsImmediately"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all checkpoints immediately, ie empty trash.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory of the Trash Policy
+ This API does not work with files deleted from encryption zone when HDFS
+ data encryption at rest feature is enabled as rename file between
+ encryption zones or encryption zone and non-encryption zone is not allowed.
+
+ The caller is recommend to use the new API
+ TrashPolicy#getCurrentTrashDir(Path path).
+ It returns the trash location correctly for the path specified no matter
+ the path is in encryption zone or not.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current trash directory for path specified based on the Trash
+ Policy
+ @param path path to be deleted
+ @return current trash directory for the path to be deleted
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @param home the home directory
+ @return an instance of TrashPolicy
+ @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @return an instance of TrashPolicy]]>
+      </doc>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trash" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="deletionInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface is used for implementing different Trash policies.
+ Provides factory method to create instances of the configured Trash policy.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.TrashPolicy -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <class name="UnsupportedFileSystemException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedFileSystemException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[File system for a given file system name/scheme is not supported]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <class name="UnsupportedMultipartUploaderException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedMultipartUploaderException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[MultipartUploader for a given file system name/scheme is not supported.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <!-- start interface org.apache.hadoop.fs.UploadHandle -->
+  <interface name="UploadHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an uploadId for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.UploadHandle -->
+  <!-- start class org.apache.hadoop.fs.XAttrCodec -->
+  <class name="XAttrCodec" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrCodec[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrCodec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="decodeValue" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode string representation of a value and check whether it's 
+ encoded. If the given string begins with 0x or 0X, it expresses
+ a hexadecimal number. If the given string begins with 0s or 0S,
+ base64 encoding is expected. If the given string is enclosed in 
+ double quotes, the inner string is treated as text. Otherwise 
+ the given string is treated as text. 
+ @param value string representation of the value.
+ @return byte[] the value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="encodeValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte[]"/>
+      <param name="encoding" type="org.apache.hadoop.fs.XAttrCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode byte[] value to string representation with encoding. 
+ Values encoded as text strings are enclosed in double quotes (\"), 
+ while strings encoded as hexadecimal and base64 are prefixed with 
+ 0x and 0s, respectively.
+ @param value byte[] value
+ @param encoding
+ @return String string representation of value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The value of <code>XAttr</code> is byte[], this class is to 
+ covert byte[] to some kind of string representation or convert back.
+ String representation is convenient for display and input. For example
+ display in screen as shell response and json response, input as http
+ or shell parameter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrCodec -->
+  <!-- start class org.apache.hadoop.fs.XAttrSetFlag -->
+  <class name="XAttrSetFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrSetFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrSetFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="xAttrName" type="java.lang.String"/>
+      <param name="xAttrExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrSetFlag -->
+  <doc>
+  <![CDATA[An abstract file system API.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.fs.crypto">
+</package>
+<package name="org.apache.hadoop.fs.ftp">
+  <!-- start class org.apache.hadoop.fs.ftp.FTPException -->
+  <class name="FTPException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A class to wrap a {@link Throwable} into a Runtime Exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPException -->
+  <!-- start class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+  <class name="FTPFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p/>
+
+ @return <code>ftp</code>]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FTPFileSystem.
+
+ @return the default port]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A stream obtained via this call must be closed before using other APIs of
+ this class or else the invocation will block.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This optional operation is not yet supported.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BLOCK_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST_PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_PASSWORD_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_DATA_CONNECTION_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TRANSFER_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="E_SAME_DIRECTORY_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ A {@link FileSystem} backed by an FTP client provided by <a
+ href="http://commons.apache.org/net/">Apache Commons Net</a>.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+</package>
+<package name="org.apache.hadoop.fs.http">
+</package>
+<package name="org.apache.hadoop.fs.protocolPB">
+</package>
+<package name="org.apache.hadoop.fs.sftp">
+</package>
+<package name="org.apache.hadoop.fs.shell.find">
+</package>
+<package name="org.apache.hadoop.ha">
+  <!-- start class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <class name="BadFencingConfigurationException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BadFencingConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BadFencingConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the operator has specified an invalid configuration
+ for fencing methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <!-- start class org.apache.hadoop.ha.FailoverFailedException -->
+  <class name="FailoverFailedException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailoverFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FailoverFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate service failover has failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.FailoverFailedException -->
+  <!-- start interface org.apache.hadoop.ha.FenceMethod -->
+  <interface name="FenceMethod"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="checkArgs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Verify that the given fencing method's arguments are valid.
+ @param args the arguments provided in the configuration. This may
+        be null if the operator did not configure any arguments.
+ @throws BadFencingConfigurationException if the arguments are invalid]]>
+      </doc>
+    </method>
+    <method name="tryFence" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.ha.HAServiceTarget"/>
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Attempt to fence the target node.
+ @param target the target of the service to fence
+ @param args the configured arguments, which were checked at startup by
+             {@link #checkArgs(String)}
+ @return true if fencing was successful, false if unsuccessful or
+              indeterminate
+ @throws BadFencingConfigurationException if the configuration was
+         determined to be invalid only at runtime]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A fencing method is a method by which one node can forcibly prevent
+ another node from making continued progress. This might be implemented
+ by killing a process on the other node, by denying the other node's
+ access to shared storage, or by accessing a PDU to cut the other node's
+ power.
+ <p>
+ Since these methods are often vendor- or device-specific, operators
+ may implement this interface in order to achieve fencing.
+ <p>
+ Fencing is configured by the operator as an ordered list of methods to
+ attempt. Each method will be tried in turn, and the next in the list
+ will only be attempted if the previous one fails. See {@link NodeFencer}
+ for more information.
+ <p>
+ If an implementation also implements {@link Configurable} then its
+ <code>setConf</code> method will be called upon instantiation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.FenceMethod -->
+  <!-- start interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <interface name="HAServiceProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="monitorHealth"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="HealthCheckFailedException" type="org.apache.hadoop.ha.HealthCheckFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Monitor the health of service. This periodically called by the HA
+ frameworks to monitor the health of the service.
+ 
+ Service is expected to perform checks to ensure it is functional.
+ If the service is not healthy due to failure or partial failure,
+ it is expected to throw {@link HealthCheckFailedException}.
+ The definition of service not healthy is left to the service.
+ 
+ Note that when health check of an Active service fails,
+ failover to standby may be done.
+ 
+ @throws HealthCheckFailedException
+           if the health check of a service fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToActive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to active state. No operation, if the
+ service is already in active state.
+ 
+ @throws ServiceFailedException
+           if transition from standby to active fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToStandby"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to standby state. No operation, if the
+ service is already in standby state.
+ 
+ @throws ServiceFailedException
+           if transition from active to standby fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToObserver"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to observer state. No operation, if the
+ service is already in observer state.
+
+ @throws ServiceFailedException
+           if transition from standby to observer fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="getServiceStatus" return="org.apache.hadoop.ha.HAServiceStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current status of the service. The status indicates
+ the current <em>state</em> (e.g ACTIVE/STANDBY) as well as
+ some additional information.
+ 
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen
+ @see HAServiceStatus]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial version of the protocol]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides High Availability related primitives to
+ monitor and fail-over the service.
+ 
+ This interface could be used by HA frameworks to manage the service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <!-- start class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <class name="HAServiceProtocolHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceProtocolHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="monitorHealth"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToActive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToStandby"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToObserver"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Helper for making {@link HAServiceProtocol} RPC calls. This helper
+ unwraps the {@link RemoteException} to specific exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <!-- start class org.apache.hadoop.ha.HAServiceTarget -->
+  <class name="HAServiceTarget" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceTarget"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the target node.]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an optional separate RPC server address for health checks at the
+ target node.  If defined, then this address is used by the health monitor
+ for the {@link HAServiceProtocol#monitorHealth()} and
+ {@link HAServiceProtocol#getServiceStatus()} calls.  This can be useful for
+ separating out these calls onto separate RPC handlers to protect against
+ resource exhaustion in the main RPC handler pool.  If null (which is the
+ default implementation), then all RPC calls go to the address defined by
+ {@link #getAddress()}.
+
+ @return IPC address of the lifeline RPC server on the target node, or null
+     if no lifeline RPC server is used]]>
+      </doc>
+    </method>
+    <method name="getZKFCAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the ZKFC on the target node]]>
+      </doc>
+    </method>
+    <method name="getFencer" return="org.apache.hadoop.ha.NodeFencer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a Fencer implementation configured for this target node]]>
+      </doc>
+    </method>
+    <method name="checkFencingConfigured"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[@throws BadFencingConfigurationException if the fencing configuration
+ appears to be invalid. This is divorced from the above
+ {@link #getFencer()} method so that the configuration can be checked
+ during the pre-flight phase of failover.]]>
+      </doc>
+    </method>
+    <method name="getProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to connect to the target HA Service.]]>
+      </doc>
+    </method>
+    <method name="setTransitionTargetHAStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"/>
+    </method>
+    <method name="getTransitionTargetHAStatus" return="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a proxy to connect to the target HA service for health monitoring.
+ If {@link #getHealthMonitorAddress()} is implemented to return a non-null
+ address, then this proxy will connect to that address.  Otherwise, the
+ returned proxy defaults to using {@link #getAddress()}, which means this
+ method's behavior is identical to {@link #getProxy(Configuration, int)}.
+
+ @param conf Configuration
+ @param timeoutMs timeout in milliseconds
+ @return a proxy to connect to the target HA service for health monitoring
+ @throws IOException if there is an error]]>
+      </doc>
+    </method>
+    <method name="getZKFCProxy" return="org.apache.hadoop.ha.ZKFCProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to the ZKFC which is associated with this HA service.]]>
+      </doc>
+    </method>
+    <method name="getFencingParameters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addFencingParameters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ret" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Hook to allow subclasses to add any parameters they would like to
+ expose to fencing implementations/scripts. Fencing methods are free
+ to use this map as they see fit -- notably, the shell script
+ implementation takes each entry, prepends 'target_', substitutes
+ '_' for '.', and adds it to the environment of the script.
+
+ Subclass implementations should be sure to delegate to the superclass
+ implementation as well as adding their own keys.
+
+ @param ret map which can be mutated to pass parameters to the fencer]]>
+      </doc>
+    </method>
+    <method name="isAutoFailoverEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if auto failover should be considered enabled]]>
+      </doc>
+    </method>
+    <method name="supportObserver" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this target supports the Observer state, false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents a target of the client side HA administration commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceTarget -->
+  <!-- start class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <class name="HealthCheckFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HealthCheckFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="HealthCheckFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that health check of a service failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <!-- start class org.apache.hadoop.ha.ServiceFailedException -->
+  <class name="ServiceFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ServiceFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that an operation performed
+ to modify the state of a service or application failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.ServiceFailedException -->
+</package>
+<package name="org.apache.hadoop.ha.protocolPB">
+  <!-- start interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <interface name="HAServiceProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <!-- start interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+  <interface name="ZKFCProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.ZKFCProtocolProtos.ZKFCProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+</package>
+<package name="org.apache.hadoop.http.lib">
+  <doc>
+  <![CDATA[This package provides user-selectable (via configuration) classes that add
+functionality to the web UI. They are configured as a list of classes in the
+configuration parameter <b>hadoop.http.filter.initializers</b>.
+
+<ul>
+<li> <b>StaticUserWebFilter</b> - An authorization plugin that makes all
+users a static configured user.
+</ul>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io">
+  <!-- start class org.apache.hadoop.io.AbstractMapWritable -->
+  <class name="AbstractMapWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractMapWritable"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addToMap"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a Class to the maps if it is not already present.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="byte"/>
+      <doc>
+      <![CDATA[@return the Class class for the specified id]]>
+      </doc>
+    </method>
+    <method name="getId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[@return the id for the specified Class]]>
+      </doc>
+    </method>
+    <method name="copy"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Used by child copy constructors.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the conf]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[@param conf the conf to set]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Abstract base class for MapWritable and SortedMapWritable
+ 
+ Unlike org.apache.nutch.crawl.MapWritable, this class allows creation of
+ MapWritable&lt;Writable, MapWritable&gt; so the CLASS_TO_ID and ID_TO_CLASS
+ maps travel with the class instead of being static.
+ 
+ Class ids range from 1 to 127 so there can be at most 127 distinct classes
+ in any specific map instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.AbstractMapWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayFile -->
+  <class name="ArrayFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ArrayFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A dense file-based mapping from integers to values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayFile -->
+  <!-- start class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <class name="ArrayPrimitiveWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayPrimitiveWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an empty instance, for use during Writable read]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance of known type but no value yet
+ for use with type-specific wrapper classes]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Wrap an existing array of primitives
+ @param value - array of primitives]]>
+      </doc>
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original array.  
+ Client must cast it back to type componentType[]
+ (or may use type-specific wrapper classes).
+ @return - original array as Object]]>
+      </doc>
+    </method>
+    <method name="getComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDeclaredComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDeclaredComponentType" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="componentType" type="java.lang.Class"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This is a wrapper class.  It wraps a Writable implementation around
+ an array of primitives (e.g., int[], long[], etc.), with optimized 
+ wire format, and without creating new objects per element.
+ 
+ This is a wrapper class only; it does not make a copy of the 
+ underlying array.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayWritable -->
+  <class name="ArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for arrays containing instances of a class. The elements of this
+ writable must all be instances of the same class. If this writable will be
+ the input for a Reducer, you will need to create a subclass that sets the
+ value to be of the proper type.
+
+ For example:
+ <code>
+ public class IntArrayWritable extends ArrayWritable {
+   public IntArrayWritable() { 
+     super(IntWritable.class); 
+   }	
+ }
+ </code>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayWritable -->
+  <!-- start class org.apache.hadoop.io.BinaryComparable -->
+  <class name="BinaryComparable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="BinaryComparable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return n st bytes 0..n-1 from {#getBytes()} are valid.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return representative byte array for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.BinaryComparable"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#compareBytes(byte[],int,int,byte[],int,int)]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()} to those provided.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if bytes from {#getBytes()} match.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a hash of the bytes returned from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#hashBytes(byte[],int)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface supported by {@link org.apache.hadoop.io.WritableComparable}
+ types supporting ordering/permutation by a representative set of bytes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BinaryComparable -->
+  <!-- start class org.apache.hadoop.io.BloomMapFile -->
+  <class name="BloomMapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomMapFile"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BLOOM_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HASH_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends {@link MapFile} and provides very much the same
+ functionality. However, it uses dynamic Bloom filters to provide
+ quick membership test for keys, and it offers a fast version of 
+ {@link Reader#get(WritableComparable, Writable)} operation, especially in
+ case of sparsely populated MapFile-s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BloomMapFile -->
+  <!-- start class org.apache.hadoop.io.BooleanWritable -->
+  <class name="BooleanWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BooleanWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BooleanWritable" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="get" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.BooleanWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for booleans.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BooleanWritable -->
+  <!-- start interface org.apache.hadoop.io.ByteBufferPool -->
+  <interface name="ByteBufferPool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Get a new direct ByteBuffer.  The pool can provide this from
+ removing a buffer from its internal cache, or by allocating a 
+ new buffer.
+
+ @param direct     Whether the buffer should be direct.
+ @param length     The minimum length the buffer will have.
+ @return           A new ByteBuffer.  This ByteBuffer must be direct.
+                   Its capacity can be less than what was requested, but
+                   must be at least 1 byte.]]>
+      </doc>
+    </method>
+    <method name="putBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Release a buffer back to the pool.
+ The pool may choose to put this buffer into its cache.
+
+ @param buffer    a direct bytebuffer]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.ByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.BytesWritable -->
+  <class name="BytesWritable" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BytesWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a zero-size sequence.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value.
+ @param bytes This array becomes the backing storage for the object.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[], int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value
+ and length as the length. Use this constructor if the array is larger
+ than the value it represents.
+ @param bytes This array becomes the backing storage for the object.
+ @param length The number of bytes to use from array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data backing the BytesWritable. Please use {@link #copyBytes()}
+ if you need the returned array to be precisely the length of the data.
+ @return The data is only valid between 0 and getLength() - 1.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getBytes()} instead.">
+      <doc>
+      <![CDATA[Get the data from the BytesWritable.
+ @deprecated Use {@link #getBytes()} instead.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current size of the buffer.]]>
+      </doc>
+    </method>
+    <method name="getSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getLength()} instead.">
+      <doc>
+      <![CDATA[Get the current size of the buffer.
+ @deprecated Use {@link #getLength()} instead.]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <doc>
+      <![CDATA[Change the size of the buffer. The values in the old range are preserved
+ and any new values are undefined. The capacity is changed if it is 
+ necessary.
+ @param size The new number of bytes]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the capacity, which is the maximum size that could handled without
+ resizing the backing storage.
+ @return The number of bytes]]>
+      </doc>
+    </method>
+    <method name="setCapacity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_cap" type="int"/>
+      <doc>
+      <![CDATA[Change the capacity of the backing storage.
+ The data is preserved.
+ @param new_cap The new capacity in bytes.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="org.apache.hadoop.io.BytesWritable"/>
+      <doc>
+      <![CDATA[Set the BytesWritable to the contents of the given newData.
+ @param newData the value to set this BytesWritable to.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Set the value to a copy of the given byte range
+ @param newData the new values to copy in
+ @param offset the offset in newData to start at
+ @param length the number of bytes to copy]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right_obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Are the two byte sequences equal?]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate the stream of bytes as hex pairs separated by ' '.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A byte sequence that is usable as a key or value.
+ It is resizable and distinguishes between the size of the sequence and
+ the current capacity. The hash function is the front of the md5 of the 
+ buffer. The sort order is the same as memcmp.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BytesWritable -->
+  <!-- start class org.apache.hadoop.io.ByteWritable -->
+  <class name="ByteWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ByteWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ByteWritable" type="byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Set the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ByteWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ByteWritable"/>
+      <doc>
+      <![CDATA[Compares two ByteWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for a single byte.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ByteWritable -->
+  <!-- start interface org.apache.hadoop.io.Closeable -->
+  <interface name="Closeable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="use java.io.Closeable">
+    <implements name="java.io.Closeable"/>
+    <doc>
+    <![CDATA[@deprecated use java.io.Closeable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Closeable -->
+  <!-- start class org.apache.hadoop.io.CompressedWritable -->
+  <class name="CompressedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompressedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="ensureInflated"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Must be called by all methods which access fields to ensure that the data
+ has been uncompressed.]]>
+      </doc>
+    </method>
+    <method name="readFieldsCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #readFields(DataInput)}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #write(DataOutput)}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base-class for Writables which store themselves compressed and lazily
+ inflate on field access.  This is useful for large objects whose fields are
+ not be altered during a map or reduce operation: leaving the field data
+ compressed makes copying the instance from one file to another much
+ faster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.CompressedWritable -->
+  <!-- start class org.apache.hadoop.io.DataOutputOutputStream -->
+  <class name="DataOutputOutputStream" extends="java.io.OutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="constructOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <doc>
+      <![CDATA[Construct an OutputStream from the given DataOutput. If 'out'
+ is already an OutputStream, simply returns it. Otherwise, wraps
+ it in an OutputStream.
+ @param out the DataOutput to wrap
+ @return an OutputStream instance that outputs to 'out']]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[OutputStream implementation that wraps a DataOutput.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DataOutputOutputStream -->
+  <!-- start class org.apache.hadoop.io.DefaultStringifier -->
+  <class name="DefaultStringifier" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Stringifier"/>
+    <constructor name="DefaultStringifier" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="fromString" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="store"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="item" type="K"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the item in the configuration with the given keyName.
+ 
+ @param <K>  the class of the item
+ @param conf the configuration to store
+ @param item the object to be stored
+ @param keyName the name of the key to use
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="load" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="storeArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="items" type="K[]"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the array of items in the configuration with the given keyName.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use 
+ @param items the objects to be stored
+ @param keyName the name of the key to use
+ @throws IndexOutOfBoundsException if the items array is empty
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="loadArray" return="K[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the array of objects from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DefaultStringifier is the default implementation of the {@link Stringifier}
+ interface which stringifies the objects using base64 encoding of the
+ serialized version of the objects. The {@link Serializer} and
+ {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ <br>
+ DefaultStringifier offers convenience methods to store/load objects to/from
+ the configuration.
+ 
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DefaultStringifier -->
+  <!-- start class org.apache.hadoop.io.DoubleWritable -->
+  <class name="DoubleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="DoubleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="DoubleWritable" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="double"/>
+    </method>
+    <method name="get" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a DoubleWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.DoubleWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Writable for Double values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DoubleWritable -->
+  <!-- start class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <class name="ElasticByteBufferPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.ByteBufferPool"/>
+    <constructor name="ElasticByteBufferPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+    </method>
+    <method name="putBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <doc>
+    <![CDATA[This is a simple ByteBufferPool which just creates ByteBuffers as needed.
+ It also caches ByteBuffers after they're released.  It will always return
+ the smallest cached buffer with at least the capacity you request.
+ We don't try to do anything clever here like try to limit the maximum cache
+ size.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.EnumSetWritable -->
+  <class name="EnumSetWritable" extends="java.util.AbstractCollection"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. If the <tt>value</tt> argument is null or
+ its size is zero, the <tt>elementType</tt> argument must not be null. If
+ the argument <tt>value</tt>'s size is bigger than zero, the argument
+ <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </constructor>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. Argument <tt>value</tt> should not be null
+ or empty.
+ 
+ @param value]]>
+      </doc>
+    </constructor>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="E"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.util.EnumSet"/>
+      <param name="elementType" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[reset the EnumSetWritable with specified
+ <tt>value</value> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ is null or its size is zero, the <tt>elementType</tt> argument must not be
+ null. If the argument <tt>value</tt>'s size is bigger than zero, the
+ argument <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </method>
+    <method name="get" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this EnumSetWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true if <code>o</code> is an EnumSetWritable with the same value,
+ or both are null.]]>
+      </doc>
+    </method>
+    <method name="getElementType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class of all the elements of the underlying EnumSetWriable. It
+ may return null.
+ 
+ @return the element class]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable wrapper for EnumSet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.EnumSetWritable -->
+  <!-- start class org.apache.hadoop.io.FloatWritable -->
+  <class name="FloatWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="FloatWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FloatWritable" type="float"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a FloatWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.FloatWritable"/>
+      <doc>
+      <![CDATA[Compares two FloatWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for floats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.FloatWritable -->
+  <!-- start class org.apache.hadoop.io.GenericWritable -->
+  <class name="GenericWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="GenericWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Set the instance that is wrapped.
+ 
+ @param obj]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the wrapped instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTypes" return="java.lang.Class[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all classes that may be wrapped.  Subclasses should implement this
+ to return a constant array of classes.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for Writable instances.
+ <p>
+ When two sequence files, which have same Key type but different Value
+ types, are mapped out to reduce, multiple Value types is not allowed.
+ In this case, this class can help you wrap instances with different types.
+ </p>
+ 
+ <p>
+ Compared with <code>ObjectWritable</code>, this class is much more effective,
+ because <code>ObjectWritable</code> will append the class declaration as a String 
+ into the output file in every Key-Value pair.
+ </p>
+ 
+ <p>
+ Generic Writable implements {@link Configurable} interface, so that it will be 
+ configured by the framework. The configuration is passed to the wrapped objects
+ implementing {@link Configurable} interface <i>before deserialization</i>. 
+ </p>
+ 
+ how to use it: <br>
+ 1. Write your own class, such as GenericObject, which extends GenericWritable.<br> 
+ 2. Implements the abstract method <code>getTypes()</code>, defines 
+    the classes which will be wrapped in GenericObject in application.
+    Attention: this classes defined in <code>getTypes()</code> method, must
+    implement <code>Writable</code> interface.
+ <br><br>
+ 
+ The code looks like this:
+ <blockquote><pre>
+ public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+ </pre></blockquote>
+ 
+ @since Nov 8, 2006]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.GenericWritable -->
+  <!-- start class org.apache.hadoop.io.IntWritable -->
+  <class name="IntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="IntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="IntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a IntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.IntWritable"/>
+      <doc>
+      <![CDATA[Compares two IntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for ints.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IntWritable -->
+  <!-- start class org.apache.hadoop.io.IOUtils -->
+  <class name="IOUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IOUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer 
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+ 
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another. <strong>closes the input and output streams 
+ at the end</strong>.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="count" type="long"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies count bytes from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param count number of bytes to copy
+ @param close whether to close the streams
+ @throws IOException if bytes can not be read or written]]>
+      </doc>
+    </method>
+    <method name="wrappedReadForCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="is" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility wrapper for reading from {@link InputStream}. It catches any errors
+ thrown by the underlying stream (either IO or decompression-related), and
+ re-throws as an IOException.
+ 
+ @param is - InputStream to be read from
+ @param buf - buffer the data is read into
+ @param off - offset within buf
+ @param len - amount of data to be read
+ @return number of bytes read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads len bytes in a loop.
+
+ @param in InputStream to read from
+ @param buf The buffer to fill
+ @param off offset from the buffer
+ @param len the length of bytes to read
+ @throws IOException if it could not read requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to readFully(). Skips bytes in a loop.
+ @param in The InputStream to skip bytes from
+ @param len number of bytes to skip.
+ @throws IOException if it could not skip requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param log the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close
+ @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="cleanupWithLogger"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logger" type="org.slf4j.Logger"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param logger the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close]]>
+      </doc>
+    </method>
+    <method name="closeStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[Closes the stream ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param stream the Stream to close]]>
+      </doc>
+    </method>
+    <method name="closeStreams"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="streams" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Closes the streams ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param streams the Streams to close]]>
+      </doc>
+    </method>
+    <method name="closeSocket"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sock" type="java.net.Socket"/>
+      <doc>
+      <![CDATA[Closes the socket ignoring {@link IOException}
+
+ @param sock the Socket to close]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bc" type="java.nio.channels.WritableByteChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a WritableByteChannel, handling short writes.
+ 
+ @param bc               The WritableByteChannel to write to
+ @param buf              The input buffer
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="java.nio.channels.FileChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <param name="offset" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a FileChannel at a given offset, 
+ handling short writes.
+ 
+ @param fc               The FileChannel to write to
+ @param buf              The input buffer
+ @param offset           The offset in the file to start writing at
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="listDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="filter" type="java.io.FilenameFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the complete list of files in a directory as strings.<p/>
+
+ This is better than File#listDir because it does not ignore IOExceptions.
+
+ @param dir              The directory to list.
+ @param filter           If non-null, the filter to use when listing
+                         this directory.
+ @return                 The list of files in the directory.
+
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileToSync" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.<br>
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param fileToSync the file to fsync]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="channel" type="java.nio.channels.FileChannel"/>
+      <param name="isDir" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param channel Channel to sync
+ @param isDir if true, the given file is a directory (Channel should be
+          opened for read and ignore IOExceptions, because not all file
+          systems and operating systems allow to fsync on a directory)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="wrapException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="methodName" type="java.lang.String"/>
+      <param name="exception" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Takes an IOException, file/directory path, and method name and returns an
+ IOException with the input exception as the cause and also include the
+ file,method details. The new exception provides the stack trace of the
+ place where the exception is thrown and some extra diagnostics
+ information.
+
+ Return instance of same exception if exception class has a public string
+ constructor; Otherwise return an PathIOException.
+ InterruptedIOException and PathIOException are returned unwrapped.
+
+ @param path file/directory path
+ @param methodName method name
+ @param exception the caught exception.
+ @return an exception to throw]]>
+      </doc>
+    </method>
+    <method name="readFullyToByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a DataInput until EOF and returns a byte array.  Make sure not to
+ pass in an infinite DataInput or this will never return.
+
+ @param in A DataInput
+ @return a byte array containing the data from the DataInput
+ @throws IOException on I/O error, other than EOF]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An utility class for I/O related functionality.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IOUtils -->
+  <!-- start class org.apache.hadoop.io.LongWritable -->
+  <class name="LongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="LongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a LongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.LongWritable"/>
+      <doc>
+      <![CDATA[Compares two LongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.LongWritable -->
+  <!-- start class org.apache.hadoop.io.MapFile -->
+  <class name="MapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="oldName" type="java.lang.String"/>
+      <param name="newName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames an existing map directory.]]>
+      </doc>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deletes the named map file.]]>
+      </doc>
+    </method>
+    <method name="fix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <param name="dryrun" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[This method attempts to fix a corrupt MapFile by re-creating its index.
+ @param fs filesystem
+ @param dir directory containing the MapFile data and index
+ @param keyClass key class (has to be a subclass of Writable)
+ @param valueClass value class (has to be a subclass of Writable)
+ @param dryrun do not perform any changes, just report what needs to be done
+ @return number of valid entries in this MapFile, or -1 if no fixing was needed
+ @throws Exception]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="INDEX_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the index file.]]>
+      </doc>
+    </field>
+    <field name="DATA_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the data file.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A file-based map from keys to values.
+ 
+ <p>A map is a directory containing two files, the <code>data</code> file,
+ containing all keys and values in the map, and a smaller <code>index</code>
+ file, containing a fraction of the keys.  The fraction is determined by
+ {@link Writer#getIndexInterval()}.
+
+ <p>The index file is read entirely into memory.  Thus key implementations
+ should try to keep themselves small.
+
+ <p>Map files are created by adding entries in-order.  To maintain a large
+ database, perform updates by copying the previous version of a database and
+ merging in a sorted change list, to create a new version of the database in
+ a new file.  Sorting large change lists can be done with {@link
+ SequenceFile.Sorter}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapFile -->
+  <!-- start class org.apache.hadoop.io.MapWritable -->
+  <class name="MapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Map"/>
+    <constructor name="MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="MapWritable" type="org.apache.hadoop.io.MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Writable"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable Map.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapWritable -->
+  <!-- start class org.apache.hadoop.io.MD5Hash -->
+  <class name="MD5Hash" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="MD5Hash"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash from a hex string.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash with a specified value.]]>
+      </doc>
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs, reads and returns an instance.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Copy the contents of another instance into this instance.]]>
+      </doc>
+    </method>
+    <method name="getDigest" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the digest bytes.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="getDigester" return="java.security.MessageDigest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a thread local MD5 digester]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a hash value for the content from the InputStream.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dataArr" type="byte[][]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for an array of byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="org.apache.hadoop.io.UTF8"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="halfDigest" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a half-sized version of this MD5.  Fits in a long]]>
+      </doc>
+    </method>
+    <method name="quarterDigest" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a 32-bit digest of the MD5.
+ @return the first 4 bytes of the md5]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is an MD5Hash whose digest contains the
+ same values.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for this object.
+ Only uses the first 4 bytes, since md5s are evenly distributed.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Compares this object with the specified object for order.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="setDigest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the digest value from a hex string.]]>
+      </doc>
+    </method>
+    <field name="MD5_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Writable for MD5 hash values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MD5Hash -->
+  <!-- start class org.apache.hadoop.io.MultipleIOException -->
+  <class name="MultipleIOException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getExceptions" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the underlying exceptions]]>
+      </doc>
+    </method>
+    <method name="createIOException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="exceptions" type="java.util.List"/>
+      <doc>
+      <![CDATA[A convenient method to create an {@link IOException}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Encapsulate a list of {@link IOException} into an {@link IOException}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MultipleIOException -->
+  <!-- start class org.apache.hadoop.io.NullWritable -->
+  <class name="NullWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <method name="get" return="org.apache.hadoop.io.NullWritable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the single instance of this class.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.NullWritable"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Singleton Writable with no data.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.NullWritable -->
+  <!-- start class org.apache.hadoop.io.ObjectWritable -->
+  <class name="ObjectWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="ObjectWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Class, java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the instance, or null if none.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the class this is meant to be.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="instance" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reset the instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="allowCompactArrays" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.  
+ 
+ @param allowCompactArrays - set true for RPC and internal or intra-cluster
+ usages.  Set false for inter-cluster, File, and other persisted output 
+ usages, to preserve the ability to interchange files with other clusters 
+ that may not be running the same version of software.  Sometime in ~2013 
+ we can consider removing this parameter and always using the compact format.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="objectWritable" type="org.apache.hadoop.io.ObjectWritable"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find and load the class with given name <tt>className</tt> by first finding
+ it in the specified <tt>conf</tt>. If the specified <tt>conf</tt> is null,
+ try load it directly.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A polymorphic Writable that writes an instance with it's class name.
+ Handles arrays, strings and primitive types without a Writable wrapper.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ObjectWritable -->
+  <!-- start interface org.apache.hadoop.io.RawComparator -->
+  <interface name="RawComparator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <method name="compare" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Compare two objects in binary.
+ b1[s1:l1] is the first object, and b2[s2:l2] is the second object.
+ 
+ @param b1 The first byte array.
+ @param s1 The position index in b1. The object under comparison's starting index.
+ @param l1 The length of the object in b1.
+ @param b2 The second byte array.
+ @param s2 The position index in b2. The object under comparison's starting index.
+ @param l2 The length of the object under comparison in b2.
+ @return An integer result of the comparison.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ A {@link Comparator} that operates directly on byte representations of
+ objects.
+ </p>
+ @param <T>
+ @see DeserializerComparator]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.RawComparator -->
+  <!-- start class org.apache.hadoop.io.SequenceFile -->
+  <class name="SequenceFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDefaultCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the compression type for the reduce outputs
+ @param job the job config to look in
+ @return the kind of compression to use]]>
+      </doc>
+    </method>
+    <method name="setDefaultCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="val" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the default compression type for sequence files.
+ @param job the configuration to modify
+ @param val the new compression type (none, block, record)]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="opts" type="org.apache.hadoop.io.SequenceFile.Writer.Option[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new Writer with the given options.
+ @param conf the configuration to use
+ @param opts the options to create the file with
+ @return a new Writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="createParent" type="boolean"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param createParent create parent directory if non-existent
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fc The context for the specified file.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @param createFlag gives the semantics of create: overwrite, append etc.
+ @param opts file creation options; see {@link CreateOpts}.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <field name="SYNC_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes between sync points. 100 KB, default.
+ Computed as 5 KB * 20 = 100 KB]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[<code>SequenceFile</code>s are flat files consisting of binary key/value 
+ pairs.
+ 
+ <p><code>SequenceFile</code> provides {@link SequenceFile.Writer},
+ {@link SequenceFile.Reader} and {@link Sorter} classes for writing,
+ reading and sorting respectively.</p>
+ 
+ There are three <code>SequenceFile</code> <code>Writer</code>s based on the 
+ {@link CompressionType} used to compress key/value pairs:
+ <ol>
+   <li>
+   <code>Writer</code> : Uncompressed records.
+   </li>
+   <li>
+   <code>RecordCompressWriter</code> : Record-compressed files, only compress 
+                                       values.
+   </li>
+   <li>
+   <code>BlockCompressWriter</code> : Block-compressed files, both keys & 
+                                      values are collected in 'blocks' 
+                                      separately and compressed. The size of 
+                                      the 'block' is configurable.
+ </ol>
+ 
+ <p>The actual compression algorithm used to compress key and/or values can be
+ specified by using the appropriate {@link CompressionCodec}.</p>
+ 
+ <p>The recommended way is to use the static <tt>createWriter</tt> methods
+ provided by the <code>SequenceFile</code> to chose the preferred format.</p>
+
+ <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
+ above <code>SequenceFile</code> formats.</p>
+
+ <h4 id="Formats">SequenceFile Formats</h4>
+ 
+ <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
+ depending on the <code>CompressionType</code> specified. All of them share a
+ <a href="#Header">common header</a> described below.
+ 
+ <h5 id="Header">SequenceFile Header</h5>
+ <ul>
+   <li>
+   version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual 
+             version number (e.g. SEQ4 or SEQ6)
+   </li>
+   <li>
+   keyClassName -key class
+   </li>
+   <li>
+   valueClassName - value class
+   </li>
+   <li>
+   compression - A boolean which specifies if compression is turned on for 
+                 keys/values in this file.
+   </li>
+   <li>
+   blockCompression - A boolean which specifies if block-compression is 
+                      turned on for keys/values in this file.
+   </li>
+   <li>
+   compression codec - <code>CompressionCodec</code> class which is used for  
+                       compression of keys and/or values (if compression is 
+                       enabled).
+   </li>
+   <li>
+   metadata - {@link Metadata} for this file.
+   </li>
+   <li>
+   sync - A sync marker to denote end of the header.
+   </li>
+ </ul>
+ 
+ <h5 id="#UncompressedFormat">Uncompressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li>Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+
+ <h5 id="#RecordCompressedFormat">Record-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li><i>Compressed</i> Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+ 
+ <h5 id="#BlockCompressedFormat">Block-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record <i>Block</i>
+   <ul>
+     <li>Uncompressed number of records in the block</li>
+     <li>Compressed key-lengths block-size</li>
+     <li>Compressed key-lengths block</li>
+     <li>Compressed keys block-size</li>
+     <li>Compressed keys block</li>
+     <li>Compressed value-lengths block-size</li>
+     <li>Compressed value-lengths block</li>
+     <li>Compressed values block-size</li>
+     <li>Compressed values block</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every block.
+ </li>
+ </ul>
+ 
+ <p>The compressed blocks of key lengths and value lengths consist of the 
+ actual lengths of individual keys/values encoded in ZeroCompressedInteger 
+ format.</p>
+ 
+ @see CompressionCodec]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SequenceFile -->
+  <!-- start class org.apache.hadoop.io.SetFile -->
+  <class name="SetFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SetFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A file-based set of keys.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SetFile -->
+  <!-- start class org.apache.hadoop.io.ShortWritable -->
+  <class name="ShortWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ShortWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ShortWritable" type="short"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="short"/>
+      <doc>
+      <![CDATA[Set the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[read the short value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[write short value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ShortWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[hash code]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ShortWritable"/>
+      <doc>
+      <![CDATA[Compares two ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Short values in string format]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for shorts.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ShortWritable -->
+  <!-- start class org.apache.hadoop.io.SortedMapWritable -->
+  <class name="SortedMapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.SortedMap"/>
+    <constructor name="SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="SortedMapWritable" type="org.apache.hadoop.io.SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="comparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="firstKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="headMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="lastKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="subMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="tailMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable SortedMap.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SortedMapWritable -->
+  <!-- start interface org.apache.hadoop.io.Stringifier -->
+  <interface name="Stringifier"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Converts the object to a string representation
+ @param obj the object to convert
+ @return the string representation of the object
+ @throws IOException if the object cannot be converted]]>
+      </doc>
+    </method>
+    <method name="fromString" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from its string representation.
+ @param str the string representation of the object
+ @return restored object
+ @throws IOException if the object cannot be restored]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes this object. 
+ @throws IOException if an I/O error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stringifier interface offers two methods to convert an object 
+ to a string representation and restore the object given its 
+ string representation.
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Stringifier -->
+  <!-- start class org.apache.hadoop.io.Text -->
+  <class name="Text" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Text" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a string.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another text.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a byte array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the raw bytes; however, only data up to {@link #getLength()} is
+ valid. Please use {@link #copyBytes()} if you
+ need the returned array to be precisely the length of the data.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes in the byte array]]>
+      </doc>
+    </method>
+    <method name="charAt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="int"/>
+      <doc>
+      <![CDATA[Returns the Unicode Scalar Value (32-bit integer value)
+ for the character at <code>position</code>. Note that this
+ method avoids using the converter or doing String instantiation
+ @return the Unicode scalar value at position or -1
+          if the position is invalid or points to a
+          trailing byte]]>
+      </doc>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Finds any occurrence of <code>what</code> in the backing
+ buffer, starting as position <code>start</code>. The starting
+ position is measured in bytes and the return value is in
+ terms of byte position in the buffer. The backing buffer is
+ not converted to a string for this operation.
+ @return byte position of the first occurrence of the search
+         string in the UTF-8 buffer or -1 if not found]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set to contain the contents of a string.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <doc>
+      <![CDATA[Set to a utf8 byte array]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[copy a text.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Set the Text to range of bytes
+ @param utf8 the data to copy from
+ @param start the first position of the new string
+ @param len the number of bytes of the new string]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Append a range of bytes to the end of the given text
+ @param utf8 the data to copy from
+ @param start the first position to append from utf8
+ @param len the number of bytes to append]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the string to empty.
+
+ <em>Note</em>: For performance reasons, this call does not clear the
+ underlying byte array that is retrievable via {@link #getBytes()}.
+ In order to free the byte-array memory, call {@link #set(byte[])}
+ with an empty byte array (For example, <code>new byte[0]</code>).]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert text back to string
+ @see java.lang.Object#toString()]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[deserialize]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skips over one Text in the input.]]>
+      </doc>
+    </method>
+    <method name="readWithKnownLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a Text object whose length is already known.
+ This allows creating Text from a stream which uses a different serialization
+ format.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[serialize
+ write this object to out
+ length uses zero-compressed encoding
+ @see Writable#write(DataOutput)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a Text with the same contents.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If the input is malformed,
+ replace by a default value.]]>
+      </doc>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If the input is malformed,
+ invalid chars are replaced by a default value.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string from in]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string with a maximum size]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string to out]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string with a maximum size to out]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check if a byte array contains valid utf-8
+ @param utf8 byte array
+ @throws MalformedInputException if the byte array contains invalid utf-8]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check to see if a byte array is valid utf-8
+ @param utf8 the array of bytes
+ @param start the offset of the first byte in the array
+ @param len the length of the byte sequence
+ @throws MalformedInputException if the byte array contains invalid bytes]]>
+      </doc>
+    </method>
+    <method name="bytesToCodePoint" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Returns the next code point at the current position in
+ the buffer. The buffer's position will be incremented.
+ Any mark set on this buffer will be changed by this method!]]>
+      </doc>
+    </method>
+    <method name="utf8Length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[For the given string, returns the number of UTF-8 bytes
+ required to encode the string.
+ @param string text to encode
+ @return number of UTF-8 bytes required to encode]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_MAX_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class stores text using standard UTF8 encoding.  It provides methods
+ to serialize, deserialize, and compare texts at byte level.  The type of
+ length is integer and is serialized using zero-compressed format.  <p>In
+ addition, it provides methods for string traversal without converting the
+ byte array to a string.  <p>Also includes utilities for
+ serializing/deserialing a string, coding/decoding a string, checking if a
+ byte array contains valid UTF8 code, calculating the length of an encoded
+ string.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.Text -->
+  <!-- start class org.apache.hadoop.io.TwoDArrayWritable -->
+  <class name="TwoDArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[][]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[][]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[][]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for 2D arrays containing a matrix of instances of a class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.TwoDArrayWritable -->
+  <!-- start class org.apache.hadoop.io.VersionedWritable -->
+  <class name="VersionedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="VersionedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getVersion" return="byte"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the version number of the current implementation.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A base class for Writables that provides version checking.
+
+ <p>This is useful when a class may evolve, so that instances written by the
+ old version of the class may still be processed by the new version.  To
+ handle this situation, {@link #readFields(DataInput)}
+ implementations should catch {@link VersionMismatchException}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionedWritable -->
+  <!-- start class org.apache.hadoop.io.VersionMismatchException -->
+  <class name="VersionMismatchException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionMismatchException" type="byte, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Thrown by {@link VersionedWritable#readFields(DataInput)} when the
+ version of an object being read does not match the current implementation
+ version as returned by {@link VersionedWritable#getVersion()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionMismatchException -->
+  <!-- start class org.apache.hadoop.io.VIntWritable -->
+  <class name="VIntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VIntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VIntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VIntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VIntWritable"/>
+      <doc>
+      <![CDATA[Compares two VIntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for integer values stored in variable-length format.
+ Such values take between one and five bytes.  Smaller values take fewer bytes.
+ 
+ @see org.apache.hadoop.io.WritableUtils#readVInt(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VIntWritable -->
+  <!-- start class org.apache.hadoop.io.VLongWritable -->
+  <class name="VLongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VLongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VLongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VLongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VLongWritable"/>
+      <doc>
+      <![CDATA[Compares two VLongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs in a variable-length format. Such values take
+  between one and five bytes.  Smaller values take fewer bytes.
+  
+  @see org.apache.hadoop.io.WritableUtils#readVLong(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VLongWritable -->
+  <!-- start interface org.apache.hadoop.io.Writable -->
+  <interface name="Writable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize the fields of this object to <code>out</code>.
+ 
+ @param out <code>DataOuput</code> to serialize this object into.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize the fields of this object from <code>in</code>.  
+ 
+ <p>For efficiency, implementations should attempt to re-use storage in the 
+ existing object where possible.</p>
+ 
+ @param in <code>DataInput</code> to deseriablize this object from.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A serializable object which implements a simple, efficient, serialization 
+ protocol, based on {@link DataInput} and {@link DataOutput}.
+
+ <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
+ framework implements this interface.</p>
+ 
+ <p>Implementations typically implement a static <code>read(DataInput)</code>
+ method which constructs a new instance, calls {@link #readFields(DataInput)} 
+ and returns the instance.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyWritable implements Writable {
+       // Some data
+       private int counter;
+       private long timestamp;
+
+       // Default constructor to allow (de)serialization
+       MyWritable() { }
+
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+ </pre></blockquote></p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Writable -->
+  <!-- start interface org.apache.hadoop.io.WritableComparable -->
+  <interface name="WritableComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <doc>
+    <![CDATA[A {@link Writable} which is also {@link Comparable}. 
+
+ <p><code>WritableComparable</code>s can be compared to each other, typically 
+ via <code>Comparator</code>s. Any type which is to be used as a 
+ <code>key</code> in the Hadoop Map-Reduce framework should implement this
+ interface.</p>
+
+ <p>Note that <code>hashCode()</code> is frequently used in Hadoop to partition
+ keys. It's important that your implementation of hashCode() returns the same 
+ result across different instances of the JVM. Note also that the default 
+ <code>hashCode()</code> implementation in <code>Object</code> does <b>not</b>
+ satisfy this property.</p>
+  
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyWritableComparable implements WritableComparable<MyWritableComparable> {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue &lt; thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp &gt;&gt;&gt; 32));
+         return result
+       }
+     }
+ </pre></blockquote></p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableComparable -->
+  <!-- start class org.apache.hadoop.io.WritableComparator -->
+  <class name="WritableComparator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.RawComparator"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="WritableComparator"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, org.apache.hadoop.conf.Configuration, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[For backwards compatibility.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a comparator for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="define"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="comparator" type="org.apache.hadoop.io.WritableComparator"/>
+      <doc>
+      <![CDATA[Register an optimized comparator for a {@link WritableComparable}
+ implementation. Comparators registered with this method must be
+ thread-safe.]]>
+      </doc>
+    </method>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the WritableComparable implementation class.]]>
+      </doc>
+    </method>
+    <method name="newKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new {@link WritableComparable} instance.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Optimization hook.  Override this to make SequenceFile.Sorter's scream.
+
+ <p>The default implementation reads the data into two {@link
+ WritableComparable}s (using {@link
+ Writable#readFields(DataInput)}, then calls {@link
+ #compare(WritableComparable,WritableComparable)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="b" type="org.apache.hadoop.io.WritableComparable"/>
+      <doc>
+      <![CDATA[Compare two WritableComparables.
+
+ <p> The default implementation uses the natural ordering, calling {@link
+ Comparable#compareTo(Object)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="java.lang.Object"/>
+      <param name="b" type="java.lang.Object"/>
+    </method>
+    <method name="compareBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Lexicographic order of binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="readUnsignedShort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an unsigned short from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an integer from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a float from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a long from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a double from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a byte array and returns it.
+ @param bytes byte array with decode long
+ @param start starting index
+ @throws java.io.IOException 
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a byte array and returns it.
+ @param bytes byte array with the encoded integer
+ @param start start index
+ @throws java.io.IOException 
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A Comparator for {@link WritableComparable}s.
+
+ <p>This base implementation uses the natural ordering.  To define alternate
+ orderings, override {@link #compare(WritableComparable,WritableComparable)}.
+
+ <p>One may optimize compare-intensive operations by overriding
+ {@link #compare(byte[],int,int,byte[],int,int)}.  Static utility methods are
+ provided to assist in optimized implementations of this method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableComparator -->
+  <!-- start class org.apache.hadoop.io.WritableFactories -->
+  <class name="WritableFactories" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="factory" type="org.apache.hadoop.io.WritableFactory"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="getFactory" return="org.apache.hadoop.io.WritableFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Factories for non-public writables.  Defining a factory permits {@link
+ ObjectWritable} to be able to construct instances of non-public classes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableFactories -->
+  <!-- start interface org.apache.hadoop.io.WritableFactory -->
+  <interface name="WritableFactory"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a new instance.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A factory for a class of Writable.
+ @see WritableFactories]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableFactory -->
+  <!-- start class org.apache.hadoop.io.WritableUtils -->
+  <class name="WritableUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WritableUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readCompressedByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skipCompressedByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedByteArray" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="displayByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="byte[]"/>
+    </method>
+    <method name="clone" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orig" type="T"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Make a copy of a writable object using serialization to a buffer.
+ @param orig The object to copy
+ @return The copied object]]>
+      </doc>
+    </method>
+    <method name="cloneInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use ReflectionUtils.cloneInto instead.">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param dst the object to copy from
+ @param src the object to copy into, which is destroyed
+ @throws IOException
+ @deprecated use ReflectionUtils.cloneInto instead.]]>
+      </doc>
+    </method>
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes an integer to a binary stream with zero-compressed encoding.
+ For -112 <= i <= 127, only one byte is used with the actual value.
+ For other values of i, the first byte value indicates whether the
+ integer is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -116, the following integer
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -124, the following integer
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+
+ @param stream Binary output stream
+ @param i Integer to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes a long to a binary stream with zero-compressed encoding.
+ For -112 <= i <= 127, only one byte is used with the actual value.
+ For other values of i, the first byte value indicates whether the
+ long is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -120, the following long
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -128, the following long
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+ 
+ @param stream Binary output stream
+ @param i Long to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized long from stream.]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized integer from stream.]]>
+      </doc>
+    </method>
+    <method name="readVIntInRange" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <param name="lower" type="int"/>
+      <param name="upper" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads an integer from the input stream and returns it.
+
+ This function validates that the integer is between [lower, upper],
+ inclusive.
+
+ @param stream Binary input stream
+ @throws java.io.IOException
+ @return deserialized integer from stream]]>
+      </doc>
+    </method>
+    <method name="isNegativeVInt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Given the first byte of a vint/vlong, determine the sign
+ @param value the first byte
+ @return is the value negative]]>
+      </doc>
+    </method>
+    <method name="decodeVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Parse the first byte of a vint/vlong to determine the number of bytes
+ @param value the first byte of the vint/vlong
+ @return the total number of bytes (1 to 9)]]>
+      </doc>
+    </method>
+    <method name="getVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="long"/>
+      <doc>
+      <![CDATA[Get the encoded length if an integer is stored in a variable-length format
+ @return the encoded length]]>
+      </doc>
+    </method>
+    <method name="readEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="enumType" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read an Enum value from DataInput, Enums are read and written 
+ using String values. 
+ @param <T> Enum type
+ @param in DataInput to read from 
+ @param enumType Class type of Enum
+ @return Enum represented by String read from DataInput
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeEnum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="enumVal" type="java.lang.Enum"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[writes String value of enum to DataOutput. 
+ @param out Dataoutput stream
+ @param enumVal enum value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip <i>len</i> number of bytes in input stream<i>in</i>
+ @param in input stream
+ @param len number of bytes to skip
+ @throws IOException when skipped less number of bytes]]>
+      </doc>
+    </method>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writables" type="org.apache.hadoop.io.Writable[]"/>
+      <doc>
+      <![CDATA[Convert writables to a byte array]]>
+      </doc>
+    </method>
+    <method name="readStringSafely" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Read a string, but check it for sanity. The format consists of a vint
+ followed by the given number of bytes.
+ @param in the stream to read from
+ @param maxLength the largest acceptable length of the encoded string
+ @return the bytes as a string
+ @throws IOException if reading from the DataInput fails
+ @throws IllegalArgumentException if the encoded byte size for string 
+             is negative or larger than maxSize. Only the vint is read.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableUtils -->
+  <doc>
+  <![CDATA[Generic i/o code for use when reading and writing data to the network,
+to databases, and to files.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.compress">
+  <!-- start class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <class name="BlockCompressorStream" extends="org.apache.hadoop.io.compress.CompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream}.
+ 
+ @param out stream
+ @param compressor compressor to be used
+ @param bufferSize size of buffer
+ @param compressionOverhead maximum 'overhead' of the compression 
+                            algorithm with given bufferSize]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream} with given output-stream and 
+ compressor.
+ Use default of 512 as bufferSize and compressionOverhead of 
+ (1% of bufferSize + 12 bytes) =  18 bytes (zlib algorithm).
+ 
+ @param out stream
+ @param compressor compressor to be used]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the data provided to the compression codec, compressing no more
+ than the buffer size less the compression overhead as specified during
+ construction for each block.
+
+ Each block contains the uncompressed length for the block, followed by
+ one or more length-prefixed blocks of compressed data.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.CompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.
+
+ It should be noted that this wrapper does not guarantee that blocks will
+ be sized for the compressor. If the
+ {@link org.apache.hadoop.io.compress.Compressor} requires buffering to
+ effect meaningful compression, it is responsible for it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <class name="BlockDecompressorStream" extends="org.apache.hadoop.io.compress.DecompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @param bufferSize size of buffer
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.DecompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BZip2Codec -->
+  <class name="BZip2Codec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.SplittableCompressionCodec"/>
+    <constructor name="BZip2Codec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BZip2Codec.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+
+ @param conf the configuration object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+
+ @return the configuration object used by this objec.]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream}.
+
+ @param out        the location for the final output stream
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream} with the given {@link Compressor}.
+
+ @param out        the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream and return a stream for uncompressed data.
+
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ {@link InputStream} with the given {@link Decompressor}, and return a 
+ stream for uncompressed data.
+
+ @param in           the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates CompressionInputStream to be used to read off uncompressed data
+ in one of the two reading modes. i.e. Continuous or Blocked reading modes
+
+ @param seekableIn The InputStream
+ @param start The start offset into the compressed stream
+ @param end The end offset into the compressed stream
+ @param readMode Controls whether progress is reported continuously or
+                 only at block boundaries.
+
+ @return CompressionInputStream for BZip2 aligned at block boundaries]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[.bz2 is recognized as the default extension for compressed BZip2 files
+
+ @return A String telling the default bzip2 file extension]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class provides output and input streams for bzip2 compression
+ and decompression.  It uses the native bzip2 library on the system
+ if possible, else it uses a pure-Java implementation of the bzip2
+ algorithm.  The configuration parameter
+ io.compression.codec.bzip2.library can be used to control this
+ behavior.
+
+ In the pure-Java mode, the Compressor and Decompressor interfaces
+ are not implemented.  Therefore, in that mode, those methods of
+ CompressionCodec which have a Compressor or Decompressor type
+ argument, throw UnsupportedOperationException.
+
+ Currently, support for splittability is available only in the
+ pure-Java mode; therefore, if a SplitCompressionInputStream is
+ requested, the pure-Java implementation is used, regardless of the
+ setting of the configuration parameter mentioned above.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BZip2Codec -->
+  <!-- start class org.apache.hadoop.io.compress.CodecConstants -->
+  <class name="CodecConstants" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="DEFAULT_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.DefaultCodec}.]]>
+      </doc>
+    </field>
+    <field name="BZIP2_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.BZip2Codec}.]]>
+      </doc>
+    </field>
+    <field name="GZIP_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.GzipCodec}.]]>
+      </doc>
+    </field>
+    <field name="LZ4_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.Lz4Codec}.]]>
+      </doc>
+    </field>
+    <field name="SNAPPY_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.SnappyCodec}.]]>
+      </doc>
+    </field>
+    <field name="ZSTANDARD_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.ZStandardCodec}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Codec related constants.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecConstants -->
+  <!-- start class org.apache.hadoop.io.compress.CodecPool -->
+  <class name="CodecPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CodecPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a {@link Compressor} for the given {@link CompressionCodec} from the 
+ pool or a new one.
+
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Compressor</code>
+ @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
+ @return <code>Compressor</code> for the given 
+         <code>CompressionCodec</code> from the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+    </method>
+    <method name="getDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Get a {@link Decompressor} for the given {@link CompressionCodec} from the
+ pool or a new one.
+  
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Decompressor</code>
+ @return <code>Decompressor</code> for the given 
+         <code>CompressionCodec</code> the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="returnCompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <doc>
+      <![CDATA[Return the {@link Compressor} to the pool.
+ 
+ @param compressor the <code>Compressor</code> to be returned to the pool]]>
+      </doc>
+    </method>
+    <method name="returnDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <doc>
+      <![CDATA[Return the {@link Decompressor} to the pool.
+ 
+ @param decompressor the <code>Decompressor</code> to be returned to the 
+                     pool]]>
+      </doc>
+    </method>
+    <method name="getLeasedCompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Compressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <method name="getLeasedDecompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Decompressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A global compressor/decompressor pool used to save and reuse 
+ (possibly native) compression/decompression codecs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecPool -->
+  <!-- start interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <interface name="CompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream}.
+ 
+ @param out the location for the final output stream
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream} with the given {@link Compressor}.
+ 
+ @param out the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream.
+ 
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given 
+ {@link InputStream} with the given {@link Decompressor}.
+ 
+ @param in the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default filename extension for this kind of compression.
+ @return the extension including the '.']]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a streaming compression/decompression pair.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <class name="CompressionCodecFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionCodecFactory" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Find the codecs specified in the config value io.compression.codecs 
+ and register them. Defaults to gzip and deflate.]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Print the extension map out as a string.]]>
+      </doc>
+    </method>
+    <method name="getCodecClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the list of codecs discovered via a Java ServiceLoader, or
+ listed in the configuration. Codecs specified in configuration come
+ later in the returned list, and are considered to override those
+ from the ServiceLoader.
+ @param conf the configuration to look in
+ @return a list of the {@link CompressionCodec} classes]]>
+      </doc>
+    </method>
+    <method name="setCodecClasses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="classes" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets a list of codec classes in the configuration. In addition to any
+ classes specified using this method, {@link CompressionCodec} classes on
+ the classpath are discovered using a Java ServiceLoader.
+ @param conf the configuration to modify
+ @param classes the list of classes to set]]>
+      </doc>
+    </method>
+    <method name="getCodec" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the given file based on its
+ filename suffix.
+ @param file the filename to check
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByClassName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classname" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name.
+ @param classname the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias.
+ <p/>
+ Codec aliases are case insensitive.
+ <p/>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias and returns its implemetation class.
+ <p/>
+ Codec aliases are case insensitive.
+ <p/>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec class]]>
+      </doc>
+    </method>
+    <method name="removeSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Removes a suffix from a filename, if it has it.
+ @param filename the filename to strip
+ @param suffix the suffix to remove
+ @return the shortened filename]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A little test program.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory that will find the correct codec for a given filename.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <class name="CompressionInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <constructor name="CompressionInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a compression input stream that reads
+ the decompressed bytes from the given stream.
+ 
+ @param in The input stream to be compressed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the decompressor to its initial state and discard any buffered data,
+ as the underlying stream may have been repositioned.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method returns the current position in the stream.
+
+ @return Current position in stream as a long]]>
+      </doc>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <field name="in" type="java.io.InputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The input stream to be compressed.]]>
+      </doc>
+    </field>
+    <field name="maxAvailableData" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A compression input stream.
+
+ <p>Implementations are assumed to be buffered.  This permits clients to
+ reposition the underlying input stream then call {@link #resetState()},
+ without having to also synchronize client buffers.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <class name="CompressionOutputStream" extends="java.io.OutputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionOutputStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a compression output stream that writes
+ the compressed bytes to the given stream.
+ @param out]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write compressed bytes to the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finishes writing compressed data to the output stream 
+ without closing the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the compression to the initial state. 
+ Does not reset the underlying stream.]]>
+      </doc>
+    </method>
+    <field name="out" type="java.io.OutputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The output stream to be compressed.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A compression output stream.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Compressor -->
+  <interface name="Compressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for compression. 
+ This should be called whenever #needsInput() returns 
+ <code>true</code> indicating that more input data is required.
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the input data buffer is empty and 
+ #setInput() should be called to provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ #setInput() should be called in order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary 
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="getBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of uncompressed bytes input so far.]]>
+      </doc>
+    </method>
+    <method name="getBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of compressed bytes output so far.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When called, indicates that compression should end
+ with the current contents of the input buffer.]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the end of the compressed 
+ data output stream has been reached.
+ @return <code>true</code> if the end of the compressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="compress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with compressed data. Returns actual number
+ of bytes of compressed data. A return value of 0 indicates that
+ needsInput() should be called in order to determine if more input
+ data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of compressed data.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets compressor so that a new set of input data can be processed.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the compressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <method name="reinit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Prepare the compressor to be used in a new stream with settings defined in
+ the given Configuration
+ 
+ @param conf Configuration from which new setting are fetched]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'compressor' which can be  
+ plugged into a {@link CompressionOutputStream} to compress data.
+ This is modelled after {@link java.util.zip.Deflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Compressor -->
+  <!-- start class org.apache.hadoop.io.compress.CompressorStream -->
+  <class name="CompressorStream" extends="org.apache.hadoop.io.compress.CompressionOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param out Underlying output stream.]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="compressor" type="org.apache.hadoop.io.compress.Compressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressorStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Decompressor -->
+  <interface name="Decompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for decompression. 
+ This should be called if and only if {@link #needsInput()} returns 
+ <code>true</code> indicating that more input data is required.
+ (Both native and non-native versions of various Decompressors require
+ that the data passed in via <code>b[]</code> remain unmodified until
+ the caller is explicitly notified--via {@link #needsInput()}--that the
+ buffer may be safely modified.  With this requirement, an extra
+ buffer-copy can be avoided.)
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called to
+ provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called in
+ order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsDictionary" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if a preset dictionary is needed for decompression.
+ @return <code>true</code> if a preset dictionary is needed for decompression]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the end of the decompressed 
+ data output stream has been reached. Indicates a concatenated data stream
+ when finished() returns <code>true</code> and {@link #getRemaining()}
+ returns a positive value. finished() will be reset with the
+ {@link #reset()} method.
+ @return <code>true</code> if the end of the decompressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="decompress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with uncompressed data. Returns actual number
+ of bytes of uncompressed data. A return value of 0 indicates that
+ {@link #needsInput()} should be called in order to determine if more
+ input data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of uncompressed data.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes remaining in the compressed data buffer.
+ Indicates a concatenated data stream if {@link #finished()} returns
+ <code>true</code> and getRemaining() returns a positive value. If
+ {@link #finished()} returns <code>true</code> and getRemaining() returns
+ a zero value, indicates that the end of data stream has been reached and
+ is not a concatenated data stream. 
+ @return The number of bytes remaining in the compressed data buffer.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets decompressor and input and output buffers so that a new set of
+ input data can be processed. If {@link #finished()}} returns
+ <code>true</code> and {@link #getRemaining()} returns a positive value,
+ reset() is called before processing of the next data stream in the
+ concatenated data stream. {@link #finished()} will be reset and will
+ return <code>false</code> when reset() is called.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the decompressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'de-compressor' which can be  
+ plugged into a {@link CompressionInputStream} to compress data.
+ This is modelled after {@link java.util.zip.Inflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Decompressor -->
+  <!-- start class org.apache.hadoop.io.compress.DecompressorStream -->
+  <class name="DecompressorStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param in Underlying input stream.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="available" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="markSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="mark"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readlimit" type="int"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="eof" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.DefaultCodec -->
+  <class name="DefaultCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <implements name="org.apache.hadoop.io.compress.DirectDecompressionCodec"/>
+    <constructor name="DefaultCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DefaultCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <interface name="DirectDecompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link DirectDecompressor} for use by this {@link DirectDecompressionCodec}.
+ 
+ @return a new direct decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a codec which can decompress direct bytebuffers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <interface name="DirectDecompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="decompress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.nio.ByteBuffer"/>
+      <param name="dst" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Specification of a direct ByteBuffer 'de-compressor'.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <!-- start class org.apache.hadoop.io.compress.GzipCodec -->
+  <class name="GzipCodec" extends="org.apache.hadoop.io.compress.DefaultCodec"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GzipCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class creates gzip compressors/decompressors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.GzipCodec -->
+  <!-- start class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <class name="SplitCompressionInputStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitCompressionInputStream" type="java.io.InputStream, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="setStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="start" type="long"/>
+    </method>
+    <method name="setEnd"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="end" type="long"/>
+    </method>
+    <method name="getAdjustedStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of start.
+ @return The changed value of start]]>
+      </doc>
+    </method>
+    <method name="getAdjustedEnd" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of end.
+ @return The changed value of end]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputStream covering a range of compressed data. The start and end
+ offsets requested by a client may be modified by the codec to fit block
+ boundaries or other algorithm-dependent requirements.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+  <interface name="SplittableCompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a stream as dictated by the readMode.  This method is used when
+ the codecs wants the ability to work with the underlying stream positions.
+
+ @param seekableIn  The seekable input stream (seeks in compressed data)
+ @param start The start offset into the compressed stream. May be changed
+              by the underlying codec.
+ @param end The end offset into the compressed stream. May be changed by
+            the underlying codec.
+ @param readMode Controls whether stream position is reported continuously
+                 from the compressed stream only only at block boundaries.
+ @return  a stream to read uncompressed bytes from]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface is meant to be implemented by those compression codecs
+ which are capable to compress / de-compress a stream starting at any
+ arbitrary position.
+
+ Especially the process of de-compressing a stream starting at some arbitrary
+ position is challenging.  Most of the codecs are only able to successfully
+ de-compress a stream, if they start from the very beginning till the end.
+ One of the reasons is the stored state at the beginning of the stream which
+ is crucial for de-compression.
+
+ Yet there are few codecs which do not save the whole state at the beginning
+ of the stream and hence can be used to de-compress stream starting at any
+ arbitrary points.  This interface is meant to be used by such codecs.  Such
+ codecs are highly valuable, especially in the context of Hadoop, because
+ an input compressed file can be split and hence can be worked on by multiple
+ machines in parallel.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode">
+  <!-- start class org.apache.hadoop.io.erasurecode.ECSchema -->
+  <class name="ECSchema" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="ECSchema" type="java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with schema name and provided all options. Note the options may
+ contain additional information for the erasure codec to interpret further.
+ @param allOptions all schema options]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int, java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided. Note the extraOptions may contain
+ additional information for the erasure codec to interpret further.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema
+ @param extraOptions extra options to configure the codec]]>
+      </doc>
+    </constructor>
+    <method name="getCodecName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the codec name
+ @return codec name]]>
+      </doc>
+    </method>
+    <method name="getExtraOptions" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get extra options specific to a erasure code.
+ @return extra options]]>
+      </doc>
+    </method>
+    <method name="getNumDataUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required data units count in a coding group
+ @return count of data units]]>
+      </doc>
+    </method>
+    <method name="getNumParityUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required parity units count in a coding group
+ @return count of parity units]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Make a meaningful string representation for log output.
+ @return string representation]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="NUM_DATA_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_PARITY_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CODEC_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Erasure coding schema to housekeeper relevant information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.erasurecode.ECSchema -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode.coder.util">
+</package>
+<package name="org.apache.hadoop.io.erasurecode.grouper">
+</package>
+<package name="org.apache.hadoop.io.file.tfile">
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <class name="MetaBlockAlreadyExists" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - Meta Block with the same name already exists.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <class name="MetaBlockDoesNotExist" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - No such Meta Block with the given name.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <!-- start interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <interface name="RawComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="buffer" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying byte array.
+ 
+ @return The underlying byte array.]]>
+      </doc>
+    </method>
+    <method name="offset" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the offset of the first byte in the byte array.
+ 
+ @return The offset of the first byte in the byte array.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the size of the byte range in the byte array.
+ 
+ @return The size of the byte range in the byte array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for objects that can be compared through {@link RawComparator}.
+ This is useful in places where we need a single object reference to specify a
+ range of bytes in a byte array, such as {@link Comparable} or
+ {@link Collections#binarySearch(java.util.List, Object, Comparator)}
+ 
+ The actual comparison among RawComparable's requires an external
+ RawComparator and it is applications' responsibility to ensure two
+ RawComparable are supposed to be semantically comparable with the same
+ RawComparator.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <!-- start class org.apache.hadoop.io.file.tfile.TFile -->
+  <class name="TFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="makeComparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Make a raw comparator from a string name.
+ 
+ @param name
+          Comparator name
+ @return A RawComparable comparator.]]>
+      </doc>
+    </method>
+    <method name="getSupportedCompressionAlgorithms" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get names of supported compression algorithms. The names are acceptable by
+ TFile.Writer.
+ 
+ @return Array of strings, each represents a supported compression
+         algorithm. Currently, the following compression algorithms are
+         supported.
+         <ul>
+         <li>"none" - No compression.
+         <li>"lzo" - LZO compression.
+         <li>"gz" - GZIP compression.
+         </ul>]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Dumping the TFile information.
+ 
+ @param args
+          A list of TFile paths.]]>
+      </doc>
+    </method>
+    <field name="COMPRESSION_GZ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: gzip]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_LZO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: lzo]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_NONE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: none]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_MEMCMP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator: memcmp]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_JCLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator prefix: java class]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A TFile is a container of key-value pairs. Both keys and values are type-less
+ bytes. Keys are restricted to 64KB, value length is not restricted
+ (practically limited to the available disk storage). TFile further provides
+ the following features:
+ <ul>
+ <li>Block Compression.
+ <li>Named meta data blocks.
+ <li>Sorted or unsorted keys.
+ <li>Seek by key or by file offset.
+ </ul>
+ The memory footprint of a TFile includes the following:
+ <ul>
+ <li>Some constant overhead of reading or writing a compressed block.
+ <ul>
+ <li>Each compressed block requires one compression/decompression codec for
+ I/O.
+ <li>Temporary space to buffer the key.
+ <li>Temporary space to buffer the value (for TFile.Writer only). Values are
+ chunk encoded, so that we buffer at most one chunk of user data. By default,
+ the chunk buffer is 1MB. Reading chunked value does not require additional
+ memory.
+ </ul>
+ <li>TFile index, which is proportional to the total number of Data Blocks.
+ The total amount of memory needed to hold the index can be estimated as
+ (56+AvgKeySize)*NumBlocks.
+ <li>MetaBlock index, which is proportional to the total number of Meta
+ Blocks.The total amount of memory needed to hold the index for Meta Blocks
+ can be estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ </ul>
+ <p>
+ The behavior of TFile can be customized by the following variables through
+ Configuration:
+ <ul>
+ <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default
+ to 1MB. Values of the length less than the chunk size is guaranteed to have
+ known value length in read time (See
+ {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for
+ FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for
+ FSDataInputStream. Integer (in bytes). Default to 256KB.
+ </ul>
+ <p>
+ Suggestions on performance optimization.
+ <ul>
+ <li>Minimum block size. We recommend a setting of minimum block size between
+ 256KB to 1MB for general usage. Larger block size is preferred if files are
+ primarily for sequential access. However, it would lead to inefficient random
+ access (because there are more data to decompress). Smaller blocks are good
+ for random access, but require more memory to hold the block index, and may
+ be slower to create (because we must flush the compressor stream at the
+ conclusion of each data block, which leads to an FS I/O flush). Further, due
+ to the internal caching in Compression codec, the smallest possible block
+ size would be around 20KB-30KB.
+ <li>The current implementation does not offer true multi-threading for
+ reading. The implementation uses FSDataInputStream seek()+read(), which is
+ shown to be much faster than positioned-read call in single thread mode.
+ However, it also means that if multiple threads attempt to access the same
+ TFile (using multiple scanners) simultaneously, the actual I/O is carried out
+ sequentially even if they access different DFS blocks.
+ <li>Compression codec. Use "none" if the data is not very compressable (by
+ compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
+ as the starting point for experimenting. "gz" overs slightly better
+ compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
+ decompress, comparing to "lzo".
+ <li>File system buffering, if the underlying FSDataInputStream and
+ FSDataOutputStream is already adequately buffered; or if applications
+ reads/writes keys and values in large buffers, we can reduce the sizes of
+ input/output buffering in TFile layer by setting the configuration parameters
+ "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ </ul>
+ 
+ Some design rationale behind TFile can be found at <a
+ href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.TFile -->
+  <!-- start class org.apache.hadoop.io.file.tfile.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding an integer into a variable-length encoding format. Synonymous to
+ <code>Utils#writeVLong(out, n)</code>.
+ 
+ @param out
+          output stream
+ @param n
+          The integer to be encoded
+ @throws IOException
+ @see Utils#writeVLong(DataOutput, long)]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding a Long integer into a variable-length encoding format.
+ <ul>
+ <li>if n in [-32, 127): encode in one byte with the actual value.
+ Otherwise,
+ <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
+ byte[1]=n&0xff. Otherwise,
+ <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
+ 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise,
+ <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
+ byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise:
+ <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
+ (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff;
+ <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
+ (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff;
+ byte[4]=(n>>8)&0xff; byte[5]=n&0xff
+ <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
+ (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff;
+ byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff;
+ <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
+ (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff;
+ byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff;
+ byte[7]=n&0xff;
+ <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
+ (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff;
+ byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff;
+ byte[7]=(n>>8)&0xff; byte[8]=n&0xff;
+ </ul>
+ 
+ @param out
+          output stream
+ @param n
+          the integer number
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Synonymous to
+ <code>(int)Utils#readVLong(in)</code>.
+ 
+ @param in
+          input stream
+ @return the decoded integer
+ @throws IOException
+ 
+ @see Utils#readVLong(DataInput)]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Suppose the value of the first byte
+ is FB, and the following bytes are NB[*].
+ <ul>
+ <li>if (FB >= -32), return (long)FB;
+ <li>if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff;
+ <li>if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 +
+ NB[1]&0xff;
+ <li>if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 +
+ (NB[1]&0xff)<<8 + NB[2]&0xff;
+ <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
+ big-endian integer.
+ 
+ @param in
+          input stream
+ @return the decoded long integer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a String as a VInt n, followed by n Bytes as in Text format.
+ 
+ @param out
+ @param s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a String as a VInt n, followed by n Bytes in Text format.
+ 
+ @param in
+          The input stream.
+ @return The string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Supporting Utility classes used by TFile, and shared by users of TFile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.Utils -->
+</package>
+<package name="org.apache.hadoop.io.serializer">
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <class name="JavaSerialization" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="JavaSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ An experimental {@link Serialization} for Java {@link Serializable} classes.
+ </p>
+ @see JavaSerializationComparator]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <class name="JavaSerializationComparator" extends="org.apache.hadoop.io.serializer.DeserializerComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JavaSerializationComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ A {@link RawComparator} that uses a {@link JavaSerialization}
+ {@link Deserializer} to deserialize objects that are then compared via
+ their {@link Comparable} interfaces.
+ </p>
+ @param <T>
+ @see JavaSerialization]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <!-- start class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <class name="WritableSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="WritableSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A {@link Serialization} for {@link Writable}s that delegates to
+ {@link Writable#write(java.io.DataOutput)} and
+ {@link Writable#readFields(java.io.DataInput)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides a mechanism for using different serialization frameworks
+in Hadoop. The property "io.serializations" defines a list of
+{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create
+{@link org.apache.hadoop.io.serializer.Serializer}s and
+{@link org.apache.hadoop.io.serializer.Deserializer}s.
+</p>
+
+<p>
+To add a new serialization framework write an implementation of
+{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the
+"io.serializations" property.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.serializer.avro">
+  <!-- start interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <interface name="AvroReflectSerializable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Tag interface for Avro 'reflect' serializable classes. Classes implementing 
+ this interface can be serialized/deserialized using 
+ {@link AvroReflectSerialization}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <class name="AvroReflectSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroReflectSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_REFLECT_PACKAGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key to configure packages that contain classes to be serialized and 
+ deserialized using this class. Multiple packages can be specified using 
+ comma-separated list.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Serialization for Avro Reflect classes. For a class to be accepted by this 
+ serialization, it must either be in the package list configured via 
+ <code>avro.reflect.pkgs</code> or implement 
+ {@link AvroReflectSerializable} interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <class name="AvroSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="AvroSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_SCHEMA_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class for providing serialization to Avro types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <class name="AvroSpecificSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroSpecificSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Serialization for Avro Specific classes. This serialization is to be used 
+ for classes generated by Avro's 'specific' compiler.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides Avro serialization in Hadoop. This can be used to 
+serialize/deserialize Avro types in Hadoop.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for 
+serialization of classes generated by Avro's 'specific' compiler.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for 
+other classes. 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for  
+any class which is either in the package list configured via 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} 
+or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable}
+interface.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.ipc.protocolPB">
+</package>
+<package name="org.apache.hadoop.log">
+</package>
+<package name="org.apache.hadoop.log.metrics">
+  <!-- start class org.apache.hadoop.log.metrics.EventCounter -->
+  <class name="EventCounter" extends="org.apache.log4j.AppenderSkeleton"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventCounter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="requiresLayout" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A log4J Appender that simply counts logging events in three levels:
+ fatal, error and warn. The class name is used in log4j.properties]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.log.metrics.EventCounter -->
+</package>
+<package name="org.apache.hadoop.metrics2">
+  <!-- start class org.apache.hadoop.metrics2.AbstractMetric -->
+  <class name="AbstractMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="AbstractMetric" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the metric
+ @param info  about the metric]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="value" return="java.lang.Number"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the metric
+ @return the value of the metric]]>
+      </doc>
+    </method>
+    <method name="type" return="org.apache.hadoop.metrics2.MetricType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the metric
+ @return the type of the metric]]>
+      </doc>
+    </method>
+    <method name="visit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visitor" type="org.apache.hadoop.metrics2.MetricsVisitor"/>
+      <doc>
+      <![CDATA[Accept a visitor interface
+ @param visitor of the metric]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The immutable metric]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.AbstractMetric -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <interface name="MetricsCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param name  of the record
+ @return  a {@link MetricsRecordBuilder} for the record {@code name}]]>
+      </doc>
+    </method>
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param info  of the record
+ @return  a {@link MetricsRecordBuilder} for metrics {@code info}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics collector interface]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsException -->
+  <class name="MetricsException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message
+ @param message for the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message and a cause
+ @param message for the exception
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a cause
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A general metrics exception wrapper]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsException -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsFilter -->
+  <class name="MetricsFilter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <constructor name="MetricsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether to accept the name
+ @param name  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Whether to accept the tag
+ @param tag to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Whether to accept the tags
+ @param tags to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Whether to accept the record
+ @param record  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics filter interface. The MetricsFilter objects can be used either to
+ filter the metrics from {@link MetricsSource}s or to filter metrics per
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsFilter -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <interface name="MetricsInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically name corresponds to annotation {@link Metric#value()} or
+ the name of the class.
+ @return the name of the metric/tag]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically the description corresponds to annotation {@link Metric#about()}
+ or the name of the class.
+ @return the description of the metric/tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to provide immutable metainfo for metrics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <class name="MetricsJsonBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsJsonBuilder" type="org.apache.hadoop.metrics2.MetricsCollector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method]]>
+      </doc>
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Build a JSON dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <interface name="MetricsPlugin"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+      <doc>
+      <![CDATA[Initialize the plugin
+ @param conf  the configuration object for the plugin]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The plugin interface for the metrics framework]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <interface name="MetricsRecord"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="timestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the metrics
+ @return  the timestamp]]>
+      </doc>
+    </method>
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics record name]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the description of the metrics record]]>
+      </doc>
+    </method>
+    <method name="context" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the context name of the metrics record]]>
+      </doc>
+    </method>
+    <method name="tags" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tags of the record
+ Note: returning a collection instead of iterable as we
+ need to use tags as keys (hence Collection#hashCode etc.) in maps
+ @return an unmodifiable collection of tags]]>
+      </doc>
+    </method>
+    <method name="metrics" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the metrics of the record
+ @return an immutable iterable interface for metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An immutable snapshot of metrics with a timestamp]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <class name="MetricsRecordBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRecordBuilder"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics value with metrics information
+ @param info  metadata of the tag
+ @param value of the tag
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Add an immutable metrics tag object
+ @param tag a pre-made tag object (potentially save an object construction)
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+      <doc>
+      <![CDATA[Add a pre-made immutable metric object
+ @param metric  the pre-made metric to save an object construction
+ @return self]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the context tag
+ @param value of the context
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add an integer metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add an long metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add a integer gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a long gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Add a float gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Add a double gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <method name="endRecord" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Syntactic sugar to add multiple records in a collector in a one liner.
+ @return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics record builder interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSink -->
+  <interface name="MetricsSink"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <method name="putMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Put a metrics record in the sink
+ @param record  the record to put]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flush any buffered metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics sink interface. <p>
+ Implementations of this interface consume the {@link MetricsRecord} generated
+ from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ periodically pushes the {@link MetricsRecord} to the sink using
+ {@link #putMetrics(MetricsRecord)} method.  If the implementing class also
+ implements {@link Closeable}, then the MetricsSystem will close the sink when
+ it is stopped.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSink -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSource -->
+  <interface name="MetricsSource"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collector" type="org.apache.hadoop.metrics2.MetricsCollector"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get metrics from the metrics source
+ @param collector to contain the resulting metrics snapshot
+ @param all if true, return all metrics even if unchanged.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The source of metrics information. It generates and updates metrics. It
+ registers with {@link MetricsSystem}, which periodically polls it to collect
+ {@link MetricsRecord} and passes it to {@link MetricsSink}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSource -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsSystem -->
+  <class name="MetricsSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSystemMXBean"/>
+    <constructor name="MetricsSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unregisterSource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unregister a metrics source
+ @param name of the source. This is the name you use to call register()]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source (deriving name and description from the object)
+ @param <T>   the actual type of the source object
+ @param source  object to register
+ @return  the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source
+ @param <T>   the actual type of the source object
+ @param source object to register
+ @param name  of the source. Must be unique or null (then extracted from
+              the annotations of the source object.)
+ @param desc  the description of the source (or null. See above.)
+ @return the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callback" type="org.apache.hadoop.metrics2.MetricsSystem.Callback"/>
+      <doc>
+      <![CDATA[Register a callback interface for JMX events
+ @param callback  the callback object implementing the MBean interface.]]>
+      </doc>
+    </method>
+    <method name="publishMetricsNow"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests an immediate publish of all metrics from sources to sinks.
+ 
+ This is a "soft" request: the expectation is that a best effort will be
+ done to synchronously snapshot the metrics from all the sources and put
+ them in all the sinks (including flushing the sinks) before returning to
+ the caller. If this can't be accomplished in reasonable time it's OK to
+ return to the caller before everything is done.]]>
+      </doc>
+    </method>
+    <method name="shutdown" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system completely (usually during server shutdown.)
+ The MetricsSystemMXBean will be unregistered.
+ @return true if shutdown completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics system interface.
+ 
+ The following components are used for metrics.
+ <ul>
+ <li>{@link MetricsSource} generate and update metrics information.</li>
+ <li>{@link MetricsSink} consume the metrics information</li>
+ </ul>
+ 
+ {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ system. Implementations of {@link MetricsSystem} polls the
+ {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsSystem -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <interface name="MetricsSystemMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="startMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start metrics MBeans
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stopMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop metrics MBeans.
+ Note, it doesn't stop the metrics system control MBean,
+ i.e this interface.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="currentConfig" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the current config
+ Avoided getConfig, as it'll turn into a "Config" attribute,
+ which doesn't support multiple line values in jconsole.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The JMX interface to the metrics system]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsTag -->
+  <class name="MetricsTag" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="MetricsTag" type="org.apache.hadoop.metrics2.MetricsInfo, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the tag with name, description and value
+ @param info  of the tag
+ @param value of the tag]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the tag]]>
+      </doc>
+    </method>
+    <method name="value" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the tag
+ @return  the value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Immutable tag for metrics (for grouping on host/queue/username etc.)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsTag -->
+  <!-- start class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <class name="MetricStringBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricStringBuilder" type="org.apache.hadoop.metrics2.MetricsCollector, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method
+ @param prefix string before each entry
+ @param separator separator between name and value
+ @param suffix suffix after each entry]]>
+      </doc>
+    </constructor>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="tuple" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add any key,val pair to the string, between the prefix and suffix,
+ separated by the separator.
+ @param key key
+ @param value value
+ @return this instance]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Build a string dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.
+
+ Every entry is formatted as
+ {@code prefix + name + separator + value + suffix}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsVisitor -->
+  <interface name="MetricsVisitor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Callback for float value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Callback for double value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A visitor interface for metrics]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsVisitor -->
+</package>
+<package name="org.apache.hadoop.metrics2.annotation">
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metric -->
+  <class name="Metric"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a single metric used to annotate a field or a method
+ in the class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metric -->
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metrics -->
+  <class name="Metrics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a group of metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metrics -->
+</package>
+<package name="org.apache.hadoop.metrics2.filter">
+  <!-- start class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <class name="GlobFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GlobFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A glob pattern filter for metrics.
+
+ The class name is used in metrics config files]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <!-- start class org.apache.hadoop.metrics2.filter.RegexFilter -->
+  <class name="RegexFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A regex pattern filter for metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.RegexFilter -->
+</package>
+<package name="org.apache.hadoop.metrics2.lib">
+  <!-- start class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <class name="DefaultMetricsSystem" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="initialize" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method to initialize the metrics system
+ @param prefix  for the metrics system configuration
+ @return the metrics system instance]]>
+      </doc>
+    </method>
+    <method name="instance" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics system object]]>
+      </doc>
+    </method>
+    <method name="shutdown"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system]]>
+      </doc>
+    </method>
+    <method name="setMiniClusterMode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="choice" type="boolean"/>
+    </method>
+    <method name="inMiniClusterMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The default metrics system singleton. This class is used by all the daemon
+ processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ to initialize the {@link MetricsSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <!-- start class org.apache.hadoop.metrics2.lib.Interns -->
+  <class name="Interns" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Interns"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric info object.
+ @param name Name of metric info object
+ @param description Description of metric info object
+ @return an interned metric info object]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param info  of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to create interned metrics info.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.Interns -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <class name="MetricsRegistry" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRegistry" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a record name
+ @param name  of the record of the metrics]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsRegistry" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a metadata object
+ @param info  the info object for the metrics record/group]]>
+      </doc>
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the metrics registry]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.lib.MutableMetric"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric by name
+ @param name  of the metric
+ @return the metric object]]>
+      </doc>
+    </method>
+    <method name="getTag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a tag by name
+ @param name  of the tag
+ @return the tag object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newQuantiles" return="org.apache.hadoop.metrics2.lib.MutableQuantiles"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="interval" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable metric that estimates quantiles of a stream of values
+ @param name of the metric
+ @param desc metric description
+ @param sampleName of the metric (e.g., "Ops")
+ @param valueName of the metric (e.g., "Time" or "Latency")
+ @param interval rollover interval of estimator in seconds
+ @return a new quantile estimator object
+ @throws MetricsException if interval is not a positive integer]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @param extended    produce extended stat (stdev, min/max etc.) if true.
+ @return a new mutable stat metric object]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @param description of the metric
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric (for throughput measurement)
+ @param name  of the metric
+ @param desc  description
+ @param extended  produce extended stat (stdev/min/max etc.) if true
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRatesWithAggregation" return="org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="newMutableRollingAverages" return="org.apache.hadoop.metrics2.lib.MutableRollingAverages"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add sample to a stat metric by name.
+ @param name  of the metric
+ @param value of the snapshot to add]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the metrics context tag
+ @param name of the context
+ @return the registry itself as a convenience]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @param override  existing tag if true
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param info  metadata of the tag
+ @param value of the tag
+ @param override existing tag if true
+ @return the registry (for keep adding tags etc.)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Sample all the mutable metrics and put the snapshot in the builder
+ @param builder to contain the metrics snapshot
+ @param all get all the metrics even if the values are not changed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An optional metrics registry class for creating and maintaining a
+ collection of MetricsMutables, making writing metrics source easier.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <class name="MutableCounter" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounter" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the metric value by 1.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable counter (monotonically increasing) metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <class name="MutableCounterInt" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable int counter for implementing metrics sources]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <class name="MutableCounterLong" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounterLong" type="org.apache.hadoop.metrics2.MetricsInfo, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable long counter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <class name="MutableGauge" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableGauge" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the value of the metric by 1]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Decrement the value of the metric by 1]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable gauge metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <class name="MutableGaugeInt" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable int gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <class name="MutableGaugeLong" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable long gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <class name="MutableMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="snapshot"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get a snapshot of the metric
+ @param builder the metrics record builder
+ @param all if true, snapshot unchanged metrics as well]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <doc>
+      <![CDATA[Get a snapshot of metric if changed
+ @param builder the metrics record builder]]>
+      </doc>
+    </method>
+    <method name="setChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the changed flag in mutable operations]]>
+      </doc>
+    </method>
+    <method name="clearChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the changed flag in the snapshot operations]]>
+      </doc>
+    </method>
+    <method name="changed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  true if metric is changed since last snapshot/snapshot]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <class name="MutableQuantiles" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableQuantiles" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiates a new {@link MutableQuantiles} for a metric that rolls itself
+ over on the specified time interval.
+ 
+ @param name
+          of the metric
+ @param description
+          long-form textual description of the metric
+ @param sampleName
+          type of items in the stream (e.g., "Ops")
+ @param valueName
+          type of the values
+ @param interval
+          rollover interval (in seconds) of the estimator]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="getInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEstimator" return="org.apache.hadoop.metrics2.util.QuantileEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the quantile estimator.
+
+ @return the quantile estimator]]>
+      </doc>
+    </method>
+    <method name="setEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quantileEstimator" type="org.apache.hadoop.metrics2.util.QuantileEstimator"/>
+    </method>
+    <field name="quantiles" type="org.apache.hadoop.metrics2.util.Quantile[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="previousSnapshot" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Watches a stream of long values, maintaining online estimates of specific
+ quantiles with provably low error bounds. This is particularly useful for
+ accurate high-percentile (e.g. 95th, 99th) latency metrics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <class name="MutableRate" extends="org.apache.hadoop.metrics2.lib.MutableStat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[A convenient mutable metric for throughput measurement]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <class name="MutableRates" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics
+
+ This class synchronizes all accesses to the metrics it
+ contains, so it should not be used in situations where
+ there is high contention on the metrics.
+ {@link MutableRatesWithAggregation} is preferable in that
+ situation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <class name="MutableRatesWithAggregation" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableRatesWithAggregation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric.
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics.
+
+ Each thread will maintain a local rate count, and upon snapshot,
+ these values will be aggregated into a global rate. This class
+ should only be used for long running threads, as any metrics
+ produced between the last snapshot and the death of a thread
+ will be lost. This allows for significantly higher concurrency
+ than {@link MutableRates}. See HADOOP-24420.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <class name="MutableRollingAverages" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="MutableRollingAverages" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for {@link MutableRollingAverages}.
+ @param metricValueName]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="collectThreadLocalStates"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Collects states maintained in {@link ThreadLocal}, if any.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[@param name
+          name of metric
+ @param value
+          value of metric]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStats" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="minSamples" type="long"/>
+      <doc>
+      <![CDATA[Retrieve a map of metric name -> (aggregate).
+ Filter out entries that don't have at least minSamples.
+
+ @return a map of peer DataNode Id to the average latency to that
+         node seen over the measurement period.]]>
+      </doc>
+    </method>
+    <method name="setRecordValidityMs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Use for test only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class maintains a group of rolling average metrics. It implements the
+ algorithm of rolling average, i.e. a number of sliding windows are kept to
+ roll over and evict old subsets of samples. Each window has a subset of
+ samples in a stream, where sub-sum and sub-total are collected. All sub-sums
+ and sub-totals in all windows will be aggregated to final-sum and final-total
+ used to compute final average, which is called rolling average.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableStat -->
+  <class name="MutableStat" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a sample statistics metric
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")
+ @param extended    create extended stats (stdev, min/max etc.) by default.]]>
+      </doc>
+    </constructor>
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a snapshot stat metric with extended stat off by default
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")]]>
+      </doc>
+    </constructor>
+    <method name="setExtended"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to display the extended stats (stdev, min/max etc.) or not
+ @param extended enable/disable displaying extended stats]]>
+      </doc>
+    </method>
+    <method name="setUpdateTimeStamp"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateTimeStamp" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to update the snapshot time or not.
+ @param updateTimeStamp enable update stats snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numSamples" type="long"/>
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Add a number of samples and their sum to the running stat
+
+ Note that although use of this method will preserve accurate mean values,
+ large values for numSamples may result in inaccurate variance values due
+ to the use of a single step of the Welford method for variance calculation.
+ @param numSamples  number of samples
+ @param sum of the samples]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a snapshot to the metric.
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="lastStat" return="org.apache.hadoop.metrics2.util.SampleStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a SampleStat object that supports
+ calls like StdDev and Mean.
+ @return SampleStat]]>
+      </doc>
+    </method>
+    <method name="resetMinMax"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset the all time min max of the metric]]>
+      </doc>
+    </method>
+    <method name="getSnapshotTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the SampleStat snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mutable metric with stats.
+
+ Useful for keeping throughput/latency stats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableStat -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink">
+  <!-- start class org.apache.hadoop.metrics2.sink.FileSink -->
+  <class name="FileSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="FileSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a file]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.FileSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <class name="GraphiteSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="GraphiteSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a Graphite server]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <class name="RollingFileSystemSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="RollingFileSystemSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty instance.  Required for reflection.]]>
+      </doc>
+    </constructor>
+    <constructor name="RollingFileSystemSink" type="long, long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance for testing.
+
+ @param flushIntervalMillis the roll interval in millis
+ @param flushOffsetIntervalMillis the roll offset interval in millis]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics2Properties" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="getRollInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Extract the roll interval from the configuration and return it in
+ milliseconds.
+
+ @return the roll interval in millis]]>
+      </doc>
+    </method>
+    <method name="updateFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Update the {@link #nextFlush} variable to the next flush time. Add
+ an integer number of flush intervals, preserving the initial random offset.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="setInitialFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Set the {@link #nextFlush} variable to the initial flush time. The initial
+ flush will be an integer number of flush intervals past the beginning of
+ the current hour and will have a random offset added, up to
+ {@link #rollOffsetIntervalMillis}. The initial flush will be a time in
+ past that can be used from which to calculate future flush times.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="source" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ignoreError" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allowAppend" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="basePath" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollOffsetIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nextFlush" type="java.util.Calendar"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="forceFlush" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="hasFlushed" type="boolean"
+      transient="false" volatile="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedConf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedFilesystem" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>This class is a metrics sink that uses
+ {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs.  Every
+ roll interval a new directory will be created under the path specified by the
+ <code>basepath</code> property. All metrics will be logged to a file in the
+ current interval's directory in a file named &lt;hostname&gt;.log, where
+ &lt;hostname&gt; is the name of the host on which the metrics logging
+ process is running. The base path is set by the
+ <code>&lt;prefix&gt;.sink.&lt;instance&gt;.basepath</code> property.  The
+ time zone used to create the current interval's directory name is GMT.  If
+ the <code>basepath</code> property isn't specified, it will default to
+ &quot;/tmp&quot;, which is the temp directory on whatever default file
+ system is configured for the cluster.</p>
+
+ <p>The <code>&lt;prefix&gt;.sink.&lt;instance&gt;.ignore-error</code>
+ property controls whether an exception is thrown when an error is encountered
+ writing a log file.  The default value is <code>true</code>.  When set to
+ <code>false</code>, file errors are quietly swallowed.</p>
+
+ <p>The <code>roll-interval</code> property sets the amount of time before
+ rolling the directory. The default value is 1 hour. The roll interval may
+ not be less than 1 minute. The property's value should be given as
+ <i>number unit</i>, where <i>number</i> is an integer value, and
+ <i>unit</i> is a valid unit.  Valid units are <i>minute</i>, <i>hour</i>,
+ and <i>day</i>.  The units are case insensitive and may be abbreviated or
+ plural. If no units are specified, hours are assumed. For example,
+ &quot;2&quot;, &quot;2h&quot;, &quot;2 hour&quot;, and
+ &quot;2 hours&quot; are all valid ways to specify two hours.</p>
+
+ <p>The <code>roll-offset-interval-millis</code> property sets the upper
+ bound on a random time interval (in milliseconds) that is used to delay
+ before the initial roll.  All subsequent rolls will happen an integer
+ number of roll intervals after the initial roll, hence retaining the original
+ offset. The purpose of this property is to insert some variance in the roll
+ times so that large clusters using this sink on every node don't cause a
+ performance impact on HDFS by rolling simultaneously.  The default value is
+ 30000 (30s).  When writing to HDFS, as a rule of thumb, the roll offset in
+ millis should be no less than the number of sink instances times 5.
+
+ <p>The primary use of this class is for logging to HDFS.  As it uses
+ {@link org.apache.hadoop.fs.FileSystem} to access the target file system,
+ however, it can be used to write to the local file system, Amazon S3, or any
+ other supported file system.  The base path for the sink will determine the
+ file system used.  An unqualified path will write to the default file system
+ set by the configuration.</p>
+
+ <p>Not all file systems support the ability to append to files.  In file
+ systems without the ability to append to files, only one writer can write to
+ a file at a time.  To allow for concurrent writes from multiple daemons on a
+ single host, the <code>source</code> property is used to set unique headers
+ for the log files.  The property should be set to the name of
+ the source daemon, e.g. <i>namenode</i>.  The value of the
+ <code>source</code> property should typically be the same as the property's
+ prefix.  If this property is not set, the source is taken to be
+ <i>unknown</i>.</p>
+
+ <p>Instead of appending to an existing file, by default the sink
+ will create a new file with a suffix of &quot;.&lt;n&gt;&quet;, where
+ <i>n</i> is the next lowest integer that isn't already used in a file name,
+ similar to the Hadoop daemon logs.  NOTE: the file with the <b>highest</b>
+ sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
+
+ <p>For file systems that allow append, the sink supports appending to the
+ existing file instead. If the <code>allow-append</code> property is set to
+ true, the sink will instead append to the existing file on file systems that
+ support appends. By default, the <code>allow-append</code> property is
+ false.</p>
+
+ <p>Note that when writing to HDFS with <code>allow-append</code> set to true,
+ there is a minimum acceptable number of data nodes.  If the number of data
+ nodes drops below that minimum, the append will succeed, but reading the
+ data will fail with an IOException in the DataStreamer class.  The minimum
+ number of data nodes required for a successful append is generally 2 or
+ 3.</p>
+
+ <p>Note also that when writing to HDFS, the file size information is not
+ updated until the file is closed (at the end of the interval) even though
+ the data is being written successfully. This is a known HDFS limitation that
+ exists because of the performance cost of updating the metadata.  See
+ <a href="https://issues.apache.org/jira/browse/HDFS-5478">HDFS-5478</a>.</p>
+
+ <p>When using this sink in a secure (Kerberos) environment, two additional
+ properties must be set: <code>keytab-key</code> and
+ <code>principal-key</code>. <code>keytab-key</code> should contain the key by
+ which the keytab file can be found in the configuration, for example,
+ <code>yarn.nodemanager.keytab</code>. <code>principal-key</code> should
+ contain the key by which the principal can be found in the configuration,
+ for example, <code>yarn.nodemanager.principal</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.StatsDSink -->
+  <class name="StatsDSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="StatsDSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="writeMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="line" type="java.lang.String"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes metrics to a StatsD daemon.
+ This sink will produce metrics of the form
+ '[hostname].servicename.context.name.metricname:value|type'
+ where hostname is optional. This is useful when sending to
+ a daemon that is running on the localhost and will add the
+ hostname to the metric (such as the
+ <a href="https://collectd.org/">CollectD</a> StatsD plugin).
+ <br/>
+ To configure this plugin, you will need to add the following
+ entries to your hadoop-metrics2.properties file:
+ <br/>
+ <pre>
+ *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.StatsDSink -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink.ganglia">
+</package>
+<package name="org.apache.hadoop.metrics2.source">
+</package>
+<package name="org.apache.hadoop.metrics2.util">
+  <!-- start class org.apache.hadoop.metrics2.util.MBeans -->
+  <class name="MBeans" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service=<serviceName>,name=<nameName>"
+ Where the <serviceName> and <nameName> are the supplied parameters.
+
+ @param serviceName
+ @param nameName
+ @param theMbean - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="properties" type="java.util.Map"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service=<serviceName>,name=<nameName>"
+ Where the <serviceName> and <nameName> are the supplied parameters.
+
+ @param serviceName
+ @param nameName
+ @param properties - Key value pairs to define additional JMX ObjectName
+                     properties.
+ @param theMbean    - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="getMbeanNameService" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="getMbeanNameName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mbeanName" type="javax.management.ObjectName"/>
+    </method>
+    <doc>
+    <![CDATA[This util class provides a method to register an MBean using
+ our standard naming convention as described in the doc
+  for {link {@link #register(String, String, Object)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MBeans -->
+  <!-- start class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <class name="MetricsCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="MetricsCache" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a metrics cache
+ @param maxRecsPerName  limit of the number records per record name]]>
+      </doc>
+    </constructor>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <param name="includingTags" type="boolean"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cached record
+ @param mr the update record
+ @param includingTags cache tag values (for later lookup by name) if true
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cache record
+ @param mr the update record
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="tags" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Get the cached record
+ @param name of the record
+ @param tags of the record
+ @return the cached record or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A metrics cache for sinks that don't support sparse updates.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <!-- start class org.apache.hadoop.metrics2.util.Servers -->
+  <class name="Servers" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parse" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="specs" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Parses a space and/or comma separated sequence of server specifications
+ of the form <i>hostname</i> or <i>hostname:port</i>.  If
+ the specs string is null, defaults to localhost:defaultPort.
+
+ @param specs   server specs (see description)
+ @param defaultPort the default port if not specified
+ @return a list of InetSocketAddress objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to handle server addresses]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.Servers -->
+</package>
+<package name="org.apache.hadoop.net">
+  <!-- start class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <class name="AbstractDNSToSwitchMapping" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.net.DNSToSwitchMapping"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractDNSToSwitchMapping"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an unconfigured instance]]>
+      </doc>
+    </constructor>
+    <constructor name="AbstractDNSToSwitchMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance, caching the configuration file.
+ This constructor does not call {@link #setConf(Configuration)}; if
+ a subclass extracts information in that method, it must call it explicitly.
+ @param conf the configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate that indicates that the switch mapping is known to be
+ single-switch. The base class returns false: it assumes all mappings are
+ multi-rack. Subclasses may override this with methods that are more aware
+ of their topologies.
+
+ <p/>
+
+ This method is used when parts of Hadoop need know whether to apply
+ single rack vs multi-rack policies, such as during block placement.
+ Such algorithms behave differently if they are on multi-switch systems.
+ </p>
+
+ @return true if the mapping thinks that it is on a single switch]]>
+      </doc>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the map (for diagnostics)
+ @return a clone of the map or null for none known]]>
+      </doc>
+    </method>
+    <method name="dumpTopology" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a string listing the switch mapping implementation,
+ the mapping for every known node and the number of nodes and
+ unique switches known about -each entry to a separate line.
+ @return a string that can be presented to the ops team or used in
+ debug messages.]]>
+      </doc>
+    </method>
+    <method name="isSingleSwitchByScriptPolicy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMappingSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapping" type="org.apache.hadoop.net.DNSToSwitchMapping"/>
+      <doc>
+      <![CDATA[Query for a {@link DNSToSwitchMapping} instance being on a single
+ switch.
+ <p/>
+ This predicate simply assumes that all mappings not derived from
+ this class are multi-switch.
+ @param mapping the mapping to query
+ @return true if the base class says it is single switch, or the mapping
+ is not derived from this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a base class for DNS to Switch mappings. <p/> It is not mandatory to
+ derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
+ recommended, as it makes it easy for the Hadoop developers to add new methods
+ to this base class that are automatically picked up by all implementations.
+ <p/>
+
+ This class does not extend the <code>Configured</code>
+ base class, and should not be changed to do so, as it causes problems
+ for subclasses. The constructor of the <code>Configured</code> calls
+ the  {@link #setConf(Configuration)} method, which will call into the
+ subclasses before they have been fully constructed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <class name="CachedDNSToSwitchMapping" extends="org.apache.hadoop.net.AbstractDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CachedDNSToSwitchMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[cache a raw DNS mapping
+ @param rawMapping the raw mapping to cache]]>
+      </doc>
+    </constructor>
+    <method name="resolve" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the (host x switch) map.
+ @return a copy of the cached map of hosts to rack]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegate the switch topology query to the raw mapping, via
+ {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+ @return true iff the raw mapper is considered single-switch.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <field name="rawMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The uncached mapping]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A cached implementation of DNSToSwitchMapping that takes an
+ raw DNSToSwitchMapping and stores the resolved network location in 
+ a cache. The following calls to a resolved network location
+ will get its location from the cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ConnectTimeoutException -->
+  <class name="ConnectTimeoutException" extends="java.net.SocketTimeoutException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConnectTimeoutException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown by {@link NetUtils#connect(java.net.Socket, java.net.SocketAddress, int)}
+ if it times out while connecting to the remote host.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ConnectTimeoutException -->
+  <!-- start interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <interface name="DNSToSwitchMapping"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="resolve" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Resolves a list of DNS-names/IP-addresses and returns back a list of
+ switch information (network paths). One-to-one correspondence must be 
+ maintained between the elements in the lists. 
+ Consider an element in the argument list - x.y.com. The switch information
+ that is returned must be a network path of the form /foo/rack, 
+ where / is the root, and 'foo' is the switch where 'rack' is connected.
+ Note the hostname/ip-address is not part of the returned path.
+ The network topology of the cluster would determine the number of
+ components in the network path.
+ <p/>
+
+ If a name cannot be resolved to a rack, the implementation
+ should return {@link NetworkTopology#DEFAULT_RACK}. This
+ is what the bundled implementations do, though it is not a formal requirement
+
+ @param names the list of hosts to resolve (can be empty)
+ @return list of resolved network paths.
+ If <i>names</i> is empty, the returned list is also empty]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload all of the cached mappings.
+
+ If there is a cache, this method will clear it, so that future accesses
+ will get a chance to see the new data.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Reload cached mappings on specific nodes.
+
+ If there is a cache on these nodes, this method will clear it, so that 
+ future accesses will see updated data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface that must be implemented to allow pluggable
+ DNS-name/IP-address to RackID resolvers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ScriptBasedMapping -->
+  <class name="ScriptBasedMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ScriptBasedMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with the default configuration.
+ </p>
+ Calling {@link #setConf(Configuration)} will trigger a
+ re-evaluation of the configuration settings and so be used to
+ set up the mapping script.]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given raw mapping
+ @param rawMap raw DNSTOSwithMapping]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given configuration
+ @param conf configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p/>
+ This will get called in the superclass constructor, so a check is needed
+ to ensure that the raw mapping is defined before trying to relaying a null
+ configuration.
+ @param conf]]>
+      </doc>
+    </method>
+    <field name="NO_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Text used in the {@link #toString()} method if there is no string
+ {@value}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class implements the {@link DNSToSwitchMapping} interface using a 
+ script configured via the
+ {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ <p/>
+ It contains a static class <code>RawScriptBasedMapping</code> that performs
+ the work: reading the configuration parameters, executing any defined
+ script, handling errors and such like. The outer
+ class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ queries.
+ <p/>
+ This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ true if and only if a script is defined.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ScriptBasedMapping -->
+  <!-- start class org.apache.hadoop.net.SocksSocketFactory -->
+  <class name="SocksSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="SocksSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <constructor name="SocksSocketFactory" type="java.net.Proxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with a supplied Proxy
+ 
+ @param proxy the proxy to use to create sockets]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.SocksSocketFactory -->
+  <!-- start class org.apache.hadoop.net.StandardSocketFactory -->
+  <class name="StandardSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StandardSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.StandardSocketFactory -->
+  <!-- start class org.apache.hadoop.net.TableMapping -->
+  <class name="TableMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TableMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ file. The columns are separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ </p>
+ <p>
+ This class uses the configuration parameter {@code
+ net.topology.table.file.name} to locate the mapping file.
+ </p>
+ <p>
+ Calls to {@link #resolve(List)} will look up the address as defined in the
+ mapping file. If no entry corresponding to the address is found, the value
+ {@code /default-rack} is returned.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.TableMapping -->
+  <doc>
+  <![CDATA[Network-related classes.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.net.unix">
+</package>
+<package name="org.apache.hadoop.security">
+  <!-- start class org.apache.hadoop.security.AccessControlException -->
+  <class name="AccessControlException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessControlException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor is needed for unwrapping from 
+ {@link org.apache.hadoop.ipc.RemoteException}.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an {@link AccessControlException}
+ with the specified detail message.
+ @param s the detail message.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[An exception class for access control related issues.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.AccessControlException -->
+  <!-- start class org.apache.hadoop.security.Credentials -->
+  <class name="Credentials" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty credentials instance.]]>
+      </doc>
+    </constructor>
+    <constructor name="Credentials" type="org.apache.hadoop.security.Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of the given credentials.
+ @param credentials to copy]]>
+      </doc>
+    </constructor>
+    <method name="getToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the Token object for the alias.
+ @param alias the alias for the Token
+ @return token for this alias]]>
+      </doc>
+    </method>
+    <method name="addToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="t" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token in the storage (in memory).
+ @param alias the alias for the key
+ @param t the token object]]>
+      </doc>
+    </method>
+    <method name="getAllTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the tokens in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getTokenMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to Tokens.]]>
+      </doc>
+    </method>
+    <method name="numberOfTokens" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of Tokens in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="getSecretKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the key bytes for the alias.
+ @param alias the alias for the key
+ @return key for this alias]]>
+      </doc>
+    </method>
+    <method name="numberOfSecretKeys" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of keys in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="addSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Set the key for an alias.
+ @param alias the alias for the key
+ @param key the key bytes]]>
+      </doc>
+    </method>
+    <method name="removeSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Remove the key for a given alias.
+ @param alias the alias for the key]]>
+      </doc>
+    </method>
+    <method name="getAllSecretKeys" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the secret key entries in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getSecretKeyMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to secret keys.]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.io.File"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token from a DataInputStream.]]>
+      </doc>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores all the keys to DataOutput.
+ @param out
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Loads all the keys.
+ @param in
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <method name="mergeAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are not overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that provides the facilities of reading and writing
+ secret keys and Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.Credentials -->
+  <!-- start interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <interface name="GroupMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getGroups" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all various group memberships of a given user.
+ Returns EMPTY list in case of non-existing user
+ @param user User's name
+ @return group memberships of user
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsRefresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the cache of groups and user mapping
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsAdd"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groups" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Caches the group user information
+ @param groups list of groups to add to cache
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="GROUP_MAPPING_CONFIG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An interface for the implementation of a user-to-groups mapping service
+ used by {@link Groups}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <!-- start interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <interface name="IdMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getUid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getGid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getGroupName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="gid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getUidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getGidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[An interface for the implementation of <userId, userName> mapping
+ and <groupId, groupName> mapping]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <!-- start class org.apache.hadoop.security.KerberosAuthException -->
+  <class name="KerberosAuthException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosAuthException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="u" type="java.lang.String"/>
+    </method>
+    <method name="setPrincipal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+    </method>
+    <method name="setKeytabFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="java.lang.String"/>
+    </method>
+    <method name="setTicketCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.String"/>
+    </method>
+    <method name="getInitialMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The initial message, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getKeytabFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The keytab file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The principal, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getTicketCacheFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The ticket cache file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The user, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown when {@link UserGroupInformation} failed with an unrecoverable error,
+ such as failure in kerberos login/logout, invalid subject etc.
+
+ Caller should not retry when catching this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.KerberosAuthException -->
+  <!-- start class org.apache.hadoop.security.SecurityUtil -->
+  <class name="SecurityUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOriginalTGT" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ticket" type="javax.security.auth.kerberos.KerberosTicket"/>
+      <doc>
+      <![CDATA[Check whether the server principal is the TGS's principal
+ @param ticket the original TGT (the ticket that is obtained when a 
+ kinit is done)
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal
+ names. It replaces hostname pattern with hostname, which should be
+ fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
+ dynamically looked-up fqdn of the current host instead.
+ 
+ @param principalConfig
+          the Kerberos principal name conf value to convert
+ @param hostname
+          the fully-qualified domain name used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal names.
+ This method is similar to {@link #getServerPrincipal(String, String)},
+ except 1) the reverse DNS lookup from addr to hostname is done only when
+ necessary, 2) param addr can't be null (no default behavior of using local
+ hostname when addr is null).
+ 
+ @param principalConfig
+          Kerberos principal name pattern to convert
+ @param addr
+          InetAddress of the host used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in
+ user's Kerberos principal name with a dynamically looked-up fully-qualified
+ domain name of the current host.
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in user's Kerberos principal 
+ name with hostname. If non-secure mode - return. If no keytab available -
+ bail out with an exception
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @param hostname
+          hostname to use for substitution
+ @throws IOException if the config doesn't specify a keytab]]>
+      </doc>
+    </method>
+    <method name="buildDTServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="defPort" type="int"/>
+      <doc>
+      <![CDATA[create the service name for a Delegation token
+ @param uri of the service
+ @param defPort is used if the uri lacks a port
+ @return the token service, or null if no authority
+ @see #buildTokenService(InetSocketAddress)]]>
+      </doc>
+    </method>
+    <method name="getHostFromPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the host name from the principal name of format <service>/host@realm.
+ @param principalName principal name of format as described above
+ @return host name if the the string conforms to the above format, else null]]>
+      </doc>
+    </method>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the KerberosInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return the KerberosInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getClientPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the client principal for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return client principal or null if it has no client principal defined.]]>
+      </doc>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the TokenInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol The protocol class to get the information for.
+ @param conf Configuration object
+ @return the TokenInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getTokenServiceAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Decode the given token's service field into an InetAddress
+ @param token from which to obtain the service
+ @return InetAddress for the service]]>
+      </doc>
+    </method>
+    <method name="setTokenService"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the given token's service to the format expected by the RPC client 
+ @param token a delegation token
+ @param addr the socket for the rpc connection]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param addr InetSocketAddress of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param uri of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUserOrFatal" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If the login
+ user cannot be determined, this will log a FATAL error and exit
+ the whole JVM.]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="doAsCurrentUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's current user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticationMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isPrivilegedPort" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+    </method>
+    <method name="getZKAuthInfos" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="configKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility method to fetch ZK auth info from the configuration.
+ @throws java.io.IOException if the Zookeeper ACLs configuration file
+ cannot be read
+ @throws ZKUtil.BadAuthFormatException if the auth format is invalid]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HOSTNAME_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED_TO_GET_UGI_MSG_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Security Utils.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.SecurityUtil -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation -->
+  <class name="UserGroupInformation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setShouldRenewImmediatelyForTests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="immediate" type="boolean"/>
+      <doc>
+      <![CDATA[For the purposes of unit tests, we want to test login
+ from keytab and don't want to wait until the renew
+ window (controlled by TICKET_RENEW_WINDOW).
+ @param immediate true if we should login without waiting for ticket window]]>
+      </doc>
+    </method>
+    <method name="reattachMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reattach the class's metrics to a new metric system.]]>
+      </doc>
+    </method>
+    <method name="isInitialized" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the static configuration for UGI.
+ In particular, set the security authentication mechanism and the
+ group look up service.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="isSecurityEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Determine if UserGroupInformation is using Kerberos to determine
+ user identities or is relying on simple authentication
+ 
+ @return true if UGI is working in a secure environment]]>
+      </doc>
+    </method>
+    <method name="hasKerberosCredentials" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[checks if logged in using kerberos
+ @return true if the subject logged via keytab or has a Kerberos TGT]]>
+      </doc>
+    </method>
+    <method name="getCurrentUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current user, including any doAs in the current stack.
+ @return the current user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="getBestUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCachePath" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the most appropriate UserGroupInformation to use
+
+ @param ticketCachePath    The Kerberos ticket cache path, or NULL
+                           if none is specfied
+ @param user               The user name, or NULL if none is specified.
+
+ @return                   The most appropriate UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getUGIFromTicketCache" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCache" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Kerberos ticket cache.
+ 
+ @param user                The principal name to load from the ticket
+                            cache
+ @param ticketCache     the path to the ticket cache file
+
+ @throws IOException        if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getUGIFromSubject" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Subject with Kerberos principal.
+
+ @param subject             The KerberosPrincipal to use in UGI.
+                            The creator of subject is responsible for
+                            renewing credentials.
+
+ @throws IOException
+ @throws KerberosAuthException if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getLoginUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the currently logged in user.  If no explicit login has occurred,
+ the user will automatically be logged in with either kerberos credentials
+ if available, or as the local OS user, based on security settings.
+ @return the logged in user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="trimLoginMethod" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[remove the login method that is followed by a space from the username
+ e.g. "jack (auth:SIMPLE)" -> "jack"
+
+ @param userName
+ @return userName without login method]]>
+      </doc>
+    </method>
+    <method name="loginUserFromSubject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log in a user using the given subject
+ @param subject the subject to use when logging in a user, or null to
+ create a new subject.
+
+ If subject is not null, the creator of subject is responsible for renewing
+ credentials.
+
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="isFromKeytab" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this user logged in from a keytab file managed by the UGI?
+ @return true if the credentials are from a keytab file.]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="logoutUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log the current user out who previously logged in using keytab.
+ This method assumes that the user logged in by calling
+ {@link #loginUserFromKeytab(String, String)}.
+
+ @throws IOException
+ @throws KerberosAuthException if a failure occurred in logout,
+ or if the user did not log in by invoking loginUserFromKeyTab() before.]]>
+      </doc>
+    </method>
+    <method name="checkTGTAndReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-login a user from keytab if TGT is expired or is close to expiry.
+ 
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="reloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user. This
+ method assumes that {@link #loginUserFromKeytab(String, String)} had
+ happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="forceReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Force re-Login a user in from a keytab file irrespective of the last login
+ time. Loads a user identity from a keytab file and logs them in. They
+ become the currently logged-in user. This method assumes that
+ {@link #loginUserFromKeytab(String, String)} had happened already. The
+ Subject field of this UserGroupInformation object is updated to have the
+ new credentials.
+
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="reloginFromTicketCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from the ticket cache.  This
+ method assumes that login had happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytabAndReturnUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and login them in. This new user does not affect the currently
+ logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException if the keytab file can't be read]]>
+      </doc>
+    </method>
+    <method name="isLoginKeytabBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via keytab
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="isLoginTicketBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via ticket cache
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createProxyUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create a proxy user using username of the effective user and the ugi of the
+ real user.
+ @param user
+ @param realUser
+ @return proxyUser ugi]]>
+      </doc>
+    </method>
+    <method name="getRealUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get RealUser (vs. EffectiveUser)
+ @return realUser running over proxy user]]>
+      </doc>
+    </method>
+    <method name="createUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a UGI for testing HDFS and MapReduce
+ @param user the full user principal name
+ @param userGroups the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="createProxyUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a proxy user UGI for testing HDFS and MapReduce
+ 
+ @param user
+          the full user principal name for effective user
+ @param realUser
+          UGI of the real user
+ @param userGroups
+          the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="getShortUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's login name.
+ @return the user's name up to the first '/' or '@'.]]>
+      </doc>
+    </method>
+    <method name="getPrimaryGroupName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's full principal name.
+ @return the user's full principal name.]]>
+      </doc>
+    </method>
+    <method name="addTokenIdentifier" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokenId" type="org.apache.hadoop.security.token.TokenIdentifier"/>
+      <doc>
+      <![CDATA[Add a TokenIdentifier to this UGI. The TokenIdentifier has typically been
+ authenticated by the RPC layer as belonging to the user represented by this
+ UGI.
+ 
+ @param tokenId
+          tokenIdentifier to be added
+ @return true on successful add of new tokenIdentifier]]>
+      </doc>
+    </method>
+    <method name="getTokenIdentifiers" return="java.util.Set"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of TokenIdentifiers belonging to this UGI
+ 
+ @return the set of TokenIdentifiers belonging to this UGI]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token to this UGI
+ 
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a named token to this UGI
+ 
+ @param alias Name of the token
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="getTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the collection of tokens associated with this user.
+ 
+ @return an unmodifiable collection of tokens associated with user]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the tokens in credentials form associated with this user.
+ 
+ @return Credentials of tokens associated with this user]]>
+      </doc>
+    </method>
+    <method name="addCredentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="credentials" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Add the given Credentials to this user.
+ @param credentials of tokens and secrets]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user. {@link #getGroups()} is less
+ expensive alternative when checking for a contained element.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the username.]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the subject
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the real user's subject.  If there
+ is no real user, return the given user's authentication method.
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Returns the authentication method of a ugi. If the authentication method is
+ PROXY, returns the authentication method of the real user.
+ 
+ @param ugi
+ @return AuthenticationMethod]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare the subjects to see if they are equal to each other.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the hash of the subject.]]>
+      </doc>
+    </method>
+    <method name="getSubject" return="javax.security.auth.Subject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying subject from this ugi.
+ @return the subject that represents this user.]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Run the given action as the user.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the given action as the user, potentially throwing an exception.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method
+ @throws IOException if the action throws an IOException
+ @throws Error if the action throws an Error
+ @throws RuntimeException if the action throws a RuntimeException
+ @throws InterruptedException if the action throws an InterruptedException
+ @throws UndeclaredThrowableException if the action throws something else]]>
+      </doc>
+    </method>
+    <method name="logAllUserInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log all (current, real, login) UGI and token info into UGI debug log.
+ @param ugi - UGI
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A test method to print out the current user's UGI.
+ @param args if there are two arguments, read the user from the keytab
+ and print it out.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <field name="HADOOP_TOKEN_FILE_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the token cache file]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[User and group information for Hadoop.
+ This class wraps around a JAAS Subject and provides methods to determine the
+ user's username and groups. It supports both the Windows, Unix and Kerberos 
+ login modules.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+  <class name="UserGroupInformation.AuthenticationMethod" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getAuthMethod" return="org.apache.hadoop.security.SaslRpcServer.AuthMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+    </method>
+    <doc>
+    <![CDATA[existing types of authentications' methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+</package>
+<package name="org.apache.hadoop.security.alias">
+  <!-- start class org.apache.hadoop.security.alias.CredentialProvider -->
+  <class name="CredentialProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide job access to
+ passwords rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the credentials are written to persistent
+ store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the credential entry for a specific alias.
+ @param alias the name of a specific credential
+ @return the credentialEntry
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAliases" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the aliases for all credentials.
+ @return the list of alias names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="credential" type="char[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new credential. The given alias must not already exist.
+ @param name the alias of the credential
+ @param credential the credential value for the alias.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteCredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given credential.
+ @param name the alias of the credential to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of credentials or password for Hadoop applications. Provides an
+ abstraction to separate credential storage from users of them. It
+ is intended to support getting or storing passwords in a variety of ways,
+ including third party bindings.
+ 
+ <code>CredentialProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProvider -->
+  <!-- start class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+  <class name="CredentialProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.security.alias.CredentialProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of CredentialProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ CredentialProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+</package>
+<package name="org.apache.hadoop.security.authorize">
+  <!-- start class org.apache.hadoop.security.authorize.AccessControlList -->
+  <class name="AccessControlList" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="AccessControlList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constructor exists primarily for AccessControlList to be Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from a String representation of the same.
+ 
+ The String is a a comma separated list of users and groups.
+ The user list comes first and is separated by a space followed 
+ by the group list. For e.g. "user1,user2 group1,group2"
+ 
+ @param aclString String representation of the ACL]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from String representation of users and groups
+ 
+ The arguments are comma separated lists
+ 
+ @param users comma separated list of users
+ @param groups comma separated list of groups]]>
+      </doc>
+    </constructor>
+    <method name="isAllAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add user to the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="addGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add group to the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="removeUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove user from the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="removeGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove group from the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="getUsers" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of users allowed for this service.
+ @return the set of user names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of user groups allowed for this service.
+ @return the set of group names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="isUserInList" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Checks if a user represented by the provided {@link UserGroupInformation}
+ is a member of the Access Control List. If user was proxied and
+ USE_REAL_ACLS + the real user name is in the control list, then treat this
+ case as if user were in the ACL list.
+ @param ugi UserGroupInformation to check if contained in the ACL
+ @return true if ugi is member of the list or if USE_REAL_ACLS + real user
+ is in the list]]>
+      </doc>
+    </method>
+    <method name="isUserAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns descriptive way of users and groups that are part of this ACL.
+ Use {@link #getAclString()} to get the exact String that can be given to
+ the constructor of AccessControlList to create a new instance.]]>
+      </doc>
+    </method>
+    <method name="getAclString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the access control list as a String that can be used for building a
+ new instance by sending it to the constructor of {@link AccessControlList}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <field name="WILDCARD_ACL_VALUE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_REAL_ACLS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Class representing a configured access control list.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AccessControlList -->
+  <!-- start class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <class name="AuthorizationException" extends="org.apache.hadoop.security.AccessControlException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuthorizationException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <method name="getStackTrace" return="java.lang.StackTraceElement[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintStream"/>
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintWriter"/>
+    </method>
+    <doc>
+    <![CDATA[An exception class for authorization-related issues.
+ 
+ This class <em>does not</em> provide the stack trace for security purposes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <!-- start class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <class name="DefaultImpersonationProvider" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authorize.ImpersonationProvider"/>
+    <constructor name="DefaultImpersonationProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTestProvider" return="org.apache.hadoop.security.authorize.DefaultImpersonationProvider"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+    </method>
+    <method name="getProxySuperuserUserConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective usergroups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser usergroups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserGroupConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective groups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser groups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserIpConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return configuration key for superuser ip addresses
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser ip-addresses]]>
+      </doc>
+    </method>
+    <method name="getProxyGroups" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProxyHosts" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <!-- start interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+  <interface name="ImpersonationProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Specifies the configuration prefix for the proxy user properties and
+ initializes the provider.
+
+ @param configurationPrefix the configuration prefix for the proxy user
+ properties]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.lang.String"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+ {@link #authorize(UserGroupInformation, InetAddress)} should
+             be preferred to avoid possibly re-resolving the ip address.
+ @param user ugi of the effective or proxy user which contains a real user.
+ @param remoteAddress the ip address of client.
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+
+ @param user ugi of the effective or proxy user which contains a real user
+ @param remoteAddress the ip address of client
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+</package>
+<package name="org.apache.hadoop.security.http">
+  <!-- start class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <class name="RestCsrfPreventionFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="RestCsrfPreventionFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filterConfig" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="isBrowser" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userAgent" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This method interrogates the User-Agent String and returns whether it
+ refers to a browser.  If its not a browser, then the requirement for the
+ CSRF header will not be enforced; if it is a browser, the requirement will
+ be enforced.
+ <p>
+ A User-Agent String is considered to be a browser if it matches
+ any of the regex patterns from browser-useragent-regex; the default
+ behavior is to consider everything a browser that matches the following:
+ "^Mozilla.*,^Opera.*".  Subclasses can optionally override
+ this method to use different behavior.
+
+ @param userAgent The User-Agent String, or null if there isn't one
+ @return true if the User-Agent String refers to a browser, false if not]]>
+      </doc>
+    </method>
+    <method name="handleHttpInteraction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="httpInteraction" type="org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+      <doc>
+      <![CDATA[Handles an {@link HttpInteraction} by applying the filtering logic.
+
+ @param httpInteraction caller's HTTP interaction
+ @throws IOException if there is an I/O error
+ @throws ServletException if the implementation relies on the servlet API
+     and a servlet API call has failed]]>
+      </doc>
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="javax.servlet.ServletRequest"/>
+      <param name="response" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="HEADER_USER_AGENT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BROWSER_USER_AGENT_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_METHODS_TO_IGNORE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HEADER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter provides protection against cross site request forgery (CSRF)
+ attacks for REST APIs. Enabling this filter on an endpoint results in the
+ requirement of all client to send a particular (configurable) HTTP header
+ with every request. In the absense of this header the filter will reject the
+ attempt as a bad request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <!-- start class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+  <class name="XFrameOptionsFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="XFrameOptionsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="javax.servlet.ServletRequest"/>
+      <param name="res" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="X_FRAME_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter protects webapps from clickjacking attacks that
+ are possible through use of Frames to embed the resources in another
+ application and intercept clicks to accomplish nefarious things.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+</package>
+<package name="org.apache.hadoop.security.protocolPB">
+</package>
+<package name="org.apache.hadoop.security.ssl">
+</package>
+<package name="org.apache.hadoop.security.token">
+  <!-- start class org.apache.hadoop.security.token.SecretManager -->
+  <class name="SecretManager" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createPassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <doc>
+      <![CDATA[Create the password for the given identifier.
+ identifier may be modified inside this method.
+ @param identifier the identifier to use
+ @return the new password]]>
+      </doc>
+    </method>
+    <method name="retrievePassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <doc>
+      <![CDATA[Retrieve the password for the given token identifier. Should check the date
+ or registry to make sure the token hasn't expired or been revoked. Returns 
+ the relevant password.
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid]]>
+      </doc>
+    </method>
+    <method name="retriableRetrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <exception name="RetriableException" type="org.apache.hadoop.ipc.RetriableException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The same functionality with {@link #retrievePassword}, except that this 
+ method can throw a {@link RetriableException} or a {@link StandbyException}
+ to indicate that client can retry/failover the same operation because of 
+ temporary issue on the server side.
+ 
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid
+ @throws StandbyException the server is in standby state, the client can
+         try other servers
+ @throws RetriableException the token was invalid, and the server thinks 
+         this may be a temporary issue and suggests the client to retry
+ @throws IOException to allow future exceptions to be added without breaking
+         compatibility]]>
+      </doc>
+    </method>
+    <method name="createIdentifier" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty token identifier.
+ @return the newly created empty token identifier]]>
+      </doc>
+    </method>
+    <method name="checkAvailableForRead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <doc>
+      <![CDATA[No-op if the secret manager is available for reading tokens, throw a
+ StandbyException otherwise.
+ 
+ @throws StandbyException if the secret manager is not available to read
+         tokens]]>
+      </doc>
+    </method>
+    <method name="generateSecret" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a new random secret key.
+ @return the new key]]>
+      </doc>
+    </method>
+    <method name="createPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="byte[]"/>
+      <param name="key" type="javax.crypto.SecretKey"/>
+      <doc>
+      <![CDATA[Compute HMAC of the identifier using the secret key and return the 
+ output as password
+ @param identifier the bytes of the identifier
+ @param key the secret key
+ @return the bytes of the generated password]]>
+      </doc>
+    </method>
+    <method name="createSecretKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Convert the byte[] to a secret key
+ @param key the byte[] to create a secret key from
+ @return the secret key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The server-side secret manager for each token type.
+ @param <T> The type of the token identifier]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.SecretManager -->
+  <!-- start class org.apache.hadoop.security.token.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Token" type="T, org.apache.hadoop.security.token.SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token given a token identifier and a secret manager for the
+ type of the token identifier.
+ @param id the token identifier
+ @param mgr the secret manager]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="byte[], byte[], org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token from the components.
+ @param identifier the token identifier
+ @param password the token's password
+ @param kind the kind of token
+ @param service the service for this token]]>
+      </doc>
+    </constructor>
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="org.apache.hadoop.security.token.Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clone a token.
+ @param other the token to clone]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="org.apache.hadoop.security.proto.SecurityProtos.TokenProto"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Token from a TokenProto.
+ @param tokenPB the TokenProto object]]>
+      </doc>
+    </constructor>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+    </method>
+    <method name="setPassword"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newPassword" type="byte[]"/>
+    </method>
+    <method name="copyToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toTokenProto" return="org.apache.hadoop.security.proto.SecurityProtos.TokenProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a TokenProto from this Token instance.
+ @return a new TokenProto object holding copies of data in this instance]]>
+      </doc>
+    </method>
+    <method name="getIdentifier" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier's byte representation.
+ @return the token identifier's byte representation]]>
+      </doc>
+    </method>
+    <method name="decodeIdentifier" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the token identifier object, or null if it could not be constructed
+ (because the class could not be loaded, for example).
+ @return the token identifier, or null if there was no class found for it
+ @throws IOException failure to unmarshall the data
+ @throws RuntimeException if the token class could not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password/secret.
+ @return the token password/secret]]>
+      </doc>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service on which the token is supposed to be used.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="setService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Set the service on which the token is supposed to be used.
+ @param newService the service name]]>
+      </doc>
+    </method>
+    <method name="isPrivate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether this is a private token.
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="isPrivateCloneOf" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thePublicService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Whether this is a private clone of a public token.
+ @param thePublicService the public service name
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="privateClone" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Create a private clone of a public token.
+ @param newService the new service name
+ @return a private token]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="encodeToUrlString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode this token as a url safe string.
+ @return the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="decodeFromUrlString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode the given url safe string into this token.
+ @param newValue the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="buildCacheKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is this token managed so that it can be renewed or cancelled?
+ @return true, if it can be renewed and cancelled.]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew this delegation token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel this delegation token.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The client-side form of the token.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token -->
+  <!-- start class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <class name="Token.TrivialRenewer" extends="org.apache.hadoop.security.token.TokenRenewer"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrivialRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="handleKind" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[A trivial renewer for token kinds that aren't managed. Sub-classes need
+ to implement getKind for their token kind.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <!-- start class org.apache.hadoop.security.token.TokenIdentifier -->
+  <class name="TokenIdentifier" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Ugi with the username encoded in the token identifier
+ 
+ @return the username. null is returned if username in the identifier is
+         empty or null.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes for the token identifier
+ @return the bytes of the identifier]]>
+      </doc>
+    </method>
+    <method name="getTrackingId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a tracking identifier that can be used to associate usages of a
+ token across multiple client sessions.
+
+ Currently, this function just returns an MD5 of {{@link #getBytes()}.
+
+ @return tracking identifier]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An identifier that identifies a token, may contain public information 
+ about a token, including its kind (or type).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenIdentifier -->
+  <!-- start class org.apache.hadoop.security.token.TokenInfo -->
+  <class name="TokenInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Indicates Token related information to be used]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenInfo -->
+  <!-- start class org.apache.hadoop.security.token.TokenRenewer -->
+  <class name="TokenRenewer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="handleKind" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Does this renewer handle this kind of token?
+ @param kind the kind of the token
+ @return true if this renewer can renew it]]>
+      </doc>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is the given token managed? Only managed tokens may be renewed or
+ cancelled.
+ @param token the token being checked
+ @return true if the token may be renewed or cancelled
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew the given token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel the given token
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for plugins that handle tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenRenewer -->
+  <!-- start interface org.apache.hadoop.security.token.TokenSelector -->
+  <interface name="TokenSelector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+    <doc>
+    <![CDATA[Select token of type T from tokens for use with named service
+ 
+ @param <T>
+          T extends TokenIdentifier]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.token.TokenSelector -->
+</package>
+<package name="org.apache.hadoop.security.token.delegation.web">
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <class name="DelegationTokenAuthenticatedURL" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DelegationTokenAuthenticatedURL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+ <p/>
+ An instance of the default {@link DelegationTokenAuthenticator} will be
+ used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code> using the default
+ {@link DelegationTokenAuthenticator} class.
+
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator, org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <method name="setDefaultDelegationTokenAuthenticator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticator" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Sets the default {@link DelegationTokenAuthenticator} class to use when an
+ {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @param authenticator the authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="getDefaultDelegationTokenAuthenticator" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the default {@link DelegationTokenAuthenticator} class to use when
+ an {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+ <p/>
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @return the delegation token authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="setUseQueryStringForDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="useQueryString" type="boolean"/>
+      <doc>
+      <![CDATA[Sets if delegation token should be transmitted in the URL query string.
+ By default it is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ <p/>
+ This method is provided to enable WebHDFS backwards compatibility.
+
+ @param useQueryString  <code>TRUE</code> if the token is transmitted in the
+ URL query string, <code>FALSE</code> if the delegation token is transmitted
+ using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ header.]]>
+      </doc>
+    </method>
+    <method name="useQueryStringForDelegationToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns if delegation token is transmitted as a HTTP header.
+
+ @return <code>TRUE</code> if the token is transmitted in the URL query
+ string, <code>FALSE</code> if the delegation token is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
+ Token only if the given auth token is an instance of {@link Token} and
+ it contains a Delegation Token, otherwise use the configured
+ {@link DelegationTokenAuthenticator} to authenticate the connection.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAs" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>. If the <code>doAs</code> parameter is not NULL,
+ the request will be done on behalf of the specified <code>doAs</code> user.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @param doAs user to do the the request on behalf of, if NULL the request is
+ as self.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The <code>DelegationTokenAuthenticatedURL</code> is a
+ {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
+ functionality.
+ <p/>
+ The authentication mechanisms supported by default are Hadoop Simple
+ authentication (also known as pseudo authentication) and Kerberos SPNEGO
+ authentication.
+ <p/>
+ Additional authentication mechanisms can be supported via {@link
+ DelegationTokenAuthenticator} implementations.
+ <p/>
+ The default {@link DelegationTokenAuthenticator} is the {@link
+ KerberosDelegationTokenAuthenticator} class which supports
+ automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
+ the {@link PseudoDelegationTokenAuthenticator} class.
+ <p/>
+ <code>AuthenticatedURL</code> instances are not thread-safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <class name="DelegationTokenAuthenticatedURL.Token" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delegationToken" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <doc>
+    <![CDATA[Client side authentication token that handles Delegation Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <class name="DelegationTokenAuthenticator" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authentication.client.Authenticator"/>
+    <constructor name="DelegationTokenAuthenticator" type="org.apache.hadoop.security.authentication.client.Authenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConnectionConfigurator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurator" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"/>
+    </method>
+    <method name="authenticate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <field name="OP_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TOKEN_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEWER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_URL_STRING_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEW_DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@link Authenticator} wrapper that enhances an {@link Authenticator} with
+ Delegation Token support.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <class name="KerberosDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>KerberosDelegationTokenAuthenticator</code> provides support for
+ Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
+ Token operations.
+ <p/>
+ It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
+ endpoint does not trigger a SPNEGO authentication]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+  <class name="PseudoDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PseudoDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>PseudoDelegationTokenAuthenticator</code> provides support for
+ Hadoop's pseudo authentication mechanism that accepts
+ the user name specified as a query string parameter and support for Hadoop
+ Delegation Token operations.
+ <p/>
+ This mimics the model of Hadoop Simple authentication trusting the
+ {@link UserGroupInformation#getCurrentUser()} value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+</package>
+<package name="org.apache.hadoop.service">
+  <!-- start class org.apache.hadoop.service.AbstractService -->
+  <class name="AbstractService" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <constructor name="AbstractService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the service.
+ @param name service name]]>
+      </doc>
+    </constructor>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration for this service.
+ This method is called during {@link #init(Configuration)}
+ and should only be needed if for some reason a service implementation
+ needs to override that initial setting -for example replacing
+ it with a new subclass of {@link Configuration}
+ @param conf new configuration.]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ This invokes {@link #serviceInit}
+ @param conf the configuration of the service. This must not be null
+ @throws ServiceStateException if the configuration was null,
+ the state change not permitted, or something else went wrong]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws ServiceStateException if the current service state does not permit
+ this action]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Relay to {@link #stop()}
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noteFailure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <param name="exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Failure handling: record the exception
+ that triggered it -if there was not one already.
+ Services are free to call this themselves.
+ @param exception the exception]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[All initialization code needed by a service.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #init(Configuration)} prevents re-entrancy.
+
+ The base implementation checks to see if the subclass has created
+ a new configuration instance, and if so, updates the base class value
+ @param conf configuration
+ @throws Exception on a failure -these will be caught,
+ possibly wrapped, and wil; trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the INITED to STARTED transition.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #start()} prevents re-entrancy.
+
+ @throws Exception if needed -these will be caught,
+ wrapped, and trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the transition to the STOPPED state.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #stop()} prevents re-entrancy.
+
+ Implementations MUST write this to be robust against failures, including
+ checks for null references -and for the first failure to not stop other
+ attempts to shut down parts of the service.
+
+ @throws Exception if needed -these will be caught and logged.]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="registerGlobalListener"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a global listener, which receives notifications
+ from the state change events of all services in the JVM
+ @param l listener]]>
+      </doc>
+    </method>
+    <method name="unregisterGlobalListener" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[unregister a global listener.
+ @param l listener to unregister
+ @return true if the listener was found (and then deleted)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="expected" type="org.apache.hadoop.service.Service.STATE"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="putBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="details" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a blocker to the blocker map -replacing any
+ with the same name.
+ @param name blocker name
+ @param details any specifics on the block. This must be non-null.]]>
+      </doc>
+    </method>
+    <method name="removeBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a blocker from the blocker map -
+ this is a no-op if the blocker is not present
+ @param name the name of the blocker]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This is the base implementation class for services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.AbstractService -->
+  <!-- start class org.apache.hadoop.service.CompositeService -->
+  <class name="CompositeService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getServices" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a cloned list of services
+ @return a list of child services at the time of invocation -
+ added services will not be picked up.]]>
+      </doc>
+    </method>
+    <method name="addService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Add the passed {@link Service} to the list of services managed by this
+ {@link CompositeService}
+ @param service the {@link Service} to be added]]>
+      </doc>
+    </method>
+    <method name="addIfService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="object" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[If the passed object is an instance of {@link Service},
+ add it to the list of services managed by this {@link CompositeService}
+ @param object
+ @return true if a service is added, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="removeService" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="STOP_ONLY_STARTED_SERVICES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Policy on shutdown: attempt to close everything (purest) or
+ only try to close started services (which assumes
+ that the service implementations may not handle the stop() operation
+ except when started.
+ Irrespective of this policy, if a child service fails during
+ its init() or start() operations, it will have stop() called on it.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Composition of services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.CompositeService -->
+  <!-- start class org.apache.hadoop.service.LifecycleEvent -->
+  <class name="LifecycleEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="LifecycleEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="time" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Local time in milliseconds when the event occurred]]>
+      </doc>
+    </field>
+    <field name="state" type="org.apache.hadoop.service.Service.STATE"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[new state]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A serializable lifecycle event: the time a state
+ transition occurred, and what state was entered.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LifecycleEvent -->
+  <!-- start class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <class name="LoggingStateChangeListener" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    <constructor name="LoggingStateChangeListener" type="org.slf4j.Logger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the given log
+ @param log destination for events]]>
+      </doc>
+    </constructor>
+    <constructor name="LoggingStateChangeListener"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the static log for this class]]>
+      </doc>
+    </constructor>
+    <method name="stateChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback for a state change event: log it
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a state change listener that logs events at INFO level]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <!-- start interface org.apache.hadoop.service.Service -->
+  <interface name="Service"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Initialize the service.
+
+ The transition MUST be from {@link STATE#NOTINITED} to {@link STATE#INITED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @param config the configuration of the service
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the service.
+
+ The transition MUST be from {@link STATE#INITED} to {@link STATE#STARTED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the service. This MUST be a no-op if the service is already
+ in the {@link STATE#STOPPED} state. It SHOULD be a best-effort attempt
+ to stop all parts of the service.
+
+ The implementation must be designed to complete regardless of the service
+ state, including the initialized/uninitialized state of all its internal
+ fields.
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A version of stop() that is designed to be usable in Java7 closure
+ clauses.
+ Implementation classes MUST relay this directly to {@link #stop()}
+ @throws IOException never
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a listener to the service state change events.
+ If the supplied listener is already listening to this service,
+ this method is a no-op.
+ @param listener a new listener]]>
+      </doc>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Unregister a previously registered listener of the service state
+ change events. No-op if the listener is already unregistered.
+ @param listener the listener to unregister.]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this service.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration of this service.
+ This is normally not a clone and may be manipulated, though there are no
+ guarantees as to what the consequences of such actions may be
+ @return the current configuration, unless a specific implentation chooses
+ otherwise.]]>
+      </doc>
+    </method>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current service state
+ @return the state of the service]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service start time
+ @return the start time of the service. This will be zero if the service
+ has not yet been started.]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query to see if the service is in a specific state.
+ In a multi-threaded system, the state may not hold for very long.
+ @param state the expected state
+ @return true if, at the time of invocation, the service was in that state.]]>
+      </doc>
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the first exception raised during the service failure. If null,
+ no exception was logged
+ @return the failure logged during a transition to the stopped state]]>
+      </doc>
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the state in which the failure in {@link #getFailureCause()} occurred.
+ @return the state or null if there was no failure]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+      <doc>
+      <![CDATA[Block waiting for the service to stop; uses the termination notification
+ object to do so.
+
+ This method will only return after all the service stop actions
+ have been executed (to success or failure), or the timeout elapsed
+ This method can be called before the service is inited or started; this is
+ to eliminate any race condition with the service stopping before
+ this event occurs.
+ @param timeout timeout in milliseconds. A value of zero means "forever"
+ @return true iff the service stopped in the time period]]>
+      </doc>
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a snapshot of the lifecycle history; it is a static list
+ @return a possibly empty but never null list of lifecycle events.]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the blockers on a service -remote dependencies
+ that are stopping the service from being <i>live</i>.
+ @return a (snapshotted) map of blocker name-&gt;description values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Service LifeCycle.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.Service -->
+  <!-- start class org.apache.hadoop.service.ServiceOperations -->
+  <class name="ServiceOperations" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service.
+ <p/>Do nothing if the service is null or not
+ in a state in which it can be/needs to be stopped.
+ <p/>
+ The service state is checked <i>before</i> the operation begins.
+ This process is <i>not</i> thread safe.
+ @param service a service or null]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains a set of methods to work with services, especially
+ to walk them through their lifecycle.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceOperations -->
+  <!-- start interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <interface name="ServiceStateChangeListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="stateChanged"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback to notify of a state change. The service will already
+ have changed state before this callback is invoked.
+
+ This operation is invoked on the thread that initiated the state change,
+ while the service itself in in a sychronized section.
+ <ol>
+   <li>Any long-lived operation here will prevent the service state
+   change from completing in a timely manner.</li>
+   <li>If another thread is somehow invoked from the listener, and
+   that thread invokes the methods of the service (including
+   subclass-specific methods), there is a risk of a deadlock.</li>
+ </ol>
+
+
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to notify state changes of a service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <!-- start class org.apache.hadoop.service.ServiceStateException -->
+  <class name="ServiceStateException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <constructor name="ServiceStateException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate
+ @param message error message]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate with a message and cause; if the cause has an exit code
+ then it is used, otherwise the generic
+ {@link LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION} exit code
+ is used.
+ @param message exception message
+ @param cause optional inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="int, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate, using the specified exit code as the exit code
+ of the exception, irrespetive of any exit code supplied by any inner
+ cause.
+
+ @param exitCode exit code to declare
+ @param message exception message
+ @param cause inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="text" type="java.lang.String"/>
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ If the caught exception is already of that type, it is typecast to a
+ {@link RuntimeException} and returned.
+
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param text text to use if a new exception is created
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Exception that can be raised on state change operations, whose
+ exit code can be explicitly set, determined from that of any nested
+ cause, or a default value of
+ {@link  LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateException -->
+  <!-- start class org.apache.hadoop.service.ServiceStateModel -->
+  <class name="ServiceStateModel" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceStateModel" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the service state model in the {@link Service.STATE#NOTINITED}
+ state.]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateModel" type="java.lang.String, org.apache.hadoop.service.Service.STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a service state model instance in the chosen state
+ @param state the starting state]]>
+      </doc>
+    </constructor>
+    <method name="getState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Query the service state. This is a non-blocking operation.
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query that the state is in a specific state
+ @param proposed proposed new state
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="ensureCurrentState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expectedState" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Verify that that a service is in a given state.
+ @param expectedState the desired state
+ @throws ServiceStateException if the service state is different from
+ the desired state]]>
+      </doc>
+    </method>
+    <method name="enterState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Enter a state -thread safe.
+
+ @param proposed proposed new state
+ @return the original state
+ @throws ServiceStateException if the transition is not permitted]]>
+      </doc>
+    </method>
+    <method name="checkStateTransition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Check that a state tansition is valid and
+ throw an exception if not
+ @param name name of the service (can be null)
+ @param state current state
+ @param proposed proposed new state]]>
+      </doc>
+    </method>
+    <method name="isValidStateTransition" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="current" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Is a state transition valid?
+ There are no checks for current==proposed
+ as that is considered a non-transition.
+
+ using an array kills off all branch misprediction costs, at the expense
+ of cache line misses.
+
+ @param current current state
+ @param proposed proposed new state
+ @return true if the transition to a new state is valid]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return the state text as the toString() value
+ @return the current state's description]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the service state model.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateModel -->
+</package>
+<package name="org.apache.hadoop.service.launcher">
+  <!-- start class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <class name="AbstractLaunchableService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.launcher.LaunchableService"/>
+    <constructor name="AbstractLaunchableService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance with the given name.]]>
+      </doc>
+    </constructor>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The base implementation logs all arguments at the debug level,
+ then returns the passed in config unchanged.]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The action is to signal success by returning the exit code 0.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Subclass of {@link AbstractService} that provides basic implementations
+ of the {@link LaunchableService} methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <!-- start class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <class name="HadoopUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="HadoopUncaughtExceptionHandler" type="java.lang.Thread.UncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance delegating to the supplied handler if
+ the exception is considered "simple".
+ @param delegate a delegate exception handler.]]>
+      </doc>
+    </constructor>
+    <constructor name="HadoopUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Basic exception handler -logs simple exceptions, then continues.]]>
+      </doc>
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thread" type="java.lang.Thread"/>
+      <param name="exception" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Uncaught exception handler.
+ If an error is raised: shutdown
+ The state of the system is unknown at this point -attempting
+ a clean shutdown is dangerous. Instead: exit
+ @param thread thread that failed
+ @param exception the raised exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ in the main entry point. 
+
+ The base class will always attempt to shut down the process if an Error
+ was raised; the behavior on a standard Exception, raised outside 
+ process shutdown, is simply to log it. 
+
+ (Based on the class {@code YarnUncaughtExceptionHandler})]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <!-- start interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <interface name="LaunchableService"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Propagate the command line arguments.
+ <p>
+ This method is called before {@link #init(Configuration)};
+ Any non-null configuration that is returned from this operation
+ becomes the one that is passed on to that {@link #init(Configuration)}
+ operation.
+ <p>
+ This permits implementations to change the configuration before
+ the init operation. As the ServiceLauncher only creates
+ an instance of the base {@link Configuration} class, it is
+ recommended to instantiate any subclass (such as YarnConfiguration)
+ that injects new resources.
+ <p>
+ @param config the initial configuration build up by the
+ service launcher.
+ @param args list of arguments passed to the command line
+ after any launcher-specific commands have been stripped.
+ @return the configuration to init the service with.
+ Recommended: pass down the config parameter with any changes
+ @throws Exception any problem]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Run a service. This method is called after {@link Service#start()}.
+ <p>
+ The return value becomes the exit code of the launched process.
+ <p>
+ If an exception is raised, the policy is:
+ <ol>
+   <li>Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}:
+   the exception is passed up unmodified.
+   </li>
+   <li>Any exception which implements
+   {@link org.apache.hadoop.util.ExitCodeProvider}:
+   A new {@link ServiceLaunchException} is created with the exit code
+   and message of the thrown exception; the thrown exception becomes the
+   cause.</li>
+   <li>Any other exception: a new {@link ServiceLaunchException} is created
+   with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and
+   the message of the original exception (which becomes the cause).</li>
+ </ol>
+ @return the exit code
+ @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed
+  up as the exit code and error text.
+ @throws Exception any exception to report. If it provides an exit code
+ this is used in a wrapping exception.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface which services can implement to have their
+ execution managed by the ServiceLauncher.
+ <p>
+ The command line options will be passed down before the 
+ {@link Service#init(Configuration)} operation is invoked via an
+ invocation of {@link LaunchableService#bindArgs(Configuration, List)}
+ After the service has been successfully started via {@link Service#start()}
+ the {@link LaunchableService#execute()} method is called to execute the 
+ service. When this method returns, the service launcher will exit, using
+ the return code from the method as its exit option.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <!-- start interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <interface name="LauncherExitCodes"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="EXIT_SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Success: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FAIL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generic "false/fail" response: {@value}.
+ The operation worked but the result was not "true" from the viewpoint
+ of the executed code.]]>
+      </doc>
+    </field>
+    <field name="EXIT_CLIENT_INITIATED_SHUTDOWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a client requested service termination: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_TASK_LAUNCH_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when targets could not be launched: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_INTERRUPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a control-C, kill -3, signal was picked up: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_OTHER_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when something happened but we can't be specific: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_COMMAND_ARGUMENT_ERROR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the command line doesn't parse: {@value}, or
+ when it is otherwise invalid.
+ <p>
+ Approximate HTTP equivalent: {@code 400 Bad Request}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNAUTHORIZED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The request requires user authentication: {@value}.
+ <p>
+ approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]>
+      </doc>
+    </field>
+    <field name="EXIT_USAGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a usage message was printed: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FORBIDDEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Forbidden action: {@value}.
+ <p>
+ Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_FOUND" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Something was not found: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 404: Not Found}]]>
+      </doc>
+    </field>
+    <field name="EXIT_OPERATION_NOT_ALLOWED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The operation is not allowed: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 405: Not allowed}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_ACCEPTABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The command is somehow not acceptable: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 406: Not Acceptable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_CONNECTIVITY_PROBLEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code on connectivity problems: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 408: Request Timeout}]]>
+      </doc>
+    </field>
+    <field name="EXIT_BAD_CONFIGURATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the configurations in valid/incomplete: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 409: Conflict}]]>
+      </doc>
+    </field>
+    <field name="EXIT_EXCEPTION_THROWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when an exception was thrown from the service: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 500 Internal Server Error}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNIMPLEMENTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Unimplemented feature: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 501: Not Implemented}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Service Unavailable; it may be available later: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 503 Service Unavailable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNSUPPORTED_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The application does not support, or refuses to support this
+ version: {@value}.
+ <p>
+ If raised, this is expected to be raised server-side and likely due
+ to client/server version incompatibilities.
+ <p>
+ Approximate HTTP equivalent: {@code 505: Version Not Supported}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_CREATION_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_LIFECYCLE_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Common Exit codes.
+ <p>
+ Codes with a YARN prefix are YARN-related.
+ <p>
+ Many of the exit codes are designed to resemble HTTP error codes,
+ squashed into a single byte. e.g 44 , "not found" is the equivalent
+ of 404. The various 2XX HTTP error codes aren't followed;
+ the Unix standard of "0" for success is used.
+ <pre>
+    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <!-- start class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+  <class name="ServiceLaunchException" extends="org.apache.hadoop.util.ExitUtil.ExitException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <implements name="org.apache.hadoop.service.launcher.LauncherExitCodes"/>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code.
+ @param exitCode exit code
+ @param cause cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code and text.
+ @param exitCode exit code
+ @param message message to use in exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ <p>
+ If the last argument is a throwable, it becomes the cause of the exception.
+ It will also be used as a parameter for the format.
+ @param exitCode exit code
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A service launch exception that includes an exit code.
+ <p>
+ When caught by the ServiceLauncher, it will convert that
+ into a process exit code.
+ 
+ The {@link #ServiceLaunchException(int, String, Object...)} constructor
+ generates formatted exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+</package>
+<package name="org.apache.hadoop.tools">
+</package>
+<package name="org.apache.hadoop.tools.protocolPB">
+</package>
+<package name="org.apache.hadoop.tracing">
+  <!-- start class org.apache.hadoop.tracing.SpanReceiverInfo -->
+  <class name="SpanReceiverInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClassName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.tracing.SpanReceiverInfo -->
+  <!-- start class org.apache.hadoop.tracing.SpanReceiverInfoBuilder -->
+  <class name="SpanReceiverInfoBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SpanReceiverInfoBuilder" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addConfigurationPair"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="build" return="org.apache.hadoop.tracing.SpanReceiverInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.tracing.SpanReceiverInfoBuilder -->
+  <!-- start interface org.apache.hadoop.tracing.TraceAdminProtocol -->
+  <interface name="TraceAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="listSpanReceivers" return="org.apache.hadoop.tracing.SpanReceiverInfo[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the currently active trace span receivers.
+ 
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <method name="addSpanReceiver" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desc" type="org.apache.hadoop.tracing.SpanReceiverInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a new trace span receiver.
+ 
+ @param desc                The span receiver description.
+ @return                    The ID of the new trace span receiver.
+
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <method name="removeSpanReceiver"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="spanReceiverId" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove a trace span receiver.
+
+ @param spanReceiverId      The id of the span receiver to remove.
+ @throws IOException        On error.]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides tracing.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.tracing.TraceAdminProtocol -->
+  <!-- start interface org.apache.hadoop.tracing.TraceAdminProtocolPB -->
+  <interface name="TraceAdminProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.tracing.TraceAdminPB.TraceAdminService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.tracing.TraceAdminProtocolPB -->
+</package>
+<package name="org.apache.hadoop.util">
+  <!-- start class org.apache.hadoop.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="java.net.URLClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="resolve" type="boolean"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="isSystemClass" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="systemClasses" type="java.util.List"/>
+      <doc>
+      <![CDATA[Checks if a class should be included as a system class.
+
+ A class is a system class if and only if it matches one of the positive
+ patterns and none of the negative ones.
+
+ @param name the class name to check
+ @param systemClasses a list of system class configurations.
+ @return true if the class is a system class]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_CLASSES_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of the system classes if the user did not override them.
+ JDK classes, hadoop classes and resources, and some select third-party
+ classes are considered system classes, and are not loaded by the
+ application classloader.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A {@link URLClassLoader} for application isolation. Classes from the
+ application JARs are loaded in preference to the parent loader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ApplicationClassLoader -->
+  <!-- start interface org.apache.hadoop.util.IPList -->
+  <interface name="IPList"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="isIn" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ipAddress" type="java.lang.String"/>
+      <doc>
+      <![CDATA[returns true if the ipAddress is in the IPList.
+ @param ipAddress
+ @return boolean value indicating whether the ipAddress is in the IPList]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.IPList -->
+  <!-- start interface org.apache.hadoop.util.Progressable -->
+  <interface name="Progressable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="progress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Report progress to the Hadoop framework.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A facility for reporting progress.
+ 
+ <p>Clients and/or applications can use the provided <code>Progressable</code>
+ to explicitly report progress to the Hadoop framework. This is especially
+ important for operations which take significant amount of time since,
+ in-lieu of the reported progress, the framework has to assume that an error
+ has occurred and time-out the operation.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Progressable -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32 -->
+  <class name="PureJavaCrc32" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the same polynomial as the built-in native CRC32.
+
+ This is to avoid the JNI overhead for certain uses of Checksumming
+ where many small pieces of data are checksummed in succession.
+
+ The current version is ~10x to 1.8x as fast as Sun's native
+ java.util.zip.CRC32 in Java 1.6
+
+ @see java.util.zip.CRC32]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32 -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32C -->
+  <class name="PureJavaCrc32C" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32C"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the CRC32-C polynomial, the same polynomial used by iSCSI
+ and implemented on many Intel chipsets supporting SSE4.2.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32C -->
+  <!-- start class org.apache.hadoop.util.ReflectionUtils -->
+  <class name="ReflectionUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReflectionUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theObject" type="java.lang.Object"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Check and set 'configuration' if necessary.
+ 
+ @param theObject object for which to set configuration
+ @param conf Configuration]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create an object for the given class and initialize it from conf
+ 
+ @param theClass class of which an object is created
+ @param conf Configuration
+ @return a new object]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="argTypes" type="java.lang.Class[]"/>
+      <param name="values" type="java.lang.Object[]"/>
+      <doc>
+      <![CDATA[Create an object for the given class and initialize it from conf
+
+ @param theClass class of which an object is created
+ @param conf Configuration
+ @param argTypes the types of the arguments
+ @param values the values of the arguments
+ @return a new object]]>
+      </doc>
+    </method>
+    <method name="setContentionTracing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="printThreadInfo"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.PrintStream"/>
+      <param name="title" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Print all of the thread's information and stack traces.
+ 
+ @param stream the stream to
+ @param title a string title for the stack trace]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="T"/>
+      <doc>
+      <![CDATA[Return the correctly-typed {@link Class} of the given object.
+  
+ @param o object whose correctly-typed <code>Class</code> is to be obtained
+ @return the correctly typed <code>Class</code> of the given object.]]>
+      </doc>
+    </method>
+    <method name="copy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="src" type="T"/>
+      <param name="dst" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param src the object to copy from
+ @param dst the object to copy into, which is destroyed
+ @return dst param (the copy)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cloneWritableInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDeclaredFieldsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared fields of a class including fields declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredMethodsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared methods of a class including methods declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[General reflection utils]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ReflectionUtils -->
+  <!-- start class org.apache.hadoop.util.Shell -->
+  <class name="Shell" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Shell"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with no minimum interval between runs; stderr is
+ not merged with stdout.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with a minimum interval between executions; stderr is
+ not merged with stdout.
+ @param interval interval in milliseconds between command executions.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a shell instance which can be re-executed when the {@link #run()}
+ method is invoked with a given elapsed time between calls.
+
+ @param interval the minimum duration in milliseconds to wait before
+        re-executing the command. If set to 0, there is no minimum.
+ @param redirectErrorStream should the error stream be merged with
+        the normal output stream?]]>
+      </doc>
+    </constructor>
+    <method name="isJava7OrAbove" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="This call isn't needed any more: please remove uses of it.">
+      <doc>
+      <![CDATA[query to see if system is Java 7 or later.
+ Now that Hadoop requires Java 7 or later, this always returns true.
+ @deprecated This call isn't needed any more: please remove uses of it.
+ @return true, always.]]>
+      </doc>
+    </method>
+    <method name="isJavaVersionAtLeast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Query to see if major version of Java specification of the system
+ is equal or greater than the parameter.
+
+ @param version 8, 9, 10 etc.
+ @return comparison with system property, always true for 8]]>
+      </doc>
+    </method>
+    <method name="checkWindowsCommandLineLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if a given command (String[]) fits in the Windows maximum command
+ line length Note that the input is expected to already include space
+ delimiters, no extra count will be added for delimiters.
+
+ @param commands command parts, including any space delimiters]]>
+      </doc>
+    </method>
+    <method name="getGroupsCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's groups list.]]>
+      </doc>
+    </method>
+    <method name="getGroupsForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's groups list.
+ If the OS is not WINDOWS, the command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.]]>
+      </doc>
+    </method>
+    <method name="getGroupsIDForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's group id list.
+ The command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.
+ This command does not support Windows and will only return group names.]]>
+      </doc>
+    </method>
+    <method name="getUsersForNetgroupCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="netgroup" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given netgroup's user list.]]>
+      </doc>
+    </method>
+    <method name="getGetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a command to get permission information.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Return a command to set permission.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set permission for specific file.
+
+ @param perm String permission to set
+ @param recursive boolean true to apply to all sub-directories recursively
+ @param file String file to set
+ @return String[] containing command and arguments]]>
+      </doc>
+    </method>
+    <method name="getSetOwnerCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set owner.]]>
+      </doc>
+    </method>
+    <method name="getSymlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to create symbolic links.]]>
+      </doc>
+    </method>
+    <method name="getReadlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to read the target of the a symbolic link.]]>
+      </doc>
+    </method>
+    <method name="getCheckProcessIsAliveCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command for determining if process with specified pid is alive.
+ @param pid process ID
+ @return a <code>kill -0</code> command or equivalent]]>
+      </doc>
+    </method>
+    <method name="getSignalKillCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to send a signal to a given pid.]]>
+      </doc>
+    </method>
+    <method name="getEnvironmentVariableRegex" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a regular expression string that match environment variables.]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.io.File"/>
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a File referencing a script with the given basename, inside the
+ given parent directory.  The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param parent File parent directory
+ @param basename String script file basename
+ @return File referencing the script in the directory]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a script file name with the given basename.
+
+ The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param basename String script file basename
+ @return String script file name]]>
+      </doc>
+    </method>
+    <method name="getRunScriptCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="script" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns a command to run the given script.  The script interpreter is
+ inferred by platform: cmd on Windows or bash otherwise.
+
+ @param script File script to run
+ @return String[] command to run the script]]>
+      </doc>
+    </method>
+    <method name="getHadoopHome" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the Hadoop home directory. Raises an exception if not found
+ @return the home dir
+ @throws IOException if the home directory cannot be located.]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBin" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBinPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist
+ @throws IOException on path canonicalization failures]]>
+      </doc>
+    </method>
+    <method name="hasWinutilsPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate to indicate whether or not the path to winutils is known.
+
+ If true, then {@link #WINUTILS} is non-null, and both
+ {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()}
+ will successfully return this value. Always false on non-windows systems.
+ @return true if there is a valid path to the binary]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Locate the winutils binary, or fail with a meaningful
+ exception and stack trace as an RTE.
+ This method is for use in methods which don't explicitly throw
+ an <code>IOException</code>.
+ @return the path to {@link #WINUTILS_EXE}
+ @throws RuntimeException if the path is not resolvable]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Get a file reference to winutils.
+ Always raises an exception if there isn't one
+ @return the file instance referring to the winutils bin.
+ @throws FileNotFoundException on any failure to locate that file.]]>
+      </doc>
+    </method>
+    <method name="checkIsBashSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+    </method>
+    <method name="setEnvironment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the environment for the command.
+ @param env Mapping of environment variables]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Set the working directory.
+ @param dir The directory where the command will be executed]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check to see if a command needs to be executed and execute if needed.]]>
+      </doc>
+    </method>
+    <method name="getExecString" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return an array containing the command name and its parameters.]]>
+      </doc>
+    </method>
+    <method name="parseExecResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="lines" type="java.io.BufferedReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the execution result]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an environment variable.
+ @param env the environment var
+ @return the value or null if it was unset.]]>
+      </doc>
+    </method>
+    <method name="getProcess" return="java.lang.Process"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the current sub-process executing the given command.
+ @return process executing the command]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the exit code.
+ @return the exit code of the process]]>
+      </doc>
+    </method>
+    <method name="getWaitingThread" return="java.lang.Thread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the thread that is waiting on this instance of <code>Shell</code>.
+ @return the thread that ran runCommand() that spawned this shell
+ or null if no thread is waiting for this shell to complete]]>
+      </doc>
+    </method>
+    <method name="isTimedOut" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[To check if the passed script to shell command executor timed out or
+ not.
+
+ @return if the script timed out.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param cmd shell command to execute.
+ @return the output of the executed command.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <param name="timeout" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @param timeout time in milliseconds after which script should be marked timeout
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="destroyAllShellProcesses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to destroy all running <code>Shell</code> processes.
+ Iterates through a map of all currently running <code>Shell</code>
+ processes and destroys them one by one. This method is thread safe]]>
+      </doc>
+    </method>
+    <method name="getAllShells" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to return a Set of all <code>Shell</code> objects.]]>
+      </doc>
+    </method>
+    <method name="getMemlockLimit" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ulimit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Static method to return the memory lock limit for datanode.
+ @param ulimit max value at which memory locked should be capped.
+ @return long value specifying the memory lock limit.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSPROP_HADOOP_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[System property for the Hadoop home directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ENV_HADOOP_HOME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable for Hadoop's home dir: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum command line length in Windows
+ KB830473 documents this as 8191]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGHT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use the correctly spelled constant.">
+      <doc>
+      <![CDATA[mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
+ @deprecated use the correctly spelled constant.]]>
+      </doc>
+    </field>
+    <field name="USER_NAME_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's name: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WindowsProcessLaunchLock" type="java.lang.Object"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Windows <code>CreateProcess</code> synchronization object.]]>
+      </doc>
+    </field>
+    <field name="osType" type="org.apache.hadoop.util.Shell.OSType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the operating system, as determined from parsing
+ the <code>os.name</code> property.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SOLARIS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FREEBSD" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OTHER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PPC_64" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENV_NAME_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Regular expression for environment variables: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_PERMISSION_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set permission: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_OWNER_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set owner: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_GROUP_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set the change user's groups list: {@value}.]]>
+      </doc>
+    </field>
+    <field name="LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to create a link: {@value}.]]>
+      </doc>
+    </field>
+    <field name="READ_LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get a link target: {@value}.]]>
+      </doc>
+    </field>
+    <field name="timeOutInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time after which the executing script would be timedout.]]>
+      </doc>
+    </field>
+    <field name="inheritParentEnv" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if the parent env vars should be inherited or not]]>
+      </doc>
+    </field>
+    <field name="WINUTILS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}">
+      <doc>
+      <![CDATA[Location of winutils as a string; null if not found.
+ <p>
+ <i>Important: caller must check for this value being null</i>.
+ The lack of such checks has led to many support issues being raised.
+ <p>
+ @deprecated use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]>
+      </doc>
+    </field>
+    <field name="isSetsidAvailable" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag which is true if setsid exists.]]>
+      </doc>
+    </field>
+    <field name="TOKEN_SEPARATOR_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Token separator regex used to parse Shell tool outputs.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for running a Shell command.
+
+ <code>Shell</code> can be used to run shell commands like <code>du</code> or
+ <code>df</code>. It also offers facilities to gate commands by
+ time-intervals.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.Shell -->
+  <!-- start class org.apache.hadoop.util.ShutdownHookManager -->
+  <class name="ShutdownHookManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="get" return="org.apache.hadoop.util.ShutdownHookManager"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return <code>ShutdownHookManager</code> singleton.
+
+ @return <code>ShutdownHookManager</code> singleton.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority, the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority and timeout the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order. The shutdown hook will be terminated if it
+ has not been finished in the specified period of time.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook
+ @param timeout timeout of the shutdownHook
+ @param unit unit of the timeout <code>TimeUnit</code>]]>
+      </doc>
+    </method>
+    <method name="removeShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Removes a shutdownHook.
+
+ @param shutdownHook shutdownHook to remove.
+ @return TRUE if the shutdownHook was registered and removed,
+ FALSE otherwise.]]>
+      </doc>
+    </method>
+    <method name="hasShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Indicates if a shutdownHook is registered or not.
+
+ @param shutdownHook shutdownHook to check if registered.
+ @return TRUE/FALSE depending if the shutdownHook is is registered.]]>
+      </doc>
+    </method>
+    <method name="isShutdownInProgress" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if shutdown is in progress or not.
+ 
+ @return TRUE if the shutdown is in progress, otherwise FALSE.]]>
+      </doc>
+    </method>
+    <method name="clearShutdownHooks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[clear all registered shutdownHooks.]]>
+      </doc>
+    </method>
+    <field name="TIMEOUT_MINIMUM" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum shutdown timeout: {@value} second(s).]]>
+      </doc>
+    </field>
+    <field name="TIME_UNIT_DEFAULT" type="java.util.concurrent.TimeUnit"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default time unit used: seconds.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The <code>ShutdownHookManager</code> enables running shutdownHook
+ in a deterministic order, higher priority first.
+ <p/>
+ The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
+ This class registers a single JVM shutdownHook and run all the
+ shutdownHooks registered to it (to this class) in order based on their
+ priority.
+
+ Unless a hook was registered with a shutdown explicitly set through
+ {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ the shutdown time allocated to it is set by the configuration option
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in
+ {@code core-site.xml}, with a default value of
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ seconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ShutdownHookManager -->
+  <!-- start class org.apache.hadoop.util.StringInterner -->
+  <class name="StringInterner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StringInterner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="strongIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains strong reference to the instance, 
+ thus preventing it from being garbage-collected. 
+ 
+ @param sample string instance to be interned
+ @return strong reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="weakIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains weak reference to the instance, 
+ and so does not prevent it from being garbage-collected.
+ 
+ @param sample string instance to be interned
+ @return weak reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="internStringsInArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="strings" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Interns all the strings in the given array in place,
+ returning the same array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides string interning utility methods. For weak interning,
+ we use the standard String.intern() call, that performs very well
+ (no problems with PermGen overflowing, etc.) starting from JDK 7.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.StringInterner -->
+  <!-- start class org.apache.hadoop.util.SysInfo -->
+  <class name="SysInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SysInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.util.SysInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return default OS instance.
+ @throws UnsupportedOperationException If cannot determine OS.
+ @return Default instance for the detected OS.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the virtual memory present in the system.
+
+ @return virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the physical memory present in the system.
+
+ @return physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailableVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available virtual memory present
+ in the system.
+
+ @return available virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailablePhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available physical memory present
+ in the system.
+
+ @return available physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getNumProcessors" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total number of logical processors present on the system.
+
+ @return number of logical processors]]>
+      </doc>
+    </method>
+    <method name="getNumCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain total number of physical cores present on the system.
+
+ @return number of physical cores]]>
+      </doc>
+    </method>
+    <method name="getCpuFrequency" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU frequency of on the system.
+
+ @return CPU frequency in kHz]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the cumulative CPU time since the system is on.
+
+ @return cumulative CPU time in milliseconds]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU usage % of the machine. Return -1 if it is unavailable
+
+ @return CPU usage as a percentage (from 0 to 100) of available cycles.]]>
+      </doc>
+    </method>
+    <method name="getNumVCoresUsed" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the number of VCores used. Return -1 if it is unavailable
+
+ @return Number of VCores used a percentage (from 0 to #VCores).]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read over the network.
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to the network.
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read from disks.
+
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to disks.
+
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Plugin to calculate resource information on the system.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.SysInfo -->
+  <!-- start interface org.apache.hadoop.util.Tool -->
+  <interface name="Tool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="run" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Execute the command with the given arguments.
+ 
+ @param args command specific arguments.
+ @return exit code.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A tool interface that supports handling of generic command-line options.
+ 
+ <p><code>Tool</code>, is the standard for any Map-Reduce tool/application. 
+ The tool/application should delegate the handling of 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ standard command-line options</a> to {@link ToolRunner#run(Tool, String[])} 
+ and only handle its custom arguments.</p>
+ 
+ <p>Here is how a typical <code>Tool</code> is implemented:</p>
+ <p><blockquote><pre>
+     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // <code>Configuration</code> processed by <code>ToolRunner</code>
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed <code>conf</code>
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let <code>ToolRunner</code> handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+ </pre></blockquote></p>
+ 
+ @see GenericOptionsParser
+ @see ToolRunner]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Tool -->
+  <!-- start class org.apache.hadoop.util.ToolRunner -->
+  <class name="ToolRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ToolRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the given <code>Tool</code> by {@link Tool#run(String[])}, after 
+ parsing with the given generic arguments. Uses the given 
+ <code>Configuration</code>, or builds one if null.
+ 
+ Sets the <code>Tool</code>'s configuration with the possibly modified 
+ version of the <code>conf</code>.  
+ 
+ @param conf <code>Configuration</code> for the <code>Tool</code>.
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the <code>Tool</code> with its <code>Configuration</code>.
+ 
+ Equivalent to <code>run(tool.getConf(), tool, args)</code>.
+ 
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="printGenericCommandUsage"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.PrintStream"/>
+      <doc>
+      <![CDATA[Prints generic command-line argurments and usage information.
+ 
+  @param out stream to write usage information to.]]>
+      </doc>
+    </method>
+    <method name="confirmPrompt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prompt" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print out a prompt to the user, and return true if the user
+ responds with "y" or "yes". (case insensitive)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A utility to help run {@link Tool}s.
+ 
+ <p><code>ToolRunner</code> can be used to run classes implementing 
+ <code>Tool</code> interface. It works in conjunction with 
+ {@link GenericOptionsParser} to parse the 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ generic hadoop command line arguments</a> and modifies the 
+ <code>Configuration</code> of the <code>Tool</code>. The 
+ application-specific options are passed along without being modified.
+ </p>
+ 
+ @see Tool
+ @see GenericOptionsParser]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ToolRunner -->
+  <!-- start class org.apache.hadoop.util.VersionInfo -->
+  <class name="VersionInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionInfo" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="_getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Hadoop version.
+ @return the Hadoop version string, eg. "0.6.3-dev"]]>
+      </doc>
+    </method>
+    <method name="getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Git commit hash of the repository when compiled.
+ @return the commit hash, eg. "18f64065d5db6208daf50b02c1b5ed4ee3ce547a"]]>
+      </doc>
+    </method>
+    <method name="getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the branch on which this originated.
+ @return The branch name, e.g. "trunk" or "branches/branch-0.20"]]>
+      </doc>
+    </method>
+    <method name="getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The date that Hadoop was compiled.
+ @return the compilation date in unix date format]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user that compiled Hadoop.
+ @return the username of the user]]>
+      </doc>
+    </method>
+    <method name="getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL for the Hadoop repository.
+ @return the URL of the Hadoop repository]]>
+      </doc>
+    </method>
+    <method name="getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the checksum of the source files from which Hadoop was built.
+ @return the checksum of the source files]]>
+      </doc>
+    </method>
+    <method name="getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the buildVersion which includes version,
+ revision, user and date.
+ @return the buildVersion]]>
+      </doc>
+    </method>
+    <method name="getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the protoc version used for the build.
+ @return the protoc version]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <doc>
+    <![CDATA[This class returns build information about Hadoop components.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.VersionInfo -->
+  <doc>
+  <![CDATA[Common utilities.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.util.bloom">
+  <!-- start class org.apache.hadoop.util.bloom.BloomFilter -->
+  <class name="BloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="BloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVectorSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return size of the the bloomfilter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>Bloom filter</i>, as defined by Bloom in 1970.
+ <p>
+ The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by 
+ the networking research community in the past decade thanks to the bandwidth efficiencies that it
+ offers for the transmission of set membership information between networked hosts.  A sender encodes 
+ the information into a bit vector, the Bloom filter, that is more compact than a conventional 
+ representation. Computation and space costs for construction are linear in the number of elements.  
+ The receiver uses the filter to test whether various elements are members of the set. Though the 
+ filter will occasionally return a false positive, it will never return a false negative. When creating 
+ the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. 
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=362692&dl=ACM&coll=portal">Space/Time Trade-Offs in Hash Coding with Allowable Errors</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.BloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <class name="CountingBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CountingBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="CountingBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Removes a specified key from <i>this</i> counting Bloom filter.
+ <p>
+ <b>Invariant</b>: nothing happens if the specified key does not belong to <i>this</i> counter Bloom filter.
+ @param key The key to remove.]]>
+      </doc>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="approximateCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[This method calculates an approximate count of the key, i.e. how many
+ times the key was added to the filter. This allows the filter to be
+ used as an approximate <code>key -&gt; count</code> map.
+ <p>NOTE: due to the bucket size of this filter, inserting the same
+ key more than 15 times will cause an overflow at all filter positions
+ associated with this key, and it will significantly increase the error
+ rate for this and other keys. For this reason the filter can only be
+ used to store small count values <code>0 &lt;= N &lt;&lt; 15</code>.
+ @param key key to be tested
+ @return 0 if the key is not present. Otherwise, a positive value v will
+ be returned such that <code>v == count</code> with probability equal to the
+ error rate of this filter, and <code>v &gt; count</code> otherwise.
+ Additionally, if the filter experienced an underflow as a result of
+ {@link #delete(Key)} operation, the return value may be lower than the
+ <code>count</code> with the probability of the false negative rate of such
+ filter.]]>
+      </doc>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>counting Bloom filter</i>, as defined by Fan et al. in a ToN
+ 2000 paper.
+ <p>
+ A counting Bloom filter is an improvement to standard a Bloom filter as it
+ allows dynamic additions and deletions of set membership information.  This 
+ is achieved through the use of a counting vector instead of a bit vector.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=343571.343572">Summary cache: a scalable wide-area web cache sharing protocol</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <class name="DynamicBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DynamicBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zero-args constructor for the serialization.]]>
+      </doc>
+    </constructor>
+    <constructor name="DynamicBloomFilter" type="int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds an empty Dynamic Bloom filter.
+ @param vectorSize The number of bits in the vector.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).
+ @param nr The threshold for the maximum number of keys to record in a
+ dynamic Bloom filter row.]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>dynamic Bloom filter</i>, as defined in the INFOCOM 2006 paper.
+ <p>
+ A dynamic Bloom filter (DBF) makes use of a <code>s * m</code> bit matrix but
+ each of the <code>s</code> rows is a standard Bloom filter. The creation 
+ process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
+ bit matrix, i.e., it is composed of a single standard Bloom filter.
+ It assumes that <code>n<sub>r</sub></code> elements are recorded in the 
+ initial bit vector, where <code>n<sub>r</sub> <= n</code> (<code>n</code> is
+ the cardinality of the set <code>A</code> to record in the filter).  
+ <p>
+ As the size of <code>A</code> grows during the execution of the application,
+ several keys must be inserted in the DBF.  When inserting a key into the DBF,
+ one must first get an active Bloom filter in the matrix.  A Bloom filter is
+ active when the number of recorded keys, <code>n<sub>r</sub></code>, is 
+ strictly less than the current cardinality of <code>A</code>, <code>n</code>.
+ If an active Bloom filter is found, the key is inserted and 
+ <code>n<sub>r</sub></code> is incremented by one. On the other hand, if there
+ is no active Bloom filter, a new one is created (i.e., a new row is added to
+ the matrix) according to the current size of <code>A</code> and the element
+ is added in this new Bloom filter and the <code>n<sub>r</sub></code> value of
+ this new Bloom filter is set to one.  A given key is said to belong to the
+ DBF if the <code>k</code> positions are set to one in one of the matrix rows.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ 
+ @see <a href="http://www.cse.fau.edu/~jie/research/publications/Publication_files/infocom2006.pdf">Theory and Network Applications of Dynamic Bloom Filters</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.HashFunction -->
+  <class name="HashFunction" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashFunction" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds a hash function that must obey to a given maximum number of returned values and a highest value.
+ @param maxValue The maximum highest returned value.
+ @param nbHash The number of resulting hashed values.
+ @param hashType type of the hashing function (see {@link Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears <i>this</i> hash function. A NOOP]]>
+      </doc>
+    </method>
+    <method name="hash" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Hashes a specified key into several integers.
+ @param k The specified key.
+ @return The array of hashed values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements a hash object that returns a certain number of hashed values.
+ 
+ @see Key The general behavior of a key being stored in a filter
+ @see Filter The general behavior of a filter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.HashFunction -->
+  <!-- start interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <interface name="RemoveScheme"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="RANDOM" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Random selection.
+ <p>
+ The idea is to randomly select a bit to reset.]]>
+      </doc>
+    </field>
+    <field name="MINIMUM_FN" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MinimumFN Selection.
+ <p>
+ The idea is to select the bit to reset that will generate the minimum
+ number of false negative.]]>
+      </doc>
+    </field>
+    <field name="MAXIMUM_FP" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MaximumFP Selection.
+ <p>
+ The idea is to select the bit to reset that will remove the maximum number
+ of false positive.]]>
+      </doc>
+    </field>
+    <field name="RATIO" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Ratio Selection.
+ <p>
+ The idea is to select the bit to reset that will, at the same time, remove
+ the maximum number of false positve while minimizing the amount of false
+ negative generated.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Defines the different remove scheme for retouched Bloom filters.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <!-- start class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+  <class name="RetouchedBloomFilter" extends="org.apache.hadoop.util.bloom.BloomFilter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.bloom.RemoveScheme"/>
+    <constructor name="RetouchedBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="RetouchedBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Adds a false positive information to <i>this</i> retouched Bloom filter.
+ <p>
+ <b>Invariant</b>: if the false positive is <code>null</code>, nothing happens.
+ @param key The false positive key to add.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="coll" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Adds a collection of false positive information to <i>this</i> retouched Bloom filter.
+ @param coll The collection of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="java.util.List"/>
+      <doc>
+      <![CDATA[Adds a list of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The list of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="org.apache.hadoop.util.bloom.Key[]"/>
+      <doc>
+      <![CDATA[Adds an array of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The array of false positive.]]>
+      </doc>
+    </method>
+    <method name="selectiveClearing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <param name="scheme" type="short"/>
+      <doc>
+      <![CDATA[Performs the selective clearing for a given key.
+ @param k The false positive key to remove from <i>this</i> retouched Bloom filter.
+ @param scheme The selective clearing scheme to apply.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>retouched Bloom filter</i>, as defined in the CoNEXT 2006 paper.
+ <p>
+ It allows the removal of selected false positives at the cost of introducing
+ random false negatives, and with the benefit of eliminating some random false
+ positives at the same time.
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ @see RemoveScheme The different selective clearing algorithms
+ 
+ @see <a href="http://www-rp.lip6.fr/site_npa/site_rp/_publications/740-rbf_cameraready.pdf">Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+</package>
+<package name="org.apache.hadoop.util.curator">
+</package>
+<package name="org.apache.hadoop.util.hash">
+</package>
+
+</api>
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.3.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.3.xml
new file mode 100644
index 0000000000000..448df9ddd686b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.3.xml
@@ -0,0 +1,39037 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 17:58:47 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop Common 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-common-project/hadoop-common/target/classes:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/wildfly/openssl/wildfly-openssl-java/1.0.7.Final/wildfly-openssl-java-1.0.7.Final.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-common-project/hadoop-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -apidir /build/source/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 3.3.3 -->
+<package name="org.apache.hadoop">
+  <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+  <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message detailed message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+  <!-- start interface org.apache.hadoop.conf.Configurable -->
+  <interface name="Configurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+ @param conf configuration to be used]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+ @return Configuration]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.conf.Configurable -->
+  <!-- start class org.apache.hadoop.conf.Configuration -->
+  <class name="Configuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Iterable"/>
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration where the behavior of reading from the default 
+ resources can be turned off.
+ 
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files. 
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration with the same settings cloned from another.
+ 
+ @param other the configuration from which to clone settings.]]>
+      </doc>
+    </constructor>
+    <method name="addDeprecations"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+      <doc>
+      <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless.  It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context.  If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas   The deprecations to add.]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+ 
+ @param key to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @param customMessage depcrication message
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key to be deprecated
+ @param newKey key that take up the values of deprecated key
+ @param customMessage deprecation message]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+      </doc>
+    </method>
+    <method name="isDeprecated" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[checks whether the given <code>key</code> is deprecated.
+ 
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and 
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setDeprecatedProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+      </doc>
+    </method>
+    <method name="reloadExistingConfigurations"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload existing configuration instances.]]>
+      </doc>
+    </method>
+    <method name="addDefaultResource"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a default resource. Resources are loaded in the order of the resources 
+ added.
+ @param name file name. File should be present in the classpath.]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemPropertiesDefault"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="setRestrictSystemProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param name resource to be added, the classpath is examined for a file 
+             with that name.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param url url of the resource to be added, the local filesystem is 
+            examined directly to find the resource, without referring to 
+            the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param file file-path of resource to be added, the local filesystem is
+             examined directly to find the resource, without referring to 
+             the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ WARNING: The contents of the InputStream will be cached, by this method. 
+ So use this sparingly because it does increase the memory consumption.
+ 
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next.  After it is read the stream will be
+ closed.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+      </doc>
+    </method>
+    <method name="reloadConfiguration"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added 
+ resources, and final parameters. This will make the resources to 
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned.
+
+ As a side effect get loads the properties from the sources if called for
+ the first time as a lazy init.
+ 
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="setAllowNullValueProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+      <doc>
+      <![CDATA[Set Configuration to allow keys without values during setup.  Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemProps"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="onlyKeyExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>null</code> if no such property exists. 
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>defaultValue</code> if no such property exists. 
+ See @{Configuration#getTrimmed} for more details.
+ 
+ @param name          the property name.
+ @param defaultValue  the property default value.
+ @return              the value of the <code>name</code> or defaultValue
+                      if it is not set.]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is 
+ deprecated, it returns the value of the first key which replaces 
+ the deprecated key and is not null.
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> property or 
+         its replacing property and null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="source" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from 
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+      </doc>
+    </method>
+    <method name="unset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unset a previously set property.
+ @param name the property name]]>
+      </doc>
+    </method>
+    <method name="setIfUnset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+ 
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property 
+         doesn't exist.]]>
+      </doc>
+    </method>
+    <method name="getInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="int"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+   
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInts" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+ 
+ If no such property exists, an empty array is returned.
+ 
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+         <code>int</code> values]]>
+      </doc>
+    </method>
+    <method name="setInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+ 
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getLongBytes" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+ 
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="float"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getBoolean" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="boolean"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.  
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setBoolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+ 
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="setBooleanIfUnset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="setEnum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="T"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
+ @param name property name
+ @param value new value
+ @param <T> enumeration type]]>
+      </doc>
+    </method>
+    <method name="getEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="T"/>
+      <doc>
+      <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @param <T> enumeration type
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided
+ @return enumeration type]]>
+      </doc>
+    </method>
+    <method name="setTimeDuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
+ provided, the default unit is applied.
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param defaultUnit Default time unit if no valid suffix is provided.
+ @param returnUnit The unit used for the returned value.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDurationHelper" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="vStr" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param vStr The string value with time unit suffix to be converted.
+ @param unit Unit to convert the stored property, if it exists.]]>
+      </doc>
+    </method>
+    <method name="getTimeDurations" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets the Storage Size from the config, or returns the defaultValue. The
+ unit of return value is specified in target unit.
+
+ @param name - Key Name
+ @param defaultValue - Default Value -- e.g. 100MB
+ @param targetUnit - The units that we want result to be in.
+ @return double -- formatted in target Units]]>
+      </doc>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets storage size from a config file.
+
+ @param name - Key to read.
+ @param defaultValue - The default value to return in case the key is
+ not present.
+ @param targetUnit - The Storage unit that should be used
+ for the return value.
+ @return - double value in the Storage Unit specified.]]>
+      </doc>
+    </method>
+    <method name="setStorageSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <param name="unit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Sets Storage Size for the specified key.
+
+ @param name - Key to set.
+ @param value - The numeric value to set.
+ @param unit - Storage Unit to be used.]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="pattern" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+      </doc>
+    </method>
+    <method name="getPropertySources" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Gets information about why a property was set.  Typically this is the 
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programmatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise, 
+ returns a list of the sources of the resource.  The older sources are
+ the first ones in the list.  So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+      </doc>
+    </method>
+    <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+      </doc>
+    </method>
+    <method name="getStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s.  
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+ 
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then <code>null</code> is returned.
+ 
+ @param name property name.
+ @return property value as an array of <code>String</code>s, 
+         or <code>null</code>.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+ 
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s, 
+         or empty array.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="setStrings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the array of string values for the <code>name</code> property as 
+ as comma delimited values.  
+ 
+ @param name property name.
+ @param values The values]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password
+ @throws IOException when error in fetching password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromCredentialProviders" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException when error in fetching password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromConfig" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="setSocketAddr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address.  This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="getClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Load a class by name.
+ 
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+      </doc>
+    </method>
+    <method name="getClassByNameOrNull" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+ 
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+      </doc>
+    </method>
+    <method name="getClasses" return="java.lang.Class[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class[]"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the conf key name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+   
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ An exception is thrown if the returned class does not implement the named
+ interface. 
+ 
+ @param name the conf key name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+ 
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+ 
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+        <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+      </doc>
+    </method>
+    <method name="setClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the name of a 
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+ 
+ An exception is thrown if <code>theClass</code> does not implement the 
+ interface <code>xface</code>. 
+ 
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+      </doc>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link URL} for the named resource.
+ 
+ @param name resource name.
+ @return the url for the named resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsReader" return="java.io.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getFinalParameters" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+      </doc>
+    </method>
+    <method name="getProps" return="java.util.Properties"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all keys from the configuration.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code> 
+ key-value pairs in the configuration.
+ 
+ @return an iterator over the entries.]]>
+      </doc>
+    </method>
+    <method name="getPropsWithPrefix" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration and includes all properties that
+ start with the specified configuration prefix.  Property names in the
+ mapping are trimmed to remove the configuration prefix.
+
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties with prefix stripped]]>
+      </doc>
+    </method>
+    <method name="addTags"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prop" type="java.util.Properties"/>
+      <doc>
+      <![CDATA[Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
+ @param prop]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+ 
+ @param out the output stream to write to.]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the
+ given {@link Writer}.
+ <ul>
+ <li>
+ When property name is not empty and the property exists in the
+ configuration, this method writes the property and its attributes
+ to the {@link Writer}.
+ </li>
+
+ <li>
+ When property name is null or empty, this method writes all the
+ configuration properties and their attributes to the {@link Writer}.
+ </li>
+
+ <li>
+ When property name is not empty but the property doesn't exist in
+ the configuration, this method throws an {@link IllegalArgumentException}.
+ </li>
+ </ul>
+ @param out the writer to write to.]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes properties and their attributes (final and resource)
+  to the given {@link Writer}.
+  <ul>
+  <li>
+  When propertyName is not empty, and the property exists
+  in the configuration, the format of the output would be,
+  <pre>
+  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is null or empty, it behaves same as
+  {@link #dumpConfiguration(Configuration, Writer)}, the
+  output would be,
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is not empty, and the property is not
+  found in the configuration, this method will throw an
+  {@link IllegalArgumentException}.
+  </li>
+  </ul>
+  <p>
+ @param config the configuration
+ @param propertyName property name
+ @param out the Writer to write to
+ @throws IOException
+ @throws IllegalArgumentException when property name is not
+   empty and the property is not found in configuration]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes out all properties and their attributes (final and resource) to
+  the given {@link Writer}, the format of the output would be,
+
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+
+  It does not output the properties of the configuration object which
+  is loaded from an input stream.
+  <p>
+
+ @param config the configuration
+ @param out the Writer to write to
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClassLoader" return="java.lang.ClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+      </doc>
+    </method>
+    <method name="setClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classLoader" type="java.lang.ClassLoader"/>
+      <doc>
+      <![CDATA[Set the class loader that will be used to load the various objects.
+ 
+ @param classLoader the new class loader.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQuietMode"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quietmode" type="boolean"/>
+      <doc>
+      <![CDATA[Set the quietness-mode. 
+ 
+ In the quiet-mode, error and informational messages might not be logged.
+ 
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+              to turn it off.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[For debugging.  List non-default properties to the terminal and exit.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getValByRegex" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="regex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[get keys matching the the regex 
+ @param regex
+ @return {@literal Map<String,String>} with matching keys]]>
+      </doc>
+    </method>
+    <method name="dumpDeprecatedKeys"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasWarnedDeprecation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTag" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get all properties belonging to tag.
+ @param tag tag
+ @return Properties with matching tag]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTags" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get all properties belonging to list of input tags. Calls
+ getAllPropertiesByTag internally.
+ @param tagList list of input tags
+ @return Properties with matching tags]]>
+      </doc>
+    </method>
+    <method name="isPropertyTag" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagStr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get Property tag Enum corresponding to given source.
+
+ @param tagStr String representation of Enum
+ @return true if tagStr is a valid tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides access to configuration parameters.
+
+ <h3 id="Resources">Resources</h3>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a 
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
+ then the classpath is examined for a file with that name.  If named by a 
+ <code>Path</code>, then the local filesystem is examined directly, without 
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two 
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+ 
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>. 
+ Once a resource declares a value final, no subsequently-loaded 
+ resource can alter that value.  
+ For example, one might define a final parameter with:
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
+    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
+    <b>&lt;final&gt;true&lt;/final&gt;</b>
+  &lt;/property&gt;</code></pre>
+
+ Administrators typically define parameters as final in 
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Environment variables in {@link System#getenv()} if a name starts with
+ "env.", or</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions: 
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;basedir&lt;/name&gt;
+    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
+  &lt;/property&gt;
+  
+  &lt;property&gt;
+    &lt;name&gt;tempdir&lt;/name&gt;
+    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;otherdir&lt;/name&gt;
+    &lt;value&gt;${<i>env.BASE_DIR</i>}/other&lt;/value&gt;
+  &lt;/property&gt;
+  </code></pre>
+
+ <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ <p>When <tt>conf.get("otherdir")</tt> is called, then <tt>${<i>env.BASE_DIR</i>}</tt>
+ will be resolved to the value of the <tt>${<i>BASE_DIR</i>}</tt> environment variable.
+ It supports <tt>${<i>env.NAME:-default</i>}</tt> and <tt>${<i>env.NAME-default</i>}</tt> notations.
+ The former is resolved to "default" if <tt>${<i>NAME</i>}</tt> environment variable is undefined
+ or its value is empty.
+ The latter behaves the same way only if <tt>${<i>NAME</i>}</tt> is undefined.
+ <p>By default, warnings will be given to any deprecated configuration 
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.
+
+ <h4 id="Tags">Tags</h4>
+
+ <p>Optionally we can tag related properties together by using tag
+ attributes. System tags are defined by hadoop.tags.system property. Users
+ can define there own custom tags in  hadoop.tags.custom property.
+
+ <p>For example, we can tag existing property as:
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;dfs.replication&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,REQUIRED&lt;/tag&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;dfs.data.transfer.protection&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,SECURITY&lt;/tag&gt;
+  &lt;/property&gt;
+ </code></pre>
+ <p> Properties marked with tags can be retrieved with <tt>conf
+ .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
+ (Arrays.asList("YARN","SECURITY"))</tt>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configuration -->
+  <!-- start class org.apache.hadoop.conf.Configured -->
+  <class name="Configured" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="Configured"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configured -->
+  <doc>
+  <![CDATA[Configuration of system parameters.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.crypto">
+</package>
+<package name="org.apache.hadoop.crypto.key">
+  <!-- start class org.apache.hadoop.crypto.key.KeyProvider -->
+  <class name="KeyProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="KeyProvider" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ 
+ @param conf configuration for the provider]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the provider configuration.
+ 
+ @return the provider configuration]]>
+      </doc>
+    </method>
+    <method name="options" return="org.apache.hadoop.crypto.key.KeyProvider.Options"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[A helper function to create an options object.
+ @param conf the configuration to use
+ @return a new options object]]>
+      </doc>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide access to
+ keying material rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getKeyVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for a specific version of the key. This method is used
+ when decrypting data.
+ @param versionName the name of a specific version of the key
+ @return the key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeys" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key names for all keys.
+ @return the list of key names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeysMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get key metadata in bulk.
+ @param names the names of the keys to get
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeyVersions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for all versions of a specific key name.
+ @return the list of key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current version of the key, which should be used for encrypting new
+ data.
+ @param name the base name of the key
+ @return the version name of the current version of the key or null if the
+    key version doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get metadata about the key.
+ @param name the basename of the key
+ @return the key's metadata or null if the key doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key. The given key must not already exist.
+ @param name the base name of the key
+ @param material the key material for the first version of the key.
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <param name="algorithm" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <doc>
+      <![CDATA[Generates a key material.
+
+ @param size length of the key.
+ @param algorithm algorithm to use for generating the key.
+ @return the generated key.
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key generating the material for it.
+ The given key must not already exist.
+ <p>
+ This implementation generates the key material and calls the
+ {@link #createKey(String, byte[], Options)} method.
+
+ @param name the base name of the key
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="deleteKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given key.
+ @param name the name of the key to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key.
+ @param name the basename of the key
+ @param material the new key material
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to close any resources
+ that require closing]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key generating the material for it.
+ <p>
+ This implementation generates the key material and calls the
+ {@link #rollNewVersion(String, byte[])} method.
+
+ @param name the basename of the key
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="invalidateCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to invalidate the caches. This could be
+ used after rollNewVersion to provide a strong guarantee to return the new
+ version of the given key.
+
+ @param name the basename of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the keys are written to persistent store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getBaseName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Split the versionName in to a base name. Converts "/aaa/bbb/3" to
+ "/aaa/bbb".
+ @param versionName the version name to split
+ @return the base name of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="buildVersionName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Build a version string from a basename and version number. Converts
+ "/aaa/bbb" and 3 to "/aaa/bbb@3".
+ @param name the basename of the key
+ @param version the version of the key
+ @return the versionName of the key.]]>
+      </doc>
+    </method>
+    <method name="findProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerList" type="java.util.List"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the provider with the given key.
+ @param providerList the list of providers
+ @param keyName the key name we are looking for
+ @return the KeyProvider that has the key]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_CIPHER_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CIPHER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIALFILTER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIAL_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of secret key material for Hadoop applications. Provides an
+ abstraction to separate key storage from users of encryption. It
+ is intended to support getting or storing keys in a variety of ways,
+ including third party bindings.
+ <p>
+ <code>KeyProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProvider -->
+  <!-- start class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+  <class name="KeyProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a KeyProvider based on a provided URI.
+
+ @param uri key provider URI
+ @param conf configuration to initialize the key provider
+ @return the key provider for the specified URI, or <code>NULL</code> if
+         a provider for the specified URI scheme could not be found.
+ @throws IOException thrown if the provider failed to initialize.]]>
+      </doc>
+    </method>
+    <field name="KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of KeyProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ KeyProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+</package>
+<package name="org.apache.hadoop.crypto.key.kms">
+</package>
+<package name="org.apache.hadoop.crypto.random">
+</package>
+<package name="org.apache.hadoop.fs">
+  <!-- start interface org.apache.hadoop.fs.Abortable -->
+  <interface name="Abortable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="abort" return="org.apache.hadoop.fs.Abortable.AbortableResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Abort the active operation without the output becoming visible.
+
+ This is to provide ability to cancel the write on stream; once
+ a stream is aborted, the write MUST NOT become visible.
+
+ @throws UnsupportedOperationException if the operation is not supported.
+ @return the result.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abort data being written to  a stream, so that close() does
+  not write the data. It is implemented by output streams in
+  some object stores, and passed through {@link FSDataOutputStream}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Abortable -->
+  <!-- start class org.apache.hadoop.fs.AbstractFileSystem -->
+  <class name="AbstractFileSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="AbstractFileSystem" type="java.net.URI, java.lang.String, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[Constructor to be called by subclasses.
+ 
+ @param uri for this file system.
+ @param supportedScheme the scheme supported by the implementor
+ @param authorityNeeded if true then theURI must have authority, if false
+          then the URI must have null authority.
+
+ @throws URISyntaxException <code>uri</code> has syntax error]]>
+      </doc>
+    </constructor>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isValidName" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns true if the specified string is considered valid in the path part
+ of a URI by this file system.  The default implementation enforces the rules
+ of HDFS, but subclasses may override this method to implement specific
+ validation rules for specific file systems.
+ 
+ @param src String source filename to check, path part of the URI
+ @return boolean true if the specified string is considered valid]]>
+      </doc>
+    </method>
+    <method name="createFileSystem" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a file system instance for the specified uri using the conf. The
+ conf is used to find the class name that implements the file system. The
+ conf is also passed to the file system for its configuration.
+
+ @param uri URI of the file system
+ @param conf Configuration for the file system
+ 
+ @return Returns the file system for the given URI
+
+ @throws UnsupportedFileSystemException file system for <code>uri</code> is
+           not found]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ 
+ @param uri
+          used as key to lookup STATISTICS_TABLE. Only scheme and authority
+          part of the uri are used.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[The main factory method for creating a file system. Get a file system for
+ the URI's scheme and authority. The scheme of the <code>uri</code>
+ determines a configuration property name,
+ <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
+ AbstractFileSystem class.
+ 
+ The entire URI and conf is passed to the AbstractFileSystem factory method.
+ 
+ @param uri for the file system to be created.
+ @param conf which is passed to the file system impl.
+ 
+ @return file system for the given URI.
+ 
+ @throws UnsupportedFileSystemException if the file system for
+           <code>uri</code> is not supported.]]>
+      </doc>
+    </method>
+    <method name="checkScheme"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="supportedScheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check that the Uri's scheme matches
+ @param uri
+ @param supportedScheme]]>
+      </doc>
+    </method>
+    <method name="getUriDefaultPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default port of this file system.
+ 
+ @return default port of this file system's Uri scheme
+         A uri with a port of -1 =&gt; default port;]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.
+ 
+ @return the uri of this file system.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+ 
+ If the path is fully qualified URI, then its scheme and authority
+ matches that of this file system. Otherwise the path must be 
+ slash-relative name.
+ 
+ @throws InvalidPathException if the path is invalid]]>
+      </doc>
+    </method>
+    <method name="getUriPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the path-part of a pathname. Checks that URI matches this file system
+ and that the path-part is a valid name.
+ 
+ @param p path
+ 
+ @return path-part of the Path p]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified to this file system
+ @param path
+ @return the qualified path]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Some file systems like LocalFileSystem have an initial workingDir
+ that is used as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+ 
+ @return the initial workingDir if the file system has such a notion
+         otherwise return a null.]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ 
+ @return current user's home directory.]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ 
+ @return server default configuration values
+ 
+ @throws IOException an I/O error occurred
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param f path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path f resolving the path
+ through any internal symlinks or mount point
+ @param p path to be resolved
+ @return fully qualified path 
+ @throws FileNotFoundException
+ @throws AccessControlException
+ @throws IOException
+ @throws UnresolvedLinkException if symbolic link on path cannot be
+ resolved internally]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
+ that the Path f must be fully qualified and the permission is absolute
+ (i.e. umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="createInternal" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
+ have been declared explicitly.]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
+ f must be fully qualified and the permission is absolute (i.e. 
+ umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#delete(Path, boolean)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path, int)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#truncate(Path, long)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setReplication(Path, short)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system and NO OVERWRITE is performed.
+ 
+ File systems that do not have a built in overwrite need implement only this
+ method and can take advantage of the default impl of the other
+ {@link #renameInternal(Path, Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the file system supports symlinks, false otherwise.
+ @return true if filesystem supports symlinks]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of  
+ {@link FileContext#createSymlink(Path, Path, boolean)};]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partially resolves the path. This is used during symlink resolution in
+ {@link FSLinkResolver}, and differs from the similarly named method
+ {@link FileContext#getLinkTarget(Path)}.
+ @throws IOException subclass implementations may throw IOException]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setPermission(Path, FsPermission)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setOwner(Path, String, String)} except that Path f must
+ be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setTimes(Path, long, long)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileChecksum(Path)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileStatus(Path)} 
+ except that an UnresolvedLinkException may be thrown if a symlink is 
+ encountered in the path.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileLinkStatus(Path)}
+ except that an UnresolvedLinkException may be thrown if a symlink is  
+ encountered in the path leading up to the final path component.
+ If the file system does not support symlinks then the behavior is
+ equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileBlockLocations(Path, long, long)} except that
+ Path f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listLocatedStatus(Path)} except that Path f 
+ must be for this file system.
+
+ In HDFS implementation, the BlockLocation of returned LocatedFileStatus
+ will have different formats for replicated and erasure coded file. Please
+ refer to {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext.Util#listStatus(Path)} except that Path f must be 
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical name for this file system.
+ @return a URI string that uniquely identifies this file system]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing modifications, must
+ include entries for user, group, and others for compatibility with
+ permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator{@literal <AclStatus>} which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only the xattr names for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#createSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#renameSnapshot(Path, String, String)}.]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="snapshotDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#deleteSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open a file with the given set of options.
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)}in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param path path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="methodNotSupported"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Helper method that throws an {@link UnsupportedOperationException} for the
+ current {@link FileSystem} method being called.]]>
+      </doc>
+    </method>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for implementors of a Hadoop file system
+ (analogous to the VFS of Unix). Applications do not access this class;
+ instead they access files across all file systems using {@link FileContext}.
+ 
+ Pathnames passed to AbstractFileSystem can be fully qualified URI that
+ matches the "this" file system (ie same scheme and authority) 
+ or a Slash-relative name that is assumed to be relative
+ to the root of the "this" file system .]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AbstractFileSystem -->
+  <!-- start class org.apache.hadoop.fs.AvroFSInput -->
+  <class name="AvroFSInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.avro.file.SeekableInput"/>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FSDataInputStream, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct given an {@link FSDataInputStream} and its length.]]>
+      </doc>
+    </constructor>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FileContext, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct given a {@link FileContext} and a {@link Path}.]]>
+      </doc>
+    </constructor>
+    <method name="length" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="tell" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AvroFSInput -->
+  <!-- start interface org.apache.hadoop.fs.BatchListingOperations -->
+  <interface name="BatchListingOperations"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="batchedListStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="paths" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Batched listing API that returns {@link PartialListing}s for the
+ passed Paths.
+
+ @param paths List of paths to list.
+ @return RemoteIterator that returns corresponding PartialListings.
+ @throws IOException failure]]>
+      </doc>
+    </method>
+    <method name="batchedListLocatedStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="paths" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Batched listing API that returns {@link PartialListing}s for the passed
+ Paths. The PartialListing will contain {@link LocatedFileStatus} entries
+ with locations.
+
+ @param paths List of paths to list.
+ @return RemoteIterator that returns corresponding PartialListings.
+ @throws IOException failure]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface filesystems MAY implement to offer a batched list.
+ If implemented, filesystems SHOULD declare
+ {@link CommonPathCapabilities#FS_EXPERIMENTAL_BATCH_LISTING} to be a supported
+ path capability.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BatchListingOperations -->
+  <!-- start class org.apache.hadoop.fs.BlockLocation -->
+  <class name="BlockLocation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default Constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="org.apache.hadoop.fs.BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset and length.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset, length and corrupt flag.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset and length.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset, length 
+ and corrupt flag.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], org.apache.hadoop.fs.StorageType[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="getCachedHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting a cached replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of names (IP:xferPort) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="getTopologyPaths" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of network topology paths for each of the hosts.
+ The last component of the path is the "name" (IP:xferPort).]]>
+      </doc>
+    </method>
+    <method name="getStorageIds" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storageID of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storage type of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start offset of file associated with this block.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of the block.]]>
+      </doc>
+    </method>
+    <method name="isCorrupt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="isStriped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the block is striped (erasure coded).]]>
+      </doc>
+    </method>
+    <method name="setOffset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="offset" type="long"/>
+      <doc>
+      <![CDATA[Set the start offset of file associated with this block.]]>
+      </doc>
+    </method>
+    <method name="setLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[Set the length of block.]]>
+      </doc>
+    </method>
+    <method name="setCorrupt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="corrupt" type="boolean"/>
+      <doc>
+      <![CDATA[Set the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hosts" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the hosts hosting this block.]]>
+      </doc>
+    </method>
+    <method name="setCachedHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cachedHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the hosts hosting a cached replica of this block.]]>
+      </doc>
+    </method>
+    <method name="setNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the names (host:port) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="setTopologyPaths"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="topologyPaths" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the network topology paths of the hosts.]]>
+      </doc>
+    </method>
+    <method name="setStorageIds"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageIds" type="java.lang.String[]"/>
+    </method>
+    <method name="setStorageTypes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="org.apache.hadoop.fs.StorageType[]"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Represents the network location of a block, information about the hosts
+ that contain block replicas, and other block metadata (E.g. the file
+ offset associated with the block, length, whether it is corrupt, etc).
+
+ For a single BlockLocation, it will have different meanings for replicated
+ and erasure coded files.
+
+ If the file is 3-replicated, offset and length of a BlockLocation represent
+ the absolute value in the file and the hosts are the 3 datanodes that
+ holding the replicas. Here is an example:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ </pre>
+
+ And if the file is erasure-coded, each BlockLocation represents a logical
+ block groups. Value offset is the offset of a block group in the file and
+ value length is the total length of a block group. Hosts of a BlockLocation
+ are the datanodes that holding all the data blocks and parity blocks of a
+ block group.
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ A BlockLocation example will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ </pre>
+
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or
+ {@link FileContext#getFileBlockLocations(Path, long, long)}
+ for more examples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.BlockLocation -->
+  <!-- start interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <interface name="BlockStoragePolicySpi"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the storage policy. Policies are uniquely
+ identified by name.
+
+ @return the name of the storage policy.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the preferred storage types associated with this policy. These
+ storage types are used sequentially for successive block replicas.
+
+ @return preferred storage types used for placing block replicas.]]>
+      </doc>
+    </method>
+    <method name="getCreationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for creating new block replicas. Fallback
+ storage types are used if the preferred storage types are not available.
+
+ @return fallback storage types for new block replicas..]]>
+      </doc>
+    </method>
+    <method name="getReplicationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for replicating existing block replicas.
+ Fallback storage types are used if the preferred storage types are not
+ available.
+
+ @return fallback storage types for replicating existing block replicas.]]>
+      </doc>
+    </method>
+    <method name="isCopyOnCreateFile" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the policy is inherit-only and cannot be changed for
+ an existing file.
+
+ @return true if the policy is inherit-only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A storage policy specifies the placement of block replicas on specific
+ storage types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <interface name="ByteBufferPositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to {@code buf.remaining()} bytes into buf from a given position
+ in the file and returns the number of bytes read. Callers should use
+ {@code buf.limit(...)} to control the size of the desired read and
+ {@code buf.position(...)} to control the offset into the buffer the data
+ should be written to.
+ <p>
+ After a successful call, {@code buf.position()} will be advanced by the
+ number of bytes read and {@code buf.limit()} will be unchanged.
+ <p>
+ In the case of an exception, the state of the buffer (the contents of the
+ buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is
+ undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p>
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with
+ {@link StreamCapabilities#PREADBYTEBUFFER} to check if the underlying
+ stream supports this interface, otherwise they might get a
+ {@link UnsupportedOperationException}.
+ <p>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+ <p>
+ This does not change the current offset of a file, and is thread-safe.
+
+ @param position position within file
+ @param buf the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if reached
+         end-of-stream
+ @throws IOException if there is some error performing the read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads {@code buf.remaining()} bytes into buf from a given position in
+ the file or until the end of the data was reached before the read
+ operation completed. Callers should use {@code buf.limit(...)} to
+ control the size of the desired read and {@code buf.position(...)} to
+ control the offset into the buffer the data should be written to.
+ <p>
+ This operation provides similar semantics to
+ {@link #read(long, ByteBuffer)}, the difference is that this method is
+ guaranteed to read data until the {@link ByteBuffer} is full, or until
+ the end of the data stream is reached.
+
+ @param position position within file
+ @param buf the ByteBuffer to receive the results of the read operation.
+ @throws IOException if there is some error performing the read
+ @throws EOFException the end of the data was reached before
+ the read operation completed
+ @see #read(long, ByteBuffer)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a positioned read API that writes to a
+ {@link ByteBuffer} rather than a {@code byte[]}.
+
+ @see PositionedReadable
+ @see ByteBufferReadable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <interface name="ByteBufferReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to buf.remaining() bytes into buf. Callers should use
+ buf.limit(..) to control the size of the desired read.
+ <p>
+ After a successful call, {@code buf.position()} will be advanced by the
+ number of bytes read and {@code buf.limit()} will be unchanged.
+ <p>
+ In the case of an exception, the state of the buffer (the contents of the
+ buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is
+ undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p>
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with
+ {@link StreamCapabilities#READBYTEBUFFER} to check if the underlying
+ stream supports this interface, otherwise they might get a
+ {@link UnsupportedOperationException}.
+ <p>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+
+ @param buf
+          the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if 
+         reach end-of-stream
+ @throws IOException
+           if there is some error performing the read]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a read API that writes to a
+ ByteBuffer, not a byte[].]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <!-- start interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <interface name="CanSetDropBehind"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setDropBehind"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropCache" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Configure whether the stream should drop the cache.
+
+ @param dropCache     Whether to drop the cache.  null means to use the
+                      default value.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting the drop-behind.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <!-- start interface org.apache.hadoop.fs.CanSetReadahead -->
+  <interface name="CanSetReadahead"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setReadahead"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Set the readahead on this stream.
+
+ @param readahead     The readahead to use.  null means to use the default.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting readahead.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetReadahead -->
+  <!-- start interface org.apache.hadoop.fs.CanUnbuffer -->
+  <interface name="CanUnbuffer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="unbuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reduce the buffering.  This will also free sockets and file descriptors
+ held by the stream, if possible.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSDataInputStreams implement this interface to indicate that they can clear
+ their buffers on request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanUnbuffer -->
+  <!-- start class org.apache.hadoop.fs.ChecksumException -->
+  <class name="ChecksumException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumException" type="java.lang.String, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown for checksum errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumException -->
+  <!-- start class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <class name="ChecksumFileSystem" extends="org.apache.hadoop.fs.FilterFileSystem"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApproxChkSumLength" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to verify checksum.]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the raw file system]]>
+      </doc>
+    </method>
+    <method name="getChecksumFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the name of the checksum file associated with a file.]]>
+      </doc>
+    </method>
+    <method name="isChecksumFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return true iff file is a checksum file name.]]>
+      </doc>
+    </method>
+    <method name="getChecksumFileLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="fileSize" type="long"/>
+      <doc>
+      <![CDATA[Return the length of the checksum file given the size of the 
+ actual file.]]>
+      </doc>
+    </method>
+    <method name="getBytesPerSum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the bytes Per Checksum]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChecksumLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <param name="bytesPerSum" type="int"/>
+      <doc>
+      <![CDATA[Calculated the length of the checksum file in bytes.
+ @param size the length of the data file in bytes
+ @param bytesPerSum the number of bytes in a checksum block
+ @return the number of bytes in the checksum file]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename files/dirs]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Implement the delete(Path, boolean) in checksum
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given path
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given patch
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="copyCrc" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ If src and dst are directories, the copyCrc parameter
+ determines whether to copy CRC files.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Report a checksum error to the file system.
+ @param f the file name containing the error
+ @param in the stream open on the file
+ @param inPos the position of the beginning of the bad data in the file
+ @param sums the stream open on the checksum file
+ @param sumsPos the position of the beginning of the bad data in the checksum file
+ @return if retry is necessary]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's
+ {@link #openFileWithOptions}() method is called, and so ultimately
+ its {@link #open(Path, int)}.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the file as a blocking call to {@link #open(Path, int)}.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's create() method is
+ ultimately called.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's create() method is
+ ultimately called.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Disable those operations which the checksummed FS blocks.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abstract Checksumed FileSystem.
+ It provide a basic implementation of a Checksumed FileSystem,
+ which creates a checksum file for each raw file.
+ It generates &amp; verifies checksums at the client side.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <!-- start class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <class name="CommonConfigurationKeysPublic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommonConfigurationKeysPublic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DEFAULT_NAME_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DF_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DU_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_GETSPACEUSED_JITTER_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_CHECKPOINT_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_PROTECTED_DIRECTORIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Directories that cannot be removed unless empty, even by an
+ administrator.]]>
+      </doc>
+    </field>
+    <field name="FS_LOCAL_BLOCK_SIZE_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Not used anywhere, looks like default value for FS_LOCAL_BLOCK_SIZE]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_AUTOMATIC_CLOSE_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CREATION_PARALLEL_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of filesystems instances can be created in parallel.
+ <p></p>
+ A higher number here does not necessarily improve performance, especially
+ for object stores, where multiple threads may be attempting to create an FS
+ instance for the same URI.
+ <p></p>
+ Default value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="FS_CREATION_PARALLEL_COUNT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #FS_CREATION_PARALLEL_COUNT}.
+ <p></p>
+ Default value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="FS_FILE_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_PORT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_ERROR_RATE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_COMPRESSION_CODEC_LZO_CLASS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Codec class that implements Lzo compression algorithm]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_INTERVAL_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_SKIP_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_FILE_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SKIP_CHECKSUM_ERRORS_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SERIALIZATIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_IO_CHUNK_SIZE_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_INPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_OUTPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable low-latency connections from the client]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of IPC_CLIENT_LOW_LATENCY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_LISTEN_QUEUE_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_KILL_MAX_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_IDLETHRESHOLD_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_REUSEADDR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_REUSEADDR_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_REUSEADDR_KEY.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_MAX_CONNECTIONS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Logs if a RPC is really slow compared to rest of RPCs.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SOCKS_SERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_UTIL_HASH_TYPE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_MAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHENTICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHORIZATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SERVICE_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_INTERFACE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_NAMESERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKENS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_AUTHENTICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_PROTECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Sasl Properties for a connection]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Impersonation provider]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_SIZE]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_NUM_REFILL_THREADS]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is the number of providers specified.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 100 ms.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 2 secs.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_HTTP_LOGS_ENABLED]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SYSTEM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CUSTOM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TAGS_SYSTEM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_TAGS_CUSTOM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option for the shutdown hook manager shutdown time:
+  {@value}.]]>
+      </doc>
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default shutdown hook timeout: {@value} seconds.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_PROMETHEUS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_PROMETHEUS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains constants for configuration keys used
+ in the common code.
+
+ It includes all publicly documented configuration keys. In general
+ this class should not be used directly (use CommonConfigurationKeys
+ instead)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <!-- start class org.apache.hadoop.fs.ContentSummary -->
+  <class name="ContentSummary" extends="org.apache.hadoop.fs.QuotaUsage"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ContentSummary"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder
+  This constructor implicitly set spaceConsumed the same as length.
+  spaceConsumed and length must be set explicitly with
+  ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the length]]>
+      </doc>
+    </method>
+    <method name="getSnapshotLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the directory count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the file count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSnapshotSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getErasureCodingPolicy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the header of the output.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="getSnapshotHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields from the summary header.
+ 
+ @return names of fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="getQuotaHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields used in the quota summary.
+ 
+ @return names of quota fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ if hOption is false, file sizes are returned in bytes
+ if hOption is true, file sizes are returned in human readable
+ if tOption is true, display the quota by storage types
+ if tOption is false, same logic with #toString(boolean,boolean)
+ if xOption is false, output includes the calculation from snapshots
+ if xOption is true, output excludes the calculation from snapshots
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toSnapshot" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the snapshot counts in the output
+ format.
+ @param hOption flag indicating human readable or not
+ @return String representation of the snapshot counts]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Store the summary of a content (a directory or a file).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ContentSummary -->
+  <!-- start class org.apache.hadoop.fs.CreateFlag -->
+  <class name="CreateFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.CreateFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.CreateFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag and throw exception if it is invalid
+ @param flag set of CreateFlag
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.Object"/>
+      <param name="pathExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for create operation
+ @param path Object representing the path; usually String or {@link Path}
+ @param pathExists pass true if the path exists in the file system
+ @param flag set of CreateFlag
+ @throws IOException on error
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validateForAppend"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for the append operation. The flag must contain
+ APPEND, and cannot contain OVERWRITE.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[CreateFlag specifies the file create semantic. Users can combine flags like: <br>
+ <code>
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
+ </code>
+ <p>
+ 
+ Use the CreateFlag as follows:
+ <ol>
+ <li> CREATE - to create a file if it does not exist, 
+ else throw FileAlreadyExists.</li>
+ <li> APPEND - to append to a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> OVERWRITE - to truncate a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> CREATE|APPEND - to create a file if it does not exist, 
+ else append to an existing file.</li>
+ <li> CREATE|OVERWRITE - to create a file if it does not exist, 
+ else overwrite an existing file.</li>
+ <li> SYNC_BLOCK - to force closed blocks to the disk device.
+ In addition {@link Syncable#hsync()} should be called after each write,
+ if true synchronous behavior is required.</li>
+ <li> LAZY_PERSIST - Create the block on transient storage (RAM) if
+ available.</li>
+ <li> APPEND_NEWBLOCK - Append data to a new block instead of end of the last
+ partial block.</li>
+ </ol>
+ 
+ Following combinations are not valid and will result in
+ {@link HadoopIllegalArgumentException}:
+ <ol>
+ <li> APPEND|OVERWRITE</li>
+ <li> CREATE|APPEND|OVERWRITE</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CreateFlag -->
+  <!-- start class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.fs.FileChecksum -->
+  <class name="FileChecksum" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileChecksum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAlgorithmName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The checksum algorithm name]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The length of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The value of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if both the algorithms and the values are the same.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An abstract class representing file checksums for files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileChecksum -->
+  <!-- start class org.apache.hadoop.fs.FileContext -->
+  <class name="FileContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the file system of supplied path.
+ 
+ @param absOrFqPath - absolute or fully qualified path
+ @return the file system of the path
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>absOrFqPath</code> is not supported.
+ @throws IOException If the file system for <code>absOrFqPath</code> could
+         not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a FileContext with specified FS as default using the specified
+ config.
+ 
+ @param defFS
+ @param aConf
+ @return new FileContext with specified FS as default.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="defaultFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified file system using the default config.
+ 
+ @param defaultFS
+ @return a FileContext with the specified AbstractFileSystem
+                 as the default FS.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the default config read from the
+ $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
+ from core-defaults.xml in the release jar.
+ 
+ @throws UnsupportedFileSystemException If the file system from the default
+           configuration is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@return a FileContext for the local file system using the default config.
+ @throws UnsupportedFileSystemException If the file system for
+           {@link FsConstants#LOCAL_FS_URI} is not supported.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified URI using the default config.
+ 
+ @param defaultFsUri
+ @return a FileContext with the specified URI as the default FS.
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>defaultFsUri</code> is not supported]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified default URI using the specified config.
+ 
+ @param defaultFsUri
+ @param aConf
+ @return new FileContext for specified uri
+ @throws UnsupportedFileSystemException If the file system with specified is
+           not supported
+ @throws RuntimeException If the file system specified is supported but
+         could not be instantiated, or if login fails.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the passed config. Generally it is better to use
+ {@link #getFileContext(URI, Configuration)} instead of this one.
+ 
+ 
+ @param aConf
+ @return new FileContext
+ @throws UnsupportedFileSystemException If file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@param aConf - from which the FileContext is configured
+ @return a FileContext for the local file system using the specified config.
+ 
+ @throws UnsupportedFileSystemException If default file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newWDir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the working directory for wd-relative names (such a "foo/bar"). Working
+ directory feature is provided by simply prefixing relative names with the
+ working dir. Note this is different from Unix where the wd is actually set
+ to the inode. Hence setWorkingDir does not follow symlinks etc. This works
+ better in a distributed environment that has multiple independent roots.
+ {@link #getWorkingDirectory()} should return what setWorkingDir() set.
+ 
+ @param newWDir new working directory
+ @throws IOException 
+ <br>
+           NewWdir can be one of:
+           <ul>
+           <li>relative path: "foo/bar";</li>
+           <li>absolute without scheme: "/foo/bar"</li>
+           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
+           </ul>
+ <br>
+           Illegal WDs:
+           <ul>
+           <li>relative with scheme: "xx:foo/bar"</li>
+           <li>non existent directory</li>
+           </ul>]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the working directory for wd-relative names (such a "foo/bar").]]>
+      </doc>
+    </method>
+    <method name="getUgi" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the ugi in the file-context
+ @return UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ @return the home directory]]>
+      </doc>
+    </method>
+    <method name="getUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the umask of this FileContext]]>
+      </doc>
+    </method>
+    <method name="setUMask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newUmask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set umask to the supplied parameter.
+ @param newUmask  the new umask]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolve the path following any symlinks or mount points
+ @param f to be resolved
+ @return fully qualified resolved path
+ 
+ @throws FileNotFoundException  If <code>f</code> does not exist
+ @throws AccessControlException if access denied
+ @throws IOException If an IO Error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified if it is isn't. 
+ A Fully-qualified path has scheme and authority specified and an absolute
+ path.
+ Use the default file system and working dir in this FileContext to qualify.
+ @param path
+ @return qualified path]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create or overwrite file on indicated path and returns an output stream for
+ writing into the file.
+ 
+ @param f the file name to open
+ @param createFlag gives the semantics of create; see {@link CreateFlag}
+ @param opts file creation options; see {@link Options.CreateOpts}.
+          <ul>
+          <li>Progress - to report progress on the operation - default null
+          <li>Permission - umask is applied against permission: default is
+          FsPermissions:getDefault()
+ 
+          <li>CreateParent - create missing parent path; default is to not
+          to create parents
+          <li>The defaults for the following are SS defaults of the file
+          server implementing the target path. Not all parameters make sense
+          for all kinds of file system - eg. localFS ignores Blocksize,
+          replication, checksum
+          <ul>
+          <li>BufferSize - buffersize used in FSDataOutputStream
+          <li>Blocksize - block size for file blocks
+          <li>ReplicationFactor - replication for blocks
+          <li>ChecksumParam - Checksum parameters. server default is used
+          if not specified.
+          </ul>
+          </ul>
+ 
+ @return {@link FSDataOutputStream} for created file
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>f</code> already exists
+ @throws FileNotFoundException If parent of <code>f</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>f</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ a file on indicated path.
+
+ @param f the file path to create builder for.
+ @return {@link FSDataOutputStreamBuilder} to build a
+         {@link FSDataOutputStream}.
+
+ Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ builder parameters will be verified by {@link FileContext} and
+ {@link AbstractFileSystem#create}. And filesystem states will be modified.
+
+ Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ same exceptions as create(Path, EnumSet, CreateOpts...).]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make(create) a directory and all the non-existent parents.
+ 
+ @param dir - the dir to make
+ @param permission - permissions is set permission{@literal &~}umask
+ @param createParent - if true then missing parent dirs are created if false
+          then parent must exist
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If directory <code>dir</code> already
+           exists
+ @throws FileNotFoundException If parent of <code>dir</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>dir</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>dir</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+ @param f the path to delete.
+ @param recursive if path is a directory and set to 
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is invalid]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path using
+ default buffersize.
+ @param f the file name to open
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ 
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+ <li>Fails if path is a directory.
+ <li>Fails if path does not exist.
+ <li>Fails if path is not closed.
+ <li>Fails if new size is greater than current size.
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param f file name
+ @param replication new replication
+
+ @return true if successful
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+ <li>Fails if src is a file and dst is a directory.
+ <li>Fails if src is a directory and dst is a file.
+ <li>Fails if the parent of dst does not exist or is a file.
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails if the dst
+ already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites the dst if
+ it is a file or an empty directory. Rename fails if dst is a non-empty
+ directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for details
+ <p>
+ 
+ @param src path to be renamed
+ @param dst new path after rename
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If <code>dst</code> already exists and
+           <code>options</code> has {@link Options.Rename#OVERWRITE}
+           option false.
+ @throws FileNotFoundException If <code>src</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>src</code>
+           and <code>dst</code> is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param f
+ @param permission - the new absolute permission (umask is not applied)
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory). The parameters username
+ and groupname cannot both be null.
+ 
+ @param f The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws HadoopIllegalArgumentException If <code>username</code> or
+           <code>groupname</code> is invalid.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param f The path
+ @param mtime Set the modification time of this file.
+        The number of milliseconds since epoch (Jan 1, 1970). 
+        A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+        The number of milliseconds since Jan 1, 1970. 
+        A value of -1 means that this call should not set access time.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file.
+
+ @param f file path
+
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag for the  file system denoted by the path.
+ This is only applicable if the 
+ corresponding FileSystem supports checksum. By default doesn't do anything.
+ @param verifyChecksum
+ @param f set the verifyChecksum for the Filesystem containing this path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+
+ @return a FileStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path. If the path 
+ refers to a symlink then the FileStatus of the symlink is returned.
+ The behavior is equivalent to #getFileStatus() if the underlying
+ file system does not support symbolic links.
+ @param  f The path we want information from.
+ @return A FileStatus object
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the target of the given symbolic link as it was specified
+ when the link was created.  Links in the path leading up to the
+ final path component are resolved transparently.
+
+ @param f the path to return the target of
+ @return The un-interpreted target of the symbolic link.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If path <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If the given path does not refer to a symlink
+           or an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ file system denoted by the Parh argument p.
+ If the file system has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ 
+ @param f Path for which status should be obtained. null means the
+ root partition of the default file system. 
+
+ @return a FsStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a symbolic link to an existing file. An exception is thrown if 
+ the symlink exits, the user does not have permission to create symlink,
+ or the underlying file system does not support symlinks.
+ 
+ Symlink permissions are ignored, access to a symlink is determined by
+ the permissions of the symlink target.
+ 
+ Symlinks in paths leading up to the final path component are resolved 
+ transparently. If the final path component refers to a symlink some 
+ functions operate on the symlink itself, these are:
+ - delete(f) and deleteOnExit(f) - Deletes the symlink.
+ - rename(src, dst) - If src refers to a symlink, the symlink is 
+   renamed. If dst refers to a symlink, the symlink is over-written.
+ - getLinkTarget(f) - Returns the target of the symlink. 
+ - getFileLinkStatus(f) - Returns a FileStatus object describing
+   the symlink.
+ Some functions, create() and mkdir(), expect the final path component
+ does not exist. If they are given a path that refers to a symlink that 
+ does exist they behave as if the path referred to an existing file or 
+ directory. All other functions fully resolve, ie follow, the symlink. 
+ These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
+ setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
+ getFsStatus, getFileStatus, exists, and listStatus.
+ 
+ Symlink targets are stored as given to createSymlink, assuming the 
+ underlying file system is capable of storing a fully qualified URI.
+ Dangling symlinks are permitted. FileContext supports four types of 
+ symlink targets, and resolves them as follows
+ <pre>
+ Given a path referring to a symlink of form:
+ 
+   {@literal <---}X{@literal --->}
+   fs://host/A/B/link 
+   {@literal <-----}Y{@literal ----->}
+ 
+ In this path X is the scheme and authority that identify the file system,
+ and Y is the path leading up to the final path component "link". If Y is
+ a symlink  itself then let Y' be the target of Y and X' be the scheme and
+ authority of Y'. Symlink targets may:
+ 
+ 1. Fully qualified URIs
+ 
+ fs://hostX/A/B/file  Resolved according to the target file system.
+ 
+ 2. Partially qualified URIs (eg scheme but no host)
+ 
+ fs:///A/B/file  Resolved according to the target file system. Eg resolving
+                 a symlink to hdfs:///A results in an exception because
+                 HDFS URIs must be fully qualified, while a symlink to 
+                 file:///A will not since Hadoop's local file systems 
+                 require partially qualified URIs.
+ 
+ 3. Relative paths
+ 
+ path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
+       is "../B/file" then [Y'][path] is hdfs://host/B/file
+ 
+ 4. Absolute paths
+ 
+ path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
+       is "/file" then [X][path] is hdfs://host/file
+ </pre>
+ 
+ @param target the target of the symbolic link
+ @param link the path to be created that points to target
+ @param createParent if true then missing parent dirs are created if 
+                     false then parent must exist
+
+
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>link</code> already exists
+ @throws FileNotFoundException If <code>target</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>link</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for 
+           <code>target</code> or <code>link</code> is not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory. 
+ Return the file's status and block locations If the path is a file.
+ 
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+ If any IO exception (for example the input directory gets deleted while
+ listing is being executed), next() or hasNext() of the returned iterator
+ may throw a RuntimeException with the io exception as the cause.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted on JVM shutdown.
+ 
+ @param f the existing path to delete.
+
+ @return  true if deleteOnExit is successful, otherwise false.
+
+ @throws AccessControlException If access is denied
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="util" return="org.apache.hadoop.fs.FileContext.Util"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolve" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path.
+ Returns the new path object.]]>
+      </doc>
+    </method>
+    <method name="resolveIntermediate" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path leading up 
+ to, but not including the final path component.
+ @param f path to resolve
+ @return the new path object.]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system
+ 
+ @param uri
+          the uri to lookup the statistics. Only scheme and authority part
+          of the uri are used as the key to store and lookup.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all the statistics stored in AbstractFileSystem, for all the file
+ systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints the statistics to standard output. File System is identified by the
+ scheme and authority.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Map of uri and statistics for each filesystem instantiated. The uri
+         consists of scheme and authority for the filesystem.]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing entries
+ to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ modifications, must include entries for user, group, and others for
+ compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator{@literal <}AclStatus{@literal >} which returns
+         each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List{@literal <}String{@literal >} of the XAttr names of the
+ file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+
+ @param path The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param path file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(Path, int)} unless a subclass
+ executes the open command differently.
+
+ The semantics of this call are therefore the same as that of
+ {@link #open(Path, int)} with one special point: it is in
+ {@code FSDataInputStreamBuilder.build()} in which the open operation
+ takes place -it is there where all preconditions to the operation
+ are checked.
+ @param path file path
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path capabilities of the bonded {@code AbstractFileSystem}.
+ @param path path to query the capability of.
+ @param capability string to query the stream support for.
+ @return true iff the capability is supported under that FS.
+ @throws IOException path resolution or other IO failure
+ @throws IllegalArgumentException invalid arguments]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param path path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory and symlink
+ In previous versions, this default permission was also used to
+ create files, so files created end up with ugo+x permission.
+ See HADOOP-9155 for detail. 
+ Two new constants are added to solve this, please use 
+ {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ {@link FileContext#FILE_DEFAULT_PERM} for file.
+ This constant is kept for compatibility.]]>
+      </doc>
+    </field>
+    <field name="DIR_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory]]>
+      </doc>
+    </field>
+    <field name="FILE_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for file]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileContext shutdown hook.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The FileContext class provides an interface for users of the Hadoop
+ file system. It exposes a number of file system operations, e.g. create,
+ open, list.
+ 
+ <h2>Path Names</h2>
+ 
+ The Hadoop file system supports a URI namespace and URI names. This enables
+ multiple types of file systems to be referenced using fully-qualified URIs.
+ Two common Hadoop file system implementations are
+ <ul>
+ <li>the local file system: file:///path
+ <li>the HDFS file system: hdfs://nnAddress:nnPort/path
+ </ul>
+ 
+ The Hadoop file system also supports additional naming schemes besides URIs.
+ Hadoop has the concept of a <i>default file system</i>, which implies a
+ default URI scheme and authority. This enables <i>slash-relative names</i>
+ relative to the default FS, which are more convenient for users and
+ application writers. The default FS is typically set by the user's
+ environment, though it can also be manually specified.
+ <p>
+ 
+ Hadoop also supports <i>working-directory-relative</i> names, which are paths
+ relative to the current working directory (similar to Unix). The working
+ directory can be in a different file system than the default FS.
+ <p>
+ Thus, Hadoop path names can be specified as one of the following:
+ <ul>
+ <li>a fully-qualified URI: scheme://authority/path (e.g.
+ hdfs://nnAddress:nnPort/foo/bar)
+ <li>a slash-relative name: path relative to the default file system (e.g.
+ /foo/bar)
+ <li>a working-directory-relative name: path relative to the working dir (e.g.
+ foo/bar)
+ </ul>
+  Relative paths with scheme (scheme:foo/bar) are illegal.
+  
+ <h2>Role of FileContext and Configuration Defaults</h2>
+
+ The FileContext is the analogue of per-process file-related state in Unix. It
+ contains two properties:
+ 
+ <ul>
+ <li>the default file system (for resolving slash-relative names)
+ <li>the umask (for file permissions)
+ </ul>
+ In general, these properties are obtained from the default configuration file
+ in the user's environment (see {@link Configuration}).
+ 
+ Further file system properties are specified on the server-side. File system
+ operations default to using these server-side defaults unless otherwise
+ specified.
+ <p>
+ The file system related server-side defaults are:
+  <ul>
+  <li> the home directory (default is "/user/userName")
+  <li> the initial wd (only for local fs)
+  <li> replication factor
+  <li> block size
+  <li> buffer size
+  <li> encryptDataTransfer 
+  <li> checksum option. (checksumType and  bytesPerChecksum)
+  </ul>
+
+ <h2>Example Usage</h2>
+
+ Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
+   Unspecified values come from core-defaults.xml in the release jar.
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(); // uses the default config
+                                                // which has your default FS 
+  <li>  myFContext.create(path, ...);
+  <li>  myFContext.setWorkingDir(path);
+  <li>  myFContext.open (path, ...);  
+  <li>...
+  </ul>  
+ Example 2: Get a FileContext with a specific URI as the default FS
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(URI);
+  <li> myFContext.create(path, ...);
+  <li>...
+ </ul>
+ Example 3: FileContext with local file system as the default
+  <ul> 
+  <li> myFContext = FileContext.getLocalFSFileContext();
+  <li> myFContext.create(path, ...);
+  <li> ...
+  </ul> 
+ Example 4: Use a specific config, ignoring $HADOOP_CONFIG
+  Generally you should not need use a config unless you are doing
+   <ul> 
+   <li> configX = someConfigSomeOnePassedToYou;
+   <li> myFContext = getFileContext(configX); // configX is not changed,
+                                              // is passed down 
+   <li> myFContext.create(path, ...);
+   <li>...
+  </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileContext -->
+  <!-- start class org.apache.hadoop.fs.FileStatus -->
+  <class name="FileStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for file systems on which symbolic links are not supported]]>
+      </doc>
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="org.apache.hadoop.fs.FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor.
+
+ @param other FileStatus to copy]]>
+      </doc>
+    </constructor>
+    <method name="attributes" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acl" type="boolean"/>
+      <param name="crypt" type="boolean"/>
+      <param name="ec" type="boolean"/>
+      <param name="sn" type="boolean"/>
+      <doc>
+      <![CDATA[Convert boolean attributes to a set of flags.
+ @param acl   See {@link AttrFlags#HAS_ACL}.
+ @param crypt See {@link AttrFlags#HAS_CRYPT}.
+ @param ec    See {@link AttrFlags#HAS_EC}.
+ @param sn    See {@link AttrFlags#SNAPSHOT_ENABLED}.
+ @return converted set of flags.]]>
+      </doc>
+    </method>
+    <method name="getLen" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of this file, in bytes.
+ @return the length of this file, in bytes.]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a file?
+ @return true if this is a file]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a directory?
+ @return true if this is a directory]]>
+      </doc>
+    </method>
+    <method name="isDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.">
+      <doc>
+      <![CDATA[Old interface, instead use the explicit {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ @return true if this is a directory.
+ @deprecated Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.]]>
+      </doc>
+    </method>
+    <method name="isSymlink" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a symbolic link?
+ @return true if this is a symbolic link]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the block size of the file.
+ @return the number of bytes]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the replication factor of a file.
+ @return the replication factor of a file.]]>
+      </doc>
+    </method>
+    <method name="getModificationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modification time of the file.
+ @return the modification time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getAccessTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the access time of the file.
+ @return the access time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get FsPermission associated with the file.
+ @return permission. If a filesystem does not have a notion of permissions
+         or if permissions could not be determined, then default 
+         permissions equivalent of "rwxrwxrwx" is returned.]]>
+      </doc>
+    </method>
+    <method name="hasAcl" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory has ACLs set.
+
+ @return true if the underlying file or directory has ACLs set.]]>
+      </doc>
+    </method>
+    <method name="isEncrypted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is encrypted or not.
+
+ @return true if the underlying file is encrypted.]]>
+      </doc>
+    </method>
+    <method name="isErasureCoded" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is erasure coded or not.
+
+ @return true if the underlying file or directory is erasure coded.]]>
+      </doc>
+    </method>
+    <method name="isSnapshotEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if directory is Snapshot enabled or not.
+
+ @return true if directory is snapshot enabled]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the owner of the file.
+ @return owner of the file. The string could be empty if there is no
+         notion of owner of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getGroup" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group associated with the file.
+ @return group for the file. The string could be empty if there is no
+         notion of group of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Sets permission.
+ @param permission if permission is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets owner.
+ @param owner if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets group.
+ @param group if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return The contents of the symbolic link.]]>
+      </doc>
+    </method>
+    <method name="setSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus.
+ This method was added back by HADOOP-14683 to keep binary compatibility.
+
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.
+ @throws ClassCastException if the specified object is not FileStatus]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read instance encoded as protobuf from stream.
+ @param in Input stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write instance encoded as protobuf to stream.
+ @param out Output stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+    </method>
+    <field name="NONE" type="java.util.Set"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shared, empty set of attributes (a common case for FileStatus).]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface that represents the client side information for a file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileStatus -->
+  <!-- start class org.apache.hadoop.fs.FileSystem -->
+  <class name="FileSystem" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.hadoop.security.token.DelegationTokenIssuer"/>
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="FileSystem"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a FileSystem instance based on the uri, the passed in
+ configuration and the user.
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return the filesystem instance
+ @throws IOException failure to load
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+ somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the configured FileSystem implementation.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="getDefaultUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the default FileSystem URI from a configuration.
+ @param conf the configuration to use
+ @return the uri of the default filesystem]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a FileSystem.
+
+ Called after the new FileSystem instance is constructed, and before it
+ is ready for use.
+
+ FileSystem implementations overriding this method MUST forward it to
+ their superclass, though the order in which it is done, and whether
+ to alter the configuration before the invocation are options of the
+ subclass.
+ @param name a URI whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration
+ @throws IOException on any failure to initialize this instance.
+ @throws IllegalArgumentException if the URI is considered invalid.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for this FileSystem.
+ <p>
+ This implementation throws an <code>UnsupportedOperationException</code>.
+
+ @return the protocol scheme for this FileSystem.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI which identifies this FileSystem.
+
+ @return the URI of this filesystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a canonicalized form of this FileSystem's URI.
+
+ The default implementation simply calls {@link #canonicalizeUri(URI)}
+ on the filesystem's own URI, so subclasses typically only need to
+ implement that method.
+
+ @see #canonicalizeUri(URI)]]>
+      </doc>
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Canonicalize the given URI.
+
+ This is implementation-dependent, and may for example consist of
+ canonicalizing the hostname using DNS and adding the default
+ port if not specified.
+
+ The default implementation simply fills in the default port if
+ not specified and if {@link #getDefaultPort()} returns a
+ default port.
+
+ @return URI
+ @see NetUtils#getCanonicalUri(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FileSystem.
+ @return the default port or 0 if there isn't one]]>
+      </doc>
+    </method>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical service name for this FileSystem.
+ The token cache is the only user of the canonical service name,
+ and uses it to lookup this FileSystem's service tokens.
+ If the file system provides a token of its own then it must have a
+ canonical name, otherwise the canonical name can be null.
+
+ Default implementation: If the FileSystem has child file systems
+ (such as an embedded file system) then it is assumed that the FS has no
+ tokens of its own and hence returns a null name; otherwise a service
+ name is built using Uri and port.
+
+ @return a service string that uniquely identifies this file system, null
+         if the filesystem does not implement tokens
+ @see SecurityUtil#buildDTServiceName(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="call {@link #getUri()} instead.">
+      <doc>
+      <![CDATA[@deprecated call {@link #getUri()} instead.]]>
+      </doc>
+    </method>
+    <method name="getNamed" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="call {@link #get(URI, Configuration)} instead.">
+      <param name="name" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated call {@link #get(URI, Configuration)} instead.]]>
+      </doc>
+    </method>
+    <method name="getLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the local FileSystem.
+ @param conf the configuration to configure the FileSystem with
+ if it is newly instantiated.
+ @return a LocalFileSystem
+ @throws IOException if somehow the local FS cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a FileSystem for this URI's scheme and authority.
+ <ol>
+ <li>
+   If the configuration has the property
+   {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+   a new instance will be created, initialized with the supplied URI and
+   configuration, then returned without being cached.
+ </li>
+ <li>
+   If the there is a cached FS instance matching the same URI, it will
+   be returned.
+ </li>
+ <li>
+   Otherwise: a new FS instance will be created, initialized with the
+   configuration and URI, cached and returned to the caller.
+ </li>
+ </ol>
+ @throws IOException if the FileSystem cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority and the
+ given user. Internally invokes {@link #newInstance(URI, Configuration)}
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return filesystem instance
+ @throws IOException if the FileSystem cannot be instantiated.
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+         somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority.
+ The entire URI is passed to the FileSystem instance's initialize method.
+ This always returns a new FileSystem object.
+ @param uri FS URI
+ @param config configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a unique configured FileSystem implementation for the default
+ filesystem of the supplied configuration.
+ This always returns a new FileSystem object.
+ @param conf the configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstanceLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a unique local FileSystem object.
+ @param conf the configuration to configure the FileSystem with
+ @return a new LocalFileSystem object.
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="closeAll"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances. After this operation, they
+ may not be used in any operations.
+
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="closeAllForUGI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances for a given UGI.
+ Be sure those filesystems are not used anymore.
+ @param ugi user group info to close
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Qualify a path to one which uses this FileSystem and, if relative,
+ made absolute.
+ @param path to qualify.
+ @return this path if it contains a scheme and authority and is absolute, or
+ a new path that includes a path and authority and is fully qualified
+ @see Path#makeQualified(URI, Path)
+ @throws IllegalArgumentException if the path has a schema/URI different
+ from this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file with the provided permission.
+
+ The permission of the file is set to be the provided permission as in
+ setPermission, not permission{@literal &~}umask
+
+ The HDFS implementation is implemented using two RPCs.
+ It is understood that it is inefficient,
+ but the implementation is thread-safe. The other option is to change the
+ value of umask in configuration to be 0, but it is not thread-safe.
+
+ @param fs FileSystem
+ @param file the name of the file to be created
+ @param permission the permission of the file
+ @return an output stream
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a directory with the provided permission.
+ The permission of the directory is set to be the provided permission as in
+ setPermission, not permission{@literal &~}umask
+
+ @see #create(FileSystem, Path, FsPermission)
+
+ @param fs FileSystem handle
+ @param dir the name of the directory to be created
+ @param permission the permission of the directory
+ @return true if the directory creation succeeds; false otherwise
+ @throws IOException A problem creating the directories.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+
+ The base implementation performs case insensitive equality checks
+ of the URIs' schemes and authorities. Subclasses may implement slightly
+ different checks.
+ @param path to check
+ @throws IllegalArgumentException if the path is not considered to be
+ part of this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For nonexistent
+ file or regions, {@code null} is returned.
+
+ <pre>
+   if f == null :
+     result = null
+   elif f.getLen() {@literal <=} start:
+     result = []
+   else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ </pre>
+ This call is most helpful with and distributed filesystem
+ where the hostnames of machines that contain blocks of the given file
+ can be determined.
+
+ The default implementation returns an array containing one element:
+ <pre>
+ BlockLocation( { "localhost:9866" },  { "localhost" }, 0, file.getLen())
+ </pre>
+
+ In HDFS, if file is three-replicated, the returned array contains
+ elements like:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+   hosts: {"host2:9866", "host3:9866, host4:9866"})
+ </pre>
+
+ And if a file is erasure-coded, the returned BlockLocation are logical
+ block groups.
+
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then
+ there will be one BlockLocation returned, with 0 offset, actual file size
+ and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks.
+ 3. If the file size is less than one group size but greater than one
+ stripe size, then there will be one BlockLocation returned, with 0 offset,
+ actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting
+ the actual blocks.
+ 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123
+ for example, then the result will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+   "host4:9866", "host5:9866"})
+ </pre>
+
+ @param file FilesStatus to get data from
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For a nonexistent
+ file or regions, {@code null} is returned.
+
+ This call is most helpful with location-aware distributed
+ filesystems, where it returns hostnames of machines that
+ contain the given file.
+
+ A FileSystem will normally return the equivalent result
+ of passing the {@code FileStatus} of the path to
+ {@link #getFileBlockLocations(FileStatus, long, long)}
+
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @return server default configuration values
+ @throws IOException IO failure
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @return server default configuration values
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path, resolving the path
+ through any symlinks or mount point.
+ @param p path to be resolved
+ @return fully qualified path
+ @throws FileNotFoundException if the path is not present
+ @throws IOException for any other error]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file to open
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If {@link #open(PathHandle, int)}
+                                       not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @param bufferSize the size of the buffer to use
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="getPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Create a durable, serializable handle to the referent of the given
+ entity.
+ @param stat Referent in the target FileSystem
+ @param opt If absent, assume {@link HandleOpt#path()}.
+ @throws IllegalArgumentException If the FileStatus does not belong to
+         this FileSystem
+ @throws UnsupportedOperationException If {@link #createPathHandle}
+         not overridden by subclass.
+ @throws UnsupportedOperationException If this FileSystem cannot enforce
+         the specified constraints.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opt Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an exception will be thrown.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a path with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an {@link FSDataOutputStream} at the indicated Path
+ with write-progress reporting.
+
+ The frequency of callbacks is implementation-specific; it may be "none".
+ @param f the path of the file to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with a custom
+ checksum option.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @param checksumOpt checksum parameter. If null, the values
+        found in conf will be used.
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This create has been added to support the FileContext that processes
+ the permission with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @param f path
+ @param absolutePermission permissions
+ @return true if the directory was actually created.
+ @throws IOException IO failure
+ @see #mkdirs(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNewFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the given Path as a brand-new zero-length file.  If
+ create fails, or if it already existed, return false.
+ <i>Important: the default implementation is not atomic</i>
+ @param f path to use for create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as
+ {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+     IO_FILE_BUFFER_SIZE_DEFAULT), null)}
+ @param f the existing file to be appended.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as append(f, bufferSize, null).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @param progress for reporting progress if it is not null.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Concat existing files together.
+ @param trg the path to the target destination.
+ @param psrcs the paths to the sources to use for the concatenation.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the replication factor.
+
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @param src file name
+ @return file replication
+ @throws FileNotFoundException if the path does not resolve.
+ @throws IOException an IO failure]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the replication for an existing file.
+ If a filesystem does not support replication, it will always
+ return true: the check for a file existing may be bypassed.
+ This is the default behavior.
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful, or the feature in unsupported;
+         false if replication is supported but the file does not exist,
+         or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws IOException on failure
+ @return true if rename is successful]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+   <li>Fails if src is a file and dst is a directory.</li>
+   <li>Fails if src is a directory and dst is a file.</li>
+   <li>Fails if the parent of dst does not exist or is a file.</li>
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails
+ if the dst already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites
+ the dst if it is a file or an empty directory. Rename fails if dst is
+ a non-empty directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for
+ details. This default implementation is non atomic.
+ <p>
+ This method is deprecated since it is a temporary method added to
+ support the transition from FileSystem to FileContext for user
+ applications.
+
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws FileNotFoundException src path does not exist, or the parent
+ path of dst does not exist.
+ @throws FileAlreadyExistsException dest path exists and is a file
+ @throws ParentNotDirectoryException if the parent path of dest is not
+ a directory
+ @throws IOException on failure]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+   <li>Fails if path is a directory.</li>
+   <li>Fails if path does not exist.</li>
+   <li>Fails if path is not closed.</li>
+   <li>Fails if new size is greater than current size.</li>
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #delete(Path, boolean)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file/directory.
+ @deprecated Use {@link #delete(Path, boolean)} instead.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+
+ @param f the path to delete.
+ @param recursive if path is a directory and set to
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+ @return  true if delete is successful else false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted when its FileSystem is closed.
+ When the JVM shuts down cleanly, all cached FileSystem objects will be
+ closed automatically. These the marked paths will be deleted as a result.
+
+ If a FileSystem instance is not cached, i.e. has been created with
+ {@link #createFileSystem(URI, Configuration)}, then the paths will
+ be deleted in when {@link #close()} is called on that instance.
+
+ The path must exist in the filesystem at the time of the method call;
+ it does not have to exist at the time of JVM shutdown.
+
+ Notes
+ <ol>
+   <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+   <li>The time to shut down a FileSystem will depends on the number of
+   files to delete. For filesystems where the cost of checking
+   for the existence of a file/directory and the actual delete operation
+   (for example: object stores) is high, the time to shutdown the JVM can be
+   significantly extended by over-use of this feature.</li>
+   <li>Connectivity problems with a remote filesystem may delay shutdown
+   further, and may cause the files to not be deleted.</li>
+ </ol>
+ @param f the path to delete.
+ @return  true if deleteOnExit is successful, otherwise false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="cancelDeleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Cancel the scheduled deletion of the path when the FileSystem is closed.
+ @param f the path to cancel deletion
+ @return true if the path was found in the delete-on-exit list.]]>
+      </doc>
+    </method>
+    <method name="processDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delete all paths that were marked as delete-on-exit. This recursively
+ deletes all files and directories in the specified paths.
+
+ The time to process this operation is {@code O(paths)}, with the actual
+ time dependent on the time for existence and deletion operations to
+ complete, successfully or not.]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if a path exists.
+
+ It is highly discouraged to call this method back to back with other
+ {@link #getFileStatus(Path)} calls, as this will involve multiple redundant
+ RPC calls in HDFS.
+
+ @param f source path
+ @return true if the path exists
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a directory.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by getFileStatus() or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a regular file.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by {@link #getFileStatus(Path)} or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The number of bytes in a file.
+ @return the number of bytes; 0 for a directory
+ @deprecated Use {@link #getFileStatus(Path)} instead.
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getContentSummary" return="org.apache.hadoop.fs.ContentSummary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link ContentSummary} of a given {@link Path}.
+ @param f path to use
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="org.apache.hadoop.fs.QuotaUsage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link QuotaUsage} of a given {@link Path}.
+ @param f path to use
+ @return the quota usage
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="namespaceQuota" type="long"/>
+      <param name="storagespaceQuota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set quota for the given {@link Path}.
+
+ @param src the target path to set quota for
+ @param namespaceQuota the namespace quota (i.e., # of files/directories)
+                       to set
+ @param storagespaceQuota the storage space quota to set
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuotaByStorageType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <param name="quota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set per storage type quota for the given {@link Path}.
+
+ @param src the target path to set storage type quota for
+ @param type the storage type to set
+ @param quota the quota to set for the given storage type
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+ <p>
+ Will not return null. Expect IOException upon access error.
+ @param f given path
+ @return the statuses of the files/directories in the given patch
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List corrupted file blocks.
+ @return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given path using the user-supplied path
+ filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param f
+          a path name
+ @param filter
+          the user-supplied path filter
+ @return an array of FileStatus objects for the files under the given path
+         after applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using default
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @return a list of statuses for the files under the given paths after
+         applying the filter default Path filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using user-supplied
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @param filter
+          the user-supplied path filter
+ @return a list of statuses for the files under the given paths after
+         applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Return all the files that match filePattern and are not checksum
+ files. Results are sorted by their names.
+
+ <p>
+ A filename pattern is composed of <i>regular</i> characters and
+ <i>special pattern matching</i> characters, which are:
+
+ <dl>
+  <dd>
+   <dl>
+    <dt> <tt> ? </tt>
+    <dd> Matches any single character.
+
+    <p>
+    <dt> <tt> * </tt>
+    <dd> Matches zero or more characters.
+
+    <p>
+    <dt> <tt> [<i>abc</i>] </tt>
+    <dd> Matches a single character from character set
+     <tt>{<i>a,b,c</i>}</tt>.
+
+    <p>
+    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
+    <dd> Matches a single character from the character range
+     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
+     lexicographically less than or equal to character <tt><i>b</i></tt>.
+
+    <p>
+    <dt> <tt> [^<i>a</i>] </tt>
+    <dd> Matches a single character that is not from character set or range
+     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
+     immediately to the right of the opening bracket.
+
+    <p>
+    <dt> <tt> \<i>c</i> </tt>
+    <dd> Removes (escapes) any special meaning of character <i>c</i>.
+
+    <p>
+    <dt> <tt> {ab,cd} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
+
+    <p>
+    <dt> <tt> {ab,c{de,fh}} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
+
+   </dl>
+  </dd>
+ </dl>
+
+ @param pathPattern a glob specifying a path pattern
+
+ @return an array of paths that match the path pattern
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of {@link FileStatus} objects whose path names match
+ {@code pathPattern} and is accepted by the user-supplied path filter.
+ Results are sorted by their path names.
+
+ @param pathPattern a glob specifying the path pattern
+ @param filter a user-supplied path filter
+ @return null if {@code pathPattern} has no glob and the path does not exist
+         an empty array if {@code pathPattern} has a glob and no path
+         matches it else an array of {@link FileStatus} objects matching the
+         pattern
+ @throws IOException if any I/O error occurs when fetching file status]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ Return the file's status and block locations If the path is a file.
+
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List a directory.
+ The returned results include its block location if it is a file
+ The results are filtered by the given path filter
+ @param f a path
+ @param filter a path filter
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+ @throws FileNotFoundException if <code>f</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a remote iterator so that followup calls are made on demand
+ while consuming the entries. Each FileSystem implementation should
+ override this method and provide a more efficient implementation, if
+ possible.
+
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ @param p target path
+ @return remote iterator
+ @throws FileNotFoundException if <code>p</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses and block locations of the files in the given path.
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ <pre>
+ If the path is a directory,
+   if recursive is false, returns files in the directory;
+   if recursive is true, return files in the subtree rooted at the path.
+ If the path is a file, return the file's status and block locations.
+ </pre>
+ @param f is the path
+ @param recursive if the subdirectories need to be traversed recursively
+
+ @return an iterator that traverses statuses of the files
+
+ @throws FileNotFoundException when the path does not exist;
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this FileSystem.
+ The default implementation returns {@code "/user/$USER/"}.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given FileSystem. All relative
+ paths will be resolved relative to it.
+
+ @param new_dir Path of new working directory]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given FileSystem
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Note: with the new FileContext class, getWorkingDirectory()
+ will be removed.
+ The working directory is implemented in FileContext.
+
+ Some FileSystems like LocalFileSystem have an initial workingDir
+ that we use as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+
+ @return if there is built in notion of workingDir then it
+ is returned; else a null is returned.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ @param f path
+ @return true if the directory was created
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make the given file and all non-existent parents into
+ directories. Has roughly the semantics of Unix @{code mkdir -p}.
+ Existence of the directory hierarchy is not an error.
+ @param f path to create
+ @param permission to apply to f
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to filesystem at
+ the given dst name and the source is kept intact afterwards
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files is on the local disk.  Add it to filesystem at
+ the given dst name, removing the source afterwards.
+ @param srcs source paths
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name, removing the source afterwards.
+ @param src local path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param src path
+ @param dst path]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param srcs array of paths which are source
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from the remote filesystem to the local one.
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy a file to the local filesystem, then delete it from the
+ remote filesystem (if successfully copied).
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from a remote filesystem to the local one.
+ delSrc indicates if the src will be removed or not.
+ @param delSrc whether to delete the src
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="useRawLocalFileSystem" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under this filesystem, and the dst is on the local disk.
+ Copy it from the remote filesystem to the local dst name.
+ delSrc indicates if the src will be removed
+ or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
+ as the local file system or not. RawLocalFileSystem is non checksumming,
+ So, It will not create any crc files at local.
+
+ @param delSrc
+          whether to delete the src
+ @param src
+          path
+ @param dst
+          path
+ @param useRawLocalFileSystem
+          whether to use RawLocalFileSystem as local file system or not.
+
+ @throws IOException for any IO error]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local file that the user can write output to.  The caller
+ provides both the eventual target name in this FileSystem
+ and the local working file path.
+ If this FileSystem is local, we write directly into the target.  If
+ the FileSystem is not local, we write into the tmp local area.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path of local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.
+ A local FS will do nothing, because we've written to exactly the
+ right place.
+ A remote FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path to local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this FileSystem instance.
+ Will release any held locks, delete all files queued for deletion
+ through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ from the cache, if cached.
+
+ After this operation, the outcome of any method call on this FileSystem
+ instance, or any input/output stream created by it is <i>undefined</i>.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the block size for a particular file.
+ @param f the filename
+ @return the number of bytes in a block
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @throws FileNotFoundException if the path is not present
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultBlockSize(Path)} instead">
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.
+ @deprecated use {@link #getDefaultBlockSize(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.  The given path will be used to
+ locate the actual filesystem.  The full path does not have to exist.
+ @param f path of file
+ @return the default block size for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultReplication(Path)} instead">
+      <doc>
+      <![CDATA[Get the default replication.
+ @return the replication; the default value is "1".
+ @deprecated use {@link #getDefaultReplication(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the default replication for a path.
+ The given path will be used to locate the actual FileSystem to query.
+ The full path does not have to exist.
+ @param path of the file
+ @return default replication for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+ @return a FileStatus object
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="fixRelativePart" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[See {@link FileContext#fixRelativePart}.]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#createSymlink(Path, Path, boolean)}.]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getFileLinkStatus(Path)}.
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#supportsSymlinks()}.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, if the FS supports checksums.
+
+ @param f The file path
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, from the beginning of the file till the
+ specific length.
+ @param f The file path
+ @param length The length of the file range for checksum calculation
+ @return The file checksum or null if checksums are not supported.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param verifyChecksum Verify checksum flag]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the write checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param writeChecksum Write checksum flag]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the root partition is reflected.
+
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ @param p Path for which status should be obtained. null means
+ the default partition.
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param p The path
+ @param permission permission
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory).
+ The parameters username and groupname cannot both be null.
+ @param p The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param p The path
+ @param mtime Set the modification time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set access time.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+ @param path  The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List&lt;AclEntry&gt; describing modifications
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List describing entries to remove
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List describing modifications, which must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACL of a file or directory.
+
+ @param path Path to get
+ @return AclStatus describing the ACL of the file or directory
+ @throws IOException if an ACL could not be read
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr name and value for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List{@literal <String>} of the XAttr names of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the root directory of Trash for current user when the path specified
+ is deleted.
+
+ @param path the trash root of the path to be determined.
+ @return the default implementation returns {@code /user/$USER/.Trash}]]>
+      </doc>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+      <doc>
+      <![CDATA[Get all the trash roots for current user or all users.
+
+ @param allUsers return trash roots for all users if true.
+ @return all the trash root directories.
+         Default FileSystem returns .Trash under users' home directories if
+         {@code /user/$USER/.Trash} exists.]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The base FileSystem implementation generally has no knowledge
+ of the capabilities of actual implementations.
+ Unless it has a way to explicitly determine the capabilities,
+ this method returns false.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getFileSystemClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the FileSystem implementation class of a filesystem.
+ This triggers a scan and load of all FileSystem implementations listed as
+ services and discovered via the {@link ServiceLoader}
+ @param scheme URL scheme of FS
+ @param conf configuration: can be null, in which case the check for
+ a filesystem binding declaration in the configuration is skipped.
+ @return the filesystem
+ @throws UnsupportedFileSystemException if there was no known implementation
+         for the scheme.
+ @throws IOException if the filesystem could not be loaded]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Get the Map of Statistics object indexed by URI Scheme.
+ @return a Map having a key as URI scheme and value as Statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Return the FileSystem classes that have Statistics.
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ @param cls the class to lookup
+ @return a statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print all statistics for all file systems to {@code System.out}]]>
+      </doc>
+    </method>
+    <method name="areSymlinksEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="enableSymlinks"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStorageStatistics" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the StorageStatistics for this FileSystem object.  These statistics are
+ per-instance.  They are not shared with any other FileSystem object.
+
+ <p>This is a default method which is intended to be overridden by
+ subclasses. The default implementation returns an empty storage statistics
+ object.</p>
+
+ @return    The StorageStatistics for this FileSystem instance.
+            Will never be null.]]>
+      </doc>
+    </method>
+    <method name="getGlobalStorageStatistics" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="createDataOutputStreamBuilder" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileSystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create instance of the standard FSDataOutputStreamBuilder for the
+ given filesystem and path.
+ @param fileSystem owner
+ @param path path to create
+ @return a builder.]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a new FSDataOutputStreamBuilder for the file with path.
+ Files are overwritten by default.
+
+ @param path file path
+ @return a FSDataOutputStreamBuilder object to build the file
+
+ HADOOP-14384. Temporarily reduce the visibility of method before the
+ builder interface becomes stable.]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a Builder to append a file.
+ @param path file path.
+ @return a {@link FSDataOutputStreamBuilder} to build file append request.]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(Path, int)} unless a subclass
+ executes the open command differently.
+
+ The semantics of this call are therefore the same as that of
+ {@link #open(Path, int)} with one special point: it is in
+ {@code FSDataInputStreamBuilder.build()} in which the open operation
+ takes place -it is there where all preconditions to the operation
+ are checked.
+ @param path file path
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(PathHandle, int)} unless a subclass
+ executes the open command differently.
+
+ If PathHandles are unsupported, this may fail in the
+ {@code FSDataInputStreamBuilder.build()}  command,
+ rather than in this {@code openFile()} operation.
+ @param pathHandle path handle.
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Execute the actual open file operation.
+
+ This is invoked from {@code FSDataInputStreamBuilder.build()}
+ and from {@link DelegateToFileSystem} and is where
+ the action of opening the file should begin.
+
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)} in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param path path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Execute the actual open file operation.
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)} in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param pathHandle path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key
+ @throws UnsupportedOperationException PathHandles are not supported.
+ This may be deferred until the future is evaluated.]]>
+      </doc>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG" type="org.apache.commons.logging.Log"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This log is widely used in the org.apache.hadoop.fs code and tests,
+ so must be considered something to only be changed with care.]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileSystem shutdown hook: {@value}.]]>
+      </doc>
+    </field>
+    <field name="TRASH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for trash directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="USER_HOME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An abstract base class for a fairly generic filesystem.  It
+ may be implemented as a distributed filesystem, or as a "local"
+ one that reflects the locally-connected disk.  The local version
+ exists for small Hadoop instances and for testing.
+
+ <p>
+
+ All user code that may potentially use the Hadoop Distributed
+ File System should be written to use a FileSystem object or its
+ successor, {@link FileContext}.
+
+ <p>
+ The local implementation is {@link LocalFileSystem} and distributed
+ implementation is DistributedFileSystem. There are other implementations
+ for object stores and (outside the Apache Hadoop codebase),
+ third party filesystems.
+ <p>
+ Notes
+ <ol>
+ <li>The behaviour of the filesystem is
+ <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ specified in the Hadoop documentation. </a>
+ However, the normative specification of the behavior of this class is
+ actually HDFS: if HDFS does not behave the way these Javadocs or
+ the specification in the Hadoop documentations define, assume that
+ the documentation is incorrect.
+ </li>
+ <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ <li>The term {@code filesystem} refers to the distributed/local filesystem
+ itself, rather than the class used to interact with it.</li>
+ <li>The term "file" refers to a file in the remote filesystem,
+ rather than instances of {@code java.io.File}.</li>
+ </ol>
+
+ This is a carefully evolving class.
+ New methods may be marked as Unstable or Evolving for their initial release,
+ as a warning that they are new and may change based on the
+ experience of use in applications.
+ <p></p>
+ <b>Important note for developers</b>
+ <p></p>
+ If you are making changes here to the public API or protected methods,
+ you must review the following subclasses and make sure that
+ they are filtering/passing through new methods as appropriate.
+ <p></p>
+
+ {@link FilterFileSystem}: methods are passed through. If not,
+ then {@code TestFilterFileSystem.MustNotImplement} must be
+ updated with the unsupported interface.
+ Furthermore, if the new API's support is probed for via
+ {@link #hasPathCapability(Path, String)} then
+ {@link FilterFileSystem#hasPathCapability(Path, String)}
+ must return false, always.
+ <p></p>
+ {@link ChecksumFileSystem}: checksums are created and
+ verified.
+ <p></p>
+ {@code TestHarFileSystem} will need its {@code MustNotImplement}
+ interface updated.
+ <p></p>
+
+ There are some external places your changes will break things.
+ Do co-ordinate changes here.
+ <p></p>
+
+ HBase: HBoss
+ <p></p>
+ Hive: HiveShim23
+ {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileSystem -->
+  <!-- start class org.apache.hadoop.fs.FileUtil -->
+  <class name="FileUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileUtil"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path
+
+ @param stats
+          an array of FileStatus objects
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path.
+ If stats if null, return path
+ @param stats
+          an array of FileStatus objects
+ @param path
+          default path to return in stats is null
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <doc>
+      <![CDATA[Register all files recursively to be deleted on exit.
+ @param file File/directory to be deleted]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.
+ @param dir the file or directory to be deleted
+ @param tryGrantPermissions true if permissions should be modified to delete a file.
+ @return true on success false on failure.]]>
+      </doc>
+    </method>
+    <method name="readLink" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns the target of the given symlink. Returns the empty string if
+ the given path does not refer to a symlink or there is an error
+ accessing the symlink.
+ @param f File representing the symbolic link.
+ @return The target of the symbolic link, empty string on error or if not
+         a symlink.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.
+ @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ and all the underlying directories before trying to delete their contents.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link FileSystem#delete(Path, boolean)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recursively delete a directory.
+
+ @param fs {@link FileSystem} on which the path is present
+ @param dir directory to recursively delete
+ @throws IOException
+ @deprecated Use {@link FileSystem#delete(Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcStatus" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy local files to a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="java.io.File"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy FileSystem files to local files.]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param filename The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeSecureShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell
+ and avoids script injection attacks.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="makeCanonicalPath" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @param makeCanonicalPath
+          Whether to make canonical path for the file passed
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="getDU" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Takes an input dir and returns the du on that local directory. Very basic
+ implementation.
+
+ @param dir
+          The input dir to get the disk space of this local dir
+ @return The total disk space of the input local directory]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="toDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a stream input it will unzip the it in the unzip directory.
+ passed as the second parameter
+ @param inputStream The zip file as input
+ @param toDir The unzip directory where to unzip the zip file.
+ @throws IOException an exception occurred]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="unzipDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a File input it will unzip it in the unzip directory.
+ passed as the second parameter
+ @param inFile The zip file as input
+ @param unzipDir The unzip directory where to unzip the zip file.
+ @throws IOException An I/O exception has occurred]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="untarDir" type="java.io.File"/>
+      <param name="gzipped" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ExecutionException" type="java.util.concurrent.ExecutionException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inputStream The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @param gzipped The input stream is gzipped
+                TODO Use magic number and PusbackInputStream to identify
+ @throws IOException an exception occurred
+ @throws InterruptedException command interrupted
+ @throws ExecutionException task submit failed]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="untarDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inFile The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="symLink" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="linkname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a soft link between a src and destination
+ only on a local disk. HDFS does not support this.
+ On Windows, when symlink creation fails due to security
+ setting, we will log a warning. The return code in this
+ case is 2.
+
+ @param target the target for symlink
+ @param linkname the symlink
+ @return 0 on success]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Change the permissions on a filename.
+ @param filename the name of the file to change
+ @param perm the permission string
+ @return the exit code from the command
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Change the permissions on a file / directory, recursively, if
+ needed.
+ @param filename name of the file whose permissions are to change
+ @param perm permission string
+ @param recursive true, if permissions should be changed recursively
+ @return the exit code from the command.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the ownership on a file / directory. User name and group name
+ cannot both be null.
+ @param file the file to change
+ @param username the new user owner name
+ @param groupname the new group owner name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setReadable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="readable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setReadable(boolean)}
+ File#setReadable does not work as expected on Windows.
+ @param f input file
+ @param readable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setWritable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="writable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setWritable(boolean)}
+ File#setWritable does not work as expected on Windows.
+ @param f input file
+ @param writable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setExecutable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="executable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setExecutable(boolean)}
+ File#setExecutable does not work as expected on Windows.
+ Note: revoking execute permission on folders does not have the same
+ behavior on Windows as on Unix platforms. Creating, deleting or renaming
+ a file within that folder will still succeed on Windows.
+ @param f input file
+ @param executable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="canRead" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canRead()}
+ @param f input file
+ @return On Unix, same as {@link File#canRead()}
+         On Windows, true if process has read access on the path]]>
+      </doc>
+    </method>
+    <method name="canWrite" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canWrite()}
+ @param f input file
+ @return On Unix, same as {@link File#canWrite()}
+         On Windows, true if process has write access on the path]]>
+      </doc>
+    </method>
+    <method name="canExecute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canExecute()}
+ @param f input file
+ @return On Unix, same as {@link File#canExecute()}
+         On Windows, true if process has execute access on the path]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permissions to the required value. Uses the java primitives instead
+ of forking if group == other.
+ @param f the file to change
+ @param permission the new permissions
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createLocalTempFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="basefile" type="java.io.File"/>
+      <param name="prefix" type="java.lang.String"/>
+      <param name="isDeleteOnExit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a tmp file for a base file.
+ @param basefile the base file of the tmp
+ @param prefix file name prefix of tmp
+ @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
+ @return a newly created tmp file
+ @exception IOException If a tmp file cannot created
+ @see java.io.File#createTempFile(String, String, File)
+ @see java.io.File#deleteOnExit()]]>
+      </doc>
+    </method>
+    <method name="replaceFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="target" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the src file to the name specified by target.
+ @param src the source file
+ @param target the target file
+ @exception IOException If this operation fails]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="java.io.File[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#listFiles()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#listFiles() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of files or empty list
+ @exception IOException for invalid directory or for a bad disk.]]>
+      </doc>
+    </method>
+    <method name="list" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#list()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#list() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of file names or empty string list
+ @exception AccessDeniedException for unreadable directory
+ @exception IOException for invalid directory or for bad disk]]>
+      </doc>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="targetDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a jar file at the given path, containing a manifest with a classpath
+ that references all specified entries.
+
+ Some platforms may have an upper limit on command line length.  For example,
+ the maximum command line length on Windows is 8191 characters, but the
+ length of the classpath may exceed this.  To work around this limitation,
+ use this method to create a small intermediate jar with a manifest that
+ contains the full classpath.  It returns the absolute path to the new jar,
+ which the caller may set as the classpath for a new process.
+
+ Environment variable evaluation is not supported within a jar manifest, so
+ this method expands environment variables before inserting classpath entries
+ to the manifest.  The method parses environment variables according to
+ platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+ environment variables are case-insensitive.  For example, %VAR% and %var%
+ evaluate to the same value.
+
+ Specifying the classpath in a jar manifest does not support wildcards, so
+ this method expands wildcards internally.  Any classpath entry that ends
+ with * is translated to all files at that path with extension .jar or .JAR.
+
+ @param inputClassPath String input classpath to bundle into the jar manifest
+ @param pwd Path to working directory to save jar
+ @param targetDir path to where the jar execution will have its working dir
+ @param callerEnv Map {@literal <}String, String{@literal >} caller's
+ environment variables to use for expansion
+ @return String[] with absolute path to new jar in position 0 and
+   unexpanded wild card entry path in position 1
+ @throws IOException if there is an I/O error while writing the jar file]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+ It operates only on local paths.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist locally]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="useLocal" type="boolean"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist]]>
+      </doc>
+    </method>
+    <method name="compareFs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="destFs" type="org.apache.hadoop.fs.FileSystem"/>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param FileContext the file context with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fs the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fileContext the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <field name="SYMLINK_NO_PRIVILEGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A collection of file-processing util methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileUtil -->
+  <!-- start class org.apache.hadoop.fs.FilterFileSystem -->
+  <class name="FilterFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the raw file system 
+ @return FileSystem being filtered]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called after a new FileSystem instance is constructed.
+ @param name a uri whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make sure that a path specifies a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.  Can take place on local fs
+ or remote DFS.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files in a directory.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files and its block locations in a directory.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a remote iterator for listing in a directory]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given file system. All relative
+ paths will be resolved relative to it.
+ 
+ @param newDir]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given file system
+ 
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ delSrc indicates if the src will be removed or not.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local File that the user can write output to.  The caller
+ provides both the eventual FS target name and the local working
+ file.  If the FS is local, we write directly into the target.  If
+ the FS is remote, we write into the tmp local area.]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.  A local FS will
+ do nothing, because we've written to exactly the right place.  A remote
+ FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get file status.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="abdolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChildFileSystems" return="org.apache.hadoop.fs.FileSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="swapScheme" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A <code>FilterFileSystem</code> contains
+ some other file system, which it uses as
+ its  basic file system, possibly transforming
+ the data along the way or providing  additional
+ functionality. The class <code>FilterFileSystem</code>
+ itself simply overrides all  methods of
+ <code>FileSystem</code> with versions that
+ pass all requests to the contained  file
+ system. Subclasses of <code>FilterFileSystem</code>
+ may further override some of  these methods
+ and may also provide additional methods
+ and fields.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FilterFileSystem -->
+  <!-- start interface org.apache.hadoop.fs.FSBuilder -->
+  <interface name="FSBuilder"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set optional Builder parameter.]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set optional boolean parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set optional int parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set optional float parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set optional double parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set an array of string values as optional parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set mandatory option to the Builder.
+
+ If the option is not supported or unavailable,
+ the client should expect {@link #build()} throws IllegalArgumentException.]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set mandatory boolean option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set mandatory int option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set mandatory float option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set mandatory double option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set a string array as mandatory option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instantiate the object which was being built.
+
+ @throws IllegalArgumentException if the parameters are not valid.
+ @throws UnsupportedOperationException if the filesystem does not support
+ the specific operation.
+ @throws IOException on filesystem IO errors.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The base interface which various FileSystem FileContext Builder
+ interfaces can extend, and which underlying implementations
+ will then implement.
+ @param <S> Return type on the {@link #build()} call.
+ @param <B> type of builder itself.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FSBuilder -->
+  <!-- start interface org.apache.hadoop.fs.FsConstants -->
+  <interface name="FsConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="LOCAL_FS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FTP_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_PATH_LINKS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ViewFs: viewFs file system (ie the mount file system on client side)]]>
+      </doc>
+    </field>
+    <field name="VIEWFS_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FileSystem related constants.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FsConstants -->
+  <!-- start class org.apache.hadoop.fs.FSDataInputStream -->
+  <class name="FSDataInputStream" extends="java.io.DataInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+    <implements name="org.apache.hadoop.fs.HasFileDescriptor"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.CanSetReadahead"/>
+    <implements name="org.apache.hadoop.fs.HasEnhancedByteBufferAccess"/>
+    <implements name="org.apache.hadoop.fs.CanUnbuffer"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferPositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="FSDataInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desired" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset.
+
+ @param desired offset to seek to]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current position in the input stream.
+
+ @return current position in the input stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    maximum number of bytes to read
+ @return total number of bytes read into the buffer, or <code>-1</code>
+         if there is no more data because the end of the stream has been
+         reached]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+ Continues to read until <code>length</code> bytes have been read.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    the number of bytes to read
+ @throws IOException IO problems
+ @throws EOFException If the end of stream is reached while reading.
+                      If an exception is thrown an undetermined number
+                      of bytes in the buffer may have been written.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link #readFully(long, byte[], int, int)}.]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given position on an alternate copy of the data.
+
+ @param  targetPos  position to seek to
+ @return true if a new source is found, false otherwise]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileDescriptor" return="java.io.FileDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReadahead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <param name="opts" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="releaseBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String value. Includes the string value of the inner stream
+ @return the stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the IO Statistics of the nested stream, falling back to
+ null if the stream does not implement the interface
+ {@link IOStatisticsSource}.
+ @return an IOStatistics instance or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link FSInputStream} in a {@link DataInputStream}
+ and buffers input through a {@link java.io.BufferedInputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataInputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStream -->
+  <class name="FSDataOutputStream" extends="java.io.DataOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Syncable"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <implements name="org.apache.hadoop.fs.Abortable"/>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current position in the output stream.
+
+ @return the current position in the output stream]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the underlying output stream.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="hflush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the IO Statistics of the nested stream, falling back to
+ empty statistics if the stream does not implement the interface
+ {@link IOStatisticsSource}.
+ @return an IOStatistics instance.]]>
+      </doc>
+    </method>
+    <method name="abort" return="org.apache.hadoop.fs.Abortable.AbortableResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code abort()} on the wrapped stream if it
+ is Abortable, otherwise raise an
+ {@code UnsupportedOperationException}.
+ @throws UnsupportedOperationException if not available.
+ @return the result.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <class name="FSDataOutputStreamBuilder" extends="org.apache.hadoop.fs.impl.AbstractFSBuilderImpl"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FSDataOutputStreamBuilder" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="getThisBuilder" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the concrete implementation of the builder instance.]]>
+      </doc>
+    </method>
+    <method name="getFS" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="permission" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set permission for the file.]]>
+      </doc>
+    </method>
+    <method name="getBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="bufferSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufSize" type="int"/>
+      <doc>
+      <![CDATA[Set the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="replication" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="replica" type="short"/>
+      <doc>
+      <![CDATA[Set replication factor.]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="blockSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blkSize" type="long"/>
+      <doc>
+      <![CDATA[Set block size.]]>
+      </doc>
+    </method>
+    <method name="isRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true to create the parent directories if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="recursive" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the parent directory if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="org.apache.hadoop.util.Progressable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="progress" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prog" type="org.apache.hadoop.util.Progressable"/>
+      <doc>
+      <![CDATA[Set the facility of reporting progress.]]>
+      </doc>
+    </method>
+    <method name="getFlags" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="create" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the specified path.]]>
+      </doc>
+    </method>
+    <method name="overwrite" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="overwrite" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to overwrite the existing file.
+ Set it to false, an exception will be thrown when calling {@link #build()}
+ if the file exists.]]>
+      </doc>
+    </method>
+    <method name="append" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="checksumOpt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="chksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <doc>
+      <![CDATA[Set checksum opt.]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the FSDataOutputStream to write on the file system.
+
+ @throws IllegalArgumentException if the parameters are not valid.
+ @throws IOException on errors when file system creates or appends the file.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for {@link FSDataOutputStream} and its subclasses.
+
+ It is used to create {@link FSDataOutputStream} when creating a new file or
+ appending an existing file on {@link FileSystem}.
+
+ By default, it does not create parent directory that do not exist.
+ {@link FileSystem#createNonRecursive(Path, boolean, int, short, long,
+ Progressable)}.
+
+ To create missing parent directory, use {@link #recursive()}.
+
+ To be more generic, {@link #opt(String, int)} and {@link #must(String, int)}
+ variants provide implementation-agnostic way to customize the builder.
+ Each FS-specific builder implementation can interpret the FS-specific
+ options accordingly, for example:
+
+ <code>
+
+ // Don't
+ if (fs instanceof FooFileSystem) {
+   FooFileSystem fs = (FooFileSystem) fs;
+   OutputStream out = dfs.createFile(path)
+     .optionA()
+     .optionB("value")
+     .cache()
+   .build()
+ } else if (fs instanceof BarFileSystem) {
+   ...
+ }
+
+ // Do
+ OutputStream out = fs.createFile(path)
+   .permission(perm)
+   .bufferSize(bufSize)
+   .opt("foofs:option.a", true)
+   .opt("foofs:option.b", "value")
+   .opt("barfs:cache", true)
+   .must("foofs:cache", true)
+   .must("barfs:cache-size", 256 * 1024 * 1024)
+   .build();
+ </code>
+
+ If the option is not related to the file system, the option will be ignored.
+ If the option is must, but not supported by the file system, a
+ {@link IllegalArgumentException} will be thrown.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.FSError -->
+  <class name="FSError" extends="java.lang.Error"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Thrown for unexpected filesystem errors, presumed to reflect disk errors
+ in the native filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSError -->
+  <!-- start class org.apache.hadoop.fs.FSInputStream -->
+  <class name="FSInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <constructor name="FSInputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seeks a different copy of the data.  Returns true if 
+ found a new source, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="validatePositionedReadArgs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="EOFException" type="java.io.EOFException"/>
+      <doc>
+      <![CDATA[Validation code, available for use in subclasses.
+ @param position position: if negative an EOF exception is raised
+ @param buffer destination buffer
+ @param offset offset within the buffer
+ @param length length of bytes to read
+ @throws EOFException if the position is negative
+ @throws IndexOutOfBoundsException if there isn't space for the amount of
+ data requested.
+ @throws IllegalArgumentException other arguments are invalid.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[toString method returns the superclass toString, but if the subclass
+ implements {@link IOStatisticsSource} then those statistics are
+ extracted and included in the output.
+ That is: statistics of subclasses are automatically reported.
+ @return a string value.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSInputStream is a generic old InputStream with a little bit
+ of RAF-style seek ability.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSInputStream -->
+  <!-- start class org.apache.hadoop.fs.FsServerDefaults -->
+  <class name="FsServerDefaults" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsServerDefaults"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBytesPerChecksum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWritePacketSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEncryptDataTransfer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrashInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getChecksumType" return="org.apache.hadoop.util.DataChecksum.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyProviderUri" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultStoragePolicyId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Provides server default configuration values to clients.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsServerDefaults -->
+  <!-- start class org.apache.hadoop.fs.FsStatus -->
+  <class name="FsStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsStatus" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a FsStatus object, using the specified statistics]]>
+      </doc>
+    </constructor>
+    <method name="getCapacity" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the capacity in bytes of the file system]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of bytes used on the file system]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of remaining bytes on the file system]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent the capacity, free and used space on a
+ {@link FileSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsStatus -->
+  <!-- start interface org.apache.hadoop.fs.FutureDataInputStreamBuilder -->
+  <interface name="FutureDataInputStreamBuilder"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.FSBuilder"/>
+    <method name="build" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="withFileStatus" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[A FileStatus may be provided to the open request.
+ It is up to the implementation whether to use this or not.
+ @param status status.
+ @return the builder.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for input streams and subclasses whose return value is
+ actually a completable future: this allows for better asynchronous
+ operation.
+
+ To be more generic, {@link #opt(String, int)} and {@link #must(String, int)}
+ variants provide implementation-agnostic way to customize the builder.
+ Each FS-specific builder implementation can interpret the FS-specific
+ options accordingly, for example:
+
+ If the option is not related to the file system, the option will be ignored.
+ If the option is must, but not supported by the file system, a
+ {@link IllegalArgumentException} will be thrown.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FutureDataInputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <class name="GlobalStorageStatistics" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.GlobalStorageStatistics[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @return            The StorageStatistics object with the given name, or
+                      null if there is none.]]>
+      </doc>
+    </method>
+    <method name="put" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="provider" type="org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider"/>
+      <doc>
+      <![CDATA[Create or return the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @param provider    An object which can create a new StorageStatistics
+                      object if needed.
+ @return            The StorageStatistics object with the given name.
+ @throws RuntimeException  If the StorageStatisticsProvider provides a null
+                           object or a new StorageStatistics object with the
+                           wrong name.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator that we can use to iterate throw all the global storage
+ statistics objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stores global storage statistics objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.GlobFilter -->
+  <class name="GlobFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="GlobFilter" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern.
+
+ @param filePattern the file pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <constructor name="GlobFilter" type="java.lang.String, org.apache.hadoop.fs.PathFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern and an user filter.
+
+ @param filePattern the file pattern.
+ @param filter user filter in addition to the glob pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <method name="hasPattern" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[A filter for POSIX glob pattern with brace expansions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobFilter -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathException -->
+  <class name="InvalidPathException" extends="org.apache.hadoop.HadoopIllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.]]>
+      </doc>
+    </constructor>
+    <constructor name="InvalidPathException" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.
+ @param reason Reason <code>path</code> is invalid]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Path string is invalid either because it has invalid characters or due to
+ other file system specific reasons.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathException -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <class name="InvalidPathHandleException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathHandleException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidPathHandleException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown when the constraints enoded in a {@link PathHandle} do not hold.
+ For example, if a handle were created with the default
+ {@link Options.HandleOpt#path()} constraints, a call to
+ {@link FileSystem#open(PathHandle)} would succeed if the file were
+ modified, but if a different file was at that location then it would throw
+ this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <!-- start class org.apache.hadoop.fs.LocalFileSystem -->
+  <class name="LocalFileSystem" extends="org.apache.hadoop.fs.ChecksumFileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocalFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p>
+
+ @return <code>file</code>]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Moves files to a bad file directory on the same device, so that their
+ storage will not be reused.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the checksumed local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.LocatedFileStatus -->
+  <class name="LocatedFileStatus" extends="org.apache.hadoop.fs.FileStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocatedFileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocatedFileStatus" type="org.apache.hadoop.fs.FileStatus, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor 
+ @param stat a file status
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ 
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param hasAcl entity has associated ACLs
+ @param isEncrypted entity is encrypted
+ @param isErasureCoded entity is erasure coded
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param attr Attribute flags (See {@link FileStatus.AttrFlags}).
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <method name="getBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file's block locations
+
+ In HDFS, the returned BlockLocation will have different formats for
+ replicated and erasure coded file.
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.
+
+ @return the file's block locations]]>
+      </doc>
+    </method>
+    <method name="setBlockLocations"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="locations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <doc>
+      <![CDATA[Hook for subclasses to lazily set block locations. The {@link #locations}
+ field should be null before this is called.
+ @param locations Block locations for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class defines a FileStatus that includes a file's block locations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocatedFileStatus -->
+  <!-- start interface org.apache.hadoop.fs.MultipartUploader -->
+  <interface name="MultipartUploader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <method name="startUpload" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a multipart upload.
+ @param filePath Target path for upload.
+ @return unique identifier associating part uploads.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="putPart" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="partNumber" type="int"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="lengthInBytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Put part as part of a multipart upload.
+ It is possible to have parts uploaded in any order (or in parallel).
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param partNumber Index of the part relative to others.
+ @param filePath Target path for upload (as {@link #startUpload(Path)}).
+ @param inputStream Data for this part. Implementations MUST close this
+ stream after reading in the data.
+ @param lengthInBytes Target length to read from the stream.
+ @return unique PartHandle identifier for the uploaded part.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="complete" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <param name="handles" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Complete a multipart upload.
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param filePath Target path for upload (as {@link #startUpload(Path)}.
+ @param handles non-empty map of part number to part handle.
+          from {@link #putPart(UploadHandle, int, Path, InputStream, long)}.
+ @return unique PathHandle identifier for the uploaded file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="abort" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Aborts a multipart upload.
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param filePath Target path for upload (same as {@link #startUpload(Path)}.
+ @throws IOException IO failure
+ @return a future; the operation will have completed]]>
+      </doc>
+    </method>
+    <method name="abortUploadsUnderPath" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Best effort attempt to aborts multipart uploads under a path.
+ Not all implementations support this, and those which do may
+ be vulnerable to eventually consistent listings of current uploads
+ -some may be missed.
+ @param path path to abort uploads under.
+ @return a future to the number of entries aborted;
+ -1 if aborting is unsupported
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[MultipartUploader is an interface for copying files multipart and across
+ multiple nodes.
+ <p></p>
+ The interface extends {@link IOStatisticsSource} so that there is no
+ need to cast an instance to see if is a source of statistics.
+ However, implementations MAY return null for their actual statistics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.MultipartUploader -->
+  <!-- start class org.apache.hadoop.fs.Options -->
+  <class name="Options" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Options"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class contains options related to file system operations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Options -->
+  <!-- start class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <class name="ParentNotDirectoryException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ParentNotDirectoryException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ParentNotDirectoryException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the parent of specified Path is not a directory
+ as expected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <!-- start interface org.apache.hadoop.fs.PartHandle -->
+  <interface name="PartHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to a part id for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PartHandle -->
+  <!-- start class org.apache.hadoop.fs.PartialListing -->
+  <class name="PartialListing" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PartialListing" type="org.apache.hadoop.fs.Path, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="PartialListing" type="org.apache.hadoop.fs.Path, org.apache.hadoop.ipc.RemoteException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partial listing of the path being listed. In the case where the path is
+ a file. The list will be a singleton with the file itself.
+
+ @return Partial listing of the path being listed.
+ @throws IOException if there was an exception getting the listing.]]>
+      </doc>
+    </method>
+    <method name="getListedPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path being listed.
+
+ @return the path being listed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A partial listing of the children of a parent directory. Since it is a
+ partial listing, multiple PartialListing may need to be combined to obtain
+ the full listing of a parent directory.
+ <p/>
+ ListingBatch behaves similar to a Future, in that getting the result via
+ {@link #get()} will throw an Exception if there was a failure.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.PartialListing -->
+  <!-- start class org.apache.hadoop.fs.Path -->
+  <class name="Path" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="Path" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a path from a String.  Path strings are URIs, but with
+ unescaped elements and some additional normalization.
+
+ @param pathString the path string]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.net.URI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a path from a URI
+
+ @param aUri the source URI]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Path from components.
+
+ @param scheme the scheme
+ @param authority the authority
+ @param path the path]]>
+      </doc>
+    </constructor>
+    <method name="getPathWithoutSchemeAndAuthority" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return a version of the given Path without the scheme information.
+
+ @param path the source Path
+ @return a copy of this Path without the scheme information]]>
+      </doc>
+    </method>
+    <method name="mergePaths" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path1" type="org.apache.hadoop.fs.Path"/>
+      <param name="path2" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Merge 2 paths such that the second path is appended relative to the first.
+ The returned path has the scheme and authority of the first path.  On
+ Windows, the drive specification in the second path is discarded.
+ 
+ @param path1 the first path
+ @param path2 the second path, to be appended relative to path1
+ @return the merged path]]>
+      </doc>
+    </method>
+    <method name="isWindowsAbsolutePath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <param name="slashed" type="boolean"/>
+      <doc>
+      <![CDATA[Determine whether a given path string represents an absolute path on
+ Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+
+ @param pathString the path string to evaluate
+ @param slashed true if the given path is prefixed with "/"
+ @return true if the supplied path looks like an absolute path with a Windows
+ drive-specifier]]>
+      </doc>
+    </method>
+    <method name="toUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert this Path to a URI.
+
+ @return this Path as a URI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the FileSystem that owns this Path.
+
+ @param conf the configuration to use when resolving the FileSystem
+ @return the FileSystem that owns this Path
+ @throws java.io.IOException thrown if there's an issue resolving the
+ FileSystem]]>
+      </doc>
+    </method>
+    <method name="isAbsoluteAndSchemeAuthorityNull" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute <strong>and</strong> the scheme is null, <b>and</b> the authority
+ is null.
+
+ @return whether the path is absolute and the URI has no scheme nor
+ authority parts]]>
+      </doc>
+    </method>
+    <method name="isUriPathAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.  This method is a wrapper for {@link #isUriPathAbsolute()}.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isRoot" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if and only if this path represents the root of a file system.
+
+ @return true if and only if this path represents the root of a file system]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the final component of this path.
+
+ @return the final component of this path]]>
+      </doc>
+    </method>
+    <method name="getParent" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the parent of a path or null if at root.
+ @return the parent of a path or null if at root]]>
+      </doc>
+    </method>
+    <method name="suffix" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds a suffix to the final name in the path.
+
+ @param suffix the suffix to add
+ @return a new path with the suffix added]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="depth" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of elements in this path.
+ @return the number of elements in this path]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #makeQualified(URI, Path)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Returns a qualified path object for the {@link FileSystem}'s working
+ directory.
+  
+ @param fs the target FileSystem
+ @return a qualified path object for the FileSystem's working directory
+ @deprecated use {@link #makeQualified(URI, Path)}]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+      <doc>
+      <![CDATA[Validate the contents of a deserialized Path, so as
+ to defend against malicious object streams.
+ @throws InvalidObjectException if there's no URI]]>
+      </doc>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash.]]>
+      </doc>
+    </field>
+    <field name="SEPARATOR_CHAR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash, as a character.]]>
+      </doc>
+    </field>
+    <field name="CUR_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current directory, ".".]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the current host is a Windows machine.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Names a file or directory in a {@link FileSystem}.
+ Path strings use slash as the directory separator.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Path -->
+  <!-- start interface org.apache.hadoop.fs.PathFilter -->
+  <interface name="PathFilter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="accept" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Tests whether or not the specified abstract pathname should be
+ included in a pathname list.
+
+ @param  path  The abstract pathname to be tested
+ @return  <code>true</code> if and only if <code>pathname</code>
+          should be included]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathFilter -->
+  <!-- start interface org.apache.hadoop.fs.PathHandle -->
+  <interface name="PathHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized form in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes of this path handle.
+ @return the bytes to get to the process completing the upload.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an entity in the FileSystem. May contain
+ metadata sufficient to resolve or verify subsequent accesses independent of
+ other modifications to the FileSystem.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathHandle -->
+  <!-- start interface org.apache.hadoop.fs.PositionedReadable -->
+  <interface name="PositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read up to the specified number of bytes, from a given
+ position within a file, and return the number of bytes read. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @return actual number of bytes read; -1 means "none"
+ @throws IOException IO problems.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the specified number of bytes, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read number of bytes equal to the length of the buffer, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits positional reading.
+
+ Implementations are required to implement thread-safe operations; this may
+ be supported by concurrent access to the data, or by using a synchronization
+ mechanism to serialize access.
+
+ Not all implementations meet this requirement. Those that do not cannot
+ be used as a backing store for some applications, such as Apache HBase.
+
+ Independent of whether or not they are thread safe, some implementations
+ may make the intermediate state of the system, specifically the position
+ obtained in {@code Seekable.getPos()} visible.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PositionedReadable -->
+  <!-- start class org.apache.hadoop.fs.QuotaUsage -->
+  <class name="QuotaUsage" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QuotaUsage"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QuotaUsage" type="org.apache.hadoop.fs.QuotaUsage.Builder"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build the instance based on the builder.]]>
+      </doc>
+    </constructor>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="quota" type="long"/>
+    </method>
+    <method name="setSpaceConsumed"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceConsumed" type="long"/>
+    </method>
+    <method name="setSpaceQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceQuota" type="long"/>
+    </method>
+    <method name="getFileAndDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory count.]]>
+      </doc>
+    </method>
+    <method name="getQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory quota.]]>
+      </doc>
+    </method>
+    <method name="getSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space consumed.]]>
+      </doc>
+    </method>
+    <method name="getSpaceQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type consumed.]]>
+      </doc>
+    </method>
+    <method name="isTypeQuotaSet" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type quota has been set.]]>
+      </doc>
+    </method>
+    <method name="isTypeConsumedAvailable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type consumption information is available.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the header of the output.
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if hOption is false file sizes are returned in bytes
+ if hOption is true file sizes are returned in human readable
+
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="getTypesQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+    </method>
+    <method name="getStorageTypeHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="java.util.List"/>
+      <doc>
+      <![CDATA[return the header of with the StorageTypes.
+
+ @param storageTypes
+ @return storage header string]]>
+      </doc>
+    </method>
+    <field name="QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output format:
+ |----12----| |----15----| |----15----| |----15----| |-------18-------|
+    QUOTA   REMAINING_QUOTA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]>
+      </doc>
+    </field>
+    <field name="SPACE_QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER_FIELDS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Store the quota usage of a directory.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.QuotaUsage -->
+  <!-- start class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <class name="RawLocalFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RawLocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="useStatIfAvailable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStreamWithMode" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="handleEmptyDstDirectoryOnWindows" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="srcFile" type="java.io.File"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFile" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given path to a file or directory.
+ @param p the path to delete
+ @param recursive to delete sub-directories
+ @return true if the file or directory and all its contents were deleted
+ @throws IOException if p is non-empty and recursive is false]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+
+ (<b>Note</b>: Returned list is not sorted in any given order,
+ due to reliance on Java's {@link File#list()} API.)]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p2f" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDirWithMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="p2f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the specified directory hierarchy. Does not
+ treat existence as an error.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the working directory to the given directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsWorkingFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chown to set owner.]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chmod to set permission.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the {@link Path}'s last modified time and last access time to
+ the given valid times.
+
+ @param mtime the modification time to set (only if no less than zero).
+ @param atime the access time to set (only if no less than zero).
+ @throws IOException if setting the times fails.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opts Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a FileStatus representing the given path. If the path refers
+ to a symlink return a FileStatus representing the link rather than
+ the object the link refers to.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the raw local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.ReadOption -->
+  <class name="ReadOption" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.ReadOption[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.ReadOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Options that can be used when reading from a FileSystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ReadOption -->
+  <!-- start interface org.apache.hadoop.fs.Seekable -->
+  <interface name="Seekable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits seeking.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Seekable -->
+  <!-- start class org.apache.hadoop.fs.StorageStatistics -->
+  <class name="StorageStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StorageStatistics" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this StorageStatistics object.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the associated file system scheme if this is scheme specific,
+ else return null.]]>
+      </doc>
+    </method>
+    <method name="getLongStatistics" return="java.util.Iterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over all the currently tracked long statistics.
+
+ The values returned will depend on the type of FileSystem or FileContext
+ object.  The values do not necessarily reflect a snapshot in time.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="java.lang.Long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of a statistic.
+
+ @return         null if the statistic is not being tracked or is not a
+                 long statistic. The value of the statistic, otherwise.]]>
+      </doc>
+    </method>
+    <method name="isTracked" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return true if a statistic is being tracked.
+
+ @return         True only if the statistic is being tracked.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all the statistic data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[StorageStatistics contains statistics data for a FileSystem or FileContext
+ instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.StorageType -->
+  <class name="StorageType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="supportTypeQuota" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMovable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="asList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMovableTypes" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTypesSupportingQuota" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <field name="DEFAULT" type="org.apache.hadoop.fs.StorageType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.fs.StorageType[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines the types of supported storage media. The default storage
+ medium is assumed to be DISK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageType -->
+  <!-- start interface org.apache.hadoop.fs.StreamCapabilities -->
+  <interface name="StreamCapabilities"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasCapability" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Query the stream for a specific capability.
+
+ @param capability string to query the stream support for.
+ @return True if the stream supports capability.]]>
+      </doc>
+    </method>
+    <field name="HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hflush capability implemented by {@link Syncable#hflush()}.
+
+ Use the {@link #HSYNC} probe to check for the support of Syncable;
+ it's that presence of {@code hsync()} which matters.]]>
+      </doc>
+    </field>
+    <field name="HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hsync capability implemented by {@link Syncable#hsync()}.]]>
+      </doc>
+    </field>
+    <field name="READAHEAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setReadahead capability implemented by
+ {@link CanSetReadahead#setReadahead(Long)}.]]>
+      </doc>
+    </field>
+    <field name="DROPBEHIND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setDropBehind capability implemented by
+ {@link CanSetDropBehind#setDropBehind(Boolean)}.]]>
+      </doc>
+    </field>
+    <field name="UNBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.]]>
+      </doc>
+    </field>
+    <field name="READBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(ByteBuffer) capability implemented by
+ {@link ByteBufferReadable#read(java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <field name="PREADBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(long, ByteBuffer) capability implemented by
+ {@link ByteBufferPositionedReadable#read(long, java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <field name="IOSTATISTICS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[IOStatisticsSource API.]]>
+      </doc>
+    </field>
+    <field name="ABORTABLE_STREAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream abort() capability implemented by {@link Abortable#abort()}.
+ This matches the Path Capability
+ {@link CommonPathCapabilities#ABORTABLE_STREAM}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface to query streams for supported capabilities.
+
+ Capability strings must be in lower case.
+
+ Constant strings are chosen over enums in order to allow other file systems
+ to define their own capabilities.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.StreamCapabilities -->
+  <!-- start class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <class name="StreamCapabilitiesPolicy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StreamCapabilitiesPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Implement the policy for {@link CanUnbuffer#unbuffer()}.
+
+ @param in the input stream]]>
+      </doc>
+    </method>
+    <field name="CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Static methods to implement policies for {@link StreamCapabilities}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <!-- start interface org.apache.hadoop.fs.Syncable -->
+  <interface name="Syncable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hflush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Flush out the data in client's user buffer. After the return of
+ this call, new readers will see the data.
+ @throws IOException if any error occurs]]>
+      </doc>
+    </method>
+    <method name="hsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to posix fsync, flush out the data in client's user buffer 
+ all the way to the disk device (but the disk may have it in its cache).
+ @throws IOException if error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for flush/sync operations.
+ Consult the Hadoop filesystem specification for the definition of the
+ semantics of these operations.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Syncable -->
+  <!-- start class org.apache.hadoop.fs.Trash -->
+  <class name="Trash" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Trash" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor.
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <constructor name="Trash" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor for the FileSystem provided.
+ @param fs the FileSystem
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <method name="moveToAppropriateTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[In case of the symlinks or mount points, one has to move the appropriate
+ trashbin in the actual volume of the path p being deleted.
+
+ Hence we get the file system of the fully-qualified resolved-path and
+ then move the path p to the trashbin in that volume,
+ @param fs - the filesystem of path p
+ @param p - the  path being deleted - to be moved to trasg
+ @param conf - configuration
+ @return false if the item is already in the trash or trash is disabled
+ @throws IOException on error]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the trash is enabled for this filesystem]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="checkpoint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="expunge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="expungeImmediately"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all trash immediately.]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Provides a trash facility which supports pluggable Trash policies. 
+
+ See the implementation of the configured TrashPolicy for more
+ details.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Trash -->
+  <!-- start class org.apache.hadoop.fs.TrashPolicy -->
+  <class name="TrashPolicy" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrashPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #initialize(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used
+ @param home the home directory
+ @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations. Different from initialize(conf, fs, home), this one does
+ not assume trash always under /user/$USER due to HDFS encryption zone.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the Trash Policy is enabled for this filesystem.]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="createCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old trash checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpointsImmediately"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all checkpoints immediately, ie empty trash.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory of the Trash Policy
+ This API does not work with files deleted from encryption zone when HDFS
+ data encryption at rest feature is enabled as rename file between
+ encryption zones or encryption zone and non-encryption zone is not allowed.
+
+ The caller is recommend to use the new API
+ TrashPolicy#getCurrentTrashDir(Path path).
+ It returns the trash location correctly for the path specified no matter
+ the path is in encryption zone or not.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current trash directory for path specified based on the Trash
+ Policy
+ @param path path to be deleted
+ @return current trash directory for the path to be deleted
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @param home the home directory
+ @return an instance of TrashPolicy
+ @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @return an instance of TrashPolicy]]>
+      </doc>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trash" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="deletionInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface is used for implementing different Trash policies.
+ Provides factory method to create instances of the configured Trash policy.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.TrashPolicy -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <class name="UnsupportedFileSystemException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedFileSystemException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[File system for a given file system name/scheme is not supported]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <class name="UnsupportedMultipartUploaderException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedMultipartUploaderException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[MultipartUploader for a given file system name/scheme is not supported.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <!-- start interface org.apache.hadoop.fs.UploadHandle -->
+  <interface name="UploadHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an uploadId for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.UploadHandle -->
+  <!-- start class org.apache.hadoop.fs.XAttrCodec -->
+  <class name="XAttrCodec" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrCodec[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrCodec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="decodeValue" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode string representation of a value and check whether it's 
+ encoded. If the given string begins with 0x or 0X, it expresses
+ a hexadecimal number. If the given string begins with 0s or 0S,
+ base64 encoding is expected. If the given string is enclosed in 
+ double quotes, the inner string is treated as text. Otherwise 
+ the given string is treated as text. 
+ @param value string representation of the value.
+ @return byte[] the value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="encodeValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte[]"/>
+      <param name="encoding" type="org.apache.hadoop.fs.XAttrCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode byte[] value to string representation with encoding. 
+ Values encoded as text strings are enclosed in double quotes (\"), 
+ while strings encoded as hexadecimal and base64 are prefixed with 
+ 0x and 0s, respectively.
+ @param value byte[] value
+ @param encoding
+ @return String string representation of value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The value of <code>XAttr</code> is byte[], this class is to 
+ covert byte[] to some kind of string representation or convert back.
+ String representation is convenient for display and input. For example
+ display in screen as shell response and json response, input as http
+ or shell parameter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrCodec -->
+  <!-- start class org.apache.hadoop.fs.XAttrSetFlag -->
+  <class name="XAttrSetFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrSetFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrSetFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="xAttrName" type="java.lang.String"/>
+      <param name="xAttrExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrSetFlag -->
+  <doc>
+  <![CDATA[An abstract file system API.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.fs.audit">
+  <!-- start class org.apache.hadoop.fs.audit.CommonAuditContext -->
+  <class name="CommonAuditContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="put" return="java.util.function.Supplier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a context entry.
+ @param key key
+ @param value new value
+ @return old value or null]]>
+      </doc>
+    </method>
+    <method name="put" return="java.util.function.Supplier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.util.function.Supplier"/>
+      <doc>
+      <![CDATA[Put a context entry dynamically evaluated on demand.
+ @param key key
+ @param value new value
+ @return old value or null]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a context entry.
+ @param key key]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a context entry.
+ @param key key
+ @return value or null]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Rest the context; will set the standard options again.
+ Primarily for testing.]]>
+      </doc>
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Does the context contain a specific key?
+ @param key key
+ @return true if it is in the context.]]>
+      </doc>
+    </method>
+    <method name="currentAuditContext" return="org.apache.hadoop.fs.audit.CommonAuditContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current common audit context. Thread local.
+ @return the audit context of this thread.]]>
+      </doc>
+    </method>
+    <method name="currentThreadID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A thread ID which is unique for this process and shared across all
+ S3A clients on the same thread, even those using different FS instances.
+ @return a thread ID for reporting.]]>
+      </doc>
+    </method>
+    <method name="getEvaluatedEntries" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the evaluated operations.
+ This is the map unique to this context.
+ @return the operations map.]]>
+      </doc>
+    </method>
+    <method name="setGlobalContextEntry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set a global entry.
+ @param key key
+ @param value value]]>
+      </doc>
+    </method>
+    <method name="getGlobalContextEntry" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a global entry.
+ @param key key
+ @return value or null]]>
+      </doc>
+    </method>
+    <method name="removeGlobalContextEntry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a global entry.
+ @param key key to clear.]]>
+      </doc>
+    </method>
+    <method name="noteEntryPoint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add the entry point as a context entry with the key
+ {@link AuditConstants#PARAM_COMMAND}
+ if it has not  already been recorded.
+ This is called via ToolRunner but may be used at any
+ other entry point.
+ @param tool object loaded/being launched.]]>
+      </doc>
+    </method>
+    <method name="getGlobalContextEntries" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over the global entries.
+ Thread safe.
+ @return an iterable to enumerate the values.]]>
+      </doc>
+    </method>
+    <field name="PROCESS_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Process ID; currently built from UUID and timestamp.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The common audit context is a map of common context information
+ which can be used with any audit span.
+ This context is shared across all Filesystems within the
+ thread.
+ Audit spans will be created with a reference to the current
+ context of their thread;
+ That reference is retained even as they are moved across threads, so
+ context information (including thread ID Java runtime).
+
+ The Global context entries are a set of key-value pairs which span
+ all threads; the {@code HttpReferrerAuditHeader} picks these
+ up automatically. It is intended for minimal use of
+ shared constant values (process ID, entry point).
+
+ An attribute set in {@link #setGlobalContextEntry(String, String)}
+ will be set across all audit spans in all threads.
+
+ The {@link #noteEntryPoint(Object)} method should be
+ used in entry points (ToolRunner.run, etc). It extracts
+ the final element of the classname and attaches that
+ to the global context with the attribute key
+ {@link AuditConstants#PARAM_COMMAND}, if not already
+ set.
+ This helps identify the application being executued.
+
+ All other values set are specific to this context, which
+ is thread local.
+ The attributes which can be added to ths common context include
+ evaluator methods which will be evaluated in whichever thread
+ invokes {@link #getEvaluatedEntries()} and then evaluates them.
+ That map of evaluated options may evaluated later, in a different
+ thread.
+
+ For setting and clearing thread-level options, use
+ {@link #currentAuditContext()} to get the thread-local
+ context for the caller, which can then be manipulated.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.audit.CommonAuditContext -->
+</package>
+<package name="org.apache.hadoop.fs.crypto">
+</package>
+<package name="org.apache.hadoop.fs.ftp">
+  <!-- start class org.apache.hadoop.fs.ftp.FTPException -->
+  <class name="FTPException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A class to wrap a {@link Throwable} into a Runtime Exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPException -->
+  <!-- start class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+  <class name="FTPFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p>
+
+ @return <code>ftp</code>]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FTPFileSystem.
+
+ @return the default port]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A stream obtained via this call must be closed before using other APIs of
+ this class or else the invocation will block.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This optional operation is not yet supported.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BLOCK_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST_PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_PASSWORD_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_DATA_CONNECTION_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TRANSFER_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="E_SAME_DIRECTORY_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ A {@link FileSystem} backed by an FTP client provided by <a
+ href="http://commons.apache.org/net/">Apache Commons Net</a>.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+</package>
+<package name="org.apache.hadoop.fs.http">
+</package>
+<package name="org.apache.hadoop.fs.protocolPB">
+</package>
+<package name="org.apache.hadoop.fs.sftp">
+</package>
+<package name="org.apache.hadoop.fs.shell.find">
+</package>
+<package name="org.apache.hadoop.fs.statistics">
+  <!-- start class org.apache.hadoop.fs.statistics.DurationStatisticSummary -->
+  <class name="DurationStatisticSummary" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="DurationStatisticSummary" type="java.lang.String, boolean, long, long, long, org.apache.hadoop.fs.statistics.MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ @param key Statistic key.
+ @param success Are these success or failure statistics.
+ @param count Count of operation invocations.
+ @param max Max duration; -1 if unknown.
+ @param min Min duration; -1 if unknown.
+ @param mean Mean duration -may be null. (will be cloned)]]>
+      </doc>
+    </constructor>
+    <method name="getKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSuccess" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMax" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMin" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMean" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fetchDurationSummary" return="org.apache.hadoop.fs.statistics.DurationStatisticSummary"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <param name="key" type="java.lang.String"/>
+      <param name="success" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch the duration timing summary of success or failure operations
+ from an IO Statistics source.
+ If the duration key is unknown, the summary will be incomplete.
+ @param source source of data
+ @param key duration statistic key
+ @param success fetch success statistics, or if false, failure stats.
+ @return a summary of the statistics.]]>
+      </doc>
+    </method>
+    <method name="fetchSuccessSummary" return="org.apache.hadoop.fs.statistics.DurationStatisticSummary"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fetch the duration timing summary from an IOStatistics source.
+ If the duration key is unknown, the summary will be incomplete.
+ @param source source of data
+ @param key duration statistic key
+ @return a summary of the statistics.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Summary of duration tracking statistics
+ as extracted from an IOStatistics instance.
+ <p>
+ This is for reporting and testing.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.DurationStatisticSummary -->
+  <!-- start interface org.apache.hadoop.fs.statistics.IOStatistics -->
+  <interface name="IOStatistics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="counters" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of counters.
+ @return the current map of counters.]]>
+      </doc>
+    </method>
+    <method name="gauges" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of gauges.
+ @return the current map of gauges.]]>
+      </doc>
+    </method>
+    <method name="minimums" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of minimums.
+ @return the current map of minimums.]]>
+      </doc>
+    </method>
+    <method name="maximums" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of maximums.
+ @return the current map of maximums.]]>
+      </doc>
+    </method>
+    <method name="meanStatistics" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of meanStatistics.
+ @return the current map of MeanStatistic statistics.]]>
+      </doc>
+    </method>
+    <field name="MIN_UNSET_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value when a minimum value has never been set.]]>
+      </doc>
+    </field>
+    <field name="MAX_UNSET_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value when a max value has never been set.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[IO Statistics.
+ <p>
+ These are low-cost per-instance statistics provided by any Hadoop
+ I/O class instance.
+ <p>
+ Consult the filesystem specification document for the requirements
+ of an implementation of this interface.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.statistics.IOStatistics -->
+  <!-- start interface org.apache.hadoop.fs.statistics.IOStatisticsAggregator -->
+  <interface name="IOStatisticsAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="aggregate" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Aggregate the supplied statistics into the current
+ set.
+
+ @param statistics statistics; may be null
+ @return true if the statistics reference was not null and
+ so aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface exported by classes which support
+ aggregation of {@link IOStatistics}.
+ Implementations MAY aggregate all statistics
+ exported by the IOStatistics reference passed in to
+ {@link #aggregate(IOStatistics)}, or they
+ may selectively aggregate specific values/classes
+ of statistics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.statistics.IOStatisticsAggregator -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsLogging -->
+  <class name="IOStatisticsLogging" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="ioStatisticsSourceToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract the statistics from a source object -or ""
+ if it is not an instance of {@link IOStatistics},
+ {@link IOStatisticsSource} or the retrieved
+ statistics are null.
+ <p>
+ Exceptions are caught and downgraded to debug logging.
+ @param source source of statistics.
+ @return a string for logging.]]>
+      </doc>
+    </method>
+    <method name="ioStatisticsToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Convert IOStatistics to a string form.
+ @param statistics A statistics instance.
+ @return string value or the empty string if null]]>
+      </doc>
+    </method>
+    <method name="ioStatisticsToPrettyString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Convert IOStatistics to a string form, with all the metrics sorted
+ and empty value stripped.
+ This is more expensive than the simple conversion, so should only
+ be used for logging/output where it's known/highly likely that the
+ caller wants to see the values. Not for debug logging.
+ @param statistics A statistics instance.
+ @return string value or the empty string if null]]>
+      </doc>
+    </method>
+    <method name="demandStringifyIOStatisticsSource" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+      <doc>
+      <![CDATA[On demand stringifier of an IOStatisticsSource instance.
+ <p>
+ Whenever this object's toString() method is called, it evaluates the
+ statistics.
+ <p>
+ This is designed to affordable to use in log statements.
+ @param source source of statistics -may be null.
+ @return an object whose toString() operation returns the current values.]]>
+      </doc>
+    </method>
+    <method name="demandStringifyIOStatistics" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[On demand stringifier of an IOStatistics instance.
+ <p>
+ Whenever this object's toString() method is called, it evaluates the
+ statistics.
+ <p>
+ This is for use in log statements where for the cost of creation
+ of this entry is low; it is affordable to use in log statements.
+ @param statistics statistics to stringify -may be null.
+ @return an object whose toString() operation returns the current values.]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtDebug"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="message" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract any statistics from the source and log at debug, if
+ the log is set to log at debug.
+ No-op if logging is not at debug or the source is null/of
+ the wrong type/doesn't provide statistics.
+ @param log log to log to
+ @param message message for log -this must contain "{}" for the
+ statistics report to actually get logged.
+ @param source source object]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtDebug"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract any statistics from the source and log to
+ this class's log at debug, if
+ the log is set to log at debug.
+ No-op if logging is not at debug or the source is null/of
+ the wrong type/doesn't provide statistics.
+ @param message message for log -this must contain "{}" for the
+ statistics report to actually get logged.
+ @param source source object]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtLevel"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="level" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[A method to log IOStatistics from a source at different levels.
+
+ @param log    Logger for logging.
+ @param level  LOG level.
+ @param source Source to LOG.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility operations convert IO Statistics sources/instances
+ to strings, especially for robustly logging.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsLogging -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsSnapshot -->
+  <class name="IOStatisticsSnapshot" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.statistics.IOStatistics"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsAggregator"/>
+    <constructor name="IOStatisticsSnapshot"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct.]]>
+      </doc>
+    </constructor>
+    <constructor name="IOStatisticsSnapshot" type="org.apache.hadoop.fs.statistics.IOStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct, taking a snapshot of the source statistics data
+ if the source is non-null.
+ If the source is null, the empty maps are created
+ @param source statistics source. Nullable.]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear all the maps.]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Take a snapshot.
+
+ This completely overwrites the map data with the statistics
+ from the source.
+ @param source statistics source.]]>
+      </doc>
+    </method>
+    <method name="aggregate" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Aggregate the current statistics with the
+ source reference passed in.
+
+ The operation is synchronized.
+ @param source source; may be null
+ @return true if a merge took place.]]>
+      </doc>
+    </method>
+    <method name="counters" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="gauges" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="minimums" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="maximums" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="meanStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serializer" return="org.apache.hadoop.util.JsonSerialization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a JSON serializer for this class.
+ @return a serializer.]]>
+      </doc>
+    </method>
+    <method name="requiredSerializationClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[What classes are needed to deserialize this class?
+ Needed to securely unmarshall this from untrusted sources.
+ @return a list of required classes to deserialize the data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Snapshot of statistics from a different source.
+ <p>
+ It is serializable so that frameworks which can use java serialization
+ to propagate data (Spark, Flink...) can send the statistics
+ back. For this reason, TreeMaps are explicitly used as field types,
+ even though IDEs can recommend use of Map instead.
+ For security reasons, untrusted java object streams should never be
+ deserialized. If for some reason this is required, use
+ {@link #requiredSerializationClasses()} to get the list of classes
+ used when deserializing instances of this object.
+ <p>
+ <p>
+ It is annotated for correct serializations with jackson2.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsSnapshot -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsSupport -->
+  <class name="IOStatisticsSupport" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="snapshotIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatisticsSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Take a snapshot of the current statistics state.
+ <p>
+ This is not an atomic option.
+ <p>
+ The instance can be serialized, and its
+ {@code toString()} method lists all the values.
+ @param statistics statistics
+ @return a snapshot of the current values.]]>
+      </doc>
+    </method>
+    <method name="snapshotIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatisticsSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a snapshot statistics instance ready to aggregate data.
+
+ The instance can be serialized, and its
+ {@code toString()} method lists all the values.
+ @return an empty snapshot]]>
+      </doc>
+    </method>
+    <method name="retrieveIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Get the IOStatistics of the source, casting it
+ if it is of the relevant type, otherwise,
+ if it implements {@link IOStatisticsSource}
+ extracting the value.
+
+ Returns null if the source isn't of the write type
+ or the return value of
+ {@link IOStatisticsSource#getIOStatistics()} was null.
+ @return an IOStatistics instance or null]]>
+      </doc>
+    </method>
+    <method name="stubDurationTrackerFactory" return="org.apache.hadoop.fs.statistics.DurationTrackerFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a stub duration tracker factory whose returned trackers
+ are always no-ops.
+
+ As singletons are returned, this is very low-cost to use.
+ @return a duration tracker factory.]]>
+      </doc>
+    </method>
+    <method name="stubDurationTracker" return="org.apache.hadoop.fs.statistics.DurationTracker"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a stub duration tracker.
+ @return a stub tracker.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Support for working with IOStatistics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsSupport -->
+  <!-- start class org.apache.hadoop.fs.statistics.MeanStatistic -->
+  <class name="MeanStatistic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="MeanStatistic" type="long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, with some resilience against invalid sample counts.
+ If the sample count is 0 or less, the sum is set to 0 and
+ the sample count to 0.
+ @param samples sample count.
+ @param sum sum value]]>
+      </doc>
+    </constructor>
+    <constructor name="MeanStatistic" type="org.apache.hadoop.fs.statistics.MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create from another statistic.
+ @param that source]]>
+      </doc>
+    </constructor>
+    <constructor name="MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty statistic.]]>
+      </doc>
+    </constructor>
+    <method name="getSum" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the sum of samples.
+ @return the sum]]>
+      </doc>
+    </method>
+    <method name="getSamples" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the sample count.
+ @return the sample count; 0 means empty]]>
+      </doc>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is a statistic empty?
+ @return true if the sample count is 0]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the values to 0.]]>
+      </doc>
+    </method>
+    <method name="setSamplesAndSum"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sampleCount" type="long"/>
+      <param name="newSum" type="long"/>
+      <doc>
+      <![CDATA[Set the sum and samples.
+ Synchronized.
+ @param sampleCount new sample count.
+ @param newSum new sum]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.fs.statistics.MeanStatistic"/>
+      <doc>
+      <![CDATA[Set the statistic to the values of another.
+ Synchronized.
+ @param other the source.]]>
+      </doc>
+    </method>
+    <method name="setSum"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Set the sum.
+ @param sum new sum]]>
+      </doc>
+    </method>
+    <method name="setSamples"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="samples" type="long"/>
+      <doc>
+      <![CDATA[Set the sample count.
+
+ If this is less than zero, it is set to zero.
+ This stops an ill-formed JSON entry from
+ breaking deserialization, or get an invalid sample count
+ into an entry.
+ @param samples sample count.]]>
+      </doc>
+    </method>
+    <method name="mean" return="double"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arithmetic mean value.
+ @return the mean]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.fs.statistics.MeanStatistic"/>
+      <doc>
+      <![CDATA[Add another MeanStatistic.
+ @param other other value]]>
+      </doc>
+    </method>
+    <method name="addSample"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a sample.
+ Thread safe.
+ @param value value to add to the sum]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The hash code is derived from the mean
+ and sample count: if either is changed
+ the statistic cannot be used as a key
+ for hash tables/maps.
+ @return a hash value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="copy" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of this instance.
+ @return copy.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mean statistic represented as the sum and the sample count;
+ the mean is calculated on demand.
+ <p>
+ It can be used to accrue values so as to dynamically update
+ the mean. If so, know that there is no synchronization
+ on the methods.
+ </p>
+ <p>
+ If a statistic has 0 samples then it is considered to be empty.
+ </p>
+ <p>
+ All 'empty' statistics are equivalent, independent of the sum value.
+ </p>
+ <p>
+ For non-empty statistics, sum and sample values must match
+ for equality.
+ </p>
+ <p>
+ It is serializable and annotated for correct serializations with jackson2.
+ </p>
+ <p>
+ Thread safety. The operations to add/copy sample data, are thread safe.
+ </p>
+ <ol>
+   <li>{@link #add(MeanStatistic)}</li>
+   <li>{@link #addSample(long)} </li>
+   <li>{@link #clear()} </li>
+   <li>{@link #setSamplesAndSum(long, long)}</li>
+   <li>{@link #set(MeanStatistic)}</li>
+   <li>{@link #setSamples(long)} and {@link #setSum(long)}</li>
+ </ol>
+ <p>
+ So is the {@link #mean()} method. This ensures that when
+ used to aggregated statistics, the aggregate value and sample
+ count are set and evaluated consistently.
+ </p>
+ <p>
+   Other methods marked as synchronized because Findbugs overreacts
+   to the idea that some operations to update sum and sample count
+   are synchronized, but that things like equals are not.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.MeanStatistic -->
+  <!-- start class org.apache.hadoop.fs.statistics.StoreStatisticNames -->
+  <class name="StoreStatisticNames" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="OP_ABORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_ACCESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[access() API call {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_COPY_FROM_LOCAL_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_CREATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_CREATE_NON_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_DELETE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_EXISTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_CONTENT_SUMMARY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_DELEGATION_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_FILE_CHECKSUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_FILE_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GLOB_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_IS_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_IS_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_LOCATED_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_MKDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_MODIFY_ACL_ENTRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_OPEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_ACL_ENTRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_DEFAULT_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_RENAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_OWNER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_PERMISSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_TIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_TRUNCATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttrs(Path path)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_NAMED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttr(Path, String)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_NAMED_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttrs(Path path, List<String> names)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code listXAttrs(Path path)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="DELEGATION_TOKENS_ISSUED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_EXISTS_PROBE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Probe for store existing: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_THROTTLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests throttled and retried: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests made of a store: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[IO retried: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_LIST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A store's equivalent of a paged LIST request was initiated: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_CONTINUE_LIST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of continued object listings made.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_BULK_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A bulk DELETE request was made: {@value}.
+ A separate statistic from {@link #OBJECT_DELETE_REQUEST}
+ so that metrics on duration of the operations can
+ be distinguished.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A store's equivalent of a DELETE request was made: {@value}.
+ This may be an HTTP DELETE verb, or it may be some custom
+ operation which takes a list of objects to delete.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_DELETE_OBJECTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The count of objects deleted in delete requests.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_MULTIPART_UPLOAD_INITIATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object multipart upload initiated.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_MULTIPART_UPLOAD_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object multipart upload aborted.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object put/multipart upload count.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST_COMPLETED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object put/multipart upload completed count.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST_ACTIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Current number of active put requests.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of bytes uploaded.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_BYTES_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of bytes queued for upload/being actively uploaded.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_SELECT_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of S3 Select (or similar) requests issued.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a minimum value when
+ the same key is shared across min/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a maximum value when
+ the same key is shared across max/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MEAN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a mean statistic value when
+ the same key is shared across mean/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_FAILURES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String to add to counters and other stats to track failures.
+ This comes before the .min/.mean//max suffixes.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_EXECUTOR_ACQUIRED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the statistic collected for executor acquisition if
+ a duration tracker factory is passed in to the constructor.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_HEAD_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP HEAD request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_GET_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP GET request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP DELETE request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP PUT request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_PATCH_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP PATCH request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_POST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP POST request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_METADATA_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP HEAD request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_COPY_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STORE_IO_THROTTLE_RATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_INSTANTIATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_PART_PUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_PART_PUT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_COMPLETED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_STARTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Common statistic names for object store operations..
+ <p>
+ When adding new common statistic name constants, please make them unique.
+ By convention:
+ </p>
+ <ul>
+   <li>the name of the constants are uppercase, words separated by
+   underscores.</li>
+   <li>the value of the constants are lowercase of the constant names.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.StoreStatisticNames -->
+  <!-- start class org.apache.hadoop.fs.statistics.StreamStatisticNames -->
+  <class name="StreamStatisticNames" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="STREAM_READ_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the TCP stream was aborted.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Bytes read from an input stream in read() calls.
+ Does not include bytes read and then discarded in seek/close etc.
+ These are the bytes returned to the caller.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES_DISCARDED_ABORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes discarded by aborting an input stream .
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES_DISCARDED_CLOSE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes read and discarded when closing an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_CLOSED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the TCP stream was closed.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_CLOSE_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of times an attempt to close an input stream was made
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPENED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of times an input stream to was opened.
+ For object stores, that means the count a GET request was initiated.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_EXCEPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of exceptions raised during input stream reads.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_FULLY_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of readFully() operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of read() operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPERATIONS_INCOMPLETE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of incomplete read() operations in an input stream,
+ that is, when the bytes returned were less than that requested.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_VERSION_MISMATCHES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of version mismatches encountered while reading an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BACKWARD_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of executed seek operations which went backwards in a stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_BACKWARDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes moved backwards during seek operations
+ in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_DISCARDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes read and discarded during seek() in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_SKIPPED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes skipped during forward seek operations.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_FORWARD_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of executed seek operations which went forward in
+ an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_POLICY_CHANGED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the seek policy was dynamically changed
+ in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of seek operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SKIP_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of {@code InputStream.skip()} calls.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SKIP_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count bytes skipped in {@code InputStream.skip()} calls.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_TOTAL_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of bytes read from an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_UNBUFFERED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of calls of {@code CanUnbuffer.unbuffer()}.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_EXCEPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA["Count of stream write failures reported.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_EXCEPTIONS_COMPLETING_UPLOADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of failures when finalizing a multipart upload:
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of block/partition uploads complete.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of number of block uploads aborted.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_ACTIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of block/partition uploads active.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gauge of data queued to be written.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_COMMITTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of number of block uploads committed.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gauge of block/partitions uploads queued to be written.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA["Count of bytes written to output stream including all not yet uploaded.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_TOTAL_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of total time taken for uploads to complete.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_QUEUE_DURATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total queue duration of all block uploads.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_TOTAL_DATA" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BYTES_TO_UPLOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes to upload from an OutputStream.]]>
+      </doc>
+    </field>
+    <field name="BYTES_UPLOAD_SUCCESSFUL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes uploaded successfully to the object store.]]>
+      </doc>
+    </field>
+    <field name="BYTES_UPLOAD_FAILED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes failed to upload to the object store.]]>
+      </doc>
+    </field>
+    <field name="TIME_SPENT_ON_TASK_WAIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total time spent on waiting for a task to complete.]]>
+      </doc>
+    </field>
+    <field name="QUEUE_SHRUNK_OPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of task queue shrunk operations.]]>
+      </doc>
+    </field>
+    <field name="WRITE_CURRENT_BUFFER_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of times current buffer is written to the service.]]>
+      </doc>
+    </field>
+    <field name="TIME_SPENT_ON_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total time spent on completing a PUT request.]]>
+      </doc>
+    </field>
+    <field name="SEEK_IN_BUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seeks in buffer.]]>
+      </doc>
+    </field>
+    <field name="BYTES_READ_BUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes read from the buffer.]]>
+      </doc>
+    </field>
+    <field name="REMOTE_READ_OP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of remote read operations performed.]]>
+      </doc>
+    </field>
+    <field name="READ_AHEAD_BYTES_READ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of bytes read from readAhead.]]>
+      </doc>
+    </field>
+    <field name="REMOTE_BYTES_READ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of bytes read from remote operations.]]>
+      </doc>
+    </field>
+    <field name="BLOCKS_ALLOCATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of Data blocks allocated by an outputStream.]]>
+      </doc>
+    </field>
+    <field name="BLOCKS_RELEASED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of Data blocks released by an outputStream.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[These are common statistic names.
+ <p>
+ When adding new common statistic name constants, please make them unique.
+ By convention, they are implicitly unique:
+ <ul>
+   <li>
+     The name of the constants are uppercase, words separated by
+     underscores.
+   </li>
+   <li>
+     The value of the constants are lowercase of the constant names.
+   </li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.StreamStatisticNames -->
+</package>
+<package name="org.apache.hadoop.ha">
+  <!-- start class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <class name="BadFencingConfigurationException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BadFencingConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BadFencingConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the operator has specified an invalid configuration
+ for fencing methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <!-- start class org.apache.hadoop.ha.FailoverFailedException -->
+  <class name="FailoverFailedException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailoverFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FailoverFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate service failover has failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.FailoverFailedException -->
+  <!-- start interface org.apache.hadoop.ha.FenceMethod -->
+  <interface name="FenceMethod"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="checkArgs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Verify that the given fencing method's arguments are valid.
+ @param args the arguments provided in the configuration. This may
+        be null if the operator did not configure any arguments.
+ @throws BadFencingConfigurationException if the arguments are invalid]]>
+      </doc>
+    </method>
+    <method name="tryFence" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.ha.HAServiceTarget"/>
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Attempt to fence the target node.
+ @param target the target of the service to fence
+ @param args the configured arguments, which were checked at startup by
+             {@link #checkArgs(String)}
+ @return true if fencing was successful, false if unsuccessful or
+              indeterminate
+ @throws BadFencingConfigurationException if the configuration was
+         determined to be invalid only at runtime]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A fencing method is a method by which one node can forcibly prevent
+ another node from making continued progress. This might be implemented
+ by killing a process on the other node, by denying the other node's
+ access to shared storage, or by accessing a PDU to cut the other node's
+ power.
+ <p>
+ Since these methods are often vendor- or device-specific, operators
+ may implement this interface in order to achieve fencing.
+ <p>
+ Fencing is configured by the operator as an ordered list of methods to
+ attempt. Each method will be tried in turn, and the next in the list
+ will only be attempted if the previous one fails. See {@link NodeFencer}
+ for more information.
+ <p>
+ If an implementation also implements {@link Configurable} then its
+ <code>setConf</code> method will be called upon instantiation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.FenceMethod -->
+  <!-- start interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <interface name="HAServiceProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="monitorHealth"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="HealthCheckFailedException" type="org.apache.hadoop.ha.HealthCheckFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Monitor the health of service. This periodically called by the HA
+ frameworks to monitor the health of the service.
+ 
+ Service is expected to perform checks to ensure it is functional.
+ If the service is not healthy due to failure or partial failure,
+ it is expected to throw {@link HealthCheckFailedException}.
+ The definition of service not healthy is left to the service.
+ 
+ Note that when health check of an Active service fails,
+ failover to standby may be done.
+ 
+ @throws HealthCheckFailedException
+           if the health check of a service fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToActive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to active state. No operation, if the
+ service is already in active state.
+ 
+ @throws ServiceFailedException
+           if transition from standby to active fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToStandby"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to standby state. No operation, if the
+ service is already in standby state.
+ 
+ @throws ServiceFailedException
+           if transition from active to standby fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToObserver"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to observer state. No operation, if the
+ service is already in observer state.
+
+ @throws ServiceFailedException
+           if transition from standby to observer fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="getServiceStatus" return="org.apache.hadoop.ha.HAServiceStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current status of the service. The status indicates
+ the current <em>state</em> (e.g ACTIVE/STANDBY) as well as
+ some additional information.
+ 
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen
+ @see HAServiceStatus]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial version of the protocol]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides High Availability related primitives to
+ monitor and fail-over the service.
+ 
+ This interface could be used by HA frameworks to manage the service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <!-- start class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <class name="HAServiceProtocolHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceProtocolHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="monitorHealth"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToActive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToStandby"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToObserver"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Helper for making {@link HAServiceProtocol} RPC calls. This helper
+ unwraps the {@link RemoteException} to specific exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <!-- start class org.apache.hadoop.ha.HAServiceTarget -->
+  <class name="HAServiceTarget" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceTarget"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the target node.]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an optional separate RPC server address for health checks at the
+ target node.  If defined, then this address is used by the health monitor
+ for the {@link HAServiceProtocol#monitorHealth()} and
+ {@link HAServiceProtocol#getServiceStatus()} calls.  This can be useful for
+ separating out these calls onto separate RPC handlers to protect against
+ resource exhaustion in the main RPC handler pool.  If null (which is the
+ default implementation), then all RPC calls go to the address defined by
+ {@link #getAddress()}.
+
+ @return IPC address of the lifeline RPC server on the target node, or null
+     if no lifeline RPC server is used]]>
+      </doc>
+    </method>
+    <method name="getZKFCAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the ZKFC on the target node]]>
+      </doc>
+    </method>
+    <method name="getFencer" return="org.apache.hadoop.ha.NodeFencer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a Fencer implementation configured for this target node]]>
+      </doc>
+    </method>
+    <method name="checkFencingConfigured"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[@throws BadFencingConfigurationException if the fencing configuration
+ appears to be invalid. This is divorced from the above
+ {@link #getFencer()} method so that the configuration can be checked
+ during the pre-flight phase of failover.]]>
+      </doc>
+    </method>
+    <method name="getProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to connect to the target HA Service.]]>
+      </doc>
+    </method>
+    <method name="setTransitionTargetHAStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"/>
+    </method>
+    <method name="getTransitionTargetHAStatus" return="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a proxy to connect to the target HA service for health monitoring.
+ If {@link #getHealthMonitorAddress()} is implemented to return a non-null
+ address, then this proxy will connect to that address.  Otherwise, the
+ returned proxy defaults to using {@link #getAddress()}, which means this
+ method's behavior is identical to {@link #getProxy(Configuration, int)}.
+
+ @param conf Configuration
+ @param timeoutMs timeout in milliseconds
+ @return a proxy to connect to the target HA service for health monitoring
+ @throws IOException if there is an error]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <param name="retries" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getZKFCProxy" return="org.apache.hadoop.ha.ZKFCProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to the ZKFC which is associated with this HA service.]]>
+      </doc>
+    </method>
+    <method name="getFencingParameters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addFencingParameters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ret" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Hook to allow subclasses to add any parameters they would like to
+ expose to fencing implementations/scripts. Fencing methods are free
+ to use this map as they see fit -- notably, the shell script
+ implementation takes each entry, prepends 'target_', substitutes
+ '_' for '.', and adds it to the environment of the script.
+
+ Subclass implementations should be sure to delegate to the superclass
+ implementation as well as adding their own keys.
+
+ @param ret map which can be mutated to pass parameters to the fencer]]>
+      </doc>
+    </method>
+    <method name="isAutoFailoverEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if auto failover should be considered enabled]]>
+      </doc>
+    </method>
+    <method name="supportObserver" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this target supports the Observer state, false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents a target of the client side HA administration commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceTarget -->
+  <!-- start class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <class name="HealthCheckFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HealthCheckFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="HealthCheckFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that health check of a service failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <!-- start class org.apache.hadoop.ha.ServiceFailedException -->
+  <class name="ServiceFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ServiceFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that an operation performed
+ to modify the state of a service or application failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.ServiceFailedException -->
+</package>
+<package name="org.apache.hadoop.ha.protocolPB">
+  <!-- start interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <interface name="HAServiceProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <!-- start interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+  <interface name="ZKFCProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.ZKFCProtocolProtos.ZKFCProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+</package>
+<package name="org.apache.hadoop.http.lib">
+  <doc>
+  <![CDATA[This package provides user-selectable (via configuration) classes that add
+functionality to the web UI. They are configured as a list of classes in the
+configuration parameter <b>hadoop.http.filter.initializers</b>.
+
+<ul>
+<li> <b>StaticUserWebFilter</b> - An authorization plugin that makes all
+users a static configured user.
+</ul>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io">
+  <!-- start class org.apache.hadoop.io.AbstractMapWritable -->
+  <class name="AbstractMapWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractMapWritable"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addToMap"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a Class to the maps if it is not already present.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="byte"/>
+      <doc>
+      <![CDATA[@return the Class class for the specified id]]>
+      </doc>
+    </method>
+    <method name="getId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[@return the id for the specified Class]]>
+      </doc>
+    </method>
+    <method name="copy"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Used by child copy constructors.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the conf]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[@param conf the conf to set]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Abstract base class for MapWritable and SortedMapWritable
+ 
+ Unlike org.apache.nutch.crawl.MapWritable, this class allows creation of
+ MapWritable&lt;Writable, MapWritable&gt; so the CLASS_TO_ID and ID_TO_CLASS
+ maps travel with the class instead of being static.
+ 
+ Class ids range from 1 to 127 so there can be at most 127 distinct classes
+ in any specific map instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.AbstractMapWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayFile -->
+  <class name="ArrayFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ArrayFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A dense file-based mapping from integers to values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayFile -->
+  <!-- start class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <class name="ArrayPrimitiveWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayPrimitiveWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an empty instance, for use during Writable read]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance of known type but no value yet
+ for use with type-specific wrapper classes]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Wrap an existing array of primitives
+ @param value - array of primitives]]>
+      </doc>
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original array.  
+ Client must cast it back to type componentType[]
+ (or may use type-specific wrapper classes).
+ @return - original array as Object]]>
+      </doc>
+    </method>
+    <method name="getComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDeclaredComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDeclaredComponentType" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="componentType" type="java.lang.Class"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This is a wrapper class.  It wraps a Writable implementation around
+ an array of primitives (e.g., int[], long[], etc.), with optimized 
+ wire format, and without creating new objects per element.
+ 
+ This is a wrapper class only; it does not make a copy of the 
+ underlying array.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayWritable -->
+  <class name="ArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable for arrays containing instances of a class. The elements of this
+ writable must all be instances of the same class. If this writable will be
+ the input for a Reducer, you will need to create a subclass that sets the
+ value to be of the proper type.
+
+ For example:
+ <code>
+ public class IntArrayWritable extends ArrayWritable {
+   public IntArrayWritable() { 
+     super(IntWritable.class); 
+   }	
+ }
+ </code>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayWritable -->
+  <!-- start class org.apache.hadoop.io.BinaryComparable -->
+  <class name="BinaryComparable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="BinaryComparable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return n st bytes 0..n-1 from {#getBytes()} are valid.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return representative byte array for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.BinaryComparable"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#compareBytes(byte[],int,int,byte[],int,int)]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()} to those provided.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if bytes from {#getBytes()} match.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a hash of the bytes returned from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#hashBytes(byte[],int)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface supported by {@link org.apache.hadoop.io.WritableComparable}
+ types supporting ordering/permutation by a representative set of bytes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BinaryComparable -->
+  <!-- start class org.apache.hadoop.io.BloomMapFile -->
+  <class name="BloomMapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomMapFile"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BLOOM_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HASH_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends {@link MapFile} and provides very much the same
+ functionality. However, it uses dynamic Bloom filters to provide
+ quick membership test for keys, and it offers a fast version of 
+ {@link Reader#get(WritableComparable, Writable)} operation, especially in
+ case of sparsely populated MapFile-s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BloomMapFile -->
+  <!-- start class org.apache.hadoop.io.BooleanWritable -->
+  <class name="BooleanWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BooleanWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BooleanWritable" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="get" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.BooleanWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for booleans.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BooleanWritable -->
+  <!-- start interface org.apache.hadoop.io.ByteBufferPool -->
+  <interface name="ByteBufferPool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Get a new direct ByteBuffer.  The pool can provide this from
+ removing a buffer from its internal cache, or by allocating a 
+ new buffer.
+
+ @param direct     Whether the buffer should be direct.
+ @param length     The minimum length the buffer will have.
+ @return           A new ByteBuffer.  This ByteBuffer must be direct.
+                   Its capacity can be less than what was requested, but
+                   must be at least 1 byte.]]>
+      </doc>
+    </method>
+    <method name="putBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Release a buffer back to the pool.
+ The pool may choose to put this buffer into its cache.
+
+ @param buffer    a direct bytebuffer]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.ByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.BytesWritable -->
+  <class name="BytesWritable" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BytesWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a zero-size sequence.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value.
+ @param bytes This array becomes the backing storage for the object.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[], int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value
+ and length as the length. Use this constructor if the array is larger
+ than the value it represents.
+ @param bytes This array becomes the backing storage for the object.
+ @param length The number of bytes to use from array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data backing the BytesWritable. Please use {@link #copyBytes()}
+ if you need the returned array to be precisely the length of the data.
+ @return The data is only valid between 0 and getLength() - 1.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getBytes()} instead.">
+      <doc>
+      <![CDATA[Get the data from the BytesWritable.
+ @deprecated Use {@link #getBytes()} instead.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current size of the buffer.]]>
+      </doc>
+    </method>
+    <method name="getSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getLength()} instead.">
+      <doc>
+      <![CDATA[Get the current size of the buffer.
+ @deprecated Use {@link #getLength()} instead.]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <doc>
+      <![CDATA[Change the size of the buffer. The values in the old range are preserved
+ and any new values are undefined. The capacity is changed if it is 
+ necessary.
+ @param size The new number of bytes]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the capacity, which is the maximum size that could handled without
+ resizing the backing storage.
+ @return The number of bytes]]>
+      </doc>
+    </method>
+    <method name="setCapacity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_cap" type="int"/>
+      <doc>
+      <![CDATA[Change the capacity of the backing storage.
+ The data is preserved.
+ @param new_cap The new capacity in bytes.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="org.apache.hadoop.io.BytesWritable"/>
+      <doc>
+      <![CDATA[Set the BytesWritable to the contents of the given newData.
+ @param newData the value to set this BytesWritable to.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Set the value to a copy of the given byte range
+ @param newData the new values to copy in
+ @param offset the offset in newData to start at
+ @param length the number of bytes to copy]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right_obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Are the two byte sequences equal?]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate the stream of bytes as hex pairs separated by ' '.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A byte sequence that is usable as a key or value.
+ It is resizable and distinguishes between the size of the sequence and
+ the current capacity. The hash function is the front of the md5 of the 
+ buffer. The sort order is the same as memcmp.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BytesWritable -->
+  <!-- start class org.apache.hadoop.io.ByteWritable -->
+  <class name="ByteWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ByteWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ByteWritable" type="byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Set the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ByteWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ByteWritable"/>
+      <doc>
+      <![CDATA[Compares two ByteWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for a single byte.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ByteWritable -->
+  <!-- start interface org.apache.hadoop.io.Closeable -->
+  <interface name="Closeable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="use java.io.Closeable">
+    <implements name="java.io.Closeable"/>
+    <doc>
+    <![CDATA[@deprecated use java.io.Closeable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Closeable -->
+  <!-- start class org.apache.hadoop.io.CompressedWritable -->
+  <class name="CompressedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompressedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="ensureInflated"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Must be called by all methods which access fields to ensure that the data
+ has been uncompressed.]]>
+      </doc>
+    </method>
+    <method name="readFieldsCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #readFields(DataInput)}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #write(DataOutput)}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base-class for Writables which store themselves compressed and lazily
+ inflate on field access.  This is useful for large objects whose fields are
+ not be altered during a map or reduce operation: leaving the field data
+ compressed makes copying the instance from one file to another much
+ faster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.CompressedWritable -->
+  <!-- start class org.apache.hadoop.io.DataOutputOutputStream -->
+  <class name="DataOutputOutputStream" extends="java.io.OutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="constructOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <doc>
+      <![CDATA[Construct an OutputStream from the given DataOutput. If 'out'
+ is already an OutputStream, simply returns it. Otherwise, wraps
+ it in an OutputStream.
+ @param out the DataOutput to wrap
+ @return an OutputStream instance that outputs to 'out']]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[OutputStream implementation that wraps a DataOutput.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DataOutputOutputStream -->
+  <!-- start class org.apache.hadoop.io.DefaultStringifier -->
+  <class name="DefaultStringifier" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Stringifier"/>
+    <constructor name="DefaultStringifier" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="fromString" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="store"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="item" type="K"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the item in the configuration with the given keyName.
+ 
+ @param <K>  the class of the item
+ @param conf the configuration to store
+ @param item the object to be stored
+ @param keyName the name of the key to use
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="load" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="storeArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="items" type="K[]"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the array of items in the configuration with the given keyName.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use 
+ @param items the objects to be stored
+ @param keyName the name of the key to use
+ @throws IndexOutOfBoundsException if the items array is empty
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="loadArray" return="K[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the array of objects from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DefaultStringifier is the default implementation of the {@link Stringifier}
+ interface which stringifies the objects using base64 encoding of the
+ serialized version of the objects. The {@link Serializer} and
+ {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ <br>
+ DefaultStringifier offers convenience methods to store/load objects to/from
+ the configuration.
+ 
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DefaultStringifier -->
+  <!-- start class org.apache.hadoop.io.DoubleWritable -->
+  <class name="DoubleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="DoubleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="DoubleWritable" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="double"/>
+    </method>
+    <method name="get" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a DoubleWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.DoubleWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Writable for Double values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DoubleWritable -->
+  <!-- start class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <class name="ElasticByteBufferPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.ByteBufferPool"/>
+    <constructor name="ElasticByteBufferPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+    </method>
+    <method name="putBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <doc>
+    <![CDATA[This is a simple ByteBufferPool which just creates ByteBuffers as needed.
+ It also caches ByteBuffers after they're released.  It will always return
+ the smallest cached buffer with at least the capacity you request.
+ We don't try to do anything clever here like try to limit the maximum cache
+ size.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.EnumSetWritable -->
+  <class name="EnumSetWritable" extends="java.util.AbstractCollection"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. If the <tt>value</tt> argument is null or
+ its size is zero, the <tt>elementType</tt> argument must not be null. If
+ the argument <tt>value</tt>'s size is bigger than zero, the argument
+ <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </constructor>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. Argument <tt>value</tt> should not be null
+ or empty.
+ 
+ @param value]]>
+      </doc>
+    </constructor>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="E"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.util.EnumSet"/>
+      <param name="elementType" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[reset the EnumSetWritable with specified
+ <tt>value</tt> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ is null or its size is zero, the <tt>elementType</tt> argument must not be
+ null. If the argument <tt>value</tt>'s size is bigger than zero, the
+ argument <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </method>
+    <method name="get" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this EnumSetWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true if <code>o</code> is an EnumSetWritable with the same value,
+ or both are null.]]>
+      </doc>
+    </method>
+    <method name="getElementType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class of all the elements of the underlying EnumSetWriable. It
+ may return null.
+ 
+ @return the element class]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable wrapper for EnumSet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.EnumSetWritable -->
+  <!-- start class org.apache.hadoop.io.FloatWritable -->
+  <class name="FloatWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="FloatWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FloatWritable" type="float"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a FloatWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.FloatWritable"/>
+      <doc>
+      <![CDATA[Compares two FloatWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for floats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.FloatWritable -->
+  <!-- start class org.apache.hadoop.io.GenericWritable -->
+  <class name="GenericWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="GenericWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Set the instance that is wrapped.
+ 
+ @param obj]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the wrapped instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTypes" return="java.lang.Class[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all classes that may be wrapped.  Subclasses should implement this
+ to return a constant array of classes.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for Writable instances.
+ <p>
+ When two sequence files, which have same Key type but different Value
+ types, are mapped out to reduce, multiple Value types is not allowed.
+ In this case, this class can help you wrap instances with different types.
+ </p>
+ 
+ <p>
+ Compared with <code>ObjectWritable</code>, this class is much more effective,
+ because <code>ObjectWritable</code> will append the class declaration as a String 
+ into the output file in every Key-Value pair.
+ </p>
+ 
+ <p>
+ Generic Writable implements {@link Configurable} interface, so that it will be 
+ configured by the framework. The configuration is passed to the wrapped objects
+ implementing {@link Configurable} interface <i>before deserialization</i>. 
+ </p>
+ 
+ how to use it: <br>
+ 1. Write your own class, such as GenericObject, which extends GenericWritable.<br> 
+ 2. Implements the abstract method <code>getTypes()</code>, defines 
+    the classes which will be wrapped in GenericObject in application.
+    Attention: this classes defined in <code>getTypes()</code> method, must
+    implement <code>Writable</code> interface.
+ <br><br>
+ 
+ The code looks like this:
+ <blockquote><pre>
+ public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+ </pre></blockquote>
+ 
+ @since Nov 8, 2006]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.GenericWritable -->
+  <!-- start class org.apache.hadoop.io.IntWritable -->
+  <class name="IntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="IntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="IntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a IntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.IntWritable"/>
+      <doc>
+      <![CDATA[Compares two IntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for ints.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IntWritable -->
+  <!-- start class org.apache.hadoop.io.IOUtils -->
+  <class name="IOUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IOUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer 
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+ 
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another. <strong>closes the input and output streams 
+ at the end</strong>.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="count" type="long"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies count bytes from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param count number of bytes to copy
+ @param close whether to close the streams
+ @throws IOException if bytes can not be read or written]]>
+      </doc>
+    </method>
+    <method name="wrappedReadForCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="is" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility wrapper for reading from {@link InputStream}. It catches any errors
+ thrown by the underlying stream (either IO or decompression-related), and
+ re-throws as an IOException.
+ 
+ @param is - InputStream to be read from
+ @param buf - buffer the data is read into
+ @param off - offset within buf
+ @param len - amount of data to be read
+ @return number of bytes read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads len bytes in a loop.
+
+ @param in InputStream to read from
+ @param buf The buffer to fill
+ @param off offset from the buffer
+ @param len the length of bytes to read
+ @throws IOException if it could not read requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to readFully(). Skips bytes in a loop.
+ @param in The InputStream to skip bytes from
+ @param len number of bytes to skip.
+ @throws IOException if it could not skip requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param log the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close
+ @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="cleanupWithLogger"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logger" type="org.slf4j.Logger"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param logger the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close]]>
+      </doc>
+    </method>
+    <method name="closeStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[Closes the stream ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param stream the Stream to close]]>
+      </doc>
+    </method>
+    <method name="closeStreams"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="streams" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Closes the streams ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param streams the Streams to close]]>
+      </doc>
+    </method>
+    <method name="closeSocket"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sock" type="java.net.Socket"/>
+      <doc>
+      <![CDATA[Closes the socket ignoring {@link IOException}
+
+ @param sock the Socket to close]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bc" type="java.nio.channels.WritableByteChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a WritableByteChannel, handling short writes.
+ 
+ @param bc               The WritableByteChannel to write to
+ @param buf              The input buffer
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="java.nio.channels.FileChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <param name="offset" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a FileChannel at a given offset, 
+ handling short writes.
+ 
+ @param fc               The FileChannel to write to
+ @param buf              The input buffer
+ @param offset           The offset in the file to start writing at
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="listDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="filter" type="java.io.FilenameFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the complete list of files in a directory as strings.<p>
+
+ This is better than File#listDir because it does not ignore IOExceptions.
+
+ @param dir              The directory to list.
+ @param filter           If non-null, the filter to use when listing
+                         this directory.
+ @return                 The list of files in the directory.
+
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileToSync" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.<br>
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param fileToSync the file to fsync]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="channel" type="java.nio.channels.FileChannel"/>
+      <param name="isDir" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param channel Channel to sync
+ @param isDir if true, the given file is a directory (Channel should be
+          opened for read and ignore IOExceptions, because not all file
+          systems and operating systems allow to fsync on a directory)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="wrapException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="methodName" type="java.lang.String"/>
+      <param name="exception" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Takes an IOException, file/directory path, and method name and returns an
+ IOException with the input exception as the cause and also include the
+ file,method details. The new exception provides the stack trace of the
+ place where the exception is thrown and some extra diagnostics
+ information.
+
+ Return instance of same exception if exception class has a public string
+ constructor; Otherwise return an PathIOException.
+ InterruptedIOException and PathIOException are returned unwrapped.
+
+ @param path file/directory path
+ @param methodName method name
+ @param exception the caught exception.
+ @return an exception to throw]]>
+      </doc>
+    </method>
+    <method name="readFullyToByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a DataInput until EOF and returns a byte array.  Make sure not to
+ pass in an infinite DataInput or this will never return.
+
+ @param in A DataInput
+ @return a byte array containing the data from the DataInput
+ @throws IOException on I/O error, other than EOF]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An utility class for I/O related functionality.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IOUtils -->
+  <!-- start class org.apache.hadoop.io.LongWritable -->
+  <class name="LongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="LongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a LongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.LongWritable"/>
+      <doc>
+      <![CDATA[Compares two LongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.LongWritable -->
+  <!-- start class org.apache.hadoop.io.MapFile -->
+  <class name="MapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="oldName" type="java.lang.String"/>
+      <param name="newName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames an existing map directory.]]>
+      </doc>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deletes the named map file.]]>
+      </doc>
+    </method>
+    <method name="fix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <param name="dryrun" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[This method attempts to fix a corrupt MapFile by re-creating its index.
+ @param fs filesystem
+ @param dir directory containing the MapFile data and index
+ @param keyClass key class (has to be a subclass of Writable)
+ @param valueClass value class (has to be a subclass of Writable)
+ @param dryrun do not perform any changes, just report what needs to be done
+ @return number of valid entries in this MapFile, or -1 if no fixing was needed
+ @throws Exception]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="INDEX_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the index file.]]>
+      </doc>
+    </field>
+    <field name="DATA_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the data file.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A file-based map from keys to values.
+ 
+ <p>A map is a directory containing two files, the <code>data</code> file,
+ containing all keys and values in the map, and a smaller <code>index</code>
+ file, containing a fraction of the keys.  The fraction is determined by
+ {@link Writer#getIndexInterval()}.
+
+ <p>The index file is read entirely into memory.  Thus key implementations
+ should try to keep themselves small.
+
+ <p>Map files are created by adding entries in-order.  To maintain a large
+ database, perform updates by copying the previous version of a database and
+ merging in a sorted change list, to create a new version of the database in
+ a new file.  Sorting large change lists can be done with {@link
+ SequenceFile.Sorter}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapFile -->
+  <!-- start class org.apache.hadoop.io.MapWritable -->
+  <class name="MapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Map"/>
+    <constructor name="MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="MapWritable" type="org.apache.hadoop.io.MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Writable"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable Map.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapWritable -->
+  <!-- start class org.apache.hadoop.io.MD5Hash -->
+  <class name="MD5Hash" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="MD5Hash"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash from a hex string.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash with a specified value.]]>
+      </doc>
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs, reads and returns an instance.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Copy the contents of another instance into this instance.]]>
+      </doc>
+    </method>
+    <method name="getDigest" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the digest bytes.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="getDigester" return="java.security.MessageDigest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a thread local MD5 digester]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a hash value for the content from the InputStream.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dataArr" type="byte[][]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for an array of byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="org.apache.hadoop.io.UTF8"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="halfDigest" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a half-sized version of this MD5.  Fits in a long]]>
+      </doc>
+    </method>
+    <method name="quarterDigest" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a 32-bit digest of the MD5.
+ @return the first 4 bytes of the md5]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is an MD5Hash whose digest contains the
+ same values.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for this object.
+ Only uses the first 4 bytes, since md5s are evenly distributed.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Compares this object with the specified object for order.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="setDigest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the digest value from a hex string.]]>
+      </doc>
+    </method>
+    <field name="MD5_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Writable for MD5 hash values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MD5Hash -->
+  <!-- start class org.apache.hadoop.io.MultipleIOException -->
+  <class name="MultipleIOException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getExceptions" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the underlying exceptions]]>
+      </doc>
+    </method>
+    <method name="createIOException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="exceptions" type="java.util.List"/>
+      <doc>
+      <![CDATA[A convenient method to create an {@link IOException}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Encapsulate a list of {@link IOException} into an {@link IOException}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MultipleIOException -->
+  <!-- start class org.apache.hadoop.io.NullWritable -->
+  <class name="NullWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <method name="get" return="org.apache.hadoop.io.NullWritable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the single instance of this class.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.NullWritable"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Singleton Writable with no data.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.NullWritable -->
+  <!-- start class org.apache.hadoop.io.ObjectWritable -->
+  <class name="ObjectWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="ObjectWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Class, java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the instance, or null if none.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the class this is meant to be.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="instance" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reset the instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="allowCompactArrays" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.  
+ 
+ @param allowCompactArrays - set true for RPC and internal or intra-cluster
+ usages.  Set false for inter-cluster, File, and other persisted output 
+ usages, to preserve the ability to interchange files with other clusters 
+ that may not be running the same version of software.  Sometime in ~2013 
+ we can consider removing this parameter and always using the compact format.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="objectWritable" type="org.apache.hadoop.io.ObjectWritable"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find and load the class with given name <tt>className</tt> by first finding
+ it in the specified <tt>conf</tt>. If the specified <tt>conf</tt> is null,
+ try load it directly.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A polymorphic Writable that writes an instance with it's class name.
+ Handles arrays, strings and primitive types without a Writable wrapper.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ObjectWritable -->
+  <!-- start interface org.apache.hadoop.io.RawComparator -->
+  <interface name="RawComparator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <method name="compare" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Compare two objects in binary.
+ b1[s1:l1] is the first object, and b2[s2:l2] is the second object.
+ 
+ @param b1 The first byte array.
+ @param s1 The position index in b1. The object under comparison's starting index.
+ @param l1 The length of the object in b1.
+ @param b2 The second byte array.
+ @param s2 The position index in b2. The object under comparison's starting index.
+ @param l2 The length of the object under comparison in b2.
+ @return An integer result of the comparison.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ A {@link Comparator} that operates directly on byte representations of
+ objects.
+ </p>
+ @param <T>
+ @see DeserializerComparator]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.RawComparator -->
+  <!-- start class org.apache.hadoop.io.SequenceFile -->
+  <class name="SequenceFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDefaultCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the compression type for the reduce outputs
+ @param job the job config to look in
+ @return the kind of compression to use]]>
+      </doc>
+    </method>
+    <method name="setDefaultCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="val" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the default compression type for sequence files.
+ @param job the configuration to modify
+ @param val the new compression type (none, block, record)]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="opts" type="org.apache.hadoop.io.SequenceFile.Writer.Option[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new Writer with the given options.
+ @param conf the configuration to use
+ @param opts the options to create the file with
+ @return a new Writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="createParent" type="boolean"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param createParent create parent directory if non-existent
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fc The context for the specified file.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @param createFlag gives the semantics of create: overwrite, append etc.
+ @param opts file creation options; see {@link CreateOpts}.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <field name="SYNC_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes between sync points. 100 KB, default.
+ Computed as 5 KB * 20 = 100 KB]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[<code>SequenceFile</code>s are flat files consisting of binary key/value 
+ pairs.
+ 
+ <p><code>SequenceFile</code> provides {@link SequenceFile.Writer},
+ {@link SequenceFile.Reader} and {@link Sorter} classes for writing,
+ reading and sorting respectively.</p>
+ 
+ There are three <code>SequenceFile</code> <code>Writer</code>s based on the 
+ {@link CompressionType} used to compress key/value pairs:
+ <ol>
+   <li>
+   <code>Writer</code> : Uncompressed records.
+   </li>
+   <li>
+   <code>RecordCompressWriter</code> : Record-compressed files, only compress 
+                                       values.
+   </li>
+   <li>
+   <code>BlockCompressWriter</code> : Block-compressed files, both keys &amp;
+                                      values are collected in 'blocks' 
+                                      separately and compressed. The size of 
+                                      the 'block' is configurable.
+ </ol>
+ 
+ <p>The actual compression algorithm used to compress key and/or values can be
+ specified by using the appropriate {@link CompressionCodec}.</p>
+ 
+ <p>The recommended way is to use the static <tt>createWriter</tt> methods
+ provided by the <code>SequenceFile</code> to chose the preferred format.</p>
+
+ <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
+ above <code>SequenceFile</code> formats.</p>
+
+ <h3 id="Formats">SequenceFile Formats</h3>
+ 
+ <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
+ depending on the <code>CompressionType</code> specified. All of them share a
+ <a href="#Header">common header</a> described below.
+ 
+ <h4 id="Header">SequenceFile Header</h4>
+ <ul>
+   <li>
+   version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual 
+             version number (e.g. SEQ4 or SEQ6)
+   </li>
+   <li>
+   keyClassName -key class
+   </li>
+   <li>
+   valueClassName - value class
+   </li>
+   <li>
+   compression - A boolean which specifies if compression is turned on for 
+                 keys/values in this file.
+   </li>
+   <li>
+   blockCompression - A boolean which specifies if block-compression is 
+                      turned on for keys/values in this file.
+   </li>
+   <li>
+   compression codec - <code>CompressionCodec</code> class which is used for  
+                       compression of keys and/or values (if compression is 
+                       enabled).
+   </li>
+   <li>
+   metadata - {@link Metadata} for this file.
+   </li>
+   <li>
+   sync - A sync marker to denote end of the header.
+   </li>
+ </ul>
+ 
+ <h5>Uncompressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li>Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+
+ <h5>Record-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li><i>Compressed</i> Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+ 
+ <h5>Block-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record <i>Block</i>
+   <ul>
+     <li>Uncompressed number of records in the block</li>
+     <li>Compressed key-lengths block-size</li>
+     <li>Compressed key-lengths block</li>
+     <li>Compressed keys block-size</li>
+     <li>Compressed keys block</li>
+     <li>Compressed value-lengths block-size</li>
+     <li>Compressed value-lengths block</li>
+     <li>Compressed values block-size</li>
+     <li>Compressed values block</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every block.
+ </li>
+ </ul>
+ 
+ <p>The compressed blocks of key lengths and value lengths consist of the 
+ actual lengths of individual keys/values encoded in ZeroCompressedInteger 
+ format.</p>
+ 
+ @see CompressionCodec]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SequenceFile -->
+  <!-- start class org.apache.hadoop.io.SetFile -->
+  <class name="SetFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SetFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A file-based set of keys.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SetFile -->
+  <!-- start class org.apache.hadoop.io.ShortWritable -->
+  <class name="ShortWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ShortWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ShortWritable" type="short"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="short"/>
+      <doc>
+      <![CDATA[Set the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[read the short value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[write short value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ShortWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[hash code]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ShortWritable"/>
+      <doc>
+      <![CDATA[Compares two ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Short values in string format]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for shorts.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ShortWritable -->
+  <!-- start class org.apache.hadoop.io.SortedMapWritable -->
+  <class name="SortedMapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.SortedMap"/>
+    <constructor name="SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="SortedMapWritable" type="org.apache.hadoop.io.SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="comparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="firstKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="headMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="lastKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="subMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="tailMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable SortedMap.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SortedMapWritable -->
+  <!-- start interface org.apache.hadoop.io.Stringifier -->
+  <interface name="Stringifier"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Converts the object to a string representation
+ @param obj the object to convert
+ @return the string representation of the object
+ @throws IOException if the object cannot be converted]]>
+      </doc>
+    </method>
+    <method name="fromString" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from its string representation.
+ @param str the string representation of the object
+ @return restored object
+ @throws IOException if the object cannot be restored]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes this object. 
+ @throws IOException if an I/O error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stringifier interface offers two methods to convert an object 
+ to a string representation and restore the object given its 
+ string representation.
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Stringifier -->
+  <!-- start class org.apache.hadoop.io.Text -->
+  <class name="Text" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Text" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a string.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another text.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a byte array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the raw bytes; however, only data up to {@link #getLength()} is
+ valid. Please use {@link #copyBytes()} if you
+ need the returned array to be precisely the length of the data.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes in the byte array]]>
+      </doc>
+    </method>
+    <method name="charAt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="int"/>
+      <doc>
+      <![CDATA[Returns the Unicode Scalar Value (32-bit integer value)
+ for the character at <code>position</code>. Note that this
+ method avoids using the converter or doing String instantiation
+ @return the Unicode scalar value at position or -1
+          if the position is invalid or points to a
+          trailing byte]]>
+      </doc>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Finds any occurrence of <code>what</code> in the backing
+ buffer, starting as position <code>start</code>. The starting
+ position is measured in bytes and the return value is in
+ terms of byte position in the buffer. The backing buffer is
+ not converted to a string for this operation.
+ @return byte position of the first occurrence of the search
+         string in the UTF-8 buffer or -1 if not found]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set to contain the contents of a string.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <doc>
+      <![CDATA[Set to a utf8 byte array]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[copy a text.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Set the Text to range of bytes
+ @param utf8 the data to copy from
+ @param start the first position of the new string
+ @param len the number of bytes of the new string]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Append a range of bytes to the end of the given text
+ @param utf8 the data to copy from
+ @param start the first position to append from utf8
+ @param len the number of bytes to append]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the string to empty.
+
+ <em>Note</em>: For performance reasons, this call does not clear the
+ underlying byte array that is retrievable via {@link #getBytes()}.
+ In order to free the byte-array memory, call {@link #set(byte[])}
+ with an empty byte array (For example, <code>new byte[0]</code>).]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert text back to string
+ @see java.lang.Object#toString()]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[deserialize]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skips over one Text in the input.]]>
+      </doc>
+    </method>
+    <method name="readWithKnownLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a Text object whose length is already known.
+ This allows creating Text from a stream which uses a different serialization
+ format.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[serialize
+ write this object to out
+ length uses zero-compressed encoding
+ @see Writable#write(DataOutput)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a Text with the same contents.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If the input is malformed,
+ replace by a default value.]]>
+      </doc>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If the input is malformed,
+ invalid chars are replaced by a default value.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string from in]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string with a maximum size]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string to out]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string with a maximum size to out]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check if a byte array contains valid utf-8
+ @param utf8 byte array
+ @throws MalformedInputException if the byte array contains invalid utf-8]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check to see if a byte array is valid utf-8
+ @param utf8 the array of bytes
+ @param start the offset of the first byte in the array
+ @param len the length of the byte sequence
+ @throws MalformedInputException if the byte array contains invalid bytes]]>
+      </doc>
+    </method>
+    <method name="bytesToCodePoint" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Returns the next code point at the current position in
+ the buffer. The buffer's position will be incremented.
+ Any mark set on this buffer will be changed by this method!]]>
+      </doc>
+    </method>
+    <method name="utf8Length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[For the given string, returns the number of UTF-8 bytes
+ required to encode the string.
+ @param string text to encode
+ @return number of UTF-8 bytes required to encode]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_MAX_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class stores text using standard UTF8 encoding.  It provides methods
+ to serialize, deserialize, and compare texts at byte level.  The type of
+ length is integer and is serialized using zero-compressed format.  <p>In
+ addition, it provides methods for string traversal without converting the
+ byte array to a string.  <p>Also includes utilities for
+ serializing/deserialing a string, coding/decoding a string, checking if a
+ byte array contains valid UTF8 code, calculating the length of an encoded
+ string.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.Text -->
+  <!-- start class org.apache.hadoop.io.TwoDArrayWritable -->
+  <class name="TwoDArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[][]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[][]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[][]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for 2D arrays containing a matrix of instances of a class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.TwoDArrayWritable -->
+  <!-- start class org.apache.hadoop.io.VersionedWritable -->
+  <class name="VersionedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="VersionedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getVersion" return="byte"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the version number of the current implementation.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A base class for Writables that provides version checking.
+
+ <p>This is useful when a class may evolve, so that instances written by the
+ old version of the class may still be processed by the new version.  To
+ handle this situation, {@link #readFields(DataInput)}
+ implementations should catch {@link VersionMismatchException}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionedWritable -->
+  <!-- start class org.apache.hadoop.io.VersionMismatchException -->
+  <class name="VersionMismatchException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionMismatchException" type="byte, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Thrown by {@link VersionedWritable#readFields(DataInput)} when the
+ version of an object being read does not match the current implementation
+ version as returned by {@link VersionedWritable#getVersion()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionMismatchException -->
+  <!-- start class org.apache.hadoop.io.VIntWritable -->
+  <class name="VIntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VIntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VIntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VIntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VIntWritable"/>
+      <doc>
+      <![CDATA[Compares two VIntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for integer values stored in variable-length format.
+ Such values take between one and five bytes.  Smaller values take fewer bytes.
+ 
+ @see org.apache.hadoop.io.WritableUtils#readVInt(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VIntWritable -->
+  <!-- start class org.apache.hadoop.io.VLongWritable -->
+  <class name="VLongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VLongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VLongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VLongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VLongWritable"/>
+      <doc>
+      <![CDATA[Compares two VLongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs in a variable-length format. Such values take
+  between one and five bytes.  Smaller values take fewer bytes.
+  
+  @see org.apache.hadoop.io.WritableUtils#readVLong(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VLongWritable -->
+  <!-- start interface org.apache.hadoop.io.Writable -->
+  <interface name="Writable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize the fields of this object to <code>out</code>.
+ 
+ @param out <code>DataOuput</code> to serialize this object into.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize the fields of this object from <code>in</code>.  
+ 
+ <p>For efficiency, implementations should attempt to re-use storage in the 
+ existing object where possible.</p>
+ 
+ @param in <code>DataInput</code> to deseriablize this object from.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A serializable object which implements a simple, efficient, serialization 
+ protocol, based on {@link DataInput} and {@link DataOutput}.
+
+ <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
+ framework implements this interface.</p>
+ 
+ <p>Implementations typically implement a static <code>read(DataInput)</code>
+ method which constructs a new instance, calls {@link #readFields(DataInput)} 
+ and returns the instance.</p>
+ 
+ <p>Example:</p>
+ <blockquote><pre>
+     public class MyWritable implements Writable {
+       // Some data
+       private int counter;
+       private long timestamp;
+
+       // Default constructor to allow (de)serialization
+       MyWritable() { }
+
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+ </pre></blockquote>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Writable -->
+  <!-- start interface org.apache.hadoop.io.WritableComparable -->
+  <interface name="WritableComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <doc>
+    <![CDATA[A {@link Writable} which is also {@link Comparable}. 
+
+ <p><code>WritableComparable</code>s can be compared to each other, typically 
+ via <code>Comparator</code>s. Any type which is to be used as a 
+ <code>key</code> in the Hadoop Map-Reduce framework should implement this
+ interface.</p>
+
+ <p>Note that <code>hashCode()</code> is frequently used in Hadoop to partition
+ keys. It's important that your implementation of hashCode() returns the same 
+ result across different instances of the JVM. Note also that the default 
+ <code>hashCode()</code> implementation in <code>Object</code> does <b>not</b>
+ satisfy this property.</p>
+  
+ <p>Example:</p>
+ <blockquote><pre>
+     public class MyWritableComparable implements
+      WritableComparable{@literal <MyWritableComparable>} {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue &lt; thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp &gt;&gt;&gt; 32));
+         return result
+       }
+     }
+ </pre></blockquote>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableComparable -->
+  <!-- start class org.apache.hadoop.io.WritableComparator -->
+  <class name="WritableComparator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.RawComparator"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="WritableComparator"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, org.apache.hadoop.conf.Configuration, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[For backwards compatibility.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a comparator for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="define"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="comparator" type="org.apache.hadoop.io.WritableComparator"/>
+      <doc>
+      <![CDATA[Register an optimized comparator for a {@link WritableComparable}
+ implementation. Comparators registered with this method must be
+ thread-safe.]]>
+      </doc>
+    </method>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the WritableComparable implementation class.]]>
+      </doc>
+    </method>
+    <method name="newKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new {@link WritableComparable} instance.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Optimization hook.  Override this to make SequenceFile.Sorter's scream.
+
+ <p>The default implementation reads the data into two {@link
+ WritableComparable}s (using {@link
+ Writable#readFields(DataInput)}, then calls {@link
+ #compare(WritableComparable,WritableComparable)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="b" type="org.apache.hadoop.io.WritableComparable"/>
+      <doc>
+      <![CDATA[Compare two WritableComparables.
+
+ <p> The default implementation uses the natural ordering, calling {@link
+ Comparable#compareTo(Object)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="java.lang.Object"/>
+      <param name="b" type="java.lang.Object"/>
+    </method>
+    <method name="compareBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Lexicographic order of binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="readUnsignedShort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an unsigned short from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an integer from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a float from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a long from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a double from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a byte array and returns it.
+ @param bytes byte array with decode long
+ @param start starting index
+ @throws java.io.IOException 
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a byte array and returns it.
+ @param bytes byte array with the encoded integer
+ @param start start index
+ @throws java.io.IOException 
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A Comparator for {@link WritableComparable}s.
+
+ <p>This base implementation uses the natural ordering.  To define alternate
+ orderings, override {@link #compare(WritableComparable,WritableComparable)}.
+
+ <p>One may optimize compare-intensive operations by overriding
+ {@link #compare(byte[],int,int,byte[],int,int)}.  Static utility methods are
+ provided to assist in optimized implementations of this method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableComparator -->
+  <!-- start class org.apache.hadoop.io.WritableFactories -->
+  <class name="WritableFactories" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="factory" type="org.apache.hadoop.io.WritableFactory"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="getFactory" return="org.apache.hadoop.io.WritableFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Factories for non-public writables.  Defining a factory permits {@link
+ ObjectWritable} to be able to construct instances of non-public classes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableFactories -->
+  <!-- start interface org.apache.hadoop.io.WritableFactory -->
+  <interface name="WritableFactory"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a new instance.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A factory for a class of Writable.
+ @see WritableFactories]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableFactory -->
+  <!-- start class org.apache.hadoop.io.WritableUtils -->
+  <class name="WritableUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WritableUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readCompressedByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skipCompressedByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedByteArray" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="displayByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="byte[]"/>
+    </method>
+    <method name="clone" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orig" type="T"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Make a copy of a writable object using serialization to a buffer.
+ @param orig The object to copy
+ @return The copied object]]>
+      </doc>
+    </method>
+    <method name="cloneInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use ReflectionUtils.cloneInto instead.">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param dst the object to copy from
+ @param src the object to copy into, which is destroyed
+ @throws IOException
+ @deprecated use ReflectionUtils.cloneInto instead.]]>
+      </doc>
+    </method>
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes an integer to a binary stream with zero-compressed encoding.
+ For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ actual value.
+ For other values of i, the first byte value indicates whether the
+ integer is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -116, the following integer
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -124, the following integer
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+
+ @param stream Binary output stream
+ @param i Integer to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes a long to a binary stream with zero-compressed encoding.
+ For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ actual value.
+ For other values of i, the first byte value indicates whether the
+ long is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -120, the following long
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -128, the following long
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+ 
+ @param stream Binary output stream
+ @param i Long to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized long from stream.]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized integer from stream.]]>
+      </doc>
+    </method>
+    <method name="readVIntInRange" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <param name="lower" type="int"/>
+      <param name="upper" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads an integer from the input stream and returns it.
+
+ This function validates that the integer is between [lower, upper],
+ inclusive.
+
+ @param stream Binary input stream
+ @throws java.io.IOException
+ @return deserialized integer from stream]]>
+      </doc>
+    </method>
+    <method name="isNegativeVInt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Given the first byte of a vint/vlong, determine the sign
+ @param value the first byte
+ @return is the value negative]]>
+      </doc>
+    </method>
+    <method name="decodeVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Parse the first byte of a vint/vlong to determine the number of bytes
+ @param value the first byte of the vint/vlong
+ @return the total number of bytes (1 to 9)]]>
+      </doc>
+    </method>
+    <method name="getVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="long"/>
+      <doc>
+      <![CDATA[Get the encoded length if an integer is stored in a variable-length format
+ @return the encoded length]]>
+      </doc>
+    </method>
+    <method name="readEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="enumType" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read an Enum value from DataInput, Enums are read and written 
+ using String values. 
+ @param <T> Enum type
+ @param in DataInput to read from 
+ @param enumType Class type of Enum
+ @return Enum represented by String read from DataInput
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeEnum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="enumVal" type="java.lang.Enum"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[writes String value of enum to DataOutput. 
+ @param out Dataoutput stream
+ @param enumVal enum value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip <i>len</i> number of bytes in input stream<i>in</i>
+ @param in input stream
+ @param len number of bytes to skip
+ @throws IOException when skipped less number of bytes]]>
+      </doc>
+    </method>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writables" type="org.apache.hadoop.io.Writable[]"/>
+      <doc>
+      <![CDATA[Convert writables to a byte array]]>
+      </doc>
+    </method>
+    <method name="readStringSafely" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Read a string, but check it for sanity. The format consists of a vint
+ followed by the given number of bytes.
+ @param in the stream to read from
+ @param maxLength the largest acceptable length of the encoded string
+ @return the bytes as a string
+ @throws IOException if reading from the DataInput fails
+ @throws IllegalArgumentException if the encoded byte size for string 
+             is negative or larger than maxSize. Only the vint is read.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableUtils -->
+  <doc>
+  <![CDATA[Generic i/o code for use when reading and writing data to the network,
+to databases, and to files.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.compress">
+  <!-- start class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <class name="BlockCompressorStream" extends="org.apache.hadoop.io.compress.CompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream}.
+ 
+ @param out stream
+ @param compressor compressor to be used
+ @param bufferSize size of buffer
+ @param compressionOverhead maximum 'overhead' of the compression 
+                            algorithm with given bufferSize]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream} with given output-stream and 
+ compressor.
+ Use default of 512 as bufferSize and compressionOverhead of 
+ (1% of bufferSize + 12 bytes) =  18 bytes (zlib algorithm).
+ 
+ @param out stream
+ @param compressor compressor to be used]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the data provided to the compression codec, compressing no more
+ than the buffer size less the compression overhead as specified during
+ construction for each block.
+
+ Each block contains the uncompressed length for the block, followed by
+ one or more length-prefixed blocks of compressed data.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.CompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.
+
+ It should be noted that this wrapper does not guarantee that blocks will
+ be sized for the compressor. If the
+ {@link org.apache.hadoop.io.compress.Compressor} requires buffering to
+ effect meaningful compression, it is responsible for it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <class name="BlockDecompressorStream" extends="org.apache.hadoop.io.compress.DecompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @param bufferSize size of buffer
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.DecompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BZip2Codec -->
+  <class name="BZip2Codec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.SplittableCompressionCodec"/>
+    <constructor name="BZip2Codec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BZip2Codec.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+
+ @param conf the configuration object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+
+ @return the configuration object used by this objec.]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream}.
+
+ @param out        the location for the final output stream
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream} with the given {@link Compressor}.
+
+ @param out        the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream and return a stream for uncompressed data.
+
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ {@link InputStream} with the given {@link Decompressor}, and return a 
+ stream for uncompressed data.
+
+ @param in           the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates CompressionInputStream to be used to read off uncompressed data
+ in one of the two reading modes. i.e. Continuous or Blocked reading modes
+
+ @param seekableIn The InputStream
+ @param start The start offset into the compressed stream
+ @param end The end offset into the compressed stream
+ @param readMode Controls whether progress is reported continuously or
+                 only at block boundaries.
+
+ @return CompressionInputStream for BZip2 aligned at block boundaries]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[.bz2 is recognized as the default extension for compressed BZip2 files
+
+ @return A String telling the default bzip2 file extension]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class provides output and input streams for bzip2 compression
+ and decompression.  It uses the native bzip2 library on the system
+ if possible, else it uses a pure-Java implementation of the bzip2
+ algorithm.  The configuration parameter
+ io.compression.codec.bzip2.library can be used to control this
+ behavior.
+
+ In the pure-Java mode, the Compressor and Decompressor interfaces
+ are not implemented.  Therefore, in that mode, those methods of
+ CompressionCodec which have a Compressor or Decompressor type
+ argument, throw UnsupportedOperationException.
+
+ Currently, support for splittability is available only in the
+ pure-Java mode; therefore, if a SplitCompressionInputStream is
+ requested, the pure-Java implementation is used, regardless of the
+ setting of the configuration parameter mentioned above.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BZip2Codec -->
+  <!-- start class org.apache.hadoop.io.compress.CodecConstants -->
+  <class name="CodecConstants" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="DEFAULT_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.DefaultCodec}.]]>
+      </doc>
+    </field>
+    <field name="BZIP2_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.BZip2Codec}.]]>
+      </doc>
+    </field>
+    <field name="GZIP_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.GzipCodec}.]]>
+      </doc>
+    </field>
+    <field name="LZ4_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.Lz4Codec}.]]>
+      </doc>
+    </field>
+    <field name="PASSTHROUGH_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for
+ {@link org.apache.hadoop.io.compress.PassthroughCodec}.]]>
+      </doc>
+    </field>
+    <field name="SNAPPY_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.SnappyCodec}.]]>
+      </doc>
+    </field>
+    <field name="ZSTANDARD_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.ZStandardCodec}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Codec related constants.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecConstants -->
+  <!-- start class org.apache.hadoop.io.compress.CodecPool -->
+  <class name="CodecPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CodecPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a {@link Compressor} for the given {@link CompressionCodec} from the 
+ pool or a new one.
+
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Compressor</code>
+ @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
+ @return <code>Compressor</code> for the given 
+         <code>CompressionCodec</code> from the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+    </method>
+    <method name="getDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Get a {@link Decompressor} for the given {@link CompressionCodec} from the
+ pool or a new one.
+  
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Decompressor</code>
+ @return <code>Decompressor</code> for the given 
+         <code>CompressionCodec</code> the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="returnCompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <doc>
+      <![CDATA[Return the {@link Compressor} to the pool.
+ 
+ @param compressor the <code>Compressor</code> to be returned to the pool]]>
+      </doc>
+    </method>
+    <method name="returnDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <doc>
+      <![CDATA[Return the {@link Decompressor} to the pool.
+ 
+ @param decompressor the <code>Decompressor</code> to be returned to the 
+                     pool]]>
+      </doc>
+    </method>
+    <method name="getLeasedCompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Compressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <method name="getLeasedDecompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Decompressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A global compressor/decompressor pool used to save and reuse 
+ (possibly native) compression/decompression codecs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecPool -->
+  <!-- start interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <interface name="CompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream}.
+ 
+ @param out the location for the final output stream
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream} with the given {@link Compressor}.
+ 
+ @param out the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream.
+ 
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given 
+ {@link InputStream} with the given {@link Decompressor}.
+ 
+ @param in the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default filename extension for this kind of compression.
+ @return the extension including the '.']]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a streaming compression/decompression pair.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <class name="CompressionCodecFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionCodecFactory" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Find the codecs specified in the config value io.compression.codecs 
+ and register them. Defaults to gzip and deflate.]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Print the extension map out as a string.]]>
+      </doc>
+    </method>
+    <method name="getCodecClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the list of codecs discovered via a Java ServiceLoader, or
+ listed in the configuration. Codecs specified in configuration come
+ later in the returned list, and are considered to override those
+ from the ServiceLoader.
+ @param conf the configuration to look in
+ @return a list of the {@link CompressionCodec} classes]]>
+      </doc>
+    </method>
+    <method name="setCodecClasses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="classes" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets a list of codec classes in the configuration. In addition to any
+ classes specified using this method, {@link CompressionCodec} classes on
+ the classpath are discovered using a Java ServiceLoader.
+ @param conf the configuration to modify
+ @param classes the list of classes to set]]>
+      </doc>
+    </method>
+    <method name="getCodec" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the given file based on its
+ filename suffix.
+ @param file the filename to check
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByClassName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classname" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name.
+ @param classname the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias.
+ <p>
+ Codec aliases are case insensitive.
+ <p>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias and returns its implemetation class.
+ <p>
+ Codec aliases are case insensitive.
+ <p>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec class]]>
+      </doc>
+    </method>
+    <method name="removeSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Removes a suffix from a filename, if it has it.
+ @param filename the filename to strip
+ @param suffix the suffix to remove
+ @return the shortened filename]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A little test program.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory that will find the correct codec for a given filename.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <class name="CompressionInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="CompressionInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a compression input stream that reads
+ the decompressed bytes from the given stream.
+ 
+ @param in The input stream to be compressed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return any IOStatistics provided by the underlying stream.
+ @return IO stats from the inner stream.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the decompressor to its initial state and discard any buffered data,
+ as the underlying stream may have been repositioned.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method returns the current position in the stream.
+
+ @return Current position in stream as a long]]>
+      </doc>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <field name="in" type="java.io.InputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The input stream to be compressed.]]>
+      </doc>
+    </field>
+    <field name="maxAvailableData" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A compression input stream.
+
+ <p>Implementations are assumed to be buffered.  This permits clients to
+ reposition the underlying input stream then call {@link #resetState()},
+ without having to also synchronize client buffers.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <class name="CompressionOutputStream" extends="java.io.OutputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="CompressionOutputStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a compression output stream that writes
+ the compressed bytes to the given stream.
+ @param out]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write compressed bytes to the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finishes writing compressed data to the output stream 
+ without closing the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the compression to the initial state. 
+ Does not reset the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return any IOStatistics provided by the underlying stream.
+ @return IO stats from the inner stream.]]>
+      </doc>
+    </method>
+    <field name="out" type="java.io.OutputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The output stream to be compressed.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A compression output stream.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Compressor -->
+  <interface name="Compressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for compression. 
+ This should be called whenever #needsInput() returns 
+ <code>true</code> indicating that more input data is required.
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the input data buffer is empty and 
+ #setInput() should be called to provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ #setInput() should be called in order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary 
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="getBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of uncompressed bytes input so far.]]>
+      </doc>
+    </method>
+    <method name="getBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of compressed bytes output so far.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When called, indicates that compression should end
+ with the current contents of the input buffer.]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the end of the compressed 
+ data output stream has been reached.
+ @return <code>true</code> if the end of the compressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="compress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with compressed data. Returns actual number
+ of bytes of compressed data. A return value of 0 indicates that
+ needsInput() should be called in order to determine if more input
+ data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of compressed data.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets compressor so that a new set of input data can be processed.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the compressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <method name="reinit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Prepare the compressor to be used in a new stream with settings defined in
+ the given Configuration
+ 
+ @param conf Configuration from which new setting are fetched]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'compressor' which can be  
+ plugged into a {@link CompressionOutputStream} to compress data.
+ This is modelled after {@link java.util.zip.Deflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Compressor -->
+  <!-- start class org.apache.hadoop.io.compress.CompressorStream -->
+  <class name="CompressorStream" extends="org.apache.hadoop.io.compress.CompressionOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param out Underlying output stream.]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="compressor" type="org.apache.hadoop.io.compress.Compressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressorStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Decompressor -->
+  <interface name="Decompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for decompression. 
+ This should be called if and only if {@link #needsInput()} returns 
+ <code>true</code> indicating that more input data is required.
+ (Both native and non-native versions of various Decompressors require
+ that the data passed in via <code>b[]</code> remain unmodified until
+ the caller is explicitly notified--via {@link #needsInput()}--that the
+ buffer may be safely modified.  With this requirement, an extra
+ buffer-copy can be avoided.)
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called to
+ provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called in
+ order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsDictionary" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if a preset dictionary is needed for decompression.
+ @return <code>true</code> if a preset dictionary is needed for decompression]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the end of the decompressed 
+ data output stream has been reached. Indicates a concatenated data stream
+ when finished() returns <code>true</code> and {@link #getRemaining()}
+ returns a positive value. finished() will be reset with the
+ {@link #reset()} method.
+ @return <code>true</code> if the end of the decompressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="decompress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with uncompressed data. Returns actual number
+ of bytes of uncompressed data. A return value of 0 indicates that
+ {@link #needsInput()} should be called in order to determine if more
+ input data is required.
+ 
+ @param b Buffer for the uncompressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of uncompressed data.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes remaining in the compressed data buffer.
+ Indicates a concatenated data stream if {@link #finished()} returns
+ <code>true</code> and getRemaining() returns a positive value. If
+ {@link #finished()} returns <code>true</code> and getRemaining() returns
+ a zero value, indicates that the end of data stream has been reached and
+ is not a concatenated data stream. 
+ @return The number of bytes remaining in the compressed data buffer.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets decompressor and input and output buffers so that a new set of
+ input data can be processed. If {@link #finished()}} returns
+ <code>true</code> and {@link #getRemaining()} returns a positive value,
+ reset() is called before processing of the next data stream in the
+ concatenated data stream. {@link #finished()} will be reset and will
+ return <code>false</code> when reset() is called.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the decompressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'de-compressor' which can be  
+ plugged into a {@link CompressionInputStream} to compress data.
+ This is modelled after {@link java.util.zip.Inflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Decompressor -->
+  <!-- start class org.apache.hadoop.io.compress.DecompressorStream -->
+  <class name="DecompressorStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param in Underlying input stream.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="available" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="markSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="mark"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readlimit" type="int"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="eof" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.DefaultCodec -->
+  <class name="DefaultCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <implements name="org.apache.hadoop.io.compress.DirectDecompressionCodec"/>
+    <constructor name="DefaultCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DefaultCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <interface name="DirectDecompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link DirectDecompressor} for use by this {@link DirectDecompressionCodec}.
+ 
+ @return a new direct decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a codec which can decompress direct bytebuffers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <interface name="DirectDecompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="decompress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.nio.ByteBuffer"/>
+      <param name="dst" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Specification of a direct ByteBuffer 'de-compressor'.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <!-- start class org.apache.hadoop.io.compress.GzipCodec -->
+  <class name="GzipCodec" extends="org.apache.hadoop.io.compress.DefaultCodec"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GzipCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class creates gzip compressors/decompressors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.GzipCodec -->
+  <!-- start class org.apache.hadoop.io.compress.PassthroughCodec -->
+  <class name="PassthroughCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <constructor name="PassthroughCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Classname of the codec: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OPT_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Option to control the extension of the code: {@value}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This default extension is here so that if no extension has been defined,
+ some value is still returned: {@value}..]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is a special codec which does not transform the output.
+ It can be declared as a codec in the option "io.compression.codecs",
+ and then it will declare that it supports the file extension
+ set in {@link #OPT_EXTENSION}.
+
+ This allows decompression to be disabled on a job, even when there is
+ a registered/discoverable decompression codec for a file extension
+ -without having to change the standard codec binding mechanism.
+
+ For example, to disable decompression for a gzipped files, set the
+ options
+ <pre>
+   io.compression.codecs = org.apache.hadoop.io.compress.PassthroughCodec
+   io.compress.passthrough.extension = .gz
+ </pre>
+
+ <i>Note:</i> this is not a Splittable codec: it doesn't know the
+ capabilities of the passed in stream. It should be possible to
+ extend this in a subclass: the inner classes are marked as protected
+ to enable this. <i>Do not retrofit splitting to this class.</i>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.PassthroughCodec -->
+  <!-- start class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <class name="SplitCompressionInputStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitCompressionInputStream" type="java.io.InputStream, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="setStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="start" type="long"/>
+    </method>
+    <method name="setEnd"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="end" type="long"/>
+    </method>
+    <method name="getAdjustedStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of start.
+ @return The changed value of start]]>
+      </doc>
+    </method>
+    <method name="getAdjustedEnd" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of end.
+ @return The changed value of end]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputStream covering a range of compressed data. The start and end
+ offsets requested by a client may be modified by the codec to fit block
+ boundaries or other algorithm-dependent requirements.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+  <interface name="SplittableCompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a stream as dictated by the readMode.  This method is used when
+ the codecs wants the ability to work with the underlying stream positions.
+
+ @param seekableIn  The seekable input stream (seeks in compressed data)
+ @param start The start offset into the compressed stream. May be changed
+              by the underlying codec.
+ @param end The end offset into the compressed stream. May be changed by
+            the underlying codec.
+ @param readMode Controls whether stream position is reported continuously
+                 from the compressed stream only only at block boundaries.
+ @return  a stream to read uncompressed bytes from]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface is meant to be implemented by those compression codecs
+ which are capable to compress / de-compress a stream starting at any
+ arbitrary position.
+
+ Especially the process of de-compressing a stream starting at some arbitrary
+ position is challenging.  Most of the codecs are only able to successfully
+ de-compress a stream, if they start from the very beginning till the end.
+ One of the reasons is the stored state at the beginning of the stream which
+ is crucial for de-compression.
+
+ Yet there are few codecs which do not save the whole state at the beginning
+ of the stream and hence can be used to de-compress stream starting at any
+ arbitrary points.  This interface is meant to be used by such codecs.  Such
+ codecs are highly valuable, especially in the context of Hadoop, because
+ an input compressed file can be split and hence can be worked on by multiple
+ machines in parallel.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode">
+  <!-- start class org.apache.hadoop.io.erasurecode.ECSchema -->
+  <class name="ECSchema" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="ECSchema" type="java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with schema name and provided all options. Note the options may
+ contain additional information for the erasure codec to interpret further.
+ @param allOptions all schema options]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int, java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided. Note the extraOptions may contain
+ additional information for the erasure codec to interpret further.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema
+ @param extraOptions extra options to configure the codec]]>
+      </doc>
+    </constructor>
+    <method name="getCodecName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the codec name
+ @return codec name]]>
+      </doc>
+    </method>
+    <method name="getExtraOptions" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get extra options specific to a erasure code.
+ @return extra options]]>
+      </doc>
+    </method>
+    <method name="getNumDataUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required data units count in a coding group
+ @return count of data units]]>
+      </doc>
+    </method>
+    <method name="getNumParityUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required parity units count in a coding group
+ @return count of parity units]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Make a meaningful string representation for log output.
+ @return string representation]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="NUM_DATA_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_PARITY_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CODEC_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Erasure coding schema to housekeeper relevant information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.erasurecode.ECSchema -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode.coder.util">
+</package>
+<package name="org.apache.hadoop.io.erasurecode.grouper">
+</package>
+<package name="org.apache.hadoop.io.file.tfile">
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <class name="MetaBlockAlreadyExists" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - Meta Block with the same name already exists.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <class name="MetaBlockDoesNotExist" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - No such Meta Block with the given name.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <!-- start interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <interface name="RawComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="buffer" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying byte array.
+ 
+ @return The underlying byte array.]]>
+      </doc>
+    </method>
+    <method name="offset" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the offset of the first byte in the byte array.
+ 
+ @return The offset of the first byte in the byte array.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the size of the byte range in the byte array.
+ 
+ @return The size of the byte range in the byte array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for objects that can be compared through {@link RawComparator}.
+ This is useful in places where we need a single object reference to specify a
+ range of bytes in a byte array, such as {@link Comparable} or
+ {@link Collections#binarySearch(java.util.List, Object, Comparator)}
+ 
+ The actual comparison among RawComparable's requires an external
+ RawComparator and it is applications' responsibility to ensure two
+ RawComparable are supposed to be semantically comparable with the same
+ RawComparator.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <!-- start class org.apache.hadoop.io.file.tfile.TFile -->
+  <class name="TFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="makeComparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Make a raw comparator from a string name.
+ 
+ @param name
+          Comparator name
+ @return A RawComparable comparator.]]>
+      </doc>
+    </method>
+    <method name="getSupportedCompressionAlgorithms" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get names of supported compression algorithms. The names are acceptable by
+ TFile.Writer.
+ 
+ @return Array of strings, each represents a supported compression
+         algorithm. Currently, the following compression algorithms are
+         supported.
+         <ul>
+         <li>"none" - No compression.
+         <li>"lzo" - LZO compression.
+         <li>"gz" - GZIP compression.
+         </ul>]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Dumping the TFile information.
+ 
+ @param args
+          A list of TFile paths.]]>
+      </doc>
+    </method>
+    <field name="COMPRESSION_GZ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: gzip]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_LZO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: lzo]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_NONE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: none]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_MEMCMP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator: memcmp]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_JCLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator prefix: java class]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A TFile is a container of key-value pairs. Both keys and values are type-less
+ bytes. Keys are restricted to 64KB, value length is not restricted
+ (practically limited to the available disk storage). TFile further provides
+ the following features:
+ <ul>
+ <li>Block Compression.
+ <li>Named meta data blocks.
+ <li>Sorted or unsorted keys.
+ <li>Seek by key or by file offset.
+ </ul>
+ The memory footprint of a TFile includes the following:
+ <ul>
+ <li>Some constant overhead of reading or writing a compressed block.
+ <ul>
+ <li>Each compressed block requires one compression/decompression codec for
+ I/O.
+ <li>Temporary space to buffer the key.
+ <li>Temporary space to buffer the value (for TFile.Writer only). Values are
+ chunk encoded, so that we buffer at most one chunk of user data. By default,
+ the chunk buffer is 1MB. Reading chunked value does not require additional
+ memory.
+ </ul>
+ <li>TFile index, which is proportional to the total number of Data Blocks.
+ The total amount of memory needed to hold the index can be estimated as
+ (56+AvgKeySize)*NumBlocks.
+ <li>MetaBlock index, which is proportional to the total number of Meta
+ Blocks.The total amount of memory needed to hold the index for Meta Blocks
+ can be estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ </ul>
+ <p>
+ The behavior of TFile can be customized by the following variables through
+ Configuration:
+ <ul>
+ <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default
+ to 1MB. Values of the length less than the chunk size is guaranteed to have
+ known value length in read time (See
+ {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for
+ FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for
+ FSDataInputStream. Integer (in bytes). Default to 256KB.
+ </ul>
+ <p>
+ Suggestions on performance optimization.
+ <ul>
+ <li>Minimum block size. We recommend a setting of minimum block size between
+ 256KB to 1MB for general usage. Larger block size is preferred if files are
+ primarily for sequential access. However, it would lead to inefficient random
+ access (because there are more data to decompress). Smaller blocks are good
+ for random access, but require more memory to hold the block index, and may
+ be slower to create (because we must flush the compressor stream at the
+ conclusion of each data block, which leads to an FS I/O flush). Further, due
+ to the internal caching in Compression codec, the smallest possible block
+ size would be around 20KB-30KB.
+ <li>The current implementation does not offer true multi-threading for
+ reading. The implementation uses FSDataInputStream seek()+read(), which is
+ shown to be much faster than positioned-read call in single thread mode.
+ However, it also means that if multiple threads attempt to access the same
+ TFile (using multiple scanners) simultaneously, the actual I/O is carried out
+ sequentially even if they access different DFS blocks.
+ <li>Compression codec. Use "none" if the data is not very compressable (by
+ compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
+ as the starting point for experimenting. "gz" overs slightly better
+ compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
+ decompress, comparing to "lzo".
+ <li>File system buffering, if the underlying FSDataInputStream and
+ FSDataOutputStream is already adequately buffered; or if applications
+ reads/writes keys and values in large buffers, we can reduce the sizes of
+ input/output buffering in TFile layer by setting the configuration parameters
+ "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ </ul>
+ 
+ Some design rationale behind TFile can be found at <a
+ href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.TFile -->
+  <!-- start class org.apache.hadoop.io.file.tfile.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding an integer into a variable-length encoding format. Synonymous to
+ <code>Utils#writeVLong(out, n)</code>.
+ 
+ @param out
+          output stream
+ @param n
+          The integer to be encoded
+ @throws IOException
+ @see Utils#writeVLong(DataOutput, long)]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding a Long integer into a variable-length encoding format.
+ <ul>
+ <li>if n in [-32, 127): encode in one byte with the actual value.
+ Otherwise,
+ <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
+ byte[1]=n&amp;0xff. Otherwise,
+ <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
+ 88; byte[1]=(n&gt;&gt;8)&amp;0xff; byte[2]=n&amp;0xff. Otherwise,
+ <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
+ byte[1] = (n&gt;&gt;16)&amp;0xff; byte[2] = (n&gt;&gt;8)&amp;0xff;
+ byte[3]=n&amp;0xff.
+ Otherwise:
+ <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
+ (n&gt;&gt;24)&amp;0xff; byte[2]=(n&gt;&gt;16)&amp;0xff;
+ byte[3]=(n&gt;&gt;8)&amp;0xff; byte[4]=n&amp;0xff;
+ <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
+ (n&gt;&gt;32)&amp;0xff; byte[2]=(n&gt;&gt;24)&amp;0xff;
+ byte[3]=(n&gt;&gt;16)&amp;0xff; byte[4]=(n&gt;&gt;8)&amp;0xff;
+ byte[5]=n&amp;0xff
+ <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
+ (n&gt;&gt;40)&amp;0xff; byte[2]=(n&gt;&gt;32)&amp;0xff;
+ byte[3]=(n&gt;&gt;24)&amp;0xff; byte[4]=(n&gt;&gt;16)&amp;0xff;
+ byte[5]=(n&gt;&gt;8)&amp;0xff; byte[6]=n&amp;0xff;
+ <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
+ (n&gt;&gt;48)&amp;0xff; byte[2] = (n&gt;&gt;40)&amp;0xff;
+ byte[3]=(n&gt;&gt;32)&amp;0xff; byte[4]=(n&gt;&gt;24)&amp;0xff; byte[5]=
+ (n&gt;&gt;16)&amp;0xff; byte[6]=(n&gt;&gt;8)&amp;0xff; byte[7]=n&amp;0xff;
+ <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
+ (n&gt;&gt;54)&amp;0xff; byte[2] = (n&gt;&gt;48)&amp;0xff;
+ byte[3] = (n&gt;&gt;40)&amp;0xff; byte[4]=(n&gt;&gt;32)&amp;0xff;
+ byte[5]=(n&gt;&gt;24)&amp;0xff; byte[6]=(n&gt;&gt;16)&amp;0xff; byte[7]=
+ (n&gt;&gt;8)&amp;0xff; byte[8]=n&amp;0xff;
+ </ul>
+ 
+ @param out
+          output stream
+ @param n
+          the integer number
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Synonymous to
+ <code>(int)Utils#readVLong(in)</code>.
+ 
+ @param in
+          input stream
+ @return the decoded integer
+ @throws IOException
+ 
+ @see Utils#readVLong(DataInput)]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Suppose the value of the first byte
+ is FB, and the following bytes are NB[*].
+ <ul>
+ <li>if (FB &gt;= -32), return (long)FB;
+ <li>if (FB in [-72, -33]), return (FB+52)&lt;&lt;8 + NB[0]&amp;0xff;
+ <li>if (FB in [-104, -73]), return (FB+88)&lt;&lt;16 +
+ (NB[0]&amp;0xff)&lt;&lt;8 + NB[1]&amp;0xff;
+ <li>if (FB in [-120, -105]), return (FB+112)&lt;&lt;24 + (NB[0]&amp;0xff)
+ &lt;&lt;16 + (NB[1]&amp;0xff)&lt;&lt;8 + NB[2]&amp;0xff;
+ <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
+ big-endian integer.
+ </ul>
+ @param in
+          input stream
+ @return the decoded long integer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a String as a VInt n, followed by n Bytes as in Text format.
+ 
+ @param out
+ @param s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a String as a VInt n, followed by n Bytes in Text format.
+ 
+ @param in
+          The input stream.
+ @return The string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Supporting Utility classes used by TFile, and shared by users of TFile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.Utils -->
+</package>
+<package name="org.apache.hadoop.io.serializer">
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <class name="JavaSerialization" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="JavaSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ An experimental {@link Serialization} for Java {@link Serializable} classes.
+ </p>
+ @see JavaSerializationComparator]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <class name="JavaSerializationComparator" extends="org.apache.hadoop.io.serializer.DeserializerComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JavaSerializationComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ A {@link RawComparator} that uses a {@link JavaSerialization}
+ {@link Deserializer} to deserialize objects that are then compared via
+ their {@link Comparable} interfaces.
+ </p>
+ @param <T>
+ @see JavaSerialization]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <!-- start class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <class name="WritableSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="WritableSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A {@link Serialization} for {@link Writable}s that delegates to
+ {@link Writable#write(java.io.DataOutput)} and
+ {@link Writable#readFields(java.io.DataInput)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides a mechanism for using different serialization frameworks
+in Hadoop. The property "io.serializations" defines a list of
+{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create
+{@link org.apache.hadoop.io.serializer.Serializer}s and
+{@link org.apache.hadoop.io.serializer.Deserializer}s.
+</p>
+
+<p>
+To add a new serialization framework write an implementation of
+{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the
+"io.serializations" property.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.serializer.avro">
+  <!-- start interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <interface name="AvroReflectSerializable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Tag interface for Avro 'reflect' serializable classes. Classes implementing 
+ this interface can be serialized/deserialized using 
+ {@link AvroReflectSerialization}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <class name="AvroReflectSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroReflectSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_REFLECT_PACKAGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key to configure packages that contain classes to be serialized and 
+ deserialized using this class. Multiple packages can be specified using 
+ comma-separated list.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Serialization for Avro Reflect classes. For a class to be accepted by this 
+ serialization, it must either be in the package list configured via 
+ <code>avro.reflect.pkgs</code> or implement 
+ {@link AvroReflectSerializable} interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <class name="AvroSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="AvroSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_SCHEMA_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class for providing serialization to Avro types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <class name="AvroSpecificSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroSpecificSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Serialization for Avro Specific classes. This serialization is to be used 
+ for classes generated by Avro's 'specific' compiler.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides Avro serialization in Hadoop. This can be used to 
+serialize/deserialize Avro types in Hadoop.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for 
+serialization of classes generated by Avro's 'specific' compiler.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for 
+other classes. 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for  
+any class which is either in the package list configured via 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} 
+or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable}
+interface.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.ipc.protocolPB">
+</package>
+<package name="org.apache.hadoop.log">
+</package>
+<package name="org.apache.hadoop.log.metrics">
+  <!-- start class org.apache.hadoop.log.metrics.EventCounter -->
+  <class name="EventCounter" extends="org.apache.log4j.AppenderSkeleton"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventCounter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="requiresLayout" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A log4J Appender that simply counts logging events in three levels:
+ fatal, error and warn. The class name is used in log4j.properties]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.log.metrics.EventCounter -->
+</package>
+<package name="org.apache.hadoop.metrics2">
+  <!-- start class org.apache.hadoop.metrics2.AbstractMetric -->
+  <class name="AbstractMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="AbstractMetric" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the metric
+ @param info  about the metric]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="value" return="java.lang.Number"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the metric
+ @return the value of the metric]]>
+      </doc>
+    </method>
+    <method name="type" return="org.apache.hadoop.metrics2.MetricType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the metric
+ @return the type of the metric]]>
+      </doc>
+    </method>
+    <method name="visit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visitor" type="org.apache.hadoop.metrics2.MetricsVisitor"/>
+      <doc>
+      <![CDATA[Accept a visitor interface
+ @param visitor of the metric]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The immutable metric]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.AbstractMetric -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <interface name="MetricsCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param name  of the record
+ @return  a {@link MetricsRecordBuilder} for the record {@code name}]]>
+      </doc>
+    </method>
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param info  of the record
+ @return  a {@link MetricsRecordBuilder} for metrics {@code info}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics collector interface]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsException -->
+  <class name="MetricsException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message
+ @param message for the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message and a cause
+ @param message for the exception
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a cause
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A general metrics exception wrapper]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsException -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsFilter -->
+  <class name="MetricsFilter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <constructor name="MetricsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether to accept the name
+ @param name  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Whether to accept the tag
+ @param tag to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Whether to accept the tags
+ @param tags to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Whether to accept the record
+ @param record  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics filter interface. The MetricsFilter objects can be used either to
+ filter the metrics from {@link MetricsSource}s or to filter metrics per
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsFilter -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <interface name="MetricsInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically name corresponds to annotation {@link Metric#value()} or
+ the name of the class.
+ @return the name of the metric/tag]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically the description corresponds to annotation {@link Metric#about()}
+ or the name of the class.
+ @return the description of the metric/tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to provide immutable metainfo for metrics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <class name="MetricsJsonBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsJsonBuilder" type="org.apache.hadoop.metrics2.MetricsCollector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method]]>
+      </doc>
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Build a JSON dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <interface name="MetricsPlugin"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+      <doc>
+      <![CDATA[Initialize the plugin
+ @param conf  the configuration object for the plugin]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The plugin interface for the metrics framework]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <interface name="MetricsRecord"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="timestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the metrics
+ @return  the timestamp]]>
+      </doc>
+    </method>
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics record name]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the description of the metrics record]]>
+      </doc>
+    </method>
+    <method name="context" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the context name of the metrics record]]>
+      </doc>
+    </method>
+    <method name="tags" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tags of the record
+ Note: returning a collection instead of iterable as we
+ need to use tags as keys (hence Collection#hashCode etc.) in maps
+ @return an unmodifiable collection of tags]]>
+      </doc>
+    </method>
+    <method name="metrics" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the metrics of the record
+ @return an immutable iterable interface for metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An immutable snapshot of metrics with a timestamp]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <class name="MetricsRecordBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRecordBuilder"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics value with metrics information
+ @param info  metadata of the tag
+ @param value of the tag
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Add an immutable metrics tag object
+ @param tag a pre-made tag object (potentially save an object construction)
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+      <doc>
+      <![CDATA[Add a pre-made immutable metric object
+ @param metric  the pre-made metric to save an object construction
+ @return self]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the context tag
+ @param value of the context
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add an integer metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add an long metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add a integer gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a long gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Add a float gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Add a double gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <method name="endRecord" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Syntactic sugar to add multiple records in a collector in a one liner.
+ @return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics record builder interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSink -->
+  <interface name="MetricsSink"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <method name="putMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Put a metrics record in the sink
+ @param record  the record to put]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flush any buffered metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics sink interface. <p>
+ Implementations of this interface consume the {@link MetricsRecord} generated
+ from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ periodically pushes the {@link MetricsRecord} to the sink using
+ {@link #putMetrics(MetricsRecord)} method.  If the implementing class also
+ implements {@link Closeable}, then the MetricsSystem will close the sink when
+ it is stopped.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSink -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSource -->
+  <interface name="MetricsSource"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collector" type="org.apache.hadoop.metrics2.MetricsCollector"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get metrics from the metrics source
+ @param collector to contain the resulting metrics snapshot
+ @param all if true, return all metrics even if unchanged.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The source of metrics information. It generates and updates metrics. It
+ registers with {@link MetricsSystem}, which periodically polls it to collect
+ {@link MetricsRecord} and passes it to {@link MetricsSink}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSource -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsSystem -->
+  <class name="MetricsSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSystemMXBean"/>
+    <constructor name="MetricsSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unregisterSource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unregister a metrics source
+ @param name of the source. This is the name you use to call register()]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source (deriving name and description from the object)
+ @param <T>   the actual type of the source object
+ @param source  object to register
+ @return  the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source
+ @param <T>   the actual type of the source object
+ @param source object to register
+ @param name  of the source. Must be unique or null (then extracted from
+              the annotations of the source object.)
+ @param desc  the description of the source (or null. See above.)
+ @return the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callback" type="org.apache.hadoop.metrics2.MetricsSystem.Callback"/>
+      <doc>
+      <![CDATA[Register a callback interface for JMX events
+ @param callback  the callback object implementing the MBean interface.]]>
+      </doc>
+    </method>
+    <method name="publishMetricsNow"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests an immediate publish of all metrics from sources to sinks.
+ 
+ This is a "soft" request: the expectation is that a best effort will be
+ done to synchronously snapshot the metrics from all the sources and put
+ them in all the sinks (including flushing the sinks) before returning to
+ the caller. If this can't be accomplished in reasonable time it's OK to
+ return to the caller before everything is done.]]>
+      </doc>
+    </method>
+    <method name="shutdown" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system completely (usually during server shutdown.)
+ The MetricsSystemMXBean will be unregistered.
+ @return true if shutdown completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics system interface.
+ 
+ The following components are used for metrics.
+ <ul>
+ <li>{@link MetricsSource} generate and update metrics information.</li>
+ <li>{@link MetricsSink} consume the metrics information</li>
+ </ul>
+ 
+ {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ system. Implementations of {@link MetricsSystem} polls the
+ {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsSystem -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <interface name="MetricsSystemMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="startMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start metrics MBeans
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stopMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop metrics MBeans.
+ Note, it doesn't stop the metrics system control MBean,
+ i.e this interface.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="currentConfig" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the current config
+ Avoided getConfig, as it'll turn into a "Config" attribute,
+ which doesn't support multiple line values in jconsole.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The JMX interface to the metrics system]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsTag -->
+  <class name="MetricsTag" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="MetricsTag" type="org.apache.hadoop.metrics2.MetricsInfo, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the tag with name, description and value
+ @param info  of the tag
+ @param value of the tag]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the tag]]>
+      </doc>
+    </method>
+    <method name="value" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the tag
+ @return  the value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Immutable tag for metrics (for grouping on host/queue/username etc.)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsTag -->
+  <!-- start class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <class name="MetricStringBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricStringBuilder" type="org.apache.hadoop.metrics2.MetricsCollector, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method
+ @param prefix string before each entry
+ @param separator separator between name and value
+ @param suffix suffix after each entry]]>
+      </doc>
+    </constructor>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="tuple" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add any key,val pair to the string, between the prefix and suffix,
+ separated by the separator.
+ @param key key
+ @param value value
+ @return this instance]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Build a string dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.
+
+ Every entry is formatted as
+ {@code prefix + name + separator + value + suffix}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsVisitor -->
+  <interface name="MetricsVisitor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Callback for float value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Callback for double value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A visitor interface for metrics]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsVisitor -->
+</package>
+<package name="org.apache.hadoop.metrics2.annotation">
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metric -->
+  <class name="Metric"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a single metric used to annotate a field or a method
+ in the class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metric -->
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metrics -->
+  <class name="Metrics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a group of metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metrics -->
+</package>
+<package name="org.apache.hadoop.metrics2.filter">
+  <!-- start class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <class name="GlobFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GlobFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A glob pattern filter for metrics.
+
+ The class name is used in metrics config files]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <!-- start class org.apache.hadoop.metrics2.filter.RegexFilter -->
+  <class name="RegexFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A regex pattern filter for metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.RegexFilter -->
+</package>
+<package name="org.apache.hadoop.metrics2.lib">
+  <!-- start class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <class name="DefaultMetricsSystem" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="initialize" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method to initialize the metrics system
+ @param prefix  for the metrics system configuration
+ @return the metrics system instance]]>
+      </doc>
+    </method>
+    <method name="instance" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics system object]]>
+      </doc>
+    </method>
+    <method name="shutdown"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system]]>
+      </doc>
+    </method>
+    <method name="setMiniClusterMode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="choice" type="boolean"/>
+    </method>
+    <method name="inMiniClusterMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The default metrics system singleton. This class is used by all the daemon
+ processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ to initialize the {@link MetricsSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <!-- start class org.apache.hadoop.metrics2.lib.Interns -->
+  <class name="Interns" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Interns"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric info object.
+ @param name Name of metric info object
+ @param description Description of metric info object
+ @return an interned metric info object]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param info  of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to create interned metrics info.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.Interns -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <class name="MetricsRegistry" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRegistry" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a record name
+ @param name  of the record of the metrics]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsRegistry" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a metadata object
+ @param info  the info object for the metrics record/group]]>
+      </doc>
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the metrics registry]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.lib.MutableMetric"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric by name
+ @param name  of the metric
+ @return the metric object]]>
+      </doc>
+    </method>
+    <method name="getTag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a tag by name
+ @param name  of the tag
+ @return the tag object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newQuantiles" return="org.apache.hadoop.metrics2.lib.MutableQuantiles"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="interval" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable metric that estimates quantiles of a stream of values
+ @param name of the metric
+ @param desc metric description
+ @param sampleName of the metric (e.g., "Ops")
+ @param valueName of the metric (e.g., "Time" or "Latency")
+ @param interval rollover interval of estimator in seconds
+ @return a new quantile estimator object
+ @throws MetricsException if interval is not a positive integer]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @param extended    produce extended stat (stdev, min/max etc.) if true.
+ @return a new mutable stat metric object]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @param description of the metric
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric (for throughput measurement)
+ @param name  of the metric
+ @param desc  description
+ @param extended  produce extended stat (stdev/min/max etc.) if true
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRatesWithAggregation" return="org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="newMutableRollingAverages" return="org.apache.hadoop.metrics2.lib.MutableRollingAverages"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add sample to a stat metric by name.
+ @param name  of the metric
+ @param value of the snapshot to add]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the metrics context tag
+ @param name of the context
+ @return the registry itself as a convenience]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @param override  existing tag if true
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param info  metadata of the tag
+ @param value of the tag
+ @param override existing tag if true
+ @return the registry (for keep adding tags etc.)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Sample all the mutable metrics and put the snapshot in the builder
+ @param builder to contain the metrics snapshot
+ @param all get all the metrics even if the values are not changed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An optional metrics registry class for creating and maintaining a
+ collection of MetricsMutables, making writing metrics source easier.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <class name="MutableCounter" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounter" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the metric value by 1.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable counter (monotonically increasing) metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <class name="MutableCounterInt" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable int counter for implementing metrics sources]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <class name="MutableCounterLong" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounterLong" type="org.apache.hadoop.metrics2.MetricsInfo, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable long counter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <class name="MutableGauge" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableGauge" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the value of the metric by 1]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Decrement the value of the metric by 1]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable gauge metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <class name="MutableGaugeInt" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable int gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <class name="MutableGaugeLong" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable long gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <class name="MutableMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="snapshot"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get a snapshot of the metric
+ @param builder the metrics record builder
+ @param all if true, snapshot unchanged metrics as well]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <doc>
+      <![CDATA[Get a snapshot of metric if changed
+ @param builder the metrics record builder]]>
+      </doc>
+    </method>
+    <method name="setChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the changed flag in mutable operations]]>
+      </doc>
+    </method>
+    <method name="clearChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the changed flag in the snapshot operations]]>
+      </doc>
+    </method>
+    <method name="changed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  true if metric is changed since last snapshot/snapshot]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <class name="MutableQuantiles" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableQuantiles" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiates a new {@link MutableQuantiles} for a metric that rolls itself
+ over on the specified time interval.
+ 
+ @param name
+          of the metric
+ @param description
+          long-form textual description of the metric
+ @param sampleName
+          type of items in the stream (e.g., "Ops")
+ @param valueName
+          type of the values
+ @param interval
+          rollover interval (in seconds) of the estimator]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="getInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEstimator" return="org.apache.hadoop.metrics2.util.QuantileEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the quantile estimator.
+
+ @return the quantile estimator]]>
+      </doc>
+    </method>
+    <method name="setEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quantileEstimator" type="org.apache.hadoop.metrics2.util.QuantileEstimator"/>
+    </method>
+    <field name="quantiles" type="org.apache.hadoop.metrics2.util.Quantile[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="previousSnapshot" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Watches a stream of long values, maintaining online estimates of specific
+ quantiles with provably low error bounds. This is particularly useful for
+ accurate high-percentile (e.g. 95th, 99th) latency metrics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <class name="MutableRate" extends="org.apache.hadoop.metrics2.lib.MutableStat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[A convenient mutable metric for throughput measurement]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <class name="MutableRates" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics
+
+ This class synchronizes all accesses to the metrics it
+ contains, so it should not be used in situations where
+ there is high contention on the metrics.
+ {@link MutableRatesWithAggregation} is preferable in that
+ situation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <class name="MutableRatesWithAggregation" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableRatesWithAggregation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initialize the registry with all rate names passed in.
+ This is an alternative to the above init function since this metric
+ can be used more than just for rpc name.
+ @param names the array of all rate names]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric.
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="prefix" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics.
+
+ Each thread will maintain a local rate count, and upon snapshot,
+ these values will be aggregated into a global rate. This class
+ should only be used for long running threads, as any metrics
+ produced between the last snapshot and the death of a thread
+ will be lost. This allows for significantly higher concurrency
+ than {@link MutableRates}. See HADOOP-24420.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <class name="MutableRollingAverages" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="MutableRollingAverages" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for {@link MutableRollingAverages}.
+ @param metricValueName]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="collectThreadLocalStates"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Collects states maintained in {@link ThreadLocal}, if any.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[@param name
+          name of metric
+ @param value
+          value of metric]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStats" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="minSamples" type="long"/>
+      <doc>
+      <![CDATA[Retrieve a map of metric name {@literal ->} (aggregate).
+ Filter out entries that don't have at least minSamples.
+
+ @return a map of peer DataNode Id to the average latency to that
+         node seen over the measurement period.]]>
+      </doc>
+    </method>
+    <method name="setRecordValidityMs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Use for test only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class maintains a group of rolling average metrics. It implements the
+ algorithm of rolling average, i.e. a number of sliding windows are kept to
+ roll over and evict old subsets of samples. Each window has a subset of
+ samples in a stream, where sub-sum and sub-total are collected. All sub-sums
+ and sub-totals in all windows will be aggregated to final-sum and final-total
+ used to compute final average, which is called rolling average.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableStat -->
+  <class name="MutableStat" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a sample statistics metric
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")
+ @param extended    create extended stats (stdev, min/max etc.) by default.]]>
+      </doc>
+    </constructor>
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a snapshot stat metric with extended stat off by default
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")]]>
+      </doc>
+    </constructor>
+    <method name="setExtended"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to display the extended stats (stdev, min/max etc.) or not
+ @param extended enable/disable displaying extended stats]]>
+      </doc>
+    </method>
+    <method name="setUpdateTimeStamp"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateTimeStamp" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to update the snapshot time or not.
+ @param updateTimeStamp enable update stats snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numSamples" type="long"/>
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Add a number of samples and their sum to the running stat
+
+ Note that although use of this method will preserve accurate mean values,
+ large values for numSamples may result in inaccurate variance values due
+ to the use of a single step of the Welford method for variance calculation.
+ @param numSamples  number of samples
+ @param sum of the samples]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a snapshot to the metric.
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="lastStat" return="org.apache.hadoop.metrics2.util.SampleStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a SampleStat object that supports
+ calls like StdDev and Mean.
+ @return SampleStat]]>
+      </doc>
+    </method>
+    <method name="resetMinMax"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset the all time min max of the metric]]>
+      </doc>
+    </method>
+    <method name="getSnapshotTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the SampleStat snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mutable metric with stats.
+
+ Useful for keeping throughput/latency stats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableStat -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink">
+  <!-- start class org.apache.hadoop.metrics2.sink.FileSink -->
+  <class name="FileSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="FileSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a file]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.FileSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <class name="GraphiteSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="GraphiteSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a Graphite server]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <class name="RollingFileSystemSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="RollingFileSystemSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty instance.  Required for reflection.]]>
+      </doc>
+    </constructor>
+    <constructor name="RollingFileSystemSink" type="long, long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance for testing.
+
+ @param flushIntervalMillis the roll interval in millis
+ @param flushOffsetIntervalMillis the roll offset interval in millis]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics2Properties" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="getRollInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Extract the roll interval from the configuration and return it in
+ milliseconds.
+
+ @return the roll interval in millis]]>
+      </doc>
+    </method>
+    <method name="updateFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Update the {@link #nextFlush} variable to the next flush time. Add
+ an integer number of flush intervals, preserving the initial random offset.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="setInitialFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Set the {@link #nextFlush} variable to the initial flush time. The initial
+ flush will be an integer number of flush intervals past the beginning of
+ the current hour and will have a random offset added, up to
+ {@link #rollOffsetIntervalMillis}. The initial flush will be a time in
+ past that can be used from which to calculate future flush times.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="source" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ignoreError" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allowAppend" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="basePath" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollOffsetIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nextFlush" type="java.util.Calendar"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="forceFlush" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="hasFlushed" type="boolean"
+      transient="false" volatile="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedConf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedFilesystem" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>This class is a metrics sink that uses
+ {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs.  Every
+ roll interval a new directory will be created under the path specified by the
+ <code>basepath</code> property. All metrics will be logged to a file in the
+ current interval's directory in a file named &lt;hostname&gt;.log, where
+ &lt;hostname&gt; is the name of the host on which the metrics logging
+ process is running. The base path is set by the
+ <code>&lt;prefix&gt;.sink.&lt;instance&gt;.basepath</code> property.  The
+ time zone used to create the current interval's directory name is GMT.  If
+ the <code>basepath</code> property isn't specified, it will default to
+ &quot;/tmp&quot;, which is the temp directory on whatever default file
+ system is configured for the cluster.</p>
+
+ <p>The <code>&lt;prefix&gt;.sink.&lt;instance&gt;.ignore-error</code>
+ property controls whether an exception is thrown when an error is encountered
+ writing a log file.  The default value is <code>true</code>.  When set to
+ <code>false</code>, file errors are quietly swallowed.</p>
+
+ <p>The <code>roll-interval</code> property sets the amount of time before
+ rolling the directory. The default value is 1 hour. The roll interval may
+ not be less than 1 minute. The property's value should be given as
+ <i>number unit</i>, where <i>number</i> is an integer value, and
+ <i>unit</i> is a valid unit.  Valid units are <i>minute</i>, <i>hour</i>,
+ and <i>day</i>.  The units are case insensitive and may be abbreviated or
+ plural. If no units are specified, hours are assumed. For example,
+ &quot;2&quot;, &quot;2h&quot;, &quot;2 hour&quot;, and
+ &quot;2 hours&quot; are all valid ways to specify two hours.</p>
+
+ <p>The <code>roll-offset-interval-millis</code> property sets the upper
+ bound on a random time interval (in milliseconds) that is used to delay
+ before the initial roll.  All subsequent rolls will happen an integer
+ number of roll intervals after the initial roll, hence retaining the original
+ offset. The purpose of this property is to insert some variance in the roll
+ times so that large clusters using this sink on every node don't cause a
+ performance impact on HDFS by rolling simultaneously.  The default value is
+ 30000 (30s).  When writing to HDFS, as a rule of thumb, the roll offset in
+ millis should be no less than the number of sink instances times 5.
+
+ <p>The primary use of this class is for logging to HDFS.  As it uses
+ {@link org.apache.hadoop.fs.FileSystem} to access the target file system,
+ however, it can be used to write to the local file system, Amazon S3, or any
+ other supported file system.  The base path for the sink will determine the
+ file system used.  An unqualified path will write to the default file system
+ set by the configuration.</p>
+
+ <p>Not all file systems support the ability to append to files.  In file
+ systems without the ability to append to files, only one writer can write to
+ a file at a time.  To allow for concurrent writes from multiple daemons on a
+ single host, the <code>source</code> property is used to set unique headers
+ for the log files.  The property should be set to the name of
+ the source daemon, e.g. <i>namenode</i>.  The value of the
+ <code>source</code> property should typically be the same as the property's
+ prefix.  If this property is not set, the source is taken to be
+ <i>unknown</i>.</p>
+
+ <p>Instead of appending to an existing file, by default the sink
+ will create a new file with a suffix of &quot;.&lt;n&gt;&quot;, where
+ <i>n</i> is the next lowest integer that isn't already used in a file name,
+ similar to the Hadoop daemon logs.  NOTE: the file with the <b>highest</b>
+ sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
+
+ <p>For file systems that allow append, the sink supports appending to the
+ existing file instead. If the <code>allow-append</code> property is set to
+ true, the sink will instead append to the existing file on file systems that
+ support appends. By default, the <code>allow-append</code> property is
+ false.</p>
+
+ <p>Note that when writing to HDFS with <code>allow-append</code> set to true,
+ there is a minimum acceptable number of data nodes.  If the number of data
+ nodes drops below that minimum, the append will succeed, but reading the
+ data will fail with an IOException in the DataStreamer class.  The minimum
+ number of data nodes required for a successful append is generally 2 or
+ 3.</p>
+
+ <p>Note also that when writing to HDFS, the file size information is not
+ updated until the file is closed (at the end of the interval) even though
+ the data is being written successfully. This is a known HDFS limitation that
+ exists because of the performance cost of updating the metadata.  See
+ <a href="https://issues.apache.org/jira/browse/HDFS-5478">HDFS-5478</a>.</p>
+
+ <p>When using this sink in a secure (Kerberos) environment, two additional
+ properties must be set: <code>keytab-key</code> and
+ <code>principal-key</code>. <code>keytab-key</code> should contain the key by
+ which the keytab file can be found in the configuration, for example,
+ <code>yarn.nodemanager.keytab</code>. <code>principal-key</code> should
+ contain the key by which the principal can be found in the configuration,
+ for example, <code>yarn.nodemanager.principal</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.StatsDSink -->
+  <class name="StatsDSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="StatsDSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="writeMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="line" type="java.lang.String"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes metrics to a StatsD daemon.
+ This sink will produce metrics of the form
+ '[hostname].servicename.context.name.metricname:value|type'
+ where hostname is optional. This is useful when sending to
+ a daemon that is running on the localhost and will add the
+ hostname to the metric (such as the
+ <a href="https://collectd.org/">CollectD</a> StatsD plugin).
+ <br>
+ To configure this plugin, you will need to add the following
+ entries to your hadoop-metrics2.properties file:
+ <br>
+ <pre>
+ *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.StatsDSink -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink.ganglia">
+</package>
+<package name="org.apache.hadoop.metrics2.source">
+</package>
+<package name="org.apache.hadoop.metrics2.util">
+  <!-- start class org.apache.hadoop.metrics2.util.MBeans -->
+  <class name="MBeans" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ Where the {@literal <serviceName> and <nameName>} are the supplied
+ parameters.
+
+ @param serviceName
+ @param nameName
+ @param theMbean - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="properties" type="java.util.Map"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ Where the {@literal <serviceName> and <nameName>} are the supplied
+ parameters.
+
+ @param serviceName
+ @param nameName
+ @param properties - Key value pairs to define additional JMX ObjectName
+                     properties.
+ @param theMbean    - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="getMbeanNameService" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="getMbeanNameName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mbeanName" type="javax.management.ObjectName"/>
+    </method>
+    <doc>
+    <![CDATA[This util class provides a method to register an MBean using
+ our standard naming convention as described in the doc
+  for {link {@link #register(String, String, Object)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MBeans -->
+  <!-- start class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <class name="MetricsCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="MetricsCache" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a metrics cache
+ @param maxRecsPerName  limit of the number records per record name]]>
+      </doc>
+    </constructor>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <param name="includingTags" type="boolean"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cached record
+ @param mr the update record
+ @param includingTags cache tag values (for later lookup by name) if true
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cache record
+ @param mr the update record
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="tags" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Get the cached record
+ @param name of the record
+ @param tags of the record
+ @return the cached record or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A metrics cache for sinks that don't support sparse updates.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <!-- start class org.apache.hadoop.metrics2.util.Servers -->
+  <class name="Servers" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parse" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="specs" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Parses a space and/or comma separated sequence of server specifications
+ of the form <i>hostname</i> or <i>hostname:port</i>.  If
+ the specs string is null, defaults to localhost:defaultPort.
+
+ @param specs   server specs (see description)
+ @param defaultPort the default port if not specified
+ @return a list of InetSocketAddress objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to handle server addresses]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.Servers -->
+</package>
+<package name="org.apache.hadoop.net">
+  <!-- start class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <class name="AbstractDNSToSwitchMapping" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.net.DNSToSwitchMapping"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractDNSToSwitchMapping"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an unconfigured instance]]>
+      </doc>
+    </constructor>
+    <constructor name="AbstractDNSToSwitchMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance, caching the configuration file.
+ This constructor does not call {@link #setConf(Configuration)}; if
+ a subclass extracts information in that method, it must call it explicitly.
+ @param conf the configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate that indicates that the switch mapping is known to be
+ single-switch. The base class returns false: it assumes all mappings are
+ multi-rack. Subclasses may override this with methods that are more aware
+ of their topologies.
+
+ <p>
+
+ This method is used when parts of Hadoop need know whether to apply
+ single rack vs multi-rack policies, such as during block placement.
+ Such algorithms behave differently if they are on multi-switch systems.
+ </p>
+
+ @return true if the mapping thinks that it is on a single switch]]>
+      </doc>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the map (for diagnostics)
+ @return a clone of the map or null for none known]]>
+      </doc>
+    </method>
+    <method name="dumpTopology" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a string listing the switch mapping implementation,
+ the mapping for every known node and the number of nodes and
+ unique switches known about -each entry to a separate line.
+ @return a string that can be presented to the ops team or used in
+ debug messages.]]>
+      </doc>
+    </method>
+    <method name="isSingleSwitchByScriptPolicy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMappingSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapping" type="org.apache.hadoop.net.DNSToSwitchMapping"/>
+      <doc>
+      <![CDATA[Query for a {@link DNSToSwitchMapping} instance being on a single
+ switch.
+ <p>
+ This predicate simply assumes that all mappings not derived from
+ this class are multi-switch.
+ @param mapping the mapping to query
+ @return true if the base class says it is single switch, or the mapping
+ is not derived from this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a base class for DNS to Switch mappings. <p> It is not mandatory to
+ derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
+ recommended, as it makes it easy for the Hadoop developers to add new methods
+ to this base class that are automatically picked up by all implementations.
+ <p>
+
+ This class does not extend the <code>Configured</code>
+ base class, and should not be changed to do so, as it causes problems
+ for subclasses. The constructor of the <code>Configured</code> calls
+ the  {@link #setConf(Configuration)} method, which will call into the
+ subclasses before they have been fully constructed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <class name="CachedDNSToSwitchMapping" extends="org.apache.hadoop.net.AbstractDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CachedDNSToSwitchMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[cache a raw DNS mapping
+ @param rawMapping the raw mapping to cache]]>
+      </doc>
+    </constructor>
+    <method name="resolve" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the (host x switch) map.
+ @return a copy of the cached map of hosts to rack]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegate the switch topology query to the raw mapping, via
+ {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+ @return true iff the raw mapper is considered single-switch.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <field name="rawMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The uncached mapping]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A cached implementation of DNSToSwitchMapping that takes an
+ raw DNSToSwitchMapping and stores the resolved network location in 
+ a cache. The following calls to a resolved network location
+ will get its location from the cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ConnectTimeoutException -->
+  <class name="ConnectTimeoutException" extends="java.net.SocketTimeoutException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConnectTimeoutException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown by {@link NetUtils#connect(java.net.Socket, java.net.SocketAddress, int)}
+ if it times out while connecting to the remote host.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ConnectTimeoutException -->
+  <!-- start interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <interface name="DNSToSwitchMapping"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="resolve" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Resolves a list of DNS-names/IP-addresses and returns back a list of
+ switch information (network paths). One-to-one correspondence must be 
+ maintained between the elements in the lists. 
+ Consider an element in the argument list - x.y.com. The switch information
+ that is returned must be a network path of the form /foo/rack, 
+ where / is the root, and 'foo' is the switch where 'rack' is connected.
+ Note the hostname/ip-address is not part of the returned path.
+ The network topology of the cluster would determine the number of
+ components in the network path.
+ <p>
+
+ If a name cannot be resolved to a rack, the implementation
+ should return {@link NetworkTopology#DEFAULT_RACK}. This
+ is what the bundled implementations do, though it is not a formal requirement
+
+ @param names the list of hosts to resolve (can be empty)
+ @return list of resolved network paths.
+ If <i>names</i> is empty, the returned list is also empty]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload all of the cached mappings.
+
+ If there is a cache, this method will clear it, so that future accesses
+ will get a chance to see the new data.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Reload cached mappings on specific nodes.
+
+ If there is a cache on these nodes, this method will clear it, so that 
+ future accesses will see updated data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface that must be implemented to allow pluggable
+ DNS-name/IP-address to RackID resolvers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ScriptBasedMapping -->
+  <class name="ScriptBasedMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ScriptBasedMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with the default configuration.
+ <p>
+ Calling {@link #setConf(Configuration)} will trigger a
+ re-evaluation of the configuration settings and so be used to
+ set up the mapping script.]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given raw mapping
+ @param rawMap raw DNSTOSwithMapping]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given configuration
+ @param conf configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ This will get called in the superclass constructor, so a check is needed
+ to ensure that the raw mapping is defined before trying to relaying a null
+ configuration.
+ @param conf]]>
+      </doc>
+    </method>
+    <field name="NO_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Text used in the {@link #toString()} method if there is no string
+ {@value}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class implements the {@link DNSToSwitchMapping} interface using a 
+ script configured via the
+ {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ <p>
+ It contains a static class <code>RawScriptBasedMapping</code> that performs
+ the work: reading the configuration parameters, executing any defined
+ script, handling errors and such like. The outer
+ class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ queries.
+ <p>
+ This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ true if and only if a script is defined.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ScriptBasedMapping -->
+  <!-- start class org.apache.hadoop.net.SocksSocketFactory -->
+  <class name="SocksSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="SocksSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <constructor name="SocksSocketFactory" type="java.net.Proxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with a supplied Proxy
+ 
+ @param proxy the proxy to use to create sockets]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.SocksSocketFactory -->
+  <!-- start class org.apache.hadoop.net.StandardSocketFactory -->
+  <class name="StandardSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StandardSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.StandardSocketFactory -->
+  <!-- start class org.apache.hadoop.net.TableMapping -->
+  <class name="TableMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TableMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ file. The columns are separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ </p>
+ <p>
+ This class uses the configuration parameter {@code
+ net.topology.table.file.name} to locate the mapping file.
+ </p>
+ <p>
+ Calls to {@link #resolve(List)} will look up the address as defined in the
+ mapping file. If no entry corresponding to the address is found, the value
+ {@code /default-rack} is returned.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.TableMapping -->
+</package>
+<package name="org.apache.hadoop.net.unix">
+</package>
+<package name="org.apache.hadoop.security">
+  <!-- start class org.apache.hadoop.security.AccessControlException -->
+  <class name="AccessControlException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessControlException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor is needed for unwrapping from 
+ {@link org.apache.hadoop.ipc.RemoteException}.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an {@link AccessControlException}
+ with the specified detail message.
+ @param s the detail message.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[An exception class for access control related issues.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.AccessControlException -->
+  <!-- start class org.apache.hadoop.security.Credentials -->
+  <class name="Credentials" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty credentials instance.]]>
+      </doc>
+    </constructor>
+    <constructor name="Credentials" type="org.apache.hadoop.security.Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of the given credentials.
+ @param credentials to copy]]>
+      </doc>
+    </constructor>
+    <method name="getToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the Token object for the alias.
+ @param alias the alias for the Token
+ @return token for this alias]]>
+      </doc>
+    </method>
+    <method name="addToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="t" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token in the storage (in memory).
+ @param alias the alias for the key
+ @param t the token object]]>
+      </doc>
+    </method>
+    <method name="getAllTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the tokens in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getTokenMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to Tokens.]]>
+      </doc>
+    </method>
+    <method name="numberOfTokens" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of Tokens in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="getSecretKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the key bytes for the alias.
+ @param alias the alias for the key
+ @return key for this alias]]>
+      </doc>
+    </method>
+    <method name="numberOfSecretKeys" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of keys in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="addSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Set the key for an alias.
+ @param alias the alias for the key
+ @param key the key bytes]]>
+      </doc>
+    </method>
+    <method name="removeSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Remove the key for a given alias.
+ @param alias the alias for the key]]>
+      </doc>
+    </method>
+    <method name="getAllSecretKeys" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the secret key entries in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getSecretKeyMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to secret keys.]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.io.File"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token from a DataInputStream.]]>
+      </doc>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores all the keys to DataOutput.
+ @param out
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Loads all the keys.
+ @param in
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <method name="mergeAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are not overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that provides the facilities of reading and writing
+ secret keys and Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.Credentials -->
+  <!-- start interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <interface name="GroupMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getGroups" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all various group memberships of a given user.
+ Returns EMPTY list in case of non-existing user
+ @param user User's name
+ @return group memberships of user
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsRefresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the cache of groups and user mapping
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsAdd"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groups" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Caches the group user information
+ @param groups list of groups to add to cache
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="GROUP_MAPPING_CONFIG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An interface for the implementation of a user-to-groups mapping service
+ used by {@link Groups}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <!-- start interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <interface name="IdMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getUid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getGid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getGroupName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="gid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getUidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getGidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[An interface for the implementation of {@literal <}userId,
+ userName{@literal >} mapping and {@literal <}groupId, groupName{@literal >}
+ mapping.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <!-- start class org.apache.hadoop.security.KerberosAuthException -->
+  <class name="KerberosAuthException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosAuthException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="u" type="java.lang.String"/>
+    </method>
+    <method name="setPrincipal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+    </method>
+    <method name="setKeytabFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="java.lang.String"/>
+    </method>
+    <method name="setTicketCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.String"/>
+    </method>
+    <method name="getInitialMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The initial message, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getKeytabFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The keytab file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The principal, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getTicketCacheFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The ticket cache file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The user, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown when {@link UserGroupInformation} failed with an unrecoverable error,
+ such as failure in kerberos login/logout, invalid subject etc.
+
+ Caller should not retry when catching this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.KerberosAuthException -->
+  <!-- start class org.apache.hadoop.security.SecurityUtil -->
+  <class name="SecurityUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOriginalTGT" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ticket" type="javax.security.auth.kerberos.KerberosTicket"/>
+      <doc>
+      <![CDATA[Check whether the server principal is the TGS's principal
+ @param ticket the original TGT (the ticket that is obtained when a 
+ kinit is done)
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal
+ names. It replaces hostname pattern with hostname, which should be
+ fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
+ dynamically looked-up fqdn of the current host instead.
+ 
+ @param principalConfig
+          the Kerberos principal name conf value to convert
+ @param hostname
+          the fully-qualified domain name used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal names.
+ This method is similar to {@link #getServerPrincipal(String, String)},
+ except 1) the reverse DNS lookup from addr to hostname is done only when
+ necessary, 2) param addr can't be null (no default behavior of using local
+ hostname when addr is null).
+ 
+ @param principalConfig
+          Kerberos principal name pattern to convert
+ @param addr
+          InetAddress of the host used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in
+ user's Kerberos principal name with a dynamically looked-up fully-qualified
+ domain name of the current host.
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in user's Kerberos principal 
+ name with hostname. If non-secure mode - return. If no keytab available -
+ bail out with an exception
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @param hostname
+          hostname to use for substitution
+ @throws IOException if the config doesn't specify a keytab]]>
+      </doc>
+    </method>
+    <method name="buildDTServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="defPort" type="int"/>
+      <doc>
+      <![CDATA[create the service name for a Delegation token
+ @param uri of the service
+ @param defPort is used if the uri lacks a port
+ @return the token service, or null if no authority
+ @see #buildTokenService(InetSocketAddress)]]>
+      </doc>
+    </method>
+    <method name="getHostFromPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the host name from the principal name of format {@literal <}service
+ {@literal >}/host@realm.
+ @param principalName principal name of format as described above
+ @return host name if the the string conforms to the above format, else null]]>
+      </doc>
+    </method>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the KerberosInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return the KerberosInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getClientPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the client principal for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return client principal or null if it has no client principal defined.]]>
+      </doc>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the TokenInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol The protocol class to get the information for.
+ @param conf Configuration object
+ @return the TokenInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getTokenServiceAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Decode the given token's service field into an InetAddress
+ @param token from which to obtain the service
+ @return InetAddress for the service]]>
+      </doc>
+    </method>
+    <method name="setTokenService"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the given token's service to the format expected by the RPC client 
+ @param token a delegation token
+ @param addr the socket for the rpc connection]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param addr InetSocketAddress of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param uri of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUserOrFatal" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If the login
+ user cannot be determined, this will log a FATAL error and exit
+ the whole JVM.]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="doAsCurrentUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's current user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticationMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isPrivilegedPort" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+    </method>
+    <method name="getZKAuthInfos" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="configKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility method to fetch ZK auth info from the configuration.
+ @throws java.io.IOException if the Zookeeper ACLs configuration file
+ cannot be read
+ @throws ZKUtil.BadAuthFormatException if the auth format is invalid]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HOSTNAME_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED_TO_GET_UGI_MSG_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Security Utils.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.SecurityUtil -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation -->
+  <class name="UserGroupInformation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setShouldRenewImmediatelyForTests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="immediate" type="boolean"/>
+      <doc>
+      <![CDATA[For the purposes of unit tests, we want to test login
+ from keytab and don't want to wait until the renew
+ window (controlled by TICKET_RENEW_WINDOW).
+ @param immediate true if we should login without waiting for ticket window]]>
+      </doc>
+    </method>
+    <method name="reattachMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reattach the class's metrics to a new metric system.]]>
+      </doc>
+    </method>
+    <method name="isInitialized" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the static configuration for UGI.
+ In particular, set the security authentication mechanism and the
+ group look up service.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="isSecurityEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Determine if UserGroupInformation is using Kerberos to determine
+ user identities or is relying on simple authentication
+ 
+ @return true if UGI is working in a secure environment]]>
+      </doc>
+    </method>
+    <method name="hasKerberosCredentials" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[checks if logged in using kerberos
+ @return true if the subject logged via keytab or has a Kerberos TGT]]>
+      </doc>
+    </method>
+    <method name="getCurrentUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current user, including any doAs in the current stack.
+ @return the current user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="getBestUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCachePath" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the most appropriate UserGroupInformation to use
+
+ @param ticketCachePath    The Kerberos ticket cache path, or NULL
+                           if none is specfied
+ @param user               The user name, or NULL if none is specified.
+
+ @return                   The most appropriate UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getUGIFromTicketCache" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCache" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Kerberos ticket cache.
+ 
+ @param user                The principal name to load from the ticket
+                            cache
+ @param ticketCache     the path to the ticket cache file
+
+ @throws IOException        if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getUGIFromSubject" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Subject with Kerberos principal.
+
+ @param subject             The KerberosPrincipal to use in UGI.
+                            The creator of subject is responsible for
+                            renewing credentials.
+
+ @throws IOException
+ @throws KerberosAuthException if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getLoginUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the currently logged in user.  If no explicit login has occurred,
+ the user will automatically be logged in with either kerberos credentials
+ if available, or as the local OS user, based on security settings.
+ @return the logged in user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="trimLoginMethod" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[remove the login method that is followed by a space from the username
+ e.g. "jack (auth:SIMPLE)" {@literal ->} "jack"
+
+ @param userName
+ @return userName without login method]]>
+      </doc>
+    </method>
+    <method name="loginUserFromSubject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log in a user using the given subject
+ @param subject the subject to use when logging in a user, or null to
+ create a new subject.
+
+ If subject is not null, the creator of subject is responsible for renewing
+ credentials.
+
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="isFromKeytab" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this user logged in from a keytab file managed by the UGI?
+ @return true if the credentials are from a keytab file.]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="logoutUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log the current user out who previously logged in using keytab.
+ This method assumes that the user logged in by calling
+ {@link #loginUserFromKeytab(String, String)}.
+
+ @throws IOException
+ @throws KerberosAuthException if a failure occurred in logout,
+ or if the user did not log in by invoking loginUserFromKeyTab() before.]]>
+      </doc>
+    </method>
+    <method name="checkTGTAndReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-login a user from keytab if TGT is expired or is close to expiry.
+ 
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="reloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user. This
+ method assumes that {@link #loginUserFromKeytab(String, String)} had
+ happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="forceReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Force re-Login a user in from a keytab file irrespective of the last login
+ time. Loads a user identity from a keytab file and logs them in. They
+ become the currently logged-in user. This method assumes that
+ {@link #loginUserFromKeytab(String, String)} had happened already. The
+ Subject field of this UserGroupInformation object is updated to have the
+ new credentials.
+
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="reloginFromTicketCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from the ticket cache.  This
+ method assumes that login had happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytabAndReturnUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and login them in. This new user does not affect the currently
+ logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException if the keytab file can't be read]]>
+      </doc>
+    </method>
+    <method name="isLoginKeytabBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via keytab
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="isLoginTicketBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via ticket cache
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createProxyUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create a proxy user using username of the effective user and the ugi of the
+ real user.
+ @param user
+ @param realUser
+ @return proxyUser ugi]]>
+      </doc>
+    </method>
+    <method name="getRealUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get RealUser (vs. EffectiveUser)
+ @return realUser running over proxy user]]>
+      </doc>
+    </method>
+    <method name="createUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a UGI for testing HDFS and MapReduce
+ @param user the full user principal name
+ @param userGroups the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="createProxyUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a proxy user UGI for testing HDFS and MapReduce
+ 
+ @param user
+          the full user principal name for effective user
+ @param realUser
+          UGI of the real user
+ @param userGroups
+          the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="getShortUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's login name.
+ @return the user's name up to the first '/' or '@'.]]>
+      </doc>
+    </method>
+    <method name="getPrimaryGroupName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's full principal name.
+ @return the user's full principal name.]]>
+      </doc>
+    </method>
+    <method name="addTokenIdentifier" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokenId" type="org.apache.hadoop.security.token.TokenIdentifier"/>
+      <doc>
+      <![CDATA[Add a TokenIdentifier to this UGI. The TokenIdentifier has typically been
+ authenticated by the RPC layer as belonging to the user represented by this
+ UGI.
+ 
+ @param tokenId
+          tokenIdentifier to be added
+ @return true on successful add of new tokenIdentifier]]>
+      </doc>
+    </method>
+    <method name="getTokenIdentifiers" return="java.util.Set"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of TokenIdentifiers belonging to this UGI
+ 
+ @return the set of TokenIdentifiers belonging to this UGI]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token to this UGI
+ 
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a named token to this UGI
+ 
+ @param alias Name of the token
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="getTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the collection of tokens associated with this user.
+ 
+ @return an unmodifiable collection of tokens associated with user]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the tokens in credentials form associated with this user.
+ 
+ @return Credentials of tokens associated with this user]]>
+      </doc>
+    </method>
+    <method name="addCredentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="credentials" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Add the given Credentials to this user.
+ @param credentials of tokens and secrets]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user. {@link #getGroups()} is less
+ expensive alternative when checking for a contained element.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the username.]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the subject
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the real user's subject.  If there
+ is no real user, return the given user's authentication method.
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Returns the authentication method of a ugi. If the authentication method is
+ PROXY, returns the authentication method of the real user.
+ 
+ @param ugi
+ @return AuthenticationMethod]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare the subjects to see if they are equal to each other.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the hash of the subject.]]>
+      </doc>
+    </method>
+    <method name="getSubject" return="javax.security.auth.Subject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying subject from this ugi.
+ @return the subject that represents this user.]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Run the given action as the user.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the given action as the user, potentially throwing an exception.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method
+ @throws IOException if the action throws an IOException
+ @throws Error if the action throws an Error
+ @throws RuntimeException if the action throws a RuntimeException
+ @throws InterruptedException if the action throws an InterruptedException
+ @throws UndeclaredThrowableException if the action throws something else]]>
+      </doc>
+    </method>
+    <method name="logAllUserInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log all (current, real, login) UGI and token info into UGI debug log.
+ @param ugi - UGI
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A test method to print out the current user's UGI.
+ @param args if there are two arguments, read the user from the keytab
+ and print it out.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <field name="HADOOP_TOKEN_FILE_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the token cache file]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the base64 tokens.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[User and group information for Hadoop.
+ This class wraps around a JAAS Subject and provides methods to determine the
+ user's username and groups. It supports both the Windows, Unix and Kerberos 
+ login modules.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+  <class name="UserGroupInformation.AuthenticationMethod" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getAuthMethod" return="org.apache.hadoop.security.SaslRpcServer.AuthMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+    </method>
+    <doc>
+    <![CDATA[existing types of authentications' methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+</package>
+<package name="org.apache.hadoop.security.alias">
+  <!-- start class org.apache.hadoop.security.alias.CredentialProvider -->
+  <class name="CredentialProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide job access to
+ passwords rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the credentials are written to persistent
+ store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the credential entry for a specific alias.
+ @param alias the name of a specific credential
+ @return the credentialEntry
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAliases" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the aliases for all credentials.
+ @return the list of alias names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="credential" type="char[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new credential. The given alias must not already exist.
+ @param name the alias of the credential
+ @param credential the credential value for the alias.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteCredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given credential.
+ @param name the alias of the credential to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of credentials or password for Hadoop applications. Provides an
+ abstraction to separate credential storage from users of them. It
+ is intended to support getting or storing passwords in a variety of ways,
+ including third party bindings.
+ 
+ <code>CredentialProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProvider -->
+  <!-- start class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+  <class name="CredentialProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.security.alias.CredentialProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of CredentialProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ CredentialProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+</package>
+<package name="org.apache.hadoop.security.authentication.server">
+</package>
+<package name="org.apache.hadoop.security.authorize">
+  <!-- start class org.apache.hadoop.security.authorize.AccessControlList -->
+  <class name="AccessControlList" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="AccessControlList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constructor exists primarily for AccessControlList to be Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from a String representation of the same.
+ 
+ The String is a a comma separated list of users and groups.
+ The user list comes first and is separated by a space followed 
+ by the group list. For e.g. "user1,user2 group1,group2"
+ 
+ @param aclString String representation of the ACL]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from String representation of users and groups
+ 
+ The arguments are comma separated lists
+ 
+ @param users comma separated list of users
+ @param groups comma separated list of groups]]>
+      </doc>
+    </constructor>
+    <method name="isAllAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add user to the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="addGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add group to the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="removeUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove user from the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="removeGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove group from the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="getUsers" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of users allowed for this service.
+ @return the set of user names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of user groups allowed for this service.
+ @return the set of group names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="isUserInList" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Checks if a user represented by the provided {@link UserGroupInformation}
+ is a member of the Access Control List. If user was proxied and
+ USE_REAL_ACLS + the real user name is in the control list, then treat this
+ case as if user were in the ACL list.
+ @param ugi UserGroupInformation to check if contained in the ACL
+ @return true if ugi is member of the list or if USE_REAL_ACLS + real user
+ is in the list]]>
+      </doc>
+    </method>
+    <method name="isUserAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns descriptive way of users and groups that are part of this ACL.
+ Use {@link #getAclString()} to get the exact String that can be given to
+ the constructor of AccessControlList to create a new instance.]]>
+      </doc>
+    </method>
+    <method name="getAclString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the access control list as a String that can be used for building a
+ new instance by sending it to the constructor of {@link AccessControlList}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <field name="WILDCARD_ACL_VALUE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_REAL_ACLS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Class representing a configured access control list.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AccessControlList -->
+  <!-- start class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <class name="AuthorizationException" extends="org.apache.hadoop.security.AccessControlException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuthorizationException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <method name="getStackTrace" return="java.lang.StackTraceElement[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintStream"/>
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintWriter"/>
+    </method>
+    <doc>
+    <![CDATA[An exception class for authorization-related issues.
+ 
+ This class <em>does not</em> provide the stack trace for security purposes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <!-- start class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <class name="DefaultImpersonationProvider" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authorize.ImpersonationProvider"/>
+    <constructor name="DefaultImpersonationProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTestProvider" return="org.apache.hadoop.security.authorize.DefaultImpersonationProvider"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+    </method>
+    <method name="getProxySuperuserUserConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective usergroups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser usergroups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserGroupConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective groups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser groups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserIpConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return configuration key for superuser ip addresses
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser ip-addresses]]>
+      </doc>
+    </method>
+    <method name="getProxyGroups" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProxyHosts" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <!-- start interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+  <interface name="ImpersonationProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Specifies the configuration prefix for the proxy user properties and
+ initializes the provider.
+
+ @param configurationPrefix the configuration prefix for the proxy user
+ properties]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.lang.String"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+ {@link #authorize(UserGroupInformation, InetAddress)} should
+             be preferred to avoid possibly re-resolving the ip address.
+ @param user ugi of the effective or proxy user which contains a real user.
+ @param remoteAddress the ip address of client.
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+
+ @param user ugi of the effective or proxy user which contains a real user
+ @param remoteAddress the ip address of client
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+</package>
+<package name="org.apache.hadoop.security.http">
+  <!-- start class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <class name="RestCsrfPreventionFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="RestCsrfPreventionFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filterConfig" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="isBrowser" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userAgent" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This method interrogates the User-Agent String and returns whether it
+ refers to a browser.  If its not a browser, then the requirement for the
+ CSRF header will not be enforced; if it is a browser, the requirement will
+ be enforced.
+ <p>
+ A User-Agent String is considered to be a browser if it matches
+ any of the regex patterns from browser-useragent-regex; the default
+ behavior is to consider everything a browser that matches the following:
+ "^Mozilla.*,^Opera.*".  Subclasses can optionally override
+ this method to use different behavior.
+
+ @param userAgent The User-Agent String, or null if there isn't one
+ @return true if the User-Agent String refers to a browser, false if not]]>
+      </doc>
+    </method>
+    <method name="handleHttpInteraction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="httpInteraction" type="org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+      <doc>
+      <![CDATA[Handles an {@link HttpInteraction} by applying the filtering logic.
+
+ @param httpInteraction caller's HTTP interaction
+ @throws IOException if there is an I/O error
+ @throws ServletException if the implementation relies on the servlet API
+     and a servlet API call has failed]]>
+      </doc>
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="javax.servlet.ServletRequest"/>
+      <param name="response" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="HEADER_USER_AGENT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BROWSER_USER_AGENT_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_METHODS_TO_IGNORE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HEADER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter provides protection against cross site request forgery (CSRF)
+ attacks for REST APIs. Enabling this filter on an endpoint results in the
+ requirement of all client to send a particular (configurable) HTTP header
+ with every request. In the absense of this header the filter will reject the
+ attempt as a bad request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <!-- start class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+  <class name="XFrameOptionsFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="XFrameOptionsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="javax.servlet.ServletRequest"/>
+      <param name="res" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="X_FRAME_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter protects webapps from clickjacking attacks that
+ are possible through use of Frames to embed the resources in another
+ application and intercept clicks to accomplish nefarious things.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+</package>
+<package name="org.apache.hadoop.security.protocolPB">
+</package>
+<package name="org.apache.hadoop.security.ssl">
+</package>
+<package name="org.apache.hadoop.security.token">
+  <!-- start class org.apache.hadoop.security.token.SecretManager -->
+  <class name="SecretManager" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createPassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <doc>
+      <![CDATA[Create the password for the given identifier.
+ identifier may be modified inside this method.
+ @param identifier the identifier to use
+ @return the new password]]>
+      </doc>
+    </method>
+    <method name="retrievePassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <doc>
+      <![CDATA[Retrieve the password for the given token identifier. Should check the date
+ or registry to make sure the token hasn't expired or been revoked. Returns 
+ the relevant password.
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid]]>
+      </doc>
+    </method>
+    <method name="retriableRetrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <exception name="RetriableException" type="org.apache.hadoop.ipc.RetriableException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The same functionality with {@link #retrievePassword}, except that this 
+ method can throw a {@link RetriableException} or a {@link StandbyException}
+ to indicate that client can retry/failover the same operation because of 
+ temporary issue on the server side.
+ 
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid
+ @throws StandbyException the server is in standby state, the client can
+         try other servers
+ @throws RetriableException the token was invalid, and the server thinks 
+         this may be a temporary issue and suggests the client to retry
+ @throws IOException to allow future exceptions to be added without breaking
+         compatibility]]>
+      </doc>
+    </method>
+    <method name="createIdentifier" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty token identifier.
+ @return the newly created empty token identifier]]>
+      </doc>
+    </method>
+    <method name="checkAvailableForRead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <doc>
+      <![CDATA[No-op if the secret manager is available for reading tokens, throw a
+ StandbyException otherwise.
+ 
+ @throws StandbyException if the secret manager is not available to read
+         tokens]]>
+      </doc>
+    </method>
+    <method name="generateSecret" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a new random secret key.
+ @return the new key]]>
+      </doc>
+    </method>
+    <method name="createPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="byte[]"/>
+      <param name="key" type="javax.crypto.SecretKey"/>
+      <doc>
+      <![CDATA[Compute HMAC of the identifier using the secret key and return the 
+ output as password
+ @param identifier the bytes of the identifier
+ @param key the secret key
+ @return the bytes of the generated password]]>
+      </doc>
+    </method>
+    <method name="createSecretKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Convert the byte[] to a secret key
+ @param key the byte[] to create a secret key from
+ @return the secret key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The server-side secret manager for each token type.
+ @param <T> The type of the token identifier]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.SecretManager -->
+  <!-- start class org.apache.hadoop.security.token.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Token" type="T, org.apache.hadoop.security.token.SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token given a token identifier and a secret manager for the
+ type of the token identifier.
+ @param id the token identifier
+ @param mgr the secret manager]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="byte[], byte[], org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token from the components.
+ @param identifier the token identifier
+ @param password the token's password
+ @param kind the kind of token
+ @param service the service for this token]]>
+      </doc>
+    </constructor>
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="org.apache.hadoop.security.token.Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clone a token.
+ @param other the token to clone]]>
+      </doc>
+    </constructor>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+    </method>
+    <method name="setPassword"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newPassword" type="byte[]"/>
+    </method>
+    <method name="copyToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdentifier" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier's byte representation.
+ @return the token identifier's byte representation]]>
+      </doc>
+    </method>
+    <method name="decodeIdentifier" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the token identifier object, or null if it could not be constructed
+ (because the class could not be loaded, for example).
+ @return the token identifier, or null if there was no class found for it
+ @throws IOException failure to unmarshall the data
+ @throws RuntimeException if the token class could not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password/secret.
+ @return the token password/secret]]>
+      </doc>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service on which the token is supposed to be used.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="setService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Set the service on which the token is supposed to be used.
+ @param newService the service name]]>
+      </doc>
+    </method>
+    <method name="isPrivate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether this is a private token.
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="isPrivateCloneOf" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thePublicService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Whether this is a private clone of a public token.
+ @param thePublicService the public service name
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="privateClone" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Create a private clone of a public token.
+ @param newService the new service name
+ @return a private token]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="encodeToUrlString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode this token as a url safe string.
+ @return the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="decodeFromUrlString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode the given url safe string into this token.
+ @param newValue the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="buildCacheKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is this token managed so that it can be renewed or cancelled?
+ @return true, if it can be renewed and cancelled.]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew this delegation token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel this delegation token.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The client-side form of the token.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token -->
+  <!-- start class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <class name="Token.TrivialRenewer" extends="org.apache.hadoop.security.token.TokenRenewer"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrivialRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="handleKind" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[A trivial renewer for token kinds that aren't managed. Sub-classes need
+ to implement getKind for their token kind.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <!-- start class org.apache.hadoop.security.token.TokenIdentifier -->
+  <class name="TokenIdentifier" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Ugi with the username encoded in the token identifier
+ 
+ @return the username. null is returned if username in the identifier is
+         empty or null.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes for the token identifier
+ @return the bytes of the identifier]]>
+      </doc>
+    </method>
+    <method name="getTrackingId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a tracking identifier that can be used to associate usages of a
+ token across multiple client sessions.
+
+ Currently, this function just returns an MD5 of {{@link #getBytes()}.
+
+ @return tracking identifier]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An identifier that identifies a token, may contain public information 
+ about a token, including its kind (or type).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenIdentifier -->
+  <!-- start class org.apache.hadoop.security.token.TokenInfo -->
+  <class name="TokenInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Indicates Token related information to be used]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenInfo -->
+  <!-- start class org.apache.hadoop.security.token.TokenRenewer -->
+  <class name="TokenRenewer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="handleKind" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Does this renewer handle this kind of token?
+ @param kind the kind of the token
+ @return true if this renewer can renew it]]>
+      </doc>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is the given token managed? Only managed tokens may be renewed or
+ cancelled.
+ @param token the token being checked
+ @return true if the token may be renewed or cancelled
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew the given token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel the given token
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for plugins that handle tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenRenewer -->
+  <!-- start interface org.apache.hadoop.security.token.TokenSelector -->
+  <interface name="TokenSelector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+    <doc>
+    <![CDATA[Select token of type T from tokens for use with named service
+ 
+ @param <T>
+          T extends TokenIdentifier]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.token.TokenSelector -->
+</package>
+<package name="org.apache.hadoop.security.token.delegation.web">
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <class name="DelegationTokenAuthenticatedURL" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DelegationTokenAuthenticatedURL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+ <p>
+ An instance of the default {@link DelegationTokenAuthenticator} will be
+ used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code> using the default
+ {@link DelegationTokenAuthenticator} class.
+
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator, org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <method name="setDefaultDelegationTokenAuthenticator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticator" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Sets the default {@link DelegationTokenAuthenticator} class to use when an
+ {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @param authenticator the authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="getDefaultDelegationTokenAuthenticator" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the default {@link DelegationTokenAuthenticator} class to use when
+ an {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+ <p>
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @return the delegation token authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="setUseQueryStringForDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="useQueryString" type="boolean"/>
+      <doc>
+      <![CDATA[Sets if delegation token should be transmitted in the URL query string.
+ By default it is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ <p>
+ This method is provided to enable WebHDFS backwards compatibility.
+
+ @param useQueryString  <code>TRUE</code> if the token is transmitted in the
+ URL query string, <code>FALSE</code> if the delegation token is transmitted
+ using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ header.]]>
+      </doc>
+    </method>
+    <method name="useQueryStringForDelegationToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns if delegation token is transmitted as a HTTP header.
+
+ @return <code>TRUE</code> if the token is transmitted in the URL query
+ string, <code>FALSE</code> if the delegation token is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
+ Token only if the given auth token is an instance of {@link Token} and
+ it contains a Delegation Token, otherwise use the configured
+ {@link DelegationTokenAuthenticator} to authenticate the connection.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAs" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>. If the <code>doAs</code> parameter is not NULL,
+ the request will be done on behalf of the specified <code>doAs</code> user.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @param doAs user to do the the request on behalf of, if NULL the request is
+ as self.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The <code>DelegationTokenAuthenticatedURL</code> is a
+ {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
+ functionality.
+ <p>
+ The authentication mechanisms supported by default are Hadoop Simple
+ authentication (also known as pseudo authentication) and Kerberos SPNEGO
+ authentication.
+ <p>
+ Additional authentication mechanisms can be supported via {@link
+ DelegationTokenAuthenticator} implementations.
+ <p>
+ The default {@link DelegationTokenAuthenticator} is the {@link
+ KerberosDelegationTokenAuthenticator} class which supports
+ automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
+ the {@link PseudoDelegationTokenAuthenticator} class.
+ <p>
+ <code>AuthenticatedURL</code> instances are not thread-safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <class name="DelegationTokenAuthenticatedURL.Token" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delegationToken" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <doc>
+    <![CDATA[Client side authentication token that handles Delegation Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <class name="DelegationTokenAuthenticator" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authentication.client.Authenticator"/>
+    <constructor name="DelegationTokenAuthenticator" type="org.apache.hadoop.security.authentication.client.Authenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConnectionConfigurator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurator" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"/>
+    </method>
+    <method name="authenticate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <field name="OP_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TOKEN_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEWER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_URL_STRING_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEW_DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@link Authenticator} wrapper that enhances an {@link Authenticator} with
+ Delegation Token support.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <class name="KerberosDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>KerberosDelegationTokenAuthenticator</code> provides support for
+ Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
+ Token operations.
+ <p>
+ It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
+ endpoint does not trigger a SPNEGO authentication]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+  <class name="PseudoDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PseudoDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>PseudoDelegationTokenAuthenticator</code> provides support for
+ Hadoop's pseudo authentication mechanism that accepts
+ the user name specified as a query string parameter and support for Hadoop
+ Delegation Token operations.
+ <p>
+ This mimics the model of Hadoop Simple authentication trusting the
+ {@link UserGroupInformation#getCurrentUser()} value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+</package>
+<package name="org.apache.hadoop.service">
+  <!-- start class org.apache.hadoop.service.AbstractService -->
+  <class name="AbstractService" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <constructor name="AbstractService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the service.
+ @param name service name]]>
+      </doc>
+    </constructor>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration for this service.
+ This method is called during {@link #init(Configuration)}
+ and should only be needed if for some reason a service implementation
+ needs to override that initial setting -for example replacing
+ it with a new subclass of {@link Configuration}
+ @param conf new configuration.]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ This invokes {@link #serviceInit}
+ @param conf the configuration of the service. This must not be null
+ @throws ServiceStateException if the configuration was null,
+ the state change not permitted, or something else went wrong]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws ServiceStateException if the current service state does not permit
+ this action]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Relay to {@link #stop()}
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noteFailure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <param name="exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Failure handling: record the exception
+ that triggered it -if there was not one already.
+ Services are free to call this themselves.
+ @param exception the exception]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[All initialization code needed by a service.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #init(Configuration)} prevents re-entrancy.
+
+ The base implementation checks to see if the subclass has created
+ a new configuration instance, and if so, updates the base class value
+ @param conf configuration
+ @throws Exception on a failure -these will be caught,
+ possibly wrapped, and will trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the INITED to STARTED transition.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #start()} prevents re-entrancy.
+
+ @throws Exception if needed -these will be caught,
+ wrapped, and trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the transition to the STOPPED state.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #stop()} prevents re-entrancy.
+
+ Implementations MUST write this to be robust against failures, including
+ checks for null references -and for the first failure to not stop other
+ attempts to shut down parts of the service.
+
+ @throws Exception if needed -these will be caught and logged.]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="registerGlobalListener"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a global listener, which receives notifications
+ from the state change events of all services in the JVM
+ @param l listener]]>
+      </doc>
+    </method>
+    <method name="unregisterGlobalListener" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[unregister a global listener.
+ @param l listener to unregister
+ @return true if the listener was found (and then deleted)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="expected" type="org.apache.hadoop.service.Service.STATE"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="putBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="details" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a blocker to the blocker map -replacing any
+ with the same name.
+ @param name blocker name
+ @param details any specifics on the block. This must be non-null.]]>
+      </doc>
+    </method>
+    <method name="removeBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a blocker from the blocker map -
+ this is a no-op if the blocker is not present
+ @param name the name of the blocker]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This is the base implementation class for services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.AbstractService -->
+  <!-- start class org.apache.hadoop.service.CompositeService -->
+  <class name="CompositeService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getServices" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a cloned list of services
+ @return a list of child services at the time of invocation -
+ added services will not be picked up.]]>
+      </doc>
+    </method>
+    <method name="addService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Add the passed {@link Service} to the list of services managed by this
+ {@link CompositeService}
+ @param service the {@link Service} to be added]]>
+      </doc>
+    </method>
+    <method name="addIfService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="object" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[If the passed object is an instance of {@link Service},
+ add it to the list of services managed by this {@link CompositeService}
+ @param object
+ @return true if a service is added, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="removeService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="STOP_ONLY_STARTED_SERVICES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Policy on shutdown: attempt to close everything (purest) or
+ only try to close started services (which assumes
+ that the service implementations may not handle the stop() operation
+ except when started.
+ Irrespective of this policy, if a child service fails during
+ its init() or start() operations, it will have stop() called on it.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Composition of services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.CompositeService -->
+  <!-- start class org.apache.hadoop.service.LifecycleEvent -->
+  <class name="LifecycleEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="LifecycleEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="time" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Local time in milliseconds when the event occurred]]>
+      </doc>
+    </field>
+    <field name="state" type="org.apache.hadoop.service.Service.STATE"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[new state]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A serializable lifecycle event: the time a state
+ transition occurred, and what state was entered.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LifecycleEvent -->
+  <!-- start class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <class name="LoggingStateChangeListener" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    <constructor name="LoggingStateChangeListener" type="org.slf4j.Logger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the given log
+ @param log destination for events]]>
+      </doc>
+    </constructor>
+    <constructor name="LoggingStateChangeListener"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the static log for this class]]>
+      </doc>
+    </constructor>
+    <method name="stateChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback for a state change event: log it
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a state change listener that logs events at INFO level]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <!-- start interface org.apache.hadoop.service.Service -->
+  <interface name="Service"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Initialize the service.
+
+ The transition MUST be from {@link STATE#NOTINITED} to {@link STATE#INITED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @param config the configuration of the service
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the service.
+
+ The transition MUST be from {@link STATE#INITED} to {@link STATE#STARTED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the service. This MUST be a no-op if the service is already
+ in the {@link STATE#STOPPED} state. It SHOULD be a best-effort attempt
+ to stop all parts of the service.
+
+ The implementation must be designed to complete regardless of the service
+ state, including the initialized/uninitialized state of all its internal
+ fields.
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A version of stop() that is designed to be usable in Java7 closure
+ clauses.
+ Implementation classes MUST relay this directly to {@link #stop()}
+ @throws IOException never
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a listener to the service state change events.
+ If the supplied listener is already listening to this service,
+ this method is a no-op.
+ @param listener a new listener]]>
+      </doc>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Unregister a previously registered listener of the service state
+ change events. No-op if the listener is already unregistered.
+ @param listener the listener to unregister.]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this service.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration of this service.
+ This is normally not a clone and may be manipulated, though there are no
+ guarantees as to what the consequences of such actions may be
+ @return the current configuration, unless a specific implentation chooses
+ otherwise.]]>
+      </doc>
+    </method>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current service state
+ @return the state of the service]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service start time
+ @return the start time of the service. This will be zero if the service
+ has not yet been started.]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query to see if the service is in a specific state.
+ In a multi-threaded system, the state may not hold for very long.
+ @param state the expected state
+ @return true if, at the time of invocation, the service was in that state.]]>
+      </doc>
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the first exception raised during the service failure. If null,
+ no exception was logged
+ @return the failure logged during a transition to the stopped state]]>
+      </doc>
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the state in which the failure in {@link #getFailureCause()} occurred.
+ @return the state or null if there was no failure]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+      <doc>
+      <![CDATA[Block waiting for the service to stop; uses the termination notification
+ object to do so.
+
+ This method will only return after all the service stop actions
+ have been executed (to success or failure), or the timeout elapsed
+ This method can be called before the service is inited or started; this is
+ to eliminate any race condition with the service stopping before
+ this event occurs.
+ @param timeout timeout in milliseconds. A value of zero means "forever"
+ @return true iff the service stopped in the time period]]>
+      </doc>
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a snapshot of the lifecycle history; it is a static list
+ @return a possibly empty but never null list of lifecycle events.]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the blockers on a service -remote dependencies
+ that are stopping the service from being <i>live</i>.
+ @return a (snapshotted) map of blocker name-&gt;description values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Service LifeCycle.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.Service -->
+  <!-- start class org.apache.hadoop.service.ServiceOperations -->
+  <class name="ServiceOperations" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service.
+ <p>Do nothing if the service is null or not
+ in a state in which it can be/needs to be stopped.
+ <p>
+ The service state is checked <i>before</i> the operation begins.
+ This process is <i>not</i> thread safe.
+ @param service a service or null]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains a set of methods to work with services, especially
+ to walk them through their lifecycle.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceOperations -->
+  <!-- start interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <interface name="ServiceStateChangeListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="stateChanged"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback to notify of a state change. The service will already
+ have changed state before this callback is invoked.
+
+ This operation is invoked on the thread that initiated the state change,
+ while the service itself in in a sychronized section.
+ <ol>
+   <li>Any long-lived operation here will prevent the service state
+   change from completing in a timely manner.</li>
+   <li>If another thread is somehow invoked from the listener, and
+   that thread invokes the methods of the service (including
+   subclass-specific methods), there is a risk of a deadlock.</li>
+ </ol>
+
+
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to notify state changes of a service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <!-- start class org.apache.hadoop.service.ServiceStateException -->
+  <class name="ServiceStateException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <constructor name="ServiceStateException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate
+ @param message error message]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate with a message and cause; if the cause has an exit code
+ then it is used, otherwise the generic
+ {@link LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION} exit code
+ is used.
+ @param message exception message
+ @param cause optional inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="int, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate, using the specified exit code as the exit code
+ of the exception, irrespetive of any exit code supplied by any inner
+ cause.
+
+ @param exitCode exit code to declare
+ @param message exception message
+ @param cause inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="text" type="java.lang.String"/>
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ If the caught exception is already of that type, it is typecast to a
+ {@link RuntimeException} and returned.
+
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param text text to use if a new exception is created
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Exception that can be raised on state change operations, whose
+ exit code can be explicitly set, determined from that of any nested
+ cause, or a default value of
+ {@link  LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateException -->
+  <!-- start class org.apache.hadoop.service.ServiceStateModel -->
+  <class name="ServiceStateModel" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceStateModel" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the service state model in the {@link Service.STATE#NOTINITED}
+ state.]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateModel" type="java.lang.String, org.apache.hadoop.service.Service.STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a service state model instance in the chosen state
+ @param state the starting state]]>
+      </doc>
+    </constructor>
+    <method name="getState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Query the service state. This is a non-blocking operation.
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query that the state is in a specific state
+ @param proposed proposed new state
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="ensureCurrentState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expectedState" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Verify that that a service is in a given state.
+ @param expectedState the desired state
+ @throws ServiceStateException if the service state is different from
+ the desired state]]>
+      </doc>
+    </method>
+    <method name="enterState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Enter a state -thread safe.
+
+ @param proposed proposed new state
+ @return the original state
+ @throws ServiceStateException if the transition is not permitted]]>
+      </doc>
+    </method>
+    <method name="checkStateTransition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Check that a state tansition is valid and
+ throw an exception if not
+ @param name name of the service (can be null)
+ @param state current state
+ @param proposed proposed new state]]>
+      </doc>
+    </method>
+    <method name="isValidStateTransition" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="current" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Is a state transition valid?
+ There are no checks for current==proposed
+ as that is considered a non-transition.
+
+ using an array kills off all branch misprediction costs, at the expense
+ of cache line misses.
+
+ @param current current state
+ @param proposed proposed new state
+ @return true if the transition to a new state is valid]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return the state text as the toString() value
+ @return the current state's description]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the service state model.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateModel -->
+</package>
+<package name="org.apache.hadoop.service.launcher">
+  <!-- start class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <class name="AbstractLaunchableService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.launcher.LaunchableService"/>
+    <constructor name="AbstractLaunchableService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance with the given name.]]>
+      </doc>
+    </constructor>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The base implementation logs all arguments at the debug level,
+ then returns the passed in config unchanged.]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The action is to signal success by returning the exit code 0.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Subclass of {@link AbstractService} that provides basic implementations
+ of the {@link LaunchableService} methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <!-- start class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <class name="HadoopUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="HadoopUncaughtExceptionHandler" type="java.lang.Thread.UncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance delegating to the supplied handler if
+ the exception is considered "simple".
+ @param delegate a delegate exception handler.]]>
+      </doc>
+    </constructor>
+    <constructor name="HadoopUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Basic exception handler -logs simple exceptions, then continues.]]>
+      </doc>
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thread" type="java.lang.Thread"/>
+      <param name="exception" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Uncaught exception handler.
+ If an error is raised: shutdown
+ The state of the system is unknown at this point -attempting
+ a clean shutdown is dangerous. Instead: exit
+ @param thread thread that failed
+ @param exception the raised exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ in the main entry point. 
+
+ The base class will always attempt to shut down the process if an Error
+ was raised; the behavior on a standard Exception, raised outside 
+ process shutdown, is simply to log it. 
+
+ (Based on the class {@code YarnUncaughtExceptionHandler})]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <!-- start interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <interface name="LaunchableService"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Propagate the command line arguments.
+ <p>
+ This method is called before {@link #init(Configuration)};
+ Any non-null configuration that is returned from this operation
+ becomes the one that is passed on to that {@link #init(Configuration)}
+ operation.
+ <p>
+ This permits implementations to change the configuration before
+ the init operation. As the ServiceLauncher only creates
+ an instance of the base {@link Configuration} class, it is
+ recommended to instantiate any subclass (such as YarnConfiguration)
+ that injects new resources.
+ <p>
+ @param config the initial configuration build up by the
+ service launcher.
+ @param args list of arguments passed to the command line
+ after any launcher-specific commands have been stripped.
+ @return the configuration to init the service with.
+ Recommended: pass down the config parameter with any changes
+ @throws Exception any problem]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Run a service. This method is called after {@link Service#start()}.
+ <p>
+ The return value becomes the exit code of the launched process.
+ <p>
+ If an exception is raised, the policy is:
+ <ol>
+   <li>Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}:
+   the exception is passed up unmodified.
+   </li>
+   <li>Any exception which implements
+   {@link org.apache.hadoop.util.ExitCodeProvider}:
+   A new {@link ServiceLaunchException} is created with the exit code
+   and message of the thrown exception; the thrown exception becomes the
+   cause.</li>
+   <li>Any other exception: a new {@link ServiceLaunchException} is created
+   with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and
+   the message of the original exception (which becomes the cause).</li>
+ </ol>
+ @return the exit code
+ @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed
+  up as the exit code and error text.
+ @throws Exception any exception to report. If it provides an exit code
+ this is used in a wrapping exception.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface which services can implement to have their
+ execution managed by the ServiceLauncher.
+ <p>
+ The command line options will be passed down before the 
+ {@link Service#init(Configuration)} operation is invoked via an
+ invocation of {@link LaunchableService#bindArgs(Configuration, List)}
+ After the service has been successfully started via {@link Service#start()}
+ the {@link LaunchableService#execute()} method is called to execute the 
+ service. When this method returns, the service launcher will exit, using
+ the return code from the method as its exit option.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <!-- start interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <interface name="LauncherExitCodes"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="EXIT_SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Success: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FAIL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generic "false/fail" response: {@value}.
+ The operation worked but the result was not "true" from the viewpoint
+ of the executed code.]]>
+      </doc>
+    </field>
+    <field name="EXIT_CLIENT_INITIATED_SHUTDOWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a client requested service termination: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_TASK_LAUNCH_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when targets could not be launched: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_INTERRUPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a control-C, kill -3, signal was picked up: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_OTHER_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when something happened but we can't be specific: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_COMMAND_ARGUMENT_ERROR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the command line doesn't parse: {@value}, or
+ when it is otherwise invalid.
+ <p>
+ Approximate HTTP equivalent: {@code 400 Bad Request}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNAUTHORIZED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The request requires user authentication: {@value}.
+ <p>
+ approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]>
+      </doc>
+    </field>
+    <field name="EXIT_USAGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a usage message was printed: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FORBIDDEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Forbidden action: {@value}.
+ <p>
+ Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_FOUND" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Something was not found: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 404: Not Found}]]>
+      </doc>
+    </field>
+    <field name="EXIT_OPERATION_NOT_ALLOWED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The operation is not allowed: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 405: Not allowed}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_ACCEPTABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The command is somehow not acceptable: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 406: Not Acceptable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_CONNECTIVITY_PROBLEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code on connectivity problems: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 408: Request Timeout}]]>
+      </doc>
+    </field>
+    <field name="EXIT_BAD_CONFIGURATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the configurations in valid/incomplete: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 409: Conflict}]]>
+      </doc>
+    </field>
+    <field name="EXIT_EXCEPTION_THROWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when an exception was thrown from the service: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 500 Internal Server Error}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNIMPLEMENTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Unimplemented feature: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 501: Not Implemented}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Service Unavailable; it may be available later: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 503 Service Unavailable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNSUPPORTED_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The application does not support, or refuses to support this
+ version: {@value}.
+ <p>
+ If raised, this is expected to be raised server-side and likely due
+ to client/server version incompatibilities.
+ <p>
+ Approximate HTTP equivalent: {@code 505: Version Not Supported}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_CREATION_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_LIFECYCLE_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Common Exit codes.
+ <p>
+ Codes with a YARN prefix are YARN-related.
+ <p>
+ Many of the exit codes are designed to resemble HTTP error codes,
+ squashed into a single byte. e.g 44 , "not found" is the equivalent
+ of 404. The various 2XX HTTP error codes aren't followed;
+ the Unix standard of "0" for success is used.
+ <pre>
+    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <!-- start class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+  <class name="ServiceLaunchException" extends="org.apache.hadoop.util.ExitUtil.ExitException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <implements name="org.apache.hadoop.service.launcher.LauncherExitCodes"/>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code.
+ @param exitCode exit code
+ @param cause cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code and text.
+ @param exitCode exit code
+ @param message message to use in exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ <p>
+ If the last argument is a throwable, it becomes the cause of the exception.
+ It will also be used as a parameter for the format.
+ @param exitCode exit code
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ @param exitCode exit code
+ @param cause inner cause
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A service launch exception that includes an exit code.
+ <p>
+ When caught by the ServiceLauncher, it will convert that
+ into a process exit code.
+ 
+ The {@link #ServiceLaunchException(int, String, Object...)} constructor
+ generates formatted exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+</package>
+<package name="org.apache.hadoop.tools">
+</package>
+<package name="org.apache.hadoop.tools.protocolPB">
+</package>
+<package name="org.apache.hadoop.tracing">
+</package>
+<package name="org.apache.hadoop.util">
+  <!-- start class org.apache.hadoop.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="java.net.URLClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="resolve" type="boolean"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="isSystemClass" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="systemClasses" type="java.util.List"/>
+      <doc>
+      <![CDATA[Checks if a class should be included as a system class.
+
+ A class is a system class if and only if it matches one of the positive
+ patterns and none of the negative ones.
+
+ @param name the class name to check
+ @param systemClasses a list of system class configurations.
+ @return true if the class is a system class]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_CLASSES_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of the system classes if the user did not override them.
+ JDK classes, hadoop classes and resources, and some select third-party
+ classes are considered system classes, and are not loaded by the
+ application classloader.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A {@link URLClassLoader} for application isolation. Classes from the
+ application JARs are loaded in preference to the parent loader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ApplicationClassLoader -->
+  <!-- start class org.apache.hadoop.util.DurationInfo -->
+  <class name="DurationInfo" extends="org.apache.hadoop.util.OperationDuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="DurationInfo" type="org.slf4j.Logger, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the duration text from a {@code String.format()} code call;
+ log output at info level.
+ @param log log to write to
+ @param format format string
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <constructor name="DurationInfo" type="org.slf4j.Logger, boolean, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the duration text from a {@code String.format()} code call
+ and log either at info or debug.
+ @param log log to write to
+ @param logAtInfo should the log be at info, rather than debug
+ @param format format string
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A duration with logging of final state at info or debug
+ in the {@code close()} call.
+ This allows it to be used in a try-with-resources clause, and have the
+ duration automatically logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.DurationInfo -->
+  <!-- start interface org.apache.hadoop.util.IPList -->
+  <interface name="IPList"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="isIn" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ipAddress" type="java.lang.String"/>
+      <doc>
+      <![CDATA[returns true if the ipAddress is in the IPList.
+ @param ipAddress
+ @return boolean value indicating whether the ipAddress is in the IPList]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.IPList -->
+  <!-- start class org.apache.hadoop.util.OperationDuration -->
+  <class name="OperationDuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OperationDuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate.
+ The start time and finished time are both set
+ to the current clock time.]]>
+      </doc>
+    </constructor>
+    <method name="time" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Evaluate the system time.
+ @return the current clock time.]]>
+      </doc>
+    </method>
+    <method name="finished"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the finished time with the current system time.]]>
+      </doc>
+    </method>
+    <method name="getDurationString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the duration as {@link #humanTime(long)}.
+ @return a printable duration.]]>
+      </doc>
+    </method>
+    <method name="humanTime" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+      <doc>
+      <![CDATA[Convert to a human time of minutes:seconds.millis.
+ @param time time to humanize.
+ @return a printable value.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the duration as {@link #humanTime(long)}.
+ @return a printable duration.]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds.
+ <p></p>
+ This will be 0 until a call
+ to {@link #finished()} has been made.
+ @return the currently recorded duration.]]>
+      </doc>
+    </method>
+    <method name="asDuration" return="java.time.Duration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration of an operation as a java Duration
+ instance.
+ @return a duration.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Little duration counter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.OperationDuration -->
+  <!-- start interface org.apache.hadoop.util.Progressable -->
+  <interface name="Progressable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="progress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Report progress to the Hadoop framework.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A facility for reporting progress.
+ 
+ <p>Clients and/or applications can use the provided <code>Progressable</code>
+ to explicitly report progress to the Hadoop framework. This is especially
+ important for operations which take significant amount of time since,
+ in-lieu of the reported progress, the framework has to assume that an error
+ has occurred and time-out the operation.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Progressable -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32 -->
+  <class name="PureJavaCrc32" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the same polynomial as the built-in native CRC32.
+
+ This is to avoid the JNI overhead for certain uses of Checksumming
+ where many small pieces of data are checksummed in succession.
+
+ The current version is ~10x to 1.8x as fast as Sun's native
+ java.util.zip.CRC32 in Java 1.6
+
+ @see java.util.zip.CRC32]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32 -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32C -->
+  <class name="PureJavaCrc32C" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32C"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the CRC32-C polynomial, the same polynomial used by iSCSI
+ and implemented on many Intel chipsets supporting SSE4.2.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32C -->
+  <!-- start class org.apache.hadoop.util.ReflectionUtils -->
+  <class name="ReflectionUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReflectionUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theObject" type="java.lang.Object"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Check and set 'configuration' if necessary.
+ 
+ @param theObject object for which to set configuration
+ @param conf Configuration]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create an object for the given class and initialize it from conf
+ 
+ @param theClass class of which an object is created
+ @param conf Configuration
+ @return a new object]]>
+      </doc>
+    </method>
+    <method name="setContentionTracing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="printThreadInfo"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.PrintStream"/>
+      <param name="title" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Print all of the thread's information and stack traces.
+ 
+ @param stream the stream to
+ @param title a string title for the stack trace]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="T"/>
+      <doc>
+      <![CDATA[Return the correctly-typed {@link Class} of the given object.
+  
+ @param o object whose correctly-typed <code>Class</code> is to be obtained
+ @return the correctly typed <code>Class</code> of the given object.]]>
+      </doc>
+    </method>
+    <method name="copy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="src" type="T"/>
+      <param name="dst" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param src the object to copy from
+ @param dst the object to copy into, which is destroyed
+ @return dst param (the copy)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cloneWritableInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDeclaredFieldsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared fields of a class including fields declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredMethodsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared methods of a class including methods declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[General reflection utils]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ReflectionUtils -->
+  <!-- start class org.apache.hadoop.util.Shell -->
+  <class name="Shell" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Shell"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with no minimum interval between runs; stderr is
+ not merged with stdout.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with a minimum interval between executions; stderr is
+ not merged with stdout.
+ @param interval interval in milliseconds between command executions.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a shell instance which can be re-executed when the {@link #run()}
+ method is invoked with a given elapsed time between calls.
+
+ @param interval the minimum duration in milliseconds to wait before
+        re-executing the command. If set to 0, there is no minimum.
+ @param redirectErrorStream should the error stream be merged with
+        the normal output stream?]]>
+      </doc>
+    </constructor>
+    <method name="isJava7OrAbove" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="This call isn't needed any more: please remove uses of it.">
+      <doc>
+      <![CDATA[query to see if system is Java 7 or later.
+ Now that Hadoop requires Java 7 or later, this always returns true.
+ @deprecated This call isn't needed any more: please remove uses of it.
+ @return true, always.]]>
+      </doc>
+    </method>
+    <method name="isJavaVersionAtLeast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Query to see if major version of Java specification of the system
+ is equal or greater than the parameter.
+
+ @param version 8, 9, 10 etc.
+ @return comparison with system property, always true for 8]]>
+      </doc>
+    </method>
+    <method name="checkWindowsCommandLineLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if a given command (String[]) fits in the Windows maximum command
+ line length Note that the input is expected to already include space
+ delimiters, no extra count will be added for delimiters.
+
+ @param commands command parts, including any space delimiters]]>
+      </doc>
+    </method>
+    <method name="getGroupsCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's groups list.]]>
+      </doc>
+    </method>
+    <method name="getGroupsForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's groups list.
+ If the OS is not WINDOWS, the command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.]]>
+      </doc>
+    </method>
+    <method name="getGroupsIDForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's group id list.
+ The command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.
+ This command does not support Windows and will only return group names.]]>
+      </doc>
+    </method>
+    <method name="getUsersForNetgroupCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="netgroup" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given netgroup's user list.]]>
+      </doc>
+    </method>
+    <method name="getGetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a command to get permission information.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Return a command to set permission.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set permission for specific file.
+
+ @param perm String permission to set
+ @param recursive boolean true to apply to all sub-directories recursively
+ @param file String file to set
+ @return String[] containing command and arguments]]>
+      </doc>
+    </method>
+    <method name="getSetOwnerCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set owner.]]>
+      </doc>
+    </method>
+    <method name="getSymlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to create symbolic links.]]>
+      </doc>
+    </method>
+    <method name="getReadlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to read the target of the a symbolic link.]]>
+      </doc>
+    </method>
+    <method name="getCheckProcessIsAliveCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command for determining if process with specified pid is alive.
+ @param pid process ID
+ @return a <code>kill -0</code> command or equivalent]]>
+      </doc>
+    </method>
+    <method name="getSignalKillCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to send a signal to a given pid.]]>
+      </doc>
+    </method>
+    <method name="getEnvironmentVariableRegex" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a regular expression string that match environment variables.]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.io.File"/>
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a File referencing a script with the given basename, inside the
+ given parent directory.  The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param parent File parent directory
+ @param basename String script file basename
+ @return File referencing the script in the directory]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a script file name with the given basename.
+
+ The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param basename String script file basename
+ @return String script file name]]>
+      </doc>
+    </method>
+    <method name="getRunScriptCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="script" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns a command to run the given script.  The script interpreter is
+ inferred by platform: cmd on Windows or bash otherwise.
+
+ @param script File script to run
+ @return String[] command to run the script]]>
+      </doc>
+    </method>
+    <method name="getHadoopHome" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the Hadoop home directory. Raises an exception if not found
+ @return the home dir
+ @throws IOException if the home directory cannot be located.]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBin" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBinPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist
+ @throws IOException on path canonicalization failures]]>
+      </doc>
+    </method>
+    <method name="hasWinutilsPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate to indicate whether or not the path to winutils is known.
+
+ If true, then {@link #WINUTILS} is non-null, and both
+ {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()}
+ will successfully return this value. Always false on non-windows systems.
+ @return true if there is a valid path to the binary]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Locate the winutils binary, or fail with a meaningful
+ exception and stack trace as an RTE.
+ This method is for use in methods which don't explicitly throw
+ an <code>IOException</code>.
+ @return the path to {@link #WINUTILS_EXE}
+ @throws RuntimeException if the path is not resolvable]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Get a file reference to winutils.
+ Always raises an exception if there isn't one
+ @return the file instance referring to the winutils bin.
+ @throws FileNotFoundException on any failure to locate that file.]]>
+      </doc>
+    </method>
+    <method name="checkIsBashSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+    </method>
+    <method name="setEnvironment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the environment for the command.
+ @param env Mapping of environment variables]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Set the working directory.
+ @param dir The directory where the command will be executed]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check to see if a command needs to be executed and execute if needed.]]>
+      </doc>
+    </method>
+    <method name="getExecString" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return an array containing the command name and its parameters.]]>
+      </doc>
+    </method>
+    <method name="parseExecResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="lines" type="java.io.BufferedReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the execution result]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an environment variable.
+ @param env the environment var
+ @return the value or null if it was unset.]]>
+      </doc>
+    </method>
+    <method name="getProcess" return="java.lang.Process"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the current sub-process executing the given command.
+ @return process executing the command]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the exit code.
+ @return the exit code of the process]]>
+      </doc>
+    </method>
+    <method name="getWaitingThread" return="java.lang.Thread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the thread that is waiting on this instance of <code>Shell</code>.
+ @return the thread that ran runCommand() that spawned this shell
+ or null if no thread is waiting for this shell to complete]]>
+      </doc>
+    </method>
+    <method name="isTimedOut" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[To check if the passed script to shell command executor timed out or
+ not.
+
+ @return if the script timed out.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param cmd shell command to execute.
+ @return the output of the executed command.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <param name="timeout" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @param timeout time in milliseconds after which script should be marked timeout
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="destroyAllShellProcesses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to destroy all running <code>Shell</code> processes.
+ Iterates through a map of all currently running <code>Shell</code>
+ processes and destroys them one by one. This method is thread safe]]>
+      </doc>
+    </method>
+    <method name="getAllShells" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to return a Set of all <code>Shell</code> objects.]]>
+      </doc>
+    </method>
+    <method name="getMemlockLimit" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ulimit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Static method to return the memory lock limit for datanode.
+ @param ulimit max value at which memory locked should be capped.
+ @return long value specifying the memory lock limit.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSPROP_HADOOP_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[System property for the Hadoop home directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ENV_HADOOP_HOME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable for Hadoop's home dir: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum command line length in Windows
+ KB830473 documents this as 8191]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGHT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use the correctly spelled constant.">
+      <doc>
+      <![CDATA[mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
+ @deprecated use the correctly spelled constant.]]>
+      </doc>
+    </field>
+    <field name="USER_NAME_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's name: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WindowsProcessLaunchLock" type="java.lang.Object"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Windows <code>CreateProcess</code> synchronization object.]]>
+      </doc>
+    </field>
+    <field name="osType" type="org.apache.hadoop.util.Shell.OSType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the operating system, as determined from parsing
+ the <code>os.name</code> property.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SOLARIS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FREEBSD" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OTHER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PPC_64" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENV_NAME_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Regular expression for environment variables: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_PERMISSION_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set permission: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_OWNER_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set owner: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_GROUP_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set the change user's groups list: {@value}.]]>
+      </doc>
+    </field>
+    <field name="LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to create a link: {@value}.]]>
+      </doc>
+    </field>
+    <field name="READ_LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get a link target: {@value}.]]>
+      </doc>
+    </field>
+    <field name="timeOutInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time after which the executing script would be timedout.]]>
+      </doc>
+    </field>
+    <field name="inheritParentEnv" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if the parent env vars should be inherited or not]]>
+      </doc>
+    </field>
+    <field name="WINUTILS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}">
+      <doc>
+      <![CDATA[Location of winutils as a string; null if not found.
+ <p>
+ <i>Important: caller must check for this value being null</i>.
+ The lack of such checks has led to many support issues being raised.
+ <p>
+ @deprecated use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]>
+      </doc>
+    </field>
+    <field name="isSetsidAvailable" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag which is true if setsid exists.]]>
+      </doc>
+    </field>
+    <field name="TOKEN_SEPARATOR_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Token separator regex used to parse Shell tool outputs.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for running a Shell command.
+
+ <code>Shell</code> can be used to run shell commands like <code>du</code> or
+ <code>df</code>. It also offers facilities to gate commands by
+ time-intervals.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.Shell -->
+  <!-- start class org.apache.hadoop.util.ShutdownHookManager -->
+  <class name="ShutdownHookManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="get" return="org.apache.hadoop.util.ShutdownHookManager"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return <code>ShutdownHookManager</code> singleton.
+
+ @return <code>ShutdownHookManager</code> singleton.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority, the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority and timeout the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order. The shutdown hook will be terminated if it
+ has not been finished in the specified period of time.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook
+ @param timeout timeout of the shutdownHook
+ @param unit unit of the timeout <code>TimeUnit</code>]]>
+      </doc>
+    </method>
+    <method name="removeShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Removes a shutdownHook.
+
+ @param shutdownHook shutdownHook to remove.
+ @return TRUE if the shutdownHook was registered and removed,
+ FALSE otherwise.]]>
+      </doc>
+    </method>
+    <method name="hasShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Indicates if a shutdownHook is registered or not.
+
+ @param shutdownHook shutdownHook to check if registered.
+ @return TRUE/FALSE depending if the shutdownHook is is registered.]]>
+      </doc>
+    </method>
+    <method name="isShutdownInProgress" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if shutdown is in progress or not.
+ 
+ @return TRUE if the shutdown is in progress, otherwise FALSE.]]>
+      </doc>
+    </method>
+    <method name="clearShutdownHooks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[clear all registered shutdownHooks.]]>
+      </doc>
+    </method>
+    <field name="TIMEOUT_MINIMUM" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum shutdown timeout: {@value} second(s).]]>
+      </doc>
+    </field>
+    <field name="TIME_UNIT_DEFAULT" type="java.util.concurrent.TimeUnit"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default time unit used: seconds.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The <code>ShutdownHookManager</code> enables running shutdownHook
+ in a deterministic order, higher priority first.
+ <p>
+ The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
+ This class registers a single JVM shutdownHook and run all the
+ shutdownHooks registered to it (to this class) in order based on their
+ priority.
+
+ Unless a hook was registered with a shutdown explicitly set through
+ {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ the shutdown time allocated to it is set by the configuration option
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in
+ {@code core-site.xml}, with a default value of
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ seconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ShutdownHookManager -->
+  <!-- start class org.apache.hadoop.util.StringInterner -->
+  <class name="StringInterner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StringInterner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="strongIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains strong reference to the instance, 
+ thus preventing it from being garbage-collected. 
+ 
+ @param sample string instance to be interned
+ @return strong reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="weakIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains weak reference to the instance, 
+ and so does not prevent it from being garbage-collected.
+ 
+ @param sample string instance to be interned
+ @return weak reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="internStringsInArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="strings" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Interns all the strings in the given array in place,
+ returning the same array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides string interning utility methods. For weak interning,
+ we use the standard String.intern() call, that performs very well
+ (no problems with PermGen overflowing, etc.) starting from JDK 7.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.StringInterner -->
+  <!-- start class org.apache.hadoop.util.SysInfo -->
+  <class name="SysInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SysInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.util.SysInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return default OS instance.
+ @throws UnsupportedOperationException If cannot determine OS.
+ @return Default instance for the detected OS.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the virtual memory present in the system.
+
+ @return virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the physical memory present in the system.
+
+ @return physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailableVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available virtual memory present
+ in the system.
+
+ @return available virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailablePhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available physical memory present
+ in the system.
+
+ @return available physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getNumProcessors" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total number of logical processors present on the system.
+
+ @return number of logical processors]]>
+      </doc>
+    </method>
+    <method name="getNumCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain total number of physical cores present on the system.
+
+ @return number of physical cores]]>
+      </doc>
+    </method>
+    <method name="getCpuFrequency" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU frequency of on the system.
+
+ @return CPU frequency in kHz]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the cumulative CPU time since the system is on.
+
+ @return cumulative CPU time in milliseconds]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU usage % of the machine. Return -1 if it is unavailable
+
+ @return CPU usage as a percentage (from 0 to 100) of available cycles.]]>
+      </doc>
+    </method>
+    <method name="getNumVCoresUsed" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the number of VCores used. Return -1 if it is unavailable
+
+ @return Number of VCores used a percentage (from 0 to #VCores).]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read over the network.
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to the network.
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read from disks.
+
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to disks.
+
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Plugin to calculate resource information on the system.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.SysInfo -->
+  <!-- start interface org.apache.hadoop.util.Tool -->
+  <interface name="Tool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="run" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Execute the command with the given arguments.
+ 
+ @param args command specific arguments.
+ @return exit code.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A tool interface that supports handling of generic command-line options.
+ 
+ <p><code>Tool</code>, is the standard for any Map-Reduce tool/application. 
+ The tool/application should delegate the handling of 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ standard command-line options</a> to {@link ToolRunner#run(Tool, String[])} 
+ and only handle its custom arguments.</p>
+ 
+ <p>Here is how a typical <code>Tool</code> is implemented:</p>
+ <p><blockquote><pre>
+     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // <code>Configuration</code> processed by <code>ToolRunner</code>
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed <code>conf</code>
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let <code>ToolRunner</code> handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+ </pre></blockquote><p>
+ 
+ @see GenericOptionsParser
+ @see ToolRunner]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Tool -->
+  <!-- start class org.apache.hadoop.util.ToolRunner -->
+  <class name="ToolRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ToolRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the given <code>Tool</code> by {@link Tool#run(String[])}, after 
+ parsing with the given generic arguments. Uses the given 
+ <code>Configuration</code>, or builds one if null.
+ 
+ Sets the <code>Tool</code>'s configuration with the possibly modified 
+ version of the <code>conf</code>.  
+ 
+ @param conf <code>Configuration</code> for the <code>Tool</code>.
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the <code>Tool</code> with its <code>Configuration</code>.
+ 
+ Equivalent to <code>run(tool.getConf(), tool, args)</code>.
+ 
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="printGenericCommandUsage"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.PrintStream"/>
+      <doc>
+      <![CDATA[Prints generic command-line argurments and usage information.
+ 
+  @param out stream to write usage information to.]]>
+      </doc>
+    </method>
+    <method name="confirmPrompt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prompt" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print out a prompt to the user, and return true if the user
+ responds with "y" or "yes". (case insensitive)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A utility to help run {@link Tool}s.
+ 
+ <p><code>ToolRunner</code> can be used to run classes implementing 
+ <code>Tool</code> interface. It works in conjunction with 
+ {@link GenericOptionsParser} to parse the 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ generic hadoop command line arguments</a> and modifies the 
+ <code>Configuration</code> of the <code>Tool</code>. The 
+ application-specific options are passed along without being modified.
+ </p>
+ 
+ @see Tool
+ @see GenericOptionsParser]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ToolRunner -->
+  <!-- start class org.apache.hadoop.util.VersionInfo -->
+  <class name="VersionInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionInfo" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="_getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Hadoop version.
+ @return the Hadoop version string, eg. "0.6.3-dev"]]>
+      </doc>
+    </method>
+    <method name="getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Git commit hash of the repository when compiled.
+ @return the commit hash, eg. "18f64065d5db6208daf50b02c1b5ed4ee3ce547a"]]>
+      </doc>
+    </method>
+    <method name="getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the branch on which this originated.
+ @return The branch name, e.g. "trunk" or "branches/branch-0.20"]]>
+      </doc>
+    </method>
+    <method name="getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The date that Hadoop was compiled.
+ @return the compilation date in unix date format]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user that compiled Hadoop.
+ @return the username of the user]]>
+      </doc>
+    </method>
+    <method name="getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL for the Hadoop repository.
+ @return the URL of the Hadoop repository]]>
+      </doc>
+    </method>
+    <method name="getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the checksum of the source files from which Hadoop was built.
+ @return the checksum of the source files]]>
+      </doc>
+    </method>
+    <method name="getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the buildVersion which includes version,
+ revision, user and date.
+ @return the buildVersion]]>
+      </doc>
+    </method>
+    <method name="getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the protoc version used for the build.
+ @return the protoc version]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <doc>
+    <![CDATA[This class returns build information about Hadoop components.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.VersionInfo -->
+  <doc>
+  <![CDATA[Common utilities.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.util.bloom">
+  <!-- start class org.apache.hadoop.util.bloom.BloomFilter -->
+  <class name="BloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="BloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVectorSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return size of the the bloomfilter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>Bloom filter</i>, as defined by Bloom in 1970.
+ <p>
+ The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by 
+ the networking research community in the past decade thanks to the bandwidth efficiencies that it
+ offers for the transmission of set membership information between networked hosts.  A sender encodes 
+ the information into a bit vector, the Bloom filter, that is more compact than a conventional 
+ representation. Computation and space costs for construction are linear in the number of elements.  
+ The receiver uses the filter to test whether various elements are members of the set. Though the 
+ filter will occasionally return a false positive, it will never return a false negative. When creating 
+ the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. 
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=362692&dl=ACM&coll=portal">Space/Time Trade-Offs in Hash Coding with Allowable Errors</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.BloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <class name="CountingBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CountingBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="CountingBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Removes a specified key from <i>this</i> counting Bloom filter.
+ <p>
+ <b>Invariant</b>: nothing happens if the specified key does not belong to <i>this</i> counter Bloom filter.
+ @param key The key to remove.]]>
+      </doc>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="approximateCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[This method calculates an approximate count of the key, i.e. how many
+ times the key was added to the filter. This allows the filter to be
+ used as an approximate <code>key -&gt; count</code> map.
+ <p>NOTE: due to the bucket size of this filter, inserting the same
+ key more than 15 times will cause an overflow at all filter positions
+ associated with this key, and it will significantly increase the error
+ rate for this and other keys. For this reason the filter can only be
+ used to store small count values <code>0 &lt;= N &lt;&lt; 15</code>.
+ @param key key to be tested
+ @return 0 if the key is not present. Otherwise, a positive value v will
+ be returned such that <code>v == count</code> with probability equal to the
+ error rate of this filter, and <code>v &gt; count</code> otherwise.
+ Additionally, if the filter experienced an underflow as a result of
+ {@link #delete(Key)} operation, the return value may be lower than the
+ <code>count</code> with the probability of the false negative rate of such
+ filter.]]>
+      </doc>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>counting Bloom filter</i>, as defined by Fan et al. in a ToN
+ 2000 paper.
+ <p>
+ A counting Bloom filter is an improvement to standard a Bloom filter as it
+ allows dynamic additions and deletions of set membership information.  This 
+ is achieved through the use of a counting vector instead of a bit vector.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=343571.343572">Summary cache: a scalable wide-area web cache sharing protocol</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <class name="DynamicBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DynamicBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zero-args constructor for the serialization.]]>
+      </doc>
+    </constructor>
+    <constructor name="DynamicBloomFilter" type="int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds an empty Dynamic Bloom filter.
+ @param vectorSize The number of bits in the vector.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).
+ @param nr The threshold for the maximum number of keys to record in a
+ dynamic Bloom filter row.]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>dynamic Bloom filter</i>, as defined in the INFOCOM 2006 paper.
+ <p>
+ A dynamic Bloom filter (DBF) makes use of a <code>s * m</code> bit matrix but
+ each of the <code>s</code> rows is a standard Bloom filter. The creation 
+ process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
+ bit matrix, i.e., it is composed of a single standard Bloom filter.
+ It assumes that <code>n<sub>r</sub></code> elements are recorded in the 
+ initial bit vector, where <code>n<sub>r</sub> {@literal <=} n</code>
+ (<code>n</code> is the cardinality of the set <code>A</code> to record in
+ the filter).
+ <p>
+ As the size of <code>A</code> grows during the execution of the application,
+ several keys must be inserted in the DBF.  When inserting a key into the DBF,
+ one must first get an active Bloom filter in the matrix.  A Bloom filter is
+ active when the number of recorded keys, <code>n<sub>r</sub></code>, is 
+ strictly less than the current cardinality of <code>A</code>, <code>n</code>.
+ If an active Bloom filter is found, the key is inserted and 
+ <code>n<sub>r</sub></code> is incremented by one. On the other hand, if there
+ is no active Bloom filter, a new one is created (i.e., a new row is added to
+ the matrix) according to the current size of <code>A</code> and the element
+ is added in this new Bloom filter and the <code>n<sub>r</sub></code> value of
+ this new Bloom filter is set to one.  A given key is said to belong to the
+ DBF if the <code>k</code> positions are set to one in one of the matrix rows.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ 
+ @see <a href="http://www.cse.fau.edu/~jie/research/publications/Publication_files/infocom2006.pdf">Theory and Network Applications of Dynamic Bloom Filters</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.HashFunction -->
+  <class name="HashFunction" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashFunction" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds a hash function that must obey to a given maximum number of returned values and a highest value.
+ @param maxValue The maximum highest returned value.
+ @param nbHash The number of resulting hashed values.
+ @param hashType type of the hashing function (see {@link Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears <i>this</i> hash function. A NOOP]]>
+      </doc>
+    </method>
+    <method name="hash" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Hashes a specified key into several integers.
+ @param k The specified key.
+ @return The array of hashed values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements a hash object that returns a certain number of hashed values.
+ 
+ @see Key The general behavior of a key being stored in a filter
+ @see Filter The general behavior of a filter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.HashFunction -->
+  <!-- start interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <interface name="RemoveScheme"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="RANDOM" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Random selection.
+ <p>
+ The idea is to randomly select a bit to reset.]]>
+      </doc>
+    </field>
+    <field name="MINIMUM_FN" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MinimumFN Selection.
+ <p>
+ The idea is to select the bit to reset that will generate the minimum
+ number of false negative.]]>
+      </doc>
+    </field>
+    <field name="MAXIMUM_FP" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MaximumFP Selection.
+ <p>
+ The idea is to select the bit to reset that will remove the maximum number
+ of false positive.]]>
+      </doc>
+    </field>
+    <field name="RATIO" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Ratio Selection.
+ <p>
+ The idea is to select the bit to reset that will, at the same time, remove
+ the maximum number of false positve while minimizing the amount of false
+ negative generated.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Defines the different remove scheme for retouched Bloom filters.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <!-- start class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+  <class name="RetouchedBloomFilter" extends="org.apache.hadoop.util.bloom.BloomFilter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.bloom.RemoveScheme"/>
+    <constructor name="RetouchedBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="RetouchedBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Adds a false positive information to <i>this</i> retouched Bloom filter.
+ <p>
+ <b>Invariant</b>: if the false positive is <code>null</code>, nothing happens.
+ @param key The false positive key to add.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="coll" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Adds a collection of false positive information to <i>this</i> retouched Bloom filter.
+ @param coll The collection of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="java.util.List"/>
+      <doc>
+      <![CDATA[Adds a list of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The list of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="org.apache.hadoop.util.bloom.Key[]"/>
+      <doc>
+      <![CDATA[Adds an array of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The array of false positive.]]>
+      </doc>
+    </method>
+    <method name="selectiveClearing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <param name="scheme" type="short"/>
+      <doc>
+      <![CDATA[Performs the selective clearing for a given key.
+ @param k The false positive key to remove from <i>this</i> retouched Bloom filter.
+ @param scheme The selective clearing scheme to apply.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>retouched Bloom filter</i>, as defined in the CoNEXT 2006 paper.
+ <p>
+ It allows the removal of selected false positives at the cost of introducing
+ random false negatives, and with the benefit of eliminating some random false
+ positives at the same time.
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ @see RemoveScheme The different selective clearing algorithms
+ 
+ @see <a href="http://www-rp.lip6.fr/site_npa/site_rp/_publications/740-rbf_cameraready.pdf">Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+</package>
+<package name="org.apache.hadoop.util.curator">
+</package>
+<package name="org.apache.hadoop.util.functional">
+  <!-- start class org.apache.hadoop.util.functional.FutureIO -->
+  <class name="FutureIO" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="awaitFuture" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="future" type="java.util.concurrent.Future"/>
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="RuntimeException" type="java.lang.RuntimeException"/>
+      <doc>
+      <![CDATA[Given a future, evaluate it.
+ <p>
+ Any exception generated in the future is
+ extracted and rethrown.
+ </p>
+ @param future future to evaluate
+ @param <T> type of the result.
+ @return the result, if all went well.
+ @throws InterruptedIOException future was interrupted
+ @throws IOException if something went wrong
+ @throws RuntimeException any nested RTE thrown]]>
+      </doc>
+    </method>
+    <method name="awaitFuture" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="future" type="java.util.concurrent.Future"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="RuntimeException" type="java.lang.RuntimeException"/>
+      <exception name="TimeoutException" type="java.util.concurrent.TimeoutException"/>
+      <doc>
+      <![CDATA[Given a future, evaluate it.
+ <p>
+ Any exception generated in the future is
+ extracted and rethrown.
+ </p>
+ @param future future to evaluate
+ @param <T> type of the result.
+ @return the result, if all went well.
+ @throws InterruptedIOException future was interrupted
+ @throws IOException if something went wrong
+ @throws RuntimeException any nested RTE thrown
+ @throws TimeoutException the future timed out.]]>
+      </doc>
+    </method>
+    <method name="raiseInnerCause" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.util.concurrent.ExecutionException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[From the inner cause of an execution exception, extract the inner cause
+ if it is an IOE or RTE.
+ This will always raise an exception, either the inner IOException,
+ an inner RuntimeException, or a new IOException wrapping the raised
+ exception.
+
+ @param e exception.
+ @param <T> type of return value.
+ @return nothing, ever.
+ @throws IOException either the inner IOException, or a wrapper around
+ any non-Runtime-Exception
+ @throws RuntimeException if that is the inner cause.]]>
+      </doc>
+    </method>
+    <method name="raiseInnerCause" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.util.concurrent.CompletionException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Extract the cause of a completion failure and rethrow it if an IOE
+ or RTE.
+ @param e exception.
+ @param <T> type of return value.
+ @return nothing, ever.
+ @throws IOException either the inner IOException, or a wrapper around
+ any non-Runtime-Exception
+ @throws RuntimeException if that is the inner cause.]]>
+      </doc>
+    </method>
+    <method name="unwrapInnerException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[From the inner cause of an execution exception, extract the inner cause
+ to an IOException, raising RuntimeExceptions and Errors immediately.
+ <ol>
+   <li> If it is an IOE: Return.</li>
+   <li> If it is a {@link UncheckedIOException}: return the cause</li>
+   <li> Completion/Execution Exceptions: extract and repeat</li>
+   <li> If it is an RTE or Error: throw.</li>
+   <li> Any other type: wrap in an IOE</li>
+ </ol>
+
+ Recursively handles wrapped Execution and Completion Exceptions in
+ case something very complicated has happened.
+ @param e exception.
+ @return an IOException extracted or built from the cause.
+ @throws RuntimeException if that is the inner cause.
+ @throws Error if that is the inner cause.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Future IO Helper methods.
+ <p>
+ Contains methods promoted from
+ {@link org.apache.hadoop.fs.impl.FutureIOSupport} because they
+ are a key part of integrating async IO in application code.
+ </p>
+ <p>
+ One key feature is that the {@link #awaitFuture(Future)} and
+ {@link #awaitFuture(Future, long, TimeUnit)} calls will
+ extract and rethrow exceptions raised in the future's execution,
+ including extracting the inner IOException of any
+ {@code UncheckedIOException} raised in the future.
+ This makes it somewhat easier to execute IOException-raising
+ code inside futures.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.functional.FutureIO -->
+  <!-- start class org.apache.hadoop.util.functional.RemoteIterators -->
+  <class name="RemoteIterators" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="remoteIteratorFromSingleton" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="singleton" type="T"/>
+      <doc>
+      <![CDATA[Create an iterator from a singleton.
+ @param singleton instance
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="java.util.Iterator"/>
+      <doc>
+      <![CDATA[Create a remote iterator from a java.util.Iterator.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromIterable" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterable" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Create a remote iterator from a java.util.Iterable -e.g. a list
+ or other collection.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromArray" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="array" type="T[]"/>
+      <doc>
+      <![CDATA[Create a remote iterator from an array.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="mappingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="mapper" type="org.apache.hadoop.util.functional.FunctionRaisingIOE"/>
+      <doc>
+      <![CDATA[Create an iterator from an iterator and a transformation function.
+ @param <S> source type
+ @param <T> result type
+ @param iterator source
+ @param mapper transformation
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="typeCastingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <doc>
+      <![CDATA[Create a RemoteIterator from a RemoteIterator, casting the
+ type in the process. This is to help with filesystem API
+ calls where overloading causes confusion (e.g. listStatusIterator())
+ @param <S> source type
+ @param <T> result type
+ @param iterator source
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="filteringRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="filter" type="org.apache.hadoop.util.functional.FunctionRaisingIOE"/>
+      <doc>
+      <![CDATA[Create a RemoteIterator from a RemoteIterator and a filter
+ function which returns true for every element to be passed
+ through.
+ <p></p>
+ Elements are filtered in the hasNext() method; if not used
+ the filtering will be done on demand in the {@code next()}
+ call.
+ @param <S> type
+ @param iterator source
+ @param filter filter
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="closingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="toClose" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[This adds an extra close operation alongside the passthrough
+ to any Closeable.close() method supported by the source iterator.
+ @param iterator source
+ @param toClose extra object to close.
+ @param <S> source type.
+ @return a new iterator]]>
+      </doc>
+    </method>
+    <method name="toList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a list from a RemoteIterator.
+ @param source source iterator
+ @param <T> type
+ @return a list of the values.
+ @throws IOException if the source RemoteIterator raises it.]]>
+      </doc>
+    </method>
+    <method name="toArray" return="T[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="a" type="T[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build an array from a RemoteIterator.
+ @param source source iterator
+ @param a destination array; if too small a new array
+ of the same type is created
+ @param <T> type
+ @return an array of the values.
+ @throws IOException if the source RemoteIterator raises it.]]>
+      </doc>
+    </method>
+    <method name="foreach" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="consumer" type="org.apache.hadoop.util.functional.ConsumerRaisingIOE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Apply an operation to all values of a RemoteIterator.
+ <p></p>
+ If the iterator is an IOStatisticsSource returning a non-null
+ set of statistics, <i>and</i> this classes log is set to DEBUG,
+ then the statistics of the operation are evaluated and logged at
+ debug.
+ <p></p>
+ The number of entries processed is returned, as it is useful to
+ know this, especially during tests or when reporting values
+ to users.
+ <p></p>
+ This does not close the iterator afterwards.
+ @param source iterator source
+ @param consumer consumer of the values.
+ @return the number of elements processed
+ @param <T> type of source
+ @throws IOException if the source RemoteIterator or the consumer raise one.]]>
+      </doc>
+    </method>
+    <method name="cleanupRemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <doc>
+      <![CDATA[Clean up after an iteration.
+ If the log is at debug, calculate and log the IOStatistics.
+ If the iterator is closeable, cast and then cleanup the iterator
+ @param source iterator source
+ @param <T> type of source]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A set of remote iterators supporting transformation and filtering,
+ with IOStatisticsSource passthrough, and of conversions of
+ the iterators to lists/arrays and of performing actions
+ on the values.
+ <p></p>
+ This aims to make it straightforward to use lambda-expressions to
+ transform the results of an iterator, without losing the statistics
+ in the process, and to chain the operations together.
+ <p></p>
+ The closeable operation will be passed through RemoteIterators which
+ wrap other RemoteIterators. This is to support any iterator which
+ can be closed to release held connections, file handles etc.
+ Unless client code is written to assume that RemoteIterator instances
+ may be closed, this is not likely to be broadly used. It is added
+ to make it possible to adopt this feature in a managed way.
+ <p></p>
+ One notable feature is that the
+ {@link #foreach(RemoteIterator, ConsumerRaisingIOE)} method will
+ LOG at debug any IOStatistics provided by the iterator, if such
+ statistics are provided. There's no attempt at retrieval and logging
+ if the LOG is not set to debug, so it is a zero cost feature unless
+ the logger {@code org.apache.hadoop.fs.functional.RemoteIterators}
+ is at DEBUG.
+ <p></p>
+ Based on the S3A Listing code, and some some work on moving other code
+ to using iterative listings so as to pick up the statistics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.functional.RemoteIterators -->
+</package>
+<package name="org.apache.hadoop.util.hash">
+</package>
+
+</api>
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml
new file mode 100644
index 0000000000000..62a0e09f121af
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml
@@ -0,0 +1,39037 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 13:58:49 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop Common 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-common-project/hadoop-common/target/classes:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/wildfly/openssl/wildfly-openssl-java/1.0.7.Final/wildfly-openssl-java-1.0.7.Final.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/org/lz4/lz4-java/1.7.1/lz4-java-1.7.1.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-common-project/hadoop-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/build/source/hadoop-common-project/hadoop-common/target/jdiff.jar -apidir /build/source/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 3.3.4 -->
+<package name="org.apache.hadoop">
+  <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+  <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message detailed message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+  <!-- start interface org.apache.hadoop.conf.Configurable -->
+  <interface name="Configurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+ @param conf configuration to be used]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+ @return Configuration]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.conf.Configurable -->
+  <!-- start class org.apache.hadoop.conf.Configuration -->
+  <class name="Configuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Iterable"/>
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration where the behavior of reading from the default 
+ resources can be turned off.
+ 
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files. 
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new configuration with the same settings cloned from another.
+ 
+ @param other the configuration from which to clone settings.]]>
+      </doc>
+    </constructor>
+    <method name="addDeprecations"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+      <doc>
+      <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless.  It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context.  If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas   The deprecations to add.]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+ 
+ @param key to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @param customMessage depcrication message
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+      String customMessage)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <param name="customMessage" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key to be deprecated
+ @param newKey key that take up the values of deprecated key
+ @param customMessage deprecation message]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKeys" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If a key is deprecated in favor of multiple keys, they are all treated as 
+ aliases of each other, and setting any one of them resets all the others 
+ to the new value.
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+      </doc>
+    </method>
+    <method name="addDeprecation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="newKey" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+ 
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+      </doc>
+    </method>
+    <method name="isDeprecated" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[checks whether the given <code>key</code> is deprecated.
+ 
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and 
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setDeprecatedProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+      </doc>
+    </method>
+    <method name="reloadExistingConfigurations"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload existing configuration instances.]]>
+      </doc>
+    </method>
+    <method name="addDefaultResource"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a default resource. Resources are loaded in the order of the resources 
+ added.
+ @param name file name. File should be present in the classpath.]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemPropertiesDefault"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="setRestrictSystemProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param name resource to be added, the classpath is examined for a file 
+             with that name.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param url url of the resource to be added, the local filesystem is 
+            examined directly to find the resource, without referring to 
+            the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param file file-path of resource to be added, the local filesystem is
+             examined directly to find the resource, without referring to 
+             the classpath.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ WARNING: The contents of the InputStream will be cached, by this method. 
+ So use this sparingly because it does increase the memory consumption.
+ 
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next.  After it is read the stream will be
+ closed.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a configuration resource. 
+ 
+ The properties of this resource will override properties of previously 
+ added resources, unless they were marked <a href="#Final">final</a>. 
+ 
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+      </doc>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="restrictedParser" type="boolean"/>
+    </method>
+    <method name="addResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+      </doc>
+    </method>
+    <method name="reloadConfiguration"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added 
+ resources, and final parameters. This will make the resources to 
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned.
+
+ As a side effect get loads the properties from the sources if called for
+ the first time as a lazy init.
+ 
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="setAllowNullValueProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+      <doc>
+      <![CDATA[Set Configuration to allow keys without values during setup.  Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+      </doc>
+    </method>
+    <method name="setRestrictSystemProps"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="onlyKeyExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>null</code> if no such property exists. 
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+ 
+ Values are processed for <a href="#VariableExpansion">variable expansion</a> 
+ before being returned. 
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property, 
+         or null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="getTrimmed" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>, 
+ <code>defaultValue</code> if no such property exists. 
+ See @{Configuration#getTrimmed} for more details.
+ 
+ @param name          the property name.
+ @param defaultValue  the property default value.
+ @return              the value of the <code>name</code> or defaultValue
+                      if it is not set.]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is 
+ deprecated, it returns the value of the first key which replaces 
+ the deprecated key and is not null.
+ 
+ @param name the property name.
+ @return the value of the <code>name</code> property or 
+         its replacing property and null if no such property exists.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="source" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>value</code> of the <code>name</code> property. If 
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from 
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+      </doc>
+    </method>
+    <method name="unset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unset a previously set property.
+ @param name the property name]]>
+      </doc>
+    </method>
+    <method name="setIfUnset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+ 
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property 
+         doesn't exist.]]>
+      </doc>
+    </method>
+    <method name="getInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="int"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+   
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInts" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+ 
+ If no such property exists, an empty array is returned.
+ 
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+         <code>int</code> values]]>
+      </doc>
+    </method>
+    <method name="setInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+ 
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getLongBytes" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+ 
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="getFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="float"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.  
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setDouble"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+ 
+ @param name property name.
+ @param value property value.]]>
+      </doc>
+    </method>
+    <method name="getBoolean" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="boolean"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.  
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+ 
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="setBoolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+ 
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+      </doc>
+    </method>
+    <method name="setBooleanIfUnset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+      </doc>
+    </method>
+    <method name="setEnum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="T"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(&lt;name&gt;, value.toString())</code>.
+ @param name property name
+ @param value new value
+ @param <T> enumeration type]]>
+      </doc>
+    </method>
+    <method name="getEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="T"/>
+      <doc>
+      <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @param <T> enumeration type
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided
+ @return enumeration type]]>
+      </doc>
+    </method>
+    <method name="setTimeDuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(&lt;name&gt;, value + &lt;time suffix&gt;)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="long"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
+ provided, the default unit is applied.
+
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param defaultUnit Default time unit if no valid suffix is provided.
+ @param returnUnit The unit used for the returned value.
+ @throws NumberFormatException If the property stripped of its unit is not
+         a number
+ @return time duration in given time unit]]>
+      </doc>
+    </method>
+    <method name="getTimeDuration" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="defaultUnit" type="java.util.concurrent.TimeUnit"/>
+      <param name="returnUnit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getTimeDurationHelper" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="vStr" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+
+ @param name Property name
+ @param vStr The string value with time unit suffix to be converted.
+ @param unit Unit to convert the stored property, if it exists.]]>
+      </doc>
+    </method>
+    <method name="getTimeDurations" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets the Storage Size from the config, or returns the defaultValue. The
+ unit of return value is specified in target unit.
+
+ @param name - Key Name
+ @param defaultValue - Default Value -- e.g. 100MB
+ @param targetUnit - The units that we want result to be in.
+ @return double -- formatted in target Units]]>
+      </doc>
+    </method>
+    <method name="getStorageSize" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="double"/>
+      <param name="targetUnit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Gets storage size from a config file.
+
+ @param name - Key to read.
+ @param defaultValue - The default value to return in case the key is
+ not present.
+ @param targetUnit - The Storage unit that should be used
+ for the return value.
+ @return - double value in the Storage Unit specified.]]>
+      </doc>
+    </method>
+    <method name="setStorageSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <param name="unit" type="org.apache.hadoop.conf.StorageUnit"/>
+      <doc>
+      <![CDATA[Sets Storage Size for the specified key.
+
+ @param name - Key to set.
+ @param value - The numeric value to set.
+ @param unit - Storage Unit to be used.]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="pattern" type="java.util.regex.Pattern"/>
+      <doc>
+      <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+      </doc>
+    </method>
+    <method name="getPropertySources" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Gets information about why a property was set.  Typically this is the 
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programmatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise, 
+ returns a list of the sources of the resource.  The older sources are
+ the first ones in the list.  So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+      </doc>
+    </method>
+    <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+      </doc>
+    </method>
+    <method name="getStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s.  
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+ 
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then <code>null</code> is returned.
+ 
+ @param name property name.
+ @return property value as an array of <code>String</code>s, 
+         or <code>null</code>.]]>
+      </doc>
+    </method>
+    <method name="getStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s.  
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStringCollection" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.  
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+ 
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s, 
+         or empty array.]]>
+      </doc>
+    </method>
+    <method name="getTrimmedStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Get the comma delimited values of the <code>name</code> property as 
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+ 
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s, 
+         or default value.]]>
+      </doc>
+    </method>
+    <method name="setStrings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the array of string values for the <code>name</code> property as 
+ as comma delimited values.  
+ 
+ @param name property name.
+ @param values The values]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password
+ @throws IOException when error in fetching password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromCredentialProviders" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException when error in fetching password]]>
+      </doc>
+    </method>
+    <method name="getPasswordFromConfig" return="char[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="setSocketAddr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostProperty" type="java.lang.String"/>
+      <param name="addressProperty" type="java.lang.String"/>
+      <param name="defaultAddressValue" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address.  This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>.  The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+      </doc>
+    </method>
+    <method name="getClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Load a class by name.
+ 
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+      </doc>
+    </method>
+    <method name="getClassByNameOrNull" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+ 
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+      </doc>
+    </method>
+    <method name="getClasses" return="java.lang.Class[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class[]"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.  
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ @param name the conf key name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+   
+ If no such property is specified, then <code>defaultValue</code> is 
+ returned.
+ 
+ An exception is thrown if the returned class does not implement the named
+ interface. 
+ 
+ @param name the conf key name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>, 
+         or <code>defaultValue</code>.]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+ 
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+ 
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+        <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+      </doc>
+    </method>
+    <method name="setClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="xface" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value of the <code>name</code> property to the name of a 
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+ 
+ An exception is thrown if <code>theClass</code> does not implement the 
+ interface <code>xface</code>. 
+ 
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+      </doc>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dirsProp" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code.  If the selected
+ directory does not exist, an attempt is made to create it.
+ 
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link URL} for the named resource.
+ 
+ @param name resource name.
+ @return the url for the named resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getConfResourceAsReader" return="java.io.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+ 
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+      </doc>
+    </method>
+    <method name="getFinalParameters" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+      </doc>
+    </method>
+    <method name="getProps" return="java.util.Properties"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all keys from the configuration.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code> 
+ key-value pairs in the configuration.
+ 
+ @return an iterator over the entries.]]>
+      </doc>
+    </method>
+    <method name="getPropsWithPrefix" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration and includes all properties that
+ start with the specified configuration prefix.  Property names in the
+ mapping are trimmed to remove the configuration prefix.
+
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties with prefix stripped]]>
+      </doc>
+    </method>
+    <method name="addTags"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prop" type="java.util.Properties"/>
+      <doc>
+      <![CDATA[Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
+ @param prop]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+ 
+ @param out the output stream to write to.]]>
+      </doc>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeXml"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Write out the non-default properties in this configuration to the
+ given {@link Writer}.
+ <ul>
+ <li>
+ When property name is not empty and the property exists in the
+ configuration, this method writes the property and its attributes
+ to the {@link Writer}.
+ </li>
+
+ <li>
+ When property name is null or empty, this method writes all the
+ configuration properties and their attributes to the {@link Writer}.
+ </li>
+
+ <li>
+ When property name is not empty but the property doesn't exist in
+ the configuration, this method throws an {@link IllegalArgumentException}.
+ </li>
+ </ul>
+ @param out the writer to write to.]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="propertyName" type="java.lang.String"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes properties and their attributes (final and resource)
+  to the given {@link Writer}.
+  <ul>
+  <li>
+  When propertyName is not empty, and the property exists
+  in the configuration, the format of the output would be,
+  <pre>
+  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is null or empty, it behaves same as
+  {@link #dumpConfiguration(Configuration, Writer)}, the
+  output would be,
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+  </li>
+
+  <li>
+  When propertyName is not empty, and the property is not
+  found in the configuration, this method will throw an
+  {@link IllegalArgumentException}.
+  </li>
+  </ul>
+  <p>
+ @param config the configuration
+ @param propertyName property name
+ @param out the Writer to write to
+ @throws IOException
+ @throws IllegalArgumentException when property name is not
+   empty and the property is not found in configuration]]>
+      </doc>
+    </method>
+    <method name="dumpConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes out all properties and their attributes (final and resource) to
+  the given {@link Writer}, the format of the output would be,
+
+  <pre>
+  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+  </pre>
+
+  It does not output the properties of the configuration object which
+  is loaded from an input stream.
+  <p>
+
+ @param config the configuration
+ @param out the Writer to write to
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClassLoader" return="java.lang.ClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+      </doc>
+    </method>
+    <method name="setClassLoader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classLoader" type="java.lang.ClassLoader"/>
+      <doc>
+      <![CDATA[Set the class loader that will be used to load the various objects.
+ 
+ @param classLoader the new class loader.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQuietMode"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quietmode" type="boolean"/>
+      <doc>
+      <![CDATA[Set the quietness-mode. 
+ 
+ In the quiet-mode, error and informational messages might not be logged.
+ 
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+              to turn it off.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[For debugging.  List non-default properties to the terminal and exit.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getValByRegex" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="regex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[get keys matching the the regex 
+ @param regex
+ @return {@literal Map<String,String>} with matching keys]]>
+      </doc>
+    </method>
+    <method name="dumpDeprecatedKeys"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasWarnedDeprecation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTag" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get all properties belonging to tag.
+ @param tag tag
+ @return Properties with matching tag]]>
+      </doc>
+    </method>
+    <method name="getAllPropertiesByTags" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get all properties belonging to list of input tags. Calls
+ getAllPropertiesByTag internally.
+ @param tagList list of input tags
+ @return Properties with matching tags]]>
+      </doc>
+    </method>
+    <method name="isPropertyTag" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tagStr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get Property tag Enum corresponding to given source.
+
+ @param tagStr String representation of Enum
+ @return true if tagStr is a valid tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides access to configuration parameters.
+
+ <h3 id="Resources">Resources</h3>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a 
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>, 
+ then the classpath is examined for a file with that name.  If named by a 
+ <code>Path</code>, then the local filesystem is examined directly, without 
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two 
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+ 
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>. 
+ Once a resource declares a value final, no subsequently-loaded 
+ resource can alter that value.  
+ For example, one might define a final parameter with:
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;dfs.hosts.include&lt;/name&gt;
+    &lt;value&gt;/etc/hadoop/conf/hosts.include&lt;/value&gt;
+    <b>&lt;final&gt;true&lt;/final&gt;</b>
+  &lt;/property&gt;</code></pre>
+
+ Administrators typically define parameters as final in 
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Environment variables in {@link System#getenv()} if a name starts with
+ "env.", or</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions: 
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;basedir&lt;/name&gt;
+    &lt;value&gt;/user/${<i>user.name</i>}&lt;/value&gt;
+  &lt;/property&gt;
+  
+  &lt;property&gt;
+    &lt;name&gt;tempdir&lt;/name&gt;
+    &lt;value&gt;${<i>basedir</i>}/tmp&lt;/value&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;otherdir&lt;/name&gt;
+    &lt;value&gt;${<i>env.BASE_DIR</i>}/other&lt;/value&gt;
+  &lt;/property&gt;
+  </code></pre>
+
+ <p>When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ <p>When <tt>conf.get("otherdir")</tt> is called, then <tt>${<i>env.BASE_DIR</i>}</tt>
+ will be resolved to the value of the <tt>${<i>BASE_DIR</i>}</tt> environment variable.
+ It supports <tt>${<i>env.NAME:-default</i>}</tt> and <tt>${<i>env.NAME-default</i>}</tt> notations.
+ The former is resolved to "default" if <tt>${<i>NAME</i>}</tt> environment variable is undefined
+ or its value is empty.
+ The latter behaves the same way only if <tt>${<i>NAME</i>}</tt> is undefined.
+ <p>By default, warnings will be given to any deprecated configuration 
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.
+
+ <h4 id="Tags">Tags</h4>
+
+ <p>Optionally we can tag related properties together by using tag
+ attributes. System tags are defined by hadoop.tags.system property. Users
+ can define there own custom tags in  hadoop.tags.custom property.
+
+ <p>For example, we can tag existing property as:
+ <pre><code>
+  &lt;property&gt;
+    &lt;name&gt;dfs.replication&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,REQUIRED&lt;/tag&gt;
+  &lt;/property&gt;
+
+  &lt;property&gt;
+    &lt;name&gt;dfs.data.transfer.protection&lt;/name&gt;
+    &lt;value&gt;3&lt;/value&gt;
+    &lt;tag&gt;HDFS,SECURITY&lt;/tag&gt;
+  &lt;/property&gt;
+ </code></pre>
+ <p> Properties marked with tags can be retrieved with <tt>conf
+ .getAllPropertiesByTag("HDFS")</tt> or <tt>conf.getAllPropertiesByTags
+ (Arrays.asList("YARN","SECURITY"))</tt>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configuration -->
+  <!-- start class org.apache.hadoop.conf.Configured -->
+  <class name="Configured" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="Configured"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Configured.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.conf.Configured -->
+  <doc>
+  <![CDATA[Configuration of system parameters.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.crypto">
+</package>
+<package name="org.apache.hadoop.crypto.key">
+  <!-- start class org.apache.hadoop.crypto.key.KeyProvider -->
+  <class name="KeyProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="KeyProvider" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ 
+ @param conf configuration for the provider]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the provider configuration.
+ 
+ @return the provider configuration]]>
+      </doc>
+    </method>
+    <method name="options" return="org.apache.hadoop.crypto.key.KeyProvider.Options"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[A helper function to create an options object.
+ @param conf the configuration to use
+ @return a new options object]]>
+      </doc>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide access to
+ keying material rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getKeyVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for a specific version of the key. This method is used
+ when decrypting data.
+ @param versionName the name of a specific version of the key
+ @return the key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeys" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key names for all keys.
+ @return the list of key names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeysMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get key metadata in bulk.
+ @param names the names of the keys to get
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getKeyVersions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the key material for all versions of a specific key name.
+ @return the list of key material
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current version of the key, which should be used for encrypting new
+ data.
+ @param name the base name of the key
+ @return the version name of the current version of the key or null if the
+    key version doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMetadata" return="org.apache.hadoop.crypto.key.KeyProvider.Metadata"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get metadata about the key.
+ @param name the basename of the key
+ @return the key's metadata or null if the key doesn't exist
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key. The given key must not already exist.
+ @param name the base name of the key
+ @param material the key material for the first version of the key.
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <param name="algorithm" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <doc>
+      <![CDATA[Generates a key material.
+
+ @param size length of the key.
+ @param algorithm algorithm to use for generating the key.
+ @return the generated key.
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="createKey" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="options" type="org.apache.hadoop.crypto.key.KeyProvider.Options"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new key generating the material for it.
+ The given key must not already exist.
+ <p>
+ This implementation generates the key material and calls the
+ {@link #createKey(String, byte[], Options)} method.
+
+ @param name the base name of the key
+ @param options the options for the new key.
+ @return the version name of the first version of the key.
+ @throws IOException
+ @throws NoSuchAlgorithmException]]>
+      </doc>
+    </method>
+    <method name="deleteKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given key.
+ @param name the name of the key to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="material" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key.
+ @param name the basename of the key
+ @param material the new key material
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to close any resources
+ that require closing]]>
+      </doc>
+    </method>
+    <method name="rollNewVersion" return="org.apache.hadoop.crypto.key.KeyProvider.KeyVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="NoSuchAlgorithmException" type="java.security.NoSuchAlgorithmException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Roll a new version of the given key generating the material for it.
+ <p>
+ This implementation generates the key material and calls the
+ {@link #rollNewVersion(String, byte[])} method.
+
+ @param name the basename of the key
+ @return the name of the new version of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="invalidateCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Can be used by implementing classes to invalidate the caches. This could be
+ used after rollNewVersion to provide a strong guarantee to return the new
+ version of the given key.
+
+ @param name the basename of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the keys are written to persistent store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getBaseName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="versionName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Split the versionName in to a base name. Converts "/aaa/bbb/3" to
+ "/aaa/bbb".
+ @param versionName the version name to split
+ @return the base name of the key
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="buildVersionName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Build a version string from a basename and version number. Converts
+ "/aaa/bbb" and 3 to "/aaa/bbb@3".
+ @param name the basename of the key
+ @param version the version of the key
+ @return the versionName of the key.]]>
+      </doc>
+    </method>
+    <method name="findProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerList" type="java.util.List"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the provider with the given key.
+ @param providerList the list of providers
+ @param keyName the key name we are looking for
+ @return the KeyProvider that has the key]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_CIPHER_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CIPHER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BITLENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIALFILTER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JCEKS_KEY_SERIAL_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of secret key material for Hadoop applications. Provides an
+ abstraction to separate key storage from users of encryption. It
+ is intended to support getting or storing keys in a variety of ways,
+ including third party bindings.
+ <p>
+ <code>KeyProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProvider -->
+  <!-- start class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+  <class name="KeyProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a KeyProvider based on a provided URI.
+
+ @param uri key provider URI
+ @param conf configuration to initialize the key provider
+ @return the key provider for the specified URI, or <code>NULL</code> if
+         a provider for the specified URI scheme could not be found.
+ @throws IOException thrown if the provider failed to initialize.]]>
+      </doc>
+    </method>
+    <field name="KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of KeyProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ KeyProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.crypto.key.KeyProviderFactory -->
+</package>
+<package name="org.apache.hadoop.crypto.key.kms">
+</package>
+<package name="org.apache.hadoop.crypto.random">
+</package>
+<package name="org.apache.hadoop.fs">
+  <!-- start interface org.apache.hadoop.fs.Abortable -->
+  <interface name="Abortable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="abort" return="org.apache.hadoop.fs.Abortable.AbortableResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Abort the active operation without the output becoming visible.
+
+ This is to provide ability to cancel the write on stream; once
+ a stream is aborted, the write MUST NOT become visible.
+
+ @throws UnsupportedOperationException if the operation is not supported.
+ @return the result.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abort data being written to  a stream, so that close() does
+  not write the data. It is implemented by output streams in
+  some object stores, and passed through {@link FSDataOutputStream}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Abortable -->
+  <!-- start class org.apache.hadoop.fs.AbstractFileSystem -->
+  <class name="AbstractFileSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="AbstractFileSystem" type="java.net.URI, java.lang.String, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[Constructor to be called by subclasses.
+ 
+ @param uri for this file system.
+ @param supportedScheme the scheme supported by the implementor
+ @param authorityNeeded if true then theURI must have authority, if false
+          then the URI must have null authority.
+
+ @throws URISyntaxException <code>uri</code> has syntax error]]>
+      </doc>
+    </constructor>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isValidName" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns true if the specified string is considered valid in the path part
+ of a URI by this file system.  The default implementation enforces the rules
+ of HDFS, but subclasses may override this method to implement specific
+ validation rules for specific file systems.
+ 
+ @param src String source filename to check, path part of the URI
+ @return boolean true if the specified string is considered valid]]>
+      </doc>
+    </method>
+    <method name="createFileSystem" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a file system instance for the specified uri using the conf. The
+ conf is used to find the class name that implements the file system. The
+ conf is also passed to the file system for its configuration.
+
+ @param uri URI of the file system
+ @param conf Configuration for the file system
+ 
+ @return Returns the file system for the given URI
+
+ @throws UnsupportedFileSystemException file system for <code>uri</code> is
+           not found]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ 
+ @param uri
+          used as key to lookup STATISTICS_TABLE. Only scheme and authority
+          part of the uri are used.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[The main factory method for creating a file system. Get a file system for
+ the URI's scheme and authority. The scheme of the <code>uri</code>
+ determines a configuration property name,
+ <tt>fs.AbstractFileSystem.<i>scheme</i>.impl</tt> whose value names the
+ AbstractFileSystem class.
+ 
+ The entire URI and conf is passed to the AbstractFileSystem factory method.
+ 
+ @param uri for the file system to be created.
+ @param conf which is passed to the file system impl.
+ 
+ @return file system for the given URI.
+ 
+ @throws UnsupportedFileSystemException if the file system for
+           <code>uri</code> is not supported.]]>
+      </doc>
+    </method>
+    <method name="checkScheme"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="supportedScheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check that the Uri's scheme matches
+ @param uri
+ @param supportedScheme]]>
+      </doc>
+    </method>
+    <method name="getUriDefaultPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default port of this file system.
+ 
+ @return default port of this file system's Uri scheme
+         A uri with a port of -1 =&gt; default port;]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.
+ 
+ @return the uri of this file system.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+ 
+ If the path is fully qualified URI, then its scheme and authority
+ matches that of this file system. Otherwise the path must be 
+ slash-relative name.
+ 
+ @throws InvalidPathException if the path is invalid]]>
+      </doc>
+    </method>
+    <method name="getUriPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the path-part of a pathname. Checks that URI matches this file system
+ and that the path-part is a valid name.
+ 
+ @param p path
+ 
+ @return path-part of the Path p]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified to this file system
+ @param path
+ @return the qualified path]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Some file systems like LocalFileSystem have an initial workingDir
+ that is used as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+ 
+ @return the initial workingDir if the file system has such a notion
+         otherwise return a null.]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ 
+ @return current user's home directory.]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ 
+ @return server default configuration values
+ 
+ @throws IOException an I/O error occurred
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param f path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path f resolving the path
+ through any internal symlinks or mount point
+ @param p path to be resolved
+ @return fully qualified path 
+ @throws FileNotFoundException
+ @throws AccessControlException
+ @throws IOException
+ @throws UnresolvedLinkException if symbolic link on path cannot be
+ resolved internally]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
+ that the Path f must be fully qualified and the permission is absolute
+ (i.e. umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="createInternal" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
+ have been declared explicitly.]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
+ f must be fully qualified and the permission is absolute (i.e. 
+ umask has been applied).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#delete(Path, boolean)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#open(Path, int)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#truncate(Path, long)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setReplication(Path, short)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system and NO OVERWRITE is performed.
+ 
+ File systems that do not have a built in overwrite need implement only this
+ method and can take advantage of the default impl of the other
+ {@link #renameInternal(Path, Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="renameInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
+ f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the file system supports symlinks, false otherwise.
+ @return true if filesystem supports symlinks]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of  
+ {@link FileContext#createSymlink(Path, Path, boolean)};]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partially resolves the path. This is used during symlink resolution in
+ {@link FSLinkResolver}, and differs from the similarly named method
+ {@link FileContext#getLinkTarget(Path)}.
+ @throws IOException subclass implementations may throw IOException]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setPermission(Path, FsPermission)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setOwner(Path, String, String)} except that Path f must
+ be for this file system.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setTimes(Path, long, long)} except that Path f must be
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileChecksum(Path)} except that Path f must be for
+ this file system.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileStatus(Path)} 
+ except that an UnresolvedLinkException may be thrown if a symlink is 
+ encountered in the path.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileLinkStatus(Path)}
+ except that an UnresolvedLinkException may be thrown if a symlink is  
+ encountered in the path leading up to the final path component.
+ If the file system does not support symlinks then the behavior is
+ equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFileBlockLocations(Path, long, long)} except that
+ Path f must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#getFsStatus(Path)}.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listStatus(Path)} except that Path f must be for this
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#listLocatedStatus(Path)} except that Path f 
+ must be for this file system.
+
+ In HDFS implementation, the BlockLocation of returned LocatedFileStatus
+ will have different formats for replicated and erasure coded file. Please
+ refer to {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext.Util#listStatus(Path)} except that Path f must be 
+ for this file system.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
+ must be for this file system.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical name for this file system.
+ @return a URI string that uniquely identifies this file system]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing entries to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <AclEntry>} describing modifications, must
+ include entries for user, group, and others for compatibility with
+ permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator{@literal <AclStatus>} which returns each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only the xattr names for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return {@literal Map<String, byte[]>} describing the XAttrs of the file
+ or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#createSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#renameSnapshot(Path, String, String)}.]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="snapshotDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The specification of this method matches that of
+ {@link FileContext#deleteSnapshot(Path, String)}.]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open a file with the given set of options.
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)}in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param path path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="methodNotSupported"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Helper method that throws an {@link UnsupportedOperationException} for the
+ current {@link FileSystem} method being called.]]>
+      </doc>
+    </method>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for implementors of a Hadoop file system
+ (analogous to the VFS of Unix). Applications do not access this class;
+ instead they access files across all file systems using {@link FileContext}.
+ 
+ Pathnames passed to AbstractFileSystem can be fully qualified URI that
+ matches the "this" file system (ie same scheme and authority) 
+ or a Slash-relative name that is assumed to be relative
+ to the root of the "this" file system .]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AbstractFileSystem -->
+  <!-- start class org.apache.hadoop.fs.AvroFSInput -->
+  <class name="AvroFSInput" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.avro.file.SeekableInput"/>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FSDataInputStream, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct given an {@link FSDataInputStream} and its length.]]>
+      </doc>
+    </constructor>
+    <constructor name="AvroFSInput" type="org.apache.hadoop.fs.FileContext, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct given a {@link FileContext} and a {@link Path}.]]>
+      </doc>
+    </constructor>
+    <method name="length" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="tell" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.AvroFSInput -->
+  <!-- start interface org.apache.hadoop.fs.BatchListingOperations -->
+  <interface name="BatchListingOperations"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="batchedListStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="paths" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Batched listing API that returns {@link PartialListing}s for the
+ passed Paths.
+
+ @param paths List of paths to list.
+ @return RemoteIterator that returns corresponding PartialListings.
+ @throws IOException failure]]>
+      </doc>
+    </method>
+    <method name="batchedListLocatedStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="paths" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Batched listing API that returns {@link PartialListing}s for the passed
+ Paths. The PartialListing will contain {@link LocatedFileStatus} entries
+ with locations.
+
+ @param paths List of paths to list.
+ @return RemoteIterator that returns corresponding PartialListings.
+ @throws IOException failure]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface filesystems MAY implement to offer a batched list.
+ If implemented, filesystems SHOULD declare
+ {@link CommonPathCapabilities#FS_EXPERIMENTAL_BATCH_LISTING} to be a supported
+ path capability.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BatchListingOperations -->
+  <!-- start class org.apache.hadoop.fs.BlockLocation -->
+  <class name="BlockLocation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default Constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="org.apache.hadoop.fs.BlockLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset and length.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, offset, length and corrupt flag.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset and length.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with host, name, network topology, offset, length 
+ and corrupt flag.]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BlockLocation" type="java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], java.lang.String[], org.apache.hadoop.fs.StorageType[], long, long, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="getCachedHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of hosts (hostname) hosting a cached replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of names (IP:xferPort) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="getTopologyPaths" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of network topology paths for each of the hosts.
+ The last component of the path is the "name" (IP:xferPort).]]>
+      </doc>
+    </method>
+    <method name="getStorageIds" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storageID of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the storage type of each replica of the block.]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start offset of file associated with this block.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of the block.]]>
+      </doc>
+    </method>
+    <method name="isCorrupt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="isStriped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the block is striped (erasure coded).]]>
+      </doc>
+    </method>
+    <method name="setOffset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="offset" type="long"/>
+      <doc>
+      <![CDATA[Set the start offset of file associated with this block.]]>
+      </doc>
+    </method>
+    <method name="setLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[Set the length of block.]]>
+      </doc>
+    </method>
+    <method name="setCorrupt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="corrupt" type="boolean"/>
+      <doc>
+      <![CDATA[Set the corrupt flag.]]>
+      </doc>
+    </method>
+    <method name="setHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hosts" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the hosts hosting this block.]]>
+      </doc>
+    </method>
+    <method name="setCachedHosts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cachedHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set the hosts hosting a cached replica of this block.]]>
+      </doc>
+    </method>
+    <method name="setNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the names (host:port) hosting this block.]]>
+      </doc>
+    </method>
+    <method name="setTopologyPaths"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="topologyPaths" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the network topology paths of the hosts.]]>
+      </doc>
+    </method>
+    <method name="setStorageIds"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageIds" type="java.lang.String[]"/>
+    </method>
+    <method name="setStorageTypes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="org.apache.hadoop.fs.StorageType[]"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Represents the network location of a block, information about the hosts
+ that contain block replicas, and other block metadata (E.g. the file
+ offset associated with the block, length, whether it is corrupt, etc).
+
+ For a single BlockLocation, it will have different meanings for replicated
+ and erasure coded files.
+
+ If the file is 3-replicated, offset and length of a BlockLocation represent
+ the absolute value in the file and the hosts are the 3 datanodes that
+ holding the replicas. Here is an example:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ </pre>
+
+ And if the file is erasure-coded, each BlockLocation represents a logical
+ block groups. Value offset is the offset of a block group in the file and
+ value length is the total length of a block group. Hosts of a BlockLocation
+ are the datanodes that holding all the data blocks and parity blocks of a
+ block group.
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ A BlockLocation example will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ </pre>
+
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or
+ {@link FileContext#getFileBlockLocations(Path, long, long)}
+ for more examples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.BlockLocation -->
+  <!-- start interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <interface name="BlockStoragePolicySpi"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the storage policy. Policies are uniquely
+ identified by name.
+
+ @return the name of the storage policy.]]>
+      </doc>
+    </method>
+    <method name="getStorageTypes" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the preferred storage types associated with this policy. These
+ storage types are used sequentially for successive block replicas.
+
+ @return preferred storage types used for placing block replicas.]]>
+      </doc>
+    </method>
+    <method name="getCreationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for creating new block replicas. Fallback
+ storage types are used if the preferred storage types are not available.
+
+ @return fallback storage types for new block replicas..]]>
+      </doc>
+    </method>
+    <method name="getReplicationFallbacks" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the fallback storage types for replicating existing block replicas.
+ Fallback storage types are used if the preferred storage types are not
+ available.
+
+ @return fallback storage types for replicating existing block replicas.]]>
+      </doc>
+    </method>
+    <method name="isCopyOnCreateFile" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the policy is inherit-only and cannot be changed for
+ an existing file.
+
+ @return true if the policy is inherit-only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A storage policy specifies the placement of block replicas on specific
+ storage types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.BlockStoragePolicySpi -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <interface name="ByteBufferPositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to {@code buf.remaining()} bytes into buf from a given position
+ in the file and returns the number of bytes read. Callers should use
+ {@code buf.limit(...)} to control the size of the desired read and
+ {@code buf.position(...)} to control the offset into the buffer the data
+ should be written to.
+ <p>
+ After a successful call, {@code buf.position()} will be advanced by the
+ number of bytes read and {@code buf.limit()} will be unchanged.
+ <p>
+ In the case of an exception, the state of the buffer (the contents of the
+ buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is
+ undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p>
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with
+ {@link StreamCapabilities#PREADBYTEBUFFER} to check if the underlying
+ stream supports this interface, otherwise they might get a
+ {@link UnsupportedOperationException}.
+ <p>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+ <p>
+ This does not change the current offset of a file, and is thread-safe.
+
+ @param position position within file
+ @param buf the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if reached
+         end-of-stream
+ @throws IOException if there is some error performing the read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads {@code buf.remaining()} bytes into buf from a given position in
+ the file or until the end of the data was reached before the read
+ operation completed. Callers should use {@code buf.limit(...)} to
+ control the size of the desired read and {@code buf.position(...)} to
+ control the offset into the buffer the data should be written to.
+ <p>
+ This operation provides similar semantics to
+ {@link #read(long, ByteBuffer)}, the difference is that this method is
+ guaranteed to read data until the {@link ByteBuffer} is full, or until
+ the end of the data stream is reached.
+
+ @param position position within file
+ @param buf the ByteBuffer to receive the results of the read operation.
+ @throws IOException if there is some error performing the read
+ @throws EOFException the end of the data was reached before
+ the read operation completed
+ @see #read(long, ByteBuffer)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a positioned read API that writes to a
+ {@link ByteBuffer} rather than a {@code byte[]}.
+
+ @see PositionedReadable
+ @see ByteBufferReadable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferPositionedReadable -->
+  <!-- start interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <interface name="ByteBufferReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads up to buf.remaining() bytes into buf. Callers should use
+ buf.limit(..) to control the size of the desired read.
+ <p>
+ After a successful call, {@code buf.position()} will be advanced by the
+ number of bytes read and {@code buf.limit()} will be unchanged.
+ <p>
+ In the case of an exception, the state of the buffer (the contents of the
+ buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is
+ undefined, and callers should be prepared to recover from this
+ eventuality.
+ <p>
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with
+ {@link StreamCapabilities#READBYTEBUFFER} to check if the underlying
+ stream supports this interface, otherwise they might get a
+ {@link UnsupportedOperationException}.
+ <p>
+ Implementations should treat 0-length requests as legitimate, and must not
+ signal an error upon their receipt.
+
+ @param buf
+          the ByteBuffer to receive the results of the read operation.
+ @return the number of bytes read, possibly zero, or -1 if 
+         reach end-of-stream
+ @throws IOException
+           if there is some error performing the read]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implementers of this interface provide a read API that writes to a
+ ByteBuffer, not a byte[].]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.ByteBufferReadable -->
+  <!-- start interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <interface name="CanSetDropBehind"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setDropBehind"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropCache" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Configure whether the stream should drop the cache.
+
+ @param dropCache     Whether to drop the cache.  null means to use the
+                      default value.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting the drop-behind.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetDropBehind -->
+  <!-- start interface org.apache.hadoop.fs.CanSetReadahead -->
+  <interface name="CanSetReadahead"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setReadahead"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Set the readahead on this stream.
+
+ @param readahead     The readahead to use.  null means to use the default.
+ @throws IOException  If there was an error changing the dropBehind
+                      setting.
+         UnsupportedOperationException  If this stream doesn't support
+                                        setting readahead.]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanSetReadahead -->
+  <!-- start interface org.apache.hadoop.fs.CanUnbuffer -->
+  <interface name="CanUnbuffer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="unbuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reduce the buffering.  This will also free sockets and file descriptors
+ held by the stream, if possible.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSDataInputStreams implement this interface to indicate that they can clear
+ their buffers on request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.CanUnbuffer -->
+  <!-- start class org.apache.hadoop.fs.ChecksumException -->
+  <class name="ChecksumException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumException" type="java.lang.String, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown for checksum errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumException -->
+  <!-- start class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <class name="ChecksumFileSystem" extends="org.apache.hadoop.fs.FilterFileSystem"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChecksumFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApproxChkSumLength" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to verify checksum.]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the raw file system]]>
+      </doc>
+    </method>
+    <method name="getChecksumFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the name of the checksum file associated with a file.]]>
+      </doc>
+    </method>
+    <method name="isChecksumFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return true iff file is a checksum file name.]]>
+      </doc>
+    </method>
+    <method name="getChecksumFileLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="fileSize" type="long"/>
+      <doc>
+      <![CDATA[Return the length of the checksum file given the size of the 
+ actual file.]]>
+      </doc>
+    </method>
+    <method name="getBytesPerSum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the bytes Per Checksum]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChecksumLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <param name="bytesPerSum" type="int"/>
+      <doc>
+      <![CDATA[Calculated the length of the checksum file in bytes.
+ @param size the length of the data file in bytes
+ @param bytesPerSum the number of bytes in a checksum block
+ @return the number of bytes in the checksum file]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename files/dirs]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Implement the delete(Path, boolean) in checksum
+ file system.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given path
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f
+          given path
+ @return the statuses of the files/directories in the given patch
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="copyCrc" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ If src and dst are directories, the copyCrc parameter
+ determines whether to copy CRC files.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Report a checksum error to the file system.
+ @param f the file name containing the error
+ @param in the stream open on the file
+ @param inPos the position of the beginning of the bad data in the file
+ @param sums the stream open on the checksum file
+ @param sumsPos the position of the beginning of the bad data in the checksum file
+ @return if retry is necessary]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's
+ {@link #openFileWithOptions}() method is called, and so ultimately
+ its {@link #open(Path, int)}.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the file as a blocking call to {@link #open(Path, int)}.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's create() method is
+ ultimately called.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[This is overridden to ensure that this class's create() method is
+ ultimately called.
+
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Disable those operations which the checksummed FS blocks.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Abstract Checksumed FileSystem.
+ It provide a basic implementation of a Checksumed FileSystem,
+ which creates a checksum file for each raw file.
+ It generates &amp; verifies checksums at the client side.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ChecksumFileSystem -->
+  <!-- start class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <class name="CommonConfigurationKeysPublic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommonConfigurationKeysPublic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DEFAULT_NAME_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DEFAULT_NAME_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DF_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DF_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_DU_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_DU_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_GETSPACEUSED_JITTER_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_GETSPACEUSED_JITTER_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="NET_DEPENDENCY_SCRIPT_FILE_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_CHECKPOINT_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_PROTECTED_DIRECTORIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Directories that cannot be removed unless empty, even by an
+ administrator.]]>
+      </doc>
+    </field>
+    <field name="FS_LOCAL_BLOCK_SIZE_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Not used anywhere, looks like default value for FS_LOCAL_BLOCK_SIZE]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_AUTOMATIC_CLOSE_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_AUTOMATIC_CLOSE_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CREATION_PARALLEL_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of filesystems instances can be created in parallel.
+ <p></p>
+ A higher number here does not necessarily improve performance, especially
+ for object stores, where multiple threads may be attempting to create an FS
+ instance for the same URI.
+ <p></p>
+ Default value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="FS_CREATION_PARALLEL_COUNT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #FS_CREATION_PARALLEL_COUNT}.
+ <p></p>
+ Default value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="FS_FILE_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_FTP_HOST_PORT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_TRASH_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_TRASH_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAPFILE_BLOOM_ERROR_RATE_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAPFILE_BLOOM_ERROR_RATE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_COMPRESSION_CODEC_LZO_CLASS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Codec class that implements Lzo compression algorithm]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_INTERVAL_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_MAP_INDEX_SKIP_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_MAP_INDEX_SKIP_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SEQFILE_COMPRESS_BLOCKSIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_FILE_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_FILE_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IO_SKIP_CHECKSUM_ERRORS_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IO_SKIP_CHECKSUM_ERRORS_KEY]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.">
+      <doc>
+      <![CDATA[@deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
+ in mapred-default.xml
+ See https://issues.apache.org/jira/browse/HADOOP-6801
+
+ For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+ instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_MB_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_MB_KEY}.]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="SEQ_IO_SORT_FACTOR_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}.]]>
+      </doc>
+    </field>
+    <field name="IO_SERIALIZATIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_IO_CHUNK_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_IO_CHUNK_SIZE_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_INPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_INPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="TFILE_FS_OUTPUT_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for TFILE_FS_OUTPUT_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_RETRY_INTERVAL_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_RETRY_INTERVAL_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable low-latency connections from the client]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_LOW_LATENCY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of IPC_CLIENT_LOW_LATENCY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_LISTEN_QUEUE_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_KILL_MAX_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_KILL_MAX_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_CLIENT_IDLETHRESHOLD_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_CLIENT_IDLETHRESHOLD_DEFAULT]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_TCPNODELAY_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_TCPNODELAY_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_REUSEADDR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_REUSEADDR_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_REUSEADDR_KEY.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_MAX_CONNECTIONS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for IPC_SERVER_MAX_CONNECTIONS_KEY]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Logs if a RPC is really slow compared to rest of RPCs.]]>
+      </doc>
+    </field>
+    <field name="IPC_SERVER_LOG_SLOW_RPC_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_PURGE_INTERVAL_MINUTES_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SOCKS_SERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_UTIL_HASH_TYPE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_UTIL_HASH_TYPE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_MAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUPS_CACHE_BACKGROUND_RELOAD_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.">
+      <doc>
+      <![CDATA[@deprecated use
+ {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+ instead.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHENTICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTHORIZATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SERVICE_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_INTERFACE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_DNS_NAMESERVER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKENS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_AUTHENTICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_RPC_PROTECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Sasl Properties for a connection]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class to override Impersonation provider]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_KEY_DEFAULT_CIPHER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_KEY_DEFAULT_CIPHER_KEY.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_SIZE]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK_DEFAULT" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_LOW_WATERMARK]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_NUM_REFILL_THREADS]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_TIMEOUT_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is the number of providers specified.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 100 ms.]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value is 2 secs.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_LOGS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defalt value for HADOOP_HTTP_LOGS_ENABLED]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_SYSTEM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_SYSTEM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_CUSTOM_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474">
+      <doc>
+      <![CDATA[@deprecated Please use
+ {@link CommonConfigurationKeysPublic#HADOOP_TAGS_CUSTOM} instead
+ See https://issues.apache.org/jira/browse/HADOOP-15474]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TAGS_SYSTEM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_TAGS_CUSTOM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option for the shutdown hook manager shutdown time:
+  {@value}.]]>
+      </doc>
+    </field>
+    <field name="SERVICE_SHUTDOWN_TIMEOUT_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default shutdown hook timeout: {@value} seconds.]]>
+      </doc>
+    </field>
+    <field name="HADOOP_PROMETHEUS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_PROMETHEUS_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@see
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a>]]>
+      </doc>
+    </field>
+    <field name="HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains constants for configuration keys used
+ in the common code.
+
+ It includes all publicly documented configuration keys. In general
+ this class should not be used directly (use CommonConfigurationKeys
+ instead)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CommonConfigurationKeysPublic -->
+  <!-- start class org.apache.hadoop.fs.ContentSummary -->
+  <class name="ContentSummary" extends="org.apache.hadoop.fs.QuotaUsage"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ContentSummary"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder
+  This constructor implicitly set spaceConsumed the same as length.
+  spaceConsumed and length must be set explicitly with
+  ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <constructor name="ContentSummary" type="long, long, long, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, deprecated by ContentSummary.Builder]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the length]]>
+      </doc>
+    </method>
+    <method name="getSnapshotLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the directory count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the file count]]>
+      </doc>
+    </method>
+    <method name="getSnapshotFileCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSnapshotSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getErasureCodingPolicy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="to" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the header of the output.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="getSnapshotHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields from the summary header.
+ 
+ @return names of fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="getQuotaHeaderFields" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of the fields used in the quota summary.
+ 
+ @return names of quota fields as displayed in the header]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+ 
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ For description of the options,
+ @see #toString(boolean, boolean, boolean, boolean, List)
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output if to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qOption" type="boolean"/>
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="xOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if qOption is false, output directory count, file count, and content size;
+ if qOption is true, output quota and remaining quota as well.
+ if hOption is false, file sizes are returned in bytes
+ if hOption is true, file sizes are returned in human readable
+ if tOption is true, display the quota by storage types
+ if tOption is false, same logic with #toString(boolean,boolean)
+ if xOption is false, output includes the calculation from snapshots
+ if xOption is true, output excludes the calculation from snapshots
+
+ @param qOption a flag indicating if quota needs to be printed or not
+ @param hOption a flag indicating if human readable output is to be used
+ @param tOption a flag indicating if display quota by storage types
+ @param xOption a flag indicating if calculation from snapshots is to be
+                included in the output
+ @param types Storage types to display
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="toSnapshot" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <doc>
+      <![CDATA[Return the string representation of the snapshot counts in the output
+ format.
+ @param hOption flag indicating human readable or not
+ @return String representation of the snapshot counts]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Store the summary of a content (a directory or a file).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ContentSummary -->
+  <!-- start class org.apache.hadoop.fs.CreateFlag -->
+  <class name="CreateFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.CreateFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.CreateFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag and throw exception if it is invalid
+ @param flag set of CreateFlag
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.Object"/>
+      <param name="pathExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for create operation
+ @param path Object representing the path; usually String or {@link Path}
+ @param pathExists pass true if the path exists in the file system
+ @param flag set of CreateFlag
+ @throws IOException on error
+ @throws HadoopIllegalArgumentException if the CreateFlag is invalid]]>
+      </doc>
+    </method>
+    <method name="validateForAppend"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[Validate the CreateFlag for the append operation. The flag must contain
+ APPEND, and cannot contain OVERWRITE.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[CreateFlag specifies the file create semantic. Users can combine flags like: <br>
+ <code>
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
+ </code>
+ <p>
+ 
+ Use the CreateFlag as follows:
+ <ol>
+ <li> CREATE - to create a file if it does not exist, 
+ else throw FileAlreadyExists.</li>
+ <li> APPEND - to append to a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> OVERWRITE - to truncate a file if it exists, 
+ else throw FileNotFoundException.</li>
+ <li> CREATE|APPEND - to create a file if it does not exist, 
+ else append to an existing file.</li>
+ <li> CREATE|OVERWRITE - to create a file if it does not exist, 
+ else overwrite an existing file.</li>
+ <li> SYNC_BLOCK - to force closed blocks to the disk device.
+ In addition {@link Syncable#hsync()} should be called after each write,
+ if true synchronous behavior is required.</li>
+ <li> LAZY_PERSIST - Create the block on transient storage (RAM) if
+ available.</li>
+ <li> APPEND_NEWBLOCK - Append data to a new block instead of end of the last
+ partial block.</li>
+ </ol>
+ 
+ Following combinations are not valid and will result in
+ {@link HadoopIllegalArgumentException}:
+ <ol>
+ <li> APPEND|OVERWRITE</li>
+ <li> CREATE|APPEND|OVERWRITE</li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.CreateFlag -->
+  <!-- start class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.fs.FileChecksum -->
+  <class name="FileChecksum" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileChecksum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAlgorithmName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The checksum algorithm name]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The length of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The value of the checksum in bytes]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if both the algorithms and the values are the same.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An abstract class representing file checksums for files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileChecksum -->
+  <!-- start class org.apache.hadoop.fs.FileContext -->
+  <class name="FileContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.AbstractFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the file system of supplied path.
+ 
+ @param absOrFqPath - absolute or fully qualified path
+ @return the file system of the path
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>absOrFqPath</code> is not supported.
+ @throws IOException If the file system for <code>absOrFqPath</code> could
+         not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a FileContext with specified FS as default using the specified
+ config.
+ 
+ @param defFS
+ @param aConf
+ @return new FileContext with specified FS as default.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="defaultFS" type="org.apache.hadoop.fs.AbstractFileSystem"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified file system using the default config.
+ 
+ @param defaultFS
+ @return a FileContext with the specified AbstractFileSystem
+                 as the default FS.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the default config read from the
+ $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted
+ from core-defaults.xml in the release jar.
+ 
+ @throws UnsupportedFileSystemException If the file system from the default
+           configuration is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@return a FileContext for the local file system using the default config.
+ @throws UnsupportedFileSystemException If the file system for
+           {@link FsConstants#LOCAL_FS_URI} is not supported.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified URI using the default config.
+ 
+ @param defaultFsUri
+ @return a FileContext with the specified URI as the default FS.
+ 
+ @throws UnsupportedFileSystemException If the file system for
+           <code>defaultFsUri</code> is not supported]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultFsUri" type="java.net.URI"/>
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext for specified default URI using the specified config.
+ 
+ @param defaultFsUri
+ @param aConf
+ @return new FileContext for specified uri
+ @throws UnsupportedFileSystemException If the file system with specified is
+           not supported
+ @throws RuntimeException If the file system specified is supported but
+         could not be instantiated, or if login fails.]]>
+      </doc>
+    </method>
+    <method name="getFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[Create a FileContext using the passed config. Generally it is better to use
+ {@link #getFileContext(URI, Configuration)} instead of this one.
+ 
+ 
+ @param aConf
+ @return new FileContext
+ @throws UnsupportedFileSystemException If file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="getLocalFSFileContext" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <doc>
+      <![CDATA[@param aConf - from which the FileContext is configured
+ @return a FileContext for the local file system using the specified config.
+ 
+ @throws UnsupportedFileSystemException If default file system in the config
+           is not supported]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newWDir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the working directory for wd-relative names (such a "foo/bar"). Working
+ directory feature is provided by simply prefixing relative names with the
+ working dir. Note this is different from Unix where the wd is actually set
+ to the inode. Hence setWorkingDir does not follow symlinks etc. This works
+ better in a distributed environment that has multiple independent roots.
+ {@link #getWorkingDirectory()} should return what setWorkingDir() set.
+ 
+ @param newWDir new working directory
+ @throws IOException 
+ <br>
+           NewWdir can be one of:
+           <ul>
+           <li>relative path: "foo/bar";</li>
+           <li>absolute without scheme: "/foo/bar"</li>
+           <li>fully qualified with scheme: "xx://auth/foo/bar"</li>
+           </ul>
+ <br>
+           Illegal WDs:
+           <ul>
+           <li>relative with scheme: "xx:foo/bar"</li>
+           <li>non existent directory</li>
+           </ul>]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the working directory for wd-relative names (such a "foo/bar").]]>
+      </doc>
+    </method>
+    <method name="getUgi" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the ugi in the file-context
+ @return UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this file system.
+ The default implementation returns "/user/$USER/".
+ @return the home directory]]>
+      </doc>
+    </method>
+    <method name="getUMask" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the umask of this FileContext]]>
+      </doc>
+    </method>
+    <method name="setUMask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newUmask" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set umask to the supplied parameter.
+ @param newUmask  the new umask]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolve the path following any symlinks or mount points
+ @param f to be resolved
+ @return fully qualified resolved path
+ 
+ @throws FileNotFoundException  If <code>f</code> does not exist
+ @throws AccessControlException if access denied
+ @throws IOException If an IO Error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make the path fully qualified if it is isn't. 
+ A Fully-qualified path has scheme and authority specified and an absolute
+ path.
+ Use the default file system and working dir in this FileContext to qualify.
+ @param path
+ @return qualified path]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create or overwrite file on indicated path and returns an output stream for
+ writing into the file.
+ 
+ @param f the file name to open
+ @param createFlag gives the semantics of create; see {@link CreateFlag}
+ @param opts file creation options; see {@link Options.CreateOpts}.
+          <ul>
+          <li>Progress - to report progress on the operation - default null
+          <li>Permission - umask is applied against permission: default is
+          FsPermissions:getDefault()
+ 
+          <li>CreateParent - create missing parent path; default is to not
+          to create parents
+          <li>The defaults for the following are SS defaults of the file
+          server implementing the target path. Not all parameters make sense
+          for all kinds of file system - eg. localFS ignores Blocksize,
+          replication, checksum
+          <ul>
+          <li>BufferSize - buffersize used in FSDataOutputStream
+          <li>Blocksize - block size for file blocks
+          <li>ReplicationFactor - replication for blocks
+          <li>ChecksumParam - Checksum parameters. server default is used
+          if not specified.
+          </ul>
+          </ul>
+ 
+ @return {@link FSDataOutputStream} for created file
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>f</code> already exists
+ @throws FileNotFoundException If parent of <code>f</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>f</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ a file on indicated path.
+
+ @param f the file path to create builder for.
+ @return {@link FSDataOutputStreamBuilder} to build a
+         {@link FSDataOutputStream}.
+
+ Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ builder parameters will be verified by {@link FileContext} and
+ {@link AbstractFileSystem#create}. And filesystem states will be modified.
+
+ Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ same exceptions as create(Path, EnumSet, CreateOpts...).]]>
+      </doc>
+    </method>
+    <method name="mkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make(create) a directory and all the non-existent parents.
+ 
+ @param dir - the dir to make
+ @param permission - permissions is set permission{@literal &~}umask
+ @param createParent - if true then missing parent dirs are created if false
+          then parent must exist
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If directory <code>dir</code> already
+           exists
+ @throws FileNotFoundException If parent of <code>dir</code> does not exist
+           and <code>createParent</code> is false
+ @throws ParentNotDirectoryException If parent of <code>dir</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>dir</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>dir</code> is not valid]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+ @param f the path to delete.
+ @param recursive if path is a directory and set to 
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws InvalidPathException If path <code>f</code> is invalid]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path using
+ default buffersize.
+ @param f the file name to open
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ 
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+ <li>Fails if path is a directory.
+ <li>Fails if path does not exist.
+ <li>Fails if path is not closed.
+ <li>Fails if new size is greater than current size.
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param f file name
+ @param replication new replication
+
+ @return true if successful
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+ <li>Fails if src is a file and dst is a directory.
+ <li>Fails if src is a directory and dst is a file.
+ <li>Fails if the parent of dst does not exist or is a file.
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails if the dst
+ already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites the dst if
+ it is a file or an empty directory. Rename fails if dst is a non-empty
+ directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for details
+ <p>
+ 
+ @param src path to be renamed
+ @param dst new path after rename
+ 
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If <code>dst</code> already exists and
+           <code>options</code> has {@link Options.Rename#OVERWRITE}
+           option false.
+ @throws FileNotFoundException If <code>src</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>dst</code> is not a
+           directory
+ @throws UnsupportedFileSystemException If file system for <code>src</code>
+           and <code>dst</code> is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param f
+ @param permission - the new absolute permission (umask is not applied)
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code>
+         is not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory). The parameters username
+ and groupname cannot both be null.
+ 
+ @param f The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server
+ 
+ RuntimeExceptions:
+ @throws HadoopIllegalArgumentException If <code>username</code> or
+           <code>groupname</code> is invalid.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param f The path
+ @param mtime Set the modification time of this file.
+        The number of milliseconds since epoch (Jan 1, 1970). 
+        A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+        The number of milliseconds since Jan 1, 1970. 
+        A value of -1 means that this call should not set access time.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file.
+
+ @param f file path
+
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag for the  file system denoted by the path.
+ This is only applicable if the 
+ corresponding FileSystem supports checksum. By default doesn't do anything.
+ @param verifyChecksum
+ @param f set the verifyChecksum for the Filesystem containing this path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+
+ @return a FileStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path. If the path 
+ refers to a symlink then the FileStatus of the symlink is returned.
+ The behavior is equivalent to #getFileStatus() if the underlying
+ file system does not support symbolic links.
+ @param  f The path we want information from.
+ @return A FileStatus object
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the target of the given symbolic link as it was specified
+ when the link was created.  Links in the path leading up to the
+ final path component are resolved transparently.
+
+ @param f the path to return the target of
+ @return The un-interpreted target of the symbolic link.
+ 
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If path <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If the given path does not refer to a symlink
+           or an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="getFsStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ file system denoted by the Parh argument p.
+ If the file system has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ 
+ @param f Path for which status should be obtained. null means the
+ root partition of the default file system. 
+
+ @return a FsStatus object
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a symbolic link to an existing file. An exception is thrown if 
+ the symlink exits, the user does not have permission to create symlink,
+ or the underlying file system does not support symlinks.
+ 
+ Symlink permissions are ignored, access to a symlink is determined by
+ the permissions of the symlink target.
+ 
+ Symlinks in paths leading up to the final path component are resolved 
+ transparently. If the final path component refers to a symlink some 
+ functions operate on the symlink itself, these are:
+ - delete(f) and deleteOnExit(f) - Deletes the symlink.
+ - rename(src, dst) - If src refers to a symlink, the symlink is 
+   renamed. If dst refers to a symlink, the symlink is over-written.
+ - getLinkTarget(f) - Returns the target of the symlink. 
+ - getFileLinkStatus(f) - Returns a FileStatus object describing
+   the symlink.
+ Some functions, create() and mkdir(), expect the final path component
+ does not exist. If they are given a path that refers to a symlink that 
+ does exist they behave as if the path referred to an existing file or 
+ directory. All other functions fully resolve, ie follow, the symlink. 
+ These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
+ setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
+ getFsStatus, getFileStatus, exists, and listStatus.
+ 
+ Symlink targets are stored as given to createSymlink, assuming the 
+ underlying file system is capable of storing a fully qualified URI.
+ Dangling symlinks are permitted. FileContext supports four types of 
+ symlink targets, and resolves them as follows
+ <pre>
+ Given a path referring to a symlink of form:
+ 
+   {@literal <---}X{@literal --->}
+   fs://host/A/B/link 
+   {@literal <-----}Y{@literal ----->}
+ 
+ In this path X is the scheme and authority that identify the file system,
+ and Y is the path leading up to the final path component "link". If Y is
+ a symlink  itself then let Y' be the target of Y and X' be the scheme and
+ authority of Y'. Symlink targets may:
+ 
+ 1. Fully qualified URIs
+ 
+ fs://hostX/A/B/file  Resolved according to the target file system.
+ 
+ 2. Partially qualified URIs (eg scheme but no host)
+ 
+ fs:///A/B/file  Resolved according to the target file system. Eg resolving
+                 a symlink to hdfs:///A results in an exception because
+                 HDFS URIs must be fully qualified, while a symlink to 
+                 file:///A will not since Hadoop's local file systems 
+                 require partially qualified URIs.
+ 
+ 3. Relative paths
+ 
+ path  Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path 
+       is "../B/file" then [Y'][path] is hdfs://host/B/file
+ 
+ 4. Absolute paths
+ 
+ path  Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
+       is "/file" then [X][path] is hdfs://host/file
+ </pre>
+ 
+ @param target the target of the symbolic link
+ @param link the path to be created that points to target
+ @param createParent if true then missing parent dirs are created if 
+                     false then parent must exist
+
+
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If file <code>link</code> already exists
+ @throws FileNotFoundException If <code>target</code> does not exist
+ @throws ParentNotDirectoryException If parent of <code>link</code> is not a
+           directory.
+ @throws UnsupportedFileSystemException If file system for 
+           <code>target</code> or <code>link</code> is not supported
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ 
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory. 
+ Return the file's status and block locations If the path is a file.
+ 
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories 
+         in the given path
+ If any IO exception (for example the input directory gets deleted while
+ listing is being executed), next() or hasNext() of the returned iterator
+ may throw a RuntimeException with the io exception as the cause.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted on JVM shutdown.
+ 
+ @param f the existing path to delete.
+
+ @return  true if deleteOnExit is successful, otherwise false.
+
+ @throws AccessControlException If access is denied
+ @throws UnsupportedFileSystemException If file system for <code>f</code> is
+           not supported
+ @throws IOException If an I/O error occurred
+ 
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws 
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="util" return="org.apache.hadoop.fs.FileContext.Util"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="resolve" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnresolvedLinkException" type="org.apache.hadoop.fs.UnresolvedLinkException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path.
+ Returns the new path object.]]>
+      </doc>
+    </method>
+    <method name="resolveIntermediate" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Resolves all symbolic links in the specified path leading up 
+ to, but not including the final path component.
+ @param f path to resolve
+ @return the new path object.]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system
+ 
+ @param uri
+          the uri to lookup the statistics. Only scheme and authority part
+          of the uri are used as the key to store and lookup.
+ @return a statistics object]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears all the statistics stored in AbstractFileSystem, for all the file
+ systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prints the statistics to standard output. File System is identified by the
+ scheme and authority.]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Map of uri and statistics for each filesystem instantiated. The uri
+         consists of scheme and authority for the filesystem.]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ modifications
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing entries
+ to remove
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List{@literal <}AclEntry{@literal >} describing
+ modifications, must include entries for user, group, and others for
+ compatibility with permission bits.
+ @throws IOException if an ACL could not be modified]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACLs of files and directories.
+
+ @param path Path to get
+ @return RemoteIterator{@literal <}AclStatus{@literal >} which returns
+         each AclStatus
+ @throws IOException if an ACL could not be read]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs for a file or directory.
+ Only those xattrs for which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
+ of the file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List{@literal <}String{@literal >} of the XAttr names of the
+ file or directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+
+ @param path The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+
+ @throws IOException If an I/O error occurred
+
+ <p>Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+           undeclared exception to RPC server]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param path file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param path file or directory path.
+ @return storage policy for give file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(Path, int)} unless a subclass
+ executes the open command differently.
+
+ The semantics of this call are therefore the same as that of
+ {@link #open(Path, int)} with one special point: it is in
+ {@code FSDataInputStreamBuilder.build()} in which the open operation
+ takes place -it is there where all preconditions to the operation
+ are checked.
+ @param path file path
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path capabilities of the bonded {@code AbstractFileSystem}.
+ @param path path to query the capability of.
+ @param capability string to query the stream support for.
+ @return true iff the capability is supported under that FS.
+ @throws IOException path resolution or other IO failure
+ @throws IllegalArgumentException invalid arguments]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values based on path.
+ @param path path to fetch server defaults
+ @return server default configuration values for path
+ @throws IOException an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory and symlink
+ In previous versions, this default permission was also used to
+ create files, so files created end up with ugo+x permission.
+ See HADOOP-9155 for detail. 
+ Two new constants are added to solve this, please use 
+ {@link FileContext#DIR_DEFAULT_PERM} for directory, and use
+ {@link FileContext#FILE_DEFAULT_PERM} for file.
+ This constant is kept for compatibility.]]>
+      </doc>
+    </field>
+    <field name="DIR_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for directory]]>
+      </doc>
+    </field>
+    <field name="FILE_DEFAULT_PERM" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permission for file]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileContext shutdown hook.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The FileContext class provides an interface for users of the Hadoop
+ file system. It exposes a number of file system operations, e.g. create,
+ open, list.
+ 
+ <h2>Path Names</h2>
+ 
+ The Hadoop file system supports a URI namespace and URI names. This enables
+ multiple types of file systems to be referenced using fully-qualified URIs.
+ Two common Hadoop file system implementations are
+ <ul>
+ <li>the local file system: file:///path
+ <li>the HDFS file system: hdfs://nnAddress:nnPort/path
+ </ul>
+ 
+ The Hadoop file system also supports additional naming schemes besides URIs.
+ Hadoop has the concept of a <i>default file system</i>, which implies a
+ default URI scheme and authority. This enables <i>slash-relative names</i>
+ relative to the default FS, which are more convenient for users and
+ application writers. The default FS is typically set by the user's
+ environment, though it can also be manually specified.
+ <p>
+ 
+ Hadoop also supports <i>working-directory-relative</i> names, which are paths
+ relative to the current working directory (similar to Unix). The working
+ directory can be in a different file system than the default FS.
+ <p>
+ Thus, Hadoop path names can be specified as one of the following:
+ <ul>
+ <li>a fully-qualified URI: scheme://authority/path (e.g.
+ hdfs://nnAddress:nnPort/foo/bar)
+ <li>a slash-relative name: path relative to the default file system (e.g.
+ /foo/bar)
+ <li>a working-directory-relative name: path relative to the working dir (e.g.
+ foo/bar)
+ </ul>
+  Relative paths with scheme (scheme:foo/bar) are illegal.
+  
+ <h2>Role of FileContext and Configuration Defaults</h2>
+
+ The FileContext is the analogue of per-process file-related state in Unix. It
+ contains two properties:
+ 
+ <ul>
+ <li>the default file system (for resolving slash-relative names)
+ <li>the umask (for file permissions)
+ </ul>
+ In general, these properties are obtained from the default configuration file
+ in the user's environment (see {@link Configuration}).
+ 
+ Further file system properties are specified on the server-side. File system
+ operations default to using these server-side defaults unless otherwise
+ specified.
+ <p>
+ The file system related server-side defaults are:
+  <ul>
+  <li> the home directory (default is "/user/userName")
+  <li> the initial wd (only for local fs)
+  <li> replication factor
+  <li> block size
+  <li> buffer size
+  <li> encryptDataTransfer 
+  <li> checksum option. (checksumType and  bytesPerChecksum)
+  </ul>
+
+ <h2>Example Usage</h2>
+
+ Example 1: use the default config read from the $HADOOP_CONFIG/core.xml.
+   Unspecified values come from core-defaults.xml in the release jar.
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(); // uses the default config
+                                                // which has your default FS 
+  <li>  myFContext.create(path, ...);
+  <li>  myFContext.setWorkingDir(path);
+  <li>  myFContext.open (path, ...);  
+  <li>...
+  </ul>  
+ Example 2: Get a FileContext with a specific URI as the default FS
+  <ul>  
+  <li> myFContext = FileContext.getFileContext(URI);
+  <li> myFContext.create(path, ...);
+  <li>...
+ </ul>
+ Example 3: FileContext with local file system as the default
+  <ul> 
+  <li> myFContext = FileContext.getLocalFSFileContext();
+  <li> myFContext.create(path, ...);
+  <li> ...
+  </ul> 
+ Example 4: Use a specific config, ignoring $HADOOP_CONFIG
+  Generally you should not need use a config unless you are doing
+   <ul> 
+   <li> configX = someConfigSomeOnePassedToYou;
+   <li> myFContext = getFileContext(configX); // configX is not changed,
+                                              // is passed down 
+   <li> myFContext.create(path, ...);
+   <li>...
+  </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileContext -->
+  <!-- start class org.apache.hadoop.fs.FileStatus -->
+  <class name="FileStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for file systems on which symbolic links are not supported]]>
+      </doc>
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileStatus" type="org.apache.hadoop.fs.FileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor.
+
+ @param other FileStatus to copy]]>
+      </doc>
+    </constructor>
+    <method name="attributes" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acl" type="boolean"/>
+      <param name="crypt" type="boolean"/>
+      <param name="ec" type="boolean"/>
+      <param name="sn" type="boolean"/>
+      <doc>
+      <![CDATA[Convert boolean attributes to a set of flags.
+ @param acl   See {@link AttrFlags#HAS_ACL}.
+ @param crypt See {@link AttrFlags#HAS_CRYPT}.
+ @param ec    See {@link AttrFlags#HAS_EC}.
+ @param sn    See {@link AttrFlags#SNAPSHOT_ENABLED}.
+ @return converted set of flags.]]>
+      </doc>
+    </method>
+    <method name="getLen" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the length of this file, in bytes.
+ @return the length of this file, in bytes.]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a file?
+ @return true if this is a file]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a directory?
+ @return true if this is a directory]]>
+      </doc>
+    </method>
+    <method name="isDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.">
+      <doc>
+      <![CDATA[Old interface, instead use the explicit {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ @return true if this is a directory.
+ @deprecated Use {@link FileStatus#isFile()},
+ {@link FileStatus#isDirectory()}, and {@link FileStatus#isSymlink()}
+ instead.]]>
+      </doc>
+    </method>
+    <method name="isSymlink" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this a symbolic link?
+ @return true if this is a symbolic link]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the block size of the file.
+ @return the number of bytes]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the replication factor of a file.
+ @return the replication factor of a file.]]>
+      </doc>
+    </method>
+    <method name="getModificationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modification time of the file.
+ @return the modification time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getAccessTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the access time of the file.
+ @return the access time of file in milliseconds since January 1, 1970 UTC.]]>
+      </doc>
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get FsPermission associated with the file.
+ @return permission. If a filesystem does not have a notion of permissions
+         or if permissions could not be determined, then default 
+         permissions equivalent of "rwxrwxrwx" is returned.]]>
+      </doc>
+    </method>
+    <method name="hasAcl" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory has ACLs set.
+
+ @return true if the underlying file or directory has ACLs set.]]>
+      </doc>
+    </method>
+    <method name="isEncrypted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is encrypted or not.
+
+ @return true if the underlying file is encrypted.]]>
+      </doc>
+    </method>
+    <method name="isErasureCoded" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Tell whether the underlying file or directory is erasure coded or not.
+
+ @return true if the underlying file or directory is erasure coded.]]>
+      </doc>
+    </method>
+    <method name="isSnapshotEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if directory is Snapshot enabled or not.
+
+ @return true if directory is snapshot enabled]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the owner of the file.
+ @return owner of the file. The string could be empty if there is no
+         notion of owner of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getGroup" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group associated with the file.
+ @return group for the file. The string could be empty if there is no
+         notion of group of a file in a filesystem or if it could not 
+         be determined (rare).]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Sets permission.
+ @param permission if permission is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets owner.
+ @param owner if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="setGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets group.
+ @param group if it is null, default value is set]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return The contents of the symbolic link.]]>
+      </doc>
+    </method>
+    <method name="setSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus.
+ This method was added back by HADOOP-14683 to keep binary compatibility.
+
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.
+ @throws ClassCastException if the specified object is not FileStatus]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read instance encoded as protobuf from stream.
+ @param in Input stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use the {@link PBHelper} and protobuf serialization directly.">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write instance encoded as protobuf to stream.
+ @param out Output stream
+ @see PBHelper#convert(FileStatus)
+ @deprecated Use the {@link PBHelper} and protobuf serialization directly.]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+    </method>
+    <field name="NONE" type="java.util.Set"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shared, empty set of attributes (a common case for FileStatus).]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface that represents the client side information for a file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileStatus -->
+  <!-- start class org.apache.hadoop.fs.FileSystem -->
+  <class name="FileSystem" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.hadoop.security.token.DelegationTokenIssuer"/>
+    <implements name="org.apache.hadoop.fs.PathCapabilities"/>
+    <constructor name="FileSystem"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a FileSystem instance based on the uri, the passed in
+ configuration and the user.
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return the filesystem instance
+ @throws IOException failure to load
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+ somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the configured FileSystem implementation.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="getDefaultUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the default FileSystem URI from a configuration.
+ @param conf the configuration to use
+ @return the uri of the default filesystem]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="setDefaultUri"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the default FileSystem URI in a configuration.
+ @param conf the configuration to alter
+ @param uri the new default filesystem uri]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a FileSystem.
+
+ Called after the new FileSystem instance is constructed, and before it
+ is ready for use.
+
+ FileSystem implementations overriding this method MUST forward it to
+ their superclass, though the order in which it is done, and whether
+ to alter the configuration before the invocation are options of the
+ subclass.
+ @param name a URI whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration
+ @throws IOException on any failure to initialize this instance.
+ @throws IllegalArgumentException if the URI is considered invalid.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for this FileSystem.
+ <p>
+ This implementation throws an <code>UnsupportedOperationException</code>.
+
+ @return the protocol scheme for this FileSystem.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI which identifies this FileSystem.
+
+ @return the URI of this filesystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a canonicalized form of this FileSystem's URI.
+
+ The default implementation simply calls {@link #canonicalizeUri(URI)}
+ on the filesystem's own URI, so subclasses typically only need to
+ implement that method.
+
+ @see #canonicalizeUri(URI)]]>
+      </doc>
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Canonicalize the given URI.
+
+ This is implementation-dependent, and may for example consist of
+ canonicalizing the hostname using DNS and adding the default
+ port if not specified.
+
+ The default implementation simply fills in the default port if
+ not specified and if {@link #getDefaultPort()} returns a
+ default port.
+
+ @return URI
+ @see NetUtils#getCanonicalUri(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FileSystem.
+ @return the default port or 0 if there isn't one]]>
+      </doc>
+    </method>
+    <method name="getFSofPath" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="absOrFqPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCanonicalServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a canonical service name for this FileSystem.
+ The token cache is the only user of the canonical service name,
+ and uses it to lookup this FileSystem's service tokens.
+ If the file system provides a token of its own then it must have a
+ canonical name, otherwise the canonical name can be null.
+
+ Default implementation: If the FileSystem has child file systems
+ (such as an embedded file system) then it is assumed that the FS has no
+ tokens of its own and hence returns a null name; otherwise a service
+ name is built using Uri and port.
+
+ @return a service string that uniquely identifies this file system, null
+         if the filesystem does not implement tokens
+ @see SecurityUtil#buildDTServiceName(URI, int)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="call {@link #getUri()} instead.">
+      <doc>
+      <![CDATA[@deprecated call {@link #getUri()} instead.]]>
+      </doc>
+    </method>
+    <method name="getNamed" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="call {@link #get(URI, Configuration)} instead.">
+      <param name="name" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated call {@link #get(URI, Configuration)} instead.]]>
+      </doc>
+    </method>
+    <method name="getLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the local FileSystem.
+ @param conf the configuration to configure the FileSystem with
+ if it is newly instantiated.
+ @return a LocalFileSystem
+ @throws IOException if somehow the local FS cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a FileSystem for this URI's scheme and authority.
+ <ol>
+ <li>
+   If the configuration has the property
+   {@code "fs.$SCHEME.impl.disable.cache"} set to true,
+   a new instance will be created, initialized with the supplied URI and
+   configuration, then returned without being cached.
+ </li>
+ <li>
+   If the there is a cached FS instance matching the same URI, it will
+   be returned.
+ </li>
+ <li>
+   Otherwise: a new FS instance will be created, initialized with the
+   configuration and URI, cached and returned to the caller.
+ </li>
+ </ol>
+ @throws IOException if the FileSystem cannot be instantiated.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority and the
+ given user. Internally invokes {@link #newInstance(URI, Configuration)}
+ @param uri of the filesystem
+ @param conf the configuration to use
+ @param user to perform the get as
+ @return filesystem instance
+ @throws IOException if the FileSystem cannot be instantiated.
+ @throws InterruptedException If the {@code UGI.doAs()} call was
+         somehow interrupted.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the FileSystem for this URI's scheme and authority.
+ The entire URI is passed to the FileSystem instance's initialize method.
+ This always returns a new FileSystem object.
+ @param uri FS URI
+ @param config configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a unique configured FileSystem implementation for the default
+ filesystem of the supplied configuration.
+ This always returns a new FileSystem object.
+ @param conf the configuration to use
+ @return the new FS instance
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="newInstanceLocal" return="org.apache.hadoop.fs.LocalFileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a unique local FileSystem object.
+ @param conf the configuration to configure the FileSystem with
+ @return a new LocalFileSystem object.
+ @throws IOException FS creation or initialization failure.]]>
+      </doc>
+    </method>
+    <method name="closeAll"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances. After this operation, they
+ may not be used in any operations.
+
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="closeAllForUGI"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all cached FileSystem instances for a given UGI.
+ Be sure those filesystems are not used anymore.
+ @param ugi user group info to close
+ @throws IOException a problem arose closing one or more filesystem.]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Qualify a path to one which uses this FileSystem and, if relative,
+ made absolute.
+ @param path to qualify.
+ @return this path if it contains a scheme and authority and is absolute, or
+ a new path that includes a path and authority and is fully qualified
+ @see Path#makeQualified(URI, Path)
+ @throws IllegalArgumentException if the path has a schema/URI different
+ from this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file with the provided permission.
+
+ The permission of the file is set to be the provided permission as in
+ setPermission, not permission{@literal &~}umask
+
+ The HDFS implementation is implemented using two RPCs.
+ It is understood that it is inefficient,
+ but the implementation is thread-safe. The other option is to change the
+ value of umask in configuration to be 0, but it is not thread-safe.
+
+ @param fs FileSystem
+ @param file the name of the file to be created
+ @param permission the permission of the file
+ @return an output stream
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a directory with the provided permission.
+ The permission of the directory is set to be the provided permission as in
+ setPermission, not permission{@literal &~}umask
+
+ @see #create(FileSystem, Path, FsPermission)
+
+ @param fs FileSystem handle
+ @param dir the name of the directory to be created
+ @param permission the permission of the directory
+ @return true if the directory creation succeeds; false otherwise
+ @throws IOException A problem creating the directories.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.
+
+ The base implementation performs case insensitive equality checks
+ of the URIs' schemes and authorities. Subclasses may implement slightly
+ different checks.
+ @param path to check
+ @throws IllegalArgumentException if the path is not considered to be
+ part of this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For nonexistent
+ file or regions, {@code null} is returned.
+
+ <pre>
+   if f == null :
+     result = null
+   elif f.getLen() {@literal <=} start:
+     result = []
+   else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)]
+ </pre>
+ This call is most helpful with and distributed filesystem
+ where the hostnames of machines that contain blocks of the given file
+ can be determined.
+
+ The default implementation returns an array containing one element:
+ <pre>
+ BlockLocation( { "localhost:9866" },  { "localhost" }, 0, file.getLen())
+ </pre>
+
+ In HDFS, if file is three-replicated, the returned array contains
+ elements like:
+ <pre>
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+   hosts: {"host2:9866", "host3:9866, host4:9866"})
+ </pre>
+
+ And if a file is erasure-coded, the returned BlockLocation are logical
+ block groups.
+
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then
+ there will be one BlockLocation returned, with 0 offset, actual file size
+ and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks.
+ 3. If the file size is less than one group size but greater than one
+ stripe size, then there will be one BlockLocation returned, with 0 offset,
+ actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting
+ the actual blocks.
+ 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123
+ for example, then the result will be like:
+ <pre>
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+   "host4:9866", "host5:9866"})
+ </pre>
+
+ @param file FilesStatus to get data from
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array containing hostnames, offset and size of
+ portions of the given file.  For a nonexistent
+ file or regions, {@code null} is returned.
+
+ This call is most helpful with location-aware distributed
+ filesystems, where it returns hostnames of machines that
+ contain the given file.
+
+ A FileSystem will normally return the equivalent result
+ of passing the {@code FileStatus} of the path to
+ {@link #getFileBlockLocations(FileStatus, long, long)}
+
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getServerDefaults(Path)} instead">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @return server default configuration values
+ @throws IOException IO failure
+ @deprecated use {@link #getServerDefaults(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a set of server default configuration values.
+ @param p path is used to identify an FS since an FS could have
+          another FS that it could be delegating the call to
+ @return server default configuration values
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the fully-qualified path of path, resolving the path
+ through any symlinks or mount point.
+ @param p path to be resolved
+ @return fully qualified path
+ @throws FileNotFoundException if the path is not present
+ @throws IOException for any other error]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file to open
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If {@link #open(PathHandle, int)}
+                                       not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open an FSDataInputStream matching the PathHandle instance. The
+ implementation may encode metadata in PathHandle to address the
+ resource directly and verify that the resource referenced
+ satisfies constraints specified at its construciton.
+ @param fd PathHandle object returned by the FS authority.
+ @param bufferSize the size of the buffer to use
+ @throws InvalidPathHandleException If {@link PathHandle} constraints are
+                                    not satisfied
+ @throws IOException IO failure
+ @throws UnsupportedOperationException If not overridden by subclass]]>
+      </doc>
+    </method>
+    <method name="getPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Create a durable, serializable handle to the referent of the given
+ entity.
+ @param stat Referent in the target FileSystem
+ @param opt If absent, assume {@link HandleOpt#path()}.
+ @throws IllegalArgumentException If the FileStatus does not belong to
+         this FileSystem
+ @throws UnsupportedOperationException If {@link #createPathHandle}
+         not overridden by subclass.
+ @throws UnsupportedOperationException If this FileSystem cannot enforce
+         the specified constraints.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opt" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opt Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an exception will be thrown.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ Files are overwritten by default.
+ @param f the file to create
+ @param replication the replication factor
+ @param progress to report progress
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file to create
+ @param overwrite if a path with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an {@link FSDataOutputStream} at the indicated Path
+ with write-progress reporting.
+
+ The frequency of callbacks is implementation-specific; it may be "none".
+ @param f the path of the file to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+   the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with write-progress
+ reporting.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the indicated Path with a custom
+ checksum option.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @param checksumOpt checksum parameter. If null, the values
+        found in conf will be used.
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This create has been added to support the FileContext that processes
+ the permission with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.
+ @param f path
+ @param absolutePermission permissions
+ @return true if the directory was actually created.
+ @throws IOException IO failure
+ @see #mkdirs(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="primitiveMkdir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This version of the mkdirs method assumes that the permission is absolute.
+ It has been added to support the FileContext that processes the permission
+ with umask before calling this method.
+ This a temporary method added to support the transition from FileSystem
+ to FileContext for user applications.]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param overwrite if a file with this name already exists, then if true,
+ the file will be overwritten, and if false an error will be thrown.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataOutputStream at the indicated Path with write-progress
+ reporting. Same as create(), except fails if parent directory doesn't
+ already exist.
+ @param f the file name to open
+ @param permission file permission
+ @param flags {@link CreateFlag}s to use for this stream.
+ @param bufferSize the size of the buffer to be used.
+ @param replication required block replication for the file.
+ @param blockSize block size
+ @param progress the progress reporter
+ @throws IOException IO failure
+ @see #setPermission(Path, FsPermission)]]>
+      </doc>
+    </method>
+    <method name="createNewFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the given Path as a brand-new zero-length file.  If
+ create fails, or if it already existed, return false.
+ <i>Important: the default implementation is not atomic</i>
+ @param f path to use for create
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as
+ {@code append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+     IO_FILE_BUFFER_SIZE_DEFAULT), null)}
+ @param f the existing file to be appended.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ Same as append(f, bufferSize, null).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).
+ @param f the existing file to be appended.
+ @param bufferSize the size of the buffer to be used.
+ @param progress for reporting progress if it is not null.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Concat existing files together.
+ @param trg the path to the target destination.
+ @param psrcs the paths to the sources to use for the concatenation.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the replication factor.
+
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @param src file name
+ @return file replication
+ @throws FileNotFoundException if the path does not resolve.
+ @throws IOException an IO failure]]>
+      </doc>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the replication for an existing file.
+ If a filesystem does not support replication, it will always
+ return true: the check for a file existing may be bypassed.
+ This is the default behavior.
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful, or the feature in unsupported;
+         false if replication is supported but the file does not exist,
+         or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws IOException on failure
+ @return true if rename is successful]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst
+ <ul>
+   <li>Fails if src is a file and dst is a directory.</li>
+   <li>Fails if src is a directory and dst is a file.</li>
+   <li>Fails if the parent of dst does not exist or is a file.</li>
+ </ul>
+ <p>
+ If OVERWRITE option is not passed as an argument, rename fails
+ if the dst already exists.
+ <p>
+ If OVERWRITE option is passed as an argument, rename overwrites
+ the dst if it is a file or an empty directory. Rename fails if dst is
+ a non-empty directory.
+ <p>
+ Note that atomicity of rename is dependent on the file system
+ implementation. Please refer to the file system documentation for
+ details. This default implementation is non atomic.
+ <p>
+ This method is deprecated since it is a temporary method added to
+ support the transition from FileSystem to FileContext for user
+ applications.
+
+ @param src path to be renamed
+ @param dst new path after rename
+ @throws FileNotFoundException src path does not exist, or the parent
+ path of dst does not exist.
+ @throws FileAlreadyExistsException dest path exists and is a file
+ @throws ParentNotDirectoryException if the parent path of dest is not
+ a directory
+ @throws IOException on failure]]>
+      </doc>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Truncate the file in the indicated path to the indicated size.
+ <ul>
+   <li>Fails if path is a directory.</li>
+   <li>Fails if path does not exist.</li>
+   <li>Fails if path is not closed.</li>
+   <li>Fails if new size is greater than current size.</li>
+ </ul>
+ @param f The path to the file to be truncated
+ @param newLength The size the file is to be truncated to
+
+ @return <code>true</code> if the file has been truncated to the desired
+ <code>newLength</code> and is immediately available to be reused for
+ write operations such as <code>append</code>, or
+ <code>false</code> if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #delete(Path, boolean)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file/directory.
+ @deprecated Use {@link #delete(Path, boolean)} instead.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file.
+
+ @param f the path to delete.
+ @param recursive if path is a directory and set to
+ true, the directory is deleted else throws an exception. In
+ case of a file the recursive can be set to either true or false.
+ @return  true if delete is successful else false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="deleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Mark a path to be deleted when its FileSystem is closed.
+ When the JVM shuts down cleanly, all cached FileSystem objects will be
+ closed automatically. These the marked paths will be deleted as a result.
+
+ If a FileSystem instance is not cached, i.e. has been created with
+ {@link #createFileSystem(URI, Configuration)}, then the paths will
+ be deleted in when {@link #close()} is called on that instance.
+
+ The path must exist in the filesystem at the time of the method call;
+ it does not have to exist at the time of JVM shutdown.
+
+ Notes
+ <ol>
+   <li>Clean shutdown of the JVM cannot be guaranteed.</li>
+   <li>The time to shut down a FileSystem will depends on the number of
+   files to delete. For filesystems where the cost of checking
+   for the existence of a file/directory and the actual delete operation
+   (for example: object stores) is high, the time to shutdown the JVM can be
+   significantly extended by over-use of this feature.</li>
+   <li>Connectivity problems with a remote filesystem may delay shutdown
+   further, and may cause the files to not be deleted.</li>
+ </ol>
+ @param f the path to delete.
+ @return  true if deleteOnExit is successful, otherwise false.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="cancelDeleteOnExit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Cancel the scheduled deletion of the path when the FileSystem is closed.
+ @param f the path to cancel deletion
+ @return true if the path was found in the delete-on-exit list.]]>
+      </doc>
+    </method>
+    <method name="processDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delete all paths that were marked as delete-on-exit. This recursively
+ deletes all files and directories in the specified paths.
+
+ The time to process this operation is {@code O(paths)}, with the actual
+ time dependent on the time for existence and deletion operations to
+ complete, successfully or not.]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if a path exists.
+
+ It is highly discouraged to call this method back to back with other
+ {@link #getFileStatus(Path)} calls, as this will involve multiple redundant
+ RPC calls in HDFS.
+
+ @param f source path
+ @return true if the path exists
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="isDirectory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a directory.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by getFileStatus() or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="isFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[True iff the named path is a regular file.
+ Note: Avoid using this method. Instead reuse the FileStatus
+ returned by {@link #getFileStatus(Path)} or listStatus() methods.
+
+ @param f path to check
+ @throws IOException IO failure
+ @deprecated Use {@link #getFileStatus(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead.">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The number of bytes in a file.
+ @return the number of bytes; 0 for a directory
+ @deprecated Use {@link #getFileStatus(Path)} instead.
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getContentSummary" return="org.apache.hadoop.fs.ContentSummary"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link ContentSummary} of a given {@link Path}.
+ @param f path to use
+ @throws FileNotFoundException if the path does not resolve
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="org.apache.hadoop.fs.QuotaUsage"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the {@link QuotaUsage} of a given {@link Path}.
+ @param f path to use
+ @return the quota usage
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="namespaceQuota" type="long"/>
+      <param name="storagespaceQuota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set quota for the given {@link Path}.
+
+ @param src the target path to set quota for
+ @param namespaceQuota the namespace quota (i.e., # of files/directories)
+                       to set
+ @param storagespaceQuota the storage space quota to set
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setQuotaByStorageType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <param name="quota" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set per storage type quota for the given {@link Path}.
+
+ @param src the target path to set storage type quota for
+ @param type the storage type to set
+ @param quota the quota to set for the given storage type
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+ <p>
+ Will not return null. Expect IOException upon access error.
+ @param f given path
+ @return the statuses of the files/directories in the given patch
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List corrupted file blocks.
+ @return an iterator over the corrupt files under the given path
+ (may contain duplicates if a file has more than one corrupt block)
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default).
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given path using the user-supplied path
+ filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param f
+          a path name
+ @param filter
+          the user-supplied path filter
+ @return an array of FileStatus objects for the files under the given path
+         after applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using default
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @return a list of statuses for the files under the given paths after
+         applying the filter default Path filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Filter files/directories in the given list of paths using user-supplied
+ path filter.
+ <p>
+ Does not guarantee to return the List of files/directories status in a
+ sorted order.
+
+ @param files
+          a list of paths
+ @param filter
+          the user-supplied path filter
+ @return a list of statuses for the files under the given paths after
+         applying the filter
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Return all the files that match filePattern and are not checksum
+ files. Results are sorted by their names.
+
+ <p>
+ A filename pattern is composed of <i>regular</i> characters and
+ <i>special pattern matching</i> characters, which are:
+
+ <dl>
+  <dd>
+   <dl>
+    <dt> <tt> ? </tt>
+    <dd> Matches any single character.
+
+    <p>
+    <dt> <tt> * </tt>
+    <dd> Matches zero or more characters.
+
+    <p>
+    <dt> <tt> [<i>abc</i>] </tt>
+    <dd> Matches a single character from character set
+     <tt>{<i>a,b,c</i>}</tt>.
+
+    <p>
+    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
+    <dd> Matches a single character from the character range
+     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
+     lexicographically less than or equal to character <tt><i>b</i></tt>.
+
+    <p>
+    <dt> <tt> [^<i>a</i>] </tt>
+    <dd> Matches a single character that is not from character set or range
+     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
+     immediately to the right of the opening bracket.
+
+    <p>
+    <dt> <tt> \<i>c</i> </tt>
+    <dd> Removes (escapes) any special meaning of character <i>c</i>.
+
+    <p>
+    <dt> <tt> {ab,cd} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
+
+    <p>
+    <dt> <tt> {ab,c{de,fh}} </tt>
+    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
+
+   </dl>
+  </dd>
+ </dl>
+
+ @param pathPattern a glob specifying a path pattern
+
+ @return an array of paths that match the path pattern
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="globStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathPattern" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of {@link FileStatus} objects whose path names match
+ {@code pathPattern} and is accepted by the user-supplied path filter.
+ Results are sorted by their path names.
+
+ @param pathPattern a glob specifying the path pattern
+ @param filter a user-supplied path filter
+ @return null if {@code pathPattern} has no glob and the path does not exist
+         an empty array if {@code pathPattern} has a glob and no path
+         matches it else an array of {@link FileStatus} objects matching the
+         pattern
+ @throws IOException if any I/O error occurs when fetching file status]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses of the files/directories in the given path if the path is
+ a directory.
+ Return the file's status and block locations If the path is a file.
+
+ If a returned status is a file, it contains the file's block locations.
+
+ @param f is the path
+
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+
+ @throws FileNotFoundException If <code>f</code> does not exist
+ @throws IOException If an I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List a directory.
+ The returned results include its block location if it is a file
+ The results are filtered by the given path filter
+ @param f a path
+ @param filter a path filter
+ @return an iterator that traverses statuses of the files/directories
+         in the given path
+ @throws FileNotFoundException if <code>f</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a remote iterator so that followup calls are made on demand
+ while consuming the entries. Each FileSystem implementation should
+ override this method and provide a more efficient implementation, if
+ possible.
+
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ @param p target path
+ @return remote iterator
+ @throws FileNotFoundException if <code>p</code> does not exist
+ @throws IOException if any I/O error occurred]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List the statuses and block locations of the files in the given path.
+ Does not guarantee to return the iterator that traverses statuses
+ of the files in a sorted order.
+
+ <pre>
+ If the path is a directory,
+   if recursive is false, returns files in the directory;
+   if recursive is true, return files in the subtree rooted at the path.
+ If the path is a file, return the file's status and block locations.
+ </pre>
+ @param f is the path
+ @param recursive if the subdirectories need to be traversed recursively
+
+ @return an iterator that traverses statuses of the files
+
+ @throws FileNotFoundException when the path does not exist;
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the current user's home directory in this FileSystem.
+ The default implementation returns {@code "/user/$USER/"}.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given FileSystem. All relative
+ paths will be resolved relative to it.
+
+ @param new_dir Path of new working directory]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given FileSystem
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Note: with the new FileContext class, getWorkingDirectory()
+ will be removed.
+ The working directory is implemented in FileContext.
+
+ Some FileSystems like LocalFileSystem have an initial workingDir
+ that we use as the starting workingDir. For other file systems
+ like HDFS there is no built in notion of an initial workingDir.
+
+ @return if there is built in notion of workingDir then it
+ is returned; else a null is returned.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Call {@link #mkdirs(Path, FsPermission)} with default permission.
+ @param f path
+ @return true if the directory was created
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make the given file and all non-existent parents into
+ directories. Has roughly the semantics of Unix @{code mkdir -p}.
+ Existence of the directory hierarchy is not an error.
+ @param f path to create
+ @param permission to apply to f
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to filesystem at
+ the given dst name and the source is kept intact afterwards
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files is on the local disk.  Add it to filesystem at
+ the given dst name, removing the source afterwards.
+ @param srcs source paths
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name, removing the source afterwards.
+ @param src local path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param src path
+ @param dst path]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param srcs array of paths which are source
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to the filesystem at
+ the given dst name.
+ delSrc indicates if the source should be removed
+ @param delSrc whether to delete the src
+ @param overwrite whether to overwrite an existing file
+ @param src path
+ @param dst path
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from the remote filesystem to the local one.
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="moveToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy a file to the local filesystem, then delete it from the
+ remote filesystem (if successfully copied).
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy it a file from a remote filesystem to the local one.
+ delSrc indicates if the src will be removed or not.
+ @param delSrc whether to delete the src
+ @param src path src file in the remote filesystem
+ @param dst path local destination
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="useRawLocalFileSystem" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under this filesystem, and the dst is on the local disk.
+ Copy it from the remote filesystem to the local dst name.
+ delSrc indicates if the src will be removed
+ or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
+ as the local file system or not. RawLocalFileSystem is non checksumming,
+ So, It will not create any crc files at local.
+
+ @param delSrc
+          whether to delete the src
+ @param src
+          path
+ @param dst
+          path
+ @param useRawLocalFileSystem
+          whether to use RawLocalFileSystem as local file system or not.
+
+ @throws IOException for any IO error]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local file that the user can write output to.  The caller
+ provides both the eventual target name in this FileSystem
+ and the local working file path.
+ If this FileSystem is local, we write directly into the target.  If
+ the FileSystem is not local, we write into the tmp local area.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path of local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.
+ A local FS will do nothing, because we've written to exactly the
+ right place.
+ A remote FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.
+ @param fsOutputFile path of output file
+ @param tmpLocalFile path to local tmp file
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this FileSystem instance.
+ Will release any held locks, delete all files queued for deletion
+ through calls to {@link #deleteOnExit(Path)}, and remove this FS instance
+ from the cache, if cached.
+
+ After this operation, the outcome of any method call on this FileSystem
+ instance, or any input/output stream created by it is <i>undefined</i>.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getFileStatus(Path)} instead">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the block size for a particular file.
+ @param f the filename
+ @return the number of bytes in a block
+ @deprecated Use {@link #getFileStatus(Path)} instead
+ @throws FileNotFoundException if the path is not present
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultBlockSize(Path)} instead">
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.
+ @deprecated use {@link #getDefaultBlockSize(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return the number of bytes that large input files should be optimally
+ be split into to minimize I/O time.  The given path will be used to
+ locate the actual filesystem.  The full path does not have to exist.
+ @param f path of file
+ @return the default block size for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getDefaultReplication(Path)} instead">
+      <doc>
+      <![CDATA[Get the default replication.
+ @return the replication; the default value is "1".
+ @deprecated use {@link #getDefaultReplication(Path)} instead]]>
+      </doc>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the default replication for a path.
+ The given path will be used to locate the actual FileSystem to query.
+ The full path does not have to exist.
+ @param path of the file
+ @return default replication for the path's filesystem]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a file status object that represents the path.
+ @param f The path we want information from
+ @return a FileStatus object
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Synchronize client metadata state.
+ <p>
+ In some FileSystem implementations such as HDFS metadata
+ synchronization is essential to guarantee consistency of read requests
+ particularly in HA setting.
+ @throws IOException
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="fixRelativePart" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[See {@link FileContext#fixRelativePart}.]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#createSymlink(Path, Path, boolean)}.]]>
+      </doc>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getFileLinkStatus(Path)}.
+ @throws FileNotFoundException when the path does not exist
+ @throws IOException see specific implementation]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#supportsSymlinks()}.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link FileContext#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link AbstractFileSystem#getLinkTarget(Path)}.
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, if the FS supports checksums.
+
+ @param f The file path
+ @return The file checksum.  The default return value is null,
+  which indicates that no checksum algorithm is implemented
+  in the corresponding FileSystem.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the checksum of a file, from the beginning of the file till the
+ specific length.
+ @param f The file path
+ @param length The length of the file range for checksum calculation
+ @return The file checksum or null if checksums are not supported.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the verify checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param verifyChecksum Verify checksum flag]]>
+      </doc>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+      <doc>
+      <![CDATA[Set the write checksum flag. This is only applicable if the
+ corresponding filesystem supports checksums.
+ By default doesn't do anything.
+ @param writeChecksum Write checksum flag]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the root partition is reflected.
+
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a status object describing the use and capacity of the
+ filesystem. If the filesystem has multiple partitions, the
+ use and capacity of the partition pointed to by the specified
+ path is reflected.
+ @param p Path for which status should be obtained. null means
+ the default partition.
+ @return a FsStatus object
+ @throws IOException
+           see specific implementation]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permission of a path.
+ @param p The path
+ @param permission permission
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set owner of a path (i.e. a file or a directory).
+ The parameters username and groupname cannot both be null.
+ @param p The path
+ @param username If it is null, the original username remains unchanged.
+ @param groupname If it is null, the original groupname remains unchanged.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set access time of a file.
+ @param p The path
+ @param mtime Set the modification time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set modification time.
+ @param atime Set the access time of this file.
+              The number of milliseconds since Jan 1, 1970.
+              A value of -1 means that this call should not set access time.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot with a default name.
+ @param path The directory where snapshots will be taken.
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a snapshot.
+ @param path The directory where snapshots will be taken.
+ @param snapshotName The name of the snapshot
+ @return the snapshot path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported]]>
+      </doc>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rename a snapshot.
+ @param path The directory path where the snapshot was taken
+ @param snapshotOldName Old name of the snapshot
+ @param snapshotNewName New name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a snapshot of a directory.
+ @param path  The directory that the to-be-deleted snapshot belongs to
+ @param snapshotName The name of the snapshot
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Modifies ACL entries of files and directories.  This method can add new ACL
+ entries or modify the permissions on existing ACL entries.  All existing
+ ACL entries that are not specified in this call are retained without
+ changes.  (Modifications are merged into the current ACL.)
+
+ @param path Path to modify
+ @param aclSpec List&lt;AclEntry&gt; describing modifications
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes ACL entries from files and directories.  Other ACL entries are
+ retained.
+
+ @param path Path to modify
+ @param aclSpec List describing entries to remove
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all default ACL entries from files and directories.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Removes all but the base ACL entries of files and directories.  The entries
+ for user, group, and others are retained for compatibility with permission
+ bits.
+
+ @param path Path to modify
+ @throws IOException if an ACL could not be removed
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully replaces ACL of files and directories, discarding all existing
+ entries.
+
+ @param path Path to modify
+ @param aclSpec List describing modifications, which must include entries
+   for user, group, and others for compatibility with permission bits.
+ @throws IOException if an ACL could not be modified
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the ACL of a file or directory.
+
+ @param path Path to get
+ @return AclStatus describing the ACL of the file or directory
+ @throws IOException if an ACL could not be read
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to modify
+ @param name xattr name.
+ @param value xattr value.
+ @param flag xattr set flag
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an xattr name and value for a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attribute
+ @param name xattr name.
+ @return byte[] xattr value.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattrs name/value pairs for a file or directory.
+ Only those xattrs which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @param names XAttr names.
+ @return Map describing the XAttrs of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all of the xattr names for a file or directory.
+ Only those xattr names which the logged-in user has permissions to view
+ are returned.
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to get extended attributes
+ @return List{@literal <String>} of the XAttr names of the file or directory
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove an xattr of a file or directory.
+ The name must be prefixed with the namespace followed by ".". For example,
+ "user.attr".
+ <p>
+ Refer to the HDFS extended attributes user documentation for details.
+
+ @param path Path to remove extended attribute
+ @param name xattr name
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the source path to satisfy storage policy.
+ @param path The source path referring to either a directory or a file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the storage policy for a given file or directory.
+
+ @param src file or directory path.
+ @param policyName the name of the target storage policy. The list
+                   of supported Storage policies can be retrieved
+                   via {@link #getAllStoragePolicies}.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unset the storage policy set for a given file or directory.
+ @param src file or directory path.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Query the effective storage policy ID for the given file or directory.
+
+ @param src file or directory path.
+ @return storage policy for give file.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Retrieve all the storage policies supported by this file system.
+
+ @return all storage policies supported by this filesystem.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+         (default outcome).]]>
+      </doc>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get the root directory of Trash for current user when the path specified
+ is deleted.
+
+ @param path the trash root of the path to be determined.
+ @return the default implementation returns {@code /user/$USER/.Trash}]]>
+      </doc>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+      <doc>
+      <![CDATA[Get all the trash roots for current user or all users.
+
+ @param allUsers return trash roots for all users if true.
+ @return all the trash root directories.
+         Default FileSystem returns .Trash under users' home directories if
+         {@code /user/$USER/.Trash} exists.]]>
+      </doc>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The base FileSystem implementation generally has no knowledge
+ of the capabilities of actual implementations.
+ Unless it has a way to explicitly determine the capabilities,
+ this method returns false.
+ {@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getFileSystemClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the FileSystem implementation class of a filesystem.
+ This triggers a scan and load of all FileSystem implementations listed as
+ services and discovered via the {@link ServiceLoader}
+ @param scheme URL scheme of FS
+ @param conf configuration: can be null, in which case the check for
+ a filesystem binding declaration in the configuration is skipped.
+ @return the filesystem
+ @throws UnsupportedFileSystemException if there was no known implementation
+         for the scheme.
+ @throws IOException if the filesystem could not be loaded]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Get the Map of Statistics object indexed by URI Scheme.
+ @return a Map having a key as URI scheme and value as Statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getAllStatistics" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <doc>
+      <![CDATA[Return the FileSystem classes that have Statistics.
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="getStatistics" return="org.apache.hadoop.fs.FileSystem.Statistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #getGlobalStorageStatistics()}">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the statistics for a particular file system.
+ @param cls the class to lookup
+ @return a statistics object
+ @deprecated use {@link #getGlobalStorageStatistics()}]]>
+      </doc>
+    </method>
+    <method name="clearStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all statistics for all file systems.]]>
+      </doc>
+    </method>
+    <method name="printStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print all statistics for all file systems to {@code System.out}]]>
+      </doc>
+    </method>
+    <method name="areSymlinksEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="enableSymlinks"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStorageStatistics" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the StorageStatistics for this FileSystem object.  These statistics are
+ per-instance.  They are not shared with any other FileSystem object.
+
+ <p>This is a default method which is intended to be overridden by
+ subclasses. The default implementation returns an empty storage statistics
+ object.</p>
+
+ @return    The StorageStatistics for this FileSystem instance.
+            Will never be null.]]>
+      </doc>
+    </method>
+    <method name="getGlobalStorageStatistics" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="createDataOutputStreamBuilder" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileSystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create instance of the standard FSDataOutputStreamBuilder for the
+ given filesystem and path.
+ @param fileSystem owner
+ @param path path to create
+ @return a builder.]]>
+      </doc>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a new FSDataOutputStreamBuilder for the file with path.
+ Files are overwritten by default.
+
+ @param path file path
+ @return a FSDataOutputStreamBuilder object to build the file
+
+ HADOOP-14384. Temporarily reduce the visibility of method before the
+ builder interface becomes stable.]]>
+      </doc>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Create a Builder to append a file.
+ @param path file path.
+ @return a {@link FSDataOutputStreamBuilder} to build file append request.]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(Path, int)} unless a subclass
+ executes the open command differently.
+
+ The semantics of this call are therefore the same as that of
+ {@link #open(Path, int)} with one special point: it is in
+ {@code FSDataInputStreamBuilder.build()} in which the open operation
+ takes place -it is there where all preconditions to the operation
+ are checked.
+ @param path file path
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Open a file for reading through a builder API.
+ Ultimately calls {@link #open(PathHandle, int)} unless a subclass
+ executes the open command differently.
+
+ If PathHandles are unsupported, this may fail in the
+ {@code FSDataInputStreamBuilder.build()}  command,
+ rather than in this {@code openFile()} operation.
+ @param pathHandle path handle.
+ @return a FSDataInputStreamBuilder object to build the input stream
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Execute the actual open file operation.
+
+ This is invoked from {@code FSDataInputStreamBuilder.build()}
+ and from {@link DelegateToFileSystem} and is where
+ the action of opening the file should begin.
+
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)} in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param path path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key]]>
+      </doc>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Execute the actual open file operation.
+ The base implementation performs a blocking
+ call to {@link #open(Path, int)} in this call;
+ the actual outcome is in the returned {@code CompletableFuture}.
+ This avoids having to create some thread pool, while still
+ setting up the expectation that the {@code get()} call
+ is needed to evaluate the result.
+ @param pathHandle path to the file
+ @param parameters open file parameters from the builder.
+ @return a future which will evaluate to the opened file.
+ @throws IOException failure to resolve the link.
+ @throws IllegalArgumentException unknown mandatory key
+ @throws UnsupportedOperationException PathHandles are not supported.
+ This may be deferred until the future is evaluated.]]>
+      </doc>
+    </method>
+    <method name="createMultipartUploader" return="org.apache.hadoop.fs.MultipartUploaderBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a multipart uploader.
+ @param basePath file path under which all files are uploaded
+ @return a MultipartUploaderBuilder object to build the uploader
+ @throws IOException if some early checks cause IO failures.
+ @throws UnsupportedOperationException if support is checked early.]]>
+      </doc>
+    </method>
+    <field name="FS_DEFAULT_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG" type="org.apache.commons.logging.Log"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This log is widely used in the org.apache.hadoop.fs code and tests,
+ so must be considered something to only be changed with care.]]>
+      </doc>
+    </field>
+    <field name="SHUTDOWN_HOOK_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the FileSystem shutdown hook: {@value}.]]>
+      </doc>
+    </field>
+    <field name="TRASH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for trash directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="USER_HOME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="statistics" type="org.apache.hadoop.fs.FileSystem.Statistics"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The statistics for this file system.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An abstract base class for a fairly generic filesystem.  It
+ may be implemented as a distributed filesystem, or as a "local"
+ one that reflects the locally-connected disk.  The local version
+ exists for small Hadoop instances and for testing.
+
+ <p>
+
+ All user code that may potentially use the Hadoop Distributed
+ File System should be written to use a FileSystem object or its
+ successor, {@link FileContext}.
+
+ <p>
+ The local implementation is {@link LocalFileSystem} and distributed
+ implementation is DistributedFileSystem. There are other implementations
+ for object stores and (outside the Apache Hadoop codebase),
+ third party filesystems.
+ <p>
+ Notes
+ <ol>
+ <li>The behaviour of the filesystem is
+ <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/filesystem/filesystem.html">
+ specified in the Hadoop documentation. </a>
+ However, the normative specification of the behavior of this class is
+ actually HDFS: if HDFS does not behave the way these Javadocs or
+ the specification in the Hadoop documentations define, assume that
+ the documentation is incorrect.
+ </li>
+ <li>The term {@code FileSystem} refers to an instance of this class.</li>
+ <li>The acronym "FS" is used as an abbreviation of FileSystem.</li>
+ <li>The term {@code filesystem} refers to the distributed/local filesystem
+ itself, rather than the class used to interact with it.</li>
+ <li>The term "file" refers to a file in the remote filesystem,
+ rather than instances of {@code java.io.File}.</li>
+ </ol>
+
+ This is a carefully evolving class.
+ New methods may be marked as Unstable or Evolving for their initial release,
+ as a warning that they are new and may change based on the
+ experience of use in applications.
+ <p></p>
+ <b>Important note for developers</b>
+ <p></p>
+ If you are making changes here to the public API or protected methods,
+ you must review the following subclasses and make sure that
+ they are filtering/passing through new methods as appropriate.
+ <p></p>
+
+ {@link FilterFileSystem}: methods are passed through. If not,
+ then {@code TestFilterFileSystem.MustNotImplement} must be
+ updated with the unsupported interface.
+ Furthermore, if the new API's support is probed for via
+ {@link #hasPathCapability(Path, String)} then
+ {@link FilterFileSystem#hasPathCapability(Path, String)}
+ must return false, always.
+ <p></p>
+ {@link ChecksumFileSystem}: checksums are created and
+ verified.
+ <p></p>
+ {@code TestHarFileSystem} will need its {@code MustNotImplement}
+ interface updated.
+ <p></p>
+
+ There are some external places your changes will break things.
+ Do co-ordinate changes here.
+ <p></p>
+
+ HBase: HBoss
+ <p></p>
+ Hive: HiveShim23
+ {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileSystem -->
+  <!-- start class org.apache.hadoop.fs.FileUtil -->
+  <class name="FileUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileUtil"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path
+
+ @param stats
+          an array of FileStatus objects
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="stat2Paths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.fs.FileStatus[]"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[convert an array of FileStatus to an array of Path.
+ If stats if null, return path
+ @param stats
+          an array of FileStatus objects
+ @param path
+          default path to return in stats is null
+ @return an array of paths corresponding to the input]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteOnExit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <doc>
+      <![CDATA[Register all files recursively to be deleted on exit.
+ @param file File/directory to be deleted]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete a directory and all its contents.  If
+ we return false, the directory may be partially-deleted.
+ (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+     to by the symlink is not deleted.
+ (2) If dir is symlink to a directory, symlink is deleted. The directory
+     pointed to by symlink is not deleted.
+ (3) If dir is a normal file, it is deleted.
+ (4) If dir is a normal directory, then dir and all its contents recursively
+     are deleted.
+ @param dir the file or directory to be deleted
+ @param tryGrantPermissions true if permissions should be modified to delete a file.
+ @return true on success false on failure.]]>
+      </doc>
+    </method>
+    <method name="readLink" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns the target of the given symlink. Returns the empty string if
+ the given path does not refer to a symlink or there is an error
+ accessing the symlink.
+ @param f File representing the symbolic link.
+ @return The target of the symbolic link, empty string on error or if not
+         a symlink.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.]]>
+      </doc>
+    </method>
+    <method name="fullyDeleteContents" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="tryGrantPermissions" type="boolean"/>
+      <doc>
+      <![CDATA[Delete the contents of a directory, not the directory itself.  If
+ we return false, the directory may be partially-deleted.
+ If dir is a symlink to a directory, all the contents of the actual
+ directory pointed to by dir will be deleted.
+ @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ and all the underlying directories before trying to delete their contents.]]>
+      </doc>
+    </method>
+    <method name="fullyDelete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link FileSystem#delete(Path, boolean)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recursively delete a directory.
+
+ @param fs {@link FileSystem} on which the path is present
+ @param dir directory to recursively delete
+ @throws IOException
+ @deprecated Use {@link FileSystem#delete(Path, boolean)}]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="srcStatus" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy files between FileSystems.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="dstFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy local files to a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="copy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFS" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="java.io.File"/>
+      <param name="deleteSource" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy FileSystem files to local files.]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param filename The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeSecureShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell
+ and avoids script injection attacks.
+ @param file The filename to convert
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="makeShellPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="makeCanonicalPath" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert a os-native filename to a path that works for the shell.
+ @param file The filename to convert
+ @param makeCanonicalPath
+          Whether to make canonical path for the file passed
+ @return The unix pathname
+ @throws IOException on windows, there can be problems with the subprocess]]>
+      </doc>
+    </method>
+    <method name="getDU" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Takes an input dir and returns the du on that local directory. Very basic
+ implementation.
+
+ @param dir
+          The input dir to get the disk space of this local dir
+ @return The total disk space of the input local directory]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="toDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a stream input it will unzip the it in the unzip directory.
+ passed as the second parameter
+ @param inputStream The zip file as input
+ @param toDir The unzip directory where to unzip the zip file.
+ @throws IOException an exception occurred]]>
+      </doc>
+    </method>
+    <method name="unZip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="unzipDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a File input it will unzip it in the unzip directory.
+ passed as the second parameter
+ @param inFile The zip file as input
+ @param unzipDir The unzip directory where to unzip the zip file.
+ @throws IOException An I/O exception has occurred]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="untarDir" type="java.io.File"/>
+      <param name="gzipped" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ExecutionException" type="java.util.concurrent.ExecutionException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inputStream The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @param gzipped The input stream is gzipped
+                TODO Use magic number and PusbackInputStream to identify
+ @throws IOException an exception occurred
+ @throws InterruptedException command interrupted
+ @throws ExecutionException task submit failed]]>
+      </doc>
+    </method>
+    <method name="unTar"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inFile" type="java.io.File"/>
+      <param name="untarDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Given a Tar File as input it will untar the file in a the untar directory
+ passed as the second parameter
+
+ This utility will untar ".tar" files and ".tar.gz","tgz" files.
+
+ @param inFile The tar file as input.
+ @param untarDir The untar directory where to untar the tar file.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="symLink" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="linkname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a soft link between a src and destination
+ only on a local disk. HDFS does not support this.
+ On Windows, when symlink creation fails due to security
+ setting, we will log a warning. The return code in this
+ case is 2.
+
+ @param target the target for symlink
+ @param linkname the symlink
+ @return 0 on success]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Change the permissions on a filename.
+ @param filename the name of the file to change
+ @param perm the permission string
+ @return the exit code from the command
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="chmod" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Change the permissions on a file / directory, recursively, if
+ needed.
+ @param filename name of the file whose permissions are to change
+ @param perm permission string
+ @param recursive true, if permissions should be changed recursively
+ @return the exit code from the command.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.io.File"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the ownership on a file / directory. User name and group name
+ cannot both be null.
+ @param file the file to change
+ @param username the new user owner name
+ @param groupname the new group owner name
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setReadable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="readable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setReadable(boolean)}
+ File#setReadable does not work as expected on Windows.
+ @param f input file
+ @param readable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setWritable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="writable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setWritable(boolean)}
+ File#setWritable does not work as expected on Windows.
+ @param f input file
+ @param writable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setExecutable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="executable" type="boolean"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#setExecutable(boolean)}
+ File#setExecutable does not work as expected on Windows.
+ Note: revoking execute permission on folders does not have the same
+ behavior on Windows as on Unix platforms. Creating, deleting or renaming
+ a file within that folder will still succeed on Windows.
+ @param f input file
+ @param executable
+ @return true on success, false otherwise]]>
+      </doc>
+    </method>
+    <method name="canRead" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canRead()}
+ @param f input file
+ @return On Unix, same as {@link File#canRead()}
+         On Windows, true if process has read access on the path]]>
+      </doc>
+    </method>
+    <method name="canWrite" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canWrite()}
+ @param f input file
+ @return On Unix, same as {@link File#canWrite()}
+         On Windows, true if process has write access on the path]]>
+      </doc>
+    </method>
+    <method name="canExecute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <doc>
+      <![CDATA[Platform independent implementation for {@link File#canExecute()}
+ @param f input file
+ @return On Unix, same as {@link File#canExecute()}
+         On Windows, true if process has execute access on the path]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set permissions to the required value. Uses the java primitives instead
+ of forking if group == other.
+ @param f the file to change
+ @param permission the new permissions
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createLocalTempFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="basefile" type="java.io.File"/>
+      <param name="prefix" type="java.lang.String"/>
+      <param name="isDeleteOnExit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a tmp file for a base file.
+ @param basefile the base file of the tmp
+ @param prefix file name prefix of tmp
+ @param isDeleteOnExit if true, the tmp will be deleted when the VM exits
+ @return a newly created tmp file
+ @exception IOException If a tmp file cannot created
+ @see java.io.File#createTempFile(String, String, File)
+ @see java.io.File#deleteOnExit()]]>
+      </doc>
+    </method>
+    <method name="replaceFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.io.File"/>
+      <param name="target" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the src file to the name specified by target.
+ @param src the source file
+ @param target the target file
+ @exception IOException If this operation fails]]>
+      </doc>
+    </method>
+    <method name="listFiles" return="java.io.File[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#listFiles()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#listFiles() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of files or empty list
+ @exception IOException for invalid directory or for a bad disk.]]>
+      </doc>
+    </method>
+    <method name="list" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A wrapper for {@link File#list()}. This java.io API returns null
+ when a dir is not a directory or for any I/O error. Instead of having
+ null check everywhere File#list() is used, we will add utility API
+ to get around this problem. For the majority of cases where we prefer
+ an IOException to be thrown.
+ @param dir directory for which listing should be performed
+ @return list of file names or empty string list
+ @exception AccessDeniedException for unreadable directory
+ @exception IOException for invalid directory or for bad disk]]>
+      </doc>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createJarWithClassPath" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClassPath" type="java.lang.String"/>
+      <param name="pwd" type="org.apache.hadoop.fs.Path"/>
+      <param name="targetDir" type="org.apache.hadoop.fs.Path"/>
+      <param name="callerEnv" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a jar file at the given path, containing a manifest with a classpath
+ that references all specified entries.
+
+ Some platforms may have an upper limit on command line length.  For example,
+ the maximum command line length on Windows is 8191 characters, but the
+ length of the classpath may exceed this.  To work around this limitation,
+ use this method to create a small intermediate jar with a manifest that
+ contains the full classpath.  It returns the absolute path to the new jar,
+ which the caller may set as the classpath for a new process.
+
+ Environment variable evaluation is not supported within a jar manifest, so
+ this method expands environment variables before inserting classpath entries
+ to the manifest.  The method parses environment variables according to
+ platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
+ environment variables are case-insensitive.  For example, %VAR% and %var%
+ evaluate to the same value.
+
+ Specifying the classpath in a jar manifest does not support wildcards, so
+ this method expands wildcards internally.  Any classpath entry that ends
+ with * is translated to all files at that path with extension .jar or .JAR.
+
+ @param inputClassPath String input classpath to bundle into the jar manifest
+ @param pwd Path to working directory to save jar
+ @param targetDir path to where the jar execution will have its working dir
+ @param callerEnv Map {@literal <}String, String{@literal >} caller's
+ environment variables to use for expansion
+ @return String[] with absolute path to new jar in position 0 and
+   unexpanded wild card entry path in position 1
+ @throws IOException if there is an I/O error while writing the jar file]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+ It operates only on local paths.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist locally]]>
+      </doc>
+    </method>
+    <method name="getJarsInDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="useLocal" type="boolean"/>
+      <doc>
+      <![CDATA[Returns all jars that are in the directory. It is useful in expanding a
+ wildcard path to return all jars from the directory to use in a classpath.
+
+ @param path the path to the directory. The path may include the wildcard.
+ @return the list of jars as URLs, or an empty list if there are no jars, or
+ the directory does not exist]]>
+      </doc>
+    </method>
+    <method name="compareFs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="srcFs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="destFs" type="org.apache.hadoop.fs.FileSystem"/>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes bytes to a file. This utility method opens the file for writing,
+ creating the file if it does not exist, or overwrites an existing file. All
+ bytes in the byte array are written to the file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param bytes the byte array with the bytes to write
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="lines" type="java.lang.Iterable"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write lines of text to a file. Each line is a char sequence and is written
+ to the file in sequence with each line terminated by the platform's line
+ separator, as defined by the system property {@code
+ line.separator}. Characters are encoded into bytes using the specified
+ charset. This utility method opens the file for writing, creating the file
+ if it does not exist, or overwrites an existing file.
+
+ @param fileContext the file context with which to create the file
+ @param path the path to the file
+ @param lines a Collection to iterate over the char sequences
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param fs the file system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <param name="cs" type="java.nio.charset.Charset"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using the
+ specified charset. This utility method opens the file for writing, creating
+ the file if it does not exist, or overwrites an existing file.
+
+ @param FileContext the file context with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+ @param cs the charset to use for encoding
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fs the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file system
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <method name="write" return="org.apache.hadoop.fs.FileContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileContext" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="charseq" type="java.lang.CharSequence"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a line of text to a file. Characters are encoded into bytes using
+ UTF-8. This utility method opens the file for writing, creating the file if
+ it does not exist, or overwrites an existing file.
+
+ @param fileContext the files system with which to create the file
+ @param path the path to the file
+ @param charseq the char sequence to write to the file
+
+ @return the file context
+
+ @throws NullPointerException if any of the arguments are {@code null}
+ @throws IOException if an I/O error occurs creating or writing to the file]]>
+      </doc>
+    </method>
+    <field name="SYMLINK_NO_PRIVILEGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A collection of file-processing util methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FileUtil -->
+  <!-- start class org.apache.hadoop.fs.FilterFileSystem -->
+  <class name="FilterFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRawFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the raw file system 
+ @return FileSystem being filtered]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called after a new FileSystem instance is constructed.
+ @param name a uri whose authority section names the host, port, etc.
+   for this FileSystem
+ @param conf the configuration]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a URI whose scheme and authority identify this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getCanonicalUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="canonicalizeUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Make sure that a path specifies a FileSystem.]]>
+      </doc>
+    </method>
+    <method name="checkPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Check that a Path belongs to this FileSystem.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="start" type="long"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolvePath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Opens an FSDataInputStream at the indicated Path.
+ @param f the file name to open
+ @param bufferSize the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="filter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReplication" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="replication" type="short"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set replication for an existing file.
+ 
+ @param src file name
+ @param replication new replication
+ @throws IOException
+ @return true if successful;
+         false if file does not exist or is a directory]]>
+      </doc>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames Path src to Path dst.  Can take place on local fs
+ or remote DFS.]]>
+      </doc>
+    </method>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="options" type="org.apache.hadoop.fs.Options.Rename[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete a file]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files in a directory.]]>
+      </doc>
+    </method>
+    <method name="listCorruptFileBlocks" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listLocatedStatus" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List files and its block locations in a directory.]]>
+      </doc>
+    </method>
+    <method name="listStatusIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a remote iterator for listing in a directory]]>
+      </doc>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the given file system. All relative
+ paths will be resolved relative to it.
+ 
+ @param newDir]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the given file system
+ 
+ @return the directory pathname]]>
+      </doc>
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="srcs" type="org.apache.hadoop.fs.Path[]"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src files are on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is on the local disk.  Add it to FS at
+ the given dst name.
+ delSrc indicates if the source should be removed]]>
+      </doc>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The src file is under FS, and the dst is on the local disk.
+ Copy it from FS control to the local dst name.
+ delSrc indicates if the src will be removed or not.]]>
+      </doc>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a local File that the user can write output to.  The caller
+ provides both the eventual FS target name and the local working
+ file.  If the FS is local, we write directly into the target.  If
+ the FS is remote, we write into the tmp local area.]]>
+      </doc>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Called when we're all done writing to the target.  A local FS will
+ do nothing, because we've written to exactly the right place.  A remote
+ FS will copy the contents of tmpLocalFile to the correct target at
+ fsOutputFile.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files in the filesystem.]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the total size of all files from a specified path.]]>
+      </doc>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getDefaultReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getServerDefaults" return="org.apache.hadoop.fs.FsServerDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get file status.]]>
+      </doc>
+    </method>
+    <method name="msync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="access"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="mode" type="org.apache.hadoop.fs.permission.FsAction"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.fs.FileAlreadyExistsException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="ParentNotDirectoryException" type="org.apache.hadoop.fs.ParentNotDirectoryException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="UnsupportedFileSystemException" type="org.apache.hadoop.fs.UnsupportedFileSystemException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resolveLink" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileChecksum" return="org.apache.hadoop.fs.FileChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="length" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setVerifyChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verifyChecksum" type="boolean"/>
+    </method>
+    <method name="setWriteChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeChecksum" type="boolean"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveCreate" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="absolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="checksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="primitiveMkdir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="abdolutePermission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getChildFileSystems" return="org.apache.hadoop.fs.FileSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSnapshot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="renameSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotOldName" type="java.lang.String"/>
+      <param name="snapshotNewName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="snapshotName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="modifyAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAclEntries"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeDefaultAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAcl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="aclSpec" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAclStatus" return="org.apache.hadoop.fs.permission.AclStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="byte[]"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttr" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getXAttrs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="names" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listXAttrs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="removeXAttr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="satisfyStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="policyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="unsetStoragePolicy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStoragePolicy" return="org.apache.hadoop.fs.BlockStoragePolicySpi"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getAllStoragePolicies" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTrashRoot" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getTrashRoots" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allUsers" type="boolean"/>
+    </method>
+    <method name="createFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="appendFile" return="org.apache.hadoop.fs.FSDataOutputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="openFile" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="openFileWithOptions" return="java.util.concurrent.CompletableFuture"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pathHandle" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="parameters" type="org.apache.hadoop.fs.impl.OpenFileParameters"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="swapScheme" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A <code>FilterFileSystem</code> contains
+ some other file system, which it uses as
+ its  basic file system, possibly transforming
+ the data along the way or providing  additional
+ functionality. The class <code>FilterFileSystem</code>
+ itself simply overrides all  methods of
+ <code>FileSystem</code> with versions that
+ pass all requests to the contained  file
+ system. Subclasses of <code>FilterFileSystem</code>
+ may further override some of  these methods
+ and may also provide additional methods
+ and fields.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FilterFileSystem -->
+  <!-- start interface org.apache.hadoop.fs.FSBuilder -->
+  <interface name="FSBuilder"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set optional Builder parameter.]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set optional boolean parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set optional int parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set optional float parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set optional double parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="opt" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set an array of string values as optional parameter for the Builder.
+
+ @see #opt(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set mandatory option to the Builder.
+
+ If the option is not supported or unavailable,
+ the client should expect {@link #build()} throws IllegalArgumentException.]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set mandatory boolean option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set mandatory int option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set mandatory float option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Set mandatory double option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="must" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="values" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Set a string array as mandatory option.
+
+ @see #must(String, String)]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instantiate the object which was being built.
+
+ @throws IllegalArgumentException if the parameters are not valid.
+ @throws UnsupportedOperationException if the filesystem does not support
+ the specific operation.
+ @throws IOException on filesystem IO errors.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The base interface which various FileSystem FileContext Builder
+ interfaces can extend, and which underlying implementations
+ will then implement.
+ @param <S> Return type on the {@link #build()} call.
+ @param <B> type of builder itself.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FSBuilder -->
+  <!-- start interface org.apache.hadoop.fs.FsConstants -->
+  <interface name="FsConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="LOCAL_FS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FTP_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_PATH_LINKS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_URI" type="java.net.URI"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ViewFs: viewFs file system (ie the mount file system on client side)]]>
+      </doc>
+    </field>
+    <field name="VIEWFS_SCHEME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VIEWFS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FileSystem related constants.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FsConstants -->
+  <!-- start class org.apache.hadoop.fs.FSDataInputStream -->
+  <class name="FSDataInputStream" extends="java.io.DataInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+    <implements name="org.apache.hadoop.fs.HasFileDescriptor"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.CanSetReadahead"/>
+    <implements name="org.apache.hadoop.fs.HasEnhancedByteBufferAccess"/>
+    <implements name="org.apache.hadoop.fs.CanUnbuffer"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <implements name="org.apache.hadoop.fs.ByteBufferPositionedReadable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="FSDataInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="desired" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset.
+
+ @param desired offset to seek to]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current position in the input stream.
+
+ @return current position in the input stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    maximum number of bytes to read
+ @return total number of bytes read into the buffer, or <code>-1</code>
+         if there is no more data because the end of the stream has been
+         reached]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the given position in the stream to the given buffer.
+ Continues to read until <code>length</code> bytes have been read.
+
+ @param position  position in the input stream to seek
+ @param buffer    buffer into which data is read
+ @param offset    offset into the buffer in which data is written
+ @param length    the number of bytes to read
+ @throws IOException IO problems
+ @throws EOFException If the end of stream is reached while reading.
+                      If an exception is thrown an undetermined number
+                      of bytes in the buffer may have been written.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[See {@link #readFully(long, byte[], int, int)}.]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given position on an alternate copy of the data.
+
+ @param  targetPos  position to seek to
+ @return true if a new source is found, false otherwise]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileDescriptor" return="java.io.FileDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setReadahead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readahead" type="java.lang.Long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <param name="opts" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="read" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufferPool" type="org.apache.hadoop.io.ByteBufferPool"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+    </method>
+    <method name="releaseBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String value. Includes the string value of the inner stream
+ @return the stream]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the IO Statistics of the nested stream, falling back to
+ null if the stream does not implement the interface
+ {@link IOStatisticsSource}.
+ @return an IOStatistics instance or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link FSInputStream} in a {@link DataInputStream}
+ and buffers input through a {@link java.io.BufferedInputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataInputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStream -->
+  <class name="FSDataOutputStream" extends="java.io.DataOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Syncable"/>
+    <implements name="org.apache.hadoop.fs.CanSetDropBehind"/>
+    <implements name="org.apache.hadoop.fs.StreamCapabilities"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <implements name="org.apache.hadoop.fs.Abortable"/>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FSDataOutputStream" type="java.io.OutputStream, org.apache.hadoop.fs.FileSystem.Statistics, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current position in the output stream.
+
+ @return the current position in the output stream]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the underlying output stream.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+    </method>
+    <method name="hflush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setDropBehind"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dropBehind" type="java.lang.Boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the IO Statistics of the nested stream, falling back to
+ empty statistics if the stream does not implement the interface
+ {@link IOStatisticsSource}.
+ @return an IOStatistics instance.]]>
+      </doc>
+    </method>
+    <method name="abort" return="org.apache.hadoop.fs.Abortable.AbortableResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code abort()} on the wrapped stream if it
+ is Abortable, otherwise raise an
+ {@code UnsupportedOperationException}.
+ @throws UnsupportedOperationException if not available.
+ @return the result.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStream -->
+  <!-- start class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <class name="FSDataOutputStreamBuilder" extends="org.apache.hadoop.fs.impl.AbstractFSBuilderImpl"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FSDataOutputStreamBuilder" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="getThisBuilder" return="B"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the concrete implementation of the builder instance.]]>
+      </doc>
+    </method>
+    <method name="getFS" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPermission" return="org.apache.hadoop.fs.permission.FsPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="permission" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <doc>
+      <![CDATA[Set permission for the file.]]>
+      </doc>
+    </method>
+    <method name="getBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="bufferSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bufSize" type="int"/>
+      <doc>
+      <![CDATA[Set the size of the buffer to be used.]]>
+      </doc>
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="replication" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="replica" type="short"/>
+      <doc>
+      <![CDATA[Set replication factor.]]>
+      </doc>
+    </method>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="blockSize" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blkSize" type="long"/>
+      <doc>
+      <![CDATA[Set block size.]]>
+      </doc>
+    </method>
+    <method name="isRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true to create the parent directories if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="recursive" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the parent directory if they do not exist.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="org.apache.hadoop.util.Progressable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="progress" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prog" type="org.apache.hadoop.util.Progressable"/>
+      <doc>
+      <![CDATA[Set the facility of reporting progress.]]>
+      </doc>
+    </method>
+    <method name="getFlags" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="create" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an FSDataOutputStream at the specified path.]]>
+      </doc>
+    </method>
+    <method name="overwrite" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="overwrite" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to overwrite the existing file.
+ Set it to false, an exception will be thrown when calling {@link #build()}
+ if the file exists.]]>
+      </doc>
+    </method>
+    <method name="append" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Append to an existing file (optional operation).]]>
+      </doc>
+    </method>
+    <method name="getChecksumOpt" return="org.apache.hadoop.fs.Options.ChecksumOpt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="checksumOpt" return="B"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="chksumOpt" type="org.apache.hadoop.fs.Options.ChecksumOpt"/>
+      <doc>
+      <![CDATA[Set checksum opt.]]>
+      </doc>
+    </method>
+    <method name="build" return="S"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the FSDataOutputStream to write on the file system.
+
+ @throws IllegalArgumentException if the parameters are not valid.
+ @throws IOException on errors when file system creates or appends the file.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for {@link FSDataOutputStream} and its subclasses.
+
+ It is used to create {@link FSDataOutputStream} when creating a new file or
+ appending an existing file on {@link FileSystem}.
+
+ By default, it does not create parent directory that do not exist.
+ {@link FileSystem#createNonRecursive(Path, boolean, int, short, long,
+ Progressable)}.
+
+ To create missing parent directory, use {@link #recursive()}.
+
+ To be more generic, {@link #opt(String, int)} and {@link #must(String, int)}
+ variants provide implementation-agnostic way to customize the builder.
+ Each FS-specific builder implementation can interpret the FS-specific
+ options accordingly, for example:
+
+ <code>
+
+ // Don't
+ if (fs instanceof FooFileSystem) {
+   FooFileSystem fs = (FooFileSystem) fs;
+   OutputStream out = dfs.createFile(path)
+     .optionA()
+     .optionB("value")
+     .cache()
+   .build()
+ } else if (fs instanceof BarFileSystem) {
+   ...
+ }
+
+ // Do
+ OutputStream out = fs.createFile(path)
+   .permission(perm)
+   .bufferSize(bufSize)
+   .opt("foofs:option.a", true)
+   .opt("foofs:option.b", "value")
+   .opt("barfs:cache", true)
+   .must("foofs:cache", true)
+   .must("barfs:cache-size", 256 * 1024 * 1024)
+   .build();
+ </code>
+
+ If the option is not related to the file system, the option will be ignored.
+ If the option is must, but not supported by the file system, a
+ {@link IllegalArgumentException} will be thrown.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSDataOutputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.FSError -->
+  <class name="FSError" extends="java.lang.Error"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Thrown for unexpected filesystem errors, presumed to reflect disk errors
+ in the native filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSError -->
+  <!-- start class org.apache.hadoop.fs.FSInputStream -->
+  <class name="FSInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.PositionedReadable"/>
+    <constructor name="FSInputStream"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seeks a different copy of the data.  Returns true if 
+ found a new source, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="validatePositionedReadArgs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="EOFException" type="java.io.EOFException"/>
+      <doc>
+      <![CDATA[Validation code, available for use in subclasses.
+ @param position position: if negative an EOF exception is raised
+ @param buffer destination buffer
+ @param offset offset within the buffer
+ @param length length of bytes to read
+ @throws EOFException if the position is negative
+ @throws IndexOutOfBoundsException if there isn't space for the amount of
+ data requested.
+ @throws IllegalArgumentException other arguments are invalid.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[toString method returns the superclass toString, but if the subclass
+ implements {@link IOStatisticsSource} then those statistics are
+ extracted and included in the output.
+ That is: statistics of subclasses are automatically reported.
+ @return a string value.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[FSInputStream is a generic old InputStream with a little bit
+ of RAF-style seek ability.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FSInputStream -->
+  <!-- start class org.apache.hadoop.fs.FsServerDefaults -->
+  <class name="FsServerDefaults" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsServerDefaults"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FsServerDefaults" type="long, int, int, short, int, boolean, long, org.apache.hadoop.util.DataChecksum.Type, java.lang.String, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlockSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getBytesPerChecksum" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWritePacketSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReplication" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileBufferSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEncryptDataTransfer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrashInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getChecksumType" return="org.apache.hadoop.util.DataChecksum.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyProviderUri" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultStoragePolicyId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Provides server default configuration values to clients.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsServerDefaults -->
+  <!-- start class org.apache.hadoop.fs.FsStatus -->
+  <class name="FsStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FsStatus" type="long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a FsStatus object, using the specified statistics]]>
+      </doc>
+    </constructor>
+    <method name="getCapacity" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the capacity in bytes of the file system]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of bytes used on the file system]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the number of remaining bytes on the file system]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent the capacity, free and used space on a
+ {@link FileSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.FsStatus -->
+  <!-- start interface org.apache.hadoop.fs.FutureDataInputStreamBuilder -->
+  <interface name="FutureDataInputStreamBuilder"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.FSBuilder"/>
+    <method name="build" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="withFileStatus" return="org.apache.hadoop.fs.FutureDataInputStreamBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[A FileStatus may be provided to the open request.
+ It is up to the implementation whether to use this or not.
+ @param status status.
+ @return the builder.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Builder for input streams and subclasses whose return value is
+ actually a completable future: this allows for better asynchronous
+ operation.
+
+ To be more generic, {@link #opt(String, int)} and {@link #must(String, int)}
+ variants provide implementation-agnostic way to customize the builder.
+ Each FS-specific builder implementation can interpret the FS-specific
+ options accordingly, for example:
+
+ If the option is not related to the file system, the option will be ignored.
+ If the option is must, but not supported by the file system, a
+ {@link IllegalArgumentException} will be thrown.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.FutureDataInputStreamBuilder -->
+  <!-- start class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <class name="GlobalStorageStatistics" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.GlobalStorageStatistics[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.GlobalStorageStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @return            The StorageStatistics object with the given name, or
+                      null if there is none.]]>
+      </doc>
+    </method>
+    <method name="put" return="org.apache.hadoop.fs.StorageStatistics"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="provider" type="org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider"/>
+      <doc>
+      <![CDATA[Create or return the StorageStatistics object with the given name.
+
+ @param name        The storage statistics object name.
+ @param provider    An object which can create a new StorageStatistics
+                      object if needed.
+ @return            The StorageStatistics object with the given name.
+ @throws RuntimeException  If the StorageStatisticsProvider provides a null
+                           object or a new StorageStatistics object with the
+                           wrong name.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all global storage statistics.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator that we can use to iterate throw all the global storage
+ statistics objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stores global storage statistics objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobalStorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.GlobFilter -->
+  <class name="GlobFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="GlobFilter" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern.
+
+ @param filePattern the file pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <constructor name="GlobFilter" type="java.lang.String, org.apache.hadoop.fs.PathFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a glob filter with the specified file pattern and an user filter.
+
+ @param filePattern the file pattern.
+ @param filter user filter in addition to the glob pattern.
+ @throws IOException thrown if the file pattern is incorrect.]]>
+      </doc>
+    </constructor>
+    <method name="hasPattern" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[A filter for POSIX glob pattern with brace expansions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.GlobFilter -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathException -->
+  <class name="InvalidPathException" extends="org.apache.hadoop.HadoopIllegalArgumentException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.]]>
+      </doc>
+    </constructor>
+    <constructor name="InvalidPathException" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+ 
+ @param path invalid path.
+ @param reason Reason <code>path</code> is invalid]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Path string is invalid either because it has invalid characters or due to
+ other file system specific reasons.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathException -->
+  <!-- start class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <class name="InvalidPathHandleException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidPathHandleException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidPathHandleException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown when the constraints enoded in a {@link PathHandle} do not hold.
+ For example, if a handle were created with the default
+ {@link Options.HandleOpt#path()} constraints, a call to
+ {@link FileSystem#open(PathHandle)} would succeed if the file were
+ modified, but if a different file was at that location then it would throw
+ this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.InvalidPathHandleException -->
+  <!-- start class org.apache.hadoop.fs.LocalFileSystem -->
+  <class name="LocalFileSystem" extends="org.apache.hadoop.fs.ChecksumFileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocalFileSystem" type="org.apache.hadoop.fs.FileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p>
+
+ @return <code>file</code>]]>
+      </doc>
+    </method>
+    <method name="getRaw" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="copyFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="copyToLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delSrc" type="boolean"/>
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reportChecksumFailure" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="in" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="inPos" type="long"/>
+      <param name="sums" type="org.apache.hadoop.fs.FSDataInputStream"/>
+      <param name="sumsPos" type="long"/>
+      <doc>
+      <![CDATA[Moves files to a bad file directory on the same device, so that their
+ storage will not be reused.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the checksumed local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.LocatedFileStatus -->
+  <class name="LocatedFileStatus" extends="org.apache.hadoop.fs.FileStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocatedFileStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LocatedFileStatus" type="org.apache.hadoop.fs.FileStatus, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor 
+ @param stat a file status
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ 
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, boolean, boolean, boolean, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param hasAcl entity has associated ACLs
+ @param isEncrypted entity is encrypted
+ @param isErasureCoded entity is erasure coded
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <constructor name="LocatedFileStatus" type="long, boolean, int, long, long, long, org.apache.hadoop.fs.permission.FsPermission, java.lang.String, java.lang.String, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path, java.util.Set, org.apache.hadoop.fs.BlockLocation[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param length a file's length
+ @param isdir if the path is a directory
+ @param block_replication the file's replication factor
+ @param blocksize a file's block size
+ @param modification_time a file's modification time
+ @param access_time a file's access time
+ @param permission a file's permission
+ @param owner a file's owner
+ @param group a file's group
+ @param symlink symlink if the path is a symbolic link
+ @param path the path's qualified name
+ @param attr Attribute flags (See {@link FileStatus.AttrFlags}).
+ @param locations a file's block locations]]>
+      </doc>
+    </constructor>
+    <method name="getBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file's block locations
+
+ In HDFS, the returned BlockLocation will have different formats for
+ replicated and erasure coded file.
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
+ for more details.
+
+ @return the file's block locations]]>
+      </doc>
+    </method>
+    <method name="setBlockLocations"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="locations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <doc>
+      <![CDATA[Hook for subclasses to lazily set block locations. The {@link #locations}
+ field should be null before this is called.
+ @param locations Block locations for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Compare this FileStatus to another FileStatus
+ @param   o the FileStatus to be compared.
+ @return  a negative integer, zero, or a positive integer as this object
+   is less than, equal to, or greater than the specified object.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare if this object is equal to another object
+ @param   o the object to be compared.
+ @return  true if two file status has the same path name; false if not.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for the object, which is defined as
+ the hash code of the path name.
+
+ @return  a hash code value for the path name.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class defines a FileStatus that includes a file's block locations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.LocatedFileStatus -->
+  <!-- start interface org.apache.hadoop.fs.MultipartUploader -->
+  <interface name="MultipartUploader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <method name="startUpload" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize a multipart upload.
+ @param filePath Target path for upload.
+ @return unique identifier associating part uploads.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="putPart" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="partNumber" type="int"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputStream" type="java.io.InputStream"/>
+      <param name="lengthInBytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Put part as part of a multipart upload.
+ It is possible to have parts uploaded in any order (or in parallel).
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param partNumber Index of the part relative to others.
+ @param filePath Target path for upload (as {@link #startUpload(Path)}).
+ @param inputStream Data for this part. Implementations MUST close this
+ stream after reading in the data.
+ @param lengthInBytes Target length to read from the stream.
+ @return unique PartHandle identifier for the uploaded part.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="complete" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <param name="handles" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Complete a multipart upload.
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param filePath Target path for upload (as {@link #startUpload(Path)}.
+ @param handles non-empty map of part number to part handle.
+          from {@link #putPart(UploadHandle, int, Path, InputStream, long)}.
+ @return unique PathHandle identifier for the uploaded file.
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <method name="abort" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uploadId" type="org.apache.hadoop.fs.UploadHandle"/>
+      <param name="filePath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Aborts a multipart upload.
+ @param uploadId Identifier from {@link #startUpload(Path)}.
+ @param filePath Target path for upload (same as {@link #startUpload(Path)}.
+ @throws IOException IO failure
+ @return a future; the operation will have completed]]>
+      </doc>
+    </method>
+    <method name="abortUploadsUnderPath" return="java.util.concurrent.CompletableFuture"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Best effort attempt to aborts multipart uploads under a path.
+ Not all implementations support this, and those which do may
+ be vulnerable to eventually consistent listings of current uploads
+ -some may be missed.
+ @param path path to abort uploads under.
+ @return a future to the number of entries aborted;
+ -1 if aborting is unsupported
+ @throws IOException IO failure]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[MultipartUploader is an interface for copying files multipart and across
+ multiple nodes.
+ <p></p>
+ The interface extends {@link IOStatisticsSource} so that there is no
+ need to cast an instance to see if is a source of statistics.
+ However, implementations MAY return null for their actual statistics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.MultipartUploader -->
+  <!-- start class org.apache.hadoop.fs.Options -->
+  <class name="Options" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Options"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class contains options related to file system operations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Options -->
+  <!-- start class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <class name="ParentNotDirectoryException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ParentNotDirectoryException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ParentNotDirectoryException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the parent of specified Path is not a directory
+ as expected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ParentNotDirectoryException -->
+  <!-- start interface org.apache.hadoop.fs.PartHandle -->
+  <interface name="PartHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to a part id for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PartHandle -->
+  <!-- start class org.apache.hadoop.fs.PartialListing -->
+  <class name="PartialListing" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PartialListing" type="org.apache.hadoop.fs.Path, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="PartialListing" type="org.apache.hadoop.fs.Path, org.apache.hadoop.ipc.RemoteException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Partial listing of the path being listed. In the case where the path is
+ a file. The list will be a singleton with the file itself.
+
+ @return Partial listing of the path being listed.
+ @throws IOException if there was an exception getting the listing.]]>
+      </doc>
+    </method>
+    <method name="getListedPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path being listed.
+
+ @return the path being listed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A partial listing of the children of a parent directory. Since it is a
+ partial listing, multiple PartialListing may need to be combined to obtain
+ the full listing of a parent directory.
+ <p/>
+ ListingBatch behaves similar to a Future, in that getting the result via
+ {@link #get()} will throw an Exception if there was a failure.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.PartialListing -->
+  <!-- start class org.apache.hadoop.fs.Path -->
+  <class name="Path" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="java.io.ObjectInputValidation"/>
+    <constructor name="Path" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new Path based on the child path resolved against the parent path.
+
+ @param parent the parent path
+ @param child the child path]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a path from a String.  Path strings are URIs, but with
+ unescaped elements and some additional normalization.
+
+ @param pathString the path string]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.net.URI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a path from a URI
+
+ @param aUri the source URI]]>
+      </doc>
+    </constructor>
+    <constructor name="Path" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a Path from components.
+
+ @param scheme the scheme
+ @param authority the authority
+ @param path the path]]>
+      </doc>
+    </constructor>
+    <method name="getPathWithoutSchemeAndAuthority" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Return a version of the given Path without the scheme information.
+
+ @param path the source Path
+ @return a copy of this Path without the scheme information]]>
+      </doc>
+    </method>
+    <method name="mergePaths" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path1" type="org.apache.hadoop.fs.Path"/>
+      <param name="path2" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Merge 2 paths such that the second path is appended relative to the first.
+ The returned path has the scheme and authority of the first path.  On
+ Windows, the drive specification in the second path is discarded.
+ 
+ @param path1 the first path
+ @param path2 the second path, to be appended relative to path1
+ @return the merged path]]>
+      </doc>
+    </method>
+    <method name="isWindowsAbsolutePath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <param name="slashed" type="boolean"/>
+      <doc>
+      <![CDATA[Determine whether a given path string represents an absolute path on
+ Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not.
+
+ @param pathString the path string to evaluate
+ @param slashed true if the given path is prefixed with "/"
+ @return true if the supplied path looks like an absolute path with a Windows
+ drive-specifier]]>
+      </doc>
+    </method>
+    <method name="toUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert this Path to a URI.
+
+ @return this Path as a URI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the FileSystem that owns this Path.
+
+ @param conf the configuration to use when resolving the FileSystem
+ @return the FileSystem that owns this Path
+ @throws java.io.IOException thrown if there's an issue resolving the
+ FileSystem]]>
+      </doc>
+    </method>
+    <method name="isAbsoluteAndSchemeAuthorityNull" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute <strong>and</strong> the scheme is null, <b>and</b> the authority
+ is null.
+
+ @return whether the path is absolute and the URI has no scheme nor
+ authority parts]]>
+      </doc>
+    </method>
+    <method name="isUriPathAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isAbsolute" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the path component (i.e. directory) of this URI is
+ absolute.  This method is a wrapper for {@link #isUriPathAbsolute()}.
+
+ @return whether this URI's path is absolute]]>
+      </doc>
+    </method>
+    <method name="isRoot" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if and only if this path represents the root of a file system.
+
+ @return true if and only if this path represents the root of a file system]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the final component of this path.
+
+ @return the final component of this path]]>
+      </doc>
+    </method>
+    <method name="getParent" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the parent of a path or null if at root.
+ @return the parent of a path or null if at root]]>
+      </doc>
+    </method>
+    <method name="suffix" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Adds a suffix to the final name in the path.
+
+ @param suffix the suffix to add
+ @return a new path with the suffix added]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="depth" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of elements in this path.
+ @return the number of elements in this path]]>
+      </doc>
+    </method>
+    <method name="makeQualified" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #makeQualified(URI, Path)}">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Returns a qualified path object for the {@link FileSystem}'s working
+ directory.
+  
+ @param fs the target FileSystem
+ @return a qualified path object for the FileSystem's working directory
+ @deprecated use {@link #makeQualified(URI, Path)}]]>
+      </doc>
+    </method>
+    <method name="validateObject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InvalidObjectException" type="java.io.InvalidObjectException"/>
+      <doc>
+      <![CDATA[Validate the contents of a deserialized Path, so as
+ to defend against malicious object streams.
+ @throws InvalidObjectException if there's no URI]]>
+      </doc>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash.]]>
+      </doc>
+    </field>
+    <field name="SEPARATOR_CHAR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The directory separator, a slash, as a character.]]>
+      </doc>
+    </field>
+    <field name="CUR_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current directory, ".".]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the current host is a Windows machine.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Names a file or directory in a {@link FileSystem}.
+ Path strings use slash as the directory separator.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Path -->
+  <!-- start interface org.apache.hadoop.fs.PathFilter -->
+  <interface name="PathFilter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="accept" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Tests whether or not the specified abstract pathname should be
+ included in a pathname list.
+
+ @param  path  The abstract pathname to be tested
+ @return  <code>true</code> if and only if <code>pathname</code>
+          should be included]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathFilter -->
+  <!-- start interface org.apache.hadoop.fs.PathHandle -->
+  <interface name="PathHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized form in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes of this path handle.
+ @return the bytes to get to the process completing the upload.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an entity in the FileSystem. May contain
+ metadata sufficient to resolve or verify subsequent accesses independent of
+ other modifications to the FileSystem.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PathHandle -->
+  <!-- start interface org.apache.hadoop.fs.PositionedReadable -->
+  <interface name="PositionedReadable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read up to the specified number of bytes, from a given
+ position within a file, and return the number of bytes read. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @return actual number of bytes read; -1 means "none"
+ @throws IOException IO problems.]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the specified number of bytes, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @param offset offset in the buffer
+ @param length number of bytes to read
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="long"/>
+      <param name="buffer" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read number of bytes equal to the length of the buffer, from a given
+ position within a file. This does not
+ change the current offset of a file, and is thread-safe.
+
+ <i>Warning: Not all filesystems satisfy the thread-safety requirement.</i>
+ @param position position within file
+ @param buffer destination buffer
+ @throws IOException IO problems.
+ @throws EOFException the end of the data was reached before
+ the read operation completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits positional reading.
+
+ Implementations are required to implement thread-safe operations; this may
+ be supported by concurrent access to the data, or by using a synchronization
+ mechanism to serialize access.
+
+ Not all implementations meet this requirement. Those that do not cannot
+ be used as a backing store for some applications, such as Apache HBase.
+
+ Independent of whether or not they are thread safe, some implementations
+ may make the intermediate state of the system, specifically the position
+ obtained in {@code Seekable.getPos()} visible.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.PositionedReadable -->
+  <!-- start class org.apache.hadoop.fs.QuotaUsage -->
+  <class name="QuotaUsage" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QuotaUsage"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QuotaUsage" type="org.apache.hadoop.fs.QuotaUsage.Builder"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build the instance based on the builder.]]>
+      </doc>
+    </constructor>
+    <method name="setQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="quota" type="long"/>
+    </method>
+    <method name="setSpaceConsumed"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceConsumed" type="long"/>
+    </method>
+    <method name="setSpaceQuota"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spaceQuota" type="long"/>
+    </method>
+    <method name="getFileAndDirectoryCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory count.]]>
+      </doc>
+    </method>
+    <method name="getQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the directory quota.]]>
+      </doc>
+    </method>
+    <method name="getSpaceConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space consumed.]]>
+      </doc>
+    </method>
+    <method name="getSpaceQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return (disk) space quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeQuota" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type quota.]]>
+      </doc>
+    </method>
+    <method name="getTypeConsumed" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+      <doc>
+      <![CDATA[Return storage type consumed.]]>
+      </doc>
+    </method>
+    <method name="isTypeQuotaSet" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type quota has been set.]]>
+      </doc>
+    </method>
+    <method name="isTypeConsumedAvailable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if any storage type consumption information is available.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the header of the output.
+ @return the header of the output]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="tOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+      <doc>
+      <![CDATA[Return the string representation of the object in the output format.
+ if hOption is false file sizes are returned in bytes
+ if hOption is true file sizes are returned in human readable
+
+ @param hOption a flag indicating if human readable output if to be used
+ @return the string representation of the object]]>
+      </doc>
+    </method>
+    <method name="getQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+    </method>
+    <method name="getTypesQuotaUsage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hOption" type="boolean"/>
+      <param name="types" type="java.util.List"/>
+    </method>
+    <method name="getStorageTypeHeader" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="storageTypes" type="java.util.List"/>
+      <doc>
+      <![CDATA[return the header of with the StorageTypes.
+
+ @param storageTypes
+ @return storage header string]]>
+      </doc>
+    </method>
+    <field name="QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output format:
+ |----12----| |----15----| |----15----| |----15----| |-------18-------|
+    QUOTA   REMAINING_QUOTA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]>
+      </doc>
+    </field>
+    <field name="SPACE_QUOTA_STRING_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER_FIELDS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUOTA_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Store the quota usage of a directory.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.QuotaUsage -->
+  <!-- start class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <class name="RawLocalFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RawLocalFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="useStatIfAvailable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="pathToFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Convert a path to a File.]]>
+      </doc>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fd" type="org.apache.hadoop.fs.PathHandle"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStreamWithMode" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="append" type="boolean"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="flags" type="java.util.EnumSet"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createNonRecursive" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="concat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trg" type="org.apache.hadoop.fs.Path"/>
+      <param name="psrcs" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="handleEmptyDstDirectoryOnWindows" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="srcFile" type="java.io.File"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <param name="dstFile" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="truncate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="newLength" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given path to a file or directory.
+ @param p the path to delete
+ @param recursive to delete sub-directories
+ @return true if the file or directory and all its contents were deleted
+ @throws IOException if p is non-empty and recursive is false]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+
+ (<b>Note</b>: Returned list is not sorted in any given order,
+ due to reliance on Java's {@link File#list()} API.)]]>
+      </doc>
+    </method>
+    <method name="exists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDir" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p2f" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkOneDirWithMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="p2f" type="java.io.File"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates the specified directory hierarchy. Does not
+ treat existence as an error.]]>
+      </doc>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the working directory to the given directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInitialWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.fs.FsStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="moveFromLocalFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="startLocalOutput" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsOutputFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="completeLocalOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fsWorkingFile" type="org.apache.hadoop.fs.Path"/>
+      <param name="tmpLocalFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="username" type="java.lang.String"/>
+      <param name="groupname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chown to set owner.]]>
+      </doc>
+    </method>
+    <method name="setPermission"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use the command chmod to set permission.]]>
+      </doc>
+    </method>
+    <method name="setTimes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="mtime" type="long"/>
+      <param name="atime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the {@link Path}'s last modified time and last access time to
+ the given valid times.
+
+ @param mtime the modification time to set (only if no less than zero).
+ @param atime the access time to set (only if no less than zero).
+ @throws IOException if setting the times fails.]]>
+      </doc>
+    </method>
+    <method name="createPathHandle" return="org.apache.hadoop.fs.PathHandle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.HandleOpt[]"/>
+      <doc>
+      <![CDATA[Hook to implement support for {@link PathHandle} operations.
+ @param stat Referent in the target FileSystem
+ @param opts Constraints that determine the validity of the
+            {@link PathHandle} reference.]]>
+      </doc>
+    </method>
+    <method name="supportsSymlinks" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.fs.Path"/>
+      <param name="link" type="org.apache.hadoop.fs.Path"/>
+      <param name="createParent" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileLinkStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a FileStatus representing the given path. If the path refers
+ to a symlink return a FileStatus representing the link rather than
+ the object the link refers to.]]>
+      </doc>
+    </method>
+    <method name="getLinkTarget" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasPathCapability" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="capability" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement the FileSystem API for the raw local filesystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.RawLocalFileSystem -->
+  <!-- start class org.apache.hadoop.fs.ReadOption -->
+  <class name="ReadOption" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.ReadOption[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.ReadOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Options that can be used when reading from a FileSystem.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ReadOption -->
+  <!-- start interface org.apache.hadoop.fs.Seekable -->
+  <interface name="Seekable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="seek"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Seek to the given offset from the start of the file.
+ The next read() will be from that location.  Can't
+ seek past the end of the file.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current offset from the start of the file]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stream that permits seeking.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Seekable -->
+  <!-- start class org.apache.hadoop.fs.StorageStatistics -->
+  <class name="StorageStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StorageStatistics" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this StorageStatistics object.]]>
+      </doc>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the associated file system scheme if this is scheme specific,
+ else return null.]]>
+      </doc>
+    </method>
+    <method name="getLongStatistics" return="java.util.Iterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over all the currently tracked long statistics.
+
+ The values returned will depend on the type of FileSystem or FileContext
+ object.  The values do not necessarily reflect a snapshot in time.]]>
+      </doc>
+    </method>
+    <method name="getLong" return="java.lang.Long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value of a statistic.
+
+ @return         null if the statistic is not being tracked or is not a
+                 long statistic. The value of the statistic, otherwise.]]>
+      </doc>
+    </method>
+    <method name="isTracked" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return true if a statistic is being tracked.
+
+ @return         True only if the statistic is being tracked.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset all the statistic data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[StorageStatistics contains statistics data for a FileSystem or FileContext
+ instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageStatistics -->
+  <!-- start class org.apache.hadoop.fs.StorageType -->
+  <class name="StorageType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.StorageType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="supportTypeQuota" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMovable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="asList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMovableTypes" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTypesSupportingQuota" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+    </method>
+    <method name="parseStorageType" return="org.apache.hadoop.fs.StorageType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <field name="DEFAULT" type="org.apache.hadoop.fs.StorageType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.fs.StorageType[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines the types of supported storage media. The default storage
+ medium is assumed to be DISK.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StorageType -->
+  <!-- start interface org.apache.hadoop.fs.StreamCapabilities -->
+  <interface name="StreamCapabilities"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasCapability" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Query the stream for a specific capability.
+
+ @param capability string to query the stream support for.
+ @return True if the stream supports capability.]]>
+      </doc>
+    </method>
+    <field name="HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hflush capability implemented by {@link Syncable#hflush()}.
+
+ Use the {@link #HSYNC} probe to check for the support of Syncable;
+ it's that presence of {@code hsync()} which matters.]]>
+      </doc>
+    </field>
+    <field name="HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream hsync capability implemented by {@link Syncable#hsync()}.]]>
+      </doc>
+    </field>
+    <field name="READAHEAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setReadahead capability implemented by
+ {@link CanSetReadahead#setReadahead(Long)}.]]>
+      </doc>
+    </field>
+    <field name="DROPBEHIND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream setDropBehind capability implemented by
+ {@link CanSetDropBehind#setDropBehind(Boolean)}.]]>
+      </doc>
+    </field>
+    <field name="UNBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.]]>
+      </doc>
+    </field>
+    <field name="READBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(ByteBuffer) capability implemented by
+ {@link ByteBufferReadable#read(java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <field name="PREADBYTEBUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream read(long, ByteBuffer) capability implemented by
+ {@link ByteBufferPositionedReadable#read(long, java.nio.ByteBuffer)}.]]>
+      </doc>
+    </field>
+    <field name="IOSTATISTICS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[IOStatisticsSource API.]]>
+      </doc>
+    </field>
+    <field name="ABORTABLE_STREAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stream abort() capability implemented by {@link Abortable#abort()}.
+ This matches the Path Capability
+ {@link CommonPathCapabilities#ABORTABLE_STREAM}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Interface to query streams for supported capabilities.
+
+ Capability strings must be in lower case.
+
+ Constant strings are chosen over enums in order to allow other file systems
+ to define their own capabilities.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.StreamCapabilities -->
+  <!-- start class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <class name="StreamCapabilitiesPolicy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StreamCapabilitiesPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unbuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <doc>
+      <![CDATA[Implement the policy for {@link CanUnbuffer#unbuffer()}.
+
+ @param in the input stream]]>
+      </doc>
+    </method>
+    <field name="CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Static methods to implement policies for {@link StreamCapabilities}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.StreamCapabilitiesPolicy -->
+  <!-- start interface org.apache.hadoop.fs.Syncable -->
+  <interface name="Syncable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hflush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Flush out the data in client's user buffer. After the return of
+ this call, new readers will see the data.
+ @throws IOException if any error occurs]]>
+      </doc>
+    </method>
+    <method name="hsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to posix fsync, flush out the data in client's user buffer 
+ all the way to the disk device (but the disk may have it in its cache).
+ @throws IOException if error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for flush/sync operations.
+ Consult the Hadoop filesystem specification for the definition of the
+ semantics of these operations.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.Syncable -->
+  <!-- start class org.apache.hadoop.fs.Trash -->
+  <class name="Trash" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Trash" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor.
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <constructor name="Trash" type="org.apache.hadoop.fs.FileSystem, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a trash can accessor for the FileSystem provided.
+ @param fs the FileSystem
+ @param conf a Configuration]]>
+      </doc>
+    </constructor>
+    <method name="moveToAppropriateTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[In case of the symlinks or mount points, one has to move the appropriate
+ trashbin in the actual volume of the path p being deleted.
+
+ Hence we get the file system of the fully-qualified resolved-path and
+ then move the path p to the trashbin in that volume,
+ @param fs - the filesystem of path p
+ @param p - the  path being deleted - to be moved to trasg
+ @param conf - configuration
+ @return false if the item is already in the trash or trash is disabled
+ @throws IOException on error]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the trash is enabled for this filesystem]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="checkpoint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="expunge"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="expungeImmediately"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all trash immediately.]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Provides a trash facility which supports pluggable Trash policies. 
+
+ See the implementation of the configured TrashPolicy for more
+ details.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.Trash -->
+  <!-- start class org.apache.hadoop.fs.TrashPolicy -->
+  <class name="TrashPolicy" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrashPolicy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #initialize(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used
+ @param home the home directory
+ @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Used to setup the trash policy. Must be implemented by all TrashPolicy
+ implementations. Different from initialize(conf, fs, home), this one does
+ not assume trash always under /user/$USER due to HDFS encryption zone.
+ @param conf the configuration to be used
+ @param fs the filesystem to be used]]>
+      </doc>
+    </method>
+    <method name="isEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether the Trash Policy is enabled for this filesystem.]]>
+      </doc>
+    </method>
+    <method name="moveToTrash" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move a file or directory to the current trash directory.
+ @return false if the item is already in the trash or trash is disabled]]>
+      </doc>
+    </method>
+    <method name="createCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a trash checkpoint.]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpoint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete old trash checkpoint(s).]]>
+      </doc>
+    </method>
+    <method name="deleteCheckpointsImmediately"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete all checkpoints immediately, ie empty trash.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory of the Trash Policy
+ This API does not work with files deleted from encryption zone when HDFS
+ data encryption at rest feature is enabled as rename file between
+ encryption zones or encryption zone and non-encryption zone is not allowed.
+
+ The caller is recommend to use the new API
+ TrashPolicy#getCurrentTrashDir(Path path).
+ It returns the trash location correctly for the path specified no matter
+ the path is in encryption zone or not.]]>
+      </doc>
+    </method>
+    <method name="getCurrentTrashDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current trash directory for path specified based on the Trash
+ Policy
+ @param path path to be deleted
+ @return current trash directory for the path to be deleted
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getEmptier" return="java.lang.Runnable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a {@link Runnable} that periodically empties the trash of all
+ users, intended to be run by the superuser.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, FileSystem)} instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="home" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @param home the home directory
+ @return an instance of TrashPolicy
+ @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.fs.TrashPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <doc>
+      <![CDATA[Get an instance of the configured TrashPolicy based on the value
+ of the configuration parameter fs.trash.classname.
+
+ @param conf the configuration to be used
+ @param fs the file system to be used
+ @return an instance of TrashPolicy]]>
+      </doc>
+    </method>
+    <field name="fs" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trash" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="deletionInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface is used for implementing different Trash policies.
+ Provides factory method to create instances of the configured Trash policy.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.TrashPolicy -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <class name="UnsupportedFileSystemException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedFileSystemException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message. 
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[File system for a given file system name/scheme is not supported]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedFileSystemException -->
+  <!-- start class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <class name="UnsupportedMultipartUploaderException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnsupportedMultipartUploaderException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs exception with the specified detail message.
+
+ @param message exception message.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[MultipartUploader for a given file system name/scheme is not supported.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.UnsupportedMultipartUploaderException -->
+  <!-- start interface org.apache.hadoop.fs.UploadHandle -->
+  <interface name="UploadHandle"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Serialized from in bytes.]]>
+      </doc>
+    </method>
+    <method name="bytes" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[Opaque, serializable reference to an uploadId for multipart uploads.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.UploadHandle -->
+  <!-- start class org.apache.hadoop.fs.XAttrCodec -->
+  <class name="XAttrCodec" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrCodec[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrCodec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="decodeValue" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode string representation of a value and check whether it's 
+ encoded. If the given string begins with 0x or 0X, it expresses
+ a hexadecimal number. If the given string begins with 0s or 0S,
+ base64 encoding is expected. If the given string is enclosed in 
+ double quotes, the inner string is treated as text. Otherwise 
+ the given string is treated as text. 
+ @param value string representation of the value.
+ @return byte[] the value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="encodeValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte[]"/>
+      <param name="encoding" type="org.apache.hadoop.fs.XAttrCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode byte[] value to string representation with encoding. 
+ Values encoded as text strings are enclosed in double quotes (\"), 
+ while strings encoded as hexadecimal and base64 are prefixed with 
+ 0x and 0s, respectively.
+ @param value byte[] value
+ @param encoding
+ @return String string representation of value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The value of <code>XAttr</code> is byte[], this class is to 
+ covert byte[] to some kind of string representation or convert back.
+ String representation is convenient for display and input. For example
+ display in screen as shell response and json response, input as http
+ or shell parameter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrCodec -->
+  <!-- start class org.apache.hadoop.fs.XAttrSetFlag -->
+  <class name="XAttrSetFlag" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.fs.XAttrSetFlag[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.fs.XAttrSetFlag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="validate"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="xAttrName" type="java.lang.String"/>
+      <param name="xAttrExists" type="boolean"/>
+      <param name="flag" type="java.util.EnumSet"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.fs.XAttrSetFlag -->
+  <doc>
+  <![CDATA[An abstract file system API.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.fs.audit">
+  <!-- start class org.apache.hadoop.fs.audit.CommonAuditContext -->
+  <class name="CommonAuditContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="put" return="java.util.function.Supplier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a context entry.
+ @param key key
+ @param value new value
+ @return old value or null]]>
+      </doc>
+    </method>
+    <method name="put" return="java.util.function.Supplier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.util.function.Supplier"/>
+      <doc>
+      <![CDATA[Put a context entry dynamically evaluated on demand.
+ @param key key
+ @param value new value
+ @return old value or null]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a context entry.
+ @param key key]]>
+      </doc>
+    </method>
+    <method name="get" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a context entry.
+ @param key key
+ @return value or null]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Rest the context; will set the standard options again.
+ Primarily for testing.]]>
+      </doc>
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Does the context contain a specific key?
+ @param key key
+ @return true if it is in the context.]]>
+      </doc>
+    </method>
+    <method name="currentAuditContext" return="org.apache.hadoop.fs.audit.CommonAuditContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current common audit context. Thread local.
+ @return the audit context of this thread.]]>
+      </doc>
+    </method>
+    <method name="currentThreadID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A thread ID which is unique for this process and shared across all
+ S3A clients on the same thread, even those using different FS instances.
+ @return a thread ID for reporting.]]>
+      </doc>
+    </method>
+    <method name="getEvaluatedEntries" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the evaluated operations.
+ This is the map unique to this context.
+ @return the operations map.]]>
+      </doc>
+    </method>
+    <method name="setGlobalContextEntry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set a global entry.
+ @param key key
+ @param value value]]>
+      </doc>
+    </method>
+    <method name="getGlobalContextEntry" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a global entry.
+ @param key key
+ @return value or null]]>
+      </doc>
+    </method>
+    <method name="removeGlobalContextEntry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a global entry.
+ @param key key to clear.]]>
+      </doc>
+    </method>
+    <method name="noteEntryPoint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add the entry point as a context entry with the key
+ {@link AuditConstants#PARAM_COMMAND}
+ if it has not  already been recorded.
+ This is called via ToolRunner but may be used at any
+ other entry point.
+ @param tool object loaded/being launched.]]>
+      </doc>
+    </method>
+    <method name="getGlobalContextEntries" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get an iterator over the global entries.
+ Thread safe.
+ @return an iterable to enumerate the values.]]>
+      </doc>
+    </method>
+    <field name="PROCESS_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Process ID; currently built from UUID and timestamp.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The common audit context is a map of common context information
+ which can be used with any audit span.
+ This context is shared across all Filesystems within the
+ thread.
+ Audit spans will be created with a reference to the current
+ context of their thread;
+ That reference is retained even as they are moved across threads, so
+ context information (including thread ID Java runtime).
+
+ The Global context entries are a set of key-value pairs which span
+ all threads; the {@code HttpReferrerAuditHeader} picks these
+ up automatically. It is intended for minimal use of
+ shared constant values (process ID, entry point).
+
+ An attribute set in {@link #setGlobalContextEntry(String, String)}
+ will be set across all audit spans in all threads.
+
+ The {@link #noteEntryPoint(Object)} method should be
+ used in entry points (ToolRunner.run, etc). It extracts
+ the final element of the classname and attaches that
+ to the global context with the attribute key
+ {@link AuditConstants#PARAM_COMMAND}, if not already
+ set.
+ This helps identify the application being executued.
+
+ All other values set are specific to this context, which
+ is thread local.
+ The attributes which can be added to ths common context include
+ evaluator methods which will be evaluated in whichever thread
+ invokes {@link #getEvaluatedEntries()} and then evaluates them.
+ That map of evaluated options may evaluated later, in a different
+ thread.
+
+ For setting and clearing thread-level options, use
+ {@link #currentAuditContext()} to get the thread-local
+ context for the caller, which can then be manipulated.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.audit.CommonAuditContext -->
+</package>
+<package name="org.apache.hadoop.fs.crypto">
+</package>
+<package name="org.apache.hadoop.fs.ftp">
+  <!-- start class org.apache.hadoop.fs.ftp.FTPException -->
+  <class name="FTPException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FTPException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A class to wrap a {@link Throwable} into a Runtime Exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPException -->
+  <!-- start class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+  <class name="FTPFileSystem" extends="org.apache.hadoop.fs.FileSystem"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FTPFileSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getScheme" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the protocol scheme for the FileSystem.
+ <p>
+
+ @return <code>ftp</code>]]>
+      </doc>
+    </method>
+    <method name="getDefaultPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default port for this FTPFileSystem.
+
+ @return the default port]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="open" return="org.apache.hadoop.fs.FSDataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="create" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <param name="overwrite" type="boolean"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A stream obtained via this call must be closed before using other APIs of
+ this class or else the invocation will block.]]>
+      </doc>
+    </method>
+    <method name="append" return="org.apache.hadoop.fs.FSDataOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="f" type="org.apache.hadoop.fs.Path"/>
+      <param name="bufferSize" type="int"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This optional operation is not yet supported.]]>
+      </doc>
+    </method>
+    <method name="delete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="recursive" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUri" return="java.net.URI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="mkdirs" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="permission" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="rename" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="org.apache.hadoop.fs.Path"/>
+      <param name="dst" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHomeDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newDir" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_BLOCK_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_HOST_PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_PASSWORD_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_DATA_CONNECTION_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TRANSFER_MODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="E_SAME_DIRECTORY_ONLY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_FTP_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ A {@link FileSystem} backed by an FTP client provided by <a
+ href="http://commons.apache.org/net/">Apache Commons Net</a>.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.ftp.FTPFileSystem -->
+</package>
+<package name="org.apache.hadoop.fs.http">
+</package>
+<package name="org.apache.hadoop.fs.protocolPB">
+</package>
+<package name="org.apache.hadoop.fs.sftp">
+</package>
+<package name="org.apache.hadoop.fs.shell.find">
+</package>
+<package name="org.apache.hadoop.fs.statistics">
+  <!-- start class org.apache.hadoop.fs.statistics.DurationStatisticSummary -->
+  <class name="DurationStatisticSummary" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="DurationStatisticSummary" type="java.lang.String, boolean, long, long, long, org.apache.hadoop.fs.statistics.MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ @param key Statistic key.
+ @param success Are these success or failure statistics.
+ @param count Count of operation invocations.
+ @param max Max duration; -1 if unknown.
+ @param min Min duration; -1 if unknown.
+ @param mean Mean duration -may be null. (will be cloned)]]>
+      </doc>
+    </constructor>
+    <method name="getKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSuccess" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCount" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMax" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMin" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMean" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fetchDurationSummary" return="org.apache.hadoop.fs.statistics.DurationStatisticSummary"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <param name="key" type="java.lang.String"/>
+      <param name="success" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch the duration timing summary of success or failure operations
+ from an IO Statistics source.
+ If the duration key is unknown, the summary will be incomplete.
+ @param source source of data
+ @param key duration statistic key
+ @param success fetch success statistics, or if false, failure stats.
+ @return a summary of the statistics.]]>
+      </doc>
+    </method>
+    <method name="fetchSuccessSummary" return="org.apache.hadoop.fs.statistics.DurationStatisticSummary"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Fetch the duration timing summary from an IOStatistics source.
+ If the duration key is unknown, the summary will be incomplete.
+ @param source source of data
+ @param key duration statistic key
+ @return a summary of the statistics.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Summary of duration tracking statistics
+ as extracted from an IOStatistics instance.
+ <p>
+ This is for reporting and testing.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.DurationStatisticSummary -->
+  <!-- start interface org.apache.hadoop.fs.statistics.IOStatistics -->
+  <interface name="IOStatistics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="counters" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of counters.
+ @return the current map of counters.]]>
+      </doc>
+    </method>
+    <method name="gauges" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of gauges.
+ @return the current map of gauges.]]>
+      </doc>
+    </method>
+    <method name="minimums" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of minimums.
+ @return the current map of minimums.]]>
+      </doc>
+    </method>
+    <method name="maximums" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of maximums.
+ @return the current map of maximums.]]>
+      </doc>
+    </method>
+    <method name="meanStatistics" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Map of meanStatistics.
+ @return the current map of MeanStatistic statistics.]]>
+      </doc>
+    </method>
+    <field name="MIN_UNSET_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value when a minimum value has never been set.]]>
+      </doc>
+    </field>
+    <field name="MAX_UNSET_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value when a max value has never been set.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[IO Statistics.
+ <p>
+ These are low-cost per-instance statistics provided by any Hadoop
+ I/O class instance.
+ <p>
+ Consult the filesystem specification document for the requirements
+ of an implementation of this interface.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.statistics.IOStatistics -->
+  <!-- start interface org.apache.hadoop.fs.statistics.IOStatisticsAggregator -->
+  <interface name="IOStatisticsAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="aggregate" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Aggregate the supplied statistics into the current
+ set.
+
+ @param statistics statistics; may be null
+ @return true if the statistics reference was not null and
+ so aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface exported by classes which support
+ aggregation of {@link IOStatistics}.
+ Implementations MAY aggregate all statistics
+ exported by the IOStatistics reference passed in to
+ {@link #aggregate(IOStatistics)}, or they
+ may selectively aggregate specific values/classes
+ of statistics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.fs.statistics.IOStatisticsAggregator -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsLogging -->
+  <class name="IOStatisticsLogging" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="ioStatisticsSourceToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract the statistics from a source object -or ""
+ if it is not an instance of {@link IOStatistics},
+ {@link IOStatisticsSource} or the retrieved
+ statistics are null.
+ <p>
+ Exceptions are caught and downgraded to debug logging.
+ @param source source of statistics.
+ @return a string for logging.]]>
+      </doc>
+    </method>
+    <method name="ioStatisticsToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Convert IOStatistics to a string form.
+ @param statistics A statistics instance.
+ @return string value or the empty string if null]]>
+      </doc>
+    </method>
+    <method name="ioStatisticsToPrettyString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Convert IOStatistics to a string form, with all the metrics sorted
+ and empty value stripped.
+ This is more expensive than the simple conversion, so should only
+ be used for logging/output where it's known/highly likely that the
+ caller wants to see the values. Not for debug logging.
+ @param statistics A statistics instance.
+ @return string value or the empty string if null]]>
+      </doc>
+    </method>
+    <method name="demandStringifyIOStatisticsSource" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+      <doc>
+      <![CDATA[On demand stringifier of an IOStatisticsSource instance.
+ <p>
+ Whenever this object's toString() method is called, it evaluates the
+ statistics.
+ <p>
+ This is designed to affordable to use in log statements.
+ @param source source of statistics -may be null.
+ @return an object whose toString() operation returns the current values.]]>
+      </doc>
+    </method>
+    <method name="demandStringifyIOStatistics" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[On demand stringifier of an IOStatistics instance.
+ <p>
+ Whenever this object's toString() method is called, it evaluates the
+ statistics.
+ <p>
+ This is for use in log statements where for the cost of creation
+ of this entry is low; it is affordable to use in log statements.
+ @param statistics statistics to stringify -may be null.
+ @return an object whose toString() operation returns the current values.]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtDebug"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="message" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract any statistics from the source and log at debug, if
+ the log is set to log at debug.
+ No-op if logging is not at debug or the source is null/of
+ the wrong type/doesn't provide statistics.
+ @param log log to log to
+ @param message message for log -this must contain "{}" for the
+ statistics report to actually get logged.
+ @param source source object]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtDebug"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Extract any statistics from the source and log to
+ this class's log at debug, if
+ the log is set to log at debug.
+ No-op if logging is not at debug or the source is null/of
+ the wrong type/doesn't provide statistics.
+ @param message message for log -this must contain "{}" for the
+ statistics report to actually get logged.
+ @param source source object]]>
+      </doc>
+    </method>
+    <method name="logIOStatisticsAtLevel"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="level" type="java.lang.String"/>
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[A method to log IOStatistics from a source at different levels.
+
+ @param log    Logger for logging.
+ @param level  LOG level.
+ @param source Source to LOG.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Utility operations convert IO Statistics sources/instances
+ to strings, especially for robustly logging.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsLogging -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsSnapshot -->
+  <class name="IOStatisticsSnapshot" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.statistics.IOStatistics"/>
+    <implements name="java.io.Serializable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsAggregator"/>
+    <constructor name="IOStatisticsSnapshot"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct.]]>
+      </doc>
+    </constructor>
+    <constructor name="IOStatisticsSnapshot" type="org.apache.hadoop.fs.statistics.IOStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct, taking a snapshot of the source statistics data
+ if the source is non-null.
+ If the source is null, the empty maps are created
+ @param source statistics source. Nullable.]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear all the maps.]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Take a snapshot.
+
+ This completely overwrites the map data with the statistics
+ from the source.
+ @param source statistics source.]]>
+      </doc>
+    </method>
+    <method name="aggregate" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Aggregate the current statistics with the
+ source reference passed in.
+
+ The operation is synchronized.
+ @param source source; may be null
+ @return true if a merge took place.]]>
+      </doc>
+    </method>
+    <method name="counters" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="gauges" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="minimums" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="maximums" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="meanStatistics" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serializer" return="org.apache.hadoop.util.JsonSerialization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a JSON serializer for this class.
+ @return a serializer.]]>
+      </doc>
+    </method>
+    <method name="requiredSerializationClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[What classes are needed to deserialize this class?
+ Needed to securely unmarshall this from untrusted sources.
+ @return a list of required classes to deserialize the data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Snapshot of statistics from a different source.
+ <p>
+ It is serializable so that frameworks which can use java serialization
+ to propagate data (Spark, Flink...) can send the statistics
+ back. For this reason, TreeMaps are explicitly used as field types,
+ even though IDEs can recommend use of Map instead.
+ For security reasons, untrusted java object streams should never be
+ deserialized. If for some reason this is required, use
+ {@link #requiredSerializationClasses()} to get the list of classes
+ used when deserializing instances of this object.
+ <p>
+ <p>
+ It is annotated for correct serializations with jackson2.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsSnapshot -->
+  <!-- start class org.apache.hadoop.fs.statistics.IOStatisticsSupport -->
+  <class name="IOStatisticsSupport" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="snapshotIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatisticsSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statistics" type="org.apache.hadoop.fs.statistics.IOStatistics"/>
+      <doc>
+      <![CDATA[Take a snapshot of the current statistics state.
+ <p>
+ This is not an atomic option.
+ <p>
+ The instance can be serialized, and its
+ {@code toString()} method lists all the values.
+ @param statistics statistics
+ @return a snapshot of the current values.]]>
+      </doc>
+    </method>
+    <method name="snapshotIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatisticsSnapshot"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a snapshot statistics instance ready to aggregate data.
+
+ The instance can be serialized, and its
+ {@code toString()} method lists all the values.
+ @return an empty snapshot]]>
+      </doc>
+    </method>
+    <method name="retrieveIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Get the IOStatistics of the source, casting it
+ if it is of the relevant type, otherwise,
+ if it implements {@link IOStatisticsSource}
+ extracting the value.
+
+ Returns null if the source isn't of the write type
+ or the return value of
+ {@link IOStatisticsSource#getIOStatistics()} was null.
+ @return an IOStatistics instance or null]]>
+      </doc>
+    </method>
+    <method name="stubDurationTrackerFactory" return="org.apache.hadoop.fs.statistics.DurationTrackerFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a stub duration tracker factory whose returned trackers
+ are always no-ops.
+
+ As singletons are returned, this is very low-cost to use.
+ @return a duration tracker factory.]]>
+      </doc>
+    </method>
+    <method name="stubDurationTracker" return="org.apache.hadoop.fs.statistics.DurationTracker"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a stub duration tracker.
+ @return a stub tracker.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Support for working with IOStatistics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.IOStatisticsSupport -->
+  <!-- start class org.apache.hadoop.fs.statistics.MeanStatistic -->
+  <class name="MeanStatistic" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="MeanStatistic" type="long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor, with some resilience against invalid sample counts.
+ If the sample count is 0 or less, the sum is set to 0 and
+ the sample count to 0.
+ @param samples sample count.
+ @param sum sum value]]>
+      </doc>
+    </constructor>
+    <constructor name="MeanStatistic" type="org.apache.hadoop.fs.statistics.MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create from another statistic.
+ @param that source]]>
+      </doc>
+    </constructor>
+    <constructor name="MeanStatistic"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty statistic.]]>
+      </doc>
+    </constructor>
+    <method name="getSum" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the sum of samples.
+ @return the sum]]>
+      </doc>
+    </method>
+    <method name="getSamples" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the sample count.
+ @return the sample count; 0 means empty]]>
+      </doc>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is a statistic empty?
+ @return true if the sample count is 0]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the values to 0.]]>
+      </doc>
+    </method>
+    <method name="setSamplesAndSum"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sampleCount" type="long"/>
+      <param name="newSum" type="long"/>
+      <doc>
+      <![CDATA[Set the sum and samples.
+ Synchronized.
+ @param sampleCount new sample count.
+ @param newSum new sum]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.fs.statistics.MeanStatistic"/>
+      <doc>
+      <![CDATA[Set the statistic to the values of another.
+ Synchronized.
+ @param other the source.]]>
+      </doc>
+    </method>
+    <method name="setSum"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Set the sum.
+ @param sum new sum]]>
+      </doc>
+    </method>
+    <method name="setSamples"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="samples" type="long"/>
+      <doc>
+      <![CDATA[Set the sample count.
+
+ If this is less than zero, it is set to zero.
+ This stops an ill-formed JSON entry from
+ breaking deserialization, or get an invalid sample count
+ into an entry.
+ @param samples sample count.]]>
+      </doc>
+    </method>
+    <method name="mean" return="double"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arithmetic mean value.
+ @return the mean]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.fs.statistics.MeanStatistic"/>
+      <doc>
+      <![CDATA[Add another MeanStatistic.
+ @param other other value]]>
+      </doc>
+    </method>
+    <method name="addSample"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a sample.
+ Thread safe.
+ @param value value to add to the sum]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The hash code is derived from the mean
+ and sample count: if either is changed
+ the statistic cannot be used as a key
+ for hash tables/maps.
+ @return a hash value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="copy" return="org.apache.hadoop.fs.statistics.MeanStatistic"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of this instance.
+ @return copy.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mean statistic represented as the sum and the sample count;
+ the mean is calculated on demand.
+ <p>
+ It can be used to accrue values so as to dynamically update
+ the mean. If so, know that there is no synchronization
+ on the methods.
+ </p>
+ <p>
+ If a statistic has 0 samples then it is considered to be empty.
+ </p>
+ <p>
+ All 'empty' statistics are equivalent, independent of the sum value.
+ </p>
+ <p>
+ For non-empty statistics, sum and sample values must match
+ for equality.
+ </p>
+ <p>
+ It is serializable and annotated for correct serializations with jackson2.
+ </p>
+ <p>
+ Thread safety. The operations to add/copy sample data, are thread safe.
+ </p>
+ <ol>
+   <li>{@link #add(MeanStatistic)}</li>
+   <li>{@link #addSample(long)} </li>
+   <li>{@link #clear()} </li>
+   <li>{@link #setSamplesAndSum(long, long)}</li>
+   <li>{@link #set(MeanStatistic)}</li>
+   <li>{@link #setSamples(long)} and {@link #setSum(long)}</li>
+ </ol>
+ <p>
+ So is the {@link #mean()} method. This ensures that when
+ used to aggregated statistics, the aggregate value and sample
+ count are set and evaluated consistently.
+ </p>
+ <p>
+   Other methods marked as synchronized because Findbugs overreacts
+   to the idea that some operations to update sum and sample count
+   are synchronized, but that things like equals are not.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.MeanStatistic -->
+  <!-- start class org.apache.hadoop.fs.statistics.StoreStatisticNames -->
+  <class name="StoreStatisticNames" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="OP_ABORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_ACCESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[access() API call {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_COPY_FROM_LOCAL_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_CREATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_CREATE_NON_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_DELETE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_EXISTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_CONTENT_SUMMARY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_DELEGATION_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_FILE_CHECKSUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_FILE_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GET_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_GLOB_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_IS_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_HFLUSH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_HSYNC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_IS_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_LOCATED_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_LIST_STATUS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_MKDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_MODIFY_ACL_ENTRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_OPEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_ACL_ENTRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_REMOVE_DEFAULT_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_RENAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_OWNER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_PERMISSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_SET_TIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_TRUNCATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttrs(Path path)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_NAMED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttr(Path, String)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_GET_NAMED_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code getXAttrs(Path path, List<String> names)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OP_XATTR_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Invoke {@code listXAttrs(Path path)}: {@value}.]]>
+      </doc>
+    </field>
+    <field name="DELEGATION_TOKENS_ISSUED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_EXISTS_PROBE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Probe for store existing: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_THROTTLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests throttled and retried: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests made of a store: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STORE_IO_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[IO retried: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_LIST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A store's equivalent of a paged LIST request was initiated: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_CONTINUE_LIST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of continued object listings made.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_BULK_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A bulk DELETE request was made: {@value}.
+ A separate statistic from {@link #OBJECT_DELETE_REQUEST}
+ so that metrics on duration of the operations can
+ be distinguished.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A store's equivalent of a DELETE request was made: {@value}.
+ This may be an HTTP DELETE verb, or it may be some custom
+ operation which takes a list of objects to delete.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_DELETE_OBJECTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The count of objects deleted in delete requests.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_MULTIPART_UPLOAD_INITIATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object multipart upload initiated.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_MULTIPART_UPLOAD_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object multipart upload aborted.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object put/multipart upload count.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST_COMPLETED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Object put/multipart upload completed count.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_REQUEST_ACTIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Current number of active put requests.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of bytes uploaded.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_PUT_BYTES_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of bytes queued for upload/being actively uploaded.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_SELECT_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of S3 Select (or similar) requests issued.
+ Value :{@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a minimum value when
+ the same key is shared across min/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a maximum value when
+ the same key is shared across max/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_MEAN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Suffix to use for a mean statistic value when
+ the same key is shared across mean/mean/max
+ statistics.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="SUFFIX_FAILURES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[String to add to counters and other stats to track failures.
+ This comes before the .min/.mean//max suffixes.
+
+ Value {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_EXECUTOR_ACQUIRED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the statistic collected for executor acquisition if
+ a duration tracker factory is passed in to the constructor.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_HEAD_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP HEAD request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_GET_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP GET request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_DELETE_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP DELETE request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP PUT request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_PATCH_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP PATCH request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ACTION_HTTP_POST_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP POST request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_METADATA_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[An HTTP HEAD request was made: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OBJECT_COPY_REQUESTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STORE_IO_THROTTLE_RATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_INSTANTIATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_PART_PUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_PART_PUT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_COMPLETED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_STARTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MULTIPART_UPLOAD_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Common statistic names for object store operations..
+ <p>
+ When adding new common statistic name constants, please make them unique.
+ By convention:
+ </p>
+ <ul>
+   <li>the name of the constants are uppercase, words separated by
+   underscores.</li>
+   <li>the value of the constants are lowercase of the constant names.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.StoreStatisticNames -->
+  <!-- start class org.apache.hadoop.fs.statistics.StreamStatisticNames -->
+  <class name="StreamStatisticNames" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="STREAM_READ_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the TCP stream was aborted.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Bytes read from an input stream in read() calls.
+ Does not include bytes read and then discarded in seek/close etc.
+ These are the bytes returned to the caller.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES_DISCARDED_ABORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes discarded by aborting an input stream .
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_BYTES_DISCARDED_CLOSE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes read and discarded when closing an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_CLOSED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the TCP stream was closed.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_CLOSE_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of times an attempt to close an input stream was made
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPENED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of times an input stream to was opened.
+ For object stores, that means the count a GET request was initiated.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_EXCEPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of exceptions raised during input stream reads.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_FULLY_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of readFully() operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of read() operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_OPERATIONS_INCOMPLETE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of incomplete read() operations in an input stream,
+ that is, when the bytes returned were less than that requested.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_VERSION_MISMATCHES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of version mismatches encountered while reading an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BACKWARD_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of executed seek operations which went backwards in a stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_BACKWARDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes moved backwards during seek operations
+ in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_DISCARDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes read and discarded during seek() in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_BYTES_SKIPPED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of bytes skipped during forward seek operations.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_FORWARD_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of executed seek operations which went forward in
+ an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_POLICY_CHANGED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of times the seek policy was dynamically changed
+ in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SEEK_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of seek operations in an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SKIP_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of {@code InputStream.skip()} calls.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_SKIP_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count bytes skipped in {@code InputStream.skip()} calls.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_TOTAL_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total count of bytes read from an input stream.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_READ_UNBUFFERED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of calls of {@code CanUnbuffer.unbuffer()}.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_EXCEPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA["Count of stream write failures reported.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_EXCEPTIONS_COMPLETING_UPLOADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of failures when finalizing a multipart upload:
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of block/partition uploads complete.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_ABORTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of number of block uploads aborted.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_ACTIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of block/partition uploads active.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gauge of data queued to be written.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_COMMITTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of number of block uploads committed.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BLOCK_UPLOADS_PENDING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gauge of block/partitions uploads queued to be written.
+ Value: {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA["Count of bytes written to output stream including all not yet uploaded.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_TOTAL_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count of total time taken for uploads to complete.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_QUEUE_DURATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total queue duration of all block uploads.
+ {@value}.]]>
+      </doc>
+    </field>
+    <field name="STREAM_WRITE_TOTAL_DATA" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BYTES_TO_UPLOAD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes to upload from an OutputStream.]]>
+      </doc>
+    </field>
+    <field name="BYTES_UPLOAD_SUCCESSFUL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes uploaded successfully to the object store.]]>
+      </doc>
+    </field>
+    <field name="BYTES_UPLOAD_FAILED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes failed to upload to the object store.]]>
+      </doc>
+    </field>
+    <field name="TIME_SPENT_ON_TASK_WAIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total time spent on waiting for a task to complete.]]>
+      </doc>
+    </field>
+    <field name="QUEUE_SHRUNK_OPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of task queue shrunk operations.]]>
+      </doc>
+    </field>
+    <field name="WRITE_CURRENT_BUFFER_OPERATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of times current buffer is written to the service.]]>
+      </doc>
+    </field>
+    <field name="TIME_SPENT_ON_PUT_REQUEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total time spent on completing a PUT request.]]>
+      </doc>
+    </field>
+    <field name="SEEK_IN_BUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seeks in buffer.]]>
+      </doc>
+    </field>
+    <field name="BYTES_READ_BUFFER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of bytes read from the buffer.]]>
+      </doc>
+    </field>
+    <field name="REMOTE_READ_OP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of remote read operations performed.]]>
+      </doc>
+    </field>
+    <field name="READ_AHEAD_BYTES_READ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of bytes read from readAhead.]]>
+      </doc>
+    </field>
+    <field name="REMOTE_BYTES_READ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of bytes read from remote operations.]]>
+      </doc>
+    </field>
+    <field name="BLOCKS_ALLOCATED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of Data blocks allocated by an outputStream.]]>
+      </doc>
+    </field>
+    <field name="BLOCKS_RELEASED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Total number of Data blocks released by an outputStream.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[These are common statistic names.
+ <p>
+ When adding new common statistic name constants, please make them unique.
+ By convention, they are implicitly unique:
+ <ul>
+   <li>
+     The name of the constants are uppercase, words separated by
+     underscores.
+   </li>
+   <li>
+     The value of the constants are lowercase of the constant names.
+   </li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.fs.statistics.StreamStatisticNames -->
+</package>
+<package name="org.apache.hadoop.ha">
+  <!-- start class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <class name="BadFencingConfigurationException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BadFencingConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BadFencingConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Indicates that the operator has specified an invalid configuration
+ for fencing methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.BadFencingConfigurationException -->
+  <!-- start class org.apache.hadoop.ha.FailoverFailedException -->
+  <class name="FailoverFailedException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailoverFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FailoverFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate service failover has failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.FailoverFailedException -->
+  <!-- start interface org.apache.hadoop.ha.FenceMethod -->
+  <interface name="FenceMethod"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="checkArgs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Verify that the given fencing method's arguments are valid.
+ @param args the arguments provided in the configuration. This may
+        be null if the operator did not configure any arguments.
+ @throws BadFencingConfigurationException if the arguments are invalid]]>
+      </doc>
+    </method>
+    <method name="tryFence" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="org.apache.hadoop.ha.HAServiceTarget"/>
+      <param name="args" type="java.lang.String"/>
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[Attempt to fence the target node.
+ @param target the target of the service to fence
+ @param args the configured arguments, which were checked at startup by
+             {@link #checkArgs(String)}
+ @return true if fencing was successful, false if unsuccessful or
+              indeterminate
+ @throws BadFencingConfigurationException if the configuration was
+         determined to be invalid only at runtime]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A fencing method is a method by which one node can forcibly prevent
+ another node from making continued progress. This might be implemented
+ by killing a process on the other node, by denying the other node's
+ access to shared storage, or by accessing a PDU to cut the other node's
+ power.
+ <p>
+ Since these methods are often vendor- or device-specific, operators
+ may implement this interface in order to achieve fencing.
+ <p>
+ Fencing is configured by the operator as an ordered list of methods to
+ attempt. Each method will be tried in turn, and the next in the list
+ will only be attempted if the previous one fails. See {@link NodeFencer}
+ for more information.
+ <p>
+ If an implementation also implements {@link Configurable} then its
+ <code>setConf</code> method will be called upon instantiation.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.FenceMethod -->
+  <!-- start interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <interface name="HAServiceProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="monitorHealth"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="HealthCheckFailedException" type="org.apache.hadoop.ha.HealthCheckFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Monitor the health of service. This periodically called by the HA
+ frameworks to monitor the health of the service.
+ 
+ Service is expected to perform checks to ensure it is functional.
+ If the service is not healthy due to failure or partial failure,
+ it is expected to throw {@link HealthCheckFailedException}.
+ The definition of service not healthy is left to the service.
+ 
+ Note that when health check of an Active service fails,
+ failover to standby may be done.
+ 
+ @throws HealthCheckFailedException
+           if the health check of a service fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToActive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to active state. No operation, if the
+ service is already in active state.
+ 
+ @throws ServiceFailedException
+           if transition from standby to active fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToStandby"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to standby state. No operation, if the
+ service is already in standby state.
+ 
+ @throws ServiceFailedException
+           if transition from active to standby fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="transitionToObserver"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="ServiceFailedException" type="org.apache.hadoop.ha.ServiceFailedException"/>
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request service to transition to observer state. No operation, if the
+ service is already in observer state.
+
+ @throws ServiceFailedException
+           if transition from standby to observer fails.
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen]]>
+      </doc>
+    </method>
+    <method name="getServiceStatus" return="org.apache.hadoop.ha.HAServiceStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current status of the service. The status indicates
+ the current <em>state</em> (e.g ACTIVE/STANDBY) as well as
+ some additional information.
+ 
+ @throws AccessControlException
+           if access is denied.
+ @throws IOException
+           if other errors happen
+ @see HAServiceStatus]]>
+      </doc>
+    </method>
+    <field name="versionID" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial version of the protocol]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Protocol interface that provides High Availability related primitives to
+ monitor and fail-over the service.
+ 
+ This interface could be used by HA frameworks to manage the service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.HAServiceProtocol -->
+  <!-- start class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <class name="HAServiceProtocolHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceProtocolHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="monitorHealth"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToActive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToStandby"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transitionToObserver"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="svc" type="org.apache.hadoop.ha.HAServiceProtocol"/>
+      <param name="reqInfo" type="org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Helper for making {@link HAServiceProtocol} RPC calls. This helper
+ unwraps the {@link RemoteException} to specific exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceProtocolHelper -->
+  <!-- start class org.apache.hadoop.ha.HAServiceTarget -->
+  <class name="HAServiceTarget" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HAServiceTarget"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the target node.]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an optional separate RPC server address for health checks at the
+ target node.  If defined, then this address is used by the health monitor
+ for the {@link HAServiceProtocol#monitorHealth()} and
+ {@link HAServiceProtocol#getServiceStatus()} calls.  This can be useful for
+ separating out these calls onto separate RPC handlers to protect against
+ resource exhaustion in the main RPC handler pool.  If null (which is the
+ default implementation), then all RPC calls go to the address defined by
+ {@link #getAddress()}.
+
+ @return IPC address of the lifeline RPC server on the target node, or null
+     if no lifeline RPC server is used]]>
+      </doc>
+    </method>
+    <method name="getZKFCAddress" return="java.net.InetSocketAddress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the IPC address of the ZKFC on the target node]]>
+      </doc>
+    </method>
+    <method name="getFencer" return="org.apache.hadoop.ha.NodeFencer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a Fencer implementation configured for this target node]]>
+      </doc>
+    </method>
+    <method name="checkFencingConfigured"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="BadFencingConfigurationException" type="org.apache.hadoop.ha.BadFencingConfigurationException"/>
+      <doc>
+      <![CDATA[@throws BadFencingConfigurationException if the fencing configuration
+ appears to be invalid. This is divorced from the above
+ {@link #getFencer()} method so that the configuration can be checked
+ during the pre-flight phase of failover.]]>
+      </doc>
+    </method>
+    <method name="getProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to connect to the target HA Service.]]>
+      </doc>
+    </method>
+    <method name="setTransitionTargetHAStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"/>
+    </method>
+    <method name="getTransitionTargetHAStatus" return="org.apache.hadoop.ha.HAServiceProtocol.HAServiceState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a proxy to connect to the target HA service for health monitoring.
+ If {@link #getHealthMonitorAddress()} is implemented to return a non-null
+ address, then this proxy will connect to that address.  Otherwise, the
+ returned proxy defaults to using {@link #getAddress()}, which means this
+ method's behavior is identical to {@link #getProxy(Configuration, int)}.
+
+ @param conf Configuration
+ @param timeoutMs timeout in milliseconds
+ @return a proxy to connect to the target HA service for health monitoring
+ @throws IOException if there is an error]]>
+      </doc>
+    </method>
+    <method name="getHealthMonitorProxy" return="org.apache.hadoop.ha.HAServiceProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <param name="retries" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getZKFCProxy" return="org.apache.hadoop.ha.ZKFCProtocol"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timeoutMs" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@return a proxy to the ZKFC which is associated with this HA service.]]>
+      </doc>
+    </method>
+    <method name="getFencingParameters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addFencingParameters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ret" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Hook to allow subclasses to add any parameters they would like to
+ expose to fencing implementations/scripts. Fencing methods are free
+ to use this map as they see fit -- notably, the shell script
+ implementation takes each entry, prepends 'target_', substitutes
+ '_' for '.', and adds it to the environment of the script.
+
+ Subclass implementations should be sure to delegate to the superclass
+ implementation as well as adding their own keys.
+
+ @param ret map which can be mutated to pass parameters to the fencer]]>
+      </doc>
+    </method>
+    <method name="isAutoFailoverEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if auto failover should be considered enabled]]>
+      </doc>
+    </method>
+    <method name="supportObserver" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this target supports the Observer state, false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents a target of the client side HA administration commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HAServiceTarget -->
+  <!-- start class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <class name="HealthCheckFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HealthCheckFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="HealthCheckFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that health check of a service failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.HealthCheckFailedException -->
+  <!-- start class org.apache.hadoop.ha.ServiceFailedException -->
+  <class name="ServiceFailedException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceFailedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ServiceFailedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception thrown to indicate that an operation performed
+ to modify the state of a service or application failed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.ha.ServiceFailedException -->
+</package>
+<package name="org.apache.hadoop.ha.protocolPB">
+  <!-- start interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <interface name="HAServiceProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB -->
+  <!-- start interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+  <interface name="ZKFCProtocolPB"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.ha.proto.ZKFCProtocolProtos.ZKFCProtocolService.BlockingInterface"/>
+    <implements name="org.apache.hadoop.ipc.VersionedProtocol"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.ha.protocolPB.ZKFCProtocolPB -->
+</package>
+<package name="org.apache.hadoop.http.lib">
+  <doc>
+  <![CDATA[This package provides user-selectable (via configuration) classes that add
+functionality to the web UI. They are configured as a list of classes in the
+configuration parameter <b>hadoop.http.filter.initializers</b>.
+
+<ul>
+<li> <b>StaticUserWebFilter</b> - An authorization plugin that makes all
+users a static configured user.
+</ul>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io">
+  <!-- start class org.apache.hadoop.io.AbstractMapWritable -->
+  <class name="AbstractMapWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractMapWritable"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addToMap"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a Class to the maps if it is not already present.]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="byte"/>
+      <doc>
+      <![CDATA[@return the Class class for the specified id]]>
+      </doc>
+    </method>
+    <method name="getId" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[@return the id for the specified Class]]>
+      </doc>
+    </method>
+    <method name="copy"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Used by child copy constructors.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the conf]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[@param conf the conf to set]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Abstract base class for MapWritable and SortedMapWritable
+ 
+ Unlike org.apache.nutch.crawl.MapWritable, this class allows creation of
+ MapWritable&lt;Writable, MapWritable&gt; so the CLASS_TO_ID and ID_TO_CLASS
+ maps travel with the class instead of being static.
+ 
+ Class ids range from 1 to 127 so there can be at most 127 distinct classes
+ in any specific map instance.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.AbstractMapWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayFile -->
+  <class name="ArrayFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ArrayFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A dense file-based mapping from integers to values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayFile -->
+  <!-- start class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <class name="ArrayPrimitiveWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayPrimitiveWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an empty instance, for use during Writable read]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance of known type but no value yet
+ for use with type-specific wrapper classes]]>
+      </doc>
+    </constructor>
+    <constructor name="ArrayPrimitiveWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Wrap an existing array of primitives
+ @param value - array of primitives]]>
+      </doc>
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original array.  
+ Client must cast it back to type componentType[]
+ (or may use type-specific wrapper classes).
+ @return - original array as Object]]>
+      </doc>
+    </method>
+    <method name="getComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDeclaredComponentType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDeclaredComponentType" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="componentType" type="java.lang.Class"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This is a wrapper class.  It wraps a Writable implementation around
+ an array of primitives (e.g., int[], long[], etc.), with optimized 
+ wire format, and without creating new objects per element.
+ 
+ This is a wrapper class only; it does not make a copy of the 
+ underlying array.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayPrimitiveWritable -->
+  <!-- start class org.apache.hadoop.io.ArrayWritable -->
+  <class name="ArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayWritable" type="java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toStrings" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable for arrays containing instances of a class. The elements of this
+ writable must all be instances of the same class. If this writable will be
+ the input for a Reducer, you will need to create a subclass that sets the
+ value to be of the proper type.
+
+ For example:
+ <code>
+ public class IntArrayWritable extends ArrayWritable {
+   public IntArrayWritable() { 
+     super(IntWritable.class); 
+   }	
+ }
+ </code>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ArrayWritable -->
+  <!-- start class org.apache.hadoop.io.BinaryComparable -->
+  <class name="BinaryComparable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="BinaryComparable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return n st bytes 0..n-1 from {#getBytes()} are valid.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return representative byte array for this instance.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.BinaryComparable"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#compareBytes(byte[],int,int,byte[],int,int)]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Compare bytes from {#getBytes()} to those provided.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true if bytes from {#getBytes()} match.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a hash of the bytes returned from {#getBytes()}.
+ @see org.apache.hadoop.io.WritableComparator#hashBytes(byte[],int)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface supported by {@link org.apache.hadoop.io.WritableComparable}
+ types supporting ordering/permutation by a representative set of bytes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BinaryComparable -->
+  <!-- start class org.apache.hadoop.io.BloomMapFile -->
+  <class name="BloomMapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomMapFile"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BLOOM_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HASH_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends {@link MapFile} and provides very much the same
+ functionality. However, it uses dynamic Bloom filters to provide
+ quick membership test for keys, and it offers a fast version of 
+ {@link Reader#get(WritableComparable, Writable)} operation, especially in
+ case of sparsely populated MapFile-s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BloomMapFile -->
+  <!-- start class org.apache.hadoop.io.BooleanWritable -->
+  <class name="BooleanWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BooleanWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="BooleanWritable" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="get" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the value of the BooleanWritable]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.BooleanWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for booleans.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BooleanWritable -->
+  <!-- start interface org.apache.hadoop.io.ByteBufferPool -->
+  <interface name="ByteBufferPool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Get a new direct ByteBuffer.  The pool can provide this from
+ removing a buffer from its internal cache, or by allocating a 
+ new buffer.
+
+ @param direct     Whether the buffer should be direct.
+ @param length     The minimum length the buffer will have.
+ @return           A new ByteBuffer.  This ByteBuffer must be direct.
+                   Its capacity can be less than what was requested, but
+                   must be at least 1 byte.]]>
+      </doc>
+    </method>
+    <method name="putBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Release a buffer back to the pool.
+ The pool may choose to put this buffer into its cache.
+
+ @param buffer    a direct bytebuffer]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.ByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.BytesWritable -->
+  <class name="BytesWritable" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="BytesWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a zero-size sequence.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value.
+ @param bytes This array becomes the backing storage for the object.]]>
+      </doc>
+    </constructor>
+    <constructor name="BytesWritable" type="byte[], int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a BytesWritable using the byte array as the initial value
+ and length as the length. Use this constructor if the array is larger
+ than the value it represents.
+ @param bytes This array becomes the backing storage for the object.
+ @param length The number of bytes to use from array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data backing the BytesWritable. Please use {@link #copyBytes()}
+ if you need the returned array to be precisely the length of the data.
+ @return The data is only valid between 0 and getLength() - 1.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getBytes()} instead.">
+      <doc>
+      <![CDATA[Get the data from the BytesWritable.
+ @deprecated Use {@link #getBytes()} instead.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current size of the buffer.]]>
+      </doc>
+    </method>
+    <method name="getSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getLength()} instead.">
+      <doc>
+      <![CDATA[Get the current size of the buffer.
+ @deprecated Use {@link #getLength()} instead.]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="int"/>
+      <doc>
+      <![CDATA[Change the size of the buffer. The values in the old range are preserved
+ and any new values are undefined. The capacity is changed if it is 
+ necessary.
+ @param size The new number of bytes]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the capacity, which is the maximum size that could handled without
+ resizing the backing storage.
+ @return The number of bytes]]>
+      </doc>
+    </method>
+    <method name="setCapacity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="new_cap" type="int"/>
+      <doc>
+      <![CDATA[Change the capacity of the backing storage.
+ The data is preserved.
+ @param new_cap The new capacity in bytes.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="org.apache.hadoop.io.BytesWritable"/>
+      <doc>
+      <![CDATA[Set the BytesWritable to the contents of the given newData.
+ @param newData the value to set this BytesWritable to.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newData" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Set the value to a copy of the given byte range
+ @param newData the new values to copy in
+ @param offset the offset in newData to start at
+ @param length the number of bytes to copy]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right_obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Are the two byte sequences equal?]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate the stream of bytes as hex pairs separated by ' '.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A byte sequence that is usable as a key or value.
+ It is resizable and distinguishes between the size of the sequence and
+ the current capacity. The hash function is the front of the md5 of the 
+ buffer. The sort order is the same as memcmp.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.BytesWritable -->
+  <!-- start class org.apache.hadoop.io.ByteWritable -->
+  <class name="ByteWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ByteWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ByteWritable" type="byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Set the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="byte"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ByteWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ByteWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ByteWritable"/>
+      <doc>
+      <![CDATA[Compares two ByteWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for a single byte.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ByteWritable -->
+  <!-- start interface org.apache.hadoop.io.Closeable -->
+  <interface name="Closeable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="use java.io.Closeable">
+    <implements name="java.io.Closeable"/>
+    <doc>
+    <![CDATA[@deprecated use java.io.Closeable]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Closeable -->
+  <!-- start class org.apache.hadoop.io.CompressedWritable -->
+  <class name="CompressedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompressedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="ensureInflated"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Must be called by all methods which access fields to ensure that the data
+ has been uncompressed.]]>
+      </doc>
+    </method>
+    <method name="readFieldsCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #readFields(DataInput)}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Subclasses implement this instead of {@link #write(DataOutput)}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base-class for Writables which store themselves compressed and lazily
+ inflate on field access.  This is useful for large objects whose fields are
+ not be altered during a map or reduce operation: leaving the field data
+ compressed makes copying the instance from one file to another much
+ faster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.CompressedWritable -->
+  <!-- start class org.apache.hadoop.io.DataOutputOutputStream -->
+  <class name="DataOutputOutputStream" extends="java.io.OutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="constructOutputStream" return="java.io.OutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <doc>
+      <![CDATA[Construct an OutputStream from the given DataOutput. If 'out'
+ is already an OutputStream, simply returns it. Otherwise, wraps
+ it in an OutputStream.
+ @param out the DataOutput to wrap
+ @return an OutputStream instance that outputs to 'out']]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[OutputStream implementation that wraps a DataOutput.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DataOutputOutputStream -->
+  <!-- start class org.apache.hadoop.io.DefaultStringifier -->
+  <class name="DefaultStringifier" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Stringifier"/>
+    <constructor name="DefaultStringifier" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="fromString" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="store"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="item" type="K"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the item in the configuration with the given keyName.
+ 
+ @param <K>  the class of the item
+ @param conf the configuration to store
+ @param item the object to be stored
+ @param keyName the name of the key to use
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="load" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="storeArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="items" type="K[]"/>
+      <param name="keyName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores the array of items in the configuration with the given keyName.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use 
+ @param items the objects to be stored
+ @param keyName the name of the key to use
+ @throws IndexOutOfBoundsException if the items array is empty
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <method name="loadArray" return="K[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keyName" type="java.lang.String"/>
+      <param name="itemClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the array of objects from the configuration.
+ 
+ @param <K> the class of the item
+ @param conf the configuration to use
+ @param keyName the name of the key to use
+ @param itemClass the class of the item
+ @return restored object
+ @throws IOException : forwards Exceptions from the underlying 
+ {@link Serialization} classes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DefaultStringifier is the default implementation of the {@link Stringifier}
+ interface which stringifies the objects using base64 encoding of the
+ serialized version of the objects. The {@link Serializer} and
+ {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ <br>
+ DefaultStringifier offers convenience methods to store/load objects to/from
+ the configuration.
+ 
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DefaultStringifier -->
+  <!-- start class org.apache.hadoop.io.DoubleWritable -->
+  <class name="DoubleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="DoubleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="DoubleWritable" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="double"/>
+    </method>
+    <method name="get" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a DoubleWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.DoubleWritable"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Writable for Double values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.DoubleWritable -->
+  <!-- start class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <class name="ElasticByteBufferPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.ByteBufferPool"/>
+    <constructor name="ElasticByteBufferPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBuffer" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="direct" type="boolean"/>
+      <param name="length" type="int"/>
+    </method>
+    <method name="putBuffer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="buffer" type="java.nio.ByteBuffer"/>
+    </method>
+    <doc>
+    <![CDATA[This is a simple ByteBufferPool which just creates ByteBuffers as needed.
+ It also caches ByteBuffers after they're released.  It will always return
+ the smallest cached buffer with at least the capacity you request.
+ We don't try to do anything clever here like try to limit the maximum cache
+ size.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ElasticByteBufferPool -->
+  <!-- start class org.apache.hadoop.io.EnumSetWritable -->
+  <class name="EnumSetWritable" extends="java.util.AbstractCollection"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. If the <tt>value</tt> argument is null or
+ its size is zero, the <tt>elementType</tt> argument must not be null. If
+ the argument <tt>value</tt>'s size is bigger than zero, the argument
+ <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </constructor>
+    <constructor name="EnumSetWritable" type="java.util.EnumSet"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new EnumSetWritable. Argument <tt>value</tt> should not be null
+ or empty.
+ 
+ @param value]]>
+      </doc>
+    </constructor>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="E"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.util.EnumSet"/>
+      <param name="elementType" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[reset the EnumSetWritable with specified
+ <tt>value</tt> and <tt>elementType</tt>. If the <tt>value</tt> argument
+ is null or its size is zero, the <tt>elementType</tt> argument must not be
+ null. If the argument <tt>value</tt>'s size is bigger than zero, the
+ argument <tt>elementType</tt> is not be used.
+ 
+ @param value
+ @param elementType]]>
+      </doc>
+    </method>
+    <method name="get" return="java.util.EnumSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this EnumSetWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true if <code>o</code> is an EnumSetWritable with the same value,
+ or both are null.]]>
+      </doc>
+    </method>
+    <method name="getElementType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class of all the elements of the underlying EnumSetWriable. It
+ may return null.
+ 
+ @return the element class]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable wrapper for EnumSet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.EnumSetWritable -->
+  <!-- start class org.apache.hadoop.io.FloatWritable -->
+  <class name="FloatWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="FloatWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FloatWritable" type="float"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Set the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this FloatWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a FloatWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.FloatWritable"/>
+      <doc>
+      <![CDATA[Compares two FloatWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for floats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.FloatWritable -->
+  <!-- start class org.apache.hadoop.io.GenericWritable -->
+  <class name="GenericWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="GenericWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="org.apache.hadoop.io.Writable"/>
+      <doc>
+      <![CDATA[Set the instance that is wrapped.
+ 
+ @param obj]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the wrapped instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTypes" return="java.lang.Class[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all classes that may be wrapped.  Subclasses should implement this
+ to return a constant array of classes.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for Writable instances.
+ <p>
+ When two sequence files, which have same Key type but different Value
+ types, are mapped out to reduce, multiple Value types is not allowed.
+ In this case, this class can help you wrap instances with different types.
+ </p>
+ 
+ <p>
+ Compared with <code>ObjectWritable</code>, this class is much more effective,
+ because <code>ObjectWritable</code> will append the class declaration as a String 
+ into the output file in every Key-Value pair.
+ </p>
+ 
+ <p>
+ Generic Writable implements {@link Configurable} interface, so that it will be 
+ configured by the framework. The configuration is passed to the wrapped objects
+ implementing {@link Configurable} interface <i>before deserialization</i>. 
+ </p>
+ 
+ how to use it: <br>
+ 1. Write your own class, such as GenericObject, which extends GenericWritable.<br> 
+ 2. Implements the abstract method <code>getTypes()</code>, defines 
+    the classes which will be wrapped in GenericObject in application.
+    Attention: this classes defined in <code>getTypes()</code> method, must
+    implement <code>Writable</code> interface.
+ <br><br>
+ 
+ The code looks like this:
+ <blockquote><pre>
+ public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+ </pre></blockquote>
+ 
+ @since Nov 8, 2006]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.GenericWritable -->
+  <!-- start class org.apache.hadoop.io.IntWritable -->
+  <class name="IntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="IntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="IntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this IntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a IntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.IntWritable"/>
+      <doc>
+      <![CDATA[Compares two IntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for ints.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IntWritable -->
+  <!-- start class org.apache.hadoop.io.IOUtils -->
+  <class name="IOUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IOUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer 
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="buffSize" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+ 
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param buffSize the size of the buffer]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another. <strong>closes the input and output streams 
+ at the end</strong>.
+
+ @param in InputStrem to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param conf the Configuration object
+ @param close whether or not close the InputStream and 
+ OutputStream at the end. The streams are closed in the finally clause.]]>
+      </doc>
+    </method>
+    <method name="copyBytes"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="count" type="long"/>
+      <param name="close" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copies count bytes from one stream to another.
+
+ @param in InputStream to read from
+ @param out OutputStream to write to
+ @param count number of bytes to copy
+ @param close whether to close the streams
+ @throws IOException if bytes can not be read or written]]>
+      </doc>
+    </method>
+    <method name="wrappedReadForCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="is" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility wrapper for reading from {@link InputStream}. It catches any errors
+ thrown by the underlying stream (either IO or decompression-related), and
+ re-throws as an IOException.
+ 
+ @param is - InputStream to be read from
+ @param buf - buffer the data is read into
+ @param off - offset within buf
+ @param len - amount of data to be read
+ @return number of bytes read]]>
+      </doc>
+    </method>
+    <method name="readFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="buf" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads len bytes in a loop.
+
+ @param in InputStream to read from
+ @param buf The buffer to fill
+ @param off offset from the buffer
+ @param len the length of bytes to read
+ @throws IOException if it could not read requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="len" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Similar to readFully(). Skips bytes in a loop.
+ @param in The InputStream to skip bytes from
+ @param len number of bytes to skip.
+ @throws IOException if it could not skip requested number of bytes 
+ for any reason (including EOF)]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param log the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close
+ @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="cleanupWithLogger"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logger" type="org.slf4j.Logger"/>
+      <param name="closeables" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Close the Closeable objects and <b>ignore</b> any {@link Throwable} or
+ null pointers. Must only be used for cleanup in exception handlers.
+
+ @param logger the log to record problems to at debug level. Can be null.
+ @param closeables the objects to close]]>
+      </doc>
+    </method>
+    <method name="closeStream"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[Closes the stream ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param stream the Stream to close]]>
+      </doc>
+    </method>
+    <method name="closeStreams"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="streams" type="java.io.Closeable[]"/>
+      <doc>
+      <![CDATA[Closes the streams ignoring {@link Throwable}.
+ Must only be called in cleaning up from exception handlers.
+
+ @param streams the Streams to close]]>
+      </doc>
+    </method>
+    <method name="closeSocket"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sock" type="java.net.Socket"/>
+      <doc>
+      <![CDATA[Closes the socket ignoring {@link IOException}
+
+ @param sock the Socket to close]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bc" type="java.nio.channels.WritableByteChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a WritableByteChannel, handling short writes.
+ 
+ @param bc               The WritableByteChannel to write to
+ @param buf              The input buffer
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="writeFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="java.nio.channels.FileChannel"/>
+      <param name="buf" type="java.nio.ByteBuffer"/>
+      <param name="offset" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a ByteBuffer to a FileChannel at a given offset, 
+ handling short writes.
+ 
+ @param fc               The FileChannel to write to
+ @param buf              The input buffer
+ @param offset           The offset in the file to start writing at
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="listDirectory" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <param name="filter" type="java.io.FilenameFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the complete list of files in a directory as strings.<p>
+
+ This is better than File#listDir because it does not ignore IOExceptions.
+
+ @param dir              The directory to list.
+ @param filter           If non-null, the filter to use when listing
+                         this directory.
+ @return                 The list of files in the directory.
+
+ @throws IOException     On I/O error]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileToSync" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.<br>
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param fileToSync the file to fsync]]>
+      </doc>
+    </method>
+    <method name="fsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="channel" type="java.nio.channels.FileChannel"/>
+      <param name="isDir" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensure that any writes to the given file is written to the storage device
+ that contains it. This method opens channel on given File and closes it
+ once the sync is done.
+ Borrowed from Uwe Schindler in LUCENE-5588
+ @param channel Channel to sync
+ @param isDir if true, the given file is a directory (Channel should be
+          opened for read and ignore IOExceptions, because not all file
+          systems and operating systems allow to fsync on a directory)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="wrapException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="methodName" type="java.lang.String"/>
+      <param name="exception" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Takes an IOException, file/directory path, and method name and returns an
+ IOException with the input exception as the cause and also include the
+ file,method details. The new exception provides the stack trace of the
+ place where the exception is thrown and some extra diagnostics
+ information.
+
+ Return instance of same exception if exception class has a public string
+ constructor; Otherwise return an PathIOException.
+ InterruptedIOException and PathIOException are returned unwrapped.
+
+ @param path file/directory path
+ @param methodName method name
+ @param exception the caught exception.
+ @return an exception to throw]]>
+      </doc>
+    </method>
+    <method name="readFullyToByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a DataInput until EOF and returns a byte array.  Make sure not to
+ pass in an infinite DataInput or this will never return.
+
+ @param in A DataInput
+ @return a byte array containing the data from the DataInput
+ @throws IOException on I/O error, other than EOF]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An utility class for I/O related functionality.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.IOUtils -->
+  <!-- start class org.apache.hadoop.io.LongWritable -->
+  <class name="LongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="LongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a LongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.LongWritable"/>
+      <doc>
+      <![CDATA[Compares two LongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.LongWritable -->
+  <!-- start class org.apache.hadoop.io.MapFile -->
+  <class name="MapFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="rename"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="oldName" type="java.lang.String"/>
+      <param name="newName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Renames an existing map directory.]]>
+      </doc>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deletes the named map file.]]>
+      </doc>
+    </method>
+    <method name="fix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <param name="dryrun" type="boolean"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[This method attempts to fix a corrupt MapFile by re-creating its index.
+ @param fs filesystem
+ @param dir directory containing the MapFile data and index
+ @param keyClass key class (has to be a subclass of Writable)
+ @param valueClass value class (has to be a subclass of Writable)
+ @param dryrun do not perform any changes, just report what needs to be done
+ @return number of valid entries in this MapFile, or -1 if no fixing was needed
+ @throws Exception]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="INDEX_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the index file.]]>
+      </doc>
+    </field>
+    <field name="DATA_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the data file.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A file-based map from keys to values.
+ 
+ <p>A map is a directory containing two files, the <code>data</code> file,
+ containing all keys and values in the map, and a smaller <code>index</code>
+ file, containing a fraction of the keys.  The fraction is determined by
+ {@link Writer#getIndexInterval()}.
+
+ <p>The index file is read entirely into memory.  Thus key implementations
+ should try to keep themselves small.
+
+ <p>Map files are created by adding entries in-order.  To maintain a large
+ database, perform updates by copying the previous version of a database and
+ merging in a sorted change list, to create a new version of the database in
+ a new file.  Sorting large change lists can be done with {@link
+ SequenceFile.Sorter}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapFile -->
+  <!-- start class org.apache.hadoop.io.MapWritable -->
+  <class name="MapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Map"/>
+    <constructor name="MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="MapWritable" type="org.apache.hadoop.io.MapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Writable"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable Map.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MapWritable -->
+  <!-- start class org.apache.hadoop.io.MD5Hash -->
+  <class name="MD5Hash" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="MD5Hash"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash from a hex string.]]>
+      </doc>
+    </constructor>
+    <constructor name="MD5Hash" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an MD5Hash with a specified value.]]>
+      </doc>
+    </constructor>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs, reads and returns an instance.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Copy the contents of another instance into this instance.]]>
+      </doc>
+    </method>
+    <method name="getDigest" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the digest bytes.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="getDigester" return="java.security.MessageDigest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a thread local MD5 digester]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a hash value for the content from the InputStream.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="data" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for a byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dataArr" type="byte[][]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Construct a hash value for an array of byte array.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="digest" return="org.apache.hadoop.io.MD5Hash"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="org.apache.hadoop.io.UTF8"/>
+      <doc>
+      <![CDATA[Construct a hash value for a String.]]>
+      </doc>
+    </method>
+    <method name="halfDigest" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a half-sized version of this MD5.  Fits in a long]]>
+      </doc>
+    </method>
+    <method name="quarterDigest" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a 32-bit digest of the MD5.
+ @return the first 4 bytes of the md5]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is an MD5Hash whose digest contains the
+ same values.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a hash code value for this object.
+ Only uses the first 4 bytes, since md5s are evenly distributed.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.io.MD5Hash"/>
+      <doc>
+      <![CDATA[Compares this object with the specified object for order.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="setDigest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hex" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the digest value from a hex string.]]>
+      </doc>
+    </method>
+    <field name="MD5_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Writable for MD5 hash values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MD5Hash -->
+  <!-- start class org.apache.hadoop.io.MultipleIOException -->
+  <class name="MultipleIOException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getExceptions" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the underlying exceptions]]>
+      </doc>
+    </method>
+    <method name="createIOException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="exceptions" type="java.util.List"/>
+      <doc>
+      <![CDATA[A convenient method to create an {@link IOException}.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Encapsulate a list of {@link IOException} into an {@link IOException}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.MultipleIOException -->
+  <!-- start class org.apache.hadoop.io.NullWritable -->
+  <class name="NullWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <method name="get" return="org.apache.hadoop.io.NullWritable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the single instance of this class.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.NullWritable"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Singleton Writable with no data.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.NullWritable -->
+  <!-- start class org.apache.hadoop.io.ObjectWritable -->
+  <class name="ObjectWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="ObjectWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ObjectWritable" type="java.lang.Class, java.lang.Object"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the instance, or null if none.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the class this is meant to be.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="instance" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reset the instance.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="writeObject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="instance" type="java.lang.Object"/>
+      <param name="declaredClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="allowCompactArrays" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.  
+ 
+ @param allowCompactArrays - set true for RPC and internal or intra-cluster
+ usages.  Set false for inter-cluster, File, and other persisted output 
+ usages, to preserve the ability to interchange files with other clusters 
+ that may not be running the same version of software.  Sometime in ~2013 
+ we can consider removing this parameter and always using the compact format.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="readObject" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="objectWritable" type="org.apache.hadoop.io.ObjectWritable"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a {@link Writable}, {@link String}, primitive type, or an array of
+ the preceding.]]>
+      </doc>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find and load the class with given name <tt>className</tt> by first finding
+ it in the specified <tt>conf</tt>. If the specified <tt>conf</tt> is null,
+ try load it directly.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A polymorphic Writable that writes an instance with it's class name.
+ Handles arrays, strings and primitive types without a Writable wrapper.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ObjectWritable -->
+  <!-- start interface org.apache.hadoop.io.RawComparator -->
+  <interface name="RawComparator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <method name="compare" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Compare two objects in binary.
+ b1[s1:l1] is the first object, and b2[s2:l2] is the second object.
+ 
+ @param b1 The first byte array.
+ @param s1 The position index in b1. The object under comparison's starting index.
+ @param l1 The length of the object in b1.
+ @param b2 The second byte array.
+ @param s2 The position index in b2. The object under comparison's starting index.
+ @param l2 The length of the object under comparison in b2.
+ @return An integer result of the comparison.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ A {@link Comparator} that operates directly on byte representations of
+ objects.
+ </p>
+ @param <T>
+ @see DeserializerComparator]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.RawComparator -->
+  <!-- start class org.apache.hadoop.io.SequenceFile -->
+  <class name="SequenceFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDefaultCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the compression type for the reduce outputs
+ @param job the job config to look in
+ @return the kind of compression to use]]>
+      </doc>
+    </method>
+    <method name="setDefaultCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="val" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the default compression type for sequence files.
+ @param job the configuration to modify
+ @param val the new compression type (none, block, record)]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="opts" type="org.apache.hadoop.io.SequenceFile.Writer.Option[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new Writer with the given options.
+ @param conf the configuration to use
+ @param opts the options to create the file with
+ @return a new Writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="bufferSize" type="int"/>
+      <param name="replication" type="short"/>
+      <param name="blockSize" type="long"/>
+      <param name="createParent" type="boolean"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param bufferSize buffer size for the underlaying outputstream.
+ @param replication replication factor for the file.
+ @param blockSize block size for the file.
+ @param createParent create parent directory if non-existent
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fc" type="org.apache.hadoop.fs.FileContext"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <param name="createFlag" type="java.util.EnumSet"/>
+      <param name="opts" type="org.apache.hadoop.fs.Options.CreateOpts[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fc The context for the specified file.
+ @param conf The configuration.
+ @param name The name of the file.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @param createFlag gives the semantics of create: overwrite, append etc.
+ @param opts file creation options; see {@link CreateOpts}.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="name" type="org.apache.hadoop.fs.Path"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of SequenceFile Writer.
+ @param fs The configured filesystem. 
+ @param conf The configuration.
+ @param name The name of the file. 
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param progress The Progressable object to track progress.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="metadata" type="org.apache.hadoop.io.SequenceFile.Metadata"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @param metadata The metadata of the file.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <method name="createWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="out" type="org.apache.hadoop.fs.FSDataOutputStream"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valClass" type="java.lang.Class"/>
+      <param name="compressionType" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct the preferred type of 'raw' SequenceFile Writer.
+ @param conf The configuration.
+ @param out The stream on top which the writer is to be constructed.
+ @param keyClass The 'key' type.
+ @param valClass The 'value' type.
+ @param compressionType The compression type.
+ @param codec The compression codec.
+ @return Returns the handle to the constructed SequenceFile Writer.
+ @throws IOException
+ @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
+     instead.]]>
+      </doc>
+    </method>
+    <field name="SYNC_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes between sync points. 100 KB, default.
+ Computed as 5 KB * 20 = 100 KB]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[<code>SequenceFile</code>s are flat files consisting of binary key/value 
+ pairs.
+ 
+ <p><code>SequenceFile</code> provides {@link SequenceFile.Writer},
+ {@link SequenceFile.Reader} and {@link Sorter} classes for writing,
+ reading and sorting respectively.</p>
+ 
+ There are three <code>SequenceFile</code> <code>Writer</code>s based on the 
+ {@link CompressionType} used to compress key/value pairs:
+ <ol>
+   <li>
+   <code>Writer</code> : Uncompressed records.
+   </li>
+   <li>
+   <code>RecordCompressWriter</code> : Record-compressed files, only compress 
+                                       values.
+   </li>
+   <li>
+   <code>BlockCompressWriter</code> : Block-compressed files, both keys &amp;
+                                      values are collected in 'blocks' 
+                                      separately and compressed. The size of 
+                                      the 'block' is configurable.
+ </ol>
+ 
+ <p>The actual compression algorithm used to compress key and/or values can be
+ specified by using the appropriate {@link CompressionCodec}.</p>
+ 
+ <p>The recommended way is to use the static <tt>createWriter</tt> methods
+ provided by the <code>SequenceFile</code> to chose the preferred format.</p>
+
+ <p>The {@link SequenceFile.Reader} acts as the bridge and can read any of the
+ above <code>SequenceFile</code> formats.</p>
+
+ <h3 id="Formats">SequenceFile Formats</h3>
+ 
+ <p>Essentially there are 3 different formats for <code>SequenceFile</code>s
+ depending on the <code>CompressionType</code> specified. All of them share a
+ <a href="#Header">common header</a> described below.
+ 
+ <h4 id="Header">SequenceFile Header</h4>
+ <ul>
+   <li>
+   version - 3 bytes of magic header <b>SEQ</b>, followed by 1 byte of actual 
+             version number (e.g. SEQ4 or SEQ6)
+   </li>
+   <li>
+   keyClassName -key class
+   </li>
+   <li>
+   valueClassName - value class
+   </li>
+   <li>
+   compression - A boolean which specifies if compression is turned on for 
+                 keys/values in this file.
+   </li>
+   <li>
+   blockCompression - A boolean which specifies if block-compression is 
+                      turned on for keys/values in this file.
+   </li>
+   <li>
+   compression codec - <code>CompressionCodec</code> class which is used for  
+                       compression of keys and/or values (if compression is 
+                       enabled).
+   </li>
+   <li>
+   metadata - {@link Metadata} for this file.
+   </li>
+   <li>
+   sync - A sync marker to denote end of the header.
+   </li>
+ </ul>
+ 
+ <h5>Uncompressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li>Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+
+ <h5>Record-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record
+   <ul>
+     <li>Record length</li>
+     <li>Key length</li>
+     <li>Key</li>
+     <li><i>Compressed</i> Value</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every few <code>100</code> kilobytes or so.
+ </li>
+ </ul>
+ 
+ <h5>Block-Compressed SequenceFile Format</h5>
+ <ul>
+ <li>
+ <a href="#Header">Header</a>
+ </li>
+ <li>
+ Record <i>Block</i>
+   <ul>
+     <li>Uncompressed number of records in the block</li>
+     <li>Compressed key-lengths block-size</li>
+     <li>Compressed key-lengths block</li>
+     <li>Compressed keys block-size</li>
+     <li>Compressed keys block</li>
+     <li>Compressed value-lengths block-size</li>
+     <li>Compressed value-lengths block</li>
+     <li>Compressed values block-size</li>
+     <li>Compressed values block</li>
+   </ul>
+ </li>
+ <li>
+ A sync-marker every block.
+ </li>
+ </ul>
+ 
+ <p>The compressed blocks of key lengths and value lengths consist of the 
+ actual lengths of individual keys/values encoded in ZeroCompressedInteger 
+ format.</p>
+ 
+ @see CompressionCodec]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SequenceFile -->
+  <!-- start class org.apache.hadoop.io.SetFile -->
+  <class name="SetFile" extends="org.apache.hadoop.io.MapFile"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SetFile"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A file-based set of keys.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SetFile -->
+  <!-- start class org.apache.hadoop.io.ShortWritable -->
+  <class name="ShortWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ShortWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ShortWritable" type="short"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="short"/>
+      <doc>
+      <![CDATA[Set the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="short"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[read the short value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[write short value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a ShortWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[hash code]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.ShortWritable"/>
+      <doc>
+      <![CDATA[Compares two ShortWritable.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Short values in string format]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for shorts.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.ShortWritable -->
+  <!-- start class org.apache.hadoop.io.SortedMapWritable -->
+  <class name="SortedMapWritable" extends="org.apache.hadoop.io.AbstractMapWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.SortedMap"/>
+    <constructor name="SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="SortedMapWritable" type="org.apache.hadoop.io.SortedMapWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Copy constructor.
+ 
+ @param other the map to copy from]]>
+      </doc>
+    </constructor>
+    <method name="comparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="firstKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="headMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="lastKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="subMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+      <param name="toKey" type="K"/>
+    </method>
+    <method name="tailMap" return="java.util.SortedMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fromKey" type="K"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="containsKey" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="containsValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="entrySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="isEmpty" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="keySet" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="put" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Writable"/>
+    </method>
+    <method name="putAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.util.Map"/>
+    </method>
+    <method name="remove" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="values" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A Writable SortedMap.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.SortedMapWritable -->
+  <!-- start interface org.apache.hadoop.io.Stringifier -->
+  <interface name="Stringifier"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Converts the object to a string representation
+ @param obj the object to convert
+ @return the string representation of the object
+ @throws IOException if the object cannot be converted]]>
+      </doc>
+    </method>
+    <method name="fromString" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restores the object from its string representation.
+ @param str the string representation of the object
+ @return restored object
+ @throws IOException if the object cannot be restored]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes this object. 
+ @throws IOException if an I/O error occurs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Stringifier interface offers two methods to convert an object 
+ to a string representation and restore the object given its 
+ string representation.
+ @param <T> the class of the objects to stringify]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Stringifier -->
+  <!-- start class org.apache.hadoop.io.Text -->
+  <class name="Text" extends="org.apache.hadoop.io.BinaryComparable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Text" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a string.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another text.]]>
+      </doc>
+    </constructor>
+    <constructor name="Text" type="byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from a byte array.]]>
+      </doc>
+    </constructor>
+    <method name="copyBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the bytes that is exactly the length of the data.
+ See {@link #getBytes()} for faster access to the underlying array.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the raw bytes; however, only data up to {@link #getLength()} is
+ valid. Please use {@link #copyBytes()} if you
+ need the returned array to be precisely the length of the data.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes in the byte array]]>
+      </doc>
+    </method>
+    <method name="charAt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="position" type="int"/>
+      <doc>
+      <![CDATA[Returns the Unicode Scalar Value (32-bit integer value)
+ for the character at <code>position</code>. Note that this
+ method avoids using the converter or doing String instantiation
+ @return the Unicode scalar value at position or -1
+          if the position is invalid or points to a
+          trailing byte]]>
+      </doc>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+    </method>
+    <method name="find" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="what" type="java.lang.String"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Finds any occurrence of <code>what</code> in the backing
+ buffer, starting as position <code>start</code>. The starting
+ position is measured in bytes and the return value is in
+ terms of byte position in the buffer. The backing buffer is
+ not converted to a string for this operation.
+ @return byte position of the first occurrence of the search
+         string in the UTF-8 buffer or -1 if not found]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set to contain the contents of a string.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <doc>
+      <![CDATA[Set to a utf8 byte array]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[copy a text.]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Set the Text to range of bytes
+ @param utf8 the data to copy from
+ @param start the first position of the new string
+ @param len the number of bytes of the new string]]>
+      </doc>
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Append a range of bytes to the end of the given text
+ @param utf8 the data to copy from
+ @param start the first position to append from utf8
+ @param len the number of bytes to append]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the string to empty.
+
+ <em>Note</em>: For performance reasons, this call does not clear the
+ underlying byte array that is retrievable via {@link #getBytes()}.
+ In order to free the byte-array memory, call {@link #set(byte[])}
+ with an empty byte array (For example, <code>new byte[0]</code>).]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert text back to string
+ @see java.lang.Object#toString()]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[deserialize]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skips over one Text in the input.]]>
+      </doc>
+    </method>
+    <method name="readWithKnownLength"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a Text object whose length is already known.
+ This allows creating Text from a stream which uses a different serialization
+ format.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[serialize
+ write this object to out
+ length uses zero-compressed encoding
+ @see Writable#write(DataOutput)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a Text with the same contents.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If the input is malformed,
+ replace by a default value.]]>
+      </doc>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+    </method>
+    <method name="decode" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided byte array to a String using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If the input is malformed,
+ invalid chars are replaced by a default value.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="encode" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <param name="replace" type="boolean"/>
+      <exception name="CharacterCodingException" type="java.nio.charset.CharacterCodingException"/>
+      <doc>
+      <![CDATA[Converts the provided String to bytes using the
+ UTF-8 encoding. If <code>replace</code> is true, then
+ malformed input is replaced with the
+ substitution character, which is U+FFFD. Otherwise the
+ method throws a MalformedInputException.
+ @return ByteBuffer: bytes stores at ByteBuffer.array() 
+                     and length is ByteBuffer.limit()]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string from in]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a UTF8 encoded string with a maximum size]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string to out]]>
+      </doc>
+    </method>
+    <method name="writeString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a UTF8 encoded string with a maximum size to out]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check if a byte array contains valid utf-8
+ @param utf8 byte array
+ @throws MalformedInputException if the byte array contains invalid utf-8]]>
+      </doc>
+    </method>
+    <method name="validateUTF8"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf8" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="MalformedInputException" type="java.nio.charset.MalformedInputException"/>
+      <doc>
+      <![CDATA[Check to see if a byte array is valid utf-8
+ @param utf8 the array of bytes
+ @param start the offset of the first byte in the array
+ @param len the length of the byte sequence
+ @throws MalformedInputException if the byte array contains invalid bytes]]>
+      </doc>
+    </method>
+    <method name="bytesToCodePoint" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Returns the next code point at the current position in
+ the buffer. The buffer's position will be incremented.
+ Any mark set on this buffer will be changed by this method!]]>
+      </doc>
+    </method>
+    <method name="utf8Length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="string" type="java.lang.String"/>
+      <doc>
+      <![CDATA[For the given string, returns the number of UTF-8 bytes
+ required to encode the string.
+ @param string text to encode
+ @return number of UTF-8 bytes required to encode]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_MAX_LEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class stores text using standard UTF8 encoding.  It provides methods
+ to serialize, deserialize, and compare texts at byte level.  The type of
+ length is integer and is serialized using zero-compressed format.  <p>In
+ addition, it provides methods for string traversal without converting the
+ byte array to a string.  <p>Also includes utilities for
+ serializing/deserialing a string, coding/decoding a string, checking if a
+ byte array contains valid UTF8 code, calculating the length of an encoded
+ string.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.Text -->
+  <!-- start class org.apache.hadoop.io.TwoDArrayWritable -->
+  <class name="TwoDArrayWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TwoDArrayWritable" type="java.lang.Class, org.apache.hadoop.io.Writable[][]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toArray" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="values" type="org.apache.hadoop.io.Writable[][]"/>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable[][]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Writable for 2D arrays containing a matrix of instances of a class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.TwoDArrayWritable -->
+  <!-- start class org.apache.hadoop.io.VersionedWritable -->
+  <class name="VersionedWritable" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="VersionedWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getVersion" return="byte"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the version number of the current implementation.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A base class for Writables that provides version checking.
+
+ <p>This is useful when a class may evolve, so that instances written by the
+ old version of the class may still be processed by the new version.  To
+ handle this situation, {@link #readFields(DataInput)}
+ implementations should catch {@link VersionMismatchException}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionedWritable -->
+  <!-- start class org.apache.hadoop.io.VersionMismatchException -->
+  <class name="VersionMismatchException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionMismatchException" type="byte, byte"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string representation of this object.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Thrown by {@link VersionedWritable#readFields(DataInput)} when the
+ version of an object being read does not match the current implementation
+ version as returned by {@link VersionedWritable#getVersion()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VersionMismatchException -->
+  <!-- start class org.apache.hadoop.io.VIntWritable -->
+  <class name="VIntWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VIntWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VIntWritable" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this VIntWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VIntWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VIntWritable"/>
+      <doc>
+      <![CDATA[Compares two VIntWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for integer values stored in variable-length format.
+ Such values take between one and five bytes.  Smaller values take fewer bytes.
+ 
+ @see org.apache.hadoop.io.WritableUtils#readVInt(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VIntWritable -->
+  <!-- start class org.apache.hadoop.io.VLongWritable -->
+  <class name="VLongWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="VLongWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="VLongWritable" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="get" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the value of this LongWritable.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Returns true iff <code>o</code> is a VLongWritable with the same value.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.io.VLongWritable"/>
+      <doc>
+      <![CDATA[Compares two VLongWritables.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A WritableComparable for longs in a variable-length format. Such values take
+  between one and five bytes.  Smaller values take fewer bytes.
+  
+  @see org.apache.hadoop.io.WritableUtils#readVLong(DataInput)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.VLongWritable -->
+  <!-- start interface org.apache.hadoop.io.Writable -->
+  <interface name="Writable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize the fields of this object to <code>out</code>.
+ 
+ @param out <code>DataOuput</code> to serialize this object into.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserialize the fields of this object from <code>in</code>.  
+ 
+ <p>For efficiency, implementations should attempt to re-use storage in the 
+ existing object where possible.</p>
+ 
+ @param in <code>DataInput</code> to deseriablize this object from.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A serializable object which implements a simple, efficient, serialization 
+ protocol, based on {@link DataInput} and {@link DataOutput}.
+
+ <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
+ framework implements this interface.</p>
+ 
+ <p>Implementations typically implement a static <code>read(DataInput)</code>
+ method which constructs a new instance, calls {@link #readFields(DataInput)} 
+ and returns the instance.</p>
+ 
+ <p>Example:</p>
+ <blockquote><pre>
+     public class MyWritable implements Writable {
+       // Some data
+       private int counter;
+       private long timestamp;
+
+       // Default constructor to allow (de)serialization
+       MyWritable() { }
+
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+ </pre></blockquote>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.Writable -->
+  <!-- start interface org.apache.hadoop.io.WritableComparable -->
+  <interface name="WritableComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Comparable"/>
+    <doc>
+    <![CDATA[A {@link Writable} which is also {@link Comparable}. 
+
+ <p><code>WritableComparable</code>s can be compared to each other, typically 
+ via <code>Comparator</code>s. Any type which is to be used as a 
+ <code>key</code> in the Hadoop Map-Reduce framework should implement this
+ interface.</p>
+
+ <p>Note that <code>hashCode()</code> is frequently used in Hadoop to partition
+ keys. It's important that your implementation of hashCode() returns the same 
+ result across different instances of the JVM. Note also that the default 
+ <code>hashCode()</code> implementation in <code>Object</code> does <b>not</b>
+ satisfy this property.</p>
+  
+ <p>Example:</p>
+ <blockquote><pre>
+     public class MyWritableComparable implements
+      WritableComparable{@literal <MyWritableComparable>} {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue &lt; thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp &gt;&gt;&gt; 32));
+         return result
+       }
+     }
+ </pre></blockquote>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableComparable -->
+  <!-- start class org.apache.hadoop.io.WritableComparator -->
+  <class name="WritableComparator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.RawComparator"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="WritableComparator"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="WritableComparator" type="java.lang.Class, org.apache.hadoop.conf.Configuration, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[For backwards compatibility.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a comparator for a {@link WritableComparable} implementation.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="define"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="comparator" type="org.apache.hadoop.io.WritableComparator"/>
+      <doc>
+      <![CDATA[Register an optimized comparator for a {@link WritableComparable}
+ implementation. Comparators registered with this method must be
+ thread-safe.]]>
+      </doc>
+    </method>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the WritableComparable implementation class.]]>
+      </doc>
+    </method>
+    <method name="newKey" return="org.apache.hadoop.io.WritableComparable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new {@link WritableComparable} instance.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Optimization hook.  Override this to make SequenceFile.Sorter's scream.
+
+ <p>The default implementation reads the data into two {@link
+ WritableComparable}s (using {@link
+ Writable#readFields(DataInput)}, then calls {@link
+ #compare(WritableComparable,WritableComparable)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.io.WritableComparable"/>
+      <param name="b" type="org.apache.hadoop.io.WritableComparable"/>
+      <doc>
+      <![CDATA[Compare two WritableComparables.
+
+ <p> The default implementation uses the natural ordering, calling {@link
+ Comparable#compareTo(Object)}.]]>
+      </doc>
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="java.lang.Object"/>
+      <param name="b" type="java.lang.Object"/>
+    </method>
+    <method name="compareBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+      <doc>
+      <![CDATA[Lexicographic order of binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="hashBytes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="length" type="int"/>
+      <doc>
+      <![CDATA[Compute hash for binary data.]]>
+      </doc>
+    </method>
+    <method name="readUnsignedShort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an unsigned short from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse an integer from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readFloat" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a float from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a long from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readDouble" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <doc>
+      <![CDATA[Parse a double from a byte array.]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from a byte array and returns it.
+ @param bytes byte array with decode long
+ @param start starting index
+ @throws java.io.IOException 
+ @return deserialized long]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+      <param name="start" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from a byte array and returns it.
+ @param bytes byte array with the encoded integer
+ @param start start index
+ @throws java.io.IOException 
+ @return deserialized integer]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A Comparator for {@link WritableComparable}s.
+
+ <p>This base implementation uses the natural ordering.  To define alternate
+ orderings, override {@link #compare(WritableComparable,WritableComparable)}.
+
+ <p>One may optimize compare-intensive operations by overriding
+ {@link #compare(byte[],int,int,byte[],int,int)}.  Static utility methods are
+ provided to assist in optimized implementations of this method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableComparator -->
+  <!-- start class org.apache.hadoop.io.WritableFactories -->
+  <class name="WritableFactories" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="factory" type="org.apache.hadoop.io.WritableFactory"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="getFactory" return="org.apache.hadoop.io.WritableFactory"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Define a factory for a class.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Create a new instance of a class with a defined factory.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Factories for non-public writables.  Defining a factory permits {@link
+ ObjectWritable} to be able to construct instances of non-public classes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableFactories -->
+  <!-- start interface org.apache.hadoop.io.WritableFactory -->
+  <interface name="WritableFactory"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="newInstance" return="org.apache.hadoop.io.Writable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a new instance.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A factory for a class of Writable.
+ @see WritableFactories]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.WritableFactory -->
+  <!-- start class org.apache.hadoop.io.WritableUtils -->
+  <class name="WritableUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WritableUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="readCompressedByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skipCompressedByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedByteArray" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="bytes" type="byte[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedString" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeCompressedStringArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readCompressedStringArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="displayByteArray"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="byte[]"/>
+    </method>
+    <method name="clone" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orig" type="T"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Make a copy of a writable object using serialization to a buffer.
+ @param orig The object to copy
+ @return The copied object]]>
+      </doc>
+    </method>
+    <method name="cloneInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="use ReflectionUtils.cloneInto instead.">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param dst the object to copy from
+ @param src the object to copy into, which is destroyed
+ @throws IOException
+ @deprecated use ReflectionUtils.cloneInto instead.]]>
+      </doc>
+    </method>
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes an integer to a binary stream with zero-compressed encoding.
+ For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ actual value.
+ For other values of i, the first byte value indicates whether the
+ integer is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -116, the following integer
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -124, the following integer
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+
+ @param stream Binary output stream
+ @param i Integer to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataOutput"/>
+      <param name="i" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes a long to a binary stream with zero-compressed encoding.
+ For -112 {@literal <=} i {@literal <=} 127, only one byte is used with the
+ actual value.
+ For other values of i, the first byte value indicates whether the
+ long is positive or negative, and the number of bytes that follow.
+ If the first byte value v is between -113 and -120, the following long
+ is positive, with number of bytes that follow are -(v+112).
+ If the first byte value v is between -121 and -128, the following long
+ is negative, with number of bytes that follow are -(v+120). Bytes are
+ stored in the high-non-zero-byte-first order.
+ 
+ @param stream Binary output stream
+ @param i Long to be serialized
+ @throws java.io.IOException]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded long from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized long from stream.]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads a zero-compressed encoded integer from input stream and returns it.
+ @param stream Binary input stream
+ @throws java.io.IOException 
+ @return deserialized integer from stream.]]>
+      </doc>
+    </method>
+    <method name="readVIntInRange" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.DataInput"/>
+      <param name="lower" type="int"/>
+      <param name="upper" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads an integer from the input stream and returns it.
+
+ This function validates that the integer is between [lower, upper],
+ inclusive.
+
+ @param stream Binary input stream
+ @throws java.io.IOException
+ @return deserialized integer from stream]]>
+      </doc>
+    </method>
+    <method name="isNegativeVInt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Given the first byte of a vint/vlong, determine the sign
+ @param value the first byte
+ @return is the value negative]]>
+      </doc>
+    </method>
+    <method name="decodeVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="byte"/>
+      <doc>
+      <![CDATA[Parse the first byte of a vint/vlong to determine the number of bytes
+ @param value the first byte of the vint/vlong
+ @return the total number of bytes (1 to 9)]]>
+      </doc>
+    </method>
+    <method name="getVIntSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="long"/>
+      <doc>
+      <![CDATA[Get the encoded length if an integer is stored in a variable-length format
+ @return the encoded length]]>
+      </doc>
+    </method>
+    <method name="readEnum" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="enumType" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read an Enum value from DataInput, Enums are read and written 
+ using String values. 
+ @param <T> Enum type
+ @param in DataInput to read from 
+ @param enumType Class type of Enum
+ @return Enum represented by String read from DataInput
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeEnum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="enumVal" type="java.lang.Enum"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[writes String value of enum to DataOutput. 
+ @param out Dataoutput stream
+ @param enumVal enum value
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="skipFully"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip <i>len</i> number of bytes in input stream<i>in</i>
+ @param in input stream
+ @param len number of bytes to skip
+ @throws IOException when skipped less number of bytes]]>
+      </doc>
+    </method>
+    <method name="toByteArray" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writables" type="org.apache.hadoop.io.Writable[]"/>
+      <doc>
+      <![CDATA[Convert writables to a byte array]]>
+      </doc>
+    </method>
+    <method name="readStringSafely" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <param name="maxLength" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Read a string, but check it for sanity. The format consists of a vint
+ followed by the given number of bytes.
+ @param in the stream to read from
+ @param maxLength the largest acceptable length of the encoded string
+ @return the bytes as a string
+ @throws IOException if reading from the DataInput fails
+ @throws IllegalArgumentException if the encoded byte size for string 
+             is negative or larger than maxSize. Only the vint is read.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.WritableUtils -->
+  <doc>
+  <![CDATA[Generic i/o code for use when reading and writing data to the network,
+to databases, and to files.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.compress">
+  <!-- start class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <class name="BlockCompressorStream" extends="org.apache.hadoop.io.compress.CompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream}.
+ 
+ @param out stream
+ @param compressor compressor to be used
+ @param bufferSize size of buffer
+ @param compressionOverhead maximum 'overhead' of the compression 
+                            algorithm with given bufferSize]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockCompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a {@link BlockCompressorStream} with given output-stream and 
+ compressor.
+ Use default of 512 as bufferSize and compressionOverhead of 
+ (1% of bufferSize + 12 bytes) =  18 bytes (zlib algorithm).
+ 
+ @param out stream
+ @param compressor compressor to be used]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the data provided to the compression codec, compressing no more
+ than the buffer size less the compression overhead as specified during
+ construction for each block.
+
+ Each block contains the uncompressed length for the block, followed by
+ one or more length-prefixed blocks of compressed data.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.CompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.
+
+ It should be noted that this wrapper does not guarantee that blocks will
+ be sized for the compressor. If the
+ {@link org.apache.hadoop.io.compress.Compressor} requires buffering to
+ effect meaningful compression, it is responsible for it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockCompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <class name="BlockDecompressorStream" extends="org.apache.hadoop.io.compress.DecompressorStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @param bufferSize size of buffer
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link BlockDecompressorStream}.
+ 
+ @param in input stream
+ @param decompressor decompressor to use
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="BlockDecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link org.apache.hadoop.io.compress.DecompressorStream} which works
+ with 'block-based' based compression algorithms, as opposed to 
+ 'stream-based' compression algorithms.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BlockDecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.BZip2Codec -->
+  <class name="BZip2Codec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.SplittableCompressionCodec"/>
+    <constructor name="BZip2Codec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a new instance of BZip2Codec.]]>
+      </doc>
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration to be used by this object.
+
+ @param conf the configuration object.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration used by this object.
+
+ @return the configuration object used by this objec.]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream}.
+
+ @param out        the location for the final output stream
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given
+ {@link OutputStream} with the given {@link Compressor}.
+
+ @param out        the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to, to have it 
+         compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream and return a stream for uncompressed data.
+
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ {@link InputStream} with the given {@link Decompressor}, and return a 
+ stream for uncompressed data.
+
+ @param in           the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates CompressionInputStream to be used to read off uncompressed data
+ in one of the two reading modes. i.e. Continuous or Blocked reading modes
+
+ @param seekableIn The InputStream
+ @param start The start offset into the compressed stream
+ @param end The end offset into the compressed stream
+ @param readMode Controls whether progress is reported continuously or
+                 only at block boundaries.
+
+ @return CompressionInputStream for BZip2 aligned at block boundaries]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[.bz2 is recognized as the default extension for compressed BZip2 files
+
+ @return A String telling the default bzip2 file extension]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class provides output and input streams for bzip2 compression
+ and decompression.  It uses the native bzip2 library on the system
+ if possible, else it uses a pure-Java implementation of the bzip2
+ algorithm.  The configuration parameter
+ io.compression.codec.bzip2.library can be used to control this
+ behavior.
+
+ In the pure-Java mode, the Compressor and Decompressor interfaces
+ are not implemented.  Therefore, in that mode, those methods of
+ CompressionCodec which have a Compressor or Decompressor type
+ argument, throw UnsupportedOperationException.
+
+ Currently, support for splittability is available only in the
+ pure-Java mode; therefore, if a SplitCompressionInputStream is
+ requested, the pure-Java implementation is used, regardless of the
+ setting of the configuration parameter mentioned above.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.BZip2Codec -->
+  <!-- start class org.apache.hadoop.io.compress.CodecConstants -->
+  <class name="CodecConstants" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <field name="DEFAULT_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.DefaultCodec}.]]>
+      </doc>
+    </field>
+    <field name="BZIP2_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.BZip2Codec}.]]>
+      </doc>
+    </field>
+    <field name="GZIP_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.GzipCodec}.]]>
+      </doc>
+    </field>
+    <field name="LZ4_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.Lz4Codec}.]]>
+      </doc>
+    </field>
+    <field name="PASSTHROUGH_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for
+ {@link org.apache.hadoop.io.compress.PassthroughCodec}.]]>
+      </doc>
+    </field>
+    <field name="SNAPPY_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.SnappyCodec}.]]>
+      </doc>
+    </field>
+    <field name="ZSTANDARD_CODEC_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default extension for {@link org.apache.hadoop.io.compress.ZStandardCodec}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Codec related constants.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecConstants -->
+  <!-- start class org.apache.hadoop.io.compress.CodecPool -->
+  <class name="CodecPool" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CodecPool"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a {@link Compressor} for the given {@link CompressionCodec} from the 
+ pool or a new one.
+
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Compressor</code>
+ @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
+ @return <code>Compressor</code> for the given 
+         <code>CompressionCodec</code> from the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="getCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+    </method>
+    <method name="getDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Get a {@link Decompressor} for the given {@link CompressionCodec} from the
+ pool or a new one.
+  
+ @param codec the <code>CompressionCodec</code> for which to get the 
+              <code>Decompressor</code>
+ @return <code>Decompressor</code> for the given 
+         <code>CompressionCodec</code> the pool or a new one]]>
+      </doc>
+    </method>
+    <method name="returnCompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <doc>
+      <![CDATA[Return the {@link Compressor} to the pool.
+ 
+ @param compressor the <code>Compressor</code> to be returned to the pool]]>
+      </doc>
+    </method>
+    <method name="returnDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <doc>
+      <![CDATA[Return the {@link Decompressor} to the pool.
+ 
+ @param decompressor the <code>Decompressor</code> to be returned to the 
+                     pool]]>
+      </doc>
+    </method>
+    <method name="getLeasedCompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Compressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <method name="getLeasedDecompressorsCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codec" type="org.apache.hadoop.io.compress.CompressionCodec"/>
+      <doc>
+      <![CDATA[Return the number of leased {@link Decompressor}s for this
+ {@link CompressionCodec}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A global compressor/decompressor pool used to save and reuse 
+ (possibly native) compression/decompression codecs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CodecPool -->
+  <!-- start interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <interface name="CompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream}.
+ 
+ @param out the location for the final output stream
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionOutputStream} that will write to the given 
+ {@link OutputStream} with the given {@link Compressor}.
+ 
+ @param out the location for the final output stream
+ @param compressor compressor to use
+ @return a stream the user can write uncompressed data to have it compressed
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Compressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of compressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Compressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new compressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given
+ input stream.
+ 
+ @param in the stream to read compressed bytes from
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a {@link CompressionInputStream} that will read from the given 
+ {@link InputStream} with the given {@link Decompressor}.
+ 
+ @param in the stream to read compressed bytes from
+ @param decompressor decompressor to use
+ @return a stream to read uncompressed bytes from
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of {@link Decompressor} needed by this {@link CompressionCodec}.
+ 
+ @return the type of decompressor needed by this codec.]]>
+      </doc>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link Decompressor} for use by this {@link CompressionCodec}.
+ 
+ @return a new decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default filename extension for this kind of compression.
+ @return the extension including the '.']]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a streaming compression/decompression pair.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.CompressionCodec -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <class name="CompressionCodecFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressionCodecFactory" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Find the codecs specified in the config value io.compression.codecs 
+ and register them. Defaults to gzip and deflate.]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Print the extension map out as a string.]]>
+      </doc>
+    </method>
+    <method name="getCodecClasses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the list of codecs discovered via a Java ServiceLoader, or
+ listed in the configuration. Codecs specified in configuration come
+ later in the returned list, and are considered to override those
+ from the ServiceLoader.
+ @param conf the configuration to look in
+ @return a list of the {@link CompressionCodec} classes]]>
+      </doc>
+    </method>
+    <method name="setCodecClasses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="classes" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets a list of codec classes in the configuration. In addition to any
+ classes specified using this method, {@link CompressionCodec} classes on
+ the classpath are discovered using a Java ServiceLoader.
+ @param conf the configuration to modify
+ @param classes the list of classes to set]]>
+      </doc>
+    </method>
+    <method name="getCodec" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the given file based on its
+ filename suffix.
+ @param file the filename to check
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByClassName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="classname" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name.
+ @param classname the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecByName" return="org.apache.hadoop.io.compress.CompressionCodec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias.
+ <p>
+ Codec aliases are case insensitive.
+ <p>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec object]]>
+      </doc>
+    </method>
+    <method name="getCodecClassByName" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find the relevant compression codec for the codec's canonical class name
+ or by codec alias and returns its implemetation class.
+ <p>
+ Codec aliases are case insensitive.
+ <p>
+ The code alias is the short class name (without the package name).
+ If the short class name ends with 'Codec', then there are two aliases for
+ the codec, the complete short class name and the short class name without
+ the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+ alias are 'gzip' and 'gzipcodec'.
+
+ @param codecName the canonical class name of the codec
+ @return the codec class]]>
+      </doc>
+    </method>
+    <method name="removeSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.lang.String"/>
+      <param name="suffix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Removes a suffix from a filename, if it has it.
+ @param filename the filename to strip
+ @param suffix the suffix to remove
+ @return the shortened filename]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A little test program.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory that will find the correct codec for a given filename.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionCodecFactory -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <class name="CompressionInputStream" extends="java.io.InputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.Seekable"/>
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="CompressionInputStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a compression input stream that reads
+ the decompressed bytes from the given stream.
+ 
+ @param in The input stream to be compressed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return any IOStatistics provided by the underlying stream.
+ @return IO stats from the inner stream.]]>
+      </doc>
+    </method>
+    <method name="read" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read bytes from the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the decompressor to its initial state and discard any buffered data,
+ as the underlying stream may have been repositioned.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method returns the current position in the stream.
+
+ @return Current position in stream as a long]]>
+      </doc>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <method name="seekToNewSource" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="targetPos" type="long"/>
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[This method is current not supported.
+
+ @throws UnsupportedOperationException]]>
+      </doc>
+    </method>
+    <field name="in" type="java.io.InputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The input stream to be compressed.]]>
+      </doc>
+    </field>
+    <field name="maxAvailableData" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A compression input stream.
+
+ <p>Implementations are assumed to be buffered.  This permits clients to
+ reposition the underlying input stream then call {@link #resetState()},
+ without having to also synchronize client buffers.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionInputStream -->
+  <!-- start class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <class name="CompressionOutputStream" extends="java.io.OutputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.statistics.IOStatisticsSource"/>
+    <constructor name="CompressionOutputStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a compression output stream that writes
+ the compressed bytes to the given stream.
+ @param out]]>
+      </doc>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write compressed bytes to the stream.
+ Made abstract to prevent leakage to underlying stream.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finishes writing compressed data to the output stream 
+ without closing the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="resetState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reset the compression to the initial state. 
+ Does not reset the underlying stream.]]>
+      </doc>
+    </method>
+    <method name="getIOStatistics" return="org.apache.hadoop.fs.statistics.IOStatistics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return any IOStatistics provided by the underlying stream.
+ @return IO stats from the inner stream.]]>
+      </doc>
+    </method>
+    <field name="out" type="java.io.OutputStream"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The output stream to be compressed.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A compression output stream.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressionOutputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Compressor -->
+  <interface name="Compressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for compression. 
+ This should be called whenever #needsInput() returns 
+ <code>true</code> indicating that more input data is required.
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the input data buffer is empty and 
+ #setInput() should be called to provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ #setInput() should be called in order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary 
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="getBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of uncompressed bytes input so far.]]>
+      </doc>
+    </method>
+    <method name="getBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return number of compressed bytes output so far.]]>
+      </doc>
+    </method>
+    <method name="finish"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When called, indicates that compression should end
+ with the current contents of the input buffer.]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the end of the compressed 
+ data output stream has been reached.
+ @return <code>true</code> if the end of the compressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="compress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with compressed data. Returns actual number
+ of bytes of compressed data. A return value of 0 indicates that
+ needsInput() should be called in order to determine if more input
+ data is required.
+ 
+ @param b Buffer for the compressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of compressed data.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets compressor so that a new set of input data can be processed.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the compressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <method name="reinit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Prepare the compressor to be used in a new stream with settings defined in
+ the given Configuration
+ 
+ @param conf Configuration from which new setting are fetched]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'compressor' which can be  
+ plugged into a {@link CompressionOutputStream} to compress data.
+ This is modelled after {@link java.util.zip.Deflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Compressor -->
+  <!-- start class org.apache.hadoop.io.compress.CompressorStream -->
+  <class name="CompressorStream" extends="org.apache.hadoop.io.compress.CompressionOutputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream, org.apache.hadoop.io.compress.Compressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompressorStream" type="java.io.OutputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param out Underlying output stream.]]>
+      </doc>
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="compress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="finish"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="compressor" type="org.apache.hadoop.io.compress.Compressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.CompressorStream -->
+  <!-- start interface org.apache.hadoop.io.compress.Decompressor -->
+  <interface name="Decompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setInput"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets input data for decompression. 
+ This should be called if and only if {@link #needsInput()} returns 
+ <code>true</code> indicating that more input data is required.
+ (Both native and non-native versions of various Decompressors require
+ that the data passed in via <code>b[]</code> remain unmodified until
+ the caller is explicitly notified--via {@link #needsInput()}--that the
+ buffer may be safely modified.  With this requirement, an extra
+ buffer-copy can be avoided.)
+ 
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsInput" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called to
+ provide more input. 
+ 
+ @return <code>true</code> if the input data buffer is empty and 
+ {@link #setInput(byte[], int, int)} should be called in
+ order to provide more input.]]>
+      </doc>
+    </method>
+    <method name="setDictionary"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <doc>
+      <![CDATA[Sets preset dictionary for compression. A preset dictionary
+ is used when the history buffer can be predetermined. 
+
+ @param b Dictionary data bytes
+ @param off Start offset
+ @param len Length]]>
+      </doc>
+    </method>
+    <method name="needsDictionary" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if a preset dictionary is needed for decompression.
+ @return <code>true</code> if a preset dictionary is needed for decompression]]>
+      </doc>
+    </method>
+    <method name="finished" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns <code>true</code> if the end of the decompressed 
+ data output stream has been reached. Indicates a concatenated data stream
+ when finished() returns <code>true</code> and {@link #getRemaining()}
+ returns a positive value. finished() will be reset with the
+ {@link #reset()} method.
+ @return <code>true</code> if the end of the decompressed
+ data output stream has been reached.]]>
+      </doc>
+    </method>
+    <method name="decompress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fills specified buffer with uncompressed data. Returns actual number
+ of bytes of uncompressed data. A return value of 0 indicates that
+ {@link #needsInput()} should be called in order to determine if more
+ input data is required.
+ 
+ @param b Buffer for the uncompressed data
+ @param off Start offset of the data
+ @param len Size of the buffer
+ @return The actual number of bytes of uncompressed data.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRemaining" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of bytes remaining in the compressed data buffer.
+ Indicates a concatenated data stream if {@link #finished()} returns
+ <code>true</code> and getRemaining() returns a positive value. If
+ {@link #finished()} returns <code>true</code> and getRemaining() returns
+ a zero value, indicates that the end of data stream has been reached and
+ is not a concatenated data stream. 
+ @return The number of bytes remaining in the compressed data buffer.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resets decompressor and input and output buffers so that a new set of
+ input data can be processed. If {@link #finished()}} returns
+ <code>true</code> and {@link #getRemaining()} returns a positive value,
+ reset() is called before processing of the next data stream in the
+ concatenated data stream. {@link #finished()} will be reset and will
+ return <code>false</code> when reset() is called.]]>
+      </doc>
+    </method>
+    <method name="end"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Closes the decompressor and discards any unprocessed input.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specification of a stream-based 'de-compressor' which can be  
+ plugged into a {@link CompressionInputStream} to compress data.
+ This is modelled after {@link java.util.zip.Inflater}]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.Decompressor -->
+  <!-- start class org.apache.hadoop.io.compress.DecompressorStream -->
+  <class name="DecompressorStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream, org.apache.hadoop.io.compress.Decompressor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="DecompressorStream" type="java.io.InputStream"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Allow derived classes to directly set the underlying stream.
+ 
+ @param in Underlying input stream.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="decompress" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressedData" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="resetState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="skip" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="available" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="markSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="mark"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readlimit" type="int"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="buffer" type="byte[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="eof" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="closed" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DecompressorStream -->
+  <!-- start class org.apache.hadoop.io.compress.DefaultCodec -->
+  <class name="DefaultCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <implements name="org.apache.hadoop.io.compress.DirectDecompressionCodec"/>
+    <constructor name="DefaultCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.DefaultCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <interface name="DirectDecompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new {@link DirectDecompressor} for use by this {@link DirectDecompressionCodec}.
+ 
+ @return a new direct decompressor for use by this codec]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a codec which can decompress direct bytebuffers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressionCodec -->
+  <!-- start interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <interface name="DirectDecompressor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="decompress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="src" type="java.nio.ByteBuffer"/>
+      <param name="dst" type="java.nio.ByteBuffer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Specification of a direct ByteBuffer 'de-compressor'.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.DirectDecompressor -->
+  <!-- start class org.apache.hadoop.io.compress.GzipCodec -->
+  <class name="GzipCodec" extends="org.apache.hadoop.io.compress.DefaultCodec"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GzipCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDirectDecompressor" return="org.apache.hadoop.io.compress.DirectDecompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class creates gzip compressors/decompressors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.GzipCodec -->
+  <!-- start class org.apache.hadoop.io.compress.PassthroughCodec -->
+  <class name="PassthroughCodec" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <constructor name="PassthroughCodec"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getDefaultExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createOutputStream" return="org.apache.hadoop.io.compress.CompressionOutputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+      <param name="compressor" type="org.apache.hadoop.io.compress.Compressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createCompressor" return="org.apache.hadoop.io.compress.Compressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.CompressionInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDecompressorType" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDecompressor" return="org.apache.hadoop.io.compress.Decompressor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLASSNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Classname of the codec: {@value}.]]>
+      </doc>
+    </field>
+    <field name="OPT_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Option to control the extension of the code: {@value}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_EXTENSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This default extension is here so that if no extension has been defined,
+ some value is still returned: {@value}..]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is a special codec which does not transform the output.
+ It can be declared as a codec in the option "io.compression.codecs",
+ and then it will declare that it supports the file extension
+ set in {@link #OPT_EXTENSION}.
+
+ This allows decompression to be disabled on a job, even when there is
+ a registered/discoverable decompression codec for a file extension
+ -without having to change the standard codec binding mechanism.
+
+ For example, to disable decompression for a gzipped files, set the
+ options
+ <pre>
+   io.compression.codecs = org.apache.hadoop.io.compress.PassthroughCodec
+   io.compress.passthrough.extension = .gz
+ </pre>
+
+ <i>Note:</i> this is not a Splittable codec: it doesn't know the
+ capabilities of the passed in stream. It should be possible to
+ extend this in a subclass: the inner classes are marked as protected
+ to enable this. <i>Do not retrofit splitting to this class.</i>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.PassthroughCodec -->
+  <!-- start class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <class name="SplitCompressionInputStream" extends="org.apache.hadoop.io.compress.CompressionInputStream"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitCompressionInputStream" type="java.io.InputStream, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="setStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="start" type="long"/>
+    </method>
+    <method name="setEnd"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="end" type="long"/>
+    </method>
+    <method name="getAdjustedStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of start.
+ @return The changed value of start]]>
+      </doc>
+    </method>
+    <method name="getAdjustedEnd" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[After calling createInputStream, the values of start or end
+ might change.  So this method can be used to get the new value of end.
+ @return The changed value of end]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputStream covering a range of compressed data. The start and end
+ offsets requested by a client may be modified by the codec to fit block
+ boundaries or other algorithm-dependent requirements.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.compress.SplitCompressionInputStream -->
+  <!-- start interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+  <interface name="SplittableCompressionCodec"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.compress.CompressionCodec"/>
+    <method name="createInputStream" return="org.apache.hadoop.io.compress.SplitCompressionInputStream"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="seekableIn" type="java.io.InputStream"/>
+      <param name="decompressor" type="org.apache.hadoop.io.compress.Decompressor"/>
+      <param name="start" type="long"/>
+      <param name="end" type="long"/>
+      <param name="readMode" type="org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a stream as dictated by the readMode.  This method is used when
+ the codecs wants the ability to work with the underlying stream positions.
+
+ @param seekableIn  The seekable input stream (seeks in compressed data)
+ @param start The start offset into the compressed stream. May be changed
+              by the underlying codec.
+ @param end The end offset into the compressed stream. May be changed by
+            the underlying codec.
+ @param readMode Controls whether stream position is reported continuously
+                 from the compressed stream only only at block boundaries.
+ @return  a stream to read uncompressed bytes from]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface is meant to be implemented by those compression codecs
+ which are capable to compress / de-compress a stream starting at any
+ arbitrary position.
+
+ Especially the process of de-compressing a stream starting at some arbitrary
+ position is challenging.  Most of the codecs are only able to successfully
+ de-compress a stream, if they start from the very beginning till the end.
+ One of the reasons is the stored state at the beginning of the stream which
+ is crucial for de-compression.
+
+ Yet there are few codecs which do not save the whole state at the beginning
+ of the stream and hence can be used to de-compress stream starting at any
+ arbitrary points.  This interface is meant to be used by such codecs.  Such
+ codecs are highly valuable, especially in the context of Hadoop, because
+ an input compressed file can be split and hence can be worked on by multiple
+ machines in parallel.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.compress.SplittableCompressionCodec -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode">
+  <!-- start class org.apache.hadoop.io.erasurecode.ECSchema -->
+  <class name="ECSchema" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="ECSchema" type="java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with schema name and provided all options. Note the options may
+ contain additional information for the erasure codec to interpret further.
+ @param allOptions all schema options]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema]]>
+      </doc>
+    </constructor>
+    <constructor name="ECSchema" type="java.lang.String, int, int, java.util.Map"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with key parameters provided. Note the extraOptions may contain
+ additional information for the erasure codec to interpret further.
+ @param codecName codec name
+ @param numDataUnits number of data units used in the schema
+ @param numParityUnits number os parity units used in the schema
+ @param extraOptions extra options to configure the codec]]>
+      </doc>
+    </constructor>
+    <method name="getCodecName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the codec name
+ @return codec name]]>
+      </doc>
+    </method>
+    <method name="getExtraOptions" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get extra options specific to a erasure code.
+ @return extra options]]>
+      </doc>
+    </method>
+    <method name="getNumDataUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required data units count in a coding group
+ @return count of data units]]>
+      </doc>
+    </method>
+    <method name="getNumParityUnits" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get required parity units count in a coding group
+ @return count of parity units]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Make a meaningful string representation for log output.
+ @return string representation]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="NUM_DATA_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_PARITY_UNITS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CODEC_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Erasure coding schema to housekeeper relevant information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.erasurecode.ECSchema -->
+</package>
+<package name="org.apache.hadoop.io.erasurecode.coder.util">
+</package>
+<package name="org.apache.hadoop.io.erasurecode.grouper">
+</package>
+<package name="org.apache.hadoop.io.file.tfile">
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <class name="MetaBlockAlreadyExists" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - Meta Block with the same name already exists.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockAlreadyExists -->
+  <!-- start class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <class name="MetaBlockDoesNotExist" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Exception - No such Meta Block with the given name.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.MetaBlockDoesNotExist -->
+  <!-- start interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <interface name="RawComparable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="buffer" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying byte array.
+ 
+ @return The underlying byte array.]]>
+      </doc>
+    </method>
+    <method name="offset" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the offset of the first byte in the byte array.
+ 
+ @return The offset of the first byte in the byte array.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the size of the byte range in the byte array.
+ 
+ @return The size of the byte range in the byte array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for objects that can be compared through {@link RawComparator}.
+ This is useful in places where we need a single object reference to specify a
+ range of bytes in a byte array, such as {@link Comparable} or
+ {@link Collections#binarySearch(java.util.List, Object, Comparator)}
+ 
+ The actual comparison among RawComparable's requires an external
+ RawComparator and it is applications' responsibility to ensure two
+ RawComparable are supposed to be semantically comparable with the same
+ RawComparator.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.file.tfile.RawComparable -->
+  <!-- start class org.apache.hadoop.io.file.tfile.TFile -->
+  <class name="TFile" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="makeComparator" return="java.util.Comparator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Make a raw comparator from a string name.
+ 
+ @param name
+          Comparator name
+ @return A RawComparable comparator.]]>
+      </doc>
+    </method>
+    <method name="getSupportedCompressionAlgorithms" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get names of supported compression algorithms. The names are acceptable by
+ TFile.Writer.
+ 
+ @return Array of strings, each represents a supported compression
+         algorithm. Currently, the following compression algorithms are
+         supported.
+         <ul>
+         <li>"none" - No compression.
+         <li>"lzo" - LZO compression.
+         <li>"gz" - GZIP compression.
+         </ul>]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Dumping the TFile information.
+ 
+ @param args
+          A list of TFile paths.]]>
+      </doc>
+    </method>
+    <field name="COMPRESSION_GZ" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: gzip]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_LZO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: lzo]]>
+      </doc>
+    </field>
+    <field name="COMPRESSION_NONE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[compression: none]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_MEMCMP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator: memcmp]]>
+      </doc>
+    </field>
+    <field name="COMPARATOR_JCLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[comparator prefix: java class]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A TFile is a container of key-value pairs. Both keys and values are type-less
+ bytes. Keys are restricted to 64KB, value length is not restricted
+ (practically limited to the available disk storage). TFile further provides
+ the following features:
+ <ul>
+ <li>Block Compression.
+ <li>Named meta data blocks.
+ <li>Sorted or unsorted keys.
+ <li>Seek by key or by file offset.
+ </ul>
+ The memory footprint of a TFile includes the following:
+ <ul>
+ <li>Some constant overhead of reading or writing a compressed block.
+ <ul>
+ <li>Each compressed block requires one compression/decompression codec for
+ I/O.
+ <li>Temporary space to buffer the key.
+ <li>Temporary space to buffer the value (for TFile.Writer only). Values are
+ chunk encoded, so that we buffer at most one chunk of user data. By default,
+ the chunk buffer is 1MB. Reading chunked value does not require additional
+ memory.
+ </ul>
+ <li>TFile index, which is proportional to the total number of Data Blocks.
+ The total amount of memory needed to hold the index can be estimated as
+ (56+AvgKeySize)*NumBlocks.
+ <li>MetaBlock index, which is proportional to the total number of Meta
+ Blocks.The total amount of memory needed to hold the index for Meta Blocks
+ can be estimated as (40+AvgMetaBlockName)*NumMetaBlock.
+ </ul>
+ <p>
+ The behavior of TFile can be customized by the following variables through
+ Configuration:
+ <ul>
+ <li><b>tfile.io.chunk.size</b>: Value chunk size. Integer (in bytes). Default
+ to 1MB. Values of the length less than the chunk size is guaranteed to have
+ known value length in read time (See
+ {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}).
+ <li><b>tfile.fs.output.buffer.size</b>: Buffer size used for
+ FSDataOutputStream. Integer (in bytes). Default to 256KB.
+ <li><b>tfile.fs.input.buffer.size</b>: Buffer size used for
+ FSDataInputStream. Integer (in bytes). Default to 256KB.
+ </ul>
+ <p>
+ Suggestions on performance optimization.
+ <ul>
+ <li>Minimum block size. We recommend a setting of minimum block size between
+ 256KB to 1MB for general usage. Larger block size is preferred if files are
+ primarily for sequential access. However, it would lead to inefficient random
+ access (because there are more data to decompress). Smaller blocks are good
+ for random access, but require more memory to hold the block index, and may
+ be slower to create (because we must flush the compressor stream at the
+ conclusion of each data block, which leads to an FS I/O flush). Further, due
+ to the internal caching in Compression codec, the smallest possible block
+ size would be around 20KB-30KB.
+ <li>The current implementation does not offer true multi-threading for
+ reading. The implementation uses FSDataInputStream seek()+read(), which is
+ shown to be much faster than positioned-read call in single thread mode.
+ However, it also means that if multiple threads attempt to access the same
+ TFile (using multiple scanners) simultaneously, the actual I/O is carried out
+ sequentially even if they access different DFS blocks.
+ <li>Compression codec. Use "none" if the data is not very compressable (by
+ compressable, I mean a compression ratio at least 2:1). Generally, use "lzo"
+ as the starting point for experimenting. "gz" overs slightly better
+ compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to
+ decompress, comparing to "lzo".
+ <li>File system buffering, if the underlying FSDataInputStream and
+ FSDataOutputStream is already adequately buffered; or if applications
+ reads/writes keys and values in large buffers, we can reduce the sizes of
+ input/output buffering in TFile layer by setting the configuration parameters
+ "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size".
+ </ul>
+ 
+ Some design rationale behind TFile can be found at <a
+ href=https://issues.apache.org/jira/browse/HADOOP-3315>Hadoop-3315</a>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.TFile -->
+  <!-- start class org.apache.hadoop.io.file.tfile.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="writeVInt"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding an integer into a variable-length encoding format. Synonymous to
+ <code>Utils#writeVLong(out, n)</code>.
+ 
+ @param out
+          output stream
+ @param n
+          The integer to be encoded
+ @throws IOException
+ @see Utils#writeVLong(DataOutput, long)]]>
+      </doc>
+    </method>
+    <method name="writeVLong"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="n" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encoding a Long integer into a variable-length encoding format.
+ <ul>
+ <li>if n in [-32, 127): encode in one byte with the actual value.
+ Otherwise,
+ <li>if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52;
+ byte[1]=n&amp;0xff. Otherwise,
+ <li>if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 -
+ 88; byte[1]=(n&gt;&gt;8)&amp;0xff; byte[2]=n&amp;0xff. Otherwise,
+ <li>if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112;
+ byte[1] = (n&gt;&gt;16)&amp;0xff; byte[2] = (n&gt;&gt;8)&amp;0xff;
+ byte[3]=n&amp;0xff.
+ Otherwise:
+ <li>if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] =
+ (n&gt;&gt;24)&amp;0xff; byte[2]=(n&gt;&gt;16)&amp;0xff;
+ byte[3]=(n&gt;&gt;8)&amp;0xff; byte[4]=n&amp;0xff;
+ <li>if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] =
+ (n&gt;&gt;32)&amp;0xff; byte[2]=(n&gt;&gt;24)&amp;0xff;
+ byte[3]=(n&gt;&gt;16)&amp;0xff; byte[4]=(n&gt;&gt;8)&amp;0xff;
+ byte[5]=n&amp;0xff
+ <li>if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] =
+ (n&gt;&gt;40)&amp;0xff; byte[2]=(n&gt;&gt;32)&amp;0xff;
+ byte[3]=(n&gt;&gt;24)&amp;0xff; byte[4]=(n&gt;&gt;16)&amp;0xff;
+ byte[5]=(n&gt;&gt;8)&amp;0xff; byte[6]=n&amp;0xff;
+ <li>if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] =
+ (n&gt;&gt;48)&amp;0xff; byte[2] = (n&gt;&gt;40)&amp;0xff;
+ byte[3]=(n&gt;&gt;32)&amp;0xff; byte[4]=(n&gt;&gt;24)&amp;0xff; byte[5]=
+ (n&gt;&gt;16)&amp;0xff; byte[6]=(n&gt;&gt;8)&amp;0xff; byte[7]=n&amp;0xff;
+ <li>if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] =
+ (n&gt;&gt;54)&amp;0xff; byte[2] = (n&gt;&gt;48)&amp;0xff;
+ byte[3] = (n&gt;&gt;40)&amp;0xff; byte[4]=(n&gt;&gt;32)&amp;0xff;
+ byte[5]=(n&gt;&gt;24)&amp;0xff; byte[6]=(n&gt;&gt;16)&amp;0xff; byte[7]=
+ (n&gt;&gt;8)&amp;0xff; byte[8]=n&amp;0xff;
+ </ul>
+ 
+ @param out
+          output stream
+ @param n
+          the integer number
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readVInt" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Synonymous to
+ <code>(int)Utils#readVLong(in)</code>.
+ 
+ @param in
+          input stream
+ @return the decoded integer
+ @throws IOException
+ 
+ @see Utils#readVLong(DataInput)]]>
+      </doc>
+    </method>
+    <method name="readVLong" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decoding the variable-length integer. Suppose the value of the first byte
+ is FB, and the following bytes are NB[*].
+ <ul>
+ <li>if (FB &gt;= -32), return (long)FB;
+ <li>if (FB in [-72, -33]), return (FB+52)&lt;&lt;8 + NB[0]&amp;0xff;
+ <li>if (FB in [-104, -73]), return (FB+88)&lt;&lt;16 +
+ (NB[0]&amp;0xff)&lt;&lt;8 + NB[1]&amp;0xff;
+ <li>if (FB in [-120, -105]), return (FB+112)&lt;&lt;24 + (NB[0]&amp;0xff)
+ &lt;&lt;16 + (NB[1]&amp;0xff)&lt;&lt;8 + NB[2]&amp;0xff;
+ <li>if (FB in [-128, -121]), return interpret NB[FB+129] as a signed
+ big-endian integer.
+ </ul>
+ @param in
+          input stream
+ @return the decoded long integer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="writeString"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <param name="s" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write a String as a VInt n, followed by n Bytes as in Text format.
+ 
+ @param out
+ @param s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read a String as a VInt n, followed by n Bytes in Text format.
+ 
+ @param in
+          The input stream.
+ @return The string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <param name="cmp" type="java.util.Comparator"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @param cmp
+          Comparator for the key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="lowerBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Lower bound binary search. Find the index to the first element in the list
+ that compares greater than or equal to key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <method name="upperBound" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="list" type="java.util.List"/>
+      <param name="key" type="T"/>
+      <doc>
+      <![CDATA[Upper bound binary search. Find the index to the first element in the list
+ that compares greater than the input key.
+ 
+ @param <T>
+          Type of the input key.
+ @param list
+          The list
+ @param key
+          The input key.
+ @return The index to the desired element if it exists; or list.size()
+         otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Supporting Utility classes used by TFile, and shared by users of TFile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.file.tfile.Utils -->
+</package>
+<package name="org.apache.hadoop.io.serializer">
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <class name="JavaSerialization" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="JavaSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ An experimental {@link Serialization} for Java {@link Serializable} classes.
+ </p>
+ @see JavaSerializationComparator]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <class name="JavaSerializationComparator" extends="org.apache.hadoop.io.serializer.DeserializerComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JavaSerializationComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ A {@link RawComparator} that uses a {@link JavaSerialization}
+ {@link Deserializer} to deserialize objects that are then compared via
+ their {@link Comparable} interfaces.
+ </p>
+ @param <T>
+ @see JavaSerialization]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.JavaSerializationComparator -->
+  <!-- start class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <class name="WritableSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="WritableSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A {@link Serialization} for {@link Writable}s that delegates to
+ {@link Writable#write(java.io.DataOutput)} and
+ {@link Writable#readFields(java.io.DataInput)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.WritableSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides a mechanism for using different serialization frameworks
+in Hadoop. The property "io.serializations" defines a list of
+{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create
+{@link org.apache.hadoop.io.serializer.Serializer}s and
+{@link org.apache.hadoop.io.serializer.Deserializer}s.
+</p>
+
+<p>
+To add a new serialization framework write an implementation of
+{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the
+"io.serializations" property.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.io.serializer.avro">
+  <!-- start interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <interface name="AvroReflectSerializable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[Tag interface for Avro 'reflect' serializable classes. Classes implementing 
+ this interface can be serialized/deserialized using 
+ {@link AvroReflectSerialization}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.io.serializer.avro.AvroReflectSerializable -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <class name="AvroReflectSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroReflectSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_REFLECT_PACKAGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key to configure packages that contain classes to be serialized and 
+ deserialized using this class. Multiple packages can be specified using 
+ comma-separated list.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Serialization for Avro Reflect classes. For a class to be accepted by this 
+ serialization, it must either be in the package list configured via 
+ <code>avro.reflect.pkgs</code> or implement 
+ {@link AvroReflectSerializable} interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <class name="AvroSerialization" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.serializer.Serialization"/>
+    <constructor name="AvroSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="AVRO_SCHEMA_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class for providing serialization to Avro types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSerialization -->
+  <!-- start class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <class name="AvroSpecificSerialization" extends="org.apache.hadoop.io.serializer.avro.AvroSerialization"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AvroSpecificSerialization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Serialization for Avro Specific classes. This serialization is to be used 
+ for classes generated by Avro's 'specific' compiler.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization -->
+  <doc>
+  <![CDATA[<p>
+This package provides Avro serialization in Hadoop. This can be used to 
+serialize/deserialize Avro types in Hadoop.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for 
+serialization of classes generated by Avro's 'specific' compiler.
+</p>
+
+<p>
+Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for 
+other classes. 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for  
+any class which is either in the package list configured via 
+{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} 
+or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable}
+interface.
+</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.ipc.protocolPB">
+</package>
+<package name="org.apache.hadoop.log">
+</package>
+<package name="org.apache.hadoop.log.metrics">
+  <!-- start class org.apache.hadoop.log.metrics.EventCounter -->
+  <class name="EventCounter" extends="org.apache.log4j.AppenderSkeleton"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventCounter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="requiresLayout" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A log4J Appender that simply counts logging events in three levels:
+ fatal, error and warn. The class name is used in log4j.properties]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.log.metrics.EventCounter -->
+</package>
+<package name="org.apache.hadoop.metrics2">
+  <!-- start class org.apache.hadoop.metrics2.AbstractMetric -->
+  <class name="AbstractMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="AbstractMetric" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the metric
+ @param info  about the metric]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="value" return="java.lang.Number"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the metric
+ @return the value of the metric]]>
+      </doc>
+    </method>
+    <method name="type" return="org.apache.hadoop.metrics2.MetricType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the metric
+ @return the type of the metric]]>
+      </doc>
+    </method>
+    <method name="visit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visitor" type="org.apache.hadoop.metrics2.MetricsVisitor"/>
+      <doc>
+      <![CDATA[Accept a visitor interface
+ @param visitor of the metric]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The immutable metric]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.AbstractMetric -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <interface name="MetricsCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param name  of the record
+ @return  a {@link MetricsRecordBuilder} for the record {@code name}]]>
+      </doc>
+    </method>
+    <method name="addRecord" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <doc>
+      <![CDATA[Add a metrics record
+ @param info  of the record
+ @return  a {@link MetricsRecordBuilder} for metrics {@code info}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics collector interface]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsCollector -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsException -->
+  <class name="MetricsException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message
+ @param message for the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a message and a cause
+ @param message for the exception
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the exception with a cause
+ @param cause of the exception]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A general metrics exception wrapper]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsException -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsFilter -->
+  <class name="MetricsFilter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <constructor name="MetricsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether to accept the name
+ @param name  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Whether to accept the tag
+ @param tag to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Whether to accept the tags
+ @param tags to filter on
+ @return  true to accept; false otherwise]]>
+      </doc>
+    </method>
+    <method name="accepts" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Whether to accept the record
+ @param record  to filter on
+ @return  true to accept; false otherwise.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics filter interface. The MetricsFilter objects can be used either to
+ filter the metrics from {@link MetricsSource}s or to filter metrics per
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsFilter -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <interface name="MetricsInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically name corresponds to annotation {@link Metric#value()} or
+ the name of the class.
+ @return the name of the metric/tag]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Typically the description corresponds to annotation {@link Metric#about()}
+ or the name of the class.
+ @return the description of the metric/tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to provide immutable metainfo for metrics.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsInfo -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <class name="MetricsJsonBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsJsonBuilder" type="org.apache.hadoop.metrics2.MetricsCollector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method]]>
+      </doc>
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Build a JSON dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsJsonBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <interface name="MetricsPlugin"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+      <doc>
+      <![CDATA[Initialize the plugin
+ @param conf  the configuration object for the plugin]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The plugin interface for the metrics framework]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsPlugin -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <interface name="MetricsRecord"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="timestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the metrics
+ @return  the timestamp]]>
+      </doc>
+    </method>
+    <method name="name" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics record name]]>
+      </doc>
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the description of the metrics record]]>
+      </doc>
+    </method>
+    <method name="context" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the context name of the metrics record]]>
+      </doc>
+    </method>
+    <method name="tags" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tags of the record
+ Note: returning a collection instead of iterable as we
+ need to use tags as keys (hence Collection#hashCode etc.) in maps
+ @return an unmodifiable collection of tags]]>
+      </doc>
+    </method>
+    <method name="metrics" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the metrics of the record
+ @return an immutable iterable interface for metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An immutable snapshot of metrics with a timestamp]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsRecord -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <class name="MetricsRecordBuilder" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRecordBuilder"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a metrics value with metrics information
+ @param info  metadata of the tag
+ @param value of the tag
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+      <doc>
+      <![CDATA[Add an immutable metrics tag object
+ @param tag a pre-made tag object (potentially save an object construction)
+ @return self]]>
+      </doc>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+      <doc>
+      <![CDATA[Add a pre-made immutable metric object
+ @param metric  the pre-made metric to save an object construction
+ @return self]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the context tag
+ @param value of the context
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add an integer metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add an long metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Add a integer gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a long gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Add a float gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Add a double gauge metric
+ @param info  metadata of the metric
+ @param value of the metric
+ @return self]]>
+      </doc>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <method name="endRecord" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Syntactic sugar to add multiple records in a collector in a one liner.
+ @return the parent metrics collector object]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics record builder interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsRecordBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSink -->
+  <interface name="MetricsSink"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsPlugin"/>
+    <method name="putMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Put a metrics record in the sink
+ @param record  the record to put]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flush any buffered metrics]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics sink interface. <p>
+ Implementations of this interface consume the {@link MetricsRecord} generated
+ from {@link MetricsSource}. It registers with {@link MetricsSystem} which
+ periodically pushes the {@link MetricsRecord} to the sink using
+ {@link #putMetrics(MetricsRecord)} method.  If the implementing class also
+ implements {@link Closeable}, then the MetricsSystem will close the sink when
+ it is stopped.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSink -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSource -->
+  <interface name="MetricsSource"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collector" type="org.apache.hadoop.metrics2.MetricsCollector"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get metrics from the metrics source
+ @param collector to contain the resulting metrics snapshot
+ @param all if true, return all metrics even if unchanged.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The source of metrics information. It generates and updates metrics. It
+ registers with {@link MetricsSystem}, which periodically polls it to collect
+ {@link MetricsRecord} and passes it to {@link MetricsSink}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSource -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsSystem -->
+  <class name="MetricsSystem" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSystemMXBean"/>
+    <constructor name="MetricsSystem"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="unregisterSource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Unregister a metrics source
+ @param name of the source. This is the name you use to call register()]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source (deriving name and description from the object)
+ @param <T>   the actual type of the source object
+ @param source  object to register
+ @return  the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="source" type="T"/>
+      <doc>
+      <![CDATA[Register a metrics source
+ @param <T>   the actual type of the source object
+ @param source object to register
+ @param name  of the source. Must be unique or null (then extracted from
+              the annotations of the source object.)
+ @param desc  the description of the source (or null. See above.)
+ @return the source object
+ @exception MetricsException]]>
+      </doc>
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callback" type="org.apache.hadoop.metrics2.MetricsSystem.Callback"/>
+      <doc>
+      <![CDATA[Register a callback interface for JMX events
+ @param callback  the callback object implementing the MBean interface.]]>
+      </doc>
+    </method>
+    <method name="publishMetricsNow"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Requests an immediate publish of all metrics from sources to sinks.
+ 
+ This is a "soft" request: the expectation is that a best effort will be
+ done to synchronously snapshot the metrics from all the sources and put
+ them in all the sinks (including flushing the sinks) before returning to
+ the caller. If this can't be accomplished in reasonable time it's OK to
+ return to the caller before everything is done.]]>
+      </doc>
+    </method>
+    <method name="shutdown" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system completely (usually during server shutdown.)
+ The MetricsSystemMXBean will be unregistered.
+ @return true if shutdown completed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The metrics system interface.
+ 
+ The following components are used for metrics.
+ <ul>
+ <li>{@link MetricsSource} generate and update metrics information.</li>
+ <li>{@link MetricsSink} consume the metrics information</li>
+ </ul>
+ 
+ {@link MetricsSource} and {@link MetricsSink} register with the metrics
+ system. Implementations of {@link MetricsSystem} polls the
+ {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to
+ {@link MetricsSink}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsSystem -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <interface name="MetricsSystemMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the metrics system
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="startMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start metrics MBeans
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="stopMetricsMBeans"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop metrics MBeans.
+ Note, it doesn't stop the metrics system control MBean,
+ i.e this interface.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <method name="currentConfig" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the current config
+ Avoided getConfig, as it'll turn into a "Config" attribute,
+ which doesn't support multiple line values in jconsole.
+ @throws MetricsException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The JMX interface to the metrics system]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsSystemMXBean -->
+  <!-- start class org.apache.hadoop.metrics2.MetricsTag -->
+  <class name="MetricsTag" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsInfo"/>
+    <constructor name="MetricsTag" type="org.apache.hadoop.metrics2.MetricsInfo, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the tag with name, description and value
+ @param info  of the tag
+ @param value of the tag]]>
+      </doc>
+    </constructor>
+    <method name="name" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="description" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the tag]]>
+      </doc>
+    </method>
+    <method name="value" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value of the tag
+ @return  the value]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Immutable tag for metrics (for grouping on host/queue/username etc.)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricsTag -->
+  <!-- start class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <class name="MetricStringBuilder" extends="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricStringBuilder" type="org.apache.hadoop.metrics2.MetricsCollector, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Build an instance.
+ @param parent parent collector. Unused in this instance; only used for
+ the {@link #parent()} method
+ @param prefix string before each entry
+ @param separator separator between name and value
+ @param suffix suffix after each entry]]>
+      </doc>
+    </constructor>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="tuple" return="org.apache.hadoop.metrics2.MetricStringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add any key,val pair to the string, between the prefix and suffix,
+ separated by the separator.
+ @param key key
+ @param value value
+ @return this instance]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tag" type="org.apache.hadoop.metrics2.MetricsTag"/>
+    </method>
+    <method name="add" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.metrics2.AbstractMetric"/>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+    </method>
+    <method name="addGauge" return="org.apache.hadoop.metrics2.MetricsRecordBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+    </method>
+    <method name="parent" return="org.apache.hadoop.metrics2.MetricsCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Build a string dump of the metrics.
+
+ The {@link #toString()} operator dumps out all values collected.
+
+ Every entry is formatted as
+ {@code prefix + name + separator + value + suffix}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.MetricStringBuilder -->
+  <!-- start interface org.apache.hadoop.metrics2.MetricsVisitor -->
+  <interface name="MetricsVisitor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="float"/>
+      <doc>
+      <![CDATA[Callback for float value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="gauge"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="double"/>
+      <doc>
+      <![CDATA[Callback for double value gauges
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Callback for integer value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Callback for long value counters
+ @param info  the metric info
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A visitor interface for metrics]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.metrics2.MetricsVisitor -->
+</package>
+<package name="org.apache.hadoop.metrics2.annotation">
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metric -->
+  <class name="Metric"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a single metric used to annotate a field or a method
+ in the class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metric -->
+  <!-- start class org.apache.hadoop.metrics2.annotation.Metrics -->
+  <class name="Metrics"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Annotation interface for a group of metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.annotation.Metrics -->
+</package>
+<package name="org.apache.hadoop.metrics2.filter">
+  <!-- start class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <class name="GlobFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GlobFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A glob pattern filter for metrics.
+
+ The class name is used in metrics config files]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.GlobFilter -->
+  <!-- start class org.apache.hadoop.metrics2.filter.RegexFilter -->
+  <class name="RegexFilter" extends="org.apache.hadoop.metrics2.filter.AbstractPatternFilter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compile" return="com.google.re2j.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="s" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A regex pattern filter for metrics]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.filter.RegexFilter -->
+</package>
+<package name="org.apache.hadoop.metrics2.lib">
+  <!-- start class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <class name="DefaultMetricsSystem" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.metrics2.lib.DefaultMetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="initialize" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method to initialize the metrics system
+ @param prefix  for the metrics system configuration
+ @return the metrics system instance]]>
+      </doc>
+    </method>
+    <method name="instance" return="org.apache.hadoop.metrics2.MetricsSystem"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the metrics system object]]>
+      </doc>
+    </method>
+    <method name="shutdown"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the metrics system]]>
+      </doc>
+    </method>
+    <method name="setMiniClusterMode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="choice" type="boolean"/>
+    </method>
+    <method name="inMiniClusterMode" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The default metrics system singleton. This class is used by all the daemon
+ processes(such as NameNode, DataNode, JobTracker etc.). During daemon process
+ initialization the processes call {@link DefaultMetricsSystem#init(String)}
+ to initialize the {@link MetricsSystem}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.DefaultMetricsSystem -->
+  <!-- start class org.apache.hadoop.metrics2.lib.Interns -->
+  <class name="Interns" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Interns"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric info object.
+ @param name Name of metric info object
+ @param description Description of metric info object
+ @return an interned metric info object]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param info  of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metrics tag.
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return an interned metrics tag]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to create interned metrics info.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.Interns -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <class name="MetricsRegistry" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsRegistry" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a record name
+ @param name  of the record of the metrics]]>
+      </doc>
+    </constructor>
+    <constructor name="MetricsRegistry" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the registry with a metadata object
+ @param info  the info object for the metrics record/group]]>
+      </doc>
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the info object of the metrics registry]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.lib.MutableMetric"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a metric by name
+ @param name  of the metric
+ @return the metric object]]>
+      </doc>
+    </method>
+    <method name="getTag" return="org.apache.hadoop.metrics2.MetricsTag"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get a tag by name
+ @param name  of the tag
+ @return the tag object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newCounter" return="org.apache.hadoop.metrics2.lib.MutableCounterLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer counter
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new counter object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeInt"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeLong"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="long"/>
+      <doc>
+      <![CDATA[Create a mutable long integer gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param name  of the metric
+ @param desc  metric description
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newGauge" return="org.apache.hadoop.metrics2.lib.MutableGaugeFloat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="iVal" type="float"/>
+      <doc>
+      <![CDATA[Create a mutable float gauge
+ @param info  metadata of the metric
+ @param iVal  initial value
+ @return a new gauge object]]>
+      </doc>
+    </method>
+    <method name="newQuantiles" return="org.apache.hadoop.metrics2.lib.MutableQuantiles"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="interval" type="int"/>
+      <doc>
+      <![CDATA[Create a mutable metric that estimates quantiles of a stream of values
+ @param name of the metric
+ @param desc metric description
+ @param sampleName of the metric (e.g., "Ops")
+ @param valueName of the metric (e.g., "Time" or "Latency")
+ @param interval rollover interval of estimator in seconds
+ @return a new quantile estimator object
+ @throws MetricsException if interval is not a positive integer]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @param extended    produce extended stat (stdev, min/max etc.) if true.
+ @return a new mutable stat metric object]]>
+      </doc>
+    </method>
+    <method name="newStat" return="org.apache.hadoop.metrics2.lib.MutableStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="sampleName" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable metric with stats
+ @param name  of the metric
+ @param desc  metric description
+ @param sampleName  of the metric (e.g., "Ops")
+ @param valueName   of the metric (e.g., "Time" or "Latency")
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @return a new mutable metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric
+ @param name  of the metric
+ @param description of the metric
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRate" return="org.apache.hadoop.metrics2.lib.MutableRate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="desc" type="java.lang.String"/>
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Create a mutable rate metric (for throughput measurement)
+ @param name  of the metric
+ @param desc  description
+ @param extended  produce extended stat (stdev/min/max etc.) if true
+ @return a new mutable rate metric object]]>
+      </doc>
+    </method>
+    <method name="newRatesWithAggregation" return="org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="newMutableRollingAverages" return="org.apache.hadoop.metrics2.lib.MutableRollingAverages"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="valueName" type="java.lang.String"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add sample to a stat metric by name.
+ @param name  of the metric
+ @param value of the snapshot to add]]>
+      </doc>
+    </method>
+    <method name="setContext" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the metrics context tag
+ @param name of the context
+ @return the registry itself as a convenience]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="description" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param name  of the tag
+ @param description of the tag
+ @param value of the tag
+ @param override  existing tag if true
+ @return the registry (for keep adding tags)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+      <param name="override" type="boolean"/>
+      <doc>
+      <![CDATA[Add a tag to the metrics
+ @param info  metadata of the tag
+ @param value of the tag
+ @param override existing tag if true
+ @return the registry (for keep adding tags etc.)]]>
+      </doc>
+    </method>
+    <method name="tag" return="org.apache.hadoop.metrics2.lib.MetricsRegistry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.metrics2.MetricsInfo"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Sample all the mutable metrics and put the snapshot in the builder
+ @param builder to contain the metrics snapshot
+ @param all get all the metrics even if the values are not changed.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[An optional metrics registry class for creating and maintaining a
+ collection of MetricsMutables, making writing metrics source easier.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MetricsRegistry -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <class name="MutableCounter" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounter" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the metric value by 1.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable counter (monotonically increasing) metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounter -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <class name="MutableCounterInt" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable int counter for implementing metrics sources]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <class name="MutableCounterLong" extends="org.apache.hadoop.metrics2.lib.MutableCounter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableCounterLong" type="org.apache.hadoop.metrics2.MetricsInfo, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment the value by a delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[A mutable long counter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableCounterLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <class name="MutableGauge" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableGauge" type="org.apache.hadoop.metrics2.MetricsInfo"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="info" return="org.apache.hadoop.metrics2.MetricsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Increment the value of the metric by 1]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Decrement the value of the metric by 1]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable gauge metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGauge -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <class name="MutableGaugeInt" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="int"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="int"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable int gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeInt -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <class name="MutableGaugeLong" extends="org.apache.hadoop.metrics2.lib.MutableGauge"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[Increment by delta
+ @param delta of the increment]]>
+      </doc>
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="decr"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delta" type="long"/>
+      <doc>
+      <![CDATA[decrement by delta
+ @param delta of the decrement]]>
+      </doc>
+    </method>
+    <method name="set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of the metric
+ @param value to set]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  the value of the metric]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A mutable long gauge]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableGaugeLong -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <class name="MutableMetric" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="snapshot"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+      <doc>
+      <![CDATA[Get a snapshot of the metric
+ @param builder the metrics record builder
+ @param all if true, snapshot unchanged metrics as well]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <doc>
+      <![CDATA[Get a snapshot of metric if changed
+ @param builder the metrics record builder]]>
+      </doc>
+    </method>
+    <method name="setChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the changed flag in mutable operations]]>
+      </doc>
+    </method>
+    <method name="clearChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clear the changed flag in the snapshot operations]]>
+      </doc>
+    </method>
+    <method name="changed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return  true if metric is changed since last snapshot/snapshot]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The mutable metric interface]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableMetric -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <class name="MutableQuantiles" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableQuantiles" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiates a new {@link MutableQuantiles} for a metric that rolls itself
+ over on the specified time interval.
+ 
+ @param name
+          of the metric
+ @param description
+          long-form textual description of the metric
+ @param sampleName
+          type of items in the stream (e.g., "Ops")
+ @param valueName
+          type of the values
+ @param interval
+          rollover interval (in seconds) of the estimator]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="getInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEstimator" return="org.apache.hadoop.metrics2.util.QuantileEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the quantile estimator.
+
+ @return the quantile estimator]]>
+      </doc>
+    </method>
+    <method name="setEstimator"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="quantileEstimator" type="org.apache.hadoop.metrics2.util.QuantileEstimator"/>
+    </method>
+    <field name="quantiles" type="org.apache.hadoop.metrics2.util.Quantile[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="previousSnapshot" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Watches a stream of long values, maintaining online estimates of specific
+ quantiles with provably low error bounds. This is particularly useful for
+ accurate high-percentile (e.g. 95th, 99th) latency metrics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableQuantiles -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <class name="MutableRate" extends="org.apache.hadoop.metrics2.lib.MutableStat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[A convenient mutable metric for throughput measurement]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRate -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <class name="MutableRates" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics
+
+ This class synchronizes all accesses to the metrics it
+ contains, so it should not be used in situations where
+ there is high contention on the metrics.
+ {@link MutableRatesWithAggregation} is preferable in that
+ situation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRates -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <class name="MutableRatesWithAggregation" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableRatesWithAggregation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Initialize the registry with all the methods in a protocol
+ so they all show up in the first snapshot.
+ Convenient for JMX implementations.
+ @param protocol the protocol class]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initialize the registry with all rate names passed in.
+ This is an alternative to the above init function since this metric
+ can be used more than just for rpc name.
+ @param names the array of all rate names]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="elapsed" type="long"/>
+      <doc>
+      <![CDATA[Add a rate sample for a rate metric.
+ @param name of the rate metric
+ @param elapsed time]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rb" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="prefix" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Helper class to manage a group of mutable rate metrics.
+
+ Each thread will maintain a local rate count, and upon snapshot,
+ these values will be aggregated into a global rate. This class
+ should only be used for long running threads, as any metrics
+ produced between the last snapshot and the death of a thread
+ will be lost. This allows for significantly higher concurrency
+ than {@link MutableRates}. See HADOOP-24420.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <class name="MutableRollingAverages" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="MutableRollingAverages" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor for {@link MutableRollingAverages}.
+ @param metricValueName]]>
+      </doc>
+    </constructor>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="collectThreadLocalStates"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Collects states maintained in {@link ThreadLocal}, if any.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[@param name
+          name of metric
+ @param value
+          value of metric]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStats" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="minSamples" type="long"/>
+      <doc>
+      <![CDATA[Retrieve a map of metric name {@literal ->} (aggregate).
+ Filter out entries that don't have at least minSamples.
+
+ @return a map of peer DataNode Id to the average latency to that
+         node seen over the measurement period.]]>
+      </doc>
+    </method>
+    <method name="setRecordValidityMs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Use for test only.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class maintains a group of rolling average metrics. It implements the
+ algorithm of rolling average, i.e. a number of sliding windows are kept to
+ roll over and evict old subsets of samples. Each window has a subset of
+ samples in a stream, where sub-sum and sub-total are collected. All sub-sums
+ and sub-totals in all windows will be aggregated to final-sum and final-total
+ used to compute final average, which is called rolling average.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableRollingAverages -->
+  <!-- start class org.apache.hadoop.metrics2.lib.MutableStat -->
+  <class name="MutableStat" extends="org.apache.hadoop.metrics2.lib.MutableMetric"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a sample statistics metric
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")
+ @param extended    create extended stats (stdev, min/max etc.) by default.]]>
+      </doc>
+    </constructor>
+    <constructor name="MutableStat" type="java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a snapshot stat metric with extended stat off by default
+ @param name        of the metric
+ @param description of the metric
+ @param sampleName  of the metric (e.g. "Ops")
+ @param valueName   of the metric (e.g. "Time", "Latency")]]>
+      </doc>
+    </constructor>
+    <method name="setExtended"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="extended" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to display the extended stats (stdev, min/max etc.) or not
+ @param extended enable/disable displaying extended stats]]>
+      </doc>
+    </method>
+    <method name="setUpdateTimeStamp"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateTimeStamp" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to update the snapshot time or not.
+ @param updateTimeStamp enable update stats snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numSamples" type="long"/>
+      <param name="sum" type="long"/>
+      <doc>
+      <![CDATA[Add a number of samples and their sum to the running stat
+
+ Note that although use of this method will preserve accurate mean values,
+ large values for numSamples may result in inaccurate variance values due
+ to the use of a single step of the Welford method for variance calculation.
+ @param numSamples  number of samples
+ @param sum of the samples]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a snapshot to the metric.
+ @param value of the metric]]>
+      </doc>
+    </method>
+    <method name="snapshot"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="org.apache.hadoop.metrics2.MetricsRecordBuilder"/>
+      <param name="all" type="boolean"/>
+    </method>
+    <method name="lastStat" return="org.apache.hadoop.metrics2.util.SampleStat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a SampleStat object that supports
+ calls like StdDev and Mean.
+ @return SampleStat]]>
+      </doc>
+    </method>
+    <method name="resetMinMax"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reset the all time min max of the metric]]>
+      </doc>
+    </method>
+    <method name="getSnapshotTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the SampleStat snapshot timestamp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A mutable metric with stats.
+
+ Useful for keeping throughput/latency stats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.lib.MutableStat -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink">
+  <!-- start class org.apache.hadoop.metrics2.sink.FileSink -->
+  <class name="FileSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="FileSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a file]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.FileSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <class name="GraphiteSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="GraphiteSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes to a Graphite server]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.GraphiteSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <class name="RollingFileSystemSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="RollingFileSystemSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty instance.  Required for reflection.]]>
+      </doc>
+    </constructor>
+    <constructor name="RollingFileSystemSink" type="long, long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance for testing.
+
+ @param flushIntervalMillis the roll interval in millis
+ @param flushOffsetIntervalMillis the roll offset interval in millis]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics2Properties" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="getRollInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Extract the roll interval from the configuration and return it in
+ milliseconds.
+
+ @return the roll interval in millis]]>
+      </doc>
+    </method>
+    <method name="updateFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Update the {@link #nextFlush} variable to the next flush time. Add
+ an integer number of flush intervals, preserving the initial random offset.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="setInitialFlushTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="now" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Set the {@link #nextFlush} variable to the initial flush time. The initial
+ flush will be an integer number of flush intervals past the beginning of
+ the current hour and will have a random offset added, up to
+ {@link #rollOffsetIntervalMillis}. The initial flush will be a time in
+ past that can be used from which to calculate future flush times.
+
+ @param now the current time]]>
+      </doc>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="source" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ignoreError" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allowAppend" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="basePath" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rollOffsetIntervalMillis" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nextFlush" type="java.util.Calendar"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="forceFlush" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="hasFlushed" type="boolean"
+      transient="false" volatile="true"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedConf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="suppliedFilesystem" type="org.apache.hadoop.fs.FileSystem"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>This class is a metrics sink that uses
+ {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs.  Every
+ roll interval a new directory will be created under the path specified by the
+ <code>basepath</code> property. All metrics will be logged to a file in the
+ current interval's directory in a file named &lt;hostname&gt;.log, where
+ &lt;hostname&gt; is the name of the host on which the metrics logging
+ process is running. The base path is set by the
+ <code>&lt;prefix&gt;.sink.&lt;instance&gt;.basepath</code> property.  The
+ time zone used to create the current interval's directory name is GMT.  If
+ the <code>basepath</code> property isn't specified, it will default to
+ &quot;/tmp&quot;, which is the temp directory on whatever default file
+ system is configured for the cluster.</p>
+
+ <p>The <code>&lt;prefix&gt;.sink.&lt;instance&gt;.ignore-error</code>
+ property controls whether an exception is thrown when an error is encountered
+ writing a log file.  The default value is <code>true</code>.  When set to
+ <code>false</code>, file errors are quietly swallowed.</p>
+
+ <p>The <code>roll-interval</code> property sets the amount of time before
+ rolling the directory. The default value is 1 hour. The roll interval may
+ not be less than 1 minute. The property's value should be given as
+ <i>number unit</i>, where <i>number</i> is an integer value, and
+ <i>unit</i> is a valid unit.  Valid units are <i>minute</i>, <i>hour</i>,
+ and <i>day</i>.  The units are case insensitive and may be abbreviated or
+ plural. If no units are specified, hours are assumed. For example,
+ &quot;2&quot;, &quot;2h&quot;, &quot;2 hour&quot;, and
+ &quot;2 hours&quot; are all valid ways to specify two hours.</p>
+
+ <p>The <code>roll-offset-interval-millis</code> property sets the upper
+ bound on a random time interval (in milliseconds) that is used to delay
+ before the initial roll.  All subsequent rolls will happen an integer
+ number of roll intervals after the initial roll, hence retaining the original
+ offset. The purpose of this property is to insert some variance in the roll
+ times so that large clusters using this sink on every node don't cause a
+ performance impact on HDFS by rolling simultaneously.  The default value is
+ 30000 (30s).  When writing to HDFS, as a rule of thumb, the roll offset in
+ millis should be no less than the number of sink instances times 5.
+
+ <p>The primary use of this class is for logging to HDFS.  As it uses
+ {@link org.apache.hadoop.fs.FileSystem} to access the target file system,
+ however, it can be used to write to the local file system, Amazon S3, or any
+ other supported file system.  The base path for the sink will determine the
+ file system used.  An unqualified path will write to the default file system
+ set by the configuration.</p>
+
+ <p>Not all file systems support the ability to append to files.  In file
+ systems without the ability to append to files, only one writer can write to
+ a file at a time.  To allow for concurrent writes from multiple daemons on a
+ single host, the <code>source</code> property is used to set unique headers
+ for the log files.  The property should be set to the name of
+ the source daemon, e.g. <i>namenode</i>.  The value of the
+ <code>source</code> property should typically be the same as the property's
+ prefix.  If this property is not set, the source is taken to be
+ <i>unknown</i>.</p>
+
+ <p>Instead of appending to an existing file, by default the sink
+ will create a new file with a suffix of &quot;.&lt;n&gt;&quot;, where
+ <i>n</i> is the next lowest integer that isn't already used in a file name,
+ similar to the Hadoop daemon logs.  NOTE: the file with the <b>highest</b>
+ sequence number is the <b>newest</b> file, unlike the Hadoop daemon logs.</p>
+
+ <p>For file systems that allow append, the sink supports appending to the
+ existing file instead. If the <code>allow-append</code> property is set to
+ true, the sink will instead append to the existing file on file systems that
+ support appends. By default, the <code>allow-append</code> property is
+ false.</p>
+
+ <p>Note that when writing to HDFS with <code>allow-append</code> set to true,
+ there is a minimum acceptable number of data nodes.  If the number of data
+ nodes drops below that minimum, the append will succeed, but reading the
+ data will fail with an IOException in the DataStreamer class.  The minimum
+ number of data nodes required for a successful append is generally 2 or
+ 3.</p>
+
+ <p>Note also that when writing to HDFS, the file size information is not
+ updated until the file is closed (at the end of the interval) even though
+ the data is being written successfully. This is a known HDFS limitation that
+ exists because of the performance cost of updating the metadata.  See
+ <a href="https://issues.apache.org/jira/browse/HDFS-5478">HDFS-5478</a>.</p>
+
+ <p>When using this sink in a secure (Kerberos) environment, two additional
+ properties must be set: <code>keytab-key</code> and
+ <code>principal-key</code>. <code>keytab-key</code> should contain the key by
+ which the keytab file can be found in the configuration, for example,
+ <code>yarn.nodemanager.keytab</code>. <code>principal-key</code> should
+ contain the key by which the principal can be found in the configuration,
+ for example, <code>yarn.nodemanager.principal</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.RollingFileSystemSink -->
+  <!-- start class org.apache.hadoop.metrics2.sink.StatsDSink -->
+  <class name="StatsDSink" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.metrics2.MetricsSink"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="StatsDSink"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.commons.configuration2.SubsetConfiguration"/>
+    </method>
+    <method name="putMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+    </method>
+    <method name="writeMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="line" type="java.lang.String"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A metrics sink that writes metrics to a StatsD daemon.
+ This sink will produce metrics of the form
+ '[hostname].servicename.context.name.metricname:value|type'
+ where hostname is optional. This is useful when sending to
+ a daemon that is running on the localhost and will add the
+ hostname to the metric (such as the
+ <a href="https://collectd.org/">CollectD</a> StatsD plugin).
+ <br>
+ To configure this plugin, you will need to add the following
+ entries to your hadoop-metrics2.properties file:
+ <br>
+ <pre>
+ *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.sink.StatsDSink -->
+</package>
+<package name="org.apache.hadoop.metrics2.sink.ganglia">
+</package>
+<package name="org.apache.hadoop.metrics2.source">
+</package>
+<package name="org.apache.hadoop.metrics2.util">
+  <!-- start class org.apache.hadoop.metrics2.util.MBeans -->
+  <class name="MBeans" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ Where the {@literal <serviceName> and <nameName>} are the supplied
+ parameters.
+
+ @param serviceName
+ @param nameName
+ @param theMbean - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="register" return="javax.management.ObjectName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceName" type="java.lang.String"/>
+      <param name="nameName" type="java.lang.String"/>
+      <param name="properties" type="java.util.Map"/>
+      <param name="theMbean" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Register the MBean using our standard MBeanName format
+ "hadoop:service={@literal <serviceName>,name=<nameName>}"
+ Where the {@literal <serviceName> and <nameName>} are the supplied
+ parameters.
+
+ @param serviceName
+ @param nameName
+ @param properties - Key value pairs to define additional JMX ObjectName
+                     properties.
+ @param theMbean    - the MBean to register
+ @return the named used to register the MBean]]>
+      </doc>
+    </method>
+    <method name="getMbeanNameService" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="getMbeanNameName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="objectName" type="javax.management.ObjectName"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mbeanName" type="javax.management.ObjectName"/>
+    </method>
+    <doc>
+    <![CDATA[This util class provides a method to register an MBean using
+ our standard naming convention as described in the doc
+  for {link {@link #register(String, String, Object)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MBeans -->
+  <!-- start class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <class name="MetricsCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MetricsCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="MetricsCache" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a metrics cache
+ @param maxRecsPerName  limit of the number records per record name]]>
+      </doc>
+    </constructor>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <param name="includingTags" type="boolean"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cached record
+ @param mr the update record
+ @param includingTags cache tag values (for later lookup by name) if true
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="update" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mr" type="org.apache.hadoop.metrics2.MetricsRecord"/>
+      <doc>
+      <![CDATA[Update the cache and return the current cache record
+ @param mr the update record
+ @return the updated cache record]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.metrics2.util.MetricsCache.Record"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="tags" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Get the cached record
+ @param name of the record
+ @param tags of the record
+ @return the cached record or null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A metrics cache for sinks that don't support sparse updates.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.MetricsCache -->
+  <!-- start class org.apache.hadoop.metrics2.util.Servers -->
+  <class name="Servers" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parse" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="specs" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Parses a space and/or comma separated sequence of server specifications
+ of the form <i>hostname</i> or <i>hostname:port</i>.  If
+ the specs string is null, defaults to localhost:defaultPort.
+
+ @param specs   server specs (see description)
+ @param defaultPort the default port if not specified
+ @return a list of InetSocketAddress objects.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Helpers to handle server addresses]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.metrics2.util.Servers -->
+</package>
+<package name="org.apache.hadoop.net">
+  <!-- start class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <class name="AbstractDNSToSwitchMapping" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.net.DNSToSwitchMapping"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="AbstractDNSToSwitchMapping"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an unconfigured instance]]>
+      </doc>
+    </constructor>
+    <constructor name="AbstractDNSToSwitchMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance, caching the configuration file.
+ This constructor does not call {@link #setConf(Configuration)}; if
+ a subclass extracts information in that method, it must call it explicitly.
+ @param conf the configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate that indicates that the switch mapping is known to be
+ single-switch. The base class returns false: it assumes all mappings are
+ multi-rack. Subclasses may override this with methods that are more aware
+ of their topologies.
+
+ <p>
+
+ This method is used when parts of Hadoop need know whether to apply
+ single rack vs multi-rack policies, such as during block placement.
+ Such algorithms behave differently if they are on multi-switch systems.
+ </p>
+
+ @return true if the mapping thinks that it is on a single switch]]>
+      </doc>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a copy of the map (for diagnostics)
+ @return a clone of the map or null for none known]]>
+      </doc>
+    </method>
+    <method name="dumpTopology" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a string listing the switch mapping implementation,
+ the mapping for every known node and the number of nodes and
+ unique switches known about -each entry to a separate line.
+ @return a string that can be presented to the ops team or used in
+ debug messages.]]>
+      </doc>
+    </method>
+    <method name="isSingleSwitchByScriptPolicy" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMappingSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapping" type="org.apache.hadoop.net.DNSToSwitchMapping"/>
+      <doc>
+      <![CDATA[Query for a {@link DNSToSwitchMapping} instance being on a single
+ switch.
+ <p>
+ This predicate simply assumes that all mappings not derived from
+ this class are multi-switch.
+ @param mapping the mapping to query
+ @return true if the base class says it is single switch, or the mapping
+ is not derived from this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a base class for DNS to Switch mappings. <p> It is not mandatory to
+ derive {@link DNSToSwitchMapping} implementations from it, but it is strongly
+ recommended, as it makes it easy for the Hadoop developers to add new methods
+ to this base class that are automatically picked up by all implementations.
+ <p>
+
+ This class does not extend the <code>Configured</code>
+ base class, and should not be changed to do so, as it causes problems
+ for subclasses. The constructor of the <code>Configured</code> calls
+ the  {@link #setConf(Configuration)} method, which will call into the
+ subclasses before they have been fully constructed.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.AbstractDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <class name="CachedDNSToSwitchMapping" extends="org.apache.hadoop.net.AbstractDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CachedDNSToSwitchMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[cache a raw DNS mapping
+ @param rawMapping the raw mapping to cache]]>
+      </doc>
+    </constructor>
+    <method name="resolve" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <method name="getSwitchMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the (host x switch) map.
+ @return a copy of the cached map of hosts to rack]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isSingleSwitch" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegate the switch topology query to the raw mapping, via
+ {@link AbstractDNSToSwitchMapping#isMappingSingleSwitch(DNSToSwitchMapping)}
+ @return true iff the raw mapper is considered single-switch.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+    </method>
+    <field name="rawMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The uncached mapping]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A cached implementation of DNSToSwitchMapping that takes an
+ raw DNSToSwitchMapping and stores the resolved network location in 
+ a cache. The following calls to a resolved network location
+ will get its location from the cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.CachedDNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ConnectTimeoutException -->
+  <class name="ConnectTimeoutException" extends="java.net.SocketTimeoutException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConnectTimeoutException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Thrown by {@link NetUtils#connect(java.net.Socket, java.net.SocketAddress, int)}
+ if it times out while connecting to the remote host.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ConnectTimeoutException -->
+  <!-- start interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <interface name="DNSToSwitchMapping"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="resolve" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Resolves a list of DNS-names/IP-addresses and returns back a list of
+ switch information (network paths). One-to-one correspondence must be 
+ maintained between the elements in the lists. 
+ Consider an element in the argument list - x.y.com. The switch information
+ that is returned must be a network path of the form /foo/rack, 
+ where / is the root, and 'foo' is the switch where 'rack' is connected.
+ Note the hostname/ip-address is not part of the returned path.
+ The network topology of the cluster would determine the number of
+ components in the network path.
+ <p>
+
+ If a name cannot be resolved to a rack, the implementation
+ should return {@link NetworkTopology#DEFAULT_RACK}. This
+ is what the bundled implementations do, though it is not a formal requirement
+
+ @param names the list of hosts to resolve (can be empty)
+ @return list of resolved network paths.
+ If <i>names</i> is empty, the returned list is also empty]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reload all of the cached mappings.
+
+ If there is a cache, this method will clear it, so that future accesses
+ will get a chance to see the new data.]]>
+      </doc>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="names" type="java.util.List"/>
+      <doc>
+      <![CDATA[Reload cached mappings on specific nodes.
+
+ If there is a cache on these nodes, this method will clear it, so that 
+ future accesses will see updated data.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface that must be implemented to allow pluggable
+ DNS-name/IP-address to RackID resolvers.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.net.DNSToSwitchMapping -->
+  <!-- start class org.apache.hadoop.net.ScriptBasedMapping -->
+  <class name="ScriptBasedMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ScriptBasedMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with the default configuration.
+ <p>
+ Calling {@link #setConf(Configuration)} will trigger a
+ re-evaluation of the configuration settings and so be used to
+ set up the mapping script.]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.net.DNSToSwitchMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given raw mapping
+ @param rawMap raw DNSTOSwithMapping]]>
+      </doc>
+    </constructor>
+    <constructor name="ScriptBasedMapping" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance from the given configuration
+ @param conf configuration]]>
+      </doc>
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ This will get called in the superclass constructor, so a check is needed
+ to ensure that the raw mapping is defined before trying to relaying a null
+ configuration.
+ @param conf]]>
+      </doc>
+    </method>
+    <field name="NO_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Text used in the {@link #toString()} method if there is no string
+ {@value}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class implements the {@link DNSToSwitchMapping} interface using a 
+ script configured via the
+ {@link CommonConfigurationKeys#NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY} option.
+ <p>
+ It contains a static class <code>RawScriptBasedMapping</code> that performs
+ the work: reading the configuration parameters, executing any defined
+ script, handling errors and such like. The outer
+ class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ queries.
+ <p>
+ This DNS mapper's {@link #isSingleSwitch()} predicate returns
+ true if and only if a script is defined.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.ScriptBasedMapping -->
+  <!-- start class org.apache.hadoop.net.SocksSocketFactory -->
+  <class name="SocksSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="SocksSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <constructor name="SocksSocketFactory" type="java.net.Proxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor with a supplied Proxy
+ 
+ @param proxy the proxy to use to create sockets]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.SocksSocketFactory -->
+  <!-- start class org.apache.hadoop.net.StandardSocketFactory -->
+  <class name="StandardSocketFactory" extends="javax.net.SocketFactory"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StandardSocketFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default empty constructor (for use with the reflection API).]]>
+      </doc>
+    </constructor>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="createSocket" return="java.net.Socket"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="localHostAddr" type="java.net.InetAddress"/>
+      <param name="localPort" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="UnknownHostException" type="java.net.UnknownHostException"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Specialized SocketFactory to create sockets with a SOCKS proxy]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.StandardSocketFactory -->
+  <!-- start class org.apache.hadoop.net.TableMapping -->
+  <class name="TableMapping" extends="org.apache.hadoop.net.CachedDNSToSwitchMapping"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TableMapping"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="reloadCachedMappings"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text
+ file. The columns are separated by whitespace. The first column is a DNS or
+ IP address and the second column specifies the rack where the address maps.
+ </p>
+ <p>
+ This class uses the configuration parameter {@code
+ net.topology.table.file.name} to locate the mapping file.
+ </p>
+ <p>
+ Calls to {@link #resolve(List)} will look up the address as defined in the
+ mapping file. If no entry corresponding to the address is found, the value
+ {@code /default-rack} is returned.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.net.TableMapping -->
+</package>
+<package name="org.apache.hadoop.net.unix">
+</package>
+<package name="org.apache.hadoop.security">
+  <!-- start class org.apache.hadoop.security.AccessControlException -->
+  <class name="AccessControlException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessControlException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor is needed for unwrapping from 
+ {@link org.apache.hadoop.ipc.RemoteException}.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs an {@link AccessControlException}
+ with the specified detail message.
+ @param s the detail message.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[An exception class for access control related issues.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.AccessControlException -->
+  <!-- start class org.apache.hadoop.security.Credentials -->
+  <class name="Credentials" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty credentials instance.]]>
+      </doc>
+    </constructor>
+    <constructor name="Credentials" type="org.apache.hadoop.security.Credentials"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a copy of the given credentials.
+ @param credentials to copy]]>
+      </doc>
+    </constructor>
+    <method name="getToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the Token object for the alias.
+ @param alias the alias for the Token
+ @return token for this alias]]>
+      </doc>
+    </method>
+    <method name="addToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="t" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token in the storage (in memory).
+ @param alias the alias for the key
+ @param t the token object]]>
+      </doc>
+    </method>
+    <method name="getAllTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the tokens in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getTokenMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to Tokens.]]>
+      </doc>
+    </method>
+    <method name="numberOfTokens" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of Tokens in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="getSecretKey" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Returns the key bytes for the alias.
+ @param alias the alias for the key
+ @return key for this alias]]>
+      </doc>
+    </method>
+    <method name="numberOfSecretKeys" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of keys in the in-memory map]]>
+      </doc>
+    </method>
+    <method name="addSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Set the key for an alias.
+ @param alias the alias for the key
+ @param key the key bytes]]>
+      </doc>
+    </method>
+    <method name="removeSecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Remove the key for a given alias.
+ @param alias the alias for the key]]>
+      </doc>
+    </method>
+    <method name="getAllSecretKeys" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all the secret key entries in the in-memory map.]]>
+      </doc>
+    </method>
+    <method name="getSecretKeyMap" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an unmodifiable version of the full map of aliases to secret keys.]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageFile" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="java.io.File"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token storage file and loading its Tokens.
+ @param filename
+ @param conf
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readTokenStorageStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convenience method for reading a token from a DataInputStream.]]>
+      </doc>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageToStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="os" type="java.io.DataOutputStream"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="writeTokenStorageFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="format" type="org.apache.hadoop.security.Credentials.SerializedFormat"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Stores all the keys to DataOutput.
+ @param out
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Loads all the keys.
+ @param in
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <method name="mergeAll"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Copy all of the credentials from one credential object into another.
+ Existing secrets and tokens are not overwritten.
+ @param other the credentials to copy]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that provides the facilities of reading and writing
+ secret keys and Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.Credentials -->
+  <!-- start interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <interface name="GroupMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getGroups" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all various group memberships of a given user.
+ Returns EMPTY list in case of non-existing user
+ @param user User's name
+ @return group memberships of user
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsRefresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the cache of groups and user mapping
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cacheGroupsAdd"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groups" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Caches the group user information
+ @param groups list of groups to add to cache
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="GROUP_MAPPING_CONFIG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An interface for the implementation of a user-to-groups mapping service
+ used by {@link Groups}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.GroupMappingServiceProvider -->
+  <!-- start interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <interface name="IdMappingServiceProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getUid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getGid" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getGroupName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="gid" type="int"/>
+      <param name="unknown" type="java.lang.String"/>
+    </method>
+    <method name="getUidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getGidAllowingUnknown" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[An interface for the implementation of {@literal <}userId,
+ userName{@literal >} mapping and {@literal <}groupId, groupName{@literal >}
+ mapping.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.IdMappingServiceProvider -->
+  <!-- start class org.apache.hadoop.security.KerberosAuthException -->
+  <class name="KerberosAuthException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosAuthException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="KerberosAuthException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="u" type="java.lang.String"/>
+    </method>
+    <method name="setPrincipal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+    </method>
+    <method name="setKeytabFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="java.lang.String"/>
+    </method>
+    <method name="setTicketCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.String"/>
+    </method>
+    <method name="getInitialMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The initial message, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getKeytabFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The keytab file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The principal, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getTicketCacheFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The ticket cache file path, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The user, or null if not set.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Thrown when {@link UserGroupInformation} failed with an unrecoverable error,
+ such as failure in kerberos login/logout, invalid subject etc.
+
+ Caller should not retry when catching this exception.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.KerberosAuthException -->
+  <!-- start class org.apache.hadoop.security.SecurityUtil -->
+  <class name="SecurityUtil" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOriginalTGT" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ticket" type="javax.security.auth.kerberos.KerberosTicket"/>
+      <doc>
+      <![CDATA[Check whether the server principal is the TGS's principal
+ @param ticket the original TGT (the ticket that is obtained when a 
+ kinit is done)
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal
+ names. It replaces hostname pattern with hostname, which should be
+ fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
+ dynamically looked-up fqdn of the current host instead.
+ 
+ @param principalConfig
+          the Kerberos principal name conf value to convert
+ @param hostname
+          the fully-qualified domain name used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="getServerPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalConfig" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Convert Kerberos principal name pattern to valid Kerberos principal names.
+ This method is similar to {@link #getServerPrincipal(String, String)},
+ except 1) the reverse DNS lookup from addr to hostname is done only when
+ necessary, 2) param addr can't be null (no default behavior of using local
+ hostname when addr is null).
+ 
+ @param principalConfig
+          Kerberos principal name pattern to convert
+ @param addr
+          InetAddress of the host used for substitution
+ @return converted Kerberos principal name
+ @throws IOException if the client address cannot be determined]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in
+ user's Kerberos principal name with a dynamically looked-up fully-qualified
+ domain name of the current host.
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="login"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="keytabFileKey" type="java.lang.String"/>
+      <param name="userNameKey" type="java.lang.String"/>
+      <param name="hostname" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Login as a principal specified in config. Substitute $host in user's Kerberos principal 
+ name with hostname. If non-secure mode - return. If no keytab available -
+ bail out with an exception
+ 
+ @param conf
+          conf to use
+ @param keytabFileKey
+          the key to look for keytab file in conf
+ @param userNameKey
+          the key to look for user's Kerberos principal name in conf
+ @param hostname
+          hostname to use for substitution
+ @throws IOException if the config doesn't specify a keytab]]>
+      </doc>
+    </method>
+    <method name="buildDTServiceName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <param name="defPort" type="int"/>
+      <doc>
+      <![CDATA[create the service name for a Delegation token
+ @param uri of the service
+ @param defPort is used if the uri lacks a port
+ @return the token service, or null if no authority
+ @see #buildTokenService(InetSocketAddress)]]>
+      </doc>
+    </method>
+    <method name="getHostFromPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="principalName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the host name from the principal name of format {@literal <}service
+ {@literal >}/host@realm.
+ @param principalName principal name of format as described above
+ @return host name if the the string conforms to the above format, else null]]>
+      </doc>
+    </method>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the KerberosInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return the KerberosInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getClientPrincipal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the client principal for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol the protocol class to get the information for
+ @param conf configuration object
+ @return client principal or null if it has no client principal defined.]]>
+      </doc>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Look up the TokenInfo for a given protocol. It searches all known
+ SecurityInfo providers.
+ @param protocol The protocol class to get the information for.
+ @param conf Configuration object
+ @return the TokenInfo or null if it has no KerberosInfo defined]]>
+      </doc>
+    </method>
+    <method name="getTokenServiceAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Decode the given token's service field into an InetAddress
+ @param token from which to obtain the service
+ @return InetAddress for the service]]>
+      </doc>
+    </method>
+    <method name="setTokenService"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Set the given token's service to the format expected by the RPC client 
+ @param token a delegation token
+ @param addr the socket for the rpc connection]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param addr InetSocketAddress of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="buildTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Construct the service key for a token
+ @param uri of remote connection with a token
+ @return "ip:port" or "host:port" depending on the value of
+          hadoop.security.token.service.use_ip]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUserOrFatal" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If the login
+ user cannot be determined, this will log a FATAL error and exit
+ the whole JVM.]]>
+      </doc>
+    </method>
+    <method name="doAsLoginUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's login user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="doAsCurrentUser" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Perform the given action as the daemon's current user. If an
+ InterruptedException is thrown, it is converted to an IOException.
+
+ @param action the action to perform
+ @return the result of the action
+ @throws IOException in the event of error]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticationMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isPrivilegedPort" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+    </method>
+    <method name="getZKAuthInfos" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="configKey" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility method to fetch ZK auth info from the configuration.
+ @throws java.io.IOException if the Zookeeper ACLs configuration file
+ cannot be read
+ @throws ZKUtil.BadAuthFormatException if the auth format is invalid]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HOSTNAME_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED_TO_GET_UGI_MSG_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Security Utils.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.SecurityUtil -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation -->
+  <class name="UserGroupInformation" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="setShouldRenewImmediatelyForTests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="immediate" type="boolean"/>
+      <doc>
+      <![CDATA[For the purposes of unit tests, we want to test login
+ from keytab and don't want to wait until the renew
+ window (controlled by TICKET_RENEW_WINDOW).
+ @param immediate true if we should login without waiting for ticket window]]>
+      </doc>
+    </method>
+    <method name="reattachMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reattach the class's metrics to a new metric system.]]>
+      </doc>
+    </method>
+    <method name="isInitialized" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the static configuration for UGI.
+ In particular, set the security authentication mechanism and the
+ group look up service.
+ @param conf the configuration to use]]>
+      </doc>
+    </method>
+    <method name="isSecurityEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Determine if UserGroupInformation is using Kerberos to determine
+ user identities or is relying on simple authentication
+ 
+ @return true if UGI is working in a secure environment]]>
+      </doc>
+    </method>
+    <method name="hasKerberosCredentials" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[checks if logged in using kerberos
+ @return true if the subject logged via keytab or has a Kerberos TGT]]>
+      </doc>
+    </method>
+    <method name="getCurrentUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the current user, including any doAs in the current stack.
+ @return the current user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="getBestUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCachePath" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Find the most appropriate UserGroupInformation to use
+
+ @param ticketCachePath    The Kerberos ticket cache path, or NULL
+                           if none is specfied
+ @param user               The user name, or NULL if none is specified.
+
+ @return                   The most appropriate UserGroupInformation]]>
+      </doc>
+    </method>
+    <method name="getUGIFromTicketCache" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ticketCache" type="java.lang.String"/>
+      <param name="user" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Kerberos ticket cache.
+ 
+ @param user                The principal name to load from the ticket
+                            cache
+ @param ticketCache     the path to the ticket cache file
+
+ @throws IOException        if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getUGIFromSubject" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a UserGroupInformation from a Subject with Kerberos principal.
+
+ @param subject             The KerberosPrincipal to use in UGI.
+                            The creator of subject is responsible for
+                            renewing credentials.
+
+ @throws IOException
+ @throws KerberosAuthException if the kerberos login fails]]>
+      </doc>
+    </method>
+    <method name="getLoginUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the currently logged in user.  If no explicit login has occurred,
+ the user will automatically be logged in with either kerberos credentials
+ if available, or as the local OS user, based on security settings.
+ @return the logged in user
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="trimLoginMethod" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[remove the login method that is followed by a space from the username
+ e.g. "jack (auth:SIMPLE)" {@literal ->} "jack"
+
+ @param userName
+ @return userName without login method]]>
+      </doc>
+    </method>
+    <method name="loginUserFromSubject"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subject" type="javax.security.auth.Subject"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log in a user using the given subject
+ @param subject the subject to use when logging in a user, or null to
+ create a new subject.
+
+ If subject is not null, the creator of subject is responsible for renewing
+ credentials.
+
+ @throws IOException if login fails]]>
+      </doc>
+    </method>
+    <method name="isFromKeytab" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is this user logged in from a keytab file managed by the UGI?
+ @return true if the credentials are from a keytab file.]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="logoutUserFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log the current user out who previously logged in using keytab.
+ This method assumes that the user logged in by calling
+ {@link #loginUserFromKeytab(String, String)}.
+
+ @throws IOException
+ @throws KerberosAuthException if a failure occurred in logout,
+ or if the user did not log in by invoking loginUserFromKeyTab() before.]]>
+      </doc>
+    </method>
+    <method name="checkTGTAndReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-login a user from keytab if TGT is expired or is close to expiry.
+ 
+ @throws IOException
+ @throws KerberosAuthException if it's a kerberos login exception.]]>
+      </doc>
+    </method>
+    <method name="reloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from a keytab file. Loads a user identity from a keytab
+ file and logs them in. They become the currently logged-in user. This
+ method assumes that {@link #loginUserFromKeytab(String, String)} had
+ happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="forceReloginFromKeytab"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Force re-Login a user in from a keytab file irrespective of the last login
+ time. Loads a user identity from a keytab file and logs them in. They
+ become the currently logged-in user. This method assumes that
+ {@link #loginUserFromKeytab(String, String)} had happened already. The
+ Subject field of this UserGroupInformation object is updated to have the
+ new credentials.
+
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="reloginFromTicketCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-Login a user in from the ticket cache.  This
+ method assumes that login had happened already.
+ The Subject field of this UserGroupInformation object is updated to have
+ the new credentials.
+ @throws IOException
+ @throws KerberosAuthException on a failure]]>
+      </doc>
+    </method>
+    <method name="loginUserFromKeytabAndReturnUGI" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log a user in from a keytab file. Loads a user identity from a keytab
+ file and login them in. This new user does not affect the currently
+ logged-in user.
+ @param user the principal name to load from the keytab
+ @param path the path to the keytab file
+ @throws IOException if the keytab file can't be read]]>
+      </doc>
+    </method>
+    <method name="isLoginKeytabBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via keytab
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="isLoginTicketBased" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did the login happen via ticket cache
+ @return true or false]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createRemoteUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Create a user from a login name. It is intended to be used for remote
+ users in RPC, since it won't have any credentials.
+ @param user the full user principal name, must not be empty or null
+ @return the UserGroupInformation for the remote user.]]>
+      </doc>
+    </method>
+    <method name="createProxyUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create a proxy user using username of the effective user and the ugi of the
+ real user.
+ @param user
+ @param realUser
+ @return proxyUser ugi]]>
+      </doc>
+    </method>
+    <method name="getRealUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get RealUser (vs. EffectiveUser)
+ @return realUser running over proxy user]]>
+      </doc>
+    </method>
+    <method name="createUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a UGI for testing HDFS and MapReduce
+ @param user the full user principal name
+ @param userGroups the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="createProxyUserForTesting" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="realUser" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="userGroups" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Create a proxy user UGI for testing HDFS and MapReduce
+ 
+ @param user
+          the full user principal name for effective user
+ @param realUser
+          UGI of the real user
+ @param userGroups
+          the names of the groups that the user belongs to
+ @return a fake user for running unit tests]]>
+      </doc>
+    </method>
+    <method name="getShortUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's login name.
+ @return the user's name up to the first '/' or '@'.]]>
+      </doc>
+    </method>
+    <method name="getPrimaryGroupName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getUserName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user's full principal name.
+ @return the user's full principal name.]]>
+      </doc>
+    </method>
+    <method name="addTokenIdentifier" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokenId" type="org.apache.hadoop.security.token.TokenIdentifier"/>
+      <doc>
+      <![CDATA[Add a TokenIdentifier to this UGI. The TokenIdentifier has typically been
+ authenticated by the RPC layer as belonging to the user represented by this
+ UGI.
+ 
+ @param tokenId
+          tokenIdentifier to be added
+ @return true on successful add of new tokenIdentifier]]>
+      </doc>
+    </method>
+    <method name="getTokenIdentifiers" return="java.util.Set"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of TokenIdentifiers belonging to this UGI
+ 
+ @return the set of TokenIdentifiers belonging to this UGI]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a token to this UGI
+ 
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="addToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="org.apache.hadoop.io.Text"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <doc>
+      <![CDATA[Add a named token to this UGI
+ 
+ @param alias Name of the token
+ @param token Token to be added
+ @return true on successful add of new token]]>
+      </doc>
+    </method>
+    <method name="getTokens" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the collection of tokens associated with this user.
+ 
+ @return an unmodifiable collection of tokens associated with user]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the tokens in credentials form associated with this user.
+ 
+ @return Credentials of tokens associated with this user]]>
+      </doc>
+    </method>
+    <method name="addCredentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="credentials" type="org.apache.hadoop.security.Credentials"/>
+      <doc>
+      <![CDATA[Add the given Credentials to this user.
+ @param credentials of tokens and secrets]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user. {@link #getGroups()} is less
+ expensive alternative when checking for a contained element.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the group names for this user.
+ @return the list of users with the primary group first. If the command
+    fails, it returns an empty list.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the username.]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="setAuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+      <doc>
+      <![CDATA[Sets the authentication method in the subject
+ 
+ @param authMethod]]>
+      </doc>
+    </method>
+    <method name="getAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the subject
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the authentication method from the real user's subject.  If there
+ is no real user, return the given user's authentication method.
+ 
+ @return AuthenticationMethod in the subject, null if not present.]]>
+      </doc>
+    </method>
+    <method name="getRealAuthenticationMethod" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Returns the authentication method of a ugi. If the authentication method is
+ PROXY, returns the authentication method of the real user.
+ 
+ @param ugi
+ @return AuthenticationMethod]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Compare the subjects to see if they are equal to each other.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the hash of the subject.]]>
+      </doc>
+    </method>
+    <method name="getSubject" return="javax.security.auth.Subject"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying subject from this ugi.
+ @return the subject that represents this user.]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedAction"/>
+      <doc>
+      <![CDATA[Run the given action as the user.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method]]>
+      </doc>
+    </method>
+    <method name="doAs" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="action" type="java.security.PrivilegedExceptionAction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the given action as the user, potentially throwing an exception.
+ @param <T> the return type of the run method
+ @param action the method to execute
+ @return the value from the run method
+ @throws IOException if the action throws an IOException
+ @throws Error if the action throws an Error
+ @throws RuntimeException if the action throws a RuntimeException
+ @throws InterruptedException if the action throws an InterruptedException
+ @throws UndeclaredThrowableException if the action throws something else]]>
+      </doc>
+    </method>
+    <method name="logAllUserInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Log all (current, real, login) UGI and token info into UGI debug log.
+ @param ugi - UGI
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[A test method to print out the current user's UGI.
+ @param args if there are two arguments, read the user from the keytab
+ and print it out.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <field name="HADOOP_TOKEN_FILE_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the token cache file]]>
+      </doc>
+    </field>
+    <field name="HADOOP_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable pointing to the base64 tokens.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[User and group information for Hadoop.
+ This class wraps around a JAAS Subject and provides methods to determine the
+ user's username and groups. It supports both the Windows, Unix and Kerberos 
+ login modules.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation -->
+  <!-- start class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+  <class name="UserGroupInformation.AuthenticationMethod" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getAuthMethod" return="org.apache.hadoop.security.SaslRpcServer.AuthMethod"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authMethod" type="org.apache.hadoop.security.SaslRpcServer.AuthMethod"/>
+    </method>
+    <doc>
+    <![CDATA[existing types of authentications' methods]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod -->
+</package>
+<package name="org.apache.hadoop.security.alias">
+  <!-- start class org.apache.hadoop.security.alias.CredentialProvider -->
+  <class name="CredentialProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isTransient" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates whether this provider represents a store
+ that is intended for transient use - such as the UserProvider
+ is. These providers are generally used to provide job access to
+ passwords rather than for long term storage.
+ @return true if transient, false otherwise]]>
+      </doc>
+    </method>
+    <method name="flush"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Ensures that any changes to the credentials are written to persistent
+ store.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="alias" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the credential entry for a specific alias.
+ @param alias the name of a specific credential
+ @return the credentialEntry
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAliases" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the aliases for all credentials.
+ @return the list of alias names
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createCredentialEntry" return="org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="credential" type="char[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new credential. The given alias must not already exist.
+ @param name the alias of the credential
+ @param credential the credential value for the alias.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteCredentialEntry"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the given credential.
+ @param name the alias of the credential to delete
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsPassword" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Does this provider require a password? This means that a password is
+ required for normal operation, and it has not been found through normal
+ means. If true, the password should be provided by the caller using
+ setPassword().
+ @return Whether or not the provider requires a password
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noPasswordWarning" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return a warning and instructions for supplying said password to the
+ provider.
+ @return A warning and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <method name="noPasswordError" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If a password for the provider is needed, but is not provided, this will
+ return an error message and instructions for supplying said password to
+ the provider.
+ @return An error message and instructions for supplying the password]]>
+      </doc>
+    </method>
+    <field name="CLEAR_TEXT_FALLBACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A provider of credentials or password for Hadoop applications. Provides an
+ abstraction to separate credential storage from users of them. It
+ is intended to support getting or storing passwords in a variety of ways,
+ including third party bindings.
+ 
+ <code>CredentialProvider</code> implementations must be thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProvider -->
+  <!-- start class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+  <class name="CredentialProviderFactory" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CredentialProviderFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createProvider" return="org.apache.hadoop.security.alias.CredentialProvider"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providerName" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProviders" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="CREDENTIAL_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A factory to create a list of CredentialProvider based on the path given in a
+ Configuration. It uses a service loader interface to find the available
+ CredentialProviders and create them based on the list of URIs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.alias.CredentialProviderFactory -->
+</package>
+<package name="org.apache.hadoop.security.authentication.server">
+</package>
+<package name="org.apache.hadoop.security.authorize">
+  <!-- start class org.apache.hadoop.security.authorize.AccessControlList -->
+  <class name="AccessControlList" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="AccessControlList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constructor exists primarily for AccessControlList to be Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from a String representation of the same.
+ 
+ The String is a a comma separated list of users and groups.
+ The user list comes first and is separated by a space followed 
+ by the group list. For e.g. "user1,user2 group1,group2"
+ 
+ @param aclString String representation of the ACL]]>
+      </doc>
+    </constructor>
+    <constructor name="AccessControlList" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new ACL from String representation of users and groups
+ 
+ The arguments are comma separated lists
+ 
+ @param users comma separated list of users
+ @param groups comma separated list of groups]]>
+      </doc>
+    </constructor>
+    <method name="isAllAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add user to the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="addGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add group to the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="removeUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove user from the names of users allowed for this service.
+ 
+ @param user
+          The user name]]>
+      </doc>
+    </method>
+    <method name="removeGroup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove group from the names of groups allowed for this service.
+ 
+ @param group
+          The group name]]>
+      </doc>
+    </method>
+    <method name="getUsers" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of users allowed for this service.
+ @return the set of user names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="getGroups" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of user groups allowed for this service.
+ @return the set of group names. the set must not be modified.]]>
+      </doc>
+    </method>
+    <method name="isUserInList" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Checks if a user represented by the provided {@link UserGroupInformation}
+ is a member of the Access Control List. If user was proxied and
+ USE_REAL_ACLS + the real user name is in the control list, then treat this
+ case as if user were in the ACL list.
+ @param ugi UserGroupInformation to check if contained in the ACL
+ @return true if ugi is member of the list or if USE_REAL_ACLS + real user
+ is in the list]]>
+      </doc>
+    </method>
+    <method name="isUserAllowed" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns descriptive way of users and groups that are part of this ACL.
+ Use {@link #getAclString()} to get the exact String that can be given to
+ the constructor of AccessControlList to create a new instance.]]>
+      </doc>
+    </method>
+    <method name="getAclString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the access control list as a String that can be used for building a
+ new instance by sending it to the constructor of {@link AccessControlList}.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Deserializes the AccessControlList object]]>
+      </doc>
+    </method>
+    <field name="WILDCARD_ACL_VALUE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_REAL_ACLS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Class representing a configured access control list.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AccessControlList -->
+  <!-- start class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <class name="AuthorizationException" extends="org.apache.hadoop.security.AccessControlException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuthorizationException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AuthorizationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a new exception with the specified cause and a detail
+ message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ typically contains the class and detail message of <tt>cause</tt>).
+ @param  cause the cause (which is saved for later retrieval by the
+         {@link #getCause()} method).  (A <tt>null</tt> value is
+         permitted, and indicates that the cause is nonexistent or
+         unknown.)]]>
+      </doc>
+    </constructor>
+    <method name="getStackTrace" return="java.lang.StackTraceElement[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintStream"/>
+    </method>
+    <method name="printStackTrace"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="java.io.PrintWriter"/>
+    </method>
+    <doc>
+    <![CDATA[An exception class for authorization-related issues.
+ 
+ This class <em>does not</em> provide the stack trace for security purposes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.AuthorizationException -->
+  <!-- start class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <class name="DefaultImpersonationProvider" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authorize.ImpersonationProvider"/>
+    <constructor name="DefaultImpersonationProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTestProvider" return="org.apache.hadoop.security.authorize.DefaultImpersonationProvider"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+    </method>
+    <method name="getProxySuperuserUserConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective usergroups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser usergroups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserGroupConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns configuration key for effective groups allowed for a superuser
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser groups]]>
+      </doc>
+    </method>
+    <method name="getProxySuperuserIpConfKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return configuration key for superuser ip addresses
+ 
+ @param userName name of the superuser
+ @return configuration key for superuser ip-addresses]]>
+      </doc>
+    </method>
+    <method name="getProxyGroups" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProxyHosts" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.security.authorize.DefaultImpersonationProvider -->
+  <!-- start interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+  <interface name="ImpersonationProvider"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurationPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Specifies the configuration prefix for the proxy user properties and
+ initializes the provider.
+
+ @param configurationPrefix the configuration prefix for the proxy user
+ properties]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.lang.String"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+ {@link #authorize(UserGroupInformation, InetAddress)} should
+             be preferred to avoid possibly re-resolving the ip address.
+ @param user ugi of the effective or proxy user which contains a real user.
+ @param remoteAddress the ip address of client.
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+    <method name="authorize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="remoteAddress" type="java.net.InetAddress"/>
+      <exception name="AuthorizationException" type="org.apache.hadoop.security.authorize.AuthorizationException"/>
+      <doc>
+      <![CDATA[Authorize the superuser which is doing doAs.
+
+ @param user ugi of the effective or proxy user which contains a real user
+ @param remoteAddress the ip address of client
+ @throws AuthorizationException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.authorize.ImpersonationProvider -->
+</package>
+<package name="org.apache.hadoop.security.http">
+  <!-- start class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <class name="RestCsrfPreventionFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="RestCsrfPreventionFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filterConfig" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="isBrowser" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userAgent" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This method interrogates the User-Agent String and returns whether it
+ refers to a browser.  If its not a browser, then the requirement for the
+ CSRF header will not be enforced; if it is a browser, the requirement will
+ be enforced.
+ <p>
+ A User-Agent String is considered to be a browser if it matches
+ any of the regex patterns from browser-useragent-regex; the default
+ behavior is to consider everything a browser that matches the following:
+ "^Mozilla.*,^Opera.*".  Subclasses can optionally override
+ this method to use different behavior.
+
+ @param userAgent The User-Agent String, or null if there isn't one
+ @return true if the User-Agent String refers to a browser, false if not]]>
+      </doc>
+    </method>
+    <method name="handleHttpInteraction"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="httpInteraction" type="org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+      <doc>
+      <![CDATA[Handles an {@link HttpInteraction} by applying the filtering logic.
+
+ @param httpInteraction caller's HTTP interaction
+ @throws IOException if there is an I/O error
+ @throws ServletException if the implementation relies on the servlet API
+     and a servlet API call has failed]]>
+      </doc>
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="javax.servlet.ServletRequest"/>
+      <param name="response" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="HEADER_USER_AGENT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="BROWSER_USER_AGENT_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_METHODS_TO_IGNORE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HEADER_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter provides protection against cross site request forgery (CSRF)
+ attacks for REST APIs. Enabling this filter on an endpoint results in the
+ requirement of all client to send a particular (configurable) HTTP header
+ with every request. In the absense of this header the filter will reject the
+ attempt as a bad request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.RestCsrfPreventionFilter -->
+  <!-- start class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+  <class name="XFrameOptionsFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="javax.servlet.Filter"/>
+    <constructor name="XFrameOptionsFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="destroy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="javax.servlet.ServletRequest"/>
+      <param name="res" type="javax.servlet.ServletResponse"/>
+      <param name="chain" type="javax.servlet.FilterChain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="javax.servlet.FilterConfig"/>
+      <exception name="ServletException" type="javax.servlet.ServletException"/>
+    </method>
+    <method name="getFilterParams" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="confPrefix" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Constructs a mapping of configuration properties to be used for filter
+ initialization.  The mapping includes all properties that start with the
+ specified configuration prefix.  Property names in the mapping are trimmed
+ to remove the configuration prefix.
+
+ @param conf configuration to read
+ @param confPrefix configuration prefix
+ @return mapping of configuration properties to be used for filter
+     initialization]]>
+      </doc>
+    </method>
+    <field name="X_FRAME_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CUSTOM_HEADER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This filter protects webapps from clickjacking attacks that
+ are possible through use of Frames to embed the resources in another
+ application and intercept clicks to accomplish nefarious things.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.http.XFrameOptionsFilter -->
+</package>
+<package name="org.apache.hadoop.security.protocolPB">
+</package>
+<package name="org.apache.hadoop.security.ssl">
+</package>
+<package name="org.apache.hadoop.security.token">
+  <!-- start class org.apache.hadoop.security.token.SecretManager -->
+  <class name="SecretManager" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createPassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <doc>
+      <![CDATA[Create the password for the given identifier.
+ identifier may be modified inside this method.
+ @param identifier the identifier to use
+ @return the new password]]>
+      </doc>
+    </method>
+    <method name="retrievePassword" return="byte[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <doc>
+      <![CDATA[Retrieve the password for the given token identifier. Should check the date
+ or registry to make sure the token hasn't expired or been revoked. Returns 
+ the relevant password.
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid]]>
+      </doc>
+    </method>
+    <method name="retriableRetrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="T"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <exception name="RetriableException" type="org.apache.hadoop.ipc.RetriableException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The same functionality with {@link #retrievePassword}, except that this 
+ method can throw a {@link RetriableException} or a {@link StandbyException}
+ to indicate that client can retry/failover the same operation because of 
+ temporary issue on the server side.
+ 
+ @param identifier the identifier to validate
+ @return the password to use
+ @throws InvalidToken the token was invalid
+ @throws StandbyException the server is in standby state, the client can
+         try other servers
+ @throws RetriableException the token was invalid, and the server thinks 
+         this may be a temporary issue and suggests the client to retry
+ @throws IOException to allow future exceptions to be added without breaking
+         compatibility]]>
+      </doc>
+    </method>
+    <method name="createIdentifier" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty token identifier.
+ @return the newly created empty token identifier]]>
+      </doc>
+    </method>
+    <method name="checkAvailableForRead"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="StandbyException" type="org.apache.hadoop.ipc.StandbyException"/>
+      <doc>
+      <![CDATA[No-op if the secret manager is available for reading tokens, throw a
+ StandbyException otherwise.
+ 
+ @throws StandbyException if the secret manager is not available to read
+         tokens]]>
+      </doc>
+    </method>
+    <method name="generateSecret" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generate a new random secret key.
+ @return the new key]]>
+      </doc>
+    </method>
+    <method name="createPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="byte[]"/>
+      <param name="key" type="javax.crypto.SecretKey"/>
+      <doc>
+      <![CDATA[Compute HMAC of the identifier using the secret key and return the 
+ output as password
+ @param identifier the bytes of the identifier
+ @param key the secret key
+ @return the bytes of the generated password]]>
+      </doc>
+    </method>
+    <method name="createSecretKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <doc>
+      <![CDATA[Convert the byte[] to a secret key
+ @param key the byte[] to create a secret key from
+ @return the secret key]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The server-side secret manager for each token type.
+ @param <T> The type of the token identifier]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.SecretManager -->
+  <!-- start class org.apache.hadoop.security.token.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="Token" type="T, org.apache.hadoop.security.token.SecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token given a token identifier and a secret manager for the
+ type of the token identifier.
+ @param id the token identifier
+ @param mgr the secret manager]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="byte[], byte[], org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a token from the components.
+ @param identifier the token identifier
+ @param password the token's password
+ @param kind the kind of token
+ @param service the service for this token]]>
+      </doc>
+    </constructor>
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor.]]>
+      </doc>
+    </constructor>
+    <constructor name="Token" type="org.apache.hadoop.security.token.Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clone a token.
+ @param other the token to clone]]>
+      </doc>
+    </constructor>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="bytes" type="byte[]"/>
+    </method>
+    <method name="setPassword"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newPassword" type="byte[]"/>
+    </method>
+    <method name="copyToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdentifier" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier's byte representation.
+ @return the token identifier's byte representation]]>
+      </doc>
+    </method>
+    <method name="decodeIdentifier" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the token identifier object, or null if it could not be constructed
+ (because the class could not be loaded, for example).
+ @return the token identifier, or null if there was no class found for it
+ @throws IOException failure to unmarshall the data
+ @throws RuntimeException if the token class could not be instantiated.]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password/secret.
+ @return the token password/secret]]>
+      </doc>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service on which the token is supposed to be used.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="setService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Set the service on which the token is supposed to be used.
+ @param newService the service name]]>
+      </doc>
+    </method>
+    <method name="isPrivate" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether this is a private token.
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="isPrivateCloneOf" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thePublicService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Whether this is a private clone of a public token.
+ @param thePublicService the public service name
+ @return false always for non-private tokens]]>
+      </doc>
+    </method>
+    <method name="privateClone" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newService" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Create a private clone of a public token.
+ @param newService the new service name
+ @return a private token]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="encodeToUrlString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Encode this token as a url safe string.
+ @return the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="decodeFromUrlString"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Decode the given url safe string into this token.
+ @param newValue the encoded string
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="right" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="buildCacheKey" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is this token managed so that it can be renewed or cancelled?
+ @return true, if it can be renewed and cancelled.]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew this delegation token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel this delegation token.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The client-side form of the token.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token -->
+  <!-- start class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <class name="Token.TrivialRenewer" extends="org.apache.hadoop.security.token.TokenRenewer"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TrivialRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="handleKind" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <method name="renew" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="cancel"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[A trivial renewer for token kinds that aren't managed. Sub-classes need
+ to implement getKind for their token kind.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.Token.TrivialRenewer -->
+  <!-- start class org.apache.hadoop.security.token.TokenIdentifier -->
+  <class name="TokenIdentifier" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind
+ @return the kind of the token]]>
+      </doc>
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Ugi with the username encoded in the token identifier
+ 
+ @return the username. null is returned if username in the identifier is
+         empty or null.]]>
+      </doc>
+    </method>
+    <method name="getBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the bytes for the token identifier
+ @return the bytes of the identifier]]>
+      </doc>
+    </method>
+    <method name="getTrackingId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a tracking identifier that can be used to associate usages of a
+ token across multiple client sessions.
+
+ Currently, this function just returns an MD5 of {{@link #getBytes()}.
+
+ @return tracking identifier]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An identifier that identifies a token, may contain public information 
+ about a token, including its kind (or type).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenIdentifier -->
+  <!-- start class org.apache.hadoop.security.token.TokenInfo -->
+  <class name="TokenInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Indicates Token related information to be used]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenInfo -->
+  <!-- start class org.apache.hadoop.security.token.TokenRenewer -->
+  <class name="TokenRenewer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenRenewer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="handleKind" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="kind" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Does this renewer handle this kind of token?
+ @param kind the kind of the token
+ @return true if this renewer can renew it]]>
+      </doc>
+    </method>
+    <method name="isManaged" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is the given token managed? Only managed tokens may be renewed or
+ cancelled.
+ @param token the token being checked
+ @return true if the token may be renewed or cancelled
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renew" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew the given token.
+ @return the new expiration time
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="cancel"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel the given token
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the interface for plugins that handle tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.TokenRenewer -->
+  <!-- start interface org.apache.hadoop.security.token.TokenSelector -->
+  <interface name="TokenSelector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+    <doc>
+    <![CDATA[Select token of type T from tokens for use with named service
+ 
+ @param <T>
+          T extends TokenIdentifier]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.security.token.TokenSelector -->
+</package>
+<package name="org.apache.hadoop.security.token.delegation.web">
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <class name="DelegationTokenAuthenticatedURL" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DelegationTokenAuthenticatedURL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+ <p>
+ An instance of the default {@link DelegationTokenAuthenticator} will be
+ used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code> using the default
+ {@link DelegationTokenAuthenticator} class.
+
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <constructor name="DelegationTokenAuthenticatedURL" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator, org.apache.hadoop.security.authentication.client.ConnectionConfigurator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an <code>DelegationTokenAuthenticatedURL</code>.
+
+ @param authenticator the {@link DelegationTokenAuthenticator} instance to
+ use, if <code>null</code> the default one will be used.
+ @param connConfigurator a connection configurator.]]>
+      </doc>
+    </constructor>
+    <method name="setDefaultDelegationTokenAuthenticator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="authenticator" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Sets the default {@link DelegationTokenAuthenticator} class to use when an
+ {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @param authenticator the authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="getDefaultDelegationTokenAuthenticator" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the default {@link DelegationTokenAuthenticator} class to use when
+ an {@link DelegationTokenAuthenticatedURL} instance is created without
+ specifying one.
+ <p>
+ The default class is {@link KerberosDelegationTokenAuthenticator}
+
+ @return the delegation token authenticator class to use as default.]]>
+      </doc>
+    </method>
+    <method name="setUseQueryStringForDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="useQueryString" type="boolean"/>
+      <doc>
+      <![CDATA[Sets if delegation token should be transmitted in the URL query string.
+ By default it is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.
+ <p>
+ This method is provided to enable WebHDFS backwards compatibility.
+
+ @param useQueryString  <code>TRUE</code> if the token is transmitted in the
+ URL query string, <code>FALSE</code> if the delegation token is transmitted
+ using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ header.]]>
+      </doc>
+    </method>
+    <method name="useQueryStringForDelegationToken" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns if delegation token is transmitted as a HTTP header.
+
+ @return <code>TRUE</code> if the token is transmitted in the URL query
+ string, <code>FALSE</code> if the delegation token is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}, it uses a Delegation
+ Token only if the given auth token is an instance of {@link Token} and
+ it contains a Delegation Token, otherwise use the configured
+ {@link DelegationTokenAuthenticator} to authenticate the connection.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="openConnection" return="java.net.HttpURLConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAs" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Returns an authenticated {@link HttpURLConnection}. If the Delegation
+ Token is present, it will be used taking precedence over the configured
+ <code>Authenticator</code>. If the <code>doAs</code> parameter is not NULL,
+ the request will be done on behalf of the specified <code>doAs</code> user.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @param doAs user to do the the request on behalf of, if NULL the request is
+ as self.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @return a delegation token.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The <code>DelegationTokenAuthenticatedURL</code> is a
+ {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token
+ functionality.
+ <p>
+ The authentication mechanisms supported by default are Hadoop Simple
+ authentication (also known as pseudo authentication) and Kerberos SPNEGO
+ authentication.
+ <p>
+ Additional authentication mechanisms can be supported via {@link
+ DelegationTokenAuthenticator} implementations.
+ <p>
+ The default {@link DelegationTokenAuthenticator} is the {@link
+ KerberosDelegationTokenAuthenticator} class which supports
+ automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via
+ the {@link PseudoDelegationTokenAuthenticator} class.
+ <p>
+ <code>AuthenticatedURL</code> instances are not thread-safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <class name="DelegationTokenAuthenticatedURL.Token" extends="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="delegationToken" type="org.apache.hadoop.security.token.Token"/>
+    </method>
+    <doc>
+    <![CDATA[Client side authentication token that handles Delegation Tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.Token -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <class name="DelegationTokenAuthenticator" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.authentication.client.Authenticator"/>
+    <constructor name="DelegationTokenAuthenticator" type="org.apache.hadoop.security.authentication.client.Authenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConnectionConfigurator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configurator" type="org.apache.hadoop.security.authentication.client.ConnectionConfigurator"/>
+    </method>
+    <method name="authenticate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="renewer" type="java.lang.String"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Requests a delegation token using the configured <code>Authenticator</code>
+ for authentication.
+
+ @param url the URL to get the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token being used for the user where the
+ Delegation token will be stored.
+ @param renewer the renewer user.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="AuthenticationException" type="org.apache.hadoop.security.authentication.client.AuthenticationException"/>
+      <doc>
+      <![CDATA[Renews a delegation token from the server end-point using the
+ configured <code>Authenticator</code> for authentication.
+
+ @param url the URL to renew the delegation token from. Only HTTP/S URLs are
+ supported.
+ @param token the authentication token with the Delegation Token to renew.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.net.URL"/>
+      <param name="token" type="org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token"/>
+      <param name="dToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="doAsUser" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Cancels a delegation token from the server end-point. It does not require
+ being authenticated by the configured <code>Authenticator</code>.
+
+ @param url the URL to cancel the delegation token from. Only HTTP/S URLs
+ are supported.
+ @param token the authentication token with the Delegation Token to cancel.
+ @param doAsUser the user to do as, which will be the token owner.
+ @throws IOException if an IO error occurred.]]>
+      </doc>
+    </method>
+    <field name="OP_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TOKEN_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEWER_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SERVICE_PARAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATION_TOKEN_URL_STRING_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RENEW_DELEGATION_TOKEN_JSON" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@link Authenticator} wrapper that enhances an {@link Authenticator} with
+ Delegation Token support.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <class name="KerberosDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KerberosDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>KerberosDelegationTokenAuthenticator</code> provides support for
+ Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation
+ Token operations.
+ <p>
+ It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP
+ endpoint does not trigger a SPNEGO authentication]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator -->
+  <!-- start class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+  <class name="PseudoDelegationTokenAuthenticator" extends="org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PseudoDelegationTokenAuthenticator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The <code>PseudoDelegationTokenAuthenticator</code> provides support for
+ Hadoop's pseudo authentication mechanism that accepts
+ the user name specified as a query string parameter and support for Hadoop
+ Delegation Token operations.
+ <p>
+ This mimics the model of Hadoop Simple authentication trusting the
+ {@link UserGroupInformation#getCurrentUser()} value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticator -->
+</package>
+<package name="org.apache.hadoop.service">
+  <!-- start class org.apache.hadoop.service.AbstractService -->
+  <class name="AbstractService" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <constructor name="AbstractService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the service.
+ @param name service name]]>
+      </doc>
+    </constructor>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Set the configuration for this service.
+ This method is called during {@link #init(Configuration)}
+ and should only be needed if for some reason a service implementation
+ needs to override that initial setting -for example replacing
+ it with a new subclass of {@link Configuration}
+ @param conf new configuration.]]>
+      </doc>
+    </method>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ This invokes {@link #serviceInit}
+ @param conf the configuration of the service. This must not be null
+ @throws ServiceStateException if the configuration was null,
+ the state change not permitted, or something else went wrong]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws ServiceStateException if the current service state does not permit
+ this action]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Relay to {@link #stop()}
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="noteFailure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <param name="exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Failure handling: record the exception
+ that triggered it -if there was not one already.
+ Services are free to call this themselves.
+ @param exception the exception]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[All initialization code needed by a service.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #init(Configuration)} prevents re-entrancy.
+
+ The base implementation checks to see if the subclass has created
+ a new configuration instance, and if so, updates the base class value
+ @param conf configuration
+ @throws Exception on a failure -these will be caught,
+ possibly wrapped, and will trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the INITED to STARTED transition.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #start()} prevents re-entrancy.
+
+ @throws Exception if needed -these will be caught,
+ wrapped, and trigger a service stop]]>
+      </doc>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Actions called during the transition to the STOPPED state.
+
+ This method will only ever be called once during the lifecycle of
+ a specific service instance.
+
+ Implementations do not need to be synchronized as the logic
+ in {@link #stop()} prevents re-entrancy.
+
+ Implementations MUST write this to be robust against failures, including
+ checks for null references -and for the first failure to not stop other
+ attempts to shut down parts of the service.
+
+ @throws Exception if needed -these will be caught and logged.]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    </method>
+    <method name="registerGlobalListener"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a global listener, which receives notifications
+ from the state change events of all services in the JVM
+ @param l listener]]>
+      </doc>
+    </method>
+    <method name="unregisterGlobalListener" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="l" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[unregister a global listener.
+ @param l listener to unregister
+ @return true if the listener was found (and then deleted)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="expected" type="org.apache.hadoop.service.Service.STATE"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="putBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="details" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Put a blocker to the blocker map -replacing any
+ with the same name.
+ @param name blocker name
+ @param details any specifics on the block. This must be non-null.]]>
+      </doc>
+    </method>
+    <method name="removeBlocker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Remove a blocker from the blocker map -
+ this is a no-op if the blocker is not present
+ @param name the name of the blocker]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This is the base implementation class for services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.AbstractService -->
+  <!-- start class org.apache.hadoop.service.CompositeService -->
+  <class name="CompositeService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeService" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getServices" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a cloned list of services
+ @return a list of child services at the time of invocation -
+ added services will not be picked up.]]>
+      </doc>
+    </method>
+    <method name="addService"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Add the passed {@link Service} to the list of services managed by this
+ {@link CompositeService}
+ @param service the {@link Service} to be added]]>
+      </doc>
+    </method>
+    <method name="addIfService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="object" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[If the passed object is an instance of {@link Service},
+ add it to the list of services managed by this {@link CompositeService}
+ @param object
+ @return true if a service is added, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="removeService" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="STOP_ONLY_STARTED_SERVICES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Policy on shutdown: attempt to close everything (purest) or
+ only try to close started services (which assumes
+ that the service implementations may not handle the stop() operation
+ except when started.
+ Irrespective of this policy, if a child service fails during
+ its init() or start() operations, it will have stop() called on it.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Composition of services.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.CompositeService -->
+  <!-- start class org.apache.hadoop.service.LifecycleEvent -->
+  <class name="LifecycleEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Serializable"/>
+    <constructor name="LifecycleEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="time" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Local time in milliseconds when the event occurred]]>
+      </doc>
+    </field>
+    <field name="state" type="org.apache.hadoop.service.Service.STATE"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[new state]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A serializable lifecycle event: the time a state
+ transition occurred, and what state was entered.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LifecycleEvent -->
+  <!-- start class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <class name="LoggingStateChangeListener" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.ServiceStateChangeListener"/>
+    <constructor name="LoggingStateChangeListener" type="org.slf4j.Logger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the given log
+ @param log destination for events]]>
+      </doc>
+    </constructor>
+    <constructor name="LoggingStateChangeListener"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Log events to the static log for this class]]>
+      </doc>
+    </constructor>
+    <method name="stateChanged"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback for a state change event: log it
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is a state change listener that logs events at INFO level]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.LoggingStateChangeListener -->
+  <!-- start interface org.apache.hadoop.service.Service -->
+  <interface name="Service"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="init"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Initialize the service.
+
+ The transition MUST be from {@link STATE#NOTINITED} to {@link STATE#INITED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @param config the configuration of the service
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Start the service.
+
+ The transition MUST be from {@link STATE#INITED} to {@link STATE#STARTED}
+ unless the operation failed and an exception was raised, in which case
+ {@link #stop()} MUST be invoked and the service enter the state
+ {@link STATE#STOPPED}.
+ @throws RuntimeException on any failure during the operation]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Stop the service. This MUST be a no-op if the service is already
+ in the {@link STATE#STOPPED} state. It SHOULD be a best-effort attempt
+ to stop all parts of the service.
+
+ The implementation must be designed to complete regardless of the service
+ state, including the initialized/uninitialized state of all its internal
+ fields.
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A version of stop() that is designed to be usable in Java7 closure
+ clauses.
+ Implementation classes MUST relay this directly to {@link #stop()}
+ @throws IOException never
+ @throws RuntimeException on any failure during the stop operation]]>
+      </doc>
+    </method>
+    <method name="registerServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Register a listener to the service state change events.
+ If the supplied listener is already listening to this service,
+ this method is a no-op.
+ @param listener a new listener]]>
+      </doc>
+    </method>
+    <method name="unregisterServiceListener"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="listener" type="org.apache.hadoop.service.ServiceStateChangeListener"/>
+      <doc>
+      <![CDATA[Unregister a previously registered listener of the service state
+ change events. No-op if the listener is already unregistered.
+ @param listener the listener to unregister.]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of this service.
+ @return the service name]]>
+      </doc>
+    </method>
+    <method name="getConfig" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration of this service.
+ This is normally not a clone and may be manipulated, though there are no
+ guarantees as to what the consequences of such actions may be
+ @return the current configuration, unless a specific implentation chooses
+ otherwise.]]>
+      </doc>
+    </method>
+    <method name="getServiceState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current service state
+ @return the state of the service]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service start time
+ @return the start time of the service. This will be zero if the service
+ has not yet been started.]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query to see if the service is in a specific state.
+ In a multi-threaded system, the state may not hold for very long.
+ @param state the expected state
+ @return true if, at the time of invocation, the service was in that state.]]>
+      </doc>
+    </method>
+    <method name="getFailureCause" return="java.lang.Throwable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the first exception raised during the service failure. If null,
+ no exception was logged
+ @return the failure logged during a transition to the stopped state]]>
+      </doc>
+    </method>
+    <method name="getFailureState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the state in which the failure in {@link #getFailureCause()} occurred.
+ @return the state or null if there was no failure]]>
+      </doc>
+    </method>
+    <method name="waitForServiceToStop" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeout" type="long"/>
+      <doc>
+      <![CDATA[Block waiting for the service to stop; uses the termination notification
+ object to do so.
+
+ This method will only return after all the service stop actions
+ have been executed (to success or failure), or the timeout elapsed
+ This method can be called before the service is inited or started; this is
+ to eliminate any race condition with the service stopping before
+ this event occurs.
+ @param timeout timeout in milliseconds. A value of zero means "forever"
+ @return true iff the service stopped in the time period]]>
+      </doc>
+    </method>
+    <method name="getLifecycleHistory" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a snapshot of the lifecycle history; it is a static list
+ @return a possibly empty but never null list of lifecycle events.]]>
+      </doc>
+    </method>
+    <method name="getBlockers" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the blockers on a service -remote dependencies
+ that are stopping the service from being <i>live</i>.
+ @return a (snapshotted) map of blocker name-&gt;description values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Service LifeCycle.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.Service -->
+  <!-- start class org.apache.hadoop.service.ServiceOperations -->
+  <class name="ServiceOperations" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service.
+ <p>Do nothing if the service is null or not
+ in a state in which it can be/needs to be stopped.
+ <p>
+ The service state is checked <i>before</i> the operation begins.
+ This process is <i>not</i> thread safe.
+ @param service a service or null]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <method name="stopQuietly" return="java.lang.Exception"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Stop a service; if it is null do nothing. Exceptions are caught and
+ logged at warn level. (but not Throwables). This operation is intended to
+ be used in cleanup operations
+
+ @param log the log to warn at
+ @param service a service; may be null
+ @return any exception that was caught; null if none was.
+ @see ServiceOperations#stopQuietly(Service)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains a set of methods to work with services, especially
+ to walk them through their lifecycle.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceOperations -->
+  <!-- start interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <interface name="ServiceStateChangeListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="stateChanged"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.service.Service"/>
+      <doc>
+      <![CDATA[Callback to notify of a state change. The service will already
+ have changed state before this callback is invoked.
+
+ This operation is invoked on the thread that initiated the state change,
+ while the service itself in in a sychronized section.
+ <ol>
+   <li>Any long-lived operation here will prevent the service state
+   change from completing in a timely manner.</li>
+   <li>If another thread is somehow invoked from the listener, and
+   that thread invokes the methods of the service (including
+   subclass-specific methods), there is a risk of a deadlock.</li>
+ </ol>
+
+
+ @param service the service that has changed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface to notify state changes of a service.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.ServiceStateChangeListener -->
+  <!-- start class org.apache.hadoop.service.ServiceStateException -->
+  <class name="ServiceStateException" extends="java.lang.RuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <constructor name="ServiceStateException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate
+ @param message error message]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate with a message and cause; if the cause has an exit code
+ then it is used, otherwise the generic
+ {@link LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION} exit code
+ is used.
+ @param message exception message
+ @param cause optional inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="int, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate, using the specified exit code as the exit code
+ of the exception, irrespetive of any exit code supplied by any inner
+ cause.
+
+ @param exitCode exit code to declare
+ @param message exception message
+ @param cause inner cause]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <method name="convert" return="java.lang.RuntimeException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="text" type="java.lang.String"/>
+      <param name="fault" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Convert any exception into a {@link RuntimeException}.
+ If the caught exception is already of that type, it is typecast to a
+ {@link RuntimeException} and returned.
+
+ All other exception types are wrapped in a new instance of
+ {@code ServiceStateException}.
+ @param text text to use if a new exception is created
+ @param fault exception or throwable
+ @return a {@link RuntimeException} to rethrow]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Exception that can be raised on state change operations, whose
+ exit code can be explicitly set, determined from that of any nested
+ cause, or a default value of
+ {@link  LauncherExitCodes#EXIT_SERVICE_LIFECYCLE_EXCEPTION}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateException -->
+  <!-- start class org.apache.hadoop.service.ServiceStateModel -->
+  <class name="ServiceStateModel" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServiceStateModel" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the service state model in the {@link Service.STATE#NOTINITED}
+ state.]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceStateModel" type="java.lang.String, org.apache.hadoop.service.Service.STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a service state model instance in the chosen state
+ @param state the starting state]]>
+      </doc>
+    </constructor>
+    <method name="getState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Query the service state. This is a non-blocking operation.
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="isInState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Query that the state is in a specific state
+ @param proposed proposed new state
+ @return the state]]>
+      </doc>
+    </method>
+    <method name="ensureCurrentState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expectedState" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Verify that that a service is in a given state.
+ @param expectedState the desired state
+ @throws ServiceStateException if the service state is different from
+ the desired state]]>
+      </doc>
+    </method>
+    <method name="enterState" return="org.apache.hadoop.service.Service.STATE"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Enter a state -thread safe.
+
+ @param proposed proposed new state
+ @return the original state
+ @throws ServiceStateException if the transition is not permitted]]>
+      </doc>
+    </method>
+    <method name="checkStateTransition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Check that a state tansition is valid and
+ throw an exception if not
+ @param name name of the service (can be null)
+ @param state current state
+ @param proposed proposed new state]]>
+      </doc>
+    </method>
+    <method name="isValidStateTransition" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="current" type="org.apache.hadoop.service.Service.STATE"/>
+      <param name="proposed" type="org.apache.hadoop.service.Service.STATE"/>
+      <doc>
+      <![CDATA[Is a state transition valid?
+ There are no checks for current==proposed
+ as that is considered a non-transition.
+
+ using an array kills off all branch misprediction costs, at the expense
+ of cache line misses.
+
+ @param current current state
+ @param proposed proposed new state
+ @return true if the transition to a new state is valid]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return the state text as the toString() value
+ @return the current state's description]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the service state model.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.ServiceStateModel -->
+</package>
+<package name="org.apache.hadoop.service.launcher">
+  <!-- start class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <class name="AbstractLaunchableService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.launcher.LaunchableService"/>
+    <constructor name="AbstractLaunchableService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct an instance with the given name.]]>
+      </doc>
+    </constructor>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The base implementation logs all arguments at the debug level,
+ then returns the passed in config unchanged.]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ <p>
+ The action is to signal success by returning the exit code 0.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Subclass of {@link AbstractService} that provides basic implementations
+ of the {@link LaunchableService} methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.AbstractLaunchableService -->
+  <!-- start class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <class name="HadoopUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="HadoopUncaughtExceptionHandler" type="java.lang.Thread.UncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance delegating to the supplied handler if
+ the exception is considered "simple".
+ @param delegate a delegate exception handler.]]>
+      </doc>
+    </constructor>
+    <constructor name="HadoopUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Basic exception handler -logs simple exceptions, then continues.]]>
+      </doc>
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="thread" type="java.lang.Thread"/>
+      <param name="exception" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[Uncaught exception handler.
+ If an error is raised: shutdown
+ The state of the system is unknown at this point -attempting
+ a clean shutdown is dangerous. Instead: exit
+ @param thread thread that failed
+ @param exception the raised exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ in the main entry point. 
+
+ The base class will always attempt to shut down the process if an Error
+ was raised; the behavior on a standard Exception, raised outside 
+ process shutdown, is simply to log it. 
+
+ (Based on the class {@code YarnUncaughtExceptionHandler})]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.HadoopUncaughtExceptionHandler -->
+  <!-- start interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <interface name="LaunchableService"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.service.Service"/>
+    <method name="bindArgs" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.util.List"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Propagate the command line arguments.
+ <p>
+ This method is called before {@link #init(Configuration)};
+ Any non-null configuration that is returned from this operation
+ becomes the one that is passed on to that {@link #init(Configuration)}
+ operation.
+ <p>
+ This permits implementations to change the configuration before
+ the init operation. As the ServiceLauncher only creates
+ an instance of the base {@link Configuration} class, it is
+ recommended to instantiate any subclass (such as YarnConfiguration)
+ that injects new resources.
+ <p>
+ @param config the initial configuration build up by the
+ service launcher.
+ @param args list of arguments passed to the command line
+ after any launcher-specific commands have been stripped.
+ @return the configuration to init the service with.
+ Recommended: pass down the config parameter with any changes
+ @throws Exception any problem]]>
+      </doc>
+    </method>
+    <method name="execute" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Run a service. This method is called after {@link Service#start()}.
+ <p>
+ The return value becomes the exit code of the launched process.
+ <p>
+ If an exception is raised, the policy is:
+ <ol>
+   <li>Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}:
+   the exception is passed up unmodified.
+   </li>
+   <li>Any exception which implements
+   {@link org.apache.hadoop.util.ExitCodeProvider}:
+   A new {@link ServiceLaunchException} is created with the exit code
+   and message of the thrown exception; the thrown exception becomes the
+   cause.</li>
+   <li>Any other exception: a new {@link ServiceLaunchException} is created
+   with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and
+   the message of the original exception (which becomes the cause).</li>
+ </ol>
+ @return the exit code
+ @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed
+  up as the exit code and error text.
+ @throws Exception any exception to report. If it provides an exit code
+ this is used in a wrapping exception.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An interface which services can implement to have their
+ execution managed by the ServiceLauncher.
+ <p>
+ The command line options will be passed down before the 
+ {@link Service#init(Configuration)} operation is invoked via an
+ invocation of {@link LaunchableService#bindArgs(Configuration, List)}
+ After the service has been successfully started via {@link Service#start()}
+ the {@link LaunchableService#execute()} method is called to execute the 
+ service. When this method returns, the service launcher will exit, using
+ the return code from the method as its exit option.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LaunchableService -->
+  <!-- start interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <interface name="LauncherExitCodes"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="EXIT_SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Success: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FAIL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Generic "false/fail" response: {@value}.
+ The operation worked but the result was not "true" from the viewpoint
+ of the executed code.]]>
+      </doc>
+    </field>
+    <field name="EXIT_CLIENT_INITIATED_SHUTDOWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a client requested service termination: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_TASK_LAUNCH_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when targets could not be launched: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_INTERRUPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a control-C, kill -3, signal was picked up: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_OTHER_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when something happened but we can't be specific: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_COMMAND_ARGUMENT_ERROR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the command line doesn't parse: {@value}, or
+ when it is otherwise invalid.
+ <p>
+ Approximate HTTP equivalent: {@code 400 Bad Request}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNAUTHORIZED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The request requires user authentication: {@value}.
+ <p>
+ approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]>
+      </doc>
+    </field>
+    <field name="EXIT_USAGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when a usage message was printed: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_FORBIDDEN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Forbidden action: {@value}.
+ <p>
+ Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_FOUND" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Something was not found: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 404: Not Found}]]>
+      </doc>
+    </field>
+    <field name="EXIT_OPERATION_NOT_ALLOWED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The operation is not allowed: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 405: Not allowed}]]>
+      </doc>
+    </field>
+    <field name="EXIT_NOT_ACCEPTABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The command is somehow not acceptable: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 406: Not Acceptable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_CONNECTIVITY_PROBLEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code on connectivity problems: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 408: Request Timeout}]]>
+      </doc>
+    </field>
+    <field name="EXIT_BAD_CONFIGURATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when the configurations in valid/incomplete: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 409: Conflict}]]>
+      </doc>
+    </field>
+    <field name="EXIT_EXCEPTION_THROWN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Exit code when an exception was thrown from the service: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 500 Internal Server Error}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNIMPLEMENTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Unimplemented feature: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 501: Not Implemented}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Service Unavailable; it may be available later: {@value}.
+ <p>
+ Approximate HTTP equivalent: {@code 503 Service Unavailable}]]>
+      </doc>
+    </field>
+    <field name="EXIT_UNSUPPORTED_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The application does not support, or refuses to support this
+ version: {@value}.
+ <p>
+ If raised, this is expected to be raised server-side and likely due
+ to client/server version incompatibilities.
+ <p>
+ Approximate HTTP equivalent: {@code 505: Version Not Supported}]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_CREATION_FAILURE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <field name="EXIT_SERVICE_LIFECYCLE_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The service instance could not be created: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Common Exit codes.
+ <p>
+ Codes with a YARN prefix are YARN-related.
+ <p>
+ Many of the exit codes are designed to resemble HTTP error codes,
+ squashed into a single byte. e.g 44 , "not found" is the equivalent
+ of 404. The various 2XX HTTP error codes aren't followed;
+ the Unix standard of "0" for success is used.
+ <pre>
+    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.service.launcher.LauncherExitCodes -->
+  <!-- start class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+  <class name="ServiceLaunchException" extends="org.apache.hadoop.util.ExitUtil.ExitException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.ExitCodeProvider"/>
+    <implements name="org.apache.hadoop.service.launcher.LauncherExitCodes"/>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code.
+ @param exitCode exit code
+ @param cause cause of the exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an exception with the specific exit code and text.
+ @param exitCode exit code
+ @param message message to use in exception]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ <p>
+ If the last argument is a throwable, it becomes the cause of the exception.
+ It will also be used as a parameter for the format.
+ @param exitCode exit code
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <constructor name="ServiceLaunchException" type="int, java.lang.Throwable, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a formatted exception.
+ <p>
+ This uses {@link String#format(String, Object...)}
+ to build the formatted exception in the ENGLISH locale.
+ @param exitCode exit code
+ @param cause inner cause
+ @param format format for message to use in exception
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[A service launch exception that includes an exit code.
+ <p>
+ When caught by the ServiceLauncher, it will convert that
+ into a process exit code.
+ 
+ The {@link #ServiceLaunchException(int, String, Object...)} constructor
+ generates formatted exceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.service.launcher.ServiceLaunchException -->
+</package>
+<package name="org.apache.hadoop.tools">
+</package>
+<package name="org.apache.hadoop.tools.protocolPB">
+</package>
+<package name="org.apache.hadoop.tracing">
+</package>
+<package name="org.apache.hadoop.util">
+  <!-- start class org.apache.hadoop.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="java.net.URLClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <method name="getResource" return="java.net.URL"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="loadClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="resolve" type="boolean"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+    </method>
+    <method name="isSystemClass" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="systemClasses" type="java.util.List"/>
+      <doc>
+      <![CDATA[Checks if a class should be included as a system class.
+
+ A class is a system class if and only if it matches one of the positive
+ patterns and none of the negative ones.
+
+ @param name the class name to check
+ @param systemClasses a list of system class configurations.
+ @return true if the class is a system class]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_CLASSES_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value of the system classes if the user did not override them.
+ JDK classes, hadoop classes and resources, and some select third-party
+ classes are considered system classes, and are not loaded by the
+ application classloader.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A {@link URLClassLoader} for application isolation. Classes from the
+ application JARs are loaded in preference to the parent loader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ApplicationClassLoader -->
+  <!-- start class org.apache.hadoop.util.DurationInfo -->
+  <class name="DurationInfo" extends="org.apache.hadoop.util.OperationDuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="DurationInfo" type="org.slf4j.Logger, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the duration text from a {@code String.format()} code call;
+ log output at info level.
+ @param log log to write to
+ @param format format string
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <constructor name="DurationInfo" type="org.slf4j.Logger, boolean, java.lang.String, java.lang.Object[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the duration text from a {@code String.format()} code call
+ and log either at info or debug.
+ @param log log to write to
+ @param logAtInfo should the log be at info, rather than debug
+ @param format format string
+ @param args list of arguments]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A duration with logging of final state at info or debug
+ in the {@code close()} call.
+ This allows it to be used in a try-with-resources clause, and have the
+ duration automatically logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.DurationInfo -->
+  <!-- start interface org.apache.hadoop.util.IPList -->
+  <interface name="IPList"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="isIn" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ipAddress" type="java.lang.String"/>
+      <doc>
+      <![CDATA[returns true if the ipAddress is in the IPList.
+ @param ipAddress
+ @return boolean value indicating whether the ipAddress is in the IPList]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.IPList -->
+  <!-- start class org.apache.hadoop.util.OperationDuration -->
+  <class name="OperationDuration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OperationDuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Instantiate.
+ The start time and finished time are both set
+ to the current clock time.]]>
+      </doc>
+    </constructor>
+    <method name="time" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Evaluate the system time.
+ @return the current clock time.]]>
+      </doc>
+    </method>
+    <method name="finished"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the finished time with the current system time.]]>
+      </doc>
+    </method>
+    <method name="getDurationString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the duration as {@link #humanTime(long)}.
+ @return a printable duration.]]>
+      </doc>
+    </method>
+    <method name="humanTime" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+      <doc>
+      <![CDATA[Convert to a human time of minutes:seconds.millis.
+ @param time time to humanize.
+ @return a printable value.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the duration as {@link #humanTime(long)}.
+ @return a printable duration.]]>
+      </doc>
+    </method>
+    <method name="value" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds.
+ <p></p>
+ This will be 0 until a call
+ to {@link #finished()} has been made.
+ @return the currently recorded duration.]]>
+      </doc>
+    </method>
+    <method name="asDuration" return="java.time.Duration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration of an operation as a java Duration
+ instance.
+ @return a duration.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Little duration counter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.OperationDuration -->
+  <!-- start interface org.apache.hadoop.util.Progressable -->
+  <interface name="Progressable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="progress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Report progress to the Hadoop framework.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A facility for reporting progress.
+ 
+ <p>Clients and/or applications can use the provided <code>Progressable</code>
+ to explicitly report progress to the Hadoop framework. This is especially
+ important for operations which take significant amount of time since,
+ in-lieu of the reported progress, the framework has to assume that an error
+ has occurred and time-out the operation.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Progressable -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32 -->
+  <class name="PureJavaCrc32" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="offset" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the same polynomial as the built-in native CRC32.
+
+ This is to avoid the JNI overhead for certain uses of Checksumming
+ where many small pieces of data are checksummed in succession.
+
+ The current version is ~10x to 1.8x as fast as Sun's native
+ java.util.zip.CRC32 in Java 1.6
+
+ @see java.util.zip.CRC32]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32 -->
+  <!-- start class org.apache.hadoop.util.PureJavaCrc32C -->
+  <class name="PureJavaCrc32C" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.zip.Checksum"/>
+    <constructor name="PureJavaCrc32C"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new PureJavaCrc32 object.]]>
+      </doc>
+    </constructor>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="off" type="int"/>
+      <param name="len" type="int"/>
+    </method>
+    <method name="update"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="b" type="int"/>
+    </method>
+    <doc>
+    <![CDATA[A pure-java implementation of the CRC32 checksum that uses
+ the CRC32-C polynomial, the same polynomial used by iSCSI
+ and implemented on many Intel chipsets supporting SSE4.2.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.PureJavaCrc32C -->
+  <!-- start class org.apache.hadoop.util.ReflectionUtils -->
+  <class name="ReflectionUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReflectionUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theObject" type="java.lang.Object"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Check and set 'configuration' if necessary.
+ 
+ @param theObject object for which to set configuration
+ @param conf Configuration]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create an object for the given class and initialize it from conf
+ 
+ @param theClass class of which an object is created
+ @param conf Configuration
+ @return a new object]]>
+      </doc>
+    </method>
+    <method name="setContentionTracing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="boolean"/>
+    </method>
+    <method name="printThreadInfo"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stream" type="java.io.PrintStream"/>
+      <param name="title" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Print all of the thread's information and stack traces.
+ 
+ @param stream the stream to
+ @param title a string title for the stack trace]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.apache.commons.logging.Log"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="logThreadInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <param name="title" type="java.lang.String"/>
+      <param name="minInterval" type="long"/>
+      <doc>
+      <![CDATA[Log the current thread stacks at INFO level.
+ @param log the logger that logs the stack trace
+ @param title a descriptive title for the call stacks
+ @param minInterval the minimum time from the last]]>
+      </doc>
+    </method>
+    <method name="getClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="T"/>
+      <doc>
+      <![CDATA[Return the correctly-typed {@link Class} of the given object.
+  
+ @param o object whose correctly-typed <code>Class</code> is to be obtained
+ @return the correctly typed <code>Class</code> of the given object.]]>
+      </doc>
+    </method>
+    <method name="copy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="src" type="T"/>
+      <param name="dst" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Make a copy of the writable object using serialization to a buffer
+ @param src the object to copy from
+ @param dst the object to copy into, which is destroyed
+ @return dst param (the copy)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cloneWritableInto"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.io.Writable"/>
+      <param name="src" type="org.apache.hadoop.io.Writable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDeclaredFieldsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared fields of a class including fields declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <method name="getDeclaredMethodsIncludingInherited" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clazz" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Gets all the declared methods of a class including methods declared in
+ superclasses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[General reflection utils]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ReflectionUtils -->
+  <!-- start class org.apache.hadoop.util.Shell -->
+  <class name="Shell" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Shell"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with no minimum interval between runs; stderr is
+ not merged with stdout.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an instance with a minimum interval between executions; stderr is
+ not merged with stdout.
+ @param interval interval in milliseconds between command executions.]]>
+      </doc>
+    </constructor>
+    <constructor name="Shell" type="long, boolean"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a shell instance which can be re-executed when the {@link #run()}
+ method is invoked with a given elapsed time between calls.
+
+ @param interval the minimum duration in milliseconds to wait before
+        re-executing the command. If set to 0, there is no minimum.
+ @param redirectErrorStream should the error stream be merged with
+        the normal output stream?]]>
+      </doc>
+    </constructor>
+    <method name="isJava7OrAbove" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="This call isn't needed any more: please remove uses of it.">
+      <doc>
+      <![CDATA[query to see if system is Java 7 or later.
+ Now that Hadoop requires Java 7 or later, this always returns true.
+ @deprecated This call isn't needed any more: please remove uses of it.
+ @return true, always.]]>
+      </doc>
+    </method>
+    <method name="isJavaVersionAtLeast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <doc>
+      <![CDATA[Query to see if major version of Java specification of the system
+ is equal or greater than the parameter.
+
+ @param version 8, 9, 10 etc.
+ @return comparison with system property, always true for 8]]>
+      </doc>
+    </method>
+    <method name="checkWindowsCommandLineLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if a given command (String[]) fits in the Windows maximum command
+ line length Note that the input is expected to already include space
+ delimiters, no extra count will be added for delimiters.
+
+ @param commands command parts, including any space delimiters]]>
+      </doc>
+    </method>
+    <method name="getGroupsCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's groups list.]]>
+      </doc>
+    </method>
+    <method name="getGroupsForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's groups list.
+ If the OS is not WINDOWS, the command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.]]>
+      </doc>
+    </method>
+    <method name="getGroupsIDForUserCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given user's group id list.
+ The command will get the user's primary group
+ first and finally get the groups list which includes the primary group.
+ i.e. the user's primary group will be included twice.
+ This command does not support Windows and will only return group names.]]>
+      </doc>
+    </method>
+    <method name="getUsersForNetgroupCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="netgroup" type="java.lang.String"/>
+      <doc>
+      <![CDATA[A command to get a given netgroup's user list.]]>
+      </doc>
+    </method>
+    <method name="getGetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a command to get permission information.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Return a command to set permission.]]>
+      </doc>
+    </method>
+    <method name="getSetPermissionCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="perm" type="java.lang.String"/>
+      <param name="recursive" type="boolean"/>
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set permission for specific file.
+
+ @param perm String permission to set
+ @param recursive boolean true to apply to all sub-directories recursively
+ @param file String file to set
+ @return String[] containing command and arguments]]>
+      </doc>
+    </method>
+    <method name="getSetOwnerCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to set owner.]]>
+      </doc>
+    </method>
+    <method name="getSymlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="target" type="java.lang.String"/>
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to create symbolic links.]]>
+      </doc>
+    </method>
+    <method name="getReadlinkCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="link" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to read the target of the a symbolic link.]]>
+      </doc>
+    </method>
+    <method name="getCheckProcessIsAliveCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command for determining if process with specified pid is alive.
+ @param pid process ID
+ @return a <code>kill -0</code> command or equivalent]]>
+      </doc>
+    </method>
+    <method name="getSignalKillCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <param name="pid" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return a command to send a signal to a given pid.]]>
+      </doc>
+    </method>
+    <method name="getEnvironmentVariableRegex" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a regular expression string that match environment variables.]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.io.File"/>
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a File referencing a script with the given basename, inside the
+ given parent directory.  The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param parent File parent directory
+ @param basename String script file basename
+ @return File referencing the script in the directory]]>
+      </doc>
+    </method>
+    <method name="appendScriptExtension" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="basename" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns a script file name with the given basename.
+
+ The file extension is inferred by platform:
+ <code>".cmd"</code> on Windows, or <code>".sh"</code> otherwise.
+
+ @param basename String script file basename
+ @return String script file name]]>
+      </doc>
+    </method>
+    <method name="getRunScriptCommand" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="script" type="java.io.File"/>
+      <doc>
+      <![CDATA[Returns a command to run the given script.  The script interpreter is
+ inferred by platform: cmd on Windows or bash otherwise.
+
+ @param script File script to run
+ @return String[] command to run the script]]>
+      </doc>
+    </method>
+    <method name="getHadoopHome" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the Hadoop home directory. Raises an exception if not found
+ @return the home dir
+ @throws IOException if the home directory cannot be located.]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBin" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist]]>
+      </doc>
+    </method>
+    <method name="getQualifiedBinPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executable" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fully qualify the path to a binary that should be in a known hadoop
+  bin location. This is primarily useful for disambiguating call-outs
+  to executable sub-components of Hadoop to avoid clashes with other
+  executables that may be in the path.  Caveat:  this call doesn't
+  just format the path to the bin directory.  It also checks for file
+  existence of the composed path. The output of this call should be
+  cached by callers.
+
+ @param executable executable
+ @return executable file reference
+ @throws FileNotFoundException if the path does not exist
+ @throws IOException on path canonicalization failures]]>
+      </doc>
+    </method>
+    <method name="hasWinutilsPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate to indicate whether or not the path to winutils is known.
+
+ If true, then {@link #WINUTILS} is non-null, and both
+ {@link #getWinUtilsPath()} and {@link #getWinUtilsFile()}
+ will successfully return this value. Always false on non-windows systems.
+ @return true if there is a valid path to the binary]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsPath" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Locate the winutils binary, or fail with a meaningful
+ exception and stack trace as an RTE.
+ This method is for use in methods which don't explicitly throw
+ an <code>IOException</code>.
+ @return the path to {@link #WINUTILS_EXE}
+ @throws RuntimeException if the path is not resolvable]]>
+      </doc>
+    </method>
+    <method name="getWinUtilsFile" return="java.io.File"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <doc>
+      <![CDATA[Get a file reference to winutils.
+ Always raises an exception if there isn't one
+ @return the file instance referring to the winutils bin.
+ @throws FileNotFoundException on any failure to locate that file.]]>
+      </doc>
+    </method>
+    <method name="checkIsBashSupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+    </method>
+    <method name="setEnvironment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the environment for the command.
+ @param env Mapping of environment variables]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dir" type="java.io.File"/>
+      <doc>
+      <![CDATA[Set the working directory.
+ @param dir The directory where the command will be executed]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check to see if a command needs to be executed and execute if needed.]]>
+      </doc>
+    </method>
+    <method name="getExecString" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[return an array containing the command name and its parameters.]]>
+      </doc>
+    </method>
+    <method name="parseExecResult"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="lines" type="java.io.BufferedReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the execution result]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get an environment variable.
+ @param env the environment var
+ @return the value or null if it was unset.]]>
+      </doc>
+    </method>
+    <method name="getProcess" return="java.lang.Process"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the current sub-process executing the given command.
+ @return process executing the command]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the exit code.
+ @return the exit code of the process]]>
+      </doc>
+    </method>
+    <method name="getWaitingThread" return="java.lang.Thread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[get the thread that is waiting on this instance of <code>Shell</code>.
+ @return the thread that ran runCommand() that spawned this shell
+ or null if no thread is waiting for this shell to complete]]>
+      </doc>
+    </method>
+    <method name="isTimedOut" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[To check if the passed script to shell command executor timed out or
+ not.
+
+ @return if the script timed out.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param cmd shell command to execute.
+ @return the output of the executed command.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <param name="timeout" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @param timeout time in milliseconds after which script should be marked timeout
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="execCommand" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="env" type="java.util.Map"/>
+      <param name="cmd" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Static method to execute a shell command.
+ Covers most of the simple cases without requiring the user to implement
+ the <code>Shell</code> interface.
+ @param env the map of environment key=value
+ @param cmd shell command to execute.
+ @return the output of the executed command.
+ @throws IOException on any problem.]]>
+      </doc>
+    </method>
+    <method name="destroyAllShellProcesses"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to destroy all running <code>Shell</code> processes.
+ Iterates through a map of all currently running <code>Shell</code>
+ processes and destroys them one by one. This method is thread safe]]>
+      </doc>
+    </method>
+    <method name="getAllShells" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Static method to return a Set of all <code>Shell</code> objects.]]>
+      </doc>
+    </method>
+    <method name="getMemlockLimit" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ulimit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Static method to return the memory lock limit for datanode.
+ @param ulimit max value at which memory locked should be capped.
+ @return long value specifying the memory lock limit.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSPROP_HADOOP_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[System property for the Hadoop home directory: {@value}.]]>
+      </doc>
+    </field>
+    <field name="ENV_HADOOP_HOME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variable for Hadoop's home dir: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum command line length in Windows
+ KB830473 documents this as 8191]]>
+      </doc>
+    </field>
+    <field name="WINDOWS_MAX_SHELL_LENGHT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use the correctly spelled constant.">
+      <doc>
+      <![CDATA[mis-spelling of {@link #WINDOWS_MAX_SHELL_LENGTH}.
+ @deprecated use the correctly spelled constant.]]>
+      </doc>
+    </field>
+    <field name="USER_NAME_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get the current user's name: {@value}.]]>
+      </doc>
+    </field>
+    <field name="WindowsProcessLaunchLock" type="java.lang.Object"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Windows <code>CreateProcess</code> synchronization object.]]>
+      </doc>
+    </field>
+    <field name="osType" type="org.apache.hadoop.util.Shell.OSType"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the operating system, as determined from parsing
+ the <code>os.name</code> property.]]>
+      </doc>
+    </field>
+    <field name="WINDOWS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SOLARIS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FREEBSD" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OTHER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PPC_64" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENV_NAME_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Regular expression for environment variables: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_PERMISSION_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set permission: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_OWNER_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set owner: {@value}.]]>
+      </doc>
+    </field>
+    <field name="SET_GROUP_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to set the change user's groups list: {@value}.]]>
+      </doc>
+    </field>
+    <field name="LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to create a link: {@value}.]]>
+      </doc>
+    </field>
+    <field name="READ_LINK_COMMAND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[a Unix command to get a link target: {@value}.]]>
+      </doc>
+    </field>
+    <field name="timeOutInterval" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time after which the executing script would be timedout.]]>
+      </doc>
+    </field>
+    <field name="inheritParentEnv" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if the parent env vars should be inherited or not]]>
+      </doc>
+    </field>
+    <field name="WINUTILS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}">
+      <doc>
+      <![CDATA[Location of winutils as a string; null if not found.
+ <p>
+ <i>Important: caller must check for this value being null</i>.
+ The lack of such checks has led to many support issues being raised.
+ <p>
+ @deprecated use one of the exception-raising getter methods,
+ specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]>
+      </doc>
+    </field>
+    <field name="isSetsidAvailable" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag which is true if setsid exists.]]>
+      </doc>
+    </field>
+    <field name="TOKEN_SEPARATOR_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Token separator regex used to parse Shell tool outputs.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for running a Shell command.
+
+ <code>Shell</code> can be used to run shell commands like <code>du</code> or
+ <code>df</code>. It also offers facilities to gate commands by
+ time-intervals.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.Shell -->
+  <!-- start class org.apache.hadoop.util.ShutdownHookManager -->
+  <class name="ShutdownHookManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="get" return="org.apache.hadoop.util.ShutdownHookManager"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return <code>ShutdownHookManager</code> singleton.
+
+ @return <code>ShutdownHookManager</code> singleton.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority, the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook.]]>
+      </doc>
+    </method>
+    <method name="addShutdownHook"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <param name="priority" type="int"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Adds a shutdownHook with a priority and timeout the higher the priority
+ the earlier will run. ShutdownHooks with same priority run
+ in a non-deterministic order. The shutdown hook will be terminated if it
+ has not been finished in the specified period of time.
+
+ @param shutdownHook shutdownHook <code>Runnable</code>
+ @param priority priority of the shutdownHook
+ @param timeout timeout of the shutdownHook
+ @param unit unit of the timeout <code>TimeUnit</code>]]>
+      </doc>
+    </method>
+    <method name="removeShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Removes a shutdownHook.
+
+ @param shutdownHook shutdownHook to remove.
+ @return TRUE if the shutdownHook was registered and removed,
+ FALSE otherwise.]]>
+      </doc>
+    </method>
+    <method name="hasShutdownHook" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shutdownHook" type="java.lang.Runnable"/>
+      <doc>
+      <![CDATA[Indicates if a shutdownHook is registered or not.
+
+ @param shutdownHook shutdownHook to check if registered.
+ @return TRUE/FALSE depending if the shutdownHook is is registered.]]>
+      </doc>
+    </method>
+    <method name="isShutdownInProgress" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if shutdown is in progress or not.
+ 
+ @return TRUE if the shutdown is in progress, otherwise FALSE.]]>
+      </doc>
+    </method>
+    <method name="clearShutdownHooks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[clear all registered shutdownHooks.]]>
+      </doc>
+    </method>
+    <field name="TIMEOUT_MINIMUM" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum shutdown timeout: {@value} second(s).]]>
+      </doc>
+    </field>
+    <field name="TIME_UNIT_DEFAULT" type="java.util.concurrent.TimeUnit"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default time unit used: seconds.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[The <code>ShutdownHookManager</code> enables running shutdownHook
+ in a deterministic order, higher priority first.
+ <p>
+ The JVM runs ShutdownHooks in a non-deterministic order or in parallel.
+ This class registers a single JVM shutdownHook and run all the
+ shutdownHooks registered to it (to this class) in order based on their
+ priority.
+
+ Unless a hook was registered with a shutdown explicitly set through
+ {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ the shutdown time allocated to it is set by the configuration option
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in
+ {@code core-site.xml}, with a default value of
+ {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ seconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ShutdownHookManager -->
+  <!-- start class org.apache.hadoop.util.StringInterner -->
+  <class name="StringInterner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StringInterner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="strongIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains strong reference to the instance, 
+ thus preventing it from being garbage-collected. 
+ 
+ @param sample string instance to be interned
+ @return strong reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="weakIntern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sample" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Interns and returns a reference to the representative instance 
+ for any of a collection of string instances that are equal to each other.
+ Retains weak reference to the instance, 
+ and so does not prevent it from being garbage-collected.
+ 
+ @param sample string instance to be interned
+ @return weak reference to interned string instance]]>
+      </doc>
+    </method>
+    <method name="internStringsInArray" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="strings" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Interns all the strings in the given array in place,
+ returning the same array.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides string interning utility methods. For weak interning,
+ we use the standard String.intern() call, that performs very well
+ (no problems with PermGen overflowing, etc.) starting from JDK 7.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.StringInterner -->
+  <!-- start class org.apache.hadoop.util.SysInfo -->
+  <class name="SysInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SysInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.util.SysInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return default OS instance.
+ @throws UnsupportedOperationException If cannot determine OS.
+ @return Default instance for the detected OS.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the virtual memory present in the system.
+
+ @return virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the physical memory present in the system.
+
+ @return physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailableVirtualMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available virtual memory present
+ in the system.
+
+ @return available virtual memory size in bytes.]]>
+      </doc>
+    </method>
+    <method name="getAvailablePhysicalMemorySize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total size of the available physical memory present
+ in the system.
+
+ @return available physical memory size bytes.]]>
+      </doc>
+    </method>
+    <method name="getNumProcessors" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the total number of logical processors present on the system.
+
+ @return number of logical processors]]>
+      </doc>
+    </method>
+    <method name="getNumCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain total number of physical cores present on the system.
+
+ @return number of physical cores]]>
+      </doc>
+    </method>
+    <method name="getCpuFrequency" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU frequency of on the system.
+
+ @return CPU frequency in kHz]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the cumulative CPU time since the system is on.
+
+ @return cumulative CPU time in milliseconds]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the CPU usage % of the machine. Return -1 if it is unavailable
+
+ @return CPU usage as a percentage (from 0 to 100) of available cycles.]]>
+      </doc>
+    </method>
+    <method name="getNumVCoresUsed" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the number of VCores used. Return -1 if it is unavailable
+
+ @return Number of VCores used a percentage (from 0 to #VCores).]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read over the network.
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getNetworkBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to the network.
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesRead" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes read from disks.
+
+ @return total number of bytes read.]]>
+      </doc>
+    </method>
+    <method name="getStorageBytesWritten" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain the aggregated number of bytes written to disks.
+
+ @return total number of bytes written.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Plugin to calculate resource information on the system.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.SysInfo -->
+  <!-- start interface org.apache.hadoop.util.Tool -->
+  <interface name="Tool"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="run" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Execute the command with the given arguments.
+ 
+ @param args command specific arguments.
+ @return exit code.
+ @throws Exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A tool interface that supports handling of generic command-line options.
+ 
+ <p><code>Tool</code>, is the standard for any Map-Reduce tool/application. 
+ The tool/application should delegate the handling of 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ standard command-line options</a> to {@link ToolRunner#run(Tool, String[])} 
+ and only handle its custom arguments.</p>
+ 
+ <p>Here is how a typical <code>Tool</code> is implemented:</p>
+ <p><blockquote><pre>
+     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // <code>Configuration</code> processed by <code>ToolRunner</code>
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed <code>conf</code>
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let <code>ToolRunner</code> handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+ </pre></blockquote><p>
+ 
+ @see GenericOptionsParser
+ @see ToolRunner]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.Tool -->
+  <!-- start class org.apache.hadoop.util.ToolRunner -->
+  <class name="ToolRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ToolRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the given <code>Tool</code> by {@link Tool#run(String[])}, after 
+ parsing with the given generic arguments. Uses the given 
+ <code>Configuration</code>, or builds one if null.
+ 
+ Sets the <code>Tool</code>'s configuration with the possibly modified 
+ version of the <code>conf</code>.  
+ 
+ @param conf <code>Configuration</code> for the <code>Tool</code>.
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tool" type="org.apache.hadoop.util.Tool"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Runs the <code>Tool</code> with its <code>Configuration</code>.
+ 
+ Equivalent to <code>run(tool.getConf(), tool, args)</code>.
+ 
+ @param tool <code>Tool</code> to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.]]>
+      </doc>
+    </method>
+    <method name="printGenericCommandUsage"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.PrintStream"/>
+      <doc>
+      <![CDATA[Prints generic command-line argurments and usage information.
+ 
+  @param out stream to write usage information to.]]>
+      </doc>
+    </method>
+    <method name="confirmPrompt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prompt" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Print out a prompt to the user, and return true if the user
+ responds with "y" or "yes". (case insensitive)]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A utility to help run {@link Tool}s.
+ 
+ <p><code>ToolRunner</code> can be used to run classes implementing 
+ <code>Tool</code> interface. It works in conjunction with 
+ {@link GenericOptionsParser} to parse the 
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
+ generic hadoop command line arguments</a> and modifies the 
+ <code>Configuration</code> of the <code>Tool</code>. The 
+ application-specific options are passed along without being modified.
+ </p>
+ 
+ @see Tool
+ @see GenericOptionsParser]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.ToolRunner -->
+  <!-- start class org.apache.hadoop.util.VersionInfo -->
+  <class name="VersionInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="VersionInfo" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="_getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="_getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Hadoop version.
+ @return the Hadoop version string, eg. "0.6.3-dev"]]>
+      </doc>
+    </method>
+    <method name="getRevision" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Git commit hash of the repository when compiled.
+ @return the commit hash, eg. "18f64065d5db6208daf50b02c1b5ed4ee3ce547a"]]>
+      </doc>
+    </method>
+    <method name="getBranch" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the branch on which this originated.
+ @return The branch name, e.g. "trunk" or "branches/branch-0.20"]]>
+      </doc>
+    </method>
+    <method name="getDate" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The date that Hadoop was compiled.
+ @return the compilation date in unix date format]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user that compiled Hadoop.
+ @return the username of the user]]>
+      </doc>
+    </method>
+    <method name="getUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL for the Hadoop repository.
+ @return the URL of the Hadoop repository]]>
+      </doc>
+    </method>
+    <method name="getSrcChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the checksum of the source files from which Hadoop was built.
+ @return the checksum of the source files]]>
+      </doc>
+    </method>
+    <method name="getBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the buildVersion which includes version,
+ revision, user and date.
+ @return the buildVersion]]>
+      </doc>
+    </method>
+    <method name="getProtocVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the protoc version used for the build.
+ @return the protoc version]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+    </method>
+    <doc>
+    <![CDATA[This class returns build information about Hadoop components.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.VersionInfo -->
+  <doc>
+  <![CDATA[Common utilities.]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.util.bloom">
+  <!-- start class org.apache.hadoop.util.bloom.BloomFilter -->
+  <class name="BloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="BloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVectorSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return size of the the bloomfilter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>Bloom filter</i>, as defined by Bloom in 1970.
+ <p>
+ The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by 
+ the networking research community in the past decade thanks to the bandwidth efficiencies that it
+ offers for the transmission of set membership information between networked hosts.  A sender encodes 
+ the information into a bit vector, the Bloom filter, that is more compact than a conventional 
+ representation. Computation and space costs for construction are linear in the number of elements.  
+ The receiver uses the filter to test whether various elements are members of the set. Though the 
+ filter will occasionally return a false positive, it will never return a false negative. When creating 
+ the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. 
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=362692&dl=ACM&coll=portal">Space/Time Trade-Offs in Hash Coding with Allowable Errors</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.BloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <class name="CountingBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CountingBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="CountingBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="delete"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Removes a specified key from <i>this</i> counting Bloom filter.
+ <p>
+ <b>Invariant</b>: nothing happens if the specified key does not belong to <i>this</i> counter Bloom filter.
+ @param key The key to remove.]]>
+      </doc>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="approximateCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[This method calculates an approximate count of the key, i.e. how many
+ times the key was added to the filter. This allows the filter to be
+ used as an approximate <code>key -&gt; count</code> map.
+ <p>NOTE: due to the bucket size of this filter, inserting the same
+ key more than 15 times will cause an overflow at all filter positions
+ associated with this key, and it will significantly increase the error
+ rate for this and other keys. For this reason the filter can only be
+ used to store small count values <code>0 &lt;= N &lt;&lt; 15</code>.
+ @param key key to be tested
+ @return 0 if the key is not present. Otherwise, a positive value v will
+ be returned such that <code>v == count</code> with probability equal to the
+ error rate of this filter, and <code>v &gt; count</code> otherwise.
+ Additionally, if the filter experienced an underflow as a result of
+ {@link #delete(Key)} operation, the return value may be lower than the
+ <code>count</code> with the probability of the false negative rate of such
+ filter.]]>
+      </doc>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>counting Bloom filter</i>, as defined by Fan et al. in a ToN
+ 2000 paper.
+ <p>
+ A counting Bloom filter is an improvement to standard a Bloom filter as it
+ allows dynamic additions and deletions of set membership information.  This 
+ is achieved through the use of a counting vector instead of a bit vector.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ 
+ @see <a href="http://portal.acm.org/citation.cfm?id=343571.343572">Summary cache: a scalable wide-area web cache sharing protocol</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.CountingBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <class name="DynamicBloomFilter" extends="org.apache.hadoop.util.bloom.Filter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DynamicBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zero-args constructor for the serialization.]]>
+      </doc>
+    </constructor>
+    <constructor name="DynamicBloomFilter" type="int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds an empty Dynamic Bloom filter.
+ @param vectorSize The number of bits in the vector.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).
+ @param nr The threshold for the maximum number of keys to record in a
+ dynamic Bloom filter row.]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="and"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="membershipTest" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="not"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="or"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="xor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filter" type="org.apache.hadoop.util.bloom.Filter"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>dynamic Bloom filter</i>, as defined in the INFOCOM 2006 paper.
+ <p>
+ A dynamic Bloom filter (DBF) makes use of a <code>s * m</code> bit matrix but
+ each of the <code>s</code> rows is a standard Bloom filter. The creation 
+ process of a DBF is iterative. At the start, the DBF is a <code>1 * m</code>
+ bit matrix, i.e., it is composed of a single standard Bloom filter.
+ It assumes that <code>n<sub>r</sub></code> elements are recorded in the 
+ initial bit vector, where <code>n<sub>r</sub> {@literal <=} n</code>
+ (<code>n</code> is the cardinality of the set <code>A</code> to record in
+ the filter).
+ <p>
+ As the size of <code>A</code> grows during the execution of the application,
+ several keys must be inserted in the DBF.  When inserting a key into the DBF,
+ one must first get an active Bloom filter in the matrix.  A Bloom filter is
+ active when the number of recorded keys, <code>n<sub>r</sub></code>, is 
+ strictly less than the current cardinality of <code>A</code>, <code>n</code>.
+ If an active Bloom filter is found, the key is inserted and 
+ <code>n<sub>r</sub></code> is incremented by one. On the other hand, if there
+ is no active Bloom filter, a new one is created (i.e., a new row is added to
+ the matrix) according to the current size of <code>A</code> and the element
+ is added in this new Bloom filter and the <code>n<sub>r</sub></code> value of
+ this new Bloom filter is set to one.  A given key is said to belong to the
+ DBF if the <code>k</code> positions are set to one in one of the matrix rows.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ 
+ @see <a href="http://www.cse.fau.edu/~jie/research/publications/Publication_files/infocom2006.pdf">Theory and Network Applications of Dynamic Bloom Filters</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.DynamicBloomFilter -->
+  <!-- start class org.apache.hadoop.util.bloom.HashFunction -->
+  <class name="HashFunction" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashFunction" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+ <p>
+ Builds a hash function that must obey to a given maximum number of returned values and a highest value.
+ @param maxValue The maximum highest returned value.
+ @param nbHash The number of resulting hashed values.
+ @param hashType type of the hashing function (see {@link Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Clears <i>this</i> hash function. A NOOP]]>
+      </doc>
+    </method>
+    <method name="hash" return="int[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Hashes a specified key into several integers.
+ @param k The specified key.
+ @return The array of hashed values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements a hash object that returns a certain number of hashed values.
+ 
+ @see Key The general behavior of a key being stored in a filter
+ @see Filter The general behavior of a filter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.HashFunction -->
+  <!-- start interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <interface name="RemoveScheme"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="RANDOM" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Random selection.
+ <p>
+ The idea is to randomly select a bit to reset.]]>
+      </doc>
+    </field>
+    <field name="MINIMUM_FN" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MinimumFN Selection.
+ <p>
+ The idea is to select the bit to reset that will generate the minimum
+ number of false negative.]]>
+      </doc>
+    </field>
+    <field name="MAXIMUM_FP" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[MaximumFP Selection.
+ <p>
+ The idea is to select the bit to reset that will remove the maximum number
+ of false positive.]]>
+      </doc>
+    </field>
+    <field name="RATIO" type="short"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Ratio Selection.
+ <p>
+ The idea is to select the bit to reset that will, at the same time, remove
+ the maximum number of false positve while minimizing the amount of false
+ negative generated.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Defines the different remove scheme for retouched Bloom filters.
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.util.bloom.RemoveScheme -->
+  <!-- start class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+  <class name="RetouchedBloomFilter" extends="org.apache.hadoop.util.bloom.BloomFilter"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.bloom.RemoveScheme"/>
+    <constructor name="RetouchedBloomFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor - use with readFields]]>
+      </doc>
+    </constructor>
+    <constructor name="RetouchedBloomFilter" type="int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+ @param vectorSize The vector size of <i>this</i> filter.
+ @param nbHash The number of hash function to consider.
+ @param hashType type of the hashing function (see
+ {@link org.apache.hadoop.util.hash.Hash}).]]>
+      </doc>
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.util.bloom.Key"/>
+      <doc>
+      <![CDATA[Adds a false positive information to <i>this</i> retouched Bloom filter.
+ <p>
+ <b>Invariant</b>: if the false positive is <code>null</code>, nothing happens.
+ @param key The false positive key to add.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="coll" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Adds a collection of false positive information to <i>this</i> retouched Bloom filter.
+ @param coll The collection of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="java.util.List"/>
+      <doc>
+      <![CDATA[Adds a list of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The list of false positive.]]>
+      </doc>
+    </method>
+    <method name="addFalsePositive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keys" type="org.apache.hadoop.util.bloom.Key[]"/>
+      <doc>
+      <![CDATA[Adds an array of false positive information to <i>this</i> retouched Bloom filter.
+ @param keys The array of false positive.]]>
+      </doc>
+    </method>
+    <method name="selectiveClearing"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="k" type="org.apache.hadoop.util.bloom.Key"/>
+      <param name="scheme" type="short"/>
+      <doc>
+      <![CDATA[Performs the selective clearing for a given key.
+ @param k The false positive key to remove from <i>this</i> retouched Bloom filter.
+ @param scheme The selective clearing scheme to apply.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implements a <i>retouched Bloom filter</i>, as defined in the CoNEXT 2006 paper.
+ <p>
+ It allows the removal of selected false positives at the cost of introducing
+ random false negatives, and with the benefit of eliminating some random false
+ positives at the same time.
+ 
+ <p>
+ Originally created by
+ <a href="http://www.one-lab.org">European Commission One-Lab Project 034819</a>.
+ 
+ @see Filter The general behavior of a filter
+ @see BloomFilter A Bloom filter
+ @see RemoveScheme The different selective clearing algorithms
+ 
+ @see <a href="http://www-rp.lip6.fr/site_npa/site_rp/_publications/740-rbf_cameraready.pdf">Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives</a>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.bloom.RetouchedBloomFilter -->
+</package>
+<package name="org.apache.hadoop.util.curator">
+</package>
+<package name="org.apache.hadoop.util.functional">
+  <!-- start class org.apache.hadoop.util.functional.FutureIO -->
+  <class name="FutureIO" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="awaitFuture" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="future" type="java.util.concurrent.Future"/>
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="RuntimeException" type="java.lang.RuntimeException"/>
+      <doc>
+      <![CDATA[Given a future, evaluate it.
+ <p>
+ Any exception generated in the future is
+ extracted and rethrown.
+ </p>
+ @param future future to evaluate
+ @param <T> type of the result.
+ @return the result, if all went well.
+ @throws InterruptedIOException future was interrupted
+ @throws IOException if something went wrong
+ @throws RuntimeException any nested RTE thrown]]>
+      </doc>
+    </method>
+    <method name="awaitFuture" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="future" type="java.util.concurrent.Future"/>
+      <param name="timeout" type="long"/>
+      <param name="unit" type="java.util.concurrent.TimeUnit"/>
+      <exception name="InterruptedIOException" type="java.io.InterruptedIOException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="RuntimeException" type="java.lang.RuntimeException"/>
+      <exception name="TimeoutException" type="java.util.concurrent.TimeoutException"/>
+      <doc>
+      <![CDATA[Given a future, evaluate it.
+ <p>
+ Any exception generated in the future is
+ extracted and rethrown.
+ </p>
+ @param future future to evaluate
+ @param <T> type of the result.
+ @return the result, if all went well.
+ @throws InterruptedIOException future was interrupted
+ @throws IOException if something went wrong
+ @throws RuntimeException any nested RTE thrown
+ @throws TimeoutException the future timed out.]]>
+      </doc>
+    </method>
+    <method name="raiseInnerCause" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.util.concurrent.ExecutionException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[From the inner cause of an execution exception, extract the inner cause
+ if it is an IOE or RTE.
+ This will always raise an exception, either the inner IOException,
+ an inner RuntimeException, or a new IOException wrapping the raised
+ exception.
+
+ @param e exception.
+ @param <T> type of return value.
+ @return nothing, ever.
+ @throws IOException either the inner IOException, or a wrapper around
+ any non-Runtime-Exception
+ @throws RuntimeException if that is the inner cause.]]>
+      </doc>
+    </method>
+    <method name="raiseInnerCause" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.util.concurrent.CompletionException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Extract the cause of a completion failure and rethrow it if an IOE
+ or RTE.
+ @param e exception.
+ @param <T> type of return value.
+ @return nothing, ever.
+ @throws IOException either the inner IOException, or a wrapper around
+ any non-Runtime-Exception
+ @throws RuntimeException if that is the inner cause.]]>
+      </doc>
+    </method>
+    <method name="unwrapInnerException" return="java.io.IOException"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="e" type="java.lang.Throwable"/>
+      <doc>
+      <![CDATA[From the inner cause of an execution exception, extract the inner cause
+ to an IOException, raising RuntimeExceptions and Errors immediately.
+ <ol>
+   <li> If it is an IOE: Return.</li>
+   <li> If it is a {@link UncheckedIOException}: return the cause</li>
+   <li> Completion/Execution Exceptions: extract and repeat</li>
+   <li> If it is an RTE or Error: throw.</li>
+   <li> Any other type: wrap in an IOE</li>
+ </ol>
+
+ Recursively handles wrapped Execution and Completion Exceptions in
+ case something very complicated has happened.
+ @param e exception.
+ @return an IOException extracted or built from the cause.
+ @throws RuntimeException if that is the inner cause.
+ @throws Error if that is the inner cause.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Future IO Helper methods.
+ <p>
+ Contains methods promoted from
+ {@link org.apache.hadoop.fs.impl.FutureIOSupport} because they
+ are a key part of integrating async IO in application code.
+ </p>
+ <p>
+ One key feature is that the {@link #awaitFuture(Future)} and
+ {@link #awaitFuture(Future, long, TimeUnit)} calls will
+ extract and rethrow exceptions raised in the future's execution,
+ including extracting the inner IOException of any
+ {@code UncheckedIOException} raised in the future.
+ This makes it somewhat easier to execute IOException-raising
+ code inside futures.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.functional.FutureIO -->
+  <!-- start class org.apache.hadoop.util.functional.RemoteIterators -->
+  <class name="RemoteIterators" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="remoteIteratorFromSingleton" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="singleton" type="T"/>
+      <doc>
+      <![CDATA[Create an iterator from a singleton.
+ @param singleton instance
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="java.util.Iterator"/>
+      <doc>
+      <![CDATA[Create a remote iterator from a java.util.Iterator.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromIterable" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterable" type="java.lang.Iterable"/>
+      <doc>
+      <![CDATA[Create a remote iterator from a java.util.Iterable -e.g. a list
+ or other collection.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="remoteIteratorFromArray" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="array" type="T[]"/>
+      <doc>
+      <![CDATA[Create a remote iterator from an array.
+ @param <T> type
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="mappingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="mapper" type="org.apache.hadoop.util.functional.FunctionRaisingIOE"/>
+      <doc>
+      <![CDATA[Create an iterator from an iterator and a transformation function.
+ @param <S> source type
+ @param <T> result type
+ @param iterator source
+ @param mapper transformation
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="typeCastingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <doc>
+      <![CDATA[Create a RemoteIterator from a RemoteIterator, casting the
+ type in the process. This is to help with filesystem API
+ calls where overloading causes confusion (e.g. listStatusIterator())
+ @param <S> source type
+ @param <T> result type
+ @param iterator source
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="filteringRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="filter" type="org.apache.hadoop.util.functional.FunctionRaisingIOE"/>
+      <doc>
+      <![CDATA[Create a RemoteIterator from a RemoteIterator and a filter
+ function which returns true for every element to be passed
+ through.
+ <p></p>
+ Elements are filtered in the hasNext() method; if not used
+ the filtering will be done on demand in the {@code next()}
+ call.
+ @param <S> type
+ @param iterator source
+ @param filter filter
+ @return a remote iterator]]>
+      </doc>
+    </method>
+    <method name="closingRemoteIterator" return="org.apache.hadoop.fs.RemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="iterator" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="toClose" type="java.io.Closeable"/>
+      <doc>
+      <![CDATA[This adds an extra close operation alongside the passthrough
+ to any Closeable.close() method supported by the source iterator.
+ @param iterator source
+ @param toClose extra object to close.
+ @param <S> source type.
+ @return a new iterator]]>
+      </doc>
+    </method>
+    <method name="toList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a list from a RemoteIterator.
+ @param source source iterator
+ @param <T> type
+ @return a list of the values.
+ @throws IOException if the source RemoteIterator raises it.]]>
+      </doc>
+    </method>
+    <method name="toArray" return="T[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="a" type="T[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build an array from a RemoteIterator.
+ @param source source iterator
+ @param a destination array; if too small a new array
+ of the same type is created
+ @param <T> type
+ @return an array of the values.
+ @throws IOException if the source RemoteIterator raises it.]]>
+      </doc>
+    </method>
+    <method name="foreach" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <param name="consumer" type="org.apache.hadoop.util.functional.ConsumerRaisingIOE"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Apply an operation to all values of a RemoteIterator.
+ <p></p>
+ If the iterator is an IOStatisticsSource returning a non-null
+ set of statistics, <i>and</i> this classes log is set to DEBUG,
+ then the statistics of the operation are evaluated and logged at
+ debug.
+ <p></p>
+ The number of entries processed is returned, as it is useful to
+ know this, especially during tests or when reporting values
+ to users.
+ <p></p>
+ This does not close the iterator afterwards.
+ @param source iterator source
+ @param consumer consumer of the values.
+ @return the number of elements processed
+ @param <T> type of source
+ @throws IOException if the source RemoteIterator or the consumer raise one.]]>
+      </doc>
+    </method>
+    <method name="cleanupRemoteIterator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="source" type="org.apache.hadoop.fs.RemoteIterator"/>
+      <doc>
+      <![CDATA[Clean up after an iteration.
+ If the log is at debug, calculate and log the IOStatistics.
+ If the iterator is closeable, cast and then cleanup the iterator
+ @param source iterator source
+ @param <T> type of source]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A set of remote iterators supporting transformation and filtering,
+ with IOStatisticsSource passthrough, and of conversions of
+ the iterators to lists/arrays and of performing actions
+ on the values.
+ <p></p>
+ This aims to make it straightforward to use lambda-expressions to
+ transform the results of an iterator, without losing the statistics
+ in the process, and to chain the operations together.
+ <p></p>
+ The closeable operation will be passed through RemoteIterators which
+ wrap other RemoteIterators. This is to support any iterator which
+ can be closed to release held connections, file handles etc.
+ Unless client code is written to assume that RemoteIterator instances
+ may be closed, this is not likely to be broadly used. It is added
+ to make it possible to adopt this feature in a managed way.
+ <p></p>
+ One notable feature is that the
+ {@link #foreach(RemoteIterator, ConsumerRaisingIOE)} method will
+ LOG at debug any IOStatistics provided by the iterator, if such
+ statistics are provided. There's no attempt at retrieval and logging
+ if the LOG is not set to debug, so it is a zero cost feature unless
+ the logger {@code org.apache.hadoop.fs.functional.RemoteIterators}
+ is at DEBUG.
+ <p></p>
+ Based on the S3A Listing code, and some some work on moving other code
+ to using iterative listings so as to pick up the statistics.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.util.functional.RemoteIterators -->
+</package>
+<package name="org.apache.hadoop.util.hash">
+</package>
+
+</api>
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 6db6c2f5483a2..4391995d209f8 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -141,12 +141,39 @@
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-servlet</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.enterprise</groupId>
+          <artifactId>cdi-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.cal10n</groupId>
+          <artifactId>cal10n-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
-      <!-- Used, even though 'mvn dependency:analyze' doesn't find it -->
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey</groupId>
@@ -187,6 +214,12 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-configuration2</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -350,6 +383,11 @@
       <artifactId>mockwebserver</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio-jvm</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>dnsjava</groupId>
       <artifactId>dnsjava</artifactId>
@@ -651,40 +689,6 @@
           </filesets>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided implementations rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
-                    <bannedImport>com.google.common.base.Preconditions</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
@@ -842,6 +846,36 @@
               </execution>
             </executions>
           </plugin>
+          <plugin>
+            <!--Sets the skip.platformToolsetDetection to true if use.platformToolsetVersion is specified.
+            This implies that the automatic detection of which platform toolset to use will be skipped
+            and the one specified with use.platformToolsetVersion will be used.-->
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <version>1.8</version>
+            <executions>
+              <execution>
+                <phase>validate</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <exportAntProperties>true</exportAntProperties>
+                  <target>
+                    <condition property="skip.platformToolsetDetection" value="true" else="false">
+                      <isset property="use.platformToolsetVersion"/>
+                    </condition>
+                    <!--Unfortunately, Maven doesn't have a way to negate a flag, thus we declare a
+                     property which holds the negated value of skip.platformToolsetDetection.-->
+                    <condition property="skip.platformToolsetDetection.negated" value="false" else="true">
+                      <isset property="use.platformToolsetVersion"/>
+                    </condition>
+                    <echo>Skip platform toolset version detection = ${skip.platformToolsetDetection}</echo>
+                  </target>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>
@@ -853,6 +887,7 @@
                   <goal>exec</goal>
                 </goals>
                 <configuration>
+                  <skip>${skip.platformToolsetDetection}</skip>
                   <executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
                   <arguments>
                     <argument>${basedir}\src\main\winutils</argument>
@@ -867,6 +902,7 @@
                   <goal>exec</goal>
                 </goals>
                 <configuration>
+                  <skip>${skip.platformToolsetDetection}</skip>
                   <executable>msbuild</executable>
                   <arguments>
                     <argument>${basedir}/src/main/winutils/winutils.sln</argument>
@@ -879,6 +915,27 @@
                   </arguments>
                 </configuration>
               </execution>
+              <execution>
+                <id>compile-ms-winutils-using-build-tools</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <skip>${skip.platformToolsetDetection.negated}</skip>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/winutils/winutils.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                    <argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
+                    <argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
+                    <argument>/p:WsceConfigFile=${wsce.config.file}</argument>
+                    <argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
+                  </arguments>
+                </configuration>
+              </execution>
               <execution>
                 <id>convert-ms-native-dll</id>
                 <phase>generate-sources</phase>
@@ -886,6 +943,7 @@
                   <goal>exec</goal>
                 </goals>
                 <configuration>
+                  <skip>${skip.platformToolsetDetection}</skip>
                   <executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
                   <arguments>
                     <argument>${basedir}\src\main\native</argument>
@@ -900,6 +958,35 @@
                   <goal>exec</goal>
                 </goals>
                 <configuration>
+                  <skip>${skip.platformToolsetDetection}</skip>
+                  <executable>msbuild</executable>
+                  <arguments>
+                    <argument>${basedir}/src/main/native/native.sln</argument>
+                    <argument>/nologo</argument>
+                    <argument>/p:Configuration=Release</argument>
+                    <argument>/p:OutDir=${project.build.directory}/bin/</argument>
+                    <argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
+                    <argument>/p:CustomZstdLib=${zstd.lib}</argument>
+                    <argument>/p:CustomZstdInclude=${zstd.include}</argument>
+                    <argument>/p:RequireZstd=${require.zstd}</argument>
+                    <argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
+                    <argument>/p:CustomOpensslLib=${openssl.lib}</argument>
+                    <argument>/p:CustomOpensslInclude=${openssl.include}</argument>
+                    <argument>/p:RequireOpenssl=${require.openssl}</argument>
+                    <argument>/p:RequireIsal=${require.isal}</argument>
+                    <argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
+                    <argument>/p:CustomIsalLib=${isal.lib}</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+              <execution>
+                <id>compile-ms-native-dll-using-build-tools</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <skip>${skip.platformToolsetDetection.negated}</skip>
                   <executable>msbuild</executable>
                   <arguments>
                     <argument>${basedir}/src/main/native/native.sln</argument>
@@ -917,6 +1004,7 @@
                     <argument>/p:RequireIsal=${require.isal}</argument>
                     <argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
                     <argument>/p:CustomIsalLib=${isal.lib}</argument>
+                    <argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
                   </arguments>
                 </configuration>
               </execution>
@@ -949,7 +1037,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
               <forkCount>${testsThreadCount}</forkCount>
               <reuseForks>false</reuseForks>
               <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
@@ -1153,7 +1240,7 @@
                 <id>src-test-compile-protoc-legacy</id>
                 <phase>generate-test-sources</phase>
                 <goals>
-                  <goal>compile</goal>
+                  <goal>test-compile</goal>
                 </goals>
                 <configuration>
                   <skip>false</skip>
@@ -1162,7 +1249,7 @@
                     com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
                   </protocArtifact>
                   <includeDependenciesInDescriptorSet>false</includeDependenciesInDescriptorSet>
-                  <protoSourceRoot>${basedir}/src/test/proto</protoSourceRoot>
+                  <protoTestSourceRoot>${basedir}/src/test/proto</protoTestSourceRoot>
                   <outputDirectory>${project.build.directory}/generated-test-sources/java</outputDirectory>
                   <clearOutputDirectory>false</clearOutputDirectory>
                   <includes>
@@ -1173,6 +1260,16 @@
               </execution>
              </executions>
           </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <configuration>
+              <sourceFileExcludes>
+                <sourceFileExclude>**/FSProtos.java</sourceFileExclude>
+              </sourceFileExcludes>
+              <excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
+            </configuration>
+          </plugin>
         </plugins>
       </build>
     </profile>
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 5a2ca4d922852..54d5c729848c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -251,30 +251,45 @@ log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex}
 #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd
 
 # Http Server Request Logs
-#log4j.logger.http.requests.namenode=INFO,namenoderequestlog
-#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log
-#log4j.appender.namenoderequestlog.RetainDays=3
-
-#log4j.logger.http.requests.datanode=INFO,datanoderequestlog
-#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log
-#log4j.appender.datanoderequestlog.RetainDays=3
-
-#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog
-#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log
-#log4j.appender.resourcemanagerrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog
-#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log
-#log4j.appender.jobhistoryrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog
-#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
-#log4j.appender.nodemanagerrequestlog.RetainDays=3
+#log4j.appender.AccessNNDRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.AccessNNDRFA.File=${hadoop.log.dir}/jetty-namenode.log
+#log4j.appender.AccessNNDRFA.DatePattern=.yyyy-MM-dd
+#log4j.appender.AccessNNDRFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.AccessNNDRFA.layout.ConversionPattern=%m%n
+
+#log4j.logger.http.requests.namenode=INFO,AccessNNDRFA
+
+#log4j.appender.AccessDNDRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.AccessDNDRFA.File=${hadoop.log.dir}/jetty-datanode.log
+#log4j.appender.AccessDNDRFA.DatePattern=.yyyy-MM-dd
+#log4j.appender.AccessDNDRFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.AccessDNDRFA.layout.ConversionPattern=%m%n
+
+#log4j.logger.http.requests.datanode=INFO,AccessDNDRFA
+
+#log4j.appender.AccessRMDRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.AccessRMDRFA.File=${hadoop.log.dir}/jetty-resourcemanager.log
+#log4j.appender.AccessRMDRFA.DatePattern=.yyyy-MM-dd
+#log4j.appender.AccessRMDRFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.AccessRMDRFA.layout.ConversionPattern=%m%n
+
+#log4j.logger.http.requests.resourcemanager=INFO,AccessRMDRFA
+
+#log4j.appender.AccessJHDRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.AccessJHDRFA.File=${hadoop.log.dir}/jetty-jobhistory.log
+#log4j.appender.AccessJHDRFA.DatePattern=.yyyy-MM-dd
+#log4j.appender.AccessJHDRFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.AccessJHDRFA.layout.ConversionPattern=%m%n
+
+#log4j.logger.http.requests.jobhistory=INFO,AccessJHDRFA
+
+#log4j.appender.AccessNMDRFA=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.AccessNMDRFA.File=${hadoop.log.dir}/jetty-jobhistory.log
+#log4j.appender.AccessNMDRFA.DatePattern=.yyyy-MM-dd
+#log4j.appender.AccessNMDRFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.AccessNMDRFA.layout.ConversionPattern=%m%n
+
+#log4j.logger.http.requests.nodemanager=INFO,AccessNMDRFA
 
 # WebHdfs request log on datanodes
 # Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index 67d0e6301320a..b427038fdddc9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -98,7 +98,7 @@ static void writeResponse(Configuration conf,
     if (FORMAT_JSON.equals(format)) {
       Configuration.dumpConfiguration(conf, propertyName, out);
     } else if (FORMAT_XML.equals(format)) {
-      conf.writeXml(propertyName, out);
+      conf.writeXml(propertyName, out, conf);
     } else {
       throw new BadFormatException("Bad format: " + format);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
index 5b2d1449f9c86..1e74077c13977 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -37,6 +37,7 @@
 public class ConfigRedactor {
 
   private static final String REDACTED_TEXT = "<redacted>";
+  private static final String REDACTED_XML = "******";
 
   private List<Pattern> compiledPatterns;
 
@@ -57,8 +58,8 @@ public ConfigRedactor(Configuration conf) {
    * Given a key / value pair, decides whether or not to redact and returns
    * either the original value or text indicating it has been redacted.
    *
-   * @param key
-   * @param value
+   * @param key param key.
+   * @param value param value, will return if conditions permit.
    * @return Original value, or text indicating it has been redacted
    */
   public String redact(String key, String value) {
@@ -84,4 +85,19 @@ private boolean configIsSensitive(String key) {
     }
     return false;
   }
+
+  /**
+   * Given a key / value pair, decides whether or not to redact and returns
+   * either the original value or text indicating it has been redacted.
+   *
+   * @param key param key.
+   * @param value param value, will return if conditions permit.
+   * @return Original value, or text indicating it has been redacted
+   */
+  public String redactXml(String key, String value) {
+    if (configIsSensitive(key)) {
+      return REDACTED_XML;
+    }
+    return value;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 7ceaad37b1dcd..ab7ff0bd40cc2 100755
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -24,7 +24,6 @@
 import com.ctc.wstx.stax.WstxInputFactory;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.apache.hadoop.classification.VisibleForTesting;
 
 import java.io.BufferedInputStream;
 import java.io.DataInput;
@@ -87,6 +86,7 @@
 import org.apache.commons.collections.map.UnmodifiableMap;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -98,18 +98,19 @@
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringInterner;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.XMLUtils;
+
 import org.codehaus.stax2.XMLStreamReader2;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 
-import org.apache.hadoop.util.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
-
 import static org.apache.commons.lang3.StringUtils.isBlank;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
 
@@ -317,7 +318,7 @@ private static boolean getRestrictParserDefault(Object resource) {
   private boolean loadDefaults = true;
 
   /**
-   * Configuration objects
+   * Configuration objects.
    */
   private static final WeakHashMap<Configuration,Object> REGISTRY = 
     new WeakHashMap<Configuration,Object>();
@@ -774,7 +775,7 @@ private void updatePropertiesWithDeprecatedKeys(
   private void handleDeprecation() {
     LOG.debug("Handling deprecation for all properties in config...");
     DeprecationContext deprecations = deprecationContext.get();
-    Set<Object> keys = new HashSet<Object>();
+    Set<Object> keys = new HashSet<>();
     keys.addAll(getProps().keySet());
     for (Object item: keys) {
       LOG.debug("Handling deprecation for " + (String)item);
@@ -1099,6 +1100,20 @@ private static int[] findSubVariable(String eval) {
     return result;
   }
 
+  /**
+   * Provides a public wrapper over substituteVars in order to avoid compatibility issues.
+   * See HADOOP-18021 for further details.
+   *
+   * @param expr the literal value of a config key
+   * @return null if expr is null, otherwise the value resulting from expanding
+   * expr using the algorithm above.
+   * @throws IllegalArgumentException when more than
+   * {@link Configuration#MAX_SUBST} replacements are required
+   */
+  public String substituteCommonVariables(String expr) {
+    return substituteVars(expr);
+  }
+
   /**
    * Attempts to repeatedly expand the value {@code expr} by replacing the
    * left-most substring of the form "${var}" in the following precedence order
@@ -1120,13 +1135,17 @@ private static int[] findSubVariable(String eval) {
    * If a cycle is detected then the original expr is returned. Loops
    * involving multiple substitutions are not detected.
    *
+   * In order not to introduce breaking changes (as Oozie for example contains a method with the
+   * same name and same signature) do not make this method public, use substituteCommonVariables
+   * in this case.
+   *
    * @param expr the literal value of a config key
    * @return null if expr is null, otherwise the value resulting from expanding
    * expr using the algorithm above.
    * @throws IllegalArgumentException when more than
    * {@link Configuration#MAX_SUBST} replacements are required
    */
-  public String substituteVars(String expr) {
+  private String substituteVars(String expr) {
     if (expr == null) {
       return null;
     }
@@ -1890,6 +1909,7 @@ public long getTimeDuration(String name, String defaultValue,
    * @param name Property name
    * @param vStr The string value with time unit suffix to be converted.
    * @param unit Unit to convert the stored property, if it exists.
+   * @return time duration in given time unit.
    */
   public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
     return getTimeDurationHelper(name, vStr, unit, unit);
@@ -1904,6 +1924,7 @@ public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
    * @param vStr The string value with time unit suffix to be converted.
    * @param defaultUnit Unit to convert the stored property, if it exists.
    * @param returnUnit Unit for the returned value.
+   * @return time duration in given time unit.
    */
   private long getTimeDurationHelper(String name, String vStr,
       TimeUnit defaultUnit, TimeUnit returnUnit) {
@@ -2188,7 +2209,7 @@ private static int convertToInt(String value, int defaultValue) {
     }
 
     /**
-     * Is the given value in the set of ranges
+     * Is the given value in the set of ranges.
      * @param value the value to check
      * @return is the value in the ranges?
      */
@@ -2245,7 +2266,7 @@ public Iterator<Integer> iterator() {
   }
 
   /**
-   * Parse the given attribute as a set of integer ranges
+   * Parse the given attribute as a set of integer ranges.
    * @param name the attribute name
    * @param defaultValue the default value if it is not set
    * @return a new set of ranges from the configured value
@@ -2464,7 +2485,7 @@ public char[] getPasswordFromCredentialProviders(String name)
 
   /**
    * Fallback to clear text passwords in configuration.
-   * @param name
+   * @param name the property name.
    * @return clear text password or null
    */
   protected char[] getPasswordFromConfig(String name) {
@@ -2529,6 +2550,8 @@ public InetSocketAddress getSocketAddr(
   /**
    * Set the socket address for the <code>name</code> property as
    * a <code>host:port</code>.
+   * @param name property name.
+   * @param addr inetSocketAddress addr.
    */
   public void setSocketAddr(String name, InetSocketAddress addr) {
     set(name, NetUtils.getHostPortString(addr));
@@ -2706,6 +2729,7 @@ public Class<?> getClass(String name, Class<?> defaultValue) {
    * @param name the conf key name.
    * @param defaultValue default value.
    * @param xface the interface implemented by the named class.
+   * @param <U> Interface class type.
    * @return property value as a <code>Class</code>, 
    *         or <code>defaultValue</code>.
    */
@@ -2735,6 +2759,7 @@ else if (theClass != null)
    * @param name the property name.
    * @param xface the interface implemented by the classes named by
    *        <code>name</code>.
+   * @param <U> Interface class type.
    * @return a <code>List</code> of objects implementing <code>xface</code>.
    */
   @SuppressWarnings("unchecked")
@@ -2767,15 +2792,16 @@ public void setClass(String name, Class<?> theClass, Class<?> xface) {
     set(name, theClass.getName());
   }
 
-  /** 
+  /**
    * Get a local file under a directory named by <i>dirsProp</i> with
    * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
    * then one is chosen based on <i>path</i>'s hash code.  If the selected
    * directory does not exist, an attempt is made to create it.
-   * 
+   *
    * @param dirsProp directory in which to locate the file.
    * @param path file-path.
    * @return local file under the directory with the given path.
+   * @throws IOException raised on errors performing I/O.
    */
   public Path getLocalPath(String dirsProp, String path)
     throws IOException {
@@ -2799,15 +2825,16 @@ public Path getLocalPath(String dirsProp, String path)
     throw new IOException("No valid local directories in property: "+dirsProp);
   }
 
-  /** 
+  /**
    * Get a local file name under a directory named in <i>dirsProp</i> with
    * the given <i>path</i>.  If <i>dirsProp</i> contains multiple directories,
    * then one is chosen based on <i>path</i>'s hash code.  If the selected
    * directory does not exist, an attempt is made to create it.
-   * 
+   *
    * @param dirsProp directory in which to locate the file.
    * @param path file-path.
    * @return local file under the directory with the given path.
+   * @throws IOException raised on errors performing I/O.
    */
   public File getFile(String dirsProp, String path)
     throws IOException {
@@ -2960,11 +2987,13 @@ public Iterator<Map.Entry<String, String>> iterator() {
     // methods that allow non-strings to be put into configurations are removed,
     // we could replace properties with a Map<String,String> and get rid of this
     // code.
-    Map<String,String> result = new HashMap<String,String>();
-    for(Map.Entry<Object,Object> item: getProps().entrySet()) {
-      if (item.getKey() instanceof String &&
-          item.getValue() instanceof String) {
+    Properties props = getProps();
+    Map<String, String> result = new HashMap<>();
+    synchronized (props) {
+      for (Map.Entry<Object, Object> item : props.entrySet()) {
+        if (item.getKey() instanceof String && item.getValue() instanceof String) {
           result.put((String) item.getKey(), (String) item.getValue());
+        }
       }
     }
     return result.entrySet().iterator();
@@ -3417,7 +3446,7 @@ void parseNext() throws IOException, XMLStreamException {
 
   /**
    * Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
-   * @param prop
+   * @param prop properties.
    */
   public void addTags(Properties prop) {
     // Get all system tags
@@ -3518,7 +3547,7 @@ private void loadProperty(Properties properties, String name, String attr,
 
   /**
    * Print a warning if a property with a given name already exists with a
-   * different value
+   * different value.
    */
   private void checkForOverride(Properties properties, String name, String attr, String value) {
     String propertyValue = properties.getProperty(attr);
@@ -3528,11 +3557,12 @@ private void checkForOverride(Properties properties, String name, String attr, S
     }
   }
 
-  /** 
+  /**
    * Write out the non-default properties in this configuration to the given
    * {@link OutputStream} using UTF-8 encoding.
-   * 
+   *
    * @param out the output stream to write to.
+   * @throws IOException raised on errors performing I/O.
    */
   public void writeXml(OutputStream out) throws IOException {
     writeXml(new OutputStreamWriter(out, "UTF-8"));
@@ -3562,16 +3592,20 @@ public void writeXml(Writer out) throws IOException {
    * the configuration, this method throws an {@link IllegalArgumentException}.
    * </li>
    * </ul>
+   * @param propertyName xml property name.
    * @param out the writer to write to.
+   * @param config configuration.
+   * @throws IOException raised on errors performing I/O.
    */
-  public void writeXml(@Nullable String propertyName, Writer out)
+  public void writeXml(@Nullable String propertyName, Writer out, Configuration config)
       throws IOException, IllegalArgumentException {
-    Document doc = asXmlDocument(propertyName);
+    ConfigRedactor redactor = config != null ? new ConfigRedactor(this) : null;
+    Document doc = asXmlDocument(propertyName, redactor);
 
     try {
       DOMSource source = new DOMSource(doc);
       StreamResult result = new StreamResult(out);
-      TransformerFactory transFactory = TransformerFactory.newInstance();
+      TransformerFactory transFactory = XMLUtils.newSecureTransformerFactory();
       Transformer transformer = transFactory.newTransformer();
 
       // Important to not hold Configuration log while writing result, since
@@ -3583,11 +3617,16 @@ public void writeXml(@Nullable String propertyName, Writer out)
     }
   }
 
+  public void writeXml(@Nullable String propertyName, Writer out)
+      throws IOException, IllegalArgumentException {
+    writeXml(propertyName, out, null);
+  }
+
   /**
    * Return the XML DOM corresponding to this Configuration.
    */
-  private synchronized Document asXmlDocument(@Nullable String propertyName)
-      throws IOException, IllegalArgumentException {
+  private synchronized Document asXmlDocument(@Nullable String propertyName,
+      ConfigRedactor redactor) throws IOException, IllegalArgumentException {
     Document doc;
     try {
       doc = DocumentBuilderFactory
@@ -3610,13 +3649,13 @@ private synchronized Document asXmlDocument(@Nullable String propertyName)
             propertyName + " not found");
       } else {
         // given property is found, write single property
-        appendXMLProperty(doc, conf, propertyName);
+        appendXMLProperty(doc, conf, propertyName, redactor);
         conf.appendChild(doc.createTextNode("\n"));
       }
     } else {
       // append all elements
       for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
-        appendXMLProperty(doc, conf, (String)e.nextElement());
+        appendXMLProperty(doc, conf, (String)e.nextElement(), redactor);
         conf.appendChild(doc.createTextNode("\n"));
       }
     }
@@ -3632,7 +3671,7 @@ private synchronized Document asXmlDocument(@Nullable String propertyName)
    * @param propertyName
    */
   private synchronized void appendXMLProperty(Document doc, Element conf,
-      String propertyName) {
+      String propertyName, ConfigRedactor redactor) {
     // skip writing if given property name is empty or null
     if (!Strings.isNullOrEmpty(propertyName)) {
       String value = properties.getProperty(propertyName);
@@ -3645,8 +3684,11 @@ private synchronized void appendXMLProperty(Document doc, Element conf,
         propNode.appendChild(nameNode);
 
         Element valueNode = doc.createElement("value");
-        valueNode.appendChild(doc.createTextNode(
-            properties.getProperty(propertyName)));
+        String propertyValue = properties.getProperty(propertyName);
+        if (redactor != null) {
+          propertyValue = redactor.redactXml(propertyName, propertyValue);
+        }
+        valueNode.appendChild(doc.createTextNode(propertyValue));
         propNode.appendChild(valueNode);
 
         Element finalNode = doc.createElement("final");
@@ -3716,7 +3758,7 @@ private synchronized void appendXMLProperty(Document doc, Element conf,
    * @param config the configuration
    * @param propertyName property name
    * @param out the Writer to write to
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws IllegalArgumentException when property name is not
    *   empty and the property is not found in configuration
    **/
@@ -3763,7 +3805,7 @@ public static void dumpConfiguration(Configuration config,
    *
    * @param config the configuration
    * @param out the Writer to write to
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void dumpConfiguration(Configuration config,
       Writer out) throws IOException {
@@ -3792,7 +3834,7 @@ public static void dumpConfiguration(Configuration config,
    * @param jsonGen json writer
    * @param config configuration
    * @param name property name
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   private static void appendJSONProperty(JsonGenerator jsonGen,
       Configuration config, String name, ConfigRedactor redactor)
@@ -3874,7 +3916,10 @@ synchronized boolean getQuietMode() {
     return this.quietmode;
   }
   
-  /** For debugging.  List non-default properties to the terminal and exit. */
+  /** For debugging.  List non-default properties to the terminal and exit.
+   * @param args the argument to be parsed.
+   * @throws Exception exception.
+   */
   public static void main(String[] args) throws Exception {
     new Configuration().writeXml(System.out);
   }
@@ -3908,8 +3953,8 @@ public void write(DataOutput out) throws IOException {
   }
   
   /**
-   * get keys matching the the regex 
-   * @param regex
+   * get keys matching the the regex.
+   * @param regex the regex to match against.
    * @return {@literal Map<String,String>} with matching keys
    */
   public Map<String,String> getValByRegex(String regex) {
@@ -3954,6 +3999,8 @@ public static void dumpDeprecatedKeys() {
   /**
    * Returns whether or not a deprecated name has been warned. If the name is not
    * deprecated then always return false
+   * @param name proprties.
+   * @return true if name is a warned deprecation.
    */
   public static boolean hasWarnedDeprecation(String name) {
     DeprecationContext deprecations = deprecationContext.get();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configured.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configured.java
index f06af2b98df14..77a7117d19665 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configured.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configured.java
@@ -33,7 +33,9 @@ public Configured() {
     this(null);
   }
   
-  /** Construct a Configured. */
+  /** Construct a Configured.
+   * @param conf the Configuration object.
+   */
   public Configured(Configuration conf) {
     setConf(conf);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Reconfigurable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Reconfigurable.java
index c93dc31a881a9..915faf4c237ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Reconfigurable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Reconfigurable.java
@@ -33,6 +33,9 @@ public interface Reconfigurable extends Configurable {
    * (or null if it was not previously set). If newVal is null, set the property
    * to its default value;
    *
+   * @param property property name.
+   * @param newVal new value.
+   * @throws ReconfigurationException if there was an error applying newVal.
    * If the property cannot be changed, throw a 
    * {@link ReconfigurationException}.
    */
@@ -45,11 +48,14 @@ void reconfigureProperty(String property, String newVal)
    * If isPropertyReconfigurable returns true for a property,
    * then changeConf should not throw an exception when changing
    * this property.
+   * @param property property name.
+   * @return true if property reconfigurable; false if not.
    */
   boolean isPropertyReconfigurable(String property);
 
   /**
    * Return all the properties that can be changed at run time.
+   * @return reconfigurable propertys.
    */
   Collection<String> getReconfigurableProperties();
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
index 35dfeb99f0ba6..1c451ca6d30b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
@@ -79,6 +79,7 @@ public ReconfigurableBase() {
   /**
    * Construct a ReconfigurableBase with the {@link Configuration}
    * conf.
+   * @param conf configuration.
    */
   public ReconfigurableBase(Configuration conf) {
     super((conf == null) ? new Configuration() : conf);
@@ -91,6 +92,7 @@ public void setReconfigurationUtil(ReconfigurationUtil ru) {
 
   /**
    * Create a new configuration.
+   * @return configuration.
    */
   protected abstract Configuration getNewConf();
 
@@ -162,6 +164,7 @@ public void run() {
 
   /**
    * Start a reconfiguration task to reload configuration in background.
+   * @throws IOException raised on errors performing I/O.
    */
   public void startReconfigurationTask() throws IOException {
     synchronized (reconfigLock) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationException.java
index 0935bf025fd30..b22af76c9eb6b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationException.java
@@ -59,6 +59,10 @@ public ReconfigurationException() {
 
   /**
    * Create a new instance of {@link ReconfigurationException}.
+   * @param property property name.
+   * @param newVal new value.
+   * @param oldVal old value.
+   * @param cause original exception.
    */
   public ReconfigurationException(String property, 
                                   String newVal, String oldVal,
@@ -71,6 +75,9 @@ public ReconfigurationException(String property,
 
   /**
    * Create a new instance of {@link ReconfigurationException}.
+   * @param property property name.
+   * @param newVal new value.
+   * @param oldVal old value.
    */
   public ReconfigurationException(String property, 
                                   String newVal, String oldVal) {
@@ -82,6 +89,7 @@ public ReconfigurationException(String property,
 
   /**
    * Get property that cannot be changed.
+   * @return property info.
    */
   public String getProperty() {
     return property;
@@ -89,6 +97,7 @@ public String getProperty() {
 
   /**
    * Get value to which property was supposed to be changed.
+   * @return new value.
    */
   public String getNewValue() {
     return newVal;
@@ -96,6 +105,7 @@ public String getNewValue() {
 
   /**
    * Get old value of property that cannot be changed.
+   * @return old value.
    */
   public String getOldValue() {
     return oldVal;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationTaskStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationTaskStatus.java
index 05ec90758e5fa..ca9ddb61566ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationTaskStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationTaskStatus.java
@@ -42,7 +42,8 @@ public ReconfigurationTaskStatus(long startTime, long endTime,
   /**
    * Return true if
    *   - A reconfiguration task has finished or
-   *   - an active reconfiguration task is running
+   *   - an active reconfiguration task is running.
+   * @return true if startTime &gt; 0; false if not.
    */
   public boolean hasTask() {
     return startTime > 0;
@@ -51,6 +52,7 @@ public boolean hasTask() {
   /**
    * Return true if the latest reconfiguration task has finished and there is
    * no another active task running.
+   * @return true if endTime &gt; 0; false if not.
    */
   public boolean stopped() {
     return endTime > 0;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
index 64c754faa59d8..e6813b96a2670 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
@@ -145,14 +145,18 @@ private static List<Class<? extends CryptoCodec>> getCodecClasses(
   public abstract CipherSuite getCipherSuite();
 
   /**
-   * Create a {@link org.apache.hadoop.crypto.Encryptor}. 
-   * @return Encryptor the encryptor
+   * Create a {@link org.apache.hadoop.crypto.Encryptor}.
+   *
+   * @return Encryptor the encryptor.
+   * @throws GeneralSecurityException thrown if create encryptor error.
    */
   public abstract Encryptor createEncryptor() throws GeneralSecurityException;
-  
+
   /**
    * Create a {@link org.apache.hadoop.crypto.Decryptor}.
+   *
    * @return Decryptor the decryptor
+   * @throws GeneralSecurityException thrown if create decryptor error.
    */
   public abstract Decryptor createDecryptor() throws GeneralSecurityException;
   
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index 5ab5d341fb826..067abde9dfbb8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -157,7 +157,7 @@ public InputStream getWrappedStream() {
    * @param off the buffer offset.
    * @param len the maximum number of decrypted data bytes to read.
    * @return int the total number of decrypted data bytes read into the buffer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public int read(byte[] b, int off, int len) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
index 8e7522112551e..2a1335b6e745a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
@@ -146,7 +146,7 @@ public OutputStream getWrappedStream() {
    * @param b the data.
    * @param off the start offset in the data.
    * @param len the number of bytes to write.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public synchronized void write(byte[] b, int off, int len) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
index 318975fd6cebd..dad4d20df2afd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
@@ -39,7 +39,11 @@ public class CryptoStreamUtils {
   private static final Logger LOG =
       LoggerFactory.getLogger(CryptoStreamUtils.class);
 
-  /** Forcibly free the direct buffer. */
+  /**
+   * Forcibly free the direct buffer.
+   *
+   * @param buffer buffer.
+   */
   public static void freeDB(ByteBuffer buffer) {
     if (CleanerUtil.UNMAP_SUPPORTED) {
       try {
@@ -52,13 +56,22 @@ public static void freeDB(ByteBuffer buffer) {
     }
   }
 
-  /** Read crypto buffer size */
+  /**
+   * Read crypto buffer size.
+   *
+   * @param conf configuration.
+   * @return hadoop.security.crypto.buffer.size.
+   */
   public static int getBufferSize(Configuration conf) {
     return conf.getInt(HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY, 
         HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT);
   }
-  
-  /** AES/CTR/NoPadding or SM4/CTR/NoPadding is required. */
+
+  /**
+   * AES/CTR/NoPadding or SM4/CTR/NoPadding is required.
+   *
+   * @param codec crypto codec.
+   */
   public static void checkCodec(CryptoCodec codec) {
     if (codec.getCipherSuite() != CipherSuite.AES_CTR_NOPADDING &&
             codec.getCipherSuite() != CipherSuite.SM4_CTR_NOPADDING) {
@@ -67,17 +80,27 @@ public static void checkCodec(CryptoCodec codec) {
     }
   }
 
-  /** Check and floor buffer size */
+  /**
+   * Check and floor buffer size.
+   *
+   * @param codec crypto codec.
+   * @param bufferSize the size of the buffer to be used.
+   * @return calc buffer size.
+   */
   public static int checkBufferSize(CryptoCodec codec, int bufferSize) {
     Preconditions.checkArgument(bufferSize >= MIN_BUFFER_SIZE, 
         "Minimum value of buffer size is " + MIN_BUFFER_SIZE + ".");
     return bufferSize - bufferSize % codec.getCipherSuite()
         .getAlgorithmBlockSize();
   }
-  
+
   /**
    * If input stream is {@link org.apache.hadoop.fs.Seekable}, return it's
    * current position, otherwise return 0;
+   *
+   * @param in wrapper.
+   * @return current position, otherwise return 0.
+   * @throws IOException raised on errors performing I/O.
    */
   public static long getInputStreamOffset(InputStream in) throws IOException {
     if (in instanceof Seekable) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
index d22e91442cca4..b166cfc8611b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
@@ -84,14 +84,14 @@ static int get(String padding) throws NoSuchPaddingException {
     String loadingFailure = null;
     try {
       if (!NativeCodeLoader.buildSupportsOpenssl()) {
-        PerformanceAdvisory.LOG.debug("Build does not support openssl");
+        PerformanceAdvisory.LOG.warn("Build does not support openssl");
         loadingFailure = "build does not support openssl.";
       } else {
         initIDs();
       }
     } catch (Throwable t) {
       loadingFailure = t.getMessage();
-      LOG.debug("Failed to load OpenSSL Cipher.", t);
+      LOG.warn("Failed to load OpenSSL Cipher.", t);
     } finally {
       loadingFailureReason = loadingFailure;
     }
@@ -225,34 +225,33 @@ public int update(ByteBuffer input, ByteBuffer output)
     output.position(output.position() + len);
     return len;
   }
-  
+
   /**
    * Finishes a multiple-part operation. The data is encrypted or decrypted,
    * depending on how this cipher was initialized.
    * <p>
-   * 
    * The result is stored in the output buffer. Upon return, the output buffer's
    * position will have advanced by n, where n is the value returned by this
    * method; the output buffer's limit will not have changed.
-   * <p>
-   * 
+   * </p>
    * If <code>output.remaining()</code> bytes are insufficient to hold the result,
    * a <code>ShortBufferException</code> is thrown.
    * <p>
-   * 
    * Upon finishing, this method resets this cipher object to the state it was
    * in when previously initialized. That is, the object is available to encrypt
    * or decrypt more data.
-   * <p>
-   * 
-   * If any exception is thrown, this cipher object need to be reset before it 
+   * </p>
+   * If any exception is thrown, this cipher object need to be reset before it
    * can be used again.
-   * 
+   *
    * @param output the output ByteBuffer
    * @return int number of bytes stored in <code>output</code>
-   * @throws ShortBufferException
-   * @throws IllegalBlockSizeException
-   * @throws BadPaddingException
+   * @throws ShortBufferException      if there is insufficient space in the output buffer.
+   * @throws IllegalBlockSizeException This exception is thrown when the length
+   *                                   of data provided to a block cipher is incorrect.
+   * @throws BadPaddingException       This exception is thrown when a particular
+   *                                   padding mechanism is expected for the input
+   *                                   data but the data is not padded properly.
    */
   public int doFinal(ByteBuffer output) throws ShortBufferException, 
       IllegalBlockSizeException, BadPaddingException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index dafdaf7e15b25..5e207251805fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -242,7 +242,7 @@ protected int addVersion() {
     /**
      * Serialize the metadata to a set of bytes.
      * @return the serialized bytes
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     protected byte[] serialize() throws IOException {
       ByteArrayOutputStream buffer = new ByteArrayOutputStream();
@@ -281,7 +281,7 @@ protected byte[] serialize() throws IOException {
     /**
      * Deserialize a new metadata object from a set of bytes.
      * @param bytes the serialized metadata
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     protected Metadata(byte[] bytes) throws IOException {
       String cipher = null;
@@ -450,7 +450,7 @@ public boolean isTransient() {
    * when decrypting data.
    * @param versionName the name of a specific version of the key
    * @return the key material
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract KeyVersion getKeyVersion(String versionName
                                             ) throws IOException;
@@ -458,14 +458,15 @@ public abstract KeyVersion getKeyVersion(String versionName
   /**
    * Get the key names for all keys.
    * @return the list of key names
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract List<String> getKeys() throws IOException;
 
   /**
    * Get key metadata in bulk.
    * @param names the names of the keys to get
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
+   * @return Metadata Array.
    */
   public Metadata[] getKeysMetadata(String... names) throws IOException {
     Metadata[] result = new Metadata[names.length];
@@ -477,8 +478,10 @@ public Metadata[] getKeysMetadata(String... names) throws IOException {
 
   /**
    * Get the key material for all versions of a specific key name.
+   *
+   * @param name the base name of the key.
    * @return the list of key material
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract List<KeyVersion> getKeyVersions(String name) throws IOException;
 
@@ -488,7 +491,7 @@ public Metadata[] getKeysMetadata(String... names) throws IOException {
    * @param name the base name of the key
    * @return the version name of the current version of the key or null if the
    *    key version doesn't exist
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public KeyVersion getCurrentKey(String name) throws IOException {
     Metadata meta = getMetadata(name);
@@ -502,7 +505,7 @@ public KeyVersion getCurrentKey(String name) throws IOException {
    * Get metadata about the key.
    * @param name the basename of the key
    * @return the key's metadata or null if the key doesn't exist
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract Metadata getMetadata(String name) throws IOException;
 
@@ -512,7 +515,7 @@ public KeyVersion getCurrentKey(String name) throws IOException {
    * @param material the key material for the first version of the key.
    * @param options the options for the new key.
    * @return the version name of the first version of the key.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract KeyVersion createKey(String name, byte[] material,
                                        Options options) throws IOException;
@@ -537,7 +540,7 @@ private String getAlgorithm(String cipher) {
    * @param size length of the key.
    * @param algorithm algorithm to use for generating the key.
    * @return the generated key.
-   * @throws NoSuchAlgorithmException
+   * @throws NoSuchAlgorithmException no such algorithm exception.
    */
   protected byte[] generateKey(int size, String algorithm)
       throws NoSuchAlgorithmException {
@@ -558,8 +561,8 @@ protected byte[] generateKey(int size, String algorithm)
    * @param name the base name of the key
    * @param options the options for the new key.
    * @return the version name of the first version of the key.
-   * @throws IOException
-   * @throws NoSuchAlgorithmException
+   * @throws IOException raised on errors performing I/O.
+   * @throws NoSuchAlgorithmException no such algorithm exception.
    */
   public KeyVersion createKey(String name, Options options)
       throws NoSuchAlgorithmException, IOException {
@@ -570,7 +573,7 @@ public KeyVersion createKey(String name, Options options)
   /**
    * Delete the given key.
    * @param name the name of the key to delete
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void deleteKey(String name) throws IOException;
 
@@ -579,7 +582,7 @@ public KeyVersion createKey(String name, Options options)
    * @param name the basename of the key
    * @param material the new key material
    * @return the name of the new version of the key
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract KeyVersion rollNewVersion(String name,
                                              byte[] material
@@ -601,7 +604,10 @@ public void close() throws IOException {
    *
    * @param name the basename of the key
    * @return the name of the new version of the key
-   * @throws IOException
+   * @throws IOException              raised on errors performing I/O.
+   * @throws NoSuchAlgorithmException This exception is thrown when a particular
+   *                                  cryptographic algorithm is requested
+   *                                  but is not available in the environment.
    */
   public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
                                                        IOException {
@@ -620,7 +626,7 @@ public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
    * version of the given key.
    *
    * @param name the basename of the key
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void invalidateCache(String name) throws IOException {
     // NOP
@@ -628,18 +634,19 @@ public void invalidateCache(String name) throws IOException {
 
   /**
    * Ensures that any changes to the keys are written to persistent store.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void flush() throws IOException;
 
   /**
-   * Split the versionName in to a base name. Converts "/aaa/bbb/3" to
+   * Split the versionName in to a base name. Converts "/aaa/bbb@3" to
    * "/aaa/bbb".
    * @param versionName the version name to split
    * @return the base name of the key
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static String getBaseName(String versionName) throws IOException {
+    Objects.requireNonNull(versionName, "VersionName cannot be null");
     int div = versionName.lastIndexOf('@');
     if (div == -1) {
       throw new IOException("No version in key path " + versionName);
@@ -660,9 +667,11 @@ protected static String buildVersionName(String name, int version) {
 
   /**
    * Find the provider with the given key.
+   *
    * @param providerList the list of providers
-   * @param keyName the key name we are looking for
+   * @param keyName the key name we are looking for.
    * @return the KeyProvider that has the key
+   * @throws IOException raised on errors performing I/O.
    */
   public static KeyProvider findProvider(List<KeyProvider> providerList,
                                          String keyName) throws IOException {
@@ -680,7 +689,7 @@ public static KeyProvider findProvider(List<KeyProvider> providerList,
    * means. If true, the password should be provided by the caller using
    * setPassword().
    * @return Whether or not the provider requires a password
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public boolean needsPassword() throws IOException {
     return false;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 3f3c367fc3933..f1bb314582038 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -25,10 +25,6 @@
 import java.util.List;
 import java.util.ListIterator;
 
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.crypto.CryptoCodec;
@@ -178,6 +174,7 @@ public interface CryptoExtension extends KeyProviderExtension.Extension {
      * Calls to this method allows the underlying KeyProvider to warm-up any
      * implementation specific caches used to store the Encrypted Keys.
      * @param keyNames Array of Key Names
+     * @throws IOException thrown if the key material could not be encrypted.
      */
     public void warmUpEncryptedKeys(String... keyNames)
         throws IOException;
@@ -474,8 +471,9 @@ public void drain(String keyName) {
   /**
    * This constructor is to be used by sub classes that provide
    * delegating/proxying functionality to the {@link KeyProviderCryptoExtension}
-   * @param keyProvider
-   * @param extension
+   *
+   * @param keyProvider key provider.
+   * @param extension crypto extension.
    */
   protected KeyProviderCryptoExtension(KeyProvider keyProvider,
       CryptoExtension extension) {
@@ -486,6 +484,7 @@ protected KeyProviderCryptoExtension(KeyProvider keyProvider,
    * Notifies the Underlying CryptoExtension implementation to warm up any
    * implementation specific caches for the specified KeyVersions
    * @param keyNames Arrays of key Names
+   * @throws IOException raised on errors performing I/O.
    */
   public void warmUpEncryptedKeys(String... keyNames)
       throws IOException {
@@ -557,7 +556,7 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
    * Calls {@link CryptoExtension#drain(String)} for the given key name on the
    * underlying {@link CryptoExtension}.
    *
-   * @param keyName
+   * @param keyName key name.
    */
   public void drain(String keyName) {
     getExtension().drain(keyName);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
index 1fdc2fe12455b..3c1af424eb7cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
@@ -48,14 +48,14 @@ public interface DelegationTokenExtension
      * Renews the given token.
      * @param token The token to be renewed.
      * @return The token's lifetime after renewal, or 0 if it can't be renewed.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     long renewDelegationToken(final Token<?> token) throws IOException;
 
     /**
      * Cancels the given token.
      * @param token The token to be cancelled.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     Void cancelDelegationToken(final Token<?> token) throws IOException;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index a75f7d3aa63bd..c18d0d41bc08a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -75,7 +75,7 @@ public class KeyShell extends CommandShell {
    * </pre>
    * @param args Command line arguments.
    * @return 0 on success, 1 on failure.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   protected int init(String[] args) throws IOException {
@@ -547,7 +547,7 @@ private String prettifyException(Exception e) {
    * success and 1 for failure.
    *
    * @param args Command line arguments.
-   * @throws Exception
+   * @throws Exception raised on errors performing I/O.
    */
   public static void main(String[] args) throws Exception {
     int res = ToolRunner.run(new Configuration(), new KeyShell(), args);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 6f8f4585ee75f..f9cc3f4524ff5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.net.ConnectException;
+import java.net.SocketException;
 import java.net.URI;
 import java.security.GeneralSecurityException;
 import java.security.NoSuchAlgorithmException;
@@ -29,7 +30,7 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
@@ -182,10 +183,10 @@ private <T> T doOp(ProviderCallable<T> op, int currPos,
       } catch (IOException ioe) {
         LOG.warn("KMS provider at [{}] threw an IOException: ",
             provider.getKMSUrl(), ioe);
-        // SSLHandshakeException can occur here because of lost connection
+        // SSLException can occur here because of lost connection
         // with the KMS server, creating a ConnectException from it,
         // so that the FailoverOnNetworkExceptionRetry policy will retry
-        if (ioe instanceof SSLHandshakeException) {
+        if (ioe instanceof SSLException || ioe instanceof SocketException) {
           Exception cause = ioe;
           ioe = new ConnectException("SSLHandshakeException: "
               + cause.getMessage());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index be2db05842c8e..65eded918d60d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -63,7 +63,7 @@ public interface QueueRefiller <E> {
      * @param keyName Key name
      * @param keyQueue Queue that needs to be filled
      * @param numValues number of Values to be added to the queue.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void fillQueueForKey(String keyName,
         Queue<E> keyQueue, int numValues) throws IOException;
@@ -268,7 +268,7 @@ public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
    * Initializes the Value Queues for the provided keys by calling the
    * fill Method with "numInitValues" values
    * @param keyNames Array of key Names
-   * @throws ExecutionException
+   * @throws ExecutionException executionException.
    */
   public void initializeQueuesForKeys(String... keyNames)
       throws ExecutionException {
@@ -285,8 +285,8 @@ public void initializeQueuesForKeys(String... keyNames)
    * function to add 1 value to Queue and then drain it.
    * @param keyName String key name
    * @return E the next value in the Queue
-   * @throws IOException
-   * @throws ExecutionException
+   * @throws IOException raised on errors performing I/O.
+   * @throws ExecutionException executionException.
    */
   public E getNext(String keyName)
       throws IOException, ExecutionException {
@@ -344,8 +344,8 @@ public int getSize(String keyName) {
    * @param keyName String key name
    * @param num Minimum number of values to return.
    * @return {@literal List<E>} values returned
-   * @throws IOException
-   * @throws ExecutionException
+   * @throws IOException raised on errors performing I/O.
+   * @throws ExecutionException execution exception.
    */
   public List<E> getAtMost(String keyName, int num) throws IOException,
       ExecutionException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index d9818b472f0e5..a4737c548c8fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -272,7 +272,7 @@ public static AbstractFileSystem get(final URI uri, final Configuration conf)
    * @param supportedScheme the scheme supported by the implementor
    * @param authorityNeeded if true then theURI must have authority, if false
    *          then the URI must have null authority.
-   *
+   * @param defaultPort default port to use if port is not specified in the URI.
    * @throws URISyntaxException <code>uri</code> has syntax error
    */
   public AbstractFileSystem(final URI uri, final String supportedScheme,
@@ -281,11 +281,12 @@ public AbstractFileSystem(final URI uri, final String supportedScheme,
     myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
     statistics = getStatistics(uri); 
   }
-  
+
   /**
-   * Check that the Uri's scheme matches
-   * @param uri
-   * @param supportedScheme
+   * Check that the Uri's scheme matches.
+   *
+   * @param uri name URI of the FS.
+   * @param supportedScheme supported scheme.
    */
   public void checkScheme(URI uri, String supportedScheme) {
     String scheme = uri.getScheme();
@@ -362,7 +363,7 @@ public URI getUri() {
    * If the path is fully qualified URI, then its scheme and authority
    * matches that of this file system. Otherwise the path must be 
    * slash-relative name.
-   * 
+   * @param path the path.
    * @throws InvalidPathException if the path is invalid
    */
   public void checkPath(Path path) {
@@ -431,7 +432,7 @@ public String getUriPath(final Path p) {
   
   /**
    * Make the path fully qualified to this file system
-   * @param path
+   * @param path the path.
    * @return the qualified path
    */
   public Path makeQualified(Path path) {
@@ -496,9 +497,9 @@ public FsServerDefaults getServerDefaults(final Path f) throws IOException {
    * through any internal symlinks or mount point
    * @param p path to be resolved
    * @return fully qualified path 
-   * @throws FileNotFoundException
-   * @throws AccessControlException
-   * @throws IOException
+   * @throws FileNotFoundException when file not find throw.
+   * @throws AccessControlException when accees control error throw.
+   * @throws IOException raised on errors performing I/O.
    * @throws UnresolvedLinkException if symbolic link on path cannot be
    * resolved internally
    */
@@ -513,6 +514,18 @@ public Path resolvePath(final Path p) throws FileNotFoundException,
    * {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
    * that the Path f must be fully qualified and the permission is absolute
    * (i.e. umask has been applied).
+   *
+   * @param f the path.
+   * @param createFlag create_flag.
+   * @param opts create ops.
+   * @throws AccessControlException access controll exception.
+   * @throws FileAlreadyExistsException file already exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws ParentNotDirectoryException parent not dir exception.
+   * @throws UnsupportedFileSystemException unsupported file system exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return output stream.
    */
   public final FSDataOutputStream create(final Path f,
       final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
@@ -630,6 +643,24 @@ public final FSDataOutputStream create(final Path f,
    * The specification of this method matches that of
    * {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
    * have been declared explicitly.
+   *
+   * @param f the path.
+   * @param flag create flag.
+   * @param absolutePermission absolute permission.
+   * @param bufferSize buffer size.
+   * @param replication replications.
+   * @param blockSize block size.
+   * @param progress progress.
+   * @param checksumOpt check sum opt.
+   * @param createParent create parent.
+   * @throws AccessControlException access control exception.
+   * @throws FileAlreadyExistsException file already exists exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws ParentNotDirectoryException parent not directory exception.
+   * @throws UnsupportedFileSystemException unsupported filesystem exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return output stream.
    */
   public abstract FSDataOutputStream createInternal(Path f,
       EnumSet<CreateFlag> flag, FsPermission absolutePermission,
@@ -644,6 +675,14 @@ public abstract FSDataOutputStream createInternal(Path f,
    * {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
    * f must be fully qualified and the permission is absolute (i.e. 
    * umask has been applied).
+   * @param dir directory.
+   * @param permission permission.
+   * @param createParent create parent flag.
+   * @throws AccessControlException access control exception.
+   * @throws FileAlreadyExistsException file already exists exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void mkdir(final Path dir, final FsPermission permission,
       final boolean createParent) throws AccessControlException,
@@ -654,6 +693,14 @@ public abstract void mkdir(final Path dir, final FsPermission permission,
    * The specification of this method matches that of
    * {@link FileContext#delete(Path, boolean)} except that Path f must be for
    * this file system.
+   *
+   * @param f the path.
+   * @param recursive recursive flag.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return if successfully deleted success true, not false.
    */
   public abstract boolean delete(final Path f, final boolean recursive)
       throws AccessControlException, FileNotFoundException,
@@ -663,6 +710,13 @@ public abstract boolean delete(final Path f, final boolean recursive)
    * The specification of this method matches that of
    * {@link FileContext#open(Path)} except that Path f must be for this
    * file system.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return input stream.
    */
   public FSDataInputStream open(final Path f) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
@@ -673,6 +727,14 @@ public FSDataInputStream open(final Path f) throws AccessControlException,
    * The specification of this method matches that of
    * {@link FileContext#open(Path, int)} except that Path f must be for this
    * file system.
+   *
+   * @param f the path.
+   * @param bufferSize buffer size.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return if successfully open success true, not false.
    */
   public abstract FSDataInputStream open(final Path f, int bufferSize)
       throws AccessControlException, FileNotFoundException,
@@ -682,6 +744,14 @@ public abstract FSDataInputStream open(final Path f, int bufferSize)
    * The specification of this method matches that of
    * {@link FileContext#truncate(Path, long)} except that Path f must be for
    * this file system.
+   *
+   * @param f the path.
+   * @param newLength new length.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return if successfully truncate success true, not false.
    */
   public boolean truncate(Path f, long newLength)
       throws AccessControlException, FileNotFoundException,
@@ -694,6 +764,14 @@ public boolean truncate(Path f, long newLength)
    * The specification of this method matches that of
    * {@link FileContext#setReplication(Path, short)} except that Path f must be
    * for this file system.
+   *
+   * @param f the path.
+   * @param replication replication.
+   * @return if successfully set replication success true, not false.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract boolean setReplication(final Path f,
       final short replication) throws AccessControlException,
@@ -703,6 +781,16 @@ public abstract boolean setReplication(final Path f,
    * The specification of this method matches that of
    * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
    * f must be for this file system.
+   *
+   * @param src src.
+   * @param dst dst.
+   * @param options options.
+   * @throws AccessControlException access control exception.
+   * @throws FileAlreadyExistsException file already exists exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws ParentNotDirectoryException parent not directory exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public final void rename(final Path src, final Path dst,
       final Options.Rename... options) throws AccessControlException,
@@ -727,6 +815,15 @@ public final void rename(final Path src, final Path dst,
    * File systems that do not have a built in overwrite need implement only this
    * method and can take advantage of the default impl of the other
    * {@link #renameInternal(Path, Path, boolean)}
+   *
+   * @param src src.
+   * @param dst dst.
+   * @throws AccessControlException access control exception.
+   * @throws FileAlreadyExistsException file already exists exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws ParentNotDirectoryException parent not directory exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void renameInternal(final Path src, final Path dst)
       throws AccessControlException, FileAlreadyExistsException,
@@ -737,6 +834,16 @@ public abstract void renameInternal(final Path src, final Path dst)
    * The specification of this method matches that of
    * {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
    * f must be for this file system.
+   *
+   * @param src src.
+   * @param dst dst.
+   * @param overwrite overwrite flag.
+   * @throws AccessControlException access control exception.
+   * @throws FileAlreadyExistsException file already exists exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws ParentNotDirectoryException parent not directory exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public void renameInternal(final Path src, final Path dst,
       boolean overwrite) throws AccessControlException,
@@ -800,6 +907,12 @@ public boolean supportsSymlinks() {
   /**
    * The specification of this method matches that of  
    * {@link FileContext#createSymlink(Path, Path, boolean)};
+   *
+   * @param target target.
+   * @param link link.
+   * @param createParent create parent.
+   * @throws IOException raised on errors performing I/O.
+   * @throws UnresolvedLinkException unresolved link exception.
    */
   public void createSymlink(final Path target, final Path link,
       final boolean createParent) throws IOException, UnresolvedLinkException {
@@ -810,6 +923,8 @@ public void createSymlink(final Path target, final Path link,
    * Partially resolves the path. This is used during symlink resolution in
    * {@link FSLinkResolver}, and differs from the similarly named method
    * {@link FileContext#getLinkTarget(Path)}.
+   * @param f the path.
+   * @return target path.
    * @throws IOException subclass implementations may throw IOException 
    */
   public Path getLinkTarget(final Path f) throws IOException {
@@ -822,6 +937,13 @@ public Path getLinkTarget(final Path f) throws IOException {
    * The specification of this method matches that of
    * {@link FileContext#setPermission(Path, FsPermission)} except that Path f
    * must be for this file system.
+   *
+   * @param f the path.
+   * @param permission permission.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void setPermission(final Path f,
       final FsPermission permission) throws AccessControlException,
@@ -831,6 +953,14 @@ public abstract void setPermission(final Path f,
    * The specification of this method matches that of
    * {@link FileContext#setOwner(Path, String, String)} except that Path f must
    * be for this file system.
+   *
+   * @param f the path.
+   * @param username username.
+   * @param groupname groupname.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void setOwner(final Path f, final String username,
       final String groupname) throws AccessControlException,
@@ -840,6 +970,14 @@ public abstract void setOwner(final Path f, final String username,
    * The specification of this method matches that of
    * {@link FileContext#setTimes(Path, long, long)} except that Path f must be
    * for this file system.
+   *
+   * @param f the path.
+   * @param mtime modify time.
+   * @param atime access time.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void setTimes(final Path f, final long mtime,
     final long atime) throws AccessControlException, FileNotFoundException,
@@ -849,6 +987,13 @@ public abstract void setTimes(final Path f, final long mtime,
    * The specification of this method matches that of
    * {@link FileContext#getFileChecksum(Path)} except that Path f must be for
    * this file system.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return File Check sum.
    */
   public abstract FileChecksum getFileChecksum(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -859,6 +1004,13 @@ public abstract FileChecksum getFileChecksum(final Path f)
    * {@link FileContext#getFileStatus(Path)} 
    * except that an UnresolvedLinkException may be thrown if a symlink is 
    * encountered in the path.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return File Status
    */
   public abstract FileStatus getFileStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -870,8 +1022,8 @@ public abstract FileStatus getFileStatus(final Path f)
    * In some FileSystem implementations such as HDFS metadata
    * synchronization is essential to guarantee consistency of read requests
    * particularly in HA setting.
-   * @throws IOException
-   * @throws UnsupportedOperationException
+   * @throws IOException raised on errors performing I/O.
+   * @throws UnsupportedOperationException Unsupported Operation Exception.
    */
   public void msync() throws IOException, UnsupportedOperationException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
@@ -883,6 +1035,13 @@ public void msync() throws IOException, UnsupportedOperationException {
    * {@link FileContext#access(Path, FsAction)}
    * except that an UnresolvedLinkException may be thrown if a symlink is
    * encountered in the path.
+   *
+   * @param path the path.
+   * @param mode fsaction mode.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
   public void access(Path path, FsAction mode) throws AccessControlException,
@@ -897,6 +1056,13 @@ public void access(Path path, FsAction mode) throws AccessControlException,
    * encountered in the path leading up to the final path component.
    * If the file system does not support symlinks then the behavior is
    * equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnsupportedFileSystemException UnSupported File System Exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return file status.
    */
   public FileStatus getFileLinkStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -908,6 +1074,15 @@ public FileStatus getFileLinkStatus(final Path f)
    * The specification of this method matches that of
    * {@link FileContext#getFileBlockLocations(Path, long, long)} except that
    * Path f must be for this file system.
+   *
+   * @param f the path.
+   * @param start start.
+   * @param len length.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return BlockLocation Array.
    */
   public abstract BlockLocation[] getFileBlockLocations(final Path f,
       final long start, final long len) throws AccessControlException,
@@ -917,6 +1092,13 @@ public abstract BlockLocation[] getFileBlockLocations(final Path f,
    * The specification of this method matches that of
    * {@link FileContext#getFsStatus(Path)} except that Path f must be for this
    * file system.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return Fs Status.
    */
   public FsStatus getFsStatus(final Path f) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
@@ -927,6 +1109,11 @@ public FsStatus getFsStatus(final Path f) throws AccessControlException,
   /**
    * The specification of this method matches that of
    * {@link FileContext#getFsStatus(Path)}.
+   *
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return Fs Status.
    */
   public abstract FsStatus getFsStatus() throws AccessControlException,
       FileNotFoundException, IOException;
@@ -935,6 +1122,13 @@ public abstract FsStatus getFsStatus() throws AccessControlException,
    * The specification of this method matches that of
    * {@link FileContext#listStatus(Path)} except that Path f must be for this
    * file system.
+   *
+   * @param f path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return FileStatus Iterator.
    */
   public RemoteIterator<FileStatus> listStatusIterator(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -967,6 +1161,13 @@ public FileStatus next() {
    * will have different formats for replicated and erasure coded file. Please
    * refer to {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
    * for more details.
+   *
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return FileStatus Iterator.
    */
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -999,6 +1200,12 @@ public LocatedFileStatus next() throws IOException {
    * The specification of this method matches that of
    * {@link FileContext.Util#listStatus(Path)} except that Path f must be 
    * for this file system.
+   * @param f the path.
+   * @throws AccessControlException access control exception.
+   * @throws FileNotFoundException file not found exception.
+   * @throws UnresolvedLinkException unresolved link exception.
+   * @throws IOException raised on errors performing I/O.
+   * @return FileStatus Iterator.
    */
   public abstract FileStatus[] listStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -1007,7 +1214,8 @@ public abstract FileStatus[] listStatus(final Path f)
   /**
    * @return an iterator over the corrupt files under the given path
    * (may contain duplicates if a file has more than one corrupt block)
-   * @throws IOException
+   * @param path the path.
+   * @throws IOException raised on errors performing I/O.
    */
   public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
@@ -1020,6 +1228,10 @@ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
    * The specification of this method matches that of
    * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
    * must be for this file system.
+   *
+   * @param verifyChecksum verify check sum flag.
+   * @throws AccessControlException access control exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void setVerifyChecksum(final boolean verifyChecksum)
       throws AccessControlException, IOException;
@@ -1041,7 +1253,7 @@ public String getCanonicalServiceName() {
    * @param renewer the account name that is allowed to renew the token.
    * @return List of delegation tokens.
    *   If delegation tokens not supported then return a list of size zero.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
   public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
@@ -1141,7 +1353,7 @@ public AclStatus getAclStatus(Path path) throws IOException {
    * @param path Path to modify
    * @param name xattr name.
    * @param value xattr value.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void setXAttr(Path path, String name, byte[] value)
       throws IOException {
@@ -1160,7 +1372,7 @@ public void setXAttr(Path path, String name, byte[] value)
    * @param name xattr name.
    * @param value xattr value.
    * @param flag xattr set flag
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void setXAttr(Path path, String name, byte[] value,
       EnumSet<XAttrSetFlag> flag) throws IOException {
@@ -1178,7 +1390,7 @@ public void setXAttr(Path path, String name, byte[] value,
    * @param path Path to get extended attribute
    * @param name xattr name.
    * @return byte[] xattr value.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public byte[] getXAttr(Path path, String name) throws IOException {
     throw new UnsupportedOperationException(getClass().getSimpleName()
@@ -1196,7 +1408,7 @@ public byte[] getXAttr(Path path, String name) throws IOException {
    *
    * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
    * or directory
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Map<String, byte[]> getXAttrs(Path path) throws IOException {
     throw new UnsupportedOperationException(getClass().getSimpleName()
@@ -1214,7 +1426,7 @@ public Map<String, byte[]> getXAttrs(Path path) throws IOException {
    * @param names XAttr names.
    * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
    * or directory
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Map<String, byte[]> getXAttrs(Path path, List<String> names)
       throws IOException {
@@ -1232,7 +1444,7 @@ public Map<String, byte[]> getXAttrs(Path path, List<String> names)
    * @param path Path to get extended attributes
    * @return {@literal Map<String, byte[]>} describing the XAttrs of the file
    * or directory
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public List<String> listXAttrs(Path path)
           throws IOException {
@@ -1249,7 +1461,7 @@ public List<String> listXAttrs(Path path)
    *
    * @param path Path to remove extended attribute
    * @param name xattr name
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void removeXAttr(Path path, String name) throws IOException {
     throw new UnsupportedOperationException(getClass().getSimpleName()
@@ -1259,6 +1471,11 @@ public void removeXAttr(Path path, String name) throws IOException {
   /**
    * The specification of this method matches that of
    * {@link FileContext#createSnapshot(Path, String)}.
+   *
+   * @param path the path.
+   * @param snapshotName snapshot name.
+   * @throws IOException raised on errors performing I/O.
+   * @return path.
    */
   public Path createSnapshot(final Path path, final String snapshotName)
       throws IOException {
@@ -1269,6 +1486,11 @@ public Path createSnapshot(final Path path, final String snapshotName)
   /**
    * The specification of this method matches that of
    * {@link FileContext#renameSnapshot(Path, String, String)}.
+   *
+   * @param path the path.
+   * @param snapshotOldName snapshot old name.
+   * @param snapshotNewName snapshot new name.
+   * @throws IOException raised on errors performing I/O.
    */
   public void renameSnapshot(final Path path, final String snapshotOldName,
       final String snapshotNewName) throws IOException {
@@ -1279,6 +1501,10 @@ public void renameSnapshot(final Path path, final String snapshotOldName,
   /**
    * The specification of this method matches that of
    * {@link FileContext#deleteSnapshot(Path, String)}.
+   *
+   * @param snapshotDir snapshot dir.
+   * @param snapshotName snapshot name.
+   * @throws IOException raised on errors performing I/O.
    */
   public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
       throws IOException {
@@ -1289,7 +1515,7 @@ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
   /**
    * Set the source path to satisfy storage policy.
    * @param path The source path referring to either a directory or a file.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void satisfyStoragePolicy(final Path path) throws IOException {
     throw new UnsupportedOperationException(
@@ -1303,6 +1529,7 @@ public void satisfyStoragePolicy(final Path path) throws IOException {
    * @param policyName the name of the target storage policy. The list
    *                   of supported Storage policies can be retrieved
    *                   via {@link #getAllStoragePolicies}.
+   * @throws IOException raised on errors performing I/O.
    */
   public void setStoragePolicy(final Path path, final String policyName)
       throws IOException {
@@ -1314,7 +1541,7 @@ public void setStoragePolicy(final Path path, final String policyName)
   /**
    * Unset the storage policy set for a given file or directory.
    * @param src file or directory path.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void unsetStoragePolicy(final Path src) throws IOException {
     throw new UnsupportedOperationException(getClass().getSimpleName()
@@ -1326,7 +1553,7 @@ public void unsetStoragePolicy(final Path src) throws IOException {
    *
    * @param src file or directory path.
    * @return storage policy for give file.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public BlockStoragePolicySpi getStoragePolicy(final Path src)
       throws IOException {
@@ -1338,7 +1565,7 @@ public BlockStoragePolicySpi getStoragePolicy(final Path src)
    * Retrieve all the storage policies supported by this file system.
    *
    * @return all storage policies supported by this filesystem.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
index b4a4a85674dfa..7518dd2f7ef74 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
@@ -25,6 +25,10 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
 /** Adapts an {@link FSDataInputStream} to Avro's SeekableInput interface. */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
@@ -32,17 +36,31 @@ public class AvroFSInput implements Closeable, SeekableInput {
   private final FSDataInputStream stream;
   private final long len;
 
-  /** Construct given an {@link FSDataInputStream} and its length. */
+  /**
+   * Construct given an {@link FSDataInputStream} and its length.
+   *
+   * @param in inputstream.
+   * @param len len.
+   */
   public AvroFSInput(final FSDataInputStream in, final long len) {
     this.stream = in;
     this.len = len;
   }
 
-  /** Construct given a {@link FileContext} and a {@link Path}. */
+  /** Construct given a {@link FileContext} and a {@link Path}.
+   * @param fc filecontext.
+   * @param p the path.
+   * @throws IOException If an I/O error occurred.
+   * */
   public AvroFSInput(final FileContext fc, final Path p) throws IOException {
     FileStatus status = fc.getFileStatus(p);
     this.len = status.getLen();
-    this.stream = fc.open(p);
+    this.stream = awaitFuture(fc.openFile(p)
+        .opt(FS_OPTION_OPENFILE_READ_POLICY,
+            FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+        .withFileStatus(status)
+        .build());
+    fc.open(p);
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java
index 607fffbcc701a..e693bcbfe89fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java
@@ -68,6 +68,7 @@ public BatchedRemoteIterator(K prevKey) {
    * 
    * @param prevKey The key to send.
    * @return A list of replies.
+   * @throws IOException If an I/O error occurred.
    */
   public abstract BatchedEntries<E> makeRequest(K prevKey) throws IOException;
 
@@ -102,6 +103,8 @@ public boolean hasNext() throws IOException {
 
   /**
    * Return the next list key associated with an element.
+   * @param element element.
+   * @return K Generics Type.
    */
   public abstract K elementToPrevKey(E element);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
index 29358dd7d1086..67687c1f0e04c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
@@ -85,6 +85,7 @@ public BlockLocation() {
 
   /**
    * Copy constructor.
+   * @param that blocklocation.
    */
   public BlockLocation(BlockLocation that) {
     this.hosts = that.hosts;
@@ -100,6 +101,10 @@ public BlockLocation(BlockLocation that) {
 
   /**
    * Constructor with host, name, offset and length.
+   * @param names names array.
+   * @param hosts host array.
+   * @param offset offset.
+   * @param length length.
    */
   public BlockLocation(String[] names, String[] hosts, long offset, 
                        long length) {
@@ -108,6 +113,11 @@ public BlockLocation(String[] names, String[] hosts, long offset,
 
   /**
    * Constructor with host, name, offset, length and corrupt flag.
+   * @param names names.
+   * @param hosts hosts.
+   * @param offset offset.
+   * @param length length.
+   * @param corrupt corrupt.
    */
   public BlockLocation(String[] names, String[] hosts, long offset, 
                        long length, boolean corrupt) {
@@ -116,6 +126,11 @@ public BlockLocation(String[] names, String[] hosts, long offset,
 
   /**
    * Constructor with host, name, network topology, offset and length.
+   * @param names names.
+   * @param hosts hosts.
+   * @param topologyPaths topologyPaths.
+   * @param offset offset.
+   * @param length length.
    */
   public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
                        long offset, long length) {
@@ -125,6 +140,12 @@ public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
   /**
    * Constructor with host, name, network topology, offset, length 
    * and corrupt flag.
+   * @param names names.
+   * @param hosts hosts.
+   * @param topologyPaths topologyPaths.
+   * @param offset offset.
+   * @param length length.
+   * @param corrupt corrupt.
    */
   public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
                        long offset, long length, boolean corrupt) {
@@ -177,6 +198,8 @@ public BlockLocation(String[] names, String[] hosts, String[] cachedHosts,
 
   /**
    * Get the list of hosts (hostname) hosting this block.
+   * @return hosts array.
+   * @throws IOException If an I/O error occurred.
    */
   public String[] getHosts() throws IOException {
     return hosts;
@@ -184,6 +207,7 @@ public String[] getHosts() throws IOException {
 
   /**
    * Get the list of hosts (hostname) hosting a cached replica of the block.
+   * @return cached hosts.
    */
   public String[] getCachedHosts() {
     return cachedHosts;
@@ -191,6 +215,8 @@ public String[] getCachedHosts() {
 
   /**
    * Get the list of names (IP:xferPort) hosting this block.
+   * @return names array.
+   * @throws IOException If an I/O error occurred.
    */
   public String[] getNames() throws IOException {
     return names;
@@ -199,6 +225,8 @@ public String[] getNames() throws IOException {
   /**
    * Get the list of network topology paths for each of the hosts.
    * The last component of the path is the "name" (IP:xferPort).
+   * @return topology paths.
+   * @throws IOException If an I/O error occurred.
    */
   public String[] getTopologyPaths() throws IOException {
     return topologyPaths;
@@ -206,6 +234,7 @@ public String[] getTopologyPaths() throws IOException {
 
   /**
    * Get the storageID of each replica of the block.
+   * @return storage ids.
    */
   public String[] getStorageIds() {
     return storageIds;
@@ -213,6 +242,7 @@ public String[] getStorageIds() {
 
   /**
    * Get the storage type of each replica of the block.
+   * @return storage type of each replica of the block.
    */
   public StorageType[] getStorageTypes() {
     return storageTypes;
@@ -220,6 +250,7 @@ public StorageType[] getStorageTypes() {
 
   /**
    * Get the start offset of file associated with this block.
+   * @return start offset of file associated with this block.
    */
   public long getOffset() {
     return offset;
@@ -227,6 +258,7 @@ public long getOffset() {
   
   /**
    * Get the length of the block.
+   * @return length of the block.
    */
   public long getLength() {
     return length;
@@ -234,6 +266,7 @@ public long getLength() {
 
   /**
    * Get the corrupt flag.
+   * @return corrupt flag.
    */
   public boolean isCorrupt() {
     return corrupt;
@@ -241,6 +274,7 @@ public boolean isCorrupt() {
 
   /**
    * Return true if the block is striped (erasure coded).
+   * @return if the block is striped true, not false.
    */
   public boolean isStriped() {
     return false;
@@ -248,6 +282,7 @@ public boolean isStriped() {
 
   /**
    * Set the start offset of file associated with this block.
+   * @param offset start offset.
    */
   public void setOffset(long offset) {
     this.offset = offset;
@@ -255,6 +290,7 @@ public void setOffset(long offset) {
 
   /**
    * Set the length of block.
+   * @param length length of block.
    */
   public void setLength(long length) {
     this.length = length;
@@ -262,6 +298,7 @@ public void setLength(long length) {
 
   /**
    * Set the corrupt flag.
+   * @param corrupt corrupt flag.
    */
   public void setCorrupt(boolean corrupt) {
     this.corrupt = corrupt;
@@ -269,6 +306,8 @@ public void setCorrupt(boolean corrupt) {
 
   /**
    * Set the hosts hosting this block.
+   * @param hosts hosts array.
+   * @throws IOException If an I/O error occurred.
    */
   public void setHosts(String[] hosts) throws IOException {
     if (hosts == null) {
@@ -280,6 +319,7 @@ public void setHosts(String[] hosts) throws IOException {
 
   /**
    * Set the hosts hosting a cached replica of this block.
+   * @param cachedHosts cached hosts.
    */
   public void setCachedHosts(String[] cachedHosts) {
     if (cachedHosts == null) {
@@ -291,6 +331,8 @@ public void setCachedHosts(String[] cachedHosts) {
 
   /**
    * Set the names (host:port) hosting this block.
+   * @param names names.
+   * @throws IOException If an I/O error occurred.
    */
   public void setNames(String[] names) throws IOException {
     if (names == null) {
@@ -302,6 +344,9 @@ public void setNames(String[] names) throws IOException {
 
   /**
    * Set the network topology paths of the hosts.
+   *
+   * @param topologyPaths topology paths.
+   * @throws IOException If an I/O error occurred.
    */
   public void setTopologyPaths(String[] topologyPaths) throws IOException {
     if (topologyPaths == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
index 59345f5d25caf..7f3171235c8f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -22,6 +22,9 @@
 import java.io.FileDescriptor;
 import java.io.IOException;
 import java.util.StringJoiner;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.function.IntFunction;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -158,8 +161,24 @@ public IOStatistics getIOStatistics() {
   @Override
   public String toString() {
     return new StringJoiner(", ",
-        BufferedFSInputStream.class.getSimpleName() + "[", "]")
-        .add("in=" + in)
-        .toString();
+            BufferedFSInputStream.class.getSimpleName() + "[", "]")
+            .add("in=" + in)
+            .toString();
+  }
+
+  @Override
+  public int minSeekForVectorReads() {
+    return ((PositionedReadable) in).minSeekForVectorReads();
+  }
+
+  @Override
+  public int maxReadSizeForVectorReads() {
+    return ((PositionedReadable) in).maxReadSizeForVectorReads();
+  }
+
+  @Override
+  public void readVectored(List<? extends FileRange> ranges,
+                           IntFunction<ByteBuffer> allocate) throws IOException {
+    ((PositionedReadable) in).readVectored(ranges, allocate);
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java
index 6576fe5827d94..f577649dd5fce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java
@@ -47,6 +47,12 @@ private static boolean streamHasByteBufferRead(InputStream stream) {
 
   /**
    * Perform a fallback read.
+   *
+   * @param stream input stream.
+   * @param bufferPool bufferPool.
+   * @param maxLength maxLength.
+   * @throws IOException raised on errors performing I/O.
+   * @return byte buffer.
    */
   public static ByteBuffer fallbackRead(
       InputStream stream, ByteBufferPool bufferPool, int maxLength)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
index 58dc82d2efb2d..d7b61346d4e3b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,6 +53,9 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
   /**
    * This is the constructor used by the builder.
    * All overriding classes should implement this.
+   *
+   * @param builder builder.
+   * @throws IOException raised on errors performing I/O.
    */
   public CachingGetSpaceUsed(CachingGetSpaceUsed.Builder builder)
       throws IOException {
@@ -89,19 +93,19 @@ void init() {
       if (!shouldFirstRefresh) {
         // Skip initial refresh operation, so we need to do first refresh
         // operation immediately in refresh thread.
-        initRefeshThread(true);
+        initRefreshThread(true);
         return;
       }
       refresh();
     }
-    initRefeshThread(false);
+    initRefreshThread(false);
   }
 
   /**
    * RunImmediately should set true, if we skip the first refresh.
    * @param runImmediately The param default should be false.
    */
-  private void initRefeshThread (boolean runImmediately) {
+  private void initRefreshThread(boolean runImmediately) {
     if (refreshInterval > 0) {
       refreshUsed = new Thread(new RefreshThread(this, runImmediately),
           "refreshUsed-" + dirPath);
@@ -139,6 +143,8 @@ public String getDirPath() {
 
   /**
    * Increment the cached value of used space.
+   *
+   * @param value dfs used value.
    */
   public void incDfsUsed(long value) {
     used.addAndGet(value);
@@ -153,11 +159,25 @@ boolean running() {
 
   /**
    * How long in between runs of the background refresh.
+   *
+   * @return refresh interval.
    */
-  long getRefreshInterval() {
+  @VisibleForTesting
+  public long getRefreshInterval() {
     return refreshInterval;
   }
 
+  /**
+   * Randomize the refresh interval timing by this amount, the actual interval will be chosen
+   * uniformly between {@code interval-jitter} and {@code interval+jitter}.
+   *
+   * @return between interval-jitter and interval+jitter.
+   */
+  @VisibleForTesting
+  public long getJitter() {
+    return jitter;
+  }
+
   /**
    * Reset the current used data amount. This should be called
    * when the cached value is re-computed.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index c7f8e36c3f675..4c7569d6ecd81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -22,18 +22,24 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
 import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.function.IntFunction;
+import java.util.zip.CRC32;
 
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
+import org.apache.hadoop.fs.impl.CombinedFileRange;
 import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
 import org.apache.hadoop.fs.impl.OpenFileParameters;
 import org.apache.hadoop.fs.permission.AclEntry;
@@ -45,8 +51,10 @@
 import org.apache.hadoop.util.LambdaUtils;
 import org.apache.hadoop.util.Progressable;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
 import static org.apache.hadoop.fs.impl.StoreImplementationUtils.isProbeForSyncable;
+import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
 
 /****************************************************************
  * Abstract Checksumed FileSystem.
@@ -66,7 +74,7 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
   public static double getApproxChkSumLength(long size) {
     return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size;
   }
-  
+
   public ChecksumFileSystem(FileSystem fs) {
     super(fs);
   }
@@ -82,7 +90,7 @@ public void setConf(Configuration conf) {
           bytesPerChecksum);
     }
   }
-  
+
   /**
    * Set whether to verify checksum.
    */
@@ -95,32 +103,51 @@ public void setVerifyChecksum(boolean verifyChecksum) {
   public void setWriteChecksum(boolean writeChecksum) {
     this.writeChecksum = writeChecksum;
   }
-  
+
   /** get the raw file system */
   @Override
   public FileSystem getRawFileSystem() {
     return fs;
   }
 
-  /** Return the name of the checksum file associated with a file.*/
+  /**
+   * Return the name of the checksum file associated with a file.
+   *
+   * @param file the file path.
+   * @return name of the checksum file associated with a file.
+   */
   public Path getChecksumFile(Path file) {
     return new Path(file.getParent(), "." + file.getName() + ".crc");
   }
 
-  /** Return true iff file is a checksum file name.*/
+  /**
+   * Return true if file is a checksum file name.
+   *
+   * @param file the file path.
+   * @return if file is a checksum file true, not false.
+   */
   public static boolean isChecksumFile(Path file) {
     String name = file.getName();
     return name.startsWith(".") && name.endsWith(".crc");
   }
 
-  /** Return the length of the checksum file given the size of the 
+  /**
+   * Return the length of the checksum file given the size of the
    * actual file.
-   **/
+   *
+   * @param file the file path.
+   * @param fileSize file size.
+   * @return checksum length.
+   */
   public long getChecksumFileLength(Path file, long fileSize) {
     return getChecksumLength(fileSize, getBytesPerSum());
   }
 
-  /** Return the bytes Per Checksum */
+  /**
+   * Return the bytes Per Checksum.
+   *
+   * @return bytes per check sum.
+   */
   public int getBytesPerSum() {
     return bytesPerChecksum;
   }
@@ -139,22 +166,23 @@ private int getSumBufferSize(int bytesPerSum, int bufferSize) {
    * It verifies that data matches checksums.
    *******************************************************/
   private static class ChecksumFSInputChecker extends FSInputChecker implements
-      IOStatisticsSource {
+      IOStatisticsSource, StreamCapabilities {
     private ChecksumFileSystem fs;
     private FSDataInputStream datas;
     private FSDataInputStream sums;
-    
+
     private static final int HEADER_LENGTH = 8;
-    
+
     private int bytesPerSum = 1;
-    
+    private long fileLen = -1L;
+
     public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
       throws IOException {
       this(fs, file, fs.getConf().getInt(
-                       LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, 
+                       LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
                        LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT));
     }
-    
+
     public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
       throws IOException {
       super( file, fs.getFileStatus(file).getReplication() );
@@ -170,7 +198,8 @@ public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
         if (!Arrays.equals(version, CHECKSUM_VERSION))
           throw new IOException("Not a checksum file: "+sumFile);
         this.bytesPerSum = sums.readInt();
-        set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4);
+        set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum,
+            FSInputChecker.CHECKSUM_SIZE);
       } catch (IOException e) {
         // mincing the message is terrible, but java throws permission
         // exceptions as FNF because that's all the method signatures allow!
@@ -182,21 +211,21 @@ public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
         set(fs.verifyChecksum, null, 1, 0);
       }
     }
-    
+
     private long getChecksumFilePos( long dataPos ) {
-      return HEADER_LENGTH + 4*(dataPos/bytesPerSum);
+      return HEADER_LENGTH + FSInputChecker.CHECKSUM_SIZE*(dataPos/bytesPerSum);
     }
-    
+
     @Override
     protected long getChunkPosition( long dataPos ) {
       return dataPos/bytesPerSum*bytesPerSum;
     }
-    
+
     @Override
     public int available() throws IOException {
       return datas.available() + super.available();
     }
-    
+
     @Override
     public int read(long position, byte[] b, int off, int len)
       throws IOException {
@@ -214,7 +243,7 @@ public int read(long position, byte[] b, int off, int len)
       }
       return nread;
     }
-    
+
     @Override
     public void close() throws IOException {
       datas.close();
@@ -223,7 +252,7 @@ public void close() throws IOException {
       }
       set(fs.verifyChecksum, null, 1, 0);
     }
-    
+
 
     @Override
     public boolean seekToNewSource(long targetPos) throws IOException {
@@ -246,7 +275,7 @@ protected int readChunk(long pos, byte[] buf, int offset, int len,
         final int checksumsToRead = Math.min(
           len/bytesPerSum, // number of checksums based on len to read
           checksum.length / CHECKSUM_SIZE); // size of checksum buffer
-        long checksumPos = getChecksumFilePos(pos); 
+        long checksumPos = getChecksumFilePos(pos);
         if(checksumPos != sums.getPos()) {
           sums.seek(checksumPos);
         }
@@ -286,8 +315,187 @@ protected int readChunk(long pos, byte[] buf, int offset, int len,
     public IOStatistics getIOStatistics() {
       return IOStatisticsSupport.retrieveIOStatistics(datas);
     }
+
+    public static long findChecksumOffset(long dataOffset,
+                                          int bytesPerSum) {
+      return HEADER_LENGTH + (dataOffset/bytesPerSum) * FSInputChecker.CHECKSUM_SIZE;
+    }
+
+    /**
+     * Calculate length of file if not already cached.
+     * @return file length.
+     * @throws IOException any IOE.
+     */
+    private long getFileLength() throws IOException {
+      if (fileLen == -1L) {
+        fileLen = fs.getFileStatus(file).getLen();
+      }
+      return fileLen;
+    }
+
+    /**
+     * Find the checksum ranges that correspond to the given data ranges.
+     * @param dataRanges the input data ranges, which are assumed to be sorted
+     *                   and non-overlapping
+     * @return a list of AsyncReaderUtils.CombinedFileRange that correspond to
+     *         the checksum ranges
+     */
+    public static List<CombinedFileRange> findChecksumRanges(
+        List<? extends FileRange> dataRanges,
+        int bytesPerSum,
+        int minSeek,
+        int maxSize) {
+      List<CombinedFileRange> result = new ArrayList<>();
+      CombinedFileRange currentCrc = null;
+      for(FileRange range: dataRanges) {
+        long crcOffset = findChecksumOffset(range.getOffset(), bytesPerSum);
+        long crcEnd = findChecksumOffset(range.getOffset() + range.getLength() +
+                                             bytesPerSum - 1, bytesPerSum);
+        if (currentCrc == null ||
+                !currentCrc.merge(crcOffset, crcEnd, range, minSeek, maxSize)) {
+          currentCrc = new CombinedFileRange(crcOffset, crcEnd, range);
+          result.add(currentCrc);
+        }
+      }
+      return result;
+    }
+
+    /**
+     * Check the data against the checksums.
+     * @param sumsBytes the checksum data
+     * @param sumsOffset where from the checksum file this buffer started
+     * @param data the file data
+     * @param dataOffset where the file data started (must be a multiple of
+     *                  bytesPerSum)
+     * @param bytesPerSum how many bytes per a checksum
+     * @param file the path of the filename
+     * @return the data buffer
+     * @throws CompletionException if the checksums don't match
+     */
+    static ByteBuffer checkBytes(ByteBuffer sumsBytes,
+                                 long sumsOffset,
+                                 ByteBuffer data,
+                                 long dataOffset,
+                                 int bytesPerSum,
+                                 Path file) {
+      // determine how many bytes we need to skip at the start of the sums
+      int offset =
+          (int) (findChecksumOffset(dataOffset, bytesPerSum) - sumsOffset);
+      IntBuffer sums = sumsBytes.asIntBuffer();
+      sums.position(offset / FSInputChecker.CHECKSUM_SIZE);
+      ByteBuffer current = data.duplicate();
+      int numFullChunks = data.remaining() / bytesPerSum;
+      boolean partialChunk = ((data.remaining() % bytesPerSum) != 0);
+      int totalChunks = numFullChunks;
+      if (partialChunk) {
+        totalChunks++;
+      }
+      CRC32 crc = new CRC32();
+      // check each chunk to ensure they match
+      for(int c = 0; c < totalChunks; ++c) {
+        // set the buffer position to the start of every chunk.
+        current.position(c * bytesPerSum);
+
+        if (c == numFullChunks) {
+          // During last chunk, there may be less than chunk size
+          // data preset, so setting the limit accordingly.
+          int lastIncompleteChunk = data.remaining() % bytesPerSum;
+          current.limit((c * bytesPerSum) + lastIncompleteChunk);
+        } else {
+          // set the buffer limit to end of every chunk.
+          current.limit((c + 1) * bytesPerSum);
+        }
+
+        // compute the crc
+        crc.reset();
+        crc.update(current);
+        int expected = sums.get();
+        int calculated = (int) crc.getValue();
+
+        if (calculated != expected) {
+          // cast of c added to silence findbugs
+          long errPosn = dataOffset + (long) c * bytesPerSum;
+          throw new CompletionException(new ChecksumException(
+              "Checksum error: " + file + " at " + errPosn +
+                  " exp: " + expected + " got: " + calculated, errPosn));
+        }
+      }
+      // if everything matches, we return the data
+      return data;
+    }
+
+    /**
+     * Validates range parameters.
+     * In case of CheckSum FS, we already have calculated
+     * fileLength so failing fast here.
+     * @param ranges requested ranges.
+     * @param fileLength length of file.
+     * @throws EOFException end of file exception.
+     */
+    private void validateRangeRequest(List<? extends FileRange> ranges,
+                                      final long fileLength) throws EOFException {
+      for (FileRange range : ranges) {
+        VectoredReadUtils.validateRangeRequest(range);
+        if (range.getOffset() + range.getLength() > fileLength) {
+          final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s",
+                  range.getOffset(), range.getLength(), file);
+          LOG.warn(errMsg);
+          throw new EOFException(errMsg);
+        }
+      }
+    }
+
+    @Override
+    public void readVectored(List<? extends FileRange> ranges,
+                             IntFunction<ByteBuffer> allocate) throws IOException {
+      final long length = getFileLength();
+      validateRangeRequest(ranges, length);
+
+      // If the stream doesn't have checksums, just delegate.
+      if (sums == null) {
+        datas.readVectored(ranges, allocate);
+        return;
+      }
+      int minSeek = minSeekForVectorReads();
+      int maxSize = maxReadSizeForVectorReads();
+      List<CombinedFileRange> dataRanges =
+          VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(ranges)), bytesPerSum,
+              minSeek, maxReadSizeForVectorReads());
+      // While merging the ranges above, they are rounded up based on the value of bytesPerSum
+      // which leads to some ranges crossing the EOF thus they need to be fixed else it will
+      // cause EOFException during actual reads.
+      for (CombinedFileRange range : dataRanges) {
+        if (range.getOffset() + range.getLength() > length) {
+          range.setLength((int) (length - range.getOffset()));
+        }
+      }
+      List<CombinedFileRange> checksumRanges = findChecksumRanges(dataRanges,
+          bytesPerSum, minSeek, maxSize);
+      sums.readVectored(checksumRanges, allocate);
+      datas.readVectored(dataRanges, allocate);
+      for(CombinedFileRange checksumRange: checksumRanges) {
+        for(FileRange dataRange: checksumRange.getUnderlying()) {
+          // when we have both the ranges, validate the checksum
+          CompletableFuture<ByteBuffer> result =
+              checksumRange.getData().thenCombineAsync(dataRange.getData(),
+                  (sumBuffer, dataBuffer) ->
+                      checkBytes(sumBuffer, checksumRange.getOffset(),
+                          dataBuffer, dataRange.getOffset(), bytesPerSum, file));
+          // Now, slice the read data range to the user's ranges
+          for(FileRange original: ((CombinedFileRange) dataRange).getUnderlying()) {
+            original.setData(result.thenApply(
+                (b) -> VectoredReadUtils.sliceTo(b, dataRange.getOffset(), original)));
+          }
+        }
+      }
+    }
+
+    @Override
+    public boolean hasCapability(String capability) {
+      return datas.hasCapability(capability);
+    }
   }
-  
+
   private static class FSDataBoundedInputStream extends FSDataInputStream {
     private FileSystem fs;
     private Path file;
@@ -298,12 +506,12 @@ private static class FSDataBoundedInputStream extends FSDataInputStream {
       this.fs = fs;
       this.file = file;
     }
-    
+
     @Override
     public boolean markSupported() {
       return false;
     }
-    
+
     /* Return the file length */
     private long getFileLength() throws IOException {
       if( fileLen==-1L ) {
@@ -311,7 +519,7 @@ private long getFileLength() throws IOException {
       }
       return fileLen;
     }
-    
+
     /**
      * Skips over and discards <code>n</code> bytes of data from the
      * input stream.
@@ -335,11 +543,11 @@ public synchronized long skip(long n) throws IOException {
       }
       return super.skip(n);
     }
-    
+
     /**
      * Seek to the given position in the stream.
      * The next read() will be from that position.
-     * 
+     *
      * <p>This method does not allow seek past the end of the file.
      * This produces IOException.
      *
@@ -362,6 +570,7 @@ public synchronized void seek(long pos) throws IOException {
    * Opens an FSDataInputStream at the indicated Path.
    * @param f the file name to open
    * @param bufferSize the size of the buffer to be used.
+   * @throws IOException if an I/O error occurs.
    */
   @Override
   public FSDataInputStream open(Path f, int bufferSize) throws IOException {
@@ -404,22 +613,22 @@ public void concat(final Path f, final Path[] psrcs) throws IOException {
    */
   public static long getChecksumLength(long size, int bytesPerSum) {
     //the checksum length is equal to size passed divided by bytesPerSum +
-    //bytes written in the beginning of the checksum file.  
-    return ((size + bytesPerSum - 1) / bytesPerSum) * 4 +
-             CHECKSUM_VERSION.length + 4;  
+    //bytes written in the beginning of the checksum file.
+    return ((size + bytesPerSum - 1) / bytesPerSum) * FSInputChecker.CHECKSUM_SIZE +
+             ChecksumFSInputChecker.HEADER_LENGTH;
   }
 
   /** This class provides an output stream for a checksummed file.
    * It generates checksums for data. */
   private static class ChecksumFSOutputSummer extends FSOutputSummer
       implements IOStatisticsSource, StreamCapabilities {
-    private FSDataOutputStream datas;    
+    private FSDataOutputStream datas;
     private FSDataOutputStream sums;
     private static final float CHKSUM_AS_FRACTION = 0.01f;
     private boolean isClosed = false;
-    
-    public ChecksumFSOutputSummer(ChecksumFileSystem fs, 
-                          Path file, 
+
+    ChecksumFSOutputSummer(ChecksumFileSystem fs,
+                          Path file,
                           boolean overwrite,
                           int bufferSize,
                           short replication,
@@ -440,7 +649,7 @@ public ChecksumFSOutputSummer(ChecksumFileSystem fs,
       sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
       sums.writeInt(bytesPerSum);
     }
-    
+
     @Override
     public void close() throws IOException {
       try {
@@ -451,7 +660,7 @@ public void close() throws IOException {
         isClosed = true;
       }
     }
-    
+
     @Override
     protected void writeChunk(byte[] b, int offset, int len, byte[] checksum,
         int ckoff, int cklen)
@@ -669,7 +878,7 @@ boolean apply(Path p) throws IOException {
    * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
    * @param src file name
    * @param replication new replication
-   * @throws IOException
+   * @throws IOException if an I/O error occurs.
    * @return true if successful;
    *         false if file does not exist or is a directory
    */
@@ -707,7 +916,7 @@ public boolean rename(Path src, Path dst) throws IOException {
         value = fs.rename(srcCheckFile, dstCheckFile);
       } else if (fs.exists(dstCheckFile)) {
         // no src checksum, so remove dst checksum
-        value = fs.delete(dstCheckFile, true); 
+        value = fs.delete(dstCheckFile, true);
       }
 
       return value;
@@ -739,7 +948,7 @@ public boolean delete(Path f, boolean recursive) throws IOException{
       return fs.delete(f, true);
     }
   }
-    
+
   final private static PathFilter DEFAULT_FILTER = new PathFilter() {
     @Override
     public boolean accept(Path file) {
@@ -750,11 +959,11 @@ public boolean accept(Path file) {
   /**
    * List the statuses of the files/directories in the given path if the path is
    * a directory.
-   * 
+   *
    * @param f
    *          given path
    * @return the statuses of the files/directories in the given path
-   * @throws IOException
+   * @throws IOException if an I/O error occurs.
    */
   @Override
   public FileStatus[] listStatus(Path f) throws IOException {
@@ -771,18 +980,18 @@ public RemoteIterator<FileStatus> listStatusIterator(final Path p)
   /**
    * List the statuses of the files/directories in the given path if the path is
    * a directory.
-   * 
+   *
    * @param f
    *          given path
    * @return the statuses of the files/directories in the given patch
-   * @throws IOException
+   * @throws IOException if an I/O error occurs.
    */
   @Override
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
   throws IOException {
     return fs.listLocatedStatus(f, DEFAULT_FILTER);
   }
-  
+
   @Override
   public boolean mkdirs(Path f) throws IOException {
     return fs.mkdirs(f);
@@ -811,6 +1020,10 @@ public void copyToLocalFile(boolean delSrc, Path src, Path dst)
    * Copy it from FS control to the local dst name.
    * If src and dst are directories, the copyCrc parameter
    * determines whether to copy CRC files.
+   * @param src src path.
+   * @param dst dst path.
+   * @param copyCrc copy csc flag.
+   * @throws IOException if an I/O error occurs.
    */
   @SuppressWarnings("deprecation")
   public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
@@ -832,7 +1045,7 @@ public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
     } else {
       FileStatus[] srcs = listStatus(src);
       for (FileStatus srcFile : srcs) {
-        copyToLocalFile(srcFile.getPath(), 
+        copyToLocalFile(srcFile.getPath(),
                         new Path(dst, srcFile.getPath().getName()), copyCrc);
       }
     }
@@ -889,7 +1102,7 @@ protected CompletableFuture<FSDataInputStream> openFileWithOptions(
       final OpenFileParameters parameters) throws IOException {
     AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
         parameters.getMandatoryKeys(),
-        Collections.emptySet(),
+        FS_OPTION_OPENFILE_STANDARD_OPTIONS,
         "for " + path);
     return LambdaUtils.eval(
         new CompletableFuture<>(),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index bc1122c56a2bd..4820c5c3045d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -70,30 +70,53 @@ public void setVerifyChecksum(boolean inVerifyChecksum) {
     this.verifyChecksum = inVerifyChecksum;
   }
 
-  /** get the raw file system. */
+  /**
+   * get the raw file system.
+   *
+   * @return abstract file system.
+   */
   public AbstractFileSystem getRawFs() {
     return getMyFs();
   }
 
-  /** Return the name of the checksum file associated with a file.*/
+  /**
+   * Return the name of the checksum file associated with a file.
+   *
+   * @param file the file path.
+   * @return the checksum file associated with a file.
+   */
   public Path getChecksumFile(Path file) {
     return new Path(file.getParent(), "." + file.getName() + ".crc");
   }
 
-  /** Return true iff file is a checksum file name.*/
+  /**
+   * Return true iff file is a checksum file name.
+   *
+   * @param file the file path.
+   * @return if is checksum file true,not false.
+   */
   public static boolean isChecksumFile(Path file) {
     String name = file.getName();
     return name.startsWith(".") && name.endsWith(".crc");
   }
 
-  /** Return the length of the checksum file given the size of the 
+  /**
+   * Return the length of the checksum file given the size of the
    * actual file.
-   **/
+   *
+   * @param file the file path.
+   * @param fileSize file size.
+   * @return check sum file length.
+   */
   public long getChecksumFileLength(Path file, long fileSize) {
     return getChecksumLength(fileSize, getBytesPerSum());
   }
 
-  /** Return the bytes Per Checksum. */
+  /**
+   * Return the bytes Per Checksum.
+   *
+   * @return bytes per sum.
+   */
   public int getBytesPerSum() {
     return defaultBytesPerChecksum;
   }
@@ -433,7 +456,7 @@ private boolean isDirectory(Path f)
    * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
    * @param src file name
    * @param replication new replication
-   * @throws IOException
+   * @throws IOException if an I/O error occurs.
    * @return true if successful;
    *         false if file does not exist or is a directory
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 6949c67f278d1..9d6224366d1ba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -399,6 +399,12 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
   public static final String ZK_ACL_DEFAULT = "world:anyone:rwcda";
   /** Authentication for the ZooKeeper ensemble. */
   public static final String ZK_AUTH = ZK_PREFIX + "auth";
+  /** Principal name for zookeeper servers. */
+  public static final String ZK_SERVER_PRINCIPAL = ZK_PREFIX + "server.principal";
+  /** Kerberos principal name for zookeeper connection. */
+  public static final String ZK_KERBEROS_PRINCIPAL = ZK_PREFIX + "kerberos.principal";
+  /** Kerberos keytab for zookeeper connection. */
+  public static final String ZK_KERBEROS_KEYTAB = ZK_PREFIX + "kerberos.keytab";
 
   /** Address of the ZooKeeper ensemble. */
   public static final String ZK_ADDRESS = ZK_PREFIX + "address";
@@ -469,4 +475,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
    * default hadoop temp dir on local system: {@value}.
    */
   public static final String HADOOP_TMP_DIR = "hadoop.tmp.dir";
+
+  /**
+   * Thread-level IOStats Support.
+   * {@value}
+   */
+  public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
+      "fs.iostatistics.thread.level.enabled";
+
+  /**
+   * Default value for Thread-level IOStats Support is true.
+   */
+  public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
+      true;
+
+  public static final String HADOOP_SECURITY_RESOLVER_IMPL =
+      "hadoop.security.resolver.impl";
+
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index a799e883bcf2a..67cd81ee91a96 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -169,11 +169,11 @@ public class CommonConfigurationKeysPublic {
 
   /**
    * Number of filesystems instances can be created in parallel.
-   * <p></p>
+   * <p>
    * A higher number here does not necessarily improve performance, especially
    * for object stores, where multiple threads may be attempting to create an FS
    * instance for the same URI.
-   * <p></p>
+   * </p>
    * Default value: {@value}.
    */
   public static final String FS_CREATION_PARALLEL_COUNT =
@@ -181,8 +181,9 @@ public class CommonConfigurationKeysPublic {
 
   /**
    * Default value for {@link #FS_CREATION_PARALLEL_COUNT}.
-   * <p></p>
+   * <p>
    * Default value: {@value}.
+   * </p>
    */
   public static final int FS_CREATION_PARALLEL_COUNT_DEFAULT =
       64;
@@ -999,6 +1000,7 @@ public class CommonConfigurationKeysPublic {
       String.join(",",
           "secret$",
           "password$",
+          "username$",
           "ssl.keystore.pass$",
           "fs.s3.*[Ss]ecret.?[Kk]ey",
           "fs.s3a.*.server-side-encryption.key",
@@ -1053,5 +1055,13 @@ public class CommonConfigurationKeysPublic {
   public static final String HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY =
       "hadoop.http.idle_timeout.ms";
   public static final int HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT = 60000;
+
+  /**
+   * To configure scheduling of server metrics update thread. This config is used to indicate
+   * initial delay and delay between each execution of the metric update runnable thread.
+   */
+  public static final String IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL =
+      "ipc.server.metrics.update.runner.interval";
+  public static final int IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL_DEFAULT = 5000;
 }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
index df932df43aebd..aa231554eb0cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
@@ -146,4 +146,22 @@ private CommonPathCapabilities() {
    */
   public static final String ABORTABLE_STREAM =
       "fs.capability.outputstream.abortable";
+
+  /**
+   * Does this FS support etags?
+   * That is: will FileStatus entries from listing/getFileStatus
+   * probes support EtagSource and return real values.
+   */
+  public static final String ETAGS_AVAILABLE =
+      "fs.capability.etags.available";
+
+  /**
+   * Are etags guaranteed to be preserved across rename() operations..
+   * FileSystems MUST NOT declare support for this feature
+   * unless this holds.
+   */
+  public static final String ETAGS_PRESERVED_IN_RENAME =
+      "fs.capability.etags.preserved.in.rename";
+
+
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
index e1ed5cbcfcaa6..bdbc8f3a33f4b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
@@ -37,7 +37,13 @@ public class CompositeCrcFileChecksum extends FileChecksum {
   private DataChecksum.Type crcType;
   private int bytesPerCrc;
 
-  /** Create a CompositeCrcFileChecksum. */
+  /**
+   * Create a CompositeCrcFileChecksum.
+   *
+   * @param crc crc.
+   * @param crcType crcType.
+   * @param bytesPerCrc bytesPerCrc.
+   */
   public CompositeCrcFileChecksum(
       int crc, DataChecksum.Type crcType, int bytesPerCrc) {
     this.crc = crc;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 79850e1a2f291..9f97a12fa6088 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -149,17 +149,31 @@ public ContentSummary build() {
   @Deprecated
   public ContentSummary() {}
   
-  /** Constructor, deprecated by ContentSummary.Builder
+  /**
+   *  Constructor, deprecated by ContentSummary.Builder
    *  This constructor implicitly set spaceConsumed the same as length.
    *  spaceConsumed and length must be set explicitly with
-   *  ContentSummary.Builder
+   *  ContentSummary.Builder.
+   *
+   * @param length length.
+   * @param fileCount file count.
+   * @param directoryCount directory count.
    * */
   @Deprecated
   public ContentSummary(long length, long fileCount, long directoryCount) {
     this(length, fileCount, directoryCount, -1L, length, -1L);
   }
 
-  /** Constructor, deprecated by ContentSummary.Builder */
+  /**
+   * Constructor, deprecated by ContentSummary.Builder.
+   *
+   * @param length length.
+   * @param fileCount file count.
+   * @param directoryCount directory count.
+   * @param quota quota.
+   * @param spaceConsumed space consumed.
+   * @param spaceQuota space quota.
+   * */
   @Deprecated
   public ContentSummary(
       long length, long fileCount, long directoryCount, long quota,
@@ -172,7 +186,11 @@ public ContentSummary(
     setSpaceQuota(spaceQuota);
   }
 
-  /** Constructor for ContentSummary.Builder*/
+  /**
+   * Constructor for ContentSummary.Builder.
+   *
+   * @param builder builder.
+   */
   private ContentSummary(Builder builder) {
     super(builder);
     this.length = builder.length;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index 71993713ad2eb..ca008e536931d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -189,6 +189,8 @@ public static void validate(Object path, boolean pathExists,
   /**
    * Validate the CreateFlag for the append operation. The flag must contain
    * APPEND, and cannot contain OVERWRITE.
+   *
+   * @param flag enum set flag.
    */
   public static void validateForAppend(EnumSet<CreateFlag> flag) {
     validate(flag);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
index da4636b2c0fbe..c5a052f3de4be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
@@ -65,7 +65,10 @@ public String getDirPath() {
     return dirPath;
   }
 
-  /** @return a string indicating which filesystem volume we're checking. */
+  /**
+   * @return a string indicating which filesystem volume we're checking.
+   * @throws IOException raised on errors performing I/O.
+   */
   public String getFilesystem() throws IOException {
     if (Shell.WINDOWS) {
       this.filesystem = dirFile.getCanonicalPath().substring(0, 2);
@@ -100,7 +103,10 @@ public int getPercentUsed() {
     return (int) (used * 100.0 / cap);
   }
 
-  /** @return the filesystem mount point for the indicated volume */
+  /**
+   * @return the filesystem mount point for the indicated volume.
+   * @throws IOException raised on errors performing I/O.
+   */
   public String getMount() throws IOException {
     // Abort early if specified path does not exist
     if (!dirFile.exists()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
index 33905dcbb77fd..794855508c63f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
@@ -47,7 +47,11 @@ public interface Renewable {
     /** @return the renew token. */
     public Token<?> getRenewToken();
 
-    /** Set delegation token. */
+    /**
+     * Set delegation token.
+     * @param <T> generic type T.
+     * @param token token.
+     */
     public <T extends TokenIdentifier> void setDelegationToken(Token<T> token);
   }
 
@@ -172,7 +176,11 @@ public String toString() {
   /** Queue to maintain the RenewActions to be processed by the {@link #run()} */
   private volatile DelayQueue<RenewAction<?>> queue = new DelayQueue<RenewAction<?>>();
   
-  /** For testing purposes */
+  /**
+   * For testing purposes.
+   *
+   * @return renew queue length.
+   */
   @VisibleForTesting
   protected int getRenewQueueLength() {
     return queue.size();
@@ -211,7 +219,13 @@ static synchronized void reset() {
     }
   }
   
-  /** Add a renew action to the queue. */
+  /**
+   * Add a renew action to the queue.
+   *
+   * @param <T> generic type T.
+   * @param fs file system.
+   * @return renew action.
+   * */
   @SuppressWarnings("static-access")
   public <T extends FileSystem & Renewable> RenewAction<T> addRenewAction(final T fs) {
     synchronized (this) {
@@ -230,8 +244,10 @@ public <T extends FileSystem & Renewable> RenewAction<T> addRenewAction(final T
 
   /**
    * Remove the associated renew action from the queue
-   * 
-   * @throws IOException
+   *
+   * @param <T> generic type T.
+   * @param fs file system.
+   * @throws IOException raised on errors performing I/O.
    */
   public <T extends FileSystem & Renewable> void removeRenewAction(
       final T fs) throws IOException {
@@ -240,9 +256,8 @@ public <T extends FileSystem & Renewable> void removeRenewAction(
       try {
         action.cancel();
       } catch (InterruptedException ie) {
-        LOG.error("Interrupted while canceling token for " + fs.getUri()
-            + "filesystem");
-        LOG.debug("Exception in removeRenewAction: {}", ie);
+        LOG.error("Interrupted while canceling token for {} filesystem.", fs.getUri());
+        LOG.debug("Exception in removeRenewAction.", ie);
       }
     }
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java
new file mode 100644
index 0000000000000..d7efdc705d8e5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/EtagSource.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+/**
+ * An optional interface for {@link FileStatus} subclasses to implement
+ * to provide access to etags.
+ * If available FS SHOULD also implement the matching PathCapabilities
+ *   -- etag supported: {@link CommonPathCapabilities#ETAGS_AVAILABLE}.
+ *   -- etag consistent over rename:
+ *      {@link CommonPathCapabilities#ETAGS_PRESERVED_IN_RENAME}.
+ */
+public interface EtagSource {
+
+  /**
+   * Return an etag of this file status.
+   * A return value of null or "" means "no etag"
+   * @return a possibly null or empty etag.
+   */
+  String getEtag();
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java
index b7757a62e28ad..56ef51f128db8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java
@@ -37,12 +37,17 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
 
   /**
    * Set optional Builder parameter.
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    */
   B opt(@Nonnull String key, @Nonnull String value);
 
   /**
    * Set optional boolean parameter for the Builder.
-   *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #opt(String, String)
    */
   B opt(@Nonnull String key, boolean value);
@@ -50,6 +55,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set optional int parameter for the Builder.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #opt(String, String)
    */
   B opt(@Nonnull String key, int value);
@@ -57,13 +65,29 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set optional float parameter for the Builder.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #opt(String, String)
    */
   B opt(@Nonnull String key, float value);
 
+  /**
+   * Set optional long parameter for the Builder.
+   *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
+   * @see #opt(String, String)
+   */
+  B opt(@Nonnull String key, long value);
+
   /**
    * Set optional double parameter for the Builder.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #opt(String, String)
    */
   B opt(@Nonnull String key, double value);
@@ -71,6 +95,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set an array of string values as optional parameter for the Builder.
    *
+   * @param key key.
+   * @param values values.
+   * @return generic type B.
    * @see #opt(String, String)
    */
   B opt(@Nonnull String key, @Nonnull String... values);
@@ -80,12 +107,19 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
    *
    * If the option is not supported or unavailable,
    * the client should expect {@link #build()} throws IllegalArgumentException.
+   *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    */
   B must(@Nonnull String key, @Nonnull String value);
 
   /**
    * Set mandatory boolean option.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #must(String, String)
    */
   B must(@Nonnull String key, boolean value);
@@ -93,6 +127,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set mandatory int option.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #must(String, String)
    */
   B must(@Nonnull String key, int value);
@@ -100,13 +137,29 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set mandatory float option.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #must(String, String)
    */
   B must(@Nonnull String key, float value);
 
+  /**
+   * Set mandatory long option.
+   *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
+   * @see #must(String, String)
+   */
+  B must(@Nonnull String key, long value);
+
   /**
    * Set mandatory double option.
    *
+   * @param key key.
+   * @param value value.
+   * @return generic type B.
    * @see #must(String, String)
    */
   B must(@Nonnull String key, double value);
@@ -114,6 +167,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
   /**
    * Set a string array as mandatory option.
    *
+   * @param key key.
+   * @param values values.
+   * @return generic type B.
    * @see #must(String, String)
    */
   B must(@Nonnull String key, @Nonnull String... values);
@@ -125,6 +181,7 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
    * @throws UnsupportedOperationException if the filesystem does not support
    * the specific operation.
    * @throws IOException on filesystem IO errors.
+   * @return generic type S.
    */
   S build() throws IllegalArgumentException,
       UnsupportedOperationException, IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
index b143a4cb63d19..cca6c28da11a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
@@ -1,4 +1,4 @@
-/**
+/*
  * 
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -26,6 +26,8 @@
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.EnumSet;
+import java.util.List;
+import java.util.function.IntFunction;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -51,7 +53,7 @@ public class FSDataInputStream extends DataInputStream
    */
   private final IdentityHashStore<ByteBuffer, ByteBufferPool>
     extendedReadBuffers
-      = new IdentityHashStore<ByteBuffer, ByteBufferPool>(0);
+      = new IdentityHashStore<>(0);
 
   public FSDataInputStream(InputStream in) {
     super(in);
@@ -142,7 +144,8 @@ public boolean seekToNewSource(long targetPos) throws IOException {
    *
    * @return the underlying input stream
    */
-  @InterfaceAudience.LimitedPrivate({"HDFS"})
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public InputStream getWrappedStream() {
     return in;
   }
@@ -279,4 +282,20 @@ public void readFully(long position, ByteBuffer buf) throws IOException {
   public IOStatistics getIOStatistics() {
     return IOStatisticsSupport.retrieveIOStatistics(in);
   }
+
+  @Override
+  public int minSeekForVectorReads() {
+    return ((PositionedReadable) in).minSeekForVectorReads();
+  }
+
+  @Override
+  public int maxReadSizeForVectorReads() {
+    return ((PositionedReadable) in).maxReadSizeForVectorReads();
+  }
+
+  @Override
+  public void readVectored(List<? extends FileRange> ranges,
+                           IntFunction<ByteBuffer> allocate) throws IOException {
+    ((PositionedReadable) in).readVectored(ranges, allocate);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index c96d499d17ba6..16938a83a69c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -123,6 +123,9 @@ public abstract class FSDataOutputStreamBuilder
 
   /**
    * Constructor.
+   *
+   * @param fileSystem file system.
+   * @param p the path.
    */
   protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
       @Nonnull Path p) {
@@ -149,6 +152,9 @@ protected FsPermission getPermission() {
 
   /**
    * Set permission for the file.
+   *
+   * @param perm permission.
+   * @return B Generics Type.
    */
   public B permission(@Nonnull final FsPermission perm) {
     checkNotNull(perm);
@@ -162,6 +168,9 @@ protected int getBufferSize() {
 
   /**
    * Set the size of the buffer to be used.
+   *
+   * @param bufSize buffer size.
+   * @return Generics Type B.
    */
   public B bufferSize(int bufSize) {
     bufferSize = bufSize;
@@ -174,6 +183,9 @@ protected short getReplication() {
 
   /**
    * Set replication factor.
+   *
+   * @param replica replica.
+   * @return Generics Type B.
    */
   public B replication(short replica) {
     replication = replica;
@@ -186,6 +198,9 @@ protected long getBlockSize() {
 
   /**
    * Set block size.
+   *
+   * @param blkSize block size.
+   * @return B Generics Type.
    */
   public B blockSize(long blkSize) {
     blockSize = blkSize;
@@ -194,6 +209,8 @@ public B blockSize(long blkSize) {
 
   /**
    * Return true to create the parent directories if they do not exist.
+   *
+   * @return if create the parent directories if they do not exist true,not false.
    */
   protected boolean isRecursive() {
     return recursive;
@@ -201,6 +218,8 @@ protected boolean isRecursive() {
 
   /**
    * Create the parent directory if they do not exist.
+   *
+   * @return B Generics Type.
    */
   public B recursive() {
     recursive = true;
@@ -213,6 +232,9 @@ protected Progressable getProgress() {
 
   /**
    * Set the facility of reporting progress.
+   *
+   * @param prog progress.
+   * @return B Generics Type.
    */
   public B progress(@Nonnull final Progressable prog) {
     checkNotNull(prog);
@@ -226,6 +248,8 @@ protected EnumSet<CreateFlag> getFlags() {
 
   /**
    * Create an FSDataOutputStream at the specified path.
+   *
+   * @return return Generics Type B.
    */
   public B create() {
     flags.add(CreateFlag.CREATE);
@@ -236,6 +260,9 @@ public B create() {
    * Set to true to overwrite the existing file.
    * Set it to false, an exception will be thrown when calling {@link #build()}
    * if the file exists.
+   *
+   * @param overwrite overrite.
+   * @return Generics Type B.
    */
   public B overwrite(boolean overwrite) {
     if (overwrite) {
@@ -248,6 +275,8 @@ public B overwrite(boolean overwrite) {
 
   /**
    * Append to an existing file (optional operation).
+   *
+   * @return Generics Type B.
    */
   public B append() {
     flags.add(CreateFlag.APPEND);
@@ -260,6 +289,9 @@ protected ChecksumOpt getChecksumOpt() {
 
   /**
    * Set checksum opt.
+   *
+   * @param chksumOpt check sum opt.
+   * @return Generics Type B.
    */
   public B checksumOpt(@Nonnull final ChecksumOpt chksumOpt) {
     checkNotNull(chksumOpt);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
index de66eab713ab6..ee16ca8a2cd50 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
@@ -82,6 +82,7 @@ protected FSInputChecker( Path file, int numOfRetries) {
    * @param sum the type of Checksum engine
    * @param chunkSize maximun chunk size
    * @param checksumSize the number byte of each checksum
+   * @param verifyChecksum verify check sum.
    */
   protected FSInputChecker( Path file, int numOfRetries, 
       boolean verifyChecksum, Checksum sum, int chunkSize, int checksumSize ) {
@@ -118,6 +119,7 @@ protected FSInputChecker( Path file, int numOfRetries,
    * @param len maximum number of bytes to read
    * @param checksum the data buffer into which to write checksums
    * @return number of bytes read
+   * @throws IOException raised on errors performing I/O.
    */
   abstract protected int readChunk(long pos, byte[] buf, int offset, int len,
       byte[] checksum) throws IOException;
@@ -129,7 +131,10 @@ abstract protected int readChunk(long pos, byte[] buf, int offset, int len,
    */
   abstract protected long getChunkPosition(long pos);
 
-  /** Return true if there is a need for checksum verification */
+  /**
+   * Return true if there is a need for checksum verification.
+   * @return if there is a need for checksum verification true, not false.
+   */
   protected synchronized boolean needChecksum() {
     return verifyChecksum && sum != null;
   }
@@ -357,6 +362,9 @@ private void verifySums(final byte b[], final int off, int read)
    * Convert a checksum byte array to a long
    * This is deprecated since 0.22 since it is no longer in use
    * by this class.
+   *
+   * @param checksum check sum.
+   * @return crc.
    */
   @Deprecated
   static public long checksum2long(byte[] checksum) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
index ffe4b34ca5fdb..f85cf7a858152 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
@@ -74,7 +74,7 @@ abstract public T next(final AbstractFileSystem fs, final Path p)
    * @param fc FileContext used to access file systems.
    * @param path The path to resolve symlinks on.
    * @return Generic type determined by the implementation of next.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public T resolve(final FileContext fc, final Path path) throws IOException {
     int count = 0;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index 6de026b9d17c0..4ef512dc257a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -186,6 +186,8 @@ public void flush() throws IOException {
 
   /**
    * Return the number of valid bytes currently in the buffer.
+   *
+   * @return buffer data size.
    */
   protected synchronized int getBufferedDataSize() {
     return count;
@@ -227,6 +229,10 @@ private void writeChecksumChunks(byte b[], int off, int len)
 
   /**
    * Converts a checksum integer value to a byte stream
+   *
+   * @param sum check sum.
+   * @param checksumSize check sum size.
+   * @return byte stream.
    */
   static public byte[] convertToByteStream(Checksum sum, int checksumSize) {
     return int2byte((int)sum.getValue(), new byte[checksumSize]);
@@ -245,6 +251,8 @@ static byte[] int2byte(int integer, byte[] bytes) {
 
   /**
    * Resets existing buffer with a new one of the specified size.
+   *
+   * @param size size.
    */
   protected synchronized void setChecksumBufSize(int size) {
     this.buf = new byte[size];
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
index 6822fa485622f..62d2e3af78671 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
@@ -28,20 +28,37 @@
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public abstract class FileChecksum implements Writable {
-  /** The checksum algorithm name */
+  /**
+   * The checksum algorithm name.
+   *
+   * @return algorithm name.
+   */
   public abstract String getAlgorithmName();
 
-  /** The length of the checksum in bytes */
+  /**
+   * The length of the checksum in bytes.
+   *
+   * @return length.
+   */
   public abstract int getLength();
 
-  /** The value of the checksum in bytes */
+  /**
+   * The value of the checksum in bytes.
+   *
+   * @return byte array.
+   */
   public abstract byte[] getBytes();
 
   public ChecksumOpt getChecksumOpt() {
     return null;
   }
 
-  /** Return true if both the algorithms and the values are the same. */
+  /**
+   * Return true if both the algorithms and the values are the same.
+   *
+   * @param other other.
+   * @return if equal true, not false.
+   */
   @Override
   public boolean equals(Object other) {
     if (other == this) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 9922dfa0ac8b8..22ac2ecbd7949 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -70,7 +70,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * The FileContext class provides an interface for users of the Hadoop
@@ -361,8 +366,8 @@ public AbstractFileSystem run() throws UnsupportedFileSystemException {
    * Create a FileContext with specified FS as default using the specified
    * config.
    * 
-   * @param defFS
-   * @param aConf
+   * @param defFS default fs.
+   * @param aConf configutration.
    * @return new FileContext with specified FS as default.
    */
   public static FileContext getFileContext(final AbstractFileSystem defFS,
@@ -373,7 +378,7 @@ public static FileContext getFileContext(final AbstractFileSystem defFS,
   /**
    * Create a FileContext for specified file system using the default config.
    * 
-   * @param defaultFS
+   * @param defaultFS default fs.
    * @return a FileContext with the specified AbstractFileSystem
    *                 as the default FS.
    */
@@ -406,6 +411,7 @@ protected static FileContext getFileContext(
    * 
    * @throws UnsupportedFileSystemException If the file system from the default
    *           configuration is not supported
+   * @return file context.
    */
   public static FileContext getFileContext()
       throws UnsupportedFileSystemException {
@@ -425,7 +431,7 @@ public static FileContext getLocalFSFileContext()
   /**
    * Create a FileContext for specified URI using the default config.
    * 
-   * @param defaultFsUri
+   * @param defaultFsUri defaultFsUri.
    * @return a FileContext with the specified URI as the default FS.
    * 
    * @throws UnsupportedFileSystemException If the file system for
@@ -439,8 +445,8 @@ public static FileContext getFileContext(final URI defaultFsUri)
   /**
    * Create a FileContext for specified default URI using the specified config.
    * 
-   * @param defaultFsUri
-   * @param aConf
+   * @param defaultFsUri defaultFsUri.
+   * @param aConf configrution.
    * @return new FileContext for specified uri
    * @throws UnsupportedFileSystemException If the file system with specified is
    *           not supported
@@ -471,7 +477,7 @@ public static FileContext getFileContext(final URI defaultFsUri,
    * {@link #getFileContext(URI, Configuration)} instead of this one.
    * 
    * 
-   * @param aConf
+   * @param aConf configration.
    * @return new FileContext
    * @throws UnsupportedFileSystemException If file system in the config
    *           is not supported
@@ -549,6 +555,7 @@ public void setWorkingDirectory(final Path newWDir) throws IOException {
   
   /**
    * Gets the working directory for wd-relative names (such a "foo/bar").
+   * @return the path.
    */
   public Path getWorkingDirectory() {
     return workingDir;
@@ -595,13 +602,14 @@ public void setUMask(final FsPermission newUmask) {
    * @throws FileNotFoundException  If <code>f</code> does not exist
    * @throws AccessControlException if access denied
    * @throws IOException If an IO Error occurred
-   * 
+   * @throws UnresolvedLinkException If unresolved link occurred.
+   *
    * Exceptions applicable to file systems accessed over RPC:
    * @throws RpcClientException If an exception occurred in the RPC client
    * @throws RpcServerException If an exception occurred in the RPC server
    * @throws UnexpectedServerException If server implementation throws
    *           undeclared exception to RPC server
-   * 
+   *
    * RuntimeExceptions:
    * @throws InvalidPathException If path <code>f</code> is not valid
    */
@@ -615,7 +623,7 @@ public Path resolvePath(final Path f) throws FileNotFoundException,
    * A Fully-qualified path has scheme and authority specified and an absolute
    * path.
    * Use the default file system and working dir in this FileContext to qualify.
-   * @param path
+   * @param path the path.
    * @return qualified path
    */
   public Path makeQualified(final Path path) {
@@ -754,6 +762,7 @@ public FSDataOutputStream build() throws IOException {
    *
    * Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
    * same exceptions as create(Path, EnumSet, CreateOpts...).
+   * @throws IOException If an I/O error occurred.
    */
   public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
       throws IOException {
@@ -827,6 +836,8 @@ public Void next(final AbstractFileSystem fs, final Path p)
    * 
    * RuntimeExceptions:
    * @throws InvalidPathException If path <code>f</code> is invalid
+   *
+   * @return if delete success true, not false.
    */
   public boolean delete(final Path f, final boolean recursive)
       throws AccessControlException, FileNotFoundException,
@@ -857,6 +868,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
    * @throws RpcServerException If an exception occurred in the RPC server
    * @throws UnexpectedServerException If server implementation throws 
    *           undeclared exception to RPC server
+   * @return input stream.
    */
   public FSDataInputStream open(final Path f) throws AccessControlException,
       FileNotFoundException, UnsupportedFileSystemException, IOException {
@@ -887,6 +899,7 @@ public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
    * @throws RpcServerException If an exception occurred in the RPC server
    * @throws UnexpectedServerException If server implementation throws 
    *           undeclared exception to RPC server
+   * @return output stream.
    */
   public FSDataInputStream open(final Path f, final int bufferSize)
       throws AccessControlException, FileNotFoundException,
@@ -996,6 +1009,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
    * 
    * @param src path to be renamed
    * @param dst new path after rename
+   * @param options rename options.
    * 
    * @throws AccessControlException If access is denied
    * @throws FileAlreadyExistsException If <code>dst</code> already exists and
@@ -1047,7 +1061,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
   
   /**
    * Set permission of a path.
-   * @param f
+   * @param f the path.
    * @param permission - the new absolute permission (umask is not applied)
    *
    * @throws AccessControlException If access is denied
@@ -1191,7 +1205,7 @@ public FileChecksum next(final AbstractFileSystem fs, final Path p)
    * Set the verify checksum flag for the  file system denoted by the path.
    * This is only applicable if the 
    * corresponding FileSystem supports checksum. By default doesn't do anything.
-   * @param verifyChecksum
+   * @param verifyChecksum verify check sum.
    * @param f set the verifyChecksum for the Filesystem containing this path
    *
    * @throws AccessControlException If access is denied
@@ -1246,8 +1260,9 @@ public FileStatus next(final AbstractFileSystem fs, final Path p)
   /**
    * Synchronize client metadata state.
    *
-   * @throws IOException
-   * @throws UnsupportedOperationException
+   * @throws IOException If an I/O error occurred.
+   * @throws UnsupportedOperationException If file system for <code>f</code> is
+   *                                       not supported.
    */
   public void msync() throws IOException, UnsupportedOperationException {
     defaultFS.msync();
@@ -1608,9 +1623,12 @@ public RemoteIterator<FileStatus> next(
   }
 
   /**
+   * List CorruptFile Blocks.
+   *
+   * @param path the path.
    * @return an iterator over the corrupt files under the given path
    * (may contain duplicates if a file has more than one corrupt block)
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
@@ -1734,6 +1752,7 @@ public class Util {
      * @throws RpcServerException If an exception occurred in the RPC server
      * @throws UnexpectedServerException If server implementation throws 
      *           undeclared exception to RPC server
+     * @return if f exists true, not false.
      */
     public boolean exists(final Path f) throws AccessControlException,
       UnsupportedFileSystemException, IOException {
@@ -1794,6 +1813,12 @@ public ContentSummary getContentSummary(Path f)
     
     /**
      * See {@link #listStatus(Path[], PathFilter)}
+     *
+     * @param files files.
+     * @throws AccessControlException If access is denied.
+     * @throws FileNotFoundException If <code>files</code> does not exist.
+     * @throws IOException If an I/O error occurred.
+     * @return file status array.
      */
     public FileStatus[] listStatus(Path[] files) throws AccessControlException,
         FileNotFoundException, IOException {
@@ -2049,36 +2074,29 @@ public LocatedFileStatus next() throws IOException {
      *    <dt> <tt> ? </tt>
      *    <dd> Matches any single character.
      *
-     *    <p>
      *    <dt> <tt> * </tt>
      *    <dd> Matches zero or more characters.
      *
-     *    <p>
      *    <dt> <tt> [<i>abc</i>] </tt>
      *    <dd> Matches a single character from character set
      *     <tt>{<i>a,b,c</i>}</tt>.
      *
-     *    <p>
      *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
      *    <dd> Matches a single character from the character range
      *     <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be
      *     lexicographically less than or equal to character <tt><i>b</i></tt>.
      *
-     *    <p>
      *    <dt> <tt> [^<i>a</i>] </tt>
      *    <dd> Matches a single char that is not from character set or range
      *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
      *     immediately to the right of the opening bracket.
      *
-     *    <p>
      *    <dt> <tt> \<i>c</i> </tt>
      *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
      *
-     *    <p>
      *    <dt> <tt> {ab,cd} </tt>
      *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
-     *    
-     *    <p>
+     *
      *    <dt> <tt> {ab,c{de,fh}} </tt>
      *    <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt>
      *
@@ -2139,6 +2157,18 @@ public FileStatus[] globStatus(final Path pathPattern,
     /**
      * Copy file from src to dest. See
      * {@link #copy(Path, Path, boolean, boolean)}
+     *
+     * @param src src.
+     * @param dst dst.
+     * @throws AccessControlException If access is denied.
+     * @throws FileAlreadyExistsException If file <code>src</code> already exists.
+     * @throws FileNotFoundException if next file does not exist any more.
+     * @throws ParentNotDirectoryException If parent of <code>src</code> is not a
+     * directory.
+     * @throws UnsupportedFileSystemException If file system for
+     * <code>src/dst</code> is not supported.
+     * @throws IOException If an I/O error occurred.
+     * @return if success copy true, not false.
      */
     public boolean copy(final Path src, final Path dst)
         throws AccessControlException, FileAlreadyExistsException,
@@ -2149,8 +2179,8 @@ public boolean copy(final Path src, final Path dst)
     
     /**
      * Copy from src to dst, optionally deleting src and overwriting dst.
-     * @param src
-     * @param dst
+     * @param src src.
+     * @param dst dst.
      * @param deleteSource - delete src if true
      * @param overwrite  overwrite dst if true; throw IOException if dst exists
      *         and overwrite is false.
@@ -2198,7 +2228,12 @@ public boolean copy(final Path src, final Path dst, boolean deleteSource,
         EnumSet<CreateFlag> createFlag = overwrite ? EnumSet.of(
             CreateFlag.CREATE, CreateFlag.OVERWRITE) :
             EnumSet.of(CreateFlag.CREATE);
-        InputStream in = open(qSrc);
+        InputStream in = awaitFuture(openFile(qSrc)
+            .opt(FS_OPTION_OPENFILE_READ_POLICY,
+                FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .opt(FS_OPTION_OPENFILE_LENGTH,
+                fs.getLen())   // file length hint for object stores
+            .build());
         try (OutputStream out = create(qDst, createFlag)) {
           IOUtils.copyBytes(in, out, conf, true);
         } finally {
@@ -2266,7 +2301,7 @@ private static void checkDependencies(Path qualSrc, Path qualDst)
    * Are qualSrc and qualDst of the same file system?
    * @param qualPath1 - fully qualified path
    * @param qualPath2 - fully qualified path
-   * @return
+   * @return is same fs true,not false.
    */
   private static boolean isSameFS(Path qualPath1, Path qualPath2) {
     URI srcUri = qualPath1.toUri();
@@ -2289,6 +2324,13 @@ public synchronized void run() {
   /**
    * Resolves all symbolic links in the specified path.
    * Returns the new path object.
+   *
+   * @param f the path.
+   * @throws FileNotFoundException If <code>f</code> does not exist.
+   * @throws UnresolvedLinkException If unresolved link occurred.
+   * @throws AccessControlException If access is denied.
+   * @throws IOException If an I/O error occurred.
+   * @return resolve path.
    */
   protected Path resolve(final Path f) throws FileNotFoundException,
       UnresolvedLinkException, AccessControlException, IOException {
@@ -2306,6 +2348,7 @@ public Path next(final AbstractFileSystem fs, final Path p)
    * to, but not including the final path component.
    * @param f path to resolve
    * @return the new path object.
+   * @throws IOException If an I/O error occurred.
    */
   protected Path resolveIntermediate(final Path f) throws IOException {
     return new FSLinkResolver<FileStatus>() {
@@ -2324,13 +2367,12 @@ public FileStatus next(final AbstractFileSystem fs, final Path p)
    * @param f
    *          Path which needs to be resolved
    * @return List of AbstractFileSystems accessed in the path
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f)
       throws IOException {
     final Path absF = fixRelativePart(f);
-    final HashSet<AbstractFileSystem> result 
-      = new HashSet<AbstractFileSystem>();
+    final HashSet<AbstractFileSystem> result = new HashSet<>();
     new FSLinkResolver<Void>() {
       @Override
       public Void next(final AbstractFileSystem fs, final Path p)
@@ -2385,7 +2427,7 @@ public static Map<URI, Statistics> getAllStatistics() {
    * @param p Path for which delegations tokens are requested.
    * @param renewer the account name that is allowed to renew the token.
    * @return List of delegation tokens.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
   public List<Token<?>> getDelegationTokens(
@@ -2537,7 +2579,7 @@ public AclStatus next(final AbstractFileSystem fs, final Path p)
    * @param path Path to modify
    * @param name xattr name.
    * @param value xattr value.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void setXAttr(Path path, String name, byte[] value)
       throws IOException {
@@ -2556,7 +2598,7 @@ public void setXAttr(Path path, String name, byte[] value)
    * @param name xattr name.
    * @param value xattr value.
    * @param flag xattr set flag
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void setXAttr(Path path, final String name, final byte[] value,
       final EnumSet<XAttrSetFlag> flag) throws IOException {
@@ -2581,7 +2623,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
    * @param path Path to get extended attribute
    * @param name xattr name.
    * @return byte[] xattr value.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public byte[] getXAttr(Path path, final String name) throws IOException {
     final Path absF = fixRelativePart(path);
@@ -2604,7 +2646,7 @@ public byte[] next(final AbstractFileSystem fs, final Path p)
    * @param path Path to get extended attributes
    * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
    * of the file or directory
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public Map<String, byte[]> getXAttrs(Path path) throws IOException {
     final Path absF = fixRelativePart(path);
@@ -2628,7 +2670,7 @@ public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p)
    * @param names XAttr names.
    * @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
    * of the file or directory
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public Map<String, byte[]> getXAttrs(Path path, final List<String> names)
       throws IOException {
@@ -2651,7 +2693,7 @@ public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p)
    *
    * @param path Path to remove extended attribute
    * @param name xattr name
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void removeXAttr(Path path, final String name) throws IOException {
     final Path absF = fixRelativePart(path);
@@ -2675,7 +2717,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
    * @param path Path to get extended attributes
    * @return List{@literal <}String{@literal >} of the XAttr names of the
    * file or directory
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public List<String> listXAttrs(Path path) throws IOException {
     final Path absF = fixRelativePart(path);
@@ -2792,7 +2834,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
   /**
    * Set the source path to satisfy storage policy.
    * @param path The source path referring to either a directory or a file.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void satisfyStoragePolicy(final Path path)
       throws IOException {
@@ -2814,6 +2856,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
    * @param policyName the name of the target storage policy. The list
    *                   of supported Storage policies can be retrieved
    *                   via {@link #getAllStoragePolicies}.
+   * @throws IOException If an I/O error occurred.
    */
   public void setStoragePolicy(final Path path, final String policyName)
       throws IOException {
@@ -2831,7 +2874,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
   /**
    * Unset the storage policy set for a given file or directory.
    * @param src file or directory path.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void unsetStoragePolicy(final Path src) throws IOException {
     final Path absF = fixRelativePart(src);
@@ -2850,7 +2893,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
    *
    * @param path file or directory path.
    * @return storage policy for give file.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
     final Path absF = fixRelativePart(path);
@@ -2868,7 +2911,7 @@ public BlockStoragePolicySpi next(final AbstractFileSystem fs,
    * Retrieve all the storage policies supported by this file system.
    *
    * @return all storage policies supported by this filesystem.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
       throws IOException {
@@ -2930,9 +2973,11 @@ public CompletableFuture<FSDataInputStream> build() throws IOException {
       final Path absF = fixRelativePart(getPath());
       OpenFileParameters parameters = new OpenFileParameters()
           .withMandatoryKeys(getMandatoryKeys())
+          .withOptionalKeys(getOptionalKeys())
           .withOptions(getOptions())
-          .withBufferSize(getBufferSize())
-          .withStatus(getStatus());
+          .withStatus(getStatus())
+          .withBufferSize(
+              getOptions().getInt(FS_OPTION_OPENFILE_BUFFER_SIZE, getBufferSize()));
       return new FSLinkResolver<CompletableFuture<FSDataInputStream>>() {
         @Override
         public CompletableFuture<FSDataInputStream> next(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
index 9260b9a62c62e..f50c06cec3810 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -52,6 +52,7 @@ public class FileEncryptionInfo implements Serializable {
    * @param keyName name of the key used for the encryption zone
    * @param ezKeyVersionName name of the KeyVersion used to encrypt the
    *                         encrypted data encryption key.
+   * @param version version.
    */
   public FileEncryptionInfo(final CipherSuite suite,
       final CryptoProtocolVersion version, final byte[] edek,
@@ -134,6 +135,8 @@ public String toString() {
    *
    * NOTE:
    * Currently this method is used by CLI for backward compatibility.
+   *
+   * @return stable string.
    */
   public String toStringStable() {
     StringBuilder builder = new StringBuilder("{")
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java
new file mode 100644
index 0000000000000..97da65585d6d2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.fs.impl.FileRangeImpl;
+
+/**
+ * A byte range of a file.
+ * This is used for the asynchronous gather read API of
+ * {@link PositionedReadable#readVectored}.
+ */
+public interface FileRange {
+
+  /**
+   * Get the starting offset of the range.
+   * @return the byte offset of the start
+   */
+  long getOffset();
+
+  /**
+   * Get the length of the range.
+   * @return the number of bytes in the range.
+   */
+  int getLength();
+
+  /**
+   * Get the future data for this range.
+   * @return the future for the {@link ByteBuffer} that contains the data
+   */
+  CompletableFuture<ByteBuffer> getData();
+
+  /**
+   * Set a future for this range's data.
+   * This method is called by {@link PositionedReadable#readVectored} to store the
+   * data for the user to pick up later via {@link #getData}.
+   * @param data the future of the ByteBuffer that will have the data
+   */
+  void setData(CompletableFuture<ByteBuffer> data);
+
+  /**
+   * Get any reference passed in to the file range constructor.
+   * This is not used by any implementation code; it is to help
+   * bind this API to libraries retrieving multiple stripes of
+   * data in parallel.
+   * @return a reference or null.
+   */
+  Object getReference();
+
+  /**
+   * Factory method to create a FileRange object.
+   * @param offset starting offset of the range.
+   * @param length length of the range.
+   * @return a new instance of FileRangeImpl.
+   */
+  static FileRange createFileRange(long offset, int length) {
+    return new FileRangeImpl(offset, length, null);
+  }
+
+  /**
+   * Factory method to create a FileRange object.
+   * @param offset starting offset of the range.
+   * @param length length of the range.
+   * @param reference nullable reference to store in the range.
+   * @return a new instance of FileRangeImpl.
+   */
+  static FileRange createFileRange(long offset, int length, Object reference) {
+    return new FileRangeImpl(offset, length, reference);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index d7ca8f172f8e2..fcef578b072f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -116,6 +116,17 @@ public FileStatus(long length, boolean isdir, int block_replication,
 
   /**
    * Constructor for file systems on which symbolic links are not supported
+   *
+   * @param length length.
+   * @param isdir isdir.
+   * @param block_replication block replication.
+   * @param blocksize block size.
+   * @param modification_time modification time.
+   * @param access_time access_time.
+   * @param permission permission.
+   * @param owner owner.
+   * @param group group.
+   * @param path the path.
    */
   public FileStatus(long length, boolean isdir,
                     int block_replication,
@@ -182,6 +193,7 @@ public FileStatus(long length, boolean isdir, int block_replication,
    * Copy constructor.
    *
    * @param other FileStatus to copy
+   * @throws IOException raised on errors performing I/O.
    */
   public FileStatus(FileStatus other) throws IOException {
     // It's important to call the getters here instead of directly accessing the
@@ -375,6 +387,8 @@ protected void setGroup(String group) {
 
   /**
    * @return The contents of the symbolic link.
+   *
+   * @throws IOException raised on errors performing I/O.
    */
   public Path getSymlink() throws IOException {
     if (!isSymlink()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index fdb1a47552025..df8530784615d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -21,7 +21,6 @@
 import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.lang.ref.WeakReference;
 import java.lang.ref.ReferenceQueue;
 import java.net.URI;
@@ -88,6 +87,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
 import static org.apache.hadoop.util.Preconditions.checkArgument;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
@@ -103,13 +103,13 @@
  * All user code that may potentially use the Hadoop Distributed
  * File System should be written to use a FileSystem object or its
  * successor, {@link FileContext}.
- *
+ * </p>
  * <p>
  * The local implementation is {@link LocalFileSystem} and distributed
  * implementation is DistributedFileSystem. There are other implementations
  * for object stores and (outside the Apache Hadoop codebase),
  * third party filesystems.
- * <p>
+ * </p>
  * Notes
  * <ol>
  * <li>The behaviour of the filesystem is
@@ -132,13 +132,12 @@
  * New methods may be marked as Unstable or Evolving for their initial release,
  * as a warning that they are new and may change based on the
  * experience of use in applications.
- * <p></p>
+ * <p>
  * <b>Important note for developers</b>
- * <p></p>
+ * </p>
  * If you are making changes here to the public API or protected methods,
  * you must review the following subclasses and make sure that
  * they are filtering/passing through new methods as appropriate.
- * <p></p>
  *
  * {@link FilterFileSystem}: methods are passed through. If not,
  * then {@code TestFilterFileSystem.MustNotImplement} must be
@@ -147,21 +146,22 @@
  * {@link #hasPathCapability(Path, String)} then
  * {@link FilterFileSystem#hasPathCapability(Path, String)}
  * must return false, always.
- * <p></p>
+ * <p>
  * {@link ChecksumFileSystem}: checksums are created and
  * verified.
- * <p></p>
+ * </p>
  * {@code TestHarFileSystem} will need its {@code MustNotImplement}
  * interface updated.
- * <p></p>
  *
+ * <p>
  * There are some external places your changes will break things.
  * Do co-ordinate changes here.
- * <p></p>
+ * </p>
  *
  * HBase: HBoss
- * <p></p>
+ * <p>
  * Hive: HiveShim23
+ * </p>
  * {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}
  *
  *****************************************************************/
@@ -280,6 +280,8 @@ public FileSystem run() throws IOException {
   /**
    * Returns the configured FileSystem implementation.
    * @param conf the configuration to use
+   * @return FileSystem.
+   * @throws IOException If an I/O error occurred.
    */
   public static FileSystem get(Configuration conf) throws IOException {
     return get(getDefaultUri(conf), conf);
@@ -374,6 +376,7 @@ public String getScheme() {
    * implement that method.
    *
    * @see #canonicalizeUri(URI)
+   * @return the URI of this filesystem.
    */
   protected URI getCanonicalUri() {
     return canonicalizeUri(getUri());
@@ -390,6 +393,7 @@ protected URI getCanonicalUri() {
    * not specified and if {@link #getDefaultPort()} returns a
    * default port.
    *
+   * @param uri url.
    * @return URI
    * @see NetUtils#getCanonicalUri(URI, int)
    */
@@ -453,11 +457,21 @@ public String getCanonicalServiceName() {
       : null;
   }
 
-  /** @deprecated call {@link #getUri()} instead.*/
+  /**
+   * @return uri to string.
+   * @deprecated call {@link #getUri()} instead.
+   */
   @Deprecated
   public String getName() { return getUri().toString(); }
 
-  /** @deprecated call {@link #get(URI, Configuration)} instead. */
+  /**
+   * @deprecated call {@link #get(URI, Configuration)} instead.
+   *
+   * @param name name.
+   * @param conf configuration.
+   * @return file system.
+   * @throws IOException If an I/O error occurred.
+   */
   @Deprecated
   public static FileSystem getNamed(String name, Configuration conf)
     throws IOException {
@@ -512,6 +526,9 @@ public static LocalFileSystem getLocal(Configuration conf)
    *   configuration and URI, cached and returned to the caller.
    * </li>
    * </ol>
+   * @param uri uri of the filesystem.
+   * @param conf configrution.
+   * @return filesystem instance.
    * @throws IOException if the FileSystem cannot be instantiated.
    */
   public static FileSystem get(URI uri, Configuration conf) throws IOException {
@@ -541,7 +558,7 @@ public static FileSystem get(URI uri, Configuration conf) throws IOException {
   /**
    * Returns the FileSystem for this URI's scheme and authority and the
    * given user. Internally invokes {@link #newInstance(URI, Configuration)}
-   * @param uri of the filesystem
+   * @param uri uri of the filesystem.
    * @param conf the configuration to use
    * @param user to perform the get as
    * @return filesystem instance
@@ -859,6 +876,7 @@ protected void checkPath(Path path) {
    * @param start offset into the given file
    * @param len length for which to get locations for
    * @throws IOException IO failure
+   * @return block location array.
    */
   public BlockLocation[] getFileBlockLocations(FileStatus file,
       long start, long len) throws IOException {
@@ -899,6 +917,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus file,
    * @param len length for which to get locations for
    * @throws FileNotFoundException when the path does not exist
    * @throws IOException IO failure
+   * @return block location array.
    */
   public BlockLocation[] getFileBlockLocations(Path p,
       long start, long len) throws IOException {
@@ -961,6 +980,7 @@ public Path resolvePath(final Path p) throws IOException {
    * @param f the file name to open
    * @param bufferSize the size of the buffer to be used.
    * @throws IOException IO failure
+   * @return input stream.
    */
   public abstract FSDataInputStream open(Path f, int bufferSize)
     throws IOException;
@@ -969,6 +989,7 @@ public abstract FSDataInputStream open(Path f, int bufferSize)
    * Opens an FSDataInputStream at the indicated Path.
    * @param f the file to open
    * @throws IOException IO failure
+   * @return input stream.
    */
   public FSDataInputStream open(Path f) throws IOException {
     return open(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@@ -986,6 +1007,7 @@ public FSDataInputStream open(Path f) throws IOException {
    * @throws IOException IO failure
    * @throws UnsupportedOperationException If {@link #open(PathHandle, int)}
    *                                       not overridden by subclass
+   * @return input stream.
    */
   public FSDataInputStream open(PathHandle fd) throws IOException {
     return open(fd, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@@ -1003,6 +1025,7 @@ public FSDataInputStream open(PathHandle fd) throws IOException {
    *                                    not satisfied
    * @throws IOException IO failure
    * @throws UnsupportedOperationException If not overridden by subclass
+   * @return input stream.
    */
   public FSDataInputStream open(PathHandle fd, int bufferSize)
       throws IOException {
@@ -1020,6 +1043,7 @@ public FSDataInputStream open(PathHandle fd, int bufferSize)
    *         not overridden by subclass.
    * @throws UnsupportedOperationException If this FileSystem cannot enforce
    *         the specified constraints.
+   * @return path handle.
    */
   public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
     // method is final with a default so clients calling getPathHandle(stat)
@@ -1035,6 +1059,7 @@ public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
    * @param stat Referent in the target FileSystem
    * @param opt Constraints that determine the validity of the
    *            {@link PathHandle} reference.
+   * @return path handle.
    */
   protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
     throw new UnsupportedOperationException();
@@ -1045,6 +1070,7 @@ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
    * Files are overwritten by default.
    * @param f the file to create
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f) throws IOException {
     return create(f, true);
@@ -1056,6 +1082,7 @@ public FSDataOutputStream create(Path f) throws IOException {
    * @param overwrite if a file with this name already exists, then if true,
    *   the file will be overwritten, and if false an exception will be thrown.
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f, boolean overwrite)
       throws IOException {
@@ -1073,6 +1100,7 @@ public FSDataOutputStream create(Path f, boolean overwrite)
    * @param f the file to create
    * @param progress to report progress
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f, Progressable progress)
       throws IOException {
@@ -1089,6 +1117,7 @@ public FSDataOutputStream create(Path f, Progressable progress)
    * @param f the file to create
    * @param replication the replication factor
    * @throws IOException IO failure
+   * @return output stream1
    */
   public FSDataOutputStream create(Path f, short replication)
       throws IOException {
@@ -1107,6 +1136,7 @@ public FSDataOutputStream create(Path f, short replication)
    * @param replication the replication factor
    * @param progress to report progress
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f, short replication,
       Progressable progress) throws IOException {
@@ -1124,6 +1154,7 @@ public FSDataOutputStream create(Path f, short replication,
    *   the file will be overwritten, and if false an error will be thrown.
    * @param bufferSize the size of the buffer to be used.
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
                                    boolean overwrite,
@@ -1143,7 +1174,9 @@ public FSDataOutputStream create(Path f,
    * @param overwrite if a file with this name already exists, then if true,
    *   the file will be overwritten, and if false an error will be thrown.
    * @param bufferSize the size of the buffer to be used.
+   * @param progress to report progress.
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
                                    boolean overwrite,
@@ -1163,7 +1196,9 @@ public FSDataOutputStream create(Path f,
    *   the file will be overwritten, and if false an error will be thrown.
    * @param bufferSize the size of the buffer to be used.
    * @param replication required block replication for the file.
+   * @param blockSize the size of the buffer to be used.
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
       boolean overwrite,
@@ -1181,7 +1216,10 @@ public FSDataOutputStream create(Path f,
    *   the file will be overwritten, and if false an error will be thrown.
    * @param bufferSize the size of the buffer to be used.
    * @param replication required block replication for the file.
+   * @param blockSize the size of the buffer to be used.
+   * @param progress to report progress.
    * @throws IOException IO failure
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
                                             boolean overwrite,
@@ -1208,6 +1246,7 @@ public FSDataOutputStream create(Path f,
    * @param progress the progress reporter
    * @throws IOException IO failure
    * @see #setPermission(Path, FsPermission)
+   * @return output stream.
    */
   public abstract FSDataOutputStream create(Path f,
       FsPermission permission,
@@ -1229,6 +1268,7 @@ public abstract FSDataOutputStream create(Path f,
    * @param progress the progress reporter
    * @throws IOException IO failure
    * @see #setPermission(Path, FsPermission)
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
       FsPermission permission,
@@ -1255,6 +1295,7 @@ public FSDataOutputStream create(Path f,
    *        found in conf will be used.
    * @throws IOException IO failure
    * @see #setPermission(Path, FsPermission)
+   * @return output stream.
    */
   public FSDataOutputStream create(Path f,
       FsPermission permission,
@@ -1276,6 +1317,16 @@ public FSDataOutputStream create(Path f,
    * the permission with umask before calling this method.
    * This a temporary method added to support the transition from FileSystem
    * to FileContext for user applications.
+   *
+   * @param f path.
+   * @param absolutePermission permission.
+   * @param flag create flag.
+   * @param bufferSize buffer size.
+   * @param replication replication.
+   * @param blockSize block size.
+   * @param progress progress.
+   * @param checksumOpt check sum opt.
+   * @return output stream.
    * @throws IOException IO failure
    */
   @Deprecated
@@ -1330,6 +1381,11 @@ protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
    * with umask before calling this method.
    * This a temporary method added to support the transition from FileSystem
    * to FileContext for user applications.
+   *
+   * @param f the path.
+   * @param absolutePermission permission.
+   * @param createParent create parent.
+   * @throws IOException IO failure.
    */
   @Deprecated
   protected void primitiveMkdir(Path f, FsPermission absolutePermission,
@@ -1369,6 +1425,7 @@ protected void primitiveMkdir(Path f, FsPermission absolutePermission,
    * @param progress the progress reporter
    * @throws IOException IO failure
    * @see #setPermission(Path, FsPermission)
+   * @return output stream.
    */
   public FSDataOutputStream createNonRecursive(Path f,
       boolean overwrite,
@@ -1392,6 +1449,7 @@ public FSDataOutputStream createNonRecursive(Path f,
    * @param progress the progress reporter
    * @throws IOException IO failure
    * @see #setPermission(Path, FsPermission)
+   * @return output stream.
    */
    public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
        boolean overwrite, int bufferSize, short replication, long blockSize,
@@ -1415,6 +1473,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
     * @param progress the progress reporter
     * @throws IOException IO failure
     * @see #setPermission(Path, FsPermission)
+    * @return output stream.
     */
     public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
         EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
@@ -1429,6 +1488,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
    * <i>Important: the default implementation is not atomic</i>
    * @param f path to use for create
    * @throws IOException IO failure
+   * @return if create new file success true,not false.
    */
   public boolean createNewFile(Path f) throws IOException {
     if (exists(f)) {
@@ -1449,6 +1509,7 @@ public boolean createNewFile(Path f) throws IOException {
    * @throws IOException IO failure
    * @throws UnsupportedOperationException if the operation is unsupported
    *         (default).
+   * @return output stream.
    */
   public FSDataOutputStream append(Path f) throws IOException {
     return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@@ -1463,6 +1524,7 @@ public FSDataOutputStream append(Path f) throws IOException {
    * @throws IOException IO failure
    * @throws UnsupportedOperationException if the operation is unsupported
    *         (default).
+   * @return output stream.
    */
   public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
     return append(f, bufferSize, null);
@@ -1476,10 +1538,44 @@ public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
    * @throws IOException IO failure
    * @throws UnsupportedOperationException if the operation is unsupported
    *         (default).
+   * @return output stream.
    */
   public abstract FSDataOutputStream append(Path f, int bufferSize,
       Progressable progress) throws IOException;
 
+  /**
+   * Append to an existing file (optional operation).
+   * @param f the existing file to be appended.
+   * @param appendToNewBlock whether to append data to a new block
+   * instead of the end of the last partial block
+   * @throws IOException IO failure
+   * @throws UnsupportedOperationException if the operation is unsupported
+   *         (default).
+   * @return output stream.
+   */
+  public FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException {
+    return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+        IO_FILE_BUFFER_SIZE_DEFAULT), null, appendToNewBlock);
+  }
+
+  /**
+   * Append to an existing file (optional operation).
+   * This function is used for being overridden by some FileSystem like DistributedFileSystem
+   * @param f the existing file to be appended.
+   * @param bufferSize the size of the buffer to be used.
+   * @param progress for reporting progress if it is not null.
+   * @param appendToNewBlock whether to append data to a new block
+   * instead of the end of the last partial block
+   * @throws IOException IO failure
+   * @throws UnsupportedOperationException if the operation is unsupported
+   *         (default).
+   * @return output stream.
+   */
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress, boolean appendToNewBlock) throws IOException {
+    return append(f, bufferSize, progress);
+  }
+
   /**
    * Concat existing files together.
    * @param trg the path to the target destination.
@@ -1514,7 +1610,7 @@ public short getReplication(Path src) throws IOException {
    * This is the default behavior.
    * @param src file name
    * @param replication new replication
-   * @throws IOException
+   * @throws IOException an IO failure.
    * @return true if successful, or the feature in unsupported;
    *         false if replication is supported but the file does not exist,
    *         or is a directory
@@ -1543,11 +1639,12 @@ public boolean setReplication(Path src, short replication)
    * <p>
    * If OVERWRITE option is not passed as an argument, rename fails
    * if the dst already exists.
+   * </p>
    * <p>
    * If OVERWRITE option is passed as an argument, rename overwrites
    * the dst if it is a file or an empty directory. Rename fails if dst is
    * a non-empty directory.
-   * <p>
+   * </p>
    * Note that atomicity of rename is dependent on the file system
    * implementation. Please refer to the file system documentation for
    * details. This default implementation is non atomic.
@@ -1555,9 +1652,11 @@ public boolean setReplication(Path src, short replication)
    * This method is deprecated since it is a temporary method added to
    * support the transition from FileSystem to FileContext for user
    * applications.
+   * </p>
    *
    * @param src path to be renamed
    * @param dst new path after rename
+   * @param options rename options.
    * @throws FileNotFoundException src path does not exist, or the parent
    * path of dst does not exist.
    * @throws FileAlreadyExistsException dest path exists and is a file
@@ -1652,6 +1751,9 @@ public boolean truncate(Path f, long newLength) throws IOException {
 
   /**
    * Delete a file/directory.
+   * @param f the path.
+   * @throws IOException IO failure.
+   * @return if delete success true, not false.
    * @deprecated Use {@link #delete(Path, boolean)} instead.
    */
   @Deprecated
@@ -1768,6 +1870,7 @@ public boolean exists(Path f) throws IOException {
    * @param f path to check
    * @throws IOException IO failure
    * @deprecated Use {@link #getFileStatus(Path)} instead
+   * @return if f is directory true, not false.
    */
   @Deprecated
   public boolean isDirectory(Path f) throws IOException {
@@ -1785,6 +1888,7 @@ public boolean isDirectory(Path f) throws IOException {
    * @param f path to check
    * @throws IOException IO failure
    * @deprecated Use {@link #getFileStatus(Path)} instead
+   * @return if f is file true, not false.
    */
   @Deprecated
   public boolean isFile(Path f) throws IOException {
@@ -1797,6 +1901,7 @@ public boolean isFile(Path f) throws IOException {
 
   /**
    * The number of bytes in a file.
+   * @param f the path.
    * @return the number of bytes; 0 for a directory
    * @deprecated Use {@link #getFileStatus(Path)} instead.
    * @throws FileNotFoundException if the path does not resolve
@@ -1811,6 +1916,7 @@ public long getLength(Path f) throws IOException {
    * @param f path to use
    * @throws FileNotFoundException if the path does not resolve
    * @throws IOException IO failure
+   * @return content summary.
    */
   public ContentSummary getContentSummary(Path f) throws IOException {
     FileStatus status = getFileStatus(f);
@@ -1945,9 +2051,9 @@ public boolean hasMore() {
    * @param f Path to list
    * @param token opaque iteration token returned by previous call, or null
    *              if this is the first call.
-   * @return
-   * @throws FileNotFoundException
-   * @throws IOException
+   * @return directory entries.
+   * @throws FileNotFoundException when the path does not exist.
+   * @throws IOException If an I/O error occurred.
    */
   @InterfaceAudience.Private
   protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
@@ -1978,6 +2084,8 @@ private void listStatus(ArrayList<FileStatus> results, Path f,
 
   /**
    * List corrupted file blocks.
+   *
+   * @param path the path.
    * @return an iterator over the corrupt files under the given path
    * (may contain duplicates if a file has more than one corrupt block)
    * @throws UnsupportedOperationException if the operation is unsupported
@@ -2071,36 +2179,29 @@ public FileStatus[] listStatus(Path[] files, PathFilter filter)
    *    <dt> <tt> ? </tt>
    *    <dd> Matches any single character.
    *
-   *    <p>
    *    <dt> <tt> * </tt>
    *    <dd> Matches zero or more characters.
    *
-   *    <p>
    *    <dt> <tt> [<i>abc</i>] </tt>
    *    <dd> Matches a single character from character set
    *     <tt>{<i>a,b,c</i>}</tt>.
    *
-   *    <p>
    *    <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
    *    <dd> Matches a single character from the character range
    *     <tt>{<i>a...b</i>}</tt>.  Note that character <tt><i>a</i></tt> must be
    *     lexicographically less than or equal to character <tt><i>b</i></tt>.
    *
-   *    <p>
    *    <dt> <tt> [^<i>a</i>] </tt>
    *    <dd> Matches a single character that is not from character set or range
    *     <tt>{<i>a</i>}</tt>.  Note that the <tt>^</tt> character must occur
    *     immediately to the right of the opening bracket.
    *
-   *    <p>
    *    <dt> <tt> \<i>c</i> </tt>
    *    <dd> Removes (escapes) any special meaning of character <i>c</i>.
    *
-   *    <p>
    *    <dt> <tt> {ab,cd} </tt>
    *    <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
    *
-   *    <p>
    *    <dt> <tt> {ab,c{de,fh}} </tt>
    *    <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
    *
@@ -2331,6 +2432,7 @@ public LocatedFileStatus next() throws IOException {
 
   /** Return the current user's home directory in this FileSystem.
    * The default implementation returns {@code "/user/$USER/"}.
+   * @return the path.
    */
   public Path getHomeDirectory() {
     String username;
@@ -2393,6 +2495,7 @@ public boolean mkdirs(Path f) throws IOException {
    * @param f path to create
    * @param permission to apply to f
    * @throws IOException IO failure
+   * @return if mkdir success true, not false.
    */
   public abstract boolean mkdirs(Path f, FsPermission permission
       ) throws IOException;
@@ -2440,6 +2543,7 @@ public void moveFromLocalFile(Path src, Path dst)
    * @param delSrc whether to delete the src
    * @param src path
    * @param dst path
+   * @throws IOException IO failure.
    */
   public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
     throws IOException {
@@ -2554,6 +2658,7 @@ public void copyToLocalFile(boolean delSrc, Path src, Path dst,
    * @param fsOutputFile path of output file
    * @param tmpLocalFile path of local tmp file
    * @throws IOException IO failure
+   * @return the path.
    */
   public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
     throws IOException {
@@ -2601,6 +2706,7 @@ public void close() throws IOException {
   /**
    * Return the total size of all files in the filesystem.
    * @throws IOException IO failure
+   * @return the number of path used.
    */
   public long getUsed() throws IOException {
     Path path = new Path("/");
@@ -2609,7 +2715,9 @@ public long getUsed() throws IOException {
 
   /**
    * Return the total size of all files from a specified path.
+   * @param path the path.
    * @throws IOException IO failure
+   * @return the number of path content summary.
    */
   public long getUsed(Path path) throws IOException {
     return getContentSummary(path).getLength();
@@ -2632,6 +2740,7 @@ public long getBlockSize(Path f) throws IOException {
    * Return the number of bytes that large input files should be optimally
    * be split into to minimize I/O time.
    * @deprecated use {@link #getDefaultBlockSize(Path)} instead
+   * @return default block size.
    */
   @Deprecated
   public long getDefaultBlockSize() {
@@ -2684,8 +2793,8 @@ public short getDefaultReplication(Path path) {
    * In some FileSystem implementations such as HDFS metadata
    * synchronization is essential to guarantee consistency of read requests
    * particularly in HA setting.
-   * @throws IOException
-   * @throws UnsupportedOperationException
+   * @throws IOException If an I/O error occurred.
+   * @throws UnsupportedOperationException if the operation is unsupported.
    */
   public void msync() throws IOException, UnsupportedOperationException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
@@ -2761,6 +2870,8 @@ static void checkAccessPermissions(FileStatus stat, FsAction mode)
 
   /**
    * See {@link FileContext#fixRelativePart}.
+   * @param p the path.
+   * @return relative part.
    */
   protected Path fixRelativePart(Path p) {
     if (p.isUriPathAbsolute()) {
@@ -2772,6 +2883,18 @@ protected Path fixRelativePart(Path p) {
 
   /**
    * See {@link FileContext#createSymlink(Path, Path, boolean)}.
+   *
+   * @param target target path.
+   * @param link link.
+   * @param createParent create parent.
+   * @throws AccessControlException if access is denied.
+   * @throws FileAlreadyExistsException when the path does not exist.
+   * @throws FileNotFoundException when the path does not exist.
+   * @throws ParentNotDirectoryException if the parent path of dest is not
+   *                                     a directory.
+   * @throws UnsupportedFileSystemException if there was no known implementation
+   *                                        for the scheme.
+   * @throws IOException raised on errors performing I/O.
    */
   public void createSymlink(final Path target, final Path link,
       final boolean createParent) throws AccessControlException,
@@ -2785,8 +2908,14 @@ public void createSymlink(final Path target, final Path link,
 
   /**
    * See {@link FileContext#getFileLinkStatus(Path)}.
-   * @throws FileNotFoundException when the path does not exist
-   * @throws IOException see specific implementation
+   *
+   * @param f the path.
+   * @throws AccessControlException if access is denied.
+   * @throws FileNotFoundException when the path does not exist.
+   * @throws IOException raised on errors performing I/O.
+   * @throws UnsupportedFileSystemException if there was no known implementation
+   *                                        for the scheme.
+   * @return file status
    */
   public FileStatus getFileLinkStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
@@ -2797,6 +2926,7 @@ public FileStatus getFileLinkStatus(final Path f)
 
   /**
    * See {@link AbstractFileSystem#supportsSymlinks()}.
+   * @return if support symlinkls true, not false.
    */
   public boolean supportsSymlinks() {
     return false;
@@ -2804,8 +2934,11 @@ public boolean supportsSymlinks() {
 
   /**
    * See {@link FileContext#getLinkTarget(Path)}.
+   * @param f the path.
    * @throws UnsupportedOperationException if the operation is unsupported
    *         (default outcome).
+   * @throws IOException IO failure.
+   * @return the path.
    */
   public Path getLinkTarget(Path f) throws IOException {
     // Supporting filesystems should override this method
@@ -2815,8 +2948,11 @@ public Path getLinkTarget(Path f) throws IOException {
 
   /**
    * See {@link AbstractFileSystem#getLinkTarget(Path)}.
+   * @param f the path.
    * @throws UnsupportedOperationException if the operation is unsupported
    *         (default outcome).
+   * @throws IOException IO failure.
+   * @return the path.
    */
   protected Path resolveLink(Path f) throws IOException {
     // Supporting filesystems should override this method
@@ -3220,7 +3356,7 @@ public void removeXAttr(Path path, String name) throws IOException {
   /**
    * Set the source path to satisfy storage policy.
    * @param path The source path referring to either a directory or a file.
-   * @throws IOException
+   * @throws IOException If an I/O error occurred.
    */
   public void satisfyStoragePolicy(final Path path) throws IOException {
     throw new UnsupportedOperationException(
@@ -3528,7 +3664,7 @@ FileSystem getUnique(URI uri, Configuration conf) throws IOException{
      * @param conf configuration
      * @param key key to store/retrieve this FileSystem in the cache
      * @return a cached or newly instantiated FileSystem.
-     * @throws IOException
+     * @throws IOException If an I/O error occurred.
      */
     private FileSystem getInternal(URI uri, Configuration conf, Key key)
         throws IOException{
@@ -3543,11 +3679,7 @@ private FileSystem getInternal(URI uri, Configuration conf, Key key)
       // to construct an instance.
       try (DurationInfo d = new DurationInfo(LOGGER, false,
           "Acquiring creator semaphore for %s", uri)) {
-        creatorPermits.acquire();
-      } catch (InterruptedException e) {
-        // acquisition was interrupted; convert to an IOE.
-        throw (IOException)new InterruptedIOException(e.toString())
-            .initCause(e);
+        creatorPermits.acquireUninterruptibly();
       }
       FileSystem fsToClose = null;
       try {
@@ -4023,6 +4155,7 @@ public void run() {
 
     /**
      * Get or create the thread-local data associated with the current thread.
+     * @return statistics data.
      */
     public StatisticsData getThreadStatistics() {
       StatisticsData data = threadData.get();
@@ -4381,6 +4514,7 @@ public static synchronized Map<String, Statistics> getStatistics() {
   /**
    * Return the FileSystem classes that have Statistics.
    * @deprecated use {@link #getGlobalStorageStatistics()}
+   * @return statistics lists.
    */
   @Deprecated
   public static synchronized List<Statistics> getAllStatistics() {
@@ -4389,6 +4523,7 @@ public static synchronized List<Statistics> getAllStatistics() {
 
   /**
    * Get the statistics for a particular file system.
+   * @param scheme scheme.
    * @param cls the class to lookup
    * @return a statistics object
    * @deprecated use {@link #getGlobalStorageStatistics()}
@@ -4423,6 +4558,7 @@ public static synchronized void clearStatistics() {
 
   /**
    * Print all statistics for all file systems to {@code System.out}
+   * @throws IOException If an I/O error occurred.
    */
   public static synchronized
   void printStatistics() throws IOException {
@@ -4463,6 +4599,7 @@ public StorageStatistics getStorageStatistics() {
 
   /**
    * Get the global storage statistics.
+   * @return global storage statistics.
    */
   public static GlobalStorageStatistics getGlobalStorageStatistics() {
     return GlobalStorageStatistics.INSTANCE;
@@ -4616,7 +4753,7 @@ protected CompletableFuture<FSDataInputStream> openFileWithOptions(
       final OpenFileParameters parameters) throws IOException {
     AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
         parameters.getMandatoryKeys(),
-        Collections.emptySet(),
+        Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS,
         "for " + path);
     return LambdaUtils.eval(
         new CompletableFuture<>(), () ->
@@ -4644,7 +4781,7 @@ protected CompletableFuture<FSDataInputStream> openFileWithOptions(
       final OpenFileParameters parameters) throws IOException {
     AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
         parameters.getMandatoryKeys(),
-        Collections.emptySet(), "");
+        Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS, "");
     CompletableFuture<FSDataInputStream> result = new CompletableFuture<>();
     try {
       result.complete(open(pathHandle, parameters.getBufferSize()));
@@ -4751,9 +4888,11 @@ public CompletableFuture<FSDataInputStream> build() throws IOException {
       Optional<Path> optionalPath = getOptionalPath();
       OpenFileParameters parameters = new OpenFileParameters()
           .withMandatoryKeys(getMandatoryKeys())
+          .withOptionalKeys(getOptionalKeys())
           .withOptions(getOptions())
-          .withBufferSize(getBufferSize())
-          .withStatus(super.getStatus());  // explicit to avoid IDE warnings
+          .withStatus(super.getStatus())
+          .withBufferSize(
+              getOptions().getInt(FS_OPTION_OPENFILE_BUFFER_SIZE, getBufferSize()));
       if(optionalPath.isPresent()) {
         return getFS().openFileWithOptions(optionalPath.get(),
             parameters);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
index 7eec0eb7cec54..593495a1daa88 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
@@ -38,8 +38,8 @@ public abstract class FileSystemLinkResolver<T> {
    * an UnresolvedLinkException if called on an unresolved {@link Path}.
    * @param p Path on which to perform an operation
    * @return Generic type returned by operation
-   * @throws IOException
-   * @throws UnresolvedLinkException
+   * @throws IOException raised on errors performing I/O.
+   * @throws UnresolvedLinkException unresolved link exception.
    */
   abstract public T doCall(final Path p) throws IOException,
       UnresolvedLinkException;
@@ -54,7 +54,7 @@ abstract public T doCall(final Path p) throws IOException,
    * @param p
    *          Resolved Target of path
    * @return Generic type determined by implementation
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   abstract public T next(final FileSystem fs, final Path p) throws IOException;
 
@@ -66,7 +66,7 @@ abstract public T doCall(final Path p) throws IOException,
    * @param filesys FileSystem with which to try call
    * @param path Path with which to try call
    * @return Generic type determined by implementation
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public T resolve(final FileSystem filesys, final Path path)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 3c88e5d21bf6b..2af0a7b9e742f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -36,13 +36,18 @@
 import java.nio.charset.CharsetEncoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
+import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Enumeration;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -51,13 +56,13 @@
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipInputStream;
 
 import org.apache.commons.collections.map.CaseInsensitiveMap;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -72,6 +77,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
 /**
  * A collection of file-processing util methods
  */
@@ -152,6 +162,8 @@ public static void fullyDeleteOnExit(final File file) {
    * (3) If dir is a normal file, it is deleted.
    * (4) If dir is a normal directory, then dir and all its contents recursively
    *     are deleted.
+   * @param dir dir.
+   * @return fully delete status.
    */
   public static boolean fullyDelete(final File dir) {
     return fullyDelete(dir, false);
@@ -247,6 +259,9 @@ private static boolean deleteImpl(final File f, final boolean doLog) {
    * we return false, the directory may be partially-deleted.
    * If dir is a symlink to a directory, all the contents of the actual
    * directory pointed to by dir will be deleted.
+   *
+   * @param dir dir.
+   * @return fullyDeleteContents Status.
    */
   public static boolean fullyDeleteContents(final File dir) {
     return fullyDeleteContents(dir, false);
@@ -257,8 +272,11 @@ public static boolean fullyDeleteContents(final File dir) {
    * we return false, the directory may be partially-deleted.
    * If dir is a symlink to a directory, all the contents of the actual
    * directory pointed to by dir will be deleted.
+   *
+   * @param dir dir.
    * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
    * and all the underlying directories before trying to delete their contents.
+   * @return fully delete contents status.
    */
   public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
     if (tryGrantPermissions) {
@@ -301,7 +319,7 @@ public static boolean fullyDeleteContents(final File dir, final boolean tryGrant
    *
    * @param fs {@link FileSystem} on which the path is present
    * @param dir directory to recursively delete
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link FileSystem#delete(Path, boolean)}
    */
   @Deprecated
@@ -333,7 +351,17 @@ private static void checkDependencies(FileSystem srcFS,
     }
   }
 
-  /** Copy files between FileSystems. */
+  /**
+   * Copy files between FileSystems.
+   * @param srcFS src fs.
+   * @param src src.
+   * @param dstFS dst fs.
+   * @param dst dst.
+   * @param deleteSource delete source.
+   * @param conf configuration.
+   * @return if copy success true, not false.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static boolean copy(FileSystem srcFS, Path src,
                              FileSystem dstFS, Path dst,
                              boolean deleteSource,
@@ -381,7 +409,19 @@ public static boolean copy(FileSystem srcFS, Path[] srcs,
     return returnVal;
   }
 
-  /** Copy files between FileSystems. */
+  /**
+   * Copy files between FileSystems.
+   *
+   * @param srcFS srcFs.
+   * @param src src.
+   * @param dstFS dstFs.
+   * @param dst dst.
+   * @param deleteSource delete source.
+   * @param overwrite overwrite.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return true if the operation succeeded.
+   */
   public static boolean copy(FileSystem srcFS, Path src,
                              FileSystem dstFS, Path dst,
                              boolean deleteSource,
@@ -391,7 +431,33 @@ public static boolean copy(FileSystem srcFS, Path src,
     return copy(srcFS, fileStatus, dstFS, dst, deleteSource, overwrite, conf);
   }
 
-  /** Copy files between FileSystems. */
+  /**
+   * Copy a file/directory tree within/between filesystems.
+   * <p>
+   * returns true if the operation succeeded. When deleteSource is true,
+   * this means "after the copy, delete(source) returned true"
+   * If the destination is a directory, and mkdirs (dest) fails,
+   * the operation will return false rather than raise any exception.
+   * </p>
+   * The overwrite flag is about overwriting files; it has no effect about
+   * handing an attempt to copy a file atop a directory (expect an IOException),
+   * or a directory over a path which contains a file (mkdir will fail, so
+   * "false").
+   * <p>
+   * The operation is recursive, and the deleteSource operation takes place
+   * as each subdirectory is copied. Therefore, if an operation fails partway
+   * through, the source tree may be partially deleted.
+   * </p>
+   * @param srcFS source filesystem
+   * @param srcStatus status of source
+   * @param dstFS destination filesystem
+   * @param dst path of source
+   * @param deleteSource delete the source?
+   * @param overwrite overwrite files at destination?
+   * @param conf configuration to use when opening files
+   * @return true if the operation succeeded.
+   * @throws IOException failure
+   */
   public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
                              FileSystem dstFS, Path dst,
                              boolean deleteSource,
@@ -404,22 +470,27 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
       if (!dstFS.mkdirs(dst)) {
         return false;
       }
-      FileStatus contents[] = srcFS.listStatus(src);
-      for (int i = 0; i < contents.length; i++) {
-        copy(srcFS, contents[i], dstFS,
-             new Path(dst, contents[i].getPath().getName()),
-             deleteSource, overwrite, conf);
+      RemoteIterator<FileStatus> contents = srcFS.listStatusIterator(src);
+      while (contents.hasNext()) {
+        FileStatus next = contents.next();
+        copy(srcFS, next, dstFS,
+            new Path(dst, next.getPath().getName()),
+            deleteSource, overwrite, conf);
       }
     } else {
-      InputStream in=null;
+      InputStream in = null;
       OutputStream out = null;
       try {
-        in = srcFS.open(src);
+        in = awaitFuture(srcFS.openFile(src)
+            .opt(FS_OPTION_OPENFILE_READ_POLICY,
+                FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .opt(FS_OPTION_OPENFILE_LENGTH,
+                srcStatus.getLen())   // file length hint for object stores
+            .build());
         out = dstFS.create(dst, overwrite);
         IOUtils.copyBytes(in, out, conf, true);
       } catch (IOException e) {
-        IOUtils.closeStream(out);
-        IOUtils.closeStream(in);
+        IOUtils.cleanupWithLogger(LOG, in, out);
         throw e;
       }
     }
@@ -431,7 +502,17 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
 
   }
 
-  /** Copy local files to a FileSystem. */
+  /**
+   * Copy local files to a FileSystem.
+   *
+   * @param src src.
+   * @param dstFS dstFs.
+   * @param dst dst.
+   * @param deleteSource delete source.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return true if the operation succeeded.
+   */
   public static boolean copy(File src,
                              FileSystem dstFS, Path dst,
                              boolean deleteSource,
@@ -474,7 +555,17 @@ public static boolean copy(File src,
     }
   }
 
-  /** Copy FileSystem files to local files. */
+  /**
+   * Copy FileSystem files to local files.
+   *
+   * @param srcFS srcFs.
+   * @param src src.
+   * @param dst dst.
+   * @param deleteSource delete source.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return true if the operation succeeded.
+   */
   public static boolean copy(FileSystem srcFS, Path src,
                              File dst, boolean deleteSource,
                              Configuration conf) throws IOException {
@@ -498,7 +589,11 @@ private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
              deleteSource, conf);
       }
     } else {
-      InputStream in = srcFS.open(src);
+      InputStream in = awaitFuture(srcFS.openFile(src)
+          .withFileStatus(srcStatus)
+          .opt(FS_OPTION_OPENFILE_READ_POLICY,
+              FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+          .build());
       IOUtils.copyBytes(in, Files.newOutputStream(dst.toPath()), conf);
     }
     if (deleteSource) {
@@ -533,6 +628,26 @@ private static Path checkDest(String srcName, FileSystem dstFS, Path dst,
     return dst;
   }
 
+  public static boolean isRegularFile(File file) {
+    return isRegularFile(file, true);
+  }
+
+  /**
+   * Check if the file is regular.
+   * @param file The file being checked.
+   * @param allowLinks Whether to allow matching links.
+   * @return Returns the result of checking whether the file is a regular file.
+   */
+  public static boolean isRegularFile(File file, boolean allowLinks) {
+    if (file != null) {
+      if (allowLinks) {
+        return Files.isRegularFile(file.toPath());
+      }
+      return Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS);
+    }
+    return true;
+  }
+
   /**
    * Convert a os-native filename to a path that works for the shell.
    * @param filename The filename to convert
@@ -622,12 +737,12 @@ public static long getDU(File dir) {
    */
   public static void unZip(InputStream inputStream, File toDir)
       throws IOException {
-    try (ZipInputStream zip = new ZipInputStream(inputStream)) {
+    try (ZipArchiveInputStream zip = new ZipArchiveInputStream(inputStream)) {
       int numOfFailedLastModifiedSet = 0;
       String targetDirPath = toDir.getCanonicalPath() + File.separator;
-      for(ZipEntry entry = zip.getNextEntry();
+      for(ZipArchiveEntry entry = zip.getNextZipEntry();
           entry != null;
-          entry = zip.getNextEntry()) {
+          entry = zip.getNextZipEntry()) {
         if (!entry.isDirectory()) {
           File file = new File(toDir, entry.getName());
           if (!file.getCanonicalPath().startsWith(targetDirPath)) {
@@ -646,6 +761,9 @@ public static void unZip(InputStream inputStream, File toDir)
           if (!file.setLastModified(entry.getTime())) {
             numOfFailedLastModifiedSet++;
           }
+          if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) {
+            Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode()));
+          }
         }
       }
       if (numOfFailedLastModifiedSet > 0) {
@@ -655,6 +773,49 @@ public static void unZip(InputStream inputStream, File toDir)
     }
   }
 
+  /**
+   * The permission operation of this method only involves users, user groups, and others.
+   * If SUID is set, only executable permissions are reserved.
+   * @param mode Permissions are represented by numerical values
+   * @return The original permissions for files are stored in collections
+   */
+  private static Set<PosixFilePermission> permissionsFromMode(int mode) {
+    EnumSet<PosixFilePermission> permissions =
+        EnumSet.noneOf(PosixFilePermission.class);
+    addPermissions(permissions, mode, PosixFilePermission.OTHERS_READ,
+        PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE);
+    addPermissions(permissions, mode >> 3, PosixFilePermission.GROUP_READ,
+        PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE);
+    addPermissions(permissions, mode >> 6, PosixFilePermission.OWNER_READ,
+        PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE);
+    return permissions;
+  }
+
+  /**
+   * Assign the original permissions to the file
+   * @param permissions The original permissions for files are stored in collections
+   * @param mode Use a value of type int to indicate permissions
+   * @param r Read permission
+   * @param w Write permission
+   * @param x Execute permission
+   */
+  private static void addPermissions(
+      Set<PosixFilePermission> permissions,
+      int mode,
+      PosixFilePermission r,
+      PosixFilePermission w,
+      PosixFilePermission x) {
+    if ((mode & 1L) == 1L) {
+      permissions.add(x);
+    }
+    if ((mode & 2L) == 2L) {
+      permissions.add(w);
+    }
+    if ((mode & 4L) == 4L) {
+      permissions.add(r);
+    }
+  }
+
   /**
    * Given a File input it will unzip it in the unzip directory.
    * passed as the second parameter
@@ -663,14 +824,14 @@ public static void unZip(InputStream inputStream, File toDir)
    * @throws IOException An I/O exception has occurred
    */
   public static void unZip(File inFile, File unzipDir) throws IOException {
-    Enumeration<? extends ZipEntry> entries;
+    Enumeration<? extends ZipArchiveEntry> entries;
     ZipFile zipFile = new ZipFile(inFile);
 
     try {
-      entries = zipFile.entries();
+      entries = zipFile.getEntries();
       String targetDirPath = unzipDir.getCanonicalPath() + File.separator;
       while (entries.hasMoreElements()) {
-        ZipEntry entry = entries.nextElement();
+        ZipArchiveEntry entry = entries.nextElement();
         if (!entry.isDirectory()) {
           InputStream in = zipFile.getInputStream(entry);
           try {
@@ -695,6 +856,9 @@ public static void unZip(File inFile, File unzipDir) throws IOException {
             } finally {
               out.close();
             }
+            if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) {
+              Files.setPosixFilePermissions(file.toPath(), permissionsFromMode(entry.getUnixMode()));
+            }
           } finally {
             in.close();
           }
@@ -845,7 +1009,7 @@ public static void unTar(InputStream inputStream, File untarDir,
    *
    * @param inFile The tar file as input.
    * @param untarDir The untar directory where to untar the tar file.
-   * @throws IOException
+   * @throws IOException an exception occurred.
    */
   public static void unTar(File inFile, File untarDir) throws IOException {
     if (!untarDir.mkdirs()) {
@@ -888,10 +1052,13 @@ private static void unTarUsingTar(InputStream inputStream, File untarDir,
   private static void unTarUsingTar(File inFile, File untarDir,
       boolean gzipped) throws IOException {
     StringBuffer untarCommand = new StringBuffer();
+    // not using canonical path here; this postpones relative path
+    // resolution until bash is executed.
+    final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'";
     if (gzipped) {
-      untarCommand.append(" gzip -dc '")
-          .append(FileUtil.makeSecureShellPath(inFile))
-          .append("' | (");
+      untarCommand.append(" gzip -dc ")
+          .append(source)
+          .append(" | (");
     }
     untarCommand.append("cd '")
         .append(FileUtil.makeSecureShellPath(untarDir))
@@ -901,15 +1068,17 @@ private static void unTarUsingTar(File inFile, File untarDir,
     if (gzipped) {
       untarCommand.append(" -)");
     } else {
-      untarCommand.append(FileUtil.makeSecureShellPath(inFile));
+      untarCommand.append(source);
     }
+    LOG.debug("executing [{}]", untarCommand);
     String[] shellCmd = { "bash", "-c", untarCommand.toString() };
     ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
     shexec.execute();
     int exitcode = shexec.getExitCode();
     if (exitcode != 0) {
       throw new IOException("Error untarring file " + inFile +
-                  ". Tar process exited with exit code " + exitcode);
+          ". Tar process exited with exit code " + exitcode
+          + " from command " + untarCommand);
     }
   }
 
@@ -966,6 +1135,14 @@ private static void unpackEntries(TarArchiveInputStream tis,
           + " would create entry outside of " + outputDir);
     }
 
+    if (entry.isSymbolicLink() || entry.isLink()) {
+      String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+      if (!canonicalTargetPath.startsWith(targetDirPath)) {
+        throw new IOException(
+            "expanding " + entry.getName() + " would create entry outside of " + outputDir);
+      }
+    }
+
     if (entry.isDirectory()) {
       File subDir = new File(outputDir, entry.getName());
       if (!subDir.mkdirs() && !subDir.isDirectory()) {
@@ -981,10 +1158,12 @@ private static void unpackEntries(TarArchiveInputStream tis,
     }
 
     if (entry.isSymbolicLink()) {
-      // Create symbolic link relative to tar parent dir
-      Files.createSymbolicLink(FileSystems.getDefault()
-              .getPath(outputDir.getPath(), entry.getName()),
-          FileSystems.getDefault().getPath(entry.getLinkName()));
+      // Create symlink with canonical target path to ensure that we don't extract
+      // outside targetDirPath
+      String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+      Files.createSymbolicLink(
+          FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()),
+          FileSystems.getDefault().getPath(canonicalTargetPath));
       return;
     }
 
@@ -996,7 +1175,8 @@ private static void unpackEntries(TarArchiveInputStream tis,
     }
 
     if (entry.isLink()) {
-      File src = new File(outputDir, entry.getLinkName());
+      String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+      File src = new File(canonicalTargetPath);
       HardLink.createHardLink(src, outputFile);
       return;
     }
@@ -1004,6 +1184,20 @@ private static void unpackEntries(TarArchiveInputStream tis,
     org.apache.commons.io.FileUtils.copyToFile(tis, outputFile);
   }
 
+  /**
+   * Gets the canonical path for the given path.
+   *
+   * @param path      The path for which the canonical path needs to be computed.
+   * @param parentDir The parent directory to use if the path is a relative path.
+   * @return The canonical path of the given path.
+   */
+  private static String getCanonicalPath(String path, File parentDir) throws IOException {
+    java.nio.file.Path targetPath = Paths.get(path);
+    return (targetPath.isAbsolute() ?
+        new File(path) :
+        new File(parentDir, path)).getCanonicalPath();
+  }
+
   /**
    * Class for creating hardlinks.
    * Supports Unix, WindXP.
@@ -1026,6 +1220,7 @@ public static class HardLink extends org.apache.hadoop.fs.HardLink {
    * @param target the target for symlink
    * @param linkname the symlink
    * @return 0 on success
+   * @throws IOException raised on errors performing I/O.
    */
   public static int symLink(String target, String linkname) throws IOException{
 
@@ -1087,8 +1282,8 @@ public static int symLink(String target, String linkname) throws IOException{
    * @param filename the name of the file to change
    * @param perm the permission string
    * @return the exit code from the command
-   * @throws IOException
-   * @throws InterruptedException
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException command interrupted.
    */
   public static int chmod(String filename, String perm
                           ) throws IOException, InterruptedException {
@@ -1102,7 +1297,7 @@ public static int chmod(String filename, String perm
    * @param perm permission string
    * @param recursive true, if permissions should be changed recursively
    * @return the exit code from the command.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static int chmod(String filename, String perm, boolean recursive)
                             throws IOException {
@@ -1128,7 +1323,7 @@ public static int chmod(String filename, String perm, boolean recursive)
    * @param file the file to change
    * @param username the new user owner name
    * @param groupname the new group owner name
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void setOwner(File file, String username,
       String groupname) throws IOException {
@@ -1145,7 +1340,7 @@ public static void setOwner(File file, String username,
    * Platform independent implementation for {@link File#setReadable(boolean)}
    * File#setReadable does not work as expected on Windows.
    * @param f input file
-   * @param readable
+   * @param readable readable.
    * @return true on success, false otherwise
    */
   public static boolean setReadable(File f, boolean readable) {
@@ -1166,7 +1361,7 @@ public static boolean setReadable(File f, boolean readable) {
    * Platform independent implementation for {@link File#setWritable(boolean)}
    * File#setWritable does not work as expected on Windows.
    * @param f input file
-   * @param writable
+   * @param writable writable.
    * @return true on success, false otherwise
    */
   public static boolean setWritable(File f, boolean writable) {
@@ -1190,7 +1385,7 @@ public static boolean setWritable(File f, boolean writable) {
    * behavior on Windows as on Unix platforms. Creating, deleting or renaming
    * a file within that folder will still succeed on Windows.
    * @param f input file
-   * @param executable
+   * @param executable executable.
    * @return true on success, false otherwise
    */
   public static boolean setExecutable(File f, boolean executable) {
@@ -1269,7 +1464,7 @@ public static boolean canExecute(File f) {
    * of forking if group == other.
    * @param f the file to change
    * @param permission the new permissions
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void setPermission(File f, FsPermission permission
                                    ) throws IOException {
@@ -1574,6 +1769,7 @@ public static List<Path> getJarsInDirectory(String path) {
    * wildcard path to return all jars from the directory to use in a classpath.
    *
    * @param path the path to the directory. The path may include the wildcard.
+   * @param useLocal use local.
    * @return the list of jars as URLs, or an empty list if there are no jars, or
    * the directory does not exist
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 607aa263622f6..cdbe51e330701 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -233,7 +233,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
    * 
    * @param src file name
    * @param replication new replication
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @return true if successful;
    *         false if file does not exist or is a directory
    */
@@ -304,7 +304,7 @@ public Path getHomeDirectory() {
    * Set the current working directory for the given file system. All relative
    * paths will be resolved relative to it.
    * 
-   * @param newDir
+   * @param newDir new dir.
    */
   @Override
   public void setWorkingDirectory(Path newDir) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 7275b70227f99..73258661ec191 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -130,7 +130,7 @@ public Path getCurrentTrashDir() throws IOException {
    * Returns the current trash location for the path specified
    * @param path to be deleted
    * @return path to the trash
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Path getCurrentTrashDir(Path path) throws IOException {
     return getTrash().getCurrentTrashDir(path);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsStatus.java
index d392c7d765d72..c4bc341bf4f7c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsStatus.java
@@ -35,24 +35,39 @@ public class FsStatus implements Writable {
   private long used;
   private long remaining;
 
-  /** Construct a FsStatus object, using the specified statistics */
+  /**
+   * Construct a FsStatus object, using the specified statistics.
+   *
+   * @param capacity capacity.
+   * @param used used.
+   * @param remaining remaining.
+   */
   public FsStatus(long capacity, long used, long remaining) {
     this.capacity = capacity;
     this.used = used;
     this.remaining = remaining;
   }
 
-  /** Return the capacity in bytes of the file system */
+  /**
+   * Return the capacity in bytes of the file system.
+   * @return capacity.
+   */
   public long getCapacity() {
     return capacity;
   }
 
-  /** Return the number of bytes used on the file system */
+  /**
+   * Return the number of bytes used on the file system.
+   * @return used.
+   */
   public long getUsed() {
     return used;
   }
 
-  /** Return the number of remaining bytes on the file system */
+  /**
+   * Return the number of remaining bytes on the file system.
+   * @return remaining.
+   */
   public long getRemaining() {
     return remaining;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
index 27a522e593001..e7f441a75d3c8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 
@@ -34,7 +35,7 @@
  * options accordingly, for example:
  *
  * If the option is not related to the file system, the option will be ignored.
- * If the option is must, but not supported by the file system, a
+ * If the option is must, but not supported/known by the file system, an
  * {@link IllegalArgumentException} will be thrown.
  *
  */
@@ -51,10 +52,11 @@ CompletableFuture<FSDataInputStream> build()
   /**
    * A FileStatus may be provided to the open request.
    * It is up to the implementation whether to use this or not.
-   * @param status status.
+   * @param status status: may be null
    * @return the builder.
    */
-  default FutureDataInputStreamBuilder withFileStatus(FileStatus status) {
+  default FutureDataInputStreamBuilder withFileStatus(
+      @Nullable FileStatus status) {
     return this;
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java
index cb430ed3f6251..c87444c6c87f7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java
@@ -56,9 +56,9 @@ public StringWithOffset(String string, int offset) {
    * {a,b}/{c/\d}        - {a,b}/c/d
    * </pre>
    * 
-   * @param filePattern
+   * @param filePattern file pattern.
    * @return expanded file patterns
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static List<String> expand(String filePattern) throws IOException {
     List<String> fullyExpanded = new ArrayList<String>();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
index 30ce07a422e6e..d94339034447a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
@@ -104,6 +104,8 @@ public synchronized void reset() {
   /**
    * Get an iterator that we can use to iterate throw all the global storage
    * statistics objects.
+   *
+   * @return StorageStatistics Iterator.
    */
   synchronized public Iterator<StorageStatistics> iterator() {
     Entry<String, StorageStatistics> first = map.firstEntry();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index 7e12d0a11e953..1d64b0bcbe921 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -463,7 +463,7 @@ static BlockLocation[] fixBlockLocations(BlockLocation[] locations,
    * @param start the start of the desired range in the contained file
    * @param len the length of the desired range
    * @return block locations for this segment of file
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
@@ -525,7 +525,7 @@ private void fileStatusesInIndex(HarStatus parent, List<FileStatus> statuses)
    * Combine the status stored in the index and the underlying status. 
    * @param h status stored in the index
    * @return the combined file status
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   private FileStatus toFileStatus(HarStatus h) throws IOException {
     final Path p = h.isDir ? archivePath : new Path(archivePath, h.partName);
@@ -635,7 +635,7 @@ public long getModificationTime() {
    * while creating a hadoop archive.
    * @param f the path in har filesystem
    * @return filestatus.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public FileStatus getFileStatus(Path f) throws IOException {
@@ -1104,7 +1104,7 @@ public void setDropBehind(Boolean dropBehind) throws IOException {
      * @param start the start position in the part file
      * @param length the length of valid data in the part file
      * @param bufsize the buffer size
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public HarFSDataInputStream(FileSystem fs, Path  p, long start, 
         long length, int bufsize) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
index 855fbb04e59b4..1624c5d395aec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
@@ -156,6 +156,7 @@ String[] linkCount(File file) throws IOException {
    * Creates a hardlink.
    * @param file - existing source file
    * @param linkName - desired target link file
+   * @throws IOException raised on errors performing I/O.
    */
   public static void createHardLink(File file, File linkName)
       throws IOException {
@@ -177,6 +178,7 @@ public static void createHardLink(File file, File linkName)
    * @param fileBaseNames - list of path-less file names, as returned by 
    *                        parentDir.list()
    * @param linkDir - where the hardlinks should be put. It must already exist.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
       File linkDir) throws IOException {
@@ -204,6 +206,10 @@ public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
 
    /**
    * Retrieves the number of links to the specified file.
+    *
+    * @param fileName file name.
+    * @throws IOException raised on errors performing I/O.
+    * @return link count.
    */
   public static int getLinkCount(File fileName) throws IOException {
     if (fileName == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java
index bcf325ceca5df..a0e89d6aeac44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HasFileDescriptor.java
@@ -33,7 +33,7 @@ public interface HasFileDescriptor {
 
   /**
    * @return the FileDescriptor
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public FileDescriptor getFileDescriptor() throws IOException;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index 5f266a7b82555..774e015b37343 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -78,8 +78,9 @@ public class LocalDirAllocator {
 
   private final DiskValidator diskValidator;
 
-  /**Create an allocator object
-   * @param contextCfgItemName
+  /**
+   * Create an allocator object.
+   * @param contextCfgItemName contextCfgItemName.
    */
   public LocalDirAllocator(String contextCfgItemName) {
     this.contextCfgItemName = contextCfgItemName;
@@ -123,7 +124,7 @@ private AllocatorPerContext obtainContext(String contextCfgItemName) {
    *  available disk)
    *  @param conf the Configuration object
    *  @return the complete path to the file on a local disk
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public Path getLocalPathForWrite(String pathStr, 
       Configuration conf) throws IOException {
@@ -139,7 +140,7 @@ public Path getLocalPathForWrite(String pathStr,
    *  @param size the size of the file that is going to be written
    *  @param conf the Configuration object
    *  @return the complete path to the file on a local disk
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public Path getLocalPathForWrite(String pathStr, long size, 
       Configuration conf) throws IOException {
@@ -156,7 +157,7 @@ public Path getLocalPathForWrite(String pathStr, long size,
    *  @param conf the Configuration object
    *  @param checkWrite ensure that the path is writable
    *  @return the complete path to the file on a local disk
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public Path getLocalPathForWrite(String pathStr, long size, 
                                    Configuration conf,
@@ -171,7 +172,7 @@ public Path getLocalPathForWrite(String pathStr, long size,
    *  @param pathStr the requested file (this will be searched)
    *  @param conf the Configuration object
    *  @return the complete path to the file on a local disk
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public Path getLocalPathToRead(String pathStr, 
       Configuration conf) throws IOException {
@@ -184,7 +185,7 @@ public Path getLocalPathToRead(String pathStr,
    * @param pathStr the path underneath the roots
    * @param conf the configuration to look up the roots in
    * @return all of the paths that exist under any of the roots
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Iterable<Path> getAllLocalPathsToRead(String pathStr, 
                                                Configuration conf
@@ -205,7 +206,7 @@ public Iterable<Path> getAllLocalPathsToRead(String pathStr,
    *  @param size the size of the file that is going to be written
    *  @param conf the Configuration object
    *  @return a unique temporary file
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public File createTmpFileForWrite(String pathStr, long size, 
       Configuration conf) throws IOException {
@@ -213,8 +214,9 @@ public File createTmpFileForWrite(String pathStr, long size,
     return context.createTmpFileForWrite(pathStr, size, conf);
   }
   
-  /** Method to check whether a context is valid
-   * @param contextCfgItemName
+  /**
+   * Method to check whether a context is valid.
+   * @param contextCfgItemName contextCfgItemName.
    * @return true/false
    */
   public static boolean isContextValid(String contextCfgItemName) {
@@ -224,9 +226,9 @@ public static boolean isContextValid(String contextCfgItemName) {
   }
   
   /**
-   * Removes the context from the context config items
+   * Removes the context from the context config items.
    * 
-   * @param contextCfgItemName
+   * @param contextCfgItemName contextCfgItemName.
    */
   @Deprecated
   @InterfaceAudience.LimitedPrivate({"MapReduce"})
@@ -236,8 +238,9 @@ public static void removeContext(String contextCfgItemName) {
     }
   }
     
-  /** We search through all the configured dirs for the file's existence
-   *  and return true when we find  
+  /**
+   *  We search through all the configured dirs for the file's existence
+   *  and return true when we find.
    *  @param pathStr the requested file (this will be searched)
    *  @param conf the Configuration object
    *  @return true if files exist. false otherwise
@@ -393,6 +396,10 @@ public Path getLocalPathForWrite(String pathStr, long size,
       Context ctx = confChanged(conf);
       int numDirs = ctx.localDirs.length;
       int numDirsSearched = 0;
+      // Max capacity in any directory
+      long maxCapacity = 0;
+      String errorText = null;
+      IOException diskException = null;
       //remove the leading slash from the path (to make sure that the uri
       //resolution results in a valid path on the dir being checked)
       if (pathStr.startsWith("/")) {
@@ -441,9 +448,18 @@ public Path getLocalPathForWrite(String pathStr, long size,
         int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc);
         while (numDirsSearched < numDirs) {
           long capacity = ctx.dirDF[dirNum].getAvailable();
+          if (capacity > maxCapacity) {
+            maxCapacity = capacity;
+          }
           if (capacity > size) {
-            returnPath =
-                createPath(ctx.localDirs[dirNum], pathStr, checkWrite);
+            try {
+              returnPath = createPath(ctx.localDirs[dirNum], pathStr,
+                  checkWrite);
+            } catch (IOException e) {
+              errorText = e.getMessage();
+              diskException = e;
+              LOG.debug("DiskException caught for dir {}", ctx.localDirs[dirNum], e);
+            }
             if (returnPath != null) {
               ctx.getAndIncrDirNumLastAccessed(numDirsSearched);
               break;
@@ -459,8 +475,13 @@ public Path getLocalPathForWrite(String pathStr, long size,
       }
       
       //no path found
-      throw new DiskErrorException("Could not find any valid local " +
-          "directory for " + pathStr);
+      String newErrorText = "Could not find any valid local directory for " +
+          pathStr + " with requested size " + size +
+          " as the max capacity in any directory is " + maxCapacity;
+      if (errorText != null) {
+        newErrorText = newErrorText + " due to " + errorText;
+      }
+      throw new DiskErrorException(newErrorText, diskException);
     }
 
     /** Creates a file on the local FS. Pass size as 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
index c41190a7b360b..590cbd9a49ece 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java
@@ -71,7 +71,11 @@ public LocalFileSystem(FileSystem rawLocalFileSystem) {
     super(rawLocalFileSystem);
   }
     
-  /** Convert a path to a File. */
+  /**
+   * Convert a path to a File.
+   * @param path the path.
+   * @return file.
+   */
   public File pathToFile(Path path) {
     return ((RawLocalFileSystem)fs).pathToFile(path);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
index 5a4a6a97cc4f7..354e4a6b4657d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
@@ -28,7 +28,13 @@ public MD5MD5CRC32CastagnoliFileChecksum() {
     this(0, 0, null);
   }
 
-  /** Create a MD5FileChecksum */
+  /**
+   * Create a MD5FileChecksum.
+   *
+   * @param bytesPerCRC bytesPerCRC.
+   * @param crcPerBlock crcPerBlock.
+   * @param md5 md5.
+   */
   public MD5MD5CRC32CastagnoliFileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) {
     super(bytesPerCRC, crcPerBlock, md5);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
index 3fdb7e982621c..c5ac381f78238 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
@@ -44,7 +44,13 @@ public MD5MD5CRC32FileChecksum() {
     this(0, 0, null);
   }
 
-  /** Create a MD5FileChecksum */
+  /**
+   * Create a MD5FileChecksum.
+   *
+   * @param bytesPerCRC bytesPerCRC.
+   * @param crcPerBlock crcPerBlock.
+   * @param md5 md5.
+   */
   public MD5MD5CRC32FileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) {
     this.bytesPerCRC = bytesPerCRC;
     this.crcPerBlock = crcPerBlock;
@@ -76,7 +82,10 @@ public byte[] getBytes() {
     return WritableUtils.toByteArray(this);
   }
 
-  /** returns the CRC type */
+  /**
+   * returns the CRC type.
+   * @return data check sum type.
+   */
   public DataChecksum.Type getCrcType() {
     // default to the one that is understood by all releases.
     return DataChecksum.Type.CRC32;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java
index 5164d0200d28d..f7996c8623717 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java
@@ -28,7 +28,13 @@ public MD5MD5CRC32GzipFileChecksum() {
     this(0, 0, null);
   }
 
-  /** Create a MD5FileChecksum */
+  /**
+   * Create a MD5FileChecksum.
+   *
+   * @param bytesPerCRC bytesPerCRC.
+   * @param crcPerBlock crcPerBlock.
+   * @param md5 md5.
+   */
   public MD5MD5CRC32GzipFileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) {
     super(bytesPerCRC, crcPerBlock, md5);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
index dcb76b50b3429..5e4eda26c7f1d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
@@ -31,10 +31,11 @@
 /**
  * MultipartUploader is an interface for copying files multipart and across
  * multiple nodes.
- * <p></p>
+ * <p>
  * The interface extends {@link IOStatisticsSource} so that there is no
  * need to cast an instance to see if is a source of statistics.
  * However, implementations MAY return null for their actual statistics.
+ * </p>
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java
index 381bfaa07f6d1..e7b0865063ee5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java
@@ -25,34 +25,43 @@
 
 /**
  * Builder interface for Multipart readers.
- * @param <S>
- * @param <B>
+ * @param <S> MultipartUploader Generic Type.
+ * @param <B> MultipartUploaderBuilder Generic Type.
  */
 public interface MultipartUploaderBuilder<S extends MultipartUploader, B extends MultipartUploaderBuilder<S, B>>
     extends FSBuilder<S, B> {
 
   /**
    * Set permission for the file.
+   * @param perm permission.
+   * @return B Generics Type.
    */
   B permission(@Nonnull FsPermission perm);
 
   /**
    * Set the size of the buffer to be used.
+   * @param bufSize buffer size.
+   * @return B Generics Type.
    */
   B bufferSize(int bufSize);
 
   /**
    * Set replication factor.
+   * @param replica replica.
+   * @return B Generics Type.
    */
   B replication(short replica);
 
   /**
    * Set block size.
+   * @param blkSize blkSize.
+   * @return B Generics Type.
    */
   B blockSize(long blkSize);
 
   /**
    * Create an FSDataOutputStream at the specified path.
+   * @return B Generics Type.
    */
   B create();
 
@@ -60,16 +69,21 @@ public interface MultipartUploaderBuilder<S extends MultipartUploader, B extends
    * Set to true to overwrite the existing file.
    * Set it to false, an exception will be thrown when calling {@link #build()}
    * if the file exists.
+   * @param overwrite overwrite.
+   * @return B Generics Type.
    */
   B overwrite(boolean overwrite);
 
   /**
    * Append to an existing file (optional operation).
+   * @return B Generics Type.
    */
   B append();
 
   /**
    * Set checksum opt.
+   * @param chksumOpt chk sum opt.
+   * @return B Generics Type.
    */
   B checksumOpt(@Nonnull Options.ChecksumOpt chksumOpt);
 
@@ -78,6 +92,7 @@ public interface MultipartUploaderBuilder<S extends MultipartUploader, B extends
    *
    * @throws IllegalArgumentException if the parameters are not valid.
    * @throws IOException on errors when file system creates or appends the file.
+   * @return S Generics Type.
    */
   S build() throws IllegalArgumentException, IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 75bc12df8fdcf..9ef7de657dc15 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -17,9 +17,13 @@
  */
 package org.apache.hadoop.fs;
 
+import java.util.Collections;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -276,7 +280,9 @@ public String toString() {
     }
 
     /**
-     * Create a ChecksumOpts that disables checksum
+     * Create a ChecksumOpts that disables checksum.
+     *
+     * @return ChecksumOpt.
      */
     public static ChecksumOpt createDisabled() {
       return new ChecksumOpt(DataChecksum.Type.NULL, -1);
@@ -291,6 +297,7 @@ public static ChecksumOpt createDisabled() {
      * @param userOpt User-specified checksum option. Ignored if null.
      * @param userBytesPerChecksum User-specified bytesPerChecksum
      *                Ignored if {@literal <} 0.
+     * @return ChecksumOpt.
      */
     public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt, 
         ChecksumOpt userOpt, int userBytesPerChecksum) {
@@ -326,6 +333,8 @@ public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
      *
      * @param defaultOpt Default checksum option
      * @param userOpt User-specified checksum option
+     *
+     * @return ChecksumOpt.
      */
     public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt,
         ChecksumOpt userOpt) {
@@ -518,4 +527,119 @@ public enum ChecksumCombineMode {
     MD5MD5CRC,  // MD5 of block checksums, which are MD5 over chunk CRCs
     COMPOSITE_CRC  // Block/chunk-independent composite CRC
   }
+
+  /**
+   * The standard {@code openFile()} options.
+   */
+  @InterfaceAudience.Public
+  @InterfaceStability.Evolving
+  public static final class OpenFileOptions {
+
+    private OpenFileOptions() {
+    }
+
+    /**
+     * Prefix for all standard filesystem options: {@value}.
+     */
+    private static final String FILESYSTEM_OPTION = "fs.option.";
+
+    /**
+     * Prefix for all openFile options: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE =
+        FILESYSTEM_OPTION + "openfile.";
+
+    /**
+     * OpenFile option for file length: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_LENGTH =
+        FS_OPTION_OPENFILE + "length";
+
+    /**
+     * OpenFile option for split start: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_SPLIT_START =
+        FS_OPTION_OPENFILE + "split.start";
+
+    /**
+     * OpenFile option for split end: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_SPLIT_END =
+        FS_OPTION_OPENFILE + "split.end";
+
+    /**
+     * OpenFile option for buffer size: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_BUFFER_SIZE =
+        FS_OPTION_OPENFILE + "buffer.size";
+
+    /**
+     * OpenFile option for read policies: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY =
+        FS_OPTION_OPENFILE + "read.policy";
+
+    /**
+     * Set of standard options which openFile implementations
+     * MUST recognize, even if they ignore the actual values.
+     */
+    public static final Set<String> FS_OPTION_OPENFILE_STANDARD_OPTIONS =
+        Collections.unmodifiableSet(Stream.of(
+                FS_OPTION_OPENFILE_BUFFER_SIZE,
+                FS_OPTION_OPENFILE_READ_POLICY,
+                FS_OPTION_OPENFILE_LENGTH,
+                FS_OPTION_OPENFILE_SPLIT_START,
+                FS_OPTION_OPENFILE_SPLIT_END)
+            .collect(Collectors.toSet()));
+
+    /**
+     * Read policy for adaptive IO: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE =
+        "adaptive";
+
+    /**
+     * Read policy {@value} -whateve the implementation does by default.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_DEFAULT =
+        "default";
+
+    /**
+     * Read policy for random IO: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_RANDOM =
+        "random";
+
+    /**
+     * Read policy for sequential IO: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL =
+        "sequential";
+
+    /**
+     * Vectored IO API to be used: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_VECTOR =
+        "vector";
+
+    /**
+     * Whole file to be read, end-to-end: {@value}.
+     */
+    public static final String FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE =
+        "whole-file";
+
+    /**
+     * All the current read policies as a set.
+     */
+    public static final Set<String> FS_OPTION_OPENFILE_READ_POLICIES =
+        Collections.unmodifiableSet(Stream.of(
+                FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE,
+                FS_OPTION_OPENFILE_READ_POLICY_DEFAULT,
+                FS_OPTION_OPENFILE_READ_POLICY_RANDOM,
+                FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL,
+                FS_OPTION_OPENFILE_READ_POLICY_VECTOR,
+                FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .collect(Collectors.toSet()));
+
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java
index deb3880ee4195..f32f2a93544bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java
@@ -64,7 +64,13 @@ public PathIOException(String path, String error) {
     this.path = path;
   }
 
-  protected PathIOException(String path, String error, Throwable cause) {
+  /**
+   * Use a subclass of PathIOException if possible.
+   * @param path for the exception
+   * @param error custom string to use an the error text
+   * @param cause cause of exception.
+   */
+  public PathIOException(String path, String error, Throwable cause) {
     super(error, cause);
     this.path = path;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
index 6744d17a72666..7380402eb6156 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,7 +17,11 @@
  */
 package org.apache.hadoop.fs;
 
-import java.io.*;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.function.IntFunction;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -85,4 +89,47 @@ void readFully(long position, byte[] buffer, int offset, int length)
    * the read operation completed
    */
   void readFully(long position, byte[] buffer) throws IOException;
+
+  /**
+   * What is the smallest reasonable seek?
+   * @return the minimum number of bytes
+   */
+  default int minSeekForVectorReads() {
+    return 4 * 1024;
+  }
+
+  /**
+   * What is the largest size that we should group ranges together as?
+   * @return the number of bytes to read at once
+   */
+  default int maxReadSizeForVectorReads() {
+    return 1024 * 1024;
+  }
+
+  /**
+   * Read fully a list of file ranges asynchronously from this file.
+   * The default iterates through the ranges to read each synchronously, but
+   * the intent is that FSDataInputStream subclasses can make more efficient
+   * readers.
+   * As a result of the call, each range will have FileRange.setData(CompletableFuture)
+   * called with a future that when complete will have a ByteBuffer with the
+   * data from the file's range.
+   * <p>
+   *   The position returned by getPos() after readVectored() is undefined.
+   * </p>
+   * <p>
+   *   If a file is changed while the readVectored() operation is in progress, the output is
+   *   undefined. Some ranges may have old data, some may have new and some may have both.
+   * </p>
+   * <p>
+   *   While a readVectored() operation is in progress, normal read api calls may block.
+   * </p>
+   * @param ranges the byte ranges to read
+   * @param allocate the function to allocate ByteBuffer
+   * @throws IOException any IOE.
+   */
+  default void readVectored(List<? extends FileRange> ranges,
+                            IntFunction<ByteBuffer> allocate) throws IOException {
+    VectoredReadUtils.readVectored(this, ranges, allocate);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index b00a31891c867..b0103562123f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -105,7 +105,9 @@ public QuotaUsage build() {
   // Make it protected for the deprecated ContentSummary constructor.
   protected QuotaUsage() { }
 
-  /** Build the instance based on the builder. */
+  /** Build the instance based on the builder.
+   * @param builder bulider.
+   */
   protected QuotaUsage(Builder builder) {
     this.fileAndDirectoryCount = builder.fileAndDirectoryCount;
     this.quota = builder.quota;
@@ -127,37 +129,67 @@ protected void setSpaceQuota(long spaceQuota) {
     this.spaceQuota = spaceQuota;
   }
 
-  /** Return the directory count. */
+  /**
+   * Return the directory count.
+   *
+   * @return file and directory count.
+   */
   public long getFileAndDirectoryCount() {
     return fileAndDirectoryCount;
   }
 
-  /** Return the directory quota. */
+  /**
+   * Return the directory quota.
+   *
+   * @return quota.
+   */
   public long getQuota() {
     return quota;
   }
 
-  /** Return (disk) space consumed. */
+  /**
+   * Return (disk) space consumed.
+   *
+   * @return space consumed.
+   */
   public long getSpaceConsumed() {
     return spaceConsumed;
   }
 
-  /** Return (disk) space quota. */
+  /**
+   * Return (disk) space quota.
+   *
+   * @return space quota.
+   */
   public long getSpaceQuota() {
     return spaceQuota;
   }
 
-  /** Return storage type quota. */
+  /**
+   * Return storage type quota.
+   *
+   * @param type storage type.
+   * @return type quota.
+   */
   public long getTypeQuota(StorageType type) {
     return (typeQuota != null) ? typeQuota[type.ordinal()] : -1L;
   }
 
-  /** Return storage type consumed. */
+  /**
+   * Return storage type consumed.
+   *
+   * @param type storage type.
+   * @return type consumed.
+   */
   public long getTypeConsumed(StorageType type) {
     return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0L;
   }
 
-  /** Return true if any storage type quota has been set. */
+  /**
+   * Return true if any storage type quota has been set.
+   *
+   * @return if any storage type quota has been set true, not false.
+   * */
   public boolean isTypeQuotaSet() {
     if (typeQuota != null) {
       for (StorageType t : StorageType.getTypesSupportingQuota()) {
@@ -169,7 +201,12 @@ public boolean isTypeQuotaSet() {
     return false;
   }
 
-  /** Return true if any storage type consumption information is available. */
+  /**
+   * Return true if any storage type consumption information is available.
+   *
+   * @return if any storage type consumption information
+   * is available, not false.
+   */
   public boolean isTypeConsumedAvailable() {
     if (typeConsumed != null) {
       for (StorageType t : StorageType.getTypesSupportingQuota()) {
@@ -271,12 +308,15 @@ public String toString(boolean hOption) {
     return toString(hOption, false, null);
   }
 
-  /** Return the string representation of the object in the output format.
+  /**
+   * Return the string representation of the object in the output format.
    * if hOption is false file sizes are returned in bytes
    * if hOption is true file sizes are returned in human readable
    *
    * @param hOption a flag indicating if human readable output if to be used
-   * @return the string representation of the object
+   * @param tOption type option.
+   * @param types storage types.
+   * @return the string representation of the object.
    */
   public String toString(boolean hOption,
       boolean tOption, List<StorageType> types) {
@@ -328,7 +368,7 @@ protected String getTypesQuotaUsage(boolean hOption,
   /**
    * return the header of with the StorageTypes.
    *
-   * @param storageTypes
+   * @param storageTypes storage types.
    * @return storage header string
    */
   public static String getStorageTypeHeader(List<StorageType> storageTypes) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index edcc4a8b99e77..2f4f93099b5c9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -33,8 +33,11 @@
 import java.io.FileDescriptor;
 import java.net.URI;
 import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.CompletionHandler;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
+import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.BasicFileAttributeView;
 import java.nio.file.attribute.FileTime;
@@ -44,6 +47,9 @@
 import java.util.Optional;
 import java.util.StringTokenizer;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.IntFunction;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -51,6 +57,8 @@
 import org.apache.hadoop.fs.impl.StoreImplementationUtils;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.fs.statistics.BufferedIOStatisticsOutputStream;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
@@ -61,6 +69,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
+import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
 import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BYTES;
 import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_EXCEPTIONS;
 import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SEEK_OPERATIONS;
@@ -100,7 +109,12 @@ private Path makeAbsolute(Path f) {
     }
   }
   
-  /** Convert a path to a File. */
+  /**
+   * Convert a path to a File.
+   *
+   * @param path the path.
+   * @return file.
+   */
   public File pathToFile(Path path) {
     checkPath(path);
     if (!path.isAbsolute()) {
@@ -125,7 +139,9 @@ public void initialize(URI uri, Configuration conf) throws IOException {
   class LocalFSFileInputStream extends FSInputStream implements
       HasFileDescriptor, IOStatisticsSource, StreamCapabilities {
     private FileInputStream fis;
+    private final File name;
     private long position;
+    private AsynchronousFileChannel asyncChannel = null;
 
     /**
      * Minimal set of counters.
@@ -142,10 +158,19 @@ class LocalFSFileInputStream extends FSInputStream implements
     /** Reference to the bytes read counter for slightly faster counting. */
     private final AtomicLong bytesRead;
 
+    /**
+     * Thread level IOStatistics aggregator to update in close().
+     */
+    private final IOStatisticsAggregator
+        ioStatisticsAggregator;
+
     public LocalFSFileInputStream(Path f) throws IOException {
-      fis = new FileInputStream(pathToFile(f));
+      name = pathToFile(f);
+      fis = new FileInputStream(name);
       bytesRead = ioStatistics.getCounterReference(
           STREAM_READ_BYTES);
+      ioStatisticsAggregator =
+          IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator();
     }
     
     @Override
@@ -174,10 +199,20 @@ public boolean seekToNewSource(long targetPos) throws IOException {
     @Override
     public int available() throws IOException { return fis.available(); }
     @Override
-    public void close() throws IOException { fis.close(); }
-    @Override
     public boolean markSupported() { return false; }
-    
+
+    @Override
+    public void close() throws IOException {
+      try {
+        fis.close();
+        if (asyncChannel != null) {
+          asyncChannel.close();
+        }
+      } finally {
+        ioStatisticsAggregator.aggregate(ioStatistics);
+      }
+    }
+
     @Override
     public int read() throws IOException {
       try {
@@ -257,6 +292,8 @@ public boolean hasCapability(String capability) {
       // new capabilities.
       switch (capability.toLowerCase(Locale.ENGLISH)) {
       case StreamCapabilities.IOSTATISTICS:
+      case StreamCapabilities.IOSTATISTICS_CONTEXT:
+      case StreamCapabilities.VECTOREDIO:
         return true;
       default:
         return false;
@@ -267,8 +304,89 @@ public boolean hasCapability(String capability) {
     public IOStatistics getIOStatistics() {
       return ioStatistics;
     }
+
+    AsynchronousFileChannel getAsyncChannel() throws IOException {
+      if (asyncChannel == null) {
+        synchronized (this) {
+          asyncChannel = AsynchronousFileChannel.open(name.toPath(),
+                  StandardOpenOption.READ);
+        }
+      }
+      return asyncChannel;
+    }
+
+    @Override
+    public void readVectored(List<? extends FileRange> ranges,
+                             IntFunction<ByteBuffer> allocate) throws IOException {
+
+      List<? extends FileRange> sortedRanges = Arrays.asList(sortRanges(ranges));
+      // Set up all of the futures, so that we can use them if things fail
+      for(FileRange range: sortedRanges) {
+        VectoredReadUtils.validateRangeRequest(range);
+        range.setData(new CompletableFuture<>());
+      }
+      try {
+        AsynchronousFileChannel channel = getAsyncChannel();
+        ByteBuffer[] buffers = new ByteBuffer[sortedRanges.size()];
+        AsyncHandler asyncHandler = new AsyncHandler(channel, sortedRanges, buffers);
+        for(int i = 0; i < sortedRanges.size(); ++i) {
+          FileRange range = sortedRanges.get(i);
+          buffers[i] = allocate.apply(range.getLength());
+          channel.read(buffers[i], range.getOffset(), i, asyncHandler);
+        }
+      } catch (IOException ioe) {
+        LOG.debug("Exception occurred during vectored read ", ioe);
+        for(FileRange range: sortedRanges) {
+          range.getData().completeExceptionally(ioe);
+        }
+      }
+    }
   }
-  
+
+  /**
+   * A CompletionHandler that implements readFully and translates back
+   * into the form of CompletionHandler that our users expect.
+   */
+  static class AsyncHandler implements CompletionHandler<Integer, Integer> {
+    private final AsynchronousFileChannel channel;
+    private final List<? extends FileRange> ranges;
+    private final ByteBuffer[] buffers;
+
+    AsyncHandler(AsynchronousFileChannel channel,
+                 List<? extends FileRange> ranges,
+                 ByteBuffer[] buffers) {
+      this.channel = channel;
+      this.ranges = ranges;
+      this.buffers = buffers;
+    }
+
+    @Override
+    public void completed(Integer result, Integer r) {
+      FileRange range = ranges.get(r);
+      ByteBuffer buffer = buffers[r];
+      if (result == -1) {
+        failed(new EOFException("Read past End of File"), r);
+      } else {
+        if (buffer.remaining() > 0) {
+          // issue a read for the rest of the buffer
+          // QQ: What if this fails? It has the same handler.
+          channel.read(buffer, range.getOffset() + buffer.position(), r, this);
+        } else {
+          // QQ: Why  is this required? I think because we don't want the
+          // user to read data beyond limit.
+          buffer.flip();
+          range.getData().complete(buffer);
+        }
+      }
+    }
+
+    @Override
+    public void failed(Throwable exc, Integer r) {
+      LOG.debug("Failed while reading range {} ", r, exc);
+      ranges.get(r).getData().completeExceptionally(exc);
+    }
+  }
+
   @Override
   public FSDataInputStream open(Path f, int bufferSize) throws IOException {
     getFileStatus(f);
@@ -304,9 +422,19 @@ final class LocalFSFileOutputStream extends OutputStream implements
             STREAM_WRITE_EXCEPTIONS)
         .build();
 
+    /**
+     * Thread level IOStatistics aggregator to update in close().
+     */
+    private final IOStatisticsAggregator
+        ioStatisticsAggregator;
+
     private LocalFSFileOutputStream(Path f, boolean append,
         FsPermission permission) throws IOException {
       File file = pathToFile(f);
+      // store the aggregator before attempting any IO.
+      ioStatisticsAggregator =
+          IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator();
+
       if (!append && permission == null) {
         permission = FsPermission.getFileDefault();
       }
@@ -333,10 +461,17 @@ private LocalFSFileOutputStream(Path f, boolean append,
     }
 
     /*
-     * Just forward to the fos
+     * Close the fos; update the IOStatisticsContext.
      */
     @Override
-    public void close() throws IOException { fos.close(); }
+    public void close() throws IOException {
+      try {
+        fos.close();
+      } finally {
+        ioStatisticsAggregator.aggregate(ioStatistics);
+      }
+    }
+
     @Override
     public void flush() throws IOException { fos.flush(); }
     @Override
@@ -382,6 +517,7 @@ public boolean hasCapability(String capability) {
       // new capabilities.
       switch (capability.toLowerCase(Locale.ENGLISH)) {
       case StreamCapabilities.IOSTATISTICS:
+      case StreamCapabilities.IOSTATISTICS_CONTEXT:
         return true;
       default:
         return StoreImplementationUtils.isProbeForSyncable(capability);
@@ -1190,4 +1326,9 @@ public boolean hasPathCapability(final Path path, final String capability)
       return super.hasPathCapability(path, capability);
     }
   }
+
+  @VisibleForTesting
+  static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) {
+    RawLocalFileSystem.useDeprecatedFileStatus = useDeprecatedFileStatus;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java
index 919c857ffa628..f7546d58e6084 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java
@@ -32,17 +32,27 @@ public interface Seekable {
    * Seek to the given offset from the start of the file.
    * The next read() will be from that location.  Can't
    * seek past the end of the file.
+   *
+   * @param pos offset from the start of the file.
+   * @throws IOException raised on errors performing I/O.
    */
   void seek(long pos) throws IOException;
-  
+
   /**
    * Return the current offset from the start of the file
+   *
+   * @return offset from the start of the file.
+   * @throws IOException raised on errors performing I/O.
    */
   long getPos() throws IOException;
 
   /**
-   * Seeks a different copy of the data.  Returns true if 
+   * Seeks a different copy of the data.  Returns true if
    * found a new source, false otherwise.
+   *
+   * @param targetPos target position.
+   * @return true if found a new source, false otherwise.
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.Private
   boolean seekToNewSource(long targetPos) throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
index 07f0513290014..72a45309b17c9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Stat.java
@@ -73,8 +73,8 @@ public FileStatus getFileStatus() throws IOException {
   }
 
   /**
-   * Whether Stat is supported on the current platform
-   * @return
+   * Whether Stat is supported on the current platform.
+   * @return if is available true, not false.
    */
   public static boolean isAvailable() {
     if (Shell.LINUX || Shell.FREEBSD || Shell.MAC) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
index 2efe4566344ee..b4a86ab781280 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
@@ -127,6 +127,7 @@ public StorageStatistics(String name) {
 
   /**
    * Get the name of this StorageStatistics object.
+   * @return name of this StorageStatistics object
    */
   public String getName() {
     return name;
@@ -145,12 +146,15 @@ public String getScheme() {
    *
    * The values returned will depend on the type of FileSystem or FileContext
    * object.  The values do not necessarily reflect a snapshot in time.
+   *
+   * @return LongStatistic Iterator.
    */
   public abstract Iterator<LongStatistic> getLongStatistics();
 
   /**
    * Get the value of a statistic.
    *
+   * @param key key.
    * @return         null if the statistic is not being tracked or is not a
    *                 long statistic. The value of the statistic, otherwise.
    */
@@ -159,6 +163,7 @@ public String getScheme() {
   /**
    * Return true if a statistic is being tracked.
    *
+   * @param key key.
    * @return         True only if the statistic is being tracked.
    */
   public abstract boolean isTracked(String key);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
index 861178019505e..93ed57ef83057 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
@@ -80,6 +80,12 @@ public interface StreamCapabilities {
    */
   String IOSTATISTICS = "iostatistics";
 
+  /**
+   * Support for vectored IO api.
+   * See {@code PositionedReadable#readVectored(List, IntFunction)}.
+   */
+  String VECTOREDIO = "in:readvectored";
+
   /**
    * Stream abort() capability implemented by {@link Abortable#abort()}.
    * This matches the Path Capability
@@ -87,6 +93,12 @@ public interface StreamCapabilities {
    */
   String ABORTABLE_STREAM =  CommonPathCapabilities.ABORTABLE_STREAM;
 
+  /**
+   * Streams that support IOStatistics context and capture thread-level
+   * IOStatistics.
+   */
+  String IOSTATISTICS_CONTEXT = "fs.capability.iocontext.supported";
+
   /**
    * Capabilities that a stream can support and be queried for.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
index e29cb9a4e0e33..5c5fa0237ea66 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
@@ -23,8 +23,10 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.viewfs.Constants.*;
 
 /** 
  * Provides a trash facility which supports pluggable Trash policies. 
@@ -43,6 +45,7 @@ public class Trash extends Configured {
   /** 
    * Construct a trash can accessor.
    * @param conf a Configuration
+   * @throws IOException raised on errors performing I/O.
    */
   public Trash(Configuration conf) throws IOException {
     this(FileSystem.get(conf), conf);
@@ -52,6 +55,7 @@ public Trash(Configuration conf) throws IOException {
    * Construct a trash can accessor for the FileSystem provided.
    * @param fs the FileSystem
    * @param conf a Configuration
+   * @throws IOException raised on errors performing I/O.
    */
   public Trash(FileSystem fs, Configuration conf) throws IOException {
     super(conf);
@@ -92,52 +96,100 @@ public static boolean moveToAppropriateTrash(FileSystem fs, Path p,
       LOG.warn("Failed to get server trash configuration", e);
       throw new IOException("Failed to get server trash configuration", e);
     }
+
+    /*
+     * In HADOOP-18144, we changed getTrashRoot() in ViewFileSystem to return a
+     * viewFS path, instead of a targetFS path. moveToTrash works for
+     * ViewFileSystem now. ViewFileSystem will do path resolution internally by
+     * itself.
+     *
+     * When localized trash flag is enabled:
+     *    1). if fs is a ViewFileSystem, we can initialize Trash() with a
+     *        ViewFileSystem object;
+     *    2). When fs is not a ViewFileSystem, the only place we would need to
+     *        resolve a path is for symbolic links. However, symlink is not
+     *        enabled in Hadoop due to the complexity to support it
+     *        (HADOOP-10019).
+     */
+    if (conf.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT,
+        CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) {
+      Trash trash = new Trash(fs, conf);
+      return trash.moveToTrash(p);
+    }
+
     Trash trash = new Trash(fullyResolvedFs, conf);
     return trash.moveToTrash(fullyResolvedPath);
   }
   
   /**
-   * Returns whether the trash is enabled for this filesystem
+   * Returns whether the trash is enabled for this filesystem.
+   *
+   * @return return if isEnabled true,not false.
    */
   public boolean isEnabled() {
     return trashPolicy.isEnabled();
   }
 
   /** Move a file or directory to the current trash directory.
+   *
+   * @param path the path.
    * @return false if the item is already in the trash or trash is disabled
+   * @throws IOException raised on errors performing I/O.
    */ 
   public boolean moveToTrash(Path path) throws IOException {
     return trashPolicy.moveToTrash(path);
   }
 
-  /** Create a trash checkpoint. */
+  /**
+   * Create a trash checkpoint.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void checkpoint() throws IOException {
     trashPolicy.createCheckpoint();
   }
 
-  /** Delete old checkpoint(s). */
+  /**
+   * Delete old checkpoint(s).
+   * @throws IOException raised on errors performing I/O.
+   */
   public void expunge() throws IOException {
     trashPolicy.deleteCheckpoint();
   }
 
-  /** Delete all trash immediately. */
+  /**
+   * Delete all trash immediately.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void expungeImmediately() throws IOException {
     trashPolicy.createCheckpoint();
     trashPolicy.deleteCheckpointsImmediately();
   }
 
-  /** get the current working directory */
+  /**
+   * get the current working directory.
+   *
+   * @throws IOException on raised on errors performing I/O.
+   * @return Trash Dir.
+   */
   Path getCurrentTrashDir() throws IOException {
     return trashPolicy.getCurrentTrashDir();
   }
 
-  /** get the configured trash policy */
+  /**
+   * get the configured trash policy.
+   *
+   * @return TrashPolicy.
+   */
   TrashPolicy getTrashPolicy() {
     return trashPolicy;
   }
 
-  /** Return a {@link Runnable} that periodically empties the trash of all
+  /**
+   * Return a {@link Runnable} that periodically empties the trash of all
    * users, intended to be run by the superuser.
+   *
+   * @throws IOException on raised on errors performing I/O.
+   * @return Runnable.
    */
   public Runnable getEmptier() throws IOException {
     return trashPolicy.getEmptier();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
index 64fb81be99ee3..35e51f9e1cfb1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -60,27 +60,34 @@ public void initialize(Configuration conf, FileSystem fs) {
 
   /**
    * Returns whether the Trash Policy is enabled for this filesystem.
+   *
+   * @return if isEnabled true,not false.
    */
   public abstract boolean isEnabled();
 
   /** 
    * Move a file or directory to the current trash directory.
+   * @param path the path.
    * @return false if the item is already in the trash or trash is disabled
+   * @throws IOException raised on errors performing I/O.
    */ 
   public abstract boolean moveToTrash(Path path) throws IOException;
 
   /** 
-   * Create a trash checkpoint. 
+   * Create a trash checkpoint.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void createCheckpoint() throws IOException;
 
   /** 
    * Delete old trash checkpoint(s).
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void deleteCheckpoint() throws IOException;
 
   /**
    * Delete all checkpoints immediately, ie empty trash.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void deleteCheckpointsImmediately() throws IOException;
 
@@ -94,6 +101,8 @@ public void initialize(Configuration conf, FileSystem fs) {
    * TrashPolicy#getCurrentTrashDir(Path path).
    * It returns the trash location correctly for the path specified no matter
    * the path is in encryption zone or not.
+   *
+   * @return the path.
    */
   public abstract Path getCurrentTrashDir();
 
@@ -102,7 +111,7 @@ public void initialize(Configuration conf, FileSystem fs) {
    * Policy
    * @param path path to be deleted
    * @return current trash directory for the path to be deleted
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public Path getCurrentTrashDir(Path path) throws IOException {
     throw new UnsupportedOperationException();
@@ -111,6 +120,9 @@ public Path getCurrentTrashDir(Path path) throws IOException {
   /** 
    * Return a {@link Runnable} that periodically empties the trash of all
    * users, intended to be run by the superuser.
+   *
+   * @throws IOException raised on errors performing I/O.
+   * @return Runnable.
    */
   public abstract Runnable getEmptier() throws IOException;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 99467f5633625..f4228dea69f49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -191,8 +191,8 @@ public boolean moveToTrash(Path path) throws IOException {
         cause = e;
       }
     }
-    throw (IOException)
-      new IOException("Failed to move to trash: " + path).initCause(cause);
+    throw new IOException("Failed to move " + path + " to trash " + trashPath,
+        cause);
   }
 
   @SuppressWarnings("deprecation")
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
new file mode 100644
index 0000000000000..cf1b1ef969863
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.IntFunction;
+
+import org.apache.hadoop.fs.impl.CombinedFileRange;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.functional.Function4RaisingIOE;
+
+/**
+ * Utility class which implements helper methods used
+ * in vectored IO implementation.
+ */
+public final class VectoredReadUtils {
+
+  private static final int TMP_BUFFER_MAX_SIZE = 64 * 1024;
+
+  /**
+   * Validate a single range.
+   * @param range file range.
+   * @throws EOFException any EOF Exception.
+   */
+  public static void validateRangeRequest(FileRange range)
+          throws EOFException {
+
+    Preconditions.checkArgument(range.getLength() >= 0, "length is negative");
+    if (range.getOffset() < 0) {
+      throw new EOFException("position is negative");
+    }
+  }
+
+  /**
+   * Validate a list of vectored read ranges.
+   * @param ranges list of ranges.
+   * @throws EOFException any EOF exception.
+   */
+  public static void validateVectoredReadRanges(List<? extends FileRange> ranges)
+          throws EOFException {
+    for (FileRange range : ranges) {
+      validateRangeRequest(range);
+    }
+  }
+
+
+
+  /**
+   * This is the default implementation which iterates through the ranges
+   * to read each synchronously, but the intent is that subclasses
+   * can make more efficient readers.
+   * The data or exceptions are pushed into {@link FileRange#getData()}.
+   * @param stream the stream to read the data from
+   * @param ranges the byte ranges to read
+   * @param allocate the byte buffer allocation
+   */
+  public static void readVectored(PositionedReadable stream,
+                                  List<? extends FileRange> ranges,
+                                  IntFunction<ByteBuffer> allocate) {
+    for (FileRange range: ranges) {
+      range.setData(readRangeFrom(stream, range, allocate));
+    }
+  }
+
+  /**
+   * Synchronously reads a range from the stream dealing with the combinations
+   * of ByteBuffers buffers and PositionedReadable streams.
+   * @param stream the stream to read from
+   * @param range the range to read
+   * @param allocate the function to allocate ByteBuffers
+   * @return the CompletableFuture that contains the read data
+   */
+  public static CompletableFuture<ByteBuffer> readRangeFrom(PositionedReadable stream,
+                                                            FileRange range,
+                                                            IntFunction<ByteBuffer> allocate) {
+    CompletableFuture<ByteBuffer> result = new CompletableFuture<>();
+    try {
+      ByteBuffer buffer = allocate.apply(range.getLength());
+      if (stream instanceof ByteBufferPositionedReadable) {
+        ((ByteBufferPositionedReadable) stream).readFully(range.getOffset(),
+            buffer);
+        buffer.flip();
+      } else {
+        readNonByteBufferPositionedReadable(stream, range, buffer);
+      }
+      result.complete(buffer);
+    } catch (IOException ioe) {
+      result.completeExceptionally(ioe);
+    }
+    return result;
+  }
+
+  private static void readNonByteBufferPositionedReadable(PositionedReadable stream,
+                                                          FileRange range,
+                                                          ByteBuffer buffer) throws IOException {
+    if (buffer.isDirect()) {
+      readInDirectBuffer(range.getLength(),
+          buffer,
+          (position, buffer1, offset, length) -> {
+            stream.readFully(position, buffer1, offset, length);
+            return null;
+          });
+      buffer.flip();
+    } else {
+      stream.readFully(range.getOffset(), buffer.array(),
+              buffer.arrayOffset(), range.getLength());
+    }
+  }
+
+  /**
+   * Read bytes from stream into a byte buffer using an
+   * intermediate byte array.
+   * @param length number of bytes to read.
+   * @param buffer buffer to fill.
+   * @param operation operation to use for reading data.
+   * @throws IOException any IOE.
+   */
+  public static void readInDirectBuffer(int length,
+                                        ByteBuffer buffer,
+                                        Function4RaisingIOE<Integer, byte[], Integer,
+                                                Integer, Void> operation) throws IOException {
+    if (length == 0) {
+      return;
+    }
+    int readBytes = 0;
+    int position = 0;
+    int tmpBufferMaxSize = Math.min(TMP_BUFFER_MAX_SIZE, length);
+    byte[] tmp = new byte[tmpBufferMaxSize];
+    while (readBytes < length) {
+      int currentLength = (readBytes + tmpBufferMaxSize) < length ?
+              tmpBufferMaxSize
+              : (length - readBytes);
+      operation.apply(position, tmp, 0, currentLength);
+      buffer.put(tmp, 0, currentLength);
+      position = position + currentLength;
+      readBytes = readBytes + currentLength;
+    }
+  }
+
+  /**
+   * Is the given input list.
+   * <ul>
+   *   <li>already sorted by offset</li>
+   *   <li>each range is more than minimumSeek apart</li>
+   *   <li>the start and end of each range is a multiple of chunkSize</li>
+   * </ul>
+   *
+   * @param input the list of input ranges.
+   * @param chunkSize the size of the chunks that the offset and end must align to.
+   * @param minimumSeek the minimum distance between ranges.
+   * @return true if we can use the input list as is.
+   */
+  public static boolean isOrderedDisjoint(List<? extends FileRange> input,
+                                          int chunkSize,
+                                          int minimumSeek) {
+    long previous = -minimumSeek;
+    for (FileRange range: input) {
+      long offset = range.getOffset();
+      long end = range.getOffset() + range.getLength();
+      if (offset % chunkSize != 0 ||
+              end % chunkSize != 0 ||
+              (offset - previous < minimumSeek)) {
+        return false;
+      }
+      previous = end;
+    }
+    return true;
+  }
+
+  /**
+   * Calculates floor value of offset based on chunk size.
+   * @param offset file offset.
+   * @param chunkSize file chunk size.
+   * @return  floor value.
+   */
+  public static long roundDown(long offset, int chunkSize) {
+    if (chunkSize > 1) {
+      return offset - (offset % chunkSize);
+    } else {
+      return offset;
+    }
+  }
+
+  /**
+   * Calculates the ceil value of offset based on chunk size.
+   * @param offset file offset.
+   * @param chunkSize file chunk size.
+   * @return ceil value.
+   */
+  public static long roundUp(long offset, int chunkSize) {
+    if (chunkSize > 1) {
+      long next = offset + chunkSize - 1;
+      return next - (next % chunkSize);
+    } else {
+      return offset;
+    }
+  }
+
+  /**
+   * Check if the input ranges are overlapping in nature.
+   * We call two ranges to be overlapping when start offset
+   * of second is less than the end offset of first.
+   * End offset is calculated as start offset + length.
+   * @param input list if input ranges.
+   * @return true/false based on logic explained above.
+   */
+  public static List<? extends FileRange> validateNonOverlappingAndReturnSortedRanges(
+          List<? extends FileRange> input) {
+
+    if (input.size() <= 1) {
+      return input;
+    }
+    FileRange[] sortedRanges = sortRanges(input);
+    FileRange prev = sortedRanges[0];
+    for (int i=1; i<sortedRanges.length; i++) {
+      if (sortedRanges[i].getOffset() < prev.getOffset() + prev.getLength()) {
+        throw new UnsupportedOperationException("Overlapping ranges are not supported");
+      }
+      prev = sortedRanges[i];
+    }
+    return Arrays.asList(sortedRanges);
+  }
+
+  /**
+   * Sort the input ranges by offset.
+   * @param input input ranges.
+   * @return sorted ranges.
+   */
+  public static FileRange[] sortRanges(List<? extends FileRange> input) {
+    FileRange[] sortedRanges = input.toArray(new FileRange[0]);
+    Arrays.sort(sortedRanges, Comparator.comparingLong(FileRange::getOffset));
+    return sortedRanges;
+  }
+
+  /**
+   * Merge sorted ranges to optimize the access from the underlying file
+   * system.
+   * The motivations are that:
+   * <ul>
+   *   <li>Upper layers want to pass down logical file ranges.</li>
+   *   <li>Fewer reads have better performance.</li>
+   *   <li>Applications want callbacks as ranges are read.</li>
+   *   <li>Some file systems want to round ranges to be at checksum boundaries.</li>
+   * </ul>
+   *
+   * @param sortedRanges already sorted list of ranges based on offset.
+   * @param chunkSize round the start and end points to multiples of chunkSize
+   * @param minimumSeek the smallest gap that we should seek over in bytes
+   * @param maxSize the largest combined file range in bytes
+   * @return the list of sorted CombinedFileRanges that cover the input
+   */
+  public static List<CombinedFileRange> mergeSortedRanges(List<? extends FileRange> sortedRanges,
+                                                          int chunkSize,
+                                                          int minimumSeek,
+                                                          int maxSize) {
+
+    CombinedFileRange current = null;
+    List<CombinedFileRange> result = new ArrayList<>(sortedRanges.size());
+
+    // now merge together the ones that merge
+    for (FileRange range: sortedRanges) {
+      long start = roundDown(range.getOffset(), chunkSize);
+      long end = roundUp(range.getOffset() + range.getLength(), chunkSize);
+      if (current == null || !current.merge(start, end, range, minimumSeek, maxSize)) {
+        current = new CombinedFileRange(start, end, range);
+        result.add(current);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Slice the data that was read to the user's request.
+   * This function assumes that the user's request is completely subsumed by the
+   * read data. This always creates a new buffer pointing to the same underlying
+   * data but with its own mark and position fields such that reading one buffer
+   * can't effect other's mark and position.
+   * @param readData the buffer with the readData
+   * @param readOffset the offset in the file for the readData
+   * @param request the user's request
+   * @return the readData buffer that is sliced to the user's request
+   */
+  public static ByteBuffer sliceTo(ByteBuffer readData, long readOffset,
+                                   FileRange request) {
+    int offsetChange = (int) (request.getOffset() - readOffset);
+    int requestLength = request.getLength();
+    // Create a new buffer that is backed by the original contents
+    // The buffer will have position 0 and the same limit as the original one
+    readData = readData.slice();
+    // Change the offset and the limit of the buffer as the reader wants to see
+    // only relevant data
+    readData.position(offsetChange);
+    readData.limit(offsetChange + requestLength);
+    // Create a new buffer after the limit change so that only that portion of the data is
+    // returned to the reader.
+    readData = readData.slice();
+    return readData;
+  }
+
+  /**
+   * private constructor.
+   */
+  private VectoredReadUtils() {
+    throw new UnsupportedOperationException();
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java
index 3d65275e673d6..df878d998706c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrCodec.java
@@ -67,7 +67,7 @@ public enum XAttrCodec {
    * the given string is treated as text. 
    * @param value string representation of the value.
    * @return byte[] the value
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static byte[] decodeValue(String value) throws IOException {
     byte[] result = null;
@@ -102,9 +102,9 @@ public static byte[] decodeValue(String value) throws IOException {
    * while strings encoded as hexadecimal and base64 are prefixed with 
    * 0x and 0s, respectively.
    * @param value byte[] value
-   * @param encoding
+   * @param encoding encoding.
    * @return String string representation of value
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static String encodeValue(byte[] value, XAttrCodec encoding) 
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java
index d9629e388b384..0929c2be03acf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java
@@ -90,6 +90,11 @@ private AuditConstants() {
    */
   public static final String PARAM_PROCESS = "ps";
 
+  /**
+   * Task Attempt ID query header: {@value}.
+   */
+  public static final String PARAM_TASK_ATTEMPT_ID = "ta";
+
   /**
    * Thread 0: the thread which created a span {@value}.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java
index 11681546e3d0a..2dcd4f8b3f570 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java
@@ -24,6 +24,9 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Supplier;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -69,11 +72,16 @@
  * {@link #currentAuditContext()} to get the thread-local
  * context for the caller, which can then be manipulated.
  *
+ * For further information, especially related to memory consumption,
+ * read the document `auditing_architecture` in the `hadoop-aws` module.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
 public final class CommonAuditContext {
 
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CommonAuditContext.class);
+
   /**
    * Process ID; currently built from UUID and timestamp.
    */
@@ -92,7 +100,7 @@ public final class CommonAuditContext {
    * Supplier operations must themselves be thread safe.
    */
   private final Map<String, Supplier<String>> evaluatedEntries =
-      new ConcurrentHashMap<>();
+      new ConcurrentHashMap<>(1);
 
   static {
     // process ID is fixed.
@@ -108,7 +116,7 @@ public final class CommonAuditContext {
    * the span is finalized.
    */
   private static final ThreadLocal<CommonAuditContext> ACTIVE_CONTEXT =
-      ThreadLocal.withInitial(() -> createInstance());
+      ThreadLocal.withInitial(CommonAuditContext::createInstance);
 
   private CommonAuditContext() {
   }
@@ -116,20 +124,34 @@ private CommonAuditContext() {
   /**
    * Put a context entry.
    * @param key key
-   * @param value new value
+   * @param value new value., If null, triggers removal.
    * @return old value or null
    */
   public Supplier<String> put(String key, String value) {
-    return evaluatedEntries.put(key, () -> value);
+    if (value != null) {
+      return evaluatedEntries.put(key, () -> value);
+    } else {
+      return evaluatedEntries.remove(key);
+    }
   }
 
   /**
    * Put a context entry dynamically evaluated on demand.
+   * Important: as these supplier methods are long-lived,
+   * the supplier function <i>MUST NOT</i> be part of/refer to
+   * any object instance of significant memory size.
+   * Applications SHOULD remove references when they are
+   * no longer needed.
+   * When logged at TRACE, prints the key and stack trace of the caller,
+   * to allow for debugging of any problems.
    * @param key key
    * @param value new value
    * @return old value or null
    */
   public Supplier<String> put(String key, Supplier<String> value) {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Adding context entry {}", key, new Exception(key));
+    }
     return evaluatedEntries.put(key, value);
   }
 
@@ -138,6 +160,9 @@ public Supplier<String> put(String key, Supplier<String> value) {
    * @param key key
    */
   public void remove(String key) {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Remove context entry {}", key);
+    }
     evaluatedEntries.remove(key);
   }
 
@@ -168,7 +193,7 @@ public void reset() {
   private void init() {
 
     // thread 1 is dynamic
-    put(PARAM_THREAD1, () -> currentThreadID());
+    put(PARAM_THREAD1, CommonAuditContext::currentThreadID);
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java
index c69e7afe4e36e..4256522b2a372 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractFSBuilderImpl.java
@@ -46,7 +46,7 @@
  * <code>
  *   .opt("foofs:option.a", true)
  *   .opt("foofs:option.b", "value")
- *   .opt("barfs:cache", true)
+ *   .opt("fs.s3a.open.option.etag", "9fe4c37c25b")
  *   .must("foofs:cache", true)
  *   .must("barfs:cache-size", 256 * 1024 * 1024)
  *   .build();
@@ -88,6 +88,9 @@
   /** Keep track of the keys for mandatory options. */
   private final Set<String> mandatoryKeys = new HashSet<>();
 
+  /** Keep track of the optional keys. */
+  private final Set<String> optionalKeys = new HashSet<>();
+
   /**
    * Constructor with both optional path and path handle.
    * Either or both argument may be empty, but it is an error for
@@ -163,6 +166,7 @@ public PathHandle getPathHandle() {
   @Override
   public B opt(@Nonnull final String key, @Nonnull final String value) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.set(key, value);
     return getThisBuilder();
   }
@@ -175,6 +179,7 @@ public B opt(@Nonnull final String key, @Nonnull final String value) {
   @Override
   public B opt(@Nonnull final String key, boolean value) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.setBoolean(key, value);
     return getThisBuilder();
   }
@@ -187,10 +192,19 @@ public B opt(@Nonnull final String key, boolean value) {
   @Override
   public B opt(@Nonnull final String key, int value) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.setInt(key, value);
     return getThisBuilder();
   }
 
+  @Override
+  public B opt(@Nonnull final String key, final long value) {
+    mandatoryKeys.remove(key);
+    optionalKeys.add(key);
+    options.setLong(key, value);
+    return getThisBuilder();
+  }
+
   /**
    * Set optional float parameter for the Builder.
    *
@@ -199,6 +213,7 @@ public B opt(@Nonnull final String key, int value) {
   @Override
   public B opt(@Nonnull final String key, float value) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.setFloat(key, value);
     return getThisBuilder();
   }
@@ -211,6 +226,7 @@ public B opt(@Nonnull final String key, float value) {
   @Override
   public B opt(@Nonnull final String key, double value) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.setDouble(key, value);
     return getThisBuilder();
   }
@@ -223,6 +239,7 @@ public B opt(@Nonnull final String key, double value) {
   @Override
   public B opt(@Nonnull final String key, @Nonnull final String... values) {
     mandatoryKeys.remove(key);
+    optionalKeys.add(key);
     options.setStrings(key, values);
     return getThisBuilder();
   }
@@ -248,6 +265,7 @@ public B must(@Nonnull final String key, @Nonnull final String value) {
   @Override
   public B must(@Nonnull final String key, boolean value) {
     mandatoryKeys.add(key);
+    optionalKeys.remove(key);
     options.setBoolean(key, value);
     return getThisBuilder();
   }
@@ -260,10 +278,19 @@ public B must(@Nonnull final String key, boolean value) {
   @Override
   public B must(@Nonnull final String key, int value) {
     mandatoryKeys.add(key);
+    optionalKeys.remove(key);
     options.setInt(key, value);
     return getThisBuilder();
   }
 
+  @Override
+  public B must(@Nonnull final String key, final long value) {
+    mandatoryKeys.add(key);
+    optionalKeys.remove(key);
+    options.setLong(key, value);
+    return getThisBuilder();
+  }
+
   /**
    * Set mandatory float option.
    *
@@ -272,6 +299,7 @@ public B must(@Nonnull final String key, int value) {
   @Override
   public B must(@Nonnull final String key, float value) {
     mandatoryKeys.add(key);
+    optionalKeys.remove(key);
     options.setFloat(key, value);
     return getThisBuilder();
   }
@@ -284,6 +312,7 @@ public B must(@Nonnull final String key, float value) {
   @Override
   public B must(@Nonnull final String key, double value) {
     mandatoryKeys.add(key);
+    optionalKeys.remove(key);
     options.setDouble(key, value);
     return getThisBuilder();
   }
@@ -296,6 +325,7 @@ public B must(@Nonnull final String key, double value) {
   @Override
   public B must(@Nonnull final String key, @Nonnull final String... values) {
     mandatoryKeys.add(key);
+    optionalKeys.remove(key);
     options.setStrings(key, values);
     return getThisBuilder();
   }
@@ -310,10 +340,18 @@ public Configuration getOptions() {
 
   /**
    * Get all the keys that are set as mandatory keys.
+   * @return mandatory keys.
    */
   public Set<String> getMandatoryKeys() {
     return Collections.unmodifiableSet(mandatoryKeys);
   }
+  /**
+   * Get all the keys that are set as optional keys.
+   * @return optional keys.
+   */
+  public Set<String> getOptionalKeys() {
+    return Collections.unmodifiableSet(optionalKeys);
+  }
 
   /**
    * Reject a configuration if one or more mandatory keys are
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
index 416924e18d87c..f9ae9f55cc17f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
@@ -127,7 +127,7 @@ protected void checkPutArguments(Path filePath,
    * {@inheritDoc}.
    * @param path path to abort uploads under.
    * @return a future to -1.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public CompletableFuture<Integer> abortUploadsUnderPath(Path path)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/CombinedFileRange.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/CombinedFileRange.java
new file mode 100644
index 0000000000000..c9555a1e5414e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/CombinedFileRange.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.impl;
+
+import org.apache.hadoop.fs.FileRange;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A file range that represents a set of underlying file ranges.
+ * This is used when we combine the user's FileRange objects
+ * together into a single read for efficiency.
+ */
+public class CombinedFileRange extends FileRangeImpl {
+  private List<FileRange> underlying = new ArrayList<>();
+
+  public CombinedFileRange(long offset, long end, FileRange original) {
+    super(offset, (int) (end - offset), null);
+    this.underlying.add(original);
+  }
+
+  /**
+   * Get the list of ranges that were merged together to form this one.
+   * @return the list of input ranges
+   */
+  public List<FileRange> getUnderlying() {
+    return underlying;
+  }
+
+  /**
+   * Merge this input range into the current one, if it is compatible.
+   * It is assumed that otherOffset is greater or equal the current offset,
+   * which typically happens by sorting the input ranges on offset.
+   * @param otherOffset the offset to consider merging
+   * @param otherEnd the end to consider merging
+   * @param other the underlying FileRange to add if we merge
+   * @param minSeek the minimum distance that we'll seek without merging the
+   *                ranges together
+   * @param maxSize the maximum size that we'll merge into a single range
+   * @return true if we have merged the range into this one
+   */
+  public boolean merge(long otherOffset, long otherEnd, FileRange other,
+                       int minSeek, int maxSize) {
+    long end = this.getOffset() + this.getLength();
+    long newEnd = Math.max(end, otherEnd);
+    if (otherOffset - end >= minSeek || newEnd - this.getOffset() > maxSize) {
+      return false;
+    }
+    this.setLength((int) (newEnd - this.getOffset()));
+    underlying.add(other);
+    return true;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileRangeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileRangeImpl.java
new file mode 100644
index 0000000000000..1239be764ba5c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileRangeImpl.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.impl;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.FileRange;
+
+/**
+ * A range of bytes from a file with an optional buffer to read those bytes
+ * for zero copy. This shouldn't be created directly via constructor rather
+ * factory defined in {@code FileRange#createFileRange} should be used.
+ */
+@InterfaceAudience.Private
+public class FileRangeImpl implements FileRange {
+  private long offset;
+  private int length;
+  private CompletableFuture<ByteBuffer> reader;
+
+  /**
+   * nullable reference to store in the range.
+   */
+  private final Object reference;
+
+  /**
+   * Create.
+   * @param offset offset in file
+   * @param length length of data to read.
+   * @param reference nullable reference to store in the range.
+   */
+  public FileRangeImpl(long offset, int length, Object reference) {
+    this.offset = offset;
+    this.length = length;
+    this.reference = reference;
+  }
+
+  @Override
+  public String toString() {
+    return "range[" + offset + "," + (offset + length) + ")";
+  }
+
+  @Override
+  public long getOffset() {
+    return offset;
+  }
+
+  @Override
+  public int getLength() {
+    return length;
+  }
+
+  public void setOffset(long offset) {
+    this.offset = offset;
+  }
+
+  public void setLength(int length) {
+    this.length = length;
+  }
+
+  @Override
+  public void setData(CompletableFuture<ByteBuffer> pReader) {
+    this.reader = pReader;
+  }
+
+  @Override
+  public CompletableFuture<ByteBuffer> getData() {
+    return reader;
+  }
+
+  @Override
+  public Object getReference() {
+    return reference;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
index 2339e42128973..481d927672dc3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.fs.PathHandle;
 import org.apache.hadoop.fs.UploadHandle;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.functional.FutureIO;
 
 import static org.apache.hadoop.fs.Path.mergePaths;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
@@ -98,7 +99,7 @@ public FileSystemMultipartUploader(
   public CompletableFuture<UploadHandle> startUpload(Path filePath)
       throws IOException {
     checkPath(filePath);
-    return FutureIOSupport.eval(() -> {
+    return FutureIO.eval(() -> {
       Path collectorPath = createCollectorPath(filePath);
       fs.mkdirs(collectorPath, FsPermission.getDirDefault());
 
@@ -116,7 +117,7 @@ public CompletableFuture<PartHandle> putPart(UploadHandle uploadId,
       throws IOException {
     checkPutArguments(filePath, inputStream, partNumber, uploadId,
         lengthInBytes);
-    return FutureIOSupport.eval(() -> innerPutPart(filePath,
+    return FutureIO.eval(() -> innerPutPart(filePath,
         inputStream, partNumber, uploadId, lengthInBytes));
   }
 
@@ -179,7 +180,7 @@ public CompletableFuture<PathHandle> complete(
       Map<Integer, PartHandle> handleMap) throws IOException {
 
     checkPath(filePath);
-    return FutureIOSupport.eval(() ->
+    return FutureIO.eval(() ->
         innerComplete(uploadId, filePath, handleMap));
   }
 
@@ -251,7 +252,7 @@ public CompletableFuture<Void> abort(UploadHandle uploadId,
     Path collectorPath = new Path(new String(uploadIdByteArray, 0,
         uploadIdByteArray.length, Charsets.UTF_8));
 
-    return FutureIOSupport.eval(() -> {
+    return FutureIO.eval(() -> {
       // force a check for a file existing; raises FNFE if not found
       fs.getFileStatus(collectorPath);
       fs.delete(collectorPath, true);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
index 24a8d49747fe6..833c21ec1a67f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.impl;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
 
@@ -47,7 +48,7 @@
  * options accordingly, for example:
  *
  * If the option is not related to the file system, the option will be ignored.
- * If the option is must, but not supported by the file system, a
+ * If the option is must, but not supported/known by the file system, an
  * {@link IllegalArgumentException} will be thrown.
  *
  */
@@ -125,6 +126,9 @@ protected int getBufferSize() {
 
   /**
    * Set the size of the buffer to be used.
+   *
+   * @param bufSize buffer size.
+   * @return FutureDataInputStreamBuilder.
    */
   public FutureDataInputStreamBuilder bufferSize(int bufSize) {
     bufferSize = bufSize;
@@ -136,6 +140,8 @@ public FutureDataInputStreamBuilder bufferSize(int bufSize) {
    * This must be used after the constructor has been invoked to create
    * the actual builder: it allows for subclasses to do things after
    * construction.
+   *
+   * @return FutureDataInputStreamBuilder.
    */
   public FutureDataInputStreamBuilder builder() {
     return getThisBuilder();
@@ -147,8 +153,9 @@ public FutureDataInputStreamBuilder getThisBuilder() {
   }
 
   @Override
-  public FutureDataInputStreamBuilder withFileStatus(FileStatus st) {
-    this.status = requireNonNull(st, "status");
+  public FutureDataInputStreamBuilder withFileStatus(
+      @Nullable FileStatus st) {
+    this.status = st;
     return this;
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
index 18f5187cb6134..0a080426c2b24 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
@@ -20,7 +20,6 @@
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
@@ -37,14 +36,16 @@
 
 /**
  * Support for future IO and the FS Builder subclasses.
- * If methods in here are needed for applications, promote
- * to {@link FutureIO} for public use -with the original
- * method relaying to it. This is to ensure that external
- * filesystem implementations can safely use these methods
+ * All methods in this class have been superceded by those in
+ * {@link FutureIO}.
+ * The methods here are retained but all marked as deprecated.
+ * This is to ensure that any external
+ * filesystem implementations can still use these methods
  * without linkage problems surfacing.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
+@Deprecated
 public final class FutureIOSupport {
 
   private FutureIOSupport() {
@@ -53,6 +54,7 @@ private FutureIOSupport() {
   /**
    * Given a future, evaluate it. Raised exceptions are
    * extracted and handled.
+   * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}.
    * @param future future to evaluate
    * @param <T> type of the result.
    * @return the result, if all went well.
@@ -60,7 +62,8 @@ private FutureIOSupport() {
    * @throws IOException if something went wrong
    * @throws RuntimeException any nested RTE thrown
    */
-  public static <T> T  awaitFuture(final Future<T> future)
+  @Deprecated
+  public static <T> T awaitFuture(final Future<T> future)
       throws InterruptedIOException, IOException, RuntimeException {
     return FutureIO.awaitFuture(future);
   }
@@ -69,14 +72,18 @@ public static <T> T  awaitFuture(final Future<T> future)
   /**
    * Given a future, evaluate it. Raised exceptions are
    * extracted and handled.
+   * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}.
    * @param future future to evaluate
    * @param <T> type of the result.
+   * @param timeout timeout.
+   * @param unit unit.
    * @return the result, if all went well.
    * @throws InterruptedIOException future was interrupted
    * @throws IOException if something went wrong
    * @throws RuntimeException any nested RTE thrown
    * @throws TimeoutException the future timed out.
    */
+  @Deprecated
   public static <T> T awaitFuture(final Future<T> future,
       final long timeout,
       final TimeUnit unit)
@@ -88,10 +95,7 @@ public static <T> T awaitFuture(final Future<T> future,
   /**
    * From the inner cause of an execution exception, extract the inner cause
    * if it is an IOE or RTE.
-   * This will always raise an exception, either the inner IOException,
-   * an inner RuntimeException, or a new IOException wrapping the raised
-   * exception.
-   *
+   * See {@link FutureIO#raiseInnerCause(ExecutionException)}.
    * @param e exception.
    * @param <T> type of return value.
    * @return nothing, ever.
@@ -99,6 +103,7 @@ public static <T> T awaitFuture(final Future<T> future,
    * any non-Runtime-Exception
    * @throws RuntimeException if that is the inner cause.
    */
+  @Deprecated
   public static <T> T raiseInnerCause(final ExecutionException e)
       throws IOException {
     return FutureIO.raiseInnerCause(e);
@@ -107,6 +112,7 @@ public static <T> T raiseInnerCause(final ExecutionException e)
   /**
    * Extract the cause of a completion failure and rethrow it if an IOE
    * or RTE.
+   * See {@link FutureIO#raiseInnerCause(CompletionException)}.
    * @param e exception.
    * @param <T> type of return value.
    * @return nothing, ever.
@@ -114,20 +120,15 @@ public static <T> T raiseInnerCause(final ExecutionException e)
    * any non-Runtime-Exception
    * @throws RuntimeException if that is the inner cause.
    */
+  @Deprecated
   public static <T> T raiseInnerCause(final CompletionException e)
       throws IOException {
     return FutureIO.raiseInnerCause(e);
   }
 
   /**
-   * Propagate options to any builder, converting everything with the
-   * prefix to an option where, if there were 2+ dot-separated elements,
-   * it is converted to a schema.
-   * <pre>{@code
-   *   fs.example.s3a.option => s3a:option
-   *   fs.example.fs.io.policy => s3a.io.policy
-   *   fs.example.something => something
-   * }</pre>
+   * Propagate options to any builder.
+   * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, String)}
    * @param builder builder to modify
    * @param conf configuration to read
    * @param optionalPrefix prefix for optional settings
@@ -136,56 +137,39 @@ public static <T> T raiseInnerCause(final CompletionException e)
    * @param <U> type of builder
    * @return the builder passed in.
    */
+  @Deprecated
   public static <T, U extends FSBuilder<T, U>>
         FSBuilder<T, U> propagateOptions(
       final FSBuilder<T, U> builder,
       final Configuration conf,
       final String optionalPrefix,
       final String mandatoryPrefix) {
-    propagateOptions(builder, conf,
-        optionalPrefix, false);
-    propagateOptions(builder, conf,
-        mandatoryPrefix, true);
-    return builder;
+    return FutureIO.propagateOptions(builder,
+        conf, optionalPrefix, mandatoryPrefix);
   }
 
   /**
-   * Propagate options to any builder, converting everything with the
-   * prefix to an option where, if there were 2+ dot-separated elements,
-   * it is converted to a schema.
-   * <pre>{@code
-   *   fs.example.s3a.option => s3a:option
-   *   fs.example.fs.io.policy => s3a.io.policy
-   *   fs.example.something => something
-   * }</pre>
+   * Propagate options to any builder.
+   * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, boolean)}
    * @param builder builder to modify
    * @param conf configuration to read
    * @param prefix prefix to scan/strip
    * @param mandatory are the options to be mandatory or optional?
    */
+  @Deprecated
   public static void propagateOptions(
       final FSBuilder<?, ?> builder,
       final Configuration conf,
       final String prefix,
       final boolean mandatory) {
-
-    final String p = prefix.endsWith(".") ? prefix : (prefix + ".");
-    final Map<String, String> propsWithPrefix = conf.getPropsWithPrefix(p);
-    for (Map.Entry<String, String> entry : propsWithPrefix.entrySet()) {
-      // change the schema off each entry
-      String key = entry.getKey();
-      String val = entry.getValue();
-      if (mandatory) {
-        builder.must(key, val);
-      } else {
-        builder.opt(key, val);
-      }
-    }
+    FutureIO.propagateOptions(builder, conf, prefix, mandatory);
   }
 
   /**
    * Evaluate a CallableRaisingIOE in the current thread,
    * converting IOEs to RTEs and propagating.
+   * See {@link FutureIO#eval(CallableRaisingIOE)}.
+   *
    * @param callable callable to invoke
    * @param <T> Return type.
    * @return the evaluated result.
@@ -194,17 +178,6 @@ public static void propagateOptions(
    */
   public static <T> CompletableFuture<T> eval(
       CallableRaisingIOE<T> callable) {
-    CompletableFuture<T> result = new CompletableFuture<>();
-    try {
-      result.complete(callable.apply());
-    } catch (UnsupportedOperationException | IllegalArgumentException tx) {
-      // fail fast here
-      throw tx;
-    } catch (Throwable tx) {
-      // fail lazily here to ensure callers expect all File IO operations to
-      // surface later
-      result.completeExceptionally(tx);
-    }
-    return result;
+    return FutureIO.eval(callable);
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
index 5584e647849f5..665bcc6a95660 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
@@ -88,6 +88,9 @@ protected MultipartUploaderBuilderImpl(@Nonnull FileContext fc,
 
   /**
    * Constructor.
+   *
+   * @param fileSystem fileSystem.
+   * @param p path.
    */
   protected MultipartUploaderBuilderImpl(@Nonnull FileSystem fileSystem,
       @Nonnull Path p) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
index 77b4ff52696a3..a19c5faff4d90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
@@ -38,6 +38,9 @@ public class OpenFileParameters {
    */
   private Set<String> mandatoryKeys;
 
+  /** The optional keys. */
+  private Set<String> optionalKeys;
+
   /**
    * Options set during the build sequence.
    */
@@ -61,6 +64,11 @@ public OpenFileParameters withMandatoryKeys(final Set<String> keys) {
     return this;
   }
 
+  public OpenFileParameters withOptionalKeys(final Set<String> keys) {
+    this.optionalKeys = requireNonNull(keys);
+    return this;
+  }
+
   public OpenFileParameters withOptions(final Configuration opts) {
     this.options = requireNonNull(opts);
     return this;
@@ -80,6 +88,10 @@ public Set<String> getMandatoryKeys() {
     return mandatoryKeys;
   }
 
+  public Set<String> getOptionalKeys() {
+    return optionalKeys;
+  }
+
   public Configuration getOptions() {
     return options;
   }
@@ -91,4 +103,5 @@ public int getBufferSize() {
   public FileStatus getStatus() {
     return status;
   }
+
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java
new file mode 100644
index 0000000000000..06be20310e43e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.impl;
+
+import java.lang.ref.WeakReference;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.util.WeakReferenceMap;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A WeakReferenceMap for threads.
+ * @param <V> value type of the map
+ */
+public class WeakReferenceThreadMap<V> extends WeakReferenceMap<Long, V> {
+
+  public WeakReferenceThreadMap(final Function<? super Long, ? extends V> factory,
+      @Nullable final Consumer<? super Long> referenceLost) {
+    super(factory, referenceLost);
+  }
+
+  /**
+   * Get the value for the current thread, creating if needed.
+   * @return an instance.
+   */
+  public V getForCurrentThread() {
+    return get(currentThreadId());
+  }
+
+  /**
+   * Remove the reference for the current thread.
+   * @return any reference value which existed.
+   */
+  public V removeForCurrentThread() {
+    return remove(currentThreadId());
+  }
+
+  /**
+   * Get the current thread ID.
+   * @return thread ID.
+   */
+  public long currentThreadId() {
+    return Thread.currentThread().getId();
+  }
+
+  /**
+   * Set the new value for the current thread.
+   * @param newVal new reference to set for the active thread.
+   * @return the previously set value, possibly null
+   */
+  public V setForCurrentThread(V newVal) {
+    requireNonNull(newVal);
+    long id = currentThreadId();
+
+    // if the same object is already in the map, just return it.
+    WeakReference<V> existingWeakRef = lookup(id);
+
+    // The looked up reference could be one of
+    // 1. null: nothing there
+    // 2. valid but get() == null : reference lost by GC.
+    // 3. different from the new value
+    // 4. the same as the old value
+    if (resolve(existingWeakRef) == newVal) {
+      // case 4: do nothing, return the new value
+      return newVal;
+    } else {
+      // cases 1, 2, 3: update the map and return the old value
+      return put(id, newVal);
+    }
+
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
index d2c999683c6c6..d5144b5e9c531 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
@@ -20,20 +20,17 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.concurrent.ExecutionException;
 
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * A wrapper for an IOException which
- * {@link FutureIOSupport#raiseInnerCause(ExecutionException)} knows to
- * always extract the exception.
+ * A wrapper for an IOException.
  *
  * The constructor signature guarantees the cause will be an IOException,
  * and as it checks for a null-argument, non-null.
- * @deprecated use the {@code UncheckedIOException}.
+ * @deprecated use the {@code UncheckedIOException} directly.]
  */
 @Deprecated
 @InterfaceAudience.Private
@@ -51,8 +48,4 @@ public WrappedIOException(final IOException cause) {
     super(Preconditions.checkNotNull(cause));
   }
 
-  @Override
-  public synchronized IOException getCause() {
-    return (IOException) super.getCause();
-  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java
new file mode 100644
index 0000000000000..c18dc519188ba
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Provides functionality necessary for caching blocks of data read from FileSystem.
+ */
+public interface BlockCache extends Closeable {
+
+  /**
+   * Indicates whether the given block is in this cache.
+   *
+   * @param blockNumber the id of the given block.
+   * @return true if the given block is in this cache, false otherwise.
+   */
+  boolean containsBlock(int blockNumber);
+
+  /**
+   * Gets the blocks in this cache.
+   *
+   * @return the blocks in this cache.
+   */
+  Iterable<Integer> blocks();
+
+  /**
+   * Gets the number of blocks in this cache.
+   *
+   * @return the number of blocks in this cache.
+   */
+  int size();
+
+  /**
+   * Gets the block having the given {@code blockNumber}.
+   *
+   * @param blockNumber the id of the desired block.
+   * @param buffer contents of the desired block are copied to this buffer.
+   * @throws IOException if there is an error reading the given block.
+   */
+  void get(int blockNumber, ByteBuffer buffer) throws IOException;
+
+  /**
+   * Puts the given block in this cache.
+   *
+   * @param blockNumber the id of the given block.
+   * @param buffer contents of the given block to be added to this cache.
+   * @throws IOException if there is an error writing the given block.
+   */
+  void put(int blockNumber, ByteBuffer buffer) throws IOException;
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java
new file mode 100644
index 0000000000000..ecb8bc7243be0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkWithinRange;
+
+/**
+ * Holds information about blocks of data in a file.
+ */
+public final class BlockData {
+
+  // State of each block of data.
+  enum State {
+
+    /** Data is not yet ready to be read from this block (still being prefetched). */
+    NOT_READY,
+
+    /** A read of this block has been enqueued in the prefetch queue. */
+    QUEUED,
+
+    /** A read of this block has been enqueued in the prefetch queue. */
+    READY,
+
+    /** This block has been cached in the local disk cache. */
+    CACHED
+  }
+
+  /**
+   * State of all blocks in a file.
+   */
+  private State[] state;
+
+  /**
+   * The size of a file.
+   */
+  private final long fileSize;
+
+  /**
+   * The file is divided into blocks of this size.
+   */
+  private final int blockSize;
+
+  /**
+   * The file has these many blocks.
+   */
+  private final int numBlocks;
+
+  /**
+   * Constructs an instance of {@link BlockData}.
+   * @param fileSize the size of a file.
+   * @param blockSize the file is divided into blocks of this size.
+   * @throws IllegalArgumentException if fileSize is negative.
+   * @throws IllegalArgumentException if blockSize is negative.
+   * @throws IllegalArgumentException if blockSize is zero or negative.
+   */
+  public BlockData(long fileSize, int blockSize) {
+    checkNotNegative(fileSize, "fileSize");
+    if (fileSize == 0) {
+      checkNotNegative(blockSize, "blockSize");
+    } else {
+      checkPositiveInteger(blockSize, "blockSize");
+    }
+
+    this.fileSize = fileSize;
+    this.blockSize = blockSize;
+    this.numBlocks =
+        (fileSize == 0)
+            ? 0
+            : ((int) (fileSize / blockSize)) + (fileSize % blockSize > 0
+                ? 1
+                : 0);
+    this.state = new State[this.numBlocks];
+    for (int b = 0; b < this.numBlocks; b++) {
+      setState(b, State.NOT_READY);
+    }
+  }
+
+  /**
+   * Gets the size of each block.
+   * @return the size of each block.
+   */
+  public int getBlockSize() {
+    return blockSize;
+  }
+
+  /**
+   * Gets the size of the associated file.
+   * @return the size of the associated file.
+   */
+  public long getFileSize() {
+    return fileSize;
+  }
+
+  /**
+   * Gets the number of blocks in the associated file.
+   * @return the number of blocks in the associated file.
+   */
+  public int getNumBlocks() {
+    return numBlocks;
+  }
+
+  /**
+   * Indicates whether the given block is the last block in the associated file.
+   * @param blockNumber the id of the desired block.
+   * @return true if the given block is the last block in the associated file, false otherwise.
+   * @throws IllegalArgumentException if blockNumber is invalid.
+   */
+  public boolean isLastBlock(int blockNumber) {
+    if (fileSize == 0) {
+      return false;
+    }
+
+    throwIfInvalidBlockNumber(blockNumber);
+
+    return blockNumber == (numBlocks - 1);
+  }
+
+  /**
+   * Gets the id of the block that contains the given absolute offset.
+   * @param offset the absolute offset to check.
+   * @return the id of the block that contains the given absolute offset.
+   * @throws IllegalArgumentException if offset is invalid.
+   */
+  public int getBlockNumber(long offset) {
+    throwIfInvalidOffset(offset);
+
+    return (int) (offset / blockSize);
+  }
+
+  /**
+   * Gets the size of the given block.
+   * @param blockNumber the id of the desired block.
+   * @return the size of the given block.
+   */
+  public int getSize(int blockNumber) {
+    if (fileSize == 0) {
+      return 0;
+    }
+
+    if (isLastBlock(blockNumber)) {
+      return (int) (fileSize - (((long) blockSize) * (numBlocks - 1)));
+    } else {
+      return blockSize;
+    }
+  }
+
+  /**
+   * Indicates whether the given absolute offset is valid.
+   * @param offset absolute offset in the file..
+   * @return true if the given absolute offset is valid, false otherwise.
+   */
+  public boolean isValidOffset(long offset) {
+    return (offset >= 0) && (offset < fileSize);
+  }
+
+  /**
+   * Gets the start offset of the given block.
+   * @param blockNumber the id of the given block.
+   * @return the start offset of the given block.
+   * @throws IllegalArgumentException if blockNumber is invalid.
+   */
+  public long getStartOffset(int blockNumber) {
+    throwIfInvalidBlockNumber(blockNumber);
+
+    return blockNumber * (long) blockSize;
+  }
+
+  /**
+   * Gets the relative offset corresponding to the given block and the absolute offset.
+   * @param blockNumber the id of the given block.
+   * @param offset absolute offset in the file.
+   * @return the relative offset corresponding to the given block and the absolute offset.
+   * @throws IllegalArgumentException if either blockNumber or offset is invalid.
+   */
+  public int getRelativeOffset(int blockNumber, long offset) {
+    throwIfInvalidOffset(offset);
+
+    return (int) (offset - getStartOffset(blockNumber));
+  }
+
+  /**
+   * Gets the state of the given block.
+   * @param blockNumber the id of the given block.
+   * @return the state of the given block.
+   * @throws IllegalArgumentException if blockNumber is invalid.
+   */
+  public State getState(int blockNumber) {
+    throwIfInvalidBlockNumber(blockNumber);
+
+    return state[blockNumber];
+  }
+
+  /**
+   * Sets the state of the given block to the given value.
+   * @param blockNumber the id of the given block.
+   * @param blockState the target state.
+   * @throws IllegalArgumentException if blockNumber is invalid.
+   */
+  public void setState(int blockNumber, State blockState) {
+    throwIfInvalidBlockNumber(blockNumber);
+
+    state[blockNumber] = blockState;
+  }
+
+  // Debug helper.
+  public String getStateString() {
+    StringBuilder sb = new StringBuilder();
+    int blockNumber = 0;
+    while (blockNumber < numBlocks) {
+      State tstate = getState(blockNumber);
+      int endBlockNumber = blockNumber;
+      while ((endBlockNumber < numBlocks) && (getState(endBlockNumber)
+          == tstate)) {
+        endBlockNumber++;
+      }
+      sb.append(
+          String.format("[%03d ~ %03d] %s%n", blockNumber, endBlockNumber - 1,
+              tstate));
+      blockNumber = endBlockNumber;
+    }
+    return sb.toString();
+  }
+
+  private void throwIfInvalidBlockNumber(int blockNumber) {
+    checkWithinRange(blockNumber, "blockNumber", 0, numBlocks - 1);
+  }
+
+  private void throwIfInvalidOffset(long offset) {
+    checkWithinRange(offset, "offset", 0, fileSize - 1);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java
new file mode 100644
index 0000000000000..45f0aabe7dcd9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull;
+
+/**
+ * Provides read access to the underlying file one block at a time.
+ *
+ * This class is the simplest form of a {@code BlockManager} that does
+ * perform prefetching or caching.
+ */
+public abstract class BlockManager implements Closeable {
+
+  /**
+   * Information about each block of the underlying file.
+   */
+  private final BlockData blockData;
+
+  /**
+   * Constructs an instance of {@code BlockManager}.
+   *
+   * @param blockData information about each block of the underlying file.
+   *
+   * @throws IllegalArgumentException if blockData is null.
+   */
+  public BlockManager(BlockData blockData) {
+    checkNotNull(blockData, "blockData");
+
+    this.blockData = blockData;
+  }
+
+  /**
+   * Gets block data information.
+   *
+   * @return instance of {@code BlockData}.
+   */
+  public BlockData getBlockData() {
+    return blockData;
+  }
+
+  /**
+   * Gets the block having the given {@code blockNumber}.
+   *
+   * The entire block is read into memory and returned as a {@code BufferData}.
+   * The blocks are treated as a limited resource and must be released when
+   * one is done reading them.
+   *
+   * @param blockNumber the number of the block to be read and returned.
+   * @return {@code BufferData} having data from the given block.
+   *
+   * @throws IOException if there an error reading the given block.
+   * @throws IllegalArgumentException if blockNumber is negative.
+   */
+  public BufferData get(int blockNumber) throws IOException {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    int size = blockData.getSize(blockNumber);
+    ByteBuffer buffer = ByteBuffer.allocate(size);
+    long startOffset = blockData.getStartOffset(blockNumber);
+    read(buffer, startOffset, size);
+    buffer.flip();
+    return new BufferData(blockNumber, buffer);
+  }
+
+  /**
+   * Reads into the given {@code buffer} {@code size} bytes from the underlying file
+   * starting at {@code startOffset}.
+   *
+   * @param buffer the buffer to read data in to.
+   * @param startOffset the offset at which reading starts.
+   * @param size the number bytes to read.
+   * @return number of bytes read.
+   * @throws IOException if there an error reading the given block.
+   */
+  public abstract int read(ByteBuffer buffer, long startOffset, int size) throws IOException;
+
+  /**
+   * Releases resources allocated to the given block.
+   *
+   * @param data the {@code BufferData} to release.
+   *
+   * @throws IllegalArgumentException if data is null.
+   */
+  public void release(BufferData data) {
+    checkNotNull(data, "data");
+
+    // Do nothing because we allocate a new buffer each time.
+  }
+
+  /**
+   * Requests optional prefetching of the given block.
+   *
+   * @param blockNumber the id of the block to prefetch.
+   *
+   * @throws IllegalArgumentException if blockNumber is negative.
+   */
+  public void requestPrefetch(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    // Do nothing because we do not support prefetches.
+  }
+
+  /**
+   * Requests cancellation of any previously issued prefetch requests.
+   */
+  public void cancelPrefetches() {
+    // Do nothing because we do not support prefetches.
+  }
+
+  /**
+   * Requests that the given block should be copied to the cache. Optional operation.
+   *
+   * @param data the {@code BufferData} instance to optionally cache.
+   */
+  public void requestCaching(BufferData data) {
+    // Do nothing because we do not support caching.
+  }
+
+  @Override
+  public void close() {
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java
new file mode 100644
index 0000000000000..2744334a3bd7a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.DoubleSummaryStatistics;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+
+/**
+ * Block level operations performed on a file.
+ * This class is meant to be used by {@code BlockManager}.
+ * It is separated out in its own file due to its size.
+ *
+ * This class is used for debugging/logging. Calls to this class
+ * can be safely removed without affecting the overall operation.
+ */
+public final class BlockOperations {
+  private static final Logger LOG = LoggerFactory.getLogger(BlockOperations.class);
+
+  /**
+   * Operation kind.
+   */
+  public enum Kind {
+    UNKNOWN("??", "unknown", false),
+    CANCEL_PREFETCHES("CP", "cancelPrefetches", false),
+    CLOSE("CX", "close", false),
+    CACHE_PUT("C+", "putC", true),
+    GET_CACHED("GC", "getCached", true),
+    GET_PREFETCHED("GP", "getPrefetched", true),
+    GET_READ("GR", "getRead", true),
+    PREFETCH("PF", "prefetch", true),
+    RELEASE("RL", "release", true),
+    REQUEST_CACHING("RC", "requestCaching", true),
+    REQUEST_PREFETCH("RP", "requestPrefetch", true);
+
+    private String shortName;
+    private String name;
+    private boolean hasBlock;
+
+    Kind(String shortName, String name, boolean hasBlock) {
+      this.shortName = shortName;
+      this.name = name;
+      this.hasBlock = hasBlock;
+    }
+
+    private static Map<String, Kind> shortNameToKind = new HashMap<>();
+
+    public static Kind fromShortName(String shortName) {
+      if (shortNameToKind.isEmpty()) {
+        for (Kind kind : Kind.values()) {
+          shortNameToKind.put(kind.shortName, kind);
+        }
+      }
+      return shortNameToKind.get(shortName);
+    }
+  }
+
+  public static class Operation {
+    private final Kind kind;
+    private final int blockNumber;
+    private final long timestamp;
+
+    public Operation(Kind kind, int blockNumber) {
+      this.kind = kind;
+      this.blockNumber = blockNumber;
+      this.timestamp = System.nanoTime();
+    }
+
+    public Kind getKind() {
+      return kind;
+    }
+
+    public int getBlockNumber() {
+      return blockNumber;
+    }
+
+    public long getTimestamp() {
+      return timestamp;
+    }
+
+    public void getSummary(StringBuilder sb) {
+      if (kind.hasBlock) {
+        sb.append(String.format("%s(%d)", kind.shortName, blockNumber));
+      } else {
+        sb.append(String.format("%s", kind.shortName));
+      }
+    }
+
+    public String getDebugInfo() {
+      if (kind.hasBlock) {
+        return String.format("--- %s(%d)", kind.name, blockNumber);
+      } else {
+        return String.format("... %s()", kind.name);
+      }
+    }
+  }
+
+  public static class End extends Operation {
+    private Operation op;
+
+    public End(Operation op) {
+      super(op.kind, op.blockNumber);
+      this.op = op;
+    }
+
+    @Override
+    public void getSummary(StringBuilder sb) {
+      sb.append("E");
+      super.getSummary(sb);
+    }
+
+    @Override
+    public String getDebugInfo() {
+      return "***" + super.getDebugInfo().substring(3);
+    }
+
+    public double duration() {
+      return (getTimestamp() - op.getTimestamp()) / 1e9;
+    }
+  }
+
+  private ArrayList<Operation> ops;
+  private boolean debugMode;
+
+  public BlockOperations() {
+    this.ops = new ArrayList<>();
+  }
+
+  public synchronized void setDebug(boolean state) {
+    debugMode = state;
+  }
+
+  private synchronized Operation add(Operation op) {
+    if (debugMode) {
+      LOG.info(op.getDebugInfo());
+    }
+    ops.add(op);
+    return op;
+  }
+
+  public Operation getPrefetched(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.GET_PREFETCHED, blockNumber));
+  }
+
+  public Operation getCached(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.GET_CACHED, blockNumber));
+  }
+
+  public Operation getRead(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.GET_READ, blockNumber));
+  }
+
+  public Operation release(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.RELEASE, blockNumber));
+  }
+
+  public Operation requestPrefetch(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.REQUEST_PREFETCH, blockNumber));
+  }
+
+  public Operation prefetch(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.PREFETCH, blockNumber));
+  }
+
+  public Operation cancelPrefetches() {
+    return add(new Operation(Kind.CANCEL_PREFETCHES, -1));
+  }
+
+  public Operation close() {
+    return add(new Operation(Kind.CLOSE, -1));
+  }
+
+  public Operation requestCaching(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.REQUEST_CACHING, blockNumber));
+  }
+
+  public Operation addToCache(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    return add(new Operation(Kind.CACHE_PUT, blockNumber));
+  }
+
+  public Operation end(Operation op) {
+    return add(new End(op));
+  }
+
+  private static void append(StringBuilder sb, String format, Object... args) {
+    sb.append(String.format(format, args));
+  }
+
+  public synchronized String getSummary(boolean showDebugInfo) {
+    StringBuilder sb = new StringBuilder();
+    for (Operation op : ops) {
+      if (op != null) {
+        if (showDebugInfo) {
+          sb.append(op.getDebugInfo());
+          sb.append("\n");
+        } else {
+          op.getSummary(sb);
+          sb.append(";");
+        }
+      }
+    }
+
+    sb.append("\n");
+    getDurationInfo(sb);
+
+    return sb.toString();
+  }
+
+  public synchronized void getDurationInfo(StringBuilder sb) {
+    Map<Kind, DoubleSummaryStatistics> durations = new HashMap<>();
+    for (Operation op : ops) {
+      if (op instanceof End) {
+        End endOp = (End) op;
+        DoubleSummaryStatistics stats = durations.get(endOp.getKind());
+        if (stats == null) {
+          stats = new DoubleSummaryStatistics();
+          durations.put(endOp.getKind(), stats);
+        }
+        stats.accept(endOp.duration());
+      }
+    }
+
+    List<Kind> kinds = Arrays.asList(
+        Kind.GET_CACHED,
+        Kind.GET_PREFETCHED,
+        Kind.GET_READ,
+        Kind.CACHE_PUT,
+        Kind.PREFETCH,
+        Kind.REQUEST_CACHING,
+        Kind.REQUEST_PREFETCH,
+        Kind.CANCEL_PREFETCHES,
+        Kind.RELEASE,
+        Kind.CLOSE
+    );
+
+    for (Kind kind : kinds) {
+      append(sb, "%-18s : ", kind);
+      DoubleSummaryStatistics stats = durations.get(kind);
+      if (stats == null) {
+        append(sb, "--\n");
+      } else {
+        append(
+            sb,
+            "#ops = %3d, total = %5.1f, min: %3.1f, avg: %3.1f, max: %3.1f\n",
+            stats.getCount(),
+            stats.getSum(),
+            stats.getMin(),
+            stats.getAverage(),
+            stats.getMax());
+      }
+    }
+  }
+
+  public synchronized void analyze(StringBuilder sb) {
+    Map<Integer, List<Operation>> blockOps = new HashMap<>();
+
+    // Group-by block number.
+    for (Operation op : ops) {
+      if (op.blockNumber < 0) {
+        continue;
+      }
+
+      List<Operation> perBlockOps;
+      if (!blockOps.containsKey(op.blockNumber)) {
+        perBlockOps = new ArrayList<>();
+        blockOps.put(op.blockNumber, perBlockOps);
+      }
+
+      perBlockOps = blockOps.get(op.blockNumber);
+      perBlockOps.add(op);
+    }
+
+    List<Integer> prefetchedNotUsed = new ArrayList<>();
+    List<Integer> cachedNotUsed = new ArrayList<>();
+
+    for (Map.Entry<Integer, List<Operation>> entry : blockOps.entrySet()) {
+      Integer blockNumber = entry.getKey();
+      List<Operation> perBlockOps = entry.getValue();
+      Map<Kind, Integer> kindCounts = new HashMap<>();
+      Map<Kind, Integer> endKindCounts = new HashMap<>();
+
+      for (Operation op : perBlockOps) {
+        if (op instanceof End) {
+          int endCount = endKindCounts.getOrDefault(op.kind, 0) + 1;
+          endKindCounts.put(op.kind, endCount);
+        } else {
+          int count = kindCounts.getOrDefault(op.kind, 0) + 1;
+          kindCounts.put(op.kind, count);
+        }
+      }
+
+      for (Kind kind : kindCounts.keySet()) {
+        int count = kindCounts.getOrDefault(kind, 0);
+        int endCount = endKindCounts.getOrDefault(kind, 0);
+        if (count != endCount) {
+          append(sb, "[%d] %s : #ops(%d) != #end-ops(%d)\n", blockNumber, kind, count, endCount);
+        }
+
+        if (count > 1) {
+          append(sb, "[%d] %s = %d\n", blockNumber, kind, count);
+        }
+      }
+
+      int prefetchCount = kindCounts.getOrDefault(Kind.PREFETCH, 0);
+      int getPrefetchedCount = kindCounts.getOrDefault(Kind.GET_PREFETCHED, 0);
+      if ((prefetchCount > 0) && (getPrefetchedCount < prefetchCount)) {
+        prefetchedNotUsed.add(blockNumber);
+      }
+
+      int cacheCount = kindCounts.getOrDefault(Kind.CACHE_PUT, 0);
+      int getCachedCount = kindCounts.getOrDefault(Kind.GET_CACHED, 0);
+      if ((cacheCount > 0) && (getCachedCount < cacheCount)) {
+        cachedNotUsed.add(blockNumber);
+      }
+    }
+
+    if (!prefetchedNotUsed.isEmpty()) {
+      append(sb, "Prefetched but not used: %s\n", getIntList(prefetchedNotUsed));
+    }
+
+    if (!cachedNotUsed.isEmpty()) {
+      append(sb, "Cached but not used: %s\n", getIntList(cachedNotUsed));
+    }
+  }
+
+  private static String getIntList(Iterable<Integer> nums) {
+    List<String> numList = new ArrayList<>();
+    for (Integer n : nums) {
+      numList.add(n.toString());
+    }
+    return String.join(", ", numList);
+  }
+
+  public static BlockOperations fromSummary(String summary) {
+    BlockOperations ops = new BlockOperations();
+    ops.setDebug(true);
+    Pattern blockOpPattern = Pattern.compile("([A-Z+]+)(\\(([0-9]+)?\\))?");
+    String[] tokens = summary.split(";");
+    for (String token : tokens) {
+      Matcher matcher = blockOpPattern.matcher(token);
+      if (!matcher.matches()) {
+        String message = String.format("Unknown summary format: %s", token);
+        throw new IllegalArgumentException(message);
+      }
+
+      String shortName = matcher.group(1);
+      String blockNumberStr = matcher.group(3);
+      int blockNumber = (blockNumberStr == null) ? -1 : Integer.parseInt(blockNumberStr);
+      Kind kind = Kind.fromShortName(shortName);
+      Kind endKind = null;
+      if (kind == null) {
+        if (shortName.charAt(0) == 'E') {
+          endKind = Kind.fromShortName(shortName.substring(1));
+        }
+      }
+
+      if (kind == null && endKind == null) {
+        String message = String.format("Unknown short name: %s (token = %s)", shortName, token);
+        throw new IllegalArgumentException(message);
+      }
+
+      if (kind != null) {
+        ops.add(new Operation(kind, blockNumber));
+      } else {
+        Operation op = null;
+        for (int i = ops.ops.size() - 1; i >= 0; i--) {
+          op = ops.ops.get(i);
+          if ((op.blockNumber == blockNumber) && (op.kind == endKind) && !(op instanceof End)) {
+            ops.add(new End(op));
+            break;
+          }
+        }
+
+        if (op == null) {
+          LOG.warn("Start op not found: {}({})", endKind, blockNumber);
+        }
+      }
+    }
+
+    return ops;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java
new file mode 100644
index 0000000000000..a871f8237729f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull;
+
+/**
+ * Manages a fixed pool of resources.
+ *
+ * Avoids creating a new resource if a previously created instance is already available.
+ */
+public abstract class BoundedResourcePool<T> extends ResourcePool<T> {
+  /**
+   * The size of this pool. Fixed at creation time.
+   */
+  private final int size;
+
+  /**
+   * Items currently available in the pool.
+   */
+  private ArrayBlockingQueue<T> items;
+
+  /**
+   * Items that have been created so far (regardless of whether they are currently available).
+   */
+  private Set<T> createdItems;
+
+  /**
+   * Constructs a resource pool of the given size.
+   *
+   * @param size the size of this pool. Cannot be changed post creation.
+   *
+   * @throws IllegalArgumentException if size is zero or negative.
+   */
+  public BoundedResourcePool(int size) {
+    Validate.checkPositiveInteger(size, "size");
+
+    this.size = size;
+    this.items = new ArrayBlockingQueue<>(size);
+
+    // The created items are identified based on their object reference.
+    this.createdItems = Collections.newSetFromMap(new IdentityHashMap<T, Boolean>());
+  }
+
+  /**
+   * Acquires a resource blocking if necessary until one becomes available.
+   */
+  @Override
+  public T acquire() {
+    return this.acquireHelper(true);
+  }
+
+  /**
+   * Acquires a resource blocking if one is immediately available. Otherwise returns null.
+   */
+  @Override
+  public T tryAcquire() {
+    return this.acquireHelper(false);
+  }
+
+  /**
+   * Releases a previously acquired resource.
+   *
+   * @throws IllegalArgumentException if item is null.
+   */
+  @Override
+  public void release(T item) {
+    checkNotNull(item, "item");
+
+    synchronized (createdItems) {
+      if (!createdItems.contains(item)) {
+        throw new IllegalArgumentException("This item is not a part of this pool");
+      }
+    }
+
+    // Return if this item was released earlier.
+    // We cannot use items.contains() because that check is not based on reference equality.
+    for (T entry : items) {
+      if (entry == item) {
+        return;
+      }
+    }
+
+    try {
+      items.put(item);
+    } catch (InterruptedException e) {
+      throw new IllegalStateException("release() should never block", e);
+    }
+  }
+
+  @Override
+  public synchronized void close() {
+    for (T item : createdItems) {
+      close(item);
+    }
+
+    items.clear();
+    items = null;
+
+    createdItems.clear();
+    createdItems = null;
+  }
+
+  /**
+   * Derived classes may implement a way to cleanup each item.
+   */
+  @Override
+  protected synchronized void close(T item) {
+    // Do nothing in this class. Allow overriding classes to take any cleanup action.
+  }
+
+  /**
+   * Number of items created so far. Mostly for testing purposes.
+   * @return the count.
+   */
+  public int numCreated() {
+    synchronized (createdItems) {
+      return createdItems.size();
+    }
+  }
+
+  /**
+   * Number of items available to be acquired. Mostly for testing purposes.
+   * @return the number available.
+   */
+  public synchronized int numAvailable() {
+    return (size - numCreated()) + items.size();
+  }
+
+  // For debugging purposes.
+  @Override
+  public synchronized String toString() {
+    return String.format(
+        "size = %d, #created = %d, #in-queue = %d, #available = %d",
+        size, numCreated(), items.size(), numAvailable());
+  }
+
+  /**
+   * Derived classes must implement a way to create an instance of a resource.
+   */
+  protected abstract T createNew();
+
+  private T acquireHelper(boolean canBlock) {
+
+    // Prefer reusing an item if one is available.
+    // That avoids unnecessarily creating new instances.
+    T result = items.poll();
+    if (result != null) {
+      return result;
+    }
+
+    synchronized (createdItems) {
+      // Create a new instance if allowed by the capacity of this pool.
+      if (createdItems.size() < size) {
+        T item = createNew();
+        createdItems.add(item);
+        return item;
+      }
+    }
+
+    if (canBlock) {
+      try {
+        // Block for an instance to be available.
+        return items.take();
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        return null;
+      }
+    } else {
+      return null;
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java
new file mode 100644
index 0000000000000..de68269ab700c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+import java.util.zip.CRC32;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Holds the state of a ByteBuffer that is in use by {@code CachingBlockManager}.
+ *
+ * This class is not meant to be of general use. It exists into its own file due to its size.
+ * We use the term block and buffer interchangeably in this file because one buffer
+ * holds exactly one block of data.
+ *
+ * Holding all of the state associated with a block allows us to validate and control
+ * state transitions in a synchronized fashion.
+ */
+public final class BufferData {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BufferData.class);
+
+  public enum State {
+    /**
+     * Unknown / invalid state.
+     */
+    UNKNOWN,
+
+    /**
+     * Buffer has been acquired but has no data.
+     */
+    BLANK,
+
+    /**
+     * This block is being prefetched.
+     */
+    PREFETCHING,
+
+    /**
+     * This block is being added to the local cache.
+     */
+    CACHING,
+
+    /**
+     * This block has data and is ready to be read.
+     */
+    READY,
+
+    /**
+     * This block is no longer in-use and should not be used once in this state.
+     */
+    DONE
+  }
+
+  /**
+   * Number of the block associated with this buffer.
+   */
+  private final int blockNumber;
+
+  /**
+   * The buffer associated with this block.
+   */
+  private ByteBuffer buffer;
+
+  /**
+   * Current state of this block.
+   */
+  private volatile State state;
+
+  /**
+   * Future of the action being performed on this block (eg, prefetching or caching).
+   */
+  private Future<Void> action;
+
+  /**
+   * Checksum of the buffer contents once in READY state.
+   */
+  private long checksum = 0;
+
+  /**
+   * Constructs an instances of this class.
+   *
+   * @param blockNumber Number of the block associated with this buffer.
+   * @param buffer The buffer associated with this block.
+   *
+   * @throws IllegalArgumentException if blockNumber is negative.
+   * @throws IllegalArgumentException if buffer is null.
+   */
+  public BufferData(int blockNumber, ByteBuffer buffer) {
+    Validate.checkNotNegative(blockNumber, "blockNumber");
+    Validate.checkNotNull(buffer, "buffer");
+
+    this.blockNumber = blockNumber;
+    this.buffer = buffer;
+    this.state = State.BLANK;
+  }
+
+  /**
+   * Gets the id of this block.
+   *
+   * @return the id of this block.
+   */
+  public int getBlockNumber() {
+    return this.blockNumber;
+  }
+
+  /**
+   * Gets the buffer associated with this block.
+   *
+   * @return the buffer associated with this block.
+   */
+  public ByteBuffer getBuffer() {
+    return this.buffer;
+  }
+
+  /**
+   * Gets the state of this block.
+   *
+   * @return the state of this block.
+   */
+  public State getState() {
+    return this.state;
+  }
+
+  /**
+   * Gets the checksum of data in this block.
+   *
+   * @return the checksum of data in this block.
+   */
+  public long getChecksum() {
+    return this.checksum;
+  }
+
+  /**
+   * Computes CRC32 checksum of the given buffer's contents.
+   *
+   * @param buffer the buffer whose content's checksum is to be computed.
+   * @return the computed checksum.
+   */
+  public static long getChecksum(ByteBuffer buffer) {
+    ByteBuffer tempBuffer = buffer.duplicate();
+    tempBuffer.rewind();
+    CRC32 crc32 = new CRC32();
+    crc32.update(tempBuffer);
+    return crc32.getValue();
+  }
+
+  public synchronized Future<Void> getActionFuture() {
+    return this.action;
+  }
+
+  /**
+   * Indicates that a prefetch operation is in progress.
+   *
+   * @param actionFuture the {@code Future} of a prefetch action.
+   *
+   * @throws IllegalArgumentException if actionFuture is null.
+   */
+  public synchronized void setPrefetch(Future<Void> actionFuture) {
+    Validate.checkNotNull(actionFuture, "actionFuture");
+
+    this.updateState(State.PREFETCHING, State.BLANK);
+    this.action = actionFuture;
+  }
+
+  /**
+   * Indicates that a caching operation is in progress.
+   *
+   * @param actionFuture the {@code Future} of a caching action.
+   *
+   * @throws IllegalArgumentException if actionFuture is null.
+   */
+  public synchronized void setCaching(Future<Void> actionFuture) {
+    Validate.checkNotNull(actionFuture, "actionFuture");
+
+    this.throwIfStateIncorrect(State.PREFETCHING, State.READY);
+    this.state = State.CACHING;
+    this.action = actionFuture;
+  }
+
+  /**
+   * Marks the completion of reading data into the buffer.
+   * The buffer cannot be modified once in this state.
+   *
+   * @param expectedCurrentState the collection of states from which transition to READY is allowed.
+   */
+  public synchronized void setReady(State... expectedCurrentState) {
+    if (this.checksum != 0) {
+      throw new IllegalStateException("Checksum cannot be changed once set");
+    }
+
+    this.buffer = this.buffer.asReadOnlyBuffer();
+    this.checksum = getChecksum(this.buffer);
+    this.buffer.rewind();
+    this.updateState(State.READY, expectedCurrentState);
+  }
+
+  /**
+   * Indicates that this block is no longer of use and can be reclaimed.
+   */
+  public synchronized void setDone() {
+    if (this.checksum != 0) {
+      if (getChecksum(this.buffer) != this.checksum) {
+        throw new IllegalStateException("checksum changed after setReady()");
+      }
+    }
+    this.state = State.DONE;
+    this.action = null;
+  }
+
+  /**
+   * Updates the current state to the specified value.
+   * Asserts that the current state is as expected.
+   * @param newState the state to transition to.
+   * @param expectedCurrentState the collection of states from which
+   *        transition to {@code newState} is allowed.
+   *
+   * @throws IllegalArgumentException if newState is null.
+   * @throws IllegalArgumentException if expectedCurrentState is null.
+   */
+  public synchronized void updateState(State newState,
+      State... expectedCurrentState) {
+    Validate.checkNotNull(newState, "newState");
+    Validate.checkNotNull(expectedCurrentState, "expectedCurrentState");
+
+    this.throwIfStateIncorrect(expectedCurrentState);
+    this.state = newState;
+  }
+
+  /**
+   * Helper that asserts the current state is one of the expected values.
+   *
+   * @param states the collection of allowed states.
+   *
+   * @throws IllegalArgumentException if states is null.
+   */
+  public void throwIfStateIncorrect(State... states) {
+    Validate.checkNotNull(states, "states");
+
+    if (this.stateEqualsOneOf(states)) {
+      return;
+    }
+
+    List<String> statesStr = new ArrayList<String>();
+    for (State s : states) {
+      statesStr.add(s.toString());
+    }
+
+    String message = String.format(
+        "Expected buffer state to be '%s' but found: %s",
+        String.join(" or ", statesStr), this);
+    throw new IllegalStateException(message);
+  }
+
+  public boolean stateEqualsOneOf(State... states) {
+    State currentState = this.state;
+
+    for (State s : states) {
+      if (currentState == s) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  public String toString() {
+
+    return String.format(
+        "[%03d] id: %03d, %s: buf: %s, checksum: %d, future: %s",
+        this.blockNumber,
+        System.identityHashCode(this),
+        this.state,
+        this.getBufferStr(this.buffer),
+        this.checksum,
+        this.getFutureStr(this.action));
+  }
+
+  private String getFutureStr(Future<Void> f) {
+    if (f == null) {
+      return "--";
+    } else {
+      return this.action.isDone() ? "done" : "not done";
+    }
+  }
+
+  private String getBufferStr(ByteBuffer buf) {
+    if (buf == null) {
+      return "--";
+    } else {
+      return String.format(
+          "(id = %d, pos = %d, lim = %d)",
+          System.identityHashCode(buf),
+          buf.position(), buf.limit());
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java
new file mode 100644
index 0000000000000..189357f6bd04f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkState;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
+
+/**
+ * Manages a fixed pool of {@code ByteBuffer} instances.
+ * <p>
+ * Avoids creating a new buffer if a previously created buffer is already available.
+ */
+public class BufferPool implements Closeable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BufferPool.class);
+
+  /**
+   * Max number of buffers in this pool.
+   */
+  private final int size;
+
+  /**
+   * Size in bytes of each buffer.
+   */
+  private final int bufferSize;
+
+  /*
+   Invariants for internal state.
+   -- a buffer is either in this.pool or in this.allocated
+   -- transition between this.pool <==> this.allocated must be atomic
+   -- only one buffer allocated for a given blockNumber
+  */
+
+
+  /**
+   * Underlying bounded resource pool.
+   */
+  private BoundedResourcePool<ByteBuffer> pool;
+
+  /**
+   * Allows associating metadata to each buffer in the pool.
+   */
+  private Map<BufferData, ByteBuffer> allocated;
+
+  /**
+   * Prefetching stats.
+   */
+  private PrefetchingStatistics prefetchingStatistics;
+
+  /**
+   * Initializes a new instance of the {@code BufferPool} class.
+   * @param size number of buffer in this pool.
+   * @param bufferSize size in bytes of each buffer.
+   * @param prefetchingStatistics statistics for this stream.
+   * @throws IllegalArgumentException if size is zero or negative.
+   * @throws IllegalArgumentException if bufferSize is zero or negative.
+   */
+  public BufferPool(int size,
+      int bufferSize,
+      PrefetchingStatistics prefetchingStatistics) {
+    Validate.checkPositiveInteger(size, "size");
+    Validate.checkPositiveInteger(bufferSize, "bufferSize");
+
+    this.size = size;
+    this.bufferSize = bufferSize;
+    this.allocated = new IdentityHashMap<BufferData, ByteBuffer>();
+    this.prefetchingStatistics = requireNonNull(prefetchingStatistics);
+    this.pool = new BoundedResourcePool<ByteBuffer>(size) {
+      @Override
+      public ByteBuffer createNew() {
+        ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+        prefetchingStatistics.memoryAllocated(bufferSize);
+        return buffer;
+      }
+    };
+  }
+
+  /**
+   * Gets a list of all blocks in this pool.
+   * @return a list of all blocks in this pool.
+   */
+  public List<BufferData> getAll() {
+    synchronized (allocated) {
+      return Collections.unmodifiableList(new ArrayList<>(allocated.keySet()));
+    }
+  }
+
+  /**
+   * Acquires a {@code ByteBuffer}; blocking if necessary until one becomes available.
+   * @param blockNumber the id of the block to acquire.
+   * @return the acquired block's {@code BufferData}.
+   */
+  public synchronized BufferData acquire(int blockNumber) {
+    BufferData data;
+    final int maxRetryDelayMs = 600 * 1000;
+    final int statusUpdateDelayMs = 120 * 1000;
+    Retryer retryer = new Retryer(10, maxRetryDelayMs, statusUpdateDelayMs);
+
+    do {
+      if (retryer.updateStatus()) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("waiting to acquire block: {}", blockNumber);
+          LOG.debug("state = {}", this);
+        }
+        releaseReadyBlock(blockNumber);
+      }
+      data = tryAcquire(blockNumber);
+    }
+    while ((data == null) && retryer.continueRetry());
+
+    if (data != null) {
+      return data;
+    } else {
+      String message =
+          String.format("Wait failed for acquire(%d)", blockNumber);
+      throw new IllegalStateException(message);
+    }
+  }
+
+  /**
+   * Acquires a buffer if one is immediately available. Otherwise returns null.
+   * @param blockNumber the id of the block to try acquire.
+   * @return the acquired block's {@code BufferData} or null.
+   */
+  public synchronized BufferData tryAcquire(int blockNumber) {
+    return acquireHelper(blockNumber, false);
+  }
+
+  private synchronized BufferData acquireHelper(int blockNumber,
+      boolean canBlock) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    releaseDoneBlocks();
+
+    BufferData data = find(blockNumber);
+    if (data != null) {
+      return data;
+    }
+
+    ByteBuffer buffer = canBlock ? pool.acquire() : pool.tryAcquire();
+    if (buffer == null) {
+      return null;
+    }
+
+    buffer.clear();
+    data = new BufferData(blockNumber, buffer.duplicate());
+
+    synchronized (allocated) {
+      checkState(find(blockNumber) == null, "buffer data already exists");
+
+      allocated.put(data, buffer);
+    }
+
+    return data;
+  }
+
+  /**
+   * Releases resources for any blocks marked as 'done'.
+   */
+  private synchronized void releaseDoneBlocks() {
+    for (BufferData data : getAll()) {
+      if (data.stateEqualsOneOf(BufferData.State.DONE)) {
+        release(data);
+      }
+    }
+  }
+
+  /**
+   * If no blocks were released after calling releaseDoneBlocks() a few times,
+   * we may end up waiting forever. To avoid that situation, we try releasing
+   * a 'ready' block farthest away from the given block.
+   */
+  private synchronized void releaseReadyBlock(int blockNumber) {
+    BufferData releaseTarget = null;
+    for (BufferData data : getAll()) {
+      if (data.stateEqualsOneOf(BufferData.State.READY)) {
+        if (releaseTarget == null) {
+          releaseTarget = data;
+        } else {
+          if (distance(data, blockNumber) > distance(releaseTarget,
+              blockNumber)) {
+            releaseTarget = data;
+          }
+        }
+      }
+    }
+
+    if (releaseTarget != null) {
+      LOG.warn("releasing 'ready' block: {}", releaseTarget);
+      releaseTarget.setDone();
+    }
+  }
+
+  private int distance(BufferData data, int blockNumber) {
+    return Math.abs(data.getBlockNumber() - blockNumber);
+  }
+
+  /**
+   * Releases a previously acquired resource.
+   * @param data the {@code BufferData} instance to release.
+   * @throws IllegalArgumentException if data is null.
+   * @throws IllegalArgumentException if data cannot be released due to its state.
+   */
+  public synchronized void release(BufferData data) {
+    checkNotNull(data, "data");
+
+    synchronized (data) {
+      checkArgument(
+          canRelease(data),
+          String.format("Unable to release buffer: %s", data));
+
+      ByteBuffer buffer = allocated.get(data);
+      if (buffer == null) {
+        // Likely released earlier.
+        return;
+      }
+      buffer.clear();
+      pool.release(buffer);
+      allocated.remove(data);
+    }
+
+    releaseDoneBlocks();
+  }
+
+  @Override
+  public synchronized void close() {
+    for (BufferData data : getAll()) {
+      Future<Void> actionFuture = data.getActionFuture();
+      if (actionFuture != null) {
+        actionFuture.cancel(true);
+      }
+    }
+
+    int currentPoolSize = pool.numCreated();
+
+    pool.close();
+    pool = null;
+
+    allocated.clear();
+    allocated = null;
+
+    prefetchingStatistics.memoryFreed(currentPoolSize * bufferSize);
+  }
+
+  // For debugging purposes.
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(pool.toString());
+    sb.append("\n");
+    List<BufferData> allData = new ArrayList<>(getAll());
+    Collections.sort(allData,
+        (d1, d2) -> d1.getBlockNumber() - d2.getBlockNumber());
+    for (BufferData data : allData) {
+      sb.append(data.toString());
+      sb.append("\n");
+    }
+
+    return sb.toString();
+  }
+
+  // Number of ByteBuffers created so far.
+  public synchronized int numCreated() {
+    return pool.numCreated();
+  }
+
+  // Number of ByteBuffers available to be acquired.
+  public synchronized int numAvailable() {
+    releaseDoneBlocks();
+    return pool.numAvailable();
+  }
+
+  private BufferData find(int blockNumber) {
+    synchronized (allocated) {
+      for (BufferData data : allocated.keySet()) {
+        if ((data.getBlockNumber() == blockNumber)
+            && !data.stateEqualsOneOf(BufferData.State.DONE)) {
+          return data;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private boolean canRelease(BufferData data) {
+    return data.stateEqualsOneOf(
+        BufferData.State.DONE,
+        BufferData.State.READY);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java
new file mode 100644
index 0000000000000..31084c7bf2648
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java
@@ -0,0 +1,638 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.statistics.DurationTracker;
+
+import static java.util.Objects.requireNonNull;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+
+/**
+ * Provides read access to the underlying file one block at a time.
+ * Improve read performance by prefetching and locall caching blocks.
+ */
+public abstract class CachingBlockManager extends BlockManager {
+  private static final Logger LOG = LoggerFactory.getLogger(CachingBlockManager.class);
+  private static final int TIMEOUT_MINUTES = 60;
+
+  /**
+   * Asynchronous tasks are performed in this pool.
+   */
+  private final ExecutorServiceFuturePool futurePool;
+
+  /**
+   * Pool of shared ByteBuffer instances.
+   */
+  private BufferPool bufferPool;
+
+  /**
+   * Size of the in-memory cache in terms of number of blocks.
+   * Total memory consumption is up to bufferPoolSize * blockSize.
+   */
+  private final int bufferPoolSize;
+
+  /**
+   * Local block cache.
+   */
+  private BlockCache cache;
+
+  /**
+   * Error counts. For testing purposes.
+   */
+  private final AtomicInteger numCachingErrors;
+  private final AtomicInteger numReadErrors;
+
+  /**
+   * Operations performed by this block manager.
+   */
+  private final BlockOperations ops;
+
+  private boolean closed;
+
+  /**
+   * If a single caching operation takes more than this time (in seconds),
+   * we disable caching to prevent further perf degradation due to caching.
+   */
+  private static final int SLOW_CACHING_THRESHOLD = 5;
+
+  /**
+   * Once set to true, any further caching requests will be ignored.
+   */
+  private final AtomicBoolean cachingDisabled;
+
+  private final PrefetchingStatistics prefetchingStatistics;
+
+  /**
+   * Constructs an instance of a {@code CachingBlockManager}.
+   *
+   * @param futurePool asynchronous tasks are performed in this pool.
+   * @param blockData information about each block of the underlying file.
+   * @param bufferPoolSize size of the in-memory cache in terms of number of blocks.
+   * @param prefetchingStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if bufferPoolSize is zero or negative.
+   */
+  public CachingBlockManager(
+      ExecutorServiceFuturePool futurePool,
+      BlockData blockData,
+      int bufferPoolSize,
+      PrefetchingStatistics prefetchingStatistics) {
+    super(blockData);
+
+    Validate.checkPositiveInteger(bufferPoolSize, "bufferPoolSize");
+
+    this.futurePool = requireNonNull(futurePool);
+    this.bufferPoolSize = bufferPoolSize;
+    this.numCachingErrors = new AtomicInteger();
+    this.numReadErrors = new AtomicInteger();
+    this.cachingDisabled = new AtomicBoolean();
+    this.prefetchingStatistics = requireNonNull(prefetchingStatistics);
+
+    if (this.getBlockData().getFileSize() > 0) {
+      this.bufferPool = new BufferPool(bufferPoolSize, this.getBlockData().getBlockSize(),
+          this.prefetchingStatistics);
+      this.cache = this.createCache();
+    }
+
+    this.ops = new BlockOperations();
+    this.ops.setDebug(false);
+  }
+
+  /**
+   * Gets the block having the given {@code blockNumber}.
+   *
+   * @throws IllegalArgumentException if blockNumber is negative.
+   */
+  @Override
+  public BufferData get(int blockNumber) throws IOException {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    BufferData data;
+    final int maxRetryDelayMs = bufferPoolSize * 120 * 1000;
+    final int statusUpdateDelayMs = 120 * 1000;
+    Retryer retryer = new Retryer(10, maxRetryDelayMs, statusUpdateDelayMs);
+    boolean done;
+
+    do {
+      if (closed) {
+        throw new IOException("this stream is already closed");
+      }
+
+      data = bufferPool.acquire(blockNumber);
+      done = getInternal(data);
+
+      if (retryer.updateStatus()) {
+        LOG.warn("waiting to get block: {}", blockNumber);
+        LOG.info("state = {}", this.toString());
+      }
+    }
+    while (!done && retryer.continueRetry());
+
+    if (done) {
+      return data;
+    } else {
+      String message = String.format("Wait failed for get(%d)", blockNumber);
+      throw new IllegalStateException(message);
+    }
+  }
+
+  private boolean getInternal(BufferData data) throws IOException {
+    Validate.checkNotNull(data, "data");
+
+    // Opportunistic check without locking.
+    if (data.stateEqualsOneOf(
+        BufferData.State.PREFETCHING,
+        BufferData.State.CACHING,
+        BufferData.State.DONE)) {
+      return false;
+    }
+
+    synchronized (data) {
+      // Reconfirm state after locking.
+      if (data.stateEqualsOneOf(
+          BufferData.State.PREFETCHING,
+          BufferData.State.CACHING,
+          BufferData.State.DONE)) {
+        return false;
+      }
+
+      int blockNumber = data.getBlockNumber();
+      if (data.getState() == BufferData.State.READY) {
+        BlockOperations.Operation op = ops.getPrefetched(blockNumber);
+        ops.end(op);
+        return true;
+      }
+
+      data.throwIfStateIncorrect(BufferData.State.BLANK);
+      read(data);
+      return true;
+    }
+  }
+
+  /**
+   * Releases resources allocated to the given block.
+   *
+   * @throws IllegalArgumentException if data is null.
+   */
+  @Override
+  public void release(BufferData data) {
+    if (closed) {
+      return;
+    }
+
+    Validate.checkNotNull(data, "data");
+
+    BlockOperations.Operation op = ops.release(data.getBlockNumber());
+    bufferPool.release(data);
+    ops.end(op);
+  }
+
+  @Override
+  public synchronized void close() {
+    if (closed) {
+      return;
+    }
+
+    closed = true;
+
+    final BlockOperations.Operation op = ops.close();
+
+    // Cancel any prefetches in progress.
+    cancelPrefetches();
+
+    cleanupWithLogger(LOG, cache);
+
+    ops.end(op);
+    LOG.info(ops.getSummary(false));
+
+    bufferPool.close();
+    bufferPool = null;
+  }
+
+  /**
+   * Requests optional prefetching of the given block.
+   * The block is prefetched only if we can acquire a free buffer.
+   *
+   * @throws IllegalArgumentException if blockNumber is negative.
+   */
+  @Override
+  public void requestPrefetch(int blockNumber) {
+    checkNotNegative(blockNumber, "blockNumber");
+
+    if (closed) {
+      return;
+    }
+
+    // We initiate a prefetch only if we can acquire a buffer from the shared pool.
+    BufferData data = bufferPool.tryAcquire(blockNumber);
+    if (data == null) {
+      return;
+    }
+
+    // Opportunistic check without locking.
+    if (!data.stateEqualsOneOf(BufferData.State.BLANK)) {
+      // The block is ready or being prefetched/cached.
+      return;
+    }
+
+    synchronized (data) {
+      // Reconfirm state after locking.
+      if (!data.stateEqualsOneOf(BufferData.State.BLANK)) {
+        // The block is ready or being prefetched/cached.
+        return;
+      }
+
+      BlockOperations.Operation op = ops.requestPrefetch(blockNumber);
+      PrefetchTask prefetchTask = new PrefetchTask(data, this, Instant.now());
+      Future<Void> prefetchFuture = futurePool.executeFunction(prefetchTask);
+      data.setPrefetch(prefetchFuture);
+      ops.end(op);
+    }
+  }
+
+  /**
+   * Requests cancellation of any previously issued prefetch requests.
+   */
+  @Override
+  public void cancelPrefetches() {
+    BlockOperations.Operation op = ops.cancelPrefetches();
+
+    for (BufferData data : bufferPool.getAll()) {
+      // We add blocks being prefetched to the local cache so that the prefetch is not wasted.
+      if (data.stateEqualsOneOf(BufferData.State.PREFETCHING, BufferData.State.READY)) {
+        requestCaching(data);
+      }
+    }
+
+    ops.end(op);
+  }
+
+  private void read(BufferData data) throws IOException {
+    synchronized (data) {
+      readBlock(data, false, BufferData.State.BLANK);
+    }
+  }
+
+  private void prefetch(BufferData data, Instant taskQueuedStartTime) throws IOException {
+    synchronized (data) {
+      prefetchingStatistics.executorAcquired(
+          Duration.between(taskQueuedStartTime, Instant.now()));
+      readBlock(
+          data,
+          true,
+          BufferData.State.PREFETCHING,
+          BufferData.State.CACHING);
+    }
+  }
+
+  private void readBlock(BufferData data, boolean isPrefetch, BufferData.State... expectedState)
+      throws IOException {
+
+    if (closed) {
+      return;
+    }
+
+    BlockOperations.Operation op = null;
+    DurationTracker tracker = null;
+
+    synchronized (data) {
+      try {
+        if (data.stateEqualsOneOf(BufferData.State.DONE, BufferData.State.READY)) {
+          // DONE  : Block was released, likely due to caching being disabled on slow perf.
+          // READY : Block was already fetched by another thread. No need to re-read.
+          return;
+        }
+
+        data.throwIfStateIncorrect(expectedState);
+        int blockNumber = data.getBlockNumber();
+
+        // Prefer reading from cache over reading from network.
+        if (cache.containsBlock(blockNumber)) {
+          op = ops.getCached(blockNumber);
+          cache.get(blockNumber, data.getBuffer());
+          data.setReady(expectedState);
+          return;
+        }
+
+        if (isPrefetch) {
+          tracker = prefetchingStatistics.prefetchOperationStarted();
+          op = ops.prefetch(data.getBlockNumber());
+        } else {
+          op = ops.getRead(data.getBlockNumber());
+        }
+
+        long offset = getBlockData().getStartOffset(data.getBlockNumber());
+        int size = getBlockData().getSize(data.getBlockNumber());
+        ByteBuffer buffer = data.getBuffer();
+        buffer.clear();
+        read(buffer, offset, size);
+        buffer.flip();
+        data.setReady(expectedState);
+      } catch (Exception e) {
+        String message = String.format("error during readBlock(%s)", data.getBlockNumber());
+        LOG.error(message, e);
+
+        if (isPrefetch && tracker != null) {
+          tracker.failed();
+        }
+
+        numReadErrors.incrementAndGet();
+        data.setDone();
+        throw e;
+      } finally {
+        if (op != null) {
+          ops.end(op);
+        }
+
+        if (isPrefetch) {
+          prefetchingStatistics.prefetchOperationCompleted();
+          if (tracker != null) {
+            tracker.close();
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Read task that is submitted to the future pool.
+   */
+  private static class PrefetchTask implements Supplier<Void> {
+    private final BufferData data;
+    private final CachingBlockManager blockManager;
+    private final Instant taskQueuedStartTime;
+
+    PrefetchTask(BufferData data, CachingBlockManager blockManager, Instant taskQueuedStartTime) {
+      this.data = data;
+      this.blockManager = blockManager;
+      this.taskQueuedStartTime = taskQueuedStartTime;
+    }
+
+    @Override
+    public Void get() {
+      try {
+        blockManager.prefetch(data, taskQueuedStartTime);
+      } catch (Exception e) {
+        LOG.error("error during prefetch", e);
+      }
+      return null;
+    }
+  }
+
+  private static final BufferData.State[] EXPECTED_STATE_AT_CACHING =
+      new BufferData.State[] {
+          BufferData.State.PREFETCHING, BufferData.State.READY
+      };
+
+  /**
+   * Requests that the given block should be copied to the local cache.
+   * The block must not be accessed by the caller after calling this method
+   * because it will released asynchronously relative to the caller.
+   *
+   * @throws IllegalArgumentException if data is null.
+   */
+  @Override
+  public void requestCaching(BufferData data) {
+    if (closed) {
+      return;
+    }
+
+    if (cachingDisabled.get()) {
+      data.setDone();
+      return;
+    }
+
+    Validate.checkNotNull(data, "data");
+
+    // Opportunistic check without locking.
+    if (!data.stateEqualsOneOf(EXPECTED_STATE_AT_CACHING)) {
+      return;
+    }
+
+    synchronized (data) {
+      // Reconfirm state after locking.
+      if (!data.stateEqualsOneOf(EXPECTED_STATE_AT_CACHING)) {
+        return;
+      }
+
+      if (cache.containsBlock(data.getBlockNumber())) {
+        data.setDone();
+        return;
+      }
+
+      BufferData.State state = data.getState();
+
+      BlockOperations.Operation op = ops.requestCaching(data.getBlockNumber());
+      Future<Void> blockFuture;
+      if (state == BufferData.State.PREFETCHING) {
+        blockFuture = data.getActionFuture();
+      } else {
+        CompletableFuture<Void> cf = new CompletableFuture<>();
+        cf.complete(null);
+        blockFuture = cf;
+      }
+
+      CachePutTask task = new CachePutTask(data, blockFuture, this, Instant.now());
+      Future<Void> actionFuture = futurePool.executeFunction(task);
+      data.setCaching(actionFuture);
+      ops.end(op);
+    }
+  }
+
+  private void addToCacheAndRelease(BufferData data, Future<Void> blockFuture,
+      Instant taskQueuedStartTime) {
+    prefetchingStatistics.executorAcquired(
+        Duration.between(taskQueuedStartTime, Instant.now()));
+
+    if (closed) {
+      return;
+    }
+
+    if (cachingDisabled.get()) {
+      data.setDone();
+      return;
+    }
+
+    try {
+      blockFuture.get(TIMEOUT_MINUTES, TimeUnit.MINUTES);
+      if (data.stateEqualsOneOf(BufferData.State.DONE)) {
+        // There was an error during prefetch.
+        return;
+      }
+    } catch (Exception e) {
+      LOG.error("error waiting on blockFuture: {}", data, e);
+      data.setDone();
+      return;
+    }
+
+    if (cachingDisabled.get()) {
+      data.setDone();
+      return;
+    }
+
+    BlockOperations.Operation op = null;
+
+    synchronized (data) {
+      try {
+        if (data.stateEqualsOneOf(BufferData.State.DONE)) {
+          return;
+        }
+
+        if (cache.containsBlock(data.getBlockNumber())) {
+          data.setDone();
+          return;
+        }
+
+        op = ops.addToCache(data.getBlockNumber());
+        ByteBuffer buffer = data.getBuffer().duplicate();
+        buffer.rewind();
+        cachePut(data.getBlockNumber(), buffer);
+        data.setDone();
+      } catch (Exception e) {
+        numCachingErrors.incrementAndGet();
+        String message = String.format("error adding block to cache after wait: %s", data);
+        LOG.error(message, e);
+        data.setDone();
+      }
+
+      if (op != null) {
+        BlockOperations.End endOp = (BlockOperations.End) ops.end(op);
+        if (endOp.duration() > SLOW_CACHING_THRESHOLD) {
+          if (!cachingDisabled.getAndSet(true)) {
+            String message = String.format(
+                "Caching disabled because of slow operation (%.1f sec)", endOp.duration());
+            LOG.warn(message);
+          }
+        }
+      }
+    }
+  }
+
+  protected BlockCache createCache() {
+    return new SingleFilePerBlockCache(prefetchingStatistics);
+  }
+
+  protected void cachePut(int blockNumber, ByteBuffer buffer) throws IOException {
+    if (closed) {
+      return;
+    }
+
+    cache.put(blockNumber, buffer);
+  }
+
+  private static class CachePutTask implements Supplier<Void> {
+    private final BufferData data;
+
+    // Block being asynchronously fetched.
+    private final Future<Void> blockFuture;
+
+    // Block manager that manages this block.
+    private final CachingBlockManager blockManager;
+
+    private final Instant taskQueuedStartTime;
+
+    CachePutTask(
+        BufferData data,
+        Future<Void> blockFuture,
+        CachingBlockManager blockManager,
+        Instant taskQueuedStartTime) {
+      this.data = data;
+      this.blockFuture = blockFuture;
+      this.blockManager = blockManager;
+      this.taskQueuedStartTime = taskQueuedStartTime;
+    }
+
+    @Override
+    public Void get() {
+      blockManager.addToCacheAndRelease(data, blockFuture, taskQueuedStartTime);
+      return null;
+    }
+  }
+
+  /**
+   * Number of ByteBuffers available to be acquired.
+   *
+   * @return the number of available buffers.
+   */
+  public int numAvailable() {
+    return bufferPool.numAvailable();
+  }
+
+  /**
+   * Number of caching operations completed.
+   *
+   * @return the number of cached buffers.
+   */
+  public int numCached() {
+    return cache.size();
+  }
+
+  /**
+   * Number of errors encountered when caching.
+   *
+   * @return the number of errors encountered when caching.
+   */
+  public int numCachingErrors() {
+    return numCachingErrors.get();
+  }
+
+  /**
+   * Number of errors encountered when reading.
+   *
+   * @return the number of errors encountered when reading.
+   */
+  public int numReadErrors() {
+    return numReadErrors.get();
+  }
+
+  BufferData getData(int blockNumber) {
+    return bufferPool.tryAcquire(blockNumber);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+
+    sb.append("cache(");
+    sb.append(cache.toString());
+    sb.append("); ");
+
+    sb.append("pool: ");
+    sb.append(bufferPool.toString());
+
+    return sb.toString();
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java
new file mode 100644
index 0000000000000..177ff7abab8b7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.time.Duration;
+
+import org.apache.hadoop.fs.statistics.DurationTracker;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTracker;
+
+/**
+ * Empty implementation of the prefetching statistics interface.
+ */
+public final class EmptyPrefetchingStatistics
+    implements PrefetchingStatistics {
+
+  private static final EmptyPrefetchingStatistics
+      EMPTY_PREFETCHING_STATISTICS =
+      new EmptyPrefetchingStatistics();
+
+  private EmptyPrefetchingStatistics() {
+  }
+
+  public static EmptyPrefetchingStatistics getInstance() {
+    return EMPTY_PREFETCHING_STATISTICS;
+  }
+
+  @Override
+  public DurationTracker prefetchOperationStarted() {
+    return stubDurationTracker();
+  }
+
+  @Override
+  public void blockAddedToFileCache() {
+
+  }
+
+  @Override
+  public void blockRemovedFromFileCache() {
+
+  }
+
+  @Override
+  public void prefetchOperationCompleted() {
+
+  }
+
+  @Override
+  public void executorAcquired(Duration timeInQueue) {
+
+  }
+
+  @Override
+  public void memoryAllocated(int size) {
+
+  }
+
+  @Override
+  public void memoryFreed(int size) {
+
+  }
+}
+
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java
new file mode 100644
index 0000000000000..645de280394c6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.util.Locale;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.slf4j.Logger;
+
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+/**
+ * A FuturePool implementation backed by a java.util.concurrent.ExecutorService.
+ *
+ * If a piece of work has started, it cannot (currently) be cancelled.
+ *
+ * This class is a simplified version of <code>com.twitter:util-core_2.11</code>
+ * ExecutorServiceFuturePool designed to avoid depending on that Scala library.
+ * One problem with using a Scala library is that many downstream projects
+ * (eg Apache Spark) use Scala, and they might want to use a different version of Scala
+ * from the version that Hadoop chooses to use.
+ *
+ */
+public class ExecutorServiceFuturePool {
+
+  private final ExecutorService executor;
+
+  public ExecutorServiceFuturePool(ExecutorService executor) {
+    this.executor = executor;
+  }
+
+  /**
+   * @param f function to run in future on executor pool
+   * @return future
+   * @throws java.util.concurrent.RejectedExecutionException can be thrown
+   * @throws NullPointerException if f param is null
+   */
+  public Future<Void> executeFunction(final Supplier<Void> f) {
+    return executor.submit(f::get);
+  }
+
+  /**
+   * @param r runnable to run in future on executor pool
+   * @return future
+   * @throws java.util.concurrent.RejectedExecutionException can be thrown
+   * @throws NullPointerException if r param is null
+   */
+  @SuppressWarnings("unchecked")
+  public Future<Void> executeRunnable(final Runnable r) {
+    return (Future<Void>) executor.submit(r::run);
+  }
+
+  /**
+   * Utility to shutdown the {@link ExecutorService} used by this class. Will wait up to a
+   * certain timeout for the ExecutorService to gracefully shutdown.
+   *
+   * @param logger Logger
+   * @param timeout the maximum time to wait
+   * @param unit the time unit of the timeout argument
+   */
+  public void shutdown(Logger logger, long timeout, TimeUnit unit) {
+    HadoopExecutors.shutdown(executor, logger, timeout, unit);
+  }
+
+  public String toString() {
+    return String.format(Locale.ROOT, "ExecutorServiceFuturePool(executor=%s)", executor);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java
new file mode 100644
index 0000000000000..286bdd7ae8996
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkState;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkWithinRange;
+
+/**
+ * Provides functionality related to tracking the position within a file.
+ *
+ * The file is accessed through an in memory buffer. The absolute position within
+ * the file is the sum of start offset of the buffer within the file and the relative
+ * offset of the current access location within the buffer.
+ *
+ * A file is made up of equal sized blocks. The last block may be of a smaller size.
+ * The size of a buffer associated with this file is typically the same as block size.
+ */
+public final class FilePosition {
+
+  /**
+   * Holds block based information about a file.
+   */
+  private BlockData blockData;
+
+  /**
+   * Information about the buffer in use.
+   */
+  private BufferData data;
+
+  /**
+   * Provides access to the underlying file.
+   */
+  private ByteBuffer buffer;
+
+  /**
+   * Start offset of the buffer relative to the start of a file.
+   */
+  private long bufferStartOffset;
+
+  /**
+   * Offset where reading starts relative to the start of a file.
+   */
+  private long readStartOffset;
+
+  // Read stats after a seek (mostly for debugging use).
+  private int numSingleByteReads;
+
+  private int numBytesRead;
+
+  private int numBufferReads;
+
+  /**
+   * Constructs an instance of {@link FilePosition}.
+   *
+   * @param fileSize size of the associated file.
+   * @param blockSize size of each block within the file.
+   *
+   * @throws IllegalArgumentException if fileSize is negative.
+   * @throws IllegalArgumentException if blockSize is zero or negative.
+   */
+  public FilePosition(long fileSize, int blockSize) {
+    checkNotNegative(fileSize, "fileSize");
+    if (fileSize == 0) {
+      checkNotNegative(blockSize, "blockSize");
+    } else {
+      checkPositiveInteger(blockSize, "blockSize");
+    }
+
+    this.blockData = new BlockData(fileSize, blockSize);
+
+    // The position is valid only when a valid buffer is associated with this file.
+    this.invalidate();
+  }
+
+  /**
+   * Associates a buffer with this file.
+   *
+   * @param bufferData the buffer associated with this file.
+   * @param startOffset Start offset of the buffer relative to the start of a file.
+   * @param readOffset Offset where reading starts relative to the start of a file.
+   *
+   * @throws IllegalArgumentException if bufferData is null.
+   * @throws IllegalArgumentException if startOffset is negative.
+   * @throws IllegalArgumentException if readOffset is negative.
+   * @throws IllegalArgumentException if readOffset is outside the range [startOffset, buffer end].
+   */
+  public void setData(BufferData bufferData,
+      long startOffset,
+      long readOffset) {
+    checkNotNull(bufferData, "bufferData");
+    checkNotNegative(startOffset, "startOffset");
+    checkNotNegative(readOffset, "readOffset");
+    checkWithinRange(
+        readOffset,
+        "readOffset",
+        startOffset,
+        startOffset + bufferData.getBuffer().limit());
+
+    data = bufferData;
+    buffer = bufferData.getBuffer().duplicate();
+    bufferStartOffset = startOffset;
+    readStartOffset = readOffset;
+    setAbsolute(readOffset);
+
+    resetReadStats();
+  }
+
+  public ByteBuffer buffer() {
+    throwIfInvalidBuffer();
+    return buffer;
+  }
+
+  public BufferData data() {
+    throwIfInvalidBuffer();
+    return data;
+  }
+
+  /**
+   * Gets the current absolute position within this file.
+   *
+   * @return the current absolute position within this file.
+   */
+  public long absolute() {
+    throwIfInvalidBuffer();
+    return bufferStartOffset + relative();
+  }
+
+  /**
+   * If the given {@code pos} lies within the current buffer, updates the current position to
+   * the specified value and returns true; otherwise returns false without changing the position.
+   *
+   * @param pos the absolute position to change the current position to if possible.
+   * @return true if the given current position was updated, false otherwise.
+   */
+  public boolean setAbsolute(long pos) {
+    if (isValid() && isWithinCurrentBuffer(pos)) {
+      int relativePos = (int) (pos - bufferStartOffset);
+      buffer.position(relativePos);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  /**
+   * Gets the current position within this file relative to the start of the associated buffer.
+   *
+   * @return the current position within this file relative to the start of the associated buffer.
+   */
+  public int relative() {
+    throwIfInvalidBuffer();
+    return buffer.position();
+  }
+
+  /**
+   * Determines whether the given absolute position lies within the current buffer.
+   *
+   * @param pos the position to check.
+   * @return true if the given absolute position lies within the current buffer, false otherwise.
+   */
+  public boolean isWithinCurrentBuffer(long pos) {
+    throwIfInvalidBuffer();
+    long bufferEndOffset = bufferStartOffset + buffer.limit();
+    return (pos >= bufferStartOffset) && (pos <= bufferEndOffset);
+  }
+
+  /**
+   * Gets the id of the current block.
+   *
+   * @return the id of the current block.
+   */
+  public int blockNumber() {
+    throwIfInvalidBuffer();
+    return blockData.getBlockNumber(bufferStartOffset);
+  }
+
+  /**
+   * Determines whether the current block is the last block in this file.
+   *
+   * @return true if the current block is the last block in this file, false otherwise.
+   */
+  public boolean isLastBlock() {
+    return blockData.isLastBlock(blockNumber());
+  }
+
+  /**
+   * Determines if the current position is valid.
+   *
+   * @return true if the current position is valid, false otherwise.
+   */
+  public boolean isValid() {
+    return buffer != null;
+  }
+
+  /**
+   * Marks the current position as invalid.
+   */
+  public void invalidate() {
+    buffer = null;
+    bufferStartOffset = -1;
+    data = null;
+  }
+
+  /**
+   * Gets the start of the current block's absolute offset.
+   *
+   * @return the start of the current block's absolute offset.
+   */
+  public long bufferStartOffset() {
+    throwIfInvalidBuffer();
+    return bufferStartOffset;
+  }
+
+  /**
+   * Determines whether the current buffer has been fully read.
+   *
+   * @return true if the current buffer has been fully read, false otherwise.
+   */
+  public boolean bufferFullyRead() {
+    throwIfInvalidBuffer();
+    return (bufferStartOffset == readStartOffset)
+        && (relative() == buffer.limit())
+        && (numBytesRead == buffer.limit());
+  }
+
+  public void incrementBytesRead(int n) {
+    numBytesRead += n;
+    if (n == 1) {
+      numSingleByteReads++;
+    } else {
+      numBufferReads++;
+    }
+  }
+
+  public int numBytesRead() {
+    return numBytesRead;
+  }
+
+  public int numSingleByteReads() {
+    return numSingleByteReads;
+  }
+
+  public int numBufferReads() {
+    return numBufferReads;
+  }
+
+  private void resetReadStats() {
+    numBytesRead = 0;
+    numSingleByteReads = 0;
+    numBufferReads = 0;
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    if (buffer == null) {
+      sb.append("currentBuffer = null");
+    } else {
+      int pos = buffer.position();
+      int val;
+      if (pos >= buffer.limit()) {
+        val = -1;
+      } else {
+        val = buffer.get(pos);
+      }
+      String currentBufferState =
+          String.format("%d at pos: %d, lim: %d", val, pos, buffer.limit());
+      sb.append(String.format(
+          "block: %d, pos: %d (CBuf: %s)%n",
+          blockNumber(), absolute(),
+          currentBufferState));
+      sb.append("\n");
+    }
+    return sb.toString();
+  }
+
+  private void throwIfInvalidBuffer() {
+    checkState(buffer != null, "'buffer' must not be null");
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java
new file mode 100644
index 0000000000000..9ce2dec5889f1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java
@@ -0,0 +1,67 @@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements. See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership. The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License. You may obtain a copy of the License at
+  *
+  *   http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied. See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.time.Duration;
+
+import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+
+public interface PrefetchingStatistics extends IOStatisticsSource {
+
+  /**
+   * A prefetch operation has started.
+   * @return duration tracker
+   */
+  DurationTracker prefetchOperationStarted();
+
+  /**
+   * A block has been saved to the file cache.
+   */
+  void blockAddedToFileCache();
+
+  /**
+   * A block has been removed from the file cache.
+   */
+  void blockRemovedFromFileCache();
+
+  /**
+   * A prefetch operation has completed.
+   */
+  void prefetchOperationCompleted();
+
+  /**
+   * An executor has been acquired, either for prefetching or caching.
+   * @param timeInQueue time taken to acquire an executor.
+   */
+  void executorAcquired(Duration timeInQueue);
+
+  /**
+   * A new buffer has been added to the buffer pool.
+   * @param size size of the new buffer
+   */
+  void memoryAllocated(int size);
+
+  /**
+   * Previously allocated memory has been freed.
+   * @param size size of memory freed.
+   */
+  void memoryFreed(int size);
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java
new file mode 100644
index 0000000000000..77e00972d08c3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.Closeable;
+
+/**
+ * Manages a fixed pool of resources.
+ *
+ * Avoids creating a new resource if a previously created instance is already available.
+ */
+public abstract class ResourcePool<T> implements Closeable {
+
+  /**
+   * Acquires a resource blocking if necessary until one becomes available.
+   *
+   * @return the acquired resource instance.
+   */
+  public abstract T acquire();
+
+  /**
+   * Acquires a resource blocking if one is immediately available. Otherwise returns null.
+
+   * @return the acquired resource instance (if immediately available) or null.
+   */
+  public abstract T tryAcquire();
+
+  /**
+   * Releases a previously acquired resource.
+   *
+   * @param item the resource to release.
+   */
+  public abstract void release(T item);
+
+  @Override
+  public void close() {
+  }
+
+  /**
+   * Derived classes may implement a way to cleanup each item.
+   *
+   * @param item the resource to close.
+   */
+  protected void close(T item) {
+    // Do nothing in this class. Allow overriding classes to take any cleanup action.
+  }
+
+  /**
+   * Derived classes must implement a way to create an instance of a resource.
+   *
+   * @return the created instance.
+   */
+  protected abstract T createNew();
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java
new file mode 100644
index 0000000000000..84c17ef9dde8a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkGreater;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger;
+
+/**
+ * Provides retry related functionality.
+ */
+public class Retryer {
+
+  /* Maximum amount of delay (in ms) before retry fails. */
+  private int maxDelay;
+
+  /* Per retry delay (in ms). */
+  private int perRetryDelay;
+
+  /**
+   * The time interval (in ms) at which status update would be made.
+   */
+  private int statusUpdateInterval;
+
+  /* Current delay. */
+  private int delay;
+
+  /**
+   * Initializes a new instance of the {@code Retryer} class.
+   *
+   * @param perRetryDelay per retry delay (in ms).
+   * @param maxDelay maximum amount of delay (in ms) before retry fails.
+   * @param statusUpdateInterval time interval (in ms) at which status update would be made.
+   *
+   * @throws IllegalArgumentException if perRetryDelay is zero or negative.
+   * @throws IllegalArgumentException if maxDelay is less than or equal to perRetryDelay.
+   * @throws IllegalArgumentException if statusUpdateInterval is zero or negative.
+   */
+  public Retryer(int perRetryDelay, int maxDelay, int statusUpdateInterval) {
+    checkPositiveInteger(perRetryDelay, "perRetryDelay");
+    checkGreater(maxDelay, "maxDelay", perRetryDelay, "perRetryDelay");
+    checkPositiveInteger(statusUpdateInterval, "statusUpdateInterval");
+
+    this.perRetryDelay = perRetryDelay;
+    this.maxDelay = maxDelay;
+    this.statusUpdateInterval = statusUpdateInterval;
+  }
+
+  /**
+   * Returns true if retrying should continue, false otherwise.
+   *
+   * @return true if the caller should retry, false otherwise.
+   */
+  public boolean continueRetry() {
+    if (this.delay >= this.maxDelay) {
+      return false;
+    }
+
+    try {
+      Thread.sleep(this.perRetryDelay);
+    } catch (InterruptedException e) {
+      // Ignore the exception as required by the semantic of this class;
+    }
+
+    this.delay += this.perRetryDelay;
+    return true;
+  }
+
+  /**
+   * Returns true if status update interval has been reached.
+   *
+   * @return true if status update interval has been reached.
+   */
+  public boolean updateStatus() {
+    return (this.delay > 0) && this.delay % this.statusUpdateInterval == 0;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java
new file mode 100644
index 0000000000000..c84335a763e87
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull;
+
+/**
+ * Provides functionality necessary for caching blocks of data read from FileSystem.
+ * Each cache block is stored on the local disk as a separate file.
+ */
+public class SingleFilePerBlockCache implements BlockCache {
+  private static final Logger LOG = LoggerFactory.getLogger(SingleFilePerBlockCache.class);
+
+  /**
+   * Blocks stored in this cache.
+   */
+  private final Map<Integer, Entry> blocks = new ConcurrentHashMap<>();
+
+  /**
+   * Number of times a block was read from this cache.
+   * Used for determining cache utilization factor.
+   */
+  private int numGets = 0;
+
+  private boolean closed;
+
+  private final PrefetchingStatistics prefetchingStatistics;
+
+  /**
+   * Cache entry.
+   * Each block is stored as a separate file.
+   */
+  private static final class Entry {
+    private final int blockNumber;
+    private final Path path;
+    private final int size;
+    private final long checksum;
+
+    Entry(int blockNumber, Path path, int size, long checksum) {
+      this.blockNumber = blockNumber;
+      this.path = path;
+      this.size = size;
+      this.checksum = checksum;
+    }
+
+    @Override
+    public String toString() {
+      return String.format(
+          "([%03d] %s: size = %d, checksum = %d)",
+          blockNumber, path, size, checksum);
+    }
+  }
+
+  /**
+   * Constructs an instance of a {@code SingleFilePerBlockCache}.
+   *
+   * @param prefetchingStatistics statistics for this stream.
+   */
+  public SingleFilePerBlockCache(PrefetchingStatistics prefetchingStatistics) {
+    this.prefetchingStatistics = requireNonNull(prefetchingStatistics);
+  }
+
+  /**
+   * Indicates whether the given block is in this cache.
+   */
+  @Override
+  public boolean containsBlock(int blockNumber) {
+    return blocks.containsKey(blockNumber);
+  }
+
+  /**
+   * Gets the blocks in this cache.
+   */
+  @Override
+  public Iterable<Integer> blocks() {
+    return Collections.unmodifiableList(new ArrayList<>(blocks.keySet()));
+  }
+
+  /**
+   * Gets the number of blocks in this cache.
+   */
+  @Override
+  public int size() {
+    return blocks.size();
+  }
+
+  /**
+   * Gets the block having the given {@code blockNumber}.
+   *
+   * @throws IllegalArgumentException if buffer is null.
+   */
+  @Override
+  public void get(int blockNumber, ByteBuffer buffer) throws IOException {
+    if (closed) {
+      return;
+    }
+
+    checkNotNull(buffer, "buffer");
+
+    Entry entry = getEntry(blockNumber);
+    buffer.clear();
+    readFile(entry.path, buffer);
+    buffer.rewind();
+
+    validateEntry(entry, buffer);
+  }
+
+  protected int readFile(Path path, ByteBuffer buffer) throws IOException {
+    int numBytesRead = 0;
+    int numBytes;
+    FileChannel channel = FileChannel.open(path, StandardOpenOption.READ);
+    while ((numBytes = channel.read(buffer)) > 0) {
+      numBytesRead += numBytes;
+    }
+    buffer.limit(buffer.position());
+    channel.close();
+    return numBytesRead;
+  }
+
+  private Entry getEntry(int blockNumber) {
+    Validate.checkNotNegative(blockNumber, "blockNumber");
+
+    Entry entry = blocks.get(blockNumber);
+    if (entry == null) {
+      throw new IllegalStateException(String.format("block %d not found in cache", blockNumber));
+    }
+    numGets++;
+    return entry;
+  }
+
+  /**
+   * Puts the given block in this cache.
+   *
+   * @throws IllegalArgumentException if buffer is null.
+   * @throws IllegalArgumentException if buffer.limit() is zero or negative.
+   */
+  @Override
+  public void put(int blockNumber, ByteBuffer buffer) throws IOException {
+    if (closed) {
+      return;
+    }
+
+    checkNotNull(buffer, "buffer");
+
+    if (blocks.containsKey(blockNumber)) {
+      Entry entry = blocks.get(blockNumber);
+      validateEntry(entry, buffer);
+      return;
+    }
+
+    Validate.checkPositiveInteger(buffer.limit(), "buffer.limit()");
+
+    Path blockFilePath = getCacheFilePath();
+    long size = Files.size(blockFilePath);
+    if (size != 0) {
+      String message =
+          String.format("[%d] temp file already has data. %s (%d)",
+          blockNumber, blockFilePath, size);
+      throw new IllegalStateException(message);
+    }
+
+    writeFile(blockFilePath, buffer);
+    prefetchingStatistics.blockAddedToFileCache();
+    long checksum = BufferData.getChecksum(buffer);
+    Entry entry = new Entry(blockNumber, blockFilePath, buffer.limit(), checksum);
+    blocks.put(blockNumber, entry);
+  }
+
+  private static final Set<? extends OpenOption> CREATE_OPTIONS =
+      EnumSet.of(StandardOpenOption.WRITE,
+          StandardOpenOption.CREATE,
+          StandardOpenOption.TRUNCATE_EXISTING);
+
+  protected void writeFile(Path path, ByteBuffer buffer) throws IOException {
+    buffer.rewind();
+    WritableByteChannel writeChannel = Files.newByteChannel(path, CREATE_OPTIONS);
+    while (buffer.hasRemaining()) {
+      writeChannel.write(buffer);
+    }
+    writeChannel.close();
+  }
+
+  protected Path getCacheFilePath() throws IOException {
+    return getTempFilePath();
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (closed) {
+      return;
+    }
+
+    closed = true;
+
+    LOG.info(getStats());
+    int numFilesDeleted = 0;
+
+    for (Entry entry : blocks.values()) {
+      try {
+        Files.deleteIfExists(entry.path);
+        prefetchingStatistics.blockRemovedFromFileCache();
+        numFilesDeleted++;
+      } catch (IOException e) {
+        // Ignore while closing so that we can delete as many cache files as possible.
+      }
+    }
+
+    if (numFilesDeleted > 0) {
+      LOG.info("Deleted {} cache files", numFilesDeleted);
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("stats: ");
+    sb.append(getStats());
+    sb.append(", blocks:[");
+    sb.append(getIntList(blocks()));
+    sb.append("]");
+    return sb.toString();
+  }
+
+  private void validateEntry(Entry entry, ByteBuffer buffer) {
+    if (entry.size != buffer.limit()) {
+      String message = String.format(
+          "[%d] entry.size(%d) != buffer.limit(%d)",
+          entry.blockNumber, entry.size, buffer.limit());
+      throw new IllegalStateException(message);
+    }
+
+    long checksum = BufferData.getChecksum(buffer);
+    if (entry.checksum != checksum) {
+      String message = String.format(
+          "[%d] entry.checksum(%d) != buffer checksum(%d)",
+          entry.blockNumber, entry.checksum, checksum);
+      throw new IllegalStateException(message);
+    }
+  }
+
+  /**
+   * Produces a human readable list of blocks for the purpose of logging.
+   * This method minimizes the length of returned list by converting
+   * a contiguous list of blocks into a range.
+   * for example,
+   * 1, 3, 4, 5, 6, 8 becomes 1, 3~6, 8
+   */
+  private String getIntList(Iterable<Integer> nums) {
+    List<String> numList = new ArrayList<>();
+    List<Integer> numbers = new ArrayList<Integer>();
+    for (Integer n : nums) {
+      numbers.add(n);
+    }
+    Collections.sort(numbers);
+
+    int index = 0;
+    while (index < numbers.size()) {
+      int start = numbers.get(index);
+      int prev = start;
+      int end = start;
+      while ((++index < numbers.size()) && ((end = numbers.get(index)) == prev + 1)) {
+        prev = end;
+      }
+
+      if (start == prev) {
+        numList.add(Integer.toString(start));
+      } else {
+        numList.add(String.format("%d~%d", start, prev));
+      }
+    }
+
+    return String.join(", ", numList);
+  }
+
+  private String getStats() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(String.format(
+        "#entries = %d, #gets = %d",
+        blocks.size(), numGets));
+    return sb.toString();
+  }
+
+  private static final String CACHE_FILE_PREFIX = "fs-cache-";
+
+  public static boolean isCacheSpaceAvailable(long fileSize) {
+    try {
+      Path cacheFilePath = getTempFilePath();
+      long freeSpace = new File(cacheFilePath.toString()).getUsableSpace();
+      LOG.info("fileSize = {}, freeSpace = {}", fileSize, freeSpace);
+      Files.deleteIfExists(cacheFilePath);
+      return fileSize < freeSpace;
+    } catch (IOException e) {
+      LOG.error("isCacheSpaceAvailable", e);
+      return false;
+    }
+  }
+
+  // The suffix (file extension) of each serialized index file.
+  private static final String BINARY_FILE_SUFFIX = ".bin";
+
+  // File attributes attached to any intermediate temporary file created during index creation.
+  private static final FileAttribute<Set<PosixFilePermission>> TEMP_FILE_ATTRS =
+      PosixFilePermissions.asFileAttribute(EnumSet.of(PosixFilePermission.OWNER_READ,
+          PosixFilePermission.OWNER_WRITE));
+
+  private static Path getTempFilePath() throws IOException {
+    return Files.createTempFile(
+        CACHE_FILE_PREFIX,
+        BINARY_FILE_SUFFIX,
+        TEMP_FILE_ATTRS
+    );
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java
new file mode 100644
index 0000000000000..17a668a0d3bc3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java
@@ -0,0 +1,399 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
+/**
+ * A superset of Validate class in Apache commons lang3.
+ * <p>
+ * It provides consistent message strings for frequently encountered checks.
+ * That simplifies callers because they have to supply only the name of the argument
+ * that failed a check instead of having to supply the entire message.
+ */
+public final class Validate {
+
+  private Validate() {
+  }
+
+  /**
+   * Validates that the given reference argument is not null.
+   * @param obj the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNull(Object obj, String argName) {
+    checkArgument(obj != null, "'%s' must not be null.", argName);
+  }
+
+  /**
+   * Validates that the given integer argument is not zero or negative.
+   * @param value the argument value to validate
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkPositiveInteger(long value, String argName) {
+    checkArgument(value > 0, "'%s' must be a positive integer.", argName);
+  }
+
+  /**
+   * Validates that the given integer argument is not negative.
+   * @param value the argument value to validate
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNegative(long value, String argName) {
+    checkArgument(value >= 0, "'%s' must not be negative.", argName);
+  }
+
+  /**
+   * Validates that the expression (that checks a required field is present) is true.
+   * @param isPresent indicates whether the given argument is present.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkRequired(boolean isPresent, String argName) {
+    checkArgument(isPresent, "'%s' is required.", argName);
+  }
+
+  /**
+   * Validates that the expression (that checks a field is valid) is true.
+   * @param isValid indicates whether the given argument is valid.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkValid(boolean isValid, String argName) {
+    checkArgument(isValid, "'%s' is invalid.", argName);
+  }
+
+  /**
+   * Validates that the expression (that checks a field is valid) is true.
+   * @param isValid indicates whether the given argument is valid.
+   * @param argName the name of the argument being validated.
+   * @param validValues the list of values that are allowed.
+   */
+  public static void checkValid(boolean isValid,
+      String argName,
+      String validValues) {
+    checkArgument(isValid, "'%s' is invalid. Valid values are: %s.", argName,
+        validValues);
+  }
+
+  /**
+   * Validates that the given string is not null and has non-zero length.
+   * @param arg the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNullAndNotEmpty(String arg, String argName) {
+    checkNotNull(arg, argName);
+    checkArgument(
+        !arg.isEmpty(),
+        "'%s' must not be empty.",
+        argName);
+  }
+
+  /**
+   * Validates that the given array is not null and has at least one element.
+   * @param <T> the type of array's elements.
+   * @param array the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static <T> void checkNotNullAndNotEmpty(T[] array, String argName) {
+    checkNotNull(array, argName);
+    checkNotEmpty(array.length, argName);
+  }
+
+  /**
+   * Validates that the given array is not null and has at least one element.
+   * @param array the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNullAndNotEmpty(byte[] array, String argName) {
+    checkNotNull(array, argName);
+    checkNotEmpty(array.length, argName);
+  }
+
+  /**
+   * Validates that the given array is not null and has at least one element.
+   * @param array the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNullAndNotEmpty(short[] array, String argName) {
+    checkNotNull(array, argName);
+    checkNotEmpty(array.length, argName);
+  }
+
+  /**
+   * Validates that the given array is not null and has at least one element.
+   * @param array the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNullAndNotEmpty(int[] array, String argName) {
+    checkNotNull(array, argName);
+    checkNotEmpty(array.length, argName);
+  }
+
+  /**
+   * Validates that the given array is not null and has at least one element.
+   * @param array the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkNotNullAndNotEmpty(long[] array, String argName) {
+    checkNotNull(array, argName);
+    checkNotEmpty(array.length, argName);
+  }
+
+  /**
+   * Validates that the given buffer is not null and has non-zero capacity.
+   * @param <T> the type of iterable's elements.
+   * @param iter the argument reference to validate.
+   * @param argName the name of the argument being validated.
+   */
+  public static <T> void checkNotNullAndNotEmpty(Iterable<T> iter,
+      String argName) {
+    checkNotNull(iter, argName);
+    int minNumElements = iter.iterator().hasNext() ? 1 : 0;
+    checkNotEmpty(minNumElements, argName);
+  }
+
+  /**
+   * Validates that the given set is not null and has an exact number of items.
+   * @param <T> the type of collection's elements.
+   * @param collection the argument reference to validate.
+   * @param numElements the expected number of elements in the collection.
+   * @param argName the name of the argument being validated.
+   */
+  public static <T> void checkNotNullAndNumberOfElements(
+      Collection<T> collection, int numElements, String argName) {
+    checkNotNull(collection, argName);
+    checkArgument(
+        collection.size() == numElements,
+        "Number of elements in '%s' must be exactly %s, %s given.",
+        argName,
+        numElements,
+        collection.size()
+    );
+  }
+
+  /**
+   * Validates that the given two values are equal.
+   * @param value1 the first value to check.
+   * @param value1Name the name of the first argument.
+   * @param value2 the second value to check.
+   * @param value2Name the name of the second argument.
+   */
+  public static void checkValuesEqual(
+      long value1,
+      String value1Name,
+      long value2,
+      String value2Name) {
+    checkArgument(
+        value1 == value2,
+        "'%s' (%s) must equal '%s' (%s).",
+        value1Name,
+        value1,
+        value2Name,
+        value2);
+  }
+
+  /**
+   * Validates that the first value is an integer multiple of the second value.
+   * @param value1 the first value to check.
+   * @param value1Name the name of the first argument.
+   * @param value2 the second value to check.
+   * @param value2Name the name of the second argument.
+   */
+  public static void checkIntegerMultiple(
+      long value1,
+      String value1Name,
+      long value2,
+      String value2Name) {
+    checkArgument(
+        (value1 % value2) == 0,
+        "'%s' (%s) must be an integer multiple of '%s' (%s).",
+        value1Name,
+        value1,
+        value2Name,
+        value2);
+  }
+
+  /**
+   * Validates that the first value is greater than the second value.
+   * @param value1 the first value to check.
+   * @param value1Name the name of the first argument.
+   * @param value2 the second value to check.
+   * @param value2Name the name of the second argument.
+   */
+  public static void checkGreater(
+      long value1,
+      String value1Name,
+      long value2,
+      String value2Name) {
+    checkArgument(
+        value1 > value2,
+        "'%s' (%s) must be greater than '%s' (%s).",
+        value1Name,
+        value1,
+        value2Name,
+        value2);
+  }
+
+  /**
+   * Validates that the first value is greater than or equal to the second value.
+   * @param value1 the first value to check.
+   * @param value1Name the name of the first argument.
+   * @param value2 the second value to check.
+   * @param value2Name the name of the second argument.
+   */
+  public static void checkGreaterOrEqual(
+      long value1,
+      String value1Name,
+      long value2,
+      String value2Name) {
+    checkArgument(
+        value1 >= value2,
+        "'%s' (%s) must be greater than or equal to '%s' (%s).",
+        value1Name,
+        value1,
+        value2Name,
+        value2);
+  }
+
+  /**
+   * Validates that the first value is less than or equal to the second value.
+   * @param value1 the first value to check.
+   * @param value1Name the name of the first argument.
+   * @param value2 the second value to check.
+   * @param value2Name the name of the second argument.
+   */
+  public static void checkLessOrEqual(
+      long value1,
+      String value1Name,
+      long value2,
+      String value2Name) {
+    checkArgument(
+        value1 <= value2,
+        "'%s' (%s) must be less than or equal to '%s' (%s).",
+        value1Name,
+        value1,
+        value2Name,
+        value2);
+  }
+
+  /**
+   * Validates that the given value is within the given range of values.
+   * @param value the value to check.
+   * @param valueName the name of the argument.
+   * @param minValueInclusive inclusive lower limit for the value.
+   * @param maxValueInclusive inclusive upper limit for the value.
+   */
+  public static void checkWithinRange(
+      long value,
+      String valueName,
+      long minValueInclusive,
+      long maxValueInclusive) {
+    checkArgument(
+        (value >= minValueInclusive) && (value <= maxValueInclusive),
+        "'%s' (%s) must be within the range [%s, %s].",
+        valueName,
+        value,
+        minValueInclusive,
+        maxValueInclusive);
+  }
+
+  /**
+   * Validates that the given value is within the given range of values.
+   * @param value the value to check.
+   * @param valueName the name of the argument.
+   * @param minValueInclusive inclusive lower limit for the value.
+   * @param maxValueInclusive inclusive upper limit for the value.
+   */
+  public static void checkWithinRange(
+      double value,
+      String valueName,
+      double minValueInclusive,
+      double maxValueInclusive) {
+    checkArgument(
+        (value >= minValueInclusive) && (value <= maxValueInclusive),
+        "'%s' (%s) must be within the range [%s, %s].",
+        valueName,
+        value,
+        minValueInclusive,
+        maxValueInclusive);
+  }
+
+  /**
+   * Validates that the given path exists.
+   * @param path the path to check.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkPathExists(Path path, String argName) {
+    checkNotNull(path, argName);
+    checkArgument(Files.exists(path), "Path %s (%s) does not exist.", argName,
+        path);
+  }
+
+  /**
+   * Validates that the given path exists and is a directory.
+   * @param path the path to check.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkPathExistsAsDir(Path path, String argName) {
+    checkPathExists(path, argName);
+    checkArgument(
+        Files.isDirectory(path),
+        "Path %s (%s) must point to a directory.",
+        argName,
+        path);
+  }
+
+  /**
+   * Validates that the given path exists and is a file.
+   * @param path the path to check.
+   * @param argName the name of the argument being validated.
+   */
+  public static void checkPathExistsAsFile(Path path, String argName) {
+    checkPathExists(path, argName);
+    checkArgument(Files.isRegularFile(path),
+        "Path %s (%s) must point to a file.", argName, path);
+  }
+
+
+  /**
+   * Check state.
+   * @param expression expression which must hold.
+   * @param format format string
+   * @param args arguments for the error string
+   * @throws IllegalStateException if the state is not valid.
+   */
+  public static void checkState(boolean expression,
+      String format,
+      Object... args) {
+    if (!expression) {
+      throw new IllegalStateException(String.format(format, args));
+    }
+  }
+
+  private static void checkNotEmpty(int arraySize, String argName) {
+    checkArgument(
+        arraySize > 0,
+        "'%s' must have at least one element.",
+        argName);
+  }
+}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java
similarity index 74%
rename from hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java
rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java
index 16c9da25777d2..1b26da85d95fb 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java
@@ -15,17 +15,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 
 /**
- * A path filter that accepts everything
+ * block caching for use in object store clients.
  */
-public class AcceptAllFilter implements PathFilter {
-  @Override
-  public boolean accept(Path file) {
-    return true;
-  }
-}
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
index 25b9ba659048a..260ee7e570c9b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
@@ -185,7 +185,8 @@ public Builder stickyBit(boolean stickyBit) {
 
     /**
      * Sets the permission for the file.
-     * @param permission
+     * @param permission permission.
+     * @return Builder.
      */
     public Builder setPermission(FsPermission permission) {
       this.permission = permission;
@@ -224,6 +225,7 @@ private AclStatus(String owner, String group, boolean stickyBit,
   /**
    * Get the effective permission for the AclEntry
    * @param entry AclEntry to get the effective action
+   * @return FsAction.
    */
   public FsAction getEffectivePermission(AclEntry entry) {
     return getEffectivePermission(entry, permission);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
index 97dcf816c16ad..746e0e1e238f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
@@ -48,7 +48,8 @@ private FsAction(String s) {
 
   /**
    * Return true if this action implies that action.
-   * @param that
+   * @param that FsAction that.
+   * @return if implies true,not false.
    */
   public boolean implies(FsAction that) {
     if (that != null) {
@@ -57,15 +58,26 @@ public boolean implies(FsAction that) {
     return false;
   }
 
-  /** AND operation. */
+  /**
+   * AND operation.
+   * @param that FsAction that.
+   * @return FsAction.
+   */
   public FsAction and(FsAction that) {
     return vals[ordinal() & that.ordinal()];
   }
-  /** OR operation. */
+  /**
+   * OR operation.
+   * @param that FsAction that.
+   * @return FsAction.
+   */
   public FsAction or(FsAction that) {
     return vals[ordinal() | that.ordinal()];
   }
-  /** NOT operation. */
+  /**
+   * NOT operation.
+   * @return FsAction.
+   */
   public FsAction not() {
     return vals[7 - ordinal()];
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
index 2bd6f1f3b9126..ff3b4f6d65a49 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -35,7 +35,10 @@ public final class FsCreateModes extends FsPermission {
   /**
    * Create from unmasked mode and umask.
    *
-   * If the mode is already an FsCreateModes object, return it.
+   * @param mode mode.
+   * @param umask umask.
+   * @return If the mode is already
+   * an FsCreateModes object, return it.
    */
   public static FsPermission applyUMask(FsPermission mode,
                                         FsPermission umask) {
@@ -47,6 +50,10 @@ public static FsPermission applyUMask(FsPermission mode,
 
   /**
    * Create from masked and unmasked modes.
+   *
+   * @param masked masked.
+   * @param unmasked unmasked.
+   * @return FsCreateModes.
    */
   public static FsCreateModes create(FsPermission masked,
                                      FsPermission unmasked) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 51c113af2702e..33fed1d303990 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -56,7 +56,11 @@ public class FsPermission implements Writable, Serializable,
   /** Maximum acceptable length of a permission string to parse */
   public static final int MAX_PERMISSION_LENGTH = 10;
 
-  /** Create an immutable {@link FsPermission} object. */
+  /**
+   * Create an immutable {@link FsPermission} object.
+   * @param permission permission.
+   * @return FsPermission.
+   */
   public static FsPermission createImmutable(short permission) {
     return new ImmutableFsPermission(permission);
   }
@@ -85,7 +89,7 @@ public FsPermission(FsAction u, FsAction g, FsAction o, boolean sb) {
 
   /**
    * Construct by the given mode.
-   * @param mode
+   * @param mode mode.
    * @see #toShort()
    */
   public FsPermission(short mode) { fromShort(mode); }
@@ -145,13 +149,19 @@ public FsPermission(String mode) {
     this(new RawParser(mode).getPermission());
   }
 
-  /** Return user {@link FsAction}. */
+  /**
+   * @return Return user {@link FsAction}.
+   */
   public FsAction getUserAction() {return useraction;}
 
-  /** Return group {@link FsAction}. */
+  /**
+   * @return Return group {@link FsAction}.
+   */
   public FsAction getGroupAction() {return groupaction;}
 
-  /** Return other {@link FsAction}. */
+  /**
+   * @return Return other {@link FsAction}.
+   */
   public FsAction getOtherAction() {return otheraction;}
 
   private void set(FsAction u, FsAction g, FsAction o, boolean sb) {
@@ -180,6 +190,7 @@ public void readFields(DataInput in) throws IOException {
 
   /**
    * Get masked permission if exists.
+   * @return masked.
    */
   public FsPermission getMasked() {
     return null;
@@ -187,6 +198,7 @@ public FsPermission getMasked() {
 
   /**
    * Get unmasked permission if exists.
+   * @return unmasked.
    */
   public FsPermission getUnmasked() {
     return null;
@@ -194,6 +206,10 @@ public FsPermission getUnmasked() {
 
   /**
    * Create and initialize a {@link FsPermission} from {@link DataInput}.
+   *
+   * @param in data input.
+   * @throws IOException raised on errors performing I/O.
+   * @return FsPermission.
    */
   public static FsPermission read(DataInput in) throws IOException {
     FsPermission p = new FsPermission();
@@ -203,6 +219,7 @@ public static FsPermission read(DataInput in) throws IOException {
 
   /**
    * Encode the object to a short.
+   * @return object to a short.
    */
   public short toShort() {
     int s =  (stickyBit ? 1 << 9 : 0)     |
@@ -301,6 +318,9 @@ public FsPermission applyUMask(FsPermission umask) {
    * '-' sets bits in the mask.
    * 
    * Octal umask, the specified bits are set in the file mode creation mask.
+   *
+   * @param conf configuration.
+   * @return FsPermission UMask.
    */
   public static FsPermission getUMask(Configuration conf) {
     int umask = DEFAULT_UMASK;
@@ -346,7 +366,11 @@ public boolean getAclBit() {
   }
 
   /**
-   * Returns true if the file is encrypted or directory is in an encryption zone
+   * Returns true if the file is encrypted or directory is in an encryption zone.
+   *
+   * @return if the file is encrypted or directory
+   * is in an encryption zone true, not false.
+   *
    * @deprecated Get encryption bit from the
    * {@link org.apache.hadoop.fs.FileStatus} object.
    */
@@ -357,6 +381,9 @@ public boolean getEncryptedBit() {
 
   /**
    * Returns true if the file or directory is erasure coded.
+   *
+   * @return if the file or directory is
+   * erasure coded true, not false.
    * @deprecated Get ec bit from the {@link org.apache.hadoop.fs.FileStatus}
    * object.
    */
@@ -365,7 +392,11 @@ public boolean getErasureCodedBit() {
     return false;
   }
 
-  /** Set the user file creation mask (umask) */
+  /**
+   * Set the user file creation mask (umask)
+   * @param conf configuration.
+   * @param umask umask.
+   */
   public static void setUMask(Configuration conf, FsPermission umask) {
     conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort()));
   }
@@ -379,6 +410,8 @@ public static void setUMask(Configuration conf, FsPermission umask) {
    * {@link FsPermission#getDirDefault()} for directory, and use
    * {@link FsPermission#getFileDefault()} for file.
    * This method is kept for compatibility.
+   *
+   * @return Default FsPermission.
    */
   public static FsPermission getDefault() {
     return new FsPermission((short)00777);
@@ -386,6 +419,8 @@ public static FsPermission getDefault() {
 
   /**
    * Get the default permission for directory.
+   *
+   * @return DirDefault FsPermission.
    */
   public static FsPermission getDirDefault() {
     return new FsPermission((short)00777);
@@ -393,6 +428,8 @@ public static FsPermission getDirDefault() {
 
   /**
    * Get the default permission for file.
+   *
+   * @return FileDefault FsPermission.
    */
   public static FsPermission getFileDefault() {
     return new FsPermission((short)00666);
@@ -400,6 +437,8 @@ public static FsPermission getFileDefault() {
 
   /**
    * Get the default permission for cache pools.
+   *
+   * @return CachePoolDefault FsPermission.
    */
   public static FsPermission getCachePoolDefault() {
     return new FsPermission((short)00755);
@@ -408,6 +447,7 @@ public static FsPermission getCachePoolDefault() {
   /**
    * Create a FsPermission from a Unix symbolic permission string
    * @param unixSymbolicPermission e.g. "-rw-rw-rw-"
+   * @return FsPermission.
    */
   public static FsPermission valueOf(String unixSymbolicPermission) {
     if (unixSymbolicPermission == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
index 3c3693f613baf..be4beb506a63f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
@@ -39,7 +39,13 @@ public class PermissionStatus implements Writable {
     WritableFactories.setFactory(PermissionStatus.class, FACTORY);
   }
 
-  /** Create an immutable {@link PermissionStatus} object. */
+  /**
+   * Create an immutable {@link PermissionStatus} object.
+   * @param user user.
+   * @param group group.
+   * @param permission permission.
+   * @return PermissionStatus.
+   */
   public static PermissionStatus createImmutable(
       String user, String group, FsPermission permission) {
     return new PermissionStatus(user, group, permission) {
@@ -56,20 +62,35 @@ public void readFields(DataInput in) throws IOException {
 
   private PermissionStatus() {}
 
-  /** Constructor */
+  /**
+   * Constructor.
+   *
+   * @param user user.
+   * @param group group.
+   * @param permission permission.
+   */
   public PermissionStatus(String user, String group, FsPermission permission) {
     username = user;
     groupname = group;
     this.permission = permission;
   }
 
-  /** Return user name */
+  /**
+   * Return user name.
+   * @return user name.
+   */
   public String getUserName() {return username;}
 
-  /** Return group name */
+  /**
+   * Return group name.
+   * @return group name.
+   */
   public String getGroupName() {return groupname;}
 
-  /** Return permission */
+  /**
+   * Return permission.
+   * @return FsPermission.
+   */
   public FsPermission getPermission() {return permission;}
 
   @Override
@@ -86,6 +107,9 @@ public void write(DataOutput out) throws IOException {
 
   /**
    * Create and initialize a {@link PermissionStatus} from {@link DataInput}.
+   * @param in data input.
+   * @throws IOException raised on errors performing I/O.
+   * @return PermissionStatus.
    */
   public static PermissionStatus read(DataInput in) throws IOException {
     PermissionStatus p = new PermissionStatus();
@@ -95,6 +119,11 @@ public static PermissionStatus read(DataInput in) throws IOException {
 
   /**
    * Serialize a {@link PermissionStatus} from its base components.
+   * @param out out.
+   * @param username username.
+   * @param groupname groupname.
+   * @param permission FsPermission.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void write(DataOutput out,
                            String username, 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPConnectionPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPConnectionPool.java
index de86bab6d3324..eace6417dcd68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPConnectionPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPConnectionPool.java
@@ -76,7 +76,7 @@ synchronized void returnToPool(ChannelSftp channel) {
     ConnectionInfo info = con2infoMap.get(channel);
     HashSet<ChannelSftp> cons = idleConnections.get(info);
     if (cons == null) {
-      cons = new HashSet<ChannelSftp>();
+      cons = new HashSet<>();
       idleConnections.put(info, cons);
     }
     cons.add(channel);
@@ -94,7 +94,7 @@ synchronized void shutdown() {
     Set<ChannelSftp> cons = con2infoMap.keySet();
     if (cons != null && cons.size() > 0) {
       // make a copy since we need to modify the underlying Map
-      Set<ChannelSftp> copy = new HashSet<ChannelSftp>(cons);
+      Set<ChannelSftp> copy = new HashSet<>(cons);
       // Initiate disconnect from all outstanding connections
       for (ChannelSftp con : copy) {
         try {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 46b829f14a15c..3260a0db51479 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -709,11 +709,11 @@ public FileStatus getFileStatus(Path f) throws IOException {
 
   @Override
   public void close() throws IOException {
-    if (closed.getAndSet(true)) {
-      return;
-    }
     try {
       super.close();
+      if (closed.getAndSet(true)) {
+        return;
+      }
     } finally {
       if (connectionPool != null) {
         connectionPool.shutdown();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index 0bdb47730a929..7858238ee71fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -77,7 +77,11 @@ protected Command() {
     err = System.err;
   }
   
-  /** Constructor */
+  /**
+   * Constructor.
+   *
+   * @param conf configuration.
+   */
   protected Command(Configuration conf) {
     super(conf);
   }
@@ -109,7 +113,7 @@ protected int getDepth() {
    * Execute the command on the input path data. Commands can override to make
    * use of the resolved filesystem.
    * @param pathData The input path with resolved filesystem
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected void run(PathData pathData) throws IOException {
     run(pathData.path);
@@ -136,11 +140,19 @@ public int runAll() {
     return exitCode;
   }
 
-  /** sets the command factory for later use */
+  /**
+   * sets the command factory for later use.
+   * @param factory factory.
+   */
   public void setCommandFactory(CommandFactory factory) {
     this.commandFactory = factory;
   }
-  /** retrieves the command factory */
+
+  /**
+   * retrieves the command factory.
+   *
+   * @return command factory.
+   */
   protected CommandFactory getCommandFactory() {
     return this.commandFactory;
   }
@@ -201,7 +213,7 @@ public int run(String...argv) {
    * IllegalArgumentException is thrown, the FsShell object will print the
    * short usage of the command.
    * @param args the command line arguments
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected void processOptions(LinkedList<String> args) throws IOException {}
 
@@ -211,7 +223,7 @@ protected void processOptions(LinkedList<String> args) throws IOException {}
    * {@link #expandArguments(LinkedList)} and pass the resulting list to
    * {@link #processArguments(LinkedList)} 
    * @param args the list of argument strings
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected void processRawArguments(LinkedList<String> args)
   throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
index 4dd20d108428e..1228f76d846ab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFormat.java
@@ -165,7 +165,7 @@ public String getOptValue(String option) {
    * @return Set{@literal <}String{@literal >} of the enabled options
    */
   public Set<String> getOpts() {
-    Set<String> optSet = new HashSet<String>();
+    Set<String> optSet = new HashSet<>();
     for (Map.Entry<String, Boolean> entry : options.entrySet()) {
       if (entry.getValue()) {
         optSet.add(entry.getKey());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 90a709dffc0c1..69a418c1925eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -55,6 +55,9 @@
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
 import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
 import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * Provides: argument processing to ensure the destination is valid
@@ -116,6 +119,8 @@ protected void setDirectWrite(boolean flag) {
    * owner, group and permission information of the source
    * file will be preserved as far as target {@link FileSystem}
    * implementation allows.
+   *
+   * @param preserve preserve.
    */
   protected void setPreserve(boolean preserve) {
     if (preserve) {
@@ -172,6 +177,7 @@ protected void preserve(FileAttribute fileAttribute) {
    *  The last arg is expected to be a local path, if only one argument is
    *  given then the destination will be the current directory 
    *  @param args is the list of arguments
+   * @throws IOException raised on errors performing I/O.
    */
   protected void getLocalDestination(LinkedList<String> args)
   throws IOException {
@@ -348,7 +354,11 @@ protected void copyFileToTarget(PathData src, PathData target)
     src.fs.setVerifyChecksum(verifyChecksum);
     InputStream in = null;
     try {
-      in = src.fs.open(src.path);
+      in = awaitFuture(src.fs.openFile(src.path)
+          .withFileStatus(src.stat)
+          .opt(FS_OPTION_OPENFILE_READ_POLICY,
+              FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+          .build());
       copyStreamToTarget(in, target);
       preserveAttributes(src, target, preserveRawXattrs);
     } finally {
@@ -397,11 +407,11 @@ private boolean checkPathsForReservedRaw(Path src, Path target)
 
   /**
    * If direct write is disabled ,copies the stream contents to a temporary
-   * file "<target>._COPYING_". If the copy is
-   * successful, the temporary file will be renamed to the real path,
-   * else the temporary file will be deleted.
+   * file "target._COPYING_". If the copy is successful, the temporary file
+   * will be renamed to the real path, else the temporary file will be deleted.
    * if direct write is enabled , then creation temporary file is skipped.
-   * @param in the input stream for the copy
+   *
+   * @param in     the input stream for the copy
    * @param target where to store the contents of the stream
    * @throws IOException if copy fails
    */ 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
new file mode 100644
index 0000000000000..aed4030540baf
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommandWithMultiThread.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Abstract command to enable sub copy commands run with multi-thread.
+ */
+public abstract class CopyCommandWithMultiThread
+    extends CommandWithDestination {
+
+  private int threadCount = 1;
+  private ThreadPoolExecutor executor = null;
+  private int threadPoolQueueSize = DEFAULT_QUEUE_SIZE;
+
+  public static final int DEFAULT_QUEUE_SIZE = 1024;
+
+  /**
+   * set thread count by option value, if the value less than 1,
+   * use 1 instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadCount(String optValue) {
+    if (optValue != null) {
+      threadCount = Math.max(Integer.parseInt(optValue), 1);
+    }
+  }
+
+  /**
+   * set thread pool queue size by option value, if the value less than 1,
+   * use DEFAULT_QUEUE_SIZE instead.
+   *
+   * @param optValue option value
+   */
+  protected void setThreadPoolQueueSize(String optValue) {
+    if (optValue != null) {
+      int size = Integer.parseInt(optValue);
+      threadPoolQueueSize = size < 1 ? DEFAULT_QUEUE_SIZE : size;
+    }
+  }
+
+  @VisibleForTesting
+  protected int getThreadCount() {
+    return this.threadCount;
+  }
+
+  @VisibleForTesting
+  protected int getThreadPoolQueueSize() {
+    return this.threadPoolQueueSize;
+  }
+
+  @VisibleForTesting
+  protected ThreadPoolExecutor getExecutor() {
+    return this.executor;
+  }
+
+  @Override
+  protected void processArguments(LinkedList<PathData> args)
+      throws IOException {
+
+    if (isMultiThreadNecessary(args)) {
+      initThreadPoolExecutor();
+    }
+
+    super.processArguments(args);
+
+    if (executor != null) {
+      waitForCompletion();
+    }
+  }
+
+  // if thread count is 1 or the source is only one single file,
+  // don't init executor to avoid threading overhead.
+  @VisibleForTesting
+  protected boolean isMultiThreadNecessary(LinkedList<PathData> args)
+      throws IOException {
+    return this.threadCount > 1 && hasMoreThanOneSourcePaths(args);
+  }
+
+  // check if source is only one single file.
+  private boolean hasMoreThanOneSourcePaths(LinkedList<PathData> args)
+      throws IOException {
+    if (args.size() > 1) {
+      return true;
+    }
+    if (args.size() == 1) {
+      PathData src = args.get(0);
+      if (src.stat == null) {
+        src.refreshStatus();
+      }
+      return isPathRecursable(src);
+    }
+    return false;
+  }
+
+  private void initThreadPoolExecutor() {
+    executor =
+        new ThreadPoolExecutor(threadCount, threadCount, 1, TimeUnit.SECONDS,
+            new ArrayBlockingQueue<>(threadPoolQueueSize),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+  }
+
+  private void waitForCompletion() {
+    if (executor != null) {
+      executor.shutdown();
+      try {
+        executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+      } catch (InterruptedException e) {
+        executor.shutdownNow();
+        displayError(e);
+        Thread.currentThread().interrupt();
+      }
+    }
+  }
+
+  @Override
+  protected void copyFileToTarget(PathData src, PathData target)
+      throws IOException {
+    if (executor == null) {
+      super.copyFileToTarget(src, target);
+    } else {
+      executor.submit(() -> {
+        try {
+          super.copyFileToTarget(src, target);
+        } catch (IOException e) {
+          displayError(e);
+        }
+      });
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 96d5cad2bf6bd..1ac204f5f8a82 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -26,11 +26,7 @@
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -38,8 +34,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIsDirectoryException;
 import org.apache.hadoop.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /** Various commands for copy files */
 @InterfaceAudience.Private
@@ -104,7 +98,8 @@ protected void processArguments(LinkedList<PathData> items)
       try {
         for (PathData src : srcs) {
           if (src.stat.getLen() != 0) {
-            try (FSDataInputStream in = src.fs.open(src.path)) {
+            // Always do sequential reads.
+            try (FSDataInputStream in = src.openForSequentialIO()) {
               IOUtils.copyBytes(in, out, getConf(), false);
               writeDelimiter(out);
             }
@@ -153,33 +148,40 @@ protected boolean isSorted() {
     }
   }
 
-  static class Cp extends CommandWithDestination {
+  static class Cp extends CopyCommandWithMultiThread {
     public static final String NAME = "cp";
     public static final String USAGE =
-        "[-f] [-p | -p[topax]] [-d] <src> ... <dst>";
+        "[-f] [-p | -p[topax]] [-d] [-t <thread count>]"
+            + " [-q <thread pool queue size>] <src> ... <dst>";
     public static final String DESCRIPTION =
-      "Copy files that match the file pattern <src> to a " +
-      "destination.  When copying multiple files, the destination " +
-      "must be a directory. Passing -p preserves status " +
-      "[topax] (timestamps, ownership, permission, ACLs, XAttr). " +
-      "If -p is specified with no <arg>, then preserves " +
-      "timestamps, ownership, permission. If -pa is specified, " +
-      "then preserves permission also because ACL is a super-set of " +
-      "permission. Passing -f overwrites the destination if it " +
-      "already exists. raw namespace extended attributes are preserved " +
-      "if (1) they are supported (HDFS only) and, (2) all of the source and " +
-      "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
-      "xattr preservation is determined solely by the presence (or absence) " +
-        "of the /.reserved/raw prefix and not by the -p option. Passing -d "+
-        "will skip creation of temporary file(<dst>._COPYING_).\n";
+        "Copy files that match the file pattern <src> to a destination."
+            + " When copying multiple files, the destination must be a "
+            + "directory.\nFlags :\n"
+            + "  -p[topax] : Preserve file attributes [topx] (timestamps, "
+            + "ownership, permission, ACL, XAttr). If -p is specified with "
+            + "no arg, then preserves timestamps, ownership, permission. "
+            + "If -pa is specified, then preserves permission also because "
+            + "ACL is a super-set of permission. Determination of whether raw "
+            + "namespace extended attributes are preserved is independent of "
+            + "the -p flag.\n"
+            + "  -f : Overwrite the destination if it already exists.\n"
+            + "  -d : Skip creation of temporary file(<dst>._COPYING_).\n"
+            + "  -t <thread count> : Number of threads to be used, "
+            + "default is 1.\n"
+            + "  -q <thread pool queue size> : Thread pool queue size to be "
+            + "used, default is 1024.\n";
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
       popPreserveOption(args);
       CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "f", "d");
+      cf.addOptionWithValue("t");
+      cf.addOptionWithValue("q");
       cf.parse(args);
       setDirectWrite(cf.getOpt("d"));
       setOverwrite(cf.getOpt("f"));
+      setThreadCount(cf.getOptValue("t"));
+      setThreadPoolQueueSize(cf.getOptValue("q"));
       // should have a -r option
       setRecursive(true);
       getRemoteDestination(args);
@@ -210,28 +212,37 @@ private void popPreserveOption(List<String> args) {
   /** 
    * Copy local files to a remote filesystem
    */
-  public static class Get extends CommandWithDestination {
+  public static class Get extends CopyCommandWithMultiThread {
     public static final String NAME = "get";
     public static final String USAGE =
-      "[-f] [-p] [-ignoreCrc] [-crc] <src> ... <localdst>";
+        "[-f] [-p] [-crc] [-ignoreCrc] [-t <thread count>]"
+            + " [-q <thread pool queue size>] <src> ... <localdst>";
     public static final String DESCRIPTION =
-      "Copy files that match the file pattern <src> " +
-      "to the local name.  <src> is kept.  When copying multiple " +
-      "files, the destination must be a directory. Passing " +
-      "-f overwrites the destination if it already exists and " +
-      "-p preserves access and modification times, " +
-      "ownership and the mode.\n";
+        "Copy files that match the file pattern <src> to the local name. "
+            + "<src> is kept.\nWhen copying multiple files, the destination"
+            + " must be a directory.\nFlags:\n"
+            + "  -p : Preserves timestamps, ownership and the mode.\n"
+            + "  -f : Overwrites the destination if it already exists.\n"
+            + "  -crc : write CRC checksums for the files downloaded.\n"
+            + "  -ignoreCrc : Skip CRC checks on the file(s) downloaded.\n"
+            + "  -t <thread count> : Number of threads to be used,"
+            + " default is 1.\n"
+            + "  -q <thread pool queue size> : Thread pool queue size to be"
+            + " used, default is 1024.\n";
 
     @Override
-    protected void processOptions(LinkedList<String> args)
-    throws IOException {
-      CommandFormat cf = new CommandFormat(
-          1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p", "f");
+    protected void processOptions(LinkedList<String> args) throws IOException {
+      CommandFormat cf =
+          new CommandFormat(1, Integer.MAX_VALUE, "crc", "ignoreCrc", "p", "f");
+      cf.addOptionWithValue("t");
+      cf.addOptionWithValue("q");
       cf.parse(args);
       setWriteChecksum(cf.getOpt("crc"));
       setVerifyChecksum(!cf.getOpt("ignoreCrc"));
       setPreserve(cf.getOpt("p"));
       setOverwrite(cf.getOpt("f"));
+      setThreadCount(cf.getOptValue("t"));
+      setThreadPoolQueueSize(cf.getOptValue("q"));
       setRecursive(true);
       getLocalDestination(args);
     }
@@ -240,21 +251,12 @@ protected void processOptions(LinkedList<String> args)
   /**
    *  Copy local files to a remote filesystem
    */
-  public static class Put extends CommandWithDestination {
-
-    public static final Logger LOG = LoggerFactory.getLogger(Put.class);
-
-    private ThreadPoolExecutor executor = null;
-    private int threadPoolQueueSize = 1024;
-    private int numThreads = 1;
-
-    private static final int MAX_THREADS =
-        Runtime.getRuntime().availableProcessors() * 2;
+  public static class Put extends CopyCommandWithMultiThread {
 
     public static final String NAME = "put";
     public static final String USAGE =
-        "[-f] [-p] [-l] [-d] [-t <thread count>] [-q <threadPool queue size>] " +
-        "<localsrc> ... <dst>";
+        "[-f] [-p] [-l] [-d] [-t <thread count>] [-q <thread pool queue size>]"
+            + " <localsrc> ... <dst>";
     public static final String DESCRIPTION =
         "Copy files from the local file system " +
         "into fs. Copying fails if the file already " +
@@ -263,11 +265,11 @@ public static class Put extends CommandWithDestination {
         "  -p : Preserves timestamps, ownership and the mode.\n" +
         "  -f : Overwrites the destination if it already exists.\n" +
         "  -t <thread count> : Number of threads to be used, default is 1.\n" +
-        "  -q <threadPool size> : ThreadPool queue size to be used, " +
+        "  -q <thread pool queue size> : Thread pool queue size to be used, " +
         "default is 1024.\n" +
-        "  -l : Allow DataNode to lazily persist the file to disk. Forces" +
-        "  replication factor of 1. This flag will result in reduced" +
-        "  durability. Use with care.\n" +
+        "  -l : Allow DataNode to lazily persist the file to disk. Forces " +
+        "replication factor of 1. This flag will result in reduced " +
+        "durability. Use with care.\n" +
         "  -d : Skip creation of temporary file(<dst>._COPYING_).\n";
 
     @Override
@@ -277,7 +279,7 @@ protected void processOptions(LinkedList<String> args) throws IOException {
       cf.addOptionWithValue("t");
       cf.addOptionWithValue("q");
       cf.parse(args);
-      setNumberThreads(cf.getOptValue("t"));
+      setThreadCount(cf.getOptValue("t"));
       setThreadPoolQueueSize(cf.getOptValue("q"));
       setOverwrite(cf.getOpt("f"));
       setPreserve(cf.getOpt("p"));
@@ -308,92 +310,9 @@ protected void processArguments(LinkedList<PathData> args)
         copyStreamToTarget(System.in, getTargetPath(args.get(0)));
         return;
       }
-
-      executor = new ThreadPoolExecutor(numThreads, numThreads, 1,
-          TimeUnit.SECONDS, new ArrayBlockingQueue<>(threadPoolQueueSize),
-          new ThreadPoolExecutor.CallerRunsPolicy());
       super.processArguments(args);
-
-      // issue the command and then wait for it to finish
-      executor.shutdown();
-      try {
-        executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
-      } catch (InterruptedException e) {
-        executor.shutdownNow();
-        displayError(e);
-        Thread.currentThread().interrupt();
-      }
-    }
-
-    private void setNumberThreads(String numberThreadsString) {
-      if (numberThreadsString == null) {
-        numThreads = 1;
-      } else {
-        int parsedValue = Integer.parseInt(numberThreadsString);
-        if (parsedValue <= 1) {
-          numThreads = 1;
-        } else if (parsedValue > MAX_THREADS) {
-          numThreads = MAX_THREADS;
-        } else {
-          numThreads = parsedValue;
-        }
-      }
-    }
-
-    private void setThreadPoolQueueSize(String numThreadPoolQueueSize) {
-      if (numThreadPoolQueueSize != null) {
-        int parsedValue = Integer.parseInt(numThreadPoolQueueSize);
-        if (parsedValue < 1) {
-          LOG.warn("The value of the thread pool queue size cannot be " +
-              "less than 1, and the default value is used here. " +
-              "The default size is 1024.");
-          threadPoolQueueSize = 1024;
-        } else {
-          threadPoolQueueSize = parsedValue;
-        }
-      }
-    }
-
-    @VisibleForTesting
-    protected int getThreadPoolQueueSize() {
-      return threadPoolQueueSize;
-    }
-
-    private void copyFile(PathData src, PathData target) throws IOException {
-      if (isPathRecursable(src)) {
-        throw new PathIsDirectoryException(src.toString());
-      }
-      super.copyFileToTarget(src, target);
-    }
-
-    @Override
-    protected void copyFileToTarget(PathData src, PathData target)
-        throws IOException {
-      // if number of thread is 1, mimic put and avoid threading overhead
-      if (numThreads == 1) {
-        copyFile(src, target);
-        return;
-      }
-
-      Runnable task = () -> {
-        try {
-          copyFile(src, target);
-        } catch (IOException e) {
-          displayError(e);
-        }
-      };
-      executor.submit(task);
-    }
-
-    @VisibleForTesting
-    public int getNumThreads() {
-      return numThreads;
     }
 
-    @VisibleForTesting
-    public ThreadPoolExecutor getExecutor() {
-      return executor;
-    }
   }
 
   public static class CopyFromLocal extends Put {
@@ -414,15 +333,24 @@ public static class CopyToLocal extends Get {
    */
   public static class AppendToFile extends CommandWithDestination {
     public static final String NAME = "appendToFile";
-    public static final String USAGE = "<localsrc> ... <dst>";
+    public static final String USAGE = "[-n] <localsrc> ... <dst>";
     public static final String DESCRIPTION =
         "Appends the contents of all the given local files to the " +
             "given dst file. The dst file will be created if it does " +
             "not exist. If <localSrc> is -, then the input is read " +
-            "from stdin.";
+            "from stdin. Option -n represents that use NEW_BLOCK create flag to append file.";
 
     private static final int DEFAULT_IO_LENGTH = 1024 * 1024;
     boolean readStdin = false;
+    private boolean appendToNewBlock = false;
+
+    public boolean isAppendToNewBlock() {
+      return appendToNewBlock;
+    }
+
+    public void setAppendToNewBlock(boolean appendToNewBlock) {
+      this.appendToNewBlock = appendToNewBlock;
+    }
 
     // commands operating on local paths have no need for glob expansion
     @Override
@@ -453,6 +381,9 @@ protected void processOptions(LinkedList<String> args)
         throw new IOException("missing destination argument");
       }
 
+      CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "n");
+      cf.parse(args);
+      appendToNewBlock = cf.getOpt("n");
       getRemoteDestination(args);
       super.processOptions(args);
     }
@@ -466,7 +397,8 @@ protected void processArguments(LinkedList<PathData> args)
       }
 
       InputStream is = null;
-      try (FSDataOutputStream fos = dst.fs.append(dst.path)) {
+      try (FSDataOutputStream fos = appendToNewBlock ?
+          dst.fs.append(dst.path, true) : dst.fs.append(dst.path)) {
         if (readStdin) {
           if (args.size() == 0) {
             IOUtils.copyBytes(System.in, fos, DEFAULT_IO_LENGTH);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index 670fa152f72ed..1a3f18a658d22 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -47,7 +47,6 @@
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -105,7 +104,8 @@ private void printToStdout(InputStream in) throws IOException {
     }
 
     protected InputStream getInputStream(PathData item) throws IOException {
-      return item.fs.open(item.path);
+      // Always do sequential reads;
+      return item.openForSequentialIO();
     }
   }
   
@@ -216,8 +216,8 @@ protected void processPath(PathData item) throws IOException {
 
   protected class TextRecordInputStream extends InputStream {
     SequenceFile.Reader r;
-    Writable key;
-    Writable val;
+    Object key;
+    Object val;
 
     DataInputBuffer inbuf;
     DataOutputBuffer outbuf;
@@ -227,10 +227,8 @@ public TextRecordInputStream(FileStatus f) throws IOException {
       final Configuration lconf = getConf();
       r = new SequenceFile.Reader(lconf, 
           SequenceFile.Reader.file(fpath));
-      key = ReflectionUtils.newInstance(
-          r.getKeyClass().asSubclass(Writable.class), lconf);
-      val = ReflectionUtils.newInstance(
-          r.getValueClass().asSubclass(Writable.class), lconf);
+      key = ReflectionUtils.newInstance(r.getKeyClass(), lconf);
+      val = ReflectionUtils.newInstance(r.getValueClass(), lconf);
       inbuf = new DataInputBuffer();
       outbuf = new DataOutputBuffer();
     }
@@ -239,8 +237,11 @@ public TextRecordInputStream(FileStatus f) throws IOException {
     public int read() throws IOException {
       int ret;
       if (null == inbuf || -1 == (ret = inbuf.read())) {
-        if (!r.next(key, val)) {
+        key = r.next(key);
+        if (key == null) {
           return -1;
+        } else {
+          val = r.getCurrentValue(val);
         }
         byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8);
         outbuf.write(tmp, 0, tmp.length);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
index 2280225b5ae32..7242f261801d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
@@ -28,6 +28,8 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+
 /**
  * Show the first 1KB of the file.
  */
@@ -68,11 +70,9 @@ protected void processPath(PathData item) throws IOException {
   }
 
   private void dumpToOffset(PathData item) throws IOException {
-    FSDataInputStream in = item.fs.open(item.path);
-    try {
+    try (FSDataInputStream in = item.openFile(
+        FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) {
       IOUtils.copyBytes(in, System.out, endingOffset, false);
-    } finally {
-      in.close();
     }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
index 1ff8d8f0494a1..da99ac212563e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
@@ -39,6 +40,10 @@
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.fs.RemoteIterator;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
 
 /**
@@ -601,4 +606,35 @@ public boolean equals(Object o) {
   public int hashCode() {
     return path.hashCode();
   }
+
+
+  /**
+   * Open a file for sequential IO.
+   * <p>
+   * This uses FileSystem.openFile() to request sequential IO;
+   * the file status is also passed in.
+   * Filesystems may use to optimize their IO.
+   * </p>
+   * @return an input stream
+   * @throws IOException failure
+   */
+  protected FSDataInputStream openForSequentialIO()
+      throws IOException {
+    return openFile(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL);
+  }
+
+  /**
+   * Open a file.
+   * @param policy fadvise policy.
+   * @return an input stream
+   * @throws IOException failure
+   */
+  protected FSDataInputStream openFile(final String policy) throws IOException {
+    return awaitFuture(fs.openFile(path)
+        .opt(FS_OPTION_OPENFILE_READ_POLICY,
+            policy)
+        .opt(FS_OPTION_OPENFILE_LENGTH,
+            stat.getLen())   // file length hint for object stores
+        .build());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
index 98db1c3b7b605..26ac3fee4713b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
@@ -30,6 +30,8 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+
 /**
  * Get a listing of all files in that match the file patterns.
  */
@@ -107,16 +109,15 @@ private long dumpFromOffset(PathData item, long offset) throws IOException {
     if (offset < 0) {
       offset = Math.max(fileSize + offset, 0);
     }
-    
-    FSDataInputStream in = item.fs.open(item.path);
-    try {
+    // Always do sequential reads.
+    try (FSDataInputStream in = item.openFile(
+        FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) {
       in.seek(offset);
       // use conf so the system configured io block size is used
       IOUtils.copyBytes(in, System.out, getConf(), false);
       offset = in.getPos();
-    } finally {
-      in.close();
     }
     return offset;
   }
+
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java
index 902eada98dbd1..457cd86ab703c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/TouchCommands.java
@@ -147,9 +147,6 @@ protected void processOptions(LinkedList<String> args) {
 
     @Override
     protected void processPath(PathData item) throws IOException {
-      if (item.stat.isDirectory()) {
-        throw new PathIsDirectoryException(item.toString());
-      }
       touch(item);
     }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
index 5069d2d34e51c..cd9bbe2bc884e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
@@ -38,12 +38,18 @@ public abstract class BaseExpression implements Expression, Configurable {
   private String[] usage = { "Not yet implemented" };
   private String[] help = { "Not yet implemented" };
 
-  /** Sets the usage text for this {@link Expression} */
+  /**
+   * Sets the usage text for this {@link Expression} .
+   * @param usage usage array.
+   */
   protected void setUsage(String[] usage) {
     this.usage = usage;
   }
 
-  /** Sets the help text for this {@link Expression} */
+  /**
+   * Sets the help text for this {@link Expression} .
+   * @param help help.
+   */
   protected void setHelp(String[] help) {
     this.help = help;
   }
@@ -92,7 +98,10 @@ public void finish() throws IOException {
   /** Children of this expression. */
   private LinkedList<Expression> children = new LinkedList<Expression>();
 
-  /** Return the options to be used by this expression. */
+  /**
+   * Return the options to be used by this expression.
+   * @return options.
+   */
   protected FindOptions getOptions() {
     return (this.options == null) ? new FindOptions() : this.options;
   }
@@ -265,6 +274,7 @@ protected void addArgument(String arg) {
    * @param depth
    *          current depth in the process directories
    * @return FileStatus
+   * @throws IOException raised on errors performing I/O.
    */
   protected FileStatus getFileStatus(PathData item, int depth)
       throws IOException {
@@ -285,6 +295,8 @@ protected FileStatus getFileStatus(PathData item, int depth)
    * @param item
    *          PathData
    * @return Path
+   *
+   * @throws IOException raised on errors performing I/O.
    */
   protected Path getPath(PathData item) throws IOException {
     return item.path;
@@ -295,6 +307,7 @@ protected Path getPath(PathData item) throws IOException {
    *
    * @param item PathData
    * @return FileSystem
+   * @throws IOException raised on errors performing I/O.
    */
   protected FileSystem getFileSystem(PathData item) throws IOException {
     return item.fs;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
index ccad631028cc9..353fe685cc9cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
@@ -30,13 +30,15 @@ public interface Expression {
   /**
    * Set the options for this expression, called once before processing any
    * items.
+   * @param options options.
+   * @throws IOException raised on errors performing I/O.
    */
   public void setOptions(FindOptions options) throws IOException;
 
   /**
    * Prepares the expression for execution, called once after setting options
    * and before processing any options.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void prepare() throws IOException;
 
@@ -46,13 +48,14 @@ public interface Expression {
    * @param item {@link PathData} item to be processed
    * @param depth distance of the item from the command line argument
    * @return {@link Result} of applying the expression to the item
+   * @throws IOException raised on errors performing I/O.
    */
   public Result apply(PathData item, int depth) throws IOException;
 
   /**
    * Finishes the expression, called once after processing all items.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void finish() throws IOException;
 
@@ -76,15 +79,21 @@ public interface Expression {
   /**
    * Indicates whether this expression performs an action, i.e. provides output
    * back to the user.
+   * @return if is action true, not false.
    */
   public boolean isAction();
 
-  /** Identifies the expression as an operator rather than a primary. */
+  /**
+   * Identifies the expression as an operator rather than a primary.
+   * @return if is operator true, not false.
+   */
   public boolean isOperator();
 
   /**
    * Returns the precedence of this expression
    * (only applicable to operators).
+   *
+   * @return precedence.
    */
   public int getPrecedence();
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
index 199038a751226..07baea89dd604 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
@@ -96,7 +96,7 @@ private static void addExpression(Class<?> clazz) {
   private Expression rootExpression;
 
   /** Set of path items returning a {@link Result#STOP} result. */
-  private HashSet<Path> stopPaths = new HashSet<Path>();
+  private HashSet<Path> stopPaths = new HashSet<>();
 
   /** Register the expressions with the expression factory. */
   private static void registerExpressions(ExpressionFactory factory) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
index b0f1be5c35c93..c605186230590 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
@@ -264,6 +264,7 @@ public void setConfiguration(Configuration configuration) {
 
   /**
    * Return the {@link Configuration} return configuration {@link Configuration}
+   * @return configuration.
    */
   public Configuration getConfiguration() {
     return this.configuration;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
index 2ef9cb4a801d6..a242681acd030 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
@@ -35,23 +35,36 @@ private Result(boolean success, boolean recurse) {
     this.descend = recurse;
   }
 
-  /** Should further directories be descended. */
+  /**
+   * Should further directories be descended.
+   * @return if is pass true,not false.
+   */
   public boolean isDescend() {
     return this.descend;
   }
 
-  /** Should processing continue. */
+  /**
+   * Should processing continue.
+   * @return if is pass true,not false.
+   */
   public boolean isPass() {
     return this.success;
   }
 
-  /** Returns the combination of this and another result. */
+  /**
+   * Returns the combination of this and another result.
+   * @param other other.
+   * @return result.
+   */
   public Result combine(Result other) {
     return new Result(this.isPass() && other.isPass(), this.isDescend()
         && other.isDescend());
   }
 
-  /** Negate this result. */
+  /**
+   * Negate this result.
+   * @return Result.
+   */
   public Result negate() {
     return new Result(!this.isPass(), this.isDescend());
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
new file mode 100644
index 0000000000000..1876a48bc5122
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics;
+
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * An interface defined to capture thread-level IOStatistics by using per
+ * thread context.
+ * <p>
+ * The aggregator should be collected in their constructor by statistics-generating
+ * classes to obtain the aggregator to update <i>across all threads</i>.
+ * <p>
+ * The {@link #snapshot()} call creates a snapshot of the statistics;
+ * <p>
+ * The {@link #reset()} call resets the statistics in the context so
+ * that later snapshots will get the incremental data.
+ */
+public interface IOStatisticsContext extends IOStatisticsSource {
+
+  /**
+   * Get the IOStatisticsAggregator for the context.
+   *
+   * @return return the aggregator for the context.
+   */
+  IOStatisticsAggregator getAggregator();
+
+  /**
+   * Capture the snapshot of the context's IOStatistics.
+   *
+   * @return IOStatisticsSnapshot for the context.
+   */
+  IOStatisticsSnapshot snapshot();
+
+  /**
+   * Get a unique ID for this context, for logging
+   * purposes.
+   *
+   * @return an ID unique for all contexts in this process.
+   */
+  long getID();
+
+  /**
+   * Reset the context's IOStatistics.
+   */
+  void reset();
+
+  /**
+   * Get the context's IOStatisticsContext.
+   *
+   * @return instance of IOStatisticsContext for the context.
+   */
+  static IOStatisticsContext getCurrentIOStatisticsContext() {
+    // the null check is just a safety check to highlight exactly where a null value would
+    // be returned if HADOOP-18456 has resurfaced.
+    return requireNonNull(
+        IOStatisticsContextIntegration.getCurrentIOStatisticsContext(),
+        "Null IOStatisticsContext");
+  }
+
+  /**
+   * Set the IOStatisticsContext for the current thread.
+   * @param statisticsContext IOStatistics context instance for the
+   * current thread. If null, the context is reset.
+   */
+  static void setThreadIOStatisticsContext(
+      IOStatisticsContext statisticsContext) {
+    IOStatisticsContextIntegration.setThreadIOStatisticsContext(
+        statisticsContext);
+  }
+
+  /**
+   * Static probe to check if the thread-level IO statistics enabled.
+   *
+   * @return if the thread-level IO statistics enabled.
+   */
+  static boolean enabled() {
+    return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled();
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
index 63d37e97c98b8..88606eb4b3055 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
@@ -53,7 +53,7 @@
  * deserialized. If for some reason this is required, use
  * {@link #requiredSerializationClasses()} to get the list of classes
  * used when deserializing instances of this object.
- * <p>
+ * </p>
  * <p>
  * It is annotated for correct serializations with jackson2.
  * </p>
@@ -238,6 +238,8 @@ public static JsonSerialization<IOStatisticsSnapshot> serializer() {
   /**
    * Serialize by converting each map to a TreeMap, and saving that
    * to the stream.
+   * @param s ObjectOutputStream.
+   * @throws IOException raised on errors performing I/O.
    */
   private synchronized void writeObject(ObjectOutputStream s)
       throws IOException {
@@ -253,6 +255,10 @@ private synchronized void writeObject(ObjectOutputStream s)
   /**
    * Deserialize by loading each TreeMap, and building concurrent
    * hash maps from them.
+   *
+   * @param s ObjectInputStream.
+   * @throws IOException raised on errors performing I/O.
+   * @throws ClassNotFoundException class not found exception
    */
   private void readObject(final ObjectInputStream s)
       throws IOException, ClassNotFoundException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
index 75977047c0f2a..bb4d9a44587a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
@@ -71,6 +71,7 @@ private IOStatisticsSupport() {
    * Returns null if the source isn't of the write type
    * or the return value of
    * {@link IOStatisticsSource#getIOStatistics()} was null.
+   * @param source source.
    * @return an IOStatistics instance or null
    */
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
index d9ff0c25c6a21..369db49654382 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
@@ -207,6 +207,7 @@ public synchronized double mean() {
   /**
    * Add another MeanStatistic.
    * @param other other value
+   * @return mean statistic.
    */
   public synchronized MeanStatistic add(final MeanStatistic other) {
     if (other.isEmpty()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index 9ec8dcdb3dc9b..c04c1bb47fcea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -53,6 +53,9 @@ public final class StoreStatisticNames {
   /** {@value}. */
   public static final String OP_CREATE = "op_create";
 
+  /** {@value}. */
+  public static final String OP_CREATE_FILE = "op_createfile";
+
   /** {@value}. */
   public static final String OP_CREATE_NON_RECURSIVE =
       "op_create_non_recursive";
@@ -112,9 +115,15 @@ public final class StoreStatisticNames {
   /** {@value}. */
   public static final String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
 
+  /** {@value}. */
+  public static final String OP_MSYNC = "op_msync";
+
   /** {@value}. */
   public static final String OP_OPEN = "op_open";
 
+  /** Call to openFile() {@value}. */
+  public static final String OP_OPENFILE = "op_openfile";
+
   /** {@value}. */
   public static final String OP_REMOVE_ACL = "op_remove_acl";
 
@@ -172,6 +181,9 @@ public final class StoreStatisticNames {
   public static final String STORE_IO_THROTTLED
       = "store_io_throttled";
 
+  /** Rate limiting was reported {@value}. */
+  public static final String STORE_IO_RATE_LIMITED = "store_io_rate_limited";
+
   /** Requests made of a store: {@value}. */
   public static final String STORE_IO_REQUEST
       = "store_io_request";
@@ -317,6 +329,12 @@ public final class StoreStatisticNames {
   public static final String ACTION_EXECUTOR_ACQUIRED =
       "action_executor_acquired";
 
+  /**
+   * A file was opened: {@value}.
+   */
+  public static final String ACTION_FILE_OPENED
+      = "action_file_opened";
+
   /**
    * An HTTP HEAD request was made: {@value}.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
index 7e9137294c1ef..50bbf45505cec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
@@ -47,7 +47,7 @@ public final class StreamStatisticNames {
   public static final String STREAM_READ_ABORTED = "stream_aborted";
 
   /**
-   * Bytes read from an input stream in read() calls.
+   * Bytes read from an input stream in read()/readVectored() calls.
    * Does not include bytes read and then discarded in seek/close etc.
    * These are the bytes returned to the caller.
    * Value: {@value}.
@@ -76,7 +76,7 @@ public final class StreamStatisticNames {
   public static final String STREAM_READ_CLOSED = "stream_read_closed";
 
   /**
-   * Total count of times an attempt to close an input stream was made
+   * Total count of times an attempt to close an input stream was made.
    * Value: {@value}.
    */
   public static final String STREAM_READ_CLOSE_OPERATIONS
@@ -110,6 +110,34 @@ public final class StreamStatisticNames {
   public static final String STREAM_READ_OPERATIONS =
       "stream_read_operations";
 
+  /**
+   * Count of readVectored() operations in an input stream.
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_VECTORED_OPERATIONS =
+          "stream_read_vectored_operations";
+
+  /**
+   * Count of bytes discarded during readVectored() operation
+   * in an input stream.
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_VECTORED_READ_BYTES_DISCARDED =
+          "stream_read_vectored_read_bytes_discarded";
+
+  /**
+   * Count of incoming file ranges during readVectored() operation.
+   * Value: {@value}
+   */
+  public static final String STREAM_READ_VECTORED_INCOMING_RANGES =
+          "stream_read_vectored_incoming_ranges";
+  /**
+   * Count of combined file ranges during readVectored() operation.
+   * Value: {@value}
+   */
+  public static final String STREAM_READ_VECTORED_COMBINED_RANGES =
+          "stream_read_vectored_combined_ranges";
+
   /**
    * Count of incomplete read() operations in an input stream,
    * that is, when the bytes returned were less than that requested.
@@ -118,6 +146,23 @@ public final class StreamStatisticNames {
   public static final String STREAM_READ_OPERATIONS_INCOMPLETE
       = "stream_read_operations_incomplete";
 
+  /**
+   * count/duration of aborting a remote stream during stream IO
+   * IO.
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_REMOTE_STREAM_ABORTED
+      = "stream_read_remote_stream_aborted";
+
+  /**
+   * count/duration of closing a remote stream,
+   * possibly including draining the stream to recycle
+   * the HTTP connection.
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_REMOTE_STREAM_DRAINED
+      = "stream_read_remote_stream_drain";
+
   /**
    * Count of version mismatches encountered while reading an input stream.
    * Value: {@value}.
@@ -370,6 +415,46 @@ public final class StreamStatisticNames {
   public static final String BLOCKS_RELEASED
       = "blocks_released";
 
+  /**
+   * Total number of prefetching operations executed.
+   */
+  public static final String STREAM_READ_PREFETCH_OPERATIONS
+      = "stream_read_prefetch_operations";
+
+  /**
+   * Total number of block in disk cache.
+   */
+  public static final String STREAM_READ_BLOCKS_IN_FILE_CACHE
+      = "stream_read_blocks_in_cache";
+
+  /**
+   * Total number of active prefetch operations.
+   */
+  public static final String STREAM_READ_ACTIVE_PREFETCH_OPERATIONS
+      = "stream_read_active_prefetch_operations";
+
+  /**
+   * Total bytes of memory in use by this input stream.
+   */
+  public static final String STREAM_READ_ACTIVE_MEMORY_IN_USE
+      = "stream_read_active_memory_in_use";
+
+  /**
+   * count/duration of reading a remote block.
+   *
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_REMOTE_BLOCK_READ
+      = "stream_read_block_read";
+
+  /**
+   * count/duration of acquiring a buffer and reading to it.
+   *
+   * Value: {@value}.
+   */
+  public static final String STREAM_READ_BLOCK_ACQUIRE_AND_READ
+      = "stream_read_block_acquire_read";
+
   private StreamStatisticNames() {
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java
new file mode 100644
index 0000000000000..b672f6639cb93
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+
+/**
+ * Empty IOStatistics context which serves no-op for all the operations and
+ * returns an empty Snapshot if asked.
+ *
+ */
+final class EmptyIOStatisticsContextImpl implements IOStatisticsContext {
+
+  private static final IOStatisticsContext EMPTY_CONTEXT = new EmptyIOStatisticsContextImpl();
+
+  private EmptyIOStatisticsContextImpl() {
+  }
+
+  /**
+   * Create a new empty snapshot.
+   * A new one is always created for isolation.
+   *
+   * @return a statistics snapshot
+   */
+  @Override
+  public IOStatisticsSnapshot snapshot() {
+    return new IOStatisticsSnapshot();
+  }
+
+  @Override
+  public IOStatisticsAggregator getAggregator() {
+    return EmptyIOStatisticsStore.getInstance();
+  }
+
+  @Override
+  public IOStatistics getIOStatistics() {
+    return EmptyIOStatistics.getInstance();
+  }
+
+  @Override
+  public void reset() {}
+
+  /**
+   * The ID is always 0.
+   * As the real context implementation counter starts at 1,
+   * we are guaranteed to have unique IDs even between them and
+   * the empty context.
+   * @return 0
+   */
+  @Override
+  public long getID() {
+    return 0;
+  }
+
+  /**
+   * Get the single instance.
+   * @return an instance.
+   */
+  static IOStatisticsContext getInstance() {
+    return EMPTY_CONTEXT;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
index c2eab9d772a66..6a5d01fb3b074 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
@@ -21,6 +21,7 @@
 import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.Serializable;
+import java.time.Duration;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -140,6 +141,7 @@ public static <E> String entryToString(
   /**
    * Convert entry values to the string format used in logging.
    *
+   * @param <E> type of values.
    * @param name statistic name
    * @param value stat value
    * @return formatted string
@@ -177,6 +179,8 @@ private static <E> Map<String, E> copyMap(
   /**
    * A passthrough copy operation suitable for immutable
    * types, including numbers.
+   *
+   * @param <E> type of values.
    * @param src source object
    * @return the source object
    */
@@ -436,6 +440,7 @@ public static <A, B> Function<A, B> trackJavaFunctionDuration(
    * @param input input callable.
    * @param <B> return type.
    * @return the result of the operation.
+   * @throws IOException raised on errors performing I/O.
    */
   public static <B> B trackDuration(
       DurationTrackerFactory factory,
@@ -450,12 +455,37 @@ public static <B> B trackDuration(
    * @param factory factory of duration trackers
    * @param statistic statistic key
    * @param input input callable.
+   * @throws IOException IO failure.
    */
   public static void trackDurationOfInvocation(
       DurationTrackerFactory factory,
       String statistic,
       InvocationRaisingIOE input) throws IOException {
 
+    measureDurationOfInvocation(factory, statistic, input);
+  }
+
+  /**
+   * Given an IOException raising callable/lambda expression,
+   * execute it and update the relevant statistic,
+   * returning the measured duration.
+   *
+   * {@link #trackDurationOfInvocation(DurationTrackerFactory, String, InvocationRaisingIOE)}
+   * with the duration returned for logging etc.; added as a new
+   * method to avoid linking problems with any code calling the existing
+   * method.
+   *
+   * @param factory factory of duration trackers
+   * @param statistic statistic key
+   * @param input input callable.
+   * @return the duration of the operation, as measured by the duration tracker.
+   * @throws IOException IO failure.
+   */
+  public static Duration measureDurationOfInvocation(
+      DurationTrackerFactory factory,
+      String statistic,
+      InvocationRaisingIOE input) throws IOException {
+
     // create the tracker outside try-with-resources so
     // that failures can be set in the catcher.
     DurationTracker tracker = createTracker(factory, statistic);
@@ -473,6 +503,7 @@ public static void trackDurationOfInvocation(
       // set the failed flag.
       tracker.close();
     }
+    return tracker.asDuration();
   }
 
   /**
@@ -494,23 +525,39 @@ public static <B> CallableRaisingIOE<B> trackDurationOfOperation(
       // create the tracker outside try-with-resources so
       // that failures can be set in the catcher.
       DurationTracker tracker = createTracker(factory, statistic);
-      try {
-        // exec the input function and return its value
-        return input.apply();
-      } catch (IOException | RuntimeException e) {
-        // input function failed: note it
-        tracker.failed();
-        // and rethrow
-        throw e;
-      } finally {
-        // update the tracker.
-        // this is called after the catch() call would have
-        // set the failed flag.
-        tracker.close();
-      }
+      return invokeTrackingDuration(tracker, input);
     };
   }
 
+  /**
+   * Given an IOException raising callable/lambda expression,
+   * execute it, updating the tracker on success/failure.
+   * @param tracker duration tracker.
+   * @param input input callable.
+   * @param <B> return type.
+   * @return the result of the invocation
+   * @throws IOException on failure.
+   */
+  public static <B> B invokeTrackingDuration(
+      final DurationTracker tracker,
+      final CallableRaisingIOE<B> input)
+      throws IOException {
+    try {
+      // exec the input function and return its value
+      return input.apply();
+    } catch (IOException | RuntimeException e) {
+      // input function failed: note it
+      tracker.failed();
+      // and rethrow
+      throw e;
+    } finally {
+      // update the tracker.
+      // this is called after the catch() call would have
+      // set the failed flag.
+      tracker.close();
+    }
+  }
+
   /**
    * Given an IOException raising Consumer,
    * return a new one which wraps the inner and tracks
@@ -622,7 +669,7 @@ public static <B> B trackDurationOfSupplier(
    * @param statistic statistic to track
    * @return a duration tracker.
    */
-  private static DurationTracker createTracker(
+  public static DurationTracker createTracker(
       @Nullable final DurationTrackerFactory factory,
       final String statistic) {
     return factory != null
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java
new file mode 100644
index 0000000000000..97a85281c4fb8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+
+/**
+ * Implementing the IOStatisticsContext.
+ *
+ * A Context defined for IOStatistics collection per thread which captures
+ * each worker thread's work in FS streams and stores it in the form of
+ * IOStatisticsSnapshot.
+ *
+ * For the current thread the IOStatisticsSnapshot can be used as a way to
+ * move the IOStatistics data between applications using the Serializable
+ * nature of the class.
+ */
+public final class IOStatisticsContextImpl implements IOStatisticsContext {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(IOStatisticsContextImpl.class);
+
+  /**
+   * Thread ID.
+   */
+  private final long threadId;
+
+  /**
+   * Unique ID.
+   */
+  private final long id;
+
+  /**
+   * IOStatistics to aggregate.
+   */
+  private final IOStatisticsSnapshot ioStatistics = new IOStatisticsSnapshot();
+
+  /**
+   * Constructor.
+   * @param threadId thread ID
+   * @param id instance ID.
+   */
+  public IOStatisticsContextImpl(final long threadId, final long id) {
+    this.threadId = threadId;
+    this.id = id;
+  }
+
+  @Override
+  public String toString() {
+    return "IOStatisticsContextImpl{" +
+        "id=" + id +
+        ", threadId=" + threadId +
+        ", ioStatistics=" + ioStatistics +
+        '}';
+  }
+
+  /**
+   * Get the IOStatisticsAggregator of the context.
+   * @return the instance of IOStatisticsAggregator for this context.
+   */
+  @Override
+  public IOStatisticsAggregator getAggregator() {
+    return ioStatistics;
+  }
+
+  /**
+   * Returns a snapshot of the current thread's IOStatistics.
+   *
+   * @return IOStatisticsSnapshot of the context.
+   */
+  @Override
+  public IOStatisticsSnapshot snapshot() {
+    LOG.debug("Taking snapshot of IOStatisticsContext id {}", id);
+    return new IOStatisticsSnapshot(ioStatistics);
+  }
+
+  /**
+   * Reset the thread +.
+   */
+  @Override
+  public void reset() {
+    LOG.debug("clearing IOStatisticsContext id {}", id);
+    ioStatistics.clear();
+  }
+
+  @Override
+  public IOStatistics getIOStatistics() {
+    return ioStatistics;
+  }
+
+  /**
+   * ID of this context.
+   * @return ID.
+   */
+  @Override
+  public long getID() {
+    return id;
+  }
+
+  /**
+   * Get the thread ID.
+   * @return thread ID.
+   */
+  public long getThreadID() {
+    return threadId;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
new file mode 100644
index 0000000000000..71fdb1f17b1f5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.statistics.impl;
+
+import java.lang.ref.WeakReference;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT;
+
+/**
+ * A Utility class for IOStatisticsContext, which helps in creating and
+ * getting the current active context. Static methods in this class allows to
+ * get the current context to start aggregating the IOStatistics.
+ *
+ * Static initializer is used to work out if the feature to collect
+ * thread-level IOStatistics is enabled or not and the corresponding
+ * implementation class is called for it.
+ *
+ * Weak Reference thread map to be used to keep track of different context's
+ * to avoid long-lived memory leakages as these references would be cleaned
+ * up at GC.
+ */
+public final class IOStatisticsContextIntegration {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(IOStatisticsContextIntegration.class);
+
+  /**
+   * Is thread-level IO Statistics enabled?
+   */
+  private static boolean isThreadIOStatsEnabled;
+
+  /**
+   * ID for next instance to create.
+   */
+  public static final AtomicLong INSTANCE_ID = new AtomicLong(1);
+
+  /**
+   * Active IOStatistics Context containing different worker thread's
+   * statistics. Weak Reference so that it gets cleaned up during GC and we
+   * avoid any memory leak issues due to long lived references.
+   */
+  private static final WeakReferenceThreadMap<IOStatisticsContext>
+      ACTIVE_IOSTATS_CONTEXT =
+      new WeakReferenceThreadMap<>(
+          IOStatisticsContextIntegration::createNewInstance,
+          IOStatisticsContextIntegration::referenceLostContext
+      );
+
+  static {
+    // Work out if the current context has thread level IOStatistics enabled.
+    final Configuration configuration = new Configuration();
+    isThreadIOStatsEnabled =
+        configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED,
+            IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT);
+  }
+
+  /**
+   * Static probe to check if the thread-level IO statistics enabled.
+   *
+   * @return if the thread-level IO statistics enabled.
+   */
+  public static boolean isIOStatisticsThreadLevelEnabled() {
+    return isThreadIOStatsEnabled;
+  }
+
+  /**
+   * Private constructor for a utility class to be used in IOStatisticsContext.
+   */
+  private IOStatisticsContextIntegration() {}
+
+  /**
+   * Creating a new IOStatisticsContext instance for a FS to be used.
+   * @param key Thread ID that represents which thread the context belongs to.
+   * @return an instance of IOStatisticsContext.
+   */
+  private static IOStatisticsContext createNewInstance(Long key) {
+    IOStatisticsContextImpl instance =
+        new IOStatisticsContextImpl(key, INSTANCE_ID.getAndIncrement());
+    LOG.debug("Created instance {}", instance);
+    return instance;
+  }
+
+  /**
+   * In case of reference loss for IOStatisticsContext.
+   * @param key ThreadID.
+   */
+  private static void referenceLostContext(Long key) {
+    LOG.debug("Reference lost for threadID for the context: {}", key);
+  }
+
+  /**
+   * Get the current thread's IOStatisticsContext instance. If no instance is
+   * present for this thread ID, create one using the factory.
+   * @return instance of IOStatisticsContext.
+   */
+  public static IOStatisticsContext getCurrentIOStatisticsContext() {
+    return isThreadIOStatsEnabled
+        ? ACTIVE_IOSTATS_CONTEXT.getForCurrentThread()
+        : EmptyIOStatisticsContextImpl.getInstance();
+  }
+
+  /**
+   * Set the IOStatisticsContext for the current thread.
+   * @param statisticsContext IOStatistics context instance for the
+   * current thread. If null, the context is reset.
+   */
+  public static void setThreadIOStatisticsContext(
+      IOStatisticsContext statisticsContext) {
+    if (isThreadIOStatsEnabled) {
+      if (statisticsContext == null) {
+        // new value is null, so remove it
+        ACTIVE_IOSTATS_CONTEXT.removeForCurrentThread();
+      } else {
+        // the setter is efficient in that it does not create a new
+        // reference if the context is unchanged.
+        ACTIVE_IOSTATS_CONTEXT.setForCurrentThread(statisticsContext);
+      }
+    }
+  }
+
+  /**
+   * Get thread ID specific IOStatistics values if
+   * statistics are enabled and the thread ID is in the map.
+   * @param testThreadId thread ID.
+   * @return IOStatisticsContext if found in the map.
+   */
+  @VisibleForTesting
+  public static IOStatisticsContext getThreadSpecificIOStatisticsContext(long testThreadId) {
+    LOG.debug("IOStatsContext thread ID required: {}", testThreadId);
+
+    if (!isThreadIOStatsEnabled) {
+      return null;
+    }
+    // lookup the weakRef IOStatisticsContext for the thread ID in the
+    // ThreadMap.
+    WeakReference<IOStatisticsContext> ioStatisticsSnapshotWeakReference =
+        ACTIVE_IOSTATS_CONTEXT.lookup(testThreadId);
+    if (ioStatisticsSnapshotWeakReference != null) {
+      return ioStatisticsSnapshotWeakReference.get();
+    }
+    return null;
+  }
+
+  /**
+   * A method to enable IOStatisticsContext to override if set otherwise in
+   * the configurations for tests.
+   */
+  @VisibleForTesting
+  public static void enableIOStatisticsContext() {
+    if (!isThreadIOStatsEnabled) {
+      LOG.info("Enabling Thread IOStatistics..");
+      isThreadIOStatsEnabled = true;
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
index 1b4139e463a9e..c083ad8c3c2ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
@@ -255,4 +255,15 @@ default long incrementCounter(String key) {
    */
   void addTimedOperation(String prefix, Duration duration);
 
+  /**
+   * Add a statistics sample as a min, max and mean and count.
+   * @param key key to add.
+   * @param count count.
+   */
+  default void addSample(String key, long count) {
+    incrementCounter(key, count);
+    addMeanStatisticSample(key, count);
+    addMaximumSample(key, count);
+    addMinimumSample(key, count);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
index 33b13f78418a9..9bc01338a1497 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
@@ -88,6 +88,11 @@ public void close() {
     public Duration asDuration() {
       return firstDuration.asDuration();
     }
+
+    @Override
+    public String toString() {
+      return firstDuration.toString();
+    }
   }
 
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
index ef9e7cb107a0d..04d30135f6bd3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
@@ -103,4 +103,11 @@ public void close() {
     }
     iostats.addTimedOperation(name, asDuration());
   }
+
+  @Override
+  public String toString() {
+    return " Duration of " +
+        (failed? (key + StoreStatisticNames.SUFFIX_FAILURES) : key)
+        + ": " + super.toString();
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
index d9d3850ef4e2e..c70d0ee91e15e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
@@ -107,6 +107,7 @@ private DataBlocks() {
    * @param len number of bytes to be written.
    * @throws NullPointerException      for a null buffer
    * @throws IndexOutOfBoundsException if indices are out of range
+   * @throws IOException raised on errors performing I/O.
    */
   public static void validateWriteArgs(byte[] b, int off, int len)
       throws IOException {
@@ -287,6 +288,7 @@ protected BlockFactory(String keyToBufferDir, Configuration conf) {
      * @param limit      limit of the block.
      * @param statistics stats to work with
      * @return a new block.
+     * @throws IOException raised on errors performing I/O.
      */
     public abstract DataBlock create(long index, int limit,
         BlockUploadStatistics statistics)
@@ -482,6 +484,8 @@ public void close() throws IOException {
 
     /**
      * Inner close logic for subclasses to implement.
+     *
+     * @throws IOException raised on errors performing I/O.
      */
     protected void innerClose() throws IOException {
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
index acc82766be190..9ad727f5a2df2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
@@ -86,6 +86,8 @@ public static InvocationRaisingIOE withinAuditSpan(
    * activates and deactivates the span around the inner one.
    * @param auditSpan audit span
    * @param operation operation
+   * @param <T> Generics Type T.
+   * @param <R> Generics Type R.
    * @return a new invocation.
    */
   public static <T, R> FunctionRaisingIOE<T, R> withinAuditSpan(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
index 09ec5d2933058..c9ee5e232d944 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
@@ -48,7 +48,7 @@ public static String getConfigViewFsPrefix() {
   /**
    * Add a link to the config for the specified mount table
    * @param conf - add the link to this conf
-   * @param mountTableName
+   * @param mountTableName mountTable.
    * @param src - the src path name
    * @param target - the target URI link
    */
@@ -71,9 +71,10 @@ public static void addLink(final Configuration conf, final String src,
 
   /**
    * Add a LinkMergeSlash to the config for the specified mount table.
-   * @param conf
-   * @param mountTableName
-   * @param target
+   *
+   * @param conf configuration.
+   * @param mountTableName mountTable.
+   * @param target target.
    */
   public static void addLinkMergeSlash(Configuration conf,
       final String mountTableName, final URI target) {
@@ -83,8 +84,9 @@ public static void addLinkMergeSlash(Configuration conf,
 
   /**
    * Add a LinkMergeSlash to the config for the default mount table.
-   * @param conf
-   * @param target
+   *
+   * @param conf configuration.
+   * @param target targets.
    */
   public static void addLinkMergeSlash(Configuration conf, final URI target) {
     addLinkMergeSlash(conf, getDefaultMountTableName(conf), target);
@@ -92,9 +94,10 @@ public static void addLinkMergeSlash(Configuration conf, final URI target) {
 
   /**
    * Add a LinkFallback to the config for the specified mount table.
-   * @param conf
-   * @param mountTableName
-   * @param target
+   *
+   * @param conf configuration.
+   * @param mountTableName mountTable.
+   * @param target targets.
    */
   public static void addLinkFallback(Configuration conf,
       final String mountTableName, final URI target) {
@@ -104,8 +107,9 @@ public static void addLinkFallback(Configuration conf,
 
   /**
    * Add a LinkFallback to the config for the default mount table.
-   * @param conf
-   * @param target
+   *
+   * @param conf configuration.
+   * @param target targets.
    */
   public static void addLinkFallback(Configuration conf, final URI target) {
     addLinkFallback(conf, getDefaultMountTableName(conf), target);
@@ -113,9 +117,10 @@ public static void addLinkFallback(Configuration conf, final URI target) {
 
   /**
    * Add a LinkMerge to the config for the specified mount table.
-   * @param conf
-   * @param mountTableName
-   * @param targets
+   *
+   * @param conf configuration.
+   * @param mountTableName mountTable.
+   * @param targets targets.
    */
   public static void addLinkMerge(Configuration conf,
       final String mountTableName, final URI[] targets) {
@@ -125,8 +130,9 @@ public static void addLinkMerge(Configuration conf,
 
   /**
    * Add a LinkMerge to the config for the default mount table.
-   * @param conf
-   * @param targets
+   *
+   * @param conf configuration.
+   * @param targets targets array.
    */
   public static void addLinkMerge(Configuration conf, final URI[] targets) {
     addLinkMerge(conf, getDefaultMountTableName(conf), targets);
@@ -134,6 +140,12 @@ public static void addLinkMerge(Configuration conf, final URI[] targets) {
 
   /**
    * Add nfly link to configuration for the given mount table.
+   *
+   * @param conf configuration.
+   * @param mountTableName mount table.
+   * @param src src.
+   * @param settings settings.
+   * @param targets targets.
    */
   public static void addLinkNfly(Configuration conf, String mountTableName,
       String src, String settings, final String targets) {
@@ -144,12 +156,13 @@ public static void addLinkNfly(Configuration conf, String mountTableName,
   }
 
   /**
+   * Add nfly link to configuration for the given mount table.
    *
-   * @param conf
-   * @param mountTableName
-   * @param src
-   * @param settings
-   * @param targets
+   * @param conf configuration.
+   * @param mountTableName mount table.
+   * @param src src.
+   * @param settings settings.
+   * @param targets targets.
    */
   public static void addLinkNfly(Configuration conf, String mountTableName,
       String src, String settings, final URI ... targets) {
@@ -202,6 +215,7 @@ public static void setHomeDirConf(final Configuration conf,
    * Add config variable for homedir the specified mount table
    * @param conf - add to this conf
    * @param homedir - the home dir path starting with slash
+   * @param mountTableName - the mount table.
    */
   public static void setHomeDirConf(final Configuration conf,
               final String mountTableName, final String homedir) {
@@ -247,4 +261,22 @@ public static String getDefaultMountTableName(final Configuration conf) {
     return conf.get(Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE_NAME_KEY,
         Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE);
   }
+
+  /**
+   * Check the bool config whether nested mount point is supported. Default: true
+   * @param conf - from this conf
+   * @return whether nested mount point is supported
+   */
+  public static boolean isNestedMountPointSupported(final Configuration conf) {
+    return conf.getBoolean(Constants.CONFIG_NESTED_MOUNT_POINT_SUPPORTED, true);
+  }
+
+  /**
+   * Set the bool value isNestedMountPointSupported in config.
+   * @param conf - from this conf
+   * @param isNestedMountPointSupported - whether nested mount point is supported
+   */
+  public static void setIsNestedMountPointSupported(final Configuration conf, boolean isNestedMountPointSupported) {
+    conf.setBoolean(Constants.CONFIG_NESTED_MOUNT_POINT_SUPPORTED, isNestedMountPointSupported);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 8235e937bddbf..806e69f32c6b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -35,7 +35,7 @@ public interface Constants {
    * Prefix for the config variable for the ViewFs mount-table path.
    */
   String CONFIG_VIEWFS_MOUNTTABLE_PATH = CONFIG_VIEWFS_PREFIX + ".path";
- 
+
   /**
    * Prefix for the home dir for the mount table - if not specified
    * then the hadoop default value (/user) is used.
@@ -53,12 +53,17 @@ public interface Constants {
    */
   public static final String CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE = "default";
 
+  /**
+   * Config to enable nested mount point in viewfs
+   */
+  String CONFIG_NESTED_MOUNT_POINT_SUPPORTED = CONFIG_VIEWFS_PREFIX + ".nested.mount.point.supported";
+
   /**
    * Config variable full prefix for the default mount table.
    */
-  public static final String CONFIG_VIEWFS_PREFIX_DEFAULT_MOUNT_TABLE = 
+  public static final String CONFIG_VIEWFS_PREFIX_DEFAULT_MOUNT_TABLE =
           CONFIG_VIEWFS_PREFIX + "." + CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
-  
+
   /**
    * Config variable for specifying a simple link
    */
@@ -82,7 +87,7 @@ public interface Constants {
 
   /**
    * Config variable for specifying a merge of the root of the mount-table
-   *  with the root of another file system. 
+   *  with the root of another file system.
    */
   String CONFIG_VIEWFS_LINK_MERGE_SLASH = "linkMergeSlash";
 
@@ -132,4 +137,11 @@ public interface Constants {
   Class<? extends MountTableConfigLoader>
       DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL =
       HCFSMountTableConfigLoader.class;
+
+  /**
+   * Force ViewFileSystem to return a trashRoot that is inside a mount point.
+   */
+  String CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT =
+      "fs.viewfs.trash.force-inside-mount-point";
+  boolean CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT = false;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
index 071af11e63bf2..f723f238e199e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
@@ -20,18 +20,24 @@
 import java.io.IOException;
 import java.net.URI;
 
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 
 /**
  * File system instance getter.
  */
-@Private
-class FsGetter {
+@InterfaceAudience.LimitedPrivate({"Common"})
+@InterfaceStability.Unstable
+public class FsGetter {
 
   /**
    * Gets new file system instance of given uri.
+   * @param uri uri.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return file system.
    */
   public FileSystem getNewInstance(URI uri, Configuration conf)
       throws IOException {
@@ -40,6 +46,11 @@ public FileSystem getNewInstance(URI uri, Configuration conf)
 
   /**
    * Gets file system instance of given uri.
+   *
+   * @param uri uri.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return FileSystem.
    */
   public FileSystem get(URI uri, Configuration conf) throws IOException {
     return FileSystem.get(uri, conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index e13c7366741c5..5360d55e10644 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.fs.viewfs;
 
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.Function;
 import org.apache.hadoop.util.Preconditions;
 import java.io.FileNotFoundException;
@@ -55,13 +59,13 @@
  * @param <T> is AbstractFileSystem or FileSystem
  *
  * The two main methods are
- * {@link #InodeTree(Configuration, String)} // constructor
+ * {@link #InodeTree(Configuration, String, URI, boolean)} // constructor
  * {@link #resolve(String, boolean)}
  */
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-abstract class InodeTree<T> {
+public abstract class InodeTree<T> {
   private static final Logger LOGGER =
       LoggerFactory.getLogger(InodeTree.class.getName());
 
@@ -81,7 +85,9 @@ enum ResultKind {
   private List<RegexMountPoint<T>> regexMountPointList =
       new ArrayList<RegexMountPoint<T>>();
 
-  static class MountPoint<T> {
+  private final boolean isNestedMountPointSupported;
+
+  public static class MountPoint<T> {
     String src;
     INodeLink<T> target;
 
@@ -89,6 +95,22 @@ static class MountPoint<T> {
       src = srcPath;
       target = mountLink;
     }
+
+    /**
+     * Returns the source of mount point.
+     * @return The source
+     */
+    public String getSource() {
+      return this.src;
+    }
+
+    /**
+     * Returns the target INode link.
+     * @return The target INode link
+     */
+    public INodeLink<T> getTarget() {
+      return this.target;
+    }
   }
 
   /**
@@ -122,6 +144,14 @@ public INode(String pathToNode, UserGroupInformation aUgi) {
     boolean isLink() {
       return !isInternalDir();
     }
+
+    /**
+     * Return the link if isLink.
+     * @return will return null, for non links.
+     */
+    INodeLink<T> getLink() {
+      return null;
+    }
   }
 
   /**
@@ -196,6 +226,51 @@ void addLink(final String pathComponent, final INodeLink<T> link)
       }
       children.put(pathComponent, link);
     }
+
+    void addDirLink(final String pathComponent, final INodeDirLink<T> dirLink) {
+      children.put(pathComponent, dirLink);
+    }
+  }
+
+  /**
+   * Internal class to represent an INodeDir which also contains a INodeLink. This is used to support nested mount points
+   * where an INode is internalDir but points to a mount link. The class is a subclass of INodeDir and the semantics are
+   * as follows:
+   * isLink(): true
+   * isInternalDir(): true
+   * @param <T>
+   */
+  static class INodeDirLink<T> extends INodeDir<T> {
+    /**
+     * INodeLink wrapped in the INodeDir
+     */
+    private final INodeLink<T> link;
+
+    INodeDirLink(String pathToNode, UserGroupInformation aUgi, INodeLink<T> link) {
+      super(pathToNode, aUgi);
+      this.link = link;
+    }
+
+    @Override
+    INodeLink<T> getLink() {
+      return link;
+    }
+
+    /**
+     * True because the INodeDirLink also contains a INodeLink
+     */
+    @Override
+    boolean isLink() {
+      return true;
+    }
+
+    /**
+     * True because the INodeDirLink is internal node
+     */
+    @Override
+    boolean isInternalDir() {
+      return true;
+    }
   }
 
   /**
@@ -250,14 +325,14 @@ enum LinkType {
 
    * A merge dir link is  a merge (junction) of links to dirs:
    * example : merge of 2 dirs
-   *     /users -> hdfs:nn1//users
-   *     /users -> hdfs:nn2//users
+   *     /users -&gt; hdfs:nn1//users
+   *     /users -&gt; hdfs:nn2//users
    *
    * For a merge, each target is checked to be dir when created but if target
    * is changed later it is then ignored (a dir with null entries)
    */
-  static class INodeLink<T> extends INode<T> {
-    final URI[] targetDirLinkList;
+  public static class INodeLink<T> extends INode<T> {
+    final String[] targetDirLinkList;
     private T targetFileSystem;   // file system object created from the link.
     // Function to initialize file system. Only applicable for simple links
     private Function<URI, T> fileSystemInitMethod;
@@ -267,7 +342,7 @@ static class INodeLink<T> extends INode<T> {
      * Construct a mergeLink or nfly.
      */
     INodeLink(final String pathToNode, final UserGroupInformation aUgi,
-        final T targetMergeFs, final URI[] aTargetDirLinkList) {
+        final T targetMergeFs, final String[] aTargetDirLinkList) {
       super(pathToNode, aUgi);
       targetFileSystem = targetMergeFs;
       targetDirLinkList = aTargetDirLinkList;
@@ -278,19 +353,21 @@ static class INodeLink<T> extends INode<T> {
      */
     INodeLink(final String pathToNode, final UserGroupInformation aUgi,
         Function<URI, T> createFileSystemMethod,
-        final URI aTargetDirLink) {
+        final String aTargetDirLink) throws URISyntaxException {
       super(pathToNode, aUgi);
       targetFileSystem = null;
-      targetDirLinkList = new URI[1];
-      targetDirLinkList[0] = aTargetDirLink;
+      targetDirLinkList = new String[1];
+      targetDirLinkList[0] = new URI(aTargetDirLink).toString();
       this.fileSystemInitMethod = createFileSystemMethod;
     }
 
     /**
      * Get the target of the link. If a merge link then it returned
      * as "," separated URI list.
+     *
+     * @return the path.
      */
-    Path getTargetLink() {
+    public Path getTargetLink() {
       StringBuilder result = new StringBuilder(targetDirLinkList[0].toString());
       // If merge link, use "," as separator between the merged URIs
       for (int i = 1; i < targetDirLinkList.length; ++i) {
@@ -304,10 +381,15 @@ boolean isInternalDir() {
       return false;
     }
 
+    @Override
+    INodeLink<T> getLink() {
+      return this;
+    }
+
     /**
      * Get the instance of FileSystem to use, creating one if needed.
      * @return An Initialized instance of T
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public T getTargetFileSystem() throws IOException {
       if (targetFileSystem != null) {
@@ -320,7 +402,8 @@ public T getTargetFileSystem() throws IOException {
           if (targetFileSystem != null) {
             return targetFileSystem;
           }
-          targetFileSystem = fileSystemInitMethod.apply(targetDirLinkList[0]);
+          targetFileSystem =
+              fileSystemInitMethod.apply(URI.create(targetDirLinkList[0]));
           if (targetFileSystem == null) {
             throw new IOException(
                 "Could not initialize target File System for URI : " +
@@ -359,10 +442,17 @@ private void createLink(final String src, final String target,
         newDir.setInternalDirFs(getTargetFileSystem(newDir));
         nextInode = newDir;
       }
-      if (nextInode.isLink()) {
-        // Error - expected a dir but got a link
-        throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
-            " already exists as link");
+      if (!nextInode.isInternalDir()) {
+        if (isNestedMountPointSupported) {
+          // nested mount detected, add a new INodeDirLink that wraps existing INodeLink to INodeTree and override existing INodelink
+          INodeDirLink<T> dirLink = new INodeDirLink<T>(nextInode.fullPath, aUgi, (INodeLink<T>) nextInode);
+          curInode.addDirLink(iPath, dirLink);
+          curInode = dirLink;
+        } else {
+          // Error - expected a dir but got a link
+          throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
+              " already exists as link");
+        }
       } else {
         assert(nextInode.isInternalDir());
         curInode = (INodeDir<T>) nextInode;
@@ -388,7 +478,7 @@ private void createLink(final String src, final String target,
     switch (linkType) {
     case SINGLE:
       newLink = new INodeLink<T>(fullPath, aUgi,
-          initAndGetTargetFs(), new URI(target));
+          initAndGetTargetFs(), target);
       break;
     case SINGLE_FALLBACK:
     case MERGE_SLASH:
@@ -397,10 +487,10 @@ private void createLink(final String src, final String target,
       throw new IllegalArgumentException("Unexpected linkType: " + linkType);
     case MERGE:
     case NFLY:
-      final URI[] targetUris = StringUtils.stringToURI(
-          StringUtils.getStrings(target));
+      final String[] targetUris = StringUtils.getStrings(target);
       newLink = new INodeLink<T>(fullPath, aUgi,
-            getTargetFileSystem(settings, targetUris), targetUris);
+          getTargetFileSystem(settings, StringUtils.stringToURI(targetUris)),
+          targetUris);
       break;
     default:
       throw new IllegalArgumentException(linkType + ": Infeasible linkType");
@@ -412,7 +502,7 @@ private void createLink(final String src, final String target,
   /**
    * The user of this class must subclass and implement the following
    * 3 abstract methods.
-   * @throws IOException
+   * @return Function.
    */
   protected abstract Function<URI, T> initAndGetTargetFs();
 
@@ -428,7 +518,7 @@ private INodeDir<T> getRootDir() {
   }
 
   private INodeLink<T> getRootLink() {
-    Preconditions.checkState(root.isLink());
+    Preconditions.checkState(!root.isInternalDir());
     return (INodeLink<T>)root;
   }
 
@@ -441,11 +531,11 @@ private boolean hasFallbackLink() {
    * there will be root to root mapping. So, root does not represent as
    * internalDir.
    */
-  protected boolean isRootInternalDir() {
+  public boolean isRootInternalDir() {
     return root.isInternalDir();
   }
 
-  protected INodeLink<T> getRootFallbackLink() {
+  public INodeLink<T> getRootFallbackLink() {
     Preconditions.checkState(root.isInternalDir());
     return rootFallbackLink;
   }
@@ -503,14 +593,21 @@ Configuration getConfig() {
   }
 
   /**
-   * Create Inode Tree from the specified mount-table specified in Config
-   * @param config - the mount table keys are prefixed with
-   *       FsConstants.CONFIG_VIEWFS_PREFIX
-   * @param viewName - the name of the mount table - if null use defaultMT name
-   * @throws UnsupportedFileSystemException
-   * @throws URISyntaxException
-   * @throws FileAlreadyExistsException
-   * @throws IOException
+   * Create Inode Tree from the specified mount-table specified in Config.
+   *
+   * @param config the mount table keys are prefixed with
+   *               FsConstants.CONFIG_VIEWFS_PREFIX.
+   * @param viewName the name of the mount table
+   *                 if null use defaultMT name.
+   * @param theUri heUri.
+   * @param initingUriAsFallbackOnNoMounts initingUriAsFallbackOnNoMounts.
+   * @throws UnsupportedFileSystemException file system for <code>uri</code> is
+   *                                        not found.
+   * @throws URISyntaxException if the URI does not have an authority
+   *                            it is badly formed.
+   * @throws FileAlreadyExistsException there is a file at the path specified
+   *                                    or is discovered on one of its ancestors.
+   * @throws IOException raised on errors performing I/O.
    */
   protected InodeTree(final Configuration config, final String viewName,
       final URI theUri, boolean initingUriAsFallbackOnNoMounts)
@@ -521,6 +618,7 @@ protected InodeTree(final Configuration config, final String viewName,
       mountTableName = ConfigUtil.getDefaultMountTableName(config);
     }
     homedirPrefix = ConfigUtil.getHomeDirValue(config, mountTableName);
+    isNestedMountPointSupported = ConfigUtil.isNestedMountPointSupported(config);
 
     boolean isMergeSlashConfigured = false;
     String mergeSlashTarget = null;
@@ -617,8 +715,7 @@ protected InodeTree(final Configuration config, final String viewName,
     if (isMergeSlashConfigured) {
       Preconditions.checkNotNull(mergeSlashTarget);
       root = new INodeLink<T>(mountTableName, ugi,
-          initAndGetTargetFs(),
-          new URI(mergeSlashTarget));
+          initAndGetTargetFs(), mergeSlashTarget);
       mountPoints.add(new MountPoint<T>("/", (INodeLink<T>) root));
       rootFallbackLink = null;
     } else {
@@ -626,7 +723,8 @@ protected InodeTree(final Configuration config, final String viewName,
       getRootDir().setInternalDirFs(getTargetFileSystem(getRootDir()));
       getRootDir().setRoot(true);
       INodeLink<T> fallbackLink = null;
-      for (LinkEntry le : linkEntries) {
+
+      for (LinkEntry le : getLinkEntries(linkEntries)) {
         switch (le.getLinkType()) {
         case SINGLE_FALLBACK:
           if (fallbackLink != null) {
@@ -636,7 +734,7 @@ protected InodeTree(final Configuration config, final String viewName,
                 + "not allowed.");
           }
           fallbackLink = new INodeLink<T>(mountTableName, ugi,
-              initAndGetTargetFs(), new URI(le.getTarget()));
+              initAndGetTargetFs(), le.getTarget());
           continue;
         case REGEX:
           addRegexMountEntry(le);
@@ -661,11 +759,37 @@ protected InodeTree(final Configuration config, final String viewName,
           .info("Empty mount table detected for {} and considering itself "
               + "as a linkFallback.", theUri);
       rootFallbackLink = new INodeLink<T>(mountTableName, ugi,
-          initAndGetTargetFs(), theUri);
+          initAndGetTargetFs(), theUri.toString());
       getRootDir().addFallbackLink(rootFallbackLink);
     }
   }
 
+  /**
+   * Get collection of linkEntry. Sort mount point based on alphabetical order of the src paths.
+   * The purpose is to group nested paths(shortest path always comes first) during INodeTree creation.
+   * E.g. /foo is nested with /foo/bar so an INodeDirLink will be created at /foo.
+   * @param linkEntries input linkEntries
+   * @return sorted linkEntries
+   */
+  private Collection<LinkEntry> getLinkEntries(List<LinkEntry> linkEntries) {
+    Set<LinkEntry> sortedLinkEntries = new TreeSet<>(new Comparator<LinkEntry>() {
+      @Override
+      public int compare(LinkEntry o1, LinkEntry o2) {
+        if (o1 == null) {
+          return -1;
+        }
+        if (o2 == null) {
+          return 1;
+        }
+        String src1 = o1.getSrc();
+        String src2=  o2.getSrc();
+        return src1.compareTo(src2);
+      }
+    });
+    sortedLinkEntries.addAll(linkEntries);
+    return sortedLinkEntries;
+  }
+
   private void checkMntEntryKeyEqualsTarget(
       String mntEntryKey, String targetMntEntryKey) throws IOException {
     if (!mntEntryKey.equals(targetMntEntryKey)) {
@@ -726,7 +850,7 @@ private LinkEntry buildLinkRegexEntry(
    * If the input pathname leads to an internal mount-table entry then
    * the target file system is one that represents the internal inode.
    */
-  static class ResolveResult<T> {
+  public static class ResolveResult<T> {
     final ResultKind kind;
     final T targetFileSystem;
     final String resolvedPath;
@@ -757,11 +881,11 @@ boolean isLastInternalDirLink() {
   /**
    * Resolve the pathname p relative to root InodeDir.
    * @param p - input path
-   * @param resolveLastComponent
+   * @param resolveLastComponent resolveLastComponent.
    * @return ResolveResult which allows further resolution of the remaining path
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
-  ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
+  public ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
       throws IOException {
     ResolveResult<T> resolveResult = null;
     String[] path = breakIntoPathComponents(p);
@@ -779,7 +903,7 @@ ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
      * been linked to the root directory of a file system.
      * The first non-slash path component should be name of the mount table.
      */
-    if (root.isLink()) {
+    if (!root.isInternalDir()) {
       Path remainingPath;
       StringBuilder remainingPathStr = new StringBuilder();
       // ignore first slash
@@ -802,10 +926,17 @@ ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
     }
 
     int i;
+    INodeDirLink<T> lastResolvedDirLink = null;
+    int lastResolvedDirLinkIndex = -1;
     // ignore first slash
     for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
       INode<T> nextInode = curInode.resolveInternal(path[i]);
       if (nextInode == null) {
+        // first resolve to dirlink for nested mount point
+        if (isNestedMountPointSupported && lastResolvedDirLink != null) {
+          return new ResolveResult<T>(ResultKind.EXTERNAL_DIR, lastResolvedDirLink.getLink().getTargetFileSystem(),
+              lastResolvedDirLink.fullPath, getRemainingPath(path, i),true);
+        }
         if (hasFallbackLink()) {
           resolveResult = new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
               getRootFallbackLink().getTargetFileSystem(), root.fullPath,
@@ -821,59 +952,67 @@ ResolveResult<T> resolve(final String p, final boolean resolveLastComponent)
         }
       }
 
-      if (nextInode.isLink()) {
+      if (!nextInode.isInternalDir()) {
         final INodeLink<T> link = (INodeLink<T>) nextInode;
-        final Path remainingPath;
-        if (i >= path.length - 1) {
-          remainingPath = SlashPath;
-        } else {
-          StringBuilder remainingPathStr =
-              new StringBuilder("/" + path[i + 1]);
-          for (int j = i + 2; j < path.length; ++j) {
-            remainingPathStr.append('/').append(path[j]);
-          }
-          remainingPath = new Path(remainingPathStr.toString());
-        }
+        final Path remainingPath = getRemainingPath(path, i + 1);
         resolveResult = new ResolveResult<T>(ResultKind.EXTERNAL_DIR,
             link.getTargetFileSystem(), nextInode.fullPath, remainingPath,
             true);
         return resolveResult;
-      } else if (nextInode.isInternalDir()) {
+      } else {
         curInode = (INodeDir<T>) nextInode;
+        // track last resolved nest mount point.
+        if (isNestedMountPointSupported && nextInode.isLink()) {
+          lastResolvedDirLink = (INodeDirLink<T>) nextInode;
+          lastResolvedDirLinkIndex = i;
+        }
       }
     }
 
-    // We have resolved to an internal dir in mount table.
     Path remainingPath;
-    if (resolveLastComponent) {
+    if (isNestedMountPointSupported && lastResolvedDirLink != null) {
+      remainingPath = getRemainingPath(path, lastResolvedDirLinkIndex + 1);
+      resolveResult = new ResolveResult<T>(ResultKind.EXTERNAL_DIR, lastResolvedDirLink.getLink().getTargetFileSystem(),
+          lastResolvedDirLink.fullPath, remainingPath,true);
+    } else {
+      remainingPath = resolveLastComponent ? SlashPath : getRemainingPath(path, i);
+      resolveResult = new ResolveResult<T>(ResultKind.INTERNAL_DIR, curInode.getInternalDirFs(),
+          curInode.fullPath, remainingPath, false);
+    }
+    return resolveResult;
+  }
+
+  /**
+   * Return remaining path from specified index to the end of the path array.
+   * @param path An array of path components split by slash
+   * @param startIndex the specified start index of the path array
+   * @return remaining path.
+   */
+  private Path getRemainingPath(String[] path, int startIndex) {
+    Path remainingPath;
+    if (startIndex >= path.length) {
       remainingPath = SlashPath;
     } else {
-      // note we have taken care of when path is "/" above
-      // for internal dirs rem-path does not start with / since the lookup
-      // that follows will do a children.get(remaningPath) and will have to
-      // strip-out the initial /
-      StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
-      for (int j = i + 1; j < path.length; ++j) {
-        remainingPathStr.append('/').append(path[j]);
+      StringBuilder remainingPathStr = new StringBuilder();
+      for (int j = startIndex; j < path.length; j++) {
+        remainingPathStr.append("/").append(path[j]);
       }
       remainingPath = new Path(remainingPathStr.toString());
     }
-    resolveResult = new ResolveResult<T>(ResultKind.INTERNAL_DIR,
-        curInode.getInternalDirFs(), curInode.fullPath, remainingPath, false);
-    return resolveResult;
+    return remainingPath;
   }
 
   /**
    * Walk through all regex mount points to see
    * whether the path match any regex expressions.
-   *  E.g. link: ^/user/(?<username>\\w+) => s3://$user.apache.com/_${user}
+   *  E.g. link: ^/user/(?&lt;username&gt;\\w+) =&gt; s3://$user.apache.com/_${user}
    *  srcPath: is /user/hadoop/dir1
    *  resolveLastComponent: true
    *  then return value is s3://hadoop.apache.com/_hadoop
    *
-   * @param srcPath
-   * @param resolveLastComponent
-   * @return
+   * @param srcPath srcPath.
+   * @param resolveLastComponent resolveLastComponent.
+   * @return ResolveResult.
    */
   protected ResolveResult<T> tryResolveInRegexMountpoint(final String srcPath,
       final boolean resolveLastComponent) {
@@ -891,7 +1030,7 @@ protected ResolveResult<T> tryResolveInRegexMountpoint(final String srcPath,
    * Build resolve result.
    * Here's an example
    * Mountpoint: fs.viewfs.mounttable.mt
-   *     .linkRegex.replaceresolveddstpath:_:-#.^/user/(?<username>\w+)
+   *     .linkRegex.replaceresolveddstpath:_:-#.^/user/(??&lt;username&gt;\w+)
    * Value: /targetTestRoot/$username
    * Dir path to test:
    * viewfs://mt/user/hadoop_user1/hadoop_dir1
@@ -900,6 +1039,10 @@ protected ResolveResult<T> tryResolveInRegexMountpoint(final String srcPath,
    * targetOfResolvedPathStr: /targetTestRoot/hadoop-user1
    * remainingPath: /hadoop_dir1
    *
+   * @param resultKind resultKind.
+   * @param resolvedPathStr resolvedPathStr.
+   * @param targetOfResolvedPathStr targetOfResolvedPathStr.
+   * @param remainingPath remainingPath.
    * @return targetFileSystem or null on exceptions.
    */
   protected ResolveResult<T> buildResolveResultForRegexMountPoint(
@@ -932,7 +1075,7 @@ protected ResolveResult<T> buildResolveResultForRegexMountPoint(
     }
   }
 
-  List<MountPoint<T>> getMountPoints() {
+  public List<MountPoint<T>> getMountPoints() {
     return mountPoints;
   }
 
@@ -941,7 +1084,7 @@ List<MountPoint<T>> getMountPoints() {
    * @return home dir value from mount table; null if no config value
    * was found.
    */
-  String getHomeDirPrefixValue() {
+  public String getHomeDirPrefixValue() {
     return homedirPrefix;
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java
index bc2c3ea93c58c..5fcd77cd29155 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java
@@ -38,6 +38,7 @@ public interface MountTableConfigLoader {
    *          a directory in the case of multiple versions of mount-table
    *          files(Recommended option).
    * @param conf - Configuration object to add mount table.
+   * @throws IOException raised on errors performing I/O.
    */
   void load(String mountTableConfigPath, Configuration conf)
       throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index ce918a13ea2ac..e31a701a6eaa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -25,6 +25,8 @@
 import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
 import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
 import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
+import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT;
+import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT;
 
 import java.util.function.Function;
 import java.io.FileNotFoundException;
@@ -105,6 +107,8 @@ static AccessControlException readOnlyMountTable(final String operation,
 
   /**
    * Gets file system creator instance.
+   *
+   * @return fs getter.
    */
   protected FsGetter fsGetter() {
     return new FsGetter();
@@ -214,11 +218,11 @@ public static class MountPoint {
     /**
      * Array of target FileSystem URIs.
      */
-    private final URI[] targetFileSystemURIs;
+    private final String[] targetFileSystemPaths;
 
-    MountPoint(Path srcPath, URI[] targetFs) {
+    MountPoint(Path srcPath, String[] targetFs) {
       mountedOnPath = srcPath;
-      targetFileSystemURIs = targetFs;
+      targetFileSystemPaths = targetFs;
     }
 
     public Path getMountedOnPath() {
@@ -226,7 +230,15 @@ public Path getMountedOnPath() {
     }
 
     public URI[] getTargetFileSystemURIs() {
-      return targetFileSystemURIs;
+      URI[] targetUris = new URI[targetFileSystemPaths.length];
+      for (int i = 0; i < targetFileSystemPaths.length; i++) {
+        targetUris[i] = URI.create(targetFileSystemPaths[i]);
+      }
+      return targetUris;
+    }
+
+    public String[] getTargetFileSystemPaths() {
+      return targetFileSystemPaths;
     }
   }
 
@@ -243,9 +255,9 @@ public URI[] getTargetFileSystemURIs() {
   private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT;
   /**
    * Make the path Absolute and get the path-part of a pathname.
-   * Checks that URI matches this file system 
+   * Checks that URI matches this file system
    * and that the path-part is a valid name.
-   * 
+   *
    * @param p path
    * @return path-part of the Path p
    */
@@ -253,17 +265,17 @@ String getUriPath(final Path p) {
     checkPath(p);
     return makeAbsolute(p).toUri().getPath();
   }
-  
+
   private Path makeAbsolute(final Path f) {
     return f.isAbsolute() ? f : new Path(workingDir, f);
   }
-  
+
   /**
    * This is the  constructor with the signature needed by
    * {@link FileSystem#createFileSystem(URI, Configuration)}
-   * 
+   *
    * After this constructor is called initialize() is called.
-   * @throws IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public ViewFileSystem() throws IOException {
     ugi = UserGroupInformation.getCurrentUser();
@@ -372,31 +384,31 @@ protected FileSystem getTargetFileSystem(final String settings,
   }
 
   /**
-   * Convenience Constructor for apps to call directly
+   * Convenience Constructor for apps to call directly.
    * @param theUri which must be that of ViewFileSystem
-   * @param conf
-   * @throws IOException
+   * @param conf conf configuration.
+   * @throws IOException raised on errors performing I/O.
    */
   ViewFileSystem(final URI theUri, final Configuration conf)
       throws IOException {
     this();
     initialize(theUri, conf);
   }
-  
+
   /**
-   * Convenience Constructor for apps to call directly
-   * @param conf
-   * @throws IOException
+   * Convenience Constructor for apps to call directly.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
    */
   public ViewFileSystem(final Configuration conf) throws IOException {
     this(FsConstants.VIEWFS_URI, conf);
   }
-  
+
   @Override
   public URI getUri() {
     return myUri;
   }
-  
+
   @Override
   public Path resolvePath(final Path f) throws IOException {
     final InodeTree.ResolveResult<FileSystem> res;
@@ -406,7 +418,7 @@ public Path resolvePath(final Path f) throws IOException {
     }
     return res.targetFileSystem.resolvePath(res.remainingPath);
   }
-  
+
   @Override
   public Path getHomeDirectory() {
     if (homeDir == null) {
@@ -414,13 +426,13 @@ public Path getHomeDirectory() {
       if (base == null) {
         base = "/user";
       }
-      homeDir = (base.equals("/") ? 
+      homeDir = (base.equals("/") ?
           this.makeQualified(new Path(base + ugi.getShortUserName())):
           this.makeQualified(new Path(base + "/" + ugi.getShortUserName())));
     }
     return homeDir;
   }
-  
+
   @Override
   public Path getWorkingDirectory() {
     return workingDir;
@@ -431,11 +443,11 @@ public void setWorkingDirectory(final Path new_dir) {
     getUriPath(new_dir); // this validates the path
     workingDir = makeAbsolute(new_dir);
   }
-  
+
   @Override
   public FSDataOutputStream append(final Path f, final int bufferSize,
       final Progressable progress) throws IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.append(res.remainingPath, bufferSize, progress);
   }
@@ -454,7 +466,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
     return res.targetFileSystem.createNonRecursive(res.remainingPath,
         permission, flags, bufferSize, replication, blockSize, progress);
   }
-  
+
   @Override
   public FSDataOutputStream create(final Path f, final FsPermission permission,
       final boolean overwrite, final int bufferSize, final short replication,
@@ -470,11 +482,11 @@ public FSDataOutputStream create(final Path f, final FsPermission permission,
         overwrite, bufferSize, replication, blockSize, progress);
   }
 
-  
+
   @Override
   public boolean delete(final Path f, final boolean recursive)
       throws AccessControlException, FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
         fsState.resolve(getUriPath(f), true);
     // If internal dir or target is a mount link (ie remainingPath is Slash)
     if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) {
@@ -482,18 +494,18 @@ public boolean delete(final Path f, final boolean recursive)
     }
     return res.targetFileSystem.delete(res.remainingPath, recursive);
   }
-  
+
   @Override
   @SuppressWarnings("deprecation")
   public boolean delete(final Path f)
       throws AccessControlException, FileNotFoundException, IOException {
     return delete(f, true);
   }
-  
+
   @Override
-  public BlockLocation[] getFileBlockLocations(FileStatus fs, 
+  public BlockLocation[] getFileBlockLocations(FileStatus fs,
       long start, long len) throws IOException {
-    final InodeTree.ResolveResult<FileSystem> res = 
+    final InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(fs.getPath()), true);
     return res.targetFileSystem.getFileBlockLocations(
         new ViewFsFileStatus(fs, res.remainingPath), start, len);
@@ -503,7 +515,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus fs,
   public FileChecksum getFileChecksum(final Path f)
       throws AccessControlException, FileNotFoundException,
       IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.getFileChecksum(res.remainingPath);
   }
@@ -560,7 +572,7 @@ public FileStatus getFileStatus(final Path f) throws AccessControlException,
     FileStatus status =  res.targetFileSystem.getFileStatus(res.remainingPath);
     return fixFileStatus(status, this.makeQualified(f));
   }
-  
+
   @Override
   public void access(Path path, FsAction mode) throws AccessControlException,
       FileNotFoundException, IOException {
@@ -601,7 +613,7 @@ public FileStatus[] listStatus(final Path f) throws AccessControlException,
       FileNotFoundException, IOException {
     InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    
+
     FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
     if (!res.isInternalDir()) {
       // We need to change the name in the FileStatus as described in
@@ -665,7 +677,7 @@ public boolean mkdirs(Path dir) throws IOException {
   @Override
   public boolean mkdirs(final Path dir, final FsPermission permission)
       throws IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
         fsState.resolve(getUriPath(dir), false);
     return res.targetFileSystem.mkdirs(res.remainingPath, permission);
   }
@@ -673,15 +685,15 @@ public boolean mkdirs(final Path dir, final FsPermission permission)
   @Override
   public FSDataInputStream open(final Path f, final int bufferSize)
       throws AccessControlException, FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
         fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.open(res.remainingPath, bufferSize);
   }
 
-  
+
   @Override
   public boolean rename(final Path src, final Path dst) throws IOException {
-    // passing resolveLastComponet as false to catch renaming a mount point to 
+    // passing resolveLastComponet as false to catch renaming a mount point to
     // itself. We need to catch this as an internal operation and fail if no
     // fallback.
     InodeTree.ResolveResult<FileSystem> resSrc =
@@ -792,28 +804,28 @@ public boolean truncate(final Path f, final long newLength)
         fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.truncate(res.remainingPath, newLength);
   }
-  
+
   @Override
   public void setOwner(final Path f, final String username,
       final String groupname) throws AccessControlException,
       FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setOwner(res.remainingPath, username, groupname); 
+    res.targetFileSystem.setOwner(res.remainingPath, username, groupname);
   }
 
   @Override
   public void setPermission(final Path f, final FsPermission permission)
       throws AccessControlException, FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setPermission(res.remainingPath, permission); 
+    res.targetFileSystem.setPermission(res.remainingPath, permission);
   }
 
   @Override
   public boolean setReplication(final Path f, final short replication)
       throws AccessControlException, FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.setReplication(res.remainingPath, replication);
   }
@@ -821,9 +833,9 @@ public boolean setReplication(final Path f, final short replication)
   @Override
   public void setTimes(final Path f, final long mtime, final long atime)
       throws AccessControlException, FileNotFoundException, IOException {
-    InodeTree.ResolveResult<FileSystem> res = 
+    InodeTree.ResolveResult<FileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
+    res.targetFileSystem.setTimes(res.remainingPath, mtime, atime);
   }
 
   @Override
@@ -917,13 +929,9 @@ public void removeXAttr(Path path, String name) throws IOException {
   }
 
   @Override
-  public void setVerifyChecksum(final boolean verifyChecksum) { 
-    List<InodeTree.MountPoint<FileSystem>> mountPoints = 
-        fsState.getMountPoints();
-    Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
-    for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
-      fsMap.get(mount.src).setVerifyChecksum(verifyChecksum);
-    }
+  public void setVerifyChecksum(final boolean verifyChecksum) {
+    // This is a file system level operations, however ViewFileSystem
+    // points to many file systems. Noop for ViewFileSystem.
   }
 
   /**
@@ -949,7 +957,7 @@ private Map<String, FileSystem> initializeMountedFileSystems(
     }
     return fsMap;
   }
-  
+
   @Override
   public long getDefaultBlockSize() {
     throw new NotInMountpointException("getDefaultBlockSize");
@@ -972,7 +980,7 @@ public long getDefaultBlockSize(Path f) {
         fsState.resolve(getUriPath(f), true);
       return res.targetFileSystem.getDefaultBlockSize(res.remainingPath);
     } catch (FileNotFoundException e) {
-      throw new NotInMountpointException(f, "getDefaultBlockSize"); 
+      throw new NotInMountpointException(f, "getDefaultBlockSize");
     } catch (IOException e) {
       throw new RuntimeException("Not able to initialize fs in "
           + " getDefaultBlockSize for path " + f + " with exception", e);
@@ -986,7 +994,7 @@ public short getDefaultReplication(Path f) {
         fsState.resolve(getUriPath(f), true);
       return res.targetFileSystem.getDefaultReplication(res.remainingPath);
     } catch (FileNotFoundException e) {
-      throw new NotInMountpointException(f, "getDefaultReplication"); 
+      throw new NotInMountpointException(f, "getDefaultReplication");
     } catch (IOException e) {
       throw new RuntimeException("Not able to initialize fs in "
           + " getDefaultReplication for path " + f + " with exception", e);
@@ -1019,13 +1027,9 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException {
   }
 
   @Override
-  public void setWriteChecksum(final boolean writeChecksum) { 
-    List<InodeTree.MountPoint<FileSystem>> mountPoints = 
-        fsState.getMountPoints();
-    Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
-    for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
-      fsMap.get(mount.src).setWriteChecksum(writeChecksum);
-    }
+  public void setWriteChecksum(final boolean writeChecksum) {
+    // This is a file system level operations, however ViewFileSystem
+    // points to many file systems. Noop for ViewFileSystem.
   }
 
   @Override
@@ -1033,7 +1037,7 @@ public FileSystem[] getChildFileSystems() {
     List<InodeTree.MountPoint<FileSystem>> mountPoints =
         fsState.getMountPoints();
     Map<String, FileSystem> fsMap = initializeMountedFileSystems(mountPoints);
-    Set<FileSystem> children = new HashSet<FileSystem>();
+    Set<FileSystem> children = new HashSet<>();
     for (InodeTree.MountPoint<FileSystem> mountPoint : mountPoints) {
       FileSystem targetFs = fsMap.get(mountPoint.src);
       children.addAll(Arrays.asList(targetFs.getChildFileSystems()));
@@ -1052,11 +1056,11 @@ public FileSystem[] getChildFileSystems() {
     }
     return children.toArray(new FileSystem[]{});
   }
-  
+
   public MountPoint[] getMountPoints() {
-    List<InodeTree.MountPoint<FileSystem>> mountPoints = 
+    List<InodeTree.MountPoint<FileSystem>> mountPoints =
                   fsState.getMountPoints();
-    
+
     MountPoint[] result = new MountPoint[mountPoints.size()];
     for ( int i = 0; i < mountPoints.size(); ++i ) {
       result[i] = new MountPoint(new Path(mountPoints.get(i).src),
@@ -1138,16 +1142,79 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
    * Get the trash root directory for current user when the path
    * specified is deleted.
    *
+   * If FORCE_INSIDE_MOUNT_POINT flag is not set, return the default trash root
+   * from targetFS.
+   *
+   * When FORCE_INSIDE_MOUNT_POINT is set to true,
+   * <ol>
+   *   <li>
+   *     If the trash root for path p is in the same mount point as path p,
+   *       and one of:
+   *       <ol>
+   *         <li>The mount point isn't at the top of the target fs.</li>
+   *         <li>The resolved path of path is root (in fallback FS).</li>
+   *         <li>The trash isn't in user's target fs home directory
+   *            get the corresponding viewFS path for the trash root and return
+   *            it.
+   *         </li>
+   *       </ol>
+   *   </li>
+   *   <li>
+   *     else, return the trash root under the root of the mount point
+   *     (/{mntpoint}/.Trash/{user}).
+   *   </li>
+   * </ol>
+   *
+   * These conditions handle several different important cases:
+   * <ul>
+   *   <li>File systems may need to have more local trash roots, such as
+   *         encryption zones or snapshot roots.</li>
+   *   <li>The fallback mount should use the user's home directory.</li>
+   *   <li>Cloud storage systems should not use trash in an implicity defined
+   *        home directory, per a container, unless it is the fallback fs.</li>
+   * </ul>
+   *
    * @param path the trash root of the path to be determined.
    * @return the trash root path.
    */
   @Override
   public Path getTrashRoot(Path path) {
+
     try {
       InodeTree.ResolveResult<FileSystem> res =
           fsState.resolve(getUriPath(path), true);
-      return res.targetFileSystem.getTrashRoot(res.remainingPath);
-    } catch (Exception e) {
+      Path targetFSTrashRoot =
+          res.targetFileSystem.getTrashRoot(res.remainingPath);
+
+      // Allow clients to use old behavior of delegating to target fs.
+      if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT,
+          CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) {
+        return targetFSTrashRoot;
+      }
+
+      // The trash root path from the target fs
+      String targetFSTrashRootPath = targetFSTrashRoot.toUri().getPath();
+      // The mount point path in the target fs
+      String mountTargetPath = res.targetFileSystem.getUri().getPath();
+      if (!mountTargetPath.endsWith("/")) {
+        mountTargetPath = mountTargetPath + "/";
+      }
+
+      Path targetFsUserHome = res.targetFileSystem.getHomeDirectory();
+      if (targetFSTrashRootPath.startsWith(mountTargetPath) &&
+          !(mountTargetPath.equals(ROOT_PATH.toString()) &&
+              !res.resolvedPath.equals(ROOT_PATH.toString()) &&
+              (targetFsUserHome != null && targetFSTrashRootPath.startsWith(
+                  targetFsUserHome.toUri().getPath())))) {
+        String relativeTrashRoot =
+            targetFSTrashRootPath.substring(mountTargetPath.length());
+        return makeQualified(new Path(res.resolvedPath, relativeTrashRoot));
+      } else {
+        // Return the trash root for the mount point.
+        return makeQualified(new Path(res.resolvedPath,
+            TRASH_PREFIX + "/" + ugi.getShortUserName()));
+      }
+    } catch (IOException | IllegalArgumentException e) {
       throw new NotInMountpointException(path, "getTrashRoot");
     }
   }
@@ -1155,16 +1222,78 @@ public Path getTrashRoot(Path path) {
   /**
    * Get all the trash roots for current user or all users.
    *
+   * When FORCE_INSIDE_MOUNT_POINT is set to true, we also return trash roots
+   * under the root of each mount point, with their viewFS paths.
+   *
    * @param allUsers return trash roots for all users if true.
    * @return all Trash root directories.
    */
   @Override
   public Collection<FileStatus> getTrashRoots(boolean allUsers) {
-    List<FileStatus> trashRoots = new ArrayList<>();
+    // A map from targetFSPath -> FileStatus.
+    // FileStatus can be from targetFS or viewFS.
+    HashMap<Path, FileStatus> trashRoots = new HashMap<>();
     for (FileSystem fs : getChildFileSystems()) {
-      trashRoots.addAll(fs.getTrashRoots(allUsers));
+      for (FileStatus trash : fs.getTrashRoots(allUsers)) {
+        trashRoots.put(trash.getPath(), trash);
+      }
+    }
+
+    // Return trashRoots if FORCE_INSIDE_MOUNT_POINT is disabled.
+    if (!config.getBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT,
+        CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT_DEFAULT)) {
+      return trashRoots.values();
     }
-    return trashRoots;
+
+    // Get trash roots in TRASH_PREFIX dir inside mount points and fallback FS.
+    List<InodeTree.MountPoint<FileSystem>> mountPoints =
+        fsState.getMountPoints();
+    // If we have a fallback FS, add a mount point for it as <"", fallback FS>.
+    // The source path of a mount point shall not end with '/', thus for
+    // fallback fs, we set its mount point src as "".
+    if (fsState.getRootFallbackLink() != null) {
+      mountPoints.add(new InodeTree.MountPoint<>("",
+          fsState.getRootFallbackLink()));
+    }
+
+    try {
+      for (InodeTree.MountPoint<FileSystem> mountPoint : mountPoints) {
+
+        Path trashRoot =
+            makeQualified(new Path(mountPoint.src + "/" + TRASH_PREFIX));
+
+        // Continue if trashRoot does not exist for this mount point
+        if (!exists(trashRoot)) {
+          continue;
+        }
+
+        FileSystem targetFS = mountPoint.target.getTargetFileSystem();
+        if (!allUsers) {
+          Path userTrashRoot = new Path(trashRoot, ugi.getShortUserName());
+          if (exists(userTrashRoot)) {
+            Path targetFSUserTrashRoot = targetFS.makeQualified(
+                new Path(targetFS.getUri().getPath(),
+                    TRASH_PREFIX + "/" + ugi.getShortUserName()));
+            trashRoots.put(targetFSUserTrashRoot, getFileStatus(userTrashRoot));
+          }
+        } else {
+          FileStatus[] mountPointTrashRoots = listStatus(trashRoot);
+          for (FileStatus trash : mountPointTrashRoots) {
+            // Remove the mountPoint and the leading '/' to get the
+            // relative targetFsTrash path
+            String targetFsTrash = trash.getPath().toUri().getPath()
+                .substring(mountPoint.src.length() + 1);
+            Path targetFsTrashPath = targetFS.makeQualified(
+                new Path(targetFS.getUri().getPath(), targetFsTrash));
+            trashRoots.put(targetFsTrashPath, trash);
+          }
+        }
+      }
+    } catch (IOException e) {
+      LOG.warn("Exception in get all trash roots for mount points", e);
+    }
+
+    return trashRoots.values();
   }
 
   @Override
@@ -1187,7 +1316,7 @@ public FsStatus getStatus(Path p) throws IOException {
    * Constants#CONFIG_VIEWFS_LINK_MERGE_SLASH} is supported and is a valid
    * mount point. Else, throw NotInMountpointException.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public long getUsed() throws IOException {
@@ -1248,9 +1377,9 @@ public boolean hasPathCapability(Path path, String capability)
    * are not allowed.
    * If called on create or mkdir then this target is the parent of the
    * directory in which one is trying to create or mkdir; hence
-   * in this case the path name passed in is the last component. 
+   * in this case the path name passed in is the last component.
    * Otherwise this target is the end point of the path and hence
-   * the path name passed in is null. 
+   * the path name passed in is null.
    */
   static class InternalDirOfViewFs extends FileSystem {
     final InodeTree.INodeDir<FileSystem>  theInternalDir;
@@ -1259,7 +1388,7 @@ static class InternalDirOfViewFs extends FileSystem {
     final URI myUri;
     private final boolean showMountLinksAsSymlinks;
     private InodeTree<FileSystem> fsState;
-    
+
     public InternalDirOfViewFs(final InodeTree.INodeDir<FileSystem> dir,
         final long cTime, final UserGroupInformation ugi, URI uri,
         Configuration config, InodeTree fsState) throws URISyntaxException {
@@ -1284,7 +1413,7 @@ static private void checkPathIsSlash(final Path f) throws IOException {
             "Internal implementation error: expected file name to be /");
       }
     }
-    
+
     @Override
     public URI getUri() {
       return myUri;
@@ -1354,7 +1483,7 @@ public boolean delete(final Path f, final boolean recursive)
       checkPathIsSlash(f);
       throw readOnlyMountTable("delete", f);
     }
-    
+
     @Override
     @SuppressWarnings("deprecation")
     public boolean delete(final Path f)
@@ -1402,7 +1531,7 @@ public FileStatus getFileStatus(Path f) throws IOException {
           new Path(theInternalDir.fullPath).makeQualified(
               myUri, ROOT_PATH));
     }
-    
+
 
     @Override
     public FileStatus[] listStatus(Path f) throws AccessControlException,
@@ -1417,7 +1546,7 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
         INode<FileSystem> inode = iEntry.getValue();
         Path path = new Path(inode.fullPath).makeQualified(myUri, null);
         if (inode.isLink()) {
-          INodeLink<FileSystem> link = (INodeLink<FileSystem>) inode;
+          INodeLink<FileSystem> link = inode.getLink();
 
           if (showMountLinksAsSymlinks) {
             // To maintain backward compatibility, with default option(showing
@@ -1582,11 +1711,6 @@ public boolean mkdirs(Path dir, FsPermission permission)
       throw readOnlyMountTable("mkdirs",  dir);
     }
 
-    @Override
-    public boolean mkdirs(Path dir) throws IOException {
-      return mkdirs(dir, null);
-    }
-
     @Override
     public FSDataInputStream open(Path f, int bufferSize)
         throws AccessControlException, FileNotFoundException, IOException {
@@ -1599,7 +1723,7 @@ public boolean rename(Path src, Path dst) throws AccessControlException,
         IOException {
       checkPathIsSlash(src);
       checkPathIsSlash(dst);
-      throw readOnlyMountTable("rename", src);     
+      throw readOnlyMountTable("rename", src);
     }
 
     @Override
@@ -1618,7 +1742,7 @@ public void setOwner(Path f, String username, String groupname)
     public void setPermission(Path f, FsPermission permission)
         throws AccessControlException, IOException {
       checkPathIsSlash(f);
-      throw readOnlyMountTable("setPermission", f);    
+      throw readOnlyMountTable("setPermission", f);
     }
 
     @Override
@@ -1632,7 +1756,7 @@ public boolean setReplication(Path f, short replication)
     public void setTimes(Path f, long mtime, long atime)
         throws AccessControlException, IOException {
       checkPathIsSlash(f);
-      throw readOnlyMountTable("setTimes", f);    
+      throw readOnlyMountTable("setTimes", f);
     }
 
     @Override
@@ -1644,7 +1768,7 @@ public void setVerifyChecksum(boolean verifyChecksum) {
     public FsServerDefaults getServerDefaults(Path f) throws IOException {
       throw new NotInMountpointException(f, "getServerDefaults");
     }
-    
+
     @Override
     public long getDefaultBlockSize(Path f) {
       throw new NotInMountpointException(f, "getDefaultBlockSize");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
index e91b66512d5bf..1c25a9536e121 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
@@ -139,6 +139,8 @@ public boolean supportAutoAddingFallbackOnNoMounts() {
 
   /**
    * Sets whether to add fallback automatically when no mount points found.
+   *
+   * @param addAutoFallbackOnNoMounts addAutoFallbackOnNoMounts.
    */
   public void setSupportAutoAddingFallbackOnNoMounts(
       boolean addAutoFallbackOnNoMounts) {
@@ -320,7 +322,8 @@ private <T> T newInstance(Class<T> theClass, URI uri, Configuration conf) {
    *
    * @param path - fs uri path
    * @param conf - configuration
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
+   * @return file system.
    */
   public FileSystem getRawFileSystem(Path path, Configuration conf)
       throws IOException {
@@ -339,6 +342,11 @@ public FileSystem getRawFileSystem(Path path, Configuration conf)
   /**
    * Gets the mount path info, which contains the target file system and
    * remaining path to pass to the target file system.
+   *
+   * @param path the path.
+   * @param conf configuration.
+   * @return mount path info.
+   * @throws IOException raised on errors performing I/O.
    */
   public MountPathInfo<FileSystem> getMountPathInfo(Path path,
       Configuration conf) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
index f486a10b4c8f9..c9c6767097b87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemUtil.java
@@ -44,7 +44,7 @@ private ViewFileSystemUtil() {
   /**
    * Check if the FileSystem is a ViewFileSystem.
    *
-   * @param fileSystem
+   * @param fileSystem file system.
    * @return true if the fileSystem is ViewFileSystem
    */
   public static boolean isViewFileSystem(final FileSystem fileSystem) {
@@ -54,7 +54,7 @@ public static boolean isViewFileSystem(final FileSystem fileSystem) {
   /**
    * Check if the FileSystem is a ViewFileSystemOverloadScheme.
    *
-   * @param fileSystem
+   * @param fileSystem file system.
    * @return true if the fileSystem is ViewFileSystemOverloadScheme
    */
   public static boolean isViewFileSystemOverloadScheme(
@@ -101,6 +101,7 @@ public static boolean isViewFileSystemOverloadScheme(
    * @param fileSystem - ViewFileSystem on which mount point exists
    * @param path - URI for which FsStatus is requested
    * @return Map of ViewFsMountPoint and FsStatus
+   * @throws IOException raised on errors performing I/O.
    */
   public static Map<MountPoint, FsStatus> getStatus(
       FileSystem fileSystem, Path path) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index deaf4f4108a3e..5f54c9cdd06aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -80,8 +80,8 @@
 /**
  * ViewFs (extends the AbstractFileSystem interface) implements a client-side
  * mount table. The viewFs file system is implemented completely in memory on
- * the client side. The client-side mount table allows a client to provide a 
- * customized view of a file system namespace that is composed from 
+ * the client side. The client-side mount table allows a client to provide a
+ * customized view of a file system namespace that is composed from
  * one or more individual file systems (a localFs or Hdfs, S3fs, etc).
  * For example one could have a mount table that provides links such as
  * <ul>
@@ -89,26 +89,26 @@
  * <li>  /project/foo   {@literal ->} hdfs://nnProject1/projects/foo
  * <li>  /project/bar   {@literal ->} hdfs://nnProject2/projects/bar
  * <li>  /tmp           {@literal ->} hdfs://nnTmp/privateTmpForUserXXX
- * </ul> 
- * 
- * ViewFs is specified with the following URI: <b>viewfs:///</b> 
+ * </ul>
+ *
+ * ViewFs is specified with the following URI: <b>viewfs:///</b>
  * <p>
  * To use viewfs one would typically set the default file system in the
  * config  (i.e. fs.defaultFS {@literal <} = viewfs:///) along with the
- * mount table config variables as described below. 
- * 
+ * mount table config variables as described below.
+ *
  * <p>
  * <b> ** Config variables to specify the mount table entries ** </b>
  * <p>
- * 
+ *
  * The file system is initialized from the standard Hadoop config through
  * config variables.
- * See {@link FsConstants} for URI and Scheme constants; 
- * See {@link Constants} for config var constants; 
+ * See {@link FsConstants} for URI and Scheme constants;
+ * See {@link Constants} for config var constants;
  * see {@link ConfigUtil} for convenient lib.
- * 
+ *
  * <p>
- * All the mount table config entries for view fs are prefixed by 
+ * All the mount table config entries for view fs are prefixed by
  * <b>fs.viewfs.mounttable.</b>
  * For example the above example can be specified with the following
  *  config variables:
@@ -122,8 +122,8 @@
  *  <li> fs.viewfs.mounttable.default.link./tmp=
  *  hdfs://nnTmp/privateTmpForUserXXX
  *  </ul>
- *  
- * The default mount table (when no authority is specified) is 
+ *
+ * The default mount table (when no authority is specified) is
  * from config variables prefixed by <b>fs.viewFs.mounttable.default </b>
  * The authority component of a URI can be used to specify a different mount
  * table. For example,
@@ -131,11 +131,11 @@
  * <li>  viewfs://sanjayMountable/
  * </ul>
  * is initialized from fs.viewFs.mounttable.sanjayMountable.* config variables.
- * 
- *  <p> 
+ *
+ *  <p>
  *  <b> **** Merge Mounts **** </b>(NOTE: merge mounts are not implemented yet.)
  *  <p>
- *  
+ *
  *   One can also use "MergeMounts" to merge several directories (this is
  *   sometimes  called union-mounts or junction-mounts in the literature.
  *   For example of the home directories are stored on say two file systems
@@ -156,7 +156,7 @@
  *   <li>    fs.viewfs.mounttable.default.linkMergeSlash=hdfs://nn99/
  *   </ul>
  *   In this cases the root of the mount table is merged with the root of
- *            <b>hdfs://nn99/ </b> 
+ *            <b>hdfs://nn99/ </b>
  */
 
 @InterfaceAudience.Public
@@ -182,19 +182,21 @@ static AccessControlException readOnlyMountTable(final String operation,
       final Path p) {
     return readOnlyMountTable(operation, p.toString());
   }
-  
-  
+
+
   static public class MountPoint {
-    private Path src;       // the src of the mount
-    private URI[] targets; //  target of the mount; Multiple targets imply mergeMount
-    MountPoint(Path srcPath, URI[] targetURIs) {
+    // the src of the mount
+    private Path src;
+    // Target of the mount; Multiple targets imply mergeMount
+    private String[] targets;
+    MountPoint(Path srcPath, String[] targetURIs) {
       src = srcPath;
       targets = targetURIs;
     }
     Path getSrc() {
       return src;
     }
-    URI[] getTargets() {
+    String[] getTargets() {
       return targets;
     }
   }
@@ -212,15 +214,15 @@ public ViewFs(final Configuration conf) throws IOException,
       URISyntaxException {
     this(FsConstants.VIEWFS_URI, conf);
   }
-  
+
   /**
    * This constructor has the signature needed by
    * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
-   * 
+   *
    * @param theUri which must be that of ViewFs
    * @param conf
    * @throws IOException
-   * @throws URISyntaxException 
+   * @throws URISyntaxException
    */
   ViewFs(final URI theUri, final Configuration conf) throws IOException,
       URISyntaxException {
@@ -290,7 +292,7 @@ protected AbstractFileSystem getTargetFileSystem(final String settings,
   @Override
   @Deprecated
   public FsServerDefaults getServerDefaults() throws IOException {
-    return LocalConfigKeys.getServerDefaults(); 
+    return LocalConfigKeys.getServerDefaults();
   }
 
   @Override
@@ -308,7 +310,7 @@ public FsServerDefaults getServerDefaults(final Path f) throws IOException {
   public int getUriDefaultPort() {
     return -1;
   }
- 
+
   @Override
   public Path getHomeDirectory() {
     if (homeDir == null) {
@@ -316,13 +318,13 @@ public Path getHomeDirectory() {
       if (base == null) {
         base = "/user";
       }
-      homeDir = (base.equals("/") ? 
+      homeDir = (base.equals("/") ?
         this.makeQualified(new Path(base + ugi.getShortUserName())):
         this.makeQualified(new Path(base + "/" + ugi.getShortUserName())));
     }
     return homeDir;
   }
-  
+
   @Override
   public Path resolvePath(final Path f) throws FileNotFoundException,
           AccessControlException, UnresolvedLinkException, IOException {
@@ -334,7 +336,7 @@ public Path resolvePath(final Path f) throws FileNotFoundException,
     return res.targetFileSystem.resolvePath(res.remainingPath);
 
   }
-  
+
   @Override
   public FSDataOutputStream createInternal(final Path f,
       final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
@@ -365,7 +367,7 @@ public FSDataOutputStream createInternal(final Path f,
   public boolean delete(final Path f, final boolean recursive)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
     // If internal dir or target is a mount link (ie remainingPath is Slash)
     if (res.isInternalDir() || res.remainingPath == InodeTree.SlashPath) {
@@ -379,7 +381,7 @@ public boolean delete(final Path f, final boolean recursive)
   public BlockLocation[] getFileBlockLocations(final Path f, final long start,
       final long len) throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return
       res.targetFileSystem.getFileBlockLocations(res.remainingPath, start, len);
@@ -389,7 +391,7 @@ public BlockLocation[] getFileBlockLocations(final Path f, final long start,
   public FileChecksum getFileChecksum(final Path f)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.getFileChecksum(res.remainingPath);
   }
@@ -405,20 +407,20 @@ public FileChecksum getFileChecksum(final Path f)
   @Override
   public FileStatus getFileStatus(final Path f) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
 
-    //  FileStatus#getPath is a fully qualified path relative to the root of 
+    //  FileStatus#getPath is a fully qualified path relative to the root of
     // target file system.
     // We need to change it to viewfs URI - relative to root of mount table.
-    
+
     // The implementors of RawLocalFileSystem were trying to be very smart.
     // They implement FileStatus#getOwener lazily -- the object
     // returned is really a RawLocalFileSystem that expect the
     // FileStatus#getPath to be unchanged so that it can get owner when needed.
     // Hence we need to interpose a new ViewFsFileStatus that works around.
-    
-    
+
+
     FileStatus status =  res.targetFileSystem.getFileStatus(res.remainingPath);
     return new ViewFsFileStatus(status, this.makeQualified(f));
   }
@@ -435,11 +437,11 @@ public void access(Path path, FsAction mode) throws AccessControlException,
   public FileStatus getFileLinkStatus(final Path f)
      throws AccessControlException, FileNotFoundException,
      UnsupportedFileSystemException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), false); // do not follow mount link
     return res.targetFileSystem.getFileLinkStatus(res.remainingPath);
   }
-  
+
   @Override
   public FsStatus getFsStatus() throws AccessControlException,
       FileNotFoundException, IOException {
@@ -486,7 +488,7 @@ public LocatedFileStatus getViewFsFileStatus(LocatedFileStatus stat,
       }
     };
   }
-  
+
   /**
    * {@inheritDoc}
    *
@@ -518,7 +520,7 @@ public FileStatus[] listStatus(final Path f) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
     InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    
+
     FileStatus[] statusLst = res.targetFileSystem.listStatus(res.remainingPath);
     if (!res.isInternalDir()) {
       // We need to change the name in the FileStatus as described in
@@ -540,7 +542,7 @@ public void mkdir(final Path dir, final FsPermission permission,
       final boolean createParent) throws AccessControlException,
       FileAlreadyExistsException,
       FileNotFoundException, UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(dir), false);
     res.targetFileSystem.mkdir(res.remainingPath, permission, createParent);
   }
@@ -549,7 +551,7 @@ public void mkdir(final Path dir, final FsPermission permission,
   public FSDataInputStream open(final Path f, final int bufferSize)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
         fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.open(res.remainingPath, bufferSize);
   }
@@ -566,7 +568,7 @@ public boolean truncate(final Path f, final long newLength)
   @Override
   public void renameInternal(final Path src, final Path dst,
       final boolean overwrite) throws IOException, UnresolvedLinkException {
-    // passing resolveLastComponet as false to catch renaming a mount point 
+    // passing resolveLastComponet as false to catch renaming a mount point
     // itself we need to catch this as an internal operation and fail if no
     // fallback.
     InodeTree.ResolveResult<AbstractFileSystem> resSrc =
@@ -640,12 +642,12 @@ public void renameInternal(final Path src, final Path dst)
       UnresolvedLinkException, IOException {
     renameInternal(src, dst, false);
   }
-  
+
   @Override
   public boolean supportsSymlinks() {
     return true;
   }
-  
+
   @Override
   public void createSymlink(final Path target, final Path link,
       final boolean createParent) throws IOException, UnresolvedLinkException {
@@ -661,12 +663,12 @@ public void createSymlink(final Path target, final Path link,
     }
     assert(res.remainingPath != null);
     res.targetFileSystem.createSymlink(target, res.remainingPath,
-        createParent);  
+        createParent);
   }
 
   @Override
   public Path getLinkTarget(final Path f) throws IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), false); // do not follow mount link
     return res.targetFileSystem.getLinkTarget(res.remainingPath);
   }
@@ -675,26 +677,26 @@ public Path getLinkTarget(final Path f) throws IOException {
   public void setOwner(final Path f, final String username,
       final String groupname) throws AccessControlException,
       FileNotFoundException, UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setOwner(res.remainingPath, username, groupname); 
+    res.targetFileSystem.setOwner(res.remainingPath, username, groupname);
   }
 
   @Override
   public void setPermission(final Path f, final FsPermission permission)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setPermission(res.remainingPath, permission); 
-    
+    res.targetFileSystem.setPermission(res.remainingPath, permission);
+
   }
 
   @Override
   public boolean setReplication(final Path f, final short replication)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
     return res.targetFileSystem.setReplication(res.remainingPath, replication);
   }
@@ -703,47 +705,58 @@ public boolean setReplication(final Path f, final short replication)
   public void setTimes(final Path f, final long mtime, final long atime)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException {
-    InodeTree.ResolveResult<AbstractFileSystem> res = 
+    InodeTree.ResolveResult<AbstractFileSystem> res =
       fsState.resolve(getUriPath(f), true);
-    res.targetFileSystem.setTimes(res.remainingPath, mtime, atime); 
+    res.targetFileSystem.setTimes(res.remainingPath, mtime, atime);
   }
 
   @Override
   public void setVerifyChecksum(final boolean verifyChecksum)
       throws AccessControlException, IOException {
-    // This is a file system level operations, however ViewFs 
-    // points to many file systems. Noop for ViewFs. 
+    // This is a file system level operations, however ViewFs
+    // points to many file systems. Noop for ViewFs.
   }
-  
+
   public MountPoint[] getMountPoints() {
-    List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
+    List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints =
                   fsState.getMountPoints();
-    
+
     MountPoint[] result = new MountPoint[mountPoints.size()];
     for ( int i = 0; i < mountPoints.size(); ++i ) {
-      result[i] = new MountPoint(new Path(mountPoints.get(i).src), 
+      result[i] = new MountPoint(new Path(mountPoints.get(i).src),
                               mountPoints.get(i).target.targetDirLinkList);
     }
     return result;
   }
-  
+
   @Override
   public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
-    List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints = 
+    List<InodeTree.MountPoint<AbstractFileSystem>> mountPoints =
                 fsState.getMountPoints();
     int initialListSize  = 0;
     for (InodeTree.MountPoint<AbstractFileSystem> im : mountPoints) {
-      initialListSize += im.target.targetDirLinkList.length; 
+      initialListSize += im.target.targetDirLinkList.length;
     }
     List<Token<?>> result = new ArrayList<Token<?>>(initialListSize);
     for ( int i = 0; i < mountPoints.size(); ++i ) {
-      List<Token<?>> tokens = 
+      List<Token<?>> tokens =
           mountPoints.get(i).target.getTargetFileSystem()
               .getDelegationTokens(renewer);
       if (tokens != null) {
         result.addAll(tokens);
       }
     }
+
+    // Add tokens from fallback FS
+    if (this.fsState.getRootFallbackLink() != null) {
+      AbstractFileSystem rootFallbackFs =
+          this.fsState.getRootFallbackLink().getTargetFileSystem();
+      List<Token<?>> tokens = rootFallbackFs.getDelegationTokens(renewer);
+      if (tokens != null) {
+        result.addAll(tokens);
+      }
+    }
+
     return result;
   }
 
@@ -896,7 +909,7 @@ public void unsetStoragePolicy(final Path src)
    *
    * @param src file or directory path.
    * @return storage policy for give file.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public BlockStoragePolicySpi getStoragePolicy(final Path src)
       throws IOException {
@@ -942,18 +955,18 @@ public T next() throws IOException {
   }
 
   /*
-   * An instance of this class represents an internal dir of the viewFs 
+   * An instance of this class represents an internal dir of the viewFs
    * ie internal dir of the mount table.
    * It is a ready only mount tbale and create, mkdir or delete operations
    * are not allowed.
    * If called on create or mkdir then this target is the parent of the
    * directory in which one is trying to create or mkdir; hence
-   * in this case the path name passed in is the last component. 
+   * in this case the path name passed in is the last component.
    * Otherwise this target is the end point of the path and hence
-   * the path name passed in is null. 
+   * the path name passed in is null.
    */
   static class InternalDirOfViewFs extends AbstractFileSystem {
-    
+
     final InodeTree.INodeDir<AbstractFileSystem>  theInternalDir;
     final long creationTime; // of the the mount table
     final UserGroupInformation ugi; // the user/group of user who created mtable
@@ -1072,7 +1085,7 @@ public FileStatus getFileStatus(final Path f) throws IOException {
           new Path(theInternalDir.fullPath).makeQualified(
               myUri, null));
     }
-    
+
     @Override
     public FileStatus getFileLinkStatus(final Path f)
         throws IOException {
@@ -1085,8 +1098,7 @@ public FileStatus getFileLinkStatus(final Path f)
       }
       FileStatus result;
       if (inode.isLink()) {
-        INodeLink<AbstractFileSystem> inodelink = 
-          (INodeLink<AbstractFileSystem>) inode;
+        INodeLink<AbstractFileSystem> inodelink = inode.getLink();
         try {
           String linkedPath = inodelink.getTargetFileSystem()
               .getUri().getPath();
@@ -1114,7 +1126,7 @@ public FileStatus getFileLinkStatus(final Path f)
       }
       return result;
     }
-    
+
     @Override
     public FsStatus getFsStatus() {
       return new FsStatus(0, 0, 0);
@@ -1156,8 +1168,7 @@ public FileStatus[] listStatus(final Path f) throws IOException {
         INode<AbstractFileSystem> inode = iEntry.getValue();
         Path path = new Path(inode.fullPath).makeQualified(myUri, null);
         if (inode.isLink()) {
-          INodeLink<AbstractFileSystem> link = 
-            (INodeLink<AbstractFileSystem>) inode;
+          INodeLink<AbstractFileSystem> link = inode.getLink();
 
           if (showMountLinksAsSymlinks) {
             // To maintain backward compatibility, with default option(showing
@@ -1306,18 +1317,18 @@ public void renameInternal(final Path src, final Path dst)
         throws AccessControlException, IOException {
       checkPathIsSlash(src);
       checkPathIsSlash(dst);
-      throw readOnlyMountTable("rename", src);     
+      throw readOnlyMountTable("rename", src);
     }
 
     @Override
     public boolean supportsSymlinks() {
       return true;
     }
-    
+
     @Override
     public void createSymlink(final Path target, final Path link,
         final boolean createParent) throws AccessControlException {
-      throw readOnlyMountTable("createSymlink", link);    
+      throw readOnlyMountTable("createSymlink", link);
     }
 
     @Override
@@ -1337,7 +1348,7 @@ public void setOwner(final Path f, final String username,
     public void setPermission(final Path f, final FsPermission permission)
         throws AccessControlException, IOException {
       checkPathIsSlash(f);
-      throw readOnlyMountTable("setPermission", f);    
+      throw readOnlyMountTable("setPermission", f);
     }
 
     @Override
@@ -1351,13 +1362,13 @@ public boolean setReplication(final Path f, final short replication)
     public void setTimes(final Path f, final long mtime, final long atime)
         throws AccessControlException, IOException {
       checkPathIsSlash(f);
-      throw readOnlyMountTable("setTimes", f);    
+      throw readOnlyMountTable("setTimes", f);
     }
 
     @Override
     public void setVerifyChecksum(final boolean verifyChecksum)
         throws AccessControlException {
-      throw readOnlyMountTable("setVerifyChecksum", "");   
+      throw readOnlyMountTable("setVerifyChecksum", "");
     }
 
     @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 041f8cab49c4d..edd15af534a76 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -91,6 +91,8 @@ public interface ActiveStandbyElectorCallback {
      * 
      * Callback implementations are expected to manage their own
      * timeouts (e.g. when making an RPC to a remote node).
+     *
+     * @throws ServiceFailedException Service Failed Exception.
      */
     void becomeActive() throws ServiceFailedException;
 
@@ -119,6 +121,8 @@ public interface ActiveStandbyElectorCallback {
      * If there is any fatal error (e.g. wrong ACL's, unexpected Zookeeper
      * errors or Zookeeper persistent unavailability) then notifyFatalError is
      * called to notify the app about it.
+     *
+     * @param errorMessage error message.
      */
     void notifyFatalError(String errorMessage);
 
@@ -204,8 +208,12 @@ enum State {
    *                 ZK connection
    * @param app
    *          reference to callback interface object
-   * @throws IOException
+   * @param maxRetryNum maxRetryNum.
+   * @throws IOException raised on errors performing I/O.
    * @throws HadoopIllegalArgumentException
+   *         if valid data is not supplied.
+   * @throws KeeperException
+   *         other zookeeper operation errors.
    */
   public ActiveStandbyElector(String zookeeperHostPorts,
       int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
@@ -245,8 +253,13 @@ public ActiveStandbyElector(String zookeeperHostPorts,
    *          reference to callback interface object
    * @param failFast
    *          whether need to add the retry when establishing ZK connection.
+   * @param maxRetryNum max Retry Num
    * @throws IOException
+   *          raised on errors performing I/O.
    * @throws HadoopIllegalArgumentException
+   *          if valid data is not supplied.
+   * @throws KeeperException
+   *          other zookeeper operation errors.
    */
   public ActiveStandbyElector(String zookeeperHostPorts,
       int zookeeperSessionTimeout, String parentZnodeName, List<ACL> acl,
@@ -312,6 +325,8 @@ public synchronized void joinElection(byte[] data)
   
   /**
    * @return true if the configured parent znode exists
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException interrupted exception.
    */
   public synchronized boolean parentZNodeExists()
       throws IOException, InterruptedException {
@@ -327,6 +342,10 @@ public synchronized boolean parentZNodeExists()
   /**
    * Utility function to ensure that the configured base znode exists.
    * This recursively creates the znode as well as all of its parents.
+   *
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException interrupted exception.
+   * @throws KeeperException other zookeeper operation errors.
    */
   public synchronized void ensureParentZNode()
       throws IOException, InterruptedException, KeeperException {
@@ -371,6 +390,9 @@ public synchronized void ensureParentZNode()
    * This recursively deletes everything within the znode as well as the
    * parent znode itself. It should only be used when it's certain that
    * no electors are currently participating in the election.
+   *
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException interrupted exception.
    */
   public synchronized void clearParentZNode()
       throws IOException, InterruptedException {
@@ -435,6 +457,7 @@ public static class ActiveNotFoundException extends Exception {
    * @throws KeeperException
    *           other zookeeper operation errors
    * @throws InterruptedException
+   *           interrupted exception.
    * @throws IOException
    *           when ZooKeeper connection could not be established
    */
@@ -684,7 +707,7 @@ synchronized void processWatchEvent(ZooKeeper zk, WatchedEvent event) {
    * inherit and mock out the zookeeper instance
    * 
    * @return new zookeeper client instance
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KeeperException zookeeper connectionloss exception
    */
   protected synchronized ZooKeeper connectToZooKeeper() throws IOException,
@@ -714,7 +737,7 @@ protected synchronized ZooKeeper connectToZooKeeper() throws IOException,
    * inherit and pass in a mock object for zookeeper
    *
    * @return new zookeeper client instance
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected ZooKeeper createZooKeeper() throws IOException {
     return new ZooKeeper(zkHostPort, zkSessionTimeout, watcher);
@@ -781,6 +804,8 @@ private void reJoinElection(int sleepTime) {
    * Sleep for the given number of milliseconds.
    * This is non-static, and separated out, so that unit tests
    * can override the behavior not to sleep.
+   *
+   * @param sleepMs sleep ms.
    */
   @VisibleForTesting
   protected void sleepFor(int sleepMs) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index c6949e561e2a2..9eeaacd76bca5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -326,6 +326,9 @@ private int getServiceState(final CommandLine cmd)
   /**
    * Return the serviceId as is, we are assuming it was
    * given as a service address of form {@literal <}host:ipcport{@literal >}.
+   *
+   * @param serviceId serviceId.
+   * @return service addr.
    */
   protected String getServiceAddr(String serviceId) {
     return serviceId;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
index 74a3d121a1abe..56c848617ffbc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceProtocol.java
@@ -118,7 +118,8 @@ public void monitorHealth() throws HealthCheckFailedException,
   /**
    * Request service to transition to active state. No operation, if the
    * service is already in active state.
-   * 
+   *
+   * @param reqInfo reqInfo.
    * @throws ServiceFailedException
    *           if transition from standby to active fails.
    * @throws AccessControlException
@@ -135,7 +136,8 @@ public void transitionToActive(StateChangeRequestInfo reqInfo)
   /**
    * Request service to transition to standby state. No operation, if the
    * service is already in standby state.
-   * 
+   *
+   * @param reqInfo reqInfo.
    * @throws ServiceFailedException
    *           if transition from active to standby fails.
    * @throws AccessControlException
@@ -153,6 +155,7 @@ public void transitionToStandby(StateChangeRequestInfo reqInfo)
    * Request service to transition to observer state. No operation, if the
    * service is already in observer state.
    *
+   * @param reqInfo reqInfo.
    * @throws ServiceFailedException
    *           if transition from standby to observer fails.
    * @throws AccessControlException
@@ -176,6 +179,7 @@ void transitionToObserver(StateChangeRequestInfo reqInfo)
    * @throws IOException
    *           if other errors happen
    * @see HAServiceStatus
+   * @return HAServiceStatus.
    */
   @Idempotent
   public HAServiceStatus getServiceStatus() throws AccessControlException,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java
index 2e6b1fe113479..288a9dcbe0e53 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAServiceTarget.java
@@ -93,6 +93,9 @@ public abstract void checkFencingConfigured()
   
   /**
    * @return a proxy to connect to the target HA Service.
+   * @param timeoutMs timeout in milliseconds.
+   * @param conf Configuration.
+   * @throws IOException raised on errors performing I/O.
    */
   public HAServiceProtocol getProxy(Configuration conf, int timeoutMs)
       throws IOException {
@@ -115,7 +118,7 @@ public HAServiceProtocol.HAServiceState getTransitionTargetHAStatus() {
    * returned proxy defaults to using {@link #getAddress()}, which means this
    * method's behavior is identical to {@link #getProxy(Configuration, int)}.
    *
-   * @param conf Configuration
+   * @param conf configuration.
    * @param timeoutMs timeout in milliseconds
    * @return a proxy to connect to the target HA service for health monitoring
    * @throws IOException if there is an error
@@ -154,6 +157,9 @@ private HAServiceProtocol getProxyForAddress(Configuration conf,
 
   /**
    * @return a proxy to the ZKFC which is associated with this HA service.
+   * @param conf configuration.
+   * @param timeoutMs timeout in milliseconds.
+   * @throws IOException raised on errors performing I/O.
    */
   public ZKFCProtocol getZKFCProxy(Configuration conf, int timeoutMs)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
index 7e90fb77a0702..d222d52e37349 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
@@ -184,6 +184,9 @@ private void tryConnect() {
   
   /**
    * Connect to the service to be monitored. Stubbed out for easier testing.
+   *
+   * @throws IOException raised on errors performing I/O.
+   * @return HAServiceProtocol.
    */
   protected HAServiceProtocol createProxy() throws IOException {
     return targetToMonitor.getHealthMonitorProxy(conf, rpcTimeout, rpcConnectRetries);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
index fb78a4c47dcde..f42db1d0c5561 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
@@ -48,6 +48,7 @@
  * <ul>
  * <li><code>shell(/path/to/some/script.sh args...)</code></li>
  * <li><code>sshfence(...)</code> (see {@link SshFenceByTcpPort})
+ * <li><code>powershell(...)</code> (see {@link PowerShellFencer})
  * </ul>
  */
 @InterfaceAudience.Private
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/PowerShellFencer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/PowerShellFencer.java
index 6de618c8fcc52..47e526f65e658 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/PowerShellFencer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/PowerShellFencer.java
@@ -34,6 +34,10 @@
  * Fencer method that uses PowerShell to remotely connect to a machine and kill
  * the required process. This only works in Windows.
  *
+ * Fencing method uses PowerShell command. It should be specified
+ * in the fencing configuration like:<br>
+ * <code>powershell(NameNode)</code>
+ *
  * The argument passed to this fencer should be a unique string in the
  * "CommandLine" attribute for the "java.exe" process. For example, the full
  * path for the Namenode: "org.apache.hadoop.hdfs.server.namenode.NameNode".
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 87a80b868cdb1..d24d5630c5917 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -153,6 +153,8 @@ protected abstract void checkRpcAdminAccess()
    * the ZKFC will do all of its work. This is so that multiple federated
    * nameservices can run on the same ZK quorum without having to manually
    * configure them to separate subdirectories.
+   *
+   * @return ScopeInsideParentNode.
    */
   protected abstract String getScopeInsideParentNode();
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
index 51db21c185f20..5f47ddb339212 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HtmlQuoting.java
@@ -80,6 +80,7 @@ public static boolean needsQuoting(String str) {
    * @param buffer the byte array to take the characters from
    * @param off the index of the first byte to quote
    * @param len the number of bytes to quote
+   * @throws IOException raised on errors performing I/O.
    */
   public static void quoteHtmlChars(OutputStream output, byte[] buffer,
                                     int off, int len) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java
index b2f18538b6c7d..863afdf1f2e53 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLog.java
@@ -17,16 +17,12 @@
  */
 package org.apache.hadoop.http;
 
+import java.util.Collections;
 import java.util.HashMap;
-
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogConfigurationException;
-import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Appender;
-import org.eclipse.jetty.server.AsyncRequestLogWriter;
+import java.util.Map;
 import org.eclipse.jetty.server.CustomRequestLog;
 import org.eclipse.jetty.server.RequestLog;
+import org.eclipse.jetty.server.Slf4jRequestLogWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,67 +33,27 @@ public class HttpRequestLog {
 
   public static final Logger LOG =
       LoggerFactory.getLogger(HttpRequestLog.class);
-  private static final HashMap<String, String> serverToComponent;
+  private static final Map<String, String> serverToComponent;
 
   static {
-    serverToComponent = new HashMap<String, String>();
-    serverToComponent.put("cluster", "resourcemanager");
-    serverToComponent.put("hdfs", "namenode");
-    serverToComponent.put("node", "nodemanager");
+    Map<String, String > map = new HashMap<String, String>();
+    map.put("cluster", "resourcemanager");
+    map.put("hdfs", "namenode");
+    map.put("node", "nodemanager");
+    serverToComponent = Collections.unmodifiableMap(map);
   }
 
   public static RequestLog getRequestLog(String name) {
-
     String lookup = serverToComponent.get(name);
     if (lookup != null) {
       name = lookup;
     }
     String loggerName = "http.requests." + name;
-    String appenderName = name + "requestlog";
-    Log logger = LogFactory.getLog(loggerName);
-
-    boolean isLog4JLogger;;
-    try {
-      isLog4JLogger = logger instanceof Log4JLogger;
-    } catch (NoClassDefFoundError err) {
-      // In some dependent projects, log4j may not even be on the classpath at
-      // runtime, in which case the above instanceof check will throw
-      // NoClassDefFoundError.
-      LOG.debug("Could not load Log4JLogger class", err);
-      isLog4JLogger = false;
-    }
-    if (isLog4JLogger) {
-      Log4JLogger httpLog4JLog = (Log4JLogger)logger;
-      org.apache.log4j.Logger httpLogger = httpLog4JLog.getLogger();
-      Appender appender = null;
-
-      try {
-        appender = httpLogger.getAppender(appenderName);
-      } catch (LogConfigurationException e) {
-        LOG.warn("Http request log for {} could not be created", loggerName);
-        throw e;
-      }
-
-      if (appender == null) {
-        LOG.info("Http request log for {} is not defined", loggerName);
-        return null;
-      }
+    Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter();
+    writer.setLoggerName(loggerName);
+    return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT);
+  }
 
-      if (appender instanceof HttpRequestLogAppender) {
-        HttpRequestLogAppender requestLogAppender
-          = (HttpRequestLogAppender)appender;
-        AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter();
-        logWriter.setFilename(requestLogAppender.getFilename());
-        logWriter.setRetainDays(requestLogAppender.getRetainDays());
-        return new CustomRequestLog(logWriter,
-            CustomRequestLog.EXTENDED_NCSA_FORMAT);
-      } else {
-        LOG.warn("Jetty request log for {} was of the wrong class", loggerName);
-        return null;
-      }
-    } else {
-      LOG.warn("Jetty request log can only be enabled using Log4j");
-      return null;
-    }
+  private HttpRequestLog() {
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 76e77560a58bf..178f761191b1e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -27,6 +27,7 @@
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -96,7 +97,7 @@
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
-import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker;
+import org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.HandlerCollection;
 import org.eclipse.jetty.server.handler.RequestLogHandler;
@@ -143,7 +144,7 @@ public final class HttpServer2 implements FilterContainer {
 
   public static final String HTTP_SOCKET_BACKLOG_SIZE_KEY =
       "hadoop.http.socket.backlog.size";
-  public static final int HTTP_SOCKET_BACKLOG_SIZE_DEFAULT = 128;
+  public static final int HTTP_SOCKET_BACKLOG_SIZE_DEFAULT = 500;
   public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads";
   public static final String HTTP_ACCEPTOR_COUNT_KEY =
       "hadoop.http.acceptor.count";
@@ -269,6 +270,7 @@ public Builder setName(String name){
      *          specifies the binding address, and the port specifies the
      *          listening port. Unspecified or zero port means that the server
      *          can listen to any port.
+     * @return Builder.
      */
     public Builder addEndpoint(URI endpoint) {
       endpoints.add(endpoint);
@@ -279,6 +281,9 @@ public Builder addEndpoint(URI endpoint) {
      * Set the hostname of the http server. The host name is used to resolve the
      * _HOST field in Kerberos principals. The hostname of the first listener
      * will be used if the name is unspecified.
+     *
+     * @param hostName hostName.
+     * @return Builder.
      */
     public Builder hostName(String hostName) {
       this.hostName = hostName;
@@ -307,6 +312,9 @@ public Builder keyPassword(String password) {
     /**
      * Specify whether the server should authorize the client in SSL
      * connections.
+     *
+     * @param value value.
+     * @return Builder.
      */
     public Builder needsClientAuth(boolean value) {
       this.needsClientAuth = value;
@@ -331,6 +339,9 @@ public Builder setConf(Configuration conf) {
     /**
      * Specify the SSL configuration to load. This API provides an alternative
      * to keyStore/keyPassword/trustStore.
+     *
+     * @param sslCnf sslCnf.
+     * @return Builder.
      */
     public Builder setSSLConf(Configuration sslCnf) {
       this.sslConf = sslCnf;
@@ -771,6 +782,28 @@ private void initializeWebServer(String name, String hostName,
 
     addDefaultServlets();
     addPrometheusServlet(conf);
+    addAsyncProfilerServlet(contexts, conf);
+  }
+
+  private void addAsyncProfilerServlet(ContextHandlerCollection contexts, Configuration conf)
+      throws IOException {
+    final String asyncProfilerHome = ProfileServlet.getAsyncProfilerHome();
+    if (asyncProfilerHome != null && !asyncProfilerHome.trim().isEmpty()) {
+      addServlet("prof", "/prof", ProfileServlet.class);
+      Path tmpDir = Paths.get(ProfileServlet.OUTPUT_DIR);
+      if (Files.notExists(tmpDir)) {
+        Files.createDirectories(tmpDir);
+      }
+      ServletContextHandler genCtx = new ServletContextHandler(contexts, "/prof-output-hadoop");
+      genCtx.addServlet(ProfileOutputServlet.class, "/*");
+      genCtx.setResourceBase(tmpDir.toAbsolutePath().toString());
+      genCtx.setDisplayName("prof-output-hadoop");
+      setContextAttributes(genCtx, conf);
+    } else {
+      addServlet("prof", "/prof", ProfilerDisabledServlet.class);
+      LOG.info("ASYNC_PROFILER_HOME environment variable and async.profiler.home system property "
+          + "not specified. Disabling /prof endpoint.");
+    }
   }
 
   private void addPrometheusServlet(Configuration conf) {
@@ -875,8 +908,11 @@ private static FilterInitializer[] getFilterInitializers(Configuration conf) {
 
   /**
    * Add default apps.
+   *
+   * @param parent contexthandlercollection.
    * @param appDir The application directory
-   * @throws IOException
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
    */
   protected void addDefaultApps(ContextHandlerCollection parent,
       final String appDir, Configuration conf) throws IOException {
@@ -903,7 +939,7 @@ protected void addDefaultApps(ContextHandlerCollection parent,
       handler.setHttpOnly(true);
       handler.getSessionCookieConfig().setSecure(true);
       logContext.setSessionHandler(handler);
-      logContext.addAliasCheck(new AllowSymLinkAliasChecker());
+      logContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(logContext));
       setContextAttributes(logContext, conf);
       addNoCacheFilter(logContext);
       defaultContexts.put(logContext, true);
@@ -922,7 +958,7 @@ protected void addDefaultApps(ContextHandlerCollection parent,
     handler.setHttpOnly(true);
     handler.getSessionCookieConfig().setSecure(true);
     staticContext.setSessionHandler(handler);
-    staticContext.addAliasCheck(new AllowSymLinkAliasChecker());
+    staticContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(staticContext));
     setContextAttributes(staticContext, conf);
     defaultContexts.put(staticContext, true);
   }
@@ -1157,6 +1193,12 @@ public void addGlobalFilter(String name, String classname,
 
   /**
    * Define a filter for a context and set up default url mappings.
+   *
+   * @param ctx ctx.
+   * @param name name.
+   * @param classname classname.
+   * @param parameters parameters.
+   * @param urls urls.
    */
   public static void defineFilter(ServletContextHandler ctx, String name,
       String classname, Map<String,String> parameters, String[] urls) {
@@ -1267,6 +1309,7 @@ public int getPort() {
   /**
    * Get the address that corresponds to a particular connector.
    *
+   * @param index index.
    * @return the corresponding address for the connector, or null if there's no
    *         such connector or the connector is not bounded or was closed.
    */
@@ -1286,6 +1329,9 @@ public InetSocketAddress getConnectorAddress(int index) {
 
   /**
    * Set the min, max number of worker threads (simultaneous connections).
+   *
+   * @param min min.
+   * @param max max.
    */
   public void setThreads(int min, int max) {
     QueuedThreadPool pool = (QueuedThreadPool) webServer.getThreadPool();
@@ -1312,6 +1358,8 @@ private void initSpnego(Configuration conf, String hostName,
 
   /**
    * Start the server. Does not wait for the server to start.
+   *
+   * @throws IOException raised on errors performing I/O.
    */
   public void start() throws IOException {
     try {
@@ -1486,7 +1534,9 @@ void openListeners() throws Exception {
   }
 
   /**
-   * stop the server
+   * stop the server.
+   *
+   * @throws Exception exception.
    */
   public void stop() throws Exception {
     MultiException exception = null;
@@ -1587,6 +1637,7 @@ public String toString() {
    * @param request the servlet request.
    * @param response the servlet response.
    * @return TRUE/FALSE based on the logic decribed above.
+   * @throws IOException raised on errors performing I/O.
    */
   public static boolean isInstrumentationAccessAllowed(
     ServletContext servletContext, HttpServletRequest request,
@@ -1608,9 +1659,11 @@ public static boolean isInstrumentationAccessAllowed(
    * Does the user sending the HttpServletRequest has the administrator ACLs? If
    * it isn't the case, response will be modified to send an error to the user.
    *
+   * @param servletContext servletContext.
+   * @param request request.
    * @param response used to send the error response if user does not have admin access.
    * @return true if admin-authorized, false otherwise
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static boolean hasAdministratorAccess(
       ServletContext servletContext, HttpServletRequest request,
@@ -1914,4 +1967,8 @@ HttpServer2Metrics getMetrics() {
     return metrics;
   }
 
+  @VisibleForTesting
+  List<ServerConnector> getListeners() {
+    return listeners;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java
new file mode 100644
index 0000000000000..1ecc21f3753ce
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileOutputServlet.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.regex.Pattern;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Servlet to serve files generated by {@link ProfileServlet}.
+ */
+@InterfaceAudience.Private
+public class ProfileOutputServlet extends DefaultServlet {
+
+  private static final long serialVersionUID = 1L;
+
+  private static final Logger LOG = LoggerFactory.getLogger(ProfileOutputServlet.class);
+  // default refresh period 2 sec
+  private static final int REFRESH_PERIOD = 2;
+  // Alphanumeric characters, plus percent (url-encoding), equals, ampersand, dot and hyphen
+  private static final Pattern ALPHA_NUMERIC = Pattern.compile("[a-zA-Z0-9%=&.\\-]*");
+
+  @Override
+  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
+      throws ServletException, IOException {
+    if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
+      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      ProfileServlet.setResponseHeader(resp);
+      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
+      return;
+    }
+
+    String absoluteDiskPath = getServletContext().getRealPath(req.getPathInfo());
+    File requestedFile = new File(absoluteDiskPath);
+    // async-profiler version 1.4 writes 'Started [cpu] profiling' to output file when profiler is
+    // running which gets replaced by final output. If final output is not ready yet, the file size
+    // will be <100 bytes (in all modes).
+    if (requestedFile.length() < 100) {
+      LOG.info("{} is incomplete. Sending auto-refresh header.", requestedFile);
+      String refreshUrl = req.getRequestURI();
+      // Rebuild the query string (if we have one)
+      if (req.getQueryString() != null) {
+        refreshUrl += "?" + sanitize(req.getQueryString());
+      }
+      ProfileServlet.setResponseHeader(resp);
+      resp.setHeader("Refresh", REFRESH_PERIOD + ";" + refreshUrl);
+      resp.getWriter().write("This page will be auto-refreshed every " + REFRESH_PERIOD
+          + " seconds until the output file is ready. Redirecting to " + refreshUrl);
+    } else {
+      super.doGet(req, resp);
+    }
+  }
+
+  static String sanitize(String input) {
+    // Basic test to try to avoid any XSS attacks or HTML content showing up.
+    // Duplicates HtmlQuoting a little, but avoid destroying ampersand.
+    if (ALPHA_NUMERIC.matcher(input).matches()) {
+      return input;
+    }
+    throw new RuntimeException("Non-alphanumeric data found in input, aborting.");
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
new file mode 100644
index 0000000000000..ce532741512fc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.ProcessUtils;
+
+/**
+ * Servlet that runs async-profiler as web-endpoint.
+ * <p>
+ * Following options from async-profiler can be specified as query paramater.
+ * //  -e event          profiling event: cpu|alloc|lock|cache-misses etc.
+ * //  -d duration       run profiling for 'duration' seconds (integer)
+ * //  -i interval       sampling interval in nanoseconds (long)
+ * //  -j jstackdepth    maximum Java stack depth (integer)
+ * //  -b bufsize        frame buffer size (long)
+ * //  -t                profile different threads separately
+ * //  -s                simple class names instead of FQN
+ * //  -o fmt[,fmt...]   output format: summary|traces|flat|collapsed|svg|tree|jfr|html
+ * //  --width px        SVG width pixels (integer)
+ * //  --height px       SVG frame height pixels (integer)
+ * //  --minwidth px     skip frames smaller than px (double)
+ * //  --reverse         generate stack-reversed FlameGraph / Call tree
+ * <p>
+ * Example:
+ * If Namenode http address is localhost:9870, and ResourceManager http address is localhost:8088,
+ * ProfileServlet running with async-profiler setup can be accessed with
+ * http://localhost:9870/prof and http://localhost:8088/prof for Namenode and ResourceManager
+ * processes respectively.
+ * Deep dive into some params:
+ * - To collect 10 second CPU profile of current process i.e. Namenode (returns FlameGraph svg)
+ * curl "http://localhost:9870/prof"
+ * - To collect 10 second CPU profile of pid 12345 (returns FlameGraph svg)
+ * curl "http://localhost:9870/prof?pid=12345" (For instance, provide pid of Datanode)
+ * - To collect 30 second CPU profile of pid 12345 (returns FlameGraph svg)
+ * curl "http://localhost:9870/prof?pid=12345&amp;duration=30"
+ * - To collect 1 minute CPU profile of current process and output in tree format (html)
+ * curl "http://localhost:9870/prof?output=tree&amp;duration=60"
+ * - To collect 10 second heap allocation profile of current process (returns FlameGraph svg)
+ * curl "http://localhost:9870/prof?event=alloc"
+ * - To collect lock contention profile of current process (returns FlameGraph svg)
+ * curl "http://localhost:9870/prof?event=lock"
+ * <p>
+ * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events)
+ * // Perf events:
+ * //    cpu
+ * //    page-faults
+ * //    context-switches
+ * //    cycles
+ * //    instructions
+ * //    cache-references
+ * //    cache-misses
+ * //    branches
+ * //    branch-misses
+ * //    bus-cycles
+ * //    L1-dcache-load-misses
+ * //    LLC-load-misses
+ * //    dTLB-load-misses
+ * //    mem:breakpoint
+ * //    trace:tracepoint
+ * // Java events:
+ * //    alloc
+ * //    lock
+ */
+@InterfaceAudience.Private
+public class ProfileServlet extends HttpServlet {
+
+  private static final long serialVersionUID = 1L;
+  private static final Logger LOG = LoggerFactory.getLogger(ProfileServlet.class);
+
+  static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+  static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+  private static final String ALLOWED_METHODS = "GET";
+  private static final String CONTENT_TYPE_TEXT = "text/plain; charset=utf-8";
+  private static final String ASYNC_PROFILER_HOME_ENV = "ASYNC_PROFILER_HOME";
+  private static final String ASYNC_PROFILER_HOME_SYSTEM_PROPERTY = "async.profiler.home";
+  private static final String PROFILER_SCRIPT = "/profiler.sh";
+  private static final int DEFAULT_DURATION_SECONDS = 10;
+  private static final AtomicInteger ID_GEN = new AtomicInteger(0);
+
+  static final String OUTPUT_DIR = System.getProperty("java.io.tmpdir") + "/prof-output-hadoop";
+
+  // This flag is only allowed to be reset by tests.
+  private static boolean isTestRun = false;
+
+  private enum Event {
+
+    CPU("cpu"),
+    ALLOC("alloc"),
+    LOCK("lock"),
+    PAGE_FAULTS("page-faults"),
+    CONTEXT_SWITCHES("context-switches"),
+    CYCLES("cycles"),
+    INSTRUCTIONS("instructions"),
+    CACHE_REFERENCES("cache-references"),
+    CACHE_MISSES("cache-misses"),
+    BRANCHES("branches"),
+    BRANCH_MISSES("branch-misses"),
+    BUS_CYCLES("bus-cycles"),
+    L1_DCACHE_LOAD_MISSES("L1-dcache-load-misses"),
+    LLC_LOAD_MISSES("LLC-load-misses"),
+    DTLB_LOAD_MISSES("dTLB-load-misses"),
+    MEM_BREAKPOINT("mem:breakpoint"),
+    TRACE_TRACEPOINT("trace:tracepoint");
+
+    private final String internalName;
+
+    Event(final String internalName) {
+      this.internalName = internalName;
+    }
+
+    public String getInternalName() {
+      return internalName;
+    }
+
+    public static Event fromInternalName(final String name) {
+      for (Event event : values()) {
+        if (event.getInternalName().equalsIgnoreCase(name)) {
+          return event;
+        }
+      }
+
+      return null;
+    }
+  }
+
+  private enum Output {
+    SUMMARY,
+    TRACES,
+    FLAT,
+    COLLAPSED,
+    // No SVG in 2.x asyncprofiler.
+    SVG,
+    TREE,
+    JFR,
+    // In 2.x asyncprofiler, this is how you get flamegraphs.
+    HTML
+  }
+
+  private final Lock profilerLock = new ReentrantLock();
+  private transient volatile Process process;
+  private final String asyncProfilerHome;
+  private Integer pid;
+
+  public ProfileServlet() {
+    this.asyncProfilerHome = getAsyncProfilerHome();
+    this.pid = ProcessUtils.getPid();
+    LOG.info("Servlet process PID: {} asyncProfilerHome: {}", pid, asyncProfilerHome);
+  }
+
+  static void setIsTestRun(boolean isTestRun) {
+    ProfileServlet.isTestRun = isTestRun;
+  }
+
+  @Override
+  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
+      throws IOException {
+    if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(), req, resp)) {
+      resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      setResponseHeader(resp);
+      resp.getWriter().write("Unauthorized: Instrumentation access is not allowed!");
+      return;
+    }
+
+    // make sure async profiler home is set
+    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      setResponseHeader(resp);
+      resp.getWriter().write("ASYNC_PROFILER_HOME env is not set.\n\n"
+          + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n"
+          + "environment is properly configured.");
+      return;
+    }
+
+    // if pid is explicitly specified, use it else default to current process
+    pid = getInteger(req, "pid", pid);
+
+    // if pid is not specified in query param and if current process pid cannot be determined
+    if (pid == null) {
+      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      setResponseHeader(resp);
+      resp.getWriter().write(
+          "'pid' query parameter unspecified or unable to determine PID of current process.");
+      return;
+    }
+
+    final int duration = getInteger(req, "duration", DEFAULT_DURATION_SECONDS);
+    final Output output = getOutput(req);
+    final Event event = getEvent(req);
+    final Long interval = getLong(req, "interval");
+    final Integer jstackDepth = getInteger(req, "jstackdepth", null);
+    final Long bufsize = getLong(req, "bufsize");
+    final boolean thread = req.getParameterMap().containsKey("thread");
+    final boolean simple = req.getParameterMap().containsKey("simple");
+    final Integer width = getInteger(req, "width", null);
+    final Integer height = getInteger(req, "height", null);
+    final Double minwidth = getMinWidth(req);
+    final boolean reverse = req.getParameterMap().containsKey("reverse");
+
+    if (process == null || !process.isAlive()) {
+      try {
+        int lockTimeoutSecs = 3;
+        if (profilerLock.tryLock(lockTimeoutSecs, TimeUnit.SECONDS)) {
+          try {
+            File outputFile = new File(OUTPUT_DIR,
+                "async-prof-pid-" + pid + "-" + event.name().toLowerCase() + "-" + ID_GEN
+                    .incrementAndGet() + "." + output.name().toLowerCase());
+            List<String> cmd = new ArrayList<>();
+            cmd.add(asyncProfilerHome + PROFILER_SCRIPT);
+            cmd.add("-e");
+            cmd.add(event.getInternalName());
+            cmd.add("-d");
+            cmd.add("" + duration);
+            cmd.add("-o");
+            cmd.add(output.name().toLowerCase());
+            cmd.add("-f");
+            cmd.add(outputFile.getAbsolutePath());
+            if (interval != null) {
+              cmd.add("-i");
+              cmd.add(interval.toString());
+            }
+            if (jstackDepth != null) {
+              cmd.add("-j");
+              cmd.add(jstackDepth.toString());
+            }
+            if (bufsize != null) {
+              cmd.add("-b");
+              cmd.add(bufsize.toString());
+            }
+            if (thread) {
+              cmd.add("-t");
+            }
+            if (simple) {
+              cmd.add("-s");
+            }
+            if (width != null) {
+              cmd.add("--width");
+              cmd.add(width.toString());
+            }
+            if (height != null) {
+              cmd.add("--height");
+              cmd.add(height.toString());
+            }
+            if (minwidth != null) {
+              cmd.add("--minwidth");
+              cmd.add(minwidth.toString());
+            }
+            if (reverse) {
+              cmd.add("--reverse");
+            }
+            cmd.add(pid.toString());
+            if (!isTestRun) {
+              process = ProcessUtils.runCmdAsync(cmd);
+            }
+
+            // set response and set refresh header to output location
+            setResponseHeader(resp);
+            resp.setStatus(HttpServletResponse.SC_ACCEPTED);
+            String relativeUrl = "/prof-output-hadoop/" + outputFile.getName();
+            resp.getWriter().write("Started [" + event.getInternalName()
+                + "] profiling. This page will automatically redirect to " + relativeUrl + " after "
+                + duration + " seconds. "
+                + "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async "
+                + "Profiler Home Page, https://github.com/jvm-profiling-tools/async-profiler."
+                + "\n\nCommand:\n" + Joiner.on(" ").join(cmd));
+
+            // to avoid auto-refresh by ProfileOutputServlet, refreshDelay can be specified
+            // via url param
+            int refreshDelay = getInteger(req, "refreshDelay", 0);
+
+            // instead of sending redirect, set auto-refresh so that browsers will refresh
+            // with redirected url
+            resp.setHeader("Refresh", (duration + refreshDelay) + ";" + relativeUrl);
+            resp.getWriter().flush();
+          } finally {
+            profilerLock.unlock();
+          }
+        } else {
+          setResponseHeader(resp);
+          resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+          resp.getWriter()
+              .write("Unable to acquire lock. Another instance of profiler might be running.");
+          LOG.warn("Unable to acquire lock in {} seconds. Another instance of profiler might be"
+              + " running.", lockTimeoutSecs);
+        }
+      } catch (InterruptedException e) {
+        LOG.warn("Interrupted while acquiring profile lock.", e);
+        resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      }
+    } else {
+      setResponseHeader(resp);
+      resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      resp.getWriter().write("Another instance of profiler is already running.");
+    }
+  }
+
+  private Integer getInteger(final HttpServletRequest req, final String param,
+      final Integer defaultValue) {
+    final String value = req.getParameter(param);
+    if (value != null) {
+      try {
+        return Integer.valueOf(value);
+      } catch (NumberFormatException e) {
+        return defaultValue;
+      }
+    }
+    return defaultValue;
+  }
+
+  private Long getLong(final HttpServletRequest req, final String param) {
+    final String value = req.getParameter(param);
+    if (value != null) {
+      try {
+        return Long.valueOf(value);
+      } catch (NumberFormatException e) {
+        return null;
+      }
+    }
+    return null;
+  }
+
+  private Double getMinWidth(final HttpServletRequest req) {
+    final String value = req.getParameter("minwidth");
+    if (value != null) {
+      try {
+        return Double.valueOf(value);
+      } catch (NumberFormatException e) {
+        return null;
+      }
+    }
+    return null;
+  }
+
+  private Event getEvent(final HttpServletRequest req) {
+    final String eventArg = req.getParameter("event");
+    if (eventArg != null) {
+      Event event = Event.fromInternalName(eventArg);
+      return event == null ? Event.CPU : event;
+    }
+    return Event.CPU;
+  }
+
+  private Output getOutput(final HttpServletRequest req) {
+    final String outputArg = req.getParameter("output");
+    if (req.getParameter("output") != null) {
+      try {
+        return Output.valueOf(outputArg.trim().toUpperCase());
+      } catch (IllegalArgumentException e) {
+        return Output.HTML;
+      }
+    }
+    return Output.HTML;
+  }
+
+  static void setResponseHeader(final HttpServletResponse response) {
+    response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
+    response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+    response.setContentType(CONTENT_TYPE_TEXT);
+  }
+
+  static String getAsyncProfilerHome() {
+    String asyncProfilerHome = System.getenv(ASYNC_PROFILER_HOME_ENV);
+    // if ENV is not set, see if -Dasync.profiler.home=/path/to/async/profiler/home is set
+    if (asyncProfilerHome == null || asyncProfilerHome.trim().isEmpty()) {
+      asyncProfilerHome = System.getProperty(ASYNC_PROFILER_HOME_SYSTEM_PROPERTY);
+    }
+
+    return asyncProfilerHome;
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
new file mode 100644
index 0000000000000..c488b574990cc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfilerDisabledServlet.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.io.IOException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Servlet for disabled async-profiler.
+ */
+@InterfaceAudience.Private
+public class ProfilerDisabledServlet extends HttpServlet {
+
+  @Override
+  protected void doGet(final HttpServletRequest req, final HttpServletResponse resp)
+      throws IOException {
+    resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+    ProfileServlet.setResponseHeader(resp);
+    // TODO : Replace github.com link with
+    //  https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/
+    //  AsyncProfilerServlet.html once Async profiler changes are released
+    //  in 3.x (3.4.0 as of today).
+    resp.getWriter().write("The profiler servlet was disabled at startup.\n\n"
+        + "Please ensure the prerequisites for the Profiler Servlet have been installed and the\n"
+        + "environment is properly configured. \n\n"
+        + "For more details, please refer to: https://github.com/apache/hadoop/blob/trunk/"
+        + "hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md");
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
index eef74628e16b1..8cf82f425090d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/AbstractMapWritable.java
@@ -84,7 +84,10 @@ private synchronized void addToMap(Class<?> clazz, byte id) {
     idToClassMap.put(id, clazz);
   }
   
-  /** Add a Class to the maps if it is not already present. */ 
+  /**
+   * Add a Class to the maps if it is not already present.
+   * @param clazz clazz.
+   */
   protected synchronized void addToMap(Class<?> clazz) {
     if (classToIdMap.containsKey(clazz)) {
       return;
@@ -97,17 +100,28 @@ protected synchronized void addToMap(Class<?> clazz) {
     addToMap(clazz, id);
   }
 
-  /** @return the Class class for the specified id */
+  /**
+   * the Class class for the specified id.
+   * @param id id.
+   * @return the Class class for the specified id.
+   */
   protected Class<?> getClass(byte id) {
     return idToClassMap.get(id);
   }
 
-  /** @return the id for the specified Class */
+  /**
+   * get id.
+   * @return the id for the specified Class.
+   * @param clazz clazz.
+   */
   protected byte getId(Class<?> clazz) {
     return classToIdMap.containsKey(clazz) ? classToIdMap.get(clazz) : -1;
   }
 
-  /** Used by child copy constructors. */
+  /**
+   * Used by child copy constructors.
+   * @param other other.
+   */
   protected synchronized void copy(Writable other) {
     if (other != null) {
       try {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayFile.java
index bee5fd2cb430c..313caa6360827 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayFile.java
@@ -38,7 +38,15 @@ protected ArrayFile() {}                            // no public ctor
   public static class Writer extends MapFile.Writer {
     private LongWritable count = new LongWritable(0);
 
-    /** Create the named file for values of the named class. */
+    /**
+     * Create the named file for values of the named class.
+     *
+     * @param conf configuration.
+     * @param fs file system.
+     * @param file file.
+     * @param valClass valClass.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Writer(Configuration conf, FileSystem fs,
                   String file, Class<? extends Writable> valClass)
       throws IOException {
@@ -46,7 +54,17 @@ public Writer(Configuration conf, FileSystem fs,
             valueClass(valClass));
     }
 
-    /** Create the named file for values of the named class. */
+    /**
+     * Create the named file for values of the named class.
+     *
+     * @param conf configuration.
+     * @param fs file system.
+     * @param file file.
+     * @param valClass valClass.
+     * @param compress compress.
+     * @param progress progress.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Writer(Configuration conf, FileSystem fs,
                   String file, Class<? extends Writable> valClass,
                   CompressionType compress, Progressable progress)
@@ -58,7 +76,11 @@ public Writer(Configuration conf, FileSystem fs,
             progressable(progress));
     }
 
-    /** Append a value to the file. */
+    /**
+     * Append a value to the file.
+     * @param value value.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized void append(Writable value) throws IOException {
       super.append(count, value);                 // add to map
       count.set(count.get()+1);                   // increment count
@@ -69,31 +91,59 @@ public synchronized void append(Writable value) throws IOException {
   public static class Reader extends MapFile.Reader {
     private LongWritable key = new LongWritable();
 
-    /** Construct an array reader for the named file.*/
+    /**
+     * Construct an array reader for the named file.
+     * @param fs FileSystem.
+     * @param file file.
+     * @param conf configuration.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Reader(FileSystem fs, String file, 
                   Configuration conf) throws IOException {
       super(new Path(file), conf);
     }
 
-    /** Positions the reader before its <code>n</code>th value. */
+    /**
+     * Positions the reader before its <code>n</code>th value.
+     *
+     * @param n n key.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized void seek(long n) throws IOException {
       key.set(n);
       seek(key);
     }
 
-    /** Read and return the next value in the file. */
+    /**
+     * Read and return the next value in the file.
+     *
+     * @param value value.
+     * @throws IOException raised on errors performing I/O.
+     * @return Writable.
+     */
     public synchronized Writable next(Writable value) throws IOException {
       return next(key, value) ? value : null;
     }
 
-    /** Returns the key associated with the most recent call to {@link
+    /**
+     * Returns the key associated with the most recent call to {@link
      * #seek(long)}, {@link #next(Writable)}, or {@link
-     * #get(long,Writable)}. */
+     * #get(long,Writable)}.
+     *
+     * @return key key.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized long key() throws IOException {
       return key.get();
     }
 
-    /** Return the <code>n</code>th value in the file. */
+    /**
+     * Return the <code>n</code>th value in the file.
+     * @param n n key.
+     * @param value value.
+     * @throws IOException raised on errors performing I/O.
+     * @return writable.
+     */
     public synchronized Writable get(long n, Writable value)
       throws IOException {
       key.set(n);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayPrimitiveWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayPrimitiveWritable.java
index 2b6f3166bc282..ce7813e7483a6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayPrimitiveWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayPrimitiveWritable.java
@@ -106,7 +106,9 @@ public ArrayPrimitiveWritable() {
   
   /**
    * Construct an instance of known type but no value yet
-   * for use with type-specific wrapper classes
+   * for use with type-specific wrapper classes.
+   *
+   * @param componentType componentType.
    */
   public ArrayPrimitiveWritable(Class<?> componentType) {
     checkPrimitive(componentType);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BinaryComparable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BinaryComparable.java
index a32c44c8e5058..a78ff8b6c583e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BinaryComparable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BinaryComparable.java
@@ -31,11 +31,15 @@ public abstract class BinaryComparable implements Comparable<BinaryComparable> {
 
   /**
    * Return n st bytes 0..n-1 from {#getBytes()} are valid.
+   *
+   * @return length.
    */
   public abstract int getLength();
 
   /**
    * Return representative byte array for this instance.
+   *
+   * @return getBytes.
    */
   public abstract byte[] getBytes();
 
@@ -53,6 +57,11 @@ public int compareTo(BinaryComparable other) {
 
   /**
    * Compare bytes from {#getBytes()} to those provided.
+   *
+   * @param other other.
+   * @param off off.
+   * @param len len.
+   * @return compareBytes.
    */
   public int compareTo(byte[] other, int off, int len) {
     return WritableComparator.compareBytes(getBytes(), 0, getLength(),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BloomMapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BloomMapFile.java
index 519fcd74cbb71..91ea07d5de412 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BloomMapFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BloomMapFile.java
@@ -259,7 +259,7 @@ private void initBloomFilter(Path dirName,
      * probability of false positives.
      * @param key key to check
      * @return  false iff key doesn't exist, true if key probably exists.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public boolean probablyHasKey(WritableComparable key) throws IOException {
       if (bloomFilter == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BooleanWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BooleanWritable.java
index 0079079a7921d..789b866255b01 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BooleanWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BooleanWritable.java
@@ -35,21 +35,24 @@ public class BooleanWritable implements WritableComparable<BooleanWritable> {
    */
   public BooleanWritable() {};
 
-  /** 
+  /**
+   * @param value value.
    */
   public BooleanWritable(boolean value) {
     set(value);
   }
 
   /** 
-   * Set the value of the BooleanWritable
+   * Set the value of the BooleanWritable.
+   * @param value value.
    */    
   public void set(boolean value) {
     this.value = value;
   }
 
   /**
-   * Returns the value of the BooleanWritable
+   * Returns the value of the BooleanWritable.
+   * @return the value of the BooleanWritable.
    */
   public boolean get() {
     return value;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java
index c27449d36189c..542721f318d0a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BoundedByteArrayOutputStream.java
@@ -114,20 +114,28 @@ public void reset() {
     this.currentPointer = startOffset;
   }
 
-  /** Return the current limit */
+  /**
+   * Return the current limit.
+   * @return limit.
+   */
   public int getLimit() {
     return limit;
   }
 
-  /** Returns the underlying buffer.
+  /**
+   * Returns the underlying buffer.
    *  Data is only valid to {@link #size()}.
+   * @return the underlying buffer.
    */
   public byte[] getBuffer() {
     return buffer;
   }
 
-  /** Returns the length of the valid data 
+  /**
+   * Returns the length of the valid data
    * currently in the buffer.
+   *
+   * @return the length of the valid data.
    */
   public int size() {
     return currentPointer - startOffset;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteBufferPool.java
index aa5f8731c54a7..b30e7cfb9c5f0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteBufferPool.java
@@ -45,4 +45,9 @@ public interface ByteBufferPool {
    * @param buffer    a direct bytebuffer
    */
   void putBuffer(ByteBuffer buffer);
+
+  /**
+   * Clear the buffer pool thus releasing all the buffers.
+   */
+  default void release() { }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteWritable.java
index ffcdea2c9a3ab..c4b88f4b5c98b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ByteWritable.java
@@ -33,10 +33,16 @@ public ByteWritable() {}
 
   public ByteWritable(byte value) { set(value); }
 
-  /** Set the value of this ByteWritable. */
+  /**
+   * Set the value of this ByteWritable.
+   * @param value value.
+   */
   public void set(byte value) { this.value = value; }
 
-  /** Return the value of this ByteWritable. */
+  /**
+   * Return the value of this ByteWritable.
+   * @return value bytes.
+   */
   public byte get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
index c5538c9e56e85..80a23f86ce80b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
@@ -77,6 +77,8 @@ public BytesWritable(byte[] bytes, int length) {
   /**
    * Get a copy of the bytes that is exactly the length of the data.
    * See {@link #getBytes()} for faster access to the underlying array.
+   *
+   * @return copyBytes.
    */
   public byte[] copyBytes() {
     return Arrays.copyOf(bytes, size);
@@ -95,6 +97,7 @@ public byte[] getBytes() {
   /**
    * Get the data from the BytesWritable.
    * @deprecated Use {@link #getBytes()} instead.
+   * @return data from the BytesWritable.
    */
   @Deprecated
   public byte[] get() {
@@ -112,6 +115,7 @@ public int getLength() {
   /**
    * Get the current size of the buffer.
    * @deprecated Use {@link #getLength()} instead.
+   * @return current size of the buffer.
    */
   @Deprecated
   public int getSize() {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/CompressedWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/CompressedWritable.java
index 6550e1f2fde04..c0315ab828c3b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/CompressedWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/CompressedWritable.java
@@ -67,7 +67,11 @@ protected void ensureInflated() {
     }
   }
 
-  /** Subclasses implement this instead of {@link #readFields(DataInput)}. */
+  /**
+   * Subclasses implement this instead of {@link #readFields(DataInput)}.
+   * @param in data input.
+   * @throws IOException raised on errors performing I/O.
+   */
   protected abstract void readFieldsCompressed(DataInput in)
     throws IOException;
 
@@ -87,7 +91,12 @@ public final void write(DataOutput out) throws IOException {
     out.write(compressed);
   }
 
-  /** Subclasses implement this instead of {@link #write(DataOutput)}. */
+  /**
+   * Subclasses implement this instead of {@link #write(DataOutput)}.
+   *
+   * @param out data output.
+   * @throws IOException raised on errors performing I/O.
+   */
   protected abstract void writeCompressed(DataOutput out) throws IOException;
 
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataInputBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataInputBuffer.java
index 63c41c2e75008..85e905d870096 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataInputBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataInputBuffer.java
@@ -140,12 +140,23 @@ private DataInputBuffer(Buffer buffer) {
     this.buffer = buffer;
   }
 
-  /** Resets the data that the buffer reads. */
+  /**
+   * Resets the data that the buffer reads.
+   *
+   * @param input input.
+   * @param length length.
+   */
   public void reset(byte[] input, int length) {
     buffer.reset(input, 0, length);
   }
 
-  /** Resets the data that the buffer reads. */
+  /**
+   * Resets the data that the buffer reads.
+   *
+   * @param input input.
+   * @param start start.
+   * @param length length.
+   */
   public void reset(byte[] input, int start, int length) {
     buffer.reset(input, start, length);
   }
@@ -154,12 +165,18 @@ public byte[] getData() {
     return buffer.getData();
   }
 
-  /** Returns the current position in the input. */
+  /**
+   * Returns the current position in the input.
+   *
+   * @return position.
+   */
   public int getPosition() { return buffer.getPosition(); }
 
   /**
    * Returns the index one greater than the last valid character in the input
    * stream buffer.
+   *
+   * @return length.
    */
   public int getLength() { return buffer.getLength(); }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java
index 1d86b89701c03..4c1fa41e149c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputBuffer.java
@@ -99,27 +99,45 @@ private DataOutputBuffer(Buffer buffer) {
     this.buffer = buffer;
   }
 
-  /** Returns the current contents of the buffer.
+  /**
+   * Returns the current contents of the buffer.
    *  Data is only valid to {@link #getLength()}.
+   *
+   * @return data byte.
    */
   public byte[] getData() { return buffer.getData(); }
 
-  /** Returns the length of the valid data currently in the buffer. */
+  /**
+   * Returns the length of the valid data currently in the buffer.
+   * @return length.
+   */
   public int getLength() { return buffer.getLength(); }
 
-  /** Resets the buffer to empty. */
+  /**
+   * Resets the buffer to empty.
+   * @return DataOutputBuffer.
+   */
   public DataOutputBuffer reset() {
     this.written = 0;
     buffer.reset();
     return this;
   }
 
-  /** Writes bytes from a DataInput directly into the buffer. */
+  /**
+   * Writes bytes from a DataInput directly into the buffer.
+   * @param in data input.
+   * @param length length.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void write(DataInput in, int length) throws IOException {
     buffer.write(in, length);
   }
 
-  /** Write to a file stream */
+  /**
+   * Write to a file stream.
+   * @param out OutputStream.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void writeTo(OutputStream out) throws IOException {
     buffer.writeTo(out);
   }
@@ -128,6 +146,10 @@ public void writeTo(OutputStream out) throws IOException {
    * Overwrite an integer into the internal buffer. Note that this call can only
    * be used to overwrite existing data in the buffer, i.e., buffer#count cannot
    * be increased, and DataOutputStream#written cannot be increased.
+   *
+   * @param v v.
+   * @param offset offset.
+   * @throws IOException raised on errors performing I/O.
    */
   public void writeInt(int v, int offset) throws IOException {
     Preconditions.checkState(offset + 4 <= buffer.getLength());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
index 7453996ecab1c..7be50b0c539b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DefaultStringifier.java
@@ -158,6 +158,9 @@ public static <K> K load(Configuration conf, String keyName,
   public static <K> void storeArray(Configuration conf, K[] items,
       String keyName) throws IOException {
 
+    if (items.length == 0) {
+      throw new IndexOutOfBoundsException();
+    }
     DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
         GenericsUtil.getClass(items[0]));
     try {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
index 6a162c3ff2087..c4c2940622729 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
@@ -36,8 +36,8 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public final class ElasticByteBufferPool implements ByteBufferPool {
-  private static final class Key implements Comparable<Key> {
+public class ElasticByteBufferPool implements ByteBufferPool {
+  protected static final class Key implements Comparable<Key> {
     private final int capacity;
     private final long insertionTime;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
index be86159519b87..4b1dc7513d054 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/EnumSetWritable.java
@@ -64,8 +64,8 @@ public boolean add(E e) {
    * the argument <tt>value</tt>'s size is bigger than zero, the argument
    * <tt>elementType</tt> is not be used.
    * 
-   * @param value
-   * @param elementType
+   * @param value enumSet value.
+   * @param elementType elementType.
    */
   public EnumSetWritable(EnumSet<E> value, Class<E> elementType) {
     set(value, elementType);
@@ -75,7 +75,7 @@ public EnumSetWritable(EnumSet<E> value, Class<E> elementType) {
    * Construct a new EnumSetWritable. Argument <tt>value</tt> should not be null
    * or empty.
    * 
-   * @param value
+   * @param value enumSet value.
    */
   public EnumSetWritable(EnumSet<E> value) {
     this(value, null);
@@ -88,8 +88,8 @@ public EnumSetWritable(EnumSet<E> value) {
    * null. If the argument <tt>value</tt>'s size is bigger than zero, the
    * argument <tt>elementType</tt> is not be used.
    * 
-   * @param value
-   * @param elementType
+   * @param value enumSet Value.
+   * @param elementType elementType.
    */
   public void set(EnumSet<E> value, Class<E> elementType) {
     if ((value == null || value.size() == 0)
@@ -106,7 +106,10 @@ public void set(EnumSet<E> value, Class<E> elementType) {
     }
   }
 
-  /** Return the value of this EnumSetWritable. */
+  /**
+   * Return the value of this EnumSetWritable.
+   * @return EnumSet.
+   */
   public EnumSet<E> get() {
     return value;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FloatWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FloatWritable.java
index 367fc946da135..864bb8752f5c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FloatWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FloatWritable.java
@@ -33,10 +33,16 @@ public FloatWritable() {}
 
   public FloatWritable(float value) { set(value); }
 
-  /** Set the value of this FloatWritable. */
+  /**
+   * Set the value of this FloatWritable.
+   * @param value value.
+   */
   public void set(float value) { this.value = value; }
 
-  /** Return the value of this FloatWritable. */
+  /**
+   * Return the value of this FloatWritable.
+   * @return value.
+   */
   public float get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
index 7cfeed7f931d7..6de927467e478 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/GenericWritable.java
@@ -90,7 +90,7 @@ public abstract class GenericWritable implements Writable, Configurable {
   /**
    * Set the instance that is wrapped.
    * 
-   * @param obj
+   * @param obj input obj.
    */
   public void set(Writable obj) {
     instance = obj;
@@ -109,6 +109,7 @@ public void set(Writable obj) {
 
   /**
    * Return the wrapped instance.
+   * @return the wrapped instance.
    */
   public Writable get() {
     return instance;
@@ -145,6 +146,7 @@ public void write(DataOutput out) throws IOException {
   /**
    * Return all classes that may be wrapped.  Subclasses should implement this
    * to return a constant array of classes.
+   * @return all classes that may be wrapped.
    */
   abstract protected Class<? extends Writable>[] getTypes();
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
index 121af64b01182..f0a9b0b6952f2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
@@ -59,7 +59,8 @@ public class IOUtils {
    * @param out OutputStream to write to
    * @param buffSize the size of the buffer 
    * @param close whether or not close the InputStream and 
-   * OutputStream at the end. The streams are closed in the finally clause.  
+   * OutputStream at the end. The streams are closed in the finally clause.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void copyBytes(InputStream in, OutputStream out,
                                int buffSize, boolean close)
@@ -85,7 +86,8 @@ public static void copyBytes(InputStream in, OutputStream out,
    * 
    * @param in InputStrem to read from
    * @param out OutputStream to write to
-   * @param buffSize the size of the buffer 
+   * @param buffSize the size of the buffer.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void copyBytes(InputStream in, OutputStream out, int buffSize) 
     throws IOException {
@@ -107,7 +109,8 @@ public static void copyBytes(InputStream in, OutputStream out, int buffSize)
    *
    * @param in InputStrem to read from
    * @param out OutputStream to write to
-   * @param conf the Configuration object 
+   * @param conf the Configuration object.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void copyBytes(InputStream in, OutputStream out, Configuration conf)
     throws IOException {
@@ -123,6 +126,7 @@ public static void copyBytes(InputStream in, OutputStream out, Configuration con
    * @param conf the Configuration object
    * @param close whether or not close the InputStream and 
    * OutputStream at the end. The streams are closed in the finally clause.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void copyBytes(InputStream in, OutputStream out, Configuration conf, boolean close)
     throws IOException {
@@ -181,6 +185,7 @@ public static void copyBytes(InputStream in, OutputStream out, long count,
    * @param off - offset within buf
    * @param len - amount of data to be read
    * @return number of bytes read
+   * @throws IOException raised on errors performing I/O.
    */
   public static int wrappedReadForCompressedData(InputStream is, byte[] buf,
       int off, int len) throws IOException {
@@ -407,6 +412,7 @@ public static List<String> listDirectory(File dir, FilenameFilter filter)
    * once the sync is done.<br>
    * Borrowed from Uwe Schindler in LUCENE-5588
    * @param fileToSync the file to fsync
+   * @throws IOException raised on errors performing I/O.
    */
   public static void fsync(File fileToSync) throws IOException {
     if (!fileToSync.exists()) {
@@ -440,7 +446,7 @@ public static void fsync(File fileToSync) throws IOException {
    * @param isDir if true, the given file is a directory (Channel should be
    *          opened for read and ignore IOExceptions, because not all file
    *          systems and operating systems allow to fsync on a directory)
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void fsync(FileChannel channel, boolean isDir)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/InputBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/InputBuffer.java
index 0d084b8396f16..686b359f57d32 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/InputBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/InputBuffer.java
@@ -75,20 +75,35 @@ private InputBuffer(Buffer buffer) {
     this.buffer = buffer;
   }
 
-  /** Resets the data that the buffer reads. */
+  /**
+   * Resets the data that the buffer reads.
+   * @param input input.
+   * @param length length.
+   */
   public void reset(byte[] input, int length) {
     buffer.reset(input, 0, length);
   }
 
-  /** Resets the data that the buffer reads. */
+  /**
+   * Resets the data that the buffer reads.
+   * @param input input.
+   * @param start start.
+   * @param length length.
+   */
   public void reset(byte[] input, int start, int length) {
     buffer.reset(input, start, length);
   }
 
-  /** Returns the current position in the input. */
+  /**
+   * Returns the current position in the input.
+   * @return the current position in the input.
+   */
   public int getPosition() { return buffer.getPosition(); }
 
-  /** Returns the length of the input. */
+  /**
+   * Returns the length of the input.
+   * @return length of the input.
+   */
   public int getLength() { return buffer.getLength(); }
 
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IntWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IntWritable.java
index f656d028cb054..ffcf93946d06a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IntWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IntWritable.java
@@ -36,10 +36,16 @@ public IntWritable() {}
 
   public IntWritable(int value) { set(value); }
 
-  /** Set the value of this IntWritable. */
+  /**
+   * Set the value of this IntWritable.
+   * @param value input value.
+   */
   public void set(int value) { this.value = value; }
 
-  /** Return the value of this IntWritable. */
+  /**
+   * Return the value of this IntWritable.
+   * @return value of this IntWritable.
+   */
   public int get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
index b77ca6781a639..9262af87bc2e1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/LongWritable.java
@@ -36,10 +36,16 @@ public LongWritable() {}
 
   public LongWritable(long value) { set(value); }
 
-  /** Set the value of this LongWritable. */
+  /**
+   * Set the value of this LongWritable.
+   * @param value value.
+   */
   public void set(long value) { this.value = value; }
 
-  /** Return the value of this LongWritable. */
+  /**
+   * Return the value of this LongWritable.
+   * @return value of this LongWritable.
+   */
   public long get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
index 99c17acdd43d2..edfcf6e1e7754 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MD5Hash.java
@@ -54,12 +54,18 @@ public MD5Hash() {
     this.digest = new byte[MD5_LEN];
   }
 
-  /** Constructs an MD5Hash from a hex string. */
+  /**
+   * Constructs an MD5Hash from a hex string.
+   * @param hex input hex.
+   */
   public MD5Hash(String hex) {
     setDigest(hex);
   }
   
-  /** Constructs an MD5Hash with a specified value. */
+  /**
+   * Constructs an MD5Hash with a specified value.
+   * @param digest digest.
+   */
   public MD5Hash(byte[] digest) {
     if (digest.length != MD5_LEN)
       throw new IllegalArgumentException("Wrong length: " + digest.length);
@@ -72,7 +78,12 @@ public void readFields(DataInput in) throws IOException {
     in.readFully(digest);
   }
 
-  /** Constructs, reads and returns an instance. */
+  /**
+   * Constructs, reads and returns an instance.
+   * @param in in.
+   * @throws IOException raised on errors performing I/O.
+   * @return MD5Hash.
+   */
   public static MD5Hash read(DataInput in) throws IOException {
     MD5Hash result = new MD5Hash();
     result.readFields(in);
@@ -85,21 +96,32 @@ public void write(DataOutput out) throws IOException {
     out.write(digest);
   }
 
-  /** Copy the contents of another instance into this instance. */
+  /**
+   * Copy the contents of another instance into this instance.
+   * @param that that.
+   */
   public void set(MD5Hash that) {
     System.arraycopy(that.digest, 0, this.digest, 0, MD5_LEN);
   }
 
-  /** Returns the digest bytes. */
+  /**
+   * Returns the digest bytes.
+   * @return digest.
+   */
   public byte[] getDigest() { return digest; }
 
-  /** Construct a hash value for a byte array. */
+  /**
+   * Construct a hash value for a byte array.
+   * @param data data.
+   * @return MD5Hash.
+   */
   public static MD5Hash digest(byte[] data) {
     return digest(data, 0, data.length);
   }
 
   /**
-   * Create a thread local MD5 digester
+   * Create a thread local MD5 digester.
+   * @return MessageDigest.
    */
   public static MessageDigest getDigester() {
     MessageDigest digester = DIGESTER_FACTORY.get();
@@ -107,7 +129,12 @@ public static MessageDigest getDigester() {
     return digester;
   }
 
-  /** Construct a hash value for the content from the InputStream. */
+  /**
+   * Construct a hash value for the content from the InputStream.
+   * @param in input stream.
+   * @return MD5Hash.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static MD5Hash digest(InputStream in) throws IOException {
     final byte[] buffer = new byte[4*1024]; 
 
@@ -119,7 +146,13 @@ public static MD5Hash digest(InputStream in) throws IOException {
     return new MD5Hash(digester.digest());
   }
 
-  /** Construct a hash value for a byte array. */
+  /**
+   * Construct a hash value for a byte array.
+   * @param data data.
+   * @param start start.
+   * @param len len.
+   * @return MD5Hash.
+   */
   public static MD5Hash digest(byte[] data, int start, int len) {
     byte[] digest;
     MessageDigest digester = getDigester();
@@ -128,7 +161,13 @@ public static MD5Hash digest(byte[] data, int start, int len) {
     return new MD5Hash(digest);
   }
 
-  /** Construct a hash value for an array of byte array. */
+  /**
+   * Construct a hash value for an array of byte array.
+   * @param dataArr dataArr.
+   * @param start start.
+   * @param len len.
+   * @return MD5Hash.
+   */
   public static MD5Hash digest(byte[][] dataArr, int start, int len) {
     byte[] digest;
     MessageDigest digester = getDigester();
@@ -139,17 +178,28 @@ public static MD5Hash digest(byte[][] dataArr, int start, int len) {
     return new MD5Hash(digest);
   }
 
-  /** Construct a hash value for a String. */
+  /**
+   * Construct a hash value for a String.
+   * @param string string.
+   * @return MD5Hash.
+   */
   public static MD5Hash digest(String string) {
     return digest(UTF8.getBytes(string));
   }
 
-  /** Construct a hash value for a String. */
+  /**
+   * Construct a hash value for a String.
+   * @param utf8 utf8.
+   * @return MD5Hash.
+   */
   public static MD5Hash digest(UTF8 utf8) {
     return digest(utf8.getBytes(), 0, utf8.getLength());
   }
 
-  /** Construct a half-sized version of this MD5.  Fits in a long **/
+  /**
+   * Construct a half-sized version of this MD5.  Fits in a long.
+   * @return halfDigest.
+   */
   public long halfDigest() {
     long value = 0;
     for (int i = 0; i < 8; i++)
@@ -226,7 +276,10 @@ public String toString() {
     return buf.toString();
   }
 
-  /** Sets the digest value from a hex string. */
+  /**
+   * Sets the digest value from a hex string.
+   * @param hex hex.
+   */
   public void setDigest(String hex) {
     if (hex.length() != MD5_LEN*2)
       throw new IllegalArgumentException("Wrong length: " + hex.length());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
index 51db0b3f0afef..7b3cd78e3ccf3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
@@ -98,8 +98,16 @@ public static class Writer implements java.io.Closeable {
     private long lastIndexKeyCount = Long.MIN_VALUE;
 
 
-    /** Create the named map for keys of the named class. 
+    /**
+     * Create the named map for keys of the named class.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     *
+     * @param conf configuration.
+     * @param fs filesystem.
+     * @param dirName dirName.
+     * @param keyClass keyClass.
+     * @param valClass valClass.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -108,8 +116,18 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
       this(conf, new Path(dirName), keyClass(keyClass), valueClass(valClass));
     }
 
-    /** Create the named map for keys of the named class. 
+    /**
+     * Create the named map for keys of the named class.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     *
+     * @param conf configuration.
+     * @param fs fs.
+     * @param dirName dirName.
+     * @param keyClass keyClass.
+     * @param valClass valClass.
+     * @param compress compress.
+     * @param progress progress.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -120,8 +138,19 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
            compression(compress), progressable(progress));
     }
 
-    /** Create the named map for keys of the named class. 
+    /**
+     * Create the named map for keys of the named class.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     *
+     * @param conf configuration.
+     * @param fs FileSystem.
+     * @param dirName dirName.
+     * @param keyClass keyClass.
+     * @param valClass valClass.
+     * @param compress compress.
+     * @param codec codec.
+     * @param progress progress.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -132,8 +161,16 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
            compression(compress, codec), progressable(progress));
     }
 
-    /** Create the named map for keys of the named class. 
+    /**
+     * Create the named map for keys of the named class.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     * @param conf configuration.
+     * @param fs fs.
+     * @param dirName dirName.
+     * @param keyClass keyClass.
+     * @param valClass valClass.
+     * @param compress compress.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -145,6 +182,12 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
 
     /** Create the named map using the named key comparator. 
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     * @param conf configuration.
+     * @param fs fs.
+     * @param dirName dirName.
+     * @param comparator comparator.
+     * @param valClass valClass.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -154,7 +197,14 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
            valueClass(valClass));
     }
 
-    /** Create the named map using the named key comparator. 
+    /** Create the named map using the named key comparator.
+     * @param conf configuration.
+     * @param fs filesystem.
+     * @param dirName dirName.
+     * @param comparator comparator.
+     * @param valClass valClass.
+     * @param compress compress.
+     * @throws IOException raised on errors performing I/O.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
      */
     @Deprecated
@@ -165,8 +215,18 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
            valueClass(valClass), compression(compress));
     }
 
-    /** Create the named map using the named key comparator. 
+    /**
+     * Create the named map using the named key comparator.
      * @deprecated Use Writer(Configuration, Path, Option...)} instead.
+     *
+     * @param conf configuration.
+     * @param fs filesystem.
+     * @param dirName dirName.
+     * @param comparator comparator.
+     * @param valClass valClass.
+     * @param compress CompressionType.
+     * @param progress progress.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -178,8 +238,19 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
            progressable(progress));
     }
 
-    /** Create the named map using the named key comparator. 
+    /**
+     * Create the named map using the named key comparator.
      * @deprecated Use Writer(Configuration, Path, Option...) instead.
+     *
+     * @param conf configuration.
+     * @param fs FileSystem.
+     * @param dirName dirName.
+     * @param comparator comparator.
+     * @param valClass valClass.
+     * @param compress CompressionType.
+     * @param codec codec.
+     * @param progress progress.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(Configuration conf, FileSystem fs, String dirName,
@@ -285,16 +356,26 @@ public Writer(Configuration conf,
       this.index = SequenceFile.createWriter(conf, indexOptions);      
     }
 
-    /** The number of entries that are added before an index entry is added.*/
+    /**
+     * The number of entries that are added before an index entry is added.
+     * @return indexInterval
+     */
     public int getIndexInterval() { return indexInterval; }
 
-    /** Sets the index interval.
+    /**
+     * Sets the index interval.
      * @see #getIndexInterval()
+     *
+     * @param interval interval.
      */
     public void setIndexInterval(int interval) { indexInterval = interval; }
 
-    /** Sets the index interval and stores it in conf
+    /**
+     * Sets the index interval and stores it in conf.
      * @see #getIndexInterval()
+     *
+     * @param conf configuration.
+     * @param interval interval.
      */
     public static void setIndexInterval(Configuration conf, int interval) {
       conf.setInt(INDEX_INTERVAL, interval);
@@ -307,8 +388,14 @@ public synchronized void close() throws IOException {
       index.close();
     }
 
-    /** Append a key/value pair to the map.  The key must be greater or equal
-     * to the previous key added to the map. */
+    /**
+     * Append a key/value pair to the map.  The key must be greater or equal
+     * to the previous key added to the map.
+     *
+     * @param key key.
+     * @param val value.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized void append(WritableComparable key, Writable val)
       throws IOException {
 
@@ -370,10 +457,18 @@ public static class Reader implements java.io.Closeable {
     private WritableComparable[] keys;
     private long[] positions;
 
-    /** Returns the class of keys in this file. */
+    /**
+     * Returns the class of keys in this file.
+     *
+     * @return keyClass.
+     */
     public Class<?> getKeyClass() { return data.getKeyClass(); }
 
-    /** Returns the class of values in this file. */
+    /**
+     * Returns the class of values in this file.
+     *
+     * @return Value Class.
+     */
     public Class<?> getValueClass() { return data.getValueClass(); }
 
     public static interface Option extends SequenceFile.Reader.Option {}
@@ -403,8 +498,14 @@ public Reader(Path dir, Configuration conf,
       open(dir, comparator, conf, opts);
     }
  
-    /** Construct a map reader for the named map.
+    /**
+     * Construct a map reader for the named map.
      * @deprecated
+     *
+     * @param fs FileSystem.
+     * @param dirName dirName.
+     * @param conf configuration.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Reader(FileSystem fs, String dirName, 
@@ -412,8 +513,15 @@ public Reader(FileSystem fs, String dirName,
       this(new Path(dirName), conf);
     }
 
-    /** Construct a map reader for the named map using the named comparator.
+    /**
+     * Construct a map reader for the named map using the named comparator.
      * @deprecated
+     *
+     * @param fs FileSystem.
+     * @param dirName dirName.
+     * @param comparator WritableComparator.
+     * @param conf Configuration.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Reader(FileSystem fs, String dirName, WritableComparator comparator, 
@@ -450,6 +558,12 @@ protected synchronized void open(Path dir,
     /**
      * Override this method to specialize the type of
      * {@link SequenceFile.Reader} returned.
+     *
+     * @param dataFile data file.
+     * @param conf configuration.
+     * @param options options.
+     * @throws IOException raised on errors performing I/O.
+     * @return SequenceFile.Reader.
      */
     protected SequenceFile.Reader 
       createDataFileReader(Path dataFile, Configuration conf,
@@ -516,13 +630,21 @@ private void readIndex() throws IOException {
       }
     }
 
-    /** Re-positions the reader before its first key. */
+    /**
+     * Re-positions the reader before its first key.
+     *
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized void reset() throws IOException {
       data.seek(firstPosition);
     }
 
-    /** Get the key at approximately the middle of the file. Or null if the
-     *  file is empty. 
+    /**
+     * Get the key at approximately the middle of the file. Or null if the
+     *  file is empty.
+     *
+     * @throws IOException raised on errors performing I/O.
+     * @return WritableComparable.
      */
     public synchronized WritableComparable midKey() throws IOException {
 
@@ -534,9 +656,11 @@ public synchronized WritableComparable midKey() throws IOException {
       return keys[(count - 1) / 2];
     }
     
-    /** Reads the final key from the file.
+    /**
+     * Reads the final key from the file.
      *
      * @param key key to read into
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized void finalKey(WritableComparable key)
       throws IOException {
@@ -556,9 +680,14 @@ public synchronized void finalKey(WritableComparable key)
       }
     }
 
-    /** Positions the reader at the named key, or if none such exists, at the
+    /**
+     * Positions the reader at the named key, or if none such exists, at the
      * first entry after the named key.  Returns true iff the named key exists
      * in this map.
+     *
+     * @param key key.
+     * @throws IOException raised on errors performing I/O.
+     * @return if the named key exists in this map true, not false.
      */
     public synchronized boolean seek(WritableComparable key) throws IOException {
       return seekInternal(key) == 0;
@@ -669,15 +798,28 @@ else if (cmp > 0)
       return -(low + 1);                          // key not found.
     }
 
-    /** Read the next key/value pair in the map into <code>key</code> and
+    /**
+     * Read the next key/value pair in the map into <code>key</code> and
      * <code>val</code>.  Returns true if such a pair exists and false when at
-     * the end of the map */
+     * the end of the map.
+     *
+     * @param key WritableComparable.
+     * @param val Writable.
+     * @return if such a pair exists true,not false.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized boolean next(WritableComparable key, Writable val)
       throws IOException {
       return data.next(key, val);
     }
 
-    /** Return the value for the named key, or null if none exists. */
+    /**
+     * Return the value for the named key, or null if none exists.
+     * @param key key.
+     * @param val val.
+     * @return Writable if such a pair exists true,not false.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized Writable get(WritableComparable key, Writable val)
       throws IOException {
       if (seek(key)) {
@@ -692,9 +834,10 @@ public synchronized Writable get(WritableComparable key, Writable val)
      * Returns <code>key</code> or if it does not exist, at the first entry
      * after the named key.
      * 
--     * @param key       - key that we're trying to find
--     * @param val       - data value if key is found
--     * @return          - the key that was the closest match or null if eof.
+     * @param key key that we're trying to find.
+     * @param val data value if key is found.
+     * @return the key that was the closest match or null if eof.
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized WritableComparable getClosest(WritableComparable key,
       Writable val)
@@ -711,6 +854,7 @@ public synchronized WritableComparable getClosest(WritableComparable key,
      * the first entry that falls just before the <code>key</code>.  Otherwise,
      * return the record that sorts just after.
      * @return          - the key that was the closest match or null if eof.
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized WritableComparable getClosest(WritableComparable key,
         Writable val, final boolean before)
@@ -730,7 +874,10 @@ public synchronized WritableComparable getClosest(WritableComparable key,
       return nextKey;
     }
 
-    /** Close the map. */
+    /**
+     * Close the map.
+     * @throws IOException raised on errors performing I/O.
+     */
     @Override
     public synchronized void close() throws IOException {
       if (!indexClosed) {
@@ -741,7 +888,13 @@ public synchronized void close() throws IOException {
 
   }
 
-  /** Renames an existing map directory. */
+  /**
+   * Renames an existing map directory.
+   * @param fs fs.
+   * @param oldName oldName.
+   * @param newName newName.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static void rename(FileSystem fs, String oldName, String newName)
     throws IOException {
     Path oldDir = new Path(oldName);
@@ -751,7 +904,12 @@ public static void rename(FileSystem fs, String oldName, String newName)
     }
   }
 
-  /** Deletes the named map file. */
+  /**
+   * Deletes the named map file.
+   * @param fs input fs.
+   * @param name input name.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static void delete(FileSystem fs, String name) throws IOException {
     Path dir = new Path(name);
     Path data = new Path(dir, DATA_FILE_NAME);
@@ -769,8 +927,9 @@ public static void delete(FileSystem fs, String name) throws IOException {
    * @param keyClass key class (has to be a subclass of Writable)
    * @param valueClass value class (has to be a subclass of Writable)
    * @param dryrun do not perform any changes, just report what needs to be done
+   * @param conf configuration.
    * @return number of valid entries in this MapFile, or -1 if no fixing was needed
-   * @throws Exception
+   * @throws Exception Exception.
    */
   public static long fix(FileSystem fs, Path dir,
                          Class<? extends Writable> keyClass,
@@ -870,11 +1029,12 @@ public Merger(Configuration conf) throws IOException {
     }
 
     /**
-     * Merge multiple MapFiles to one Mapfile
+     * Merge multiple MapFiles to one Mapfile.
      *
-     * @param inMapFiles
-     * @param outMapFile
-     * @throws IOException
+     * @param inMapFiles input inMapFiles.
+     * @param deleteInputs deleteInputs.
+     * @param outMapFile input outMapFile.
+     * @throws IOException raised on errors performing I/O.
      */
     public void merge(Path[] inMapFiles, boolean deleteInputs,
         Path outMapFile) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
index c9d7ade43064f..452965b7c8220 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
@@ -42,7 +42,11 @@ private MultipleIOException(List<IOException> exceptions) {
   /** @return the underlying exceptions */
   public List<IOException> getExceptions() {return exceptions;}
 
-  /** A convenient method to create an {@link IOException}. */
+  /**
+   * A convenient method to create an {@link IOException}.
+   * @param exceptions IOException List.
+   * @return IOException.
+   */
   public static IOException createIOException(List<IOException> exceptions) {
     if (exceptions == null || exceptions.isEmpty()) {
       return null;
@@ -60,7 +64,10 @@ public static IOException createIOException(List<IOException> exceptions) {
   public static class Builder {
     private List<IOException> exceptions;
     
-    /** Add the given {@link Throwable} to the exception list. */
+    /**
+     * Add the given {@link Throwable} to the exception list.
+     * @param t Throwable.
+     */
     public void add(Throwable t) {
       if (exceptions == null) {
         exceptions = new ArrayList<>();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/NullWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/NullWritable.java
index 77c590fdb6344..d6e4846264f98 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/NullWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/NullWritable.java
@@ -32,7 +32,10 @@ public class NullWritable implements WritableComparable<NullWritable> {
 
   private NullWritable() {}                       // no public ctor
 
-  /** Returns the single instance of this class. */
+  /**
+   * Returns the single instance of this class.
+   * @return the single instance of this class.
+   */
   public static NullWritable get() { return THIS; }
   
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
index b35a32f288b4b..29c06a01ad6e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
@@ -54,13 +54,22 @@ public ObjectWritable(Class declaredClass, Object instance) {
     this.instance = instance;
   }
 
-  /** Return the instance, or null if none. */
+  /**
+   * Return the instance, or null if none.
+   * @return the instance, or null if none.
+   */
   public Object get() { return instance; }
   
-  /** Return the class this is meant to be. */
+  /**
+   * Return the class this is meant to be.
+   * @return the class this is meant to be.
+   */
   public Class getDeclaredClass() { return declaredClass; }
   
-  /** Reset the instance. */
+  /**
+   * Reset the instance.
+   * @param instance instance.
+   */
   public void set(Object instance) {
     this.declaredClass = instance.getClass();
     this.instance = instance;
@@ -120,8 +129,16 @@ public void write(DataOutput out) throws IOException {
     }
   }
 
-  /** Write a {@link Writable}, {@link String}, primitive type, or an array of
-   * the preceding. */
+  /**
+   * Write a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   *
+   * @param out DataOutput.
+   * @param instance instance.
+   * @param conf Configuration.
+   * @param declaredClass declaredClass.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static void writeObject(DataOutput out, Object instance,
                                  Class declaredClass, 
                                  Configuration conf) throws IOException {
@@ -137,6 +154,13 @@ public static void writeObject(DataOutput out, Object instance,
      * usages, to preserve the ability to interchange files with other clusters 
      * that may not be running the same version of software.  Sometime in ~2013 
      * we can consider removing this parameter and always using the compact format.
+     *
+     * @param conf configuration.
+     * @param out dataoutput.
+     * @param declaredClass declaredClass.
+     * @param instance instance.
+     * @throws IOException raised on errors performing I/O.
+     *
      */
     public static void writeObject(DataOutput out, Object instance,
         Class declaredClass, Configuration conf, boolean allowCompactArrays) 
@@ -210,15 +234,30 @@ public static void writeObject(DataOutput out, Object instance,
   }
   
   
-  /** Read a {@link Writable}, {@link String}, primitive type, or an array of
-   * the preceding. */
+  /**
+   * Read a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   *
+   * @param conf configuration.
+   * @param in DataInput.
+   * @return Object.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static Object readObject(DataInput in, Configuration conf)
     throws IOException {
     return readObject(in, null, conf);
   }
     
-  /** Read a {@link Writable}, {@link String}, primitive type, or an array of
-   * the preceding. */
+  /**
+   * Read a {@link Writable}, {@link String}, primitive type, or an array of
+   * the preceding.
+   *
+   * @param in DataInput.
+   * @param objectWritable objectWritable.
+   * @param conf configuration.
+   * @return Object.
+   * @throws IOException raised on errors performing I/O.
+   */
   @SuppressWarnings("unchecked")
   public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
     throws IOException {
@@ -365,6 +404,10 @@ static Method getStaticProtobufMethod(Class<?> declaredClass, String method,
    * Find and load the class with given name <tt>className</tt> by first finding
    * it in the specified <tt>conf</tt>. If the specified <tt>conf</tt> is null,
    * try load it directly.
+   *
+   * @param conf configuration.
+   * @param className classname.
+   * @return Class.
    */
   public static Class<?> loadClass(Configuration conf, String className) {
     Class<?> declaredClass = null;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/OutputBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/OutputBuffer.java
index 15a396dc2bf55..f80c0a71883d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/OutputBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/OutputBuffer.java
@@ -77,21 +77,33 @@ private OutputBuffer(Buffer buffer) {
     this.buffer = buffer;
   }
 
-  /** Returns the current contents of the buffer.
+  /**
+   * Returns the current contents of the buffer.
    *  Data is only valid to {@link #getLength()}.
+   *
+   * @return the current contents of the buffer.
    */
   public byte[] getData() { return buffer.getData(); }
 
-  /** Returns the length of the valid data currently in the buffer. */
+  /**
+   * Returns the length of the valid data currently in the buffer.
+   * @return the length of the valid data
+   *          currently in the buffer.
+   */
   public int getLength() { return buffer.getLength(); }
 
-  /** Resets the buffer to empty. */
+  /** @return Resets the buffer to empty. */
   public OutputBuffer reset() {
     buffer.reset();
     return this;
   }
 
-  /** Writes bytes from a InputStream directly into the buffer. */
+  /**
+   * Writes bytes from a InputStream directly into the buffer.
+   * @param in input in.
+   * @param length input length.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void write(InputStream in, int length) throws IOException {
     buffer.write(in, length);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/RawComparator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/RawComparator.java
index a52190db5f4d1..354dda964e92b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/RawComparator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/RawComparator.java
@@ -29,7 +29,7 @@
  * A {@link Comparator} that operates directly on byte representations of
  * objects.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  * @see DeserializerComparator
  */
 @InterfaceAudience.Public
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
index 65e751eca417e..2a6fafce545f3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
@@ -50,7 +50,7 @@ public class ReadaheadPool {
   private static ReadaheadPool instance;
 
   /**
-   * Return the singleton instance for the current process.
+   * @return Return the singleton instance for the current process.
    */
   public static ReadaheadPool getInstance() {
     synchronized (ReadaheadPool.class) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
index 016daf9f352c1..cddddcc6c9a45 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java
@@ -90,7 +90,7 @@ public class SecureIOUtils {
   private final static FileSystem rawFilesystem;
 
   /**
-   * Open the given File for random read access, verifying the expected user/
+   * @return Open the given File for random read access, verifying the expected user/
    * group constraints if security is enabled.
    * 
    * Note that this function provides no additional security checks if hadoop
@@ -114,8 +114,14 @@ public static RandomAccessFile openForRandomRead(File f,
   }
 
   /**
-   * Same as openForRandomRead except that it will run even if security is off.
+   * @return Same as openForRandomRead except that it will run even if security is off.
    * This is used by unit tests.
+   *
+   * @param f input f.
+   * @param mode input mode.
+   * @param expectedOwner input expectedOwner.
+   * @param expectedGroup input expectedGroup.
+   * @throws IOException raised on errors performing I/O.
    */
   @VisibleForTesting
   protected static RandomAccessFile forceSecureOpenForRandomRead(File f,
@@ -145,6 +151,7 @@ protected static RandomAccessFile forceSecureOpenForRandomRead(File f,
    * @param expectedGroup the expected group owner for the file
    * @throws IOException if an IO Error occurred or the user/group does not
    * match if security is enabled
+   * @return FSDataInputStream.
    */
   public static FSDataInputStream openFSDataInputStream(File file,
       String expectedOwner, String expectedGroup) throws IOException {
@@ -157,6 +164,12 @@ public static FSDataInputStream openFSDataInputStream(File file,
   /**
    * Same as openFSDataInputStream except that it will run even if security is
    * off. This is used by unit tests.
+   *
+   * @param file input file.
+   * @param expectedOwner input expectedOwner.
+   * @param expectedGroup input expectedGroup.
+   * @throws IOException raised on errors performing I/O.
+   * @return FSDataInputStream.
    */
   @VisibleForTesting
   protected static FSDataInputStream forceSecureOpenFSDataInputStream(
@@ -182,7 +195,7 @@ protected static FSDataInputStream forceSecureOpenFSDataInputStream(
    * Open the given File for read access, verifying the expected user/group
    * constraints if security is enabled.
    *
-   * Note that this function provides no additional checks if Hadoop
+   * @return Note that this function provides no additional checks if Hadoop
    * security is disabled, since doing the checks would be too expensive
    * when native libraries are not available.
    *
@@ -201,8 +214,12 @@ public static FileInputStream openForRead(File f, String expectedOwner,
   }
 
   /**
-   * Same as openForRead() except that it will run even if security is off.
+   * @return Same as openForRead() except that it will run even if security is off.
    * This is used by unit tests.
+   * @param f input f.
+   * @param expectedOwner input expectedOwner.
+   * @param expectedGroup input expectedGroup.
+   * @throws IOException raised on errors performing I/O.
    */
   @VisibleForTesting
   protected static FileInputStream forceSecureOpenForRead(File f, String expectedOwner,
@@ -251,6 +268,7 @@ private static FileOutputStream insecureCreateForWrite(File f,
    *
    * @throws AlreadyExistsException if the file already exists
    * @throws IOException if any other error occurred
+   * @return createForWrite FileOutputStream.
    */
   public static FileOutputStream createForWrite(File f, int permissions)
   throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 037bbd3fd0f21..a0b45814f1c77 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -57,6 +57,11 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SEQFILE_COMPRESS_BLOCKSIZE_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSUM_ERRORS_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_SKIP_CHECKSUM_ERRORS_KEY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /** 
  * <code>SequenceFile</code>s are flat files consisting of binary key/value 
@@ -264,7 +269,7 @@ static public void setDefaultCompressionType(Configuration job,
    * @param conf the configuration to use
    * @param opts the options to create the file with
    * @return a new Writer
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static Writer createWriter(Configuration conf, Writer.Option... opts
                                     ) throws IOException {
@@ -296,7 +301,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param keyClass The 'key' type.
    * @param valClass The 'value' type.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -318,7 +323,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param valClass The 'value' type.
    * @param compressionType The compression type.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -343,7 +348,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param compressionType The compression type.
    * @param progress The Progressable object to track progress.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -370,7 +375,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param compressionType The compression type.
    * @param codec The compression codec.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -398,7 +403,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param progress The Progressable object to track progress.
    * @param metadata The metadata of the file.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -432,7 +437,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param progress The Progressable object to track progress.
    * @param metadata The metadata of the file.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -470,7 +475,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param codec The compression codec.
    * @param metadata The metadata of the file.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Deprecated
   public static Writer
@@ -503,7 +508,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param createFlag gives the semantics of create: overwrite, append etc.
    * @param opts file creation options; see {@link CreateOpts}.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static Writer
   createWriter(FileContext fc, Configuration conf, Path name,
@@ -527,7 +532,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param codec The compression codec.
    * @param progress The Progressable object to track progress.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -555,7 +560,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param codec The compression codec.
    * @param metadata The metadata of the file.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -580,7 +585,7 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
    * @param compressionType The compression type.
    * @param codec The compression codec.
    * @return Returns the handle to the constructed SequenceFile Writer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated Use {@link #createWriter(Configuration, Writer.Option...)}
    *     instead.
    */
@@ -598,22 +603,26 @@ public static Writer createWriter(Configuration conf, Writer.Option... opts
   /** The interface to 'raw' values of SequenceFiles. */
   public static interface ValueBytes {
 
-    /** Writes the uncompressed bytes to the outStream.
+    /**
+     * Writes the uncompressed bytes to the outStream.
      * @param outStream : Stream to write uncompressed bytes into.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void writeUncompressedBytes(DataOutputStream outStream)
       throws IOException;
 
-    /** Write compressed bytes to outStream. 
+    /**
+     * Write compressed bytes to outStream.
      * Note: that it will NOT compress the bytes if they are not compressed.
      * @param outStream : Stream to write compressed bytes into.
+     * @throws IllegalArgumentException an illegal or inappropriate argument.
+     * @throws IOException raised on errors performing I/O.
      */
     public void writeCompressedBytes(DataOutputStream outStream) 
       throws IllegalArgumentException, IOException;
 
     /**
-     * Size of stored data.
+     * @return Size of stored data.
      */
     public int getSize();
   }
@@ -1185,10 +1194,17 @@ public static Option syncInterval(int value) {
           codec, metadata, syncInterval);
     }
 
-    /** Create the named file.
+    /**
+     * Create the named file.
      * @deprecated Use 
      *   {@link SequenceFile#createWriter(Configuration, Writer.Option...)} 
      *   instead.
+     * @param fs input filesystem.
+     * @param conf input configuration.
+     * @param name input name.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(FileSystem fs, Configuration conf, Path name, 
@@ -1198,10 +1214,19 @@ public Writer(FileSystem fs, Configuration conf, Path name,
            new Metadata(), SYNC_INTERVAL);
     }
     
-    /** Create the named file with write-progress reporter.
+    /**
+     * Create the named file with write-progress reporter.
      * @deprecated Use 
      *   {@link SequenceFile#createWriter(Configuration, Writer.Option...)} 
      *   instead.
+     * @param fs input filesystem.
+     * @param conf input configuration.
+     * @param name input name.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @param progress input progress.
+     * @param metadata input metadata.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(FileSystem fs, Configuration conf, Path name, 
@@ -1212,10 +1237,22 @@ public Writer(FileSystem fs, Configuration conf, Path name,
            null, metadata, SYNC_INTERVAL);
     }
     
-    /** Create the named file with write-progress reporter. 
+    /**
+     * Create the named file with write-progress reporter.
      * @deprecated Use 
      *   {@link SequenceFile#createWriter(Configuration, Writer.Option...)} 
      *   instead.
+     * @param fs input filesystem.
+     * @param conf input configuration.
+     * @param name input name.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @param bufferSize input bufferSize.
+     * @param replication input replication.
+     * @param blockSize input blockSize.
+     * @param progress input progress.
+     * @param metadata input metadata.
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Writer(FileSystem fs, Configuration conf, Path name,
@@ -1316,16 +1353,19 @@ void init(Configuration config, FSDataOutputStream outStream,
       }
     }
     
-    /** Returns the class of keys in this file. */
+    /** @return Returns the class of keys in this file. */
     public Class getKeyClass() { return keyClass; }
 
-    /** Returns the class of values in this file. */
+    /** @return Returns the class of values in this file. */
     public Class getValueClass() { return valClass; }
 
-    /** Returns the compression codec of data in this file. */
+    /** @return Returns the compression codec of data in this file. */
     public CompressionCodec getCompressionCodec() { return codec; }
     
-    /** create a sync point */
+    /**
+     * create a sync point.
+     * @throws IOException raised on errors performing I/O.
+     */
     public void sync() throws IOException {
       if (sync != null && lastSyncPos != out.getPos()) {
         out.writeInt(SYNC_ESCAPE);                // mark the start of the sync
@@ -1335,8 +1375,9 @@ public void sync() throws IOException {
     }
 
     /**
-     * flush all currently written data to the file system
+     * flush all currently written data to the file system.
      * @deprecated Use {@link #hsync()} or {@link #hflush()} instead
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public void syncFs() throws IOException {
@@ -1408,13 +1449,23 @@ synchronized void checkAndWriteSync() throws IOException {
       }
     }
 
-    /** Append a key/value pair. */
+    /**
+     * Append a key/value pair.
+     * @param key input Writable key.
+     * @param val input Writable val.
+     * @throws IOException raised on errors performing I/O.
+     */
     public void append(Writable key, Writable val)
       throws IOException {
       append((Object) key, (Object) val);
     }
 
-    /** Append a key/value pair. */
+    /**
+     * Append a key/value pair.
+     * @param key input Object key.
+     * @param val input Object val.
+     * @throws IOException raised on errors performing I/O.
+     */
     @SuppressWarnings("unchecked")
     public synchronized void append(Object key, Object val)
       throws IOException {
@@ -1465,14 +1516,16 @@ public synchronized void appendRaw(byte[] keyData, int keyOffset,
       val.writeUncompressedBytes(out);            // value
     }
 
-    /** Returns the current length of the output file.
+    /** @return Returns the current length of the output file.
      *
      * <p>This always returns a synchronized position.  In other words,
      * immediately after calling {@link SequenceFile.Reader#seek(long)} with a position
      * returned by this method, {@link SequenceFile.Reader#next(Writable)} may be called.  However
      * the key may be earlier in the file than key last written when this
      * method was called (e.g., with block-compression, it may be the first key
-     * in the block that was being written when this method was called).
+     * in the block that was being written when this method was called).</p>
+     *
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized long getLength() throws IOException {
       return out.getPos();
@@ -1883,7 +1936,7 @@ public Reader(Configuration conf, Option... opts) throws IOException {
      * @param fs The file system used to open the file.
      * @param file The file being read.
      * @param conf Configuration
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * @deprecated Use Reader(Configuration, Option...) instead.
      */
     @Deprecated
@@ -1899,7 +1952,7 @@ public Reader(FileSystem fs, Path file,
      * @param start The starting position.
      * @param length The length being read.
      * @param conf Configuration
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * @deprecated Use Reader(Configuration, Reader.Option...) instead.
      */
     @Deprecated
@@ -1944,11 +1997,18 @@ private void initialize(Path filename, FSDataInputStream in,
      * @param length The length being read if it is {@literal >=} 0.
      *               Otherwise, the length is not available.
      * @return The opened stream.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     protected FSDataInputStream openFile(FileSystem fs, Path file,
         int bufferSize, long length) throws IOException {
-      return fs.open(file, bufferSize);
+      FutureDataInputStreamBuilder builder = fs.openFile(file)
+          .opt(FS_OPTION_OPENFILE_READ_POLICY,
+              FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+          .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, bufferSize);
+      if (length >= 0) {
+        builder.opt(FS_OPTION_OPENFILE_LENGTH, length);
+      }
+      return awaitFuture(builder.build());
     }
     
     /**
@@ -2127,12 +2187,12 @@ public synchronized void close() throws IOException {
       in.close();
     }
 
-    /** Returns the name of the key class. */
+    /** @return Returns the name of the key class. */
     public String getKeyClassName() {
       return keyClassName;
     }
 
-    /** Returns the class of keys in this file. */
+    /** @return Returns the class of keys in this file. */
     public synchronized Class<?> getKeyClass() {
       if (null == keyClass) {
         try {
@@ -2144,12 +2204,12 @@ public synchronized Class<?> getKeyClass() {
       return keyClass;
     }
 
-    /** Returns the name of the value class. */
+    /** @return Returns the name of the value class. */
     public String getValueClassName() {
       return valClassName;
     }
 
-    /** Returns the class of values in this file. */
+    /** @return Returns the class of values in this file. */
     public synchronized Class<?> getValueClass() {
       if (null == valClass) {
         try {
@@ -2161,13 +2221,22 @@ public synchronized Class<?> getValueClass() {
       return valClass;
     }
 
-    /** Returns true if values are compressed. */
+    /**
+     * Returns true if values are compressed.
+     * @return if values are compressed true, not false.
+     */
     public boolean isCompressed() { return decompress; }
     
-    /** Returns true if records are block-compressed. */
+    /**
+     * Returns true if records are block-compressed.
+     * @return if records are block-compressed true, not false.
+     */
     public boolean isBlockCompressed() { return blockCompressed; }
     
-    /** Returns the compression codec of data in this file. */
+    /**
+     * Returns the compression codec of data in this file.
+     * @return CompressionCodec.
+     */
     public CompressionCodec getCompressionCodec() { return codec; }
     
     private byte[] getSync() {
@@ -2190,7 +2259,10 @@ public CompressionType getCompressionType() {
       }
     }
 
-    /** Returns the metadata object of the file */
+    /**
+     * Returns the metadata object of the file.
+     * @return metadata.
+     */
     public Metadata getMetadata() {
       return this.metadata;
     }
@@ -2299,7 +2371,7 @@ private synchronized void seekToCurrentValue() throws IOException {
     /**
      * Get the 'value' corresponding to the last read 'key'.
      * @param val : The 'value' to be read.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized void getCurrentValue(Writable val) 
       throws IOException {
@@ -2336,9 +2408,9 @@ public synchronized void getCurrentValue(Writable val)
     }
     
     /**
-     * Get the 'value' corresponding to the last read 'key'.
+     * @return Get the 'value' corresponding to the last read 'key'.
      * @param val : The 'value' to be read.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized Object getCurrentValue(Object val) 
       throws IOException {
@@ -2380,8 +2452,13 @@ private Object deserializeValue(Object val) throws IOException {
       return valDeserializer.deserialize(val);
     }
     
-    /** Read the next key in the file into <code>key</code>, skipping its
-     * value.  True if another entry exists, and false at end of file. */
+    /**
+     * @return Read the next key in the file into <code>key</code>, skipping its
+     * value.True if another entry exists, and false at end of file.
+     *
+     * @param key key.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized boolean next(Writable key) throws IOException {
       if (key.getClass() != getKeyClass())
         throw new IOException("wrong key class: "+key.getClass().getName()
@@ -2428,9 +2505,16 @@ public synchronized boolean next(Writable key) throws IOException {
       return true;
     }
 
-    /** Read the next key/value pair in the file into <code>key</code> and
-     * <code>val</code>.  Returns true if such a pair exists and false when at
-     * end of file */
+    /**
+     * Read the next key/value pair in the file into <code>key</code> and
+     * <code>val</code>.
+     * @return Returns true if such a pair exists and false when at
+     * end of file.
+     *
+     * @param key input key.
+     * @param val input val.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized boolean next(Writable key, Writable val)
       throws IOException {
       if (val.getClass() != getValueClass())
@@ -2514,7 +2598,7 @@ public ValueBytes createValueBytes() {
      * @param key - The buffer into which the key is read
      * @param val - The 'raw' value
      * @return Returns the total record length or -1 for end of file
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized int nextRaw(DataOutputBuffer key, ValueBytes val) 
       throws IOException {
@@ -2573,7 +2657,7 @@ public synchronized int nextRaw(DataOutputBuffer key, ValueBytes val)
      * Read 'raw' keys.
      * @param key - The buffer into which the key is read
      * @return Returns the key length or -1 for end of file
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized int nextRawKey(DataOutputBuffer key) 
       throws IOException {
@@ -2612,8 +2696,14 @@ public synchronized int nextRawKey(DataOutputBuffer key)
       
     }
 
-    /** Read the next key in the file, skipping its
-     * value.  Return null at end of file. */
+    /**
+     * Read the next key in the file, skipping its
+     * value.
+     *
+     * @param key input Object key.
+     * @throws IOException raised on errors performing I/O.
+     * @return Return null at end of file.
+     */
     public synchronized Object next(Object key) throws IOException {
       if (key != null && key.getClass() != getKeyClass()) {
         throw new IOException("wrong key class: "+key.getClass().getName()
@@ -2670,7 +2760,7 @@ private Object deserializeKey(Object key) throws IOException {
      * Read 'raw' values.
      * @param val - The 'raw' value
      * @return Returns the value length
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized int nextRawValue(ValueBytes val) 
       throws IOException {
@@ -2710,16 +2800,20 @@ private void handleChecksumException(ChecksumException e)
       }
     }
 
-    /** disables sync. often invoked for tmp files */
+    /** disables sync. often invoked for tmp files. */
     synchronized void ignoreSync() {
       sync = null;
     }
     
-    /** Set the current byte position in the input file.
+    /**
+     * Set the current byte position in the input file.
      *
      * <p>The position passed must be a position returned by {@link
      * SequenceFile.Writer#getLength()} when writing this file.  To seek to an arbitrary
-     * position, use {@link SequenceFile.Reader#sync(long)}.
+     * position, use {@link SequenceFile.Reader#sync(long)}. </p>
+     *
+     * @param position input position.
+     * @throws IOException raised on errors performing I/O.
      */
     public synchronized void seek(long position) throws IOException {
       in.seek(position);
@@ -2729,7 +2823,11 @@ public synchronized void seek(long position) throws IOException {
       }
     }
 
-    /** Seek to the next sync mark past a given position.*/
+    /**
+     * Seek to the next sync mark past a given position.
+     * @param position position.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized void sync(long position) throws IOException {
       if (position+SYNC_SIZE >= end) {
         seek(end);
@@ -2765,10 +2863,13 @@ public synchronized void sync(long position) throws IOException {
       }
     }
 
-    /** Returns true iff the previous call to next passed a sync mark.*/
+    /** @return Returns true iff the previous call to next passed a sync mark.*/
     public synchronized boolean syncSeen() { return syncSeen; }
 
-    /** Return the current byte position in the input file. */
+    /**
+     * @return Return the current byte position in the input file.
+     * @throws IOException raised on errors performing I/O.
+     */
     public synchronized long getPosition() throws IOException {
       return in.getPos();
     }
@@ -2810,19 +2911,40 @@ public static class Sorter {
     
     private Progressable progressable = null;
 
-    /** Sort and merge files containing the named classes. */
+    /**
+     * Sort and merge files containing the named classes.
+     * @param fs input FileSystem.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @param conf input Configuration.
+     */
     public Sorter(FileSystem fs, Class<? extends WritableComparable> keyClass,
                   Class valClass, Configuration conf)  {
       this(fs, WritableComparator.get(keyClass, conf), keyClass, valClass, conf);
     }
 
-    /** Sort and merge using an arbitrary {@link RawComparator}. */
+    /**
+     * Sort and merge using an arbitrary {@link RawComparator}.
+     * @param fs input FileSystem.
+     * @param comparator input RawComparator.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @param conf input Configuration.
+     */
     public Sorter(FileSystem fs, RawComparator comparator, Class keyClass, 
                   Class valClass, Configuration conf) {
       this(fs, comparator, keyClass, valClass, conf, new Metadata());
     }
 
-    /** Sort and merge using an arbitrary {@link RawComparator}. */
+    /**
+     * Sort and merge using an arbitrary {@link RawComparator}.
+     * @param fs input FileSystem.
+     * @param comparator input RawComparator.
+     * @param keyClass input keyClass.
+     * @param valClass input valClass.
+     * @param conf input Configuration.
+     * @param metadata input metadata.
+     */
     @SuppressWarnings("deprecation")
     public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
                   Class valClass, Configuration conf, Metadata metadata) {
@@ -2851,19 +2973,28 @@ public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
       this.metadata = metadata;
     }
 
-    /** Set the number of streams to merge at once.*/
+    /**
+     * Set the number of streams to merge at once.
+     * @param factor factor.
+     */
     public void setFactor(int factor) { this.factor = factor; }
 
-    /** Get the number of streams to merge at once.*/
+    /** @return Get the number of streams to merge at once.*/
     public int getFactor() { return factor; }
 
-    /** Set the total amount of buffer memory, in bytes.*/
+    /**
+     * Set the total amount of buffer memory, in bytes.
+     * @param memory buffer memory.
+     */
     public void setMemory(int memory) { this.memory = memory; }
 
-    /** Get the total amount of buffer memory, in bytes.*/
+    /** @return Get the total amount of buffer memory, in bytes.*/
     public int getMemory() { return memory; }
 
-    /** Set the progressable object in order to report progress. */
+    /**
+     * Set the progressable object in order to report progress.
+     * @param progressable input Progressable.
+     */
     public void setProgressable(Progressable progressable) {
       this.progressable = progressable;
     }
@@ -2873,6 +3004,7 @@ public void setProgressable(Progressable progressable) {
      * @param inFiles the files to be sorted
      * @param outFile the sorted output file
      * @param deleteInput should the input files be deleted as they are read?
+     * @throws IOException raised on errors performing I/O.
      */
     public void sort(Path[] inFiles, Path outFile,
                      boolean deleteInput) throws IOException {
@@ -2895,6 +3027,7 @@ public void sort(Path[] inFiles, Path outFile,
      * @param tempDir the directory where temp files are created during sort
      * @param deleteInput should the input files be deleted as they are read?
      * @return iterator the RawKeyValueIterator
+     * @throws IOException raised on errors performing I/O.
      */
     public RawKeyValueIterator sortAndIterate(Path[] inFiles, Path tempDir, 
                                               boolean deleteInput) throws IOException {
@@ -2920,8 +3053,9 @@ else if (segments == 1)
 
     /**
      * The backwards compatible interface to sort.
-     * @param inFile the input file to sort
-     * @param outFile the sorted output file
+     * @param inFile the input file to sort.
+     * @param outFile the sorted output file.
+     * @throws IOException raised on errors performing I/O.
      */
     public void sort(Path inFile, Path outFile) throws IOException {
       sort(new Path[]{inFile}, outFile, false);
@@ -3139,27 +3273,32 @@ public void setProgressable(Progressable progressable)
 
     /** The interface to iterate over raw keys/values of SequenceFiles. */
     public static interface RawKeyValueIterator {
-      /** Gets the current raw key
+      /**
+       * Gets the current raw key.
        * @return DataOutputBuffer
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       DataOutputBuffer getKey() throws IOException; 
-      /** Gets the current raw value
+      /**
+       * Gets the current raw value.
        * @return ValueBytes 
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       ValueBytes getValue() throws IOException; 
-      /** Sets up the current key and value (for getKey and getValue)
+      /**
+       * Sets up the current key and value (for getKey and getValue).
        * @return true if there exists a key/value, false otherwise 
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       boolean next() throws IOException;
-      /** closes the iterator so that the underlying streams can be closed
-       * @throws IOException
+      /**
+       * closes the iterator so that the underlying streams can be closed.
+       * @throws IOException raised on errors performing I/O.
        */
       void close() throws IOException;
-      /** Gets the Progress object; this has a float (0.0 - 1.0) 
-       * indicating the bytes processed by the iterator so far
+      /**
+       * @return Gets the Progress object; this has a float (0.0 - 1.0)
+       * indicating the bytes processed by the iterator so far.
        */
       Progress getProgress();
     }    
@@ -3169,7 +3308,7 @@ public static interface RawKeyValueIterator {
      * @param segments the list of SegmentDescriptors
      * @param tmpDir the directory to write temporary files into
      * @return RawKeyValueIterator
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawKeyValueIterator merge(List <SegmentDescriptor> segments, 
                                      Path tmpDir) 
@@ -3187,7 +3326,7 @@ public RawKeyValueIterator merge(List <SegmentDescriptor> segments,
      * unnecessary
      * @param tmpDir the directory to write temporary files into
      * @return RawKeyValueIteratorMergeQueue
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs,
                                      Path tmpDir) 
@@ -3205,7 +3344,7 @@ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs,
      * @param factor the factor that will be used as the maximum merge fan-in
      * @param tmpDir the directory to write temporary files into
      * @return RawKeyValueIteratorMergeQueue
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs,
                                      int factor, Path tmpDir) 
@@ -3231,7 +3370,7 @@ public RawKeyValueIterator merge(Path [] inNames, boolean deleteInputs,
      * @param deleteInputs true if the input files should be deleted when 
      * unnecessary
      * @return RawKeyValueIteratorMergeQueue
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawKeyValueIterator merge(Path [] inNames, Path tempDir, 
                                      boolean deleteInputs) 
@@ -3262,7 +3401,7 @@ public RawKeyValueIterator merge(Path [] inNames, Path tempDir,
      * @param outputFile the path of the output file 
      * @param prog the Progressable to report status during the file write
      * @return Writer
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Writer cloneFileAttributes(Path inputFile, Path outputFile, 
                                       Progressable prog) throws IOException {
@@ -3284,10 +3423,10 @@ public Writer cloneFileAttributes(Path inputFile, Path outputFile,
 
     /**
      * Writes records from RawKeyValueIterator into a file represented by the 
-     * passed writer
+     * passed writer.
      * @param records the RawKeyValueIterator
      * @param writer the Writer created earlier 
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void writeFile(RawKeyValueIterator records, Writer writer) 
       throws IOException {
@@ -3301,7 +3440,7 @@ public void writeFile(RawKeyValueIterator records, Writer writer)
     /** Merge the provided files.
      * @param inFiles the array of input path names
      * @param outFile the final output file
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void merge(Path[] inFiles, Path outFile) throws IOException {
       if (fs.exists(outFile)) {
@@ -3637,10 +3776,13 @@ public SegmentDescriptor (long segmentOffset, long segmentLength,
         this.segmentPathName = segmentPathName;
       }
       
-      /** Do the sync checks */
+      /** Do the sync checks. */
       public void doSync() {ignoreSync = false;}
       
-      /** Whether to delete the files when no longer needed */
+      /**
+       * Whether to delete the files when no longer needed.
+       * @param preserve input boolean preserve.
+       */
       public void preserveInput(boolean preserve) {
         preserveInput = preserve;
       }
@@ -3682,9 +3824,10 @@ public int hashCode() {
         return 37 * 17 + (int) (segmentOffset^(segmentOffset>>>32));
       }
 
-      /** Fills up the rawKey object with the key returned by the Reader
+      /**
+       * Fills up the rawKey object with the key returned by the Reader.
        * @return true if there is a key returned; false, otherwise
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public boolean nextRawKey() throws IOException {
         if (in == null) {
@@ -3713,18 +3856,19 @@ public boolean nextRawKey() throws IOException {
         return (keyLength >= 0);
       }
 
-      /** Fills up the passed rawValue with the value corresponding to the key
-       * read earlier
-       * @param rawValue
+      /**
+       * Fills up the passed rawValue with the value corresponding to the key
+       * read earlier.
+       * @param rawValue input ValueBytes rawValue.
        * @return the length of the value
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public int nextRawValue(ValueBytes rawValue) throws IOException {
         int valLength = in.nextRawValue(rawValue);
         return valLength;
       }
       
-      /** Returns the stored rawKey */
+      /** @return Returns the stored rawKey */
       public DataOutputBuffer getKey() {
         return rawKey;
       }
@@ -3735,8 +3879,10 @@ private void close() throws IOException {
         this.in = null;
       }
 
-      /** The default cleanup. Subclasses can override this with a custom 
-       * cleanup 
+      /**
+       * The default cleanup. Subclasses can override this with a custom
+       * cleanup.
+       * @throws IOException raised on errors performing I/O.
        */
       public void cleanup() throws IOException {
         close();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
index 118cce75136ed..de75810df0f70 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SetFile.java
@@ -39,15 +39,29 @@ protected SetFile() {}                            // no public ctor
    */
   public static class Writer extends MapFile.Writer {
 
-    /** Create the named set for keys of the named class. 
-     *  @deprecated pass a Configuration too
+    /**
+     * Create the named set for keys of the named class.
+     * @deprecated pass a Configuration too
+     * @param fs input FileSystem.
+     * @param dirName input dirName.
+     * @param keyClass input keyClass.
+     * @throws IOException raised on errors performing I/O.
      */
     public Writer(FileSystem fs, String dirName,
 	Class<? extends WritableComparable> keyClass) throws IOException {
       super(new Configuration(), fs, dirName, keyClass, NullWritable.class);
     }
 
-    /** Create a set naming the element class and compression type. */
+    /**
+     * Create a set naming the element class and compression type.
+     *
+     * @param conf input Configuration.
+     * @param fs input FileSystem.
+     * @param dirName input dirName.
+     * @param keyClass input keyClass.
+     * @param compress input compress.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Writer(Configuration conf, FileSystem fs, String dirName,
                   Class<? extends WritableComparable> keyClass,
                   SequenceFile.CompressionType compress)
@@ -55,7 +69,16 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
       this(conf, fs, dirName, WritableComparator.get(keyClass, conf), compress);
     }
 
-    /** Create a set naming the element comparator and compression type. */
+    /**
+     * Create a set naming the element comparator and compression type.
+     *
+     * @param conf input Configuration.
+     * @param fs input FileSystem.
+     * @param dirName input dirName.
+     * @param comparator input comparator.
+     * @param compress input compress.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Writer(Configuration conf, FileSystem fs, String dirName,
                   WritableComparator comparator,
                   SequenceFile.CompressionType compress) throws IOException {
@@ -65,8 +88,12 @@ public Writer(Configuration conf, FileSystem fs, String dirName,
             compression(compress));
     }
 
-    /** Append a key to a set.  The key must be strictly greater than the
-     * previous key added to the set. */
+    /**
+     * Append a key to a set.  The key must be strictly greater than the
+     * previous key added to the set.
+     * @param key input key.
+     * @throws IOException raised on errors performing I/O.
+     */
     public void append(WritableComparable key) throws IOException{
       append(key, NullWritable.get());
     }
@@ -75,12 +102,25 @@ public void append(WritableComparable key) throws IOException{
   /** Provide access to an existing set file. */
   public static class Reader extends MapFile.Reader {
 
-    /** Construct a set reader for the named set.*/
+    /**
+     * Construct a set reader for the named set.
+     * @param fs input FileSystem.
+     * @param dirName input dirName.
+     * @param conf input Configuration.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Reader(FileSystem fs, String dirName, Configuration conf) throws IOException {
       super(fs, dirName, conf);
     }
 
-    /** Construct a set reader for the named set using the named comparator.*/
+    /**
+     * Construct a set reader for the named set using the named comparator.
+     * @param fs input FileSystem.
+     * @param dirName input dirName.
+     * @param comparator input comparator.
+     * @param conf input Configuration.
+     * @throws IOException raised on errors performing I/O.
+     */
     public Reader(FileSystem fs, String dirName, WritableComparator comparator, Configuration conf)
       throws IOException {
       super(new Path(dirName), conf, comparator(comparator));
@@ -93,15 +133,26 @@ public boolean seek(WritableComparable key)
       return super.seek(key);
     }
 
-    /** Read the next key in a set into <code>key</code>.  Returns
-     * true if such a key exists and false when at the end of the set. */
+    /**
+     * Read the next key in a set into <code>key</code>.
+     *
+     * @param key input key.
+     * @return Returns true if such a key exists
+     *    and false when at the end of the set.
+     * @throws IOException raised on errors performing I/O.
+     */
     public boolean next(WritableComparable key)
       throws IOException {
       return next(key, NullWritable.get());
     }
 
-    /** Read the matching key from a set into <code>key</code>.
-     * Returns <code>key</code>, or null if no match exists. */
+    /**
+     * Read the matching key from a set into <code>key</code>.
+     *
+     * @param key input key.
+     * @return Returns <code>key</code>, or null if no match exists.
+     * @throws IOException raised on errors performing I/O.
+     */
     public WritableComparable get(WritableComparable key)
       throws IOException {
       if (seek(key)) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ShortWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ShortWritable.java
index be09df18017b7..96e6cacae8773 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ShortWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ShortWritable.java
@@ -38,12 +38,15 @@ public ShortWritable(short value) {
     set(value);
   }
 
-  /** Set the value of this ShortWritable. */
+  /**
+   * Set the value of this ShortWritable.
+   * @param value input value.
+   */
   public void set(short value) {
     this.value = value;
   }
 
-  /** Return the value of this ShortWritable. */
+  /** @return Return the value of this ShortWritable. */
   public short get() {
     return value;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index 5ca7f3c84cab0..86fb1ff9a54a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -90,6 +90,7 @@ public Text() {
 
   /**
    * Construct from a string.
+   * @param string input string.
    */
   public Text(String string) {
     set(string);
@@ -97,6 +98,7 @@ public Text(String string) {
 
   /**
    * Construct from another text.
+   * @param utf8 input utf8.
    */
   public Text(Text utf8) {
     set(utf8);
@@ -104,13 +106,15 @@ public Text(Text utf8) {
 
   /**
    * Construct from a byte array.
+   *
+   * @param utf8 input utf8.
    */
   public Text(byte[] utf8)  {
     set(utf8);
   }
 
   /**
-   * Get a copy of the bytes that is exactly the length of the data.
+   * @return Get a copy of the bytes that is exactly the length of the data.
    * See {@link #getBytes()} for faster access to the underlying array.
    */
   public byte[] copyBytes() {
@@ -136,7 +140,7 @@ public int getLength() {
   }
 
   /**
-   * Returns the length of this text. The length is equal to the number of
+   * @return Returns the length of this text. The length is equal to the number of
    * Unicode code units in the text.
    */
   public int getTextLength() {
@@ -149,7 +153,9 @@ public int getTextLength() {
   /**
    * Returns the Unicode Scalar Value (32-bit integer value)
    * for the character at <code>position</code>. Note that this
-   * method avoids using the converter or doing String instantiation
+   * method avoids using the converter or doing String instantiation.
+   *
+   * @param position input position.
    * @return the Unicode scalar value at position or -1
    *          if the position is invalid or points to a
    *          trailing byte
@@ -172,6 +178,9 @@ public int find(String what) {
    * position is measured in bytes and the return value is in
    * terms of byte position in the buffer. The backing buffer is
    * not converted to a string for this operation.
+   *
+   * @param what input what.
+   * @param start input start.
    * @return byte position of the first occurrence of the search
    *         string in the UTF-8 buffer or -1 if not found
    */
@@ -213,6 +222,8 @@ public int find(String what, int start) {
 
   /**
    * Set to contain the contents of a string.
+   *
+   * @param string input string.
    */
   public void set(String string) {
     try {
@@ -229,6 +240,8 @@ public void set(String string) {
    * Set to a utf8 byte array. If the length of <code>utf8</code> is
    * <em>zero</em>, actually clear {@link #bytes} and any existing
    * data is lost.
+   *
+   * @param utf8 input utf8.
    */
   public void set(byte[] utf8) {
     if (utf8.length == 0) {
@@ -242,6 +255,7 @@ public void set(byte[] utf8) {
 
   /**
    * Copy a text.
+   * @param other other.
    */
   public void set(Text other) {
     set(other.getBytes(), 0, other.getLength());
@@ -349,6 +363,8 @@ public void readFields(DataInput in, int maxLength) throws IOException {
 
   /**
    * Skips over one Text in the input.
+   * @param in input in.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void skip(DataInput in) throws IOException {
     int length = WritableUtils.readVInt(in);
@@ -359,6 +375,10 @@ public static void skip(DataInput in) throws IOException {
    * Read a Text object whose length is already known.
    * This allows creating Text from a stream which uses a different serialization
    * format.
+   *
+   * @param in input in.
+   * @param len input len.
+   * @throws IOException raised on errors performing I/O.
    */
   public void readWithKnownLength(DataInput in, int len) throws IOException {
     ensureCapacity(len);
@@ -426,9 +446,13 @@ public int compare(byte[] b1, int s1, int l1,
 
   /// STATIC UTILITIES FROM HERE DOWN
   /**
-   * Converts the provided byte array to a String using the
+   * @return Converts the provided byte array to a String using the
    * UTF-8 encoding. If the input is malformed,
    * replace by a default value.
+   *
+   * @param utf8 input utf8.
+   * @throws CharacterCodingException when a character
+   *                                  encoding or decoding error occurs.
    */
   public static String decode(byte[] utf8) throws CharacterCodingException {
     return decode(ByteBuffer.wrap(utf8), true);
@@ -440,11 +464,18 @@ public static String decode(byte[] utf8, int start, int length)
   }
 
   /**
-   * Converts the provided byte array to a String using the
+   * @return Converts the provided byte array to a String using the
    * UTF-8 encoding. If <code>replace</code> is true, then
    * malformed input is replaced with the
    * substitution character, which is U+FFFD. Otherwise the
    * method throws a MalformedInputException.
+   *
+   * @param utf8 input utf8.
+   * @param start input start.
+   * @param length input length.
+   * @param replace input replace.
+   * @throws CharacterCodingException when a character
+   *                                  encoding or decoding error occurs.
    */
   public static String decode(byte[] utf8, int start, int length, boolean replace) 
     throws CharacterCodingException {
@@ -472,8 +503,12 @@ private static String decode(ByteBuffer utf8, boolean replace)
    * Converts the provided String to bytes using the
    * UTF-8 encoding. If the input is malformed,
    * invalid chars are replaced by a default value.
+   *
+   * @param string input string.
    * @return ByteBuffer: bytes stores at ByteBuffer.array() 
    *                     and length is ByteBuffer.limit()
+   * @throws CharacterCodingException when a character
+   *                                  encoding or decoding error occurs.
    */
 
   public static ByteBuffer encode(String string)
@@ -487,8 +522,13 @@ public static ByteBuffer encode(String string)
    * malformed input is replaced with the
    * substitution character, which is U+FFFD. Otherwise the
    * method throws a MalformedInputException.
+   *
+   * @param string input string.
+   * @param replace input replace.
    * @return ByteBuffer: bytes stores at ByteBuffer.array() 
    *                     and length is ByteBuffer.limit()
+   * @throws CharacterCodingException when a character
+   *                                  encoding or decoding error occurs.
    */
   public static ByteBuffer encode(String string, boolean replace)
     throws CharacterCodingException {
@@ -508,13 +548,20 @@ public static ByteBuffer encode(String string, boolean replace)
 
   static final public int DEFAULT_MAX_LEN = 1024 * 1024;
 
-  /** Read a UTF8 encoded string from in
+  /**
+   * @return Read a UTF8 encoded string from in.
+   * @param in input in.
+   * @throws IOException raised on errors performing I/O.
    */
   public static String readString(DataInput in) throws IOException {
     return readString(in, Integer.MAX_VALUE);
   }
 
-  /** Read a UTF8 encoded string with a maximum size
+  /**
+   * @return Read a UTF8 encoded string with a maximum size.
+   * @param in input datainput.
+   * @param maxLength input maxLength.
+   * @throws IOException raised on errors performing I/O.
    */
   public static String readString(DataInput in, int maxLength)
       throws IOException {
@@ -526,6 +573,11 @@ public static String readString(DataInput in, int maxLength)
 
   /**
    * Write a UTF8 encoded string to out.
+   *
+   * @param out input out.
+   * @param s input s.
+   * @throws IOException raised on errors performing I/O.
+   * @return a UTF8 encoded string to out.
    */
   public static int writeString(DataOutput out, String s) throws IOException {
     ByteBuffer bytes = encode(s);
@@ -536,7 +588,12 @@ public static int writeString(DataOutput out, String s) throws IOException {
   }
 
   /**
-   * Write a UTF8 encoded string with a maximum size to out.
+   * @return Write a UTF8 encoded string with a maximum size to out.
+   *
+   * @param out input out.
+   * @param s input s.
+   * @param maxLength input maxLength.
+   * @throws IOException raised on errors performing I/O.
    */
   public static int writeString(DataOutput out, String s, int maxLength)
       throws IOException {
@@ -670,9 +727,11 @@ public static void validateUTF8(byte[] utf8, int start, int len)
     3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5 };
 
   /**
-   * Returns the next code point at the current position in
+   * @return Returns the next code point at the current position in
    * the buffer. The buffer's position will be incremented.
    * Any mark set on this buffer will be changed by this method!
+   *
+   * @param bytes input bytes.
    */
   public static int bytesToCodePoint(ByteBuffer bytes) {
     bytes.mark();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
index f5d33a13005d7..fdee830e6fea8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/UTF8.java
@@ -63,27 +63,36 @@ public UTF8() {
     //set("");
   }
 
-  /** Construct from a given string. */
+  /**
+   * Construct from a given string.
+   * @param string input string.
+   */
   public UTF8(String string) {
     set(string);
   }
 
-  /** Construct from a given string. */
+  /**
+   * Construct from a given string.
+   * @param utf8 input utf8.
+   */
   public UTF8(UTF8 utf8) {
     set(utf8);
   }
 
-  /** The raw bytes. */
+  /** @return The raw bytes. */
   public byte[] getBytes() {
     return bytes;
   }
 
-  /** The number of bytes in the encoded string. */
+  /** @return The number of bytes in the encoded string. */
   public int getLength() {
     return length;
   }
 
-  /** Set to contain the contents of a string. */
+  /**
+   * Set to contain the contents of a string.
+   * @param string input string.
+   */
   public void set(String string) {
     if (string.length() > 0xffff/3) {             // maybe too long
       LOG.warn("truncating long string: " + string.length()
@@ -108,7 +117,10 @@ public void set(String string) {
     }
   }
 
-  /** Set to contain the contents of a string. */
+  /**
+   * Set to contain the contents of a string.
+   * @param other input other.
+   */
   public void set(UTF8 other) {
     length = other.length;
     if (bytes == null || length > bytes.length)   // grow buffer
@@ -124,7 +136,11 @@ public void readFields(DataInput in) throws IOException {
     in.readFully(bytes, 0, length);
   }
 
-  /** Skips over one UTF8 in the input. */
+  /**
+   * Skips over one UTF8 in the input.
+   * @param in datainput.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static void skip(DataInput in) throws IOException {
     int length = in.readUnsignedShort();
     WritableUtils.skipFully(in, length);
@@ -214,8 +230,10 @@ public int compare(byte[] b1, int s1, int l1,
 
   /// These are probably not used much anymore, and might be removed...
 
-  /** Convert a string to a UTF-8 encoded byte array.
+  /**
+   * @return Convert a string to a UTF-8 encoded byte array.
    * @see String#getBytes(String)
+   * @param string input string.
    */
   public static byte[] getBytes(String string) {
     byte[] result = new byte[utf8Length(string)];
@@ -231,8 +249,9 @@ public static byte[] getBytes(String string) {
   }
 
   /**
-   * Convert a UTF-8 encoded byte array back into a string.
+   * @return Convert a UTF-8 encoded byte array back into a string.
    *
+   * @param bytes input bytes.
    * @throws IOException if the byte array is invalid UTF8
    */
   public static String fromBytes(byte[] bytes) throws IOException {
@@ -243,9 +262,12 @@ public static String fromBytes(byte[] bytes) throws IOException {
     return buf.toString();
   }
 
-  /** Read a UTF-8 encoded string.
+  /**
+   * @return Read a UTF-8 encoded string.
    *
    * @see DataInput#readUTF()
+   * @param in DataInput.
+   * @throws IOException raised on errors performing I/O.
    */
   public static String readString(DataInput in) throws IOException {
     int bytes = in.readUnsignedShort();
@@ -318,9 +340,13 @@ private static char lowSurrogate(int codePoint) {
     return (char) ((codePoint & 0x3ff) + Character.MIN_LOW_SURROGATE);
   }
 
-  /** Write a UTF-8 encoded string.
+  /**
+   * @return Write a UTF-8 encoded string.
    *
    * @see DataOutput#writeUTF(String)
+   * @param out input out.
+   * @param s input s.
+   * @throws IOException raised on errors performing I/O.
    */
   public static int writeString(DataOutput out, String s) throws IOException {
     if (s.length() > 0xffff/3) {         // maybe too long
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VIntWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VIntWritable.java
index f537524c4b40a..7d3f680858ec3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VIntWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VIntWritable.java
@@ -37,10 +37,13 @@ public VIntWritable() {}
 
   public VIntWritable(int value) { set(value); }
 
-  /** Set the value of this VIntWritable. */
+  /**
+   * Set the value of this VIntWritable.
+   * @param value input value.
+   */
   public void set(int value) { this.value = value; }
 
-  /** Return the value of this VIntWritable. */
+  /** @return Return the value of this VIntWritable. */
   public int get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VLongWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VLongWritable.java
index a9fac30605be6..a72a7fc2fd39a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VLongWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VLongWritable.java
@@ -37,10 +37,13 @@ public VLongWritable() {}
 
   public VLongWritable(long value) { set(value); }
 
-  /** Set the value of this LongWritable. */
+  /**
+   * Set the value of this LongWritable.
+   * @param value input value.
+   */
   public void set(long value) { this.value = value; }
 
-  /** Return the value of this LongWritable. */
+  /** @return Return the value of this LongWritable. */
   public long get() { return value; }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionedWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionedWritable.java
index c2db55520c918..421b8daeeae8e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionedWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/VersionedWritable.java
@@ -36,7 +36,7 @@
 @InterfaceStability.Stable
 public abstract class VersionedWritable implements Writable {
 
-  /** Return the version number of the current implementation. */
+  /** @return Return the version number of the current implementation. */
   public abstract byte getVersion();
     
   // javadoc from Writable
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WeakReferencedElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WeakReferencedElasticByteBufferPool.java
new file mode 100644
index 0000000000000..c71c44e798a65
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WeakReferencedElasticByteBufferPool.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+
+/**
+ * Buffer pool implementation which uses weak references to store
+ * buffers in the pool, such that they are garbage collected when
+ * there are no references to the buffer during a gc run. This is
+ * important as direct buffers don't get garbage collected automatically
+ * during a gc run as they are not stored on heap memory.
+ * Also the buffers are stored in a tree map which helps in returning
+ * smallest buffer whose size is just greater than requested length.
+ * This is a thread safe implementation.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class WeakReferencedElasticByteBufferPool extends ElasticByteBufferPool {
+
+  /**
+   * Map to store direct byte buffers of different sizes in the pool.
+   * Used tree map such that we can return next greater than capacity
+   * buffer if buffer with exact capacity is unavailable.
+   * This must be accessed in synchronized blocks.
+   */
+  private final TreeMap<Key, WeakReference<ByteBuffer>> directBuffers =
+          new TreeMap<>();
+
+  /**
+   * Map to store heap based byte buffers of different sizes in the pool.
+   * Used tree map such that we can return next greater than capacity
+   * buffer if buffer with exact capacity is unavailable.
+   * This must be accessed in synchronized blocks.
+   */
+  private final TreeMap<Key, WeakReference<ByteBuffer>> heapBuffers =
+          new TreeMap<>();
+
+  /**
+   * Method to get desired buffer tree.
+   * @param isDirect whether the buffer is heap based or direct.
+   * @return corresponding buffer tree.
+   */
+  private TreeMap<Key, WeakReference<ByteBuffer>> getBufferTree(boolean isDirect) {
+    return isDirect
+            ? directBuffers
+            : heapBuffers;
+  }
+
+  /**
+   * {@inheritDoc}
+   *
+   * @param direct whether we want a direct byte buffer or a heap one.
+   * @param length length of requested buffer.
+   * @return returns equal or next greater than capacity buffer from
+   * pool if already available and not garbage collected else creates
+   * a new buffer and return it.
+   */
+  @Override
+  public synchronized ByteBuffer getBuffer(boolean direct, int length) {
+    TreeMap<Key, WeakReference<ByteBuffer>> buffersTree = getBufferTree(direct);
+
+    // Scan the entire tree and remove all weak null references.
+    buffersTree.entrySet().removeIf(next -> next.getValue().get() == null);
+
+    Map.Entry<Key, WeakReference<ByteBuffer>> entry =
+            buffersTree.ceilingEntry(new Key(length, 0));
+    // If there is no buffer present in the pool with desired size.
+    if (entry == null) {
+      return direct ? ByteBuffer.allocateDirect(length) :
+                      ByteBuffer.allocate(length);
+    }
+    // buffer is available in the pool and not garbage collected.
+    WeakReference<ByteBuffer> bufferInPool = entry.getValue();
+    buffersTree.remove(entry.getKey());
+    ByteBuffer buffer = bufferInPool.get();
+    if (buffer != null) {
+      return buffer;
+    }
+    // buffer was in pool but already got garbage collected.
+    return direct
+            ? ByteBuffer.allocateDirect(length)
+            : ByteBuffer.allocate(length);
+  }
+
+  /**
+   * Return buffer to the pool.
+   * @param buffer buffer to be returned.
+   */
+  @Override
+  public synchronized void putBuffer(ByteBuffer buffer) {
+    buffer.clear();
+    TreeMap<Key, WeakReference<ByteBuffer>> buffersTree = getBufferTree(buffer.isDirect());
+    // Buffers are indexed by (capacity, time).
+    // If our key is not unique on the first try, we try again, since the
+    // time will be different.  Since we use nanoseconds, it's pretty
+    // unlikely that we'll loop even once, unless the system clock has a
+    // poor granularity or multi-socket systems have clocks slightly out
+    // of sync.
+    while (true) {
+      Key keyToInsert = new Key(buffer.capacity(), System.nanoTime());
+      if (!buffersTree.containsKey(keyToInsert)) {
+        buffersTree.put(keyToInsert, new WeakReference<>(buffer));
+        return;
+      }
+    }
+  }
+
+  /**
+   * Clear the buffer pool thus releasing all the buffers.
+   * The caller must remove all references of
+   * existing buffers before calling this method to avoid
+   * memory leaks.
+   */
+  @Override
+  public synchronized void release() {
+    heapBuffers.clear();
+    directBuffers.clear();
+  }
+
+  /**
+   * Get current buffers count in the pool.
+   * @param isDirect whether we want to count the heap or direct buffers.
+   * @return count of buffers.
+   */
+  @VisibleForTesting
+  public synchronized int getCurrentBuffersCount(boolean isDirect) {
+    return isDirect
+            ? directBuffers.size()
+            : heapBuffers.size();
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
index b94de6c3c72bd..56b46d554fee6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Writable.java
@@ -71,7 +71,7 @@ public interface Writable {
    * Serialize the fields of this object to <code>out</code>.
    * 
    * @param out <code>DataOuput</code> to serialize this object into.
-   * @throws IOException
+   * @throws IOException any other problem for write.
    */
   void write(DataOutput out) throws IOException;
 
@@ -82,7 +82,7 @@ public interface Writable {
    * existing object where possible.</p>
    * 
    * @param in <code>DataInput</code> to deseriablize this object from.
-   * @throws IOException
+   * @throws IOException any other problem for readFields.
    */
   void readFields(DataInput in) throws IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
index 1754b8d06f6fa..05d4e3c5c533f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
@@ -46,12 +46,22 @@ public class WritableComparator implements RawComparator, Configurable {
 
   private Configuration conf;
 
-  /** For backwards compatibility. **/
+  /**
+   * For backwards compatibility.
+   *
+   * @param c WritableComparable Type.
+   * @return WritableComparator.
+   */
   public static WritableComparator get(Class<? extends WritableComparable> c) {
     return get(c, null);
   }
 
-  /** Get a comparator for a {@link WritableComparable} implementation. */
+  /**
+   * Get a comparator for a {@link WritableComparable} implementation.
+   * @param c class.
+   * @param conf configuration.
+   * @return WritableComparator.
+   */
   public static WritableComparator get(
       Class<? extends WritableComparable> c, Configuration conf) {
     WritableComparator comparator = comparators.get(c);
@@ -95,9 +105,13 @@ private static void forceInit(Class<?> cls) {
     }
   } 
 
-  /** Register an optimized comparator for a {@link WritableComparable}
+  /**
+   * Register an optimized comparator for a {@link WritableComparable}
    * implementation. Comparators registered with this method must be
-   * thread-safe. */
+   * thread-safe.
+   * @param c class.
+   * @param comparator WritableComparator.
+   */
   public static void define(Class c, WritableComparator comparator) {
     comparators.put(c, comparator);
   }
@@ -111,7 +125,10 @@ protected WritableComparator() {
     this(null);
   }
 
-  /** Construct for a {@link WritableComparable} implementation. */
+  /**
+   * Construct for a {@link WritableComparable} implementation.
+   * @param keyClass WritableComparable Class.
+   */
   protected WritableComparator(Class<? extends WritableComparable> keyClass) {
     this(keyClass, null, false);
   }
@@ -136,10 +153,16 @@ protected WritableComparator(Class<? extends WritableComparable> keyClass,
     }
   }
 
-  /** Returns the WritableComparable implementation class. */
+  /**
+   * Returns the WritableComparable implementation class.
+   * @return WritableComparable.
+   */
   public Class<? extends WritableComparable> getKeyClass() { return keyClass; }
 
-  /** Construct a new {@link WritableComparable} instance. */
+  /**
+   * Construct a new {@link WritableComparable} instance.
+   * @return WritableComparable.
+   */
   public WritableComparable newKey() {
     return ReflectionUtils.newInstance(keyClass, conf);
   }
@@ -168,27 +191,54 @@ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
     return compare(key1, key2);                   // compare them
   }
 
-  /** Compare two WritableComparables.
+  /**
+   * Compare two WritableComparables.
    *
-   * <p> The default implementation uses the natural ordering, calling {@link
-   * Comparable#compareTo(Object)}. */
+   * The default implementation uses the natural ordering, calling {@link
+   * Comparable#compareTo(Object)}.
+   * @param a the first object to be compared.
+   * @param b the second object to be compared.
+   * @return compare result.
+   */
   @SuppressWarnings("unchecked")
   public int compare(WritableComparable a, WritableComparable b) {
     return a.compareTo(b);
   }
 
+  /**
+   * Compare two Object.
+   *
+   * @param a the first object to be compared.
+   * @param b the second object to be compared.
+   * @return compare result.
+   */
   @Override
   public int compare(Object a, Object b) {
     return compare((WritableComparable)a, (WritableComparable)b);
   }
 
-  /** Lexicographic order of binary data. */
+  /**
+   * Lexicographic order of binary data.
+   * @param b1 b1.
+   * @param s1 s1.
+   * @param l1 l1.
+   * @param b2 b2.
+   * @param s2 s2.
+   * @param l2 l2.
+   * @return compare bytes.
+   */
   public static int compareBytes(byte[] b1, int s1, int l1,
                                  byte[] b2, int s2, int l2) {
     return FastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2);
   }
 
-  /** Compute hash for binary data. */
+  /**
+   * Compute hash for binary data.
+   * @param bytes bytes.
+   * @param offset offset.
+   * @param length length.
+   * @return hash for binary data.
+   */
   public static int hashBytes(byte[] bytes, int offset, int length) {
     int hash = 1;
     for (int i = offset; i < offset + length; i++)
@@ -196,18 +246,33 @@ public static int hashBytes(byte[] bytes, int offset, int length) {
     return hash;
   }
   
-  /** Compute hash for binary data. */
+  /**
+   * Compute hash for binary data.
+   * @param bytes bytes.
+   * @param length length.
+   * @return hash for binary data.
+   */
   public static int hashBytes(byte[] bytes, int length) {
     return hashBytes(bytes, 0, length);
   }
 
-  /** Parse an unsigned short from a byte array. */
+  /**
+   * Parse an unsigned short from a byte array.
+   * @param bytes bytes.
+   * @param start start.
+   * @return unsigned short from a byte array
+   */
   public static int readUnsignedShort(byte[] bytes, int start) {
     return (((bytes[start]   & 0xff) <<  8) +
             ((bytes[start+1] & 0xff)));
   }
 
-  /** Parse an integer from a byte array. */
+  /**
+   * Parse an integer from a byte array.
+   * @param bytes bytes.
+   * @param start start.
+   * @return integer from a byte array
+   */
   public static int readInt(byte[] bytes, int start) {
     return (((bytes[start  ] & 0xff) << 24) +
             ((bytes[start+1] & 0xff) << 16) +
@@ -216,18 +281,33 @@ public static int readInt(byte[] bytes, int start) {
 
   }
 
-  /** Parse a float from a byte array. */
+  /**
+   * Parse a float from a byte array.
+   * @param bytes bytes.
+   * @param start start.
+   * @return float from a byte array
+   */
   public static float readFloat(byte[] bytes, int start) {
     return Float.intBitsToFloat(readInt(bytes, start));
   }
 
-  /** Parse a long from a byte array. */
+  /**
+   * Parse a long from a byte array.
+   * @param bytes bytes.
+   * @param start start.
+   * @return long from a byte array
+   */
   public static long readLong(byte[] bytes, int start) {
     return ((long)(readInt(bytes, start)) << 32) +
       (readInt(bytes, start+4) & 0xFFFFFFFFL);
   }
 
-  /** Parse a double from a byte array. */
+  /**
+   * Parse a double from a byte array.
+   * @param bytes bytes.
+   * @param start start.
+   * @return double from a byte array.
+   */
   public static double readDouble(byte[] bytes, int start) {
     return Double.longBitsToDouble(readLong(bytes, start));
   }
@@ -236,7 +316,7 @@ public static double readDouble(byte[] bytes, int start) {
    * Reads a zero-compressed encoded long from a byte array and returns it.
    * @param bytes byte array with decode long
    * @param start starting index
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    * @return deserialized long
    */
   public static long readVLong(byte[] bytes, int start) throws IOException {
@@ -261,7 +341,7 @@ public static long readVLong(byte[] bytes, int start) throws IOException {
    * Reads a zero-compressed encoded integer from a byte array and returns it.
    * @param bytes byte array with the encoded integer
    * @param start start index
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    * @return deserialized integer
    */
   public static int readVInt(byte[] bytes, int start) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java
index a8fdbfe98dfdc..9dd231e488780 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactories.java
@@ -35,17 +35,31 @@ public class WritableFactories {
 
   private WritableFactories() {}                  // singleton
 
-  /** Define a factory for a class. */
+  /**
+   * Define a factory for a class.
+   * @param c input c.
+   * @param factory input factory.
+   */
   public static void setFactory(Class c, WritableFactory factory) {
     CLASS_TO_FACTORY.put(c, factory);
   }
 
-  /** Define a factory for a class. */
+  /**
+   * Define a factory for a class.
+   * @param c input c.
+   * @return a factory for a class.
+   */
   public static WritableFactory getFactory(Class c) {
     return CLASS_TO_FACTORY.get(c);
   }
 
-  /** Create a new instance of a class with a defined factory. */
+  /**
+   * Create a new instance of a class with a defined factory.
+   *
+   * @param c input c.
+   * @param conf input configuration.
+   * @return a new instance of a class with a defined factory.
+   */
   public static Writable newInstance(Class<? extends Writable> c, Configuration conf) {
     WritableFactory factory = WritableFactories.getFactory(c);
     if (factory != null) {
@@ -59,7 +73,11 @@ public static Writable newInstance(Class<? extends Writable> c, Configuration co
     }
   }
   
-  /** Create a new instance of a class with a defined factory. */
+  /**
+   * Create a new instance of a class with a defined factory.
+   * @param c input c.
+   * @return a new instance of a class with a defined factory.
+   */
   public static Writable newInstance(Class<? extends Writable> c) {
     return newInstance(c, null);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactory.java
index bb8af974f051e..d9e9b543c7d96 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableFactory.java
@@ -27,7 +27,7 @@
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public interface WritableFactory {
-  /** Return a new instance. */
+  /** @return Return a new instance. */
   Writable newInstance();
 }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
index 43d396edad7ab..e5e74875225cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java
@@ -45,19 +45,33 @@ public class WritableName {
 
   private WritableName() {}                      // no public ctor
 
-  /** Set the name that a class should be known as to something other than the
-   * class name. */
+  /**
+   * Set the name that a class should be known as to something other than the
+   * class name.
+   *
+   * @param writableClass input writableClass.
+   * @param name input name.
+   */
   public static synchronized void setName(Class<?> writableClass, String name) {
     CLASS_TO_NAME.put(writableClass, name);
     NAME_TO_CLASS.put(name, writableClass);
   }
 
-  /** Add an alternate name for a class. */
+  /**
+   * Add an alternate name for a class.
+   * @param writableClass input writableClass.
+   * @param name input name.
+   */
   public static synchronized void addName(Class<?> writableClass, String name) {
     NAME_TO_CLASS.put(name, writableClass);
   }
 
-  /** Return the name for a class.  Default is {@link Class#getName()}. */
+  /**
+   * Return the name for a class.
+   * Default is {@link Class#getName()}.
+   * @param writableClass input writableClass.
+   * @return name for a class.
+   */
   public static synchronized String getName(Class<?> writableClass) {
     String name = CLASS_TO_NAME.get(writableClass);
     if (name != null)
@@ -65,7 +79,15 @@ public static synchronized String getName(Class<?> writableClass) {
     return writableClass.getName();
   }
 
-  /** Return the class for a name.  Default is {@link Class#forName(String)}.*/
+  /**
+   * Return the class for a name.
+   * Default is {@link Class#forName(String)}.
+   *
+   * @param name input name.
+   * @param conf input configuration.
+   * @return class for a name.
+   * @throws IOException raised on errors performing I/O.
+   */
   public static synchronized Class<?> getClass(String name, Configuration conf
                                             ) throws IOException {
     Class<?> writableClass = NAME_TO_CLASS.get(name);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
index 2062fb6fe3705..187398de0ec86 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableUtils.java
@@ -208,7 +208,10 @@ public static void displayByteArray(byte[] record){
 
   /**
    * Make a copy of a writable object using serialization to a buffer.
+   *
+   * @param <T> Generics Type T.
    * @param orig The object to copy
+   * @param conf input Configuration.
    * @return The copied object
    */
   public static <T extends Writable> T clone(T orig, Configuration conf) {
@@ -223,10 +226,10 @@ public static <T extends Writable> T clone(T orig, Configuration conf) {
   }
 
   /**
-   * Make a copy of the writable object using serialization to a buffer
+   * Make a copy of the writable object using serialization to a buffer.
    * @param dst the object to copy from
    * @param src the object to copy into, which is destroyed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @deprecated use ReflectionUtils.cloneInto instead.
    */
   @Deprecated
@@ -248,7 +251,7 @@ public static void cloneInto(Writable dst, Writable src) throws IOException {
    *
    * @param stream Binary output stream
    * @param i Integer to be serialized
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public static void writeVInt(DataOutput stream, int i) throws IOException {
     writeVLong(stream, i);
@@ -268,7 +271,7 @@ public static void writeVInt(DataOutput stream, int i) throws IOException {
    * 
    * @param stream Binary output stream
    * @param i Long to be serialized
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public static void writeVLong(DataOutput stream, long i) throws IOException {
     if (i >= -112 && i <= 127) {
@@ -303,7 +306,7 @@ public static void writeVLong(DataOutput stream, long i) throws IOException {
   /**
    * Reads a zero-compressed encoded long from input stream and returns it.
    * @param stream Binary input stream
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    * @return deserialized long from stream.
    */
   public static long readVLong(DataInput stream) throws IOException {
@@ -324,7 +327,7 @@ public static long readVLong(DataInput stream) throws IOException {
   /**
    * Reads a zero-compressed encoded integer from input stream and returns it.
    * @param stream Binary input stream
-   * @throws java.io.IOException 
+   * @throws IOException raised on errors performing I/O.
    * @return deserialized integer from stream.
    */
   public static int readVInt(DataInput stream) throws IOException {
@@ -342,8 +345,10 @@ public static int readVInt(DataInput stream) throws IOException {
    * inclusive.
    *
    * @param stream Binary input stream
-   * @throws java.io.IOException
-   * @return deserialized integer from stream
+   * @param lower input lower.
+   * @param upper input upper.
+   * @throws IOException raised on errors performing I/O.
+   * @return deserialized integer from stream.
    */
   public static int readVIntInRange(DataInput stream, int lower, int upper)
       throws IOException {
@@ -387,7 +392,8 @@ public static int decodeVIntSize(byte value) {
   }
 
   /**
-   * Get the encoded length if an integer is stored in a variable-length format
+   * Get the encoded length if an integer is stored in a variable-length format.
+   * @param i input i.
    * @return the encoded length 
    */
   public static int getVIntSize(long i) {
@@ -410,7 +416,7 @@ public static int getVIntSize(long i) {
    * @param in DataInput to read from 
    * @param enumType Class type of Enum
    * @return Enum represented by String read from DataInput
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static <T extends Enum<T>> T readEnum(DataInput in, Class<T> enumType)
     throws IOException{
@@ -420,7 +426,7 @@ public static <T extends Enum<T>> T readEnum(DataInput in, Class<T> enumType)
    * writes String value of enum to DataOutput. 
    * @param out Dataoutput stream
    * @param enumVal enum value
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void writeEnum(DataOutput out,  Enum<?> enumVal) 
     throws IOException{
@@ -446,7 +452,11 @@ public static void skipFully(DataInput in, int len) throws IOException {
     }
   }
 
-  /** Convert writables to a byte array */
+  /**
+   * Convert writables to a byte array.
+   * @param writables input writables.
+   * @return ByteArray.
+   */
   public static byte[] toByteArray(Writable... writables) {
     final DataOutputBuffer out = new DataOutputBuffer();
     try {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java
similarity index 69%
rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java
index 921d4a686e05a..993d2678d2a10 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractMSContract.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java
@@ -16,18 +16,18 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.fs.FileSystem;
+package org.apache.hadoop.io.compress;
 
 import java.io.IOException;
 
 /**
- * Test specification for MetadataStore contract tests. Supplies configuration
- * and MetadataStore instance.
+ * An exception class for when a closed compressor/decopressor is being used
+ * {@link org.apache.hadoop.io.compress.Compressor}
+ * {@link org.apache.hadoop.io.compress.Decompressor}
  */
-public abstract class AbstractMSContract {
+public class AlreadyClosedException extends IOException {
 
-  public abstract FileSystem getFileSystem() throws IOException;
-  public abstract MetadataStore getMetadataStore() throws IOException;
+  public AlreadyClosedException(String message) {
+    super(message);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
index 7fd5633daa698..7508def9a75c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
@@ -24,6 +24,7 @@
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 
@@ -99,7 +100,7 @@ public BZip2Codec() { }
    * @param out        the location for the final output stream
    * @return a stream the user can write uncompressed data to, to have it 
    *         compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
@@ -116,7 +117,7 @@ public CompressionOutputStream createOutputStream(OutputStream out)
    * @param compressor compressor to use
    * @return a stream the user can write uncompressed data to, to have it 
    *         compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
@@ -154,7 +155,7 @@ public Compressor createCompressor() {
    *
    * @param in the stream to read compressed bytes from
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
@@ -171,7 +172,7 @@ public CompressionInputStream createInputStream(InputStream in)
    * @param in           the stream to read compressed bytes from
    * @param decompressor decompressor to use
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
@@ -255,10 +256,7 @@ public BZip2CompressionOutputStream(OutputStream out)
 
     private void writeStreamHeader() throws IOException {
       if (super.out != null) {
-        // The compressed bzip2 stream should start with the
-        // identifying characters BZ. Caller of CBZip2OutputStream
-        // i.e. this class must write these characters.
-        out.write(HEADER.getBytes(StandardCharsets.UTF_8));
+        writeHeader(out);
       }
     }
 
@@ -337,6 +335,7 @@ private static class BZip2CompressionInputStream extends
     private boolean isSubHeaderStripped = false;
     private READ_MODE readMode = READ_MODE.CONTINUOUS;
     private long startingPos = 0L;
+    private boolean didInitialRead;
 
     // Following state machine handles different states of compressed stream
     // position
@@ -482,24 +481,42 @@ public void close() throws IOException {
     */
 
     public int read(byte[] b, int off, int len) throws IOException {
+      if (b == null) {
+        throw new NullPointerException();
+      }
+      if (off < 0 || len < 0 || len > b.length - off) {
+        throw new IndexOutOfBoundsException();
+      }
+      if (len == 0) {
+        return 0;
+      }
       if (needsReset) {
         internalReset();
       }
-
-      int result = 0;
-      result = this.input.read(b, off, len);
+      // When startingPos > 0, the stream should be initialized at the end of
+      // one block (which would correspond to be the start of another block).
+      // Thus, the initial read would technically be reading one byte passed a
+      // BZip2 end of block marker. To be consistent, we should also be
+      // updating the position to be one byte after the end of an block on the
+      // initial read.
+      boolean initializedAtEndOfBlock =
+          !didInitialRead && startingPos > 0 && readMode == READ_MODE.BYBLOCK;
+      int result = initializedAtEndOfBlock
+          ? BZip2Constants.END_OF_BLOCK
+          : this.input.read(b, off, len);
       if (result == BZip2Constants.END_OF_BLOCK) {
         this.posSM = POS_ADVERTISEMENT_STATE_MACHINE.ADVERTISE;
       }
 
       if (this.posSM == POS_ADVERTISEMENT_STATE_MACHINE.ADVERTISE) {
-        result = this.input.read(b, off, off + 1);
+        result = this.input.read(b, off, 1);
         // This is the precise time to update compressed stream position
         // to the client of this code.
         this.updatePos(true);
         this.posSM = POS_ADVERTISEMENT_STATE_MACHINE.HOLD;
       }
 
+      didInitialRead = true;
       return result;
 
     }
@@ -515,6 +532,7 @@ private void internalReset() throws IOException {
         needsReset = false;
         BufferedInputStream bufferedIn = readStreamHeader();
         input = new CBZip2InputStream(bufferedIn, this.readMode);
+        didInitialRead = false;
       }
     }    
     
@@ -547,4 +565,11 @@ private void updatePos(boolean shouldAddOn) {
 
   }// end of BZip2CompressionInputStream
 
+  @VisibleForTesting
+  public static void writeHeader(OutputStream out) throws IOException {
+    // The compressed bzip2 stream should start with the
+    // identifying characters BZ. Caller of CBZip2OutputStream
+    // i.e. this class must write these characters.
+    out.write(HEADER.getBytes(StandardCharsets.UTF_8));
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java
index de457d192400d..ff10332ea8d5a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java
@@ -43,7 +43,7 @@ public class BlockDecompressorStream extends DecompressorStream {
    * @param in input stream
    * @param decompressor decompressor to use
    * @param bufferSize size of buffer
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public BlockDecompressorStream(InputStream in, Decompressor decompressor, 
                                  int bufferSize) throws IOException {
@@ -55,7 +55,7 @@ public BlockDecompressorStream(InputStream in, Decompressor decompressor,
    * 
    * @param in input stream
    * @param decompressor decompressor to use
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public BlockDecompressorStream(InputStream in, Decompressor decompressor) throws IOException {
     super(in, decompressor);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
index 2ac2ca65173f0..5b1826f9e30a8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
@@ -109,7 +109,7 @@ private static <T> boolean payback(Map<Class<T>, Set<T>> pool, T codec) {
       synchronized (pool) {
         codecSet = pool.get(codecClass);
         if (codecSet == null) {
-          codecSet = new HashSet<T>();
+          codecSet = new HashSet<>();
           pool.put(codecClass, codecSet);
         }
       }
@@ -205,6 +205,7 @@ public static void returnCompressor(Compressor compressor) {
     }
     // if the compressor can't be reused, don't pool it.
     if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) {
+      compressor.end();
       return;
     }
     compressor.reset();
@@ -225,6 +226,7 @@ public static void returnDecompressor(Decompressor decompressor) {
     }
     // if the decompressor can't be reused, don't pool it.
     if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
+      decompressor.end();
       return;
     }
     decompressor.reset();
@@ -235,7 +237,10 @@ public static void returnDecompressor(Decompressor decompressor) {
 
   /**
    * Return the number of leased {@link Compressor}s for this
-   * {@link CompressionCodec}
+   * {@link CompressionCodec}.
+   *
+   * @param codec codec.
+   * @return the number of leased.
    */
   public static int getLeasedCompressorsCount(CompressionCodec codec) {
     return (codec == null) ? 0 : getLeaseCount(compressorCounts,
@@ -244,7 +249,10 @@ public static int getLeasedCompressorsCount(CompressionCodec codec) {
 
   /**
    * Return the number of leased {@link Decompressor}s for this
-   * {@link CompressionCodec}
+   * {@link CompressionCodec}.
+   *
+   * @param codec codec.
+   * @return the number of leased
    */
   public static int getLeasedDecompressorsCount(CompressionCodec codec) {
     return (codec == null) ? 0 : getLeaseCount(decompressorCounts,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodec.java
index f37aadfcb57f3..d064e1b914707 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodec.java
@@ -39,7 +39,7 @@ public interface CompressionCodec {
    * 
    * @param out the location for the final output stream
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   CompressionOutputStream createOutputStream(OutputStream out) 
   throws IOException;
@@ -51,7 +51,7 @@ CompressionOutputStream createOutputStream(OutputStream out)
    * @param out the location for the final output stream
    * @param compressor compressor to use
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   CompressionOutputStream createOutputStream(OutputStream out, 
                                              Compressor compressor) 
@@ -77,7 +77,7 @@ CompressionOutputStream createOutputStream(OutputStream out,
    * 
    * @param in the stream to read compressed bytes from
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   CompressionInputStream createInputStream(InputStream in) throws IOException;
   
@@ -88,7 +88,7 @@ CompressionOutputStream createOutputStream(OutputStream out,
    * @param in the stream to read compressed bytes from
    * @param decompressor decompressor to use
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   CompressionInputStream createInputStream(InputStream in, 
                                            Decompressor decompressor) 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
index 1fa7fd4b52be5..8e920a2e64c27 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
@@ -40,7 +40,7 @@ public class CompressionCodecFactory {
       LoggerFactory.getLogger(CompressionCodecFactory.class.getName());
   
   private static final ServiceLoader<CompressionCodec> CODEC_PROVIDERS =
-    ServiceLoader.load(CompressionCodec.class);
+          ServiceLoader.load(CompressionCodec.class);
 
   /**
    * A map from the reversed filename suffixes to the codecs.
@@ -49,15 +49,15 @@ public class CompressionCodecFactory {
    */
   private SortedMap<String, CompressionCodec> codecs = null;
 
-    /**
-     * A map from the reversed filename suffixes to the codecs.
-     * This is probably overkill, because the maps should be small, but it
-     * automatically supports finding the longest matching suffix.
-     */
-    private Map<String, CompressionCodec> codecsByName = null;
+  /**
+   * A map from the reversed filename suffixes to the codecs.
+   * This is probably overkill, because the maps should be small, but it
+   * automatically supports finding the longest matching suffix.
+   */
+  private Map<String, CompressionCodec> codecsByName = null;
 
   /**
-   * A map from class names to the codecs
+   * A map from class names to the codecs.
    */
   private HashMap<String, CompressionCodec> codecsByClassName = null;
 
@@ -80,8 +80,8 @@ private void addCodec(CompressionCodec codec) {
   @Override
   public String toString() {
     StringBuilder buf = new StringBuilder();
-    Iterator<Map.Entry<String, CompressionCodec>> itr = 
-      codecs.entrySet().iterator();
+    Iterator<Map.Entry<String, CompressionCodec>> itr =
+            codecs.entrySet().iterator();
     buf.append("{ ");
     if (itr.hasNext()) {
       Map.Entry<String, CompressionCodec> entry = itr.next();
@@ -110,8 +110,8 @@ public String toString() {
    */
   public static List<Class<? extends CompressionCodec>> getCodecClasses(
       Configuration conf) {
-    List<Class<? extends CompressionCodec>> result
-      = new ArrayList<Class<? extends CompressionCodec>>();
+    List<Class<? extends CompressionCodec>> result =
+            new ArrayList<Class<? extends CompressionCodec>>();
     // Add codec classes discovered via service loading
     synchronized (CODEC_PROVIDERS) {
       // CODEC_PROVIDERS is a lazy collection. Synchronize so it is
@@ -171,6 +171,8 @@ public static void setCodecClasses(Configuration conf,
   /**
    * Find the codecs specified in the config value io.compression.codecs 
    * and register them. Defaults to gzip and deflate.
+   *
+   * @param conf configuration.
    */
   public CompressionCodecFactory(Configuration conf) {
     codecs = new TreeMap<String, CompressionCodec>();
@@ -200,11 +202,13 @@ public CompressionCodec getCodec(Path file) {
       String filename = file.getName();
       String reversedFilename =
           new StringBuilder(filename).reverse().toString();
-      SortedMap<String, CompressionCodec> subMap = 
-        codecs.headMap(reversedFilename);
+      String lowerReversedFilename =
+              StringUtils.toLowerCase(reversedFilename);
+      SortedMap<String, CompressionCodec> subMap =
+              codecs.headMap(lowerReversedFilename);
       if (!subMap.isEmpty()) {
         String potentialSuffix = subMap.lastKey();
-        if (reversedFilename.startsWith(potentialSuffix)) {
+        if (lowerReversedFilename.startsWith(potentialSuffix)) {
           result = codecs.get(potentialSuffix);
         }
       }
@@ -224,57 +228,57 @@ public CompressionCodec getCodecByClassName(String classname) {
     return codecsByClassName.get(classname);
   }
 
-    /**
-     * Find the relevant compression codec for the codec's canonical class name
-     * or by codec alias.
-     * <p>
-     * Codec aliases are case insensitive.
-     * <p>
-     * The code alias is the short class name (without the package name).
-     * If the short class name ends with 'Codec', then there are two aliases for
-     * the codec, the complete short class name and the short class name without
-     * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
-     * alias are 'gzip' and 'gzipcodec'.
-     *
-     * @param codecName the canonical class name of the codec
-     * @return the codec object
-     */
-    public CompressionCodec getCodecByName(String codecName) {
-      if (codecsByClassName == null) {
-        return null;
-      }
-      CompressionCodec codec = getCodecByClassName(codecName);
-      if (codec == null) {
-        // trying to get the codec by name in case the name was specified
-        // instead a class
-        codec = codecsByName.get(StringUtils.toLowerCase(codecName));
-      }
-      return codec;
+  /**
+   * Find the relevant compression codec for the codec's canonical class name
+   * or by codec alias.
+   * <p>
+   * Codec aliases are case insensitive.
+   * <p>
+   * The code alias is the short class name (without the package name).
+   * If the short class name ends with 'Codec', then there are two aliases for
+   * the codec, the complete short class name and the short class name without
+   * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+   * alias are 'gzip' and 'gzipcodec'.
+   *
+   * @param codecName the canonical class name of the codec
+   * @return the codec object
+   */
+  public CompressionCodec getCodecByName(String codecName) {
+    if (codecsByClassName == null) {
+      return null;
     }
+    CompressionCodec codec = getCodecByClassName(codecName);
+    if (codec == null) {
+      // trying to get the codec by name in case the name was specified
+      // instead a class
+      codec = codecsByName.get(StringUtils.toLowerCase(codecName));
+    }
+    return codec;
+  }
 
-    /**
-     * Find the relevant compression codec for the codec's canonical class name
-     * or by codec alias and returns its implemetation class.
-     * <p>
-     * Codec aliases are case insensitive.
-     * <p>
-     * The code alias is the short class name (without the package name).
-     * If the short class name ends with 'Codec', then there are two aliases for
-     * the codec, the complete short class name and the short class name without
-     * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
-     * alias are 'gzip' and 'gzipcodec'.
-     *
-     * @param codecName the canonical class name of the codec
-     * @return the codec class
-     */
-    public Class<? extends CompressionCodec> getCodecClassByName(
-        String codecName) {
-      CompressionCodec codec = getCodecByName(codecName);
-      if (codec == null) {
-        return null;
-      }
-      return codec.getClass();
+  /**
+   * Find the relevant compression codec for the codec's canonical class name
+   * or by codec alias and returns its implemetation class.
+   * <p>
+   * Codec aliases are case insensitive.
+   * <p>
+   * The code alias is the short class name (without the package name).
+   * If the short class name ends with 'Codec', then there are two aliases for
+   * the codec, the complete short class name and the short class name without
+   * the 'Codec' ending. For example for the 'GzipCodec' codec class name the
+   * alias are 'gzip' and 'gzipcodec'.
+   *
+   * @param codecName the canonical class name of the codec
+   * @return the codec class
+   */
+  public Class<? extends CompressionCodec> getCodecClassByName(
+      String codecName) {
+    CompressionCodec codec = getCodecByName(codecName);
+    if (codec == null) {
+      return null;
     }
+    return codec.getClass();
+  }
 
   /**
    * Removes a suffix from a filename, if it has it.
@@ -291,7 +295,8 @@ public static String removeSuffix(String filename, String suffix) {
   
   /**
    * A little test program.
-   * @param args
+   * @param args arguments.
+   * @throws Exception exception.
    */
   public static void main(String[] args) throws Exception {
     Configuration conf = new Configuration();
@@ -323,8 +328,12 @@ public static void main(String[] args) throws Exception {
                 len = in.read(buffer);
               }
             } finally {
-              if(out != null) { out.close(); }
-              if(in  != null) { in.close(); }
+              if(out != null) {
+                out.close();
+              }
+              if(in  != null) {
+                in.close();
+              }
             }
           } else {
             CompressionInputStream in = null;
@@ -338,7 +347,9 @@ public static void main(String[] args) throws Exception {
                 len = in.read(buffer);
               }
             } finally {
-              if(in != null) { in.close(); }
+              if(in != null) {
+                in.close();
+              }
             }
           }
         }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
index 55bb132e9c87c..5bfec01ec945d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionInputStream.java
@@ -53,7 +53,7 @@ public abstract class CompressionInputStream extends InputStream
    * the decompressed bytes from the given stream.
    * 
    * @param in The input stream to be compressed.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected CompressionInputStream(InputStream in) throws IOException {
     if (!(in instanceof Seekable) || !(in instanceof PositionedReadable)) {
@@ -93,6 +93,8 @@ public IOStatistics getIOStatistics() {
   /**
    * Reset the decompressor to its initial state and discard any buffered data,
    * as the underlying stream may have been repositioned.
+   *
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void resetState() throws IOException;
   
@@ -118,7 +120,7 @@ public long getPos() throws IOException {
   /**
    * This method is current not supported.
    *
-   * @throws UnsupportedOperationException
+   * @throws UnsupportedOperationException Unsupported Operation Exception.
    */
 
   @Override
@@ -129,7 +131,7 @@ public void seek(long pos) throws UnsupportedOperationException {
   /**
    * This method is current not supported.
    *
-   * @throws UnsupportedOperationException
+   * @throws UnsupportedOperationException Unsupported Operation Exception.
    */
   @Override
   public boolean seekToNewSource(long targetPos) throws UnsupportedOperationException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
index 2a11ace81702c..2e412dcd58fce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionOutputStream.java
@@ -48,7 +48,7 @@ public abstract class CompressionOutputStream extends OutputStream
   /**
    * Create a compression output stream that writes
    * the compressed bytes to the given stream.
-   * @param out
+   * @param out out.
    */
   protected CompressionOutputStream(OutputStream out) {
     this.out = out;
@@ -89,12 +89,14 @@ public void flush() throws IOException {
   /**
    * Finishes writing compressed data to the output stream 
    * without closing the underlying stream.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void finish() throws IOException;
   
   /**
    * Reset the compression to the initial state. 
    * Does not reset the underlying stream.
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void resetState() throws IOException;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Compressor.java
index 537837faa0a51..7e2a6e679f43c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Compressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Compressor.java
@@ -65,11 +65,13 @@ public interface Compressor {
 
   /**
    * Return number of uncompressed bytes input so far.
+   * @return bytes read.
    */
   public long getBytesRead();
 
   /**
    * Return number of compressed bytes output so far.
+   * @return bytes written.
    */
   public long getBytesWritten();
 
@@ -97,6 +99,7 @@ public interface Compressor {
    * @param off Start offset of the data
    * @param len Size of the buffer
    * @return The actual number of bytes of compressed data.
+   * @throws IOException raised on errors performing I/O.
    */
   public int compress(byte[] b, int off, int len) throws IOException;
   
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
index e9558fab87325..30d4e29892eb7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Decompressor.java
@@ -96,7 +96,7 @@ public interface Decompressor {
    * @param off Start offset of the data
    * @param len Size of the buffer
    * @return The actual number of bytes of uncompressed data.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public int decompress(byte[] b, int off, int len) throws IOException;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
index 570d15c7f16aa..745105ce873af 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DecompressorStream.java
@@ -80,7 +80,7 @@ public DecompressorStream(InputStream in, Decompressor decompressor)
    * Allow derived classes to directly set the underlying stream.
    * 
    * @param in Underlying input stream.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected DecompressorStream(InputStream in) throws IOException {
     super(in);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
index d2ffb22eaafb3..b407ddb11046c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
index 8bfb7fe95c4e2..a5afb706c99c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
@@ -61,7 +61,7 @@ public Configuration getConf() {
    *
    * @param out the location for the final output stream
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
@@ -77,7 +77,7 @@ public CompressionOutputStream createOutputStream(OutputStream out)
    * @param out        the location for the final output stream
    * @param compressor compressor to use
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
@@ -125,7 +125,7 @@ public Compressor createCompressor() {
    *
    * @param in the stream to read compressed bytes from
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
@@ -141,7 +141,7 @@ public CompressionInputStream createInputStream(InputStream in)
    * @param in           the stream to read compressed bytes from
    * @param decompressor decompressor to use
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
index 77cf36a339b34..d64c6e512f87c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
@@ -61,7 +61,7 @@ public Configuration getConf() {
    *
    * @param out the location for the final output stream
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
@@ -77,7 +77,7 @@ public CompressionOutputStream createOutputStream(OutputStream out)
    * @param out        the location for the final output stream
    * @param compressor compressor to use
    * @return a stream the user can write uncompressed data to have it compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
@@ -122,7 +122,7 @@ public Compressor createCompressor() {
    *
    * @param in the stream to read compressed bytes from
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
@@ -138,7 +138,7 @@ public CompressionInputStream createInputStream(InputStream in)
    * @param in           the stream to read compressed bytes from
    * @param decompressor decompressor to use
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SplittableCompressionCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SplittableCompressionCodec.java
index a756f47260c33..f2e28774a46db 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SplittableCompressionCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SplittableCompressionCodec.java
@@ -61,6 +61,7 @@ public enum READ_MODE {CONTINUOUS, BYBLOCK};
    * Create a stream as dictated by the readMode.  This method is used when
    * the codecs wants the ability to work with the underlying stream positions.
    *
+   * @param decompressor decompressor.
    * @param seekableIn  The seekable input stream (seeks in compressed data)
    * @param start The start offset into the compressed stream. May be changed
    *              by the underlying codec.
@@ -69,6 +70,7 @@ public enum READ_MODE {CONTINUOUS, BYBLOCK};
    * @param readMode Controls whether stream position is reported continuously
    *                 from the compressed stream only only at block boundaries.
    * @return  a stream to read uncompressed bytes from
+   * @throws IOException raised on errors performing I/O.
    */
   SplitCompressionInputStream createInputStream(InputStream seekableIn,
       Decompressor decompressor, long start, long end, READ_MODE readMode)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
index a7afebc0c49ae..139e81eb73cc2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
@@ -116,7 +116,7 @@ private static int getBufferSize(Configuration conf) {
    *
    * @param out the location for the final output stream
    * @return a stream the user can write uncompressed data to have compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out)
@@ -132,7 +132,7 @@ public CompressionOutputStream createOutputStream(OutputStream out)
    * @param out        the location for the final output stream
    * @param compressor compressor to use
    * @return a stream the user can write uncompressed data to have compressed
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionOutputStream createOutputStream(OutputStream out,
@@ -173,7 +173,7 @@ public Compressor createCompressor() {
    *
    * @param in the stream to read compressed bytes from
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in)
@@ -189,7 +189,7 @@ public CompressionInputStream createInputStream(InputStream in)
    * @param in           the stream to read compressed bytes from
    * @param decompressor decompressor to use
    * @return a stream to read uncompressed bytes from
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public CompressionInputStream createInputStream(InputStream in,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
index 5713c56df6aef..9d1d85332489a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
@@ -67,6 +67,7 @@ public Bzip2Compressor() {
 
   /**
    * Creates a new compressor, taking settings from the configuration.
+   * @param conf configuration.
    */
   public Bzip2Compressor(Configuration conf) {
     this(Bzip2Factory.getBlockSize(conf),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
index 72ba97630e206..acd806b9b300a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java
@@ -50,6 +50,8 @@ public class Bzip2Decompressor implements Decompressor {
 
   /**
    * Creates a new decompressor.
+   * @param conserveMemory conserveMemory.
+   * @param directBufferSize directBufferSize.
    */
   public Bzip2Decompressor(boolean conserveMemory, int directBufferSize) {
     this.conserveMemory = conserveMemory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
index 8426d25c2950e..61e88d80d8ce4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java
@@ -27,6 +27,7 @@
 import java.io.InputStream;
 import java.io.IOException;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE;
 
 
@@ -152,6 +153,7 @@ public enum STATE {
   * This method reports the processed bytes so far. Please note that this
   * statistic is only updated on block boundaries and only when the stream is
   * initiated in BYBLOCK mode.
+  * @return ProcessedByteCount.
   */
   public long getProcessedByteCount() {
     return reportedBytesReadFromCompressedStream;
@@ -209,7 +211,7 @@ private int readAByte(InputStream inStream) throws IOException {
    * @param marker The bit pattern to be found in the stream
    * @param markerBitLength No of bits in the marker
    * @return true if the marker was found otherwise false
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws IllegalArgumentException if marketBitLength is greater than 63
    */
   public boolean skipToNextMarker(long marker, int markerBitLength)
@@ -282,7 +284,8 @@ private void makeMaps() {
   * the magic. Thus callers have to skip the first two bytes. Otherwise this
   * constructor will throw an exception.
   * </p>
-  *
+  * @param in in.
+  * @param readMode READ_MODE.
   * @throws IOException
   *             if the stream content is malformed or an I/O error occurs.
   * @throws NullPointerException
@@ -310,13 +313,24 @@ private CBZip2InputStream(final InputStream in, READ_MODE readMode, boolean skip
   }
     } else if (readMode == READ_MODE.BYBLOCK) {
       this.currentState = STATE.NO_PROCESS_STATE;
-      skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER,DELIMITER_BIT_LENGTH);
+      skipResult = skipToNextBlockMarker();
       if(!skipDecompression){
         changeStateToProcessABlock();
       }
     }
   }
 
+  /**
+   * Skips bytes in the stream until the start marker of a block is reached
+   * or end of stream is reached. Used for testing purposes to identify the
+   * start offsets of blocks.
+   */
+  @VisibleForTesting
+  boolean skipToNextBlockMarker() throws IOException {
+    return skipToNextMarker(
+        CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH);
+  }
+
   /**
    * Returns the number of bytes between the current stream position
    * and the immediate next BZip2 block marker.
@@ -326,7 +340,7 @@ private CBZip2InputStream(final InputStream in, READ_MODE readMode, boolean skip
    *
    * @return long Number of bytes between current stream position and the
    * next BZip2 block start marker.
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
    *
    */
   public static long numberOfBytesTillNextMarker(final InputStream in) throws IOException{
@@ -426,7 +440,7 @@ public int read(final byte[] dest, final int offs, final int len)
       //report 'end of block' or 'end of stream'
       result = b;
 
-      skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH);
+      skipResult = skipToNextBlockMarker();
 
       changeStateToProcessABlock();
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
index 850fec77c5109..50bdddb8136fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java
@@ -27,6 +27,7 @@
 import java.io.OutputStream;
 import java.io.IOException;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.io.IOUtils;
 
 /**
@@ -64,13 +65,10 @@
  * </pre>
  *
  * <table width="100%" border="1">
- *   <caption></caption>
+ * <caption>Memory usage by blocksize</caption>
  * <colgroup> <col width="33%" > <col width="33%" > <col width="33%" >
  * </colgroup>
  * <tr>
- * <th colspan="3">Memory usage by blocksize</th>
- * </tr>
- * <tr>
  * <th align="right">Blocksize</th> <th align="right">Compression<br>
  * memory usage</th> <th align="right">Decompression<br>
  * memory usage</th>
@@ -213,6 +211,10 @@ public class CBZip2OutputStream extends OutputStream implements BZip2Constants {
   /**
   * This method is accessible by subclasses for historical purposes. If you
   * don't know what it does then you don't need it.
+  * @param len len.
+  * @param freq freq.
+  * @param alphaSize alphaSize.
+  * @param maxLen maxLen.
   */
   protected static void hbMakeCodeLengths(char[] len, int[] freq,
       int alphaSize, int maxLen) {
@@ -780,8 +782,7 @@ private void initBlock() {
       inUse[i] = false;
     }
 
-    /* 20 is just a paranoia constant */
-    this.allowableBlockSize = (this.blockSize100k * BZip2Constants.baseBlockSize) - 20;
+    this.allowableBlockSize = getAllowableBlockSize(this.blockSize100k);
   }
 
   private void endBlock() throws IOException {
@@ -849,6 +850,7 @@ private void endCompression() throws IOException {
 
   /**
   * Returns the blocksize parameter specified at construction time.
+  * @return blocksize.
   */
   public final int getBlockSize() {
     return this.blockSize100k;
@@ -2091,4 +2093,9 @@ private static final class Data extends Object {
 
   }
 
+  @VisibleForTesting
+  static int getAllowableBlockSize(int blockSize100k) {
+    /* 20 is just a paranoia constant */
+    return (blockSize100k * BZip2Constants.baseBlockSize) - 20;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java
index 2b62ef78b2859..719d216abaed0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java
@@ -199,7 +199,7 @@ public synchronized boolean finished() {
    * @param off Start offset of the data
    * @param len Size of the buffer
    * @return The actual number of bytes of uncompressed data.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public synchronized int decompress(byte[] b, int off, int len)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
index d3775e286e895..58987c4dda3de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
@@ -187,7 +187,7 @@ public boolean finished() {
    * @param off Start offset of the data
    * @param len Size of the buffer
    * @return The actual number of bytes of compressed data.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public int decompress(byte[] b, int off, int len)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java
index fcb431dce86ca..d44413cc30912 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java
@@ -24,6 +24,7 @@
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.AlreadyClosedException;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.DoNotPool;
 import org.apache.hadoop.util.DataChecksum;
@@ -83,6 +84,10 @@ public int compress(byte[] b, int off, int len) throws IOException {
       throw new IOException("compress called on finished compressor");
     }
 
+    if (state == BuiltInGzipDecompressor.GzipStateLabel.ENDED) {
+      throw new AlreadyClosedException("compress called on closed compressor");
+    }
+
     int compressedBytesWritten = 0;
 
     // If we are not within uncompressed data yet, output the header.
@@ -139,6 +144,8 @@ public long getBytesWritten() {
   @Override
   public void end() {
     deflater.end();
+
+    state = BuiltInGzipDecompressor.GzipStateLabel.ENDED;
   }
 
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
index 47c21b4e3ea98..d47864a71f481 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java
@@ -23,6 +23,7 @@
 import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 
+import org.apache.hadoop.io.compress.AlreadyClosedException;
 import org.apache.hadoop.io.compress.Decompressor;
 import org.apache.hadoop.io.compress.DoNotPool;
 import org.apache.hadoop.util.DataChecksum;
@@ -109,7 +110,11 @@ public enum GzipStateLabel {
      * Immediately after the trailer (and potentially prior to the next gzip
      * member/substream header), without reset() having been called.
      */
-    FINISHED;
+    FINISHED,
+    /**
+     * Immediately after end() has been called.
+     */
+    ENDED;
   }
 
   /**
@@ -186,6 +191,10 @@ public synchronized int decompress(byte[] b, int off, int len)
   throws IOException {
     int numAvailBytes = 0;
 
+    if (state == GzipStateLabel.ENDED) {
+      throw new AlreadyClosedException("decompress called on closed decompressor");
+    }
+
     if (state != GzipStateLabel.DEFLATE_STREAM) {
       executeHeaderState();
 
@@ -476,6 +485,8 @@ public synchronized void reset() {
   @Override
   public synchronized void end() {
     inflater.end();
+
+    state = GzipStateLabel.ENDED;
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
index da8a90bb3170e..89e05fc6d07be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
@@ -240,6 +240,7 @@ public ZlibCompressor() {
 
   /**
    * Creates a new compressor, taking settings from the configuration.
+   * @param conf configuration.
    */
   public ZlibCompressor(Configuration conf) {
     this(ZlibFactory.getCompressionLevel(conf),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
index f642d7713035d..c2615548d23ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.java
@@ -101,6 +101,8 @@ static boolean isNativeZlibLoaded() {
 
   /**
    * Creates a new decompressor.
+   * @param header header.
+   * @param directBufferSize directBufferSize.
    */
   public ZlibDecompressor(CompressionHeader header, int directBufferSize) {
     this.header = header;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
index 883f1717eea93..c2de494457ccd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
@@ -66,7 +66,7 @@ public static void loadNativeZLib() {
   /**
    * Set the flag whether to use native library. Used for testing non-native
    * libraries
-   *
+   * @param isLoaded isLoaded.
    */
   @VisibleForTesting
   public static void setNativeZlibLoaded(final boolean isLoaded) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
index bc51f3d98a505..a77b59640cdaf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
@@ -84,6 +84,8 @@ public static int getRecommendedBufferSize() {
   /**
    * Creates a new compressor with the default compression level.
    * Compressed data will be generated in ZStandard format.
+   * @param level level.
+   * @param bufferSize bufferSize.
    */
   public ZStandardCompressor(int level, int bufferSize) {
     this(level, bufferSize, bufferSize);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
index bc9d29cb4f294..792547a62faea 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
@@ -73,6 +73,7 @@ public ZStandardDecompressor() {
 
   /**
    * Creates a new decompressor.
+   * @param bufferSize bufferSize.
    */
   public ZStandardDecompressor(int bufferSize) {
     this.directBufferSize = bufferSize;
@@ -113,6 +114,12 @@ private void setInputFromSavedData() {
     compressedDirectBuf.put(
         userBuf, userBufOff, bytesInCompressedBuffer);
 
+    // Set the finished to false when compressedDirectBuf still
+    // contains some bytes.
+    if (compressedDirectBuf.position() > 0 && finished) {
+      finished = false;
+    }
+
     userBufOff += bytesInCompressedBuffer;
     userBufferBytesToConsume -= bytesInCompressedBuffer;
   }
@@ -186,6 +193,13 @@ public int decompress(byte[] b, int off, int len)
         0,
         directBufferSize
     );
+
+    // Set the finished to false when compressedDirectBuf still
+    // contains some bytes.
+    if (remaining > 0 && finished) {
+      finished = false;
+    }
+
     uncompressedDirectBuf.limit(n);
 
     // Get at most 'len' bytes
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 2632f4b82f070..f89a0d9812d7c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -83,6 +83,7 @@ private CodecUtil() { }
   /**
    * Create encoder corresponding to given codec.
    * @param options Erasure codec options
+   * @param conf configuration.
    * @return erasure encoder
    */
   public static ErasureEncoder createEncoder(Configuration conf,
@@ -100,6 +101,7 @@ public static ErasureEncoder createEncoder(Configuration conf,
   /**
    * Create decoder corresponding to given codec.
    * @param options Erasure codec options
+   * @param conf configuration.
    * @return erasure decoder
    */
   public static ErasureDecoder createDecoder(Configuration conf,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
index ec317eee4dc3e..83a3151282096 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
@@ -61,6 +61,7 @@ private ErasureCodeNative() {}
 
   /**
    * Are native libraries loaded?
+   * @return if is native code loaded true,not false.
    */
   public static boolean isNativeCodeLoaded() {
     return LOADING_FAILURE_REASON == null;
@@ -82,6 +83,7 @@ public static void checkNativeCodeLoaded() {
 
   /**
    * Get the native library name that's available or supported.
+   * @return library name.
    */
   public static native String getLibraryName();
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
index c75eaead83d01..22ab632a49512 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
@@ -19,7 +19,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.erasurecode.CodecUtil;
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodecOptions;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
index b5ae1f1e399a8..ab1775538bd5f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -68,6 +68,7 @@ public interface ErasureCoder extends Configurable {
    *
    * @param blockGroup the erasure coding block group containing all necessary
    *                   information for codec calculation
+   * @return ErasureCodingStep.
    */
   ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
index fb89d99a0540c..333647c982b9f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
@@ -46,8 +46,9 @@ public interface ErasureCodingStep {
   /**
    * Perform encoding or decoding given the input chunks, and generated results
    * will be written to the output chunks.
-   * @param inputChunks
-   * @param outputChunks
+   * @param inputChunks inputChunks.
+   * @param outputChunks outputChunks.
+   * @throws IOException raised on errors performing I/O.
    */
   void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks)
       throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
index 004fd38df1141..30020b9959f0b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
@@ -65,7 +65,7 @@ public ErasureCoderOptions getOptions() {
   /**
    * We have all the data blocks and parity blocks as input blocks for
    * recovering by default. It's codec specific
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return input blocks
    */
   protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
@@ -83,7 +83,7 @@ protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
 
   /**
    * Which blocks were erased ?
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return output blocks to recover
    */
   protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
@@ -118,7 +118,7 @@ public void release() {
 
   /**
    * Perform decoding against a block blockGroup.
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return decoding step for caller to do the real work
    */
   protected abstract ErasureCodingStep prepareDecodingStep(
@@ -126,7 +126,7 @@ protected abstract ErasureCodingStep prepareDecodingStep(
 
   /**
    * Get the number of erased blocks in the block group.
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return number of erased blocks
    */
   protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
@@ -153,7 +153,7 @@ protected static int getNumErasedBlocks(ECBlock[] inputBlocks) {
 
   /**
    * Get indexes of erased blocks from inputBlocks
-   * @param inputBlocks
+   * @param inputBlocks inputBlocks.
    * @return indexes of erased blocks from inputBlocks
    */
   protected int[] getErasedIndexes(ECBlock[] inputBlocks) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
index 24f55470e1727..20a396d313678 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
@@ -37,10 +37,10 @@ public class ErasureDecodingStep implements ErasureCodingStep {
 
   /**
    * The constructor with all the necessary info.
-   * @param inputBlocks
+   * @param inputBlocks inputBlocks.
    * @param erasedIndexes the indexes of erased blocks in inputBlocks array
-   * @param outputBlocks
-   * @param rawDecoder
+   * @param outputBlocks outputBlocks.
+   * @param rawDecoder rawDecoder.
    */
   public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
                              ECBlock[] outputBlocks,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
index 81666e9b76b2e..cca272f69a28d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
@@ -83,7 +83,7 @@ public void release() {
 
   /**
    * Perform encoding against a block group.
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return encoding step for caller to do the real work
    */
   protected abstract ErasureCodingStep prepareEncodingStep(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
index 5fc5c7a09928f..9e696d2c58477 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
@@ -36,9 +36,9 @@ public class ErasureEncodingStep implements ErasureCodingStep {
 
   /**
    * The constructor with all the necessary info.
-   * @param inputBlocks
-   * @param outputBlocks
-   * @param rawEncoder
+   * @param inputBlocks inputBlocks.
+   * @param outputBlocks outputBlocks.
+   * @param rawEncoder rawEncoder.
    */
   public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
                              RawErasureEncoder rawEncoder) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
index a0f5b72710679..46f0a76da17df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHErasureCodingStep.java
@@ -38,8 +38,8 @@ public abstract class HHErasureCodingStep
   /**
    * Constructor given input blocks and output blocks.
    *
-   * @param inputBlocks
-   * @param outputBlocks
+   * @param inputBlocks inputBlocks.
+   * @param outputBlocks outputBlocks.
    */
   public HHErasureCodingStep(ECBlock[] inputBlocks,
                                      ECBlock[] outputBlocks) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
index 16a3c0fa61c4b..4d594f476dfd9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecodingStep.java
@@ -43,9 +43,9 @@ public class HHXORErasureDecodingStep extends HHErasureCodingStep {
 
   /**
    * The constructor with all the necessary info.
-   * @param inputBlocks
+   * @param inputBlocks inputBlocks.
    * @param erasedIndexes the indexes of erased blocks in inputBlocks array
-   * @param outputBlocks
+   * @param outputBlocks outputBlocks.
    * @param rawDecoder underlying RS decoder for hitchhiker decoding
    * @param rawEncoder underlying XOR encoder for hitchhiker decoding
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
index 6a5644270117b..f571e932b6a85 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncodingStep.java
@@ -40,8 +40,8 @@ public class HHXORErasureEncodingStep extends HHErasureCodingStep {
   /**
    * The constructor with all the necessary info.
    *
-   * @param inputBlocks
-   * @param outputBlocks
+   * @param inputBlocks inputBlocks.
+   * @param outputBlocks outputBlocks.
    * @param rsRawEncoder  underlying RS encoder for hitchhiker encoding
    * @param xorRawEncoder underlying XOR encoder for hitchhiker encoding
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
index 1a0e5c030e070..9aae5e43c86f7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
@@ -53,7 +53,7 @@ protected ErasureCodingStep prepareDecodingStep(
   /**
    * Which blocks were erased ? For XOR it's simple we only allow and return one
    * erased block, either data or parity.
-   * @param blockGroup
+   * @param blockGroup blockGroup.
    * @return output blocks to recover
    */
   @Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
index 91d02415bfd93..7f771c9677da8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/util/HHUtil.java
@@ -202,6 +202,8 @@ public static ByteBuffer getPiggyBackForDecode(ByteBuffer[][] inputs,
 
   /**
    * Find the valid input from all the inputs.
+   *
+   * @param <T> Generics Type T.
    * @param inputs input buffers to look for valid input
    * @return the first valid input
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
index 3f1b0c22941bd..0407d16120819 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
@@ -33,7 +33,7 @@ public class BlockGrouper {
 
   /**
    * Set EC schema.
-   * @param schema
+   * @param schema schema.
    */
   public void setSchema(ECSchema schema) {
     this.schema = schema;
@@ -41,7 +41,7 @@ public void setSchema(ECSchema schema) {
 
   /**
    * Get EC schema.
-   * @return
+   * @return ECSchema.
    */
   protected ECSchema getSchema() {
     return schema;
@@ -67,7 +67,7 @@ public int getRequiredNumParityBlocks() {
    * Calculating and organizing BlockGroup, to be called by ECManager
    * @param dataBlocks Data blocks to compute parity blocks against
    * @param parityBlocks To be computed parity blocks
-   * @return
+   * @return ECBlockGroup.
    */
   public ECBlockGroup makeBlockGroup(ECBlock[] dataBlocks,
                                      ECBlock[] parityBlocks) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DecodingValidator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DecodingValidator.java
index 396aac08cc517..a9bc297739810 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DecodingValidator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DecodingValidator.java
@@ -68,7 +68,7 @@ public DecodingValidator(RawErasureDecoder decoder) {
    * @param erasedIndexes indexes of erased units used for decoding
    * @param outputs decoded output buffers, which are ready to be read after
    *                the call
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void validate(ByteBuffer[] inputs, int[] erasedIndexes,
       ByteBuffer[] outputs) throws IOException {
@@ -133,7 +133,7 @@ public void validate(ByteBuffer[] inputs, int[] erasedIndexes,
    * @param inputs input buffers used for decoding
    * @param erasedIndexes indexes of erased units used for decoding
    * @param outputs decoded output buffers
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void validate(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
index 249930ebe3f22..329bf7c3aaf7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
@@ -80,8 +80,9 @@ public RawErasureDecoder(ErasureCoderOptions coderOptions) {
    * @param erasedIndexes indexes of erased units in the inputs array
    * @param outputs output buffers to put decoded data into according to
    *                erasedIndexes, ready for read after the call
+   * @throws IOException raised on errors performing I/O.
    */
-  public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+  public synchronized void decode(ByteBuffer[] inputs, int[] erasedIndexes,
                      ByteBuffer[] outputs) throws IOException {
     ByteBufferDecodingState decodingState = new ByteBufferDecodingState(this,
         inputs, erasedIndexes, outputs);
@@ -117,6 +118,7 @@ public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
   /**
    * Perform the real decoding using Direct ByteBuffer.
    * @param decodingState the decoding state
+   * @throws IOException  raised on errors performing I/O.
    */
   protected abstract void doDecode(ByteBufferDecodingState decodingState)
       throws IOException;
@@ -130,7 +132,7 @@ protected abstract void doDecode(ByteBufferDecodingState decodingState)
    *                erasedIndexes, ready for read after the call
    * @throws IOException if the decoder is closed.
    */
-  public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
+  public synchronized void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs)
       throws IOException {
     ByteArrayDecodingState decodingState = new ByteArrayDecodingState(this,
         inputs, erasedIndexes, outputs);
@@ -163,7 +165,7 @@ protected abstract void doDecode(ByteArrayDecodingState decodingState)
    *                erasedIndexes, ready for read after the call
    * @throws IOException if the decoder is closed
    */
-  public void decode(ECChunk[] inputs, int[] erasedIndexes,
+  public synchronized void decode(ECChunk[] inputs, int[] erasedIndexes,
                      ECChunk[] outputs) throws IOException {
     ByteBuffer[] newInputs = CoderUtil.toBuffers(inputs);
     ByteBuffer[] newOutputs = CoderUtil.toBuffers(outputs);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
index 6d2ecd20525f4..d5ccb12c9d6a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
@@ -100,7 +100,8 @@ public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs)
 
   /**
    * Perform the real encoding work using direct ByteBuffer.
-   * @param encodingState the encoding state
+   * @param encodingState the encoding state.
+   * @throws IOException raised on errors performing I/O.
    */
   protected abstract void doEncode(ByteBufferEncodingState encodingState)
       throws IOException;
@@ -111,6 +112,7 @@ protected abstract void doEncode(ByteBufferEncodingState encodingState)
    * @param inputs input buffers to read data from
    * @param outputs output buffers to put the encoded data into, read to read
    *                after the call
+   * @throws IOException raised on errors performing I/O.
    */
   public void encode(byte[][] inputs, byte[][] outputs) throws IOException {
     ByteArrayEncodingState baeState = new ByteArrayEncodingState(
@@ -128,6 +130,7 @@ public void encode(byte[][] inputs, byte[][] outputs) throws IOException {
    * Perform the real encoding work using bytes array, supporting offsets
    * and lengths.
    * @param encodingState the encoding state
+   * @throws IOException  raised on errors performing I/O.
    */
   protected abstract void doEncode(ByteArrayEncodingState encodingState)
       throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
index 6de0716174319..90e57201c545b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
@@ -36,6 +36,10 @@ private DumpUtil() {
   /**
    * Convert bytes into format like 0x02 02 00 80.
    * If limit is negative or too large, then all bytes will be converted.
+   *
+   * @param bytes bytes.
+   * @param limit limit.
+   * @return bytesToHex.
    */
   public static String bytesToHex(byte[] bytes, int limit) {
     if (limit <= 0 || limit > bytes.length) {
@@ -70,8 +74,8 @@ public static void dumpMatrix(byte[] matrix,
 
   /**
    * Print data in hex format in an array of chunks.
-   * @param header
-   * @param chunks
+   * @param header header.
+   * @param chunks chunks.
    */
   public static void dumpChunks(String header, ECChunk[] chunks) {
     System.out.println();
@@ -84,7 +88,7 @@ public static void dumpChunks(String header, ECChunk[] chunks) {
 
   /**
    * Print data in hex format in a chunk.
-   * @param chunk
+   * @param chunk chunk.
    */
   public static void dumpChunk(ECChunk chunk) {
     String str;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GF256.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GF256.java
index 35534f307a7a0..b48a23f8b7085 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GF256.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GF256.java
@@ -195,6 +195,10 @@ public static byte gfInv(byte a) {
    * Invert a matrix assuming it's invertible.
    *
    * Ported from Intel ISA-L library.
+   *
+   * @param inMatrix inMatrix.
+   * @param outMatrix outMatrix.
+   * @param n n
    */
   public static void gfInvertMatrix(byte[] inMatrix, byte[] outMatrix, int n) {
     byte temp;
@@ -262,7 +266,11 @@ public static void gfInvertMatrix(byte[] inMatrix, byte[] outMatrix, int n) {
    *
    * Calculates const table gftbl in GF(2^8) from single input A
    * gftbl(A) = {A{00}, A{01}, A{02}, ... , A{0f} }, {A{00}, A{10}, A{20},
-   * ... , A{f0} } -- from ISA-L implementation
+   * ... , A{f0} } -- from ISA-L implementation.
+   *
+   * @param c c.
+   * @param tbl tbl.
+   * @param offset offset.
    */
   public static void gfVectMulInit(byte c, byte[] tbl, int offset) {
     byte c2 = (byte) ((c << 1) ^ ((c & 0x80) != 0 ? 0x1d : 0));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
index f80fceca94c34..6d22ff0f62eb3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
@@ -93,10 +93,11 @@ private GaloisField(int fieldSize, int primitivePolynomial) {
   }
 
   /**
-   * Get the object performs Galois field arithmetics
+   * Get the object performs Galois field arithmetics.
    *
    * @param fieldSize           size of the field
    * @param primitivePolynomial a primitive polynomial corresponds to the size
+   * @return GaloisField.
    */
   public static GaloisField getInstance(int fieldSize,
                                         int primitivePolynomial) {
@@ -114,7 +115,8 @@ public static GaloisField getInstance(int fieldSize,
   }
 
   /**
-   * Get the object performs Galois field arithmetic with default setting
+   * Get the object performs Galois field arithmetic with default setting.
+   * @return GaloisField.
    */
   public static GaloisField getInstance() {
     return getInstance(DEFAULT_FIELD_SIZE, DEFAULT_PRIMITIVE_POLYNOMIAL);
@@ -236,7 +238,13 @@ public void solveVandermondeSystem(int[] x, int[] y, int len) {
   }
 
   /**
-   * A "bulk" version to the solving of Vandermonde System
+   * A "bulk" version to the solving of Vandermonde System.
+   *
+   * @param x input x.
+   * @param y input y.
+   * @param outputOffsets input outputOffsets.
+   * @param len input len.
+   * @param dataLen input dataLen.
    */
   public void solveVandermondeSystem(int[] x, byte[][] y, int[] outputOffsets,
                                      int len, int dataLen) {
@@ -269,6 +277,10 @@ public void solveVandermondeSystem(int[] x, byte[][] y, int[] outputOffsets,
 
   /**
    * A "bulk" version of the solveVandermondeSystem, using ByteBuffer.
+   *
+   * @param x input x.
+   * @param y input y.
+   * @param len input len.
    */
   public void solveVandermondeSystem(int[] x, ByteBuffer[] y, int len) {
     ByteBuffer p;
@@ -413,10 +425,10 @@ public void substitute(byte[][] p, byte[] q, int x) {
    * Tends to be 2X faster than the "int" substitute in a loop.
    *
    * @param p input polynomial
-   * @param offsets
-   * @param len
+   * @param offsets input offset.
+   * @param len input len.
    * @param q store the return result
-   * @param offset
+   * @param offset input offset.
    * @param x input field
    */
   public void substitute(byte[][] p, int[] offsets,
@@ -440,6 +452,7 @@ public void substitute(byte[][] p, int[] offsets,
    * @param p input polynomial
    * @param q store the return result
    * @param x input field
+   * @param len input len.
    */
   public void substitute(ByteBuffer[] p, int len, ByteBuffer q, int x) {
     int y = 1, iIdx, oIdx;
@@ -459,6 +472,9 @@ public void substitute(ByteBuffer[] p, int len, ByteBuffer q, int x) {
   /**
    * The "bulk" version of the remainder.
    * Warning: This function will modify the "dividend" inputs.
+   *
+   * @param divisor divisor.
+   * @param dividend dividend.
    */
   public void remainder(byte[][] dividend, int[] divisor) {
     for (int i = dividend.length - divisor.length; i >= 0; i--) {
@@ -476,6 +492,11 @@ public void remainder(byte[][] dividend, int[] divisor) {
   /**
    * The "bulk" version of the remainder.
    * Warning: This function will modify the "dividend" inputs.
+   *
+   * @param dividend dividend.
+   * @param offsets offsets.
+   * @param len len.
+   * @param divisor divisor.
    */
   public void remainder(byte[][] dividend, int[] offsets,
                         int len, int[] divisor) {
@@ -497,6 +518,9 @@ public void remainder(byte[][] dividend, int[] offsets,
   /**
    * The "bulk" version of the remainder, using ByteBuffer.
    * Warning: This function will modify the "dividend" inputs.
+   *
+   * @param dividend dividend.
+   * @param divisor divisor.
    */
   public void remainder(ByteBuffer[] dividend, int[] divisor) {
     int idx1, idx2;
@@ -519,6 +543,8 @@ public void remainder(ByteBuffer[] dividend, int[] divisor) {
   /**
    * Perform Gaussian elimination on the given matrix. This matrix has to be a
    * fat matrix (number of rows &gt; number of columns).
+   *
+   * @param matrix matrix.
    */
   public void gaussianElimination(int[][] matrix) {
     assert(matrix != null && matrix.length > 0 && matrix[0].length > 0
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
index 43823d0f8c300..b1fdc82a11628 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
@@ -59,6 +59,10 @@ public static void initTables(int k, int rows, byte[] codingMatrix,
 
   /**
    * Ported from Intel ISA-L library.
+   *
+   * @param k k.
+   * @param a a.
+   * @param m m.
    */
   public static void genCauchyMatrix(byte[] a, int m, int k) {
     // Identity matrix in high position
@@ -82,6 +86,13 @@ public static void genCauchyMatrix(byte[] a, int m, int k) {
    *
    * The algorithm is ported from Intel ISA-L library for compatible. It
    * leverages Java auto-vectorization support for performance.
+   *
+   * @param gfTables gfTables.
+   * @param dataLen dataLen.
+   * @param inputs inputs.
+   * @param inputOffsets inputOffsets.
+   * @param outputs outputs.
+   * @param outputOffsets outputOffsets.
    */
   public static void encodeData(byte[] gfTables, int dataLen, byte[][] inputs,
       int[] inputOffsets, byte[][] outputs,
@@ -133,6 +144,10 @@ public static void encodeData(byte[] gfTables, int dataLen, byte[][] inputs,
 
   /**
    * See above. Try to use the byte[] version when possible.
+   *
+   * @param gfTables gfTables.
+   * @param inputs inputs.
+   * @param outputs outputs.
    */
   public static void encodeData(byte[] gfTables, ByteBuffer[] inputs,
       ByteBuffer[] outputs) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/ByteArray.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/ByteArray.java
index c6c8b3fe3e1fe..964fb04c1b976 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/ByteArray.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/ByteArray.java
@@ -35,7 +35,7 @@ public final class ByteArray implements RawComparable {
   /**
    * Constructing a ByteArray from a {@link BytesWritable}.
    * 
-   * @param other
+   * @param other other.
    */
   public ByteArray(BytesWritable other) {
     this(other.getBytes(), 0, other.getLength());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
index 09cd2825e3cf2..aeacc16a78f9b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/TFile.java
@@ -276,7 +276,7 @@ private enum State {
      *          </ul>
      * @param conf
      *          The configuration object.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Writer(FSDataOutputStream fsdos, int minBlockSize,
         String compressName, String comparator, Configuration conf)
@@ -350,7 +350,7 @@ public void close() throws IOException {
      *          Buffer for key.
      * @param value
      *          Buffer for value.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void append(byte[] key, byte[] value) throws IOException {
       append(key, 0, key.length, value, 0, value.length);
@@ -521,7 +521,7 @@ public void close() throws IOException {
      *          exactly as many bytes as specified here before calling close on
      *          the returned output stream.
      * @return The key appending output stream.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * 
      */
     public DataOutputStream prepareAppendKey(int length) throws IOException {
@@ -548,8 +548,8 @@ public DataOutputStream prepareAppendKey(int length) throws IOException {
      *          the returned output stream. Advertising the value size up-front
      *          guarantees that the value is encoded in one chunk, and avoids
      *          intermediate chunk buffering.
-     * @throws IOException
-     * 
+     * @throws IOException raised on errors performing I/O.
+     * @return DataOutputStream.
      */
     public DataOutputStream prepareAppendValue(int length) throws IOException {
       if (state != State.END_KEY) {
@@ -588,7 +588,7 @@ public DataOutputStream prepareAppendValue(int length) throws IOException {
      *          {@link TFile#getSupportedCompressionAlgorithms()}.
      * @return A DataOutputStream that can be used to write Meta Block data.
      *         Closing the stream would signal the ending of the block.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * @throws MetaBlockAlreadyExists
      *           the Meta Block with the same name already exists.
      */
@@ -616,7 +616,7 @@ public DataOutputStream prepareMetaBlock(String name, String compressName)
      *          Name of the meta block.
      * @return A DataOutputStream that can be used to write Meta Block data.
      *         Closing the stream would signal the ending of the block.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * @throws MetaBlockAlreadyExists
      *           the Meta Block with the same name already exists.
      */
@@ -796,8 +796,8 @@ public boolean equals(Object obj) {
      *          The length of TFile. This is required because we have no easy
      *          way of knowing the actual size of the input file through the
      *          File input stream.
-     * @param conf
-     * @throws IOException
+     * @param conf configuration.
+     * @throws IOException raised on errors performing I/O.
      */
     public Reader(FSDataInputStream fsdis, long fileLength, Configuration conf)
         throws IOException {
@@ -896,7 +896,7 @@ synchronized void checkTFileDataIndex() throws IOException {
      * Get the first key in the TFile.
      * 
      * @return The first key in the TFile.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawComparable getFirstKey() throws IOException {
       checkTFileDataIndex();
@@ -907,7 +907,7 @@ public RawComparable getFirstKey() throws IOException {
      * Get the last key in the TFile.
      * 
      * @return The last key in the TFile.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawComparable getLastKey() throws IOException {
       checkTFileDataIndex();
@@ -1043,7 +1043,7 @@ Location getLocationNear(long offset) {
      *          the user supplied offset.
      * @return the RecordNum to the corresponding entry. If no such entry
      *         exists, it returns the total entry count.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public long getRecordNumNear(long offset) throws IOException {
       return getRecordNumByLocation(getLocationNear(offset));
@@ -1058,7 +1058,7 @@ public long getRecordNumNear(long offset) throws IOException {
      * @return the key that fits the requirement; or null if no such key exists
      *         (which could happen if the offset is close to the end of the
      *         TFile).
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public RawComparable getKeyNear(long offset) throws IOException {
       int blockIndex = readerBCF.getBlockIndexNear(offset);
@@ -1072,7 +1072,7 @@ public RawComparable getKeyNear(long offset) throws IOException {
      * 
      * @return The scanner object. A valid Scanner is always returned even if
      *         the TFile is empty.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Scanner createScanner() throws IOException {
       return new Scanner(this, begin, end);
@@ -1089,7 +1089,7 @@ public Scanner createScanner() throws IOException {
      *         specified byte-region but always round up to the compression
      *         block boundaries. It is possible that the returned scanner
      *         contains zero key-value pairs even if length is positive.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Scanner createScannerByByteRange(long offset, long length) throws IOException {
       return new Scanner(this, offset, offset + length);
@@ -1106,7 +1106,7 @@ public Scanner createScannerByByteRange(long offset, long length) throws IOExcep
      *          key-value entry of the TFile.
      * @return The actual coverage of the returned scanner will cover all keys
      *         greater than or equal to the beginKey and less than the endKey.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * 
      * @deprecated Use {@link #createScannerByKey(byte[], byte[])} instead.
      */
@@ -1127,7 +1127,7 @@ public Scanner createScanner(byte[] beginKey, byte[] endKey)
      *          key-value entry of the TFile.
      * @return The actual coverage of the returned scanner will cover all keys
      *         greater than or equal to the beginKey and less than the endKey.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Scanner createScannerByKey(byte[] beginKey, byte[] endKey)
         throws IOException {
@@ -1147,7 +1147,7 @@ public Scanner createScannerByKey(byte[] beginKey, byte[] endKey)
      *          key-value entry of the TFile.
      * @return The actual coverage of the returned scanner will cover all keys
      *         greater than or equal to the beginKey and less than the endKey.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      * 
      * @deprecated Use {@link #createScannerByKey(RawComparable, RawComparable)}
      *             instead.
@@ -1169,7 +1169,7 @@ public Scanner createScanner(RawComparable beginKey, RawComparable endKey)
      *          key-value entry of the TFile.
      * @return The actual coverage of the returned scanner will cover all keys
      *         greater than or equal to the beginKey and less than the endKey.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Scanner createScannerByKey(RawComparable beginKey, RawComparable endKey)
         throws IOException {
@@ -1189,7 +1189,7 @@ public Scanner createScannerByKey(RawComparable beginKey, RawComparable endKey)
      *          The RecordNum for the last record (exclusive). To scan the whole
      *          file, either specify endRecNum==-1 or endRecNum==getEntryCount().
      * @return The TFile scanner that covers the specified range of records.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Scanner createScannerByRecordNum(long beginRecNum, long endRecNum)
         throws IOException {
@@ -1313,7 +1313,7 @@ protected Scanner(Reader reader, long offBegin, long offEnd)
        * @param endKey
        *          End key of the scan. If null, scan up to the last &lt;K, V&gt;
        *          entry of the TFile.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       protected Scanner(Reader reader, RawComparable beginKey,
           RawComparable endKey) throws IOException {
@@ -1338,7 +1338,7 @@ protected Scanner(Reader reader, RawComparable beginKey,
        * @param key
        *          The input key
        * @return true if we find an equal key.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public boolean seekTo(byte[] key) throws IOException {
         return seekTo(key, 0, key.length);
@@ -1356,7 +1356,7 @@ public boolean seekTo(byte[] key) throws IOException {
        * @param keyLen
        *          key buffer length.
        * @return true if we find an equal key; false otherwise.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public boolean seekTo(byte[] key, int keyOffset, int keyLen)
           throws IOException {
@@ -1432,7 +1432,7 @@ private void seekTo(Location l) throws IOException {
        * Rewind to the first entry in the scanner. The entry returned by the
        * previous entry() call will be invalid.
        * 
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void rewind() throws IOException {
         seekTo(beginLocation);
@@ -1442,7 +1442,7 @@ public void rewind() throws IOException {
        * Seek to the end of the scanner. The entry returned by the previous
        * entry() call will be invalid.
        * 
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void seekToEnd() throws IOException {
         parkCursorAtEnd();
@@ -1455,7 +1455,7 @@ public void seekToEnd() throws IOException {
        * 
        * @param key
        *          The input key
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void lowerBound(byte[] key) throws IOException {
         lowerBound(key, 0, key.length);
@@ -1472,7 +1472,7 @@ public void lowerBound(byte[] key) throws IOException {
        *          offset in the key buffer.
        * @param keyLen
        *          key buffer length.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void lowerBound(byte[] key, int keyOffset, int keyLen)
           throws IOException {
@@ -1486,7 +1486,7 @@ public void lowerBound(byte[] key, int keyOffset, int keyLen)
        * 
        * @param key
        *          The input key
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void upperBound(byte[] key) throws IOException {
         upperBound(key, 0, key.length);
@@ -1503,7 +1503,7 @@ public void upperBound(byte[] key) throws IOException {
        *          offset in the key buffer.
        * @param keyLen
        *          key buffer length.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public void upperBound(byte[] key, int keyOffset, int keyLen)
           throws IOException {
@@ -1516,7 +1516,7 @@ public void upperBound(byte[] key, int keyOffset, int keyLen)
        * 
        * @return true if the cursor successfully moves. False when cursor is
        *         already at the end location and cannot be advanced.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public boolean advance() throws IOException {
         if (atEnd()) {
@@ -1614,7 +1614,7 @@ void checkKey() throws IOException {
        * Get an entry to access the key and value.
        * 
        * @return The Entry object to access the key and value.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public Entry entry() throws IOException {
         checkKey();
@@ -1624,7 +1624,7 @@ public Entry entry() throws IOException {
       /**
        * Get the RecordNum corresponding to the entry pointed by the cursor.
        * @return The RecordNum corresponding to the entry pointed by the cursor.
-       * @throws IOException
+       * @throws IOException raised on errors performing I/O.
        */
       public long getRecordNum() throws IOException {
         return reader.getRecordNumByLocation(currentLocation);
@@ -1670,7 +1670,7 @@ byte[] getKeyBuffer() {
          *          BytesWritable to hold key.
          * @param value
          *          BytesWritable to hold value
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public void get(BytesWritable key, BytesWritable value)
             throws IOException {
@@ -1684,7 +1684,8 @@ public void get(BytesWritable key, BytesWritable value)
          * 
          * @param key
          *          BytesWritable to hold the key.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
+         * @return the key into BytesWritable.
          */
         public int getKey(BytesWritable key) throws IOException {
           key.setSize(getKeyLength());
@@ -1698,8 +1699,9 @@ public int getKey(BytesWritable key) throws IOException {
          * directly uses the buffer inside BytesWritable for storing the value.
          * The call does not require the value length to be known.
          * 
-         * @param value
-         * @throws IOException
+         * @param value value.
+         * @throws IOException raised on errors performing I/O.
+         * @return long value.
          */
         public long getValue(BytesWritable value) throws IOException {
           DataInputStream dis = getValueStream();
@@ -1725,7 +1727,7 @@ public long getValue(BytesWritable value) throws IOException {
          * @param out
          *          The output stream
          * @return the length of the key.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public int writeKey(OutputStream out) throws IOException {
           out.write(keyBuffer, 0, klen);
@@ -1740,7 +1742,7 @@ public int writeKey(OutputStream out) throws IOException {
          * @param out
          *          The output stream
          * @return the length of the value
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public long writeValue(OutputStream out) throws IOException {
           DataInputStream dis = getValueStream();
@@ -1768,7 +1770,7 @@ public long writeValue(OutputStream out) throws IOException {
          *          not be shorter than the key length.
          * @return The length of the key.
          * 
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public int getKey(byte[] buf) throws IOException {
           return getKey(buf, 0);
@@ -1784,7 +1786,7 @@ public int getKey(byte[] buf) throws IOException {
          *          the key into. Requiring the key-length + offset no greater
          *          than the buffer length.
          * @return The length of the key.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public int getKey(byte[] buf, int offset) throws IOException {
           if ((offset | (buf.length - offset - klen)) < 0) {
@@ -1828,10 +1830,11 @@ public int getValueLength() {
          * without moving the cursor will result in exception:
          * {@link #getValue(byte[])}, {@link #getValue(byte[], int)},
          * {@link #getValueStream}.
-         * 
+         *
+         * @param buf buf.
          * @return the length of the value. Does not require
          *         isValueLengthKnown() to be true.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          * 
          */
         public int getValue(byte[] buf) throws IOException {
@@ -1846,10 +1849,12 @@ public int getValue(byte[] buf) throws IOException {
          * functions more than once without moving the cursor will result in
          * exception: {@link #getValue(byte[])}, {@link #getValue(byte[], int)},
          * {@link #getValueStream}.
-         * 
+         *
+         * @param buf buf.
+         * @param offset offset.
          * @return the length of the value. Does not require
          *         isValueLengthKnown() to be true.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public int getValue(byte[] buf, int offset) throws IOException {
           DataInputStream dis = getValueStream();
@@ -1892,7 +1897,7 @@ public int getValue(byte[] buf, int offset) throws IOException {
          * {@link #getValue(byte[], int)}, {@link #getValueStream}.
          * 
          * @return The input stream for reading the value.
-         * @throws IOException
+         * @throws IOException raised on errors performing I/O.
          */
         public DataInputStream getValueStream() throws IOException {
           if (valueChecked == true) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
index 17a27f16b9a4a..714dc5a12acd2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/file/tfile/Utils.java
@@ -49,7 +49,7 @@ private Utils() {
    *          output stream
    * @param n
    *          The integer to be encoded
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @see Utils#writeVLong(DataOutput, long)
    */
   public static void writeVInt(DataOutput out, int n) throws IOException {
@@ -95,7 +95,7 @@ public static void writeVInt(DataOutput out, int n) throws IOException {
    *          output stream
    * @param n
    *          the integer number
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @SuppressWarnings("fallthrough")
   public static void writeVLong(DataOutput out, long n) throws IOException {
@@ -170,7 +170,7 @@ public static void writeVLong(DataOutput out, long n) throws IOException {
    * @param in
    *          input stream
    * @return the decoded integer
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * 
    * @see Utils#readVLong(DataInput)
    */
@@ -199,7 +199,7 @@ public static int readVInt(DataInput in) throws IOException {
    * @param in
    *          input stream
    * @return the decoded long integer.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
 
   public static long readVLong(DataInput in) throws IOException {
@@ -249,9 +249,9 @@ public static long readVLong(DataInput in) throws IOException {
   /**
    * Write a String as a VInt n, followed by n Bytes as in Text format.
    * 
-   * @param out
-   * @param s
-   * @throws IOException
+   * @param out out.
+   * @param s s.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void writeString(DataOutput out, String s) throws IOException {
     if (s != null) {
@@ -271,7 +271,7 @@ public static void writeString(DataOutput out, String s) throws IOException {
    * @param in
    *          The input stream.
    * @return The string
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static String readString(DataInput in) throws IOException {
     int length = readVInt(in);
@@ -299,7 +299,7 @@ public static final class Version implements Comparable<Version> {
      * 
      * @param in
      *          input stream
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public Version(DataInput in) throws IOException {
       major = in.readShort();
@@ -326,7 +326,7 @@ public Version(short major, short minor) {
      * 
      * @param out
      *          The DataOutput object.
-     * @throws IOException
+     * @throws IOException raised on errors performing I/O.
      */
     public void write(DataOutput out) throws IOException {
       out.writeShort(major);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 79b489b3d1a98..5cf820c50ca7c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -141,7 +141,7 @@ public String getMessage() {
       }
     }
 
-    // Denotes the state of supporting PMDK. The value is set by JNI.
+    // Denotes the state of supporting PMDK. The actual value is set via JNI.
     private static SupportState pmdkSupportState =
         SupportState.UNSUPPORTED;
 
@@ -355,7 +355,7 @@ public boolean verifyCanMlock() {
     }
 
     /**
-     * Return true if the JNI-based native IO extensions are available.
+     * @return Return true if the JNI-based native IO extensions are available.
      */
     public static boolean isAvailable() {
       return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
@@ -367,7 +367,14 @@ private static void assertCodeLoaded() throws IOException {
       }
     }
 
-    /** Wrapper around open(2) */
+    /**
+     * Wrapper around open(2) .
+     * @param path input path.
+     * @param flags input flags.
+     * @param mode input mode.
+     * @return FileDescriptor.
+     * @throws IOException raised on errors performing I/O.
+     */
     public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
     /** Wrapper around fstat(2) */
     private static native Stat fstat(FileDescriptor fd) throws IOException;
@@ -428,6 +435,10 @@ static void posixFadviseIfPossible(String identifier,
      * for this syscall for more information. On systems where this
      * call is not available, does nothing.
      *
+     * @param fd input fd.
+     * @param offset input offset.
+     * @param nbytes input nbytes.
+     * @param flags input flag.
      * @throws NativeIOException if there is an error with the syscall
      */
     public static void syncFileRangeIfPossible(
@@ -712,7 +723,14 @@ public static void createDirectoryWithMode(File path, int mode)
     private static native void createDirectoryWithMode0(String path, int mode)
         throws NativeIOException;
 
-    /** Wrapper around CreateFile() on Windows */
+    /**
+     * @return Wrapper around CreateFile() on Windows.
+     * @param path input path.
+     * @param desiredAccess input desiredAccess.
+     * @param shareMode input shareMode.
+     * @param creationDisposition input creationDisposition.
+     * @throws IOException raised on errors performing I/O.
+     */
     public static native FileDescriptor createFile(String path,
         long desiredAccess, long shareMode, long creationDisposition)
         throws IOException;
@@ -749,7 +767,13 @@ private static native FileDescriptor createFileWithMode0(String path,
         long desiredAccess, long shareMode, long creationDisposition, int mode)
         throws NativeIOException;
 
-    /** Wrapper around SetFilePointer() on Windows */
+    /**
+     * @return Wrapper around SetFilePointer() on Windows.
+     * @param fd input fd.
+     * @param distanceToMove input distanceToMove.
+     * @param moveMethod input moveMethod.
+     * @throws IOException raised on errors performing I/O.
+     */
     public static native long setFilePointer(FileDescriptor fd,
         long distanceToMove, long moveMethod) throws IOException;
 
@@ -840,7 +864,7 @@ public static boolean access(String path, AccessRight desiredAccess)
   }
 
   /**
-   * Return true if the JNI-based native IO extensions are available.
+   * @return Return true if the JNI-based native IO extensions are available.
    */
   public static boolean isAvailable() {
     return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
@@ -898,6 +922,7 @@ public CachedUid(String username, long timestamp) {
    *
    * @param name the full principal name containing the domain
    * @return name with domain removed
+   * @throws IOException raised on errors performing I/O.
    */
   private static String stripDomain(String name) {
     int i = name.indexOf('\\');
@@ -933,6 +958,11 @@ public static String getOwner(FileDescriptor fd) throws IOException {
    * file opened at a given offset, i.e. other process can delete
    * the file the FileDescriptor is reading. Only Windows implementation
    * uses the native interface.
+   *
+   * @param f input f.
+   * @param seekOffset input seekOffset.
+   * @return FileDescriptor.
+   * @throws IOException raised on errors performing I/O.
    */
   public static FileDescriptor getShareDeleteFileDescriptor(
       File f, long seekOffset) throws IOException {
@@ -961,7 +991,7 @@ public static FileDescriptor getShareDeleteFileDescriptor(
   }
 
   /**
-   * Create the specified File for write access, ensuring that it does not exist.
+   * @return Create the specified File for write access, ensuring that it does not exist.
    * @param f the file that we want to create
    * @param permissions we want to have on the file (if security is enabled)
    *
@@ -1045,7 +1075,7 @@ public static void renameTo(File src, File dst)
    *
    * @param src source file
    * @param dst hardlink location
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Deprecated
   public static void link(File src, File dst) throws IOException {
@@ -1103,7 +1133,7 @@ private static native void link0(String src, String dst)
    *
    * @param src                  The source path
    * @param dst                  The destination path
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void copyFileUnbuffered(File src, File dst) throws IOException {
     if (nativeLoaded && Shell.WINDOWS) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java
index 06dd0d45b3c2d..60210ccd920c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/AsyncCallHandler.java
@@ -28,7 +28,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.InterruptedIOException;
 import java.lang.reflect.Method;
 import java.util.Iterator;
 import java.util.Queue;
@@ -49,7 +48,11 @@ public class AsyncCallHandler {
   private static final ThreadLocal<AsyncGet<Object, Throwable>>
       ASYNC_RETURN = new ThreadLocal<>();
 
-  /** @return the async return value from {@link AsyncCallHandler}. */
+  /**
+   * @return the async return value from {@link AsyncCallHandler}.
+   * @param <T> T.
+   * @param <R> R.
+   */
   @InterfaceStability.Unstable
   @SuppressWarnings("unchecked")
   public static <R, T extends  Throwable> AsyncGet<R, T> getAsyncReturn() {
@@ -62,7 +65,10 @@ public static <R, T extends  Throwable> AsyncGet<R, T> getAsyncReturn() {
     }
   }
 
-  /** For the lower rpc layers to set the async return value. */
+  /**
+   * For the lower rpc layers to set the async return value.
+   * @param asyncReturn asyncReturn.
+   */
   @InterfaceStability.Unstable
   public static void setLowerLayerAsyncReturn(
       AsyncGet<?, Exception> asyncReturn) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index fcc5975987ebc..9707ee388e1d2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -46,6 +46,10 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
   public static final Logger LOG = LoggerFactory.getLogger(
       RetryInvocationHandler.class);
 
+  @VisibleForTesting
+  public static final ThreadLocal<Boolean> SET_CALL_ID_FOR_TEST =
+      ThreadLocal.withInitial(() -> true);
+
   static class Call {
     private final Method method;
     private final Object[] args;
@@ -159,7 +163,7 @@ CallReturn invoke() throws Throwable {
     }
 
     Object invokeMethod() throws Throwable {
-      if (isRpc) {
+      if (isRpc && SET_CALL_ID_FOR_TEST.get()) {
         Client.setCallIdAndRetryCount(callId, counters.retries,
             retryInvocationHandler.asyncCallHandler);
       }
@@ -387,12 +391,12 @@ private RetryInfo handleException(final Method method, final int callId,
       throw retryInfo.getFailException();
     }
 
-    log(method, retryInfo.isFailover(), counters.failovers, retryInfo.delay, e);
+    log(method, retryInfo.isFailover(), counters.failovers, counters.retries, retryInfo.delay, e);
     return retryInfo;
   }
 
-  private void log(final Method method, final boolean isFailover,
-      final int failovers, final long delay, final Exception ex) {
+  private void log(final Method method, final boolean isFailover, final int failovers,
+      final int retries, final long delay, final Exception ex) {
     boolean info = true;
     // If this is the first failover to this proxy, skip logging at INFO level
     if (!failedAtLeastOnce.contains(proxyDescriptor.getProxyInfo().toString()))
@@ -408,13 +412,15 @@ private void log(final Method method, final boolean isFailover,
     }
 
     final StringBuilder b = new StringBuilder()
-        .append(ex + ", while invoking ")
+        .append(ex)
+        .append(", while invoking ")
         .append(proxyDescriptor.getProxyInfo().getString(method.getName()));
     if (failovers > 0) {
       b.append(" after ").append(failovers).append(" failover attempts");
     }
     b.append(isFailover? ". Trying to failover ": ". Retrying ");
     b.append(delay > 0? "after sleeping for " + delay + "ms.": "immediately.");
+    b.append(" Current retry count: ").append(retries).append(".");
 
     if (info) {
       LOG.info(b.toString());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index 842811edb399a..e14459123016b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -41,7 +41,6 @@
 import org.apache.hadoop.net.ConnectTimeoutException;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
-import org.ietf.jgss.GSSException;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
@@ -75,6 +74,10 @@ public class RetryPolicies {
    * <p>
    * Keep trying forever with a fixed time between attempts.
    * </p>
+   *
+   * @param sleepTime sleepTime.
+   * @param timeUnit timeUnit.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryForeverWithFixedSleep(long sleepTime,
       TimeUnit timeUnit) {
@@ -87,6 +90,11 @@ public static final RetryPolicy retryForeverWithFixedSleep(long sleepTime,
    * Keep trying a limited number of times, waiting a fixed time between attempts,
    * and then fail by re-throwing the exception.
    * </p>
+   *
+   * @param maxRetries maxRetries.
+   * @param sleepTime sleepTime.
+   * @param timeUnit timeUnit.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryUpToMaximumCountWithFixedSleep(int maxRetries, long sleepTime, TimeUnit timeUnit) {
     return new RetryUpToMaximumCountWithFixedSleep(maxRetries, sleepTime, timeUnit);
@@ -97,6 +105,11 @@ public static final RetryPolicy retryUpToMaximumCountWithFixedSleep(int maxRetri
    * Keep trying for a maximum time, waiting a fixed time between attempts,
    * and then fail by re-throwing the exception.
    * </p>
+   *
+   * @param timeUnit timeUnit.
+   * @param sleepTime sleepTime.
+   * @param maxTime maxTime.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryUpToMaximumTimeWithFixedSleep(long maxTime, long sleepTime, TimeUnit timeUnit) {
     return new RetryUpToMaximumTimeWithFixedSleep(maxTime, sleepTime, timeUnit);
@@ -108,6 +121,11 @@ public static final RetryPolicy retryUpToMaximumTimeWithFixedSleep(long maxTime,
    * and then fail by re-throwing the exception.
    * The time between attempts is <code>sleepTime</code> mutliplied by the number of tries so far.
    * </p>
+   *
+   * @param sleepTime sleepTime.
+   * @param maxRetries maxRetries.
+   * @param timeUnit timeUnit.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryUpToMaximumCountWithProportionalSleep(int maxRetries, long sleepTime, TimeUnit timeUnit) {
     return new RetryUpToMaximumCountWithProportionalSleep(maxRetries, sleepTime, timeUnit);
@@ -120,6 +138,12 @@ public static final RetryPolicy retryUpToMaximumCountWithProportionalSleep(int m
    * The time between attempts is <code>sleepTime</code> mutliplied by a random
    * number in the range of [0, 2 to the number of retries)
    * </p>
+   *
+   *
+   * @param timeUnit timeUnit.
+   * @param maxRetries maxRetries.
+   * @param sleepTime sleepTime.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy exponentialBackoffRetry(
       int maxRetries, long sleepTime, TimeUnit timeUnit) {
@@ -130,6 +154,10 @@ public static final RetryPolicy exponentialBackoffRetry(
    * <p>
    * Set a default policy with some explicit handlers for specific exceptions.
    * </p>
+   *
+   * @param exceptionToPolicyMap exceptionToPolicyMap.
+   * @param defaultPolicy defaultPolicy.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryByException(RetryPolicy defaultPolicy,
                                                    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
@@ -141,6 +169,10 @@ public static final RetryPolicy retryByException(RetryPolicy defaultPolicy,
    * A retry policy for RemoteException
    * Set a default policy with some explicit handlers for specific exceptions.
    * </p>
+   *
+   * @param defaultPolicy defaultPolicy.
+   * @param exceptionToPolicyMap exceptionToPolicyMap.
+   * @return RetryPolicy.
    */
   public static final RetryPolicy retryByRemoteException(
       RetryPolicy defaultPolicy,
@@ -149,12 +181,20 @@ public static final RetryPolicy retryByRemoteException(
   }
 
   /**
-   * A retry policy for exceptions other than RemoteException.
+   * <p>
+   * A retry policy where RemoteException and SaslException are not retried, other individual
+   * exception types can have RetryPolicy overrides, and any other exception type without an
+   * override is not retried.
+   * </p>
+   *
+   * @param defaultPolicy defaultPolicy.
+   * @param exceptionToPolicyMap exceptionToPolicyMap.
+   * @return RetryPolicy.
    */
-  public static final RetryPolicy retryOtherThanRemoteException(
+  public static final RetryPolicy retryOtherThanRemoteAndSaslException(
       RetryPolicy defaultPolicy,
       Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
-    return new OtherThanRemoteExceptionDependentRetry(defaultPolicy,
+    return new OtherThanRemoteAndSaslExceptionDependentRetry(defaultPolicy,
         exceptionToPolicyMap);
   }
 
@@ -437,6 +477,7 @@ public String toString() {
      * where t_i and n_i are the i-th pair of sleep time and number of retries.
      * Note that the white spaces in the string are ignored.
      *
+     * @param s input string.
      * @return the parsed object, or null if the parsing fails.
      */
     public static MultipleLinearRandomRetry parseCommaSeparatedString(String s) {
@@ -553,12 +594,12 @@ public RetryAction shouldRetry(Exception e, int retries, int failovers,
     }
   }
 
-  static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy {
+  static class OtherThanRemoteAndSaslExceptionDependentRetry implements RetryPolicy {
 
     private RetryPolicy defaultPolicy;
     private Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;
 
-    public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
+    OtherThanRemoteAndSaslExceptionDependentRetry(RetryPolicy defaultPolicy,
         Map<Class<? extends Exception>,
         RetryPolicy> exceptionToPolicyMap) {
       this.defaultPolicy = defaultPolicy;
@@ -569,10 +610,8 @@ public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
     public RetryAction shouldRetry(Exception e, int retries, int failovers,
         boolean isIdempotentOrAtMostOnce) throws Exception {
       RetryPolicy policy = null;
-      // ignore Remote Exception
-      if (e instanceof RemoteException) {
-        // do nothing
-      } else {
+      // ignore RemoteException and SaslException
+      if (!(e instanceof RemoteException || isSaslFailure(e))) {
         policy = exceptionToPolicyMap.get(e.getClass());
       }
       if (policy == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
index 7fcd5fd4b0080..eaff5bbd528d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryProxy.java
@@ -34,6 +34,7 @@ public class RetryProxy {
    * @param iface the interface that the retry will implement
    * @param implementation the instance whose methods should be retried
    * @param retryPolicy the policy for retrying method call failures
+   * @param <T> T.
    * @return the retry proxy
    */
   public static <T> Object create(Class<T> iface, T implementation,
@@ -51,6 +52,7 @@ public static <T> Object create(Class<T> iface, T implementation,
    * @param iface the interface that the retry will implement
    * @param proxyProvider provides implementation instances whose methods should be retried
    * @param retryPolicy the policy for retrying or failing over method call failures
+   * @param <T> T.
    * @return the retry proxy
    */
   public static <T> Object create(Class<T> iface,
@@ -69,6 +71,7 @@ public static <T> Object create(Class<T> iface,
    * {@link RetryPolicies#TRY_ONCE_THEN_FAIL} is used.
    * 
    * @param iface the interface that the retry will implement
+   * @param <T> T.
    * @param implementation the instance whose methods should be retried
    * @param methodNameToPolicyMap a map of method names to retry policies
    * @return the retry proxy
@@ -90,6 +93,8 @@ public static <T> Object create(Class<T> iface, T implementation,
    * @param iface the interface that the retry will implement
    * @param proxyProvider provides implementation instances whose methods should be retried
    * @param methodNameToPolicyMap map of method names to retry policies
+   * @param defaultPolicy defaultPolicy.
+   * @param <T> T.
    * @return the retry proxy
    */
   public static <T> Object create(Class<T> iface,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
index c035a42d4a751..d2fb070ee2c25 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
@@ -48,7 +48,7 @@ public class RetryUtils {
    *     - non-IOException.
    *     
    *
-   * @param conf
+   * @param conf configuration.
    * @param retryPolicyEnabledKey     conf property key for enabling retry
    * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value 
    * @param retryPolicySpecKey        conf property key for retry policy spec
@@ -168,7 +168,7 @@ public String toString() {
    * Retry policy spec:
    *   N pairs of sleep-time and number-of-retries "s1,n1,s2,n2,..."
    * 
-   * @param conf
+   * @param conf configuration.
    * @param retryPolicyEnabledKey     conf property key for enabling retry
    * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value 
    * @param retryPolicySpecKey        conf property key for retry policy spec
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
index 3c8dfccafa8bb..4bdd60d90c382 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Deserializer.java
@@ -35,13 +35,15 @@
  * other producers may read from the input between calls to
  * {@link #deserialize(Object)}.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public interface Deserializer<T> {
   /**
    * <p>Prepare the deserializer for reading.</p>
+   * @param in input stream.
+   * @throws IOException raised on errors performing I/O.
    */
   void open(InputStream in) throws IOException;
   
@@ -53,12 +55,15 @@ public interface Deserializer<T> {
    * stream. Otherwise, if the object <code>t</code> is null a new
    * deserialized object will be created.
    * </p>
+   * @param t t.
    * @return the deserialized object
+   * @throws IOException raised on errors performing I/O.
    */
   T deserialize(T t) throws IOException;
   
   /**
    * <p>Close the underlying input stream and clear up any resources.</p>
+   * @throws IOException raised on errors performing I/O.
    */
   void close() throws IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/DeserializerComparator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/DeserializerComparator.java
index 05205c5523cc6..29c04f66d4370 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/DeserializerComparator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/DeserializerComparator.java
@@ -37,7 +37,7 @@
  * implementation of {@link RawComparator} that operates directly
  * on byte representations.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/JavaSerializationComparator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/JavaSerializationComparator.java
index f9bf692f1fcc8..d53f7ab75c503 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/JavaSerializationComparator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/JavaSerializationComparator.java
@@ -31,7 +31,7 @@
  * {@link Deserializer} to deserialize objects that are then compared via
  * their {@link Comparable} interfaces.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  * @see JavaSerialization
  */
 @InterfaceAudience.Public
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serialization.java
index 6f2097f7bf9da..0793dc1ca0184 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serialization.java
@@ -25,7 +25,7 @@
  * <p>
  * Encapsulates a {@link Serializer}/{@link Deserializer} pair.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
@@ -34,16 +34,21 @@ public interface Serialization<T> {
   /**
    * Allows clients to test whether this {@link Serialization}
    * supports the given class.
+   *
+   * @param c class.
+   * @return if accept true,not false.
    */
   boolean accept(Class<?> c);
   
   /**
    * @return a {@link Serializer} for the given class.
+   * @param c class.
    */
   Serializer<T> getSerializer(Class<T> c);
 
   /**
    * @return a {@link Deserializer} for the given class.
+   * @param c class.
    */
   Deserializer<T> getDeserializer(Class<T> c);
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
index ce0c3fe398eed..b531ae85233e8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
@@ -52,6 +52,8 @@ public class SerializationFactory extends Configured {
    * property from <code>conf</code>, which is a comma-delimited list of
    * classnames.
    * </p>
+   *
+   * @param conf configuration.
    */
   public SerializationFactory(Configuration conf) {
     super(conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
index 5ada541370ee0..c44b3678fc3be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/Serializer.java
@@ -35,23 +35,28 @@
  * other producers may write to the output between calls to
  * {@link #serialize(Object)}.
  * </p>
- * @param <T>
+ * @param <T> generic type.
  */
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public interface Serializer<T> {
   /**
    * <p>Prepare the serializer for writing.</p>
+   * @param out output stream.
+   * @throws IOException raised on errors performing I/O.
    */
   void open(OutputStream out) throws IOException;
   
   /**
    * <p>Serialize <code>t</code> to the underlying output stream.</p>
+   * @param t t.
+   * @throws IOException raised on errors performing I/O.
    */
   void serialize(T t) throws IOException;
   
   /**
    * <p>Close the underlying output stream and clear up any resources.</p>
+   * @throws IOException raised on errors performing I/O.
    */  
   void close() throws IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
index cfbc60d10452b..544958e682a50 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
@@ -64,7 +64,7 @@ public synchronized boolean accept(Class<?> c) {
 
   private void getPackages() {
     String[] pkgList  = getConf().getStrings(AVRO_REFLECT_PACKAGES);
-    packages = new HashSet<String>();
+    packages = new HashSet<>();
     if (pkgList != null) {
       for (String pkg : pkgList) {
         packages.add(pkg.trim());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
index f340cb3a98a44..2327fd2d55a2e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
@@ -61,18 +61,24 @@ public Serializer<T> getSerializer(Class<T> c) {
 
   /**
    * Return an Avro Schema instance for the given class.
+   * @param t Generics Type T.
+   * @return schema.
    */
   @InterfaceAudience.Private
   public abstract Schema getSchema(T t);
 
   /**
    * Create and return Avro DatumWriter for the given class.
+   * @param clazz clazz.
+   * @return DatumWriter.
    */
   @InterfaceAudience.Private
   public abstract DatumWriter<T> getWriter(Class<T> clazz);
 
   /**
    * Create and return Avro DatumReader for the given class.
+   * @param clazz clazz.
+   * @return DatumReader.
    */
   @InterfaceAudience.Private
   public abstract DatumReader<T> getReader(Class<T> clazz);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java
index fbf825bcb91b1..8d43fd74a843c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java
@@ -46,7 +46,7 @@ public interface AlignmentContext {
   void updateResponseState(RpcResponseHeaderProto.Builder header);
 
   /**
-   * This is the intended client method call to implement to recieve state info
+   * This is the intended client method call to implement to receive state info
    * during RPC response processing.
    *
    * @param header The RPC response header.
@@ -71,7 +71,7 @@ public interface AlignmentContext {
    *        misaligned with the client state.
    *        See implementation for more details.
    * @return state id required for the server to execute the call.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   long receiveRequestState(RpcRequestHeaderProto header, long threshold)
       throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 6cc2540c174d3..fa6f34adaf3bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -405,6 +405,12 @@ public static int[] getDefaultQueueCapacityWeights(int priorityLevels) {
   /**
    * Replaces active queue with the newly requested one and transfers
    * all calls to the newQ before returning.
+   *
+   * @param schedulerClass input schedulerClass.
+   * @param queueClassToUse input queueClassToUse.
+   * @param maxSize input maxSize.
+   * @param ns input ns.
+   * @param conf input configuration.
    */
   public synchronized void swapQueue(
       Class<? extends RpcScheduler> schedulerClass,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 378b83d13b0c7..b6e6f0c57a80d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -43,6 +43,13 @@
 @InterfaceStability.Evolving
 public final class CallerContext {
   public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;
+
+  // field names
+  public static final String CLIENT_IP_STR = "clientIp";
+  public static final String CLIENT_PORT_STR = "clientPort";
+  public static final String CLIENT_ID_STR = "clientId";
+  public static final String CLIENT_CALL_ID_STR = "clientCallId";
+
   /** The caller context.
    *
    * It will be truncated if it exceeds the maximum allowed length in
@@ -116,7 +123,7 @@ public String toString() {
 
   /** The caller context builder. */
   public static final class Builder {
-    private static final String KEY_VALUE_SEPARATOR = ":";
+    public static final String KEY_VALUE_SEPARATOR = ":";
     /**
      * The illegal separators include '\t', '\n', '='.
      * User should not set illegal separator.
@@ -133,12 +140,8 @@ public Builder(String context) {
     }
 
     public Builder(String context, Configuration conf) {
-      if (isValid(context)) {
-        sb.append(context);
-      }
-      fieldSeparator = conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
-          HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
-      checkFieldSeparator(fieldSeparator);
+      this(context, conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+              HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT));
     }
 
     public Builder(String context, String separator) {
@@ -164,8 +167,6 @@ private void checkFieldSeparator(String separator) {
 
     /**
      * Whether the field is valid.
-     * The field should not contain '\t', '\n', '='.
-     * Because the context could be written to audit log.
      * @param field one of the fields in context.
      * @return true if the field is not null or empty.
      */
@@ -228,6 +229,26 @@ public Builder append(String key, String value) {
       return this;
     }
 
+    /**
+     * Append new field which contains key and value to the context
+     * if the key("key:") is absent.
+     * @param key the key of field.
+     * @param value the value of field.
+     * @return the builder.
+     */
+    public Builder appendIfAbsent(String key, String value) {
+      if (sb.toString().contains(key + KEY_VALUE_SEPARATOR)) {
+        return this;
+      }
+      if (isValid(key) && isValid(value)) {
+        if (sb.length() > 0) {
+          sb.append(fieldSeparator);
+        }
+        sb.append(key).append(KEY_VALUE_SEPARATOR).append(value);
+      }
+      return this;
+    }
+
     public CallerContext build() {
       return new CallerContext(this);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 96f925f0f2c0f..c43f922477643 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -18,10 +18,10 @@
 
 package org.apache.hadoop.ipc;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.util.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -61,6 +61,7 @@
 
 import javax.net.SocketFactory;
 import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
 import java.io.*;
 import java.net.*;
 import java.nio.ByteBuffer;
@@ -112,18 +113,39 @@ protected Boolean initialValue() {
     return (AsyncGet<T, IOException>) ASYNC_RPC_RESPONSE.get();
   }
 
-  /** Set call id and retry count for the next call. */
+  /**
+   * Set call id and retry count for the next call.
+   * @param cid input cid.
+   * @param rc input rc.
+   * @param externalHandler input externalHandler.
+   */
   public static void setCallIdAndRetryCount(int cid, int rc,
                                             Object externalHandler) {
     Preconditions.checkArgument(cid != RpcConstants.INVALID_CALL_ID);
     Preconditions.checkState(callId.get() == null);
     Preconditions.checkArgument(rc != RpcConstants.INVALID_RETRY_COUNT);
+    setCallIdAndRetryCountUnprotected(cid, rc, externalHandler);
+  }
 
+  public static void setCallIdAndRetryCountUnprotected(Integer cid, int rc,
+      Object externalHandler) {
     callId.set(cid);
     retryCount.set(rc);
     EXTERNAL_CALL_HANDLER.set(externalHandler);
   }
 
+  public static int getCallId() {
+    return callId.get() != null ? callId.get() : nextCallId();
+  }
+
+  public static int getRetryCount() {
+    return retryCount.get() != null ? retryCount.get() : 0;
+  }
+
+  public static Object getExternalHandler() {
+    return EXTERNAL_CALL_HANDLER.get();
+  }
+
   private final ConcurrentMap<ConnectionId, Connection> connections =
       new ConcurrentHashMap<>();
   private final Object putLock = new Object();
@@ -144,73 +166,6 @@ public static void setCallIdAndRetryCount(int cid, int rc,
   private final int maxAsyncCalls;
   private final AtomicInteger asyncCallCounter = new AtomicInteger(0);
 
-  /**
-   * Executor on which IPC calls' parameters are sent.
-   * Deferring the sending of parameters to a separate
-   * thread isolates them from thread interruptions in the
-   * calling code.
-   */
-  private final ExecutorService sendParamsExecutor;
-  private final static ClientExecutorServiceFactory clientExcecutorFactory =
-      new ClientExecutorServiceFactory();
-
-  private static class ClientExecutorServiceFactory {
-    private int executorRefCount = 0;
-    private ExecutorService clientExecutor = null;
-    
-    /**
-     * Get Executor on which IPC calls' parameters are sent.
-     * If the internal reference counter is zero, this method
-     * creates the instance of Executor. If not, this method
-     * just returns the reference of clientExecutor.
-     * 
-     * @return An ExecutorService instance
-     */
-    synchronized ExecutorService refAndGetInstance() {
-      if (executorRefCount == 0) {
-        clientExecutor = Executors.newCachedThreadPool(
-            new ThreadFactoryBuilder()
-            .setDaemon(true)
-            .setNameFormat("IPC Parameter Sending Thread #%d")
-            .build());
-      }
-      executorRefCount++;
-      
-      return clientExecutor;
-    }
-    
-    /**
-     * Cleanup Executor on which IPC calls' parameters are sent.
-     * If reference counter is zero, this method discards the
-     * instance of the Executor. If not, this method
-     * just decrements the internal reference counter.
-     * 
-     * @return An ExecutorService instance if it exists.
-     *   Null is returned if not.
-     */
-    synchronized ExecutorService unrefAndCleanup() {
-      executorRefCount--;
-      assert(executorRefCount >= 0);
-      
-      if (executorRefCount == 0) {
-        clientExecutor.shutdown();
-        try {
-          if (!clientExecutor.awaitTermination(1, TimeUnit.MINUTES)) {
-            clientExecutor.shutdownNow();
-          }
-        } catch (InterruptedException e) {
-          LOG.warn("Interrupted while waiting for clientExecutor" +
-              " to stop");
-          clientExecutor.shutdownNow();
-          Thread.currentThread().interrupt();
-        }
-        clientExecutor = null;
-      }
-      
-      return clientExecutor;
-    }
-  }
-  
   /**
    * set the ping interval value in configuration
    * 
@@ -279,11 +234,6 @@ public static final void setConnectTimeout(Configuration conf, int timeout) {
     conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY, timeout);
   }
 
-  @VisibleForTesting
-  public static final ExecutorService getClientExecutor() {
-    return Client.clientExcecutorFactory.clientExecutor;
-  }
-
   /**
    * Increment this client's reference count
    */
@@ -413,7 +363,7 @@ public synchronized Writable getRpcResponse() {
    * socket: responses may be delivered out of order. */
   private class Connection extends Thread {
     private InetSocketAddress server;             // server ip:port
-    private final ConnectionId remoteId;                // connection id
+    private final ConnectionId remoteId;          // connection id
     private AuthMethod authMethod; // authentication method
     private AuthProtocol authProtocol;
     private int serviceClass;
@@ -440,8 +390,10 @@ private class Connection extends Thread {
     private AtomicLong lastActivity = new AtomicLong();// last I/O activity time
     private AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
     private IOException closeException; // close reason
-    
-    private final Object sendRpcRequestLock = new Object();
+
+    private final Thread rpcRequestThread;
+    private final SynchronousQueue<Pair<Call, ResponseBuffer>> rpcRequestQueue =
+        new SynchronousQueue<>(true);
 
     private AtomicReference<Thread> connectingThread = new AtomicReference<>();
     private final Consumer<Connection> removeMethod;
@@ -450,6 +402,9 @@ private class Connection extends Thread {
         Consumer<Connection> removeMethod) {
       this.remoteId = remoteId;
       this.server = remoteId.getAddress();
+      this.rpcRequestThread = new Thread(new RpcRequestSender(),
+          "IPC Parameter Sending Thread for " + remoteId);
+      this.rpcRequestThread.setDaemon(true);
 
       this.maxResponseLength = remoteId.conf.getInt(
           CommonConfigurationKeys.IPC_MAXIMUM_RESPONSE_LENGTH,
@@ -639,6 +594,9 @@ private synchronized boolean updateAddress() throws IOException {
         LOG.warn("Address change detected. Old: " + server.toString() +
                                  " New: " + currentAddr.toString());
         server = currentAddr;
+        // Update the remote address so that reconnections are with the updated address.
+        // This avoids thrashing.
+        remoteId.setAddress(currentAddr);
         UserGroupInformation ticket = remoteId.getTicket();
         this.setName("IPC Client (" + socketFactory.hashCode()
             + ") connection to " + server.toString() + " from "
@@ -807,17 +765,18 @@ public Object run() throws IOException, InterruptedException {
      */
     private synchronized void setupIOstreams(
         AtomicBoolean fallbackToSimpleAuth) {
-      if (socket != null || shouldCloseConnection.get()) {
-        return;
-      }
-      UserGroupInformation ticket = remoteId.getTicket();
-      if (ticket != null) {
-        final UserGroupInformation realUser = ticket.getRealUser();
-        if (realUser != null) {
-          ticket = realUser;
-        }
-      }
       try {
+        if (socket != null || shouldCloseConnection.get()) {
+          setFallBackToSimpleAuth(fallbackToSimpleAuth);
+          return;
+        }
+        UserGroupInformation ticket = remoteId.getTicket();
+        if (ticket != null) {
+          final UserGroupInformation realUser = ticket.getRealUser();
+          if (realUser != null) {
+            ticket = realUser;
+          }
+        }
         connectingThread.set(Thread.currentThread());
         if (LOG.isDebugEnabled()) {
           LOG.debug("Connecting to "+server);
@@ -863,20 +822,8 @@ public AuthMethod run()
               remoteId.saslQop =
                   (String)saslRpcClient.getNegotiatedProperty(Sasl.QOP);
               LOG.debug("Negotiated QOP is :" + remoteId.saslQop);
-              if (fallbackToSimpleAuth != null) {
-                fallbackToSimpleAuth.set(false);
-              }
-            } else if (UserGroupInformation.isSecurityEnabled()) {
-              if (!fallbackAllowed) {
-                throw new AccessControlException(
-                    "Server asks us to fall back to SIMPLE " +
-                    "auth, but this client is configured to only allow secure " +
-                    "connections.");
-              }
-              if (fallbackToSimpleAuth != null) {
-                fallbackToSimpleAuth.set(true);
-              }
             }
+            setFallBackToSimpleAuth(fallbackToSimpleAuth);
           }
 
           if (doPing) {
@@ -909,7 +856,41 @@ public AuthMethod run()
         connectingThread.set(null);
       }
     }
-    
+
+    private void setFallBackToSimpleAuth(AtomicBoolean fallbackToSimpleAuth)
+        throws AccessControlException {
+      if (authMethod == null || authProtocol != AuthProtocol.SASL) {
+        if (authProtocol == AuthProtocol.SASL) {
+          LOG.trace("Auth method is not set, yield from setting auth fallback.");
+        }
+        return;
+      }
+      if (fallbackToSimpleAuth == null) {
+        // this should happen only during testing.
+        LOG.trace("Connection {} will skip to set fallbackToSimpleAuth as it is null.", remoteId);
+      } else {
+        if (fallbackToSimpleAuth.get()) {
+          // we already set the value to true, we do not need to examine again.
+          return;
+        }
+      }
+      if (authMethod != AuthMethod.SIMPLE) {
+        if (fallbackToSimpleAuth != null) {
+          LOG.trace("Disabling fallbackToSimpleAuth, target does not use SIMPLE authentication.");
+          fallbackToSimpleAuth.set(false);
+        }
+      } else if (UserGroupInformation.isSecurityEnabled()) {
+        if (!fallbackAllowed) {
+          throw new AccessControlException("Server asks us to fall back to SIMPLE auth, but this "
+              + "client is configured to only allow secure connections.");
+        }
+        if (fallbackToSimpleAuth != null) {
+          LOG.trace("Enabling fallbackToSimpleAuth for target, as we are allowed to fall back.");
+          fallbackToSimpleAuth.set(true);
+        }
+      }
+    }
+
     private void closeConnection() {
       if (socket == null) {
         return;
@@ -1102,6 +1083,10 @@ private synchronized void sendPing() throws IOException {
 
     @Override
     public void run() {
+      // Don't start the ipc parameter sending thread until we start this
+      // thread, because the shutdown logic only gets triggered if this
+      // thread is started.
+      rpcRequestThread.start();
       if (LOG.isDebugEnabled())
         LOG.debug(getName() + ": starting, having connections " 
             + connections.size());
@@ -1125,9 +1110,52 @@ public void run() {
             + connections.size());
     }
 
+    /**
+     * A thread to write rpc requests to the socket.
+     */
+    private class RpcRequestSender implements Runnable {
+      @Override
+      public void run() {
+        while (!shouldCloseConnection.get()) {
+          ResponseBuffer buf = null;
+          try {
+            Pair<Call, ResponseBuffer> pair =
+                rpcRequestQueue.poll(maxIdleTime, TimeUnit.MILLISECONDS);
+            if (pair == null || shouldCloseConnection.get()) {
+              continue;
+            }
+            buf = pair.getRight();
+            synchronized (ipcStreams.out) {
+              if (LOG.isDebugEnabled()) {
+                Call call = pair.getLeft();
+                LOG.debug(getName() + "{} sending #{} {}", getName(), call.id,
+                    call.rpcRequest);
+              }
+              // RpcRequestHeader + RpcRequest
+              ipcStreams.sendRequest(buf.toByteArray());
+              ipcStreams.flush();
+            }
+          } catch (InterruptedException ie) {
+            // stop this thread
+            return;
+          } catch (IOException e) {
+            // exception at this point would leave the connection in an
+            // unrecoverable state (eg half a call left on the wire).
+            // So, close the connection, killing any outstanding calls
+            markClosed(e);
+          } finally {
+            //the buffer is just an in-memory buffer, but it is still polite to
+            // close early
+            IOUtils.closeStream(buf);
+          }
+        }
+      }
+    }
+
     /** Initiates a rpc call by sending the rpc request to the remote server.
-     * Note: this is not called from the Connection thread, but by other
-     * threads.
+     * Note: this is not called from the current thread, but by another
+     * thread, so that if the current thread is interrupted that the socket
+     * state isn't corrupted with a partially written message.
      * @param call - the rpc request
      */
     public void sendRpcRequest(final Call call)
@@ -1137,8 +1165,7 @@ public void sendRpcRequest(final Call call)
       }
 
       // Serialize the call to be sent. This is done from the actual
-      // caller thread, rather than the sendParamsExecutor thread,
-      
+      // caller thread, rather than the rpcRequestThread in the connection,
       // so that if the serialization throws an error, it is reported
       // properly. This also parallelizes the serialization.
       //
@@ -1155,51 +1182,7 @@ public void sendRpcRequest(final Call call)
       final ResponseBuffer buf = new ResponseBuffer();
       header.writeDelimitedTo(buf);
       RpcWritable.wrap(call.rpcRequest).writeTo(buf);
-
-      synchronized (sendRpcRequestLock) {
-        Future<?> senderFuture = sendParamsExecutor.submit(new Runnable() {
-          @Override
-          public void run() {
-            try {
-              synchronized (ipcStreams.out) {
-                if (shouldCloseConnection.get()) {
-                  return;
-                }
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug(getName() + " sending #" + call.id
-                      + " " + call.rpcRequest);
-                }
-                // RpcRequestHeader + RpcRequest
-                ipcStreams.sendRequest(buf.toByteArray());
-                ipcStreams.flush();
-              }
-            } catch (IOException e) {
-              // exception at this point would leave the connection in an
-              // unrecoverable state (eg half a call left on the wire).
-              // So, close the connection, killing any outstanding calls
-              markClosed(e);
-            } finally {
-              //the buffer is just an in-memory buffer, but it is still polite to
-              // close early
-              IOUtils.closeStream(buf);
-            }
-          }
-        });
-      
-        try {
-          senderFuture.get();
-        } catch (ExecutionException e) {
-          Throwable cause = e.getCause();
-          
-          // cause should only be a RuntimeException as the Runnable above
-          // catches IOException
-          if (cause instanceof RuntimeException) {
-            throw (RuntimeException) cause;
-          } else {
-            throw new RuntimeException("unexpected checked exception", cause);
-          }
-        }
-      }
+      rpcRequestQueue.put(Pair.of(call, buf));
     }
 
     /* Receive a response.
@@ -1326,8 +1309,14 @@ private void cleanupCalls() {
     }
   }
 
-  /** Construct an IPC client whose values are of the given {@link Writable}
-   * class. */
+  /**
+   * Construct an IPC client whose values are of the given {@link Writable}
+   * class.
+   *
+   * @param valueClass input valueClass.
+   * @param conf input configuration.
+   * @param factory input factory.
+   */
   public Client(Class<? extends Writable> valueClass, Configuration conf, 
       SocketFactory factory) {
     this.valueClass = valueClass;
@@ -1342,16 +1331,15 @@ public Client(Class<? extends Writable> valueClass, Configuration conf,
             CommonConfigurationKeys.IPC_CLIENT_BIND_WILDCARD_ADDR_DEFAULT);
 
     this.clientId = ClientId.getClientId();
-    this.sendParamsExecutor = clientExcecutorFactory.refAndGetInstance();
     this.maxAsyncCalls = conf.getInt(
         CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_KEY,
         CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_DEFAULT);
   }
 
   /**
-   * Construct an IPC client with the default SocketFactory
-   * @param valueClass
-   * @param conf
+   * Construct an IPC client with the default SocketFactory.
+   * @param valueClass input valueClass.
+   * @param conf input Configuration.
    */
   public Client(Class<? extends Writable> valueClass, Configuration conf) {
     this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
@@ -1386,6 +1374,7 @@ public void stop() {
     // wake up all connections
     for (Connection conn : connections.values()) {
       conn.interrupt();
+      conn.rpcRequestThread.interrupt();
       conn.interruptConnectingThread();
     }
     
@@ -1402,14 +1391,13 @@ public void stop() {
         }
       }
     }
-    clientExcecutorFactory.unrefAndCleanup();
   }
 
   /** 
    * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
    * <code>remoteId</code>, returning the rpc respond.
    *
-   * @param rpcKind
+   * @param rpcKind - input rpcKind.
    * @param rpcRequest -  contains serialized method and method parameters
    * @param remoteId - the target rpc server
    * @param fallbackToSimpleAuth - set to true or false during this method to
@@ -1417,6 +1405,7 @@ public void stop() {
    * @return the rpc response
    * Throws exceptions if there are network problems or if the remote code
    * threw an exception.
+   * @throws IOException raised on errors performing I/O.
    */
   public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
       ConnectionId remoteId, AtomicBoolean fallbackToSimpleAuth)
@@ -1585,7 +1574,8 @@ private Writable getRpcResponse(final Call call, final Connection connection,
       }
 
       if (call.error != null) {
-        if (call.error instanceof RemoteException) {
+        if (call.error instanceof RemoteException ||
+            call.error instanceof SaslException) {
           call.error.fillInStackTrace();
           throw call.error;
         } else { // local exception
@@ -1663,9 +1653,9 @@ private Connection getConnection(ConnectionId remoteId,
   @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
   @InterfaceStability.Evolving
   public static class ConnectionId {
-    InetSocketAddress address;
-    UserGroupInformation ticket;
-    final Class<?> protocol;
+    private InetSocketAddress address;
+    private final UserGroupInformation ticket;
+    private final Class<?> protocol;
     private static final int PRIME = 16777619;
     private final int rpcTimeout;
     private final int maxIdleTime; //connections will be culled if it was idle for 
@@ -1680,8 +1670,8 @@ public static class ConnectionId {
     private final int pingInterval; // how often sends ping to the server in msecs
     private String saslQop; // here for testing
     private final Configuration conf; // used to get the expected kerberos principal name
-    
-    ConnectionId(InetSocketAddress address, Class<?> protocol, 
+
+    public ConnectionId(InetSocketAddress address, Class<?> protocol,
                  UserGroupInformation ticket, int rpcTimeout,
                  RetryPolicy connectionRetryPolicy, Configuration conf) {
       this.protocol = protocol;
@@ -1716,7 +1706,28 @@ public static class ConnectionId {
     InetSocketAddress getAddress() {
       return address;
     }
-    
+
+    /**
+     * This is used to update the remote address when an address change is detected.  This method
+     * ensures that the {@link #hashCode()} won't change.
+     *
+     * @param address the updated address
+     * @throws IllegalArgumentException if the hostname or port doesn't match
+     * @see Connection#updateAddress()
+     */
+    void setAddress(InetSocketAddress address) {
+      if (!Objects.equals(this.address.getHostName(), address.getHostName())) {
+        throw new IllegalArgumentException("Hostname must match: " + this.address + " vs "
+            + address);
+      }
+      if (this.address.getPort() != address.getPort()) {
+        throw new IllegalArgumentException("Port must match: " + this.address + " vs " + address);
+      }
+
+      this.address = address;
+    }
+
+
     Class<?> getProtocol() {
       return protocol;
     }
@@ -1725,7 +1736,7 @@ UserGroupInformation getTicket() {
       return ticket;
     }
     
-    private int getRpcTimeout() {
+    int getRpcTimeout() {
       return rpcTimeout;
     }
     
@@ -1737,7 +1748,7 @@ public int getMaxRetriesOnSasl() {
       return maxRetriesOnSasl;
     }
 
-    /** max connection retries on socket time outs */
+    /** @return max connection retries on socket time outs */
     public int getMaxRetriesOnSocketTimeouts() {
       return maxRetriesOnSocketTimeouts;
     }
@@ -1759,6 +1770,10 @@ boolean getDoPing() {
     int getPingInterval() {
       return pingInterval;
     }
+
+    RetryPolicy getRetryPolicy() {
+      return connectionRetryPolicy;
+    }
     
     @VisibleForTesting
     String getSaslQop() {
@@ -1823,7 +1838,11 @@ && isEqual(this.protocol, that.protocol)
     @Override
     public int hashCode() {
       int result = connectionRetryPolicy.hashCode();
-      result = PRIME * result + ((address == null) ? 0 : address.hashCode());
+      // We calculate based on the host name and port without the IP address, since the hashCode
+      // must be stable even if the IP address is updated.
+      result = PRIME * result + ((address == null || address.getHostName() == null) ? 0 :
+          address.getHostName().hashCode());
+      result = PRIME * result + ((address == null) ? 0 : address.getPort());
       result = PRIME * result + (doPing ? 1231 : 1237);
       result = PRIME * result + maxIdleTime;
       result = PRIME * result + pingInterval;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
index b7257c8b2a69c..c5d0183dba25d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientCache.java
@@ -93,6 +93,8 @@ public synchronized Client getClient(Configuration conf, SocketFactory factory)
   /**
    * Stop a RPC client connection 
    * A RPC client is closed only when its reference count becomes zero.
+   *
+   * @param client input client.
    */
   public void stopClient(Client client) {
     if (Client.LOG.isDebugEnabled()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java
index 152e062392fcc..bab1de753f265 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ClientId.java
@@ -35,7 +35,7 @@ public class ClientId {
   private static final int shiftWidth = 8;
   
   /**
-   * Return clientId as byte[]
+   * @return Return clientId as byte[].
    */
   public static byte[] getClientId() {
     UUID uuid = UUID.randomUUID();
@@ -45,7 +45,10 @@ public static byte[] getClientId() {
     return buf.array();
   }
   
-  /** Convert a clientId byte[] to string */
+  /**
+   * @return Convert a clientId byte[] to string.
+   * @param clientId input clientId.
+   */
   public static String toString(byte[] clientId) {
     // clientId can be null or an empty array
     if (clientId == null || clientId.length == 0) {
@@ -74,7 +77,10 @@ public static long getLsb(byte[] clientId) {
     return lsb;
   }
   
-  /** Convert from clientId string byte[] representation of clientId */
+  /**
+   * @return Convert from clientId string byte[] representation of clientId.
+   * @param id input id.
+   */
   public static byte[] toBytes(String id) {
     if (id == null || "".equals(id)) {
       return new byte[0];
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/GenericRefreshProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/GenericRefreshProtocol.java
index bfa055bcb0997..10e661a3095cd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/GenericRefreshProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/GenericRefreshProtocol.java
@@ -41,7 +41,11 @@ public interface GenericRefreshProtocol {
 
   /**
    * Refresh the resource based on identity passed in.
-   * @throws IOException
+   *
+   * @param identifier input identifier.
+   * @param args input args.
+   * @throws IOException raised on errors performing I/O.
+   * @return Collection RefreshResponse.
    */
   @Idempotent
   Collection<RefreshResponse> refresh(String identifier, String[] args)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
index 1e110b9011313..9ed0640c8dcfa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
@@ -85,7 +85,7 @@ public static IOException getRemoteException(
   /**
    * Get the ByteString for frequently used fixed and small set strings.
    * @param key string
-   * @return
+   * @return the ByteString for frequently used fixed and small set strings.
    */
   public static ByteString getFixedByteString(Text key) {
     ByteString value = FIXED_BYTESTRING_CACHE.get(key);
@@ -99,7 +99,7 @@ public static ByteString getFixedByteString(Text key) {
   /**
    * Get the ByteString for frequently used fixed and small set strings.
    * @param key string
-   * @return
+   * @return ByteString for frequently used fixed and small set strings.
    */
   public static ByteString getFixedByteString(String key) {
     ByteString value = FIXED_BYTESTRING_CACHE.get(key);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index c4457a653e35f..df0f734d08016 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -77,6 +77,16 @@ public static AsyncGet<Message, Exception> getAsyncReturnMessage() {
     return ASYNC_RETURN_MESSAGE.get();
   }
 
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+      ConnectionId connId, Configuration conf, SocketFactory factory,
+      AlignmentContext alignmentContext) throws IOException {
+    final Invoker invoker = new Invoker(protocol, connId, conf, factory, alignmentContext);
+    return new ProtocolProxy<T>(protocol, (T) Proxy.newProxyInstance(
+        protocol.getClassLoader(), new Class[] {protocol}, invoker), false);
+  }
+
   public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
       InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
       SocketFactory factory, int rpcTimeout) throws IOException {
@@ -116,7 +126,7 @@ public ProtocolProxy<ProtocolMetaInfoPB> getProtocolMetaInfoProxy(
     return new ProtocolProxy<ProtocolMetaInfoPB>(protocol,
         (ProtocolMetaInfoPB) Proxy.newProxyInstance(protocol.getClassLoader(),
             new Class[] { protocol }, new Invoker(protocol, connId, conf,
-                factory)), false);
+                factory, null)), false);
   }
 
   protected static class Invoker implements RpcInvocationHandler {
@@ -137,21 +147,26 @@ protected Invoker(Class<?> protocol, InetSocketAddress addr,
         throws IOException {
       this(protocol, Client.ConnectionId.getConnectionId(
           addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf),
-          conf, factory);
+          conf, factory, alignmentContext);
       this.fallbackToSimpleAuth = fallbackToSimpleAuth;
-      this.alignmentContext = alignmentContext;
     }
     
     /**
      * This constructor takes a connectionId, instead of creating a new one.
+     * @param protocol input protocol.
+     * @param connId input connId.
+     * @param conf input Configuration.
+     * @param factory input factory.
+     * @param alignmentContext Alignment context
      */
     protected Invoker(Class<?> protocol, Client.ConnectionId connId,
-        Configuration conf, SocketFactory factory) {
+        Configuration conf, SocketFactory factory, AlignmentContext alignmentContext) {
       this.remoteId = connId;
       this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class);
       this.protocolName = RPC.getProtocolName(protocol);
       this.clientProtocolVersion = RPC
           .getProtocolVersion(protocol);
+      this.alignmentContext = alignmentContext;
     }
 
     private RequestHeaderProto constructRpcRequestHeader(Method method) {
@@ -423,6 +438,10 @@ public static ProtobufRpcEngineCallback registerForDeferredResponse() {
      * @param portRangeConfig A config parameter that can be used to restrict
      * the range of ports used when port is 0 (an ephemeral port)
      * @param alignmentContext provides server state info on client responses
+     * @param secretManager input secretManager.
+     * @param queueSizePerHandler input queueSizePerHandler.
+     * @param numReaders input numReaders.
+     * @throws IOException raised on errors performing I/O.
      */
     public Server(Class<?> protocolClass, Object protocolImpl,
         Configuration conf, String bindAddress, int port, int numHandlers,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java
index e1ee374282897..bedecc8851d6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java
@@ -100,6 +100,16 @@ public <T> ProtocolProxy<T> getProxy(
       rpcTimeout, connectionRetryPolicy, null, null);
   }
 
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+      ConnectionId connId, Configuration conf, SocketFactory factory,
+      AlignmentContext alignmentContext) throws IOException {
+    final Invoker invoker = new Invoker(protocol, connId, conf, factory, alignmentContext);
+    return new ProtocolProxy<T>(protocol, (T) Proxy.newProxyInstance(
+        protocol.getClassLoader(), new Class[] {protocol}, invoker), false);
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
@@ -123,7 +133,7 @@ public ProtocolProxy<ProtocolMetaInfoPB> getProtocolMetaInfoProxy(
     return new ProtocolProxy<ProtocolMetaInfoPB>(protocol,
         (ProtocolMetaInfoPB) Proxy.newProxyInstance(protocol.getClassLoader(),
             new Class[]{protocol}, new Invoker(protocol, connId, conf,
-                factory)), false);
+                factory, null)), false);
   }
 
   protected static class Invoker implements RpcInvocationHandler {
@@ -144,21 +154,27 @@ protected Invoker(Class<?> protocol, InetSocketAddress addr,
         throws IOException {
       this(protocol, Client.ConnectionId.getConnectionId(
           addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf),
-          conf, factory);
+          conf, factory, alignmentContext);
       this.fallbackToSimpleAuth = fallbackToSimpleAuth;
-      this.alignmentContext = alignmentContext;
     }
 
     /**
      * This constructor takes a connectionId, instead of creating a new one.
+     *
+     * @param protocol input protocol.
+     * @param connId input connId.
+     * @param conf input Configuration.
+     * @param factory input factory.
+     * @param alignmentContext Alignment context
      */
     protected Invoker(Class<?> protocol, Client.ConnectionId connId,
-        Configuration conf, SocketFactory factory) {
+        Configuration conf, SocketFactory factory, AlignmentContext alignmentContext) {
       this.remoteId = connId;
       this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class);
       this.protocolName = RPC.getProtocolName(protocol);
       this.clientProtocolVersion = RPC
           .getProtocolVersion(protocol);
+      this.alignmentContext = alignmentContext;
     }
 
     private RequestHeaderProto constructRpcRequestHeader(Method method) {
@@ -458,6 +474,7 @@ public static ProtobufRpcEngineCallback2 registerForDeferredResponse2() {
      * @param portRangeConfig A config parameter that can be used to restrict
      * the range of ports used when port is 0 (an ephemeral port)
      * @param alignmentContext provides server state info on client responses
+     * @throws IOException raised on errors performing I/O.
      */
     public Server(Class<?> protocolClass, Object protocolImpl,
         Configuration conf, String bindAddress, int port, int numHandlers,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInterface.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInterface.java
index 29c07ac29cb87..f23c05936a356 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInterface.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInterface.java
@@ -36,7 +36,7 @@ public interface ProtocolMetaInterface {
    * It is assumed that all method names are unique for a protocol.
    * @param methodName The name of the method
    * @return true if method is supported, otherwise false.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public boolean isMethodSupported(String methodName) throws IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolProxy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolProxy.java
index cc66958d14e08..f5f212b29276d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolProxy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolProxy.java
@@ -69,7 +69,7 @@ private void fetchServerMethods(Method method) throws IOException {
     }
     int[] serverMethodsCodes = serverInfo.getMethods();
     if (serverMethodsCodes != null) {
-      serverMethods = new HashSet<Integer>(serverMethodsCodes.length);
+      serverMethods = new HashSet<>(serverMethodsCodes.length);
       for (int m : serverMethodsCodes) {
         this.serverMethods.add(Integer.valueOf(m));
       }
@@ -85,11 +85,12 @@ public T getProxy() {
   }
   
   /**
-   * Check if a method is supported by the server or not
+   * Check if a method is supported by the server or not.
    * 
    * @param methodName a method's name in String format
    * @param parameterTypes a method's parameter types
    * @return true if the method is supported by the server
+   * @throws IOException raised on errors performing I/O.
    */
   public synchronized boolean isMethodSupported(String methodName,
                                    Class<?>... parameterTypes)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 3bbd82d153a09..fc562b525ad6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -150,6 +150,9 @@ static Class<?>[] getProtocolInterfaces(Class<?> protocol) {
    * Get the protocol name.
    *  If the protocol class has a ProtocolAnnotation, then get the protocol
    *  name from the annotation; otherwise the class name is the protocol name.
+   *
+   * @param protocol input protocol.
+   * @return protocol name.
    */
   static public String getProtocolName(Class<?> protocol) {
     if (protocol == null) {
@@ -164,6 +167,9 @@ static public String getProtocolName(Class<?> protocol) {
    * If the protocol class has a ProtocolAnnotation,
    * then get the protocol version from the annotation;
    * otherwise get it from the versionID field of the protocol class.
+   *
+   * @param protocol input protocol.
+   * @return ProtocolVersion.
    */
   static public long getProtocolVersion(Class<?> protocol) {
     if (protocol == null) {
@@ -258,14 +264,14 @@ public String getInterfaceName() {
     }
     
     /**
-     * Get the client's preferred version
+     * @return Get the client's preferred version.
      */
     public long getClientVersion() {
       return clientVersion;
     }
     
     /**
-     * Get the server's agreed to version.
+     * @return Get the server's agreed to version.
      */
     public long getServerVersion() {
       return serverVersion;
@@ -286,8 +292,9 @@ public RpcErrorCodeProto getRpcErrorCodeProto() {
   }
 
   /**
-   * Get a proxy connection to a remote server
-   * 
+   * Get a proxy connection to a remote server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -306,8 +313,9 @@ public static <T> T waitForProxy(
 
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
-   * and a set of methods that are supported by the server
-   * 
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -324,8 +332,9 @@ public static <T> ProtocolProxy<T> waitForProtocolProxy(Class<T> protocol,
   }
 
   /**
-   * Get a proxy connection to a remote server
-   * 
+   * Get a proxy connection to a remote server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -344,7 +353,8 @@ public static <T> T waitForProxy(Class<T> protocol, long clientVersion,
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
    * and a set of methods that are supported by the server
-   * 
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -362,8 +372,9 @@ public static <T> ProtocolProxy<T> waitForProtocolProxy(Class<T> protocol,
   }
   
   /**
-   * Get a proxy connection to a remote server
-   * 
+   * Get a proxy connection to a remote server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -384,16 +395,18 @@ public static <T> T waitForProxy(Class<T> protocol,
 
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
-   * and a set of methods that are supported by the server
-   * 
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
    * @param conf configuration to use
    * @param rpcTimeout timeout for each RPC
+   * @param connectionRetryPolicy input connectionRetryPolicy.
    * @param timeout time in milliseconds before giving up
    * @return the proxy
-   * @throws IOException if the far end through a RemoteException
+   * @throws IOException if the far end through a RemoteException.
    */
   public static <T> ProtocolProxy<T> waitForProtocolProxy(Class<T> protocol,
                                long clientVersion,
@@ -439,9 +452,18 @@ public static <T> ProtocolProxy<T> waitForProtocolProxy(Class<T> protocol,
     }
   }
 
-  /** Construct a client-side proxy object that implements the named protocol,
+  /**
+   * Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address. 
-   * @param <T>*/
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param conf input Configuration.
+   * @param factory input factory.
+   * @throws IOException raised on errors performing I/O.
+   * @return proxy.
+   */
   public static <T> T getProxy(Class<T> protocol,
                                 long clientVersion,
                                 InetSocketAddress addr, Configuration conf,
@@ -452,8 +474,9 @@ public static <T> T getProxy(Class<T> protocol,
 
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
-   * and a set of methods that are supported by the server
-   * 
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -470,9 +493,21 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
     return getProtocolProxy(protocol, clientVersion, addr, ugi, conf, factory);
   }
   
-  /** Construct a client-side proxy object that implements the named protocol,
+  /**
+   * Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address. 
-   * @param <T>*/
+   *
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param ticket input tocket.
+   * @param conf input conf.
+   * @param factory input factory.
+   * @return the protocol proxy.
+   * @throws IOException raised on errors performing I/O.
+   *
+   */
   public static <T> T getProxy(Class<T> protocol,
                                 long clientVersion,
                                 InetSocketAddress addr,
@@ -486,7 +521,8 @@ public static <T> T getProxy(Class<T> protocol,
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
    * and a set of methods that are supported by the server
-   * 
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol class
    * @param clientVersion client version
    * @param addr remote address
@@ -505,12 +541,56 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
     return getProtocolProxy(protocol, clientVersion, addr, ticket, conf,
         factory, getRpcTimeout(conf), null);
   }
+
+  /**
+   * Get a protocol proxy that contains a proxy connection to a remote server
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type T
+   * @param protocol protocol class
+   * @param clientVersion client's version
+   * @param connId client connection identifier
+   * @param conf configuration
+   * @param factory socket factory
+   * @return the protocol proxy
+   * @throws IOException if the far end through a RemoteException
+   */
+  public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
+      long clientVersion, ConnectionId connId, Configuration conf,
+      SocketFactory factory) throws IOException {
+    return getProtocolProxy(protocol, clientVersion, connId, conf,
+        factory, null);
+  }
+
+  /**
+   * Get a protocol proxy that contains a proxy connection to a remote server
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type T
+   * @param protocol protocol class
+   * @param clientVersion client's version
+   * @param connId client connection identifier
+   * @param conf configuration
+   * @param factory socket factory
+   * @param alignmentContext StateID alignment context
+   * @return the protocol proxy
+   * @throws IOException if the far end through a RemoteException
+   */
+  public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
+      long clientVersion, ConnectionId connId, Configuration conf,
+      SocketFactory factory, AlignmentContext alignmentContext) throws IOException {
+    if (UserGroupInformation.isSecurityEnabled()) {
+      SaslRpcServer.init(conf);
+    }
+    return getProtocolEngine(protocol, conf).getProxy(
+        protocol, clientVersion, connId, conf, factory, alignmentContext);
+  }
   
   /**
    * Construct a client-side proxy that implements the named protocol,
    * talking to a server at the named address.
-   * @param <T>
-   * 
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol
    * @param clientVersion client's version
    * @param addr server address
@@ -534,8 +614,9 @@ public static <T> T getProxy(Class<T> protocol,
   
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
-   * and a set of methods that are supported by the server
-   * 
+   * and a set of methods that are supported by the server.
+   *
+   * @param <T> Generics Type T.
    * @param protocol protocol
    * @param clientVersion client's version
    * @param addr server address
@@ -561,8 +642,9 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
 
   /**
    * Get a protocol proxy that contains a proxy connection to a remote server
-   * and a set of methods that are supported by the server
+   * and a set of methods that are supported by the server.
    *
+   * @param <T> Generics Type T.
    * @param protocol protocol
    * @param clientVersion client's version
    * @param addr server address
@@ -609,6 +691,7 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
    * @param fallbackToSimpleAuth set to true or false during calls to indicate
    *   if a secure client falls back to simple auth
    * @param alignmentContext state alignment context
+   * @param <T> Generics Type T.
    * @return the proxy
    * @throws IOException if any error occurs
    */
@@ -632,15 +715,15 @@ public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
   }
 
    /**
-    * Construct a client-side proxy object with the default SocketFactory
-    * @param <T>
-    * 
-    * @param protocol
-    * @param clientVersion
-    * @param addr
-    * @param conf
+    * Construct a client-side proxy object with the default SocketFactory.
+    *
+    * @param <T> Generics Type T.
+    * @param protocol input protocol.
+    * @param clientVersion input clientVersion.
+    * @param addr input addr.
+    * @param conf input Configuration.
     * @return a proxy instance
-    * @throws IOException
+    * @throws IOException  if the thread is interrupted.
     */
    public static <T> T getProxy(Class<T> protocol,
                                  long clientVersion,
@@ -651,7 +734,8 @@ public static <T> T getProxy(Class<T> protocol,
    }
   
   /**
-   * Returns the server address for a given proxy.
+   * @return Returns the server address for a given proxy.
+   * @param proxy input proxy.
    */
   public static InetSocketAddress getServerAddress(Object proxy) {
     return getConnectionIdForProxy(proxy).getAddress();
@@ -678,12 +762,13 @@ public static ConnectionId getConnectionIdForProxy(Object proxy) {
    * Get a protocol proxy that contains a proxy connection to a remote server
    * and a set of methods that are supported by the server
    * 
-   * @param protocol
-   * @param clientVersion
-   * @param addr
-   * @param conf
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param conf input configuration.
+   * @param <T> Generics Type T.
    * @return a protocol proxy
-   * @throws IOException
+   * @throws IOException if the thread is interrupted.
    */
   public static <T> ProtocolProxy<T> getProtocolProxy(Class<T> protocol,
                                 long clientVersion,
@@ -767,75 +852,109 @@ public Builder(Configuration conf) {
       this.conf = conf;
     }
 
-    /** Mandatory field */
+    /**
+     * @return Mandatory field.
+     * @param protocol input protocol.
+     */
     public Builder setProtocol(Class<?> protocol) {
       this.protocol = protocol;
       return this;
     }
     
-    /** Mandatory field */
+    /**
+     * @return Mandatory field.
+     * @param instance input instance.
+     */
     public Builder setInstance(Object instance) {
       this.instance = instance;
       return this;
     }
     
-    /** Default: 0.0.0.0 */
+    /**
+     * @return Default: 0.0.0.0.
+     * @param bindAddress input bindAddress.
+     */
     public Builder setBindAddress(String bindAddress) {
       this.bindAddress = bindAddress;
       return this;
     }
     
-    /** Default: 0 */
+    /**
+     * @return Default: 0.
+     * @param port input port.
+     */
     public Builder setPort(int port) {
       this.port = port;
       return this;
     }
     
-    /** Default: 1 */
+    /**
+     * @return Default: 1.
+     * @param numHandlers input numHandlers.
+     */
     public Builder setNumHandlers(int numHandlers) {
       this.numHandlers = numHandlers;
       return this;
     }
     
-    /** Default: -1 */
+    /**
+     * @return Default: -1.
+     * @param numReaders input numReaders.
+     */
     public Builder setnumReaders(int numReaders) {
       this.numReaders = numReaders;
       return this;
     }
     
-    /** Default: -1 */
+    /**
+     * @return Default: -1.
+     * @param queueSizePerHandler
+     *        input queueSizePerHandler.
+     */
     public Builder setQueueSizePerHandler(int queueSizePerHandler) {
       this.queueSizePerHandler = queueSizePerHandler;
       return this;
     }
     
-    /** Default: false */
+    /**
+     * @return Default: false.
+     * @param verbose input verbose.
+     */
     public Builder setVerbose(boolean verbose) {
       this.verbose = verbose;
       return this;
     }
     
-    /** Default: null */
+    /**
+     * @return Default: null.
+     * @param secretManager input secretManager.
+     */
     public Builder setSecretManager(
         SecretManager<? extends TokenIdentifier> secretManager) {
       this.secretManager = secretManager;
       return this;
     }
     
-    /** Default: null */
+    /**
+     * @return Default: null.
+     * @param portRangeConfig input portRangeConfig.
+     */
     public Builder setPortRangeConfig(String portRangeConfig) {
       this.portRangeConfig = portRangeConfig;
       return this;
     }
     
-    /** Default: null */
+    /**
+     * @return Default: null.
+     * @param alignmentContext input alignmentContext.
+     */
     public Builder setAlignmentContext(AlignmentContext alignmentContext) {
       this.alignmentContext = alignmentContext;
       return this;
     }
 
     /**
-     * Build the RPC Server. 
+     * @return Build the RPC Server.
      * @throws IOException on error
      * @throws HadoopIllegalArgumentException when mandatory fields are not set
      */
@@ -1077,6 +1196,7 @@ private void initProtocolMetaInfo(Configuration conf) {
     
     /**
      * Add a protocol to the existing server.
+     * @param rpcKind - input rpcKind
      * @param protocolClass - the protocol class
      * @param protocolImpl - the impl of the protocol that will be called
      * @return the server (for convenience)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshCallQueueProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshCallQueueProtocol.java
index 553f9a00d4cca..b1aa0197040a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshCallQueueProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshCallQueueProtocol.java
@@ -41,7 +41,7 @@ public interface RefreshCallQueueProtocol {
 
   /**
    * Refresh the callqueue.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Idempotent
   void refreshCallQueue() throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java
index 95f1323410fe0..0cc0b8ba3d8b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RefreshRegistry.java
@@ -72,6 +72,7 @@ public synchronized void register(String identifier, RefreshHandler handler) {
   /**
    * Remove the registered object for a given identity.
    * @param identifier the resource to unregister
+   * @param handler input handler.
    * @return the true if removed
    */
   public synchronized boolean unregister(String identifier, RefreshHandler handler) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
index f1142d35e72c2..da08c3d152e61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
@@ -124,8 +124,9 @@ private IOException instantiateException(Class<? extends IOException> cls)
   }
 
   /**
-   * Create RemoteException from attributes
-   * @param attrs may not be null
+   * Create RemoteException from attributes.
+   * @param attrs may not be null.
+   * @return RemoteException.
    */
   public static RemoteException valueOf(Attributes attrs) {
     return new RemoteException(attrs.getValue("class"),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
index c9e04ab82b615..624cc08ac25be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
@@ -49,20 +49,20 @@ public class RetryCache {
   private static final int MAX_CAPACITY = 16;
 
   /**
-   * CacheEntry is tracked using unique client ID and callId of the RPC request
+   * CacheEntry is tracked using unique client ID and callId of the RPC request.
    */
   public static class CacheEntry implements LightWeightCache.Entry {
     /**
-     * Processing state of the requests
+     * Processing state of the requests.
      */
-    private static byte INPROGRESS = 0;
-    private static byte SUCCESS = 1;
-    private static byte FAILED = 2;
+    private static final byte INPROGRESS = 0;
+    private static final byte SUCCESS = 1;
+    private static final byte FAILED = 2;
 
     private byte state = INPROGRESS;
     
     // Store uuid as two long for better memory utilization
-    private final long clientIdMsb; // Most signficant bytes
+    private final long clientIdMsb; // Most significant bytes
     private final long clientIdLsb; // Least significant bytes
     
     private final int callId;
@@ -140,8 +140,8 @@ public long getExpirationTime() {
     
     @Override
     public String toString() {
-      return (new UUID(this.clientIdMsb, this.clientIdLsb)).toString() + ":"
-          + this.callId + ":" + this.state;
+      return String.format("%s:%s:%s", new UUID(this.clientIdMsb, this.clientIdLsb),
+          this.callId, this.state);
     }
   }
 
@@ -183,7 +183,7 @@ public Object getPayload() {
 
   private final LightWeightGSet<CacheEntry, CacheEntry> set;
   private final long expirationTime;
-  private String cacheName;
+  private final String cacheName;
 
   private final ReentrantLock lock = new ReentrantLock();
 
@@ -195,7 +195,7 @@ public Object getPayload() {
    */
   public RetryCache(String cacheName, double percentage, long expirationTime) {
     int capacity = LightWeightGSet.computeCapacity(percentage, cacheName);
-    capacity = capacity > MAX_CAPACITY ? capacity : MAX_CAPACITY;
+    capacity = Math.max(capacity, MAX_CAPACITY);
     this.set = new LightWeightCache<CacheEntry, CacheEntry>(capacity, capacity,
         expirationTime, 0);
     this.expirationTime = expirationTime;
@@ -203,11 +203,11 @@ public RetryCache(String cacheName, double percentage, long expirationTime) {
     this.retryCacheMetrics =  RetryCacheMetrics.create(this);
   }
 
-  private static boolean skipRetryCache() {
+  private static boolean skipRetryCache(byte[] clientId, int callId) {
     // Do not track non RPC invocation or RPC requests with
     // invalid callId or clientId in retry cache
-    return !Server.isRpcInvocation() || Server.getCallId() < 0
-        || Arrays.equals(Server.getClientId(), RpcConstants.DUMMY_CLIENT_ID);
+    return !Server.isRpcInvocation() || callId < 0
+        || Arrays.equals(clientId, RpcConstants.DUMMY_CLIENT_ID);
   }
 
   public void lock() {
@@ -233,7 +233,7 @@ public RetryCacheMetrics getMetricsForTests() {
   }
 
   /**
-   * This method returns cache name for metrics.
+   * @return This method returns cache name for metrics.
    */
   public String getCacheName() {
     return cacheName;
@@ -302,6 +302,9 @@ private CacheEntry waitForCompletion(CacheEntry newEntry) {
   /** 
    * Add a new cache entry into the retry cache. The cache entry consists of 
    * clientId and callId extracted from editlog.
+   *
+   * @param clientId input clientId.
+   * @param callId input callId.
    */
   public void addCacheEntry(byte[] clientId, int callId) {
     CacheEntry newEntry = new CacheEntry(clientId, callId, System.nanoTime()
@@ -329,34 +332,51 @@ public void addCacheEntryWithPayload(byte[] clientId, int callId,
     retryCacheMetrics.incrCacheUpdated();
   }
 
-  private static CacheEntry newEntry(long expirationTime) {
-    return new CacheEntry(Server.getClientId(), Server.getCallId(),
+  private static CacheEntry newEntry(long expirationTime,
+      byte[] clientId, int callId) {
+    return new CacheEntry(clientId, callId,
         System.nanoTime() + expirationTime);
   }
 
   private static CacheEntryWithPayload newEntry(Object payload,
-      long expirationTime) {
-    return new CacheEntryWithPayload(Server.getClientId(), Server.getCallId(),
+      long expirationTime, byte[] clientId, int callId) {
+    return new CacheEntryWithPayload(clientId, callId,
         payload, System.nanoTime() + expirationTime);
   }
 
-  /** Static method that provides null check for retryCache */
-  public static CacheEntry waitForCompletion(RetryCache cache) {
-    if (skipRetryCache()) {
+  /**
+   * Static method that provides null check for retryCache.
+   * @param cache input Cache.
+   * @param clientId client id of this request
+   * @param callId client call id of this request
+   * @return CacheEntry.
+   */
+  public static CacheEntry waitForCompletion(RetryCache cache,
+      byte[] clientId, int callId) {
+    if (skipRetryCache(clientId, callId)) {
       return null;
     }
     return cache != null ? cache
-        .waitForCompletion(newEntry(cache.expirationTime)) : null;
+        .waitForCompletion(newEntry(cache.expirationTime,
+            clientId, callId)) : null;
   }
 
-  /** Static method that provides null check for retryCache */
+  /**
+   * Static method that provides null check for retryCache.
+   * @param cache input cache.
+   * @param payload input payload.
+   * @param clientId client id of this request
+   * @param callId client call id of this request
+   * @return CacheEntryWithPayload.
+   */
   public static CacheEntryWithPayload waitForCompletion(RetryCache cache,
-      Object payload) {
-    if (skipRetryCache()) {
+      Object payload, byte[] clientId, int callId) {
+    if (skipRetryCache(clientId, callId)) {
       return null;
     }
     return (CacheEntryWithPayload) (cache != null ? cache
-        .waitForCompletion(newEntry(payload, cache.expirationTime)) : null);
+        .waitForCompletion(newEntry(payload, cache.expirationTime,
+            clientId, callId)) : null);
   }
 
   public static void setState(CacheEntry e, boolean success) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
index 0ce78e54a43a0..4af35ad9270f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
@@ -103,7 +103,7 @@ private static Map<Long, ProtocolSignature> getVersionSignatureMap(
    * @param version The version at the client.
    * @param methodName Name of the method.
    * @return true if the method is supported, false otherwise.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static boolean isMethodSupported(Object rpcProxy, Class<?> protocol,
       RPC.RpcKind rpcKind, long version, String methodName) throws IOException {
@@ -200,6 +200,8 @@ private static ProtocolMetaInfoPB getProtocolMetaInfoProxy(Object proxy,
    *
    * the format we want is:
    *   ClientNamenodeProtocol#getServerDefaults
+   * @param method input method.
+   * @return methodToTraceString.
    */
   public static String methodToTraceString(Method method) {
     Class<?> clazz = method.getDeclaringClass();
@@ -221,6 +223,8 @@ public static String methodToTraceString(Method method) {
    *
    * the format we want is:
    *   ClientProtocol#getBlockLocations
+   * @param fullName input fullName.
+   * @return toTraceName.
    */
   public static String toTraceName(String fullName) {
     int lastPeriod = fullName.lastIndexOf('.');
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
index 0f5769e705028..f322f6eb98abb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java
@@ -36,15 +36,62 @@
 @InterfaceStability.Evolving
 public interface RpcEngine {
 
-  /** Construct a client-side proxy object. 
-   * @param <T>*/
+  /**
+   * Construct a client-side proxy object.
+   *
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param ticket input ticket.
+   * @param conf input Configuration.
+   * @param factory input factory.
+   * @param rpcTimeout input rpcTimeout.
+   * @param connectionRetryPolicy input connectionRetryPolicy.
+   * @throws IOException raised on errors performing I/O.
+   * @return ProtocolProxy.
+   */
   <T> ProtocolProxy<T> getProxy(Class<T> protocol,
                   long clientVersion, InetSocketAddress addr,
                   UserGroupInformation ticket, Configuration conf,
                   SocketFactory factory, int rpcTimeout,
                   RetryPolicy connectionRetryPolicy) throws IOException;
 
-  /** Construct a client-side proxy object. */
+  /**
+   * Construct a client-side proxy object with a ConnectionId.
+   *
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param connId input ConnectionId.
+   * @param conf input Configuration.
+   * @param factory input factory.
+   * @param alignmentContext Alignment context
+   * @throws IOException raised on errors performing I/O.
+   * @return ProtocolProxy.
+   */
+  <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+      Client.ConnectionId connId, Configuration conf, SocketFactory factory,
+      AlignmentContext alignmentContext)
+      throws IOException;
+
+  /**
+   * Construct a client-side proxy object.
+   *
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param ticket input tocket.
+   * @param conf input Configuration.
+   * @param factory input factory.
+   * @param rpcTimeout input rpcTimeout.
+   * @param connectionRetryPolicy input connectionRetryPolicy.
+   * @param fallbackToSimpleAuth input fallbackToSimpleAuth.
+   * @param alignmentContext input alignmentContext.
+   * @throws IOException raised on errors performing I/O.
+   * @return ProtocolProxy.
+   */
   <T> ProtocolProxy<T> getProxy(Class<T> protocol,
                   long clientVersion, InetSocketAddress addr,
                   UserGroupInformation ticket, Configuration conf,
@@ -87,7 +134,7 @@ RPC.Server getServer(Class<?> protocol, Object instance, String bindAddress,
    * @param conf, Configuration.
    * @param factory, Socket factory.
    * @return Proxy object.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   ProtocolProxy<ProtocolMetaInfoPB> getProtocolMetaInfoProxy(
       ConnectionId connId, Configuration conf, SocketFactory factory)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
index 8c423b8e5e1bd..bffe5f2d257fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcScheduler.java
@@ -26,7 +26,8 @@
  */
 public interface RpcScheduler {
   /**
-   * Returns priority level greater than zero as a hint for scheduling.
+   * @return Returns priority level greater than zero as a hint for scheduling.
+   * @param obj input obj.
    */
   int getPriorityLevel(Schedulable obj);
 
@@ -37,6 +38,12 @@ public interface RpcScheduler {
    * implementations. It will not be called by any Hadoop code, and should not
    * be implemented by new implementations.
    *
+   * @param name input name.
+   * @param priorityLevel input priorityLevel.
+   * @param queueTime input queueTime.
+   * @param processingTime input processingTime.
+   * @throws UnsupportedOperationException
+   *         the requested operation is not supported.
    * @deprecated Use
    * {@link #addResponseTime(String, Schedulable, ProcessingDetails)} instead.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcServerException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcServerException.java
index 992997ead25de..ce4aac54b6cd2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcServerException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcServerException.java
@@ -47,14 +47,14 @@ public RpcServerException(final String message, final Throwable cause) {
   }
   
   /**
-   * get the rpc status corresponding to this exception
+   * @return get the rpc status corresponding to this exception.
    */
   public RpcStatusProto getRpcStatusProto() {
     return RpcStatusProto.ERROR;
   }
 
   /**
-   * get the detailed rpc status corresponding to this exception
+   * @return get the detailed rpc status corresponding to this exception.
    */
   public RpcErrorCodeProto getRpcErrorCodeProto() {
     return RpcErrorCodeProto.ERROR_RPC_SERVER;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 230cad5d3aa46..e10e7bfd7c17a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -65,9 +65,12 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.stream.Collectors;
 
 import javax.security.sasl.Sasl;
@@ -127,6 +130,8 @@
 import org.apache.hadoop.tracing.TraceUtils;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.hadoop.classification.VisibleForTesting;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream;
 import org.apache.hadoop.thirdparty.protobuf.Message;
@@ -266,10 +271,10 @@ static class RpcKindMapValue {
    * Register a RPC kind and the class to deserialize the rpc request.
    * 
    * Called by static initializers of rpcKind Engines
-   * @param rpcKind
+   * @param rpcKind - input rpcKind.
    * @param rpcRequestWrapperClass - this class is used to deserialze the
    *  the rpc request.
-   *  @param rpcInvoker - use to process the calls on SS.
+   * @param rpcInvoker - use to process the calls on SS.
    */
   
   public static void registerProtocolEngine(RPC.RpcKind rpcKind, 
@@ -282,11 +287,8 @@ public static void registerProtocolEngine(RPC.RpcKind rpcKind,
       throw new IllegalArgumentException("ReRegistration of rpcKind: " +
           rpcKind);      
     }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("rpcKind=" + rpcKind +
-          ", rpcRequestWrapperClass=" + rpcRequestWrapperClass +
-          ", rpcInvoker=" + rpcInvoker);
-    }
+    LOG.debug("rpcKind={}, rpcRequestWrapperClass={}, rpcInvoker={}.",
+        rpcKind, rpcRequestWrapperClass, rpcInvoker);
   }
   
   public Class<? extends Writable> getRpcRequestWrapper(
@@ -328,7 +330,7 @@ static Class<?> getProtocolClass(String protocolName, Configuration conf)
     return protocol;
   }
   
-  /** Returns the server instance called under or null.  May be called under
+  /** @return Returns the server instance called under or null.  May be called under
    * {@link #call(Writable, long)} implementations, and under {@link Writable}
    * methods of paramters and return values.  Permits applications to access
    * the server context.*/
@@ -341,7 +343,7 @@ public static Server get() {
    */
   private static final ThreadLocal<Call> CurCall = new ThreadLocal<Call>();
   
-  /** Get the current call */
+  /** @return Get the current call. */
   @VisibleForTesting
   public static ThreadLocal<Call> getCurCall() {
     return CurCall;
@@ -368,12 +370,22 @@ public static int getCallRetryCount() {
     return call != null ? call.retryCount : RpcConstants.INVALID_RETRY_COUNT;
   }
 
-  /** Returns the remote side ip address when invoked inside an RPC 
-   *  Returns null incase of an error.
+  /**
+   * @return Returns the remote side ip address when invoked inside an RPC
+   *  Returns null in case of an error.
    */
   public static InetAddress getRemoteIp() {
     Call call = CurCall.get();
-    return (call != null ) ? call.getHostInetAddress() : null;
+    return (call != null) ? call.getHostInetAddress() : null;
+  }
+
+  /**
+   * @return Returns the remote side port when invoked inside an RPC
+   * Returns 0 in case of an error.
+   */
+  public static int getRemotePort() {
+    Call call = CurCall.get();
+    return (call != null) ? call.getRemotePort() : 0;
   }
 
   /**
@@ -403,14 +415,14 @@ public static String getAuxiliaryPortEstablishedQOP() {
   }
 
   /**
-   * Returns the clientId from the current RPC request
+   * @return Returns the clientId from the current RPC request.
    */
   public static byte[] getClientId() {
     Call call = CurCall.get();
     return call != null ? call.clientId : RpcConstants.DUMMY_CLIENT_ID;
   }
-  
-  /** Returns remote address as a string when invoked inside an RPC.
+
+  /** @return Returns remote address as a string when invoked inside an RPC.
    *  Returns null in case of an error.
    */
   public static String getRemoteAddress() {
@@ -432,14 +444,14 @@ public static String getProtocol() {
     return (call != null) ? call.getProtocol() : null;
   }
 
-  /** Return true if the invocation was through an RPC.
+  /** @return Return true if the invocation was through an RPC.
    */
   public static boolean isRpcInvocation() {
     return CurCall.get() != null;
   }
 
   /**
-   * Return the priority level assigned by call queue to an RPC
+   * @return Return the priority level assigned by call queue to an RPC
    * Returns 0 in case no priority is assigned.
    */
   public static int getPriorityLevel() {
@@ -447,7 +459,7 @@ public static int getPriorityLevel() {
     return call != null? call.getPriorityLevel() : 0;
   }
 
-  private String bindAddress; 
+  private String bindAddress;
   private int port;                               // port we listen on
   private int handlerCount;                       // number of handler threads
   private int readThreads;                        // number of read threads
@@ -455,7 +467,7 @@ public static int getPriorityLevel() {
   private Class<? extends Writable> rpcRequestClass;   // class used for deserializing the rpc request
   final protected RpcMetrics rpcMetrics;
   final protected RpcDetailedMetrics rpcDetailedMetrics;
-  
+
   private Configuration conf;
   private String portRangeConfig = null;
   private SecretManager<TokenIdentifier> secretManager;
@@ -489,6 +501,12 @@ protected ResponseBuffer initialValue() {
   private Map<Integer, Listener> auxiliaryListenerMap;
   private Responder responder = null;
   private Handler[] handlers = null;
+  private final AtomicInteger numInProcessHandler = new AtomicInteger();
+  private final LongAdder totalRequests = new LongAdder();
+  private long lastSeenTotalRequests = 0;
+  private long totalRequestsPerSecond = 0;
+  private final long metricsUpdaterInterval;
+  private final ScheduledExecutorService scheduledExecutorService;
 
   private boolean logSlowRPC = false;
 
@@ -500,9 +518,21 @@ protected boolean isLogSlowRPC() {
     return logSlowRPC;
   }
 
+  public int getNumInProcessHandler() {
+    return numInProcessHandler.get();
+  }
+
+  public long getTotalRequests() {
+    return totalRequests.sum();
+  }
+
+  public long getTotalRequestsPerSecond() {
+    return totalRequestsPerSecond;
+  }
+
   /**
    * Sets slow RPC flag.
-   * @param logSlowRPCFlag
+   * @param logSlowRPCFlag input logSlowRPCFlag.
    */
   @VisibleForTesting
   protected void setLogSlowRPC(boolean logSlowRPCFlag) {
@@ -563,6 +593,7 @@ void logSlowRpcCalls(String methodName, Call call,
   }
 
   void updateMetrics(Call call, long startTime, boolean connDropped) {
+    totalRequests.increment();
     // delta = handler + processing + response
     long deltaNanos = Time.monotonicNowNanos() - startTime;
     long timestampNanos = call.timestampNanos;
@@ -652,17 +683,17 @@ public static void bind(ServerSocket socket, InetSocketAddress address,
     }
   }
 
-  @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting
+  @VisibleForTesting
   int getPriorityLevel(Schedulable e) {
     return callQueue.getPriorityLevel(e);
   }
 
-  @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting
+  @VisibleForTesting
   int getPriorityLevel(UserGroupInformation ugi) {
     return callQueue.getPriorityLevel(ugi);
   }
 
-  @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting
+  @VisibleForTesting
   void setPriorityLevel(UserGroupInformation ugi, int priority) {
     callQueue.setPriorityLevel(ugi, priority);
   }
@@ -693,6 +724,9 @@ Connection[] getConnections() {
 
   /**
    * Refresh the service authorization ACL for the service handled by this server.
+   *
+   * @param conf input Configuration.
+   * @param provider input PolicyProvider.
    */
   public void refreshServiceAcl(Configuration conf, PolicyProvider provider) {
     serviceAuthorizationManager.refresh(conf, provider);
@@ -701,6 +735,9 @@ public void refreshServiceAcl(Configuration conf, PolicyProvider provider) {
   /**
    * Refresh the service authorization ACL for the service handled by this server
    * using the specified Configuration.
+   *
+   * @param conf input Configuration.
+   * @param provider input provider.
    */
   @Private
   public void refreshServiceAclWithLoadedConfiguration(Configuration conf,
@@ -888,7 +925,7 @@ public static class Call implements Schedulable,
     private volatile String detailedMetricsName = "";
     final int callId;            // the client's call id
     final int retryCount;        // the retry count of the call
-    long timestampNanos;         // time the call was received
+    private final long timestampNanos; // time the call was received
     long responseTimestampNanos; // time the call was served
     private AtomicInteger responseWaitCount = new AtomicInteger(1);
     final RPC.RpcKind rpcKind;
@@ -900,6 +937,9 @@ public static class Call implements Schedulable,
     // the priority level assigned by scheduler, 0 by default
     private long clientStateId;
     private boolean isCallCoordinated;
+    // Serialized RouterFederatedStateProto message to
+    // store last seen states for multiple namespaces.
+    private ByteString federatedNamespaceState;
 
     Call() {
       this(RpcConstants.INVALID_CALL_ID, RpcConstants.INVALID_RETRY_COUNT,
@@ -957,6 +997,14 @@ public ProcessingDetails getProcessingDetails() {
       return processingDetails;
     }
 
+    public void setFederatedNamespaceState(ByteString federatedNamespaceState) {
+      this.federatedNamespaceState = federatedNamespaceState;
+    }
+
+    public ByteString getFederatedNamespaceState() {
+      return this.federatedNamespaceState;
+    }
+
     @Override
     public String toString() {
       return "Call#" + callId + " Retry#" + retryCount;
@@ -973,6 +1021,9 @@ public UserGroupInformation getRemoteUser() {
     public InetAddress getHostInetAddress() {
       return null;
     }
+    public int getRemotePort() {
+      return 0;
+    }
     public String getHostAddress() {
       InetAddress addr = getHostInetAddress();
       return (addr != null) ? addr.getHostAddress() : null;
@@ -1067,6 +1118,10 @@ public void setDeferredResponse(Writable response) {
 
     public void setDeferredError(Throwable t) {
     }
+
+    public long getTimestampNanos() {
+      return timestampNanos;
+    }
   }
 
   /** A RPC extended call queued for handling. */
@@ -1130,6 +1185,11 @@ public InetAddress getHostInetAddress() {
       return connection.getHostInetAddress();
     }
 
+    @Override
+    public int getRemotePort() {
+      return connection.getRemotePort();
+    }
+
     @Override
     public Void run() throws Exception {
       if (!connection.channel.isOpen()) {
@@ -1143,7 +1203,7 @@ public Void run() throws Exception {
 
       try {
         value = call(
-            rpcKind, connection.protocolName, rpcRequest, timestampNanos);
+            rpcKind, connection.protocolName, rpcRequest, getTimestampNanos());
       } catch (Throwable e) {
         populateResponseParamsOnError(e, responseParams);
       }
@@ -1164,9 +1224,7 @@ public Void run() throws Exception {
         deltaNanos = Time.monotonicNowNanos() - startNanos;
         details.set(Timing.RESPONSE, deltaNanos, TimeUnit.NANOSECONDS);
       } else {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Deferring response for callId: " + this.callId);
-        }
+        LOG.debug("Deferring response for callId: {}", this.callId);
       }
       return null;
     }
@@ -1335,8 +1393,7 @@ private class Listener extends Thread {
       bind(acceptChannel.socket(), address, backlogLength, conf, portRangeConfig);
       //Could be an ephemeral port
       this.listenPort = acceptChannel.socket().getLocalPort();
-      Thread.currentThread().setName("Listener at " +
-          bindAddress + "/" + this.listenPort);
+      LOG.info("Listener at {}:{}", bindAddress, this.listenPort);
       // create a selector;
       selector= Selector.open();
       readers = new Reader[readThreads];
@@ -1663,9 +1720,7 @@ private void doRunLoop() {
           // If there were some calls that have not been sent out for a
           // long time, discard them.
           //
-          if(LOG.isDebugEnabled()) {
-            LOG.debug("Checking for old call responses.");
-          }
+          LOG.debug("Checking for old call responses.");
           ArrayList<RpcCall> calls;
           
           // get the list of channels from list of keys.
@@ -1765,9 +1820,8 @@ private boolean processResponse(LinkedList<RpcCall> responseQueue,
           //
           call = responseQueue.removeFirst();
           SocketChannel channel = call.connection.channel;
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(Thread.currentThread().getName() + ": responding to " + call);
-          }
+
+          LOG.debug("{}: responding to {}.", Thread.currentThread().getName(), call);
           //
           // Send as much data as we can in the non-blocking fashion
           //
@@ -1784,10 +1838,8 @@ private boolean processResponse(LinkedList<RpcCall> responseQueue,
             } else {
               done = false;            // more calls pending to be sent.
             }
-            if (LOG.isDebugEnabled()) {
-              LOG.debug(Thread.currentThread().getName() + ": responding to " + call
-                  + " Wrote " + numBytes + " bytes.");
-            }
+            LOG.debug("{}: responding to {} Wrote {} bytes.",
+                Thread.currentThread().getName(), call, numBytes);
           } else {
             //
             // If we were unable to write the entire response out, then 
@@ -1812,10 +1864,8 @@ private boolean processResponse(LinkedList<RpcCall> responseQueue,
                 decPending();
               }
             }
-            if (LOG.isDebugEnabled()) {
-              LOG.debug(Thread.currentThread().getName() + ": responding to " + call
-                  + " Wrote partial " + numBytes + " bytes.");
-            }
+            LOG.debug("{}: responding to {} Wrote partial {} bytes.",
+                Thread.currentThread().getName(), call, numBytes);
           }
           error = false;              // everything went off well
         }
@@ -2011,6 +2061,10 @@ public int getIngressPort() {
       return ingressPort;
     }
 
+    public int getRemotePort() {
+      return remotePort;
+    }
+
     public InetAddress getHostInetAddress() {
       return addr;
     }
@@ -2157,13 +2211,11 @@ private void saslProcess(RpcSaslProto saslMessage)
         
         if (saslServer != null && saslServer.isComplete()) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("SASL server context established. Negotiated QoP is "
-                + saslServer.getNegotiatedProperty(Sasl.QOP));
+            LOG.debug("SASL server context established. Negotiated QoP is {}.",
+                saslServer.getNegotiatedProperty(Sasl.QOP));
           }
           user = getAuthorizedUgi(saslServer.getAuthorizationID());
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("SASL server successfully authenticated client: " + user);
-          }
+          LOG.debug("SASL server successfully authenticated client: {}.", user);
           rpcMetrics.incrAuthenticationSuccesses();
           AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user + " from " + toString());
           saslContextEstablished = true;
@@ -2268,10 +2320,8 @@ private RpcSaslProto processSaslToken(RpcSaslProto saslMessage)
         throw new SaslException("Client did not send a token");
       }
       byte[] saslToken = saslMessage.getToken().toByteArray();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Have read input token of size " + saslToken.length
-            + " for processing by saslServer.evaluateResponse()");
-      }
+      LOG.debug("Have read input token of size {} for processing by saslServer.evaluateResponse()",
+          saslToken.length);
       saslToken = saslServer.evaluateResponse(saslToken);
       return buildSaslResponse(
           saslServer.isComplete() ? SaslState.SUCCESS : SaslState.CHALLENGE,
@@ -2286,9 +2336,8 @@ private void switchToSimple() {
 
     private RpcSaslProto buildSaslResponse(SaslState state, byte[] replyToken) {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Will send " + state + " token of size "
-            + ((replyToken != null) ? replyToken.length : null)
-            + " from saslServer.");
+        LOG.debug("Will send {} token of size {} from saslServer.", state,
+            ((replyToken != null) ? replyToken.length : null));
       }
       RpcSaslProto.Builder response = RpcSaslProto.newBuilder();
       response.setState(state);
@@ -2354,7 +2403,7 @@ private void checkDataLength(int dataLength) throws IOException {
      * @return -1 in case of error, else num bytes read so far
      * @throws IOException - internal error that should not be returned to
      *         client, typically failure to respond to client
-     * @throws InterruptedException
+     * @throws InterruptedException - if the thread is interrupted.
      */
     public int readAndProcess() throws IOException, InterruptedException {
       while (!shouldClose()) { // stop if a fatal response has been sent.
@@ -2612,10 +2661,8 @@ private void processConnectionContext(RpcWritable.Buffer buffer)
      */    
     private void unwrapPacketAndProcessRpcs(byte[] inBuf)
         throws IOException, InterruptedException {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Have read input token of size " + inBuf.length
-            + " for processing by saslServer.unwrap()");
-      }
+      LOG.debug("Have read input token of size {} for processing by saslServer.unwrap()",
+          inBuf.length);
       inBuf = saslServer.unwrap(inBuf, 0, inBuf.length);
       ReadableByteChannel ch = Channels.newChannel(new ByteArrayInputStream(
           inBuf));
@@ -2677,9 +2724,7 @@ private void processOneRpc(ByteBuffer bb)
             getMessage(RpcRequestHeaderProto.getDefaultInstance(), buffer);
         callId = header.getCallId();
         retry = header.getRetryCount();
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(" got #" + callId);
-        }
+        LOG.debug(" got #{}", callId);
         checkRpcHeaders(header);
 
         if (callId < 0) { // callIds typically used during connection setup
@@ -2694,11 +2739,8 @@ private void processOneRpc(ByteBuffer bb)
       } catch (RpcServerException rse) {
         // inform client of error, but do not rethrow else non-fatal
         // exceptions will close connection!
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(Thread.currentThread().getName() +
-              ": processOneRpc from client " + this +
-              " threw exception [" + rse + "]");
-        }
+        LOG.debug("{}: processOneRpc from client {} threw exception [{}]",
+            Thread.currentThread().getName(), this, rse);
         // use the wrapped exception if there is one.
         Throwable t = (rse.getCause() != null) ? rse.getCause() : rse;
         final RpcCall call = new RpcCall(this, callId, retry);
@@ -2836,6 +2878,9 @@ private void processRpcRequest(RpcRequestHeaderProto header,
             stateId = alignmentContext.receiveRequestState(
                 header, getMaxIdleTime());
             call.setClientStateId(stateId);
+            if (header.hasRouterFederatedState()) {
+              call.setFederatedNamespaceState(header.getRouterFederatedState());
+            }
           }
         } catch (IOException ioe) {
           throw new RpcServerException("Processing RPC request caught ", ioe);
@@ -2910,9 +2955,7 @@ private void authorizeConnection() throws RpcServerException {
           ProxyUsers.authorize(user, this.getHostAddress());
         }
         authorize(user, protocolName, getHostInetAddress());
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Successfully authorized " + connectionContext);
-        }
+        LOG.debug("Successfully authorized {}.", connectionContext);
         rpcMetrics.incrAuthorizationSuccesses();
       } catch (AuthorizationException ae) {
         LOG.info("Connection from " + this
@@ -3029,7 +3072,7 @@ public Handler(int instanceNumber) {
 
     @Override
     public void run() {
-      LOG.debug(Thread.currentThread().getName() + ": starting");
+      LOG.debug("{}: starting", Thread.currentThread().getName());
       SERVER.set(Server.this);
       while (running) {
         TraceScope traceScope = null;
@@ -3042,6 +3085,7 @@ public void run() {
 
         try {
           call = callQueue.take(); // pop the queue; maybe blocked here
+          numInProcessHandler.incrementAndGet();
           startTimeNanos = Time.monotonicNowNanos();
           if (alignmentContext != null && call.isCallCoordinated() &&
               call.getClientStateId() > alignmentContext.getLastSeenStateId()) {
@@ -3062,9 +3106,7 @@ public void run() {
             call = null;
             continue;
           }
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);
-          }
+          LOG.debug("{}: {} for RpcKind {}.", Thread.currentThread().getName(), call, call.rpcKind);
           CurCall.set(call);
           if (call.span != null) {
             traceScope = tracer.activateSpan(call.span);
@@ -3095,24 +3137,25 @@ public void run() {
           }
         } finally {
           CurCall.set(null);
+          numInProcessHandler.decrementAndGet();
           IOUtils.cleanupWithLogger(LOG, traceScope);
           if (call != null) {
             updateMetrics(call, startTimeNanos, connDropped);
-            ProcessingDetails.LOG.debug(
-                "Served: [{}]{} name={} user={} details={}",
+            ProcessingDetails.LOG.debug("Served: [{}]{} name={} user={} details={}",
                 call, (call.isResponseDeferred() ? ", deferred" : ""),
                 call.getDetailedMetricsName(), call.getRemoteUser(),
                 call.getProcessingDetails());
           }
         }
       }
-      LOG.debug(Thread.currentThread().getName() + ": exiting");
+      LOG.debug("{}: exiting", Thread.currentThread().getName());
     }
 
     private void requeueCall(Call call)
         throws IOException, InterruptedException {
       try {
         internalQueueCall(call, false);
+        rpcMetrics.incrRequeueCalls();
       } catch (RpcServerException rse) {
         call.doResponse(rse.getCause(), rse.getRpcStatusProto());
       }
@@ -3170,6 +3213,18 @@ protected Server(String bindAddress, int port,
    *  Class, RPC.RpcInvoker)}
    * This parameter has been retained for compatibility with existing tests
    * and usage.
+   *
+   * @param bindAddress input bindAddress.
+   * @param port input port.
+   * @param rpcRequestClass input rpcRequestClass.
+   * @param handlerCount input handlerCount.
+   * @param numReaders input numReaders.
+   * @param queueSizePerHandler input queueSizePerHandler.
+   * @param conf input Configuration.
+   * @param serverName input serverName.
+   * @param secretManager input secretManager.
+   * @param portRangeConfig input portRangeConfig.
+   * @throws IOException raised on errors performing I/O.
    */
   @SuppressWarnings("unchecked")
   protected Server(String bindAddress, int port,
@@ -3257,6 +3312,14 @@ protected Server(String bindAddress, int port,
     this.exceptionsHandler.addTerseLoggingExceptions(StandbyException.class);
     this.exceptionsHandler.addTerseLoggingExceptions(
         HealthCheckFailedException.class);
+    this.metricsUpdaterInterval =
+        conf.getLong(CommonConfigurationKeysPublic.IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL,
+            CommonConfigurationKeysPublic.IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL_DEFAULT);
+    this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
+        new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Hadoop-Metrics-Updater-%d")
+            .build());
+    this.scheduledExecutorService.scheduleWithFixedDelay(new MetricsUpdateRunner(),
+        metricsUpdaterInterval, metricsUpdaterInterval, TimeUnit.MILLISECONDS);
   }
 
   public synchronized void addAuxiliaryListener(int auxiliaryPort)
@@ -3315,14 +3378,13 @@ private List<AuthMethod> getAuthMethods(SecretManager<?> secretManager,
             " authentication requires a secret manager");
       } 
     } else if (secretManager != null) {
-      LOG.debug(AuthenticationMethod.TOKEN +
-          " authentication enabled for secret manager");
+      LOG.debug("{} authentication enabled for secret manager", AuthenticationMethod.TOKEN);
       // most preferred, go to the front of the line!
       authMethods.add(AuthenticationMethod.TOKEN.getAuthMethod());
     }
     authMethods.add(confAuthenticationMethod.getAuthMethod());        
     
-    LOG.debug("Server accepts auth methods:" + authMethods);
+    LOG.debug("Server accepts auth methods:{}", authMethods);
     return authMethods;
   }
   
@@ -3482,9 +3544,7 @@ private void wrapWithSasl(RpcCall call) throws IOException {
       synchronized (call.connection.saslServer) {
         token = call.connection.saslServer.wrap(token, 0, token.length);
       }
-      if (LOG.isDebugEnabled())
-        LOG.debug("Adding saslServer wrapped token of size " + token.length
-            + " as call response.");
+      LOG.debug("Adding saslServer wrapped token of size {} as call response.", token.length);
       // rebuild with sasl header and payload
       RpcResponseHeaderProto saslHeader = RpcResponseHeaderProto.newBuilder()
           .setCallId(AuthProtocol.SASL.callId)
@@ -3502,7 +3562,10 @@ Configuration getConf() {
     return conf;
   }
   
-  /** Sets the socket buffer size used for responding to RPCs */
+  /**
+   * Sets the socket buffer size used for responding to RPCs.
+   * @param size input size.
+   */
   public void setSocketSendBufSize(int size) { this.socketSendBufferSize = size; }
 
   public void setTracer(Tracer t) {
@@ -3548,13 +3611,30 @@ public synchronized void stop() {
     }
     responder.interrupt();
     notifyAll();
+    shutdownMetricsUpdaterExecutor();
     this.rpcMetrics.shutdown();
     this.rpcDetailedMetrics.shutdown();
   }
 
-  /** Wait for the server to be stopped.
+  private void shutdownMetricsUpdaterExecutor() {
+    this.scheduledExecutorService.shutdown();
+    try {
+      boolean isExecutorShutdown =
+          this.scheduledExecutorService.awaitTermination(3, TimeUnit.SECONDS);
+      if (!isExecutorShutdown) {
+        LOG.info("Hadoop Metrics Updater executor could not be shutdown.");
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      LOG.info("Hadoop Metrics Updater executor shutdown interrupted.", e);
+    }
+  }
+
+  /**
+   * Wait for the server to be stopped.
    * Does not wait for all subthreads to finish.
    *  See {@link #stop()}.
+   * @throws InterruptedException if the thread is interrupted.
    */
   public synchronized void join() throws InterruptedException {
     while (running) {
@@ -3591,13 +3671,25 @@ public synchronized Set<InetSocketAddress> getAuxiliaryListenerAddresses() {
    * Called for each call. 
    * @deprecated Use  {@link #call(RPC.RpcKind, String,
    *  Writable, long)} instead
+   * @param param input param.
+   * @param receiveTime input receiveTime.
+   * @throws Exception if any error occurs.
+   * @return Call
    */
   @Deprecated
   public Writable call(Writable param, long receiveTime) throws Exception {
     return call(RPC.RpcKind.RPC_BUILTIN, null, param, receiveTime);
   }
   
-  /** Called for each call. */
+  /**
+   * Called for each call.
+   * @param rpcKind input rpcKind.
+   * @param protocol input protocol.
+   * @param param input param.
+   * @param receiveTime input receiveTime.
+   * @return Call.
+   * @throws Exception raised on errors performing I/O.
+   */
   public abstract Writable call(RPC.RpcKind rpcKind, String protocol,
       Writable param, long receiveTime) throws Exception;
   
@@ -3645,7 +3737,7 @@ public int getNumOpenConnections() {
   }
 
   /**
-   * Get the NumOpenConnections/User.
+   * @return Get the NumOpenConnections/User.
    */
   public String getNumOpenConnectionsPerUser() {
     ObjectMapper mapper = new ObjectMapper();
@@ -3898,11 +3990,8 @@ Connection register(SocketChannel channel, int ingressPort,
       Connection connection = new Connection(channel, Time.now(),
           ingressPort, isOnAuxiliaryPort);
       add(connection);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Server connection from " + connection +
-            "; # active connections: " + size() +
-            "; # queued calls: " + callQueue.size());
-      }      
+      LOG.debug("Server connection from {}; # active connections: {}; # queued calls: {}.",
+          connection, size(), callQueue.size());
       return connection;
     }
     
@@ -3910,9 +3999,8 @@ boolean close(Connection connection) {
       boolean exists = remove(connection);
       if (exists) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug(Thread.currentThread().getName() +
-              ": disconnecting client " + connection +
-              ". Number of active connections: "+ size());
+          LOG.debug("{}: disconnecting client {}. Number of active connections: {}.",
+              Thread.currentThread().getName(), connection, size());
         }
         // only close if actually removed to avoid double-closing due
         // to possible races
@@ -3974,9 +4062,7 @@ public void run() {
           if (!running) {
             return;
           }
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(Thread.currentThread().getName()+": task running");
-          }
+          LOG.debug("{}: task running", Thread.currentThread().getName());
           try {
             closeIdle(false);
           } finally {
@@ -3997,4 +4083,46 @@ protected int getMaxIdleTime() {
   public String getServerName() {
     return serverName;
   }
+
+  /**
+   * Server metrics updater thread, used to update some metrics on a regular basis.
+   * For instance, requests per second.
+   */
+  private class MetricsUpdateRunner implements Runnable {
+
+    private long lastExecuted = 0;
+
+    @Override
+    public synchronized void run() {
+      long currentTime = Time.monotonicNow();
+      if (lastExecuted == 0) {
+        lastExecuted = currentTime - metricsUpdaterInterval;
+      }
+      long currentTotalRequests = totalRequests.sum();
+      long totalRequestsDiff = currentTotalRequests - lastSeenTotalRequests;
+      lastSeenTotalRequests = currentTotalRequests;
+      if ((currentTime - lastExecuted) > 0) {
+        double totalRequestsPerSecInDouble =
+            (double) totalRequestsDiff / TimeUnit.MILLISECONDS.toSeconds(
+                currentTime - lastExecuted);
+        totalRequestsPerSecond = ((long) totalRequestsPerSecInDouble);
+      }
+      lastExecuted = currentTime;
+    }
+  }
+
+  @VisibleForTesting
+  CallQueueManager<Call> getCallQueue() {
+    return callQueue;
+  }
+
+  @VisibleForTesting
+  void setCallQueue(CallQueueManager<Call> callQueue) {
+    this.callQueue = callQueue;
+  }
+
+  @VisibleForTesting
+  void setRpcRequestClass(Class<? extends Writable> rpcRequestClass) {
+    this.rpcRequestClass = rpcRequestClass;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/VersionedProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/VersionedProtocol.java
index 4d02027a0e688..98daa84187464 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/VersionedProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/VersionedProtocol.java
@@ -46,6 +46,7 @@ public long getProtocolVersion(String protocol,
    *         a list of its supported methods
    * @see ProtocolSignature#getProtocolSignature(VersionedProtocol, String, 
    *                long, int) for a default implementation
+   * @throws IOException raised on errors performing I/O.
    */
   public ProtocolSignature getProtocolSignature(String protocol, 
                                  long clientVersion,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
index d790e49f5dcf2..d92bcea5d2eff 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java
@@ -282,9 +282,20 @@ static Client getClient(Configuration conf) {
     return CLIENTS.getClient(conf);
   }
   
-  /** Construct a client-side proxy object that implements the named protocol,
+  /**
+   * Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address. 
-   * @param <T>*/
+   * @param <T> Generics Type T
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param ticket input ticket.
+   * @param conf input configuration.
+   * @param factory input factory.
+   * @param rpcTimeout input rpcTimeout.
+   * @param connectionRetryPolicy input connectionRetryPolicy.
+   * @throws IOException raised on errors performing I/O.
+   */
   @Override
   public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
                          InetSocketAddress addr, UserGroupInformation ticket,
@@ -295,9 +306,45 @@ public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
       rpcTimeout, connectionRetryPolicy, null, null);
   }
 
-  /** Construct a client-side proxy object that implements the named protocol,
+  /**
+   * Construct a client-side proxy object with a ConnectionId.
+   *
+   * @param <T> Generics Type T.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param connId input ConnectionId.
+   * @param conf input Configuration.
+   * @param factory input factory.
+   * @param alignmentContext Alignment context
+   * @throws IOException raised on errors performing I/O.
+   * @return ProtocolProxy.
+   */
+  @Override
+  public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+      Client.ConnectionId connId, Configuration conf, SocketFactory factory,
+      AlignmentContext alignmentContext)
+      throws IOException {
+    return getProxy(protocol, clientVersion, connId.getAddress(),
+        connId.getTicket(), conf, factory, connId.getRpcTimeout(),
+        connId.getRetryPolicy(), null, alignmentContext);
+  }
+
+  /**
+   * Construct a client-side proxy object that implements the named protocol,
    * talking to a server at the named address. 
-   * @param <T>*/
+   * @param <T> Generics Type.
+   * @param protocol input protocol.
+   * @param clientVersion input clientVersion.
+   * @param addr input addr.
+   * @param ticket input ticket.
+   * @param conf input configuration.
+   * @param factory input factory.
+   * @param rpcTimeout input rpcTimeout.
+   * @param connectionRetryPolicy input connectionRetryPolicy.
+   * @param fallbackToSimpleAuth input fallbackToSimpleAuth.
+   * @param alignmentContext input alignmentContext.
+   * @return ProtocolProxy.
+   */
   @Override
   @SuppressWarnings("unchecked")
   public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
@@ -345,7 +392,8 @@ public static class Server extends RPC.Server {
      * @param bindAddress the address to bind on to listen for connection
      * @param port the port to listen for connections on
      * 
-     * @deprecated Use #Server(Class, Object, Configuration, String, int)    
+     * @deprecated Use #Server(Class, Object, Configuration, String, int)
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Server(Object instance, Configuration conf, String bindAddress,
@@ -360,6 +408,7 @@ public Server(Object instance, Configuration conf, String bindAddress,
      * @param conf the configuration to use
      * @param bindAddress the address to bind on to listen for connection
      * @param port the port to listen for connections on
+     * @throws IOException raised on errors performing I/O.
      */
     public Server(Class<?> protocolClass, Object protocolImpl, 
         Configuration conf, String bindAddress, int port) 
@@ -376,9 +425,13 @@ public Server(Class<?> protocolClass, Object protocolImpl,
      * @param port the port to listen for connections on
      * @param numHandlers the number of method handler threads to run
      * @param verbose whether each call should be logged
+     * @param numReaders input numberReaders.
+     * @param queueSizePerHandler input queueSizePerHandler.
+     * @param secretManager input secretManager.
      * 
      * @deprecated use Server#Server(Class, Object, 
      *      Configuration, String, int, int, int, int, boolean, SecretManager)
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Server(Object protocolImpl, Configuration conf, String bindAddress,
@@ -401,9 +454,14 @@ public Server(Object protocolImpl, Configuration conf, String bindAddress,
      * @param port the port to listen for connections on
      * @param numHandlers the number of method handler threads to run
      * @param verbose whether each call should be logged
+     * @param secretManager input secretManager.
+     * @param queueSizePerHandler input queueSizePerHandler.
+     * @param portRangeConfig input portRangeConfig.
+     * @param numReaders input numReaders.
      *
      * @deprecated use Server#Server(Class, Object,
      *      Configuration, String, int, int, int, int, boolean, SecretManager)
+     * @throws IOException raised on errors performing I/O.
      */
     @Deprecated
     public Server(Class<?> protocolClass, Object protocolImpl,
@@ -428,6 +486,11 @@ public Server(Class<?> protocolClass, Object protocolImpl,
      * @param numHandlers the number of method handler threads to run
      * @param verbose whether each call should be logged
      * @param alignmentContext provides server state info on client responses
+     * @param numReaders input numReaders.
+     * @param portRangeConfig input portRangeConfig.
+     * @param queueSizePerHandler input queueSizePerHandler.
+     * @param secretManager input secretManager.
+     * @throws IOException raised on errors performing I/O.
      */
     public Server(Class<?> protocolClass, Object protocolImpl,
         Configuration conf, String bindAddress,  int port,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java
index b86381706d67b..0bfe5c7d88000 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java
@@ -65,6 +65,7 @@ public static DecayRpcSchedulerDetailedMetrics create(String ns) {
 
   /**
    * Initialize the metrics for JMX with priority levels.
+   * @param numLevels input numLevels.
    */
   public void init(int numLevels) {
     LOG.info("Initializing RPC stats for {} priority levels", numLevels);
@@ -106,14 +107,16 @@ public void shutdown() {
   }
 
   /**
-   * Returns the rate name inside the metric.
+   * @return Returns the rate name inside the metric.
+   * @param priority input priority.
    */
   public String getQueueName(int priority) {
     return "DecayRPCSchedulerPriority."+priority+".RpcQueueTime";
   }
 
   /**
-   * Returns the rate name inside the metric.
+   * @return Returns the rate name inside the metric.
+   * @param priority input priority.
    */
   public String getProcessingName(int priority) {
     return "DecayRPCSchedulerPriority."+priority+".RpcProcessingTime";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
index cbb4bb36d1da5..282eca3cf8373 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java
@@ -128,11 +128,18 @@ public static RpcMetrics create(Server server, Configuration conf) {
   MutableCounterLong rpcClientBackoff;
   @Metric("Number of Slow RPC calls")
   MutableCounterLong rpcSlowCalls;
+  @Metric("Number of requeue calls")
+  MutableCounterLong rpcRequeueCalls;
 
   @Metric("Number of open connections") public int numOpenConnections() {
     return server.getNumOpenConnections();
   }
 
+  @Metric("Number of in process handlers")
+  public int getNumInProcessHandler() {
+    return server.getNumInProcessHandler();
+  }
+
   @Metric("Number of open connections per user")
   public String numOpenConnectionsPerUser() {
     return server.getNumOpenConnectionsPerUser();
@@ -146,6 +153,16 @@ public String numOpenConnectionsPerUser() {
     return server.getNumDroppedConnections();
   }
 
+  @Metric("Number of total requests")
+  public long getTotalRequests() {
+    return server.getTotalRequests();
+  }
+
+  @Metric("Number of total requests per second")
+  public long getTotalRequestsPerSecond() {
+    return server.getTotalRequestsPerSecond();
+  }
+
   public TimeUnit getMetricsTimeUnit() {
     return metricsTimeUnit;
   }
@@ -288,6 +305,14 @@ public void incrClientBackoff() {
   public  void incrSlowRpc() {
     rpcSlowCalls.incr();
   }
+
+  /**
+   * Increments the Requeue Calls counter.
+   */
+  public void incrRequeueCalls() {
+    rpcRequeueCalls.incr();
+  }
+
   /**
    * Returns a MutableRate Counter.
    * @return Mutable Rate
@@ -328,6 +353,15 @@ public long getRpcSlowCalls() {
     return rpcSlowCalls.value();
   }
 
+  /**
+   * Returns the number of requeue calls.
+   * @return long
+   */
+  @VisibleForTesting
+  public long getRpcRequeueCalls() {
+    return rpcRequeueCalls.value();
+  }
+
   public MutableRate getDeferredRpcProcessingTime() {
     return deferredRpcProcessingTime;
   }
@@ -348,4 +382,9 @@ public double getDeferredRpcProcessingStdDev() {
   public MetricsTag getTag(String tagName) {
     return registry.getTag(tagName);
   }
+
+  @VisibleForTesting
+  public MutableCounterLong getRpcAuthorizationSuccesses() {
+    return rpcAuthorizationSuccesses;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
index f20933b5c8668..85f2d2828562d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
@@ -64,27 +64,31 @@
  * functionality is provided through the
  * {@link MBeanServer#queryNames(ObjectName, javax.management.QueryExp)}
  * method.
+ * </p>
  * <p>
  * For example <code>http://.../jmx?qry=Hadoop:*</code> will return
  * all hadoop metrics exposed through JMX.
+ * </p>
  * <p>
  * The optional <code>get</code> parameter is used to query an specific 
  * attribute of a JMX bean.  The format of the URL is
  * <code>http://.../jmx?get=MXBeanName::AttributeName</code>
+ * </p>
  * <p>
  * For example 
  * <code>
  * http://../jmx?get=Hadoop:service=NameNode,name=NameNodeInfo::ClusterId
  * </code> will return the cluster id of the namenode mxbean.
+ * </p>
  * <p>
  * If the <code>qry</code> or the <code>get</code> parameter is not formatted 
- * correctly then a 400 BAD REQUEST http response code will be returned. 
+ * correctly then a 400 BAD REQUEST http response code will be returned.
+ * </p>
  * <p>
  * If a resouce such as a mbean or attribute can not be found, 
  * a 404 SC_NOT_FOUND http response code will be returned. 
- * <p>
+ * </p>
  * The return format is JSON and in the form
- * <p>
  *  <pre><code>
  *  {
  *    "beans" : [
@@ -95,7 +99,6 @@
  *    ]
  *  }
  *  </code></pre>
- *  <p>
  *  The servlet attempts to convert the the JMXBeans into JSON. Each
  *  bean's attributes will be converted to a JSON object member.
  *  
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
index c8a88236aeb39..e2ad16fce2c57 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -66,6 +66,8 @@ public class LogLevel {
   public static final String PROTOCOL_HTTPS = "https";
   /**
    * A command line implementation
+   * @param args input args.
+   * @throws Exception exception.
    */
   public static void main(String[] args) throws Exception {
     CLI cli = new CLI(new Configuration());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
index 622ee5405c892..af5f852143389 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -88,21 +88,22 @@ public class LogThrottlingHelper {
   public interface LogAction {
 
     /**
-     * Return the number of records encapsulated in this action; that is, the
+     * @return Return the number of records encapsulated in this action; that is, the
      * number of times {@code record} was called to produce this action,
      * including the current one.
      */
     int getCount();
 
     /**
-     * Return summary information for the value that was recorded at index
+     * @return Return summary information for the value that was recorded at index
      * {@code idx}. Corresponds to the ordering of values passed to
      * {@link #record(double...)}.
+     * @param idx input idx.
      */
     SummaryStatistics getStats(int idx);
 
     /**
-     * If this is true, the caller should write to its log. Otherwise, the
+     * @return If this is true, the caller should write to its log. Otherwise, the
      * caller should take no action, and it is an error to call other methods
      * on this object.
      */
@@ -139,6 +140,7 @@ public interface LogAction {
    * Create a log helper without any primary recorder.
    *
    * @see #LogThrottlingHelper(long, String)
+   * @param minLogPeriodMs input minLogPeriodMs.
    */
   public LogThrottlingHelper(long minLogPeriodMs) {
     this(minLogPeriodMs, null);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
index 1d62c0a29fca4..3a9be12803143 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
@@ -21,8 +21,9 @@
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectWriter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
index a277abd6e1384..fef8c4b7e4ba9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
@@ -50,7 +50,7 @@ public abstract class MetricsSystem implements MetricsSystemMXBean {
    *              the annotations of the source object.)
    * @param desc  the description of the source (or null. See above.)
    * @return the source object
-   * @exception MetricsException
+   * @exception MetricsException Metrics Exception.
    */
   public abstract <T> T register(String name, String desc, T source);
 
@@ -65,7 +65,7 @@ public abstract class MetricsSystem implements MetricsSystemMXBean {
    * @param <T>   the actual type of the source object
    * @param source  object to register
    * @return  the source object
-   * @exception MetricsException
+   * @exception MetricsException Metrics Exception.
    */
   public <T> T register(T source) {
     return register(null, null, source);
@@ -85,7 +85,7 @@ public <T> T register(T source) {
    * @param name  of the sink. Must be unique.
    * @param desc  the description of the sink
    * @return the sink
-   * @exception MetricsException
+   * @exception MetricsException Metrics Exception.
    */
   public abstract <T extends MetricsSink>
   T register(String name, String desc, T sink);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystemMXBean.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystemMXBean.java
index e471ab7498ce4..8656da6f316c0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystemMXBean.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystemMXBean.java
@@ -29,19 +29,19 @@
 public interface MetricsSystemMXBean {
   /**
    * Start the metrics system
-   * @throws MetricsException
+   * @throws MetricsException Metrics Exception.
    */
   public void start();
 
   /**
    * Stop the metrics system
-   * @throws MetricsException
+   * @throws MetricsException Metrics Exception.
    */
   public void stop();
 
   /**
    * Start metrics MBeans
-   * @throws MetricsException
+   * @throws MetricsException Metrics Exception.
    */
   public void startMetricsMBeans();
 
@@ -49,7 +49,7 @@ public interface MetricsSystemMXBean {
    * Stop metrics MBeans.
    * Note, it doesn't stop the metrics system control MBean,
    * i.e this interface.
-   * @throws MetricsException
+   * @throws MetricsException Metrics Exception.
    */
   public void stopMetricsMBeans();
 
@@ -57,7 +57,7 @@ public interface MetricsSystemMXBean {
    * @return the current config
    * Avoided getConfig, as it'll turn into a "Config" attribute,
    * which doesn't support multiple line values in jconsole.
-   * @throws MetricsException
+   * @throws MetricsException Metrics Exception.
    */
   public String currentConfig();
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java
index 9ffceaaa0ddda..b11f775a73db3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java
@@ -22,12 +22,14 @@
 
 import static org.apache.hadoop.util.Preconditions.*;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.AbstractMetric;
 import org.apache.hadoop.metrics2.MetricsTag;
 import static org.apache.hadoop.metrics2.util.Contracts.*;
 
-class MetricsRecordImpl extends AbstractMetricsRecord {
+@VisibleForTesting
+public class MetricsRecordImpl extends AbstractMetricsRecord {
   protected static final String DEFAULT_CONTEXT = "default";
 
   private final long timestamp;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index 8837c02b99d4b..6c5a71a708fda 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -280,7 +280,6 @@ T register(final String name, final String description, final T sink) {
       }
       return sink;
     }
-    allSinks.put(name, sink);
     if (config != null) {
       registerSink(name, description, sink);
     }
@@ -301,6 +300,7 @@ synchronized void registerSink(String name, String desc, MetricsSink sink) {
         ? newSink(name, desc, sink, conf)
         : newSink(name, desc, sink, config.subset(SINK_KEY));
     sinks.put(name, sa);
+    allSinks.put(name, sink);
     sa.start();
     LOG.info("Registered sink "+ name);
   }
@@ -508,6 +508,7 @@ private synchronized void configureSinks() {
             conf.getString(DESC_KEY, sinkName), conf);
         sa.start();
         sinks.put(sinkName, sa);
+        allSinks.put(sinkName, sa.sink());
       } catch (Exception e) {
         LOG.warn("Error creating sink '"+ sinkName +"'", e);
       }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
index 6a52bf382df46..126601fcbb618 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
@@ -69,7 +69,7 @@ private final boolean compareAndSet(float expect, float update) {
 
   private void incr(float delta) {
     while (true) {
-      float current = value.get();
+      float current = Float.intBitsToFloat(value.get());
       float next = current + delta;
       if (compareAndSet(current, next)) {
         setChanged();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
index c7adaa5d9917f..e4886cb603e4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
@@ -146,8 +146,10 @@ protected MetricsInfo getInfo(Class<?> cls, Metrics annotation) {
   }
 
   /**
-   * Remove the prefix "get", if any, from the method name. Return the
+   * @return Remove the prefix "get", if any, from the method name. Return the
    * capacitalized method name."
+   *
+   * @param method input method.
    */
   protected String getName(Method method) {
     String methodName = method.getName();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java
index 19696bd839400..90b5da01c062f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java
@@ -19,11 +19,10 @@
 package org.apache.hadoop.metrics2.lib;
 
 import java.lang.reflect.Method;
+import java.util.HashSet;
 import java.util.Set;
 
 import static org.apache.hadoop.util.Preconditions.*;
-import org.apache.hadoop.util.Sets;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -44,7 +43,7 @@
 public class MutableRates extends MutableMetric {
   static final Logger LOG = LoggerFactory.getLogger(MutableRates.class);
   private final MetricsRegistry registry;
-  private final Set<Class<?>> protocolCache = Sets.newHashSet();
+  private final Set<Class<?>> protocolCache = new HashSet<>();
 
   MutableRates(MetricsRegistry registry) {
     this.registry = checkNotNull(registry, "metrics registry");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
index dc37f96f4f449..60b33a84b519b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
@@ -18,9 +18,9 @@
 
 package org.apache.hadoop.metrics2.lib;
 
-import org.apache.hadoop.util.Sets;
 import java.lang.ref.WeakReference;
 import java.lang.reflect.Method;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
@@ -52,7 +52,7 @@ public class MutableRatesWithAggregation extends MutableMetric {
       LoggerFactory.getLogger(MutableRatesWithAggregation.class);
   private final Map<String, MutableRate> globalMetrics =
       new ConcurrentHashMap<>();
-  private final Set<Class<?>> protocolCache = Sets.newHashSet();
+  private final Set<Class<?>> protocolCache = new HashSet<>();
 
   private final ConcurrentLinkedDeque<WeakReference<ConcurrentMap<String, ThreadSafeSampleStat>>>
       weakReferenceQueue = new ConcurrentLinkedDeque<>();
@@ -72,7 +72,7 @@ public void init(Class<?> protocol) {
       return;
     }
     protocolCache.add(protocol);
-    for (Method method : protocol.getDeclaredMethods()) {
+    for (Method method : protocol.getMethods()) {
       String name = method.getName();
       LOG.debug(name);
       addMetricIfNotExists(name);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
index 193ed0f71d7c7..016ecdd4d1e99 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
@@ -139,7 +139,7 @@ public long getSnapshotTimeStamp() {
 
   /**
    * Constructor for {@link MutableRollingAverages}.
-   * @param metricValueName
+   * @param metricValueName input metricValueName.
    */
   public MutableRollingAverages(String metricValueName) {
     if (metricValueName == null) {
@@ -167,7 +167,7 @@ synchronized void replaceScheduledTask(int windows, long interval,
   }
 
   @Override
-  public void snapshot(MetricsRecordBuilder builder, boolean all) {
+  public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
       for (final Entry<String, LinkedBlockingDeque<SumAndCount>> entry
           : averages.entrySet()) {
@@ -179,8 +179,11 @@ public void snapshot(MetricsRecordBuilder builder, boolean all) {
         long totalCount = 0;
 
         for (final SumAndCount sumAndCount : entry.getValue()) {
-          totalCount += sumAndCount.getCount();
-          totalSum += sumAndCount.getSum();
+          if (Time.monotonicNow() - sumAndCount.getSnapshotTimeStamp()
+              < recordValidityMs) {
+            totalCount += sumAndCount.getCount();
+            totalSum += sumAndCount.getSum();
+          }
         }
 
         if (totalCount != 0) {
@@ -282,6 +285,7 @@ public void close() throws IOException {
    * Retrieve a map of metric name {@literal ->} (aggregate).
    * Filter out entries that don't have at least minSamples.
    *
+   * @param minSamples input minSamples.
    * @return a map of peer DataNode Id to the average latency to that
    *         node seen over the measurement period.
    */
@@ -311,6 +315,7 @@ public synchronized Map<String, Double> getStats(long minSamples) {
 
   /**
    * Use for test only.
+   * @param value input value.
    */
   @VisibleForTesting
   public synchronized void setRecordValidityMs(long value) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
index e04b4b58ece0b..b130aa6ada398 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
@@ -140,14 +140,14 @@ public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
     if (all || changed()) {
       numSamples += intervalStat.numSamples();
       builder.addCounter(numInfo, numSamples)
-             .addGauge(avgInfo, lastStat().mean());
+             .addGauge(avgInfo, intervalStat.mean());
       if (extended) {
-        builder.addGauge(stdevInfo, lastStat().stddev())
-               .addGauge(iMinInfo, lastStat().min())
-               .addGauge(iMaxInfo, lastStat().max())
+        builder.addGauge(stdevInfo, intervalStat.stddev())
+               .addGauge(iMinInfo, intervalStat.min())
+               .addGauge(iMaxInfo, intervalStat.max())
                .addGauge(minInfo, minMax.min())
                .addGauge(maxInfo, minMax.max())
-               .addGauge(iNumInfo, lastStat().numSamples());
+               .addGauge(iNumInfo, intervalStat.numSamples());
       }
       if (changed()) {
         if (numSamples > 0) {
@@ -179,7 +179,7 @@ public void resetMinMax() {
   }
 
   /**
-   * Return the SampleStat snapshot timestamp
+   * @return Return the SampleStat snapshot timestamp.
    */
   public long getSnapshotTimeStamp() {
     return snapshotTimeStamp;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
index 8fd3b33b3a253..196469be9dce2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/package-info.java
@@ -87,7 +87,7 @@ usually does not need to reference any class here.
   <h2><a name="gettingstarted">Getting started</a></h2>
   <h3>Implementing metrics sources</h3>
   <table width="99%" border="1" cellspacing="0" cellpadding="4">
-    <caption></caption>
+    <caption>Implementing metrics sources</caption>
     <tbody>
       <tr>
         <th>Using annotations</th><th>Using MetricsSource interface</th>
@@ -290,10 +290,10 @@ metrics system decouples the concept for context (for grouping) with the
     backend that can handle multiple contexts (file, gangalia etc.):
   </p>
   <table width="99%" border="1" cellspacing="0" cellpadding="4">
-    <caption></caption>
+    <caption>Migration from previous system</caption>
     <tbody>
       <tr>
-        <th width="40%">Before</th><th>After</th>
+        <th style='width="40%"'>Before</th><th>After</th>
       </tr>
       <tr>
         <td><pre>
@@ -312,10 +312,10 @@ backend that can handle multiple contexts (file, gangalia etc.):
     using the context option in the sink options like the following:
   </p>
   <table width="99%" border="1" cellspacing="0" cellpadding="4">
-    <caption></caption>
+    <caption>Metrics2</caption>
     <tbody>
       <tr>
-        <th width="40%">Before</th><th>After</th>
+        <th style='width="40%"'>Before</th><th>After</th>
       </tr>
       <tr>
         <td><pre>
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
index ea1bde3a75e03..e07260c99936f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
@@ -21,6 +21,7 @@
 import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.metrics2.AbstractMetric;
 import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsRecord;
@@ -37,171 +38,173 @@
 import java.nio.charset.StandardCharsets;
 
 /**
- * A metrics sink that writes to a Graphite server
+ * A metrics sink that writes to a Graphite server.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class GraphiteSink implements MetricsSink, Closeable {
-    private static final Logger LOG =
-        LoggerFactory.getLogger(GraphiteSink.class);
-    private static final String SERVER_HOST_KEY = "server_host";
-    private static final String SERVER_PORT_KEY = "server_port";
-    private static final String METRICS_PREFIX = "metrics_prefix";
-    private String metricsPrefix = null;
-    private Graphite graphite = null;
-
-    @Override
-    public void init(SubsetConfiguration conf) {
-        // Get Graphite host configurations.
-        final String serverHost = conf.getString(SERVER_HOST_KEY);
-        final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY));
-
-        // Get Graphite metrics graph prefix.
-        metricsPrefix = conf.getString(METRICS_PREFIX);
-        if (metricsPrefix == null)
-            metricsPrefix = "";
-
-        graphite = new Graphite(serverHost, serverPort);
-        graphite.connect();
+  private static final Logger LOG =
+      LoggerFactory.getLogger(GraphiteSink.class);
+  private static final String SERVER_HOST_KEY = "server_host";
+  private static final String SERVER_PORT_KEY = "server_port";
+  private static final String METRICS_PREFIX = "metrics_prefix";
+  private String metricsPrefix = null;
+  private Graphite graphite = null;
+
+  @Override
+  public void init(SubsetConfiguration conf) {
+    // Get Graphite host configurations.
+    final String serverHost = conf.getString(SERVER_HOST_KEY);
+    final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY));
+
+    // Get Graphite metrics graph prefix.
+    metricsPrefix = conf.getString(METRICS_PREFIX);
+    if (metricsPrefix == null) {
+      metricsPrefix = "";
     }
 
-    @Override
-    public void putMetrics(MetricsRecord record) {
-        StringBuilder lines = new StringBuilder();
-        StringBuilder metricsPathPrefix = new StringBuilder();
-
-        // Configure the hierarchical place to display the graph.
-        metricsPathPrefix.append(metricsPrefix).append(".")
-                .append(record.context()).append(".").append(record.name());
-
-        for (MetricsTag tag : record.tags()) {
-            if (tag.value() != null) {
-                metricsPathPrefix.append(".")
-                    .append(tag.name())
-                    .append("=")
-                    .append(tag.value());
-            }
-        }
-
-        // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds.
-        long timestamp = record.timestamp() / 1000L;
+    graphite = new Graphite(serverHost, serverPort);
+    graphite.connect();
+  }
+
+  @Override
+  public void putMetrics(MetricsRecord record) {
+    StringBuilder lines = new StringBuilder();
+    StringBuilder metricsPathPrefix = new StringBuilder();
+
+    // Configure the hierarchical place to display the graph.
+    metricsPathPrefix.append(metricsPrefix).append(".")
+        .append(record.context()).append(".").append(record.name());
+
+    for (MetricsTag tag : record.tags()) {
+      if (tag.value() != null) {
+        metricsPathPrefix.append(".")
+            .append(tag.name())
+            .append("=")
+            .append(tag.value());
+      }
+    }
 
-        // Collect datapoints.
-        for (AbstractMetric metric : record.metrics()) {
-            lines.append(
-                    metricsPathPrefix.toString() + "."
-                            + metric.name().replace(' ', '.')).append(" ")
-                    .append(metric.value()).append(" ").append(timestamp)
-                    .append("\n");
-        }
+    // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds.
+    long timestamp = record.timestamp() / 1000L;
 
-        try {
-          graphite.write(lines.toString());
-        } catch (Exception e) {
-          LOG.warn("Error sending metrics to Graphite", e);
-          try {
-            graphite.close();
-          } catch (Exception e1) {
-            throw new MetricsException("Error closing connection to Graphite", e1);
-          }
-        }
+    // Collect datapoints.
+    for (AbstractMetric metric : record.metrics()) {
+      lines.append(metricsPathPrefix + "." + metric.name().replace(' ', '.')).append(" ")
+           .append(metric.value()).append(" ").append(timestamp)
+           .append("\n");
     }
 
-    @Override
-    public void flush() {
+    try {
+      graphite.write(lines.toString());
+    } catch (Exception e) {
+      LOG.warn("Error sending metrics to Graphite.", e);
       try {
-        graphite.flush();
-      } catch (Exception e) {
-        LOG.warn("Error flushing metrics to Graphite", e);
-        try {
-          graphite.close();
-        } catch (Exception e1) {
-          throw new MetricsException("Error closing connection to Graphite", e1);
-        }
+        graphite.close();
+      } catch (Exception e1) {
+        throw new MetricsException("Error closing connection to Graphite", e1);
       }
     }
-
-    @Override
-    public void close() throws IOException {
-      graphite.close();
+  }
+
+  @Override
+  public void flush() {
+    try {
+      graphite.flush();
+    } catch (Exception e) {
+      LOG.warn("Error flushing metrics to Graphite.", e);
+      try {
+        graphite.close();
+      } catch (Exception e1) {
+        throw new MetricsException("Error closing connection to Graphite.", e1);
+      }
     }
+  }
 
-    public static class Graphite {
-      private final static int MAX_CONNECTION_FAILURES = 5;
+  @Override
+  public void close() throws IOException {
+    graphite.close();
+  }
 
-      private String serverHost;
-      private int serverPort;
-      private Writer writer = null;
-      private Socket socket = null;
-      private int connectionFailures = 0;
+  public static class Graphite {
+    private final static int MAX_CONNECTION_FAILURES = 5;
 
-      public Graphite(String serverHost, int serverPort) {
-        this.serverHost = serverHost;
-        this.serverPort = serverPort;
-      }
+    private String serverHost;
+    private int serverPort;
+    private Writer writer = null;
+    private Socket socket = null;
+    private int connectionFailures = 0;
 
-      public void connect() {
-        if (isConnected()) {
-          throw new MetricsException("Already connected to Graphite");
-        }
-        if (tooManyConnectionFailures()) {
-          // return silently (there was ERROR in logs when we reached limit for the first time)
-          return;
-        }
-        try {
+    public Graphite(String serverHost, int serverPort) {
+      this.serverHost = serverHost;
+      this.serverPort = serverPort;
+    }
+
+    public void connect() {
+      if (isConnected()) {
+        throw new MetricsException("Already connected to Graphite");
+      }
+      if (tooManyConnectionFailures()) {
+        // return silently (there was ERROR in logs when we reached limit for the first time)
+        return;
+      }
+      try {
           // Open a connection to Graphite server.
-          socket = new Socket(serverHost, serverPort);
+        socket = new Socket(serverHost, serverPort);
         writer = new OutputStreamWriter(socket.getOutputStream(),
                 StandardCharsets.UTF_8);
-        } catch (Exception e) {
-          connectionFailures++;
-          if (tooManyConnectionFailures()) {
-            // first time when connection limit reached, report to logs
-            LOG.error("Too many connection failures, would not try to connect again.");
-          }
-          throw new MetricsException("Error creating connection, "
-              + serverHost + ":" + serverPort, e);
+      } catch (Exception e) {
+        connectionFailures++;
+        if (tooManyConnectionFailures()) {
+          // first time when connection limit reached, report to logs
+          LOG.error("Too many connection failures, would not try to connect again.");
         }
+        throw new MetricsException("Error creating connection, " +
+            serverHost + ":" + serverPort, e);
       }
+    }
 
-      public void write(String msg) throws IOException {
-        if (!isConnected()) {
-          connect();
-        }
-        if (isConnected()) {
-          writer.write(msg);
-        }
+    public void write(String msg) throws IOException {
+      if (!isConnected()) {
+        connect();
       }
-
-      public void flush() throws IOException {
-        if (isConnected()) {
-          writer.flush();
-        }
+      if (isConnected()) {
+        writer.write(msg);
       }
+    }
 
-      public boolean isConnected() {
-        return socket != null && socket.isConnected() && !socket.isClosed();
+    public void flush() throws IOException {
+      if (isConnected()) {
+        writer.flush();
       }
+    }
 
-      public void close() throws IOException {
-        try {
-          if (writer != null) {
-            writer.close();
-          }
-        } catch (IOException ex) {
-          if (socket != null) {
-            socket.close();
-          }
-        } finally {
-          socket = null;
-          writer = null;
-        }
-      }
+    public boolean isConnected() {
+      return socket != null && socket.isConnected() && !socket.isClosed();
+    }
 
-      private boolean tooManyConnectionFailures() {
-        return connectionFailures > MAX_CONNECTION_FAILURES;
+    public void close() throws IOException {
+      try {
+        if (writer != null) {
+          writer.close();
+        }
+      } catch (IOException ex) {
+        if (socket != null) {
+          socket.close();
+        }
+      } finally {
+        socket = null;
+        writer = null;
       }
+    }
 
+    private boolean tooManyConnectionFailures() {
+      return connectionFailures > MAX_CONNECTION_FAILURES;
     }
+  }
 
+  @VisibleForTesting
+  void setGraphite(Graphite graphite) {
+    this.graphite = graphite;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java
index a59ad5f227be5..9024203700ee1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/PrometheusMetricsSink.java
@@ -84,6 +84,10 @@ public void putMetrics(MetricsRecord metricsRecord) {
   /**
    * Convert CamelCase based names to lower-case names where the separator
    * is the underscore, to follow prometheus naming conventions.
+   *
+   * @param metricName metricName.
+   * @param recordName recordName.
+   * @return prometheusName.
    */
   public String prometheusName(String recordName,
                                String metricName) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
index d1ec47fdecb31..4f41c0b0057ce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java
@@ -28,6 +28,7 @@
 import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.metrics2.AbstractMetric;
 import org.apache.hadoop.metrics2.MetricType;
 import org.apache.hadoop.metrics2.MetricsException;
@@ -214,5 +215,8 @@ public void close() throws IOException {
     }
 
   }
-
+  @VisibleForTesting
+  void setStatsd(StatsD statsd) {
+    this.statsd = statsd;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
index 804e90330fba3..620f2f2faa679 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -78,6 +79,10 @@ public abstract class AbstractGangliaSink implements MetricsSink {
   private int offset;
   private boolean supportSparseMetrics = SUPPORT_SPARSE_METRICS_DEFAULT;
 
+  public List<? extends SocketAddress> getMetricsServers() {
+    return metricsServers;
+  }
+
   /**
    * Used for visiting Metrics
    */
@@ -133,8 +138,11 @@ public void init(SubsetConfiguration conf) {
     }
 
     // load the gannglia servers from properties
-    metricsServers = Servers.parse(conf.getString(SERVERS_PROPERTY),
-        DEFAULT_PORT);
+    List<String> serversFromConf =
+        conf.getList(String.class, SERVERS_PROPERTY, new ArrayList<String>());
+    metricsServers =
+        Servers.parse(serversFromConf.size() > 0 ? String.join(",", serversFromConf) : null,
+            DEFAULT_PORT);
     multicastEnabled = conf.getBoolean(MULTICAST_ENABLED_PROPERTY,
             DEFAULT_MULTICAST_ENABLED);
     multicastTtl = conf.getInt(MULTICAST_TTL_PROPERTY, DEFAULT_MULTICAST_TTL);
@@ -212,7 +220,7 @@ private void loadGangliaConf(GangliaConfType gtype) {
   /**
    * Lookup GangliaConf from cache. If not found, return default values
    *
-   * @param metricName
+   * @param metricName metricName.
    * @return looked up GangliaConf
    */
   protected GangliaConf getGangliaConfForMetric(String metricName) {
@@ -253,6 +261,7 @@ private void pad() {
 
   /**
    * Puts an integer into the buffer as 4 bytes, big-endian.
+   * @param i i.
    */
   protected void xdr_int(int i) {
     buffer[offset++] = (byte) ((i >> 24) & 0xff);
@@ -263,7 +272,7 @@ protected void xdr_int(int i) {
 
   /**
    * Sends Ganglia Metrics to the configured hosts
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected void emitToGangliaHosts() throws IOException {
     try {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
index 3e8314ee884d8..196824f433c81 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
@@ -216,7 +216,7 @@ private GangliaSlope calculateSlope(GangliaConf gConf,
    * @param value The value of the metric
    * @param gConf The GangliaConf for this metric
    * @param gSlope The slope for this metric
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected void emitMetric(String groupName, String name, String type,
       String value, GangliaConf gConf, GangliaSlope gSlope) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink31.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink31.java
index 5aebff8c031a9..fae0d4e85e1ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink31.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink31.java
@@ -42,7 +42,7 @@ public class GangliaSink31 extends GangliaSink30 {
    * @param value The value of the metric
    * @param gConf The GangliaConf for this metric
    * @param gSlope The slope for this metric
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   protected void emitMetric(String groupName, String name, String type,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
index 20b1cd6051961..58081449ee74f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java
@@ -63,8 +63,8 @@ private MBeans() {
    * Where the {@literal <serviceName> and <nameName>} are the supplied
    * parameters.
    *
-   * @param serviceName
-   * @param nameName
+   * @param serviceName serviceName.
+   * @param nameName nameName.
    * @param theMbean - the MBean to register
    * @return the named used to register the MBean
    */
@@ -79,8 +79,8 @@ static public ObjectName register(String serviceName, String nameName,
    * Where the {@literal <serviceName> and <nameName>} are the supplied
    * parameters.
    *
-   * @param serviceName
-   * @param nameName
+   * @param serviceName serviceName.
+   * @param nameName nameName.
    * @param properties - Key value pairs to define additional JMX ObjectName
    *                     properties.
    * @param theMbean    - the MBean to register
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
index 737ccc0d788dd..46a9d35f9d242 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleQuantiles.java
@@ -108,7 +108,7 @@ private double allowableError(int rank) {
   /**
    * Add a new value from the stream.
    * 
-   * @param v
+   * @param v v.
    */
   synchronized public void insert(long v) {
     buffer[bufferCount] = v;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java
index 23abfc4bedc31..dec7033424fbe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java
@@ -27,33 +27,29 @@
 public class SampleStat {
   private final MinMax minmax = new MinMax();
   private long numSamples = 0;
-  private double a0, a1, s0, s1, total;
+  private double mean, s;
 
   /**
    * Construct a new running sample stat
    */
   public SampleStat() {
-    a0 = s0 = 0.0;
-    total = 0.0;
+    mean = 0.0;
+    s = 0.0;
   }
 
   public void reset() {
     numSamples = 0;
-    a0 = s0 = 0.0;
-    total = 0.0;
+    mean = 0.0;
+    s = 0.0;
     minmax.reset();
   }
 
   // We want to reuse the object, sometimes.
-  void reset(long numSamples, double a0, double a1, double s0, double s1,
-      double total, MinMax minmax) {
-    this.numSamples = numSamples;
-    this.a0 = a0;
-    this.a1 = a1;
-    this.s0 = s0;
-    this.s1 = s1;
-    this.total = total;
-    this.minmax.reset(minmax);
+  void reset(long numSamples1, double mean1, double s1, MinMax minmax1) {
+    numSamples = numSamples1;
+    mean = mean1;
+    s = s1;
+    minmax.reset(minmax1);
   }
 
   /**
@@ -61,7 +57,7 @@ void reset(long numSamples, double a0, double a1, double s0, double s1,
    * @param other the destination to hold our values
    */
   public void copyTo(SampleStat other) {
-    other.reset(numSamples, a0, a1, s0, s1, total, minmax);
+    other.reset(numSamples, mean, s, minmax);
   }
 
   /**
@@ -78,24 +74,22 @@ public SampleStat add(double x) {
    * Add some sample and a partial sum to the running stat.
    * Note, min/max is not evaluated using this method.
    * @param nSamples  number of samples
-   * @param x the partial sum
+   * @param xTotal the partial sum
    * @return  self
    */
-  public SampleStat add(long nSamples, double x) {
+  public SampleStat add(long nSamples, double xTotal) {
     numSamples += nSamples;
-    total += x;
 
-    if (numSamples == 1) {
-      a0 = a1 = x;
-      s0 = 0.0;
-    }
-    else {
-      // The Welford method for numerical stability
-      a1 = a0 + (x - a0) / numSamples;
-      s1 = s0 + (x - a0) * (x - a1);
-      a0 = a1;
-      s0 = s1;
-    }
+    // use the weighted incremental version of Welford's algorithm to get
+    // numerical stability while treating the samples as being weighted
+    // by nSamples
+    // see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+
+    double x = xTotal / nSamples;
+    double meanOld = mean;
+
+    mean += ((double) nSamples / numSamples) * (x - meanOld);
+    s += nSamples * (x - meanOld) * (x - mean);
     return this;
   }
 
@@ -110,21 +104,21 @@ public long numSamples() {
    * @return the total of all samples added
    */
   public double total() {
-    return total;
+    return mean * numSamples;
   }
 
   /**
    * @return  the arithmetic mean of the samples
    */
   public double mean() {
-    return numSamples > 0 ? (total / numSamples) : 0.0;
+    return numSamples > 0 ? mean : 0.0;
   }
 
   /**
    * @return  the variance of the samples
    */
   public double variance() {
-    return numSamples > 1 ? s1 / (numSamples - 1) : 0.0;
+    return numSamples > 1 ? s / (numSamples - 1) : 0.0;
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
index f050219398721..5a13b00098a44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/AbstractDNSToSwitchMapping.java
@@ -115,7 +115,7 @@ public String dumpTopology() {
     builder.append("Mapping: ").append(toString()).append("\n");
     if (rack != null) {
       builder.append("Map:\n");
-      Set<String> switches = new HashSet<String>();
+      Set<String> switches = new HashSet<>();
       for (Map.Entry<String, String> entry : rack.entrySet()) {
         builder.append("  ")
             .append(entry.getKey())
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
index 83be2f1579f53..53bb44fb2e9ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
@@ -142,8 +142,12 @@ private static LinkedHashSet<InetAddress> getSubinterfaceInetAddrs(
   }
 
   /**
-   * Like {@link DNS#getIPs(String, boolean)}, but returns all
+   * @return Like {@link DNS#getIPs(String, boolean)}, but returns all
    * IPs associated with the given interface and its subinterfaces.
+   *
+   * @param strInterface input strInterface.
+   * @throws UnknownHostException
+   * If no IP address for the local host could be found.
    */
   public static String[] getIPs(String strInterface)
       throws UnknownHostException {
@@ -346,6 +350,8 @@ public static String[] getHosts(String strInterface)
    *            The name of the network interface to query (e.g. eth0)
    * @param nameserver
    *            The DNS host name
+   * @param tryfallbackResolution
+   *            Input tryfallbackResolution.
    * @return The default host names associated with IPs bound to the network
    *         interface
    * @throws UnknownHostException
@@ -385,7 +391,7 @@ public static String getDefaultHost(@Nullable String strInterface)
   }
 
   /**
-   * Returns the default (first) host name associated by the provided
+   * @return Returns the default (first) host name associated by the provided
    * nameserver with the address bound to the specified network interface.
    *
    * @param strInterface
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java
index 5866e2960fe1b..ce962bf9e8c6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java
@@ -18,6 +18,10 @@
 
 package org.apache.hadoop.net;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.naming.NamingException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
@@ -27,6 +31,10 @@
  * fully qualified domain names belonging to the IPs from this host name
  */
 public class DNSDomainNameResolver implements DomainNameResolver {
+
+  private final static Logger LOG =
+      LoggerFactory.getLogger(DNSDomainNameResolver.class.getName());
+
   @Override
   public InetAddress[] getAllByDomainName(String domainName)
       throws UnknownHostException {
@@ -40,6 +48,16 @@ public String getHostnameByIP(InetAddress address) {
         && host.charAt(host.length()-1) == '.') {
       host = host.substring(0, host.length()-1);
     }
+    // Protect against the Java behaviour of returning the IP address as a string from a cache
+    // instead of performing a reverse lookup.
+    if (host != null && host.equals(address.getHostAddress())) {
+      LOG.debug("IP address returned for FQDN detected: {}", address.getHostAddress());
+      try {
+        return DNS.reverseDns(address, null);
+      } catch (NamingException lookupFailure) {
+        LOG.warn("Failed to perform reverse lookup: {}", address);
+      }
+    }
     return host;
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
index 1e6f5f500849f..d29c6e3077df5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSToSwitchMapping.java
@@ -65,6 +65,8 @@ public interface DNSToSwitchMapping {
    *
    * If there is a cache on these nodes, this method will clear it, so that 
    * future accesses will see updated data.
+   *
+   * @param names input names.
    */
   public void reloadCachedMappings(List<String> names);
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java
index 4c44e9da4c063..debfe2feaa8b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java
@@ -30,9 +30,10 @@ public interface DomainNameResolver {
    * Takes one domain name and returns its IP addresses based on the actual
    * service discovery methods.
    *
-   * @param domainName
+   * @param domainName input domainName.
    * @return all IP addresses
-   * @throws UnknownHostException
+   * @throws UnknownHostException indicate that the IP address of a
+   * host could not be determined.
    */
   InetAddress[] getAllByDomainName(String domainName)
       throws UnknownHostException;
@@ -40,7 +41,7 @@ InetAddress[] getAllByDomainName(String domainName)
   /**
    * Reverse lookup an IP address and get the fully qualified domain name(fqdn).
    *
-   * @param address
+   * @param address input address.
    * @return fully qualified domain name
    */
   String getHostnameByIP(InetAddress address);
@@ -52,10 +53,12 @@ InetAddress[] getAllByDomainName(String domainName)
    * This function is necessary in secure environment since Kerberos uses fqdn
    * in the service principal instead of IP.
    *
-   * @param domainName
+   * @param domainName input domainName.
+   * @param useFQDN input useFQDN.
    * @return all fully qualified domain names belonging to the IPs resolved from
    * the input domainName
-   * @throws UnknownHostException
+   * @throws UnknownHostException indicate that the IP address of a
+   * host could not be determined.
    */
    String[] getAllResolvedHostnameByDomainName(
        String domainName, boolean useFQDN) throws UnknownHostException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
index efd1cc07d44bd..df4a01af27bc9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNode.java
@@ -27,7 +27,10 @@
 @InterfaceStability.Unstable
 public interface InnerNode extends Node {
   interface Factory<N extends InnerNode> {
-    /** Construct an InnerNode from a path-like string */
+    /**
+     * @return Construct an InnerNode from a path-like string.
+     * @param path input path.
+     */
     N newInnerNode(String path);
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
index 923515b6efe7e..1dd3105080778 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java
@@ -41,13 +41,22 @@ public InnerNodeImpl newInnerNode(String path) {
   protected final Map<String, Node> childrenMap = new HashMap<>();
   protected int numOfLeaves;
 
-  /** Construct an InnerNode from a path-like string. */
+  /**
+   * Construct an InnerNode from a path-like string.
+   * @param path input path.
+   */
   protected InnerNodeImpl(String path) {
     super(path);
   }
 
-  /** Construct an InnerNode
-   * from its name, its network location, its parent, and its level. */
+  /**
+   * Construct an InnerNode
+   * from its name, its network location, its parent, and its level.
+   * @param name input name.
+   * @param location input location.
+   * @param parent input parent.
+   * @param level input level.
+   */
   protected InnerNodeImpl(String name, String location,
       InnerNode parent, int level) {
     super(name, location, parent, level);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index 4b924af03c196..c49706d66f27d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -133,7 +133,8 @@ public static SocketFactory getDefaultSocketFactory(Configuration conf) {
    * Get the socket factory corresponding to the given proxy URI. If the
    * given proxy URI corresponds to an absence of configuration parameter,
    * returns null. If the URI is malformed raises an exception.
-   * 
+   *
+   * @param conf configuration.
    * @param propValue the property which is the class name of the
    *        SocketFactory to instantiate; assumed non null and non empty.
    * @return a socket factory as defined in the property value.
@@ -151,19 +152,26 @@ public static SocketFactory getSocketFactoryFromProperty(
   }
 
   /**
-   * Util method to build socket addr from either:
+   * Util method to build socket addr from either.
    *   {@literal <host>:<port>}
    *   {@literal <fs>://<host>:<port>/<path>}
+   *
+   * @param target target.
+   * @return socket addr.
    */
   public static InetSocketAddress createSocketAddr(String target) {
     return createSocketAddr(target, -1);
   }
 
   /**
-   * Util method to build socket addr from either:
+   * Util method to build socket addr from either.
    *   {@literal <host>}
    *   {@literal <host>:<port>}
    *   {@literal <fs>://<host>:<port>/<path>}
+   *
+   * @param target target.
+   * @param defaultPort default port.
+   * @return socket addr.
    */
   public static InetSocketAddress createSocketAddr(String target,
                                                    int defaultPort) {
@@ -183,6 +191,7 @@ public static InetSocketAddress createSocketAddr(String target,
    * @param configName the name of the configuration from which
    *                   <code>target</code> was loaded. This is used in the
    *                   exception message in the case that parsing fails.
+   * @return socket addr.
    */
   public static InetSocketAddress createSocketAddr(String target,
                                                    int defaultPort,
@@ -204,6 +213,7 @@ public static InetSocketAddress createSocketAddr(String target,
    *                   <code>target</code> was loaded. This is used in the
    *                   exception message in the case that parsing fails.
    * @param useCacheIfPresent Whether use cache when create URI
+   * @return  socket addr
    */
   public static InetSocketAddress createSocketAddr(String target,
                                                    int defaultPort,
@@ -361,8 +371,8 @@ private static String canonicalizeHost(String host) {
    * daemons, one can set up mappings from those hostnames to "localhost".
    * {@link NetUtils#getStaticResolution(String)} can be used to query for
    * the actual hostname. 
-   * @param host
-   * @param resolvedName
+   * @param host the hostname or IP use to instantiate the object.
+   * @param resolvedName resolved name.
    */
   public static void addStaticResolution(String host, String resolvedName) {
     synchronized (hostToResolved) {
@@ -374,7 +384,7 @@ public static void addStaticResolution(String host, String resolvedName) {
    * Retrieves the resolved name for the passed host. The resolved name must
    * have been set earlier using 
    * {@link NetUtils#addStaticResolution(String, String)}
-   * @param host
+   * @param host the hostname or IP use to instantiate the object.
    * @return the resolution
    */
   public static String getStaticResolution(String host) {
@@ -410,7 +420,7 @@ public static List <String[]> getAllStaticResolutions() {
    * the server binds to "0.0.0.0". This returns "hostname:port" of the server,
    * or "127.0.0.1:port" when the getListenerAddress() returns "0.0.0.0:port".
    * 
-   * @param server
+   * @param server server.
    * @return socket address that a client can use to connect to the server.
    */
   public static InetSocketAddress getConnectAddress(Server server) {
@@ -438,8 +448,10 @@ public static InetSocketAddress getConnectAddress(InetSocketAddress addr) {
   
   /**
    * Same as <code>getInputStream(socket, socket.getSoTimeout()).</code>
-   * <br><br>
-   * 
+   *
+   * @param socket socket.
+   * @throws IOException raised on errors performing I/O.
+   * @return SocketInputWrapper for reading from the socket.
    * @see #getInputStream(Socket, long)
    */
   public static SocketInputWrapper getInputStream(Socket socket) 
@@ -462,11 +474,11 @@ public static SocketInputWrapper getInputStream(Socket socket)
    *
    * @see Socket#getChannel()
    * 
-   * @param socket
+   * @param socket socket.
    * @param timeout timeout in milliseconds. zero for waiting as
    *                long as necessary.
    * @return SocketInputWrapper for reading from the socket.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static SocketInputWrapper getInputStream(Socket socket, long timeout) 
                                            throws IOException {
@@ -494,9 +506,9 @@ public static SocketInputWrapper getInputStream(Socket socket, long timeout)
    * 
    * @see #getOutputStream(Socket, long)
    * 
-   * @param socket
+   * @param socket socket.
    * @return OutputStream for writing to the socket.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */  
   public static OutputStream getOutputStream(Socket socket) 
                                              throws IOException {
@@ -516,11 +528,11 @@ public static OutputStream getOutputStream(Socket socket)
    * 
    * @see Socket#getChannel()
    * 
-   * @param socket
+   * @param socket socket.
    * @param timeout timeout in milliseconds. This may not always apply. zero
    *        for waiting as long as necessary.
    * @return OutputStream for writing to the socket.
-   * @throws IOException   
+   * @throws IOException raised on errors performing I/O.
    */
   public static OutputStream getOutputStream(Socket socket, long timeout) 
                                              throws IOException {
@@ -541,9 +553,10 @@ public static OutputStream getOutputStream(Socket socket, long timeout)
    * 
    * @see java.net.Socket#connect(java.net.SocketAddress, int)
    * 
-   * @param socket
+   * @param socket socket.
    * @param address the remote address
    * @param timeout timeout in milliseconds
+   * @throws IOException raised on errors performing I/O.
    */
   public static void connect(Socket socket,
       SocketAddress address,
@@ -555,10 +568,11 @@ public static void connect(Socket socket,
    * Like {@link NetUtils#connect(Socket, SocketAddress, int)} but
    * also takes a local address and port to bind the socket to. 
    * 
-   * @param socket
+   * @param socket socket.
    * @param endpoint the remote address
    * @param localAddr the local address to bind the socket to
    * @param timeout timeout in milliseconds
+   * @throws IOException raised on errors performing I/O.
    */
   public static void connect(Socket socket, 
                              SocketAddress endpoint,
@@ -644,7 +658,7 @@ public static List<String> normalizeHostNames(Collection<String> names) {
    * Performs a sanity check on the list of hostnames/IPs to verify they at least
    * appear to be valid.
    * @param names - List of hostnames/IPs
-   * @throws UnknownHostException
+   * @throws UnknownHostException Unknown Host Exception.
    */
   public static void verifyHostnames(String[] names) throws UnknownHostException {
     for (String name: names) {
@@ -735,6 +749,9 @@ public static String getHostname() {
 
   /**
    * Compose a "host:port" string from the address.
+   *
+   * @param addr address.
+   * @return hort port string.
    */
   public static String getHostPortString(InetSocketAddress addr) {
     return addr.getHostName() + ":" + addr.getPort();
@@ -969,6 +986,8 @@ private static String quoteHost(final String hostname) {
   }
 
   /**
+   * isValidSubnet.
+   * @param subnet subnet.
    * @return true if the given string is a subnet specified
    *     using CIDR notation, false otherwise
    */
@@ -1004,6 +1023,7 @@ private static void addMatchingAddrs(NetworkInterface nif,
    * @param returnSubinterfaces
    *            whether to return IPs associated with subinterfaces
    * @throws IllegalArgumentException if subnet is invalid
+   * @return ips.
    */
   public static List<InetAddress> getIPs(String subnet,
       boolean returnSubinterfaces) {
@@ -1053,12 +1073,38 @@ public static int getFreeSocketPort() {
     return port;
   }
 
+  /**
+   * Return free ports. There is no guarantee they will remain free, so
+   * ports should be used immediately. The number of free ports returned by
+   * this method should match argument {@code numOfPorts}. Num of ports
+   * provided in the argument should not exceed 25.
+   *
+   * @param numOfPorts Number of free ports to acquire.
+   * @return Free ports for binding a local socket.
+   */
+  public static Set<Integer> getFreeSocketPorts(int numOfPorts) {
+    Preconditions.checkArgument(numOfPorts > 0 && numOfPorts <= 25,
+        "Valid range for num of ports is between 0 and 26");
+    final Set<Integer> freePorts = new HashSet<>(numOfPorts);
+    for (int i = 0; i < numOfPorts * 5; i++) {
+      int port = getFreeSocketPort();
+      if (port == 0) {
+        continue;
+      }
+      freePorts.add(port);
+      if (freePorts.size() == numOfPorts) {
+        return freePorts;
+      }
+    }
+    throw new IllegalStateException(numOfPorts + " free ports could not be acquired.");
+  }
+
   /**
    * Return an @{@link InetAddress} to bind to. If bindWildCardAddress is true
    * than returns null.
    *
-   * @param localAddr
-   * @param bindWildCardAddress
+   * @param localAddr local addr.
+   * @param bindWildCardAddress bind wildcard address.
    * @return InetAddress
    */
   public static InetAddress bindToLocalAddress(InetAddress localAddr, boolean
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 6e0d88f2a7b7b..50be1ab759f36 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -101,6 +101,13 @@ protected NetworkTopology init(InnerNode.Factory factory) {
   private int depthOfAllLeaves = -1;
   /** rack counter */
   protected int numOfRacks = 0;
+  /** empty rack map, rackname->nodenumber. */
+  private HashMap<String, Set<String>> rackMap =
+      new HashMap<String, Set<String>>();
+  /** decommission nodes, contained stoped nodes. */
+  private HashSet<String> decommissionNodes = new HashSet<>();
+  /** empty rack counter. */
+  private int numOfEmptyRacks = 0;
 
   /**
    * Whether or not this cluster has ever consisted of more than 1 rack,
@@ -150,6 +157,7 @@ public void add(Node node) {
         if (rack == null) {
           incrementRacks();
         }
+        interAddNodeWithEmptyRack(node);
         if (depthOfAllLeaves == -1) {
           depthOfAllLeaves = node.getLevel();
         }
@@ -224,6 +232,7 @@ public void remove(Node node) {
         if (rack == null) {
           numOfRacks--;
         }
+        interRemoveNodeWithEmptyRack(node);
       }
       LOG.debug("NetworkTopology became:\n{}", this);
     } finally {
@@ -406,14 +415,16 @@ public boolean isOnSameRack(Node node1, Node node2) {
   }
   
   /**
-   * Check if network topology is aware of NodeGroup
+   * @return Check if network topology is aware of NodeGroup.
    */
   public boolean isNodeGroupAware() {
     return false;
   }
   
   /** 
-   * Return false directly as not aware of NodeGroup, to be override in sub-class
+   * @return Return false directly as not aware of NodeGroup, to be override in sub-class.
+   * @param node1 input node1.
+   * @param node2 input node2.
    */
   public boolean isOnSameNodeGroup(Node node1, Node node2) {
     return false;
@@ -511,8 +522,7 @@ protected Node chooseRandom(final String scope, String excludedScope,
       }
     }
     if (numOfDatanodes <= 0) {
-      LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\")."
-              + " numOfDatanodes={}",
+      LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\"). numOfDatanodes={}",
           scope, excludedScope, numOfDatanodes);
       return null;
     }
@@ -528,10 +538,12 @@ protected Node chooseRandom(final String scope, String excludedScope,
         netlock.readLock().unlock();
       }
     }
-    LOG.debug("Choosing random from {} available nodes on node {},"
-        + " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.",
-        availableNodes, innerNode, scope, excludedScope, excludedNodes,
-        numOfDatanodes);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Choosing random from {} available nodes on node {}, scope={},"
+              + " excludedScope={}, excludeNodes={}. numOfDatanodes={}.",
+          availableNodes, innerNode, scope, excludedScope, excludedNodes,
+          numOfDatanodes);
+    }
     Node ret = null;
     if (availableNodes > 0) {
       ret = chooseRandom(innerNode, node, excludedNodes, numOfDatanodes,
@@ -720,11 +732,10 @@ public String toString() {
   }
   
   /**
-   * Divide networklocation string into two parts by last separator, and get 
+   * @return Divide networklocation string into two parts by last separator, and get
    * the first part here.
    * 
-   * @param networkLocation
-   * @return
+   * @param networkLocation input networkLocation.
    */
   public static String getFirstHalf(String networkLocation) {
     int index = networkLocation.lastIndexOf(NodeBase.PATH_SEPARATOR_STR);
@@ -732,11 +743,10 @@ public static String getFirstHalf(String networkLocation) {
   }
 
   /**
-   * Divide networklocation string into two parts by last separator, and get 
+   * @return Divide networklocation string into two parts by last separator, and get
    * the second part here.
    * 
-   * @param networkLocation
-   * @return
+   * @param networkLocation input networkLocation.
    */
   public static String getLastHalf(String networkLocation) {
     int index = networkLocation.lastIndexOf(NodeBase.PATH_SEPARATOR_STR);
@@ -888,7 +898,7 @@ public void sortByDistance(Node reader, Node[] nodes, int activeLen) {
    * or on a different rack from the reader. Sorting the nodes based on network
    * distance from the reader reduces network traffic and improves
    * performance.
-   * <p>
+   * </p>
    * As an additional twist, we also randomize the nodes at each network
    * distance. This helps with load balancing when there is data skew.
    *
@@ -897,6 +907,7 @@ public void sortByDistance(Node reader, Node[] nodes, int activeLen) {
    * @param activeLen Number of active nodes at the front of the array
    * @param secondarySort a secondary sorting strategy which can inject into
    *     that point from outside to help sort the same distance.
+   * @param <T> Generics Type T
    */
   public <T extends Node> void sortByDistance(Node reader, T[] nodes,
       int activeLen, Consumer<List<T>> secondarySort){
@@ -909,7 +920,7 @@ public <T extends Node> void sortByDistance(Node reader, T[] nodes,
    * is not a datanode. Sorting the nodes based on network distance
    * from the reader reduces network traffic and improves
    * performance.
-   * <p>
+   * </p>
    *
    * @param reader    Node where data will be read
    * @param nodes     Available replicas with the requested data
@@ -930,13 +941,14 @@ public void sortByDistanceUsingNetworkLocation(Node reader, Node[] nodes,
    * is not a datanode. Sorting the nodes based on network distance
    * from the reader reduces network traffic and improves
    * performance.
-   * <p>
+   * </p>
    *
    * @param reader    Node where data will be read
    * @param nodes     Available replicas with the requested data
    * @param activeLen Number of active nodes at the front of the array
    * @param secondarySort a secondary sorting strategy which can inject into
    *     that point from outside to help sort the same distance.
+   * @param <T> Generics Type T.
    */
   public <T extends Node> void sortByDistanceUsingNetworkLocation(Node reader,
       T[] nodes, int activeLen, Consumer<List<T>> secondarySort) {
@@ -1015,4 +1027,108 @@ protected static boolean isNodeInScope(Node node, String scope) {
     String nodeLocation = NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR;
     return nodeLocation.startsWith(scope);
   }
-}
\ No newline at end of file
+
+  /** @return the number of nonempty racks */
+  public int getNumOfNonEmptyRacks() {
+    return numOfRacks - numOfEmptyRacks;
+  }
+
+  /**
+   * Update empty rack number when add a node like recommission.
+   * @param node node to be added; can be null
+   */
+  public void recommissionNode(Node node) {
+    if (node == null) {
+      return;
+    }
+    if (node instanceof InnerNode) {
+      throw new IllegalArgumentException(
+          "Not allow to remove an inner node: " + NodeBase.getPath(node));
+    }
+    netlock.writeLock().lock();
+    try {
+      decommissionNodes.remove(node.getName());
+      interAddNodeWithEmptyRack(node);
+    } finally {
+      netlock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Update empty rack number when remove a node like decommission.
+   * @param node node to be added; can be null
+   */
+  public void decommissionNode(Node node) {
+    if (node == null) {
+      return;
+    }
+    if (node instanceof InnerNode) {
+      throw new IllegalArgumentException(
+          "Not allow to remove an inner node: " + NodeBase.getPath(node));
+    }
+    netlock.writeLock().lock();
+    try {
+      decommissionNodes.add(node.getName());
+      interRemoveNodeWithEmptyRack(node);
+    } finally {
+      netlock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Internal function for update empty rack number
+   * for add or recommission a node.
+   * @param node node to be added; can be null
+   */
+  private void interAddNodeWithEmptyRack(Node node) {
+    if (node == null) {
+      return;
+    }
+    String rackname = node.getNetworkLocation();
+    Set<String> nodes = rackMap.get(rackname);
+    if (nodes == null) {
+      nodes = new HashSet<>();
+    }
+    if (!decommissionNodes.contains(node.getName())) {
+      nodes.add(node.getName());
+    }
+    rackMap.put(rackname, nodes);
+    countEmptyRacks();
+  }
+
+  /**
+   * Internal function for update empty rack number
+   * for remove or decommission a node.
+   * @param node node to be removed; can be null
+   */
+  private void interRemoveNodeWithEmptyRack(Node node) {
+    if (node == null) {
+      return;
+    }
+    String rackname = node.getNetworkLocation();
+    Set<String> nodes = rackMap.get(rackname);
+    if (nodes != null) {
+      InnerNode rack = (InnerNode) getNode(node.getNetworkLocation());
+      if (rack == null) {
+        // this node and its rack are both removed.
+        rackMap.remove(rackname);
+      } else if (nodes.contains(node.getName())) {
+        // this node is decommissioned or removed.
+        nodes.remove(node.getName());
+        rackMap.put(rackname, nodes);
+      }
+      countEmptyRacks();
+    }
+  }
+
+  private void countEmptyRacks() {
+    int count = 0;
+    for (Set<String> nodes : rackMap.values()) {
+      if (nodes != null && nodes.isEmpty()) {
+        count++;
+      }
+    }
+    numOfEmptyRacks = count;
+    LOG.debug("Current numOfEmptyRacks is {}", numOfEmptyRacks);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
index 4db8155ffed3b..60ae442b4f602 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMapping.java
@@ -124,12 +124,13 @@ public String toString() {
   }
 
   /**
-   * {@inheritDoc}
+   * {@inheritDoc}.
    * <p>
    * This will get called in the superclass constructor, so a check is needed
    * to ensure that the raw mapping is defined before trying to relaying a null
    * configuration.
-   * @param conf
+   * </p>
+   * @param conf input Configuration.
    */
   @Override
   public void setConf(Configuration conf) {
@@ -212,8 +213,9 @@ public List<String> resolve(List<String> names) {
     /**
      * Build and execute the resolution command. The command is
      * executed in the directory specified by the system property
-     * "user.dir" if set; otherwise the current working directory is used
+     * "user.dir" if set; otherwise the current working directory is used.
      * @param args a list of arguments
+     * @param commandScriptName input commandScriptName.
      * @return null if the number of arguments is out of range,
      * or the output of the command.
      */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
index e05fae6496a15..4c1a547baabe0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/ScriptBasedMappingWithDependency.java
@@ -74,12 +74,13 @@ public String toString() {
   }
 
   /**
-   * {@inheritDoc}
+   * {@inheritDoc}.
    * <p>
    * This will get called in the superclass constructor, so a check is needed
    * to ensure that the raw mapping is defined before trying to relaying a null
    * configuration.
-   * @param conf
+   * </p>
+   * @param conf input Configuration.
    */
   @Override
   public void setConf(Configuration conf) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
index cfa7b01e8136a..99e646a975b22 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketInputStream.java
@@ -67,7 +67,7 @@ int performIO(ByteBuffer buf) throws IOException {
    *        Channel for reading, should also be a {@link SelectableChannel}.
    *        The channel will be configured to be non-blocking.
    * @param timeout timeout in milliseconds. must not be negative.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public SocketInputStream(ReadableByteChannel channel, long timeout)
                                                         throws IOException {
@@ -86,7 +86,7 @@ public SocketInputStream(ReadableByteChannel channel, long timeout)
    *  
    * @param socket should have a channel associated with it.
    * @param timeout timeout timeout in milliseconds. must not be negative.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public SocketInputStream(Socket socket, long timeout) 
                                          throws IOException {
@@ -103,7 +103,7 @@ public SocketInputStream(Socket socket, long timeout)
    * @see SocketInputStream#SocketInputStream(ReadableByteChannel, long)
    *  
    * @param socket should have a channel associated with it.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public SocketInputStream(Socket socket) throws IOException {
     this(socket.getChannel(), socket.getSoTimeout());
@@ -141,7 +141,7 @@ public synchronized void close() throws IOException {
   }
 
   /**
-   * Returns underlying channel used by inputstream.
+   * @return Returns underlying channel used by inputstream.
    * This is useful in certain cases like channel for 
    * {@link FileChannel#transferFrom(ReadableByteChannel, long, long)}.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
index 93f4f56d78d63..3f6ea098a7200 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketOutputStream.java
@@ -72,7 +72,7 @@ int performIO(ByteBuffer buf) throws IOException {
    *        Channel for writing, should also be a {@link SelectableChannel}.  
    *        The channel will be configured to be non-blocking.
    * @param timeout timeout in milliseconds. must not be negative.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public SocketOutputStream(WritableByteChannel channel, long timeout) 
                                                          throws IOException {
@@ -91,7 +91,7 @@ public SocketOutputStream(WritableByteChannel channel, long timeout)
    *  
    * @param socket should have a channel associated with it.
    * @param timeout timeout timeout in milliseconds. must not be negative.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public SocketOutputStream(Socket socket, long timeout) 
                                          throws IOException {
@@ -138,7 +138,7 @@ public synchronized void close() throws IOException {
   }
 
   /**
-   * Returns underlying channel used by this stream.
+   * @return Returns underlying channel used by this stream.
    * This is useful in certain cases like channel for 
    * {@link FileChannel#transferTo(long, long, WritableByteChannel)}
    */
@@ -254,7 +254,12 @@ public void transferToFully(FileChannel fileCh, long position, int count,
    * Call
    * {@link #transferToFully(FileChannel, long, int, LongWritable, LongWritable)
    * }
-   * with null <code>waitForWritableTime</code> and <code>transferToTime</code>
+   * with null <code>waitForWritableTime</code> and <code>transferToTime</code>.
+   *
+   * @param fileCh input fileCh.
+   * @param position input position.
+   * @param count input count.
+   * @throws IOException raised on errors performing I/O.
    */
   public void transferToFully(FileChannel fileCh, long position, int count)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
index 82c087737cbad..73fff0313a58c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
@@ -106,6 +106,8 @@ native static void validateSocketPathSecurity0(String path,
 
   /**
    * Return true only if UNIX domain sockets are available.
+   *
+   * @return loadingFailureReason.
    */
   public static String getLoadingFailureReason() {
     return loadingFailureReason;
@@ -184,6 +186,7 @@ private void unreference(boolean checkClosed) throws ClosedChannelException {
    *
    * @param path         The path to bind and listen on.
    * @return             The new DomainSocket.
+   * @throws IOException raised on errors performing I/O.
    */
   public static DomainSocket bindAndListen(String path) throws IOException {
     if (loadingFailureReason != null) {
@@ -387,7 +390,7 @@ public void close() throws IOException {
   /**
    * Call shutdown(SHUT_RDWR) on the UNIX domain socket.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void shutdown() throws IOException {
     refCount.reference();
@@ -413,6 +416,7 @@ private native static void sendFileDescriptors0(int fd,
    *                          one byte.
    * @param offset            The offset in the jbuf array to start at.
    * @param length            Length of the jbuf array to use.
+   * @throws IOException raised on errors performing I/O.
    */
   public void sendFileDescriptors(FileDescriptor descriptors[],
       byte jbuf[], int offset, int length) throws IOException {
@@ -433,6 +437,13 @@ private static native int receiveFileDescriptors0(int fd,
   /**
    * Receive some FileDescriptor objects from the process on the other side of
    * this socket, and wrap them in FileInputStream objects.
+   *
+   * @param streams input stream.
+   * @param buf input buf.
+   * @param offset input offset.
+   * @param length input length.
+   * @return wrap them in FileInputStream objects.
+   * @throws IOException raised on errors performing I/O.
    */
   public int recvFileInputStreams(FileInputStream[] streams, byte buf[],
         int offset, int length) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
index 6f799c1542095..deca6f1152ba4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CompositeGroupsMapping.java
@@ -109,7 +109,7 @@ public void cacheGroupsAdd(List<String> groups) throws IOException {
 
   @Override
   public synchronized Set<String> getGroupsSet(String user) throws IOException {
-    Set<String> groupSet = new HashSet<String>();
+    Set<String> groupSet = new HashSet<>();
 
     Set<String> groups = null;
     for (GroupMappingServiceProvider provider : providersList) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index e71bf6d40dd21..ef309cb2247fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -138,6 +138,8 @@ public void addToken(Text alias, Token<? extends TokenIdentifier> t) {
 
   /**
    * Return all the tokens in the in-memory map.
+   *
+   * @return all the tokens in the in-memory map.
    */
   public Collection<Token<? extends TokenIdentifier>> getAllTokens() {
     return tokenMap.values();
@@ -145,6 +147,8 @@ public Collection<Token<? extends TokenIdentifier>> getAllTokens() {
 
   /**
    * Returns an unmodifiable version of the full map of aliases to Tokens.
+   *
+   * @return TokenMap.
    */
   public Map<Text, Token<? extends TokenIdentifier>> getTokenMap() {
     return Collections.unmodifiableMap(tokenMap);
@@ -192,6 +196,8 @@ public void removeSecretKey(Text alias) {
 
   /**
    * Return all the secret key entries in the in-memory map.
+   *
+   * @return Text List.
    */
   public List<Text> getAllSecretKeys() {
     List<Text> list = new java.util.ArrayList<Text>();
@@ -202,6 +208,8 @@ public List<Text> getAllSecretKeys() {
 
   /**
    * Returns an unmodifiable version of the full map of aliases to secret keys.
+   *
+   * @return SecretKeyMap.
    */
   public Map<Text, byte[]> getSecretKeyMap() {
     return Collections.unmodifiableMap(secretKeysMap);
@@ -209,9 +217,10 @@ public Map<Text, byte[]> getSecretKeyMap() {
 
   /**
    * Convenience method for reading a token storage file and loading its Tokens.
-   * @param filename
-   * @param conf
-   * @throws IOException
+   * @param filename filename.
+   * @param conf configuration.
+   * @throws IOException  raised on errors performing I/O.
+   * @return Credentials.
    */
   public static Credentials readTokenStorageFile(Path filename,
                                                  Configuration conf)
@@ -233,9 +242,10 @@ public static Credentials readTokenStorageFile(Path filename,
 
   /**
    * Convenience method for reading a token storage file and loading its Tokens.
-   * @param filename
-   * @param conf
-   * @throws IOException
+   * @param filename filename.
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @return Token.
    */
   public static Credentials readTokenStorageFile(File filename,
                                                  Configuration conf)
@@ -256,6 +266,9 @@ public static Credentials readTokenStorageFile(File filename,
 
   /**
    * Convenience method for reading a token from a DataInputStream.
+   *
+   * @param in DataInputStream.
+   * @throws IOException raised on errors performing I/O.
    */
   public void readTokenStorageStream(DataInputStream in) throws IOException {
     byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length];
@@ -335,8 +348,8 @@ public void writeTokenStorageFile(Path filename, Configuration conf,
 
   /**
    * Stores all the keys to DataOutput.
-   * @param out
-   * @throws IOException
+   * @param out DataOutput.
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public void write(DataOutput out) throws IOException {
@@ -401,8 +414,8 @@ void readProto(DataInput in) throws IOException {
 
   /**
    * Loads all the keys.
-   * @param in
-   * @throws IOException
+   * @param in DataInput.
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public void readFields(DataInput in) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
index 3a9073bbffaba..f37089fb55a79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/GroupMappingServiceProvider.java
@@ -40,18 +40,18 @@ public interface GroupMappingServiceProvider {
    * Returns EMPTY list in case of non-existing user
    * @param user User's name
    * @return group memberships of user
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public List<String> getGroups(String user) throws IOException;
   /**
    * Refresh the cache of groups and user mapping
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void cacheGroupsRefresh() throws IOException;
   /**
    * Caches the group user information
    * @param groups list of groups to add to cache
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void cacheGroupsAdd(List<String> groups) throws IOException;
 
@@ -60,7 +60,7 @@ public interface GroupMappingServiceProvider {
    * Returns EMPTY set in case of non-existing user
    * @param user User's name
    * @return set of group memberships of user
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   default Set<String> getGroupsSet(String user) throws IOException {
     //Override to form the set directly to avoid another conversion
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index bc2ea458b33f3..1b3adc14283ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -465,7 +465,7 @@ public static Groups getUserToGroupsMappingService() {
 
   /**
    * Get the groups being used to map user-to-groups.
-   * @param conf
+   * @param conf configuration.
    * @return the groups being used to map user-to-groups.
    */
   public static synchronized Groups getUserToGroupsMappingService(
@@ -482,7 +482,7 @@ public static synchronized Groups getUserToGroupsMappingService(
 
   /**
    * Create new groups used to map user-to-groups with loaded configuration.
-   * @param conf
+   * @param conf configuration.
    * @return the groups being used to map user-to-groups.
    */
   @Private
@@ -493,4 +493,9 @@ public static synchronized Groups getUserToGroupsMappingService(
     GROUPS = new Groups(conf);
     return GROUPS;
   }
+
+  @VisibleForTesting
+  public static void reset() {
+    GROUPS = null;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
index df96c500cd08b..b66f8444528a4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java
@@ -45,7 +45,7 @@ public class HadoopKerberosName extends KerberosName {
 
   /**
    * Create a name from the full Kerberos principal name.
-   * @param name
+   * @param name name.
    */
   public HadoopKerberosName(String name) {
     super(name);
@@ -58,7 +58,7 @@ public HadoopKerberosName(String name) {
    * method should be invoked directly.
    * 
    * @param conf the new configuration
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void setConfiguration(Configuration conf) throws IOException {
     final String defaultRule;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
index 86edab7de7097..08cacdc248fa4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IdMappingServiceProvider.java
@@ -18,11 +18,9 @@
 package org.apache.hadoop.security;
 
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 
 /**
  * An interface for the implementation of {@literal <}userId,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index b2797871339e3..ee6a127f0e24f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -435,7 +435,8 @@ protected boolean isSimpleAuthentication(Configuration conf) {
    * This is a recurrent problem
    * (that is: it keeps creeping back with JVM updates);
    * a fast failure is the best tactic.
-   * @throws NoSuchAlgorithmException
+   * @throws NoSuchAlgorithmException when a particular cryptographic algorithm is
+   *                          requested but is not available in the environment.
    */
 
   protected void validateKeyLength() throws NoSuchAlgorithmException {
@@ -1046,7 +1047,7 @@ private void failif(boolean condition,
    * @param conf configuration
    * @param argv argument list
    * @return an exception
-   * @throws Exception
+   * @throws Exception Exception.
    */
   public static int exec(Configuration conf, String... argv) throws Exception {
     try(KDiag kdiag = new KDiag()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosInfo.java
index 062dcff61e1d9..e79492adf94e9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KerberosInfo.java
@@ -31,7 +31,10 @@
 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
 @InterfaceStability.Evolving
 public @interface KerberosInfo {
-  /** Key for getting server's Kerberos principal name from Configuration */
+  /**
+   * Key for getting server's Kerberos principal name from Configuration.
+   * @return serverPrincipal.
+   */
   String serverPrincipal();
   String clientPrincipal() default "";
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 17934add81413..f0ff5bd700877 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -30,6 +30,7 @@
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Hashtable;
 import java.util.Iterator;
@@ -59,6 +60,7 @@
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -251,6 +253,10 @@ public class LdapGroupsMapping
   public static final String POSIX_GID_ATTR_KEY = LDAP_CONFIG_PREFIX + ".posix.attr.gid.name";
   public static final String POSIX_GID_ATTR_DEFAULT = "gidNumber";
 
+  public static final String GROUP_SEARCH_FILTER_PATTERN =
+      LDAP_CONFIG_PREFIX + ".group.search.filter.pattern";
+  public static final String GROUP_SEARCH_FILTER_PATTERN_DEFAULT = "";
+
   /*
    * Posix attributes
    */
@@ -336,6 +342,7 @@ public class LdapGroupsMapping
   private int numAttempts;
   private volatile int numAttemptsBeforeFailover;
   private volatile String ldapCtxFactoryClassName;
+  private volatile String[] groupSearchFilterParams;
 
   /**
    * Returns list of groups for a user.
@@ -428,15 +435,22 @@ private NamingEnumeration<SearchResult> lookupPosixGroup(SearchResult result,
    * @return a list of strings representing group names of the user.
    * @throws NamingException if unable to find group names
    */
-  private Set<String> lookupGroup(SearchResult result, DirContext c,
+  @VisibleForTesting
+  Set<String> lookupGroup(SearchResult result, DirContext c,
       int goUpHierarchy)
       throws NamingException {
     Set<String> groups = new LinkedHashSet<>();
     Set<String> groupDNs = new HashSet<>();
 
     NamingEnumeration<SearchResult> groupResults;
-    // perform the second LDAP query
-    if (isPosix) {
+
+    String[] resolved = resolveCustomGroupFilterArgs(result);
+    // If custom group filter argument is supplied, use that!!!
+    if (resolved != null) {
+      groupResults =
+          c.search(groupbaseDN, groupSearchFilter, resolved, SEARCH_CONTROLS);
+    } else if (isPosix) {
+      // perform the second LDAP query
       groupResults = lookupPosixGroup(result, c);
     } else {
       String userDn = result.getNameInNamespace();
@@ -460,6 +474,25 @@ private Set<String> lookupGroup(SearchResult result, DirContext c,
     return groups;
   }
 
+  private String[] resolveCustomGroupFilterArgs(SearchResult result)
+      throws NamingException {
+    if (groupSearchFilterParams != null) {
+      String[] filterElems = new String[groupSearchFilterParams.length];
+      for (int i = 0; i < groupSearchFilterParams.length; i++) {
+        // Specific handling for userDN.
+        if (groupSearchFilterParams[i].equalsIgnoreCase("userDN")) {
+          filterElems[i] = result.getNameInNamespace();
+        } else {
+          filterElems[i] =
+              result.getAttributes().get(groupSearchFilterParams[i]).get()
+                  .toString();
+        }
+      }
+      return filterElems;
+    }
+    return null;
+  }
+
   /**
    * Perform LDAP queries to get group names of a user.
    *
@@ -510,6 +543,8 @@ Set<String> doGetGroups(String user, int goUpHierarchy)
         }
       } catch (NamingException e) {
         // If the first lookup failed, fall back to the typical scenario.
+        // In order to force the fallback, we need to reset groups collection.
+        groups.clear();
         LOG.info("Failed to get groups from the first lookup. Initiating " +
                 "the second LDAP query using the user's DN.", e);
       }
@@ -777,6 +812,12 @@ public synchronized void setConf(Configuration conf) {
         conf.get(POSIX_UID_ATTR_KEY, POSIX_UID_ATTR_DEFAULT);
     posixGidAttr =
         conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
+    String groupSearchFilterParamCSV = conf.get(GROUP_SEARCH_FILTER_PATTERN,
+        GROUP_SEARCH_FILTER_PATTERN_DEFAULT);
+    if(groupSearchFilterParamCSV!=null && !groupSearchFilterParamCSV.isEmpty()) {
+      LOG.debug("Using custom group search filters: {}", groupSearchFilterParamCSV);
+      groupSearchFilterParams = groupSearchFilterParamCSV.split(",");
+    }
 
     int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
         DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
@@ -791,7 +832,16 @@ public synchronized void setConf(Configuration conf) {
       returningAttributes = new String[] {
           groupNameAttr, posixUidAttr, posixGidAttr};
     }
-    SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
+
+    // If custom group filter is being used, fetch attributes in the filter
+    // as well.
+    ArrayList<String> customAttributes = new ArrayList<>();
+    if (groupSearchFilterParams != null) {
+      customAttributes.addAll(Arrays.asList(groupSearchFilterParams));
+    }
+    customAttributes.addAll(Arrays.asList(returningAttributes));
+    SEARCH_CONTROLS
+        .setReturningAttributes(customAttributes.toArray(new String[0]));
 
     // LDAP_CTX_FACTORY_CLASS_DEFAULT is not open to unnamed modules
     // in Java 11+, so the default value is set to null to avoid
@@ -849,7 +899,7 @@ String getPasswordFromCredentialProviders(
         password = new String(passchars);
       }
     } catch (IOException ioe) {
-      LOG.warn("Exception while trying to get password for alias {}: {}",
+      LOG.warn("Exception while trying to get password for alias {}: ",
           alias, ioe);
     }
     return password;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
index aa06c59a64814..5e466033fb713 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java
@@ -65,7 +65,7 @@ public static List<String> getNetgroupNames() {
   }
 
   private static Set<String> getGroups() {
-    Set<String> allGroups = new HashSet<String> ();
+    Set<String> allGroups = new HashSet<>();
     for (Set<String> userGroups : userToNetgroupsMap.values()) {
       allGroups.addAll(userGroups);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
index 9592ecc32c012..aebb50a0af4a4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NullGroupsMapping.java
@@ -39,7 +39,7 @@ public void cacheGroupsAdd(List<String> groups) {
    *
    * @param user User's name
    * @return set of group memberships of user
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public Set<String> getGroupsSet(String user) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
index 9c4fb64d149c3..9cd85499f5803 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
@@ -136,6 +136,7 @@ public static URI nestURIForLocalJavaKeyStoreProvider(final URI localFile)
    * @param config the existing configuration with provider path
    * @param fileSystemClass the class which providers must be compatible
    * @return Configuration clone with new provider path
+   * @throws IOException raised on errors performing I/O.
    */
   public static Configuration excludeIncompatibleCredentialProviders(
       Configuration config, Class<? extends FileSystem> fileSystemClass)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java
index 005b2948ea2a6..c4f636e374519 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java
@@ -42,14 +42,14 @@ public interface RefreshUserMappingsProtocol {
 
   /**
    * Refresh user to group mappings.
-   * @throws IOException
+   * @throws IOException  raised on errors performing I/O.
    */
   @Idempotent
   public void refreshUserToGroupsMappings() throws IOException;
   
   /**
    * Refresh superuser proxy group list
-   * @throws IOException
+   * @throws IOException  raised on errors performing I/O.
    */
   @Idempotent
   public void refreshSuperUserGroupsConfiguration() throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java
index a91a90ac7c901..2a8c3bf30c75f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslInputStream.java
@@ -281,7 +281,7 @@ public int read(byte[] b, int off, int len) throws IOException {
    * <p>
    * Fewer bytes than requested might be skipped. The actual number of bytes
    * skipped is equal to <code>n</code> or the result of a call to
-   * {@link #available() <code>available</code>}, whichever is smaller. If
+   * {@link #available()}, whichever is smaller. If
    * <code>n</code> is less than zero, no bytes are skipped.
    * 
    * <p>
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
index dd6c42e1491a8..25cc4a8144f05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java
@@ -46,7 +46,7 @@ public class SaslPropertiesResolver implements Configurable{
    * Looks up the configuration to see if there is custom class specified.
    * Constructs the instance by passing the configuration directly to the
    * constructor to achieve thread safety using final fields.
-   * @param conf
+   * @param conf configuration.
    * @return SaslPropertiesResolver
    */
   public static SaslPropertiesResolver getInstance(Configuration conf) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index 938eeeba96786..ce7878480e22c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -237,7 +237,14 @@ private SaslClient createSaslClient(SaslAuth authType)
           LOG.debug("client isn't using kerberos");
           return null;
         }
-        String serverPrincipal = getServerPrincipal(authType);
+        final String serverPrincipal;
+        try {
+          serverPrincipal = getServerPrincipal(authType);
+        } catch (IllegalArgumentException ex) {
+          // YARN-11210: getServerPrincipal can throw IllegalArgumentException if Kerberos
+          // configuration is bad, this is surfaced as a non-retryable SaslException
+          throw new SaslException("Bad Kerberos server principal configuration", ex);
+        }
         if (serverPrincipal == null) {
           LOG.debug("protocol doesn't use kerberos");
           return null;
@@ -351,9 +358,9 @@ String getServerPrincipal(SaslAuth authType) throws IOException {
   /**
    * Do client side SASL authentication with server via the given IpcStreams.
    *
-   * @param ipcStreams
+   * @param ipcStreams ipcStreams.
    * @return AuthMethod used to negotiate the connection
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public AuthMethod saslConnect(IpcStreams ipcStreams) throws IOException {
     // redefined if/when a SASL negotiation starts, can be queried if the
@@ -521,7 +528,7 @@ private boolean useWrap() {
    * 
    * @param in - InputStream used to make the connection
    * @return InputStream that may be using SASL unwrap
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public InputStream getInputStream(InputStream in) throws IOException {
     if (useWrap()) {
@@ -537,7 +544,7 @@ public InputStream getInputStream(InputStream in) throws IOException {
    * 
    * @param out - OutputStream used to make the connection
    * @return OutputStream that may be using wrapping
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public OutputStream getOutputStream(OutputStream out) throws IOException {
     if (useWrap()) {
@@ -638,7 +645,11 @@ public void write(byte[] buf, int off, int len) throws IOException {
     }
   }
 
-  /** Release resources used by wrapped saslClient */
+  /**
+   * Release resources used by wrapped saslClient.
+   * @throws SaslException if authentication or generating response fails,
+   *                       or SASL protocol mixup
+   */
   public void dispose() throws SaslException {
     if (saslClient != null) {
       saslClient.dispose();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
index 7c3f14da21cf5..b61b6cc18414d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
@@ -208,7 +208,11 @@ static char[] encodePassword(byte[] password) {
                       StandardCharsets.UTF_8).toCharArray();
   }
 
-  /** Splitting fully qualified Kerberos name into parts */
+  /**
+   * Splitting fully qualified Kerberos name into parts.
+   * @param fullName fullName.
+   * @return splitKerberosName.
+   */
   public static String[] splitKerberosName(String fullName) {
     return fullName.split("[/@]");
   }
@@ -240,17 +244,30 @@ private static AuthMethod valueOf(byte code) {
       return i < 0 || i >= values().length ? null : values()[i];
     }
 
-    /** Return the SASL mechanism name */
+    /**
+     * Return the SASL mechanism name.
+     * @return mechanismName.
+     */
     public String getMechanismName() {
       return mechanismName;
     }
 
-    /** Read from in */
+    /**
+     * Read from in.
+     *
+     * @param in DataInput.
+     * @throws IOException raised on errors performing I/O.
+     * @return AuthMethod.
+     */
     public static AuthMethod read(DataInput in) throws IOException {
       return valueOf(in.readByte());
     }
 
-    /** Write to out */
+    /**
+     * Write to out.
+     * @param out DataOutput.
+     * @throws IOException raised on errors performing I/O.
+     */
     public void write(DataOutput out) throws IOException {
       out.write(code);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index c9423490635cb..3369869bde24d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -44,6 +44,8 @@
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.net.DNS;
+import org.apache.hadoop.net.DomainNameResolver;
+import org.apache.hadoop.net.DomainNameResolverFactory;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.token.Token;
@@ -81,6 +83,8 @@ private SecurityUtil() {
   @VisibleForTesting
   static HostResolver hostResolver;
 
+  private static DomainNameResolver domainNameResolver;
+
   private static boolean logSlowLookups;
   private static int slowLookupThresholdMs;
 
@@ -112,10 +116,15 @@ private static void setConfigurationInternal(Configuration conf) {
             .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY,
         CommonConfigurationKeys
             .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT);
+
+    domainNameResolver = DomainNameResolverFactory.newInstance(conf,
+        CommonConfigurationKeys.HADOOP_SECURITY_RESOLVER_IMPL);
   }
 
   /**
-   * For use only by tests and initialization
+   * For use only by tests and initialization.
+   *
+   * @param flag flag.
    */
   @InterfaceAudience.Private
   @VisibleForTesting
@@ -210,7 +219,7 @@ public static String getServerPrincipal(String principalConfig,
         throw new IOException("Can't replace " + HOSTNAME_PATTERN
             + " pattern since client address is null");
       }
-      return replacePattern(components, addr.getCanonicalHostName());
+      return replacePattern(components, domainNameResolver.getHostnameByIP(addr));
     }
   }
   
@@ -487,6 +496,10 @@ public static Text buildTokenService(URI uri) {
    * Perform the given action as the daemon's login user. If the login
    * user cannot be determined, this will log a FATAL error and exit
    * the whole JVM.
+   *
+   * @param action action.
+   * @param <T> generic type T.
+   * @return generic type T.
    */
   public static <T> T doAsLoginUserOrFatal(PrivilegedAction<T> action) { 
     if (UserGroupInformation.isSecurityEnabled()) {
@@ -509,6 +522,7 @@ public static <T> T doAsLoginUserOrFatal(PrivilegedAction<T> action) {
    * InterruptedException is thrown, it is converted to an IOException.
    *
    * @param action the action to perform
+   * @param <T> Generics Type T.
    * @return the result of the action
    * @throws IOException in the event of error
    */
@@ -522,6 +536,7 @@ public static <T> T doAsLoginUser(PrivilegedExceptionAction<T> action)
    * InterruptedException is thrown, it is converted to an IOException.
    *
    * @param action the action to perform
+   * @param <T> generic type T.
    * @return the result of the action
    * @throws IOException in the event of error
    */
@@ -745,9 +760,13 @@ public static boolean isPrivilegedPort(final int port) {
 
   /**
    * Utility method to fetch ZK auth info from the configuration.
+   *
+   * @param conf configuration.
+   * @param configKey config key.
    * @throws java.io.IOException if the Zookeeper ACLs configuration file
    * cannot be read
    * @throws ZKUtil.BadAuthFormatException if the auth format is invalid
+   * @return ZKAuthInfo List.
    */
   public static List<ZKUtil.ZKAuthInfo> getZKAuthInfos(Configuration conf,
       String configKey) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index d1eab8f4e1bbd..c28471a3bdad9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -210,7 +210,14 @@ private static Integer parseId(final String idStr) {
   /**
    * Get the list of users or groups returned by the specified command,
    * and save them in the corresponding map.
-   * @throws IOException 
+   *
+   * @param map map.
+   * @param mapName mapName.
+   * @param command command.
+   * @param staticMapping staticMapping.
+   * @param regex regex.
+   * @throws IOException raised on errors performing I/O.
+   * @return updateMapInternal.
    */
   @VisibleForTesting
   public static boolean updateMapInternal(BiMap<Integer, String> map,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index f4db520ac24c0..d0c4e11cbefc3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -215,7 +215,7 @@ private Set<String> getUnixGroups(String user) throws IOException {
           groups = resolvePartialGroupNames(user, e.getMessage(),
               executor.getOutput());
         } catch (PartialGroupNameException pge) {
-          LOG.warn("unable to return groups for user {}", user, pge);
+          LOG.debug("unable to return groups for user {}", user, pge);
           return EMPTY_GROUPS_SET;
         }
       }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
index eff6985471b4c..01d6f299d17d5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsNetgroupMapping.java
@@ -92,6 +92,7 @@ public void cacheGroupsAdd(List<String> groups) throws IOException {
    *
    * @param netgroup return users for this netgroup
    * @return list of users for a given netgroup
+   * @throws IOException raised on errors performing I/O.
    */
   protected List<String> getUsersForNetgroup(String netgroup) 
     throws IOException {
@@ -128,6 +129,7 @@ protected List<String> getUsersForNetgroup(String netgroup)
    *
    * @param netgroup get users for this netgroup
    * @return string of users for a given netgroup in getent netgroups format
+   * @throws IOException raised on errors performing I/O.
    */
   protected String execShellGetUserForNetgroup(final String netgroup)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index a3b1cbd14d822..9671d8da38fd3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -589,6 +589,7 @@ public static UserGroupInformation getCurrentUser() throws IOException {
    * @param user               The user name, or NULL if none is specified.
    *
    * @return                   The most appropriate UserGroupInformation
+   * @throws IOException raised on errors performing I/O.
    */ 
   public static UserGroupInformation getBestUGI(
       String ticketCachePath, String user) throws IOException {
@@ -609,6 +610,7 @@ public static UserGroupInformation getBestUGI(
    * @param ticketCache     the path to the ticket cache file
    *
    * @throws IOException        if the kerberos login fails
+   * @return UserGroupInformation.
    */
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
@@ -630,8 +632,9 @@ public static UserGroupInformation getUGIFromTicketCache(
    *                            The creator of subject is responsible for
    *                            renewing credentials.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException if the kerberos login fails
+   * @return UserGroupInformation
    */
   public static UserGroupInformation getUGIFromSubject(Subject subject)
       throws IOException {
@@ -686,7 +689,7 @@ public static UserGroupInformation getLoginUser() throws IOException {
    * remove the login method that is followed by a space from the username
    * e.g. "jack (auth:SIMPLE)" {@literal ->} "jack"
    *
-   * @param userName
+   * @param userName userName.
    * @return userName without login method
    */
   public static String trimLoginMethod(String userName) {
@@ -1106,7 +1109,7 @@ static long getNextTgtRenewalTime(final long tgtEndTime, final long now,
    * file and logs them in. They become the currently logged-in user.
    * @param user the principal name to load from the keytab
    * @param path the path to the keytab file
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException if it's a kerberos login exception.
    */
   @InterfaceAudience.Public
@@ -1136,7 +1139,7 @@ static void loginUserFromKeytab(String user,
    * This method assumes that the user logged in by calling
    * {@link #loginUserFromKeytab(String, String)}.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException if a failure occurred in logout,
    * or if the user did not log in by invoking loginUserFromKeyTab() before.
    */
@@ -1176,7 +1179,7 @@ public void logoutUserFromKeytab() throws IOException {
   /**
    * Re-login a user from keytab if TGT is expired or is close to expiry.
    * 
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException if it's a kerberos login exception.
    */
   public void checkTGTAndReloginFromKeytab() throws IOException {
@@ -1224,7 +1227,7 @@ void fixKerberosTicketOrder() {
    * happened already.
    * The Subject field of this UserGroupInformation object is updated to have
    * the new credentials.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException on a failure
    */
   @InterfaceAudience.Public
@@ -1241,7 +1244,7 @@ public void reloginFromKeytab() throws IOException {
    * Subject field of this UserGroupInformation object is updated to have the
    * new credentials.
    *
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException on a failure
    */
   @InterfaceAudience.Public
@@ -1278,7 +1281,7 @@ private void reloginFromKeytab(boolean checkTGT, boolean ignoreLastLoginTime)
    * method assumes that login had happened already.
    * The Subject field of this UserGroupInformation object is updated to have
    * the new credentials.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    * @throws KerberosAuthException on a failure
    */
   @InterfaceAudience.Public
@@ -1346,6 +1349,7 @@ private void unprotectedRelogin(HadoopLoginContext login,
    * @param user the principal name to load from the keytab
    * @param path the path to the keytab file
    * @throws IOException if the keytab file can't be read
+   * @return UserGroupInformation.
    */
   public
   static UserGroupInformation loginUserFromKeytabAndReturnUGI(String user,
@@ -1372,8 +1376,9 @@ private boolean hasSufficientTimeElapsed(long now) {
   }
   
   /**
-   * Did the login happen via keytab
+   * Did the login happen via keytab.
    * @return true or false
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
@@ -1382,8 +1387,9 @@ public static boolean isLoginKeytabBased() throws IOException {
   }
 
   /**
-   * Did the login happen via ticket cache
+   * Did the login happen via ticket cache.
    * @return true or false
+   * @throws IOException raised on errors performing I/O.
    */
   public static boolean isLoginTicketBased()  throws IOException {
     return getLoginUser().isFromTicket();
@@ -1405,6 +1411,7 @@ public static UserGroupInformation createRemoteUser(String user) {
    * Create a user from a login name. It is intended to be used for remote
    * users in RPC, since it won't have any credentials.
    * @param user the full user principal name, must not be empty or null
+   * @param authMethod authMethod.
    * @return the UserGroupInformation for the remote user.
    */
   @InterfaceAudience.Public
@@ -1474,8 +1481,8 @@ public static AuthenticationMethod valueOf(AuthMethod authMethod) {
   /**
    * Create a proxy user using username of the effective user and the ugi of the
    * real user.
-   * @param user
-   * @param realUser
+   * @param user user.
+   * @param realUser realUser.
    * @return proxyUser ugi
    */
   @InterfaceAudience.Public
@@ -1508,7 +1515,19 @@ public UserGroupInformation getRealUser() {
     return null;
   }
 
-
+  /**
+   * If this is a proxy user, get the real user. Otherwise, return
+   * this user.
+   * @param user the user to check
+   * @return the real user or self
+   */
+  public static UserGroupInformation getRealUserOrSelf(UserGroupInformation user) {
+    if (user == null) {
+      return null;
+    }
+    UserGroupInformation real = user.getRealUser();
+    return real != null ? real : user;
+  }
   
   /**
    * This class is used for storing the groups for testing. It stores a local
@@ -1776,7 +1795,7 @@ public String toString() {
   /**
    * Sets the authentication method in the subject
    * 
-   * @param authMethod
+   * @param authMethod authMethod.
    */
   public synchronized 
   void setAuthenticationMethod(AuthenticationMethod authMethod) {
@@ -1786,7 +1805,7 @@ void setAuthenticationMethod(AuthenticationMethod authMethod) {
   /**
    * Sets the authentication method in the subject
    * 
-   * @param authMethod
+   * @param authMethod authMethod.
    */
   public void setAuthenticationMethod(AuthMethod authMethod) {
     user.setAuthenticationMethod(AuthenticationMethod.valueOf(authMethod));
@@ -1819,7 +1838,7 @@ public synchronized AuthenticationMethod getRealAuthenticationMethod() {
    * Returns the authentication method of a ugi. If the authentication method is
    * PROXY, returns the authentication method of the real user.
    * 
-   * @param ugi
+   * @param ugi ugi.
    * @return AuthenticationMethod
    */
   public static AuthenticationMethod getRealAuthenticationMethod(
@@ -1921,6 +1940,8 @@ public <T> T doAs(PrivilegedExceptionAction<T> action
   /**
    * Log current UGI and token information into specified log.
    * @param ugi - UGI
+   * @param log log.
+   * @param caption caption.
    */
   @InterfaceAudience.LimitedPrivate({"HDFS", "KMS"})
   @InterfaceStability.Unstable
@@ -1938,7 +1959,8 @@ public static void logUserInfo(Logger log, String caption,
   /**
    * Log all (current, real, login) UGI and token info into specified log.
    * @param ugi - UGI
-   * @throws IOException
+   * @param log - log.
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.LimitedPrivate({"HDFS", "KMS"})
   @InterfaceStability.Unstable
@@ -1956,7 +1978,7 @@ public static void logAllUserInfo(Logger log, UserGroupInformation ugi) throws
   /**
    * Log all (current, real, login) UGI and token info into UGI debug log.
    * @param ugi - UGI
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void logAllUserInfo(UserGroupInformation ugi) throws
       IOException {
@@ -2234,7 +2256,7 @@ private static String prependFileAuthority(String keytabPath) {
    * A test method to print out the current user's UGI.
    * @param args if there are two arguments, read the user from the keytab
    * and print it out.
-   * @throws Exception
+   * @throws Exception Exception.
    */
   public static void main(String [] args) throws Exception {
   System.out.println("Getting UGI for current user");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
index 113dcaeb5e644..2779194d85e00 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
@@ -91,7 +91,7 @@ public boolean isTransient() {
   /**
    * Ensures that any changes to the credentials are written to persistent
    * store.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void flush() throws IOException;
 
@@ -99,7 +99,7 @@ public boolean isTransient() {
    * Get the credential entry for a specific alias.
    * @param alias the name of a specific credential
    * @return the credentialEntry
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract CredentialEntry getCredentialEntry(String alias) 
       throws IOException;
@@ -107,7 +107,7 @@ public abstract CredentialEntry getCredentialEntry(String alias)
   /**
    * Get the aliases for all credentials.
    * @return the list of alias names
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract List<String> getAliases() throws IOException;
 
@@ -115,7 +115,8 @@ public abstract CredentialEntry getCredentialEntry(String alias)
    * Create a new credential. The given alias must not already exist.
    * @param name the alias of the credential
    * @param credential the credential value for the alias.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
+   * @return CredentialEntry.
    */
   public abstract CredentialEntry createCredentialEntry(String name, 
       char[] credential) throws IOException;
@@ -123,7 +124,7 @@ public abstract CredentialEntry createCredentialEntry(String name,
   /**
    * Delete the given credential.
    * @param name the alias of the credential to delete
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract void deleteCredentialEntry(String name) throws IOException;
 
@@ -133,7 +134,7 @@ public abstract CredentialEntry createCredentialEntry(String name,
    * means. If true, the password should be provided by the caller using
    * setPassword().
    * @return Whether or not the provider requires a password
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public boolean needsPassword() throws IOException {
     return false;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index 06d42207ecba5..66df17a181e54 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -70,9 +70,9 @@ public class CredentialShell extends CommandShell {
    * % hadoop credential check alias [-provider providerPath]
    * % hadoop credential delete alias [-provider providerPath] [-f]
    * </pre>
-   * @param args
+   * @param args args.
    * @return 0 if the argument(s) were recognized, 1 otherwise
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   protected int init(String[] args) throws IOException {
@@ -523,7 +523,7 @@ public void format(String message) {
    *
    * @param args
    *          Command line arguments
-   * @throws Exception
+   * @throws Exception exception.
    */
   public static void main(String[] args) throws Exception {
     int res = ToolRunner.run(new Configuration(), new CredentialShell(), args);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
index aa5b01fbed113..6fabbfb47b9f8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/AccessControlList.java
@@ -105,8 +105,8 @@ public AccessControlList(String users, String groups) {
    * @param userGroupStrings build ACL from array of Strings
    */
   private void buildACL(String[] userGroupStrings) {
-    users = new HashSet<String>();
-    groups = new HashSet<String>();
+    users = new HashSet<>();
+    groups = new HashSet<>();
     for (String aclPart : userGroupStrings) {
       if (aclPart != null && isWildCardACLValue(aclPart)) {
         allAllowed = true;
@@ -296,6 +296,7 @@ else if (!users.isEmpty()) {
   /**
    * Returns the access control list as a String that can be used for building a
    * new instance by sending it to the constructor of {@link AccessControlList}.
+   * @return acl string.
    */
   public String getAclString() {
     StringBuilder sb = new StringBuilder(INITIAL_CAPACITY);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java
index eff77d8942cf7..129e1e4dad26a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ImpersonationProvider.java
@@ -46,7 +46,7 @@ public interface ImpersonationProvider  extends Configurable {
    *             be preferred to avoid possibly re-resolving the ip address.
    * @param user ugi of the effective or proxy user which contains a real user.
    * @param remoteAddress the ip address of client.
-   * @throws AuthorizationException
+   * @throws AuthorizationException Authorization Exception.
    */
   default void authorize(UserGroupInformation user, String remoteAddress)
       throws AuthorizationException {
@@ -62,7 +62,7 @@ default void authorize(UserGroupInformation user, String remoteAddress)
    *
    * @param user ugi of the effective or proxy user which contains a real user
    * @param remoteAddress the ip address of client
-   * @throws AuthorizationException
+   * @throws AuthorizationException Authorization Exception.
    */
   void authorize(UserGroupInformation user, InetAddress remoteAddress)
       throws AuthorizationException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyServers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyServers.java
index 410e25f583966..6f5283074dca6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyServers.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyServers.java
@@ -33,7 +33,7 @@ public static void refresh() {
   }
 
   public static void refresh(Configuration conf){
-    Collection<String> tempServers = new HashSet<String>();
+    Collection<String> tempServers = new HashSet<>();
     // trusted proxy servers such as http proxies
     for (String host : conf.getTrimmedStrings(CONF_HADOOP_PROXYSERVERS)) {
       InetSocketAddress addr = new InetSocketAddress(host, 0);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
index be05e110b59cf..cc80708f1854e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
@@ -94,7 +94,7 @@ public static void refreshSuperUserGroupsConfiguration(Configuration conf) {
    *
    * @param user ugi of the effective or proxy user which contains a real user
    * @param remoteAddress the ip address of client
-   * @throws AuthorizationException
+   * @throws AuthorizationException Authorization Exception.
    */
   public static void authorize(UserGroupInformation user, 
       String remoteAddress) throws AuthorizationException {
@@ -106,7 +106,7 @@ public static void authorize(UserGroupInformation user,
    *
    * @param user ugi of the effective or proxy user which contains a real user
    * @param remoteAddress the inet address of client
-   * @throws AuthorizationException
+   * @throws AuthorizationException Authorization Exception.
    */
   public static void authorize(UserGroupInformation user,
       InetAddress remoteAddress) throws AuthorizationException {
@@ -125,10 +125,10 @@ private static ImpersonationProvider getSip() {
 
   /**
    * This function is kept to provide backward compatibility.
-   * @param user
-   * @param remoteAddress
-   * @param conf
-   * @throws AuthorizationException
+   * @param user user.
+   * @param remoteAddress remote address.
+   * @param conf configuration.
+   * @throws AuthorizationException Authorization Exception.
    * @deprecated use {@link #authorize(UserGroupInformation, String)} instead.
    */
   @Deprecated
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/RefreshAuthorizationPolicyProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/RefreshAuthorizationPolicyProtocol.java
index 0f0b25d8344e2..51a900fa71cb0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/RefreshAuthorizationPolicyProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/RefreshAuthorizationPolicyProtocol.java
@@ -41,7 +41,7 @@ public interface RefreshAuthorizationPolicyProtocol {
 
   /**
    * Refresh the service-level authorization policy in-effect.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Idempotent
   void refreshServiceAcl() throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
index 059cdc4b653de..ef342f257a937 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
@@ -22,7 +22,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.servlet.Filter;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
index b81ed8e90155e..7363ca0ba6450 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java
@@ -94,7 +94,7 @@ public void init(FilterConfig filterConfig) throws ServletException {
 
   void parseBrowserUserAgents(String userAgents) {
     String[] agentsArray =  userAgents.split(",");
-    browserUserAgents = new HashSet<Pattern>();
+    browserUserAgents = new HashSet<>();
     for (String patternString : agentsArray) {
       browserUserAgents.add(Pattern.compile(patternString));
     }
@@ -102,7 +102,7 @@ void parseBrowserUserAgents(String userAgents) {
 
   void parseMethodsToIgnore(String mti) {
     String[] methods = mti.split(",");
-    methodsToIgnore = new HashSet<String>();
+    methodsToIgnore = new HashSet<>();
     for (int i = 0; i < methods.length; i++) {
       methodsToIgnore.add(methods[i]);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
index 520047b3a0414..4fbb3c8aea6fc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java
@@ -319,8 +319,10 @@ String getPassword(Configuration conf, String alias, String defaultPass) {
    */
   @Override
   public synchronized void destroy() {
-    if (trustManager != null) {
+    if (fileMonitoringTimer != null) {
       fileMonitoringTimer.cancel();
+    }
+    if (trustManager != null) {
       trustManager = null;
       keyManagers = null;
       trustManagers = null;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509KeystoreManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509KeystoreManager.java
index 216d949de1048..429304ef64c1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509KeystoreManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509KeystoreManager.java
@@ -63,8 +63,8 @@ public class ReloadingX509KeystoreManager extends X509ExtendedKeyManager {
    * @param location local path to the keystore file.
    * @param storePassword password of the keystore file.
    * @param keyPassword The password of the key.
-   * @throws IOException
-   * @throws GeneralSecurityException
+   * @throws IOException raised on errors performing I/O.
+   * @throws GeneralSecurityException thrown if create encryptor error.
    */
   public ReloadingX509KeystoreManager(String type, String location,
                                       String storePassword, String keyPassword)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
index c6049a91b5a51..3dc5017ba6377 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
@@ -27,7 +27,6 @@
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java
index 7b0a78bcd3c0d..77e74a271fc0e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DelegationTokenIssuer.java
@@ -39,17 +39,24 @@ public interface DelegationTokenIssuer {
    * The service name used as the alias for the  token in the credential
    * token map.  addDelegationTokens will use this to determine if
    * a token exists, and if not, add a new token with this alias.
+   * @return the token.
    */
   String getCanonicalServiceName();
 
   /**
    * Unconditionally get a new token with the optional renewer.  Returning
    * null indicates the service does not issue tokens.
+   * @param renewer renewer.
+   * @return the token.
+   * @throws IOException raised on errors performing I/O.
    */
   Token<?> getDelegationToken(String renewer) throws IOException;
 
   /**
    * Issuers may need tokens from additional services.
+   *
+   * @return delegation token issuer.
+   * @throws IOException raised on errors performing I/O.
    */
   default DelegationTokenIssuer[] getAdditionalTokenIssuers()
       throws IOException {
@@ -81,6 +88,12 @@ default Token<?>[] addDelegationTokens(
 
   /**
    * NEVER call this method directly.
+   *
+   * @param issuer issuer.
+   * @param renewer renewer.
+   * @param credentials cache in which to add new delegation tokens.
+   * @param tokens list of new delegation tokens.
+   * @throws IOException raised on errors performing I/O.
    */
   @InterfaceAudience.Private
   static void collectDelegationTokens(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFetcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFetcher.java
index d74e7bdb10272..4b22df2043e8c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFetcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFetcher.java
@@ -28,14 +28,29 @@
  *  compilation units.  Resolution of fetcher impl will be done at runtime.
  */
 public interface DtFetcher {
-  /** Return a key used to identify the object/service implementation. */
+  /**
+   * Return a key used to identify the object/service implementation.
+   * @return ServiceName.
+   */
   Text getServiceName();
 
-  /** Used to allow the service API to indicate whether a token is required. */
+  /**
+   * Used to allow the service API to indicate whether a token is required.
+   * @return isTokenRequired.
+   */
   boolean isTokenRequired();
 
-  /** Add any number of delegation tokens to Credentials object and return
-   *  a token instance that is appropriate for aliasing, or null if none. */
+  /**
+   * Add any number of delegation tokens to Credentials object and return
+   *  a token instance that is appropriate for aliasing, or null if none.
+   *
+   * @param conf configuration.
+   * @param creds credentials.
+   * @param renewer renewer.
+   * @param url url.
+   * @throws Exception Exception.
+   * @return DelegationTokens.
+   */
   Token<?> addDelegationTokens(Configuration conf, Credentials creds,
                                String renewer, String url) throws Exception;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index 2160d8b6a82a1..5d80a45f79f22 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -99,7 +99,7 @@ private static Path fileToPath(File f) {
    *  @param format a string equal to FORMAT_PB or FORMAT_JAVA.
    *  @param creds the Credentials object to be written out.
    *  @param conf a Configuration object passed along.
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public static void doFormattedWrite(
       File f, String format, Credentials creds, Configuration conf)
@@ -118,7 +118,7 @@ public static void doFormattedWrite(
    *  @param alias print only tokens matching alias (null matches all).
    *  @param conf Configuration object passed along.
    *  @param out print to this stream.
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public static void printTokenFile(
       File tokenFile, Text alias, Configuration conf, PrintStream out)
@@ -170,7 +170,7 @@ public static void printCredentials(
    *  @param url pass this URL to fetcher after stripping any http/s prefix.
    *  @param renewer pass this renewer to the fetcher.
    *  @param conf Configuration object passed along.
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public static void getTokenFile(File tokenFile, String fileFormat,
       Text alias, Text service, String url, String renewer, Configuration conf)
@@ -225,7 +225,7 @@ public static void getTokenFile(File tokenFile, String fileFormat,
    *  @param alias overwrite service field of fetched token with this text.
    *  @param service only apply alias to tokens matching this service text.
    *  @param conf Configuration object passed along.
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public static void aliasTokenFile(File tokenFile, String fileFormat,
       Text alias, Text service, Configuration conf) throws Exception {
@@ -246,7 +246,7 @@ public static void aliasTokenFile(File tokenFile, String fileFormat,
    *  @param tokenFiles list of local File objects.  Last file holds the output.
    *  @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
    *  @param conf Configuration object passed along.
-   *  @throws IOException
+   *  @throws IOException raised on errors performing I/O.
    */
   public static void appendTokenFiles(
       ArrayList<File> tokenFiles, String fileFormat, Configuration conf)
@@ -269,8 +269,8 @@ public static void appendTokenFiles(
    *  @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
    *  @param alias remove only tokens matching alias; null matches all.
    *  @param conf Configuration object passed along.
-   *  @throws IOException
-   *  @throws InterruptedException
+   *  @throws IOException raised on errors performing I/O.
+   *  @throws InterruptedException if the thread is interrupted.
    */
   public static void removeTokenFromFile(boolean cancel,
       File tokenFile, String fileFormat, Text alias, Configuration conf)
@@ -295,8 +295,8 @@ public static void removeTokenFromFile(boolean cancel,
    *  @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
    *  @param alias renew only tokens matching alias; null matches all.
    *  @param conf Configuration object passed along.
-   *  @throws IOException
-   *  @throws InterruptedException
+   *  @throws IOException raised on errors performing I/O.
+   *  @throws InterruptedException if the thread is interrupted.
    */
   public static void renewTokenFile(
       File tokenFile, String fileFormat, Text alias, Configuration conf)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
index bc2d1b6e11a7e..9e34ebf4a2a58 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
@@ -109,9 +109,9 @@ private String[] maybeDoLoginFromKeytabAndPrincipal(String[] args)
    * Parse the command line arguments and initialize subcommand.
    * Also will attempt to perform Kerberos login if both -principal and -keytab
    * flags are passed in args array.
-   * @param args
+   * @param args args.
    * @return 0 if the argument(s) were recognized, 1 otherwise
-   * @throws Exception
+   * @throws Exception Exception.
    */
   @Override
   protected int init(String[] args) throws Exception {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
index 0141af8237b1b..33314060a5563 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
@@ -193,7 +193,7 @@ public synchronized Text getKind() {
   /**
    * Set the token kind. This is only intended to be used by services that
    * wrap another service's token.
-   * @param newKind
+   * @param newKind newKind.
    */
   @InterfaceAudience.Private
   public synchronized void setKind(Text newKind) {
@@ -367,7 +367,7 @@ private static void decodeWritable(Writable obj,
   /**
    * Encode this token as a url safe string.
    * @return the encoded string
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public String encodeToUrlString() throws IOException {
     return encodeWritable(this);
@@ -376,7 +376,7 @@ public String encodeToUrlString() throws IOException {
   /**
    * Decode the given url safe string into this token.
    * @param newValue the encoded string
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void decodeFromUrlString(String newValue) throws IOException {
     decodeWritable(this, newValue);
@@ -481,6 +481,7 @@ private synchronized TokenRenewer getRenewer() throws IOException {
   /**
    * Is this token managed so that it can be renewed or cancelled?
    * @return true, if it can be renewed and cancelled.
+   * @throws IOException raised on errors performing I/O.
    */
   public boolean isManaged() throws IOException {
     return getRenewer().isManaged(this);
@@ -488,9 +489,10 @@ public boolean isManaged() throws IOException {
 
   /**
    * Renew this delegation token.
+   * @param conf configuration.
    * @return the new expiration time
-   * @throws IOException
-   * @throws InterruptedException
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException if the thread is interrupted.
    */
   public long renew(Configuration conf
                     ) throws IOException, InterruptedException {
@@ -499,8 +501,10 @@ public long renew(Configuration conf
 
   /**
    * Cancel this delegation token.
-   * @throws IOException
-   * @throws InterruptedException
+   *
+   * @param conf configuration.
+   * @throws IOException raised on errors performing I/O.
+   * @throws InterruptedException if the thread is interrupted.
    */
   public void cancel(Configuration conf
                      ) throws IOException, InterruptedException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenInfo.java
index cc76824eb0e13..9234b23202eca 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenInfo.java
@@ -31,6 +31,10 @@
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public @interface TokenInfo {
-  /** The type of TokenSelector to be used */
+  /**
+   * The type of TokenSelector to be used.
+   *
+   * @return TokenSelector
+   */
   Class<? extends TokenSelector<? extends TokenIdentifier>> value();
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenRenewer.java
index 11e275f3213d2..eba4bf6daa42f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenRenewer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/TokenRenewer.java
@@ -44,25 +44,37 @@ public abstract class TokenRenewer {
    * cancelled.
    * @param token the token being checked
    * @return true if the token may be renewed or cancelled
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public abstract boolean isManaged(Token<?> token) throws IOException;
-  
-  /**
-   * Renew the given token.
-   * @return the new expiration time
-   * @throws IOException
-   * @throws InterruptedException 
-   */
+
+    /**
+     * Renew the given token.
+     *
+     * @param token the token being checked.
+     * @param conf configuration.
+     *
+     * @return the new expiration time.
+     * @throws IOException raised on errors performing I/O.
+     * @throws InterruptedException thrown when a thread is waiting, sleeping,
+     *                              or otherwise occupied, and the thread is interrupted,
+     *                              either before or during the activity.
+     */
   public abstract long renew(Token<?> token,
                              Configuration conf
                              ) throws IOException, InterruptedException;
-  
-  /**
-   * Cancel the given token
-   * @throws IOException
-   * @throws InterruptedException 
-   */
+
+    /**
+     * Cancel the given token.
+     *
+     * @param token the token being checked.
+     * @param conf configuration.
+     *
+     * @throws IOException raised on errors performing I/O.
+     * @throws InterruptedException thrown when a thread is waiting, sleeping,
+     *                              or otherwise occupied, and the thread is interrupted,
+     *                              either before or during the activity.
+     */
   public abstract void cancel(Token<?> token,
                               Configuration conf
                               ) throws IOException, InterruptedException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index b7283adcada56..d0c0fac6e88df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -36,7 +36,17 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair;
 import org.apache.hadoop.metrics2.util.Metrics2Util.TopN;
 import org.apache.hadoop.security.AccessControlException;
@@ -47,6 +57,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.functional.InvocationRaisingIOE;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -59,6 +70,12 @@ class AbstractDelegationTokenSecretManager<TokenIdent
   private static final Logger LOG = LoggerFactory
       .getLogger(AbstractDelegationTokenSecretManager.class);
 
+  /**
+   * Metrics to track token management operations.
+   */
+  private static final DelegationTokenSecretManagerMetrics METRICS
+      = DelegationTokenSecretManagerMetrics.create();
+
   private String formatTokenId(TokenIdent id) {
     return "(" + id + ")";
   }
@@ -136,7 +153,10 @@ public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval,
     this.storeTokenTrackingId = false;
   }
 
-  /** should be called before this object is used */
+  /**
+   * should be called before this object is used.
+   * @throws IOException raised on errors performing I/O.
+   */
   public void startThreads() throws IOException {
     Preconditions.checkState(!running);
     updateCurrentKey();
@@ -159,6 +179,8 @@ public synchronized void reset() {
 
   /**
    * Total count of active delegation tokens.
+   *
+   * @return currentTokens.size.
    */
   public long getCurrentTokensSize() {
     return currentTokens.size();
@@ -166,8 +188,11 @@ public long getCurrentTokensSize() {
 
   /** 
    * Add a previously used master key to cache (when NN restarts), 
-   * should be called before activate(). 
-   * */
+   * should be called before activate().
+   *
+   * @param key delegation key.
+   * @throws IOException raised on errors performing I/O.
+   */
   public synchronized void addKey(DelegationKey key) throws IOException {
     if (running) // a safety check
       throw new IOException("Can't add delegation key to a running SecretManager.");
@@ -217,7 +242,9 @@ protected void updateStoredToken(TokenIdent ident, long renewDate) throws IOExce
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @return currentId.
    */
   protected synchronized int getCurrentKeyId() {
     return currentId;
@@ -225,7 +252,9 @@ protected synchronized int getCurrentKeyId() {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @return currentId.
    */
   protected synchronized int incrementCurrentKeyId() {
     return ++currentId;
@@ -233,7 +262,9 @@ protected synchronized int incrementCurrentKeyId() {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param keyId keyId.
    */
   protected synchronized void setCurrentKeyId(int keyId) {
     currentId = keyId;
@@ -241,7 +272,9 @@ protected synchronized void setCurrentKeyId(int keyId) {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @return delegationTokenSequenceNumber.
    */
   protected synchronized int getDelegationTokenSeqNum() {
     return delegationTokenSequenceNumber;
@@ -249,7 +282,9 @@ protected synchronized int getDelegationTokenSeqNum() {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @return delegationTokenSequenceNumber.
    */
   protected synchronized int incrementDelegationTokenSeqNum() {
     return ++delegationTokenSequenceNumber;
@@ -257,7 +292,9 @@ protected synchronized int incrementDelegationTokenSeqNum() {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param seqNum seqNum.
    */
   protected synchronized void setDelegationTokenSeqNum(int seqNum) {
     delegationTokenSequenceNumber = seqNum;
@@ -265,7 +302,10 @@ protected synchronized void setDelegationTokenSeqNum(int seqNum) {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param keyId keyId.
+   * @return DelegationKey.
    */
   protected DelegationKey getDelegationKey(int keyId) {
     return allKeys.get(keyId);
@@ -273,7 +313,10 @@ protected DelegationKey getDelegationKey(int keyId) {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param key DelegationKey.
+   * @throws IOException raised on errors performing I/O.
    */
   protected void storeDelegationKey(DelegationKey key) throws IOException {
     allKeys.put(key.getKeyId(), key);
@@ -282,7 +325,10 @@ protected void storeDelegationKey(DelegationKey key) throws IOException {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param key DelegationKey.
+   * @throws IOException raised on errors performing I/O.
    */
   protected void updateDelegationKey(DelegationKey key) throws IOException {
     allKeys.put(key.getKeyId(), key);
@@ -291,6 +337,9 @@ protected void updateDelegationKey(DelegationKey key) throws IOException {
   /**
    * For subclasses externalizing the storage, for example Zookeeper
    * based implementations
+   *
+   * @param ident ident.
+   * @return DelegationTokenInformation.
    */
   protected DelegationTokenInformation getTokenInfo(TokenIdent ident) {
     return currentTokens.get(ident);
@@ -298,7 +347,11 @@ protected DelegationTokenInformation getTokenInfo(TokenIdent ident) {
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param ident ident.
+   * @param tokenInfo tokenInfo.
+   * @throws IOException raised on errors performing I/O.
    */
   protected void storeToken(TokenIdent ident,
       DelegationTokenInformation tokenInfo) throws IOException {
@@ -309,7 +362,11 @@ protected void storeToken(TokenIdent ident,
 
   /**
    * For subclasses externalizing the storage, for example Zookeeper
-   * based implementations
+   * based implementations.
+   *
+   * @param ident ident.
+   * @param tokenInfo tokenInfo.
+   * @throws IOException raised on errors performing I/O.
    */
   protected void updateToken(TokenIdent ident,
       DelegationTokenInformation tokenInfo) throws IOException {
@@ -325,7 +382,7 @@ protected void updateToken(TokenIdent ident,
    * startThreads() is called)
    * @param identifier identifier read from persistent storage
    * @param renewDate token renew time
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public synchronized void addPersistedDelegationToken(
       TokenIdent identifier, long renewDate) throws IOException {
@@ -430,34 +487,42 @@ protected synchronized byte[] createPassword(TokenIdent identifier) {
     DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
         + tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
     try {
-      storeToken(identifier, tokenInfo);
+      METRICS.trackStoreToken(() -> storeToken(identifier, tokenInfo));
     } catch (IOException ioe) {
       LOG.error("Could not store token " + formatTokenId(identifier) + "!!",
           ioe);
     }
     return password;
   }
-  
+
 
 
   /**
    * Find the DelegationTokenInformation for the given token id, and verify that
    * if the token is expired. Note that this method should be called with 
    * acquiring the secret manager's monitor.
+   *
+   * @param identifier identifier.
+   * @throws InvalidToken invalid token exception.
+   * @return DelegationTokenInformation.
    */
   protected DelegationTokenInformation checkToken(TokenIdent identifier)
       throws InvalidToken {
     assert Thread.holdsLock(this);
     DelegationTokenInformation info = getTokenInfo(identifier);
+    String err;
     if (info == null) {
-      throw new InvalidToken("token " + formatTokenId(identifier)
-          + " can't be found in cache");
+      err = "Token for real user: " + identifier.getRealUser() + ", can't be found in cache";
+      LOG.warn("{}, Token={}", err, formatTokenId(identifier));
+      throw new InvalidToken(err);
     }
     long now = Time.now();
     if (info.getRenewDate() < now) {
-      throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
-          "expired, current time: " + Time.formatTime(now) +
-          " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+      err = "Token " + identifier.getRealUser() + " has expired, current time: "
+          + Time.formatTime(now) + " expected renewal time: " + Time
+          .formatTime(info.getRenewDate());
+      LOG.info("{}, Token={}", err, formatTokenId(identifier));
+      throw new InvalidToken(err);
     }
     return info;
   }
@@ -487,7 +552,7 @@ public synchronized String getTokenTrackingId(TokenIdent identifier) {
    * Verifies that the given identifier and password are valid and match.
    * @param identifier Token identifier.
    * @param password Password in the token.
-   * @throws InvalidToken
+   * @throws InvalidToken InvalidToken.
    */
   public synchronized void verifyToken(TokenIdent identifier, byte[] password)
       throws InvalidToken {
@@ -555,12 +620,15 @@ public synchronized long renewToken(Token<TokenIdent> token,
       throw new InvalidToken("Renewal request for unknown token "
           + formatTokenId(id));
     }
-    updateToken(id, info);
+    METRICS.trackUpdateToken(() -> updateToken(id, info));
     return renewTime;
   }
   
   /**
    * Cancel a token by removing it from cache.
+   *
+   * @param token token.
+   * @param canceller canceller.
    * @return Identifier of the canceled token
    * @throws InvalidToken for invalid token
    * @throws AccessControlException if the user isn't allowed to cancel
@@ -591,8 +659,10 @@ public synchronized TokenIdent cancelToken(Token<TokenIdent> token,
     if (info == null) {
       throw new InvalidToken("Token not found " + formatTokenId(id));
     }
-    removeTokenForOwnerStats(id);
-    removeStoredToken(id);
+    METRICS.trackRemoveToken(() -> {
+      removeTokenForOwnerStats(id);
+      removeStoredToken(id);
+    });
     return id;
   }
   
@@ -622,15 +692,22 @@ public DelegationTokenInformation(long renewDate, byte[] password,
       this.password = password;
       this.trackingId = trackingId;
     }
-    /** returns renew date */
+    /**
+     * @return returns renew date.
+     */
     public long getRenewDate() {
       return renewDate;
     }
-    /** returns password */
+    /**
+     * @return returns password.
+     */
     byte[] getPassword() {
       return password;
     }
-    /** returns tracking id */
+
+    /**
+     * @return returns tracking id.
+     */
     public String getTrackingId() {
       return trackingId;
     }
@@ -639,7 +716,7 @@ public String getTrackingId() {
   /** Remove expired delegation tokens from cache */
   private void removeExpiredToken() throws IOException {
     long now = Time.now();
-    Set<TokenIdent> expiredTokens = new HashSet<TokenIdent>();
+    Set<TokenIdent> expiredTokens = new HashSet<>();
     synchronized (this) {
       Iterator<Map.Entry<TokenIdent, DelegationTokenInformation>> i =
           currentTokens.entrySet().iterator();
@@ -735,7 +812,7 @@ public void run() {
    * 
    * @param token the token where to extract the identifier
    * @return the delegation token identifier
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public TokenIdent decodeTokenIdentifier(Token<TokenIdent> token) throws IOException {
     return token.decodeIdentifier();
@@ -825,4 +902,97 @@ protected void syncTokenOwnerStats() {
       addTokenForOwnerStats(id);
     }
   }
+
+  protected DelegationTokenSecretManagerMetrics getMetrics() {
+    return METRICS;
+  }
+
+  /**
+   * DelegationTokenSecretManagerMetrics tracks token management operations
+   * and publishes them through the metrics interfaces.
+   */
+  @Metrics(about="Delegation token secret manager metrics", context="token")
+  static class DelegationTokenSecretManagerMetrics implements DurationTrackerFactory {
+    private static final Logger LOG = LoggerFactory.getLogger(
+        DelegationTokenSecretManagerMetrics.class);
+
+    final static String STORE_TOKEN_STAT = "storeToken";
+    final static String UPDATE_TOKEN_STAT = "updateToken";
+    final static String REMOVE_TOKEN_STAT = "removeToken";
+    final static String TOKEN_FAILURE_STAT = "tokenFailure";
+
+    private final MetricsRegistry registry;
+    private final IOStatisticsStore ioStatistics;
+
+    @Metric("Rate of storage of delegation tokens and latency (milliseconds)")
+    private MutableRate storeToken;
+    @Metric("Rate of update of delegation tokens and latency (milliseconds)")
+    private MutableRate updateToken;
+    @Metric("Rate of removal of delegation tokens and latency (milliseconds)")
+    private MutableRate removeToken;
+    @Metric("Counter of delegation tokens operation failures")
+    private MutableCounterLong tokenFailure;
+
+    static DelegationTokenSecretManagerMetrics create() {
+      return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics());
+    }
+
+    DelegationTokenSecretManagerMetrics() {
+      ioStatistics = IOStatisticsBinding.iostatisticsStore()
+          .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT)
+          .withCounters(TOKEN_FAILURE_STAT)
+          .build();
+      registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
+      LOG.debug("Initialized {}", registry);
+    }
+
+    public void trackStoreToken(InvocationRaisingIOE invocation) throws IOException {
+      trackInvocation(invocation, STORE_TOKEN_STAT, storeToken);
+    }
+
+    public void trackUpdateToken(InvocationRaisingIOE invocation) throws IOException {
+      trackInvocation(invocation, UPDATE_TOKEN_STAT, updateToken);
+    }
+
+    public void trackRemoveToken(InvocationRaisingIOE invocation) throws IOException {
+      trackInvocation(invocation, REMOVE_TOKEN_STAT, removeToken);
+    }
+
+    public void trackInvocation(InvocationRaisingIOE invocation, String statistic,
+        MutableRate metric) throws IOException {
+      try {
+        long start = Time.monotonicNow();
+        IOStatisticsBinding.trackDurationOfInvocation(this, statistic, invocation);
+        metric.add(Time.monotonicNow() - start);
+      } catch (Exception ex) {
+        tokenFailure.incr();
+        throw ex;
+      }
+    }
+
+    @Override
+    public DurationTracker trackDuration(String key, long count) {
+      return ioStatistics.trackDuration(key, count);
+    }
+
+    protected MutableRate getStoreToken() {
+      return storeToken;
+    }
+
+    protected MutableRate getUpdateToken() {
+      return updateToken;
+    }
+
+    protected MutableRate getRemoveToken() {
+      return removeToken;
+    }
+
+    protected MutableCounterLong getTokenFailure() {
+      return tokenFailure;
+    }
+
+    protected IOStatisticsStore getIoStatistics() {
+      return ioStatistics;
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index 5e5ea8cebc6f7..fb9a2951f598a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -25,14 +25,10 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
-import javax.security.auth.login.AppConfigurationEntry;
-
 import org.apache.curator.ensemble.fixed.FixedEnsembleProvider;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -46,15 +42,16 @@
 import org.apache.curator.framework.recipes.shared.SharedCount;
 import org.apache.curator.framework.recipes.shared.VersionedValue;
 import org.apache.curator.retry.RetryNTimes;
-import org.apache.curator.utils.EnsurePath;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.util.JaasConfiguration;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
 import static org.apache.hadoop.util.Time.now;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -62,6 +59,7 @@
 import org.apache.zookeeper.client.ZKClientConfig;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -98,6 +96,8 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
       + "kerberos.keytab";
   public static final String ZK_DTSM_ZK_KERBEROS_PRINCIPAL = ZK_CONF_PREFIX
       + "kerberos.principal";
+  public static final String ZK_DTSM_ZK_KERBEROS_SERVER_PRINCIPAL = ZK_CONF_PREFIX
+      + "kerberos.server.principal";
   public static final String ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE = ZK_CONF_PREFIX
       + "token.seqnum.batch.size";
   public static final String ZK_DTSM_TOKEN_WATCHER_ENABLED = ZK_CONF_PREFIX
@@ -134,6 +134,11 @@ public static void setCurator(CuratorFramework curator) {
     CURATOR_TL.set(curator);
   }
 
+  @VisibleForTesting
+  protected static CuratorFramework getCurator() {
+    return CURATOR_TL.get();
+  }
+
   private final boolean isExternalClient;
   protected final CuratorFramework zkClient;
   private SharedCount delTokSeqCounter;
@@ -202,6 +207,8 @@ public ZKDelegationTokenSecretManager(Configuration conf) {
         builder =
             CuratorFrameworkFactory
                 .builder()
+                .zookeeperFactory(new ZKCuratorManager.HadoopZookeeperFactory(
+                    conf.get(ZK_DTSM_ZK_KERBEROS_SERVER_PRINCIPAL)))
                 .aclProvider(aclProvider)
                 .namespace(
                     conf.get(ZK_DTSM_ZNODE_WORKING_PATH,
@@ -246,68 +253,6 @@ private String setJaasConfiguration(Configuration config) throws Exception {
     return principal.split("[/@]")[0];
   }
 
-  /**
-   * Creates a programmatic version of a jaas.conf file. This can be used
-   * instead of writing a jaas.conf file and setting the system property,
-   * "java.security.auth.login.config", to point to that file. It is meant to be
-   * used for connecting to ZooKeeper.
-   */
-  @InterfaceAudience.Private
-  public static class JaasConfiguration extends
-      javax.security.auth.login.Configuration {
-
-    private final javax.security.auth.login.Configuration baseConfig =
-        javax.security.auth.login.Configuration.getConfiguration();
-    private static AppConfigurationEntry[] entry;
-    private String entryName;
-
-    /**
-     * Add an entry to the jaas configuration with the passed in name,
-     * principal, and keytab. The other necessary options will be set for you.
-     *
-     * @param entryName
-     *          The name of the entry (e.g. "Client")
-     * @param principal
-     *          The principal of the user
-     * @param keytab
-     *          The location of the keytab
-     */
-    public JaasConfiguration(String entryName, String principal, String keytab) {
-      this.entryName = entryName;
-      Map<String, String> options = new HashMap<String, String>();
-      options.put("keyTab", keytab);
-      options.put("principal", principal);
-      options.put("useKeyTab", "true");
-      options.put("storeKey", "true");
-      options.put("useTicketCache", "false");
-      options.put("refreshKrb5Config", "true");
-      String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
-      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
-        options.put("debug", "true");
-      }
-      entry = new AppConfigurationEntry[] {
-          new AppConfigurationEntry(getKrb5LoginModuleName(),
-              AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
-              options) };
-    }
-
-    @Override
-    public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
-      return (entryName.equals(name)) ? entry : ((baseConfig != null)
-        ? baseConfig.getAppConfigurationEntry(name) : null);
-    }
-
-    private String getKrb5LoginModuleName() {
-      String krb5LoginModuleName;
-      if (System.getProperty("java.vendor").contains("IBM")) {
-        krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
-      } else {
-        krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
-      }
-      return krb5LoginModuleName;
-    }
-  }
-
   @Override
   public void startThreads() throws IOException {
     if (!isExternalClient) {
@@ -320,10 +265,12 @@ public void startThreads() throws IOException {
       // If namespace parents are implicitly created, they won't have ACLs.
       // So, let's explicitly create them.
       CuratorFramework nullNsFw = zkClient.usingNamespace(null);
-      EnsurePath ensureNs =
-        nullNsFw.newNamespaceAwareEnsurePath("/" + zkClient.getNamespace());
       try {
-        ensureNs.ensure(nullNsFw.getZookeeperClient());
+        String nameSpace = "/" + zkClient.getNamespace();
+        Stat stat = nullNsFw.checkExists().forPath(nameSpace);
+        if (stat == null) {
+          nullNsFw.create().creatingParentContainersIfNeeded().forPath(nameSpace);
+        }
       } catch (Exception e) {
         throw new IOException("Could not create namespace", e);
       }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 0988826605fbb..31eef61c8f57c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -336,6 +336,10 @@ public HttpURLConnection openConnection(URL url, Token token, String doAs)
 
   /**
    * Select a delegation token from all tokens in credentials, based on url.
+   *
+   * @param url url.
+   * @param creds credentials.
+   * @return token.
    */
   @InterfaceAudience.Private
   public org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
@@ -407,6 +411,7 @@ public HttpURLConnection openConnection(URL url, Token token, String doAs)
    * @param token the authentication token with the Delegation Token to renew.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return delegation token long value.
    */
   public long renewDelegationToken(URL url, Token token)
       throws IOException, AuthenticationException {
@@ -423,6 +428,7 @@ public long renewDelegationToken(URL url, Token token)
    * @param doAsUser the user to do as, which will be the token owner.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return delegation token long value.
    */
   public long renewDelegationToken(URL url, Token token, String doAsUser)
       throws IOException, AuthenticationException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index be061bb63f3ee..571e54c5f907c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -125,6 +125,7 @@ protected Properties getConfiguration(String configPrefix,
    * Set AUTH_TYPE property to the name of the corresponding authentication
    * handler class based on the input properties.
    * @param props input properties.
+   * @throws ServletException servlet exception.
    */
   protected void setAuthHandlerClass(Properties props)
       throws ServletException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index b55214451ec25..f4ede6f35edb0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -89,7 +89,7 @@ public abstract class DelegationTokenAuthenticationHandler
 
   public static final String TOKEN_KIND = PREFIX + "token-kind";
 
-  private static final Set<String> DELEGATION_TOKEN_OPS = new HashSet<String>();
+  private static final Set<String> DELEGATION_TOKEN_OPS = new HashSet<>();
 
   public static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
       "hadoop.security.delegation-token.ugi";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 19427dcfafeb4..1b2b6ca1ef693 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -163,6 +163,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
    * @param renewer the renewer user.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return abstract delegation token identifier.
    */
   public Token<AbstractDelegationTokenIdentifier> getDelegationToken(URL url,
       AuthenticatedURL.Token token, String renewer)
@@ -182,6 +183,7 @@ public Token<AbstractDelegationTokenIdentifier> getDelegationToken(URL url,
    * @param doAsUser the user to do as, which will be the token owner.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return abstract delegation token identifier.
    */
   public Token<AbstractDelegationTokenIdentifier> getDelegationToken(URL url,
       AuthenticatedURL.Token token, String renewer, String doAsUser)
@@ -207,8 +209,10 @@ public Token<AbstractDelegationTokenIdentifier> getDelegationToken(URL url,
    * @param url the URL to renew the delegation token from. Only HTTP/S URLs are
    * supported.
    * @param token the authentication token with the Delegation Token to renew.
+   * @param dToken abstract delegation token identifier.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return delegation token long value.
    */
   public long renewDelegationToken(URL url,
       AuthenticatedURL.Token token,
@@ -225,8 +229,10 @@ public long renewDelegationToken(URL url,
    * supported.
    * @param token the authentication token with the Delegation Token to renew.
    * @param doAsUser the user to do as, which will be the token owner.
+   * @param dToken abstract delegation token identifier.
    * @throws IOException if an IO error occurred.
    * @throws AuthenticationException if an authentication exception occurred.
+   * @return delegation token long value.
    */
   public long renewDelegationToken(URL url,
       AuthenticatedURL.Token token,
@@ -245,6 +251,7 @@ public long renewDelegationToken(URL url,
    * @param url the URL to cancel the delegation token from. Only HTTP/S URLs
    * are supported.
    * @param token the authentication token with the Delegation Token to cancel.
+   * @param dToken abstract delegation token identifier.
    * @throws IOException if an IO error occurred.
    */
   public void cancelDelegationToken(URL url,
@@ -261,6 +268,7 @@ public void cancelDelegationToken(URL url,
    * @param url the URL to cancel the delegation token from. Only HTTP/S URLs
    * are supported.
    * @param token the authentication token with the Delegation Token to cancel.
+   * @param dToken abstract delegation token identifier.
    * @param doAsUser the user to do as, which will be the token owner.
    * @throws IOException if an IO error occurred.
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/HttpUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/HttpUserGroupInformation.java
index 614c0d3b36bcf..1f18b1c2f6ae0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/HttpUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/HttpUserGroupInformation.java
@@ -20,8 +20,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.security.UserGroupInformation;
 
-import javax.servlet.http.HttpServletRequest;
-
 /**
  * Util class that returns the remote {@link UserGroupInformation} in scope
  * for the HTTP request.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java
index 9b50e7c524270..bd85a740186b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/AbstractService.java
@@ -241,7 +241,7 @@ public void stop() {
 
   /**
    * Relay to {@link #stop()}
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Override
   public final void close() throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
index ee66e90f7c4ee..b08f9952d863e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/CompositeService.java
@@ -82,7 +82,7 @@ protected void addService(Service service) {
   /**
    * If the passed object is an instance of {@link Service},
    * add it to the list of services managed by this {@link CompositeService}
-   * @param object
+   * @param object object.
    * @return true if a service is added, false otherwise.
    */
   protected boolean addIfService(Object object) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceStateModel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceStateModel.java
index 9f282b9f93483..c075cbb89a8a6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceStateModel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/ServiceStateModel.java
@@ -54,6 +54,8 @@ public class ServiceStateModel {
   /**
    * Create the service state model in the {@link Service.STATE#NOTINITED}
    * state.
+   *
+   * @param name input name.
    */
   public ServiceStateModel(String name) {
     this(name, Service.STATE.NOTINITED);
@@ -62,6 +64,7 @@ public ServiceStateModel(String name) {
   /**
    * Create a service state model instance in the chosen state
    * @param state the starting state
+   * @param name input name.
    */
   public ServiceStateModel(String name, Service.STATE state) {
     this.state = state;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/AbstractLaunchableService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/AbstractLaunchableService.java
index be28c5be2d017..66f8ee430f6e1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/AbstractLaunchableService.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/AbstractLaunchableService.java
@@ -42,6 +42,8 @@ public abstract class AbstractLaunchableService extends AbstractService
 
   /**
    * Construct an instance with the given name.
+   *
+   * @param name input name.
    */
   protected AbstractLaunchableService(String name) {
     super(name);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
index 7fd4657f3fd39..cef69ebd4b789 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/ServiceLauncher.java
@@ -413,7 +413,7 @@ protected List<String> getConfigurationsToCreate() {
   }
 
   /**
-   * This creates all the configurations defined by
+   * @return This creates all the configurations defined by
    * {@link #getConfigurationsToCreate()} , ensuring that
    * the resources have been pushed in.
    * If one cannot be loaded it is logged and the operation continues
@@ -567,6 +567,7 @@ public ExitUtil.ExitException launchService(Configuration conf,
    * @throws Exception any other failure -if it implements
    * {@link ExitCodeProvider} then it defines the exit code for any
    * containing exception
+   * @return status code.
    */
 
   protected int coreServiceLaunch(Configuration conf,
@@ -648,7 +649,7 @@ protected int coreServiceLaunch(Configuration conf,
   }
 
   /**
-   * Instantiate the service defined in {@code serviceClassName}.
+   * @return Instantiate the service defined in {@code serviceClassName}.
    *
    * Sets the {@code configuration} field
    * to the the value of {@code conf},
@@ -852,6 +853,7 @@ protected void error(String message, Throwable thrown) {
    * The service launcher code assumes that after this method is invoked,
    * no other code in the same method is called.
    * @param exitCode code to exit
+   * @param message input message.
    */
   protected void exit(int exitCode, String message) {
     ExitUtil.terminate(exitCode, message);
@@ -1003,7 +1005,7 @@ protected void verifyConfigurationFilesExist(String[] filenames) {
   }
 
   /**
-   * Build a log message for starting up and shutting down. 
+   * @return Build a log message for starting up and shutting down.
    * @param classname the class of the server
    * @param args arguments
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
index f582fa2d97875..ae7acfc1b17f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/package-info.java
@@ -192,9 +192,8 @@ At this point, rather than block waiting for the service to terminate (as
  during the {@code execute()} method takes priority over any exit codes
  returned by the method. This allows services to signal failures simply
  by raising exceptions with exit codes.
- <p>
+ </p>
 
- <p>
  To view the workflow in sequence, it is:
  <ol>
  <li>(prepare configuration files &mdash;covered later)</li>
@@ -221,7 +220,7 @@ At this point, rather than block waiting for the service to terminate (as
  <p>
  For a basic service, the return code is 0 unless an exception
  was raised. 
- <p>
+ </p>
  For a {@link org.apache.hadoop.service.launcher.LaunchableService}, the return
  code is the number returned from the
  {@link org.apache.hadoop.service.launcher.LaunchableService#execute()}
@@ -235,7 +234,7 @@ At this point, rather than block waiting for the service to terminate (as
  of returning error codes to signal failures and for
  normal Services to return any error code at all.
 
- <p>
+ </p>
  Any exception which implements the
  {@link org.apache.hadoop.util.ExitCodeProvider}
  interface is considered be a provider of the exit code: the method
@@ -269,7 +268,7 @@ interface listing common exception codes. These are exception codes
  Note that {@link org.apache.hadoop.util.ExitUtil.ExitException} itself
  implements {@link org.apache.hadoop.util.ExitCodeProvider#getExitCode()}
 
- <p>
+ </p>
  If an exception does not implement
  {@link org.apache.hadoop.util.ExitCodeProvider#getExitCode()},
  it will be wrapped in an {@link org.apache.hadoop.util.ExitUtil.ExitException}
@@ -324,7 +323,7 @@ interface listing common exception codes. These are exception codes
  when received, attempts to stop the service in a limited period of time.
  It then triggers a JVM shutdown by way of
  {@link org.apache.hadoop.util.ExitUtil#terminate(int, String)}
- <p>
+
  If a second signal is received, the
  {@link org.apache.hadoop.service.launcher.InterruptEscalator}
  reacts by triggering an immediate JVM halt, invoking 
@@ -342,7 +341,7 @@ interface listing common exception codes. These are exception codes
  stop the service if a shutdown request is received, so ensuring that
  if the JVM is exited by any thread, an attempt to shut down the service
  will be made.
- 
+ </p>
 
  <h2>Configuration class creation</h2>
 
@@ -355,12 +354,12 @@ interface listing common exception codes. These are exception codes
  What the launcher does do is use reflection to try and create instances of
  these classes simply to force in the common resources. If the classes are
  not on the classpath this fact will be logged.
- <p>
+ </p>
  Applications may consider it essential to either force load in the relevant
  configuration, or pass it down to the service being created. In which
  case further measures may be needed.
  
- <p><b>1: Creation in an extended {@code ServiceLauncher}</b>
+ <b>1: Creation in an extended {@code ServiceLauncher}</b>
  
  <p>
  Subclass the Service launcher and override its
@@ -371,9 +370,9 @@ interface listing common exception codes. These are exception codes
  HDFS or YARN. It does imply a dedicated script to invoke the custom
  {@code main()} method.
 
- <p><b>2: Creation in {@code bindArgs()}</b>
+ <b>2: Creation in {@code bindArgs()}</b>
 
- <p>
+ </p>
  In
  {@link org.apache.hadoop.service.launcher.LaunchableService#bindArgs(Configuration, List)},
  a new configuration is created:
@@ -390,7 +389,7 @@ interface listing common exception codes. These are exception codes
  instances created via the service launcher. It does imply that this is
  expected to be only way that services will be launched.
 
- <p><b>3: Creation in {@code serviceInit()}</b>
+ <p><b>3: Creation in {@code serviceInit()}</b></p>
 
  <pre>
  protected void serviceInit(Configuration conf) throws Exception {
@@ -406,7 +405,7 @@ protected void serviceInit(Configuration conf) throws Exception {
  propagating information between peer services in a
  {@link org.apache.hadoop.service.CompositeService}.
  While a dangerous practice, it does happen.
-
+ </p>
 
  <b>Summary</b>: the ServiceLauncher makes a best-effort attempt to load the
  standard Configuration subclasses, but does not fail if they are not present.
@@ -429,7 +428,7 @@ class (the one created by the
  If this argument is repeated multiple times, all configuration
  files are merged with the latest file on the command line being the
  last one to be applied.
- <p>
+ </p>
  All the {@code --conf &lt;file&gt;} argument pairs are stripped off
  the argument list provided to the instantiated service; they get the
  merged configuration, but not the commands used to create it.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/CommandShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/CommandShell.java
index a53e2259e0e25..25f79a63a233c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/CommandShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/CommandShell.java
@@ -36,6 +36,7 @@ public abstract class CommandShell extends Configured implements Tool {
 
   /**
    * Return usage string for the command including any summary of subcommands.
+   * @return command usage.
    */
   public abstract String getCommandUsage();
 
@@ -84,8 +85,9 @@ public int run(String[] args) throws Exception {
 
   /**
    * Parse the command line arguments and initialize subcommand instance.
-   * @param args
+   * @param args arguments.
    * @return 0 if the argument(s) were recognized, 1 otherwise
+   * @throws Exception init exception.
    */
   protected abstract int init(String[] args) throws Exception;
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetGroupsBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetGroupsBase.java
index 92cdb5835e77d..548409ea58e09 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetGroupsBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetGroupsBase.java
@@ -38,7 +38,7 @@ public abstract class GetGroupsBase extends Configured implements Tool {
   
   /**
    * Create an instance of this tool using the given configuration.
-   * @param conf
+   * @param conf configuration.
    */
   protected GetGroupsBase(Configuration conf) {
     this(conf, System.out);
@@ -84,7 +84,7 @@ public int run(String[] args) throws Exception {
    * 
    * @param conf The configuration to use.
    * @return The address where the service is listening.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected abstract InetSocketAddress getProtocolAddress(Configuration conf)
       throws IOException;
@@ -92,7 +92,7 @@ protected abstract InetSocketAddress getProtocolAddress(Configuration conf)
   /**
    * Get a client of the {@link GetUserMappingsProtocol}.
    * @return A {@link GetUserMappingsProtocol} client proxy.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
     GetUserMappingsProtocol userGroupMappingProtocol =
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetUserMappingsProtocol.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetUserMappingsProtocol.java
index 3e80ac030d3ef..8b1922372ba3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetUserMappingsProtocol.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/GetUserMappingsProtocol.java
@@ -40,7 +40,7 @@ public interface GetUserMappingsProtocol {
    * Get the groups which are mapped to the given user.
    * @param user The user to get the groups for.
    * @return The set of groups the user belongs to.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @Idempotent
   public String[] getGroupsForUser(String user) throws IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
index 348f86fe1368b..d1d933c737d2b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
@@ -155,7 +155,9 @@ public Builder addField(String title, Justification justification,
     }
 
     /**
-     * Whether to hide column headers in table output
+     * Whether to hide column headers in table output.
+     *
+     * @return Builder.
      */
     public Builder hideHeaders() {
       this.showHeader = false;
@@ -164,6 +166,8 @@ public Builder hideHeaders() {
 
     /**
      * Whether to show column headers in table output. This is the default.
+     *
+     * @return Builder.
      */
     public Builder showHeaders() {
       this.showHeader = true;
@@ -173,6 +177,9 @@ public Builder showHeaders() {
     /**
      * Set the maximum width of a row in the TableListing. Must have one or
      * more wrappable fields for this to take effect.
+     *
+     * @param width width.
+     * @return Builder.
      */
     public Builder wrapWidth(int width) {
       this.wrapWidth = width;
@@ -181,6 +188,8 @@ public Builder wrapWidth(int width) {
 
     /**
      * Create a new TableListing.
+     *
+     * @return TableListing.
      */
     public TableListing build() {
       return new TableListing(columns.toArray(new Column[0]), showHeader,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AsyncDiskService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AsyncDiskService.java
index 8e48cb955a3a7..a3bf4faf0a980 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AsyncDiskService.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/AsyncDiskService.java
@@ -94,6 +94,9 @@ public Thread newThread(Runnable r) {
   
   /**
    * Execute the task sometime in the future, using ThreadPools.
+   *
+   * @param root root.
+   * @param task task.
    */
   public synchronized void execute(String root, Runnable task) {
     ThreadPoolExecutor executor = executors.get(root);
@@ -123,7 +126,7 @@ public synchronized void shutdown() {
    * 
    * @param milliseconds  The number of milliseconds to wait
    * @return   true if all thread pools are terminated without time limit
-   * @throws InterruptedException 
+   * @throws InterruptedException if the thread is interrupted.
    */
   public synchronized boolean awaitTermination(long milliseconds) 
       throws InterruptedException {
@@ -145,6 +148,8 @@ public synchronized boolean awaitTermination(long milliseconds)
   
   /**
    * Shut down all ThreadPools immediately.
+   *
+   * @return Runnable List.
    */
   public synchronized List<Runnable> shutdownNow() {
     
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java
index d08e84f99de29..5c90e4bd2d601 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BlockingThreadPoolExecutorService.java
@@ -117,6 +117,7 @@ private BlockingThreadPoolExecutorService(int permitCount,
    * @param keepAliveTime time until threads are cleaned up in {@code unit}
    * @param unit time unit
    * @param prefixName prefix of name for threads
+   * @return BlockingThreadPoolExecutorService.
    */
   public static BlockingThreadPoolExecutorService newInstance(
       int activeTasks,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
index 4023995941f5c..5bf773cef3836 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcComposer.java
@@ -51,6 +51,11 @@ public class CrcComposer {
   /**
    * Returns a CrcComposer which will collapse all ingested CRCs into a single
    * value.
+   *
+   * @param type type.
+   * @param bytesPerCrcHint bytesPerCrcHint.
+   * @throws IOException raised on errors performing I/O.
+   * @return a CrcComposer which will collapse all ingested CRCs into a single value.
    */
   public static CrcComposer newCrcComposer(
       DataChecksum.Type type, long bytesPerCrcHint)
@@ -67,6 +72,13 @@ public static CrcComposer newCrcComposer(
    * final digest, each corresponding to 10 underlying data bytes. Using
    * a stripeLength greater than the total underlying data size is equivalent
    * to using a non-striped CrcComposer.
+   *
+   * @param type type.
+   * @param bytesPerCrcHint bytesPerCrcHint.
+   * @param stripeLength stripeLength.
+   * @return a CrcComposer which will collapse CRCs for every combined.
+   * underlying data size which aligns with the specified stripe boundary.
+   * @throws IOException raised on errors performing I/O.
    */
   public static CrcComposer newStripedCrcComposer(
       DataChecksum.Type type, long bytesPerCrcHint, long stripeLength)
@@ -102,7 +114,11 @@ public static CrcComposer newStripedCrcComposer(
    * each CRC expected to correspond to exactly {@code bytesPerCrc} underlying
    * data bytes.
    *
+   * @param crcBuffer crcBuffer.
+   * @param offset offset.
    * @param length must be a multiple of the expected byte-size of a CRC.
+   * @param bytesPerCrc bytesPerCrc.
+   * @throws IOException raised on errors performing I/O.
    */
   public void update(
       byte[] crcBuffer, int offset, int length, long bytesPerCrc)
@@ -125,6 +141,11 @@ public void update(
    * Composes {@code numChecksumsToRead} additional CRCs into the current digest
    * out of {@code checksumIn}, with each CRC expected to correspond to exactly
    * {@code bytesPerCrc} underlying data bytes.
+   *
+   * @param checksumIn checksumIn.
+   * @param numChecksumsToRead numChecksumsToRead.
+   * @param bytesPerCrc bytesPerCrc.
+   * @throws IOException raised on errors performing I/O.
    */
   public void update(
       DataInputStream checksumIn, long numChecksumsToRead, long bytesPerCrc)
@@ -138,6 +159,10 @@ public void update(
   /**
    * Updates with a single additional CRC which corresponds to an underlying
    * data size of {@code bytesPerCrc}.
+   *
+   * @param crcB crcB.
+   * @param bytesPerCrc bytesPerCrc.
+   * @throws IOException raised on errors performing I/O.
    */
   public void update(int crcB, long bytesPerCrc) throws IOException {
     if (curCompositeCrc == 0) {
@@ -173,6 +198,8 @@ public void update(int crcB, long bytesPerCrc) throws IOException {
    * total sum bytesPerCrc divided by stripeLength. If the sum of bytesPerCrc
    * is not a multiple of stripeLength, then the last CRC in the array
    * corresponds to totalLength % stripeLength underlying data bytes.
+   *
+   * @return byte representation of composed CRCs.
    */
   public byte[] digest() {
     if (curPositionInStripe > 0) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
index 42eaf148d64c3..c8183b042fb1a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CrcUtil.java
@@ -44,6 +44,10 @@ private CrcUtil() {
    * Compute x^({@code lengthBytes} * 8) mod {@code mod}, where {@code mod} is
    * in "reversed" (little-endian) format such that {@code mod & 1} represents
    * x^31 and has an implicit term x^32.
+   *
+   * @param lengthBytes lengthBytes.
+   * @param mod mod.
+   * @return monomial.
    */
   public static int getMonomial(long lengthBytes, int mod) {
     if (lengthBytes == 0) {
@@ -73,7 +77,13 @@ public static int getMonomial(long lengthBytes, int mod) {
   }
 
   /**
+   * composeWithMonomial.
+   *
+   * @param crcA crcA.
+   * @param crcB crcB.
    * @param monomial Precomputed x^(lengthBInBytes * 8) mod {@code mod}
+   * @param mod mod.
+   * @return compose with monomial.
    */
   public static int composeWithMonomial(
       int crcA, int crcB, int monomial, int mod) {
@@ -81,7 +91,13 @@ public static int composeWithMonomial(
   }
 
   /**
+   * compose.
+   *
+   * @param crcA crcA.
+   * @param crcB crcB.
    * @param lengthB length of content corresponding to {@code crcB}, in bytes.
+   * @param mod mod.
+   * @return compose result.
    */
   public static int compose(int crcA, int crcB, long lengthB, int mod) {
     int monomial = getMonomial(lengthB, mod);
@@ -91,6 +107,8 @@ public static int compose(int crcA, int crcB, long lengthB, int mod) {
   /**
    * @return 4-byte array holding the big-endian representation of
    *     {@code value}.
+   *
+   * @param value value.
    */
   public static byte[] intToBytes(int value) {
     byte[] buf = new byte[4];
@@ -110,6 +128,11 @@ public static byte[] intToBytes(int value) {
    * Writes big-endian representation of {@code value} into {@code buf}
    * starting at {@code offset}. buf.length must be greater than or
    * equal to offset + 4.
+   *
+   * @param buf buf size.
+   * @param offset offset.
+   * @param value value.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void writeInt(byte[] buf, int offset, int value)
       throws IOException {
@@ -127,6 +150,11 @@ public static void writeInt(byte[] buf, int offset, int value)
   /**
    * Reads 4-byte big-endian int value from {@code buf} starting at
    * {@code offset}. buf.length must be greater than or equal to offset + 4.
+   *
+   * @param offset offset.
+   * @param buf buf.
+   * @return int.
+   * @throws IOException raised on errors performing I/O.
    */
   public static int readInt(byte[] buf, int offset)
       throws IOException {
@@ -146,6 +174,10 @@ public static int readInt(byte[] buf, int offset)
    * For use with debug statements; verifies bytes.length on creation,
    * expecting it to represent exactly one CRC, and returns a hex
    * formatted value.
+   *
+   * @param bytes bytes.
+   * @throws IOException raised on errors performing I/O.
+   * @return a list of hex formatted values.
    */
   public static String toSingleCrcString(final byte[] bytes)
       throws IOException {
@@ -161,6 +193,10 @@ public static String toSingleCrcString(final byte[] bytes)
    * For use with debug statements; verifies bytes.length on creation,
    * expecting it to be divisible by CRC byte size, and returns a list of
    * hex formatted values.
+   *
+   * @param bytes bytes.
+   * @throws IOException raised on errors performing I/O.
+   * @return a list of hex formatted values.
    */
   public static String toMultiCrcString(final byte[] bytes)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Daemon.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Daemon.java
index 3b95db6693e03..f735b82e4289b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Daemon.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Daemon.java
@@ -52,14 +52,21 @@ public Daemon() {
     super();
   }
 
-  /** Construct a daemon thread. */
+  /**
+   * Construct a daemon thread.
+   * @param runnable runnable.
+   */
   public Daemon(Runnable runnable) {
     super(runnable);
     this.runnable = runnable;
     this.setName(((Object)runnable).toString());
   }
 
-  /** Construct a daemon thread to be part of a specified thread group. */
+  /**
+   * Construct a daemon thread to be part of a specified thread group.
+   * @param group thread group.
+   * @param runnable runnable.
+   */
   public Daemon(ThreadGroup group, Runnable runnable) {
     super(group, runnable);
     this.runnable = runnable;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
index 32a0adca1979a..1c37d5944c6f2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
@@ -69,7 +69,12 @@ public enum Type {
       this.size = size;
     }
 
-    /** @return the type corresponding to the id. */
+    /**
+     * the type corresponding to the id.
+     *
+     * @return the type corresponding to the id.
+     * @param id id.
+     */
     public static Type valueOf(int id) {
       if (id < 0 || id >= values().length) {
         throw new IllegalArgumentException("id=" + id
@@ -82,6 +87,8 @@ public static Type valueOf(int id) {
   /**
    * Create a Crc32 Checksum object. The implementation of the Crc32 algorithm
    * is chosen depending on the platform.
+   *
+   * @return Checksum.
    */
   public static Checksum newCrc32() {
     return new CRC32();
@@ -105,6 +112,9 @@ static Checksum newCrc32C() {
   }
 
   /**
+   * getCrcPolynomialForType.
+   *
+   * @param type type.
    * @return the int representation of the polynomial associated with the
    *     CRC {@code type}, suitable for use with further CRC arithmetic.
    * @throws IOException if there is no CRC polynomial applicable
@@ -141,7 +151,11 @@ public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
   
   /**
    * Creates a DataChecksum from HEADER_LEN bytes from arr[offset].
+   *
+   * @param bytes bytes.
+   * @param offset offset.
    * @return DataChecksum of the type in the array or null in case of an error.
+   * @throws IOException raised on errors performing I/O.
    */
   public static DataChecksum newDataChecksum(byte[] bytes, int offset)
       throws IOException {
@@ -168,7 +182,12 @@ public static DataChecksum newDataChecksum(byte[] bytes, int offset)
   
   /**
    * This constructs a DataChecksum by reading HEADER_LEN bytes from input
-   * stream <i>in</i>
+   * stream <i>in</i>.
+   *
+   * @param in data input stream.
+   * @throws IOException raised on errors performing I/O.
+   * @return DataChecksum by reading HEADER_LEN
+   *         bytes from input stream.
    */
   public static DataChecksum newDataChecksum( DataInputStream in )
                                  throws IOException {
@@ -194,6 +213,9 @@ private static Type mapByteToChecksumType(int type)
   
   /**
    * Writes the checksum header to the output stream <i>out</i>.
+   *
+   * @param out output stream.
+   * @throws IOException raised on errors performing I/O.
    */
   public void writeHeader( DataOutputStream out ) 
                            throws IOException { 
@@ -215,7 +237,11 @@ public byte[] getHeader() {
   /**
    * Writes the current checksum to the stream.
    * If <i>reset</i> is true, then resets the checksum.
+   *
+   * @param out out.
+   * @param reset reset.
    * @return number of bytes written. Will be equal to getChecksumSize();
+   * @throws IOException raised on errors performing I/O.
    */
    public int writeValue( DataOutputStream out, boolean reset )
                           throws IOException {
@@ -239,7 +265,12 @@ public int writeValue( DataOutputStream out, boolean reset )
    /**
     * Writes the current checksum to a buffer.
     * If <i>reset</i> is true, then resets the checksum.
+    *
+    * @param buf buf.
+    * @param offset offset.
+    * @param reset reset.
     * @return number of bytes written. Will be equal to getChecksumSize();
+    * @throws IOException raised on errors performing I/O.
     */
     public int writeValue( byte[] buf, int offset, boolean reset )
                            throws IOException {
@@ -266,6 +297,9 @@ public int writeValue( byte[] buf, int offset, boolean reset )
    
    /**
     * Compares the checksum located at buf[offset] with the current checksum.
+    *
+    * @param buf buf.
+    * @param offset offset.
     * @return true if the checksum matches and false otherwise.
     */
    public boolean compare( byte buf[], int offset ) {
@@ -295,12 +329,19 @@ public Type getChecksumType() {
     return type;
   }
 
-  /** @return the size for a checksum. */
+  /**
+   * the size for a checksum.
+   * @return the size for a checksum.
+   */
   public int getChecksumSize() {
     return type.size;
   }
 
-  /** @return the required checksum size given the data length. */
+  /**
+   * the required checksum size given the data length.
+   * @param dataSize data size.
+   * @return the required checksum size given the data length.
+   */
   public int getChecksumSize(int dataSize) {
     return ((dataSize - 1)/getBytesPerChecksum() + 1) * getChecksumSize(); 
   }
@@ -525,6 +566,12 @@ public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) {
   /**
    * Implementation of chunked calculation specifically on byte arrays. This
    * is to avoid the copy when dealing with ByteBuffers that have array backing.
+   *
+   * @param data data.
+   * @param dataOffset dataOffset.
+   * @param dataLength dataLength.
+   * @param sums sums.
+   * @param sumsOffset sumsOffset.
    */
   public void calculateChunkedSums(
       byte[] data, int dataOffset, int dataLength,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java
index 3951ec2609c2c..8808e1067284a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java
@@ -53,6 +53,9 @@ public class DirectBufferPool {
    * Allocate a direct buffer of the specified size, in bytes.
    * If a pooled buffer is available, returns that. Otherwise
    * allocates a new one.
+   *
+   * @param size size.
+   * @return ByteBuffer.
    */
   public ByteBuffer getBuffer(int size) {
     Queue<WeakReference<ByteBuffer>> list = buffersBySize.get(size);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
index 446aad949a2f6..9fc18ca3bbf92 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
@@ -70,8 +70,8 @@ public DiskOutOfSpaceException(String msg) {
    * Create the directory if it doesn't exist and check that dir is readable,
    * writable and executable
    *  
-   * @param dir
-   * @throws DiskErrorException
+   * @param dir dir.
+   * @throws DiskErrorException disk problem.
    */
   public static void checkDir(File dir) throws DiskErrorException {
     checkDirInternal(dir);
@@ -82,8 +82,8 @@ public static void checkDir(File dir) throws DiskErrorException {
    * readable, writable and executable. Perform some disk IO to
    * ensure that the disk is usable for writes.
    *
-   * @param dir
-   * @throws DiskErrorException
+   * @param dir dir.
+   * @throws DiskErrorException disk problem.
    */
   public static void checkDirWithDiskIo(File dir)
       throws DiskErrorException {
@@ -107,8 +107,8 @@ private static void checkDirInternal(File dir)
    * @param localFS local filesystem
    * @param dir directory
    * @param expected permission
-   * @throws DiskErrorException
-   * @throws IOException
+   * @throws DiskErrorException disk problem.
+   * @throws IOException raised on errors performing I/O.
    */
   public static void checkDir(LocalFileSystem localFS, Path dir,
                               FsPermission expected)
@@ -125,8 +125,8 @@ public static void checkDir(LocalFileSystem localFS, Path dir,
    * @param localFS local filesystem
    * @param dir directory
    * @param expected permission
-   * @throws DiskErrorException
-   * @throws IOException
+   * @throws DiskErrorException disk problem.
+   * @throws IOException raised on errors performing I/O.
    */  
   public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir,
                                         FsPermission expected) 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
index 67ded618d25b1..97d0c812667a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java
@@ -40,6 +40,7 @@ private DiskValidatorFactory() {
   /**
    * Returns a {@link DiskValidator} instance corresponding to the passed clazz.
    * @param clazz a class extends {@link DiskValidator}
+   * @return disk validator.
    */
   public static DiskValidator
       getInstance(Class<? extends DiskValidator> clazz) {
@@ -66,6 +67,7 @@ private DiskValidatorFactory() {
    * or "read-write" for {@link ReadWriteDiskValidator}.
    * @param diskValidator canonical class name, for example, "basic"
    * @throws DiskErrorException if the class cannot be located
+   * @return disk validator.
    */
   @SuppressWarnings("unchecked")
   public static DiskValidator getInstance(String diskValidator)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
index 32f4b5b7a72d4..dd47aeeefac2c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.util;
 
+import java.util.concurrent.atomic.AtomicReference;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.slf4j.Logger;
@@ -36,8 +38,10 @@ public final class ExitUtil {
       LOG = LoggerFactory.getLogger(ExitUtil.class.getName());
   private static volatile boolean systemExitDisabled = false;
   private static volatile boolean systemHaltDisabled = false;
-  private static volatile ExitException firstExitException;
-  private static volatile HaltException firstHaltException;
+  private static final AtomicReference<ExitException> FIRST_EXIT_EXCEPTION =
+      new AtomicReference<>();
+  private static final AtomicReference<HaltException> FIRST_HALT_EXCEPTION =
+      new AtomicReference<>();
   /** Message raised from an exit exception if none were provided: {@value}. */
   public static final String EXIT_EXCEPTION_MESSAGE = "ExitException";
   /** Message raised from a halt exception if none were provided: {@value}. */
@@ -159,28 +163,29 @@ public static void disableSystemHalt() {
    */
   public static boolean terminateCalled() {
     // Either we set this member or we actually called System#exit
-    return firstExitException != null;
+    return FIRST_EXIT_EXCEPTION.get() != null;
   }
 
   /**
    * @return true if halt has been called.
    */
   public static boolean haltCalled() {
-    return firstHaltException != null;
+    // Either we set this member or we actually called Runtime#halt
+    return FIRST_HALT_EXCEPTION.get() != null;
   }
 
   /**
-   * @return the first ExitException thrown, null if none thrown yet.
+   * @return the first {@code ExitException} thrown, null if none thrown yet.
    */
   public static ExitException getFirstExitException() {
-    return firstExitException;
+    return FIRST_EXIT_EXCEPTION.get();
   }
 
   /**
    * @return the first {@code HaltException} thrown, null if none thrown yet.
    */
   public static HaltException getFirstHaltException() {
-    return firstHaltException;
+    return FIRST_HALT_EXCEPTION.get();
   }
 
   /**
@@ -188,63 +193,136 @@ public static HaltException getFirstHaltException() {
    * where one test in the suite expects an exit but others do not.
    */
   public static void resetFirstExitException() {
-    firstExitException = null;
+    FIRST_EXIT_EXCEPTION.set(null);
   }
 
+  /**
+   * Reset the tracking of process termination. This is for use in unit tests
+   * where one test in the suite expects a halt but others do not.
+   */
   public static void resetFirstHaltException() {
-    firstHaltException = null;
+    FIRST_HALT_EXCEPTION.set(null);
   }
 
   /**
+   * Suppresses if legit and returns the first non-null of the two. Legit means
+   * <code>suppressor</code> if neither <code>null</code> nor <code>suppressed</code>.
+   * @param suppressor <code>Throwable</code> that suppresses <code>suppressed</code>
+   * @param suppressed <code>Throwable</code> that is suppressed by <code>suppressor</code>
+   * @return <code>suppressor</code> if not <code>null</code>, <code>suppressed</code> otherwise
+   */
+  private static <T extends Throwable> T addSuppressed(T suppressor, T suppressed) {
+    if (suppressor == null) {
+      return suppressed;
+    }
+    if (suppressor != suppressed) {
+      suppressor.addSuppressed(suppressed);
+    }
+    return suppressor;
+  }
+
+  /**
+   * Exits the JVM if exit is enabled, rethrow provided exception or any raised error otherwise.
    * Inner termination: either exit with the exception's exit code,
    * or, if system exits are disabled, rethrow the exception.
    * @param ee exit exception
+   * @throws ExitException if {@link System#exit(int)} is disabled and not suppressed by an Error
+   * @throws Error if {@link System#exit(int)} is disabled and one Error arise, suppressing
+   * anything else, even <code>ee</code>
    */
-  public static synchronized void terminate(ExitException ee)
-      throws ExitException {
-    int status = ee.getExitCode();
-    String msg = ee.getMessage();
+  public static void terminate(final ExitException ee) throws ExitException {
+    final int status = ee.getExitCode();
+    Error caught = null;
     if (status != 0) {
-      //exit indicates a problem, log it
-      LOG.debug("Exiting with status {}: {}",  status, msg, ee);
-      LOG.info("Exiting with status {}: {}", status, msg);
+      try {
+        // exit indicates a problem, log it
+        String msg = ee.getMessage();
+        LOG.debug("Exiting with status {}: {}",  status, msg, ee);
+        LOG.info("Exiting with status {}: {}", status, msg);
+      } catch (Error e) {
+        // errors have higher priority than HaltException, it may be re-thrown.
+        // OOM and ThreadDeath are 2 examples of Errors to re-throw
+        caught = e;
+      } catch (Throwable t) {
+        // all other kind of throwables are suppressed
+        addSuppressed(ee, t);
+      }
     }
     if (systemExitDisabled) {
-      LOG.error("Terminate called", ee);
-      if (!terminateCalled()) {
-        firstExitException = ee;
+      try {
+        LOG.error("Terminate called", ee);
+      } catch (Error e) {
+        // errors have higher priority again, if it's a 2nd error, the 1st one suprpesses it
+        caught = addSuppressed(caught, e);
+      } catch (Throwable t) {
+        // all other kind of throwables are suppressed
+        addSuppressed(ee, t);
       }
+      FIRST_EXIT_EXCEPTION.compareAndSet(null, ee);
+      if (caught != null) {
+        caught.addSuppressed(ee);
+        throw caught;
+      }
+      // not suppressed by a higher prority error
       throw ee;
+    } else {
+      // when exit is enabled, whatever Throwable happened, we exit the VM
+      System.exit(status);
     }
-    System.exit(status);
   }
 
   /**
-   * Forcibly terminates the currently running Java virtual machine.
-   * The exception argument is rethrown if JVM halting is disabled.
-   * @param ee the exception containing the status code, message and any stack
+   * Halts the JVM if halt is enabled, rethrow provided exception or any raised error otherwise.
+   * If halt is disabled, this method throws either the exception argument if no
+   * error arise, the first error if at least one arise, suppressing <code>he</code>.
+   * If halt is enabled, all throwables are caught, even errors.
+   *
+   * @param he the exception containing the status code, message and any stack
    * trace.
-   * @throws HaltException if {@link Runtime#halt(int)} is disabled.
+   * @throws HaltException if {@link Runtime#halt(int)} is disabled and not suppressed by an Error
+   * @throws Error if {@link Runtime#halt(int)} is disabled and one Error arise, suppressing
+   * anyuthing else, even <code>he</code>
    */
-  public static synchronized void halt(HaltException ee) throws HaltException {
-    int status = ee.getExitCode();
-    String msg = ee.getMessage();
-    try {
-      if (status != 0) {
-        //exit indicates a problem, log it
-        LOG.info("Halt with status {}: {}", status, msg, ee);
+  public static void halt(final HaltException he) throws HaltException {
+    final int status = he.getExitCode();
+    Error caught = null;
+    if (status != 0) {
+      try {
+        // exit indicates a problem, log it
+        String msg = he.getMessage();
+        LOG.info("Halt with status {}: {}", status, msg, he);
+      } catch (Error e) {
+        // errors have higher priority than HaltException, it may be re-thrown.
+        // OOM and ThreadDeath are 2 examples of Errors to re-throw
+        caught = e;
+      } catch (Throwable t) {
+        // all other kind of throwables are suppressed
+        addSuppressed(he, t);
       }
-    } catch (Exception ignored) {
-      // ignore exceptions here, as it may be due to an out of memory situation
     }
+    // systemHaltDisabled is volatile and not used in scenario nheding atomicty,
+    // thus it does not nhed a synchronized access nor a atomic access
     if (systemHaltDisabled) {
-      LOG.error("Halt called", ee);
-      if (!haltCalled()) {
-        firstHaltException = ee;
+      try {
+        LOG.error("Halt called", he);
+      } catch (Error e) {
+        // errors have higher priority again, if it's a 2nd error, the 1st one suprpesses it
+        caught = addSuppressed(caught, e);
+      } catch (Throwable t) {
+        // all other kind of throwables are suppressed
+        addSuppressed(he, t);
       }
-      throw ee;
+      FIRST_HALT_EXCEPTION.compareAndSet(null, he);
+      if (caught != null) {
+        caught.addSuppressed(he);
+        throw caught;
+      }
+      // not suppressed by a higher prority error
+      throw he;
+    } else {
+      // when halt is enabled, whatever Throwable happened, we halt the VM
+      Runtime.getRuntime().halt(status);
     }
-    Runtime.getRuntime().halt(status);
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
index 47aa9cc71a12e..31dfe594207be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FileBasedIPList.java
@@ -58,7 +58,7 @@ public FileBasedIPList(String fileName) {
       lines = null;
     }
     if (lines != null) {
-      addressList = new MachineList(new HashSet<String>(Arrays.asList(lines)));
+      addressList = new MachineList(new HashSet<>(Arrays.asList(lines)));
     } else {
       addressList = null;
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
index e51f7b14a60e0..268b4e166e8a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/FindClass.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import java.io.IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
index f189708692d8b..95d0d4d290ccd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
@@ -54,6 +54,8 @@ public static class Builder {
 
     /**
      * Set observation window size in milliseconds.
+     * @param value value.
+     * @return window size in milliseconds.
      */
     public Builder observationWindowMs(long value) {
       this.observationWindowMs = value;
@@ -62,6 +64,8 @@ public Builder observationWindowMs(long value) {
 
     /**
      * Set sleep interval in milliseconds.
+     * @param value value.
+     * @return IntervalMs.
      */
     public Builder sleepIntervalMs(long value) {
       this.sleepIntervalMs = value;
@@ -70,6 +74,8 @@ public Builder sleepIntervalMs(long value) {
 
     /**
      * Set the max GC time percentage that triggers the alert handler.
+     * @param value value.
+     * @return max GC time percentage.
      */
     public Builder maxGcTimePercentage(int value) {
       this.maxGcTimePercentage = value;
@@ -78,6 +84,8 @@ public Builder maxGcTimePercentage(int value) {
 
     /**
      * Set the GC alert handler.
+     * @param value value.
+     * @return GC alert handler.
      */
     public Builder gcTimeAlertHandler(GcTimeAlertHandler value) {
       this.handler = value;
@@ -167,7 +175,10 @@ public void shutdown() {
     shouldRun = false;
   }
 
-  /** Returns a copy of the most recent data measured by this monitor. */
+  /**
+   * Returns a copy of the most recent data measured by this monitor.
+   * @return a copy of the most recent data measured by this monitor
+   */
   public GcData getLatestGcData() {
     return curData.clone();
   }
@@ -227,22 +238,34 @@ public static class GcData implements Cloneable {
     private long gcMonitorRunTime, totalGcTime, totalGcCount;
     private int gcTimePercentage;
 
-    /** Returns the absolute timestamp when this measurement was taken. */
+    /**
+     * Returns the absolute timestamp when this measurement was taken.
+     * @return timestamp.
+     */
     public long getTimestamp() {
       return timestamp;
     }
 
-    /** Returns the time since the start of the associated GcTimeMonitor. */
+    /**
+     * Returns the time since the start of the associated GcTimeMonitor.
+     * @return GcMonitorRunTime.
+     */
     public long getGcMonitorRunTime() {
       return gcMonitorRunTime;
     }
 
-    /** Returns accumulated GC time since this JVM started. */
+    /**
+     * Returns accumulated GC time since this JVM started.
+     * @return AccumulatedGcTime.
+     */
     public long getAccumulatedGcTime() {
       return totalGcTime;
     }
 
-    /** Returns the accumulated number of GC pauses since this JVM started. */
+    /**
+     * Returns the accumulated number of GC pauses since this JVM started.
+     * @return AccumulatedGcCount.
+     */
     public long getAccumulatedGcCount() {
       return totalGcCount;
     }
@@ -250,6 +273,8 @@ public long getAccumulatedGcCount() {
     /**
      * Returns the percentage (0..100) of time that the JVM spent in GC pauses
      * within the observation window of the associated GcTimeMonitor.
+     *
+     * @return GcTimePercentage.
      */
     public int getGcTimePercentage() {
       return gcTimePercentage;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
index 8ca7a904fdc84..81e1fb5d21234 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericOptionsParser.java
@@ -57,8 +57,9 @@
  * 
  * <h3 id="GenericOptions">Generic Options</h3>
  * 
- * <p>The supported generic options are:
- * <p><blockquote><pre>
+ * <p>The supported generic options are:</p>
+ * <blockquote>
+ * <pre>
  *     -conf &lt;configuration file&gt;     specify a configuration file
  *     -D &lt;property=value&gt;            use value for given property
  *     -fs &lt;local|namenode:port&gt;      specify a namenode
@@ -69,13 +70,15 @@
  *                            jar files to include in the classpath.
  *     -archives &lt;comma separated list of archives&gt;    specify comma
  *             separated archives to be unarchived on the compute machines.
-
- * </pre></blockquote><p>
+ * </pre>
+ * </blockquote>
  * 
  * <p>The general command line syntax is:</p>
- * <p><pre><code>
+ * <pre>
+ * <code>
  * bin/hadoop command [genericOptions] [commandOptions]
- * </code></pre><p>
+ * </code>
+ * </pre>
  * 
  * <p>Generic command line arguments <strong>might</strong> modify 
  * <code>Configuration </code> objects, given to constructors.</p>
@@ -83,7 +86,9 @@
  * <p>The functionality is implemented using Commons CLI.</p>
  *
  * <p>Examples:</p>
- * <p><blockquote><pre>
+ *
+ * <blockquote>
+ * <pre>
  * $ bin/hadoop dfs -fs darwin:8020 -ls /data
  * list /data directory in dfs with namenode darwin:8020
  * 
@@ -105,7 +110,9 @@
  * $ bin/hadoop jar -libjars testlib.jar 
  * -archives test.tgz -files file.txt inputjar args
  * job submission with libjars, files and archives
- * </pre></blockquote><p>
+ * </pre>
+ * </blockquote>
+ *
  *
  * @see Tool
  * @see ToolRunner
@@ -124,7 +131,7 @@ public class GenericOptionsParser {
    * Create an options parser with the given options to parse the args.
    * @param opts the options
    * @param args the command line arguments
-   * @throws IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public GenericOptionsParser(Options opts, String[] args) 
       throws IOException {
@@ -134,7 +141,7 @@ public GenericOptionsParser(Options opts, String[] args)
   /**
    * Create an options parser to parse the args.
    * @param args the command line arguments
-   * @throws IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public GenericOptionsParser(String[] args) 
       throws IOException {
@@ -150,7 +157,7 @@ public GenericOptionsParser(String[] args)
    * 
    * @param conf the <code>Configuration</code> to modify.
    * @param args command-line arguments.
-   * @throws IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public GenericOptionsParser(Configuration conf, String[] args) 
       throws IOException {
@@ -167,7 +174,7 @@ public GenericOptionsParser(Configuration conf, String[] args)
    * @param conf the configuration to modify  
    * @param options options built by the caller 
    * @param args User-specified arguments
-   * @throws IOException 
+   * @throws IOException raised on errors performing I/O.
    */
   public GenericOptionsParser(Configuration conf,
       Options options, String[] args) throws IOException {
@@ -217,9 +224,10 @@ public boolean isParseSuccessful() {
   }
 
   /**
-   * Specify properties of each generic option.
+   * @return Specify properties of each generic option.
    * <i>Important</i>: as {@link OptionBuilder} is not thread safe, subclasses
    * must synchronize use on {@code OptionBuilder.class}
+   * @param opts input opts.
    */
   @SuppressWarnings("static-access")
   protected Options buildGeneralOptions(Options opts) {
@@ -359,9 +367,9 @@ private void processGeneralOptions(CommandLine line) throws IOException {
   
   /**
    * If libjars are set in the conf, parse the libjars.
-   * @param conf
+   * @param conf input Configuration.
    * @return libjar urls
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static URL[] getLibJars(Configuration conf) throws IOException {
     String jars = conf.get("tmpjars");
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
index 0aba34845a676..2d35b15bc5900 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
@@ -51,6 +51,8 @@ public static <T> Class<T> getClass(T t) {
    * <code>T[]</code>.
    * @param c the Class object of the items in the list
    * @param list the list to convert
+   * @param <T> Generics Type T.
+   * @return T Array.
    */
   public static <T> T[] toArray(Class<T> c, List<T> list)
   {
@@ -67,8 +69,10 @@ public static <T> T[] toArray(Class<T> c, List<T> list)
    * Converts the given <code>List&lt;T&gt;</code> to a an array of 
    * <code>T[]</code>. 
    * @param list the list to convert
+   * @param <T> Generics Type T.
    * @throws ArrayIndexOutOfBoundsException if the list is empty. 
    * Use {@link #toArray(Class, List)} if the list may be empty.
+   * @return T Array.
    */
   public static <T> T[] toArray(List<T> list) {
     return toArray(getClass(list.get(0)), list);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index 5141740a3d23e..300f8145c3162 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -135,7 +135,7 @@ public static void readFileToMapWithFileInputStream(String type,
     if (xmlInput) {
       readXmlFileToMapWithFileInputStream(type, filename, inputStream, map);
     } else {
-      HashSet<String> nodes = new HashSet<String>();
+      HashSet<String> nodes = new HashSet<>();
       readFileToSetWithFileInputStream(type, filename, inputStream, nodes);
       for (String node : nodes) {
         map.put(node, null);
@@ -147,8 +147,8 @@ public static void readXmlFileToMapWithFileInputStream(String type,
       String filename, InputStream fileInputStream, Map<String, Integer> map)
           throws IOException {
     Document dom;
-    DocumentBuilderFactory builder = DocumentBuilderFactory.newInstance();
     try {
+      DocumentBuilderFactory builder = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = builder.newDocumentBuilder();
       dom = db.parse(fileInputStream);
       // Examples:
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java
index 3a2616376fbac..71cdcf11656ae 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IPList.java
@@ -26,7 +26,7 @@ public interface IPList {
 
   /**
    * returns true if the ipAddress is in the IPList.
-   * @param ipAddress
+   * @param ipAddress ipAddress.
    * @return boolean value indicating whether the ipAddress is in the IPList
    */
   public abstract boolean isIn(String ipAddress);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
index c14727a3771da..49ae8a8605e98 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdGenerator.java
@@ -26,6 +26,9 @@
 @InterfaceAudience.Private
 public interface IdGenerator {
 
-  /** Increment and then return the next value. */
+  /**
+   * Increment and then return the next value.
+   * @return long value.
+   */
   public long nextValue();
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java
index ecf099feff9d0..25e79f651b944 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IdentityHashStore.java
@@ -111,6 +111,9 @@ private void putInternal(Object k, Object v) {
    * Inserting a new (key, value) never overwrites a previous one.
    * In other words, you can insert the same key multiple times and it will
    * lead to multiple entries.
+   *
+   * @param k Generics Type k.
+   * @param v Generics Type v.
    */
   public void put(K k, V v) {
     Preconditions.checkNotNull(k);
@@ -142,6 +145,9 @@ private int getElementIndex(K k) {
 
   /**
    * Retrieve a value associated with a given key.
+   *
+   * @param k Generics Type k.
+   * @return Generics Type V.
    */
   public V get(K k) {
     int index = getElementIndex(k);
@@ -154,6 +160,9 @@ public V get(K k) {
   /**
    * Retrieve a value associated with a given key, and delete the
    * relevant entry.
+   *
+   * @param k Generics Type k.
+   * @return Generics Type V.
    */
   public V remove(K k) {
     int index = getElementIndex(k);
@@ -185,6 +194,8 @@ public interface Visitor<K, V> {
 
   /**
    * Visit all key, value pairs in the IdentityHashStore.
+   *
+   * @param visitor visitor.
    */
   public void visitAll(Visitor<K, V> visitor) {
     int length = buffer == null ? 0 : buffer.length;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSortable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSortable.java
index 1aa036e95b735..99472e18f1daf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSortable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSortable.java
@@ -31,11 +31,18 @@ public interface IndexedSortable {
   /**
    * Compare items at the given addresses consistent with the semantics of
    * {@link java.util.Comparator#compare(Object, Object)}.
+   *
+   * @param i(int).
+   * @param j(int).
+   * @return compare result.
    */
   int compare(int i, int j);
 
   /**
    * Swap items at the given addresses.
+   *
+   * @param i i(int).
+   * @param j j(int).
    */
   void swap(int i, int j);
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSorter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSorter.java
index bdd024302c99f..252efe3e4de44 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSorter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IndexedSorter.java
@@ -38,6 +38,10 @@ public interface IndexedSorter {
    * entry.
    * @see IndexedSortable#compare
    * @see IndexedSortable#swap
+   *
+   * @param r r.
+   * @param l l.
+   * @param s s.
    */
   void sort(IndexedSortable s, int l, int r);
 
@@ -45,6 +49,10 @@ public interface IndexedSorter {
    * Same as {@link #sort(IndexedSortable,int,int)}, but indicate progress
    * periodically.
    * @see #sort(IndexedSortable,int,int)
+   * @param s s.
+   * @param l l.
+   * @param r r.
+   * @param rep rep.
    */
   void sort(IndexedSortable s, int l, int r, Progressable rep);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
index e83736cd3e35a..35d127af425c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
@@ -185,6 +185,7 @@ protected void startLockTiming() {
    *
    * @param acquireTime  - timestamp just after acquiring the lock.
    * @param releaseTime - timestamp just before releasing the lock.
+   * @param checkLockHeld checkLockHeld.
    */
   protected void check(long acquireTime, long releaseTime,
        boolean checkLockHeld) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
index 18f6ccfdb176b..c99290bc3d3ac 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
@@ -44,12 +44,7 @@ public class InstrumentedReadLock extends InstrumentedLock {
    * there can be multiple threads that hold the read lock concurrently.
    */
   private final ThreadLocal<Long> readLockHeldTimeStamp =
-      new ThreadLocal<Long>() {
-    @Override
-    protected Long initialValue() {
-      return Long.MAX_VALUE;
-    };
-  };
+      ThreadLocal.withInitial(() -> Long.MAX_VALUE);
 
   public InstrumentedReadLock(String name, Logger logger,
       ReentrantReadWriteLock readWriteLock,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
index 667b1ca6a4b60..4637b5efe535b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
@@ -37,6 +37,9 @@
 @InterfaceStability.Unstable
 public class InstrumentedWriteLock extends InstrumentedLock {
 
+  private final ReentrantReadWriteLock readWriteLock;
+  private volatile long writeLockHeldTimeStamp = 0;
+
   public InstrumentedWriteLock(String name, Logger logger,
       ReentrantReadWriteLock readWriteLock,
       long minLoggingGapMs, long lockWarningThresholdMs) {
@@ -50,5 +53,28 @@ public InstrumentedWriteLock(String name, Logger logger,
       long minLoggingGapMs, long lockWarningThresholdMs, Timer clock) {
     super(name, logger, readWriteLock.writeLock(), minLoggingGapMs,
         lockWarningThresholdMs, clock);
+    this.readWriteLock = readWriteLock;
+  }
+
+  @Override
+  public void unlock() {
+    boolean needReport = readWriteLock.getWriteHoldCount() == 1;
+    long localWriteReleaseTime = getTimer().monotonicNow();
+    long localWriteAcquireTime = writeLockHeldTimeStamp;
+    getLock().unlock();
+    if (needReport) {
+      writeLockHeldTimeStamp = 0;
+      check(localWriteAcquireTime, localWriteReleaseTime, true);
+    }
+  }
+
+  /**
+   * Starts timing for the instrumented write lock.
+   */
+  @Override
+  protected void startLockTiming() {
+    if (readWriteLock.getWriteHoldCount() == 1) {
+      writeLockHeldTimeStamp = getTimer().monotonicNow();
+    }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java
index 21d8ad34a8784..c27a42d763ed1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/IntrusiveCollection.java
@@ -49,38 +49,59 @@ public interface Element {
     /**
      * Insert this element into the list.  This is the first thing that will
      * be called on the element.
+     *
+     * @param list list.
+     * @param prev prev.
+     * @param next next.
      */
     void insertInternal(IntrusiveCollection<? extends Element> list,
         Element prev, Element next);
 
     /**
      * Set the prev pointer of an element already in the list.
+     *
+     * @param list list.
+     * @param prev prev.
      */
     void setPrev(IntrusiveCollection<? extends Element> list, Element prev);
 
     /**
      * Set the next pointer of an element already in the list.
+     *
+     * @param list list.
+     * @param next next.
      */
     void setNext(IntrusiveCollection<? extends Element> list, Element next);
 
     /**
      * Remove an element from the list.  This is the last thing that will be
      * called on an element.
+     *
+     * @param list list.
      */
     void removeInternal(IntrusiveCollection<? extends Element> list);
 
     /**
      * Get the prev pointer of an element.
+     *
+     * @param list list.
+     * @return Element.
      */
     Element getPrev(IntrusiveCollection<? extends Element> list);
 
     /**
      * Get the next pointer of an element.
+     *
+     * @param list list.
+     * @return Element.
      */
     Element getNext(IntrusiveCollection<? extends Element> list);
 
     /**
      * Returns true if this element is in the provided list.
+     *
+     * @param list list.
+     * @return if this element is in the provided list true, not false.
      */
     boolean isInList(IntrusiveCollection<? extends Element> list);
   }
@@ -260,6 +281,7 @@ public <T> T[] toArray(T[] array) {
    * Add an element to the end of the list.
    * 
    * @param elem     The new element to add.
+   * @return add result.
    */
   @Override
   public boolean add(E elem) {
@@ -281,6 +303,7 @@ public boolean add(E elem) {
    * Add an element to the front of the list.
    *
    * @param elem     The new element to add.
+   * @return if addFirst success true, not false.
    */
   public boolean addFirst(Element elem) {
     if (elem == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index 81a356c7a835a..52c6c4505226a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.util;
 
+import javax.annotation.Nullable;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -42,8 +43,15 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * Support for marshalling objects to and from JSON.
@@ -166,6 +174,9 @@ public synchronized T fromJsonStream(InputStream stream) throws IOException {
   @SuppressWarnings("unchecked")
   public synchronized T load(File jsonFile)
       throws IOException, JsonParseException, JsonMappingException {
+    if (!jsonFile.exists()) {
+      throw new FileNotFoundException("No such file: " + jsonFile);
+    }
     if (!jsonFile.isFile()) {
       throw new FileNotFoundException("Not a file: " + jsonFile);
     }
@@ -175,7 +186,7 @@ public synchronized T load(File jsonFile)
     try {
       return mapper.readValue(jsonFile, classType);
     } catch (IOException e) {
-      LOG.error("Exception while parsing json file {}", jsonFile, e);
+      LOG.warn("Exception while parsing json file {}", jsonFile, e);
       throw e;
     }
   }
@@ -228,30 +239,46 @@ public T fromInstance(T instance) throws IOException {
 
   /**
    * Load from a Hadoop filesystem.
-   * There's a check for data availability after the file is open, by
-   * raising an EOFException if stream.available == 0.
-   * This allows for a meaningful exception without the round trip overhead
-   * of a getFileStatus call before opening the file. It may be brittle
-   * against an FS stream which doesn't return a value here, but the
-   * standard filesystems all do.
-   * JSON parsing and mapping problems
-   * are converted to IOEs.
    * @param fs filesystem
    * @param path path
    * @return a loaded object
-   * @throws IOException IO or JSON parse problems
+   * @throws PathIOException JSON parse problem
+   * @throws IOException IO problems
    */
   public T load(FileSystem fs, Path path) throws IOException {
-    try (FSDataInputStream dataInputStream = fs.open(path)) {
-      // throw an EOF exception if there is no data available.
-      if (dataInputStream.available() == 0) {
-        throw new EOFException("No data in " + path);
-      }
+    return load(fs, path, null);
+  }
+
+  /**
+   * Load from a Hadoop filesystem.
+   * If a file status is supplied, it's passed in to the openFile()
+   * call so that FS implementations can optimize their opening.
+   * @param fs filesystem
+   * @param path path
+   * @param status status of the file to open.
+   * @return a loaded object
+   * @throws PathIOException JSON parse problem
+   * @throws EOFException file status references an empty file
+   * @throws IOException IO problems
+   */
+  public T load(FileSystem fs, Path path, @Nullable FileStatus status)
+      throws IOException {
+
+    if (status != null && status.getLen() == 0) {
+      throw new EOFException("No data in " + path);
+    }
+    FutureDataInputStreamBuilder builder = fs.openFile(path)
+        .opt(FS_OPTION_OPENFILE_READ_POLICY,
+            FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE);
+    if (status != null) {
+      builder.withFileStatus(status);
+    }
+    try (FSDataInputStream dataInputStream =
+             awaitFuture(builder.build())) {
       return fromJsonStream(dataInputStream);
     } catch (JsonProcessingException e) {
-      throw new IOException(
-          String.format("Failed to read JSON file \"%s\": %s", path, e),
-          e);
+      throw new PathIOException(path.toString(),
+          "Failed to read JSON file " + e, e);
     }
   }
 
@@ -260,7 +287,8 @@ public T load(FileSystem fs, Path path) throws IOException {
    * @param fs filesystem
    * @param path path
    * @param overwrite should any existing file be overwritten
-   * @throws IOException IO exception
+   * @param instance instance
+   * @throws IOException IO exception.
    */
   public void save(FileSystem fs, Path path, T instance,
       boolean overwrite) throws
@@ -269,11 +297,12 @@ public void save(FileSystem fs, Path path, T instance,
   }
 
   /**
-   * Write the JSON as bytes, then close the file.
+   * Write the JSON as bytes, then close the stream.
+   * @param instance instance to write
    * @param dataOutputStream an output stream that will always be closed
    * @throws IOException on any failure
    */
-  private void writeJsonAsBytes(T instance,
+  public void writeJsonAsBytes(T instance,
       OutputStream dataOutputStream) throws IOException {
     try {
       dataOutputStream.write(toBytes(instance));
@@ -297,6 +326,7 @@ public byte[] toBytes(T instance) throws IOException {
    * @param bytes byte array
    * @throws IOException IO problems
    * @throws EOFException not enough data
+   * @return byte array.
    */
   public T fromBytes(byte[] bytes) throws IOException {
     return fromJson(new String(bytes, 0, bytes.length, UTF_8));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index feb4f9b9d3f01..382266a99401f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -214,6 +214,9 @@ public void run() {
    * This main function just leaks memory into a list. Running this class
    * with a 1GB heap will very quickly go into "GC hell" and result in
    * log messages about the GC pauses.
+   *
+   * @param args args.
+   * @throws Exception Exception.
    */
   @SuppressWarnings("resource")
   public static void main(String []args) throws Exception {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
index f1897ea2368b2..0ce612ccca3e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightCache.java
@@ -58,10 +58,13 @@ public class LightWeightCache<K, E extends K> extends LightWeightGSet<K, E> {
    * Entries of {@link LightWeightCache}.
    */
   public static interface Entry extends LinkedElement {
-    /** Set the expiration time. */
+    /**
+     * Set the expiration time.
+     * @param timeNano input timeNano.
+     */
     public void setExpirationTime(long timeNano);
 
-    /** Get the expiration time. */
+    /** @return Get the expiration time. */
     public long getExpirationTime();
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java
index d32d1f37b42fa..5151cdec5cd76 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightGSet.java
@@ -52,10 +52,13 @@ public class LightWeightGSet<K, E extends K> implements GSet<K, E> {
    * Elements of {@link LightWeightGSet}.
    */
   public interface LinkedElement {
-    /** Set the next element. */
+    /**
+     * Set the next element.
+     * @param next inputNext.
+     */
     void setNext(LinkedElement next);
 
-    /** Get the next element. */
+    /** @return Get the next element. */
     LinkedElement getNext();
   }
 
@@ -177,6 +180,8 @@ public E put(final E element) {
    * Remove the element corresponding to the key,
    * given key.hashCode() == index.
    *
+   * @param key key.
+   * @param index index.
    * @return If such element exists, return it.
    *         Otherwise, return null.
    */
@@ -270,7 +275,11 @@ public String toString() {
     return b.toString();
   }
 
-  /** Print detailed information of this object. */
+  /**
+   * Print detailed information of this object.
+   *
+   * @param out out.
+   */
   public void printDetails(final PrintStream out) {
     out.print(this + ", entries = [");
     for(int i = 0; i < entries.length; i++) {
@@ -357,6 +366,10 @@ public void setTrackModification(boolean trackModification) {
    * Let e = round(log_2 t).
    * Then, we choose capacity = 2^e/(size of reference),
    * unless it is outside the close interval [1, 2^30].
+   *
+   * @param mapName mapName.
+   * @param percentage percentage.
+   * @return compute capacity.
    */
   public static int computeCapacity(double percentage, String mapName) {
     return computeCapacity(Runtime.getRuntime().maxMemory(), percentage,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
index 0abcf989d1500..051e2680bc306 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
@@ -20,6 +20,9 @@
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 
+import java.util.Iterator;
+import java.util.function.Consumer;
+
 /**
  * A low memory footprint {@link GSet} implementation,
  * which uses an array for storing the elements
@@ -86,17 +89,38 @@ public LightWeightResizableGSet(int initCapacity) {
   }
 
   @Override
-  public E put(final E element) {
+  public synchronized E put(final E element) {
     E existing = super.put(element);
     expandIfNecessary();
     return existing;
   }
 
+  @Override
+  public synchronized E get(K key) {
+    return super.get(key);
+  }
+
+  @Override
+  public synchronized E remove(K key) {
+    return super.remove(key);
+  }
+
+  @Override
+  public synchronized int size() {
+    return super.size();
+  }
+
+  public synchronized void getIterator(Consumer<Iterator<E>> consumer) {
+    consumer.accept(super.values().iterator());
+  }
+
   /**
    * Resize the internal table to given capacity.
+   *
+   * @param cap capacity.
    */
   @SuppressWarnings("unchecked")
-  protected void resize(int cap) {
+  protected synchronized void resize(int cap) {
     int newCapacity = actualArrayLength(cap);
     if (newCapacity == this.capacity) {
       return;
@@ -121,7 +145,7 @@ protected void resize(int cap) {
   /**
    * Checks if we need to expand, and expands if necessary.
    */
-  protected void expandIfNecessary() {
+  protected synchronized void expandIfNecessary() {
     if (size > this.threshold && capacity < MAX_ARRAY_LENGTH) {
       resize(capacity * 2);
     }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
index 520ddf6bdf401..08bd8102b1388 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
@@ -89,7 +89,7 @@ public LineReader(InputStream in, int bufferSize) {
    * <code>Configuration</code>.
    * @param in input stream
    * @param conf configuration
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public LineReader(InputStream in, Configuration conf) throws IOException {
     this(in, conf.getInt(IO_FILE_BUFFER_SIZE_KEY, DEFAULT_BUFFER_SIZE));
@@ -133,7 +133,7 @@ public LineReader(InputStream in, int bufferSize,
    * @param in input stream
    * @param conf configuration
    * @param recordDelimiterBytes The delimiter
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public LineReader(InputStream in, Configuration conf,
       byte[] recordDelimiterBytes) throws IOException {
@@ -146,7 +146,7 @@ public LineReader(InputStream in, Configuration conf,
 
   /**
    * Close the underlying stream.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public void close() throws IOException {
     in.close();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java
index b6d74ee679281..a9d0756e8e381 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java
@@ -44,6 +44,9 @@ private Lists() {
 
   /**
    * Creates a <i>mutable</i>, empty {@code ArrayList} instance.
+   *
+   * @param <E> Generics Type E.
+   * @return ArrayList Generics Type E.
    */
   public static <E> ArrayList<E> newArrayList() {
     return new ArrayList<>();
@@ -59,6 +62,10 @@ public static <E> ArrayList<E> newArrayList() {
    * {@link Arrays#asList asList}
    * {@code (...))}, or for creating an empty list then calling
    * {@link Collections#addAll}.
+   *
+   * @param <E> Generics Type E.
+   * @param elements elements.
+   * @return ArrayList Generics Type E.
    */
   @SafeVarargs
   public static <E> ArrayList<E> newArrayList(E... elements) {
@@ -76,6 +83,10 @@ public static <E> ArrayList<E> newArrayList(E... elements) {
    * Creates a <i>mutable</i> {@code ArrayList} instance containing the
    * given elements; a very thin shortcut for creating an empty list then
    * calling Iterables#addAll.
+   *
+   * @param <E> Generics Type E.
+   * @param elements elements.
+   * @return ArrayList Generics Type E.
    */
   public static <E> ArrayList<E> newArrayList(Iterable<? extends E> elements) {
     if (elements == null) {
@@ -90,6 +101,10 @@ public static <E> ArrayList<E> newArrayList(Iterable<? extends E> elements) {
    * Creates a <i>mutable</i> {@code ArrayList} instance containing the
    * given elements; a very thin shortcut for creating an empty list
    * and then calling Iterators#addAll.
+   *
+   * @param <E> Generics Type E.
+   * @param elements elements.
+   * @return ArrayList Generics Type E.
    */
   public static <E> ArrayList<E> newArrayList(Iterator<? extends E> elements) {
     ArrayList<E> list = newArrayList();
@@ -102,6 +117,7 @@ public static <E> ArrayList<E> newArrayList(Iterator<? extends E> elements) {
    * specified initial size;
    * simply delegates to {@link ArrayList#ArrayList(int)}.
    *
+   * @param <E> Generics Type E.
    * @param initialArraySize the exact size of the initial backing array for
    *     the returned array list
    *     ({@code ArrayList} documentation calls this value the "capacity").
@@ -126,6 +142,8 @@ public static <E> ArrayList<E> newArrayListWithCapacity(
    * @return a new, empty {@code ArrayList}, sized appropriately to hold the
    *     estimated number of elements.
    * @throws IllegalArgumentException if {@code estimatedSize} is negative.
+   *
+   * @param <E> Generics Type E.
    */
   public static <E> ArrayList<E> newArrayListWithExpectedSize(
       int estimatedSize) {
@@ -140,7 +158,10 @@ public static <E> ArrayList<E> newArrayListWithExpectedSize(
    * outperform {@code LinkedList} except in certain rare and specific
    * situations. Unless you have
    * spent a lot of time benchmarking your specific needs, use one of those
-   * instead.
+   * instead.</p>
+   *
+   * @param <E> Generics Type E.
+   * @return Generics Type E List.
    */
   public static <E> LinkedList<E> newLinkedList() {
     return new LinkedList<>();
@@ -155,7 +176,11 @@ public static <E> LinkedList<E> newLinkedList() {
    * {@link java.util.ArrayDeque} consistently
    * outperform {@code LinkedList} except in certain rare and specific
    * situations. Unless you have spent a lot of time benchmarking your
-   * specific needs, use one of those instead.
+   * specific needs, use one of those instead.</p>
+   *
+   * @param elements elements.
+   * @param <E> Generics Type E.
+   * @return Generics Type E List.
    */
   public static <E> LinkedList<E> newLinkedList(
       Iterable<? extends E> elements) {
@@ -232,4 +257,29 @@ private static <T> boolean addAll(Collection<T> addTo,
     return addAll(addTo, elementsToAdd.iterator());
   }
 
+  /**
+   * Returns consecutive sub-lists of a list, each of the same size
+   * (the final list may be smaller).
+   * @param originalList original big list.
+   * @param pageSize desired size of each sublist ( last one
+   *                 may be smaller)
+   * @param <T> Generics Type.
+   * @return a list of sub lists.
+   */
+  public static <T> List<List<T>> partition(List<T> originalList, int pageSize) {
+
+    Preconditions.checkArgument(originalList != null && originalList.size() > 0,
+            "Invalid original list");
+    Preconditions.checkArgument(pageSize > 0, "Page size should " +
+            "be greater than 0 for performing partition");
+
+    List<List<T>> result = new ArrayList<>();
+    int i=0;
+    while (i < originalList.size()) {
+      result.add(originalList.subList(i,
+              Math.min(i + pageSize, originalList.size())));
+      i = i + pageSize;
+    }
+    return result;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
index d07264c88ddd8..68c5c3ca47ff9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java
@@ -89,7 +89,7 @@ public MachineList(Collection<String> hostEntries) {
   /**
    * Accepts a collection of ip/cidr/host addresses
    * 
-   * @param hostEntries
+   * @param hostEntries hostEntries.
    * @param addressFactory addressFactory to convert host to InetAddress
    */
   public MachineList(Collection<String> hostEntries,
@@ -139,7 +139,7 @@ public MachineList(Collection<String> hostEntries,
    * {@link #includes(InetAddress)} should be preferred
    * to avoid possibly re-resolving the ip address.
    *
-   * @param ipAddress
+   * @param ipAddress ipAddress.
    * @return true if ipAddress is part of the list
    */
   public boolean includes(String ipAddress) {
@@ -161,7 +161,7 @@ public boolean includes(String ipAddress) {
 
   /**
    * Accepts an inet address and return true if address is in the list.
-   * @param address
+   * @param address address.
    * @return true if address is part of the list
    */
   public boolean includes(InetAddress address) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
index 11d1176f92a59..b5550f58ae218 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
@@ -76,16 +76,22 @@ public static boolean isNativeCodeLoaded() {
 
   /**
    * Returns true only if this build was compiled with support for ISA-L.
+   *
+   * @return if this build was compiled with support for ISA-L true, not false.
    */
   public static native boolean buildSupportsIsal();
 
   /**
-  * Returns true only if this build was compiled with support for ZStandard.
+   * Returns true only if this build was compiled with support for ZStandard.
+   *
+   * @return if this build was compiled with support for ZStandard true,not false.
    */
   public static native boolean buildSupportsZstd();
 
   /**
    * Returns true only if this build was compiled with support for openssl.
+   *
+   * @return if this build was compiled with support for openssl true,not false.
    */
   public static native boolean buildSupportsOpenssl();
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index 3847902e79743..9843a9d4057dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -39,7 +39,8 @@ public class NativeLibraryChecker {
       LoggerFactory.getLogger(NativeLibraryChecker.class);
 
   /**
-   * A tool to test native library availability, 
+   * A tool to test native library availability.
+   * @param args args.
    */
   public static void main(String[] args) {
     String usage = "NativeLibraryChecker [-a|-h]\n"
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/OperationDuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/OperationDuration.java
index fdd25286a2300..1fb920e99f08e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/OperationDuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/OperationDuration.java
@@ -95,9 +95,11 @@ public String toString() {
 
   /**
    * Get the duration in milliseconds.
-   * <p></p>
+   *
+   * <p>
    * This will be 0 until a call
    * to {@link #finished()} has been made.
+   * </p>
    * @return the currently recorded duration.
    */
   public long value() {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Options.java
index 23169e3af3533..ccd494e5e40d9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Options.java
@@ -126,7 +126,7 @@ public Progressable getValue() {
    * @param cls the dynamic class to find
    * @param opts the list of options to look through
    * @return the first option that matches
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @SuppressWarnings("unchecked")
   public static <base, T extends base> T getOption(Class<T> cls, base [] opts
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
index df571f35e2fcd..f7822e3f788d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PrintJarMainClass.java
@@ -31,7 +31,7 @@
 public class PrintJarMainClass {
   
   /**
-   * @param args
+   * @param args args.
    */
   public static void main(String[] args) {
     try (JarFile jar_file = new JarFile(args[0])) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PriorityQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PriorityQueue.java
index ebb943bcb6285..d149d5d811914 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PriorityQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PriorityQueue.java
@@ -31,11 +31,19 @@ public abstract class PriorityQueue<T> {
   private int size;
   private int maxSize;
 
-  /** Determines the ordering of objects in this priority queue.  Subclasses
-      must define this one method. */
+  /**
+   * Determines the ordering of objects in this priority queue.  Subclasses
+      must define this one method.
+   * @param a object a.
+   * @param b object b.
+   * @return if a less than b true, not false
+   */
   protected abstract boolean lessThan(Object a, Object b);
 
-  /** Subclass constructors must call this. */
+  /**
+   * Subclass constructors must call this.
+   * @param maxSize max size.
+   */
   @SuppressWarnings("unchecked")
   protected final void initialize(int maxSize) {
     size = 0;
@@ -48,6 +56,7 @@ protected final void initialize(int maxSize) {
    * Adds an Object to a PriorityQueue in log(size) time.
    * If one tries to add more objects than maxSize from initialize
    * a RuntimeException (ArrayIndexOutOfBound) is thrown.
+   * @param element element.
    */
   public final void put(T element) {
     size++;
@@ -58,7 +67,7 @@ public final void put(T element) {
   /**
    * Adds element to the PriorityQueue in log(size) time if either
    * the PriorityQueue is not full, or not lessThan(element, top()).
-   * @param element
+   * @param element element.
    * @return true if element is added, false otherwise.
    */
   public boolean insert(T element){
@@ -75,7 +84,11 @@ else if (size > 0 && !lessThan(element, top())){
       return false;
   }
 
-  /** Returns the least element of the PriorityQueue in constant time. */
+  /**
+   * Returns the least element of the PriorityQueue in constant time.
+   *
+   * @return T Generics Type T.
+   */
   public final T top() {
     if (size > 0)
       return heap[1];
@@ -83,8 +96,11 @@ public final T top() {
       return null;
   }
 
-  /** Removes and returns the least element of the PriorityQueue in log(size)
-      time. */
+  /**
+   * Removes and returns the least element of the PriorityQueue in log(size)
+      time.
+   * @return T Generics Type T.
+   */
   public final T pop() {
     if (size > 0) {
       T result = heap[1];			  // save first value
@@ -109,7 +125,11 @@ public final void adjustTop() {
   }
 
 
-  /** Returns the number of elements currently stored in the PriorityQueue. */
+  /**
+   * Returns the number of elements currently stored in the PriorityQueue.
+   *
+   * @return size.
+   */
   public final int size() {
     return size;
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java
new file mode 100644
index 0000000000000..cf653b9c912c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProcessUtils.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Process related utilities.
+ */
+@InterfaceAudience.Private
+public final class ProcessUtils {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ProcessUtils.class);
+
+  private ProcessUtils() {
+    // no-op
+  }
+
+  public static Integer getPid() {
+    // JVM_PID can be exported in service start script
+    String pidStr = System.getenv("JVM_PID");
+
+    // In case if it is not set correctly, fallback to mxbean which is implementation specific.
+    if (pidStr == null || pidStr.trim().isEmpty()) {
+      String name = ManagementFactory.getRuntimeMXBean().getName();
+      if (name != null) {
+        int idx = name.indexOf("@");
+        if (idx != -1) {
+          pidStr = name.substring(0, name.indexOf("@"));
+        }
+      }
+    }
+    try {
+      if (pidStr != null) {
+        return Integer.valueOf(pidStr);
+      }
+    } catch (NumberFormatException ignored) {
+      // ignore
+    }
+    return null;
+  }
+
+  public static Process runCmdAsync(List<String> cmd) {
+    try {
+      LOG.info("Running command async: {}", cmd);
+      return new ProcessBuilder(cmd).inheritIO().start();
+    } catch (IOException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
index 347e5087eaa37..e49cf57705344 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProgramDriver.java
@@ -91,12 +91,12 @@ private static void printUsage(Map<String, ProgramDescription> programs) {
   }
     
   /**
-   * This is the method that adds the classed to the repository
+   * This is the method that adds the classed to the repository.
    * @param name The name of the string you want the class instance to be called with
    * @param mainClass The class that you want to add to the repository
    * @param description The description of the class
-   * @throws NoSuchMethodException 
-   * @throws SecurityException 
+   * @throws NoSuchMethodException when a particular method cannot be found.
+   * @throws SecurityException security manager to indicate a security violation.
    */
   public void addClass(String name, Class<?> mainClass, String description)
       throws Throwable {
@@ -111,10 +111,10 @@ public void addClass(String name, Class<?> mainClass, String description)
    * of the command line arguments.
    * @param args The argument from the user. args[0] is the command to run.
    * @return -1 on error, 0 on success
-   * @throws NoSuchMethodException 
-   * @throws SecurityException 
-   * @throws IllegalAccessException 
-   * @throws IllegalArgumentException 
+   * @throws NoSuchMethodException  when a particular method cannot be found.
+   * @throws SecurityException security manager to indicate a security violation.
+   * @throws IllegalAccessException for backward compatibility.
+   * @throws IllegalArgumentException if the arg is invalid.
    * @throws Throwable Anything thrown by the example program's main
    */
   public int run(String[] args)
@@ -146,7 +146,11 @@ public int run(String[] args)
   }
 
   /**
-   * API compatible with Hadoop 1.x
+   * API compatible with Hadoop 1.x.
+   *
+   * @param argv argv.
+   * @throws Throwable Anything thrown
+   *                   by the example program's main
    */
   public void driver(String[] argv) throws Throwable {
     if (run(argv) == -1) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
index bd1c0f4a62a78..a839c04e99e63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
@@ -53,14 +53,21 @@ public class Progress {
   /** Creates a new root node. */
   public Progress() {}
 
-  /** Adds a named node to the tree. */
+  /**
+   * Adds a named node to the tree.
+   * @param status status.
+   * @return Progress.
+   */
   public Progress addPhase(String status) {
     Progress phase = addPhase();
     phase.setStatus(status);
     return phase;
   }
 
-  /** Adds a node to the tree. Gives equal weightage to all phases */
+  /**
+   * Adds a node to the tree. Gives equal weightage to all phases.
+   * @return Progress.
+   */
   public synchronized Progress addPhase() {
     Progress phase = addNewPhase();
     // set equal weightage for all phases
@@ -77,7 +84,13 @@ private synchronized Progress addNewPhase() {
     return phase;
   }
 
-  /** Adds a named node with a specified progress weightage to the tree. */
+  /**
+   * Adds a named node with a specified progress weightage to the tree.
+   *
+   * @param status status.
+   * @param weightage weightage.
+   * @return Progress.
+   */
   public Progress addPhase(String status, float weightage) {
     Progress phase = addPhase(weightage);
     phase.setStatus(status);
@@ -85,7 +98,12 @@ public Progress addPhase(String status, float weightage) {
     return phase;
   }
 
-  /** Adds a node with a specified progress weightage to the tree. */
+  /**
+   * Adds a node with a specified progress weightage to the tree.
+   *
+   * @param weightage weightage.
+   * @return Progress.
+   */
   public synchronized Progress addPhase(float weightage) {
     Progress phase = new Progress();
     progressWeightagesForPhases.add(weightage);
@@ -104,7 +122,11 @@ public synchronized Progress addPhase(float weightage) {
     return phase;
   }
 
-  /** Adds n nodes to the tree. Gives equal weightage to all phases */
+  /**
+   * Adds n nodes to the tree. Gives equal weightage to all phases.
+   *
+   * @param n n.
+   */
   public synchronized void addPhases(int n) {
     for (int i = 0; i < n; i++) {
       addNewPhase();
@@ -136,7 +158,10 @@ public synchronized void startNextPhase() {
     currentPhase++;
   }
 
-  /** Returns the current sub-node executing. */
+  /**
+   * Returns the current sub-node executing.
+   * @return Progress.
+   */
   public synchronized Progress phase() {
     return phases.get(currentPhase);
   }
@@ -158,7 +183,10 @@ public void complete() {
     }
   }
 
-  /** Called during execution on a leaf node to set its progress. */
+  /**
+   * Called during execution on a leaf node to set its progress.
+   * @param progress progress.
+   */
   public synchronized void set(float progress) {
     if (Float.isNaN(progress)) {
       progress = 0;
@@ -188,7 +216,10 @@ else if (progress == Float.POSITIVE_INFINITY) {
     this.progress = progress;
   }
 
-  /** Returns the overall progress of the root. */
+  /**
+   * Returns the overall progress of the root.
+   * @return progress.
+   */
   // this method probably does not need to be synchronized as getInternal() is
   // synchronized and the node's parent never changes. Still, it doesn't hurt. 
   public synchronized float get() {
@@ -202,6 +233,8 @@ public synchronized float get() {
   /**
    * Returns progress in this node. get() would give overall progress of the
    * root node(not just given current node).
+   *
+   * @return progress.
    */
   public synchronized float getProgress() {
     return getInternal();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
index 9807adc50d6d1..883c19c5e7750 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
@@ -83,6 +83,10 @@ public static int readRawVarint32(DataInput in) throws IOException {
    * as the old connection context as was done for writable where
    * the effective and real users are set based on the auth method.
    *
+   * @param protocol protocol.
+   * @param ugi ugi.
+   * @param authMethod authMethod.
+   * @return IpcConnectionContextProto.
    */
   public static IpcConnectionContextProto makeIpcConnectionContext(
       final String protocol,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/QuickSort.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/QuickSort.java
index 73d8d90d42507..f3f8b839a0500 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/QuickSort.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/QuickSort.java
@@ -40,6 +40,9 @@ private static void fix(IndexedSortable s, int p, int r) {
   /**
    * Deepest recursion before giving up and doing a heapsort.
    * Returns 2 * ceil(log(n)).
+   *
+   * @param x x.
+   * @return MaxDepth.
    */
   protected static int getMaxDepth(int x) {
     if (x <= 0)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java
new file mode 100644
index 0000000000000..ae119c0e630f4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Minimal subset of google rate limiter class.
+ * Can be used to throttle use of object stores where excess load
+ * will trigger cluster-wide throttling, backoff etc. and so collapse
+ * performance.
+ * The time waited is returned as a Duration type.
+ * The google rate limiter implements this by allowing a caller to ask for
+ * more capacity than is available. This will be granted
+ * but the subsequent request will be blocked if the bucket of
+ * capacity hasn't let refilled to the point where there is
+ * capacity again.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface RateLimiting {
+
+  /**
+   * Acquire rate limiter capacity.
+   * If there is not enough space, the permits will be acquired,
+   * but the subsequent call will block until the capacity has been
+   * refilled.
+   * @param requestedCapacity capacity to acquire.
+   * @return time spent waiting for output.
+   */
+  Duration acquire(int requestedCapacity);
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java
new file mode 100644
index 0000000000000..621415456e125
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter;
+
+/**
+ * Factory for Rate Limiting.
+ * This should be only place in the code where the guava RateLimiter is imported.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class RateLimitingFactory {
+
+  private static final RateLimiting UNLIMITED = new NoRateLimiting();
+
+  /**
+   * No waiting took place.
+   */
+  private static final Duration INSTANTLY = Duration.ofMillis(0);
+
+  private RateLimitingFactory() {
+  }
+
+  /**
+   * No Rate Limiting.
+   */
+  private static class NoRateLimiting implements RateLimiting {
+
+
+    @Override
+    public Duration acquire(int requestedCapacity) {
+      return INSTANTLY;
+    }
+  }
+
+  /**
+   * Rate limiting restricted to that of a google rate limiter.
+   */
+  private static final class RestrictedRateLimiting implements RateLimiting {
+    private final RateLimiter limiter;
+
+    /**
+     * Constructor.
+     * @param capacityPerSecond capacity in permits/second.
+     */
+    private RestrictedRateLimiting(int capacityPerSecond) {
+      this.limiter = RateLimiter.create(capacityPerSecond);
+    }
+
+    @Override
+    public Duration acquire(int requestedCapacity) {
+      final double delayMillis = limiter.acquire(requestedCapacity);
+      return delayMillis == 0
+          ? INSTANTLY
+          : Duration.ofMillis((long) (delayMillis * 1000));
+    }
+
+  }
+
+  /**
+   * Get the unlimited rate.
+   * @return a rate limiter which always has capacity.
+   */
+  public static RateLimiting unlimitedRate() {
+    return UNLIMITED;
+  }
+
+  /**
+   * Create an instance.
+   * If the rate is 0; return the unlimited rate.
+   * @param capacity capacity in permits/second.
+   * @return limiter restricted to the given capacity.
+   */
+  public static RateLimiting create(int capacity) {
+
+    return capacity == 0
+        ? unlimitedRate()
+        : new RestrictedRateLimiting(capacity);
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
index 47e44c9e09f8d..155c4f9c5f498 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
@@ -21,7 +21,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
-import java.io.PrintWriter;
 import java.io.UnsupportedEncodingException;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
@@ -120,6 +119,7 @@ private static void setJobConf(Object theObject, Configuration conf) {
    * 
    * @param theClass class of which an object is created
    * @param conf Configuration
+   * @param <T> Generics Type T.
    * @return a new object
    */
   @SuppressWarnings("unchecked")
@@ -133,6 +133,7 @@ public static <T> T newInstance(Class<T> theClass, Configuration conf) {
    * @param conf Configuration
    * @param argTypes the types of the arguments
    * @param values the values of the arguments
+   * @param <T> Generics Type.
    * @return a new object
    */
   @SuppressWarnings("unchecked")
@@ -284,6 +285,7 @@ public static void logThreadInfo(Logger log,
    * Return the correctly-typed {@link Class} of the given object.
    *  
    * @param o object whose correctly-typed <code>Class</code> is to be obtained
+   * @param <T> Generics Type T.
    * @return the correctly typed <code>Class</code> of the given object.
    */
   @SuppressWarnings("unchecked")
@@ -332,11 +334,13 @@ private static SerializationFactory getFactory(Configuration conf) {
   }
   
   /**
-   * Make a copy of the writable object using serialization to a buffer
+   * Make a copy of the writable object using serialization to a buffer.
    * @param src the object to copy from
    * @param dst the object to copy into, which is destroyed
+   * @param <T> Generics Type.
+   * @param conf configuration.
    * @return dst param (the copy)
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   @SuppressWarnings("unchecked")
   public static <T> T copy(Configuration conf, 
@@ -368,6 +372,9 @@ public static void cloneWritableInto(Writable dst,
   /**
    * Gets all the declared fields of a class including fields declared in
    * superclasses.
+   *
+   * @param clazz clazz
+   * @return field List
    */
   public static List<Field> getDeclaredFieldsIncludingInherited(Class<?> clazz) {
     List<Field> fields = new ArrayList<Field>();
@@ -390,6 +397,9 @@ public int compare(Field a, Field b) {
   /**
    * Gets all the declared methods of a class including methods declared in
    * superclasses.
+   *
+   * @param clazz clazz.
+   * @return Method List.
    */
   public static List<Method> getDeclaredMethodsIncludingInherited(Class<?> clazz) {
     List<Method> methods = new ArrayList<Method>();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 50126002b7be7..c28e69f54611e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -156,6 +156,7 @@ public static void unJar(InputStream inputStream, File toDir,
    * @param inputStream the jar stream to unpack
    * @param toDir the destination directory into which to unpack the jar
    * @param unpackRegex the pattern to match jar entries against
+   * @param name name.
    *
    * @throws IOException if an I/O error has occurred or toDir
    * cannot be created and does not already exist
@@ -231,7 +232,11 @@ private static void ensureDirectory(File dir) throws IOException {
   }
 
   /** Run a Hadoop job jar.  If the main class is not in the jar's manifest,
-   * then it must be provided on the command line. */
+   * then it must be provided on the command line.
+   *
+   * @param args args.
+   * @throws Throwable error.
+   */
   public static void main(String[] args) throws Throwable {
     new RunJar().run(args);
   }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
index 685e92d628136..c3c04493d8be5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
@@ -30,7 +30,10 @@
 public abstract class SequentialNumber implements IdGenerator {
   private final AtomicLong currentValue;
 
-  /** Create a new instance with the given initial value. */
+  /**
+   * Create a new instance with the given initial value.
+   * @param initialValue initialValue.
+   */
   protected SequentialNumber(final long initialValue) {
     currentValue = new AtomicLong(initialValue);
   }
@@ -40,7 +43,10 @@ public long getCurrentValue() {
     return currentValue.get();
   }
 
-  /** Set current value. */
+  /**
+   * Set current value.
+   * @param value value.
+   */
   public void setCurrentValue(long value) {
     currentValue.set(value);
   }
@@ -63,7 +69,12 @@ public long nextValue() {
     return currentValue.incrementAndGet();
   }
 
-  /** Skip to the new value. */
+  /**
+   * Skip to the new value.
+   * @param newValue newValue.
+   * @throws IllegalStateException
+   *         Cannot skip to less than the current value.
+   */
   public void skipTo(long newValue) throws IllegalStateException {
     for(;;) {
       final long c = getCurrentValue();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
index bb367278e537e..10cac5f49f7e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ServletUtil.java
@@ -30,7 +30,12 @@
 @InterfaceStability.Unstable
 public class ServletUtil {
   /**
-   * Initial HTML header
+   * Initial HTML header.
+   *
+   * @param response response.
+   * @param title title.
+   * @throws IOException raised on errors performing I/O.
+   * @return PrintWriter.
    */
   public static PrintWriter initHTML(ServletResponse response, String title
       ) throws IOException {
@@ -47,6 +52,10 @@ public static PrintWriter initHTML(ServletResponse response, String title
   /**
    * Get a parameter from a ServletRequest.
    * Return null if the parameter contains only white spaces.
+   *
+   * @param request request.
+   * @param name name.
+   * @return get a parameter from a ServletRequest.
    */
   public static String getParameter(ServletRequest request, String name) {
     String s = request.getParameter(name);
@@ -58,8 +67,13 @@ public static String getParameter(ServletRequest request, String name) {
   }
   
   /**
+   * parseLongParam.
+   *
+   * @param request request.
+   * @param param param.
    * @return a long value as passed in the given parameter, throwing
    * an exception if it is not present or if it is not a valid number.
+   * @throws IOException raised on errors performing I/O.
    */
   public static long parseLongParam(ServletRequest request, String param)
       throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java
index bddcbeb21f26a..8867900d0b692 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java
@@ -55,7 +55,10 @@ private Sets() {
    * instead. If {@code E} is an {@link Enum} type, use {@link EnumSet#noneOf}
    * instead. Otherwise, strongly consider using a {@code LinkedHashSet}
    * instead, at the cost of increased memory footprint, to get
-   * deterministic iteration behavior.
+   * deterministic iteration behavior.</p>
+   *
+   * @param <E> Generics Type E.
+   * @return a new, empty {@code TreeSet}
    */
   public static <E> HashSet<E> newHashSet() {
     return new HashSet<E>();
@@ -66,8 +69,9 @@ public static <E> HashSet<E> newHashSet() {
    * natural sort ordering of its elements.
    *
    * <p><b>Note:</b> if mutability is not required, use ImmutableSortedSet#of()
-   * instead.
+   * instead.</p>
    *
+   * @param <E> Generics Type E
    * @return a new, empty {@code TreeSet}
    */
   public static <E extends Comparable> TreeSet<E> newTreeSet() {
@@ -83,11 +87,15 @@ public static <E extends Comparable> TreeSet<E> newTreeSet() {
    * instead. If {@code E} is an {@link Enum} type, use
    * {@link EnumSet#of(Enum, Enum[])} instead. Otherwise, strongly consider
    * using a {@code LinkedHashSet} instead, at the cost of increased memory
-   * footprint, to get deterministic iteration behavior.
+   * footprint, to get deterministic iteration behavior.</p>
    *
    * <p>This method is just a small convenience, either for
    * {@code newHashSet(}{@link Arrays#asList}{@code (...))}, or for creating an
-   * empty set then calling {@link Collections#addAll}.
+   * empty set then calling {@link Collections#addAll}.</p>
+   *
+   * @param <E> Generics Type E.
+   * @param elements the elements that the set should contain.
+   * @return a new, empty thread-safe {@code Set}
    */
   @SafeVarargs
   public static <E> HashSet<E> newHashSet(E... elements) {
@@ -103,10 +111,14 @@ public static <E> HashSet<E> newHashSet(E... elements) {
    *
    * <p><b>Note:</b> if mutability is not required and the elements are
    * non-null, use ImmutableSet#copyOf(Iterable) instead. (Or, change
-   * {@code elements} to be a FluentIterable and call {@code elements.toSet()}.)
+   * {@code elements} to be a FluentIterable and call {@code elements.toSet()}.)</p>
    *
    * <p><b>Note:</b> if {@code E} is an {@link Enum} type, use
-   * newEnumSet(Iterable, Class) instead.
+   * newEnumSet(Iterable, Class) instead.</p>
+   *
+   * @param <E> Generics Type E.
+   * @param elements the elements that the set should contain.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> HashSet<E> newHashSet(Iterable<? extends E> elements) {
     return (elements instanceof Collection)
@@ -135,6 +147,7 @@ public static <E> HashSet<E> newHashSet(Iterable<? extends E> elements) {
    * then calling Iterables#addAll. This method is not very useful and will
    * likely be deprecated in the future.
    *
+   * @param <E> Generics Type E.
    * @param elements the elements that the set should contain
    * @return a new {@code TreeSet} containing those elements (minus duplicates)
    */
@@ -163,13 +176,17 @@ private static <E extends Comparable> boolean addAll(TreeSet<E> addTo,
    * calling Iterators#addAll.
    *
    * <p><b>Note:</b> if mutability is not required and the elements are
-   * non-null, use ImmutableSet#copyOf(Iterator) instead.
+   * non-null, use ImmutableSet#copyOf(Iterator) instead.</p>
    *
    * <p><b>Note:</b> if {@code E} is an {@link Enum} type, you should create
-   * an {@link EnumSet} instead.
+   * an {@link EnumSet} instead.</p>
    *
    * <p>Overall, this method is not very useful and will likely be deprecated
-   * in the future.
+   * in the future.</p>
+   *
+   * @param <E> Generics Type E.
+   * @param elements elements.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> HashSet<E> newHashSet(Iterator<? extends E> elements) {
     HashSet<E> set = newHashSet();
@@ -184,10 +201,11 @@ public static <E> HashSet<E> newHashSet(Iterator<? extends E> elements) {
    * expect it to do.
    *
    * <p>This behavior can't be broadly guaranteed, but has been tested with
-   * OpenJDK 1.7 and 1.8.
+   * OpenJDK 1.7 and 1.8.</p>
    *
    * @param expectedSize the number of elements you expect to add to the
    *     returned set
+   * @param <E> Generics Type E.
    * @return a new, empty hash set with enough capacity to hold
    *     {@code expectedSize} elements without resizing
    * @throws IllegalArgumentException if {@code expectedSize} is negative
@@ -223,6 +241,11 @@ private static <E> boolean addAll(Collection<E> addTo,
    * <p>Results are undefined if {@code set1} and {@code set2} are sets based
    * on different equivalence relations (as {@code HashSet}, {@code TreeSet},
    * and the keySet of an {@code IdentityHashMap} all are).
+   *
+   * @param set1 set1.
+   * @param set2 set2.
+   * @param <E> Generics Type E.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> Set<E> intersection(final Set<E> set1,
       final Set<E> set2) {
@@ -246,6 +269,11 @@ public static <E> Set<E> intersection(final Set<E> set1,
    * based on different equivalence relations (as {@link HashSet},
    * {@link TreeSet}, and the {@link Map#keySet} of an
    * {@code IdentityHashMap} all are).
+   *
+   * @param set1 set1.
+   * @param set2 set2.
+   * @param <E> Generics Type E.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> Set<E> union(
       final Set<E> set1, final Set<E> set2) {
@@ -272,6 +300,11 @@ public static <E> Set<E> union(
    * This method is used to find difference for HashSets. For TreeSets with
    * strict order requirement, recommended method is
    * {@link #differenceInTreeSets(Set, Set)}.
+   *
+   * @param set1 set1.
+   * @param set2 set2.
+   * @param <E> Generics Type E.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> Set<E> difference(
       final Set<E> set1, final Set<E> set2) {
@@ -297,6 +330,11 @@ public static <E> Set<E> difference(
    *
    * This method is used to find difference for TreeSets. For HashSets,
    * recommended method is {@link #difference(Set, Set)}.
+   *
+   * @param <E> Generics Type E.
+   * @param set1 set1.
+   * @param set2 set2.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> Set<E> differenceInTreeSets(
       final Set<E> set1, final Set<E> set2) {
@@ -320,6 +358,11 @@ public static <E> Set<E> differenceInTreeSets(
    * <p>Results are undefined if {@code set1} and {@code set2} are sets based
    * on different equivalence relations (as {@code HashSet}, {@code TreeSet},
    * and the keySet of an {@code IdentityHashMap} all are).
+   *
+   * @param set1 set1.
+   * @param set2 set2.
+   * @param <E> Generics Type E.
+   * @return a new, empty thread-safe {@code Set}.
    */
   public static <E> Set<E> symmetricDifference(
       final Set<E> set1, final Set<E> set2) {
@@ -345,6 +388,7 @@ public static <E> Set<E> symmetricDifference(
    * <p>Unlike {@code HashSet}, this class does NOT allow {@code null} to be
    * used as an element. The set is serializable.
    *
+   * @param <E> Generics Type.
    * @return a new, empty thread-safe {@code Set}
    */
   public static <E> Set<E> newConcurrentHashSet() {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 49c5a38765eed..65978f3c5f59c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -38,7 +38,6 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,7 +60,7 @@ public abstract class Shell {
    * {@value}
    */
   private static final String WINDOWS_PROBLEMS =
-      "https://wiki.apache.org/hadoop/WindowsProblems";
+      "https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems";
 
   /**
    * Name of the windows utils binary: {@value}.
@@ -123,6 +122,7 @@ public static boolean isJavaVersionAtLeast(int version) {
    * delimiters, no extra count will be added for delimiters.
    *
    * @param commands command parts, including any space delimiters
+   * @throws IOException raised on errors performing I/O.
    */
   public static void checkWindowsCommandLineLength(String...commands)
       throws IOException {
@@ -206,7 +206,11 @@ private static OSType getOSType() {
   public static final boolean PPC_64
                 = System.getProperties().getProperty("os.arch").contains("ppc64");
 
-  /** a Unix command to get the current user's groups list. */
+  /**
+   * a Unix command to get the current user's groups list.
+   *
+   * @return group command array.
+   */
   public static String[] getGroupsCommand() {
     return (WINDOWS)? new String[]{"cmd", "/c", "groups"}
                     : new String[]{"groups"};
@@ -217,6 +221,9 @@ public static String[] getGroupsCommand() {
    * If the OS is not WINDOWS, the command will get the user's primary group
    * first and finally get the groups list which includes the primary group.
    * i.e. the user's primary group will be included twice.
+   *
+   * @param user user.
+   * @return groups for user command.
    */
   public static String[] getGroupsForUserCommand(final String user) {
     //'groups username' command return is inconsistent across different unixes
@@ -236,6 +243,9 @@ public static String[] getGroupsForUserCommand(final String user) {
    * first and finally get the groups list which includes the primary group.
    * i.e. the user's primary group will be included twice.
    * This command does not support Windows and will only return group names.
+   *
+   * @param user user.
+   * @return groups id for user command.
    */
   public static String[] getGroupsIDForUserCommand(final String user) {
     //'groups username' command return is inconsistent across different unixes
@@ -249,19 +259,34 @@ public static String[] getGroupsIDForUserCommand(final String user) {
     }
   }
 
-  /** A command to get a given netgroup's user list. */
+  /**
+   * A command to get a given netgroup's user list.
+   *
+   * @param netgroup net group.
+   * @return users for net group command.
+   */
   public static String[] getUsersForNetgroupCommand(final String netgroup) {
     //'groups username' command return is non-consistent across different unixes
     return new String[] {"getent", "netgroup", netgroup};
   }
 
-  /** Return a command to get permission information. */
+  /**
+   * Return a command to get permission information.
+   *
+   * @return permission command.
+   */
   public static String[] getGetPermissionCommand() {
     return (WINDOWS) ? new String[] { getWinUtilsPath(), "ls", "-F" }
                      : new String[] { "ls", "-ld" };
   }
 
-  /** Return a command to set permission. */
+  /**
+   * Return a command to set permission.
+   *
+   * @param perm permission.
+   * @param recursive recursive.
+   * @return set permission command.
+   */
   public static String[] getSetPermissionCommand(String perm, boolean recursive) {
     if (recursive) {
       return (WINDOWS) ?
@@ -291,21 +316,37 @@ public static String[] getSetPermissionCommand(String perm,
     return cmdWithFile;
   }
 
-  /** Return a command to set owner. */
+  /**
+   * Return a command to set owner.
+   *
+   * @param owner owner.
+   * @return set owner command.
+   */
   public static String[] getSetOwnerCommand(String owner) {
     return (WINDOWS) ?
         new String[] { getWinUtilsPath(), "chown", "\"" + owner + "\"" }
         : new String[] { "chown", owner };
   }
 
-  /** Return a command to create symbolic links. */
+  /**
+   * Return a command to create symbolic links.
+   *
+   * @param target target.
+   * @param link link.
+   * @return symlink command.
+   */
   public static String[] getSymlinkCommand(String target, String link) {
     return WINDOWS ?
        new String[] { getWinUtilsPath(), "symlink", link, target }
        : new String[] { "ln", "-s", target, link };
   }
 
-  /** Return a command to read the target of the a symbolic link. */
+  /**
+   * Return a command to read the target of the a symbolic link.
+   *
+   * @param link link.
+   * @return read link command.
+   */
   public static String[] getReadlinkCommand(String link) {
     return WINDOWS ?
         new String[] { getWinUtilsPath(), "readlink", link }
@@ -321,7 +362,13 @@ public static String[] getCheckProcessIsAliveCommand(String pid) {
     return getSignalKillCommand(0, pid);
   }
 
-  /** Return a command to send a signal to a given pid. */
+  /**
+   * Return a command to send a signal to a given pid.
+   *
+   * @param code code.
+   * @param pid pid.
+   * @return signal kill command.
+   */
   public static String[] getSignalKillCommand(int code, String pid) {
     // Code == 0 means check alive
     if (Shell.WINDOWS) {
@@ -348,7 +395,11 @@ public static String[] getSignalKillCommand(int code, String pid) {
   /** Regular expression for environment variables: {@value}. */
   public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
 
-  /** Return a regular expression string that match environment variables. */
+  /**
+   * Return a regular expression string that match environment variables.
+   *
+   * @return environment variable regex.
+   */
   public static String getEnvironmentVariableRegex() {
     return (WINDOWS)
         ? "%(" + ENV_NAME_REGEX + "?)%"
@@ -891,7 +942,11 @@ protected void setWorkingDirectory(File dir) {
     this.dir = dir;
   }
 
-  /** Check to see if a command needs to be executed and execute if needed. */
+  /**
+   * Check to see if a command needs to be executed and execute if needed.
+   *
+   * @throws IOException raised on errors performing I/O.
+   */
   protected void run() throws IOException {
     if (lastTime + interval > Time.monotonicNow()) {
       return;
@@ -903,7 +958,11 @@ protected void run() throws IOException {
     runCommand();
   }
 
-  /** Run the command. */
+  /**
+   * Run the command.
+   *
+   * @throws IOException raised on errors performing I/O.
+   */
   private void runCommand() throws IOException {
     ProcessBuilder builder = new ProcessBuilder(getExecString());
     Timer timeOutTimer = null;
@@ -1050,10 +1109,19 @@ private static void joinThread(Thread t) {
     }
   }
 
-  /** return an array containing the command name and its parameters. */
+  /**
+   * return an array containing the command name and its parameters.
+   *
+   * @return exec string array.
+   */
   protected abstract String[] getExecString();
 
-  /** Parse the execution result */
+  /**
+   * Parse the execution result.
+   *
+   * @param lines lines.
+   * @throws IOException raised on errors performing I/O.
+   * */
   protected abstract void parseExecResult(BufferedReader lines)
   throws IOException;
 
@@ -1284,6 +1352,7 @@ private void setTimedOut() {
    * the <code>Shell</code> interface.
    * @param cmd shell command to execute.
    * @return the output of the executed command.
+   * @throws IOException raised on errors performing I/O.
    */
   public static String execCommand(String ... cmd) throws IOException {
     return execCommand(null, cmd, 0L);
@@ -1368,6 +1437,8 @@ public static void destroyAllShellProcesses() {
 
   /**
    * Static method to return a Set of all <code>Shell</code> objects.
+   *
+   * @return all shells set.
    */
   public static Set<Shell> getAllShells() {
     synchronized (CHILD_SHELLS) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index fbdd33331b62b..e85f850514b16 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -249,7 +249,7 @@ TimeUnit getTimeUnit() {
   }
 
   private final Set<HookEntry> hooks =
-      Collections.synchronizedSet(new HashSet<HookEntry>());
+      Collections.synchronizedSet(new HashSet<>());
 
   private AtomicBoolean shutdownInProgress = new AtomicBoolean(false);
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
index 16673129cb7fe..dc13697f158ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownThreadsHelper.java
@@ -68,10 +68,12 @@ public static boolean shutdownThread(Thread thread,
   }
 
   /**
+   * shutdownExecutorService.
+   *
    * @param service {@link ExecutorService to be shutdown}
    * @return <tt>true</tt> if the service is terminated,
    * <tt>false</tt> otherwise
-   * @throws InterruptedException
+   * @throws InterruptedException if the thread is interrupted.
    */
   public static boolean shutdownExecutorService(ExecutorService service)
       throws InterruptedException {
@@ -79,13 +81,15 @@ public static boolean shutdownExecutorService(ExecutorService service)
   }
 
   /**
+   * shutdownExecutorService.
+   *
    * @param service {@link ExecutorService to be shutdown}
    * @param timeoutInMs time to wait for {@link
    * ExecutorService#awaitTermination(long, java.util.concurrent.TimeUnit)}
    *                    calls in milli seconds.
    * @return <tt>true</tt> if the service is terminated,
    * <tt>false</tt> otherwise
-   * @throws InterruptedException
+   * @throws InterruptedException if the thread is interrupted.
    */
   public static boolean shutdownExecutorService(ExecutorService service,
                                         long timeoutInMs)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
index c0eedf6110d7f..7ccaebedb5662 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StopWatch.java
@@ -89,6 +89,9 @@ public StopWatch reset() {
   }
 
   /**
+   * now.
+   *
+   * @param timeUnit timeUnit.
    * @return current elapsed time in specified timeunit.
    */
   public long now(TimeUnit timeUnit) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
index 4a30ee2800aa9..2cbaa2ac1f1a6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
@@ -74,6 +74,9 @@ public static String weakIntern(String sample) {
   /**
    * Interns all the strings in the given array in place,
    * returning the same array.
+   *
+   * @param strings strings.
+   * @return internStringsInArray.
    */
   public static String[] internStringsInArray(String[] strings) {
     for (int i = 0; i < strings.length; i++) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index baae69f2e1791..b620ba73222ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -120,7 +120,12 @@ public static String humanReadableInt(long number) {
     return TraditionalBinaryPrefix.long2String(number, "", 1);
   }
 
-  /** The same as String.format(Locale.ENGLISH, format, objects). */
+  /**
+   * The same as String.format(Locale.ENGLISH, format, objects).
+   * @param format format.
+   * @param objects objects.
+   * @return format string.
+   */
   public static String format(final String format, final Object... objects) {
     return String.format(Locale.ENGLISH, format, objects);
   }
@@ -156,7 +161,7 @@ public static String arrayToString(String[] strs) {
   /**
    * Given an array of bytes it will convert the bytes to a hex string
    * representation of the bytes
-   * @param bytes
+   * @param bytes bytes.
    * @param start start index, inclusively
    * @param end end index, exclusively
    * @return hex string representation of the byte array
@@ -172,7 +177,11 @@ public static String byteToHexString(byte[] bytes, int start, int end) {
     return s.toString();
   }
 
-  /** Same as byteToHexString(bytes, 0, bytes.length). */
+  /**
+   * Same as byteToHexString(bytes, 0, bytes.length).
+   * @param bytes bytes.
+   * @return byteToHexString.
+   */
   public static String byteToHexString(byte bytes[]) {
     return byteToHexString(bytes, 0, bytes.length);
   }
@@ -203,8 +212,9 @@ public static byte[] hexStringToByte(String hex) {
     return bts;
   }
   /**
-   * 
-   * @param uris
+   * uriToString.
+   * @param uris uris.
+   * @return uriToString.
    */
   public static String uriToString(URI[] uris){
     if (uris == null) {
@@ -242,8 +252,9 @@ public static URI[] stringToURI(String[] str){
   }
   
   /**
-   * 
-   * @param str
+   * stringToPath.
+   * @param str str.
+   * @return path array.
    */
   public static Path[] stringToPath(String[] str){
     if (str == null) {
@@ -263,6 +274,8 @@ public static Path[] stringToPath(String[] str){
    * 
    * @param finishTime finish time
    * @param startTime start time
+   * @return a String in the format Xhrs, Ymins, Z sec,
+   *         for the time difference between two times.
    */
   public static String formatTimeDiff(long finishTime, long startTime){
     long timeDiff = finishTime - startTime; 
@@ -275,6 +288,7 @@ public static String formatTimeDiff(long finishTime, long startTime){
    * String in the format Xhrs, Ymins, Z sec. 
    * 
    * @param timeDiff The time difference to format
+   * @return formatTime String.
    */
   public static String formatTime(long timeDiff){
     StringBuilder buf = new StringBuilder();
@@ -305,6 +319,7 @@ public static String formatTime(long timeDiff){
    * more than 100 hours ,it is displayed as 99hrs, 59mins, 59sec.
    *
    * @param timeDiff The time difference to format
+   * @return format time sortable.
    */
   public static String formatTimeSortable(long timeDiff) {
     StringBuilder buf = new StringBuilder();
@@ -563,6 +578,7 @@ public static String[] split(
    * @param escapeChar character used to escape
    * @param start from where to search
    * @param split used to pass back the extracted string
+   * @return index.
    */
   public static int findNext(String str, char separator, char escapeChar, 
                              int start, StringBuilder split) {
@@ -615,7 +631,12 @@ private static boolean hasChar(char[] chars, char character) {
   }
   
   /**
+   * escapeString.
+   *
+   * @param str str.
+   * @param escapeChar escapeChar.
    * @param charsToEscape array of characters to be escaped
+   * @return escapeString.
    */
   public static String escapeString(String str, char escapeChar, 
                                     char[] charsToEscape) {
@@ -658,7 +679,11 @@ public static String unEscapeString(
   }
   
   /**
+   * unEscapeString.
+   * @param str str.
+   * @param escapeChar escapeChar.
    * @param charsToEscape array of characters to unescape
+   * @return escape string.
    */
   public static String unEscapeString(String str, char escapeChar, 
                                       char[] charsToEscape) {
@@ -807,7 +832,10 @@ private TraditionalBinaryPrefix(int bitShift) {
     }
 
     /**
-     * @return The TraditionalBinaryPrefix object corresponding to the symbol.
+     * The TraditionalBinaryPrefix object corresponding to the symbol.
+     *
+     * @param symbol symbol.
+     * @return traditional binary prefix object.
      */
     public static TraditionalBinaryPrefix valueOf(char symbol) {
       symbol = Character.toUpperCase(symbol);
@@ -907,7 +935,7 @@ public static String long2String(long n, String unit, int decimalPlaces) {
 
     /**
      * Escapes HTML Special characters present in the string.
-     * @param string
+     * @param string param string.
      * @return HTML Escaped String representation
      */
     public static String escapeHTML(String string) {
@@ -942,13 +970,22 @@ public static String escapeHTML(String string) {
     }
 
   /**
+   * a byte description of the given long interger value.
+   *
+   * @param len len.
    * @return a byte description of the given long interger value.
    */
   public static String byteDesc(long len) {
     return TraditionalBinaryPrefix.long2String(len, "B", 2);
   }
 
-  /** @deprecated use StringUtils.format("%.2f", d). */
+  /**
+   * limitDecimalTo2.
+   *
+   * @param d double param.
+   * @return string value ("%.2f").
+   * @deprecated use StringUtils.format("%.2f", d).
+   */
   @Deprecated
   public static String limitDecimalTo2(double d) {
     return format("%.2f", d);
@@ -959,6 +996,7 @@ public static String limitDecimalTo2(double d) {
    *
    * @param separator Separator to join with.
    * @param strings Strings to join.
+   * @return join string.
    */
   public static String join(CharSequence separator, Iterable<?> strings) {
     Iterator<?> i = strings.iterator();
@@ -1054,6 +1092,8 @@ public static String replaceTokens(String template, Pattern pattern,
   
   /**
    * Get stack trace for a given thread.
+   * @param t thread.
+   * @return stack trace string.
    */
   public static String getStackTrace(Thread t) {
     final StackTraceElement[] stackTrace = t.getStackTrace();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
index 42005f0b09b3e..f0ce85bbac873 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Time.java
@@ -81,6 +81,8 @@ public static long monotonicNowNanos() {
 
   /**
    * Convert time in millisecond to human readable format.
+   *
+   * @param millis millisecond.
    * @return a human readable string for the input time
    */
   public static String formatTime(long millis) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
index a4fbce4ace86f..63c275a1b06bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
@@ -32,7 +32,7 @@
  * and only handle its custom arguments.</p>
  * 
  * <p>Here is how a typical <code>Tool</code> is implemented:</p>
- * <p><blockquote><pre>
+ * <blockquote><pre>
  *     public class MyApp extends Configured implements Tool {
  *     
  *       public int run(String[] args) throws Exception {
@@ -69,7 +69,7 @@
  *         System.exit(res);
  *       }
  *     }
- * </pre></blockquote><p>
+ * </pre></blockquote>
  * 
  * @see GenericOptionsParser
  * @see ToolRunner
@@ -82,7 +82,7 @@ public interface Tool extends Configurable {
    * 
    * @param args command specific arguments.
    * @return exit code.
-   * @throws Exception
+   * @throws Exception command exception.
    */
   int run(String [] args) throws Exception;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java
index 336700a6e276a..b2b57f233a7b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java
@@ -57,6 +57,7 @@ public class ToolRunner {
    * @param tool <code>Tool</code> to run.
    * @param args command-line arguments to the tool.
    * @return exit code of the {@link Tool#run(String[])} method.
+   * @throws Exception Exception.
    */
   public static int run(Configuration conf, Tool tool, String[] args) 
     throws Exception{
@@ -89,6 +90,7 @@ public static int run(Configuration conf, Tool tool, String[] args)
    * @param tool <code>Tool</code> to run.
    * @param args command-line arguments to the tool.
    * @return exit code of the {@link Tool#run(String[])} method.
+   * @throws Exception exception.
    */
   public static int run(Tool tool, String[] args) 
     throws Exception{
@@ -107,7 +109,13 @@ public static void printGenericCommandUsage(PrintStream out) {
   
   /**
    * Print out a prompt to the user, and return true if the user
-   * responds with "y" or "yes". (case insensitive)
+   * responds with "y" or "yes". (case insensitive).
+   *
+   * @param prompt prompt.
+   * @throws IOException raised on errors performing I/O.
+   * @return if the user
+   *         responds with "y" or "yes". (case insensitive) true,
+   *         not false.
    */
   public static boolean confirmPrompt(String prompt) throws IOException {
     while (true) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java
new file mode 100644
index 0000000000000..18d180ee47024
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/WeakReferenceMap.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.lang.ref.WeakReference;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A map of keys type K to objects of type V which uses weak references,
+ * so does lot leak memory through long-lived references
+ * <i>at the expense of losing references when GC takes place.</i>.
+ *
+ * This class is intended be used instead of ThreadLocal storage when
+ * references are to be cleaned up when the instance holding.
+ * In this use case, the key is the Long key.
+ *
+ * Concurrency.
+ * The class assumes that map entries are rarely contended for when writing,
+ * and that not blocking other threads is more important than atomicity.
+ * - a ConcurrentHashMap is used to map keys to weak references, with
+ *   all its guarantees.
+ * - there is no automatic pruning.
+ * - see {@link #create(Object)} for the concurrency semantics on entry creation.
+ */
+@InterfaceAudience.Private
+public class WeakReferenceMap<K, V> {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(WeakReferenceMap.class);
+
+  /**
+   * The reference map.
+   */
+  private final Map<K, WeakReference<V>> map = new ConcurrentHashMap<>();
+
+  /**
+   * Supplier of new instances.
+   */
+  private final Function<? super K, ? extends V> factory;
+
+  /**
+   * Nullable callback when a get on a key got a weak reference back.
+   * The assumption is that this is for logging/stats, which is why
+   * no attempt is made to use the call as a supplier of a new value.
+   */
+  private final Consumer<? super K> referenceLost;
+
+  /**
+   * Counter of references lost.
+   */
+  private final AtomicLong referenceLostCount = new AtomicLong();
+
+  /**
+   * Counter of entries created.
+   */
+  private final AtomicLong entriesCreatedCount = new AtomicLong();
+
+  /**
+   * Log to report loss of a reference during the create phase, which
+   * is believed to be a cause of HADOOP-18456.
+   */
+  private final LogExactlyOnce referenceLostDuringCreation = new LogExactlyOnce(LOG);
+
+  /**
+   * instantiate.
+   * @param factory supplier of new instances
+   * @param referenceLost optional callback on lost references.
+   */
+  public WeakReferenceMap(
+      Function<? super K, ? extends V> factory,
+      @Nullable final Consumer<? super K> referenceLost) {
+
+    this.factory = requireNonNull(factory);
+    this.referenceLost = referenceLost;
+  }
+
+  @Override
+  public String toString() {
+    return "WeakReferenceMap{" +
+        "size=" + size() +
+        ", referenceLostCount=" + referenceLostCount +
+        ", entriesCreatedCount=" + entriesCreatedCount +
+        '}';
+  }
+
+  /**
+   * Map size.
+   * @return the current map size.
+   */
+  public int size() {
+    return map.size();
+  }
+
+  /**
+   * Clear all entries.
+   */
+  public void clear() {
+    map.clear();
+  }
+
+  /**
+   * look up the value, returning the possibly empty weak reference
+   * to a value, or null if no value was found.
+   * @param key key to look up
+   * @return null if there is no entry, a weak reference if found
+   */
+  public WeakReference<V> lookup(K key) {
+    return map.get(key);
+  }
+
+  /**
+   * Get the value, creating if needed.
+   * @param key key.
+   * @return an instance.
+   */
+  public V get(K key) {
+    final WeakReference<V> currentWeakRef = lookup(key);
+    // resolve it, after which if not null, we have a strong reference
+    V strongVal = resolve(currentWeakRef);
+    if (strongVal != null) {
+      // all good.
+      return  strongVal;
+    }
+
+    // here, either currentWeakRef was null, or its reference was GC'd.
+    if (currentWeakRef != null) {
+      // garbage collection removed the reference.
+
+      // explicitly remove the weak ref from the map if it has not
+      // been updated by this point
+      // this is here just for completeness.
+      map.remove(key, currentWeakRef);
+
+      // log/report the loss.
+      noteLost(key);
+    }
+
+    // create a new value and add it to the map
+    return create(key);
+  }
+
+  /**
+   * Create a new instance under a key.
+   * <p>
+   * The instance is created, added to the map and then the
+   * map value retrieved.
+   * This ensures that the reference returned is that in the map,
+   * even if there is more than one entry being created at the same time.
+   * If that race does occur, it will be logged the first time it happens
+   * for this specific map instance.
+   * <p>
+   * HADOOP-18456 highlighted the risk of a concurrent GC resulting a null
+   * value being retrieved and so returned.
+   * To prevent this:
+   * <ol>
+   *   <li>A strong reference is retained to the newly created instance
+   *       in a local variable.</li>
+   *   <li>That variable is used after the resolution process, to ensure
+   *       the JVM doesn't consider it "unreachable" and so eligible for GC.</li>
+   *   <li>A check is made for the resolved reference being null, and if so,
+   *       the put() is repeated</li>
+   * </ol>
+   * @param key key
+   * @return the created value
+   */
+  public V create(K key) {
+    entriesCreatedCount.incrementAndGet();
+    /*
+     Get a strong ref so even if a GC happens in this method the reference is not lost.
+     It is NOT enough to have a reference in a field, it MUST be used
+     so as to ensure the reference isn't optimized away prematurely.
+     "A reachable object is any object that can be accessed in any potential continuing
+      computation from any live thread."
+    */
+
+    final V strongRef = requireNonNull(factory.apply(key),
+        "factory returned a null instance");
+    V resolvedStrongRef;
+    do {
+      WeakReference<V> newWeakRef = new WeakReference<>(strongRef);
+
+      // put it in the map
+      map.put(key, newWeakRef);
+
+      // get it back from the map
+      WeakReference<V> retrievedWeakRef = map.get(key);
+      // resolve that reference, handling the situation where somehow it was removed from the map
+      // between the put() and the get()
+      resolvedStrongRef = resolve(retrievedWeakRef);
+      if (resolvedStrongRef == null) {
+        referenceLostDuringCreation.warn("reference to %s lost during creation", key);
+        noteLost(key);
+      }
+    } while (resolvedStrongRef == null);
+
+    // note if there was any change in the reference.
+    // as this forces strongRef to be kept in scope
+    if (strongRef != resolvedStrongRef) {
+      LOG.debug("Created instance for key {}: {} overwritten by {}",
+          key, strongRef, resolvedStrongRef);
+    }
+
+    return resolvedStrongRef;
+  }
+
+  /**
+   * Put a value under the key.
+   * A null value can be put, though on a get() call
+   * a new entry is generated
+   *
+   * @param key key
+   * @param value value
+   * @return any old non-null reference.
+   */
+  public V put(K key, V value) {
+    return resolve(map.put(key, new WeakReference<>(value)));
+  }
+
+  /**
+   * Remove any value under the key.
+   * @param key key
+   * @return any old non-null reference.
+   */
+  public V remove(K key) {
+    return resolve(map.remove(key));
+  }
+
+  /**
+   * Does the map have a valid reference for this object?
+   * no-side effects: there's no attempt to notify or cleanup
+   * if the reference is null.
+   * @param key key to look up
+   * @return true if there is a valid reference.
+   */
+  public boolean containsKey(K key) {
+    final WeakReference<V> current = lookup(key);
+    return resolve(current) != null;
+  }
+
+  /**
+   * Given a possibly null weak reference, resolve
+   * its value.
+   * @param r reference to resolve
+   * @return the value or null
+   */
+  protected V resolve(WeakReference<V> r) {
+    return r == null ? null : r.get();
+  }
+
+  /**
+   * Prune all null weak references, calling the referenceLost
+   * callback for each one.
+   *
+   * non-atomic and non-blocking.
+   * @return the number of entries pruned.
+   */
+  public int prune() {
+    int count = 0;
+    final Iterator<Map.Entry<K, WeakReference<V>>> it = map.entrySet().iterator();
+    while (it.hasNext()) {
+      final Map.Entry<K, WeakReference<V>> next = it.next();
+      if (next.getValue().get() == null) {
+        it.remove();
+        count++;
+        noteLost(next.getKey());
+      }
+    }
+    return count;
+  }
+
+  /**
+   * Notify the reference lost callback.
+   * @param key key of lost reference
+   */
+  private void noteLost(final K key) {
+    // increment local counter
+    referenceLostCount.incrementAndGet();
+
+    // and call any notification function supplied in the constructor
+    if (referenceLost != null) {
+      referenceLost.accept(key);
+    }
+  }
+
+  /**
+   * Get count of references lost as detected
+   * during prune() or get() calls.
+   * @return count of references lost
+   */
+  public final long getReferenceLostCount() {
+    return referenceLostCount.get();
+  }
+
+  /**
+   * Get count of entries created on demand.
+   * @return count of entries created
+   */
+  public final long getEntriesCreatedCount() {
+    return entriesCreatedCount.get();
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
index 6d3894f5b4ca5..a8040ad90a893 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
@@ -18,12 +18,19 @@
 
 package org.apache.hadoop.util;
 
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
 import javax.xml.transform.*;
+import javax.xml.transform.sax.SAXTransformerFactory;
 import javax.xml.transform.stream.*;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import org.xml.sax.SAXException;
+
 import java.io.*;
 
 /**
@@ -33,21 +40,37 @@
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class XMLUtils {
+
+  public static final String DISALLOW_DOCTYPE_DECL =
+      "http://apache.org/xml/features/disallow-doctype-decl";
+  public static final String LOAD_EXTERNAL_DECL =
+      "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+  public static final String EXTERNAL_GENERAL_ENTITIES =
+      "http://xml.org/sax/features/external-general-entities";
+  public static final String EXTERNAL_PARAMETER_ENTITIES =
+      "http://xml.org/sax/features/external-parameter-entities";
+  public static final String CREATE_ENTITY_REF_NODES =
+      "http://apache.org/xml/features/dom/create-entity-ref-nodes";
+  public static final String VALIDATION =
+      "http://xml.org/sax/features/validation";
+
   /**
    * Transform input xml given a stylesheet.
    * 
    * @param styleSheet the style-sheet
    * @param xml input xml data
    * @param out output
-   * @throws TransformerConfigurationException
-   * @throws TransformerException
+   * @throws TransformerConfigurationException synopsis signals a problem
+   *         creating a transformer object.
+   * @throws TransformerException this is used for throwing processor
+   *          exceptions before the processing has started.
    */
   public static void transform(
                                InputStream styleSheet, InputStream xml, Writer out
                                ) 
     throws TransformerConfigurationException, TransformerException {
     // Instantiate a TransformerFactory
-    TransformerFactory tFactory = TransformerFactory.newInstance();
+    TransformerFactory tFactory = newSecureTransformerFactory();
 
     // Use the TransformerFactory to process the  
     // stylesheet and generate a Transformer
@@ -59,4 +82,82 @@ public static void transform(
     // and send the output to a Result object.
     transformer.transform(new StreamSource(xml), new StreamResult(out));
   }
+
+  /**
+   * This method should be used if you need a {@link DocumentBuilderFactory}. Use this method
+   * instead of {@link DocumentBuilderFactory#newInstance()}. The factory that is returned has
+   * secure configuration enabled.
+   *
+   * @return a {@link DocumentBuilderFactory} with secure configuration enabled
+   * @throws ParserConfigurationException if the {@code JAXP} parser does not support the
+   * secure configuration
+   */
+  public static DocumentBuilderFactory newSecureDocumentBuilderFactory()
+          throws ParserConfigurationException {
+    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    dbf.setFeature(DISALLOW_DOCTYPE_DECL, true);
+    dbf.setFeature(LOAD_EXTERNAL_DECL, false);
+    dbf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    dbf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    dbf.setFeature(CREATE_ENTITY_REF_NODES, false);
+    return dbf;
+  }
+
+  /**
+   * This method should be used if you need a {@link SAXParserFactory}. Use this method
+   * instead of {@link SAXParserFactory#newInstance()}. The factory that is returned has
+   * secure configuration enabled.
+   *
+   * @return a {@link SAXParserFactory} with secure configuration enabled
+   * @throws ParserConfigurationException if the {@code JAXP} parser does not support the
+   * secure configuration
+   * @throws SAXException if there are another issues when creating the factory
+   */
+  public static SAXParserFactory newSecureSAXParserFactory()
+          throws SAXException, ParserConfigurationException {
+    SAXParserFactory spf = SAXParserFactory.newInstance();
+    spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    spf.setFeature(DISALLOW_DOCTYPE_DECL, true);
+    spf.setFeature(LOAD_EXTERNAL_DECL, false);
+    spf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    spf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    return spf;
+  }
+
+  /**
+   * This method should be used if you need a {@link TransformerFactory}. Use this method
+   * instead of {@link TransformerFactory#newInstance()}. The factory that is returned has
+   * secure configuration enabled.
+   *
+   * @return a {@link TransformerFactory} with secure configuration enabled
+   * @throws TransformerConfigurationException if the {@code JAXP} transformer does not
+   * support the secure configuration
+   */
+  public static TransformerFactory newSecureTransformerFactory()
+          throws TransformerConfigurationException {
+    TransformerFactory trfactory = TransformerFactory.newInstance();
+    trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    trfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+    trfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+    return trfactory;
+  }
+
+  /**
+   * This method should be used if you need a {@link SAXTransformerFactory}. Use this method
+   * instead of {@link SAXTransformerFactory#newInstance()}. The factory that is returned has
+   * secure configuration enabled.
+   *
+   * @return a {@link SAXTransformerFactory} with secure configuration enabled
+   * @throws TransformerConfigurationException if the {@code JAXP} transformer does not
+   * support the secure configuration
+   */
+  public static SAXTransformerFactory newSecureSAXTransformerFactory()
+          throws TransformerConfigurationException {
+    SAXTransformerFactory trfactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
+    trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    trfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+    trfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+    return trfactory;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
index 6d38c606c8c4c..17d8233342743 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
@@ -88,6 +88,7 @@ public static int removeSpecificPerms(int perms, int remove) {
    * Parse comma separated list of ACL entries to secure generated nodes, e.g.
    * <code>sasl:hdfs/host1@MY.DOMAIN:cdrwa,sasl:hdfs/host2@MY.DOMAIN:cdrwa</code>
    *
+   * @param aclString aclString.
    * @return ACL list
    * @throws BadAclFormatException if an ACL is invalid
    */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
index e8ad18cfc87e3..e2299365f8e22 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/Key.java
@@ -100,8 +100,8 @@ public Key(byte[] value, double weight) {
   }
 
   /**
-   * @param value
-   * @param weight
+   * @param value value.
+   * @param weight weight.
    */
   public void set(byte[] value, double weight) {
     if (value == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
index 9304b483952d0..35ec9115a5f4c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/AsyncGet.java
@@ -52,7 +52,13 @@ R get(long timeout, TimeUnit unit)
 
   /** Utility */
   class Util {
-    /** Use {@link #get(long, TimeUnit)} timeout parameters to wait. */
+    /**
+     * Use {@link #get(long, TimeUnit)} timeout parameters to wait.
+     * @param obj object.
+     * @param timeout timeout.
+     * @param unit unit.
+     * @throws InterruptedException if the thread is interrupted.
+     */
     public static void wait(Object obj, long timeout, TimeUnit unit)
         throws InterruptedException {
       if (timeout < 0) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
index 0b349518e4c2b..5cc92eb71bb64 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/ExecutorHelper.java
@@ -47,12 +47,12 @@ static void logThrowableFromAfterExecute(Runnable r, Throwable t) {
       try {
         ((Future<?>) r).get();
       } catch (ExecutionException ee) {
-        LOG.warn(
-            "Execution exception when running task in " + Thread.currentThread()
+        LOG.debug(
+            "Execution exception when running task in {}", Thread.currentThread()
                 .getName());
         t = ee.getCause();
       } catch (InterruptedException ie) {
-        LOG.warn("Thread (" + Thread.currentThread() + ") interrupted: ", ie);
+        LOG.debug("Thread ( {} ) interrupted: ", Thread.currentThread(), ie);
         Thread.currentThread().interrupt();
       } catch (Throwable throwable) {
         t = throwable;
@@ -60,8 +60,8 @@ static void logThrowableFromAfterExecute(Runnable r, Throwable t) {
     }
 
     if (t != null) {
-      LOG.warn("Caught exception in thread " + Thread
-          .currentThread().getName() + ": ", t);
+      LOG.warn("Caught exception in thread {}  + : ", Thread
+              .currentThread().getName(), t);
     }
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
index 0bbceb59c31e7..6e2838bfe9c97 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
@@ -115,9 +115,8 @@ public static void shutdown(ExecutorService executorService, Logger logger,
     try {
       executorService.shutdown();
 
-      logger.debug(
-          "Gracefully shutting down executor service. Waiting max {} {}",
-          timeout, unit);
+      logger.debug("Gracefully shutting down executor service {}. Waiting max {} {}",
+          executorService, timeout, unit);
       if (!executorService.awaitTermination(timeout, unit)) {
         logger.debug(
             "Executor service has not shutdown yet. Forcing. "
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index edb4eb6ac1dea..c11b86838673a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -28,12 +28,17 @@
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.api.transaction.CuratorOp;
 import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.utils.ZookeeperFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.util.JaasConfiguration;
 import org.apache.hadoop.util.ZKUtil;
 import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.client.ZKClientConfig;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
@@ -79,8 +84,11 @@ public void close() {
 
   /**
    * Utility method to fetch the ZK ACLs from the configuration.
+   *
+   * @param conf configuration.
    * @throws java.io.IOException if the Zookeeper ACLs configuration file
    * cannot be read
+   * @return acl list.
    */
   public static List<ACL> getZKAcls(Configuration conf) throws IOException {
     // Parse authentication from configuration.
@@ -98,9 +106,12 @@ public static List<ACL> getZKAcls(Configuration conf) throws IOException {
 
   /**
    * Utility method to fetch ZK auth info from the configuration.
+   *
+   * @param conf configuration.
    * @throws java.io.IOException if the Zookeeper ACLs configuration file
    * cannot be read
    * @throws ZKUtil.BadAuthFormatException if the auth format is invalid
+   * @return ZKAuthInfo List.
    */
   public static List<ZKUtil.ZKAuthInfo> getZKAuths(Configuration conf)
       throws IOException {
@@ -148,6 +159,10 @@ public void start(List<AuthInfo> authInfos) throws IOException {
 
     CuratorFramework client = CuratorFrameworkFactory.builder()
         .connectString(zkHostPort)
+        .zookeeperFactory(new HadoopZookeeperFactory(
+            conf.get(CommonConfigurationKeys.ZK_SERVER_PRINCIPAL),
+            conf.get(CommonConfigurationKeys.ZK_KERBEROS_PRINCIPAL),
+            conf.get(CommonConfigurationKeys.ZK_KERBEROS_KEYTAB)))
         .sessionTimeoutMs(zkSessionTimeout)
         .retryPolicy(retryPolicy)
         .authorization(authInfos)
@@ -161,7 +176,7 @@ public void start(List<AuthInfo> authInfos) throws IOException {
    * Get ACLs for a ZNode.
    * @param path Path of the ZNode.
    * @return The list of ACLs.
-   * @throws Exception
+   * @throws Exception If it cannot contact Zookeeper.
    */
   public List<ACL> getACL(final String path) throws Exception {
     return curator.getACL().forPath(path);
@@ -180,7 +195,7 @@ public byte[] getData(final String path) throws Exception {
   /**
    * Get the data in a ZNode.
    * @param path Path of the ZNode.
-   * @param stat
+   * @param stat stat.
    * @return The data in the ZNode.
    * @throws Exception If it cannot contact Zookeeper.
    */
@@ -357,7 +372,10 @@ public void safeCreate(String path, byte[] data, List<ACL> acl,
 
   /**
    * Deletes the path. Checks for existence of path as well.
+   *
    * @param path Path to be deleted.
+   * @param fencingNodePath fencingNodePath.
+   * @param fencingACL fencingACL.
    * @throws Exception if any problem occurs while performing deletion.
    */
   public void safeDelete(final String path, List<ACL> fencingACL,
@@ -428,4 +446,61 @@ public void setData(String path, byte[] data, int version)
                               .forPath(path, data));
     }
   }
+
+  public static class HadoopZookeeperFactory implements ZookeeperFactory {
+    public final static String JAAS_CLIENT_ENTRY = "HadoopZookeeperClient";
+    private final String zkPrincipal;
+    private final String kerberosPrincipal;
+    private final String kerberosKeytab;
+
+    public HadoopZookeeperFactory(String zkPrincipal) {
+      this(zkPrincipal, null, null);
+    }
+
+    public HadoopZookeeperFactory(String zkPrincipal, String kerberosPrincipal,
+        String kerberosKeytab) {
+      this.zkPrincipal = zkPrincipal;
+      this.kerberosPrincipal = kerberosPrincipal;
+      this.kerberosKeytab = kerberosKeytab;
+    }
+
+    @Override
+    public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
+                                  Watcher watcher, boolean canBeReadOnly
+    ) throws Exception {
+      ZKClientConfig zkClientConfig = new ZKClientConfig();
+      if (zkPrincipal != null) {
+        LOG.info("Configuring zookeeper to use {} as the server principal",
+            zkPrincipal);
+        zkClientConfig.setProperty(ZKClientConfig.ZK_SASL_CLIENT_USERNAME,
+            zkPrincipal);
+      }
+      if (zkClientConfig.isSaslClientEnabled() && !isJaasConfigurationSet(zkClientConfig)) {
+        setJaasConfiguration(zkClientConfig);
+      }
+      return new ZooKeeper(connectString, sessionTimeout, watcher,
+          canBeReadOnly, zkClientConfig);
+    }
+
+    private boolean isJaasConfigurationSet(ZKClientConfig zkClientConfig) {
+      String clientConfig = zkClientConfig.getProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY,
+          ZKClientConfig.LOGIN_CONTEXT_NAME_KEY_DEFAULT);
+      return javax.security.auth.login.Configuration.getConfiguration()
+          .getAppConfigurationEntry(clientConfig) != null;
+    }
+
+    private void setJaasConfiguration(ZKClientConfig zkClientConfig) throws IOException {
+      if (kerberosPrincipal == null || kerberosKeytab == null) {
+        LOG.warn("JaasConfiguration has not been set since kerberos principal "
+            + "or keytab is not specified");
+        return;
+      }
+
+      String principal = SecurityUtil.getServerPrincipal(kerberosPrincipal, "");
+      JaasConfiguration jconf = new JaasConfiguration(JAAS_CLIENT_ENTRY, principal,
+          kerberosKeytab);
+      javax.security.auth.login.Configuration.setConfiguration(jconf);
+      zkClientConfig.setProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY, JAAS_CLIENT_ENTRY);
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java
new file mode 100644
index 0000000000000..695da7e932279
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util.functional;
+
+import java.io.Closeable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A task submitter which is closeable, and whose close() call
+ * shuts down the pool. This can help manage
+ * thread pool lifecycles.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class CloseableTaskPoolSubmitter implements TaskPool.Submitter,
+    Closeable {
+
+  /** Executors. */
+  private ExecutorService pool;
+
+  /**
+   * Constructor.
+   * @param pool non-null executor.
+   */
+  public CloseableTaskPoolSubmitter(final ExecutorService pool) {
+    this.pool = requireNonNull(pool);
+  }
+
+  /**
+   * Get the pool.
+   * @return the pool.
+   */
+  public ExecutorService getPool() {
+    return pool;
+  }
+
+  /**
+   * Shut down the pool.
+   */
+  @Override
+  public void close() {
+    if (pool != null) {
+      pool.shutdown();
+      pool = null;
+    }
+  }
+
+  @Override
+  public Future<?> submit(final Runnable task) {
+    return pool.submit(task);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java
index e2cdc0fd41472..67299ef96aec6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CommonCallableSupplier.java
@@ -34,7 +34,7 @@
 
 import org.apache.hadoop.util.DurationInfo;
 
-import static org.apache.hadoop.fs.impl.FutureIOSupport.raiseInnerCause;
+import static org.apache.hadoop.util.functional.FutureIO.raiseInnerCause;
 
 /**
  * A bridge from Callable to Supplier; catching exceptions
@@ -89,7 +89,9 @@ public static <T> CompletableFuture<T> submit(final Executor executor,
   /**
    * Wait for a list of futures to complete. If the list is empty,
    * return immediately.
+   *
    * @param futures list of futures.
+   * @param <T> Generics Type T.
    * @throws IOException      if one of the called futures raised an IOE.
    * @throws RuntimeException if one of the futures raised one.
    */
@@ -105,6 +107,8 @@ public static <T> void waitForCompletion(
 
   /**
    * Wait for a single of future to complete, extracting IOEs afterwards.
+   *
+   * @param <T> Generics Type T.
    * @param future future to wait for.
    * @throws IOException      if one of the called futures raised an IOE.
    * @throws RuntimeException if one of the futures raised one.
@@ -124,6 +128,7 @@ public static <T> void waitForCompletion(final CompletableFuture<T> future)
   /**
    * Wait for a single of future to complete, ignoring exceptions raised.
    * @param future future to wait for.
+   * @param <T> Generics Type T.
    */
   public static <T> void waitForCompletionIgnoringExceptions(
       @Nullable final CompletableFuture<T> future) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java
similarity index 60%
rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java
rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java
index 7477adeeb07b5..f0cd5c08c572b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestLocalMetadataStoreScale.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java
@@ -16,23 +16,28 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
+package org.apache.hadoop.util.functional;
 
 import java.io.IOException;
 
 /**
- * Scale test for LocalMetadataStore.
+ * Function of arity 4 which may raise an IOException.
+ * @param <I1> type of arg1.
+ * @param <I2> type of arg2.
+ * @param <I3> type of arg3.
+ * @param <I4> type of arg4.
+ * @param <R> return type.
  */
-public class ITestLocalMetadataStoreScale
-    extends AbstractITestS3AMetadataStoreScale {
-  @Override
-  public MetadataStore createMetadataStore() throws IOException {
-    MetadataStore ms = new LocalMetadataStore();
-    ms.initialize(getFileSystem(), new S3Guard.TtlTimeProvider(getConf()));
-    return ms;
-  }
+public interface Function4RaisingIOE<I1, I2, I3, I4, R> {
+
+  /**
+   * Apply the function.
+   * @param i1 argument 1.
+   * @param i2 argument 2.
+   * @param i3 argument 3.
+   * @param i4 argument 4.
+   * @return return value.
+   * @throws IOException any IOE.
+   */
+  R apply(I1 i1, I2 i2, I3 i3, I4 i4) throws IOException;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java
index 3f7218baa759f..c3fda19d8d73b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FutureIO.java
@@ -21,6 +21,8 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.UncheckedIOException;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -29,6 +31,8 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSBuilder;
 
 /**
  * Future IO Helper methods.
@@ -86,6 +90,8 @@ public static <T> T awaitFuture(final Future<T> future)
    * extracted and rethrown.
    * </p>
    * @param future future to evaluate
+   * @param timeout timeout to wait
+   * @param unit time unit.
    * @param <T> type of the result.
    * @return the result, if all went well.
    * @throws InterruptedIOException future was interrupted
@@ -185,4 +191,88 @@ public static IOException unwrapInnerException(final Throwable e) {
     }
   }
 
+  /**
+   * Propagate options to any builder, converting everything with the
+   * prefix to an option where, if there were 2+ dot-separated elements,
+   * it is converted to a schema.
+   * See {@link #propagateOptions(FSBuilder, Configuration, String, boolean)}.
+   * @param builder builder to modify
+   * @param conf configuration to read
+   * @param optionalPrefix prefix for optional settings
+   * @param mandatoryPrefix prefix for mandatory settings
+   * @param <T> type of result
+   * @param <U> type of builder
+   * @return the builder passed in.
+   */
+  public static <T, U extends FSBuilder<T, U>>
+      FSBuilder<T, U> propagateOptions(
+        final FSBuilder<T, U> builder,
+        final Configuration conf,
+        final String optionalPrefix,
+        final String mandatoryPrefix) {
+    propagateOptions(builder, conf,
+        optionalPrefix, false);
+    propagateOptions(builder, conf,
+        mandatoryPrefix, true);
+    return builder;
+  }
+
+  /**
+   * Propagate options to any builder, converting everything with the
+   * prefix to an option where, if there were 2+ dot-separated elements,
+   * it is converted to a schema.
+   * <pre>
+   *   fs.example.s3a.option becomes "s3a.option"
+   *   fs.example.fs.io.policy becomes "fs.io.policy"
+   *   fs.example.something becomes "something"
+   * </pre>
+   * @param builder builder to modify
+   * @param conf configuration to read
+   * @param prefix prefix to scan/strip
+   * @param mandatory are the options to be mandatory or optional?
+   */
+  public static void propagateOptions(
+      final FSBuilder<?, ?> builder,
+      final Configuration conf,
+      final String prefix,
+      final boolean mandatory) {
+
+    final String p = prefix.endsWith(".") ? prefix : (prefix + ".");
+    final Map<String, String> propsWithPrefix = conf.getPropsWithPrefix(p);
+    for (Map.Entry<String, String> entry : propsWithPrefix.entrySet()) {
+      // change the schema off each entry
+      String key = entry.getKey();
+      String val = entry.getValue();
+      if (mandatory) {
+        builder.must(key, val);
+      } else {
+        builder.opt(key, val);
+      }
+    }
+  }
+
+  /**
+   * Evaluate a CallableRaisingIOE in the current thread,
+   * converting IOEs to RTEs and propagating.
+   * @param callable callable to invoke
+   * @param <T> Return type.
+   * @return the evaluated result.
+   * @throws UnsupportedOperationException fail fast if unsupported
+   * @throws IllegalArgumentException invalid argument
+   */
+  public static <T> CompletableFuture<T> eval(
+      CallableRaisingIOE<T> callable) {
+    CompletableFuture<T> result = new CompletableFuture<>();
+    try {
+      result.complete(callable.apply());
+    } catch (UnsupportedOperationException | IllegalArgumentException tx) {
+      // fail fast here
+      throw tx;
+    } catch (Throwable tx) {
+      // fail lazily here to ensure callers expect all File IO operations to
+      // surface later
+      result.completeExceptionally(tx);
+    }
+    return result;
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RemoteIterators.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RemoteIterators.java
index 5fdea4f5b747a..d2f7742d3d988 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RemoteIterators.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RemoteIterators.java
@@ -47,18 +47,18 @@
  * with IOStatisticsSource passthrough, and of conversions of
  * the iterators to lists/arrays and of performing actions
  * on the values.
- * <p></p>
+ * <p>
  * This aims to make it straightforward to use lambda-expressions to
  * transform the results of an iterator, without losing the statistics
  * in the process, and to chain the operations together.
- * <p></p>
+ * </p>
  * The closeable operation will be passed through RemoteIterators which
  * wrap other RemoteIterators. This is to support any iterator which
  * can be closed to release held connections, file handles etc.
  * Unless client code is written to assume that RemoteIterator instances
  * may be closed, this is not likely to be broadly used. It is added
  * to make it possible to adopt this feature in a managed way.
- * <p></p>
+ * <p>
  * One notable feature is that the
  * {@link #foreach(RemoteIterator, ConsumerRaisingIOE)} method will
  * LOG at debug any IOStatistics provided by the iterator, if such
@@ -66,7 +66,7 @@
  * if the LOG is not set to debug, so it is a zero cost feature unless
  * the logger {@code org.apache.hadoop.fs.functional.RemoteIterators}
  * is at DEBUG.
- * <p></p>
+ * </p>
  * Based on the S3A Listing code, and some some work on moving other code
  * to using iterative listings so as to pick up the statistics.
  */
@@ -99,6 +99,7 @@ public static <T> RemoteIterator<T> remoteIteratorFromSingleton(
   /**
    * Create a remote iterator from a java.util.Iterator.
    * @param <T> type
+   * @param iterator iterator.
    * @return a remote iterator
    */
   public static <T> RemoteIterator<T> remoteIteratorFromIterator(
@@ -110,6 +111,7 @@ public static <T> RemoteIterator<T> remoteIteratorFromIterator(
    * Create a remote iterator from a java.util.Iterable -e.g. a list
    * or other collection.
    * @param <T> type
+   * @param iterable iterable.
    * @return a remote iterator
    */
   public static <T> RemoteIterator<T> remoteIteratorFromIterable(
@@ -120,6 +122,7 @@ public static <T> RemoteIterator<T> remoteIteratorFromIterable(
   /**
    * Create a remote iterator from an array.
    * @param <T> type
+   * @param array array.
    * @return a remote iterator
    */
   public static <T> RemoteIterator<T> remoteIteratorFromArray(T[] array) {
@@ -158,10 +161,11 @@ public static <S, T> RemoteIterator<T> typeCastingRemoteIterator(
    * Create a RemoteIterator from a RemoteIterator and a filter
    * function which returns true for every element to be passed
    * through.
-   * <p></p>
+   * <p>
    * Elements are filtered in the hasNext() method; if not used
    * the filtering will be done on demand in the {@code next()}
    * call.
+   * </p>
    * @param <S> type
    * @param iterator source
    * @param filter filter
@@ -218,16 +222,16 @@ public static <T> T[] toArray(RemoteIterator<T> source,
 
   /**
    * Apply an operation to all values of a RemoteIterator.
-   * <p></p>
+   *
    * If the iterator is an IOStatisticsSource returning a non-null
    * set of statistics, <i>and</i> this classes log is set to DEBUG,
    * then the statistics of the operation are evaluated and logged at
    * debug.
-   * <p></p>
+   * <p>
    * The number of entries processed is returned, as it is useful to
    * know this, especially during tests or when reporting values
    * to users.
-   * <p></p>
+   * </p>
    * This does not close the iterator afterwards.
    * @param source iterator source
    * @param consumer consumer of the values.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java
new file mode 100644
index 0000000000000..c9e6d0b78ac11
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java
@@ -0,0 +1,652 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util.functional;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.util.functional.RemoteIterators.remoteIteratorFromIterable;
+
+/**
+ * Utility class for parallel execution, takes closures for the various
+ * actions.
+ * There is no retry logic: it is expected to be handled by the closures.
+ * From {@code org.apache.hadoop.fs.s3a.commit.Tasks} which came from
+ * the Netflix committer patch.
+ * Apache Iceberg has its own version of this, with a common ancestor
+ * at some point in its history.
+ * A key difference with this class is that the iterator is always,
+ * internally, an {@link RemoteIterator}.
+ * This is to allow tasks to be scheduled while incremental operations
+ * such as paged directory listings are still collecting in results.
+ *
+ * While awaiting completion, this thread spins and sleeps a time of
+ * {@link #SLEEP_INTERVAL_AWAITING_COMPLETION}, which, being a
+ * busy-wait, is inefficient.
+ * There's an implicit assumption that remote IO is being performed, and
+ * so this is not impacting throughput/performance.
+ *
+ * History:
+ * This class came with the Netflix contributions to the S3A committers
+ * in HADOOP-13786.
+ * It was moved into hadoop-common for use in the manifest committer and
+ * anywhere else it is needed, and renamed in the process as
+ * "Tasks" has too many meanings in the hadoop source.
+ * The iterator was then changed from a normal java iterable
+ * to a hadoop {@link org.apache.hadoop.fs.RemoteIterator}.
+ * This allows a task pool to be supplied with incremental listings
+ * from object stores, scheduling work as pages of listing
+ * results come in, rather than blocking until the entire
+ * directory/directory tree etc has been enumerated.
+ *
+ * There is a variant of this in Apache Iceberg in
+ * {@code org.apache.iceberg.util.Tasks}
+ * That is not derived from any version in the hadoop codebase, it
+ * just shares a common ancestor somewhere in the Netflix codebase.
+ * It is the more sophisticated version.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class TaskPool {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TaskPool.class);
+
+  /**
+   * Interval in milliseconds to await completion.
+   */
+  private static final int SLEEP_INTERVAL_AWAITING_COMPLETION = 10;
+
+  private TaskPool() {
+  }
+
+  /**
+   * Callback invoked to process an item.
+   * @param <I> item type being processed
+   * @param <E> exception class which may be raised
+   */
+  @FunctionalInterface
+  public interface Task<I, E extends Exception> {
+    void run(I item) throws E;
+  }
+
+  /**
+   * Callback invoked on a failure.
+   * @param <I> item type being processed
+   * @param <E> exception class which may be raised
+   */
+  @FunctionalInterface
+  public interface FailureTask<I, E extends Exception> {
+
+    /**
+     * process a failure.
+     * @param item item the task is processing
+     * @param exception the exception which was raised.
+     * @throws E Exception of type E
+     */
+    void run(I item, Exception exception) throws E;
+  }
+
+  /**
+   * Builder for task execution.
+   * @param <I> item type
+   */
+  public static class Builder<I> {
+    private final RemoteIterator<I> items;
+    private Submitter service = null;
+    private FailureTask<I, ?> onFailure = null;
+    private boolean stopOnFailure = false;
+    private boolean suppressExceptions = false;
+    private Task<I, ?> revertTask = null;
+    private boolean stopRevertsOnFailure = false;
+    private Task<I, ?> abortTask = null;
+    private boolean stopAbortsOnFailure = false;
+    private int sleepInterval = SLEEP_INTERVAL_AWAITING_COMPLETION;
+
+    /**
+     * IOStatisticsContext to switch to in all threads
+     * taking part in the commit operation.
+     * This ensures that the IOStatistics collected in the
+     * worker threads will be aggregated into the total statistics
+     * of the thread calling the committer commit/abort methods.
+     */
+    private IOStatisticsContext ioStatisticsContext = null;
+
+    /**
+     * Create the builder.
+     * @param items items to process
+     */
+    Builder(RemoteIterator<I> items) {
+      this.items = requireNonNull(items, "items");
+    }
+
+    /**
+     * Create the builder.
+     * @param items items to process
+     */
+    Builder(Iterable<I> items) {
+      this(remoteIteratorFromIterable(items));
+    }
+
+    /**
+     * Declare executor service: if null, the tasks are executed in a single
+     * thread.
+     * @param submitter service to schedule tasks with.
+     * @return this builder.
+     */
+    public Builder<I> executeWith(@Nullable Submitter submitter) {
+
+      this.service = submitter;
+      return this;
+    }
+
+    /**
+     * Task to invoke on failure.
+     * @param task task
+     * @return the builder
+     */
+    public Builder<I> onFailure(FailureTask<I, ?> task) {
+      this.onFailure = task;
+      return this;
+    }
+
+    public Builder<I> stopOnFailure() {
+      this.stopOnFailure = true;
+      return this;
+    }
+
+    /**
+     * Suppress exceptions from tasks.
+     * RemoteIterator exceptions are not suppressable.
+     * @return the builder.
+     */
+    public Builder<I> suppressExceptions() {
+      return suppressExceptions(true);
+    }
+
+    /**
+     * Suppress exceptions from tasks.
+     * RemoteIterator exceptions are not suppressable.
+     * @param suppress new value
+     * @return the builder.
+     */
+    public Builder<I> suppressExceptions(boolean suppress) {
+      this.suppressExceptions = suppress;
+      return this;
+    }
+
+    /**
+     * Task to revert with after another task failed.
+     * @param task task to execute
+     * @return the builder
+     */
+    public Builder<I> revertWith(Task<I, ?> task) {
+      this.revertTask = task;
+      return this;
+    }
+
+    /**
+     * Stop trying to revert if one operation fails.
+     * @return the builder
+     */
+    public Builder<I> stopRevertsOnFailure() {
+      this.stopRevertsOnFailure = true;
+      return this;
+    }
+
+    /**
+     * Task to abort with after another task failed.
+     * @param task task to execute
+     * @return the builder
+     */
+    public Builder<I> abortWith(Task<I, ?> task) {
+      this.abortTask = task;
+      return this;
+    }
+
+    /**
+     * Stop trying to abort if one operation fails.
+     * @return the builder
+     */
+    public Builder<I> stopAbortsOnFailure() {
+      this.stopAbortsOnFailure = true;
+      return this;
+    }
+
+    /**
+     * Set the sleep interval.
+     * @param value new value
+     * @return the builder
+     */
+    public Builder<I> sleepInterval(final int value) {
+      sleepInterval = value;
+      return this;
+    }
+
+    /**
+     * Execute the task across the data.
+     * @param task task to execute
+     * @param <E> exception which may be raised in execution.
+     * @return true if the operation executed successfully
+     * @throws E any exception raised.
+     * @throws IOException IOExceptions raised by remote iterator or in execution.
+     */
+    public <E extends Exception> boolean run(Task<I, E> task) throws E, IOException {
+      requireNonNull(items, "items");
+      if (!items.hasNext()) {
+        // if there are no items, return without worrying about
+        // execution pools, errors etc.
+        return true;
+      }
+      if (service != null) {
+        // thread pool, so run in parallel
+        return runParallel(task);
+      } else {
+        // single threaded execution.
+        return runSingleThreaded(task);
+      }
+    }
+
+    /**
+     * Single threaded execution.
+     * @param task task to execute
+     * @param <E> exception which may be raised in execution.
+     * @return true if the operation executed successfully
+     * @throws E any exception raised.
+     * @throws IOException IOExceptions raised by remote iterator or in execution.
+     */
+    private <E extends Exception> boolean runSingleThreaded(Task<I, E> task)
+        throws E, IOException {
+      List<I> succeeded = new ArrayList<>();
+      List<Exception> exceptions = new ArrayList<>();
+
+      RemoteIterator<I> iterator = items;
+      boolean threw = true;
+      try {
+        while (iterator.hasNext()) {
+          I item = iterator.next();
+          try {
+            task.run(item);
+            succeeded.add(item);
+
+          } catch (Exception e) {
+            exceptions.add(e);
+
+            if (onFailure != null) {
+              try {
+                onFailure.run(item, e);
+              } catch (Exception failException) {
+                LOG.error("Failed to clean up on failure", e);
+                // keep going
+              }
+            }
+
+            if (stopOnFailure) {
+              break;
+            }
+          }
+        }
+
+        threw = false;
+      } catch (IOException iteratorIOE) {
+        // an IOE is reaised here during iteration
+        LOG.debug("IOException when iterating through {}", iterator, iteratorIOE);
+        throw iteratorIOE;
+      } finally {
+        // threw handles exceptions that were *not* caught by the catch block,
+        // and exceptions that were caught and possibly handled by onFailure
+        // are kept in exceptions.
+        if (threw || !exceptions.isEmpty()) {
+          if (revertTask != null) {
+            boolean failed = false;
+            for (I item : succeeded) {
+              try {
+                revertTask.run(item);
+              } catch (Exception e) {
+                LOG.error("Failed to revert task", e);
+                failed = true;
+                // keep going
+              }
+              if (stopRevertsOnFailure && failed) {
+                break;
+              }
+            }
+          }
+
+          if (abortTask != null) {
+            boolean failed = false;
+            while (iterator.hasNext()) {
+              try {
+                abortTask.run(iterator.next());
+              } catch (Exception e) {
+                failed = true;
+                LOG.error("Failed to abort task", e);
+                // keep going
+              }
+              if (stopAbortsOnFailure && failed) {
+                break;
+              }
+            }
+          }
+        }
+      }
+
+      if (!suppressExceptions && !exceptions.isEmpty()) {
+        TaskPool.<E>throwOne(exceptions);
+      }
+
+      return exceptions.isEmpty();
+    }
+
+    /**
+     * Parallel execution.
+     * All tasks run within the same IOStatisticsContext as the
+     * thread calling this method.
+     * @param task task to execute
+     * @param <E> exception which may be raised in execution.
+     * @return true if the operation executed successfully
+     * @throws E any exception raised.
+     * @throws IOException IOExceptions raised by remote iterator or in execution.
+     */
+    private <E extends Exception> boolean runParallel(final Task<I, E> task)
+        throws E, IOException {
+      final Queue<I> succeeded = new ConcurrentLinkedQueue<>();
+      final Queue<Exception> exceptions = new ConcurrentLinkedQueue<>();
+      final AtomicBoolean taskFailed = new AtomicBoolean(false);
+      final AtomicBoolean abortFailed = new AtomicBoolean(false);
+      final AtomicBoolean revertFailed = new AtomicBoolean(false);
+
+      List<Future<?>> futures = new ArrayList<>();
+      ioStatisticsContext = IOStatisticsContext.getCurrentIOStatisticsContext();
+
+      IOException iteratorIOE = null;
+      final RemoteIterator<I> iterator = this.items;
+      try {
+        while (iterator.hasNext()) {
+          final I item = iterator.next();
+          // submit a task for each item that will either run or abort the task
+          futures.add(service.submit(() -> {
+            setStatisticsContext();
+            try {
+              if (!(stopOnFailure && taskFailed.get())) {
+                // prepare and run the task
+                boolean threw = true;
+                try {
+                  LOG.debug("Executing task");
+                  task.run(item);
+                  succeeded.add(item);
+                  LOG.debug("Task succeeded");
+
+                  threw = false;
+
+                } catch (Exception e) {
+                  taskFailed.set(true);
+                  exceptions.add(e);
+                  LOG.info("Task failed {}", e.toString());
+                  LOG.debug("Task failed", e);
+
+                  if (onFailure != null) {
+                    try {
+                      onFailure.run(item, e);
+                    } catch (Exception failException) {
+                      LOG.warn("Failed to clean up on failure", e);
+                      // swallow the exception
+                    }
+                  }
+                } finally {
+                  if (threw) {
+                    taskFailed.set(true);
+                  }
+                }
+
+              } else if (abortTask != null) {
+                // abort the task instead of running it
+                if (stopAbortsOnFailure && abortFailed.get()) {
+                  return;
+                }
+
+                boolean failed = true;
+                try {
+                  LOG.info("Aborting task");
+                  abortTask.run(item);
+                  failed = false;
+                } catch (Exception e) {
+                  LOG.error("Failed to abort task", e);
+                  // swallow the exception
+                } finally {
+                  if (failed) {
+                    abortFailed.set(true);
+                  }
+                }
+              }
+            } finally {
+              resetStatisticsContext();
+            }
+          }));
+        }
+      } catch (IOException e) {
+        // iterator failure.
+        LOG.debug("IOException when iterating through {}", iterator, e);
+        iteratorIOE = e;
+        // mark as a task failure so all submitted tasks will halt/abort
+        taskFailed.set(true);
+      }
+      // let the above tasks complete (or abort)
+      waitFor(futures, sleepInterval);
+      int futureCount = futures.size();
+      futures.clear();
+
+      if (taskFailed.get() && revertTask != null) {
+        // at least one task failed, revert any that succeeded
+        LOG.info("Reverting all {} succeeded tasks from {} futures",
+            succeeded.size(), futureCount);
+        for (final I item : succeeded) {
+          futures.add(service.submit(() -> {
+            if (stopRevertsOnFailure && revertFailed.get()) {
+              return;
+            }
+
+            boolean failed = true;
+            setStatisticsContext();
+            try {
+              revertTask.run(item);
+              failed = false;
+            } catch (Exception e) {
+              LOG.error("Failed to revert task", e);
+              // swallow the exception
+            } finally {
+              if (failed) {
+                revertFailed.set(true);
+              }
+              resetStatisticsContext();
+            }
+          }));
+        }
+
+        // let the revert tasks complete
+        waitFor(futures, sleepInterval);
+      }
+
+      // give priority to execution exceptions over
+      // iterator exceptions.
+      if (!suppressExceptions && !exceptions.isEmpty()) {
+        // there's an exception list to build up, cast and throw.
+        TaskPool.<E>throwOne(exceptions);
+      }
+
+      // raise any iterator exception.
+      // this can not be suppressed.
+      if (iteratorIOE != null) {
+        throw iteratorIOE;
+      }
+
+      // return true if all tasks succeeded.
+      return !taskFailed.get();
+    }
+
+    /**
+     * Set the statistics context for this thread.
+     */
+    private void setStatisticsContext() {
+      if (ioStatisticsContext != null) {
+        IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext);
+      }
+    }
+
+    /**
+     * Reset the statistics context if it was set earlier.
+     * This unbinds the current thread from any statistics
+     * context.
+     */
+    private void resetStatisticsContext() {
+      if (ioStatisticsContext != null) {
+        IOStatisticsContext.setThreadIOStatisticsContext(null);
+      }
+    }
+  }
+
+  /**
+   * Wait for all the futures to complete; there's a small sleep between
+   * each iteration; enough to yield the CPU.
+   * @param futures futures.
+   * @param sleepInterval Interval in milliseconds to await completion.
+   */
+  private static void waitFor(Collection<Future<?>> futures, int sleepInterval) {
+    int size = futures.size();
+    LOG.debug("Waiting for {} tasks to complete", size);
+    int oldNumFinished = 0;
+    while (true) {
+      int numFinished = (int) futures.stream().filter(Future::isDone).count();
+
+      if (oldNumFinished != numFinished) {
+        LOG.debug("Finished count -> {}/{}", numFinished, size);
+        oldNumFinished = numFinished;
+      }
+
+      if (numFinished == size) {
+        // all of the futures are done, stop looping
+        break;
+      } else {
+        try {
+          Thread.sleep(sleepInterval);
+        } catch (InterruptedException e) {
+          futures.forEach(future -> future.cancel(true));
+          Thread.currentThread().interrupt();
+          break;
+        }
+      }
+    }
+  }
+
+  /**
+   * Create a task builder for the iterable.
+   * @param items item source.
+   * @param <I> type of result.
+   * @return builder.
+   */
+  public static <I> Builder<I> foreach(Iterable<I> items) {
+    return new Builder<>(requireNonNull(items, "items"));
+  }
+
+  /**
+   * Create a task builder for the remote iterator.
+   * @param items item source.
+   * @param <I> type of result.
+   * @return builder.
+   */
+  public static <I> Builder<I> foreach(RemoteIterator<I> items) {
+    return new Builder<>(items);
+  }
+
+  public static <I> Builder<I> foreach(I[] items) {
+    return new Builder<>(Arrays.asList(requireNonNull(items, "items")));
+  }
+
+  /**
+   * Throw one exception, adding the others as suppressed
+   * exceptions attached to the one thrown.
+   * This method never completes normally.
+   * @param exceptions collection of exceptions
+   * @param <E> class of exceptions
+   * @throws E an extracted exception.
+   */
+  private static <E extends Exception> void throwOne(
+      Collection<Exception> exceptions)
+      throws E {
+    Iterator<Exception> iter = exceptions.iterator();
+    Exception e = iter.next();
+    Class<? extends Exception> exceptionClass = e.getClass();
+
+    while (iter.hasNext()) {
+      Exception other = iter.next();
+      if (!exceptionClass.isInstance(other)) {
+        e.addSuppressed(other);
+      }
+    }
+
+    TaskPool.<E>castAndThrow(e);
+  }
+
+  /**
+   * Raise an exception of the declared type.
+   * This method never completes normally.
+   * @param e exception
+   * @param <E> class of exceptions
+   * @throws E a recast exception.
+   */
+  @SuppressWarnings("unchecked")
+  private static <E extends Exception> void castAndThrow(Exception e) throws E {
+    if (e instanceof RuntimeException) {
+      throw (RuntimeException) e;
+    }
+    throw (E) e;
+  }
+
+  /**
+   * Interface to whatever lets us submit tasks.
+   */
+  public interface Submitter {
+
+    /**
+     * Submit work.
+     * @param task task to execute
+     * @return the future of the submitted task.
+     */
+    Future<?> submit(Runnable task);
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/package-info.java
index 1c204bb9979a8..18d23b31ff6d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/package-info.java
@@ -18,14 +18,14 @@
 
 /**
  * Support for functional programming within the Hadoop APIs.
- * <p></p>
+ *
  * Much of this is needed simply to cope with Java's checked exceptions and
  * the fact that the java.util.function can only throw runtime exceptions.
- * <p></p>
+ * <p>
  * Pretty much all the Hadoop FS APIs raise IOExceptions, hence the need
  * for these classes. If Java had made a different decision about the
  * nature of exceptions, life would be better.
- * <p></p>
+ * </p>
  * Do note that the {@link org.apache.hadoop.util.functional.RemoteIterators}
  * iterators go beyond that of the java ones, in terms of declaring themselves
  * Closeable and implementors of
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
index 3f62aef00a5f6..595a09db3f824 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/hash/JenkinsHash.java
@@ -247,7 +247,7 @@ public int hash(byte[] key, int nbytes, int initval) {
   /**
    * Compute the hash of the specified file
    * @param args name of file to compute hash of.
-   * @throws IOException
+   * @throws IOException raised on errors performing I/O.
    */
   public static void main(String[] args) throws IOException {
     if (args.length != 1) {
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
index 502508cbf3b86..f1a1df5c9dbe3 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/pmdk_load.c
@@ -35,13 +35,14 @@
 #endif
 
 PmdkLibLoader * pmdkLoader;
+// 1 represents loaded. Otherwise, not loaded.
+int pmdkLoaded;
 
 /**
  *  pmdk_load.c
  *  Utility of loading the libpmem library and the required functions.
  *  Building of this codes won't rely on any libpmem source codes, but running
  *  into this will rely on successfully loading of the dynamic library.
- *
  */
 
 static const char* load_functions() {
@@ -56,6 +57,10 @@ static const char* load_functions() {
   return NULL;
 }
 
+/**
+ * It should be idempotent to call this function for checking
+ * whether PMDK lib is successfully loaded.
+ */
 void load_pmdk_lib(char* err, size_t err_len) {
   const char* errMsg;
   const char* library = NULL;
@@ -67,10 +72,13 @@ void load_pmdk_lib(char* err, size_t err_len) {
 
   err[0] = '\0';
 
-  if (pmdkLoader != NULL) {
+  if (pmdkLoaded == 1) {
     return;
   }
-  pmdkLoader = calloc(1, sizeof(PmdkLibLoader));
+
+  if (pmdkLoader == NULL) {
+    pmdkLoader = calloc(1, sizeof(PmdkLibLoader));
+  }
 
   // Load PMDK library
   #ifdef UNIX
@@ -103,4 +111,5 @@ void load_pmdk_lib(char* err, size_t err_len) {
   }
 
   pmdkLoader->libname = strdup(library);
+  pmdkLoaded = 1;
 }
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
index c895bce757b77..17bbcf8f48707 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto
index 6296f88da69b8..91d2e2e6c4c4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/GenericRefreshProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto
index cb91a13b04875..bccb57dd86832 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto
index 5a88a7ff03f02..d9c7d70c0e904 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/HAServiceProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto b/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto
index 16e2fb7c4db75..d853cf3afb3ca 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/IpcConnectionContext.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto
index f72cf1a8da16e..8cace2d454ae9 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto
index c3023ec26dfd4..0e38070ab33fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine2.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto
index 0e9d0d4baa413..77d883227e5fe 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtocolInfo.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto
index f57c6d6303916..7bf69a70ef07f 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto
index 463b7c548fe22..138ede842d554 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshCallQueueProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto
index a1130f5c2d96d..a0f9e41b7082d 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto
index 760e8261b4ea7..d9becf722e982 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
@@ -91,6 +91,10 @@ message RpcRequestHeaderProto { // the header for the RpcRequest
   optional RPCTraceInfoProto traceInfo = 6; // tracing info
   optional RPCCallerContextProto callerContext = 7; // call context
   optional int64 stateId = 8; // The last seen Global State ID
+  // Alignment context info for use with routers.
+  // The client should not interpret these bytes, but only forward bytes
+  // received from RpcResponseHeaderProto.routerFederatedState.
+  optional bytes routerFederatedState = 9;
 }
 
 
@@ -157,6 +161,10 @@ message RpcResponseHeaderProto {
   optional bytes clientId = 7; // Globally unique client ID
   optional sint32 retryCount = 8 [default = -1];
   optional int64 stateId = 9; // The last written Global State ID
+  // Alignment context info for use with routers.
+  // The client should not interpret these bytes, but only
+  // forward them to the router using RpcRequestHeaderProto.routerFederatedState.
+  optional bytes routerFederatedState = 10;
 }
 
 message RpcSaslProto {
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/Security.proto b/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
index 5177a86ef113e..37dbf7f18a9c7 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/Security.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto b/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto
index 8cf131bfb460a..390040cc9d152 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/TraceAdmin.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto b/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto
index 98bc05f4a360e..8f9d9f0d9e7f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/ZKFCProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f2c04427d2dff..047c5482062be 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -78,7 +78,7 @@
   <value>false</value>
   <description>
     Indicates if administrator ACLs are required to access
-    instrumentation servlets (JMX, METRICS, CONF, STACKS).
+    instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF).
   </description>
 </property>
 
@@ -130,6 +130,14 @@
   </description>
 </property>
 
+<property>
+  <name>hadoop.security.resolver.impl</name>
+  <value>org.apache.hadoop.net.DNSDomainNameResolver</value>
+  <description>
+    The resolver implementation used to resolve FQDN for Kerberos
+  </description>
+</property>
+
 <property>
   <name>hadoop.security.dns.log-slow-lookups.enabled</name>
   <value>false</value>
@@ -577,6 +585,18 @@
   </description>
 </property>
 
+<property>
+  <name>hadoop.security.group.mapping.ldap.group.search.filter.pattern</name>
+  <value></value>
+  <description>
+    Comma separated values that needs to be substituted in the group search
+    filter during group lookup. The values are substituted in the order they
+    appear in the list, the first value will replace {0} the second {1} and
+    so on.
+  </description>
+</property>
+
+
 <property>
   <name>hadoop.security.group.mapping.providers</name>
   <value></value>
@@ -793,7 +813,8 @@
   <description>The size of buffer for use in sequence files.
   The size of this buffer should probably be a multiple of hardware
   page size (4096 on Intel x86), and it determines how much data is
-  buffered during read and write operations.</description>
+  buffered during read and write operations. Must be greater than zero.
+  </description>
 </property>
 
 <property>
@@ -1073,14 +1094,6 @@
   </description>
 </property>
 
-<property>
-  <name>fs.viewfs.overload.scheme.target.swift.impl</name>
-  <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
-  <description>The SwiftNativeFileSystem for view file system overload scheme
-   when child file system and ViewFSOverloadScheme's schemes are swift.
-  </description>
-</property>
-
 <property>
   <name>fs.viewfs.overload.scheme.target.oss.impl</name>
   <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
@@ -1190,12 +1203,6 @@
   <description>File space usage statistics refresh interval in msec.</description>
 </property>
 
-<property>
-  <name>fs.swift.impl</name>
-  <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
-  <description>The implementation class of the OpenStack Swift Filesystem</description>
-</property>
-
 <property>
   <name>fs.automatic.close</name>
   <value>true</value>
@@ -1229,7 +1236,7 @@
     com.amazonaws.auth.AWSCredentialsProvider.
 
     When S3A delegation tokens are not enabled, this list will be used
-    to directly authenticate with S3 and DynamoDB services.
+    to directly authenticate with S3 and other AWS services.
     When S3A Delegation tokens are enabled, depending upon the delegation
     token binding it may be used
     to communicate wih the STS endpoint to request session/role
@@ -1358,12 +1365,6 @@
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.delegation.tokens.enabled</name>
-  <value>false</value>
-  <description></description>
-</property>
-
 <property>
   <name>fs.s3a.delegation.token.binding</name>
   <value></value>
@@ -1623,9 +1624,12 @@
 
 <property>
   <name>fs.s3a.buffer.dir</name>
-  <value>${hadoop.tmp.dir}/s3a</value>
+  <value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value>
   <description>Comma separated list of directories that will be used to buffer file
-    uploads to.</description>
+    uploads to.
+    Yarn container path will be used as default value on yarn applications,
+    otherwise fall back to hadoop.tmp.dir
+  </description>
 </property>
 
 <property>
@@ -1692,180 +1696,18 @@
   </description>
 </property>
 
-<property>
-    <name>fs.s3a.metadatastore.authoritative</name>
-    <value>false</value>
-    <description>
-        When true, allow MetadataStore implementations to act as source of
-        truth for getting file status and directory listings.  Even if this
-        is set to true, MetadataStore implementations may choose not to
-        return authoritative results.  If the configured MetadataStore does
-        not support being authoritative, this setting will have no effect.
-    </description>
-</property>
-
-<property>
-    <name>fs.s3a.metadatastore.metadata.ttl</name>
-    <value>15m</value>
-    <description>
-        This value sets how long an entry in a MetadataStore is valid.
-    </description>
-</property>
-
-<property>
-    <name>fs.s3a.metadatastore.impl</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
-    <description>
-        Fully-qualified name of the class that implements the MetadataStore
-        to be used by s3a.  The default class, NullMetadataStore, has no
-        effect: s3a will continue to treat the backing S3 service as the one
-        and only source of truth for file and directory metadata.
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.metadatastore.fail.on.write.error</name>
-  <value>true</value>
-  <description>
-    When true (default), FileSystem write operations generate
-    org.apache.hadoop.fs.s3a.MetadataPersistenceException if the metadata
-    cannot be saved to the metadata store.  When false, failures to save to
-    metadata store are logged at ERROR level, but the overall FileSystem
-    write operation succeeds.
-  </description>
-</property>
-
-<property>
-    <name>fs.s3a.s3guard.cli.prune.age</name>
-    <value>86400000</value>
-    <description>
-        Default age (in milliseconds) after which to prune metadata from the
-        metadatastore when the prune command is run.  Can be overridden on the
-        command-line.
-    </description>
-</property>
-
-
 <property>
   <name>fs.s3a.impl</name>
   <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
   <description>The implementation class of the S3A Filesystem</description>
 </property>
 
-<property>
-  <name>fs.s3a.s3guard.ddb.region</name>
-  <value></value>
-  <description>
-    AWS DynamoDB region to connect to. An up-to-date list is
-    provided in the AWS Documentation: regions and endpoints. Without this
-    property, the S3Guard will operate table in the associated S3 bucket region.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table</name>
-  <value></value>
-  <description>
-    The DynamoDB table name to operate. Without this property, the respective
-    S3 bucket name will be used.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.create</name>
-  <value>false</value>
-  <description>
-    If true, the S3A client will create the table if it does not already exist.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for read operations in terms of capacity
-    units for the DynamoDB table. This config value will only be used when
-    creating a new DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    If a positive integer is provided for this and the write capacity, then
-    a table with "provisioned capacity" will be created.
-    You can change the capacity of an existing provisioned-capacity table
-    through the "s3guard set-capacity" command.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for write operations in terms of
-    capacity units for the DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    Refer to related configuration option fs.s3a.s3guard.ddb.table.capacity.read
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.enabled</name>
-  <value>false</value>
-  <description>
-    Whether server-side encryption (SSE) is enabled or disabled on the table.
-    By default it's disabled, meaning SSE is set to AWS owned CMK.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.cmk</name>
-  <value/>
-  <description>
-    The KMS Customer Master Key (CMK) used for the KMS encryption on the table.
-    To specify a CMK, this config value can be its key ID, Amazon Resource Name
-    (ARN), alias name, or alias ARN. Users only need to provide this config if
-    the key is different from the default DynamoDB KMS Master Key, which is
-    alias/aws/dynamodb.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.max.retries</name>
-  <value>9</value>
-    <description>
-      Max retries on throttled/incompleted DynamoDB operations
-      before giving up and throwing an IOException.
-      Each retry is delayed with an exponential
-      backoff timer which starts at 100 milliseconds and approximately
-      doubles each time.  The minimum wait before throwing an exception is
-      sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.throttle.retry.interval</name>
-  <value>100ms</value>
-    <description>
-      Initial interval to retry after a request is throttled events;
-      the back-off policy is exponential until the number of retries of
-      fs.s3a.s3guard.ddb.max.retries is reached.
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.background.sleep</name>
-  <value>25ms</value>
-  <description>
-    Length (in milliseconds) of pause between each batch of deletes when
-    pruning metadata.  Prevents prune operations (which can typically be low
-    priority background operations) from overly interfering with other I/O
-    operations.
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.retry.limit</name>
   <value>7</value>
   <description>
     Number of times to retry any repeatable S3 client request on failure,
-    excluding throttling requests and S3Guard inconsistency resolution.
+    excluding throttling requests.
   </description>
 </property>
 
@@ -1874,7 +1716,7 @@
   <value>500ms</value>
   <description>
     Initial retry interval when retrying operations for any reason other
-    than S3 throttle errors and S3Guard inconsistency resolution.
+    than S3 throttle errors.
   </description>
 </property>
 
@@ -1897,27 +1739,6 @@
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.limit</name>
-  <value>7</value>
-  <description>
-    Number of times to retry attempts to read/open/copy files when
-    S3Guard believes a specific version of the file to be available,
-    but the S3 request does not find any version of a file, or a different
-    version.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.interval</name>
-  <value>2s</value>
-  <description>
-    Initial interval between attempts to retry operations while waiting for S3
-    to become consistent with the S3Guard data.
-    An exponential back-off is used here: every failure doubles the delay.
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.committer.name</name>
   <value>file</value>
@@ -2205,7 +2026,27 @@
   </description>
 </property>
 
-<!-- Azure file system properties -->
+<property>
+  <name>fs.s3a.downgrade.syncable.exceptions</name>
+  <value>true</value>
+  <description>
+    Warn but continue when applications use Syncable.hsync when writing
+    to S3A.
+  </description>
+</property>
+
+<!--
+The switch to turn S3A auditing on or off.
+-->
+<property>
+  <name>fs.s3a.audit.enabled</name>
+  <value>true</value>
+  <description>
+    Should auditing of S3A requests be enabled?
+  </description>
+</property>
+
+  <!-- Azure file system properties -->
 <property>
   <name>fs.AbstractFileSystem.wasb.impl</name>
   <value>org.apache.hadoop.fs.azure.Wasb</value>
@@ -2325,6 +2166,13 @@
   <description>The AbstractFileSystem for gs: uris.</description>
 </property>
 
+<property>
+  <name>fs.azure.enable.readahead</name>
+  <value>false</value>
+  <description>Disable readahead/prefetching in AbfsInputStream.
+    See HADOOP-18521</description>
+</property>
+
 <property>
   <name>io.seqfile.compress.blocksize</name>
   <value>1000000</value>
@@ -3095,7 +2943,7 @@
   <value></value>
   <description>
     List of fencing methods to use for service fencing. May contain
-    builtin methods (eg shell and sshfence) or user-defined method.
+    builtin methods (eg shell, sshfence and powershell) or user-defined method.
   </description>
 </property>
 
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md
index 67f9c907ff97c..5d2c38d4c5646 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md
@@ -137,7 +137,8 @@ internal state stores:
 
 * The internal MapReduce state data will remain compatible across minor releases within the same major version to facilitate rolling upgrades while MapReduce workloads execute.
 * HDFS maintains metadata about the data stored in HDFS in a private, internal format that is versioned. In the event of an incompatible change, the store's version number will be incremented. When upgrading an existing cluster, the metadata store will automatically be upgraded if possible. After the metadata store has been upgraded, it is always possible to reverse the upgrade process.
-* The AWS S3A guard keeps a private, internal metadata store that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
+* The AWS S3A guard kept a private, internal metadata store.
+  Now that the feature has been removed, the store is obsolete and can be deleted.
 * The YARN resource manager keeps a private, internal state store of application and scheduler information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
 * The YARN node manager keeps a private, internal state store of application information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
 * The YARN federation service keeps a private, internal state store of application and cluster information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md
new file mode 100644
index 0000000000000..4b93cc219a5ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/AsyncProfilerServlet.md
@@ -0,0 +1,145 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+Async Profiler Servlet for Hadoop
+========================================
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+Purpose
+-------
+
+This document describes how to configure and use async profiler
+with Hadoop applications.
+Async profiler is a low overhead sampling profiler for Java that
+does not suffer from Safepoint bias problem. It features
+HotSpot-specific APIs to collect stack traces and to track memory
+allocations. The profiler works with OpenJDK, Oracle JDK and other
+Java runtimes based on the HotSpot JVM.
+
+Hadoop profiler servlet supports Async Profiler major versions
+1.x and 2.x.
+
+Prerequisites
+-------------
+
+Make sure Hadoop is installed, configured and setup correctly.
+For more information see:
+
+* [Single Node Setup](./SingleCluster.html) for first-time users.
+* [Cluster Setup](./ClusterSetup.html) for large, distributed clusters.
+
+Go to https://github.com/jvm-profiling-tools/async-profiler,
+download a release appropriate for your platform, and install
+on every cluster host.
+
+Set `ASYNC_PROFILER_HOME` in the environment (put it in hadoop-env.sh)
+to the root directory of the async-profiler install location, or pass
+it on the Hadoop daemon's command line as a system property as
+`-Dasync.profiler.home=/path/to/async-profiler`.
+
+
+Usage
+--------
+
+Once the prerequisites have been satisfied, access to the async-profiler
+is available by using Namenode or ResourceManager UI.
+
+Following options from async-profiler can be specified as query paramater.
+*  `-e event`          profiling event: cpu|alloc|lock|cache-misses etc.
+*  `-d duration`       run profiling for 'duration' seconds (integer)
+*  `-i interval`       sampling interval in nanoseconds (long)
+*  `-j jstackdepth`    maximum Java stack depth (integer)
+*  `-b bufsize`        frame buffer size (long)
+*  `-t`                profile different threads separately
+*  `-s`                simple class names instead of FQN
+*  `-o fmt[,fmt...]`   output format: summary|traces|flat|collapsed|svg|tree|jfr|html
+*  `--width px`        SVG width pixels (integer)
+*  `--height px`       SVG frame height pixels (integer)
+*  `--minwidth px`     skip frames smaller than px (double)
+*  `--reverse`         generate stack-reversed FlameGraph / Call tree
+
+
+Example:
+If Namenode http address is localhost:9870, and ResourceManager http
+address is localhost:8088, ProfileServlet running with async-profiler
+setup can be accessed with http://localhost:9870/prof and
+http://localhost:8088/prof for Namenode and ResourceManager processes
+respectively.
+
+Diving deep into some params:
+
+* To collect 10 second CPU profile of current process
+  (returns FlameGraph svg)
+  * `curl http://localhost:9870/prof` (FlameGraph svg for Namenode)
+  * `curl http://localhost:8088/prof` (FlameGraph svg for ResourceManager)
+* To collect 10 second CPU profile of pid 12345 (returns FlameGraph svg)
+  * `curl http://localhost:9870/prof?pid=12345` (For instance, provide
+   pid of Datanode here)
+* To collect 30 second CPU profile of pid 12345 (returns FlameGraph svg)
+  * `curl http://localhost:9870/prof?pid=12345&duration=30`
+* To collect 1 minute CPU profile of current process and output in tree
+  format (html)
+  * `curl http://localhost:9870/prof?output=tree&amp;duration=60`
+* To collect 10 second heap allocation profile of current process
+  (returns FlameGraph svg)
+  * `curl http://localhost:9870/prof?event=alloc`
+* To collect lock contention profile of current process
+  (returns FlameGraph svg)
+  * `curl http://localhost:9870/prof?event=lock`
+
+
+The following event types are supported by async-profiler.
+Use the 'event' parameter to specify. Default is 'cpu'.
+Not all operating systems will support all types.
+
+Perf events:
+
+* cpu
+* page-faults
+* context-switches
+* cycles
+* instructions
+* cache-references
+* cache-misses
+* branches
+* branch-misses
+* bus-cycles
+* L1-dcache-load-misses
+* LLC-load-misses
+* dTLB-load-misses
+
+Java events:
+
+* alloc
+* lock
+
+The following output formats are supported.
+Use the 'output' parameter to specify. Default is 'flamegraph'.
+
+Output formats:
+
+* summary: A dump of basic profiling statistics.
+* traces: Call traces.
+* flat: Flat profile (top N hot methods).
+* collapsed: Collapsed call traces in the format used by FlameGraph
+  script. This is a collection of call stacks, where each line is a
+  semicolon separated list of frames followed by a counter.
+* svg: FlameGraph in SVG format.
+* tree: Call tree in HTML format.
+* jfr: Call traces in Java Flight Recorder format.
+
+The 'duration' parameter specifies how long to collect trace data
+before generating output, specified in seconds. The default is 10 seconds.
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
index 03d162a18acc2..0ccb6a8b5c3ca 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
@@ -477,19 +477,12 @@ rolled back to the older layout.
 
 ##### AWS S3A Guard Metadata
 
-For each operation in the Hadoop S3 client (s3a) that reads or modifies
-file metadata, a shadow copy of that file metadata is stored in a separate
-metadata store, which offers HDFS-like consistency for the metadata, and may
-also provide faster lookups for things like file status or directory listings.
-S3A guard tables are created with a version marker which indicates
-compatibility.
+The S3Guard metastore used to store metadata in DynamoDB tables;
+as such it had to maintain a compatibility strategy.
+Now that S3Guard is removed, the tables are not needed.
 
-###### Policy
-
-The S3A guard metadata schema SHALL be considered
-[Private](./InterfaceClassification.html#Private) and
-[Unstable](./InterfaceClassification.html#Unstable). Any incompatible change
-to the schema MUST result in the version number of the schema being incremented.
+Applications configured to use an S3A metadata store other than
+the "null" store will fail.
 
 ##### YARN Resource Manager State Store
 
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index 281e42dad887f..a00feb039d889 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -208,7 +208,8 @@ The following table lists the configuration property names that are deprecated i
 | mapred.task.profile.params | mapreduce.task.profile.params |
 | mapred.task.profile.reduces | mapreduce.task.profile.reduces |
 | mapred.task.timeout | mapreduce.task.timeout |
-| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
+| mapred.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
+| mapreduce.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
 | mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum |
 | mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin |
 | mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin |
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md b/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md
index b04bc2488f8ae..e38dfc4c88bc7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DownstreamDev.md
@@ -300,7 +300,7 @@ that conflicts with a property defined by Hadoop can lead to unexpected and
 undesirable results. Users are encouraged to avoid using custom configuration
 property names that conflict with the namespace of Hadoop-defined properties
 and thus should avoid using any prefixes used by Hadoop,
-e.g. hadoop, io, ipc, fs, net, file, ftp, kfs, ha, file, dfs, mapred,
+e.g. hadoop, io, ipc, fs, net, ftp, ha, file, dfs, mapred,
 mapreduce, and yarn.
 
 ### Logging Configuration Files
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 9f634b645d0a1..16ecf3b25eb53 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -59,7 +59,7 @@ Copies source paths to stdout.
 
 Options
 
-* The `-ignoreCrc` option disables checkshum verification.
+* The `-ignoreCrc` option disables checksum verification.
 
 Example:
 
@@ -73,18 +73,19 @@ Returns 0 on success and -1 on error.
 checksum
 --------
 
-Usage: `hadoop fs -checksum [-v] URI`
+Usage: `hadoop fs -checksum [-v] URI [URI ...]`
 
-Returns the checksum information of a file.
+Returns the checksum information of the file(s).
 
 Options
 
-* The `-v` option displays blocks size for the file.
+* The `-v` option displays blocks size for the file(s).
 
 Example:
 
 * `hadoop fs -checksum hdfs://nn1.example.com/file1`
 * `hadoop fs -checksum file:///etc/hosts`
+* `hadoop fs -checksum file:///etc/hosts hdfs://nn1.example.com/file1`
 
 chgrp
 -----
@@ -177,7 +178,7 @@ Returns 0 on success and -1 on error.
 cp
 ----
 
-Usage: `hadoop fs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> `
+Usage: `hadoop fs -cp [-f] [-p | -p[topax]] [-d] [-t <thread count>] [-q <thread pool queue size>] URI [URI ...] <dest>`
 
 Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory.
 
@@ -185,13 +186,19 @@ Copy files from source to destination. This command allows multiple sources as w
 
 Options:
 
-* The -f option will overwrite the destination if it already exists.
-* The -p option will preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag.
+* `-f` : Overwrite the destination if it already exists.
+* `-d` : Skip creation of temporary file with the suffix `._COPYING_`.
+* `-p` : Preserve file attributes [topax] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag.
+* `-t <thread count>` : Number of threads to be used, default is 1. Useful when copying directories containing more than 1 file.
+* `-q <thread pool queue size>` : Thread pool queue size to be used, default is 1024. It takes effect only when thread count greater than 1.
 
 Example:
 
 * `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2`
+* `hadoop fs -cp -f -d /user/hadoop/file1 /user/hadoop/file2`
 * `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
+* `hadoop fs -cp -t 5 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
+* `hadoop fs -cp -t 10 -q 2048 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
 
 Exit Code:
 
@@ -323,27 +330,33 @@ Returns 0 on success and -1 on error.
 get
 ---
 
-Usage: `hadoop fs -get [-ignorecrc] [-crc] [-p] [-f] <src> <localdst> `
+Usage: `hadoop fs -get [-ignoreCrc] [-crc] [-p] [-f] [-t <thread count>] [-q <thread pool queue size>] <src> ... <localdst> `
+
+Copy files to the local file system. Files that fail the CRC check may be copied with the -ignoreCrc option. Files and CRCs may be copied using the -crc option.
+
+Options:
 
-Copy files to the local file system. Files that fail the CRC check may be copied with the -ignorecrc option. Files and CRCs may be copied using the -crc option.
+* `-p` : Preserves access and modification times, ownership and the permissions.
+  (assuming the permissions can be propagated across filesystems)
+* `-f` : Overwrites the destination if it already exists.
+* `-ignoreCrc` : Skip CRC checks on the file(s) downloaded.
+* `-crc`: write CRC checksums for the files downloaded.
+* `-t <thread count>` : Number of threads to be used, default is 1.
+  Useful when downloading directories containing more than 1 file.
+* `-q <thread pool queue size>` : Thread pool queue size to be used, default is 1024.
+  It takes effect only when thread count greater than 1.
 
 Example:
 
 * `hadoop fs -get /user/hadoop/file localfile`
 * `hadoop fs -get hdfs://nn.example.com/user/hadoop/file localfile`
+* `hadoop fs -get -t 10 hdfs://nn.example.com/user/hadoop/dir1 localdir`
+* `hadoop fs -get -t 10 -q 2048 hdfs://nn.example.com/user/hadoop/dir* localdir`
 
 Exit Code:
 
 Returns 0 on success and -1 on error.
 
-Options:
-
-* `-p` : Preserves access and modification times, ownership and the permissions.
-(assuming the permissions can be propagated across filesystems)
-* `-f` : Overwrites the destination if it already exists.
-* `-ignorecrc` : Skip CRC checks on the file(s) downloaded.
-* `-crc`: write CRC checksums for the files downloaded.
-
 getfacl
 -------
 
@@ -392,7 +405,7 @@ Returns 0 on success and non-zero on error.
 getmerge
 --------
 
-Usage: `hadoop fs -getmerge [-nl] <src> <localdst>`
+Usage: `hadoop fs -getmerge [-nl] [-skip-empty-file] <src> <localdst>`
 
 Takes a source directory and a destination file as input and concatenates files in src into the destination local file. Optionally -nl can be set to enable adding a newline character (LF) at the end of each file.
 -skip-empty-file can be used to avoid unwanted newline characters in case of empty files.
@@ -401,6 +414,7 @@ Examples:
 
 * `hadoop fs -getmerge -nl  /src  /opt/output.txt`
 * `hadoop fs -getmerge -nl  /src/file1.txt /src/file2.txt  /output.txt`
+* `hadoop fs -getmerge -nl -skip-empty-file /src/file1.txt /src/file2.txt /output.txt`
 
 Exit Code:
 
@@ -525,7 +539,7 @@ Returns 0 on success and -1 on error.
 put
 ---
 
-Usage: `hadoop fs -put  [-f] [-p] [-l] [-d] [-t <thread count>] [-q <threadPool queue size>] [ - | <localsrc1>  .. ]. <dst>`
+Usage: `hadoop fs -put  [-f] [-p] [-l] [-d] [-t <thread count>] [-q <thread pool queue size>] [ - | <localsrc> ...] <dst>`
 
 Copy single src, or multiple srcs from local file system to the destination file system.
 Also reads input from stdin and writes to destination file system if the source is set to "-"
@@ -537,12 +551,13 @@ Options:
 * `-p` : Preserves access and modification times, ownership and the permissions.
 (assuming the permissions can be propagated across filesystems)
 * `-f` : Overwrites the destination if it already exists.
-* `-t <thread count>` : Number of threads to be used, default is 1. Useful
- when uploading a directory containing more than 1 file.
 * `-l` : Allow DataNode to lazily persist the file to disk, Forces a replication
  factor of 1. This flag will result in reduced durability. Use with care.
 * `-d` : Skip creation of temporary file with the suffix `._COPYING_`.
-* `-q <threadPool queue size>` : ThreadPool queue size to be used, default is 1024.
+* `-t <thread count>` : Number of threads to be used, default is 1.
+ Useful when uploading directories containing more than 1 file.
+* `-q <thread pool queue size>` : Thread pool queue size to be used, default is 1024.
+ It takes effect only when thread count greater than 1.
 
 
 Examples:
@@ -551,7 +566,8 @@ Examples:
 * `hadoop fs -put -f localfile1 localfile2 /user/hadoop/hadoopdir`
 * `hadoop fs -put -d localfile hdfs://nn.example.com/hadoop/hadoopfile`
 * `hadoop fs -put - hdfs://nn.example.com/hadoop/hadoopfile` Reads the input from stdin.
-* `hadoop fs -put -q 500 localfile3 hdfs://nn.example.com/hadoop/hadoopfile3`
+* `hadoop fs -put -t 5 localdir hdfs://nn.example.com/hadoop/hadoopdir`
+* `hadoop fs -put -t 10 -q 2048 localdir1 localdir2 hdfs://nn.example.com/hadoop/hadoopdir`
 
 Exit Code:
 
@@ -839,7 +855,7 @@ Return the help for an individual command.
 ====================================================
 
 The Hadoop FileSystem shell works with Object Stores such as Amazon S3,
-Azure WASB and OpenStack Swift.
+Azure ABFS and Google GCS.
 
 
 
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index 03759d80092cd..cd6e6fecb1389 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -85,6 +85,14 @@ This is the limit for each ldap query.  If `hadoop.security.group.mapping.ldap.s
 `hadoop.security.group.mapping.ldap.base` configures how far to walk up the groups hierarchy when resolving groups.
 By default, with a limit of 0, in order to be considered a member of a group, the user must be an explicit member in LDAP.  Otherwise, it will traverse the group hierarchy `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` levels up.
 
+It is possible to have custom group search filters with different arguments using
+the configuration `hadoop.security.group.mapping.ldap.group.search.filter.pattern`, we can configure comma separated values here and the values configured will be fetched from the LDAP attributes and will be replaced in the group
+search filter in the order they appear here, say if the first entry here is uid, so uid will be fetched from the attributes and the value fetched
+will be used in place of {0} in the group search filter, similarly the second value configured will replace {1} and so on.
+
+Note: If `hadoop.security.group.mapping.ldap.group.search.filter.pattern` is configured, the group search will always be done assuming this group
+search filter pattern irrespective of any other parameters.
+
 ### Bind user(s) ###
 If the LDAP server does not support anonymous binds,
 set the distinguished name of the user to bind in `hadoop.security.group.mapping.ldap.bind.user`.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 2002e7a98fe68..04cbd9fedf83c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -83,6 +83,7 @@ The default timeunit used for RPC metrics is milliseconds (as per the below desc
 | `RpcAuthorizationFailures` | Total number of authorization failures |
 | `RpcAuthorizationSuccesses` | Total number of authorization successes |
 | `NumOpenConnections` | Current number of open connections |
+| `NumInProcessHandler` | Current number of handlers on working |
 | `CallQueueLength` | Current length of the call queue |
 | `numDroppedConnections` | Total number of dropped connections |
 | `rpcQueueTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
@@ -103,6 +104,8 @@ The default timeunit used for RPC metrics is milliseconds (as per the below desc
 | `rpcLockWaitTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
 | `rpcLockWaitTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
 | `rpcLockWaitTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of RPC lock wait time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `TotalRequests` | Total num of requests served by the RPC server. |
+| `TotalRequestsPerSeconds` | Total num of requests per second served by the RPC server. |
 
 RetryCache/NameNodeRetryCache
 -----------------------------
@@ -285,6 +288,8 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `HAState` | (HA-only) Current state of the NameNode: initializing or active or standby or stopping state |
 | `FSState` | Current state of the file system: Safemode or Operational |
 | `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock |
+| `ReadLockLongHoldCount` | The number of time the read lock has been held for longer than the threshold |
+| `WriteLockLongHoldCount` | The number of time the write lock has been held for longer than the threshold |
 | `TotalSyncCount` | Total number of sync operations performed by edit log |
 | `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation|
 | `NameDirSize` | NameNode name directories size in bytes |
@@ -296,6 +301,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `FSN(Read/Write)Lock`*OperationName*`NanosAvgTime` | Average time of holding the lock by operations in nanoseconds |
 | `FSN(Read/Write)LockOverallNanosNumOps`  | Total number of acquiring lock by all operations |
 | `FSN(Read/Write)LockOverallNanosAvgTime` | Average time of holding the lock by all operations in nanoseconds |
+| `PendingSPSPaths` | The number of paths to be processed by storage policy satisfier |
 
 JournalNode
 -----------
@@ -477,6 +483,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
 | `PacketsSlowWriteToMirror` | Total number of packets whose write to other Datanodes in the pipeline takes more than a certain time (300ms by default) |
 | `PacketsSlowWriteToDisk` | Total number of packets whose write to disk takes more than a certain time (300ms by default) |
 | `PacketsSlowWriteToOsCache` | Total number of packets whose write to os cache takes more than a certain time (300ms by default) |
+| `slowFlushOrSyncCount` | Total number of packets whose sync/flush takes more than a certain time (300ms by default) |
+| `slowAckToUpstreamCount` | Total number of packets whose upstream ack takes more than a certain time (300ms by default) |
 
 FsVolume
 --------
@@ -510,6 +518,12 @@ contains tags such as Hostname as additional information along with metrics.
 | `WriteIoRateNumOps` | The number of file write io operations within an interval time of metric |
 | `WriteIoRateAvgTime` | Mean time of file write io operations in milliseconds |
 | `WriteIoLatency`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of file write io operations latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `TransferIoRateNumOps` | The number of file transfer io operations within an interval time of metric |
+| `TransferIoRateAvgTime` | Mean time of file transfer io operations in milliseconds |
+| `TransferIoLatency`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of file transfer io operations latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `NativeCopyIoRateNumOps` | The number of file nativeCopy io operations within an interval time of metric |
+| `NativeCopyIoRateAvgTime` | Mean time of file nativeCopy io operations in milliseconds |
+| `NativeCopyIoLatency`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of file nativeCopy io operations latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
 | `TotalFileIoErrors` | Total number (monotonically increasing) of file io error operations |
 | `FileIoErrorRateNumOps` | The number of file io error operations within an interval time of metric |
 | `FileIoErrorRateAvgTime` | It measures the mean time in milliseconds from the start of an operation to hitting a failure |
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
index ffe2aec96ab7a..ca32fd8ee2f95 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
@@ -134,7 +134,7 @@ Apache Hadoop's shell code has a [function library](./UnixShellAPI.html) that is
 
 The shell code allows for core functions to be overridden. However, not all functions can be or are safe to be replaced.  If a function is not safe to replace, it will have an attribute of Replaceable: No.  If a function is safe to replace, it will have the attribute of Replaceable: Yes.
 
-In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory.  Simply define the new, replacement function in this file and the system will pick it up automatically.  There may be as many replacement functions as needed in this file.  Examples of function replacement are in the `hadoop-user-functions.sh.examples` file.
+In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory.  Simply define the new, replacement function in this file and the system will pick it up automatically.  There may be as many replacement functions as needed in this file.  Examples of function replacement are in the `hadoop-user-functions.sh.example` file.
 
 Functions that are marked Public and Stable are safe to use in shell profiles as-is.  Other functions may change in a minor release.
 
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 2eb0bc07196d6..9fd14f2218939 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -453,6 +453,26 @@ The function `getLocatedFileStatus(FS, d)` is as defined in
 The atomicity and consistency constraints are as for
 `listStatus(Path, PathFilter)`.
 
+
+### `ContentSummary getContentSummary(Path path)`
+
+Given a path return its content summary.
+
+`getContentSummary()` first checks if the given path is a file and if yes, it returns 0 for directory count
+and 1 for file count.
+
+#### Preconditions
+
+    exists(FS, path) else raise FileNotFoundException
+
+#### Postconditions
+
+Returns a `ContentSummary` object with information such as directory count
+and file count for a given path.
+
+The atomicity and consistency constraints are as for
+`listStatus(Path, PathFilter)`.
+
 ### `BlockLocation[] getFileBlockLocations(FileStatus f, int s, int l)`
 
 #### Preconditions
@@ -681,7 +701,7 @@ The behavior of the returned stream is covered in [Output](outputstream.html).
  clients creating files with `overwrite==true` to fail if the file is created
  by another client between the two tests.
 
-* S3A, Swift and potentially other Object Stores do not currently change the `FS` state
+* The S3A and potentially other Object Stores connectors not currently change the `FS` state
 until the output stream `close()` operation is completed.
 This is a significant difference between the behavior of object stores
 and that of filesystems, as it allows &gt;1 client to create a file with `overwrite=false`,
@@ -794,97 +814,11 @@ exists in the metadata, but no copies of any its blocks can be located;
 
 ### `FSDataInputStreamBuilder openFile(Path path)`
 
-Creates a [`FSDataInputStreamBuilder`](fsdatainputstreambuilder.html)
-to construct a operation to open the file at `path` for reading.
-
-When `build()` is invoked on the returned `FSDataInputStreamBuilder` instance,
-the builder parameters are verified and
-`openFileWithOptions(Path, OpenFileParameters)` invoked.
-
-This (protected) operation returns a `CompletableFuture<FSDataInputStream>`
-which, when its `get()` method is called, either returns an input
-stream of the contents of opened file, or raises an exception.
-
-The base implementation of the `openFileWithOptions(PathHandle, OpenFileParameters)`
-ultimately invokes `open(Path, int)`.
-
-Thus the chain `openFile(path).build().get()` has the same preconditions
-and postconditions as `open(Path p, int bufferSize)`
+See [openFile()](openfile.html).
 
-However, there is one difference which implementations are free to
-take advantage of: 
-
-The returned stream MAY implement a lazy open where file non-existence or
-access permission failures may not surface until the first `read()` of the
-actual data.
-
-The `openFile()` operation may check the state of the filesystem during its
-invocation, but as the state of the filesystem may change betwen this call and
-the actual `build()` and `get()` operations, this file-specific
-preconditions (file exists, file is readable, etc) MUST NOT be checked here.
-
-FileSystem implementations which do not implement `open(Path, int)`
-MAY postpone raising an `UnsupportedOperationException` until either the
-`FSDataInputStreamBuilder.build()` or the subsequent `get()` call,
-else they MAY fail fast in the `openFile()` call.
-
-### Implementors notes
-
-The base implementation of `openFileWithOptions()` actually executes
-the `open(path)` operation synchronously, yet still returns the result
-or any failures in the `CompletableFuture<>`, so as to ensure that users
-code expecting this.
-
-Any filesystem where the time to open a file may be significant SHOULD
-execute it asynchronously by submitting the operation in some executor/thread
-pool. This is particularly recommended for object stores and other filesystems
-likely to be accessed over long-haul connections.
-
-Arbitrary filesystem-specific options MAY be supported; these MUST
-be prefixed with either the filesystem schema, e.g. `hdfs.`
-or in the "fs.SCHEMA" format as normal configuration settings `fs.hdfs`). The
-latter style allows the same configuration option to be used for both
-filesystem configuration and file-specific configuration.
-
-It SHOULD be possible to always open a file without specifying any options,
-so as to present a consistent model to users. However, an implementation MAY
-opt to require one or more mandatory options to be set.
-
-The returned stream may perform "lazy" evaluation of file access. This is
-relevant for object stores where the probes for existence are expensive, and,
-even with an asynchronous open, may be considered needless.
- 
 ### `FSDataInputStreamBuilder openFile(PathHandle)`
 
-Creates a `FSDataInputStreamBuilder` to build an operation to open a file.
-Creates a [`FSDataInputStreamBuilder`](fsdatainputstreambuilder.html)
-to construct a operation to open the file identified by the given `PathHandle` for reading.
-
-When `build()` is invoked on the returned `FSDataInputStreamBuilder` instance,
-the builder parameters are verified and
-`openFileWithOptions(PathHandle, OpenFileParameters)` invoked.
-
-This (protected) operation returns a `CompletableFuture<FSDataInputStream>`
-which, when its `get()` method is called, either returns an input
-stream of the contents of opened file, or raises an exception.
-
-The base implementation of the `openFileWithOptions(PathHandle, OpenFileParameters)` method
-returns a future which invokes `open(Path, int)`.
-
-Thus the chain `openFile(pathhandle).build().get()` has the same preconditions
-and postconditions as `open(Pathhandle, int)`
-
-As with `FSDataInputStreamBuilder openFile(PathHandle)`, the `openFile()`
-call must not be where path-specific preconditions are checked -that
-is postponed to the `build()` and `get()` calls.
-
-FileSystem implementations which do not implement `open(PathHandle handle, int bufferSize)`
-MAY postpone raising an `UnsupportedOperationException` until either the
-`FSDataInputStreamBuilder.build()` or the subsequent `get()` call,
-else they MAY fail fast in the `openFile()` call.
-
-The base implementation raises this exception in the `build()` operation;
-other implementations SHOULD copy this.
+See [openFile()](openfile.html).
 
 ### `PathHandle getPathHandle(FileStatus stat, HandleOpt... options)`
 
@@ -1240,7 +1174,7 @@ Renaming a file where the destination is a directory moves the file as a child
         FS' where:
             not exists(FS', src)
             and exists(FS', dest)
-            and data(FS', dest) == data (FS, dest)
+            and data(FS', dest) == data (FS, source)
         result = True
 
 
@@ -1291,7 +1225,7 @@ the parent directories of the destination then exist:
 There is a check for and rejection if the `parent(dest)` is a file, but
 no checks for any other ancestors.
 
-*Other Filesystems (including Swift) *
+*Other Filesystems*
 
 Other filesystems strictly reject the operation, raising a `FileNotFoundException`
 
@@ -1698,3 +1632,92 @@ in:readahead | READAHEAD  | CanSetReadahead  | Set the readahead on the input st
 dropbehind   | DROPBEHIND | CanSetDropBehind | Drop the cache.
 in:unbuffer  | UNBUFFER   | CanUnbuffer      | Reduce the buffering on the input stream.
 
+## <a name="etagsource"></a> Etag probes through the interface `EtagSource`
+
+FileSystem implementations MAY support querying HTTP etags from `FileStatus`
+entries. If so, the requirements are as follows
+
+### Etag support MUST BE across all list/`getFileStatus()` calls.
+
+That is: when adding etag support, all operations which return `FileStatus` or `ListLocatedStatus`
+entries MUST return subclasses which are instances of `EtagSource`.
+
+### FileStatus instances MUST have etags whenever the remote store provides them.
+
+To support etags, they MUST BE to be provided in both `getFileStatus()`
+and list calls.
+
+Implementors note: the core APIs which MUST BE overridden to achieve this are as follows:
+
+```java
+FileStatus getFileStatus(Path)
+FileStatus[] listStatus(Path)
+RemoteIterator<FileStatus> listStatusIterator(Path)
+RemoteIterator<LocatedFileStatus> listFiles([Path, boolean)
+```
+
+
+### Etags of files MUST BE Consistent across all list/getFileStatus operations.
+
+The value of `EtagSource.getEtag()` MUST be the same for list* queries which return etags for calls of `getFileStatus()` for the specific object.
+
+```java
+((EtagSource)getFileStatus(path)).getEtag() == ((EtagSource)listStatus(path)[0]).getEtag()
+```
+
+Similarly, the same value MUST BE returned for `listFiles()`, `listStatusIncremental()` of the path and
+when listing the parent path, of all files in the listing.
+
+### Etags MUST BE different for different file contents.
+
+Two different arrays of data written to the same path MUST have different etag values when probed.
+This is a requirement of the HTTP specification.
+
+### Etags of files SHOULD BE preserved across rename operations
+
+After a file is renamed, the value of `((EtagSource)getFileStatus(dest)).getEtag()`
+SHOULD be the same as the value of `((EtagSource)getFileStatus(source)).getEtag()`
+was before the rename took place.
+
+This is an implementation detail of the store; it does not hold for AWS S3.
+
+If and only if the store consistently meets this requirement, the filesystem SHOULD
+declare in `hasPathCapability()` that it supports
+`fs.capability.etags.preserved.in.rename`
+
+### Directories MAY have etags
+
+Directory entries MAY return etags in listing/probe operations; these entries MAY be preserved across renames.
+
+Equally, directory entries MAY NOT provide such entries, MAY NOT preserve them acrosss renames,
+and MAY NOT guarantee consistency over time.
+
+Note: special mention of the root path "/".
+As that isn't a real "directory", nobody should expect it to have an etag.
+
+### All etag-aware `FileStatus` subclass MUST BE `Serializable`; MAY BE `Writable`
+
+The base `FileStatus` class implements `Serializable` and  `Writable` and marshalls its fields appropriately.
+
+Subclasses MUST support java serialization (Some Apache Spark applications use it), preserving the etag.
+This is a matter of making the etag field non-static and adding a `serialVersionUID`.
+
+The `Writable` support was used for marshalling status data over Hadoop IPC calls;
+in Hadoop 3 that is implemented through `org/apache/hadoop/fs/protocolPB/PBHelper.java`and the methods deprecated.
+Subclasses MAY override the deprecated methods to add etag marshalling.
+However -but there is no expectation of this and such marshalling is unlikely to ever take place.
+
+### Appropriate etag Path Capabilities SHOULD BE declared
+
+1. `hasPathCapability(path, "fs.capability.etags.available")` MUST return true iff
+    the filesystem returns valid (non-empty etags) on file status/listing operations.
+2. `hasPathCapability(path, "fs.capability.etags.consistent.across.rename")` MUST return
+   true if and only if etags are preserved across renames.
+
+### Non-requirements of etag support
+
+* There is no requirement/expectation that `FileSystem.getFileChecksum(Path)` returns
+  a checksum value related to the etag of an object, if any value is returned.
+* If the same data is uploaded to the twice to the same or a different path,
+  the etag of the second upload MAY NOT match that of the first upload.
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
index 090696483be34..f64a2bd03b63b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md
@@ -443,6 +443,52 @@ The semantics of this are exactly equivalent to
 That is, the buffer is filled entirely with the contents of the input source
 from position `position`
 
+### `default void readVectored(List<? extends FileRange> ranges, IntFunction<ByteBuffer> allocate)`
+
+Read fully data for a list of ranges asynchronously. The default implementation
+iterates through the ranges, tries to coalesce the ranges based on values of
+`minSeekForVectorReads` and `maxReadSizeForVectorReads` and then read each merged
+ranges synchronously, but the intent is sub classes can implement efficient
+implementation. Reading in both direct and heap byte buffers are supported.
+Also, clients are encouraged to use `WeakReferencedElasticByteBufferPool` for
+allocating buffers such that even direct buffers are garbage collected when
+they are no longer referenced.
+
+The position returned by `getPos()` after `readVectored()` is undefined.
+
+If a file is changed while the `readVectored()` operation is in progress, the output is
+undefined. Some ranges may have old data, some may have new, and some may have both.
+
+While a `readVectored()` operation is in progress, normal read api calls may block.
+
+Note: Don't use direct buffers for reading from ChecksumFileSystem as that may
+lead to memory fragmentation explained in HADOOP-18296.
+
+
+#### Preconditions
+
+For each requested range:
+
+    range.getOffset >= 0 else raise IllegalArgumentException
+    range.getLength >= 0 else raise EOFException
+
+#### Postconditions
+
+For each requested range:
+
+    range.getData() returns CompletableFuture<ByteBuffer> which will have data
+    from range.getOffset to range.getLength.
+
+### `minSeekForVectorReads()`
+
+The smallest reasonable seek. Two ranges won't be merged together if the difference between
+end of first and start of next range is more than this value.
+
+### `maxReadSizeForVectorReads()`
+
+Maximum number of bytes which can be read in one go after merging the ranges.
+Two ranges won't be merged if the combined data to be read is more than this value.
+Essentially setting this to 0 will disable the merging of ranges.
 
 ## Consistency
 
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
index eadba174fc1a6..16a14150ef949 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
@@ -13,10 +13,10 @@
 -->
 
 <!--  ============================================================= -->
-<!--  CLASS: FSDataInputStreamBuilder -->
+<!--  CLASS: FutureDataInputStreamBuilder -->
 <!--  ============================================================= -->
 
-# class `org.apache.hadoop.fs.FSDataInputStreamBuilder`
+# class `org.apache.hadoop.fs.FutureDataInputStreamBuilder`
 
 <!-- MACRO{toc|fromDepth=1|toDepth=2} -->
 
@@ -27,7 +27,7 @@ file for reading.
 
 ## Invariants
 
-The `FSDataInputStreamBuilder` interface does not require parameters or
+The `FutureDataInputStreamBuilder` interface does not require parameters or
 or the state of `FileSystem` until [`build()`](#build) is
 invoked and/or during the asynchronous open operation itself.
 
@@ -39,11 +39,11 @@ path validation.
 ## Implementation-agnostic parameters.
 
 
-### <a name="Builder.bufferSize"></a> `FSDataInputStreamBuilder bufferSize(int bufSize)`
+### <a name="Builder.bufferSize"></a> `FutureDataInputStreamBuilder bufferSize(int bufSize)`
 
 Set the size of the buffer to be used.
 
-### <a name="Builder.withFileStatus"></a> `FSDataInputStreamBuilder withFileStatus(FileStatus status)`
+### <a name="Builder.withFileStatus"></a> `FutureDataInputStreamBuilder withFileStatus(FileStatus status)`
 
 A `FileStatus` instance which refers to the file being opened.
 
@@ -53,7 +53,7 @@ So potentially saving on remote calls especially to object stores.
 Requirements:
 
 * `status != null`
-* `status.getPath()` == the resolved path of the file being opened.
+* `status.getPath().getName()` == the name of the file being opened.
 
 The path validation MUST take place if the store uses the `FileStatus` when
 it opens files, and MAY be performed otherwise. The validation
@@ -65,27 +65,85 @@ If a filesystem implementation extends the `FileStatus` returned in its
 implementation MAY use this information when opening the file.
 
 This is relevant with those stores which return version/etag information,
-including the S3A and ABFS connectors -they MAY use this to guarantee that
-the file they opened is exactly the one returned in the listing.
+-they MAY use this to guarantee that the file they opened
+is exactly the one returned in the listing.
+
+
+The final `status.getPath().getName()` element of the supplied status MUST equal
+the name value of the path supplied to the  `openFile(path)` call.
+
+Filesystems MUST NOT validate the rest of the path.
+This is needed to support viewfs and other mount-point wrapper filesystems
+where schemas and paths are different. These often create their own FileStatus results
+
+Preconditions
+
+```python
+status == null or status.getPath().getName() == path.getName()
+
+```
+
+Filesystems MUST NOT require the class of `status` to equal
+that of any specific subclass their implementation returns in filestatus/list
+operations. This is to support wrapper filesystems and serialization/deserialization
+of the status.
+
 
 ### Set optional or mandatory parameters
 
-    FSDataInputStreamBuilder opt(String key, ...)
-    FSDataInputStreamBuilder must(String key, ...)
+    FutureDataInputStreamBuilder opt(String key, ...)
+    FutureDataInputStreamBuilder must(String key, ...)
 
 Set optional or mandatory parameters to the builder. Using `opt()` or `must()`,
 client can specify FS-specific parameters without inspecting the concrete type
 of `FileSystem`.
 
+Example:
+
 ```java
 out = fs.openFile(path)
-    .opt("fs.s3a.experimental.input.fadvise", "random")
-    .must("fs.s3a.readahead.range", 256 * 1024)
+    .must("fs.option.openfile.read.policy", "random")
+    .opt("fs.http.connection.timeout", 30_000L)
     .withFileStatus(statusFromListing)
     .build()
     .get();
 ```
 
+Here the read policy of `random` has been specified,
+with the requirement that the filesystem implementation must understand the option.
+An http-specific option has been supplied which may be interpreted by any store;
+If the filesystem opening the file does not recognize the option, it can safely be
+ignored.
+
+### When to use `opt()` versus `must()`
+
+The difference between `opt()` versus `must()` is how the FileSystem opening
+the file must react to an option which it does not recognize.
+
+```python
+
+def must(name, value):
+  if not name in known_keys:
+    raise IllegalArgumentException
+  if not name in supported_keys:
+    raise UnsupportedException
+
+
+def opt(name, value):
+  if not name in known_keys:
+     # ignore option
+
+```
+
+For any known key, the validation of the `value` argument MUST be the same
+irrespective of how the (key, value) pair was declared.
+
+1. For a filesystem-specific option, it is the choice of the implementation
+   how to validate the entry.
+1. For standard options, the specification of what is a valid `value` is
+   defined in this filesystem specification, validated through contract
+   tests.
+
 #### Implementation Notes
 
 Checking for supported options must be performed in the `build()` operation.
@@ -93,9 +151,9 @@ Checking for supported options must be performed in the `build()` operation.
 1. If a mandatory parameter declared via `must(key, value)`) is not recognized,
 `IllegalArgumentException` MUST be thrown.
 
-1. If a mandatory parameter declared via `must(key, value)`) relies on
+1. If a mandatory parameter declared via `must(key, value)` relies on
 a feature which is recognized but not supported in the specific
-Filesystem/FileContext instance `UnsupportedException` MUST be thrown.
+`FileSystem`/`FileContext` instance `UnsupportedException` MUST be thrown.
 
 The behavior of resolving the conflicts between the parameters set by
 builder methods (i.e., `bufferSize()`) and `opt()`/`must()` is as follows:
@@ -110,13 +168,18 @@ custom subclasses.
 
 This is critical to ensure safe use of the feature: directory listing/
 status serialization/deserialization can result result in the `withFileStatus()`
-argumennt not being the custom subclass returned by the Filesystem instance's
+argument not being the custom subclass returned by the Filesystem instance's
 own `getFileStatus()`, `listFiles()`, `listLocatedStatus()` calls, etc.
 
 In such a situation the implementations must:
 
-1. Validate the path (always).
-1. Use the status/convert to the custom type, *or* simply discard it.
+1. Verify that `status.getPath().getName()` matches the current `path.getName()`
+   value. The rest of the path MUST NOT be validated.
+1. Use any status fields as desired -for example the file length.
+
+Even if not values of the status are used, the presence of the argument
+can be interpreted as the caller declaring that they believe the file
+to be present and of the given size.
 
 ## Builder interface
 
@@ -128,26 +191,499 @@ completed, returns an input stream which can read data from the filesystem.
 
 The `build()` operation MAY perform the validation of the file's existence,
 its kind, so rejecting attempts to read from a directory or non-existent
-file. **Alternatively**, the `build()` operation may delay all checks
-until an asynchronous operation whose outcome is provided by the `Future`
+file. Alternatively
+* file existence/status checks MAY be performed asynchronously within the returned
+    `CompletableFuture<>`.
+* file existence/status checks MAY be postponed until the first byte is read in
+  any of the read such as `read()` or `PositionedRead`.
 
 That is, the precondition  `exists(FS, path)` and `isFile(FS, path)` are
-only guaranteed to have been met after the `get()` on the returned future is successful.
+only guaranteed to have been met after the `get()` called on returned future
+and an attempt has been made to read the stream.
 
-Thus, if even a file does not exist, the following call will still succeed, returning
-a future to be evaluated.
+Thus, if even when file does not exist, or is a directory rather than a file,
+the following call MUST succeed, returning a `CompletableFuture` to be evaluated.
 
 ```java
 Path p = new Path("file://tmp/file-which-does-not-exist");
 
 CompletableFuture<FSDataInputStream> future = p.getFileSystem(conf)
       .openFile(p)
-      .build;
+      .build();
 ```
 
-The preconditions for opening the file are checked during the asynchronous
-evaluation, and so will surface when the future is completed:
+The inability to access/read a file MUST raise an `IOException`or subclass
+in either the future's `get()` call, or, for late binding operations,
+when an operation to read data is invoked.
+
+Therefore the following sequence SHALL fail when invoked on the
+`future` returned by the previous example.
 
 ```java
-FSDataInputStream in = future.get();
+  future.get().read();
 ```
+
+Access permission checks have the same visibility requirements: permission failures
+MUST be delayed until the `get()` call and MAY be delayed into subsequent operations.
+
+Note: some operations on the input stream, such as `seek()` may not attempt any IO
+at all. Such operations MAY NOT raise exceotions when interacting with
+nonexistent/unreadable files.
+
+## <a name="options"></a> Standard `openFile()` options since hadoop branch-3.3
+
+These are options which `FileSystem` and `FileContext` implementation
+MUST recognise and MAY support by changing the behavior of
+their input streams as appropriate.
+
+Hadoop 3.3.0 added the `openFile()` API; these standard options were defined in
+a later release. Therefore, although they are "well known", unless confident that
+the application will only be executed against releases of Hadoop which knows of
+the options -applications SHOULD set the options via `opt()` calls rather than `must()`.
+
+When opening a file through the `openFile()` builder API, callers MAY use
+both `.opt(key, value)` and `.must(key, value)` calls to set standard and
+filesystem-specific options.
+
+If set as an `opt()` parameter, unsupported "standard" options MUST be ignored,
+as MUST unrecognized standard options.
+
+If set as a `must()` parameter, unsupported "standard" options MUST be ignored.
+unrecognized standard options MUST be rejected.
+
+The standard `openFile()` options are defined
+in `org.apache.hadoop.fs.OpenFileOptions`; they all SHALL start
+with `fs.option.openfile.`.
+
+Note that while all `FileSystem`/`FileContext` instances SHALL support these
+options to the extent that `must()` declarations SHALL NOT fail, the
+implementations MAY support them to the extent of interpreting the values. This
+means that it is not a requirement for the stores to actually read the read
+policy or file length values and use them when opening files.
+
+Unless otherwise stated, they SHOULD be viewed as hints.
+
+Note: if a standard option is added such that if set but not
+supported would be an error, then implementations SHALL reject it. For example,
+the S3A filesystem client supports the ability to push down SQL commands. If
+something like that were ever standardized, then the use of the option, either
+in `opt()` or `must()` argument MUST be rejected for filesystems which don't
+support the feature.
+
+### <a name="buffer.size"></a>  Option: `fs.option.openfile.buffer.size`
+
+Read buffer size in bytes.
+
+This overrides the default value set in the configuration with the option
+`io.file.buffer.size`.
+
+It is supported by all filesystem clients which allow for stream-specific buffer
+sizes to be set via `FileSystem.open(path, buffersize)`.
+
+### <a name="read.policy"></a> Option: `fs.option.openfile.read.policy`
+
+Declare the read policy of the input stream. This is a hint as to what the
+expected read pattern of an input stream will be. This MAY control readahead,
+buffering and other optimizations.
+
+Sequential reads may be optimized with prefetching data and/or reading data in
+larger blocks. Some applications (e.g. distCp) perform sequential IO even over
+columnar data.
+
+In contrast, random IO reads data in different parts of the file using a
+sequence of `seek()/read()`
+or via the `PositionedReadable` or `ByteBufferPositionedReadable` APIs.
+
+Random IO performance may be best if little/no prefetching takes place, along
+with other possible optimizations
+
+Queries over columnar formats such as Apache ORC and Apache Parquet perform such
+random IO; other data formats may be best read with sequential or whole-file
+policies.
+
+What is key is that optimizing reads for seqential reads may impair random
+performance -and vice versa.
+
+1. The seek policy is a hint; even if declared as a `must()` option, the
+   filesystem MAY ignore it.
+1. The interpretation/implementation of a policy is a filesystem specific
+   behavior -and it may change with Hadoop releases and/or specific storage
+   subsystems.
+1. If a policy is not recognized, the filesystem client MUST ignore it.
+
+| Policy       | Meaning                                                  |
+|--------------|----------------------------------------------------------|
+| `adaptive`   | Any adaptive policy implemented by the store.            |
+| `default`    | The default policy for this store. Generally "adaptive". |
+| `random`     | Optimize for random access.                              |
+| `sequential` | Optimize for sequential access.                          |
+| `vector`     | The Vectored IO API is intended to be used.              |
+| `whole-file` | The whole file will be read.                             |
+
+Choosing the wrong read policy for an input source may be inefficient.
+
+A list of read policies MAY be supplied; the first one recognized/supported by
+the filesystem SHALL be the one used. This allows for custom policies to be
+supported, for example an `hbase-hfile` policy optimized for HBase HFiles.
+
+The S3A and ABFS input streams both implement
+the [IOStatisticsSource](iostatistics.html) API, and can be queried for their IO
+Performance.
+
+*Tip:* log the `toString()` value of input streams at `DEBUG`. The S3A and ABFS
+Input Streams log read statistics, which can provide insight about whether reads
+are being performed efficiently or not.
+
+_Futher reading_
+
+* [Linux fadvise()](https://linux.die.net/man/2/fadvise).
+* [Windows `CreateFile()`](https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#caching-behavior)
+
+#### <a name="read.policy."></a> Read Policy `adaptive`
+
+Try to adapt the seek policy to the read pattern of the application.
+
+The `normal` policy of the S3A client and the sole policy supported by
+the `wasb:` client are both adaptive -they assume sequential IO, but once a
+backwards seek/positioned read call is made the stream switches to random IO.
+
+Other filesystem implementations may wish to adopt similar strategies, and/or
+extend the algorithms to detect forward seeks and/or switch from random to
+sequential IO if that is considered more efficient.
+
+Adaptive read policies are the absence of the ability to
+declare the seek policy in the `open()` API, so requiring it to be declared, if
+configurable, in the cluster/application configuration. However, the switch from
+sequential to random seek policies may be exensive.
+
+When applications explicitly set the `fs.option.openfile.read.policy` option, if
+they know their read plan, they SHOULD declare which policy is most appropriate.
+
+#### <a name="read.policy.default"></a> Read Policy ``
+
+The default policy for the filesystem instance.
+Implementation/installation-specific.
+
+#### <a name="read.policy.sequential"></a> Read Policy `sequential`
+
+Expect sequential reads from the first byte read to the end of the file/until
+the stream is closed.
+
+#### <a name="read.policy.random"></a> Read Policy `random`
+
+Expect `seek()/read()` sequences, or use of `PositionedReadable`
+or `ByteBufferPositionedReadable` APIs.
+
+
+#### <a name="read.policy.vector"></a> Read Policy `vector`
+
+This declares that the caller intends to use the Vectored read API of
+[HADOOP-11867](https://issues.apache.org/jira/browse/HADOOP-11867)
+_Add a high-performance vectored read API_.
+
+This is a hint: it is not a requirement when using the API.
+It does inform the implemenations that the stream should be
+configured for optimal vectored IO performance, if such a
+feature has been implemented.
+
+It is *not* exclusive: the same stream may still be used for
+classic `InputStream` and `PositionedRead` API calls.
+Implementations SHOULD use the `random` read policy
+with these operations.
+
+#### <a name="read.policy.whole-file"></a> Read Policy `whole-file`
+
+
+This declares that the whole file is to be read end-to-end; the file system client is free to enable
+whatever strategies maximise performance for this. In particular, larger ranged reads/GETs can
+deliver high bandwidth by reducing socket/TLS setup costs and providing a connection long-lived
+enough for TCP flow control to determine the optimal download rate.
+
+Strategies can include:
+
+* Initiate an HTTP GET of the entire file in `openFile()` operation.
+* Prefech data in large blocks, possibly in parallel read operations.
+
+Applications which know that the entire file is to be read from an opened stream SHOULD declare this
+read policy.
+
+### <a name="openfile.length"></a> Option: `fs.option.openfile.length`
+
+Declare the length of a file.
+
+This can be used by clients to skip querying a remote store for the size
+of/existence of a file when opening it, similar to declaring a file status
+through the `withFileStatus()` option.
+
+If supported by a filesystem connector, this option MUST be interpreted as
+declaring the minimum length of the file:
+
+1. If the value is negative, the option SHALL be considered unset.
+2. It SHALL NOT be an error if the actual length of the file is greater than
+   this value.
+3. `read()`, `seek()` and positioned read calls MAY use a position across/beyond
+   this length but below the actual length of the file. Implementations MAY
+   raise `EOFExceptions` in such cases, or they MAY return data.
+
+If this option is used by the FileSystem implementation
+
+*Implementor's Notes*
+
+* A value of `fs.option.openfile.length` &lt; 0 MUST be rejected.
+* If a file status is supplied along with a value in `fs.opt.openfile.length`;
+  the file status values take precedence.
+
+### <a name="split.start"></a> Options: `fs.option.openfile.split.start` and `fs.option.openfile.split.end`
+
+Declare the start and end of the split when a file has been split for processing
+in pieces.
+
+1. If a value is negative, the option SHALL be considered unset.
+1. Filesystems MAY assume that the length of the file is greater than or equal
+   to the value of `fs.option.openfile.split.end`.
+1. And that they MAY raise an exception if the client application reads past the
+   value set in `fs.option.openfile.split.end`.
+1. The pair of options MAY be used to optimise the read plan, such as setting
+   the content range for GET requests, or using the split end as an implicit
+   declaration of the guaranteed minimum length of the file.
+1. If both options are set, and the split start is declared as greater than the
+   split end, then the split start SHOULD just be reset to zero, rather than
+   rejecting the operation.
+
+The split end value can provide a hint as to the end of the input stream. The
+split start can be used to optimize any initial read offset for filesystem
+clients.
+
+*Note for implementors: applications will read past the end of a split when they
+need to read to the end of a record/line which begins before the end of the
+split.
+
+Therefore clients MUST be allowed to `seek()`/`read()` past the length
+set in `fs.option.openfile.split.end` if the file is actually longer
+than that value.
+
+## <a name="s3a"></a> S3A-specific options
+
+The S3A Connector supports custom options for readahead and seek policy.
+
+| Name                                 | Type     | Meaning                                                     |
+|--------------------------------------|----------|-------------------------------------------------------------|
+| `fs.s3a.readahead.range`             | `long`   | readahead range in bytes                                    |
+| `fs.s3a.input.async.drain.threshold` | `long`   | threshold to switch to asynchronous draining of the stream  |
+| `fs.s3a.experimental.input.fadvise`  | `String` | seek policy. Superceded by `fs.option.openfile.read.policy` |
+
+If the option set contains a SQL statement in the `fs.s3a.select.sql` statement,
+then the file is opened as an S3 Select query.
+Consult the S3A documentation for more details.
+
+## <a name="abfs"></a> ABFS-specific options
+
+The ABFS Connector supports custom input stream options.
+
+| Name                              | Type      | Meaning                                            |
+|-----------------------------------|-----------|----------------------------------------------------|
+| `fs.azure.buffered.pread.disable` | `boolean` | disable caching on the positioned read operations. |
+
+
+Disables caching on data read through the [PositionedReadable](fsdatainputstream.html#PositionedReadable)
+APIs.
+
+Consult the ABFS Documentation for more details.
+
+## <a name="examples"></a> Examples
+
+#### Declaring seek policy and split limits when opening a file.
+
+Here is an example from a proof of
+concept `org.apache.parquet.hadoop.util.HadoopInputFile`
+reader which uses a (nullable) file status and a split start/end.
+
+The `FileStatus` value is always passed in -but if it is null, then the split
+end is used to declare the length of the file.
+
+```java
+protected SeekableInputStream newStream(Path path, FileStatus stat,
+     long splitStart, long splitEnd)
+     throws IOException {
+
+   FutureDataInputStreamBuilder builder = fs.openFile(path)
+   .opt("fs.option.openfile.read.policy", "vector, random")
+   .withFileStatus(stat);
+
+   builder.opt("fs.option.openfile.split.start", splitStart);
+   builder.opt("fs.option.openfile.split.end", splitEnd);
+   CompletableFuture<FSDataInputStream> streamF = builder.build();
+   return HadoopStreams.wrap(FutureIO.awaitFuture(streamF));
+}
+```
+
+As a result, whether driven directly by a file listing, or when opening a file
+from a query plan of `(path, splitStart, splitEnd)`, there is no need to probe
+the remote store for the length of the file. When working with remote object
+stores, this can save tens to hundreds of milliseconds, even if such a probe is
+done asynchronously.
+
+If both the file length and the split end is set, then the file length MUST be
+considered "more" authoritative, that is it really SHOULD be defining the file
+length. If the split end is set, the caller MAY ot read past it.
+
+The `CompressedSplitLineReader` can read past the end of a split if it is
+partway through processing a compressed record. That is: it assumes an
+incomplete record read means that the file length is greater than the split
+length, and that it MUST read the entirety of the partially read record. Other
+readers may behave similarly.
+
+Therefore
+
+1. File length as supplied in a `FileStatus` or in `fs.option.openfile.length`
+   SHALL set the strict upper limit on the length of a file
+2. The split end as set in `fs.option.openfile.split.end` MUST be viewed as a
+   hint, rather than the strict end of the file.
+
+### Opening a file with both standard and non-standard options
+
+Standard and non-standard options MAY be combined in the same `openFile()`
+operation.
+
+```java
+Future<FSDataInputStream> f = openFile(path)
+  .must("fs.option.openfile.read.policy", "random, adaptive")
+  .opt("fs.s3a.readahead.range", 1024 * 1024)
+  .build();
+
+FSDataInputStream is = f.get();
+```
+
+The option set in `must()` MUST be understood, or at least recognized and
+ignored by all filesystems. In this example, S3A-specific option MAY be
+ignored by all other filesystem clients.
+
+### Opening a file with older releases
+
+Not all hadoop releases recognize the `fs.option.openfile.read.policy` option.
+
+The option can be safely used in application code if it is added via the `opt()`
+builder argument, as it will be treated as an unknown optional key which can
+then be discarded.
+
+```java
+Future<FSDataInputStream> f = openFile(path)
+  .opt("fs.option.openfile.read.policy", "vector, random, adaptive")
+  .build();
+
+FSDataInputStream is = f.get();
+```
+
+*Note 1* if the option name is set by a reference to a constant in
+`org.apache.hadoop.fs.Options.OpenFileOptions`, then the program will not link
+against versions of Hadoop without the specific option. Therefore for resilient
+linking against older releases -use a copy of the value.
+
+*Note 2* as option validation is performed in the FileSystem connector,
+a third-party connector designed to work with multiple hadoop versions
+MAY NOT support the option.
+
+### Passing options in to MapReduce
+
+Hadoop MapReduce will automatically read MR Job Options with the prefixes
+`mapreduce.job.input.file.option.` and `mapreduce.job.input.file.must.`
+prefixes, and apply these values as `.opt()` and `must()` respectively, after
+remove the mapreduce-specific prefixes.
+
+This makes passing options in to MR jobs straightforward. For example, to
+declare that a job should read its data using random IO:
+
+```java
+JobConf jobConf = (JobConf) job.getConfiguration()
+jobConf.set(
+    "mapreduce.job.input.file.option.fs.option.openfile.read.policy",
+    "random");
+```
+
+### MapReduce input format propagating options
+
+An example of a record reader passing in options to the file it opens.
+
+```java
+  public void initialize(InputSplit genericSplit,
+                     TaskAttemptContext context) throws IOException {
+    FileSplit split = (FileSplit)genericSplit;
+    Configuration job = context.getConfiguration();
+    start = split.getStart();
+    end = start + split.getLength();
+    Path file = split.getPath();
+
+    // open the file and seek to the start of the split
+    FutureDataInputStreamBuilder builder =
+      file.getFileSystem(job).openFile(file);
+    // the start and end of the split may be used to build
+    // an input strategy.
+    builder.opt("fs.option.openfile.split.start", start);
+    builder.opt("fs.option.openfile.split.end", end);
+    FutureIO.propagateOptions(builder, job,
+        "mapreduce.job.input.file.option",
+        "mapreduce.job.input.file.must");
+
+    fileIn = FutureIO.awaitFuture(builder.build());
+    fileIn.seek(start)
+    /* Rest of the operation on the opened stream */
+  }
+```
+
+### `FileContext.openFile`
+
+From `org.apache.hadoop.fs.AvroFSInput`; a file is opened with sequential input.
+Because the file length has already been probed for, the length is passd down
+
+```java
+  public AvroFSInput(FileContext fc, Path p) throws IOException {
+    FileStatus status = fc.getFileStatus(p);
+    this.len = status.getLen();
+    this.stream = awaitFuture(fc.openFile(p)
+        .opt("fs.option.openfile.read.policy",
+            "sequential")
+        .opt("fs.option.openfile.length",
+            Long.toString(status.getLen()))
+        .build());
+    fc.open(p);
+  }
+```
+
+In this example, the length is passed down as a string (via `Long.toString()`)
+rather than directly as a long. This is to ensure that the input format will
+link against versions of $Hadoop which do not have the
+`opt(String, long)` and `must(String, long)` builder parameters. Similarly, the
+values are passed as optional, so that if unrecognized the application will
+still succeed.
+
+### Example: reading a whole file
+
+This is from `org.apache.hadoop.util.JsonSerialization`.
+
+Its `load(FileSystem, Path, FileStatus)` method
+* declares the whole file is to be read end to end.
+* passes down the file status
+
+```java
+public T load(FileSystem fs,
+        Path path,
+        status)
+        throws IOException {
+
+ try (FSDataInputStream dataInputStream =
+          awaitFuture(fs.openFile(path)
+              .opt("fs.option.openfile.read.policy", "whole-file")
+              .withFileStatus(status)
+              .build())) {
+   return fromJsonStream(dataInputStream);
+ } catch (JsonProcessingException e) {
+   throw new PathIOException(path.toString(),
+       "Failed to read JSON file " + e, e);
+ }
+}
+```
+
+*Note:* : in Hadoop 3.3.2 and earlier, the `withFileStatus(status)` call
+required a non-null parameter; this has since been relaxed.
+For maximum compatibility across versions, only invoke the method
+when the file status is known to be non-null.
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
index 64dda2df8c63c..59a93c5887a1f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
@@ -26,7 +26,7 @@ create a new file or open an existing file on `FileSystem` for write.
 ## Invariants
 
 The `FSDataOutputStreamBuilder` interface does not validate parameters
-and modify the state of `FileSystem` until [`build()`](#Builder.build) is
+and modify the state of `FileSystem` until `build()` is
 invoked.
 
 ## Implementation-agnostic parameters.
@@ -110,7 +110,7 @@ of `FileSystem`.
 #### Implementation Notes
 
 The concrete `FileSystem` and/or `FSDataOutputStreamBuilder` implementation
-MUST verify that implementation-agnostic parameters (i.e., "syncable") or
+MUST verify that implementation-agnostic parameters (i.e., "syncable`) or
 implementation-specific parameters (i.e., "foofs:cache")
 are supported. `FileSystem` will satisfy optional parameters (via `opt(key, ...)`)
 on best effort. If the mandatory parameters (via `must(key, ...)`) can not be satisfied
@@ -182,3 +182,58 @@ see `FileSystem#create(path, ...)` and `FileSystem#append()`.
     result = FSDataOutputStream
 
 The result is `FSDataOutputStream` to be used to write data to filesystem.
+
+
+## <a name="s3a"></a> S3A-specific options
+
+Here are the custom options which the S3A Connector supports.
+
+| Name                        | Type      | Meaning                                |
+|-----------------------------|-----------|----------------------------------------|
+| `fs.s3a.create.performance` | `boolean` | create a file with maximum performance |
+| `fs.s3a.create.header`      | `string`  | prefix for user supplied headers       |
+
+### `fs.s3a.create.performance`
+
+Prioritize file creation performance over safety checks for filesystem consistency.
+
+This:
+1. Skips the `LIST` call which makes sure a file is being created over a directory.
+   Risk: a file is created over a directory.
+1. Ignores the overwrite flag.
+1. Never issues a `DELETE` call to delete parent directory markers.
+
+It is possible to probe an S3A Filesystem instance for this capability through
+the `hasPathCapability(path, "fs.s3a.create.performance")` check.
+
+Creating files with this option over existing directories is likely
+to make S3A filesystem clients behave inconsistently.
+
+Operations optimized for directories (e.g. listing calls) are likely
+to see the directory tree not the file; operations optimized for
+files (`getFileStatus()`, `isFile()`) more likely to see the file.
+The exact form of the inconsistencies, and which operations/parameters
+trigger this are undefined and may change between even minor releases.
+
+Using this option is the equivalent of pressing and holding down the
+"Electronic Stability Control"
+button on a rear-wheel drive car for five seconds: the safety checks are off.
+Things wil be faster if the driver knew what they were doing.
+If they didn't, the fact they had held the button down will
+be used as evidence at the inquest as proof that they made a
+conscious decision to choose speed over safety and
+that the outcome was their own fault.
+
+Accordingly: *Use if and only if you are confident that the conditions are met.*
+
+### `fs.s3a.create.header` User-supplied header support
+
+Options with the prefix `fs.s3a.create.header.` will be added to to the
+S3 object metadata as "user defined metadata".
+This metadata is visible to all applications. It can also be retrieved through the
+FileSystem/FileContext `listXAttrs()` and `getXAttrs()` API calls with the prefix `header.`
+
+When an object is renamed, the metadata is propagated the copy created.
+
+It is possible to probe an S3A Filesystem instance for this capability through
+the `hasPathCapability(path, "fs.s3a.create.header")` check.
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
index a4aa136033a0c..e18f4c3bf4ab2 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
@@ -41,3 +41,4 @@ HDFS as these are commonly expected by Hadoop client applications.
 2. [Extending the specification and its tests](extending.html)
 1. [Uploading a file using Multiple Parts](multipartuploader.html)
 1. [IOStatistics](iostatistics.html)
+1. [openFile()](openfile.html).
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
index 37191a5b2a69a..76782b45409ad 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
@@ -30,8 +30,8 @@ are places where HDFS diverges from the expected behaviour of a POSIX
 filesystem.
 
 The bundled S3A FileSystem clients make Amazon's S3 Object Store ("blobstore")
-accessible through the FileSystem API. The Swift FileSystem driver provides similar
-functionality for the OpenStack Swift blobstore. The Azure WASB and ADL object
+accessible through the FileSystem API. 
+The Azure ABFS, WASB and ADL object
 storage FileSystems talks to Microsoft's Azure storage. All of these
 bind to object stores, which do have different behaviors, especially regarding
 consistency guarantees, and atomicity of operations.
@@ -314,10 +314,10 @@ child entries
 
 This specification refers to *Object Stores* in places, often using the
 term *Blobstore*. Hadoop does provide FileSystem client classes for some of these
-even though they violate many of the requirements. This is why, although
-Hadoop can read and write data in an object store, the two which Hadoop ships
-with direct support for &mdash; Amazon S3 and OpenStack Swift &mdash; cannot
-be used as direct replacements for HDFS.
+even though they violate many of the requirements.
+
+Consult the documentation for a specific store to determine its compatibility
+with specific applications and services.
 
 *What is an Object Store?*
 
@@ -343,7 +343,7 @@ stores pretend that they are a FileSystem, a FileSystem with the same
 features and operations as HDFS. This is &mdash;ultimately&mdash;a pretence:
 they have different characteristics and occasionally the illusion fails.
 
-1. **Consistency**. Object stores are generally *Eventually Consistent*: it
+1. **Consistency**. Object may be *Eventually Consistent*: it
 can take time for changes to objects &mdash;creation, deletion and updates&mdash;
 to become visible to all callers. Indeed, there is no guarantee a change is
 immediately visible to the client which just made the change. As an example,
@@ -447,10 +447,6 @@ Object stores have an even vaguer view of time, which can be summarized as
  * The timestamp is likely to be in UTC or the TZ of the object store. If the
    client is in a different timezone, the timestamp of objects may be ahead or
    behind that of the client.
- * Object stores with cached metadata databases (for example: AWS S3 with
-   an in-memory or a DynamoDB metadata store) may have timestamps generated
-   from the local system clock, rather than that of the service.
-   This is an optimization to avoid round-trip calls to the object stores.
  + A file's modification time is often the same as its creation time.
  + The `FileSystem.setTimes()` operation to set file timestamps *may* be ignored.
  * `FileSystem.chmod()` may update modification times (example: Azure `wasb://`).
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md
new file mode 100644
index 0000000000000..afb3245c5105f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/openfile.md
@@ -0,0 +1,122 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# `FileSystem.openFile()`/`FileContext.openFile()`
+
+This is a method provided by both FileSystem and FileContext for
+advanced file opening options and, where implemented,
+an asynchrounous/lazy opening of a file.
+
+Creates a builder to open a file, supporting options
+both standard and filesystem specific. The return
+value of the `build()` call is a `Future<FSDataInputStream>`,
+which must be waited on. The file opening may be
+asynchronous, and it may actually be postponed (including
+permission/existence checks) until reads are actually
+performed.
+
+This API call was added to `FileSystem` and `FileContext` in
+Hadoop 3.3.0; it was tuned in Hadoop 3.3.1 as follows.
+
+* Added `opt(key, long)` and `must(key, long)`.
+* Declared that `withFileStatus(null)` is allowed.
+* Declared that `withFileStatus(status)` only checks
+  the filename of the path, not the full path.
+  This is needed to support passthrough/mounted filesystems.
+* Added standard option keys.
+
+###  <a name="openfile_path_"></a> `FutureDataInputStreamBuilder openFile(Path path)`
+
+Creates a [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html)
+to construct a operation to open the file at `path` for reading.
+
+When `build()` is invoked on the returned `FutureDataInputStreamBuilder` instance,
+the builder parameters are verified and
+`FileSystem.openFileWithOptions(Path, OpenFileParameters)` or
+`AbstractFileSystem.openFileWithOptions(Path, OpenFileParameters)` invoked.
+
+These protected methods returns a `CompletableFuture<FSDataInputStream>`
+which, when its `get()` method is called, either returns an input
+stream of the contents of opened file, or raises an exception.
+
+The base implementation of the `FileSystem.openFileWithOptions(PathHandle, OpenFileParameters)`
+ultimately invokes `FileSystem.open(Path, int)`.
+
+Thus the chain `FileSystem.openFile(path).build().get()` has the same preconditions
+and postconditions as `FileSystem.open(Path p, int bufferSize)`
+
+However, there is one difference which implementations are free to
+take advantage of:
+
+The returned stream MAY implement a lazy open where file non-existence or
+access permission failures may not surface until the first `read()` of the
+actual data.
+
+This saves network IO on object stores.
+
+The `openFile()` operation MAY check the state of the filesystem during its
+invocation, but as the state of the filesystem may change between this call and
+the actual `build()` and `get()` operations, this file-specific
+preconditions (file exists, file is readable, etc) MUST NOT be checked here.
+
+FileSystem implementations which do not implement `open(Path, int)`
+MAY postpone raising an `UnsupportedOperationException` until either the
+`FutureDataInputStreamBuilder.build()` or the subsequent `get()` call,
+else they MAY fail fast in the `openFile()` call.
+
+Consult [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html) for details
+on how to use the builder, and for standard options which may be passed in.
+
+### <a name="openfile_pathhandle_"></a> `FutureDataInputStreamBuilder openFile(PathHandle)`
+
+Creates a [`FutureDataInputStreamBuilder`](fsdatainputstreambuilder.html)
+to construct a operation to open the file identified by the given `PathHandle` for reading.
+
+If implemented by a filesystem, the semantics of  [`openFile(Path)`](#openfile_path_)
+Thus the chain `openFile(pathhandle).build().get()` has the same preconditions and postconditions
+as `open(Pathhandle, int)`
+
+FileSystem implementations which do not implement `open(PathHandle handle, int bufferSize)`
+MAY postpone raising an `UnsupportedOperationException` until either the
+`FutureDataInputStreamBuilder.build()` or the subsequent `get()` call, else they MAY fail fast in
+the `openFile(PathHandle)` call.
+
+The base implementation raises this exception in the `build()` operation; other implementations
+SHOULD copy this.
+
+### Implementors notes
+
+The base implementation of `openFileWithOptions()` actually executes
+the `open(path)` operation synchronously, yet still returns the result
+or any failures in the `CompletableFuture<>`, so as to provide a consistent
+lifecycle across all filesystems.
+
+Any filesystem client where the time to open a file may be significant SHOULD
+execute it asynchronously by submitting the operation in some executor/thread
+pool. This is particularly recommended for object stores and other filesystems
+likely to be accessed over long-haul connections.
+
+Arbitrary filesystem-specific options MAY be supported; these MUST
+be prefixed with either the filesystem schema, e.g. `hdfs.`
+or in the `fs.SCHEMA` format as normal configuration settings `fs.hdfs`. The
+latter style allows the same configuration option to be used for both
+filesystem configuration and file-specific configuration.
+
+It SHOULD be possible to always open a file without specifying any options,
+so as to present a consistent model to users. However, an implementation MAY
+opt to require one or more mandatory options to be set.
+
+The returned stream may perform "lazy" evaluation of file access. This is
+relevant for object stores where the probes for existence are expensive, and,
+even with an asynchronous open, may be considered needless.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
index 8d0d4c4354f0b..1498d8db2e23b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
@@ -42,7 +42,7 @@ or `FSDataOutputStreamBuilder.build()`.
 These all return instances of `FSDataOutputStream`, through which data
 can be written through various `write()` methods.
 After a stream's `close()` method is called, all data written to the
-stream MUST BE persisted to the fileysystem and visible to oll other
+stream MUST BE persisted to the filesystem and visible to oll other
 clients attempting to read data from that path via `FileSystem.open()`.
 
 As well as operations to write the data, Hadoop's `OutputStream` implementations
@@ -126,7 +126,7 @@ consistent with the data at the filesystem.
 
 The output stream returned from a call of
  `FileSystem.append(path, buffersize, progress)` within a filesystem `FS`,
-can be modelled as a stream whose `buffer` is intialized to that of
+can be modelled as a stream whose `buffer` is initialized to that of
 the original file:
 
 ```python
@@ -184,7 +184,7 @@ This document covers the requirements of such implementations.
 
 HDFS's `FileSystem` implementation, `DistributedFileSystem`, returns an instance
 of `HdfsDataOutputStream`. This implementation has at least two behaviors
-which are not explicitly declared by the base Java implmentation
+which are not explicitly declared by the base Java implementation
 
 1. Writes are synchronized: more than one thread can write to the same
 output stream. This is a use pattern which HBase relies on.
@@ -648,7 +648,7 @@ in production.
 1. `OutputStream.write()` MAY persist the data, synchronously or asynchronously
 1. `OutputStream.flush()` flushes data to the destination. There
 are no strict persistence requirements.
-1. `Syncable.hflush()` synchronously sends all outstaning data to the destination
+1. `Syncable.hflush()` synchronously sends all outstanding data to the destination
 filesystem. After returning to the caller, the data MUST be visible to other readers,
 it MAY be durable. That is: it does not have to be persisted, merely guaranteed
 to be consistently visible to all clients attempting to open a new stream reading
@@ -678,7 +678,7 @@ public void hflush() throws IOException {
 ```
 
 This is perfectly acceptable as an implementation: the semantics of `hflush()`
-are satisifed.
+are satisfied.
 What is not acceptable is downgrading `hsync()` to `hflush()`, as the durability guarantee is no longer met.
 
 
@@ -863,7 +863,7 @@ local data as can be written to full checksummed blocks of data.
 That is, the hsync/hflush operations are not guaranteed to write all the pending
 data until the file is finally closed.
 
-For this reason, the local fileystem accessed via `file://` URLs
+For this reason, the local filesystem accessed via `file://` URLs
 does not support `Syncable` unless `setWriteChecksum(false)` was
 called on that FileSystem instance so as to disable checksum creation.
 After which, obviously, checksums are not generated for any file.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
index 4c6fa3ff0f6d4..53eb9870bc17d 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
@@ -66,55 +66,6 @@ Example:
       </property>
     </configuration>
 
-
-### swift://
-
-The OpenStack Swift login details must be defined in the file
-`/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml`.
-The standard hadoop-common `contract-test-options.xml` resource file cannot be
-used, as that file does not get included in `hadoop-common-test.jar`.
-
-
-In `/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml`
-the Swift bucket name must be defined in the property `fs.contract.test.fs.swift`,
-along with the login details for the specific Swift service provider in which the
-bucket is posted.
-
-    <configuration>
-      <property>
-        <name>fs.contract.test.fs.swift</name>
-        <value>swift://swiftbucket.rackspace/</value>
-      </property>
-
-      <property>
-        <name>fs.swift.service.rackspace.auth.url</name>
-        <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
-        <description>Rackspace US (multiregion)</description>
-      </property>
-
-      <property>
-        <name>fs.swift.service.rackspace.username</name>
-        <value>this-is-your-username</value>
-      </property>
-
-      <property>
-        <name>fs.swift.service.rackspace.region</name>
-        <value>DFW</value>
-      </property>
-
-      <property>
-        <name>fs.swift.service.rackspace.apikey</name>
-        <value>ab0bceyoursecretapikeyffef</value>
-      </property>
-
-    </configuration>
-
-1. Often the different public cloud Swift infrastructures exhibit different behaviors
-(authentication and throttling in particular). We recommand that testers create
-accounts on as many of these providers as possible and test against each of them.
-1. They can be slow, especially remotely. Remote links are also the most likely
-to make eventual-consistency behaviors visible, which is a mixed benefit.
-
 ## Testing a new filesystem
 
 The core of adding a new FileSystem to the contract tests is adding a
@@ -228,8 +179,6 @@ Passing all the FileSystem contract tests does not mean that a filesystem can be
 * Scalability: does it support files as large as HDFS, or as many in a single directory?
 * Durability: do files actually last -and how long for?
 
-Proof that this is is true is the fact that the Amazon S3 and OpenStack Swift object stores are eventually consistent object stores with non-atomic rename and delete operations. Single threaded test cases are unlikely to see some of the concurrency issues, while consistency is very often only visible in tests that span a datacenter.
-
 There are also some specific aspects of the use of the FileSystem API:
 
 * Compatibility with the `hadoop -fs` CLI.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md
index 03af8e5716e36..d8dd2daca09cc 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md
@@ -16,9 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 -->
-# Apache Hadoop Changelog
+# "Apache Hadoop" Changelog
 
-## Release 2.10.0 - Unreleased (as of 2018-09-02)
+## Release 2.10.0 - 2019-10-22
 
 ### INCOMPATIBLE CHANGES:
 
@@ -27,6 +27,8 @@
 | [HDFS-12883](https://issues.apache.org/jira/browse/HDFS-12883) | RBF: Document Router and State Store metrics |  Major | documentation | Yiqun Lin | Yiqun Lin |
 | [HDFS-12895](https://issues.apache.org/jira/browse/HDFS-12895) | RBF: Add ACL support for mount table |  Major | . | Yiqun Lin | Yiqun Lin |
 | [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | RBF: Use the ZooKeeper as the default State Store |  Minor | documentation | Yiqun Lin | Yiqun Lin |
+| [HADOOP-16055](https://issues.apache.org/jira/browse/HADOOP-16055) | Upgrade AWS SDK to 1.11.271 in branch-2 |  Blocker | fs/s3 | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16053](https://issues.apache.org/jira/browse/HADOOP-16053) | Backport HADOOP-14816 to branch-2 |  Major | build | Akira Ajisaka | Akira Ajisaka |
 
 
 ### IMPORTANT ISSUES:
@@ -42,6 +44,9 @@
 |:---- |:---- | :--- |:---- |:---- |:---- |
 | [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode |  Major | datanode, hdfs | Lukas Majercak | Lukas Majercak |
 | [HDFS-13553](https://issues.apache.org/jira/browse/HDFS-13553) | RBF: Support global quota |  Major | . | Íñigo Goiri | Yiqun Lin |
+| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping |  Major | common, security | Lukas Majercak | Lukas Majercak |
+| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs |  Major | . | Jonathan Hung | pralabhkumar |
+| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level |  Major | . | Jonathan Hung | Varun Saxena |
 
 
 ### IMPROVEMENTS:
@@ -53,10 +58,11 @@
 | [HADOOP-14960](https://issues.apache.org/jira/browse/HADOOP-14960) | Add GC time percentage monitor/alerter |  Major | . | Misha Dmitriev | Misha Dmitriev |
 | [HADOOP-15023](https://issues.apache.org/jira/browse/HADOOP-15023) | ValueQueue should also validate (lowWatermark \* numValues) \> 0 on construction |  Minor | . | Xiao Chen | Xiao Chen |
 | [YARN-6851](https://issues.apache.org/jira/browse/YARN-6851) | Capacity Scheduler: document configs for controlling # containers allowed to be allocated per node heartbeat |  Minor | . | Wei Yan | Wei Yan |
-| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService |  Major | log-aggregation | Jonathan Eagles | Jonathan Eagles |
+| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService |  Major | log-aggregation | Jonathan Turner Eagles | Jonathan Turner Eagles |
 | [HADOOP-15056](https://issues.apache.org/jira/browse/HADOOP-15056) | Fix TestUnbuffer#testUnbufferException failure |  Minor | test | Jack Bearden | Jack Bearden |
 | [HADOOP-15012](https://issues.apache.org/jira/browse/HADOOP-15012) | Add readahead, dropbehind, and unbuffer to StreamCapabilities |  Major | fs | John Zhuge | John Zhuge |
 | [HADOOP-15104](https://issues.apache.org/jira/browse/HADOOP-15104) | AliyunOSS: change the default value of max error retry |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level |  Major | capacityscheduler | Scott Brokaw | Zian Chen |
 | [YARN-7642](https://issues.apache.org/jira/browse/YARN-7642) | Add test case to verify context update after container promotion or demotion with or without auto update |  Minor | nodemanager | Weiwei Yang | Weiwei Yang |
 | [HADOOP-15111](https://issues.apache.org/jira/browse/HADOOP-15111) | AliyunOSS: backport HADOOP-14993 to branch-2 |  Major | fs/oss | Genmao Yu | Genmao Yu |
 | [HDFS-12818](https://issues.apache.org/jira/browse/HDFS-12818) | Support multiple storages in DataNodeCluster / SimulatedFSDataset |  Minor | datanode, test | Erik Krogen | Erik Krogen |
@@ -88,14 +94,16 @@
 | [HDFS-13492](https://issues.apache.org/jira/browse/HDFS-13492) | Limit httpfs binds to certain IP addresses in branch-2 |  Major | httpfs | Wei-Chiu Chuang | Wei-Chiu Chuang |
 | [HDFS-12981](https://issues.apache.org/jira/browse/HDFS-12981) | renameSnapshot a Non-Existent snapshot to itself should throw error |  Minor | hdfs | Sailesh Patel | Kitti Nanasi |
 | [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. |  Minor | . | Wei-Chiu Chuang | Gabor Bota |
-| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check |  Critical | webapp, yarn | Sunil Govindan | Sunil Govindan |
+| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster |  Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru |
+| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check |  Critical | webapp, yarn | Sunil G | Sunil G |
 | [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair |  Major | net | Nanda kumar | Nanda kumar |
 | [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover |  Critical | common | Karthik Palanisamy | Karthik Palanisamy |
 | [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem |  Major | ha, namenode | Erik Krogen | Chao Sun |
 | [HDFS-13644](https://issues.apache.org/jira/browse/HDFS-13644) | Backport HDFS-10376 to branch-2 |  Major | . | Yiqun Lin | Zsolt Venczel |
 | [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration |  Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
+| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
 | [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows |  Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak |
-| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading |  Major | . | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading |  Major | documentation | Kitti Nanasi | Kitti Nanasi |
 | [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable |  Minor | hdfs | Lantao Jin | Lantao Jin |
 | [YARN-8155](https://issues.apache.org/jira/browse/YARN-8155) | Improve ATSv2 client logging in RM and NM publisher |  Major | . | Rohith Sharma K S | Abhishek Modi |
 | [HDFS-13814](https://issues.apache.org/jira/browse/HDFS-13814) | Remove super user privilege requirement for NameNode.getServiceStatus |  Minor | namenode | Chao Sun | Chao Sun |
@@ -105,17 +113,84 @@
 | [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 |  Major | . | Rui Gao | Rui Gao |
 | [HDFS-13831](https://issues.apache.org/jira/browse/HDFS-13831) | Make block increment deletion number configurable |  Major | . | Yiqun Lin | Ryan Wu |
 | [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey |  Major | test | Robert Kanter | Robert Kanter |
+| [HADOOP-15547](https://issues.apache.org/jira/browse/HADOOP-15547) | WASB: improve listStatus performance |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
+| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation |  Trivial | documentation | David Mollitor | Hrishikesh Gadre |
+| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation |  Major | metrics | Sushil Ks | Sushil Ks |
+| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) |  Add JMX, conf and stacks menus to the datanode page |  Minor | datanode | fengchuang | fengchuang |
+| [HADOOP-15726](https://issues.apache.org/jira/browse/HADOOP-15726) | Create utility to limit frequency of log statements |  Major | common, util | Erik Krogen | Erik Krogen |
+| [YARN-7974](https://issues.apache.org/jira/browse/YARN-7974) | Allow updating application tracking url after registration |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics |  Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth |
+| [YARN-8896](https://issues.apache.org/jira/browse/YARN-8896) | Limit the maximum number of container assignments per heartbeat |  Major | . | Weiwei Yang | Zhankun Tang |
+| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 |  Major | . | PJ Fanning | Akira Ajisaka |
+| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log |  Major | resourcemanager | Jonathan Hung | Eric Payne |
+| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI |  Major | . | Jonathan Hung | Keqiu Hu |
+| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog |  Major | namenode | Kenneth Yang | Kenneth Yang |
+| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics |  Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian |
+| [HADOOP-15617](https://issues.apache.org/jira/browse/HADOOP-15617) | Node.js and npm package loading in the Dockerfile failing on branch-2 |  Major | build | Allen Wittenauer | Akhil PB |
+| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically |  Major | . | Eric Maynard | Ahmed Hussein |
+| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' |  Trivial | . | Szilard Nemeth | Charan Hebri |
+| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
+| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology |  Minor | datanode | HuangTao | HuangTao |
+| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 |  Major | ATSv2 | Sushil Ks | Sushil Ks |
+| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance |  Major | hdfs | Chao Sun | Chao Sun |
+| [HDFS-14205](https://issues.apache.org/jira/browse/HDFS-14205) | Backport HDFS-6440 to branch-2 |  Major | . | Chen Liang | Chao Sun |
+| [HDFS-14391](https://issues.apache.org/jira/browse/HDFS-14391) | Backport HDFS-9659 to branch-2 |  Minor | . | Chao Sun | Chao Sun |
+| [HDFS-14415](https://issues.apache.org/jira/browse/HDFS-14415) | Backport HDFS-13799 to branch-2 |  Trivial | . | Chao Sun | Chao Sun |
+| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml |  Minor | hdfs | puleya7 | puleya7 |
+| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. |  Major | nodemanager | Jonathan Hung | Jonathan Hung |
+| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() |  Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings |  Minor | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts |  Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
+| [HADOOP-16359](https://issues.apache.org/jira/browse/HADOOP-16359) | Bundle ZSTD native in branch-2 |  Major | native | Chao Sun | Chao Sun |
+| [HADOOP-15253](https://issues.apache.org/jira/browse/HADOOP-15253) | Should update maxQueueSize when refresh call queue |  Minor | . | Tao Jie | Tao Jie |
+| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer |  Minor | ipc | Christopher Gregorian | Erik Krogen |
+| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading |  Major | . | zhouyingchao | Lisheng Sun |
+| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. |  Major | . | Jinglun | Jinglun |
+| [HDFS-14697](https://issues.apache.org/jira/browse/HDFS-14697) | Backport HDFS-14513 to branch-2 |  Minor | namenode | Xiaoqiao He | Xiaoqiao He |
+| [YARN-8045](https://issues.apache.org/jira/browse/YARN-8045) | Reduce log output from container status calls |  Major | . | Shane Kumpf | Craig Condit |
+| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory  instead of df/du |  Major | datanode, performance | Lisheng Sun | Lisheng Sun |
+| [HDFS-14696](https://issues.apache.org/jira/browse/HDFS-14696) | Backport HDFS-11273 to branch-2 (Move TransferFsImage#doGetUrl function to a Util class) |  Major | . | Siyao Meng | Siyao Meng |
+| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
+| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-16459](https://issues.apache.org/jira/browse/HADOOP-16459) | Backport [HADOOP-16266] "Add more fine-grained processing time metrics to the RPC layer" to branch-2 |  Major | . | Erik Krogen | Erik Krogen |
+| [HDFS-14707](https://issues.apache.org/jira/browse/HDFS-14707) |  Add JAVA\_LIBRARY\_PATH to HTTPFS startup options in branch-2 |  Major | httpfs | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-14723](https://issues.apache.org/jira/browse/HDFS-14723) | Add helper method FSNamesystem#setBlockManagerForTesting() in branch-2 |  Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead |  Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena |
+| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round |  Major | capacity scheduler | Eric Payne | Manikandan R |
+| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename |  Major | . | Jinglun | Jinglun |
+| [HADOOP-16439](https://issues.apache.org/jira/browse/HADOOP-16439) | Upgrade bundled Tomcat in branch-2 |  Major | httpfs, kms | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics |  Major | . | Jonathan Hung | Shubham Gupta |
+| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [HADOOP-16530](https://issues.apache.org/jira/browse/HADOOP-16530) | Update xercesImpl in branch-2 |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9825](https://issues.apache.org/jira/browse/YARN-9825) | Changes for initializing placement rules with ResourceScheduler in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs |  Major | . | Jonathan Hung | Manoj Kumar |
+| [HDFS-14667](https://issues.apache.org/jira/browse/HDFS-14667) | Backport [HDFS-14403] "Cost-based FairCallQueue" to branch-2 |  Major | . | Erik Krogen | Erik Krogen |
 
 
 ### BUG FIXES:
 
 | JIRA | Summary | Priority | Component | Reporter | Contributor |
 |:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-6896](https://issues.apache.org/jira/browse/MAPREDUCE-6896) | Document wrong spelling in usage of MapredTestDriver tools. |  Major | documentation | LiXin Ge | LiXin Ge |
 | [HDFS-12052](https://issues.apache.org/jira/browse/HDFS-12052) | Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS |  Major | httpfs, webhdfs | Zoran Dimitrijevic | Zoran Dimitrijevic |
 | [HDFS-12318](https://issues.apache.org/jira/browse/HDFS-12318) | Fix IOException condition for openInfo in DFSInputStream |  Major | . | legend | legend |
 | [HDFS-12614](https://issues.apache.org/jira/browse/HDFS-12614) | FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider configured |  Major | . | Manoj Govindassamy | Manoj Govindassamy |
+| [YARN-7396](https://issues.apache.org/jira/browse/YARN-7396) | NPE when accessing container logs due to null dirsHandler |  Major | . | Jonathan Hung | Jonathan Hung |
 | [YARN-7370](https://issues.apache.org/jira/browse/YARN-7370) | Preemption properties should be refreshable |  Major | capacity scheduler, scheduler preemption | Eric Payne | Gergely Novák |
 | [YARN-7428](https://issues.apache.org/jira/browse/YARN-7428) | Add containerId to Localizer failed logs |  Minor | nodemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-7410](https://issues.apache.org/jira/browse/YARN-7410) | Cleanup FixedValueResource to avoid dependency to ResourceUtils |  Major | resourcemanager | Sunil G | Wangda Tan |
 | [HDFS-12783](https://issues.apache.org/jira/browse/HDFS-12783) | [branch-2] "dfsrouter" should use hdfsScript |  Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
 | [HDFS-12788](https://issues.apache.org/jira/browse/HDFS-12788) | Reset the upload button when file upload fails |  Critical | ui, webhdfs | Brahma Reddy Battula | Brahma Reddy Battula |
 | [YARN-7469](https://issues.apache.org/jira/browse/YARN-7469) | Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit |  Major | capacity scheduler, yarn | Eric Payne | Eric Payne |
@@ -128,36 +203,37 @@
 | [YARN-7363](https://issues.apache.org/jira/browse/YARN-7363) | ContainerLocalizer doesn't have a valid log4j config when using LinuxContainerExecutor |  Major | nodemanager | Yufei Gu | Yufei Gu |
 | [HDFS-12754](https://issues.apache.org/jira/browse/HDFS-12754) | Lease renewal can hit a deadlock |  Major | . | Kuhu Shukla | Kuhu Shukla |
 | [HDFS-12832](https://issues.apache.org/jira/browse/HDFS-12832) | INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit |  Critical | namenode | DENG FEI | Konstantin Shvachko |
-| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. |  Minor | . | Rushabh S Shah | Mikhail Erofeev |
+| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. |  Minor | . | Rushabh Shah | Mikhail Erofeev |
 | [HADOOP-15042](https://issues.apache.org/jira/browse/HADOOP-15042) | Azure PageBlobInputStream.skip() can return negative value when numberOfPagesRemaining is 0 |  Minor | fs/azure | Rajesh Balamohan | Rajesh Balamohan |
 | [HDFS-12638](https://issues.apache.org/jira/browse/HDFS-12638) | Delete copy-on-truncate block along with the original block, when deleting a file being truncated |  Blocker | hdfs | Jiandan Yang | Konstantin Shvachko |
 | [YARN-4813](https://issues.apache.org/jira/browse/YARN-4813) | TestRMWebServicesDelegationTokenAuthentication.testDoAs fails intermittently |  Major | resourcemanager | Daniel Templeton | Gergo Repas |
-| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events |  Major | mr-am | Jason Lowe | Peter Bacsko |
-| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer |  Major | nodemanager | Jason Lowe | Jim Brennan |
+| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events |  Major | mr-am | Jason Darrell Lowe | Peter Bacsko |
+| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer |  Major | nodemanager | Jason Darrell Lowe | Jim Brennan |
+| [YARN-7594](https://issues.apache.org/jira/browse/YARN-7594) | TestNMWebServices#testGetNMResourceInfo fails on trunk |  Major | nodemanager, webapp | Gergely Novák | Gergely Novák |
 | [YARN-5594](https://issues.apache.org/jira/browse/YARN-5594) | Handle old RMDelegationToken format when recovering RM |  Major | resourcemanager | Tatyana But | Robert Kanter |
 | [HADOOP-14985](https://issues.apache.org/jira/browse/HADOOP-14985) | Remove subversion related code from VersionInfoMojo.java |  Minor | build | Akira Ajisaka | Ajay Kumar |
 | [HDFS-11751](https://issues.apache.org/jira/browse/HDFS-11751) | DFSZKFailoverController daemon exits with wrong status code |  Major | auto-failover | Doris Gu | Bharat Viswanadham |
 | [HDFS-12889](https://issues.apache.org/jira/browse/HDFS-12889) | Router UI is missing robots.txt file |  Major | . | Bharat Viswanadham | Bharat Viswanadham |
 | [YARN-7607](https://issues.apache.org/jira/browse/YARN-7607) | Remove the trailing duplicated timestamp in container diagnostics message |  Minor | nodemanager | Weiwei Yang | Weiwei Yang |
-| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" |  Blocker | fs/oss | Chris Douglas | Sammi Chen |
+| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" |  Blocker | fs/oss | Christopher Douglas | Sammi Chen |
 | [YARN-7608](https://issues.apache.org/jira/browse/YARN-7608) | Incorrect sTarget column causing DataTable warning on RM application and scheduler web page |  Major | resourcemanager, webapp | Weiwei Yang | Gergely Novák |
 | [HDFS-12833](https://issues.apache.org/jira/browse/HDFS-12833) | Distcp : Update the usage of delete option for dependency with update and overwrite option |  Minor | distcp, hdfs | Harshakiran Reddy | usharani |
 | [YARN-7647](https://issues.apache.org/jira/browse/YARN-7647) | NM print inappropriate error log when node-labels is enabled |  Minor | . | Yang Wang | Yang Wang |
-| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers |  Major | namenode | Daryn Sharp | Rushabh S Shah |
-| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Lowe | Ajay Kumar |
-| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes |  Major | nodemanager | Jason Lowe | Jim Brennan |
-| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Lowe | Jim Brennan |
+| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers |  Major | namenode | Daryn Sharp | Rushabh Shah |
+| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Darrell Lowe | Ajay Kumar |
+| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes |  Major | nodemanager | Jason Darrell Lowe | Jim Brennan |
+| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Darrell Lowe | Jim Brennan |
 | [HADOOP-15123](https://issues.apache.org/jira/browse/HADOOP-15123) | KDiag tries to load krb5.conf from KRB5CCNAME instead of KRB5\_CONFIG |  Minor | security | Vipin Rathor | Vipin Rathor |
 | [YARN-7661](https://issues.apache.org/jira/browse/YARN-7661) | NodeManager metrics return wrong value after update node resource |  Major | . | Yang Wang | Yang Wang |
 | [HDFS-12347](https://issues.apache.org/jira/browse/HDFS-12347) | TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently |  Critical | test | Xiao Chen | Bharat Viswanadham |
 | [YARN-7662](https://issues.apache.org/jira/browse/YARN-7662) | [Atsv2] Define new set of configurations for reader and collectors to bind. |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 |  Major | . | Rohith Sharma K S | Sunil Govindan |
+| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 |  Major | . | Rohith Sharma K S | Sunil G |
 | [YARN-7542](https://issues.apache.org/jira/browse/YARN-7542) | Fix issue that causes some Running Opportunistic Containers to be recovered as PAUSED |  Major | . | Arun Suresh | Sampada Dehankar |
 | [HADOOP-15143](https://issues.apache.org/jira/browse/HADOOP-15143) | NPE due to Invalid KerberosTicket in UGI |  Major | . | Jitendra Nath Pandey | Mukul Kumar Singh |
-| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications |  Blocker | resourcemanager | Charan Hebri | Sunil Govindan |
+| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications |  Blocker | resourcemanager | Charan Hebri | Sunil G |
 | [MAPREDUCE-7028](https://issues.apache.org/jira/browse/MAPREDUCE-7028) | Concurrent task progress updates causing NPE in Application Master |  Blocker | mr-am | Gergo Repas | Gergo Repas |
 | [YARN-7619](https://issues.apache.org/jira/browse/YARN-7619) | Max AM Resource value in Capacity Scheduler UI has to be refreshed for every user |  Major | capacity scheduler, yarn | Eric Payne | Eric Payne |
-| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call |  Major | webapp | Sunil Govindan | Sunil Govindan |
+| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call |  Major | webapp | Sunil G | Sunil G |
 | [HDFS-12985](https://issues.apache.org/jira/browse/HDFS-12985) | NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted |  Major | hdfs | Manoj Govindassamy | Manoj Govindassamy |
 | [YARN-4227](https://issues.apache.org/jira/browse/YARN-4227) | Ignore expired containers from removed nodes in FairScheduler |  Critical | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
 | [YARN-7508](https://issues.apache.org/jira/browse/YARN-7508) | NPE in FiCaSchedulerApp when debug log enabled in async-scheduling mode |  Major | capacityscheduler | Tao Yang | Tao Yang |
@@ -185,27 +261,27 @@
 | [YARN-7698](https://issues.apache.org/jira/browse/YARN-7698) | A misleading variable's name in ApplicationAttemptEventDispatcher |  Minor | resourcemanager | Jinjiang Ling | Jinjiang Ling |
 | [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | Add an option to not disable short-circuit reads on failures |  Major | hdfs-client, performance | Andre Araujo | Xiao Chen |
 | [HDFS-13100](https://issues.apache.org/jira/browse/HDFS-13100) | Handle IllegalArgumentException when GETSERVERDEFAULTS is not implemented in webhdfs. |  Critical | hdfs, webhdfs | Yongjun Zhang | Yongjun Zhang |
-| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error |  Major | test | Jason Lowe | Botong Huang |
+| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error |  Major | test | Jason Darrell Lowe | Botong Huang |
 | [YARN-7801](https://issues.apache.org/jira/browse/YARN-7801) | AmFilterInitializer should addFilter after fill all parameters |  Critical | . | Sumana Sathish | Wangda Tan |
-| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch |  Major | . | Billie Rinaldi | Jason Lowe |
+| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch |  Major | . | Billie Rinaldi | Jason Darrell Lowe |
 | [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted |  Major | . | Yongjun Zhang | Yongjun Zhang |
 | [HDFS-12935](https://issues.apache.org/jira/browse/HDFS-12935) | Get ambiguous result for DFSAdmin command in HA mode when only one namenode is up |  Major | tools | Jianfei Jiang | Jianfei Jiang |
 | [HDFS-13120](https://issues.apache.org/jira/browse/HDFS-13120) | Snapshot diff could be corrupted after concat |  Major | namenode, snapshots | Xiaoyu Yao | Xiaoyu Yao |
-| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. |  Major | namenode | He Xiaoqiao | He Xiaoqiao |
+| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
 | [HDFS-8693](https://issues.apache.org/jira/browse/HDFS-8693) | refreshNamenodes does not support adding a new standby to a running DN |  Critical | datanode, ha | Jian Fang | Ajith S |
 | [MAPREDUCE-7052](https://issues.apache.org/jira/browse/MAPREDUCE-7052) | TestFixedLengthInputFormat#testFormatCompressedIn is flaky |  Major | client, test | Peter Bacsko | Peter Bacsko |
 | [HDFS-13112](https://issues.apache.org/jira/browse/HDFS-13112) | Token expiration edits may cause log corruption or deadlock |  Critical | namenode | Daryn Sharp | Daryn Sharp |
-| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump |  Major | . | Jason Lowe | Jason Lowe |
+| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump |  Major | . | Jason Darrell Lowe | Jason Darrell Lowe |
 | [HADOOP-15206](https://issues.apache.org/jira/browse/HADOOP-15206) | BZip2 drops and duplicates records when input split size is small |  Major | . | Aki Tanaka | Aki Tanaka |
 | [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request |  Minor | docs, documentation | Charan Hebri | Charan Hebri |
 | [YARN-7947](https://issues.apache.org/jira/browse/YARN-7947) | Capacity Scheduler intra-queue preemption can NPE for non-schedulable apps |  Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne |
 | [YARN-7945](https://issues.apache.org/jira/browse/YARN-7945) | Java Doc error in UnmanagedAMPoolManager for branch-2 |  Major | . | Rohith Sharma K S | Botong Huang |
 | [HADOOP-14903](https://issues.apache.org/jira/browse/HADOOP-14903) | Add json-smart explicitly to pom.xml |  Major | common | Ray Chiang | Ray Chiang |
 | [HADOOP-15236](https://issues.apache.org/jira/browse/HADOOP-15236) | Fix typo in RequestHedgingProxyProvider and RequestHedgingRMFailoverProxyProvider |  Trivial | documentation | Akira Ajisaka | Gabor Bota |
-| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails |  Critical | mrv2 | Gergely Novák | Gergely Novák |
+| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails |  Critical | harchive | Gergely Novák | Gergely Novák |
 | [HDFS-12781](https://issues.apache.org/jira/browse/HDFS-12781) | After Datanode down, In Namenode UI Datanode tab is throwing warning message. |  Major | datanode | Harshakiran Reddy | Brahma Reddy Battula |
 | [HDFS-12070](https://issues.apache.org/jira/browse/HDFS-12070) | Failed block recovery leaves files open indefinitely and at risk for data loss |  Major | . | Daryn Sharp | Kihwal Lee |
-| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 |  Major | test | Chris Douglas | Chris Douglas |
+| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 |  Major | test | Christopher Douglas | Christopher Douglas |
 | [HDFS-13194](https://issues.apache.org/jira/browse/HDFS-13194) | CachePool permissions incorrectly checked |  Major | . | Yiqun Lin | Jianfei Jiang |
 | [HADOOP-15276](https://issues.apache.org/jira/browse/HADOOP-15276) | branch-2 site not building after ADL troubleshooting doc added |  Major | documentation | Steve Loughran | Steve Loughran |
 | [YARN-7835](https://issues.apache.org/jira/browse/YARN-7835) | [Atsv2] Race condition in NM while publishing events if second attempt is launched on the same node |  Critical | . | Rohith Sharma K S | Rohith Sharma K S |
@@ -227,11 +303,11 @@
 | [YARN-8039](https://issues.apache.org/jira/browse/YARN-8039) | Clean up log dir configuration in TestLinuxContainerExecutorWithMocks.testStartLocalizer |  Minor | . | Miklos Szegedi | Miklos Szegedi |
 | [HDFS-13296](https://issues.apache.org/jira/browse/HDFS-13296) | GenericTestUtils generates paths with drive letter in Windows and fail webhdfs related test cases |  Major | . | Xiao Liang | Xiao Liang |
 | [HDFS-13268](https://issues.apache.org/jira/browse/HDFS-13268) | TestWebHdfsFileContextMainOperations fails on Windows |  Major | . | Íñigo Goiri | Xiao Liang |
-| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread |  Major | . | Jonathan Eagles | Jonathan Eagles |
+| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread |  Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
 | [YARN-7873](https://issues.apache.org/jira/browse/YARN-7873) | Revert YARN-6078 |  Blocker | . | Billie Rinaldi | Billie Rinaldi |
 | [HDFS-13195](https://issues.apache.org/jira/browse/HDFS-13195) | DataNode conf page  cannot display the current value after reconfig |  Minor | datanode | maobaolong | maobaolong |
 | [YARN-8063](https://issues.apache.org/jira/browse/YARN-8063) | DistributedShellTimelinePlugin wrongly check for entityId instead of entityType |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ |  Blocker | yarn | Sunil Govindan | Sunil Govindan |
+| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ |  Blocker | yarn | Sunil G | Sunil G |
 | [HADOOP-12862](https://issues.apache.org/jira/browse/HADOOP-12862) | LDAP Group Mapping over SSL can not specify trust store |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
 | [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop |  Major | . | Xiao Chen | Xiao Chen |
 | [HADOOP-15355](https://issues.apache.org/jira/browse/HADOOP-15355) | TestCommonConfigurationFields is broken by HADOOP-15312 |  Major | test | Konstantin Shvachko | LiXin Ge |
@@ -244,8 +320,8 @@
 | [HDFS-13427](https://issues.apache.org/jira/browse/HDFS-13427) | Fix the section titles of transparent encryption document |  Minor | documentation | Akira Ajisaka | Akira Ajisaka |
 | [YARN-7527](https://issues.apache.org/jira/browse/YARN-7527) | Over-allocate node resource in async-scheduling mode of CapacityScheduler |  Major | capacityscheduler | Tao Yang | Tao Yang |
 | [HDFS-7101](https://issues.apache.org/jira/browse/HDFS-7101) | Potential null dereference in DFSck#doWork() |  Minor | . | Ted Yu | skrho |
-| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger |  Major | nodemanager, resourcemanager | Jason Lowe | Jason Lowe |
-| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails |  Major | test | Jason Lowe | Jason Lowe |
+| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger |  Major | nodemanager, resourcemanager | Jason Darrell Lowe | Jason Darrell Lowe |
+| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails |  Major | test | Jason Darrell Lowe | Jason Darrell Lowe |
 | [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option |  Minor | . | Erik Krogen | Erik Krogen |
 | [YARN-8156](https://issues.apache.org/jira/browse/YARN-8156) | Increase the default value of yarn.timeline-service.app-collector.linger-period.ms |  Major | . | Rohith Sharma K S | Charan Hebri |
 | [YARN-8165](https://issues.apache.org/jira/browse/YARN-8165) | Incorrect queue name logging in AbstractContainerAllocator |  Trivial | capacityscheduler | Weiwei Yang | Weiwei Yang |
@@ -259,48 +335,50 @@
 | [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens |  Critical | . | Xiao Chen | Xiao Chen |
 | [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows |  Major | . | Xiao Liang | Xiao Liang |
 | [YARN-7598](https://issues.apache.org/jira/browse/YARN-7598) | Document how to use classpath isolation for aux-services in YARN |  Major | . | Xuan Gong | Xuan Gong |
-| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 |  Critical | tools/distcp | Rushabh S Shah | Jason Lowe |
+| [YARN-8183](https://issues.apache.org/jira/browse/YARN-8183) | Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap |  Critical | yarn | Sumana Sathish | Suma Shivaprasad |
+| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 |  Critical | tools/distcp | Rushabh Shah | Jason Darrell Lowe |
 | [MAPREDUCE-7042](https://issues.apache.org/jira/browse/MAPREDUCE-7042) | Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled |  Major | . | Yesha Vora | Xuan Gong |
 | [YARN-8205](https://issues.apache.org/jira/browse/YARN-8205) | Application State is not updated to ATS if AM launching is delayed. |  Critical | . | Sumana Sathish | Rohith Sharma K S |
 | [MAPREDUCE-7072](https://issues.apache.org/jira/browse/MAPREDUCE-7072) | mapred job -history prints duplicate counter in human output |  Major | client | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
-| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps |  Major | webapp | Sunil Govindan | Sunil Govindan |
+| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps |  Major | webapp | Sunil G | Sunil G |
 | [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
-| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal |  Major | . | Bibin A Chundatt | Bibin A Chundatt |
+| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-8025](https://issues.apache.org/jira/browse/YARN-8025) | UsersManangers#getComputedResourceLimitForActiveUsers throws NPE due to preComputedActiveUserLimit is empty |  Major | yarn | Jiandan Yang | Tao Yang |
 | [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens |  Major | resourcemanager | Hu Ziqian | Hu Ziqian |
 | [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows |  Major | . | Xiao Liang | Xiao Liang |
-| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication |  Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
 | [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) |  TestContainerSchedulerQueuing.testStartMultipleContainers failed |  Major | . | Miklos Szegedi | Jim Brennan |
 | [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows |  Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak |
 | [HDFS-13590](https://issues.apache.org/jira/browse/HDFS-13590) | Backport HDFS-12378 to branch-2 |  Major | datanode, hdfs, test | Lukas Majercak | Lukas Majercak |
-| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression |  Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
 | [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation |  Blocker | . | Kihwal Lee | Arpit Agarwal |
 | [HDFS-13601](https://issues.apache.org/jira/browse/HDFS-13601) | Optimize ByteString conversions in PBHelper |  Major | . | Andrew Wang | Andrew Wang |
 | [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
 | [YARN-8310](https://issues.apache.org/jira/browse/YARN-8310) | Handle old NMTokenIdentifier, AMRMTokenIdentifier, and ContainerTokenIdentifier formats |  Major | . | Robert Kanter | Robert Kanter |
 | [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync |  Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
 | [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows |  Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" |  Blocker | . | Rohith Sharma K S | Jason Lowe |
+| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" |  Blocker | . | Rohith Sharma K S | Jason Darrell Lowe |
 | [HDFS-13611](https://issues.apache.org/jira/browse/HDFS-13611) | Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient |  Major | . | Andrew Wang | Andrew Wang |
 | [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows |  Major | test | Xiao Liang | Xiao Liang |
 | [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 |  Critical | kms | Gabor Bota | Gabor Bota |
 | [HDFS-13626](https://issues.apache.org/jira/browse/HDFS-13626) | Fix incorrect username when deny the setOwner operation |  Minor | namenode | luhuachao | Zsolt Venczel |
 | [MAPREDUCE-7103](https://issues.apache.org/jira/browse/MAPREDUCE-7103) | Fix TestHistoryViewerPrinter on windows due to a mismatch line separator |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows |  Major | . | Giovanni Matteo Fumarola | Jason Lowe |
+| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows |  Major | . | Giovanni Matteo Fumarola | Jason Darrell Lowe |
 | [HDFS-13664](https://issues.apache.org/jira/browse/HDFS-13664) | Refactor ConfiguredFailoverProxyProvider to make inheritance easier |  Minor | hdfs-client | Chao Sun | Chao Sun |
 | [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 |  Major | . | Rohith Sharma K S | Íñigo Goiri |
 | [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
-| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime |  Critical | . | Wangda Tan | Thomas Marquardt |
+| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime |  Critical | . | Wangda Tan | Thomas Marqardt |
 | [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
 | [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
 | [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
 | [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
 | [HADOOP-15523](https://issues.apache.org/jira/browse/HADOOP-15523) | Shell command timeout given is in seconds whereas it is taken as millisec while scheduling |  Major | . | Bilwa S T | Bilwa S T |
 | [YARN-8444](https://issues.apache.org/jira/browse/YARN-8444) | NodeResourceMonitor crashes on bad swapFree value |  Major | . | Jim Brennan | Jim Brennan |
-| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui |  Major | webapp | Sunil Govindan | Sunil Govindan |
-| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection |  Blocker | . | Bibin A Chundatt | Bibin A Chundatt |
+| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui |  Major | webapp | Sunil G | Sunil G |
+| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection |  Blocker | . | Bibin Chundatt | Bibin Chundatt |
 | [YARN-8451](https://issues.apache.org/jira/browse/YARN-8451) | Multiple NM heartbeat thread created when a slow NM resync with RM |  Major | nodemanager | Botong Huang | Botong Huang |
 | [HADOOP-15548](https://issues.apache.org/jira/browse/HADOOP-15548) | Randomize local dirs |  Minor | . | Jim Brennan | Jim Brennan |
-| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state |  Major | nodemanager | Jason Lowe | Jason Lowe |
+| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state |  Major | nodemanager | Jason Darrell Lowe | Jason Darrell Lowe |
 | [HDFS-13729](https://issues.apache.org/jira/browse/HDFS-13729) | Fix broken links to RBF documentation |  Minor | documentation | jwhitter | Gabor Bota |
 | [YARN-8515](https://issues.apache.org/jira/browse/YARN-8515) | container-executor can crash with SIGPIPE after nodemanager restart |  Major | . | Jim Brennan | Jim Brennan |
 | [YARN-8421](https://issues.apache.org/jira/browse/YARN-8421) | when moving app, activeUsers is increased, even though app does not have outstanding request |  Major | . | kyungwan nam |  |
@@ -317,6 +395,131 @@
 | [YARN-8679](https://issues.apache.org/jira/browse/YARN-8679) | [ATSv2] If HBase cluster is down for long time, high chances that NM ContainerManager dispatcher get blocked |  Major | . | Rohith Sharma K S | Wangda Tan |
 | [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md |  Major | documentation | Daniel Templeton | Rui Gao |
 | [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing |  Major | . | lujie | lujie |
+| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns |  Major | ipc | Steve Loughran | Kihwal Lee |
+| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate |  Major | . | Anthony Hsu | Anthony Hsu |
+| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HDFS-12716](https://issues.apache.org/jira/browse/HDFS-12716) |  'dfs.datanode.failed.volumes.tolerated' to support minimum number of volumes to be available |  Major | datanode | usharani | Ranith Sardar |
+| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted |  Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang |
+| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full |  Major | namenode | zhangwei | Daryn Sharp |
+| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 |  Minor | datanode | Jianfei Jiang | Jianfei Jiang |
+| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data |  Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level |  Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi |
+| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js |  Minor | hdfs, ui | yanghuafeng | yanghuafeng |
+| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null |  Major | . | Lokesh Jain | Dinesh Chitlangia |
+| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently |  Major | mrv2 | Robert Joseph Evans | Jason Darrell Lowe |
+| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky |  Major | . | Íñigo Goiri | Ayush Saxena |
+| [HADOOP-15772](https://issues.apache.org/jira/browse/HADOOP-15772) | Remove the 'Path ... should be specified as a URI' warnings on startup |  Major | conf | Arpit Agarwal | Ayush Saxena |
+| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-8840](https://issues.apache.org/jira/browse/YARN-8840) | Add missing cleanupSSLConfig() call for TestTimelineClient test |  Minor | test, timelineclient | Aki Tanaka | Aki Tanaka |
+| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader |  Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles |
+| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long |  Blocker | . | Jason Darrell Lowe | Jason Darrell Lowe |
+| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-13768](https://issues.apache.org/jira/browse/HDFS-13768) |  Adding replicas to volume map makes DataNode start slowly |  Major | . | Yiqun Lin | Surendra Singh Lilhore |
+| [HADOOP-15818](https://issues.apache.org/jira/browse/HADOOP-15818) | Fix deprecated maven-surefire-plugin configuration in hadoop-kms module |  Minor | kms | Akira Ajisaka | Vidura Bhathiya Mudalige |
+| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI |  Major | . | Fei Hui | Fei Hui |
+| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance |  Blocker | . | Ben Lau | Jason Darrell Lowe |
+| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 |  Critical | tools/distcp | Ted Yu | Ted Yu |
+| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property |  Major | documentation | Jason Darrell Lowe | Suma Shivaprasad |
+| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager |  Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli |
+| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer |  Major | . | Jason Darrell Lowe | Jason Darrell Lowe |
+| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
+| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. |  Major | . | Sumana Sathish | Wangda Tan |
+| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart |  Major | hdfs-client | Erik Krogen | Erik Krogen |
+| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections |  Major | mrv2 | Peter Bacsko | Peter Bacsko |
+| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact |  Critical | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script |  Major | build | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 |  Minor | hdfs | Adam Antal | Ayush Saxena |
+| [YARN-7794](https://issues.apache.org/jira/browse/YARN-7794) | SLSRunner is not loading timeline service jars causing failure |  Blocker | scheduler-load-simulator | Sunil G | Yufei Gu |
+| [HADOOP-16008](https://issues.apache.org/jira/browse/HADOOP-16008) | Fix typo in CommandsManual.md |  Minor | . | Akira Ajisaka | Shweta |
+| [HADOOP-15973](https://issues.apache.org/jira/browse/HADOOP-15973) | Configuration: Included properties are not cached if resource is a stream |  Critical | . | Eric Payne | Eric Payne |
+| [YARN-9175](https://issues.apache.org/jira/browse/YARN-9175) | Null resources check in ResourceInfo for branch-3.0 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-15992](https://issues.apache.org/jira/browse/HADOOP-15992) | JSON License is included in the transitive dependency of aliyun-sdk-oss 3.0.0 |  Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 |  Blocker | fs/oss | wujinhu | wujinhu |
+| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp |  Major | resourcemanager, test | Ayush Saxena | Ayush Saxena |
+| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares |  Major | fairscheduler | liyakun | liyakun |
+| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds |  Major | security, test | Jason Darrell Lowe | Akira Ajisaka |
+| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon |  Major | ipc | Erik Krogen | Erik Krogen |
+| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration |  Critical | webapp | collinma | collinma |
+| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info |  Blocker | yarn | Jiandan Yang | Jiandan Yang |
+| [YARN-7088](https://issues.apache.org/jira/browse/YARN-7088) | Add application launch time to Resource Manager REST API |  Major | . | Abdullah Yousufi | Kanwaljeet Sachdev |
+| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO |  Major | . | Kihwal Lee | Zhaohui Xin |
+| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference |  Minor | documentation | Szilard Nemeth | Adam Antal |
+| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse |  Major | common | Daryn Sharp | Daryn Sharp |
+| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN |  Critical | datanode | star | star |
+| [YARN-5714](https://issues.apache.org/jira/browse/YARN-5714) | ContainerExecutor does not order environment map |  Trivial | nodemanager | Remi Catherinot | Remi Catherinot |
+| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed |  Major | ipc | Erik Krogen | Erik Krogen |
+| [HDFS-14399](https://issues.apache.org/jira/browse/HDFS-14399) | Backport HDFS-10536 to branch-2 |  Critical | . | Chao Sun | Chao Sun |
+| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes |  Minor | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-14414](https://issues.apache.org/jira/browse/HDFS-14414) | Clean up findbugs warning in branch-2 |  Major | . | Wei-Chiu Chuang | Dinesh Chitlangia |
+| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. |  Minor | documentation | Chris Nauroth | Masatake Iwasaki |
+| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby |  Major | namenode | yunjiong zhao | yunjiong zhao |
+| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap |  Blocker | . | xuzq | xuzq |
+| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type |  Minor | yarn | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished |  Major | namenode | Erik Krogen | Erik Krogen |
+| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml |  Major | . | Wanqiang Ji | Akira Ajisaka |
+| [HDFS-14514](https://issues.apache.org/jira/browse/HDFS-14514) | Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2 |  Major | encryption, hdfs, snapshots | Siyao Meng | Siyao Meng |
+| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) |  Major | . | Shen Yinjie | Ayush Saxena |
+| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics |  Major | ipc | Erik Krogen | Erik Krogen |
+| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() |  Major | . | Konstantin Shvachko | Shweta |
+| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read |  Major | hdfs-client | Zheng Hu | Zheng Hu |
+| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE |  Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu |
+| [HDFS-13770](https://issues.apache.org/jira/browse/HDFS-13770) | dfsadmin -report does not always decrease "missing blocks (with replication factor 1)" metrics when file is deleted |  Major | hdfs | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock |  Major | test | Kihwal Lee | Zsolt Venczel |
+| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. |  Major | . | Yicong Cai | Yicong Cai |
+| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) |  When evictableMmapped or evictable size is zero, do not throw NoSuchElementException |  Major | hdfs-client, performance | Zheng Hu | Lisheng Sun |
+| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java |  Trivial | . | hemanthboyina | hemanthboyina |
+| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor |  Critical | namenode | Daryn Sharp | Xiaoqiao He |
+| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY |  Major | hdfs | Jiandan Yang | Weiwei Yang |
+| [HADOOP-16386](https://issues.apache.org/jira/browse/HADOOP-16386) | FindBugs warning in branch-2: GlobalStorageStatistics defines non-transient non-serializable instance field map |  Major | fs | Wei-Chiu Chuang | Masatake Iwasaki |
+| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test |  Major | test | Masatake Iwasaki | Masatake Iwasaki |
+| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build |  Minor | test | Rushabh Shah | kevin su |
+| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC |  Major | ipc | Steve Loughran | Steve Loughran |
+| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs |  Major | common, security | Erik Krogen | Erik Krogen |
+| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission |  Major | . | Yicong Cai | Yicong Cai |
+| [HDFS-14672](https://issues.apache.org/jira/browse/HDFS-14672) | Backport HDFS-12703 to branch-2 |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever |  Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich |
+| [HDFS-14464](https://issues.apache.org/jira/browse/HDFS-14464) | Remove unnecessary log message from DFSInputStream |  Trivial | . | Kihwal Lee | Chao Sun |
+| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved |  Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan |
+| [HADOOP-15237](https://issues.apache.org/jira/browse/HADOOP-15237) | In KMS docs there should be one space between KMS\_LOG and NOTE |  Minor | kms | Snigdhanjali Mishra | Snigdhanjali Mishra |
+| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException |  Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira |
+| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. |  Major | . | Jinglun | Jinglun |
+| [HDFS-14724](https://issues.apache.org/jira/browse/HDFS-14724) | Fix JDK7 compatibility in branch-2 |  Blocker | . | Wei-Chiu Chuang | Chen Liang |
+| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS |  Major | webhdfs | Jing Wang | Masatake Iwasaki |
+| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot |  Major | snapshots | Yongjun Zhang | Shashikant Banerjee |
+| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage |  Major | rolling upgrades | Yicong Cai | Yicong Cai |
+| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
+| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-7585](https://issues.apache.org/jira/browse/YARN-7585) | NodeManager should go unhealthy when state store throws DBException |  Major | nodemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14726](https://issues.apache.org/jira/browse/HDFS-14726) | Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519 |  Blocker | journal-node | Chen Liang | Chen Liang |
+| [YARN-9806](https://issues.apache.org/jira/browse/YARN-9806) | TestNMSimulator#testNMSimulator fails in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted |  Critical | . | Rohith Sharma K S | Prabhu Joseph |
+| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log |  Minor | datanode, hdfs | qiang Liu | qiang Liu |
+| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark |  Major | common, kms | Yuval Degani | Yuval Degani |
+| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present |  Major | . | Ranith Sardar | Ranith Sardar |
+| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down |  Critical | . | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider |  Major | . | Shen Yinjie | Erik Krogen |
+| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x |  Blocker | . | Yuxuan Wang | Yuxuan Wang |
+| [HADOOP-16655](https://issues.apache.org/jira/browse/HADOOP-16655) | Change cipher suite when fetching tomcat tarball for branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
 
 
 ### TESTS:
@@ -351,22 +554,51 @@
 | [HADOOP-15532](https://issues.apache.org/jira/browse/HADOOP-15532) | TestBasicDiskValidator fails with NoSuchFileException |  Minor | . | Íñigo Goiri | Giovanni Matteo Fumarola |
 | [HDFS-13563](https://issues.apache.org/jira/browse/HDFS-13563) | TestDFSAdminWithHA times out on Windows |  Minor | . | Anbang Hu | Lukas Majercak |
 | [HDFS-13681](https://issues.apache.org/jira/browse/HDFS-13681) | Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows |  Major | test | Xiao Liang | Xiao Liang |
+| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 |  Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac |  Minor | libhdfs | John Zhuge | Akira Ajisaka |
 
 
 ### SUB-TASKS:
 
 | JIRA | Summary | Priority | Component | Reporter | Contributor |
 |:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-4081](https://issues.apache.org/jira/browse/YARN-4081) | Add support for multiple resource types in the Resource class |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-4172](https://issues.apache.org/jira/browse/YARN-4172) | Extend DominantResourceCalculator to account for all resources |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-4715](https://issues.apache.org/jira/browse/YARN-4715) | Add support to read resource types from a config file |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-4829](https://issues.apache.org/jira/browse/YARN-4829) | Add support for binary units |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-4830](https://issues.apache.org/jira/browse/YARN-4830) | Add support for resource types in the nodemanager |  Major | nodemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-5242](https://issues.apache.org/jira/browse/YARN-5242) | Update DominantResourceCalculator to consider all resource types in calculations |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-5586](https://issues.apache.org/jira/browse/YARN-5586) | Update the Resources class to consider all resource types |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-6761](https://issues.apache.org/jira/browse/YARN-6761) | Fix build for YARN-3926 branch |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-6786](https://issues.apache.org/jira/browse/YARN-6786) | ResourcePBImpl imports cleanup |  Trivial | resourcemanager | Daniel Templeton | Yeliang Cang |
+| [YARN-6788](https://issues.apache.org/jira/browse/YARN-6788) | Improve performance of resource profile branch |  Blocker | nodemanager, resourcemanager | Sunil G | Sunil G |
+| [YARN-6994](https://issues.apache.org/jira/browse/YARN-6994) | Remove last uses of Long from resource types code |  Minor | resourcemanager | Daniel Templeton | Daniel Templeton |
+| [YARN-6892](https://issues.apache.org/jira/browse/YARN-6892) | Improve API implementation in Resources and DominantResourceCalculator class |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
+| [YARN-6610](https://issues.apache.org/jira/browse/YARN-6610) | DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources |  Critical | resourcemanager | Daniel Templeton | Daniel Templeton |
+| [YARN-7030](https://issues.apache.org/jira/browse/YARN-7030) | Performance optimizations in Resource and ResourceUtils class |  Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
+| [YARN-7042](https://issues.apache.org/jira/browse/YARN-7042) | Clean up unit tests after YARN-6610 |  Major | test | Daniel Templeton | Daniel Templeton |
+| [YARN-6789](https://issues.apache.org/jira/browse/YARN-6789) | Add Client API to get all supported resource types from RM |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
+| [YARN-6781](https://issues.apache.org/jira/browse/YARN-6781) | ResourceUtils#initializeResourcesMap takes an unnecessary Map parameter |  Minor | resourcemanager | Daniel Templeton | Yu-Tang Lin |
+| [YARN-7067](https://issues.apache.org/jira/browse/YARN-7067) | Optimize ResourceType information display in UI |  Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
+| [YARN-7039](https://issues.apache.org/jira/browse/YARN-7039) | Fix javac and javadoc errors in YARN-3926 branch |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
+| [YARN-7093](https://issues.apache.org/jira/browse/YARN-7093) | Improve log message in ResourceUtils |  Trivial | nodemanager, resourcemanager | Sunil G | Sunil G |
+| [YARN-6933](https://issues.apache.org/jira/browse/YARN-6933) | ResourceUtils.DISALLOWED\_NAMES check is duplicated |  Major | resourcemanager | Daniel Templeton | Manikandan R |
+| [YARN-7137](https://issues.apache.org/jira/browse/YARN-7137) | Move newly added APIs to unstable in YARN-3926 branch |  Blocker | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
 | [HADOOP-14799](https://issues.apache.org/jira/browse/HADOOP-14799) | Update nimbus-jose-jwt to 4.41.1 |  Major | . | Ray Chiang | Ray Chiang |
+| [YARN-7345](https://issues.apache.org/jira/browse/YARN-7345) | GPU Isolation: Incorrect minor device numbers written to devices.deny file |  Major | . | Jonathan Hung | Jonathan Hung |
 | [HADOOP-14997](https://issues.apache.org/jira/browse/HADOOP-14997) |  Add hadoop-aliyun as dependency of hadoop-cloud-storage |  Minor | fs/oss | Genmao Yu | Genmao Yu |
+| [YARN-7143](https://issues.apache.org/jira/browse/YARN-7143) | FileNotFound handling in ResourceUtils is inconsistent |  Major | resourcemanager | Daniel Templeton | Daniel Templeton |
+| [YARN-6909](https://issues.apache.org/jira/browse/YARN-6909) | Use LightWeightedResource when number of resource types more than two |  Critical | resourcemanager | Daniel Templeton | Sunil G |
 | [HDFS-12801](https://issues.apache.org/jira/browse/HDFS-12801) | RBF: Set MountTableResolver as default file resolver |  Minor | . | Íñigo Goiri | Íñigo Goiri |
 | [YARN-7430](https://issues.apache.org/jira/browse/YARN-7430) | Enable user re-mapping for Docker containers by default |  Blocker | security, yarn | Eric Yang | Eric Yang |
 | [HADOOP-15024](https://issues.apache.org/jira/browse/HADOOP-15024) | AliyunOSS: support user agent configuration and include that & Hadoop version information to oss server |  Major | fs, fs/oss | Sammi Chen | Sammi Chen |
 | [HDFS-12858](https://issues.apache.org/jira/browse/HDFS-12858) | RBF: Add router admin commands usage in HDFS commands reference doc |  Minor | documentation | Yiqun Lin | Yiqun Lin |
 | [HDFS-12835](https://issues.apache.org/jira/browse/HDFS-12835) | RBF: Fix Javadoc parameter errors |  Minor | . | Wei Yan | Wei Yan |
-| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. |  Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah |
+| [YARN-7573](https://issues.apache.org/jira/browse/YARN-7573) | Gpu Information page could be empty for nodes without GPU |  Major | webapp, yarn-ui-v2 | Sunil G | Sunil G |
+| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. |  Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah |
 | [YARN-6704](https://issues.apache.org/jira/browse/YARN-6704) | Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService |  Major | . | Botong Huang | Botong Huang |
 | [HDFS-12875](https://issues.apache.org/jira/browse/HDFS-12875) | RBF: Complete logic for -readonly option of dfsrouteradmin add command |  Major | . | Yiqun Lin | Íñigo Goiri |
+| [YARN-7383](https://issues.apache.org/jira/browse/YARN-7383) | Node resource is not parsed correctly for resource names containing dot |  Major | nodemanager, resourcemanager | Jonathan Hung | Gergely Novák |
 | [YARN-7630](https://issues.apache.org/jira/browse/YARN-7630) | Fix AMRMToken rollover handling in AMRMProxy |  Minor | . | Botong Huang | Botong Huang |
 | [HDFS-12937](https://issues.apache.org/jira/browse/HDFS-12937) | RBF: Add more unit tests for router admin commands |  Major | test | Yiqun Lin | Yiqun Lin |
 | [YARN-7032](https://issues.apache.org/jira/browse/YARN-7032) | [ATSv2] NPE while starting hbase co-processor when HBase authorization is enabled. |  Critical | . | Rohith Sharma K S | Rohith Sharma K S |
@@ -381,11 +613,14 @@
 | [HDFS-12772](https://issues.apache.org/jira/browse/HDFS-12772) | RBF: Federation Router State State Store internal API |  Major | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13042](https://issues.apache.org/jira/browse/HDFS-13042) | RBF: Heartbeat Router State |  Major | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13049](https://issues.apache.org/jira/browse/HDFS-13049) | RBF: Inconsistent Router OPTS config in branch-2 and branch-3 |  Minor | . | Wei Yan | Wei Yan |
-| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. |  Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah |
+| [YARN-7817](https://issues.apache.org/jira/browse/YARN-7817) | Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. |  Major | . | Sumana Sathish | Sunil G |
+| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. |  Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah |
 | [HDFS-13044](https://issues.apache.org/jira/browse/HDFS-13044) | RBF: Add a safe mode for the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13043](https://issues.apache.org/jira/browse/HDFS-13043) | RBF: Expose the state of the Routers in the federation |  Major | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode |  Major | . | Íñigo Goiri | Yiqun Lin |
+| [YARN-7860](https://issues.apache.org/jira/browse/YARN-7860) | Fix UT failure TestRMWebServiceAppsNodelabel#testAppsRunning |  Major | . | Weiwei Yang | Sunil G |
 | [HDFS-13119](https://issues.apache.org/jira/browse/HDFS-13119) | RBF: Manage unavailable clusters |  Major | . | Íñigo Goiri | Yiqun Lin |
+| [YARN-7223](https://issues.apache.org/jira/browse/YARN-7223) | Document GPU isolation feature |  Blocker | . | Wangda Tan | Wangda Tan |
 | [HDFS-13187](https://issues.apache.org/jira/browse/HDFS-13187) | RBF: Fix Routers information shown in the web UI |  Minor | . | Wei Yan | Wei Yan |
 | [HDFS-13184](https://issues.apache.org/jira/browse/HDFS-13184) | RBF: Improve the unit test TestRouterRPCClientRetries |  Minor | test | Yiqun Lin | Yiqun Lin |
 | [HDFS-13199](https://issues.apache.org/jira/browse/HDFS-13199) | RBF: Fix the hdfs router page missing label icon issue |  Major | federation, hdfs | maobaolong | maobaolong |
@@ -396,7 +631,7 @@
 | [HADOOP-15267](https://issues.apache.org/jira/browse/HADOOP-15267) | S3A multipart upload fails when SSE-C encryption is enabled |  Critical | fs/s3 | Anis Elleuch | Anis Elleuch |
 | [HDFS-13230](https://issues.apache.org/jira/browse/HDFS-13230) | RBF: ConnectionManager's cleanup task will compare each pool's own active conns with its total conns |  Minor | . | Wei Yan | Chao Sun |
 | [HDFS-13233](https://issues.apache.org/jira/browse/HDFS-13233) | RBF: MountTableResolver doesn't return the correct mount point of the given path |  Major | hdfs | wangzhiyuan | wangzhiyuan |
-| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue |  Major | federation, hdfs | Weiwei Wu | Weiwei Wu |
+| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue |  Major | federation, hdfs | Wu Weiwei | Wu Weiwei |
 | [HDFS-13232](https://issues.apache.org/jira/browse/HDFS-13232) | RBF: ConnectionPool should return first usable connection |  Minor | . | Wei Yan | Ekanth Sethuramalingam |
 | [HDFS-13240](https://issues.apache.org/jira/browse/HDFS-13240) | RBF: Update some inaccurate document descriptions |  Minor | . | Yiqun Lin | Yiqun Lin |
 | [HDFS-11399](https://issues.apache.org/jira/browse/HDFS-11399) | Many tests fails in Windows due to injecting disk failures |  Major | . | Yiqun Lin | Yiqun Lin |
@@ -407,6 +642,7 @@
 | [HDFS-12773](https://issues.apache.org/jira/browse/HDFS-12773) | RBF: Improve State Store FS implementation |  Major | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13198](https://issues.apache.org/jira/browse/HDFS-13198) | RBF: RouterHeartbeatService throws out CachedStateStore related exceptions when starting router |  Minor | . | Wei Yan | Wei Yan |
 | [HDFS-13224](https://issues.apache.org/jira/browse/HDFS-13224) | RBF: Resolvers to support mount points across multiple subclusters |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) |  Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
 | [HADOOP-15262](https://issues.apache.org/jira/browse/HADOOP-15262) | AliyunOSS: move files under a directory in parallel when rename a directory |  Major | fs/oss | wujinhu | wujinhu |
 | [HDFS-13215](https://issues.apache.org/jira/browse/HDFS-13215) | RBF: Move Router to its own module |  Major | . | Íñigo Goiri | Wei Yan |
 | [HDFS-13307](https://issues.apache.org/jira/browse/HDFS-13307) | RBF: Improve the use of setQuota command |  Major | . | liuhongtong | liuhongtong |
@@ -423,16 +659,17 @@
 | [HDFS-13347](https://issues.apache.org/jira/browse/HDFS-13347) | RBF: Cache datanode reports |  Minor | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13289](https://issues.apache.org/jira/browse/HDFS-13289) | RBF: TestConnectionManager#testCleanup() test case need correction |  Minor | . | Dibyendu Karmakar | Dibyendu Karmakar |
 | [HDFS-13364](https://issues.apache.org/jira/browse/HDFS-13364) | RBF: Support NamenodeProtocol in the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-14651](https://issues.apache.org/jira/browse/HADOOP-14651) | Update okhttp version to 2.7.5 |  Major | fs/adl | Ray Chiang | Ray Chiang |
 | [YARN-6936](https://issues.apache.org/jira/browse/YARN-6936) | [Atsv2] Retrospect storing entities into sub application table from client perspective |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
 | [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
 | [YARN-8107](https://issues.apache.org/jira/browse/YARN-8107) | Give an informative message when incorrect format is used in ATSv2 filter attributes |  Major | ATSv2 | Charan Hebri | Rohith Sharma K S |
 | [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit |  Major | . | Botong Huang | Botong Huang |
 | [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix  java doc for StateStoreFileSystemImpl |  Minor | hdfs | Yiran Wu | Yiran Wu |
-| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit |  Major | . | Weiwei Wu | Yiqun Lin |
+| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit |  Major | . | Wu Weiwei | Yiqun Lin |
 | [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters |  Minor | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism |  Minor | . | Íñigo Goiri | Íñigo Goiri |
 | [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster |  Minor | . | Wei Yan | Íñigo Goiri |
-| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java |  Trivial | federation | BELUGA BEHR | BELUGA BEHR |
+| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java |  Trivial | federation | David Mollitor | David Mollitor |
 | [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism |  Major | fs/oss | Genmao Yu | Genmao Yu |
 | [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user |  Major | . | Shane Kumpf | Shane Kumpf |
 | [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace |  Major | . | Yiqun Lin | Yiqun Lin |
@@ -456,12 +693,14 @@
 | [YARN-8130](https://issues.apache.org/jira/browse/YARN-8130) | Race condition when container events are published for KILLED applications |  Major | ATSv2 | Charan Hebri | Rohith Sharma K S |
 | [YARN-7900](https://issues.apache.org/jira/browse/YARN-7900) | [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor |  Major | . | Botong Huang | Botong Huang |
 | [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows |  Minor | . | Anbang Hu | Anbang Hu |
 | [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE |  Critical | federation | CR Hota | CR Hota |
 | [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. |  Major | scheduler | Wangda Tan | Eric Payne |
 | [HADOOP-15506](https://issues.apache.org/jira/browse/HADOOP-15506) | Upgrade Azure Storage Sdk version to 7.0.0 and update corresponding code blocks |  Minor | fs/azure | Esfandiar Manii | Esfandiar Manii |
 | [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
 | [HADOOP-15533](https://issues.apache.org/jira/browse/HADOOP-15533) | Make WASB listStatus messages consistent |  Trivial | fs/azure | Esfandiar Manii | Esfandiar Manii |
 | [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows |  Minor | test | Xiao Liang | Xiao Liang |
+| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters |  Minor | amrmproxy, federation | Botong Huang | Botong Huang |
 | [HDFS-13528](https://issues.apache.org/jira/browse/HDFS-13528) | RBF: If a directory exceeds quota limit then quota usage is not refreshed for other mount entries |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
 | [HDFS-13710](https://issues.apache.org/jira/browse/HDFS-13710) | RBF:  setQuota and getQuotaUsage should check the dfs.federation.router.quota.enable |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
 | [HDFS-13726](https://issues.apache.org/jira/browse/HDFS-13726) | RBF: Fix RBF configuration links |  Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
@@ -471,8 +710,58 @@
 | [HDFS-13583](https://issues.apache.org/jira/browse/HDFS-13583) | RBF: Router admin clrQuota is not synchronized with nameservice |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
 | [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
 | [YARN-8129](https://issues.apache.org/jira/browse/YARN-8129) | Improve error message for invalid value in fields attribute |  Minor | ATSv2 | Charan Hebri | Abhishek Modi |
+| [YARN-8581](https://issues.apache.org/jira/browse/YARN-8581) | [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy |  Major | amrmproxy, federation | Botong Huang | Botong Huang |
+| [YARN-8673](https://issues.apache.org/jira/browse/YARN-8673) | [AMRMProxy] More robust responseId resync after an YarnRM master slave switch |  Major | amrmproxy | Botong Huang | Botong Huang |
 | [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers |  Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
 | [HDFS-13634](https://issues.apache.org/jira/browse/HDFS-13634) | RBF: Configurable value in xml for async connection request queue size. |  Major | federation | CR Hota | CR Hota |
+| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows |  Major | . | Botong Huang | Botong Huang |
+| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-15748](https://issues.apache.org/jira/browse/HADOOP-15748) | S3 listing inconsistency can raise NPE in globber |  Major | fs | Steve Loughran | Steve Loughran |
+| [YARN-8696](https://issues.apache.org/jira/browse/YARN-8696) | [AMRMProxy] FederationInterceptor upgrade: home sub-cluster heartbeat async |  Major | nodemanager | Botong Huang | Botong Huang |
+| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module |  Major | . | Íñigo Goiri | Chao Sun |
+| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor |  Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang |
+| [YARN-6989](https://issues.apache.org/jira/browse/YARN-6989) | Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way |  Major | timelineserver | Vrushali C | Abhishek Modi |
+| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads |  Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi |
+| [HADOOP-15837](https://issues.apache.org/jira/browse/HADOOP-15837) | DynamoDB table Update can fail S3A FS init |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream |  Critical | . | wujinhu | wujinhu |
+| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client |  Major | amrmproxy, federation | Botong Huang | Botong Huang |
+| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor |  Minor | . | Bibin Chundatt | Bilwa S T |
+| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code |  Major | documentation | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9182](https://issues.apache.org/jira/browse/YARN-9182) | Backport YARN-6445 resource profile performance improvements to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9181](https://issues.apache.org/jira/browse/YARN-9181) | Backport YARN-6232 for generic resource type usage to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9177](https://issues.apache.org/jira/browse/YARN-9177) | Use resource map for app metrics in TestCombinedSystemMetricsPublisher for branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9188](https://issues.apache.org/jira/browse/YARN-9188) | Port YARN-7136 to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9187](https://issues.apache.org/jira/browse/YARN-9187) | Backport YARN-6852 for GPU-specific native changes to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9180](https://issues.apache.org/jira/browse/YARN-9180) | Port YARN-7033 NM recovery of assigned resources to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9280](https://issues.apache.org/jira/browse/YARN-9280) | Backport YARN-6620 to YARN-8200/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9174](https://issues.apache.org/jira/browse/YARN-9174) | Backport YARN-7224 for refactoring of GpuDevice class |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9289](https://issues.apache.org/jira/browse/YARN-9289) | Backport YARN-7330 for GPU in UI to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext |  Major | hdfs | Shweta | Shweta |
+| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 |  Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli |
+| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF |  Blocker | fs/s3 | Dave Christianson | Steve Loughran |
+| [YARN-9397](https://issues.apache.org/jira/browse/YARN-9397) | Fix empty NMResourceInfo object test failures in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9271](https://issues.apache.org/jira/browse/YARN-9271) | Backport YARN-6927 for resource type support in MapReduce |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9291](https://issues.apache.org/jira/browse/YARN-9291) | Backport YARN-7637 to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9409](https://issues.apache.org/jira/browse/YARN-9409) | Port resource type changes from YARN-7237 to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9272](https://issues.apache.org/jira/browse/YARN-9272) | Backport YARN-7738 for refreshing max allocation for multiple resource types |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16205](https://issues.apache.org/jira/browse/HADOOP-16205) | Backporting ABFS driver from trunk to branch 2.0 |  Major | fs/azure | Esfandiar Manii | Yuan Gao |
+| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom |  Major | fs/azure | Da Zhou | Da Zhou |
+| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS |  Major | fs, webhdfs | Erik Krogen | Chao Sun |
+| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore |  Major | restapi | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14771](https://issues.apache.org/jira/browse/HDFS-14771) | Backport HDFS-14617 to branch-2 (Improve fsimage load time by writing sub-sections to the fsimage index) |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. |  Major | hdfs | Konstantin Shvachko | Chen Liang |
+| [HADOOP-16630](https://issues.apache.org/jira/browse/HADOOP-16630) | Backport HADOOP-16548 - "ABFS: Config to enable/disable flush operation" to branch-2 |  Minor | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HADOOP-16631](https://issues.apache.org/jira/browse/HADOOP-16631) | Backport HADOOP-16578 - "ABFS: fileSystemExists() should not call container level apis" to Branch-2 |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode |  Major | . | Konstantin Shvachko | Erik Krogen |
 
 
 ### OTHER:
@@ -482,3 +771,18 @@
 | [HADOOP-15149](https://issues.apache.org/jira/browse/HADOOP-15149) | CryptoOutputStream should implement StreamCapabilities |  Major | fs | Mike Drob | Xiao Chen |
 | [HADOOP-15177](https://issues.apache.org/jira/browse/HADOOP-15177) | Update the release year to 2018 |  Blocker | build | Akira Ajisaka | Bharat Viswanadham |
 | [YARN-8412](https://issues.apache.org/jira/browse/YARN-8412) | Move ResourceRequest.clone logic everywhere into a proper API |  Minor | . | Botong Huang | Botong Huang |
+| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc |  Minor | documentation, webhdfs | Siyao Meng | Siyao Meng |
+| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS |  Major | documentation | Christopher Douglas | Masatake Iwasaki |
+| [HADOOP-15711](https://issues.apache.org/jira/browse/HADOOP-15711) | Move branch-2 precommit/nightly test builds to java 8 |  Critical | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14510](https://issues.apache.org/jira/browse/HDFS-14510) | Backport HDFS-13087 to branch-2 (Snapshotted encryption zone information should be immutable) |  Major | encryption, snapshots | Wei-Chiu Chuang | Siyao Meng |
+| [HDFS-14585](https://issues.apache.org/jira/browse/HDFS-14585) | Backport HDFS-8901 Use ByteBuffer in DFSInputStream#read to branch2.9 |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14483](https://issues.apache.org/jira/browse/HDFS-14483) | Backport HDFS-14111,HDFS-3246 ByteBuffer pread interface to branch-2.9 |  Major | . | Zheng Hu | Lisheng Sun |
+| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14725](https://issues.apache.org/jira/browse/HDFS-14725) | Backport HDFS-12914 to branch-2 (Block report leases cause missing blocks until next report) |  Major | namenode | Wei-Chiu Chuang | Xiaoqiao He |
+| [YARN-8200](https://issues.apache.org/jira/browse/YARN-8200) | Backport resource types/GPU features to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 |  Major | . | Wei-Chiu Chuang | YiSheng Lien |
+| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16544](https://issues.apache.org/jira/browse/HADOOP-16544) | update io.netty in branch-2 |  Major | . | Wei-Chiu Chuang | Masatake Iwasaki |
+| [HADOOP-16588](https://issues.apache.org/jira/browse/HADOOP-16588) | Update commons-beanutils version to 1.9.4 in branch-2 |  Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md
deleted file mode 100644
index d8dd2daca09cc..0000000000000
--- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md
+++ /dev/null
@@ -1,788 +0,0 @@
-
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-# "Apache Hadoop" Changelog
-
-## Release 2.10.0 - 2019-10-22
-
-### INCOMPATIBLE CHANGES:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HDFS-12883](https://issues.apache.org/jira/browse/HDFS-12883) | RBF: Document Router and State Store metrics |  Major | documentation | Yiqun Lin | Yiqun Lin |
-| [HDFS-12895](https://issues.apache.org/jira/browse/HDFS-12895) | RBF: Add ACL support for mount table |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | RBF: Use the ZooKeeper as the default State Store |  Minor | documentation | Yiqun Lin | Yiqun Lin |
-| [HADOOP-16055](https://issues.apache.org/jira/browse/HADOOP-16055) | Upgrade AWS SDK to 1.11.271 in branch-2 |  Blocker | fs/s3 | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-16053](https://issues.apache.org/jira/browse/HADOOP-16053) | Backport HADOOP-14816 to branch-2 |  Major | build | Akira Ajisaka | Akira Ajisaka |
-
-
-### IMPORTANT ISSUES:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HDFS-13083](https://issues.apache.org/jira/browse/HDFS-13083) | RBF: Fix doc error setting up client |  Major | federation | tartarus | tartarus |
-
-
-### NEW FEATURES:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode |  Major | datanode, hdfs | Lukas Majercak | Lukas Majercak |
-| [HDFS-13553](https://issues.apache.org/jira/browse/HDFS-13553) | RBF: Support global quota |  Major | . | Íñigo Goiri | Yiqun Lin |
-| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping |  Major | common, security | Lukas Majercak | Lukas Majercak |
-| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs |  Major | . | Jonathan Hung | pralabhkumar |
-| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level |  Major | . | Jonathan Hung | Varun Saxena |
-
-
-### IMPROVEMENTS:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HADOOP-14987](https://issues.apache.org/jira/browse/HADOOP-14987) | Improve KMSClientProvider log around delegation token checking |  Major | . | Xiaoyu Yao | Xiaoyu Yao |
-| [HADOOP-14872](https://issues.apache.org/jira/browse/HADOOP-14872) | CryptoInputStream should implement unbuffer |  Major | fs, security | John Zhuge | John Zhuge |
-| [HADOOP-14960](https://issues.apache.org/jira/browse/HADOOP-14960) | Add GC time percentage monitor/alerter |  Major | . | Misha Dmitriev | Misha Dmitriev |
-| [HADOOP-15023](https://issues.apache.org/jira/browse/HADOOP-15023) | ValueQueue should also validate (lowWatermark \* numValues) \> 0 on construction |  Minor | . | Xiao Chen | Xiao Chen |
-| [YARN-6851](https://issues.apache.org/jira/browse/YARN-6851) | Capacity Scheduler: document configs for controlling # containers allowed to be allocated per node heartbeat |  Minor | . | Wei Yan | Wei Yan |
-| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService |  Major | log-aggregation | Jonathan Turner Eagles | Jonathan Turner Eagles |
-| [HADOOP-15056](https://issues.apache.org/jira/browse/HADOOP-15056) | Fix TestUnbuffer#testUnbufferException failure |  Minor | test | Jack Bearden | Jack Bearden |
-| [HADOOP-15012](https://issues.apache.org/jira/browse/HADOOP-15012) | Add readahead, dropbehind, and unbuffer to StreamCapabilities |  Major | fs | John Zhuge | John Zhuge |
-| [HADOOP-15104](https://issues.apache.org/jira/browse/HADOOP-15104) | AliyunOSS: change the default value of max error retry |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level |  Major | capacityscheduler | Scott Brokaw | Zian Chen |
-| [YARN-7642](https://issues.apache.org/jira/browse/YARN-7642) | Add test case to verify context update after container promotion or demotion with or without auto update |  Minor | nodemanager | Weiwei Yang | Weiwei Yang |
-| [HADOOP-15111](https://issues.apache.org/jira/browse/HADOOP-15111) | AliyunOSS: backport HADOOP-14993 to branch-2 |  Major | fs/oss | Genmao Yu | Genmao Yu |
-| [HDFS-12818](https://issues.apache.org/jira/browse/HDFS-12818) | Support multiple storages in DataNodeCluster / SimulatedFSDataset |  Minor | datanode, test | Erik Krogen | Erik Krogen |
-| [HDFS-9023](https://issues.apache.org/jira/browse/HDFS-9023) | When NN is not able to identify DN for replication, reason behind it can be logged |  Critical | hdfs-client, namenode | Surendra Singh Lilhore | Xiao Chen |
-| [YARN-7678](https://issues.apache.org/jira/browse/YARN-7678) | Ability to enable logging of container memory stats |  Major | nodemanager | Jim Brennan | Jim Brennan |
-| [HDFS-12945](https://issues.apache.org/jira/browse/HDFS-12945) | Switch to ClientProtocol instead of NamenodeProtocols in NamenodeWebHdfsMethods |  Minor | . | Wei Yan | Wei Yan |
-| [YARN-7622](https://issues.apache.org/jira/browse/YARN-7622) | Allow fair-scheduler configuration on HDFS |  Minor | fairscheduler, resourcemanager | Greg Phillips | Greg Phillips |
-| [YARN-7590](https://issues.apache.org/jira/browse/YARN-7590) | Improve container-executor validation check |  Major | security, yarn | Eric Yang | Eric Yang |
-| [MAPREDUCE-7029](https://issues.apache.org/jira/browse/MAPREDUCE-7029) | FileOutputCommitter is slow on filesystems lacking recursive delete |  Minor | . | Karthik Palaniappan | Karthik Palaniappan |
-| [MAPREDUCE-6984](https://issues.apache.org/jira/browse/MAPREDUCE-6984) | MR AM to clean up temporary files from previous attempt in case of no recovery |  Major | applicationmaster | Gergo Repas | Gergo Repas |
-| [HADOOP-15189](https://issues.apache.org/jira/browse/HADOOP-15189) | backport HADOOP-15039 to branch-2 and branch-3 |  Blocker | . | Genmao Yu | Genmao Yu |
-| [HADOOP-15212](https://issues.apache.org/jira/browse/HADOOP-15212) | Add independent secret manager method for logging expired tokens |  Major | security | Daryn Sharp | Daryn Sharp |
-| [YARN-7728](https://issues.apache.org/jira/browse/YARN-7728) | Expose container preemptions related information in Capacity Scheduler queue metrics |  Major | . | Eric Payne | Eric Payne |
-| [MAPREDUCE-7048](https://issues.apache.org/jira/browse/MAPREDUCE-7048) | Uber AM can crash due to unknown task in statusUpdate |  Major | mr-am | Peter Bacsko | Peter Bacsko |
-| [HADOOP-13972](https://issues.apache.org/jira/browse/HADOOP-13972) | ADLS to support per-store configuration |  Major | fs/adl | John Zhuge | Sharad Sonker |
-| [YARN-7813](https://issues.apache.org/jira/browse/YARN-7813) | Capacity Scheduler Intra-queue Preemption should be configurable for each queue |  Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne |
-| [HADOOP-15235](https://issues.apache.org/jira/browse/HADOOP-15235) | Authentication Tokens should use HMAC instead of MAC |  Major | security | Robert Kanter | Robert Kanter |
-| [HDFS-11187](https://issues.apache.org/jira/browse/HDFS-11187) | Optimize disk access for last partial chunk checksum of Finalized replica |  Major | datanode | Wei-Chiu Chuang | Gabor Bota |
-| [HADOOP-15266](https://issues.apache.org/jira/browse/HADOOP-15266) |  [branch-2] Upper/Lower case conversion support for group names in LdapGroupsMapping |  Major | . | Nanda kumar | Nanda kumar |
-| [HADOOP-15279](https://issues.apache.org/jira/browse/HADOOP-15279) | increase maven heap size recommendations |  Minor | build, documentation, test | Allen Wittenauer | Allen Wittenauer |
-| [HDFS-12884](https://issues.apache.org/jira/browse/HDFS-12884) | BlockUnderConstructionFeature.truncateBlock should be of type BlockInfo |  Major | namenode | Konstantin Shvachko | chencan |
-| [HADOOP-15334](https://issues.apache.org/jira/browse/HADOOP-15334) | Upgrade Maven surefire plugin |  Major | build | Arpit Agarwal | Arpit Agarwal |
-| [HADOOP-15312](https://issues.apache.org/jira/browse/HADOOP-15312) | Undocumented KeyProvider configuration keys |  Major | . | Wei-Chiu Chuang | LiXin Ge |
-| [YARN-7623](https://issues.apache.org/jira/browse/YARN-7623) | Fix the CapacityScheduler Queue configuration documentation |  Major | . | Arun Suresh | Jonathan Hung |
-| [HDFS-13314](https://issues.apache.org/jira/browse/HDFS-13314) | NameNode should optionally exit if it detects FsImage corruption |  Major | namenode | Arpit Agarwal | Arpit Agarwal |
-| [HDFS-13418](https://issues.apache.org/jira/browse/HDFS-13418) |  NetworkTopology should be configurable when enable DFSNetworkTopology |  Major | . | Tao Jie | Tao Jie |
-| [HADOOP-15394](https://issues.apache.org/jira/browse/HADOOP-15394) | Backport PowerShell NodeFencer HADOOP-14309 to branch-2 |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13462](https://issues.apache.org/jira/browse/HDFS-13462) | Add BIND\_HOST configuration for JournalNode's HTTP and RPC Servers |  Major | hdfs, journal-node | Lukas Majercak | Lukas Majercak |
-| [HDFS-13492](https://issues.apache.org/jira/browse/HDFS-13492) | Limit httpfs binds to certain IP addresses in branch-2 |  Major | httpfs | Wei-Chiu Chuang | Wei-Chiu Chuang |
-| [HDFS-12981](https://issues.apache.org/jira/browse/HDFS-12981) | renameSnapshot a Non-Existent snapshot to itself should throw error |  Minor | hdfs | Sailesh Patel | Kitti Nanasi |
-| [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. |  Minor | . | Wei-Chiu Chuang | Gabor Bota |
-| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster |  Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru |
-| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check |  Critical | webapp, yarn | Sunil G | Sunil G |
-| [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair |  Major | net | Nanda kumar | Nanda kumar |
-| [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover |  Critical | common | Karthik Palanisamy | Karthik Palanisamy |
-| [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem |  Major | ha, namenode | Erik Krogen | Chao Sun |
-| [HDFS-13644](https://issues.apache.org/jira/browse/HDFS-13644) | Backport HDFS-10376 to branch-2 |  Major | . | Yiqun Lin | Zsolt Venczel |
-| [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration |  Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
-| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
-| [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows |  Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak |
-| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading |  Major | documentation | Kitti Nanasi | Kitti Nanasi |
-| [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable |  Minor | hdfs | Lantao Jin | Lantao Jin |
-| [YARN-8155](https://issues.apache.org/jira/browse/YARN-8155) | Improve ATSv2 client logging in RM and NM publisher |  Major | . | Rohith Sharma K S | Abhishek Modi |
-| [HDFS-13814](https://issues.apache.org/jira/browse/HDFS-13814) | Remove super user privilege requirement for NameNode.getServiceStatus |  Minor | namenode | Chao Sun | Chao Sun |
-| [YARN-8559](https://issues.apache.org/jira/browse/YARN-8559) | Expose mutable-conf scheduler's configuration in RM /scheduler-conf endpoint |  Major | resourcemanager | Anna Savarin | Weiwei Yang |
-| [HDFS-13813](https://issues.apache.org/jira/browse/HDFS-13813) | Exit NameNode if dangling child inode is detected when saving FsImage |  Major | hdfs, namenode | Siyao Meng | Siyao Meng |
-| [HDFS-13821](https://issues.apache.org/jira/browse/HDFS-13821) | RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache |  Major | hdfs | Fei Hui | Fei Hui |
-| [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 |  Major | . | Rui Gao | Rui Gao |
-| [HDFS-13831](https://issues.apache.org/jira/browse/HDFS-13831) | Make block increment deletion number configurable |  Major | . | Yiqun Lin | Ryan Wu |
-| [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey |  Major | test | Robert Kanter | Robert Kanter |
-| [HADOOP-15547](https://issues.apache.org/jira/browse/HADOOP-15547) | WASB: improve listStatus performance |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
-| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
-| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation |  Trivial | documentation | David Mollitor | Hrishikesh Gadre |
-| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation |  Major | metrics | Sushil Ks | Sushil Ks |
-| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) |  Add JMX, conf and stacks menus to the datanode page |  Minor | datanode | fengchuang | fengchuang |
-| [HADOOP-15726](https://issues.apache.org/jira/browse/HADOOP-15726) | Create utility to limit frequency of log statements |  Major | common, util | Erik Krogen | Erik Krogen |
-| [YARN-7974](https://issues.apache.org/jira/browse/YARN-7974) | Allow updating application tracking url after registration |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics |  Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth |
-| [YARN-8896](https://issues.apache.org/jira/browse/YARN-8896) | Limit the maximum number of container assignments per heartbeat |  Major | . | Weiwei Yang | Zhankun Tang |
-| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 |  Major | . | PJ Fanning | Akira Ajisaka |
-| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler |  Minor | . | Zhankun Tang | Zhankun Tang |
-| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log |  Major | resourcemanager | Jonathan Hung | Eric Payne |
-| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS |  Major | fs/oss | wujinhu | wujinhu |
-| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI |  Major | . | Jonathan Hung | Keqiu Hu |
-| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog |  Major | namenode | Kenneth Yang | Kenneth Yang |
-| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics |  Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian |
-| [HADOOP-15617](https://issues.apache.org/jira/browse/HADOOP-15617) | Node.js and npm package loading in the Dockerfile failing on branch-2 |  Major | build | Allen Wittenauer | Akhil PB |
-| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically |  Major | . | Eric Maynard | Ahmed Hussein |
-| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' |  Trivial | . | Szilard Nemeth | Charan Hebri |
-| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
-| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology |  Minor | datanode | HuangTao | HuangTao |
-| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 |  Major | ATSv2 | Sushil Ks | Sushil Ks |
-| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance |  Major | hdfs | Chao Sun | Chao Sun |
-| [HDFS-14205](https://issues.apache.org/jira/browse/HDFS-14205) | Backport HDFS-6440 to branch-2 |  Major | . | Chen Liang | Chao Sun |
-| [HDFS-14391](https://issues.apache.org/jira/browse/HDFS-14391) | Backport HDFS-9659 to branch-2 |  Minor | . | Chao Sun | Chao Sun |
-| [HDFS-14415](https://issues.apache.org/jira/browse/HDFS-14415) | Backport HDFS-13799 to branch-2 |  Trivial | . | Chao Sun | Chao Sun |
-| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml |  Minor | hdfs | puleya7 | puleya7 |
-| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. |  Major | nodemanager | Jonathan Hung | Jonathan Hung |
-| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() |  Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko |
-| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings |  Minor | build | Akira Ajisaka | Akira Ajisaka |
-| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf |  Minor | . | Ahmed Hussein | Ahmed Hussein |
-| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
-| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts |  Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
-| [HADOOP-16359](https://issues.apache.org/jira/browse/HADOOP-16359) | Bundle ZSTD native in branch-2 |  Major | native | Chao Sun | Chao Sun |
-| [HADOOP-15253](https://issues.apache.org/jira/browse/HADOOP-15253) | Should update maxQueueSize when refresh call queue |  Minor | . | Tao Jie | Tao Jie |
-| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer |  Minor | ipc | Christopher Gregorian | Erik Krogen |
-| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading |  Major | . | zhouyingchao | Lisheng Sun |
-| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
-| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. |  Major | . | Jinglun | Jinglun |
-| [HDFS-14697](https://issues.apache.org/jira/browse/HDFS-14697) | Backport HDFS-14513 to branch-2 |  Minor | namenode | Xiaoqiao He | Xiaoqiao He |
-| [YARN-8045](https://issues.apache.org/jira/browse/YARN-8045) | Reduce log output from container status calls |  Major | . | Shane Kumpf | Craig Condit |
-| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory  instead of df/du |  Major | datanode, performance | Lisheng Sun | Lisheng Sun |
-| [HDFS-14696](https://issues.apache.org/jira/browse/HDFS-14696) | Backport HDFS-11273 to branch-2 (Move TransferFsImage#doGetUrl function to a Util class) |  Major | . | Siyao Meng | Siyao Meng |
-| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
-| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions |  Minor | yarn | Jim Brennan | Jim Brennan |
-| [HADOOP-16459](https://issues.apache.org/jira/browse/HADOOP-16459) | Backport [HADOOP-16266] "Add more fine-grained processing time metrics to the RPC layer" to branch-2 |  Major | . | Erik Krogen | Erik Krogen |
-| [HDFS-14707](https://issues.apache.org/jira/browse/HDFS-14707) |  Add JAVA\_LIBRARY\_PATH to HTTPFS startup options in branch-2 |  Major | httpfs | Masatake Iwasaki | Masatake Iwasaki |
-| [HDFS-14723](https://issues.apache.org/jira/browse/HDFS-14723) | Add helper method FSNamesystem#setBlockManagerForTesting() in branch-2 |  Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
-| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead |  Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena |
-| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
-| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round |  Major | capacity scheduler | Eric Payne | Manikandan R |
-| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename |  Major | . | Jinglun | Jinglun |
-| [HADOOP-16439](https://issues.apache.org/jira/browse/HADOOP-16439) | Upgrade bundled Tomcat in branch-2 |  Major | httpfs, kms | Masatake Iwasaki | Masatake Iwasaki |
-| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics |  Major | . | Jonathan Hung | Shubham Gupta |
-| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
-| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
-| [HADOOP-16530](https://issues.apache.org/jira/browse/HADOOP-16530) | Update xercesImpl in branch-2 |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
-| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9825](https://issues.apache.org/jira/browse/YARN-9825) | Changes for initializing placement rules with ResourceScheduler in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs |  Major | . | Jonathan Hung | Manoj Kumar |
-| [HDFS-14667](https://issues.apache.org/jira/browse/HDFS-14667) | Backport [HDFS-14403] "Cost-based FairCallQueue" to branch-2 |  Major | . | Erik Krogen | Erik Krogen |
-
-
-### BUG FIXES:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [MAPREDUCE-6896](https://issues.apache.org/jira/browse/MAPREDUCE-6896) | Document wrong spelling in usage of MapredTestDriver tools. |  Major | documentation | LiXin Ge | LiXin Ge |
-| [HDFS-12052](https://issues.apache.org/jira/browse/HDFS-12052) | Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS |  Major | httpfs, webhdfs | Zoran Dimitrijevic | Zoran Dimitrijevic |
-| [HDFS-12318](https://issues.apache.org/jira/browse/HDFS-12318) | Fix IOException condition for openInfo in DFSInputStream |  Major | . | legend | legend |
-| [HDFS-12614](https://issues.apache.org/jira/browse/HDFS-12614) | FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider configured |  Major | . | Manoj Govindassamy | Manoj Govindassamy |
-| [YARN-7396](https://issues.apache.org/jira/browse/YARN-7396) | NPE when accessing container logs due to null dirsHandler |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-7370](https://issues.apache.org/jira/browse/YARN-7370) | Preemption properties should be refreshable |  Major | capacity scheduler, scheduler preemption | Eric Payne | Gergely Novák |
-| [YARN-7428](https://issues.apache.org/jira/browse/YARN-7428) | Add containerId to Localizer failed logs |  Minor | nodemanager | Prabhu Joseph | Prabhu Joseph |
-| [YARN-7410](https://issues.apache.org/jira/browse/YARN-7410) | Cleanup FixedValueResource to avoid dependency to ResourceUtils |  Major | resourcemanager | Sunil G | Wangda Tan |
-| [HDFS-12783](https://issues.apache.org/jira/browse/HDFS-12783) | [branch-2] "dfsrouter" should use hdfsScript |  Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
-| [HDFS-12788](https://issues.apache.org/jira/browse/HDFS-12788) | Reset the upload button when file upload fails |  Critical | ui, webhdfs | Brahma Reddy Battula | Brahma Reddy Battula |
-| [YARN-7469](https://issues.apache.org/jira/browse/YARN-7469) | Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit |  Major | capacity scheduler, yarn | Eric Payne | Eric Payne |
-| [HADOOP-14982](https://issues.apache.org/jira/browse/HADOOP-14982) | Clients using FailoverOnNetworkExceptionRetry can go into a loop if they're used without authenticating with kerberos in HA env |  Major | common | Peter Bacsko | Peter Bacsko |
-| [YARN-7489](https://issues.apache.org/jira/browse/YARN-7489) | ConcurrentModificationException in RMAppImpl#getRMAppMetrics |  Major | capacityscheduler | Tao Yang | Tao Yang |
-| [YARN-7525](https://issues.apache.org/jira/browse/YARN-7525) | Incorrect query parameters in cluster nodes REST API document |  Minor | documentation | Tao Yang | Tao Yang |
-| [HDFS-12813](https://issues.apache.org/jira/browse/HDFS-12813) | RequestHedgingProxyProvider can hide Exception thrown from the Namenode for proxy size of 1 |  Major | ha | Mukul Kumar Singh | Mukul Kumar Singh |
-| [HADOOP-15045](https://issues.apache.org/jira/browse/HADOOP-15045) | ISA-L build options are documented in branch-2 |  Major | build, documentation | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-15067](https://issues.apache.org/jira/browse/HADOOP-15067) | GC time percentage reported in JvmMetrics should be a gauge, not counter |  Major | . | Misha Dmitriev | Misha Dmitriev |
-| [YARN-7363](https://issues.apache.org/jira/browse/YARN-7363) | ContainerLocalizer doesn't have a valid log4j config when using LinuxContainerExecutor |  Major | nodemanager | Yufei Gu | Yufei Gu |
-| [HDFS-12754](https://issues.apache.org/jira/browse/HDFS-12754) | Lease renewal can hit a deadlock |  Major | . | Kuhu Shukla | Kuhu Shukla |
-| [HDFS-12832](https://issues.apache.org/jira/browse/HDFS-12832) | INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit |  Critical | namenode | DENG FEI | Konstantin Shvachko |
-| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. |  Minor | . | Rushabh Shah | Mikhail Erofeev |
-| [HADOOP-15042](https://issues.apache.org/jira/browse/HADOOP-15042) | Azure PageBlobInputStream.skip() can return negative value when numberOfPagesRemaining is 0 |  Minor | fs/azure | Rajesh Balamohan | Rajesh Balamohan |
-| [HDFS-12638](https://issues.apache.org/jira/browse/HDFS-12638) | Delete copy-on-truncate block along with the original block, when deleting a file being truncated |  Blocker | hdfs | Jiandan Yang | Konstantin Shvachko |
-| [YARN-4813](https://issues.apache.org/jira/browse/YARN-4813) | TestRMWebServicesDelegationTokenAuthentication.testDoAs fails intermittently |  Major | resourcemanager | Daniel Templeton | Gergo Repas |
-| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events |  Major | mr-am | Jason Darrell Lowe | Peter Bacsko |
-| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer |  Major | nodemanager | Jason Darrell Lowe | Jim Brennan |
-| [YARN-7594](https://issues.apache.org/jira/browse/YARN-7594) | TestNMWebServices#testGetNMResourceInfo fails on trunk |  Major | nodemanager, webapp | Gergely Novák | Gergely Novák |
-| [YARN-5594](https://issues.apache.org/jira/browse/YARN-5594) | Handle old RMDelegationToken format when recovering RM |  Major | resourcemanager | Tatyana But | Robert Kanter |
-| [HADOOP-14985](https://issues.apache.org/jira/browse/HADOOP-14985) | Remove subversion related code from VersionInfoMojo.java |  Minor | build | Akira Ajisaka | Ajay Kumar |
-| [HDFS-11751](https://issues.apache.org/jira/browse/HDFS-11751) | DFSZKFailoverController daemon exits with wrong status code |  Major | auto-failover | Doris Gu | Bharat Viswanadham |
-| [HDFS-12889](https://issues.apache.org/jira/browse/HDFS-12889) | Router UI is missing robots.txt file |  Major | . | Bharat Viswanadham | Bharat Viswanadham |
-| [YARN-7607](https://issues.apache.org/jira/browse/YARN-7607) | Remove the trailing duplicated timestamp in container diagnostics message |  Minor | nodemanager | Weiwei Yang | Weiwei Yang |
-| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" |  Blocker | fs/oss | Christopher Douglas | Sammi Chen |
-| [YARN-7608](https://issues.apache.org/jira/browse/YARN-7608) | Incorrect sTarget column causing DataTable warning on RM application and scheduler web page |  Major | resourcemanager, webapp | Weiwei Yang | Gergely Novák |
-| [HDFS-12833](https://issues.apache.org/jira/browse/HDFS-12833) | Distcp : Update the usage of delete option for dependency with update and overwrite option |  Minor | distcp, hdfs | Harshakiran Reddy | usharani |
-| [YARN-7647](https://issues.apache.org/jira/browse/YARN-7647) | NM print inappropriate error log when node-labels is enabled |  Minor | . | Yang Wang | Yang Wang |
-| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers |  Major | namenode | Daryn Sharp | Rushabh Shah |
-| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Darrell Lowe | Ajay Kumar |
-| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes |  Major | nodemanager | Jason Darrell Lowe | Jim Brennan |
-| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors |  Major | . | Jason Darrell Lowe | Jim Brennan |
-| [HADOOP-15123](https://issues.apache.org/jira/browse/HADOOP-15123) | KDiag tries to load krb5.conf from KRB5CCNAME instead of KRB5\_CONFIG |  Minor | security | Vipin Rathor | Vipin Rathor |
-| [YARN-7661](https://issues.apache.org/jira/browse/YARN-7661) | NodeManager metrics return wrong value after update node resource |  Major | . | Yang Wang | Yang Wang |
-| [HDFS-12347](https://issues.apache.org/jira/browse/HDFS-12347) | TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently |  Critical | test | Xiao Chen | Bharat Viswanadham |
-| [YARN-7662](https://issues.apache.org/jira/browse/YARN-7662) | [Atsv2] Define new set of configurations for reader and collectors to bind. |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 |  Major | . | Rohith Sharma K S | Sunil G |
-| [YARN-7542](https://issues.apache.org/jira/browse/YARN-7542) | Fix issue that causes some Running Opportunistic Containers to be recovered as PAUSED |  Major | . | Arun Suresh | Sampada Dehankar |
-| [HADOOP-15143](https://issues.apache.org/jira/browse/HADOOP-15143) | NPE due to Invalid KerberosTicket in UGI |  Major | . | Jitendra Nath Pandey | Mukul Kumar Singh |
-| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications |  Blocker | resourcemanager | Charan Hebri | Sunil G |
-| [MAPREDUCE-7028](https://issues.apache.org/jira/browse/MAPREDUCE-7028) | Concurrent task progress updates causing NPE in Application Master |  Blocker | mr-am | Gergo Repas | Gergo Repas |
-| [YARN-7619](https://issues.apache.org/jira/browse/YARN-7619) | Max AM Resource value in Capacity Scheduler UI has to be refreshed for every user |  Major | capacity scheduler, yarn | Eric Payne | Eric Payne |
-| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call |  Major | webapp | Sunil G | Sunil G |
-| [HDFS-12985](https://issues.apache.org/jira/browse/HDFS-12985) | NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted |  Major | hdfs | Manoj Govindassamy | Manoj Govindassamy |
-| [YARN-4227](https://issues.apache.org/jira/browse/YARN-4227) | Ignore expired containers from removed nodes in FairScheduler |  Critical | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
-| [YARN-7508](https://issues.apache.org/jira/browse/YARN-7508) | NPE in FiCaSchedulerApp when debug log enabled in async-scheduling mode |  Major | capacityscheduler | Tao Yang | Tao Yang |
-| [YARN-7663](https://issues.apache.org/jira/browse/YARN-7663) | RMAppImpl:Invalid event: START at KILLED |  Major | resourcemanager | lujie | lujie |
-| [YARN-6948](https://issues.apache.org/jira/browse/YARN-6948) | Invalid event: ATTEMPT\_ADDED at FINAL\_SAVING |  Major | yarn | lujie | lujie |
-| [HADOOP-15060](https://issues.apache.org/jira/browse/HADOOP-15060) | TestShellBasedUnixGroupsMapping.testFiniteGroupResolutionTime flaky |  Major | . | Miklos Szegedi | Miklos Szegedi |
-| [YARN-7735](https://issues.apache.org/jira/browse/YARN-7735) | Fix typo in YARN documentation |  Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
-| [YARN-7727](https://issues.apache.org/jira/browse/YARN-7727) | Incorrect log levels in few logs with QueuePriorityContainerCandidateSelector |  Minor | yarn | Prabhu Joseph | Prabhu Joseph |
-| [HDFS-11915](https://issues.apache.org/jira/browse/HDFS-11915) | Sync rbw dir on the first hsync() to avoid file lost on power failure |  Critical | . | Kanaka Kumar Avvaru | Vinayakumar B |
-| [YARN-7705](https://issues.apache.org/jira/browse/YARN-7705) | Create the container log directory with correct sticky bit in C code |  Major | nodemanager | Yufei Gu | Yufei Gu |
-| [HDFS-9049](https://issues.apache.org/jira/browse/HDFS-9049) | Make Datanode Netty reverse proxy port to be configurable |  Major | datanode | Vinayakumar B | Vinayakumar B |
-| [YARN-7758](https://issues.apache.org/jira/browse/YARN-7758) | Add an additional check to the validity of container and application ids passed to container-executor |  Major | nodemanager | Miklos Szegedi | Yufei Gu |
-| [HADOOP-15150](https://issues.apache.org/jira/browse/HADOOP-15150) | in FsShell, UGI params should be overidden through env vars(-D arg) |  Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
-| [HADOOP-15181](https://issues.apache.org/jira/browse/HADOOP-15181) | Typo in SecureMode.md |  Trivial | documentation | Masahiro Tanaka | Masahiro Tanaka |
-| [YARN-7806](https://issues.apache.org/jira/browse/YARN-7806) | Distributed Shell should use timeline async api's |  Major | distributed-shell | Sumana Sathish | Rohith Sharma K S |
-| [HADOOP-15121](https://issues.apache.org/jira/browse/HADOOP-15121) | Encounter NullPointerException when using DecayRpcScheduler |  Major | . | Tao Jie | Tao Jie |
-| [MAPREDUCE-7015](https://issues.apache.org/jira/browse/MAPREDUCE-7015) | Possible race condition in JHS if the job is not loaded |  Major | jobhistoryserver | Peter Bacsko | Peter Bacsko |
-| [YARN-7737](https://issues.apache.org/jira/browse/YARN-7737) | prelaunch.err file not found exception on container failure |  Major | . | Jonathan Hung | Keqiu Hu |
-| [HDFS-13063](https://issues.apache.org/jira/browse/HDFS-13063) | Fix the incorrect spelling in HDFSHighAvailabilityWithQJM.md |  Trivial | documentation | Jianfei Jiang | Jianfei Jiang |
-| [YARN-7102](https://issues.apache.org/jira/browse/YARN-7102) | NM heartbeat stuck when responseId overflows MAX\_INT |  Critical | . | Botong Huang | Botong Huang |
-| [MAPREDUCE-7041](https://issues.apache.org/jira/browse/MAPREDUCE-7041) | MR should not try to clean up at first job attempt |  Major | . | Takanobu Asanuma | Gergo Repas |
-| [MAPREDUCE-7020](https://issues.apache.org/jira/browse/MAPREDUCE-7020) | Task timeout in uber mode can crash AM |  Major | mr-am | Akira Ajisaka | Peter Bacsko |
-| [YARN-7765](https://issues.apache.org/jira/browse/YARN-7765) | [Atsv2] GSSException: No valid credentials provided - Failed to find any Kerberos tgt thrown by Timelinev2Client & HBaseClient in NM |  Blocker | . | Sumana Sathish | Rohith Sharma K S |
-| [HDFS-12974](https://issues.apache.org/jira/browse/HDFS-12974) | Exception message is not printed when creating an encryption zone fails with AuthorizationException |  Minor | encryption | fang zhenyi | fang zhenyi |
-| [YARN-7698](https://issues.apache.org/jira/browse/YARN-7698) | A misleading variable's name in ApplicationAttemptEventDispatcher |  Minor | resourcemanager | Jinjiang Ling | Jinjiang Ling |
-| [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | Add an option to not disable short-circuit reads on failures |  Major | hdfs-client, performance | Andre Araujo | Xiao Chen |
-| [HDFS-13100](https://issues.apache.org/jira/browse/HDFS-13100) | Handle IllegalArgumentException when GETSERVERDEFAULTS is not implemented in webhdfs. |  Critical | hdfs, webhdfs | Yongjun Zhang | Yongjun Zhang |
-| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error |  Major | test | Jason Darrell Lowe | Botong Huang |
-| [YARN-7801](https://issues.apache.org/jira/browse/YARN-7801) | AmFilterInitializer should addFilter after fill all parameters |  Critical | . | Sumana Sathish | Wangda Tan |
-| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch |  Major | . | Billie Rinaldi | Jason Darrell Lowe |
-| [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted |  Major | . | Yongjun Zhang | Yongjun Zhang |
-| [HDFS-12935](https://issues.apache.org/jira/browse/HDFS-12935) | Get ambiguous result for DFSAdmin command in HA mode when only one namenode is up |  Major | tools | Jianfei Jiang | Jianfei Jiang |
-| [HDFS-13120](https://issues.apache.org/jira/browse/HDFS-13120) | Snapshot diff could be corrupted after concat |  Major | namenode, snapshots | Xiaoyu Yao | Xiaoyu Yao |
-| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
-| [HDFS-8693](https://issues.apache.org/jira/browse/HDFS-8693) | refreshNamenodes does not support adding a new standby to a running DN |  Critical | datanode, ha | Jian Fang | Ajith S |
-| [MAPREDUCE-7052](https://issues.apache.org/jira/browse/MAPREDUCE-7052) | TestFixedLengthInputFormat#testFormatCompressedIn is flaky |  Major | client, test | Peter Bacsko | Peter Bacsko |
-| [HDFS-13112](https://issues.apache.org/jira/browse/HDFS-13112) | Token expiration edits may cause log corruption or deadlock |  Critical | namenode | Daryn Sharp | Daryn Sharp |
-| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump |  Major | . | Jason Darrell Lowe | Jason Darrell Lowe |
-| [HADOOP-15206](https://issues.apache.org/jira/browse/HADOOP-15206) | BZip2 drops and duplicates records when input split size is small |  Major | . | Aki Tanaka | Aki Tanaka |
-| [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request |  Minor | docs, documentation | Charan Hebri | Charan Hebri |
-| [YARN-7947](https://issues.apache.org/jira/browse/YARN-7947) | Capacity Scheduler intra-queue preemption can NPE for non-schedulable apps |  Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne |
-| [YARN-7945](https://issues.apache.org/jira/browse/YARN-7945) | Java Doc error in UnmanagedAMPoolManager for branch-2 |  Major | . | Rohith Sharma K S | Botong Huang |
-| [HADOOP-14903](https://issues.apache.org/jira/browse/HADOOP-14903) | Add json-smart explicitly to pom.xml |  Major | common | Ray Chiang | Ray Chiang |
-| [HADOOP-15236](https://issues.apache.org/jira/browse/HADOOP-15236) | Fix typo in RequestHedgingProxyProvider and RequestHedgingRMFailoverProxyProvider |  Trivial | documentation | Akira Ajisaka | Gabor Bota |
-| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails |  Critical | harchive | Gergely Novák | Gergely Novák |
-| [HDFS-12781](https://issues.apache.org/jira/browse/HDFS-12781) | After Datanode down, In Namenode UI Datanode tab is throwing warning message. |  Major | datanode | Harshakiran Reddy | Brahma Reddy Battula |
-| [HDFS-12070](https://issues.apache.org/jira/browse/HDFS-12070) | Failed block recovery leaves files open indefinitely and at risk for data loss |  Major | . | Daryn Sharp | Kihwal Lee |
-| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 |  Major | test | Christopher Douglas | Christopher Douglas |
-| [HDFS-13194](https://issues.apache.org/jira/browse/HDFS-13194) | CachePool permissions incorrectly checked |  Major | . | Yiqun Lin | Jianfei Jiang |
-| [HADOOP-15276](https://issues.apache.org/jira/browse/HADOOP-15276) | branch-2 site not building after ADL troubleshooting doc added |  Major | documentation | Steve Loughran | Steve Loughran |
-| [YARN-7835](https://issues.apache.org/jira/browse/YARN-7835) | [Atsv2] Race condition in NM while publishing events if second attempt is launched on the same node |  Critical | . | Rohith Sharma K S | Rohith Sharma K S |
-| [HADOOP-15275](https://issues.apache.org/jira/browse/HADOOP-15275) | Incorrect javadoc for return type of RetryPolicy#shouldRetry |  Minor | documentation | Nanda kumar | Nanda kumar |
-| [YARN-7511](https://issues.apache.org/jira/browse/YARN-7511) | NPE in ContainerLocalizer when localization failed for running container |  Major | nodemanager | Tao Yang | Tao Yang |
-| [MAPREDUCE-7023](https://issues.apache.org/jira/browse/MAPREDUCE-7023) | TestHadoopArchiveLogs.testCheckFilesAndSeedApps fails on rerun |  Minor | test | Gergely Novák | Gergely Novák |
-| [HADOOP-15283](https://issues.apache.org/jira/browse/HADOOP-15283) | Upgrade from findbugs 3.0.1 to spotbugs 3.1.2 in branch-2 to fix docker image build |  Major | . | Xiao Chen | Akira Ajisaka |
-| [HADOOP-15286](https://issues.apache.org/jira/browse/HADOOP-15286) | Remove unused imports from TestKMSWithZK.java |  Minor | test | Akira Ajisaka | Ajay Kumar |
-| [HDFS-13040](https://issues.apache.org/jira/browse/HDFS-13040) | Kerberized inotify client fails despite kinit properly |  Major | namenode | Wei-Chiu Chuang | Xiao Chen |
-| [YARN-7736](https://issues.apache.org/jira/browse/YARN-7736) | Fix itemization in YARN federation document |  Minor | documentation | Akira Ajisaka | Sen Zhao |
-| [HDFS-13164](https://issues.apache.org/jira/browse/HDFS-13164) | File not closed if streamer fail with DSQuotaExceededException |  Major | hdfs-client | Xiao Chen | Xiao Chen |
-| [HDFS-13109](https://issues.apache.org/jira/browse/HDFS-13109) | Support fully qualified hdfs path in EZ commands |  Major | hdfs | Hanisha Koneru | Hanisha Koneru |
-| [MAPREDUCE-6930](https://issues.apache.org/jira/browse/MAPREDUCE-6930) | mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores are both present twice in mapred-default.xml |  Major | mrv2 | Daniel Templeton | Sen Zhao |
-| [HDFS-10618](https://issues.apache.org/jira/browse/HDFS-10618) | TestPendingReconstruction#testPendingAndInvalidate is flaky due to race condition |  Major | . | Eric Badger | Eric Badger |
-| [HDFS-10803](https://issues.apache.org/jira/browse/HDFS-10803) | TestBalancerWithMultipleNameNodes#testBalancing2OutOf3Blockpools fails intermittently due to no free space available |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-12156](https://issues.apache.org/jira/browse/HDFS-12156) | TestFSImage fails without -Pnative |  Major | test | Akira Ajisaka | Akira Ajisaka |
-| [HDFS-13261](https://issues.apache.org/jira/browse/HDFS-13261) | Fix incorrect null value check |  Minor | hdfs | Jianfei Jiang | Jianfei Jiang |
-| [HDFS-12886](https://issues.apache.org/jira/browse/HDFS-12886) | Ignore minReplication for block recovery |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
-| [YARN-8039](https://issues.apache.org/jira/browse/YARN-8039) | Clean up log dir configuration in TestLinuxContainerExecutorWithMocks.testStartLocalizer |  Minor | . | Miklos Szegedi | Miklos Szegedi |
-| [HDFS-13296](https://issues.apache.org/jira/browse/HDFS-13296) | GenericTestUtils generates paths with drive letter in Windows and fail webhdfs related test cases |  Major | . | Xiao Liang | Xiao Liang |
-| [HDFS-13268](https://issues.apache.org/jira/browse/HDFS-13268) | TestWebHdfsFileContextMainOperations fails on Windows |  Major | . | Íñigo Goiri | Xiao Liang |
-| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread |  Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
-| [YARN-7873](https://issues.apache.org/jira/browse/YARN-7873) | Revert YARN-6078 |  Blocker | . | Billie Rinaldi | Billie Rinaldi |
-| [HDFS-13195](https://issues.apache.org/jira/browse/HDFS-13195) | DataNode conf page  cannot display the current value after reconfig |  Minor | datanode | maobaolong | maobaolong |
-| [YARN-8063](https://issues.apache.org/jira/browse/YARN-8063) | DistributedShellTimelinePlugin wrongly check for entityId instead of entityType |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ |  Blocker | yarn | Sunil G | Sunil G |
-| [HADOOP-12862](https://issues.apache.org/jira/browse/HADOOP-12862) | LDAP Group Mapping over SSL can not specify trust store |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
-| [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop |  Major | . | Xiao Chen | Xiao Chen |
-| [HADOOP-15355](https://issues.apache.org/jira/browse/HADOOP-15355) | TestCommonConfigurationFields is broken by HADOOP-15312 |  Major | test | Konstantin Shvachko | LiXin Ge |
-| [HDFS-13176](https://issues.apache.org/jira/browse/HDFS-13176) | WebHdfs file path gets truncated when having semicolon (;) inside |  Major | webhdfs | Zsolt Venczel | Zsolt Venczel |
-| [HADOOP-15375](https://issues.apache.org/jira/browse/HADOOP-15375) | Branch-2 pre-commit failed to build docker image |  Major | . | Xiao Chen | Xiao Chen |
-| [HADOOP-15357](https://issues.apache.org/jira/browse/HADOOP-15357) | Configuration.getPropsWithPrefix no longer does variable substitution |  Major | . | Jim Brennan | Jim Brennan |
-| [MAPREDUCE-7062](https://issues.apache.org/jira/browse/MAPREDUCE-7062) | Update mapreduce.job.tags description for making use for ATSv2 purpose. |  Major | . | Charan Hebri | Charan Hebri |
-| [YARN-8073](https://issues.apache.org/jira/browse/YARN-8073) | TimelineClientImpl doesn't honor yarn.timeline-service.versions configuration |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-6629](https://issues.apache.org/jira/browse/YARN-6629) | NPE occurred when container allocation proposal is applied but its resource requests are removed before |  Critical | . | Tao Yang | Tao Yang |
-| [HDFS-13427](https://issues.apache.org/jira/browse/HDFS-13427) | Fix the section titles of transparent encryption document |  Minor | documentation | Akira Ajisaka | Akira Ajisaka |
-| [YARN-7527](https://issues.apache.org/jira/browse/YARN-7527) | Over-allocate node resource in async-scheduling mode of CapacityScheduler |  Major | capacityscheduler | Tao Yang | Tao Yang |
-| [HDFS-7101](https://issues.apache.org/jira/browse/HDFS-7101) | Potential null dereference in DFSck#doWork() |  Minor | . | Ted Yu | skrho |
-| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger |  Major | nodemanager, resourcemanager | Jason Darrell Lowe | Jason Darrell Lowe |
-| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails |  Major | test | Jason Darrell Lowe | Jason Darrell Lowe |
-| [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option |  Minor | . | Erik Krogen | Erik Krogen |
-| [YARN-8156](https://issues.apache.org/jira/browse/YARN-8156) | Increase the default value of yarn.timeline-service.app-collector.linger-period.ms |  Major | . | Rohith Sharma K S | Charan Hebri |
-| [YARN-8165](https://issues.apache.org/jira/browse/YARN-8165) | Incorrect queue name logging in AbstractContainerAllocator |  Trivial | capacityscheduler | Weiwei Yang | Weiwei Yang |
-| [YARN-8164](https://issues.apache.org/jira/browse/YARN-8164) | Fix a potential NPE in AbstractSchedulerPlanFollower |  Major | . | lujie | lujie |
-| [HDFS-12828](https://issues.apache.org/jira/browse/HDFS-12828) | OIV ReverseXML Processor fails with escaped characters |  Critical | hdfs | Erik Krogen | Erik Krogen |
-| [HADOOP-15180](https://issues.apache.org/jira/browse/HADOOP-15180) | branch-2 : daemon processes' sysout overwrites 'ulimit -a' in daemon's out file |  Minor | scripts | Ranith Sardar | Ranith Sardar |
-| [HADOOP-15396](https://issues.apache.org/jira/browse/HADOOP-15396) | Some java source files are executable |  Minor | . | Akira Ajisaka | Shashikant Banerjee |
-| [YARN-6827](https://issues.apache.org/jira/browse/YARN-6827) | [ATS1/1.5] NPE exception while publishing recovering applications into ATS during RM restart. |  Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S |
-| [YARN-7786](https://issues.apache.org/jira/browse/YARN-7786) | NullPointerException while launching ApplicationMaster |  Major | . | lujie | lujie |
-| [HDFS-13408](https://issues.apache.org/jira/browse/HDFS-13408) | MiniDFSCluster to support being built on randomized base directory |  Major | test | Xiao Liang | Xiao Liang |
-| [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens |  Critical | . | Xiao Chen | Xiao Chen |
-| [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows |  Major | . | Xiao Liang | Xiao Liang |
-| [YARN-7598](https://issues.apache.org/jira/browse/YARN-7598) | Document how to use classpath isolation for aux-services in YARN |  Major | . | Xuan Gong | Xuan Gong |
-| [YARN-8183](https://issues.apache.org/jira/browse/YARN-8183) | Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap |  Critical | yarn | Sumana Sathish | Suma Shivaprasad |
-| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 |  Critical | tools/distcp | Rushabh Shah | Jason Darrell Lowe |
-| [MAPREDUCE-7042](https://issues.apache.org/jira/browse/MAPREDUCE-7042) | Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled |  Major | . | Yesha Vora | Xuan Gong |
-| [YARN-8205](https://issues.apache.org/jira/browse/YARN-8205) | Application State is not updated to ATS if AM launching is delayed. |  Critical | . | Sumana Sathish | Rohith Sharma K S |
-| [MAPREDUCE-7072](https://issues.apache.org/jira/browse/MAPREDUCE-7072) | mapred job -history prints duplicate counter in human output |  Major | client | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
-| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps |  Major | webapp | Sunil G | Sunil G |
-| [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
-| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal |  Major | . | Bibin Chundatt | Bibin Chundatt |
-| [YARN-8025](https://issues.apache.org/jira/browse/YARN-8025) | UsersManangers#getComputedResourceLimitForActiveUsers throws NPE due to preComputedActiveUserLimit is empty |  Major | yarn | Jiandan Yang | Tao Yang |
-| [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens |  Major | resourcemanager | Hu Ziqian | Hu Ziqian |
-| [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows |  Major | . | Xiao Liang | Xiao Liang |
-| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
-| [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) |  TestContainerSchedulerQueuing.testStartMultipleContainers failed |  Major | . | Miklos Szegedi | Jim Brennan |
-| [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows |  Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak |
-| [HDFS-13590](https://issues.apache.org/jira/browse/HDFS-13590) | Backport HDFS-12378 to branch-2 |  Major | datanode, hdfs, test | Lukas Majercak | Lukas Majercak |
-| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression |  Major | fs/azure | Thomas Marqardt | Thomas Marqardt |
-| [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation |  Blocker | . | Kihwal Lee | Arpit Agarwal |
-| [HDFS-13601](https://issues.apache.org/jira/browse/HDFS-13601) | Optimize ByteString conversions in PBHelper |  Major | . | Andrew Wang | Andrew Wang |
-| [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
-| [YARN-8310](https://issues.apache.org/jira/browse/YARN-8310) | Handle old NMTokenIdentifier, AMRMTokenIdentifier, and ContainerTokenIdentifier formats |  Major | . | Robert Kanter | Robert Kanter |
-| [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync |  Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows |  Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" |  Blocker | . | Rohith Sharma K S | Jason Darrell Lowe |
-| [HDFS-13611](https://issues.apache.org/jira/browse/HDFS-13611) | Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient |  Major | . | Andrew Wang | Andrew Wang |
-| [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows |  Major | test | Xiao Liang | Xiao Liang |
-| [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 |  Critical | kms | Gabor Bota | Gabor Bota |
-| [HDFS-13626](https://issues.apache.org/jira/browse/HDFS-13626) | Fix incorrect username when deny the setOwner operation |  Minor | namenode | luhuachao | Zsolt Venczel |
-| [MAPREDUCE-7103](https://issues.apache.org/jira/browse/MAPREDUCE-7103) | Fix TestHistoryViewerPrinter on windows due to a mismatch line separator |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows |  Major | . | Giovanni Matteo Fumarola | Jason Darrell Lowe |
-| [HDFS-13664](https://issues.apache.org/jira/browse/HDFS-13664) | Refactor ConfiguredFailoverProxyProvider to make inheritance easier |  Minor | hdfs-client | Chao Sun | Chao Sun |
-| [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 |  Major | . | Rohith Sharma K S | Íñigo Goiri |
-| [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
-| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime |  Critical | . | Wangda Tan | Thomas Marqardt |
-| [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
-| [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
-| [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
-| [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
-| [HADOOP-15523](https://issues.apache.org/jira/browse/HADOOP-15523) | Shell command timeout given is in seconds whereas it is taken as millisec while scheduling |  Major | . | Bilwa S T | Bilwa S T |
-| [YARN-8444](https://issues.apache.org/jira/browse/YARN-8444) | NodeResourceMonitor crashes on bad swapFree value |  Major | . | Jim Brennan | Jim Brennan |
-| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui |  Major | webapp | Sunil G | Sunil G |
-| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection |  Blocker | . | Bibin Chundatt | Bibin Chundatt |
-| [YARN-8451](https://issues.apache.org/jira/browse/YARN-8451) | Multiple NM heartbeat thread created when a slow NM resync with RM |  Major | nodemanager | Botong Huang | Botong Huang |
-| [HADOOP-15548](https://issues.apache.org/jira/browse/HADOOP-15548) | Randomize local dirs |  Minor | . | Jim Brennan | Jim Brennan |
-| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state |  Major | nodemanager | Jason Darrell Lowe | Jason Darrell Lowe |
-| [HDFS-13729](https://issues.apache.org/jira/browse/HDFS-13729) | Fix broken links to RBF documentation |  Minor | documentation | jwhitter | Gabor Bota |
-| [YARN-8515](https://issues.apache.org/jira/browse/YARN-8515) | container-executor can crash with SIGPIPE after nodemanager restart |  Major | . | Jim Brennan | Jim Brennan |
-| [YARN-8421](https://issues.apache.org/jira/browse/YARN-8421) | when moving app, activeUsers is increased, even though app does not have outstanding request |  Major | . | kyungwan nam |  |
-| [HADOOP-15614](https://issues.apache.org/jira/browse/HADOOP-15614) | TestGroupsCaching.testExceptionOnBackgroundRefreshHandled reliably fails |  Major | . | Kihwal Lee | Weiwei Yang |
-| [YARN-4606](https://issues.apache.org/jira/browse/YARN-4606) | CapacityScheduler: applications could get starved because computation of #activeUsers considers pending apps |  Critical | capacity scheduler, capacityscheduler | Karam Singh | Manikandan R |
-| [HADOOP-15637](https://issues.apache.org/jira/browse/HADOOP-15637) | LocalFs#listLocatedStatus does not filter out hidden .crc files |  Minor | fs | Erik Krogen | Erik Krogen |
-| [HADOOP-15644](https://issues.apache.org/jira/browse/HADOOP-15644) | Hadoop Docker Image Pip Install Fails on branch-2 |  Critical | build | Haibo Chen | Haibo Chen |
-| [YARN-6966](https://issues.apache.org/jira/browse/YARN-6966) | NodeManager metrics may return wrong negative values when NM restart |  Major | . | Yang Wang | Szilard Nemeth |
-| [YARN-8331](https://issues.apache.org/jira/browse/YARN-8331) | Race condition in NM container launched after done |  Major | . | Yang Wang | Pradeep Ambati |
-| [HDFS-13758](https://issues.apache.org/jira/browse/HDFS-13758) | DatanodeManager should throw exception if it has BlockRecoveryCommand but the block is not under construction |  Major | namenode | Wei-Chiu Chuang | chencan |
-| [YARN-8612](https://issues.apache.org/jira/browse/YARN-8612) | Fix NM Collector Service Port issue in YarnConfiguration |  Major | ATSv2 | Prabha Manepalli | Prabha Manepalli |
-| [HADOOP-15674](https://issues.apache.org/jira/browse/HADOOP-15674) | Test failure TestSSLHttpServer.testExcludedCiphers with TLS\_ECDHE\_RSA\_WITH\_AES\_128\_CBC\_SHA256 cipher suite |  Major | common | Gabor Bota | Szilard Nemeth |
-| [YARN-8640](https://issues.apache.org/jira/browse/YARN-8640) | Restore previous state in container-executor after failure |  Major | . | Jim Brennan | Jim Brennan |
-| [YARN-8679](https://issues.apache.org/jira/browse/YARN-8679) | [ATSv2] If HBase cluster is down for long time, high chances that NM ContainerManager dispatcher get blocked |  Major | . | Rohith Sharma K S | Wangda Tan |
-| [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md |  Major | documentation | Daniel Templeton | Rui Gao |
-| [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing |  Major | . | lujie | lujie |
-| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns |  Major | ipc | Steve Loughran | Kihwal Lee |
-| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate |  Major | . | Anthony Hsu | Anthony Hsu |
-| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
-| [HDFS-12716](https://issues.apache.org/jira/browse/HDFS-12716) |  'dfs.datanode.failed.volumes.tolerated' to support minimum number of volumes to be available |  Major | datanode | usharani | Ranith Sardar |
-| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted |  Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang |
-| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full |  Major | namenode | zhangwei | Daryn Sharp |
-| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 |  Minor | datanode | Jianfei Jiang | Jianfei Jiang |
-| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data |  Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko |
-| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level |  Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi |
-| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js |  Minor | hdfs, ui | yanghuafeng | yanghuafeng |
-| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null |  Major | . | Lokesh Jain | Dinesh Chitlangia |
-| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently |  Major | mrv2 | Robert Joseph Evans | Jason Darrell Lowe |
-| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
-| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
-| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky |  Major | . | Íñigo Goiri | Ayush Saxena |
-| [HADOOP-15772](https://issues.apache.org/jira/browse/HADOOP-15772) | Remove the 'Path ... should be specified as a URI' warnings on startup |  Major | conf | Arpit Agarwal | Ayush Saxena |
-| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues |  Critical | capacityscheduler | Tao Yang | Tao Yang |
-| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label |  Critical | capacityscheduler | Tao Yang | Tao Yang |
-| [YARN-8840](https://issues.apache.org/jira/browse/YARN-8840) | Add missing cleanupSSLConfig() call for TestTimelineClient test |  Minor | test, timelineclient | Aki Tanaka | Aki Tanaka |
-| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader |  Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles |
-| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long |  Blocker | . | Jason Darrell Lowe | Jason Darrell Lowe |
-| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice |  Major | . | Ayush Saxena | Ayush Saxena |
-| [HDFS-13768](https://issues.apache.org/jira/browse/HDFS-13768) |  Adding replicas to volume map makes DataNode start slowly |  Major | . | Yiqun Lin | Surendra Singh Lilhore |
-| [HADOOP-15818](https://issues.apache.org/jira/browse/HADOOP-15818) | Fix deprecated maven-surefire-plugin configuration in hadoop-kms module |  Minor | kms | Akira Ajisaka | Vidura Bhathiya Mudalige |
-| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI |  Major | . | Fei Hui | Fei Hui |
-| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance |  Blocker | . | Ben Lau | Jason Darrell Lowe |
-| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 |  Critical | tools/distcp | Ted Yu | Ted Yu |
-| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property |  Major | documentation | Jason Darrell Lowe | Suma Shivaprasad |
-| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
-| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager |  Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli |
-| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer |  Major | . | Jason Darrell Lowe | Jason Darrell Lowe |
-| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky |  Major | . | Ayush Saxena | Ayush Saxena |
-| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
-| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
-| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. |  Major | . | Sumana Sathish | Wangda Tan |
-| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart |  Major | hdfs-client | Erik Krogen | Erik Krogen |
-| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections |  Major | mrv2 | Peter Bacsko | Peter Bacsko |
-| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null |  Critical | capacityscheduler | Tao Yang | Tao Yang |
-| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact |  Critical | build | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script |  Major | build | Akira Ajisaka | Dinesh Chitlangia |
-| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 |  Minor | hdfs | Adam Antal | Ayush Saxena |
-| [YARN-7794](https://issues.apache.org/jira/browse/YARN-7794) | SLSRunner is not loading timeline service jars causing failure |  Blocker | scheduler-load-simulator | Sunil G | Yufei Gu |
-| [HADOOP-16008](https://issues.apache.org/jira/browse/HADOOP-16008) | Fix typo in CommandsManual.md |  Minor | . | Akira Ajisaka | Shweta |
-| [HADOOP-15973](https://issues.apache.org/jira/browse/HADOOP-15973) | Configuration: Included properties are not cached if resource is a stream |  Critical | . | Eric Payne | Eric Payne |
-| [YARN-9175](https://issues.apache.org/jira/browse/YARN-9175) | Null resources check in ResourceInfo for branch-3.0 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-15992](https://issues.apache.org/jira/browse/HADOOP-15992) | JSON License is included in the transitive dependency of aliyun-sdk-oss 3.0.0 |  Blocker | . | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 |  Blocker | fs/oss | wujinhu | wujinhu |
-| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp |  Major | resourcemanager, test | Ayush Saxena | Ayush Saxena |
-| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares |  Major | fairscheduler | liyakun | liyakun |
-| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds |  Major | security, test | Jason Darrell Lowe | Akira Ajisaka |
-| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon |  Major | ipc | Erik Krogen | Erik Krogen |
-| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration |  Critical | webapp | collinma | collinma |
-| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info |  Blocker | yarn | Jiandan Yang | Jiandan Yang |
-| [YARN-7088](https://issues.apache.org/jira/browse/YARN-7088) | Add application launch time to Resource Manager REST API |  Major | . | Abdullah Yousufi | Kanwaljeet Sachdev |
-| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime |  Minor | . | Prabhu Joseph | Prabhu Joseph |
-| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO |  Major | . | Kihwal Lee | Zhaohui Xin |
-| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state |  Major | . | Kuhu Shukla | Kuhu Shukla |
-| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference |  Minor | documentation | Szilard Nemeth | Adam Antal |
-| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse |  Major | common | Daryn Sharp | Daryn Sharp |
-| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN |  Critical | datanode | star | star |
-| [YARN-5714](https://issues.apache.org/jira/browse/YARN-5714) | ContainerExecutor does not order environment map |  Trivial | nodemanager | Remi Catherinot | Remi Catherinot |
-| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed |  Major | ipc | Erik Krogen | Erik Krogen |
-| [HDFS-14399](https://issues.apache.org/jira/browse/HDFS-14399) | Backport HDFS-10536 to branch-2 |  Critical | . | Chao Sun | Chao Sun |
-| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes |  Minor | . | Wanqiang Ji | Wanqiang Ji |
-| [HDFS-14414](https://issues.apache.org/jira/browse/HDFS-14414) | Clean up findbugs warning in branch-2 |  Major | . | Wei-Chiu Chuang | Dinesh Chitlangia |
-| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. |  Minor | documentation | Chris Nauroth | Masatake Iwasaki |
-| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby |  Major | namenode | yunjiong zhao | yunjiong zhao |
-| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap |  Blocker | . | xuzq | xuzq |
-| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type |  Minor | yarn | Ahmed Hussein | Ahmed Hussein |
-| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished |  Major | namenode | Erik Krogen | Erik Krogen |
-| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml |  Major | . | Wanqiang Ji | Akira Ajisaka |
-| [HDFS-14514](https://issues.apache.org/jira/browse/HDFS-14514) | Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2 |  Major | encryption, hdfs, snapshots | Siyao Meng | Siyao Meng |
-| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) |  Major | . | Shen Yinjie | Ayush Saxena |
-| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. |  Major | . | Kihwal Lee | Kihwal Lee |
-| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
-| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics |  Major | ipc | Erik Krogen | Erik Krogen |
-| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() |  Major | . | Konstantin Shvachko | Shweta |
-| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read |  Major | hdfs-client | Zheng Hu | Zheng Hu |
-| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE |  Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu |
-| [HDFS-13770](https://issues.apache.org/jira/browse/HDFS-13770) | dfsadmin -report does not always decrease "missing blocks (with replication factor 1)" metrics when file is deleted |  Major | hdfs | Kitti Nanasi | Kitti Nanasi |
-| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock |  Major | test | Kihwal Lee | Zsolt Venczel |
-| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. |  Major | . | Yicong Cai | Yicong Cai |
-| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) |  When evictableMmapped or evictable size is zero, do not throw NoSuchElementException |  Major | hdfs-client, performance | Zheng Hu | Lisheng Sun |
-| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java |  Trivial | . | hemanthboyina | hemanthboyina |
-| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor |  Critical | namenode | Daryn Sharp | Xiaoqiao He |
-| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY |  Major | hdfs | Jiandan Yang | Weiwei Yang |
-| [HADOOP-16386](https://issues.apache.org/jira/browse/HADOOP-16386) | FindBugs warning in branch-2: GlobalStorageStatistics defines non-transient non-serializable instance field map |  Major | fs | Wei-Chiu Chuang | Masatake Iwasaki |
-| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test |  Major | test | Masatake Iwasaki | Masatake Iwasaki |
-| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build |  Minor | test | Rushabh Shah | kevin su |
-| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC |  Major | ipc | Steve Loughran | Steve Loughran |
-| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs |  Major | common, security | Erik Krogen | Erik Krogen |
-| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider |  Major | . | Chao Sun | Chao Sun |
-| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission |  Major | . | Yicong Cai | Yicong Cai |
-| [HDFS-14672](https://issues.apache.org/jira/browse/HDFS-14672) | Backport HDFS-12703 to branch-2 |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
-| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever |  Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich |
-| [HDFS-14464](https://issues.apache.org/jira/browse/HDFS-14464) | Remove unnecessary log message from DFSInputStream |  Trivial | . | Kihwal Lee | Chao Sun |
-| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly |  Major | . | hemanthboyina | hemanthboyina |
-| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved |  Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan |
-| [HADOOP-15237](https://issues.apache.org/jira/browse/HADOOP-15237) | In KMS docs there should be one space between KMS\_LOG and NOTE |  Minor | kms | Snigdhanjali Mishra | Snigdhanjali Mishra |
-| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException |  Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira |
-| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. |  Major | . | Jinglun | Jinglun |
-| [HDFS-14724](https://issues.apache.org/jira/browse/HDFS-14724) | Fix JDK7 compatibility in branch-2 |  Blocker | . | Wei-Chiu Chuang | Chen Liang |
-| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS |  Major | webhdfs | Jing Wang | Masatake Iwasaki |
-| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot |  Major | snapshots | Yongjun Zhang | Shashikant Banerjee |
-| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage |  Major | rolling upgrades | Yicong Cai | Yicong Cai |
-| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
-| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
-| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-7585](https://issues.apache.org/jira/browse/YARN-7585) | NodeManager should go unhealthy when state store throws DBException |  Major | nodemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
-| [HDFS-14726](https://issues.apache.org/jira/browse/HDFS-14726) | Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519 |  Blocker | journal-node | Chen Liang | Chen Liang |
-| [YARN-9806](https://issues.apache.org/jira/browse/YARN-9806) | TestNMSimulator#testNMSimulator fails in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted |  Critical | . | Rohith Sharma K S | Prabhu Joseph |
-| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log |  Minor | datanode, hdfs | qiang Liu | qiang Liu |
-| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. |  Major | . | Kihwal Lee | Kihwal Lee |
-| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark |  Major | common, kms | Yuval Degani | Yuval Degani |
-| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present |  Major | . | Ranith Sardar | Ranith Sardar |
-| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down |  Critical | . | Harshakiran Reddy | Ayush Saxena |
-| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider |  Major | . | Shen Yinjie | Erik Krogen |
-| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x |  Blocker | . | Yuxuan Wang | Yuxuan Wang |
-| [HADOOP-16655](https://issues.apache.org/jira/browse/HADOOP-16655) | Change cipher suite when fetching tomcat tarball for branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-
-
-### TESTS:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HDFS-13337](https://issues.apache.org/jira/browse/HDFS-13337) | Backport HDFS-4275 to branch-2.9 |  Minor | . | Íñigo Goiri | Xiao Liang |
-| [HDFS-13503](https://issues.apache.org/jira/browse/HDFS-13503) | Fix TestFsck test failures on Windows |  Major | hdfs | Xiao Liang | Xiao Liang |
-| [HDFS-13542](https://issues.apache.org/jira/browse/HDFS-13542) | TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13551](https://issues.apache.org/jira/browse/HDFS-13551) | TestMiniDFSCluster#testClusterSetStorageCapacity does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-11700](https://issues.apache.org/jira/browse/HDFS-11700) | TestHDFSServerPorts#testBackupNodePorts doesn't pass on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13548](https://issues.apache.org/jira/browse/HDFS-13548) | TestResolveHdfsSymlink#testFcResolveAfs fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13567](https://issues.apache.org/jira/browse/HDFS-13567) | TestNameNodeMetrics#testGenerateEDEKTime,TestNameNodeMetrics#testResourceCheck should use a different cluster basedir |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13557](https://issues.apache.org/jira/browse/HDFS-13557) | TestDFSAdmin#testListOpenFiles fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13550](https://issues.apache.org/jira/browse/HDFS-13550) | TestDebugAdmin#testComputeMetaCommand fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13559](https://issues.apache.org/jira/browse/HDFS-13559) | TestBlockScanner does not close TestContext properly |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13570](https://issues.apache.org/jira/browse/HDFS-13570) | TestQuotaByStorageType,TestQuota,TestDFSOutputStream fail on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13558](https://issues.apache.org/jira/browse/HDFS-13558) | TestDatanodeHttpXFrame does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13554](https://issues.apache.org/jira/browse/HDFS-13554) | TestDatanodeRegistration#testForcedRegistration does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13556](https://issues.apache.org/jira/browse/HDFS-13556) | TestNestedEncryptionZones does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13560](https://issues.apache.org/jira/browse/HDFS-13560) | Insufficient system resources exist to complete the requested service for some tests on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13592](https://issues.apache.org/jira/browse/HDFS-13592) | TestNameNodePrunesMissingStorages#testNameNodePrunesUnreportedStorages does not shut down cluster properly |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13593](https://issues.apache.org/jira/browse/HDFS-13593) | TestBlockReaderLocalLegacy#testBlockReaderLocalLegacyWithAppend fails on Windows |  Minor | test | Anbang Hu | Anbang Hu |
-| [HDFS-13587](https://issues.apache.org/jira/browse/HDFS-13587) | TestQuorumJournalManager fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13620](https://issues.apache.org/jira/browse/HDFS-13620) | Randomize the test directory path for TestHDFSFileSystemContract |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13591](https://issues.apache.org/jira/browse/HDFS-13591) | TestDFSShell#testSetrepLow fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13632](https://issues.apache.org/jira/browse/HDFS-13632) | Randomize baseDir for MiniJournalCluster in MiniQJMHACluster for TestDFSAdminWithHA |  Minor | . | Anbang Hu | Anbang Hu |
-| [MAPREDUCE-7102](https://issues.apache.org/jira/browse/MAPREDUCE-7102) | Fix TestJavaSerialization for Windows due a mismatch line separator |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [HDFS-13652](https://issues.apache.org/jira/browse/HDFS-13652) | Randomize baseDir for MiniDFSCluster in TestBlockScanner |  Minor | . | Anbang Hu | Anbang Hu |
-| [YARN-8370](https://issues.apache.org/jira/browse/YARN-8370) | Some Node Manager tests fail on Windows due to improper path/file separator |  Minor | . | Anbang Hu | Anbang Hu |
-| [YARN-8422](https://issues.apache.org/jira/browse/YARN-8422) | TestAMSimulator failing with NPE |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [HADOOP-15532](https://issues.apache.org/jira/browse/HADOOP-15532) | TestBasicDiskValidator fails with NoSuchFileException |  Minor | . | Íñigo Goiri | Giovanni Matteo Fumarola |
-| [HDFS-13563](https://issues.apache.org/jira/browse/HDFS-13563) | TestDFSAdminWithHA times out on Windows |  Minor | . | Anbang Hu | Lukas Majercak |
-| [HDFS-13681](https://issues.apache.org/jira/browse/HDFS-13681) | Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows |  Major | test | Xiao Liang | Xiao Liang |
-| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 |  Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
-| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac |  Minor | libhdfs | John Zhuge | Akira Ajisaka |
-
-
-### SUB-TASKS:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [YARN-4081](https://issues.apache.org/jira/browse/YARN-4081) | Add support for multiple resource types in the Resource class |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-4172](https://issues.apache.org/jira/browse/YARN-4172) | Extend DominantResourceCalculator to account for all resources |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-4715](https://issues.apache.org/jira/browse/YARN-4715) | Add support to read resource types from a config file |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-4829](https://issues.apache.org/jira/browse/YARN-4829) | Add support for binary units |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-4830](https://issues.apache.org/jira/browse/YARN-4830) | Add support for resource types in the nodemanager |  Major | nodemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-5242](https://issues.apache.org/jira/browse/YARN-5242) | Update DominantResourceCalculator to consider all resource types in calculations |  Major | resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-5586](https://issues.apache.org/jira/browse/YARN-5586) | Update the Resources class to consider all resource types |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-6761](https://issues.apache.org/jira/browse/YARN-6761) | Fix build for YARN-3926 branch |  Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
-| [YARN-6786](https://issues.apache.org/jira/browse/YARN-6786) | ResourcePBImpl imports cleanup |  Trivial | resourcemanager | Daniel Templeton | Yeliang Cang |
-| [YARN-6788](https://issues.apache.org/jira/browse/YARN-6788) | Improve performance of resource profile branch |  Blocker | nodemanager, resourcemanager | Sunil G | Sunil G |
-| [YARN-6994](https://issues.apache.org/jira/browse/YARN-6994) | Remove last uses of Long from resource types code |  Minor | resourcemanager | Daniel Templeton | Daniel Templeton |
-| [YARN-6892](https://issues.apache.org/jira/browse/YARN-6892) | Improve API implementation in Resources and DominantResourceCalculator class |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
-| [YARN-6610](https://issues.apache.org/jira/browse/YARN-6610) | DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources |  Critical | resourcemanager | Daniel Templeton | Daniel Templeton |
-| [YARN-7030](https://issues.apache.org/jira/browse/YARN-7030) | Performance optimizations in Resource and ResourceUtils class |  Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
-| [YARN-7042](https://issues.apache.org/jira/browse/YARN-7042) | Clean up unit tests after YARN-6610 |  Major | test | Daniel Templeton | Daniel Templeton |
-| [YARN-6789](https://issues.apache.org/jira/browse/YARN-6789) | Add Client API to get all supported resource types from RM |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
-| [YARN-6781](https://issues.apache.org/jira/browse/YARN-6781) | ResourceUtils#initializeResourcesMap takes an unnecessary Map parameter |  Minor | resourcemanager | Daniel Templeton | Yu-Tang Lin |
-| [YARN-7067](https://issues.apache.org/jira/browse/YARN-7067) | Optimize ResourceType information display in UI |  Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
-| [YARN-7039](https://issues.apache.org/jira/browse/YARN-7039) | Fix javac and javadoc errors in YARN-3926 branch |  Major | nodemanager, resourcemanager | Sunil G | Sunil G |
-| [YARN-7093](https://issues.apache.org/jira/browse/YARN-7093) | Improve log message in ResourceUtils |  Trivial | nodemanager, resourcemanager | Sunil G | Sunil G |
-| [YARN-6933](https://issues.apache.org/jira/browse/YARN-6933) | ResourceUtils.DISALLOWED\_NAMES check is duplicated |  Major | resourcemanager | Daniel Templeton | Manikandan R |
-| [YARN-7137](https://issues.apache.org/jira/browse/YARN-7137) | Move newly added APIs to unstable in YARN-3926 branch |  Blocker | nodemanager, resourcemanager | Wangda Tan | Wangda Tan |
-| [HADOOP-14799](https://issues.apache.org/jira/browse/HADOOP-14799) | Update nimbus-jose-jwt to 4.41.1 |  Major | . | Ray Chiang | Ray Chiang |
-| [YARN-7345](https://issues.apache.org/jira/browse/YARN-7345) | GPU Isolation: Incorrect minor device numbers written to devices.deny file |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-14997](https://issues.apache.org/jira/browse/HADOOP-14997) |  Add hadoop-aliyun as dependency of hadoop-cloud-storage |  Minor | fs/oss | Genmao Yu | Genmao Yu |
-| [YARN-7143](https://issues.apache.org/jira/browse/YARN-7143) | FileNotFound handling in ResourceUtils is inconsistent |  Major | resourcemanager | Daniel Templeton | Daniel Templeton |
-| [YARN-6909](https://issues.apache.org/jira/browse/YARN-6909) | Use LightWeightedResource when number of resource types more than two |  Critical | resourcemanager | Daniel Templeton | Sunil G |
-| [HDFS-12801](https://issues.apache.org/jira/browse/HDFS-12801) | RBF: Set MountTableResolver as default file resolver |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [YARN-7430](https://issues.apache.org/jira/browse/YARN-7430) | Enable user re-mapping for Docker containers by default |  Blocker | security, yarn | Eric Yang | Eric Yang |
-| [HADOOP-15024](https://issues.apache.org/jira/browse/HADOOP-15024) | AliyunOSS: support user agent configuration and include that & Hadoop version information to oss server |  Major | fs, fs/oss | Sammi Chen | Sammi Chen |
-| [HDFS-12858](https://issues.apache.org/jira/browse/HDFS-12858) | RBF: Add router admin commands usage in HDFS commands reference doc |  Minor | documentation | Yiqun Lin | Yiqun Lin |
-| [HDFS-12835](https://issues.apache.org/jira/browse/HDFS-12835) | RBF: Fix Javadoc parameter errors |  Minor | . | Wei Yan | Wei Yan |
-| [YARN-7573](https://issues.apache.org/jira/browse/YARN-7573) | Gpu Information page could be empty for nodes without GPU |  Major | webapp, yarn-ui-v2 | Sunil G | Sunil G |
-| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. |  Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah |
-| [YARN-6704](https://issues.apache.org/jira/browse/YARN-6704) | Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService |  Major | . | Botong Huang | Botong Huang |
-| [HDFS-12875](https://issues.apache.org/jira/browse/HDFS-12875) | RBF: Complete logic for -readonly option of dfsrouteradmin add command |  Major | . | Yiqun Lin | Íñigo Goiri |
-| [YARN-7383](https://issues.apache.org/jira/browse/YARN-7383) | Node resource is not parsed correctly for resource names containing dot |  Major | nodemanager, resourcemanager | Jonathan Hung | Gergely Novák |
-| [YARN-7630](https://issues.apache.org/jira/browse/YARN-7630) | Fix AMRMToken rollover handling in AMRMProxy |  Minor | . | Botong Huang | Botong Huang |
-| [HDFS-12937](https://issues.apache.org/jira/browse/HDFS-12937) | RBF: Add more unit tests for router admin commands |  Major | test | Yiqun Lin | Yiqun Lin |
-| [YARN-7032](https://issues.apache.org/jira/browse/YARN-7032) | [ATSv2] NPE while starting hbase co-processor when HBase authorization is enabled. |  Critical | . | Rohith Sharma K S | Rohith Sharma K S |
-| [HDFS-12988](https://issues.apache.org/jira/browse/HDFS-12988) | RBF: Mount table entries not properly updated in the local cache |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-12802](https://issues.apache.org/jira/browse/HDFS-12802) | RBF: Control MountTableResolver cache size |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-12934](https://issues.apache.org/jira/browse/HDFS-12934) | RBF: Federation supports global quota |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-12972](https://issues.apache.org/jira/browse/HDFS-12972) | RBF: Display mount table quota info in Web UI and admin command |  Major | . | Yiqun Lin | Yiqun Lin |
-| [YARN-6736](https://issues.apache.org/jira/browse/YARN-6736) | Consider writing to both ats v1 & v2 from RM for smoother upgrades |  Major | timelineserver | Vrushali C | Aaron Gresch |
-| [HADOOP-15027](https://issues.apache.org/jira/browse/HADOOP-15027) | AliyunOSS: Support multi-thread pre-read to improve sequential read from Hadoop to Aliyun OSS performance |  Major | fs/oss | wujinhu | wujinhu |
-| [HDFS-12973](https://issues.apache.org/jira/browse/HDFS-12973) | RBF: Document global quota supporting in federation |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-13028](https://issues.apache.org/jira/browse/HDFS-13028) | RBF: Fix spurious TestRouterRpc#testProxyGetStats |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-12772](https://issues.apache.org/jira/browse/HDFS-12772) | RBF: Federation Router State State Store internal API |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13042](https://issues.apache.org/jira/browse/HDFS-13042) | RBF: Heartbeat Router State |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13049](https://issues.apache.org/jira/browse/HDFS-13049) | RBF: Inconsistent Router OPTS config in branch-2 and branch-3 |  Minor | . | Wei Yan | Wei Yan |
-| [YARN-7817](https://issues.apache.org/jira/browse/YARN-7817) | Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. |  Major | . | Sumana Sathish | Sunil G |
-| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. |  Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah |
-| [HDFS-13044](https://issues.apache.org/jira/browse/HDFS-13044) | RBF: Add a safe mode for the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13043](https://issues.apache.org/jira/browse/HDFS-13043) | RBF: Expose the state of the Routers in the federation |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode |  Major | . | Íñigo Goiri | Yiqun Lin |
-| [YARN-7860](https://issues.apache.org/jira/browse/YARN-7860) | Fix UT failure TestRMWebServiceAppsNodelabel#testAppsRunning |  Major | . | Weiwei Yang | Sunil G |
-| [HDFS-13119](https://issues.apache.org/jira/browse/HDFS-13119) | RBF: Manage unavailable clusters |  Major | . | Íñigo Goiri | Yiqun Lin |
-| [YARN-7223](https://issues.apache.org/jira/browse/YARN-7223) | Document GPU isolation feature |  Blocker | . | Wangda Tan | Wangda Tan |
-| [HDFS-13187](https://issues.apache.org/jira/browse/HDFS-13187) | RBF: Fix Routers information shown in the web UI |  Minor | . | Wei Yan | Wei Yan |
-| [HDFS-13184](https://issues.apache.org/jira/browse/HDFS-13184) | RBF: Improve the unit test TestRouterRPCClientRetries |  Minor | test | Yiqun Lin | Yiqun Lin |
-| [HDFS-13199](https://issues.apache.org/jira/browse/HDFS-13199) | RBF: Fix the hdfs router page missing label icon issue |  Major | federation, hdfs | maobaolong | maobaolong |
-| [HADOOP-15090](https://issues.apache.org/jira/browse/HADOOP-15090) | Add ADL troubleshooting doc |  Major | documentation, fs/adl | Steve Loughran | Steve Loughran |
-| [YARN-7919](https://issues.apache.org/jira/browse/YARN-7919) | Refactor timelineservice-hbase module into submodules |  Major | timelineservice | Haibo Chen | Haibo Chen |
-| [YARN-8003](https://issues.apache.org/jira/browse/YARN-8003) | Backport the code structure changes in YARN-7346 to branch-2 |  Major | . | Haibo Chen | Haibo Chen |
-| [HDFS-13214](https://issues.apache.org/jira/browse/HDFS-13214) | RBF: Complete document of Router configuration |  Major | . | Tao Jie | Yiqun Lin |
-| [HADOOP-15267](https://issues.apache.org/jira/browse/HADOOP-15267) | S3A multipart upload fails when SSE-C encryption is enabled |  Critical | fs/s3 | Anis Elleuch | Anis Elleuch |
-| [HDFS-13230](https://issues.apache.org/jira/browse/HDFS-13230) | RBF: ConnectionManager's cleanup task will compare each pool's own active conns with its total conns |  Minor | . | Wei Yan | Chao Sun |
-| [HDFS-13233](https://issues.apache.org/jira/browse/HDFS-13233) | RBF: MountTableResolver doesn't return the correct mount point of the given path |  Major | hdfs | wangzhiyuan | wangzhiyuan |
-| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue |  Major | federation, hdfs | Wu Weiwei | Wu Weiwei |
-| [HDFS-13232](https://issues.apache.org/jira/browse/HDFS-13232) | RBF: ConnectionPool should return first usable connection |  Minor | . | Wei Yan | Ekanth Sethuramalingam |
-| [HDFS-13240](https://issues.apache.org/jira/browse/HDFS-13240) | RBF: Update some inaccurate document descriptions |  Minor | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-11399](https://issues.apache.org/jira/browse/HDFS-11399) | Many tests fails in Windows due to injecting disk failures |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-13241](https://issues.apache.org/jira/browse/HDFS-13241) | RBF: TestRouterSafemode failed if the port 8888 is in use |  Major | hdfs, test | maobaolong | maobaolong |
-| [HDFS-13253](https://issues.apache.org/jira/browse/HDFS-13253) | RBF: Quota management incorrect parent-child relationship judgement |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-13226](https://issues.apache.org/jira/browse/HDFS-13226) | RBF: Throw the exception if mount table entry validated failed |  Major | hdfs | maobaolong | maobaolong |
-| [HADOOP-15308](https://issues.apache.org/jira/browse/HADOOP-15308) | TestConfiguration fails on Windows because of paths |  Major | test | Íñigo Goiri | Xiao Liang |
-| [HDFS-12773](https://issues.apache.org/jira/browse/HDFS-12773) | RBF: Improve State Store FS implementation |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13198](https://issues.apache.org/jira/browse/HDFS-13198) | RBF: RouterHeartbeatService throws out CachedStateStore related exceptions when starting router |  Minor | . | Wei Yan | Wei Yan |
-| [HDFS-13224](https://issues.apache.org/jira/browse/HDFS-13224) | RBF: Resolvers to support mount points across multiple subclusters |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) |  Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
-| [HADOOP-15262](https://issues.apache.org/jira/browse/HADOOP-15262) | AliyunOSS: move files under a directory in parallel when rename a directory |  Major | fs/oss | wujinhu | wujinhu |
-| [HDFS-13215](https://issues.apache.org/jira/browse/HDFS-13215) | RBF: Move Router to its own module |  Major | . | Íñigo Goiri | Wei Yan |
-| [HDFS-13307](https://issues.apache.org/jira/browse/HDFS-13307) | RBF: Improve the use of setQuota command |  Major | . | liuhongtong | liuhongtong |
-| [HDFS-13250](https://issues.apache.org/jira/browse/HDFS-13250) | RBF: Router to manage requests across multiple subclusters |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13318](https://issues.apache.org/jira/browse/HDFS-13318) | RBF: Fix FindBugs in hadoop-hdfs-rbf |  Minor | . | Íñigo Goiri | Ekanth Sethuramalingam |
-| [HDFS-12792](https://issues.apache.org/jira/browse/HDFS-12792) | RBF: Test Router-based federation using HDFSContract |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [YARN-7581](https://issues.apache.org/jira/browse/YARN-7581) | HBase filters are not constructed correctly in ATSv2 |  Major | ATSv2 | Haibo Chen | Haibo Chen |
-| [YARN-7986](https://issues.apache.org/jira/browse/YARN-7986) | ATSv2 REST API queries do not return results for uppercase application tags |  Critical | . | Charan Hebri | Charan Hebri |
-| [HDFS-12512](https://issues.apache.org/jira/browse/HDFS-12512) | RBF: Add WebHDFS |  Major | fs | Íñigo Goiri | Wei Yan |
-| [HDFS-13291](https://issues.apache.org/jira/browse/HDFS-13291) | RBF: Implement available space based OrderResolver |  Major | . | Yiqun Lin | Yiqun Lin |
-| [HDFS-13204](https://issues.apache.org/jira/browse/HDFS-13204) | RBF: Optimize name service safe mode icon |  Minor | . | liuhongtong | liuhongtong |
-| [HDFS-13352](https://issues.apache.org/jira/browse/HDFS-13352) | RBF: Add xsl stylesheet for hdfs-rbf-default.xml |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
-| [YARN-8010](https://issues.apache.org/jira/browse/YARN-8010) | Add config in FederationRMFailoverProxy to not bypass facade cache when failing over |  Minor | . | Botong Huang | Botong Huang |
-| [HDFS-13347](https://issues.apache.org/jira/browse/HDFS-13347) | RBF: Cache datanode reports |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13289](https://issues.apache.org/jira/browse/HDFS-13289) | RBF: TestConnectionManager#testCleanup() test case need correction |  Minor | . | Dibyendu Karmakar | Dibyendu Karmakar |
-| [HDFS-13364](https://issues.apache.org/jira/browse/HDFS-13364) | RBF: Support NamenodeProtocol in the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HADOOP-14651](https://issues.apache.org/jira/browse/HADOOP-14651) | Update okhttp version to 2.7.5 |  Major | fs/adl | Ray Chiang | Ray Chiang |
-| [YARN-6936](https://issues.apache.org/jira/browse/YARN-6936) | [Atsv2] Retrospect storing entities into sub application table from client perspective |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
-| [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
-| [YARN-8107](https://issues.apache.org/jira/browse/YARN-8107) | Give an informative message when incorrect format is used in ATSv2 filter attributes |  Major | ATSv2 | Charan Hebri | Rohith Sharma K S |
-| [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit |  Major | . | Botong Huang | Botong Huang |
-| [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix  java doc for StateStoreFileSystemImpl |  Minor | hdfs | Yiran Wu | Yiran Wu |
-| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit |  Major | . | Wu Weiwei | Yiqun Lin |
-| [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster |  Minor | . | Wei Yan | Íñigo Goiri |
-| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java |  Trivial | federation | David Mollitor | David Mollitor |
-| [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism |  Major | fs/oss | Genmao Yu | Genmao Yu |
-| [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user |  Major | . | Shane Kumpf | Shane Kumpf |
-| [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace |  Major | . | Yiqun Lin | Yiqun Lin |
-| [YARN-7189](https://issues.apache.org/jira/browse/YARN-7189) | Container-executor doesn't remove Docker containers that error out early |  Major | yarn | Eric Badger | Eric Badger |
-| [HDFS-13466](https://issues.apache.org/jira/browse/HDFS-13466) | RBF: Add more router-related information to the UI |  Minor | . | Wei Yan | Wei Yan |
-| [HDFS-13453](https://issues.apache.org/jira/browse/HDFS-13453) | RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
-| [HDFS-13478](https://issues.apache.org/jira/browse/HDFS-13478) | RBF: Disabled Nameservice store API |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13490](https://issues.apache.org/jira/browse/HDFS-13490) | RBF: Fix setSafeMode in the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13484](https://issues.apache.org/jira/browse/HDFS-13484) | RBF: Disable Nameservices from the federation |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13326](https://issues.apache.org/jira/browse/HDFS-13326) | RBF: Improve the interfaces to modify and view mount tables |  Minor | . | Wei Yan | Gang Li |
-| [HDFS-13499](https://issues.apache.org/jira/browse/HDFS-13499) | RBF: Show disabled name services in the UI |  Minor | . | Íñigo Goiri | Íñigo Goiri |
-| [YARN-8215](https://issues.apache.org/jira/browse/YARN-8215) | ATS v2 returns invalid YARN\_CONTAINER\_ALLOCATED\_HOST\_HTTP\_ADDRESS from NM |  Critical | ATSv2 | Yesha Vora | Rohith Sharma K S |
-| [HDFS-13508](https://issues.apache.org/jira/browse/HDFS-13508) | RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries |  Minor | . | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
-| [HDFS-13434](https://issues.apache.org/jira/browse/HDFS-13434) | RBF: Fix dead links in RBF document |  Major | documentation | Akira Ajisaka | Chetna Chaudhari |
-| [HDFS-13488](https://issues.apache.org/jira/browse/HDFS-13488) | RBF: Reject requests when a Router is overloaded |  Major | . | Íñigo Goiri | Íñigo Goiri |
-| [HDFS-13525](https://issues.apache.org/jira/browse/HDFS-13525) | RBF: Add unit test TestStateStoreDisabledNameservice |  Major | . | Yiqun Lin | Yiqun Lin |
-| [YARN-8253](https://issues.apache.org/jira/browse/YARN-8253) | HTTPS Ats v2 api call fails with "bad HTTP parsed" |  Critical | ATSv2 | Yesha Vora | Charan Hebri |
-| [HADOOP-15454](https://issues.apache.org/jira/browse/HADOOP-15454) | TestRollingFileSystemSinkWithLocal fails on Windows |  Major | test | Xiao Liang | Xiao Liang |
-| [HDFS-13346](https://issues.apache.org/jira/browse/HDFS-13346) | RBF: Fix synchronization of router quota and nameservice quota |  Major | . | liuhongtong | Yiqun Lin |
-| [YARN-8247](https://issues.apache.org/jira/browse/YARN-8247) | Incorrect HTTP status code returned by ATSv2 for non-whitelisted users |  Critical | ATSv2 | Charan Hebri | Rohith Sharma K S |
-| [YARN-8130](https://issues.apache.org/jira/browse/YARN-8130) | Race condition when container events are published for KILLED applications |  Major | ATSv2 | Charan Hebri | Rohith Sharma K S |
-| [YARN-7900](https://issues.apache.org/jira/browse/YARN-7900) | [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor |  Major | . | Botong Huang | Botong Huang |
-| [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows |  Minor | . | Anbang Hu | Anbang Hu |
-| [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE |  Critical | federation | CR Hota | CR Hota |
-| [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. |  Major | scheduler | Wangda Tan | Eric Payne |
-| [HADOOP-15506](https://issues.apache.org/jira/browse/HADOOP-15506) | Upgrade Azure Storage Sdk version to 7.0.0 and update corresponding code blocks |  Minor | fs/azure | Esfandiar Manii | Esfandiar Manii |
-| [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
-| [HADOOP-15533](https://issues.apache.org/jira/browse/HADOOP-15533) | Make WASB listStatus messages consistent |  Trivial | fs/azure | Esfandiar Manii | Esfandiar Manii |
-| [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows |  Minor | test | Xiao Liang | Xiao Liang |
-| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters |  Minor | amrmproxy, federation | Botong Huang | Botong Huang |
-| [HDFS-13528](https://issues.apache.org/jira/browse/HDFS-13528) | RBF: If a directory exceeds quota limit then quota usage is not refreshed for other mount entries |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
-| [HDFS-13710](https://issues.apache.org/jira/browse/HDFS-13710) | RBF:  setQuota and getQuotaUsage should check the dfs.federation.router.quota.enable |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
-| [HDFS-13726](https://issues.apache.org/jira/browse/HDFS-13726) | RBF: Fix RBF configuration links |  Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
-| [HDFS-13475](https://issues.apache.org/jira/browse/HDFS-13475) | RBF: Admin cannot enforce Router enter SafeMode |  Major | . | Wei Yan | Chao Sun |
-| [HDFS-13733](https://issues.apache.org/jira/browse/HDFS-13733) | RBF: Add Web UI configurations and descriptions to RBF document |  Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
-| [HDFS-13743](https://issues.apache.org/jira/browse/HDFS-13743) | RBF: Router throws NullPointerException due to the invalid initialization of MountTableResolver |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
-| [HDFS-13583](https://issues.apache.org/jira/browse/HDFS-13583) | RBF: Router admin clrQuota is not synchronized with nameservice |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
-| [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
-| [YARN-8129](https://issues.apache.org/jira/browse/YARN-8129) | Improve error message for invalid value in fields attribute |  Minor | ATSv2 | Charan Hebri | Abhishek Modi |
-| [YARN-8581](https://issues.apache.org/jira/browse/YARN-8581) | [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy |  Major | amrmproxy, federation | Botong Huang | Botong Huang |
-| [YARN-8673](https://issues.apache.org/jira/browse/YARN-8673) | [AMRMProxy] More robust responseId resync after an YarnRM master slave switch |  Major | amrmproxy | Botong Huang | Botong Huang |
-| [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers |  Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
-| [HDFS-13634](https://issues.apache.org/jira/browse/HDFS-13634) | RBF: Configurable value in xml for async connection request queue size. |  Major | federation | CR Hota | CR Hota |
-| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows |  Major | . | Botong Huang | Botong Huang |
-| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 |  Major | fs/oss | wujinhu | wujinhu |
-| [HADOOP-15748](https://issues.apache.org/jira/browse/HADOOP-15748) | S3 listing inconsistency can raise NPE in globber |  Major | fs | Steve Loughran | Steve Loughran |
-| [YARN-8696](https://issues.apache.org/jira/browse/YARN-8696) | [AMRMProxy] FederationInterceptor upgrade: home sub-cluster heartbeat async |  Major | nodemanager | Botong Huang | Botong Huang |
-| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS |  Major | fs/oss | wujinhu | wujinhu |
-| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module |  Major | . | Íñigo Goiri | Chao Sun |
-| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor |  Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang |
-| [YARN-6989](https://issues.apache.org/jira/browse/YARN-6989) | Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way |  Major | timelineserver | Vrushali C | Abhishek Modi |
-| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads |  Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi |
-| [HADOOP-15837](https://issues.apache.org/jira/browse/HADOOP-15837) | DynamoDB table Update can fail S3A FS init |  Major | fs/s3 | Steve Loughran | Steve Loughran |
-| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream |  Critical | . | wujinhu | wujinhu |
-| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client |  Major | amrmproxy, federation | Botong Huang | Botong Huang |
-| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor |  Minor | . | Bibin Chundatt | Bilwa S T |
-| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics |  Major | fs/oss | wujinhu | wujinhu |
-| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code |  Major | documentation | Akira Ajisaka | Akira Ajisaka |
-| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-9182](https://issues.apache.org/jira/browse/YARN-9182) | Backport YARN-6445 resource profile performance improvements to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9181](https://issues.apache.org/jira/browse/YARN-9181) | Backport YARN-6232 for generic resource type usage to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9177](https://issues.apache.org/jira/browse/YARN-9177) | Use resource map for app metrics in TestCombinedSystemMetricsPublisher for branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9188](https://issues.apache.org/jira/browse/YARN-9188) | Port YARN-7136 to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9187](https://issues.apache.org/jira/browse/YARN-9187) | Backport YARN-6852 for GPU-specific native changes to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9180](https://issues.apache.org/jira/browse/YARN-9180) | Port YARN-7033 NM recovery of assigned resources to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9280](https://issues.apache.org/jira/browse/YARN-9280) | Backport YARN-6620 to YARN-8200/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9174](https://issues.apache.org/jira/browse/YARN-9174) | Backport YARN-7224 for refactoring of GpuDevice class |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9289](https://issues.apache.org/jira/browse/YARN-9289) | Backport YARN-7330 for GPU in UI to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext |  Major | hdfs | Shweta | Shweta |
-| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 |  Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli |
-| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF |  Blocker | fs/s3 | Dave Christianson | Steve Loughran |
-| [YARN-9397](https://issues.apache.org/jira/browse/YARN-9397) | Fix empty NMResourceInfo object test failures in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging |  Major | fs/oss | wujinhu | wujinhu |
-| [YARN-9271](https://issues.apache.org/jira/browse/YARN-9271) | Backport YARN-6927 for resource type support in MapReduce |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9291](https://issues.apache.org/jira/browse/YARN-9291) | Backport YARN-7637 to branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9409](https://issues.apache.org/jira/browse/YARN-9409) | Port resource type changes from YARN-7237 to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [YARN-9272](https://issues.apache.org/jira/browse/YARN-9272) | Backport YARN-7738 for refreshing max allocation for multiple resource types |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-16205](https://issues.apache.org/jira/browse/HADOOP-16205) | Backporting ABFS driver from trunk to branch 2.0 |  Major | fs/azure | Esfandiar Manii | Yuan Gao |
-| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom |  Major | fs/azure | Da Zhou | Da Zhou |
-| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS |  Major | fs/oss | wujinhu | wujinhu |
-| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS |  Major | fs, webhdfs | Erik Krogen | Chao Sun |
-| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore |  Major | restapi | Prabhu Joseph | Prabhu Joseph |
-| [HDFS-14771](https://issues.apache.org/jira/browse/HDFS-14771) | Backport HDFS-14617 to branch-2 (Improve fsimage load time by writing sub-sections to the fsimage index) |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
-| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id |  Major | hdfs | Chen Liang | Chen Liang |
-| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive |  Major | hdfs | Chen Liang | Chen Liang |
-| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode |  Major | hdfs | Chen Liang | Chen Liang |
-| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. |  Major | hdfs | Konstantin Shvachko | Chen Liang |
-| [HADOOP-16630](https://issues.apache.org/jira/browse/HADOOP-16630) | Backport HADOOP-16548 - "ABFS: Config to enable/disable flush operation" to branch-2 |  Minor | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
-| [HADOOP-16631](https://issues.apache.org/jira/browse/HADOOP-16631) | Backport HADOOP-16578 - "ABFS: fileSystemExists() should not call container level apis" to Branch-2 |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
-| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode |  Major | . | Konstantin Shvachko | Erik Krogen |
-
-
-### OTHER:
-
-| JIRA | Summary | Priority | Component | Reporter | Contributor |
-|:---- |:---- | :--- |:---- |:---- |:---- |
-| [HADOOP-15149](https://issues.apache.org/jira/browse/HADOOP-15149) | CryptoOutputStream should implement StreamCapabilities |  Major | fs | Mike Drob | Xiao Chen |
-| [HADOOP-15177](https://issues.apache.org/jira/browse/HADOOP-15177) | Update the release year to 2018 |  Blocker | build | Akira Ajisaka | Bharat Viswanadham |
-| [YARN-8412](https://issues.apache.org/jira/browse/YARN-8412) | Move ResourceRequest.clone logic everywhere into a proper API |  Minor | . | Botong Huang | Botong Huang |
-| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc |  Minor | documentation, webhdfs | Siyao Meng | Siyao Meng |
-| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS |  Major | documentation | Christopher Douglas | Masatake Iwasaki |
-| [HADOOP-15711](https://issues.apache.org/jira/browse/HADOOP-15711) | Move branch-2 precommit/nightly test builds to java 8 |  Critical | . | Jonathan Hung | Jonathan Hung |
-| [HDFS-14510](https://issues.apache.org/jira/browse/HDFS-14510) | Backport HDFS-13087 to branch-2 (Snapshotted encryption zone information should be immutable) |  Major | encryption, snapshots | Wei-Chiu Chuang | Siyao Meng |
-| [HDFS-14585](https://issues.apache.org/jira/browse/HDFS-14585) | Backport HDFS-8901 Use ByteBuffer in DFSInputStream#read to branch2.9 |  Major | . | Lisheng Sun | Lisheng Sun |
-| [HDFS-14483](https://issues.apache.org/jira/browse/HDFS-14483) | Backport HDFS-14111,HDFS-3246 ByteBuffer pread interface to branch-2.9 |  Major | . | Zheng Hu | Lisheng Sun |
-| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HDFS-14725](https://issues.apache.org/jira/browse/HDFS-14725) | Backport HDFS-12914 to branch-2 (Block report leases cause missing blocks until next report) |  Major | namenode | Wei-Chiu Chuang | Xiaoqiao He |
-| [YARN-8200](https://issues.apache.org/jira/browse/YARN-8200) | Backport resource types/GPU features to branch-3.0/branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 |  Major | . | Wei-Chiu Chuang | YiSheng Lien |
-| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label |  Major | . | Jonathan Hung | Jonathan Hung |
-| [HADOOP-16544](https://issues.apache.org/jira/browse/HADOOP-16544) | update io.netty in branch-2 |  Major | . | Wei-Chiu Chuang | Masatake Iwasaki |
-| [HADOOP-16588](https://issues.apache.org/jira/browse/HADOOP-16588) | Update commons-beanutils version to 1.9.4 in branch-2 |  Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
-
-
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md
new file mode 100644
index 0000000000000..584bced722366
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md
@@ -0,0 +1,288 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 2.10.1 - 2020-09-14
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-14597](https://issues.apache.org/jira/browse/HADOOP-14597) | Native compilation broken with OpenSSL-1.1.0 because EVP\_CIPHER\_CTX has been made opaque |  Major | . | Ravi Prakash | Ravi Prakash |
+| [MAPREDUCE-7069](https://issues.apache.org/jira/browse/MAPREDUCE-7069) | Add ability to specify user environment variables individually |  Major | . | Jim Brennan | Jim Brennan |
+| [HDFS-13511](https://issues.apache.org/jira/browse/HDFS-13511) | Provide specialized exception when block length cannot be obtained |  Major | . | Ted Yu | Gabor Bota |
+| [HDFS-10659](https://issues.apache.org/jira/browse/HDFS-10659) | Namenode crashes after Journalnode re-installation in an HA cluster due to missing paxos directory |  Major | ha, journal-node | Amit Anand | star |
+| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible |  Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen |
+| [HDFS-14991](https://issues.apache.org/jira/browse/HDFS-14991) | Backport better time precision of configuration#getTimeDuration to branch-2 to support SBN read |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN |  Trivial | namenode | Rajesh Balamohan | Xiaoqiao He |
+| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN |  Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu |
+| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources |  Major | . | Jonathan Hung | Manikandan R |
+| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException |  Major | dfsclient | Xiaoqiao He | Xiaoqiao He |
+| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. |  Major | capacity scheduler, test | Eric Payne | Eric Payne |
+| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined |  Critical | capacity scheduler | Eric Payne | Eric Payne |
+| [HADOOP-16753](https://issues.apache.org/jira/browse/HADOOP-16753) | Refactor HAAdmin |  Major | ha | Akira Ajisaka | Xieming Li |
+| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness |  Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) |  Minor | mr-am | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime |  Major | capacity scheduler | Eric Payne | Eric Payne |
+| [HDFS-15046](https://issues.apache.org/jira/browse/HDFS-15046) | Backport HDFS-7060 to branch-2.10 |  Major | datanode | Wei-Chiu Chuang | Lisheng Sun |
+| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs |  Major | libhdfs | John Zhuge | Muhammad Samir Khan |
+| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16739](https://issues.apache.org/jira/browse/HADOOP-16739) | Fix native build failure of hadoop-pipes on CentOS 8 |  Major | tools/pipes | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit |  Minor | . | Eric Payne | Hemanth Boyina |
+| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [HADOOP-16882](https://issues.apache.org/jira/browse/HADOOP-16882) | Update jackson-databind to 2.9.10.2 in branch-3.1, branch-2.10 |  Blocker | . | Wei-Chiu Chuang | Lisheng Sun |
+| [HADOOP-16776](https://issues.apache.org/jira/browse/HADOOP-16776) | backport HADOOP-16775: distcp copies to s3 are randomly corrupted |  Blocker | tools/distcp | Amir Shenavandeh | Amir Shenavandeh |
+| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug |  Minor | hdfs | Chen Liang | Chen Liang |
+| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-8213](https://issues.apache.org/jira/browse/YARN-8213) | Add Capacity Scheduler performance metrics |  Critical | capacityscheduler, metrics | Weiwei Yang | Weiwei Yang |
+| [HADOOP-16952](https://issues.apache.org/jira/browse/HADOOP-16952) | Add .diff to gitignore |  Minor | . | Ayush Saxena | Ayush Saxena |
+| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts |  Major | . | Jonathan Hung | Bilwa S T |
+| [HDFS-14476](https://issues.apache.org/jira/browse/HDFS-14476) | lock too long when fix inconsistent blocks between disk and in-memory |  Major | datanode | Sean Chow | Sean Chow |
+| [YARN-8680](https://issues.apache.org/jira/browse/YARN-8680) | YARN NM: Implement Iterable Abstraction for LocalResourceTracker state |  Critical | yarn | Pradeep Ambati | Pradeep Ambati |
+| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length |  Major | . | Jonathan Hung | Bilwa S T |
+| [YARN-10260](https://issues.apache.org/jira/browse/YARN-10260) | Allow transitioning queue from DRAINING to RUNNING state |  Major | . | Jonathan Hung | Bilwa S T |
+| [HDFS-15264](https://issues.apache.org/jira/browse/HDFS-15264) | Backport Datanode detection to branch-2.10 |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-6492](https://issues.apache.org/jira/browse/YARN-6492) | Generate queue metrics for each partition |  Major | capacity scheduler | Jonathan Hung | Manikandan R |
+| [HADOOP-17047](https://issues.apache.org/jira/browse/HADOOP-17047) | TODO comments exist in trunk while the related issues are already fixed. |  Trivial | . | Rungroj Maipradit | Rungroj Maipradit |
+| [HADOOP-17090](https://issues.apache.org/jira/browse/HADOOP-17090) | Increase precommit job timeout from 5 hours to 20 hours |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10297](https://issues.apache.org/jira/browse/YARN-10297) | TestContinuousScheduling#testFairSchedulerContinuousSchedulingInitTime fails intermittently |  Major | . | Jonathan Hung | Jim Brennan |
+| [HADOOP-17127](https://issues.apache.org/jira/browse/HADOOP-17127) | Use RpcMetrics.TIMEUNIT to initialize rpc queueTime and processingTime |  Minor | common | Jim Brennan | Jim Brennan |
+| [HDFS-15404](https://issues.apache.org/jira/browse/HDFS-15404) | ShellCommandFencer should expose info about source |  Major | . | Chen Liang | Chen Liang |
+| [YARN-10343](https://issues.apache.org/jira/browse/YARN-10343) | Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. |  Major | . | Eric Payne | Eric Payne |
+| [YARN-1529](https://issues.apache.org/jira/browse/YARN-1529) | Add Localization overhead metrics to NM |  Major | nodemanager | Gera Shegalov | Jim Brennan |
+| [YARN-10251](https://issues.apache.org/jira/browse/YARN-10251) | Show extended resources on legacy RM UI. |  Major | . | Eric Payne | Eric Payne |
+| [HADOOP-17159](https://issues.apache.org/jira/browse/HADOOP-17159) | Make UGI support forceful relogin from keytab ignoring the last login time |  Major | security | Sandeep Guggilam | Sandeep Guggilam |
+| [HADOOP-17251](https://issues.apache.org/jira/browse/HADOOP-17251) | Upgrade netty-all to 4.1.50.Final on branch-2.10 |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17249](https://issues.apache.org/jira/browse/HADOOP-17249) | Upgrade jackson-databind to 2.9.10.6 on branch-2.10 |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17253](https://issues.apache.org/jira/browse/HADOOP-17253) | Upgrade zookeeper to 3.4.14 on branch-2.10 |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17254](https://issues.apache.org/jira/browse/HADOOP-17254) | Upgrade hbase to 1.4.13 on branch-2.10 |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-16918](https://issues.apache.org/jira/browse/HADOOP-16918) | Dependency update for Hadoop 2.10 |  Major | . | Wei-Chiu Chuang |  |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-9376](https://issues.apache.org/jira/browse/HDFS-9376) | TestSeveralNameNodes fails occasionally |  Major | test | Kihwal Lee | Masatake Iwasaki |
+| [MAPREDUCE-6881](https://issues.apache.org/jira/browse/MAPREDUCE-6881) | Fix warnings from Spotbugs in hadoop-mapreduce |  Major | . | Weiwei Yang | Weiwei Yang |
+| [HADOOP-14354](https://issues.apache.org/jira/browse/HADOOP-14354) | SysInfoWindows is not thread safe |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-14400](https://issues.apache.org/jira/browse/HADOOP-14400) | Fix warnings from spotbugs in hadoop-tools |  Major | tools | Weiwei Yang | Weiwei Yang |
+| [YARN-6515](https://issues.apache.org/jira/browse/YARN-6515) | Fix warnings from Spotbugs in hadoop-yarn-server-nodemanager |  Major | nodemanager | Naganarasimha G R | Naganarasimha G R |
+| [HDFS-11696](https://issues.apache.org/jira/browse/HDFS-11696) | Fix warnings from Spotbugs in hadoop-hdfs |  Major | . | Yiqun Lin | Yiqun Lin |
+| [HADOOP-14910](https://issues.apache.org/jira/browse/HADOOP-14910) | Upgrade netty-all jar to latest 4.0.x.Final |  Critical | . | Vinayakumar B | Vinayakumar B |
+| [YARN-7589](https://issues.apache.org/jira/browse/YARN-7589) | TestPBImplRecords fails with NullPointerException |  Major | . | Jason Darrell Lowe | Daniel Templeton |
+| [YARN-7739](https://issues.apache.org/jira/browse/YARN-7739) | DefaultAMSProcessor should properly check customized resource types against minimum/maximum allocation |  Blocker | . | Wangda Tan | Wangda Tan |
+| [YARN-8011](https://issues.apache.org/jira/browse/YARN-8011) | TestOpportunisticContainerAllocatorAMService#testContainerPromoteAndDemoteBeforeContainerStart fails sometimes in trunk |  Minor | . | Tao Yang | Tao Yang |
+| [HADOOP-15062](https://issues.apache.org/jira/browse/HADOOP-15062) | TestCryptoStreamsWithOpensslAesCtrCryptoCodec fails on Debian 9 |  Major | . | Miklos Szegedi | Miklos Szegedi |
+| [YARN-8004](https://issues.apache.org/jira/browse/YARN-8004) | Add unit tests for inter queue preemption for dominant resource calculator |  Critical | yarn | Sumana Sathish | Zian Chen |
+| [YARN-8210](https://issues.apache.org/jira/browse/YARN-8210) | AMRMClient logging on every heartbeat to track updation of AM RM token causes too many log lines to be generated in AM logs |  Major | yarn | Suma Shivaprasad | Suma Shivaprasad |
+| [YARN-8202](https://issues.apache.org/jira/browse/YARN-8202) | DefaultAMSProcessor should properly check units of requested custom resource types against minimum/maximum allocation |  Blocker | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-8179](https://issues.apache.org/jira/browse/YARN-8179) | Preemption does not happen due to natural\_termination\_factor when DRF is used |  Major | . | kyungwan nam | kyungwan nam |
+| [YARN-8369](https://issues.apache.org/jira/browse/YARN-8369) | Javadoc build failed due to "bad use of '\>'" |  Critical | build, docs | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-8382](https://issues.apache.org/jira/browse/YARN-8382) | cgroup file leak in NM |  Major | nodemanager | Hu Ziqian | Hu Ziqian |
+| [HDFS-12459](https://issues.apache.org/jira/browse/HDFS-12459) | Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API |  Major | webhdfs | Weiwei Yang | Weiwei Yang |
+| [HDFS-14054](https://issues.apache.org/jira/browse/HDFS-14054) | TestLeaseRecovery2: testHardLeaseRecoveryAfterNameNodeRestart2 and testHardLeaseRecoveryWithRenameAfterNameNodeRestart are flaky |  Major | . | Zsolt Venczel | Zsolt Venczel |
+| [YARN-9205](https://issues.apache.org/jira/browse/YARN-9205) | When using custom resource type, application will fail to run due to the CapacityScheduler throws InvalidResourceRequestException(GREATER\_THEN\_MAX\_ALLOCATION) |  Critical | . | Zhankun Tang | Zhankun Tang |
+| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized |  Major | scheduler | Daniel Templeton | Peter Bacsko |
+| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup |  Major | hdfs | Fengnan Li | Fengnan Li |
+| [HADOOP-16255](https://issues.apache.org/jira/browse/HADOOP-16255) | ChecksumFS.Make FileSystem.rename(path, path, options) doesn't rename checksum |  Major | fs | Steve Loughran | Jungtaek Lim |
+| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart |  Major | datanode | TanYuxin | Xiaoqiao He |
+| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename |  Major | . | Jinglun | Jinglun |
+| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width |  Major | . | Eric Badger | Eric Badger |
+| [YARN-9945](https://issues.apache.org/jira/browse/YARN-9945) | Fix javadoc in FederationProxyProviderUtil in branch-2 |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-14941](https://issues.apache.org/jira/browse/HDFS-14941) | Potential editlog race condition can cause corrupted file |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup |  Minor | hdfs | Jim Brennan | Jim Brennan |
+| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs |  Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh |
+| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path |  Minor | fs, test | zhoutai.zt | Xieming Li |
+| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later |  Minor | . | xuzq | xuzq |
+| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue |  Critical | capacity scheduler | jiulongzhu | jiulongzhu |
+| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error |  Critical | . | luhuachao | luhuachao |
+| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws  ConcurrentModificationException |  Major | datanode, performance | Ryan Wu | Aiphago |
+| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure |  Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-15005](https://issues.apache.org/jira/browse/HDFS-15005) | Backport HDFS-12300 to branch-2 |  Major | hdfs | Chao Sun | Chao Sun |
+| [HDFS-15017](https://issues.apache.org/jira/browse/HDFS-15017) | Remove redundant import of AtomicBoolean in NameNodeConnector. |  Major | balancer & mover, hdfs | Konstantin Shvachko | Chao Sun |
+| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop |  Blocker | build | Kevin Su | Kevin Su |
+| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HDFS-15048](https://issues.apache.org/jira/browse/HDFS-15048) | Fix findbug in DirectoryScanner |  Major | . | Takanobu Asanuma | Masatake Iwasaki |
+| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. |  Major | test | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 |  Major | . | Takanobu Asanuma | Ayush Saxena |
+| [HADOOP-16789](https://issues.apache.org/jira/browse/HADOOP-16789) | In TestZKFailoverController, restore changes from HADOOP-11149 that were dropped by HDFS-6440 |  Minor | common | Jim Brennan | Jim Brennan |
+| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout |  Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently |  Major | . | Miklos Szegedi | Jim Brennan |
+| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out |  Major | nodemanager | Jason Darrell Lowe | Chandni Singh |
+| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank |  Major | conf | Jonathan Turner Eagles | Daryn Sharp |
+| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [HDFS-13339](https://issues.apache.org/jira/browse/HDFS-13339) | Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume |  Critical | datanode | liaoyuxiangqin | Zsolt Venczel |
+| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int |  Major | documentation | zhaoshengjie | zhaoshengjie |
+| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) |  Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero |  Major | . | kyungwan nam | kyungwan nam |
+| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently |  Critical | fs | Gabor Bota | Ahmed Hussein |
+| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15146](https://issues.apache.org/jira/browse/HDFS-15146) | TestBalancerRPCDelay.testBalancerRPCDelay fails intermittently |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect |  Major | . | Jason Darrell Lowe | Ahmed Hussein |
+| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. |  Major | hdfs-client | Konstantin Shvachko | Chen Liang |
+| [HDFS-15148](https://issues.apache.org/jira/browse/HDFS-15148) | dfs.namenode.send.qop.enabled should not apply to primary NN port |  Major | . | Chen Liang | Chen Liang |
+| [YARN-8292](https://issues.apache.org/jira/browse/YARN-8292) | Fix the dominant resource preemption cannot happen when some of the resource vector becomes negative |  Critical | yarn | Sumana Sathish | Wangda Tan |
+| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug |  Major | . | wangzhixiang | wangzhixiang |
+| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only |  Minor | build | Jihyun Cho | Jihyun Cho |
+| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException |  Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze |
+| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) |  Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation |  Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki |
+| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes |  Major | namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog |  Major | . | Konstantin Shvachko | Xieming Li |
+| [YARN-10140](https://issues.apache.org/jira/browse/YARN-10140) | TestTimelineAuthFilterForV2 fails due to login failures in branch-2.10 |  Major | timelineservice | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md |  Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook |
+| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 |  Blocker | . | Siyao Meng | Siyao Meng |
+| [HDFS-15204](https://issues.apache.org/jira/browse/HDFS-15204) | TestRetryCacheWithHA testRemoveCacheDescriptor fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically |  Major | scheduler, test | Prabhu Joseph | Ahmed Hussein |
+| [HDFS-11396](https://issues.apache.org/jira/browse/HDFS-11396) | TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out |  Minor | namenode, test | John Zhuge | Ayush Saxena |
+| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment |  Critical | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10227](https://issues.apache.org/jira/browse/YARN-10227) | Pull YARN-8242 back to branch-2.10 |  Major | yarn | Jim Brennan | Jim Brennan |
+| [YARN-2710](https://issues.apache.org/jira/browse/YARN-2710) | RM HA tests failed intermittently on trunk |  Major | client | Wangda Tan | Ahmed Hussein |
+| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception |  Critical | . | Hemanth Boyina | Hemanth Boyina |
+| [HADOOP-16361](https://issues.apache.org/jira/browse/HADOOP-16361) | TestSecureLogins#testValidKerberosName fails on branch-2 |  Major | security | Jim Brennan | Jim Brennan |
+| [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address |  Major | ha, namenode | Dhiraj Hegde | Dhiraj Hegde |
+| [HDFS-15297](https://issues.apache.org/jira/browse/HDFS-15297) | TestNNHandlesBlockReportPerStorage::blockReport\_02 fails intermittently in trunk |  Major | datanode, test | Mingliang Liu | Ayush Saxena |
+| [YARN-8193](https://issues.apache.org/jira/browse/YARN-8193) | YARN RM hangs abruptly (stops allocating resources) when running successive applications. |  Blocker | yarn | Zian Chen | Zian Chen |
+| [YARN-10255](https://issues.apache.org/jira/browse/YARN-10255) | fix intermittent failure TestContainerSchedulerQueuing.testContainerUpdateExecTypeGuaranteedToOpportunistic in branch-2.10 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15302](https://issues.apache.org/jira/browse/HDFS-15302) | Backport HDFS-15286 to branch-2.x |  Blocker | . | Akira Ajisaka | Hemanth Boyina |
+| [YARN-8959](https://issues.apache.org/jira/browse/YARN-8959) | TestContainerResizing fails randomly |  Minor | . | Bibin Chundatt | Ahmed Hussein |
+| [HDFS-15323](https://issues.apache.org/jira/browse/HDFS-15323) | StandbyNode fails transition to active due to insufficient transaction tailing |  Major | namenode, qjm | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. |  Major | . | Jinglun | Jinglun |
+| [YARN-9444](https://issues.apache.org/jira/browse/YARN-9444) | YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource |  Minor | api | Gergely Pollak | Gergely Pollak |
+| [HDFS-15038](https://issues.apache.org/jira/browse/HDFS-15038) | TestFsck testFsckListCorruptSnapshotFiles is failing in trunk |  Major | . | Hemanth Boyina | Hemanth Boyina |
+| [HADOOP-17045](https://issues.apache.org/jira/browse/HADOOP-17045) | `Configuration` javadoc describes supporting environment variables, but the feature is not available |  Minor | . | Nick Dimiduk | Masatake Iwasaki |
+| [HDFS-15293](https://issues.apache.org/jira/browse/HDFS-15293) | Relax the condition for accepting a fsimage when receiving a checkpoint |  Critical | namenode | Chen Liang | Chen Liang |
+| [HADOOP-17052](https://issues.apache.org/jira/browse/HADOOP-17052) | NetUtils.connect() throws unchecked exception (UnresolvedAddressException) causing clients to abort |  Major | net | Dhiraj Hegde | Dhiraj Hegde |
+| [HADOOP-17062](https://issues.apache.org/jira/browse/HADOOP-17062) | Fix shelldocs path in Jenkinsfile |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15386](https://issues.apache.org/jira/browse/HDFS-15386) | ReplicaNotFoundException keeps happening in DN after removing multiple DN's data directories |  Major | . | Toshihiro Suzuki | Toshihiro Suzuki |
+| [YARN-10300](https://issues.apache.org/jira/browse/YARN-10300) | appMasterHost not set in RM ApplicationSummary when AM fails before first heartbeat |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17059](https://issues.apache.org/jira/browse/HADOOP-17059) | ArrayIndexOfboundsException in ViewFileSystem#listStatus |  Major | viewfs | Hemanth Boyina | Hemanth Boyina |
+| [YARN-10312](https://issues.apache.org/jira/browse/YARN-10312) | Add support for yarn logs -logFile to retain backward compatibility |  Major | client | Jim Brennan | Jim Brennan |
+| [HDFS-12453](https://issues.apache.org/jira/browse/HDFS-12453) | TestDataNodeHotSwapVolumes fails in trunk Jenkins runs |  Critical | test | Arpit Agarwal | Lei (Eddy) Xu |
+| [HADOOP-17089](https://issues.apache.org/jira/browse/HADOOP-17089) | WASB: Update azure-storage-java SDK |  Critical | fs/azure | Thomas Marqardt | Thomas Marqardt |
+| [HDFS-15421](https://issues.apache.org/jira/browse/HDFS-15421) | IBR leak causes standby NN to be stuck in safe mode |  Blocker | namenode | Kihwal Lee | Akira Ajisaka |
+| [YARN-9903](https://issues.apache.org/jira/browse/YARN-9903) | Support reservations continue looking for Node Labels |  Major | . | Tarun Parimi | Jim Brennan |
+| [HADOOP-17094](https://issues.apache.org/jira/browse/HADOOP-17094) | vulnerabilities reported in jackson and jackson-databind in branch-2.10 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17120](https://issues.apache.org/jira/browse/HADOOP-17120) | Fix failure of docker image creation due to pip2 install error |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10348](https://issues.apache.org/jira/browse/YARN-10348) | Allow RM to always cancel tokens after app completes |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-14498](https://issues.apache.org/jira/browse/HDFS-14498) | LeaseManager can loop forever on the file for which create has failed |  Major | namenode | Sergey Shelukhin | Stephen O'Donnell |
+| [YARN-4771](https://issues.apache.org/jira/browse/YARN-4771) | Some containers can be skipped during log aggregation after NM restart |  Major | nodemanager | Jason Darrell Lowe | Jim Brennan |
+| [HDFS-15313](https://issues.apache.org/jira/browse/HDFS-15313) | Ensure inodes in active filesystem are not deleted during snapshot delete |  Major | snapshots | Shashikant Banerjee | Shashikant Banerjee |
+| [YARN-10363](https://issues.apache.org/jira/browse/YARN-10363) | TestRMAdminCLI.testHelp is failing in branch-2.10 |  Major | yarn | Jim Brennan | Bilwa S T |
+| [HADOOP-17184](https://issues.apache.org/jira/browse/HADOOP-17184) | Add --mvn-custom-repos parameter to yetus calls |  Major | build | Mingliang Liu | Mingliang Liu |
+| [HADOOP-17185](https://issues.apache.org/jira/browse/HADOOP-17185) | Set MAVEN\_OPTS in Jenkinsfile |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15499](https://issues.apache.org/jira/browse/HDFS-15499) | Clean up httpfs/pom.xml to remove aws-java-sdk-s3 exclusion |  Major | httpfs | Mingliang Liu | Mingliang Liu |
+| [HADOOP-17164](https://issues.apache.org/jira/browse/HADOOP-17164) | UGI loginUserFromKeytab doesn't set the last login time |  Major | security | Sandeep Guggilam | Sandeep Guggilam |
+| [YARN-4575](https://issues.apache.org/jira/browse/YARN-4575) | ApplicationResourceUsageReport should return ALL  reserved resource |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-7677](https://issues.apache.org/jira/browse/YARN-7677) | Docker image cannot set HADOOP\_CONF\_DIR |  Major | . | Eric Badger | Jim Brennan |
+| [YARN-8459](https://issues.apache.org/jira/browse/YARN-8459) | Improve Capacity Scheduler logs to debug invalid states |  Major | capacity scheduler | Wangda Tan | Wangda Tan |
+| [MAPREDUCE-7286](https://issues.apache.org/jira/browse/MAPREDUCE-7286) | Fix findbugs warnings in hadoop-mapreduce-project on branch-2.10 |  Minor | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17204](https://issues.apache.org/jira/browse/HADOOP-17204) | Fix typo in Hadoop KMS document |  Trivial | documentation, kms | Akira Ajisaka | Xieming Li |
+| [HADOOP-17202](https://issues.apache.org/jira/browse/HADOOP-17202) | Fix findbugs warnings in hadoop-tools on branch-2.10 |  Minor | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-15290](https://issues.apache.org/jira/browse/HDFS-15290) | NPE in HttpServer during NameNode startup |  Major | namenode | Konstantin Shvachko | Simbarashe Dzinamarira |
+| [YARN-10358](https://issues.apache.org/jira/browse/YARN-10358) | Fix findbugs warnings in hadoop-yarn-project on branch-2.10 |  Minor | . | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10177](https://issues.apache.org/jira/browse/YARN-10177) | Backport YARN-7307 to branch-2.10 Allow client/AM update supported resource types via YARN APIs |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-9404](https://issues.apache.org/jira/browse/YARN-9404) | TestApplicationLifetimeMonitor#testApplicationLifetimeMonitor fails intermittent |  Major | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15125](https://issues.apache.org/jira/browse/HDFS-15125) | Pull back HDFS-11353, HDFS-13993, HDFS-13945, and HDFS-14324 to branch-2.10 |  Minor | hdfs | Jim Brennan | Jim Brennan |
+| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature |  Major | fs, test | John Zhuge | Jim Brennan |
+| [HADOOP-17205](https://issues.apache.org/jira/browse/HADOOP-17205) | Move personality file from Yetus to Hadoop repository |  Major | test, yetus | Chao Sun | Chao Sun |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-14056](https://issues.apache.org/jira/browse/HADOOP-14056) | Update maven-javadoc-plugin to 2.10.4 |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12940](https://issues.apache.org/jira/browse/HADOOP-12940) | Fix warnings from Spotbugs in hadoop-common |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-7411](https://issues.apache.org/jira/browse/YARN-7411) | Inter-Queue preemption's computeFixpointAllocation need to handle absolute resources while computing normalizedGuarantee |  Major | resourcemanager | Sunil G | Sunil G |
+| [YARN-7541](https://issues.apache.org/jira/browse/YARN-7541) | Node updates don't update the maximum cluster capability for resources other than CPU and memory |  Critical | resourcemanager | Daniel Templeton | Daniel Templeton |
+| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails |  Major | . | Akira Ajisaka | Zsolt Venczel |
+| [HDFS-13404](https://issues.apache.org/jira/browse/HDFS-13404) | RBF: TestRouterWebHDFSContractAppend.testRenameFileBeingAppended fails |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16652](https://issues.apache.org/jira/browse/HADOOP-16652) | Backport HADOOP-16587 - "Make AAD endpoint configurable on all Auth flows" to branch-2 |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources |  Major | . | Manikandan R | Manikandan R |
+| [HADOOP-16598](https://issues.apache.org/jira/browse/HADOOP-16598) | Backport "HADOOP-16558 [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes" to all active branches |  Major | common | Duo Zhang | Duo Zhang |
+| [HDFS-14792](https://issues.apache.org/jira/browse/HDFS-14792) | [SBN read] StanbyNode does not come out of safemode while adding new blocks. |  Major | namenode | Konstantin Shvachko |  |
+| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit |  Major | build | Duo Zhang | Duo Zhang |
+| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml |  Minor | fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HADOOP-16609](https://issues.apache.org/jira/browse/HADOOP-16609) | Add Jenkinsfile for all active branches |  Major | build | Duo Zhang | Akira Ajisaka |
+| [HADOOP-16734](https://issues.apache.org/jira/browse/HADOOP-16734) | Backport HADOOP-16455- "ABFS: Implement FileSystem.access() method" to branch-2 |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [HADOOP-16778](https://issues.apache.org/jira/browse/HADOOP-16778) | ABFS: Backport HADOOP-16660  ABFS: Make RetryCount in ExponentialRetryPolicy Configurable to Branch-2 |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HADOOP-14918](https://issues.apache.org/jira/browse/HADOOP-14918) | Remove the Local Dynamo DB test option |  Major | fs/s3 | Steve Loughran | Gabor Bota |
+| [HADOOP-16933](https://issues.apache.org/jira/browse/HADOOP-16933) | Backport HADOOP-16890- "ABFS: Change in expiry calculation for MSI token provider" & HADOOP-16825 "ITestAzureBlobFileSystemCheckAccess failing" to branch-2 |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [HDFS-10499](https://issues.apache.org/jira/browse/HDFS-10499) | TestNameNodeMetadataConsistency#testGenerationStampInFuture Fails Intermittently |  Major | namenode, test | Hanisha Koneru | Yiqun Lin |
+| [HADOOP-17199](https://issues.apache.org/jira/browse/HADOOP-17199) | Backport HADOOP-13230 list/getFileStatus changes for preserved directory markers |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16647](https://issues.apache.org/jira/browse/HADOOP-16647) | Support OpenSSL 1.1.1 LTS |  Critical | security | Wei-Chiu Chuang | Rakesh Radhakrishnan |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md
new file mode 100644
index 0000000000000..73f8f1ea2f424
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md
@@ -0,0 +1,64 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop"  2.10.1 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [MAPREDUCE-7069](https://issues.apache.org/jira/browse/MAPREDUCE-7069) | *Major* | **Add ability to specify user environment variables individually**
+
+Environment variables for MapReduce tasks can now be specified as separate properties, e.g.:
+mapreduce.map.env.VARNAME=value
+mapreduce.reduce.env.VARNAME=value
+yarn.app.mapreduce.am.env.VARNAME=value
+yarn.app.mapreduce.am.admin.user.env.VARNAME=value
+This form of specifying environment variables is useful when the value of an environment variable contains commas.
+
+
+---
+
+* [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | *Major* | **ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address**
+
+ZKFC binds host address to "dfs.namenode.servicerpc-bind-host", if configured. Otherwise, it binds to "dfs.namenode.rpc-bind-host". If neither of those is configured, ZKFC binds itself to NameNode RPC server address (effectively "dfs.namenode.rpc-address").
+
+
+---
+
+* [HDFS-12453](https://issues.apache.org/jira/browse/HDFS-12453) | *Critical* | **TestDataNodeHotSwapVolumes fails in trunk Jenkins runs**
+
+Submitted version of patch for branch-2.10
+
+
+---
+
+* [HADOOP-17089](https://issues.apache.org/jira/browse/HADOOP-17089) | *Critical* | **WASB: Update azure-storage-java SDK**
+
+Azure WASB bug fix that can cause list results to appear empty.
+
+
+---
+
+* [YARN-7677](https://issues.apache.org/jira/browse/YARN-7677) | *Major* | **Docker image cannot set HADOOP\_CONF\_DIR**
+
+The HADOOP\_CONF\_DIR environment variable is no longer unconditionally inherited by containers even if it does not appear in the nodemanager whitelist variables specified by the yarn.nodemanager.env-whitelist property. If the whitelist property has been modified from the default to not include HADOOP\_CONF\_DIR yet containers need it to be inherited from the nodemanager's environment then the whitelist settings need to be updated to include HADOOP\_CONF\_DIR.
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/CHANGELOG.2.10.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/CHANGELOG.2.10.2.md
new file mode 100644
index 0000000000000..a72f1ba6e9894
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/CHANGELOG.2.10.2.md
@@ -0,0 +1,277 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 2.10.2 - 2022-05-31
+
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-17338](https://issues.apache.org/jira/browse/HADOOP-17338) | Intermittent S3AInputStream failures: Premature end of Content-Length delimited message body etc |  Major | fs/s3 | Yongjun Zhang | Yongjun Zhang |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-12077](https://issues.apache.org/jira/browse/HADOOP-12077) | Provide a multi-URI replication Inode for ViewFs |  Major | fs | Gera Shegalov | Gera Shegalov |
+| [HADOOP-13055](https://issues.apache.org/jira/browse/HADOOP-13055) | Implement linkMergeSlash and linkFallback for ViewFileSystem |  Major | fs, viewfs | Zhe Zhang | Manoj Govindassamy |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-13722](https://issues.apache.org/jira/browse/HADOOP-13722) | Code cleanup -- ViewFileSystem and InodeTree |  Minor | viewfs | Manoj Govindassamy | Manoj Govindassamy |
+| [YARN-1151](https://issues.apache.org/jira/browse/YARN-1151) | Ability to configure auxiliary services from HDFS-based JAR files |  Major | nodemanager | john lilley | Xuan Gong |
+| [HADOOP-15584](https://issues.apache.org/jira/browse/HADOOP-15584) | move httpcomponents version in pom.xml |  Minor | build | Brandon Scheller | Brandon Scheller |
+| [HADOOP-16208](https://issues.apache.org/jira/browse/HADOOP-16208) | Do Not Log InterruptedException in Client |  Minor | common | David Mollitor | David Mollitor |
+| [HADOOP-16052](https://issues.apache.org/jira/browse/HADOOP-16052) | Remove Subversion and Forrest from Dockerfile |  Minor | build | Akira Ajisaka | Xieming Li |
+| [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | Install yarnpkg and upgrade nodejs in Dockerfile |  Major | buid, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16811](https://issues.apache.org/jira/browse/HADOOP-16811) | Use JUnit TemporaryFolder Rule in TestFileUtils |  Minor | common, test | David Mollitor | David Mollitor |
+| [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | Update Dockerfile to use Bionic |  Major | build, test | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10451](https://issues.apache.org/jira/browse/YARN-10451) | RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. |  Major | . | Eric Payne | Eric Payne |
+| [YARN-9667](https://issues.apache.org/jira/browse/YARN-9667) | Container-executor.c duplicates messages to stdout |  Major | nodemanager, yarn | Adam Antal | Peter Bacsko |
+| [MAPREDUCE-7301](https://issues.apache.org/jira/browse/MAPREDUCE-7301) | Expose Mini MR Cluster attribute for testing |  Minor | test | Swaroopa Kadam | Swaroopa Kadam |
+| [HDFS-15567](https://issues.apache.org/jira/browse/HDFS-15567) | [SBN Read] HDFS should expose msync() API to allow downstream applications call it explicitly. |  Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
+| [YARN-10450](https://issues.apache.org/jira/browse/YARN-10450) | Add cpu and memory utilization per node and cluster-wide metrics |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15652](https://issues.apache.org/jira/browse/HDFS-15652) | Make block size from NNThroughputBenchmark configurable |  Minor | benchmarks | Hui Fei | Hui Fei |
+| [HDFS-15665](https://issues.apache.org/jira/browse/HDFS-15665) | Balancer logging improvement |  Major | balancer & mover | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-17342](https://issues.apache.org/jira/browse/HADOOP-17342) | Creating a token identifier should not do kerberos name resolution |  Major | common | Jim Brennan | Jim Brennan |
+| [YARN-10479](https://issues.apache.org/jira/browse/YARN-10479) | RMProxy should retry on SocketTimeout Exceptions |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15623](https://issues.apache.org/jira/browse/HDFS-15623) | Respect configured values of rpc.engine |  Major | hdfs | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HDFS-15717](https://issues.apache.org/jira/browse/HDFS-15717) | Improve fsck logging |  Major | logging, namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15751](https://issues.apache.org/jira/browse/HDFS-15751) | Add documentation for msync() API to filesystem.md |  Major | documentation | Konstantin Shvachko | Konstantin Shvachko |
+| [YARN-10562](https://issues.apache.org/jira/browse/YARN-10562) | Follow up changes for YARN-9833 |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-17478](https://issues.apache.org/jira/browse/HADOOP-17478) | Improve the description of hadoop.http.authentication.signature.secret.file |  Minor | documentation | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17501](https://issues.apache.org/jira/browse/HADOOP-17501) | Fix logging typo in ShutdownHookManager |  Major | common | Konstantin Shvachko | Fengnan Li |
+| [HADOOP-17354](https://issues.apache.org/jira/browse/HADOOP-17354) | Move Jenkinsfile outside of the root directory |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [MAPREDUCE-7319](https://issues.apache.org/jira/browse/MAPREDUCE-7319) | Log list of mappers at trace level in ShuffleHandler audit log |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [YARN-10626](https://issues.apache.org/jira/browse/YARN-10626) | Log resource allocation in NM log at container start time |  Major | . | Eric Badger | Eric Badger |
+| [YARN-10613](https://issues.apache.org/jira/browse/YARN-10613) | Config to allow Intra- and Inter-queue preemption to  enable/disable conservativeDRF |  Minor | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne |
+| [MAPREDUCE-7324](https://issues.apache.org/jira/browse/MAPREDUCE-7324) | ClientHSSecurityInfo class is in wrong META-INF file |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17546](https://issues.apache.org/jira/browse/HADOOP-17546) | Update Description of hadoop-http-auth-signature-secret in HttpAuthentication.md |  Minor | . | Ravuri Sushma sree | Ravuri Sushma sree |
+| [HADOOP-17570](https://issues.apache.org/jira/browse/HADOOP-17570) | Apply YETUS-1102 to re-enable GitHub comments |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | Use spotbugs-maven-plugin instead of findbugs-maven-plugin |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15931](https://issues.apache.org/jira/browse/HDFS-15931) | Fix non-static inner classes for better memory management |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17680](https://issues.apache.org/jira/browse/HADOOP-17680) | Allow ProtobufRpcEngine to be extensible |  Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HADOOP-17756](https://issues.apache.org/jira/browse/HADOOP-17756) | Increase precommit job timeout from 20 hours to 24 hours. |  Major | build | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-10834](https://issues.apache.org/jira/browse/YARN-10834) | Intra-queue preemption: apps that don't use defined custom resource won't be preempted. |  Major | . | Eric Payne | Eric Payne |
+| [HADOOP-17775](https://issues.apache.org/jira/browse/HADOOP-17775) | Remove JavaScript package from Docker environment |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-12665](https://issues.apache.org/jira/browse/HADOOP-12665) | Document hadoop.security.token.service.use\_ip |  Major | documentation | Arpit Agarwal | Akira Ajisaka |
+| [YARN-10456](https://issues.apache.org/jira/browse/YARN-10456) | RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry |  Major | resourcemanager | Eric Payne | Eric Payne |
+| [YARN-10860](https://issues.apache.org/jira/browse/YARN-10860) | Make max container per heartbeat configs refreshable |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17813](https://issues.apache.org/jira/browse/HADOOP-17813) | Checkstyle - Allow line length: 100 |  Major | . | Akira Ajisaka | Viraj Jasani |
+| [HADOOP-17819](https://issues.apache.org/jira/browse/HADOOP-17819) | Add extensions to ProtobufRpcEngine RequestHeaderProto |  Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HADOOP-17897](https://issues.apache.org/jira/browse/HADOOP-17897) | Allow nested blocks in switch case in checkstyle settings |  Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17857](https://issues.apache.org/jira/browse/HADOOP-17857) | Check real user ACLs in addition to proxied user ACLs |  Major | . | Eric Payne | Eric Payne |
+| [HADOOP-17926](https://issues.apache.org/jira/browse/HADOOP-17926) | Maven-eclipse-plugin is no longer needed since Eclipse can import Maven projects by itself. |  Minor | documentation | Rintaro Ikeda | Rintaro Ikeda |
+| [YARN-10935](https://issues.apache.org/jira/browse/YARN-10935) | AM Total Queue Limit goes below per-user AM Limit if parent is full. |  Major | capacity scheduler, capacityscheduler | Eric Payne | Eric Payne |
+| [HDFS-16257](https://issues.apache.org/jira/browse/HDFS-16257) | [HDFS] [RBF] Guava cache performance issue in Router MountTableResolver |  Major | . | Janus Chow | Janus Chow |
+| [YARN-1115](https://issues.apache.org/jira/browse/YARN-1115) | Provide optional means for a scheduler to check real user ACLs |  Major | capacity scheduler, scheduler | Eric Payne |  |
+| [HDFS-16294](https://issues.apache.org/jira/browse/HDFS-16294) | Remove invalid DataNode#CONFIG\_PROPERTY\_SIMULATED |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16301](https://issues.apache.org/jira/browse/HDFS-16301) | Improve BenchmarkThroughput#SIZE naming standardization |  Minor | benchmarks, test | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-18035](https://issues.apache.org/jira/browse/HADOOP-18035) | Skip unit test failures to run all the unit tests |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-18040](https://issues.apache.org/jira/browse/HADOOP-18040) | Use maven.test.failure.ignore instead of ignoreTestFailure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | Improve RM system metrics publisher's performance by pushing events to timeline server in batch |  Critical | resourcemanager, timelineserver | Hu Ziqian | Ashutosh Gupta |
+| [HADOOP-18093](https://issues.apache.org/jira/browse/HADOOP-18093) | Better exception handling for testFileStatusOnMountLink() in ViewFsBaseTest.java |  Trivial | . | Xing Lin | Xing Lin |
+| [HADOOP-18099](https://issues.apache.org/jira/browse/HADOOP-18099) | Upgrade bundled Tomcat to 8.5.75 |  Major | httpfs, kms | Akira Ajisaka | Ashutosh Gupta |
+| [HADOOP-18110](https://issues.apache.org/jira/browse/HADOOP-18110) | ViewFileSystem: Add Support for Localized Trash Root |  Major | common | Xing Lin | Xing Lin |
+| [HADOOP-18144](https://issues.apache.org/jira/browse/HADOOP-18144) | getTrashRoot/s in ViewFileSystem should return viewFS path, not targetFS path |  Major | common | Xing Lin | Xing Lin |
+| [HADOOP-18136](https://issues.apache.org/jira/browse/HADOOP-18136) | Verify FileUtils.unTar() handling of missing .tar files |  Minor | test, util | Steve Loughran | Steve Loughran |
+| [HDFS-16529](https://issues.apache.org/jira/browse/HDFS-16529) | Remove unnecessary setObserverRead in TestConsistentReadsObserver |  Trivial | test | wangzhaohui | wangzhaohui |
+| [HADOOP-18155](https://issues.apache.org/jira/browse/HADOOP-18155) | Refactor tests in TestFileUtil |  Trivial | common | Gautham Banasandra | Gautham Banasandra |
+| [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | Replace log4j 1.x with reload4j |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-11125](https://issues.apache.org/jira/browse/YARN-11125) | Backport YARN-6483 to branch-2.10 |  Major | resourcemanager | Ashutosh Gupta | Ashutosh Gupta |
+| [HADOOP-18172](https://issues.apache.org/jira/browse/HADOOP-18172) | Change scope of getRootFallbackLink for InodeTree to make them accessible from outside package |  Minor | . | Xing Lin | Xing Lin |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-10650](https://issues.apache.org/jira/browse/HDFS-10650) | DFSClient#mkdirs and DFSClient#primitiveMkdir should use default directory permission |  Minor | . | John Zhuge | John Zhuge |
+| [HADOOP-13965](https://issues.apache.org/jira/browse/HADOOP-13965) | Groups should be consistent in using default group mapping class |  Minor | security | Yiqun Lin | Yiqun Lin |
+| [YARN-6977](https://issues.apache.org/jira/browse/YARN-6977) | Node information is not provided for non am containers in RM logs |  Major | capacity scheduler | Sumana Sathish | Suma Shivaprasad |
+| [HADOOP-15261](https://issues.apache.org/jira/browse/HADOOP-15261) | Upgrade commons-io from 2.4 to 2.5 |  Major | minikdc | PandaMonkey | PandaMonkey |
+| [HADOOP-15331](https://issues.apache.org/jira/browse/HADOOP-15331) | Fix a race condition causing parsing error of java.io.BufferedInputStream in class org.apache.hadoop.conf.Configuration |  Major | common | Miklos Szegedi | Miklos Szegedi |
+| [YARN-8222](https://issues.apache.org/jira/browse/YARN-8222) | Fix potential NPE when gets RMApp from RM context |  Critical | . | Tao Yang | Tao Yang |
+| [HDFS-13174](https://issues.apache.org/jira/browse/HDFS-13174) | hdfs mover -p /path times out after 20 min |  Major | balancer & mover | István Fajth | István Fajth |
+| [HDFS-13723](https://issues.apache.org/jira/browse/HDFS-13723) | Occasional "Should be different group" error in TestRefreshUserMappings#testGroupMappingRefresh |  Major | security, test | Siyao Meng | Siyao Meng |
+| [YARN-7266](https://issues.apache.org/jira/browse/YARN-7266) | Timeline Server event handler threads locked |  Major | ATSv2, timelineserver | Venkata Puneet Ravuri | Prabhu Joseph |
+| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap |  Blocker | . | ZanderXu | ZanderXu |
+| [HADOOP-16334](https://issues.apache.org/jira/browse/HADOOP-16334) | Fix yetus-wrapper not working when HADOOP\_YETUS\_VERSION \>= 0.9.0 |  Major | yetus | Wanqiang Ji | Wanqiang Ji |
+| [YARN-9594](https://issues.apache.org/jira/browse/YARN-9594) | Fix missing break statement in ContainerScheduler#handle |  Major | . | lujie | lujie |
+| [YARN-9744](https://issues.apache.org/jira/browse/YARN-9744) | RollingLevelDBTimelineStore.getEntityByTime fails with NPE |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9785](https://issues.apache.org/jira/browse/YARN-9785) | Fix DominantResourceCalculator when one resource is zero |  Blocker | . | Bilwa S T | Bilwa S T |
+| [YARN-9833](https://issues.apache.org/jira/browse/YARN-9833) | Race condition when DirectoryCollection.checkDirs() runs during container launch |  Major | . | Peter Bacsko | Peter Bacsko |
+| [HDFS-14216](https://issues.apache.org/jira/browse/HDFS-14216) | NullPointerException happens in NamenodeWebHdfs |  Critical | . | lujie | lujie |
+| [YARN-9984](https://issues.apache.org/jira/browse/YARN-9984) | FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node |  Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HADOOP-16841](https://issues.apache.org/jira/browse/HADOOP-16841) | The description of hadoop.http.authentication.signature.secret.file contains outdated information |  Minor | documentation | Akira Ajisaka | Xieming Li |
+| [HADOOP-16768](https://issues.apache.org/jira/browse/HADOOP-16768) | SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data |  Major | io, test | zhao bo | Akira Ajisaka |
+| [HADOOP-17068](https://issues.apache.org/jira/browse/HADOOP-17068) | client fails forever when namenode ipaddr changed |  Major | hdfs-client | Sean Chow | Sean Chow |
+| [HADOOP-17116](https://issues.apache.org/jira/browse/HADOOP-17116) | Skip Retry INFO logging on first failover from a proxy |  Major | ha | Hanisha Koneru | Hanisha Koneru |
+| [MAPREDUCE-7294](https://issues.apache.org/jira/browse/MAPREDUCE-7294) | Only application master should upload resource to Yarn Shared Cache |  Major | mrv2 | zhenzhao wang | zhenzhao wang |
+| [YARN-10438](https://issues.apache.org/jira/browse/YARN-10438) | Handle null containerId in ClientRMService#getContainerReport() |  Major | resourcemanager | Raghvendra Singh | Shubham Gupta |
+| [MAPREDUCE-7289](https://issues.apache.org/jira/browse/MAPREDUCE-7289) | Fix wrong comment in LongLong.java |  Trivial | documentation, examples | Akira Ajisaka | Wanqiang Ji |
+| [YARN-10393](https://issues.apache.org/jira/browse/YARN-10393) | MR job live lock caused by completed state container leak in heartbeat between node manager and RM |  Major | nodemanager, yarn | zhenzhao wang | Jim Brennan |
+| [YARN-10455](https://issues.apache.org/jira/browse/YARN-10455) | TestNMProxy.testNMProxyRPCRetry is not consistent |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17223](https://issues.apache.org/jira/browse/HADOOP-17223) | update  org.apache.httpcomponents:httpclient to 4.5.13 and httpcore to 4.4.13 |  Blocker | . | Pranav Bheda | Pranav Bheda |
+| [HADOOP-17309](https://issues.apache.org/jira/browse/HADOOP-17309) | Javadoc warnings and errors are ignored in the precommit jobs |  Major | build, documentation | Akira Ajisaka | Akira Ajisaka |
+| [MAPREDUCE-7302](https://issues.apache.org/jira/browse/MAPREDUCE-7302) | Upgrading to JUnit 4.13 causes testcase TestFetcher.testCorruptedIFile() to fail |  Major | test | Peter Bacsko | Peter Bacsko |
+| [HDFS-15644](https://issues.apache.org/jira/browse/HDFS-15644) | Failed volumes can cause DNs to stop block reporting |  Major | block placement, datanode | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10467](https://issues.apache.org/jira/browse/YARN-10467) | ContainerIdPBImpl objects can be leaked in RMNodeImpl.completedContainers |  Major | resourcemanager | Haibo Chen | Haibo Chen |
+| [HADOOP-17340](https://issues.apache.org/jira/browse/HADOOP-17340) | TestLdapGroupsMapping failing -string mismatch in exception validation |  Major | test | Steve Loughran | Steve Loughran |
+| [HADOOP-17352](https://issues.apache.org/jira/browse/HADOOP-17352) | Update PATCH\_NAMING\_RULE in the personality file |  Minor | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17358](https://issues.apache.org/jira/browse/HADOOP-17358) | Improve excessive reloading of Configurations |  Major | conf | Ahmed Hussein | Ahmed Hussein |
+| [YARN-8558](https://issues.apache.org/jira/browse/YARN-8558) | NM recovery level db not cleaned up properly on container finish |  Critical | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-10498](https://issues.apache.org/jira/browse/YARN-10498) | Fix Yarn CapacityScheduler Markdown document |  Trivial | documentation | zhaoshengjie | zhaoshengjie |
+| [HDFS-15660](https://issues.apache.org/jira/browse/HDFS-15660) | StorageTypeProto is not compatiable between 3.x and 2.6 |  Major | . | Ryan Wu | Ryan Wu |
+| [HDFS-15725](https://issues.apache.org/jira/browse/HDFS-15725) | Lease Recovery never completes for a committed block which the DNs never finalize |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-17438](https://issues.apache.org/jira/browse/HADOOP-17438) | Increase docker memory limit in Jenkins |  Major | build, scripts, test, yetus | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16947](https://issues.apache.org/jira/browse/HADOOP-16947) | Stale record should be remove when MutableRollingAverages generating aggregate data. |  Major | . | Haibin Huang | Haibin Huang |
+| [HDFS-15632](https://issues.apache.org/jira/browse/HDFS-15632) | AbstractContractDeleteTest should set recursive parameter to true for recursive test cases. |  Major | . | Konstantin Shvachko | Anton Kutuzov |
+| [HDFS-10498](https://issues.apache.org/jira/browse/HDFS-10498) | Intermittent test failure org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotFileLength.testSnapshotfileLength |  Major | hdfs, snapshots | Hanisha Koneru | Jim Brennan |
+| [HADOOP-17495](https://issues.apache.org/jira/browse/HADOOP-17495) | Backport HADOOP-16947 "Stale record should be remove when MutableRollingAverages generating aggregate data." to branch 2.10 |  Major | . | Felix N | Felix N |
+| [HDFS-15801](https://issues.apache.org/jira/browse/HDFS-15801) | Backport HDFS-14582 to branch-2.10 (Failed to start DN with ArithmeticException when NULL checksum used) |  Major | . | Janus Chow | Janus Chow |
+| [YARN-10428](https://issues.apache.org/jira/browse/YARN-10428) | Zombie applications in the YARN queue using FAIR + sizebasedweight |  Critical | capacityscheduler | Guang Yang | Andras Gyori |
+| [HDFS-15792](https://issues.apache.org/jira/browse/HDFS-15792) | ClasscastException while loading FSImage |  Major | nn | Renukaprasad C | Renukaprasad C |
+| [HADOOP-17516](https://issues.apache.org/jira/browse/HADOOP-17516) | Upgrade ant to 1.10.9 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10500](https://issues.apache.org/jira/browse/YARN-10500) | TestDelegationTokenRenewer fails intermittently |  Major | test | Akira Ajisaka | Masatake Iwasaki |
+| [MAPREDUCE-7323](https://issues.apache.org/jira/browse/MAPREDUCE-7323) | Remove job\_history\_summary.py |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17510](https://issues.apache.org/jira/browse/HADOOP-17510) | Hadoop prints sensitive Cookie information. |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-15422](https://issues.apache.org/jira/browse/HDFS-15422) | Reported IBR is partially replaced with stored info when queuing. |  Critical | namenode | Kihwal Lee | Stephen O'Donnell |
+| [YARN-10651](https://issues.apache.org/jira/browse/YARN-10651) | CapacityScheduler crashed with NPE in AbstractYarnScheduler.updateNodeResource() |  Major | . | Haibo Chen | Haibo Chen |
+| [MAPREDUCE-7320](https://issues.apache.org/jira/browse/MAPREDUCE-7320) | ClusterMapReduceTestCase does not clean directories |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15849](https://issues.apache.org/jira/browse/HDFS-15849) | ExpiredHeartbeats metric should be of Type.COUNTER |  Major | metrics | Konstantin Shvachko | Qi Zhu |
+| [HADOOP-17557](https://issues.apache.org/jira/browse/HADOOP-17557) | skip-dir option is not processed by Yetus |  Major | build, precommit, yetus | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17572](https://issues.apache.org/jira/browse/HADOOP-17572) | [branch-2.10] Docker image build fails due to the removal of openjdk-7-jdk package |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17582](https://issues.apache.org/jira/browse/HADOOP-17582) | Replace GitHub App Token with GitHub OAuth token |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17586](https://issues.apache.org/jira/browse/HADOOP-17586) | Upgrade org.codehaus.woodstox:stax2-api to 4.2.1 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-10588](https://issues.apache.org/jira/browse/YARN-10588) | Percentage of queue and cluster is zero in WebUI |  Major | . | Bilwa S T | Bilwa S T |
+| [MAPREDUCE-7322](https://issues.apache.org/jira/browse/MAPREDUCE-7322) | revisiting TestMRIntermediateDataEncryption |  Major | job submission, security, test | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17592](https://issues.apache.org/jira/browse/HADOOP-17592) | Fix the wrong CIDR range example in Proxy User documentation |  Minor | documentation | Kwangsun Noh | Kwangsun Noh |
+| [MAPREDUCE-7325](https://issues.apache.org/jira/browse/MAPREDUCE-7325) | Intermediate data encryption is broken in LocalJobRunner |  Major | job submission, security | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10697](https://issues.apache.org/jira/browse/YARN-10697) | Resources are displayed in bytes in UI for schedulers other than capacity |  Major | . | Bilwa S T | Bilwa S T |
+| [HADOOP-17602](https://issues.apache.org/jira/browse/HADOOP-17602) | Upgrade JUnit to 4.13.1 |  Major | build, security, test | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7332](https://issues.apache.org/jira/browse/MAPREDUCE-7332) | Fix SpillCallBackPathsFinder to use JDK7 on branch-2.10 |  Minor | job submission, security | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10501](https://issues.apache.org/jira/browse/YARN-10501) | Can't remove all node labels after add node label without nodemanager port |  Critical | yarn | caozhiqiang | caozhiqiang |
+| [YARN-10716](https://issues.apache.org/jira/browse/YARN-10716) | Fix typo in ContainerRuntime |  Trivial | documentation | Wanqiang Ji | xishuhai |
+| [HADOOP-17603](https://issues.apache.org/jira/browse/HADOOP-17603) | Upgrade tomcat-embed-core to 7.0.108 |  Major | build, security | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17601](https://issues.apache.org/jira/browse/HADOOP-17601) | Upgrade Jackson databind in branch-2.10 to 2.9.10.7 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10733](https://issues.apache.org/jira/browse/YARN-10733) | TimelineService Hbase tests are failing with timeout error on branch-2.10 |  Major | test, timelineserver, yarn | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15963](https://issues.apache.org/jira/browse/HDFS-15963) | Unreleased volume references cause an infinite loop |  Critical | datanode | Shuyan Zhang | Shuyan Zhang |
+| [YARN-10460](https://issues.apache.org/jira/browse/YARN-10460) | Upgrading to JUnit 4.13 causes tests in TestNodeStatusUpdater to fail |  Major | nodemanager, test | Peter Bacsko | Peter Bacsko |
+| [YARN-10749](https://issues.apache.org/jira/browse/YARN-10749) | Can't remove all node labels after add node label without nodemanager port, broken by YARN-10647 |  Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy |
+| [YARN-10555](https://issues.apache.org/jira/browse/YARN-10555) |  Missing access check before getAppAttempts |  Critical | webapp | lujie | lujie |
+| [HADOOP-17718](https://issues.apache.org/jira/browse/HADOOP-17718) | Explicitly set locale in the Dockerfile |  Blocker | build | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-10770](https://issues.apache.org/jira/browse/YARN-10770) | container-executor permission is wrong in SecureContainer.md |  Major | documentation | Akira Ajisaka | Siddharth Ahuja |
+| [HDFS-15915](https://issues.apache.org/jira/browse/HDFS-15915) | Race condition with async edits logging due to updating txId outside of the namesystem log |  Major | hdfs, namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-16040](https://issues.apache.org/jira/browse/HDFS-16040) | RpcQueueTime metric counts requeued calls as unique events. |  Major | hdfs | Simbarashe Dzinamarira | Simbarashe Dzinamarira |
+| [HDFS-16068](https://issues.apache.org/jira/browse/HDFS-16068) | WebHdfsFileSystem has a possible connection leak in connection with HttpFS |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-15618](https://issues.apache.org/jira/browse/HDFS-15618) | Improve datanode shutdown latency |  Major | datanode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17769](https://issues.apache.org/jira/browse/HADOOP-17769) | Upgrade JUnit to 4.13.2 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10824](https://issues.apache.org/jira/browse/YARN-10824) | Title not set for JHS and NM webpages |  Major | . | Rajshree Mishra | Bilwa S T |
+| [MAPREDUCE-7353](https://issues.apache.org/jira/browse/MAPREDUCE-7353) | Mapreduce job fails when NM is stopped |  Major | . | Bilwa S T | Bilwa S T |
+| [HADOOP-17793](https://issues.apache.org/jira/browse/HADOOP-17793) | Better token validation |  Major | . | Artem Smotrakov | Artem Smotrakov |
+| [HDFS-16042](https://issues.apache.org/jira/browse/HDFS-16042) | DatanodeAdminMonitor scan should be delay based |  Major | datanode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17028](https://issues.apache.org/jira/browse/HADOOP-17028) | ViewFS should initialize target filesystems lazily |  Major | client-mounts, fs, viewfs | Uma Maheswara Rao G | Abhishek Das |
+| [HADOOP-17370](https://issues.apache.org/jira/browse/HADOOP-17370) | Upgrade commons-compress to 1.21 |  Major | common | Dongjoon Hyun | Akira Ajisaka |
+| [HADOOP-17886](https://issues.apache.org/jira/browse/HADOOP-17886) | Upgrade ant to 1.10.11 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17885](https://issues.apache.org/jira/browse/HADOOP-17885) | Upgrade JSON smart to 1.3.3 on branch-2.10 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-16207](https://issues.apache.org/jira/browse/HDFS-16207) | Remove NN logs stack trace for non-existent xattr query |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown |  Major | . | Eungsop Yoo | Eungsop Yoo |
+| [HDFS-16233](https://issues.apache.org/jira/browse/HDFS-16233) | Do not use exception handler to implement copy-on-write for EnumCounters |  Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16181](https://issues.apache.org/jira/browse/HDFS-16181) | [SBN Read] Fix metric of RpcRequestCacheMissAmount can't display when tailEditLog form JN |  Critical | . | wangzhaohui | wangzhaohui |
+| [YARN-8127](https://issues.apache.org/jira/browse/YARN-8127) | Resource leak when async scheduling is enabled |  Critical | . | Weiwei Yang | Tao Yang |
+| [HADOOP-17964](https://issues.apache.org/jira/browse/HADOOP-17964) | Increase Java heap size for running Maven in Dockerfile of branch-2.10 |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-16532](https://issues.apache.org/jira/browse/HADOOP-16532) | Fix TestViewFsTrash to use the correct homeDir. |  Minor | test, viewfs | Steve Loughran | Xing Lin |
+| [HADOOP-17965](https://issues.apache.org/jira/browse/HADOOP-17965) | Fix documentation build failure using JDK 7 on branch-2.10 |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-7612](https://issues.apache.org/jira/browse/HDFS-7612) | TestOfflineEditsViewer.testStored() uses incorrect default value for cacheDir |  Major | test | Konstantin Shvachko | Michael Kuchenbecker |
+| [HADOOP-17978](https://issues.apache.org/jira/browse/HADOOP-17978) | Exclude ASF license check for pkg-resolver JSON |  Major | build | Gautham Banasandra | Gautham Banasandra |
+| [HADOOP-17880](https://issues.apache.org/jira/browse/HADOOP-17880) | Build Hadoop on Centos 7 |  Major | build | baizhendong |  |
+| [HADOOP-17988](https://issues.apache.org/jira/browse/HADOOP-17988) | Disable JIRA plugin for YETUS on Hadoop |  Critical | build | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-16311](https://issues.apache.org/jira/browse/HDFS-16311) | Metric metadataOperationRate calculation error in DataNodeVolumeMetrics |  Major | . | Tao Li | Tao Li |
+| [HADOOP-17999](https://issues.apache.org/jira/browse/HADOOP-17999) | No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem |  Major | . | Abhishek Das | Abhishek Das |
+| [YARN-9063](https://issues.apache.org/jira/browse/YARN-9063) | ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing |  Major | timelineserver, timelineservice | Tarun Parimi | Ashutosh Gupta |
+| [HADOOP-18049](https://issues.apache.org/jira/browse/HADOOP-18049) | Pin python lazy-object-proxy to 1.6.0 in Docker file as newer versions are incompatible with python2.7 |  Major | build | Dhananjay Badaya | Dhananjay Badaya |
+| [HADOOP-13500](https://issues.apache.org/jira/browse/HADOOP-13500) | Synchronizing iteration of Configuration properties object |  Major | conf | Jason Darrell Lowe | Dhananjay Badaya |
+| [YARN-10178](https://issues.apache.org/jira/browse/YARN-10178) | Global Scheduler async thread crash caused by 'Comparison method violates its general contract |  Major | capacity scheduler | tuyu | Andras Gyori |
+| [HDFS-16410](https://issues.apache.org/jira/browse/HDFS-16410) | Insecure Xml parsing in OfflineEditsXmlLoader |  Minor | . | Ashutosh Gupta | Ashutosh Gupta |
+| [HADOOP-18158](https://issues.apache.org/jira/browse/HADOOP-18158) | Fix failure of create-release script due to releasedocmaker changes in branch-2.10 |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-18129](https://issues.apache.org/jira/browse/HADOOP-18129) | Change URI[] in INodeLink to String[] to reduce memory footprint of ViewFileSystem |  Major | . | Abhishek Das | Abhishek Das |
+| [HDFS-16517](https://issues.apache.org/jira/browse/HDFS-16517) | In 2.10 the distance metric is wrong for non-DN machines |  Major | . | Owen O'Malley | Owen O'Malley |
+| [HDFS-16518](https://issues.apache.org/jira/browse/HDFS-16518) | KeyProviderCache close cached KeyProvider with Hadoop ShutdownHookManager |  Major | hdfs | Lei Yang | Lei Yang |
+| [HADOOP-18169](https://issues.apache.org/jira/browse/HADOOP-18169) | getDelegationTokens in ViewFs should also fetch the token from the fallback FS |  Major | . | Xing Lin | Xing Lin |
+| [YARN-10720](https://issues.apache.org/jira/browse/YARN-10720) | YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging |  Critical | . | Qi Zhu | Qi Zhu |
+| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice |  Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [MAPREDUCE-7246](https://issues.apache.org/jira/browse/MAPREDUCE-7246) |  In MapredAppMasterRest#Mapreduce\_Application\_Master\_Info\_API, the datatype of appId should be "string". |  Major | documentation | jenny | Ashutosh Gupta |
+| [YARN-11126](https://issues.apache.org/jira/browse/YARN-11126) | ZKConfigurationStore Java deserialisation vulnerability |  Major | yarn | Tamas Domok | Tamas Domok |
+| [YARN-11162](https://issues.apache.org/jira/browse/YARN-11162) | Set the zk acl for nodes created by ZKConfigurationStore. |  Major | resourcemanager | Owen O'Malley | Owen O'Malley |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-7011](https://issues.apache.org/jira/browse/MAPREDUCE-7011) | TestClientDistributedCacheManager::testDetermineCacheVisibilities assumes all parent dirs set other exec |  Trivial | . | Christopher Douglas | Christopher Douglas |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15133](https://issues.apache.org/jira/browse/HADOOP-15133) | [JDK9] Ignore com.sun.javadoc.\* and com.sun.tools.\* in animal-sniffer-maven-plugin to compile with Java 9 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15293](https://issues.apache.org/jira/browse/HADOOP-15293) | TestLogLevel fails on Java 9 |  Major | test | Akira Ajisaka | Takanobu Asanuma |
+| [HADOOP-15513](https://issues.apache.org/jira/browse/HADOOP-15513) | Add additional test cases to cover some corner cases for FileUtil#symlink |  Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [HADOOP-15783](https://issues.apache.org/jira/browse/HADOOP-15783) | [JDK10] TestSFTPFileSystem.testGetModifyTime fails |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16764](https://issues.apache.org/jira/browse/HADOOP-16764) | Rewrite Python example codes using Python3 |  Minor | documentation | Kengo Seki | Kengo Seki |
+| [HADOOP-16767](https://issues.apache.org/jira/browse/HADOOP-16767) | S3AInputStream reopening does not handle non IO exceptions properly |  Major | . | Sergei Poganshev | Sergei Poganshev |
+| [HADOOP-17336](https://issues.apache.org/jira/browse/HADOOP-17336) | Backport HADOOP-16005-"NativeAzureFileSystem does not support setXAttr" and HADOOP-16785. "Improve wasb and abfs resilience on double close() calls. followup to abfs close() fix." to branch-2.10 |  Major | fs/azure | Sally Zuo | Sally Zuo |
+| [HDFS-15716](https://issues.apache.org/jira/browse/HDFS-15716) | TestUpgradeDomainBlockPlacementPolicy flaky |  Major | namenode, test | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | Migrate to Python 3 and upgrade Yetus to 0.13.0 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-13975](https://issues.apache.org/jira/browse/HDFS-13975) | TestBalancer#testMaxIterationTime fails sporadically |  Major | . | Jason Darrell Lowe | Toshihiko Uchida |
+| [HDFS-16072](https://issues.apache.org/jira/browse/HDFS-16072) | TestBlockRecovery fails consistently on Branch-2.10 |  Major | datanode, test | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15659](https://issues.apache.org/jira/browse/HDFS-15659) | Set dfs.namenode.redundancy.considerLoad to false in MiniDFSCluster |  Major | test | Akira Ajisaka | Ahmed Hussein |
+| [HADOOP-17952](https://issues.apache.org/jira/browse/HADOOP-17952) | Replace Guava VisibleForTesting by Hadoop's own annotation in hadoop-common-project modules |  Major | . | Viraj Jasani | Viraj Jasani |
+| [YARN-8546](https://issues.apache.org/jira/browse/YARN-8546) | Resource leak caused by a reserved container being released more than once under async scheduling |  Major | capacity scheduler | Weiwei Yang | Tao Yang |
+| [HDFS-13248](https://issues.apache.org/jira/browse/HDFS-13248) | RBF: Namenode need to choose block location for the client |  Major | . | Wu Weiwei | Owen O'Malley |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15808](https://issues.apache.org/jira/browse/HDFS-15808) | Add metrics for FSNamesystem read/write lock hold long time |  Major | hdfs | Tao Li | Tao Li |
+| [HDFS-16298](https://issues.apache.org/jira/browse/HDFS-16298) | Improve error msg for BlockMissingException |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16312](https://issues.apache.org/jira/browse/HDFS-16312) | Fix typo for DataNodeVolumeMetrics and ProfilingFileIoEvents |  Minor | . | Tao Li | Tao Li |
+| [YARN-10540](https://issues.apache.org/jira/browse/YARN-10540) | Node page is broken in YARN UI1 and UI2 including RMWebService api for nodes |  Critical | webapp | Sunil G | Jim Brennan |
+| [HADOOP-17445](https://issues.apache.org/jira/browse/HADOOP-17445) | Update the year to 2021 |  Major | . | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-17571](https://issues.apache.org/jira/browse/HADOOP-17571) | Upgrade com.fasterxml.woodstox:woodstox-core for security reasons |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-15895](https://issues.apache.org/jira/browse/HDFS-15895) | DFSAdmin#printOpenFiles has redundant String#format usage |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-18061](https://issues.apache.org/jira/browse/HADOOP-18061) | Update the year to 2022 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-18191](https://issues.apache.org/jira/browse/HADOOP-18191) | Log retry count while handling exceptions in RetryInvocationHandler |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-18125](https://issues.apache.org/jira/browse/HADOOP-18125) | Utility to identify git commit / Jira fixVersion discrepancies for RC preparation |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16551](https://issues.apache.org/jira/browse/HDFS-16551) | Backport HADOOP-17588 to 3.3 and other active old branches. |  Major | . | Renukaprasad C | Renukaprasad C |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/RELEASENOTES.2.10.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/RELEASENOTES.2.10.2.md
new file mode 100644
index 0000000000000..ab4936c18fa32
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.2/RELEASENOTES.2.10.2.md
@@ -0,0 +1,102 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop"  2.10.2 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-10650](https://issues.apache.org/jira/browse/HDFS-10650) | *Minor* | **DFSClient#mkdirs and DFSClient#primitiveMkdir should use default directory permission**
+
+If the caller does not supply a permission, DFSClient#mkdirs and DFSClient#primitiveMkdir will create a new directory with the default directory permission 00777 now, instead of 00666.
+
+
+---
+
+* [HDFS-13174](https://issues.apache.org/jira/browse/HDFS-13174) | *Major* | **hdfs mover -p /path times out after 20 min**
+
+Mover could have fail after 20+ minutes if a block move was enqueued for this long, between two DataNodes due to an internal constant that was introduced for Balancer, but affected Mover as well.
+The internal constant can be configured with the dfs.balancer.max-iteration-time parameter after the patch, and affects only the Balancer. Default is 20 minutes.
+
+
+---
+
+* [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | *Major* | **Install yarnpkg and upgrade nodejs in Dockerfile**
+
+In the Dockerfile, nodejs is upgraded to 8.17.0 and yarn 1.12.1 is installed.
+
+
+---
+
+* [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | *Major* | **Update Dockerfile to use Bionic**
+
+The build image has been upgraded to Bionic.
+
+
+---
+
+* [HADOOP-17338](https://issues.apache.org/jira/browse/HADOOP-17338) | *Major* | **Intermittent S3AInputStream failures: Premature end of Content-Length delimited message body etc**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | *Major* | **Migrate to Python 3 and upgrade Yetus to 0.13.0**
+
+<!-- markdown -->
+- Upgraded Yetus to 0.13.0.
+- Removed determine-flaky-tests-hadoop.py.
+- Temporarily disabled shelldocs check in the Jenkins jobs due to YETUS-1099.
+
+
+---
+
+* [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | *Major* | **Use spotbugs-maven-plugin instead of findbugs-maven-plugin**
+
+Removed findbugs from the hadoop build images and added spotbugs instead.
+Upgraded SpotBugs to 4.2.2 and spotbugs-maven-plugin to 4.2.0.
+
+
+---
+
+* [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | *Critical* | **Improve RM system metrics publisher's performance by pushing events to timeline server in batch**
+
+When Timeline Service V1 or V1.5 is used, if "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch" is set to true, ResourceManager sends timeline events in batch. The default value is false. If this functionality is enabled, the maximum number that events published in batch is configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size". The default value is 1000. The interval of publishing events can be configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds". By default, it is set to 60 seconds.
+
+
+---
+
+* [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | *Major* | **Replace log4j 1.x with reload4j**
+
+log4j 1 was replaced with reload4j which is fork of log4j 1.2.17 with the goal of fixing pressing security issues.
+
+If you are depending on the hadoop artifacts in your build were explicitly excluding log4 artifacts, and now want to exclude the reload4j files, you will need to update your exclusion lists
+\<exclusion\>
+  \<groupId\>org.slf4j\</groupId\>
+  \<artifactId\>slf4j-reload4j\</artifactId\>
+\</exclusion\>
+\<exclusion\>
+  \<groupId\>ch.qos.reload4j\</groupId\>
+  \<artifactId\>reload4j\</artifactId\>
+\</exclusion\>
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGES.3.1.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGELOG.3.1.2.md
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGES.3.1.2.md
rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGELOG.3.1.2.md
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGES.3.1.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGELOG.3.1.3.md
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGES.3.1.3.md
rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGELOG.3.1.3.md
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md
new file mode 100644
index 0000000000000..ee3547ebfd2c8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md
@@ -0,0 +1,363 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.1.4 - 2020-07-21
+
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs |  Major | . | Jonathan Hung | pralabhkumar |
+| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level |  Major | . | Jonathan Hung | Varun Saxena |
+| [HDFS-14745](https://issues.apache.org/jira/browse/HDFS-14745) | Backport HDFS persistent memory read cache support to branch-3.1 |  Major | . | Feilong He | Feilong He |
+| [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | Consistent Reads from Standby Node |  Major | hdfs | Konstantin Shvachko | Konstantin Shvachko |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-8361](https://issues.apache.org/jira/browse/YARN-8361) | Change App Name Placement Rule to use App Name instead of App Id for configuration |  Major | yarn | Zian Chen | Zian Chen |
+| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics |  Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-15849](https://issues.apache.org/jira/browse/HADOOP-15849) | Upgrade netty version to 3.10.6 |  Major | . | Xiao Chen | Xiao Chen |
+| [HDFS-14064](https://issues.apache.org/jira/browse/HDFS-14064) | WEBHDFS: Support Enable/Disable EC Policy |  Major | erasure-coding, webhdfs | Ayush Saxena | Ayush Saxena |
+| [HDFS-14113](https://issues.apache.org/jira/browse/HDFS-14113) | EC : Add Configuration to restrict UserDefined Policies |  Major | erasure-coding | Ayush Saxena | Ayush Saxena |
+| [HDFS-14124](https://issues.apache.org/jira/browse/HDFS-14124) | EC : Support EC Commands (set/get/unset EcPolicy) via WebHdfs |  Major | erasure-coding, httpfs, webhdfs | Souryakanta Dwivedy | Ayush Saxena |
+| [HDFS-14006](https://issues.apache.org/jira/browse/HDFS-14006) | Refactor name node to allow different token verification implementations |  Major | . | CR Hota | CR Hota |
+| [HADOOP-15909](https://issues.apache.org/jira/browse/HADOOP-15909) | KeyProvider class should implement Closeable |  Major | kms | Kuhu Shukla | Kuhu Shukla |
+| [HDFS-14187](https://issues.apache.org/jira/browse/HDFS-14187) | Make warning message more clear when there are not enough data nodes for EC write |  Major | erasure-coding | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-14235](https://issues.apache.org/jira/browse/HDFS-14235) | Handle ArrayIndexOutOfBoundsException in DataNodeDiskMetrics#slowDiskDetectionDaemon |  Major | . | Surendra Singh Lilhore | Ranith Sardar |
+| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
+| [HADOOP-16140](https://issues.apache.org/jira/browse/HADOOP-16140) | hadoop fs expunge to add -immediate option to purge trash immediately |  Major | fs | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-15014](https://issues.apache.org/jira/browse/HADOOP-15014) | KMS should log the IP address of the clients |  Major | kms | Zsombor Gegesy | Zsombor Gegesy |
+| [HDFS-14460](https://issues.apache.org/jira/browse/HDFS-14460) | DFSUtil#getNamenodeWebAddr should return HTTPS address based on policy configured |  Major | . | CR Hota | CR Hota |
+| [HDFS-14624](https://issues.apache.org/jira/browse/HDFS-14624) | When decommissioning a node, log remaining blocks to replicate periodically |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-13693](https://issues.apache.org/jira/browse/HDFS-13693) | Remove unnecessary search in INodeDirectory.addChild during image loading |  Major | namenode | zhouyingchao | Lisheng Sun |
+| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory  instead of df/du |  Major | datanode, performance | Lisheng Sun | Lisheng Sun |
+| [HDFS-14678](https://issues.apache.org/jira/browse/HDFS-14678) | Allow triggerBlockReport to a specific namenode |  Major | datanode | Leon Gao | Leon Gao |
+| [HDFS-14523](https://issues.apache.org/jira/browse/HDFS-14523) | Remove excess read lock for NetworkToplogy |  Major | . | Wu Weiwei | Wu Weiwei |
+| [HDFS-14497](https://issues.apache.org/jira/browse/HDFS-14497) | Write lock held by metasave impact following RPC processing |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round |  Major | capacity scheduler | Eric Payne | Manikandan R |
+| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics |  Major | . | Jonathan Hung | Shubham Gupta |
+| [HADOOP-16531](https://issues.apache.org/jira/browse/HADOOP-16531) | Log more detail for slow RPC |  Major | . | Chen Zhang | Chen Zhang |
+| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16069](https://issues.apache.org/jira/browse/HADOOP-16069) | Support configure ZK\_DTSM\_ZK\_KERBEROS\_PRINCIPAL in ZKDelegationTokenSecretManager using principal with Schema /\_HOST |  Minor | common | luhuachao | luhuachao |
+| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs |  Major | . | Jonathan Hung | Manoj Kumar |
+| [HDFS-14850](https://issues.apache.org/jira/browse/HDFS-14850) | Optimize FileSystemAccessService#getFileSystemConfiguration |  Major | httpfs, performance | Lisheng Sun | Lisheng Sun |
+| [HDFS-14192](https://issues.apache.org/jira/browse/HDFS-14192) | Track missing DFS operations in Statistics and StorageStatistics |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16625](https://issues.apache.org/jira/browse/HADOOP-16625) | Backport HADOOP-14624 to branch-3.1 |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-9356](https://issues.apache.org/jira/browse/YARN-9356) | Add more tests to ratio method in TestResourceCalculator |  Major | . | Szilard Nemeth | Zoltan Siegl |
+| [HDFS-14915](https://issues.apache.org/jira/browse/HDFS-14915) | Move Superuser Check Before Taking Lock For Encryption API |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14921](https://issues.apache.org/jira/browse/HDFS-14921) | Remove SuperUser Check in Setting Storage Policy in FileStatus During Listing |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14923](https://issues.apache.org/jira/browse/HDFS-14923) | Remove dead code from HealthMonitor |  Minor | . | Fei Hui | Fei Hui |
+| [YARN-9914](https://issues.apache.org/jira/browse/YARN-9914) | Use separate configs for free disk space checking for full and not-full disks |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14942](https://issues.apache.org/jira/browse/HDFS-14942) | Change Log Level to debug in JournalNodeSyncer#syncWithJournalAtIndex |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible |  Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen |
+| [HADOOP-16705](https://issues.apache.org/jira/browse/HADOOP-16705) | MBeanInfoBuilder puts unnecessary memory pressure on the system with a debug log |  Major | metrics | Lukas Majercak | Lukas Majercak |
+| [HADOOP-16712](https://issues.apache.org/jira/browse/HADOOP-16712) | Config ha.failover-controller.active-standby-elector.zk.op.retries is not in core-default.xml |  Trivial | . | Wei-Chiu Chuang | Xieming Li |
+| [HADOOP-16703](https://issues.apache.org/jira/browse/HADOOP-16703) | Backport HADOOP-16152 to branch-3.1 |  Major | . | Siyao Meng | Siyao Meng |
+| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN |  Trivial | namenode | Rajesh Balamohan | Xiaoqiao He |
+| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9966](https://issues.apache.org/jira/browse/YARN-9966) | Code duplication in UserGroupMappingPlacementRule |  Major | . | Szilard Nemeth | Kevin Su |
+| [YARN-9937](https://issues.apache.org/jira/browse/YARN-9937) | Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16718](https://issues.apache.org/jira/browse/HADOOP-16718) | Allow disabling Server Name Indication (SNI) for Jetty |  Major | . | Siyao Meng | Aravindan Vijayan |
+| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN |  Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu |
+| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources |  Major | . | Jonathan Hung | Manikandan R |
+| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException |  Major | dfsclient | Xiaoqiao He | Xiaoqiao He |
+| [YARN-10033](https://issues.apache.org/jira/browse/YARN-10033) | TestProportionalCapacityPreemptionPolicy not initializing vcores for effective max resources |  Major | capacity scheduler, test | Eric Payne | Eric Payne |
+| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. |  Major | capacity scheduler, test | Eric Payne | Eric Payne |
+| [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0 |  Major | build | Andras Bokor | Andras Bokor |
+| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined |  Critical | capacity scheduler | Eric Payne | Eric Payne |
+| [HDFS-12999](https://issues.apache.org/jira/browse/HDFS-12999) | When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. |  Major | erasure-coding, hdfs-client | lufei | lufei |
+| [HDFS-15074](https://issues.apache.org/jira/browse/HDFS-15074) | DataNode.DataTransfer thread should catch all the expception and log it. |  Major | datanode | Surendra Singh Lilhore | Hemanth Boyina |
+| [HDFS-14740](https://issues.apache.org/jira/browse/HDFS-14740) | Recover data blocks from persistent memory read cache during datanode restarts |  Major | caching, datanode | Feilong He | Feilong He |
+| [HDFS-15097](https://issues.apache.org/jira/browse/HDFS-15097) | Purge log in KMS and HttpFS |  Minor | httpfs, kms | Doris Gu | Doris Gu |
+| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness |  Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein |
+| [YARN-7913](https://issues.apache.org/jira/browse/YARN-7913) | Improve error handling when application recovery fails with exception |  Major | resourcemanager | Gergo Repas | Wilfred Spiegelenburg |
+| [HDFS-15119](https://issues.apache.org/jira/browse/HDFS-15119) | Allow expiration of cached locations in DFSInputStream |  Minor | dfsclient | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) |  Minor | mr-am | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime |  Major | capacity scheduler | Eric Payne | Eric Payne |
+| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs |  Major | libhdfs | John Zhuge | Muhammad Samir Khan |
+| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit |  Minor | . | Eric Payne | Hemanth Boyina |
+| [HDFS-15086](https://issues.apache.org/jira/browse/HDFS-15086) | Block scheduled counter never get decremet if the block got deleted before replication. |  Major | 3.1.1 | Surendra Singh Lilhore | Hemanth Boyina |
+| [HDFS-15174](https://issues.apache.org/jira/browse/HDFS-15174) | Optimize ReplicaCachingGetSpaceUsed by reducing unnecessary io operations |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [HDFS-14861](https://issues.apache.org/jira/browse/HDFS-14861) | Reset LowRedundancyBlocks Iterator periodically |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-16882](https://issues.apache.org/jira/browse/HADOOP-16882) | Update jackson-databind to 2.9.10.2 in branch-3.1, branch-2.10 |  Blocker | . | Wei-Chiu Chuang | Lisheng Sun |
+| [HADOOP-16776](https://issues.apache.org/jira/browse/HADOOP-16776) | backport HADOOP-16775: distcp copies to s3 are randomly corrupted |  Blocker | tools/distcp | Amir Shenavandeh | Amir Shenavandeh |
+| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug |  Minor | hdfs | Chen Liang | Chen Liang |
+| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16952](https://issues.apache.org/jira/browse/HADOOP-16952) | Add .diff to gitignore |  Minor | . | Ayush Saxena | Ayush Saxena |
+| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts |  Major | . | Jonathan Hung | Bilwa S T |
+| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length |  Major | . | Jonathan Hung | Bilwa S T |
+| [HADOOP-17001](https://issues.apache.org/jira/browse/HADOOP-17001) | The suffix name of the unified compression class |  Major | io | bianqi | bianqi |
+| [HDFS-15295](https://issues.apache.org/jira/browse/HDFS-15295) | AvailableSpaceBlockPlacementPolicy should use chooseRandomWithStorageTypeTwoTrial() for better performance. |  Minor | . | Jinglun | Jinglun |
+| [HADOOP-17127](https://issues.apache.org/jira/browse/HADOOP-17127) | Use RpcMetrics.TIMEUNIT to initialize rpc queueTime and processingTime |  Minor | common | Jim Brennan | Jim Brennan |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13806](https://issues.apache.org/jira/browse/HDFS-13806) | EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory |  Minor | erasure-coding | Souryakanta Dwivedy | Ayush Saxena |
+| [HDFS-12459](https://issues.apache.org/jira/browse/HDFS-12459) | Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API |  Major | webhdfs | Weiwei Yang | Weiwei Yang |
+| [HADOOP-15418](https://issues.apache.org/jira/browse/HADOOP-15418) | Hadoop KMSAuthenticationFilter needs to use getPropsByPrefix instead of iterator to avoid ConcurrentModificationException |  Major | common | Suma Shivaprasad | Suma Shivaprasad |
+| [HDFS-14004](https://issues.apache.org/jira/browse/HDFS-14004) | TestLeaseRecovery2#testCloseWhileRecoverLease fails intermittently in trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-8948](https://issues.apache.org/jira/browse/YARN-8948) | PlacementRule interface should be for all YarnSchedulers |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon |  Major | ipc | Erik Krogen | Erik Krogen |
+| [HDFS-14175](https://issues.apache.org/jira/browse/HDFS-14175) | EC: Native XOR decoder should reset the output buffer before using it. |  Major | ec, hdfs | Surendra Singh Lilhore | Ayush Saxena |
+| [HDFS-14202](https://issues.apache.org/jira/browse/HDFS-14202) | "dfs.disk.balancer.max.disk.throughputInMBperSec" property is not working as per set value. |  Major | diskbalancer | Ranith Sardar | Ranith Sardar |
+| [HADOOP-16032](https://issues.apache.org/jira/browse/HADOOP-16032) | Distcp It should clear sub directory ACL before applying new ACL on it. |  Major | tools/distcp | Ranith Sardar | Ranith Sardar |
+| [HADOOP-16127](https://issues.apache.org/jira/browse/HADOOP-16127) | In ipc.Client, put a new connection could happen after stop |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
+| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized |  Major | scheduler | Daniel Templeton | Peter Bacsko |
+| [HADOOP-16161](https://issues.apache.org/jira/browse/HADOOP-16161) | NetworkTopology#getWeightUsingNetworkLocation return unexpected result |  Major | net | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14434](https://issues.apache.org/jira/browse/HDFS-14434) | webhdfs that connect secure hdfs should not use user.name parameter |  Minor | webhdfs | KWON BYUNGCHANG | KWON BYUNGCHANG |
+| [HDFS-14527](https://issues.apache.org/jira/browse/HDFS-14527) | Stop all DataNodes may result in NN terminate |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() |  Major | . | Konstantin Shvachko | Shweta |
+| [HDFS-14618](https://issues.apache.org/jira/browse/HDFS-14618) | Incorrect synchronization of ArrayList field (ArrayList is thread-unsafe). |  Critical | . | Paul Ward | Paul Ward |
+| [HDFS-14610](https://issues.apache.org/jira/browse/HDFS-14610) | HashMap is not thread safe. Field storageMap is typically synchronized by storageMap. However, in one place, field storageMap is not protected with synchronized. |  Critical | . | Paul Ward | Paul Ward |
+| [HDFS-14499](https://issues.apache.org/jira/browse/HDFS-14499) | Misleading REM\_QUOTA value with snapshot and trash feature enabled for a directory |  Major | snapshots | Shashikant Banerjee | Shashikant Banerjee |
+| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup |  Major | hdfs | Fengnan Li | Fengnan Li |
+| [HADOOP-16461](https://issues.apache.org/jira/browse/HADOOP-16461) | Regression: FileSystem cache lock parses XML within the lock |  Major | fs | Gopal Vijayaraghavan | Gopal Vijayaraghavan |
+| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly |  Major | . | Hemanth Boyina | Hemanth Boyina |
+| [HADOOP-12282](https://issues.apache.org/jira/browse/HADOOP-12282) | Connection thread's name should be updated after address changing is detected |  Major | ipc | zhouyingchao | Lisheng Sun |
+| [HDFS-14686](https://issues.apache.org/jira/browse/HDFS-14686) | HttpFS: HttpFSFileSystem#getErasureCodingPolicy always returns null |  Major | httpfs | Siyao Meng | Siyao Meng |
+| [HADOOP-15865](https://issues.apache.org/jira/browse/HADOOP-15865) | ConcurrentModificationException in Configuration.overlay() method |  Major | . | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-14759](https://issues.apache.org/jira/browse/HDFS-14759) | HDFS cat logs an info message |  Major | . | Eric Badger | Eric Badger |
+| [HDFS-2470](https://issues.apache.org/jira/browse/HDFS-2470) | NN should automatically set permissions on dfs.namenode.\*.dir |  Major | namenode | Aaron Myers | Siddharth Wagle |
+| [YARN-9718](https://issues.apache.org/jira/browse/YARN-9718) | Yarn REST API, services endpoint remote command ejection |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9813](https://issues.apache.org/jira/browse/YARN-9813) | RM does not start on JDK11 when UIv2 is enabled |  Critical | resourcemanager, yarn | Adam Antal | Adam Antal |
+| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted |  Critical | . | Rohith Sharma K S | Prabhu Joseph |
+| [HDFS-14838](https://issues.apache.org/jira/browse/HDFS-14838) | RBF: Display RPC (instead of HTTP) Port Number in RBF web UI |  Minor | rbf, ui | Xieming Li | Xieming Li |
+| [HDFS-14699](https://issues.apache.org/jira/browse/HDFS-14699) | Erasure Coding: Storage not considered in live replica when replication streams hard limit reached to threshold |  Critical | ec | Zhao Yi Ming | Zhao Yi Ming |
+| [YARN-9833](https://issues.apache.org/jira/browse/YARN-9833) | Race condition when DirectoryCollection.checkDirs() runs during container launch |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-9837](https://issues.apache.org/jira/browse/YARN-9837) | YARN Service fails to fetch status for Stopped apps with bigger spec files |  Major | yarn-native-services | Tarun Parimi | Tarun Parimi |
+| [YARN-2255](https://issues.apache.org/jira/browse/YARN-2255) | YARN Audit logging not added to log4j.properties |  Major | . | Varun Saxena | Aihua Xu |
+| [HDFS-14836](https://issues.apache.org/jira/browse/HDFS-14836) | FileIoProvider should not increase FileIoErrors metric in datanode volume metric |  Minor | . | Aiphago | Aiphago |
+| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark |  Major | common, kms | Yuval Degani | Yuval Degani |
+| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-13660](https://issues.apache.org/jira/browse/HDFS-13660) | DistCp job fails when new data is appended in the file while the distCp copy job is running |  Critical | distcp | Mukund Thakur | Mukund Thakur |
+| [HDFS-14808](https://issues.apache.org/jira/browse/HDFS-14808) | EC: Improper size values for corrupt ec block in LOG |  Major | ec | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14849](https://issues.apache.org/jira/browse/HDFS-14849) | Erasure Coding: the internal block is replicated many times when datanode is decommissioning |  Major | ec, erasure-coding | HuangTao | HuangTao |
+| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14492](https://issues.apache.org/jira/browse/HDFS-14492) | Snapshot memory leak |  Major | snapshots | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-14418](https://issues.apache.org/jira/browse/HDFS-14418) | Remove redundant super user priveledge checks from namenode. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14216](https://issues.apache.org/jira/browse/HDFS-14216) | NullPointerException happens in NamenodeWebHdfs |  Critical | . | lujie | lujie |
+| [HDFS-14637](https://issues.apache.org/jira/browse/HDFS-14637) | Namenode may not replicate blocks to meet the policy after enabling upgradeDomain |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14879](https://issues.apache.org/jira/browse/HDFS-14879) | Header was wrong in Snapshot web UI |  Major | . | Hemanth Boyina | Hemanth Boyina |
+| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down |  Critical | . | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14859](https://issues.apache.org/jira/browse/HDFS-14859) | Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero |  Major | hdfs | Srinivasu Majeti | Srinivasu Majeti |
+| [YARN-6715](https://issues.apache.org/jira/browse/YARN-6715) | Fix documentation about NodeHealthScriptRunner |  Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko |
+| [YARN-9552](https://issues.apache.org/jira/browse/YARN-9552) | FairScheduler: NODE\_UPDATE can cause NoSuchElementException |  Major | fairscheduler | Peter Bacsko | Peter Bacsko |
+| [HDFS-14754](https://issues.apache.org/jira/browse/HDFS-14754) | Erasure Coding :  The number of Under-Replicated Blocks never reduced |  Critical | ec | Hemanth Boyina | Hemanth Boyina |
+| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider |  Major | . | Shen Yinjie | Erik Krogen |
+| [HDFS-14373](https://issues.apache.org/jira/browse/HDFS-14373) | EC : Decoding is failing when block group last incomplete cell fall in to AlignedStripe |  Critical | ec, hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x |  Blocker | . | Yuxuan Wang | Yuxuan Wang |
+| [HDFS-14886](https://issues.apache.org/jira/browse/HDFS-14886) | In NameNode Web UI's Startup Progress page, Loading edits always shows 0 sec |  Major | . | Hemanth Boyina | Hemanth Boyina |
+| [YARN-8453](https://issues.apache.org/jira/browse/YARN-8453) | Additional Unit  tests to verify queue limit and max-limit with multiple resource types |  Major | capacity scheduler | Sunil G | Adam Antal |
+| [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | Setting permissions on name directory fails on non posix compliant filesystems |  Blocker | . | hirik | Siddharth Wagle |
+| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [HDFS-14909](https://issues.apache.org/jira/browse/HDFS-14909) | DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope |  Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16600](https://issues.apache.org/jira/browse/HADOOP-16600) | StagingTestBase uses methods not available in Mockito 1.8.5 in branch-3.1 |  Major | . | Lisheng Sun | Duo Zhang |
+| [MAPREDUCE-6441](https://issues.apache.org/jira/browse/MAPREDUCE-6441) | Improve temporary directory name generation in LocalDistributedCacheManager for concurrent processes |  Major | . | William Watson | Haibo Chen |
+| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14847](https://issues.apache.org/jira/browse/HDFS-14847) | Erasure Coding: Blocks are over-replicated while EC decommissioning |  Critical | ec | Fei Hui | Fei Hui |
+| [HDFS-14913](https://issues.apache.org/jira/browse/HDFS-14913) | Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType() |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart |  Major | datanode | TanYuxin | Xiaoqiao He |
+| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename |  Major | . | Jinglun | Jinglun |
+| [YARN-9921](https://issues.apache.org/jira/browse/YARN-9921) | Issue in PlacementConstraint when YARN Service AM retries allocation on component failure. |  Major | . | Tarun Parimi | Tarun Parimi |
+| [HDFS-14910](https://issues.apache.org/jira/browse/HDFS-14910) | Rename Snapshot with Pre Descendants Fail With IllegalArgumentException. |  Blocker | . | Íñigo Goiri | Wei-Chiu Chuang |
+| [HDFS-14308](https://issues.apache.org/jira/browse/HDFS-14308) | DFSStripedInputStream curStripeBuf is not freed by unbuffer() |  Major | ec | Joe McDonnell | Zhao Yi Ming |
+| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-16669](https://issues.apache.org/jira/browse/HADOOP-16669) | TestRawLocalFileSystemContract.testPermission fails if no native library |  Minor | common, test | Steve Loughran | Steve Loughran |
+| [HDFS-14920](https://issues.apache.org/jira/browse/HDFS-14920) | Erasure Coding: Decommission may hang If one or more datanodes are out of service during decommission |  Major | ec | Fei Hui | Fei Hui |
+| [HDFS-13736](https://issues.apache.org/jira/browse/HDFS-13736) | BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false |  Major | . | hu xiaodong | hu xiaodong |
+| [HDFS-14925](https://issues.apache.org/jira/browse/HDFS-14925) | rename operation should check nest snapshot |  Major | namenode | Junwang Zhao | Junwang Zhao |
+| [YARN-9949](https://issues.apache.org/jira/browse/YARN-9949) | Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14945](https://issues.apache.org/jira/browse/HDFS-14945) | Revise PacketResponder's log. |  Minor | datanode | Xudong Cao | Xudong Cao |
+| [HDFS-14946](https://issues.apache.org/jira/browse/HDFS-14946) | Erasure Coding: Block recovery failed during decommissioning |  Major | . | Fei Hui | Fei Hui |
+| [HDFS-14384](https://issues.apache.org/jira/browse/HDFS-14384) | When lastLocatedBlock token expire, it will take 1~3s second to refetch it. |  Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup |  Minor | hdfs | Jim Brennan | Jim Brennan |
+| [HDFS-14720](https://issues.apache.org/jira/browse/HDFS-14720) | DataNode shouldn't report block as bad block if the block length is Long.MAX\_VALUE. |  Major | datanode | Surendra Singh Lilhore | Hemanth Boyina |
+| [HADOOP-16677](https://issues.apache.org/jira/browse/HADOOP-16677) | Recalculate the remaining timeout millis correctly while throwing an InterupptedException in SocketIOWithTimeout. |  Minor | common | Xudong Cao | Xudong Cao |
+| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs |  Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh |
+| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path |  Minor | fs, test | zhoutai.zt | Xieming Li |
+| [YARN-9984](https://issues.apache.org/jira/browse/YARN-9984) | FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node |  Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-9983](https://issues.apache.org/jira/browse/YARN-9983) | Typo in YARN Service overview documentation |  Trivial | documentation | Denes Gerencser | Denes Gerencser |
+| [HADOOP-16719](https://issues.apache.org/jira/browse/HADOOP-16719) | Remove the disallowed element config within maven-checkstyle-plugin |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later |  Minor | . | xuzq | xuzq |
+| [HADOOP-15686](https://issues.apache.org/jira/browse/HADOOP-15686) | Supress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr |  Major | kms | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-14940](https://issues.apache.org/jira/browse/HDFS-14940) | HDFS Balancer : Do not allow to set balancer maximum network bandwidth more than 1TB |  Minor | balancer & mover | Souryakanta Dwivedy | Hemanth Boyina |
+| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue |  Critical | capacity scheduler | jiulongzhu | jiulongzhu |
+| [YARN-9968](https://issues.apache.org/jira/browse/YARN-9968) | Public Localizer is exiting in NodeManager due to NullPointerException |  Major | nodemanager | Tarun Parimi | Tarun Parimi |
+| [YARN-9011](https://issues.apache.org/jira/browse/YARN-9011) | Race condition during decommissioning |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error |  Critical | . | luhuachao | luhuachao |
+| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws  ConcurrentModificationException |  Major | datanode, performance | Ryan Wu | Aiphago |
+| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure |  Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-15010](https://issues.apache.org/jira/browse/HDFS-15010) | BlockPoolSlice#addReplicaThreadPool static pool should be initialized by static method |  Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16744](https://issues.apache.org/jira/browse/HADOOP-16744) |  Fix building instruction to enable zstd |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-14869](https://issues.apache.org/jira/browse/HDFS-14869) | Data loss in case of distcp using snapshot diff. Replication should include rename records if file was skipped in the previous iteration |  Major | distcp | Aasha Medhi | Aasha Medhi |
+| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop |  Blocker | build | Kevin Su | Kevin Su |
+| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong |  Major | . | Ranith Sardar | Ranith Sardar |
+| [YARN-10055](https://issues.apache.org/jira/browse/YARN-10055) | bower install fails |  Blocker | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. |  Major | test | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15073](https://issues.apache.org/jira/browse/HDFS-15073) | Replace curator-shaded guava import with the standard one |  Minor | hdfs-client | Akira Ajisaka | Chandra Sanivarapu |
+| [HADOOP-16042](https://issues.apache.org/jira/browse/HADOOP-16042) | Update the link to HadoopJavaVersion |  Minor | documentation | Akira Ajisaka | Chandra Sanivarapu |
+| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 |  Major | . | Takanobu Asanuma | Ayush Saxena |
+| [YARN-10053](https://issues.apache.org/jira/browse/YARN-10053) | Placement rules do not use correct group service init |  Major | yarn | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-15068](https://issues.apache.org/jira/browse/HDFS-15068) | DataNode could meet deadlock if invoke refreshVolumes when register |  Major | datanode | Xiaoqiao He | Aiphago |
+| [MAPREDUCE-7255](https://issues.apache.org/jira/browse/MAPREDUCE-7255) | Fix typo in MapReduce documentaion example |  Trivial | documentation | Sergey Pogorelov | Sergey Pogorelov |
+| [HDFS-15072](https://issues.apache.org/jira/browse/HDFS-15072) | HDFS MiniCluster fails to start when run in directory path with a % |  Minor | . | Geoffrey Jacoby | Masatake Iwasaki |
+| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout |  Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-15080](https://issues.apache.org/jira/browse/HDFS-15080) | Fix the issue in reading persistent memory cached data with an offset |  Major | caching, datanode | Feilong He | Feilong He |
+| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently |  Major | . | Miklos Szegedi | Jim Brennan |
+| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out |  Major | nodemanager | Jason Darrell Lowe | Chandni Singh |
+| [HDFS-14957](https://issues.apache.org/jira/browse/HDFS-14957) | INodeReference Space Consumed was not same in QuotaUsage and ContentSummary |  Major | namenode | Hemanth Boyina | Hemanth Boyina |
+| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank |  Major | conf | Jonathan Turner Eagles | Daryn Sharp |
+| [HDFS-15095](https://issues.apache.org/jira/browse/HDFS-15095) | Fix accidental comment in flaky test TestDecommissioningStatus |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HDFS-14578](https://issues.apache.org/jira/browse/HDFS-14578) | AvailableSpaceBlockPlacementPolicy always prefers local node |  Major | block placement | Wei-Chiu Chuang | Ayush Saxena |
+| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-8373](https://issues.apache.org/jira/browse/YARN-8373) | RM  Received RMFatalEvent of type CRITICAL\_THREAD\_CRASH |  Major | fairscheduler, resourcemanager | Girish Bhat | Wilfred Spiegelenburg |
+| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int |  Major | documentation | zhaoshengjie |  |
+| [YARN-8148](https://issues.apache.org/jira/browse/YARN-8148) | Update decimal values for queue capacities shown on queue status CLI |  Major | client | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) |  Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero |  Major | . | kyungwan nam | kyungwan nam |
+| [HDFS-14993](https://issues.apache.org/jira/browse/HDFS-14993) | checkDiskError doesn't work during datanode startup |  Major | datanode | Yang Yun | Yang Yun |
+| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently |  Critical | fs | Gabor Bota | Ahmed Hussein |
+| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect |  Major | . | Jason Darrell Lowe | Ahmed Hussein |
+| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. |  Major | hdfs-client | Konstantin Shvachko | Chen Liang |
+| [HDFS-7175](https://issues.apache.org/jira/browse/HDFS-7175) | Client-side SocketTimeoutException during Fsck |  Major | namenode | Carl Steinbach | Stephen O'Donnell |
+| [HDFS-15148](https://issues.apache.org/jira/browse/HDFS-15148) | dfs.namenode.send.qop.enabled should not apply to primary NN port |  Major | . | Chen Liang | Chen Liang |
+| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug |  Major | . | wangzhixiang | wangzhixiang |
+| [HDFS-15157](https://issues.apache.org/jira/browse/HDFS-15157) | Fix compilation failure for branch-3.1 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15158](https://issues.apache.org/jira/browse/HDFS-15158) | The number of failed volumes mismatch  with volumeFailures of Datanode metrics |  Minor | datanode | Yang Yun | Yang Yun |
+| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only |  Minor | build | Jihyun Cho | Jihyun Cho |
+| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException |  Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze |
+| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) |  Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation |  Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki |
+| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes |  Major | namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog |  Major | . | Konstantin Shvachko | Xieming Li |
+| [HADOOP-16841](https://issues.apache.org/jira/browse/HADOOP-16841) | The description of hadoop.http.authentication.signature.secret.file contains outdated information |  Minor | documentation | Akira Ajisaka | Xieming Li |
+| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md |  Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook |
+| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14668](https://issues.apache.org/jira/browse/HDFS-14668) | Support Fuse with Users from multiple Security Realms |  Critical | fuse-dfs | Sailesh Patel | Istvan Fajth |
+| [HDFS-15111](https://issues.apache.org/jira/browse/HDFS-15111) | stopStandbyServices() should log which service state it is transitioning from. |  Major | hdfs, logging | Konstantin Shvachko | Xieming Li |
+| [HDFS-15199](https://issues.apache.org/jira/browse/HDFS-15199) | NPE in BlockSender |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 |  Blocker | . | Siyao Meng | Siyao Meng |
+| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically |  Major | scheduler, test | Prabhu Joseph | Ahmed Hussein |
+| [HDFS-15216](https://issues.apache.org/jira/browse/HDFS-15216) | Wrong Use Case of -showprogress in fsck |  Major | . | Ravuri Sushma sree | Ravuri Sushma sree |
+| [HDFS-15211](https://issues.apache.org/jira/browse/HDFS-15211) | EC: File write hangs during close in case of Exception during updatePipeline |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15208](https://issues.apache.org/jira/browse/HDFS-15208) | Suppress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr in hdfs |  Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-15219](https://issues.apache.org/jira/browse/HDFS-15219) | DFS Client will stuck when ResponseProcessor.run throw Error |  Major | hdfs-client | zhengchenyu | zhengchenyu |
+| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment |  Critical | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-14836](https://issues.apache.org/jira/browse/HADOOP-14836) | Upgrade maven-clean-plugin to 3.1.0 |  Major | build | Allen Wittenauer | Akira Ajisaka |
+| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception |  Critical | . | Hemanth Boyina | Hemanth Boyina |
+| [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address |  Major | ha, namenode | Dhiraj Hegde | Dhiraj Hegde |
+| [HDFS-15297](https://issues.apache.org/jira/browse/HDFS-15297) | TestNNHandlesBlockReportPerStorage::blockReport\_02 fails intermittently in trunk |  Major | datanode, test | Mingliang Liu | Ayush Saxena |
+| [HADOOP-17014](https://issues.apache.org/jira/browse/HADOOP-17014) | Upgrade jackson-databind to 2.9.10.4 |  Blocker | . | Siyao Meng | Siyao Meng |
+| [HDFS-15286](https://issues.apache.org/jira/browse/HDFS-15286) | Concat on a same files deleting the file |  Critical | . | Hemanth Boyina | Hemanth Boyina |
+| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. |  Major | . | Jinglun | Jinglun |
+| [HADOOP-17052](https://issues.apache.org/jira/browse/HADOOP-17052) | NetUtils.connect() throws unchecked exception (UnresolvedAddressException) causing clients to abort |  Major | net | Dhiraj Hegde | Dhiraj Hegde |
+| [YARN-10347](https://issues.apache.org/jira/browse/YARN-10347) | Fix double locking in CapacityScheduler#reinitialize in branch-3.1 |  Critical | capacity scheduler | Masatake Iwasaki | Masatake Iwasaki |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures |  Major | yarn | Jim Brennan | Jim Brennan |
+| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature |  Major | fs, test | John Zhuge | Jim Brennan |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15636](https://issues.apache.org/jira/browse/HADOOP-15636) | Add ITestDynamoDBMetadataStore |  Minor | fs/s3, test | Sean Mackrory | Gabor Bota |
+| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails |  Major | . | Akira Ajisaka | Zsolt Venczel |
+| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext |  Major | hdfs | Shweta | Shweta |
+| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-9791](https://issues.apache.org/jira/browse/YARN-9791) | Queue Mutation API does not allow to remove a config |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive |  Major | hdfs | Chen Liang | Chen Liang |
+| [YARN-9864](https://issues.apache.org/jira/browse/YARN-9864) | Format CS Configuration present in Configuration Store |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9801](https://issues.apache.org/jira/browse/YARN-9801) | SchedConfCli does not work with https mode |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode |  Major | hdfs | Chen Liang | Chen Liang |
+| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. |  Major | hdfs | Konstantin Shvachko | Chen Liang |
+| [YARN-9873](https://issues.apache.org/jira/browse/YARN-9873) | Mutation API Config Change need to update Version Number |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode |  Major | . | Konstantin Shvachko | Erik Krogen |
+| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources |  Major | . | Manikandan R | Manikandan R |
+| [HADOOP-16598](https://issues.apache.org/jira/browse/HADOOP-16598) | Backport "HADOOP-16558 [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes" to all active branches |  Major | common | Duo Zhang | Duo Zhang |
+| [YARN-9950](https://issues.apache.org/jira/browse/YARN-9950) | Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9900](https://issues.apache.org/jira/browse/YARN-9900) | Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit |  Major | build | Duo Zhang | Duo Zhang |
+| [YARN-9909](https://issues.apache.org/jira/browse/YARN-9909) | Offline format of YarnConfigurationStore |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9836](https://issues.apache.org/jira/browse/YARN-9836) | General usability improvements in showSimulationTrace.html |  Minor | scheduler-load-simulator | Adam Antal | Adam Antal |
+| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml |  Minor | fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HADOOP-16609](https://issues.apache.org/jira/browse/HADOOP-16609) | Add Jenkinsfile for all active branches |  Major | build | Duo Zhang | Akira Ajisaka |
+| [YARN-10022](https://issues.apache.org/jira/browse/YARN-10022) | Create RM Rest API to validate a CapacityScheduler Configuration |  Major | . | Kinga Marton | Kinga Marton |
+| [HDFS-15173](https://issues.apache.org/jira/browse/HDFS-15173) | RBF: Delete repeated configuration 'dfs.federation.router.metrics.enable' |  Minor | documentation, rbf | panlijie | panlijie |
+| [YARN-10139](https://issues.apache.org/jira/browse/YARN-10139) | ValidateAndGetSchedulerConfiguration API fails when cluster max allocation \> default 8GB |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14731](https://issues.apache.org/jira/browse/HDFS-14731) | [FGL] Remove redundant locking on NameNode. |  Major | namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-14918](https://issues.apache.org/jira/browse/HADOOP-14918) | Remove the Local Dynamo DB test option |  Major | fs/s3 | Steve Loughran | Gabor Bota |
+| [HDFS-15305](https://issues.apache.org/jira/browse/HDFS-15305) | Extend ViewFS and provide ViewFSOverloadScheme implementation with scheme configurable. |  Major | fs, hadoop-client, hdfs-client, viewfs | Uma Maheswara Rao G | Uma Maheswara Rao G |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-8016](https://issues.apache.org/jira/browse/YARN-8016) | Refine PlacementRule interface and add a app-name queue mapping rule as an example |  Major | . | Zian Chen | Zian Chen |
+| [HADOOP-16491](https://issues.apache.org/jira/browse/HADOOP-16491) | Upgrade jetty version to 9.3.27 |  Major | . | Hrishikesh Gadre | Hrishikesh Gadre |
+| [HADOOP-16542](https://issues.apache.org/jira/browse/HADOOP-16542) | Update commons-beanutils version to 1.9.4 |  Major | . | Wei-Chiu Chuang | Kevin Su |
+| [HADOOP-16551](https://issues.apache.org/jira/browse/HADOOP-16551) | The changelog\*.md seems not generated when create-release |  Blocker | . | Zhankun Tang |  |
+| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14959](https://issues.apache.org/jira/browse/HDFS-14959) | [SBNN read] access time should be turned off |  Major | documentation | Wei-Chiu Chuang | Chao Sun |
+| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16871](https://issues.apache.org/jira/browse/HADOOP-16871) | Upgrade Netty version to 4.1.45.Final to handle CVE-2019-20444,CVE-2019-16869 |  Major | . | Aray Chenchu Sukesh | Aray Chenchu Sukesh |
+| [HADOOP-16982](https://issues.apache.org/jira/browse/HADOOP-16982) | Update Netty to 4.1.48.Final |  Blocker | . | Wei-Chiu Chuang | Lisheng Sun |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md
new file mode 100644
index 0000000000000..b22827bf8c8de
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md
@@ -0,0 +1,72 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.1.4 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-13806](https://issues.apache.org/jira/browse/HDFS-13806) | *Minor* | **EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory**
+
+After this change, attempt to unsetErasureCodingPolicy() on a directory without EC policy explicitly set on it, will get NoECPolicySetException.
+
+
+---
+
+* [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | *Blocker* | **Setting permissions on name directory fails on non posix compliant filesystems**
+
+- Fixed namenode/journal startup on Windows.
+
+
+---
+
+* [HDFS-14745](https://issues.apache.org/jira/browse/HDFS-14745) | *Major* | **Backport HDFS persistent memory read cache support to branch-3.1**
+
+Non-volatile storage class memory (SCM, also known as persistent memory) is supported in HDFS cache. To enable SCM cache, user just needs to configure SCM volume for property “dfs.datanode.cache.pmem.dirs” in hdfs-site.xml. And all HDFS cache directives keep unchanged. There are two implementations for HDFS SCM Cache, one is pure java code implementation and the other is native PMDK based implementation. The latter implementation can bring user better performance gain in cache write and cache read. If PMDK native libs could be loaded, it will use PMDK based implementation otherwise it will fallback to java code implementation. To enable PMDK based implementation, user should install PMDK library by referring to the official site http://pmem.io/. Then, build Hadoop with PMDK support by referring to "PMDK library build options" section in \`BUILDING.txt\` in the source code. If multiple SCM volumes are configured, a round-robin policy is used to select an available volume for caching a block. Consistent with DRAM cache, SCM cache also has no cache eviction mechanism. When DataNode receives a data read request from a client, if the corresponding block is cached into SCM, DataNode will instantiate an InputStream with the block location path on SCM (pure java implementation) or cache address on SCM (PMDK based implementation). Once the InputStream is created, DataNode will send the cached data to the client. Please refer "Centralized Cache Management" guide for more details.
+
+
+---
+
+* [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | *Major* | **Consistent Reads from Standby Node**
+
+Observer is a new type of a NameNode in addition to Active and Standby Nodes in HA settings. An Observer Node maintains a replica of the namespace same as a Standby Node. It additionally allows execution of clients read requests.
+
+To ensure read-after-write consistency within a single client, a state ID is introduced in RPC headers. The Observer responds to the client request only after its own state has caught up with the client’s state ID, which it previously received from the Active NameNode.
+
+Clients can explicitly invoke a new client protocol call msync(), which ensures that subsequent reads by this client from an Observer are consistent.
+
+A new client-side ObserverReadProxyProvider is introduced to provide automatic switching between Active and Observer NameNodes for submitting respectively write and read requests.
+
+
+---
+
+* [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | *Major* | **Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0**
+
+Updated checkstyle to 8.26 and updated maven-checkstyle-plugin to 3.1.0.
+
+
+---
+
+* [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | *Major* | **ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address**
+
+ZKFC binds host address to "dfs.namenode.servicerpc-bind-host", if configured. Otherwise, it binds to "dfs.namenode.rpc-bind-host". If neither of those is configured, ZKFC binds itself to NameNode RPC server address (effectively "dfs.namenode.rpc-address").
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.3.2.2.md
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.md
rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/CHANGELOG.3.2.2.md
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.3.2.2.md
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.md
rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.2/RELEASENOTES.3.2.2.md
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md
new file mode 100644
index 0000000000000..34928bf54e50d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/CHANGELOG.3.2.3.md
@@ -0,0 +1,386 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.2.3 - 2022-03-02
+
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15691](https://issues.apache.org/jira/browse/HADOOP-15691) | Add PathCapabilities to FS and FC to complement StreamCapabilities |  Major | . | Steve Loughran | Steve Loughran |
+| [HDFS-15711](https://issues.apache.org/jira/browse/HDFS-15711) | Add Metrics to HttpFS Server |  Major | httpfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15759](https://issues.apache.org/jira/browse/HDFS-15759) | EC: Verify EC reconstruction correctness on DataNode |  Major | datanode, ec, erasure-coding | Toshihiko Uchida | Toshihiko Uchida |
+| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web |  Minor | . | tomscut | tomscut |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-16052](https://issues.apache.org/jira/browse/HADOOP-16052) | Remove Subversion and Forrest from Dockerfile |  Minor | build | Akira Ajisaka | Xieming Li |
+| [YARN-9783](https://issues.apache.org/jira/browse/YARN-9783) | Remove low-level zookeeper test to be able to build Hadoop against zookeeper 3.5.5 |  Major | test | Mate Szalay-Beko | Mate Szalay-Beko |
+| [HADOOP-16717](https://issues.apache.org/jira/browse/HADOOP-16717) | Remove GenericsUtil isLog4jLogger dependency on Log4jLoggerAdapter |  Major | . | David Mollitor | Xieming Li |
+| [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | Install yarnpkg and upgrade nodejs in Dockerfile |  Major | buid, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16811](https://issues.apache.org/jira/browse/HADOOP-16811) | Use JUnit TemporaryFolder Rule in TestFileUtils |  Minor | common, test | David Mollitor | David Mollitor |
+| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor |  Major | . | Íñigo Goiri | Xiaoqiao He |
+| [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | Update Dockerfile to use Bionic |  Major | build, test | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15574](https://issues.apache.org/jira/browse/HDFS-15574) | Remove unnecessary sort of block list in DirectoryScanner |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15583](https://issues.apache.org/jira/browse/HDFS-15583) | Backport DirectoryScanner improvements HDFS-14476, HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1 |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15567](https://issues.apache.org/jira/browse/HDFS-15567) | [SBN Read] HDFS should expose msync() API to allow downstream applications call it explicitly. |  Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15633](https://issues.apache.org/jira/browse/HDFS-15633) | Avoid redundant RPC calls for getDiskStatus |  Major | dfsclient | Ayush Saxena | Ayush Saxena |
+| [YARN-10450](https://issues.apache.org/jira/browse/YARN-10450) | Add cpu and memory utilization per node and cluster-wide metrics |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15652](https://issues.apache.org/jira/browse/HDFS-15652) | Make block size from NNThroughputBenchmark configurable |  Minor | benchmarks | Hui Fei | Hui Fei |
+| [YARN-10475](https://issues.apache.org/jira/browse/YARN-10475) | Scale RM-NM heartbeat interval based on node utilization |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15665](https://issues.apache.org/jira/browse/HDFS-15665) | Balancer logging improvement |  Major | balancer & mover | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-17342](https://issues.apache.org/jira/browse/HADOOP-17342) | Creating a token identifier should not do kerberos name resolution |  Major | common | Jim Brennan | Jim Brennan |
+| [YARN-10479](https://issues.apache.org/jira/browse/YARN-10479) | RMProxy should retry on SocketTimeout Exceptions |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15623](https://issues.apache.org/jira/browse/HDFS-15623) | Respect configured values of rpc.engine |  Major | hdfs | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HDFS-14395](https://issues.apache.org/jira/browse/HDFS-14395) | Remove WARN Logging From Interrupts in DataStreamer |  Minor | hdfs-client | David Mollitor | David Mollitor |
+| [HADOOP-17367](https://issues.apache.org/jira/browse/HADOOP-17367) | Add InetAddress api to ProxyUsers.authorize |  Major | performance, security | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15694](https://issues.apache.org/jira/browse/HDFS-15694) | Avoid calling UpdateHeartBeatState inside DataNodeDescriptor |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15703](https://issues.apache.org/jira/browse/HDFS-15703) | Don't generate edits for set operations that are no-op |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17392](https://issues.apache.org/jira/browse/HADOOP-17392) | Remote exception messages should not include the exception class |  Major | ipc | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15706](https://issues.apache.org/jira/browse/HDFS-15706) | HttpFS: Log more information on request failures |  Major | httpfs | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17389](https://issues.apache.org/jira/browse/HADOOP-17389) | KMS should log full UGI principal |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15720](https://issues.apache.org/jira/browse/HDFS-15720) | namenode audit async logger should add some log4j config |  Minor | hdfs | Max  Xie |  |
+| [HDFS-15704](https://issues.apache.org/jira/browse/HDFS-15704) | Mitigate lease monitor's rapid infinite loop |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15751](https://issues.apache.org/jira/browse/HDFS-15751) | Add documentation for msync() API to filesystem.md |  Major | documentation | Konstantin Shvachko | Konstantin Shvachko |
+| [YARN-10538](https://issues.apache.org/jira/browse/YARN-10538) | Add recommissioning nodes to the list of updated nodes returned to the AM |  Major | . | Srinivas S T | Srinivas S T |
+| [YARN-4589](https://issues.apache.org/jira/browse/YARN-4589) | Diagnostics for localization timeouts is lacking |  Major | . | Chang Li | Chang Li |
+| [YARN-10562](https://issues.apache.org/jira/browse/YARN-10562) | Follow up changes for YARN-9833 |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15783](https://issues.apache.org/jira/browse/HDFS-15783) | Speed up BlockPlacementPolicyRackFaultTolerant#verifyBlockPlacement |  Major | block placement | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17478](https://issues.apache.org/jira/browse/HADOOP-17478) | Improve the description of hadoop.http.authentication.signature.secret.file |  Minor | documentation | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15789](https://issues.apache.org/jira/browse/HDFS-15789) | Lease renewal does not require namesystem lock |  Major | hdfs | Jim Brennan | Jim Brennan |
+| [HADOOP-17501](https://issues.apache.org/jira/browse/HADOOP-17501) | Fix logging typo in ShutdownHookManager |  Major | common | Konstantin Shvachko | Fengnan Li |
+| [HADOOP-17354](https://issues.apache.org/jira/browse/HADOOP-17354) | Move Jenkinsfile outside of the root directory |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15799](https://issues.apache.org/jira/browse/HDFS-15799) | Make DisallowedDatanodeException terse |  Minor | hdfs | Richard | Richard |
+| [HDFS-15813](https://issues.apache.org/jira/browse/HDFS-15813) | DataStreamer: keep sending heartbeat packets while streaming |  Major | hdfs | Jim Brennan | Jim Brennan |
+| [MAPREDUCE-7319](https://issues.apache.org/jira/browse/MAPREDUCE-7319) | Log list of mappers at trace level in ShuffleHandler audit log |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15821](https://issues.apache.org/jira/browse/HDFS-15821) | Add metrics for in-service datanodes |  Minor | . | Zehao Chen | Zehao Chen |
+| [YARN-10626](https://issues.apache.org/jira/browse/YARN-10626) | Log resource allocation in NM log at container start time |  Major | . | Eric Badger | Eric Badger |
+| [HDFS-15815](https://issues.apache.org/jira/browse/HDFS-15815) |  if required storageType are unavailable, log the failed reason during choosing Datanode |  Minor | block placement | Yang Yun | Yang Yun |
+| [HDFS-15826](https://issues.apache.org/jira/browse/HDFS-15826) | Solve the problem of incorrect progress of delegation tokens when loading FsImage |  Major | . | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-15734](https://issues.apache.org/jira/browse/HDFS-15734) | [READ] DirectoryScanner#scan need not check StorageType.PROVIDED |  Minor | datanode | Yuxuan Wang | Yuxuan Wang |
+| [HADOOP-17538](https://issues.apache.org/jira/browse/HADOOP-17538) | Add kms-default.xml and httpfs-default.xml to site index |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10613](https://issues.apache.org/jira/browse/YARN-10613) | Config to allow Intra- and Inter-queue preemption to  enable/disable conservativeDRF |  Minor | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne |
+| [YARN-10653](https://issues.apache.org/jira/browse/YARN-10653) | Fixed the findbugs issues introduced by YARN-10647. |  Major | . | Qi Zhu | Qi Zhu |
+| [MAPREDUCE-7324](https://issues.apache.org/jira/browse/MAPREDUCE-7324) | ClientHSSecurityInfo class is in wrong META-INF file |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17546](https://issues.apache.org/jira/browse/HADOOP-17546) | Update Description of hadoop-http-auth-signature-secret in HttpAuthentication.md |  Minor | . | Ravuri Sushma sree | Ravuri Sushma sree |
+| [YARN-10664](https://issues.apache.org/jira/browse/YARN-10664) | Allow parameter expansion in NM\_ADMIN\_USER\_ENV |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-17570](https://issues.apache.org/jira/browse/HADOOP-17570) | Apply YETUS-1102 to re-enable GitHub comments |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17594](https://issues.apache.org/jira/browse/HADOOP-17594) | DistCp: Expose the JobId for applications executing through run method |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15911](https://issues.apache.org/jira/browse/HDFS-15911) | Provide blocks moved count in Balancer iteration result |  Major | balancer & mover | Viraj Jasani | Viraj Jasani |
+| [HDFS-15919](https://issues.apache.org/jira/browse/HDFS-15919) | BlockPoolManager should log stack trace if unable to get Namenode addresses |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | Use spotbugs-maven-plugin instead of findbugs-maven-plugin |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15932](https://issues.apache.org/jira/browse/HDFS-15932) | Improve the balancer error message when process exits abnormally. |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-15931](https://issues.apache.org/jira/browse/HDFS-15931) | Fix non-static inner classes for better memory management |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-15942](https://issues.apache.org/jira/browse/HDFS-15942) | Increase Quota initialization threads |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15937](https://issues.apache.org/jira/browse/HDFS-15937) | Reduce memory used during datanode layout upgrade |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-17569](https://issues.apache.org/jira/browse/HADOOP-17569) | Building native code fails on Fedora 33 |  Major | build, common | Kengo Seki | Masatake Iwasaki |
+| [HADOOP-17633](https://issues.apache.org/jira/browse/HADOOP-17633) | Bump json-smart to 2.4.2 and nimbus-jose-jwt to 9.8 due to CVEs |  Major | auth, build | helen huang | Viraj Jasani |
+| [HADOOP-16822](https://issues.apache.org/jira/browse/HADOOP-16822) | Provide source artifacts for hadoop-client-api |  Major | . | Karel Kolman | Karel Kolman |
+| [HADOOP-17680](https://issues.apache.org/jira/browse/HADOOP-17680) | Allow ProtobufRpcEngine to be extensible |  Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [YARN-10123](https://issues.apache.org/jira/browse/YARN-10123) | Error message around yarn app -stop/start can be improved to highlight that an implementation at framework level is needed for the stop/start functionality to work |  Minor | client, documentation | Siddharth Ahuja | Siddharth Ahuja |
+| [HADOOP-17756](https://issues.apache.org/jira/browse/HADOOP-17756) | Increase precommit job timeout from 20 hours to 24 hours. |  Major | build | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-16073](https://issues.apache.org/jira/browse/HDFS-16073) | Remove redundant RPC requests for getFileLinkInfo in ClientNamenodeProtocolTranslatorPB |  Minor | . | lei w | lei w |
+| [HDFS-16074](https://issues.apache.org/jira/browse/HDFS-16074) | Remove an expensive debug string concatenation |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-10834](https://issues.apache.org/jira/browse/YARN-10834) | Intra-queue preemption: apps that don't use defined custom resource won't be preempted. |  Major | . | Eric Payne | Eric Payne |
+| [HADOOP-17749](https://issues.apache.org/jira/browse/HADOOP-17749) | Remove lock contention in SelectorPool of SocketIOWithTimeout |  Major | common | Xuesen Liang | Xuesen Liang |
+| [HADOOP-17775](https://issues.apache.org/jira/browse/HADOOP-17775) | Remove JavaScript package from Docker environment |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17794](https://issues.apache.org/jira/browse/HADOOP-17794) | Add a sample configuration to use ZKDelegationTokenSecretManager in Hadoop KMS |  Major | documentation, kms, security | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12665](https://issues.apache.org/jira/browse/HADOOP-12665) | Document hadoop.security.token.service.use\_ip |  Major | documentation | Arpit Agarwal | Akira Ajisaka |
+| [YARN-10456](https://issues.apache.org/jira/browse/YARN-10456) | RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry |  Major | resourcemanager | Eric Payne | Eric Payne |
+| [HDFS-15650](https://issues.apache.org/jira/browse/HDFS-15650) | Make the socket timeout for computing checksum of striped blocks configurable |  Minor | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka |
+| [YARN-10858](https://issues.apache.org/jira/browse/YARN-10858) | [UI2] YARN-10826 breaks Queue view |  Major | yarn-ui-v2 | Andras Gyori | Masatake Iwasaki |
+| [YARN-10860](https://issues.apache.org/jira/browse/YARN-10860) | Make max container per heartbeat configs refreshable |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17813](https://issues.apache.org/jira/browse/HADOOP-17813) | Checkstyle - Allow line length: 100 |  Major | . | Akira Ajisaka | Viraj Jasani |
+| [HADOOP-17819](https://issues.apache.org/jira/browse/HADOOP-17819) | Add extensions to ProtobufRpcEngine RequestHeaderProto |  Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HDFS-16153](https://issues.apache.org/jira/browse/HDFS-16153) | Avoid evaluation of LOG.debug statement in QuorumJournalManager |  Trivial | . | wangzhaohui | wangzhaohui |
+| [HDFS-16154](https://issues.apache.org/jira/browse/HDFS-16154) | TestMiniJournalCluster failing intermittently because of not reseting UserGroupInformation completely |  Minor | . | wangzhaohui | wangzhaohui |
+| [HADOOP-17849](https://issues.apache.org/jira/browse/HADOOP-17849) | Exclude spotbugs-annotations from transitive dependencies on branch-3.2 |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability |  Major | fs | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-15160](https://issues.apache.org/jira/browse/HDFS-15160) | ReplicaMap, Disk Balancer, Directory Scanner and various FsDatasetImpl methods should use datanode readlock |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously |  Major | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread |  Major | . | zhanghuazong | zhanghuazong |
+| [HDFS-16286](https://issues.apache.org/jira/browse/HDFS-16286) | Debug tool to verify the correctness of erasure coding on file |  Minor | erasure-coding, tools | daimin | daimin |
+| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. |  Major | fs | Chengwei Wang | Chengwei Wang |
+| [HADOOP-18023](https://issues.apache.org/jira/browse/HADOOP-18023) | Allow cp command to run with multi threads. |  Major | fs | Chengwei Wang | Chengwei Wang |
+| [HADOOP-17643](https://issues.apache.org/jira/browse/HADOOP-17643) | WASB : Make metadata checks case insensitive |  Major | . | Anoop Sam John | Anoop Sam John |
+| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16430](https://issues.apache.org/jira/browse/HDFS-16430) | Validate maximum blocks in EC group when adding an EC policy |  Minor | ec, erasure-coding | daimin | daimin |
+| [HDFS-16403](https://issues.apache.org/jira/browse/HDFS-16403) | Improve FUSE IO performance by supporting FUSE parameter max\_background |  Minor | fuse-dfs | daimin | daimin |
+| [HADOOP-18093](https://issues.apache.org/jira/browse/HADOOP-18093) | Better exception handling for testFileStatusOnMountLink() in ViewFsBaseTest.java |  Trivial | . | Xing Lin | Xing Lin |
+| [HADOOP-18155](https://issues.apache.org/jira/browse/HADOOP-18155) | Refactor tests in TestFileUtil |  Trivial | common | Gautham Banasandra | Gautham Banasandra |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15939](https://issues.apache.org/jira/browse/HADOOP-15939) | Filter overlapping objenesis class in hadoop-client-minicluster |  Minor | build | Xiaoyu Yao | Xiaoyu Yao |
+| [YARN-8936](https://issues.apache.org/jira/browse/YARN-8936) | Bump up Atsv2 hbase versions |  Major | . | Rohith Sharma K S | Vrushali C |
+| [HDFS-14189](https://issues.apache.org/jira/browse/HDFS-14189) | Fix intermittent failure of TestNameNodeMetrics |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9246](https://issues.apache.org/jira/browse/YARN-9246) | NPE when executing a command yarn node -status or -states without additional arguments |  Minor | client | Masahiro Tanaka | Masahiro Tanaka |
+| [YARN-7266](https://issues.apache.org/jira/browse/YARN-7266) | Timeline Server event handler threads locked |  Major | ATSv2, timelineserver | Venkata Puneet Ravuri | Prabhu Joseph |
+| [YARN-9990](https://issues.apache.org/jira/browse/YARN-9990) | Testcase fails with "Insufficient configured threads: required=16 \< max=10" |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-10020](https://issues.apache.org/jira/browse/YARN-10020) | Fix build instruction of hadoop-yarn-ui |  Minor | yarn-ui-v2 | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10037](https://issues.apache.org/jira/browse/YARN-10037) | Upgrade build tools for YARN Web UI v2 |  Major | build, security, yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki |
+| [HDFS-15187](https://issues.apache.org/jira/browse/HDFS-15187) | CORRUPT replica mismatch between namenodes after failover |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15200](https://issues.apache.org/jira/browse/HDFS-15200) | Delete Corrupt Replica Immediately Irrespective of Replicas On Stale Storage |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature |  Blocker | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-15210](https://issues.apache.org/jira/browse/HDFS-15210) | EC : File write hanged when DN is shutdown by admin command. |  Major | ec | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16768](https://issues.apache.org/jira/browse/HADOOP-16768) | SnappyCompressor test cases wrongly assume that the compressed data is always smaller than the input data |  Major | io, test | zhao bo | Akira Ajisaka |
+| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice |  Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-17068](https://issues.apache.org/jira/browse/HADOOP-17068) | client fails forever when namenode ipaddr changed |  Major | hdfs-client | Sean Chow | Sean Chow |
+| [HDFS-15378](https://issues.apache.org/jira/browse/HDFS-15378) | TestReconstructStripedFile#testErasureCodingWorkerXmitsWeight is failing on trunk |  Major | . | Hemanth Boyina | Hemanth Boyina |
+| [YARN-10331](https://issues.apache.org/jira/browse/YARN-10331) | Upgrade node.js to 10.21.0 |  Critical | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17119](https://issues.apache.org/jira/browse/HADOOP-17119) | Jetty upgrade to 9.4.x causes MR app fail with IOException |  Major | . | Bilwa S T | Bilwa S T |
+| [HADOOP-17138](https://issues.apache.org/jira/browse/HADOOP-17138) | Fix spotbugs warnings surfaced after upgrade to 4.0.6 |  Minor | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-15439](https://issues.apache.org/jira/browse/HDFS-15439) | Setting dfs.mover.retry.max.attempts to negative value will retry forever. |  Major | balancer & mover | AMC-team | AMC-team |
+| [YARN-10430](https://issues.apache.org/jira/browse/YARN-10430) | Log improvements in NodeStatusUpdaterImpl |  Minor | nodemanager | Bilwa S T | Bilwa S T |
+| [HDFS-15438](https://issues.apache.org/jira/browse/HDFS-15438) | Setting dfs.disk.balancer.max.disk.errors = 0 will fail the block copy |  Major | balancer & mover | AMC-team | AMC-team |
+| [YARN-10438](https://issues.apache.org/jira/browse/YARN-10438) | Handle null containerId in ClientRMService#getContainerReport() |  Major | resourcemanager | Raghvendra Singh | Shubham Gupta |
+| [HDFS-15628](https://issues.apache.org/jira/browse/HDFS-15628) | HttpFS server throws NPE if a file is a symlink |  Major | fs, httpfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15627](https://issues.apache.org/jira/browse/HDFS-15627) | Audit log deletes before collecting blocks |  Major | logging, namenode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17309](https://issues.apache.org/jira/browse/HADOOP-17309) | Javadoc warnings and errors are ignored in the precommit jobs |  Major | build, documentation | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17310](https://issues.apache.org/jira/browse/HADOOP-17310) | Touch command with -c  option is broken |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15639](https://issues.apache.org/jira/browse/HDFS-15639) | [JDK 11] Fix Javadoc errors in hadoop-hdfs-client |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-15622](https://issues.apache.org/jira/browse/HDFS-15622) | Deleted blocks linger in the replications queue |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15641](https://issues.apache.org/jira/browse/HDFS-15641) | DataNode could meet deadlock if invoke refreshNameNode |  Critical | . | Hongbing Wang | Hongbing Wang |
+| [MAPREDUCE-7302](https://issues.apache.org/jira/browse/MAPREDUCE-7302) | Upgrading to JUnit 4.13 causes testcase TestFetcher.testCorruptedIFile() to fail |  Major | test | Peter Bacsko | Peter Bacsko |
+| [HDFS-15644](https://issues.apache.org/jira/browse/HDFS-15644) | Failed volumes can cause DNs to stop block reporting |  Major | block placement, datanode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17236](https://issues.apache.org/jira/browse/HADOOP-17236) | Bump up snakeyaml to 1.26 to mitigate CVE-2017-18640 |  Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [YARN-10467](https://issues.apache.org/jira/browse/YARN-10467) | ContainerIdPBImpl objects can be leaked in RMNodeImpl.completedContainers |  Major | resourcemanager | Haibo Chen | Haibo Chen |
+| [HADOOP-17329](https://issues.apache.org/jira/browse/HADOOP-17329) | mvn site commands fails due to MetricsSystemImpl changes |  Major | . | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-15651](https://issues.apache.org/jira/browse/HDFS-15651) | Client could not obtain block when DN CommandProcessingThread exit |  Major | . | Yiqun Lin | Mingxiang Li |
+| [HADOOP-17340](https://issues.apache.org/jira/browse/HADOOP-17340) | TestLdapGroupsMapping failing -string mismatch in exception validation |  Major | test | Steve Loughran | Steve Loughran |
+| [HADOOP-17352](https://issues.apache.org/jira/browse/HADOOP-17352) | Update PATCH\_NAMING\_RULE in the personality file |  Minor | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15485](https://issues.apache.org/jira/browse/HDFS-15485) | Fix outdated properties of JournalNode when performing rollback |  Minor | . | Deegue | Deegue |
+| [HADOOP-17358](https://issues.apache.org/jira/browse/HADOOP-17358) | Improve excessive reloading of Configurations |  Major | conf | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15538](https://issues.apache.org/jira/browse/HDFS-15538) | Fix the documentation for dfs.namenode.replication.max-streams in hdfs-default.xml |  Major | . | Xieming Li | Xieming Li |
+| [HADOOP-17362](https://issues.apache.org/jira/browse/HADOOP-17362) | Doing hadoop ls on Har file triggers too many RPC calls |  Major | fs | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10485](https://issues.apache.org/jira/browse/YARN-10485) | TimelineConnector swallows InterruptedException |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17360](https://issues.apache.org/jira/browse/HADOOP-17360) | Log the remote address for authentication success |  Minor | ipc | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17346](https://issues.apache.org/jira/browse/HADOOP-17346) | Fair call queue is defeated by abusive service principals |  Major | common, ipc | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10470](https://issues.apache.org/jira/browse/YARN-10470) | When building new web ui with root user, the bower install should support it. |  Major | build, yarn-ui-v2 | Qi Zhu | Qi Zhu |
+| [YARN-10498](https://issues.apache.org/jira/browse/YARN-10498) | Fix Yarn CapacityScheduler Markdown document |  Trivial | documentation | zhaoshengjie | zhaoshengjie |
+| [HDFS-15695](https://issues.apache.org/jira/browse/HDFS-15695) | NN should not let the balancer run in safemode |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10511](https://issues.apache.org/jira/browse/YARN-10511) | Update yarn.nodemanager.env-whitelist value in docs |  Minor | documentation | Andrea Scarpino | Andrea Scarpino |
+| [HDFS-15707](https://issues.apache.org/jira/browse/HDFS-15707) | NNTop counts don't add up as expected |  Major | hdfs, metrics, namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15709](https://issues.apache.org/jira/browse/HDFS-15709) | EC: Socket file descriptor leak in StripedBlockChecksumReconstructor |  Major | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka |
+| [YARN-10491](https://issues.apache.org/jira/browse/YARN-10491) | Fix deprecation warnings in SLSWebApp.java |  Minor | build | Akira Ajisaka | Ankit Kumar |
+| [HADOOP-13571](https://issues.apache.org/jira/browse/HADOOP-13571) | ServerSocketUtil.getPort() should use loopback address, not 0.0.0.0 |  Major | . | Eric Badger | Eric Badger |
+| [HDFS-15725](https://issues.apache.org/jira/browse/HDFS-15725) | Lease Recovery never completes for a committed block which the DNs never finalize |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15170](https://issues.apache.org/jira/browse/HDFS-15170) | EC: Block gets marked as CORRUPT in case of failover and pipeline recovery |  Critical | erasure-coding | Ayush Saxena | Ayush Saxena |
+| [HDFS-15719](https://issues.apache.org/jira/browse/HDFS-15719) | [Hadoop 3] Both NameNodes can crash simultaneously due to the short JN socket timeout |  Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-10560](https://issues.apache.org/jira/browse/YARN-10560) | Upgrade node.js to 10.23.1 and yarn to 1.22.5 in Web UI v2 |  Major | webapp, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10528](https://issues.apache.org/jira/browse/YARN-10528) | maxAMShare should only be accepted for leaf queues, not parent queues |  Major | . | Siddharth Ahuja | Siddharth Ahuja |
+| [HADOOP-17438](https://issues.apache.org/jira/browse/HADOOP-17438) | Increase docker memory limit in Jenkins |  Major | build, scripts, test, yetus | Ahmed Hussein | Ahmed Hussein |
+| [MAPREDUCE-7310](https://issues.apache.org/jira/browse/MAPREDUCE-7310) | Clear the fileMap in JHEventHandlerForSigtermTest |  Minor | test | Zhengxi Li | Zhengxi Li |
+| [HADOOP-16947](https://issues.apache.org/jira/browse/HADOOP-16947) | Stale record should be remove when MutableRollingAverages generating aggregate data. |  Major | . | Haibin Huang | Haibin Huang |
+| [HDFS-15632](https://issues.apache.org/jira/browse/HDFS-15632) | AbstractContractDeleteTest should set recursive parameter to true for recursive test cases. |  Major | . | Konstantin Shvachko | Anton Kutuzov |
+| [HDFS-10498](https://issues.apache.org/jira/browse/HDFS-10498) | Intermittent test failure org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotFileLength.testSnapshotfileLength |  Major | hdfs, snapshots | Hanisha Koneru | Jim Brennan |
+| [HADOOP-17506](https://issues.apache.org/jira/browse/HADOOP-17506) | Fix typo in BUILDING.txt |  Trivial | documentation | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-15795](https://issues.apache.org/jira/browse/HDFS-15795) | EC: Wrong checksum when reconstruction was failed by exception |  Major | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka |
+| [HDFS-15779](https://issues.apache.org/jira/browse/HDFS-15779) | EC: fix NPE caused by StripedWriter.clearBuffers during reconstruct block |  Major | . | Hongbing Wang | Hongbing Wang |
+| [HDFS-15798](https://issues.apache.org/jira/browse/HDFS-15798) | EC: Reconstruct task failed, and It would be XmitsInProgress of DN has negative number |  Major | . | Haiyang Hu | Haiyang Hu |
+| [YARN-10428](https://issues.apache.org/jira/browse/YARN-10428) | Zombie applications in the YARN queue using FAIR + sizebasedweight |  Critical | capacityscheduler | Guang Yang | Andras Gyori |
+| [YARN-10607](https://issues.apache.org/jira/browse/YARN-10607) | User environment is unable to prepend PATH when mapreduce.admin.user.env also sets PATH |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17516](https://issues.apache.org/jira/browse/HADOOP-17516) | Upgrade ant to 1.10.9 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10500](https://issues.apache.org/jira/browse/YARN-10500) | TestDelegationTokenRenewer fails intermittently |  Major | test | Akira Ajisaka | Masatake Iwasaki |
+| [HADOOP-17534](https://issues.apache.org/jira/browse/HADOOP-17534) | Upgrade Jackson databind to 2.10.5.1 |  Major | build | Adam Roberts | Akira Ajisaka |
+| [MAPREDUCE-7323](https://issues.apache.org/jira/browse/MAPREDUCE-7323) | Remove job\_history\_summary.py |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10647](https://issues.apache.org/jira/browse/YARN-10647) | Fix TestRMNodeLabelsManager failed after YARN-10501. |  Major | . | Qi Zhu | Qi Zhu |
+| [HADOOP-17510](https://issues.apache.org/jira/browse/HADOOP-17510) | Hadoop prints sensitive Cookie information. |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-15422](https://issues.apache.org/jira/browse/HDFS-15422) | Reported IBR is partially replaced with stored info when queuing. |  Critical | namenode | Kihwal Lee | Stephen O'Donnell |
+| [YARN-10651](https://issues.apache.org/jira/browse/YARN-10651) | CapacityScheduler crashed with NPE in AbstractYarnScheduler.updateNodeResource() |  Major | . | Haibo Chen | Haibo Chen |
+| [MAPREDUCE-7320](https://issues.apache.org/jira/browse/MAPREDUCE-7320) | ClusterMapReduceTestCase does not clean directories |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14013](https://issues.apache.org/jira/browse/HDFS-14013) | Skip any credentials stored in HDFS when starting ZKFC |  Major | hdfs | Krzysztof Adamski | Stephen O'Donnell |
+| [HDFS-15849](https://issues.apache.org/jira/browse/HDFS-15849) | ExpiredHeartbeats metric should be of Type.COUNTER |  Major | metrics | Konstantin Shvachko | Qi Zhu |
+| [YARN-10672](https://issues.apache.org/jira/browse/YARN-10672) | All testcases in TestReservations are flaky |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-17557](https://issues.apache.org/jira/browse/HADOOP-17557) | skip-dir option is not processed by Yetus |  Major | build, precommit, yetus | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15875](https://issues.apache.org/jira/browse/HDFS-15875) | Check whether file is being truncated before truncate |  Major | . | Hui Fei | Hui Fei |
+| [HADOOP-17582](https://issues.apache.org/jira/browse/HADOOP-17582) | Replace GitHub App Token with GitHub OAuth token |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10687](https://issues.apache.org/jira/browse/YARN-10687) | Add option to disable/enable free disk space checking and percentage checking for full and not-full disks |  Major | nodemanager | Qi Zhu | Qi Zhu |
+| [HADOOP-17586](https://issues.apache.org/jira/browse/HADOOP-17586) | Upgrade org.codehaus.woodstox:stax2-api to 4.2.1 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-17585](https://issues.apache.org/jira/browse/HADOOP-17585) | Correct timestamp format in the docs for the touch command |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-10588](https://issues.apache.org/jira/browse/YARN-10588) | Percentage of queue and cluster is zero in WebUI |  Major | . | Bilwa S T | Bilwa S T |
+| [MAPREDUCE-7322](https://issues.apache.org/jira/browse/MAPREDUCE-7322) | revisiting TestMRIntermediateDataEncryption |  Major | job submission, security, test | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17592](https://issues.apache.org/jira/browse/HADOOP-17592) | Fix the wrong CIDR range example in Proxy User documentation |  Minor | documentation | Kwangsun Noh | Kwangsun Noh |
+| [YARN-10706](https://issues.apache.org/jira/browse/YARN-10706) | Upgrade com.github.eirslett:frontend-maven-plugin to 1.11.2 |  Major | buid | Mingliang Liu | Mingliang Liu |
+| [MAPREDUCE-7325](https://issues.apache.org/jira/browse/MAPREDUCE-7325) | Intermediate data encryption is broken in LocalJobRunner |  Major | job submission, security | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10697](https://issues.apache.org/jira/browse/YARN-10697) | Resources are displayed in bytes in UI for schedulers other than capacity |  Major | . | Bilwa S T | Bilwa S T |
+| [HADOOP-17602](https://issues.apache.org/jira/browse/HADOOP-17602) | Upgrade JUnit to 4.13.1 |  Major | build, security, test | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15900](https://issues.apache.org/jira/browse/HDFS-15900) | RBF: empty blockpool id on dfsrouter caused by UNAVAILABLE NameNode |  Major | rbf | Harunobu Daikoku | Harunobu Daikoku |
+| [YARN-10501](https://issues.apache.org/jira/browse/YARN-10501) | Can't remove all node labels after add node label without nodemanager port |  Critical | yarn | caozhiqiang | caozhiqiang |
+| [YARN-10716](https://issues.apache.org/jira/browse/YARN-10716) | Fix typo in ContainerRuntime |  Trivial | documentation | Wanqiang Ji | xishuhai |
+| [HDFS-15950](https://issues.apache.org/jira/browse/HDFS-15950) | Remove unused hdfs.proto import |  Major | hdfs-client | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-15949](https://issues.apache.org/jira/browse/HDFS-15949) | Fix integer overflow |  Major | libhdfs++ | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-15948](https://issues.apache.org/jira/browse/HDFS-15948) | Fix test4tests for libhdfspp |  Critical | build, libhdfs++ | Gautham Banasandra | Gautham Banasandra |
+| [HADOOP-17608](https://issues.apache.org/jira/browse/HADOOP-17608) | Fix TestKMS failure |  Major | kms | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10460](https://issues.apache.org/jira/browse/YARN-10460) | Upgrading to JUnit 4.13 causes tests in TestNodeStatusUpdater to fail |  Major | nodemanager, test | Peter Bacsko | Peter Bacsko |
+| [HADOOP-17641](https://issues.apache.org/jira/browse/HADOOP-17641) | ITestWasbUriAndConfiguration.testCanonicalServiceName() failing now mockaccount exists |  Minor | fs/azure, test | Steve Loughran | Steve Loughran |
+| [HADOOP-17655](https://issues.apache.org/jira/browse/HADOOP-17655) | Upgrade Jetty to 9.4.40 |  Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10749](https://issues.apache.org/jira/browse/YARN-10749) | Can't remove all node labels after add node label without nodemanager port, broken by YARN-10647 |  Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy |
+| [HDFS-15621](https://issues.apache.org/jira/browse/HDFS-15621) | Datanode DirectoryScanner uses excessive memory |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-10756](https://issues.apache.org/jira/browse/YARN-10756) | Remove additional junit 4.11 dependency from javadoc |  Major | build, test, timelineservice | ANANDA G B | Akira Ajisaka |
+| [YARN-10555](https://issues.apache.org/jira/browse/YARN-10555) |  Missing access check before getAppAttempts |  Critical | webapp | lujie | lujie |
+| [HADOOP-17703](https://issues.apache.org/jira/browse/HADOOP-17703) | checkcompatibility.py errors out when specifying annotations |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-14922](https://issues.apache.org/jira/browse/HADOOP-14922) | Build of Mapreduce Native Task module fails with unknown opcode "bswap" |  Major | . | Anup Halarnkar | Anup Halarnkar |
+| [HADOOP-17718](https://issues.apache.org/jira/browse/HADOOP-17718) | Explicitly set locale in the Dockerfile |  Blocker | build | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-17700](https://issues.apache.org/jira/browse/HADOOP-17700) | ExitUtil#halt info log should log HaltException |  Major | . | Viraj Jasani | Viraj Jasani |
+| [YARN-10770](https://issues.apache.org/jira/browse/YARN-10770) | container-executor permission is wrong in SecureContainer.md |  Major | documentation | Akira Ajisaka | Siddharth Ahuja |
+| [HDFS-15915](https://issues.apache.org/jira/browse/HDFS-15915) | Race condition with async edits logging due to updating txId outside of the namesystem log |  Major | hdfs, namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-16040](https://issues.apache.org/jira/browse/HDFS-16040) | RpcQueueTime metric counts requeued calls as unique events. |  Major | hdfs | Simbarashe Dzinamarira | Simbarashe Dzinamarira |
+| [YARN-10809](https://issues.apache.org/jira/browse/YARN-10809) | testWithHbaseConfAtHdfsFileSystem consistently failing |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16055](https://issues.apache.org/jira/browse/HDFS-16055) | Quota is not preserved in snapshot INode |  Major | hdfs | Siyao Meng | Siyao Meng |
+| [HDFS-16068](https://issues.apache.org/jira/browse/HDFS-16068) | WebHdfsFileSystem has a possible connection leak in connection with HttpFS |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-10767](https://issues.apache.org/jira/browse/YARN-10767) | Yarn Logs Command retrying on Standby RM for 30 times |  Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy |
+| [HDFS-15618](https://issues.apache.org/jira/browse/HDFS-15618) | Improve datanode shutdown latency |  Major | datanode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17760](https://issues.apache.org/jira/browse/HADOOP-17760) | Delete hadoop.ssl.enabled and dfs.https.enable from docs and core-default.xml |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13671](https://issues.apache.org/jira/browse/HDFS-13671) | Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet |  Major | . | Yiqun Lin | Haibin Huang |
+| [HDFS-16061](https://issues.apache.org/jira/browse/HDFS-16061) | DFTestUtil.waitReplication can produce false positives |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient |  Major | . | Tao Yang | Renukaprasad C |
+| [YARN-10826](https://issues.apache.org/jira/browse/YARN-10826) | [UI2] Upgrade Node.js to at least v12.22.1 |  Major | yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki |
+| [YARN-10828](https://issues.apache.org/jira/browse/YARN-10828) | Backport YARN-9789 to branch-3.2 |  Major | . | Tarun Parimi | Tarun Parimi |
+| [HADOOP-17769](https://issues.apache.org/jira/browse/HADOOP-17769) | Upgrade JUnit to 4.13.2 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10824](https://issues.apache.org/jira/browse/YARN-10824) | Title not set for JHS and NM webpages |  Major | . | Rajshree Mishra | Bilwa S T |
+| [HDFS-16092](https://issues.apache.org/jira/browse/HDFS-16092) | Avoid creating LayoutFlags redundant objects |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16108](https://issues.apache.org/jira/browse/HDFS-16108) | Incorrect log placeholders used in JournalNodeSyncer |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [MAPREDUCE-7353](https://issues.apache.org/jira/browse/MAPREDUCE-7353) | Mapreduce job fails when NM is stopped |  Major | . | Bilwa S T | Bilwa S T |
+| [HDFS-16121](https://issues.apache.org/jira/browse/HDFS-16121) | Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames |  Major | snapshots | Srinivasu Majeti | Shashikant Banerjee |
+| [HDFS-15796](https://issues.apache.org/jira/browse/HDFS-15796) | ConcurrentModificationException error happens on NameNode occasionally |  Critical | hdfs | Daniel Ma | Daniel Ma |
+| [HADOOP-17793](https://issues.apache.org/jira/browse/HADOOP-17793) | Better token validation |  Major | . | Artem Smotrakov | Artem Smotrakov |
+| [HDFS-16042](https://issues.apache.org/jira/browse/HDFS-16042) | DatanodeAdminMonitor scan should be delay based |  Major | datanode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-16127](https://issues.apache.org/jira/browse/HDFS-16127) | Improper pipeline close recovery causes a permanent write failure or data loss. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-17028](https://issues.apache.org/jira/browse/HADOOP-17028) | ViewFS should initialize target filesystems lazily |  Major | client-mounts, fs, viewfs | Uma Maheswara Rao G | Abhishek Das |
+| [HDFS-12920](https://issues.apache.org/jira/browse/HDFS-12920) | HDFS default value change (with adding time unit) breaks old version MR tarball work with Hadoop 3.x |  Critical | configuration, hdfs | Junping Du | Akira Ajisaka |
+| [YARN-10813](https://issues.apache.org/jira/browse/YARN-10813) | Set default capacity of root for node labels |  Major | . | Andras Gyori | Andras Gyori |
+| [YARN-9551](https://issues.apache.org/jira/browse/YARN-9551) | TestTimelineClientV2Impl.testSyncCall fails intermittently |  Minor | ATSv2, test | Prabhu Joseph | Andras Gyori |
+| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog |  Critical | . | Yicong Cai | Wan Chang |
+| [YARN-10789](https://issues.apache.org/jira/browse/YARN-10789) | RM HA startup can fail due to race conditions in ZKConfigurationStore |  Major | . | Tarun Parimi | Tarun Parimi |
+| [YARN-6221](https://issues.apache.org/jira/browse/YARN-6221) | Entities missing from ATS when summary log file info got returned to the ATS before the domain log |  Critical | yarn | Sushmitha Sreenivasan | Xiaomin Zhang |
+| [MAPREDUCE-7258](https://issues.apache.org/jira/browse/MAPREDUCE-7258) | HistoryServerRest.html#Task\_Counters\_API, modify the jobTaskCounters's itemName from "taskcounterGroup" to "taskCounterGroup". |  Minor | documentation | jenny | jenny |
+| [YARN-8990](https://issues.apache.org/jira/browse/YARN-8990) | Fix fair scheduler race condition in app submit and queue cleanup |  Blocker | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-8992](https://issues.apache.org/jira/browse/YARN-8992) | Fair scheduler can delete a dynamic queue while an application attempt is being added to the queue |  Major | fairscheduler | Haibo Chen | Wilfred Spiegelenburg |
+| [HADOOP-17370](https://issues.apache.org/jira/browse/HADOOP-17370) | Upgrade commons-compress to 1.21 |  Major | common | Dongjoon Hyun | Akira Ajisaka |
+| [HADOOP-17844](https://issues.apache.org/jira/browse/HADOOP-17844) | Upgrade JSON smart to 2.4.7 |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HADOOP-17850](https://issues.apache.org/jira/browse/HADOOP-17850) | Upgrade ZooKeeper to 3.4.14 in branch-3.2 |  Major | . | Akira Ajisaka | Masatake Iwasaki |
+| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile |  Minor | . | tomscut | tomscut |
+| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file |  Major | . | Benjamin Teke | Tamas Domok |
+| [HADOOP-17858](https://issues.apache.org/jira/browse/HADOOP-17858) | Avoid possible class loading deadlock with VerifierNone initialization |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17886](https://issues.apache.org/jira/browse/HADOOP-17886) | Upgrade ant to 1.10.11 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir |  Major | nodemanager | Tamas Domok | Tamas Domok |
+| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing |  Major | snapshots | Srinivasu Majeti | Shashikant Banerjee |
+| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown |  Major | . | Eungsop Yoo | Eungsop Yoo |
+| [HADOOP-17917](https://issues.apache.org/jira/browse/HADOOP-17917) | Backport HADOOP-15993 to branch-3.2 which address CVE-2014-4611 |  Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-16233](https://issues.apache.org/jira/browse/HDFS-16233) | Do not use exception handler to implement copy-on-write for EnumCounters |  Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method |  Major | hdfs | angerszhu | angerszhu |
+| [HADOOP-17940](https://issues.apache.org/jira/browse/HADOOP-17940) | Upgrade Kafka to 2.8.1 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-16272](https://issues.apache.org/jira/browse/HDFS-16272) | Int overflow in computing safe length during EC block recovery |  Critical | 3.1.1 | daimin | daimin |
+| [HADOOP-17971](https://issues.apache.org/jira/browse/HADOOP-17971) | Exclude IBM Java security classes from being shaded/relocated |  Major | build | Nicholas Marion | Nicholas Marion |
+| [HADOOP-17972](https://issues.apache.org/jira/browse/HADOOP-17972) | Backport HADOOP-17683 for branch-3.2 |  Major | security | Ananya Singh | Ananya Singh |
+| [HADOOP-17993](https://issues.apache.org/jira/browse/HADOOP-17993) | Disable JIRA plugin for YETUS on Hadoop |  Critical | build | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in  BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage |  Major | namanode | Max  Xie | Max  Xie |
+| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-13500](https://issues.apache.org/jira/browse/HADOOP-13500) | Synchronizing iteration of Configuration properties object |  Major | conf | Jason Darrell Lowe | Dhananjay Badaya |
+| [HDFS-16317](https://issues.apache.org/jira/browse/HDFS-16317) | Backport HDFS-14729 for branch-3.2 |  Major | security | Ananya Singh | Ananya Singh |
+| [HDFS-14099](https://issues.apache.org/jira/browse/HDFS-14099) | Unknown frame descriptor when decompressing multiple frames in ZStandardDecompressor |  Major | . | xuzq | xuzq |
+| [HDFS-16410](https://issues.apache.org/jira/browse/HDFS-16410) | Insecure Xml parsing in OfflineEditsXmlLoader |  Minor | . | Ashutosh Gupta | Ashutosh Gupta |
+| [HDFS-16420](https://issues.apache.org/jira/browse/HDFS-16420) | Avoid deleting unique data blocks when deleting redundancy striped blocks |  Critical | ec, erasure-coding | qinyuren | Jackson Wang |
+| [HDFS-16428](https://issues.apache.org/jira/browse/HDFS-16428) | Source path with storagePolicy cause wrong typeConsumed while rename |  Major | hdfs, namenode | lei w | lei w |
+| [HDFS-16437](https://issues.apache.org/jira/browse/HDFS-16437) | ReverseXML processor doesn't accept XML files without the SnapshotDiffSection. |  Critical | hdfs | yanbin.zhang | yanbin.zhang |
+| [HDFS-16422](https://issues.apache.org/jira/browse/HDFS-16422) | Fix thread safety of EC decoding during concurrent preads |  Critical | dfsclient, ec, erasure-coding | daimin | daimin |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-9338](https://issues.apache.org/jira/browse/YARN-9338) | Timeline related testcases are failing |  Major | . | Prabhu Joseph | Abhishek Modi |
+| [HDFS-15092](https://issues.apache.org/jira/browse/HDFS-15092) | TestRedudantBlocks#testProcessOverReplicatedAndRedudantBlock sometimes fails |  Minor | test | Hui Fei | Hui Fei |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15775](https://issues.apache.org/jira/browse/HADOOP-15775) | [JDK9] Add missing javax.activation-api dependency |  Critical | test | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9875](https://issues.apache.org/jira/browse/YARN-9875) | FSSchedulerConfigurationStore fails to update with hdfs path |  Major | capacityscheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16764](https://issues.apache.org/jira/browse/HADOOP-16764) | Rewrite Python example codes using Python3 |  Minor | documentation | Kengo Seki | Kengo Seki |
+| [HADOOP-16905](https://issues.apache.org/jira/browse/HADOOP-16905) | Update jackson-databind to 2.10.3 to relieve us from the endless CVE patches |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-10337](https://issues.apache.org/jira/browse/YARN-10337) | TestRMHATimelineCollectors fails on hadoop trunk |  Major | test, yarn | Ahmed Hussein | Bilwa S T |
+| [HDFS-15464](https://issues.apache.org/jira/browse/HDFS-15464) | ViewFsOverloadScheme should work when -fs option pointing to remote cluster without mount links |  Major | viewfsOverloadScheme | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HDFS-15478](https://issues.apache.org/jira/browse/HDFS-15478) | When Empty mount points, we are assigning fallback link to self. But it should not use full URI for target fs. |  Major | . | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HDFS-15459](https://issues.apache.org/jira/browse/HDFS-15459) | TestBlockTokenWithDFSStriped fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15461](https://issues.apache.org/jira/browse/HDFS-15461) | TestDFSClientRetries#testGetFileChecksum fails intermittently |  Major | dfsclient, test | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-9776](https://issues.apache.org/jira/browse/HDFS-9776) | TestHAAppend#testMultipleAppendsDuringCatchupTailing is flaky |  Major | . | Vinayakumar B | Ahmed Hussein |
+| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17330](https://issues.apache.org/jira/browse/HADOOP-17330) | Backport HADOOP-16005-"NativeAzureFileSystem does not support setXAttr" to branch-3.2 |  Major | fs/azure | Sally Zuo | Sally Zuo |
+| [HDFS-15643](https://issues.apache.org/jira/browse/HDFS-15643) | EC: Fix checksum computation in case of native encoders |  Blocker | . | Ahmed Hussein | Ayush Saxena |
+| [HADOOP-17325](https://issues.apache.org/jira/browse/HADOOP-17325) | WASB: Test failures |  Major | fs/azure, test | Sneha Vijayarajan | Steve Loughran |
+| [HDFS-15716](https://issues.apache.org/jira/browse/HDFS-15716) | TestUpgradeDomainBlockPlacementPolicy flaky |  Major | namenode, test | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15762](https://issues.apache.org/jira/browse/HDFS-15762) | TestMultipleNNPortQOP#testMultipleNNPortOverwriteDownStream fails intermittently |  Minor | . | Toshihiko Uchida | Toshihiko Uchida |
+| [HDFS-15672](https://issues.apache.org/jira/browse/HDFS-15672) | TestBalancerWithMultipleNameNodes#testBalancingBlockpoolsWithBlockPoolPolicy fails on trunk |  Major | . | Ahmed Hussein | Masatake Iwasaki |
+| [HDFS-15818](https://issues.apache.org/jira/browse/HDFS-15818) | Fix TestFsDatasetImpl.testReadLockCanBeDisabledByConfig |  Minor | test | Leon Gao | Leon Gao |
+| [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | Migrate to Python 3 and upgrade Yetus to 0.13.0 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15890](https://issues.apache.org/jira/browse/HDFS-15890) | Improve the Logs for File Concat Operation |  Minor | namenode | Bhavik Patel | Bhavik Patel |
+| [HDFS-13975](https://issues.apache.org/jira/browse/HDFS-13975) | TestBalancer#testMaxIterationTime fails sporadically |  Major | . | Jason Darrell Lowe | Toshihiko Uchida |
+| [YARN-10688](https://issues.apache.org/jira/browse/YARN-10688) | ClusterMetrics should support GPU capacity related metrics. |  Major | metrics, resourcemanager | Qi Zhu | Qi Zhu |
+| [HDFS-15902](https://issues.apache.org/jira/browse/HDFS-15902) | Improve the log for HTTPFS server operation |  Minor | httpfs | Bhavik Patel | Bhavik Patel |
+| [HDFS-15940](https://issues.apache.org/jira/browse/HDFS-15940) | Some tests in TestBlockRecovery are consistently failing |  Major | . | Viraj Jasani | Viraj Jasani |
+| [YARN-10702](https://issues.apache.org/jira/browse/YARN-10702) | Add cluster metric for amount of CPU used by RM Event Processor |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-17630](https://issues.apache.org/jira/browse/HADOOP-17630) | [JDK 15] TestPrintableString fails due to Unicode 13.0 support |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10723](https://issues.apache.org/jira/browse/YARN-10723) | Change CS nodes page in UI to support custom resource. |  Major | . | Qi Zhu | Qi Zhu |
+| [HADOOP-17112](https://issues.apache.org/jira/browse/HADOOP-17112) | whitespace not allowed in paths when saving files to s3a via committer |  Blocker | fs/s3 | Krzysztof Adamski | Krzysztof Adamski |
+| [HADOOP-17661](https://issues.apache.org/jira/browse/HADOOP-17661) | mvn versions:set fails to parse pom.xml |  Blocker | build | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-10642](https://issues.apache.org/jira/browse/YARN-10642) | Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995 |  Critical | resourcemanager | zhengchenyu | zhengchenyu |
+| [HDFS-15659](https://issues.apache.org/jira/browse/HDFS-15659) | Set dfs.namenode.redundancy.considerLoad to false in MiniDFSCluster |  Major | test | Akira Ajisaka | Ahmed Hussein |
+| [HADOOP-17840](https://issues.apache.org/jira/browse/HADOOP-17840) | Backport HADOOP-17837 to branch-3.2 |  Minor | . | Bryan Beaudreault | Bryan Beaudreault |
+| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull |  Major | . | Ahmed Hussein | Ahmed Hussein |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15870](https://issues.apache.org/jira/browse/HDFS-15870) | Remove unused configuration dfs.namenode.stripe.min |  Minor | . | tomscut | tomscut |
+| [HDFS-15808](https://issues.apache.org/jira/browse/HDFS-15808) | Add metrics for FSNamesystem read/write lock hold long time |  Major | hdfs | tomscut | tomscut |
+| [HDFS-15873](https://issues.apache.org/jira/browse/HDFS-15873) | Add namenode address in logs for block report |  Minor | datanode, hdfs | tomscut | tomscut |
+| [HDFS-15906](https://issues.apache.org/jira/browse/HDFS-15906) | Close FSImage and FSNamesystem after formatting is complete |  Minor | . | tomscut | tomscut |
+| [HDFS-15892](https://issues.apache.org/jira/browse/HDFS-15892) | Add metric for editPendingQ in FSEditLogAsync |  Minor | . | tomscut | tomscut |
+| [HDFS-16078](https://issues.apache.org/jira/browse/HDFS-16078) | Remove unused parameters for DatanodeManager.handleLifeline() |  Minor | . | tomscut | tomscut |
+| [YARN-10278](https://issues.apache.org/jira/browse/YARN-10278) | CapacityScheduler test framework ProportionalCapacityPreemptionPolicyMockFramework need some review |  Major | . | Gergely Pollák | Szilard Nemeth |
+| [HDFS-15731](https://issues.apache.org/jira/browse/HDFS-15731) | Reduce threadCount for unit tests to reduce the memory usage |  Major | build, test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17571](https://issues.apache.org/jira/browse/HADOOP-17571) | Upgrade com.fasterxml.woodstox:woodstox-core for security reasons |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-15895](https://issues.apache.org/jira/browse/HDFS-15895) | DFSAdmin#printOpenFiles has redundant String#format usage |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17614](https://issues.apache.org/jira/browse/HADOOP-17614) | Bump netty to the latest 4.1.61 |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-17627](https://issues.apache.org/jira/browse/HADOOP-17627) | Backport to branch-3.2 HADOOP-17371, HADOOP-17621, HADOOP-17625 to update Jetty to 9.4.39 |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-15989](https://issues.apache.org/jira/browse/HDFS-15989) | Split TestBalancer into two classes |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17808](https://issues.apache.org/jira/browse/HADOOP-17808) | ipc.Client not setting interrupt flag after catching InterruptedException |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17834](https://issues.apache.org/jira/browse/HADOOP-17834) | Bump aliyun-sdk-oss to 3.13.0 |  Major | . | Siyao Meng | Siyao Meng |
+| [HADOOP-17955](https://issues.apache.org/jira/browse/HADOOP-17955) | Bump netty to the latest 4.1.68 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-18061](https://issues.apache.org/jira/browse/HADOOP-18061) | Update the year to 2022 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-18125](https://issues.apache.org/jira/browse/HADOOP-18125) | Utility to identify git commit / Jira fixVersion discrepancies for RC preparation |  Major | . | Viraj Jasani | Viraj Jasani |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md
new file mode 100644
index 0000000000000..5c53bb4cb876b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.3/RELEASENOTES.3.2.3.md
@@ -0,0 +1,71 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.2.3 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | *Major* | **Install yarnpkg and upgrade nodejs in Dockerfile**
+
+In the Dockerfile, nodejs is upgraded to 8.17.0 and yarn 1.12.1 is installed.
+
+
+---
+
+* [HADOOP-16054](https://issues.apache.org/jira/browse/HADOOP-16054) | *Major* | **Update Dockerfile to use Bionic**
+
+The build image has been upgraded to Bionic.
+
+
+---
+
+* [HDFS-15719](https://issues.apache.org/jira/browse/HDFS-15719) | *Critical* | **[Hadoop 3] Both NameNodes can crash simultaneously due to the short JN socket timeout**
+
+The default value of the configuration hadoop.http.idle\_timeout.ms (how long does Jetty disconnect an idle connection) is changed from 10000 to 60000. 
+This property is inlined during compile time, so an application that references this property must be recompiled in order for it to take effect.
+
+
+---
+
+* [HADOOP-16748](https://issues.apache.org/jira/browse/HADOOP-16748) | *Major* | **Migrate to Python 3 and upgrade Yetus to 0.13.0**
+
+<!-- markdown -->
+- Upgraded Yetus to 0.13.0.
+- Removed determine-flaky-tests-hadoop.py.
+- Temporarily disabled shelldocs check in the Jenkins jobs due to YETUS-1099.
+
+
+---
+
+* [HADOOP-16870](https://issues.apache.org/jira/browse/HADOOP-16870) | *Major* | **Use spotbugs-maven-plugin instead of findbugs-maven-plugin**
+
+Removed findbugs from the hadoop build images and added spotbugs instead.
+Upgraded SpotBugs to 4.2.2 and spotbugs-maven-plugin to 4.2.0.
+
+
+---
+
+* [HDFS-15942](https://issues.apache.org/jira/browse/HDFS-15942) | *Major* | **Increase Quota initialization threads**
+
+The default quota initialization thread count during the NameNode startup process (dfs.namenode.quota.init-threads) is increased from 4 to 12.
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md
new file mode 100644
index 0000000000000..fc0079d1c9bd8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md
@@ -0,0 +1,213 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.2.4 - 2022-07-12
+
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web |  Minor | . | Tao Li | Tao Li |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor |  Major | . | Íñigo Goiri | Xiaoqiao He |
+| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-16175](https://issues.apache.org/jira/browse/HDFS-16175) | Improve the configurable value of Server #PURGE\_INTERVAL\_NANOS |  Major | ipc | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability |  Major | fs | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-17897](https://issues.apache.org/jira/browse/HADOOP-17897) | Allow nested blocks in switch case in checkstyle settings |  Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17857](https://issues.apache.org/jira/browse/HADOOP-17857) | Check real user ACLs in addition to proxied user ACLs |  Major | . | Eric Payne | Eric Payne |
+| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously |  Major | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-17926](https://issues.apache.org/jira/browse/HADOOP-17926) | Maven-eclipse-plugin is no longer needed since Eclipse can import Maven projects by itself. |  Minor | documentation | Rintaro Ikeda | Rintaro Ikeda |
+| [YARN-10935](https://issues.apache.org/jira/browse/YARN-10935) | AM Total Queue Limit goes below per-user AM Limit if parent is full. |  Major | capacity scheduler, capacityscheduler | Eric Payne | Eric Payne |
+| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread |  Major | . | zhanghuazong | zhanghuazong |
+| [YARN-1115](https://issues.apache.org/jira/browse/YARN-1115) | Provide optional means for a scheduler to check real user ACLs |  Major | capacity scheduler, scheduler | Eric Payne |  |
+| [HDFS-16279](https://issues.apache.org/jira/browse/HDFS-16279) | Print detail datanode info when process first storage report |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16294](https://issues.apache.org/jira/browse/HDFS-16294) | Remove invalid DataNode#CONFIG\_PROPERTY\_SIMULATED |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16299](https://issues.apache.org/jira/browse/HDFS-16299) | Fix bug for TestDataNodeVolumeMetrics#verifyDataNodeVolumeMetrics |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16301](https://issues.apache.org/jira/browse/HDFS-16301) | Improve BenchmarkThroughput#SIZE naming standardization |  Minor | benchmarks, test | JiangHua Zhu | JiangHua Zhu |
+| [YARN-10997](https://issues.apache.org/jira/browse/YARN-10997) | Revisit allocation and reservation logging |  Major | . | Andras Gyori | Andras Gyori |
+| [HDFS-16315](https://issues.apache.org/jira/browse/HDFS-16315) | Add metrics related to Transfer and NativeCopy for DataNode |  Major | . | Tao Li | Tao Li |
+| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. |  Major | fs | Chengwei Wang | Chengwei Wang |
+| [HDFS-16345](https://issues.apache.org/jira/browse/HDFS-16345) | Fix test cases fail in TestBlockStoragePolicy |  Major | build | guophilipse | guophilipse |
+| [HADOOP-18035](https://issues.apache.org/jira/browse/HADOOP-18035) | Skip unit test failures to run all the unit tests |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-18040](https://issues.apache.org/jira/browse/HADOOP-18040) | Use maven.test.failure.ignore instead of ignoreTestFailure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-16352](https://issues.apache.org/jira/browse/HDFS-16352) | return the real datanode numBlocks in #getDatanodeStorageReport |  Major | . | qinyuren | qinyuren |
+| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16391](https://issues.apache.org/jira/browse/HDFS-16391) | Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService |  Trivial | . | wangzhaohui | wangzhaohui |
+| [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | Improve RM system metrics publisher's performance by pushing events to timeline server in batch |  Critical | resourcemanager, timelineserver | Hu Ziqian | Ashutosh Gupta |
+| [HDFS-16430](https://issues.apache.org/jira/browse/HDFS-16430) | Validate maximum blocks in EC group when adding an EC policy |  Minor | ec, erasure-coding | daimin | daimin |
+| [HDFS-16403](https://issues.apache.org/jira/browse/HDFS-16403) | Improve FUSE IO performance by supporting FUSE parameter max\_background |  Minor | fuse-dfs | daimin | daimin |
+| [HADOOP-18136](https://issues.apache.org/jira/browse/HADOOP-18136) | Verify FileUtils.unTar() handling of missing .tar files |  Minor | test, util | Steve Loughran | Steve Loughran |
+| [HDFS-16529](https://issues.apache.org/jira/browse/HDFS-16529) | Remove unnecessary setObserverRead in TestConsistentReadsObserver |  Trivial | test | wangzhaohui | wangzhaohui |
+| [HDFS-16530](https://issues.apache.org/jira/browse/HDFS-16530) | setReplication debug log creates a new string even if debug is disabled |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-16427](https://issues.apache.org/jira/browse/HDFS-16427) | Add debug log for BlockManager#chooseExcessRedundancyStriped |  Minor | erasure-coding | Tao Li | Tao Li |
+| [HDFS-16389](https://issues.apache.org/jira/browse/HDFS-16389) | Improve NNThroughputBenchmark test mkdirs |  Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu |
+| [MAPREDUCE-7373](https://issues.apache.org/jira/browse/MAPREDUCE-7373) | Building MapReduce NativeTask fails on Fedora 34+ |  Major | build, nativetask | Kengo Seki | Kengo Seki |
+| [HDFS-16355](https://issues.apache.org/jira/browse/HDFS-16355) | Improve the description of dfs.block.scanner.volume.bytes.per.second |  Minor | documentation, hdfs | guophilipse | guophilipse |
+| [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | Replace log4j 1.x with reload4j |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16501](https://issues.apache.org/jira/browse/HDFS-16501) | Print the exception when reporting a bad block |  Major | datanode | qinyuren | qinyuren |
+| [YARN-11116](https://issues.apache.org/jira/browse/YARN-11116) | Migrate Times util from SimpleDateFormat to thread-safe DateTimeFormatter class |  Minor | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
+| [YARN-10080](https://issues.apache.org/jira/browse/YARN-10080) | Support show app id on localizer thread pool |  Major | nodemanager | zhoukang | Ashutosh Gupta |
+| [HADOOP-18240](https://issues.apache.org/jira/browse/HADOOP-18240) | Upgrade Yetus to 0.14.0 |  Major | build | Akira Ajisaka | Ashutosh Gupta |
+| [HDFS-16585](https://issues.apache.org/jira/browse/HDFS-16585) | Add @VisibleForTesting in Dispatcher.java after HDFS-16268 |  Trivial | . | Wei-Chiu Chuang | Ashutosh Gupta |
+| [HDFS-16610](https://issues.apache.org/jira/browse/HDFS-16610) | Make fsck read timeout configurable |  Major | hdfs-client | Stephen O'Donnell | Stephen O'Donnell |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13983](https://issues.apache.org/jira/browse/HDFS-13983) | TestOfflineImageViewer crashes in windows |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-9744](https://issues.apache.org/jira/browse/YARN-9744) | RollingLevelDBTimelineStore.getEntityByTime fails with NPE |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature |  Blocker | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-16985](https://issues.apache.org/jira/browse/HADOOP-16985) | Handle release package related issues |  Major | . | Vinayakumar B | Vinayakumar B |
+| [HADOOP-17116](https://issues.apache.org/jira/browse/HADOOP-17116) | Skip Retry INFO logging on first failover from a proxy |  Major | ha | Hanisha Koneru | Hanisha Koneru |
+| [HDFS-15651](https://issues.apache.org/jira/browse/HDFS-15651) | Client could not obtain block when DN CommandProcessingThread exit |  Major | . | Yiqun Lin | Mingxiang Li |
+| [HDFS-15963](https://issues.apache.org/jira/browse/HDFS-15963) | Unreleased volume references cause an infinite loop |  Critical | datanode | Shuyan Zhang | Shuyan Zhang |
+| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient |  Major | . | Tao Yang | Renukaprasad C |
+| [HADOOP-17796](https://issues.apache.org/jira/browse/HADOOP-17796) | Upgrade jetty version to 9.4.43 |  Major | . | Wei-Chiu Chuang | Renukaprasad C |
+| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog |  Critical | . | Yicong Cai | Wan Chang |
+| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile |  Minor | . | Tao Li | Tao Li |
+| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file |  Major | . | Benjamin Teke | Tamas Domok |
+| [HADOOP-17874](https://issues.apache.org/jira/browse/HADOOP-17874) | ExceptionsHandler to add terse/suppressed Exceptions in thread-safe manner |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-15129](https://issues.apache.org/jira/browse/HADOOP-15129) | Datanode caches namenode DNS lookup failure and cannot startup |  Minor | ipc | Karthik Palaniappan | Chris Nauroth |
+| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir |  Major | nodemanager | Tamas Domok | Tamas Domok |
+| [HDFS-16207](https://issues.apache.org/jira/browse/HDFS-16207) | Remove NN logs stack trace for non-existent xattr query |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing |  Major | snapshots | Srinivasu Majeti | Shashikant Banerjee |
+| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown |  Major | . | Eungsop Yoo | Eungsop Yoo |
+| [YARN-10870](https://issues.apache.org/jira/browse/YARN-10870) | Missing user filtering check -\> yarn.webapp.filter-entity-list-by-user for RM Scheduler page |  Major | yarn | Siddharth Ahuja | Gergely Pollák |
+| [HADOOP-17919](https://issues.apache.org/jira/browse/HADOOP-17919) | Fix command line example in Hadoop Cluster Setup documentation |  Minor | documentation | Rintaro Ikeda | Rintaro Ikeda |
+| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method |  Major | hdfs | angerszhu | angerszhu |
+| [HDFS-16181](https://issues.apache.org/jira/browse/HDFS-16181) | [SBN Read] Fix metric of RpcRequestCacheMissAmount can't display when tailEditLog form JN |  Critical | . | wangzhaohui | wangzhaohui |
+| [HADOOP-17925](https://issues.apache.org/jira/browse/HADOOP-17925) | BUILDING.txt should not encourage to activate docs profile on building binary artifacts |  Minor | documentation | Rintaro Ikeda | Masatake Iwasaki |
+| [HADOOP-16532](https://issues.apache.org/jira/browse/HADOOP-16532) | Fix TestViewFsTrash to use the correct homeDir. |  Minor | test, viewfs | Steve Loughran | Xing Lin |
+| [HDFS-16268](https://issues.apache.org/jira/browse/HDFS-16268) | Balancer stuck when moving striped blocks due to NPE |  Major | balancer & mover, erasure-coding | Leon Gao | Leon Gao |
+| [HDFS-7612](https://issues.apache.org/jira/browse/HDFS-7612) | TestOfflineEditsViewer.testStored() uses incorrect default value for cacheDir |  Major | test | Konstantin Shvachko | Michael Kuchenbecker |
+| [HDFS-16311](https://issues.apache.org/jira/browse/HDFS-16311) | Metric metadataOperationRate calculation error in DataNodeVolumeMetrics |  Major | . | Tao Li | Tao Li |
+| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in  BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage |  Major | namanode | Max  Xie | Max  Xie |
+| [HADOOP-17999](https://issues.apache.org/jira/browse/HADOOP-17999) | No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem |  Major | . | Abhishek Das | Abhishek Das |
+| [HDFS-16329](https://issues.apache.org/jira/browse/HDFS-16329) | Fix log format for BlockManager |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16330](https://issues.apache.org/jira/browse/HDFS-16330) | Fix incorrect placeholder for Exception logs in DiskBalancer |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16328](https://issues.apache.org/jira/browse/HDFS-16328) | Correct disk balancer param desc |  Minor | documentation, hdfs | guophilipse | guophilipse |
+| [HDFS-16343](https://issues.apache.org/jira/browse/HDFS-16343) | Add some debug logs when the dfsUsed are not used during Datanode startup |  Major | datanode | Mukul Kumar Singh | Mukul Kumar Singh |
+| [YARN-10991](https://issues.apache.org/jira/browse/YARN-10991) | Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method |  Minor | distributed-shell | Ashutosh Gupta | Ashutosh Gupta |
+| [HADOOP-17975](https://issues.apache.org/jira/browse/HADOOP-17975) | Fallback to simple auth does not work for a secondary DistributedFileSystem instance |  Major | ipc | István Fajth | István Fajth |
+| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [YARN-11007](https://issues.apache.org/jira/browse/YARN-11007) | Correct words in YARN documents |  Minor | documentation | guophilipse | guophilipse |
+| [HDFS-16332](https://issues.apache.org/jira/browse/HDFS-16332) | Expired block token causes slow read due to missing handling in sasl handshake |  Major | datanode, dfs, dfsclient | Shinya Yoshida | Shinya Yoshida |
+| [YARN-9063](https://issues.apache.org/jira/browse/YARN-9063) | ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing |  Major | timelineserver, timelineservice | Tarun Parimi | Ashutosh Gupta |
+| [HDFS-16333](https://issues.apache.org/jira/browse/HDFS-16333) | fix balancer bug when transfer an EC block |  Major | balancer & mover, erasure-coding | qinyuren | qinyuren |
+| [HDFS-16373](https://issues.apache.org/jira/browse/HDFS-16373) | Fix MiniDFSCluster restart in case of multiple namenodes |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-16377](https://issues.apache.org/jira/browse/HDFS-16377) | Should CheckNotNull before access FsDatasetSpi |  Major | . | Tao Li | Tao Li |
+| [YARN-6862](https://issues.apache.org/jira/browse/YARN-6862) | Nodemanager resource usage metrics sometimes are negative |  Major | nodemanager | YunFan Zhou | Benjamin Teke |
+| [YARN-10178](https://issues.apache.org/jira/browse/YARN-10178) | Global Scheduler async thread crash caused by 'Comparison method violates its general contract |  Major | capacity scheduler | tuyu | Andras Gyori |
+| [HDFS-16395](https://issues.apache.org/jira/browse/HDFS-16395) | Remove useless NNThroughputBenchmark#dummyActionNoSynch() |  Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-18063](https://issues.apache.org/jira/browse/HADOOP-18063) | Remove unused import AbstractJavaKeyStoreProvider in Shell class |  Minor | . | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16409](https://issues.apache.org/jira/browse/HDFS-16409) | Fix typo: testHasExeceptionsReturnsCorrectValue -\> testHasExceptionsReturnsCorrectValue |  Trivial | . | Ashutosh Gupta | Ashutosh Gupta |
+| [HDFS-16408](https://issues.apache.org/jira/browse/HDFS-16408) | Ensure LeaseRecheckIntervalMs is greater than zero |  Major | namenode | Jingxuan Fu | Jingxuan Fu |
+| [YARN-11055](https://issues.apache.org/jira/browse/YARN-11055) | In cgroups-operations.c some fprintf format strings don't end with "\\n" |  Minor | nodemanager | Gera Shegalov | Gera Shegalov |
+| [HDFS-16303](https://issues.apache.org/jira/browse/HDFS-16303) | Losing over 100 datanodes in state decommissioning results in full blockage of all datanode decommissioning |  Major | . | Kevin Wikant | Kevin Wikant |
+| [HDFS-16443](https://issues.apache.org/jira/browse/HDFS-16443) | Fix edge case where DatanodeAdminDefaultMonitor doubly enqueues a DatanodeDescriptor on exception |  Major | hdfs | Kevin Wikant | Kevin Wikant |
+| [HDFS-16449](https://issues.apache.org/jira/browse/HDFS-16449) | Fix hadoop web site release notes and changelog not available |  Minor | documentation | guophilipse | guophilipse |
+| [HADOOP-18192](https://issues.apache.org/jira/browse/HADOOP-18192) | Fix multiple\_bindings warning about slf4j-reload4j |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-16479](https://issues.apache.org/jira/browse/HDFS-16479) | EC: NameNode should not send a reconstruction work when the source datanodes are insufficient |  Critical | ec, erasure-coding | Yuanbo Liu | Takanobu Asanuma |
+| [HDFS-16509](https://issues.apache.org/jira/browse/HDFS-16509) | Fix decommission UnsupportedOperationException: Remove unsupported |  Major | namenode | daimin | daimin |
+| [HDFS-16456](https://issues.apache.org/jira/browse/HDFS-16456) | EC: Decommission a rack with only on dn will fail when the rack number is equal with replication |  Critical | ec, namenode | caozhiqiang | caozhiqiang |
+| [HDFS-16437](https://issues.apache.org/jira/browse/HDFS-16437) | ReverseXML processor doesn't accept XML files without the SnapshotDiffSection. |  Critical | hdfs | yanbin.zhang | yanbin.zhang |
+| [HDFS-16507](https://issues.apache.org/jira/browse/HDFS-16507) | [SBN read] Avoid purging edit log which is in progress |  Critical | . | Tao Li | Tao Li |
+| [YARN-10720](https://issues.apache.org/jira/browse/YARN-10720) | YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging |  Critical | . | Qi Zhu | Qi Zhu |
+| [HDFS-16428](https://issues.apache.org/jira/browse/HDFS-16428) | Source path with storagePolicy cause wrong typeConsumed while rename |  Major | hdfs, namenode | lei w | lei w |
+| [YARN-11014](https://issues.apache.org/jira/browse/YARN-11014) | YARN incorrectly validates maximum capacity resources on the validation API |  Major | . | Benjamin Teke | Benjamin Teke |
+| [YARN-11075](https://issues.apache.org/jira/browse/YARN-11075) | Explicitly declare serialVersionUID in LogMutation class |  Major | . | Benjamin Teke | Benjamin Teke |
+| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice |  Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16538](https://issues.apache.org/jira/browse/HDFS-16538) |  EC decoding failed due to not enough valid inputs |  Major | erasure-coding | qinyuren | qinyuren |
+| [HDFS-16544](https://issues.apache.org/jira/browse/HDFS-16544) | EC decoding failed due to invalid buffer |  Major | erasure-coding | qinyuren | qinyuren |
+| [HDFS-16546](https://issues.apache.org/jira/browse/HDFS-16546) | Fix UT TestOfflineImageViewer#testReverseXmlWithoutSnapshotDiffSection to branch branch-3.2 |  Major | test | daimin | daimin |
+| [HDFS-16552](https://issues.apache.org/jira/browse/HDFS-16552) | Fix NPE for TestBlockManager |  Major | . | Tao Li | Tao Li |
+| [MAPREDUCE-7246](https://issues.apache.org/jira/browse/MAPREDUCE-7246) |  In MapredAppMasterRest#Mapreduce\_Application\_Master\_Info\_API, the datatype of appId should be "string". |  Major | documentation | jenny | Ashutosh Gupta |
+| [YARN-10187](https://issues.apache.org/jira/browse/YARN-10187) | Removing hadoop-yarn-project/hadoop-yarn/README as it is no longer maintained. |  Minor | documentation | N Sanketh Reddy | Ashutosh Gupta |
+| [HDFS-16185](https://issues.apache.org/jira/browse/HDFS-16185) | Fix comment in LowRedundancyBlocks.java |  Minor | documentation | Akira Ajisaka | Ashutosh Gupta |
+| [HADOOP-17479](https://issues.apache.org/jira/browse/HADOOP-17479) | Fix the examples of hadoop config prefix |  Minor | documentation | Akira Ajisaka | Ashutosh Gupta |
+| [HDFS-16579](https://issues.apache.org/jira/browse/HDFS-16579) | Fix build failure for TestBlockManager on branch-3.2 |  Major | . | Tao Li | Tao Li |
+| [YARN-11092](https://issues.apache.org/jira/browse/YARN-11092) | Upgrade jquery ui to 1.13.1 |  Major | . | D M Murali Krishna Reddy | Ashutosh Gupta |
+| [YARN-11133](https://issues.apache.org/jira/browse/YARN-11133) | YarnClient gets the wrong EffectiveMinCapacity value |  Major | api | Zilong Zhu | Zilong Zhu |
+| [YARN-10850](https://issues.apache.org/jira/browse/YARN-10850) | TimelineService v2 lists containers for all attempts when filtering for one |  Major | timelinereader | Benjamin Teke | Benjamin Teke |
+| [YARN-11126](https://issues.apache.org/jira/browse/YARN-11126) | ZKConfigurationStore Java deserialisation vulnerability |  Major | yarn | Tamas Domok | Tamas Domok |
+| [YARN-11162](https://issues.apache.org/jira/browse/YARN-11162) | Set the zk acl for nodes created by ZKConfigurationStore. |  Major | resourcemanager | Owen O'Malley | Owen O'Malley |
+| [HDFS-16586](https://issues.apache.org/jira/browse/HDFS-16586) | Purge FsDatasetAsyncDiskService threadgroup; it causes BPServiceActor$CommandProcessingThread IllegalThreadStateException 'fatal exception and exit' |  Major | datanode | Michael Stack | Michael Stack |
+| [HADOOP-18251](https://issues.apache.org/jira/browse/HADOOP-18251) | Fix failure of extracting JIRA id from commit message in git\_jira\_fix\_version\_check.py |  Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-16583](https://issues.apache.org/jira/browse/HDFS-16583) | DatanodeAdminDefaultMonitor can get stuck in an infinite loop |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-16623](https://issues.apache.org/jira/browse/HDFS-16623) | IllegalArgumentException in LifelineSender |  Major | . | ZanderXu | ZanderXu |
+| [HDFS-16064](https://issues.apache.org/jira/browse/HDFS-16064) | Determine when to invalidate corrupt replicas based on number of usable replicas |  Major | datanode, namenode | Kevin Wikant | Kevin Wikant |
+| [HADOOP-18100](https://issues.apache.org/jira/browse/HADOOP-18100) | Change scope of inner classes in InodeTree to make them accessible outside package |  Major | . | Abhishek Das | Abhishek Das |
+| [HADOOP-18334](https://issues.apache.org/jira/browse/HADOOP-18334) | Fix create-release to address removal of GPG\_AGENT\_INFO in branch-3.2 |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-7342](https://issues.apache.org/jira/browse/MAPREDUCE-7342) | Stop RMService in TestClientRedirect.testRedirect() |  Minor | . | Zhengxi Li | Zhengxi Li |
+| [MAPREDUCE-7311](https://issues.apache.org/jira/browse/MAPREDUCE-7311) | Fix non-idempotent test in TestTaskProgressReporter |  Minor | . | Zhengxi Li | Zhengxi Li |
+| [HDFS-15862](https://issues.apache.org/jira/browse/HDFS-15862) | Make TestViewfsWithNfs3.testNfsRenameSingleNN() idempotent |  Minor | nfs | Zhengxi Li | Zhengxi Li |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15818](https://issues.apache.org/jira/browse/HDFS-15818) | Fix TestFsDatasetImpl.testReadLockCanBeDisabledByConfig |  Minor | test | Leon Gao | Leon Gao |
+| [YARN-10503](https://issues.apache.org/jira/browse/YARN-10503) | Support queue capacity in terms of absolute resources with custom resourceType. |  Critical | . | Qi Zhu | Qi Zhu |
+| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17929](https://issues.apache.org/jira/browse/HADOOP-17929) | implement non-guava Precondition checkArgument |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17947](https://issues.apache.org/jira/browse/HADOOP-17947) | Provide alternative to Guava VisibleForTesting |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17930](https://issues.apache.org/jira/browse/HADOOP-17930) | implement non-guava Precondition checkState |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17374](https://issues.apache.org/jira/browse/HADOOP-17374) | AliyunOSS: support ListObjectsV2 |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-16336](https://issues.apache.org/jira/browse/HDFS-16336) | De-flake TestRollingUpgrade#testRollback |  Minor | hdfs, test | Kevin Wikant | Viraj Jasani |
+| [HDFS-16171](https://issues.apache.org/jira/browse/HDFS-16171) | De-flake testDecommissionStatus |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16169](https://issues.apache.org/jira/browse/HDFS-16169) | Fix TestBlockTokenWithDFSStriped#testEnd2End failure |  Major | test | Hui Fei | secfree |
+| [HDFS-16484](https://issues.apache.org/jira/browse/HDFS-16484) | [SPS]: Fix an infinite loop bug in SPSPathIdProcessor thread |  Major | . | qinyuren | qinyuren |
+| [HADOOP-16663](https://issues.apache.org/jira/browse/HADOOP-16663) | Backport "HADOOP-16560 [YARN] use protobuf-maven-plugin to generate protobuf classes" to all active branches |  Major | . | Duo Zhang | Duo Zhang |
+| [HADOOP-16664](https://issues.apache.org/jira/browse/HADOOP-16664) | Backport "HADOOP-16561 [MAPREDUCE] use protobuf-maven-plugin to generate protobuf classes" to all active branches |  Major | . | Duo Zhang | Duo Zhang |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-16298](https://issues.apache.org/jira/browse/HDFS-16298) | Improve error msg for BlockMissingException |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16312](https://issues.apache.org/jira/browse/HDFS-16312) | Fix typo for DataNodeVolumeMetrics and ProfilingFileIoEvents |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16326](https://issues.apache.org/jira/browse/HDFS-16326) | Simplify the code for DiskBalancer |  Minor | . | Tao Li | Tao Li |
+| [HDFS-16339](https://issues.apache.org/jira/browse/HDFS-16339) | Show the threshold when mover threads quota is exceeded |  Minor | . | Tao Li | Tao Li |
+| [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | Make GetClusterNodesRequestPBImpl thread safe |  Major | client | Prabhu Joseph | SwathiChandrashekar |
+| [HADOOP-13464](https://issues.apache.org/jira/browse/HADOOP-13464) | update GSON to 2.7+ |  Minor | build | Sean Busbey | Igor Dvorzhak |
+| [HADOOP-18191](https://issues.apache.org/jira/browse/HADOOP-18191) | Log retry count while handling exceptions in RetryInvocationHandler |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16551](https://issues.apache.org/jira/browse/HDFS-16551) | Backport HADOOP-17588 to 3.3 and other active old branches. |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-16618](https://issues.apache.org/jira/browse/HDFS-16618) | sync\_file\_range error should include more volume and file info |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-18300](https://issues.apache.org/jira/browse/HADOOP-18300) | Update Gson to 2.9.0 |  Minor | build | Igor Dvorzhak | Igor Dvorzhak |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md
new file mode 100644
index 0000000000000..fac976d655da1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md
@@ -0,0 +1,55 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.2.4 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | *Major* | **Make GetClusterNodesRequestPBImpl thread safe**
+
+Added syncronization so that the "yarn node list" command does not fail intermittently
+
+
+---
+
+* [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | *Critical* | **Improve RM system metrics publisher's performance by pushing events to timeline server in batch**
+
+When Timeline Service V1 or V1.5 is used, if "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch" is set to true, ResourceManager sends timeline events in batch. The default value is false. If this functionality is enabled, the maximum number that events published in batch is configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size". The default value is 1000. The interval of publishing events can be configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds". By default, it is set to 60 seconds.
+
+
+---
+
+* [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | *Major* | **Replace log4j 1.x with reload4j**
+
+log4j 1 was replaced with reload4j which is fork of log4j 1.2.17 with the goal of fixing pressing security issues.
+
+If you are depending on the hadoop artifacts in your build were explicitly excluding log4 artifacts, and now want to exclude the reload4j files, you will need to update your exclusion lists
+\<exclusion\>
+  \<groupId\>org.slf4j\</groupId\>
+  \<artifactId\>slf4j-reload4j\</artifactId\>
+\</exclusion\>
+\<exclusion\>
+  \<groupId\>ch.qos.reload4j\</groupId\>
+  \<artifactId\>reload4j\</artifactId\>
+\</exclusion\>
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md
new file mode 100644
index 0000000000000..6ac3f1df25862
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md
@@ -0,0 +1,2186 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.3.0 - 2020-07-06
+
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-14339](https://issues.apache.org/jira/browse/HDFS-14339) | Inconsistent log level practices in RpcProgramNfs3.java |  Major | nfs | Anuhan Torgonshar | Anuhan Torgonshar |
+| [HDFS-15186](https://issues.apache.org/jira/browse/HDFS-15186) | Erasure Coding: Decommission may generate the parity block's content with all 0 in some case |  Critical | datanode, erasure-coding | Yao Guangdong | Yao Guangdong |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15885](https://issues.apache.org/jira/browse/HADOOP-15885) | Add base64 (urlString) support to DTUtil |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping |  Major | common, security | Lukas Majercak | Lukas Majercak |
+| [HDFS-14001](https://issues.apache.org/jira/browse/HDFS-14001) | [PROVIDED Storage] bootstrapStandby should manage the InMemoryAliasMap |  Major | . | Íñigo Goiri | Virajith Jalaparti |
+| [YARN-8762](https://issues.apache.org/jira/browse/YARN-8762) | [Umbrella] Support Interactive Docker Shell to running Containers |  Major | . | Zian Chen | Eric Yang |
+| [HADOOP-15996](https://issues.apache.org/jira/browse/HADOOP-15996) | Plugin interface to support more complex usernames in Hadoop |  Major | security | Eric Yang | Bolke de Bruin |
+| [HADOOP-15229](https://issues.apache.org/jira/browse/HADOOP-15229) | Add FileSystem builder-based openFile() API to match createFile(); S3A to implement S3 Select through this API. |  Major | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14118](https://issues.apache.org/jira/browse/HDFS-14118) | Support using DNS to resolve nameservices to IP addresses |  Major | . | Fengnan Li | Fengnan Li |
+| [HADOOP-16125](https://issues.apache.org/jira/browse/HADOOP-16125) | Support multiple bind users in LdapGroupsMapping |  Major | common, security | Lukas Majercak | Lukas Majercak |
+| [YARN-9228](https://issues.apache.org/jira/browse/YARN-9228) | [Umbrella] Docker image life cycle management on HDFS |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9016](https://issues.apache.org/jira/browse/YARN-9016) | DocumentStore as a backend for ATSv2 |  Major | ATSv2 | Sushil Ks | Sushil Ks |
+| [HDFS-14234](https://issues.apache.org/jira/browse/HDFS-14234) | Limit WebHDFS to specifc user, host, directory triples |  Trivial | webhdfs | Clay B. | Clay B. |
+| [HADOOP-16095](https://issues.apache.org/jira/browse/HADOOP-16095) | Support impersonation for AuthenticationFilter |  Major | security | Eric Yang | Eric Yang |
+| [HDFS-12345](https://issues.apache.org/jira/browse/HDFS-12345) | Scale testing HDFS NameNode with real metadata and workloads (Dynamometer) |  Major | namenode, test | Zhe Zhang | Erik Krogen |
+| [YARN-9473](https://issues.apache.org/jira/browse/YARN-9473) | [Umbrella] Support Vector Engine ( a new accelerator hardware) based on pluggable device framework |  Major | nodemanager | Zhankun Tang | Peter Bacsko |
+| [HDFS-13783](https://issues.apache.org/jira/browse/HDFS-13783) | Balancer: make balancer to be a long service process for easy to monitor it. |  Major | balancer & mover | maobaolong | Chen Zhang |
+| [HADOOP-16398](https://issues.apache.org/jira/browse/HADOOP-16398) | Exports Hadoop metrics to Prometheus |  Major | metrics | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16527](https://issues.apache.org/jira/browse/HADOOP-16527) | Add a whitelist of endpoints to skip Kerberos authentication |  Major | security | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-12904](https://issues.apache.org/jira/browse/HDFS-12904) | Add DataTransferThrottler to the Datanode transfers |  Minor | datanode | Íñigo Goiri | Lisheng Sun |
+| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs |  Major | . | Jonathan Hung | pralabhkumar |
+| [YARN-9808](https://issues.apache.org/jira/browse/YARN-9808) | Zero length files in container log output haven't got a header |  Major | log-aggregation, yarn | Adam Antal | Adam Antal |
+| [HADOOP-15691](https://issues.apache.org/jira/browse/HADOOP-15691) | Add PathCapabilities to FS and FC to complement StreamCapabilities |  Major | . | Steve Loughran | Steve Loughran |
+| [HADOOP-15616](https://issues.apache.org/jira/browse/HADOOP-15616) | Incorporate Tencent Cloud COS File System Implementation |  Major | fs/cos | Junping Du | Yang Yu |
+| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level |  Major | . | Jonathan Hung | Varun Saxena |
+| [HDFS-13762](https://issues.apache.org/jira/browse/HDFS-13762) | Support non-volatile storage class memory(SCM) in HDFS cache directives |  Major | caching, datanode | Sammi Chen | Feilong He |
+| [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | Consistent Reads from Standby Node |  Major | hdfs | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-13571](https://issues.apache.org/jira/browse/HDFS-13571) | Deadnode detection |  Major | hdfs-client | Gang Xie | Lisheng Sun |
+| [YARN-9923](https://issues.apache.org/jira/browse/YARN-9923) | Introduce HealthReporter interface to support multiple health checker files |  Major | nodemanager, yarn | Adam Antal | Adam Antal |
+| [YARN-8851](https://issues.apache.org/jira/browse/YARN-8851) | [Umbrella] A pluggable device plugin framework to ease vendor plugin development |  Major | yarn | Zhankun Tang | Zhankun Tang |
+| [YARN-9414](https://issues.apache.org/jira/browse/YARN-9414) | Application Catalog for YARN applications |  Major | . | Eric Yang | Eric Yang |
+| [YARN-5542](https://issues.apache.org/jira/browse/YARN-5542) | Scheduling of opportunistic containers |  Major | . | Konstantinos Karanasos |  |
+| [HDFS-13616](https://issues.apache.org/jira/browse/HDFS-13616) | Batch listing of multiple directories |  Major | . | Andrew Wang | Chao Sun |
+| [HDFS-14743](https://issues.apache.org/jira/browse/HDFS-14743) | Enhance INodeAttributeProvider/ AccessControlEnforcer Interface in HDFS to support Authorization of mkdir, rm, rmdir, copy, move etc... |  Critical | hdfs | Ramesh Mani | Wei-Chiu Chuang |
+| [MAPREDUCE-7237](https://issues.apache.org/jira/browse/MAPREDUCE-7237) | Supports config the shuffle's path cache related parameters |  Major | mrv2 | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16661](https://issues.apache.org/jira/browse/HADOOP-16661) | Support TLS 1.3 |  Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-16912](https://issues.apache.org/jira/browse/HADOOP-16912) | Emit per priority RPC queue time and processing time from DecayRpcScheduler |  Major | common | Fengnan Li | Fengnan Li |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-8226](https://issues.apache.org/jira/browse/YARN-8226) | Improve anti-affinity section description in YARN Service API doc |  Major | docs, documentation | Charan Hebri | Gour Saha |
+| [HADOOP-15356](https://issues.apache.org/jira/browse/HADOOP-15356) | Make HTTP timeout configurable in ADLS Connector |  Major | fs/adl | Atul Sikaria | Atul Sikaria |
+| [HADOOP-15586](https://issues.apache.org/jira/browse/HADOOP-15586) | Fix wrong log statement in AbstractService |  Minor | util | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation |  Major | metrics | Sushil Ks | Sushil Ks |
+| [YARN-8621](https://issues.apache.org/jira/browse/YARN-8621) | Add test coverage of custom Resource Types for the apps/\<appId\> REST API endpoint |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HDFS-13947](https://issues.apache.org/jira/browse/HDFS-13947) | Review of DirectoryScanner Class |  Major | datanode | David Mollitor | David Mollitor |
+| [YARN-8732](https://issues.apache.org/jira/browse/YARN-8732) | Add unit tests of min/max allocation for custom resource types in FairScheduler |  Minor | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics |  Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth |
+| [HDFS-13950](https://issues.apache.org/jira/browse/HDFS-13950) | ACL documentation update to indicate that ACL entries are capped by 32 |  Minor | hdfs | Adam Antal | Adam Antal |
+| [HDFS-13958](https://issues.apache.org/jira/browse/HDFS-13958) | Miscellaneous Improvements for FsVolumeSpi |  Major | datanode | David Mollitor | David Mollitor |
+| [YARN-8644](https://issues.apache.org/jira/browse/YARN-8644) | Improve unit test for RMAppImpl.FinalTransition |  Minor | . | Szilard Nemeth | Szilard Nemeth |
+| [HDFS-13967](https://issues.apache.org/jira/browse/HDFS-13967) | HDFS Router Quota Class Review |  Minor | federation, hdfs | David Mollitor | David Mollitor |
+| [HADOOP-15832](https://issues.apache.org/jira/browse/HADOOP-15832) | Upgrade BouncyCastle to 1.60 |  Major | . | Robert Kanter | Robert Kanter |
+| [HDFS-13882](https://issues.apache.org/jira/browse/HDFS-13882) | Set a maximum delay for retrying locateFollowingBlock |  Major | . | Kitti Nanasi | Kitti Nanasi |
+| [MAPREDUCE-7149](https://issues.apache.org/jira/browse/MAPREDUCE-7149) | javadocs for FileInputFormat and OutputFormat to mention DT collection |  Minor | client | Steve Loughran | Steve Loughran |
+| [HDFS-13968](https://issues.apache.org/jira/browse/HDFS-13968) | BlockReceiver Array-Based Queue |  Minor | datanode | David Mollitor | David Mollitor |
+| [HADOOP-15717](https://issues.apache.org/jira/browse/HADOOP-15717) | TGT renewal thread does not log IOException |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-15831](https://issues.apache.org/jira/browse/HADOOP-15831) | Include modificationTime in the toString method of CopyListingFileStatus |  Minor | . | Ted Yu | Ted Yu |
+| [HDFS-13156](https://issues.apache.org/jira/browse/HDFS-13156) | HDFS Block Placement Policy - Client Local Rack |  Minor | documentation | David Mollitor | Ayush Saxena |
+| [HADOOP-15849](https://issues.apache.org/jira/browse/HADOOP-15849) | Upgrade netty version to 3.10.6 |  Major | . | Xiao Chen | Xiao Chen |
+| [YARN-8836](https://issues.apache.org/jira/browse/YARN-8836) | Add tags and attributes in resource definition |  Major | . | Weiwei Yang | Weiwei Yang |
+| [HDFS-13987](https://issues.apache.org/jira/browse/HDFS-13987) | RBF: Review of RandomResolver Class |  Minor | federation | David Mollitor | David Mollitor |
+| [MAPREDUCE-7150](https://issues.apache.org/jira/browse/MAPREDUCE-7150) | Optimize collections used by MR JHS to reduce its memory |  Major | jobhistoryserver, mrv2 | Misha Dmitriev | Misha Dmitriev |
+| [HADOOP-15854](https://issues.apache.org/jira/browse/HADOOP-15854) | AuthToken Use StringBuilder instead of StringBuffer |  Trivial | auth | David Mollitor | David Mollitor |
+| [HADOOP-11100](https://issues.apache.org/jira/browse/HADOOP-11100) | Support to configure ftpClient.setControlKeepAliveTimeout |  Minor | fs | Krishnamoorthy Dharmalingam | Adam Antal |
+| [YARN-8899](https://issues.apache.org/jira/browse/YARN-8899) | TestCleanupAfterKIll is failing due to unsatisfied dependencies |  Blocker | yarn-native-services | Eric Yang | Robert Kanter |
+| [YARN-8618](https://issues.apache.org/jira/browse/YARN-8618) | Yarn Service: When all the components of a service have restart policy NEVER then initiation of service upgrade should fail |  Major | . | Chandni Singh | Chandni Singh |
+| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 |  Major | . | PJ Fanning | Akira Ajisaka |
+| [YARN-8916](https://issues.apache.org/jira/browse/YARN-8916) | Define a constant "docker" string in "ContainerRuntimeConstants.java" for better maintainability |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8908](https://issues.apache.org/jira/browse/YARN-8908) | Fix errors in yarn-default.xml related to GPU/FPGA |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-13994](https://issues.apache.org/jira/browse/HDFS-13994) | Improve DataNode BlockSender waitForMinLength |  Minor | datanode | David Mollitor | David Mollitor |
+| [HADOOP-15821](https://issues.apache.org/jira/browse/HADOOP-15821) | Move Hadoop YARN Registry to Hadoop Registry |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-8542](https://issues.apache.org/jira/browse/YARN-8542) | Yarn Service: Add component name to container json |  Major | . | Chandni Singh | Chandni Singh |
+| [YARN-8923](https://issues.apache.org/jira/browse/YARN-8923) | Cleanup references to ENV file type in YARN service code |  Minor | yarn-native-services | Suma Shivaprasad | Suma Shivaprasad |
+| [YARN-6586](https://issues.apache.org/jira/browse/YARN-6586) | YARN to facilitate HTTPS in AM web server |  Major | yarn | Haibo Chen | Robert Kanter |
+| [HDFS-13941](https://issues.apache.org/jira/browse/HDFS-13941) | make storageId in BlockPoolTokenSecretManager.checkAccess optional |  Major | . | Ajay Kumar | Ajay Kumar |
+| [HDFS-14026](https://issues.apache.org/jira/browse/HDFS-14026) | Overload BlockPoolTokenSecretManager.checkAccess to make storageId and storageType optional |  Major | . | Ajay Kumar | Ajay Kumar |
+| [HDFS-14029](https://issues.apache.org/jira/browse/HDFS-14029) | Sleep in TestLazyPersistFiles should be put into a loop |  Trivial | hdfs | Adam Antal | Adam Antal |
+| [HADOOP-9567](https://issues.apache.org/jira/browse/HADOOP-9567) | Provide auto-renewal for keytab based logins |  Minor | security | Harsh J | Hrishikesh Gadre |
+| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-14008](https://issues.apache.org/jira/browse/HDFS-14008) | NN should log snapshotdiff report |  Major | namenode | Pranay Singh | Pranay Singh |
+| [HDFS-13996](https://issues.apache.org/jira/browse/HDFS-13996) | Make HttpFS' ACLs RegEx configurable |  Major | httpfs | Siyao Meng | Siyao Meng |
+| [YARN-8954](https://issues.apache.org/jira/browse/YARN-8954) | Reservations list field in ReservationListInfo is not accessible |  Minor | resourcemanager, restapi | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log |  Major | resourcemanager | Jonathan Hung | Eric Payne |
+| [HADOOP-15687](https://issues.apache.org/jira/browse/HADOOP-15687) | Credentials class should allow access to aliases |  Trivial | . | Lars Francke | Lars Francke |
+| [YARN-8969](https://issues.apache.org/jira/browse/YARN-8969) | AbstractYarnScheduler#getNodeTracker should return generic type to avoid type casting |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-14053](https://issues.apache.org/jira/browse/HDFS-14053) | Provide ability for NN to re-replicate based on topology changes. |  Major | . | ellen johansen | Hrishikesh Gadre |
+| [HDFS-14051](https://issues.apache.org/jira/browse/HDFS-14051) | Refactor NameNodeHttpServer#initWebHdfs to specify local keytab |  Major | . | Íñigo Goiri | CR Hota |
+| [YARN-8957](https://issues.apache.org/jira/browse/YARN-8957) | Add Serializable interface to ComponentContainers |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8976](https://issues.apache.org/jira/browse/YARN-8976) | Remove redundant modifiers in interface "ApplicationConstants" |  Trivial | . | Zhankun Tang | Zhankun Tang |
+| [HADOOP-15907](https://issues.apache.org/jira/browse/HADOOP-15907) | Add missing maven modules in BUILDING.txt |  Trivial | . | Wanqiang Ji | Wanqiang Ji |
+| [MAPREDUCE-7148](https://issues.apache.org/jira/browse/MAPREDUCE-7148) | Fast fail jobs when exceeds dfs quota limitation |  Major | task | Wang Yan | Wang Yan |
+| [YARN-8977](https://issues.apache.org/jira/browse/YARN-8977) | Remove unnecessary type casting when calling AbstractYarnScheduler#getSchedulerNode |  Trivial | . | Wanqiang Ji | Wanqiang Ji |
+| [YARN-8997](https://issues.apache.org/jira/browse/YARN-8997) | [Submarine] Small refactors of modifier, condition check and redundant local variables |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-14070](https://issues.apache.org/jira/browse/HDFS-14070) | Refactor NameNodeWebHdfsMethods to allow better extensibility |  Major | . | CR Hota | CR Hota |
+| [HADOOP-15926](https://issues.apache.org/jira/browse/HADOOP-15926) | Document upgrading the section in NOTICE.txt when upgrading the version of AWS SDK |  Minor | documentation | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-14045](https://issues.apache.org/jira/browse/HDFS-14045) | Use different metrics in DataNode to better measure latency of heartbeat/blockReports/incrementalBlockReports of Active/Standby NN |  Major | datanode | Jiandan Yang | Jiandan Yang |
+| [HADOOP-12558](https://issues.apache.org/jira/browse/HADOOP-12558) | distcp documentation is woefully out of date |  Critical | documentation, tools/distcp | Allen Wittenauer | Dinesh Chitlangia |
+| [HDFS-14063](https://issues.apache.org/jira/browse/HDFS-14063) | Support noredirect param for CREATE/APPEND/OPEN/GETFILECHECKSUM in HttpFS |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-8860](https://issues.apache.org/jira/browse/YARN-8860) | Federation client intercepter class contains unwanted character |  Minor | router | Rakesh Shah | Abhishek Modi |
+| [HDFS-14015](https://issues.apache.org/jira/browse/HDFS-14015) | Improve error handling in hdfsThreadDestructor in native thread local storage |  Major | native | Daniel Templeton | Daniel Templeton |
+| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-14739](https://issues.apache.org/jira/browse/HADOOP-14739) | Update start-build-env.sh and build instruction for docker for Mac instead of docker toolbox. |  Minor | build, documentation | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-14064](https://issues.apache.org/jira/browse/HDFS-14064) | WEBHDFS: Support Enable/Disable EC Policy |  Major | erasure-coding, webhdfs | Ayush Saxena | Ayush Saxena |
+| [YARN-8964](https://issues.apache.org/jira/browse/YARN-8964) | [UI2] YARN ui2 should use clusters/{cluster name} for all ATSv2 REST APIs |  Major | . | Rohith Sharma K S | Akhil PB |
+| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14108](https://issues.apache.org/jira/browse/HDFS-14108) | Performance improvement in BlockManager Data Structures |  Minor | hdfs | David Mollitor | David Mollitor |
+| [HDFS-14102](https://issues.apache.org/jira/browse/HDFS-14102) | Performance improvement in BlockPlacementPolicyDefault |  Minor | . | David Mollitor | David Mollitor |
+| [MAPREDUCE-7164](https://issues.apache.org/jira/browse/MAPREDUCE-7164) | FileOutputCommitter does not report progress while merging paths. |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [YARN-8975](https://issues.apache.org/jira/browse/YARN-8975) | [Submarine] Use predefined Charset object StandardCharsets.UTF\_8 instead of String "UTF-8" |  Trivial | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8974](https://issues.apache.org/jira/browse/YARN-8974) | Improve the assertion message in TestGPUResourceHandler |  Trivial | . | Zhankun Tang | Zhankun Tang |
+| [YARN-9061](https://issues.apache.org/jira/browse/YARN-9061) | Improve the GPU/FPGA module log message of container-executor |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [YARN-9069](https://issues.apache.org/jira/browse/YARN-9069) | Fix SchedulerInfo#getSchedulerType for custom schedulers |  Minor | . | Bilwa S T | Bilwa S T |
+| [HDFS-14095](https://issues.apache.org/jira/browse/HDFS-14095) | EC: Track Erasure Coding commands in DFS statistics |  Major | erasure-coding | Ayush Saxena | Ayush Saxena |
+| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI |  Major | . | Jonathan Hung | Keqiu Hu |
+| [HDFS-14106](https://issues.apache.org/jira/browse/HDFS-14106) | Refactor NamenodeFsck#copyBlock |  Minor | namenode | David Mollitor | David Mollitor |
+| [HDFS-12946](https://issues.apache.org/jira/browse/HDFS-12946) | Add a tool to check rack configuration against EC policies |  Major | erasure-coding | Xiao Chen | Kitti Nanasi |
+| [HDFS-13818](https://issues.apache.org/jira/browse/HDFS-13818) | Extend OIV to detect FSImage corruption |  Major | hdfs | Adam Antal | Adam Antal |
+| [HDFS-14119](https://issues.apache.org/jira/browse/HDFS-14119) | Improve GreedyPlanner Parameter Logging |  Trivial | hdfs | David Mollitor | David Mollitor |
+| [HDFS-14105](https://issues.apache.org/jira/browse/HDFS-14105) | Replace TreeSet in NamenodeFsck with HashSet |  Trivial | . | David Mollitor | David Mollitor |
+| [YARN-8985](https://issues.apache.org/jira/browse/YARN-8985) | Improve debug log in FSParentQueue when assigning container |  Minor | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14113](https://issues.apache.org/jira/browse/HDFS-14113) | EC : Add Configuration to restrict UserDefined Policies |  Major | erasure-coding | Ayush Saxena | Ayush Saxena |
+| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14124](https://issues.apache.org/jira/browse/HDFS-14124) | EC : Support EC Commands (set/get/unset EcPolicy) via WebHdfs |  Major | erasure-coding, httpfs, webhdfs | Souryakanta Dwivedy | Ayush Saxena |
+| [YARN-9051](https://issues.apache.org/jira/browse/YARN-9051) | Integrate multiple CustomResourceTypesConfigurationProvider implementations into one |  Minor | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9008](https://issues.apache.org/jira/browse/YARN-9008) | Extend YARN distributed shell with file localization feature |  Major | yarn | Peter Bacsko | Peter Bacsko |
+| [HDFS-13985](https://issues.apache.org/jira/browse/HDFS-13985) | Clearer error message for ReplicaNotFoundException |  Major | hdfs | Adam Antal | Adam Antal |
+| [HDFS-13970](https://issues.apache.org/jira/browse/HDFS-13970) | Use MultiMap for CacheManager Directives to simplify the code |  Minor | caching, hdfs | David Mollitor | David Mollitor |
+| [HDFS-14006](https://issues.apache.org/jira/browse/HDFS-14006) | Refactor name node to allow different token verification implementations |  Major | . | CR Hota | CR Hota |
+| [YARN-9122](https://issues.apache.org/jira/browse/YARN-9122) | Add table of contents to YARN Service API document |  Minor | documentation | Akira Ajisaka | Zhankun Tang |
+| [HADOOP-16000](https://issues.apache.org/jira/browse/HADOOP-16000) | Remove TLSv1 and SSLv2Hello from the default value of hadoop.ssl.enabled.protocols |  Major | security | Akira Ajisaka | Gabor Bota |
+| [YARN-9095](https://issues.apache.org/jira/browse/YARN-9095) | Removed Unused field from Resource: NUM\_MANDATORY\_RESOURCES |  Minor | . | Szilard Nemeth | Vidura Bhathiya Mudalige |
+| [MAPREDUCE-7166](https://issues.apache.org/jira/browse/MAPREDUCE-7166) | map-only job should ignore node lost event when task is already succeeded |  Major | mrv2 | Zhaohui Xin | Lei Li |
+| [YARN-9130](https://issues.apache.org/jira/browse/YARN-9130) | Add Bind\_HOST configuration for Yarn Web Proxy |  Major | yarn | Rong Tang | Rong Tang |
+| [HADOOP-15965](https://issues.apache.org/jira/browse/HADOOP-15965) | Upgrade to ADLS SDK which has major performance improvement for ingress/egress |  Major | fs/adl | Vishwajeet Dusane | Vishwajeet Dusane |
+| [HADOOP-16014](https://issues.apache.org/jira/browse/HADOOP-16014) | Fix test, checkstyle and javadoc issues in TestKerberosAuthenticationHandler |  Major | test | Dinesh Chitlangia | Dinesh Chitlangia |
+| [HDFS-13946](https://issues.apache.org/jira/browse/HDFS-13946) | Log longest FSN write/read lock held stack trace |  Minor | . | Yiqun Lin | Yiqun Lin |
+| [HADOOP-15962](https://issues.apache.org/jira/browse/HADOOP-15962) | The buffer size is small when unpacking tar archives |  Minor | common, util | David Mollitor | David Mollitor |
+| [YARN-8878](https://issues.apache.org/jira/browse/YARN-8878) | Remove StringBuffer from ManagedParentQueue.java |  Trivial | resourcemanager | David Mollitor | David Mollitor |
+| [YARN-8894](https://issues.apache.org/jira/browse/YARN-8894) | Improve InMemoryPlan#toString |  Minor | reservation system | David Mollitor | David Mollitor |
+| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog |  Major | namenode | Kenneth Yang | Kenneth Yang |
+| [HDFS-14184](https://issues.apache.org/jira/browse/HDFS-14184) | [SPS] Add support for URI based path in satisfystoragepolicy command |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14163](https://issues.apache.org/jira/browse/HDFS-14163) | Debug Admin Command Should Support Generic Options. |  Major | . | Harshakiran Reddy | Ayush Saxena |
+| [YARN-6523](https://issues.apache.org/jira/browse/YARN-6523) | Optimize system credentials sent in node heartbeat responses |  Major | RM | Naganarasimha G R | Manikandan R |
+| [HADOOP-15909](https://issues.apache.org/jira/browse/HADOOP-15909) | KeyProvider class should implement Closeable |  Major | kms | Kuhu Shukla | Kuhu Shukla |
+| [HADOOP-16029](https://issues.apache.org/jira/browse/HADOOP-16029) | Consecutive StringBuilder.append can be reused |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics |  Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian |
+| [HADOOP-15994](https://issues.apache.org/jira/browse/HADOOP-15994) | Upgrade Jackson2 to 2.9.8 |  Major | security | Akira Ajisaka | lqjacklee |
+| [HDFS-14213](https://issues.apache.org/jira/browse/HDFS-14213) | Remove Jansson from BUILDING.txt |  Minor | documentation | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-14221](https://issues.apache.org/jira/browse/HDFS-14221) | Replace Guava Optional with Java Optional |  Major | . | Arpit Agarwal | Arpit Agarwal |
+| [HDFS-14222](https://issues.apache.org/jira/browse/HDFS-14222) | Make ThrottledAsyncChecker constructor public |  Major | . | Arpit Agarwal | Arpit Agarwal |
+| [HDFS-14153](https://issues.apache.org/jira/browse/HDFS-14153) | [SPS] : Add Support for Storage Policy Satisfier in WEBHDFS |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14061](https://issues.apache.org/jira/browse/HDFS-14061) | Check if the cluster topology supports the EC policy before setting, enabling or adding it |  Major | erasure-coding, hdfs | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-14185](https://issues.apache.org/jira/browse/HDFS-14185) | Cleanup method calls to static Assert methods in TestAddStripedBlocks |  Minor | hdfs | Shweta | Shweta |
+| [HADOOP-16075](https://issues.apache.org/jira/browse/HADOOP-16075) | Upgrade checkstyle version to 8.16 |  Minor | build | Dinesh Chitlangia | Dinesh Chitlangia |
+| [HDFS-14187](https://issues.apache.org/jira/browse/HDFS-14187) | Make warning message more clear when there are not enough data nodes for EC write |  Major | erasure-coding | Kitti Nanasi | Kitti Nanasi |
+| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14125](https://issues.apache.org/jira/browse/HDFS-14125) | Use parameterized log format in ECTopologyVerifier |  Trivial | erasure-coding | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-14231](https://issues.apache.org/jira/browse/HDFS-14231) | DataXceiver#run() should not log exceptions caused by InvalidToken exception as an error |  Major | hdfs | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-14250](https://issues.apache.org/jira/browse/HDFS-14250) | [SBN read] msync should sync with active NameNode to fetch the latest stateID |  Major | namenode | Chao Sun | Chao Sun |
+| [YARN-8219](https://issues.apache.org/jira/browse/YARN-8219) | Add application launch time to ATSV2 |  Major | timelineserver | Kanwaljeet Sachdev | Abhishek Modi |
+| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically |  Major | . | Eric Maynard | Ahmed Hussein |
+| [HDFS-14172](https://issues.apache.org/jira/browse/HDFS-14172) | Avoid NPE when SectionName#fromString returns null |  Minor | . | Xiang Li | Xiang Li |
+| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' |  Trivial | . | Szilard Nemeth | Charan Hebri |
+| [HDFS-14260](https://issues.apache.org/jira/browse/HDFS-14260) | Replace synchronized method in BlockReceiver with atomic value |  Minor | datanode | David Mollitor | David Mollitor |
+| [HADOOP-16097](https://issues.apache.org/jira/browse/HADOOP-16097) | Provide proper documentation for FairCallQueue |  Major | documentation, ipc | Erik Krogen | Erik Krogen |
+| [HADOOP-16108](https://issues.apache.org/jira/browse/HADOOP-16108) | Tail Follow Interval Should Allow To Specify The Sleep Interval To Save Unnecessary RPC's |  Major | . | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14241](https://issues.apache.org/jira/browse/HDFS-14241) | Provide feedback on successful renameSnapshot and deleteSnapshot |  Minor | hdfs, shell | Siyao Meng | Siyao Meng |
+| [HDFS-13209](https://issues.apache.org/jira/browse/HDFS-13209) | DistributedFileSystem.create should allow an option to provide StoragePolicy |  Major | hdfs | Jean-Marc Spaggiari | Ayush Saxena |
+| [YARN-8295](https://issues.apache.org/jira/browse/YARN-8295) | [UI2] Improve "Resource Usage" tab error message when there are no data available. |  Minor | yarn-ui-v2 | Gergely Novák | Charan Hebri |
+| [YARN-8927](https://issues.apache.org/jira/browse/YARN-8927) | Support trust top-level image like "centos" when "library" is configured in "docker.trusted.registries" |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-14258](https://issues.apache.org/jira/browse/HDFS-14258) | Introduce Java Concurrent Package To DataXceiverServer Class |  Minor | datanode | David Mollitor | David Mollitor |
+| [YARN-7824](https://issues.apache.org/jira/browse/YARN-7824) | [UI2] Yarn Component Instance page should include link to container logs |  Major | yarn-ui-v2 | Yesha Vora | Akhil PB |
+| [HADOOP-15281](https://issues.apache.org/jira/browse/HADOOP-15281) | Distcp to add no-rename copy option |  Major | tools/distcp | Steve Loughran | Andrew Olson |
+| [HDFS-9596](https://issues.apache.org/jira/browse/HDFS-9596) | Remove Shuffle Method From DFSUtil |  Trivial | . | David Mollitor | David Mollitor |
+| [HDFS-14296](https://issues.apache.org/jira/browse/HDFS-14296) | Prefer ArrayList over LinkedList in VolumeScanner |  Minor | datanode | David Mollitor | David Mollitor |
+| [YARN-9309](https://issues.apache.org/jira/browse/YARN-9309) | Improve graph text in SLS to avoid overlapping |  Minor | . | Bilwa S T | Bilwa S T |
+| [HDFS-14188](https://issues.apache.org/jira/browse/HDFS-14188) | Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter |  Major | erasure-coding | Kitti Nanasi | Kitti Nanasi |
+| [HADOOP-15967](https://issues.apache.org/jira/browse/HADOOP-15967) | KMS Benchmark Tool |  Major | . | Wei-Chiu Chuang | George Huang |
+| [HDFS-14286](https://issues.apache.org/jira/browse/HDFS-14286) | Logging stale datanode information |  Trivial | hdfs | Karthik Palanisamy | Karthik Palanisamy |
+| [HDFS-14235](https://issues.apache.org/jira/browse/HDFS-14235) | Handle ArrayIndexOutOfBoundsException in DataNodeDiskMetrics#slowDiskDetectionDaemon |  Major | . | Surendra Singh Lilhore | Ranith Sardar |
+| [HDFS-14267](https://issues.apache.org/jira/browse/HDFS-14267) | Add test\_libhdfs\_ops to libhdfs tests, mark libhdfs\_read/write.c as examples |  Major | libhdfs, native, test | Sahil Takiar | Sahil Takiar |
+| [HDFS-14302](https://issues.apache.org/jira/browse/HDFS-14302) | Refactor NameNodeWebHdfsMethods#generateDelegationToken() to allow better extensibility |  Major | . | CR Hota | CR Hota |
+| [HADOOP-16035](https://issues.apache.org/jira/browse/HADOOP-16035) | Jenkinsfile for Hadoop |  Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-14298](https://issues.apache.org/jira/browse/HDFS-14298) | Improve log messages of ECTopologyVerifier |  Minor | . | Kitti Nanasi | Kitti Nanasi |
+| [YARN-9168](https://issues.apache.org/jira/browse/YARN-9168) | DistributedShell client timeout should be -1 by default |  Minor | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-7133](https://issues.apache.org/jira/browse/HDFS-7133) | Support clearing namespace quota on "/" |  Major | namenode | Guo Ruijing | Ayush Saxena |
+| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
+| [YARN-9287](https://issues.apache.org/jira/browse/YARN-9287) | Consecutive StringBuilder append should be reused |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9087](https://issues.apache.org/jira/browse/YARN-9087) | Improve logging for initialization of Resource plugins |  Major | yarn | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9121](https://issues.apache.org/jira/browse/YARN-9121) | Replace GpuDiscoverer.getInstance() to a readable object for easy access control |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9322](https://issues.apache.org/jira/browse/YARN-9322) | Store metrics for custom resource types into FSQueueMetrics and query them in FairSchedulerQueueInfo |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9139](https://issues.apache.org/jira/browse/YARN-9139) | Simplify initializer code of GpuDiscoverer |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology |  Minor | datanode | HuangTao | HuangTao |
+| [YARN-9332](https://issues.apache.org/jira/browse/YARN-9332) | RackResolver tool should accept multiple hosts |  Minor | yarn | Lantao Jin | Lantao Jin |
+| [HDFS-14182](https://issues.apache.org/jira/browse/HDFS-14182) | Datanode usage histogram is clicked to show ip list |  Major | . | fengchuang | fengchuang |
+| [HDFS-14321](https://issues.apache.org/jira/browse/HDFS-14321) | Fix -Xcheck:jni issues in libhdfs, run ctest with -Xcheck:jni enabled |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [HADOOP-16148](https://issues.apache.org/jira/browse/HADOOP-16148) | Cleanup LineReader Unit Test |  Trivial | common | David Mollitor | David Mollitor |
+| [HADOOP-16162](https://issues.apache.org/jira/browse/HADOOP-16162) | Remove unused Job Summary Appender configurations from log4j.properties |  Major | conf | Chen Zhi | Chen Zhi |
+| [HDFS-14336](https://issues.apache.org/jira/browse/HDFS-14336) | Fix checkstyle for NameNodeMXBean |  Trivial | namenode | Danny Becker | Danny Becker |
+| [YARN-9298](https://issues.apache.org/jira/browse/YARN-9298) | Implement FS placement rules using PlacementRule interface |  Major | scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14326](https://issues.apache.org/jira/browse/HDFS-14326) | Add CorruptFilesCount to JMX |  Minor | fs, metrics, namenode | Danny Becker | Danny Becker |
+| [YARN-9138](https://issues.apache.org/jira/browse/YARN-9138) | Improve test coverage for nvidia-smi binary execution of GpuDiscoverer |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-8218](https://issues.apache.org/jira/browse/YARN-8218) | Add application launch time to ATSV1 |  Major | . | Kanwaljeet Sachdev | Abhishek Modi |
+| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 |  Major | ATSv2 | Sushil Ks | Sushil Ks |
+| [HADOOP-16157](https://issues.apache.org/jira/browse/HADOOP-16157) | [Clean-up] Remove NULL check before instanceof in AzureNativeFileSystemStore |  Minor | tools | Shweta | Shweta |
+| [MAPREDUCE-7191](https://issues.apache.org/jira/browse/MAPREDUCE-7191) | JobHistoryServer should log exception when loading/parsing history file failed |  Minor | mrv2 | Jiandan Yang | Jiandan Yang |
+| [YARN-9381](https://issues.apache.org/jira/browse/YARN-9381) | The yarn-default.xml has two identical property named yarn.timeline-service.http-cross-origin.enabled |  Trivial | yarn | jenny | Abhishek Modi |
+| [MAPREDUCE-7192](https://issues.apache.org/jira/browse/MAPREDUCE-7192) | JobHistoryServer attempts page support jump to  containers log page in NM when logAggregation is disable |  Major | mrv2 | Jiandan Yang | Jiandan Yang |
+| [HDFS-14346](https://issues.apache.org/jira/browse/HDFS-14346) | Better time precision in getTimeDuration |  Minor | namenode | Chao Sun | Chao Sun |
+| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance |  Major | hdfs | Chao Sun | Chao Sun |
+| [YARN-4404](https://issues.apache.org/jira/browse/YARN-4404) | Typo in comment in SchedulerUtils |  Trivial | resourcemanager | Daniel Templeton | Yesha Vora |
+| [MAPREDUCE-7188](https://issues.apache.org/jira/browse/MAPREDUCE-7188) | [Clean-up] Remove NULL check before instanceof and fix checkstyle issue in TaskResult |  Minor | . | Shweta | Shweta |
+| [YARN-9340](https://issues.apache.org/jira/browse/YARN-9340) | [Clean-up] Remove NULL check before instanceof in ResourceRequestSetKey |  Minor | yarn | Shweta | Shweta |
+| [HDFS-14328](https://issues.apache.org/jira/browse/HDFS-14328) | [Clean-up] Remove NULL check before instanceof in TestGSet |  Minor | . | Shweta | Shweta |
+| [HADOOP-16167](https://issues.apache.org/jira/browse/HADOOP-16167) | "hadoop CLASSFILE" prints error messages on Ubuntu 18 |  Major | scripts | Daniel Templeton | Daniel Templeton |
+| [YARN-9385](https://issues.apache.org/jira/browse/YARN-9385) | YARN Services with simple authentication doesn't respect current UGI |  Major | security, yarn-native-services | Todd Lipcon | Eric Yang |
+| [HDFS-14211](https://issues.apache.org/jira/browse/HDFS-14211) | [Consistent Observer Reads] Allow for configurable "always msync" mode |  Major | hdfs-client | Erik Krogen | Erik Krogen |
+| [YARN-9370](https://issues.apache.org/jira/browse/YARN-9370) | Better logging in recoverAssignedGpus in class GpuResourceAllocator |  Trivial | . | Szilard Nemeth | Yesha Vora |
+| [HADOOP-16147](https://issues.apache.org/jira/browse/HADOOP-16147) | Allow CopyListing sequence file keys and values to be more easily customized |  Major | tools/distcp | Andrew Olson | Andrew Olson |
+| [YARN-9358](https://issues.apache.org/jira/browse/YARN-9358) | Add javadoc to new methods introduced in FSQueueMetrics with YARN-9322 |  Major | . | Szilard Nemeth | Zoltan Siegl |
+| [YARN-8967](https://issues.apache.org/jira/browse/YARN-8967) | Change FairScheduler to use PlacementRule interface |  Major | capacityscheduler, fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14304](https://issues.apache.org/jira/browse/HDFS-14304) | High lock contention on hdfsHashMutex in libhdfs |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [HDFS-14295](https://issues.apache.org/jira/browse/HDFS-14295) | Add Threadpool for DataTransfers |  Major | datanode | David Mollitor | David Mollitor |
+| [HDFS-14395](https://issues.apache.org/jira/browse/HDFS-14395) | Remove WARN Logging From Interrupts |  Minor | hdfs-client | David Mollitor | David Mollitor |
+| [YARN-9264](https://issues.apache.org/jira/browse/YARN-9264) | [Umbrella] Follow-up on IntelOpenCL FPGA plugin |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [MAPREDUCE-7190](https://issues.apache.org/jira/browse/MAPREDUCE-7190) | Add SleepJob additional parameter to make parallel runs distinguishable |  Major | . | Adam Antal | Adam Antal |
+| [YARN-9214](https://issues.apache.org/jira/browse/YARN-9214) | Add AbstractYarnScheduler#getValidQueues method to remove duplication |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-13960](https://issues.apache.org/jira/browse/HDFS-13960) | hdfs dfs -checksum command should optionally show block size in output |  Minor | hdfs | Adam Antal | Lokesh Jain |
+| [HDFS-14327](https://issues.apache.org/jira/browse/HDFS-14327) | Using FQDN instead of IP to access servers with DNS resolving |  Major | . | Fengnan Li | Fengnan Li |
+| [YARN-9394](https://issues.apache.org/jira/browse/YARN-9394) | Use new API of RackResolver to get better performance |  Major | yarn | Lantao Jin | Lantao Jin |
+| [HADOOP-16208](https://issues.apache.org/jira/browse/HADOOP-16208) | Do Not Log InterruptedException in Client |  Minor | common | David Mollitor | David Mollitor |
+| [HDFS-14371](https://issues.apache.org/jira/browse/HDFS-14371) | Improve Logging in FSNamesystem by adding parameterized logging |  Minor | hdfs | Shweta | Shweta |
+| [HADOOP-10848](https://issues.apache.org/jira/browse/HADOOP-10848) | Cleanup calling of sun.security.krb5.Config |  Minor | . | Kai Zheng | Akira Ajisaka |
+| [YARN-9463](https://issues.apache.org/jira/browse/YARN-9463) | Add queueName info when failing with queue capacity sanity check |  Trivial | capacity scheduler | Aihua Xu | Aihua Xu |
+| [HADOOP-16179](https://issues.apache.org/jira/browse/HADOOP-16179) | hadoop-common pom should not depend on kerb-simplekdc |  Major | common | Todd Lipcon | Todd Lipcon |
+| [HADOOP-16052](https://issues.apache.org/jira/browse/HADOOP-16052) | Remove Subversion and Forrest from Dockerfile |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HADOOP-16243](https://issues.apache.org/jira/browse/HADOOP-16243) | Change Log Level to trace in NetUtils.java |  Major | . | Bharat Viswanadham | chencan |
+| [HADOOP-16227](https://issues.apache.org/jira/browse/HADOOP-16227) | Upgrade checkstyle to 8.19 |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16249](https://issues.apache.org/jira/browse/HADOOP-16249) | Make CallerContext LimitedPrivate scope to Public |  Minor | ipc | Kenneth Yang | Kenneth Yang |
+| [HADOOP-15014](https://issues.apache.org/jira/browse/HADOOP-15014) | KMS should log the IP address of the clients |  Major | kms | Zsombor Gegesy | Zsombor Gegesy |
+| [YARN-9123](https://issues.apache.org/jira/browse/YARN-9123) | Clean up and split testcases in TestNMWebServices for GPU support |  Minor | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9349](https://issues.apache.org/jira/browse/YARN-9349) | When doTransition() method occurs exception, the log level practices are inconsistent |  Major | nodemanager | Anuhan Torgonshar |  |
+| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml |  Minor | hdfs | puleya7 | puleya7 |
+| [HDFS-14374](https://issues.apache.org/jira/browse/HDFS-14374) | Expose total number of delegation tokens in AbstractDelegationTokenSecretManager |  Major | . | CR Hota | CR Hota |
+| [HADOOP-16026](https://issues.apache.org/jira/browse/HADOOP-16026) | Replace incorrect use of system property user.name |  Major | . | Dinesh Chitlangia | Dinesh Chitlangia |
+| [YARN-9081](https://issues.apache.org/jira/browse/YARN-9081) | Update jackson from 1.9.13 to 2.x in hadoop-yarn-services-core |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-3246](https://issues.apache.org/jira/browse/HDFS-3246) | pRead equivalent for direct read path |  Major | hdfs-client, performance | Henry Robinson | Sahil Takiar |
+| [HDFS-14463](https://issues.apache.org/jira/browse/HDFS-14463) | Add Log Level link under NameNode and DataNode Web UI Utilities dropdown |  Trivial | webhdfs | Siyao Meng | Siyao Meng |
+| [HDFS-14460](https://issues.apache.org/jira/browse/HDFS-14460) | DFSUtil#getNamenodeWebAddr should return HTTPS address based on policy configured |  Major | . | CR Hota | CR Hota |
+| [HADOOP-16282](https://issues.apache.org/jira/browse/HADOOP-16282) | Avoid FileStream to improve performance |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14453](https://issues.apache.org/jira/browse/HDFS-14453) | Improve Bad Sequence Number Error Message |  Minor | ipc | David Mollitor | Shweta |
+| [HADOOP-16059](https://issues.apache.org/jira/browse/HADOOP-16059) | Use SASL Factories Cache to Improve Performance |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16292](https://issues.apache.org/jira/browse/HADOOP-16292) | Refactor checkTrustAndSend in SaslDataTransferClient to make it cleaner |  Major | . | Sherwood Zheng | Sherwood Zheng |
+| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. |  Major | nodemanager | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16289](https://issues.apache.org/jira/browse/HADOOP-16289) | Allow extra jsvc startup option in hadoop\_start\_secure\_daemon in hadoop-functions.sh |  Major | scripts | Siyao Meng | Siyao Meng |
+| [HADOOP-16238](https://issues.apache.org/jira/browse/HADOOP-16238) | Add the possbility to set SO\_REUSEADDR in IPC Server Listener |  Minor | ipc | Peter Bacsko | Peter Bacsko |
+| [YARN-9453](https://issues.apache.org/jira/browse/YARN-9453) | Clean up code long if-else chain in ApplicationCLI#run |  Major | . | Szilard Nemeth | Wanqiang Ji |
+| [YARN-9546](https://issues.apache.org/jira/browse/YARN-9546) | Add configuration option for YARN Native services AM classpath |  Major | . | Gergely Pollak | Gergely Pollak |
+| [HDFS-14507](https://issues.apache.org/jira/browse/HDFS-14507) | Document -blockingDecommission option for hdfs dfsadmin -listOpenFiles |  Minor | documentation | Siyao Meng | Siyao Meng |
+| [YARN-9145](https://issues.apache.org/jira/browse/YARN-9145) | [Umbrella] Dynamically add or remove auxiliary services |  Major | nodemanager | Billie Rinaldi | Billie Rinaldi |
+| [HDFS-14451](https://issues.apache.org/jira/browse/HDFS-14451) | Incorrect header or version mismatch log message |  Minor | ipc | David Mollitor | Shweta |
+| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() |  Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings |  Minor | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-13654](https://issues.apache.org/jira/browse/HDFS-13654) | Use a random secret when a secret file doesn't  exist in HttpFS. This should be default. |  Major | httpfs, security | Pulkit Bhardwaj | Takanobu Asanuma |
+| [YARN-9592](https://issues.apache.org/jira/browse/YARN-9592) | Use Logger format in ContainersMonitorImpl |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9545](https://issues.apache.org/jira/browse/YARN-9545) | Create healthcheck REST endpoint for ATSv2 |  Major | ATSv2 | Zoltan Siegl | Zoltan Siegl |
+| [HADOOP-16344](https://issues.apache.org/jira/browse/HADOOP-16344) | Make DurationInfo  "public unstable" |  Minor | util | Kevin Risden | Kevin Su |
+| [YARN-9471](https://issues.apache.org/jira/browse/YARN-9471) | Cleanup in TestLogAggregationIndexFileController |  Major | log-aggregation, yarn | Adam Antal | Adam Antal |
+| [HDFS-10659](https://issues.apache.org/jira/browse/HDFS-10659) | Namenode crashes after Journalnode re-installation in an HA cluster due to missing paxos directory |  Major | ha, journal-node | Amit Anand | star |
+| [HDFS-10210](https://issues.apache.org/jira/browse/HDFS-10210) | Remove the defunct startKdc profile from hdfs |  Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-9569](https://issues.apache.org/jira/browse/YARN-9569) | Auto-created leaf queues do not honor cluster-wide min/max memory/vcores |  Major | capacity scheduler | Craig Condit | Craig Condit |
+| [YARN-9602](https://issues.apache.org/jira/browse/YARN-9602) | Use logger format in Container Executor. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [YARN-9543](https://issues.apache.org/jira/browse/YARN-9543) | [UI2] Handle ATSv2 server down or failures cases gracefully in YARN UI v2 |  Major | ATSv2, yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl |
+| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts |  Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles |
+| [HDFS-14560](https://issues.apache.org/jira/browse/HDFS-14560) | Allow block replication parameters to be refreshable |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14203](https://issues.apache.org/jira/browse/HDFS-14203) | Refactor OIV Delimited output entry building mechanism |  Minor | tools | Adam Antal | Adam Antal |
+| [HADOOP-14807](https://issues.apache.org/jira/browse/HADOOP-14807) | should prevent the possibility of  NPE about ReconfigurableBase.java |  Minor | . | hu xiaodong | hu xiaodong |
+| [HDFS-12770](https://issues.apache.org/jira/browse/HDFS-12770) | Add doc about how to disable client socket cache |  Trivial | hdfs-client | Weiwei Yang | Weiwei Yang |
+| [HADOOP-9157](https://issues.apache.org/jira/browse/HADOOP-9157) | Better option for curl in hadoop-auth-examples |  Minor | documentation | Jingguo Yao | Andras Bokor |
+| [HDFS-14340](https://issues.apache.org/jira/browse/HDFS-14340) | Lower the log level when can't get postOpAttr |  Minor | nfs | Anuhan Torgonshar | Anuhan Torgonshar |
+| [HADOOP-15914](https://issues.apache.org/jira/browse/HADOOP-15914) | hadoop jar command has no help argument |  Major | common | Adam Antal | Adam Antal |
+| [YARN-9630](https://issues.apache.org/jira/browse/YARN-9630) | [UI2] Add a link in docs's top page |  Major | documentation, yarn-ui-v2 | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16156](https://issues.apache.org/jira/browse/HADOOP-16156) | [Clean-up] Remove NULL check before instanceof and fix checkstyle in InnerNodeImpl |  Minor | . | Shweta | Shweta |
+| [HDFS-14201](https://issues.apache.org/jira/browse/HDFS-14201) | Ability to disallow safemode NN to become active |  Major | auto-failover | Xiao Liang | Xiao Liang |
+| [HDFS-14487](https://issues.apache.org/jira/browse/HDFS-14487) | Missing Space in Client Error Message |  Minor | hdfs-client | David Mollitor | Shweta |
+| [HDFS-14398](https://issues.apache.org/jira/browse/HDFS-14398) | Update HAState.java to fix typos. |  Trivial | namenode | bianqi | Nikhil Navadiya |
+| [HDFS-14103](https://issues.apache.org/jira/browse/HDFS-14103) | Review Logging of BlockPlacementPolicyDefault |  Minor | . | David Mollitor | David Mollitor |
+| [YARN-9631](https://issues.apache.org/jira/browse/YARN-9631) | hadoop-yarn-applications-catalog-webapp doesn't respect mvn test -D parameter |  Major | . | Wei-Chiu Chuang | Eric Yang |
+| [HADOOP-14385](https://issues.apache.org/jira/browse/HADOOP-14385) | HttpExceptionUtils#validateResponse swallows exceptions |  Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-12564](https://issues.apache.org/jira/browse/HDFS-12564) | Add the documents of swebhdfs configurations on the client side |  Major | documentation, webhdfs | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14403](https://issues.apache.org/jira/browse/HDFS-14403) | Cost-Based RPC FairCallQueue |  Major | ipc, namenode | Erik Krogen | Christopher Gregorian |
+| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer |  Minor | ipc | Christopher Gregorian | Erik Krogen |
+| [HADOOP-16350](https://issues.apache.org/jira/browse/HADOOP-16350) | Ability to tell HDFS client not to request KMS Information from NameNode |  Major | common, kms | Greg Senia |  |
+| [HADOOP-16396](https://issues.apache.org/jira/browse/HADOOP-16396) | Allow authoritative mode on a subdirectory |  Major | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [YARN-9629](https://issues.apache.org/jira/browse/YARN-9629) | Support configurable MIN\_LOG\_ROLLING\_INTERVAL |  Minor | log-aggregation, nodemanager, yarn | Adam Antal | Adam Antal |
+| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading |  Major | . | zhouyingchao | Lisheng Sun |
+| [HADOOP-16409](https://issues.apache.org/jira/browse/HADOOP-16409) | Allow authoritative mode on non-qualified paths |  Major | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14624](https://issues.apache.org/jira/browse/HDFS-14624) | When decommissioning a node, log remaining blocks to replicate periodically |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-9573](https://issues.apache.org/jira/browse/YARN-9573) | DistributedShell cannot specify LogAggregationContext |  Major | distributed-shell, log-aggregation, yarn | Adam Antal | Adam Antal |
+| [YARN-9337](https://issues.apache.org/jira/browse/YARN-9337) | GPU auto-discovery script runs even when the resource is given by hand |  Major | yarn | Adam Antal | Adam Antal |
+| [YARN-9360](https://issues.apache.org/jira/browse/YARN-9360) | Do not expose innards of QueueMetrics object into FSLeafQueue#computeMaxAMResource |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9127](https://issues.apache.org/jira/browse/YARN-9127) | Create more tests to verify GpuDeviceInformationParser |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9326](https://issues.apache.org/jira/browse/YARN-9326) | Fair Scheduler configuration defaults are not documented in case of min and maxResources |  Major | docs, documentation, fairscheduler, yarn | Adam Antal | Adam Antal |
+| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. |  Major | . | Jinglun | Jinglun |
+| [HDFS-13693](https://issues.apache.org/jira/browse/HDFS-13693) | Remove unnecessary search in INodeDirectory.addChild during image loading |  Major | namenode | zhouyingchao | Lisheng Sun |
+| [HADOOP-16431](https://issues.apache.org/jira/browse/HADOOP-16431) | Remove useless log in IOUtils.java and ExceptionDiags.java |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14673](https://issues.apache.org/jira/browse/HDFS-14673) | The console log is noisy when using DNSDomainNameResolver to resolve NameNode. |  Minor | . | Akira Ajisaka | Kevin Su |
+| [HDFS-12967](https://issues.apache.org/jira/browse/HDFS-12967) | NNBench should support multi-cluster access |  Major | benchmarks | Chen Zhang | Chen Zhang |
+| [HADOOP-16452](https://issues.apache.org/jira/browse/HADOOP-16452) | Increase ipc.maximum.data.length default from 64MB to 128MB |  Major | ipc | Wei-Chiu Chuang | Siyao Meng |
+| [HDFS-14449](https://issues.apache.org/jira/browse/HDFS-14449) | Expose total number of DT in JMX for Namenode |  Major | . | Fengnan Li | Fengnan Li |
+| [HDFS-14419](https://issues.apache.org/jira/browse/HDFS-14419) | Avoid repeated calls to the listOpenFiles function |  Minor | namenode, performance | HuangTao | HuangTao |
+| [HDFS-14683](https://issues.apache.org/jira/browse/HDFS-14683) | WebHDFS: Add erasureCodingPolicy field to GETCONTENTSUMMARY response |  Major | . | Siyao Meng | Siyao Meng |
+| [YARN-9375](https://issues.apache.org/jira/browse/YARN-9375) | Use Configured in GpuDiscoverer and FpgaDiscoverer |  Major | nodemanager, yarn | Adam Antal | Adam Antal |
+| [YARN-9093](https://issues.apache.org/jira/browse/YARN-9093) | Remove commented code block from the beginning of TestDefaultContainerExecutor |  Trivial | . | Szilard Nemeth | Vidura Bhathiya Mudalige |
+| [HADOOP-15942](https://issues.apache.org/jira/browse/HADOOP-15942) | Change the logging level form DEBUG to ERROR for RuntimeErrorException in JMXJsonServlet |  Major | common | Anuhan Torgonshar | Anuhan Torgonshar |
+| [YARN-9667](https://issues.apache.org/jira/browse/YARN-9667) | Container-executor.c duplicates messages to stdout |  Major | nodemanager, yarn | Adam Antal | Peter Bacsko |
+| [YARN-9678](https://issues.apache.org/jira/browse/YARN-9678) | TestGpuResourceHandler / TestFpgaResourceHandler should be renamed |  Major | . | Szilard Nemeth | Kevin Su |
+| [HDFS-14652](https://issues.apache.org/jira/browse/HDFS-14652) | HealthMonitor connection retry times should be configurable |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory  instead of df/du |  Major | datanode, performance | Lisheng Sun | Lisheng Sun |
+| [HDFS-14608](https://issues.apache.org/jira/browse/HDFS-14608) | DataNode#DataTransfer should be named |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14616](https://issues.apache.org/jira/browse/HDFS-14616) | Add the warn log when the volume available space isn't enough |  Minor | hdfs | liying | liying |
+| [YARN-9711](https://issues.apache.org/jira/browse/YARN-9711) | Missing spaces in NMClientImpl |  Trivial | client | Charles Xu | Charles Xu |
+| [HDFS-14662](https://issues.apache.org/jira/browse/HDFS-14662) | Document the usage of the new Balancer "asService" parameter |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-14701](https://issues.apache.org/jira/browse/HDFS-14701) | Change Log Level to warn in SlotReleaser |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14705](https://issues.apache.org/jira/browse/HDFS-14705) | Remove unused configuration dfs.min.replication |  Trivial | . | Wei-Chiu Chuang | CR Hota |
+| [HDFS-14693](https://issues.apache.org/jira/browse/HDFS-14693) | NameNode should log a warning when EditLog IPC logger's pending size exceeds limit. |  Minor | namenode | Xudong Cao | Xudong Cao |
+| [YARN-9094](https://issues.apache.org/jira/browse/YARN-9094) | Remove unused interface method: NodeResourceUpdaterPlugin#handleUpdatedResourceFromRM |  Trivial | . | Szilard Nemeth | Gergely Pollak |
+| [YARN-9096](https://issues.apache.org/jira/browse/YARN-9096) | Some GpuResourcePlugin and ResourcePluginManager methods are synchronized unnecessarily |  Major | . | Szilard Nemeth | Gergely Pollak |
+| [YARN-9092](https://issues.apache.org/jira/browse/YARN-9092) | Create an object for cgroups mount enable and cgroups mount path as they belong together |  Minor | . | Szilard Nemeth | Gergely Pollak |
+| [YARN-9124](https://issues.apache.org/jira/browse/YARN-9124) | Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently |  Minor | . | Szilard Nemeth | Adam Antal |
+| [YARN-8199](https://issues.apache.org/jira/browse/YARN-8199) | Logging fileSize of log files under NM Local Dir |  Major | log-aggregation | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14195](https://issues.apache.org/jira/browse/HDFS-14195) | OIV: print out storage policy id in oiv Delimited output |  Minor | tools | Wang, Xinglong | Wang, Xinglong |
+| [YARN-9729](https://issues.apache.org/jira/browse/YARN-9729) | [UI2] Fix error message for logs when ATSv2 is offline |  Major | yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl |
+| [YARN-9657](https://issues.apache.org/jira/browse/YARN-9657) | AbstractLivelinessMonitor add serviceName to PingChecker thread |  Minor | . | Bibin Chundatt | Bilwa S T |
+| [YARN-9715](https://issues.apache.org/jira/browse/YARN-9715) | [UI2] yarn-container-log URI need to be encoded to avoid potential misuses |  Major | . | Prabhu Joseph | Akhil PB |
+| [HADOOP-16453](https://issues.apache.org/jira/browse/HADOOP-16453) | Update how exceptions are handled in NetUtils |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9464](https://issues.apache.org/jira/browse/YARN-9464) | Support "Pending Resource" metrics in RM's RESTful API |  Major | . | Zhankun Tang | Prabhu Joseph |
+| [YARN-9135](https://issues.apache.org/jira/browse/YARN-9135) | NM State store ResourceMappings serialization are tested with Strings instead of real Device objects |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [HDFS-13505](https://issues.apache.org/jira/browse/HDFS-13505) | Turn on HDFS ACLs by default. |  Major | . | Ajay Kumar | Siyao Meng |
+| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
+| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-14625](https://issues.apache.org/jira/browse/HDFS-14625) | Make DefaultAuditLogger class in FSnamesystem to Abstract |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-14491](https://issues.apache.org/jira/browse/HDFS-14491) | More Clarity on Namenode UI Around Blocks and Replicas |  Minor | . | Alan Jackoway | Siyao Meng |
+| [YARN-9134](https://issues.apache.org/jira/browse/YARN-9134) | No test coverage for redefining FPGA / GPU resource types in TestResourceUtils |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9133](https://issues.apache.org/jira/browse/YARN-9133) | Make tests more easy to comprehend in TestGpuResourceHandler |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9140](https://issues.apache.org/jira/browse/YARN-9140) | Code cleanup in ResourcePluginManager.initialize and in TestResourcePluginManager |  Trivial | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9676](https://issues.apache.org/jira/browse/YARN-9676) | Add DEBUG and TRACE level messages to AppLogAggregatorImpl and connected classes |  Major | . | Adam Antal | Adam Antal |
+| [YARN-9488](https://issues.apache.org/jira/browse/YARN-9488) | Skip YARNFeatureNotEnabledException from ClientRMService |  Minor | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9679](https://issues.apache.org/jira/browse/YARN-9679) | Regular code cleanup in TestResourcePluginManager |  Major | . | Szilard Nemeth | Adam Antal |
+| [HADOOP-16504](https://issues.apache.org/jira/browse/HADOOP-16504) | Increase ipc.server.listen.queue.size default from 128 to 256 |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-8586](https://issues.apache.org/jira/browse/YARN-8586) | Extract log aggregation related fields and methods from RMAppImpl |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [YARN-9100](https://issues.apache.org/jira/browse/YARN-9100) | Add tests for GpuResourceAllocator and do minor code cleanup |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [HDFS-14678](https://issues.apache.org/jira/browse/HDFS-14678) | Allow triggerBlockReport to a specific namenode |  Major | datanode | Leon Gao | Leon Gao |
+| [HDFS-14523](https://issues.apache.org/jira/browse/HDFS-14523) | Remove excess read lock for NetworkToplogy |  Major | . | Wu Weiwei | Wu Weiwei |
+| [HADOOP-15246](https://issues.apache.org/jira/browse/HADOOP-15246) | SpanReceiverInfo - Prefer ArrayList over LinkedList |  Trivial | common | David Mollitor | David Mollitor |
+| [HADOOP-16158](https://issues.apache.org/jira/browse/HADOOP-16158) | DistCp to support checksum validation when copy blocks in parallel |  Major | tools/distcp | Kai Xie | Kai Xie |
+| [HADOOP-14784](https://issues.apache.org/jira/browse/HADOOP-14784) | [KMS] Improve KeyAuthorizationKeyProvider#toString() |  Trivial | . | Wei-Chiu Chuang | Yeliang Cang |
+| [HDFS-14746](https://issues.apache.org/jira/browse/HDFS-14746) | Trivial test code update after HDFS-14687 |  Trivial | ec | Wei-Chiu Chuang | Kevin Su |
+| [HDFS-13709](https://issues.apache.org/jira/browse/HDFS-13709) | Report bad block to NN when transfer block encounter EIO exception |  Major | datanode | Chen Zhang | Chen Zhang |
+| [HDFS-14665](https://issues.apache.org/jira/browse/HDFS-14665) | HttpFS: LISTSTATUS response is missing HDFS-specific fields |  Major | httpfs | Siyao Meng | Siyao Meng |
+| [HADOOP-16523](https://issues.apache.org/jira/browse/HADOOP-16523) | Minor spell mistake in comment (PR#388) |  Major | . | Wei-Chiu Chuang |  |
+| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead |  Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena |
+| [HADOOP-16061](https://issues.apache.org/jira/browse/HADOOP-16061) | Update Apache Yetus to 0.10.0 |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14358](https://issues.apache.org/jira/browse/HDFS-14358) | Provide LiveNode and DeadNode filter in DataNode UI |  Major | . | Ravuri Sushma sree | hemanthboyina |
+| [HDFS-14675](https://issues.apache.org/jira/browse/HDFS-14675) | Increase Balancer Defaults Further |  Major | balancer & mover | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14497](https://issues.apache.org/jira/browse/HDFS-14497) | Write lock held by metasave impact following RPC processing |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14760](https://issues.apache.org/jira/browse/HDFS-14760) | Log INFO mode if snapshot usage and actual usage differ |  Major | . | CR Hota | CR Hota |
+| [HDFS-14710](https://issues.apache.org/jira/browse/HDFS-14710) | RBF: Improve some RPC performance by using previous block |  Minor | rbf | xuzq | xuzq |
+| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round |  Major | capacity scheduler | Eric Payne | Manikandan R |
+| [HDFS-14104](https://issues.apache.org/jira/browse/HDFS-14104) | Review getImageTxIdToRetain |  Minor | namenode | David Mollitor | David Mollitor |
+| [HDFS-14256](https://issues.apache.org/jira/browse/HDFS-14256) | Review Logging of NameNode Class |  Minor | namenode | David Mollitor | David Mollitor |
+| [YARN-9783](https://issues.apache.org/jira/browse/YARN-9783) | Remove low-level zookeeper test to be able to build Hadoop against zookeeper 3.5.5 |  Major | test | Mate Szalay-Beko | Mate Szalay-Beko |
+| [HDFS-14748](https://issues.apache.org/jira/browse/HDFS-14748) | Make DataNodePeerMetrics#minOutlierDetectionSamples configurable |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-15998](https://issues.apache.org/jira/browse/HADOOP-15998) | Ensure jar validation works on Windows. |  Blocker | build | Brian Grunkemeyer | Brian Grunkemeyer |
+| [HDFS-13843](https://issues.apache.org/jira/browse/HDFS-13843) | RBF: Add optional parameter -d for detailed listing of mount points. |  Major | federation | Soumyapn | Ayush Saxena |
+| [YARN-9400](https://issues.apache.org/jira/browse/YARN-9400) | Remove unnecessary if at EntityGroupFSTimelineStore#parseApplicationId |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename |  Major | . | Jinglun | Jinglun |
+| [HADOOP-16268](https://issues.apache.org/jira/browse/HADOOP-16268) | Allow custom wrapped exception to be thrown by server if RPC call queue is filled up |  Major | . | CR Hota | CR Hota |
+| [HDFS-14812](https://issues.apache.org/jira/browse/HDFS-14812) | RBF: MountTableRefresherService should load cache when refresh |  Major | . | xuzq | xuzq |
+| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics |  Major | . | Jonathan Hung | Shubham Gupta |
+| [HDFS-14784](https://issues.apache.org/jira/browse/HDFS-14784) | Add more methods to WebHdfsTestUtil to support tests outside of package |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-13913](https://issues.apache.org/jira/browse/HDFS-13913) | LazyPersistFileScrubber.run() should log meaningful warn message |  Minor | namenode | Daniel Templeton | Daniel Green |
+| [HADOOP-16531](https://issues.apache.org/jira/browse/HADOOP-16531) | Log more detail for slow RPC |  Major | . | Chen Zhang | Chen Zhang |
+| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [YARN-9795](https://issues.apache.org/jira/browse/YARN-9795) | ClusterMetrics to include AM allocation delay |  Minor | . | Fengnan Li | Fengnan Li |
+| [YARN-8995](https://issues.apache.org/jira/browse/YARN-8995) | Log events info in AsyncDispatcher when event queue size cumulatively reaches a certain number every time. |  Major | metrics, nodemanager, resourcemanager | zhuqi | zhuqi |
+| [YARN-9787](https://issues.apache.org/jira/browse/YARN-9787) | Typo in analysesErrorMsg |  Trivial | yarn | David Mollitor | Kevin Su |
+| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary |  Major | . | Jonathan Hung | Manoj Kumar |
+| [HADOOP-16549](https://issues.apache.org/jira/browse/HADOOP-16549) | Remove Unsupported SSL/TLS Versions from Docs/Properties |  Minor | documentation, security | Daisuke Kobayashi | Daisuke Kobayashi |
+| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-15184](https://issues.apache.org/jira/browse/HADOOP-15184) | Add GitHub pull request template |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9815](https://issues.apache.org/jira/browse/YARN-9815) | ReservationACLsTestBase fails with NPE |  Minor | yarn | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14840](https://issues.apache.org/jira/browse/HDFS-14840) | Use Java Conccurent Instead of Synchronization in BlockPoolTokenSecretManager |  Minor | hdfs | David Mollitor | David Mollitor |
+| [HDFS-14799](https://issues.apache.org/jira/browse/HDFS-14799) | Do Not Call Map containsKey In Conjunction with get |  Minor | namenode | David Mollitor | hemanthboyina |
+| [HDFS-14795](https://issues.apache.org/jira/browse/HDFS-14795) | Add Throttler for writing block |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16556](https://issues.apache.org/jira/browse/HADOOP-16556) | Fix some LGTM alerts |  Minor | common | Malcolm Taylor | Malcolm Taylor |
+| [HADOOP-16069](https://issues.apache.org/jira/browse/HADOOP-16069) | Support configure ZK\_DTSM\_ZK\_KERBEROS\_PRINCIPAL in ZKDelegationTokenSecretManager using principal with Schema /\_HOST |  Minor | common | luhuachao | luhuachao |
+| [HDFS-14844](https://issues.apache.org/jira/browse/HDFS-14844) | Make buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream  configurable |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16445](https://issues.apache.org/jira/browse/HADOOP-16445) | Allow separate custom signing algorithms for S3 and DDB |  Major | fs/s3 | Siddharth Seth | Siddharth Seth |
+| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs |  Major | . | Jonathan Hung | Manoj Kumar |
+| [HDFS-14837](https://issues.apache.org/jira/browse/HDFS-14837) | Review of Block.java |  Minor | hdfs-client | David Mollitor | David Mollitor |
+| [HDFS-14843](https://issues.apache.org/jira/browse/HDFS-14843) | Double Synchronization in BlockReportLeaseManager |  Minor | . | David Mollitor | David Mollitor |
+| [HDFS-14832](https://issues.apache.org/jira/browse/HDFS-14832) | RBF : Add Icon for ReadOnly False |  Minor | . | hemanthboyina | hemanthboyina |
+| [HDFS-11934](https://issues.apache.org/jira/browse/HDFS-11934) | Add assertion to TestDefaultNameNodePort#testGetAddressFromConf |  Minor | hdfs | legend | Nikhil Navadiya |
+| [YARN-9857](https://issues.apache.org/jira/browse/YARN-9857) | TestDelegationTokenRenewer throws NPE but tests pass |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14564](https://issues.apache.org/jira/browse/HDFS-14564) | Add libhdfs APIs for readFully; add readFully to ByteBufferPositionedReadable |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [HDFS-14850](https://issues.apache.org/jira/browse/HDFS-14850) | Optimize FileSystemAccessService#getFileSystemConfiguration |  Major | httpfs, performance | Lisheng Sun | Lisheng Sun |
+| [HDFS-14192](https://issues.apache.org/jira/browse/HDFS-14192) | Track missing DFS operations in Statistics and StorageStatistics |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16599](https://issues.apache.org/jira/browse/HADOOP-16599) | Allow a SignerInitializer to be specified along with a Custom Signer |  Major | fs/s3 | Siddharth Seth | Siddharth Seth |
+| [HDFS-14888](https://issues.apache.org/jira/browse/HDFS-14888) | RBF: Enable Parallel Test Profile for builds |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16624](https://issues.apache.org/jira/browse/HADOOP-16624) | Upgrade hugo to the latest version in Dockerfile |  Minor | build | Akira Ajisaka | Kevin Su |
+| [HDFS-14814](https://issues.apache.org/jira/browse/HDFS-14814) | RBF: RouterQuotaUpdateService supports inherited rule. |  Major | . | Jinglun | Jinglun |
+| [YARN-9356](https://issues.apache.org/jira/browse/YARN-9356) | Add more tests to ratio method in TestResourceCalculator |  Major | . | Szilard Nemeth | Zoltan Siegl |
+| [HDFS-14898](https://issues.apache.org/jira/browse/HDFS-14898) | Use Relative URLS in Hadoop HDFS HTTP FS |  Major | httpfs | David Mollitor | David Mollitor |
+| [YARN-7291](https://issues.apache.org/jira/browse/YARN-7291) | Better input parsing for resource in allocation file |  Minor | fairscheduler | Yufei Gu | Zoltan Siegl |
+| [YARN-9860](https://issues.apache.org/jira/browse/YARN-9860) | Enable service mode for Docker containers on YARN |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14899](https://issues.apache.org/jira/browse/HDFS-14899) | Use Relative URLS in Hadoop HDFS RBF |  Major | rbf | David Mollitor | David Mollitor |
+| [HDFS-14238](https://issues.apache.org/jira/browse/HDFS-14238) | A log in NNThroughputBenchmark should  change log level to "INFO" instead of "ERROR" |  Major | . | Shen Yinjie | Shen Yinjie |
+| [HADOOP-16643](https://issues.apache.org/jira/browse/HADOOP-16643) | Update netty4 to the latest 4.1.42 |  Major | . | Wei-Chiu Chuang | Lisheng Sun |
+| [HDFS-14810](https://issues.apache.org/jira/browse/HDFS-14810) | Review FSNameSystem editlog sync |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-16152](https://issues.apache.org/jira/browse/HADOOP-16152) | Upgrade Eclipse Jetty version to 9.4.x |  Major | . | Yuming Wang | Siyao Meng |
+| [HADOOP-16579](https://issues.apache.org/jira/browse/HADOOP-16579) | Upgrade to Apache Curator 4.2.0 and ZooKeeper 3.5.6 in Hadoop |  Major | . | Mate Szalay-Beko | Norbert Kalmár |
+| [HDFS-14918](https://issues.apache.org/jira/browse/HDFS-14918) | Remove useless getRedundancyThread from BlockManagerTestUtil |  Minor | . | Fei Hui | Fei Hui |
+| [HDFS-14915](https://issues.apache.org/jira/browse/HDFS-14915) | Move Superuser Check Before Taking Lock For Encryption API |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14921](https://issues.apache.org/jira/browse/HDFS-14921) | Remove SuperUser Check in Setting Storage Policy in FileStatus During Listing |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16615](https://issues.apache.org/jira/browse/HADOOP-16615) | Add password check for credential provider |  Major | . | hong dongdong | hong dongdong |
+| [HDFS-14917](https://issues.apache.org/jira/browse/HDFS-14917) | Change the ICON of "Decommissioned & dead" datanode on "dfshealth.html" |  Trivial | ui | Xieming Li | Xieming Li |
+| [HDFS-14923](https://issues.apache.org/jira/browse/HDFS-14923) | Remove dead code from HealthMonitor |  Minor | . | Fei Hui | Fei Hui |
+| [YARN-9914](https://issues.apache.org/jira/browse/YARN-9914) | Use separate configs for free disk space checking for full and not-full disks |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-14935](https://issues.apache.org/jira/browse/HDFS-14935) | Refactor DFSNetworkTopology#isNodeInScope |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-2442](https://issues.apache.org/jira/browse/YARN-2442) | ResourceManager JMX UI does not give HA State |  Major | resourcemanager | Nishan Shetty | Rohith Sharma K S |
+| [YARN-9889](https://issues.apache.org/jira/browse/YARN-9889) | [UI] Add Application Tag column to RM All Applications table |  Major | . | Kinga Marton | Kinga Marton |
+| [HDFS-14936](https://issues.apache.org/jira/browse/HDFS-14936) | Add getNumOfChildren() for interface InnerNode |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14927](https://issues.apache.org/jira/browse/HDFS-14927) | RBF: Add metrics for async callers thread pool |  Minor | rbf | Leon Gao | Leon Gao |
+| [HADOOP-16678](https://issues.apache.org/jira/browse/HADOOP-16678) | Review of ArrayWritable |  Minor | common | David Mollitor | David Mollitor |
+| [HDFS-14775](https://issues.apache.org/jira/browse/HDFS-14775) | Add Timestamp for longest FSN write/read lock held log |  Major | . | Chen Zhang | Chen Zhang |
+| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14942](https://issues.apache.org/jira/browse/HDFS-14942) | Change Log Level to debug in JournalNodeSyncer#syncWithJournalAtIndex |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14815](https://issues.apache.org/jira/browse/HDFS-14815) | RBF: Update the quota in MountTable when calling setQuota on a MountTable src. |  Major | . | Jinglun | Jinglun |
+| [YARN-9677](https://issues.apache.org/jira/browse/YARN-9677) | Make FpgaDevice and GpuDevice classes more similar to each other |  Major | . | Szilard Nemeth | Kevin Su |
+| [YARN-9890](https://issues.apache.org/jira/browse/YARN-9890) | [UI2] Add Application tag to the app table and app detail page. |  Major | . | Kinga Marton | Kinga Marton |
+| [HDFS-14928](https://issues.apache.org/jira/browse/HDFS-14928) | UI: unifying the WebUI across different components. |  Trivial | ui | Xieming Li | Xieming Li |
+| [HDFS-14975](https://issues.apache.org/jira/browse/HDFS-14975) | Add LF for SetECPolicyCommand usage |  Trivial | . | Fei Hui | Fei Hui |
+| [YARN-9537](https://issues.apache.org/jira/browse/YARN-9537) | Add configuration to disable AM preemption |  Major | fairscheduler | zhoukang | zhoukang |
+| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible |  Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen |
+| [HADOOP-16705](https://issues.apache.org/jira/browse/HADOOP-16705) | MBeanInfoBuilder puts unnecessary memory pressure on the system with a debug log |  Major | metrics | Lukas Majercak | Lukas Majercak |
+| [HADOOP-16691](https://issues.apache.org/jira/browse/HADOOP-16691) | Unify Logging in UserGroupInformation |  Minor | common | David Mollitor | David Mollitor |
+| [HDFS-14882](https://issues.apache.org/jira/browse/HDFS-14882) | Consider DataNode load when #getBlockLocation |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14974](https://issues.apache.org/jira/browse/HDFS-14974) | RBF: Make tests use free ports |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14955](https://issues.apache.org/jira/browse/HDFS-14955) | RBF: getQuotaUsage() on mount point should return global quota. |  Minor | . | Jinglun | Jinglun |
+| [HADOOP-16712](https://issues.apache.org/jira/browse/HADOOP-16712) | Config ha.failover-controller.active-standby-elector.zk.op.retries is not in core-default.xml |  Trivial | . | Wei-Chiu Chuang | Xieming Li |
+| [YARN-9886](https://issues.apache.org/jira/browse/YARN-9886) | Queue mapping based on userid passed through application tag |  Major | scheduler | Kinga Marton | Szilard Nemeth |
+| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN |  Trivial | namenode | Rajesh Balamohan | Xiaoqiao He |
+| [HDFS-14995](https://issues.apache.org/jira/browse/HDFS-14995) | Use log variable directly instead of passing as argument in InvalidateBlocks#printBlockDeletionTime() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14949](https://issues.apache.org/jira/browse/HDFS-14949) | Add getServerDefaults() support to HttpFS |  Major | . | Kihwal Lee | hemanthboyina |
+| [HADOOP-15852](https://issues.apache.org/jira/browse/HADOOP-15852) | Refactor QuotaUsage |  Minor | common | David Mollitor | David Mollitor |
+| [HDFS-15002](https://issues.apache.org/jira/browse/HDFS-15002) |  RBF: Fix annotation in RouterAdmin |  Trivial | . | Jinglun | Jinglun |
+| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-16717](https://issues.apache.org/jira/browse/HADOOP-16717) | Remove GenericsUtil isLog4jLogger dependency on Log4jLoggerAdapter |  Major | . | David Mollitor | Xieming Li |
+| [YARN-9966](https://issues.apache.org/jira/browse/YARN-9966) | Code duplication in UserGroupMappingPlacementRule |  Major | . | Szilard Nemeth | Kevin Su |
+| [YARN-9991](https://issues.apache.org/jira/browse/YARN-9991) | Queue mapping based on userid passed through application tag: Change prefix to 'userid' |  Major | scheduler | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9362](https://issues.apache.org/jira/browse/YARN-9362) | Code cleanup in TestNMLeveldbStateStoreService |  Minor | . | Szilard Nemeth | Denes Gerencser |
+| [YARN-9937](https://issues.apache.org/jira/browse/YARN-9937) | Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-15013](https://issues.apache.org/jira/browse/HDFS-15013) | Reduce NameNode overview tab response time |  Minor | namenode | HuangTao | HuangTao |
+| [HADOOP-16718](https://issues.apache.org/jira/browse/HADOOP-16718) | Allow disabling Server Name Indication (SNI) for Jetty |  Major | . | Siyao Meng | Aravindan Vijayan |
+| [YARN-9958](https://issues.apache.org/jira/browse/YARN-9958) | Remove the invalid lock in ContainerExecutor |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16729](https://issues.apache.org/jira/browse/HADOOP-16729) | Extract version numbers to head of pom.xml |  Minor | build | Tamas Penzes | Tamas Penzes |
+| [MAPREDUCE-7250](https://issues.apache.org/jira/browse/MAPREDUCE-7250) | FrameworkUploader: skip replication check entirely if timeout == 0 |  Major | mrv2 | Peter Bacsko | Peter Bacsko |
+| [YARN-9052](https://issues.apache.org/jira/browse/YARN-9052) | Replace all MockRM submit method definitions with a builder |  Minor | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9128](https://issues.apache.org/jira/browse/YARN-9128) | Use SerializationUtils from apache commons to serialize / deserialize ResourceMappings |  Major | . | Szilard Nemeth | Adam Antal |
+| [HDFS-15023](https://issues.apache.org/jira/browse/HDFS-15023) | [SBN read] ZKFC should check the state before joining the election |  Major | . | Fei Hui | Fei Hui |
+| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN |  Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HDFS-15028](https://issues.apache.org/jira/browse/HDFS-15028) | Keep the capacity of volume and reduce a system call |  Minor | datanode | Yang Yun | Yang Yun |
+| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources |  Major | . | Jonathan Hung | Manikandan R |
+| [HDFS-14522](https://issues.apache.org/jira/browse/HDFS-14522) | Allow compact property description in xml in httpfs |  Major | httpfs | Akira Ajisaka | Masatake Iwasaki |
+| [YARN-5106](https://issues.apache.org/jira/browse/YARN-5106) | Provide a builder interface for FairScheduler allocations for use in tests |  Major | fairscheduler | Karthik Kambatla | Adam Antal |
+| [HDFS-14854](https://issues.apache.org/jira/browse/HDFS-14854) | Create improved decommission monitor implementation |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException |  Major | dfsclient | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-15016](https://issues.apache.org/jira/browse/HDFS-15016) | RBF: getDatanodeReport() should return the latest update |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-10033](https://issues.apache.org/jira/browse/YARN-10033) | TestProportionalCapacityPreemptionPolicy not initializing vcores for effective max resources |  Major | capacity scheduler, test | Eric Payne | Eric Payne |
+| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. |  Major | capacity scheduler, test | Eric Payne | Eric Payne |
+| [YARN-10038](https://issues.apache.org/jira/browse/YARN-10038) | [UI] Finish Time is not correctly parsed in the RM Apps page |  Minor | webapp | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0 |  Major | build | Andras Bokor | Andras Bokor |
+| [HDFS-15062](https://issues.apache.org/jira/browse/HDFS-15062) | Add LOG when sendIBRs failed |  Major | datanode | Fei Hui | Fei Hui |
+| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined |  Critical | capacity scheduler | Eric Payne | Eric Payne |
+| [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | Install yarnpkg and upgrade nodejs in Dockerfile |  Major | buid, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-12999](https://issues.apache.org/jira/browse/HDFS-12999) | When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. |  Major | erasure-coding, hdfs-client | lufei | lufei |
+| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously |  Major | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-15003](https://issues.apache.org/jira/browse/HDFS-15003) | RBF: Make Router support storage type quota. |  Major | . | Jinglun | Jinglun |
+| [HDFS-15081](https://issues.apache.org/jira/browse/HDFS-15081) | Typo in RetryCache#waitForCompletion annotation |  Trivial | namenode | Fei Hui | Fei Hui |
+| [HDFS-15074](https://issues.apache.org/jira/browse/HDFS-15074) | DataNode.DataTransfer thread should catch all the expception and log it. |  Major | datanode | Surendra Singh Lilhore | hemanthboyina |
+| [HDFS-14937](https://issues.apache.org/jira/browse/HDFS-14937) | [SBN read] ObserverReadProxyProvider should throw InterruptException |  Major | . | xuzq | xuzq |
+| [HDFS-14740](https://issues.apache.org/jira/browse/HDFS-14740) | Recover data blocks from persistent memory read cache during datanode restarts |  Major | caching, datanode | Feilong He | Feilong He |
+| [HADOOP-16777](https://issues.apache.org/jira/browse/HADOOP-16777) | Add Tez to LimitedPrivate of ClusterStorageCapacityExceededException |  Minor | common | Wang Yan | Wang Yan |
+| [HDFS-15091](https://issues.apache.org/jira/browse/HDFS-15091) | Cache Admin and Quota Commands Should Check SuperUser Before Taking Lock |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15090](https://issues.apache.org/jira/browse/HDFS-15090) | RBF: MountPoint Listing Should Return Flag Values Of Destination |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15096](https://issues.apache.org/jira/browse/HDFS-15096) | RBF: GetServerDefaults Should be Cached At Router |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15094](https://issues.apache.org/jira/browse/HDFS-15094) | RBF: Reuse ugi string in ConnectionPoolID |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15993](https://issues.apache.org/jira/browse/HADOOP-15993) | Upgrade Kafka version in hadoop-kafka module |  Major | build, security | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15107](https://issues.apache.org/jira/browse/HDFS-15107) | dfs.client.server-defaults.validity.period.ms to support time units |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9989](https://issues.apache.org/jira/browse/YARN-9989) | Typo in CapacityScheduler documentation:  Runtime Configuration |  Major | . | Szilard Nemeth | Kevin Su |
+| [YARN-9868](https://issues.apache.org/jira/browse/YARN-9868) | Validate %primary\_group queue in CS queue manager |  Major | . | Manikandan R | Manikandan R |
+| [YARN-9912](https://issues.apache.org/jira/browse/YARN-9912) | Capacity scheduler: support u:user2:%secondary\_group queue mapping |  Major | capacity scheduler, capacityscheduler | Manikandan R | Manikandan R |
+| [HDFS-15097](https://issues.apache.org/jira/browse/HDFS-15097) | Purge log in KMS and HttpFS |  Minor | httpfs, kms | Doris Gu | Doris Gu |
+| [HDFS-15112](https://issues.apache.org/jira/browse/HDFS-15112) | RBF: Do not return FileNotFoundException when a subcluster is unavailable |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-16753](https://issues.apache.org/jira/browse/HADOOP-16753) | Refactor HAAdmin |  Major | ha | Akira Ajisaka | Xieming Li |
+| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness |  Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein |
+| [YARN-7913](https://issues.apache.org/jira/browse/YARN-7913) | Improve error handling when application recovery fails with exception |  Major | resourcemanager | Gergo Repas | Wilfred Spiegelenburg |
+| [YARN-8472](https://issues.apache.org/jira/browse/YARN-8472) | YARN Container Phase 2 |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-15117](https://issues.apache.org/jira/browse/HDFS-15117) | EC: Add getECTopologyResultForPolicies to DistributedFileSystem |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15119](https://issues.apache.org/jira/browse/HDFS-15119) | Allow expiration of cached locations in DFSInputStream |  Minor | dfsclient | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16811](https://issues.apache.org/jira/browse/HADOOP-16811) | Use JUnit TemporaryFolder Rule in TestFileUtils |  Minor | common, test | David Mollitor | David Mollitor |
+| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) |  Minor | mr-am | Ahmed Hussein | Ahmed Hussein |
+| [YARN-9768](https://issues.apache.org/jira/browse/YARN-9768) | RM Renew Delegation token thread should timeout and retry |  Major | . | CR Hota | Manikandan R |
+| [MAPREDUCE-7260](https://issues.apache.org/jira/browse/MAPREDUCE-7260) | Cross origin request support for Job history server web UI |  Critical | jobhistoryserver | Adam Antal | Adam Antal |
+| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime |  Major | capacity scheduler | Eric Payne | Eric Payne |
+| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs |  Major | libhdfs | John Zhuge | Muhammad Samir Khan |
+| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9624](https://issues.apache.org/jira/browse/YARN-9624) | Use switch case for ProtoUtils#convertFromProtoFormat containerState |  Major | . | Bibin Chundatt | Bilwa S T |
+| [HADOOP-16739](https://issues.apache.org/jira/browse/HADOOP-16739) | Fix native build failure of hadoop-pipes on CentOS 8 |  Major | tools/pipes | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-16847](https://issues.apache.org/jira/browse/HADOOP-16847) | Test TestGroupsCaching fail if HashSet iterates in a different order |  Minor | test | testfixer0 | testfixer0 |
+| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit |  Minor | . | Eric Payne | hemanthboyina |
+| [HDFS-15127](https://issues.apache.org/jira/browse/HDFS-15127) | RBF: Do not allow writes when a subcluster is unavailable for HASH\_ALL mount points. |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [MAPREDUCE-7263](https://issues.apache.org/jira/browse/MAPREDUCE-7263) | Remove obsolete validateTargetPath() from FrameworkUploader |  Major | mrv2 | Adam Antal | Hudáky Márton Gyula |
+| [YARN-10137](https://issues.apache.org/jira/browse/YARN-10137) | UIv2 build is broken in trunk |  Critical | yarn, yarn-ui-v2 | Adam Antal | Adam Antal |
+| [HDFS-15086](https://issues.apache.org/jira/browse/HDFS-15086) | Block scheduled counter never get decremet if the block got deleted before replication. |  Major | 3.1.1 | Surendra Singh Lilhore | hemanthboyina |
+| [HADOOP-16850](https://issues.apache.org/jira/browse/HADOOP-16850) | Support getting thread info from thread group for JvmMetrics to improve the performance |  Major | metrics, performance | Tao Yang | Tao Yang |
+| [HADOOP-13666](https://issues.apache.org/jira/browse/HADOOP-13666) | Supporting rack exclusion in countNumOfAvailableNodes in NetworkTopology |  Major | net | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-16833](https://issues.apache.org/jira/browse/HADOOP-16833) | InstrumentedLock should log lock queue time |  Major | util | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-8374](https://issues.apache.org/jira/browse/YARN-8374) | Upgrade objenesis to 2.6 |  Major | build, timelineservice | Jason Darrell Lowe | Akira Ajisaka |
+| [HDFS-13739](https://issues.apache.org/jira/browse/HDFS-13739) | Add option to disable rack local write preference |  Major | balancer & mover, block placement, datanode, fs, hdfs, hdfs-client, namenode, nn, performance | Hari Sekhon | Ayush Saxena |
+| [HDFS-15176](https://issues.apache.org/jira/browse/HDFS-15176) | Enable GcTimePercentage Metric in NameNode's JvmMetrics. |  Minor | . | Jinglun | Jinglun |
+| [HDFS-15174](https://issues.apache.org/jira/browse/HDFS-15174) | Optimize ReplicaCachingGetSpaceUsed by reducing unnecessary io operations |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-10074](https://issues.apache.org/jira/browse/YARN-10074) | Update netty to 4.1.42Final in yarn-csi |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [HDFS-14861](https://issues.apache.org/jira/browse/HDFS-14861) | Reset LowRedundancyBlocks Iterator periodically |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15120](https://issues.apache.org/jira/browse/HDFS-15120) | Refresh BlockPlacementPolicy at runtime. |  Major | . | Jinglun | Jinglun |
+| [HDFS-15190](https://issues.apache.org/jira/browse/HDFS-15190) | HttpFS : Add Support for Storage Policy Satisfier |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-15033](https://issues.apache.org/jira/browse/HDFS-15033) | Support to save replica cached files to other place and make expired time configurable |  Minor | datanode | Yang Yun | Yang Yun |
+| [YARN-10148](https://issues.apache.org/jira/browse/YARN-10148) | Add Unit test for queue ACL for both FS and CS |  Major | scheduler | Kinga Marton | Kinga Marton |
+| [HADOOP-16899](https://issues.apache.org/jira/browse/HADOOP-16899) | Update HdfsDesign.md to reduce ambiguity |  Minor | documentation | Akshay Nehe | Akshay Nehe |
+| [HADOOP-14630](https://issues.apache.org/jira/browse/HADOOP-14630) | Contract Tests to verify create, mkdirs and rename under a file is forbidden |  Major | fs, fs/azure, fs/s3, fs/swift | Steve Loughran | Steve Loughran |
+| [HDFS-12101](https://issues.apache.org/jira/browse/HDFS-12101) | DFSClient.rename() to unwrap ParentNotDirectoryException; define policy for renames under a file |  Minor | fs | Steve Loughran | Steve Loughran |
+| [HADOOP-16898](https://issues.apache.org/jira/browse/HADOOP-16898) | Batch listing of multiple directories to be an unstable interface |  Major | fs | Steve Loughran | Steve Loughran |
+| [HADOOP-16772](https://issues.apache.org/jira/browse/HADOOP-16772) | Extract version numbers to head of pom.xml (addendum) |  Major | build | Tamas Penzes | Tamas Penzes |
+| [YARN-9050](https://issues.apache.org/jira/browse/YARN-9050) | [Umbrella] Usability improvements for scheduler activities |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HDFS-15159](https://issues.apache.org/jira/browse/HDFS-15159) | Prevent adding same DN multiple times in PendingReconstructionBlocks |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug |  Minor | hdfs | Chen Liang | Chen Liang |
+| [HADOOP-16927](https://issues.apache.org/jira/browse/HADOOP-16927) | Update hadoop-thirdparty dependency version to 1.0.0 |  Major | . | Vinayakumar B | Vinayakumar B |
+| [HADOOP-15620](https://issues.apache.org/jira/browse/HADOOP-15620) | Über-jira: S3A phase VI: Hadoop 3.3 features |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-13377](https://issues.apache.org/jira/browse/HDFS-13377) | The owner of folder can set quota for his sub folder |  Minor | namenode | Yang Yun | Yang Yun |
+| [HADOOP-16938](https://issues.apache.org/jira/browse/HADOOP-16938) | Make non-HA proxy providers pluggable |  Major | . | Roger Liu | Roger Liu |
+| [HDFS-15154](https://issues.apache.org/jira/browse/HDFS-15154) | Allow only hdfs superusers the ability to assign HDFS storage policies |  Major | hdfs | Bob Cauthen | Siddharth Wagle |
+| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor |  Major | . | Íñigo Goiri | Xiaoqiao He |
+| [YARN-10210](https://issues.apache.org/jira/browse/YARN-10210) | Add a RMFailoverProxyProvider that does DNS resolution on failover |  Major | . | Roger Liu | Roger Liu |
+| [HDFS-15238](https://issues.apache.org/jira/browse/HDFS-15238) | RBF：NamenodeHeartbeatService caused memory to grow rapidly |  Major | . | xuzq | xuzq |
+| [HDFS-15239](https://issues.apache.org/jira/browse/HDFS-15239) | Add button to go to the parent directory in the explorer |  Major | . | Íñigo Goiri | hemanthboyina |
+| [HADOOP-16910](https://issues.apache.org/jira/browse/HADOOP-16910) | ABFS Streams to update FileSystem.Statistics counters on IO. |  Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-15619](https://issues.apache.org/jira/browse/HADOOP-15619) | Über-JIRA: S3Guard Phase IV: Hadoop 3.3 features |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-15258](https://issues.apache.org/jira/browse/HDFS-15258) | RBF: Mark Router FSCK unstable |  Major | rbf | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10063](https://issues.apache.org/jira/browse/YARN-10063) | Usage output of container-executor binary needs to include --http/--https argument |  Minor | . | Siddharth Ahuja | Siddharth Ahuja |
+| [MAPREDUCE-7266](https://issues.apache.org/jira/browse/MAPREDUCE-7266) | historyContext doesn't need to be a class attribute inside JobHistoryServer |  Minor | jobhistoryserver | Siddharth Ahuja | Siddharth Ahuja |
+| [YARN-10003](https://issues.apache.org/jira/browse/YARN-10003) | YarnConfigurationStore#checkVersion throws exception that belongs to RMStateStore |  Major | . | Szilard Nemeth | Benjamin Teke |
+| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts |  Major | . | Jonathan Hung | Bilwa S T |
+| [HDFS-14476](https://issues.apache.org/jira/browse/HDFS-14476) | lock too long when fix inconsistent blocks between disk and in-memory |  Major | datanode | Sean Chow | Sean Chow |
+| [YARN-5277](https://issues.apache.org/jira/browse/YARN-5277) | When localizers fail due to resource timestamps being out, provide more diagnostics |  Major | nodemanager | Steve Loughran | Siddharth Ahuja |
+| [YARN-9995](https://issues.apache.org/jira/browse/YARN-9995) | Code cleanup in TestSchedConfCLI |  Minor | . | Szilard Nemeth | Bilwa S T |
+| [YARN-9354](https://issues.apache.org/jira/browse/YARN-9354) | Resources should be created with ResourceTypesTestHelper instead of TestUtils |  Trivial | . | Szilard Nemeth | Andras Gyori |
+| [YARN-10002](https://issues.apache.org/jira/browse/YARN-10002) | Code cleanup and improvements in ConfigurationStoreBaseTest |  Minor | . | Szilard Nemeth | Benjamin Teke |
+| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length |  Major | . | Jonathan Hung | Bilwa S T |
+| [HADOOP-16972](https://issues.apache.org/jira/browse/HADOOP-16972) | Ignore AuthenticationFilterInitializer for KMSWebServer |  Blocker | kms | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10001](https://issues.apache.org/jira/browse/YARN-10001) | Add explanation of unimplemented methods in InMemoryConfigurationStore |  Major | . | Szilard Nemeth | Siddharth Ahuja |
+| [HADOOP-17001](https://issues.apache.org/jira/browse/HADOOP-17001) | The suffix name of the unified compression class |  Major | io | bianqi | bianqi |
+| [YARN-9997](https://issues.apache.org/jira/browse/YARN-9997) | Code cleanup in ZKConfigurationStore |  Minor | . | Szilard Nemeth | Andras Gyori |
+| [YARN-9996](https://issues.apache.org/jira/browse/YARN-9996) | Code cleanup in QueueAdminConfigurationMutationACLPolicy |  Major | . | Szilard Nemeth | Siddharth Ahuja |
+| [HADOOP-16914](https://issues.apache.org/jira/browse/HADOOP-16914) | Adding Output Stream Counters in ABFS |  Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh |
+| [YARN-9998](https://issues.apache.org/jira/browse/YARN-9998) | Code cleanup in LeveldbConfigurationStore |  Minor | . | Szilard Nemeth | Benjamin Teke |
+| [YARN-9999](https://issues.apache.org/jira/browse/YARN-9999) | TestFSSchedulerConfigurationStore: Extend from ConfigurationStoreBaseTest, general code cleanup |  Minor | . | Szilard Nemeth | Benjamin Teke |
+| [YARN-10189](https://issues.apache.org/jira/browse/YARN-10189) | Code cleanup in LeveldbRMStateStore |  Minor | . | Benjamin Teke | Benjamin Teke |
+| [YARN-10237](https://issues.apache.org/jira/browse/YARN-10237) | Add isAbsoluteResource config for queue in scheduler response |  Minor | scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-10160](https://issues.apache.org/jira/browse/YARN-10160) | Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-17046](https://issues.apache.org/jira/browse/HADOOP-17046) | Support downstreams' existing Hadoop-rpc implementations using non-shaded protobuf classes. |  Major | rpc-server | Vinayakumar B | Vinayakumar B |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-8035](https://issues.apache.org/jira/browse/YARN-8035) | Uncaught exception in ContainersMonitorImpl during relaunch due to the process ID changing |  Major | . | Shane Kumpf | Shane Kumpf |
+| [HDFS-13376](https://issues.apache.org/jira/browse/HDFS-13376) | Specify minimum GCC version to avoid TLS support error in Build of hadoop-hdfs-native-client |  Major | build, documentation, native | LiXin Ge | LiXin Ge |
+| [YARN-8388](https://issues.apache.org/jira/browse/YARN-8388) | TestCGroupElasticMemoryController.testNormalExit() hangs on Linux |  Major | nodemanager | Haibo Chen | Miklos Szegedi |
+| [YARN-8108](https://issues.apache.org/jira/browse/YARN-8108) | RM metrics rest API throws GSSException in kerberized environment |  Blocker | . | Kshitij Badani | Sunil G |
+| [HADOOP-15814](https://issues.apache.org/jira/browse/HADOOP-15814) | Maven 3.3.3 unable to parse pom file |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader |  Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles |
+| [HDFS-13957](https://issues.apache.org/jira/browse/HDFS-13957) | Fix incorrect option used in description of InMemoryAliasMap |  Minor | documentation | Yiqun Lin | Yiqun Lin |
+| [YARN-4254](https://issues.apache.org/jira/browse/YARN-4254) | ApplicationAttempt stuck for ever due to UnknownHostException |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-8788](https://issues.apache.org/jira/browse/YARN-8788) | mvn package -Pyarn-ui fails on JDK9 |  Major | . | Akira Ajisaka | Vidura Bhathiya Mudalige |
+| [YARN-7825](https://issues.apache.org/jira/browse/YARN-7825) | [UI2] Maintain constant horizontal application info bar for Application Attempt page |  Major | yarn-ui-v2 | Yesha Vora | Akhil PB |
+| [YARN-8659](https://issues.apache.org/jira/browse/YARN-8659) | RMWebServices returns only RUNNING apps when filtered with queue |  Major | yarn | Prabhu Joseph | Szilard Nemeth |
+| [YARN-8843](https://issues.apache.org/jira/browse/YARN-8843) | updateNodeResource does not support units for memory |  Minor | . | Íñigo Goiri | Manikandan R |
+| [HDFS-13962](https://issues.apache.org/jira/browse/HDFS-13962) | Add null check for add-replica pool to avoid lock acquiring |  Major | . | Yiqun Lin | Surendra Singh Lilhore |
+| [MAPREDUCE-7035](https://issues.apache.org/jira/browse/MAPREDUCE-7035) | Skip javadoc build for auto-generated sources in hadoop-mapreduce-client |  Minor | client, documentation | Mukul Kumar Singh | Mukul Kumar Singh |
+| [YARN-8853](https://issues.apache.org/jira/browse/YARN-8853) | [UI2] Application Attempts tab is not shown correctly when there are no attempts |  Major | yarn-ui-v2 | Charan Hebri | Akhil PB |
+| [MAPREDUCE-7130](https://issues.apache.org/jira/browse/MAPREDUCE-7130) | Rumen crashes trying to handle MRAppMaster recovery events |  Minor | tools/rumen | Jonathan Bender | Peter Bacsko |
+| [HDFS-13949](https://issues.apache.org/jira/browse/HDFS-13949) | Correct the description of dfs.datanode.disk.check.timeout in hdfs-default.xml |  Minor | documentation | Toshihiro Suzuki | Toshihiro Suzuki |
+| [HADOOP-15708](https://issues.apache.org/jira/browse/HADOOP-15708) | Reading values from Configuration before adding deprecations make it impossible to read value with deprecated key |  Minor | conf | Szilard Nemeth | Zoltan Siegl |
+| [YARN-8666](https://issues.apache.org/jira/browse/YARN-8666) | [UI2] Remove application tab from YARN Queue Page |  Major | yarn-ui-v2 | Yesha Vora | Yesha Vora |
+| [YARN-8753](https://issues.apache.org/jira/browse/YARN-8753) | [UI2] Lost nodes representation missing from Nodemanagers Chart |  Major | yarn-ui-v2 | Yesha Vora | Yesha Vora |
+| [YARN-8710](https://issues.apache.org/jira/browse/YARN-8710) | Service AM should set a finite limit on NM container max retries |  Major | yarn-native-services | Suma Shivaprasad | Suma Shivaprasad |
+| [HDFS-13951](https://issues.apache.org/jira/browse/HDFS-13951) | HDFS DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile |  Minor | tools | Steve Loughran | Steve Loughran |
+| [HDFS-13945](https://issues.apache.org/jira/browse/HDFS-13945) | TestDataNodeVolumeFailure is Flaky |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI |  Major | . | Fei Hui | Fei Hui |
+| [YARN-8830](https://issues.apache.org/jira/browse/YARN-8830) | SLS tool fix node addition |  Critical | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-8869](https://issues.apache.org/jira/browse/YARN-8869) | YARN Service Client might not work correctly with RM REST API for Kerberos authentication |  Blocker | . | Eric Yang | Eric Yang |
+| [YARN-8775](https://issues.apache.org/jira/browse/YARN-8775) | TestDiskFailures.testLocalDirsFailures sometimes can fail on concurrent File modifications |  Major | test, yarn | Antal Bálint Steinbach | Antal Bálint Steinbach |
+| [HADOOP-15853](https://issues.apache.org/jira/browse/HADOOP-15853) | TestConfigurationDeprecation leaves behind a temp file, resulting in a license issue |  Major | test | Robert Kanter | Ayush Saxena |
+| [YARN-8879](https://issues.apache.org/jira/browse/YARN-8879) | Kerberos principal is needed when submitting a submarine job |  Critical | . | Zac Zhou | Zac Zhou |
+| [HDFS-13993](https://issues.apache.org/jira/browse/HDFS-13993) | TestDataNodeVolumeFailure#testTolerateVolumeFailuresAfterAddingMoreVolumes is flaky |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-8810](https://issues.apache.org/jira/browse/YARN-8810) | Yarn Service: discrepancy between hashcode and equals of ConfigFile |  Minor | . | Chandni Singh | Chandni Singh |
+| [YARN-8759](https://issues.apache.org/jira/browse/YARN-8759) | Copy of resource-types.xml is not deleted if test fails, causes other test failures |  Major | yarn | Antal Bálint Steinbach | Antal Bálint Steinbach |
+| [HDFS-14000](https://issues.apache.org/jira/browse/HDFS-14000) | RBF: Documentation should reflect right scripts for v3.0 and above |  Major | . | CR Hota | CR Hota |
+| [YARN-8868](https://issues.apache.org/jira/browse/YARN-8868) | Set HTTPOnly attribute to Cookie |  Major | . | Chandni Singh | Chandni Singh |
+| [YARN-8864](https://issues.apache.org/jira/browse/YARN-8864) | NM incorrectly logs container user as the user who sent a start/stop container request in its audit log |  Major | nodemanager | Haibo Chen | Wilfred Spiegelenburg |
+| [HDFS-13990](https://issues.apache.org/jira/browse/HDFS-13990) | Synchronization Issue With HashResolver |  Minor | federation | David Mollitor | David Mollitor |
+| [HDFS-14003](https://issues.apache.org/jira/browse/HDFS-14003) | Fix findbugs warning in trunk for FSImageFormatPBINode |  Major | . | Yiqun Lin | Yiqun Lin |
+| [HDFS-14005](https://issues.apache.org/jira/browse/HDFS-14005) | RBF: Web UI update to bootstrap-3.3.7 |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14002](https://issues.apache.org/jira/browse/HDFS-14002) | TestLayoutVersion#testNameNodeFeatureMinimumCompatibleLayoutVersions fails |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-15418](https://issues.apache.org/jira/browse/HADOOP-15418) | Hadoop KMSAuthenticationFilter needs to use getPropsByPrefix instead of iterator to avoid ConcurrentModificationException |  Major | common | Suma Shivaprasad | Suma Shivaprasad |
+| [HDFS-14007](https://issues.apache.org/jira/browse/HDFS-14007) | Incompatible layout when generating FSImage |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-9872](https://issues.apache.org/jira/browse/HDFS-9872) | HDFS bytes-default configurations should accept multiple size units |  Major | . | Yiqun Lin | Yiqun Lin |
+| [YARN-8910](https://issues.apache.org/jira/browse/YARN-8910) | Misleading log statement in NM when max retries is -1 |  Minor | . | Chandni Singh | Chandni Singh |
+| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 |  Critical | tools/distcp | Ted Yu | Ted Yu |
+| [HDFS-13983](https://issues.apache.org/jira/browse/HDFS-13983) | TestOfflineImageViewer crashes in windows |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property |  Major | documentation | Jason Darrell Lowe | Suma Shivaprasad |
+| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-8922](https://issues.apache.org/jira/browse/YARN-8922) | Fix test-container-executor |  Major | test | Robert Kanter | Robert Kanter |
+| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager |  Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli |
+| [HADOOP-15873](https://issues.apache.org/jira/browse/HADOOP-15873) | Add JavaBeans Activation Framework API to LICENSE.txt |  Major | . | Wei-Chiu Chuang | Akira Ajisaka |
+| [YARN-8919](https://issues.apache.org/jira/browse/YARN-8919) | Some tests fail due to NoClassDefFoundError for OperatorCreationException |  Critical | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14004](https://issues.apache.org/jira/browse/HDFS-14004) | TestLeaseRecovery2#testCloseWhileRecoverLease fails intermittently in trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [MAPREDUCE-4669](https://issues.apache.org/jira/browse/MAPREDUCE-4669) | MRAM web UI does not work with HTTPS |  Major | mr-am | Alejandro Abdelnur | Robert Kanter |
+| [YARN-8814](https://issues.apache.org/jira/browse/YARN-8814) | Yarn Service Upgrade: Update the swagger definition |  Major | . | Chandni Singh | Chandni Singh |
+| [HADOOP-15856](https://issues.apache.org/jira/browse/HADOOP-15856) | Trunk build fails to compile native on Windows |  Blocker | native | Vinayakumar B | Vinayakumar B |
+| [YARN-8939](https://issues.apache.org/jira/browse/YARN-8939) | Javadoc build fails in hadoop-yarn-csi |  Major | . | Takanobu Asanuma | Weiwei Yang |
+| [YARN-8938](https://issues.apache.org/jira/browse/YARN-8938) | Add service upgrade cancel and express examples to the service upgrade doc |  Minor | . | Chandni Singh | Chandni Singh |
+| [HDFS-14021](https://issues.apache.org/jira/browse/HDFS-14021) | TestReconstructStripedBlocksWithRackAwareness#testReconstructForNotEnoughRacks fails intermittently |  Major | erasure-coding, test | Xiao Chen | Xiao Chen |
+| [MAPREDUCE-7151](https://issues.apache.org/jira/browse/MAPREDUCE-7151) | RMContainerAllocator#handleJobPriorityChange expects application\_priority always |  Major | . | Bibin Chundatt | Bilwa S T |
+| [YARN-8929](https://issues.apache.org/jira/browse/YARN-8929) | DefaultOOMHandler should only pick running containers to kill upon oom events |  Major | nodemanager | Haibo Chen | Haibo Chen |
+| [YARN-8587](https://issues.apache.org/jira/browse/YARN-8587) | Delays are noticed to launch docker container |  Major | . | Yesha Vora | dockerzhang |
+| [HDFS-14025](https://issues.apache.org/jira/browse/HDFS-14025) | TestPendingReconstruction.testPendingAndInvalidate fails |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-8941](https://issues.apache.org/jira/browse/YARN-8941) | Fix findbugs warnings in hadoop-yarn-csi |  Minor | . | Takanobu Asanuma | Weiwei Yang |
+| [YARN-8930](https://issues.apache.org/jira/browse/YARN-8930) | CGroup-based strict container memory enforcement does not work with CGroupElasticMemoryController |  Major | nodemanager | Haibo Chen | Haibo Chen |
+| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14028](https://issues.apache.org/jira/browse/HDFS-14028) | HDFS OIV temporary dir deletes folder |  Major | hdfs | Adam Antal | Adam Antal |
+| [HDFS-14027](https://issues.apache.org/jira/browse/HDFS-14027) | DFSStripedOutputStream should implement both hsync methods |  Critical | erasure-coding | Xiao Chen | Xiao Chen |
+| [YARN-8950](https://issues.apache.org/jira/browse/YARN-8950) | Compilation fails with dependency convergence error for hbase.profile=2.0 |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-8854](https://issues.apache.org/jira/browse/YARN-8854) | Upgrade jquery datatable version references to v1.10.19 |  Critical | . | Akhil PB | Akhil PB |
+| [HADOOP-15886](https://issues.apache.org/jira/browse/HADOOP-15886) | Fix findbugs warnings in RegistryDNS.java |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8897](https://issues.apache.org/jira/browse/YARN-8897) | LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability |  Minor | federation, router | Akshay Agarwal | Bilwa S T |
+| [HDFS-14049](https://issues.apache.org/jira/browse/HDFS-14049) | TestHttpFSServerWebServer fails on Windows because of missing winutils.exe |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14042](https://issues.apache.org/jira/browse/HDFS-14042) | Fix NPE when PROVIDED storage is missing |  Major | . | Íñigo Goiri | Virajith Jalaparti |
+| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
+| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. |  Major | . | Sumana Sathish | Wangda Tan |
+| [YARN-8970](https://issues.apache.org/jira/browse/YARN-8970) | Improve the debug message in CS#allocateContainerOnSingleNode |  Trivial | . | Weiwei Yang | Zhankun Tang |
+| [YARN-8865](https://issues.apache.org/jira/browse/YARN-8865) | RMStateStore contains large number of expired RMDelegationToken |  Major | resourcemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart |  Major | hdfs-client | Erik Krogen | Erik Krogen |
+| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections |  Major | mrv2 | Peter Bacsko | Peter Bacsko |
+| [YARN-8866](https://issues.apache.org/jira/browse/YARN-8866) | Fix a parsing error for crossdomain.xml |  Major | build, yarn-ui-v2 | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-6729](https://issues.apache.org/jira/browse/YARN-6729) | Clarify documentation on how to enable cgroup support |  Major | nodemanager | Yufei Gu | Zhankun Tang |
+| [HDFS-14039](https://issues.apache.org/jira/browse/HDFS-14039) | ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6,3) |  Major | erasure-coding | Xiao Chen | Kitti Nanasi |
+| [HADOOP-15903](https://issues.apache.org/jira/browse/HADOOP-15903) | Allow HttpServer2 to discover resources in /static when symlinks are used |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-8990](https://issues.apache.org/jira/browse/YARN-8990) | Fix fair scheduler race condition in app submit and queue cleanup |  Blocker | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HADOOP-15916](https://issues.apache.org/jira/browse/HADOOP-15916) | Upgrade Maven Surefire plugin to 3.0.0-M1 |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9002](https://issues.apache.org/jira/browse/YARN-9002) | YARN Service keytab does not support s3, wasb, gs and is restricted to HDFS and local filesystem only |  Major | yarn-native-services | Gour Saha | Gour Saha |
+| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HDFS-14065](https://issues.apache.org/jira/browse/HDFS-14065) | Failed Storage Locations shows nothing in the Datanode Volume Failures |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15912](https://issues.apache.org/jira/browse/HADOOP-15912) | start-build-env.sh still creates an invalid /etc/sudoers.d/hadoop-build-${USER\_ID} file entry after HADOOP-15802 |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15869](https://issues.apache.org/jira/browse/HADOOP-15869) | BlockDecompressorStream#decompress should not return -1 in case of IOException. |  Major | . | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [MAPREDUCE-7158](https://issues.apache.org/jira/browse/MAPREDUCE-7158) | Inefficient Flush Logic in JobHistory EventWriter |  Major | . | Zichen Sun | Zichen Sun |
+| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact |  Critical | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8856](https://issues.apache.org/jira/browse/YARN-8856) | TestTimelineReaderWebServicesHBaseStorage tests failing with NoClassDefFoundError |  Major | . | Jason Darrell Lowe | Sushil Ks |
+| [HDFS-14054](https://issues.apache.org/jira/browse/HDFS-14054) | TestLeaseRecovery2: testHardLeaseRecoveryAfterNameNodeRestart2 and testHardLeaseRecoveryWithRenameAfterNameNodeRestart are flaky |  Major | . | Zsolt Venczel | Zsolt Venczel |
+| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script |  Major | build | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-13963](https://issues.apache.org/jira/browse/HDFS-13963) | NN UI is broken with IE11 |  Minor | namenode, ui | Daisuke Kobayashi | Ayush Saxena |
+| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 |  Minor | hdfs | Adam Antal | Ayush Saxena |
+| [HADOOP-15939](https://issues.apache.org/jira/browse/HADOOP-15939) | Filter overlapping objenesis class in hadoop-client-minicluster |  Minor | build | Xiaoyu Yao | Xiaoyu Yao |
+| [YARN-8936](https://issues.apache.org/jira/browse/YARN-8936) | Bump up Atsv2 hbase versions |  Major | . | Rohith Sharma K S | Vrushali C |
+| [YARN-8984](https://issues.apache.org/jira/browse/YARN-8984) | AMRMClient#OutstandingSchedRequests leaks when AllocationTags is null or empty |  Critical | . | Yang Wang | Yang Wang |
+| [YARN-9042](https://issues.apache.org/jira/browse/YARN-9042) | Javadoc error in deviceplugin package |  Major | . | Rohith Sharma K S | Zhankun Tang |
+| [HADOOP-15358](https://issues.apache.org/jira/browse/HADOOP-15358) | SFTPConnectionPool connections leakage |  Critical | fs | Mikhail Pryakhin | Mikhail Pryakhin |
+| [HADOOP-15948](https://issues.apache.org/jira/browse/HADOOP-15948) | Inconsistency in get and put syntax if filename/dirname contains space |  Minor | fs | vivek kumar | Ayush Saxena |
+| [HDFS-13816](https://issues.apache.org/jira/browse/HDFS-13816) | dfs.getQuotaUsage() throws NPE on non-existent dir instead of FileNotFoundException |  Major | namenode | Vinayakumar B | Vinayakumar B |
+| [MAPREDUCE-7162](https://issues.apache.org/jira/browse/MAPREDUCE-7162) | TestEvents#testEvents fails |  Critical | jobhistoryserver, test | Zhaohui Xin | Zhaohui Xin |
+| [YARN-9056](https://issues.apache.org/jira/browse/YARN-9056) | Yarn Service Upgrade: Instance state changes from UPGRADING to READY without performing a readiness check |  Critical | . | Chandni Singh | Chandni Singh |
+| [MAPREDUCE-6190](https://issues.apache.org/jira/browse/MAPREDUCE-6190) | If a task stucks before its first heartbeat, it never timeouts and the MR job becomes stuck |  Major | . | Ankit Malhotra | Zhaohui Xin |
+| [YARN-8812](https://issues.apache.org/jira/browse/YARN-8812) | Containers fail during creating a symlink which started with hyphen for a resource file |  Minor | . | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [YARN-9044](https://issues.apache.org/jira/browse/YARN-9044) | LogsCLI should contact ATSv2 for "-am" option |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
+| [HADOOP-15947](https://issues.apache.org/jira/browse/HADOOP-15947) | Fix ITestDynamoDBMetadataStore test error issues |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [YARN-9030](https://issues.apache.org/jira/browse/YARN-9030) | Log aggregation changes to handle filesystems which do not support setting permissions |  Major | log-aggregation | Suma Shivaprasad | Suma Shivaprasad |
+| [YARN-8948](https://issues.apache.org/jira/browse/YARN-8948) | PlacementRule interface should be for all YarnSchedulers |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-9067](https://issues.apache.org/jira/browse/YARN-9067) | YARN Resource Manager is running OOM because of leak of Configuration Object |  Major | yarn-native-services | Eric Yang | Eric Yang |
+| [YARN-9010](https://issues.apache.org/jira/browse/YARN-9010) | Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl |  Major | . | Zhankun Tang | Zhankun Tang |
+| [MAPREDUCE-7165](https://issues.apache.org/jira/browse/MAPREDUCE-7165) | mapred-site.xml is misformatted in single node setup document |  Major | documentation | Akira Ajisaka | Zhaohui Xin |
+| [HDFS-14075](https://issues.apache.org/jira/browse/HDFS-14075) | Terminate the namenode when failed to start log segment |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15970](https://issues.apache.org/jira/browse/HADOOP-15970) | Upgrade plexus-utils from 2.0.5 to 3.1.0 |  Major | security | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15974](https://issues.apache.org/jira/browse/HADOOP-15974) | Upgrade Curator version to 2.13.0 to fix ZK tests |  Major | . | Jason Darrell Lowe | Akira Ajisaka |
+| [YARN-9071](https://issues.apache.org/jira/browse/YARN-9071) | NM and service AM don't have updated status for reinitialized containers |  Critical | . | Billie Rinaldi | Chandni Singh |
+| [YARN-8994](https://issues.apache.org/jira/browse/YARN-8994) | Fix race condition between move app and queue cleanup in Fair Scheduler |  Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-9019](https://issues.apache.org/jira/browse/YARN-9019) | Ratio calculation of ResourceCalculator implementations could return NaN |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9025](https://issues.apache.org/jira/browse/YARN-9025) | TestFairScheduler#testChildMaxResources is flaky |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [MAPREDUCE-7159](https://issues.apache.org/jira/browse/MAPREDUCE-7159) | FrameworkUploader: ensure proper permissions of generated framework tar.gz if restrictive umask is used |  Major | mrv2 | Peter Bacsko | Peter Bacsko |
+| [YARN-9009](https://issues.apache.org/jira/browse/YARN-9009) | Fix flaky test TestEntityGroupFSTimelineStore.testCleanLogs |  Minor | . | OrDTesters | OrDTesters |
+| [YARN-8738](https://issues.apache.org/jira/browse/YARN-8738) | FairScheduler should not parse negative maxResources or minResources values as positive |  Major | fairscheduler | Sen Zhao | Szilard Nemeth |
+| [HDFS-14137](https://issues.apache.org/jira/browse/HDFS-14137) | TestMaintenanceState fails with ArrayIndexOutOfBound Exception |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9114](https://issues.apache.org/jira/browse/YARN-9114) | [UI2] log service redirect url need to support user name |  Major | webapp, yarn-ui-v2 | Sunil G | Akhil PB |
+| [HADOOP-15995](https://issues.apache.org/jira/browse/HADOOP-15995) | Add ldap.bind.password.alias in LdapGroupsMapping to distinguish aliases when using multiple providers through CompositeGroupsMapping |  Major | common | Lukas Majercak | Lukas Majercak |
+| [HDFS-14144](https://issues.apache.org/jira/browse/HDFS-14144) | TestPred fails in Trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [MAPREDUCE-7170](https://issues.apache.org/jira/browse/MAPREDUCE-7170) | Doc typo in PluggableShuffleAndPluggableSort.md |  Minor | documentation | Zhaohui Xin | Zhaohui Xin |
+| [HDFS-14145](https://issues.apache.org/jira/browse/HDFS-14145) | TestBlockStorageMovementAttemptedItems#testNoBlockMovementAttemptFinishedReportAdded fails sporadically in Trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14088](https://issues.apache.org/jira/browse/HDFS-14088) | RequestHedgingProxyProvider can throw NullPointerException when failover due to no lock on currentUsedProxy |  Major | hdfs-client | Yuxuan Wang | Yuxuan Wang |
+| [YARN-9040](https://issues.apache.org/jira/browse/YARN-9040) | LevelDBCacheTimelineStore in ATS 1.5 leaks native memory |  Major | timelineserver | Tarun Parimi | Tarun Parimi |
+| [YARN-9084](https://issues.apache.org/jira/browse/YARN-9084) | Service Upgrade: With default readiness check, the status of upgrade is reported to be successful prematurely |  Major | . | Chandni Singh | Chandni Singh |
+| [HDFS-13661](https://issues.apache.org/jira/browse/HDFS-13661) | Ls command with e option fails when the filesystem is not HDFS |  Major | erasure-coding, tools | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-9154](https://issues.apache.org/jira/browse/YARN-9154) | Fix itemization in YARN service quickstart document |  Minor | documentation | Akira Ajisaka | Ayush Saxena |
+| [HDFS-14166](https://issues.apache.org/jira/browse/HDFS-14166) | Ls with -e option not giving the result in proper format |  Major | . | Soumyapn | Shubham Dewan |
+| [HDFS-14165](https://issues.apache.org/jira/browse/HDFS-14165) | In NameNode UI under DataNode tab ,the Capacity column is Non-Aligned |  Minor | . | Shubham Dewan | Shubham Dewan |
+| [HDFS-14046](https://issues.apache.org/jira/browse/HDFS-14046) | In-Maintenance ICON is missing in datanode info page |  Major | datanode | Harshakiran Reddy | Ranith Sardar |
+| [HDFS-14183](https://issues.apache.org/jira/browse/HDFS-14183) | [SPS] Remove the -w parameter from the -satisfystoragepolicy usage |  Major | . | Ayush Saxena | Ayush Saxena |
+| [MAPREDUCE-7174](https://issues.apache.org/jira/browse/MAPREDUCE-7174) | MapReduce example wordmedian should handle generic options |  Major | . | Fei Hui | Fei Hui |
+| [YARN-9164](https://issues.apache.org/jira/browse/YARN-9164) | Shutdown NM may cause NPE when opportunistic container scheduling is enabled |  Critical | . | lujie | lujie |
+| [YARN-8567](https://issues.apache.org/jira/browse/YARN-8567) | Fetching yarn logs fails for long running application if it is not present in timeline store |  Major | log-aggregation | Tarun Parimi | Tarun Parimi |
+| [HADOOP-15997](https://issues.apache.org/jira/browse/HADOOP-15997) | KMS client uses wrong UGI after HADOOP-14445 |  Blocker | kms | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-16028](https://issues.apache.org/jira/browse/HADOOP-16028) | Fix NetworkTopology chooseRandom function to support excluded nodes |  Major | . | Sihai Ke | Sihai Ke |
+| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 |  Blocker | fs/oss | wujinhu | wujinhu |
+| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp |  Major | resourcemanager, test | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16031](https://issues.apache.org/jira/browse/HADOOP-16031) | TestSecureLogins#testValidKerberosName fails |  Major | security | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9173](https://issues.apache.org/jira/browse/YARN-9173) | FairShare calculation broken for large values after YARN-8833 |  Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14189](https://issues.apache.org/jira/browse/HDFS-14189) | Fix intermittent failure of TestNameNodeMetrics |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14132](https://issues.apache.org/jira/browse/HDFS-14132) | Add BlockLocation.isStriped() to determine if block is replicated or Striped |  Major | hdfs | Shweta | Shweta |
+| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares |  Major | fairscheduler | liyakun | liyakun |
+| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds |  Major | security, test | Jason Darrell Lowe | Akira Ajisaka |
+| [HDFS-14198](https://issues.apache.org/jira/browse/HDFS-14198) | Upload and Create button doesn't get enabled after getting reset. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon |  Major | ipc | Erik Krogen | Erik Krogen |
+| [HADOOP-16043](https://issues.apache.org/jira/browse/HADOOP-16043) | NPE in ITestDynamoDBMetadataStore when fs.s3a.s3guard.ddb.table is not set |  Major | fs/s3 | Adam Antal | Gabor Bota |
+| [YARN-9179](https://issues.apache.org/jira/browse/YARN-9179) | Fix NPE in AbstractYarnScheduler#updateNewContainerInfo |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration |  Critical | webapp | collinma | collinma |
+| [YARN-9203](https://issues.apache.org/jira/browse/YARN-9203) | Fix typos in yarn-default.xml |  Trivial | documentation | Rahul Padmanabhan | Rahul Padmanabhan |
+| [YARN-9194](https://issues.apache.org/jira/browse/YARN-9194) | Invalid event: REGISTERED and LAUNCH\_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM |  Critical | . | lujie | lujie |
+| [HDFS-14175](https://issues.apache.org/jira/browse/HDFS-14175) | EC: Native XOR decoder should reset the output buffer before using it. |  Major | ec, hdfs | Surendra Singh Lilhore | Ayush Saxena |
+| [YARN-9197](https://issues.apache.org/jira/browse/YARN-9197) | NPE in service AM when failed to launch container |  Major | yarn-native-services | kyungwan nam | kyungwan nam |
+| [YARN-9204](https://issues.apache.org/jira/browse/YARN-9204) |  RM fails to start if absolute resource is specified for partition capacity in CS queues |  Blocker | yarn | Jiandan Yang | Jiandan Yang |
+| [HDFS-14207](https://issues.apache.org/jira/browse/HDFS-14207) | ZKFC should catch exception when ha configuration missing |  Major | hdfs | Fei Hui | Fei Hui |
+| [HADOOP-15922](https://issues.apache.org/jira/browse/HADOOP-15922) | DelegationTokenAuthenticationFilter get wrong doAsUser since it does not decode URL |  Major | common, kms | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14218](https://issues.apache.org/jira/browse/HDFS-14218) | EC: Ls -e throw NPE when directory ec policy is disabled |  Major | . | Surendra Singh Lilhore | Ayush Saxena |
+| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info |  Blocker | yarn | Jiandan Yang | Jiandan Yang |
+| [YARN-9205](https://issues.apache.org/jira/browse/YARN-9205) | When using custom resource type, application will fail to run due to the CapacityScheduler throws InvalidResourceRequestException(GREATER\_THEN\_MAX\_ALLOCATION) |  Critical | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8961](https://issues.apache.org/jira/browse/YARN-8961) | [UI2] Flow Run End Time shows 'Invalid date' |  Major | . | Charan Hebri | Akhil PB |
+| [HADOOP-16065](https://issues.apache.org/jira/browse/HADOOP-16065) | -Ddynamodb should be -Ddynamo in AWS SDK testing document |  Minor | documentation | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14228](https://issues.apache.org/jira/browse/HDFS-14228) | Incorrect getSnapshottableDirListing() javadoc |  Major | snapshots | Wei-Chiu Chuang | Dinesh Chitlangia |
+| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-8901](https://issues.apache.org/jira/browse/YARN-8901) | Restart "NEVER" policy does not work with component dependency |  Critical | . | Yesha Vora | Suma Shivaprasad |
+| [YARN-9237](https://issues.apache.org/jira/browse/YARN-9237) | NM should ignore sending finished apps to RM during RM fail-over |  Major | yarn | Jiandan Yang | Jiandan Yang |
+| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14236](https://issues.apache.org/jira/browse/HDFS-14236) | Lazy persist copy/ put fails with ViewFs |  Major | fs | Hanisha Koneru | Hanisha Koneru |
+| [YARN-9251](https://issues.apache.org/jira/browse/YARN-9251) | Build failure for -Dhbase.profile=2.0 |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-9099](https://issues.apache.org/jira/browse/YARN-9099) | GpuResourceAllocator#getReleasingGpus calculates number of GPUs in a wrong way |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-16084](https://issues.apache.org/jira/browse/HADOOP-16084) | Fix the comment for getClass in Configuration |  Trivial | . | Fengnan Li | Fengnan Li |
+| [YARN-9262](https://issues.apache.org/jira/browse/YARN-9262) | TestRMAppAttemptTransitions is failing with an NPE |  Critical | resourcemanager | Sunil G | lujie |
+| [YARN-9231](https://issues.apache.org/jira/browse/YARN-9231) | TestDistributedShell fix timeout |  Major | distributed-shell | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14232](https://issues.apache.org/jira/browse/HDFS-14232) | libhdfs is not included in binary tarball |  Critical | build, libhdfs | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14158](https://issues.apache.org/jira/browse/HDFS-14158) | Checkpointer ignores configured time period \> 5 minutes |  Minor | namenode | Timo Walter | Timo Walter |
+| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO |  Major | . | Kihwal Lee | Zhaohui Xin |
+| [HADOOP-16076](https://issues.apache.org/jira/browse/HADOOP-16076) | SPNEGO+SSL Client Connections with HttpClient Broken |  Major | build, security | Larry McCay | Larry McCay |
+| [HDFS-14202](https://issues.apache.org/jira/browse/HDFS-14202) | "dfs.disk.balancer.max.disk.throughputInMBperSec" property is not working as per set value. |  Major | diskbalancer | Ranith Sardar | Ranith Sardar |
+| [YARN-9149](https://issues.apache.org/jira/browse/YARN-9149) | yarn container -status misses logUrl when integrated with ATSv2 |  Major | . | Rohith Sharma K S | Abhishek Modi |
+| [YARN-9246](https://issues.apache.org/jira/browse/YARN-9246) | NPE when executing a command yarn node -status or -states without additional arguments |  Minor | client | Masahiro Tanaka | Masahiro Tanaka |
+| [HDFS-14242](https://issues.apache.org/jira/browse/HDFS-14242) | OIV WebImageViewer: NPE when param op is not specified |  Major | tools | Siyao Meng | Siyao Meng |
+| [YARN-7627](https://issues.apache.org/jira/browse/YARN-7627) | [ATSv2] When passing a non-number as metricslimit, the error message is wrong |  Trivial | api | Grant Sohn | Charan Hebri |
+| [YARN-8498](https://issues.apache.org/jira/browse/YARN-8498) | Yarn NodeManager OOM Listener Fails Compilation on Ubuntu 18.04 |  Blocker | . | Jack Bearden | Ayush Saxena |
+| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state |  Major | . | Kuhu Shukla | Kuhu Shukla |
+| [HADOOP-16032](https://issues.apache.org/jira/browse/HADOOP-16032) | Distcp It should clear sub directory ACL before applying new ACL on it. |  Major | tools/distcp | Ranith Sardar | Ranith Sardar |
+| [HDFS-14140](https://issues.apache.org/jira/browse/HDFS-14140) | JournalNodeSyncer authentication is failing in secure cluster |  Major | journal-node, security | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-9257](https://issues.apache.org/jira/browse/YARN-9257) | Distributed Shell client throws a NPE for a non-existent queue |  Major | distributed-shell | Charan Hebri | Charan Hebri |
+| [YARN-8761](https://issues.apache.org/jira/browse/YARN-8761) | Service AM support for decommissioning component instances |  Major | . | Billie Rinaldi | Billie Rinaldi |
+| [HADOOP-16098](https://issues.apache.org/jira/browse/HADOOP-16098) | Fix javadoc warnings in hadoop-aws |  Minor | fs/s3 | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-14266](https://issues.apache.org/jira/browse/HDFS-14266) | EC : Fsck -blockId shows null for EC Blocks if One Block Is Not Available. |  Major | . | Harshakiran Reddy | Ayush Saxena |
+| [HADOOP-10007](https://issues.apache.org/jira/browse/HADOOP-10007) | distcp / mv is not working on ftp |  Major | fs | Fabian Zimmermann |  |
+| [HDFS-14274](https://issues.apache.org/jira/browse/HDFS-14274) | EC: NPE While Listing EC Policy For A Directory Following Replication Policy. |  Major | erasure-coding | Souryakanta Dwivedy | Ayush Saxena |
+| [HDFS-14263](https://issues.apache.org/jira/browse/HDFS-14263) | Remove unnecessary block file exists check from FsDatasetImpl#getBlockInputStream() |  Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-7761](https://issues.apache.org/jira/browse/YARN-7761) | [UI2] Clicking 'master container log' or 'Link' next to 'log' under application's appAttempt goes to Old UI's Log link |  Major | yarn-ui-v2 | Sumana Sathish | Akhil PB |
+| [YARN-9295](https://issues.apache.org/jira/browse/YARN-9295) | [UI2] Fix label typo in Cluster Overview page |  Trivial | yarn-ui-v2 | Charan Hebri | Charan Hebri |
+| [YARN-9308](https://issues.apache.org/jira/browse/YARN-9308) | fairscheduler-statedump.log gets generated regardless of service again after the merge of HDFS-7240 |  Blocker | fairscheduler, scheduler | Akira Ajisaka | Wilfred Spiegelenburg |
+| [YARN-9284](https://issues.apache.org/jira/browse/YARN-9284) | Fix the unit of yarn.service.am-resource.memory in the document |  Minor | documentation, yarn-native-services | Masahiro Tanaka | Masahiro Tanaka |
+| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference |  Minor | documentation | Szilard Nemeth | Adam Antal |
+| [HADOOP-16116](https://issues.apache.org/jira/browse/HADOOP-16116) | Fix Spelling Mistakes - DECOMISSIONED |  Trivial | . | David Mollitor | David Mollitor |
+| [HDFS-14287](https://issues.apache.org/jira/browse/HDFS-14287) | DataXceiverServer May Double-Close PeerServer |  Minor | datanode | David Mollitor | David Mollitor |
+| [YARN-9286](https://issues.apache.org/jira/browse/YARN-9286) | [Timeline Server] Sorting based on FinalStatus shows pop-up message |  Minor | timelineserver | Nallasivan | Bilwa S T |
+| [HDFS-14081](https://issues.apache.org/jira/browse/HDFS-14081) | hdfs dfsadmin -metasave metasave\_test results NPE |  Major | hdfs | Shweta | Shweta |
+| [HDFS-14273](https://issues.apache.org/jira/browse/HDFS-14273) | Fix checkstyle issues in BlockLocation's method javadoc |  Trivial | . | Shweta | Shweta |
+| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse |  Major | common | Daryn Sharp | Daryn Sharp |
+| [YARN-9319](https://issues.apache.org/jira/browse/YARN-9319) | Fix compilation issue of  handling typedef an existing name by gcc compiler |  Blocker | . | Wei-Chiu Chuang | Zhankun Tang |
+| [YARN-9238](https://issues.apache.org/jira/browse/YARN-9238) | Avoid allocating opportunistic containers to previous/removed/non-exist application attempt |  Critical | . | lujie | lujie |
+| [YARN-9118](https://issues.apache.org/jira/browse/YARN-9118) | Handle exceptions with parsing user defined GPU devices in GpuDiscoverer |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HDFS-14279](https://issues.apache.org/jira/browse/HDFS-14279) | [SBN Read] Race condition in ObserverReadProxyProvider |  Major | hdfs, namenode | Erik Krogen | Erik Krogen |
+| [HADOOP-16129](https://issues.apache.org/jira/browse/HADOOP-16129) | Misc. bug fixes for KMS Benchmark |  Major | kms | Wei-Chiu Chuang | George Huang |
+| [HDFS-14285](https://issues.apache.org/jira/browse/HDFS-14285) | libhdfs hdfsRead copies entire array even if its only partially filled |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [YARN-9317](https://issues.apache.org/jira/browse/YARN-9317) | Avoid repeated YarnConfiguration#timelineServiceV2Enabled check |  Major | . | Bibin Chundatt | Prabhu Joseph |
+| [YARN-9300](https://issues.apache.org/jira/browse/YARN-9300) | Lazy preemption should trigger an update on queue preemption metrics for CapacityScheduler |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-9213](https://issues.apache.org/jira/browse/YARN-9213) | RM Web UI v1 does not show custom resource allocations for containers page |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [YARN-9329](https://issues.apache.org/jira/browse/YARN-9329) | updatePriority is blocked when using FairScheduler |  Major | . | Jiandan Yang | Jiandan Yang |
+| [HDFS-14299](https://issues.apache.org/jira/browse/HDFS-14299) | ViewFs: Correct error message for read only operations |  Minor | federation | hu xiaodong | hu xiaodong |
+| [HADOOP-16127](https://issues.apache.org/jira/browse/HADOOP-16127) | In ipc.Client, put a new connection could happen after stop |  Major | ipc | Tsz-wo Sze | Tsz-wo Sze |
+| [YARN-8378](https://issues.apache.org/jira/browse/YARN-8378) | ApplicationHistoryManagerImpl#getApplications doesn't honor filters |  Minor | resourcemanager, yarn | Lantao Jin | Lantao Jin |
+| [YARN-9311](https://issues.apache.org/jira/browse/YARN-9311) | TestRMRestart hangs due to a deadlock |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9248](https://issues.apache.org/jira/browse/YARN-9248) | RMContainerImpl:Invalid event: ACQUIRED at KILLED |  Major | . | lujie | lujie |
+| [YARN-9318](https://issues.apache.org/jira/browse/YARN-9318) | Resources#multiplyAndRoundUp does not consider Resource Types |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-16107](https://issues.apache.org/jira/browse/HADOOP-16107) | FilterFileSystem doesn't wrap all create() or new builder calls; may skip CRC logic |  Blocker | fs | Steve Loughran | Steve Loughran |
+| [HADOOP-16149](https://issues.apache.org/jira/browse/HADOOP-16149) | hadoop-mapreduce-client-app build not converging due to transient dependencies |  Major | build | Steve Loughran | Steve Loughran |
+| [YARN-9334](https://issues.apache.org/jira/browse/YARN-9334) | YARN Service Client does not work with SPNEGO when knox is configured |  Major | yarn-native-services | Tarun Parimi | Billie Rinaldi |
+| [YARN-9323](https://issues.apache.org/jira/browse/YARN-9323) | FSLeafQueue#computeMaxAMResource does not override zero values for custom resources |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [HADOOP-16067](https://issues.apache.org/jira/browse/HADOOP-16067) | Incorrect Format Debug Statement KMSACLs |  Trivial | kms | David Mollitor | Charan Hebri |
+| [HDFS-14324](https://issues.apache.org/jira/browse/HDFS-14324) | Fix TestDataNodeVolumeFailure |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-13997](https://issues.apache.org/jira/browse/HDFS-13997) | Secondary NN Web UI displays nothing, and the console log shows moment is not defined. |  Major | namenode | Rui Chen | Ayush Saxena |
+| [HDFS-14272](https://issues.apache.org/jira/browse/HDFS-14272) | [SBN read] ObserverReadProxyProvider should sync with active txnID on startup |  Major | tools | Wei-Chiu Chuang | Erik Krogen |
+| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN |  Critical | datanode | star | star |
+| [YARN-7266](https://issues.apache.org/jira/browse/YARN-7266) | Timeline Server event handler threads locked |  Major | ATSv2, timelineserver | Venkata Puneet Ravuri | Prabhu Joseph |
+| [HADOOP-16150](https://issues.apache.org/jira/browse/HADOOP-16150) | checksumFS doesn't wrap concat(): concatenated files don't have checksums |  Major | fs | Steve Loughran | Steve Loughran |
+| [YARN-8803](https://issues.apache.org/jira/browse/YARN-8803) | [UI2] Show flow runs in the order of recently created time in graph widgets |  Major | yarn-ui-v2 | Akhil PB | Akhil PB |
+| [HDFS-14111](https://issues.apache.org/jira/browse/HDFS-14111) | hdfsOpenFile on HDFS causes unnecessary IO from file offset 0 |  Major | hdfs-client, libhdfs | Todd Lipcon | Sahil Takiar |
+| [YARN-9341](https://issues.apache.org/jira/browse/YARN-9341) | Reentrant lock() before try |  Minor | yarn | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14317](https://issues.apache.org/jira/browse/HDFS-14317) | Standby does not trigger edit log rolling when in-progress edit log tailing is enabled |  Critical | . | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
+| [HADOOP-16174](https://issues.apache.org/jira/browse/HADOOP-16174) | Disable wildfly logs to the console |  Major | fs/azure | Denes Gerencser | Denes Gerencser |
+| [HDFS-14347](https://issues.apache.org/jira/browse/HDFS-14347) | Restore a comment line mistakenly removed in ProtobufRpcEngine |  Major | . | Konstantin Shvachko | Fengnan Li |
+| [HDFS-14333](https://issues.apache.org/jira/browse/HDFS-14333) | Datanode fails to start if any disk has errors during Namenode registration |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed |  Major | ipc | Erik Krogen | Erik Krogen |
+| [YARN-9365](https://issues.apache.org/jira/browse/YARN-9365) | fix wrong command in TimelineServiceV2.md |  Major | timelineserver | Runlin Zhang | Runlin Zhang |
+| [YARN-9357](https://issues.apache.org/jira/browse/YARN-9357) | Modify HBase Liveness monitor log to debug |  Minor | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14359](https://issues.apache.org/jira/browse/HDFS-14359) | Inherited ACL permissions masked when parent directory does not exist (mkdir -p) |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14037](https://issues.apache.org/jira/browse/HDFS-14037) | Fix SSLFactory truststore reloader thread leak in URLConnectionFactory |  Major | hdfs-client, webhdfs | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14348](https://issues.apache.org/jira/browse/HDFS-14348) | Fix JNI exception handling issues in libhdfs |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [YARN-9411](https://issues.apache.org/jira/browse/YARN-9411) | TestYarnNativeServices fails sporadically with bind address in use |  Major | test, yarn-native-services | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-15652](https://issues.apache.org/jira/browse/HADOOP-15652) | Fix typos SPENGO into SPNEGO |  Trivial | documentation | okumin | okumin |
+| [HADOOP-16199](https://issues.apache.org/jira/browse/HADOOP-16199) | KMSLoadBlanceClientProvider does not select token correctly |  Major | . | Xiaoyu Yao | Xiaoyu Yao |
+| [YARN-9227](https://issues.apache.org/jira/browse/YARN-9227) | DistributedShell RelativePath is not removed at end |  Minor | distributed-shell | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9431](https://issues.apache.org/jira/browse/YARN-9431) | Fix flaky junit test fair.TestAppRunnability after YARN-8967 |  Minor | fairscheduler, test | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HADOOP-16225](https://issues.apache.org/jira/browse/HADOOP-16225) | Fix links to the developer mailing lists in DownstreamDev.md |  Minor | documentation | Akira Ajisaka | Wanqiang Ji |
+| [HADOOP-16226](https://issues.apache.org/jira/browse/HADOOP-16226) | new Path(String str) does not remove all the trailing slashes of str |  Minor | fs | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16232](https://issues.apache.org/jira/browse/HADOOP-16232) | Fix errors in the checkstyle configration xmls |  Major | build | Akira Ajisaka | Wanqiang Ji |
+| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized |  Major | scheduler | Daniel Templeton | Peter Bacsko |
+| [HADOOP-16011](https://issues.apache.org/jira/browse/HADOOP-16011) | OsSecureRandom very slow compared to other SecureRandom implementations |  Major | security | Todd Lipcon | Siyao Meng |
+| [HDFS-14389](https://issues.apache.org/jira/browse/HDFS-14389) | getAclStatus returns incorrect permissions and owner when an iNodeAttributeProvider is configured |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-9396](https://issues.apache.org/jira/browse/YARN-9396) | YARN\_RM\_CONTAINER\_CREATED published twice to ATS |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes |  Minor | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-14416](https://issues.apache.org/jira/browse/HDFS-14416) | Fix TestHdfsConfigFields for field dfs.client.failover.resolver.useFQDN |  Major | . | Íñigo Goiri | Fengnan Li |
+| [YARN-9413](https://issues.apache.org/jira/browse/YARN-9413) | Queue resource leak after app fail for CapacityScheduler |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-14635](https://issues.apache.org/jira/browse/HADOOP-14635) | Javadoc correction for AccessControlList#buildACL |  Minor | documentation | Bibin Chundatt | Yeliang Cang |
+| [HADOOP-12890](https://issues.apache.org/jira/browse/HADOOP-12890) | Fix typo in AbstractService |  Trivial | documentation | Mike Drob | Gabor Liptak |
+| [HADOOP-16240](https://issues.apache.org/jira/browse/HADOOP-16240) | start-build-env.sh can consume all disk space during image creation |  Minor | build | Craig Condit | Craig Condit |
+| [HDFS-12245](https://issues.apache.org/jira/browse/HDFS-12245) | Fix INodeId javadoc |  Major | documentation, namenode | Wei-Chiu Chuang | Adam Antal |
+| [HDFS-14420](https://issues.apache.org/jira/browse/HDFS-14420) | Fix typo in KeyShell console |  Minor | kms | hu xiaodong | hu xiaodong |
+| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. |  Minor | documentation | Chris Nauroth | Masatake Iwasaki |
+| [YARN-9481](https://issues.apache.org/jira/browse/YARN-9481) | [JDK 11] Build fails due to hard-coded target version in hadoop-yarn-applications-catalog-webapp |  Major | build | Kei Kori | Kei Kori |
+| [YARN-9379](https://issues.apache.org/jira/browse/YARN-9379) | Can't specify docker runtime through environment |  Minor | nodemanager | caozhiqiang | caozhiqiang |
+| [YARN-9336](https://issues.apache.org/jira/browse/YARN-9336) | JobHistoryServer leaks CLOSE\_WAIT tcp connections when using LogAggregationIndexedFileController |  Major | log-aggregation | Tarun Parimi | Tarun Parimi |
+| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby |  Major | namenode | yunjiong zhao | yunjiong zhao |
+| [HADOOP-15881](https://issues.apache.org/jira/browse/HADOOP-15881) | Remove JUnit from LICENSE.txt |  Major | . | Akira Ajisaka | Kei Kori |
+| [YARN-9487](https://issues.apache.org/jira/browse/YARN-9487) | NodeManager native build shouldn't link against librt on macOS |  Major | nodemanager | Siyao Meng | Siyao Meng |
+| [YARN-6695](https://issues.apache.org/jira/browse/YARN-6695) | Race condition in RM for publishing container events vs appFinished events causes NPE |  Critical | . | Rohith Sharma K S | Prabhu Joseph |
+| [YARN-8622](https://issues.apache.org/jira/browse/YARN-8622) | NodeManager native build fails due to getgrouplist not found on macOS |  Major | nodemanager | Ewan Higgs | Siyao Meng |
+| [YARN-9495](https://issues.apache.org/jira/browse/YARN-9495) | Fix findbugs warnings in hadoop-yarn-server-resourcemanager module |  Minor | . | Tao Yang | Tao Yang |
+| [HADOOP-16265](https://issues.apache.org/jira/browse/HADOOP-16265) | Configuration#getTimeDuration is not consistent between default value and manual settings. |  Major | . | star | star |
+| [HDFS-14445](https://issues.apache.org/jira/browse/HDFS-14445) | TestTrySendErrorReportWhenNNThrowsIOException fails in trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14435](https://issues.apache.org/jira/browse/HDFS-14435) | ObserverReadProxyProvider is unable to properly fetch HAState from Standby NNs |  Major | ha, nn | Erik Krogen | Erik Krogen |
+| [YARN-9339](https://issues.apache.org/jira/browse/YARN-9339) | Apps pending metric incorrect after moving app to a new queue |  Minor | . | Billie Rinaldi | Abhishek Modi |
+| [YARN-9491](https://issues.apache.org/jira/browse/YARN-9491) | TestApplicationMasterServiceFair\>ApplicationMasterServiceTestBase.testUpdateTrackingUrl fails intermittent |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9501](https://issues.apache.org/jira/browse/YARN-9501) | TestCapacitySchedulerOvercommit#testReducePreemptAndCancel fails intermittent |  Minor | capacityscheduler, test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9424](https://issues.apache.org/jira/browse/YARN-9424) | Change getDeclaredMethods to getMethods in FederationClientInterceptor#invokeConcurrent() |  Major | federation | Shen Yinjie | Shen Yinjie |
+| [MAPREDUCE-7200](https://issues.apache.org/jira/browse/MAPREDUCE-7200) | Remove stale eclipse templates |  Minor | . | Akira Ajisaka | Wanqiang Ji |
+| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap |  Blocker | . | xuzq | xuzq |
+| [YARN-6929](https://issues.apache.org/jira/browse/YARN-6929) | yarn.nodemanager.remote-app-log-dir structure is not scalable |  Major | log-aggregation | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type |  Minor | yarn | Ahmed Hussein | Ahmed Hussein |
+| [YARN-9528](https://issues.apache.org/jira/browse/YARN-9528) | Federation RMs starting up at the same time can give duplicate application IDs |  Minor | . | Young Chen | Young Chen |
+| [HDFS-14438](https://issues.apache.org/jira/browse/HDFS-14438) | Fix typo in OfflineEditsVisitorFactory |  Major | hdfs | bianqi | bianqi |
+| [HDFS-14372](https://issues.apache.org/jira/browse/HDFS-14372) | NPE while DN is shutting down |  Major | . | lujie | lujie |
+| [YARN-9524](https://issues.apache.org/jira/browse/YARN-9524) | TestAHSWebServices and TestLogsCLI test case failures |  Major | log-aggregation, test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9432](https://issues.apache.org/jira/browse/YARN-9432) | Reserved containers leak after its request has been cancelled or satisfied when multi-nodes enabled |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-9513](https://issues.apache.org/jira/browse/YARN-9513) | [JDK11] Fix TestMetricsInvariantChecker#testManyRuns in case of JDK greater than 8 |  Major | test | Siyao Meng | Adam Antal |
+| [HADOOP-16293](https://issues.apache.org/jira/browse/HADOOP-16293) | AuthenticationFilterInitializer doc has speudo instead of pseudo |  Trivial | auth, documentation | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9535](https://issues.apache.org/jira/browse/YARN-9535) | Typos in Docker documentation |  Major | . | Szilard Nemeth | Charan Hebri |
+| [HADOOP-16299](https://issues.apache.org/jira/browse/HADOOP-16299) | [JDK 11] Build fails without specifying -Djavac.version=11 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9483](https://issues.apache.org/jira/browse/YARN-9483) | DistributedShell does not release container when failed to localize at launch |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16278](https://issues.apache.org/jira/browse/HADOOP-16278) | With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die |  Major | common, fs/s3, metrics | Rajat Khandelwal | Rajat Khandelwal |
+| [YARN-9522](https://issues.apache.org/jira/browse/YARN-9522) | AppBlock ignores full qualified class name of PseudoAuthenticationHandler |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9504](https://issues.apache.org/jira/browse/YARN-9504) | [UI2] Fair scheduler queue view page does not show actual capacity |  Major | fairscheduler, yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl |
+| [YARN-9493](https://issues.apache.org/jira/browse/YARN-9493) | Scheduler Page does not display the right page by query string |  Major | capacity scheduler, resourcemanager, webapp | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16161](https://issues.apache.org/jira/browse/HADOOP-16161) | NetworkTopology#getWeightUsingNetworkLocation return unexpected result |  Major | net | Xiaoqiao He | Xiaoqiao He |
+| [YARN-9519](https://issues.apache.org/jira/browse/YARN-9519) | TFile log aggregation file format is not working for yarn.log-aggregation.TFile.remote-app-log-dir config |  Major | log-aggregation | Adam Antal | Adam Antal |
+| [HDFS-14482](https://issues.apache.org/jira/browse/HDFS-14482) | Crash when using libhdfs with bad classpath |  Major | . | Todd Lipcon | Sahil Takiar |
+| [YARN-9508](https://issues.apache.org/jira/browse/YARN-9508) | YarnConfiguration areNodeLabel enabled is costly in allocation flow |  Critical | . | Bibin Chundatt | Bilwa S T |
+| [HADOOP-16247](https://issues.apache.org/jira/browse/HADOOP-16247) | NPE in FsUrlConnection |  Major | hdfs-client | Karthik Palanisamy | Karthik Palanisamy |
+| [YARN-9554](https://issues.apache.org/jira/browse/YARN-9554) | TimelineEntity DAO has java.util.Set interface which JAXB can't handle |  Major | timelineservice | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14323](https://issues.apache.org/jira/browse/HDFS-14323) | Distcp fails in Hadoop 3.x when 2.x source webhdfs url has special characters in hdfs file path |  Major | webhdfs | Srinivasu Majeti | Srinivasu Majeti |
+| [YARN-9575](https://issues.apache.org/jira/browse/YARN-9575) | Fix TestYarnConfigurationFields testcase failing |  Major | test, yarn-native-services | Prabhu Joseph | Prabhu Joseph |
+| [MAPREDUCE-7205](https://issues.apache.org/jira/browse/MAPREDUCE-7205) | Treat container scheduler kill exit code as a task attempt killing event |  Major | applicationmaster, mr-am, mrv2 | Wanqiang Ji | Wanqiang Ji |
+| [MAPREDUCE-7198](https://issues.apache.org/jira/browse/MAPREDUCE-7198) | mapreduce.task.timeout=0 configuration used to disable timeout doesn't work |  Minor | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12948](https://issues.apache.org/jira/browse/HADOOP-12948) | Remove the defunct startKdc profile from hadoop-common |  Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-9080](https://issues.apache.org/jira/browse/YARN-9080) | Bucket Directories as part of ATS done accumulates |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9482](https://issues.apache.org/jira/browse/YARN-9482) | DistributedShell job with localization fails in unsecure cluster |  Major | distributed-shell | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9558](https://issues.apache.org/jira/browse/YARN-9558) | Log Aggregation testcases failing |  Major | log-aggregation, test | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished |  Major | namenode | Erik Krogen | Erik Krogen |
+| [YARN-9503](https://issues.apache.org/jira/browse/YARN-9503) | Fix JavaDoc error in TestSchedulerOvercommit |  Minor | documentation, test | Wanqiang Ji | Wanqiang Ji |
+| [YARN-9500](https://issues.apache.org/jira/browse/YARN-9500) | Fix typos in ResourceModel.md |  Trivial | documentation | leiqiang | leiqiang |
+| [HDFS-14434](https://issues.apache.org/jira/browse/HDFS-14434) | webhdfs that connect secure hdfs should not use user.name parameter |  Minor | webhdfs | KWON BYUNGCHANG | KWON BYUNGCHANG |
+| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml |  Major | . | Wanqiang Ji | Akira Ajisaka |
+| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) |  Major | . | Shen Yinjie | Ayush Saxena |
+| [HADOOP-16334](https://issues.apache.org/jira/browse/HADOOP-16334) | Fix yetus-wrapper not working when HADOOP\_YETUS\_VERSION \>= 0.9.0 |  Major | yetus | Wanqiang Ji | Wanqiang Ji |
+| [YARN-9553](https://issues.apache.org/jira/browse/YARN-9553) | Fix NPE in EntityGroupFSTimelineStore#getEntityTimelines |  Major | timelineservice | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [YARN-9033](https://issues.apache.org/jira/browse/YARN-9033) | ResourceHandlerChain#bootstrap is invoked twice during NM start if LinuxContainerExecutor enabled |  Major | yarn | Zhankun Tang | Zhankun Tang |
+| [YARN-9027](https://issues.apache.org/jira/browse/YARN-9027) | EntityGroupFSTimelineStore fails to init LevelDBCacheTimelineStore |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9507](https://issues.apache.org/jira/browse/YARN-9507) | Fix NPE in NodeManager#serviceStop on startup failure |  Minor | . | Bilwa S T | Bilwa S T |
+| [YARN-8947](https://issues.apache.org/jira/browse/YARN-8947) | [UI2] Active User info missing from UI2 |  Major | yarn-ui-v2 | Akhil PB | Akhil PB |
+| [YARN-8906](https://issues.apache.org/jira/browse/YARN-8906) | [UI2] NM hostnames not displayed correctly in Node Heatmap Chart |  Major | . | Charan Hebri | Akhil PB |
+| [YARN-9580](https://issues.apache.org/jira/browse/YARN-9580) | Fulfilled reservation information in assignment is lost when transferring in ParentQueue#assignContainers |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-9595](https://issues.apache.org/jira/browse/YARN-9595) | FPGA plugin: NullPointerException in FpgaNodeResourceUpdateHandler.updateConfiguredResource() |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9600](https://issues.apache.org/jira/browse/YARN-9600) | Support self-adaption width for columns of containers table on app attempt page |  Minor | webapp | Tao Yang | Tao Yang |
+| [HDFS-14527](https://issues.apache.org/jira/browse/HDFS-14527) | Stop all DataNodes may result in NN terminate |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-14486](https://issues.apache.org/jira/browse/HDFS-14486) | The exception classes in some throw statements do not accurately describe why they are thrown |  Minor | . | Haicheng Chen | Ayush Saxena |
+| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics |  Major | ipc | Erik Krogen | Erik Krogen |
+| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() |  Major | . | Konstantin Shvachko | Shweta |
+| [YARN-9594](https://issues.apache.org/jira/browse/YARN-9594) | Fix missing break statement in ContainerScheduler#handle |  Major | . | lujie | lujie |
+| [YARN-9565](https://issues.apache.org/jira/browse/YARN-9565) | RMAppImpl#ranNodes not cleared on FinalTransition |  Major | . | Bibin Chundatt | Bilwa S T |
+| [YARN-9547](https://issues.apache.org/jira/browse/YARN-9547) | ContainerStatusPBImpl default execution type is not returned |  Major | . | Bibin Chundatt | Bilwa S T |
+| [HDFS-13231](https://issues.apache.org/jira/browse/HDFS-13231) | Extend visualization for Decommissioning, Maintenance Mode under Datanode tab in the NameNode UI |  Major | datanode, namenode | Haibo Yan | Stephen O'Donnell |
+| [HADOOP-15960](https://issues.apache.org/jira/browse/HADOOP-15960) | Update guava to 27.0-jre in hadoop-project |  Critical | common, security | Gabor Bota | Gabor Bota |
+| [HDFS-14549](https://issues.apache.org/jira/browse/HDFS-14549) | EditLogTailer shouldn't output full stack trace when interrupted |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-16373](https://issues.apache.org/jira/browse/HADOOP-16373) | Fix typo in FileSystemShell#test documentation |  Trivial | documentation | Dinesh Chitlangia | Dinesh Chitlangia |
+| [HDFS-14556](https://issues.apache.org/jira/browse/HDFS-14556) | Spelling Mistake "gloablly" |  Trivial | hdfs-client | David Mollitor | David Mollitor |
+| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read |  Major | hdfs-client | Zheng Hu | Zheng Hu |
+| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE |  Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu |
+| [HDFS-12315](https://issues.apache.org/jira/browse/HDFS-12315) | Use Path instead of String in the TestHdfsAdmin.verifyOpenFiles() |  Trivial | . | Oleg Danilov | Oleg Danilov |
+| [HDFS-12314](https://issues.apache.org/jira/browse/HDFS-12314) | Typo in the TestDataNodeHotSwapVolumes.testAddOneNewVolume() |  Trivial | . | Oleg Danilov | Oleg Danilov |
+| [YARN-9584](https://issues.apache.org/jira/browse/YARN-9584) | Should put initializeProcessTrees method call before get pid |  Critical | nodemanager | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-14010](https://issues.apache.org/jira/browse/HDFS-14010) | Pass correct DF usage to ReservedSpaceCalculator builder |  Minor | . | Lukas Majercak | Lukas Majercak |
+| [HDFS-14078](https://issues.apache.org/jira/browse/HDFS-14078) | Admin helper fails to prettify NullPointerExceptions |  Major | . | Marton Elek | Marton Elek |
+| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock |  Major | test | Kihwal Lee | Zsolt Venczel |
+| [HDFS-14537](https://issues.apache.org/jira/browse/HDFS-14537) | Journaled Edits Cache is not cleared when formatting the JN |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-14581](https://issues.apache.org/jira/browse/HDFS-14581) | Appending to EC files crashes NameNode |  Critical | erasure-coding | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. |  Major | . | Yicong Cai | Yicong Cai |
+| [HDFS-13893](https://issues.apache.org/jira/browse/HDFS-13893) | DiskBalancer: no validations for Disk balancer commands |  Major | diskbalancer | Harshakiran Reddy | Lokesh Jain |
+| [YARN-9209](https://issues.apache.org/jira/browse/YARN-9209) | When nodePartition is not set in Placement Constraints, containers are allocated only in default partition |  Major | capacity scheduler, scheduler | Tarun Parimi | Tarun Parimi |
+| [HADOOP-15989](https://issues.apache.org/jira/browse/HADOOP-15989) | Synchronized at CompositeService#removeService is not required |  Major | common | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-12487](https://issues.apache.org/jira/browse/HDFS-12487) | FsDatasetSpi.isValidBlock() lacks null pointer check inside and neither do the callers |  Major | balancer & mover, diskbalancer | liumi | liumi |
+| [HDFS-14074](https://issues.apache.org/jira/browse/HDFS-14074) | DataNode runs async disk checks  maybe  throws NullPointerException, and DataNode failed to register to NameSpace. |  Major | hdfs | guangyi lu | guangyi lu |
+| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) |  When evictableMmapped or evictable size is zero, do not throw NoSuchElementException |  Major | hdfs-client, performance | Zheng Hu | Lisheng Sun |
+| [HDFS-13371](https://issues.apache.org/jira/browse/HDFS-13371) | NPE for FsServerDefaults.getKeyProviderUri() for clientProtocol communication between 2.7 and 3.X |  Minor | . | Sherwood Zheng | Sherwood Zheng |
+| [HDFS-14598](https://issues.apache.org/jira/browse/HDFS-14598) | Findbugs warning caused by HDFS-12487 |  Minor | diskbalancer | Wei-Chiu Chuang | Xiaoqiao He |
+| [HADOOP-16390](https://issues.apache.org/jira/browse/HADOOP-16390) | Build fails due to bad use of '\>' in javadoc |  Critical | build | Kei Kori | Kei Kori |
+| [YARN-9639](https://issues.apache.org/jira/browse/YARN-9639) | DecommissioningNodesWatcher cause memory leak |  Blocker | . | Bibin Chundatt | Bilwa S T |
+| [HDFS-14599](https://issues.apache.org/jira/browse/HDFS-14599) | HDFS-12487 breaks test TestDiskBalancer.testDiskBalancerWithFedClusterWithOneNameServiceEmpty |  Major | diskbalancer | Wei-Chiu Chuang | Xiaoqiao He |
+| [YARN-9581](https://issues.apache.org/jira/browse/YARN-9581) | Fix WebAppUtils#getRMWebAppURLWithScheme ignores rm2 |  Major | client | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14618](https://issues.apache.org/jira/browse/HDFS-14618) | Incorrect synchronization of ArrayList field (ArrayList is thread-unsafe). |  Critical | . | Paul Ward | Paul Ward |
+| [YARN-9661](https://issues.apache.org/jira/browse/YARN-9661) | Fix typos in LocalityMulticastAMRMProxyPolicy and AbstractConfigurableFederationPolicy |  Major | federation, yarn | hunshenshi | hunshenshi |
+| [HDFS-14610](https://issues.apache.org/jira/browse/HDFS-14610) | HashMap is not thread safe. Field storageMap is typically synchronized by storageMap. However, in one place, field storageMap is not protected with synchronized. |  Critical | . | Paul Ward | Paul Ward |
+| [YARN-9327](https://issues.apache.org/jira/browse/YARN-9327) | Improve synchronisation in ProtoUtils#convertToProtoFormat block |  Critical | . | Bibin Chundatt | Bibin Chundatt |
+| [YARN-9655](https://issues.apache.org/jira/browse/YARN-9655) | AllocateResponse in FederationInterceptor lost  applicationPriority |  Major | federation | hunshenshi | hunshenshi |
+| [HADOOP-16385](https://issues.apache.org/jira/browse/HADOOP-16385) | Namenode crashes with "RedundancyMonitor thread received Runtime exception" |  Major | . | krishna reddy | Ayush Saxena |
+| [YARN-9658](https://issues.apache.org/jira/browse/YARN-9658) | Fix UT failures in TestLeafQueue |  Minor | . | Tao Yang | Tao Yang |
+| [YARN-9644](https://issues.apache.org/jira/browse/YARN-9644) | First RMContext object is always leaked during switch over |  Blocker | . | Bibin Chundatt | Bibin Chundatt |
+| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java |  Trivial | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16411](https://issues.apache.org/jira/browse/HADOOP-16411) | Fix javadoc warnings in hadoop-dynamometer |  Minor | tools | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-9557](https://issues.apache.org/jira/browse/YARN-9557) | Application fails in diskchecker when ReadWriteDiskValidator is configured. |  Critical | nodemanager | Anuruddh Nayak | Bilwa S T |
+| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor |  Critical | namenode | Daryn Sharp | Xiaoqiao He |
+| [HADOOP-16381](https://issues.apache.org/jira/browse/HADOOP-16381) | The JSON License is included in binary tarball via azure-documentdb:1.16.2 |  Blocker | . | Akira Ajisaka | Sushil Ks |
+| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY |  Major | hdfs | Jiandan Yang | Weiwei Yang |
+| [HADOOP-16418](https://issues.apache.org/jira/browse/HADOOP-16418) | Fix checkstyle and findbugs warnings in hadoop-dynamometer |  Minor | tools | Masatake Iwasaki | Erik Krogen |
+| [YARN-9625](https://issues.apache.org/jira/browse/YARN-9625) | UI2 - No link to a queue on the Queues page for Fair Scheduler |  Major | . | Charan Hebri | Zoltan Siegl |
+| [HDFS-14466](https://issues.apache.org/jira/browse/HDFS-14466) | Add a regression test for HDFS-14323 |  Minor | fs, test, webhdfs | Yuya Ebihara | Masatake Iwasaki |
+| [HDFS-14499](https://issues.apache.org/jira/browse/HDFS-14499) | Misleading REM\_QUOTA value with snapshot and trash feature enabled for a directory |  Major | snapshots | Shashikant Banerjee | Shashikant Banerjee |
+| [YARN-9235](https://issues.apache.org/jira/browse/YARN-9235) | If linux container executor is not set for a GPU cluster GpuResourceHandlerImpl is not initialized and NPE is thrown |  Major | yarn | Antal Bálint Steinbach | Adam Antal |
+| [YARN-9626](https://issues.apache.org/jira/browse/YARN-9626) | UI2 - Fair scheduler queue apps page issues |  Major | . | Charan Hebri | Zoltan Siegl |
+| [HDFS-14642](https://issues.apache.org/jira/browse/HDFS-14642) | processMisReplicatedBlocks does not return correct processed count |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-9645](https://issues.apache.org/jira/browse/YARN-9645) | Fix Invalid event FINISHED\_CONTAINERS\_PULLED\_BY\_AM at NEW on NM restart |  Major | . | krishna reddy | Bilwa S T |
+| [YARN-9646](https://issues.apache.org/jira/browse/YARN-9646) | DistributedShell tests failed to bind to a local host name |  Major | test | Ray Yang | Ray Yang |
+| [YARN-9682](https://issues.apache.org/jira/browse/YARN-9682) | Wrong log message when finalizing the upgrade |  Trivial | . | kyungwan nam | kyungwan nam |
+| [HDFS-13647](https://issues.apache.org/jira/browse/HDFS-13647) | Fix the description of storageType option for space quota |  Major | documentation, tools | Takanobu Asanuma | Takanobu Asanuma |
+| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test |  Major | test | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-9568](https://issues.apache.org/jira/browse/YARN-9568) | NPE in MiniYarnCluster during FileSystemNodeAttributeStore.recover |  Minor | resourcemanager, test | Steve Loughran | Steve Loughran |
+| [YARN-6046](https://issues.apache.org/jira/browse/YARN-6046) | Documentation correction in YarnApplicationSecurity |  Trivial | . | Bibin Chundatt | Yousef Abu-Salah |
+| [HADOOP-16440](https://issues.apache.org/jira/browse/HADOOP-16440) | Distcp can not preserve timestamp with -delete  option |  Major | . | ludun | ludun |
+| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build |  Minor | test | Rushabh Shah | Kevin Su |
+| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16451](https://issues.apache.org/jira/browse/HADOOP-16451) | Update jackson-databind to 2.9.9.1 |  Major | . | Wei-Chiu Chuang | Siyao Meng |
+| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup |  Major | hdfs | Fengnan Li | Fengnan Li |
+| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC |  Major | ipc | Steve Loughran | Steve Loughran |
+| [HADOOP-16461](https://issues.apache.org/jira/browse/HADOOP-16461) | Regression: FileSystem cache lock parses XML within the lock |  Major | fs | Gopal Vijayaraghavan | Gopal Vijayaraghavan |
+| [HDFS-14135](https://issues.apache.org/jira/browse/HDFS-14135) | TestWebHdfsTimeouts Fails intermittently in trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [MAPREDUCE-6973](https://issues.apache.org/jira/browse/MAPREDUCE-6973) | Fix comments on creating \_SUCCESS file. |  Trivial | documentation | Mehul Garnara (MG) | Mehul Garnara (MG) |
+| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs |  Major | common, security | Erik Krogen | Erik Krogen |
+| [HDFS-14425](https://issues.apache.org/jira/browse/HDFS-14425) | Native build fails on macos due to jlong in hdfs.c |  Major | . | hunshenshi | hunshenshi |
+| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission |  Major | . | Yicong Cai | Yicong Cai |
+| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever |  Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich |
+| [HADOOP-15910](https://issues.apache.org/jira/browse/HADOOP-15910) | Javadoc for LdapAuthenticationHandler#ENABLE\_START\_TLS is wrong |  Trivial | . | Ted Yu | Don Jeba |
+| [HDFS-14677](https://issues.apache.org/jira/browse/HDFS-14677) | TestDataNodeHotSwapVolumes#testAddVolumesConcurrently fails intermittently in trunk |  Major | . | Chen Zhang | Chen Zhang |
+| [HADOOP-16460](https://issues.apache.org/jira/browse/HADOOP-16460) | ABFS: fix for Sever Name Indication (SNI) |  Major | fs/azure | Thomas Marqardt | Sneha Vijayarajan |
+| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved |  Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan |
+| [HDFS-14681](https://issues.apache.org/jira/browse/HDFS-14681) | RBF: TestDisableRouterQuota failed because port 8888 was occupied |  Minor | rbf | Wei-Chiu Chuang | Chao Sun |
+| [HDFS-14661](https://issues.apache.org/jira/browse/HDFS-14661) | RBF: updateMountTableEntry shouldn't update mountTableEntry if targetPath not exist |  Major | rbf | xuzq | xuzq |
+| [MAPREDUCE-7225](https://issues.apache.org/jira/browse/MAPREDUCE-7225) | Fix broken current folder expansion during MR job start |  Major | mrv2 | Adam Antal | Peter Bacsko |
+| [HDFS-13529](https://issues.apache.org/jira/browse/HDFS-13529) | Fix default trash policy emptier trigger time correctly |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [HADOOP-12282](https://issues.apache.org/jira/browse/HADOOP-12282) | Connection thread's name should be updated after address changing is detected |  Major | ipc | zhouyingchao | Lisheng Sun |
+| [HADOOP-15410](https://issues.apache.org/jira/browse/HADOOP-15410) | Update scope of log4j in hadoop-auth to provided |  Major | . | lqjack | lqjacklee |
+| [HDFS-14686](https://issues.apache.org/jira/browse/HDFS-14686) | HttpFS: HttpFSFileSystem#getErasureCodingPolicy always returns null |  Major | httpfs | Siyao Meng | Siyao Meng |
+| [HADOOP-15681](https://issues.apache.org/jira/browse/HADOOP-15681) | AuthenticationFilter should generate valid date format for Set-Cookie header regardless of default Locale |  Minor | security | Cao Manh Dat | Cao Manh Dat |
+| [HDFS-13131](https://issues.apache.org/jira/browse/HDFS-13131) | Modifying testcase testEnableAndDisableErasureCodingPolicy |  Minor | . | chencan | chencan |
+| [HADOOP-15865](https://issues.apache.org/jira/browse/HADOOP-15865) | ConcurrentModificationException in Configuration.overlay() method |  Major | . | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-14685](https://issues.apache.org/jira/browse/HDFS-14685) | DefaultAuditLogger doesn't print CallerContext |  Major | hdfs | xuzq | xuzq |
+| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException |  Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira |
+| [HDFS-12826](https://issues.apache.org/jira/browse/HDFS-12826) | Document Saying the RPC port, But it's required IPC port in HDFS Federation Document. |  Minor | balancer & mover, documentation | Harshakiran Reddy | usharani |
+| [HDFS-14669](https://issues.apache.org/jira/browse/HDFS-14669) | TestDirectoryScanner#testDirectoryScannerInFederatedCluster fails intermittently in trunk |  Minor | datanode | qiang Liu | qiang Liu |
+| [HADOOP-16487](https://issues.apache.org/jira/browse/HADOOP-16487) | Update jackson-databind to 2.9.9.2 |  Critical | . | Siyao Meng | Siyao Meng |
+| [HDFS-14679](https://issues.apache.org/jira/browse/HDFS-14679) | Failed to add erasure code policies with example template |  Minor | ec | Yuan Zhou | Yuan Zhou |
+| [YARN-9410](https://issues.apache.org/jira/browse/YARN-9410) | Typo in documentation: Using FPGA On YARN |  Major | . | Szilard Nemeth | Kevin Su |
+| [HDFS-14557](https://issues.apache.org/jira/browse/HDFS-14557) | JournalNode error: Can't scan a pre-transactional edit log |  Major | ha | Wei-Chiu Chuang | Stephen O'Donnell |
+| [HADOOP-16457](https://issues.apache.org/jira/browse/HADOOP-16457) | Hadoop does not work with Kerberos config in hdfs-site.xml for simple security |  Minor | . | Eric Yang | Prabhu Joseph |
+| [HDFS-14692](https://issues.apache.org/jira/browse/HDFS-14692) | Upload button should not encode complete url |  Major | . | Lokesh Jain | Lokesh Jain |
+| [HADOOP-15908](https://issues.apache.org/jira/browse/HADOOP-15908) | hadoop-build-tools jar is downloaded from remote repository instead of using from local |  Minor | build | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. |  Major | . | Jinglun | Jinglun |
+| [YARN-9601](https://issues.apache.org/jira/browse/YARN-9601) | Potential NPE in ZookeeperFederationStateStore#getPoliciesConfigurations |  Major | federation, yarn | hunshenshi | hunshenshi |
+| [YARN-9685](https://issues.apache.org/jira/browse/YARN-9685) | NPE when rendering the info table of leaf queue in non-accessible partitions |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HDFS-14459](https://issues.apache.org/jira/browse/HDFS-14459) | ClosedChannelException silently ignored in FsVolumeList.addBlockPool() |  Major | datanode | Stephen O'Donnell | Stephen O'Donnell |
+| [YARN-9732](https://issues.apache.org/jira/browse/YARN-9732) | yarn.system-metrics-publisher.enabled=false is not honored by RM |  Major | resourcemanager, timelineclient | KWON BYUNGCHANG | KWON BYUNGCHANG |
+| [YARN-9527](https://issues.apache.org/jira/browse/YARN-9527) | Rogue LocalizerRunner/ContainerLocalizer repeatedly downloading same file |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-14623](https://issues.apache.org/jira/browse/HDFS-14623) | In NameNode Web UI, for Head the file (first 32K) old data is showing |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-12125](https://issues.apache.org/jira/browse/HDFS-12125) | Document the missing EC removePolicy command |  Major | documentation, erasure-coding | Wenxin He | Siyao Meng |
+| [HDFS-13359](https://issues.apache.org/jira/browse/HDFS-13359) | DataXceiver hung due to the lock in FsDatasetImpl#getBlockInputStream |  Major | datanode | Yiqun Lin | Yiqun Lin |
+| [YARN-9722](https://issues.apache.org/jira/browse/YARN-9722) | PlacementRule logs object ID in place of queue name. |  Minor | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9451](https://issues.apache.org/jira/browse/YARN-9451) | AggregatedLogsBlock shows wrong NM http port |  Minor | nodemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9723](https://issues.apache.org/jira/browse/YARN-9723) | ApplicationPlacementContext is not required for terminated jobs during recovery |  Major | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-12914](https://issues.apache.org/jira/browse/HDFS-12914) | Block report leases cause missing blocks until next report |  Critical | namenode | Daryn Sharp | Santosh Marella |
+| [YARN-9719](https://issues.apache.org/jira/browse/YARN-9719) | Failed to restart yarn-service if it doesn’t exist in RM |  Major | yarn-native-services | kyungwan nam | kyungwan nam |
+| [HDFS-14148](https://issues.apache.org/jira/browse/HDFS-14148) | HDFS OIV ReverseXML SnapshotSection parser throws exception when there are more than one snapshottable directory |  Major | hdfs | Siyao Meng | Siyao Meng |
+| [HDFS-14708](https://issues.apache.org/jira/browse/HDFS-14708) | TestLargeBlockReport#testBlockReportSucceedsWithLargerLengthLimit fails in trunk |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9744](https://issues.apache.org/jira/browse/YARN-9744) | RollingLevelDBTimelineStore.getEntityByTime fails with NPE |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16495](https://issues.apache.org/jira/browse/HADOOP-16495) | Fix invalid metric types in PrometheusMetricsSink |  Major | metrics | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9747](https://issues.apache.org/jira/browse/YARN-9747) | Reduce additional namenode call by EntityGroupFSTimelineStore#cleanLogs |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14595](https://issues.apache.org/jira/browse/HDFS-14595) | HDFS-11848 breaks API compatibility |  Blocker | . | Wei-Chiu Chuang | Siyao Meng |
+| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS |  Major | webhdfs | Jing Wang | Masatake Iwasaki |
+| [HDFS-14719](https://issues.apache.org/jira/browse/HDFS-14719) | Correct the safemode threshold value in BlockManagerSafeMode |  Major | namenode | Surendra Singh Lilhore | hemanthboyina |
+| [MAPREDUCE-7230](https://issues.apache.org/jira/browse/MAPREDUCE-7230) | TestHSWebApp.testLogsViewSingle fails |  Major | jobhistoryserver, test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9749](https://issues.apache.org/jira/browse/YARN-9749) | TestAppLogAggregatorImpl#testDFSQuotaExceeded fails on trunk |  Major | log-aggregation, test | Peter Bacsko | Adam Antal |
+| [YARN-9461](https://issues.apache.org/jira/browse/YARN-9461) | TestRMWebServicesDelegationTokenAuthentication.testCancelledDelegationToken fails with HTTP 400 |  Minor | resourcemanager, test | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16391](https://issues.apache.org/jira/browse/HADOOP-16391) | Duplicate values in rpcDetailedMetrics |  Major | . | Bilwa S T | Bilwa S T |
+| [HDFS-14456](https://issues.apache.org/jira/browse/HDFS-14456) | HAState#prepareToEnterState needn't a lock |  Major | hdfs | hunshenshi | hunshenshi |
+| [YARN-2599](https://issues.apache.org/jira/browse/YARN-2599) | Standby RM should expose jmx endpoint |  Major | resourcemanager | Karthik Kambatla | Rohith Sharma K S |
+| [HDFS-12012](https://issues.apache.org/jira/browse/HDFS-12012) | Fix spelling mistakes in BPServiceActor.java. |  Major | datanode | chencan | chencan |
+| [HDFS-14127](https://issues.apache.org/jira/browse/HDFS-14127) | Add a description about the observer read configuration |  Minor | . | xiangheng | xiangheng |
+| [HDFS-14687](https://issues.apache.org/jira/browse/HDFS-14687) | Standby Namenode never come out of safemode when EC files are being written. |  Critical | ec, namenode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot |  Critical | snapshots | Yongjun Zhang | Shashikant Banerjee |
+| [YARN-9758](https://issues.apache.org/jira/browse/YARN-9758) | Upgrade JQuery to latest version for YARN UI |  Major | yarn | Akhil PB | Akhil PB |
+| [HDFS-13201](https://issues.apache.org/jira/browse/HDFS-13201) | Fix prompt message in testPolicyAndStateCantBeNull |  Minor | . | chencan | chencan |
+| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage |  Major | rolling upgrades | Yicong Cai | Yicong Cai |
+| [HDFS-14582](https://issues.apache.org/jira/browse/HDFS-14582) | Failed to start DN with ArithmeticException when NULL checksum used |  Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16496](https://issues.apache.org/jira/browse/HADOOP-16496) | Apply HDDS-1870 (ConcurrentModification at PrometheusMetricsSink) to Hadoop common |  Major | metrics | Akira Ajisaka | Akira Ajisaka |
+| [YARN-5857](https://issues.apache.org/jira/browse/YARN-5857) | TestLogAggregationService.testFixedSizeThreadPool fails intermittently on trunk |  Minor | . | Varun Saxena | Bilwa S T |
+| [YARN-9217](https://issues.apache.org/jira/browse/YARN-9217) | Nodemanager will fail to start if GPU is misconfigured on the node or GPU drivers missing |  Major | yarn | Antal Bálint Steinbach | Peter Bacsko |
+| [HDFS-14759](https://issues.apache.org/jira/browse/HDFS-14759) | HDFS cat logs an info message |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14741](https://issues.apache.org/jira/browse/HDFS-14741) | RBF: RecoverLease should be return false when the file is open in multiple destination |  Major | rbf | xuzq | xuzq |
+| [HDFS-14583](https://issues.apache.org/jira/browse/HDFS-14583) | FileStatus#toString() will throw IllegalArgumentException |  Major | . | xuzq | xuzq |
+| [YARN-9774](https://issues.apache.org/jira/browse/YARN-9774) | Fix order of arguments for assertEquals in TestSLSUtils |  Minor | test | Nikhil Navadiya | Nikhil Navadiya |
+| [HDFS-13596](https://issues.apache.org/jira/browse/HDFS-13596) | NN restart fails after RollingUpgrade from 2.x to 3.x |  Blocker | hdfs | Hanisha Koneru | Fei Hui |
+| [HDFS-14396](https://issues.apache.org/jira/browse/HDFS-14396) | Failed to load image from FSImageFile when downgrade from 3.x to 2.x |  Blocker | rolling upgrades | Fei Hui | Fei Hui |
+| [HDFS-14747](https://issues.apache.org/jira/browse/HDFS-14747) | RBF: IsFileClosed should be return false when the file is open in multiple destination |  Major | rbf | xuzq | xuzq |
+| [HDFS-14761](https://issues.apache.org/jira/browse/HDFS-14761) | RBF: MountTableResolver cannot invalidate cache correctly |  Major | rbf | Yuxuan Wang | Yuxuan Wang |
+| [HDFS-14722](https://issues.apache.org/jira/browse/HDFS-14722) | RBF: GetMountPointStatus should return mountTable information when getFileInfoAll throw IOException |  Major | rbf | xuzq | xuzq |
+| [YARN-8917](https://issues.apache.org/jira/browse/YARN-8917) | Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-9771](https://issues.apache.org/jira/browse/YARN-9771) | Add GPU in the container-executor.cfg example |  Trivial | nodemanager, yarn | Adam Antal | Kinga Marton |
+| [YARN-9642](https://issues.apache.org/jira/browse/YARN-9642) | Fix Memory Leak in AbstractYarnScheduler caused by timer |  Blocker | resourcemanager | Bibin Chundatt | Bibin Chundatt |
+| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously |  Major | namenode, qjm | Erik Krogen | Erik Krogen |
+| [HDFS-2470](https://issues.apache.org/jira/browse/HDFS-2470) | NN should automatically set permissions on dfs.namenode.\*.dir |  Major | namenode | Aaron Myers | Siddharth Wagle |
+| [HADOOP-15958](https://issues.apache.org/jira/browse/HADOOP-15958) | Revisiting LICENSE and NOTICE files |  Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16533](https://issues.apache.org/jira/browse/HADOOP-16533) | Update jackson-databind to 2.9.9.3 |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications |  Major | . | Jonathan Hung | Jonathan Hung |
+| [YARN-9640](https://issues.apache.org/jira/browse/YARN-9640) | Slow event processing could cause too many attempt unregister events |  Critical | . | Bibin Chundatt | Bibin Chundatt |
+| [HDFS-14721](https://issues.apache.org/jira/browse/HDFS-14721) | RBF: ProxyOpComplete is not accurate in FederationRPCPerformanceMonitor |  Major | rbf | xuzq | xuzq |
+| [HDFS-11246](https://issues.apache.org/jira/browse/HDFS-11246) | FSNameSystem#logAuditEvent should be called outside the read or write locks |  Major | . | Kuhu Shukla | Xiaoqiao He |
+| [HDFS-12212](https://issues.apache.org/jira/browse/HDFS-12212) | Options.Rename.To\_TRASH is considered even when Options.Rename.NONE is specified |  Major | namenode | Vinayakumar B | Vinayakumar B |
+| [YARN-9714](https://issues.apache.org/jira/browse/YARN-9714) | ZooKeeper connection in ZKRMStateStore leaks after RM transitioned to standby |  Major | resourcemanager | Tao Yang | Tao Yang |
+| [HDFS-14796](https://issues.apache.org/jira/browse/HDFS-14796) | Define LOG instead of BlockManager.LOG in ErasureCodingWork/ReplicationWork |  Major | . | Fei Hui | Fei Hui |
+| [YARN-9540](https://issues.apache.org/jira/browse/YARN-9540) | TestRMAppTransitions fails intermittently |  Minor | resourcemanager, test | Prabhu Joseph | Tao Yang |
+| [HDFS-8178](https://issues.apache.org/jira/browse/HDFS-8178) | QJM doesn't move aside stale inprogress edits files |  Major | qjm | Zhe Zhang | Istvan Fajth |
+| [YARN-9798](https://issues.apache.org/jira/browse/YARN-9798) | ApplicationMasterServiceTestBase#testRepeatedFinishApplicationMaster fails intermittently |  Minor | test | Tao Yang | Tao Yang |
+| [YARN-9800](https://issues.apache.org/jira/browse/YARN-9800) | TestRMDelegationTokens can fail in testRemoveExpiredMasterKeyInRMStateStore |  Major | test, yarn | Adam Antal | Adam Antal |
+| [YARN-9793](https://issues.apache.org/jira/browse/YARN-9793) | Remove duplicate sentence from TimelineServiceV2.md |  Major | ATSv2, docs | Kinga Marton | Kinga Marton |
+| [YARN-8174](https://issues.apache.org/jira/browse/YARN-8174) | Add containerId to ResourceLocalizationService fetch failure log statement |  Minor | nodemanager | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14706](https://issues.apache.org/jira/browse/HDFS-14706) | Checksums are not checked if block meta file is less than 7 bytes |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14630](https://issues.apache.org/jira/browse/HDFS-14630) | Configuration.getTimeDurationHelper() should not log time unit warning in info log. |  Minor | hdfs | Surendra Singh Lilhore | hemanthboyina |
+| [YARN-9797](https://issues.apache.org/jira/browse/YARN-9797) | LeafQueue#activateApplications should use resourceCalculator#fitsIn |  Blocker | . | Bibin Chundatt | Bilwa S T |
+| [HADOOP-16534](https://issues.apache.org/jira/browse/HADOOP-16534) | Exclude submarine from hadoop source build |  Major | . | Nanda kumar | Nanda kumar |
+| [HDFS-14807](https://issues.apache.org/jira/browse/HDFS-14807) | SetTimes updates all negative values apart from -1 |  Major | . | Harshakiran Reddy | Ayush Saxena |
+| [YARN-9785](https://issues.apache.org/jira/browse/YARN-9785) | Fix DominantResourceCalculator when one resource is zero |  Blocker | . | Bilwa S T | Bilwa S T |
+| [HDFS-14777](https://issues.apache.org/jira/browse/HDFS-14777) | RBF: Set ReadOnly is failing for mount Table but actually readonly succeed to set |  Major | . | Ranith Sardar | Ranith Sardar |
+| [YARN-9718](https://issues.apache.org/jira/browse/YARN-9718) | Yarn REST API, services endpoint remote command ejection |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-14826](https://issues.apache.org/jira/browse/HDFS-14826) | dfs.ha.zkfc.port property duplicated in hdfs-default.xml |  Major | hdfs | Renukaprasad C | Renukaprasad C |
+| [YARN-9817](https://issues.apache.org/jira/browse/YARN-9817) | Fix failing testcases due to not initialized AsyncDispatcher -  ArithmeticException: / by zero |  Major | test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9813](https://issues.apache.org/jira/browse/YARN-9813) | RM does not start on JDK11 when UIv2 is enabled |  Critical | resourcemanager, yarn | Adam Antal | Adam Antal |
+| [YARN-9812](https://issues.apache.org/jira/browse/YARN-9812) | mvn javadoc:javadoc fails in hadoop-sls |  Major | documentation | Akira Ajisaka | Abhishek Modi |
+| [YARN-9784](https://issues.apache.org/jira/browse/YARN-9784) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue is flaky |  Major | test | Kinga Marton | Kinga Marton |
+| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted |  Critical | . | Rohith Sharma K S | Prabhu Joseph |
+| [HADOOP-16554](https://issues.apache.org/jira/browse/HADOOP-16554) | mvn javadoc:javadoc fails in hadoop-aws |  Major | documentation | Akira Ajisaka | Xieming Li |
+| [YARN-9728](https://issues.apache.org/jira/browse/YARN-9728) |  ResourceManager REST API can produce an illegal xml response |  Major | api, resourcemanager | Thomas | Prabhu Joseph |
+| [HDFS-14835](https://issues.apache.org/jira/browse/HDFS-14835) | RBF: Secured Router should not run when it can't initialize DelegationTokenSecretManager |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14838](https://issues.apache.org/jira/browse/HDFS-14838) | RBF: Display RPC (instead of HTTP) Port Number in RBF web UI |  Minor | rbf, ui | Xieming Li | Xieming Li |
+| [YARN-9816](https://issues.apache.org/jira/browse/YARN-9816) | EntityGroupFSTimelineStore#scanActiveLogs fails when undesired files are present under /ats/active. |  Major | timelineserver | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14699](https://issues.apache.org/jira/browse/HDFS-14699) | Erasure Coding: Storage not considered in live replica when replication streams hard limit reached to threshold |  Critical | ec | Zhao Yi Ming | Zhao Yi Ming |
+| [HDFS-14798](https://issues.apache.org/jira/browse/HDFS-14798) | Synchronize invalidateBlocks in DatanodeDescriptor |  Minor | namenode | David Mollitor | hemanthboyina |
+| [HDFS-14821](https://issues.apache.org/jira/browse/HDFS-14821) | Make HDFS-14617 (fsimage sub-sections) off by default |  Blocker | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log |  Minor | datanode, hdfs | qiang Liu | qiang Liu |
+| [YARN-9833](https://issues.apache.org/jira/browse/YARN-9833) | Race condition when DirectoryCollection.checkDirs() runs during container launch |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-9837](https://issues.apache.org/jira/browse/YARN-9837) | YARN Service fails to fetch status for Stopped apps with bigger spec files |  Major | yarn-native-services | Tarun Parimi | Tarun Parimi |
+| [YARN-2255](https://issues.apache.org/jira/browse/YARN-2255) | YARN Audit logging not added to log4j.properties |  Major | . | Varun Saxena | Aihua Xu |
+| [YARN-9814](https://issues.apache.org/jira/browse/YARN-9814) | JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager |  Minor | log-aggregation, yarn | Adam Antal | Adam Antal |
+| [HDFS-14836](https://issues.apache.org/jira/browse/HDFS-14836) | FileIoProvider should not increase FileIoErrors metric in datanode volume metric |  Minor | . | Aiphago | Aiphago |
+| [HDFS-14846](https://issues.apache.org/jira/browse/HDFS-14846) | libhdfs tests are failing on trunk due to jni usage bugs |  Major | libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [HDFS-14609](https://issues.apache.org/jira/browse/HDFS-14609) | RBF: Security should use common AuthenticationFilter |  Major | . | CR Hota | Chen Zhang |
+| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark |  Major | common, kms | Yuval Degani | Yuval Degani |
+| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-13660](https://issues.apache.org/jira/browse/HDFS-13660) | DistCp job fails when new data is appended in the file while the distCp copy job is running |  Critical | distcp | Mukund Thakur | Mukund Thakur |
+| [HDFS-14868](https://issues.apache.org/jira/browse/HDFS-14868) | RBF: Fix typo in TestRouterQuota |  Trivial | . | Jinglun | Jinglun |
+| [HDFS-14808](https://issues.apache.org/jira/browse/HDFS-14808) | EC: Improper size values for corrupt ec block in LOG |  Major | ec | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14845](https://issues.apache.org/jira/browse/HDFS-14845) | Ignore AuthenticationFilterInitializer for HttpFSServerWebServer and honor hadoop.http.authentication configs |  Critical | httpfs | Akira Ajisaka | Prabhu Joseph |
+| [HADOOP-16602](https://issues.apache.org/jira/browse/HADOOP-16602) | mvn package fails in hadoop-aws |  Major | documentation | Xieming Li | Xieming Li |
+| [HDFS-14874](https://issues.apache.org/jira/browse/HDFS-14874) | Fix TestHDFSCLI and TestDFSShell test break because of logging change in mkdir |  Major | hdfs | Gabor Bota | Gabor Bota |
+| [HDFS-14873](https://issues.apache.org/jira/browse/HDFS-14873) | Fix dfsadmin doc for triggerBlockReport |  Major | documentation | Fei Hui | Fei Hui |
+| [HDFS-14849](https://issues.apache.org/jira/browse/HDFS-14849) | Erasure Coding: the internal block is replicated many times when datanode is decommissioning |  Major | ec, erasure-coding | HuangTao | HuangTao |
+| [HDFS-14876](https://issues.apache.org/jira/browse/HDFS-14876) | Remove unused imports from TestBlockMissingException.java and TestClose.java |  Minor | test | Lisheng Sun | Lisheng Sun |
+| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-14492](https://issues.apache.org/jira/browse/HDFS-14492) | Snapshot memory leak |  Major | snapshots | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-14418](https://issues.apache.org/jira/browse/HDFS-14418) | Remove redundant super user priveledge checks from namenode. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9862](https://issues.apache.org/jira/browse/YARN-9862) | yarn-services-core test timeout |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14885](https://issues.apache.org/jira/browse/HDFS-14885) | UI: Fix a typo on WebUI of DataNode. |  Trivial | datanode, ui | Xieming Li | Xieming Li |
+| [HADOOP-16619](https://issues.apache.org/jira/browse/HADOOP-16619) | Upgrade jackson and jackson-databind to 2.9.10 |  Major | . | Siyao Meng | Siyao Meng |
+| [HDFS-14216](https://issues.apache.org/jira/browse/HDFS-14216) | NullPointerException happens in NamenodeWebHdfs |  Critical | . | lujie | lujie |
+| [HADOOP-16605](https://issues.apache.org/jira/browse/HADOOP-16605) | NPE in TestAdlSdkConfiguration failing in yetus |  Major | fs/adl | Steve Loughran | Sneha Vijayarajan |
+| [HDFS-14881](https://issues.apache.org/jira/browse/HDFS-14881) | Safemode 'forceExit' option, doesn’t shown in help message |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-14637](https://issues.apache.org/jira/browse/HDFS-14637) | Namenode may not replicate blocks to meet the policy after enabling upgradeDomain |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-14879](https://issues.apache.org/jira/browse/HDFS-14879) | Header was wrong in Snapshot web UI |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down |  Critical | . | Harshakiran Reddy | Ayush Saxena |
+| [HDFS-14859](https://issues.apache.org/jira/browse/HDFS-14859) | Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero |  Major | hdfs | Srinivasu Majeti | Srinivasu Majeti |
+| [YARN-6715](https://issues.apache.org/jira/browse/YARN-6715) | Fix documentation about NodeHealthScriptRunner |  Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko |
+| [YARN-9552](https://issues.apache.org/jira/browse/YARN-9552) | FairScheduler: NODE\_UPDATE can cause NoSuchElementException |  Major | fairscheduler | Peter Bacsko | Peter Bacsko |
+| [HDFS-14754](https://issues.apache.org/jira/browse/HDFS-14754) | Erasure Coding :  The number of Under-Replicated Blocks never reduced |  Critical | ec | hemanthboyina | hemanthboyina |
+| [HDFS-14900](https://issues.apache.org/jira/browse/HDFS-14900) |  Fix build failure of hadoop-hdfs-native-client |  Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider |  Major | . | Shen Yinjie | Erik Krogen |
+| [HDFS-14373](https://issues.apache.org/jira/browse/HDFS-14373) | EC : Decoding is failing when block group last incomplete cell fall in to AlignedStripe |  Critical | ec, hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x |  Blocker | . | Yuxuan Wang | Yuxuan Wang |
+| [HADOOP-13907](https://issues.apache.org/jira/browse/HADOOP-13907) | Fix TestWebDelegationToken#testKerberosDelegationTokenAuthenticator on Windows |  Major | security | Xiaoyu Yao | Kitti Nanasi |
+| [HDFS-14886](https://issues.apache.org/jira/browse/HDFS-14886) | In NameNode Web UI's Startup Progress page, Loading edits always shows 0 sec |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-8453](https://issues.apache.org/jira/browse/YARN-8453) | Additional Unit  tests to verify queue limit and max-limit with multiple resource types |  Major | capacity scheduler | Sunil G | Adam Antal |
+| [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | Setting permissions on name directory fails on non posix compliant filesystems |  Blocker | . | hirik | Siddharth Wagle |
+| [HADOOP-15169](https://issues.apache.org/jira/browse/HADOOP-15169) | "hadoop.ssl.enabled.protocols" should be considered in httpserver2 |  Major | security | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [HDFS-14739](https://issues.apache.org/jira/browse/HDFS-14739) | RBF: LS command for mount point shows wrong owner and permission information. |  Major | . | xuzq | Jinglun |
+| [HDFS-14909](https://issues.apache.org/jira/browse/HDFS-14909) | DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope |  Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-14887](https://issues.apache.org/jira/browse/HDFS-14887) | RBF: In Router Web UI, Observer Namenode Information displaying as Unavailable |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-14847](https://issues.apache.org/jira/browse/HDFS-14847) | Erasure Coding: Blocks are over-replicated while EC decommissioning |  Critical | ec | Fei Hui | Fei Hui |
+| [HDFS-14916](https://issues.apache.org/jira/browse/HDFS-14916) | RBF: line breaks are missing from the output of 'hdfs dfsrouteradmin -ls' |  Minor | rbf, ui | Xieming Li | Xieming Li |
+| [HDFS-14913](https://issues.apache.org/jira/browse/HDFS-14913) | Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType() |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics |  Minor | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart |  Major | datanode | TanYuxin | Xiaoqiao He |
+| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename |  Major | . | Jinglun | Jinglun |
+| [HADOOP-16658](https://issues.apache.org/jira/browse/HADOOP-16658) | S3A connector does not support including the token renewer in the token identifier |  Major | fs/s3 | Philip Zampino | Philip Zampino |
+| [YARN-9921](https://issues.apache.org/jira/browse/YARN-9921) | Issue in PlacementConstraint when YARN Service AM retries allocation on component failure. |  Major | . | Tarun Parimi | Tarun Parimi |
+| [HADOOP-16614](https://issues.apache.org/jira/browse/HADOOP-16614) | Missing leveldbjni package of aarch64 platform |  Major | . | liusheng |  |
+| [HDFS-14910](https://issues.apache.org/jira/browse/HDFS-14910) | Rename Snapshot with Pre Descendants Fail With IllegalArgumentException. |  Blocker | . | Íñigo Goiri | Wei-Chiu Chuang |
+| [HDFS-14933](https://issues.apache.org/jira/browse/HDFS-14933) | Fixing a typo in documentation of Observer NameNode |  Trivial | documentation | Xieming Li | Xieming Li |
+| [HDFS-14308](https://issues.apache.org/jira/browse/HDFS-14308) | DFSStripedInputStream curStripeBuf is not freed by unbuffer() |  Major | ec | Joe McDonnell | Zhao Yi Ming |
+| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width |  Major | . | Eric Badger | Eric Badger |
+| [HDFS-14730](https://issues.apache.org/jira/browse/HDFS-14730) | Remove unused configuration dfs.web.authentication.filter |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-14920](https://issues.apache.org/jira/browse/HDFS-14920) | Erasure Coding: Decommission may hang If one or more datanodes are out of service during decommission |  Major | ec | Fei Hui | Fei Hui |
+| [HDFS-14768](https://issues.apache.org/jira/browse/HDFS-14768) | EC : Busy DN replica should be consider in live replica check. |  Major | datanode, erasure-coding, hdfs, namenode | guojh | guojh |
+| [HDFS-13736](https://issues.apache.org/jira/browse/HDFS-13736) | BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false |  Major | . | hu xiaodong | hu xiaodong |
+| [HDFS-14925](https://issues.apache.org/jira/browse/HDFS-14925) | rename operation should check nest snapshot |  Major | namenode | Junwang Zhao | Junwang Zhao |
+| [YARN-9949](https://issues.apache.org/jira/browse/YARN-9949) | Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14945](https://issues.apache.org/jira/browse/HDFS-14945) | Revise PacketResponder's log. |  Minor | datanode | Xudong Cao | Xudong Cao |
+| [HDFS-14946](https://issues.apache.org/jira/browse/HDFS-14946) | Erasure Coding: Block recovery failed during decommissioning |  Major | . | Fei Hui | Fei Hui |
+| [HDFS-14938](https://issues.apache.org/jira/browse/HDFS-14938) | Add check if excludedNodes contain scope in DFSNetworkTopology#chooseRandomWithStorageType() |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16681](https://issues.apache.org/jira/browse/HADOOP-16681) | mvn javadoc:javadoc fails in hadoop-aws |  Major | documentation | Xieming Li | Xieming Li |
+| [HDFS-14384](https://issues.apache.org/jira/browse/HDFS-14384) | When lastLocatedBlock token expire, it will take 1~3s second to refetch it. |  Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-14880](https://issues.apache.org/jira/browse/HDFS-14880) | Correct the sequence of statistics & exit message in balencer |  Major | balancer & mover | Renukaprasad C | Renukaprasad C |
+| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-14941](https://issues.apache.org/jira/browse/HDFS-14941) | Potential editlog race condition can cause corrupted file |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup |  Minor | hdfs | Jim Brennan | Jim Brennan |
+| [HDFS-14720](https://issues.apache.org/jira/browse/HDFS-14720) | DataNode shouldn't report block as bad block if the block length is Long.MAX\_VALUE. |  Major | datanode | Surendra Singh Lilhore | hemanthboyina |
+| [HDFS-14962](https://issues.apache.org/jira/browse/HDFS-14962) | RBF: ConnectionPool#newConnection() error log wrong protocol class |  Minor | rbf | Yuxuan Wang | Yuxuan Wang |
+| [HADOOP-16701](https://issues.apache.org/jira/browse/HADOOP-16701) | Fix broken links in site index |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-16702](https://issues.apache.org/jira/browse/HADOOP-16702) | Move documentation of hadoop-cos to under src directory. |  Major | tools | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-16665](https://issues.apache.org/jira/browse/HADOOP-16665) | Filesystems to be closed if they failed during initialize() |  Major | fs, fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14922](https://issues.apache.org/jira/browse/HDFS-14922) | Prevent snapshot modification time got change on startup |  Major | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16677](https://issues.apache.org/jira/browse/HADOOP-16677) | Recalculate the remaining timeout millis correctly while throwing an InterupptedException in SocketIOWithTimeout. |  Minor | common | Xudong Cao | Xudong Cao |
+| [HADOOP-16585](https://issues.apache.org/jira/browse/HADOOP-16585) |  [Tool:NNloadGeneratorMR] Multiple threads are using same id for creating file LoadGenerator#write |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs |  Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh |
+| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path |  Minor | fs, test | zhoutai.zt | Xieming Li |
+| [HDFS-14802](https://issues.apache.org/jira/browse/HDFS-14802) | The feature of protect directories should be used in RenameOp |  Major | hdfs | Fei Hui | Fei Hui |
+| [YARN-9982](https://issues.apache.org/jira/browse/YARN-9982) | Fix Container API example link in NodeManager REST API doc |  Trivial | documentation | Charan Hebri | Charan Hebri |
+| [HDFS-14967](https://issues.apache.org/jira/browse/HDFS-14967) | TestWebHDFS  fails in Windows |  Major | . | Renukaprasad C | Renukaprasad C |
+| [YARN-9965](https://issues.apache.org/jira/browse/YARN-9965) | Fix NodeManager failing to start on subsequent times when Hdfs Auxillary Jar is set |  Major | auxservices, nodemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9984](https://issues.apache.org/jira/browse/YARN-9984) | FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node |  Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-9986](https://issues.apache.org/jira/browse/YARN-9986) | signalToContainer REST API does not work even if requested by the app owner |  Major | restapi | kyungwan nam | kyungwan nam |
+| [HDFS-14992](https://issues.apache.org/jira/browse/HDFS-14992) | TestOfflineEditsViewer is failing in Trunk |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-9983](https://issues.apache.org/jira/browse/YARN-9983) | Typo in YARN Service overview documentation |  Trivial | documentation | Denes Gerencser | Denes Gerencser |
+| [HADOOP-16719](https://issues.apache.org/jira/browse/HADOOP-16719) | Remove the disallowed element config within maven-checkstyle-plugin |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later |  Minor | . | xuzq | xuzq |
+| [HADOOP-15686](https://issues.apache.org/jira/browse/HADOOP-15686) | Supress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr |  Major | kms | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-16276](https://issues.apache.org/jira/browse/HADOOP-16276) | Fix jsvc startup command in hadoop-functions.sh due to jsvc \>= 1.0.11 changed default current working directory |  Major | scripts | Siyao Meng | Siyao Meng |
+| [HDFS-14996](https://issues.apache.org/jira/browse/HDFS-14996) | RBF: GetFileStatus fails for directory with EC policy set in case of multiple destinations |  Major | ec, rbf | Ayush Saxena | Ayush Saxena |
+| [HDFS-14940](https://issues.apache.org/jira/browse/HDFS-14940) | HDFS Balancer : Do not allow to set balancer maximum network bandwidth more than 1TB |  Minor | balancer & mover | Souryakanta Dwivedy | hemanthboyina |
+| [HDFS-14924](https://issues.apache.org/jira/browse/HDFS-14924) | RenameSnapshot not updating new modification time |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-13842](https://issues.apache.org/jira/browse/HDFS-13842) | RBF: Exceptions are conflicting when creating the same mount entry twice |  Major | federation | Soumyapn | Ranith Sardar |
+| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue |  Critical | capacity scheduler | jiulongzhu | jiulongzhu |
+| [YARN-9968](https://issues.apache.org/jira/browse/YARN-9968) | Public Localizer is exiting in NodeManager due to NullPointerException |  Major | nodemanager | Tarun Parimi | Tarun Parimi |
+| [YARN-9011](https://issues.apache.org/jira/browse/YARN-9011) | Race condition during decommissioning |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [HADOOP-16685](https://issues.apache.org/jira/browse/HADOOP-16685) | FileSystem#listStatusIterator does not check if given path exists |  Major | fs | Sahil Takiar | Sahil Takiar |
+| [YARN-9290](https://issues.apache.org/jira/browse/YARN-9290) | Invalid SchedulingRequest not rejected in Scheduler PlacementConstraintsHandler |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error |  Critical | . | luhuachao | luhuachao |
+| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws  ConcurrentModificationException |  Major | datanode, performance | Ryan Wu | Aiphago |
+| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure |  Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-14961](https://issues.apache.org/jira/browse/HDFS-14961) | [SBN read] Prevent ZKFC changing Observer Namenode state |  Major | . | Íñigo Goiri | Ayush Saxena |
+| [HDFS-15010](https://issues.apache.org/jira/browse/HDFS-15010) | BlockPoolSlice#addReplicaThreadPool static pool should be initialized by static method |  Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-9990](https://issues.apache.org/jira/browse/YARN-9990) | Testcase fails with "Insufficient configured threads: required=16 \< max=10" |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-15009](https://issues.apache.org/jira/browse/HDFS-15009) | FSCK "-list-corruptfileblocks" return Invalid Entries |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-9695](https://issues.apache.org/jira/browse/HDFS-9695) | HTTPFS - CHECKACCESS operation missing |  Major | . | Bert Hekman | hemanthboyina |
+| [HDFS-15026](https://issues.apache.org/jira/browse/HDFS-15026) | TestPendingReconstruction#testPendingReconstruction() fail in trunk |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16441](https://issues.apache.org/jira/browse/HADOOP-16441) |  if use -Dbundle.openssl=true, bundled with unnecessary libk5crypto.\* |  Major | build | KWON BYUNGCHANG | KWON BYUNGCHANG |
+| [YARN-9969](https://issues.apache.org/jira/browse/YARN-9969) | Improve yarn.scheduler.capacity.queue-mappings documentation |  Major | . | Manikandan R | Manikandan R |
+| [YARN-9938](https://issues.apache.org/jira/browse/YARN-9938) | Validate Parent Queue for QueueMapping contains dynamic group as parent queue |  Major | . | Manikandan R | Manikandan R |
+| [HADOOP-16744](https://issues.apache.org/jira/browse/HADOOP-16744) |  Fix building instruction to enable zstd |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10006](https://issues.apache.org/jira/browse/YARN-10006) | IOException used in place of YARNException in CapacitySheduler |  Minor | capacity scheduler | Prabhu Joseph | Adam Antal |
+| [HDFS-14751](https://issues.apache.org/jira/browse/HDFS-14751) | Synchronize on diffs in DirectoryScanner |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9985](https://issues.apache.org/jira/browse/YARN-9985) | Unsupported "transitionToObserver" option displaying for rmadmin command |  Minor | RM, yarn | Souryakanta Dwivedy | Ayush Saxena |
+| [HDFS-15040](https://issues.apache.org/jira/browse/HDFS-15040) | RBF: Secured Router should not run when SecretManager is not running |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-7721](https://issues.apache.org/jira/browse/YARN-7721) | TestContinuousScheduling fails sporadically with NPE |  Major | fairscheduler | Jason Darrell Lowe | Wilfred Spiegelenburg |
+| [HDFS-15045](https://issues.apache.org/jira/browse/HDFS-15045) | DataStreamer#createBlockOutputStream() should log exception in warn. |  Major | dfsclient | Surendra Singh Lilhore | Ravuri Sushma sree |
+| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop |  Blocker | build | Kevin Su | Kevin Su |
+| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider |  Major | balancer & mover | Erik Krogen | Erik Krogen |
+| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HADOOP-16755](https://issues.apache.org/jira/browse/HADOOP-16755) | Fix broken link in single node cluster setup documentation |  Trivial | documentation | Denes Gerencser | Denes Gerencser |
+| [HDFS-15048](https://issues.apache.org/jira/browse/HDFS-15048) | Fix findbug in DirectoryScanner |  Major | . | Takanobu Asanuma | Masatake Iwasaki |
+| [HADOOP-16765](https://issues.apache.org/jira/browse/HADOOP-16765) | Fix curator dependencies for gradle projects using hadoop-minicluster |  Major | . | Mate Szalay-Beko | Mate Szalay-Beko |
+| [HDFS-14908](https://issues.apache.org/jira/browse/HDFS-14908) | LeaseManager should check parent-child relationship when filter open files. |  Minor | . | Jinglun | Jinglun |
+| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong |  Major | . | Ranith Sardar | Ranith Sardar |
+| [YARN-10020](https://issues.apache.org/jira/browse/YARN-10020) | Fix build instruction of hadoop-yarn-ui |  Minor | yarn-ui-v2 | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-15012](https://issues.apache.org/jira/browse/HDFS-15012) | NN fails to parse Edit logs after applying HDFS-13101 |  Blocker | nn | Eric Lin | Shashikant Banerjee |
+| [YARN-10037](https://issues.apache.org/jira/browse/YARN-10037) | Upgrade build tools for YARN Web UI v2 |  Major | build, security, yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki |
+| [YARN-10042](https://issues.apache.org/jira/browse/YARN-10042) | Uupgrade grpc-xxx depdencies to 1.26.0 |  Major | . | liusheng | liusheng |
+| [HADOOP-16774](https://issues.apache.org/jira/browse/HADOOP-16774) | TestDiskChecker and TestReadWriteDiskValidator fails when run with -Pparallel-tests |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-10054](https://issues.apache.org/jira/browse/YARN-10054) | Upgrade yarn to the latest version in Dockerfile |  Major | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10055](https://issues.apache.org/jira/browse/YARN-10055) | bower install fails |  Blocker | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. |  Major | test | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15073](https://issues.apache.org/jira/browse/HDFS-15073) | Replace curator-shaded guava import with the standard one |  Minor | hdfs-client | Akira Ajisaka | Chandra Sanivarapu |
+| [YARN-10057](https://issues.apache.org/jira/browse/YARN-10057) | Upgrade the dependencies managed by yarnpkg |  Major | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15054](https://issues.apache.org/jira/browse/HDFS-15054) | Delete Snapshot not updating new modification time |  Major | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16042](https://issues.apache.org/jira/browse/HADOOP-16042) | Update the link to HadoopJavaVersion |  Minor | documentation | Akira Ajisaka | Chandra Sanivarapu |
+| [YARN-10041](https://issues.apache.org/jira/browse/YARN-10041) | Should not use AbstractPath to create unix domain socket |  Major | test | zhao bo | liusheng |
+| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 |  Major | . | Takanobu Asanuma | Ayush Saxena |
+| [HDFS-15063](https://issues.apache.org/jira/browse/HDFS-15063) | HttpFS : getFileStatus doesn't return ecPolicy |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-10053](https://issues.apache.org/jira/browse/YARN-10053) | Placement rules do not use correct group service init |  Major | yarn | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [HDFS-15068](https://issues.apache.org/jira/browse/HDFS-15068) | DataNode could meet deadlock if invoke refreshVolumes when register |  Major | datanode | Xiaoqiao He | Aiphago |
+| [HDFS-15089](https://issues.apache.org/jira/browse/HDFS-15089) | RBF: SmallFix for RBFMetrics in doc |  Trivial | documentation, rbf | luhuachao | luhuachao |
+| [MAPREDUCE-7255](https://issues.apache.org/jira/browse/MAPREDUCE-7255) | Fix typo in MapReduce documentaion example |  Trivial | documentation | Sergey Pogorelov | Sergey Pogorelov |
+| [YARN-9956](https://issues.apache.org/jira/browse/YARN-9956) | Improve connection error message for YARN ApiServerClient |  Major | . | Eric Yang | Prabhu Joseph |
+| [HADOOP-16773](https://issues.apache.org/jira/browse/HADOOP-16773) | Fix duplicate assertj-core dependency in hadoop-common module. |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HDFS-15072](https://issues.apache.org/jira/browse/HDFS-15072) | HDFS MiniCluster fails to start when run in directory path with a % |  Minor | . | Geoffrey Jacoby | Masatake Iwasaki |
+| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout |  Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-15080](https://issues.apache.org/jira/browse/HDFS-15080) | Fix the issue in reading persistent memory cached data with an offset |  Major | caching, datanode | Feilong He | Feilong He |
+| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently |  Major | . | Miklos Szegedi | Jim Brennan |
+| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out |  Major | nodemanager | Jason Darrell Lowe | Chandni Singh |
+| [HDFS-14957](https://issues.apache.org/jira/browse/HDFS-14957) | INodeReference Space Consumed was not same in QuotaUsage and ContentSummary |  Major | namenode | hemanthboyina | hemanthboyina |
+| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-15102](https://issues.apache.org/jira/browse/HDFS-15102) | HttpFS: put requests are not supported for path "/" |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-15110](https://issues.apache.org/jira/browse/HDFS-15110) | HttpFS: POST requests are not supported for path "/" |  Major | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank |  Major | conf | Jonathan Turner Eagles | Daryn Sharp |
+| [HDFS-15095](https://issues.apache.org/jira/browse/HDFS-15095) | Fix accidental comment in flaky test TestDecommissioningStatus |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-16590](https://issues.apache.org/jira/browse/HADOOP-16590) | IBM Java has deprecated OS login module classes and OS principal classes. |  Major | security | Nicholas Marion |  |
+| [YARN-10019](https://issues.apache.org/jira/browse/YARN-10019) | container-executor: misc improvements in child processes and exec() calls |  Minor | nodemanager | Peter Bacsko | Peter Bacsko |
+| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode |  Major | namenode | Konstantin Shvachko | Chen Liang |
+| [HDFS-15108](https://issues.apache.org/jira/browse/HDFS-15108) | RBF: MembershipNamenodeResolver should invalidate cache incase of active namenode update |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14578](https://issues.apache.org/jira/browse/HDFS-14578) | AvailableSpaceBlockPlacementPolicy always prefers local node |  Major | block placement | Wei-Chiu Chuang | Ayush Saxena |
+| [YARN-9866](https://issues.apache.org/jira/browse/YARN-9866) | u:user2:%primary\_group is not working as expected |  Major | . | Manikandan R | Manikandan R |
+| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException |  Major | common | Adam Antal | Adam Antal |
+| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing |  Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-8373](https://issues.apache.org/jira/browse/YARN-8373) | RM  Received RMFatalEvent of type CRITICAL\_THREAD\_CRASH |  Major | fairscheduler, resourcemanager | Girish Bhat | Wilfred Spiegelenburg |
+| [YARN-9512](https://issues.apache.org/jira/browse/YARN-9512) | [JDK11] TestAuxServices#testCustomizedAuxServiceClassPath fails because of ClassCastException. |  Major | test | Siyao Meng | Akira Ajisaka |
+| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int |  Major | documentation | zhaoshengjie |  |
+| [YARN-10070](https://issues.apache.org/jira/browse/YARN-10070) | NPE if no rule is defined and application-tag-based-placement is enabled |  Major | . | Kinga Marton | Kinga Marton |
+| [YARN-9970](https://issues.apache.org/jira/browse/YARN-9970) | Refactor TestUserGroupMappingPlacementRule#verifyQueueMapping |  Major | . | Manikandan R | Manikandan R |
+| [YARN-8148](https://issues.apache.org/jira/browse/YARN-8148) | Update decimal values for queue capacities shown on queue status CLI |  Major | client | Prabhu Joseph | Prabhu Joseph |
+| [YARN-10081](https://issues.apache.org/jira/browse/YARN-10081) | Exception message from ClientRMProxy#getRMAddress is misleading |  Trivial | yarn | Adam Antal | Ravuri Sushma sree |
+| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) |  Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15126](https://issues.apache.org/jira/browse/HDFS-15126) | TestDatanodeRegistration#testForcedRegistration fails intermittently |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-9462](https://issues.apache.org/jira/browse/YARN-9462) | TestResourceTrackerService.testNodeRemovalGracefully fails sporadically |  Minor | resourcemanager, test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero |  Major | . | kyungwan nam | kyungwan nam |
+| [HDFS-15128](https://issues.apache.org/jira/browse/HDFS-15128) | Unit test failing to clean testing data and crashed future Maven test run due to failure in TestDataNodeVolumeFailureToleration |  Critical | hdfs, test | Ctest | Ctest |
+| [HDFS-15143](https://issues.apache.org/jira/browse/HDFS-15143) | LocatedStripedBlock returns wrong block type |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14993](https://issues.apache.org/jira/browse/HDFS-14993) | checkDiskError doesn't work during datanode startup |  Major | datanode | Yang Yun | Yang Yun |
+| [HDFS-15145](https://issues.apache.org/jira/browse/HDFS-15145) | HttpFS: getAclStatus() returns permission as null |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently |  Critical | fs | Gabor Bota | Ahmed Hussein |
+| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10107](https://issues.apache.org/jira/browse/YARN-10107) | Invoking NMWebServices#getNMResourceInfo tries to execute gpu discovery binary even if auto discovery is turned off |  Major | . | Szilard Nemeth | Szilard Nemeth |
+| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect |  Major | . | Jason Darrell Lowe | Ahmed Hussein |
+| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. |  Major | hdfs-client | Konstantin Shvachko | Chen Liang |
+| [YARN-9743](https://issues.apache.org/jira/browse/YARN-9743) | [JDK11] TestTimelineWebServices.testContextFactory fails |  Major | timelineservice | Adam Antal | Akira Ajisaka |
+| [HDFS-7175](https://issues.apache.org/jira/browse/HDFS-7175) | Client-side SocketTimeoutException during Fsck |  Major | namenode | Carl Steinbach | Stephen O'Donnell |
+| [HADOOP-16834](https://issues.apache.org/jira/browse/HADOOP-16834) | Replace com.sun.istack.Nullable with javax.annotation.Nullable in DNS.java |  Minor | build | Akira Ajisaka | Xieming Li |
+| [HDFS-15136](https://issues.apache.org/jira/browse/HDFS-15136) | LOG flooding in secure mode when Cookies are not set in request header |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug |  Major | . | wangzhixiang | wangzhixiang |
+| [HDFS-15158](https://issues.apache.org/jira/browse/HDFS-15158) | The number of failed volumes mismatch  with volumeFailures of Datanode metrics |  Minor | datanode | Yang Yun | Yang Yun |
+| [HADOOP-16851](https://issues.apache.org/jira/browse/HADOOP-16851) | unused import in Configuration class |  Trivial | conf | runzhou wu | Jan Hentschel |
+| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only |  Minor | build | Jihyun Cho | Jihyun Cho |
+| [HADOOP-16856](https://issues.apache.org/jira/browse/HADOOP-16856) | cmake is missing in the CentOS 8 section of BUILDING.txt |  Minor | build, documentation | Akira Ajisaka | Masatake Iwasaki |
+| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9521](https://issues.apache.org/jira/browse/YARN-9521) | RM failed to start due to system services |  Major | . | kyungwan nam | kyungwan nam |
+| [YARN-10136](https://issues.apache.org/jira/browse/YARN-10136) | [Router] : Application metrics are hardcode as N/A in UI. |  Major | federation, yarn | Sushanta Sen | Bilwa S T |
+| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException |  Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze |
+| [HDFS-15165](https://issues.apache.org/jira/browse/HDFS-15165) | In Du missed calling getAttributesProvider |  Major | . | Bharat Viswanadham | Bharat Viswanadham |
+| [YARN-10119](https://issues.apache.org/jira/browse/YARN-10119) | Cannot reset the AM failure count for YARN Service |  Major | . | kyungwan nam | kyungwan nam |
+| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) |  Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation |  Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki |
+| [YARN-10147](https://issues.apache.org/jira/browse/YARN-10147) | FPGA plugin can't find the localized aocx file |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes |  Major | namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15182](https://issues.apache.org/jira/browse/HDFS-15182) | TestBlockManager#testOneOfTwoRacksDecommissioned() fail in trunk |  Minor | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog |  Major | . | Konstantin Shvachko | Xieming Li |
+| [HDFS-15187](https://issues.apache.org/jira/browse/HDFS-15187) | CORRUPT replica mismatch between namenodes after failover |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [YARN-10143](https://issues.apache.org/jira/browse/YARN-10143) | YARN-10101 broke Yarn logs CLI |  Blocker | yarn | Adam Antal | Adam Antal |
+| [HADOOP-16841](https://issues.apache.org/jira/browse/HADOOP-16841) | The description of hadoop.http.authentication.signature.secret.file contains outdated information |  Minor | documentation | Akira Ajisaka | Xieming Li |
+| [YARN-8767](https://issues.apache.org/jira/browse/YARN-8767) | TestStreamingStatus fails |  Major | . | Andras Bokor | Andras Bokor |
+| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md |  Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook |
+| [YARN-9593](https://issues.apache.org/jira/browse/YARN-9593) | Updating scheduler conf with comma in config value fails |  Major | . | Anthony Hsu | Tanu Ajmera |
+| [YARN-10141](https://issues.apache.org/jira/browse/YARN-10141) | Interceptor in FederationInterceptorREST doesnt update on RM switchover |  Major | federation, restapi | D M Murali Krishna Reddy | D M Murali Krishna Reddy |
+| [YARN-10152](https://issues.apache.org/jira/browse/YARN-10152) | Fix findbugs warnings in hadoop-yarn-applications-mawo-core module |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15167](https://issues.apache.org/jira/browse/HDFS-15167) | Block Report Interval shouldn't be reset apart from first Block Report |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone |  Minor | . | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14668](https://issues.apache.org/jira/browse/HDFS-14668) | Support Fuse with Users from multiple Security Realms |  Critical | fuse-dfs | Sailesh Patel | Istvan Fajth |
+| [HDFS-15124](https://issues.apache.org/jira/browse/HDFS-15124) | Crashing bugs in NameNode when using a valid configuration for \`dfs.namenode.audit.loggers\` |  Critical | namenode | Ctest | Ctest |
+| [YARN-10155](https://issues.apache.org/jira/browse/YARN-10155) | TestDelegationTokenRenewer.testTokenThreadTimeout fails in trunk |  Major | yarn | Adam Antal | Manikandan R |
+| [HDFS-15111](https://issues.apache.org/jira/browse/HDFS-15111) | stopStandbyServices() should log which service state it is transitioning from. |  Major | hdfs, logging | Konstantin Shvachko | Xieming Li |
+| [HDFS-15199](https://issues.apache.org/jira/browse/HDFS-15199) | NPE in BlockSender |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9452](https://issues.apache.org/jira/browse/YARN-9452) | Fix TestDistributedShell and TestTimelineAuthFilterForV2 failures |  Major | ATSv2, distributed-shell, test | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 |  Blocker | . | Siyao Meng | Siyao Meng |
+| [HDFS-15149](https://issues.apache.org/jira/browse/HDFS-15149) | TestDeadNodeDetection test cases time-out |  Major | datanode | Ahmed Hussein | Lisheng Sun |
+| [HADOOP-16897](https://issues.apache.org/jira/browse/HADOOP-16897) | Sort fields in ReflectionUtils.java |  Minor | test, util | cpugputpu | cpugputpu |
+| [HADOOP-16437](https://issues.apache.org/jira/browse/HADOOP-16437) | Documentation typos: fs.s3a.experimental.fadvise -\> fs.s3a.experimental.input.fadvise |  Major | documentation, fs/s3 | Josh Rosen | Josh Rosen |
+| [HDFS-15204](https://issues.apache.org/jira/browse/HDFS-15204) | TestRetryCacheWithHA testRemoveCacheDescriptor fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14977](https://issues.apache.org/jira/browse/HDFS-14977) | Quota Usage and Content summary are not same in Truncate with Snapshot |  Major | . | hemanthboyina | hemanthboyina |
+| [YARN-10173](https://issues.apache.org/jira/browse/YARN-10173) | Make pid file generation timeout configurable in case of reacquired container |  Minor | yarn | Adam Antal | Adam Antal |
+| [HDFS-15212](https://issues.apache.org/jira/browse/HDFS-15212) | TestEncryptionZones.testVersionAndSuiteNegotiation fails in trunk |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9419](https://issues.apache.org/jira/browse/YARN-9419) | Log a warning if GPU isolation is enabled but LinuxContainerExecutor is disabled |  Major | . | Szilard Nemeth | Andras Gyori |
+| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically |  Major | scheduler, test | Prabhu Joseph | Ahmed Hussein |
+| [HDFS-15135](https://issues.apache.org/jira/browse/HDFS-15135) | EC : ArrayIndexOutOfBoundsException in BlockRecoveryWorker#RecoveryTaskStriped. |  Major | erasure-coding | Surendra Singh Lilhore | Ravuri Sushma sree |
+| [HDFS-14442](https://issues.apache.org/jira/browse/HDFS-14442) | Disagreement between HAUtil.getAddressOfActive and RpcInvocationHandler.getConnectionId |  Major | . | Erik Krogen | Ravuri Sushma sree |
+| [HDFS-14612](https://issues.apache.org/jira/browse/HDFS-14612) | SlowDiskReport won't update when SlowDisks is always empty in heartbeat |  Major | . | Haibin Huang | Haibin Huang |
+| [HDFS-15155](https://issues.apache.org/jira/browse/HDFS-15155) | writeIoRate of DataNodeVolumeMetrics is never used |  Major | hdfs | Haibin Huang | Haibin Huang |
+| [HDFS-15216](https://issues.apache.org/jira/browse/HDFS-15216) | Wrong Use Case of -showprogress in fsck |  Major | . | Ravuri Sushma sree | Ravuri Sushma sree |
+| [HADOOP-16885](https://issues.apache.org/jira/browse/HADOOP-16885) | Encryption zone file copy failure leaks temp file .\_COPYING\_ and wrapped stream |  Major | . | Xiaoyu Yao | Xiaoyu Yao |
+| [HDFS-11396](https://issues.apache.org/jira/browse/HDFS-11396) | TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out |  Minor | namenode, test | John Zhuge | Ayush Saxena |
+| [YARN-10195](https://issues.apache.org/jira/browse/YARN-10195) | Dependency divergence building Timeline Service on HBase 2.2.0 and above |  Major | build, timelineservice | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-14820](https://issues.apache.org/jira/browse/HDFS-14820) |  The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15211](https://issues.apache.org/jira/browse/HDFS-15211) | EC: File write hangs during close in case of Exception during updatePipeline |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14919](https://issues.apache.org/jira/browse/HDFS-14919) | Provide Non DFS Used per DataNode in DataNode UI |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15208](https://issues.apache.org/jira/browse/HDFS-15208) | Suppress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr in hdfs |  Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-10034](https://issues.apache.org/jira/browse/YARN-10034) | Allocation tags are not removed when node decommission |  Major | . | kyungwan nam | kyungwan nam |
+| [HDFS-15223](https://issues.apache.org/jira/browse/HDFS-15223) | FSCK fails if one namenode is not available |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15200](https://issues.apache.org/jira/browse/HDFS-15200) | Delete Corrupt Replica Immediately Irrespective of Replicas On Stale Storage |  Critical | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-15214](https://issues.apache.org/jira/browse/HDFS-15214) | WebHDFS: Add snapshot counts to Content Summary |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-15227](https://issues.apache.org/jira/browse/HDFS-15227) | NPE if the last block changes from COMMITTED to COMPLETE during FSCK |  Major | . | krishna reddy | Ayush Saxena |
+| [YARN-10198](https://issues.apache.org/jira/browse/YARN-10198) | [managedParent].%primary\_group mapping rule doesn't work after YARN-9868 |  Major | capacity scheduler | Peter Bacsko | Peter Bacsko |
+| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature |  Blocker | datanode | Xiaoqiao He | Xiaoqiao He |
+| [HDFS-15232](https://issues.apache.org/jira/browse/HDFS-15232) | Fix libhdfspp test failures with GCC 7 |  Major | native, test | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15201](https://issues.apache.org/jira/browse/HDFS-15201) | SnapshotCounter hits MaxSnapshotID limit |  Major | snapshots | Karthik Palanisamy | Karthik Palanisamy |
+| [HDFS-15219](https://issues.apache.org/jira/browse/HDFS-15219) | DFS Client will stuck when ResponseProcessor.run throw Error |  Major | hdfs-client | zhengchenyu | zhengchenyu |
+| [HDFS-15215](https://issues.apache.org/jira/browse/HDFS-15215) | The Timestamp for longest write/read lock held log is wrong |  Major | . | Toshihiro Suzuki | Toshihiro Suzuki |
+| [HDFS-15234](https://issues.apache.org/jira/browse/HDFS-15234) | Add a default method body for the INodeAttributeProvider#checkPermissionWithContext API |  Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-15193](https://issues.apache.org/jira/browse/HDFS-15193) | Improving the error message for missing \`dfs.namenode.rpc-address.$NAMESERVICE\` |  Minor | hdfs | Ctest | Ctest |
+| [YARN-10202](https://issues.apache.org/jira/browse/YARN-10202) | Fix documentation about NodeAttributes. |  Minor | documentation | Sen Zhao | Sen Zhao |
+| [MAPREDUCE-7268](https://issues.apache.org/jira/browse/MAPREDUCE-7268) | Fix TestMapreduceConfigFields |  Major | mrv2 | Ayush Saxena | Wanqiang Ji |
+| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment |  Critical | build | Akira Ajisaka | Akira Ajisaka |
+| [MAPREDUCE-7269](https://issues.apache.org/jira/browse/MAPREDUCE-7269) | TestNetworkedJob fails |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-14836](https://issues.apache.org/jira/browse/HADOOP-14836) | Upgrade maven-clean-plugin to 3.1.0 |  Major | build | Allen Wittenauer | Akira Ajisaka |
+| [YARN-10207](https://issues.apache.org/jira/browse/YARN-10207) | CLOSE\_WAIT socket connection leaks during rendering of (corrupted) aggregated logs on the JobHistoryServer Web UI |  Major | yarn | Siddharth Ahuja | Siddharth Ahuja |
+| [YARN-10226](https://issues.apache.org/jira/browse/YARN-10226) | NPE in Capacity Scheduler while using %primary\_group queue mapping |  Critical | capacity scheduler | Peter Bacsko | Peter Bacsko |
+| [HDFS-15269](https://issues.apache.org/jira/browse/HDFS-15269) | NameNode should check the authorization API version only once during initialization |  Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-16932](https://issues.apache.org/jira/browse/HADOOP-16932) | distcp copy calls getFileStatus() needlessly and can fail against S3 |  Minor | fs/s3, tools/distcp | Thangamani Murugasamy | Steve Loughran |
+| [YARN-2710](https://issues.apache.org/jira/browse/YARN-2710) | RM HA tests failed intermittently on trunk |  Major | client | Wangda Tan | Ahmed Hussein |
+| [HDFS-12862](https://issues.apache.org/jira/browse/HDFS-12862) | CacheDirective becomes invalid when NN restart or failover |  Major | caching, hdfs | Wang XL | Wang XL |
+| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10219](https://issues.apache.org/jira/browse/YARN-10219) | YARN service placement constraints is broken |  Blocker | . | Eric Yang | Eric Yang |
+| [YARN-10233](https://issues.apache.org/jira/browse/YARN-10233) | [YARN UI2] No Logs were found in "YARN Daemon Logs" page |  Blocker | yarn-ui-v2 | Akhil PB | Akhil PB |
+| [HADOOP-16985](https://issues.apache.org/jira/browse/HADOOP-16985) | Handle release package related issues |  Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-15218](https://issues.apache.org/jira/browse/HDFS-15218) | RBF: MountTableRefresherService failed to refresh other router MountTableEntries in secure mode. |  Major | rbf | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception |  Critical | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16341](https://issues.apache.org/jira/browse/HADOOP-16341) | ShutDownHookManager: Regressed performance on Hook removals after HADOOP-15679 |  Major | common | Gopal Vijayaraghavan | Gopal Vijayaraghavan |
+| [YARN-10223](https://issues.apache.org/jira/browse/YARN-10223) | Duplicate jersey-test-framework-core dependency in yarn-server-common |  Minor | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17007](https://issues.apache.org/jira/browse/HADOOP-17007) | hadoop-cos fails to build |  Major | fs/cos | Wei-Chiu Chuang | Yang Yu |
+| [YARN-9848](https://issues.apache.org/jira/browse/YARN-9848) | revert YARN-4946 |  Blocker | log-aggregation, resourcemanager | Steven Rand | Steven Rand |
+| [HDFS-15286](https://issues.apache.org/jira/browse/HDFS-15286) | Concat on a same files deleting the file |  Critical | . | hemanthboyina | hemanthboyina |
+| [HDFS-15301](https://issues.apache.org/jira/browse/HDFS-15301) | statfs function in hdfs-fuse is not working |  Major | fuse-dfs, libhdfs | Aryan Gupta | Aryan Gupta |
+| [HDFS-15334](https://issues.apache.org/jira/browse/HDFS-15334) | INodeAttributeProvider's new API checkPermissionWithContext not getting called in for authorization |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-15323](https://issues.apache.org/jira/browse/HDFS-15323) | StandbyNode fails transition to active due to insufficient transaction tailing |  Major | namenode, qjm | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-15343](https://issues.apache.org/jira/browse/HDFS-15343) | TestConfiguredFailoverProxyProvider is failing |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. |  Major | . | Jinglun | Jinglun |
+| [YARN-9898](https://issues.apache.org/jira/browse/YARN-9898) | Dependency netty-all-4.1.27.Final doesn't support ARM platform |  Major | . | liusheng | liusheng |
+| [YARN-10265](https://issues.apache.org/jira/browse/YARN-10265) | Upgrade Netty-all dependency to latest version 4.1.50 to fix ARM support issue |  Major | . | liusheng | liusheng |
+| [YARN-9444](https://issues.apache.org/jira/browse/YARN-9444) | YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource |  Minor | api | Gergely Pollak | Gergely Pollak |
+| [HDFS-15038](https://issues.apache.org/jira/browse/HDFS-15038) | TestFsck testFsckListCorruptSnapshotFiles is failing in trunk |  Major | . | hemanthboyina | hemanthboyina |
+| [HDFS-15293](https://issues.apache.org/jira/browse/HDFS-15293) | Relax the condition for accepting a fsimage when receiving a checkpoint |  Critical | namenode | Chen Liang | Chen Liang |
+| [YARN-10228](https://issues.apache.org/jira/browse/YARN-10228) | Yarn Service fails if am java opts contains ZK authentication file path |  Major | . | Bilwa S T | Bilwa S T |
+| [MAPREDUCE-7278](https://issues.apache.org/jira/browse/MAPREDUCE-7278) | Speculative execution behavior is observed even when mapreduce.map.speculative and mapreduce.reduce.speculative are false |  Major | task | Tarun Parimi | Tarun Parimi |
+| [YARN-10314](https://issues.apache.org/jira/browse/YARN-10314) | YarnClient throws NoClassDefFoundError for WebSocketException with only shaded client jars |  Blocker | yarn | Vinayakumar B | Vinayakumar B |
+| [HDFS-15421](https://issues.apache.org/jira/browse/HDFS-15421) | IBR leak causes standby NN to be stuck in safe mode |  Blocker | namenode | Kihwal Lee | Akira Ajisaka |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-8907](https://issues.apache.org/jira/browse/YARN-8907) | Modify a logging message in TestCapacityScheduler |  Trivial | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8904](https://issues.apache.org/jira/browse/YARN-8904) | TestRMDelegationTokens can fail in testRMDTMasterKeyStateOnRollingMasterKey |  Minor | test | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 |  Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [YARN-9263](https://issues.apache.org/jira/browse/YARN-9263) | TestConfigurationNodeAttributesProvider fails after Mockito updated |  Minor | . | Weiwei Yang | Weiwei Yang |
+| [YARN-9315](https://issues.apache.org/jira/browse/YARN-9315) | TestCapacitySchedulerMetrics fails intermittently |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9316](https://issues.apache.org/jira/browse/YARN-9316) | TestPlacementConstraintsUtil#testInterAppConstraintsByAppID fails intermittently |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9324](https://issues.apache.org/jira/browse/YARN-9324) | TestSchedulingRequestContainerAllocation(Async) fails with junit-4.11 |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9338](https://issues.apache.org/jira/browse/YARN-9338) | Timeline related testcases are failing |  Major | . | Prabhu Joseph | Abhishek Modi |
+| [YARN-9299](https://issues.apache.org/jira/browse/YARN-9299) | TestTimelineReaderWhitelistAuthorizationFilter ignores Http Errors |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9404](https://issues.apache.org/jira/browse/YARN-9404) | TestApplicationLifetimeMonitor#testApplicationLifetimeMonitor fails intermittent |  Major | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9405](https://issues.apache.org/jira/browse/YARN-9405) | TestYarnNativeServices#testExpressUpgrade fails intermittent |  Major | yarn-native-services | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9325](https://issues.apache.org/jira/browse/YARN-9325) | TestQueueManagementDynamicEditPolicy fails intermittent |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac |  Minor | libhdfs | John Zhuge | Akira Ajisaka |
+| [HDFS-11949](https://issues.apache.org/jira/browse/HDFS-11949) | Add testcase for ensuring that FsShell cann't move file to the target directory that file exists |  Minor | test | legend | legend |
+| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures |  Major | yarn | Jim Brennan | Jim Brennan |
+| [HDFS-15092](https://issues.apache.org/jira/browse/HDFS-15092) | TestRedudantBlocks#testProcessOverReplicatedAndRedudantBlock sometimes fails |  Minor | test | Fei Hui | Fei Hui |
+| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT |  Minor | yarn | Jim Brennan | Jim Brennan |
+| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature |  Major | fs, test | John Zhuge | Jim Brennan |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-12975](https://issues.apache.org/jira/browse/HDFS-12975) | Changes to the NameNode to support reads from standby |  Major | namenode | Konstantin Shvachko | Chao Sun |
+| [HDFS-12977](https://issues.apache.org/jira/browse/HDFS-12977) | Add stateId to RPC headers. |  Major | ipc, namenode | Konstantin Shvachko | Plamen Jeliazkov |
+| [HDFS-13331](https://issues.apache.org/jira/browse/HDFS-13331) | Add lastSeenStateId to RpcRequestHeader. |  Major | . | Plamen Jeliazkov | Plamen Jeliazkov |
+| [HDFS-11807](https://issues.apache.org/jira/browse/HDFS-11807) | libhdfs++: Get minidfscluster tests running under valgrind |  Major | hdfs-client | James Clampffer | Anatoli Shein |
+| [HDFS-13286](https://issues.apache.org/jira/browse/HDFS-13286) | Add haadmin commands to transition between standby and observer |  Major | ha, hdfs, namenode | Chao Sun | Chao Sun |
+| [HDFS-13578](https://issues.apache.org/jira/browse/HDFS-13578) | Add ReadOnly annotation to methods in ClientProtocol |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-13399](https://issues.apache.org/jira/browse/HDFS-13399) | Make Client field AlignmentContext non-static. |  Major | . | Plamen Jeliazkov | Plamen Jeliazkov |
+| [HDFS-13607](https://issues.apache.org/jira/browse/HDFS-13607) | [Edit Tail Fast Path Pt 1] Enhance JournalNode with an in-memory cache of recent edit transactions |  Major | ha, journal-node | Erik Krogen | Erik Krogen |
+| [HDFS-13608](https://issues.apache.org/jira/browse/HDFS-13608) | [Edit Tail Fast Path Pt 2] Add ability for JournalNode to serve edits via RPC |  Major | . | Erik Krogen | Erik Krogen |
+| [HDFS-13609](https://issues.apache.org/jira/browse/HDFS-13609) | [Edit Tail Fast Path Pt 3] NameNode-side changes to support tailing edits via RPC |  Major | ha, namenode | Erik Krogen | Erik Krogen |
+| [HDFS-13706](https://issues.apache.org/jira/browse/HDFS-13706) | ClientGCIContext should be correctly named ClientGSIContext |  Major | . | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-12976](https://issues.apache.org/jira/browse/HDFS-12976) | Introduce ObserverReadProxyProvider |  Major | hdfs-client | Konstantin Shvachko | Chao Sun |
+| [HDFS-13665](https://issues.apache.org/jira/browse/HDFS-13665) | Move RPC response serialization into Server.doResponse |  Major | . | Plamen Jeliazkov | Plamen Jeliazkov |
+| [HADOOP-15349](https://issues.apache.org/jira/browse/HADOOP-15349) | S3Guard DDB retryBackoff to be more informative on limits exceeded |  Major | fs/s3 | Steve Loughran | Gabor Bota |
+| [HDFS-13610](https://issues.apache.org/jira/browse/HDFS-13610) | [Edit Tail Fast Path Pt 4] Cleanup: integration test, documentation, remove unnecessary dummy sync |  Major | ha, journal-node, namenode | Erik Krogen | Erik Krogen |
+| [HDFS-13150](https://issues.apache.org/jira/browse/HDFS-13150) | [Edit Tail Fast Path] Allow SbNN to tail in-progress edits from JN via RPC |  Major | ha, hdfs, journal-node, namenode | Erik Krogen | Erik Krogen |
+| [HDFS-13688](https://issues.apache.org/jira/browse/HDFS-13688) | Introduce msync API call |  Major | . | Chen Liang | Chen Liang |
+| [HDFS-13789](https://issues.apache.org/jira/browse/HDFS-13789) | Reduce logging frequency of QuorumJournalManager#selectInputStreams |  Trivial | namenode, qjm | Erik Krogen | Erik Krogen |
+| [HDFS-13767](https://issues.apache.org/jira/browse/HDFS-13767) | Add msync server implementation. |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-13851](https://issues.apache.org/jira/browse/HDFS-13851) | Remove AlignmentContext from AbstractNNFailoverProxyProvider |  Major | . | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-13782](https://issues.apache.org/jira/browse/HDFS-13782) | ObserverReadProxyProvider should work with IPFailoverProxyProvider |  Major | test | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-13779](https://issues.apache.org/jira/browse/HDFS-13779) | Implement performFailover logic for ObserverReadProxyProvider. |  Major | hdfs-client | Konstantin Shvachko | Erik Krogen |
+| [HADOOP-15635](https://issues.apache.org/jira/browse/HADOOP-15635) | s3guard set-capacity command to fail fast if bucket is unguarded |  Minor | fs/s3 | Steve Loughran | Gabor Bota |
+| [HADOOP-15750](https://issues.apache.org/jira/browse/HADOOP-15750) | remove obsolete S3A test ITestS3ACredentialsInURL |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-13880](https://issues.apache.org/jira/browse/HDFS-13880) | Add mechanism to allow certain RPC calls to bypass sync |  Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-13778](https://issues.apache.org/jira/browse/HDFS-13778) | TestStateAlignmentContextWithHA should use real ObserverReadProxyProvider instead of AlignmentContextProxyProvider. |  Major | test | Konstantin Shvachko | Plamen Jeliazkov |
+| [HDFS-13749](https://issues.apache.org/jira/browse/HDFS-13749) | Use getServiceStatus to discover observer namenodes |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-13898](https://issues.apache.org/jira/browse/HDFS-13898) | Throw retriable exception for getBlockLocations when ObserverNameNode is in safemode |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-13791](https://issues.apache.org/jira/browse/HDFS-13791) | Limit logging frequency of edit tail related statements |  Major | hdfs, qjm | Erik Krogen | Erik Krogen |
+| [HADOOP-15767](https://issues.apache.org/jira/browse/HADOOP-15767) | [JDK10] Building native package on JDK10 fails due to missing javah |  Major | native | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13944](https://issues.apache.org/jira/browse/HDFS-13944) | [JDK10] Fix javadoc errors in hadoop-hdfs-rbf module |  Major | documentation | Akira Ajisaka | Íñigo Goiri |
+| [HADOOP-15621](https://issues.apache.org/jira/browse/HADOOP-15621) | S3Guard: Implement time-based (TTL) expiry for Authoritative Directory Listing |  Major | fs/s3 | Aaron Fabbri | Gabor Bota |
+| [HDFS-13877](https://issues.apache.org/jira/browse/HDFS-13877) | HttpFS: Implement GETSNAPSHOTDIFF |  Major | httpfs | Siyao Meng | Siyao Meng |
+| [HDFS-13961](https://issues.apache.org/jira/browse/HDFS-13961) | TestObserverNode refactoring |  Major | test | Konstantin Shvachko | Konstantin Shvachko |
+| [YARN-8763](https://issues.apache.org/jira/browse/YARN-8763) | Add WebSocket logic to the Node Manager web server to establish servlet |  Major | . | Zian Chen | Zian Chen |
+| [HADOOP-15775](https://issues.apache.org/jira/browse/HADOOP-15775) | [JDK9] Add missing javax.activation-api dependency |  Critical | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15827](https://issues.apache.org/jira/browse/HADOOP-15827) | NPE in DynamoDBMetadataStore.lambda$listChildren for root + auth S3Guard |  Blocker | fs/s3 | Steve Loughran | Gabor Bota |
+| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor |  Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang |
+| [HADOOP-15825](https://issues.apache.org/jira/browse/HADOOP-15825) | ABFS: Enable some tests for namespace not enabled account using OAuth |  Major | fs/azure | Da Zhou | Da Zhou |
+| [HADOOP-15785](https://issues.apache.org/jira/browse/HADOOP-15785) | [JDK10] Javadoc build fails on JDK 10 in hadoop-common |  Major | build, documentation | Takanobu Asanuma | Dinesh Chitlangia |
+| [YARN-8777](https://issues.apache.org/jira/browse/YARN-8777) | Container Executor C binary change to execute interactive docker command |  Major | . | Zian Chen | Eric Yang |
+| [YARN-5742](https://issues.apache.org/jira/browse/YARN-5742) | Serve aggregated logs of historical apps from timeline service |  Critical | timelineserver | Varun Saxena | Rohith Sharma K S |
+| [YARN-8834](https://issues.apache.org/jira/browse/YARN-8834) | Provide Java client for fetching Yarn specific entities from TimelineReader |  Critical | timelinereader | Rohith Sharma K S | Abhishek Modi |
+| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads |  Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi |
+| [HDFS-13523](https://issues.apache.org/jira/browse/HDFS-13523) | Support observer nodes in MiniDFSCluster |  Major | namenode, test | Erik Krogen | Konstantin Shvachko |
+| [HDFS-13906](https://issues.apache.org/jira/browse/HDFS-13906) | RBF: Add multiple paths for dfsrouteradmin "rm" and "clrquota" commands |  Major | federation | Soumyapn | Ayush Saxena |
+| [HADOOP-15826](https://issues.apache.org/jira/browse/HADOOP-15826) | @Retries annotation of putObject() call & uses wrong |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-8448](https://issues.apache.org/jira/browse/YARN-8448) | AM HTTPS Support for AM communication with RMWeb proxy |  Major | . | Robert Kanter | Robert Kanter |
+| [HDFS-13925](https://issues.apache.org/jira/browse/HDFS-13925) | Unit Test for transitioning between different states |  Major | test | Sherwood Zheng | Sherwood Zheng |
+| [YARN-8582](https://issues.apache.org/jira/browse/YARN-8582) | Document YARN support for HTTPS in AM Web server |  Major | docs | Robert Kanter | Robert Kanter |
+| [YARN-8449](https://issues.apache.org/jira/browse/YARN-8449) | RM HA for AM web server HTTPS Support |  Major | . | Robert Kanter | Robert Kanter |
+| [YARN-8873](https://issues.apache.org/jira/browse/YARN-8873) | [YARN-8811] Add CSI java-based client library |  Major | . | Weiwei Yang | Weiwei Yang |
+| [HDFS-14011](https://issues.apache.org/jira/browse/HDFS-14011) | RBF: Add more information to HdfsFileStatus for a mount point |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-13566](https://issues.apache.org/jira/browse/HDFS-13566) | Add configurable additional RPC listener to NameNode |  Major | ipc | Chen Liang | Chen Liang |
+| [HDFS-13924](https://issues.apache.org/jira/browse/HDFS-13924) | Handle BlockMissingException when reading from observer |  Major | . | Chao Sun | Chao Sun |
+| [HADOOP-14775](https://issues.apache.org/jira/browse/HADOOP-14775) | Change junit dependency in parent pom file to junit 5 while maintaining backward compatibility to junit4. |  Major | build | Ajay Kumar | Akira Ajisaka |
+| [HADOOP-15823](https://issues.apache.org/jira/browse/HADOOP-15823) | ABFS: Stop requiring client ID and tenant ID for MSI |  Major | . | Sean Mackrory | Da Zhou |
+| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-8569](https://issues.apache.org/jira/browse/YARN-8569) | Create an interface to provide cluster information to application |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-13845](https://issues.apache.org/jira/browse/HDFS-13845) | RBF: The default MountTableResolver should fail resolving multi-destination paths |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [YARN-8871](https://issues.apache.org/jira/browse/YARN-8871) | Document behavior of YARN-5742 |  Major | . | Vrushali C | Suma Shivaprasad |
+| [YARN-7754](https://issues.apache.org/jira/browse/YARN-7754) | [Atsv2] Update document for running v1 and v2 TS |  Major | . | Rohith Sharma K S | Suma Shivaprasad |
+| [HDFS-13942](https://issues.apache.org/jira/browse/HDFS-13942) | [JDK10] Fix javadoc errors in hadoop-hdfs module |  Major | documentation | Akira Ajisaka | Dinesh Chitlangia |
+| [HDFS-14033](https://issues.apache.org/jira/browse/HDFS-14033) | [libhdfs++] Disable libhdfs++ build on systems that do not support thread\_local |  Major | . | Anatoli Shein | Anatoli Shein |
+| [HDFS-14016](https://issues.apache.org/jira/browse/HDFS-14016) | ObserverReadProxyProvider should enable observer read by default |  Major | . | Chen Liang | Chen Liang |
+| [HDFS-12026](https://issues.apache.org/jira/browse/HDFS-12026) | libhdfs++: Fix compilation errors and warnings when compiling with Clang |  Blocker | hdfs-client | Anatoli Shein | Anatoli Shein |
+| [HADOOP-15895](https://issues.apache.org/jira/browse/HADOOP-15895) | [JDK9+] Add missing javax.annotation-api dependency to hadoop-yarn-csi |  Major | build | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14024](https://issues.apache.org/jira/browse/HDFS-14024) | RBF: ProvidedCapacityTotal json exception in NamenodeHeartbeatService |  Major | . | CR Hota | CR Hota |
+| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client |  Major | amrmproxy, federation | Botong Huang | Botong Huang |
+| [HDFS-13834](https://issues.apache.org/jira/browse/HDFS-13834) | RBF: Connection creator thread should catch Throwable |  Critical | . | CR Hota | CR Hota |
+| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor |  Minor | . | Bibin Chundatt | Bilwa S T |
+| [HADOOP-15904](https://issues.apache.org/jira/browse/HADOOP-15904) | [JDK 11] Javadoc build failed due to "bad use of '\>'" in hadoop-hdfs |  Major | build | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-15902](https://issues.apache.org/jira/browse/HADOOP-15902) | [JDK 11] Specify the HTML version of Javadoc to 4.01 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14047](https://issues.apache.org/jira/browse/HDFS-14047) | [libhdfs++] Fix hdfsGetLastExceptionRootCause bug in test\_libhdfs\_threaded.c |  Major | libhdfs, native | Anatoli Shein | Anatoli Shein |
+| [HDFS-12284](https://issues.apache.org/jira/browse/HDFS-12284) | RBF: Support for Kerberos authentication |  Major | security | Zhe Zhang | Sherwood Zheng |
+| [YARN-8880](https://issues.apache.org/jira/browse/YARN-8880) | Phase 1 - Add configurations for pluggable plugin framework |  Major | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8988](https://issues.apache.org/jira/browse/YARN-8988) | Reduce the verbose log on RM heartbeat path when distributed node-attributes is enabled |  Major | . | Weiwei Yang | Tao Yang |
+| [YARN-8902](https://issues.apache.org/jira/browse/YARN-8902) | [CSI] Add volume manager that manages CSI volume lifecycle |  Major | . | Weiwei Yang | Weiwei Yang |
+| [YARN-8987](https://issues.apache.org/jira/browse/YARN-8987) | Usability improvements node-attributes CLI |  Critical | . | Weiwei Yang | Bibin Chundatt |
+| [YARN-8877](https://issues.apache.org/jira/browse/YARN-8877) | [CSI] Extend service spec to allow setting resource attributes |  Major | . | Weiwei Yang | Weiwei Yang |
+| [YARN-8776](https://issues.apache.org/jira/browse/YARN-8776) | Container Executor change to create stdin/stdout pipeline |  Major | . | Zian Chen | Eric Yang |
+| [HDFS-13852](https://issues.apache.org/jira/browse/HDFS-13852) | RBF: The DN\_REPORT\_TIME\_OUT and DN\_REPORT\_CACHE\_EXPIRE should be configured in RBFConfigKeys. |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14035](https://issues.apache.org/jira/browse/HDFS-14035) | NN status discovery does not leverage delegation token |  Major | . | Chen Liang | Chen Liang |
+| [HADOOP-15936](https://issues.apache.org/jira/browse/HADOOP-15936) | [JDK 11] MiniDFSClusterManager & MiniHadoopClusterManager compilation fails due to the usage of '\_' as identifier |  Major | build | Devaraj Kavali | Zsolt Venczel |
+| [YARN-8303](https://issues.apache.org/jira/browse/YARN-8303) | YarnClient should contact TimelineReader for application/attempt/container report |  Critical | . | Rohith Sharma K S | Abhishek Modi |
+| [HDFS-14017](https://issues.apache.org/jira/browse/HDFS-14017) | ObserverReadProxyProviderWithIPFailover should work with HA configuration |  Major | . | Chen Liang | Chen Liang |
+| [YARN-8881](https://issues.apache.org/jira/browse/YARN-8881) | [YARN-8851] Add basic pluggable device plugin framework |  Major | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8778](https://issues.apache.org/jira/browse/YARN-8778) | Add Command Line interface to invoke interactive docker shell |  Major | . | Zian Chen | Eric Yang |
+| [YARN-8953](https://issues.apache.org/jira/browse/YARN-8953) | [CSI] CSI driver adaptor module support in NodeManager |  Major | . | Weiwei Yang | Weiwei Yang |
+| [YARN-8838](https://issues.apache.org/jira/browse/YARN-8838) | Add security check for container user is same as websocket user |  Major | nodemanager | Eric Yang | Eric Yang |
+| [HDFS-13776](https://issues.apache.org/jira/browse/HDFS-13776) | RBF: Add Storage policies related ClientProtocol APIs |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
+| [HDFS-14089](https://issues.apache.org/jira/browse/HDFS-14089) | RBF: Failed to specify server's Kerberos pricipal name in NamenodeHeartbeatService |  Minor | . | Ranith Sardar | Ranith Sardar |
+| [HDFS-14067](https://issues.apache.org/jira/browse/HDFS-14067) | Allow manual failover between standby and observer |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-14094](https://issues.apache.org/jira/browse/HDFS-14094) | Fix the order of logging arguments in ObserverReadProxyProvider. |  Major | logging | Konstantin Shvachko | Ayush Saxena |
+| [YARN-9054](https://issues.apache.org/jira/browse/YARN-9054) | Fix FederationStateStoreFacade#buildGetSubClustersCacheRequest |  Major | federation | Bibin Chundatt | Bibin Chundatt |
+| [YARN-9058](https://issues.apache.org/jira/browse/YARN-9058) | [CSI] YARN service fail to launch due to CSI changes |  Blocker | . | Eric Yang | Weiwei Yang |
+| [YARN-8986](https://issues.apache.org/jira/browse/YARN-8986) | publish all exposed ports to random ports when using bridge network |  Minor | yarn | dockerzhang | dockerzhang |
+| [YARN-9034](https://issues.apache.org/jira/browse/YARN-9034) | ApplicationCLI should have option to take clusterId |  Major | . | Rohith Sharma K S | Rohith Sharma K S |
+| [HADOOP-15798](https://issues.apache.org/jira/browse/HADOOP-15798) | LocalMetadataStore put() does not retain isDeleted in parent listing |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HADOOP-15370](https://issues.apache.org/jira/browse/HADOOP-15370) | S3A log message on rm s3a://bucket/ not intuitive |  Trivial | fs/s3 | Steve Loughran | Gabor Bota |
+| [YARN-8882](https://issues.apache.org/jira/browse/YARN-8882) | [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins |  Major | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8989](https://issues.apache.org/jira/browse/YARN-8989) | Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-13713](https://issues.apache.org/jira/browse/HDFS-13713) | Add specification of Multipart Upload API to FS specification, with contract tests |  Blocker | fs, test | Steve Loughran | Ewan Higgs |
+| [HADOOP-14927](https://issues.apache.org/jira/browse/HADOOP-14927) | ITestS3GuardTool failures in testDestroyNoBucket() |  Minor | fs/s3 | Aaron Fabbri | Gabor Bota |
+| [HDFS-13547](https://issues.apache.org/jira/browse/HDFS-13547) | Add ingress port based sasl resolver |  Major | security | Chen Liang | Chen Liang |
+| [HDFS-14120](https://issues.apache.org/jira/browse/HDFS-14120) | ORFPP should also clone DT for the virtual IP |  Major | . | Chen Liang | Chen Liang |
+| [HDFS-14085](https://issues.apache.org/jira/browse/HDFS-14085) | RBF: LS command for root shows wrong owner and permission information. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14114](https://issues.apache.org/jira/browse/HDFS-14114) | RBF: MIN\_ACTIVE\_RATIO should be configurable |  Major | . | Fei Hui | Fei Hui |
+| [YARN-9057](https://issues.apache.org/jira/browse/YARN-9057) | [CSI] CSI jar file should not bundle third party dependencies |  Blocker | build | Eric Yang | Weiwei Yang |
+| [YARN-8914](https://issues.apache.org/jira/browse/YARN-8914) | Add xtermjs to YARN UI2 |  Major | yarn-ui-v2 | Eric Yang | Eric Yang |
+| [HDFS-14131](https://issues.apache.org/jira/browse/HDFS-14131) | Create user guide for "Consistent reads from Observer" feature. |  Major | documentation | Konstantin Shvachko | Chao Sun |
+| [HADOOP-15428](https://issues.apache.org/jira/browse/HADOOP-15428) | s3guard bucket-info will create s3guard table if FS is set to do this automatically |  Major | fs/s3 | Steve Loughran | Gabor Bota |
+| [HADOOP-15988](https://issues.apache.org/jira/browse/HADOOP-15988) | Should be able to set empty directory flag to TRUE in DynamoDBMetadataStore#innerGet when using authoritative directory listings |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HDFS-14142](https://issues.apache.org/jira/browse/HDFS-14142) | Move ipfailover config key out of HdfsClientConfigKeys |  Minor | . | Chen Liang | Chen Liang |
+| [YARN-8885](https://issues.apache.org/jira/browse/YARN-8885) | [DevicePlugin] Support NM APIs to query device resource allocation |  Major | . | Zhankun Tang | Zhankun Tang |
+| [YARN-9015](https://issues.apache.org/jira/browse/YARN-9015) | [DevicePlugin] Add an interface for device plugin to provide customized scheduler |  Major | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8962](https://issues.apache.org/jira/browse/YARN-8962) | Add ability to use interactive shell with normal yarn container |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-13873](https://issues.apache.org/jira/browse/HDFS-13873) | ObserverNode should reject read requests when it is too far behind. |  Major | hdfs-client, namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-14138](https://issues.apache.org/jira/browse/HDFS-14138) | Description errors in the comparison logic of transaction ID |  Minor | . | xiangheng | xiangheng |
+| [HDFS-14146](https://issues.apache.org/jira/browse/HDFS-14146) | Handle exception from internalQueueCall |  Critical | ipc | Chao Sun | Chao Sun |
+| [YARN-9032](https://issues.apache.org/jira/browse/YARN-9032) | Support sh shell for interactive container shell at command line |  Blocker | yarn | Eric Yang | Eric Yang |
+| [YARN-9089](https://issues.apache.org/jira/browse/YARN-9089) | Add Terminal Link to Service component instance page for UI2 |  Major | yarn-ui-v2 | Eric Yang | Eric Yang |
+| [YARN-8963](https://issues.apache.org/jira/browse/YARN-8963) | Add flag to disable interactive shell |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9091](https://issues.apache.org/jira/browse/YARN-9091) | Improve terminal message when connection is refused |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-14152](https://issues.apache.org/jira/browse/HDFS-14152) | RBF: Fix a typo in RouterAdmin usage |  Major | . | Takanobu Asanuma | Ayush Saxena |
+| [HDFS-13869](https://issues.apache.org/jira/browse/HDFS-13869) | RBF: Handle NPE for NamenodeBeanMetrics#getFederationMetrics |  Major | namenode | Surendra Singh Lilhore | Ranith Sardar |
+| [HDFS-14096](https://issues.apache.org/jira/browse/HDFS-14096) | [SPS] : Add Support for Storage Policy Satisfier in ViewFs |  Major | federation | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15969](https://issues.apache.org/jira/browse/HADOOP-15969) | ABFS: getNamespaceEnabled can fail blocking user access thru ACLs |  Major | fs/azure | Da Zhou | Da Zhou |
+| [HADOOP-15972](https://issues.apache.org/jira/browse/HADOOP-15972) | ABFS: reduce list page size to to 500 |  Major | fs/azure | Da Zhou | Da Zhou |
+| [HADOOP-16004](https://issues.apache.org/jira/browse/HADOOP-16004) | ABFS: Convert 404 error response in AbfsInputStream and AbfsOutPutStream to FileNotFoundException |  Major | fs/azure | Da Zhou | Da Zhou |
+| [YARN-9125](https://issues.apache.org/jira/browse/YARN-9125) | Carriage Return character in launch command cause node manager to become unhealthy |  Major | . | Eric Yang | Billie Rinaldi |
+| [HDFS-14116](https://issues.apache.org/jira/browse/HDFS-14116) | Fix class cast error in NNThroughputBenchmark with ObserverReadProxyProvider. |  Major | hdfs-client | Chen Liang | Chao Sun |
+| [HDFS-14149](https://issues.apache.org/jira/browse/HDFS-14149) | Adjust annotations on new interfaces/classes for SBN reads. |  Major | . | Konstantin Shvachko | Chao Sun |
+| [HDFS-14151](https://issues.apache.org/jira/browse/HDFS-14151) | RBF: Make the read-only column of Mount Table clearly understandable |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-9072](https://issues.apache.org/jira/browse/YARN-9072) | Web browser close without proper exit can leak shell process |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9117](https://issues.apache.org/jira/browse/YARN-9117) | Container shell does not work when using yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user is set |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9075](https://issues.apache.org/jira/browse/YARN-9075) | Dynamically add or remove auxiliary services |  Major | nodemanager | Billie Rinaldi | Billie Rinaldi |
+| [HDFS-13443](https://issues.apache.org/jira/browse/HDFS-13443) | RBF: Update mount table cache immediately after changing (add/update/remove) mount table entries. |  Major | fs | Mohammad Arshad | Mohammad Arshad |
+| [YARN-9126](https://issues.apache.org/jira/browse/YARN-9126) | Container reinit always fails in branch-3.2 and trunk |  Major | . | Eric Yang | Chandni Singh |
+| [HDFS-14160](https://issues.apache.org/jira/browse/HDFS-14160) | ObserverReadInvocationHandler should implement RpcInvocationHandler |  Major | . | Konstantin Shvachko | Konstantin Shvachko |
+| [YARN-9129](https://issues.apache.org/jira/browse/YARN-9129) | Ensure flush after printing to log plus additional cleanup |  Major | . | Billie Rinaldi | Eric Yang |
+| [YARN-8523](https://issues.apache.org/jira/browse/YARN-8523) | Interactive docker shell |  Major | . | Eric Yang | Zian Chen |
+| [HADOOP-15935](https://issues.apache.org/jira/browse/HADOOP-15935) | [JDK 11] Update maven.plugin-tools.version to 3.6.0 |  Major | build | Devaraj Kavali | Dinesh Chitlangia |
+| [HDFS-14154](https://issues.apache.org/jira/browse/HDFS-14154) | Document dfs.ha.tail-edits.period in user guide. |  Major | documentation | Chao Sun | Chao Sun |
+| [HADOOP-16015](https://issues.apache.org/jira/browse/HADOOP-16015) | Add bouncycastle jars to hadoop-aws as test dependencies |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-9131](https://issues.apache.org/jira/browse/YARN-9131) | Document usage of Dynamic auxiliary services |  Major | . | Eric Yang | Billie Rinaldi |
+| [YARN-9152](https://issues.apache.org/jira/browse/YARN-9152) | Auxiliary service REST API query does not return running services |  Major | . | Eric Yang | Billie Rinaldi |
+| [YARN-8925](https://issues.apache.org/jira/browse/YARN-8925) | Updating distributed node attributes only when necessary |  Major | resourcemanager | Tao Yang | Tao Yang |
+| [YARN-5168](https://issues.apache.org/jira/browse/YARN-5168) | Add port mapping handling when docker container use bridge network |  Major | . | Jun Gong | Xun Liu |
+| [HDFS-14170](https://issues.apache.org/jira/browse/HDFS-14170) | Fix white spaces related to SBN reads. |  Major | . | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code |  Major | documentation | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15860](https://issues.apache.org/jira/browse/HADOOP-15860) | ABFS: Throw IllegalArgumentException when Directory/File name ends with a period(.) |  Major | fs/azure | Sean Mackrory | Shweta |
+| [HDFS-14167](https://issues.apache.org/jira/browse/HDFS-14167) | RBF: Add stale nodes to federation metrics |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14161](https://issues.apache.org/jira/browse/HDFS-14161) | RBF: Throw StandbyException instead of IOException so that client can retry when can not get connection |  Major | . | Fei Hui | Fei Hui |
+| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore |  Major | fs/oss | wujinhu | wujinhu |
+| [YARN-9147](https://issues.apache.org/jira/browse/YARN-9147) | Auxiliary manifest file deleted from HDFS does not trigger service to be removed |  Major | . | Eric Yang | Billie Rinaldi |
+| [YARN-9038](https://issues.apache.org/jira/browse/YARN-9038) | [CSI] Add ability to publish/unpublish volumes on node managers |  Major | . | Weiwei Yang | Weiwei Yang |
+| [YARN-6149](https://issues.apache.org/jira/browse/YARN-6149) | Allow port range to be specified while starting NM Timeline collector manager. |  Major | timelineserver | Varun Saxena | Abhishek Modi |
+| [YARN-9166](https://issues.apache.org/jira/browse/YARN-9166) | Fix logging for preemption of Opportunistic containers for Guaranteed containers. |  Minor | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-15937](https://issues.apache.org/jira/browse/HADOOP-15937) | [JDK 11] Update maven-shade-plugin.version to 3.2.1 |  Major | build | Devaraj Kavali | Dinesh Chitlangia |
+| [YARN-9169](https://issues.apache.org/jira/browse/YARN-9169) | Add metrics for queued opportunistic and guaranteed containers. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-8822](https://issues.apache.org/jira/browse/YARN-8822) | Nvidia-docker v2 support for YARN GPU feature |  Critical | . | Zhankun Tang | dockerzhang |
+| [YARN-9037](https://issues.apache.org/jira/browse/YARN-9037) | [CSI] Ignore volume resource in resource calculators based on tags |  Major | . | Weiwei Yang | Sunil G |
+| [HDFS-14150](https://issues.apache.org/jira/browse/HDFS-14150) | RBF: Quotas of the sub-cluster should be removed when removing the mount point |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14191](https://issues.apache.org/jira/browse/HDFS-14191) | RBF: Remove hard coded router status from FederationMetrics. |  Major | . | Ranith Sardar | Ranith Sardar |
+| [HADOOP-16027](https://issues.apache.org/jira/browse/HADOOP-16027) | [DOC] Effective use of FS instances during S3A integration tests |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HDFS-13856](https://issues.apache.org/jira/browse/HDFS-13856) | RBF: RouterAdmin should support dfsrouteradmin -refreshRouterArgs command |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HADOOP-16045](https://issues.apache.org/jira/browse/HADOOP-16045) | Don't run TestDU on Windows |  Trivial | common, test | Lukas Majercak | Lukas Majercak |
+| [HADOOP-14556](https://issues.apache.org/jira/browse/HADOOP-14556) | S3A to support Delegation Tokens |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14206](https://issues.apache.org/jira/browse/HDFS-14206) | RBF: Cleanup quota modules |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14129](https://issues.apache.org/jira/browse/HDFS-14129) | RBF: Create new policy provider for router |  Major | namenode | Surendra Singh Lilhore | Ranith Sardar |
+| [HADOOP-15941](https://issues.apache.org/jira/browse/HADOOP-15941) | [JDK 11] Compilation failure: package com.sun.jndi.ldap is not visible |  Major | common | Uma Maheswara Rao G | Takanobu Asanuma |
+| [HDFS-14193](https://issues.apache.org/jira/browse/HDFS-14193) | RBF: Inconsistency with the Default Namespace |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14156](https://issues.apache.org/jira/browse/HDFS-14156) | RBF: rollEdit() command fails with Router |  Major | . | Harshakiran Reddy | Shubham Dewan |
+| [HADOOP-16046](https://issues.apache.org/jira/browse/HADOOP-16046) | [JDK 11] Correct the compiler exclusion of org/apache/hadoop/yarn/webapp/hamlet/\*\* classes for \>= Java 9 |  Major | build | Devaraj Kavali | Devaraj Kavali |
+| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails |  Major | . | Akira Ajisaka | Zsolt Venczel |
+| [YARN-9146](https://issues.apache.org/jira/browse/YARN-9146) | REST API to trigger storing auxiliary manifest file and publish to NMs |  Major | . | Eric Yang | Billie Rinaldi |
+| [YARN-8101](https://issues.apache.org/jira/browse/YARN-8101) | Add UT to verify node-attributes in RM nodes rest API |  Minor | resourcemanager, restapi | Weiwei Yang | Prabhu Joseph |
+| [HDFS-14209](https://issues.apache.org/jira/browse/HDFS-14209) | RBF: setQuota() through router is working for only the mount Points under the Source column in MountTable |  Major | . | Shubham Dewan | Shubham Dewan |
+| [YARN-9116](https://issues.apache.org/jira/browse/YARN-9116) | Capacity Scheduler: implements queue level maximum-allocation inheritance |  Major | capacity scheduler | Aihua Xu | Aihua Xu |
+| [YARN-8867](https://issues.apache.org/jira/browse/YARN-8867) | Retrieve the status of resource localization |  Major | yarn | Chandni Singh | Chandni Singh |
+| [HDFS-14223](https://issues.apache.org/jira/browse/HDFS-14223) | RBF: Add configuration documents for using multiple sub-clusters |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-9221](https://issues.apache.org/jira/browse/YARN-9221) | Add a flag to enable dynamic auxiliary service feature |  Major | . | Eric Yang | Billie Rinaldi |
+| [HDFS-14224](https://issues.apache.org/jira/browse/HDFS-14224) | RBF: NPE in getContentSummary() for getEcPolicy() in case of multiple destinations |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14215](https://issues.apache.org/jira/browse/HDFS-14215) | RBF: Remove dependency on availability of default namespace |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-9074](https://issues.apache.org/jira/browse/YARN-9074) | Docker container rm command should be executed after stop |  Major | . | Zhaohui Xin | Zhaohui Xin |
+| [YARN-9086](https://issues.apache.org/jira/browse/YARN-9086) | [CSI] Run csi-driver-adaptor as aux service |  Major | . | Weiwei Yang | Weiwei Yang |
+| [HADOOP-14178](https://issues.apache.org/jira/browse/HADOOP-14178) | Move Mockito up to version 2.23.4 |  Major | test | Steve Loughran | Akira Ajisaka |
+| [HADOOP-16041](https://issues.apache.org/jira/browse/HADOOP-16041) | UserAgent string for ABFS |  Major | fs/azure | Shweta | Shweta |
+| [HADOOP-15938](https://issues.apache.org/jira/browse/HADOOP-15938) | [JDK 11] Remove animal-sniffer-maven-plugin to fix build |  Major | build | Devaraj Kavali | Dinesh Chitlangia |
+| [HDFS-14225](https://issues.apache.org/jira/browse/HDFS-14225) | RBF : MiniRouterDFSCluster should configure the failover proxy provider for namespace |  Minor | federation | Surendra Singh Lilhore | Ranith Sardar |
+| [YARN-9275](https://issues.apache.org/jira/browse/YARN-9275) | Add link to NodeAttributes doc in PlacementConstraints document |  Minor | documentation | Weiwei Yang | Masatake Iwasaki |
+| [YARN-6735](https://issues.apache.org/jira/browse/YARN-6735) | Have a way to turn off container metrics from NMs |  Major | timelineserver | Vrushali C | Abhishek Modi |
+| [HDFS-14252](https://issues.apache.org/jira/browse/HDFS-14252) | RBF : Exceptions are exposing the actual sub cluster path |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15954](https://issues.apache.org/jira/browse/HADOOP-15954) | ABFS: Enable owner and group conversion for MSI and login user using OAuth |  Major | fs/azure | junhua gu | Da Zhou |
+| [YARN-9253](https://issues.apache.org/jira/browse/YARN-9253) | Add UT to verify Placement Constraint in Distributed Shell |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9252](https://issues.apache.org/jira/browse/YARN-9252) | Allocation Tag Namespace support in Distributed Shell |  Major | distributed-shell | Prabhu Joseph | Prabhu Joseph |
+| [YARN-8555](https://issues.apache.org/jira/browse/YARN-8555) | Parameterize TestSchedulingRequestContainerAllocation(Async) to cover both PC handler options |  Minor | . | Weiwei Yang | Prabhu Joseph |
+| [YARN-996](https://issues.apache.org/jira/browse/YARN-996) | REST API support for node resource configuration |  Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri |
+| [YARN-9229](https://issues.apache.org/jira/browse/YARN-9229) | Document docker registry deployment with NFS Gateway |  Major | . | Eric Yang | Eric Yang |
+| [HADOOP-15364](https://issues.apache.org/jira/browse/HADOOP-15364) | Add support for S3 Select to S3A |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14230](https://issues.apache.org/jira/browse/HDFS-14230) | RBF: Throw RetriableException instead of IOException when no namenodes available |  Major | . | Fei Hui | Fei Hui |
+| [YARN-9184](https://issues.apache.org/jira/browse/YARN-9184) | Docker run doesn't pull down latest image if the image exists locally |  Major | nodemanager | Zhaohui Xin | Zhaohui Xin |
+| [HDFS-13617](https://issues.apache.org/jira/browse/HDFS-13617) | Allow wrapping NN QOP into token in encrypted message |  Major | . | Chen Liang | Chen Liang |
+| [HDFS-13358](https://issues.apache.org/jira/browse/HDFS-13358) | RBF: Support for Delegation Token (RPC) |  Major | . | Sherwood Zheng | CR Hota |
+| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext |  Major | hdfs | Shweta | Shweta |
+| [YARN-9293](https://issues.apache.org/jira/browse/YARN-9293) | Optimize MockAMLauncher event handling |  Major | . | Bibin Chundatt | Bibin Chundatt |
+| [HDFS-14226](https://issues.apache.org/jira/browse/HDFS-14226) | RBF: Setting attributes should set on all subclusters' directories. |  Major | . | Takanobu Asanuma | Ayush Saxena |
+| [HDFS-14268](https://issues.apache.org/jira/browse/HDFS-14268) | RBF: Fix the location of the DNs in getDatanodeReport() |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9060](https://issues.apache.org/jira/browse/YARN-9060) | [YARN-8851] Phase 1 - Support device isolation and use the Nvidia GPU plugin as an example |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HADOOP-15843](https://issues.apache.org/jira/browse/HADOOP-15843) | s3guard bucket-info command to not print a stack trace on bucket-not-found |  Minor | fs/s3 | Steve Loughran | Adam Antal |
+| [HADOOP-16104](https://issues.apache.org/jira/browse/HADOOP-16104) | Wasb tests to downgrade to skip when test a/c is namespace enabled |  Major | fs/azure, test | Steve Loughran | Masatake Iwasaki |
+| [HDFS-14249](https://issues.apache.org/jira/browse/HDFS-14249) | RBF: Tooling to identify the subcluster location of a file |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9258](https://issues.apache.org/jira/browse/YARN-9258) | Support to specify allocation tags without constraint in distributed shell CLI |  Major | distributed-shell | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9156](https://issues.apache.org/jira/browse/YARN-9156) | [YARN-8851] Improve debug message in device plugin method compatibility check of ResourcePluginManager |  Trivial | . | Zhankun Tang | Zhankun Tang |
+| [YARN-8891](https://issues.apache.org/jira/browse/YARN-8891) | Documentation of the pluggable device framework |  Major | documentation | Zhankun Tang | Zhankun Tang |
+| [YARN-9244](https://issues.apache.org/jira/browse/YARN-9244) | Document docker registry deployment with direct S3 driver |  Major | . | Eric Yang | Suma Shivaprasad |
+| [YARN-8821](https://issues.apache.org/jira/browse/YARN-8821) | [YARN-8851] GPU hierarchy/topology scheduling support based on pluggable device framework |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HDFS-14130](https://issues.apache.org/jira/browse/HDFS-14130) | Make ZKFC ObserverNode aware |  Major | ha | Konstantin Shvachko | xiangheng |
+| [YARN-9331](https://issues.apache.org/jira/browse/YARN-9331) | [YARN-8851] Fix a bug that lacking cgroup initialization when bootstrap DeviceResourceHandlerImpl |  Major | . | Zhankun Tang | Zhankun Tang |
+| [HADOOP-16093](https://issues.apache.org/jira/browse/HADOOP-16093) | Move DurationInfo from hadoop-aws to hadoop-common org.apache.hadoop.util |  Minor | fs/s3, util | Steve Loughran | Abhishek Modi |
+| [YARN-8783](https://issues.apache.org/jira/browse/YARN-8783) | Improve the documentation for the docker.trusted.registries configuration |  Major | . | Simon Prewo | Eric Yang |
+| [HADOOP-16136](https://issues.apache.org/jira/browse/HADOOP-16136) | ABFS: Should only transform username to short name |  Major | . | Da Zhou | Da Zhou |
+| [YARN-9245](https://issues.apache.org/jira/browse/YARN-9245) | Add support for Docker Images command |  Major | yarn | Chandni Singh | Chandni Singh |
+| [YARN-5336](https://issues.apache.org/jira/browse/YARN-5336) | Limit the flow name size & consider cleanup for hex chars |  Major | timelineserver | Vrushali C | Sushil Ks |
+| [YARN-3841](https://issues.apache.org/jira/browse/YARN-3841) | [Storage implementation] Adding retry semantics to HDFS backing storage |  Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi |
+| [HADOOP-16068](https://issues.apache.org/jira/browse/HADOOP-16068) | ABFS Authentication and Delegation Token plugins to optionally be bound to specific URI of the store |  Major | fs/azure | Steve Loughran | Steve Loughran |
+| [YARN-7904](https://issues.apache.org/jira/browse/YARN-7904) | Privileged, trusted containers should be supported only in ENTRYPOINT mode |  Major | . | Eric Badger | Eric Yang |
+| [HDFS-14259](https://issues.apache.org/jira/browse/HDFS-14259) | RBF: Fix safemode message for Router |  Major | . | Íñigo Goiri | Ranith Sardar |
+| [HDFS-14329](https://issues.apache.org/jira/browse/HDFS-14329) | RBF: Add maintenance nodes to federation metrics |  Major | . | Ayush Saxena | Ayush Saxena |
+| [YARN-7477](https://issues.apache.org/jira/browse/YARN-7477) | Moving logging APIs over to slf4j in hadoop-yarn-common |  Major | . | Yeliang Cang | Prabhu Joseph |
+| [HDFS-14331](https://issues.apache.org/jira/browse/HDFS-14331) | RBF: IOE While Removing Mount Entry |  Major | rbf | Surendra Singh Lilhore | Ayush Saxena |
+| [HDFS-14335](https://issues.apache.org/jira/browse/HDFS-14335) | RBF: Fix heartbeat typos in the Router. |  Trivial | . | CR Hota | CR Hota |
+| [YARN-7243](https://issues.apache.org/jira/browse/YARN-7243) | Moving logging APIs over to slf4j in hadoop-yarn-server-resourcemanager |  Major | . | Yeliang Cang | Prabhu Joseph |
+| [HDFS-7663](https://issues.apache.org/jira/browse/HDFS-7663) | Erasure Coding: Append on striped file |  Major | . | Jing Zhao | Ayush Saxena |
+| [HADOOP-16163](https://issues.apache.org/jira/browse/HADOOP-16163) | NPE in setup/teardown of ITestAbfsDelegationTokens |  Major | fs/azure, test | Steve Loughran | Steve Loughran |
+| [HDFS-14334](https://issues.apache.org/jira/browse/HDFS-14334) | RBF: Use human readable format for long numbers in the Router UI |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9239](https://issues.apache.org/jira/browse/YARN-9239) | Document docker registry deployment with Ozone CSI driver |  Major | . | Eric Yang | Eric Yang |
+| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 |  Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli |
+| [YARN-9265](https://issues.apache.org/jira/browse/YARN-9265) | FPGA plugin fails to recognize Intel Processing Accelerator Card |  Critical | . | Peter Bacsko | Peter Bacsko |
+| [YARN-8643](https://issues.apache.org/jira/browse/YARN-8643) | Docker image life cycle management on HDFS |  Major | yarn | Eric Yang | Eric Yang |
+| [HDFS-14343](https://issues.apache.org/jira/browse/HDFS-14343) | RBF: Fix renaming folders spread across multiple subclusters |  Major | . | Íñigo Goiri | Ayush Saxena |
+| [YARN-8805](https://issues.apache.org/jira/browse/YARN-8805) | Automatically convert the launch command to the exec form when using entrypoint support |  Major | . | Shane Kumpf | Eric Yang |
+| [HDFS-14270](https://issues.apache.org/jira/browse/HDFS-14270) | [SBN Read] StateId and TrasactionId not present in Trace level logging |  Trivial | namenode | Shweta | Shweta |
+| [YARN-9266](https://issues.apache.org/jira/browse/YARN-9266) | General improvements in IntelFpgaOpenclPlugin |  Major | . | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF |  Blocker | fs/s3 | Dave Christianson | Steve Loughran |
+| [YARN-8376](https://issues.apache.org/jira/browse/YARN-8376) | Separate white list for docker.trusted.registries and docker.privileged-container.registries |  Major | . | Eric Yang | Eric Yang |
+| [HADOOP-15625](https://issues.apache.org/jira/browse/HADOOP-15625) | S3A input stream to use etags/version number to detect changed source files |  Major | fs/s3 | Brahma Reddy Battula | Ben Roling |
+| [HDFS-14354](https://issues.apache.org/jira/browse/HDFS-14354) | Refactor MappableBlock to align with the implementation of SCM cache |  Major | caching, datanode | Feilong He | Feilong He |
+| [YARN-9343](https://issues.apache.org/jira/browse/YARN-9343) | Replace isDebugEnabled with SLF4J parameterized log messages |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16182](https://issues.apache.org/jira/browse/HADOOP-16182) | Update abfs storage back-end with "close" flag when application is done writing to a file |  Major | fs/azure | Vishwajeet Dusane | Vishwajeet Dusane |
+| [YARN-9363](https://issues.apache.org/jira/browse/YARN-9363) | Replace isDebugEnabled with SLF4J parameterized log messages for remaining code |  Minor | yarn | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16124](https://issues.apache.org/jira/browse/HADOOP-16124) | Extend documentation in testing.md about endpoint constants |  Trivial | fs/s3 | Adam Antal | Adam Antal |
+| [YARN-9364](https://issues.apache.org/jira/browse/YARN-9364) | Remove commons-logging dependency from remaining hadoop-yarn |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14351](https://issues.apache.org/jira/browse/HDFS-14351) | RBF: Optimize configuration item resolving for monitor namenode |  Major | rbf | Xiaoqiao He | Xiaoqiao He |
+| [YARN-9387](https://issues.apache.org/jira/browse/YARN-9387) | Update document for ATS HBase Custom tablenames (-entityTableName) |  Critical | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9389](https://issues.apache.org/jira/browse/YARN-9389) | FlowActivity and FlowRun table prefix is wrong |  Minor | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9398](https://issues.apache.org/jira/browse/YARN-9398) | Javadoc error on FPGA related java files |  Major | . | Eric Yang | Peter Bacsko |
+| [YARN-9267](https://issues.apache.org/jira/browse/YARN-9267) | General improvements in FpgaResourceHandlerImpl |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-9402](https://issues.apache.org/jira/browse/YARN-9402) | Opportunistic containers should not be scheduled on Decommissioning nodes. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16201](https://issues.apache.org/jira/browse/HADOOP-16201) | S3AFileSystem#innerMkdirs builds needless lists |  Trivial | fs/s3 | Lokesh Jain | Lokesh Jain |
+| [HDFS-14388](https://issues.apache.org/jira/browse/HDFS-14388) | RBF: Prevent loading metric system when disabled |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9391](https://issues.apache.org/jira/browse/YARN-9391) | Disable PATH variable to be passed to Docker container |  Major | . | Eric Yang | Jim Brennan |
+| [YARN-9268](https://issues.apache.org/jira/browse/YARN-9268) | General improvements in FpgaDevice |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-9269](https://issues.apache.org/jira/browse/YARN-9269) | Minor cleanup in FpgaResourceAllocator |  Minor | . | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16186](https://issues.apache.org/jira/browse/HADOOP-16186) | S3Guard: NPE in DynamoDBMetadataStore.lambda$listChildren |  Major | fs/s3 | Steve Loughran | Gabor Bota |
+| [HADOOP-15999](https://issues.apache.org/jira/browse/HADOOP-15999) | S3Guard: Better support for out-of-band operations |  Major | fs/s3 | Sean Mackrory | Gabor Bota |
+| [HDFS-14393](https://issues.apache.org/jira/browse/HDFS-14393) | Refactor FsDatasetCache for SCM cache implementation |  Major | . | Rakesh Radhakrishnan | Rakesh Radhakrishnan |
+| [HADOOP-16058](https://issues.apache.org/jira/browse/HADOOP-16058) | S3A tests to include Terasort |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-9270](https://issues.apache.org/jira/browse/YARN-9270) | Minor cleanup in TestFpgaDiscoverer |  Minor | . | Peter Bacsko | Peter Bacsko |
+| [YARN-7129](https://issues.apache.org/jira/browse/YARN-7129) | Application Catalog initial project setup and source |  Major | applications | Eric Yang | Eric Yang |
+| [YARN-9348](https://issues.apache.org/jira/browse/YARN-9348) | Build issues on hadoop-yarn-application-catalog-webapp |  Major | . | Eric Yang | Eric Yang |
+| [HDFS-14316](https://issues.apache.org/jira/browse/HDFS-14316) | RBF: Support unavailable subclusters for mount points with multiple destinations |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-14355](https://issues.apache.org/jira/browse/HDFS-14355) | Implement HDFS cache on SCM by using pure java mapped byte buffer |  Major | caching, datanode | Feilong He | Feilong He |
+| [HADOOP-16220](https://issues.apache.org/jira/browse/HADOOP-16220) | Add findbugs ignores for unjustified issues during update to guava to 27.0-jre in hadoop-project |  Major | . | Gabor Bota | Gabor Bota |
+| [YARN-9255](https://issues.apache.org/jira/browse/YARN-9255) | Improve recommend applications order |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9418](https://issues.apache.org/jira/browse/YARN-9418) | ATSV2 /apps/appId/entities/YARN\_CONTAINER rest api does not show metrics |  Critical | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16218](https://issues.apache.org/jira/browse/HADOOP-16218) | findbugs warning of null param to non-nullable method in Configuration with Guava update |  Minor | build | Steve Loughran | Steve Loughran |
+| [YARN-9303](https://issues.apache.org/jira/browse/YARN-9303) | Username splits won't help timelineservice.app\_flow table |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16210](https://issues.apache.org/jira/browse/HADOOP-16210) | Update guava to 27.0-jre in hadoop-project trunk |  Critical | build | Gabor Bota | Gabor Bota |
+| [YARN-9441](https://issues.apache.org/jira/browse/YARN-9441) | Component name should start with Apache Hadoop for consistency |  Minor | . | Weiwei Yang | Weiwei Yang |
+| [HADOOP-16197](https://issues.apache.org/jira/browse/HADOOP-16197) | S3AUtils.translateException to map CredentialInitializationException to AccessDeniedException |  Major | . | Steve Loughran | Steve Loughran |
+| [HDFS-13853](https://issues.apache.org/jira/browse/HDFS-13853) | RBF: RouterAdmin update cmd is overwriting the entry not updating the existing |  Major | . | Dibyendu Karmakar | Ayush Saxena |
+| [YARN-9382](https://issues.apache.org/jira/browse/YARN-9382) | Publish container killed, paused and resumed events to ATSv2. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9335](https://issues.apache.org/jira/browse/YARN-9335) | [atsv2] Restrict the number of elements held in timeline collector when backend is unreachable for async calls |  Major | . | Vrushali C | Abhishek Modi |
+| [YARN-9313](https://issues.apache.org/jira/browse/YARN-9313) | Support asynchronized scheduling mode and multi-node lookup mechanism for scheduler activities |  Major | . | Tao Yang | Tao Yang |
+| [HDFS-14369](https://issues.apache.org/jira/browse/HDFS-14369) | RBF: Fix trailing "/" for webhdfs |  Major | . | CR Hota | Akira Ajisaka |
+| [YARN-999](https://issues.apache.org/jira/browse/YARN-999) | In case of long running tasks, reduce node resource should balloon out resource quickly by calling preemption API and suspending running task. |  Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri |
+| [YARN-9435](https://issues.apache.org/jira/browse/YARN-9435) | Add Opportunistic Scheduler metrics in ResourceManager. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16195](https://issues.apache.org/jira/browse/HADOOP-16195) | S3A MarshalledCredentials.toString() doesn't print full date/time of expiry |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-13699](https://issues.apache.org/jira/browse/HDFS-13699) | Add DFSClient sending handshake token to DataNode, and allow DataNode overwrite downstream QOP |  Major | . | Chen Liang | Chen Liang |
+| [HADOOP-14747](https://issues.apache.org/jira/browse/HADOOP-14747) | S3AInputStream to implement CanUnbuffer |  Major | fs/s3 | Steve Loughran | Sahil Takiar |
+| [HADOOP-16237](https://issues.apache.org/jira/browse/HADOOP-16237) | Fix new findbugs issues after update guava to 27.0-jre in hadoop-project trunk |  Critical | . | Gabor Bota | Gabor Bota |
+| [YARN-9281](https://issues.apache.org/jira/browse/YARN-9281) | Add express upgrade button to Appcatalog UI |  Major | . | Eric Yang | Eric Yang |
+| [YARN-9474](https://issues.apache.org/jira/browse/YARN-9474) | Remove hard coded sleep from Opportunistic Scheduler tests. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9439](https://issues.apache.org/jira/browse/YARN-9439) | Support asynchronized scheduling mode and multi-node lookup mechanism for app activities |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-7848](https://issues.apache.org/jira/browse/YARN-7848) | Force removal of docker containers that do not get removed on first try |  Major | . | Eric Badger | Eric Yang |
+| [YARN-8943](https://issues.apache.org/jira/browse/YARN-8943) | Upgrade JUnit from 4 to 5 in hadoop-yarn-api |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16253](https://issues.apache.org/jira/browse/HADOOP-16253) | Update AssertJ to 3.12.2 |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14422](https://issues.apache.org/jira/browse/HDFS-14422) | RBF: Router shouldn't allow READ operations in safe mode |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-8530](https://issues.apache.org/jira/browse/YARN-8530) | Add security filters to Application catalog |  Major | security, yarn-native-services | Eric Yang | Eric Yang |
+| [YARN-9466](https://issues.apache.org/jira/browse/YARN-9466) | App catalog navigation stylesheet does not display correctly in Safari |  Major | . | Eric Yang | Eric Yang |
+| [HADOOP-16508](https://issues.apache.org/jira/browse/HADOOP-16508) | [hadoop-yarn-project] Fix order of actual and expected expression in assert statements |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9448](https://issues.apache.org/jira/browse/YARN-9448) | Fix Opportunistic Scheduling for node local allocations. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9254](https://issues.apache.org/jira/browse/YARN-9254) | Externalize Solr data storage |  Major | . | Eric Yang | Eric Yang |
+| [YARN-2889](https://issues.apache.org/jira/browse/YARN-2889) | Limit the number of opportunistic container allocated per AM heartbeat |  Major | nodemanager, resourcemanager | Konstantinos Karanasos | Abhishek Modi |
+| [YARN-8551](https://issues.apache.org/jira/browse/YARN-8551) | Build Common module for MaWo application |  Major | . | Yesha Vora | Yesha Vora |
+| [YARN-9475](https://issues.apache.org/jira/browse/YARN-9475) | Create basic VE plugin |  Major | nodemanager | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16252](https://issues.apache.org/jira/browse/HADOOP-16252) | Use configurable dynamo table name prefix in S3Guard tests |  Major | fs/s3 | Ben Roling | Ben Roling |
+| [HDFS-13972](https://issues.apache.org/jira/browse/HDFS-13972) | RBF: Support for Delegation Token (WebHDFS) |  Major | . | Íñigo Goiri | CR Hota |
+| [HADOOP-16222](https://issues.apache.org/jira/browse/HADOOP-16222) | Fix new deprecations after guava 27.0 update in trunk |  Major | . | Gabor Bota | Gabor Bota |
+| [HDFS-14457](https://issues.apache.org/jira/browse/HDFS-14457) | RBF: Add order text SPACE in CLI command 'hdfs dfsrouteradmin' |  Major | rbf | luhuachao | luhuachao |
+| [YARN-9486](https://issues.apache.org/jira/browse/YARN-9486) | Docker container exited with failure does not get clean up correctly |  Major | . | Eric Yang | Eric Yang |
+| [HADOOP-16242](https://issues.apache.org/jira/browse/HADOOP-16242) | ABFS: add bufferpool to AbfsOutputStream |  Major | fs/azure | Da Zhou | Da Zhou |
+| [YARN-9476](https://issues.apache.org/jira/browse/YARN-9476) | Create unit tests for VE plugin |  Major | . | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16221](https://issues.apache.org/jira/browse/HADOOP-16221) | S3Guard: fail write that doesn't update metadata store |  Major | fs/s3 | Ben Roling | Ben Roling |
+| [HDFS-14454](https://issues.apache.org/jira/browse/HDFS-14454) | RBF: getContentSummary() should allow non-existing folders |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-9440](https://issues.apache.org/jira/browse/YARN-9440) | Improve diagnostics for scheduler and app activities |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-16270](https://issues.apache.org/jira/browse/HADOOP-16270) | [JDK 11] Remove unintentional override of the version of Maven Dependency Plugin |  Major | build | Akira Ajisaka | Xieming Li |
+| [HDFS-14401](https://issues.apache.org/jira/browse/HDFS-14401) | Refine the implementation for HDFS cache on SCM |  Major | caching, datanode | Feilong He | Feilong He |
+| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom |  Major | fs/azure | Da Zhou | Da Zhou |
+| [YARN-9489](https://issues.apache.org/jira/browse/YARN-9489) | Support filtering by request-priorities and allocation-request-ids for query results of app activities |  Major | . | Tao Yang | Tao Yang |
+| [YARN-9539](https://issues.apache.org/jira/browse/YARN-9539) | Improve cleanup process of app activities and make some conditions configurable |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HDFS-14426](https://issues.apache.org/jira/browse/HDFS-14426) | RBF: Add delegation token total count as one of the federation metrics |  Major | . | Fengnan Li | Fengnan Li |
+| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-14210](https://issues.apache.org/jira/browse/HDFS-14210) | RBF: ACL commands should work over all the destinations |  Major | . | Shubham Dewan | Ayush Saxena |
+| [HDFS-14490](https://issues.apache.org/jira/browse/HDFS-14490) | RBF: Remove unnecessary quota checks |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16050](https://issues.apache.org/jira/browse/HADOOP-16050) | S3A SSL connections should use OpenSSL |  Major | fs/s3 | Justin Uang | Sahil Takiar |
+| [HDFS-14447](https://issues.apache.org/jira/browse/HDFS-14447) | RBF: Router should support RefreshUserMappingsProtocol |  Major | rbf | Shen Yinjie | Shen Yinjie |
+| [YARN-9505](https://issues.apache.org/jira/browse/YARN-9505) | Add container allocation latency for Opportunistic Scheduler |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16085](https://issues.apache.org/jira/browse/HADOOP-16085) | S3Guard: use object version or etags to protect against inconsistent read after replace/overwrite |  Major | fs/s3 | Ben Roling | Ben Roling |
+| [HDFS-13995](https://issues.apache.org/jira/browse/HDFS-13995) | RBF: Security documentation |  Major | . | CR Hota | CR Hota |
+| [HADOOP-16287](https://issues.apache.org/jira/browse/HADOOP-16287) | KerberosAuthenticationHandler Trusted Proxy Support for Knox |  Major | auth | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14440](https://issues.apache.org/jira/browse/HDFS-14440) | RBF: Optimize the file write process in case of multiple destinations. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14402](https://issues.apache.org/jira/browse/HDFS-14402) | Use FileChannel.transferTo() method for transferring block to SCM cache |  Major | caching, datanode | Feilong He | Feilong He |
+| [YARN-9497](https://issues.apache.org/jira/browse/YARN-9497) | Support grouping by diagnostics for query results of scheduler and app activities |  Major | . | Tao Yang | Tao Yang |
+| [HDFS-13255](https://issues.apache.org/jira/browse/HDFS-13255) | RBF: Fail when try to remove mount point paths |  Major | . | Wu Weiwei | Akira Ajisaka |
+| [HADOOP-16332](https://issues.apache.org/jira/browse/HADOOP-16332) | Remove S3A's depedency on http core |  Critical | build, fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-13909](https://issues.apache.org/jira/browse/HDFS-13909) | RBF: Add Cache pools and directives related ClientProtocol APIs |  Major | . | Dibyendu Karmakar | Ayush Saxena |
+| [YARN-8693](https://issues.apache.org/jira/browse/YARN-8693) | Add signalToContainer REST API for RMWebServices |  Major | restapi | Tao Yang | Tao Yang |
+| [HDFS-14516](https://issues.apache.org/jira/browse/HDFS-14516) | RBF: Create hdfs-rbf-site.xml for RBF specific properties |  Major | rbf | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13787](https://issues.apache.org/jira/browse/HDFS-13787) | RBF: Add Snapshot related ClientProtocol APIs |  Major | federation | Ranith Sardar | Íñigo Goiri |
+| [HADOOP-13656](https://issues.apache.org/jira/browse/HADOOP-13656) | fs -expunge to take a filesystem |  Minor | fs | Steve Loughran | Shweta |
+| [HDFS-14475](https://issues.apache.org/jira/browse/HDFS-14475) | RBF: Expose router security enabled status on the UI |  Major | . | CR Hota | CR Hota |
+| [HDFS-13480](https://issues.apache.org/jira/browse/HDFS-13480) | RBF: Separate namenodeHeartbeat and routerHeartbeat to different config key |  Major | . | maobaolong | Ayush Saxena |
+| [HADOOP-16118](https://issues.apache.org/jira/browse/HADOOP-16118) | S3Guard to support on-demand DDB tables |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14508](https://issues.apache.org/jira/browse/HDFS-14508) | RBF: Clean-up and refactor UI components |  Minor | . | CR Hota | Takanobu Asanuma |
+| [YARN-7537](https://issues.apache.org/jira/browse/YARN-7537) | [Atsv2] load hbase configuration from filesystem rather than URL |  Major | . | Rohith Sharma K S | Prabhu Joseph |
+| [MAPREDUCE-7210](https://issues.apache.org/jira/browse/MAPREDUCE-7210) | Replace \`mapreduce.job.counters.limit\` with \`mapreduce.job.counters.max\` in mapred-default.xml |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-14526](https://issues.apache.org/jira/browse/HDFS-14526) | RBF: Update the document of RBF related metrics |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13404](https://issues.apache.org/jira/browse/HDFS-13404) | RBF: TestRouterWebHDFSContractAppend.testRenameFileBeingAppended fails |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16314](https://issues.apache.org/jira/browse/HADOOP-16314) | Make sure all end point URL is covered by the same AuthenticationFilter |  Major | security | Eric Yang | Prabhu Joseph |
+| [HDFS-14356](https://issues.apache.org/jira/browse/HDFS-14356) | Implement HDFS cache on SCM with native PMDK libs |  Major | caching, datanode | Feilong He | Feilong He |
+| [HADOOP-16117](https://issues.apache.org/jira/browse/HADOOP-16117) | Update AWS SDK to 1.11.563 |  Major | build, fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9590](https://issues.apache.org/jira/browse/YARN-9590) | Correct incompatible, incomplete and redundant activities |  Major | . | Tao Yang | Tao Yang |
+| [MAPREDUCE-6794](https://issues.apache.org/jira/browse/MAPREDUCE-6794) | Remove unused properties from TTConfig.java |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14550](https://issues.apache.org/jira/browse/HDFS-14550) | RBF: Failed to get statistics from NameNodes before 2.9.0 |  Major | . | Akira Ajisaka | Xiaoqiao He |
+| [HADOOP-15563](https://issues.apache.org/jira/browse/HADOOP-15563) | S3Guard to support creating on-demand DDB tables |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14553](https://issues.apache.org/jira/browse/HDFS-14553) | Make queue size of BlockReportProcessingThread configurable |  Major | namenode | Xiaoqiao He | Xiaoqiao He |
+| [MAPREDUCE-7214](https://issues.apache.org/jira/browse/MAPREDUCE-7214) | Remove unused pieces related to \`mapreduce.job.userlog.retain.hours\` |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16354](https://issues.apache.org/jira/browse/HADOOP-16354) | Enable AuthFilter as default for WebHdfs |  Major | security | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16367](https://issues.apache.org/jira/browse/HADOOP-16367) | ApplicationHistoryServer related testcases failing |  Major | security, test | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9578](https://issues.apache.org/jira/browse/YARN-9578) | Add limit/actions/summarize options for app activities REST API |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-16366](https://issues.apache.org/jira/browse/HADOOP-16366) | Fix TimelineReaderServer ignores ProxyUserAuthenticationFilterInitializer |  Major | security | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14545](https://issues.apache.org/jira/browse/HDFS-14545) | RBF: Router should support GetUserMappingsProtocol |  Major | . | Íñigo Goiri | Ayush Saxena |
+| [YARN-8499](https://issues.apache.org/jira/browse/YARN-8499) | ATS v2 Generic TimelineStorageMonitor |  Major | ATSv2 | Sunil G | Prabhu Joseph |
+| [HADOOP-16279](https://issues.apache.org/jira/browse/HADOOP-16279) | S3Guard: Implement time-based (TTL) expiry for entries (and tombstones) |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HADOOP-16376](https://issues.apache.org/jira/browse/HADOOP-16376) | ABFS: Override access() to no-op for now |  Major | fs/azure | Da Zhou | Da Zhou |
+| [YARN-9574](https://issues.apache.org/jira/browse/YARN-9574) | ArtifactId of MaWo application is wrong |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HADOOP-16340](https://issues.apache.org/jira/browse/HADOOP-16340) | ABFS driver continues to retry on IOException responses from REST operations |  Major | fs/azure | Robert Levas | Robert Levas |
+| [HADOOP-16379](https://issues.apache.org/jira/browse/HADOOP-16379) | S3AInputStream#unbuffer should merge input stream stats into fs-wide stats |  Major | fs/s3 | Sahil Takiar | Sahil Takiar |
+| [HADOOP-15183](https://issues.apache.org/jira/browse/HADOOP-15183) | S3Guard store becomes inconsistent after partial failure of rename |  Blocker | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-15658](https://issues.apache.org/jira/browse/HADOOP-15658) | Memory leak in S3AOutputStream |  Major | fs/s3 | Piotr Nowojski | Steve Loughran |
+| [HADOOP-16364](https://issues.apache.org/jira/browse/HADOOP-16364) | S3Guard table destroy to map IllegalArgumentExceptions to IOEs |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-15604](https://issues.apache.org/jira/browse/HADOOP-15604) | Bulk commits of S3A MPUs place needless excessive load on S3 & S3Guard |  Major | fs/s3 | Gabor Bota | Steve Loughran |
+| [HADOOP-16368](https://issues.apache.org/jira/browse/HADOOP-16368) | S3A list operation doesn't pick up etags from results |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9374](https://issues.apache.org/jira/browse/YARN-9374) | HBaseTimelineWriterImpl sync writes has to avoid thread blocking if storage down |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16363](https://issues.apache.org/jira/browse/HADOOP-16363) | S3Guard DDB store prune() doesn't translate AWS exceptions to IOEs |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-9477](https://issues.apache.org/jira/browse/YARN-9477) | Implement VE discovery using libudev |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-6055](https://issues.apache.org/jira/browse/YARN-6055) | ContainersMonitorImpl need be adjusted when NM resource changed. |  Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri |
+| [HDFS-14036](https://issues.apache.org/jira/browse/HDFS-14036) | RBF: Add hdfs-rbf-default.xml to HdfsConfiguration by default |  Major | . | Íñigo Goiri | Takanobu Asanuma |
+| [YARN-9623](https://issues.apache.org/jira/browse/YARN-9623) | Auto adjust max queue length of app activities to make sure activities on all nodes can be covered |  Major | . | Tao Yang | Tao Yang |
+| [YARN-9560](https://issues.apache.org/jira/browse/YARN-9560) | Restructure DockerLinuxContainerRuntime to extend a new OCIContainerRuntime |  Major | . | Eric Badger | Eric Badger |
+| [HDFS-14620](https://issues.apache.org/jira/browse/HDFS-14620) | RBF: Fix 'not a super user' error when disabling a namespace in kerberos with superuser principal |  Major | . | luhuachao | luhuachao |
+| [HDFS-14622](https://issues.apache.org/jira/browse/HDFS-14622) | [Dynamometer] State transition err when CCM( HDFS Centralized Cache Management) feature is used |  Major | tools | TanYuxin | Erik Krogen |
+| [YARN-9660](https://issues.apache.org/jira/browse/YARN-9660) | Enhance documentation of Docker on YARN support |  Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko |
+| [HDFS-14640](https://issues.apache.org/jira/browse/HDFS-14640) | [Dynamometer] Fix TestDynamometerInfra failures |  Major | test, tools | Erik Krogen | Erik Krogen |
+| [HDFS-14410](https://issues.apache.org/jira/browse/HDFS-14410) | Make Dynamometer documentation properly compile onto the Hadoop site |  Major | . | Erik Krogen | Erik Krogen |
+| [HADOOP-16357](https://issues.apache.org/jira/browse/HADOOP-16357) | TeraSort Job failing on S3 DirectoryStagingCommitter: destination path exists |  Minor | fs/s3 | Prabhu Joseph | Steve Loughran |
+| [HDFS-14611](https://issues.apache.org/jira/browse/HDFS-14611) | Move handshake secret field from Token to BlockAccessToken |  Blocker | hdfs | Chen Liang | Chen Liang |
+| [HADOOP-16384](https://issues.apache.org/jira/browse/HADOOP-16384) | S3A: Avoid inconsistencies between DDB and S3 |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16397](https://issues.apache.org/jira/browse/HADOOP-16397) | Hadoop S3Guard Prune command to support a -tombstone option. |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16406](https://issues.apache.org/jira/browse/HADOOP-16406) | ITestDynamoDBMetadataStore.testProvisionTable times out intermittently |  Minor | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HDFS-14458](https://issues.apache.org/jira/browse/HDFS-14458) | Report pmem stats to namenode |  Major | . | Feilong He | Feilong He |
+| [HDFS-14357](https://issues.apache.org/jira/browse/HDFS-14357) | Update documentation for HDFS cache on SCM support |  Major | . | Feilong He | Feilong He |
+| [HDFS-14593](https://issues.apache.org/jira/browse/HDFS-14593) | RBF: Implement deletion feature for expired records in State Store |  Major | rbf | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16383](https://issues.apache.org/jira/browse/HADOOP-16383) | Pass ITtlTimeProvider instance in initialize method in MetadataStore interface |  Major | . | Gabor Bota | Gabor Bota |
+| [HDFS-14653](https://issues.apache.org/jira/browse/HDFS-14653) | RBF: Correct the default value for dfs.federation.router.namenode.heartbeat.enable |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-14577](https://issues.apache.org/jira/browse/HDFS-14577) | RBF: FederationUtil#newInstance should allow constructor without context |  Major | . | CR Hota | CR Hota |
+| [HADOOP-15847](https://issues.apache.org/jira/browse/HADOOP-15847) | S3Guard testConcurrentTableCreations to set r & w capacity == 0 |  Major | fs/s3, test | Steve Loughran | lqjacklee |
+| [HADOOP-16380](https://issues.apache.org/jira/browse/HADOOP-16380) | S3A tombstones can confuse empty directory status |  Blocker | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HADOOP-16447](https://issues.apache.org/jira/browse/HADOOP-16447) | Upgrade JUnit5 from 5.3.1 to 5.5.1 to support global timeout |  Major | test | Akira Ajisaka | Kevin Su |
+| [HDFS-14670](https://issues.apache.org/jira/browse/HDFS-14670) | RBF: Create secret manager instance using FederationUtil#newInstance. |  Major | . | CR Hota | CR Hota |
+| [HDFS-14639](https://issues.apache.org/jira/browse/HDFS-14639) | [Dynamometer] Unnecessary duplicate bin directory appears in dist layout |  Major | namenode, test | Erik Krogen | Erik Krogen |
+| [HADOOP-16472](https://issues.apache.org/jira/browse/HADOOP-16472) | findbugs warning on LocalMetadataStore.ttlTimeProvider sync |  Major | build, fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16275](https://issues.apache.org/jira/browse/HADOOP-16275) | Upgrade Mockito to the latest version |  Minor | test | Akira Ajisaka | Kevin Su |
+| [HADOOP-16479](https://issues.apache.org/jira/browse/HADOOP-16479) | ABFS FileStatus.getModificationTime returns localized time instead of UTC |  Major | fs/azure | Joan Sala Reixach | Bilahari T H |
+| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS |  Major | fs, webhdfs | Erik Krogen | Chao Sun |
+| [HDFS-14700](https://issues.apache.org/jira/browse/HDFS-14700) | Clean up pmem cache before setting pmem cache capacity |  Minor | caching, datanode | Feilong He | Feilong He |
+| [HADOOP-16315](https://issues.apache.org/jira/browse/HADOOP-16315) | ABFS: transform full UPN for named user in AclStatus |  Major | fs/azure | Da Zhou | Da Zhou |
+| [HADOOP-16499](https://issues.apache.org/jira/browse/HADOOP-16499) | S3A retry policy to be exponential |  Critical | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16481](https://issues.apache.org/jira/browse/HADOOP-16481) | ITestS3GuardDDBRootOperations.test\_300\_MetastorePrune needs to set region |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-9694](https://issues.apache.org/jira/browse/YARN-9694) | UI always show default-rack for all the nodes while running SLS. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16509](https://issues.apache.org/jira/browse/HADOOP-16509) | [hadoop-mapreduce-project] Fix order of actual and expected expression in assert statements |  Major | . | Adam Antal | Adam Antal |
+| [HDFS-14717](https://issues.apache.org/jira/browse/HDFS-14717) | Junit not found in hadoop-dynamometer-infra |  Major | . | Kevin Su | Kevin Su |
+| [YARN-9608](https://issues.apache.org/jira/browse/YARN-9608) | DecommissioningNodesWatcher should get lists of running applications on node from RMNode. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16500](https://issues.apache.org/jira/browse/HADOOP-16500) | S3ADelegationTokens to only log at debug on startup |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9683](https://issues.apache.org/jira/browse/YARN-9683) | Remove reapDockerContainerNoPid left behind by YARN-9074 |  Trivial | yarn | Adam Antal | Kevin Su |
+| [HDFS-14713](https://issues.apache.org/jira/browse/HDFS-14713) | RBF: RouterAdmin supports refreshRouterArgs command but not on display |  Major | . | wangzhaohui | wangzhaohui |
+| [YARN-9765](https://issues.apache.org/jira/browse/YARN-9765) | SLS runner crashes when run with metrics turned off. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9752](https://issues.apache.org/jira/browse/YARN-9752) | Add support for allocation id in SLS. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HDFS-14714](https://issues.apache.org/jira/browse/HDFS-14714) | RBF: implement getReplicatedBlockStats interface |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-14756](https://issues.apache.org/jira/browse/HDFS-14756) | RBF: getQuotaUsage may ignore some folders |  Major | . | Chen Zhang | Chen Zhang |
+| [HDFS-14744](https://issues.apache.org/jira/browse/HDFS-14744) | RBF: Non secured routers should not log in error mode when UGI is default. |  Major | . | CR Hota | CR Hota |
+| [HDFS-14763](https://issues.apache.org/jira/browse/HDFS-14763) | Fix package name of audit log class in Dynamometer document |  Major | documentation, tools | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16470](https://issues.apache.org/jira/browse/HADOOP-16470) | Make last AWS credential provider in default auth chain EC2ContainerCredentialsProviderWrapper |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14755](https://issues.apache.org/jira/browse/HDFS-14755) | [Dynamometer] Hadoop-2 DataNode fail to start |  Major | tools | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16469](https://issues.apache.org/jira/browse/HADOOP-16469) | Typo in s3a committers.md doc |  Minor | documentation, fs/s3 | Steve Loughran |  |
+| [HDFS-14674](https://issues.apache.org/jira/browse/HDFS-14674) | [SBN read] Got an unexpected txid when tail editlog |  Blocker | . | wangzhaohui | wangzhaohui |
+| [HDFS-14766](https://issues.apache.org/jira/browse/HDFS-14766) | RBF: MountTableStoreImpl#getMountTableEntries returns extra entry |  Major | . | Chen Zhang | Chen Zhang |
+| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore |  Major | restapi | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9755](https://issues.apache.org/jira/browse/YARN-9755) | RM fails to start with FileSystemBasedConfigurationProvider |  Major | resourcemanager | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14779](https://issues.apache.org/jira/browse/HDFS-14779) | Fix logging error in TestEditLog#testMultiStreamsLoadEditWithConfMaxTxns |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16416](https://issues.apache.org/jira/browse/HADOOP-16416) | mark DynamoDBMetadataStore.deleteTrackingValueMap as final |  Trivial | fs/s3 | Steve Loughran | Kevin Su |
+| [HDFS-8631](https://issues.apache.org/jira/browse/HDFS-8631) | WebHDFS : Support setQuota |  Major | . | nijel | Chao Sun |
+| [YARN-9754](https://issues.apache.org/jira/browse/YARN-9754) | Add support for arbitrary DAG AM Simulator. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9664](https://issues.apache.org/jira/browse/YARN-9664) | Improve response of scheduler/app activities for better understanding |  Major | . | Tao Yang | Tao Yang |
+| [YARN-8678](https://issues.apache.org/jira/browse/YARN-8678) | Queue Management API - rephrase error messages |  Major | . | Akhil PB | Prabhu Joseph |
+| [HDFS-14711](https://issues.apache.org/jira/browse/HDFS-14711) | RBF: RBFMetrics throws NullPointerException if stateStore disabled |  Major | . | Chen Zhang | Chen Zhang |
+| [YARN-9791](https://issues.apache.org/jira/browse/YARN-9791) | Queue Mutation API does not allow to remove a config |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-7982](https://issues.apache.org/jira/browse/YARN-7982) | Do ACLs check while retrieving entity-types per application |  Major | . | Rohith Sharma K S | Prabhu Joseph |
+| [HDFS-14654](https://issues.apache.org/jira/browse/HDFS-14654) | RBF: TestRouterRpc#testNamenodeMetrics is flaky |  Major | . | Takanobu Asanuma | Chen Zhang |
+| [YARN-9804](https://issues.apache.org/jira/browse/YARN-9804) | Update ATSv2 document for latest feature supports |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-14819](https://issues.apache.org/jira/browse/HDFS-14819) | [Dynamometer] Cannot parse audit logs with ‘=‘ in unexpected places when starting a workload. |  Major | . | Soya Miyoshi | Soya Miyoshi |
+| [HDFS-14817](https://issues.apache.org/jira/browse/HDFS-14817) | [Dynamometer] start-dynamometer-cluster.sh shows its usage even if correct arguments are given. |  Major | tools | Soya Miyoshi | Soya Miyoshi |
+| [YARN-9821](https://issues.apache.org/jira/browse/YARN-9821) | NM hangs at serviceStop when ATSV2 Backend Hbase is Down |  Major | ATSv2 | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16430](https://issues.apache.org/jira/browse/HADOOP-16430) | S3AFilesystem.delete to incrementally update s3guard with deletions |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16490](https://issues.apache.org/jira/browse/HADOOP-16490) | Avoid/handle cached 404s during S3A file creation |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9819](https://issues.apache.org/jira/browse/YARN-9819) | Make TestOpportunisticContainerAllocatorAMService more resilient. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16562](https://issues.apache.org/jira/browse/HADOOP-16562) | [pb-upgrade] Update docker image to have 3.7.1 protoc executable |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-9794](https://issues.apache.org/jira/browse/YARN-9794) | RM crashes due to runtime errors in TimelineServiceV2Publisher |  Major | . | Tarun Parimi | Tarun Parimi |
+| [HADOOP-16371](https://issues.apache.org/jira/browse/HADOOP-16371) | Option to disable GCM for SSL connections when running on Java 8 |  Major | fs/s3 | Sahil Takiar | Sahil Takiar |
+| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id |  Major | hdfs | Chen Liang | Chen Liang |
+| [HADOOP-16557](https://issues.apache.org/jira/browse/HADOOP-16557) | [pb-upgrade] Upgrade protobuf.version to 3.7.1 |  Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-14833](https://issues.apache.org/jira/browse/HDFS-14833) | RBF: Router Update Doesn't Sync Quota |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16589](https://issues.apache.org/jira/browse/HADOOP-16589) | [pb-upgrade] Update docker image to make 3.7.1 protoc as default |  Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-14818](https://issues.apache.org/jira/browse/HDFS-14818) | Check native pmdk lib by 'hadoop checknative' command |  Minor | native | Feilong He | Feilong He |
+| [HADOOP-16558](https://issues.apache.org/jira/browse/HADOOP-16558) | [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes |  Major | common | Vinayakumar B | Vinayakumar B |
+| [HADOOP-16565](https://issues.apache.org/jira/browse/HADOOP-16565) | Region must be provided when requesting session credentials or SdkClientException will be thrown |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HADOOP-16138](https://issues.apache.org/jira/browse/HADOOP-16138) | hadoop fs mkdir / of nonexistent abfs container raises NPE |  Minor | fs/azure | Steve Loughran | Gabor Bota |
+| [HADOOP-16591](https://issues.apache.org/jira/browse/HADOOP-16591) | S3A ITest\*MRjob failures |  Major | fs/s3 | Siddharth Seth | Siddharth Seth |
+| [HADOOP-16560](https://issues.apache.org/jira/browse/HADOOP-16560) | [YARN] use protobuf-maven-plugin to generate protobuf classes |  Major | . | Vinayakumar B | Duo Zhang |
+| [HADOOP-16561](https://issues.apache.org/jira/browse/HADOOP-16561) | [MAPREDUCE] use protobuf-maven-plugin to generate protobuf classes |  Major | . | Vinayakumar B | Duo Zhang |
+| [HDFS-14461](https://issues.apache.org/jira/browse/HDFS-14461) | RBF: Fix intermittently failing kerberos related unit test |  Major | . | CR Hota | Xiaoqiao He |
+| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive |  Major | hdfs | Chen Liang | Chen Liang |
+| [YARN-9859](https://issues.apache.org/jira/browse/YARN-9859) | Refactor OpportunisticContainerAllocator |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9864](https://issues.apache.org/jira/browse/YARN-9864) | Format CS Configuration present in Configuration Store |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9801](https://issues.apache.org/jira/browse/YARN-9801) | SchedConfCli does not work with https mode |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16458](https://issues.apache.org/jira/browse/HADOOP-16458) | LocatedFileStatusFetcher scans failing intermittently against S3 store |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16578](https://issues.apache.org/jira/browse/HADOOP-16578) | ABFS: fileSystemExists() should not call container level apis |  Major | fs/azure | Da Zhou | Sneha Vijayarajan |
+| [YARN-9870](https://issues.apache.org/jira/browse/YARN-9870) | Remove unused function from OpportunisticContainerAllocatorAMService |  Minor | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9792](https://issues.apache.org/jira/browse/YARN-9792) | Document examples of SchedulerConf with Node Labels |  Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-13373](https://issues.apache.org/jira/browse/HADOOP-13373) | Add S3A implementation of FSMainOperationsBaseTest |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode |  Major | hdfs | Chen Liang | Chen Liang |
+| [HADOOP-16620](https://issues.apache.org/jira/browse/HADOOP-16620) | [pb-upgrade] Remove protocol buffers 3.7.1 from requirements in BUILDING.txt |  Minor | build, documentation | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15729](https://issues.apache.org/jira/browse/HADOOP-15729) | [s3a] stop treat fs.s3a.max.threads as the long-term minimum |  Major | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. |  Major | hdfs | Konstantin Shvachko | Chen Liang |
+| [YARN-9782](https://issues.apache.org/jira/browse/YARN-9782) | Avoid DNS resolution while running SLS. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16207](https://issues.apache.org/jira/browse/HADOOP-16207) | Improved S3A MR tests |  Critical | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HADOOP-16570](https://issues.apache.org/jira/browse/HADOOP-16570) | S3A committers leak threads/raises OOM on job/task commit at scale |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16626](https://issues.apache.org/jira/browse/HADOOP-16626) | S3A ITestRestrictedReadAccess fails |  Major | fs/s3 | Siddharth Seth | Steve Loughran |
+| [HADOOP-16512](https://issues.apache.org/jira/browse/HADOOP-16512) | [hadoop-tools] Fix order of actual and expected expression in assert statements |  Major | . | Adam Antal | Kevin Su |
+| [HADOOP-16587](https://issues.apache.org/jira/browse/HADOOP-16587) | Make AAD endpoint configurable on all Auth flows |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [YARN-9873](https://issues.apache.org/jira/browse/YARN-9873) | Mutation API Config Change need to update Version Number |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode |  Major | . | Konstantin Shvachko | Erik Krogen |
+| [HADOOP-16650](https://issues.apache.org/jira/browse/HADOOP-16650) | ITestS3AClosedFS failing -junit test thread |  Blocker | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-9699](https://issues.apache.org/jira/browse/YARN-9699) | Migration tool that help to generate CS config based on FS config [Phase 1] |  Major | . | Wanqiang Ji | Peter Bacsko |
+| [HADOOP-16635](https://issues.apache.org/jira/browse/HADOOP-16635) | S3A innerGetFileStatus s"directories only" scan still does a HEAD |  Blocker | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16634](https://issues.apache.org/jira/browse/HADOOP-16634) | S3A ITest failures without S3Guard |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-9840](https://issues.apache.org/jira/browse/YARN-9840) | Capacity scheduler: add support for Secondary Group rule mapping |  Major | capacity scheduler | Peter Bacsko | Manikandan R |
+| [HADOOP-16651](https://issues.apache.org/jira/browse/HADOOP-16651) | S3 getBucketLocation() can return "US" for us-east |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16478](https://issues.apache.org/jira/browse/HADOOP-16478) | S3Guard bucket-info fails if the bucket location is denied to the caller |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources |  Major | . | Manikandan R | Manikandan R |
+| [YARN-9841](https://issues.apache.org/jira/browse/YARN-9841) | Capacity scheduler: add support for combined %user + %primary\_group mapping |  Major | capacity scheduler | Peter Bacsko | Manikandan R |
+| [YARN-9884](https://issues.apache.org/jira/browse/YARN-9884) | Make container-executor mount logic modular |  Major | . | Eric Badger | Eric Badger |
+| [YARN-9875](https://issues.apache.org/jira/browse/YARN-9875) | FSSchedulerConfigurationStore fails to update with hdfs path |  Major | capacityscheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14638](https://issues.apache.org/jira/browse/HDFS-14638) | [Dynamometer] Fix scripts to refer to current build structure |  Major | namenode, test | Erik Krogen | Takanobu Asanuma |
+| [HDFS-14907](https://issues.apache.org/jira/browse/HDFS-14907) | [Dynamometer] DataNode can't find junit jar when using Hadoop-3 binary |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-14824](https://issues.apache.org/jira/browse/HDFS-14824) | [Dynamometer] Dynamometer in org.apache.hadoop.tools does not output the benchmark results. |  Major | . | Soya Miyoshi | Takanobu Asanuma |
+| [HADOOP-16510](https://issues.apache.org/jira/browse/HADOOP-16510) | [hadoop-common] Fix order of actual and expected expression in assert statements |  Major | . | Adam Antal | Adam Antal |
+| [YARN-9950](https://issues.apache.org/jira/browse/YARN-9950) | Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HDFS-14891](https://issues.apache.org/jira/browse/HDFS-14891) | RBF: namenode links in NameFederation Health page (federationhealth.html)  cannot use https scheme |  Major | rbf, ui | Xieming Li | Xieming Li |
+| [YARN-9865](https://issues.apache.org/jira/browse/YARN-9865) | Capacity scheduler: add support for combined %user + %secondary\_group mapping |  Major | . | Manikandan R | Manikandan R |
+| [YARN-9697](https://issues.apache.org/jira/browse/YARN-9697) | Efficient allocation of Opportunistic containers. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HADOOP-16477](https://issues.apache.org/jira/browse/HADOOP-16477) | S3A delegation token tests fail if fs.s3a.encryption.key set |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HDFS-14648](https://issues.apache.org/jira/browse/HDFS-14648) | Implement DeadNodeDetector basic model |  Major | . | Lisheng Sun | Lisheng Sun |
+| [YARN-9900](https://issues.apache.org/jira/browse/YARN-9900) | Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16484](https://issues.apache.org/jira/browse/HADOOP-16484) | S3A to warn or fail if S3Guard is disabled |  Minor | fs/s3 | Steve Loughran | Gabor Bota |
+| [YARN-9562](https://issues.apache.org/jira/browse/YARN-9562) | Add Java changes for the new RuncContainerRuntime |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-16657](https://issues.apache.org/jira/browse/HADOOP-16657) | Move remaining log4j APIs over to slf4j in hadoop-common. |  Major | . | Minni Mittal | Minni Mittal |
+| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit |  Major | build | Duo Zhang | Duo Zhang |
+| [YARN-9909](https://issues.apache.org/jira/browse/YARN-9909) | Offline format of YarnConfigurationStore |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16632](https://issues.apache.org/jira/browse/HADOOP-16632) | Speculating & Partitioned S3A magic committers can leave pending files under \_\_magic |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-9836](https://issues.apache.org/jira/browse/YARN-9836) | General usability improvements in showSimulationTrace.html |  Minor | scheduler-load-simulator | Adam Antal | Adam Antal |
+| [HADOOP-16707](https://issues.apache.org/jira/browse/HADOOP-16707) | NPE in UGI.getCurrentUser in ITestAbfsIdentityTransformer setup |  Major | auth, fs/azure, security | Steve Loughran | Steve Loughran |
+| [HADOOP-16687](https://issues.apache.org/jira/browse/HADOOP-16687) | ABFS: Fix testcase added for HADOOP-16138 for namespace enabled account |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HDFS-14651](https://issues.apache.org/jira/browse/HDFS-14651) | DeadNodeDetector checks dead node periodically |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16708](https://issues.apache.org/jira/browse/HADOOP-16708) | HadoopExecutors cleanup to only log at debug |  Minor | util | Steve Loughran | David Mollitor |
+| [YARN-9899](https://issues.apache.org/jira/browse/YARN-9899) | Migration tool that help to generate CS config based on FS config [Phase 2] |  Major | . | Szilard Nemeth | Peter Bacsko |
+| [HDFS-14649](https://issues.apache.org/jira/browse/HDFS-14649) | Add suspect probe for DeadNodeDetector |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HADOOP-16455](https://issues.apache.org/jira/browse/HADOOP-16455) | ABFS: Implement FileSystem.access() method |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [HADOOP-16660](https://issues.apache.org/jira/browse/HADOOP-16660) | ABFS: Make RetryCount in ExponentialRetryPolicy Configurable |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HDFS-15019](https://issues.apache.org/jira/browse/HDFS-15019) | Refactor the unit test of TestDeadNodeDetection |  Minor | . | Yiqun Lin | Lisheng Sun |
+| [HDFS-14825](https://issues.apache.org/jira/browse/HDFS-14825) | [Dynamometer] Workload doesn't start unless an absolute path of Mapper class given |  Major | . | Soya Miyoshi | Takanobu Asanuma |
+| [HDFS-13811](https://issues.apache.org/jira/browse/HDFS-13811) | RBF: Race condition between router admin quota update and periodic quota update service |  Major | . | Dibyendu Karmakar | Jinglun |
+| [YARN-9781](https://issues.apache.org/jira/browse/YARN-9781) | SchedConfCli to get current stored scheduler configuration |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9780](https://issues.apache.org/jira/browse/YARN-9780) | SchedulerConf Mutation API does not Allow Stop and Remove Queue in a single call |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9789](https://issues.apache.org/jira/browse/YARN-9789) | Disable Option for Write Ahead Logs of LogMutation |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-9607](https://issues.apache.org/jira/browse/YARN-9607) | Auto-configuring rollover-size of IFile format for non-appendable filesystems |  Major | log-aggregation, yarn | Adam Antal | Adam Antal |
+| [YARN-9561](https://issues.apache.org/jira/browse/YARN-9561) | Add C changes for the new RuncContainerRuntime |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-16612](https://issues.apache.org/jira/browse/HADOOP-16612) | Track Azure Blob File System client-perceived latency |  Major | fs/azure, hdfs-client | Jeetesh Mangwani | Jeetesh Mangwani |
+| [HDFS-15043](https://issues.apache.org/jira/browse/HDFS-15043) | RBF: The detail of the Exception is not shown in ZKDelegationTokenSecretManagerImpl |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml |  Minor | fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HDFS-14983](https://issues.apache.org/jira/browse/HDFS-14983) | RBF: Add dfsrouteradmin -refreshSuperUserGroupsConfiguration command option |  Minor | rbf | Akira Ajisaka | Xieming Li |
+| [HDFS-15044](https://issues.apache.org/jira/browse/HDFS-15044) | [Dynamometer] Show the line of audit log when parsing it unsuccessfully |  Major | tools | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-16757](https://issues.apache.org/jira/browse/HADOOP-16757) | Increase timeout unit test rule for MetadataStoreTestBase |  Major | fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HADOOP-16764](https://issues.apache.org/jira/browse/HADOOP-16764) | Rewrite Python example codes using Python3 |  Minor | documentation | Kengo Seki | Kengo Seki |
+| [HADOOP-16751](https://issues.apache.org/jira/browse/HADOOP-16751) | DurationInfo text parsing/formatting should be moved out of hotpath |  Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan |
+| [YARN-10035](https://issues.apache.org/jira/browse/YARN-10035) | Add ability to filter the Cluster Applications API request by name |  Major | yarn | Adam Antal | Adam Antal |
+| [HDFS-15066](https://issues.apache.org/jira/browse/HDFS-15066) | HttpFS: Implement setErasureCodingPolicy , unsetErasureCodingPolicy , getErasureCodingPolicy |  Major | . | hemanthboyina | hemanthboyina |
+| [HADOOP-16645](https://issues.apache.org/jira/browse/HADOOP-16645) | S3A Delegation Token extension point to use StoreContext |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16699](https://issues.apache.org/jira/browse/HADOOP-16699) | ABFS: Enhance driver debug logs |  Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan |
+| [YARN-10068](https://issues.apache.org/jira/browse/YARN-10068) | TimelineV2Client may leak file descriptors creating ClientResponse objects. |  Critical | ATSv2 | Anand Srinivasan | Anand Srinivasan |
+| [HADOOP-16642](https://issues.apache.org/jira/browse/HADOOP-16642) | ITestDynamoDBMetadataStoreScale fails when throttled. |  Major | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HDFS-15100](https://issues.apache.org/jira/browse/HDFS-15100) | RBF: Print stacktrace when DFSRouter fails to fetch/parse JMX output from NameNode |  Major | rbf | Akira Ajisaka | Akira Ajisaka |
+| [YARN-10071](https://issues.apache.org/jira/browse/YARN-10071) | Sync Mockito version with other modules |  Major | build, test | Akira Ajisaka | Adam Antal |
+| [HADOOP-16697](https://issues.apache.org/jira/browse/HADOOP-16697) | audit/tune s3a authoritative flag in s3guard DDB Table |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-10067](https://issues.apache.org/jira/browse/YARN-10067) | Add dry-run feature to FS-CS converter tool |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10026](https://issues.apache.org/jira/browse/YARN-10026) | Pull out common code pieces from ATS v1.5 and v2 |  Major | ATSv2, yarn | Adam Antal | Adam Antal |
+| [HADOOP-16797](https://issues.apache.org/jira/browse/HADOOP-16797) | Add dockerfile for ARM builds |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-9788](https://issues.apache.org/jira/browse/YARN-9788) | Queue Management API does not support parallel updates |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16005](https://issues.apache.org/jira/browse/HADOOP-16005) | NativeAzureFileSystem does not support setXAttr |  Major | fs/azure | Clemens Wolff | Clemens Wolff |
+| [YARN-10028](https://issues.apache.org/jira/browse/YARN-10028) | Integrate the new abstract log servlet to the JobHistory server |  Major | yarn | Adam Antal | Adam Antal |
+| [YARN-10082](https://issues.apache.org/jira/browse/YARN-10082) | FS-CS converter: disable terminal placement rule checking |  Critical | . | Peter Bacsko | Peter Bacsko |
+| [HADOOP-16621](https://issues.apache.org/jira/browse/HADOOP-16621) | [pb-upgrade] Remove Protobuf classes from signatures of Public APIs |  Critical | common | Steve Loughran | Vinayakumar B |
+| [YARN-9525](https://issues.apache.org/jira/browse/YARN-9525) | IFile format is not working against s3a remote folder |  Major | log-aggregation | Adam Antal | Adam Antal |
+| [HADOOP-16346](https://issues.apache.org/jira/browse/HADOOP-16346) | Stabilize S3A OpenSSL support |  Blocker | fs/s3 | Steve Loughran | Sahil Takiar |
+| [HADOOP-16759](https://issues.apache.org/jira/browse/HADOOP-16759) | Filesystem openFile() builder to take a FileStatus param |  Minor | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-10083](https://issues.apache.org/jira/browse/YARN-10083) | Provide utility to ask whether an application is in final status |  Minor | . | Adam Antal | Adam Antal |
+| [HADOOP-16792](https://issues.apache.org/jira/browse/HADOOP-16792) | Let s3 clients configure request timeout |  Major | fs/s3 | Mustafa Iman | Mustafa Iman |
+| [HADOOP-16827](https://issues.apache.org/jira/browse/HADOOP-16827) | TestHarFileSystem.testInheritedMethodsImplemented broken |  Major | fs, test | Steve Loughran | Steve Loughran |
+| [HADOOP-16746](https://issues.apache.org/jira/browse/HADOOP-16746) | S3A empty dir markers are not created in s3guard as authoritative |  Critical | fs/s3 | Steve Loughran | Steve Loughran |
+| [YARN-10085](https://issues.apache.org/jira/browse/YARN-10085) | FS-CS converter: remove mixed ordering policy check |  Critical | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10104](https://issues.apache.org/jira/browse/YARN-10104) | FS-CS converter: dry run should work without output defined |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10015](https://issues.apache.org/jira/browse/YARN-10015) | Correct the sample command in SLS README file |  Trivial | yarn | Aihua Xu | Aihua Xu |
+| [YARN-10099](https://issues.apache.org/jira/browse/YARN-10099) | FS-CS converter: handle allow-undeclared-pools and user-as-default-queue properly and fix misc issues |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-8982](https://issues.apache.org/jira/browse/YARN-8982) | [Router] Add locality policy |  Major | . | Giovanni Matteo Fumarola | Young Chen |
+| [HADOOP-16732](https://issues.apache.org/jira/browse/HADOOP-16732) | S3Guard to support encrypted DynamoDB table |  Major | fs/s3 | Mingliang Liu | Mingliang Liu |
+| [HADOOP-16825](https://issues.apache.org/jira/browse/HADOOP-16825) | ITestAzureBlobFileSystemCheckAccess failing |  Major | fs/azure, test | Steve Loughran | Bilahari T H |
+| [HADOOP-16845](https://issues.apache.org/jira/browse/HADOOP-16845) | ITestAbfsClient.testContinuationTokenHavingEqualSign failing |  Major | fs/azure, test | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HADOOP-16596](https://issues.apache.org/jira/browse/HADOOP-16596) | [pb-upgrade] Use shaded protobuf classes from hadoop-thirdparty dependency |  Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-10109](https://issues.apache.org/jira/browse/YARN-10109) | Allow stop and convert from leaf to parent queue in a single Mutation API call |  Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph |
+| [YARN-10101](https://issues.apache.org/jira/browse/YARN-10101) | Support listing of aggregated logs for containers belonging to an application attempt |  Major | log-aggregation, yarn | Adam Antal | Adam Antal |
+| [YARN-10127](https://issues.apache.org/jira/browse/YARN-10127) | FSQueueConverter should not set App Ordering Policy to Parent Queue |  Major | . | Prabhu Joseph | Peter Bacsko |
+| [YARN-10022](https://issues.apache.org/jira/browse/YARN-10022) | Create RM Rest API to validate a CapacityScheduler Configuration |  Major | . | Kinga Marton | Kinga Marton |
+| [HDFS-13989](https://issues.apache.org/jira/browse/HDFS-13989) | RBF: Add FSCK to the Router |  Major | . | Íñigo Goiri | Akira Ajisaka |
+| [YARN-10029](https://issues.apache.org/jira/browse/YARN-10029) | Add option to UIv2 to get container logs from the new JHS API |  Major | yarn | Adam Antal | Adam Antal |
+| [HADOOP-16823](https://issues.apache.org/jira/browse/HADOOP-16823) | Large DeleteObject requests are their own Thundering Herd |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-15173](https://issues.apache.org/jira/browse/HDFS-15173) | RBF: Delete repeated configuration 'dfs.federation.router.metrics.enable' |  Minor | documentation, rbf | panlijie | panlijie |
+| [HADOOP-15961](https://issues.apache.org/jira/browse/HADOOP-15961) | S3A committers: make sure there's regular progress() calls |  Minor | fs/s3 | Steve Loughran | lqjacklee |
+| [YARN-10139](https://issues.apache.org/jira/browse/YARN-10139) | ValidateAndGetSchedulerConfiguration API fails when cluster max allocation \> default 8GB |  Major | . | Prabhu Joseph | Prabhu Joseph |
+| [HADOOP-16706](https://issues.apache.org/jira/browse/HADOOP-16706) | ITestClientUrlScheme fails for accounts which don't support HTTP |  Minor | fs/azure, test | Steve Loughran | Steve Loughran |
+| [HADOOP-16711](https://issues.apache.org/jira/browse/HADOOP-16711) | S3A bucket existence checks to support v2 API and "no checks at all" |  Minor | fs/s3 | Rajesh Balamohan | Mukund Thakur |
+| [HDFS-15172](https://issues.apache.org/jira/browse/HDFS-15172) | Remove unnecessary  deadNodeDetectInterval in DeadNodeDetector#checkDeadNodes() |  Major | . | Lisheng Sun | Lisheng Sun |
+| [HDFS-15041](https://issues.apache.org/jira/browse/HDFS-15041) | Make MAX\_LOCK\_HOLD\_MS and full queue size configurable |  Major | namenode | zhuqi | zhuqi |
+| [HADOOP-16853](https://issues.apache.org/jira/browse/HADOOP-16853) | ITestS3GuardOutOfBandOperations failing on versioned S3 buckets |  Minor | fs/s3, test | Steve Loughran | Steve Loughran |
+| [YARN-10157](https://issues.apache.org/jira/browse/YARN-10157) | FS-CS converter: initPropertyActions() is not called without rules file |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10135](https://issues.apache.org/jira/browse/YARN-10135) | FS-CS converter tool: issue warning on dynamic auto-create mapping rules |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10130](https://issues.apache.org/jira/browse/YARN-10130) | FS-CS converter: Do not allow output dir to be the same as input dir |  Major | . | Szilard Nemeth | Adam Antal |
+| [HDFS-14731](https://issues.apache.org/jira/browse/HDFS-14731) | [FGL] Remove redundant locking on NameNode. |  Major | namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-16767](https://issues.apache.org/jira/browse/HADOOP-16767) | S3AInputStream reopening does not handle non IO exceptions properly |  Major | . | Sergei Poganshev | Sergei Poganshev |
+| [YARN-10175](https://issues.apache.org/jira/browse/YARN-10175) | FS-CS converter: only convert placement rules if a cmd line switch is defined |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10167](https://issues.apache.org/jira/browse/YARN-10167) | FS-CS Converter: Need to validate c-s.xml after converting |  Major | . | Wangda Tan | Peter Bacsko |
+| [HADOOP-16905](https://issues.apache.org/jira/browse/HADOOP-16905) | Update jackson-databind to 2.10.3 to relieve us from the endless CVE patches |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-6924](https://issues.apache.org/jira/browse/YARN-6924) | Metrics for Federation AMRMProxy |  Major | . | Giovanni Matteo Fumarola | Young Chen |
+| [YARN-10168](https://issues.apache.org/jira/browse/YARN-10168) | FS-CS Converter: tool doesn't handle min/max resource conversion correctly |  Blocker | . | Wangda Tan | Peter Bacsko |
+| [HADOOP-16890](https://issues.apache.org/jira/browse/HADOOP-16890) | ABFS: Change in expiry calculation for MSI token provider |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [YARN-10191](https://issues.apache.org/jira/browse/YARN-10191) | FS-CS converter: call System.exit function call for every code path in main method |  Blocker | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10193](https://issues.apache.org/jira/browse/YARN-10193) | FS-CS converter: fix incorrect capacity conversion |  Blocker | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10110](https://issues.apache.org/jira/browse/YARN-10110) | In Federation Secure cluster Application submission fails when authorization is enabled |  Blocker | federation | Sushanta Sen | Bilwa S T |
+| [YARN-9538](https://issues.apache.org/jira/browse/YARN-9538) | Document scheduler/app activities and REST APIs |  Major | documentation | Tao Yang | Tao Yang |
+| [YARN-9567](https://issues.apache.org/jira/browse/YARN-9567) | Add diagnostics for outstanding resource requests on app attempts page |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-16858](https://issues.apache.org/jira/browse/HADOOP-16858) | S3Guard fsck: Add option to remove orphaned entries |  Major | fs/s3 | Gabor Bota | Gabor Bota |
+| [HDFS-15088](https://issues.apache.org/jira/browse/HDFS-15088) | RBF: Correct annotation typo of RouterPermissionChecker#checkPermission |  Trivial | rbf | Xiaoqiao He | Xiaoqiao He |
+| [YARN-9879](https://issues.apache.org/jira/browse/YARN-9879) | Allow multiple leaf queues with the same name in CapacityScheduler |  Major | . | Gergely Pollak | Gergely Pollak |
+| [YARN-10197](https://issues.apache.org/jira/browse/YARN-10197) | FS-CS converter: fix emitted ordering policy string and max-am-resource percent value |  Major | . | Peter Bacsko | Peter Bacsko |
+| [YARN-10043](https://issues.apache.org/jira/browse/YARN-10043) | FairOrderingPolicy Improvements |  Major | . | Manikandan R | Manikandan R |
+| [HDFS-13470](https://issues.apache.org/jira/browse/HDFS-13470) | RBF: Add Browse the Filesystem button to the UI |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HADOOP-15430](https://issues.apache.org/jira/browse/HADOOP-15430) | hadoop fs -mkdir -p path-ending-with-slash/ fails with s3guard |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16939](https://issues.apache.org/jira/browse/HADOOP-16939) | fs.s3a.authoritative.path should support multiple FS URIs |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16859](https://issues.apache.org/jira/browse/HADOOP-16859) | ABFS: Add unbuffer support to AbfsInputStream |  Major | fs/azure | Sahil Takiar | Sahil Takiar |
+| [YARN-10120](https://issues.apache.org/jira/browse/YARN-10120) | In Federation Router Nodes/Applications/About pages throws 500 exception when https is enabled |  Critical | federation | Sushanta Sen | Bilwa S T |
+| [HADOOP-16465](https://issues.apache.org/jira/browse/HADOOP-16465) | Tune S3AFileSystem.listLocatedStatus |  Major | fs/s3 | Steve Loughran | Mukund Thakur |
+| [YARN-10234](https://issues.apache.org/jira/browse/YARN-10234) | FS-CS converter: don't enable auto-create queue property for root |  Critical | . | Peter Bacsko | Peter Bacsko |
+| [HADOOP-13873](https://issues.apache.org/jira/browse/HADOOP-13873) | log DNS addresses on s3a init |  Minor | fs/s3 | Steve Loughran | Mukund Thakur |
+| [HADOOP-16959](https://issues.apache.org/jira/browse/HADOOP-16959) | Resolve hadoop-cos dependency conflict |  Major | build, fs/cos | Yang Yu | Yang Yu |
+| [HADOOP-16986](https://issues.apache.org/jira/browse/HADOOP-16986) | s3a to not need wildfly on the classpath |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-14742](https://issues.apache.org/jira/browse/HDFS-14742) | RBF:TestRouterFaultTolerant tests are flaky |  Major | test | Chen Zhang | Akira Ajisaka |
+| [HADOOP-16794](https://issues.apache.org/jira/browse/HADOOP-16794) | S3A reverts KMS encryption to the bucket's default KMS key in rename/copy |  Major | fs/s3 | Mukund Thakur | Mukund Thakur |
+| [HADOOP-16920](https://issues.apache.org/jira/browse/HADOOP-16920) | ABFS: Make list page size configurable |  Minor | fs/azure | Bilahari T H | Bilahari T H |
+| [YARN-10194](https://issues.apache.org/jira/browse/YARN-10194) | YARN RMWebServices /scheduler-conf/validate leaks ZK Connections |  Blocker | capacityscheduler | Akhil PB | Prabhu Joseph |
+| [YARN-10215](https://issues.apache.org/jira/browse/YARN-10215) | Endpoint for obtaining direct URL for the logs |  Major | yarn | Adam Antal | Andras Gyori |
+| [HDFS-14353](https://issues.apache.org/jira/browse/HDFS-14353) | Erasure Coding: metrics xmitsInProgress become to negative. |  Major | datanode, erasure-coding | maobaolong | maobaolong |
+| [HADOOP-16953](https://issues.apache.org/jira/browse/HADOOP-16953) | HADOOP-16953. tune s3guard disabled warnings |  Minor | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-16835](https://issues.apache.org/jira/browse/HADOOP-16835) | catch and downgrade all exceptions trying to load openssl native libs through wildfly |  Minor | fs/azure, fs/s3 | Steve Loughran | Steve Loughran |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13338](https://issues.apache.org/jira/browse/HDFS-13338) | Update BUILDING.txt for building native libraries |  Critical | build, documentation, native | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13403](https://issues.apache.org/jira/browse/HDFS-13403) | libhdfs++: Use hdfs::IoService object rather than asio::io\_service |  Critical | . | James Clampffer | James Clampffer |
+| [HDFS-13534](https://issues.apache.org/jira/browse/HDFS-13534) | libhdfs++: Fix GCC7 build |  Major | . | James Clampffer | James Clampffer |
+| [HADOOP-15816](https://issues.apache.org/jira/browse/HADOOP-15816) | Upgrade Apache Zookeeper version due to security concerns |  Major | . | Boris Vulikh | Akira Ajisaka |
+| [HADOOP-15882](https://issues.apache.org/jira/browse/HADOOP-15882) | Upgrade maven-shade-plugin from 2.4.3 to 3.2.0 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-15815](https://issues.apache.org/jira/browse/HADOOP-15815) | Upgrade Eclipse Jetty version to 9.3.24 |  Major | . | Boris Vulikh | Boris Vulikh |
+| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc |  Minor | documentation, webhdfs | Siyao Meng | Siyao Meng |
+| [YARN-8489](https://issues.apache.org/jira/browse/YARN-8489) | Need to support "dominant" component concept inside YARN service |  Major | yarn-native-services | Wangda Tan | Zac Zhou |
+| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS |  Major | documentation | Christopher Douglas | Masatake Iwasaki |
+| [YARN-9191](https://issues.apache.org/jira/browse/YARN-9191) | Add cli option in DS to support enforceExecutionType in resource requests. |  Major | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9428](https://issues.apache.org/jira/browse/YARN-9428) | Add metrics for paused containers in NodeManager |  Major | . | Abhishek Modi | Abhishek Modi |
+| [HDFS-14394](https://issues.apache.org/jira/browse/HDFS-14394) | Add -std=c99 / -std=gnu99 to libhdfs compile flags |  Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar |
+| [HADOOP-15242](https://issues.apache.org/jira/browse/HADOOP-15242) | Fix typos in hadoop-functions.sh |  Trivial | . | Ray Chiang | Ray Chiang |
+| [YARN-9433](https://issues.apache.org/jira/browse/YARN-9433) | Remove unused constants from RMAuditLogger |  Minor | yarn | Adam Antal | Igor Rudenko |
+| [HDFS-14433](https://issues.apache.org/jira/browse/HDFS-14433) | Remove the extra empty space in the DataStreamer logging |  Trivial | hdfs | Yishuang Lu | Yishuang Lu |
+| [YARN-9469](https://issues.apache.org/jira/browse/YARN-9469) | Fix typo in YarnConfiguration: physical memory |  Trivial | yarn | Adam Antal | Igor Rudenko |
+| [HADOOP-16263](https://issues.apache.org/jira/browse/HADOOP-16263) | Update BUILDING.txt with macOS native build instructions |  Minor | . | Siyao Meng | Siyao Meng |
+| [HADOOP-16365](https://issues.apache.org/jira/browse/HADOOP-16365) | Upgrade jackson-databind to 2.9.9 |  Major | build | Shweta | Shweta |
+| [YARN-9599](https://issues.apache.org/jira/browse/YARN-9599) | TestContainerSchedulerQueuing#testQueueShedding fails intermittently. |  Minor | . | Abhishek Modi | Abhishek Modi |
+| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16491](https://issues.apache.org/jira/browse/HADOOP-16491) | Upgrade jetty version to 9.3.27 |  Major | . | Hrishikesh Gadre | Hrishikesh Gadre |
+| [HADOOP-16351](https://issues.apache.org/jira/browse/HADOOP-16351) | Change ":" to ApplicationConstants.CLASS\_PATH\_SEPARATOR |  Trivial | common | Kevin Su | Kevin Su |
+| [HDFS-14729](https://issues.apache.org/jira/browse/HDFS-14729) | Upgrade Bootstrap and jQuery versions used in HDFS UIs |  Major | ui | Vivek Ratnavel Subramanian | Vivek Ratnavel Subramanian |
+| [HADOOP-16438](https://issues.apache.org/jira/browse/HADOOP-16438) | Introduce a config to control SSL Channel mode in Azure DataLake Store Gen1 |  Major | fs/adl | Sneha Vijayarajan | Sneha Vijayarajan |
+| [HADOOP-16542](https://issues.apache.org/jira/browse/HADOOP-16542) | Update commons-beanutils version to 1.9.4 |  Major | . | Wei-Chiu Chuang | Kevin Su |
+| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 |  Major | . | Wei-Chiu Chuang | YiSheng Lien |
+| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label |  Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-16675](https://issues.apache.org/jira/browse/HADOOP-16675) | Upgrade jackson-databind to 2.9.10.1 |  Blocker | security | Wei-Chiu Chuang | Lisheng Sun |
+| [HADOOP-16656](https://issues.apache.org/jira/browse/HADOOP-16656) | Document FairCallQueue configs in core-default.xml |  Major | conf, documentation | Siyao Meng | Siyao Meng |
+| [HDFS-14959](https://issues.apache.org/jira/browse/HDFS-14959) | [SBNN read] access time should be turned off |  Major | documentation | Wei-Chiu Chuang | Chao Sun |
+| [HADOOP-16654](https://issues.apache.org/jira/browse/HADOOP-16654) | Delete hadoop-ozone and hadoop-hdds subprojects from apache trunk |  Major | . | Marton Elek | Sandeep Nemuri |
+| [HDFS-15047](https://issues.apache.org/jira/browse/HDFS-15047) | Document the new decommission monitor (HDFS-14854) |  Major | documentation | Wei-Chiu Chuang | Masatake Iwasaki |
+| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-16358](https://issues.apache.org/jira/browse/HADOOP-16358) | Add an ARM CI for Hadoop |  Major | build | Zhenyu Zheng | Zhenyu Zheng |
+| [HADOOP-16803](https://issues.apache.org/jira/browse/HADOOP-16803) | Upgrade jackson-databind to 2.9.10.2 |  Blocker | security | Akira Ajisaka | Masatake Iwasaki |
+| [HADOOP-16670](https://issues.apache.org/jira/browse/HADOOP-16670) | Stripping Submarine code from Hadoop codebase. |  Blocker | . | Wei-Chiu Chuang | Zhankun Tang |
+| [HADOOP-16871](https://issues.apache.org/jira/browse/HADOOP-16871) | Upgrade Netty version to 4.1.45.Final to handle CVE-2019-20444,CVE-2019-16869 |  Major | . | Aray Chenchu Sukesh | Aray Chenchu Sukesh |
+| [HADOOP-16647](https://issues.apache.org/jira/browse/HADOOP-16647) | Support OpenSSL 1.1.1 LTS |  Critical | security | Wei-Chiu Chuang | Rakesh Radhakrishnan |
+| [HADOOP-16982](https://issues.apache.org/jira/browse/HADOOP-16982) | Update Netty to 4.1.48.Final |  Blocker | . | Wei-Chiu Chuang | Lisheng Sun |
+| [YARN-10247](https://issues.apache.org/jira/browse/YARN-10247) | Application priority queue ACLs are not respected |  Blocker | capacity scheduler | Sunil G | Sunil G |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md
new file mode 100644
index 0000000000000..371a8b08497d9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md
@@ -0,0 +1,540 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.3.0 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [YARN-8587](https://issues.apache.org/jira/browse/YARN-8587) | *Major* | **Delays are noticed to launch docker container**
+
+add "docker inspect" retries to discover container exit code.
+
+
+---
+
+* [HDFS-14053](https://issues.apache.org/jira/browse/HDFS-14053) | *Major* | **Provide ability for NN to re-replicate based on topology changes.**
+
+A new option (-replicate) is added to fsck command to re-trigger the replication for mis-replicated blocks. This option should be used instead of previous workaround of increasing and then decreasing replication factor (using hadoop fs -setrep command).
+
+
+---
+
+* [HADOOP-15358](https://issues.apache.org/jira/browse/HADOOP-15358) | *Critical* | **SFTPConnectionPool connections leakage**
+
+Fixed SFTPConnectionPool connections leakage
+
+
+---
+
+* [YARN-8986](https://issues.apache.org/jira/browse/YARN-8986) | *Minor* | **publish all exposed ports to random ports when using bridge network**
+
+support -p and -P for bridge type network;
+
+
+---
+
+* [MAPREDUCE-6190](https://issues.apache.org/jira/browse/MAPREDUCE-6190) | *Major* | **If a task stucks before its first heartbeat, it never timeouts and the MR job becomes stuck**
+
+Added "mapreduce.task.stuck.timeout-ms" parameter to timeout a task before sending the task's first heartbeat. The default value is 600000 milliseconds (10 minutes).
+
+
+---
+
+* [YARN-9071](https://issues.apache.org/jira/browse/YARN-9071) | *Critical* | **NM and service AM don't have updated status for reinitialized containers**
+
+In progress upgrade status may show READY state sooner than actual upgrade operations.  External caller to upgrade API is recommended to wait minimum 30 seconds before querying yarn app -status.
+
+
+---
+
+* [HADOOP-15428](https://issues.apache.org/jira/browse/HADOOP-15428) | *Major* | **s3guard bucket-info will create s3guard table if FS is set to do this automatically**
+
+If -unguarded flag is passed to \`hadoop s3guard bucket-info\`, it will now proceed with S3Guard disabled instead of failing if S3Guard is not already disabled.
+
+
+---
+
+* [HADOOP-16000](https://issues.apache.org/jira/browse/HADOOP-16000) | *Major* | **Remove TLSv1 and SSLv2Hello from the default value of hadoop.ssl.enabled.protocols**
+
+TLSv1 and SSLv2Hello were removed from the default value of "hadoop.ssl.enabled.protocols".
+
+
+---
+
+* [YARN-9084](https://issues.apache.org/jira/browse/YARN-9084) | *Major* | **Service Upgrade: With default readiness check, the status of upgrade is reported to be successful prematurely**
+
+Improve transient container status accuracy for upgrade.
+
+
+---
+
+* [YARN-8762](https://issues.apache.org/jira/browse/YARN-8762) | *Major* | **[Umbrella] Support Interactive Docker Shell to running Containers**
+
+- Add shell access to YARN containers
+
+
+---
+
+* [HADOOP-15965](https://issues.apache.org/jira/browse/HADOOP-15965) | *Major* | **Upgrade to ADLS SDK which has major performance improvement for ingress/egress**
+
+
+
+
+---
+
+* [HADOOP-15996](https://issues.apache.org/jira/browse/HADOOP-15996) | *Major* | **Plugin interface to support more complex usernames in Hadoop**
+
+This patch enables "Hadoop" and "MIT" as options for "hadoop.security.auth\_to\_local.mechanism" and defaults to 'hadoop'. This should be backward compatible with pre-HADOOP-12751.
+
+This is basically HADOOP-12751 plus configurable + extended tests.
+
+
+---
+
+* [YARN-8489](https://issues.apache.org/jira/browse/YARN-8489) | *Major* | **Need to support "dominant" component concept inside YARN service**
+
+- Improved YARN service status report based on dominant component status.
+
+
+---
+
+* [HADOOP-15922](https://issues.apache.org/jira/browse/HADOOP-15922) | *Major* | **DelegationTokenAuthenticationFilter get wrong doAsUser since it does not decode URL**
+
+- Fix DelegationTokenAuthentication filter for incorrectly double encode doAs user parameter.
+
+
+---
+
+* [YARN-9116](https://issues.apache.org/jira/browse/YARN-9116) | *Major* | **Capacity Scheduler: implements queue level maximum-allocation inheritance**
+
+After this change, capacity scheduler queue is able to inherit and override max-allocation from its parent queue. The subqueue's max-allocation can be larger or smaller than the parent queue's but can't exceed the global "yarn.scheduler.capacity.maximum-allocation". User can set queue-level max-allocation on any level of queues, using configuration property "yarn.scheduler.capacity.%QUEUE\_PATH%.maximum-allocation". And this property allows user to set max-resource-allocation for customized resource types, e.g "memory=20G,vcores=20,gpu=3".
+
+
+---
+
+* [HDFS-14158](https://issues.apache.org/jira/browse/HDFS-14158) | *Minor* | **Checkpointer ignores configured time period \> 5 minutes**
+
+Fix the Checkpointer not to ignore the configured "dfs.namenode.checkpoint.period" \> 5 minutes
+
+
+---
+
+* [YARN-8761](https://issues.apache.org/jira/browse/YARN-8761) | *Major* | **Service AM support for decommissioning component instances**
+
+- Component instance number is not linear increment when decommission feature is used.  Application with assumption of linear increment component instance number maybe impacted by introduction of this feature.
+
+
+---
+
+* [HDFS-14273](https://issues.apache.org/jira/browse/HDFS-14273) | *Trivial* | **Fix checkstyle issues in BlockLocation's method javadoc**
+
+Thanks for the patch, [~shwetayakkali], and review, [~knanasi]. Committed to trunk.
+
+
+---
+
+* [HDFS-14118](https://issues.apache.org/jira/browse/HDFS-14118) | *Major* | **Support using DNS to resolve nameservices to IP addresses**
+
+HDFS clients can use a single domain name to discover servers (namenodes/routers/observers) instead of explicitly listing out all hosts in the config
+
+
+---
+
+* [HDFS-7133](https://issues.apache.org/jira/browse/HDFS-7133) | *Major* | **Support clearing namespace quota on "/"**
+
+Namespace Quota on root can be cleared now.
+
+
+---
+
+* [HADOOP-16011](https://issues.apache.org/jira/browse/HADOOP-16011) | *Major* | **OsSecureRandom very slow compared to other SecureRandom implementations**
+
+The default RNG is now OpensslSecureRandom instead of OsSecureRandom.The high-performance hardware random number generator (RDRAND instruction) will be used if available. If not, it will fall back to OpenSSL secure random generator.
+If you insist on using OsSecureRandom, set hadoop.security.secure.random.impl in core-site.xml to org.apache.hadoop.crypto.random.OsSecureRandom.
+
+
+---
+
+* [HADOOP-16210](https://issues.apache.org/jira/browse/HADOOP-16210) | *Critical* | **Update guava to 27.0-jre in hadoop-project trunk**
+
+Guava has been updated to 27.0. Code built against this new version  may not link against older releases because of new overloads of methods like Preconditions.checkArgument().
+
+
+---
+
+* [HADOOP-16085](https://issues.apache.org/jira/browse/HADOOP-16085) | *Major* | **S3Guard: use object version or etags to protect against inconsistent read after replace/overwrite**
+
+S3Guard will now track the etag of uploaded files and, if an S3 bucket is versioned, the object version. You can then control how to react to a mismatch between the data in the DynamoDB table and that in the store: warn, fail, or, when using versions, return the original value.
+
+This adds two new columns to the table: etag and version. This is transparent to older S3A clients -but when such clients add/update data to the S3Guard table, they will not add these values. As a result, the etag/version checks will not work with files uploaded by older clients.
+
+For a consistent experience, upgrade all clients to use the latest hadoop version.
+
+
+---
+
+* [HADOOP-15563](https://issues.apache.org/jira/browse/HADOOP-15563) | *Major* | **S3Guard to support creating on-demand DDB tables**
+
+S3Guard now defaults to creating DynamoDB tables as "On-Demand", rather than with a prepaid IO capacity. This reduces costs when idle to only the storage of the metadata entries, while delivering significantly faster performance during query planning and other bursts of IO. Consult the S3Guard documentation for further details.
+
+
+---
+
+* [YARN-9578](https://issues.apache.org/jira/browse/YARN-9578) | *Major* | **Add limit/actions/summarize options for app activities REST API**
+
+This patch changes the format of app activities query URL to http://{rm-host}:{port}/ws/v1/cluster/scheduler/app-activities/{app-id}/, app-id now is a path parameter instead of a query parameter.
+
+
+---
+
+* [HDFS-14339](https://issues.apache.org/jira/browse/HDFS-14339) | *Major* | **Inconsistent log level practices in RpcProgramNfs3.java**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HDFS-14403](https://issues.apache.org/jira/browse/HDFS-14403) | *Major* | **Cost-Based RPC FairCallQueue**
+
+This adds an extension to the IPC FairCallQueue which allows for the consideration of the \*cost\* of a user's operations when deciding how they should be prioritized, as opposed to the number of operations. This can be helpful for protecting the NameNode from clients which submit very expensive operations (e.g. large listStatus operations or recursive getContentSummary operations).
+
+This can be enabled by setting the \`ipc.\<port\>.costprovder.impl\` configuration to \`org.apache.hadoop.ipc.WeightedTimeCostProvider\`.
+
+
+---
+
+* [HADOOP-16460](https://issues.apache.org/jira/browse/HADOOP-16460) | *Major* | **ABFS: fix for Sever Name Indication (SNI)**
+
+ABFS: Bug fix to support Server Name Indication (SNI).
+
+
+---
+
+* [HADOOP-16452](https://issues.apache.org/jira/browse/HADOOP-16452) | *Major* | **Increase ipc.maximum.data.length default from 64MB to 128MB**
+
+Default ipc.maximum.data.length is now 128 MB in order to accommodate huge block reports.
+
+
+---
+
+* [HDFS-13783](https://issues.apache.org/jira/browse/HDFS-13783) | *Major* | **Balancer: make balancer to be a long service process for easy to monitor it.**
+
+Adds a new parameter to the Balancer CLI, "-asService", to enable the process to be long-running.
+
+
+---
+
+* [HADOOP-16398](https://issues.apache.org/jira/browse/HADOOP-16398) | *Major* | **Exports Hadoop metrics to Prometheus**
+
+If "hadoop.prometheus.endpoint.enabled" is set to true, Prometheus-friendly formatted metrics can be obtained from '/prom' endpoint of Hadoop daemons. The default value of the property is false.
+
+
+---
+
+* [HADOOP-16479](https://issues.apache.org/jira/browse/HADOOP-16479) | *Major* | **ABFS FileStatus.getModificationTime returns localized time instead of UTC**
+
+private long parseLastModifiedTime(final String lastModifiedTime)
+
+Timezone is part of lastModifiedTime String as it's last field. But when parsed it ignores timezone field and assumes JVM timezone.
+Fix: Made timezone field considered in lastModifiedTime when the same is parsed
+
+
+---
+
+* [HADOOP-16499](https://issues.apache.org/jira/browse/HADOOP-16499) | *Critical* | **S3A retry policy to be exponential**
+
+The S3A filesystem now backs off exponentially on failures. if you have customized the fs.s3a.retry.limit and fs.s3a.retry.interval options you may wish to review these settings
+
+
+---
+
+* [HDFS-13505](https://issues.apache.org/jira/browse/HDFS-13505) | *Major* | **Turn on HDFS ACLs by default.**
+
+By default, dfs.namenode.acls.enabled is now set to true. If you haven't set dfs.namenode.acls.enabled=false before and want to keep ACLs feature disabled, you must explicitly set it to false in hdfs-site.xml. i.e.
+ \<property\>
+   \<name\>dfs.namenode.acls.enabled\</name\>
+   \<value\>false\</value\>
+ \</property\>
+
+
+---
+
+* [YARN-2599](https://issues.apache.org/jira/browse/YARN-2599) | *Major* | **Standby RM should expose jmx endpoint**
+
+YARN /jmx URL end points will be accessible per resource manager process. Hence there will not be any redirection to active resource manager while accessing /jmx endpoints.
+
+
+---
+
+* [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | *Critical* | **Yet another fsimage corruption related to snapshot**
+
+Fix a corner case in deleting HDFS snapshots. A regression was later found and fixed by HDFS-15012.
+
+
+---
+
+* [HDFS-14675](https://issues.apache.org/jira/browse/HDFS-14675) | *Major* | **Increase Balancer Defaults Further**
+
+Increase default bandwidth limit for rebalancing per DataNode (dfs.datanode.balance.bandwidthPerSec) from 10MB/s to 100MB/s.
+
+Increase default maximum threads of DataNode balancer (dfs.datanode.balance.max.concurrent.moves) from 50 to 100.
+
+
+---
+
+* [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | *Major* | **Improve fsimage load time by writing sub-sections to the fsimage index**
+
+This change allows the inode and inode directory sections of the fsimage to be loaded in parallel. Tests on large images have shown this change to reduce the image load time to about 50% of the pre-change run time.
+
+It works by writing sub-section entries to the image index, effectively splitting each image section into many sub-sections which can be processed in parallel. By default 12 sub-sections per image section are created when the image is saved, and 4 threads are used to load the image at startup.
+
+This is disabled by default for any image with more than 1M inodes (dfs.image.parallel.inode.threshold) and can be enabled by setting dfs.image.parallel.load to true. When the feature is enabled, the next HDFS checkpoint will write the image sub-sections and subsequent namenode restarts can load the image in parallel.
+
+A image with the parallel sections can be read even if the feature is disabled, but HDFS versions without this Jira cannot load an image with parallel sections. OIV can process a parallel enabled image without issues.
+
+Key configuration parameters are:
+
+dfs.image.parallel.load=false - enable or disable the feature
+
+dfs.image.parallel.target.sections = 12 - The target number of subsections. Aim for 2 to 3 times the number of dfs.image.parallel.threads.
+
+dfs.image.parallel.inode.threshold = 1000000 - Only save and load in parallel if the image has more than this number of inodes.
+
+dfs.image.parallel.threads = 4 - The number of threads used to load the image. Testing has shown 4 to be optimal, but this may depends on the environment
+
+
+---
+
+* [HDFS-14396](https://issues.apache.org/jira/browse/HDFS-14396) | *Blocker* | **Failed to load image from FSImageFile when downgrade from 3.x to 2.x**
+
+During a rolling upgrade from Hadoop 2.x to 3.x, NameNode cannot persist erasure coding information, and therefore a user cannot start using erasure coding feature until finalize is done.
+
+
+---
+
+* [HADOOP-16554](https://issues.apache.org/jira/browse/HADOOP-16554) | *Major* | **mvn javadoc:javadoc fails in hadoop-aws**
+
+Fixed the link for javadoc of hadoop-aws CommitOperations
+
+
+---
+
+* [HADOOP-16557](https://issues.apache.org/jira/browse/HADOOP-16557) | *Major* | **[pb-upgrade] Upgrade protobuf.version to 3.7.1**
+
+Upgraded the protobuf to 3.7.1.
+
+
+---
+
+* [HDFS-14845](https://issues.apache.org/jira/browse/HDFS-14845) | *Critical* | **Ignore AuthenticationFilterInitializer for HttpFSServerWebServer and honor hadoop.http.authentication configs**
+
+httpfs.authentication.\* configs become deprecated and hadoop.http.authentication.\* configs are honored in HttpFS. If the both configs are set, httpfs.authentication.\* configs are effective for compatibility.
+
+
+---
+
+* [HADOOP-15616](https://issues.apache.org/jira/browse/HADOOP-15616) | *Major* | **Incorporate Tencent Cloud COS File System Implementation**
+
+Tencent cloud is top 2 cloud vendors in China market and the object store COS is widely used among China’s cloud users.  This task implements a COSN filesytem to support Tencent cloud COS natively in Hadoop.  With simple configuration, Hadoop applications, like Spark and Hive,  can read/write data from COS without any code change.
+
+
+---
+
+* [HDFS-13762](https://issues.apache.org/jira/browse/HDFS-13762) | *Major* | **Support non-volatile storage class memory(SCM) in HDFS cache directives**
+
+Non-volatile storage class memory (SCM, also known as persistent memory) is supported in HDFS cache. To enable SCM cache, user just needs to configure SCM volume for property “dfs.datanode.cache.pmem.dirs” in hdfs-site.xml. And all HDFS cache directives keep unchanged. There are two implementations for HDFS SCM Cache, one is pure java code implementation and the other is native PMDK based implementation. The latter implementation can bring user better performance gain in cache write and cache read. If PMDK native libs could be loaded, it will use PMDK based implementation otherwise it will fallback to java code implementation. To enable PMDK based implementation, user should install PMDK library by referring to the official site http://pmem.io/. Then, build Hadoop with PMDK support by referring to "PMDK library build options" section in \`BUILDING.txt\` in the source code. If multiple SCM volumes are configured, a round-robin policy is used to select an available volume for caching a block. Consistent with DRAM cache, SCM cache also has no cache eviction mechanism. When DataNode receives a data read request from a client, if the corresponding block is cached into SCM, DataNode will instantiate an InputStream with the block location path on SCM (pure java implementation) or cache address on SCM (PMDK based implementation). Once the InputStream is created, DataNode will send the cached data to the client. Please refer "Centralized Cache Management" guide for more details.
+
+
+---
+
+* [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | *Blocker* | **Setting permissions on name directory fails on non posix compliant filesystems**
+
+- Fixed namenode/journal startup on Windows.
+
+
+---
+
+* [HADOOP-16152](https://issues.apache.org/jira/browse/HADOOP-16152) | *Major* | **Upgrade Eclipse Jetty version to 9.4.x**
+
+Upgraded Jetty to 9.4.20.v20190813.
+Downstream applications that still depend on Jetty 9.3.x may break.
+
+
+---
+
+* [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | *Major* | **Consistent Reads from Standby Node**
+
+Observer is a new type of a NameNode in addition to Active and Standby Nodes in HA settings. An Observer Node maintains a replica of the namespace same as a Standby Node. It additionally allows execution of clients read requests.
+
+To ensure read-after-write consistency within a single client, a state ID is introduced in RPC headers. The Observer responds to the client request only after its own state has caught up with the client’s state ID, which it previously received from the Active NameNode.
+
+Clients can explicitly invoke a new client protocol call msync(), which ensures that subsequent reads by this client from an Observer are consistent.
+
+A new client-side ObserverReadProxyProvider is introduced to provide automatic switching between Active and Observer NameNodes for submitting respectively write and read requests.
+
+
+---
+
+* [HDFS-13571](https://issues.apache.org/jira/browse/HDFS-13571) | *Major* | **Deadnode detection**
+
+When dead node blocks DFSInputStream, Deadnode detection can find it and share this information to other DFSInputStreams in the same DFSClient. Thus, these DFSInputStreams will not read from the dead node and be blocked by this dead node.
+
+
+---
+
+* [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | *Major* | **Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0**
+
+Updated checkstyle to 8.26 and updated maven-checkstyle-plugin to 3.1.0.
+
+
+---
+
+* [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | *Major* | **Install yarnpkg and upgrade nodejs in Dockerfile**
+
+In the Dockerfile, nodejs is upgraded to 8.17.0 and yarn 1.12.1 is installed.
+
+
+---
+
+* [HADOOP-16621](https://issues.apache.org/jira/browse/HADOOP-16621) | *Critical* | **[pb-upgrade] Remove Protobuf classes from signatures of Public APIs**
+
+Following APIs have been removed from Token.java to avoid protobuf classes in signature.
+1.   o.a.h.security.token.Token(TokenProto tokenPB)
+2.   o.a.h.security.token.Token.toTokenProto()
+
+
+---
+
+* [HADOOP-16670](https://issues.apache.org/jira/browse/HADOOP-16670) | *Blocker* | **Stripping Submarine code from Hadoop codebase.**
+
+The Submarine subproject is moved to its own Apache Top Level Project. Therefore, the Submarine code is removed from Hadoop 3.3.0 and forward. Please visit https://submarine.apache.org/ for more information.
+
+
+---
+
+* [YARN-8472](https://issues.apache.org/jira/browse/YARN-8472) | *Major* | **YARN Container Phase 2**
+
+- Improved debugging Docker container on YARN
+- Improved security for running Docker containers
+- Improved cgroup management for docker container.
+
+
+---
+
+* [HADOOP-16732](https://issues.apache.org/jira/browse/HADOOP-16732) | *Major* | **S3Guard to support encrypted DynamoDB table**
+
+Support server-side encrypted DynamoDB table for S3Guard. Users don't need to do anything (provide any configuration or change application code) if they don't want to enable server side encryption. Existing tables and the default configuration values will keep existing behavior, which is encrypted using Amazon owned customer master key (CMK).
+
+To enable server side encryption, users can set "fs.s3a.s3guard.ddb.table.sse.enabled" as true. This uses Amazon managed CMK "alias/aws/dynamodb". When it's enabled, a user can also specify her own custom KMS CMK with config "fs.s3a.s3guard.ddb.table.sse.cmk".
+
+
+---
+
+* [HADOOP-16596](https://issues.apache.org/jira/browse/HADOOP-16596) | *Major* | **[pb-upgrade] Use shaded protobuf classes from hadoop-thirdparty dependency**
+
+All protobuf classes will be used from hadooop-shaded-protobuf\_3\_7 artifact with package prefix as 'org.apache.hadoop.thirdparty.protobuf' instead of 'com.google.protobuf'
+
+
+---
+
+* [HADOOP-16823](https://issues.apache.org/jira/browse/HADOOP-16823) | *Minor* | **Large DeleteObject requests are their own Thundering Herd**
+
+The page size for bulk delete operations has been reduced from 1000 to 250 to reduce the likelihood of overloading an S3 partition, especially because the retry policy on throttling is simply to try again.
+
+The page size can be set in  "fs.s3a.bulk.delete.page.size"
+
+There is also an option to control whether or not the AWS client retries requests, or whether it is handled exclusively in the S3A code. This option "fs.s3a.experimental.aws.s3.throttling" is true by default. If set to false: everything is handled in the S3A client. While this means that metrics may be more accurate, it may mean that throttling failures in helper threads of the AWS SDK (especially those used in copy/rename) may not be handled properly. This is experimental, and should be left at "true" except when seeking more detail about throttling rates.
+
+
+---
+
+* [HADOOP-16850](https://issues.apache.org/jira/browse/HADOOP-16850) | *Major* | **Support getting thread info from thread group for JvmMetrics to improve the performance**
+
+Add a new configuration "hadoop.metrics.jvm.use-thread-mxbean". If set to true, ThreadMXBean is used for getting thread info in JvmMetrics, otherwise, ThreadGroup is used. The default value is false (use ThreadGroup for better performance).
+
+
+---
+
+* [HADOOP-16711](https://issues.apache.org/jira/browse/HADOOP-16711) | *Minor* | **S3A bucket existence checks to support v2 API and "no checks at all"**
+
+The probe for an S3 bucket existing now uses the V2 API, which fails fast if the caller lacks access to the bucket.
+The property fs.s3a.bucket.probe can be used to change this. If using a third party library which doesn't support this API call, change it to "1".
+
+to skip the probe entirely, use "0". This will make filesystem instantiation slightly faster, at a cost of postponing all issues related to bucket existence and client authentication until the filesystem API calls are first used.
+
+
+---
+
+* [HDFS-15186](https://issues.apache.org/jira/browse/HDFS-15186) | *Critical* | **Erasure Coding: Decommission may generate the parity block's content with all 0 in some case**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HDFS-14743](https://issues.apache.org/jira/browse/HDFS-14743) | *Critical* | **Enhance INodeAttributeProvider/ AccessControlEnforcer Interface in HDFS to support Authorization of mkdir, rm, rmdir, copy, move etc...**
+
+A new INodeAttributeProvider API checkPermissionWithContext(AuthorizationContext) is added. Authorization provider implementations may implement this API to get additional context (operation name and caller context) of an authorization request.
+
+
+---
+
+* [HDFS-14820](https://issues.apache.org/jira/browse/HDFS-14820) | *Major* | ** The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big**
+
+Reduce the output stream buffer size of a DFSClient remote read from 8KB to 512 bytes.
+
+
+---
+
+* [HADOOP-16661](https://issues.apache.org/jira/browse/HADOOP-16661) | *Major* | **Support TLS 1.3**
+
+Starting with Hadoop 3.3.0, TLS 1.3 is supported on Java 11 Runtime (11.0.3 and above).
+This support does not include cloud connectors.
+
+To enable TLSv1.3, add to the core-site.xml:
+\<property\>
+  \<name\>hadoop.ssl.enabled.protocols\</name\>
+  \<value\>TLSv1.3\</value\>
+\</property\>
+
+
+---
+
+* [HADOOP-15620](https://issues.apache.org/jira/browse/HADOOP-15620) | *Major* | **Über-jira: S3A phase VI: Hadoop 3.3 features**
+
+Lots of enhancements to the S3A code, including
+\* Delegation Token support
+\* better handling of 404 caching
+\* S3guard performance, resilience improvements
+
+
+---
+
+* [HADOOP-16986](https://issues.apache.org/jira/browse/HADOOP-16986) | *Major* | **s3a to not need wildfly on the classpath**
+
+hadoop-aws can use native openssl libraries for better HTTPS performance -consult the S3A performance document for details.
+
+To enable this, wildfly.jar is declared as a compile-time dependency of the hadoop-aws module, so ensuring it ends up on the classpath of the hadoop command line, distribution packages and downstream modules. 
+
+It is however, still optional, unless fs.s3a.ssl.channel.mode is set to openssl
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md
new file mode 100644
index 0000000000000..162f9928489ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/CHANGELOG.3.3.2.md
@@ -0,0 +1,350 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.3.2 - 2022-02-21
+
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15814](https://issues.apache.org/jira/browse/HDFS-15814) | Make some parameters configurable for DataNodeDiskMetrics |  Major | hdfs | tomscut | tomscut |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) | Add Available Space Rack Fault Tolerant BPP |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HDFS-16048](https://issues.apache.org/jira/browse/HDFS-16048) | RBF: Print network topology on the router web |  Minor | . | tomscut | tomscut |
+| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web |  Minor | . | tomscut | tomscut |
+| [HADOOP-17979](https://issues.apache.org/jira/browse/HADOOP-17979) | Interface EtagSource to allow FileStatus subclasses to provide etags |  Major | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-10123](https://issues.apache.org/jira/browse/YARN-10123) | Error message around yarn app -stop/start can be improved to highlight that an implementation at framework level is needed for the stop/start functionality to work |  Minor | client, documentation | Siddharth Ahuja | Siddharth Ahuja |
+| [HADOOP-17756](https://issues.apache.org/jira/browse/HADOOP-17756) | Increase precommit job timeout from 20 hours to 24 hours. |  Major | build | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-16073](https://issues.apache.org/jira/browse/HDFS-16073) | Remove redundant RPC requests for getFileLinkInfo in ClientNamenodeProtocolTranslatorPB |  Minor | . | lei w | lei w |
+| [HDFS-16074](https://issues.apache.org/jira/browse/HDFS-16074) | Remove an expensive debug string concatenation |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16080](https://issues.apache.org/jira/browse/HDFS-16080) | RBF: Invoking method in all locations should break the loop after successful result |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16075](https://issues.apache.org/jira/browse/HDFS-16075) | Use empty array constants present in StorageType and DatanodeInfo to avoid creating redundant objects |  Major | . | Viraj Jasani | Viraj Jasani |
+| [MAPREDUCE-7354](https://issues.apache.org/jira/browse/MAPREDUCE-7354) | Use empty array constants present in TaskCompletionEvent to avoid creating redundant objects |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16082](https://issues.apache.org/jira/browse/HDFS-16082) | Avoid non-atomic operations on exceptionsSinceLastBalance and failedTimesSinceLastSuccessfulBalance in Balancer |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16076](https://issues.apache.org/jira/browse/HDFS-16076) | Avoid using slow DataNodes for reading by sorting locations |  Major | hdfs | tomscut | tomscut |
+| [HDFS-16085](https://issues.apache.org/jira/browse/HDFS-16085) | Move the getPermissionChecker out of the read lock |  Minor | . | tomscut | tomscut |
+| [YARN-10834](https://issues.apache.org/jira/browse/YARN-10834) | Intra-queue preemption: apps that don't use defined custom resource won't be preempted. |  Major | . | Eric Payne | Eric Payne |
+| [HADOOP-17777](https://issues.apache.org/jira/browse/HADOOP-17777) | Update clover-maven-plugin version from 3.3.0 to 4.4.1 |  Major | . | Wanqiang Ji | Wanqiang Ji |
+| [HDFS-16090](https://issues.apache.org/jira/browse/HDFS-16090) | Fine grained locking for datanodeNetworkCounts |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17749](https://issues.apache.org/jira/browse/HADOOP-17749) | Remove lock contention in SelectorPool of SocketIOWithTimeout |  Major | common | Xuesen Liang | Xuesen Liang |
+| [HADOOP-17775](https://issues.apache.org/jira/browse/HADOOP-17775) | Remove JavaScript package from Docker environment |  Major | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17402](https://issues.apache.org/jira/browse/HADOOP-17402) | Add GCS FS impl reference to core-default.xml |  Major | fs | Rafal Wojdyla | Rafal Wojdyla |
+| [HADOOP-17794](https://issues.apache.org/jira/browse/HADOOP-17794) | Add a sample configuration to use ZKDelegationTokenSecretManager in Hadoop KMS |  Major | documentation, kms, security | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-16122](https://issues.apache.org/jira/browse/HDFS-16122) | Fix DistCpContext#toString() |  Minor | . | tomscut | tomscut |
+| [HADOOP-12665](https://issues.apache.org/jira/browse/HADOOP-12665) | Document hadoop.security.token.service.use\_ip |  Major | documentation | Arpit Agarwal | Akira Ajisaka |
+| [YARN-10456](https://issues.apache.org/jira/browse/YARN-10456) | RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry |  Major | resourcemanager | Eric Payne | Eric Payne |
+| [HDFS-15650](https://issues.apache.org/jira/browse/HDFS-15650) | Make the socket timeout for computing checksum of striped blocks configurable |  Minor | datanode, ec, erasure-coding | Yushi Hayasaka | Yushi Hayasaka |
+| [YARN-10858](https://issues.apache.org/jira/browse/YARN-10858) | [UI2] YARN-10826 breaks Queue view |  Major | yarn-ui-v2 | Andras Gyori | Masatake Iwasaki |
+| [HADOOP-16290](https://issues.apache.org/jira/browse/HADOOP-16290) | Enable RpcMetrics units to be configurable |  Major | ipc, metrics | Erik Krogen | Viraj Jasani |
+| [YARN-10860](https://issues.apache.org/jira/browse/YARN-10860) | Make max container per heartbeat configs refreshable |  Major | . | Eric Badger | Eric Badger |
+| [HADOOP-17813](https://issues.apache.org/jira/browse/HADOOP-17813) | Checkstyle - Allow line length: 100 |  Major | . | Akira Ajisaka | Viraj Jasani |
+| [HADOOP-17811](https://issues.apache.org/jira/browse/HADOOP-17811) | ABFS ExponentialRetryPolicy doesn't pick up configuration values |  Minor | documentation, fs/azure | Brian Frank Loss | Brian Frank Loss |
+| [HADOOP-17819](https://issues.apache.org/jira/browse/HADOOP-17819) | Add extensions to ProtobufRpcEngine RequestHeaderProto |  Major | common | Hector Sandoval Chaverri | Hector Sandoval Chaverri |
+| [HDFS-15936](https://issues.apache.org/jira/browse/HDFS-15936) | Solve BlockSender#sendPacket() does not record SocketTimeout exception |  Minor | . | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16153](https://issues.apache.org/jira/browse/HDFS-16153) | Avoid evaluation of LOG.debug statement in QuorumJournalManager |  Trivial | . | wangzhaohui | wangzhaohui |
+| [HDFS-16154](https://issues.apache.org/jira/browse/HDFS-16154) | TestMiniJournalCluster failing intermittently because of not reseting UserGroupInformation completely |  Minor | . | wangzhaohui | wangzhaohui |
+| [HADOOP-17837](https://issues.apache.org/jira/browse/HADOOP-17837) | Make it easier to debug UnknownHostExceptions from NetUtils.connect |  Minor | . | Bryan Beaudreault | Bryan Beaudreault |
+| [HDFS-16175](https://issues.apache.org/jira/browse/HDFS-16175) | Improve the configurable value of Server #PURGE\_INTERVAL\_NANOS |  Major | ipc | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability |  Major | fs | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-17897](https://issues.apache.org/jira/browse/HADOOP-17897) | Allow nested blocks in switch case in checkstyle settings |  Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-17857](https://issues.apache.org/jira/browse/HADOOP-17857) | Check real user ACLs in addition to proxied user ACLs |  Major | . | Eric Payne | Eric Payne |
+| [HDFS-16210](https://issues.apache.org/jira/browse/HDFS-16210) | RBF: Add the option of refreshCallQueue to RouterAdmin |  Major | . | Janus Chow | Janus Chow |
+| [HDFS-16221](https://issues.apache.org/jira/browse/HDFS-16221) | RBF: Add usage of refreshCallQueue for Router |  Major | . | Janus Chow | Janus Chow |
+| [HDFS-16223](https://issues.apache.org/jira/browse/HDFS-16223) | AvailableSpaceRackFaultTolerantBlockPlacementPolicy should use chooseRandomWithStorageTypeTwoTrial() for better performance. |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-17893](https://issues.apache.org/jira/browse/HADOOP-17893) | Improve PrometheusSink for Namenode TopMetrics |  Major | metrics | Max  Xie | Max  Xie |
+| [HADOOP-17926](https://issues.apache.org/jira/browse/HADOOP-17926) | Maven-eclipse-plugin is no longer needed since Eclipse can import Maven projects by itself. |  Minor | documentation | Rintaro Ikeda | Rintaro Ikeda |
+| [YARN-10935](https://issues.apache.org/jira/browse/YARN-10935) | AM Total Queue Limit goes below per-user AM Limit if parent is full. |  Major | capacity scheduler, capacityscheduler | Eric Payne | Eric Payne |
+| [HADOOP-17939](https://issues.apache.org/jira/browse/HADOOP-17939) | Support building on Apple Silicon |  Major | build, common | Dongjoon Hyun | Dongjoon Hyun |
+| [HADOOP-17941](https://issues.apache.org/jira/browse/HADOOP-17941) | Update xerces to 2.12.1 |  Minor | . | Zhongwei Zhu | Zhongwei Zhu |
+| [HDFS-16246](https://issues.apache.org/jira/browse/HDFS-16246) | Print lockWarningThreshold in InstrumentedLock#logWarning and InstrumentedLock#logWaitWarning |  Minor | . | tomscut | tomscut |
+| [HDFS-16252](https://issues.apache.org/jira/browse/HDFS-16252) | Correct docs for dfs.http.client.retry.policy.spec |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread |  Major | . | zhanghuazong | zhanghuazong |
+| [HADOOP-17974](https://issues.apache.org/jira/browse/HADOOP-17974) | Fix the import statements in hadoop-aws module |  Minor | build, fs/azure | Tamas Domok |  |
+| [HDFS-16277](https://issues.apache.org/jira/browse/HDFS-16277) | Improve decision in AvailableSpaceBlockPlacementPolicy |  Major | block placement | guophilipse | guophilipse |
+| [HADOOP-17770](https://issues.apache.org/jira/browse/HADOOP-17770) | WASB : Support disabling buffered reads in positional reads |  Major | . | Anoop Sam John | Anoop Sam John |
+| [HDFS-16282](https://issues.apache.org/jira/browse/HDFS-16282) | Duplicate generic usage information to hdfs debug command |  Minor | tools | daimin | daimin |
+| [YARN-1115](https://issues.apache.org/jira/browse/YARN-1115) | Provide optional means for a scheduler to check real user ACLs |  Major | capacity scheduler, scheduler | Eric Payne |  |
+| [HDFS-16279](https://issues.apache.org/jira/browse/HDFS-16279) | Print detail datanode info when process first storage report |  Minor | . | tomscut | tomscut |
+| [HDFS-16286](https://issues.apache.org/jira/browse/HDFS-16286) | Debug tool to verify the correctness of erasure coding on file |  Minor | erasure-coding, tools | daimin | daimin |
+| [HDFS-16294](https://issues.apache.org/jira/browse/HDFS-16294) | Remove invalid DataNode#CONFIG\_PROPERTY\_SIMULATED |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16299](https://issues.apache.org/jira/browse/HDFS-16299) | Fix bug for TestDataNodeVolumeMetrics#verifyDataNodeVolumeMetrics |  Minor | . | tomscut | tomscut |
+| [HDFS-16301](https://issues.apache.org/jira/browse/HDFS-16301) | Improve BenchmarkThroughput#SIZE naming standardization |  Minor | benchmarks, test | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16287](https://issues.apache.org/jira/browse/HDFS-16287) | Support to make dfs.namenode.avoid.read.slow.datanode  reconfigurable |  Major | . | Haiyang Hu | Haiyang Hu |
+| [HDFS-16321](https://issues.apache.org/jira/browse/HDFS-16321) | Fix invalid config in TestAvailableSpaceRackFaultTolerantBPP |  Minor | test | guophilipse | guophilipse |
+| [HDFS-16315](https://issues.apache.org/jira/browse/HDFS-16315) | Add metrics related to Transfer and NativeCopy for DataNode |  Major | . | tomscut | tomscut |
+| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. |  Major | fs | Chengwei Wang | Chengwei Wang |
+| [HDFS-16344](https://issues.apache.org/jira/browse/HDFS-16344) | Improve DirectoryScanner.Stats#toString |  Major | . | tomscut | tomscut |
+| [HADOOP-18023](https://issues.apache.org/jira/browse/HADOOP-18023) | Allow cp command to run with multi threads. |  Major | fs | Chengwei Wang | Chengwei Wang |
+| [HDFS-16314](https://issues.apache.org/jira/browse/HDFS-16314) | Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable |  Major | . | Haiyang Hu | Haiyang Hu |
+| [HADOOP-18026](https://issues.apache.org/jira/browse/HADOOP-18026) | Fix default value of Magic committer |  Minor | common | guophilipse | guophilipse |
+| [HDFS-16345](https://issues.apache.org/jira/browse/HDFS-16345) | Fix test cases fail in TestBlockStoragePolicy |  Major | build | guophilipse | guophilipse |
+| [HADOOP-18040](https://issues.apache.org/jira/browse/HADOOP-18040) | Use maven.test.failure.ignore instead of ignoreTestFailure |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17643](https://issues.apache.org/jira/browse/HADOOP-17643) | WASB : Make metadata checks case insensitive |  Major | . | Anoop Sam John | Anoop Sam John |
+| [HADOOP-18033](https://issues.apache.org/jira/browse/HADOOP-18033) | Upgrade fasterxml Jackson to 2.13.0 |  Major | build | Akira Ajisaka | Viraj Jasani |
+| [HDFS-16327](https://issues.apache.org/jira/browse/HDFS-16327) | Make dfs.namenode.max.slowpeer.collect.nodes reconfigurable |  Major | . | tomscut | tomscut |
+| [HDFS-16375](https://issues.apache.org/jira/browse/HDFS-16375) | The FBR lease ID should be exposed to the log |  Major | . | tomscut | tomscut |
+| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working |  Major | datanode | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16391](https://issues.apache.org/jira/browse/HDFS-16391) | Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService |  Trivial | . | wangzhaohui | wangzhaohui |
+| [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | Improve RM system metrics publisher's performance by pushing events to timeline server in batch |  Critical | resourcemanager, timelineserver | Hu Ziqian | Ashutosh Gupta |
+| [HADOOP-18052](https://issues.apache.org/jira/browse/HADOOP-18052) | Support Apple Silicon in start-build-env.sh |  Major | build | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-18056](https://issues.apache.org/jira/browse/HADOOP-18056) | DistCp: Filter duplicates in the source paths |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-18065](https://issues.apache.org/jira/browse/HADOOP-18065) | ExecutorHelper.logThrowableFromAfterExecute() is too noisy. |  Minor | . | Mukund Thakur | Mukund Thakur |
+| [HDFS-16043](https://issues.apache.org/jira/browse/HDFS-16043) | Add markedDeleteBlockScrubberThread to delete blocks asynchronously |  Major | hdfs, namanode | Xiangyi Zhu | Xiangyi Zhu |
+| [HADOOP-18094](https://issues.apache.org/jira/browse/HADOOP-18094) | Disable S3A auditing by default. |  Blocker | fs/s3 | Steve Loughran | Steve Loughran |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-10438](https://issues.apache.org/jira/browse/YARN-10438) | Handle null containerId in ClientRMService#getContainerReport() |  Major | resourcemanager | Raghvendra Singh | Shubham Gupta |
+| [YARN-10428](https://issues.apache.org/jira/browse/YARN-10428) | Zombie applications in the YARN queue using FAIR + sizebasedweight |  Critical | capacityscheduler | Guang Yang | Andras Gyori |
+| [HDFS-15916](https://issues.apache.org/jira/browse/HDFS-15916) | DistCp: Backward compatibility: Distcp fails from Hadoop 3 to Hadoop 2 for snapshotdiff |  Major | distcp | Srinivasu Majeti | Ayush Saxena |
+| [HDFS-15977](https://issues.apache.org/jira/browse/HDFS-15977) | Call explicit\_bzero only if it is available |  Major | libhdfs++ | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-14922](https://issues.apache.org/jira/browse/HADOOP-14922) | Build of Mapreduce Native Task module fails with unknown opcode "bswap" |  Major | . | Anup Halarnkar | Anup Halarnkar |
+| [HADOOP-17700](https://issues.apache.org/jira/browse/HADOOP-17700) | ExitUtil#halt info log should log HaltException |  Major | . | Viraj Jasani | Viraj Jasani |
+| [YARN-10770](https://issues.apache.org/jira/browse/YARN-10770) | container-executor permission is wrong in SecureContainer.md |  Major | documentation | Akira Ajisaka | Siddharth Ahuja |
+| [YARN-10691](https://issues.apache.org/jira/browse/YARN-10691) | DominantResourceCalculator isInvalidDivisor should consider only countable resource types |  Major | . | Bilwa S T | Bilwa S T |
+| [HDFS-16031](https://issues.apache.org/jira/browse/HDFS-16031) | Possible Resource Leak in org.apache.hadoop.hdfs.server.aliasmap#InMemoryAliasMap |  Major | . | Narges Shadab | Narges Shadab |
+| [MAPREDUCE-7348](https://issues.apache.org/jira/browse/MAPREDUCE-7348) | TestFrameworkUploader#testNativeIO fails |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-15915](https://issues.apache.org/jira/browse/HDFS-15915) | Race condition with async edits logging due to updating txId outside of the namesystem log |  Major | hdfs, namenode | Konstantin Shvachko | Konstantin Shvachko |
+| [HDFS-16040](https://issues.apache.org/jira/browse/HDFS-16040) | RpcQueueTime metric counts requeued calls as unique events. |  Major | hdfs | Simbarashe Dzinamarira | Simbarashe Dzinamarira |
+| [MAPREDUCE-7287](https://issues.apache.org/jira/browse/MAPREDUCE-7287) | Distcp will delete existing file ,  If we use "-delete and -update" options and distcp file. |  Major | distcp | zhengchenyu | zhengchenyu |
+| [HDFS-15998](https://issues.apache.org/jira/browse/HDFS-15998) | Fix NullPointException In listOpenFiles |  Major | . | Haiyang Hu | Haiyang Hu |
+| [HDFS-16050](https://issues.apache.org/jira/browse/HDFS-16050) | Some dynamometer tests fail |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-17631](https://issues.apache.org/jira/browse/HADOOP-17631) | Configuration ${env.VAR:-FALLBACK} should eval FALLBACK when restrictSystemProps=true |  Minor | common | Steve Loughran | Steve Loughran |
+| [YARN-10809](https://issues.apache.org/jira/browse/YARN-10809) | testWithHbaseConfAtHdfsFileSystem consistently failing |  Major | . | Viraj Jasani | Viraj Jasani |
+| [YARN-10803](https://issues.apache.org/jira/browse/YARN-10803) | [JDK 11] TestRMFailoverProxyProvider and TestNoHaRMFailoverProxyProvider fails by ClassCastException |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-16057](https://issues.apache.org/jira/browse/HDFS-16057) | Make sure the order for location in ENTERING\_MAINTENANCE state |  Minor | . | tomscut | tomscut |
+| [HDFS-16055](https://issues.apache.org/jira/browse/HDFS-16055) | Quota is not preserved in snapshot INode |  Major | hdfs | Siyao Meng | Siyao Meng |
+| [HDFS-16068](https://issues.apache.org/jira/browse/HDFS-16068) | WebHdfsFileSystem has a possible connection leak in connection with HttpFS |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-10767](https://issues.apache.org/jira/browse/YARN-10767) | Yarn Logs Command retrying on Standby RM for 30 times |  Major | . | D M Murali Krishna Reddy | D M Murali Krishna Reddy |
+| [HADOOP-17760](https://issues.apache.org/jira/browse/HADOOP-17760) | Delete hadoop.ssl.enabled and dfs.https.enable from docs and core-default.xml |  Major | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13671](https://issues.apache.org/jira/browse/HDFS-13671) | Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet |  Major | . | Yiqun Lin | Haibin Huang |
+| [HDFS-16061](https://issues.apache.org/jira/browse/HDFS-16061) | DFTestUtil.waitReplication can produce false positives |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient |  Major | . | Tao Yang | Renukaprasad C |
+| [YARN-10826](https://issues.apache.org/jira/browse/YARN-10826) | [UI2] Upgrade Node.js to at least v12.22.1 |  Major | yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki |
+| [HADOOP-17769](https://issues.apache.org/jira/browse/HADOOP-17769) | Upgrade JUnit to 4.13.2 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [YARN-10824](https://issues.apache.org/jira/browse/YARN-10824) | Title not set for JHS and NM webpages |  Major | . | Rajshree Mishra | Bilwa S T |
+| [HDFS-16092](https://issues.apache.org/jira/browse/HDFS-16092) | Avoid creating LayoutFlags redundant objects |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17764](https://issues.apache.org/jira/browse/HADOOP-17764) | S3AInputStream read does not re-open the input stream on the second read retry attempt |  Major | fs/s3 | Zamil Majdy | Zamil Majdy |
+| [HDFS-16109](https://issues.apache.org/jira/browse/HDFS-16109) | Fix flaky some unit tests since they offen timeout |  Minor | test | tomscut | tomscut |
+| [HDFS-16108](https://issues.apache.org/jira/browse/HDFS-16108) | Incorrect log placeholders used in JournalNodeSyncer |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [MAPREDUCE-7353](https://issues.apache.org/jira/browse/MAPREDUCE-7353) | Mapreduce job fails when NM is stopped |  Major | . | Bilwa S T | Bilwa S T |
+| [HDFS-16121](https://issues.apache.org/jira/browse/HDFS-16121) | Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames |  Major | snapshots | Srinivasu Majeti | Shashikant Banerjee |
+| [HDFS-15796](https://issues.apache.org/jira/browse/HDFS-15796) | ConcurrentModificationException error happens on NameNode occasionally |  Critical | hdfs | Daniel Ma | Daniel Ma |
+| [HADOOP-17793](https://issues.apache.org/jira/browse/HADOOP-17793) | Better token validation |  Major | . | Artem Smotrakov | Artem Smotrakov |
+| [HDFS-16042](https://issues.apache.org/jira/browse/HDFS-16042) | DatanodeAdminMonitor scan should be delay based |  Major | datanode | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17803](https://issues.apache.org/jira/browse/HADOOP-17803) | Remove WARN logging from LoggingAuditor when executing a request outside an audit span |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HDFS-16127](https://issues.apache.org/jira/browse/HDFS-16127) | Improper pipeline close recovery causes a permanent write failure or data loss. |  Major | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-17028](https://issues.apache.org/jira/browse/HADOOP-17028) | ViewFS should initialize target filesystems lazily |  Major | client-mounts, fs, viewfs | Uma Maheswara Rao G | Abhishek Das |
+| [HADOOP-17801](https://issues.apache.org/jira/browse/HADOOP-17801) | No error message reported when bucket doesn't exist in S3AFS |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17796](https://issues.apache.org/jira/browse/HADOOP-17796) | Upgrade jetty version to 9.4.43 |  Major | . | Wei-Chiu Chuang | Renukaprasad C |
+| [HDFS-12920](https://issues.apache.org/jira/browse/HDFS-12920) | HDFS default value change (with adding time unit) breaks old version MR tarball work with Hadoop 3.x |  Critical | configuration, hdfs | Junping Du | Akira Ajisaka |
+| [HDFS-16145](https://issues.apache.org/jira/browse/HDFS-16145) | CopyListing fails with FNF exception with snapshot diff |  Major | distcp | Shashikant Banerjee | Shashikant Banerjee |
+| [YARN-10813](https://issues.apache.org/jira/browse/YARN-10813) | Set default capacity of root for node labels |  Major | . | Andras Gyori | Andras Gyori |
+| [HDFS-16144](https://issues.apache.org/jira/browse/HDFS-16144) | Revert HDFS-15372 (Files in snapshots no longer see attribute provider permissions) |  Major | . | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-17817](https://issues.apache.org/jira/browse/HADOOP-17817) | HADOOP-17817. S3A to raise IOE if both S3-CSE and S3Guard enabled |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [YARN-9551](https://issues.apache.org/jira/browse/YARN-9551) | TestTimelineClientV2Impl.testSyncCall fails intermittently |  Minor | ATSv2, test | Prabhu Joseph | Andras Gyori |
+| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog |  Critical | . | Yicong Cai | Wan Chang |
+| [YARN-10869](https://issues.apache.org/jira/browse/YARN-10869) | CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues |  Major | capacity scheduler | Benjamin Teke | Benjamin Teke |
+| [YARN-10789](https://issues.apache.org/jira/browse/YARN-10789) | RM HA startup can fail due to race conditions in ZKConfigurationStore |  Major | . | Tarun Parimi | Tarun Parimi |
+| [HADOOP-17812](https://issues.apache.org/jira/browse/HADOOP-17812) | NPE in S3AInputStream read() after failure to reconnect to store |  Major | fs/s3 | Bobby Wang | Bobby Wang |
+| [YARN-6221](https://issues.apache.org/jira/browse/YARN-6221) | Entities missing from ATS when summary log file info got returned to the ATS before the domain log |  Critical | yarn | Sushmitha Sreenivasan | Xiaomin Zhang |
+| [MAPREDUCE-7258](https://issues.apache.org/jira/browse/MAPREDUCE-7258) | HistoryServerRest.html#Task\_Counters\_API, modify the jobTaskCounters's itemName from "taskcounterGroup" to "taskCounterGroup". |  Minor | documentation | jenny | jenny |
+| [HADOOP-17370](https://issues.apache.org/jira/browse/HADOOP-17370) | Upgrade commons-compress to 1.21 |  Major | common | Dongjoon Hyun | Akira Ajisaka |
+| [HDFS-16151](https://issues.apache.org/jira/browse/HDFS-16151) | Improve the parameter comments related to ProtobufRpcEngine2#Server() |  Minor | documentation | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-17844](https://issues.apache.org/jira/browse/HADOOP-17844) | Upgrade JSON smart to 2.4.7 |  Major | . | Renukaprasad C | Renukaprasad C |
+| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile |  Minor | . | tomscut | tomscut |
+| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file |  Major | . | Benjamin Teke | Tamas Domok |
+| [HADOOP-17858](https://issues.apache.org/jira/browse/HADOOP-17858) | Avoid possible class loading deadlock with VerifierNone initialization |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17869](https://issues.apache.org/jira/browse/HADOOP-17869) | fs.s3a.connection.maximum should be bigger than fs.s3a.threads.max |  Major | common | Dongjoon Hyun | Dongjoon Hyun |
+| [HADOOP-17886](https://issues.apache.org/jira/browse/HADOOP-17886) | Upgrade ant to 1.10.11 |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17874](https://issues.apache.org/jira/browse/HADOOP-17874) | ExceptionsHandler to add terse/suppressed Exceptions in thread-safe manner |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-15129](https://issues.apache.org/jira/browse/HADOOP-15129) | Datanode caches namenode DNS lookup failure and cannot startup |  Minor | ipc | Karthik Palaniappan | Chris Nauroth |
+| [HADOOP-17870](https://issues.apache.org/jira/browse/HADOOP-17870) | HTTP Filesystem to qualify paths in open()/getFileStatus() |  Minor | fs | VinothKumar Raman | VinothKumar Raman |
+| [HADOOP-17899](https://issues.apache.org/jira/browse/HADOOP-17899) | Avoid using implicit dependency on junit-jupiter-api |  Major | test | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir |  Major | nodemanager | Tamas Domok | Tamas Domok |
+| [HADOOP-17804](https://issues.apache.org/jira/browse/HADOOP-17804) | Prometheus metrics only include the last set of labels |  Major | common | Adam Binford | Adam Binford |
+| [HDFS-16207](https://issues.apache.org/jira/browse/HDFS-16207) | Remove NN logs stack trace for non-existent xattr query |  Major | namenode | Ahmed Hussein | Ahmed Hussein |
+| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing |  Major | snapshots | Srinivasu Majeti | Shashikant Banerjee |
+| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown |  Major | . | Eungsop Yoo | Eungsop Yoo |
+| [YARN-10870](https://issues.apache.org/jira/browse/YARN-10870) | Missing user filtering check -\> yarn.webapp.filter-entity-list-by-user for RM Scheduler page |  Major | yarn | Siddharth Ahuja | Gergely Pollák |
+| [HADOOP-17891](https://issues.apache.org/jira/browse/HADOOP-17891) | lz4-java and snappy-java should be excluded from relocation in shaded Hadoop libraries |  Major | . | L. C. Hsieh | L. C. Hsieh |
+| [HADOOP-17919](https://issues.apache.org/jira/browse/HADOOP-17919) | Fix command line example in Hadoop Cluster Setup documentation |  Minor | documentation | Rintaro Ikeda | Rintaro Ikeda |
+| [YARN-9606](https://issues.apache.org/jira/browse/YARN-9606) | Set sslfactory for AuthenticatedURL() while creating LogsCLI#webServiceClient |  Major | . | Bilwa S T | Bilwa S T |
+| [HDFS-16233](https://issues.apache.org/jira/browse/HDFS-16233) | Do not use exception handler to implement copy-on-write for EnumCounters |  Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method |  Major | hdfs | angerszhu | angerszhu |
+| [HADOOP-17940](https://issues.apache.org/jira/browse/HADOOP-17940) | Upgrade Kafka to 2.8.1 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-10970](https://issues.apache.org/jira/browse/YARN-10970) | Standby RM should expose prom endpoint |  Major | resourcemanager | Max  Xie | Max  Xie |
+| [HADOOP-17934](https://issues.apache.org/jira/browse/HADOOP-17934) | NullPointerException when no HTTP response set on AbfsRestOperation |  Major | fs/azure | Josh Elser | Josh Elser |
+| [HDFS-16181](https://issues.apache.org/jira/browse/HDFS-16181) | [SBN Read] Fix metric of RpcRequestCacheMissAmount can't display when tailEditLog form JN |  Critical | . | wangzhaohui | wangzhaohui |
+| [HADOOP-17922](https://issues.apache.org/jira/browse/HADOOP-17922) | Lookup old S3 encryption configs for JCEKS |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17925](https://issues.apache.org/jira/browse/HADOOP-17925) | BUILDING.txt should not encourage to activate docs profile on building binary artifacts |  Minor | documentation | Rintaro Ikeda | Masatake Iwasaki |
+| [HADOOP-16532](https://issues.apache.org/jira/browse/HADOOP-16532) | Fix TestViewFsTrash to use the correct homeDir. |  Minor | test, viewfs | Steve Loughran | Xing Lin |
+| [HDFS-16268](https://issues.apache.org/jira/browse/HDFS-16268) | Balancer stuck when moving striped blocks due to NPE |  Major | balancer & mover, erasure-coding | Leon Gao | Leon Gao |
+| [HDFS-16271](https://issues.apache.org/jira/browse/HDFS-16271) | RBF: NullPointerException when setQuota through routers with quota disabled |  Major | . | Chengwei Wang | Chengwei Wang |
+| [YARN-10976](https://issues.apache.org/jira/browse/YARN-10976) | Fix resource leak due to Files.walk |  Minor | . | lujie | lujie |
+| [HADOOP-17932](https://issues.apache.org/jira/browse/HADOOP-17932) | Distcp file length comparison have no effect |  Major | common, tools, tools/distcp | yinan zhan | yinan zhan |
+| [HDFS-16272](https://issues.apache.org/jira/browse/HDFS-16272) | Int overflow in computing safe length during EC block recovery |  Critical | 3.1.1 | daimin | daimin |
+| [HADOOP-17953](https://issues.apache.org/jira/browse/HADOOP-17953) | S3A: ITestS3AFileContextStatistics test to lookup global or per-bucket configuration for encryption algorithm |  Minor | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17971](https://issues.apache.org/jira/browse/HADOOP-17971) | Exclude IBM Java security classes from being shaded/relocated |  Major | build | Nicholas Marion | Nicholas Marion |
+| [HDFS-7612](https://issues.apache.org/jira/browse/HDFS-7612) | TestOfflineEditsViewer.testStored() uses incorrect default value for cacheDir |  Major | test | Konstantin Shvachko | Michael Kuchenbecker |
+| [HDFS-16269](https://issues.apache.org/jira/browse/HDFS-16269) | [Fix] Improve NNThroughputBenchmark#blockReport operation |  Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-17945](https://issues.apache.org/jira/browse/HADOOP-17945) | JsonSerialization raises EOFException reading JSON data stored on google GCS |  Major | fs | Steve Loughran | Steve Loughran |
+| [HDFS-16259](https://issues.apache.org/jira/browse/HDFS-16259) | Catch and re-throw sub-classes of AccessControlException thrown by any permission provider plugins (eg Ranger) |  Major | namenode | Stephen O'Donnell | Stephen O'Donnell |
+| [HADOOP-17988](https://issues.apache.org/jira/browse/HADOOP-17988) | Disable JIRA plugin for YETUS on Hadoop |  Critical | build | Gautham Banasandra | Gautham Banasandra |
+| [HDFS-16311](https://issues.apache.org/jira/browse/HDFS-16311) | Metric metadataOperationRate calculation error in DataNodeVolumeMetrics |  Major | . | tomscut | tomscut |
+| [HADOOP-18002](https://issues.apache.org/jira/browse/HADOOP-18002) | abfs rename idempotency broken -remove recovery |  Major | fs/azure | Steve Loughran | Steve Loughran |
+| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in  BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage |  Major | namanode | Max  Xie | Max  Xie |
+| [HADOOP-17999](https://issues.apache.org/jira/browse/HADOOP-17999) | No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem |  Major | . | Abhishek Das | Abhishek Das |
+| [HDFS-16329](https://issues.apache.org/jira/browse/HDFS-16329) | Fix log format for BlockManager |  Minor | . | tomscut | tomscut |
+| [HDFS-16330](https://issues.apache.org/jira/browse/HDFS-16330) | Fix incorrect placeholder for Exception logs in DiskBalancer |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HDFS-16328](https://issues.apache.org/jira/browse/HDFS-16328) | Correct disk balancer param desc |  Minor | documentation, hdfs | guophilipse | guophilipse |
+| [HDFS-16334](https://issues.apache.org/jira/browse/HDFS-16334) | Correct NameNode ACL description |  Minor | documentation | guophilipse | guophilipse |
+| [HDFS-16343](https://issues.apache.org/jira/browse/HDFS-16343) | Add some debug logs when the dfsUsed are not used during Datanode startup |  Major | datanode | Mukul Kumar Singh | Mukul Kumar Singh |
+| [YARN-10991](https://issues.apache.org/jira/browse/YARN-10991) | Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method |  Minor | distributed-shell | Ashutosh Gupta | Ashutosh Gupta |
+| [HADOOP-17975](https://issues.apache.org/jira/browse/HADOOP-17975) | Fallback to simple auth does not work for a secondary DistributedFileSystem instance |  Major | ipc | István Fajth | István Fajth |
+| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [YARN-11007](https://issues.apache.org/jira/browse/YARN-11007) | Correct words in YARN documents |  Minor | documentation | guophilipse | guophilipse |
+| [YARN-10975](https://issues.apache.org/jira/browse/YARN-10975) | EntityGroupFSTimelineStore#ActiveLogParser parses already processed files |  Major | timelineserver | Prabhu Joseph | Ravuri Sushma sree |
+| [HDFS-16332](https://issues.apache.org/jira/browse/HDFS-16332) | Expired block token causes slow read due to missing handling in sasl handshake |  Major | datanode, dfs, dfsclient | Shinya Yoshida | Shinya Yoshida |
+| [HDFS-16293](https://issues.apache.org/jira/browse/HDFS-16293) | Client sleeps and holds 'dataQueue' when DataNodes are congested |  Major | hdfs-client | Yuanxin Zhu | Yuanxin Zhu |
+| [YARN-9063](https://issues.apache.org/jira/browse/YARN-9063) | ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing |  Major | timelineserver, timelineservice | Tarun Parimi | Ashutosh Gupta |
+| [HDFS-16333](https://issues.apache.org/jira/browse/HDFS-16333) | fix balancer bug when transfer an EC block |  Major | balancer & mover, erasure-coding | qinyuren | qinyuren |
+| [YARN-11020](https://issues.apache.org/jira/browse/YARN-11020) | [UI2] No container is found for an application attempt with a single AM container |  Major | yarn-ui-v2 | Andras Gyori | Andras Gyori |
+| [HDFS-16373](https://issues.apache.org/jira/browse/HDFS-16373) | Fix MiniDFSCluster restart in case of multiple namenodes |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-18048](https://issues.apache.org/jira/browse/HADOOP-18048) | [branch-3.3] Dockerfile\_aarch64 build fails with fatal error: Python.h: No such file or directory |  Major | . | Siyao Meng | Siyao Meng |
+| [HDFS-16377](https://issues.apache.org/jira/browse/HDFS-16377) | Should CheckNotNull before access FsDatasetSpi |  Major | . | tomscut | tomscut |
+| [YARN-6862](https://issues.apache.org/jira/browse/YARN-6862) | Nodemanager resource usage metrics sometimes are negative |  Major | nodemanager | YunFan Zhou | Benjamin Teke |
+| [HADOOP-13500](https://issues.apache.org/jira/browse/HADOOP-13500) | Synchronizing iteration of Configuration properties object |  Major | conf | Jason Darrell Lowe | Dhananjay Badaya |
+| [YARN-10178](https://issues.apache.org/jira/browse/YARN-10178) | Global Scheduler async thread crash caused by 'Comparison method violates its general contract |  Major | capacity scheduler | tuyu | Andras Gyori |
+| [YARN-11053](https://issues.apache.org/jira/browse/YARN-11053) | AuxService should not use class name as default system classes |  Major | auxservices | Cheng Pan | Cheng Pan |
+| [HDFS-16395](https://issues.apache.org/jira/browse/HDFS-16395) | Remove useless NNThroughputBenchmark#dummyActionNoSynch() |  Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu |
+| [HADOOP-18045](https://issues.apache.org/jira/browse/HADOOP-18045) | Disable TestDynamometerInfra |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-14099](https://issues.apache.org/jira/browse/HDFS-14099) | Unknown frame descriptor when decompressing multiple frames in ZStandardDecompressor |  Major | . | xuzq | xuzq |
+| [HADOOP-18063](https://issues.apache.org/jira/browse/HADOOP-18063) | Remove unused import AbstractJavaKeyStoreProvider in Shell class |  Minor | . | JiangHua Zhu | JiangHua Zhu |
+| [HDFS-16409](https://issues.apache.org/jira/browse/HDFS-16409) | Fix typo: testHasExeceptionsReturnsCorrectValue -\> testHasExceptionsReturnsCorrectValue |  Trivial | . | Ashutosh Gupta | Ashutosh Gupta |
+| [HDFS-16408](https://issues.apache.org/jira/browse/HDFS-16408) | Ensure LeaseRecheckIntervalMs is greater than zero |  Major | namenode | Jingxuan Fu | Jingxuan Fu |
+| [HDFS-16410](https://issues.apache.org/jira/browse/HDFS-16410) | Insecure Xml parsing in OfflineEditsXmlLoader |  Minor | . | Ashutosh Gupta | Ashutosh Gupta |
+| [HDFS-16420](https://issues.apache.org/jira/browse/HDFS-16420) | Avoid deleting unique data blocks when deleting redundancy striped blocks |  Critical | ec, erasure-coding | qinyuren | Jackson Wang |
+| [YARN-10561](https://issues.apache.org/jira/browse/YARN-10561) | Upgrade node.js to 12.22.1 and yarn to 1.22.5 in YARN application catalog webapp |  Critical | webapp | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-18096](https://issues.apache.org/jira/browse/HADOOP-18096) | Distcp: Sync moves filtered file to home directory rather than deleting |  Critical | . | Ayush Saxena | Ayush Saxena |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-7342](https://issues.apache.org/jira/browse/MAPREDUCE-7342) | Stop RMService in TestClientRedirect.testRedirect() |  Minor | . | Zhengxi Li | Zhengxi Li |
+| [MAPREDUCE-7311](https://issues.apache.org/jira/browse/MAPREDUCE-7311) | Fix non-idempotent test in TestTaskProgressReporter |  Minor | . | Zhengxi Li | Zhengxi Li |
+| [HADOOP-17936](https://issues.apache.org/jira/browse/HADOOP-17936) | TestLocalFSCopyFromLocal.testDestinationFileIsToParentDirectory failure after reverting HADOOP-16878 |  Major | . | Chao Sun | Chao Sun |
+| [HDFS-15862](https://issues.apache.org/jira/browse/HDFS-15862) | Make TestViewfsWithNfs3.testNfsRenameSingleNN() idempotent |  Minor | nfs | Zhengxi Li | Zhengxi Li |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-10337](https://issues.apache.org/jira/browse/YARN-10337) | TestRMHATimelineCollectors fails on hadoop trunk |  Major | test, yarn | Ahmed Hussein | Bilwa S T |
+| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently |  Major | hdfs | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17424](https://issues.apache.org/jira/browse/HADOOP-17424) | Replace HTrace with No-Op tracer |  Major | . | Siyao Meng | Siyao Meng |
+| [HADOOP-17705](https://issues.apache.org/jira/browse/HADOOP-17705) | S3A to add option fs.s3a.endpoint.region to set AWS region |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17670](https://issues.apache.org/jira/browse/HADOOP-17670) | S3AFS and ABFS to log IOStats at DEBUG mode or optionally at INFO level in close() |  Minor | fs/azure, fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17511](https://issues.apache.org/jira/browse/HADOOP-17511) | Add an Audit plugin point for S3A auditing/context |  Major | . | Steve Loughran | Steve Loughran |
+| [HADOOP-17470](https://issues.apache.org/jira/browse/HADOOP-17470) | Collect more S3A IOStatistics |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-17735](https://issues.apache.org/jira/browse/HADOOP-17735) | Upgrade aws-java-sdk to 1.11.1026 |  Major | build, fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-17547](https://issues.apache.org/jira/browse/HADOOP-17547) | Magic committer to downgrade abort in cleanup if list uploads fails with access denied |  Major | fs/s3 | Steve Loughran | Bogdan Stolojan |
+| [HADOOP-17771](https://issues.apache.org/jira/browse/HADOOP-17771) | S3AFS creation fails "Unable to find a region via the region provider chain." |  Blocker | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-15659](https://issues.apache.org/jira/browse/HDFS-15659) | Set dfs.namenode.redundancy.considerLoad to false in MiniDFSCluster |  Major | test | Akira Ajisaka | Ahmed Hussein |
+| [HADOOP-17774](https://issues.apache.org/jira/browse/HADOOP-17774) | bytesRead FS statistic showing twice the correct value in S3A |  Major | fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17290](https://issues.apache.org/jira/browse/HADOOP-17290) | ABFS: Add Identifiers to Client Request Header |  Major | fs/azure | Sumangala Patki | Sumangala Patki |
+| [HADOOP-17250](https://issues.apache.org/jira/browse/HADOOP-17250) | ABFS: Random read perf improvement |  Major | fs/azure | Sneha Vijayarajan | Mukund Thakur |
+| [HADOOP-17596](https://issues.apache.org/jira/browse/HADOOP-17596) | ABFS: Change default Readahead Queue Depth from num(processors) to const |  Major | fs/azure | Sumangala Patki | Sumangala Patki |
+| [HADOOP-17715](https://issues.apache.org/jira/browse/HADOOP-17715) | ABFS: Append blob tests with non HNS accounts fail |  Minor | . | Sneha Varma | Sneha Varma |
+| [HADOOP-17714](https://issues.apache.org/jira/browse/HADOOP-17714) | ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs |  Minor | test | Sneha Varma | Sneha Varma |
+| [HDFS-16140](https://issues.apache.org/jira/browse/HDFS-16140) | TestBootstrapAliasmap fails by BindException |  Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-13887](https://issues.apache.org/jira/browse/HADOOP-13887) | Encrypt S3A data client-side with AWS SDK (S3-CSE) |  Minor | fs/s3 | Jeeyoung Kim | Mehakmeet Singh |
+| [HADOOP-17458](https://issues.apache.org/jira/browse/HADOOP-17458) | S3A to treat "SdkClientException: Data read has a different length than the expected" as EOFException |  Minor | fs/s3 | Steve Loughran | Bogdan Stolojan |
+| [HADOOP-17628](https://issues.apache.org/jira/browse/HADOOP-17628) | Distcp contract test is really slow with ABFS and S3A; timing out |  Minor | fs/azure, fs/s3, test, tools/distcp | Bilahari T H | Steve Loughran |
+| [HADOOP-17822](https://issues.apache.org/jira/browse/HADOOP-17822) | fs.s3a.acl.default not working after S3A Audit feature added |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-17139](https://issues.apache.org/jira/browse/HADOOP-17139) | Re-enable optimized copyFromLocal implementation in S3AFileSystem |  Minor | fs/s3 | Sahil Takiar | Bogdan Stolojan |
+| [HADOOP-17823](https://issues.apache.org/jira/browse/HADOOP-17823) | S3A Tests to skip if S3Guard and S3-CSE are enabled. |  Major | build, fs/s3 | Mehakmeet Singh | Mehakmeet Singh |
+| [HDFS-16184](https://issues.apache.org/jira/browse/HDFS-16184) | De-flake TestBlockScanner#testSkipRecentAccessFile |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17677](https://issues.apache.org/jira/browse/HADOOP-17677) | Distcp is unable to determine region with S3 PrivateLink endpoints |  Major | fs/s3, tools/distcp | KJ |  |
+| [HDFS-16192](https://issues.apache.org/jira/browse/HDFS-16192) | ViewDistributedFileSystem#rename wrongly using src in the place of dst. |  Major | . | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HADOOP-17156](https://issues.apache.org/jira/browse/HADOOP-17156) | Clear abfs readahead requests on stream close |  Major | fs/azure | Rajesh Balamohan | Mukund Thakur |
+| [HADOOP-17618](https://issues.apache.org/jira/browse/HADOOP-17618) | ABFS: Partially obfuscate SAS object IDs in Logs |  Major | fs/azure | Sumangala Patki | Sumangala Patki |
+| [HADOOP-17894](https://issues.apache.org/jira/browse/HADOOP-17894) | CredentialProviderFactory.getProviders() recursion loading JCEKS file from s3a |  Major | conf, fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17195](https://issues.apache.org/jira/browse/HADOOP-17195) | Intermittent OutOfMemory error while performing hdfs CopyFromLocal to abfs |  Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh |
+| [HADOOP-17929](https://issues.apache.org/jira/browse/HADOOP-17929) | implement non-guava Precondition checkArgument |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17198](https://issues.apache.org/jira/browse/HADOOP-17198) | Support S3 Access Points |  Major | fs/s3 | Steve Loughran | Bogdan Stolojan |
+| [HADOOP-17871](https://issues.apache.org/jira/browse/HADOOP-17871) | S3A CSE: minor tuning |  Minor | fs/s3 | Steve Loughran | Mehakmeet Singh |
+| [HADOOP-17947](https://issues.apache.org/jira/browse/HADOOP-17947) | Provide alternative to Guava VisibleForTesting |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17930](https://issues.apache.org/jira/browse/HADOOP-17930) | implement non-guava Precondition checkState |  Major | . | Ahmed Hussein | Ahmed Hussein |
+| [HADOOP-17374](https://issues.apache.org/jira/browse/HADOOP-17374) | AliyunOSS: support ListObjectsV2 |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-17863](https://issues.apache.org/jira/browse/HADOOP-17863) | ABFS: Fix compiler deprecation warning in TextFileBasedIdentityHandler |  Minor | fs/azure | Sumangala Patki | Sumangala Patki |
+| [HADOOP-17928](https://issues.apache.org/jira/browse/HADOOP-17928) | s3a: set fs.s3a.downgrade.syncable.exceptions = true by default |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-16336](https://issues.apache.org/jira/browse/HDFS-16336) | De-flake TestRollingUpgrade#testRollback |  Minor | hdfs, test | Kevin Wikant | Viraj Jasani |
+| [HDFS-16171](https://issues.apache.org/jira/browse/HDFS-16171) | De-flake testDecommissionStatus |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17226](https://issues.apache.org/jira/browse/HADOOP-17226) | Failure of ITestAssumeRole.testRestrictedCommitActions |  Minor | fs/s3, test | Steve Loughran | Steve Loughran |
+| [HADOOP-14334](https://issues.apache.org/jira/browse/HADOOP-14334) | S3 SSEC  tests to downgrade when running against a mandatory encryption object store |  Minor | fs/s3, test | Steve Loughran | Monthon Klongklaew |
+| [HADOOP-16223](https://issues.apache.org/jira/browse/HADOOP-16223) | remove misleading fs.s3a.delegation.tokens.enabled prompt |  Minor | fs/s3 | Steve Loughran |  |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-16078](https://issues.apache.org/jira/browse/HDFS-16078) | Remove unused parameters for DatanodeManager.handleLifeline() |  Minor | . | tomscut | tomscut |
+| [HDFS-16079](https://issues.apache.org/jira/browse/HDFS-16079) | Improve the block state change log |  Minor | . | tomscut | tomscut |
+| [HDFS-16089](https://issues.apache.org/jira/browse/HDFS-16089) | EC: Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor |  Minor | . | tomscut | tomscut |
+| [HDFS-16298](https://issues.apache.org/jira/browse/HDFS-16298) | Improve error msg for BlockMissingException |  Minor | . | tomscut | tomscut |
+| [HDFS-16312](https://issues.apache.org/jira/browse/HDFS-16312) | Fix typo for DataNodeVolumeMetrics and ProfilingFileIoEvents |  Minor | . | tomscut | tomscut |
+| [HADOOP-18005](https://issues.apache.org/jira/browse/HADOOP-18005) | Correct log format for LdapGroupsMapping |  Minor | . | tomscut | tomscut |
+| [HDFS-16319](https://issues.apache.org/jira/browse/HDFS-16319) | Add metrics doc for ReadLockLongHoldCount and WriteLockLongHoldCount |  Minor | . | tomscut | tomscut |
+| [HDFS-16326](https://issues.apache.org/jira/browse/HDFS-16326) | Simplify the code for DiskBalancer |  Minor | . | tomscut | tomscut |
+| [HDFS-16335](https://issues.apache.org/jira/browse/HDFS-16335) | Fix HDFSCommands.md |  Minor | . | tomscut | tomscut |
+| [HDFS-16339](https://issues.apache.org/jira/browse/HDFS-16339) | Show the threshold when mover threads quota is exceeded |  Minor | . | tomscut | tomscut |
+| [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | Make GetClusterNodesRequestPBImpl thread safe |  Major | client | Prabhu Joseph | SwathiChandrashekar |
+| [HADOOP-17808](https://issues.apache.org/jira/browse/HADOOP-17808) | ipc.Client not setting interrupt flag after catching InterruptedException |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17834](https://issues.apache.org/jira/browse/HADOOP-17834) | Bump aliyun-sdk-oss to 3.13.0 |  Major | . | Siyao Meng | Siyao Meng |
+| [HADOOP-17950](https://issues.apache.org/jira/browse/HADOOP-17950) | Provide replacement for deprecated APIs of commons-io IOUtils |  Major | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-17955](https://issues.apache.org/jira/browse/HADOOP-17955) | Bump netty to the latest 4.1.68 |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HADOOP-17946](https://issues.apache.org/jira/browse/HADOOP-17946) | Update commons-lang to latest 3.x |  Minor | . | Sean Busbey | Renukaprasad C |
+| [HDFS-16323](https://issues.apache.org/jira/browse/HDFS-16323) | DatanodeHttpServer doesn't require handler state map while retrieving filter handlers |  Minor | . | Viraj Jasani | Viraj Jasani |
+| [HADOOP-13464](https://issues.apache.org/jira/browse/HADOOP-13464) | update GSON to 2.7+ |  Minor | build | Sean Busbey | Igor Dvorzhak |
+| [HADOOP-18061](https://issues.apache.org/jira/browse/HADOOP-18061) | Update the year to 2022 |  Major | . | Ayush Saxena | Ayush Saxena |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md
new file mode 100644
index 0000000000000..9948d8ff3222c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.2/RELEASENOTES.3.3.2.md
@@ -0,0 +1,93 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.3.2 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) | *Major* | **Add Available Space Rack Fault Tolerant BPP**
+
+Added a new BlockPlacementPolicy: "AvailableSpaceRackFaultTolerantBlockPlacementPolicy" which uses the same optimization logic as the AvailableSpaceBlockPlacementPolicy along with spreading the replicas across maximum number of racks, similar to BlockPlacementPolicyRackFaultTolerant.
+The BPP can be configured by setting the blockplacement policy class as org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy
+
+
+---
+
+* [HADOOP-17424](https://issues.apache.org/jira/browse/HADOOP-17424) | *Major* | **Replace HTrace with No-Op tracer**
+
+Dependency on HTrace and TraceAdmin protocol/utility were removed. Tracing functionality is no-op until alternative tracer implementation is added.
+
+
+---
+
+* [HDFS-15814](https://issues.apache.org/jira/browse/HDFS-15814) | *Major* | **Make some parameters configurable for DataNodeDiskMetrics**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | *Major* | **Make GetClusterNodesRequestPBImpl thread safe**
+
+Added syncronization so that the "yarn node list" command does not fail intermittently
+
+
+---
+
+* [HADOOP-13887](https://issues.apache.org/jira/browse/HADOOP-13887) | *Minor* | **Encrypt S3A data client-side with AWS SDK (S3-CSE)**
+
+Adds support for client side encryption in AWS S3,
+with keys managed by AWS-KMS.
+
+Read the documentation in encryption.md very, very carefully before
+use and consider it unstable.
+
+S3-CSE is enabled in the existing configuration option
+"fs.s3a.server-side-encryption-algorithm":
+
+fs.s3a.server-side-encryption-algorithm=CSE-KMS
+fs.s3a.server-side-encryption.key=\<KMS\_KEY\_ID\>
+
+You cannot enable CSE and SSE in the same client, although
+you can still enable a default SSE option in the S3 console.
+
+\* Not compatible with S3Guard.   
+\* Filesystem list/get status operations subtract 16 bytes from the length
+  of all files \>= 16 bytes long to compensate for the padding which CSE
+  adds.
+\* The SDK always warns about the specific algorithm chosen being
+  deprecated. It is critical to use this algorithm for ranged
+  GET requests to work (i.e. random IO). Ignore.
+\* Unencrypted files CANNOT BE READ.
+  The entire bucket SHOULD be encrypted with S3-CSE.
+\* Uploading files may be a bit slower as blocks are now
+  written sequentially.
+\* The Multipart Upload API is disabled when S3-CSE is active.
+
+
+---
+
+* [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | *Critical* | **Improve RM system metrics publisher's performance by pushing events to timeline server in batch**
+
+When Timeline Service V1 or V1.5 is used, if "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch" is set to true, ResourceManager sends timeline events in batch. The default value is false. If this functionality is enabled, the maximum number that events published in batch is configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size". The default value is 1000. The interval of publishing events can be configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds". By default, it is set to 60 seconds.
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/CHANGELOG.3.3.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/CHANGELOG.3.3.3.md
new file mode 100644
index 0000000000000..bf9498db08172
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/CHANGELOG.3.3.3.md
@@ -0,0 +1,65 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.3.3 - 2022-05-09
+
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-7373](https://issues.apache.org/jira/browse/MAPREDUCE-7373) | Building MapReduce NativeTask fails on Fedora 34+ |  Major | build, nativetask | Kengo Seki | Kengo Seki |
+| [HDFS-16355](https://issues.apache.org/jira/browse/HDFS-16355) | Improve the description of dfs.block.scanner.volume.bytes.per.second |  Minor | documentation, hdfs | guophilipse | guophilipse |
+| [HADOOP-18155](https://issues.apache.org/jira/browse/HADOOP-18155) | Refactor tests in TestFileUtil |  Trivial | common | Gautham Banasandra | Gautham Banasandra |
+| [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | Replace log4j 1.x with reload4j |  Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-16501](https://issues.apache.org/jira/browse/HDFS-16501) | Print the exception when reporting a bad block |  Major | datanode | qinyuren | qinyuren |
+| [HADOOP-18214](https://issues.apache.org/jira/browse/HADOOP-18214) | Update BUILDING.txt |  Minor | build, documentation | Steve Loughran |  |
+| [HDFS-16556](https://issues.apache.org/jira/browse/HDFS-16556) | Fix typos in distcp |  Minor | documentation | guophilipse | guophilipse |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-17341](https://issues.apache.org/jira/browse/HADOOP-17341) | Upgrade commons-codec to 1.15 |  Minor | . | Dongjoon Hyun | Dongjoon Hyun |
+| [HADOOP-17650](https://issues.apache.org/jira/browse/HADOOP-17650) | Fails to build using Maven 3.8.1 |  Major | build | Wei-Chiu Chuang | Viraj Jasani |
+| [HADOOP-18178](https://issues.apache.org/jira/browse/HADOOP-18178) | Upgrade jackson to 2.13.2 and jackson-databind to 2.13.2.2 |  Major | . | PJ Fanning | PJ Fanning |
+| [HDFS-16535](https://issues.apache.org/jira/browse/HDFS-16535) | SlotReleaser should reuse the domain socket based on socket paths |  Major | hdfs-client | Quanlong Huang |  |
+| [HADOOP-18109](https://issues.apache.org/jira/browse/HADOOP-18109) | Ensure that default permissions of directories under internal ViewFS directories are the same as directories on target filesystems |  Major | viewfs | Chentao Yu | Chentao Yu |
+| [HDFS-16422](https://issues.apache.org/jira/browse/HDFS-16422) | Fix thread safety of EC decoding during concurrent preads |  Critical | dfsclient, ec, erasure-coding | daimin | daimin |
+| [HDFS-16437](https://issues.apache.org/jira/browse/HDFS-16437) | ReverseXML processor doesn't accept XML files without the SnapshotDiffSection. |  Critical | hdfs | yanbin.zhang | yanbin.zhang |
+| [HDFS-16507](https://issues.apache.org/jira/browse/HDFS-16507) | [SBN read] Avoid purging edit log which is in progress |  Critical | . | Tao Li | Tao Li |
+| [YARN-10720](https://issues.apache.org/jira/browse/YARN-10720) | YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging |  Critical | . | Qi Zhu | Qi Zhu |
+| [HDFS-16428](https://issues.apache.org/jira/browse/HDFS-16428) | Source path with storagePolicy cause wrong typeConsumed while rename |  Major | hdfs, namenode | lei w | lei w |
+| [YARN-11014](https://issues.apache.org/jira/browse/YARN-11014) | YARN incorrectly validates maximum capacity resources on the validation API |  Major | . | Benjamin Teke | Benjamin Teke |
+| [YARN-11075](https://issues.apache.org/jira/browse/YARN-11075) | Explicitly declare serialVersionUID in LogMutation class |  Major | . | Benjamin Teke | Benjamin Teke |
+| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice |  Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-18160](https://issues.apache.org/jira/browse/HADOOP-18160) | \`org.wildfly.openssl\` should not be shaded by Hadoop build |  Major | build | André F. | André F. |
+| [HADOOP-18202](https://issues.apache.org/jira/browse/HADOOP-18202) | create-release fails fatal: unsafe repository ('/build/source' is owned by someone else) |  Major | build | Steve Loughran | Steve Loughran |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-18125](https://issues.apache.org/jira/browse/HADOOP-18125) | Utility to identify git commit / Jira fixVersion discrepancies for RC preparation |  Major | . | Viraj Jasani | Viraj Jasani |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/RELEASENOTES.3.3.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/RELEASENOTES.3.3.3.md
new file mode 100644
index 0000000000000..bd9ea8a2e49d6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.3/RELEASENOTES.3.3.3.md
@@ -0,0 +1,48 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.3.3 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HADOOP-17650](https://issues.apache.org/jira/browse/HADOOP-17650) | *Major* | **Fails to build using Maven 3.8.1**
+
+In order to resolve build issues with Maven 3.8.1, we have to bump SolrJ to latest version 8.8.2 as of now. Solr is used by YARN application catalog. Hence, we would recommend upgrading Solr cluster accordingly before upgrading entire Hadoop cluster to 3.4.0 if the YARN application catalog service is used.
+
+
+---
+
+* [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | *Major* | **Replace log4j 1.x with reload4j**
+
+log4j 1 was replaced with reload4j which is fork of log4j 1.2.17 with the goal of fixing pressing security issues.
+
+If you are depending on the hadoop artifacts in your build were explicitly excluding log4 artifacts, and now want to exclude the reload4j files, you will need to update your exclusion lists
+\<exclusion\>
+  \<groupId\>org.slf4j\</groupId\>
+  \<artifactId\>slf4j-reload4j\</artifactId\>
+\</exclusion\>
+\<exclusion\>
+  \<groupId\>ch.qos.reload4j\</groupId\>
+  \<artifactId\>reload4j\</artifactId\>
+\</exclusion\>
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md
new file mode 100644
index 0000000000000..78b805240c78e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md
@@ -0,0 +1,56 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.3.4 - 2022-07-29
+
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-18044](https://issues.apache.org/jira/browse/HADOOP-18044) | Hadoop - Upgrade to JQuery 3.6.0 |  Major | . | Yuan Luo | Yuan Luo |
+| [YARN-11195](https://issues.apache.org/jira/browse/YARN-11195) | Document how to configure NUMA in YARN |  Major | documentation | Prabhu Joseph | Samrat Deb |
+| [HADOOP-18332](https://issues.apache.org/jira/browse/HADOOP-18332) | Remove rs-api dependency by downgrading jackson to 2.12.7 |  Major | build | PJ Fanning | PJ Fanning |
+| [HADOOP-18354](https://issues.apache.org/jira/browse/HADOOP-18354) | Upgrade reload4j to 1.2.22 due to XXE vulnerability |  Major | . | PJ Fanning | PJ Fanning |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-18085](https://issues.apache.org/jira/browse/HADOOP-18085) | S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation |  Major | fs/s3, test | Bogdan Stolojan | Bogdan Stolojan |
+| [YARN-11092](https://issues.apache.org/jira/browse/YARN-11092) | Upgrade jquery ui to 1.13.1 |  Major | . | D M Murali Krishna Reddy | groot |
+| [HDFS-16453](https://issues.apache.org/jira/browse/HDFS-16453) | Upgrade okhttp from 2.7.5 to 4.9.3 |  Major | hdfs-client | Ivan Viaznikov | groot |
+| [YARN-10974](https://issues.apache.org/jira/browse/YARN-10974) | CS UI: queue filter and openQueues param do not work as expected |  Major | capacity scheduler | Chengbing Liu | Chengbing Liu |
+| [HADOOP-18237](https://issues.apache.org/jira/browse/HADOOP-18237) | Upgrade Apache Xerces Java to 2.12.2 |  Major | build | groot | groot |
+| [HADOOP-18074](https://issues.apache.org/jira/browse/HADOOP-18074) | Partial/Incomplete groups list can be returned in LDAP groups lookup |  Major | security | Philippe Lanoe | Larry McCay |
+| [HADOOP-18079](https://issues.apache.org/jira/browse/HADOOP-18079) | Upgrade Netty to 4.1.77.Final |  Major | build | Renukaprasad C | Wei-Chiu Chuang |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-18068](https://issues.apache.org/jira/browse/HADOOP-18068) | Upgrade AWS SDK to 1.12.132 |  Major | build, fs/s3 | Steve Loughran | Steve Loughran |
+| [HADOOP-18307](https://issues.apache.org/jira/browse/HADOOP-18307) | remove hadoop-cos as a dependency of hadoop-cloud-storage |  Major | bulid, fs | Steve Loughran | Steve Loughran |
+| [HADOOP-18344](https://issues.apache.org/jira/browse/HADOOP-18344) | AWS SDK update to 1.12.262 to address jackson  CVE-2018-7489 |  Major | fs/s3 | Steve Loughran | Steve Loughran |
+
+
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md
new file mode 100644
index 0000000000000..79573880423d6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md
@@ -0,0 +1,66 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop  3.3.4 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-16453](https://issues.apache.org/jira/browse/HDFS-16453) | *Major* | **Upgrade okhttp from 2.7.5 to 4.9.3**
+
+okhttp has been updated to address CVE-2021-0341
+
+
+---
+
+* [HADOOP-18237](https://issues.apache.org/jira/browse/HADOOP-18237) | *Major* | **Upgrade Apache Xerces Java to 2.12.2**
+
+Apache Xerces has been updated  to 2.12.2 to fix CVE-2022-23437
+
+
+---
+
+* [HADOOP-18307](https://issues.apache.org/jira/browse/HADOOP-18307) | *Major* | **remove hadoop-cos as a dependency of hadoop-cloud-storage**
+
+We have recently become aware that libraries which include a shaded apache httpclient libraries (hadoop-client-runtime.jar, aws-java-sdk-bundle.jar, gcs-connector-shaded.jar, cos\_api-bundle-5.6.19.jar) all load and use the unshaded resource mozilla/public-suffix-list.txt. If an out of date version of this is found on the classpath first, attempts to negotiate TLS connections may fail with the error "Certificate doesn't match any of the subject alternative names". This release does not declare the hadoop-cos library to be a dependency of the hadoop-cloud-storage POM, so applications depending on that module are no longer exposed to this issue. If an application requires use of the hadoop-cos module, please declare an explicit dependency.
+
+
+---
+
+* [HADOOP-18332](https://issues.apache.org/jira/browse/HADOOP-18332) | *Major* | **Remove rs-api dependency by downgrading jackson to 2.12.7**
+
+Downgrades Jackson from 2.13.2 to 2.12.7 to fix class conflicts in downstream projects. This version of jackson does contain the fix for CVE-2020-36518.
+
+
+---
+
+* [HADOOP-18079](https://issues.apache.org/jira/browse/HADOOP-18079) | *Major* | **Upgrade Netty to 4.1.77.Final**
+
+Netty has been updated to address CVE-2019-20444, CVE-2019-20445 and CVE-2022-24823
+
+
+---
+
+* [HADOOP-18344](https://issues.apache.org/jira/browse/HADOOP-18344) | *Major* | **AWS SDK update to 1.12.262 to address jackson  CVE-2018-7489**
+
+The AWS SDK has been updated to 1.12.262 to address jackson CVE-2018-7489
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
index ada4cd80e4882..f80c62535a1f0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
@@ -24,6 +24,8 @@
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.XMLUtils;
+
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -34,7 +36,6 @@
 import org.xml.sax.helpers.DefaultHandler;
 
 import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
 import java.io.File;
 import java.util.ArrayList;
 
@@ -76,7 +77,7 @@ protected void readTestConfigFile() {
       boolean success = false;
       testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
       try {
-        SAXParser p = (SAXParserFactory.newInstance()).newSAXParser();
+        SAXParser p = XMLUtils.newSecureSAXParserFactory().newSAXParser();
         p.parse(testConfigFile, getConfigParser());
         success = true;
       } catch (Exception e) {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
index 5ef129cdc87ed..2ccfcfebb27e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
@@ -23,7 +23,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
-import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.ArrayList;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 9fcf4a5eb55a2..74b2f55065dec 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -21,21 +21,17 @@
 import java.util.HashSet;
 
 import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
-import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.ftp.FtpConfigKeys;
 import org.apache.hadoop.fs.local.LocalConfigKeys;
 import org.apache.hadoop.ha.SshFenceByTcpPort;
 import org.apache.hadoop.ha.ZKFailoverController;
-import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.io.erasurecode.CodecUtil;
-import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.security.CompositeGroupsMapping;
 import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
 import org.apache.hadoop.security.LdapGroupsMapping;
 import org.apache.hadoop.security.RuleBasedLdapGroupsMapping;
-import org.apache.hadoop.security.http.CrossOriginFilter;
 import org.apache.hadoop.security.ssl.SSLFactory;
 
 /**
@@ -80,9 +76,9 @@ public void initializeMemberVariables() {
         };
 
     // Initialize used variables
-    xmlPropsToSkipCompare = new HashSet<String>();
-    xmlPrefixToSkipCompare = new HashSet<String>();
-    configurationPropsToSkipCompare = new HashSet<String>();
+    xmlPropsToSkipCompare = new HashSet<>();
+    xmlPrefixToSkipCompare = new HashSet<>();
+    configurationPropsToSkipCompare = new HashSet<>();
 
     // Set error modes
     errorIfMissingConfigProps = true;
@@ -139,7 +135,6 @@ public void initializeMemberVariables() {
     xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
     xmlPropsToSkipCompare.
         add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
-    xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
     xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
     xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
 
@@ -227,8 +222,7 @@ public void initializeMemberVariables() {
     xmlPropsToSkipCompare.add("hadoop.common.configuration.version");
     // - org.apache.hadoop.fs.FileSystem
     xmlPropsToSkipCompare.add("fs.har.impl.disable.cache");
-    // - org.apache.hadoop.fs.FileSystem#getFileSystemClass()
-    xmlPropsToSkipCompare.add("fs.swift.impl");
+
     // - package org.apache.hadoop.tracing.TraceUtils ?
     xmlPropsToSkipCompare.add("hadoop.htrace.span.receiver.classes");
     // Private keys
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index eae9a1fc744c0..6db47d6d22fda 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -41,9 +41,12 @@
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 
 import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.util.XMLUtils;
+
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
+
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.mock;
 import static org.junit.Assert.*;
@@ -59,6 +62,7 @@ public class TestConfServlet {
       new HashMap<String, String>();
   private static final Map<String, String> TEST_FORMATS =
       new HashMap<String, String>();
+  private static final Map<String, String> MASK_PROPERTIES = new HashMap<>();
 
   @BeforeClass
   public static void initTestProperties() {
@@ -67,6 +71,8 @@ public static void initTestProperties() {
     TEST_PROPERTIES.put("test.key3", "value3");
     TEST_FORMATS.put(ConfServlet.FORMAT_XML, "application/xml");
     TEST_FORMATS.put(ConfServlet.FORMAT_JSON, "application/json");
+    MASK_PROPERTIES.put("yarn.federation.state-store.sql.username", "admin");
+    MASK_PROPERTIES.put("yarn.federation.state-store.sql.password", "123456");
   }
 
   private Configuration getTestConf() {
@@ -80,6 +86,9 @@ private Configuration getMultiPropertiesConf() {
     for(String key : TEST_PROPERTIES.keySet()) {
       testConf.set(key, TEST_PROPERTIES.get(key));
     }
+    for(String key : MASK_PROPERTIES.keySet()) {
+      testConf.set(key, MASK_PROPERTIES.get(key));
+    }
     return testConf;
   }
 
@@ -217,8 +226,7 @@ public void testWriteXml() throws Exception {
     ConfServlet.writeResponse(getTestConf(), sw, "xml");
     String xml = sw.toString();
 
-    DocumentBuilderFactory docBuilderFactory 
-      = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
     Document doc = builder.parse(new InputSource(new StringReader(xml)));
     NodeList nameNodes = doc.getElementsByTagName("name");
@@ -247,4 +255,63 @@ public void testBadFormat() throws Exception {
     }
     assertEquals("", sw.toString());
   }
+
+  private void verifyReplaceProperty(Configuration conf, String format,
+      String propertyName) throws Exception {
+    StringWriter sw = null;
+    PrintWriter pw = null;
+    ConfServlet service = null;
+    try {
+      service = new ConfServlet();
+      ServletConfig servletConf = mock(ServletConfig.class);
+      ServletContext context = mock(ServletContext.class);
+      service.init(servletConf);
+      when(context.getAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE)).thenReturn(conf);
+      when(service.getServletContext()).thenReturn(context);
+
+      HttpServletRequest request = mock(HttpServletRequest.class);
+      when(request.getHeader(HttpHeaders.ACCEPT)).thenReturn(TEST_FORMATS.get(format));
+      when(request.getParameter("name")).thenReturn(propertyName);
+
+      HttpServletResponse response = mock(HttpServletResponse.class);
+      sw = new StringWriter();
+      pw = new PrintWriter(sw);
+      when(response.getWriter()).thenReturn(pw);
+
+      // response request
+      service.doGet(request, response);
+      String result = sw.toString().trim();
+
+      // For example, for the property yarn.federation.state-store.sql.username,
+      // we set the value to test-user,
+      // which should be replaced by a mask, which should be ******
+      // MASK_PROPERTIES.get("property yarn.federation.state-store.sql.username")
+      // is the value before replacement, test-user
+      // result contains the replaced value, which should be ******
+      assertTrue(result.contains(propertyName));
+      assertFalse(result.contains(MASK_PROPERTIES.get(propertyName)));
+
+    } finally {
+      if (sw != null) {
+        sw.close();
+      }
+      if (pw != null) {
+        pw.close();
+      }
+      if (service != null) {
+        service.destroy();
+      }
+    }
+  }
+
+  @Test
+  public void testReplaceProperty() throws Exception {
+    Configuration configurations = getMultiPropertiesConf();
+
+    for(String format : TEST_FORMATS.keySet()) {
+      for(String key : MASK_PROPERTIES.keySet()) {
+        verifyReplaceProperty(configurations, format, key);
+      }
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 3748eed12246b..b3487ef309fc9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -38,6 +38,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.ConcurrentModificationException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -45,6 +46,7 @@
 import java.util.Properties;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import static java.util.concurrent.TimeUnit.*;
 
@@ -444,6 +446,17 @@ public void testVariableSubstitution() throws IOException {
     assertTrue(mock.getInt("my.int", -1) == 42);
   }
 
+  /**
+   * Checks if variable substitution is accessible via a public API.
+   */
+  @Test
+  public void testCommonVariableSubstitution() {
+    conf.set("intvar", String.valueOf(42));
+    String intVar = conf.substituteCommonVariables("${intvar}");
+
+    assertEquals("42", intVar);
+  }
+
   @Test
   public void testEnvDefault() throws IOException {
     Configuration mock = Mockito.spy(conf);
@@ -2676,4 +2689,31 @@ private static Configuration checkCDATA(byte[] bytes) {
     assertEquals("  prefix >cdata\nsuffix  ", conf.get("cdata-whitespace"));
     return conf;
   }
+
+  @Test
+  public void testConcurrentModificationDuringIteration() throws InterruptedException {
+    Configuration configuration = new Configuration();
+    new Thread(() -> {
+      while (true) {
+        configuration.set(String.valueOf(Math.random()), String.valueOf(Math.random()));
+      }
+    }).start();
+
+    AtomicBoolean exceptionOccurred = new AtomicBoolean(false);
+
+    new Thread(() -> {
+      while (true) {
+        try {
+          configuration.iterator();
+        } catch (final ConcurrentModificationException e) {
+          exceptionOccurred.set(true);
+          break;
+        }
+      }
+    }).start();
+
+    Thread.sleep(1000); //give enough time for threads to run
+
+    assertFalse("ConcurrentModificationException occurred", exceptionOccurred.get());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
index 2c0d6025f2688..83837862ac47e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationDeprecation.java
@@ -30,7 +30,6 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Future;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
index 8453889b53a5a..072baf188de72 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
@@ -25,7 +25,6 @@
 import java.io.OutputStream;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.LocalFileSystem;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
index cb6a1fb31e6c1..b0c3b0900221b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
@@ -36,6 +36,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -62,6 +63,8 @@ public void testParseVersionName() throws Exception {
     } catch (IOException e) {
       assertTrue(true);
     }
+    intercept(NullPointerException.class, () ->
+        KeyProvider.getBaseName(null));
   }
 
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
index 5da973c6a761d..4805fca1d49f4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.crypto.key;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Queue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeoutException;
@@ -32,7 +34,6 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import org.apache.hadoop.util.Sets;
 
 public class TestValueQueue {
   Logger LOG = LoggerFactory.getLogger(TestValueQueue.class);
@@ -103,10 +104,10 @@ public void testWarmUp() throws Exception {
     Assert.assertEquals(5, fillInfos[0].num);
     Assert.assertEquals(5, fillInfos[1].num);
     Assert.assertEquals(5, fillInfos[2].num);
-    Assert.assertEquals(Sets.newHashSet("k1", "k2", "k3"),
-        Sets.newHashSet(fillInfos[0].key,
+    Assert.assertEquals(new HashSet<>(Arrays.asList("k1", "k2", "k3")),
+        new HashSet<>(Arrays.asList(fillInfos[0].key,
             fillInfos[1].key,
-            fillInfos[2].key));
+            fillInfos[2].key)));
     vq.shutdown();
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
index 9122389bd6e54..3bc96c3e2fce0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -31,6 +31,7 @@
 import java.io.IOException;
 import java.net.ConnectException;
 import java.net.NoRouteToHostException;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.net.UnknownHostException;
@@ -38,9 +39,12 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
 
 import org.apache.hadoop.conf.Configuration;
@@ -63,7 +67,6 @@
 import org.junit.rules.Timeout;
 import org.mockito.Mockito;
 
-import org.apache.hadoop.util.Sets;
 
 public class TestLoadBalancingKMSClientProvider {
 
@@ -84,8 +87,8 @@ public void testCreation() throws Exception {
     KMSClientProvider[] providers =
         ((LoadBalancingKMSClientProvider) kp).getProviders();
     assertEquals(1, providers.length);
-    assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/"),
-        Sets.newHashSet(providers[0].getKMSUrl()));
+    assertEquals(new HashSet<>(Collections.singleton("http://host1:9600/kms/foo/v1/")),
+        new HashSet<>(Collections.singleton(providers[0].getKMSUrl())));
 
     kp = new KMSClientProvider.Factory().createProvider(new URI(
         "kms://http@host1;host2;host3:9600/kms/foo"), conf);
@@ -93,12 +96,12 @@ public void testCreation() throws Exception {
     providers =
         ((LoadBalancingKMSClientProvider) kp).getProviders();
     assertEquals(3, providers.length);
-    assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/",
+    assertEquals(new HashSet<>(Arrays.asList("http://host1:9600/kms/foo/v1/",
         "http://host2:9600/kms/foo/v1/",
-        "http://host3:9600/kms/foo/v1/"),
-        Sets.newHashSet(providers[0].getKMSUrl(),
+        "http://host3:9600/kms/foo/v1/")),
+        new HashSet<>(Arrays.asList(providers[0].getKMSUrl(),
             providers[1].getKMSUrl(),
-            providers[2].getKMSUrl()));
+            providers[2].getKMSUrl())));
 
     kp = new KMSClientProvider.Factory().createProvider(new URI(
         "kms://http@host1;host2;host3:9600/kms/foo"), conf);
@@ -106,12 +109,12 @@ public void testCreation() throws Exception {
     providers =
         ((LoadBalancingKMSClientProvider) kp).getProviders();
     assertEquals(3, providers.length);
-    assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/",
+    assertEquals(new HashSet<>(Arrays.asList("http://host1:9600/kms/foo/v1/",
         "http://host2:9600/kms/foo/v1/",
-        "http://host3:9600/kms/foo/v1/"),
-        Sets.newHashSet(providers[0].getKMSUrl(),
+        "http://host3:9600/kms/foo/v1/")),
+        new HashSet<>(Arrays.asList(providers[0].getKMSUrl(),
             providers[1].getKMSUrl(),
-            providers[2].getKMSUrl()));
+            providers[2].getKMSUrl())));
   }
 
   @Test
@@ -707,16 +710,18 @@ public void testClientRetriesWithSSLHandshakeExceptionSucceedsSecondTime()
       throws Exception {
     Configuration conf = new Configuration();
     conf.setInt(
-        CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+        CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 5);
     final String keyName = "test";
     KMSClientProvider p1 = mock(KMSClientProvider.class);
     when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
         .thenThrow(new SSLHandshakeException("p1"))
+        .thenThrow(new SSLException("p1"))
         .thenReturn(new KMSClientProvider.KMSKeyVersion(keyName, "v1",
             new byte[0]));
     KMSClientProvider p2 = mock(KMSClientProvider.class);
     when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
-        .thenThrow(new ConnectException("p2"));
+        .thenThrow(new ConnectException("p2"))
+        .thenThrow(new SocketException("p1"));
 
     when(p1.getKMSUrl()).thenReturn("p1");
     when(p2.getKMSUrl()).thenReturn("p2");
@@ -725,9 +730,9 @@ public void testClientRetriesWithSSLHandshakeExceptionSucceedsSecondTime()
         new KMSClientProvider[] {p1, p2}, 0, conf);
 
     kp.createKey(keyName, new Options(conf));
-    verify(p1, Mockito.times(2)).createKey(Mockito.eq(keyName),
+    verify(p1, Mockito.times(3)).createKey(Mockito.eq(keyName),
         Mockito.any(Options.class));
-    verify(p2, Mockito.times(1)).createKey(Mockito.eq(keyName),
+    verify(p2, Mockito.times(2)).createKey(Mockito.eq(keyName),
         Mockito.any(Options.class));
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
index 2ea45231a13a1..6448a9a2fba73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
@@ -22,7 +22,7 @@
 
 import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+
 import org.junit.Assume;
 import org.junit.Test;
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 2aa5407c056af..dc12f44fc2758 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -178,7 +178,8 @@ public Boolean get() {
    * 
    * @param stats
    */
-  protected abstract void verifyWrittenBytes(Statistics stats);
+  protected abstract void verifyWrittenBytes(Statistics stats)
+      throws IOException;
   
   /**
    * Returns the filesystem uri. Should be set
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestCommandFormat.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestCommandFormat.java
index 4b855c4940440..084c6a0aef83d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestCommandFormat.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestCommandFormat.java
@@ -43,9 +43,9 @@ public class TestCommandFormat {
   
   @Before
   public void setUp() {
-    args = new ArrayList<String>();
-    expectedOpts = new HashSet<String>();
-    expectedArgs = new ArrayList<String>();
+    args = new ArrayList<>();
+    expectedOpts = new HashSet<>();
+    expectedArgs = new ArrayList<>();
   }
 
   @Test
@@ -205,6 +205,6 @@ private static List<String> listOf(String ... objects) {
   }
   
   private static Set<String> setOf(String ... objects) {
-    return new HashSet<String>(listOf(objects));
+    return new HashSet<>(listOf(objects));
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 3a88b7352f1a6..c884e223365c9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs;
 
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -41,27 +42,31 @@
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.jar.Attributes;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.tools.tar.TarEntry;
 import org.apache.tools.tar.TarOutputStream;
+
+import org.assertj.core.api.Assertions;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -157,13 +162,12 @@ public void setup() throws IOException {
     FileUtils.forceMkdir(dir1);
     FileUtils.forceMkdir(dir2);
 
-    new File(del, FILE).createNewFile();
-    File tmpFile = new File(tmp, FILE);
-    tmpFile.createNewFile();
+    Verify.createNewFile(new File(del, FILE));
+    File tmpFile = Verify.createNewFile(new File(tmp, FILE));
 
     // create files
-    new File(dir1, FILE).createNewFile();
-    new File(dir2, FILE).createNewFile();
+    Verify.createNewFile(new File(dir1, FILE));
+    Verify.createNewFile(new File(dir2, FILE));
 
     // create a symlink to file
     File link = new File(del, LINK);
@@ -172,7 +176,7 @@ public void setup() throws IOException {
     // create a symlink to dir
     File linkDir = new File(del, "tmpDir");
     FileUtil.symLink(tmp.toString(), linkDir.toString());
-    Assert.assertEquals(5, del.listFiles().length);
+    Assert.assertEquals(5, Objects.requireNonNull(del.listFiles()).length);
 
     // create files in partitioned directories
     createFile(partitioned, "part-r-00000", "foo");
@@ -199,13 +203,9 @@ public void tearDown() throws IOException {
   private File createFile(File directory, String name, String contents)
       throws IOException {
     File newFile = new File(directory, name);
-    PrintWriter pw = new PrintWriter(newFile);
-    try {
+    try (PrintWriter pw = new PrintWriter(newFile)) {
       pw.println(contents);
     }
-    finally {
-      pw.close();
-    }
     return newFile;
   }
 
@@ -217,11 +217,11 @@ public void testListFiles() throws IOException {
 
     //Test existing directory with no files case 
     File newDir = new File(tmp.getPath(),"test");
-    newDir.mkdir();
+    Verify.mkdir(newDir);
     Assert.assertTrue("Failed to create test dir", newDir.exists());
     files = FileUtil.listFiles(newDir);
     Assert.assertEquals(0, files.length);
-    newDir.delete();
+    assertTrue(newDir.delete());
     Assert.assertFalse("Failed to delete test dir", newDir.exists());
     
     //Test non-existing directory case, this throws 
@@ -243,11 +243,11 @@ public void testListAPI() throws IOException {
 
     //Test existing directory with no files case 
     File newDir = new File(tmp.getPath(),"test");
-    newDir.mkdir();
+    Verify.mkdir(newDir);
     Assert.assertTrue("Failed to create test dir", newDir.exists());
     files = FileUtil.list(newDir);
     Assert.assertEquals("New directory unexpectedly contains files", 0, files.length);
-    newDir.delete();
+    assertTrue(newDir.delete());
     Assert.assertFalse("Failed to delete test dir", newDir.exists());
     
     //Test non-existing directory case, this throws 
@@ -265,7 +265,7 @@ public void testListAPI() throws IOException {
   public void testFullyDelete() throws IOException {
     boolean ret = FileUtil.fullyDelete(del);
     Assert.assertTrue(ret);
-    Assert.assertFalse(del.exists());
+    Verify.notExists(del);
     validateTmpDir();
   }
 
@@ -278,13 +278,13 @@ public void testFullyDelete() throws IOException {
   @Test (timeout = 30000)
   public void testFullyDeleteSymlinks() throws IOException {
     File link = new File(del, LINK);
-    Assert.assertEquals(5, del.list().length);
+    assertDelListLength(5);
     // Since tmpDir is symlink to tmp, fullyDelete(tmpDir) should not
     // delete contents of tmp. See setupDirs for details.
     boolean ret = FileUtil.fullyDelete(link);
     Assert.assertTrue(ret);
-    Assert.assertFalse(link.exists());
-    Assert.assertEquals(4, del.list().length);
+    Verify.notExists(link);
+    assertDelListLength(4);
     validateTmpDir();
 
     File linkDir = new File(del, "tmpDir");
@@ -292,8 +292,8 @@ public void testFullyDeleteSymlinks() throws IOException {
     // delete contents of tmp. See setupDirs for details.
     ret = FileUtil.fullyDelete(linkDir);
     Assert.assertTrue(ret);
-    Assert.assertFalse(linkDir.exists());
-    Assert.assertEquals(3, del.list().length);
+    Verify.notExists(linkDir);
+    assertDelListLength(3);
     validateTmpDir();
   }
 
@@ -309,16 +309,16 @@ public void testFullyDeleteDanglingSymlinks() throws IOException {
     // to make y as a dangling link to file tmp/x
     boolean ret = FileUtil.fullyDelete(tmp);
     Assert.assertTrue(ret);
-    Assert.assertFalse(tmp.exists());
+    Verify.notExists(tmp);
 
     // dangling symlink to file
     File link = new File(del, LINK);
-    Assert.assertEquals(5, del.list().length);
+    assertDelListLength(5);
     // Even though 'y' is dangling symlink to file tmp/x, fullyDelete(y)
     // should delete 'y' properly.
     ret = FileUtil.fullyDelete(link);
     Assert.assertTrue(ret);
-    Assert.assertEquals(4, del.list().length);
+    assertDelListLength(4);
 
     // dangling symlink to directory
     File linkDir = new File(del, "tmpDir");
@@ -326,22 +326,22 @@ public void testFullyDeleteDanglingSymlinks() throws IOException {
     // delete tmpDir properly.
     ret = FileUtil.fullyDelete(linkDir);
     Assert.assertTrue(ret);
-    Assert.assertEquals(3, del.list().length);
+    assertDelListLength(3);
   }
 
   @Test (timeout = 30000)
   public void testFullyDeleteContents() throws IOException {
     boolean ret = FileUtil.fullyDeleteContents(del);
     Assert.assertTrue(ret);
-    Assert.assertTrue(del.exists());
-    Assert.assertEquals(0, del.listFiles().length);
+    Verify.exists(del);
+    Assert.assertEquals(0, Objects.requireNonNull(del.listFiles()).length);
     validateTmpDir();
   }
 
   private void validateTmpDir() {
-    Assert.assertTrue(tmp.exists());
-    Assert.assertEquals(1, tmp.listFiles().length);
-    Assert.assertTrue(new File(tmp, FILE).exists());
+    Verify.exists(tmp);
+    Assert.assertEquals(1, Objects.requireNonNull(tmp.listFiles()).length);
+    Verify.exists(new File(tmp, FILE));
   }
 
   /**
@@ -365,15 +365,15 @@ private void validateTmpDir() {
    * @throws IOException
    */
   private void setupDirsAndNonWritablePermissions() throws IOException {
-    new MyFile(del, FILE_1_NAME).createNewFile();
+    Verify.createNewFile(new MyFile(del, FILE_1_NAME));
 
     // "file1" is non-deletable by default, see MyFile.delete().
 
-    xSubDir.mkdirs();
-    file2.createNewFile();
+    Verify.mkdirs(xSubDir);
+    Verify.createNewFile(file2);
 
-    xSubSubDir.mkdirs();
-    file22.createNewFile();
+    Verify.mkdirs(xSubSubDir);
+    Verify.createNewFile(file22);
 
     revokePermissions(file22);
     revokePermissions(xSubSubDir);
@@ -381,8 +381,8 @@ private void setupDirsAndNonWritablePermissions() throws IOException {
     revokePermissions(file2);
     revokePermissions(xSubDir);
 
-    ySubDir.mkdirs();
-    file3.createNewFile();
+    Verify.mkdirs(ySubDir);
+    Verify.createNewFile(file3);
 
     File tmpFile = new File(tmp, FILE);
     tmpFile.createNewFile();
@@ -461,8 +461,8 @@ public void testFailFullyDeleteDirSymlinks() throws IOException {
     boolean ret = FileUtil.fullyDelete(linkDir);
     // fail symlink deletion
     Assert.assertFalse(ret);
-    Assert.assertTrue(linkDir.exists());
-    Assert.assertEquals(5, del.list().length);
+    Verify.exists(linkDir);
+    assertDelListLength(5);
     // tmp dir should exist
     validateTmpDir();
     // simulate disk recovers and turns good
@@ -470,12 +470,94 @@ public void testFailFullyDeleteDirSymlinks() throws IOException {
     ret = FileUtil.fullyDelete(linkDir);
     // success symlink deletion
     Assert.assertTrue(ret);
-    Assert.assertFalse(linkDir.exists());
-    Assert.assertEquals(4, del.list().length);
+    Verify.notExists(linkDir);
+    assertDelListLength(4);
     // tmp dir should exist
     validateTmpDir();
   }
 
+  /**
+   * Asserts if the {@link TestFileUtil#del} meets the given expected length.
+   *
+   * @param expectedLength The expected length of the {@link TestFileUtil#del}.
+   */
+  private void assertDelListLength(int expectedLength) {
+    Assertions.assertThat(del.list()).describedAs("del list").isNotNull().hasSize(expectedLength);
+  }
+
+  /**
+   * Helper class to perform {@link File} operation and also verify them.
+   */
+  public static class Verify {
+    /**
+     * Invokes {@link File#createNewFile()} on the given {@link File} instance.
+     *
+     * @param file The file to call {@link File#createNewFile()} on.
+     * @return The result of {@link File#createNewFile()}.
+     * @throws IOException As per {@link File#createNewFile()}.
+     */
+    public static File createNewFile(File file) throws IOException {
+      assertTrue("Unable to create new file " + file, file.createNewFile());
+      return file;
+    }
+
+    /**
+     * Invokes {@link File#mkdir()} on the given {@link File} instance.
+     *
+     * @param file The file to call {@link File#mkdir()} on.
+     * @return The result of {@link File#mkdir()}.
+     */
+    public static File mkdir(File file) {
+      assertTrue("Unable to mkdir for " + file, file.mkdir());
+      return file;
+    }
+
+    /**
+     * Invokes {@link File#mkdirs()} on the given {@link File} instance.
+     *
+     * @param file The file to call {@link File#mkdirs()} on.
+     * @return The result of {@link File#mkdirs()}.
+     */
+    public static File mkdirs(File file) {
+      assertTrue("Unable to mkdirs for " + file, file.mkdirs());
+      return file;
+    }
+
+    /**
+     * Invokes {@link File#delete()} on the given {@link File} instance.
+     *
+     * @param file The file to call {@link File#delete()} on.
+     * @return The result of {@link File#delete()}.
+     */
+    public static File delete(File file) {
+      assertTrue("Unable to delete " + file, file.delete());
+      return file;
+    }
+
+    /**
+     * Invokes {@link File#exists()} on the given {@link File} instance.
+     *
+     * @param file The file to call {@link File#exists()} on.
+     * @return The result of {@link File#exists()}.
+     */
+    public static File exists(File file) {
+      assertTrue("Expected file " + file + " doesn't exist", file.exists());
+      return file;
+    }
+
+    /**
+     * Invokes {@link File#exists()} on the given {@link File} instance to check if the
+     * {@link File} doesn't exists.
+     *
+     * @param file The file to call {@link File#exists()} on.
+     * @return The negation of the result of {@link File#exists()}.
+     */
+    public static File notExists(File file) {
+      assertFalse("Expected file " + file + " must not exist", file.exists());
+      return file;
+    }
+  }
+
   /**
    * Extend {@link File}. Same as {@link File} except for two things: (1) This
    * treats file1Name as a very special file which is not delete-able
@@ -608,14 +690,13 @@ public void testGetDU() throws Exception {
       FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true/*recursive*/);
     }
   }
-  
+
   @Test (timeout = 30000)
-  public void testUnTar() throws IOException {
+  public void testUnTar() throws Exception {
     // make a simple tar:
     final File simpleTar = new File(del, FILE);
-    OutputStream os = new FileOutputStream(simpleTar); 
-    TarOutputStream tos = new TarOutputStream(os);
-    try {
+    OutputStream os = new FileOutputStream(simpleTar);
+    try (TarOutputStream tos = new TarOutputStream(os)) {
       TarEntry te = new TarEntry("/bar/foo");
       byte[] data = "some-content".getBytes("UTF-8");
       te.setSize(data.length);
@@ -624,55 +705,42 @@ public void testUnTar() throws IOException {
       tos.closeEntry();
       tos.flush();
       tos.finish();
-    } finally {
-      tos.close();
     }
 
     // successfully untar it into an existing dir:
     FileUtil.unTar(simpleTar, tmp);
     // check result:
-    assertTrue(new File(tmp, "/bar/foo").exists());
+    Verify.exists(new File(tmp, "/bar/foo"));
     assertEquals(12, new File(tmp, "/bar/foo").length());
-    
-    final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog");
-    regularFile.createNewFile();
-    assertTrue(regularFile.exists());
-    try {
-      FileUtil.unTar(simpleTar, regularFile);
-      assertTrue("An IOException expected.", false);
-    } catch (IOException ioe) {
-      // okay
-    }
+
+    final File regularFile =
+        Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"));
+    LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unTar(simpleTar, regularFile));
   }
   
   @Test (timeout = 30000)
   public void testReplaceFile() throws IOException {
-    final File srcFile = new File(tmp, "src");
-    
     // src exists, and target does not exist:
-    srcFile.createNewFile();
-    assertTrue(srcFile.exists());
+    final File srcFile = Verify.createNewFile(new File(tmp, "src"));
     final File targetFile = new File(tmp, "target");
-    assertTrue(!targetFile.exists());
+    Verify.notExists(targetFile);
     FileUtil.replaceFile(srcFile, targetFile);
-    assertTrue(!srcFile.exists());
-    assertTrue(targetFile.exists());
+    Verify.notExists(srcFile);
+    Verify.exists(targetFile);
 
     // src exists and target is a regular file: 
-    srcFile.createNewFile();
-    assertTrue(srcFile.exists());
+    Verify.createNewFile(srcFile);
+    Verify.exists(srcFile);
     FileUtil.replaceFile(srcFile, targetFile);
-    assertTrue(!srcFile.exists());
-    assertTrue(targetFile.exists());
+    Verify.notExists(srcFile);
+    Verify.exists(targetFile);
     
     // src exists, and target is a non-empty directory: 
-    srcFile.createNewFile();
-    assertTrue(srcFile.exists());
-    targetFile.delete();
-    targetFile.mkdirs();
-    File obstacle = new File(targetFile, "obstacle");
-    obstacle.createNewFile();
-    assertTrue(obstacle.exists());
+    Verify.createNewFile(srcFile);
+    Verify.exists(srcFile);
+    Verify.delete(targetFile);
+    Verify.mkdirs(targetFile);
+    File obstacle = Verify.createNewFile(new File(targetFile, "obstacle"));
     assertTrue(targetFile.exists() && targetFile.isDirectory());
     try {
       FileUtil.replaceFile(srcFile, targetFile);
@@ -681,9 +749,9 @@ public void testReplaceFile() throws IOException {
       // okay
     }
     // check up the post-condition: nothing is deleted:
-    assertTrue(srcFile.exists());
+    Verify.exists(srcFile);
     assertTrue(targetFile.exists() && targetFile.isDirectory());
-    assertTrue(obstacle.exists());
+    Verify.exists(obstacle);
   }
   
   @Test (timeout = 30000)
@@ -696,45 +764,84 @@ public void testCreateLocalTempFile() throws IOException {
     assertTrue(tmp1.exists() && tmp2.exists());
     assertTrue(tmp1.canWrite() && tmp2.canWrite());
     assertTrue(tmp1.canRead() && tmp2.canRead());
-    tmp1.delete();
-    tmp2.delete();
+    Verify.delete(tmp1);
+    Verify.delete(tmp2);
     assertTrue(!tmp1.exists() && !tmp2.exists());
   }
   
   @Test (timeout = 30000)
-  public void testUnZip() throws IOException {
+  public void testUnZip() throws Exception {
     // make sa simple zip
     final File simpleZip = new File(del, FILE);
-    OutputStream os = new FileOutputStream(simpleZip); 
-    ZipOutputStream tos = new ZipOutputStream(os);
-    try {
-      ZipEntry ze = new ZipEntry("foo");
-      byte[] data = "some-content".getBytes("UTF-8");
-      ze.setSize(data.length);
-      tos.putNextEntry(ze);
-      tos.write(data);
-      tos.closeEntry();
+    try (OutputStream os = new FileOutputStream(simpleZip);
+         ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os)) {
+      List<ZipArchiveEntry> ZipArchiveList = new ArrayList<>(7);
+      int count = 0;
+      // create 7 files to verify permissions
+      for (int i = 0; i < 7; i++) {
+        ZipArchiveList.add(new ZipArchiveEntry("foo_" + i));
+        ZipArchiveEntry archiveEntry = ZipArchiveList.get(i);
+        archiveEntry.setUnixMode(count += 0100);
+        byte[] data = "some-content".getBytes("UTF-8");
+        archiveEntry.setSize(data.length);
+        tos.putArchiveEntry(archiveEntry);
+        tos.write(data);
+      }
+      tos.closeArchiveEntry();
       tos.flush();
       tos.finish();
-    } finally {
-      tos.close();
     }
-    
+
     // successfully unzip it into an existing dir:
     FileUtil.unZip(simpleZip, tmp);
+    File foo0 = new File(tmp, "foo_0");
+    File foo1 = new File(tmp, "foo_1");
+    File foo2 = new File(tmp, "foo_2");
+    File foo3 = new File(tmp, "foo_3");
+    File foo4 = new File(tmp, "foo_4");
+    File foo5 = new File(tmp, "foo_5");
+    File foo6 = new File(tmp, "foo_6");
     // check result:
-    assertTrue(new File(tmp, "foo").exists());
-    assertEquals(12, new File(tmp, "foo").length());
-    
-    final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog");
-    regularFile.createNewFile();
-    assertTrue(regularFile.exists());
-    try {
-      FileUtil.unZip(simpleZip, regularFile);
-      assertTrue("An IOException expected.", false);
-    } catch (IOException ioe) {
-      // okay
-    }
+    assertTrue(foo0.exists());
+    assertTrue(foo1.exists());
+    assertTrue(foo2.exists());
+    assertTrue(foo3.exists());
+    assertTrue(foo4.exists());
+    assertTrue(foo5.exists());
+    assertTrue(foo6.exists());
+    assertEquals(12, foo0.length());
+    // tests whether file foo_0 has executable permissions
+    assertTrue("file lacks execute permissions", foo0.canExecute());
+    assertFalse("file has write permissions", foo0.canWrite());
+    assertFalse("file has read permissions", foo0.canRead());
+    // tests whether file foo_1 has writable permissions
+    assertFalse("file has execute permissions", foo1.canExecute());
+    assertTrue("file lacks write permissions", foo1.canWrite());
+    assertFalse("file has read permissions", foo1.canRead());
+    // tests whether file foo_2 has executable and writable permissions
+    assertTrue("file lacks execute permissions", foo2.canExecute());
+    assertTrue("file lacks write permissions", foo2.canWrite());
+    assertFalse("file has read permissions", foo2.canRead());
+    // tests whether file foo_3 has readable permissions
+    assertFalse("file has execute permissions", foo3.canExecute());
+    assertFalse("file has write permissions", foo3.canWrite());
+    assertTrue("file lacks read permissions", foo3.canRead());
+    // tests whether file foo_4 has readable and executable permissions
+    assertTrue("file lacks execute permissions", foo4.canExecute());
+    assertFalse("file has write permissions", foo4.canWrite());
+    assertTrue("file lacks read permissions", foo4.canRead());
+    // tests whether file foo_5 has readable and writable permissions
+    assertFalse("file has execute permissions", foo5.canExecute());
+    assertTrue("file lacks write permissions", foo5.canWrite());
+    assertTrue("file lacks read permissions", foo5.canRead());
+    // tests whether file foo_6 has readable, writable and executable permissions
+    assertTrue("file lacks execute permissions", foo6.canExecute());
+    assertTrue("file lacks write permissions", foo6.canWrite());
+    assertTrue("file lacks read permissions", foo6.canRead());
+
+    final File regularFile =
+        Verify.createNewFile(new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"));
+    LambdaTestUtils.intercept(IOException.class, () -> FileUtil.unZip(simpleZip, regularFile));
   }
 
   @Test (timeout = 30000)
@@ -742,14 +849,14 @@ public void testUnZip2() throws IOException {
     // make a simple zip
     final File simpleZip = new File(del, FILE);
     OutputStream os = new FileOutputStream(simpleZip);
-    try (ZipOutputStream tos = new ZipOutputStream(os)) {
+    try (ZipArchiveOutputStream tos = new ZipArchiveOutputStream(os)) {
       // Add an entry that contains invalid filename
-      ZipEntry ze = new ZipEntry("../foo");
+      ZipArchiveEntry ze = new ZipArchiveEntry("../foo");
       byte[] data = "some-content".getBytes(StandardCharsets.UTF_8);
       ze.setSize(data.length);
-      tos.putNextEntry(ze);
+      tos.putArchiveEntry(ze);
       tos.write(data);
-      tos.closeEntry();
+      tos.closeArchiveEntry();
       tos.flush();
       tos.finish();
     }
@@ -780,24 +887,24 @@ public void testCopy5() throws IOException {
     final File dest = new File(del, "dest");
     boolean result = FileUtil.copy(fs, srcPath, dest, false, conf);
     assertTrue(result);
-    assertTrue(dest.exists());
+    Verify.exists(dest);
     assertEquals(content.getBytes().length 
         + System.getProperty("line.separator").getBytes().length, dest.length());
-    assertTrue(srcFile.exists()); // should not be deleted
+    Verify.exists(srcFile); // should not be deleted
     
     // copy regular file, delete src:
-    dest.delete();
-    assertTrue(!dest.exists());
+    Verify.delete(dest);
+    Verify.notExists(dest);
     result = FileUtil.copy(fs, srcPath, dest, true, conf);
     assertTrue(result);
-    assertTrue(dest.exists());
+    Verify.exists(dest);
     assertEquals(content.getBytes().length 
         + System.getProperty("line.separator").getBytes().length, dest.length());
-    assertTrue(!srcFile.exists()); // should be deleted
+    Verify.notExists(srcFile); // should be deleted
     
     // copy a dir:
-    dest.delete();
-    assertTrue(!dest.exists());
+    Verify.delete(dest);
+    Verify.notExists(dest);
     srcPath = new Path(partitioned.toURI());
     result = FileUtil.copy(fs, srcPath, dest, true, conf);
     assertTrue(result);
@@ -809,7 +916,7 @@ public void testCopy5() throws IOException {
       assertEquals(3 
           + System.getProperty("line.separator").getBytes().length, f.length());
     }
-    assertTrue(!partitioned.exists()); // should be deleted
+    Verify.notExists(partitioned); // should be deleted
   }  
 
   @Test (timeout = 30000)
@@ -897,8 +1004,8 @@ public void testSymlinkRenameTo() throws Exception {
     // create the symlink
     FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
 
-    Assert.assertTrue(file.exists());
-    Assert.assertTrue(link.exists());
+    Verify.exists(file);
+    Verify.exists(link);
 
     File link2 = new File(del, "_link2");
 
@@ -908,10 +1015,10 @@ public void testSymlinkRenameTo() throws Exception {
     // Make sure the file still exists
     // (NOTE: this would fail on Java6 on Windows if we didn't
     // copy the file in FileUtil#symlink)
-    Assert.assertTrue(file.exists());
+    Verify.exists(file);
 
-    Assert.assertTrue(link2.exists());
-    Assert.assertFalse(link.exists());
+    Verify.exists(link2);
+    Verify.notExists(link);
   }
 
   /**
@@ -926,13 +1033,13 @@ public void testSymlinkDelete() throws Exception {
     // create the symlink
     FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
 
-    Assert.assertTrue(file.exists());
-    Assert.assertTrue(link.exists());
+    Verify.exists(file);
+    Verify.exists(link);
 
     // make sure that deleting a symlink works properly
-    Assert.assertTrue(link.delete());
-    Assert.assertFalse(link.exists());
-    Assert.assertTrue(file.exists());
+    Verify.delete(link);
+    Verify.notExists(link);
+    Verify.exists(file);
   }
 
   /**
@@ -959,13 +1066,13 @@ public void testSymlinkLength() throws Exception {
     Assert.assertEquals(data.length, file.length());
     Assert.assertEquals(data.length, link.length());
 
-    file.delete();
-    Assert.assertFalse(file.exists());
+    Verify.delete(file);
+    Verify.notExists(file);
 
     Assert.assertEquals(0, link.length());
 
-    link.delete();
-    Assert.assertFalse(link.exists());
+    Verify.delete(link);
+    Verify.notExists(link);
   }
 
   /**
@@ -1031,7 +1138,7 @@ public void testSymlinkFileAlreadyExists() throws IOException {
   public void testSymlinkSameFile() throws IOException {
     File file = new File(del, FILE);
 
-    file.delete();
+    Verify.delete(file);
 
     // Create a symbolic link
     // The operation should succeed
@@ -1104,21 +1211,21 @@ private void doUntarAndVerify(File tarFile, File untarDir)
 
     String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name";
     File testFile = new File(parentDir + Path.SEPARATOR + "version");
-    Assert.assertTrue(testFile.exists());
+    Verify.exists(testFile);
     Assert.assertTrue(testFile.length() == 0);
     String imageDir = parentDir + Path.SEPARATOR + "image";
     testFile = new File(imageDir + Path.SEPARATOR + "fsimage");
-    Assert.assertTrue(testFile.exists());
+    Verify.exists(testFile);
     Assert.assertTrue(testFile.length() == 157);
     String currentDir = parentDir + Path.SEPARATOR + "current";
     testFile = new File(currentDir + Path.SEPARATOR + "fsimage");
-    Assert.assertTrue(testFile.exists());
+    Verify.exists(testFile);
     Assert.assertTrue(testFile.length() == 4331);
     testFile = new File(currentDir + Path.SEPARATOR + "edits");
-    Assert.assertTrue(testFile.exists());
+    Verify.exists(testFile);
     Assert.assertTrue(testFile.length() == 1033);
     testFile = new File(currentDir + Path.SEPARATOR + "fstime");
-    Assert.assertTrue(testFile.exists());
+    Verify.exists(testFile);
     Assert.assertTrue(testFile.length() == 8);
   }
 
@@ -1135,6 +1242,38 @@ public void testUntar() throws IOException {
     doUntarAndVerify(new File(tarFileName), untarDir);
   }
 
+  /**
+   * Verify we can't unTar a file which isn't there.
+   * This will test different codepaths on Windows from unix,
+   * but both MUST throw an IOE of some kind.
+   */
+  @Test(timeout = 30000)
+  public void testUntarMissingFile() throws Throwable {
+    File dataDir = GenericTestUtils.getTestDir();
+    File tarFile = new File(dataDir, "missing; true");
+    File untarDir = new File(dataDir, "untarDir");
+    intercept(IOException.class, () ->
+        FileUtil.unTar(tarFile, untarDir));
+  }
+
+  /**
+   * Verify we can't unTar a file which isn't there
+   * through the java untar code.
+   * This is how {@code FileUtil.unTar(File, File}
+   * will behave on Windows,
+   */
+  @Test(timeout = 30000)
+  public void testUntarMissingFileThroughJava() throws Throwable {
+    File dataDir = GenericTestUtils.getTestDir();
+    File tarFile = new File(dataDir, "missing; true");
+    File untarDir = new File(dataDir, "untarDir");
+    // java8 on unix throws java.nio.file.NoSuchFileException here;
+    // leaving as an IOE intercept in case windows throws something
+    // else.
+    intercept(IOException.class, () ->
+        FileUtil.unTarUsingJava(tarFile, untarDir, false));
+  }
+
   @Test (timeout = 30000)
   public void testCreateJarWithClassPath() throws Exception {
     // create files expected to match a wildcard
@@ -1147,9 +1286,9 @@ public void testCreateJarWithClassPath() throws Exception {
     }
 
     // create non-jar files, which we expect to not be included in the classpath
-    Assert.assertTrue(new File(tmp, "text.txt").createNewFile());
-    Assert.assertTrue(new File(tmp, "executable.exe").createNewFile());
-    Assert.assertTrue(new File(tmp, "README").createNewFile());
+    Verify.createNewFile(new File(tmp, "text.txt"));
+    Verify.createNewFile(new File(tmp, "executable.exe"));
+    Verify.createNewFile(new File(tmp, "README"));
 
     // create classpath jar
     String wildcardPath = tmp.getCanonicalPath() + File.separator + "*";
@@ -1235,9 +1374,9 @@ public void testGetJarsInDirectory() throws Exception {
     }
 
     // create non-jar files, which we expect to not be included in the result
-    assertTrue(new File(tmp, "text.txt").createNewFile());
-    assertTrue(new File(tmp, "executable.exe").createNewFile());
-    assertTrue(new File(tmp, "README").createNewFile());
+    Verify.createNewFile(new File(tmp, "text.txt"));
+    Verify.createNewFile(new File(tmp, "executable.exe"));
+    Verify.createNewFile(new File(tmp, "README"));
 
     // pass in the directory
     String directory = tmp.getCanonicalPath();
@@ -1271,7 +1410,7 @@ public void setupCompareFs() {
       uri4 = new URI(uris4);
       uri5 = new URI(uris5);
       uri6 = new URI(uris6);
-    } catch (URISyntaxException use) {
+    } catch (URISyntaxException ignored) {
     }
     // Set up InetAddress
     inet1 = mock(InetAddress.class);
@@ -1294,7 +1433,7 @@ public void setupCompareFs() {
       when(InetAddress.getByName(uris3)).thenReturn(inet3);
       when(InetAddress.getByName(uris4)).thenReturn(inet4);
       when(InetAddress.getByName(uris5)).thenReturn(inet5);
-    } catch (UnknownHostException ue) {
+    } catch (UnknownHostException ignored) {
     }
 
     fs1 = mock(FileSystem.class);
@@ -1314,62 +1453,87 @@ public void setupCompareFs() {
   @Test
   public void testCompareFsNull() throws Exception {
     setupCompareFs();
-    assertEquals(FileUtil.compareFs(null,fs1),false);
-    assertEquals(FileUtil.compareFs(fs1,null),false);
+    assertFalse(FileUtil.compareFs(null, fs1));
+    assertFalse(FileUtil.compareFs(fs1, null));
   }
 
   @Test
   public void testCompareFsDirectories() throws Exception {
     setupCompareFs();
-    assertEquals(FileUtil.compareFs(fs1,fs1),true);
-    assertEquals(FileUtil.compareFs(fs1,fs2),false);
-    assertEquals(FileUtil.compareFs(fs1,fs5),false);
-    assertEquals(FileUtil.compareFs(fs3,fs4),true);
-    assertEquals(FileUtil.compareFs(fs1,fs6),false);
+    assertTrue(FileUtil.compareFs(fs1, fs1));
+    assertFalse(FileUtil.compareFs(fs1, fs2));
+    assertFalse(FileUtil.compareFs(fs1, fs5));
+    assertTrue(FileUtil.compareFs(fs3, fs4));
+    assertFalse(FileUtil.compareFs(fs1, fs6));
   }
 
   @Test(timeout = 8000)
   public void testCreateSymbolicLinkUsingJava() throws IOException {
     final File simpleTar = new File(del, FILE);
     OutputStream os = new FileOutputStream(simpleTar);
-    TarArchiveOutputStream tos = new TarArchiveOutputStream(os);
-    File untarFile = null;
-    try {
+    try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) {
       // Files to tar
       final String tmpDir = "tmp/test";
       File tmpDir1 = new File(tmpDir, "dir1/");
       File tmpDir2 = new File(tmpDir, "dir2/");
-      // Delete the directories if they already exist
-      tmpDir1.mkdirs();
-      tmpDir2.mkdirs();
+      Verify.mkdirs(tmpDir1);
+      Verify.mkdirs(tmpDir2);
 
-      java.nio.file.Path symLink = FileSystems
-          .getDefault().getPath(tmpDir1.getPath() + "/sl");
+      java.nio.file.Path symLink = Paths.get(tmpDir1.getPath(), "sl");
 
       // Create Symbolic Link
-      Files.createSymbolicLink(symLink,
-          FileSystems.getDefault().getPath(tmpDir2.getPath())).toString();
+      Files.createSymbolicLink(symLink, Paths.get(tmpDir2.getPath()));
       assertTrue(Files.isSymbolicLink(symLink.toAbsolutePath()));
-      // put entries in tar file
+      // Put entries in tar file
       putEntriesInTar(tos, tmpDir1.getParentFile());
       tos.close();
 
-      untarFile = new File(tmpDir, "2");
-      // Untar using java
+      File untarFile = new File(tmpDir, "2");
+      // Untar using Java
       FileUtil.unTarUsingJava(simpleTar, untarFile, false);
 
       // Check symbolic link and other directories are there in untar file
       assertTrue(Files.exists(untarFile.toPath()));
-      assertTrue(Files.exists(FileSystems.getDefault().getPath(untarFile
-          .getPath(), tmpDir)));
-      assertTrue(Files.isSymbolicLink(FileSystems.getDefault().getPath(untarFile
-          .getPath().toString(), symLink.toString())));
-
+      assertTrue(Files.exists(Paths.get(untarFile.getPath(), tmpDir)));
+      assertTrue(Files.isSymbolicLink(Paths.get(untarFile.getPath(), symLink.toString())));
     } finally {
       FileUtils.deleteDirectory(new File("tmp"));
-      tos.close();
     }
+  }
+
+  @Test(expected = IOException.class)
+  public void testCreateArbitrarySymlinkUsingJava() throws IOException {
+    final File simpleTar = new File(del, FILE);
+    OutputStream os = new FileOutputStream(simpleTar);
+
+    File rootDir = new File("tmp");
+    try (TarArchiveOutputStream tos = new TarArchiveOutputStream(os)) {
+      tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+
+      // Create arbitrary dir
+      File arbitraryDir = new File(rootDir, "arbitrary-dir/");
+      Verify.mkdirs(arbitraryDir);
+
+      // We will tar from the tar-root lineage
+      File tarRoot = new File(rootDir, "tar-root/");
+      File symlinkRoot = new File(tarRoot, "dir1/");
+      Verify.mkdirs(symlinkRoot);
+
+      // Create Symbolic Link to an arbitrary dir
+      java.nio.file.Path symLink = Paths.get(symlinkRoot.getPath(), "sl");
+      Files.createSymbolicLink(symLink, arbitraryDir.toPath().toAbsolutePath());
+
+      // Put entries in tar file
+      putEntriesInTar(tos, tarRoot);
+      putEntriesInTar(tos, new File(symLink.toFile(), "dir-outside-tar-root/"));
+      tos.close();
 
+      // Untar using Java
+      File untarFile = new File(rootDir, "extracted");
+      FileUtil.unTarUsingJava(simpleTar, untarFile, false);
+    } finally {
+      FileUtils.deleteDirectory(rootDir);
+    }
   }
 
   private void putEntriesInTar(TarArchiveOutputStream tos, File f)
@@ -1433,6 +1597,23 @@ public void testReadSymlink() throws IOException {
     Assert.assertEquals(file.getAbsolutePath(), result);
   }
 
+  @Test
+  public void testRegularFile() throws IOException {
+    byte[] data = "testRegularData".getBytes();
+    File tmpFile = new File(del, "reg1");
+
+    // write some data to the file
+    FileOutputStream os = new FileOutputStream(tmpFile);
+    os.write(data);
+    os.close();
+    assertTrue(FileUtil.isRegularFile(tmpFile));
+
+    // create a symlink to file
+    File link = new File(del, "reg2");
+    FileUtil.symLink(tmpFile.toString(), link.toString());
+    assertFalse(FileUtil.isRegularFile(link, false));
+  }
+
   /**
    * This test validates the correctness of {@link FileUtil#readLink(File)} when
    * it gets a file in input.
@@ -1446,7 +1627,7 @@ public void testReadSymlinkWithAFileAsInput() throws IOException {
     String result = FileUtil.readLink(file);
     Assert.assertEquals("", result);
 
-    file.delete();
+    Verify.delete(file);
   }
 
   /**
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
index 5ed4d9bc9a7fa..3d8ea0e826cf2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
@@ -143,6 +143,11 @@ public Token<?>[] addDelegationTokens(String renewer, Credentials creds)
     of the filter such as checksums.
      */
     MultipartUploaderBuilder createMultipartUploader(Path basePath);
+
+    FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException;
+
+    FSDataOutputStream append(Path f, int bufferSize,
+        Progressable progress, boolean appendToNewBlock) throws IOException;
   }
 
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java
index 49bbd5af04f25..c2bd5b2133d47 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellTouch.java
@@ -200,6 +200,78 @@ public void testTouch() throws Exception {
     FileStatus fileStatus = lfs.getFileStatus(newFile);
     assertThat(fileStatus.getAccessTime()).isEqualTo(dateObj.getTime());
     assertThat(fileStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    lfs.delete(newFile, true);
+    assertThat(lfs.exists(newFile)).isFalse();
+
+  }
+
+  @Test
+  public void testTouchDir() throws Exception {
+    String strTime;
+    final String newFileName = "dir3/newFile3";
+    Date dateObj;
+    final Path newFile = new Path(newFileName);
+    FileStatus fstatus;
+    Path dirPath = new Path("dir3");
+    lfs.delete(dirPath, true);
+    lfs.mkdirs(dirPath);
+    lfs.delete(newFile, true);
+    assertThat(lfs.exists(newFile)).isFalse();
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-t", strTime, newFileName)).as(
+        "Expected successful touch on a new file with a specified timestamp").isEqualTo(0);
+    FileStatus newStatus = lfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    Thread.sleep(500);
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-m", "-a", "-t", strTime, "dir3")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = lfs.getFileStatus(dirPath);
+    // Verify if both modification and access times are recorded correctly
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    fstatus = lfs.getFileStatus(dirPath);
+    Thread.sleep(500);
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-m", "-t", strTime, "dir3")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = lfs.getFileStatus(dirPath);
+    // Verify if modification time is recorded correctly (and access time
+    // remains unchanged).
+    assertThat(newStatus.getAccessTime()).isEqualTo(fstatus.getAccessTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    fstatus = lfs.getFileStatus(dirPath);
+    Thread.sleep(500);
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-a", "-t", strTime, "dir3")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = lfs.getFileStatus(dirPath);
+    // Verify if access time is recorded correctly (and modification time
+    // remains unchanged).
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(fstatus.getModificationTime());
+
+    lfs.delete(newFile, true);
+    lfs.delete(dirPath, true);
+    assertThat(lfs.exists(newFile)).isFalse();
+    assertThat(lfs.exists(dirPath)).isFalse();
   }
 
   private String formatTimestamp(long timeInMillis) {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 711ab94fdf123..b227e16908828 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -250,6 +250,11 @@ CompletableFuture<FSDataInputStream> openFileWithOptions(
 
     MultipartUploaderBuilder createMultipartUploader(Path basePath)
         throws IOException;
+
+    FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException;
+
+    FSDataOutputStream append(Path f, int bufferSize,
+        Progressable progress, boolean appendToNewBlock) throws IOException;
   }
 
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
index 6415df6310fc2..471d2458f4f46 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
@@ -246,7 +246,7 @@ public void testListLocatedStatus() throws Exception {
     // test.har has the following contents:
     //   dir1/1.txt
     //   dir1/2.txt
-    Set<String> expectedFileNames = new HashSet<String>();
+    Set<String> expectedFileNames = new HashSet<>();
     expectedFileNames.add("1.txt");
     expectedFileNames.add("2.txt");
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java
index 44308ea6fc5ea..dce3b956d47ef 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestListFiles.java
@@ -152,7 +152,7 @@ public void testDirectory() throws IOException {
     writeFile(fs, FILE1, FILE_LEN);
     writeFile(fs, FILE3, FILE_LEN);
 
-    Set<Path> filesToFind = new HashSet<Path>();
+    Set<Path> filesToFind = new HashSet<>();
     filesToFind.add(fs.makeQualified(FILE1));
     filesToFind.add(fs.makeQualified(FILE2));
     filesToFind.add(fs.makeQualified(FILE3));
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
index acda898ea1342..939881f39df6d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
@@ -26,6 +26,7 @@
 import java.util.NoSuchElementException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.Shell;
 
@@ -532,4 +533,20 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception {
     }
   }
 
+  /**
+   * Test to verify LocalDirAllocator log details to provide diagnostics when file creation fails.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 30000)
+  public void testGetLocalPathForWriteForLessSpace() throws Exception {
+    String dir0 = buildBufferDir(ROOT, 0);
+    String dir1 = buildBufferDir(ROOT, 1);
+    conf.set(CONTEXT, dir0 + "," + dir1);
+    LambdaTestUtils.intercept(DiskErrorException.class,
+        String.format("Could not find any valid local directory for %s with requested size %s",
+            "p1/x", Long.MAX_VALUE - 1), "Expect a DiskErrorException.",
+        () -> dirAllocator.getLocalPathForWrite("p1/x", Long.MAX_VALUE - 1, conf));
+  }
 }
+
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
index 949ea42331527..38e16221a4518 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.util.StringUtils;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
@@ -164,7 +163,7 @@ public void testWorkingDirectory() throws IOException {
   public void testSyncable() throws IOException {
     FileSystem fs = fileSys.getRawFileSystem();
     Path file = new Path(TEST_ROOT_DIR, "syncable");
-    FSDataOutputStream out = fs.create(file);;
+    FSDataOutputStream out = fs.create(file);
     final int bytesWritten = 1;
     byte[] expectedBuf = new byte[] {'0', '1', '2', '3'};
     try {
@@ -650,7 +649,8 @@ public void testFileStatusPipeFile() throws Exception {
     RawLocalFileSystem fs = spy(origFs);
     Configuration conf = mock(Configuration.class);
     fs.setConf(conf);
-    Whitebox.setInternalState(fs, "useDeprecatedFileStatus", false);
+
+    RawLocalFileSystem.setUseDeprecatedFileStatus(false);
     Path path = new Path("/foo");
     File pipe = mock(File.class);
     when(pipe.isFile()).thenReturn(false);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 72287782baac6..5b8c10b3fa6f9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -747,7 +747,7 @@ public void testTrashEmptier() throws Exception {
     Path myPath = new Path(TEST_DIR, "test/mkdirs");
     mkdir(fs, myPath);
     int fileIndex = 0;
-    Set<String> checkpoints = new HashSet<String>();
+    Set<String> checkpoints = new HashSet<>();
     while (true)  {
       // Create a file with a new name
       Path myFile = new Path(TEST_DIR, "test/mkdirs/myFile" + fileIndex++);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
new file mode 100644
index 0000000000000..e964d23f4b750
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.IntFunction;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.impl.CombinedFileRange;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
+import static org.apache.hadoop.fs.VectoredReadUtils.validateNonOverlappingAndReturnSortedRanges;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.test.MoreAsserts.assertFutureCompletedSuccessfully;
+import static org.apache.hadoop.test.MoreAsserts.assertFutureFailedExceptionally;
+
+/**
+ * Test behavior of {@link VectoredReadUtils}.
+ */
+public class TestVectoredReadUtils extends HadoopTestBase {
+
+  @Test
+  public void testSliceTo() {
+    final int size = 64 * 1024;
+    ByteBuffer buffer = ByteBuffer.allocate(size);
+    // fill the buffer with data
+    IntBuffer intBuffer = buffer.asIntBuffer();
+    for(int i=0; i < size / Integer.BYTES; ++i) {
+      intBuffer.put(i);
+    }
+    // ensure we don't make unnecessary slices
+    ByteBuffer slice = VectoredReadUtils.sliceTo(buffer, 100,
+        FileRange.createFileRange(100, size));
+    Assertions.assertThat(buffer)
+            .describedAs("Slicing on the same offset shouldn't " +
+                    "create a new buffer")
+            .isEqualTo(slice);
+    Assertions.assertThat(slice.position())
+        .describedAs("Slicing should return buffers starting from position 0")
+        .isEqualTo(0);
+
+    // try slicing a range
+    final int offset = 100;
+    final int sliceStart = 1024;
+    final int sliceLength = 16 * 1024;
+    slice = VectoredReadUtils.sliceTo(buffer, offset,
+        FileRange.createFileRange(offset + sliceStart, sliceLength));
+    // make sure they aren't the same, but use the same backing data
+    Assertions.assertThat(buffer)
+            .describedAs("Slicing on new offset should " +
+                    "create a new buffer")
+            .isNotEqualTo(slice);
+    Assertions.assertThat(buffer.array())
+            .describedAs("Slicing should use the same underlying " +
+                    "data")
+            .isEqualTo(slice.array());
+    Assertions.assertThat(slice.position())
+        .describedAs("Slicing should return buffers starting from position 0")
+        .isEqualTo(0);
+    // test the contents of the slice
+    intBuffer = slice.asIntBuffer();
+    for(int i=0; i < sliceLength / Integer.BYTES; ++i) {
+      assertEquals("i = " + i, i + sliceStart / Integer.BYTES, intBuffer.get());
+    }
+  }
+
+  @Test
+  public void testRounding() {
+    for(int i=5; i < 10; ++i) {
+      assertEquals("i = "+ i, 5, VectoredReadUtils.roundDown(i, 5));
+      assertEquals("i = "+ i, 10, VectoredReadUtils.roundUp(i+1, 5));
+    }
+    assertEquals("Error while roundDown", 13, VectoredReadUtils.roundDown(13, 1));
+    assertEquals("Error while roundUp", 13, VectoredReadUtils.roundUp(13, 1));
+  }
+
+  @Test
+  public void testMerge() {
+    // a reference to use for tracking
+    Object tracker1 = "one";
+    Object tracker2 = "two";
+    FileRange base = FileRange.createFileRange(2000, 1000, tracker1);
+    CombinedFileRange mergeBase = new CombinedFileRange(2000, 3000, base);
+
+    // test when the gap between is too big
+    assertFalse("Large gap ranges shouldn't get merged", mergeBase.merge(5000, 6000,
+        FileRange.createFileRange(5000, 1000), 2000, 4000));
+    assertEquals("Number of ranges in merged range shouldn't increase",
+            1, mergeBase.getUnderlying().size());
+    assertFileRange(mergeBase, 2000, 1000);
+
+    // test when the total size gets exceeded
+    assertFalse("Large size ranges shouldn't get merged", mergeBase.merge(5000, 6000,
+        FileRange.createFileRange(5000, 1000), 2001, 3999));
+    assertEquals("Number of ranges in merged range shouldn't increase",
+            1, mergeBase.getUnderlying().size());
+    assertFileRange(mergeBase, 2000, 1000);
+
+    // test when the merge works
+    assertTrue("ranges should get merged ", mergeBase.merge(5000, 6000,
+        FileRange.createFileRange(5000, 1000, tracker2),
+        2001, 4000));
+    assertEquals("post merge size", 2, mergeBase.getUnderlying().size());
+    assertFileRange(mergeBase, 2000, 4000);
+
+    Assertions.assertThat(mergeBase.getUnderlying().get(0).getReference())
+        .describedAs("reference of range %s", mergeBase.getUnderlying().get(0))
+        .isSameAs(tracker1);
+    Assertions.assertThat(mergeBase.getUnderlying().get(1).getReference())
+        .describedAs("reference of range %s", mergeBase.getUnderlying().get(1))
+        .isSameAs(tracker2);
+
+    // reset the mergeBase and test with a 10:1 reduction
+    mergeBase = new CombinedFileRange(200, 300, base);
+    assertFileRange(mergeBase, 200, 100);
+
+    assertTrue("ranges should get merged ", mergeBase.merge(500, 600,
+        FileRange.createFileRange(5000, 1000), 201, 400));
+    assertEquals("post merge size", 2, mergeBase.getUnderlying().size());
+    assertFileRange(mergeBase, 200, 400);
+  }
+
+  @Test
+  public void testSortAndMerge() {
+    List<FileRange> input = Arrays.asList(
+        FileRange.createFileRange(3000, 100, "1"),
+        FileRange.createFileRange(2100, 100, null),
+        FileRange.createFileRange(1000, 100, "3")
+        );
+    assertFalse("Ranges are non disjoint", VectoredReadUtils.isOrderedDisjoint(input, 100, 800));
+    final List<CombinedFileRange> outputList = VectoredReadUtils.mergeSortedRanges(
+            Arrays.asList(sortRanges(input)), 100, 1001, 2500);
+    Assertions.assertThat(outputList)
+            .describedAs("merged range size")
+            .hasSize(1);
+    CombinedFileRange output = outputList.get(0);
+    Assertions.assertThat(output.getUnderlying())
+            .describedAs("merged range underlying size")
+            .hasSize(3);
+    // range[1000,3100)
+    assertFileRange(output, 1000, 2100);
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(outputList, 100, 800));
+
+    // the minSeek doesn't allow the first two to merge
+    assertFalse("Ranges are non disjoint",
+            VectoredReadUtils.isOrderedDisjoint(input, 100, 1000));
+    final List<CombinedFileRange> list2 = VectoredReadUtils.mergeSortedRanges(
+        Arrays.asList(sortRanges(input)),
+            100, 1000, 2100);
+    Assertions.assertThat(list2)
+            .describedAs("merged range size")
+            .hasSize(2);
+    assertFileRange(list2.get(0), 1000, 100);
+
+    // range[2100,3100)
+    assertFileRange(list2.get(1), 2100, 1000);
+
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(list2, 100, 1000));
+
+    // the maxSize doesn't allow the third range to merge
+    assertFalse("Ranges are non disjoint",
+            VectoredReadUtils.isOrderedDisjoint(input, 100, 800));
+    final List<CombinedFileRange> list3 = VectoredReadUtils.mergeSortedRanges(
+        Arrays.asList(sortRanges(input)),
+            100, 1001, 2099);
+    Assertions.assertThat(list3)
+            .describedAs("merged range size")
+            .hasSize(2);
+    // range[1000,2200)
+    CombinedFileRange range0 = list3.get(0);
+    assertFileRange(range0, 1000, 1200);
+    assertFileRange(range0.getUnderlying().get(0),
+        1000, 100, "3");
+    assertFileRange(range0.getUnderlying().get(1),
+        2100, 100, null);
+    CombinedFileRange range1 = list3.get(1);
+    // range[3000,3100)
+    assertFileRange(range1, 3000, 100);
+    assertFileRange(range1.getUnderlying().get(0),
+        3000, 100, "1");
+
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(list3, 100, 800));
+
+    // test the round up and round down (the maxSize doesn't allow any merges)
+    assertFalse("Ranges are non disjoint",
+            VectoredReadUtils.isOrderedDisjoint(input, 16, 700));
+    final List<CombinedFileRange> list4 = VectoredReadUtils.mergeSortedRanges(
+        Arrays.asList(sortRanges(input)),
+            16, 1001, 100);
+    Assertions.assertThat(list4)
+            .describedAs("merged range size")
+            .hasSize(3);
+    // range[992,1104)
+    assertFileRange(list4.get(0), 992, 112);
+    // range[2096,2208)
+    assertFileRange(list4.get(1), 2096, 112);
+    // range[2992,3104)
+    assertFileRange(list4.get(2), 2992, 112);
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(list4, 16, 700));
+  }
+
+  /**
+   * Assert that a file range satisfies the conditions.
+   * @param range range to validate
+   * @param offset offset of range
+   * @param length range length
+   */
+  private void assertFileRange(FileRange range, long offset, int length) {
+    Assertions.assertThat(range)
+        .describedAs("file range %s", range)
+        .isNotNull();
+    Assertions.assertThat(range.getOffset())
+        .describedAs("offset of %s", range)
+        .isEqualTo(offset);
+    Assertions.assertThat(range.getLength())
+        .describedAs("length of %s", range)
+        .isEqualTo(length);
+  }
+
+  /**
+   * Assert that a file range satisfies the conditions.
+   * @param range range to validate
+   * @param offset offset of range
+   * @param length range length
+   * @param reference reference; may be null.
+   */
+  private void assertFileRange(FileRange range, long offset, int length, Object reference) {
+    assertFileRange(range, offset, length);
+    Assertions.assertThat(range.getReference())
+        .describedAs("reference field of file range %s", range)
+        .isEqualTo(reference);
+  }
+
+
+  @Test
+  public void testSortAndMergeMoreCases() throws Exception {
+    List<FileRange> input = Arrays.asList(
+            FileRange.createFileRange(3000, 110),
+            FileRange.createFileRange(3000, 100),
+            FileRange.createFileRange(2100, 100),
+            FileRange.createFileRange(1000, 100)
+    );
+    assertFalse("Ranges are non disjoint",
+            VectoredReadUtils.isOrderedDisjoint(input, 100, 800));
+    List<CombinedFileRange> outputList = VectoredReadUtils.mergeSortedRanges(
+            Arrays.asList(sortRanges(input)), 1, 1001, 2500);
+    Assertions.assertThat(outputList)
+            .describedAs("merged range size")
+            .hasSize(1);
+    CombinedFileRange output = outputList.get(0);
+    Assertions.assertThat(output.getUnderlying())
+            .describedAs("merged range underlying size")
+            .hasSize(4);
+
+    assertFileRange(output, 1000, 2110);
+
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(outputList, 1, 800));
+
+    outputList = VectoredReadUtils.mergeSortedRanges(
+            Arrays.asList(sortRanges(input)), 100, 1001, 2500);
+    Assertions.assertThat(outputList)
+            .describedAs("merged range size")
+            .hasSize(1);
+    output = outputList.get(0);
+    Assertions.assertThat(output.getUnderlying())
+            .describedAs("merged range underlying size")
+            .hasSize(4);
+    assertFileRange(output, 1000, 2200);
+
+    assertTrue("merged output ranges are disjoint",
+            VectoredReadUtils.isOrderedDisjoint(outputList, 1, 800));
+
+  }
+
+  @Test
+  public void testValidateOverlappingRanges()  throws Exception {
+    List<FileRange> input = Arrays.asList(
+            FileRange.createFileRange(100, 100),
+            FileRange.createFileRange(200, 100),
+            FileRange.createFileRange(250, 100)
+    );
+
+    intercept(UnsupportedOperationException.class,
+        () -> validateNonOverlappingAndReturnSortedRanges(input));
+
+    List<FileRange> input1 = Arrays.asList(
+            FileRange.createFileRange(100, 100),
+            FileRange.createFileRange(500, 100),
+            FileRange.createFileRange(1000, 100),
+            FileRange.createFileRange(1000, 100)
+    );
+
+    intercept(UnsupportedOperationException.class,
+        () -> validateNonOverlappingAndReturnSortedRanges(input1));
+
+    List<FileRange> input2 = Arrays.asList(
+            FileRange.createFileRange(100, 100),
+            FileRange.createFileRange(200, 100),
+            FileRange.createFileRange(300, 100)
+    );
+    // consecutive ranges should pass.
+    validateNonOverlappingAndReturnSortedRanges(input2);
+  }
+
+  @Test
+  public void testMaxSizeZeroDisablesMering() throws Exception {
+    List<FileRange> randomRanges = Arrays.asList(
+            FileRange.createFileRange(3000, 110),
+            FileRange.createFileRange(3000, 100),
+            FileRange.createFileRange(2100, 100)
+    );
+    assertEqualRangeCountsAfterMerging(randomRanges, 1, 1, 0);
+    assertEqualRangeCountsAfterMerging(randomRanges, 1, 0, 0);
+    assertEqualRangeCountsAfterMerging(randomRanges, 1, 100, 0);
+  }
+
+  private void assertEqualRangeCountsAfterMerging(List<FileRange> inputRanges,
+                                                  int chunkSize,
+                                                  int minimumSeek,
+                                                  int maxSize) {
+    List<CombinedFileRange> combinedFileRanges = VectoredReadUtils
+            .mergeSortedRanges(inputRanges, chunkSize, minimumSeek, maxSize);
+    Assertions.assertThat(combinedFileRanges)
+            .describedAs("Mismatch in number of ranges post merging")
+            .hasSize(inputRanges.size());
+  }
+
+  interface Stream extends PositionedReadable, ByteBufferPositionedReadable {
+    // nothing
+  }
+
+  static void fillBuffer(ByteBuffer buffer) {
+    byte b = 0;
+    while (buffer.remaining() > 0) {
+      buffer.put(b++);
+    }
+  }
+
+  @Test
+  public void testReadRangeFromByteBufferPositionedReadable() throws Exception {
+    Stream stream = Mockito.mock(Stream.class);
+    Mockito.doAnswer(invocation -> {
+      fillBuffer(invocation.getArgument(1));
+      return null;
+    }).when(stream).readFully(ArgumentMatchers.anyLong(),
+                              ArgumentMatchers.any(ByteBuffer.class));
+    CompletableFuture<ByteBuffer> result =
+        VectoredReadUtils.readRangeFrom(stream, FileRange.createFileRange(1000, 100),
+        ByteBuffer::allocate);
+    assertFutureCompletedSuccessfully(result);
+    ByteBuffer buffer = result.get();
+    assertEquals("Size of result buffer", 100, buffer.remaining());
+    byte b = 0;
+    while (buffer.remaining() > 0) {
+      assertEquals("remain = " + buffer.remaining(), b++, buffer.get());
+    }
+
+    // test an IOException
+    Mockito.reset(stream);
+    Mockito.doThrow(new IOException("foo"))
+        .when(stream).readFully(ArgumentMatchers.anyLong(),
+                                ArgumentMatchers.any(ByteBuffer.class));
+    result =
+        VectoredReadUtils.readRangeFrom(stream, FileRange.createFileRange(1000, 100),
+            ByteBuffer::allocate);
+    assertFutureFailedExceptionally(result);
+  }
+
+  static void runReadRangeFromPositionedReadable(IntFunction<ByteBuffer> allocate)
+          throws Exception {
+    PositionedReadable stream = Mockito.mock(PositionedReadable.class);
+    Mockito.doAnswer(invocation -> {
+      byte b=0;
+      byte[] buffer = invocation.getArgument(1);
+      for(int i=0; i < buffer.length; ++i) {
+        buffer[i] = b++;
+      }
+      return null;
+    }).when(stream).readFully(ArgumentMatchers.anyLong(),
+        ArgumentMatchers.any(), ArgumentMatchers.anyInt(),
+        ArgumentMatchers.anyInt());
+    CompletableFuture<ByteBuffer> result =
+        VectoredReadUtils.readRangeFrom(stream, FileRange.createFileRange(1000, 100),
+            allocate);
+    assertFutureCompletedSuccessfully(result);
+    ByteBuffer buffer = result.get();
+    assertEquals("Size of result buffer", 100, buffer.remaining());
+    byte b = 0;
+    while (buffer.remaining() > 0) {
+      assertEquals("remain = " + buffer.remaining(), b++, buffer.get());
+    }
+
+    // test an IOException
+    Mockito.reset(stream);
+    Mockito.doThrow(new IOException("foo"))
+        .when(stream).readFully(ArgumentMatchers.anyLong(),
+        ArgumentMatchers.any(), ArgumentMatchers.anyInt(),
+        ArgumentMatchers.anyInt());
+    result =
+        VectoredReadUtils.readRangeFrom(stream, FileRange.createFileRange(1000, 100),
+            ByteBuffer::allocate);
+    assertFutureFailedExceptionally(result);
+  }
+
+  @Test
+  public void testReadRangeArray() throws Exception {
+    runReadRangeFromPositionedReadable(ByteBuffer::allocate);
+  }
+
+  @Test
+  public void testReadRangeDirect() throws Exception {
+    runReadRangeFromPositionedReadable(ByteBuffer::allocateDirect);
+  }
+
+  static void validateBuffer(String message, ByteBuffer buffer, int start) {
+    byte expected = (byte) start;
+    while (buffer.remaining() > 0) {
+      assertEquals(message + " remain: " + buffer.remaining(), expected++,
+          buffer.get());
+    }
+  }
+
+  @Test
+  public void testReadVectored() throws Exception {
+    List<FileRange> input = Arrays.asList(FileRange.createFileRange(0, 100),
+        FileRange.createFileRange(100_000, 100),
+        FileRange.createFileRange(200_000, 100));
+    runAndValidateVectoredRead(input);
+  }
+
+  @Test
+  public void testReadVectoredZeroBytes() throws Exception {
+    List<FileRange> input = Arrays.asList(FileRange.createFileRange(0, 0),
+            FileRange.createFileRange(100_000, 100),
+            FileRange.createFileRange(200_000, 0));
+    runAndValidateVectoredRead(input);
+  }
+
+
+  private void runAndValidateVectoredRead(List<FileRange> input)
+          throws Exception {
+    Stream stream = Mockito.mock(Stream.class);
+    Mockito.doAnswer(invocation -> {
+      fillBuffer(invocation.getArgument(1));
+      return null;
+    }).when(stream).readFully(ArgumentMatchers.anyLong(),
+            ArgumentMatchers.any(ByteBuffer.class));
+    // should not merge the ranges
+    VectoredReadUtils.readVectored(stream, input, ByteBuffer::allocate);
+    Mockito.verify(stream, Mockito.times(3))
+            .readFully(ArgumentMatchers.anyLong(), ArgumentMatchers.any(ByteBuffer.class));
+    for (int b = 0; b < input.size(); ++b) {
+      validateBuffer("buffer " + b, input.get(b).getData().get(), 0);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java
index 798841a2d6905..9782eb276d306 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java
@@ -132,6 +132,15 @@ private AbstractStringAssert<?> assertContextValue(final String key) {
         .describedAs("Value of context element %s", key)
         .isNotBlank();
   }
+  /**
+   * Assert a context value is null.
+   * @param key key to look up
+   */
+  private void assertContextValueIsNull(final String key) {
+    assertThat(context.get(key))
+        .describedAs("Value of context element %s", key)
+        .isNull();
+  }
 
   @Test
   public void testNoteEntryPoint() throws Throwable {
@@ -158,4 +167,13 @@ private AbstractStringAssert<?> assertGlobalEntry(final String key) {
     return anAssert;
   }
 
+  @Test
+  public void testAddRemove() throws Throwable {
+    final String key = "testAddRemove";
+    assertContextValueIsNull(key);
+    context.put(key, key);
+    assertContextValue(key).isEqualTo(key);
+    context.remove(key);
+    assertContextValueIsNull(key);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java
new file mode 100644
index 0000000000000..5e5c917395413
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractContentSummaryTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+public abstract class AbstractContractContentSummaryTest extends AbstractFSContractTestBase {
+
+  @Test
+  public void testGetContentSummary() throws Throwable {
+    FileSystem fs = getFileSystem();
+
+    Path parent = path("parent");
+    Path nested = path(parent + "/a/b/c");
+    Path filePath = path(nested + "file.txt");
+
+    fs.mkdirs(parent);
+    fs.mkdirs(nested);
+    touch(getFileSystem(), filePath);
+
+    ContentSummary summary = fs.getContentSummary(parent);
+
+    Assertions.assertThat(summary.getDirectoryCount()).as("Summary " + summary).isEqualTo(4);
+
+    Assertions.assertThat(summary.getFileCount()).as("Summary " + summary).isEqualTo(1);
+  }
+
+  @Test
+  public void testGetContentSummaryIncorrectPath() throws Throwable {
+    FileSystem fs = getFileSystem();
+
+    Path parent = path("parent");
+    Path nested = path(parent + "/a");
+
+    fs.mkdirs(parent);
+
+    intercept(FileNotFoundException.class, () -> fs.getContentSummary(nested));
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
index 85e1f849998c3..91d19ecad1ec6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java
@@ -264,13 +264,8 @@ public void testCreatedFileIsEventuallyVisible() throws Throwable {
     describe("verify a written to file is visible after the stream is closed");
     Path path = path("testCreatedFileIsEventuallyVisible");
     FileSystem fs = getFileSystem();
-    try(
-      FSDataOutputStream out = fs.create(path,
-          false,
-          4096,
-          (short) 1,
-          1024)
-      ) {
+    try(FSDataOutputStream out = fs.create(path, false, 4096, (short) 1,
+        1024)) {
       out.write(0x01);
       out.close();
       getFileStatusEventually(fs, path, CREATE_TIMEOUT);
@@ -568,14 +563,14 @@ protected void validateSyncableSemantics(final FileSystem fs,
           in.close();
 
         } catch (FileNotFoundException e) {
-          // that's OK if it's an object store, but not if its a real
-          // FS
-          if (!isSupported(IS_BLOBSTORE)) {
+          // that's OK if it's an object store or some file systems that newly created files
+          // are not immediately visible, but not if its a real FS
+          if (!isSupported(IS_BLOBSTORE) && !isSupported(CREATE_VISIBILITY_DELAYED)) {
             throw e;
           } else {
             LOG.warn(
-                "Output file was not created; this is an object store with different"
-                    + " visibility semantics");
+                "Output file was not created; this is an object store or "
+                    + "a file system with different visibility semantics");
           }
         }
       }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java
index 08df1d4d883a6..605ea45649a16 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.contract;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
+
 import org.junit.Test;
 
 import java.io.IOException;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java
new file mode 100644
index 0000000000000..e7a121b704677
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractEtagTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import java.nio.charset.StandardCharsets;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_AVAILABLE;
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME;
+
+/**
+ * For filesystems which support etags, validate correctness
+ * of their implementation.
+ */
+public abstract class AbstractContractEtagTest extends
+    AbstractFSContractTestBase {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(AbstractContractEtagTest.class);
+
+  /**
+   * basic consistency across operations, as well as being non-empty.
+   */
+  @Test
+  public void testEtagConsistencyAcrossListAndHead() throws Throwable {
+    describe("Etag values must be non-empty and consistent across LIST and HEAD Calls.");
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+
+    Assertions.assertThat(fs.hasPathCapability(path, ETAGS_AVAILABLE))
+        .describedAs("path capability %s of %s",
+            ETAGS_AVAILABLE, path)
+        .isTrue();
+
+    ContractTestUtils.touch(fs, path);
+
+    final FileStatus st = fs.getFileStatus(path);
+    final String etag = etagFromStatus(st);
+    LOG.info("etag of empty file is \"{}\"", etag);
+
+    final FileStatus[] statuses = fs.listStatus(path);
+    Assertions.assertThat(statuses)
+        .describedAs("List(%s)", path)
+        .hasSize(1);
+    final FileStatus lsStatus = statuses[0];
+    Assertions.assertThat(etagFromStatus(lsStatus))
+        .describedAs("etag of list status (%s) compared to HEAD value of %s", lsStatus, st)
+        .isEqualTo(etag);
+  }
+
+  /**
+   * Get an etag from a FileStatus which MUST BE
+   * an implementation of EtagSource and
+   * whose etag MUST NOT BE null/empty.
+   * @param st the status
+   * @return the etag
+   */
+  String etagFromStatus(FileStatus st) {
+    Assertions.assertThat(st)
+        .describedAs("FileStatus %s", st)
+        .isInstanceOf(EtagSource.class);
+    final String etag = ((EtagSource) st).getEtag();
+    Assertions.assertThat(etag)
+        .describedAs("Etag of %s", st)
+        .isNotBlank();
+    return etag;
+  }
+
+  /**
+   * Overwritten data has different etags.
+   */
+  @Test
+  public void testEtagsOfDifferentDataDifferent() throws Throwable {
+    describe("Verify that two different blocks of data written have different tags");
+
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    Path src = new Path(path, "src");
+
+    ContractTestUtils.createFile(fs, src, true,
+        "data1234".getBytes(StandardCharsets.UTF_8));
+    final FileStatus srcStatus = fs.getFileStatus(src);
+    final String srcTag = etagFromStatus(srcStatus);
+    LOG.info("etag of file 1 is \"{}\"", srcTag);
+
+    // now overwrite with data of same length
+    // (ensure that path or length aren't used exclusively as tag)
+    ContractTestUtils.createFile(fs, src, true,
+        "1234data".getBytes(StandardCharsets.UTF_8));
+
+    // validate
+    final String tag2 = etagFromStatus(fs.getFileStatus(src));
+    LOG.info("etag of file 2 is \"{}\"", tag2);
+
+    Assertions.assertThat(tag2)
+        .describedAs("etag of updated file")
+        .isNotEqualTo(srcTag);
+  }
+
+  /**
+   * If supported, rename preserves etags.
+   */
+  @Test
+  public void testEtagConsistencyAcrossRename() throws Throwable {
+    describe("Verify that when a file is renamed, the etag remains unchanged");
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    Assume.assumeTrue(
+        "Filesystem does not declare that etags are preserved across renames",
+        fs.hasPathCapability(path, ETAGS_PRESERVED_IN_RENAME));
+    Path src = new Path(path, "src");
+    Path dest = new Path(path, "dest");
+
+    ContractTestUtils.createFile(fs, src, true,
+        "sample data".getBytes(StandardCharsets.UTF_8));
+    final FileStatus srcStatus = fs.getFileStatus(src);
+    LOG.info("located file status string value " + srcStatus);
+
+    final String srcTag = etagFromStatus(srcStatus);
+    LOG.info("etag of short file is \"{}\"", srcTag);
+
+    Assertions.assertThat(srcTag)
+        .describedAs("Etag of %s", srcStatus)
+        .isNotBlank();
+
+    // rename
+    fs.rename(src, dest);
+
+    // validate
+    FileStatus destStatus = fs.getFileStatus(dest);
+    final String destTag = etagFromStatus(destStatus);
+    Assertions.assertThat(destTag)
+        .describedAs("etag of list status (%s) compared to HEAD value of %s",
+            destStatus, srcStatus)
+        .isEqualTo(srcTag);
+  }
+
+  /**
+   * For effective use of etags, listLocatedStatus SHOULD return status entries
+   * with consistent values.
+   * This ensures that listing during query planning can collect and use the etags.
+   */
+  @Test
+  public void testLocatedStatusAlsoHasEtag() throws Throwable {
+    describe("verify that listLocatedStatus() and listFiles() are etag sources");
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    Path src = new Path(path, "src");
+    ContractTestUtils.createFile(fs, src, true,
+        "sample data".getBytes(StandardCharsets.UTF_8));
+    final FileStatus srcStatus = fs.getFileStatus(src);
+    final String srcTag = etagFromStatus(srcStatus);
+    final LocatedFileStatus entry = fs.listLocatedStatus(path).next();
+    LOG.info("located file status string value " + entry);
+    final String listTag = etagFromStatus(entry);
+    Assertions.assertThat(listTag)
+        .describedAs("etag of listLocatedStatus (%s) compared to HEAD value of %s",
+            entry, srcStatus)
+        .isEqualTo(srcTag);
+
+    final LocatedFileStatus entry2 = fs.listFiles(path, false).next();
+    Assertions.assertThat(etagFromStatus(entry2))
+        .describedAs("etag of listFiles (%s) compared to HEAD value of %s",
+            entry, srcStatus)
+        .isEqualTo(srcTag);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 3e754e4578de8..c395afdb3779b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -50,11 +50,11 @@
 import org.apache.hadoop.util.DurationInfo;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
-import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
 import static org.apache.hadoop.test.LambdaTestUtils.eventually;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * Tests of multipart uploads.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
index a43053180fbf8..25bfe082b01f6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
@@ -30,14 +30,18 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.IOUtils;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.compareByteArrays;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 import org.junit.Test;
 
@@ -232,7 +236,7 @@ public void testAwaitFutureFailToFNFE() throws Throwable {
         getFileSystem().openFile(path("testAwaitFutureFailToFNFE"))
             .opt("fs.test.something", true);
     intercept(FileNotFoundException.class,
-        () -> FutureIOSupport.awaitFuture(builder.build()));
+        () -> awaitFuture(builder.build()));
   }
 
   @Test
@@ -242,7 +246,7 @@ public void testAwaitFutureTimeoutFailToFNFE() throws Throwable {
         getFileSystem().openFile(path("testAwaitFutureFailToFNFE"))
             .opt("fs.test.something", true);
     intercept(FileNotFoundException.class,
-        () -> FutureIOSupport.awaitFuture(builder.build(),
+        () -> awaitFuture(builder.build(),
             10, TimeUnit.DAYS));
   }
 
@@ -250,7 +254,7 @@ public void testAwaitFutureTimeoutFailToFNFE() throws Throwable {
   public void testOpenFileExceptionallyTranslating() throws Throwable {
     describe("openFile missing file chains into exceptionally()");
     CompletableFuture<FSDataInputStream> f = getFileSystem()
-        .openFile(path("testOpenFileUnknownOption")).build();
+        .openFile(path("testOpenFileExceptionallyTranslating")).build();
     interceptFuture(RuntimeException.class,
         "exceptionally",
         f.exceptionally(ex -> {
@@ -262,11 +266,12 @@ public void testOpenFileExceptionallyTranslating() throws Throwable {
   public void testChainedFailureAwaitFuture() throws Throwable {
     describe("await Future handles chained failures");
     CompletableFuture<FSDataInputStream> f = getFileSystem()
-        .openFile(path("testOpenFileUnknownOption"))
+        .openFile(path("testChainedFailureAwaitFuture"))
+        .withFileStatus(null)
         .build();
     intercept(RuntimeException.class,
         "exceptionally",
-        () -> FutureIOSupport.awaitFuture(
+        () -> awaitFuture(
             f.exceptionally(ex -> {
               throw new RuntimeException("exceptionally", ex);
             })));
@@ -280,13 +285,34 @@ public void testOpenFileApplyRead() throws Throwable {
     int len = 4096;
     createFile(fs, path, true,
         dataset(len, 0x40, 0x80));
+    FileStatus st = fs.getFileStatus(path);
     CompletableFuture<Long> readAllBytes = fs.openFile(path)
-        .withFileStatus(fs.getFileStatus(path))
+        .withFileStatus(st)
         .build()
         .thenApply(ContractTestUtils::readStream);
     assertEquals("Wrong number of bytes read value",
         len,
         (long) readAllBytes.get());
+    // now reattempt with a new FileStatus and a different path
+    // other than the final name element
+    // implementations MUST use path in openFile() call
+    FileStatus st2 = new FileStatus(
+        len, false,
+        st.getReplication(),
+        st.getBlockSize(),
+        st.getModificationTime(),
+        st.getAccessTime(),
+        st.getPermission(),
+        st.getOwner(),
+        st.getGroup(),
+        new Path("gopher:///localhost:/" + path.getName()));
+    assertEquals("Wrong number of bytes read value",
+        len,
+        (long) fs.openFile(path)
+            .withFileStatus(st2)
+            .build()
+            .thenApply(ContractTestUtils::readStream)
+            .get());
   }
 
   @Test
@@ -298,17 +324,47 @@ public void testOpenFileApplyAsyncRead() throws Throwable {
         dataset(4, 0x40, 0x80));
     CompletableFuture<FSDataInputStream> future = fs.openFile(path).build();
     AtomicBoolean accepted = new AtomicBoolean(false);
-    future.thenAcceptAsync(i -> accepted.set(true)).get();
+    future.thenApply(stream -> {
+      accepted.set(true);
+      return stream;
+    }).get().close();
     assertTrue("async accept operation not invoked",
         accepted.get());
   }
 
+  /**
+   * Open a file with a null status, and the length
+   * passed in as an opt() option (along with sequential IO).
+   * The file is opened, the data read, and it must match
+   * the source data.
+   * opt() is used so that integration testing with external
+   * filesystem connectors will downgrade if the option is not
+   * recognized.
+   */
   @Test
-  public void testOpenFileNullStatus() throws Throwable {
-    describe("use openFile() with a null status");
+  public void testOpenFileNullStatusButFileLength() throws Throwable {
+    describe("use openFile() with a null status and expect the status to be"
+        + " ignored. block size, fadvise and length are passed in as"
+        + " opt() options");
     Path path = path("testOpenFileNullStatus");
-    intercept(NullPointerException.class,
-        () -> getFileSystem().openFile(path).withFileStatus(null));
+    FileSystem fs = getFileSystem();
+    int len = 4;
+    byte[] result = new byte[len];
+    byte[] dataset = dataset(len, 0x40, 0x80);
+    createFile(fs, path, true,
+        dataset);
+    CompletableFuture<FSDataInputStream> future = fs.openFile(path)
+        .withFileStatus(null)
+        .opt(FS_OPTION_OPENFILE_READ_POLICY,
+            "unknown, sequential, random")
+        .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, 32768)
+        .opt(FS_OPTION_OPENFILE_LENGTH, len)
+        .build();
+
+    try (FSDataInputStream in = future.get()) {
+      in.readFully(result);
+    }
+    compareByteArrays(dataset, result, len);
   }
 
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java
new file mode 100644
index 0000000000000..a39201df24943
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.IntFunction;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileRange;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.impl.FutureIOSupport;
+import org.apache.hadoop.io.WeakReferencedElasticByteBufferPool;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.functional.FutureIO;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertCapabilities;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDatasetEquals;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.returnBuffersToPoolPostRead;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
+
+@RunWith(Parameterized.class)
+public abstract class AbstractContractVectoredReadTest extends AbstractFSContractTestBase {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractContractVectoredReadTest.class);
+
+  public static final int DATASET_LEN = 64 * 1024;
+  protected static final byte[] DATASET = ContractTestUtils.dataset(DATASET_LEN, 'a', 32);
+  protected static final String VECTORED_READ_FILE_NAME = "vectored_file.txt";
+
+  private final IntFunction<ByteBuffer> allocate;
+
+  private final WeakReferencedElasticByteBufferPool pool =
+          new WeakReferencedElasticByteBufferPool();
+
+  private final String bufferType;
+
+  @Parameterized.Parameters(name = "Buffer type : {0}")
+  public static List<String> params() {
+    return Arrays.asList("direct", "array");
+  }
+
+  public AbstractContractVectoredReadTest(String bufferType) {
+    this.bufferType = bufferType;
+    this.allocate = value -> {
+      boolean isDirect = !"array".equals(bufferType);
+      return pool.getBuffer(isDirect, value);
+    };
+  }
+
+  public IntFunction<ByteBuffer> getAllocate() {
+    return allocate;
+  }
+
+  public WeakReferencedElasticByteBufferPool getPool() {
+    return pool;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    Path path = path(VECTORED_READ_FILE_NAME);
+    FileSystem fs = getFileSystem();
+    createFile(fs, path, true, DATASET);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    super.teardown();
+    pool.release();
+  }
+
+  @Test
+  public void testVectoredReadCapability() throws Exception {
+    FileSystem fs = getFileSystem();
+    String[] vectoredReadCapability = new String[]{StreamCapabilities.VECTOREDIO};
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      assertCapabilities(in, vectoredReadCapability, null);
+    }
+  }
+
+  @Test
+  public void testVectoredReadMultipleRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      FileRange fileRange = FileRange.createFileRange(i * 100, 100);
+      fileRanges.add(fileRange);
+    }
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      CompletableFuture<?>[] completableFutures = new CompletableFuture<?>[fileRanges.size()];
+      int i = 0;
+      for (FileRange res : fileRanges) {
+        completableFutures[i++] = res.getData();
+      }
+      CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(completableFutures);
+      combinedFuture.get();
+
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testVectoredReadAndReadFully()  throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(100, 100));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      byte[] readFullRes = new byte[100];
+      in.readFully(100, readFullRes);
+      ByteBuffer vecRes = FutureIOSupport.awaitFuture(fileRanges.get(0).getData());
+      Assertions.assertThat(vecRes)
+              .describedAs("Result from vectored read and readFully must match")
+              .isEqualByComparingTo(ByteBuffer.wrap(readFullRes));
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  /**
+   * As the minimum seek value is 4*1024,none of the below ranges
+   * will get merged.
+   */
+  @Test
+  public void testDisjointRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(0, 100));
+    fileRanges.add(FileRange.createFileRange(4_000 + 101, 100));
+    fileRanges.add(FileRange.createFileRange(16_000 + 101, 100));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  /**
+   * As the minimum seek value is 4*1024, all the below ranges
+   * will get merged into one.
+   */
+  @Test
+  public void testAllRangesMergedIntoOne() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(0, 100));
+    fileRanges.add(FileRange.createFileRange(4_000 - 101, 100));
+    fileRanges.add(FileRange.createFileRange(8_000 - 101, 100));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  /**
+   * As the minimum seek value is 4*1024, the first three ranges will be
+   * merged into and other two will remain as it is.
+   */
+  @Test
+  public void testSomeRangesMergedSomeUnmerged() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(8 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(14 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(10 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(2 * 1024 - 101, 100));
+    fileRanges.add(FileRange.createFileRange(40 * 1024, 1024));
+    FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME));
+    CompletableFuture<FSDataInputStream> builder =
+            fs.openFile(path(VECTORED_READ_FILE_NAME))
+                    .withFileStatus(fileStatus)
+                    .build();
+    try (FSDataInputStream in = builder.get()) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testOverlappingRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = getSampleOverlappingRanges();
+    FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME));
+    CompletableFuture<FSDataInputStream> builder =
+            fs.openFile(path(VECTORED_READ_FILE_NAME))
+                    .withFileStatus(fileStatus)
+                    .build();
+    try (FSDataInputStream in = builder.get()) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testSameRanges() throws Exception {
+    // Same ranges are special case of overlapping only.
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = getSampleSameRanges();
+    CompletableFuture<FSDataInputStream> builder =
+            fs.openFile(path(VECTORED_READ_FILE_NAME))
+                    .build();
+    try (FSDataInputStream in = builder.get()) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testSomeRandomNonOverlappingRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(500, 100));
+    fileRanges.add(FileRange.createFileRange(1000, 200));
+    fileRanges.add(FileRange.createFileRange(50, 10));
+    fileRanges.add(FileRange.createFileRange(10, 5));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testConsecutiveRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(500, 100));
+    fileRanges.add(FileRange.createFileRange(600, 200));
+    fileRanges.add(FileRange.createFileRange(800, 100));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  /**
+   * Test to validate EOF ranges. Default implementation fails with EOFException
+   * while reading the ranges. Some implementation like s3, checksum fs fail fast
+   * as they already have the file length calculated.
+   */
+  @Test
+  public void testEOFRanges()  throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(DATASET_LEN, 100));
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      for (FileRange res : fileRanges) {
+        CompletableFuture<ByteBuffer> data = res.getData();
+        interceptFuture(EOFException.class,
+                "",
+                ContractTestUtils.VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS,
+                TimeUnit.SECONDS,
+                data);
+      }
+    }
+  }
+
+  @Test
+  public void testNegativeLengthRange()  throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(0, -50));
+    verifyExceptionalVectoredRead(fs, fileRanges, IllegalArgumentException.class);
+  }
+
+  @Test
+  public void testNegativeOffsetRange()  throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(-1, 50));
+    verifyExceptionalVectoredRead(fs, fileRanges, EOFException.class);
+  }
+
+  @Test
+  public void testNormalReadAfterVectoredRead() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = createSampleNonOverlappingRanges();
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, allocate);
+      // read starting 200 bytes
+      byte[] res = new byte[200];
+      in.read(res, 0, 200);
+      ByteBuffer buffer = ByteBuffer.wrap(res);
+      assertDatasetEquals(0, "normal_read", buffer, 200, DATASET);
+      Assertions.assertThat(in.getPos())
+              .describedAs("Vectored read shouldn't change file pointer.")
+              .isEqualTo(200);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testVectoredReadAfterNormalRead() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = createSampleNonOverlappingRanges();
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      // read starting 200 bytes
+      byte[] res = new byte[200];
+      in.read(res, 0, 200);
+      ByteBuffer buffer = ByteBuffer.wrap(res);
+      assertDatasetEquals(0, "normal_read", buffer, 200, DATASET);
+      Assertions.assertThat(in.getPos())
+              .describedAs("Vectored read shouldn't change file pointer.")
+              .isEqualTo(200);
+      in.readVectored(fileRanges, allocate);
+      validateVectoredReadResult(fileRanges, DATASET);
+      returnBuffersToPoolPostRead(fileRanges, pool);
+    }
+  }
+
+  @Test
+  public void testMultipleVectoredReads() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges1 = createSampleNonOverlappingRanges();
+    List<FileRange> fileRanges2 = createSampleNonOverlappingRanges();
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges1, allocate);
+      in.readVectored(fileRanges2, allocate);
+      validateVectoredReadResult(fileRanges2, DATASET);
+      validateVectoredReadResult(fileRanges1, DATASET);
+      returnBuffersToPoolPostRead(fileRanges1, pool);
+      returnBuffersToPoolPostRead(fileRanges2, pool);
+    }
+  }
+
+  /**
+   * This test creates list of ranges and then submit a readVectored
+   * operation and then uses a separate thread pool to process the
+   * results asynchronously.
+   */
+  @Test
+  public void testVectoredIOEndToEnd() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(8 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(14 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(10 * 1024, 100));
+    fileRanges.add(FileRange.createFileRange(2 * 1024 - 101, 100));
+    fileRanges.add(FileRange.createFileRange(40 * 1024, 1024));
+
+    ExecutorService dataProcessor = Executors.newFixedThreadPool(5);
+    CountDownLatch countDown = new CountDownLatch(fileRanges.size());
+
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))) {
+      in.readVectored(fileRanges, value -> pool.getBuffer(true, value));
+      for (FileRange res : fileRanges) {
+        dataProcessor.submit(() -> {
+          try {
+            readBufferValidateDataAndReturnToPool(res, countDown);
+          } catch (Exception e) {
+            String error = String.format("Error while processing result for %s", res);
+            LOG.error(error, e);
+            ContractTestUtils.fail(error, e);
+          }
+        });
+      }
+      // user can perform other computations while waiting for IO.
+      if (!countDown.await(VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
+        ContractTestUtils.fail("Timeout/Error while processing vectored io results");
+      }
+    } finally {
+      HadoopExecutors.shutdown(dataProcessor, LOG,
+              VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+    }
+  }
+
+  private void readBufferValidateDataAndReturnToPool(FileRange res,
+                                                     CountDownLatch countDownLatch)
+          throws IOException, TimeoutException {
+    CompletableFuture<ByteBuffer> data = res.getData();
+    // Read the data and perform custom operation. Here we are just
+    // validating it with original data.
+    FutureIO.awaitFuture(data.thenAccept(buffer -> {
+      assertDatasetEquals((int) res.getOffset(),
+              "vecRead", buffer, res.getLength(), DATASET);
+      // return buffer to the pool once read.
+      pool.putBuffer(buffer);
+    }),
+    VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+    // countdown to notify main thread that processing has been done.
+    countDownLatch.countDown();
+  }
+
+
+  protected List<FileRange> createSampleNonOverlappingRanges() {
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(0, 100));
+    fileRanges.add(FileRange.createFileRange(110, 50));
+    return fileRanges;
+  }
+
+  protected List<FileRange> getSampleSameRanges() {
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(8_000, 1000));
+    fileRanges.add(FileRange.createFileRange(8_000, 1000));
+    fileRanges.add(FileRange.createFileRange(8_000, 1000));
+    return fileRanges;
+  }
+
+  protected List<FileRange> getSampleOverlappingRanges() {
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(100, 500));
+    fileRanges.add(FileRange.createFileRange(400, 500));
+    return fileRanges;
+  }
+
+  protected List<FileRange> getConsecutiveRanges() {
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(100, 500));
+    fileRanges.add(FileRange.createFileRange(600, 500));
+    return fileRanges;
+  }
+
+  /**
+   * Validate that exceptions must be thrown during a vectored
+   * read operation with specific input ranges.
+   * @param fs FileSystem instance.
+   * @param fileRanges input file ranges.
+   * @param clazz type of exception expected.
+   * @throws Exception any other IOE.
+   */
+  protected <T extends Throwable> void verifyExceptionalVectoredRead(
+          FileSystem fs,
+          List<FileRange> fileRanges,
+          Class<T> clazz) throws Exception {
+
+    CompletableFuture<FSDataInputStream> builder =
+            fs.openFile(path(VECTORED_READ_FILE_NAME))
+                    .build();
+    try (FSDataInputStream in = builder.get()) {
+      intercept(clazz,
+          () -> in.readVectored(fileRanges, allocate));
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index e13a49ca10e70..b61abddd43426 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -21,6 +21,7 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileRange;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -28,7 +29,11 @@
 import org.apache.hadoop.fs.PathCapabilities;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.io.ByteBufferPool;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.FutureIO;
+
 import org.junit.Assert;
 import org.junit.AssumptionViolatedException;
 import org.slf4j.Logger;
@@ -39,6 +44,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -49,6 +55,9 @@
 import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
@@ -68,6 +77,11 @@ public class ContractTestUtils extends Assert {
   public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
   public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
 
+  /**
+   * Timeout in seconds for vectored read operation in tests : {@value}.
+   */
+  public static final int VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS = 5 * 60;
+
   /**
    * Assert that a property in the property set matches the expected value.
    * @param props property set
@@ -1095,6 +1109,78 @@ public static void validateFileContent(byte[] concat, byte[][] bytes) {
                 mismatch);
   }
 
+  /**
+   * Utility to validate vectored read results.
+   * @param fileRanges input ranges.
+   * @param originalData original data.
+   * @throws IOException any ioe.
+   */
+  public static void validateVectoredReadResult(List<FileRange> fileRanges,
+                                                byte[] originalData)
+          throws IOException, TimeoutException {
+    CompletableFuture<?>[] completableFutures = new CompletableFuture<?>[fileRanges.size()];
+    int i = 0;
+    for (FileRange res : fileRanges) {
+      completableFutures[i++] = res.getData();
+    }
+    CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(completableFutures);
+    FutureIO.awaitFuture(combinedFuture,
+            VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS,
+            TimeUnit.SECONDS);
+
+    for (FileRange res : fileRanges) {
+      CompletableFuture<ByteBuffer> data = res.getData();
+      ByteBuffer buffer = FutureIO.awaitFuture(data,
+              VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS,
+              TimeUnit.SECONDS);
+      assertDatasetEquals((int) res.getOffset(), "vecRead",
+              buffer, res.getLength(), originalData);
+    }
+  }
+
+  /**
+   * Utility to return buffers back to the pool once all
+   * data has been read for each file range.
+   * @param fileRanges list of file range.
+   * @param pool buffer pool.
+   * @throws IOException any IOE
+   * @throws TimeoutException ideally this should never occur.
+   */
+  public static void returnBuffersToPoolPostRead(List<FileRange> fileRanges,
+                                                 ByteBufferPool pool)
+          throws IOException, TimeoutException {
+    for (FileRange range : fileRanges) {
+      ByteBuffer buffer = FutureIO.awaitFuture(range.getData(),
+              VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS,
+              TimeUnit.SECONDS);
+      pool.putBuffer(buffer);
+    }
+  }
+
+
+  /**
+   * Assert that the data read matches the dataset at the given offset.
+   * This helps verify that the seek process is moving the read pointer
+   * to the correct location in the file.
+   *  @param readOffset the offset in the file where the read began.
+   * @param operation  operation name for the assertion.
+   * @param data       data read in.
+   * @param length     length of data to check.
+   * @param originalData original data.
+   */
+  public static void assertDatasetEquals(
+          final int readOffset,
+          final String operation,
+          final ByteBuffer data,
+          int length, byte[] originalData) {
+    for (int i = 0; i < length; i++) {
+      int o = readOffset + i;
+      assertEquals(operation + " with read offset " + readOffset
+                      + ": data[" + i + "] != DATASET[" + o + "]",
+              originalData[o], data.get());
+    }
+  }
+
   /**
    * Receives test data from the given input file and checks the size of the
    * data as well as the pattern inside the received data.
@@ -1446,11 +1532,7 @@ public static TreeScanResults treeWalk(FileSystem fs, Path path)
    */
   public static List<LocatedFileStatus> toList(
       RemoteIterator<LocatedFileStatus> iterator) throws IOException {
-    ArrayList<LocatedFileStatus> list = new ArrayList<>();
-    while (iterator.hasNext()) {
-      list.add(iterator.next());
-    }
-    return list;
+    return RemoteIterators.toList(iterator);
   }
 
   /**
@@ -1464,11 +1546,7 @@ public static List<LocatedFileStatus> toList(
    */
   public static <T extends FileStatus> List<T> iteratorToList(
           RemoteIterator<T> iterator) throws IOException {
-    List<T> list = new ArrayList<>();
-    while (iterator.hasNext()) {
-      list.add(iterator.next());
-    }
-    return list;
+    return RemoteIterators.toList(iterator);
   }
 
 
@@ -1642,17 +1720,22 @@ public static int read(InputStream in) {
 
   /**
    * Read a whole stream; downgrades an IOE to a runtime exception.
+   * Closes the stream afterwards.
    * @param in input
    * @return the number of bytes read.
    * @throws AssertionError on any IOException
    */
   public static long readStream(InputStream in) {
-    long count = 0;
+    try {
+      long count = 0;
 
-    while (read(in) >= 0) {
-      count++;
+      while (read(in) >= 0) {
+        count++;
+      }
+      return count;
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, in);
     }
-    return count;
   }
 
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java
index 1efd7fc4e95d4..62648ec58bcc7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java
@@ -22,7 +22,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
-import org.junit.Assert;
 
 import java.net.URI;
 
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java
similarity index 80%
rename from hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java
rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java
index 65d031cd3980e..7555cf85158f9 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractContentSummary.java
@@ -16,16 +16,16 @@
  *  limitations under the License.
  */
 
-package org.apache.hadoop.fs.swift.contract;
+package org.apache.hadoop.fs.contract.localfs;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractContractContentSummaryTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-public class TestSwiftContractDelete extends AbstractContractDeleteTest {
+public class TestLocalFSContractContentSummary extends AbstractContractContentSummaryTest {
 
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
+    return new LocalFSContract(conf);
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java
new file mode 100644
index 0000000000000..5ee888015315c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.localfs;
+
+import java.io.EOFException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ChecksumException;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileRange;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractVectoredReadTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+public class TestLocalFSContractVectoredRead extends AbstractContractVectoredReadTest {
+
+  public TestLocalFSContractVectoredRead(String bufferType) {
+    super(bufferType);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new LocalFSContract(conf);
+  }
+
+  @Test
+  public void testChecksumValidationDuringVectoredRead() throws Exception {
+    Path testPath = path("big_range_checksum_file");
+    List<FileRange> someRandomRanges = new ArrayList<>();
+    someRandomRanges.add(FileRange.createFileRange(10, 1024));
+    someRandomRanges.add(FileRange.createFileRange(1025, 1024));
+    validateCheckReadException(testPath, DATASET_LEN, someRandomRanges);
+  }
+
+
+  /**
+   * Test for file size less than checksum chunk size.
+   * {@code ChecksumFileSystem#bytesPerChecksum}.
+   */
+  @Test
+  public void testChecksumValidationDuringVectoredReadSmallFile() throws Exception {
+    Path testPath = path("big_range_checksum_file");
+    final int length = 471;
+    List<FileRange> smallFileRanges = new ArrayList<>();
+    smallFileRanges.add(FileRange.createFileRange(10, 50));
+    smallFileRanges.add(FileRange.createFileRange(100, 20));
+    validateCheckReadException(testPath, length, smallFileRanges);
+  }
+
+  private void validateCheckReadException(Path testPath,
+                                          int length,
+                                          List<FileRange> ranges) throws Exception {
+    LocalFileSystem localFs = (LocalFileSystem) getFileSystem();
+    final byte[] datasetCorrect = ContractTestUtils.dataset(length, 'a', 32);
+    try (FSDataOutputStream out = localFs.create(testPath, true)){
+      out.write(datasetCorrect);
+    }
+    Path checksumPath = localFs.getChecksumFile(testPath);
+    Assertions.assertThat(localFs.exists(checksumPath))
+            .describedAs("Checksum file should be present")
+            .isTrue();
+    CompletableFuture<FSDataInputStream> fis = localFs.openFile(testPath).build();
+    try (FSDataInputStream in = fis.get()){
+      in.readVectored(ranges, getAllocate());
+      validateVectoredReadResult(ranges, datasetCorrect);
+    }
+    final byte[] datasetCorrupted = ContractTestUtils.dataset(length, 'a', 64);
+    try (FSDataOutputStream out = localFs.getRaw().create(testPath, true)){
+      out.write(datasetCorrupted);
+    }
+    CompletableFuture<FSDataInputStream> fisN = localFs.openFile(testPath).build();
+    try (FSDataInputStream in = fisN.get()){
+      in.readVectored(ranges, getAllocate());
+      // Expect checksum exception when data is updated directly through
+      // raw local fs instance.
+      intercept(ChecksumException.class,
+          () -> validateVectoredReadResult(ranges, datasetCorrupted));
+    }
+  }
+  @Test
+  public void tesChecksumVectoredReadBoundaries() throws Exception {
+    Path testPath = path("boundary_range_checksum_file");
+    final int length = 1071;
+    LocalFileSystem localFs = (LocalFileSystem) getFileSystem();
+    final byte[] datasetCorrect = ContractTestUtils.dataset(length, 'a', 32);
+    try (FSDataOutputStream out = localFs.create(testPath, true)){
+      out.write(datasetCorrect);
+    }
+    Path checksumPath = localFs.getChecksumFile(testPath);
+    Assertions.assertThat(localFs.exists(checksumPath))
+            .describedAs("Checksum file should be present at {} ", checksumPath)
+            .isTrue();
+    CompletableFuture<FSDataInputStream> fis = localFs.openFile(testPath).build();
+    List<FileRange> smallRange = new ArrayList<>();
+    smallRange.add(FileRange.createFileRange(1000, 71));
+    try (FSDataInputStream in = fis.get()){
+      in.readVectored(smallRange, getAllocate());
+      validateVectoredReadResult(smallRange, datasetCorrect);
+    }
+  }
+
+
+  /**
+   * Overriding in checksum fs as vectored read api fails fast
+   * in case of EOF requested range.
+   */
+  @Override
+  public void testEOFRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(DATASET_LEN, 100));
+    verifyExceptionalVectoredRead(fs, fileRanges, EOFException.class);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java
index 2cb5414caa4c7..6eb24985f4ff3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.contract.rawlocal;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
+
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractVectoredRead.java
similarity index 61%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java
rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractVectoredRead.java
index c25a630cc298d..cbb31ffe27a59 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyRequest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractVectoredRead.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -16,26 +16,20 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.swift.http;
+package org.apache.hadoop.fs.contract.rawlocal;
 
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractVectoredReadTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-/**
- * Implementation for SwiftRestClient to make copy requests.
- * COPY is a method that came with WebDAV (RFC2518), and is not something that
- * can be handled by all proxies en-route to a filesystem.
- */
-class CopyRequest extends HttpEntityEnclosingRequestBase {
+public class TestRawLocalContractVectoredRead extends AbstractContractVectoredReadTest {
 
-  CopyRequest() {
-    super();
+  public TestRawLocalContractVectoredRead(String bufferType) {
+    super(bufferType);
   }
 
-  /**
-   * @return http method name
-   */
   @Override
-  public String getMethod() {
-    return "COPY";
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new RawlocalFSContract(conf);
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
index 3c088d278e536..c34269708ddcb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractPathHandle.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.contract.AbstractContractPathHandleTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.localfs.LocalFSContract;
 import org.apache.hadoop.fs.contract.rawlocal.RawlocalFSContract;
 
 public class TestRawlocalContractPathHandle
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java
new file mode 100644
index 0000000000000..82348d97798ea
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+public final class ExceptionAsserts {
+
+  private ExceptionAsserts() {
+  }
+
+
+  /**
+   * Asserts that the given code throws an exception of the given type
+   * and that the exception message contains the given sub-message.
+   *
+   * Usage:
+   *
+   * ExceptionAsserts.assertThrows(
+   *   IllegalArgumentException.class,
+   *   "'nullArg' must not be null",
+   *   () -> Preconditions.checkNotNull(null, "nullArg"));
+   *
+   * Note: JUnit 5 has similar functionality but it will be a long time before
+   * we move to that framework because of significant differences and lack of
+   * backward compatibility for some JUnit rules.
+   */
+  public static <E extends Exception> void assertThrows(
+      Class<E> expectedExceptionClass,
+      String partialMessage,
+      LambdaTestUtils.VoidCallable code) throws Exception {
+
+    intercept(expectedExceptionClass, partialMessage, code);
+
+  }
+
+  public static <E extends Exception> void assertThrows(
+      Class<E> expectedExceptionClass,
+      LambdaTestUtils.VoidCallable code) throws Exception {
+
+    intercept(expectedExceptionClass, code);
+
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java
new file mode 100644
index 0000000000000..b6f744582d3e2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Frequently used test data items.
+ */
+public final class SampleDataForTests {
+
+  private SampleDataForTests() {
+  }
+
+
+  // Array data.
+  public static final Object[] NULL_ARRAY = null;
+
+  public static final Object[] EMPTY_ARRAY = new Object[0];
+
+  public static final Object[] NON_EMPTY_ARRAY = new Object[1];
+
+  public static final byte[] NULL_BYTE_ARRAY = null;
+
+  public static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+
+  public static final byte[] NON_EMPTY_BYTE_ARRAY = new byte[1];
+
+  public static final short[] NULL_SHORT_ARRAY = null;
+
+  public static final short[] EMPTY_SHORT_ARRAY = new short[0];
+
+  public static final short[] NON_EMPTY_SHORT_ARRAY = new short[1];
+
+  public static final int[] NULL_INT_ARRAY = null;
+
+  public static final int[] EMPTY_INT_ARRAY = new int[0];
+
+  public static final int[] NON_EMPTY_INT_ARRAY = new int[1];
+
+  public static final long[] NULL_LONG_ARRAY = null;
+
+  public static final long[] EMPTY_LONG_ARRAY = new long[0];
+
+  public static final long[] NON_EMPTY_LONG_ARRAY = new long[1];
+
+  public static final List<Object> NULL_LIST = null;
+
+  public static final List<Object> EMPTY_LIST = new ArrayList<Object>();
+
+  public static final List<Object> VALID_LIST = Arrays.asList(new Object[1]);
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java
new file mode 100644
index 0000000000000..2ea041283a7e1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+public class TestBlockCache extends AbstractHadoopTestBase {
+
+  private static final int BUFFER_SIZE = 16;
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    BlockCache cache =
+        new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance());
+
+    ByteBuffer buffer = ByteBuffer.allocate(16);
+
+    // Verify it throws correctly.
+    intercept(IllegalArgumentException.class, "'buffer' must not be null",
+        () -> cache.put(42, null));
+
+
+    intercept(NullPointerException.class, null,
+        () -> new SingleFilePerBlockCache(null));
+
+  }
+
+
+  @Test
+  public void testPutAndGet() throws Exception {
+    BlockCache cache =
+        new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance());
+
+    ByteBuffer buffer1 = ByteBuffer.allocate(BUFFER_SIZE);
+    for (byte i = 0; i < BUFFER_SIZE; i++) {
+      buffer1.put(i);
+    }
+
+    assertEquals(0, cache.size());
+    assertFalse(cache.containsBlock(0));
+    cache.put(0, buffer1);
+    assertEquals(1, cache.size());
+    assertTrue(cache.containsBlock(0));
+    ByteBuffer buffer2 = ByteBuffer.allocate(BUFFER_SIZE);
+    cache.get(0, buffer2);
+    assertNotSame(buffer1, buffer2);
+    assertBuffersEqual(buffer1, buffer2);
+
+    assertEquals(1, cache.size());
+    assertFalse(cache.containsBlock(1));
+    cache.put(1, buffer1);
+    assertEquals(2, cache.size());
+    assertTrue(cache.containsBlock(1));
+    ByteBuffer buffer3 = ByteBuffer.allocate(BUFFER_SIZE);
+    cache.get(1, buffer3);
+    assertNotSame(buffer1, buffer3);
+    assertBuffersEqual(buffer1, buffer3);
+  }
+
+  private void assertBuffersEqual(ByteBuffer buffer1, ByteBuffer buffer2) {
+    assertNotNull(buffer1);
+    assertNotNull(buffer2);
+    assertEquals(buffer1.limit(), buffer2.limit());
+    assertEquals(BUFFER_SIZE, buffer1.limit());
+    for (int i = 0; i < BUFFER_SIZE; i++) {
+      assertEquals(buffer1.get(i), buffer2.get(i));
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java
new file mode 100644
index 0000000000000..50ce220f6527e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestBlockData extends AbstractHadoopTestBase {
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    new BlockData(10, 5);
+    new BlockData(5, 10);
+    new BlockData(0, 10);
+
+    // Verify it throws correctly.
+
+
+    intercept(IllegalArgumentException.class, "'fileSize' must not be negative",
+        () -> new BlockData(-1, 2));
+
+    intercept(IllegalArgumentException.class,
+        "'blockSize' must be a positive integer",
+        () -> new BlockData(10, 0));
+
+    intercept(IllegalArgumentException.class,
+        "'blockSize' must be a positive integer",
+        () -> new BlockData(10, -2));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' (-1) must be within the range [0, 3]",
+        () -> new BlockData(10, 3).isLastBlock(
+            -1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' (11) must be within the range [0, 3]",
+        () -> new BlockData(10, 3).isLastBlock(
+            11));
+
+  }
+
+  @Test
+  public void testComputedFields() throws Exception {
+    testComputedFieldsHelper(0, 10);
+    testComputedFieldsHelper(1, 10);
+    testComputedFieldsHelper(10, 1);
+    testComputedFieldsHelper(10, 2);
+    testComputedFieldsHelper(10, 3);
+  }
+
+  private void testComputedFieldsHelper(long fileSize, int blockSize)
+      throws Exception {
+    BlockData bd = new BlockData(fileSize, blockSize);
+
+    if (fileSize == 0) {
+      assertFalse(bd.isLastBlock(0));
+      assertFalse(bd.isLastBlock(1));
+      assertFalse(bd.isValidOffset(0));
+      assertEquals(0, bd.getSize(0));
+      assertEquals("", bd.getStateString());
+
+      ExceptionAsserts.assertThrows(
+          IllegalArgumentException.class,
+          "'offset' (0) must be within the range [0, -1]",
+          () -> bd.getBlockNumber(0));
+
+      ExceptionAsserts.assertThrows(
+          IllegalArgumentException.class,
+          "'blockNumber' (0) must be within the range [0, -1]",
+          () -> bd.getStartOffset(0));
+
+      ExceptionAsserts.assertThrows(
+          IllegalArgumentException.class,
+          "'offset' (0) must be within the range [0, -1]",
+          () -> bd.getRelativeOffset(0, 0));
+
+      ExceptionAsserts.assertThrows(
+          IllegalArgumentException.class,
+          "'blockNumber' (0) must be within the range [0, -1]",
+          () -> bd.getState(0));
+
+      ExceptionAsserts.assertThrows(
+          IllegalArgumentException.class,
+          "'blockNumber' (0) must be within the range [0, -1]",
+          () -> bd.setState(0, BlockData.State.READY));
+
+      return;
+    }
+
+    assertEquals(fileSize, bd.getFileSize());
+    assertEquals(blockSize, bd.getBlockSize());
+
+    int expectedNumBlocks = (int) (fileSize / blockSize);
+    if (fileSize % blockSize > 0) {
+      expectedNumBlocks++;
+    }
+    assertEquals(expectedNumBlocks, bd.getNumBlocks());
+
+    int lastBlockNumber = expectedNumBlocks - 1;
+    for (int b = 0; b < lastBlockNumber; b++) {
+      assertFalse(bd.isLastBlock(b));
+      assertEquals(blockSize, bd.getSize(b));
+    }
+    assertTrue(bd.isLastBlock(lastBlockNumber));
+    int lastBlockSize = (int) (fileSize - blockSize * (expectedNumBlocks - 1));
+    assertEquals(lastBlockSize, bd.getSize(lastBlockNumber));
+
+    // Offset related methods.
+    for (long offset = 0; offset < fileSize; offset++) {
+      int expectedBlockNumber = (int) (offset / blockSize);
+      assertEquals(expectedBlockNumber, bd.getBlockNumber(offset));
+
+      for (int b = 0; b < expectedNumBlocks - 1; b++) {
+        long expectedStartOffset = b * blockSize;
+        assertEquals(expectedStartOffset, bd.getStartOffset(b));
+
+        int expectedRelativeOffset = (int) (offset - expectedStartOffset);
+        assertEquals(expectedRelativeOffset, bd.getRelativeOffset(b, offset));
+      }
+    }
+
+
+    // State methods.
+    for (int b = 0; b < expectedNumBlocks; b++) {
+      assertEquals(b * blockSize, bd.getStartOffset(b));
+      assertEquals(BlockData.State.NOT_READY, bd.getState(b));
+      bd.setState(b, BlockData.State.QUEUED);
+      assertEquals(BlockData.State.QUEUED, bd.getState(b));
+      bd.setState(b, BlockData.State.READY);
+      assertEquals(BlockData.State.READY, bd.getState(b));
+      bd.setState(b, BlockData.State.CACHED);
+      assertEquals(BlockData.State.CACHED, bd.getState(b));
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java
new file mode 100644
index 0000000000000..703041379ab6e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.lang.reflect.Method;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertTrue;
+
+public class TestBlockOperations extends AbstractHadoopTestBase {
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    BlockOperations ops = new BlockOperations();
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.getPrefetched(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.getCached(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.getRead(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.release(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.requestPrefetch(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> ops.requestCaching(-1));
+
+  }
+
+  @Test
+  public void testGetSummary() throws Exception {
+    verifySummary("getPrefetched", "GP");
+    verifySummary("getCached", "GC");
+    verifySummary("getRead", "GR");
+    verifySummary("release", "RL");
+    verifySummary("requestPrefetch", "RP");
+    verifySummary("prefetch", "PF");
+    verifySummary("requestCaching", "RC");
+    verifySummary("addToCache", "C+");
+
+    verifySummaryNoArg("cancelPrefetches", "CP");
+    verifySummaryNoArg("close", "CX");
+  }
+
+  private void verifySummary(String methodName, String shortName)
+      throws Exception {
+    int blockNumber = 42;
+    BlockOperations ops = new BlockOperations();
+    Method method = ops.getClass().getDeclaredMethod(methodName, int.class);
+    BlockOperations.Operation op =
+        (BlockOperations.Operation) method.invoke(ops, blockNumber);
+    ops.end(op);
+    String summary = ops.getSummary(false);
+    String opSummary = String.format("%s(%d)", shortName, blockNumber);
+    String expectedSummary = String.format("%s;E%s;", opSummary, opSummary);
+    assertTrue(summary.startsWith(expectedSummary));
+  }
+
+  private void verifySummaryNoArg(String methodName, String shortName)
+      throws Exception {
+    BlockOperations ops = new BlockOperations();
+    Method method = ops.getClass().getDeclaredMethod(methodName);
+    BlockOperations.Operation op =
+        (BlockOperations.Operation) method.invoke(ops);
+    ops.end(op);
+    String summary = ops.getSummary(false);
+    String expectedSummary = String.format("%s;E%s;", shortName, shortName);
+    assertTrue(summary.startsWith(expectedSummary));
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java
new file mode 100644
index 0000000000000..fc29e1b725405
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class TestBoundedResourcePool extends AbstractHadoopTestBase {
+
+  static class BufferPool extends BoundedResourcePool<ByteBuffer> {
+
+    BufferPool(int size) {
+      super(size);
+    }
+
+    @Override
+    protected ByteBuffer createNew() {
+      return ByteBuffer.allocate(10);
+    }
+  }
+
+  @Test
+  public void testArgChecks() throws Exception {
+
+    // Should not throw.
+    BufferPool pool = new BufferPool(5);
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> new BufferPool(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> new BufferPool(0));
+
+    intercept(IllegalArgumentException.class, "'item' must not be null",
+        () -> pool.release(null));
+
+    intercept(IllegalArgumentException.class,
+        "This item is not a part of this pool",
+        () -> pool.release(ByteBuffer.allocate(4)));
+
+  }
+
+  @Test
+  public void testAcquireReleaseSingle() {
+    final int numBuffers = 5;
+    BufferPool pool = new BufferPool(numBuffers);
+
+    assertEquals(0, pool.numCreated());
+    assertEquals(numBuffers, pool.numAvailable());
+
+    ByteBuffer buffer1 = pool.acquire();
+    assertNotNull(buffer1);
+    assertEquals(1, pool.numCreated());
+    assertEquals(numBuffers - 1, pool.numAvailable());
+
+    // Release and immediately reacquire => should not end up creating new buffer.
+    pool.release(buffer1);
+    assertEquals(1, pool.numCreated());
+
+    ByteBuffer buffer2 = pool.acquire();
+    assertNotNull(buffer2);
+    assertSame(buffer1, buffer2);
+    assertEquals(1, pool.numCreated());
+  }
+
+  @Test
+  public void testAcquireReleaseMultiple() {
+    final int numBuffers = 5;
+    BufferPool pool = new BufferPool(numBuffers);
+    Set<ByteBuffer> buffers =
+        Collections.newSetFromMap(new IdentityHashMap<ByteBuffer, Boolean>());
+
+    assertEquals(0, pool.numCreated());
+
+    // Acquire all one by one.
+    for (int i = 0; i < numBuffers; i++) {
+      assertEquals(numBuffers - i, pool.numAvailable());
+      ByteBuffer buffer = pool.acquire();
+      assertNotNull(buffer);
+      assertFalse(buffers.contains(buffer));
+      buffers.add(buffer);
+      assertEquals(i + 1, pool.numCreated());
+    }
+
+    assertEquals(numBuffers, pool.numCreated());
+    assertEquals(0, pool.numAvailable());
+
+    int releaseCount = 0;
+
+    // Release all one by one.
+    for (ByteBuffer buffer : buffers) {
+      assertEquals(releaseCount, pool.numAvailable());
+      releaseCount++;
+      pool.release(buffer);
+      assertEquals(releaseCount, pool.numAvailable());
+
+      // Releasing the same buffer again should not have any ill effect.
+      pool.release(buffer);
+      assertEquals(releaseCount, pool.numAvailable());
+      pool.release(buffer);
+      assertEquals(releaseCount, pool.numAvailable());
+    }
+
+    // Acquire all one by one again to ensure that they are the same ones we got earlier.
+    for (int i = 0; i < numBuffers; i++) {
+      ByteBuffer buffer = pool.acquire();
+      assertTrue(buffers.contains(buffer));
+    }
+
+    assertEquals(numBuffers, pool.numCreated());
+    assertEquals(0, pool.numAvailable());
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java
new file mode 100644
index 0000000000000..ee5f95ca6bbb6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+import java.nio.ReadOnlyBufferException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+public class TestBufferData extends AbstractHadoopTestBase {
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    ByteBuffer buffer = ByteBuffer.allocate(1);
+    BufferData data = new BufferData(1, buffer);
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> new BufferData(-1, buffer));
+
+    intercept(IllegalArgumentException.class, "'buffer' must not be null",
+        () -> new BufferData(1, null));
+
+    intercept(IllegalArgumentException.class, "'actionFuture' must not be null",
+        () -> data.setPrefetch(null));
+
+    intercept(IllegalArgumentException.class, "'actionFuture' must not be null",
+        () -> data.setCaching(null));
+
+    intercept(IllegalArgumentException.class, "'states' must not be null",
+        () -> data.throwIfStateIncorrect((BufferData.State[]) null));
+
+    intercept(IllegalStateException.class,
+        "Expected buffer state to be 'READY or CACHING' but found",
+        () -> data.throwIfStateIncorrect(BufferData.State.READY,
+            BufferData.State.CACHING));
+
+  }
+
+  @Test
+  public void testValidStateUpdates() {
+    ByteBuffer buffer = ByteBuffer.allocate(1);
+    BufferData data = new BufferData(1, buffer);
+
+    assertEquals(BufferData.State.BLANK, data.getState());
+
+    CompletableFuture<Void> actionFuture = new CompletableFuture<>();
+    actionFuture.complete(null);
+    data.setPrefetch(actionFuture);
+    assertEquals(BufferData.State.PREFETCHING, data.getState());
+    assertNotNull(data.getActionFuture());
+    assertSame(actionFuture, data.getActionFuture());
+
+    CompletableFuture<Void> actionFuture2 = new CompletableFuture<>();
+    data.setCaching(actionFuture2);
+    assertEquals(BufferData.State.CACHING, data.getState());
+    assertNotNull(data.getActionFuture());
+    assertSame(actionFuture2, data.getActionFuture());
+    assertNotSame(actionFuture, actionFuture2);
+
+    List<BufferData.State> states = Arrays.asList(
+        BufferData.State.BLANK,
+        BufferData.State.PREFETCHING,
+        BufferData.State.CACHING,
+        BufferData.State.READY
+    );
+
+    BufferData data2 = new BufferData(1, buffer);
+    BufferData.State prevState = null;
+    for (BufferData.State state : states) {
+      if (prevState != null) {
+        assertEquals(prevState, data2.getState());
+        data2.updateState(state, prevState);
+        assertEquals(state, data2.getState());
+      }
+      prevState = state;
+    }
+  }
+
+  @Test
+  public void testInvalidStateUpdates() throws Exception {
+    CompletableFuture<Void> actionFuture = new CompletableFuture<>();
+    actionFuture.complete(null);
+    testInvalidStateUpdatesHelper(
+        (d) -> d.setPrefetch(actionFuture),
+        BufferData.State.BLANK,
+        BufferData.State.READY);
+
+    testInvalidStateUpdatesHelper(
+        (d) -> d.setCaching(actionFuture),
+        BufferData.State.PREFETCHING,
+        BufferData.State.READY);
+  }
+
+  @Test
+  public void testSetReady() throws Exception {
+    byte[] bytes1 = new byte[5];
+    initBytes(bytes1);
+
+    ByteBuffer buffer = ByteBuffer.allocate(10);
+    buffer.put(bytes1);
+    buffer.limit(bytes1.length);
+    BufferData data = new BufferData(1, buffer);
+    assertNotEquals(BufferData.State.READY, data.getState());
+    assertEquals(0, data.getChecksum());
+
+    data.setReady(BufferData.State.BLANK);
+    assertEquals(BufferData.State.READY, data.getState());
+    assertNotEquals(0, data.getChecksum());
+
+    // Verify that buffer cannot be modified once in READY state.
+    ExceptionAsserts.assertThrows(
+        ReadOnlyBufferException.class,
+        null,
+        () -> data.getBuffer().put(bytes1));
+
+    // Verify that buffer cannot be set to READY state more than once.
+    ExceptionAsserts.assertThrows(
+        IllegalStateException.class,
+        "Checksum cannot be changed once set",
+        () -> data.setReady(BufferData.State.BLANK));
+
+    // Verify that we detect post READY buffer modification.
+    buffer.array()[2] = (byte) 42;
+    ExceptionAsserts.assertThrows(
+        IllegalStateException.class,
+        "checksum changed after setReady()",
+        () -> data.setDone());
+  }
+
+  @Test
+  public void testChecksum() {
+    byte[] bytes1 = new byte[5];
+    byte[] bytes2 = new byte[10];
+
+    initBytes(bytes1);
+    initBytes(bytes2);
+
+    ByteBuffer buffer1 = ByteBuffer.wrap(bytes1);
+    ByteBuffer buffer2 = ByteBuffer.wrap(bytes2);
+    buffer2.limit(bytes1.length);
+
+    long checksum1 = BufferData.getChecksum(buffer1);
+    long checksum2 = BufferData.getChecksum(buffer2);
+
+    assertEquals(checksum1, checksum2);
+  }
+
+  private void initBytes(byte[] bytes) {
+    for (int i = 0; i < bytes.length; i++) {
+      bytes[i] = (byte) i;
+    }
+  }
+
+  @FunctionalInterface
+  public interface StateChanger {
+
+    void run(BufferData data) throws Exception;
+  }
+
+  private void testInvalidStateUpdatesHelper(
+      StateChanger changeState,
+      BufferData.State... validFromState) throws Exception {
+
+    ByteBuffer buffer = ByteBuffer.allocate(1);
+    BufferData data = new BufferData(1, buffer);
+    data.updateState(validFromState[0], BufferData.State.BLANK);
+    List<BufferData.State> states = this.getStatesExcept(validFromState);
+    BufferData.State prevState = validFromState[0];
+    String expectedMessage =
+        String.format("Expected buffer state to be '%s", validFromState[0]);
+    for (BufferData.State s : states) {
+      data.updateState(s, prevState);
+
+      ExceptionAsserts.assertThrows(
+          IllegalStateException.class,
+          expectedMessage,
+          () -> changeState.run(data));
+
+      assertEquals(s, data.getState());
+      prevState = s;
+    }
+  }
+
+  static final List<BufferData.State> ALL_STATES = Arrays.asList(
+      BufferData.State.UNKNOWN,
+      BufferData.State.BLANK,
+      BufferData.State.PREFETCHING,
+      BufferData.State.CACHING,
+      BufferData.State.READY
+  );
+
+  private List<BufferData.State> getStatesExcept(BufferData.State... states) {
+
+    List<BufferData.State> result = new ArrayList<>();
+    for (BufferData.State s : ALL_STATES) {
+      boolean found = false;
+      for (BufferData.State ss : states) {
+        if (s == ss) {
+          found = true;
+        }
+      }
+
+      if (!found) {
+        result.add(s);
+      }
+    }
+
+    return result;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java
new file mode 100644
index 0000000000000..b8375fe66dcb1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+public class TestBufferPool extends AbstractHadoopTestBase {
+
+  private static final int POOL_SIZE = 2;
+
+  private static final int BUFFER_SIZE = 10;
+
+  private final PrefetchingStatistics statistics =
+      EmptyPrefetchingStatistics.getInstance();
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics);
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> new BufferPool(0, 10, statistics));
+
+    intercept(IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> new BufferPool(-1, 10, statistics));
+
+    intercept(IllegalArgumentException.class,
+        "'bufferSize' must be a positive integer",
+        () -> new BufferPool(10, 0, statistics));
+
+    intercept(IllegalArgumentException.class,
+        "'bufferSize' must be a positive integer",
+        () -> new BufferPool(1, -10, statistics));
+
+    intercept(NullPointerException.class,
+        () -> new BufferPool(1, 10, null));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> pool.acquire(-1));
+
+    intercept(IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> pool.tryAcquire(-1));
+
+    intercept(NullPointerException.class, "data",
+        () -> pool.release((BufferData) null));
+
+  }
+
+  @Test
+  public void testGetAndRelease() {
+    BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics);
+    assertInitialState(pool, POOL_SIZE);
+
+    int count = 0;
+    for (BufferData data : pool.getAll()) {
+      count++;
+    }
+    assertEquals(0, count);
+
+    BufferData data1 = this.acquire(pool, 1);
+    BufferData data2 = this.acquire(pool, 2);
+    BufferData data3 = pool.tryAcquire(3);
+    assertNull(data3);
+
+    count = 0;
+    for (BufferData data : pool.getAll()) {
+      count++;
+    }
+    assertEquals(2, count);
+
+    assertEquals(2, pool.numCreated());
+    assertEquals(0, pool.numAvailable());
+
+    data1.updateState(BufferData.State.READY, BufferData.State.BLANK);
+    pool.release(data1);
+
+    assertEquals(2, pool.numCreated());
+    assertEquals(1, pool.numAvailable());
+
+    data2.updateState(BufferData.State.READY, BufferData.State.BLANK);
+    pool.release(data2);
+
+    assertEquals(2, pool.numCreated());
+    assertEquals(2, pool.numAvailable());
+  }
+
+  @Test
+  public void testRelease() throws Exception {
+    testReleaseHelper(BufferData.State.BLANK, true);
+    testReleaseHelper(BufferData.State.PREFETCHING, true);
+    testReleaseHelper(BufferData.State.CACHING, true);
+    testReleaseHelper(BufferData.State.READY, false);
+  }
+
+  private void testReleaseHelper(BufferData.State stateBeforeRelease,
+      boolean expectThrow)
+      throws Exception {
+
+    BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics);
+    assertInitialState(pool, POOL_SIZE);
+
+    BufferData data = this.acquire(pool, 1);
+    data.updateState(stateBeforeRelease, BufferData.State.BLANK);
+
+    if (expectThrow) {
+
+      intercept(IllegalArgumentException.class, "Unable to release buffer",
+          () -> pool.release(data));
+
+    } else {
+      pool.release(data);
+    }
+  }
+
+  private BufferData acquire(BufferPool pool, int blockNumber) {
+    BufferData data = pool.acquire(blockNumber);
+    assertNotNull(data);
+    assertSame(data, pool.acquire(blockNumber));
+    assertEquals(blockNumber, data.getBlockNumber());
+    return data;
+  }
+
+  private void assertInitialState(BufferPool pool, int poolSize) {
+    assertEquals(poolSize, pool.numAvailable());
+    assertEquals(0, pool.numCreated());
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java
new file mode 100644
index 0000000000000..3b8bc75f14989
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
+import static org.junit.Assert.assertTrue;
+
+public class TestExecutorServiceFuturePool extends AbstractHadoopTestBase {
+
+  private ExecutorService executorService;
+
+  @Before
+  public void setUp() {
+    executorService = Executors.newFixedThreadPool(3);
+  }
+
+  @After
+  public void tearDown() {
+    if (executorService != null) {
+      executorService.shutdownNow();
+    }
+  }
+
+  @Test
+  public void testRunnableSucceeds() throws Exception {
+    ExecutorServiceFuturePool futurePool =
+        new ExecutorServiceFuturePool(executorService);
+    final AtomicBoolean atomicBoolean = new AtomicBoolean(false);
+    Future<Void> future =
+        futurePool.executeRunnable(() -> atomicBoolean.set(true));
+    future.get(30, TimeUnit.SECONDS);
+    assertTrue("atomicBoolean set to true?", atomicBoolean.get());
+  }
+
+  @Test
+  public void testSupplierSucceeds() throws Exception {
+    ExecutorServiceFuturePool futurePool =
+        new ExecutorServiceFuturePool(executorService);
+    final AtomicBoolean atomicBoolean = new AtomicBoolean(false);
+    Future<Void> future = futurePool.executeFunction(() -> {
+      atomicBoolean.set(true);
+      return null;
+    });
+    future.get(30, TimeUnit.SECONDS);
+    assertTrue("atomicBoolean set to true?", atomicBoolean.get());
+  }
+
+  @Test
+  public void testRunnableFails() throws Exception {
+    ExecutorServiceFuturePool futurePool =
+        new ExecutorServiceFuturePool(executorService);
+    Future<Void> future = futurePool.executeRunnable(() -> {
+      throw new IllegalStateException("deliberate");
+    });
+    interceptFuture(IllegalStateException.class, "deliberate", 30,
+        TimeUnit.SECONDS, future);
+  }
+
+  @Test
+  public void testSupplierFails() throws Exception {
+    ExecutorServiceFuturePool futurePool =
+        new ExecutorServiceFuturePool(executorService);
+    Future<Void> future = futurePool.executeFunction(() -> {
+      throw new IllegalStateException("deliberate");
+    });
+    interceptFuture(IllegalStateException.class, "deliberate", 30,
+        TimeUnit.SECONDS, future);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
new file mode 100644
index 0000000000000..12ab62556a104
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestFilePosition extends AbstractHadoopTestBase {
+
+  @Test
+  public void testArgChecks() throws Exception {
+    ByteBuffer buffer = ByteBuffer.allocate(10);
+    BufferData data = new BufferData(0, buffer);
+
+    // Should not throw.
+    new FilePosition(0, 0);
+    new FilePosition(0, 5);
+    new FilePosition(10, 5);
+    new FilePosition(5, 10);
+    new FilePosition(10, 5).setData(data, 3, 4);
+    new FilePosition(10, 10).setData(data, 3, 13);
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class, "'fileSize' must not be negative",
+        () -> new FilePosition(-1, 2));
+
+    intercept(IllegalArgumentException.class,
+        "'blockSize' must be a positive integer",
+        () -> new FilePosition(1, 0));
+
+    intercept(IllegalArgumentException.class,
+        "'blockSize' must be a positive integer",
+        () -> new FilePosition(1, -1));
+
+    FilePosition pos = new FilePosition(10, 3);
+
+    // Verify that we cannot obtain buffer properties without setting buffer.
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.buffer());
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.absolute());
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.isWithinCurrentBuffer(2));
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.blockNumber());
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.isLastBlock());
+
+    intercept(IllegalStateException.class, "'buffer' must not be null",
+        () -> pos.bufferFullyRead());
+
+    // Verify that we cannot set invalid buffer parameters.
+
+    intercept(IllegalArgumentException.class, "'bufferData' must not be null",
+        () -> pos.setData(null, 4, 4));
+
+    intercept(IllegalArgumentException.class,
+        "'startOffset' must not be negative", () -> pos.setData(data, -4, 4));
+
+    intercept(IllegalArgumentException.class,
+        "'readOffset' must not be negative", () -> pos.setData(data, 4, -4));
+
+    intercept(IllegalArgumentException.class,
+        "'readOffset' must not be negative", () -> pos.setData(data, 4, -4));
+
+    intercept(IllegalArgumentException.class,
+        "'readOffset' (15) must be within the range [4, 14]",
+        () -> pos.setData(data, 4, 15));
+
+    intercept(IllegalArgumentException.class,
+        "'readOffset' (3) must be within the range [4, 14]",
+        () -> pos.setData(data, 4, 3));
+
+  }
+
+  @Test
+  public void testValidity() {
+    int bufferSize = 8;
+    long fileSize = 100;
+    long bufferStartOffset = 7;
+    long readStartOffset = 9;
+
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    BufferData data = new BufferData(0, buffer);
+    FilePosition pos = new FilePosition(fileSize, bufferSize);
+
+    assertFalse(pos.isValid());
+    pos.setData(data, bufferStartOffset, readStartOffset);
+    assertTrue(pos.isValid());
+
+    pos.invalidate();
+    assertFalse(pos.isValid());
+  }
+
+  @Test
+  public void testOffsets() {
+    int bufferSize = 8;
+    long fileSize = 100;
+    long bufferStartOffset = 7;
+    long readStartOffset = 9;
+
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    BufferData data = new BufferData(0, buffer);
+    FilePosition pos = new FilePosition(fileSize, bufferSize);
+    pos.setData(data, bufferStartOffset, readStartOffset);
+    assertTrue(pos.isValid());
+
+    assertEquals(readStartOffset, pos.absolute());
+    assertEquals(readStartOffset - bufferStartOffset, pos.relative());
+    assertTrue(pos.isWithinCurrentBuffer(8));
+    assertFalse(pos.isWithinCurrentBuffer(6));
+    assertFalse(pos.isWithinCurrentBuffer(1));
+
+    int expectedBlockNumber = (int) (bufferStartOffset / bufferSize);
+    assertEquals(expectedBlockNumber, pos.blockNumber());
+    assertFalse(pos.isLastBlock());
+
+    pos.setData(data, fileSize - 3, fileSize - 2);
+    assertTrue(pos.isLastBlock());
+  }
+
+  @Test
+  public void testBufferStats() {
+    int bufferSize = 8;
+    long fileSize = 100;
+    long bufferStartOffset = 7;
+    long readStartOffset = 9;
+
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    BufferData data = new BufferData(0, buffer);
+    FilePosition pos = new FilePosition(fileSize, bufferSize);
+    pos.setData(data, bufferStartOffset, readStartOffset);
+    assertTrue(pos.isValid());
+    assertEquals(bufferStartOffset, pos.bufferStartOffset());
+
+    assertEquals(0, pos.numBytesRead());
+    assertEquals(0, pos.numSingleByteReads());
+    assertEquals(0, pos.numBufferReads());
+
+    pos.incrementBytesRead(1);
+    pos.incrementBytesRead(1);
+    pos.incrementBytesRead(1);
+    pos.incrementBytesRead(5);
+    pos.incrementBytesRead(51);
+
+    assertEquals(59, pos.numBytesRead());
+    assertEquals(3, pos.numSingleByteReads());
+    assertEquals(2, pos.numBufferReads());
+
+    assertFalse(pos.bufferFullyRead());
+
+    pos.setData(data, bufferStartOffset, bufferStartOffset);
+    assertTrue(pos.isValid());
+
+    assertEquals(0, pos.numBytesRead());
+    assertEquals(0, pos.numSingleByteReads());
+    assertEquals(0, pos.numBufferReads());
+
+    for (int i = 0; i < bufferSize; i++) {
+      pos.buffer().get();
+      pos.incrementBytesRead(1);
+    }
+    assertTrue(pos.bufferFullyRead());
+  }
+
+  @Test
+  public void testBounds() {
+    int bufferSize = 8;
+    long fileSize = bufferSize;
+
+    ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+    BufferData data = new BufferData(0, buffer);
+    FilePosition pos = new FilePosition(fileSize, bufferSize);
+
+    long eofOffset = fileSize;
+    pos.setData(data, 0, eofOffset);
+
+    assertThat(pos.isWithinCurrentBuffer(eofOffset))
+        .describedAs("EOF offset %d should be within the current buffer", eofOffset)
+        .isTrue();
+    assertThat(pos.absolute())
+        .describedAs("absolute() should return the EOF offset")
+        .isEqualTo(eofOffset);
+
+    assertThat(pos.setAbsolute(eofOffset))
+        .describedAs("setAbsolute() should return true on the EOF offset %d", eofOffset)
+        .isTrue();
+    assertThat(pos.absolute())
+        .describedAs("absolute() should return the EOF offset")
+        .isEqualTo(eofOffset);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java
new file mode 100644
index 0000000000000..50701c717a4b0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestRetryer extends AbstractHadoopTestBase {
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    new Retryer(10, 50, 500);
+
+    // Verify it throws correctly.
+
+    intercept(IllegalArgumentException.class,
+        "'perRetryDelay' must be a positive integer",
+        () -> new Retryer(-1, 50, 500));
+
+    intercept(IllegalArgumentException.class,
+        "'perRetryDelay' must be a positive integer",
+        () -> new Retryer(0, 50, 500));
+
+    intercept(IllegalArgumentException.class,
+        "'maxDelay' (5) must be greater than 'perRetryDelay' (10)",
+        () -> new Retryer(10, 5, 500));
+
+    intercept(IllegalArgumentException.class,
+        "'statusUpdateInterval' must be a positive integer",
+        () -> new Retryer(10, 50, -1));
+
+    intercept(IllegalArgumentException.class,
+        "'statusUpdateInterval' must be a positive integer",
+        () -> new Retryer(10, 50, 0));
+
+  }
+
+  @Test
+  public void testRetry() {
+    int perRetryDelay = 1;
+    int statusUpdateInterval = 3;
+    int maxDelay = 10;
+
+    Retryer retryer =
+        new Retryer(perRetryDelay, maxDelay, statusUpdateInterval);
+    for (int t = 1; t <= maxDelay; t++) {
+      assertTrue(retryer.continueRetry());
+      if (t % statusUpdateInterval == 0) {
+        assertTrue(retryer.updateStatus());
+      } else {
+        assertFalse(retryer.updateStatus());
+      }
+    }
+
+    assertFalse(retryer.continueRetry());
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java
new file mode 100644
index 0000000000000..a42462b3355af
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_BYTE_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_INT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_LIST;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_LONG_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_SHORT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_BYTE_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_INT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_LONG_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_SHORT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_BYTE_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_INT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_LIST;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_LONG_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_SHORT_ARRAY;
+import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.VALID_LIST;
+import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+public class TestValidate extends AbstractHadoopTestBase {
+
+  @Test
+  public void testCheckNotNull() throws Exception {
+    String nonNullArg = "nonNullArg";
+    String nullArg = null;
+
+    // Should not throw.
+    Validate.checkNotNull(nonNullArg, "nonNullArg");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class, "'nullArg' must not be null",
+        () -> Validate.checkNotNull(nullArg, "nullArg"));
+
+  }
+
+  @Test
+  public void testCheckPositiveInteger() throws Exception {
+    int positiveArg = 1;
+    int zero = 0;
+    int negativeArg = -1;
+
+    // Should not throw.
+    checkPositiveInteger(positiveArg, "positiveArg");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'negativeArg' must be a positive integer",
+        () -> checkPositiveInteger(negativeArg, "negativeArg"));
+
+    intercept(IllegalArgumentException.class,
+        "'zero' must be a positive integer",
+        () -> checkPositiveInteger(zero, "zero"));
+
+  }
+
+  @Test
+  public void testCheckNotNegative() throws Exception {
+    int positiveArg = 1;
+    int zero = 0;
+    int negativeArg = -1;
+
+    // Should not throw.
+    Validate.checkNotNegative(zero, "zeroArg");
+    Validate.checkNotNegative(positiveArg, "positiveArg");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'negativeArg' must not be negative",
+        () -> Validate.checkNotNegative(negativeArg, "negativeArg"));
+
+  }
+
+  @Test
+  public void testCheckRequired() throws Exception {
+    // Should not throw.
+    Validate.checkRequired(true, "arg");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class, "'arg' is required",
+        () -> Validate.checkRequired(false, "arg"));
+
+  }
+
+  @Test
+  public void testCheckValid() throws Exception {
+    // Should not throw.
+    Validate.checkValid(true, "arg");
+
+    // Verify it throws.
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'arg' is invalid",
+        () -> Validate.checkValid(false, "arg"));
+  }
+
+  @Test
+  public void testCheckValidWithValues() throws Exception {
+    String validValues = "foo, bar";
+
+    // Should not throw.
+    Validate.checkValid(true, "arg", validValues);
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg' is invalid. Valid values are: foo, bar",
+        () -> Validate.checkValid(false, "arg", validValues));
+
+  }
+
+  @Test
+  public void testCheckNotNullAndNotEmpty() throws Exception {
+    // Should not throw.
+    Validate.checkNotNullAndNotEmpty(NON_EMPTY_ARRAY, "array");
+    Validate.checkNotNullAndNotEmpty(NON_EMPTY_BYTE_ARRAY, "array");
+    Validate.checkNotNullAndNotEmpty(NON_EMPTY_SHORT_ARRAY, "array");
+    Validate.checkNotNullAndNotEmpty(NON_EMPTY_INT_ARRAY, "array");
+    Validate.checkNotNullAndNotEmpty(NON_EMPTY_LONG_ARRAY, "array");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class, "'string' must not be empty",
+        () -> Validate.checkNotNullAndNotEmpty("", "string"));
+
+    intercept(IllegalArgumentException.class, "'array' must not be null", () ->
+        Validate.checkNotNullAndNotEmpty(SampleDataForTests.NULL_ARRAY,
+            "array"));
+
+    intercept(IllegalArgumentException.class,
+        "'array' must have at least one element", () ->
+            Validate.checkNotNullAndNotEmpty(SampleDataForTests.EMPTY_ARRAY,
+                "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must not be null",
+        () -> Validate.checkNotNullAndNotEmpty(NULL_BYTE_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must have at least one element",
+        () -> Validate.checkNotNullAndNotEmpty(EMPTY_BYTE_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must not be null",
+        () -> Validate.checkNotNullAndNotEmpty(NULL_SHORT_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must have at least one element",
+        () -> Validate.checkNotNullAndNotEmpty(EMPTY_SHORT_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must not be null",
+        () -> Validate.checkNotNullAndNotEmpty(NULL_INT_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must have at least one element",
+        () -> Validate.checkNotNullAndNotEmpty(EMPTY_INT_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must not be null",
+        () -> Validate.checkNotNullAndNotEmpty(NULL_LONG_ARRAY, "array"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'array' must have at least one element",
+        () -> Validate.checkNotNullAndNotEmpty(EMPTY_LONG_ARRAY, "array"));
+  }
+
+  @Test
+  public void testCheckListNotNullAndNotEmpty() throws Exception {
+    // Should not throw.
+    Validate.checkNotNullAndNotEmpty(VALID_LIST, "list");
+
+    // Verify it throws.
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'list' must not be null",
+        () -> Validate.checkNotNullAndNotEmpty(NULL_LIST, "list"));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'list' must have at least one element",
+        () -> Validate.checkNotNullAndNotEmpty(EMPTY_LIST, "list"));
+  }
+
+  @Test
+  public void testCheckNotNullAndNumberOfElements() throws Exception {
+    // Should not throw.
+    Validate.checkNotNullAndNumberOfElements(Arrays.asList(1, 2, 3), 3, "arg");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class, "'arg' must not be null",
+        () -> Validate.checkNotNullAndNumberOfElements(null, 3, "arg"));
+
+    // Verify it throws.
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "Number of elements in 'arg' must be exactly 3, 2 given.",
+        () -> Validate.checkNotNullAndNumberOfElements(Arrays.asList(1, 2), 3,
+            "arg")
+    );
+  }
+
+  @Test
+  public void testCheckValuesEqual() throws Exception {
+    // Should not throw.
+    Validate.checkValuesEqual(1, "arg1", 1, "arg2");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg1' (1) must equal 'arg2' (2)",
+        () -> Validate.checkValuesEqual(1, "arg1", 2, "arg2"));
+
+  }
+
+  @Test
+  public void testCheckIntegerMultiple() throws Exception {
+    // Should not throw.
+    Validate.checkIntegerMultiple(10, "arg1", 5, "arg2");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg1' (10) must be an integer multiple of 'arg2' (3)",
+        () -> Validate.checkIntegerMultiple(10, "arg1", 3, "arg2"));
+
+  }
+
+  @Test
+  public void testCheckGreater() throws Exception {
+    // Should not throw.
+    Validate.checkGreater(10, "arg1", 5, "arg2");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg1' (5) must be greater than 'arg2' (10)",
+        () -> Validate.checkGreater(5, "arg1", 10, "arg2"));
+
+  }
+
+  @Test
+  public void testCheckGreaterOrEqual() throws Exception {
+    // Should not throw.
+    Validate.checkGreaterOrEqual(10, "arg1", 5, "arg2");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg1' (5) must be greater than or equal to 'arg2' (10)",
+        () -> Validate.checkGreaterOrEqual(5, "arg1", 10, "arg2"));
+
+  }
+
+  @Test
+  public void testCheckWithinRange() throws Exception {
+    // Should not throw.
+    Validate.checkWithinRange(10, "arg", 5, 15);
+    Validate.checkWithinRange(10.0, "arg", 5.0, 15.0);
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class,
+        "'arg' (5) must be within the range [10, 20]",
+        () -> Validate.checkWithinRange(5, "arg", 10, 20));
+
+    intercept(IllegalArgumentException.class,
+        "'arg' (5.0) must be within the range [10.0, 20.0]",
+        () -> Validate.checkWithinRange(5.0, "arg", 10.0, 20.0));
+
+  }
+
+  @Test
+  public void testCheckPathExists() throws Exception {
+    Path tempFile = Files.createTempFile("foo", "bar");
+    Path tempDir = tempFile.getParent();
+    Path notFound = Paths.get("<not-found>");
+
+    // Should not throw.
+    Validate.checkPathExists(tempFile, "tempFile");
+    Validate.checkPathExists(tempDir, "tempDir");
+
+    // Verify it throws.
+
+    intercept(IllegalArgumentException.class, "'nullArg' must not be null",
+        () -> Validate.checkPathExists(null, "nullArg"));
+
+    intercept(IllegalArgumentException.class,
+        "Path notFound (<not-found>) does not exist",
+        () -> Validate.checkPathExists(notFound, "notFound"));
+
+    intercept(IllegalArgumentException.class, "must point to a directory",
+        () -> Validate.checkPathExistsAsDir(tempFile, "tempFile"));
+
+    intercept(IllegalArgumentException.class, "must point to a file",
+        () -> Validate.checkPathExistsAsFile(tempDir, "tempDir"));
+
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
index e7f36fc85013b..757c588104ea1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
@@ -17,23 +17,25 @@
  */
 package org.apache.hadoop.fs.shell;
 
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
-import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
-import org.junit.BeforeClass;
-import org.junit.AfterClass;
-import org.junit.Test;
-import org.junit.Assert;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.concurrent.ThreadPoolExecutor;
 
 import static org.junit.Assert.assertEquals;
 
@@ -48,6 +50,9 @@ public class TestCopyFromLocal {
   private static Path testDir;
   private static Configuration conf;
 
+  private Path dir = null;
+  private int numFiles = 0;
+
   public static int initialize(Path dir) throws Exception {
     fs.mkdirs(dir);
     Path fromDirPath = new Path(dir, FROM_DIR_NAME);
@@ -66,7 +71,7 @@ public static int initialize(Path dir) throws Exception {
         Path subFile = new Path(subDirPath, "file" + fileCount);
         fs.createNewFile(subFile);
         FSDataOutputStream output = fs.create(subFile, true);
-        for(int i = 0; i < 100; ++i) {
+        for (int i = 0; i < 100; ++i) {
           output.writeInt(i);
           output.writeChar('\n');
         }
@@ -96,48 +101,36 @@ public static void cleanup() throws Exception {
     fs.close();
   }
 
+  @Before
+  public void initDirectory() throws Exception {
+    dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
+    numFiles = initialize(dir);
+  }
+
+
   private void run(CommandWithDestination cmd, String... args) {
     cmd.setConf(conf);
     assertEquals(0, cmd.run(args));
   }
 
   @Test(timeout = 10000)
-  public void testCopyFromLocal() throws Exception {
-    Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
-    TestCopyFromLocal.initialize(dir);
+  public void testCopyFromLocal() {
     run(new TestMultiThreadedCopy(1, 0),
         new Path(dir, FROM_DIR_NAME).toString(),
         new Path(dir, TO_DIR_NAME).toString());
   }
 
   @Test(timeout = 10000)
-  public void testCopyFromLocalWithThreads() throws Exception {
-    Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
-    int numFiles = TestCopyFromLocal.initialize(dir);
-    int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
-    int randThreads = RandomUtils.nextInt(0, maxThreads - 1) + 1;
-    String numThreads = Integer.toString(randThreads);
-    run(new TestMultiThreadedCopy(randThreads,
-        randThreads == 1 ? 0 : numFiles), "-t", numThreads,
-        new Path(dir, FROM_DIR_NAME).toString(),
-        new Path(dir, TO_DIR_NAME).toString());
-  }
-
-  @Test(timeout = 10000)
-  public void testCopyFromLocalWithThreadWrong() throws Exception {
-    Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
-    int numFiles = TestCopyFromLocal.initialize(dir);
-    int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
-    String numThreads = Integer.toString(maxThreads * 2);
-    run(new TestMultiThreadedCopy(maxThreads, numFiles), "-t", numThreads,
+  public void testCopyFromLocalWithThreads(){
+    int threads = Runtime.getRuntime().availableProcessors() * 2 + 1;
+    run(new TestMultiThreadedCopy(threads, numFiles),
+        "-t", Integer.toString(threads),
         new Path(dir, FROM_DIR_NAME).toString(),
         new Path(dir, TO_DIR_NAME).toString());
   }
 
   @Test(timeout = 10000)
-  public void testCopyFromLocalWithZeroThreads() throws Exception {
-    Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
-    TestCopyFromLocal.initialize(dir);
+  public void testCopyFromLocalWithThreadWrong(){
     run(new TestMultiThreadedCopy(1, 0), "-t", "0",
         new Path(dir, FROM_DIR_NAME).toString(),
         new Path(dir, TO_DIR_NAME).toString());
@@ -148,8 +141,7 @@ private class TestMultiThreadedCopy extends CopyFromLocal {
     private int expectedThreads;
     private int expectedCompletedTaskCount;
 
-    TestMultiThreadedCopy(int expectedThreads,
-                          int expectedCompletedTaskCount) {
+    TestMultiThreadedCopy(int expectedThreads, int expectedCompletedTaskCount) {
       this.expectedThreads = expectedThreads;
       this.expectedCompletedTaskCount = expectedCompletedTaskCount;
     }
@@ -158,17 +150,22 @@ private class TestMultiThreadedCopy extends CopyFromLocal {
     protected void processArguments(LinkedList<PathData> args)
         throws IOException {
       // Check if the correct number of threads are spawned
-      Assert.assertEquals(expectedThreads, getNumThreads());
+      Assert.assertEquals(expectedThreads, getThreadCount());
       super.processArguments(args);
-      // Once the copy is complete, check following
-      // 1) number of completed tasks are same as expected
-      // 2) There are no active tasks in the executor
-      // 3) Executor has shutdown correctly
-      ThreadPoolExecutor executor = getExecutor();
-      Assert.assertEquals(expectedCompletedTaskCount,
-          executor.getCompletedTaskCount());
-      Assert.assertEquals(0, executor.getActiveCount());
-      Assert.assertTrue(executor.isTerminated());
+
+      if (isMultiThreadNecessary(args)) {
+        // Once the copy is complete, check following
+        // 1) number of completed tasks are same as expected
+        // 2) There are no active tasks in the executor
+        // 3) Executor has shutdown correctly
+        ThreadPoolExecutor executor = getExecutor();
+        Assert.assertEquals(expectedCompletedTaskCount,
+            executor.getCompletedTaskCount());
+        Assert.assertEquals(0, executor.getActiveCount());
+        Assert.assertTrue(executor.isTerminated());
+      } else {
+        assert getExecutor() == null;
+      }
     }
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
index 0f0ddcc4ee498..b68be243c956e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyPreserveFlag.java
@@ -17,11 +17,12 @@
  */
 package org.apache.hadoop.fs.shell;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
 import java.io.IOException;
 
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -31,13 +32,13 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
 import org.apache.hadoop.fs.shell.CopyCommands.Cp;
 import org.apache.hadoop.fs.shell.CopyCommands.Get;
 import org.apache.hadoop.fs.shell.CopyCommands.Put;
-import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
 public class TestCopyPreserveFlag {
   private static final int MODIFICATION_TIME = 12345000;
@@ -176,6 +177,34 @@ public void testGetWithoutP() throws Exception {
     assertAttributesChanged(TO);
   }
 
+  @Test(timeout = 10000)
+  public void testGetWithPQ() throws Exception {
+    Get get = new Get();
+    run(get, "-p", "-q", "100", FROM.toString(), TO.toString());
+    assertEquals(get.getThreadPoolQueueSize(), 100);
+    assertAttributesPreserved(TO);
+  }
+
+  @Test(timeout = 10000)
+  public void testGetWithQ() throws Exception {
+    Get get = new Get();
+    run(get, "-q", "100", FROM.toString(), TO.toString());
+    assertEquals(get.getThreadPoolQueueSize(), 100);
+    assertAttributesChanged(TO);
+  }
+
+  @Test(timeout = 10000)
+  public void testGetWithThreads() throws Exception {
+    run(new Get(), "-t", "10", FROM.toString(), TO.toString());
+    assertAttributesChanged(TO);
+  }
+
+  @Test(timeout = 10000)
+  public void testGetWithThreadsPreserve() throws Exception {
+    run(new Get(), "-p", "-t", "10", FROM.toString(), TO.toString());
+    assertAttributesPreserved(TO);
+  }
+
   @Test(timeout = 10000)
   public void testCpWithP() throws Exception {
       run(new Cp(), "-p", FROM.toString(), TO.toString());
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java
new file mode 100644
index 0000000000000..202b81912c104
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyToLocal.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CopyCommands.CopyToLocal;
+
+import static org.apache.hadoop.fs.shell.CopyCommandWithMultiThread.DEFAULT_QUEUE_SIZE;
+import static org.junit.Assert.assertEquals;
+
+public class TestCopyToLocal {
+
+  private static final String FROM_DIR_NAME = "fromDir";
+  private static final String TO_DIR_NAME = "toDir";
+
+  private static FileSystem fs;
+  private static Path testDir;
+  private static Configuration conf;
+
+  private Path dir = null;
+  private int numFiles = 0;
+
+  private static int initialize(Path dir) throws Exception {
+    fs.mkdirs(dir);
+    Path fromDirPath = new Path(dir, FROM_DIR_NAME);
+    fs.mkdirs(fromDirPath);
+    Path toDirPath = new Path(dir, TO_DIR_NAME);
+    fs.mkdirs(toDirPath);
+
+    int numTotalFiles = 0;
+    int numDirs = RandomUtils.nextInt(0, 5);
+    for (int dirCount = 0; dirCount < numDirs; ++dirCount) {
+      Path subDirPath = new Path(fromDirPath, "subdir" + dirCount);
+      fs.mkdirs(subDirPath);
+      int numFiles = RandomUtils.nextInt(0, 10);
+      for (int fileCount = 0; fileCount < numFiles; ++fileCount) {
+        numTotalFiles++;
+        Path subFile = new Path(subDirPath, "file" + fileCount);
+        fs.createNewFile(subFile);
+        FSDataOutputStream output = fs.create(subFile, true);
+        for (int i = 0; i < 100; ++i) {
+          output.writeInt(i);
+          output.writeChar('\n');
+        }
+        output.close();
+      }
+    }
+
+    return numTotalFiles;
+  }
+
+  @BeforeClass
+  public static void init() throws Exception {
+    conf = new Configuration(false);
+    conf.set("fs.file.impl", LocalFileSystem.class.getName());
+    fs = FileSystem.getLocal(conf);
+    testDir = new FileSystemTestHelper().getTestRootPath(fs);
+    // don't want scheme on the path, just an absolute path
+    testDir = new Path(fs.makeQualified(testDir).toUri().getPath());
+
+    FileSystem.setDefaultUri(conf, fs.getUri());
+    fs.setWorkingDirectory(testDir);
+  }
+
+  @AfterClass
+  public static void cleanup() throws Exception {
+    fs.delete(testDir, true);
+    fs.close();
+  }
+
+  private void run(CopyCommandWithMultiThread cmd, String... args) {
+    cmd.setConf(conf);
+    assertEquals(0, cmd.run(args));
+  }
+
+  @Before
+  public void initDirectory() throws Exception {
+    dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
+    numFiles = initialize(dir);
+  }
+
+  @Test(timeout = 10000)
+  public void testCopy() throws Exception {
+    MultiThreadedCopy copy = new MultiThreadedCopy(1, DEFAULT_QUEUE_SIZE, 0);
+    run(copy, new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+    assert copy.getExecutor() == null;
+  }
+
+  @Test(timeout = 10000)
+  public void testCopyWithThreads() {
+    run(new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5",
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCopyWithThreadWrong() {
+    run(new MultiThreadedCopy(1, DEFAULT_QUEUE_SIZE, 0), "-t", "0",
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCopyWithThreadsAndQueueSize() {
+    int queueSize = 256;
+    run(new MultiThreadedCopy(5, queueSize, numFiles), "-t", "5", "-q",
+        Integer.toString(queueSize),
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCopyWithThreadsAndQueueSizeWrong() {
+    int queueSize = 0;
+    run(new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", "-q",
+        Integer.toString(queueSize),
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCopySingleFile() throws Exception {
+    Path fromDirPath = new Path(dir, FROM_DIR_NAME);
+    Path subFile = new Path(fromDirPath, "file0");
+    fs.createNewFile(subFile);
+    FSDataOutputStream output = fs.create(subFile, true);
+    for (int i = 0; i < 100; ++i) {
+      output.writeInt(i);
+      output.writeChar('\n');
+    }
+    output.close();
+
+    MultiThreadedCopy copy = new MultiThreadedCopy(5, DEFAULT_QUEUE_SIZE, 0);
+    run(copy, "-t", "5", subFile.toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+    assert copy.getExecutor() == null;
+  }
+
+  private static class MultiThreadedCopy extends CopyToLocal {
+    public static final String NAME = "multiThreadCopy";
+    private final int expectedThreads;
+    private final int expectedQueuePoolSize;
+    private final int expectedCompletedTaskCount;
+
+    MultiThreadedCopy(int expectedThreads, int expectedQueuePoolSize,
+        int expectedCompletedTaskCount) {
+      this.expectedThreads = expectedThreads;
+      this.expectedQueuePoolSize = expectedQueuePoolSize;
+      this.expectedCompletedTaskCount = expectedCompletedTaskCount;
+    }
+
+    @Override
+    protected void processArguments(LinkedList<PathData> args)
+        throws IOException {
+      // Check if the number of threads are same as expected
+      Assert.assertEquals(expectedThreads, getThreadCount());
+      // Check if the queue pool size of executor is same as expected
+      Assert.assertEquals(expectedQueuePoolSize, getThreadPoolQueueSize());
+
+      super.processArguments(args);
+
+      if (isMultiThreadNecessary(args)) {
+        // Once the copy is complete, check following
+        // 1) number of completed tasks are same as expected
+        // 2) There are no active tasks in the executor
+        // 3) Executor has shutdown correctly
+        ThreadPoolExecutor executor = getExecutor();
+        Assert.assertEquals(expectedCompletedTaskCount,
+            executor.getCompletedTaskCount());
+        Assert.assertEquals(0, executor.getActiveCount());
+        Assert.assertTrue(executor.isTerminated());
+      } else {
+        assert getExecutor() == null;
+      }
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java
new file mode 100644
index 0000000000000..214f1a0686cd9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCpCommand.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CopyCommands.Cp;
+
+import static org.apache.hadoop.fs.shell.CopyCommandWithMultiThread.DEFAULT_QUEUE_SIZE;
+import static org.junit.Assert.assertEquals;
+
+public class TestCpCommand {
+
+  private static final String FROM_DIR_NAME = "fromDir";
+  private static final String TO_DIR_NAME = "toDir";
+
+  private static FileSystem fs;
+  private static Path testDir;
+  private static Configuration conf;
+
+  private Path dir = null;
+  private int numFiles = 0;
+
+  private static int initialize(Path dir) throws Exception {
+    fs.mkdirs(dir);
+    Path fromDirPath = new Path(dir, FROM_DIR_NAME);
+    fs.mkdirs(fromDirPath);
+    Path toDirPath = new Path(dir, TO_DIR_NAME);
+    fs.mkdirs(toDirPath);
+
+    int numTotalFiles = 0;
+    int numDirs = RandomUtils.nextInt(0, 5);
+    for (int dirCount = 0; dirCount < numDirs; ++dirCount) {
+      Path subDirPath = new Path(fromDirPath, "subdir" + dirCount);
+      fs.mkdirs(subDirPath);
+      int numFiles = RandomUtils.nextInt(0, 10);
+      for (int fileCount = 0; fileCount < numFiles; ++fileCount) {
+        numTotalFiles++;
+        Path subFile = new Path(subDirPath, "file" + fileCount);
+        fs.createNewFile(subFile);
+        FSDataOutputStream output = fs.create(subFile, true);
+        for (int i = 0; i < 100; ++i) {
+          output.writeInt(i);
+          output.writeChar('\n');
+        }
+        output.close();
+      }
+    }
+
+    return numTotalFiles;
+  }
+
+  @BeforeClass
+  public static void init() throws Exception {
+    conf = new Configuration(false);
+    conf.set("fs.file.impl", LocalFileSystem.class.getName());
+    fs = FileSystem.getLocal(conf);
+    testDir = new FileSystemTestHelper().getTestRootPath(fs);
+    // don't want scheme on the path, just an absolute path
+    testDir = new Path(fs.makeQualified(testDir).toUri().getPath());
+
+    FileSystem.setDefaultUri(conf, fs.getUri());
+    fs.setWorkingDirectory(testDir);
+  }
+
+  @AfterClass
+  public static void cleanup() throws Exception {
+    fs.delete(testDir, true);
+    fs.close();
+  }
+
+  private void run(CopyCommandWithMultiThread cmd, String... args) {
+    cmd.setConf(conf);
+    assertEquals(0, cmd.run(args));
+  }
+
+  @Before
+  public void initDirectory() throws Exception {
+    dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
+    numFiles = initialize(dir);
+  }
+
+  @Test(timeout = 10000)
+  public void testCp() throws Exception {
+    MultiThreadedCp copy = new MultiThreadedCp(1, DEFAULT_QUEUE_SIZE, 0);
+    run(copy, new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+    assert copy.getExecutor() == null;
+  }
+
+  @Test(timeout = 10000)
+  public void testCpWithThreads() {
+    run(new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5",
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCpWithThreadWrong() {
+    run(new MultiThreadedCp(1, DEFAULT_QUEUE_SIZE, 0), "-t", "0",
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCpWithThreadsAndQueueSize() {
+    int queueSize = 256;
+    run(new MultiThreadedCp(5, queueSize, numFiles), "-t", "5", "-q",
+        Integer.toString(queueSize),
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCpWithThreadsAndQueueSizeWrong() {
+    int queueSize = 0;
+    run(new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, numFiles), "-t", "5", "-q",
+        Integer.toString(queueSize),
+        new Path(dir, FROM_DIR_NAME).toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+  }
+
+  @Test(timeout = 10000)
+  public void testCpSingleFile() throws Exception {
+    Path fromDirPath = new Path(dir, FROM_DIR_NAME);
+    Path subFile = new Path(fromDirPath, "file0");
+    fs.createNewFile(subFile);
+    FSDataOutputStream output = fs.create(subFile, true);
+    for (int i = 0; i < 100; ++i) {
+      output.writeInt(i);
+      output.writeChar('\n');
+    }
+    output.close();
+
+    MultiThreadedCp copy = new MultiThreadedCp(5, DEFAULT_QUEUE_SIZE, 0);
+    run(copy, "-t", "5", subFile.toString(),
+        new Path(dir, TO_DIR_NAME).toString());
+    assert copy.getExecutor() == null;
+  }
+
+  private static class MultiThreadedCp extends Cp {
+    public static final String NAME = "multiThreadCp";
+    private final int expectedThreads;
+    private final int expectedQueuePoolSize;
+    private final int expectedCompletedTaskCount;
+
+    MultiThreadedCp(int expectedThreads, int expectedQueuePoolSize,
+        int expectedCompletedTaskCount) {
+      this.expectedThreads = expectedThreads;
+      this.expectedQueuePoolSize = expectedQueuePoolSize;
+      this.expectedCompletedTaskCount = expectedCompletedTaskCount;
+    }
+
+    @Override
+    protected void processArguments(LinkedList<PathData> args)
+        throws IOException {
+      // Check if the number of threads are same as expected
+      Assert.assertEquals(expectedThreads, getThreadCount());
+      // Check if the queue pool size of executor is same as expected
+      Assert.assertEquals(expectedQueuePoolSize, getThreadPoolQueueSize());
+
+      super.processArguments(args);
+
+      if (isMultiThreadNecessary(args)) {
+        // Once the copy is complete, check following
+        // 1) number of completed tasks are same as expected
+        // 2) There are no active tasks in the executor
+        // 3) Executor has shutdown correctly
+        ThreadPoolExecutor executor = getExecutor();
+        Assert.assertEquals(expectedCompletedTaskCount,
+            executor.getCompletedTaskCount());
+        Assert.assertEquals(0, executor.getActiveCount());
+        Assert.assertTrue(executor.isTerminated());
+      } else {
+        assert getExecutor() == null;
+      }
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
index 921fc18131a5c..130ee5edee768 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
@@ -24,7 +24,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URI;
 import java.util.Arrays;
 
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
index c99b97e6e4021..e806dde11b012 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
@@ -25,15 +25,14 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
-import java.lang.reflect.Method;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.nio.file.Paths;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
@@ -49,22 +48,19 @@ public class TestTextCommand {
   private static final String TEXT_FILENAME =
     new File(TEST_ROOT_DIR, "testtextfile.txt").toURI().getPath();
 
+  private static final String SEPARATOR = System.getProperty("line.separator");
+
   /**
    * Tests whether binary Avro data files are displayed correctly.
    */
   @Test (timeout = 30000)
   public void testDisplayForAvroFiles() throws Exception {
     String expectedOutput =
-      "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" +
-      System.getProperty("line.separator") +
-      "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" +
-      System.getProperty("line.separator") +
-      "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" +
-      System.getProperty("line.separator") +
-      "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" +
-      System.getProperty("line.separator") +
-      "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" +
-      System.getProperty("line.separator");
+        "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + SEPARATOR
+        + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" + SEPARATOR
+        + "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" + SEPARATOR
+        + "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" + SEPARATOR
+        + "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" + SEPARATOR;
 
     String output = readUsingTextCommand(AVRO_FILENAME,
                                          generateWeatherAvroBinaryData());
@@ -107,11 +103,16 @@ private String readUsingTextCommand(String fileName, byte[] fileContents)
           throws Exception {
     createFile(fileName, fileContents);
 
-    // Prepare and call the Text command's protected getInputStream method
-    // using reflection.
     Configuration conf = new Configuration();
     URI localPath = new URI(fileName);
-    PathData pathData = new PathData(localPath, conf);
+    return readUsingTextCommand(localPath, conf);
+  }
+  // Read a file using Display.Text class.
+  private String readUsingTextCommand(URI uri, Configuration conf)
+      throws Exception {
+    // Prepare and call the Text command's protected getInputStream method
+    // using reflection.
+    PathData pathData = new PathData(uri, conf);
     Display.Text text = new Display.Text() {
       @Override
       public InputStream getInputStream(PathData item) throws IOException {
@@ -119,7 +120,7 @@ public InputStream getInputStream(PathData item) throws IOException {
       }
     };
     text.setConf(conf);
-    InputStream stream = (InputStream) text.getInputStream(pathData);
+    InputStream stream = text.getInputStream(pathData);
     return inputStreamToString(stream);
   }
 
@@ -235,5 +236,21 @@ private byte[] generateWeatherAvroBinaryData() {
 
     return contents;
   }
+
+  @Test
+  public void testDisplayForNonWritableSequenceFile() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
+    Path path = new Path(String.valueOf(TEST_ROOT_DIR), "NonWritableSequenceFile");
+    SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(path),
+        SequenceFile.Writer.keyClass(String.class), SequenceFile.Writer.valueClass(String.class));
+    writer.append("Key1", "Value1");
+    writer.append("Key2", "Value2");
+    writer.close();
+    String expected = "Key1\tValue1" + SEPARATOR + "Key2\tValue2" + SEPARATOR;
+    URI uri = path.toUri();
+    System.out.println(expected);
+    assertEquals(expected, readUsingTextCommand(uri, conf));
+  }
 }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java
index 22f6c33d2e260..755599f0c390c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/IOStatisticAssertions.java
@@ -36,6 +36,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MAX;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MIN;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -347,6 +349,24 @@ public static AbstractLongAssert<?> assertThatStatisticMaximum(
         verifyStatisticsNotNull(stats).maximums());
   }
 
+  /**
+   * Assert that a duration is within a given minimum/maximum range.
+   * @param stats statistics source
+   * @param key statistic key without any suffix
+   * @param min minimum statistic must be equal to or greater than this.
+   * @param max maximum statistic must be equal to or less than this.
+   */
+  public static void assertDurationRange(
+      final IOStatistics stats,
+      final String key,
+      final long min,
+      final long max) {
+    assertThatStatisticMinimum(stats, key + SUFFIX_MIN)
+        .isGreaterThanOrEqualTo(min);
+    assertThatStatisticMaximum(stats, key + SUFFIX_MAX)
+        .isLessThanOrEqualTo(max);
+  }
+
   /**
    * Start an assertion chain on
    * a required mean statistic.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java
index 8258b62c1f759..cfde1583e2c21 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/statistics/TestDurationTracking.java
@@ -30,7 +30,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 import org.apache.hadoop.test.AbstractHadoopTestBase;
 import org.apache.hadoop.util.functional.FunctionRaisingIOE;
@@ -276,7 +275,7 @@ public void testCallableIOEFailureDuration() throws Throwable {
    */
   @Test
   public void testDurationThroughEval() throws Throwable {
-    CompletableFuture<Object> eval = FutureIOSupport.eval(
+    CompletableFuture<Object> eval = FutureIO.eval(
         trackDurationOfOperation(stats, REQUESTS, () -> {
           sleepf(100);
           throw new FileNotFoundException("oops");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java
new file mode 100644
index 0000000000000..4a7aafd0a2b25
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestNestedMountPoint.java
@@ -0,0 +1,365 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.net.URI;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Unit test of nested mount point support in INodeTree
+ */
+public class TestNestedMountPoint {
+  private InodeTree inodeTree;
+  private Configuration conf;
+  private String mtName;
+  private URI fsUri;
+
+  static class TestNestMountPointFileSystem {
+    public URI getUri() {
+      return uri;
+    }
+
+    private URI uri;
+
+    TestNestMountPointFileSystem(URI uri) {
+      this.uri = uri;
+    }
+  }
+
+  static class TestNestMountPointInternalFileSystem extends TestNestMountPointFileSystem {
+    TestNestMountPointInternalFileSystem(URI uri) {
+      super(uri);
+    }
+  }
+
+  private static final URI LINKFALLBACK_TARGET = URI.create("hdfs://nn00");
+  private static final URI NN1_TARGET = URI.create("hdfs://nn01/a/b");
+  private static final URI NN2_TARGET = URI.create("hdfs://nn02/a/b/e");
+  private static final URI NN3_TARGET = URI.create("hdfs://nn03/a/b/c/d");
+  private static final URI NN4_TARGET = URI.create("hdfs://nn04/a/b/c/d/e");
+  private static final URI NN5_TARGET = URI.create("hdfs://nn05/b/c/d/e");
+  private static final URI NN6_TARGET = URI.create("hdfs://nn06/b/c/d/e/f");
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    mtName = TestNestedMountPoint.class.getName();
+    ConfigUtil.setIsNestedMountPointSupported(conf, true);
+    ConfigUtil.addLink(conf, mtName, "/a/b", NN1_TARGET);
+    ConfigUtil.addLink(conf, mtName, "/a/b/e", NN2_TARGET);
+    ConfigUtil.addLink(conf, mtName, "/a/b/c/d", NN3_TARGET);
+    ConfigUtil.addLink(conf, mtName, "/a/b/c/d/e", NN4_TARGET);
+    ConfigUtil.addLink(conf, mtName, "/b/c/d/e", NN5_TARGET);
+    ConfigUtil.addLink(conf, mtName, "/b/c/d/e/f", NN6_TARGET);
+    ConfigUtil.addLinkFallback(conf, mtName, LINKFALLBACK_TARGET);
+
+    fsUri = new URI(FsConstants.VIEWFS_SCHEME, mtName, "/", null, null);
+
+    inodeTree = new InodeTree<TestNestedMountPoint.TestNestMountPointFileSystem>(conf,
+        mtName, fsUri, false) {
+      @Override
+      protected Function<URI, TestNestedMountPoint.TestNestMountPointFileSystem> initAndGetTargetFs() {
+        return new Function<URI, TestNestMountPointFileSystem>() {
+          @Override
+          public TestNestedMountPoint.TestNestMountPointFileSystem apply(URI uri) {
+            return new TestNestMountPointFileSystem(uri);
+          }
+        };
+      }
+
+      // For intenral dir fs
+      @Override
+      protected TestNestedMountPoint.TestNestMountPointInternalFileSystem getTargetFileSystem(
+          final INodeDir<TestNestedMountPoint.TestNestMountPointFileSystem> dir) {
+        return new TestNestMountPointInternalFileSystem(fsUri);
+      }
+
+      @Override
+      protected TestNestedMountPoint.TestNestMountPointInternalFileSystem getTargetFileSystem(
+          final String settings, final URI[] mergeFsURIList) {
+        return new TestNestMountPointInternalFileSystem(null);
+      }
+    };
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    inodeTree = null;
+  }
+
+  @Test
+  public void testPathResolveToLink() throws Exception {
+    // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b/c/d/e resolves to /a/b/c/d/e and /
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a/b/c/d/e", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+    // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+    InodeTree.ResolveResult resolveResult3 = inodeTree.resolve("/a/b/c/d/e/f/g/h/i", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult3.kind);
+    Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+    Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+    Assert.assertTrue(resolveResult3.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem) resolveResult3.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToLinkNotResolveLastComponent() throws Exception {
+    // /a/b/c/d/e/f resolves to /a/b/c/d/e and /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/e/f", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b/c/d/e", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b/c/d/e resolves to /a/b/c/d and /e
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d/e", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/e"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+    // /a/b/c/d/e/f/g/h/i resolves to /a/b/c/d/e and /f/g/h/i
+    InodeTree.ResolveResult resolveResult3 = inodeTree.resolve("/a/b/c/d/e/f/g/h/i", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult3.kind);
+    Assert.assertEquals("/a/b/c/d/e", resolveResult3.resolvedPath);
+    Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+    Assert.assertTrue(resolveResult3.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN4_TARGET, ((TestNestMountPointFileSystem) resolveResult3.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToDirLink() throws Exception {
+    // /a/b/c/d/f resolves to /a/b/c/d, /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b/c/d resolves to /a/b/c/d and /
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+    // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+    InodeTree.ResolveResult resolveResult3 = inodeTree.resolve("/a/b/c/d/f/g/h/i", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult3.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+    Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+    Assert.assertTrue(resolveResult3.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult3.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToDirLinkNotResolveLastComponent() throws Exception {
+    // /a/b/c/d/f resolves to /a/b/c/d, /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c/d/f", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b/c/d resolves to /a/b and /c/d
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b/c/d", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a/b", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/c/d"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+
+    // /a/b/c/d/f/g/h/i resolves to /a/b/c/d and /f/g/h/i
+    InodeTree.ResolveResult resolveResult3 = inodeTree.resolve("/a/b/c/d/f/g/h/i", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult3.kind);
+    Assert.assertEquals("/a/b/c/d", resolveResult3.resolvedPath);
+    Assert.assertEquals(new Path("/f/g/h/i"), resolveResult3.remainingPath);
+    Assert.assertTrue(resolveResult3.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN3_TARGET, ((TestNestMountPointFileSystem) resolveResult3.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult3.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testMultiNestedMountPointsPathResolveToDirLink() throws Exception {
+    // /a/b/f resolves to /a/b and /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/f", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b resolves to /a/b and /
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a/b", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult2.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testMultiNestedMountPointsPathResolveToDirLinkNotResolveLastComponent() throws Exception {
+    // /a/b/f resolves to /a/b and /f
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/f", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/f"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+
+    // /a/b resolves to /a and /b
+    InodeTree.ResolveResult resolveResult2 = inodeTree.resolve("/a/b", false);
+    Assert.assertEquals(InodeTree.ResultKind.INTERNAL_DIR, resolveResult2.kind);
+    Assert.assertEquals("/a", resolveResult2.resolvedPath);
+    Assert.assertEquals(new Path("/b"), resolveResult2.remainingPath);
+    Assert.assertTrue(resolveResult2.targetFileSystem instanceof TestNestMountPointInternalFileSystem);
+    Assert.assertEquals(fsUri, ((TestNestMountPointInternalFileSystem) resolveResult2.targetFileSystem).getUri());
+    Assert.assertFalse(resolveResult2.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToDirLinkLastComponentInternalDir() throws Exception {
+    // /a/b/c resolves to /a/b and /c
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/c"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToDirLinkLastComponentInternalDirNotResolveLastComponent() throws Exception {
+    // /a/b/c resolves to /a/b and /c
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/b/c", false);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a/b", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/c"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(NN1_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertTrue(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToLinkFallBack() throws Exception {
+    // /a/e resolves to linkfallback
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/e", true);
+    Assert.assertEquals(InodeTree.ResultKind.EXTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/a/e"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointFileSystem);
+    Assert.assertEquals(LINKFALLBACK_TARGET, ((TestNestMountPointFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertFalse(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathNotResolveToLinkFallBackNotResolveLastComponent() throws Exception {
+    // /a/e resolves to internalDir instead of linkfallback
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/a/e", false);
+    Assert.assertEquals(InodeTree.ResultKind.INTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/a", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/e"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointInternalFileSystem);
+    Assert.assertEquals(fsUri, ((TestNestMountPointInternalFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertFalse(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToInternalDir() throws Exception {
+    // /b/c resolves to internal dir
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/b/c", true);
+    Assert.assertEquals(InodeTree.ResultKind.INTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/b/c", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointInternalFileSystem);
+    Assert.assertEquals(fsUri, ((TestNestMountPointInternalFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertFalse(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testPathResolveToInternalDirNotResolveLastComponent() throws Exception {
+    // /b/c resolves to internal dir
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/b/c", false);
+    Assert.assertEquals(InodeTree.ResultKind.INTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/b", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/c"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointInternalFileSystem);
+    Assert.assertEquals(fsUri, ((TestNestMountPointInternalFileSystem) resolveResult.targetFileSystem).getUri());
+    Assert.assertFalse(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testSlashResolveToInternalDir() throws Exception {
+    // / resolves to internal dir
+    InodeTree.ResolveResult resolveResult = inodeTree.resolve("/", true);
+    Assert.assertEquals(InodeTree.ResultKind.INTERNAL_DIR, resolveResult.kind);
+    Assert.assertEquals("/", resolveResult.resolvedPath);
+    Assert.assertEquals(new Path("/"), resolveResult.remainingPath);
+    Assert.assertTrue(resolveResult.targetFileSystem instanceof TestNestMountPointInternalFileSystem);
+    Assert.assertFalse(resolveResult.isLastInternalDirLink());
+  }
+
+  @Test
+  public void testInodeTreeMountPoints() throws Exception {
+    List<InodeTree.MountPoint<FileSystem>> mountPoints = inodeTree.getMountPoints();
+    Assert.assertEquals(6, mountPoints.size());
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
index d8c39f79d0454..3a60d6ecdda94 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java
@@ -83,12 +83,6 @@ public void testSanity() throws URISyntaxException {
     assertEquals(new URI("fs2:/").getAuthority(), fs2.getUri().getAuthority());
   }
   
-  @Test
-  public void testVerifyChecksum() throws Exception {
-    checkVerifyChecksum(false);
-    checkVerifyChecksum(true);
-  }
-
   /**
    * Tests that ViewFileSystem dispatches calls for every ACL method through the
    * mount table to the correct underlying FileSystem with all Path arguments
@@ -144,12 +138,6 @@ public void testAclMethods() throws Exception {
     verify(mockFs2).getAclStatus(mockFsPath2);
   }
 
-  void checkVerifyChecksum(boolean flag) {
-    viewFs.setVerifyChecksum(flag);
-    assertEquals(flag, fs1.getVerifyChecksum());
-    assertEquals(flag, fs2.getVerifyChecksum());
-  }
-
   static class FakeFileSystem extends LocalFileSystem {
     boolean verifyChecksum = true;
     URI uri;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java
index 808d8b06c35ba..adc5db87e7725 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLocalFileSystem.java
@@ -33,6 +33,8 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+import org.apache.hadoop.security.UserGroupInformation;
 
 import org.junit.After;
 import org.junit.Before;
@@ -61,6 +63,13 @@ public void setUp() throws Exception {
     
   }
 
+  @Override
+  Path getTrashRootInFallBackFS() throws IOException {
+    return new Path(
+        "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser()
+            .getShortUserName());
+  }
+
   @Test
   public void testNflyWriteSimple() throws IOException {
     LOG.info("Starting testNflyWriteSimple");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java
index 877c2228c1eea..f2452279bc7fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithAuthorityLocalFileSystem.java
@@ -22,9 +22,11 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.FsConstants;
 import org.apache.hadoop.fs.Path;
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
+import org.apache.hadoop.security.UserGroupInformation;
+import java.io.IOException;
 
 import org.junit.After;
 import org.junit.Assert;
@@ -63,6 +65,13 @@ public void tearDown() throws Exception {
     super.tearDown();
   }
  
+  @Override
+  Path getTrashRootInFallBackFS() throws IOException {
+    return new Path(
+        "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser()
+            .getShortUserName());
+  }
+
   @Override
   @Test
   public void testBasicPaths() {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java
index 7c318654ecf1c..9d7c58f8197b3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsConfig.java
@@ -32,6 +32,7 @@ public class TestViewFsConfig {
   @Test(expected = FileAlreadyExistsException.class)
   public void testInvalidConfig() throws IOException, URISyntaxException {
     Configuration conf = new Configuration();
+    ConfigUtil.setIsNestedMountPointSupported(conf, false);
     ConfigUtil.addLink(conf, "/internalDir/linkToDir2",
         new Path("file:///dir2").toUri());
     ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir3",
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java
index 8e5fa72f7ed71..06cbdab8d210f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsTrash.java
@@ -17,15 +17,24 @@
  */
 package org.apache.hadoop.fs.viewfs;
 
+import java.io.DataOutputStream;
+import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.TestTrash;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.fs.TrashPolicyDefault;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+import static org.apache.hadoop.fs.viewfs.Constants.*;
+import static org.junit.Assert.*;
 
 public class TestViewFsTrash {
   FileSystem fsTarget;  // the target file system - the mount will point here
@@ -65,5 +74,37 @@ public void testTrash() throws Exception {
     TestTrash.trashShell(conf, fileSystemTestHelper.getTestRootPath(fsView),
         fsView, new Path(fileSystemTestHelper.getTestRootPath(fsView), ".Trash/Current"));
   }
-  
+
+  @Test
+  public void testLocalizedTrashInMoveToAppropriateTrash() throws IOException {
+    Configuration conf2 = new Configuration(conf);
+    Path testFile = new Path("/data/testfile.txt");
+
+    // Enable moveToTrash and add a mount point for /data
+    conf2.setLong(FS_TRASH_INTERVAL_KEY, 1);
+    ConfigUtil.addLink(conf2, "/data", new Path(fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget), "data").toUri());
+
+    // Default case. file should be moved to fsTarget.getTrashRoot()/resolvedPath
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, false);
+    try (FileSystem fsView2 = FileSystem.get(conf2)) {
+      FileSystemTestHelper.createFile(fsView2, testFile);
+      Path resolvedFile = fsView2.resolvePath(testFile);
+
+      Trash.moveToAppropriateTrash(fsView2, testFile, conf2);
+      Trash trash = new Trash(fsTarget, conf2);
+      Path movedPath = Path.mergePaths(trash.getCurrentTrashDir(testFile), resolvedFile);
+      ContractTestUtils.assertPathExists(fsTarget, "File not in trash", movedPath);
+    }
+
+    // Turn on localized trash. File should be moved to viewfs:/data/.Trash/{user}/Current.
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true);
+    try (FileSystem fsView2 = FileSystem.get(conf2)) {
+      FileSystemTestHelper.createFile(fsView2, testFile);
+
+      Trash.moveToAppropriateTrash(fsView2, testFile, conf2);
+      Trash trash = new Trash(fsView2, conf2);
+      Path movedPath = Path.mergePaths(trash.getCurrentTrashDir(testFile), testFile);
+      ContractTestUtils.assertPathExists(fsView2, "File not in localized trash", movedPath);
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
index be50f457be21c..0f3c8aacab7cd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
@@ -69,6 +69,8 @@
 import static org.apache.hadoop.fs.FileSystemTestHelper.*;
 import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
 import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
+import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT;
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
 
 import org.junit.After;
 import org.junit.Assert;
@@ -458,6 +460,148 @@ public void testRenameAcrossMounts4() throws IOException {
         .assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar"));
   }
 
+
+  // rename across nested mount points that point to same target also fail
+  @Test
+  public void testRenameAcrossNestedMountPointSameTarget() throws IOException {
+    setUpNestedMountPoint();
+    fileSystemTestHelper.createFile(fsView, "/user/foo");
+    try {
+      // Nested mount points point to the same target should fail
+      // /user -> /user
+      // /user/userA -> /user
+      // Rename strategy: SAME_MOUNTPOINT
+      fsView.rename(new Path("/user/foo"), new Path("/user/userA/foo"));
+      ContractTestUtils.fail("IOException is not thrown on rename operation");
+    } catch (IOException e) {
+      GenericTestUtils
+          .assertExceptionContains("Renames across Mount points not supported",
+              e);
+    }
+  }
+
+
+  // rename across nested mount points fail if the mount link targets are different
+  // even if the targets are part of the same target FS
+  @Test
+  public void testRenameAcrossMountPointDifferentTarget() throws IOException {
+    setUpNestedMountPoint();
+    fileSystemTestHelper.createFile(fsView, "/data/foo");
+    // /data -> /data
+    // /data/dataA -> /dataA
+    // Rename strategy: SAME_MOUNTPOINT
+    try {
+      fsView.rename(new Path("/data/foo"), new Path("/data/dataA/fooBar"));
+      ContractTestUtils.fail("IOException is not thrown on rename operation");
+    } catch (IOException e) {
+      GenericTestUtils
+          .assertExceptionContains("Renames across Mount points not supported",
+              e);
+    }
+  }
+
+  // RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
+  // to rename across nested mount points that point to same target URI
+  @Test
+  public void testRenameAcrossNestedMountPointSameTargetUriAcrossMountPoint() throws IOException {
+    setUpNestedMountPoint();
+    //  /user/foo -> /user
+    // /user/userA/fooBarBar -> /user
+    // Rename strategy: SAME_TARGET_URI_ACROSS_MOUNTPOINT
+    Configuration conf2 = new Configuration(conf);
+    conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+        ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
+            .toString());
+    FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+    fileSystemTestHelper.createFile(fsView2, "/user/foo");
+    fsView2.rename(new Path("/user/foo"), new Path("/user/userA/fooBarBar"));
+    ContractTestUtils.assertPathDoesNotExist(fsView2, "src should not exist after rename",
+        new Path("/user/foo"));
+    ContractTestUtils.assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+        new Path(targetTestRoot, "user/foo"));
+    ContractTestUtils.assertIsFile(fsView2, fileSystemTestHelper.getTestRootPath(fsView2, "/user/userA/fooBarBar"));
+    ContractTestUtils.assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar"));
+  }
+
+  // RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
+  // to rename across mount points where the mount link targets are different
+  // but are part of the same target FS
+  @Test
+  public void testRenameAcrossNestedMountPointSameFileSystemAcrossMountPoint() throws IOException {
+    setUpNestedMountPoint();
+    // /data/foo -> /data
+    // /data/dataA/fooBar -> /dataA
+    // Rename strategy: SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+    Configuration conf2 = new Configuration(conf);
+    conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
+        ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
+            .toString());
+    FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
+    fileSystemTestHelper.createFile(fsView2, "/data/foo");
+    fsView2.rename(new Path("/data/foo"), new Path("/data/dataB/fooBar"));
+    ContractTestUtils
+        .assertPathDoesNotExist(fsView2, "src should not exist after rename",
+            new Path("/data/foo"));
+    ContractTestUtils
+        .assertPathDoesNotExist(fsTarget, "src should not exist after rename",
+            new Path(targetTestRoot, "data/foo"));
+    ContractTestUtils.assertIsFile(fsView2,
+        fileSystemTestHelper.getTestRootPath(fsView2, "/user/fooBar"));
+    ContractTestUtils
+        .assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBar"));
+  }
+
+  @Test
+  public void testOperationsThroughNestedMountPointsInternal()
+      throws IOException {
+    setUpNestedMountPoint();
+    // Create file with nested mount point
+    fileSystemTestHelper.createFile(fsView, "/user/userB/foo");
+    Assert.assertTrue("Created file should be type file",
+        fsView.getFileStatus(new Path("/user/userB/foo")).isFile());
+    Assert.assertTrue("Target of created file should be type file",
+        fsTarget.getFileStatus(new Path(targetTestRoot,"userB/foo")).isFile());
+
+    // Delete the created file with nested mount point
+    Assert.assertTrue("Delete should succeed",
+        fsView.delete(new Path("/user/userB/foo"), false));
+    Assert.assertFalse("File should not exist after delete",
+        fsView.exists(new Path("/user/userB/foo")));
+    Assert.assertFalse("Target File should not exist after delete",
+        fsTarget.exists(new Path(targetTestRoot,"userB/foo")));
+
+    // Create file with a 2 component dirs with nested mount point
+    fileSystemTestHelper.createFile(fsView, "/internalDir/linkToDir2/linkToDir2/foo");
+    Assert.assertTrue("Created file should be type file",
+        fsView.getFileStatus(new Path("/internalDir/linkToDir2/linkToDir2/foo")).isFile());
+    Assert.assertTrue("Target of created file should be type file",
+        fsTarget.getFileStatus(new Path(targetTestRoot,"linkToDir2/foo")).isFile());
+
+    // Delete the created file with nested mount point
+    Assert.assertTrue("Delete should succeed",
+        fsView.delete(new Path("/internalDir/linkToDir2/linkToDir2/foo"), false));
+    Assert.assertFalse("File should not exist after delete",
+        fsView.exists(new Path("/internalDir/linkToDir2/linkToDir2/foo")));
+    Assert.assertFalse("Target File should not exist after delete",
+        fsTarget.exists(new Path(targetTestRoot,"linkToDir2/foo")));
+  }
+
+  private void setUpNestedMountPoint() throws IOException {
+    // Enable nested mount point, ViewFilesystem should support both non-nested and nested mount points
+    ConfigUtil.setIsNestedMountPointSupported(conf, true);
+    ConfigUtil.addLink(conf, "/user/userA",
+        new Path(targetTestRoot, "user").toUri());
+    ConfigUtil.addLink(conf, "/user/userB",
+        new Path(targetTestRoot, "userB").toUri());
+    ConfigUtil.addLink(conf, "/data/dataA",
+        new Path(targetTestRoot, "dataA").toUri());
+    ConfigUtil.addLink(conf, "/data/dataB",
+        new Path(targetTestRoot, "user").toUri());
+    ConfigUtil.addLink(conf, "/internalDir/linkToDir2/linkToDir2",
+        new Path(targetTestRoot,"linkToDir2").toUri());
+    fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
+  }
+
   static protected boolean SupportsBlocks = false; //  local fs use 1 block
                                                    // override for HDFS
   @Test
@@ -1102,6 +1246,176 @@ public void testTrashRoot() throws IOException {
     Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0);
   }
 
+  // Default implementation of getTrashRoot for a fallback FS mounted at root:
+  // e.g., fallbackFS.uri.getPath = '/'
+  Path getTrashRootInFallBackFS() throws IOException {
+    return new Path(fsTarget.getHomeDirectory().toUri().getPath(),
+        TRASH_PREFIX);
+  }
+
+  /**
+   * Test TRASH_FORCE_INSIDE_MOUNT_POINT feature for getTrashRoot.
+   */
+  @Test
+  public void testTrashRootForceInsideMountPoint() throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    Configuration conf2 = new Configuration(conf);
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true);
+    ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri());
+    FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+
+    // Case 1: path p in the /data mount point.
+    // Return a trash root within the /data mount point.
+    Path dataTestPath = new Path("/data/dir/file");
+    Path dataTrashRoot = fsView2.makeQualified(
+        new Path("/data/" + TRASH_PREFIX + "/" + ugi.getShortUserName()));
+    Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath));
+
+    // Case 2: path p not found in mount table.
+    // Return a trash root in fallback FS.
+    Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile");
+    Path expectedTrash =
+        fsView2.makeQualified(getTrashRootInFallBackFS());
+    Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(nonExistentPath));
+
+    // Case 3: turn off the CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT flag.
+    // Return a trash root in user home dir.
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, false);
+    fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+    Path targetFSUserHomeTrashRoot = fsTarget.makeQualified(
+        new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX));
+    Assert.assertEquals(targetFSUserHomeTrashRoot,
+        fsView2.getTrashRoot(dataTestPath));
+
+    // Case 4: viewFS without fallback. Expect exception for a nonExistent path
+    conf2 = new Configuration(conf);
+    fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+    try {
+      fsView2.getTrashRoot(nonExistentPath);
+    } catch (NotInMountpointException ignored) {
+    }
+  }
+
+  /**
+   * A mocked FileSystem which returns a deep trash dir.
+   */
+  static class DeepTrashRootMockFS extends MockFileSystem {
+    public static final Path TRASH =
+        new Path("/vol/very/deep/deep/trash/dir/.Trash");
+
+    @Override
+    public Path getTrashRoot(Path path) {
+      return TRASH;
+    }
+  }
+
+  /**
+   * Test getTrashRoot that is very deep inside a mount point.
+   */
+  @Test
+  public void testTrashRootDeepTrashDir() throws IOException {
+
+    Configuration conf2 = ViewFileSystemTestSetup.createConfig();
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true);
+    conf2.setClass("fs.mocktrashfs.impl", DeepTrashRootMockFS.class,
+        FileSystem.class);
+    ConfigUtil.addLink(conf2, "/mnt/datavol1",
+        URI.create("mocktrashfs://localhost/vol"));
+    Path testPath = new Path("/mnt/datavol1/projs/proj");
+    FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+    Path expectedTrash = fsView2.makeQualified(
+        new Path("/mnt/datavol1/very/deep/deep/trash/dir/.Trash"));
+    Assert.assertEquals(expectedTrash, fsView2.getTrashRoot(testPath));
+  }
+
+  /**
+   * Test getTrashRoots() for all users.
+   */
+  @Test
+  public void testTrashRootsAllUsers() throws IOException {
+    Configuration conf2 = new Configuration(conf);
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true);
+    FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+
+    // Case 1: verify correct trash roots from fsView and fsView2
+    int beforeTrashRootNum = fsView.getTrashRoots(true).size();
+    int beforeTrashRootNum2 = fsView2.getTrashRoots(true).size();
+    Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2);
+
+    fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user1"));
+    fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2"));
+    fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user3"));
+    fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4"));
+    fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5"));
+    int afterTrashRootsNum = fsView.getTrashRoots(true).size();
+    int afterTrashRootsNum2 = fsView2.getTrashRoots(true).size();
+    Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum);
+    Assert.assertEquals(beforeTrashRootNum2 + 5, afterTrashRootsNum2);
+
+    // Case 2: per-user mount point
+    fsTarget.mkdirs(new Path(targetTestRoot, "Users/userA/.Trash/userA"));
+    Configuration conf3 = new Configuration(conf2);
+    ConfigUtil.addLink(conf3, "/Users/userA",
+        new Path(targetTestRoot, "Users/userA").toUri());
+    FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3);
+    int trashRootsNum3 = fsView3.getTrashRoots(true).size();
+    Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3);
+
+    // Case 3: single /Users mount point for all users
+    fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user1"));
+    fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user2"));
+    Configuration conf4 = new Configuration(conf2);
+    ConfigUtil.addLink(conf4, "/Users",
+        new Path(targetTestRoot, "Users").toUri());
+    FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4);
+    int trashRootsNum4 = fsView4.getTrashRoots(true).size();
+    Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4);
+
+    // Case 4: test trash roots in fallback FS
+    fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user10"));
+    fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user11"));
+    fsTarget.mkdirs(new Path(targetTestRoot, ".Trash/user12"));
+    Configuration conf5 = new Configuration(conf2);
+    ConfigUtil.addLinkFallback(conf5, targetTestRoot.toUri());
+    FileSystem fsView5 = FileSystem.get(FsConstants.VIEWFS_URI, conf5);
+    int trashRootsNum5 = fsView5.getTrashRoots(true).size();
+    Assert.assertEquals(afterTrashRootsNum2 + 3, trashRootsNum5);
+  }
+
+  /**
+   * Test getTrashRoots() for current user.
+   */
+  @Test
+  public void testTrashRootsCurrentUser() throws IOException {
+    String currentUser =
+        UserGroupInformation.getCurrentUser().getShortUserName();
+    Configuration conf2 = new Configuration(conf);
+    conf2.setBoolean(CONFIG_VIEWFS_TRASH_FORCE_INSIDE_MOUNT_POINT, true);
+    FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
+
+    int beforeTrashRootNum = fsView.getTrashRoots(false).size();
+    int beforeTrashRootNum2 = fsView2.getTrashRoots(false).size();
+    Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2);
+
+    fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/" + currentUser));
+    fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2"));
+    fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/" + currentUser));
+    fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4"));
+    fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5"));
+    int afterTrashRootsNum = fsView.getTrashRoots(false).size();
+    int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size();
+    Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum);
+    Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2);
+
+    // Test trash roots in fallback FS
+    Configuration conf3 = new Configuration(conf2);
+    fsTarget.mkdirs(new Path(targetTestRoot, TRASH_PREFIX + "/" + currentUser));
+    ConfigUtil.addLinkFallback(conf3, targetTestRoot.toUri());
+    FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3);
+    int trashRootsNum3 = fsView3.getTrashRoots(false).size();
+    Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3);
+  }
+
   @Test(expected = NotInMountpointException.class)
   public void testViewFileSystemUtil() throws Exception {
     Configuration newConf = new Configuration(conf);
@@ -1472,4 +1786,65 @@ public void testTargetFileSystemLazyInitialization() throws Exception {
     // viewfs inner cache is disabled
     assertEquals(cacheSize + 2, TestFileUtil.getCacheSize());
   }
+
+  @Test
+  public void testTargetFileSystemLazyInitializationForChecksumMethods()
+      throws Exception {
+    final String clusterName = "cluster" + new Random().nextInt();
+    Configuration config = new Configuration(conf);
+    config.setBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
+    config.setClass("fs.othermockfs.impl",
+        TestChRootedFileSystem.MockFileSystem.class, FileSystem.class);
+    ConfigUtil.addLink(config, clusterName, "/user",
+        URI.create("othermockfs://mockauth1/mockpath"));
+    ConfigUtil.addLink(config, clusterName,
+        "/mock", URI.create("othermockfs://mockauth/mockpath"));
+
+    final int cacheSize = TestFileUtil.getCacheSize();
+    ViewFileSystem viewFs = (ViewFileSystem) FileSystem.get(
+        new URI("viewfs://" + clusterName + "/"), config);
+
+    // As no inner file system instance has been initialized,
+    // cache size will remain the same
+    // cache is disabled for viewfs scheme, so the viewfs:// instance won't
+    // go in the cache even after the initialization
+    assertEquals(cacheSize, TestFileUtil.getCacheSize());
+
+    // This is not going to initialize any filesystem instance
+    viewFs.setVerifyChecksum(true);
+
+    // Cache size will remain the same
+    assertEquals(cacheSize, TestFileUtil.getCacheSize());
+
+    // This resolve path will initialize the file system corresponding
+    // to the mount table entry of the path "/user"
+    viewFs.getFileChecksum(
+        new Path(String.format("viewfs://%s/%s", clusterName, "/user")));
+
+    // Cache size will increase by 1.
+    assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
+
+    viewFs.close();
+    // Initialized FileSystem instances will not be removed from cache as
+    // viewfs inner cache is disabled
+    assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
+  }
+
+  @Test
+  public void testInvalidMountPoints() throws Exception {
+    final String clusterName = "cluster" + new Random().nextInt();
+    Configuration config = new Configuration(conf);
+    config.set(ConfigUtil.getConfigViewFsPrefix(clusterName) + "." +
+        Constants.CONFIG_VIEWFS_LINK + "." + "/invalidPath",
+        "othermockfs:|mockauth/mockpath");
+
+    try {
+      FileSystem viewFs = FileSystem.get(
+          new URI("viewfs://" + clusterName + "/"), config);
+      fail("FileSystem should not initialize. Should fail with IOException");
+    } catch (IOException ex) {
+      assertTrue("Should get URISyntax Exception",
+          ex.getMessage().startsWith("URISyntax exception"));
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
index 866c03ecda9d2..5713f532be7e8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemTestSetup.java
@@ -20,7 +20,6 @@
 import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.FsConstants;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
index 73f6265eee72c..1d855ab442600 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
@@ -522,7 +522,7 @@ public void testListOnInternalDirsOfMountTable() throws IOException {
         Assert.assertTrue("A mount should appear as symlink", fs.isSymlink());
   }
       
-  @Test
+  @Test(expected = FileNotFoundException.class)
   public void testFileStatusOnMountLink() throws IOException {
     Assert.assertTrue("Slash should appear as dir", 
         fcView.getFileStatus(new Path("/")).isDirectory());
@@ -534,12 +534,7 @@ public void testFileStatusOnMountLink() throws IOException {
     checkFileStatus(fcView, "/internalDir/internalDir2/linkToDir3", fileType.isDir);
     checkFileStatus(fcView, "/linkToAFile", fileType.isFile);
 
-    try {
-      fcView.getFileStatus(new Path("/danglingLink"));
-      Assert.fail("Excepted a not found exception here");
-    } catch ( FileNotFoundException e) {
-      // as excepted
-    }
+    fcView.getFileStatus(new Path("/danglingLink"));
   }
   
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java
new file mode 100644
index 0000000000000..ce068bb6f1cf6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestDisabledProfileServlet.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import javax.servlet.http.HttpServletResponse;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Small test to cover default disabled prof endpoint.
+ */
+public class TestDisabledProfileServlet extends HttpServerFunctionalTest {
+
+  private static HttpServer2 server;
+  private static URL baseUrl;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    server = createTestServer();
+    server.start();
+    baseUrl = getServerURL(server);
+  }
+
+  @AfterClass
+  public static void cleanup() throws Exception {
+    server.stop();
+  }
+
+  @Test
+  public void testQuery() throws Exception {
+    try {
+      readOutput(new URL(baseUrl, "/prof"));
+      throw new IllegalStateException("Should not reach here");
+    } catch (IOException e) {
+      assertTrue(e.getMessage()
+          .contains(HttpServletResponse.SC_INTERNAL_SERVER_ERROR + " for URL: " + baseUrl));
+    }
+
+    // CORS headers
+    HttpURLConnection conn =
+        (HttpURLConnection) new URL(baseUrl, "/prof").openConnection();
+    assertEquals("GET", conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_METHODS));
+    assertNotNull(conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_ORIGIN));
+    conn.disconnect();
+  }
+
+  @Test
+  public void testRequestMethods() throws IOException {
+    HttpURLConnection connection = getConnection("PUT");
+    assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED,
+        connection.getResponseCode());
+    connection.disconnect();
+    connection = getConnection("POST");
+    assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED,
+        connection.getResponseCode());
+    connection.disconnect();
+    connection = getConnection("DELETE");
+    assertEquals("Unexpected response code", HttpServletResponse.SC_METHOD_NOT_ALLOWED,
+        connection.getResponseCode());
+    connection.disconnect();
+    connection = getConnection("GET");
+    assertEquals("Unexpected response code", HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
+        connection.getResponseCode());
+    connection.disconnect();
+  }
+
+  private HttpURLConnection getConnection(final String method) throws IOException {
+    URL url = new URL(baseUrl, "/prof");
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod(method);
+    return conn;
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java
index d0123e32039c9..58721c4baa8f9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLog.java
@@ -17,32 +17,25 @@
  */
 package org.apache.hadoop.http;
 
-import org.apache.log4j.Logger;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
 import org.eclipse.jetty.server.CustomRequestLog;
 import org.eclipse.jetty.server.RequestLog;
+import org.eclipse.jetty.server.Slf4jRequestLogWriter;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
 public class TestHttpRequestLog {
 
-  @Test
-  public void testAppenderUndefined() {
-    RequestLog requestLog = HttpRequestLog.getRequestLog("test");
-    assertNull("RequestLog should be null", requestLog);
-  }
-
   @Test
   public void testAppenderDefined() {
-    HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
-    requestLogAppender.setName("testrequestlog");
-    Logger.getLogger("http.requests.test").addAppender(requestLogAppender);
     RequestLog requestLog = HttpRequestLog.getRequestLog("test");
-    Logger.getLogger("http.requests.test").removeAppender(requestLogAppender);
     assertNotNull("RequestLog should not be null", requestLog);
-    assertEquals("Class mismatch",
-        CustomRequestLog.class, requestLog.getClass());
+    assertThat(requestLog, instanceOf(CustomRequestLog.class));
+    CustomRequestLog crl = (CustomRequestLog) requestLog;
+    assertThat(crl.getWriter(), instanceOf(Slf4jRequestLogWriter.class));
+    assertEquals(CustomRequestLog.EXTENDED_NCSA_FORMAT, crl.getFormatString());
   }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index b1255d19d9086..37e3415546e2e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.test.Whitebox;
 
 import org.assertj.core.api.Assertions;
 import org.eclipse.jetty.server.ServerConnector;
@@ -663,8 +662,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort)
     HttpServer2 server = createServer(host, port);
     try {
       // not bound, ephemeral should return requested port (0 for ephemeral)
-      List<?> listeners = (List<?>) Whitebox.getInternalState(server,
-          "listeners");
+      List<ServerConnector> listeners = server.getListeners();
       ServerConnector listener = (ServerConnector)listeners.get(0);
 
       assertEquals(port, listener.getPort());
@@ -740,12 +738,21 @@ public void testBacklogSize() throws Exception
     Configuration conf = new Configuration();
     conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize);
     HttpServer2 srv = createServer("test", conf);
-    List<?> listeners = (List<?>) Whitebox.getInternalState(srv,
-            "listeners");
+    List<ServerConnector> listeners = srv.getListeners();
     ServerConnector listener = (ServerConnector)listeners.get(0);
     assertEquals(backlogSize, listener.getAcceptQueueSize());
   }
 
+  @Test
+  public void testBacklogSize2() throws Exception
+  {
+    Configuration conf = new Configuration();
+    HttpServer2 srv = createServer("test", conf);
+    List<ServerConnector> listeners = srv.getListeners();
+    ServerConnector listener = (ServerConnector)listeners.get(0);
+    assertEquals(500, listener.getAcceptQueueSize());
+  }
+
   @Test
   public void testIdleTimeout() throws Exception {
     final int idleTimeout = 1000;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java
index 40f1b3df08c6e..4ae1190abd5af 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerLifecycle.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.http;
 
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Logger;
 import org.junit.Test;
 
 public class TestHttpServerLifecycle extends HttpServerFunctionalTest {
@@ -68,27 +66,6 @@ public void testStartedServerIsAlive() throws Throwable {
     stop(server);
   }
 
-  /**
-   * Test that the server with request logging enabled
-   *
-   * @throws Throwable on failure
-   */
-  @Test
-  public void testStartedServerWithRequestLog() throws Throwable {
-    HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
-    requestLogAppender.setName("httprequestlog");
-    requestLogAppender.setFilename(
-        GenericTestUtils.getTempPath("jetty-name-yyyy_mm_dd.log"));
-    Logger.getLogger(HttpServer2.class.getName() + ".test").addAppender(requestLogAppender);
-    HttpServer2 server = null;
-    server = createTestServer();
-    assertNotLive(server);
-    server.start();
-    assertAlive(server);
-    stop(server);
-    Logger.getLogger(HttpServer2.class.getName() + ".test").removeAppender(requestLogAppender);
-  }
-
   /**
    * Assert that the result of {@link HttpServer2#toString()} contains the specific text
    * @param server server to examine
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java
new file mode 100644
index 0000000000000..5c87451a49e6c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestProfileServlet.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.http;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.UUID;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test coverage for async profiler servlets: ProfileServlet and ProfileOutputServlet.
+ */
+public class TestProfileServlet extends HttpServerFunctionalTest {
+
+  private static HttpServer2 server;
+  private static URL baseUrl;
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestProfileServlet.class);
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    ProfileServlet.setIsTestRun(true);
+    System.setProperty("async.profiler.home", UUID.randomUUID().toString());
+    server = createTestServer();
+    server.start();
+    baseUrl = getServerURL(server);
+  }
+
+  @AfterClass
+  public static void cleanup() throws Exception {
+    ProfileServlet.setIsTestRun(false);
+    System.clearProperty("async.profiler.home");
+    server.stop();
+  }
+
+  @Test
+  public void testQuery() throws Exception {
+    String output = readOutput(new URL(baseUrl, "/prof"));
+    LOG.info("/prof output: {}", output);
+    assertTrue(output.startsWith(
+        "Started [cpu] profiling. This page will automatically redirect to /prof-output-hadoop/"));
+    assertTrue(output.contains(
+        "If empty diagram and Linux 4.6+, see 'Basic Usage' section on the Async Profiler Home"
+            + " Page, https://github.com/jvm-profiling-tools/async-profiler."));
+
+    HttpURLConnection conn =
+        (HttpURLConnection) new URL(baseUrl, "/prof").openConnection();
+    assertEquals("GET", conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_METHODS));
+    assertEquals(HttpURLConnection.HTTP_ACCEPTED, conn.getResponseCode());
+    assertNotNull(conn.getHeaderField(ProfileServlet.ACCESS_CONTROL_ALLOW_ORIGIN));
+    assertTrue(conn.getHeaderField("Refresh").startsWith("10;/prof-output-hadoop/async-prof-pid"));
+
+    String redirectOutput = readOutput(new URL(baseUrl, "/prof-output-hadoop"));
+    LOG.info("/prof-output-hadoop output: {}", redirectOutput);
+
+    HttpURLConnection redirectedConn =
+        (HttpURLConnection) new URL(baseUrl, "/prof-output-hadoop").openConnection();
+    assertEquals(HttpURLConnection.HTTP_OK, redirectedConn.getResponseCode());
+
+    redirectedConn.disconnect();
+    conn.disconnect();
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
index ec76ea008077d..9c9b75fa76e6c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
@@ -41,7 +41,7 @@ public static void testReflect(Object value, Type type, String schema)
 
     // check that schema matches expected
     Schema s = ReflectData.get().getSchema(type);
-    assertEquals(Schema.parse(schema), s);
+    assertEquals(new Schema.Parser().parse(schema), s);
 
     // check that value is serialized correctly
     ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(s);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
index b70e011f6aa13..c15ec8caa4f6c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
@@ -26,6 +26,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assert.assertEquals;
 
 public class TestDefaultStringifier {
@@ -98,7 +99,7 @@ public void testStoreLoad() throws IOException {
   }
 
   @Test
-  public void testStoreLoadArray() throws IOException {
+  public void testStoreLoadArray() throws Exception {
     LOG.info("Testing DefaultStringifier#storeArray() and #loadArray()");
     conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
 
@@ -107,6 +108,8 @@ public void testStoreLoadArray() throws IOException {
     Integer[] array = new Integer[] {1,2,3,4,5};
 
 
+    intercept(IndexOutOfBoundsException.class, () ->
+        DefaultStringifier.storeArray(conf, new Integer[] {}, keyName));
     DefaultStringifier.storeArray(conf, array, keyName);
 
     Integer[] claimedArray = DefaultStringifier.<Integer>loadArray(conf, keyName, Integer.class);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java
index 5e71601742f3c..11459261f5b74 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestEnumSetWritable.java
@@ -119,7 +119,7 @@ public void testSerializeAndDeserializeNull() throws IOException {
   public void testAvroReflect() throws Exception {
     String schema = "{\"type\":\"array\",\"items\":{\"type\":\"enum\","
         + "\"name\":\"TestEnumSet\","
-        + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable$\","
+        + "\"namespace\":\"org.apache.hadoop.io.TestEnumSetWritable\","
         + "\"symbols\":[\"CREATE\",\"OVERWRITE\",\"APPEND\"]},"
         + "\"java-class\":\"org.apache.hadoop.io.EnumSetWritable\"}";
     Type type =
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
index fca72d9c65a6a..51f207f97ad29 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
@@ -275,7 +275,7 @@ public void testListDirectory() throws IOException {
     File dir = new File("testListDirectory");
     Files.createDirectory(dir.toPath());
     try {
-      Set<String> entries = new HashSet<String>();
+      Set<String> entries = new HashSet<>();
       entries.add("entry1");
       entries.add("entry2");
       entries.add("entry3");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMoreWeakReferencedElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMoreWeakReferencedElasticByteBufferPool.java
new file mode 100644
index 0000000000000..6ca380ef0e46b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMoreWeakReferencedElasticByteBufferPool.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Non parameterized tests for {@code WeakReferencedElasticByteBufferPool}.
+ */
+public class TestMoreWeakReferencedElasticByteBufferPool
+        extends HadoopTestBase {
+
+  @Test
+  public void testMixedBuffersInPool() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer1 = pool.getBuffer(true, 5);
+    ByteBuffer buffer2 = pool.getBuffer(true, 10);
+    ByteBuffer buffer3 = pool.getBuffer(false, 5);
+    ByteBuffer buffer4 = pool.getBuffer(false, 10);
+    ByteBuffer buffer5 = pool.getBuffer(true, 15);
+
+    assertBufferCounts(pool, 0, 0);
+    pool.putBuffer(buffer1);
+    pool.putBuffer(buffer2);
+    assertBufferCounts(pool, 2, 0);
+    pool.putBuffer(buffer3);
+    assertBufferCounts(pool, 2, 1);
+    pool.putBuffer(buffer5);
+    assertBufferCounts(pool, 3, 1);
+    pool.putBuffer(buffer4);
+    assertBufferCounts(pool, 3, 2);
+    pool.release();
+    assertBufferCounts(pool, 0, 0);
+
+  }
+
+  @Test
+  public void testUnexpectedBufferSizes() throws Exception {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer1 = pool.getBuffer(true, 0);
+
+    // try writing a random byte in a 0 length buffer.
+    // Expected exception as buffer requested is of size 0.
+    intercept(BufferOverflowException.class,
+        () -> buffer1.put(new byte[1]));
+
+    // Expected IllegalArgumentException as negative length buffer is requested.
+    intercept(IllegalArgumentException.class,
+        () -> pool.getBuffer(true, -5));
+
+    // test returning null buffer to the pool.
+    intercept(NullPointerException.class,
+        () -> pool.putBuffer(null));
+  }
+
+  /**
+   * Utility method to assert counts of direct and heap buffers in
+   * the given buffer pool.
+   * @param pool buffer pool.
+   * @param numDirectBuffersExpected expected number of direct buffers.
+   * @param numHeapBuffersExpected expected number of heap buffers.
+   */
+  private void assertBufferCounts(WeakReferencedElasticByteBufferPool pool,
+                                  int numDirectBuffersExpected,
+                                  int numHeapBuffersExpected) {
+    Assertions.assertThat(pool.getCurrentBuffersCount(true))
+            .describedAs("Number of direct buffers in pool")
+            .isEqualTo(numDirectBuffersExpected);
+    Assertions.assertThat(pool.getCurrentBuffersCount(false))
+            .describedAs("Number of heap buffers in pool")
+            .isEqualTo(numHeapBuffersExpected);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWeakReferencedElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWeakReferencedElasticByteBufferPool.java
new file mode 100644
index 0000000000000..1434010ffa652
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWeakReferencedElasticByteBufferPool.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+/**
+ * Unit tests for {@code WeakReferencedElasticByteBufferPool}.
+ */
+@RunWith(Parameterized.class)
+public class TestWeakReferencedElasticByteBufferPool
+        extends HadoopTestBase {
+
+  private final boolean isDirect;
+
+  private final String type;
+
+  @Parameterized.Parameters(name = "Buffer type : {0}")
+  public static List<String> params() {
+    return Arrays.asList("direct", "array");
+  }
+
+  public TestWeakReferencedElasticByteBufferPool(String type) {
+    this.type = type;
+    this.isDirect = !"array".equals(type);
+  }
+
+  @Test
+  public void testGetAndPutBasic() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    int bufferSize = 5;
+    ByteBuffer buffer = pool.getBuffer(isDirect, bufferSize);
+    Assertions.assertThat(buffer.isDirect())
+            .describedAs("Buffered returned should be of correct type {}", type)
+            .isEqualTo(isDirect);
+    Assertions.assertThat(buffer.capacity())
+            .describedAs("Initial capacity of returned buffer from pool")
+            .isEqualTo(bufferSize);
+    Assertions.assertThat(buffer.position())
+            .describedAs("Initial position of returned buffer from pool")
+            .isEqualTo(0);
+
+    byte[] arr = createByteArray(bufferSize);
+    buffer.put(arr, 0, arr.length);
+    buffer.flip();
+    validateBufferContent(buffer, arr);
+    Assertions.assertThat(buffer.position())
+            .describedAs("Buffer's position after filling bytes in it")
+            .isEqualTo(bufferSize);
+    // releasing buffer to the pool.
+    pool.putBuffer(buffer);
+    Assertions.assertThat(buffer.position())
+            .describedAs("Position should be reset to 0 after returning buffer to the pool")
+            .isEqualTo(0);
+
+  }
+
+  @Test
+  public void testPoolingWithDifferentSizes() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer = pool.getBuffer(isDirect, 5);
+    ByteBuffer buffer1 = pool.getBuffer(isDirect, 10);
+    ByteBuffer buffer2 = pool.getBuffer(isDirect, 15);
+
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(0);
+
+    pool.putBuffer(buffer1);
+    pool.putBuffer(buffer2);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(2);
+    ByteBuffer buffer3 = pool.getBuffer(isDirect, 12);
+    Assertions.assertThat(buffer3.capacity())
+            .describedAs("Pooled buffer should have older capacity")
+            .isEqualTo(15);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(1);
+    pool.putBuffer(buffer);
+    ByteBuffer buffer4 = pool.getBuffer(isDirect, 6);
+    Assertions.assertThat(buffer4.capacity())
+            .describedAs("Pooled buffer should have older capacity")
+            .isEqualTo(10);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(1);
+
+    pool.release();
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool post release")
+            .isEqualTo(0);
+  }
+
+  @Test
+  public void testPoolingWithDifferentInsertionTime() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer = pool.getBuffer(isDirect, 10);
+    ByteBuffer buffer1 = pool.getBuffer(isDirect, 10);
+    ByteBuffer buffer2 = pool.getBuffer(isDirect, 10);
+
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(0);
+
+    pool.putBuffer(buffer1);
+    pool.putBuffer(buffer2);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(2);
+    ByteBuffer buffer3 = pool.getBuffer(isDirect, 10);
+    // As buffer1 is returned to the pool before buffer2, it should
+    // be returned when buffer of same size is asked again from
+    // the pool. Memory references must match not just content
+    // that is why {@code Assertions.isSameAs} is used here rather
+    // than usual {@code Assertions.isEqualTo}.
+    Assertions.assertThat(buffer3)
+            .describedAs("Buffers should be returned in order of their " +
+                    "insertion time")
+            .isSameAs(buffer1);
+    pool.putBuffer(buffer);
+    ByteBuffer buffer4 = pool.getBuffer(isDirect, 10);
+    Assertions.assertThat(buffer4)
+            .describedAs("Buffers should be returned in order of their " +
+                    "insertion time")
+            .isSameAs(buffer2);
+  }
+
+  @Test
+  public void testGarbageCollection() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer = pool.getBuffer(isDirect, 5);
+    ByteBuffer buffer1 = pool.getBuffer(isDirect, 10);
+    ByteBuffer buffer2 = pool.getBuffer(isDirect, 15);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(0);
+    pool.putBuffer(buffer1);
+    pool.putBuffer(buffer2);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(2);
+    // Before GC.
+    ByteBuffer buffer4 = pool.getBuffer(isDirect, 12);
+    Assertions.assertThat(buffer4.capacity())
+            .describedAs("Pooled buffer should have older capacity")
+            .isEqualTo(15);
+    pool.putBuffer(buffer4);
+    // Removing the references
+    buffer1 = null;
+    buffer2 = null;
+    buffer4 = null;
+    System.gc();
+    ByteBuffer buffer3 = pool.getBuffer(isDirect, 12);
+    Assertions.assertThat(buffer3.capacity())
+            .describedAs("After garbage collection new buffer should be " +
+                    "returned with fixed capacity")
+            .isEqualTo(12);
+  }
+
+  @Test
+  public void testWeakReferencesPruning() {
+    WeakReferencedElasticByteBufferPool pool = new WeakReferencedElasticByteBufferPool();
+    ByteBuffer buffer1 = pool.getBuffer(isDirect, 5);
+    ByteBuffer buffer2 = pool.getBuffer(isDirect, 10);
+    ByteBuffer buffer3 = pool.getBuffer(isDirect, 15);
+
+    pool.putBuffer(buffer2);
+    pool.putBuffer(buffer3);
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(2);
+
+    // marking only buffer2 to be garbage collected.
+    buffer2 = null;
+    System.gc();
+    ByteBuffer buffer4 = pool.getBuffer(isDirect, 10);
+    // Number of buffers in the pool is 0 as one got garbage
+    // collected and other got returned in above call.
+    Assertions.assertThat(pool.getCurrentBuffersCount(isDirect))
+            .describedAs("Number of buffers in the pool")
+            .isEqualTo(0);
+    Assertions.assertThat(buffer4.capacity())
+            .describedAs("After gc, pool should return next greater than " +
+                    "available buffer")
+            .isEqualTo(15);
+
+  }
+
+  private void validateBufferContent(ByteBuffer buffer, byte[] arr) {
+    for (int i=0; i<arr.length; i++) {
+      Assertions.assertThat(buffer.get())
+              .describedAs("Content of buffer at index {} should match " +
+                      "with content of byte array", i)
+              .isEqualTo(arr[i]);
+    }
+  }
+
+  private byte[] createByteArray(int length) {
+    byte[] arr = new byte[length];
+    Random r = new Random();
+    r.nextBytes(arr);
+    return arr;
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestBZip2Codec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestBZip2Codec.java
new file mode 100644
index 0000000000000..9dd3215f90d5e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestBZip2Codec.java
@@ -0,0 +1,203 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.io.compress;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE;
+import org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter;
+import org.apache.hadoop.io.compress.bzip2.BZip2Utils;
+
+import static org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE.BYBLOCK;
+import static org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE.CONTINUOUS;
+import static org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter.BLOCK_SIZE;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.assertj.core.api.Assertions.assertThatNullPointerException;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public final class TestBZip2Codec {
+
+  private static final long HEADER_LEN = 2;
+
+  private Configuration conf;
+  private FileSystem fs;
+  private BZip2Codec codec;
+  private Decompressor decompressor;
+  private Path tempFile;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+
+    Path workDir = new Path(System.getProperty("test.build.data", "target"),
+        "data/" + getClass().getSimpleName());
+
+    Path inputDir = new Path(workDir, "input");
+    tempFile = new Path(inputDir, "test.txt.bz2");
+
+    fs = workDir.getFileSystem(conf);
+
+    codec = new BZip2Codec();
+    codec.setConf(new Configuration(/* loadDefaults */ false));
+    decompressor = CodecPool.getDecompressor(codec);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    CodecPool.returnDecompressor(decompressor);
+    fs.delete(tempFile, /* recursive */ false);
+  }
+
+  @Test
+  public void createInputStreamWithStartAndEnd() throws Exception {
+    byte[] data1 = newAlternatingByteArray(BLOCK_SIZE, 'a', 'b');
+    byte[] data2 = newAlternatingByteArray(BLOCK_SIZE, 'c', 'd');
+    byte[] data3 = newAlternatingByteArray(BLOCK_SIZE, 'e', 'f');
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.write(data1);
+      writer.write(data2);
+      writer.write(data3);
+    }
+    long fileSize = fs.getFileStatus(tempFile).getLen();
+
+    List<Long> nextBlockOffsets = BZip2Utils.getNextBlockMarkerOffsets(tempFile, conf);
+    long block2Start = nextBlockOffsets.get(0);
+    long block3Start = nextBlockOffsets.get(1);
+
+    try (SplitCompressionInputStream stream = newCompressionStream(tempFile, 0, fileSize,
+        BYBLOCK)) {
+      assertEquals(0, stream.getPos());
+      assertCasesWhereReadDoesNotAdvanceStream(stream);
+      assertReadingAtPositionZero(stream, data1);
+      assertCasesWhereReadDoesNotAdvanceStream(stream);
+      assertReadingPastEndOfBlock(stream, block2Start, data2);
+      assertReadingPastEndOfBlock(stream, block3Start, data3);
+      assertEquals(-1, stream.read());
+    }
+
+    try (SplitCompressionInputStream stream = newCompressionStream(tempFile, 1, fileSize - 1,
+        BYBLOCK)) {
+      assertEquals(block2Start, stream.getPos());
+      assertCasesWhereReadDoesNotAdvanceStream(stream);
+      assertReadingPastEndOfBlock(stream, block2Start, data2);
+      assertCasesWhereReadDoesNotAdvanceStream(stream);
+      assertReadingPastEndOfBlock(stream, block3Start, data3);
+      assertEquals(-1, stream.read());
+    }
+
+    // With continuous mode, only starting at or after the stream header is
+    // supported.
+    byte[] allData = Bytes.concat(data1, data2, data3);
+    assertReadingWithContinuousMode(tempFile, 0, fileSize, allData);
+    assertReadingWithContinuousMode(tempFile, HEADER_LEN, fileSize - HEADER_LEN, allData);
+  }
+
+  private void assertReadingWithContinuousMode(Path file, long start, long length,
+      byte[] expectedData) throws IOException {
+    try (SplitCompressionInputStream stream = newCompressionStream(file, start, length,
+        CONTINUOUS)) {
+      assertEquals(HEADER_LEN, stream.getPos());
+
+      assertRead(stream, expectedData);
+      assertEquals(-1, stream.read());
+
+      // When specifying CONTINUOUS read mode, the position ends up not being
+      // updated at all.
+      assertEquals(HEADER_LEN, stream.getPos());
+    }
+  }
+
+  private SplitCompressionInputStream newCompressionStream(Path file, long start, long length,
+      READ_MODE readMode) throws IOException {
+    FSDataInputStream rawIn = fs.open(file);
+    rawIn.seek(start);
+    long end = start + length;
+    return codec.createInputStream(rawIn, decompressor, start, end, readMode);
+  }
+
+  private static byte[] newAlternatingByteArray(int size, int... choices) {
+    checkArgument(choices.length > 1);
+    byte[] result = new byte[size];
+    for (int i = 0; i < size; i++) {
+      result[i] = (byte) choices[i % choices.length];
+    }
+    return result;
+  }
+
+  private static void assertCasesWhereReadDoesNotAdvanceStream(SplitCompressionInputStream in)
+      throws IOException {
+    long initialPos = in.getPos();
+
+    assertEquals(0, in.read(new byte[0]));
+
+    assertThatNullPointerException().isThrownBy(() -> in.read(null, 0, 1));
+    assertThatExceptionOfType(IndexOutOfBoundsException.class).isThrownBy(
+        () -> in.read(new byte[5], -1, 2));
+    assertThatExceptionOfType(IndexOutOfBoundsException.class).isThrownBy(
+        () -> in.read(new byte[5], 0, -1));
+    assertThatExceptionOfType(IndexOutOfBoundsException.class).isThrownBy(
+        () -> in.read(new byte[5], 1, 5));
+
+    assertEquals(initialPos, in.getPos());
+  }
+
+  private static void assertReadingAtPositionZero(SplitCompressionInputStream in,
+      byte[] expectedData) throws IOException {
+    byte[] buffer = new byte[expectedData.length];
+    assertEquals(1, in.read(buffer, 0, 1));
+    assertEquals(expectedData[0], buffer[0]);
+    assertEquals(0, in.getPos());
+
+    IOUtils.readFully(in, buffer, 1, expectedData.length - 1);
+    assertArrayEquals(expectedData, buffer);
+    assertEquals(0, in.getPos());
+  }
+
+  private static void assertReadingPastEndOfBlock(SplitCompressionInputStream in,
+      long endOfBlockPos, byte[] expectedData) throws IOException {
+    byte[] buffer = new byte[expectedData.length];
+    assertEquals(1, in.read(buffer));
+    assertEquals(expectedData[0], buffer[0]);
+    assertEquals(endOfBlockPos + 1, in.getPos());
+
+    IOUtils.readFully(in, buffer, 1, expectedData.length - 1);
+    assertArrayEquals(expectedData, buffer);
+    assertEquals(endOfBlockPos + 1, in.getPos());
+  }
+
+  private static void assertRead(InputStream in, byte[] expectedData) throws IOException {
+    byte[] buffer = new byte[expectedData.length];
+    IOUtils.readFully(in, buffer);
+    assertArrayEquals(expectedData, buffer);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
index 0bda66741ba65..5a63c06515e8b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
@@ -78,7 +78,6 @@
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
index edab634a0b877..7461ea36f59a3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
@@ -29,6 +29,7 @@
 
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 
 public class TestCodecFactory {
@@ -45,8 +46,8 @@ public Configuration getConf() {
     }
     
     @Override
-    public CompressionOutputStream createOutputStream(OutputStream out) 
-    throws IOException {
+    public CompressionOutputStream createOutputStream(OutputStream out)
+            throws IOException {
       return null;
     }
     
@@ -62,21 +63,21 @@ public Compressor createCompressor() {
 
     @Override
     public CompressionInputStream createInputStream(InputStream in, 
-                                                    Decompressor decompressor) 
-    throws IOException {
+                                                    Decompressor decompressor)
+            throws IOException {
       return null;
     }
 
     @Override
-    public CompressionInputStream createInputStream(InputStream in) 
-    throws IOException {
+    public CompressionInputStream createInputStream(InputStream in)
+            throws IOException {
       return null;
     }
 
     @Override
     public CompressionOutputStream createOutputStream(OutputStream out, 
-                                                      Compressor compressor) 
-    throws IOException {
+                                                      Compressor compressor)
+            throws IOException {
       return null;
     }
 
@@ -125,7 +126,7 @@ public String getDefaultExtension() {
   }
   
   /**
-   * Returns a factory for a given set of codecs
+   * Returns a factory for a given set of codecs.
    * @param classes the codec classes to include
    * @return a new factory
    */
@@ -137,15 +138,21 @@ private static CompressionCodecFactory setClasses(Class[] classes) {
   
   private static void checkCodec(String msg, 
                                  Class expected, CompressionCodec actual) {
-    assertEquals(msg + " unexpected codec found",
-                 expected.getName(),
-                 actual.getClass().getName());
+    if (expected == null) {
+      assertNull(msg, actual);
+    } else if (actual == null) {
+      fail(msg + " result was null");
+    } else {
+      assertEquals(msg + " unexpected codec found",
+              expected.getName(),
+              actual.getClass().getName());
+    }
   }
 
   @Test
   public void testFinding() {
     CompressionCodecFactory factory =
-      new CompressionCodecFactory(new Configuration());
+            new CompressionCodecFactory(new Configuration());
     CompressionCodec codec = factory.getCodec(new Path("/tmp/foo.bar"));
     assertEquals("default factory foo codec", null, codec);
     codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName());
@@ -153,6 +160,8 @@ public void testFinding() {
     
     codec = factory.getCodec(new Path("/tmp/foo.gz"));
     checkCodec("default factory for .gz", GzipCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo.GZ"));
+    checkCodec("default factory for .GZ", GzipCodec.class, codec);
     codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
     checkCodec("default factory for gzip codec", GzipCodec.class, codec);
     codec = factory.getCodecByName("gzip");
@@ -168,6 +177,8 @@ public void testFinding() {
 
     codec = factory.getCodec(new Path("/tmp/foo.bz2"));
     checkCodec("default factory for .bz2", BZip2Codec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo.BZ2"));
+    checkCodec("default factory for .BZ2", BZip2Codec.class, codec);
     codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
     checkCodec("default factory for bzip2 codec", BZip2Codec.class, codec);
     codec = factory.getCodecByName("bzip2");
@@ -221,16 +232,22 @@ public void testFinding() {
                                      FooBarCodec.class});
     codec = factory.getCodec(new Path("/tmp/.foo.bar.gz"));
     checkCodec("full factory gz codec", GzipCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/.foo.bar.GZ"));
+    checkCodec("full factory GZ codec", GzipCodec.class, codec);
     codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
     checkCodec("full codec gz codec", GzipCodec.class, codec);
      
     codec = factory.getCodec(new Path("/tmp/foo.bz2"));
     checkCodec("full factory for .bz2", BZip2Codec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo.BZ2"));
+    checkCodec("full factory for .BZ2", BZip2Codec.class, codec);
     codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
     checkCodec("full codec bzip2 codec", BZip2Codec.class, codec);
 
     codec = factory.getCodec(new Path("/tmp/foo.bar"));
     checkCodec("full factory bar codec", BarCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo.BAR"));
+    checkCodec("full factory BAR codec", BarCodec.class, codec);
     codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName());
     checkCodec("full factory bar codec", BarCodec.class, codec);
     codec = factory.getCodecByName("bar");
@@ -240,6 +257,8 @@ public void testFinding() {
 
     codec = factory.getCodec(new Path("/tmp/foo/baz.foo.bar"));
     checkCodec("full factory foo bar codec", FooBarCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo/baz.FOO.bar"));
+    checkCodec("full factory FOO bar codec", FooBarCodec.class, codec);
     codec = factory.getCodecByClassName(FooBarCodec.class.getCanonicalName());
     checkCodec("full factory foo bar codec", FooBarCodec.class, codec);
     codec = factory.getCodecByName("foobar");
@@ -249,6 +268,8 @@ public void testFinding() {
 
     codec = factory.getCodec(new Path("/tmp/foo.foo"));
     checkCodec("full factory foo codec", FooCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/FOO.FOO"));
+    checkCodec("full factory FOO codec", FooCodec.class, codec);
     codec = factory.getCodecByClassName(FooCodec.class.getCanonicalName());
     checkCodec("full factory foo codec", FooCodec.class, codec);
     codec = factory.getCodecByName("foo");
@@ -259,6 +280,8 @@ public void testFinding() {
     factory = setClasses(new Class[]{NewGzipCodec.class});
     codec = factory.getCodec(new Path("/tmp/foo.gz"));
     checkCodec("overridden factory for .gz", NewGzipCodec.class, codec);
+    codec = factory.getCodec(new Path("/tmp/foo.GZ"));
+    checkCodec("overridden factory for .GZ", NewGzipCodec.class, codec);
     codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName());
     checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
     
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
index 1fb25cb9087c2..4b18ee6047ba4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
@@ -19,6 +19,10 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -26,6 +30,9 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.zlib.BuiltInGzipCompressor;
+import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -69,7 +76,7 @@ public void testCompressorNotReturnSameInstance() {
     Compressor comp = CodecPool.getCompressor(codec);
     CodecPool.returnCompressor(comp);
     CodecPool.returnCompressor(comp);
-    Set<Compressor> compressors = new HashSet<Compressor>();
+    Set<Compressor> compressors = new HashSet<>();
     for (int i = 0; i < 10; ++i) {
       compressors.add(CodecPool.getCompressor(codec));
     }
@@ -180,7 +187,7 @@ public void testDecompressorNotReturnSameInstance() {
     Decompressor decomp = CodecPool.getDecompressor(codec);
     CodecPool.returnDecompressor(decomp);
     CodecPool.returnDecompressor(decomp);
-    Set<Decompressor> decompressors = new HashSet<Decompressor>();
+    Set<Decompressor> decompressors = new HashSet<>();
     for (int i = 0; i < 10; ++i) {
       decompressors.add(CodecPool.getDecompressor(codec));
     }
@@ -189,4 +196,56 @@ public void testDecompressorNotReturnSameInstance() {
       CodecPool.returnDecompressor(decompressor);
     }
   }
+
+  @Test(timeout = 10000)
+  public void testDoNotPoolCompressorNotUseableAfterReturn() throws Exception {
+
+    final GzipCodec gzipCodec = new GzipCodec();
+    gzipCodec.setConf(new Configuration());
+
+    // BuiltInGzipCompressor is an explicit example of a Compressor with the @DoNotPool annotation
+    final Compressor compressor = new BuiltInGzipCompressor(new Configuration());
+    CodecPool.returnCompressor(compressor);
+
+    final CompressionOutputStream outputStream =
+            gzipCodec.createOutputStream(new ByteArrayOutputStream(), compressor);
+    LambdaTestUtils.intercept(
+            AlreadyClosedException.class,
+            "compress called on closed compressor",
+            "Compressor from Codec with @DoNotPool should not be " +
+                    "useable after returning to CodecPool",
+        () -> outputStream.write(1));
+  }
+
+  @Test(timeout = 10000)
+  public void testDoNotPoolDecompressorNotUseableAfterReturn() throws Exception {
+
+    final GzipCodec gzipCodec = new GzipCodec();
+    gzipCodec.setConf(new Configuration());
+
+    final Random random = new Random();
+    final byte[] bytes = new byte[1024];
+    random.nextBytes(bytes);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (OutputStream outputStream = gzipCodec.createOutputStream(baos)) {
+      outputStream.write(bytes);
+    }
+
+    final byte[] gzipBytes = baos.toByteArray();
+    final ByteArrayInputStream bais = new ByteArrayInputStream(gzipBytes);
+
+    // BuiltInGzipDecompressor is an explicit example of a Decompressor
+    // with the @DoNotPool annotation
+    final Decompressor decompressor = new BuiltInGzipDecompressor();
+    CodecPool.returnDecompressor(decompressor);
+
+    final CompressionInputStream inputStream = gzipCodec.createInputStream(bais, decompressor);
+    LambdaTestUtils.intercept(
+            AlreadyClosedException.class,
+            "decompress called on closed decompressor",
+            "Decompressor from Codec with @DoNotPool should not be " +
+                    "useable after returning to CodecPool",
+        () -> inputStream.read());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java
new file mode 100644
index 0000000000000..5ca99fd02dfbf
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java
@@ -0,0 +1,114 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.BZip2Codec;
+
+import static org.apache.hadoop.io.compress.bzip2.CBZip2OutputStream.MIN_BLOCKSIZE;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
+/**
+ * A writer that simplifies creating BZip2 compressed text data for testing
+ * purposes.
+ */
+public final class BZip2TextFileWriter implements Closeable {
+
+  // Use minimum block size to reduce amount of data to require to be written
+  // to CBZip2OutputStream before a new block is created.
+  private static final int BLOCK_SIZE_100K = MIN_BLOCKSIZE;
+
+  /**
+   * The amount of bytes of run-length encoded data that needs to be written
+   * to this writer in order for the next byte written starts a new BZip2 block.
+   */
+  public static final int BLOCK_SIZE =
+      // The + 1 is needed because of how CBZip2OutputStream checks whether the
+      // last offset written is less than allowable block size. Because the last
+      // offset is one less of the amount of bytes written to the block, we need
+      // to write an extra byte to trigger writing a new block.
+      CBZip2OutputStream.getAllowableBlockSize(BLOCK_SIZE_100K) + 1;
+
+  private final CBZip2OutputStream out;
+
+  public BZip2TextFileWriter(Path path, Configuration conf) throws IOException {
+    this(path.getFileSystem(conf).create(path));
+  }
+
+  public BZip2TextFileWriter(OutputStream rawOut) throws IOException {
+    try {
+      BZip2Codec.writeHeader(rawOut);
+      out = new CBZip2OutputStream(rawOut, BLOCK_SIZE_100K);
+    } catch (Throwable e) {
+      rawOut.close();
+      throw e;
+    }
+  }
+
+  public void writeManyRecords(int totalSize, int numRecords, byte[] delimiter)
+      throws IOException {
+    checkArgument(numRecords > 0);
+    checkArgument(delimiter.length > 0);
+
+    int minRecordSize = totalSize / numRecords;
+    checkArgument(minRecordSize >= delimiter.length);
+
+    int lastRecordExtraSize = totalSize % numRecords;
+
+    for (int i = 0; i < numRecords - 1; i++) {
+      writeRecord(minRecordSize, delimiter);
+    }
+    writeRecord(minRecordSize + lastRecordExtraSize, delimiter);
+  }
+
+  public void writeRecord(int totalSize, byte[] delimiter) throws IOException {
+    checkArgument(delimiter.length > 0);
+    checkArgument(totalSize >= delimiter.length);
+
+    int contentSize = totalSize - delimiter.length;
+    for (int i = 0; i < contentSize; i++) {
+      // Alternate between characters so that internals of CBZip2OutputStream
+      // cannot condensed the written bytes using run-length encoding. This
+      // allows the caller to use #BLOCK_SIZE in order to know whether the next
+      // write will end just before the end of the current block, or exceed it,
+      // and by how much.
+      out.write(i % 2 == 0 ? 'a' : 'b');
+    }
+    write(delimiter);
+  }
+
+  public void write(String bytes) throws IOException {
+    write(bytes.getBytes(StandardCharsets.UTF_8));
+  }
+
+  public void write(byte[] bytes) throws IOException {
+    out.write(bytes);
+  }
+
+  @Override
+  public void close() throws IOException {
+    out.close();
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java
new file mode 100644
index 0000000000000..d597ed0e7e0b8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE.BYBLOCK;
+
+public final class BZip2Utils {
+
+  private BZip2Utils() {
+  }
+
+  /**
+   * Returns the start offsets of blocks that follow the first block in the
+   * BZip2 compressed file at the given path. The first offset corresponds to
+   * the first byte containing the BZip2 block marker of the second block. The
+   * i-th offset corresponds to the block marker of the (i + 1)-th block.
+   */
+  public static List<Long> getNextBlockMarkerOffsets(
+      Path path, Configuration conf) throws IOException {
+    FileSystem fs = path.getFileSystem(conf);
+    try (InputStream fileIn = fs.open(path)) {
+      return getNextBlockMarkerOffsets(fileIn);
+    }
+  }
+
+  /**
+   * Returns the start offsets of blocks that follow the first block in the
+   * BZip2 compressed input stream. The first offset corresponds to
+   * the first byte containing the BZip2 block marker of the second block. The
+   * i-th offset corresponds to the block marker of the (i + 1)-th block.
+   */
+  public static List<Long> getNextBlockMarkerOffsets(InputStream rawIn)
+      throws IOException {
+    try (CBZip2InputStream in = new CBZip2InputStream(rawIn, BYBLOCK)) {
+      ArrayList<Long> offsets = new ArrayList<>();
+      while (in.skipToNextBlockMarker()) {
+        offsets.add(in.getProcessedByteCount());
+      }
+      return offsets;
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java
new file mode 100644
index 0000000000000..7d92e07f01b6a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.compress.bzip2;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter.BLOCK_SIZE;
+import static org.junit.Assert.assertEquals;
+
+public final class TestBZip2TextFileWriter {
+
+  private static final byte[] DELIMITER = new byte[] {'\0'};
+
+  private ByteArrayOutputStream rawOut;
+  private BZip2TextFileWriter writer;
+
+  @Before
+  public void setUp() throws Exception {
+    rawOut = new ByteArrayOutputStream();
+    writer = new BZip2TextFileWriter(rawOut);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    rawOut = null;
+    writer.close();
+  }
+
+  @Test
+  public void writingSingleBlockSizeOfData() throws Exception {
+    writer.writeRecord(BLOCK_SIZE, DELIMITER);
+    writer.close();
+
+    List<Long> nextBlocks = getNextBlockMarkerOffsets();
+    assertEquals(0, nextBlocks.size());
+  }
+
+  @Test
+  public void justExceedingBeyondBlockSize() throws Exception {
+    writer.writeRecord(BLOCK_SIZE + 1, DELIMITER);
+    writer.close();
+
+    List<Long> nextBlocks = getNextBlockMarkerOffsets();
+    assertEquals(1, nextBlocks.size());
+  }
+
+  @Test
+  public void writingTwoBlockSizesOfData() throws Exception {
+    writer.writeRecord(2 * BLOCK_SIZE, DELIMITER);
+    writer.close();
+
+    List<Long> nextBlocks = getNextBlockMarkerOffsets();
+    assertEquals(1, nextBlocks.size());
+  }
+
+  @Test
+  public void justExceedingBeyondTwoBlocks() throws Exception {
+    writer.writeRecord(2 * BLOCK_SIZE + 1, DELIMITER);
+    writer.close();
+
+    List<Long> nextBlocks = getNextBlockMarkerOffsets();
+    assertEquals(2, nextBlocks.size());
+  }
+
+  private List<Long> getNextBlockMarkerOffsets() throws IOException {
+    ByteArrayInputStream in = new ByteArrayInputStream(rawOut.toByteArray());
+    return BZip2Utils.getNextBlockMarkerOffsets(in);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java
index c585a463e46b1..fae5ce6de40a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java
@@ -18,9 +18,6 @@
 package org.apache.hadoop.io.compress.bzip2;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.compress.*;
 import org.apache.hadoop.io.compress.bzip2.Bzip2Compressor;
 import org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor;
 import org.apache.hadoop.test.MultithreadedTestUtil;
@@ -32,7 +29,6 @@
 
 import static org.junit.Assert.*;
 import static org.junit.Assume.*;
-import static org.junit.Assume.assumeTrue;
 
 public class TestBzip2CompressorDecompressor {
   
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
index ac9ea5e8a8468..25da4fe2375ed 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java
@@ -28,7 +28,6 @@
 import java.util.zip.DeflaterOutputStream;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.compress.CompressDecompressTester;
 import org.apache.hadoop.io.compress.Compressor;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
index f12226d897d59..d4c0718220a20 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
@@ -231,6 +231,65 @@ public void testCompressorDecompressorLogicWithCompressionStreams()
     }
   }
 
+  /**
+   * Verify decompressor logic with some finish operation in compress.
+   */
+  @Test
+  public void testCompressorDecompressorWithFinish() throws Exception {
+    DataOutputStream deflateOut = null;
+    DataInputStream inflateIn = null;
+    int byteSize = 1024 * 100;
+    byte[] bytes = generate(byteSize);
+    int firstLength = 1024 * 30;
+
+    int bufferSize = IO_FILE_BUFFER_SIZE_DEFAULT;
+    try {
+      DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
+      CompressionOutputStream deflateFilter =
+              new CompressorStream(compressedDataBuffer, new ZStandardCompressor(),
+                      bufferSize);
+
+      deflateOut =
+              new DataOutputStream(new BufferedOutputStream(deflateFilter));
+
+      // Write some data and finish.
+      deflateOut.write(bytes, 0, firstLength);
+      deflateFilter.finish();
+      deflateOut.flush();
+
+      // ResetState then write some data and finish.
+      deflateFilter.resetState();
+      deflateOut.write(bytes, firstLength, firstLength);
+      deflateFilter.finish();
+      deflateOut.flush();
+
+      // ResetState then write some data and finish.
+      deflateFilter.resetState();
+      deflateOut.write(bytes, firstLength * 2, byteSize - firstLength * 2);
+      deflateFilter.finish();
+      deflateOut.flush();
+
+      DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
+      deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
+              compressedDataBuffer.getLength());
+
+      CompressionInputStream inflateFilter =
+              new DecompressorStream(deCompressedDataBuffer,
+                      new ZStandardDecompressor(bufferSize), bufferSize);
+
+      inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
+
+      byte[] result = new byte[byteSize];
+      inflateIn.read(result);
+      assertArrayEquals(
+              "original array not equals compress/decompressed array", bytes,
+              result);
+    } finally {
+      IOUtils.closeStream(deflateOut);
+      IOUtils.closeStream(inflateIn);
+    }
+  }
+
   @Test
   public void testZStandardCompressDecompressInMultiThreads() throws Exception {
     MultithreadedTestUtil.TestContext ctx =
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
index b1bf0774974da..6b4c698551359 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestCompression.java
@@ -17,15 +17,12 @@
  */
 package org.apache.hadoop.io.file.tfile;
 
-import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.*;
 
 import java.io.IOException;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 public class TestCompression {
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
index 48924203f34e7..fdd622a7f6a37 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/file/tfile/TestTFileSeqFileComparison.java
@@ -516,7 +516,7 @@ private static String parameters2String(MyOptions options) {
   }
 
   private static class MyOptions {
-    String rootDir = GenericTestUtils.getTestDir().getAbsolutePath();;
+    String rootDir = GenericTestUtils.getTestDir().getAbsolutePath();
     String compress = "gz";
     String format = "tfile";
     int dictSize = 1000;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index e1fc29f88126b..59b9b13fbff55 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -291,7 +291,7 @@ public void testRetryOtherThanRemoteException() throws Throwable {
 
     UnreliableInterface unreliable = (UnreliableInterface)
         RetryProxy.create(UnreliableInterface.class, unreliableImpl,
-            retryOtherThanRemoteException(TRY_ONCE_THEN_FAIL,
+            retryOtherThanRemoteAndSaslException(TRY_ONCE_THEN_FAIL,
                 exceptionToPolicyMap));
     // should retry with local IOException.
     unreliable.failsOnceWithIOException();
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java
index 2290270bfba1a..70ae639091421 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java
@@ -36,7 +36,6 @@
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
-import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index 6742425e766e8..cca40f97c570b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -289,7 +289,7 @@ public int run(String[] args) throws Exception {
           long cpuNanosClient = getTotalCpuTime(ctx.getTestThreads());
           long cpuNanosServer = -1;
           if (server != null) {
-            cpuNanosServer = getTotalCpuTime(server.getHandlers());; 
+            cpuNanosServer = getTotalCpuTime(server.getHandlers());
           }
           System.out.println("====== Results ======");
           System.out.println("Options:\n" + opts);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
index fc1057b9f766d..bb4a119e7db29 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallerContext.java
@@ -42,6 +42,37 @@ public void testBuilderAppend() {
         builder.build().getContext());
   }
 
+  @Test
+  public void testBuilderAppendIfAbsent() {
+    Configuration conf = new Configuration();
+    conf.set(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, "$");
+    CallerContext.Builder builder = new CallerContext.Builder(null, conf);
+    builder.append("key1", "value1");
+    Assert.assertEquals("key1:value1",
+        builder.build().getContext());
+
+    // Append an existed key with different value.
+    builder.appendIfAbsent("key1", "value2");
+    String[] items = builder.build().getContext().split("\\$");
+    Assert.assertEquals(1, items.length);
+    Assert.assertEquals("key1:value1",
+        builder.build().getContext());
+
+    // Append an absent key.
+    builder.appendIfAbsent("key2", "value2");
+    String[] items2 = builder.build().getContext().split("\\$");
+    Assert.assertEquals(2, items2.length);
+    Assert.assertEquals("key1:value1$key2:value2",
+        builder.build().getContext());
+
+    // Append a key that is a substring of an existing key.
+    builder.appendIfAbsent("key", "value");
+    String[] items3 = builder.build().getContext().split("\\$");
+    Assert.assertEquals(3, items3.length);
+    Assert.assertEquals("key1:value1$key2:value2$key:value",
+        builder.build().getContext());
+  }
+
   @Test(expected = IllegalArgumentException.class)
   public void testNewBuilder() {
     Configuration conf = new Configuration();
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index 95ff302103d89..25c6976549413 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ipc;
 
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -91,8 +92,8 @@
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.util.StringUtils;
+import org.assertj.core.api.Condition;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
@@ -815,6 +816,81 @@ public Void call() throws IOException {
     }
   }
 
+  /**
+   * The {@link ConnectionId#hashCode} has to be stable despite updates that occur as the the
+   * address evolves over time.  The {@link ConnectionId} is used as a primary key in maps, so
+   * its hashCode can't change.
+   *
+   * @throws IOException if there is a client or server failure
+   */
+  @Test
+  public void testStableHashCode() throws IOException {
+    Server server = new TestServer(5, false);
+    try {
+      server.start();
+
+      // Leave host unresolved to start. Use "localhost" as opposed
+      // to local IP from NetUtils.getConnectAddress(server) to force
+      // resolution later
+      InetSocketAddress unresolvedAddr = InetSocketAddress.createUnresolved(
+          "localhost", NetUtils.getConnectAddress(server).getPort());
+
+      // Setup: Create a ConnectionID using an unresolved address, and get it's hashCode to serve
+      // as a point of comparison.
+      int rpcTimeout = MIN_SLEEP_TIME * 2;
+      final ConnectionId remoteId = getConnectionId(unresolvedAddr, rpcTimeout, conf);
+      int expected = remoteId.hashCode();
+
+      // Start client
+      Client.setConnectTimeout(conf, 100);
+      Client client = new Client(LongWritable.class, conf);
+      try {
+        // Test: Call should re-resolve host and succeed
+        LongWritable param = new LongWritable(RANDOM.nextLong());
+        client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId,
+            RPC.RPC_SERVICE_CLASS_DEFAULT, null);
+        int actual = remoteId.hashCode();
+
+        // Verify: The hashCode should match, although the InetAddress is different since it has
+        // now been resolved
+        assertThat(remoteId.getAddress()).isNotEqualTo(unresolvedAddr);
+        assertThat(remoteId.getAddress().getHostName()).isEqualTo(unresolvedAddr.getHostName());
+        assertThat(remoteId.hashCode()).isEqualTo(expected);
+
+        // Test: Call should succeed without having to re-resolve
+        InetSocketAddress expectedSocketAddress = remoteId.getAddress();
+        param = new LongWritable(RANDOM.nextLong());
+        client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId,
+            RPC.RPC_SERVICE_CLASS_DEFAULT, null);
+
+        // Verify: The same instance of the InetSocketAddress has been used to make the second
+        // call
+        assertThat(remoteId.getAddress()).isSameAs(expectedSocketAddress);
+
+        // Verify: The hashCode is protected against updates to the host name
+        String hostName = InetAddress.getLocalHost().getHostName();
+        InetSocketAddress mismatchedHostName = NetUtils.createSocketAddr(
+            InetAddress.getLocalHost().getHostName(),
+            remoteId.getAddress().getPort());
+        assertThatExceptionOfType(IllegalArgumentException.class)
+            .isThrownBy(() -> remoteId.setAddress(mismatchedHostName))
+            .withMessageStartingWith("Hostname must match");
+
+        // Verify: The hashCode is protected against updates to the port
+        InetSocketAddress mismatchedPort = NetUtils.createSocketAddr(
+            remoteId.getAddress().getHostName(),
+            remoteId.getAddress().getPort() + 1);
+        assertThatExceptionOfType(IllegalArgumentException.class)
+            .isThrownBy(() -> remoteId.setAddress(mismatchedPort))
+            .withMessageStartingWith("Port must match");
+      } finally {
+        client.stop();
+      }
+    } finally {
+      server.stop();
+    }
+  }
+
   @Test(timeout=60000)
   public void testIpcFlakyHostResolution() throws IOException {
     // start server
@@ -861,7 +937,6 @@ public void testIpcWithReaderQueuing() throws Exception {
   // goal is to jam a handler with a connection, fill the callq with
   // connections, in turn jamming the readers - then flood the server and
   // ensure that the listener blocks when the reader connection queues fill
-  @SuppressWarnings("unchecked")
   private void checkBlocking(int readers, int readerQ, int callQ) throws Exception {
     int handlers = 1; // makes it easier
     
@@ -881,9 +956,8 @@ private void checkBlocking(int readers, int readerQ, int callQ) throws Exception
     // start server
     final TestServerQueue server =
         new TestServerQueue(clients, readers, callQ, handlers, conf);
-    CallQueueManager<Call> spy = spy(
-        (CallQueueManager<Call>)Whitebox.getInternalState(server, "callQueue"));
-    Whitebox.setInternalState(server, "callQueue", spy);
+    CallQueueManager<Call> spy = spy(server.getCallQueue());
+    server.setCallQueue(spy);
     final InetSocketAddress addr = NetUtils.getConnectAddress(server);
     server.start();
 
@@ -1142,11 +1216,6 @@ public void testSocketLeak() throws IOException {
   @Test(timeout=30000)
   public void testInterrupted() {
     Client client = new Client(LongWritable.class, conf);
-    Client.getClientExecutor().submit(new Runnable() {
-      public void run() {
-        while(true);
-      }
-    });
     Thread.currentThread().interrupt();
     client.stop();
     try {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index b78900b609e54..084a3dbd4ae5d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.ipc;
 
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.retry.RetryUtils;
+import org.apache.hadoop.ipc.metrics.RpcMetrics;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
@@ -47,10 +52,10 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.MetricsAsserts;
 import org.apache.hadoop.test.MockitoUtil;
-import org.apache.hadoop.test.Whitebox;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.event.Level;
@@ -58,13 +63,16 @@
 import javax.net.SocketFactory;
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InterruptedIOException;
+import java.io.OutputStream;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.net.SocketTimeoutException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedAction;
@@ -83,7 +91,9 @@
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
@@ -285,6 +295,14 @@ public <T> ProtocolProxy<T> getProxy(
           rpcTimeout, connectionRetryPolicy, null, null);
     }
 
+    @Override
+    public <T> ProtocolProxy<T> getProxy(Class<T> protocol, long clientVersion,
+        ConnectionId connId, Configuration conf, SocketFactory factory,
+        AlignmentContext alignmentContext)
+        throws IOException {
+      throw new UnsupportedOperationException("This proxy is not supported");
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public <T> ProtocolProxy<T> getProxy(
@@ -295,7 +313,7 @@ public <T> ProtocolProxy<T> getProxy(
         throws IOException {
       T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(),
           new Class[] { protocol }, new StoppedInvocationHandler());
-      return new ProtocolProxy<T>(protocol, proxy, false);
+      return new ProtocolProxy<>(protocol, proxy, false);
     }
 
     @Override
@@ -386,6 +404,53 @@ public void testProxyAddress() throws Exception {
     }
   }
 
+  @Test
+  public void testConnectionWithSocketFactory() throws IOException, ServiceException {
+    TestRpcService firstProxy = null;
+    TestRpcService secondProxy = null;
+
+    Configuration newConf = new Configuration(conf);
+    newConf.set(CommonConfigurationKeysPublic.
+        HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, "");
+
+    RetryPolicy retryPolicy = RetryUtils.getDefaultRetryPolicy(
+        newConf, "Test.No.Such.Key",
+        true,
+        "Test.No.Such.Key", "10000,6",
+        null);
+
+    // create a server with two handlers
+    Server server = setupTestServer(newConf, 2);
+    try {
+      // create the first client
+      firstProxy = getClient(addr, newConf);
+      // create the second client
+      secondProxy = getClient(addr, newConf);
+
+      firstProxy.ping(null, newEmptyRequest());
+      secondProxy.ping(null, newEmptyRequest());
+
+      Client client = ProtobufRpcEngine2.getClient(newConf);
+      assertEquals(1, client.getConnectionIds().size());
+
+      stop(null, firstProxy, secondProxy);
+      ProtobufRpcEngine2.clearClientCache();
+
+      // create the first client with index 1
+      firstProxy = getMultipleClientWithIndex(addr, newConf, retryPolicy, 1);
+      // create the second client with index 2
+      secondProxy = getMultipleClientWithIndex(addr, newConf, retryPolicy, 2);
+      firstProxy.ping(null, newEmptyRequest());
+      secondProxy.ping(null, newEmptyRequest());
+
+      Client client2 = ProtobufRpcEngine2.getClient(newConf);
+      assertEquals(2, client2.getConnectionIds().size());
+    } finally {
+      System.out.println("Down slow rpc testing");
+      stop(server, firstProxy, secondProxy);
+    }
+  }
+
   @Test
   public void testSlowRpc() throws IOException, ServiceException {
     Server server;
@@ -933,6 +998,196 @@ public void run() {
     }
   }
 
+  /**
+   * This tests the case where the server isn't receiving new data and
+   * multiple threads queue up to send rpc requests. Only one of the requests
+   * should be written and all of the calling threads should be interrupted.
+   *
+   * We use a mock SocketFactory so that we can control when the input and
+   * output streams are frozen.
+   */
+  @Test(timeout=30000)
+  public void testSlowConnection() throws Exception {
+    SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
+    Socket mockSocket = Mockito.mock(Socket.class);
+    Mockito.when(mockFactory.createSocket()).thenReturn(mockSocket);
+    Mockito.when(mockSocket.getPort()).thenReturn(1234);
+    Mockito.when(mockSocket.getLocalPort()).thenReturn(2345);
+    MockOutputStream mockOutputStream = new MockOutputStream();
+    Mockito.when(mockSocket.getOutputStream()).thenReturn(mockOutputStream);
+    // Use an input stream that always blocks
+    Mockito.when(mockSocket.getInputStream()).thenReturn(new InputStream() {
+      @Override
+      public int read() throws IOException {
+        // wait forever
+        while (true) {
+          try {
+            Thread.sleep(TimeUnit.DAYS.toMillis(1));
+          } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+            throw new InterruptedIOException("test");
+          }
+        }
+      }
+    });
+    Configuration clientConf = new Configuration();
+    // disable ping & timeout to minimize traffic
+    clientConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, false);
+    clientConf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 0);
+    RPC.setProtocolEngine(clientConf, TestRpcService.class, ProtobufRpcEngine.class);
+    // set async mode so that we don't need to implement the input stream
+    final boolean wasAsync = Client.isAsynchronousMode();
+    TestRpcService client = null;
+    try {
+      Client.setAsynchronousMode(true);
+      client = RPC.getProtocolProxy(
+          TestRpcService.class,
+          0,
+          new InetSocketAddress("localhost", 1234),
+          UserGroupInformation.getCurrentUser(),
+          clientConf,
+          mockFactory).getProxy();
+      // The connection isn't actually made until the first call.
+      client.ping(null, newEmptyRequest());
+      mockOutputStream.waitForFlush(1);
+      final long headerAndFirst = mockOutputStream.getBytesWritten();
+      client.ping(null, newEmptyRequest());
+      mockOutputStream.waitForFlush(2);
+      final long second = mockOutputStream.getBytesWritten() - headerAndFirst;
+      // pause the writer thread
+      mockOutputStream.pause();
+      // create a set of threads to create calls that will back up
+      ExecutorService pool = Executors.newCachedThreadPool();
+      Future[] futures = new Future[numThreads];
+      final AtomicInteger doneThreads = new AtomicInteger(0);
+      for(int thread = 0; thread < numThreads; ++thread) {
+        final TestRpcService finalClient = client;
+        futures[thread] = pool.submit(new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            finalClient.ping(null, newEmptyRequest());
+            doneThreads.incrementAndGet();
+            return null;
+          }
+        });
+      }
+      // wait until the threads have started writing
+      mockOutputStream.waitForWriters();
+      // interrupt all the threads
+      for(int thread=0; thread < numThreads; ++thread) {
+        assertTrue("cancel thread " + thread,
+            futures[thread].cancel(true));
+      }
+      // wait until all the writers are cancelled
+      pool.shutdown();
+      pool.awaitTermination(10, TimeUnit.SECONDS);
+      mockOutputStream.resume();
+      // wait for the in flight rpc request to be flushed
+      mockOutputStream.waitForFlush(3);
+      // All the threads should have been interrupted
+      assertEquals(0, doneThreads.get());
+      // make sure that only one additional rpc request was sent
+      assertEquals(headerAndFirst + second * 2,
+          mockOutputStream.getBytesWritten());
+    } finally {
+      Client.setAsynchronousMode(wasAsync);
+      if (client != null) {
+        RPC.stopProxy(client);
+      }
+    }
+  }
+
+  private static final class MockOutputStream extends OutputStream {
+    private long bytesWritten = 0;
+    private AtomicInteger flushCount = new AtomicInteger(0);
+    private ReentrantLock lock = new ReentrantLock(true);
+
+    @Override
+    public synchronized void write(int b) throws IOException {
+      lock.lock();
+      bytesWritten += 1;
+      lock.unlock();
+    }
+
+    @Override
+    public void flush() {
+      flushCount.incrementAndGet();
+    }
+
+    public synchronized long getBytesWritten() {
+      return bytesWritten;
+    }
+
+    public void pause() {
+      lock.lock();
+    }
+
+    public void resume() {
+      lock.unlock();
+    }
+
+    private static final int DELAY_MS = 250;
+
+    /**
+     * Wait for the Nth flush, which we assume will happen exactly when the
+     * Nth RPC request is sent.
+     * @param flush the total flush count to wait for
+     * @throws InterruptedException
+     */
+    public void waitForFlush(int flush) throws InterruptedException {
+      while (flushCount.get() < flush) {
+        Thread.sleep(DELAY_MS);
+      }
+    }
+
+    public void waitForWriters() throws InterruptedException {
+      while (!lock.hasQueuedThreads()) {
+        Thread.sleep(DELAY_MS);
+      }
+    }
+  }
+
+  /**
+   * This test causes an exception in the RPC connection setup to make
+   * sure that threads aren't leaked.
+   */
+  @Test(timeout=30000)
+  public void testBadSetup() throws Exception {
+    SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
+    Mockito.when(mockFactory.createSocket())
+        .thenThrow(new IOException("can't connect"));
+    Configuration clientConf = new Configuration();
+    // Set an illegal value to cause an exception in the constructor
+    clientConf.set(CommonConfigurationKeys.IPC_MAXIMUM_RESPONSE_LENGTH,
+        "xxx");
+    RPC.setProtocolEngine(clientConf, TestRpcService.class,
+        ProtobufRpcEngine.class);
+    TestRpcService client = null;
+    int threadCount = Thread.getAllStackTraces().size();
+    try {
+      try {
+        client = RPC.getProtocolProxy(
+            TestRpcService.class,
+            0,
+            new InetSocketAddress("localhost", 1234),
+            UserGroupInformation.getCurrentUser(),
+            clientConf,
+            mockFactory).getProxy();
+        client.ping(null, newEmptyRequest());
+        assertTrue("Didn't throw exception!", false);
+      } catch (ServiceException nfe) {
+        // ensure no extra threads are running.
+        assertEquals(threadCount, Thread.getAllStackTraces().size());
+      } catch (Throwable t) {
+        assertTrue("wrong exception: " + t, false);
+      }
+    } finally {
+      if (client != null) {
+        RPC.stopProxy(client);
+      }
+    }
+  }
+
   @Test
   public void testConnectionPing() throws Exception {
     Server server;
@@ -1107,6 +1362,37 @@ public TestRpcService run() {
     }
   }
 
+  @Test
+  public void testNumInProcessHandlerMetrics() throws Exception {
+    UserGroupInformation ugi = UserGroupInformation.
+        createUserForTesting("user123", new String[0]);
+    // use 1 handler so the callq can be plugged
+    final Server server = setupTestServer(conf, 1);
+    try {
+      RpcMetrics rpcMetrics = server.getRpcMetrics();
+      assertEquals(0, rpcMetrics.getNumInProcessHandler());
+
+      ExternalCall<String> call1 = newExtCall(ugi, () -> {
+        assertEquals(1, rpcMetrics.getNumInProcessHandler());
+        return UserGroupInformation.getCurrentUser().getUserName();
+      });
+      ExternalCall<Void> call2 = newExtCall(ugi, () -> {
+        assertEquals(1, rpcMetrics.getNumInProcessHandler());
+        return null;
+      });
+
+      server.queueCall(call1);
+      server.queueCall(call2);
+
+      // Wait for call1 and call2 to enter the handler.
+      call1.get();
+      call2.get();
+      assertEquals(0, rpcMetrics.getNumInProcessHandler());
+    } finally {
+      server.stop();
+    }
+  }
+
   /**
    *  Test RPC backoff by queue full.
    */
@@ -1127,10 +1413,8 @@ public void testClientBackOff() throws Exception {
         .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
     server = setupTestServer(builder);
 
-    @SuppressWarnings("unchecked")
-    CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox
-        .getInternalState(server, "callQueue"));
-    Whitebox.setInternalState(server, "callQueue", spy);
+    CallQueueManager<Call> spy = spy(server.getCallQueue());
+    server.setCallQueue(spy);
 
     Exception lastException = null;
     proxy = getClient(addr, conf);
@@ -1182,7 +1466,7 @@ public void testClientBackOffByResponseTime() throws Exception {
     GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG);
     GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG);
 
-    final List<Future<Void>> res = new ArrayList<Future<Void>>();
+    final List<Future<Void>> res = new ArrayList<>();
     final ExecutorService executorService =
         Executors.newFixedThreadPool(numClients);
     conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
@@ -1190,10 +1474,8 @@ public void testClientBackOffByResponseTime() throws Exception {
     final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0";
     Server server = setupDecayRpcSchedulerandTestServer(ns + ".");
 
-    @SuppressWarnings("unchecked")
-    CallQueueManager<Call> spy = spy((CallQueueManager<Call>) Whitebox
-        .getInternalState(server, "callQueue"));
-    Whitebox.setInternalState(server, "callQueue", spy);
+    CallQueueManager<Call> spy = spy(server.getCallQueue());
+    server.setCallQueue(spy);
 
     Exception lastException = null;
     proxy = getClient(addr, conf);
@@ -1532,11 +1814,8 @@ public RpcStatusProto getRpcStatusProto() {
       RPC.Builder builder = newServerBuilder(conf)
           .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true);
       server = setupTestServer(builder);
-      Whitebox.setInternalState(
-          server, "rpcRequestClass", FakeRequestClass.class);
-      MutableCounterLong authMetric =
-          (MutableCounterLong)Whitebox.getInternalState(
-              server.getRpcMetrics(), "rpcAuthorizationSuccesses");
+      server.setRpcRequestClass(FakeRequestClass.class);
+      MutableCounterLong authMetric = server.getRpcMetrics().getRpcAuthorizationSuccesses();
 
       proxy = getClient(addr, conf);
       boolean isDisconnected = true;
@@ -1665,6 +1944,61 @@ public void testRpcMetricsInNanos() throws Exception {
     }
   }
 
+  @Test
+  public void testNumTotalRequestsMetrics() throws Exception {
+    UserGroupInformation ugi = UserGroupInformation.
+        createUserForTesting("userXyz", new String[0]);
+
+    final Server server = setupTestServer(conf, 1);
+
+    ExecutorService executorService = null;
+    try {
+      RpcMetrics rpcMetrics = server.getRpcMetrics();
+      assertEquals(0, rpcMetrics.getTotalRequests());
+      assertEquals(0, rpcMetrics.getTotalRequestsPerSecond());
+
+      List<ExternalCall<Void>> externalCallList = new ArrayList<>();
+
+      executorService = Executors.newSingleThreadExecutor(
+          new ThreadFactoryBuilder().setDaemon(true).setNameFormat("testNumTotalRequestsMetrics")
+              .build());
+      AtomicInteger rps = new AtomicInteger(0);
+      CountDownLatch countDownLatch = new CountDownLatch(1);
+      executorService.submit(() -> {
+        while (true) {
+          int numRps = (int) rpcMetrics.getTotalRequestsPerSecond();
+          rps.getAndSet(numRps);
+          if (rps.get() > 0) {
+            countDownLatch.countDown();
+            break;
+          }
+        }
+      });
+
+      for (int i = 0; i < 100000; i++) {
+        externalCallList.add(newExtCall(ugi, () -> null));
+      }
+      for (ExternalCall<Void> externalCall : externalCallList) {
+        server.queueCall(externalCall);
+      }
+      for (ExternalCall<Void> externalCall : externalCallList) {
+        externalCall.get();
+      }
+
+      assertEquals(100000, rpcMetrics.getTotalRequests());
+      if (countDownLatch.await(10, TimeUnit.SECONDS)) {
+        assertTrue(rps.get() > 10);
+      } else {
+        throw new AssertionError("total requests per seconds are still 0");
+      }
+    } finally {
+      if (executorService != null) {
+        executorService.shutdown();
+      }
+      server.stop();
+    }
+  }
+
 
   public static void main(String[] args) throws Exception {
     new TestRPC().testCallsInternal(conf);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java
index 64607deb908fa..b789ada5271ff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java
@@ -50,14 +50,14 @@ public void setup() {
   static class TestServer {
     AtomicInteger retryCount = new AtomicInteger();
     AtomicInteger operationCount = new AtomicInteger();
-    private RetryCache retryCache = new RetryCache("TestRetryCache", 1,
-        100 * 1000 * 1000 * 1000L);
+    private final RetryCache retryCache = new RetryCache(
+        "TestRetryCache", 1, 100 * 1000 * 1000 * 1000L);
 
     /**
      * A server method implemented using {@link RetryCache}.
      * 
      * @param input is returned back in echo, if {@code success} is true.
-     * @param failureOuput returned on failure, if {@code success} is false.
+     * @param failureOutput returned on failure, if {@code success} is false.
      * @param methodTime time taken by the operation. By passing smaller/larger
      *          value one can simulate an operation that takes short/long time.
      * @param success whether this operation completes successfully or not
@@ -67,7 +67,7 @@ static class TestServer {
     int echo(int input, int failureOutput, long methodTime, boolean success)
         throws InterruptedException {
       CacheEntryWithPayload entry = RetryCache.waitForCompletion(retryCache,
-          null);
+          null, Server.getClientId(), Server.getCallId());
       if (entry != null && entry.isSuccess()) {
         System.out.println("retryCount incremented " + retryCount.get());
         retryCount.incrementAndGet();
@@ -173,16 +173,13 @@ public void testOperations(final int input, final int numberOfThreads,
     final int failureOutput = input + 1;
     ExecutorService executorService = Executors
         .newFixedThreadPool(numberOfThreads);
-    List<Future<Integer>> list = new ArrayList<Future<Integer>>();
+    List<Future<Integer>> list = new ArrayList<>();
     for (int i = 0; i < numberOfThreads; i++) {
-      Callable<Integer> worker = new Callable<Integer>() {
-        @Override
-        public Integer call() throws Exception {
-          Server.getCurCall().set(call);
-          Assert.assertEquals(Server.getCurCall().get(), call);
-          int randomPause = pause == 0 ? pause : r.nextInt(pause);
-          return testServer.echo(input, failureOutput, randomPause, success);
-        }
+      Callable<Integer> worker = () -> {
+        Server.getCurCall().set(call);
+        Assert.assertEquals(Server.getCurCall().get(), call);
+        int randomPause = pause == 0 ? pause : r.nextInt(pause);
+        return testServer.echo(input, failureOutput, randomPause, success);
       };
       Future<Integer> submit = executorService.submit(worker);
       list.add(submit);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index 0962b50099c57..5b5c8bbaa9b73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -18,13 +18,17 @@
 
 package org.apache.hadoop.ipc;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.thirdparty.protobuf.BlockingService;
 import org.apache.hadoop.thirdparty.protobuf.RpcController;
 import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Client.ConnectionId;
 import org.apache.hadoop.ipc.protobuf.TestProtos;
 import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
 import org.apache.hadoop.net.NetUtils;
@@ -124,18 +128,19 @@ protected static RPC.Server setupTestServer(
     return server;
   }
 
-  protected static TestRpcService getClient(InetSocketAddress serverAddr,
-                                     Configuration clientConf)
+  protected static TestRpcService getClient(InetSocketAddress serverAddr, Configuration clientConf)
       throws ServiceException {
-    try {
-      return RPC.getProxy(TestRpcService.class, 0, serverAddr, clientConf);
-    } catch (IOException e) {
-      throw new ServiceException(e);
-    }
+    return getClient(serverAddr, clientConf, null);
   }
 
   protected static TestRpcService getClient(InetSocketAddress serverAddr,
-      Configuration clientConf, final RetryPolicy connectionRetryPolicy)
+      Configuration clientConf, RetryPolicy connectionRetryPolicy) throws ServiceException {
+    return getClient(serverAddr, clientConf, connectionRetryPolicy, null);
+  }
+
+  protected static TestRpcService getClient(InetSocketAddress serverAddr,
+      Configuration clientConf, final RetryPolicy connectionRetryPolicy,
+      AtomicBoolean fallbackToSimpleAuth)
       throws ServiceException {
     try {
       return RPC.getProtocolProxy(
@@ -146,17 +151,60 @@ protected static TestRpcService getClient(InetSocketAddress serverAddr,
           clientConf,
           NetUtils.getDefaultSocketFactory(clientConf),
           RPC.getRpcTimeout(clientConf),
-          connectionRetryPolicy, null).getProxy();
+          connectionRetryPolicy, fallbackToSimpleAuth).getProxy();
     } catch (IOException e) {
       throw new ServiceException(e);
     }
   }
 
-  protected static void stop(Server server, TestRpcService proxy) {
-    if (proxy != null) {
-      try {
-        RPC.stopProxy(proxy);
-      } catch (Exception ignored) {}
+  /**
+   * Try to obtain a proxy of TestRpcService with an index.
+   * @param serverAddr input server address
+   * @param clientConf input client configuration
+   * @param retryPolicy input retryPolicy
+   * @param index input index
+   * @return one proxy of TestRpcService
+   */
+  protected static TestRpcService getMultipleClientWithIndex(InetSocketAddress serverAddr,
+      Configuration clientConf, RetryPolicy retryPolicy, int index)
+      throws ServiceException, IOException {
+    MockConnectionId connectionId = new MockConnectionId(serverAddr,
+        TestRpcService.class, UserGroupInformation.getCurrentUser(),
+        RPC.getRpcTimeout(clientConf), retryPolicy, clientConf, index);
+    return getClient(connectionId, clientConf);
+  }
+
+  /**
+   * Obtain a TestRpcService Proxy by a connectionId.
+   * @param connId input connectionId
+   * @param clientConf  input configuration
+   * @return a TestRpcService Proxy
+   * @throws ServiceException a ServiceException
+   */
+  protected static TestRpcService getClient(ConnectionId connId,
+      Configuration clientConf) throws ServiceException {
+    try {
+      return RPC.getProtocolProxy(
+          TestRpcService.class,
+          0,
+          connId,
+          clientConf,
+          NetUtils.getDefaultSocketFactory(clientConf),
+          null).getProxy();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  protected static void stop(Server server, TestRpcService... proxies) {
+    if (proxies != null) {
+      for (TestRpcService proxy : proxies) {
+        if (proxy != null) {
+          try {
+            RPC.stopProxy(proxy);
+          } catch (Exception ignored) {}
+        }
+      }
     }
 
     if (server != null) {
@@ -187,6 +235,40 @@ protected static int countThreads(String search) {
     return count;
   }
 
+  public static class MockConnectionId extends ConnectionId {
+    private static final int PRIME = 16777619;
+    private final int index;
+
+    public MockConnectionId(InetSocketAddress address, Class<?> protocol,
+        UserGroupInformation ticket, int rpcTimeout, RetryPolicy connectionRetryPolicy,
+        Configuration conf, int index) {
+      super(address, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf);
+      this.index = index;
+    }
+
+    @Override
+    public int hashCode() {
+      return new HashCodeBuilder()
+          .append(PRIME * super.hashCode())
+          .append(this.index)
+          .toHashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!super.equals(obj)) {
+        return false;
+      }
+      if (obj instanceof MockConnectionId) {
+        MockConnectionId other = (MockConnectionId)obj;
+        return new EqualsBuilder()
+            .append(this.index, other.index)
+            .isEquals();
+      }
+      return false;
+    }
+  }
+
   public static class TestTokenIdentifier extends TokenIdentifier {
     private Text tokenid;
     private Text realUser;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 72085a19ec711..662faea599648 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -72,6 +72,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
@@ -569,6 +570,72 @@ public void testSimpleServer() throws Exception {
     assertAuthEquals(SIMPLE,    getAuthMethod(KERBEROS, SIMPLE, UseToken.OTHER));
   }
 
+  /**
+   * In DfsClient there is a fallback mechanism to simple auth, which passes in an atomic boolean
+   * to the ipc Client, which then sets it during setupIOStreams.
+   * SetupIOStreams were running only once per connection, so if two separate DfsClient was
+   * instantiated, then due to the connection caching inside the ipc client, the second DfsClient
+   * did not have the passed in atomic boolean set properly if the first client was not yet closed,
+   * as setupIOStreams was yielding to set up new streams as it has reused the already existing
+   * connection.
+   * This test mimics this behaviour, and asserts the fallback whether it is set correctly.
+   * @see <a href="https://issues.apache.org/jira/browse/HADOOP-17975">HADOOP-17975</a>
+   */
+  @Test
+  public void testClientFallbackToSimpleAuthForASecondClient() throws Exception {
+    Configuration serverConf = createConfForAuth(SIMPLE);
+    Server server = startServer(serverConf,
+        setupServerUgi(SIMPLE, serverConf),
+        createServerSecretManager(SIMPLE, new TestTokenSecretManager()));
+    final InetSocketAddress serverAddress = NetUtils.getConnectAddress(server);
+
+    clientFallBackToSimpleAllowed = true;
+    Configuration clientConf = createConfForAuth(KERBEROS);
+    UserGroupInformation clientUgi = setupClientUgi(KERBEROS, clientConf);
+
+    AtomicBoolean fallbackToSimpleAuth1 = new AtomicBoolean();
+    AtomicBoolean fallbackToSimpleAuth2 = new AtomicBoolean();
+    try {
+      LOG.info("trying ugi:"+ clientUgi +" tokens:"+ clientUgi.getTokens());
+      clientUgi.doAs((PrivilegedExceptionAction<Void>) () -> {
+        TestRpcService proxy1 = null;
+        TestRpcService proxy2 = null;
+        try {
+          proxy1 = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth1);
+          proxy1.ping(null, newEmptyRequest());
+          // make sure the other side thinks we are who we said we are!!!
+          assertEquals(clientUgi.getUserName(),
+              proxy1.getAuthUser(null, newEmptyRequest()).getUser());
+          AuthMethod authMethod =
+              convert(proxy1.getAuthMethod(null, newEmptyRequest()));
+          assertAuthEquals(SIMPLE, authMethod.toString());
+
+          proxy2 = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth2);
+          proxy2.ping(null, newEmptyRequest());
+          // make sure the other side thinks we are who we said we are!!!
+          assertEquals(clientUgi.getUserName(),
+              proxy2.getAuthUser(null, newEmptyRequest()).getUser());
+          AuthMethod authMethod2 =
+              convert(proxy2.getAuthMethod(null, newEmptyRequest()));
+          assertAuthEquals(SIMPLE, authMethod2.toString());
+        } finally {
+          if (proxy1 != null) {
+            RPC.stopProxy(proxy1);
+          }
+          if (proxy2 != null) {
+            RPC.stopProxy(proxy2);
+          }
+        }
+        return null;
+      });
+    } finally {
+      server.stop();
+    }
+
+    assertTrue("First client does not set to fall back properly.", fallbackToSimpleAuth1.get());
+    assertTrue("Second client does not set to fall back properly.", fallbackToSimpleAuth2.get());
+  }
+
   @Test
   public void testNoClientFallbackToSimple()
       throws Exception {
@@ -815,22 +882,44 @@ private String getAuthMethod(
       return e.toString();
     }
   }
-  
+
   private String internalGetAuthMethod(
       final AuthMethod clientAuth,
       final AuthMethod serverAuth,
       final UseToken tokenType) throws Exception {
-    
-    final Configuration serverConf = new Configuration(conf);
-    serverConf.set(HADOOP_SECURITY_AUTHENTICATION, serverAuth.toString());
-    UserGroupInformation.setConfiguration(serverConf);
-    
-    final UserGroupInformation serverUgi = (serverAuth == KERBEROS)
-        ? UserGroupInformation.createRemoteUser("server/localhost@NONE")
-        : UserGroupInformation.createRemoteUser("server");
-    serverUgi.setAuthenticationMethod(serverAuth);
 
     final TestTokenSecretManager sm = new TestTokenSecretManager();
+
+    Configuration serverConf = createConfForAuth(serverAuth);
+    Server server = startServer(
+        serverConf,
+        setupServerUgi(serverAuth, serverConf),
+        createServerSecretManager(serverAuth, sm));
+    final InetSocketAddress serverAddress = NetUtils.getConnectAddress(server);
+
+    final Configuration clientConf = createConfForAuth(clientAuth);
+    final UserGroupInformation clientUgi = setupClientUgi(clientAuth, clientConf);
+
+    setupTokenIfNeeded(tokenType, sm, clientUgi, serverAddress);
+
+    try {
+      return createClientAndQueryAuthMethod(serverAddress, clientConf, clientUgi, null);
+    } finally {
+      server.stop();
+    }
+  }
+
+  private Configuration createConfForAuth(AuthMethod clientAuth) {
+    final Configuration clientConf = new Configuration(conf);
+    clientConf.set(HADOOP_SECURITY_AUTHENTICATION, clientAuth.toString());
+    clientConf.setBoolean(
+        CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
+        clientFallBackToSimpleAllowed);
+    return clientConf;
+  }
+
+  private SecretManager<?> createServerSecretManager(
+      AuthMethod serverAuth, TestTokenSecretManager sm) {
     boolean useSecretManager = (serverAuth != SIMPLE);
     if (enableSecretManager != null) {
       useSecretManager &= enableSecretManager;
@@ -839,26 +928,43 @@ private String internalGetAuthMethod(
       useSecretManager |= forceSecretManager;
     }
     final SecretManager<?> serverSm = useSecretManager ? sm : null;
+    return serverSm;
+  }
 
+  private Server startServer(Configuration serverConf, UserGroupInformation serverUgi,
+      SecretManager<?> serverSm) throws IOException, InterruptedException {
     Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
       @Override
       public Server run() throws IOException {
         return setupTestServer(serverConf, 5, serverSm);
       }
     });
+    return server;
+  }
 
-    final Configuration clientConf = new Configuration(conf);
-    clientConf.set(HADOOP_SECURITY_AUTHENTICATION, clientAuth.toString());
-    clientConf.setBoolean(
-        CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
-        clientFallBackToSimpleAllowed);
+  private UserGroupInformation setupServerUgi(AuthMethod serverAuth,
+      Configuration serverConf) {
+    UserGroupInformation.setConfiguration(serverConf);
+
+    final UserGroupInformation serverUgi = (serverAuth == KERBEROS)
+        ? UserGroupInformation.createRemoteUser("server/localhost@NONE")
+        : UserGroupInformation.createRemoteUser("server");
+    serverUgi.setAuthenticationMethod(serverAuth);
+    return serverUgi;
+  }
+
+  private UserGroupInformation setupClientUgi(AuthMethod clientAuth,
+      Configuration clientConf) {
     UserGroupInformation.setConfiguration(clientConf);
-    
+
     final UserGroupInformation clientUgi =
         UserGroupInformation.createRemoteUser("client");
-    clientUgi.setAuthenticationMethod(clientAuth);    
+    clientUgi.setAuthenticationMethod(clientAuth);
+    return clientUgi;
+  }
 
-    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+  private void setupTokenIfNeeded(UseToken tokenType, TestTokenSecretManager sm,
+      UserGroupInformation clientUgi, InetSocketAddress addr) {
     if (tokenType != UseToken.NONE) {
       TestTokenIdentifier tokenId = new TestTokenIdentifier(
           new Text(clientUgi.getUserName()));
@@ -881,44 +987,44 @@ public Server run() throws IOException {
       }
       clientUgi.addToken(token);
     }
+  }
 
-    try {
-      LOG.info("trying ugi:"+clientUgi+" tokens:"+clientUgi.getTokens());
-      return clientUgi.doAs(new PrivilegedExceptionAction<String>() {
-        @Override
-        public String run() throws IOException {
-          TestRpcService proxy = null;
-          try {
-            proxy = getClient(addr, clientConf);
-
-            proxy.ping(null, newEmptyRequest());
-            // make sure the other side thinks we are who we said we are!!!
-            assertEquals(clientUgi.getUserName(),
-                proxy.getAuthUser(null, newEmptyRequest()).getUser());
-            AuthMethod authMethod =
-                convert(proxy.getAuthMethod(null, newEmptyRequest()));
-            // verify sasl completed with correct QOP
-            assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
-                         RPC.getConnectionIdForProxy(proxy).getSaslQop());
-            return authMethod != null ? authMethod.toString() : null;
-          } catch (ServiceException se) {
-            if (se.getCause() instanceof RemoteException) {
-              throw (RemoteException) se.getCause();
-            } else if (se.getCause() instanceof IOException) {
-              throw (IOException) se.getCause();
-            } else {
-              throw new RuntimeException(se.getCause());
-            }
-          } finally {
-            if (proxy != null) {
-              RPC.stopProxy(proxy);
-            }
+  private String createClientAndQueryAuthMethod(InetSocketAddress serverAddress,
+      Configuration clientConf, UserGroupInformation clientUgi, AtomicBoolean fallbackToSimpleAuth)
+      throws IOException, InterruptedException {
+    LOG.info("trying ugi:"+ clientUgi +" tokens:"+ clientUgi.getTokens());
+    return clientUgi.doAs(new PrivilegedExceptionAction<String>() {
+      @Override
+      public String run() throws IOException {
+        TestRpcService proxy = null;
+        try {
+          proxy = getClient(serverAddress, clientConf, null, fallbackToSimpleAuth);
+
+          proxy.ping(null, newEmptyRequest());
+          // make sure the other side thinks we are who we said we are!!!
+          assertEquals(clientUgi.getUserName(),
+              proxy.getAuthUser(null, newEmptyRequest()).getUser());
+          AuthMethod authMethod =
+              convert(proxy.getAuthMethod(null, newEmptyRequest()));
+          // verify sasl completed with correct QOP
+          assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
+              RPC.getConnectionIdForProxy(proxy).getSaslQop());
+          return authMethod != null ? authMethod.toString() : null;
+        } catch (ServiceException se) {
+          if (se.getCause() instanceof RemoteException) {
+            throw (RemoteException) se.getCause();
+          } else if (se.getCause() instanceof IOException) {
+            throw (IOException) se.getCause();
+          } else {
+            throw new RuntimeException(se.getCause());
+          }
+        } finally {
+          if (proxy != null) {
+            RPC.stopProxy(proxy);
           }
         }
-      });
-    } finally {
-      server.stop();
-    }
+      }
+    });
   }
 
   private static void assertAuthEquals(AuthMethod expect,
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
deleted file mode 100644
index 743080acd7a5e..0000000000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metrics2.impl;
-
-import org.apache.hadoop.metrics2.AbstractMetric;
-import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsTag;
-import org.apache.hadoop.metrics2.sink.GraphiteSink;
-import org.apache.hadoop.test.Whitebox;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.Collections;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.reset;
-
-
-public class TestGraphiteMetrics {
-    private AbstractMetric makeMetric(String name, Number value) {
-        AbstractMetric metric = mock(AbstractMetric.class);
-        when(metric.name()).thenReturn(name);
-        when(metric.value()).thenReturn(value);
-        return metric;
-    }
-
-    private GraphiteSink.Graphite makeGraphite() {
-      GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class);
-      when(mockGraphite.isConnected()).thenReturn(true);
-      return mockGraphite;
-    }
-
-    @Test
-    public void testPutMetrics() {
-        GraphiteSink sink = new GraphiteSink();
-        List<MetricsTag> tags = new ArrayList<MetricsTag>();
-        tags.add(new MetricsTag(MsInfo.Context, "all"));
-        tags.add(new MetricsTag(MsInfo.Hostname, "host"));
-        Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
-        metrics.add(makeMetric("foo1", 1.25));
-        metrics.add(makeMetric("foo2", 2.25));
-        MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
-
-        ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
-        final GraphiteSink.Graphite mockGraphite = makeGraphite();
-        Whitebox.setInternalState(sink, "graphite", mockGraphite);
-        sink.putMetrics(record);
-
-        try {
-          verify(mockGraphite).write(argument.capture());
-        } catch (IOException e) {
-          e.printStackTrace();
-        }
-
-        String result = argument.getValue();
-
-        assertEquals(true,
-            result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
-            "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") ||
-            result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + 
-            "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"));
-    }
-
-    @Test
-    public void testPutMetrics2() {
-        GraphiteSink sink = new GraphiteSink();
-        List<MetricsTag> tags = new ArrayList<MetricsTag>();
-        tags.add(new MetricsTag(MsInfo.Context, "all"));
-      tags.add(new MetricsTag(MsInfo.Hostname, null));
-        Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
-        metrics.add(makeMetric("foo1", 1));
-        metrics.add(makeMetric("foo2", 2));
-        MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
-
-
-        ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
-        final GraphiteSink.Graphite mockGraphite = makeGraphite();
-        Whitebox.setInternalState(sink, "graphite", mockGraphite);
-        sink.putMetrics(record);
-
-        try {
-            verify(mockGraphite).write(argument.capture());
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-
-        String result = argument.getValue();
-
-        assertEquals(true,
-            result.equals("null.all.Context.Context=all.foo1 1 10\n" + 
-            "null.all.Context.Context=all.foo2 2 10\n") ||
-            result.equals("null.all.Context.Context=all.foo2 2 10\n" + 
-            "null.all.Context.Context=all.foo1 1 10\n"));
-    }
-
-    /**
-     * Assert that timestamps are converted correctly, ticket HADOOP-11182
-     */
-    @Test
-    public void testPutMetrics3() {
-
-      // setup GraphiteSink
-      GraphiteSink sink = new GraphiteSink();
-      final GraphiteSink.Graphite mockGraphite = makeGraphite();
-      Whitebox.setInternalState(sink, "graphite", mockGraphite);
-
-      // given two metrics records with timestamps 1000 milliseconds apart.
-      List<MetricsTag> tags = Collections.emptyList();
-      Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
-      metrics.add(makeMetric("foo1", 1));
-      MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics);
-      MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics);
-
-      sink.putMetrics(record1);
-      sink.putMetrics(record2);
-
-      sink.flush();
-      try {
-        sink.close();
-      } catch(IOException e) {
-        e.printStackTrace();
-      }
-
-      // then the timestamps in the graphite stream should differ by one second.
-      try {
-        verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n"));
-        verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n"));
-      } catch (IOException e) {
-        e.printStackTrace();
-      }
-    }
-
-    @Test
-    public void testFailureAndPutMetrics() throws IOException {
-      GraphiteSink sink = new GraphiteSink();
-      List<MetricsTag> tags = new ArrayList<MetricsTag>();
-      tags.add(new MetricsTag(MsInfo.Context, "all"));
-      tags.add(new MetricsTag(MsInfo.Hostname, "host"));
-      Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
-      metrics.add(makeMetric("foo1", 1.25));
-      metrics.add(makeMetric("foo2", 2.25));
-      MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
-
-      final GraphiteSink.Graphite mockGraphite = makeGraphite();
-      Whitebox.setInternalState(sink, "graphite", mockGraphite);
-
-      // throw exception when first try
-      doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString());
-
-      sink.putMetrics(record);
-      verify(mockGraphite).write(anyString());
-      verify(mockGraphite).close();
-
-      // reset mock and try again
-      reset(mockGraphite);
-      when(mockGraphite.isConnected()).thenReturn(false);
-
-      ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
-      sink.putMetrics(record);
-
-      verify(mockGraphite).write(argument.capture());
-      String result = argument.getValue();
-
-      assertEquals(true,
-          result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
-          "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") ||
-          result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" +
-          "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"));
-    }
-
-    @Test
-    public void testClose(){
-        GraphiteSink sink = new GraphiteSink();
-        final GraphiteSink.Graphite mockGraphite = makeGraphite();
-        Whitebox.setInternalState(sink, "graphite", mockGraphite);
-        try {
-            sink.close();
-        } catch (IOException ioe) {
-            ioe.printStackTrace();
-        }
-
-        try {
-            verify(mockGraphite).close();
-        } catch (IOException ioe) {
-            ioe.printStackTrace();
-        }
-    }
-}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index 5a1f1d1376d4a..1e841a686549c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -438,6 +438,8 @@ private void checkMetricsRecords(List<MetricsRecord> recs) {
     r = recs.get(1);
     assertTrue("NumActiveSinks should be 3", Iterables.contains(r.metrics(),
                new MetricGaugeInt(MsInfo.NumActiveSinks, 3)));
+    assertTrue("NumAllSinks should be 3",
+        Iterables.contains(r.metrics(), new MetricGaugeInt(MsInfo.NumAllSinks, 3)));
   }
 
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
index 5d20abdd8bf10..9984c9b95fb24 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.metrics2.lib;
 
+import static org.apache.hadoop.metrics2.impl.MsInfo.Context;
 import static org.apache.hadoop.metrics2.lib.Interns.info;
 import static org.apache.hadoop.test.MetricsAsserts.*;
 import static org.mockito.AdditionalMatchers.eq;
@@ -29,6 +30,8 @@
 import static org.mockito.Mockito.verify;
 import static org.junit.Assert.*;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Random;
@@ -36,6 +39,7 @@
 
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.util.Quantile;
+import org.apache.hadoop.thirdparty.com.google.common.math.Stats;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -47,7 +51,7 @@ public class TestMutableMetrics {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(TestMutableMetrics.class);
-  private final double EPSILON = 1e-42;
+  private static final double EPSILON = 1e-42;
 
   /**
    * Test the snapshot method
@@ -287,6 +291,27 @@ private static void snapshotMutableRatesWithAggregation(
     }
   }
 
+  /**
+   * MutableStat should output 0 instead of the previous state when there is no change.
+   */
+  @Test public void testMutableWithoutChanged() {
+    MetricsRecordBuilder builderWithChange = mockMetricsRecordBuilder();
+    MetricsRecordBuilder builderWithoutChange = mockMetricsRecordBuilder();
+    MetricsRegistry registry = new MetricsRegistry("test");
+    MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true);
+    stat.add(1000, 1000);
+    stat.add(1000, 2000);
+    registry.snapshot(builderWithChange, true);
+
+    assertCounter("TestNumOps", 2000L, builderWithChange);
+    assertGauge("TestINumOps", 2000L, builderWithChange);
+    assertGauge("TestAvgVal", 1.5, builderWithChange);
+
+    registry.snapshot(builderWithoutChange, true);
+    assertGauge("TestINumOps", 0L, builderWithoutChange);
+    assertGauge("TestAvgVal", 0.0, builderWithoutChange);
+  }
+
   @Test
   public void testDuplicateMetrics() {
     MutableRatesWithAggregation rates = new MutableRatesWithAggregation();
@@ -306,19 +331,56 @@ public void testDuplicateMetrics() {
 
   /**
    * Tests that when using {@link MutableStat#add(long, long)}, even with a high
-   * sample count, the mean does not lose accuracy.
+   * sample count, the mean does not lose accuracy. This also validates that
+   * the std dev is correct, assuming samples of equal value.
    */
-  @Test public void testMutableStatWithBulkAdd() {
+  @Test
+  public void testMutableStatWithBulkAdd() {
+    List<Long> samples = new ArrayList<>();
+    for (int i = 0; i < 1000; i++) {
+      samples.add(1000L);
+    }
+    for (int i = 0; i < 1000; i++) {
+      samples.add(2000L);
+    }
+    Stats stats = Stats.of(samples);
+
+    for (int bulkSize : new int[] {1, 10, 100, 1000}) {
+      MetricsRecordBuilder rb = mockMetricsRecordBuilder();
+      MetricsRegistry registry = new MetricsRegistry("test");
+      MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true);
+
+      for (int i = 0; i < samples.size(); i += bulkSize) {
+        stat.add(bulkSize, samples
+            .subList(i, i + bulkSize)
+            .stream()
+            .mapToLong(Long::longValue)
+            .sum()
+        );
+      }
+      registry.snapshot(rb, false);
+
+      assertCounter("TestNumOps", 2000L, rb);
+      assertGauge("TestAvgVal", stats.mean(), rb);
+      assertGauge("TestStdevVal", stats.sampleStandardDeviation(), rb);
+    }
+  }
+
+  @Test
+  public void testLargeMutableStatAdd() {
     MetricsRecordBuilder rb = mockMetricsRecordBuilder();
     MetricsRegistry registry = new MetricsRegistry("test");
-    MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", false);
+    MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true);
 
-    stat.add(1000, 1000);
-    stat.add(1000, 2000);
+    long sample = 1000000000000009L;
+    for (int i = 0; i < 100; i++) {
+      stat.add(1, sample);
+    }
     registry.snapshot(rb, false);
 
-    assertCounter("TestNumOps", 2000L, rb);
-    assertGauge("TestAvgVal", 1.5, rb);
+    assertCounter("TestNumOps", 100L, rb);
+    assertGauge("TestAvgVal", (double) sample, rb);
+    assertGauge("TestStdevVal", 0.0, rb);
   }
 
   /**
@@ -439,4 +501,15 @@ public void testMutableQuantilesEmptyRollover() throws Exception {
     verify(mb, times(2)).addGauge(
         info("FooNumOps", "Number of ops for stat with 5s interval"), (long) 0);
   }
+
+  /**
+   * Test {@link MutableGaugeFloat#incr()}.
+   */
+  @Test(timeout = 30000)
+  public void testMutableGaugeFloat() {
+    MutableGaugeFloat mgf = new MutableGaugeFloat(Context, 3.2f);
+    assertEquals(3.2f, mgf.value(), 0.0);
+    mgf.incr();
+    assertEquals(4.2f, mgf.value(), 0.0);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java
new file mode 100644
index 0000000000000..9ea81c6e4c62e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.metrics2.sink;
+
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsTag;
+import org.apache.hadoop.metrics2.impl.MetricsRecordImpl;
+import org.apache.hadoop.metrics2.impl.MsInfo;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.reset;
+
+
+public class TestGraphiteMetrics {
+  private AbstractMetric makeMetric(String name, Number value) {
+    AbstractMetric metric = mock(AbstractMetric.class);
+    when(metric.name()).thenReturn(name);
+    when(metric.value()).thenReturn(value);
+    return metric;
+  }
+
+  private GraphiteSink.Graphite makeGraphite() {
+    GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class);
+    when(mockGraphite.isConnected()).thenReturn(true);
+    return mockGraphite;
+  }
+
+  @Test
+  public void testPutMetrics() {
+    GraphiteSink sink = new GraphiteSink();
+    List<MetricsTag> tags = new ArrayList<>();
+    tags.add(new MetricsTag(MsInfo.Context, "all"));
+    tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+    Set<AbstractMetric> metrics = new HashSet<>();
+    metrics.add(makeMetric("foo1", 1.25));
+    metrics.add(makeMetric("foo2", 2.25));
+    MetricsRecord record =
+        new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics);
+
+    ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
+    final GraphiteSink.Graphite mockGraphite = makeGraphite();
+    sink.setGraphite(mockGraphite);
+    sink.putMetrics(record);
+
+    try {
+      verify(mockGraphite).write(argument.capture());
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    String result = argument.getValue();
+
+    assertEquals(true,
+        result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
+        "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") ||
+        result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" +
+        "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"));
+    }
+
+  @Test
+  public void testPutMetrics2() throws IllegalAccessException {
+    GraphiteSink sink = new GraphiteSink();
+    List<MetricsTag> tags = new ArrayList<>();
+    tags.add(new MetricsTag(MsInfo.Context, "all"));
+    tags.add(new MetricsTag(MsInfo.Hostname, null));
+    Set<AbstractMetric> metrics = new HashSet<>();
+    metrics.add(makeMetric("foo1", 1));
+    metrics.add(makeMetric("foo2", 2));
+    MetricsRecord record =
+        new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics);
+
+    ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
+    final GraphiteSink.Graphite mockGraphite = makeGraphite();
+    sink.setGraphite(mockGraphite);
+    sink.putMetrics(record);
+
+    try {
+      verify(mockGraphite).write(argument.capture());
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    String result = argument.getValue();
+
+    assertEquals(true,
+        result.equals("null.all.Context.Context=all.foo1 1 10\n" +
+        "null.all.Context.Context=all.foo2 2 10\n") ||
+        result.equals("null.all.Context.Context=all.foo2 2 10\n" +
+        "null.all.Context.Context=all.foo1 1 10\n"));
+  }
+
+  /**
+   * Assert that timestamps are converted correctly, ticket HADOOP-11182.
+   */
+  @Test
+  public void testPutMetrics3() throws IllegalAccessException {
+
+    // setup GraphiteSink
+    GraphiteSink sink = new GraphiteSink();
+    final GraphiteSink.Graphite mockGraphite = makeGraphite();
+    sink.setGraphite(mockGraphite);
+
+    // given two metrics records with timestamps 1000 milliseconds apart.
+    List<MetricsTag> tags = Collections.emptyList();
+    Set<AbstractMetric> metrics = new HashSet<>();
+    metrics.add(makeMetric("foo1", 1));
+    MetricsRecord record1 =
+        new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics);
+    MetricsRecord record2 =
+        new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics);
+
+    sink.putMetrics(record1);
+    sink.putMetrics(record2);
+
+    sink.flush();
+    try {
+      sink.close();
+    } catch(IOException e) {
+      e.printStackTrace();
+    }
+
+    // then the timestamps in the graphite stream should differ by one second.
+    try {
+      verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n"));
+      verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n"));
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Test
+  public void testFailureAndPutMetrics() throws IOException, IllegalAccessException {
+    GraphiteSink sink = new GraphiteSink();
+    List<MetricsTag> tags = new ArrayList<>();
+    tags.add(new MetricsTag(MsInfo.Context, "all"));
+    tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+    Set<AbstractMetric> metrics = new HashSet<>();
+    metrics.add(makeMetric("foo1", 1.25));
+    metrics.add(makeMetric("foo2", 2.25));
+    MetricsRecord record =
+        new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics);
+
+    final GraphiteSink.Graphite mockGraphite = makeGraphite();
+    sink.setGraphite(mockGraphite);
+
+    // throw exception when first try
+    doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString());
+
+    sink.putMetrics(record);
+    verify(mockGraphite).write(anyString());
+    verify(mockGraphite).close();
+
+    // reset mock and try again
+    reset(mockGraphite);
+    when(mockGraphite.isConnected()).thenReturn(false);
+
+    ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
+    sink.putMetrics(record);
+
+    verify(mockGraphite).write(argument.capture());
+    String result = argument.getValue();
+
+    assertEquals(true,
+        result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" +
+        "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") ||
+        result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" +
+        "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n"));
+  }
+
+  @Test
+  public void testClose() throws IllegalAccessException {
+    GraphiteSink sink = new GraphiteSink();
+    final GraphiteSink.Graphite mockGraphite = makeGraphite();
+    sink.setGraphite(mockGraphite);
+    try {
+      sink.close();
+    } catch (IOException ioe) {
+      ioe.printStackTrace();
+    }
+
+    try {
+      verify(mockGraphite).close();
+    } catch (IOException ioe) {
+      ioe.printStackTrace();
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java
similarity index 91%
rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java
rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java
index 4cf4894ff8352..99a75787ad841 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.metrics2.impl;
+package org.apache.hadoop.metrics2.sink;
 
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
@@ -35,9 +35,9 @@
 import org.apache.hadoop.metrics2.MetricType;
 import org.apache.hadoop.metrics2.MetricsRecord;
 import org.apache.hadoop.metrics2.MetricsTag;
-import org.apache.hadoop.metrics2.sink.StatsDSink;
+import org.apache.hadoop.metrics2.impl.MetricsRecordImpl;
+import org.apache.hadoop.metrics2.impl.MsInfo;
 import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD;
-import org.apache.hadoop.test.Whitebox;
 import org.junit.Test;
 
 public class TestStatsDMetrics {
@@ -52,7 +52,7 @@ private AbstractMetric makeMetric(String name, Number value,
   }
 
   @Test(timeout=3000)
-  public void testPutMetrics() throws IOException, InterruptedException {
+  public void testPutMetrics() throws IOException, IllegalAccessException {
     final StatsDSink sink = new StatsDSink();
     List<MetricsTag> tags = new ArrayList<MetricsTag>();
     tags.add(new MetricsTag(MsInfo.Hostname, "host"));
@@ -69,7 +69,7 @@ public void testPutMetrics() throws IOException, InterruptedException {
       final StatsDSink.StatsD mockStatsD =
           new StatsD(sock.getLocalAddress().getHostName(),
               sock.getLocalPort());
-      Whitebox.setInternalState(sink, "statsd", mockStatsD);
+      sink.setStatsd(mockStatsD);
       final DatagramPacket p = new DatagramPacket(new byte[8192], 8192);
       sink.putMetrics(record);
       sock.receive(p);
@@ -87,7 +87,7 @@ public void testPutMetrics() throws IOException, InterruptedException {
   }
 
   @Test(timeout=3000)
-  public void testPutMetrics2() throws IOException {
+  public void testPutMetrics2() throws IOException, IllegalAccessException {
     StatsDSink sink = new StatsDSink();
     List<MetricsTag> tags = new ArrayList<MetricsTag>();
     tags.add(new MetricsTag(MsInfo.Hostname, null));
@@ -104,7 +104,7 @@ public void testPutMetrics2() throws IOException {
       final StatsDSink.StatsD mockStatsD =
           new StatsD(sock.getLocalAddress().getHostName(),
               sock.getLocalPort());
-      Whitebox.setInternalState(sink, "statsd", mockStatsD);
+      sink.setStatsd(mockStatsD);
       final DatagramPacket p = new DatagramPacket(new byte[8192], 8192);
       sink.putMetrics(record);
       sock.receive(p);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java
index 30e8961ef60e7..59ba18803f6bc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,6 +20,7 @@
 
 import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.hadoop.metrics2.impl.ConfigBuilder;
+
 import org.junit.Test;
 
 import java.net.DatagramSocket;
@@ -30,52 +31,61 @@
 import static org.junit.Assert.assertTrue;
 
 public class TestGangliaSink {
-    @Test
-    public void testShouldCreateDatagramSocketByDefault() throws Exception {
-        SubsetConfiguration conf = new ConfigBuilder()
-                .subset("test.sink.ganglia");
+  @Test
+  public void testShouldCreateDatagramSocketByDefault() throws Exception {
+    SubsetConfiguration conf = new ConfigBuilder().subset("test.sink.ganglia");
+
+    GangliaSink30 gangliaSink = new GangliaSink30();
+    gangliaSink.init(conf);
+    DatagramSocket socket = gangliaSink.getDatagramSocket();
+    assertFalse("Did not create DatagramSocket",
+        socket == null || socket instanceof MulticastSocket);
+  }
 
-        GangliaSink30 gangliaSink = new GangliaSink30();
-        gangliaSink.init(conf);
-        DatagramSocket socket = gangliaSink.getDatagramSocket();
-        assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket);
-    }
+  @Test
+  public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception {
+    SubsetConfiguration conf =
+        new ConfigBuilder().add("test.sink.ganglia.multicast", false).subset("test.sink.ganglia");
+    GangliaSink30 gangliaSink = new GangliaSink30();
+    gangliaSink.init(conf);
+    DatagramSocket socket = gangliaSink.getDatagramSocket();
+    assertFalse("Did not create DatagramSocket",
+        socket == null || socket instanceof MulticastSocket);
+  }
 
-    @Test
-    public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception {
-        SubsetConfiguration conf = new ConfigBuilder()
-                .add("test.sink.ganglia.multicast", false)
-                .subset("test.sink.ganglia");
-        GangliaSink30 gangliaSink = new GangliaSink30();
-        gangliaSink.init(conf);
-        DatagramSocket socket = gangliaSink.getDatagramSocket();
-        assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket);
-    }
+  @Test
+  public void testShouldCreateMulticastSocket() throws Exception {
+    SubsetConfiguration conf =
+        new ConfigBuilder().add("test.sink.ganglia.multicast", true).subset("test.sink.ganglia");
+    GangliaSink30 gangliaSink = new GangliaSink30();
+    gangliaSink.init(conf);
+    DatagramSocket socket = gangliaSink.getDatagramSocket();
+    assertTrue("Did not create MulticastSocket",
+        socket != null && socket instanceof MulticastSocket);
+    int ttl = ((MulticastSocket) socket).getTimeToLive();
+    assertEquals("Did not set default TTL", 1, ttl);
+  }
 
-    @Test
-    public void testShouldCreateMulticastSocket() throws Exception {
-        SubsetConfiguration conf = new ConfigBuilder()
-                .add("test.sink.ganglia.multicast", true)
-                .subset("test.sink.ganglia");
-        GangliaSink30 gangliaSink = new GangliaSink30();
-        gangliaSink.init(conf);
-        DatagramSocket socket = gangliaSink.getDatagramSocket();
-        assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket);
-        int ttl = ((MulticastSocket) socket).getTimeToLive();
-        assertEquals("Did not set default TTL", 1, ttl);
-    }
+  @Test
+  public void testShouldSetMulticastSocketTtl() throws Exception {
+    SubsetConfiguration conf = new ConfigBuilder().add("test.sink.ganglia.multicast", true)
+        .add("test.sink.ganglia.multicast.ttl", 3).subset("test.sink.ganglia");
+    GangliaSink30 gangliaSink = new GangliaSink30();
+    gangliaSink.init(conf);
+    DatagramSocket socket = gangliaSink.getDatagramSocket();
+    assertTrue("Did not create MulticastSocket",
+        socket != null && socket instanceof MulticastSocket);
+    int ttl = ((MulticastSocket) socket).getTimeToLive();
+    assertEquals("Did not set TTL", 3, ttl);
+  }
 
-    @Test
-    public void testShouldSetMulticastSocketTtl() throws Exception {
-        SubsetConfiguration conf = new ConfigBuilder()
-                .add("test.sink.ganglia.multicast", true)
-                .add("test.sink.ganglia.multicast.ttl", 3)
-                .subset("test.sink.ganglia");
-        GangliaSink30 gangliaSink = new GangliaSink30();
-        gangliaSink.init(conf);
-        DatagramSocket socket = gangliaSink.getDatagramSocket();
-        assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket);
-        int ttl = ((MulticastSocket) socket).getTimeToLive();
-        assertEquals("Did not set TTL", 3, ttl);
-    }
+  @Test
+  public void testMultipleMetricsServers() {
+    SubsetConfiguration conf =
+        new ConfigBuilder().add("test.sink.ganglia.servers", "server1,server2")
+            .subset("test.sink.ganglia");
+    GangliaSink30 gangliaSink = new GangliaSink30();
+    gangliaSink.init(conf);
+    assertEquals(2, gangliaSink.getMetricsServers().size());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index 328cf11c20fa6..57b620fde6c0e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Arrays;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.math3.stat.inference.ChiSquareTest;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Assert;
@@ -248,4 +249,41 @@ private NodeElement getNewNode(String name, String rackLocation) {
     node.setNetworkLocation(rackLocation);
     return node;
   }
+
+  private NodeElement getNewNode(NetworkTopology cluster,
+                                 String name, String rackLocation) {
+    NodeElement node = getNewNode(name, rackLocation);
+    cluster.add(node);
+    return node;
+  }
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testWeights() {
+    // create the topology
+    NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
+    NodeElement node1 = getNewNode(cluster, "node1", "/r1");
+    NodeElement node2 = getNewNode(cluster, "node2", "/r1");
+    NodeElement node3 = getNewNode(cluster, "node3", "/r2");
+    for (Pair<Integer, NodeElement> test: new Pair[]{Pair.of(0, node1),
+        Pair.of(2, node2), Pair.of(4, node3)}) {
+      int expect = test.getLeft();
+      assertEquals(test.toString(), expect, cluster.getWeight(node1, test.getRight()));
+      assertEquals(test.toString(), expect,
+          cluster.getWeightUsingNetworkLocation(node1, test.getRight()));
+    }
+    // Reset so that we can have 2 levels
+    cluster = NetworkTopology.getInstance(new Configuration());
+    NodeElement node5 = getNewNode(cluster, "node5", "/pod1/r1");
+    NodeElement node6 = getNewNode(cluster, "node6", "/pod1/r1");
+    NodeElement node7 = getNewNode(cluster, "node7", "/pod1/r2");
+    NodeElement node8 = getNewNode(cluster, "node8", "/pod2/r3");
+    for (Pair<Integer, NodeElement> test: new Pair[]{Pair.of(0, node5),
+        Pair.of(2, node6), Pair.of(4, node7), Pair.of(6, node8)}) {
+      int expect = test.getLeft();
+      assertEquals(test.toString(), expect, cluster.getWeight(node5, test.getRight()));
+      assertEquals(test.toString(), expect,
+          cluster.getWeightUsingNetworkLocation(node5, test.getRight()));
+    }
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
index 0e1ac1deb9612..d33545ab6fe0d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.net;
 
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
 import java.net.NetworkInterface;
 import java.net.SocketException;
 import java.net.UnknownHostException;
@@ -27,10 +25,12 @@
 
 import javax.naming.CommunicationException;
 import javax.naming.NameNotFoundException;
+import javax.naming.ServiceUnavailableException;
 
 import org.apache.hadoop.util.Time;
 
 import org.assertj.core.api.Assertions;
+import org.junit.Assume;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -169,7 +169,7 @@ public void testRDNS() throws Exception {
     try {
       String s = DNS.reverseDns(localhost, null);
       LOG.info("Local reverse DNS hostname is " + s);
-    } catch (NameNotFoundException | CommunicationException e) {
+    } catch (NameNotFoundException | CommunicationException | ServiceUnavailableException e) {
       if (!localhost.isLinkLocalAddress() || localhost.isLoopbackAddress()) {
         //these addresses probably won't work with rDNS anyway, unless someone
         //has unusual entries in their DNS server mapping 1.0.0.127 to localhost
@@ -178,6 +178,7 @@ public void testRDNS() throws Exception {
                 + " Loopback=" + localhost.isLoopbackAddress()
                 + " Linklocal=" + localhost.isLinkLocalAddress());
       }
+      Assume.assumeNoException(e);
     }
   }
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java
new file mode 100644
index 0000000000000..4729cee118818
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.net;
+
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Objects;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assume.assumeFalse;
+
+
+public class TestDNSDomainNameResolver {
+
+  static DNSDomainNameResolver DNR = new DNSDomainNameResolver();
+
+  @Test
+  public void testGetHostNameByIP() throws UnknownHostException {
+    InetAddress localhost = InetAddress.getLocalHost();
+    assumeFalse("IP lookup support required",
+        Objects.equals(localhost.getCanonicalHostName(), localhost.getHostAddress()));
+
+    // Precondition: host name and canonical host name for unresolved returns an IP address.
+    InetAddress unresolved = InetAddress.getByAddress(localhost.getHostAddress(),
+        localhost.getAddress());
+    assertEquals(localhost.getHostAddress(), unresolved.getHostName());
+
+    // Test: Get the canonical name despite InetAddress caching
+    String canonicalHostName = DNR.getHostnameByIP(unresolved);
+
+    // Verify: The canonical host name doesn't match the host address but does match the localhost.
+    assertNotEquals(localhost.getHostAddress(), canonicalHostName);
+    assertEquals(localhost.getCanonicalHostName(), canonicalHostName);
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
index 86870e1257119..697b0bad43757 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestTableMapping.java
@@ -30,7 +30,7 @@
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
+
 import org.junit.Test;
 
 public class TestTableMapping {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
index 2e266bba1f97a..03d953b5f3cc3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiagNoKDC.java
@@ -19,8 +19,7 @@
 package org.apache.hadoop.security;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.minikdc.MiniKdc;
-import org.junit.AfterClass;
+
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -31,20 +30,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
 import static org.apache.hadoop.security.KDiag.ARG_KEYLEN;
-import static org.apache.hadoop.security.KDiag.ARG_KEYTAB;
 import static org.apache.hadoop.security.KDiag.ARG_NOFAIL;
 import static org.apache.hadoop.security.KDiag.ARG_NOLOGIN;
-import static org.apache.hadoop.security.KDiag.ARG_PRINCIPAL;
-import static org.apache.hadoop.security.KDiag.ARG_SECURE;
-import static org.apache.hadoop.security.KDiag.CAT_CONFIG;
-import static org.apache.hadoop.security.KDiag.CAT_KERBEROS;
 import static org.apache.hadoop.security.KDiag.CAT_LOGIN;
 import static org.apache.hadoop.security.KDiag.CAT_TOKEN;
 import static org.apache.hadoop.security.KDiag.KerberosDiagsFailure;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index aba3997187747..82e80fd9fa504 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.security;
 
 import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT;
+import static org.apache.hadoop.security.LdapGroupsMapping.GROUP_SEARCH_FILTER_PATTERN;
 import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
 import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
@@ -27,6 +28,8 @@
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -44,6 +47,8 @@
 
 import javax.naming.CommunicationException;
 import javax.naming.NamingException;
+import javax.naming.directory.Attribute;
+import javax.naming.directory.Attributes;
 import javax.naming.directory.SearchControls;
 
 import org.apache.hadoop.conf.Configuration;
@@ -120,6 +125,49 @@ public void testGetGroupsWithDefaultBaseDN() throws Exception {
     doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim());
   }
 
+  @Test
+  public void testGetGroupsWithDynamicGroupFilter() throws Exception {
+    // Set basic mock stuff.
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    String baseDN = "dc=xxx,dc=com";
+    conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN);
+    Attributes attributes = getAttributes();
+
+    // Set the groupFilter conf to take the csv.
+    conf.set(GROUP_SEARCH_FILTER_PATTERN, "userDN,userName");
+
+    // Set the value for userName attribute that is to be used as part of the
+    // group filter at argument 1.
+    final String userName = "some_user";
+    Attribute userNameAttr = mock(Attribute.class);
+    when(userNameAttr.get()).thenReturn(userName);
+    when(attributes.get(eq("userName"))).thenReturn(userNameAttr);
+
+    // Set the dynamic group search filter.
+    final String groupSearchFilter =
+        "(|(memberUid={0})(uname={1}))" + "(objectClass=group)";
+    conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY, groupSearchFilter);
+
+    final LdapGroupsMapping groupsMapping = getGroupsMapping();
+    groupsMapping.setConf(conf);
+
+    // The group search filter should be resolved and should be passed as the
+    // below.
+    String groupFilter = "(|(memberUid={0})(uname={1}))(objectClass=group)";
+    String[] resolvedFilterArgs =
+        new String[] {"CN=some_user,DC=test,DC=com", "some_user"};
+
+    // Return groups only if the resolved filter is passed.
+    when(getContext()
+        .search(anyString(), eq(groupFilter), eq(resolvedFilterArgs),
+            any(SearchControls.class)))
+        .thenReturn(getUserNames(), getGroupNames());
+
+    // Check the group filter got resolved and get the desired values.
+    List<String> groups = groupsMapping.getGroups(userName);
+    Assert.assertEquals(Arrays.asList(getTestGroups()), groups);
+  }
+
   /**
    * Helper method to do the LDAP getGroups operation using given user base DN
    * and group base DN.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
index 7ae802e26d36d..8686d5c6e3b46 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -18,19 +18,22 @@
 
 package org.apache.hadoop.security;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Set;
 
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
+import javax.naming.directory.DirContext;
 import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
+import org.mockito.stubbing.Stubber;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -49,48 +52,121 @@
 public class TestLdapGroupsMappingWithOneQuery
     extends TestLdapGroupsMappingBase {
 
-  @Before
-  public void setupMocks() throws NamingException {
+  public void setupMocks(List<String> listOfDNs) throws NamingException {
     Attribute groupDN = mock(Attribute.class);
 
     NamingEnumeration<SearchResult> groupNames = getGroupNames();
     doReturn(groupNames).when(groupDN).getAll();
-    String groupName1 = "CN=abc,DC=foo,DC=bar,DC=com";
-    String groupName2 = "CN=xyz,DC=foo,DC=bar,DC=com";
-    String groupName3 = "CN=sss,CN=foo,DC=bar,DC=com";
-    doReturn(groupName1).doReturn(groupName2).doReturn(groupName3).
-        when(groupNames).next();
-    when(groupNames.hasMore()).thenReturn(true).thenReturn(true).
-        thenReturn(true).thenReturn(false);
+    buildListOfGroupDNs(listOfDNs).when(groupNames).next();
+    when(groupNames.hasMore()).
+      thenReturn(true).thenReturn(true).
+      thenReturn(true).thenReturn(false);
 
     when(getAttributes().get(eq("memberOf"))).thenReturn(groupDN);
   }
 
+  /**
+   * Build and return a list of individually added group DNs such
+   * that calls to .next() will result in a single value each time.
+   *
+   * @param listOfDNs
+   * @return the stubber to use for the .when().next() call
+   */
+  private Stubber buildListOfGroupDNs(List<String> listOfDNs) {
+    Stubber stubber = null;
+    for (String s : listOfDNs) {
+      if (stubber != null) {
+        stubber.doReturn(s);
+      } else {
+        stubber = doReturn(s);
+      }
+    }
+    return stubber;
+  }
+
   @Test
   public void testGetGroups() throws NamingException {
     // given a user whose ldap query returns a user object with three "memberOf"
     // properties, return an array of strings representing its groups.
     String[] testGroups = new String[] {"abc", "xyz", "sss"};
     doTestGetGroups(Arrays.asList(testGroups));
+
+    // test fallback triggered by NamingException
+    doTestGetGroupsWithFallback();
   }
 
   private void doTestGetGroups(List<String> expectedGroups)
       throws NamingException {
+    List<String> groupDns = new ArrayList<>();
+    groupDns.add("CN=abc,DC=foo,DC=bar,DC=com");
+    groupDns.add("CN=xyz,DC=foo,DC=bar,DC=com");
+    groupDns.add("CN=sss,DC=foo,DC=bar,DC=com");
+
+    setupMocks(groupDns);
     String ldapUrl = "ldap://test";
     Configuration conf = getBaseConf(ldapUrl);
     // enable single-query lookup
     conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
 
-    LdapGroupsMapping groupsMapping = getGroupsMapping();
+    TestLdapGroupsMapping groupsMapping = new TestLdapGroupsMapping();
     groupsMapping.setConf(conf);
     // Username is arbitrary, since the spy is mocked to respond the same,
     // regardless of input
     List<String> groups = groupsMapping.getGroups("some_user");
 
     Assert.assertEquals(expectedGroups, groups);
+    Assert.assertFalse("Second LDAP query should NOT have been called.",
+            groupsMapping.isSecondaryQueryCalled());
 
     // We should have only made one query because single-query lookup is enabled
     verify(getContext(), times(1)).search(anyString(), anyString(),
         any(Object[].class), any(SearchControls.class));
   }
-}
\ No newline at end of file
+
+  private void doTestGetGroupsWithFallback()
+          throws NamingException {
+    List<String> groupDns = new ArrayList<>();
+    groupDns.add("CN=abc,DC=foo,DC=bar,DC=com");
+    groupDns.add("CN=xyz,DC=foo,DC=bar,DC=com");
+    groupDns.add("ipaUniqueID=e4a9a634-bb24-11ec-aec1-06ede52b5fe1," +
+            "CN=sudo,DC=foo,DC=bar,DC=com");
+    setupMocks(groupDns);
+    String ldapUrl = "ldap://test";
+    Configuration conf = getBaseConf(ldapUrl);
+    // enable single-query lookup
+    conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
+    conf.set(LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY, "1");
+
+    TestLdapGroupsMapping groupsMapping = new TestLdapGroupsMapping();
+    groupsMapping.setConf(conf);
+    // Username is arbitrary, since the spy is mocked to respond the same,
+    // regardless of input
+    List<String> groups = groupsMapping.getGroups("some_user");
+
+    // expected to be empty due to invalid memberOf
+    Assert.assertEquals(0, groups.size());
+
+    // expect secondary query to be called: getGroups()
+    Assert.assertTrue("Second LDAP query should have been called.",
+            groupsMapping.isSecondaryQueryCalled());
+
+    // We should have fallen back to the second query because first threw
+    // NamingException expected count is 3 since testGetGroups calls
+    // doTestGetGroups and doTestGetGroupsWithFallback in succession and
+    // the count is across both test scenarios.
+    verify(getContext(), times(3)).search(anyString(), anyString(),
+            any(Object[].class), any(SearchControls.class));
+  }
+
+  private static final class TestLdapGroupsMapping extends LdapGroupsMapping {
+    private boolean secondaryQueryCalled = false;
+    public boolean isSecondaryQueryCalled() {
+      return secondaryQueryCalled;
+    }
+    Set<String> lookupGroup(SearchResult result, DirContext c,
+                                    int goUpHierarchy) throws NamingException {
+      secondaryQueryCalled = true;
+      return super.lookupGroup(result, c, goUpHierarchy);
+    }
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
index de74d17863668..f04fbe1e08926 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
@@ -19,23 +19,18 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.minikdc.MiniKdc;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.PlatformName;
+
 import org.junit.After;
 import org.junit.Test;
 import org.slf4j.event.Level;
 
-import javax.security.auth.Subject;
 import javax.security.auth.kerberos.KerberosPrincipal;
-import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.LoginContext;
 import java.io.File;
 import java.security.Principal;
-import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java
index 03fc4cbb54c16..81abc42a023f3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestWhitelistBasedResolver.java
@@ -111,16 +111,16 @@ public void testFixedAndLocalWhiteList() throws IOException {
     assertEquals (wqr.getDefaultProperties(),
         wqr.getServerProperties(InetAddress.getByName("10.119.103.112")));
 
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113"));
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.119.103.113"));
 
-    assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121"));
+    assertEquals(wqr.getDefaultProperties(), wqr.getServerProperties("10.221.103.121"));
 
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0"));
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.222.103.121"));
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0"));
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.221"));
-    assertEquals (SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222"));
-    assertEquals (wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1"));;
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.221.104.0"));
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.222.103.121"));
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.223.104.0"));
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.221"));
+    assertEquals(SASL_PRIVACY_PROPS, wqr.getServerProperties("10.113.221.222"));
+    assertEquals(wqr.getDefaultProperties(), wqr.getServerProperties("127.0.0.1"));
 
     TestFileBasedIPList.removeFile("fixedwhitelist.txt");
     TestFileBasedIPList.removeFile("variablewhitelist.txt");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
index b9662b8c6a328..0b396be48f983 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
@@ -36,9 +36,6 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-
 public class TestCrossOriginFilter {
 
   @Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509KeyManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509KeyManager.java
index bf0a6abdc676d..a0ce721ecf05b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509KeyManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestReloadingX509KeyManager.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.security.ssl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Supplier;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.BeforeClass;
@@ -32,6 +31,7 @@
 import java.security.cert.X509Certificate;
 import java.util.Timer;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
 
 import static org.apache.hadoop.security.ssl.KeyStoreTestUtil.*;
 import static org.junit.Assert.assertEquals;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
index 8bc881ae5d1da..225cc658d39ba 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java
@@ -30,6 +30,13 @@
 import java.util.List;
 import java.util.Map;
 
+import java.util.concurrent.Callable;
+import org.apache.hadoop.fs.statistics.IOStatisticAssertions;
+import org.apache.hadoop.fs.statistics.MeanStatistic;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.Assert;
 
 import org.apache.hadoop.io.DataInputBuffer;
@@ -155,6 +162,55 @@ public DelegationKey getKey(TestDelegationTokenIdentifier id) {
       return allKeys.get(id.getMasterKeyId());
     }
   }
+
+  public static class TestFailureDelegationTokenSecretManager
+      extends TestDelegationTokenSecretManager {
+    private boolean throwError = false;
+    private long errorSleepMillis;
+
+    public TestFailureDelegationTokenSecretManager(long errorSleepMillis) {
+      super(24*60*60*1000, 10*1000, 1*1000, 60*60*1000);
+      this.errorSleepMillis = errorSleepMillis;
+    }
+
+    public void setThrowError(boolean throwError) {
+      this.throwError = throwError;
+    }
+
+    private void sleepAndThrow() throws IOException {
+      try {
+        Thread.sleep(errorSleepMillis);
+        throw new IOException("Test exception");
+      } catch (InterruptedException e) {
+      }
+    }
+
+    @Override
+    protected void storeNewToken(TestDelegationTokenIdentifier ident, long renewDate)
+        throws IOException {
+      if (throwError) {
+        sleepAndThrow();
+      }
+      super.storeNewToken(ident, renewDate);
+    }
+
+    @Override
+    protected void removeStoredToken(TestDelegationTokenIdentifier ident) throws IOException {
+      if (throwError) {
+        sleepAndThrow();
+      }
+      super.removeStoredToken(ident);
+    }
+
+    @Override
+    protected void updateStoredToken(TestDelegationTokenIdentifier ident, long renewDate)
+        throws IOException {
+      if (throwError) {
+        sleepAndThrow();
+      }
+      super.updateStoredToken(ident, renewDate);
+    }
+  }
   
   public static class TokenSelector extends 
   AbstractDelegationTokenSelector<TestDelegationTokenIdentifier>{
@@ -579,4 +635,102 @@ public void testEmptyToken() throws IOException {
     assertEquals(token1, token2);
     assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString());
   }
+
+  @Test
+  public void testMultipleDelegationTokenSecretManagerMetrics() {
+    TestDelegationTokenSecretManager dtSecretManager1 =
+        new TestDelegationTokenSecretManager(0, 0, 0, 0);
+    assertNotNull(dtSecretManager1.getMetrics());
+
+    TestDelegationTokenSecretManager dtSecretManager2 =
+        new TestDelegationTokenSecretManager(0, 0, 0, 0);
+    assertNotNull(dtSecretManager2.getMetrics());
+
+    DefaultMetricsSystem.instance().init("test");
+
+    TestDelegationTokenSecretManager dtSecretManager3 =
+        new TestDelegationTokenSecretManager(0, 0, 0, 0);
+    assertNotNull(dtSecretManager3.getMetrics());
+  }
+
+  @Test
+  public void testDelegationTokenSecretManagerMetrics() throws Exception {
+    TestDelegationTokenSecretManager dtSecretManager =
+        new TestDelegationTokenSecretManager(24*60*60*1000,
+            10*1000, 1*1000, 60*60*1000);
+    try {
+      dtSecretManager.startThreads();
+
+      final Token<TestDelegationTokenIdentifier> token = callAndValidateMetrics(
+          dtSecretManager, dtSecretManager.getMetrics().getStoreToken(), "storeToken",
+          () -> generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"));
+
+      callAndValidateMetrics(dtSecretManager, dtSecretManager.getMetrics().getUpdateToken(),
+          "updateToken", () -> dtSecretManager.renewToken(token, "JobTracker"));
+
+      callAndValidateMetrics(dtSecretManager, dtSecretManager.getMetrics().getRemoveToken(),
+          "removeToken", () -> dtSecretManager.cancelToken(token, "JobTracker"));
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+
+  @Test
+  public void testDelegationTokenSecretManagerMetricsFailures() throws Exception {
+    int errorSleepMillis = 200;
+    TestFailureDelegationTokenSecretManager dtSecretManager =
+        new TestFailureDelegationTokenSecretManager(errorSleepMillis);
+
+    try {
+      dtSecretManager.startThreads();
+
+      final Token<TestDelegationTokenIdentifier> token =
+          generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+
+      dtSecretManager.setThrowError(true);
+
+      callAndValidateFailureMetrics(dtSecretManager, "storeToken", false,
+          errorSleepMillis,
+          () -> generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"));
+
+      callAndValidateFailureMetrics(dtSecretManager, "updateToken", true,
+          errorSleepMillis, () -> dtSecretManager.renewToken(token, "JobTracker"));
+
+      callAndValidateFailureMetrics(dtSecretManager, "removeToken", true,
+          errorSleepMillis, () -> dtSecretManager.cancelToken(token, "JobTracker"));
+    } finally {
+      dtSecretManager.stopThreads();
+    }
+  }
+
+  private <T> T callAndValidateMetrics(TestDelegationTokenSecretManager dtSecretManager,
+      MutableRate metric, String statName,  Callable<T> callable)
+      throws Exception {
+    MeanStatistic stat = IOStatisticAssertions.lookupMeanStatistic(
+        dtSecretManager.getMetrics().getIoStatistics(), statName + ".mean");
+    long metricBefore = metric.lastStat().numSamples();
+    long statBefore = stat.getSamples();
+    T returnedObject = callable.call();
+    assertEquals(metricBefore + 1, metric.lastStat().numSamples());
+    assertEquals(statBefore + 1, stat.getSamples());
+    return returnedObject;
+  }
+
+  private <T> void callAndValidateFailureMetrics(TestDelegationTokenSecretManager dtSecretManager,
+      String statName, boolean expectError, int errorSleepMillis, Callable<T> callable)
+      throws Exception {
+    MutableCounterLong counter = dtSecretManager.getMetrics().getTokenFailure();
+    MeanStatistic failureStat = IOStatisticAssertions.lookupMeanStatistic(
+        dtSecretManager.getMetrics().getIoStatistics(), statName + ".failures.mean");
+    long counterBefore = counter.value();
+    long statBefore = failureStat.getSamples();
+    if (expectError) {
+      LambdaTestUtils.intercept(IOException.class, callable);
+    } else {
+      callable.call();
+    }
+    assertEquals(counterBefore + 1, counter.value());
+    assertEquals(statBefore + 1, failureStat.getSamples());
+    assertTrue(failureStat.getSum() >= errorSleepMillis);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
index 63d01ccab9e47..e92a25ea0ed8f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
@@ -28,6 +28,8 @@
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.framework.api.CreateBuilder;
+import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
@@ -38,9 +40,12 @@
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.data.Stat;
 import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
 import org.junit.After;
 import org.junit.Assert;
@@ -507,4 +512,65 @@ public Boolean get() {
       }
     }, 1000, 5000);
   }
+
+  @Test
+  public void testCreatingParentContainersIfNeeded() throws Exception {
+
+    String connectString = zkServer.getConnectString();
+    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+    Configuration conf = getSecretConf(connectString);
+    CuratorFramework curatorFramework =
+        CuratorFrameworkFactory.builder()
+        .connectString(connectString)
+        .retryPolicy(retryPolicy)
+        .build();
+    curatorFramework.start();
+    ZKDelegationTokenSecretManager.setCurator(curatorFramework);
+    DelegationTokenManager tm1 = new DelegationTokenManager(conf, new Text("foo"));
+
+    // When the init method is called,
+    // the ZKDelegationTokenSecretManager#startThread method will be called,
+    // and the creatingParentContainersIfNeeded will be called to create the nameSpace.
+    tm1.init();
+
+    String workingPath = "/" + conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH,
+        ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) + "/ZKDTSMRoot";
+
+    // Check if the created NameSpace exists.
+    Stat stat = curatorFramework.checkExists().forPath(workingPath);
+    Assert.assertNotNull(stat);
+
+    tm1.destroy();
+    curatorFramework.close();
+  }
+
+  @Test
+  public void testCreateNameSpaceRepeatedly() throws Exception {
+
+    String connectString = zkServer.getConnectString();
+    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+    Configuration conf = getSecretConf(connectString);
+    CuratorFramework curatorFramework =
+        CuratorFrameworkFactory.builder().
+        connectString(connectString).
+        retryPolicy(retryPolicy).
+        build();
+    curatorFramework.start();
+
+    String workingPath = "/" + conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH,
+        ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) + "/ZKDTSMRoot-Test";
+    CreateBuilder createBuilder = curatorFramework.create();
+    ProtectACLCreateModeStatPathAndBytesable<String> createModeStat =
+        createBuilder.creatingParentContainersIfNeeded();
+    createModeStat.forPath(workingPath);
+
+    // Check if the created NameSpace exists.
+    Stat stat = curatorFramework.checkExists().forPath(workingPath);
+    Assert.assertNotNull(stat);
+
+    // Repeated creation will throw NodeExists exception
+    LambdaTestUtils.intercept(KeeperException.class,
+        "KeeperErrorCode = NodeExists for "+workingPath,
+        () -> createModeStat.forPath(workingPath));
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceLauncher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceLauncher.java
index f40051b0d178f..72757e4b1c182 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceLauncher.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceLauncher.java
@@ -29,8 +29,6 @@
 import org.apache.hadoop.service.launcher.testservices.StoppingInStartLaunchableService;
 import org.apache.hadoop.service.launcher.testservices.StringConstructorOnlyService;
 
-import static org.apache.hadoop.service.launcher.LauncherArguments.*;
-
 import static org.apache.hadoop.test.GenericTestUtils.*;
 import static org.apache.hadoop.service.launcher.testservices.ExceptionInExecuteLaunchableService.*;
 
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/testservices/NoArgsAllowedService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/testservices/NoArgsAllowedService.java
index 602cb157ed5d8..9245b1844f792 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/testservices/NoArgsAllowedService.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/testservices/NoArgsAllowedService.java
@@ -26,7 +26,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.List;
-import java.util.Map;
 
 /**
  * service that does not allow any arguments.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index f1bf4bb91e668..61d5938494c22 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -39,6 +39,7 @@
 import java.util.Random;
 import java.util.Set;
 import java.util.Enumeration;
+import java.util.TreeSet;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -72,7 +73,6 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.util.Sets;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.util.functional.CommonCallableSupplier.submit;
@@ -344,13 +344,13 @@ public static void assertExists(File f) {
   public static void assertGlobEquals(File dir, String pattern,
       String ... expectedMatches) throws IOException {
 
-    Set<String> found = Sets.newTreeSet();
+    Set<String> found = new TreeSet<>();
     for (File f : FileUtil.listFiles(dir)) {
       if (f.getName().matches(pattern)) {
         found.add(f.getName());
       }
     }
-    Set<String> expectedSet = Sets.newTreeSet(
+    Set<String> expectedSet = new TreeSet<>(
         Arrays.asList(expectedMatches));
     Assert.assertEquals("Bad files matching " + pattern + " in " + dir,
         Joiner.on(",").join(expectedSet),
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MoreAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MoreAsserts.java
index 142669b78682e..f6e6055d78e2c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MoreAsserts.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MoreAsserts.java
@@ -19,6 +19,9 @@
 package org.apache.hadoop.test;
 
 import java.util.Iterator;
+import java.util.concurrent.CompletableFuture;
+
+import org.assertj.core.api.Assertions;
 import org.junit.Assert;
 
 /**
@@ -28,17 +31,18 @@ public class MoreAsserts {
 
   /**
    * Assert equivalence for array and iterable
-   * @param <T> the type of the elements
-   * @param s the name/message for the collection
-   * @param expected  the expected array of elements
-   * @param actual    the actual iterable of elements
+   *
+   * @param <T>      the type of the elements
+   * @param s        the name/message for the collection
+   * @param expected the expected array of elements
+   * @param actual   the actual iterable of elements
    */
   public static <T> void assertEquals(String s, T[] expected,
                                       Iterable<T> actual) {
     Iterator<T> it = actual.iterator();
     int i = 0;
     for (; i < expected.length && it.hasNext(); ++i) {
-      Assert.assertEquals("Element "+ i +" for "+ s, expected[i], it.next());
+      Assert.assertEquals("Element " + i + " for " + s, expected[i], it.next());
     }
     Assert.assertTrue("Expected more elements", i == expected.length);
     Assert.assertTrue("Expected less elements", !it.hasNext());
@@ -46,7 +50,8 @@ public static <T> void assertEquals(String s, T[] expected,
 
   /**
    * Assert equality for two iterables
-   * @param <T> the type of the elements
+   *
+   * @param <T>      the type of the elements
    * @param s
    * @param expected
    * @param actual
@@ -57,10 +62,40 @@ public static <T> void assertEquals(String s, Iterable<T> expected,
     Iterator<T> ita = actual.iterator();
     int i = 0;
     while (ite.hasNext() && ita.hasNext()) {
-      Assert.assertEquals("Element "+ i +" for "+s, ite.next(), ita.next());
+      Assert.assertEquals("Element " + i + " for " + s, ite.next(), ita.next());
     }
     Assert.assertTrue("Expected more elements", !ite.hasNext());
     Assert.assertTrue("Expected less elements", !ita.hasNext());
   }
 
+
+  public static <T> void assertFutureCompletedSuccessfully(CompletableFuture<T> future) {
+    Assertions.assertThat(future.isDone())
+            .describedAs("This future is supposed to be " +
+                    "completed successfully")
+            .isTrue();
+    Assertions.assertThat(future.isCompletedExceptionally())
+            .describedAs("This future is supposed to be " +
+                    "completed successfully")
+            .isFalse();
+  }
+
+  public static <T> void assertFutureFailedExceptionally(CompletableFuture<T> future) {
+    Assertions.assertThat(future.isCompletedExceptionally())
+            .describedAs("This future is supposed to be " +
+                    "completed exceptionally")
+            .isTrue();
+  }
+
+  /**
+   * Assert two same type of values.
+   * @param actual actual value.
+   * @param expected expected value.
+   * @param message error message to print in case of mismatch.
+   */
+  public static <T> void assertEqual(T actual, T expected, String message) {
+    Assertions.assertThat(actual)
+            .describedAs("Mismatch in %s", message)
+            .isEqualTo(expected);
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
index 217c2f84eba4b..e270ee68000eb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MultithreadedTestUtil.java
@@ -70,8 +70,8 @@ public abstract class MultithreadedTestUtil {
   public static class TestContext {
     private Throwable err = null;
     private boolean stopped = false;
-    private Set<TestingThread> testThreads = new HashSet<TestingThread>();
-    private Set<TestingThread> finishedThreads = new HashSet<TestingThread>();
+    private Set<TestingThread> testThreads = new HashSet<>();
+    private Set<TestingThread> finishedThreads = new HashSet<>();
 
     /**
      * Check if the context can run threads.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
index 1334f1c95f407..42ed8c8775570 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java
@@ -19,7 +19,6 @@
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
index 606791801fe13..e9c5950b729c6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.tools.CommandShell;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import org.junit.Before;
 import org.junit.Test;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskCheckerWithDiskIo.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskCheckerWithDiskIo.java
index 082672ccd33d2..552d1319312c6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskCheckerWithDiskIo.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskCheckerWithDiskIo.java
@@ -23,15 +23,12 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java
new file mode 100644
index 0000000000000..58a1997e9bc59
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.util.ExitUtil.ExitException;
+import org.apache.hadoop.util.ExitUtil.HaltException;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+public class TestExitUtil extends AbstractHadoopTestBase {
+
+  @Before
+  public void before() {
+    ExitUtil.disableSystemExit();
+    ExitUtil.disableSystemHalt();
+    ExitUtil.resetFirstExitException();
+    ExitUtil.resetFirstHaltException();
+  }
+
+  @After
+  public void after() {
+    ExitUtil.resetFirstExitException();
+    ExitUtil.resetFirstHaltException();
+  }
+
+  @Test
+  public void testGetSetExitExceptions() throws Throwable {
+    // prepare states and exceptions
+    ExitException ee1 = new ExitException(1, "TestExitUtil forged 1st ExitException");
+    ExitException ee2 = new ExitException(2, "TestExitUtil forged 2nd ExitException");
+    // check proper initial settings
+    assertFalse("ExitUtil.terminateCalled initial value should be false",
+        ExitUtil.terminateCalled());
+    assertNull("ExitUtil.getFirstExitException initial value should be null",
+        ExitUtil.getFirstExitException());
+
+    // simulate/check 1st call
+    ExitException ee = intercept(ExitException.class, ()->ExitUtil.terminate(ee1));
+    assertSame("ExitUtil.terminate should have rethrown its ExitException argument but it "
+        + "had thrown something else", ee1, ee);
+    assertTrue("ExitUtil.terminateCalled should be true after 1st ExitUtil.terminate call",
+        ExitUtil.terminateCalled());
+    assertSame("ExitUtil.terminate should store its 1st call's ExitException",
+        ee1, ExitUtil.getFirstExitException());
+
+    // simulate/check 2nd call not overwritting 1st one
+    ee = intercept(ExitException.class, ()->ExitUtil.terminate(ee2));
+    assertSame("ExitUtil.terminate should have rethrown its HaltException argument but it "
+        + "had thrown something else", ee2, ee);
+    assertTrue("ExitUtil.terminateCalled should still be true after 2nd ExitUtil.terminate call",
+        ExitUtil.terminateCalled());
+    // 2nd call rethrown the 2nd ExitException yet only the 1st only should have been stored
+    assertSame("ExitUtil.terminate when called twice should only remember 1st call's "
+        + "ExitException", ee1, ExitUtil.getFirstExitException());
+
+    // simulate cleanup, also tries to make sure state is ok for all junit still has to do
+    ExitUtil.resetFirstExitException();
+    assertFalse("ExitUtil.terminateCalled should be false after "
+        + "ExitUtil.resetFirstExitException call", ExitUtil.terminateCalled());
+    assertNull("ExitUtil.getFirstExitException should be null after "
+        + "ExitUtil.resetFirstExitException call", ExitUtil.getFirstExitException());
+  }
+
+  @Test
+  public void testGetSetHaltExceptions() throws Throwable {
+    // prepare states and exceptions
+    ExitUtil.disableSystemHalt();
+    ExitUtil.resetFirstHaltException();
+    HaltException he1 = new HaltException(1, "TestExitUtil forged 1st HaltException");
+    HaltException he2 = new HaltException(2, "TestExitUtil forged 2nd HaltException");
+
+    // check proper initial settings
+    assertFalse("ExitUtil.haltCalled initial value should be false",
+        ExitUtil.haltCalled());
+    assertNull("ExitUtil.getFirstHaltException initial value should be null",
+        ExitUtil.getFirstHaltException());
+
+    // simulate/check 1st call
+    HaltException he = intercept(HaltException.class, ()->ExitUtil.halt(he1));
+    assertSame("ExitUtil.halt should have rethrown its HaltException argument but it had "
+        +"thrown something else", he1, he);
+    assertTrue("ExitUtil.haltCalled should be true after 1st ExitUtil.halt call",
+        ExitUtil.haltCalled());
+    assertSame("ExitUtil.halt should store its 1st call's HaltException",
+        he1, ExitUtil.getFirstHaltException());
+
+    // simulate/check 2nd call not overwritting 1st one
+    he = intercept(HaltException.class, ()->ExitUtil.halt(he2));
+    assertSame("ExitUtil.halt should have rethrown its HaltException argument but it had "
+        +"thrown something else", he2, he);
+    assertTrue("ExitUtil.haltCalled should still be true after 2nd ExitUtil.halt call",
+        ExitUtil.haltCalled());
+    assertSame("ExitUtil.halt when called twice should only remember 1st call's HaltException",
+        he1, ExitUtil.getFirstHaltException());
+
+    // simulate cleanup, also tries to make sure state is ok for all junit still has to do
+    ExitUtil.resetFirstHaltException();
+    assertFalse("ExitUtil.haltCalled should be false after "
+        + "ExitUtil.resetFirstHaltException call", ExitUtil.haltCalled());
+    assertNull("ExitUtil.getFirstHaltException should be null after "
+        + "ExitUtil.resetFirstHaltException call", ExitUtil.getFirstHaltException());
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
index 1ea3ef1860860..4d0f8d2e04f20 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
@@ -233,4 +233,111 @@ protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
     assertEquals(2, wlogged.get());
     assertEquals(1, wsuppresed.get());
   }
+
+
+  /**
+   * Tests the warning when the write lock is held longer than threshold.
+   */
+  @Test(timeout=10000)
+  public void testWriteLockLongHoldingReportWithReentrant() {
+    String testname = name.getMethodName();
+    final AtomicLong time = new AtomicLong(0);
+    Timer mclock = new Timer() {
+      @Override
+      public long monotonicNow() {
+        return time.get();
+      }
+    };
+
+    final AtomicLong wlogged = new AtomicLong(0);
+    final AtomicLong wsuppresed = new AtomicLong(0);
+    final AtomicLong totalHeldTime = new AtomicLong(0);
+    ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
+    InstrumentedWriteLock writeLock = new InstrumentedWriteLock(testname, LOG,
+        readWriteLock, 2000, 300, mclock) {
+      @Override
+      protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+        totalHeldTime.addAndGet(lockHeldTime);
+        wlogged.incrementAndGet();
+        wsuppresed.set(stats.getSuppressedCount());
+      }
+    };
+
+    InstrumentedReadLock readLock = new InstrumentedReadLock(testname, LOG,
+        readWriteLock, 2000, 300, mclock) {
+      @Override
+      protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+        totalHeldTime.addAndGet(lockHeldTime);
+        wlogged.incrementAndGet();
+        wsuppresed.set(stats.getSuppressedCount());
+      }
+    };
+
+    writeLock.lock();   // t = 0
+    time.set(100);
+
+    writeLock.lock();   // t = 100
+    time.set(500);
+
+    writeLock.lock();   // t = 500
+    time.set(2900);
+    writeLock.unlock(); // t = 2900
+
+    readLock.lock();    // t = 2900
+    time.set(3000);
+    readLock.unlock();  // t = 3000
+
+    writeLock.unlock(); // t = 3000
+
+    writeLock.unlock(); // t = 3000
+    assertEquals(1, wlogged.get());
+    assertEquals(0, wsuppresed.get());
+    assertEquals(3000, totalHeldTime.get());
+  }
+
+  /**
+   * Tests the warning when the read lock is held longer than threshold.
+   */
+  @Test(timeout=10000)
+  public void testReadLockLongHoldingReportWithReentrant() {
+    String testname = name.getMethodName();
+    final AtomicLong time = new AtomicLong(0);
+    Timer mclock = new Timer() {
+      @Override
+      public long monotonicNow() {
+        return time.get();
+      }
+    };
+
+    final AtomicLong wlogged = new AtomicLong(0);
+    final AtomicLong wsuppresed = new AtomicLong(0);
+    final AtomicLong totalHelpTime = new AtomicLong(0);
+    ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
+    InstrumentedReadLock readLock = new InstrumentedReadLock(testname, LOG,
+        readWriteLock, 2000, 300, mclock) {
+      @Override
+      protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+        totalHelpTime.addAndGet(lockHeldTime);
+        wlogged.incrementAndGet();
+        wsuppresed.set(stats.getSuppressedCount());
+      }
+    };
+
+    readLock.lock();   // t = 0
+    time.set(100);
+
+    readLock.lock();   // t = 100
+    time.set(500);
+
+    readLock.lock();   // t = 500
+    time.set(3000);
+    readLock.unlock(); // t = 3000
+
+    readLock.unlock(); // t = 3000
+
+    readLock.unlock(); // t = 3000
+    assertEquals(1, wlogged.get());
+    assertEquals(0, wsuppresed.get());
+    assertEquals(3000, totalHelpTime.get());
+  }
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
index 991697d96bc95..4a106e8fdf1f3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
@@ -28,9 +28,11 @@
 import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.test.HadoopTestBase;
 import org.apache.hadoop.test.LambdaTestUtils;
 
@@ -151,6 +153,9 @@ public void testEmptyFile() throws Throwable {
     }
   }
 
+  /**
+   * round trip through both load APIs.
+   */
   @Test
   public void testFileSystemRoundTrip() throws Throwable {
     File tempFile = File.createTempFile("Keyval", ".json");
@@ -159,19 +164,30 @@ public void testFileSystemRoundTrip() throws Throwable {
     LocalFileSystem fs = FileSystem.getLocal(new Configuration());
     try {
       serDeser.save(fs, tempPath, source, false);
-      assertEquals(source, serDeser.load(fs, tempPath));
+      assertEquals("JSON loaded with load(fs, path)",
+          source,
+          serDeser.load(fs, tempPath));
+      assertEquals("JSON loaded with load(fs, path, status)",
+          source,
+          serDeser.load(fs, tempPath, fs.getFileStatus(tempPath)));
     } finally {
       fs.delete(tempPath, false);
     }
   }
 
+  /**
+   * 0 byte file through the load(path) API will fail with a wrapped
+   * Parser exception.
+   * 0 byte file through the load(path, status) API will fail with a wrapped
+   * Parser exception.
+   */
   @Test
   public void testFileSystemEmptyPath() throws Throwable {
     File tempFile = File.createTempFile("Keyval", ".json");
     Path tempPath = new Path(tempFile.toURI());
     LocalFileSystem fs = FileSystem.getLocal(new Configuration());
     try {
-      LambdaTestUtils.intercept(EOFException.class,
+      LambdaTestUtils.intercept(PathIOException.class,
           () -> serDeser.load(fs, tempPath));
       fs.delete(tempPath, false);
       LambdaTestUtils.intercept(FileNotFoundException.class,
@@ -181,5 +197,23 @@ public void testFileSystemEmptyPath() throws Throwable {
     }
   }
 
+  /**
+   * 0 byte file through the load(path, status) API will fail with an
+   * EOFException.
+   */
+  @Test
+  public void testFileSystemEmptyStatus() throws Throwable {
+    File tempFile = File.createTempFile("Keyval", ".json");
+    Path tempPath = new Path(tempFile.toURI());
+    LocalFileSystem fs = FileSystem.getLocal(new Configuration());
+    try {
+      final FileStatus st = fs.getFileStatus(tempPath);
+      LambdaTestUtils.intercept(EOFException.class,
+          () -> serDeser.load(fs, tempPath, st));
+    } finally {
+      fs.delete(tempPath, false);
+    }
+  }
+
 
 }
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java
index 537e3781edc0e..53241da695c63 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java
@@ -18,9 +18,11 @@
 
 package org.apache.hadoop.util;
 
+import org.assertj.core.api.Assertions;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -79,6 +81,48 @@ public void testItrLinkedLists() {
     Assert.assertEquals(4, list.size());
   }
 
+  @Test
+  public void testListsPartition() {
+    List<String> list = new ArrayList<>();
+    list.add("a");
+    list.add("b");
+    list.add("c");
+    list.add("d");
+    list.add("e");
+    List<List<String>> res = Lists.
+            partition(list, 2);
+    Assertions.assertThat(res)
+            .describedAs("Number of partitions post partition")
+            .hasSize(3);
+    Assertions.assertThat(res.get(0))
+            .describedAs("Number of elements in first partition")
+            .hasSize(2);
+    Assertions.assertThat(res.get(2))
+            .describedAs("Number of elements in last partition")
+            .hasSize(1);
+
+    List<List<String>> res2 = Lists.
+            partition(list, 1);
+    Assertions.assertThat(res2)
+            .describedAs("Number of partitions post partition")
+            .hasSize(5);
+    Assertions.assertThat(res2.get(0))
+            .describedAs("Number of elements in first partition")
+            .hasSize(1);
+    Assertions.assertThat(res2.get(4))
+            .describedAs("Number of elements in last partition")
+            .hasSize(1);
+
+    List<List<String>> res3 = Lists.
+            partition(list, 6);
+    Assertions.assertThat(res3)
+            .describedAs("Number of partitions post partition")
+            .hasSize(1);
+    Assertions.assertThat(res3.get(0))
+            .describedAs("Number of elements in first partition")
+            .hasSize(5);
+  }
+
   @Test
   public void testArrayListWithSize() {
     List<String> list = Lists.newArrayListWithCapacity(3);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 6b7154b83b523..9ae52ff95cb91 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -20,7 +20,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
+
 import org.junit.Assert;
 
 import java.io.BufferedReader;
@@ -51,7 +51,7 @@
 
 public class TestShell extends Assert {
   /**
-   * Set the timeout for every test
+   * Set the timeout for every test.
    */
   @Rule
   public Timeout testTimeout = new Timeout(30000, TimeUnit.MILLISECONDS);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java
new file mode 100644
index 0000000000000..3203de8a96488
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWeakReferenceMap.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test {@link WeakReferenceMap} and {@link WeakReferenceThreadMap}.
+ * There's no attempt to force GC here, so the tests are
+ * more about the basic behavior not the handling of empty references.
+ */
+public class TestWeakReferenceMap extends AbstractHadoopTestBase {
+
+  public static final String FACTORY_STRING = "recreated %d";
+
+  /**
+   * The map to test.
+   */
+  private WeakReferenceMap<Integer, String> referenceMap;
+
+  /**
+   * List of references notified of loss.
+   */
+  private List<Integer> lostReferences;
+
+  @Before
+  public void setup() {
+    lostReferences = new ArrayList<>();
+    referenceMap = new WeakReferenceMap<>(
+        this::factory,
+        this::referenceLost);
+  }
+
+  /**
+   * Reference lost callback.
+   * @param key key lost
+   */
+  private void referenceLost(Integer key) {
+    lostReferences.add(key);
+  }
+
+
+  /**
+   * Basic insertions and lookups of those values.
+   */
+  @Test
+  public void testBasicOperationsWithValidReferences() {
+
+    referenceMap.put(1, "1");
+    referenceMap.put(2, "2");
+    assertMapSize(2);
+    assertMapContainsKey(1);
+    assertMapEntryEquals(1, "1");
+    assertMapEntryEquals(2, "2");
+    // overwrite
+    referenceMap.put(1, "3");
+    assertMapEntryEquals(1, "3");
+
+    // remove an entry
+    referenceMap.remove(1);
+    assertMapDoesNotContainKey(1);
+    assertMapSize(1);
+
+    // clear the map
+    referenceMap.clear();
+    assertMapSize(0);
+  }
+
+  /**
+   * pruning removes null entries, leaves the others alone.
+   */
+  @Test
+  public void testPruneNullEntries() {
+    referenceMap.put(1, "1");
+    assertPruned(0);
+    referenceMap.put(2, null);
+    assertMapSize(2);
+    assertPruned(1);
+    assertMapSize(1);
+    assertMapDoesNotContainKey(2);
+    assertMapEntryEquals(1, "1");
+    assertLostCount(1);
+  }
+
+  /**
+   * Demand create entries.
+   */
+  @Test
+  public void testDemandCreateEntries() {
+
+    // ask for an unknown key and expect a generated value
+    assertMapEntryEquals(1, factory(1));
+    assertMapSize(1);
+    assertMapContainsKey(1);
+    assertLostCount(0);
+
+    // an empty ref has the same outcome
+    referenceMap.put(2, null);
+    assertMapEntryEquals(2, factory(2));
+    // but the lost coun goes up
+    assertLostCount(1);
+
+  }
+
+  /**
+   * It is an error to have a factory which returns null.
+   */
+  @Test
+  public void testFactoryReturningNull() throws Throwable {
+    referenceMap = new WeakReferenceMap<>(
+        (k) -> null,
+            null);
+    intercept(NullPointerException.class, () ->
+        referenceMap.get(0));
+  }
+
+  /**
+   * Test the WeakReferenceThreadMap extension.
+   */
+  @Test
+  public void testWeakReferenceThreadMapAssignment()
+      throws Throwable {
+
+    // counters foor the callbacks
+    final AtomicLong created = new AtomicLong();
+    final AtomicLong lost = new AtomicLong();
+
+    WeakReferenceThreadMap<String> threadMap = new WeakReferenceThreadMap<>(
+        id -> "Entry for thread ID " + id + " (" + created.incrementAndGet() + ")",
+        id -> lost.incrementAndGet());
+
+    Assertions.assertThat(threadMap.setForCurrentThread("hello"))
+        .describedAs("current thread map value on first set")
+        .isNull();
+
+    // second attempt returns itself
+    Assertions.assertThat(threadMap.setForCurrentThread("hello"))
+        .describedAs("current thread map value on second set")
+        .isEqualTo("hello");
+
+    // it is forbidden to explicitly set to null via the set() call.
+    intercept(NullPointerException.class, () ->
+        threadMap.setForCurrentThread(null));
+
+    // the map is unchanged
+    Assertions.assertThat(threadMap.getForCurrentThread())
+        .describedAs("current thread map value")
+        .isEqualTo("hello");
+
+    // remove the value and assert what the removed entry was
+    Assertions.assertThat(threadMap.removeForCurrentThread())
+        .describedAs("removed thread map value")
+        .isEqualTo("hello");
+
+    // remove the value again; this time the removed value is null
+    Assertions.assertThat(threadMap.removeForCurrentThread())
+        .describedAs("removed thread map value on second call")
+        .isNull();
+
+    // lookup will return a new instance created by the factory
+    long c1 = created.get();
+    String dynamicValue = threadMap.getForCurrentThread();
+    Assertions.assertThat(dynamicValue)
+        .describedAs("dynamically created thread map value")
+        .startsWith("Entry for thread ID")
+        .contains("(" + (c1 + 1) + ")");
+
+    // and we can overwrite that
+    Assertions.assertThat(threadMap.setForCurrentThread("hello2"))
+        .describedAs("value before the thread entry is changed")
+        .isEqualTo(dynamicValue);
+
+    // simulate a weak gc
+    long threadId = threadMap.currentThreadId();
+    threadMap.put(threadId, null);
+    String updated = threadMap.getForCurrentThread();
+    Assertions.assertThat(lost.get())
+        .describedAs("lost count")
+        .isEqualTo(1);
+    Assertions.assertThat(updated)
+        .describedAs("dynamically created thread map value")
+        .startsWith("Entry for thread ID")
+        .contains("(" + (c1 + 2) + ")");
+  }
+
+  /**
+   * Assert that the value of a map entry is as expected.
+   * Will trigger entry creation if the key is absent.
+   * @param key key
+   * @param val expected value
+   */
+  private void assertMapEntryEquals(int key, String val) {
+    Assertions.assertThat(referenceMap.get(key))
+        .describedAs("map enty of key %d", key)
+        .isEqualTo(val);
+  }
+
+  /**
+   * Assert that a map entry is present.
+   * @param key key
+   */
+  private void assertMapContainsKey(int key) {
+    Assertions.assertThat(referenceMap.containsKey(key))
+        .describedAs("map entry of key %d should be present", key)
+        .isTrue();
+  }
+
+  /**
+   * Assert that a map entry is not present.
+   * @param key key
+   */
+  private void assertMapDoesNotContainKey(int key) {
+    Assertions.assertThat(referenceMap.containsKey(key))
+        .describedAs("map enty of key %d should be absent", key)
+        .isFalse();
+  }
+
+  /**
+   * Assert map size.
+   * @param size expected size.
+   */
+  private void assertMapSize(int size) {
+    Assertions.assertThat(referenceMap.size())
+        .describedAs("size of map %s", referenceMap)
+        .isEqualTo(size);
+  }
+
+  /**
+   * Assert prune returned the given count.
+   * @param count expected count.
+   */
+  private void assertPruned(int count) {
+    Assertions.assertThat(referenceMap.prune())
+        .describedAs("number of entries pruned from map %s", referenceMap)
+        .isEqualTo(count);
+  }
+
+  /**
+   * Assert number of entries lost matches expected count.
+   * @param count expected count.
+   */
+  private void assertLostCount(int count) {
+    Assertions.assertThat(lostReferences)
+        .describedAs("number of entries lost from map %s", referenceMap)
+        .hasSize(count);
+  }
+
+  /**
+   * Factory operation.
+   * @param key map key
+   * @return a string
+   */
+  private String factory(Integer key) {
+    return String.format(FACTORY_STRING, key);
+  }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java
new file mode 100644
index 0000000000000..ec1b74338a113
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.SAXParser;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+public class TestXMLUtils extends AbstractHadoopTestBase {
+
+  @Test
+  public void testSecureDocumentBuilderFactory() throws Exception {
+    DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+    Document doc = db.parse(new InputSource(new StringReader("<root/>")));
+    Assertions.assertThat(doc).describedAs("parsed document").isNotNull();
+  }
+
+  @Test(expected = SAXException.class)
+  public void testExternalDtdWithSecureDocumentBuilderFactory() throws Exception {
+    DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+    try (InputStream stream = getResourceStream("/xml/external-dtd.xml")) {
+      Document doc = db.parse(stream);
+    }
+  }
+
+  @Test(expected = SAXException.class)
+  public void testEntityDtdWithSecureDocumentBuilderFactory() throws Exception {
+    DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+    try (InputStream stream = getResourceStream("/xml/entity-dtd.xml")) {
+      Document doc = db.parse(stream);
+    }
+  }
+
+  @Test
+  public void testSecureSAXParserFactory() throws Exception {
+    SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+    parser.parse(new InputSource(new StringReader("<root/>")), new DefaultHandler());
+  }
+
+  @Test(expected = SAXException.class)
+  public void testExternalDtdWithSecureSAXParserFactory() throws Exception {
+    SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+    try (InputStream stream = getResourceStream("/xml/external-dtd.xml")) {
+      parser.parse(stream, new DefaultHandler());
+    }
+  }
+
+  @Test(expected = SAXException.class)
+  public void testEntityDtdWithSecureSAXParserFactory() throws Exception {
+    SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+    try (InputStream stream = getResourceStream("/xml/entity-dtd.xml")) {
+      parser.parse(stream, new DefaultHandler());
+    }
+  }
+
+  @Test
+  public void testSecureTransformerFactory() throws Exception {
+    Transformer transformer = XMLUtils.newSecureTransformerFactory().newTransformer();
+    DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+    Document doc = db.parse(new InputSource(new StringReader("<root/>")));
+    try (StringWriter stringWriter = new StringWriter()) {
+      transformer.transform(new DOMSource(doc), new StreamResult(stringWriter));
+      Assertions.assertThat(stringWriter.toString()).contains("<root");
+    }
+  }
+
+  @Test(expected = TransformerException.class)
+  public void testExternalDtdWithSecureTransformerFactory() throws Exception {
+    Transformer transformer = XMLUtils.newSecureTransformerFactory().newTransformer();
+    try (
+        InputStream stream = getResourceStream("/xml/external-dtd.xml");
+        StringWriter stringWriter = new StringWriter()
+    ) {
+      transformer.transform(new StreamSource(stream), new StreamResult(stringWriter));
+    }
+  }
+
+  @Test
+  public void testSecureSAXTransformerFactory() throws Exception {
+    Transformer transformer = XMLUtils.newSecureSAXTransformerFactory().newTransformer();
+    DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+    Document doc = db.parse(new InputSource(new StringReader("<root/>")));
+    try (StringWriter stringWriter = new StringWriter()) {
+      transformer.transform(new DOMSource(doc), new StreamResult(stringWriter));
+      Assertions.assertThat(stringWriter.toString()).contains("<root");
+    }
+  }
+
+  @Test(expected = TransformerException.class)
+  public void testExternalDtdWithSecureSAXTransformerFactory() throws Exception {
+    Transformer transformer = XMLUtils.newSecureSAXTransformerFactory().newTransformer();
+    try (
+        InputStream stream = getResourceStream("/xml/external-dtd.xml");
+        StringWriter stringWriter = new StringWriter()
+    ) {
+      transformer.transform(new StreamSource(stream), new StreamResult(stringWriter));
+    }
+  }
+
+  private static InputStream getResourceStream(final String filename) {
+    return TestXMLUtils.class.getResourceAsStream(filename);
+  }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
index a2156ee6d93af..fd15a0c2b1bf4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
@@ -25,11 +25,15 @@
 import java.util.Arrays;
 import java.util.List;
 
+import javax.security.auth.login.AppConfigurationEntry;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.security.authentication.util.JaasConfiguration;
 import org.apache.hadoop.util.ZKUtil;
 import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.client.ZKClientConfig;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 import org.junit.After;
@@ -154,4 +158,51 @@ public void testTransaction() throws Exception {
     assertFalse(curator.exists(node2));
     assertTrue(Arrays.equals(setData, curator.getData(node1)));
   }
+
+  @Test
+  public void testJaasConfiguration() throws Exception {
+    // Validate that HadoopZooKeeperFactory will set ZKConfig with given principals
+    ZKCuratorManager.HadoopZookeeperFactory factory1 =
+        new ZKCuratorManager.HadoopZookeeperFactory("foo1", "bar1", "bar1.keytab");
+    ZooKeeper zk1 = factory1.newZooKeeper("connString", 1000, null, false);
+    validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY,
+        "bar1", "bar1.keytab", zk1);
+
+    // Validate that a new HadoopZooKeeperFactory will use the new principals
+    ZKCuratorManager.HadoopZookeeperFactory factory2 =
+        new ZKCuratorManager.HadoopZookeeperFactory("foo2", "bar2", "bar2.keytab");
+    ZooKeeper zk2 = factory2.newZooKeeper("connString", 1000, null, false);
+    validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY,
+        "bar2", "bar2.keytab", zk2);
+
+    try {
+      // Setting global configuration
+      String testClientConfig = "TestClientConfig";
+      JaasConfiguration jconf = new JaasConfiguration(testClientConfig, "test", "test.keytab");
+      javax.security.auth.login.Configuration.setConfiguration(jconf);
+      System.setProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY, testClientConfig);
+
+      // Validate that a new HadoopZooKeeperFactory will use the global principals
+      ZKCuratorManager.HadoopZookeeperFactory factory3 =
+          new ZKCuratorManager.HadoopZookeeperFactory("foo3", "bar3", "bar3.keytab");
+      ZooKeeper zk3 = factory3.newZooKeeper("connString", 1000, null, false);
+      validateJaasConfiguration(testClientConfig, "test", "test.keytab", zk3);
+    } finally {
+      // Remove global configuration
+      System.clearProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY);
+    }
+  }
+
+  private void validateJaasConfiguration(String clientConfig, String principal, String keytab,
+      ZooKeeper zk) {
+    assertEquals("Validate that expected clientConfig is set in ZK config", clientConfig,
+        zk.getClientConfig().getProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY));
+
+    AppConfigurationEntry[] entries = javax.security.auth.login.Configuration.getConfiguration()
+        .getAppConfigurationEntry(clientConfig);
+    assertEquals("Validate that expected principal is set in Jaas config", principal,
+        entries[0].getOptions().get("principal"));
+    assertEquals("Validate that expected keytab is set in Jaas config", keytab,
+        entries[0].getOptions().get("keyTab"));
+  }
 }
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
similarity index 94%
rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java
rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
index 7ff5c3d280938..dfee6fc75dcb3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.commit;
+package org.apache.hadoop.util.functional;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -45,15 +45,21 @@
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
- * Test Tasks class.
+ * Test Task Pool class.
+ * This is pulled straight out of the S3A version.
  */
 @RunWith(Parameterized.class)
-public class TestTasks extends HadoopTestBase {
-  private static final Logger LOG = LoggerFactory.getLogger(TestTasks.class);
+public class TestTaskPool extends HadoopTestBase {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestTaskPool.class);
+
   public static final int ITEM_COUNT = 16;
+
   private static final int FAILPOINT = 8;
 
   private final int numThreads;
+
   /**
    * Thread pool for task execution.
    */
@@ -63,14 +69,17 @@ public class TestTasks extends HadoopTestBase {
    * Task submitter bonded to the thread pool, or
    * null for the 0-thread case.
    */
-  Tasks.Submitter submitter;
+  private TaskPool.Submitter submitter;
+
   private final CounterTask failingTask
       = new CounterTask("failing committer", FAILPOINT, Item::commit);
 
   private final FailureCounter failures
       = new FailureCounter("failures", 0, null);
+
   private final CounterTask reverter
       = new CounterTask("reverter", 0, Item::revert);
+
   private final CounterTask aborter
       = new CounterTask("aborter", 0, Item::abort);
 
@@ -91,15 +100,13 @@ public static Collection<Object[]> params() {
     });
   }
 
-
   private List<Item> items;
 
-
   /**
    * Construct the parameterized test.
    * @param numThreads number of threads
    */
-  public TestTasks(int numThreads) {
+  public TestTaskPool(int numThreads) {
     this.numThreads = numThreads;
   }
 
@@ -139,7 +146,7 @@ public void teardown() {
     }
   }
 
-  private class PoolSubmitter implements Tasks.Submitter {
+  private class PoolSubmitter implements TaskPool.Submitter {
 
     @Override
     public Future<?> submit(final Runnable task) {
@@ -152,17 +159,17 @@ public Future<?> submit(final Runnable task) {
    * create the builder.
    * @return pre-inited builder
    */
-  private Tasks.Builder<Item> builder() {
-    return Tasks.foreach(items).executeWith(submitter);
+  private TaskPool.Builder<Item> builder() {
+    return TaskPool.foreach(items).executeWith(submitter);
   }
 
-  private void assertRun(Tasks.Builder<Item> builder,
+  private void assertRun(TaskPool.Builder<Item> builder,
       CounterTask task) throws IOException {
     boolean b = builder.run(task);
     assertTrue("Run of " + task + " failed", b);
   }
 
-  private void assertFailed(Tasks.Builder<Item> builder,
+  private void assertFailed(TaskPool.Builder<Item> builder,
       CounterTask task) throws IOException {
     boolean b = builder.run(task);
     assertFalse("Run of " + task + " unexpectedly succeeded", b);
@@ -170,7 +177,7 @@ private void assertFailed(Tasks.Builder<Item> builder,
 
   private String itemsToString() {
     return "[" + items.stream().map(Item::toString)
-        .collect(Collectors.joining("\n")) +"]";
+        .collect(Collectors.joining("\n")) + "]";
   }
 
   @Test
@@ -377,7 +384,9 @@ public void testRevertAllSuppressed() throws Throwable {
    * The Item which tasks process.
    */
   private final class Item {
+
     private final int id;
+
     private final String text;
 
     private volatile boolean committed, aborted, reverted, failed;
@@ -409,14 +418,14 @@ boolean fail() {
 
     public Item assertCommitted() {
       assertTrue(toString() + " was not committed in\n"
-              +  itemsToString(),
+              + itemsToString(),
           committed);
       return this;
     }
 
     public Item assertCommittedOrFailed() {
       assertTrue(toString() + " was not committed nor failed in\n"
-              +  itemsToString(),
+              + itemsToString(),
           committed || failed);
       return this;
     }
@@ -455,10 +464,15 @@ public String toString() {
    * whose count == limit.
    */
   private class BaseCounter {
+
     private final AtomicInteger counter = new AtomicInteger(0);
+
     private final int limit;
+
     private final String name;
+
     private Item item;
+
     private final Optional<Function<Item, Boolean>> action;
 
     /**
@@ -531,7 +545,7 @@ public String toString() {
   }
 
   private final class CounterTask
-      extends BaseCounter implements Tasks.Task<Item, IOException> {
+      extends BaseCounter implements TaskPool.Task<Item, IOException> {
 
     private CounterTask(String name, int limit,
         Function<Item, Boolean> action) {
@@ -546,7 +560,9 @@ public void run(Item item) throws IOException {
   }
 
   private final class FailureCounter
-      extends BaseCounter implements Tasks.FailureTask<Item, IOException> {
+      extends BaseCounter
+      implements TaskPool.FailureTask<Item, IOException> {
+
     private Exception exception;
 
     private FailureCounter(String name, int limit,
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index fd6ee110c722f..574dfd2852277 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -162,38 +162,6 @@
       </comparators>
     </test>
 
-    <test> <!-- TESTED -->
-      <description>help: help for get</description>
-      <test-commands>
-        <command>-help get</command>
-      </test-commands>
-      <cleanup-commands>
-        <!-- No cleanup -->
-      </cleanup-commands>
-      <comparators>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-f\]( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>\s*Copy files that match the file pattern &lt;src&gt; to the local name.  &lt;src&gt; is kept.\s*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-           <expected-output>^( |\t)*When copying multiple files, the destination must be a directory. Passing -f( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-          <expected-output>^( |\t)*overwrites the destination if it already exists and -p preserves access and( )*</expected-output>
-        </comparator>
-        <comparator>
-          <type>RegexpComparator</type>
-         <expected-output>^( |\t)*modification times, ownership and the mode.*</expected-output>
-        </comparator>
-      </comparators>
-    </test>
-
     <test> <!-- TESTED -->
       <description>help: help for du</description>
       <test-commands>
@@ -356,51 +324,72 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-cp \[-f\] \[-p \| -p\[topax\]\] \[-d\] &lt;src&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+          <expected-output>^-cp \[-f\] \[-p \| -p\[topax\]\] \[-d\] \[-t &lt;thread count&gt;\] \[-q &lt;thread pool queue size&gt;\] &lt;src&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*Copy files that match the file pattern &lt;src&gt; to a destination.  When copying( )*</expected-output>
+          <expected-output>^\s*Copy files that match the file pattern &lt;src&gt; to a destination. When copying( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*multiple files, the destination must be a directory.( )*Passing -p preserves status( )*</expected-output>
+          <expected-output>^( |\t)*multiple files, the destination must be a directory.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*\[topax\] \(timestamps, ownership, permission, ACLs, XAttr\). If -p is specified( )*</expected-output>
+          <expected-output>^( |\t)*Flags :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*with no &lt;arg&gt;, then preserves timestamps, ownership, permission. If -pa is( )*</expected-output>
+          <expected-output>^( |\t)*-p\[topax\]\s+Preserve file attributes \[topx\] \(timestamps,( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*specified, then preserves permission also because ACL is a super-set of( )*</expected-output>
+          <expected-output>^( |\t)*ownership, permission, ACL, XAttr\). If -p is( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*permission. Passing -f overwrites the destination if it already exists. raw( )*</expected-output>
+          <expected-output>^( |\t)*specified with no arg, then preserves timestamps,( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*namespace extended attributes are preserved if \(1\) they are supported \(HDFS( )*</expected-output>
+          <expected-output>^( |\t)*ownership, permission. If -pa is specified, then( )*</expected-output>
         </comparator>
         <comparator>
             <type>RegexpComparator</type>
-            <expected-output>^( |\t)*only\) and, \(2\) all of the source and target pathnames are in the \/\.reserved\/raw( )*</expected-output>
+            <expected-output>^( |\t)*preserves permission also because ACL is a( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*hierarchy. raw namespace xattr preservation is determined solely by the presence( )*</expected-output>
+          <expected-output>^( |\t)*super-set of permission. Determination of whether( )*</expected-output>
         </comparator>
+
         <comparator>
-            <type>RegexpComparator</type>
-          <expected-output>^\s*\(or absence\) of the \/\.reserved\/raw prefix and not by the -p option\. Passing -d( )*</expected-output>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*raw namespace extended attributes are preserved is( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*independent of the -p flag.( )*</expected-output>
         </comparator>
         <comparator>
             <type>RegexpComparator</type>
-            <expected-output>^\s*will skip creation of temporary file\(&lt;dst&gt;\._COPYING_\)\.( )*</expected-output>
+          <expected-output>^\s*-f\s+Overwrite the destination if it already exists.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-d\s+Skip creation of temporary file\(&lt;dst&gt;\._COPYING_\).( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-t &lt;thread count&gt;\s+Number of threads to be used, default is 1.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-q &lt;thread pool queue size&gt;\s+Thread pool queue size to be used, default is( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*1024.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -498,7 +487,7 @@
           <type>RegexpComparator</type>
           <comparator>
             <type>RegexpComparator</type>
-            <expected-output>^-put \[-f\] \[-p\] \[-l\] \[-d\] \[-t &lt;thread count&gt;\] \[-q &lt;threadPool queue size&gt;\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+            <expected-output>^-put \[-f\] \[-p\] \[-l\] \[-d\] \[-t &lt;thread count&gt;\] \[-q &lt;thread pool queue size&gt;\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
           </comparator>
         </comparator>
         <comparator>
@@ -515,35 +504,39 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-p                    Preserves timestamps, ownership and the mode.( )*</expected-output>
+          <expected-output>^\s*-p\s+Preserves timestamps, ownership and the mode.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^\s*-f\s+Overwrites the destination if it already exists.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-f                    Overwrites the destination if it already exists.( )*</expected-output>
+          <expected-output>^\s*-t &lt;thread count&gt;\s+Number of threads to be used, default is 1.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-t &lt;thread count&gt;     Number of threads to be used, default is 1.( )*</expected-output>
+          <expected-output>^\s*-q &lt;thread pool queue size&gt;\s+Thread pool queue size to be used, default is( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-q &lt;threadPool size&gt;  ThreadPool queue size to be used, default is 1024.( )*</expected-output>
+          <expected-output>^( |\t)*1024.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-l                    Allow DataNode to lazily persist the file to disk. Forces( )*</expected-output>
+          <expected-output>^\s*-l\s+Allow DataNode to lazily persist the file to disk.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*replication factor of 1. This flag will result in reduced( )*</expected-output>
+          <expected-output>^\s*Forces replication factor of 1. This flag will( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*durability. Use with care.( )*</expected-output>
+          <expected-output>^\s*result in reduced durability. Use with care.( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^\s*-d                    Skip creation of temporary file\(&lt;dst&gt;\._COPYING_\).( )*</expected-output>
+          <expected-output>^\s*-d\s+Skip creation of temporary file\(&lt;dst&gt;\._COPYING_\).( )*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -558,7 +551,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] \[-d\] \[-t &lt;thread count&gt;\] \[-q &lt;threadPool queue size&gt;\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
+          <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] \[-d\] \[-t &lt;thread count&gt;\] \[-q &lt;thread pool queue size&gt;\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -600,7 +593,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-get( )*\[-f\]( )*\[-p\]( )*\[-ignoreCrc\]( )*\[-crc\]( )*&lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
+          <expected-output>^-get \[-f\] \[-p\] \[-crc\] \[-ignoreCrc\] \[-t &lt;thread count&gt;\] \[-q &lt;thread pool queue size&gt;\] &lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -608,15 +601,39 @@
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*When copying multiple files, the destination must be a directory. Passing -f( )*</expected-output>
+          <expected-output>^( |\t)*When copying multiple files, the destination must be a directory.( )*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*Flags:\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+         <expected-output>^( |\t)*-p\s+Preserves timestamps, ownership and the mode.\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-f\s+Overwrites the destination if it already exists.\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-crc\s+ write CRC checksums for the files downloaded.\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-ignoreCrc\s+ Skip CRC checks on the file\(s\) downloaded.\s*</expected-output>
+        </comparator>
+        <comparator>
+          <type>RegexpComparator</type>
+          <expected-output>^( |\t)*-t &lt;thread count&gt;\s+Number of threads to be used, default is 1.\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^( |\t)*overwrites the destination if it already exists and -p preserves access and( )*</expected-output>
+          <expected-output>^( |\t)*-q &lt;thread pool queue size&gt;\s+Thread pool queue size to be used, default is\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
-         <expected-output>^( |\t)*modification times, ownership and the mode.*</expected-output>
+          <expected-output>^( |\t)*1024.\s*</expected-output>
         </comparator>
       </comparators>
     </test>
@@ -723,7 +740,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-copyToLocal \[-f\] \[-p\] \[-ignoreCrc\] \[-crc\] &lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
+          <expected-output>^-copyToLocal \[-f\] \[-p\] \[-crc\] \[-ignoreCrc\] \[-t &lt;thread count&gt;\] \[-q &lt;thread pool queue size&gt;\] &lt;src&gt; \.\.\. &lt;localdst&gt; :\s*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/xml/entity-dtd.xml b/hadoop-common-project/hadoop-common/src/test/resources/xml/entity-dtd.xml
new file mode 100644
index 0000000000000..a3926bd67ad42
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/xml/entity-dtd.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!DOCTYPE lolz [
+        <!ENTITY lol "lol">
+        <!ELEMENT lolz (#PCDATA)>
+        ]>
+<lolz>&lol;</lolz>
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml b/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml
new file mode 100644
index 0000000000000..08a13938f5f76
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml
@@ -0,0 +1,23 @@
+<?xml version = "1.0" encoding = "UTF-8" standalone = "no" ?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<!DOCTYPE address SYSTEM "address.dtd">
+<address>
+    <name>First Last</name>
+    <company>Acme</company>
+    <phone>(555) 123-4567</phone>
+</address>
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
index fa34bdfc4b5dd..9ba5b7e1bbbfc 100755
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
@@ -16,6 +16,7 @@
 
 setup() {
 
+  export LD_LIBRARY_PATH=""
   RELTMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}"
   mkdir -p ${RELTMP}
   TMP=$(cd -P -- "${RELTMP}" >/dev/null && pwd -P)
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index fedb0375ef72d..96588a22b9419 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -186,7 +186,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <forkCount>1</forkCount>
           <reuseForks>false</reuseForks>
           <threadCount>1</threadCount>
@@ -244,40 +243,6 @@
           </excludeFilterFile>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided implementations rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
-                    <bannedImport>com.google.common.base.Preconditions</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
index 14c2ae907b14d..d71172e1b93ef 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -36,9 +36,9 @@
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener;
 import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -56,10 +56,10 @@
  */
 public class KMSAudit {
   @VisibleForTesting
-  static final Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
+  static final Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = new HashSet<>(Arrays.asList(
       KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY,
       KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK, KMS.KMSOp.REENCRYPT_EEK
-  );
+  ));
 
   private Cache<String, AuditEvent> cache;
 
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 6ea21d5cf407d..09375d5aab528 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -1208,9 +1208,10 @@ Name               | Description
 /logs              | Display log files
 /stacks            | Display JVM stacks
 /static/index.html | The static home page
+/prof              | Async Profiler endpoint
 
 To control the access to servlet `/conf`, `/jmx`, `/logLevel`, `/logs`,
-and `/stacks`, configure the following properties in `kms-site.xml`:
+`/stacks` and `/prof`, configure the following properties in `kms-site.xml`:
 
 ```xml
   <property>
@@ -1224,7 +1225,7 @@ and `/stacks`, configure the following properties in `kms-site.xml`:
     <value>true</value>
     <description>
       Indicates if administrator ACLs are required to access
-      instrumentation servlets (JMX, METRICS, CONF, STACKS).
+      instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF).
     </description>
   </property>
 
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index a0a58ff3567f5..f4c7fbe0b3c3c 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.crypto.key.kms.server;
 
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
@@ -48,7 +49,6 @@
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.util.Time;
 import org.apache.http.client.utils.URIBuilder;
 import org.junit.After;
@@ -929,6 +929,7 @@ public Void call() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("unchecked")
   public void testKMSProviderCaching() throws Exception {
     Configuration conf = new Configuration();
     File confDir = getTestDir();
@@ -946,11 +947,12 @@ public Void call() throws Exception {
         KMSClientProvider kmscp = createKMSClientProvider(uri, conf);
 
         // get the reference to the internal cache, to test invalidation.
-        ValueQueue vq =
-            (ValueQueue) Whitebox.getInternalState(kmscp, "encKeyVersionQueue");
+        ValueQueue vq = (ValueQueue) FieldUtils.getField(KMSClientProvider.class,
+            "encKeyVersionQueue", true).get(kmscp);
         LoadingCache<String, LinkedBlockingQueue<EncryptedKeyVersion>> kq =
-            ((LoadingCache<String, LinkedBlockingQueue<EncryptedKeyVersion>>)
-                Whitebox.getInternalState(vq, "keyQueues"));
+            (LoadingCache<String, LinkedBlockingQueue<EncryptedKeyVersion>>)
+                 FieldUtils.getField(ValueQueue.class, "keyQueues", true).get(vq);
+
         EncryptedKeyVersion mockEKV = Mockito.mock(EncryptedKeyVersion.class);
         when(mockEKV.getEncryptionKeyName()).thenReturn(keyName);
         when(mockEKV.getEncryptionKeyVersionName()).thenReturn(mockVersionName);
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
index 2f47ed794ac84..3d0fd7de6428d 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -24,13 +24,14 @@
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.util.ThreadUtil;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.PropertyConfigurator;
@@ -63,7 +64,7 @@ public void setOutputStream(OutputStream out) {
   }
 
   @Rule
-  public final Timeout testTimeout = new Timeout(180000);
+  public final Timeout testTimeout = new Timeout(180000L, TimeUnit.MILLISECONDS);
 
   @Before
   public void setUp() throws IOException {
@@ -207,8 +208,9 @@ public void testAuditLogFormat() throws Exception {
   @Test
   public void testInitAuditLoggers() throws Exception {
     // Default should be the simple logger
-    List<KMSAuditLogger> loggers = (List<KMSAuditLogger>) Whitebox
-        .getInternalState(kmsAudit, "auditLoggers");
+    List<KMSAuditLogger> loggers = (List<KMSAuditLogger>) FieldUtils.
+        getField(KMSAudit.class, "auditLoggers", true).get(kmsAudit);
+
     Assert.assertEquals(1, loggers.size());
     Assert.assertEquals(SimpleKMSAuditLogger.class, loggers.get(0).getClass());
 
@@ -218,8 +220,8 @@ public void testInitAuditLoggers() throws Exception {
         SimpleKMSAuditLogger.class.getName() + ", "
             + SimpleKMSAuditLogger.class.getName());
     final KMSAudit audit = new KMSAudit(conf);
-    loggers =
-        (List<KMSAuditLogger>) Whitebox.getInternalState(audit, "auditLoggers");
+    loggers = (List<KMSAuditLogger>) FieldUtils.
+        getField(KMSAudit.class, "auditLoggers", true).get(kmsAudit);
     Assert.assertEquals(1, loggers.size());
     Assert.assertEquals(SimpleKMSAuditLogger.class, loggers.get(0).getClass());
 
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 600b2ae1a43ea..33d8b3710c528 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -114,40 +114,6 @@
           </excludeFilterFile>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided implementations rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
-                    <bannedImport>com.google.common.base.Preconditions</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSINFO3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSINFO3Response.java
index ebd54fe08cdbe..4ec4cec610732 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSINFO3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSINFO3Response.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.nfs.nfs3.response;
 
 import org.apache.hadoop.nfs.NfsTime;
-import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
 import org.apache.hadoop.nfs.nfs3.Nfs3Status;
 import org.apache.hadoop.oncrpc.XDR;
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSSTAT3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSSTAT3Response.java
index c0d1a8a38ef06..bcc95861a9608 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSSTAT3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/FSSTAT3Response.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.nfs.nfs3.response;
 
-import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
 import org.apache.hadoop.nfs.nfs3.Nfs3Status;
 import org.apache.hadoop.oncrpc.XDR;
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/LINK3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/LINK3Response.java
index 3893aa10b2f3e..c810030c2616e 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/LINK3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/LINK3Response.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.nfs.nfs3.response;
 
-import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
-import org.apache.hadoop.nfs.nfs3.Nfs3Status;
 import org.apache.hadoop.oncrpc.XDR;
 import org.apache.hadoop.oncrpc.security.Verifier;
 
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java
index 5802b7544e6a8..f47acce57f435 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIR3Response.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.nfs.nfs3.response;
 
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java
index f1bfd56fd3908..cc27397d77c1d 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/READDIRPLUS3Response.java
@@ -25,8 +25,6 @@
 import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
 import org.apache.hadoop.nfs.nfs3.Nfs3Status;
-import org.apache.hadoop.nfs.nfs3.response.READDIR3Response.DirList3;
-import org.apache.hadoop.nfs.nfs3.response.READDIR3Response.Entry3;
 import org.apache.hadoop.oncrpc.XDR;
 import org.apache.hadoop.oncrpc.security.Verifier;
 
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/REMOVE3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/REMOVE3Response.java
index f0fcb3d705c5d..a0b51111a6368 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/REMOVE3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/REMOVE3Response.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.nfs.nfs3.response;
 
-import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
-import org.apache.hadoop.nfs.nfs3.Nfs3Status;
 import org.apache.hadoop.oncrpc.XDR;
 import org.apache.hadoop.oncrpc.security.Verifier;
 
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/WRITE3Response.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/WRITE3Response.java
index 8d4b4d909f640..3f2552a8eca2a 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/WRITE3Response.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/response/WRITE3Response.java
@@ -17,9 +17,7 @@
  */
 package org.apache.hadoop.nfs.nfs3.response;
 
-import org.apache.hadoop.nfs.nfs3.FileHandle;
 import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
-import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes;
 import org.apache.hadoop.nfs.nfs3.Nfs3Status;
 import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow;
 import org.apache.hadoop.oncrpc.XDR;
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java
index 1acefc857f830..163e26b823d04 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpClientHandler.java
@@ -18,11 +18,8 @@
 package org.apache.hadoop.oncrpc;
 
 import io.netty.buffer.ByteBuf;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInboundHandlerAdapter;
-import io.netty.util.ReferenceCountUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java
index 884bebc97561a..1ca4dbe2cca0c 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.oncrpc.security;
 
-import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
 import org.apache.hadoop.oncrpc.RpcCall;
 import org.apache.hadoop.security.IdMappingConstant;
 import org.apache.hadoop.security.IdMappingServiceProvider;
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java
index 7b33a644fbe76..a585dbc6b20b0 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.portmap;
 
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.Map;
 
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
@@ -54,7 +55,7 @@ final class RpcProgramPortmap extends IdleStateHandler {
   private static final Logger LOG =
       LoggerFactory.getLogger(RpcProgramPortmap.class);
 
-  private final ConcurrentHashMap<String, PortmapMapping> map = new ConcurrentHashMap<String, PortmapMapping>();
+  private final ConcurrentHashMap<String, PortmapMapping> map = new ConcurrentHashMap<>();
 
   /** ChannelGroup that remembers all active channels for gracefully shutdown. */
   private final ChannelGroup allChannels;
@@ -208,4 +209,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable t) {
     LOG.warn("Encountered ", t);
     ctx.channel().close();
   }
+
+  public Map<String, PortmapMapping> getMap() {
+    return map;
+  }
 }
diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
index 8ebf9d03c6c30..84fa71a269d71 100644
--- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
@@ -31,7 +31,6 @@
 import org.apache.hadoop.oncrpc.XDR;
 import org.apache.hadoop.oncrpc.security.CredentialsNone;
 import org.apache.hadoop.oncrpc.security.VerifierNone;
-import org.apache.hadoop.test.Whitebox;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -76,7 +75,7 @@ public void testIdle() throws InterruptedException, IOException {
   }
 
   @Test(timeout = 10000)
-  public void testRegistration() throws IOException, InterruptedException {
+  public void testRegistration() throws IOException, InterruptedException, IllegalAccessException {
     XDR req = new XDR();
     RpcCall.getInstance(++xid, RpcProgramPortmap.PROGRAM,
         RpcProgramPortmap.VERSION,
@@ -100,9 +99,7 @@ public void testRegistration() throws IOException, InterruptedException {
     // Give the server a chance to process the request
     Thread.sleep(100);
     boolean found = false;
-    @SuppressWarnings("unchecked")
-    Map<String, PortmapMapping> map = (Map<String, PortmapMapping>) Whitebox
-        .getInternalState(pm.getHandler(), "map");
+    Map<String, PortmapMapping> map = pm.getHandler().getMap();
 
     for (PortmapMapping m : map.values()) {
       if (m.getPort() == sent.getPort()
diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml
index 753b2b28fb0f8..725dda50f216b 100644
--- a/hadoop-common-project/hadoop-registry/pom.xml
+++ b/hadoop-common-project/hadoop-registry/pom.xml
@@ -231,7 +231,6 @@
       <groupId>org.apache.maven.plugins</groupId>
       <artifactId>maven-surefire-plugin</artifactId>
       <configuration>
-        <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
         <reuseForks>false</reuseForks>
         <forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
         <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
@@ -271,40 +270,6 @@
       </configuration>
     </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided implementations rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
-                    <bannedImport>com.google.common.base.Preconditions</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
index 9fa4b8d84d85e..b0b30bdea5faa 100644
--- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
+++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
 import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
-import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor;
 import org.apache.zookeeper.common.PathUtils;
 
 import java.net.IDN;
diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
index cff70a613783a..f451792355c50 100644
--- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
+++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
@@ -43,7 +43,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 
 /**
diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
index 49ea16bba7d62..96cf911fd9541 100644
--- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
+++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
 import org.apache.hadoop.registry.client.types.RegistryPathStatus;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.zookeeper.CreateMode;
diff --git a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index bcca604182bfd..d48920a222333 100644
--- a/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-common-project/hadoop-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -18,11 +18,11 @@
 
 package org.apache.hadoop.registry.client.impl.zk;
 
+import org.apache.hadoop.security.authentication.util.JaasConfiguration;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Splitter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.util.KerberosUtil;
@@ -33,7 +33,6 @@
 import org.apache.zookeeper.Environment;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.client.ZKClientConfig;
-import org.apache.zookeeper.client.ZooKeeperSaslClient;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Id;
 import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
@@ -46,11 +45,9 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Locale;
-import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import static org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions.*;
@@ -798,65 +795,6 @@ public void setKerberosPrincipalAndKeytab(String principal, String keytab) {
     this.keytab = keytab;
   }
 
-  /**
-   * Creates a programmatic version of a jaas.conf file. This can be used
-   * instead of writing a jaas.conf file and setting the system property,
-   * "java.security.auth.login.config", to point to that file. It is meant to be
-   * used for connecting to ZooKeeper.
-   */
-  @InterfaceAudience.Private
-  public static class JaasConfiguration extends
-      javax.security.auth.login.Configuration {
-
-    private final javax.security.auth.login.Configuration baseConfig =
-        javax.security.auth.login.Configuration.getConfiguration();
-    private static AppConfigurationEntry[] entry;
-    private String entryName;
-
-    /**
-     * Add an entry to the jaas configuration with the passed in name,
-     * principal, and keytab. The other necessary options will be set for you.
-     *
-     * @param entryName The name of the entry (e.g. "Client")
-     * @param principal The principal of the user
-     * @param keytab The location of the keytab
-     */
-    public JaasConfiguration(String entryName, String principal, String keytab) {
-      this.entryName = entryName;
-      Map<String, String> options = new HashMap<String, String>();
-      options.put("keyTab", keytab);
-      options.put("principal", principal);
-      options.put("useKeyTab", "true");
-      options.put("storeKey", "true");
-      options.put("useTicketCache", "false");
-      options.put("refreshKrb5Config", "true");
-      String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
-      if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
-        options.put("debug", "true");
-      }
-      entry = new AppConfigurationEntry[]{
-          new AppConfigurationEntry(getKrb5LoginModuleName(),
-              AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
-              options)};
-    }
-
-    @Override
-    public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
-      return (entryName.equals(name)) ? entry : ((baseConfig != null)
-          ? baseConfig.getAppConfigurationEntry(name) : null);
-    }
-
-    private String getKrb5LoginModuleName() {
-      String krb5LoginModuleName;
-      if (System.getProperty("java.vendor").contains("IBM")) {
-        krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
-      } else {
-        krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
-      }
-      return krb5LoginModuleName;
-    }
-  }
-
   /**
    * Set the client properties. This forces the ZK client into
    * failing if it can't auth.
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index b36dbf30610ff..f167a079a9b0c 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -56,5 +56,4 @@
       </plugin>
     </plugins>
   </build>
-
 </project>
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 0a5db2565b8c5..0b1c601267358 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -41,11 +41,21 @@
       <artifactId>hadoop-hdfs-client</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-native-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-app</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
index c96b3a99bd1c4..508388aa4812e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
@@ -94,4 +94,17 @@
     <Bug pattern="EI_EXPOSE_REP" />
   </Match>
 
+  <!--okhttp classes from Kotlin are not analysed for NP check. -->
+  <Match>
+    <Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
+    <Method name="refresh" />
+    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+  </Match>
+
+  <Match>
+    <Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
+    <Method name="refresh" />
+    <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
+  </Match>
+
 </FindBugsFilter>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index f521480281810..3337f7d408903 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -35,8 +35,26 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
 
   <dependencies>
     <dependency>
-      <groupId>com.squareup.okhttp</groupId>
+      <groupId>com.squareup.okhttp3</groupId>
       <artifactId>okhttp</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.squareup.okio</groupId>
+          <artifactId>okio-jvm</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio-jvm</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib-common</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -51,6 +69,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <groupId>log4j</groupId>
           <artifactId>log4j</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-ext</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -178,38 +200,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           <excludePackageNames>org.apache.hadoop.hdfs.protocol.proto</excludePackageNames>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index 5a8d033c993f7..2377baa4fec25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -43,7 +43,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,6 +69,11 @@ public class ClientContext {
    */
   private final String name;
 
+  /**
+   * The client conf used to initialize context.
+   */
+  private final DfsClientConf dfsClientConf;
+
   /**
    * String representation of the configuration.
    */
@@ -130,6 +135,17 @@ public class ClientContext {
    */
   private volatile DeadNodeDetector deadNodeDetector = null;
 
+  /**
+   * The switch for the {@link LocatedBlocksRefresher}.
+   */
+  private final boolean locatedBlocksRefresherEnabled;
+
+  /**
+   * Periodically refresh the {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks} backing
+   * registered {@link DFSInputStream}s, to take advantage of changes in block placement.
+   */
+  private volatile LocatedBlocksRefresher locatedBlocksRefresher = null;
+
   /**
    * Count the reference of ClientContext.
    */
@@ -146,6 +162,7 @@ private ClientContext(String name, DfsClientConf conf,
     final ShortCircuitConf scConf = conf.getShortCircuitConf();
 
     this.name = name;
+    this.dfsClientConf = conf;
     this.confString = scConf.confAsString();
     this.clientShortCircuitNum = conf.getClientShortCircuitNum();
     this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum];
@@ -164,6 +181,7 @@ private ClientContext(String name, DfsClientConf conf,
     this.byteArrayManager = ByteArrayManager.newInstance(
         conf.getWriteByteArrayManagerConf());
     this.deadNodeDetectionEnabled = conf.isDeadNodeDetectionEnabled();
+    this.locatedBlocksRefresherEnabled = conf.isLocatedBlocksRefresherEnabled();
     initTopologyResolution(config);
   }
 
@@ -301,6 +319,21 @@ public DeadNodeDetector getDeadNodeDetector() {
     return deadNodeDetector;
   }
 
+  /**
+   * If true, LocatedBlocksRefresher will be periodically refreshing LocatedBlocks
+   * of registered DFSInputStreams.
+   */
+  public boolean isLocatedBlocksRefresherEnabled() {
+    return locatedBlocksRefresherEnabled;
+  }
+
+  /**
+   * Obtain LocatedBlocksRefresher of the current client.
+   */
+  public LocatedBlocksRefresher getLocatedBlocksRefresher() {
+    return locatedBlocksRefresher;
+  }
+
   /**
    * Increment the counter. Start the dead node detector thread if there is no
    * reference.
@@ -311,6 +344,10 @@ synchronized void reference() {
       deadNodeDetector = new DeadNodeDetector(name, configuration);
       deadNodeDetector.start();
     }
+    if (locatedBlocksRefresherEnabled && locatedBlocksRefresher == null) {
+      locatedBlocksRefresher = new LocatedBlocksRefresher(name, configuration, dfsClientConf);
+      locatedBlocksRefresher.start();
+    }
   }
 
   /**
@@ -324,5 +361,10 @@ synchronized void unreference() {
       deadNodeDetector.shutdown();
       deadNodeDetector = null;
     }
+
+    if (counter == 0 && locatedBlocksRefresherEnabled && locatedBlocksRefresher != null) {
+      locatedBlocksRefresher.shutdown();
+      locatedBlocksRefresher = null;
+    }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java
index 4de969642d574..7b03e1f3518f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java
@@ -20,12 +20,19 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RouterFederatedStateProto;
 import org.apache.hadoop.ipc.AlignmentContext;
 import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
 import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.LongAccumulator;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 
 /**
  * Global State Id context for the client.
@@ -37,8 +44,17 @@
 @InterfaceStability.Evolving
 public class ClientGSIContext implements AlignmentContext {
 
-  private final LongAccumulator lastSeenStateId =
-      new LongAccumulator(Math::max, Long.MIN_VALUE);
+  private final LongAccumulator lastSeenStateId;
+  private ByteString routerFederatedState;
+
+  public ClientGSIContext() {
+    this(new LongAccumulator(Math::max, Long.MIN_VALUE));
+  }
+
+  public ClientGSIContext(LongAccumulator lastSeenStateId) {
+    this.lastSeenStateId = lastSeenStateId;
+    routerFederatedState = null;
+  }
 
   @Override
   public long getLastSeenStateId() {
@@ -65,16 +81,59 @@ public void updateResponseState(RpcResponseHeaderProto.Builder header) {
    * in responses.
    */
   @Override
-  public void receiveResponseState(RpcResponseHeaderProto header) {
-    lastSeenStateId.accumulate(header.getStateId());
+  public synchronized void receiveResponseState(RpcResponseHeaderProto header) {
+    if (header.hasRouterFederatedState()) {
+      routerFederatedState = mergeRouterFederatedState(
+          this.routerFederatedState, header.getRouterFederatedState());
+    } else {
+      lastSeenStateId.accumulate(header.getStateId());
+    }
+  }
+
+  /**
+   * Utility function to parse routerFederatedState field in RPC headers.
+   */
+  public static Map<String, Long> getRouterFederatedStateMap(ByteString byteString) {
+    if (byteString != null) {
+      try {
+        RouterFederatedStateProto federatedState = RouterFederatedStateProto.parseFrom(byteString);
+        return federatedState.getNamespaceStateIdsMap();
+      } catch (InvalidProtocolBufferException e) {
+        // Ignore this exception and will return an empty map
+      }
+    }
+    return Collections.emptyMap();
+  }
+
+  /**
+   * Merge state1 and state2 to get the max value for each namespace.
+   * @param state1 input ByteString.
+   * @param state2 input ByteString.
+   * @return one ByteString object which contains the max value of each namespace.
+   */
+  public static ByteString mergeRouterFederatedState(ByteString state1, ByteString state2) {
+    Map<String, Long> mapping1 = new HashMap<>(getRouterFederatedStateMap(state1));
+    Map<String, Long> mapping2 = getRouterFederatedStateMap(state2);
+    mapping2.forEach((k, v) -> {
+      long localValue = mapping1.getOrDefault(k, 0L);
+      mapping1.put(k, Math.max(v, localValue));
+    });
+    RouterFederatedStateProto.Builder federatedBuilder = RouterFederatedStateProto.newBuilder();
+    mapping1.forEach(federatedBuilder::putNamespaceStateIds);
+    return federatedBuilder.build().toByteString();
   }
 
   /**
    * Client side implementation for providing state alignment info in requests.
    */
   @Override
-  public void updateRequestState(RpcRequestHeaderProto.Builder header) {
-    header.setStateId(lastSeenStateId.longValue());
+  public synchronized void updateRequestState(RpcRequestHeaderProto.Builder header) {
+    if (lastSeenStateId.get() != Long.MIN_VALUE) {
+      header.setStateId(lastSeenStateId.get());
+    }
+    if (routerFederatedState != null) {
+      header.setRouterFederatedState(routerFederatedState);
+    }
   }
 
   /**
@@ -87,4 +146,9 @@ public long receiveRequestState(RpcRequestHeaderProto header, long threshold)
     // Do nothing.
     return 0;
   }
+
+  @VisibleForTesting
+  public ByteString getRouterFederatedState() {
+    return this.routerFederatedState;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 2fa67d9449f77..acfca6799f4f8 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -41,6 +41,7 @@
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -196,7 +197,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
 
 /********************************************************
@@ -274,7 +275,7 @@ Configuration getConfiguration() {
    * that are currently being written by this client.
    * Note that a file can only be written by a single client.
    */
-  private final Map<Long, DFSOutputStream> filesBeingWritten = new HashMap<>();
+  private final Map<String, DFSOutputStream> filesBeingWritten = new HashMap<>();
 
   /**
    * Same as this(NameNode.getNNAddress(conf), conf);
@@ -501,9 +502,9 @@ public LeaseRenewer getLeaseRenewer() {
   }
 
   /** Get a lease and start automatic renewal */
-  private void beginFileLease(final long inodeId, final DFSOutputStream out) {
+  private void beginFileLease(final String key, final DFSOutputStream out) {
     synchronized (filesBeingWritten) {
-      putFileBeingWritten(inodeId, out);
+      putFileBeingWritten(key, out);
       LeaseRenewer renewer = getLeaseRenewer();
       boolean result = renewer.put(this);
       if (!result) {
@@ -517,9 +518,9 @@ private void beginFileLease(final long inodeId, final DFSOutputStream out) {
   }
 
   /** Stop renewal of lease for the file. */
-  void endFileLease(final long inodeId) {
+  void endFileLease(final String renewLeaseKey) {
     synchronized (filesBeingWritten) {
-      removeFileBeingWritten(inodeId);
+      removeFileBeingWritten(renewLeaseKey);
       // remove client from renewer if no files are open
       if (filesBeingWritten.isEmpty()) {
         getLeaseRenewer().closeClient(this);
@@ -531,10 +532,10 @@ void endFileLease(final long inodeId) {
    *  enforced to consistently update its local dfsclients array and
    *  client's filesBeingWritten map.
    */
-  public void putFileBeingWritten(final long inodeId,
+  public void putFileBeingWritten(final String key,
       final DFSOutputStream out) {
     synchronized(filesBeingWritten) {
-      filesBeingWritten.put(inodeId, out);
+      filesBeingWritten.put(key, out);
       // update the last lease renewal time only when there was no
       // writes. once there is one write stream open, the lease renewer
       // thread keeps it updated well with in anyone's expiration time.
@@ -545,9 +546,9 @@ public void putFileBeingWritten(final long inodeId,
   }
 
   /** Remove a file. Only called from LeaseRenewer. */
-  public void removeFileBeingWritten(final long inodeId) {
+  public void removeFileBeingWritten(final String key) {
     synchronized(filesBeingWritten) {
-      filesBeingWritten.remove(inodeId);
+      filesBeingWritten.remove(key);
       if (filesBeingWritten.isEmpty()) {
         lastLeaseRenewal = 0;
       }
@@ -579,6 +580,34 @@ void updateLastLeaseRenewal() {
     }
   }
 
+  @VisibleForTesting
+  public int getNumOfFilesBeingWritten() {
+    synchronized (filesBeingWritten) {
+      return filesBeingWritten.size();
+    }
+  }
+
+  /**
+   * Get all namespaces of DFSOutputStreams.
+   */
+  private List<String> getNamespaces() {
+    HashSet<String> namespaces = new HashSet<>();
+    synchronized (filesBeingWritten) {
+      for (DFSOutputStream outputStream : filesBeingWritten.values()) {
+        String namespace = outputStream.getNamespace();
+        if (namespace == null || namespace.isEmpty()) {
+          return null;
+        } else {
+          namespaces.add(namespace);
+        }
+      }
+      if (namespaces.isEmpty()) {
+        return null;
+      }
+    }
+    return new ArrayList<>(namespaces);
+  }
+
   /**
    * Renew leases.
    * @return true if lease was renewed. May return false if this
@@ -587,7 +616,7 @@ void updateLastLeaseRenewal() {
   public boolean renewLease() throws IOException {
     if (clientRunning && !isFilesBeingWrittenEmpty()) {
       try {
-        namenode.renewLease(clientName);
+        namenode.renewLease(clientName, getNamespaces());
         updateLastLeaseRenewal();
         return true;
       } catch (IOException e) {
@@ -618,14 +647,14 @@ void closeConnectionToNamenode() {
   /** Close/abort all files being written. */
   public void closeAllFilesBeingWritten(final boolean abort) {
     for(;;) {
-      final long inodeId;
+      final String key;
       final DFSOutputStream out;
       synchronized(filesBeingWritten) {
         if (filesBeingWritten.isEmpty()) {
           return;
         }
-        inodeId = filesBeingWritten.keySet().iterator().next();
-        out = filesBeingWritten.remove(inodeId);
+        key = filesBeingWritten.keySet().iterator().next();
+        out = filesBeingWritten.remove(key);
       }
       if (out != null) {
         try {
@@ -636,7 +665,7 @@ public void closeAllFilesBeingWritten(final boolean abort) {
           }
         } catch(IOException ie) {
           LOG.error("Failed to " + (abort ? "abort" : "close") + " file: "
-              + out.getSrc() + " with inode: " + inodeId, ie);
+              + out.getSrc() + " with renewLeaseKey: " + key, ie);
         }
       }
     }
@@ -863,7 +892,7 @@ public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
   }
 
   public long getRefreshReadBlkLocationsInterval() {
-    return dfsClientConf.getRefreshReadBlockLocationsMS();
+    return dfsClientConf.getLocatedBlocksRefresherInterval();
   }
 
   /**
@@ -1275,7 +1304,7 @@ public DFSOutputStream create(String src, FsPermission permission,
         src, masked, flag, createParent, replication, blockSize, progress,
         dfsClientConf.createChecksum(checksumOpt),
         getFavoredNodesStr(favoredNodes), ecPolicyName, storagePolicy);
-    beginFileLease(result.getFileId(), result);
+    beginFileLease(result.getUniqKey(), result);
     return result;
   }
 
@@ -1330,7 +1359,7 @@ public DFSOutputStream primitiveCreate(String src, FsPermission absPermission,
           flag, createParent, replication, blockSize, progress, checksum,
           null, null, null);
     }
-    beginFileLease(result.getFileId(), result);
+    beginFileLease(result.getUniqKey(), result);
     return result;
   }
 
@@ -1475,7 +1504,7 @@ private DFSOutputStream append(String src, int buffersize,
     checkOpen();
     final DFSOutputStream result = callAppend(src, flag, progress,
         favoredNodes);
-    beginFileLease(result.getFileId(), result);
+    beginFileLease(result.getUniqKey(), result);
     return result;
   }
 
@@ -2396,8 +2425,8 @@ long rollEdits() throws IOException {
   }
 
   @VisibleForTesting
-  ExtendedBlock getPreviousBlock(long fileId) {
-    return filesBeingWritten.get(fileId).getBlock();
+  ExtendedBlock getPreviousBlock(String key) {
+    return filesBeingWritten.get(key).getBlock();
   }
 
   /**
@@ -3459,4 +3488,44 @@ private boolean isDeadNodeDetectionEnabled() {
   public DeadNodeDetector getDeadNodeDetector() {
     return clientContext.getDeadNodeDetector();
   }
+
+  /**
+   * Obtain LocatedBlocksRefresher of the current client.
+   */
+  public LocatedBlocksRefresher getLocatedBlockRefresher() {
+    return clientContext.getLocatedBlocksRefresher();
+  }
+
+  /**
+   * Adds the {@link DFSInputStream} to the {@link LocatedBlocksRefresher}, so that
+   * the underlying {@link LocatedBlocks} is periodically refreshed.
+   */
+  public void addLocatedBlocksRefresh(DFSInputStream dfsInputStream) {
+    if (isLocatedBlocksRefresherEnabled()) {
+      clientContext.getLocatedBlocksRefresher().addInputStream(dfsInputStream);
+    }
+  }
+
+  /**
+   * Removes the {@link DFSInputStream} from the {@link LocatedBlocksRefresher}, so that
+   * the underlying {@link LocatedBlocks} is no longer periodically refreshed.
+   * @param dfsInputStream
+   */
+  public void removeLocatedBlocksRefresh(DFSInputStream dfsInputStream) {
+    if (isLocatedBlocksRefresherEnabled()) {
+      clientContext.getLocatedBlocksRefresher().removeInputStream(dfsInputStream);
+    }
+  }
+
+  private boolean isLocatedBlocksRefresherEnabled() {
+    return clientContext.isLocatedBlocksRefresherEnabled();
+  }
+
+  public DatanodeInfo[] slowDatanodeReport() throws IOException {
+    checkOpen();
+    try (TraceScope ignored = tracer.newScope("slowDatanodeReport")) {
+      return namenode.getSlowDatanodeReport();
+    }
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
index ecea795ee5f98..caf8aad32e343 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 
 /**
  * Used for injecting faults in DFSClient and DFSOutputStream tests.
@@ -65,4 +66,7 @@ public boolean skipRollingRestartWait() {
   public void sleepBeforeHedgedGet() {}
 
   public void delayWhenRenewLeaseTimeout() {}
+
+  public void onCreateBlockReader(LocatedBlock block, int chunkIndex, long offset, long length) {}
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 68618952dc5e8..7b664e4f31164 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -28,6 +28,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -65,6 +66,7 @@
 import org.apache.hadoop.hdfs.protocol.BlockType;
 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -128,18 +130,18 @@ public class DFSInputStream extends FSInputStream
   private long lastBlockBeingWrittenLength = 0;
   private FileEncryptionInfo fileEncryptionInfo = null;
   protected CachingStrategy cachingStrategy;
+  // this is volatile because it will be polled outside the lock,
+  // but still only updated within the lock
+  private volatile long lastRefreshedBlocksAt = Time.monotonicNow();
   ////
 
+  private AtomicBoolean refreshingBlockLocations = new AtomicBoolean(false);
   protected final ReadStatistics readStatistics = new ReadStatistics();
   // lock for state shared between read and pread
   // Note: Never acquire a lock on <this> with this lock held to avoid deadlocks
   //       (it's OK to acquire this lock when the lock on <this> is held)
   protected final Object infoLock = new Object();
 
-  // refresh locatedBlocks periodically
-  private final long refreshReadBlockIntervals;
-  /** timeStamp of the last time a block location was refreshed. */
-  private long locatedBlocksTimeStamp;
   /**
    * Track the ByteBuffers that we have handed out to readers.
    *
@@ -156,10 +158,6 @@ public class DFSInputStream extends FSInputStream
     return extendedReadBuffers;
   }
 
-  private boolean isPeriodicRefreshEnabled() {
-    return (refreshReadBlockIntervals > 0L);
-  }
-
   /**
    * This variable tracks the number of failures since the start of the
    * most recent user-facing operation. That is to say, it should be reset
@@ -206,9 +204,6 @@ protected DFSClient getDFSClient() {
   DFSInputStream(DFSClient dfsClient, String src, boolean verifyChecksum,
       LocatedBlocks locatedBlocks) throws IOException {
     this.dfsClient = dfsClient;
-    this.refreshReadBlockIntervals =
-        this.dfsClient.getRefreshReadBlkLocationsInterval();
-    setLocatedBlocksTimeStamp();
     this.verifyChecksum = verifyChecksum;
     this.src = src;
     synchronized (infoLock) {
@@ -228,19 +223,6 @@ boolean deadNodesContain(DatanodeInfo nodeInfo) {
     return deadNodes.containsKey(nodeInfo);
   }
 
-  @VisibleForTesting
-  void setReadTimeStampsForTesting(long timeStamp) {
-    setLocatedBlocksTimeStamp(timeStamp);
-  }
-
-  private void setLocatedBlocksTimeStamp() {
-    setLocatedBlocksTimeStamp(Time.monotonicNow());
-  }
-
-  private void setLocatedBlocksTimeStamp(long timeStamp) {
-    this.locatedBlocksTimeStamp = timeStamp;
-  }
-
   /**
    * Grab the open-file info from namenode
    * @param refreshLocatedBlocks whether to re-fetch locatedblocks
@@ -248,33 +230,50 @@ private void setLocatedBlocksTimeStamp(long timeStamp) {
   void openInfo(boolean refreshLocatedBlocks) throws IOException {
     final DfsClientConf conf = dfsClient.getConf();
     synchronized(infoLock) {
-      lastBlockBeingWrittenLength =
-          fetchLocatedBlocksAndGetLastBlockLength(refreshLocatedBlocks);
       int retriesForLastBlockLength = conf.getRetryTimesForGetLastBlockLength();
-      while (retriesForLastBlockLength > 0) {
+
+      while (true) {
+        LocatedBlocks newLocatedBlocks;
+        if (locatedBlocks == null || refreshLocatedBlocks) {
+          newLocatedBlocks = fetchAndCheckLocatedBlocks(locatedBlocks);
+        } else {
+          newLocatedBlocks = locatedBlocks;
+        }
+
+        long lastBlockLength = getLastBlockLength(newLocatedBlocks);
+        if (lastBlockLength != -1) {
+          setLocatedBlocksFields(newLocatedBlocks, lastBlockLength);
+          return;
+        }
+
         // Getting last block length as -1 is a special case. When cluster
         // restarts, DNs may not report immediately. At this time partial block
         // locations will not be available with NN for getting the length. Lets
         // retry for 3 times to get the length.
-        if (lastBlockBeingWrittenLength == -1) {
-          DFSClient.LOG.warn("Last block locations not available. "
-              + "Datanodes might not have reported blocks completely."
-              + " Will retry for " + retriesForLastBlockLength + " times");
-          waitFor(conf.getRetryIntervalForGetLastBlockLength());
-          lastBlockBeingWrittenLength =
-              fetchLocatedBlocksAndGetLastBlockLength(true);
-        } else {
-          break;
+
+        if (retriesForLastBlockLength-- <= 0) {
+          throw new IOException("Could not obtain the last block locations.");
         }
-        retriesForLastBlockLength--;
-      }
-      if (lastBlockBeingWrittenLength == -1
-          && retriesForLastBlockLength == 0) {
-        throw new IOException("Could not obtain the last block locations.");
+
+        DFSClient.LOG.warn("Last block locations not available. "
+            + "Datanodes might not have reported blocks completely."
+            + " Will retry for " + retriesForLastBlockLength + " times");
+        waitFor(conf.getRetryIntervalForGetLastBlockLength());
       }
     }
   }
 
+  /**
+   * Set locatedBlocks and related fields, using the passed lastBlockLength.
+   * Should be called within infoLock.
+   */
+  private void setLocatedBlocksFields(LocatedBlocks locatedBlocksToSet, long lastBlockLength) {
+    locatedBlocks = locatedBlocksToSet;
+    lastBlockBeingWrittenLength = lastBlockLength;
+    fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();
+    setLastRefreshedBlocksAt();
+  }
+
   private void waitFor(int waitTime) throws IOException {
     try {
       Thread.sleep(waitTime);
@@ -285,62 +284,18 @@ private void waitFor(int waitTime) throws IOException {
     }
   }
 
-  /**
-   * Checks whether the block locations timestamps have expired.
-   * In the case of expired timestamp:
-   *    - clear list of deadNodes
-   *    - call openInfo(true) which will re-fetch locatedblocks
-   *    - update locatedBlocksTimeStamp
-   * @return true when the expiration feature is enabled and locatedblocks
-   *         timestamp has expired.
-   * @throws IOException
-   */
-  private boolean isLocatedBlocksExpired() {
-    if (!isPeriodicRefreshEnabled()) {
-      return false;
-    }
-    long now = Time.monotonicNow();
-    long elapsed = now - locatedBlocksTimeStamp;
-    if (elapsed < refreshReadBlockIntervals) {
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Update the block locations timestamps if they have expired.
-   * In the case of expired timestamp:
-   *    - clear list of deadNodes
-   *    - call openInfo(true) which will re-fetch locatedblocks
-   *    - update locatedBlocksTimeStamp
-   * @return true when the locatedblocks list is re-fetched from the namenode.
-   * @throws IOException
-   */
-  private boolean updateBlockLocationsStamp() throws IOException {
-    if (!isLocatedBlocksExpired()) {
-      return false;
-    }
-    // clear dead nodes
-    deadNodes.clear();
-    openInfo(true);
-    setLocatedBlocksTimeStamp();
-    return true;
-  }
-
-  private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh)
+  private LocatedBlocks fetchAndCheckLocatedBlocks(LocatedBlocks existing)
       throws IOException {
-    LocatedBlocks newInfo = locatedBlocks;
-    if (locatedBlocks == null || refresh) {
-      newInfo = dfsClient.getLocatedBlocks(src, 0);
-    }
+    LocatedBlocks newInfo = dfsClient.getLocatedBlocks(src, 0);
+
     DFSClient.LOG.debug("newInfo = {}", newInfo);
     if (newInfo == null) {
       throw new IOException("Cannot open filename " + src);
     }
 
-    if (locatedBlocks != null) {
+    if (existing != null) {
       Iterator<LocatedBlock> oldIter =
-          locatedBlocks.getLocatedBlocks().iterator();
+          existing.getLocatedBlocks().iterator();
       Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator();
       while (oldIter.hasNext() && newIter.hasNext()) {
         if (!oldIter.next().getBlock().equals(newIter.next().getBlock())) {
@@ -348,17 +303,14 @@ private long fetchLocatedBlocksAndGetLastBlockLength(boolean refresh)
         }
       }
     }
-    locatedBlocks = newInfo;
-    long lastBlkBeingWrittenLength = getLastBlockLength();
-    fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();
 
-    return lastBlkBeingWrittenLength;
+    return newInfo;
   }
 
-  private long getLastBlockLength() throws IOException{
+  private long getLastBlockLength(LocatedBlocks blocks) throws IOException{
     long lastBlockBeingWrittenLength = 0;
-    if (!locatedBlocks.isLastBlockComplete()) {
-      final LocatedBlock last = locatedBlocks.getLastLocatedBlock();
+    if (!blocks.isLastBlockComplete()) {
+      final LocatedBlock last = blocks.getLastLocatedBlock();
       if (last != null) {
         if (last.getLocations().length == 0) {
           if (last.getBlockSize() == 0) {
@@ -501,6 +453,14 @@ public List<LocatedBlock> getAllBlocks() throws IOException {
     return getBlockRange(0, getFileLength());
   }
 
+  protected String getSrc() {
+    return src;
+  }
+
+  protected LocatedBlocks getLocatedBlocks() {
+    return locatedBlocks;
+  }
+
   /**
    * Get block at the specified position.
    * Fetch it from the namenode if not cached.
@@ -543,8 +503,8 @@ protected LocatedBlock fetchBlockAt(long offset) throws IOException {
   /** Fetch a block from namenode and cache it */
   private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache)
       throws IOException {
+    maybeRegisterBlockRefresh();
     synchronized(infoLock) {
-      updateBlockLocationsStamp();
       int targetBlockIdx = locatedBlocks.findBlock(offset);
       if (targetBlockIdx < 0) { // block is not cached
         targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
@@ -559,8 +519,7 @@ private LocatedBlock fetchBlockAt(long offset, long length, boolean useCache)
         }
         // Update the LastLocatedBlock, if offset is for last block.
         if (offset >= locatedBlocks.getFileLength()) {
-          locatedBlocks = newBlocks;
-          lastBlockBeingWrittenLength = getLastBlockLength();
+          setLocatedBlocksFields(newBlocks, getLastBlockLength(newBlocks));
         } else {
           locatedBlocks.insertRange(targetBlockIdx,
               newBlocks.getLocatedBlocks());
@@ -587,6 +546,7 @@ private List<LocatedBlock> getBlockRange(long offset,
       throw new IOException("Offset: " + offset +
         " exceeds file length: " + getFileLength());
     }
+
     synchronized(infoLock) {
       final List<LocatedBlock> blocks;
       final long lengthOfCompleteBlk = locatedBlocks.getFileLength();
@@ -644,6 +604,9 @@ private synchronized DatanodeInfo blockSeekTo(long target)
     if (target >= getFileLength()) {
       throw new IOException("Attempted to read past end of file");
     }
+
+    maybeRegisterBlockRefresh();
+
     // Will be getting a new BlockReader.
     closeCurrentBlockReaders();
 
@@ -657,9 +620,6 @@ private synchronized DatanodeInfo blockSeekTo(long target)
     boolean connectFailedOnce = false;
 
     while (true) {
-      // Re-fetch the locatedBlocks from NN if the timestamp has expired.
-      updateBlockLocationsStamp();
-
       //
       // Compute desired block
       //
@@ -793,6 +753,7 @@ public void accept(ByteBuffer k, Object v) {
        * this dfsInputStream anymore.
        */
       dfsClient.removeNodeFromDeadNodeDetector(this, locatedBlocks);
+      maybeDeRegisterBlockRefresh();
     }
   }
 
@@ -871,16 +832,16 @@ protected synchronized int readWithStrategy(ReaderStrategy strategy)
     int len = strategy.getTargetLength();
     CorruptedBlocks corruptedBlocks = new CorruptedBlocks();
     failures = 0;
+
+    maybeRegisterBlockRefresh();
+
     if (pos < getFileLength()) {
       int retries = 2;
       while (retries > 0) {
         try {
           // currentNode can be left as null if previous read had a checksum
           // error on the same block. See HDFS-3067
-          // currentNode needs to be updated if the blockLocations timestamp has
-          // expired.
-          if (pos > blockEnd || currentNode == null
-              || updateBlockLocationsStamp()) {
+          if (pos > blockEnd || currentNode == null) {
             currentNode = blockSeekTo(pos);
           }
           int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
@@ -1003,7 +964,7 @@ private LocatedBlock refetchLocations(LocatedBlock block,
       String description = "Could not obtain block: " + blockInfo;
       DFSClient.LOG.warn(description + errMsg
           + ". Throwing a BlockMissingException");
-      throw new BlockMissingException(src, description,
+      throw new BlockMissingException(src, description + errMsg,
           block.getStartOffset());
     }
 
@@ -1958,4 +1919,153 @@ public boolean hasCapability(String capability) {
       return false;
     }
   }
+
+  /**
+   * Many DFSInputStreams can be opened and closed in quick succession, in which case
+   * they would be registered/deregistered but never need to be refreshed.
+   * Defers registering with the located block refresher, in order to avoid an additional
+   * source of unnecessary synchronization for short-lived DFSInputStreams.
+   */
+  protected void maybeRegisterBlockRefresh() {
+    if (!dfsClient.getConf().isRefreshReadBlockLocationsAutomatically()
+        || !dfsClient.getConf().isLocatedBlocksRefresherEnabled()) {
+      return;
+    }
+
+    if (refreshingBlockLocations.get()) {
+      return;
+    }
+
+    // not enough time elapsed to refresh
+    long timeSinceLastRefresh = Time.monotonicNow() - lastRefreshedBlocksAt;
+    if (timeSinceLastRefresh < dfsClient.getConf().getLocatedBlocksRefresherInterval()) {
+      return;
+    }
+
+    if (!refreshingBlockLocations.getAndSet(true)) {
+      dfsClient.addLocatedBlocksRefresh(this);
+    }
+  }
+
+  /**
+   * De-register periodic refresh of this inputstream, if it was added to begin with.
+   */
+  private void maybeDeRegisterBlockRefresh() {
+    if (refreshingBlockLocations.get()) {
+      dfsClient.removeLocatedBlocksRefresh(this);
+    }
+  }
+
+  /**
+   * Refresh blocks for the input stream, if necessary.
+   *
+   * @param addressCache optional map to use as a cache for resolving datanode InetSocketAddress
+   * @return whether a refresh was performed or not
+   */
+  boolean refreshBlockLocations(Map<String, InetSocketAddress> addressCache) {
+    LocatedBlocks blocks;
+    synchronized (infoLock) {
+      blocks = getLocatedBlocks();
+    }
+
+    if (getLocalDeadNodes().isEmpty() && allBlocksLocal(blocks, addressCache)) {
+      return false;
+    }
+
+    try {
+      DFSClient.LOG.debug("Refreshing {} for path {}", this, getSrc());
+      LocatedBlocks newLocatedBlocks = fetchAndCheckLocatedBlocks(blocks);
+      long lastBlockLength = getLastBlockLength(newLocatedBlocks);
+      if (lastBlockLength == -1) {
+        DFSClient.LOG.debug(
+            "Discarding refreshed blocks for path {} because lastBlockLength was -1",
+            getSrc());
+        return true;
+      }
+
+      setRefreshedValues(newLocatedBlocks, lastBlockLength);
+    } catch (IOException e) {
+      DFSClient.LOG.debug("Failed to refresh DFSInputStream for path {}", getSrc(), e);
+    }
+
+    return true;
+  }
+
+  /**
+   * Once new LocatedBlocks have been fetched, sets them on the DFSInputStream and
+   * updates stateful read location within the necessary locks.
+   */
+  private synchronized void setRefreshedValues(LocatedBlocks blocks, long lastBlockLength)
+      throws IOException {
+    synchronized (infoLock) {
+      setLocatedBlocksFields(blocks, lastBlockLength);
+    }
+
+    getLocalDeadNodes().clear();
+
+    // if a stateful read has been initialized, refresh it
+    if (currentNode != null) {
+      currentNode = blockSeekTo(pos);
+    }
+  }
+
+  private boolean allBlocksLocal(LocatedBlocks blocks,
+      Map<String, InetSocketAddress> addressCache) {
+    if (addressCache == null) {
+      addressCache = new HashMap<>();
+    }
+
+    // we only need to check the first location of each block, because the blocks are already
+    // sorted by distance from the current host
+    for (LocatedBlock lb : blocks.getLocatedBlocks()) {
+      if (lb.getLocations().length == 0) {
+        return false;
+      }
+
+      DatanodeInfoWithStorage location = lb.getLocations()[0];
+      if (location == null) {
+        return false;
+      }
+
+      InetSocketAddress targetAddr = addressCache.computeIfAbsent(
+          location.getDatanodeUuid(),
+          unused -> {
+            String dnAddr = location.getXferAddr(dfsClient.getConf().isConnectToDnViaHostname());
+            return NetUtils.createSocketAddr(
+                dnAddr,
+                -1,
+                null,
+                dfsClient.getConf().isUriCacheEnabled());
+          });
+
+      if (!isResolveableAndLocal(targetAddr)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  private boolean isResolveableAndLocal(InetSocketAddress targetAddr) {
+    try {
+      return DFSUtilClient.isLocalAddress(targetAddr);
+    } catch (IOException e) {
+      DFSClient.LOG.debug("Got an error checking if {} is local", targetAddr, e);
+      return false;
+    }
+  }
+
+  @VisibleForTesting
+  void setLastRefreshedBlocksAtForTesting(long timestamp) {
+    lastRefreshedBlocksAt = timestamp;
+  }
+
+  @VisibleForTesting
+  long getLastRefreshedBlocksAtForTesting() {
+    return lastRefreshedBlocksAt;
+  }
+
+  private void setLastRefreshedBlocksAt() {
+    lastRefreshedBlocksAt = Time.monotonicNow();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 0cf11d4c19687..6ddd56cf72703 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -73,7 +73,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY;
@@ -113,6 +113,8 @@ public class DFSOutputStream extends FSOutputSummer
 
   protected final String src;
   protected final long fileId;
+  private final String namespace;
+  private final String uniqKey;
   protected final long blockSize;
   protected final int bytesPerChecksum;
 
@@ -195,6 +197,15 @@ private DFSOutputStream(DFSClient dfsClient, String src,
     this.dfsClient = dfsClient;
     this.src = src;
     this.fileId = stat.getFileId();
+    this.namespace = stat.getNamespace();
+    if (this.namespace == null) {
+      String defaultKey = dfsClient.getConfiguration().get(
+          HdfsClientConfigKeys.DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY,
+          HdfsClientConfigKeys.DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY_DEFAULT);
+      this.uniqKey = defaultKey + "_" + this.fileId;
+    } else {
+      this.uniqKey = this.namespace + "_" + this.fileId;
+    }
     this.blockSize = stat.getBlockSize();
     this.blockReplication = stat.getReplication();
     this.fileEncryptionInfo = stat.getFileEncryptionInfo();
@@ -818,7 +829,7 @@ boolean isClosed() {
 
   void setClosed() {
     closed = true;
-    dfsClient.endFileLease(fileId);
+    dfsClient.endFileLease(getUniqKey());
     getStreamer().release();
   }
 
@@ -921,7 +932,7 @@ protected synchronized void closeImpl() throws IOException {
   protected void recoverLease(boolean recoverLeaseOnCloseException) {
     if (recoverLeaseOnCloseException) {
       try {
-        dfsClient.endFileLease(fileId);
+        dfsClient.endFileLease(getUniqKey());
         dfsClient.recoverLease(src);
         leaseRecovered = true;
       } catch (Exception e) {
@@ -1084,6 +1095,16 @@ public long getFileId() {
     return fileId;
   }
 
+  @VisibleForTesting
+  public String getNamespace() {
+    return namespace;
+  }
+
+  @VisibleForTesting
+  public String getUniqKey() {
+    return this.uniqKey;
+  }
+
   /**
    * Return the source of stream.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 95cce9d363e58..5ae517095931a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -141,14 +141,6 @@ protected ByteBuffer getCurStripeBuf() {
     return curStripeBuf;
   }
 
-  protected String getSrc() {
-    return src;
-  }
-
-  protected LocatedBlocks getLocatedBlocks() {
-    return locatedBlocks;
-  }
-
   protected ByteBufferPool getBufferPool() {
     return BUFFER_POOL;
   }
@@ -166,6 +158,8 @@ synchronized void blockSeekTo(long target) throws IOException {
       throw new IOException("Attempted to read past end of file");
     }
 
+    maybeRegisterBlockRefresh();
+
     // Will be getting a new BlockReader.
     closeCurrentBlockReaders();
 
@@ -238,7 +232,7 @@ private long getOffsetInBlockGroup(long pos) {
 
   boolean createBlockReader(LocatedBlock block, long offsetInBlock,
       LocatedBlock[] targetBlocks, BlockReaderInfo[] readerInfos,
-      int chunkIndex) throws IOException {
+      int chunkIndex, long readTo) throws IOException {
     BlockReader reader = null;
     final ReaderRetryPolicy retry = new ReaderRetryPolicy();
     DFSInputStream.DNAddrPair dnInfo =
@@ -256,9 +250,14 @@ boolean createBlockReader(LocatedBlock block, long offsetInBlock,
         if (dnInfo == null) {
           break;
         }
+        if (readTo < 0 || readTo > block.getBlockSize()) {
+          readTo = block.getBlockSize();
+        }
         reader = getBlockReader(block, offsetInBlock,
-            block.getBlockSize() - offsetInBlock,
+            readTo - offsetInBlock,
             dnInfo.addr, dnInfo.storageType, dnInfo.info);
+        DFSClientFaultInjector.get().onCreateBlockReader(block, chunkIndex, offsetInBlock,
+            readTo - offsetInBlock);
       } catch (IOException e) {
         if (e instanceof InvalidEncryptionKeyException &&
             retry.shouldRefetchEncryptionKey()) {
@@ -491,11 +490,16 @@ protected void fetchBlockByteRange(LocatedBlock block, long start,
     final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
         blockGroup, cellSize, dataBlkNum, parityBlkNum);
     final BlockReaderInfo[] preaderInfos = new BlockReaderInfo[groupSize];
+    long readTo = -1;
+    for (AlignedStripe stripe : stripes) {
+      readTo = Math.max(readTo, stripe.getOffsetInBlock() + stripe.getSpanInBlock());
+    }
     try {
       for (AlignedStripe stripe : stripes) {
         // Parse group to get chosen DN location
         StripeReader preader = new PositionStripeReader(stripe, ecPolicy, blks,
             preaderInfos, corruptedBlocks, decoder, this);
+        preader.setReadTo(readTo);
         try {
           preader.readStripe();
         } finally {
@@ -560,4 +564,5 @@ public synchronized void unbuffer() {
       parityBuf = null;
     }
   }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index bd7ff048dc8a9..1233c033ee0e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.CreateFlag;
@@ -1055,7 +1055,7 @@ void abort() throws IOException {
       }
     }
 
-    dfsClient.endFileLease(fileId);
+    dfsClient.endFileLease(getUniqKey());
     final IOException ioe = b.build();
     if (ioe != null) {
       throw ioe;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 6f54de05eb6ac..1a1819dfa2c5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -17,9 +17,11 @@
  */
 package org.apache.hadoop.hdfs;
 
+import org.apache.commons.collections.list.TreeList;
+import org.apache.hadoop.ipc.RpcNoSuchMethodException;
 import org.apache.hadoop.net.DomainNameResolver;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes;
 import java.net.URISyntaxException;
@@ -30,6 +32,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator;
 import org.apache.hadoop.hdfs.net.BasicInetPeer;
 import org.apache.hadoop.hdfs.net.NioInetPeer;
 import org.apache.hadoop.hdfs.net.Peer;
@@ -43,8 +46,11 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
 import org.apache.hadoop.hdfs.protocol.ReconfigurationProtocol;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry;
 import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
 import org.apache.hadoop.hdfs.protocolPB.ReconfigurationProtocolTranslatorPB;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
@@ -55,6 +61,7 @@
 import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.Daemon;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
@@ -145,7 +152,10 @@ public static byte[][] bytes2byteArray(byte[] bytes) {
    */
   public static byte[][] bytes2byteArray(byte[] bytes, int len,
       byte separator) {
-    Preconditions.checkPositionIndex(len, bytes.length);
+    if (len < 0 || len > bytes.length) {
+      throw new IndexOutOfBoundsException(
+          "Incorrect index [len, size] [" + len + ", " + bytes.length + "]");
+    }
     if (len == 0) {
       return new byte[][]{null};
     }
@@ -357,6 +367,13 @@ public static byte[] byteArray2bytes(byte[][] pathComponents) {
     return path;
   }
 
+  /**
+   * Given a list of path components returns a string.
+   */
+  public static String byteArray2String(byte[][] pathComponents) {
+    return bytes2String(byteArray2bytes(pathComponents));
+  }
+
   /**
    * Decode a specific range of bytes of the given byte array to a string
    * using UTF8.
@@ -716,13 +733,13 @@ public static boolean isLocalAddress(InetSocketAddress targetAddr)
     InetAddress addr = targetAddr.getAddress();
     Boolean cached = localAddrMap.get(addr.getHostAddress());
     if (cached != null) {
-      LOG.trace("Address {} is {} local", targetAddr, (cached ? "" : "not"));
+      LOG.trace("Address {} is{} local", targetAddr, (cached ? "" : " not"));
       return cached;
     }
 
     boolean local = NetUtils.isLocalAddress(addr);
 
-    LOG.trace("Address {} is {} local", targetAddr, (local ? "" : "not"));
+    LOG.trace("Address {} is{} local", targetAddr, (local ? "" : " not"));
     localAddrMap.put(addr.getHostAddress(), local);
     return local;
   }
@@ -1126,4 +1143,68 @@ public static String getSnapshotTrashRoot(String ssRoot,
     return (ssRoot.equals("/") ? ssRoot : ssRoot + Path.SEPARATOR)
         + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName();
   }
+
+  /**
+   * Returns true if the name of snapshot is vlaid.
+   * @param snapshotName name of the snapshot.
+   * @return true if the name of snapshot is vlaid.
+   */
+  public static boolean isValidSnapshotName(String snapshotName) {
+    // If any of the snapshots specified in the getSnapshotDiffReport call
+    // is null or empty, it points to the current tree.
+    return (snapshotName != null && !snapshotName.isEmpty());
+  }
+
+  public static SnapshotDiffReport getSnapshotDiffReport(
+      String snapshotDir, String fromSnapshot, String toSnapshot,
+      SnapshotDiffReportFunction withoutListing,
+      SnapshotDiffReportListingFunction withListing) throws IOException {
+    // In case the diff needs to be computed between a snapshot and the current
+    // tree, we should not do iterative diffReport computation as the iterative
+    // approach might fail if in between the rpc calls the current tree
+    // changes in absence of the global fsn lock.
+    if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName(toSnapshot)) {
+      return withoutListing.apply(snapshotDir, fromSnapshot, toSnapshot);
+    }
+    byte[] startPath = EMPTY_BYTES;
+    int index = -1;
+    SnapshotDiffReportGenerator snapshotDiffReport;
+    List<DiffReportListingEntry> modifiedList = new TreeList();
+    List<DiffReportListingEntry> createdList = new ChunkedArrayList<>();
+    List<DiffReportListingEntry> deletedList = new ChunkedArrayList<>();
+    SnapshotDiffReportListing report;
+    do {
+      try {
+        report = withListing.apply(snapshotDir, fromSnapshot, toSnapshot, startPath, index);
+      } catch (RpcNoSuchMethodException|UnsupportedOperationException e) {
+        // In case the server doesn't support getSnapshotDiffReportListing,
+        // fallback to getSnapshotDiffReport.
+        LOG.warn("Falling back to getSnapshotDiffReport {}", e.getMessage());
+        return withoutListing.apply(snapshotDir, fromSnapshot, toSnapshot);
+      }
+      startPath = report.getLastPath();
+      index = report.getLastIndex();
+      modifiedList.addAll(report.getModifyList());
+      createdList.addAll(report.getCreateList());
+      deletedList.addAll(report.getDeleteList());
+    } while (!(Arrays.equals(startPath, EMPTY_BYTES)
+        && index == -1));
+    snapshotDiffReport =
+        new SnapshotDiffReportGenerator(snapshotDir, fromSnapshot, toSnapshot,
+            report.getIsFromEarlier(), modifiedList, createdList, deletedList);
+    return snapshotDiffReport.generateReport();
+  }
+
+  @FunctionalInterface
+  public interface SnapshotDiffReportFunction {
+    SnapshotDiffReport apply(String snapshotDir, String fromSnapshot, String toSnapshot)
+        throws IOException;
+  }
+
+  @FunctionalInterface
+  public interface SnapshotDiffReportListingFunction {
+    SnapshotDiffReportListing apply(String snapshotDir, String fromSnapshot, String toSnapshot,
+        byte[] startPath, int index)
+        throws IOException;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index c1e25af466d9d..4fa578ab6c03f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -38,6 +38,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -525,11 +526,13 @@ boolean doWaitForRestart() {
   // List of congested data nodes. The stream will back off if the DataNodes
   // are congested
   private final List<DatanodeInfo> congestedNodes = new ArrayList<>();
+  private final Map<DatanodeInfo, Integer> slowNodeMap = new HashMap<>();
   private static final int CONGESTION_BACKOFF_MEAN_TIME_IN_MS = 5000;
   private static final int CONGESTION_BACK_OFF_MAX_TIME_IN_MS =
       CONGESTION_BACKOFF_MEAN_TIME_IN_MS * 10;
   private int lastCongestionBackoffTime;
   private int maxPipelineRecoveryRetries;
+  private int markSlowNodeAsBadNodeThreshold;
 
   protected final LoadingCache<DatanodeInfo, DatanodeInfo> excludedNodes;
   private final String[] favoredNodes;
@@ -559,6 +562,7 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block,
     this.errorState = new ErrorState(conf.getDatanodeRestartTimeout());
     this.addBlockFlags = flags;
     this.maxPipelineRecoveryRetries = conf.getMaxPipelineRecoveryRetries();
+    this.markSlowNodeAsBadNodeThreshold = conf.getMarkSlowNodeAsBadNodeThreshold();
   }
 
   /**
@@ -687,11 +691,6 @@ public void run() {
             continue;
           }
           // get packet to be sent.
-          try {
-            backOffIfNecessary();
-          } catch (InterruptedException e) {
-            LOG.debug("Thread interrupted", e);
-          }
           one = dataQueue.getFirst(); // regular data packet
           SpanContext[] parents = one.getTraceParents();
           if (parents != null && parents.length > 0) {
@@ -704,6 +703,14 @@ public void run() {
           }
         }
 
+        // The DataStreamer has to release the dataQueue before sleeping,
+        // otherwise it will cause the ResponseProcessor to accept the ACK delay.
+        try {
+          backOffIfNecessary();
+        } catch (InterruptedException e) {
+          LOG.debug("Thread interrupted", e);
+        }
+
         // get new block from namenode.
         LOG.debug("stage={}, {}", stage, this);
 
@@ -1152,6 +1159,7 @@ public void run() {
           long seqno = ack.getSeqno();
           // processes response status from datanodes.
           ArrayList<DatanodeInfo> congestedNodesFromAck = new ArrayList<>();
+          ArrayList<DatanodeInfo> slownodesFromAck = new ArrayList<>();
           for (int i = ack.getNumOfReplies()-1; i >=0  && dfsClient.clientRunning; i--) {
             final Status reply = PipelineAck.getStatusFromHeader(ack
                 .getHeaderFlag(i));
@@ -1159,6 +1167,10 @@ public void run() {
                 PipelineAck.ECN.CONGESTED) {
               congestedNodesFromAck.add(targets[i]);
             }
+            if (PipelineAck.getSLOWFromHeader(ack.getHeaderFlag(i)) ==
+                PipelineAck.SLOW.SLOW) {
+              slownodesFromAck.add(targets[i]);
+            }
             // Restart will not be treated differently unless it is
             // the local node or the only one in the pipeline.
             if (PipelineAck.isRestartOOBStatus(reply)) {
@@ -1188,6 +1200,16 @@ public void run() {
             }
           }
 
+          if (slownodesFromAck.isEmpty()) {
+            if (!slowNodeMap.isEmpty()) {
+              slowNodeMap.clear();
+            }
+          } else {
+            markSlowNode(slownodesFromAck);
+            LOG.debug("SlowNodeMap content: {}.", slowNodeMap);
+          }
+
+
           assert seqno != PipelineAck.UNKOWN_SEQNO :
               "Ack for unknown seqno should be a failed ack: " + ack;
           if (seqno == DFSPacket.HEART_BEAT_SEQNO) {  // a heartbeat ack
@@ -1254,10 +1276,51 @@ public void run() {
       }
     }
 
+    void markSlowNode(List<DatanodeInfo> slownodesFromAck) throws IOException {
+      Set<DatanodeInfo> discontinuousNodes = new HashSet<>(slowNodeMap.keySet());
+      for (DatanodeInfo slowNode : slownodesFromAck) {
+        if (!slowNodeMap.containsKey(slowNode)) {
+          slowNodeMap.put(slowNode, 1);
+        } else {
+          int oldCount = slowNodeMap.get(slowNode);
+          slowNodeMap.put(slowNode, ++oldCount);
+        }
+        discontinuousNodes.remove(slowNode);
+      }
+      for (DatanodeInfo discontinuousNode : discontinuousNodes) {
+        slowNodeMap.remove(discontinuousNode);
+      }
+
+      if (!slowNodeMap.isEmpty()) {
+        for (Map.Entry<DatanodeInfo, Integer> entry : slowNodeMap.entrySet()) {
+          if (entry.getValue() >= markSlowNodeAsBadNodeThreshold) {
+            DatanodeInfo slowNode = entry.getKey();
+            int index = getDatanodeIndex(slowNode);
+            if (index >= 0) {
+              errorState.setBadNodeIndex(index);
+              throw new IOException("Receive reply from slowNode " + slowNode +
+                  " for continuous " + markSlowNodeAsBadNodeThreshold +
+                  " times, treating it as badNode");
+            }
+            slowNodeMap.remove(entry.getKey());
+          }
+        }
+      }
+    }
+
     void close() {
       responderClosed = true;
       this.interrupt();
     }
+
+    int getDatanodeIndex(DatanodeInfo datanodeInfo) {
+      for (int i = 0; i < targets.length; i++) {
+        if (targets[i].equals(datanodeInfo)) {
+          return i;
+        }
+      }
+      return -1;
+    }
   }
 
   private boolean shouldHandleExternalError(){
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 449d0562b7070..9050a4bddeece 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -18,12 +18,10 @@
 
 package org.apache.hadoop.hdfs;
 
-
-import org.apache.hadoop.ipc.RpcNoSuchMethodException;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.commons.collections.list.TreeList;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -107,8 +105,6 @@
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
-import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry;
-import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -116,7 +112,6 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.DelegationTokenIssuer;
-import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
@@ -427,6 +422,16 @@ public FSDataOutputStream append(Path f, final int bufferSize,
     return append(f, EnumSet.of(CreateFlag.APPEND), bufferSize, progress);
   }
 
+  @Override
+  public FSDataOutputStream append(Path f, final int bufferSize,
+      final Progressable progress, boolean appendToNewBlock) throws IOException {
+    EnumSet<CreateFlag> flag = EnumSet.of(CreateFlag.APPEND);
+    if (appendToNewBlock) {
+      flag.add(CreateFlag.NEW_BLOCK);
+    }
+    return append(f, flag, bufferSize, progress);
+  }
+
   /**
    * Append to an existing file (optional operation).
    *
@@ -573,7 +578,7 @@ public FSDataOutputStream next(final FileSystem fs, final Path p)
 
   /**
    * Same as
-   * {@link #create(Path, FsPermission, EnumSet<CreateFlag>, int, short, long,
+   * {@link #create(Path, FsPermission, EnumSet, int, short, long,
    * Progressable, ChecksumOpt)} with a few additions. First, addition of
    * favoredNodes that is a hint to where the namenode should place the file
    * blocks. The favored nodes hint is not persisted in HDFS. Hence it may be
@@ -642,12 +647,12 @@ protected HdfsDataOutputStream primitiveCreate(Path f,
 
   /**
    * Similar to {@link #create(Path, FsPermission, EnumSet, int, short, long,
-   * Progressable, ChecksumOpt, InetSocketAddress[], String)}, it provides a
+   * Progressable, ChecksumOpt, InetSocketAddress[], String, String)}, it provides a
    * HDFS-specific version of {@link #createNonRecursive(Path, FsPermission,
    * EnumSet, int, short, long, Progressable)} with a few additions.
    *
    * @see #create(Path, FsPermission, EnumSet, int, short, long, Progressable,
-   * ChecksumOpt, InetSocketAddress[], String) for the descriptions of
+   * ChecksumOpt, InetSocketAddress[], String, String) for the descriptions of
    * additional parameters, i.e., favoredNodes, ecPolicyName and
    * storagePolicyName.
    */
@@ -2298,8 +2303,8 @@ <SnapshotDiffReportListing> snapshotDiffReportListingRemoteIterator(
       @Override
       public RemoteIterator<SnapshotDiffReportListing> doCall(final Path p)
           throws IOException {
-        if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName(
-            toSnapshot)) {
+        if (!DFSUtilClient.isValidSnapshotName(fromSnapshot) ||
+            !DFSUtilClient.isValidSnapshotName(toSnapshot)) {
           throw new UnsupportedOperationException("Remote Iterator is"
               + "supported for snapshotDiffReport between two snapshots");
         }
@@ -2364,52 +2369,11 @@ public SnapshotDiffReportListing next() throws IOException {
     }
   }
 
-  private boolean isValidSnapshotName(String snapshotName) {
-    // If any of the snapshots specified in the getSnapshotDiffReport call
-    // is null or empty, it points to the current tree.
-    return (snapshotName != null && !snapshotName.isEmpty());
-  }
-
   private SnapshotDiffReport getSnapshotDiffReportInternal(
       final String snapshotDir, final String fromSnapshot,
       final String toSnapshot) throws IOException {
-    // In case the diff needs to be computed between a snapshot and the current
-    // tree, we should not do iterative diffReport computation as the iterative
-    // approach might fail if in between the rpc calls the current tree
-    // changes in absence of the global fsn lock.
-    if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName(
-        toSnapshot)) {
-      return dfs.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
-    }
-    byte[] startPath = DFSUtilClient.EMPTY_BYTES;
-    int index = -1;
-    SnapshotDiffReportGenerator snapshotDiffReport;
-    List<DiffReportListingEntry> modifiedList = new TreeList();
-    List<DiffReportListingEntry> createdList = new ChunkedArrayList<>();
-    List<DiffReportListingEntry> deletedList = new ChunkedArrayList<>();
-    SnapshotDiffReportListing report;
-    do {
-      try {
-        report = dfs.getSnapshotDiffReportListing(snapshotDir, fromSnapshot,
-            toSnapshot, startPath, index);
-      } catch (RpcNoSuchMethodException e) {
-        // In case the server doesn't support getSnapshotDiffReportListing,
-        // fallback to getSnapshotDiffReport.
-        DFSClient.LOG.warn(
-            "Falling back to getSnapshotDiffReport {}", e.getMessage());
-        return dfs.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
-      }
-      startPath = report.getLastPath();
-      index = report.getLastIndex();
-      modifiedList.addAll(report.getModifyList());
-      createdList.addAll(report.getCreateList());
-      deletedList.addAll(report.getDeleteList());
-    } while (!(Arrays.equals(startPath, DFSUtilClient.EMPTY_BYTES)
-        && index == -1));
-    snapshotDiffReport =
-        new SnapshotDiffReportGenerator(snapshotDir, fromSnapshot, toSnapshot,
-            report.getIsFromEarlier(), modifiedList, createdList, deletedList);
-    return snapshotDiffReport.generateReport();
+    return  DFSUtilClient.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot,
+        dfs::getSnapshotDiffReport, dfs::getSnapshotDiffReportListing);
   }
 
   /**
@@ -2447,6 +2411,51 @@ public SnapshotDiffReport next(final FileSystem fs, final Path p)
     }.resolve(this, absF);
   }
 
+  /**
+   * Get the difference between two snapshots of a directory iteratively.
+   *
+   * @param snapshotDir full path of the directory where snapshots are taken.
+   * @param fromSnapshotName snapshot name of the from point. Null indicates the current tree.
+   * @param toSnapshotName snapshot name of the to point. Null indicates the current tree.
+   * @param snapshotDiffStartPath path relative to the snapshottable root directory from where
+   *     the snapshotdiff computation needs to start.
+   * @param snapshotDiffIndex index in the created or deleted list of the directory at which the
+   *     snapshotdiff computation stopped during the last rpc call. -1 indicates the diff
+   *     computation needs to start right from the start path.
+   * @return the difference report represented as a {@link SnapshotDiffReportListing}.
+   * @throws IOException if an I/O error occurred.
+   */
+  public SnapshotDiffReportListing getSnapshotDiffReportListing(Path snapshotDir,
+      String fromSnapshotName, String toSnapshotName, String snapshotDiffStartPath,
+      int snapshotDiffIndex) throws IOException {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF);
+    Path absF = fixRelativePart(snapshotDir);
+    return new FileSystemLinkResolver<SnapshotDiffReportListing>() {
+
+      @Override
+      public SnapshotDiffReportListing doCall(final Path p) throws IOException {
+        return dfs.getSnapshotDiffReportListing(getPathName(p), fromSnapshotName, toSnapshotName,
+            DFSUtilClient.string2Bytes(snapshotDiffStartPath), snapshotDiffIndex);
+      }
+
+      @Override
+      public SnapshotDiffReportListing next(final FileSystem fs, final Path p)
+          throws IOException {
+        if (fs instanceof DistributedFileSystem) {
+          DistributedFileSystem distributedFileSystem = (DistributedFileSystem)fs;
+          distributedFileSystem.getSnapshotDiffReportListing(p, fromSnapshotName, toSnapshotName,
+              snapshotDiffStartPath, snapshotDiffIndex);
+        } else {
+          throw new UnsupportedOperationException("Cannot perform snapshot"
+              + " operations on a symlink to a non-DistributedFileSystem: "
+              + snapshotDir + " -> " + p);
+        }
+        return null;
+      }
+    }.resolve(this, absF);
+  }
+
   /**
    * Get the close status of a file
    * @param src The path to the file
@@ -3889,4 +3898,47 @@ public MultipartUploaderBuilder createMultipartUploader(final Path basePath)
       throws IOException {
     return new FileSystemMultipartUploaderBuilder(this, basePath);
   }
+
+  /**
+   * Retrieve stats for slow running datanodes.
+   *
+   * @return An array of slow datanode info.
+   * @throws IOException If an I/O error occurs.
+   */
+  public DatanodeInfo[] getSlowDatanodeStats() throws IOException {
+    return dfs.slowDatanodeReport();
+  }
+
+  /**
+   * Returns LocatedBlocks of the corresponding HDFS file p from offset start
+   * for length len.
+   * This is similar to {@link #getFileBlockLocations(Path, long, long)} except
+   * that it returns LocatedBlocks rather than BlockLocation array.
+   * @param p path representing the file of interest.
+   * @param start offset
+   * @param len length
+   * @return a LocatedBlocks object
+   * @throws IOException
+   */
+  public LocatedBlocks getLocatedBlocks(Path p, long start, long len)
+      throws IOException {
+    final Path absF = fixRelativePart(p);
+    return new FileSystemLinkResolver<LocatedBlocks>() {
+      @Override
+      public LocatedBlocks doCall(final Path p) throws IOException {
+        return dfs.getLocatedBlocks(getPathName(p), start, len);
+      }
+      @Override
+      public LocatedBlocks next(final FileSystem fs, final Path p)
+          throws IOException {
+        if (fs instanceof DistributedFileSystem) {
+          DistributedFileSystem myDfs = (DistributedFileSystem)fs;
+          return myDfs.getLocatedBlocks(p, start, len);
+        }
+        throw new UnsupportedOperationException("Cannot getLocatedBlocks " +
+            "through a symlink to a non-DistributedFileSystem: " + fs + " -> "+
+            p);
+      }
+    }.resolve(this, absF);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
index c1804a2c22f44..21bc885358076 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java
@@ -303,7 +303,8 @@ FileChecksum makeCompositeCrcResult() throws IOException {
       byte[] blockChecksumBytes = blockChecksumBuf.getData();
 
       long sumBlockLengths = 0;
-      for (int i = 0; i < locatedBlocks.size() - 1; ++i) {
+      int i = 0;
+      for (; i < locatedBlocks.size() - 1; ++i) {
         LocatedBlock block = locatedBlocks.get(i);
         // For everything except the last LocatedBlock, we expect getBlockSize()
         // to accurately reflect the number of file bytes digested in the block
@@ -316,19 +317,8 @@ FileChecksum makeCompositeCrcResult() throws IOException {
             "Added blockCrc 0x{} for block index {} of size {}",
             Integer.toString(blockCrc, 16), i, block.getBlockSize());
       }
-
-      // NB: In some cases the located blocks have their block size adjusted
-      // explicitly based on the requested length, but not all cases;
-      // these numbers may or may not reflect actual sizes on disk.
-      long reportedLastBlockSize =
-          blockLocations.getLastLocatedBlock().getBlockSize();
-      long consumedLastBlockLength = reportedLastBlockSize;
-      if (length - sumBlockLengths < reportedLastBlockSize) {
-        LOG.warn(
-            "Last block length {} is less than reportedLastBlockSize {}",
-            length - sumBlockLengths, reportedLastBlockSize);
-        consumedLastBlockLength = length - sumBlockLengths;
-      }
+      LocatedBlock nextBlock = locatedBlocks.get(i);
+      long consumedLastBlockLength = Math.min(length - sumBlockLengths, nextBlock.getBlockSize());
       // NB: blockChecksumBytes.length may be much longer than actual bytes
       // written into the DataOutput.
       int lastBlockCrc = CrcUtil.readInt(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
index c70081c06fa49..d8dd485101bce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.util.KMSUtil;
 
 import org.apache.hadoop.classification.VisibleForTesting;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener;
 import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification;
 
+import org.apache.hadoop.util.ShutdownHookManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,6 +67,9 @@ public void onRemoval(
           }
         })
         .build();
+
+    ShutdownHookManager.get().addShutdownHook(new KeyProviderCacheFinalizer(),
+        SHUTDOWN_HOOK_PRIORITY);
   }
 
   public KeyProvider get(final Configuration conf,
@@ -85,6 +90,26 @@ public KeyProvider call() throws Exception {
     }
   }
 
+  public static final int SHUTDOWN_HOOK_PRIORITY = FileSystem.SHUTDOWN_HOOK_PRIORITY - 1;
+
+  private class KeyProviderCacheFinalizer implements Runnable {
+    @Override
+    public synchronized void run() {
+      invalidateCache();
+    }
+  }
+
+  /**
+   * Invalidate cache. KeyProviders in the cache will be closed by cache hook.
+   */
+  @VisibleForTesting
+  synchronized void invalidateCache() {
+    LOG.debug("Invalidating all cached KeyProviders.");
+    if (cache != null) {
+      cache.invalidateAll();
+    }
+  }
+
   private URI createKeyProviderURI(Configuration conf) {
     final String providerUriStr = conf.getTrimmed(
         CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java
new file mode 100644
index 0000000000000..454d1f9cd93e4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/LocatedBlocksRefresher.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Periodically refresh the underlying cached {@link LocatedBlocks} for eligible registered
+ * {@link DFSInputStream}s.  DFSInputStreams are eligible for refreshing if they have any
+ * deadNodes or any blocks are lacking local replicas.
+ * Disabled by default, unless an interval is configured.
+ */
+public class LocatedBlocksRefresher extends Daemon {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(LocatedBlocksRefresher.class);
+
+  private static final String THREAD_PREFIX = "located-block-refresher-";
+
+  private final String name;
+  private final long interval;
+  private final long jitter;
+  private final ExecutorService refreshThreadPool;
+
+  // Use WeakHashMap so that we don't hold onto references that might have not been explicitly
+  // closed because they were created and thrown away.
+  private final Set<DFSInputStream> registeredInputStreams =
+      Collections.newSetFromMap(new WeakHashMap<>());
+
+  private int runCount;
+  private int refreshCount;
+
+  LocatedBlocksRefresher(String name, Configuration conf, DfsClientConf dfsClientConf) {
+    this.name = name;
+    this.interval = dfsClientConf.getLocatedBlocksRefresherInterval();
+    this.jitter = Math.round(this.interval * 0.1);
+    int rpcThreads = conf.getInt(DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY,
+        DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT);
+
+    String threadPrefix;
+    if (name.equals(DFS_CLIENT_CONTEXT_DEFAULT)) {
+      threadPrefix = THREAD_PREFIX;
+    } else {
+      threadPrefix = THREAD_PREFIX + name + "-";
+    }
+
+    this.refreshThreadPool = Executors.newFixedThreadPool(rpcThreads, new Daemon.DaemonFactory() {
+      private final AtomicInteger threadIndex = new AtomicInteger(0);
+
+      @Override
+      public Thread newThread(Runnable r) {
+        Thread t = super.newThread(r);
+        t.setName(threadPrefix + threadIndex.getAndIncrement());
+        return t;
+      }
+    });
+
+    setName(threadPrefix + "main");
+
+    LOG.info("Start located block refresher for DFSClient {}.", this.name);
+  }
+
+  @Override
+  public void run() {
+    while (!Thread.currentThread().isInterrupted()) {
+
+      if (!waitForInterval()) {
+        return;
+      }
+
+      LOG.debug("Running refresh for {} streams", registeredInputStreams.size());
+      long start = Time.monotonicNow();
+      AtomicInteger neededRefresh = new AtomicInteger(0);
+
+      Phaser phaser = new Phaser(1);
+
+      Map<String, InetSocketAddress> addressCache = new ConcurrentHashMap<>();
+
+      for (DFSInputStream inputStream : getInputStreams()) {
+        phaser.register();
+        refreshThreadPool.submit(() -> {
+          try {
+            if (isInputStreamTracked(inputStream) &&
+                inputStream.refreshBlockLocations(addressCache)) {
+              neededRefresh.incrementAndGet();
+            }
+          } finally {
+            phaser.arriveAndDeregister();
+          }
+        });
+      }
+
+      phaser.arriveAndAwaitAdvance();
+
+      synchronized (this) {
+        runCount++;
+        refreshCount += neededRefresh.get();
+      }
+
+      LOG.debug(
+          "Finished refreshing {} of {} streams in {}ms",
+          neededRefresh,
+          registeredInputStreams.size(),
+          Time.monotonicNow() - start
+      );
+    }
+  }
+
+  public synchronized int getRunCount() {
+    return runCount;
+  }
+
+  public synchronized int getRefreshCount() {
+    return refreshCount;
+  }
+
+  private boolean waitForInterval() {
+    try {
+      Thread.sleep(interval + ThreadLocalRandom.current().nextLong(-jitter, jitter));
+      return true;
+    } catch (InterruptedException e) {
+      LOG.debug("Interrupted during wait interval", e);
+      Thread.currentThread().interrupt();
+      return false;
+    }
+  }
+
+  /**
+   * Shutdown all the threads.
+   */
+  public void shutdown() {
+    if (isAlive()) {
+      interrupt();
+      try {
+        join();
+      } catch (InterruptedException e) {
+      }
+    }
+    refreshThreadPool.shutdown();
+  }
+
+  /**
+   * Collects the DFSInputStreams to a list within synchronization, so that we can iterate them
+   * without potentially blocking callers to {@link #addInputStream(DFSInputStream)} or
+   * {@link #removeInputStream(DFSInputStream)}. We don't care so much about missing additions,
+   * and we'll guard against removals by doing an additional
+   * {@link #isInputStreamTracked(DFSInputStream)} track during iteration.
+   */
+  private synchronized Collection<DFSInputStream> getInputStreams() {
+    return new ArrayList<>(registeredInputStreams);
+  }
+
+  public synchronized void addInputStream(DFSInputStream dfsInputStream) {
+    LOG.trace("Registering {} for {}", dfsInputStream, dfsInputStream.getSrc());
+    registeredInputStreams.add(dfsInputStream);
+  }
+
+  public synchronized void removeInputStream(DFSInputStream dfsInputStream) {
+    if (isInputStreamTracked(dfsInputStream)) {
+      LOG.trace("De-registering {} for {}", dfsInputStream, dfsInputStream.getSrc());
+      registeredInputStreams.remove(dfsInputStream);
+    }
+  }
+
+  public synchronized boolean isInputStreamTracked(DFSInputStream dfsInputStream) {
+    return registeredInputStreams.contains(dfsInputStream);
+  }
+
+  public long getInterval() {
+    return interval;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
index 1d8c8e632a38b..2e5532381978d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
@@ -36,7 +36,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -349,6 +349,12 @@ public static ClientProtocol createProxyWithAlignmentContext(
       boolean withRetries, AtomicBoolean fallbackToSimpleAuth,
       AlignmentContext alignmentContext)
       throws IOException {
+    if (alignmentContext == null &&
+        conf.getBoolean(HdfsClientConfigKeys.DFS_RBF_OBSERVER_READ_ENABLE,
+            HdfsClientConfigKeys.DFS_RBF_OBSERVER_READ_ENABLE_DEFAULT)) {
+      alignmentContext = new ClientGSIContext();
+    }
+
     RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
         ProtobufRpcEngine2.class);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java
index abcc7e011ade5..a26a518a8395d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java
@@ -24,7 +24,7 @@
 import java.util.Map.Entry;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.LinkedListMultimap;
 
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
index efadedb8f082f..12328ebb0f01b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
index 8bc6951f593ea..730307b4434cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
@@ -52,7 +52,9 @@ void prepareDecodeInputs() {
       cur = dfsStripedInputStream.getCurStripeBuf().duplicate();
     }
 
-    this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum];
+    if (this.decodeInputs == null) {
+      this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum];
+    }
     int bufLen = (int) alignedStripe.getSpanInBlock();
     int bufOff = (int) alignedStripe.getOffsetInBlock();
     for (int i = 0; i < dataBlkNum; i++) {
@@ -72,11 +74,6 @@ void prepareDecodeInputs() {
   boolean prepareParityChunk(int index) {
     Preconditions.checkState(index >= dataBlkNum
         && alignedStripe.chunks[index] == null);
-    if (readerInfos[index] != null && readerInfos[index].shouldSkip) {
-      alignedStripe.chunks[index] = new StripingChunk(StripingChunk.MISSING);
-      // we have failed the block reader before
-      return false;
-    }
     final int parityIndex = index - dataBlkNum;
     ByteBuffer buf = dfsStripedInputStream.getParityBuffer().duplicate();
     buf.position(cellSize * parityIndex);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
index 69aad06363c6c..3fc87c7952a19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -119,6 +119,7 @@ void skip() {
   protected final int cellSize;
   protected final RawErasureDecoder decoder;
   protected final DFSStripedInputStream dfsStripedInputStream;
+  private long readTo = -1;
 
   protected ECChunk[] decodeInputs;
 
@@ -302,7 +303,7 @@ boolean readChunk(final LocatedBlock block, int chunkIndex)
     if (readerInfos[chunkIndex] == null) {
       if (!dfsStripedInputStream.createBlockReader(block,
           alignedStripe.getOffsetInBlock(), targetBlocks,
-          readerInfos, chunkIndex)) {
+          readerInfos, chunkIndex, readTo)) {
         chunk.state = StripingChunk.MISSING;
         return false;
       }
@@ -478,4 +479,9 @@ void clearFutures() {
   boolean useDirectBuffer() {
     return decoder.preferDirectBuffer();
   }
+
+  public void setReadTo(long readTo) {
+    this.readTo = readTo;
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
index 2a4469a5ae585..d34df37ae6133 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ViewDistributedFileSystem.java
@@ -2318,4 +2318,14 @@ public long getUsed() throws IOException {
     }
     return this.vfs.getUsed();
   }
+
+  @Override
+  public DatanodeInfo[] getSlowDatanodeStats() throws IOException {
+    if (this.vfs == null) {
+      return super.getSlowDatanodeStats();
+    }
+    checkDefaultDFS(defaultDFS, "getSlowDatanodeStats");
+    return defaultDFS.getSlowDatanodeStats();
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java
index ed6dd2fa2093f..7e8812fc13f9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.StringUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/CreateEncryptionZoneFlag.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/CreateEncryptionZoneFlag.java
index ad4cea6468d81..fe87158c1cd64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/CreateEncryptionZoneFlag.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/CreateEncryptionZoneFlag.java
@@ -19,6 +19,9 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import java.util.EnumSet;
 
 /**
  * CreateEncryptionZoneFlag is used in
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 0664f6414ec64..2b511bfc2ebee 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -78,6 +78,8 @@ public interface HdfsClientConfigKeys {
   int     DFS_NAMENODE_HTTPS_PORT_DEFAULT = 9871;
   String  DFS_NAMENODE_HTTPS_ADDRESS_KEY = "dfs.namenode.https-address";
   String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes";
+  String DFS_RBF_OBSERVER_READ_ENABLE = "dfs.client.rbf.observer.read.enable";
+  boolean DFS_RBF_OBSERVER_READ_ENABLE_DEFAULT = false;
   int DFS_NAMENODE_RPC_PORT_DEFAULT = 8020;
   String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY =
       "dfs.namenode.kerberos.principal";
@@ -154,6 +156,9 @@ public interface HdfsClientConfigKeys {
   String  DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
       "dfs.client.slow.io.warning.threshold.ms";
   long    DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
+  String  DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY =
+      "dfs.client.mark.slownode.as.badnode.threshold";
+  int DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT = 10;
   String  DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS =
           "dfs.client.key.provider.cache.expiry";
   long    DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
@@ -202,6 +207,19 @@ public interface HdfsClientConfigKeys {
       "dfs.client.refresh.read-block-locations.ms";
   long DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_DEFAULT = 0L;
 
+  //  Number of threads to use for refreshing LocatedBlocks of registered
+  //  DFSInputStreams. If a DFSClient opens many DFSInputStreams, increasing
+  //  this may help refresh them all in a timely manner.
+  String DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_KEY =
+      "dfs.client.refresh.read-block-locations.threads";
+  int DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_THREADS_DEFAULT = 5;
+
+  // Whether to auto-register all DFSInputStreams for background refreshes.
+  // If false, user must manually register using DFSClient#addLocatedBlocksRefresh(DFSInputStream)
+  String DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_KEY =
+      "dfs.client.refresh.read-block-locations.register-automatically";
+  boolean DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_DEFAULT = true;
+
   String  DFS_DATANODE_KERBEROS_PRINCIPAL_KEY =
       "dfs.datanode.kerberos.principal";
   String  DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes";
@@ -257,6 +275,18 @@ public interface HdfsClientConfigKeys {
   String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
   long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
 
+  String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
+      "dfs.client.fsck.connect.timeout";
+  int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
+
+  String DFS_CLIENT_FSCK_READ_TIMEOUT =
+      "dfs.client.fsck.read.timeout";
+  int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
+
+  String DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY =
+      "dfs.client.output.stream.uniq.default.key";
+  String DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY_DEFAULT = "DEFAULT";
+
   /**
    * These are deprecated config keys to client code.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
index 5a615bbd62de4..af1e92e2fb449 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * The Hdfs implementation of {@link FSDataInputStream}.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
index 8af3417ca9fdf..cc7e7cd21b090 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
@@ -28,7 +28,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DFSOutputStream;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * The Hdfs implementation of {@link FSDataOutputStream}.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index a7c49128ba524..b2cf5348cfa2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -75,7 +75,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java
index f013be6c3a6c0..50b1d61282d23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.client.impl;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.ReadOption;
 import org.apache.hadoop.fs.StorageType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index 890c23bc9bc2f..6562c81358039 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.client.impl;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -60,6 +60,8 @@
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_READ_USE_CACHE_PRIORITY;
@@ -142,9 +144,11 @@ public class DfsClientConf {
   private final int retryIntervalForGetLastBlockLength;
   private final long datanodeRestartTimeout;
   private final long slowIoWarningThresholdMs;
+  private final int markSlowNodeAsBadNodeThreshold;
 
   /** wait time window before refreshing blocklocation for inputstream. */
   private final long refreshReadBlockLocationsMS;
+  private final boolean refreshReadBlockLocationsAutomatically;
 
   private final ShortCircuitConf shortCircuitConf;
   private final int clientShortCircuitNum;
@@ -261,12 +265,19 @@ public DfsClientConf(Configuration conf) {
         DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
     readUseCachePriority = conf.getBoolean(DFS_CLIENT_READ_USE_CACHE_PRIORITY,
         DFS_CLIENT_READ_USE_CACHE_PRIORITY_DEFAULT);
+    markSlowNodeAsBadNodeThreshold = conf.getInt(
+        DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY,
+        DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_DEFAULT);
 
     refreshReadBlockLocationsMS = conf.getLong(
         HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_KEY,
         HdfsClientConfigKeys.
             DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_DEFAULT);
 
+    refreshReadBlockLocationsAutomatically = conf.getBoolean(
+        HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_KEY,
+        HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_AUTOMATICALLY_DEFAULT);
+
     hedgedReadThresholdMillis = conf.getLong(
         HedgedRead.THRESHOLD_MILLIS_KEY,
         HedgedRead.THRESHOLD_MILLIS_DEFAULT);
@@ -644,6 +655,13 @@ public long getSlowIoWarningThresholdMs() {
     return slowIoWarningThresholdMs;
   }
 
+  /**
+   * @return the continuous slowNode replies received to mark slowNode as badNode
+   */
+  public int getMarkSlowNodeAsBadNodeThreshold() {
+    return markSlowNodeAsBadNodeThreshold;
+  }
+
   /*
    * @return the clientShortCircuitNum
    */
@@ -701,13 +719,18 @@ public boolean isReadUseCachePriority() {
     return replicaAccessorBuilderClasses;
   }
 
-  /**
-   * @return the replicaAccessorBuilderClasses
-   */
-  public long getRefreshReadBlockLocationsMS() {
+  public boolean isLocatedBlocksRefresherEnabled() {
+    return refreshReadBlockLocationsMS > 0;
+  }
+
+  public long getLocatedBlocksRefresherInterval() {
     return refreshReadBlockLocationsMS;
   }
 
+  public boolean isRefreshReadBlockLocationsAutomatically() {
+    return refreshReadBlockLocationsAutomatically;
+  }
+
   /**
    * @return the shortCircuitConf
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
index 3e5a4431372ef..0661a40bbb304 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
index b3354344d7237..20ecfd4e1ec83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.fs.InvalidRequestException;
 import org.apache.hadoop.ipc.RemoteException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.tracing.TraceScope;
 import org.apache.hadoop.tracing.Tracer;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index ea90645ca082b..4f2da496a1a3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -759,11 +759,19 @@ SnapshotStatus[] getSnapshotListing(String snapshotRoot)
    * the last call to renewLease(), the NameNode assumes the
    * client has died.
    *
+   * @param namespaces The full Namespace list that the renewLease rpc
+   *                   should be forwarded by RBF.
+   *                   Tips: NN side, this value should be null.
+   *                         RBF side, if this value is null, this rpc will
+   *                         be forwarded to all available namespaces,
+   *                         else this rpc will be forwarded to
+   *                         the special namespaces.
+   *
    * @throws org.apache.hadoop.security.AccessControlException permission denied
    * @throws IOException If an I/O error occurred
    */
   @Idempotent
-  void renewLease(String clientName) throws IOException;
+  void renewLease(String clientName, List<String> namespaces) throws IOException;
 
   /**
    * Start lease recovery.
@@ -1868,4 +1876,16 @@ BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
    */
   @AtMostOnce
   void satisfyStoragePolicy(String path) throws IOException;
+
+  /**
+   * Get report on all of the slow Datanodes. Slow running datanodes are identified based on
+   * the Outlier detection algorithm, if slow peer tracking is enabled for the DFS cluster.
+   *
+   * @return Datanode report for slow running datanodes.
+   * @throws IOException If an I/O error occurs.
+   */
+  @Idempotent
+  @ReadOnly
+  DatanodeInfo[] getSlowDatanodeReport() throws IOException;
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index bba90a05794ba..fbe6bcc4629d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -698,9 +698,10 @@ public static class DatanodeInfoBuilder {
     private long nonDfsUsed = 0L;
     private long lastBlockReportTime = 0L;
     private long lastBlockReportMonotonic = 0L;
-    private int numBlocks;
-
+    private int numBlocks = 0;
 
+    // Please use setNumBlocks explicitly to set numBlocks as this method doesn't have
+    // sufficient info about numBlocks
     public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
       this.capacity = from.getCapacity();
       this.dfsUsed = from.getDfsUsed();
@@ -717,7 +718,6 @@ public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
       this.upgradeDomain = from.getUpgradeDomain();
       this.lastBlockReportTime = from.getLastBlockReportTime();
       this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
-      this.numBlocks = from.getNumBlocks();
       setNodeID(from);
       return this;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 02c4f9a1f21b1..dc84bbbeae197 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
index 48b581dfe6acb..79f10429a41d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 264e3f4050fd7..efc3b90b5a970 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -490,6 +490,10 @@ default FileStatus makeQualified(URI defaultUri, Path parent) {
    */
   int compareTo(FileStatus stat);
 
+  void setNamespace(String namespace);
+
+  String getNamespace();
+
   /**
    * Set redundant flags for compatibility with existing applications.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
index bf4e0d2f9f16e..a3d4867cff45d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -54,6 +54,8 @@ public class HdfsLocatedFileStatus
   // BlockLocations[] is the user-facing type
   private transient LocatedBlocks hdfsloc;
 
+  private String namespace = null;
+
   /**
    * Constructor.
    * @param length the number of bytes the file has
@@ -217,4 +219,14 @@ public LocatedFileStatus makeQualifiedLocated(URI defaultUri, Path path) {
     return this;
   }
 
+  @Override
+  public String getNamespace() {
+    return namespace;
+  }
+
+  @Override
+  public void setNamespace(String namespace) {
+    this.namespace = namespace;
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java
index 9434423d721b9..4c90e17e4a5d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java
@@ -44,6 +44,8 @@ public class HdfsNamedFileStatus extends FileStatus implements HdfsFileStatus {
   private final int childrenNum;
   private final byte storagePolicy;
 
+  private String namespace = null;
+
   /**
    * Constructor.
    * @param length the number of bytes the file has
@@ -177,4 +179,13 @@ public int hashCode() {
     return super.hashCode();
   }
 
+  @Override
+  public String getNamespace() {
+    return namespace;
+  }
+
+  @Override
+  public void setNamespace(String namespace) {
+    this.namespace = namespace;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java
index d96c7892b76ea..f64ca17a23b7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.protocol;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.ipc.RemoteException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
index 9085e87ce60e1..01c276ae469e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
@@ -22,7 +22,7 @@
 import java.util.Comparator;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.StorageType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
index b89786d37ec7e..d7c8e4b5e438e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.protocol;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java
index 74329bc1e8a12..3fc13dec298d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java
@@ -20,7 +20,7 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.hdfs.DFSUtilClient;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
index 10884f27f90a2..858dff6204104 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java
index cc958e35df116..ef1a3658305b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PacketHeaderProto;
 import org.apache.hadoop.hdfs.util.ByteBufferOutputStream;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Shorts;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
index 6949a9d40a656..56dc37d77e22e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.util.DirectBufferPool;
 import org.apache.hadoop.io.IOUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
index 3d9e48ceebd51..7561a40b0163b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PipelineAck.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
@@ -42,6 +43,27 @@ public class PipelineAck {
   final static int OOB_START = Status.OOB_RESTART_VALUE; // the first OOB type
   final static int OOB_END = Status.OOB_RESERVED3_VALUE; // the last OOB type
 
+  public enum SLOW {
+    DISABLED(0),
+    NORMAL(1),
+    SLOW(2),
+    RESERVED(3);
+
+    private final int value;
+    private static final SLOW[] VALUES = values();
+    static SLOW valueOf(int value) {
+      return VALUES[value];
+    }
+
+    SLOW(int value) {
+      this.value = value;
+    }
+
+    public int getValue() {
+      return value;
+    }
+  }
+
   public enum ECN {
     DISABLED(0),
     SUPPORTED(1),
@@ -66,7 +88,8 @@ public int getValue() {
   private enum StatusFormat {
     STATUS(null, 4),
     RESERVED(STATUS.BITS, 1),
-    ECN_BITS(RESERVED.BITS, 2);
+    ECN_BITS(RESERVED.BITS, 2),
+    SLOW_BITS(ECN_BITS.BITS, 2);
 
     private final LongBitFormat BITS;
 
@@ -82,6 +105,10 @@ static ECN getECN(int header) {
       return ECN.valueOf((int) ECN_BITS.BITS.retrieve(header));
     }
 
+    static SLOW getSLOW(int header) {
+      return SLOW.valueOf((int) SLOW_BITS.BITS.retrieve(header));
+    }
+
     public static int setStatus(int old, Status status) {
       return (int) STATUS.BITS.combine(status.getNumber(), old);
     }
@@ -89,6 +116,10 @@ public static int setStatus(int old, Status status) {
     public static int setECN(int old, ECN ecn) {
       return (int) ECN_BITS.BITS.combine(ecn.getValue(), old);
     }
+
+    public static int setSLOW(int old, SLOW slow) {
+      return (int) SLOW_BITS.BITS.combine(slow.getValue(), old);
+    }
   }
 
   /** default constructor **/
@@ -149,7 +180,7 @@ public int getHeaderFlag(int i) {
     if (proto.getFlagCount() > 0) {
       return proto.getFlag(i);
     } else {
-      return combineHeader(ECN.DISABLED, proto.getReply(i));
+      return combineHeader(ECN.DISABLED, proto.getReply(i), SLOW.DISABLED);
     }
   }
 
@@ -230,14 +261,28 @@ public static ECN getECNFromHeader(int header) {
     return StatusFormat.getECN(header);
   }
 
+  public static SLOW getSLOWFromHeader(int header) {
+    return StatusFormat.getSLOW(header);
+  }
+
   public static int setStatusForHeader(int old, Status status) {
     return StatusFormat.setStatus(old, status);
   }
 
+  @VisibleForTesting
+  public static int setSLOWForHeader(int old, SLOW slow) {
+    return StatusFormat.setSLOW(old, slow);
+  }
+
   public static int combineHeader(ECN ecn, Status status) {
+    return combineHeader(ecn, status, SLOW.DISABLED);
+  }
+
+  public static int combineHeader(ECN ecn, Status status, SLOW slow) {
     int header = 0;
     header = StatusFormat.setStatus(header, status);
     header = StatusFormat.setECN(header, ecn);
+    header = StatusFormat.setSLOW(header, slow);
     return header;
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
index 526f3d0c66512..ab5cd0608d9d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import javax.security.sasl.Sasl;
 
 import org.apache.commons.codec.binary.Base64;
@@ -52,6 +53,7 @@
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.HandshakeSecretProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CipherOptionProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
+import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
 import org.apache.hadoop.security.SaslPropertiesResolver;
 import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
 import org.slf4j.Logger;
@@ -204,6 +206,26 @@ public static SaslPropertiesResolver getSaslPropertiesResolver(
     return resolver;
   }
 
+  private static <T> T readSaslMessage(InputStream in,
+      Function<DataTransferEncryptorMessageProto, ? extends T> handler) throws IOException {
+    DataTransferEncryptorMessageProto proto =
+        DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
+    switch (proto.getStatus()) {
+    case ERROR_UNKNOWN_KEY:
+      throw new InvalidEncryptionKeyException(proto.getMessage());
+    case ERROR:
+      if (proto.hasAccessTokenError() && proto.getAccessTokenError()) {
+        throw new InvalidBlockTokenException(proto.getMessage());
+      }
+      throw new IOException(proto.getMessage());
+    case SUCCESS:
+      return handler.apply(proto);
+    default:
+      throw new IOException(
+          "Unknown status: " + proto.getStatus() + ", message: " + proto.getMessage());
+    }
+  }
+
   /**
    * Reads a SASL negotiation message.
    *
@@ -212,15 +234,7 @@ public static SaslPropertiesResolver getSaslPropertiesResolver(
    * @throws IOException for any error
    */
   public static byte[] readSaslMessage(InputStream in) throws IOException {
-    DataTransferEncryptorMessageProto proto =
-        DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
-    if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
-      throw new InvalidEncryptionKeyException(proto.getMessage());
-    } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) {
-      throw new IOException(proto.getMessage());
-    } else {
-      return proto.getPayload().toByteArray();
-    }
+    return readSaslMessage(in, proto -> proto.getPayload().toByteArray());
   }
 
   /**
@@ -233,13 +247,7 @@ public static byte[] readSaslMessage(InputStream in) throws IOException {
    */
   public static byte[] readSaslMessageAndNegotiationCipherOptions(
       InputStream in, List<CipherOption> cipherOptions) throws IOException {
-    DataTransferEncryptorMessageProto proto =
-        DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
-    if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
-      throw new InvalidEncryptionKeyException(proto.getMessage());
-    } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) {
-      throw new IOException(proto.getMessage());
-    } else {
+    return readSaslMessage(in, proto -> {
       List<CipherOptionProto> optionProtos = proto.getCipherOptionList();
       if (optionProtos != null) {
         for (CipherOptionProto optionProto : optionProtos) {
@@ -247,7 +255,7 @@ public static byte[] readSaslMessageAndNegotiationCipherOptions(
         }
       }
       return proto.getPayload().toByteArray();
-    }
+    });
   }
 
   static class SaslMessageWithHandshake {
@@ -276,13 +284,7 @@ String getBpid() {
 
   public static SaslMessageWithHandshake readSaslMessageWithHandshakeSecret(
       InputStream in) throws IOException {
-    DataTransferEncryptorMessageProto proto =
-        DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
-    if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
-      throw new InvalidEncryptionKeyException(proto.getMessage());
-    } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) {
-      throw new IOException(proto.getMessage());
-    } else {
+    return readSaslMessage(in, proto -> {
       byte[] payload = proto.getPayload().toByteArray();
       byte[] secret = null;
       String bpid = null;
@@ -292,7 +294,7 @@ public static SaslMessageWithHandshake readSaslMessageWithHandshakeSecret(
         bpid = handshakeSecret.getBpid();
       }
       return new SaslMessageWithHandshake(payload, secret, bpid);
-    }
+    });
   }
 
   /**
@@ -467,13 +469,7 @@ public static void sendSaslMessageAndNegotiationCipherOptions(
   public static SaslResponseWithNegotiatedCipherOption
       readSaslMessageAndNegotiatedCipherOption(InputStream in)
       throws IOException {
-    DataTransferEncryptorMessageProto proto =
-        DataTransferEncryptorMessageProto.parseFrom(vintPrefixed(in));
-    if (proto.getStatus() == DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY) {
-      throw new InvalidEncryptionKeyException(proto.getMessage());
-    } else if (proto.getStatus() == DataTransferEncryptorStatus.ERROR) {
-      throw new IOException(proto.getMessage());
-    } else {
+    return readSaslMessage(in, proto -> {
       byte[] response = proto.getPayload().toByteArray();
       List<CipherOption> options = PBHelperClient.convertCipherOptionProtos(
           proto.getCipherOptionList());
@@ -482,7 +478,7 @@ public static void sendSaslMessageAndNegotiationCipherOptions(
         option = options.get(0);
       }
       return new SaslResponseWithNegotiatedCipherOption(response, option);
-    }
+    });
   }
 
   /**
@@ -558,6 +554,13 @@ public static void sendSaslMessage(OutputStream out,
       DataTransferEncryptorStatus status, byte[] payload, String message,
       HandshakeSecretProto handshakeSecret)
       throws IOException {
+    sendSaslMessage(out, status, payload, message, handshakeSecret, false);
+  }
+
+  public static void sendSaslMessage(OutputStream out,
+      DataTransferEncryptorStatus status, byte[] payload, String message,
+      HandshakeSecretProto handshakeSecret, boolean accessTokenError)
+      throws IOException {
     DataTransferEncryptorMessageProto.Builder builder =
         DataTransferEncryptorMessageProto.newBuilder();
 
@@ -571,6 +574,9 @@ public static void sendSaslMessage(OutputStream out,
     if (handshakeSecret != null) {
       builder.setHandshakeSecret(handshakeSecret);
     }
+    if (accessTokenError) {
+      builder.setAccessTokenError(true);
+    }
 
     DataTransferEncryptorMessageProto proto = builder.build();
     proto.writeDelimitedTo(out);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
index 1a61c9664825e..641c7a0ff4790 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
@@ -588,11 +588,11 @@ private IOStreamPair doSaslHandshake(InetAddress addr,
               // the client accepts some cipher suites, but the server does not.
               LOG.debug("Client accepts cipher suites {}, "
                       + "but server {} does not accept any of them",
-                  cipherSuites, addr.toString());
+                  cipherSuites, addr);
             }
           } else {
             LOG.debug("Client using cipher suite {} with server {}",
-                cipherOption.getCipherSuite().getName(), addr.toString());
+                cipherOption.getCipherSuite().getName(), addr);
           }
         }
       }
@@ -603,7 +603,20 @@ private IOStreamPair doSaslHandshake(InetAddress addr,
           conf, cipherOption, underlyingOut, underlyingIn, false) :
           sasl.createStreamPair(out, in);
     } catch (IOException ioe) {
-      sendGenericSaslErrorMessage(out, ioe.getMessage());
+      String message = ioe.getMessage();
+      try {
+        sendGenericSaslErrorMessage(out, message);
+      } catch (Exception e) {
+        // If ioe is caused by error response from server, server will close peer connection.
+        // So sendGenericSaslErrorMessage might cause IOException due to "Broken pipe".
+        // We suppress IOException from sendGenericSaslErrorMessage
+        // and always throw `ioe` as top level.
+        // `ioe` can be InvalidEncryptionKeyException or InvalidBlockTokenException
+        // that indicates refresh key or token and are important for caller.
+        LOG.debug("Failed to send generic sasl error to server {} (message: {}), "
+                + "suppress exception", addr, message, e);
+        ioe.addSuppressed(e);
+      }
       throw ioe;
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 2668ec1cefe98..541a4361896dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -143,6 +143,7 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingRequestProto;
@@ -743,11 +744,15 @@ public BatchedDirectoryListing getBatchedListing(
 
 
   @Override
-  public void renewLease(String clientName) throws IOException {
-    RenewLeaseRequestProto req = RenewLeaseRequestProto.newBuilder()
-        .setClientName(clientName).build();
+  public void renewLease(String clientName, List<String> namespaces)
+      throws IOException {
+    RenewLeaseRequestProto.Builder builder = RenewLeaseRequestProto
+        .newBuilder().setClientName(clientName);
+    if (namespaces != null && !namespaces.isEmpty()) {
+      builder.addAllNamespaces(namespaces);
+    }
     try {
-      rpcProxy.renewLease(null, req);
+      rpcProxy.renewLease(null, builder.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -2065,6 +2070,18 @@ public void satisfyStoragePolicy(String src) throws IOException {
     }
   }
 
+  @Override
+  public DatanodeInfo[] getSlowDatanodeReport() throws IOException {
+    GetSlowDatanodeReportRequestProto req =
+        GetSlowDatanodeReportRequestProto.newBuilder().build();
+    try {
+      return PBHelperClient.convert(
+          rpcProxy.getSlowDatanodeReport(null, req).getDatanodeInfoProtoList());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
   @Override
   public HAServiceProtocol.HAServiceState getHAServiceState()
       throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 9de9c373040e7..496a5cf46146d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -28,7 +28,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
 import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
@@ -1764,7 +1764,7 @@ public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
     EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags()
         ? convertFlags(fs.getFlags())
         : convertFlags(fs.getPermission());
-    return new HdfsFileStatus.Builder()
+    HdfsFileStatus hdfsFileStatus = new HdfsFileStatus.Builder()
         .length(fs.getLength())
         .isdir(fs.getFileType().equals(FileType.IS_DIR))
         .replication(fs.getBlockReplication())
@@ -1794,6 +1794,10 @@ public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
             ? convertErasureCodingPolicy(fs.getEcPolicy())
             : null)
         .build();
+    if (fs.hasNamespace()) {
+      hdfsFileStatus.setNamespace(fs.getNamespace());
+    }
+    return hdfsFileStatus;
   }
 
   private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) {
@@ -2399,6 +2403,9 @@ public static HdfsFileStatusProto convert(HdfsFileStatus fs) {
     flags |= fs.isSnapshotEnabled() ? HdfsFileStatusProto.Flags
         .SNAPSHOT_ENABLED_VALUE : 0;
     builder.setFlags(flags);
+    if (fs.getNamespace() != null && !fs.getNamespace().isEmpty()) {
+      builder.setNamespace(fs.getNamespace());
+    }
     return builder.build();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
index 130e8c1c9c728..d1ad5a2079f5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
index 7e3954c562a58..5a5da7326a4e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
@@ -23,7 +23,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.SerializationFeature;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
index bc89830f1e14f..e1a7c2f8030cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ObserverReadProxyProvider.java
@@ -214,7 +214,6 @@ public ObserverReadProxyProvider(
         OBSERVER_PROBE_RETRY_PERIOD_KEY,
         OBSERVER_PROBE_RETRY_PERIOD_DEFAULT, TimeUnit.MILLISECONDS);
 
-    // TODO : make this configurable or remove this variable
     if (wrappedProxy instanceof ClientProtocol) {
       this.observerReadEnabled = true;
     } else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
index 9011b25eda043..5e83fff6b784a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
 import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
@@ -27,20 +26,24 @@
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.ExecutionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.MultiException;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.Client.ConnectionId;
+import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RpcInvocationHandler;
 import org.apache.hadoop.ipc.StandbyException;
 
-import org.apache.hadoop.io.retry.MultiException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 /**
  * A FailoverProxyProvider implementation that technically does not "failover"
  * per-se. It constructs a wrapper proxy that sends the request to ALL
@@ -55,7 +58,7 @@ public class RequestHedgingProxyProvider<T> extends
   public static final Logger LOG =
       LoggerFactory.getLogger(RequestHedgingProxyProvider.class);
 
-  class RequestHedgingInvocationHandler implements InvocationHandler {
+  class RequestHedgingInvocationHandler implements RpcInvocationHandler {
 
     final Map<String, ProxyInfo<T>> targetProxies;
     // Proxy of the active nn
@@ -123,11 +126,18 @@ public RequestHedgingInvocationHandler(
               }
               executor = Executors.newFixedThreadPool(proxies.size());
               completionService = new ExecutorCompletionService<>(executor);
+              // Set the callId and other informations from current thread.
+              final int callId = Client.getCallId();
+              final int retryCount = Client.getRetryCount();
+              final Object externalHandler = Client.getExternalHandler();
               for (final Map.Entry<String, ProxyInfo<T>> pEntry : targetProxies
                   .entrySet()) {
                 Callable<Object> c = new Callable<Object>() {
                   @Override
                   public Object call() throws Exception {
+                    // Call Id and other informations from parent thread.
+                    Client.setCallIdAndRetryCount(callId, retryCount,
+                        externalHandler);
                     LOG.trace("Invoking method {} on proxy {}", method,
                         pEntry.getValue().proxyInfo);
                     return method.invoke(pEntry.getValue().proxy, args);
@@ -136,7 +146,9 @@ public Object call() throws Exception {
                 proxyMap.put(completionService.submit(c), pEntry.getValue());
                 numAttempts++;
               }
-
+              // Current thread's callId will not be cleared as RPC happens in
+              // separate threads. Reset the CallId information Forcefully.
+              Client.setCallIdAndRetryCountUnprotected(null, 0, null);
               Map<String, Exception> badResults = new HashMap<>();
               while (numAttempts > 0) {
                 Future<Object> callResultFuture = completionService.take();
@@ -189,6 +201,18 @@ public Object call() throws Exception {
         throw unwrappedException;
       }
     }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    @Override
+    public ConnectionId getConnectionId() {
+      if (currentUsedProxy == null) {
+        return null;
+      }
+      return RPC.getConnectionIdForProxy(currentUsedProxy.proxy);
+    }
   }
 
   /** A proxy wrapping {@link RequestHedgingInvocationHandler}. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/OutlierMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/OutlierMetrics.java
new file mode 100644
index 0000000000000..452885d0c0538
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/OutlierMetrics.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.protocol;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Outlier detection metrics - median, median absolute deviation, upper latency limit,
+ * actual latency etc.
+ */
+@InterfaceAudience.Private
+public class OutlierMetrics {
+
+  private final Double median;
+  private final Double mad;
+  private final Double upperLimitLatency;
+  private final Double actualLatency;
+
+  public OutlierMetrics(Double median, Double mad, Double upperLimitLatency,
+      Double actualLatency) {
+    this.median = median;
+    this.mad = mad;
+    this.upperLimitLatency = upperLimitLatency;
+    this.actualLatency = actualLatency;
+  }
+
+  public Double getMedian() {
+    return median;
+  }
+
+  public Double getMad() {
+    return mad;
+  }
+
+  public Double getUpperLimitLatency() {
+    return upperLimitLatency;
+  }
+
+  public Double getActualLatency() {
+    return actualLatency;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    OutlierMetrics that = (OutlierMetrics) o;
+
+    return new EqualsBuilder()
+        .append(median, that.median)
+        .append(mad, that.mad)
+        .append(upperLimitLatency, that.upperLimitLatency)
+        .append(actualLatency, that.actualLatency)
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(17, 37)
+        .append(median)
+        .append(mad)
+        .append(upperLimitLatency)
+        .append(actualLatency)
+        .toHashCode();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java
index a3b3445d5a743..586e2f4931e28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowPeerReports.java
@@ -51,7 +51,7 @@ public final class SlowPeerReports {
    * meaningful and must be avoided.
    */
   @Nonnull
-  private final Map<String, Double> slowPeers;
+  private final Map<String, OutlierMetrics> slowPeers;
 
   /**
    * An object representing a SlowPeerReports with no entries. Should
@@ -61,19 +61,19 @@ public final class SlowPeerReports {
   public static final SlowPeerReports EMPTY_REPORT =
       new SlowPeerReports(ImmutableMap.of());
 
-  private SlowPeerReports(Map<String, Double> slowPeers) {
+  private SlowPeerReports(Map<String, OutlierMetrics> slowPeers) {
     this.slowPeers = slowPeers;
   }
 
   public static SlowPeerReports create(
-      @Nullable Map<String, Double> slowPeers) {
+      @Nullable Map<String, OutlierMetrics> slowPeers) {
     if (slowPeers == null || slowPeers.isEmpty()) {
       return EMPTY_REPORT;
     }
     return new SlowPeerReports(slowPeers);
   }
 
-  public Map<String, Double> getSlowPeers() {
+  public Map<String, OutlierMetrics> getSlowPeers() {
     return slowPeers;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java
index 7bf768935d9d9..ae445efaf566b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.net.unix.DomainSocketWatcher;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * DfsClientShm is a subclass of ShortCircuitShm which is used by the
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
index f165ab90a1d4a..2b9bf68e469c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java
@@ -44,7 +44,7 @@
 import org.apache.hadoop.net.unix.DomainSocketWatcher;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java
index 9f08f0901a7ea..46d6718adb37c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.net.unix.DomainSocket;
 import org.apache.hadoop.util.PerformanceAdvisory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index ccc39ddf5d55f..df2a92c75c962 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -55,7 +55,7 @@
 import org.apache.hadoop.util.Waitable;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import org.slf4j.Logger;
@@ -189,6 +189,7 @@ public void run() {
       final DfsClientShm shm = (DfsClientShm)slot.getShm();
       final DomainSocket shmSock = shm.getPeer().getDomainSocket();
       final String path = shmSock.getPath();
+      DomainSocket domainSocket = pathToDomainSocket.get(path);
       DataOutputStream out = null;
       boolean success = false;
       int retries = 2;
@@ -196,9 +197,10 @@ public void run() {
         while (retries > 0) {
           try {
             if (domainSocket == null || !domainSocket.isOpen()) {
-              // we are running in single thread mode, no protection needed for
-              // domainSocket
               domainSocket = DomainSocket.connect(path);
+              // we are running in single thread mode, no protection needed for
+              // pathToDomainSocket
+              pathToDomainSocket.put(path, domainSocket);
             }
 
             out = new DataOutputStream(
@@ -221,13 +223,16 @@ public void run() {
           } catch (SocketException se) {
             // the domain socket on datanode may be timed out, we retry once
             retries--;
-            domainSocket.close();
-            domainSocket = null;
+            if (domainSocket != null) {
+              domainSocket.close();
+              domainSocket = null;
+              pathToDomainSocket.remove(path);
+            }
             if (retries == 0) {
               throw new SocketException("Create domain socket failed");
             }
           }
-        }
+        } // end of while block
       } catch (IOException e) {
         LOG.warn(ShortCircuitCache.this + ": failed to release "
             + "short-circuit shared memory slot " + slot + " by sending "
@@ -240,10 +245,10 @@ public void run() {
         } else {
           shm.getEndpointShmManager().shutdown(shm);
           IOUtilsClient.cleanupWithLogger(LOG, domainSocket, out);
-          domainSocket = null;
+          pathToDomainSocket.remove(path);
         }
       }
-    }
+    } // end of run()
   }
 
   public interface ShortCircuitReplicaCreator {
@@ -354,7 +359,11 @@ public interface ShortCircuitReplicaCreator {
    */
   private final DfsClientShmManager shmManager;
 
-  private DomainSocket domainSocket = null;
+  /**
+   * A map contains all DomainSockets used in SlotReleaser. Keys are the domain socket
+   * paths of short-circuit shared memory segments.
+   */
+  private Map<String, DomainSocket> pathToDomainSocket = new HashMap<>();
 
   public static ShortCircuitCache fromConf(ShortCircuitConf conf) {
     return new ShortCircuitCache(
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java
index fd58ef187525b..2177d9108f54e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java
@@ -32,7 +32,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -160,7 +160,7 @@ boolean isStale() {
       long deltaMs = Time.monotonicNow() - creationTimeMs;
       long staleThresholdMs = cache.getStaleThresholdMs();
       if (deltaMs > staleThresholdMs) {
-        LOG.trace("{} is stale because it's {} ms old and staleThreadholdMS={}",
+        LOG.trace("{} is stale because it's {} ms old and staleThresholdMs={}",
             this, deltaMs, staleThresholdMs);
         return true;
       } else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
index 1cb123bb58f3b..c6f7a50368152 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
@@ -39,7 +39,7 @@
 
 import sun.misc.Unsafe;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
index 059280e494678..f076969c9b17b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.util.Time;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
index fb15926dce176..0d1be4b8e6790 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
@@ -20,6 +20,8 @@
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.util.XMLUtils;
+
 import org.w3c.dom.Node;
 import org.w3c.dom.Text;
 import org.w3c.dom.Element;
@@ -87,7 +89,7 @@ private List<ErasureCodingPolicy> loadECPolicies(File policyFile)
     LOG.info("Loading EC policy file " + policyFile);
 
     // Read and parse the EC policy file.
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     dbf.setIgnoringComments(true);
     DocumentBuilder builder = dbf.newDocumentBuilder();
     Document doc = builder.parse(policyFile);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 540187daa2a2c..25ee8d30c9290 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -28,7 +28,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.DFSStripedOutputStream;
@@ -245,8 +245,7 @@ public static long getSafeLength(ErasureCodingPolicy ecPolicy,
     Arrays.sort(cpy);
     // full stripe is a stripe has at least dataBlkNum full cells.
     // lastFullStripeIdx is the index of the last full stripe.
-    int lastFullStripeIdx =
-        (int) (cpy[cpy.length - dataBlkNum] / cellSize);
+    long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
     return lastFullStripeIdx * stripeSize; // return the safeLength
     // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
     // such a stripe (and it must be partial).
@@ -271,9 +270,9 @@ private static int lastCellSize(int size, int cellSize, int numDataBlocks,
    */
   public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
       long offsetInBlk, int idxInBlockGroup) {
-    int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+    long cellIdxInBlk = offsetInBlk / cellSize;
     return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
-        + idxInBlockGroup * cellSize // m full cells before offset
+        + (long)idxInBlockGroup * cellSize // m full cells before offset
         + offsetInBlk % cellSize; // partial cell
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 6acd062568e86..c3dd556ba5312 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -19,7 +19,10 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileChecksum;
@@ -39,6 +42,8 @@
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
@@ -245,6 +250,14 @@ static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
     return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp);
   }
 
+  static boolean getBoolean(Map<?, ?> m, String key, final boolean defaultValue) {
+    Object value = m.get(key);
+    if (value == null) {
+      return defaultValue;
+    }
+    return ((Boolean) value).booleanValue();
+  }
+
   static int getInt(Map<?, ?> m, String key, final int defaultValue) {
     Object value = m.get(key);
     if (value == null) {
@@ -834,6 +847,53 @@ private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry(
     return new SnapshotDiffReport.DiffReportEntry(type, sourcePath, targetPath);
   }
 
+  public static SnapshotDiffReportListing toSnapshotDiffReportListing(
+      final Map<?, ?> json) {
+    if (json == null) {
+      return null;
+    }
+
+    Map<?, ?> m =
+        (Map<?, ?>) json.get(SnapshotDiffReportListing.class.getSimpleName());
+    byte[] lastPath = DFSUtilClient.string2Bytes(getString(m, "lastPath", ""));
+    int lastIndex = getInt(m, "lastIndex", -1);
+    boolean isFromEarlier = getBoolean(m, "isFromEarlier", false);
+    List<DiffReportListingEntry> modifyList =
+        toDiffListingList(getList(m, "modifyList"));
+    List<DiffReportListingEntry> createList =
+        toDiffListingList(getList(m, "createList"));
+    List<DiffReportListingEntry> deleteList =
+        toDiffListingList(getList(m, "deleteList"));
+
+    return new SnapshotDiffReportListing(
+        lastPath, modifyList, createList, deleteList, lastIndex, isFromEarlier);
+  }
+
+  public static List<DiffReportListingEntry> toDiffListingList(List<?> objs) {
+    if (objs == null) {
+      return null;
+    }
+    List<DiffReportListingEntry> diffList = new ChunkedArrayList<>();
+    for (int i = 0; i < objs.size(); i++) {
+      diffList.add(toDiffReportListingEntry((Map<?, ?>) objs.get(i)));
+    }
+    return diffList;
+  }
+
+  private static DiffReportListingEntry toDiffReportListingEntry(
+      Map<?, ?> json) {
+    if (json == null) {
+      return null;
+    }
+    long dirId = getLong(json, "dirId", 0);
+    long fileId = getLong(json, "fileId", 0);
+    byte[] sourcePath = toByteArray(getString(json, "sourcePath", null));
+    byte[] targetPath = toByteArray(getString(json, "targetPath", null));
+    boolean isReference = getBoolean(json, "isReference", false);
+    return new DiffReportListingEntry(
+        dirId, fileId, sourcePath, isReference, targetPath);
+  }
+
   private static byte[] toByteArray(String str) {
     if (str == null) {
       return null;
@@ -908,4 +968,53 @@ private static SnapshotStatus toSnapshotStatus(
                 SnapshotStatus.getParentPath(fullPath)));
     return snapshotStatus;
   }
+
+  @VisibleForTesting
+  public static BlockLocation[] toBlockLocationArray(Map<?, ?> json)
+      throws IOException {
+    final Map<?, ?> rootmap =
+        (Map<?, ?>) json.get(BlockLocation.class.getSimpleName() + "s");
+    final List<?> array =
+        JsonUtilClient.getList(rootmap, BlockLocation.class.getSimpleName());
+    Preconditions.checkNotNull(array);
+    final BlockLocation[] locations = new BlockLocation[array.size()];
+    int i = 0;
+    for (Object object : array) {
+      final Map<?, ?> m = (Map<?, ?>) object;
+      locations[i++] = JsonUtilClient.toBlockLocation(m);
+    }
+    return locations;
+  }
+
+  /** Convert a Json map to BlockLocation. **/
+  private static BlockLocation toBlockLocation(Map<?, ?> m) throws IOException {
+    if (m == null) {
+      return null;
+    }
+    long length = ((Number) m.get("length")).longValue();
+    long offset = ((Number) m.get("offset")).longValue();
+    boolean corrupt = Boolean.getBoolean(m.get("corrupt").toString());
+    String[] storageIds = toStringArray(getList(m, "storageIds"));
+    String[] cachedHosts = toStringArray(getList(m, "cachedHosts"));
+    String[] hosts = toStringArray(getList(m, "hosts"));
+    String[] names = toStringArray(getList(m, "names"));
+    String[] topologyPaths = toStringArray(getList(m, "topologyPaths"));
+    StorageType[] storageTypes = toStorageTypeArray(getList(m, "storageTypes"));
+    return new BlockLocation(names, hosts, cachedHosts, topologyPaths,
+        storageIds, storageTypes, offset, length, corrupt);
+  }
+
+  @VisibleForTesting
+  static String[] toStringArray(List<?> list) {
+    if (list == null) {
+      return null;
+    } else {
+      final String[] array = new String[list.size()];
+      int i = 0;
+      for (Object object : list) {
+        array[i++] = object.toString();
+      }
+      return array;
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 15d1b7b4b7ff3..f0774e98d1f8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -46,7 +46,6 @@
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -63,7 +62,6 @@
 import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.CommonPathCapabilities;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.DelegationTokenRenewer;
@@ -78,7 +76,6 @@
 import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
 import org.apache.hadoop.fs.MultipartUploaderBuilder;
 import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.fs.PathCapabilities;
 import org.apache.hadoop.fs.StorageStatistics;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.fs.impl.FileSystemMultipartUploaderBuilder;
@@ -105,6 +102,7 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FileEncryptionInfoProto;
@@ -138,9 +136,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
+import org.apache.hadoop.util.Preconditions;
 
 /** A FileSystem for HDFS over the web. */
 public class WebHdfsFileSystem extends FileSystem
@@ -187,6 +183,8 @@ public class WebHdfsFileSystem extends FileSystem
   private KeyProvider testProvider;
   private boolean isTLSKrb;
 
+  private boolean isServerHCFSCompatible = true;
+
   /**
    * Return the protocol scheme for the FileSystem.
    *
@@ -480,7 +478,8 @@ private Path makeAbsolute(Path f) {
     return f.isAbsolute()? f: new Path(workingDir, f);
   }
 
-  static Map<?, ?> jsonParse(final HttpURLConnection c,
+  @VisibleForTesting
+  public static Map<?, ?> jsonParse(final HttpURLConnection c,
       final boolean useErrorStream) throws IOException {
     if (c.getContentLength() == 0) {
       return null;
@@ -1445,19 +1444,47 @@ public void renameSnapshot(final Path path, final String snapshotOldName,
         new SnapshotNameParam(snapshotNewName)).run();
   }
 
+  private SnapshotDiffReport getSnapshotDiffReport(
+      final String snapshotDir, final String fromSnapshot, final String toSnapshot)
+      throws IOException {
+    return new FsPathResponseRunner<SnapshotDiffReport>(
+        GetOpParam.Op.GETSNAPSHOTDIFF,
+        new Path(snapshotDir),
+        new OldSnapshotNameParam(fromSnapshot),
+        new SnapshotNameParam(toSnapshot)) {
+          @Override
+          SnapshotDiffReport decodeResponse(Map<?, ?> json) {
+            return JsonUtilClient.toSnapshotDiffReport(json);
+          }
+        }.run();
+  }
+
+  // This API should be treated as private to WebHdfsFileSystem. Only tests can use it directly.
+  @VisibleForTesting
+  public SnapshotDiffReportListing getSnapshotDiffReportListing(
+        String snapshotDir, final String fromSnapshot, final String toSnapshot,
+        byte[] startPath, int index) throws IOException {
+    return new FsPathResponseRunner<SnapshotDiffReportListing>(
+        GetOpParam.Op.GETSNAPSHOTDIFFLISTING,
+        new Path(snapshotDir),
+        new OldSnapshotNameParam(fromSnapshot),
+        new SnapshotNameParam(toSnapshot),
+        new SnapshotDiffStartPathParam(DFSUtilClient.bytes2String(startPath)),
+        new SnapshotDiffIndexParam(index)) {
+          @Override
+          SnapshotDiffReportListing decodeResponse(Map<?, ?> json) {
+            return JsonUtilClient.toSnapshotDiffReportListing(json);
+          }
+        }.run();
+  }
+
   public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
       final String fromSnapshot, final String toSnapshot) throws IOException {
     statistics.incrementReadOps(1);
     storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF);
-    final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF;
-    return new FsPathResponseRunner<SnapshotDiffReport>(op, snapshotDir,
-        new OldSnapshotNameParam(fromSnapshot),
-        new SnapshotNameParam(toSnapshot)) {
-      @Override
-      SnapshotDiffReport decodeResponse(Map<?, ?> json) {
-        return JsonUtilClient.toSnapshotDiffReport(json);
-      }
-    }.run();
+    return DFSUtilClient.getSnapshotDiffReport(
+        snapshotDir.toUri().getPath(), fromSnapshot, toSnapshot,
+        this::getSnapshotDiffReport, this::getSnapshotDiffReportListing);
   }
 
   public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList()
@@ -1857,18 +1884,51 @@ public BlockLocation[] getFileBlockLocations(final FileStatus status,
   }
 
   @Override
-  public BlockLocation[] getFileBlockLocations(final Path p,
-      final long offset, final long length) throws IOException {
+  public BlockLocation[] getFileBlockLocations(final Path p, final long offset,
+      final long length) throws IOException {
     statistics.incrementReadOps(1);
     storageStatistics.incrementOpCounter(OpType.GET_FILE_BLOCK_LOCATIONS);
+    BlockLocation[] locations;
+    try {
+      if (isServerHCFSCompatible) {
+        locations = getFileBlockLocations(GetOpParam.Op.GETFILEBLOCKLOCATIONS, p, offset, length);
+      } else {
+        locations = getFileBlockLocations(GetOpParam.Op.GET_BLOCK_LOCATIONS, p, offset, length);
+      }
+    } catch (RemoteException e) {
+      // parsing the exception is needed only if the client thinks the service is compatible
+      if (isServerHCFSCompatible && isGetFileBlockLocationsException(e)) {
+        LOG.warn("Server does not appear to support GETFILEBLOCKLOCATIONS." +
+                "Fallback to the old GET_BLOCK_LOCATIONS. Exception: {}",
+            e.getMessage());
+        isServerHCFSCompatible = false;
+        locations = getFileBlockLocations(GetOpParam.Op.GET_BLOCK_LOCATIONS, p, offset, length);
+      } else {
+        throw e;
+      }
+    }
+    return locations;
+  }
+
+  private boolean isGetFileBlockLocationsException(RemoteException e) {
+    return e.getMessage() != null && e.getMessage().contains("Invalid value for webhdfs parameter")
+        && e.getMessage().contains(GetOpParam.Op.GETFILEBLOCKLOCATIONS.toString());
+  }
 
-    final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
-    return new FsPathResponseRunner<BlockLocation[]>(op, p,
+  private BlockLocation[] getFileBlockLocations(final GetOpParam.Op operation,
+      final Path p, final long offset, final long length) throws IOException {
+    return new FsPathResponseRunner<BlockLocation[]>(operation, p,
         new OffsetParam(offset), new LengthParam(length)) {
       @Override
-      BlockLocation[] decodeResponse(Map<?,?> json) throws IOException {
-        return DFSUtilClient.locatedBlocks2Locations(
-            JsonUtilClient.toLocatedBlocks(json));
+      BlockLocation[] decodeResponse(Map<?, ?> json) throws IOException {
+        switch (operation) {
+        case GETFILEBLOCKLOCATIONS:
+          return JsonUtilClient.toBlockLocationArray(json);
+        case GET_BLOCK_LOCATIONS:
+          return DFSUtilClient.locatedBlocks2Locations(JsonUtilClient.toLocatedBlocks(json));
+        default:
+          throw new IOException("Unknown operation " + operation.name());
+        }
       }
     }.run();
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
index 3e3fbfbd91380..e944e8c1c8d77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
@@ -18,10 +18,15 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import com.squareup.okhttp.OkHttpClient;
-import com.squareup.okhttp.Request;
-import com.squareup.okhttp.RequestBody;
-import com.squareup.okhttp.Response;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -30,10 +35,6 @@
 import org.apache.hadoop.util.Timer;
 import org.apache.http.HttpStatus;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
 import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.ACCESS_TOKEN;
@@ -102,37 +103,34 @@ public synchronized String getAccessToken() throws IOException {
   }
 
   void refresh() throws IOException {
-    try {
-      OkHttpClient client = new OkHttpClient();
-      client.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
-          TimeUnit.MILLISECONDS);
-      client.setReadTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
-                TimeUnit.MILLISECONDS);
-
-      String bodyString = Utils.postBody(GRANT_TYPE, REFRESH_TOKEN,
-          REFRESH_TOKEN, refreshToken,
-          CLIENT_ID, clientId);
-
-      RequestBody body = RequestBody.create(URLENCODED, bodyString);
-
-      Request request = new Request.Builder()
-          .url(refreshURL)
-          .post(body)
-          .build();
-      Response responseBody = client.newCall(request).execute();
-
-      if (responseBody.code() != HttpStatus.SC_OK) {
-        throw new IllegalArgumentException("Received invalid http response: "
-            + responseBody.code() + ", text = " + responseBody.toString());
+    OkHttpClient client =
+        new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
+                TimeUnit.MILLISECONDS)
+            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+            .build();
+
+    String bodyString =
+        Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
+
+    RequestBody body = RequestBody.create(bodyString, URLENCODED);
+
+    Request request = new Request.Builder().url(refreshURL).post(body).build();
+    try (Response response = client.newCall(request).execute()) {
+      if (!response.isSuccessful()) {
+        throw new IOException("Unexpected code " + response);
+      }
+      if (response.code() != HttpStatus.SC_OK) {
+        throw new IllegalArgumentException(
+            "Received invalid http response: " + response.code() + ", text = "
+                + response.toString());
       }
 
-      Map<?, ?> response = JsonSerialization.mapReader().readValue(
-          responseBody.body().string());
+      Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());
 
-      String newExpiresIn = response.get(EXPIRES_IN).toString();
+      String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
       accessTokenTimer.setExpiresIn(newExpiresIn);
 
-      accessToken = response.get(ACCESS_TOKEN).toString();
+      accessToken = responseBody.get(ACCESS_TOKEN).toString();
     } catch (Exception e) {
       throw new IOException("Exception while refreshing access token", e);
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
index bfd7055990e81..25ceb8846092b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
@@ -18,10 +18,15 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import com.squareup.okhttp.OkHttpClient;
-import com.squareup.okhttp.Request;
-import com.squareup.okhttp.RequestBody;
-import com.squareup.okhttp.Response;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -30,10 +35,6 @@
 import org.apache.hadoop.util.Timer;
 import org.apache.http.HttpStatus;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
 import static org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants.ACCESS_TOKEN;
@@ -96,38 +97,38 @@ public synchronized String getAccessToken() throws IOException {
   }
 
   void refresh() throws IOException {
-    try {
-      OkHttpClient client = new OkHttpClient();
-      client.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
-          TimeUnit.MILLISECONDS);
-      client.setReadTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
-          TimeUnit.MILLISECONDS);
-
-      String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
-          GRANT_TYPE, CLIENT_CREDENTIALS,
-          CLIENT_ID, clientId);
-
-      RequestBody body = RequestBody.create(URLENCODED, bodyString);
-
-      Request request = new Request.Builder()
-          .url(refreshURL)
-          .post(body)
-          .build();
-      Response responseBody = client.newCall(request).execute();
-
-      if (responseBody.code() != HttpStatus.SC_OK) {
+    OkHttpClient client = new OkHttpClient.Builder()
+            .connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+            .readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
+            .build();
+
+    String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
+        GRANT_TYPE, CLIENT_CREDENTIALS,
+        CLIENT_ID, clientId);
+
+    RequestBody body = RequestBody.create(bodyString, URLENCODED);
+
+    Request request = new Request.Builder()
+        .url(refreshURL)
+        .post(body)
+        .build();
+    try (Response response = client.newCall(request).execute()) {
+      if (!response.isSuccessful()) {
+        throw new IOException("Unexpected code " + response);
+      }
+
+      if (response.code() != HttpStatus.SC_OK) {
         throw new IllegalArgumentException("Received invalid http response: "
-            + responseBody.code() + ", text = " + responseBody.toString());
+            + response.code() + ", text = " + response.toString());
       }
 
-      Map<?, ?> response = JsonSerialization.mapReader().readValue(
-          responseBody.body().string());
+      Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
+          response.body().string());
 
-      String newExpiresIn = response.get(EXPIRES_IN).toString();
+      String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
       timer.setExpiresIn(newExpiresIn);
 
-      accessToken = response.get(ACCESS_TOKEN).toString();
-
+      accessToken = responseBody.get(ACCESS_TOKEN).toString();
     } catch (Exception e) {
       throw new IOException("Unable to obtain access token from credential", e);
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java
index 3f091c2945bcc..2f28b65e40e92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java
@@ -18,7 +18,7 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import com.squareup.okhttp.MediaType;
+import okhttp3.MediaType;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index abbef9d8e9b91..14938c3c45b62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -62,6 +62,7 @@ public enum Op implements HttpOpParam.Op {
     LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK),
     GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK),
     GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK),
+    GETSNAPSHOTDIFFLISTING(false, HttpURLConnection.HTTP_OK),
     GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK),
     GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK);
 
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationWrapper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java
similarity index 52%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationWrapper.java
rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java
index 6f67a16715e77..cdec18cdcf63d 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationWrapper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java
@@ -15,33 +15,35 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.hadoop.hdfs.web.resources;
 
-package org.apache.hadoop.fs.swift.auth;
+/** resuming index of snapshotDiffReportListing operation. */
+public class SnapshotDiffIndexParam extends IntegerParam {
+  /** Parameter name. */
+  public static final String NAME = "snapshotdiffindex";
+  /** Default parameter value. */
+  public static final String DEFAULT = "-1";
 
-/**
- * This class is used for correct hierarchy mapping of
- * Keystone authentication model and java code
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class AuthenticationWrapper {
+  private static final Domain DOMAIN = new Domain(NAME);
 
   /**
-   * authentication response field
+   * Constructor.
+   * @param value the parameter value.
    */
-  private AuthenticationResponse access;
+  public SnapshotDiffIndexParam(final Integer value) {
+    super(DOMAIN, value, -1, null);
+  }
 
   /**
-   * @return authentication response
+   * Constructor.
+   * @param str a string representation of the parameter value.
    */
-  public AuthenticationResponse getAccess() {
-    return access;
+  public SnapshotDiffIndexParam(final String str) {
+    this(DOMAIN.parse(str));
   }
 
-  /**
-   * @param access sets authentication response
-   */
-  public void setAccess(AuthenticationResponse access) {
-    this.access = access;
+  @Override
+  public String getName() {
+    return NAME;
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java
new file mode 100644
index 0000000000000..22bce429b3fd5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/**
+ * The snapshot startPath parameter used by snapshotDiffReportListing.
+ */
+public class SnapshotDiffStartPathParam extends StringParam {
+  /** Parameter name. */
+  public static final String NAME = "snapshotdiffstartpath";
+
+  /** Default parameter value. */
+  public static final String DEFAULT = "";
+
+  private static final Domain DOMAIN = new Domain(NAME, null);
+
+  public SnapshotDiffStartPathParam(final String str) {
+    super(DOMAIN, str);
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
index 84cd771da4912..6c8d1c5fafb80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 20967cc13ab86..60792b5b6c94c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
@@ -332,6 +332,7 @@ message GetSnapshotDiffReportListingResponseProto {
 }
 message RenewLeaseRequestProto {
   required string clientName = 1;
+  repeated string namespaces = 2;
 }
 
 message RenewLeaseResponseProto { //void response
@@ -424,6 +425,13 @@ message GetPreferredBlockSizeResponseProto {
   required uint64 bsize = 1;
 }
 
+message GetSlowDatanodeReportRequestProto {
+}
+
+message GetSlowDatanodeReportResponseProto {
+  repeated DatanodeInfoProto datanodeInfoProto = 1;
+}
+
 enum SafeModeActionProto {
   SAFEMODE_LEAVE = 1;
   SAFEMODE_ENTER = 2;
@@ -1070,4 +1078,6 @@ service ClientNamenodeProtocol {
       returns(SatisfyStoragePolicyResponseProto);
   rpc getHAServiceState(HAServiceStateRequestProto)
       returns(HAServiceStateResponseProto);
+  rpc getSlowDatanodeReport(GetSlowDatanodeReportRequestProto)
+      returns(GetSlowDatanodeReportResponseProto);
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
index 28a292e729e2a..5356cd6961699 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/datatransfer.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
@@ -44,6 +44,7 @@ message DataTransferEncryptorMessageProto {
   optional string message = 3;
   repeated CipherOptionProto cipherOption = 4;
   optional HandshakeSecretProto handshakeSecret = 5;
+  optional bool accessTokenError = 6;
 }
 
 message HandshakeSecretProto {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
index bcd82d63e0577..d280947cf5a74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index d92dd4cb84c97..fd3618fe73175 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -108,6 +108,7 @@ message BlockECReconstructionInfoProto {
   required StorageTypesProto targetStorageTypes = 5;
   required bytes liveBlockIndices = 6;
   required ErasureCodingPolicyProto ecPolicy = 7;
+  optional bytes excludeReconstructedIndices = 8;
 }
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 08fce71560602..e1e7f7d780dac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
@@ -481,6 +481,7 @@ message HdfsFileStatusProto {
 
   // Set of flags
   optional uint32 flags = 18 [default = 0];
+  optional string namespace = 19;
 }
 
 /**
@@ -722,3 +723,15 @@ message BlockTokenSecretProto {
   repeated string storageIds = 8;
   optional bytes handshakeSecret = 9;
 }
+
+/////////////////////////////////////////////////
+// Alignment state for namespaces.
+/////////////////////////////////////////////////
+/**
+ * Clients should receive this message in RPC responses and forward it
+ * in RPC requests without interpreting it. It should be encoded
+ * as an obscure byte array when being sent to clients.
+ */
+message RouterFederatedStateProto {
+  map<string, int64> namespaceStateIds = 1; // Last seen state IDs for multiple namespaces.
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
index e1ade19b27ee8..eb7a0c3549b08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/inotify.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax="proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java
index 4e6f4e3f4ba38..f7ea7bcd76dde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestReadOnly.java
@@ -76,7 +76,8 @@ public class TestReadOnly {
           "getQuotaUsage",
           "msync",
           "getHAServiceState",
-          "getECTopologyResultForPolicies"
+          "getECTopologyResultForPolicies",
+          "getSlowDatanodeReport"
       )
   );
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
index f28bfa65b59f5..c198536d01a2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java
@@ -35,7 +35,6 @@
 import org.slf4j.event.Level;
 
 import java.io.IOException;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 346f79cee9903..5e6cdf5eedc73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -20,6 +20,7 @@
 import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.reflect.Proxy;
 import java.net.ConnectException;
 import java.net.InetSocketAddress;
 import java.net.URI;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.io.retry.MultiException;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RpcInvocationHandler;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -101,6 +103,8 @@ public long[] answer(InvocationOnMock invocation) throws Throwable {
     RequestHedgingProxyProvider<ClientProtocol> provider =
         new RequestHedgingProxyProvider<>(conf, nnUri, ClientProtocol.class,
             createFactory(badMock, goodMock));
+    Assert.assertTrue(Proxy.getInvocationHandler(
+        provider.getProxy().proxy) instanceof RpcInvocationHandler);
     long[] stats = provider.getProxy().proxy.getStats();
     Assert.assertTrue(stats.length == 1);
     Mockito.verify(badMock).getStats();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 57005e9d80b0c..b471fd062ddee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -199,6 +199,16 @@
       <artifactId>bcprov-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
@@ -247,7 +257,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <threadCount>1</threadCount>
           <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
           <systemPropertyVariables>
@@ -343,38 +352,6 @@
           <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
@@ -393,7 +370,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
               <forkCount>1</forkCount>
               <reuseForks>true</reuseForks>
               <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 549eb704ee12f..f34a27e027710 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -24,8 +24,12 @@
 import java.util.List;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.MapType;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CommonPathCapabilities;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.DelegationTokenRenewer;
@@ -53,6 +57,7 @@
 import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.web.JsonUtilClient;
@@ -73,7 +78,7 @@
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 
 import java.io.BufferedInputStream;
@@ -92,7 +97,6 @@
 import java.security.PrivilegedExceptionAction;
 import java.text.MessageFormat;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
@@ -136,8 +140,12 @@ public class HttpFSFileSystem extends FileSystem
   public static final String POLICY_NAME_PARAM = "storagepolicy";
   public static final String SNAPSHOT_NAME_PARAM = "snapshotname";
   public static final String OLD_SNAPSHOT_NAME_PARAM = "oldsnapshotname";
+  public static final String SNAPSHOT_DIFF_START_PATH = "snapshotdiffstartpath";
+  public static final String SNAPSHOT_DIFF_INDEX = "snapshotdiffindex";
   public static final String FSACTION_MODE_PARAM = "fsaction";
   public static final String EC_POLICY_NAME_PARAM = "ecpolicy";
+  public static final String OFFSET_PARAM = "offset";
+  public static final String LENGTH_PARAM = "length";
 
   public static final Short DEFAULT_PERMISSION = 0755;
   public static final String ACLSPEC_DEFAULT = "";
@@ -237,6 +245,7 @@ public static FILE_TYPE getType(FileStatus fileStatus) {
 
   public static final String STORAGE_POLICIES_JSON = "BlockStoragePolicies";
   public static final String STORAGE_POLICY_JSON = "BlockStoragePolicy";
+  public static final String BLOCK_LOCATIONS_JSON = "BlockLocations";
 
   public static final int HTTP_TEMPORARY_REDIRECT = 307;
 
@@ -266,8 +275,9 @@ public enum Operation {
     RENAMESNAPSHOT(HTTP_PUT), GETSNAPSHOTDIFF(HTTP_GET),
     GETSNAPSHOTTABLEDIRECTORYLIST(HTTP_GET), GETSNAPSHOTLIST(HTTP_GET),
     GETSERVERDEFAULTS(HTTP_GET),
-    CHECKACCESS(HTTP_GET), SETECPOLICY(HTTP_PUT), GETECPOLICY(
-        HTTP_GET), UNSETECPOLICY(HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT);
+    CHECKACCESS(HTTP_GET), SETECPOLICY(HTTP_PUT), GETECPOLICY(HTTP_GET), UNSETECPOLICY(
+        HTTP_POST), SATISFYSTORAGEPOLICY(HTTP_PUT), GETSNAPSHOTDIFFLISTING(HTTP_GET),
+    GET_BLOCK_LOCATIONS(HTTP_GET);
 
     private String httpMethod;
 
@@ -1577,6 +1587,22 @@ public SnapshotDiffReport getSnapshotDiffReport(Path path,
     return JsonUtilClient.toSnapshotDiffReport(json);
   }
 
+  public SnapshotDiffReportListing getSnapshotDiffReportListing(Path path, String snapshotOldName,
+      String snapshotNewName, byte[] snapshotDiffStartPath, Integer snapshotDiffIndex)
+      throws IOException {
+    Map<String, String> params = new HashMap<>();
+    params.put(OP_PARAM, Operation.GETSNAPSHOTDIFFLISTING.toString());
+    params.put(SNAPSHOT_NAME_PARAM, snapshotNewName);
+    params.put(OLD_SNAPSHOT_NAME_PARAM, snapshotOldName);
+    params.put(SNAPSHOT_DIFF_START_PATH, DFSUtilClient.bytes2String(snapshotDiffStartPath));
+    params.put(SNAPSHOT_DIFF_INDEX, snapshotDiffIndex.toString());
+    HttpURLConnection conn = getConnection(
+        Operation.GETSNAPSHOTDIFFLISTING.getMethod(), params, path, true);
+    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+    return JsonUtilClient.toSnapshotDiffReportListing(json);
+  }
+
   public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList()
       throws IOException {
     Map<String, String> params = new HashMap<String, String>();
@@ -1692,4 +1718,41 @@ public void satisfyStoragePolicy(final Path path) throws IOException {
         Operation.SATISFYSTORAGEPOLICY.getMethod(), params, path, true);
     HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
   }
-}
+
+  @Override
+  public BlockLocation[] getFileBlockLocations(Path path, long start, long len)
+      throws IOException {
+    Map<String, String> params = new HashMap<>();
+    params.put(OP_PARAM, Operation.GETFILEBLOCKLOCATIONS.toString());
+    params.put(OFFSET_PARAM, Long.toString(start));
+    params.put(LENGTH_PARAM, Long.toString(len));
+    HttpURLConnection conn = getConnection(
+        Operation.GETFILEBLOCKLOCATIONS.getMethod(), params, path, true);
+    HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
+    JSONObject json = (JSONObject) HttpFSUtils.jsonParse(conn);
+    return toBlockLocations(json);
+  }
+
+  @Override
+  public BlockLocation[] getFileBlockLocations(final FileStatus status,
+      final long offset, final long length) throws IOException {
+    if (status == null) {
+      return null;
+    }
+    return getFileBlockLocations(status.getPath(), offset, length);
+  }
+
+  @VisibleForTesting
+  static BlockLocation[] toBlockLocations(JSONObject json) throws IOException {
+    ObjectMapper mapper = new ObjectMapper();
+    MapType subType = mapper.getTypeFactory().constructMapType(Map.class,
+        String.class, BlockLocation[].class);
+    MapType rootType = mapper.getTypeFactory().constructMapType(Map.class,
+        mapper.constructType(String.class), mapper.constructType(subType));
+
+    Map<String, Map<String, BlockLocation[]>> jsonMap =
+        mapper.readValue(json.toJSONString(), rootType);
+    Map<String, BlockLocation[]> locationMap = jsonMap.get(BLOCK_LOCATIONS_JSON);
+    return locationMap.get(BlockLocation.class.getSimpleName());
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index e272cdc71b686..9f70351dfcae3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.BlockStoragePolicySpi;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileChecksum;
@@ -44,7 +45,9 @@
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.web.JsonUtil;
@@ -1879,6 +1882,65 @@ public String execute(FileSystem fs) throws IOException {
     }
   }
 
+  /**
+   *  Executor that performs a getSnapshotDiffListing operation.
+   */
+  @InterfaceAudience.Private
+  public static class FSGetSnapshotDiffListing implements
+      FileSystemAccess.FileSystemExecutor<String> {
+
+    private final Path path;
+    private final String oldSnapshotName;
+    private final String snapshotName;
+    private final String snapshotDiffStartPath;
+    private final int snapshotDiffIndex;
+
+    /**
+     * Creates a getSnapshotDiffListing executor.
+     *
+     * @param path directory path of the snapshots to be examined.
+     * @param oldSnapshotName Older snapshot name.
+     * @param snapshotName Newer snapshot name.
+     * @param snapshotDiffStartPath snapshot diff start path.
+     * @param snapshotDiffIndex snapshot diff index.
+     */
+    public FSGetSnapshotDiffListing(String path, String oldSnapshotName,
+        String snapshotName, String snapshotDiffStartPath, int snapshotDiffIndex) {
+      this.path = new Path(path);
+      this.oldSnapshotName = oldSnapshotName;
+      this.snapshotName = snapshotName;
+      this.snapshotDiffStartPath = snapshotDiffStartPath;
+      this.snapshotDiffIndex = snapshotDiffIndex;
+    }
+
+    /**
+     * Executes the filesystem operation.
+     *
+     * @param fs filesystem instance to use.
+     * @return A serialized JSON string of snapshot diffs.
+     * @throws IOException thrown if an IO error occurred.
+     */
+    @Override
+    public String execute(FileSystem fs) throws IOException {
+      SnapshotDiffReportListing snapshotDiffReportListing = null;
+      if (fs instanceof DistributedFileSystem) {
+        DistributedFileSystem dfs = (DistributedFileSystem) fs;
+        snapshotDiffReportListing =
+            dfs.getSnapshotDiffReportListing(path, oldSnapshotName, snapshotName,
+                snapshotDiffStartPath, snapshotDiffIndex);
+      } else {
+        throw new UnsupportedOperationException("getSnapshotDiffListing is not "
+            + "supported for HttpFs on " + fs.getClass()
+            + ". Please check your fs.defaultFS configuration");
+      }
+      if (snapshotDiffReportListing != null) {
+        return JsonUtil.toJsonString(snapshotDiffReportListing);
+      } else {
+        return "";
+      }
+    }
+  }
+
   /**
    *  Executor that performs a getSnapshottableDirListing operation.
    */
@@ -2132,4 +2194,75 @@ public Void execute(FileSystem fs) throws IOException {
       return null;
     }
   }
+
+  /**
+   * Executor that performs a getFileBlockLocations operation.
+   */
+
+  @InterfaceAudience.Private
+  @SuppressWarnings("rawtypes")
+  public static class FSFileBlockLocations implements FileSystemAccess.FileSystemExecutor<Map> {
+    final private Path path;
+    final private long offsetValue;
+    final private long lengthValue;
+
+    /**
+     * Creates a file-block-locations executor.
+     *
+     * @param path the path to retrieve the location
+     * @param offsetValue offset into the given file
+     * @param lengthValue length for which to get locations for
+     */
+    public FSFileBlockLocations(String path, long offsetValue, long lengthValue) {
+      this.path = new Path(path);
+      this.offsetValue = offsetValue;
+      this.lengthValue = lengthValue;
+    }
+
+    @Override
+    public Map execute(FileSystem fs) throws IOException {
+      BlockLocation[] locations = fs.getFileBlockLocations(this.path,
+          this.offsetValue, this.lengthValue);
+      return JsonUtil.toJsonMap(locations);
+    }
+  }
+
+  /**
+   * Executor that performs a getFileBlockLocations operation for legacy
+   * clients that supports only GET_BLOCK_LOCATIONS.
+   */
+
+  @InterfaceAudience.Private
+  @SuppressWarnings("rawtypes")
+  public static class FSFileBlockLocationsLegacy
+      implements FileSystemAccess.FileSystemExecutor<Map> {
+    final private Path path;
+    final private long offsetValue;
+    final private long lengthValue;
+
+    /**
+     * Creates a file-block-locations executor.
+     *
+     * @param path the path to retrieve the location
+     * @param offsetValue offset into the given file
+     * @param lengthValue length for which to get locations for
+     */
+    public FSFileBlockLocationsLegacy(String path, long offsetValue, long lengthValue) {
+      this.path = new Path(path);
+      this.offsetValue = offsetValue;
+      this.lengthValue = lengthValue;
+    }
+
+    @Override
+    public Map execute(FileSystem fs) throws IOException {
+      if (fs instanceof DistributedFileSystem) {
+        DistributedFileSystem dfs = (DistributedFileSystem)fs;
+        LocatedBlocks locations = dfs.getLocatedBlocks(
+            this.path, this.offsetValue, this.lengthValue);
+        return JsonUtil.toJsonMap(locations);
+      }
+      throw new IOException("Unable to support FSFileBlockLocationsLegacy " +
+          "because the file system is not DistributedFileSystem.");
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
index f6c84dcae4e07..6943636f8fc0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.lib.wsrs.BooleanParam;
 import org.apache.hadoop.lib.wsrs.EnumParam;
 import org.apache.hadoop.lib.wsrs.EnumSetParam;
+import org.apache.hadoop.lib.wsrs.IntegerParam;
 import org.apache.hadoop.lib.wsrs.LongParam;
 import org.apache.hadoop.lib.wsrs.Param;
 import org.apache.hadoop.lib.wsrs.ParametersProvider;
@@ -59,7 +60,8 @@ public class HttpFSParametersProvider extends ParametersProvider {
     PARAMS_DEF.put(Operation.GETQUOTAUSAGE, new Class[]{});
     PARAMS_DEF.put(Operation.GETFILECHECKSUM,
         new Class[]{NoRedirectParam.class});
-    PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS, new Class[]{});
+    PARAMS_DEF.put(Operation.GETFILEBLOCKLOCATIONS,
+        new Class[] {OffsetParam.class, LenParam.class});
     PARAMS_DEF.put(Operation.GETACLSTATUS, new Class[]{});
     PARAMS_DEF.put(Operation.GETTRASHROOT, new Class[]{});
     PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{});
@@ -112,6 +114,9 @@ public class HttpFSParametersProvider extends ParametersProvider {
     PARAMS_DEF.put(Operation.RENAMESNAPSHOT,
             new Class[] {OldSnapshotNameParam.class,
                 SnapshotNameParam.class});
+    PARAMS_DEF.put(Operation.GETSNAPSHOTDIFFLISTING,
+        new Class[] {OldSnapshotNameParam.class, SnapshotNameParam.class,
+            SnapshotDiffStartPathParam.class, SnapshotDiffIndexParam.class});
     PARAMS_DEF.put(Operation.GETSNAPSHOTDIFF,
         new Class[] {OldSnapshotNameParam.class,
             SnapshotNameParam.class});
@@ -123,6 +128,7 @@ public class HttpFSParametersProvider extends ParametersProvider {
     PARAMS_DEF.put(Operation.GETECPOLICY, new Class[] {});
     PARAMS_DEF.put(Operation.UNSETECPOLICY, new Class[] {});
     PARAMS_DEF.put(Operation.SATISFYSTORAGEPOLICY, new Class[] {});
+    PARAMS_DEF.put(Operation.GET_BLOCK_LOCATIONS, new Class[] {OffsetParam.class, LenParam.class});
   }
 
   public HttpFSParametersProvider() {
@@ -669,6 +675,44 @@ public OldSnapshotNameParam() {
     }
   }
 
+  /**
+   * Class for SnapshotDiffStartPath parameter.
+   */
+  public static class SnapshotDiffStartPathParam extends StringParam {
+
+    /**
+     * Parameter name.
+     */
+    public static final String NAME = HttpFSFileSystem.SNAPSHOT_DIFF_START_PATH;
+
+    /**
+     * Constructor.
+     */
+    public SnapshotDiffStartPathParam() {
+      super(NAME, "");
+    }
+
+  }
+
+  /**
+   * Class for SnapshotDiffStartPath parameter.
+   */
+  public static class SnapshotDiffIndexParam extends IntegerParam {
+
+    /**
+     * Parameter name.
+     */
+    public static final String NAME = HttpFSFileSystem.SNAPSHOT_DIFF_INDEX;
+
+    /**
+     * Constructor.
+     */
+    public SnapshotDiffIndexParam() {
+      super(NAME, null);
+    }
+
+  }
+
   /**
    * Class for FsAction parameter.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index d0d76d6289912..b50d24900ac0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -370,7 +370,23 @@ public InputStream run() throws Exception {
       break;
     }
     case GETFILEBLOCKLOCATIONS: {
-      response = Response.status(Response.Status.BAD_REQUEST).build();
+      long offset = 0;
+      long len = Long.MAX_VALUE;
+      Long offsetParam = params.get(OffsetParam.NAME, OffsetParam.class);
+      Long lenParam = params.get(LenParam.NAME, LenParam.class);
+      AUDIT_LOG.info("[{}] offset [{}] len [{}]", path, offsetParam, lenParam);
+      if (offsetParam != null && offsetParam > 0) {
+        offset = offsetParam;
+      }
+      if (lenParam != null && lenParam > 0) {
+        len = lenParam;
+      }
+      FSOperations.FSFileBlockLocations command =
+          new FSOperations.FSFileBlockLocations(path, offset, len);
+      @SuppressWarnings("rawtypes")
+      Map locations = fsExecute(user, command);
+      final String json = JsonUtil.toJsonString("BlockLocations", locations);
+      response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
       break;
     }
     case GETACLSTATUS: {
@@ -450,6 +466,24 @@ public InputStream run() throws Exception {
       response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
       break;
     }
+    case GETSNAPSHOTDIFFLISTING: {
+      String oldSnapshotName = params.get(OldSnapshotNameParam.NAME,
+          OldSnapshotNameParam.class);
+      String snapshotName = params.get(SnapshotNameParam.NAME,
+          SnapshotNameParam.class);
+      String snapshotDiffStartPath = params
+          .get(HttpFSParametersProvider.SnapshotDiffStartPathParam.NAME,
+              HttpFSParametersProvider.SnapshotDiffStartPathParam.class);
+      Integer snapshotDiffIndex = params.get(HttpFSParametersProvider.SnapshotDiffIndexParam.NAME,
+          HttpFSParametersProvider.SnapshotDiffIndexParam.class);
+      FSOperations.FSGetSnapshotDiffListing command =
+          new FSOperations.FSGetSnapshotDiffListing(path, oldSnapshotName,
+              snapshotName, snapshotDiffStartPath, snapshotDiffIndex);
+      String js = fsExecute(user, command);
+      AUDIT_LOG.info("[{}]", path);
+      response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+      break;
+    }
     case GETSNAPSHOTTABLEDIRECTORYLIST: {
       FSOperations.FSGetSnapshottableDirListing command =
           new FSOperations.FSGetSnapshottableDirListing();
@@ -492,6 +526,26 @@ public InputStream run() throws Exception {
       response = Response.ok(js).type(MediaType.APPLICATION_JSON).build();
       break;
     }
+    case GET_BLOCK_LOCATIONS: {
+      long offset = 0;
+      long len = Long.MAX_VALUE;
+      Long offsetParam = params.get(OffsetParam.NAME, OffsetParam.class);
+      Long lenParam = params.get(LenParam.NAME, LenParam.class);
+      AUDIT_LOG.info("[{}] offset [{}] len [{}]", path, offsetParam, lenParam);
+      if (offsetParam != null && offsetParam > 0) {
+        offset = offsetParam;
+      }
+      if (lenParam != null && lenParam > 0) {
+        len = lenParam;
+      }
+      FSOperations.FSFileBlockLocationsLegacy command =
+          new FSOperations.FSFileBlockLocationsLegacy(path, offset, len);
+      @SuppressWarnings("rawtypes")
+      Map locations = fsExecute(user, command);
+      final String json = JsonUtil.toJsonString("LocatedBlocks", locations);
+      response = Response.ok(json).type(MediaType.APPLICATION_JSON).build();
+      break;
+    }
     default: {
       throw new IOException(
           MessageFormat.format("Invalid HTTP GET operation [{0}]", op.value()));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
index 2d0a5b8cd2e7d..e97de0275ca22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -162,9 +162,10 @@ Name               | Description
 /logs              | Display log files
 /stacks            | Display JVM stacks
 /static/index.html | The static home page
+/prof              | Async Profiler endpoint
 
 To control the access to servlet `/conf`, `/jmx`, `/logLevel`, `/logs`,
-and `/stacks`, configure the following properties in `httpfs-site.xml`:
+`/stacks` and `/prof`, configure the following properties in `httpfs-site.xml`:
 
 ```xml
   <property>
@@ -178,7 +179,7 @@ and `/stacks`, configure the following properties in `httpfs-site.xml`:
     <value>true</value>
     <description>
       Indicates if administrator ACLs are required to access
-      instrumentation servlets (JMX, METRICS, CONF, STACKS).
+      instrumentation servlets (JMX, METRICS, CONF, STACKS, PROF).
     </description>
   </property>
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index d8dd7c7c362d7..41dc03d59e22f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.http.client;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.BlockStoragePolicySpi;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.ContentSummary;
@@ -42,6 +43,8 @@
 import org.apache.hadoop.hdfs.AppendTestUtil;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -50,6 +53,7 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
@@ -70,6 +74,10 @@
 import org.apache.hadoop.test.TestJetty;
 import org.apache.hadoop.test.TestJettyHelper;
 import org.apache.hadoop.util.Lists;
+
+import org.json.simple.JSONObject;
+import org.json.simple.parser.ContainerFactory;
+import org.json.simple.parser.JSONParser;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Test;
@@ -98,11 +106,11 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 @RunWith(value = Parameterized.class)
 public abstract class BaseTestHttpFSWith extends HFSTestCase {
-
   protected abstract Path getProxiedFSTestDir();
 
   protected abstract String getProxiedFSURI();
@@ -188,7 +196,7 @@ protected FileSystem getHttpFSFileSystem() throws Exception {
 
   protected void testGet() throws Exception {
     FileSystem fs = getHttpFSFileSystem();
-    Assert.assertNotNull(fs);
+    assertNotNull(fs);
     URI uri = new URI(getScheme() + "://" +
                       TestJettyHelper.getJettyURL().toURI().getAuthority());
     assertEquals(fs.getUri(), uri);
@@ -1198,7 +1206,7 @@ protected enum Operation {
     ALLOW_SNAPSHOT, DISALLOW_SNAPSHOT, DISALLOW_SNAPSHOT_EXCEPTION,
     FILE_STATUS_ATTR, GET_SNAPSHOT_DIFF, GET_SNAPSHOTTABLE_DIRECTORY_LIST,
     GET_SNAPSHOT_LIST, GET_SERVERDEFAULTS, CHECKACCESS, SETECPOLICY,
-    SATISFYSTORAGEPOLICY
+    SATISFYSTORAGEPOLICY, GET_SNAPSHOT_DIFF_LISTING, GETFILEBLOCKLOCATIONS
   }
 
   private void operation(Operation op) throws Exception {
@@ -1335,6 +1343,12 @@ private void operation(Operation op) throws Exception {
     case SATISFYSTORAGEPOLICY:
       testStoragePolicySatisfier();
       break;
+    case GET_SNAPSHOT_DIFF_LISTING:
+      testGetSnapshotDiffListing();
+      break;
+    case GETFILEBLOCKLOCATIONS:
+      testGetFileBlockLocations();
+      break;
     }
 
   }
@@ -1607,29 +1621,30 @@ private void testGetSnapshotDiff() throws Exception {
       Path file2 = new Path(path, "file2");
       testCreate(file2, false);
       fs.createSnapshot(path, "snap2");
-      // Get snapshot diff
-      SnapshotDiffReport diffReport = null;
-      if (fs instanceof HttpFSFileSystem) {
-        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
-        diffReport = httpFS.getSnapshotDiffReport(path, "snap1", "snap2");
-      } else if (fs instanceof WebHdfsFileSystem) {
-        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
-        diffReport = webHdfsFileSystem.getSnapshotDiffReport(path,
-            "snap1", "snap2");
-      } else {
-        Assert.fail(fs.getClass().getSimpleName() +
-            " doesn't support getSnapshotDiff");
+
+      try {
+        // Get snapshot diff
+        SnapshotDiffReport diffReport = null;
+        if (fs instanceof HttpFSFileSystem) {
+          HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
+          diffReport = httpFS.getSnapshotDiffReport(path, "snap1", "snap2");
+        } else if (fs instanceof WebHdfsFileSystem) {
+          WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
+          diffReport = webHdfsFileSystem.getSnapshotDiffReport(path, "snap1", "snap2");
+        } else {
+          Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff");
+        }
+        // Verify result with DFS
+        DistributedFileSystem dfs =
+            (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
+        SnapshotDiffReport dfsDiffReport = dfs.getSnapshotDiffReport(path, "snap1", "snap2");
+        Assert.assertEquals(diffReport.toString(), dfsDiffReport.toString());
+      } finally {
+        // Cleanup
+        fs.deleteSnapshot(path, "snap2");
+        fs.deleteSnapshot(path, "snap1");
+        fs.delete(path, true);
       }
-      // Verify result with DFS
-      DistributedFileSystem dfs = (DistributedFileSystem)
-          FileSystem.get(path.toUri(), this.getProxiedFSConf());
-      SnapshotDiffReport dfsDiffReport =
-          dfs.getSnapshotDiffReport(path, "snap1", "snap2");
-      Assert.assertEquals(diffReport.toString(), dfsDiffReport.toString());
-      // Cleanup
-      fs.deleteSnapshot(path, "snap2");
-      fs.deleteSnapshot(path, "snap1");
-      fs.delete(path, true);
     }
   }
 
@@ -1951,4 +1966,133 @@ public void testStoragePolicySatisfier() throws Exception {
       dfs.delete(path1, true);
     }
   }
+
+  private void testGetFileBlockLocations() throws Exception {
+    BlockLocation[] blockLocations;
+    Path testFile;
+    if (!this.isLocalFS()) {
+      FileSystem fs = this.getHttpFSFileSystem();
+      testFile = new Path(getProxiedFSTestDir(), "singleBlock.txt");
+      DFSTestUtil.createFile(fs, testFile, 1, (short) 1, 0L);
+      if (fs instanceof HttpFSFileSystem) {
+        HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
+        blockLocations = httpFS.getFileBlockLocations(testFile, 0, 1);
+        assertNotNull(blockLocations);
+
+        // verify HttpFSFileSystem.toBlockLocations()
+        String jsonString = JsonUtil.toJsonString(blockLocations);
+        JSONParser parser = new JSONParser();
+        JSONObject jsonObject = (JSONObject) parser.parse(jsonString, (ContainerFactory) null);
+        BlockLocation[] deserializedLocation = HttpFSFileSystem.toBlockLocations(jsonObject);
+        assertEquals(blockLocations.length, deserializedLocation.length);
+        for (int i = 0; i < blockLocations.length; i++) {
+          assertEquals(blockLocations[i].toString(), deserializedLocation[i].toString());
+        }
+      } else if (fs instanceof WebHdfsFileSystem) {
+        WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
+        blockLocations = webHdfsFileSystem.getFileBlockLocations(testFile, 0, 1);
+        assertNotNull(blockLocations);
+      } else {
+        Assert.fail(fs.getClass().getSimpleName() + " doesn't support access");
+      }
+    }
+  }
+
+  private void testGetSnapshotDiffListing() throws Exception {
+    if (!this.isLocalFS()) {
+      // Create a directory with snapshot allowed
+      Path path = new Path("/tmp/tmp-snap-test");
+      createSnapshotTestsPreconditions(path);
+      // Get the FileSystem instance that's being tested
+      FileSystem fs = this.getHttpFSFileSystem();
+      // Check FileStatus
+      Assert.assertTrue(fs.getFileStatus(path).isSnapshotEnabled());
+      // Create a file and take a snapshot
+      Path file1 = new Path(path, "file1");
+      testCreate(file1, false);
+      fs.createSnapshot(path, "snap1");
+      // Create another file and take a snapshot
+      Path file2 = new Path(path, "file2");
+      testCreate(file2, false);
+      fs.createSnapshot(path, "snap2");
+      // Get snapshot diff listing
+      try {
+        SnapshotDiffReportListing diffReportListing = null;
+        byte[] emptyBytes = new byte[] {};
+        if (fs instanceof HttpFSFileSystem) {
+          HttpFSFileSystem httpFS = (HttpFSFileSystem) fs;
+          diffReportListing =
+              httpFS.getSnapshotDiffReportListing(path, "snap1", "snap2", emptyBytes, -1);
+        } else if (fs instanceof WebHdfsFileSystem) {
+          WebHdfsFileSystem webHdfsFileSystem = (WebHdfsFileSystem) fs;
+          diffReportListing = webHdfsFileSystem
+              .getSnapshotDiffReportListing(path.toUri().getPath(), "snap1", "snap2", emptyBytes,
+                  -1);
+        } else {
+          Assert.fail(fs.getClass().getSimpleName() + " doesn't support getSnapshotDiff");
+        }
+        // Verify result with DFS
+        DistributedFileSystem dfs =
+            (DistributedFileSystem) FileSystem.get(path.toUri(), this.getProxiedFSConf());
+        SnapshotDiffReportListing dfsDiffReportListing =
+            dfs.getSnapshotDiffReportListing(path, "snap1", "snap2",
+                DFSUtil.bytes2String(emptyBytes), -1);
+        assertHttpFsReportListingWithDfsClient(diffReportListing, dfsDiffReportListing);
+      } finally {
+        // Cleanup
+        fs.deleteSnapshot(path, "snap2");
+        fs.deleteSnapshot(path, "snap1");
+        fs.delete(path, true);
+      }
+    }
+  }
+
+  private void assertHttpFsReportListingWithDfsClient(SnapshotDiffReportListing diffReportListing,
+      SnapshotDiffReportListing dfsDiffReportListing) {
+    Assert.assertEquals(diffReportListing.getCreateList().size(),
+        dfsDiffReportListing.getCreateList().size());
+    Assert.assertEquals(diffReportListing.getDeleteList().size(),
+        dfsDiffReportListing.getDeleteList().size());
+    Assert.assertEquals(diffReportListing.getModifyList().size(),
+        dfsDiffReportListing.getModifyList().size());
+    Assert.assertEquals(diffReportListing.getIsFromEarlier(),
+        dfsDiffReportListing.getIsFromEarlier());
+    Assert.assertEquals(diffReportListing.getLastIndex(), dfsDiffReportListing.getLastIndex());
+    Assert.assertEquals(DFSUtil.bytes2String(diffReportListing.getLastPath()),
+        DFSUtil.bytes2String(dfsDiffReportListing.getLastPath()));
+    int i = 0;
+    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
+        .getCreateList()) {
+      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
+          dfsDiffReportListing.getCreateList().get(i);
+      Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
+      Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
+      Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
+          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
+      i++;
+    }
+    i = 0;
+    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
+        .getDeleteList()) {
+      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
+          dfsDiffReportListing.getDeleteList().get(i);
+      Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
+      Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
+      Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
+          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
+      i++;
+    }
+    i = 0;
+    for (SnapshotDiffReportListing.DiffReportListingEntry entry : diffReportListing
+        .getModifyList()) {
+      SnapshotDiffReportListing.DiffReportListingEntry dfsDiffEntry =
+          dfsDiffReportListing.getModifyList().get(i);
+      Assert.assertEquals(entry.getDirId(), dfsDiffEntry.getDirId());
+      Assert.assertEquals(entry.getFileId(), dfsDiffEntry.getFileId());
+      Assert.assertArrayEquals(DFSUtilClient.byteArray2bytes(entry.getSourcePath()),
+          DFSUtilClient.byteArray2bytes(dfsDiffEntry.getSourcePath()));
+      i++;
+    }
+  }
+
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystemLocalFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystemLocalFileSystem.java
index 83bcb2efacc3c..955529ef9816d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystemLocalFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/TestHttpFSFileSystemLocalFileSystem.java
@@ -32,8 +32,6 @@
 import org.junit.runners.Parameterized;
 
 import java.io.File;
-import java.net.URI;
-import java.net.URISyntaxException;
 
 @RunWith(value = Parameterized.class)
 public class TestHttpFSFileSystemLocalFileSystem extends BaseTestHttpFSWith {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java
index 7a010e034d2f0..1411cbf78a180 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSAccessControlled.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.test.HTestCase;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
index f584b56ebbcb2..89efdb24edf61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
@@ -36,12 +36,14 @@
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.web.JsonUtil;
+import org.apache.hadoop.hdfs.web.JsonUtilClient;
 import org.apache.hadoop.lib.service.FileSystemAccess;
 import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
 import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
 import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
+import org.apache.hadoop.util.JsonSerialization;
 import org.json.simple.JSONArray;
 import org.junit.Assert;
 
@@ -70,6 +72,7 @@
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Path;
@@ -2003,4 +2006,38 @@ public void testContentType() throws Exception {
         () -> HttpFSUtils.jsonParse(conn));
     conn.disconnect();
   }
+
+  @Test
+  @TestDir
+  @TestJetty
+  @TestHdfs
+  public void testGetFileBlockLocations() throws Exception {
+    createHttpFSServer(false, false);
+    // Create a test directory
+    String pathStr = "/tmp/tmp-get-block-location-test";
+    createDirWithHttp(pathStr, "700", null);
+
+    Path path = new Path(pathStr);
+    DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
+        .get(path.toUri(), TestHdfsHelper.getHdfsConf());
+
+    String file1 = pathStr + "/file1";
+    createWithHttp(file1, null);
+    HttpURLConnection conn = sendRequestToHttpFSServer(file1,
+        "GETFILEBLOCKLOCATIONS", "length=10&offset10");
+    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+    BlockLocation[] locations1 = dfs.getFileBlockLocations(new Path(file1), 0, 1);
+    Assert.assertNotNull(locations1);
+
+    Map<?, ?> jsonMap = JsonSerialization.mapReader().readValue(conn.getInputStream());
+
+    BlockLocation[] httpfsBlockLocations = JsonUtilClient.toBlockLocationArray(jsonMap);
+
+    assertEquals(locations1.length, httpfsBlockLocations.length);
+    for (int i = 0; i < locations1.length; i++) {
+      assertEquals(locations1[i].toString(), httpfsBlockLocations[i].toString());
+    }
+
+    conn.getInputStream().close();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/server/TestServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/server/TestServer.java
index a6a139f23e16b..ff1d1ca0ad55c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/server/TestServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/server/TestServer.java
@@ -42,7 +42,6 @@
 import org.apache.hadoop.test.TestDir;
 import org.apache.hadoop.test.TestDirHelper;
 import org.apache.hadoop.test.TestException;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Test;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java
deleted file mode 100644
index 4af93182ed341..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/service/security/DummyGroupMapping.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.lib.service.security;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.security.GroupMappingServiceProvider;
-import org.apache.hadoop.test.HadoopUsersConfTestHelper;
-import org.apache.hadoop.util.Sets;
-
-public class DummyGroupMapping implements GroupMappingServiceProvider {
-
-  @Override
-  public List<String> getGroups(String user) throws IOException {
-    if (user.equals("root")) {
-      return Arrays.asList("admin");
-    }
-    else if (user.equals("nobody")) {
-      return Arrays.asList("nobody");
-    } else {
-      String[] groups = HadoopUsersConfTestHelper.getHadoopUserGroups(user);
-      return (groups != null) ? Arrays.asList(groups) : Collections.emptyList();
-    }
-  }
-
-  @Override
-  public void cacheGroupsRefresh() throws IOException {
-  }
-
-  @Override
-  public void cacheGroupsAdd(List<String> groups) throws IOException {
-  }
-
-  @Override
-  public Set<String> getGroupsSet(String user) throws IOException {
-    if (user.equals("root")) {
-      return Sets.newHashSet("admin");
-    } else if (user.equals("nobody")) {
-      return Sets.newHashSet("nobody");
-    } else {
-      String[] groups = HadoopUsersConfTestHelper.getHadoopUserGroups(user);
-      return (groups != null) ? Sets.newHashSet(groups) :
-          Collections.emptySet();
-    }
-  }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index 279b0e6dc1abd..3f25354e293b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -95,38 +95,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           </excludes>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
   <profiles>
@@ -179,7 +147,23 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
                     <mkdir dir="${project.build.directory}/native"/>
                     <exec executable="cmake" dir="${project.build.directory}/native"
                           failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DHADOOP_BUILD=1 -DREQUIRE_FUSE=${require.fuse} -DREQUIRE_VALGRIND=${require.valgrind} -A '${env.PLATFORM}'"/>
+                      <arg line="${basedir}/src/"/>
+                      <arg line="-DGENERATED_JAVAH=${project.build.directory}/native/javah"/>
+                      <arg line="-DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+                      <arg line="-DREQUIRE_VALGRIND=${require.valgrind}"/>
+                      <arg line="-DHADOOP_BUILD=1"/>
+                      <arg line="-DREQUIRE_LIBWEBHDFS=${require.libwebhdfs}"/>
+                      <arg line="-DREQUIRE_OPENSSL=${require.openssl}"/>
+                      <arg line="-DCUSTOM_OPENSSL_PREFIX=${openssl.prefix}"/>
+                      <arg line="-DCUSTOM_OPENSSL_LIB=${openssl.lib}"/>
+                      <arg line="-DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
+                      <arg line="-DCMAKE_PREFIX_PATH=${windows.cmake.prefix.path}"/>
+                      <arg line="-DCMAKE_TOOLCHAIN_FILE=${windows.cmake.toolchain.file}"/>
+                      <arg line="-DCMAKE_BUILD_TYPE=${windows.cmake.build.type}"/>
+                      <arg line="-DBUILD_SHARED_HDFSPP=${windows.build.hdfspp.dll}"/>
+                      <arg line="-DNO_SASL=${windows.no.sasl}"/>
+                      <arg line="-DREQUIRE_FUSE=${require.fuse}"/>
+                      <arg line="-A '${env.PLATFORM}'"/>
                       <arg line="${native_cmake_args}"/>
                     </exec>
                     <exec executable="msbuild" dir="${project.build.directory}/native"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index d8c2012fb88de..b44109f30928a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -18,12 +18,33 @@
 
 cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 
+# Need this CMake policy for using the MultiThreaded MSVC runtime library.
+cmake_policy(SET CMP0091 NEW)
+
+# Need this CMake policy for invoking target_link_libraries on a target in any directory.
+cmake_policy(SET CMP0079 NEW)
+
 project(hadoop_hdfs_native_client)
 
 enable_testing()
 
 set(CMAKE_CXX_STANDARD 17)
 
+if (MSVC)
+    # Instructs MSVC to use the static version of the runtime library.
+    # More info - https://docs.microsoft.com/en-us/cpp/build/reference/md-mt-ld-use-run-time-library?view=msvc-170
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")
+
+    # This indicates that Windows 10 and above are supported.
+    # More info - https://docs.microsoft.com/en-us/cpp/porting/modifying-winver-and-win32-winnt?view=msvc-170
+    set(NT_REQUIRED_VERSION 0x0A00)
+    add_definitions(-D_WIN32_WINNT=${NT_REQUIRED_VERSION})
+
+    # Windows.h defines the "min" and "max" macros which conflicts with the std::min and std::max from C++ STL.
+    # Adding the below definition will disable this macro.
+    add_definitions(-DNOMINMAX)
+endif (MSVC)
+
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
 include(HadoopCommon)
 
@@ -108,7 +129,7 @@ set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
 hadoop_set_find_shared_library_without_version()
 set(OPENSSL_NAME "crypto")
 if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
-    SET(OPENSSL_NAME "eay32")
+    SET(OPENSSL_NAME "libcrypto")
 endif()
 message("CUSTOM_OPENSSL_PREFIX = ${CUSTOM_OPENSSL_PREFIX}")
 find_library(OPENSSL_LIBRARY
@@ -127,7 +148,8 @@ if(OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
     include(CheckCSourceCompiles)
     set(OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
     set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR})
-    check_c_source_compiles("#include \"${OPENSSL_INCLUDE_DIR}/openssl/evp.h\"\nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL)
+    set(CMAKE_REQUIRED_LIBRARIES ${OPENSSL_LIBRARY})
+    check_c_source_compiles("#include <openssl/evp.h>\nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL)
     set(CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES})
     if(NOT HAS_NEW_ENOUGH_OPENSSL)
         message("The OpenSSL library installed at ${OPENSSL_LIBRARY} is too old.  You need a version at least new enough to have EVP_aes_256_ctr.")
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
index e8cc0e509f7ee..0dc17214348d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README
@@ -88,6 +88,7 @@ rw
 -odebug (do not daemonize - aka -d in fuse speak)
 -obig_writes (use fuse big_writes option so as to allow better performance of writes on kernels >= 2.6.26)
 -initchecks - have fuse-dfs try to connect to hdfs to ensure all is ok upon startup. recommended to have this  on
+-omax_background=%d (maximum number of pending "background" requests - see fuse docs)
 The defaults are:
 
 entry,attribute_timeouts = 60 seconds
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
index f693032d5c5ed..b9b01009beb1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c
@@ -81,6 +81,7 @@ int main(int argc, char *argv[])
   options.rdbuffer_size = 10*1024*1024; 
   options.attribute_timeout = 60; 
   options.entry_timeout = 60;
+  options.max_background = 0;
 
   if (-1 == fuse_opt_parse(&args, &options, dfs_opts, dfs_options)) {
     return -1;
@@ -114,6 +115,11 @@ int main(int argc, char *argv[])
 
     snprintf(buf, sizeof buf, "-oentry_timeout=%d",options.entry_timeout);
     fuse_opt_add_arg(&args, buf);
+
+    if (options.max_background > 0) {
+      snprintf(buf, sizeof buf, "-omax_background=%d",options.max_background);
+      fuse_opt_add_arg(&args, buf);
+    }
   }
 
   if (options.nn_uri == NULL) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.h
index 4554dbdbea584..5ab8ac4131b15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.h
@@ -25,7 +25,6 @@
 #include <string.h>
 #include <errno.h>
 #include <assert.h>
-#include <strings.h>
 #include <syslog.h>
 
 #include <fuse.h>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
index 205a7f40fded0..65e9e6d27682f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c
@@ -91,11 +91,11 @@ static void dfsPrintOptions(FILE *fp, const struct options *o)
   INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, "
           "debug=%d, read_only=%d, initchecks=%d, "
           "no_permissions=%d, usetrash=%d, entry_timeout=%d, "
-          "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]",
+          "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d, max_background=%d ]",
           (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, 
           o->debug, o->read_only, o->initchecks,
           o->no_permissions, o->usetrash, o->entry_timeout,
-          o->attribute_timeout, o->rdbuffer_size, o->direct_io);
+          o->attribute_timeout, o->rdbuffer_size, o->direct_io, o->max_background);
 }
 
 void *dfs_init(struct fuse_conn_info *conn)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
index 8461ce40f9186..b4082c63d783e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c
@@ -37,11 +37,13 @@ void print_options() {
 	 "\tentry_timeout=%d\n"
 	 "\tattribute_timeout=%d\n"
 	 "\tprivate=%d\n"
-	 "\trdbuffer_size=%d (KBs)\n", 
-	 options.protected, options.nn_uri, options.nn_port, options.debug,
+         "\trdbuffer_size=%d (KBs)\n"
+         "\tmax_background=%d\n",
+         options.protected, options.nn_uri, options.nn_port, options.debug,
 	 options.read_only, options.usetrash, options.entry_timeout, 
 	 options.attribute_timeout, options.private, 
-	 (int)options.rdbuffer_size / 1024);
+         (int)options.rdbuffer_size / 1024,
+         options.max_background);
 }
 
 const char *program;
@@ -56,7 +58,7 @@ void print_usage(const char *pname)
 	 "[-ousetrash] [-obig_writes] [-oprivate (single user)] [ro] "
 	 "[-oserver=<hadoop_servername>] [-oport=<hadoop_port>] "
 	 "[-oentry_timeout=<secs>] [-oattribute_timeout=<secs>] "
-	 "[-odirect_io] [-onopoermissions] [-o<other fuse option>] "
+         "[-odirect_io] [-onopoermissions] [-omax_background=<size>] [-o<other fuse option>] "
 	 "<mntpoint> [fuse options]\n", pname);
   printf("NOTE: debugging option for fuse is -debug\n");
 }
@@ -87,6 +89,7 @@ struct fuse_opt dfs_opts[] =
     DFSFS_OPT_KEY("protected=%s", protected, 0),
     DFSFS_OPT_KEY("port=%d", nn_port, 0),
     DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
+    DFSFS_OPT_KEY("max_background=%d", max_background, 0),
 
     FUSE_OPT_KEY("private", KEY_PRIVATE),
     FUSE_OPT_KEY("ro", KEY_RO),
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
index 4bfc2355259b3..2d00f1b30a1dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h
@@ -34,6 +34,7 @@ struct options {
   int private;
   size_t rdbuffer_size;
   int direct_io;
+  int max_background;
 } options;
 
 extern struct fuse_opt dfs_opts[];
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
index 02f1b5f282cd3..23a00a28898ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c
@@ -21,7 +21,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <strings.h>
 
 #include "fuse_context_handle.h"
 #include "fuse_dfs.h"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
index 33fe4464e65c6..68cd29a703b3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java
@@ -187,6 +187,7 @@ private static Process establishMount(URI uri) throws IOException  {
       "-ononempty",              // Don't complain about junk in mount point
       "-f",                      // Don't background the process
       "-ordbuffer=32768",        // Read buffer size in kb
+      "-omax_background=100",    // Set fuse max_background=100 (12 by default)
       "rw"
     };
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
index f16cc9eb1b033..36055cc009041 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
@@ -25,6 +25,7 @@ include_directories(
     ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs
     ${JNI_INCLUDE_DIRS}
     ${OS_DIR}
+    ../libhdfspp/lib
 )
 
 add_library(native_mini_dfs
@@ -34,6 +35,8 @@ add_library(native_mini_dfs
     ../libhdfs/jni_helper.c
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
+    $<TARGET_OBJECTS:x_platform_obj>
+    $<TARGET_OBJECTS:x_platform_obj_c_api>
 )
 
 add_executable(test_native_mini_dfs test_native_mini_dfs.c)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
index 1641470733f2c..7318aaafbae46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c
@@ -18,24 +18,27 @@
 
 #include "common/util_c.h"
 #include "expect.h"
-#include "hdfs/hdfs.h"
 #include "hdfspp/hdfs_ext.h"
 #include "native_mini_dfs.h"
 #include "os/thread.h"
 #include "x-platform/c-api/syscall.h"
+#include "hdfs/hdfs.h"
 
 #include <errno.h>
 #include <inttypes.h>
-#include <pwd.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/socket.h>
 #include <sys/types.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
+#ifndef WIN32
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <pwd.h>
+#endif
+
 #define TO_STR_HELPER(X) #X
 #define TO_STR(X) TO_STR_HELPER(X)
 
@@ -197,7 +200,7 @@ static int fileEventCallback1(const char * event, const char * cluster, const ch
   if (randomErrRatioStr) randomErrRatio = (int64_t)atoi(randomErrRatioStr);
   if (randomErrRatio == 0) return DEBUG_SIMULATE_ERROR;
   else if (randomErrRatio < 0) return LIBHDFSPP_EVENT_OK;
-  return random() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
+  return rand() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
 }
 
 static int fileEventCallback2(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie)
@@ -235,7 +238,7 @@ static int doTestHdfsMiniStress(struct tlhThreadInfo *ti, int randomErr)
     EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file));
     file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0);
     EXPECT_NONNULL(file);
-    seekPos = (((double)random()) / RAND_MAX) * (fileInfo->mSize - expected);
+    seekPos = (((double)rand()) / RAND_MAX) * (fileInfo->mSize - expected);
     seekPos = (seekPos / expected) * expected;
     ret = hdfsSeek(ti->hdfs, file, seekPos);
     if (ret < 0) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
index a3058bbe6ec06..359cc2f4e3269 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_ops.c
@@ -454,6 +454,68 @@ int main(int argc, char **argv) {
         hdfsCloseFile(lfs, localFile);
     }
 
+
+    {
+        // HDFS Open File Builder tests
+
+        exists = hdfsExists(fs, readPath);
+
+        if (exists) {
+            fprintf(stderr, "Failed to validate existence of %s\n", readPath);
+            shutdown_and_exit(cl, -1);
+        }
+
+        hdfsOpenFileBuilder *builder;
+        builder = hdfsOpenFileBuilderAlloc(fs, readPath);
+        hdfsOpenFileBuilderOpt(builder, "hello", "world");
+
+        hdfsOpenFileFuture *future;
+        future = hdfsOpenFileBuilderBuild(builder);
+
+        readFile = hdfsOpenFileFutureGet(future);
+        if (!hdfsOpenFileFutureCancel(future, 0)) {
+            fprintf(stderr, "Cancel on a completed Future should return false");
+            shutdown_and_exit(cl, -1);
+        }
+        hdfsOpenFileFutureFree(future);
+
+        memset(buffer, 0, sizeof(buffer));
+        num_read_bytes = hdfsRead(fs, readFile, (void *) buffer,
+                                  sizeof(buffer));
+        if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
+            fprintf(stderr,
+                    "Failed to read. Expected %s but got %s (%d bytes)\n",
+                    fileContents, buffer, num_read_bytes);
+            shutdown_and_exit(cl, -1);
+        }
+        hdfsCloseFile(fs, readFile);
+
+        builder = hdfsOpenFileBuilderAlloc(fs, readPath);
+        hdfsOpenFileBuilderOpt(builder, "hello", "world");
+
+        future = hdfsOpenFileBuilderBuild(builder);
+
+        readFile = hdfsOpenFileFutureGetWithTimeout(future, 1, jDays);
+        if (!hdfsOpenFileFutureCancel(future, 0)) {
+            fprintf(stderr, "Cancel on a completed Future should return "
+                            "false");
+            shutdown_and_exit(cl, -1);
+        }
+        hdfsOpenFileFutureFree(future);
+
+        memset(buffer, 0, sizeof(buffer));
+        num_read_bytes = hdfsRead(fs, readFile, (void*)buffer,
+                                  sizeof(buffer));
+        if (strncmp(fileContents, buffer, strlen(fileContents)) != 0) {
+            fprintf(stderr, "Failed to read. Expected %s but got "
+                            "%s (%d bytes)\n", fileContents, buffer,
+                            num_read_bytes);
+            shutdown_and_exit(cl, -1);
+        }
+        memset(buffer, 0, strlen(fileContents + 1));
+        hdfsCloseFile(fs, readFile);
+    }
+
     totalResult = 0;
     result = 0;
     {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt
index a7fb311125110..958bdcf2a41c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt
@@ -29,6 +29,7 @@ include_directories(
     main/native
     main/native/libhdfs
     ${OS_DIR}
+    ../libhdfspp/lib
 )
 
 hadoop_add_dual_library(hdfs
@@ -38,6 +39,8 @@ hadoop_add_dual_library(hdfs
     jclasses.c
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
+    $<TARGET_OBJECTS:x_platform_obj>
+    $<TARGET_OBJECTS:x_platform_obj_c_api>
 )
 if(NEED_LINK_DL)
    set(LIB_DL dl)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 60f2826c74173..ed150925cdb81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -38,6 +38,10 @@
 
 #define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path"
 
+// StreamCapability flags taken from o.a.h.fs.StreamCapabilities
+#define IS_READ_BYTE_BUFFER_CAPABILITY "in:readbytebuffer"
+#define IS_PREAD_BYTE_BUFFER_CAPABILITY "in:preadbytebuffer"
+
 // Bit fields for hdfsFile_internal flags
 #define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0)
 #define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<1)
@@ -1075,6 +1079,27 @@ static int hdfsHasStreamCapability(jobject jFile,
     return 0;
 }
 
+/**
+ * Sets the flags of the given hdfsFile based on the capabilities of the
+ * underlying stream.
+ *
+ * @param file file->flags will be updated based on the capabilities of jFile
+ * @param jFile the underlying stream to check for capabilities
+ */
+static void setFileFlagCapabilities(hdfsFile file, jobject jFile) {
+    // Check the StreamCapabilities of jFile to see if we can do direct
+    // reads
+    if (hdfsHasStreamCapability(jFile, IS_READ_BYTE_BUFFER_CAPABILITY)) {
+        file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ;
+    }
+
+    // Check the StreamCapabilities of jFile to see if we can do direct
+    // preads
+    if (hdfsHasStreamCapability(jFile, IS_PREAD_BYTE_BUFFER_CAPABILITY)) {
+        file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD;
+    }
+}
+
 static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags,
                   int32_t bufferSize, int16_t replication, int64_t blockSize)
 {
@@ -1245,17 +1270,7 @@ static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags,
     file->flags = 0;
 
     if ((flags & O_WRONLY) == 0) {
-        // Check the StreamCapabilities of jFile to see if we can do direct
-        // reads
-        if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) {
-            file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ;
-        }
-
-        // Check the StreamCapabilities of jFile to see if we can do direct
-        // preads
-        if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) {
-            file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD;
-        }
+        setFileFlagCapabilities(file, jFile);
     }
     ret = 0;
 
@@ -1288,6 +1303,469 @@ hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld)
     return file;
 }
 
+/**
+ * A wrapper around o.a.h.fs.FutureDataInputStreamBuilder and the file name
+ * associated with the builder.
+ */
+struct hdfsOpenFileBuilder {
+    jobject jBuilder;
+    const char *path;
+};
+
+/**
+ * A wrapper around a java.util.concurrent.Future (created by calling
+ * FutureDataInputStreamBuilder#build) and the file name associated with the
+ * builder.
+ */
+struct hdfsOpenFileFuture {
+    jobject jFuture;
+    const char *path;
+};
+
+hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs,
+        const char *path) {
+    int ret = 0;
+    jthrowable jthr;
+    jvalue jVal;
+    jobject jFS = (jobject) fs;
+
+    jobject jPath = NULL;
+    jobject jBuilder = NULL;
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        errno = EINTERNAL;
+        return NULL;
+    }
+
+    hdfsOpenFileBuilder *builder;
+    builder = calloc(1, sizeof(hdfsOpenFileBuilder));
+    if (!builder) {
+        fprintf(stderr, "hdfsOpenFileBuilderAlloc(%s): OOM when creating "
+                        "hdfsOpenFileBuilder\n", path);
+        errno = ENOMEM;
+        goto done;
+    }
+    builder->path = path;
+
+    jthr = constructNewObjectOfPath(env, path, &jPath);
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderAlloc(%s): constructNewObjectOfPath",
+                path);
+        goto done;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM,
+            "openFile", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FDISB)),
+            jPath);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderAlloc(%s): %s#openFile(Path) failed",
+                HADOOP_FS, path);
+        goto done;
+    }
+    jBuilder = jVal.l;
+
+    builder->jBuilder = (*env)->NewGlobalRef(env, jBuilder);
+    if (!builder->jBuilder) {
+        printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderAlloc(%s): NewGlobalRef(%s) failed", path,
+                HADOOP_FDISB);
+        ret = EINVAL;
+        goto done;
+    }
+
+done:
+    destroyLocalReference(env, jPath);
+    destroyLocalReference(env, jBuilder);
+    if (ret) {
+        if (builder) {
+            if (builder->jBuilder) {
+                (*env)->DeleteGlobalRef(env, builder->jBuilder);
+            }
+            free(builder);
+        }
+        errno = ret;
+        return NULL;
+    }
+    return builder;
+}
+
+/**
+ * Used internally by hdfsOpenFileBuilderWithOption to switch between
+ * FSBuilder#must and #opt.
+ */
+typedef enum { must, opt } openFileBuilderOptionType;
+
+/**
+ * Shared implementation of hdfsOpenFileBuilderMust and hdfsOpenFileBuilderOpt
+ * that switches between each method depending on the value of
+ * openFileBuilderOptionType.
+ */
+static hdfsOpenFileBuilder *hdfsOpenFileBuilderWithOption(
+        hdfsOpenFileBuilder *builder, const char *key,
+        const char *value, openFileBuilderOptionType optionType) {
+    int ret = 0;
+    jthrowable jthr;
+    jvalue jVal;
+    jobject localJBuilder = NULL;
+    jobject globalJBuilder;
+    jstring jKeyString = NULL;
+    jstring jValueString = NULL;
+
+    // If the builder was not previously created by a prior call to
+    // hdfsOpenFileBuilderAlloc then exit
+    if (builder == NULL || builder->jBuilder == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        errno = EINTERNAL;
+        return NULL;
+    }
+    jthr = newJavaStr(env, key, &jKeyString);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderWithOption(%s): newJavaStr(%s)",
+                builder->path, key);
+        goto done;
+    }
+    jthr = newJavaStr(env, value, &jValueString);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderWithOption(%s): newJavaStr(%s)",
+                builder->path, value);
+        goto done;
+    }
+
+    const char *optionTypeMethodName;
+    switch (optionType) {
+        case must:
+            optionTypeMethodName = "must";
+            break;
+        case opt:
+            optionTypeMethodName = "opt";
+            break;
+        default:
+            ret = EINTERNAL;
+            goto done;
+    }
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, builder->jBuilder,
+            JC_FUTURE_DATA_IS_BUILDER, optionTypeMethodName,
+            JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING),
+                    JPARAM(HADOOP_FS_BLDR)), jKeyString,
+            jValueString);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderWithOption(%s): %s#%s(%s, %s) failed",
+                builder->path, HADOOP_FS_BLDR, optionTypeMethodName, key,
+                value);
+        goto done;
+    }
+
+    localJBuilder = jVal.l;
+    globalJBuilder = (*env)->NewGlobalRef(env, localJBuilder);
+    if (!globalJBuilder) {
+        printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderWithOption(%s): NewGlobalRef(%s) failed",
+                builder->path, HADOOP_FDISB);
+        ret = EINVAL;
+        goto done;
+    }
+    (*env)->DeleteGlobalRef(env, builder->jBuilder);
+    builder->jBuilder = globalJBuilder;
+
+done:
+    destroyLocalReference(env, jKeyString);
+    destroyLocalReference(env, jValueString);
+    destroyLocalReference(env, localJBuilder);
+    if (ret) {
+        errno = ret;
+        return NULL;
+    }
+    return builder;
+}
+
+hdfsOpenFileBuilder *hdfsOpenFileBuilderMust(hdfsOpenFileBuilder *builder,
+        const char *key, const char *value) {
+    openFileBuilderOptionType optionType;
+    optionType = must;
+    return hdfsOpenFileBuilderWithOption(builder, key, value, optionType);
+}
+
+hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt(hdfsOpenFileBuilder *builder,
+        const char *key, const char *value) {
+    openFileBuilderOptionType optionType;
+    optionType = opt;
+    return hdfsOpenFileBuilderWithOption(builder, key, value, optionType);
+}
+
+hdfsOpenFileFuture *hdfsOpenFileBuilderBuild(hdfsOpenFileBuilder *builder) {
+    int ret = 0;
+    jthrowable jthr;
+    jvalue jVal;
+
+    jobject jFuture = NULL;
+
+    // If the builder was not previously created by a prior call to
+    // hdfsOpenFileBuilderAlloc then exit
+    if (builder == NULL || builder->jBuilder == NULL) {
+        ret = EINVAL;
+        return NULL;
+    }
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        errno = EINTERNAL;
+        return NULL;
+    }
+
+    hdfsOpenFileFuture *future;
+    future = calloc(1, sizeof(hdfsOpenFileFuture));
+    if (!future) {
+        fprintf(stderr, "hdfsOpenFileBuilderBuild: OOM when creating "
+                        "hdfsOpenFileFuture\n");
+        errno = ENOMEM;
+        goto done;
+    }
+    future->path = builder->path;
+
+    jthr = invokeMethod(env, &jVal, INSTANCE, builder->jBuilder,
+            JC_FUTURE_DATA_IS_BUILDER, "build",
+            JMETHOD1("", JPARAM(JAVA_CFUTURE)));
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderBuild(%s): %s#build() failed",
+                builder->path, HADOOP_FDISB);
+        goto done;
+    }
+    jFuture = jVal.l;
+
+    future->jFuture = (*env)->NewGlobalRef(env, jFuture);
+    if (!future->jFuture) {
+        printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "hdfsOpenFileBuilderBuild(%s): NewGlobalRef(%s) failed",
+                builder->path, JAVA_CFUTURE);
+        ret = EINVAL;
+        goto done;
+    }
+
+done:
+    destroyLocalReference(env, jFuture);
+    if (ret) {
+        if (future) {
+            if (future->jFuture) {
+                (*env)->DeleteGlobalRef(env, future->jFuture);
+            }
+            free(future);
+        }
+        hdfsOpenFileBuilderFree(builder);
+        errno = ret;
+        return NULL;
+    }
+    hdfsOpenFileBuilderFree(builder);
+    return future;
+}
+
+void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder) {
+    JNIEnv *env;
+    env = getJNIEnv();
+    if (!env) {
+        return;
+    }
+    if (builder->jBuilder) {
+        (*env)->DeleteGlobalRef(env, builder->jBuilder);
+        builder->jBuilder = NULL;
+    }
+    free(builder);
+}
+
+/**
+ * Shared implementation of hdfsOpenFileFutureGet and
+ * hdfsOpenFileFutureGetWithTimeout. If a timeout is specified, calls
+ * Future#get() otherwise it calls Future#get(long, TimeUnit).
+ */
+static hdfsFile fileFutureGetWithTimeout(hdfsOpenFileFuture *future,
+        int64_t timeout, jobject jTimeUnit) {
+    int ret = 0;
+    jthrowable jthr;
+    jvalue jVal;
+
+    hdfsFile file = NULL;
+    jobject jFile = NULL;
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        ret = EINTERNAL;
+        return NULL;
+    }
+
+    if (!jTimeUnit) {
+        jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture,
+                JC_CFUTURE, "get", JMETHOD1("", JPARAM(JAVA_OBJECT)));
+    } else {
+        jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture,
+                JC_CFUTURE, "get", JMETHOD2("J",
+                        JPARAM(JAVA_TIMEUNIT), JPARAM(JAVA_OBJECT)), timeout,
+                        jTimeUnit);
+    }
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileFutureGet(%s): %s#get failed", future->path,
+                JAVA_CFUTURE);
+        goto done;
+    }
+
+    file = calloc(1, sizeof(struct hdfsFile_internal));
+    if (!file) {
+        fprintf(stderr, "hdfsOpenFileFutureGet(%s): OOM when creating "
+                        "hdfsFile\n", future->path);
+        ret = ENOMEM;
+        goto done;
+    }
+    jFile = jVal.l;
+    file->file = (*env)->NewGlobalRef(env, jFile);
+    if (!file->file) {
+        ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL,
+                "hdfsOpenFileFutureGet(%s): NewGlobalRef(jFile) failed",
+                future->path);
+        goto done;
+    }
+
+    file->type = HDFS_STREAM_INPUT;
+    file->flags = 0;
+
+    setFileFlagCapabilities(file, jFile);
+
+done:
+    destroyLocalReference(env, jTimeUnit);
+    destroyLocalReference(env, jFile);
+    if (ret) {
+        if (file) {
+            if (file->file) {
+                (*env)->DeleteGlobalRef(env, file->file);
+            }
+            free(file);
+        }
+        errno = ret;
+        return NULL;
+    }
+    return file;
+}
+
+hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future) {
+    return fileFutureGetWithTimeout(future, -1, NULL);
+}
+
+hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future,
+        int64_t timeout, javaConcurrentTimeUnit timeUnit) {
+    int ret = 0;
+    jthrowable jthr;
+    jobject jTimeUnit = NULL;
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        ret = EINTERNAL;
+        return NULL;
+    }
+
+    const char *timeUnitEnumName;
+    switch (timeUnit) {
+        case jNanoseconds:
+            timeUnitEnumName = "NANOSECONDS";
+            break;
+        case jMicroseconds:
+            timeUnitEnumName = "MICROSECONDS";
+            break;
+        case jMilliseconds:
+            timeUnitEnumName = "MILLISECONDS";
+            break;
+        case jSeconds:
+            timeUnitEnumName = "SECONDS";
+            break;
+        case jMinutes:
+            timeUnitEnumName = "MINUTES";
+            break;
+        case jHours:
+            timeUnitEnumName = "HOURS";
+            break;
+        case jDays:
+            timeUnitEnumName = "DAYS";
+            break;
+        default:
+            ret = EINTERNAL;
+            goto done;
+    }
+
+    jthr = fetchEnumInstance(env, JAVA_TIMEUNIT, timeUnitEnumName, &jTimeUnit);
+
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileFutureGet(%s): %s#get failed", future->path,
+                JAVA_CFUTURE);
+        goto done;
+    }
+    return fileFutureGetWithTimeout(future, timeout, jTimeUnit);
+
+done:
+    if (ret) {
+        errno = ret;
+    }
+    return NULL;
+}
+
+int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future,
+        int mayInterruptIfRunning) {
+    int ret = 0;
+    jthrowable jthr;
+    jvalue jVal;
+
+    jboolean jMayInterruptIfRunning;
+
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        ret = EINTERNAL;
+        return -1;
+    }
+
+    jMayInterruptIfRunning = mayInterruptIfRunning ? JNI_TRUE : JNI_FALSE;
+    jthr = invokeMethod(env, &jVal, INSTANCE, future->jFuture, JC_CFUTURE,
+            "cancel", JMETHOD1("Z", "Z"), jMayInterruptIfRunning);
+    if (jthr) {
+        ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+                "hdfsOpenFileFutureCancel(%s): %s#cancel failed", future->path,
+                JAVA_CFUTURE);
+        goto done;
+    }
+
+done:
+    if (ret) {
+        errno = ret;
+        return -1;
+    }
+    if (!jVal.z) {
+        return -1;
+    }
+    return 0;
+}
+
+void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future) {
+    JNIEnv *env;
+    env = getJNIEnv();
+    if (!env) {
+        return;
+    }
+    if (future->jFuture) {
+        (*env)->DeleteGlobalRef(env, future->jFuture);
+        future->jFuture = NULL;
+    }
+    free(future);
+}
+
 int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength)
 {
     jobject jFS = (jobject)fs;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index e58a6232d205a..eba50ff6eb277 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -82,6 +82,29 @@ extern  "C" {
     } tObjectKind;
     struct hdfsStreamBuilder;
 
+    /**
+     * The C reflection of the enum values from java.util.concurrent.TimeUnit .
+     */
+    typedef enum javaConcurrentTimeUnit {
+        jNanoseconds,
+        jMicroseconds,
+        jMilliseconds,
+        jSeconds,
+        jMinutes,
+        jHours,
+        jDays,
+    } javaConcurrentTimeUnit;
+
+    /**
+     * The C reflection of java.util.concurrent.Future specifically used for
+     * opening HDFS files asynchronously.
+     */
+    typedef struct hdfsOpenFileFuture hdfsOpenFileFuture;
+
+    /**
+     * The C reflection of o.a.h.fs.FutureDataInputStreamBuilder .
+     */
+    typedef struct hdfsOpenFileBuilder hdfsOpenFileBuilder;
 
     /**
      * The C reflection of org.apache.org.hadoop.FileSystem .
@@ -429,6 +452,118 @@ extern  "C" {
     hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
                           int bufferSize, short replication, tSize blocksize);
 
+    /**
+     * hdfsOpenFileBuilderAlloc - Allocate a HDFS open file builder.
+     *
+     * @param fs The configured filesystem handle.
+     * @param path The full path to the file.
+     * @return Returns the hdfsOpenFileBuilder, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs,
+            const char *path);
+
+    /**
+     * hdfsOpenFileBuilderMust - Specifies a mandatory parameter for the open
+     * file builder. While the underlying FsBuilder supports various various
+     * types for the value (boolean, int, float, double), currently only
+     * strings are supported.
+     *
+     * @param builder The open file builder to set the config for.
+     * @param key The config key
+     * @param value The config value
+     * @return Returns the hdfsOpenFileBuilder, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsOpenFileBuilder *hdfsOpenFileBuilderMust(hdfsOpenFileBuilder *builder,
+            const char *key, const char *value);
+
+    /**
+     * hdfsOpenFileBuilderOpt - Specifies an optional parameter for the open
+     * file builder. While the underlying FsBuilder supports various various
+     * types for the value (boolean, int, float, double), currently only
+     * strings are supported.
+     *
+     * @param builder The open file builder to set the config for.
+     * @param key The config key
+     * @param value The config value
+     * @return Returns the hdfsOpenFileBuilder, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt(hdfsOpenFileBuilder *builder,
+            const char *key, const char *value);
+
+    /**
+     * hdfsOpenFileBuilderBuild - Builds the open file builder and returns a
+     * hdfsOpenFileFuture which tracks the asynchronous call to open the
+     * specified file.
+     *
+     * @param builder The open file builder to build.
+     * @return Returns the hdfsOpenFileFuture, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsOpenFileFuture *hdfsOpenFileBuilderBuild(hdfsOpenFileBuilder *builder);
+
+    /**
+     * hdfsOpenFileBuilderFree - Free a HDFS open file builder.
+     *
+     * It is normally not necessary to call this function since
+     * hdfsOpenFileBuilderBuild frees the builder.
+     *
+     * @param builder The hdfsOpenFileBuilder to free.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder);
+
+    /**
+     * hdfsOpenFileFutureGet - Call Future#get() on the underlying Java Future
+     * object. A call to #get() will block until the asynchronous operation has
+     * completed. In this case, until the open file call has completed. This
+     * method blocks indefinitely until blocking call completes.
+     *
+     * @param future The hdfsOpenFileFuture to call #get on
+     * @return Returns the opened hdfsFile, or NULL on error.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future);
+
+    /**
+     * hdfsOpenFileFutureGetWithTimeout - Call Future#get(long, TimeUnit) on
+     * the underlying Java Future object. A call to #get(long, TimeUnit) will
+     * block until the asynchronous operation has completed (in this case,
+     * until the open file call has completed) or the specified timeout has
+     * been reached.
+     *
+     * @param future The hdfsOpenFileFuture to call #get on
+     * @return Returns the opened hdfsFile, or NULL on error or if the timeout
+     *         has been reached.
+     */
+    LIBHDFS_EXTERNAL
+    hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future,
+            int64_t timeout, javaConcurrentTimeUnit timeUnit);
+
+    /**
+     * hdfsOpenFileFutureCancel - Call Future#cancel(boolean) on the
+     * underlying Java Future object. The value of mayInterruptedIfRunning
+     * controls whether the Java thread running the Future should be
+     * interrupted or not.
+     *
+     * @param future The hdfsOpenFileFuture to call #cancel on
+     * @param mayInterruptIfRunning if true, interrupts the running thread
+     * @return Returns 0 if the thread was successfully cancelled, else -1
+     */
+    LIBHDFS_EXTERNAL
+    int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future,
+            int mayInterruptIfRunning);
+
+    /**
+     * hdfsOpenFileFutureFree - Free a HDFS open file future.
+     *
+     * @param hdfsOpenFileFuture The hdfsOpenFileFuture to free.
+     */
+    LIBHDFS_EXTERNAL
+    void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future);
+
     /**
      * hdfsStreamBuilderAlloc - Allocate an HDFS stream builder.
      *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
index cf880e91b7596..9f589ac257aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
@@ -98,6 +98,8 @@ jthrowable initCachedClasses(JNIEnv* env) {
                 "org/apache/hadoop/hdfs/ReadStatistics";
         cachedJavaClasses[JC_HDFS_DATA_INPUT_STREAM].className =
                 "org/apache/hadoop/hdfs/client/HdfsDataInputStream";
+        cachedJavaClasses[JC_FUTURE_DATA_IS_BUILDER].className =
+                "org/apache/hadoop/fs/FutureDataInputStreamBuilder";
         cachedJavaClasses[JC_DOMAIN_SOCKET].className =
                 "org/apache/hadoop/net/unix/DomainSocket";
         cachedJavaClasses[JC_URI].className =
@@ -108,6 +110,8 @@ jthrowable initCachedClasses(JNIEnv* env) {
                 "java/util/EnumSet";
         cachedJavaClasses[JC_EXCEPTION_UTILS].className =
                 "org/apache/commons/lang3/exception/ExceptionUtils";
+        cachedJavaClasses[JC_CFUTURE].className =
+                "java/util/concurrent/CompletableFuture";
 
         // Create and set the jclass objects based on the class names set above
         jthrowable jthr;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
index 92cdd542e2371..0b174e1fecc56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
@@ -54,11 +54,13 @@ typedef enum {
     JC_FS_PERMISSION,
     JC_READ_STATISTICS,
     JC_HDFS_DATA_INPUT_STREAM,
+    JC_FUTURE_DATA_IS_BUILDER,
     JC_DOMAIN_SOCKET,
     JC_URI,
     JC_BYTE_BUFFER,
     JC_ENUM_SET,
     JC_EXCEPTION_UTILS,
+    JC_CFUTURE,
     // A special marker enum that counts the number of cached jclasses
     NUM_CACHED_CLASSES
 } CachedJavaClass;
@@ -95,6 +97,8 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
 #define HADOOP_FSPERM   "org/apache/hadoop/fs/permission/FsPermission"
 #define HADOOP_RSTAT    "org/apache/hadoop/hdfs/ReadStatistics"
 #define HADOOP_HDISTRM  "org/apache/hadoop/hdfs/client/HdfsDataInputStream"
+#define HADOOP_FDISB    "org/apache/hadoop/fs/FutureDataInputStreamBuilder"
+#define HADOOP_FS_BLDR  "org/apache/hadoop/fs/FSBuilder"
 #define HADOOP_RO       "org/apache/hadoop/fs/ReadOption"
 #define HADOOP_DS       "org/apache/hadoop/net/unix/DomainSocket"
 
@@ -104,6 +108,9 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
 #define JAVA_BYTEBUFFER "java/nio/ByteBuffer"
 #define JAVA_STRING     "java/lang/String"
 #define JAVA_ENUMSET    "java/util/EnumSet"
+#define JAVA_CFUTURE    "java/util/concurrent/CompletableFuture"
+#define JAVA_TIMEUNIT   "java/util/concurrent/TimeUnit"
+#define JAVA_OBJECT     "java/lang/Object"
 
 /* Some frequently used third-party class names */
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
index bbbc8b4602b54..8f00a08b0a98b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
@@ -23,9 +23,10 @@
 #include "platform.h"
 #include "os/mutexes.h"
 #include "os/thread_local_storage.h"
+#include "x-platform/c-api/dirent.h"
+#include "x-platform/types.h"
 
 #include <errno.h>
-#include <dirent.h>
 #include <stdio.h> 
 #include <string.h> 
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h
index b82ce48968dd5..900264240e8b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h
@@ -22,7 +22,7 @@
 /* On Windows, unistd.h does not exist, so manually define what we need. */
 
 #include <process.h> /* Declares getpid(). */
-#include <windows.h>
+#include <Windows.h>
 
 /* Re-route sleep to Sleep, converting units from seconds to milliseconds. */
 #define sleep(seconds) Sleep((seconds) * 1000)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index d23f84941db9b..a42ee2e211494 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -28,7 +28,7 @@ project (libhdfspp)
 
 cmake_minimum_required(VERSION 2.8)
 
-find_package (Boost 1.72.0 REQUIRED)
+find_package (Boost 1.72.0 REQUIRED COMPONENTS date_time)
 
 enable_testing()
 set(CMAKE_CXX_STANDARD 17)
@@ -52,31 +52,20 @@ find_package(Threads)
 
 include(CheckCXXSourceCompiles)
 include(CheckSymbolExists)
-
-# Download and build gtest
-configure_file(CMakeLists-gtest.txt.in googletest-download/CMakeLists.txt)
-execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
-  RESULT_VARIABLE result
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download)
-if(result)
-  message(FATAL_ERROR "CMake step for googletest failed: ${result}")
-endif()
-execute_process(COMMAND ${CMAKE_COMMAND} --build .
-  RESULT_VARIABLE result
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download)
-if(result)
-  message(FATAL_ERROR "Build step for googletest failed: ${result}")
-endif()
-
-# Prevent overriding the parent project's compiler/linker
-# settings on Windows
+include(FetchContent)
+
+# Install googletest
+# As per the approach documented in the GoogleTest repo -
+# https://github.com/google/googletest/blob/e649993a402d96afe25fbf3413749adf0f2947f6/googletest/README.md#incorporating-into-an-existing-cmake-project
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/703bd9caab50b139428cea1aaff9974ebee5742e.zip
+)
+# The commit SHA 703bd9caab50b139428cea1aaff9974ebee5742e in the URL above corresponds to the
+# tag release-1.10.0 in the googletest repository.
+# For Windows: Prevent overriding the parent project's compiler/linker settings
 set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
-
-# Add googletest directly to our build. This defines
-# the gtest and gtest_main targets.
-add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
-                 ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
-                 EXCLUDE_FROM_ALL)
+FetchContent_MakeAvailable(googletest)
 
 # Check if thread_local is supported
 unset (THREAD_LOCAL_SUPPORTED CACHE)
@@ -285,7 +274,8 @@ endif()
 
 set(LIBHDFSPP_VERSION "0.1.0")
 set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:x_platform_obj> $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
-if (HADOOP_BUILD)
+# HDFS-16464: We don't support building Hadoop DLL for Windows yet.
+if (HADOOP_BUILD AND NOT MSVC)
   hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
   hadoop_target_link_dual_libraries(hdfspp
     ${LIB_DL}
@@ -293,23 +283,23 @@ if (HADOOP_BUILD)
     ${OPENSSL_LIBRARIES}
     ${SASL_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
-  )
+    ${Boost_LIBRARIES})
   set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION})
   hadoop_dual_output_directory(hdfspp ${OUT_DIR})
-else (HADOOP_BUILD)
+else (HADOOP_BUILD AND NOT MSVC)
   add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
-  target_link_libraries(hdfspp_static
+  target_link_libraries(hdfspp_static PUBLIC
     ${LIB_DL}
     ${PROTOBUF_LIBRARY}
     ${OPENSSL_LIBRARIES}
     ${SASL_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
-    )
+    ${Boost_LIBRARIES})
   if(BUILD_SHARED_HDFSPP)
     add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
     set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION})
   endif(BUILD_SHARED_HDFSPP)
-endif (HADOOP_BUILD)
+endif (HADOOP_BUILD AND NOT MSVC)
 
 # Set up make install targets
 # Can be installed to a particular location via "make DESTDIR=... install"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
index 41a9ee87fde3f..d7dfb0a74438d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
@@ -20,7 +20,7 @@
 #    it by add -DLIBHDFSPP_DIR=... to your cmake invocation
 set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
 
-include_directories( ${LIBHDFSPP_DIR}/include )
+include_directories( ${LIBHDFSPP_DIR}/include ../../lib )
 link_directories( ${LIBHDFSPP_DIR}/lib )
 
 add_executable(cat_c cat.c)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
index bee5382c9ec0d..d0de0dc5fea77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
@@ -28,6 +28,7 @@
 #include "hdfspp/hdfs_ext.h"
 #include "uriparser2/uriparser2.h"
 #include "common/util_c.h"
+#include "x-platform/types.h"
 
 #define SCHEME "hdfs"
 #define BUF_SIZE 1048576 //1 MB
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
index 9d400e7b00cdc..584af44924e26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
@@ -62,7 +62,6 @@ int main(int argc, char *argv[]) {
   //wrapping file_raw into a unique pointer to guarantee deletion
   std::unique_ptr<hdfs::FileHandle> file(file_raw);
 
-  ssize_t total_bytes_read = 0;
   size_t last_bytes_read = 0;
 
   do{
@@ -71,7 +70,6 @@ int main(int argc, char *argv[]) {
     if(status.ok()) {
       //Writing file chunks to stdout
       fwrite(input_buffer, last_bytes_read, 1, stdout);
-      total_bytes_read += last_bytes_read;
     } else {
       if(status.is_invalid_offset()){
         //Reached the end of the file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
index 80f9316160216..9386453c929ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+#include "fs/filehandle.h"
+
 #include "hdfspp/hdfspp.h"
 #include "hdfspp/hdfs_ext.h"
 
@@ -23,7 +25,6 @@
 #include "common/configuration_loader.h"
 #include "common/logging.h"
 #include "fs/filesystem.h"
-#include "fs/filehandle.h"
 #include "x-platform/utils.h"
 #include "x-platform/syscall.h"
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
index 947214bdbd50a..8219740ef05d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.cc
@@ -35,7 +35,6 @@
 #include "hdfspp/uri.h"
 #include "x-platform/syscall.h"
 
-#include <strings.h>
 #include <sstream>
 #include <map>
 #include <rapidxml/rapidxml.hpp>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
index 5301137505af1..7d88550a81f71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration_loader.cc
@@ -21,7 +21,6 @@
 #include "x-platform/syscall.h"
 
 #include <fstream>
-#include <strings.h>
 #include <sstream>
 #include <map>
 #include <sys/stat.h>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
index f8f5923832711..ec7efc8f2708d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc
@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <sstream>
 #include <iomanip>
+#include <algorithm>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
index 8935287fe0108..9d75a48791da0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h
@@ -19,10 +19,10 @@
 #ifndef LIB_COMMON_LOGGING_H_
 #define LIB_COMMON_LOGGING_H_
 
-#include <boost/asio/ip/tcp.hpp>
-
 #include "hdfspp/log.h"
 
+#include <boost/asio/ip/tcp.hpp>
+
 #include <sstream>
 #include <mutex>
 #include <memory>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
index 92054fce07e31..4b375711f7419 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc
@@ -26,6 +26,7 @@
 #include <utility>
 #include <future>
 
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
index 2fb744fbddea8..072a3f36584f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc
@@ -17,10 +17,12 @@
  */
 
 #include <hdfspp/statinfo.h>
-#include <sys/stat.h>
+
 #include <sstream>
 #include <iomanip>
 
+#include "x-platform/stat.h"
+
 namespace hdfs {
 
 StatInfo::StatInfo()
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
index c0e10183297ac..45798bdd8e940 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
@@ -19,6 +19,7 @@
 #include "common/util.h"
 #include "common/util_c.h"
 
+#include <boost/system/error_code.hpp>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
index a0cb8375a8680..b5fe7b975c7fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h
@@ -18,9 +18,10 @@
 #ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
 #define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
 
+#include "ClientNamenodeProtocol.pb.h"
+
 #include "hdfspp/ioservice.h"
 #include "common/async_stream.h"
-#include "ClientNamenodeProtocol.pb.h"
 #include "common/libhdfs_events_impl.h"
 #include "common/logging.h"
 #include "common/util.h"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
index 7c9e24c0d883a..d5118089027b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
@@ -16,12 +16,14 @@
  * limitations under the License.
  */
 
+#include "reader/block_reader.h"
 #include "filehandle.h"
+
 #include "common/continuation/continuation.h"
 #include "common/logging.h"
 #include "connection/datanodeconnection.h"
-#include "reader/block_reader.h"
 #include "hdfspp/events.h"
+#include "x-platform/types.h"
 
 #include <future>
 #include <memory>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
index 724b1a14bc21d..cfff9b1b3f037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
@@ -18,16 +18,18 @@
 #ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_
 #define LIBHDFSPP_LIB_FS_FILEHANDLE_H_
 
+#include "reader/readergroup.h"
+
 #include "hdfspp/ioservice.h"
 #include "common/async_stream.h"
 #include "common/cancel_tracker.h"
 #include "common/libhdfs_events_impl.h"
 #include "common/new_delete.h"
 #include "reader/fileinfo.h"
-#include "reader/readergroup.h"
 
 #include "bad_datanode_tracker.h"
 #include "ClientNamenodeProtocol.pb.h"
+#include "x-platform/types.h"
 
 #include <memory>
 #include <mutex>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index e92a9ee48d6b9..e2b6cea30b2f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -16,21 +16,24 @@
  * limitations under the License.
  */
 
-#include "filesystem.h"
-
 #include "filehandle.h"
+#include "filesystem.h"
 #include "common/namenode_info.h"
 
 #include <functional>
 #include <limits>
 #include <future>
 #include <tuple>
-#include <pwd.h>
+#include <iostream>
 
 #include <boost/asio/ip/tcp.hpp>
 
 #include "x-platform/syscall.h"
 
+#ifndef WIN32
+#include <pwd.h>
+#endif
+
 #define FMT_THIS_ADDR "this=" << (void*)this
 
 namespace hdfs {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
index 96744e5d03d2f..4c9e398286c98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc
@@ -26,9 +26,12 @@
 #include <future>
 #include <tuple>
 #include <iostream>
-#include <pwd.h>
 #include <utility>
 
+#ifndef WIN32
+#include <pwd.h>
+#endif
+
 #define FMT_THIS_ADDR "this=" << (void*)this
 
 using boost::asio::ip::tcp;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
index 445aa08653dcf..bcccd52a56d3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h
@@ -18,17 +18,19 @@
 #ifndef LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
 #define LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
 
+#include "ClientNamenodeProtocol.pb.h"
+
 #include "rpc/rpc_engine.h"
 #include "hdfspp/statinfo.h"
 #include "hdfspp/fsinfo.h"
 #include "hdfspp/content_summary.h"
 #include "common/namenode_info.h"
-#include "ClientNamenodeProtocol.pb.h"
-#include "ClientNamenodeProtocol.hrpc.inl"
 
 #include <memory>
 #include <string>
 
+#include "ClientNamenodeProtocol.hrpc.inl"
+
 namespace hdfs {
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
index d47b0886c1ba9..35a38258f5f49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
@@ -67,10 +67,10 @@ function(GEN_HRPC SRCS)
     add_custom_command(
       OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl"
       COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
-      ARGS --plugin=protoc-gen-hrpc=${CMAKE_CURRENT_BINARY_DIR}/protoc-gen-hrpc --hrpc_out=${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      ARGS --plugin=protoc-gen-hrpc=$<TARGET_FILE:protoc-gen-hrpc> --hrpc_out=${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
       DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} protoc-gen-hrpc
       COMMENT "Running HRPC protocol buffer compiler on ${FIL}"
-      VERBATIM )
+      VERBATIM)
   endforeach()
 
   set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
index acecfce52374e..8db645e48d049 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc
@@ -15,8 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #include "reader/block_reader.h"
 #include "reader/datatransfer.h"
+
 #include "common/continuation/continuation.h"
 #include "common/continuation/asio.h"
 #include "common/logging.h"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
index 167c57d3a1704..605d641b3b317 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h
@@ -18,11 +18,12 @@
 #ifndef BLOCK_READER_H_
 #define BLOCK_READER_H_
 
+#include "datatransfer.pb.h"
+
 #include "hdfspp/status.h"
 #include "common/async_stream.h"
 #include "common/cancel_tracker.h"
 #include "common/new_delete.h"
-#include "datatransfer.pb.h"
 #include "connection/datanodeconnection.h"
 
 #include <memory>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
index cfa94bea2baf2..025cb8d37080d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h
@@ -18,6 +18,8 @@
 #ifndef LIB_READER_DATA_TRANSFER_H_
 #define LIB_READER_DATA_TRANSFER_H_
 
+#include "datatransfer.pb.h"
+
 #include "common/sasl_authenticator.h"
 #include "common/async_stream.h"
 #include "connection/datanodeconnection.h"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h
index d77685dd45a70..e3249ece51e13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h
@@ -18,10 +18,13 @@
 #ifndef LIB_READER_DATATRANFER_IMPL_H_
 #define LIB_READER_DATATRANFER_IMPL_H_
 
-#include "datatransfer.pb.h"
 #include "common/continuation/continuation.h"
 #include "common/continuation/asio.h"
 #include "common/continuation/protobuf.h"
+#include "common/sasl_authenticator.h"
+
+#include <boost/asio/read.hpp>
+#include <boost/asio/buffer.hpp>
 
 #include <boost/asio/read.hpp>
 #include <boost/asio/buffer.hpp>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h
index e6173f7fa70e2..47b6046d80fae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h
@@ -15,14 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #ifndef READER_READER_GROUP_H_
 #define READER_READER_GROUP_H_
 
 #include "block_reader.h"
 
 #include <memory>
-#include <vector>
 #include <mutex>
+#include <vector>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
index b50134eda9536..5503ef59fb1cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt
@@ -30,3 +30,4 @@ target_include_directories(rpc_obj PRIVATE ../../lib)
 add_dependencies(rpc_obj proto)
 add_library(rpc $<TARGET_OBJECTS:rpc_obj>)
 target_include_directories(rpc PRIVATE ../../lib)
+target_link_libraries(rpc PRIVATE ${Boost_LIBRARIES})
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
index 99762c89ee9e4..bf25e60c68cab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
@@ -16,16 +16,17 @@
  * limitations under the License.
  */
 
+#include "RpcHeader.pb.h"
+#include "ProtobufRpcEngine.pb.h"
+#include "IpcConnectionContext.pb.h"
+
 #include <functional>
+
 #include "request.h"
 #include "rpc_engine.h"
 #include "sasl_protocol.h"
 #include "hdfspp/ioservice.h"
 
-#include "RpcHeader.pb.h"
-#include "ProtobufRpcEngine.pb.h"
-#include "IpcConnectionContext.pb.h"
-
 #include <sstream>
 
 namespace hdfs {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
index f599d36ee5c4d..21c475c1e8bbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h
@@ -43,6 +43,7 @@
 #include <unordered_map>
 
 #include <boost/asio/ip/tcp.hpp>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
index 5d434ef370a9d..2a4e358e9bbf4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
@@ -15,15 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "rpc_engine.h"
-#include "rpc_connection_impl.h"
-#include "sasl_protocol.h"
 
 #include "RpcHeader.pb.h"
 #include "ProtobufRpcEngine.pb.h"
 #include "IpcConnectionContext.pb.h"
 
+#include "rpc_engine.h"
+#include "rpc_connection_impl.h"
+#include "sasl_protocol.h"
+
 #include <boost/asio/error.hpp>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
index 884bd64ac642c..4f90ab72f991e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h
@@ -35,6 +35,7 @@
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
 
 #include <system_error>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
index e3274cb88aacf..0bea5bf192cfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
@@ -27,6 +27,7 @@
 #include <string>
 
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/system/error_code.hpp>
 #include <openssl/rand.h>
 #include <openssl/err.h>
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
index bc9adbff313d7..c45ab84d2137b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+#include "sasl_protocol.h"
 #include "rpc_engine.h"
 #include "rpc_connection.h"
 #include "common/logging.h"
@@ -23,7 +24,6 @@
 #include "x-platform/syscall.h"
 
 #include "sasl_engine.h"
-#include "sasl_protocol.h"
 
 #if defined USE_SASL
   #if defined USE_CYRUS_SASL
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
index a46ae08074d24..2c7fd09b96b5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h
@@ -19,12 +19,12 @@
 #ifndef LIB_RPC_SASLPROTOCOL_H
 #define LIB_RPC_SASLPROTOCOL_H
 
+#include "RpcHeader.pb.h"
+
 #include <memory>
 #include <mutex>
 #include <functional>
 
-#include <RpcHeader.pb.h>
-
 #include "hdfspp/status.h"
 #include "common/auth_info.h"
 #include "common/libhdfs_events_impl.h"
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt
index 77d0216d4877a..44e2943aef3ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt
@@ -22,5 +22,6 @@ else()
   set(SYSCALL_SRC syscall_linux.cc)
 endif()
 
-add_library(x_platform_obj OBJECT ${SYSCALL_SRC} utils.cc)
-add_library(x_platform_obj_c_api OBJECT $<TARGET_OBJECTS:x_platform_obj> c-api/syscall.cc)
+add_library(x_platform_obj OBJECT ${SYSCALL_SRC} utils.cc dirent.cc)
+add_library(x_platform_obj_c_api OBJECT $<TARGET_OBJECTS:x_platform_obj> c-api/syscall.cc c-api/dirent.cc)
+target_compile_definitions(x_platform_obj_c_api PRIVATE USE_X_PLATFORM_DIRENT)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/core/dirent.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/core/dirent.h
new file mode 100644
index 0000000000000..1a592dbeeb58f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/core/dirent.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_CORE_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_CORE_DIRENT_H
+
+/**
+ * DIR struct holds the pointer to XPlatform::Dirent instance. Since this will
+ * be used in C, we can't hold the pointer to XPlatform::Dirent. We're working
+ * around this by using a void pointer and casting it to XPlatform::Dirent when
+ * needed in C++.
+ */
+typedef struct DIR {
+  void *x_platform_dirent_ptr;
+} DIR;
+
+/**
+ * dirent struct contains the name of the file/folder while iterating through
+ * the directory's children.
+ */
+struct dirent {
+  char d_name[256];
+};
+
+/**
+ * Opens a directory for iteration. Internally, it instantiates DIR struct for
+ * the given path. closedir must be called on the returned pointer to DIR struct
+ * when done.
+ *
+ * @param dir_path The path to the directory to iterate through.
+ * @return A pointer to the DIR struct.
+ */
+DIR *opendir(const char *dir_path);
+
+/**
+ * For iterating through the children of the directory pointed to by the DIR
+ * struct pointer.
+ *
+ * @param dir The pointer to the DIR struct.
+ * @return A pointer to dirent struct containing the name of the current child
+ * file/folder.
+ */
+struct dirent *readdir(DIR *dir);
+
+/**
+ * De-allocates the XPlatform::Dirent instance pointed to by the DIR pointer.
+ *
+ * @param dir The pointer to DIR struct to close.
+ * @return 0 if successful.
+ */
+int closedir(DIR *dir);
+
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.cc
new file mode 100644
index 0000000000000..24cebc1192d60
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.cc
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cerrno>
+#include <iostream>
+#include <iterator>
+#include <system_error>
+#include <variant>
+
+#include "x-platform/c-api/dirent.h"
+#include "x-platform/dirent.h"
+
+DIR *opendir(const char *dir_path) {
+  const auto dir = new DIR;
+  dir->x_platform_dirent_ptr = new XPlatform::Dirent(dir_path);
+  return dir;
+}
+
+struct dirent *readdir(DIR *dir) {
+  /*
+   * We will use a static variable to hold the dirent, so that we align with the
+   * readdir's implementation in dirent.h header file in Linux.
+   */
+  static struct dirent static_dir_entry;
+
+  // Get the XPlatform::Dirent instance and move the iterator.
+  const auto x_platform_dirent =
+      static_cast<XPlatform::Dirent *>(dir->x_platform_dirent_ptr);
+  const auto dir_entry = x_platform_dirent->NextFile();
+
+  // End of iteration.
+  if (std::holds_alternative<std::monostate>(dir_entry)) {
+    return nullptr;
+  }
+
+  // Error in iteration.
+  if (std::holds_alternative<std::error_code>(dir_entry)) {
+    const auto err = std::get<std::error_code>(dir_entry);
+    errno = err.value();
+
+#ifdef X_PLATFORM_C_API_DIRENT_DEBUG
+    std::cerr << "Error in listing directory: " << err.message() << std::endl;
+#endif
+
+    return nullptr;
+  }
+
+  // Return the current child file/folder's name.
+  if (std::holds_alternative<std::filesystem::directory_entry>(dir_entry)) {
+    const auto entry = std::get<std::filesystem::directory_entry>(dir_entry);
+    const auto filename = entry.path().filename().string();
+
+    // The file name's length shouldn't exceed 256.
+    if (filename.length() >= 256) {
+      errno = 1;
+      return nullptr;
+    }
+
+    std::fill(std::begin(static_dir_entry.d_name),
+              std::end(static_dir_entry.d_name), '\0');
+    std::copy(filename.begin(), filename.end(),
+              std::begin(static_dir_entry.d_name));
+  }
+  return &static_dir_entry;
+}
+
+int closedir(DIR *dir) {
+  const auto x_platform_dirent =
+      static_cast<XPlatform::Dirent *>(dir->x_platform_dirent_ptr);
+  delete x_platform_dirent;
+  delete dir;
+
+  // We can't use the void return type for closedir since we want to align the
+  // closedir method's signature in dirent.h header file in Linux.
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.h
new file mode 100644
index 0000000000000..23fa9d997ae11
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.h
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_DIRENT_H
+
+#if !(defined(WIN32) || defined(USE_X_PLATFORM_DIRENT))
+
+/*
+ * For non-Windows environments, we use the dirent.h header itself.
+ */
+#include <dirent.h>
+
+#else
+
+/*
+ * If it's a Windows environment or if the macro USE_X_PLATFORM_DIRENT is
+ * defined, we switch to using dirent from the XPlatform library.
+ */
+#include "x-platform/c-api/extern/dirent.h"
+
+#endif
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/extern/dirent.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/extern/dirent.h
new file mode 100644
index 0000000000000..1845a831deb92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/extern/dirent.h
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_EXTERN_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_EXTERN_DIRENT_H
+
+/*
+ * We will use extern "C" only on Windows.
+ */
+#if defined(WIN32) && defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "x-platform/c-api/core/dirent.h"
+
+#if defined(WIN32) && defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionClosedException.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.cc
similarity index 64%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionClosedException.java
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.cc
index eeaf8a5606fb3..c53e7986979f1 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionClosedException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.cc
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,22 +15,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.fs.swift.exceptions;
 
-/**
- * Exception raised when an attempt is made to use a closed stream
- */
-public class SwiftConnectionClosedException extends SwiftException {
+#include <filesystem>
+#include <system_error>
+#include <variant>
 
-  public static final String MESSAGE =
-    "Connection to Swift service has been closed";
+#include "dirent.h"
 
-  public SwiftConnectionClosedException() {
-    super(MESSAGE);
+std::variant<std::monostate, std::filesystem::directory_entry, std::error_code>
+XPlatform::Dirent::NextFile() {
+  if (dir_it_err_) {
+    return dir_it_err_;
   }
 
-  public SwiftConnectionClosedException(String reason) {
-    super(MESSAGE + ": " + reason);
+  if (dir_it_ == std::filesystem::end(dir_it_)) {
+    return std::monostate();
   }
 
+  const std::filesystem::directory_entry dir_entry = *dir_it_;
+  dir_it_ = dir_it_.increment(dir_it_err_);
+  return dir_entry;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.h
new file mode 100644
index 0000000000000..91b586d921baa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.h
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_DIRENT
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_DIRENT
+
+#include <filesystem>
+#include <string>
+#include <system_error>
+#include <variant>
+
+namespace XPlatform {
+/**
+ * {@class XPlatform::Dirent} provides the functionality to perform a one-time
+ * iteration per {@link XPlatform::Dirent} through the child files or folders
+ * under a given path.
+ */
+class Dirent {
+public:
+  Dirent(const std::string &path)
+      : dir_it_{std::filesystem::path{path}, dir_it_err_} {}
+
+  // Abiding to the Rule of 5
+  Dirent(const Dirent &) = default;
+  Dirent(Dirent &&) = default;
+  Dirent &operator=(const Dirent &) = default;
+  Dirent &operator=(Dirent &&) = default;
+  ~Dirent() = default;
+
+  /**
+   * Advances the iterator {@link XPlatform::Dirent#dir_it_} to the next file in
+   * the given path.
+   *
+   * @return An {@link std::variant} comprising of any one of the following
+   * types:
+   * 1. {@link std::monostate} which indicates the end of iteration of all the
+   * files in the given path.
+   * 2. {@link std::filesystem::directory_entry} which is the directory entry of
+   * the current file.
+   * 3. {@link std::error_code} which corresponds to the error in retrieving the
+   * file.
+   */
+  std::variant<std::monostate, std::filesystem::directory_entry,
+               std::error_code>
+  NextFile();
+
+private:
+  /**
+   * Indicates the error corresponding to the most recent invocation of
+   * directory iteration by {@link XPlatform::Dirent#dir_it_}.
+   */
+  std::error_code dir_it_err_{};
+
+  /**
+   * The iterator used for iterating through the files or folders under the
+   * given path.
+   */
+  std::filesystem::directory_iterator dir_it_;
+};
+} // namespace XPlatform
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h
new file mode 100644
index 0000000000000..c078d44a51831
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_STAT
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_STAT
+
+#include <sys/stat.h>
+
+#if defined(_WIN32)
+
+// Windows defines the macros for user RWX (_S_IREAD, _S_IWRITE and _S_IEXEC),
+// but not for group and others. We implement the permissions for group and
+// others through appropriate bit shifting.
+
+#define S_IRUSR _S_IREAD
+#define S_IWUSR _S_IWRITE
+#define S_IXUSR _S_IEXEC
+#define S_IRGRP (S_IRUSR >> 3)
+#define S_IWGRP (S_IWUSR >> 3)
+#define S_IXGRP (S_IXUSR >> 3)
+#define S_IROTH (S_IRGRP >> 3)
+#define S_IWOTH (S_IWGRP >> 3)
+#define S_IXOTH (S_IXGRP >> 3)
+
+#endif
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index 15ec620c9abd7..51907634bbac6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "syscall.h"
+#include "types.h"
 
 bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
   return WriteToStdoutImpl(message.c_str());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersion.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h
similarity index 65%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersion.java
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h
index 57439de078fbf..0719fe90dae54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h
@@ -16,16 +16,19 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.yarn.server.timeline;
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES
 
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Retention(value = RetentionPolicy.RUNTIME)
-@Target(value = {ElementType.METHOD})
-public @interface TimelineVersion {
-  float value() default TimelineVersionWatcher.DEFAULT_TIMELINE_VERSION;
-}
+#if defined(_WIN64)
+// Windows 64-bit.
+typedef long int ssize_t;
+#elif defined(_WIN32)
+// Windows 32-bit.
+typedef int ssize_t;
+#else
+// ssize_t is correctly defined by taking bit-ness into account on non-Windows
+// systems. So we just include the header file where ssize_t is defined.
+#include <sys/types.h>
+#endif
 
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index d7d20ec36007a..7eb432f31ac0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -16,16 +16,17 @@
 # limitations under the License.
 #
 
-find_package(Boost REQUIRED COMPONENTS date_time)
-
 # Delegate some functionality to libhdfs, until libhdfspp is complete.
 set (LIBHDFS_SRC_DIR ../../libhdfs)
 set (LIBHDFS_TESTS_DIR ../../libhdfs-tests)
 set (LIBHDFSPP_SRC_DIR ..)
 set (LIBHDFSPP_LIB_DIR ${LIBHDFSPP_SRC_DIR}/lib)
 set (LIBHDFSPP_BINDING_C ${LIBHDFSPP_LIB_DIR}/bindings/c)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
+
+if (NOT MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
+endif (NOT MSVC)
 
 include_directories(
     ${GENERATED_JAVAH}
@@ -136,7 +137,6 @@ target_link_libraries(hdfs_config_connect_bugs_test common gmock_main bindings_c
 add_memcheck_test(hdfs_config_connect_bugs hdfs_config_connect_bugs_test)
 
 
-
 #
 #
 #   INTEGRATION TESTS - TESTS THE FULL LIBRARY AGAINST ACTUAL SERVERS
@@ -151,7 +151,7 @@ include_directories (
     ${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/
 )
 
-add_library(hdfspp_test_shim_static STATIC $<TARGET_OBJECTS:x_platform_obj> hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
+add_library(hdfspp_test_shim_static STATIC $<TARGET_OBJECTS:x_platform_obj> hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc)
 add_dependencies(hdfspp_test_shim_static proto)
 
 add_library(hdfspp_test_static STATIC $<TARGET_OBJECTS:x_platform_obj> ${LIBHDFSPP_BINDING_C}/hdfs.cc)
@@ -170,10 +170,12 @@ build_libhdfs_test(hdfspp_mini_dfs_smoke hdfspp_test_shim_static ${CMAKE_CURRENT
 link_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static fs reader rpc proto common connection gmock_main ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
 add_libhdfs_test  (hdfspp_mini_dfs_smoke hdfspp_test_shim_static)
 
-build_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c)
-link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
-add_memcheck_test(libhdfs_mini_stress_valgrind_hdfspp_test_static libhdfs_mini_stress_valgrind_hdfspp_test_static)
-set_target_properties(libhdfs_mini_stress_valgrind_hdfspp_test_static PROPERTIES COMPILE_DEFINITIONS "VALGRIND")
+if (NOT MSVC)
+  build_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c)
+  link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
+  add_memcheck_test(libhdfs_mini_stress_valgrind_hdfspp_test_static libhdfs_mini_stress_valgrind_hdfspp_test_static)
+  set_target_properties(libhdfs_mini_stress_valgrind_hdfspp_test_static PROPERTIES COMPILE_DEFINITIONS "VALGRIND")
+endif (NOT MSVC)
 
 build_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c)
 link_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES})
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 973212647e764..225dd5e20ef4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -16,12 +16,13 @@
  * limitations under the License.
  */
 
+#include "reader/block_reader.h"
+#include "fs/filehandle.h"
+
 #include "common/libhdfs_events_impl.h"
 #include "common/util.h"
 #include "fs/filesystem.h"
-#include "fs/filehandle.h"
 #include "fs/bad_datanode_tracker.h"
-#include "reader/block_reader.h"
 
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
index bda27b9a43202..ad8ad712c9d4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
@@ -24,6 +24,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef WIN32
+#define __PRETTY_FUNCTION__ __FUNCSIG__
+#endif
+
 /* Shim structs and functions that delegate to libhdfspp and libhdfs. */
 struct hdfs_internal {
   libhdfs_hdfsFS libhdfsRep;
@@ -250,6 +254,65 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags,
   return ret;
 }
 
+hdfsOpenFileBuilder *hdfsOpenFileBuilderAlloc(hdfsFS fs,
+        const char *path) {
+  return libhdfs_hdfsOpenFileBuilderAlloc(fs->libhdfsRep, path);
+}
+
+hdfsOpenFileBuilder *hdfsOpenFileBuilderMust(
+        hdfsOpenFileBuilder *builder, const char *key,
+        const char *value) {
+  return libhdfs_hdfsOpenFileBuilderMust(builder, key, value);
+}
+
+hdfsOpenFileBuilder *hdfsOpenFileBuilderOpt(
+        hdfsOpenFileBuilder *builder, const char *key,
+        const char *value) {
+  return libhdfs_hdfsOpenFileBuilderOpt(builder, key, value);
+}
+
+hdfsOpenFileFuture *hdfsOpenFileBuilderBuild(
+        hdfsOpenFileBuilder *builder) {
+  return libhdfs_hdfsOpenFileBuilderBuild(builder);
+}
+
+void hdfsOpenFileBuilderFree(hdfsOpenFileBuilder *builder) {
+  libhdfs_hdfsOpenFileBuilderFree(builder);
+}
+
+hdfsFile hdfsOpenFileFutureGet(hdfsOpenFileFuture *future) {
+  hdfsFile ret = calloc(1, sizeof(struct hdfsFile_internal));
+  ret->libhdfsppRep = 0;
+  ret->libhdfsRep = libhdfs_hdfsOpenFileFutureGet(future);
+  if (!ret->libhdfsRep) {
+    free(ret);
+    ret = NULL;
+  }
+  return ret;
+}
+
+hdfsFile hdfsOpenFileFutureGetWithTimeout(hdfsOpenFileFuture *future,
+        int64_t timeout, javaConcurrentTimeUnit timeUnit) {
+  hdfsFile ret = calloc(1, sizeof(struct hdfsFile_internal));
+  ret->libhdfsppRep = 0;
+  ret->libhdfsRep = libhdfs_hdfsOpenFileFutureGetWithTimeout(future, timeout,
+                                                             timeUnit);
+  if (!ret->libhdfsRep) {
+    free(ret);
+    ret = NULL;
+  }
+  return ret;
+}
+
+int hdfsOpenFileFutureCancel(hdfsOpenFileFuture *future,
+        int mayInterruptIfRunning) {
+  return libhdfs_hdfsOpenFileFutureCancel(future, mayInterruptIfRunning);
+}
+
+void hdfsOpenFileFutureFree(hdfsOpenFileFuture *future) {
+  libhdfs_hdfsOpenFileFutureFree(future);
+}
+
 int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) {
   return libhdfs_hdfsTruncateFile(fs->libhdfsRep, path, newlength);
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
index 0d014341b4c57..165744142558a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
@@ -39,6 +39,23 @@
 #define hdfsConfStrFree libhdfs_hdfsConfStrFree
 #define hdfsDisconnect libhdfs_hdfsDisconnect
 #define hdfsOpenFile libhdfs_hdfsOpenFile
+#define hdfsOpenFileBuilderAlloc libhdfs_hdfsOpenFileBuilderAlloc
+#define hdfsOpenFileBuilderMust libhdfs_hdfsOpenFileBuilderMust
+#define hdfsOpenFileBuilderOpt libhdfs_hdfsOpenFileBuilderOpt
+#define hdfsOpenFileBuilderBuild libhdfs_hdfsOpenFileBuilderBuild
+#define hdfsOpenFileBuilderFree libhdfs_hdfsOpenFileBuilderFree
+#define hdfsOpenFileFutureGet libhdfs_hdfsOpenFileFutureGet
+#define javaConcurrentTimeUnit libhdfs_javaConcurrentTimeUnit
+#define jNanoseconds libhdfs_jNanoseconds
+#define jMicroseconds libhdfs_jMicroseconds
+#define jMilliseconds libhdfs_jMilliseconds
+#define jSeconds libhdfsj_jSeconds
+#define jMinutes libhdfs_jMinutes
+#define jHours libhdfs_jHours
+#define jDays libhdfs_jDays
+#define hdfsOpenFileFutureGetWithTimeout libhdfs_hdfsOpenFileFutureGetWithTimeout
+#define hdfsOpenFileFutureCancel libhdfs_hdfsOpenFileFutureCancel
+#define hdfsOpenFileFutureFree libhdfs_hdfsOpenFileFutureFree
 #define hdfsTruncateFile libhdfs_hdfsTruncateFile
 #define hdfsUnbufferFile libhdfs_hdfsUnbufferFile
 #define hdfsCloseFile libhdfs_hdfsCloseFile
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
index d46768c02ad39..d84b8ba287525 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
@@ -39,6 +39,23 @@
 #undef hdfsConfStrFree
 #undef hdfsDisconnect
 #undef hdfsOpenFile
+#undef hdfsOpenFileBuilderAlloc
+#undef hdfsOpenFileBuilderMust
+#undef hdfsOpenFileBuilderOpt
+#undef hdfsOpenFileBuilderBuild
+#undef hdfsOpenFileBuilderFree
+#undef hdfsOpenFileFutureGet
+#undef javaConcurrentTimeUnit
+#undef jNanoseconds
+#undef jMicroseconds
+#undef jMilliseconds
+#undef jSeconds
+#undef jMinutes
+#undef jHours
+#undef jDays
+#undef hdfsOpenFileFutureGetWithTimeout
+#undef hdfsOpenFileFutureCancel
+#undef hdfsOpenFileFutureFree
 #undef hdfsTruncateFile
 #undef hdfsUnbufferFile
 #undef hdfsCloseFile
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
index 4b08d0556c3aa..0a6d987409fec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
@@ -39,6 +39,23 @@
 #define hdfsConfStrFree libhdfspp_hdfsConfStrFree
 #define hdfsDisconnect libhdfspp_hdfsDisconnect
 #define hdfsOpenFile libhdfspp_hdfsOpenFile
+#define hdfsOpenFileBuilderAlloc libhdfspp_hdfsOpenFileBuilderAlloc
+#define hdfsOpenFileBuilderMust libhdfspp_hdfsOpenFileBuilderMust
+#define hdfsOpenFileBuilderOpt libhdfspp_hdfsOpenFileBuilderOpt
+#define hdfsOpenFileBuilderBuild libhdfspp_hdfsOpenFileBuilderBuild
+#define hdfsOpenFileBuilderFree libhdfspp_hdfsOpenFileBuilderFree
+#define hdfsOpenFileFutureGet libhdfspp_hdfsOpenFileFutureGet
+#define javaConcurrentTimeUnit libhdfspp_javaConcurrentTimeUnit
+#define jNanoseconds libhdfspp_jNanoseconds
+#define jMicroseconds libhdfspp_jMicroseconds
+#define jMilliseconds libhdfspp_jMilliseconds
+#define jSeconds libhdfspp_jSeconds
+#define jMinutes libhdfspp_jMinutes
+#define jHours libhdfspp_jHours
+#define jDays libhdfspp_jDays
+#define hdfsOpenFileFutureGetWithTimeout libhdfspp_hdfsOpenFileFutureGetWithTimeout
+#define hdfsOpenFileFutureCancel libhdfspp_hdfsOpenFileFutureCancel
+#define hdfsOpenFileFutureFree libhdfspp_hdfsOpenFileFutureFree
 #define hdfsTruncateFile libhdfspp_hdfsTruncateFile
 #define hdfsUnbufferFile libhdfspp_hdfsUnbufferFile
 #define hdfsCloseFile libhdfspp_hdfsCloseFile
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc
index d487bf5f0f006..b14c22b02b2e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc
@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 
-#include <common/logging.h>
-#include <bindings/c/hdfs.cc>
+#include "bindings/c/hdfs.cc"
+#include "common/logging.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc
index 37fabf568d275..c99087011bf33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc
@@ -18,6 +18,8 @@
 
 #include "mock_connection.h"
 
+#include <boost/asio/io_service.hpp>
+
 namespace hdfs {
 
 MockConnectionBase::MockConnectionBase(boost::asio::io_service *io_service)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
index 7a7b5f076ed39..eb62a2b35c7fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h
@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #ifndef LIBHDFSPP_TEST_MOCK_CONNECTION_H_
 #define LIBHDFSPP_TEST_MOCK_CONNECTION_H_
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
index ccec5812f61d1..0c22e89f4c388 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
@@ -16,13 +16,13 @@
  * limitations under the License.
  */
 
+#include "reader/block_reader.h"
+#include "reader/datatransfer.h"
+
 #include "mock_connection.h"
 
-#include "datatransfer.pb.h"
 #include "common/util.h"
 #include "common/cancel_tracker.h"
-#include "reader/block_reader.h"
-#include "reader/datatransfer.h"
 #include "reader/fileinfo.h"
 
 #include <google/protobuf/io/coded_stream.h>
@@ -124,11 +124,11 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) {
   return res;
 }
 
-static inline std::pair<error_code, string> Produce(const std::string &s) {
-  return make_pair(error_code(), s);
+static inline std::pair<boost::system::error_code, string> Produce(const std::string &s) {
+  return make_pair(boost::system::error_code(), s);
 }
 
-static inline std::pair<error_code, string> ProducePacket(
+static inline std::pair<boost::system::error_code, string> ProducePacket(
     const std::string &data, const std::string &checksum, int offset_in_block,
     int seqno, bool last_packet) {
   PacketHeaderProto proto;
@@ -148,7 +148,7 @@ static inline std::pair<error_code, string> ProducePacket(
   proto.AppendToString(&payload);
   payload += checksum;
   payload += data;
-  return std::make_pair(error_code(), std::move(payload));
+  return std::make_pair(boost::system::error_code(), std::move(payload));
 }
 
 TEST(RemoteBlockReaderTest, TestReadSingleTrunk) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index caf4842b29899..ba09e8fc90781 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -16,11 +16,11 @@
  * limitations under the License.
  */
 
-#include "hdfspp/ioservice.h"
+#include "RpcHeader.pb.h"
 
+#include "hdfspp/ioservice.h"
 #include "mock_connection.h"
 #include "test.pb.h"
-#include "RpcHeader.pb.h"
 #include "rpc/rpc_connection_impl.h"
 #include "common/namenode_info.h"
 
@@ -41,6 +41,8 @@ using ::hadoop::common::EmptyResponseProto;
 using ::hadoop::common::EchoRequestProto;
 using ::hadoop::common::EchoResponseProto;
 
+using boost::system::error_code;
+
 using ::testing::Return;
 
 using ::std::make_pair;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
index 48580a4ccab19..7bbe63153b77c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
@@ -18,23 +18,77 @@
 
 add_executable(hdfs_tool_tests
                 hdfs-allow-snapshot-mock.cc
+                hdfs-disallow-snapshot-mock.cc
+                hdfs-rename-snapshot-mock.cc
                 hdfs-delete-snapshot-mock.cc
+                hdfs-create-snapshot-mock.cc
                 hdfs-cat-mock.cc
+                hdfs-chown-mock.cc
+                hdfs-chmod-mock.cc
+                hdfs-chgrp-mock.cc
                 hdfs-tool-test-fixtures.cc
                 hdfs-tool-tests.cc
                 hdfs-df-mock.cc
+                hdfs-du-mock.cc
+                hdfs-copy-to-local-mock.cc
+                hdfs-move-to-local-mock.cc
+                hdfs-count-mock.cc
+                hdfs-mkdir-mock.cc
+                hdfs-rm-mock.cc
+                hdfs-get-mock.cc
+                hdfs-find-mock.cc
+                hdfs-ls-mock.cc
+                hdfs-setrep-mock.cc
+                hdfs-stat-mock.cc
+                hdfs-tail-mock.cc
                 main.cc)
 target_include_directories(hdfs_tool_tests PRIVATE
                             ../tools
                             ../../tools
                             ../../tools/hdfs-df
+                            ../../tools/hdfs-du
                             ../../tools/hdfs-allow-snapshot
+                            ../../tools/hdfs-disallow-snapshot
                             ../../tools/hdfs-delete-snapshot
+                            ../../tools/hdfs-create-snapshot
+                            ../../tools/hdfs-rename-snapshot
+                            ../../tools/hdfs-chown
+                            ../../tools/hdfs-chgrp
+                            ../../tools/hdfs-chmod
+                            ../../tools/hdfs-copy-to-local
+                            ../../tools/hdfs-move-to-local
+                            ../../tools/hdfs-count
+                            ../../tools/hdfs-mkdir
+                            ../../tools/hdfs-rm
+                            ../../tools/hdfs-get
+                            ../../tools/hdfs-find
+                            ../../tools/hdfs-ls
+                            ../../tools/hdfs-setrep
+                            ../../tools/hdfs-stat
+                            ../../tools/hdfs-tail
                             ../../tools/hdfs-cat)
 target_link_libraries(hdfs_tool_tests PRIVATE
                        gmock_main
                        hdfs_df_lib
+                       hdfs_du_lib
                        hdfs_allowSnapshot_lib
+                       hdfs_disallowSnapshot_lib
                        hdfs_deleteSnapshot_lib
+                       hdfs_createSnapshot_lib
+                       hdfs_renameSnapshot_lib
+                       hdfs_chown_lib
+                       hdfs_chgrp_lib
+                       hdfs_chmod_lib
+                       hdfs_copyToLocal_lib
+                       hdfs_moveToLocal_lib
+                       hdfs_count_lib
+                       hdfs_mkdir_lib
+                       hdfs_rm_lib
+                       hdfs_get_lib
+                       hdfs_find_lib
+                       hdfs_ls_lib
+                       hdfs_setrep_lib
+                       hdfs_stat_lib
+                       hdfs_tail_lib
                        hdfs_cat_lib)
 add_test(hdfs_tool_tests hdfs_tool_tests)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc
new file mode 100644
index 0000000000000..5948a7cc942bd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-chgrp-mock.h"
+#include "hdfs-chgrp.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+ChgrpMock::~ChgrpMock() = default;
+
+void ChgrpMock::SetExpectations(
+    std::function<std::unique_ptr<ChgrpMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<ChgrpMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<ChgrpMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassOwnerAndAPath<ChgrpMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+
+    EXPECT_CALL(*this, HandlePath(arg1, false, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursiveOwnerAndAPath<ChgrpMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[2];
+
+    EXPECT_CALL(*this, HandlePath(arg1, true, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h
new file mode 100644
index 0000000000000..a7fb95cc81723
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-chgrp.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class ChgrpMock} is an {@class Chgrp} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class ChgrpMock : public hdfs::tools::Chgrp {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  ChgrpMock(const int argc, char **argv) : Chgrp(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  ChgrpMock(const ChgrpMock &) = delete;
+  ChgrpMock(ChgrpMock &&) = delete;
+  ChgrpMock &operator=(const ChgrpMock &) = delete;
+  ChgrpMock &operator=(ChgrpMock &&) = delete;
+  ~ChgrpMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<ChgrpMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath,
+              (const std::string &, bool, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc
new file mode 100644
index 0000000000000..3c4ea24e0db79
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-chmod-mock.h"
+#include "hdfs-chmod.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+ChmodMock::~ChmodMock() = default;
+
+void ChmodMock::SetExpectations(
+    std::function<std::unique_ptr<ChmodMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<ChmodMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<ChmodMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassPermissionsAndAPath<ChmodMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+
+    EXPECT_CALL(*this, HandlePath(arg1, false, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassInvalidPermissionsAndAPath<ChmodMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+
+    EXPECT_CALL(*this, HandlePath(arg1, false, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(false));
+  }
+
+  if (*test_case_func == &PassRecursivePermissionsAndAPath<ChmodMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[2];
+
+    EXPECT_CALL(*this, HandlePath(arg1, true, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h
new file mode 100644
index 0000000000000..cd193b9048914
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-chmod.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class ChmodMock} is an {@class Chmod} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class ChmodMock : public hdfs::tools::Chmod {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  ChmodMock(const int argc, char **argv) : Chmod(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  ChmodMock(const ChmodMock &) = delete;
+  ChmodMock(ChmodMock &&) = delete;
+  ChmodMock &operator=(const ChmodMock &) = delete;
+  ChmodMock &operator=(ChmodMock &&) = delete;
+  ~ChmodMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<ChmodMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath,
+              (const std::string &, bool, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc
new file mode 100644
index 0000000000000..a126e9c6fe29a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-chown-mock.h"
+#include "hdfs-chown.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+ChownMock::~ChownMock() = default;
+
+void ChownMock::SetExpectations(
+    std::function<std::unique_ptr<ChownMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<ChownMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<ChownMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassOwnerAndAPath<ChownMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    const Ownership ownership(arg1);
+
+    EXPECT_CALL(*this, HandlePath(ownership, false, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursiveOwnerAndAPath<ChownMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[2];
+    const Ownership ownership(arg1);
+
+    EXPECT_CALL(*this, HandlePath(ownership, true, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h
new file mode 100644
index 0000000000000..5cde9416bb6d4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-chown.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class ChownMock} is an {@class Chown} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class ChownMock : public hdfs::tools::Chown {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  ChownMock(const int argc, char **argv) : Chown(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  ChownMock(const ChownMock &) = delete;
+  ChownMock(ChownMock &&) = delete;
+  ChownMock &operator=(const ChownMock &) = delete;
+  ChownMock &operator=(ChownMock &&) = delete;
+  ~ChownMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<ChownMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const Ownership &, bool, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc
new file mode 100644
index 0000000000000..f76a2c917795f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-copy-to-local-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+CopyToLocalMock::~CopyToLocalMock() = default;
+
+void CopyToLocalMock::SetExpectations(
+    std::function<std::unique_ptr<CopyToLocalMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<CopyToLocalMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<CopyToLocalMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass2Paths<CopyToLocalMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(arg1, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h
new file mode 100644
index 0000000000000..32973dd2f2e6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-copy-to-local.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class CopyToLocalMock} is an {@class CopyToLocal} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class CopyToLocalMock : public hdfs::tools::CopyToLocal {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CopyToLocalMock(const int argc, char **argv) : CopyToLocal(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  CopyToLocalMock(const CopyToLocalMock &) = delete;
+  CopyToLocalMock(CopyToLocalMock &&) = delete;
+  CopyToLocalMock &operator=(const CopyToLocalMock &) = delete;
+  CopyToLocalMock &operator=(CopyToLocalMock &&) = delete;
+  ~CopyToLocalMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void
+  SetExpectations(std::function<std::unique_ptr<CopyToLocalMock>()> test_case,
+                  const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc
new file mode 100644
index 0000000000000..649a71036079a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-count-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+CountMock::~CountMock() = default;
+
+void CountMock::SetExpectations(
+    std::function<std::unique_ptr<CountMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<CountMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<CountMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<CountMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandlePath(false, arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassQOptAndPath<CountMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(true, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h
new file mode 100644
index 0000000000000..6f0e5c00cea65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-count.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class CountMock} is an {@class Count} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class CountMock : public hdfs::tools::Count {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CountMock(const int argc, char **argv) : Count(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  CountMock(const CountMock &) = delete;
+  CountMock(CountMock &&) = delete;
+  CountMock &operator=(const CountMock &) = delete;
+  CountMock &operator=(CountMock &&) = delete;
+  ~CountMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<CountMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const bool, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc
new file mode 100644
index 0000000000000..6a4f96a83fd61
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc
@@ -0,0 +1,60 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-create-snapshot-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+CreateSnapshotMock::~CreateSnapshotMock() = default;
+
+void CreateSnapshotMock::SetExpectations(
+    std::function<std::unique_ptr<CreateSnapshotMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<CreateSnapshotMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<CreateSnapshotMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassNOptAndAPath<CreateSnapshotMock>) {
+    const auto opt_n = args[0];
+    const auto path = args[2];
+    const auto opt_n_value = std::optional{args[1]};
+    ASSERT_EQ(opt_n, "-n");
+    EXPECT_CALL(*this, HandleSnapshot(path, opt_n_value))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h
new file mode 100644
index 0000000000000..e159ec96c5370
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT_MOCK
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-create-snapshot.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class CreateSnapshotMock} is an {@class CreateSnapshot} whereby it mocks
+ * the HandleHelp and HandleSnapshot methods for testing their functionality.
+ */
+class CreateSnapshotMock : public hdfs::tools::CreateSnapshot {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CreateSnapshotMock(const int argc, char **argv)
+      : CreateSnapshot(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  CreateSnapshotMock(const CreateSnapshotMock &) = delete;
+  CreateSnapshotMock(CreateSnapshotMock &&) = delete;
+  CreateSnapshotMock &operator=(const CreateSnapshotMock &) = delete;
+  CreateSnapshotMock &operator=(CreateSnapshotMock &&) = delete;
+  ~CreateSnapshotMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(
+      std::function<std::unique_ptr<CreateSnapshotMock>()> test_case,
+      const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandleSnapshot,
+              (const std::string &, const std::optional<std::string> &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc
new file mode 100644
index 0000000000000..445cb3851cc69
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc
@@ -0,0 +1,56 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-disallow-snapshot-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+DisallowSnapshotMock::~DisallowSnapshotMock() = default;
+
+void DisallowSnapshotMock::SetExpectations(
+    std::function<std::unique_ptr<DisallowSnapshotMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<DisallowSnapshotMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<DisallowSnapshotMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<DisallowSnapshotMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandleSnapshot(arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h
new file mode 100644
index 0000000000000..a6e2c240781d9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-disallow-snapshot.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class DisallowSnapshotMock} is an {@class DisallowSnapshot} whereby it
+ * mocks the HandleHelp and HandleSnapshot methods for testing their
+ * functionality.
+ */
+class DisallowSnapshotMock : public hdfs::tools::DisallowSnapshot {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  DisallowSnapshotMock(const int argc, char **argv)
+      : DisallowSnapshot(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  DisallowSnapshotMock(const DisallowSnapshotMock &) = delete;
+  DisallowSnapshotMock(DisallowSnapshotMock &&) = delete;
+  DisallowSnapshotMock &operator=(const DisallowSnapshotMock &) = delete;
+  DisallowSnapshotMock &operator=(DisallowSnapshotMock &&) = delete;
+  ~DisallowSnapshotMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(
+      std::function<std::unique_ptr<DisallowSnapshotMock>()> test_case,
+      const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandleSnapshot, (const std::string &), (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc
new file mode 100644
index 0000000000000..e0c2ebf74546e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.cc
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-du-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+DuMock::~DuMock() = default;
+
+void DuMock::SetExpectations(std::function<std::unique_ptr<DuMock>()> test_case,
+                             const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func = test_case.target<std::unique_ptr<DuMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<DuMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<DuMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandlePath(arg1, false))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursivePath<DuMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    ASSERT_EQ(arg1, "-R");
+    EXPECT_CALL(*this, HandlePath(arg2, true))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursive<DuMock>) {
+    const auto arg1 = args[0];
+    ASSERT_EQ(arg1, "-R");
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h
new file mode 100644
index 0000000000000..de5caeb76b318
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-du-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_DU_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_DU_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-du.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class DuMock} is an {@class Du} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class DuMock : public hdfs::tools::Du {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  DuMock(const int argc, char **argv) : Du(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  DuMock(const DuMock &) = delete;
+  DuMock(DuMock &&) = delete;
+  DuMock &operator=(const DuMock &) = delete;
+  DuMock &operator=(DuMock &&) = delete;
+  ~DuMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<DuMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const bool),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc
new file mode 100644
index 0000000000000..9fd57ec270c07
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.cc
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-find-mock.h"
+#include "hdfs-tool-tests.h"
+#include "hdfspp/hdfspp.h"
+
+namespace hdfs::tools::test {
+FindMock::~FindMock() = default;
+
+void FindMock::SetExpectations(
+    std::function<std::unique_ptr<FindMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<FindMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<FindMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<FindMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandlePath(arg1, "*",
+                                  hdfs::FileSystem::GetDefaultFindMaxDepth()))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassNOptAndAPath<FindMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    const auto arg3 = args[2];
+    ASSERT_EQ(arg1, "-n");
+    EXPECT_CALL(*this, HandlePath(arg3, arg2,
+                                  hdfs::FileSystem::GetDefaultFindMaxDepth()))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassMOptPermissionsAndAPath<FindMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    const auto arg3 = args[2];
+    ASSERT_EQ(arg1, "-m");
+    EXPECT_CALL(*this,
+                HandlePath(arg3, "*", static_cast<uint32_t>(std::stoi(arg2))))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassNStrMNumAndAPath<FindMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    const auto arg3 = args[2];
+    const auto arg4 = args[3];
+    const auto arg5 = args[4];
+    ASSERT_EQ(arg1, "-n");
+    ASSERT_EQ(arg3, "-m");
+    EXPECT_CALL(*this,
+                HandlePath(arg5, arg2, static_cast<uint32_t>(std::stoi(arg4))))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h
new file mode 100644
index 0000000000000..7520ac7c0da03
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-find-mock.h
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_FIND_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_FIND_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-find.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class FindMock} is an {@class Find} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class FindMock : public hdfs::tools::Find {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  FindMock(const int argc, char **argv) : Find(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  FindMock(const FindMock &) = delete;
+  FindMock(FindMock &&) = delete;
+  FindMock &operator=(const FindMock &) = delete;
+  FindMock &operator=(FindMock &&) = delete;
+  ~FindMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<FindMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath,
+              (const std::string &, const std::string &, uint32_t),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc
new file mode 100644
index 0000000000000..713564e45b160
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-get-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+GetMock::~GetMock() = default;
+
+void GetMock::SetExpectations(
+    std::function<std::unique_ptr<GetMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<GetMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<GetMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass2Paths<GetMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(arg1, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h
new file mode 100644
index 0000000000000..535f7153f1f98
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_GET_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_GET_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-get.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class GetMock} is an {@class Get} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class GetMock : public hdfs::tools::Get {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  GetMock(const int argc, char **argv) : Get(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  GetMock(const GetMock &) = delete;
+  GetMock(GetMock &&) = delete;
+  GetMock &operator=(const GetMock &) = delete;
+  GetMock &operator=(GetMock &&) = delete;
+  ~GetMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<GetMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc
new file mode 100644
index 0000000000000..6f1cbcf1d074d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.cc
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-ls-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+LsMock::~LsMock() = default;
+
+void LsMock::SetExpectations(std::function<std::unique_ptr<LsMock>()> test_case,
+                             const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func = test_case.target<std::unique_ptr<LsMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<LsMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<LsMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandlePath(arg1, false))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursivePath<LsMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    ASSERT_EQ(arg1, "-R");
+    EXPECT_CALL(*this, HandlePath(arg2, true))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursive<LsMock>) {
+    const auto arg1 = args[0];
+    ASSERT_EQ(arg1, "-R");
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h
new file mode 100644
index 0000000000000..2218549e95cb7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-ls-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_LS_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_LS_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-ls.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class LsMock} is an {@class Ls} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class LsMock : public hdfs::tools::Ls {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  LsMock(const int argc, char **argv) : Ls(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  LsMock(const LsMock &) = delete;
+  LsMock(LsMock &&) = delete;
+  LsMock &operator=(const LsMock &) = delete;
+  LsMock &operator=(LsMock &&) = delete;
+  ~LsMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<LsMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const bool),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc
new file mode 100644
index 0000000000000..54ed0b0990ee1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-mkdir-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+MkdirMock::~MkdirMock() = default;
+
+void MkdirMock::SetExpectations(
+    std::function<std::unique_ptr<MkdirMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<MkdirMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<MkdirMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<MkdirMock>) {
+    const auto arg1 = args[0];
+    const std::optional<std::string> permissions = std::nullopt;
+    EXPECT_CALL(*this, HandlePath(false, permissions, arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassPOptAndPath<MkdirMock>) {
+    const auto arg1 = args[1];
+    const std::optional<std::string> permissions = std::nullopt;
+    EXPECT_CALL(*this, HandlePath(true, permissions, arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassMOptPermissionsAndAPath<MkdirMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[2];
+    const auto permissions = std::optional(arg1);
+    EXPECT_CALL(*this, HandlePath(false, permissions, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassMPOptsPermissionsAndAPath<MkdirMock>) {
+    const auto arg1 = args[1];
+    const auto arg2 = args[3];
+    const auto permissions = std::optional(arg1);
+    EXPECT_CALL(*this, HandlePath(true, permissions, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h
new file mode 100644
index 0000000000000..e112a14cd2049
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK
+
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-mkdir.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class MkdirMock} is an {@class Mkdir} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class MkdirMock : public hdfs::tools::Mkdir {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  MkdirMock(const int argc, char **argv) : Mkdir(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  MkdirMock(const MkdirMock &) = delete;
+  MkdirMock(MkdirMock &&) = delete;
+  MkdirMock &operator=(const MkdirMock &) = delete;
+  MkdirMock &operator=(MkdirMock &&) = delete;
+  ~MkdirMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<MkdirMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath,
+              (bool, const std::optional<std::string> &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc
new file mode 100644
index 0000000000000..33d1a7c99486f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-move-to-local-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+MoveToLocalMock::~MoveToLocalMock() = default;
+
+void MoveToLocalMock::SetExpectations(
+    std::function<std::unique_ptr<MoveToLocalMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<MoveToLocalMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<MoveToLocalMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass2Paths<MoveToLocalMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(arg1, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h
new file mode 100644
index 0000000000000..a068b5a9e2302
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-move-to-local.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class MoveToLocalMock} is an {@class MoveToLocal} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class MoveToLocalMock : public hdfs::tools::MoveToLocal {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  MoveToLocalMock(const int argc, char **argv) : MoveToLocal(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  MoveToLocalMock(const MoveToLocalMock &) = delete;
+  MoveToLocalMock(MoveToLocalMock &&) = delete;
+  MoveToLocalMock &operator=(const MoveToLocalMock &) = delete;
+  MoveToLocalMock &operator=(MoveToLocalMock &&) = delete;
+  ~MoveToLocalMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void
+  SetExpectations(std::function<std::unique_ptr<MoveToLocalMock>()> test_case,
+                  const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc
new file mode 100644
index 0000000000000..6ee7f173d847a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc
@@ -0,0 +1,58 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-rename-snapshot-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+RenameSnapshotMock::~RenameSnapshotMock() = default;
+
+void RenameSnapshotMock::SetExpectations(
+    std::function<std::unique_ptr<RenameSnapshotMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<RenameSnapshotMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<RenameSnapshotMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass3Paths<RenameSnapshotMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    const auto arg3 = args[2];
+    EXPECT_CALL(*this, HandleSnapshot(arg1, arg2, arg3))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h
new file mode 100644
index 0000000000000..657c2f179146a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-rename-snapshot.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class RenameSnapshotMock} is an {@class RenameSnapshot} whereby it
+ * mocks the HandleHelp and HandleSnapshot methods for testing their
+ * functionality.
+ */
+class RenameSnapshotMock : public hdfs::tools::RenameSnapshot {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  RenameSnapshotMock(const int argc, char **argv)
+      : RenameSnapshot(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  RenameSnapshotMock(const RenameSnapshotMock &) = delete;
+  RenameSnapshotMock(RenameSnapshotMock &&) = delete;
+  RenameSnapshotMock &operator=(const RenameSnapshotMock &) = delete;
+  RenameSnapshotMock &operator=(RenameSnapshotMock &&) = delete;
+  ~RenameSnapshotMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(
+      std::function<std::unique_ptr<RenameSnapshotMock>()> test_case,
+      const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandleSnapshot,
+              (const std::string &, const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc
new file mode 100644
index 0000000000000..29cf95fcced6d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-rm-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+RmMock::~RmMock() = default;
+
+void RmMock::SetExpectations(std::function<std::unique_ptr<RmMock>()> test_case,
+                             const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func = test_case.target<std::unique_ptr<RmMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<RmMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<RmMock>) {
+    const auto arg1 = args[0];
+    EXPECT_CALL(*this, HandlePath(false, arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassRecursivePath<RmMock>) {
+    const auto arg1 = args[1];
+    EXPECT_CALL(*this, HandlePath(true, arg1))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h
new file mode 100644
index 0000000000000..632716bf0a64b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_RM_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_RM_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-rm.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class RmMock} is an {@class Rm} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class RmMock : public hdfs::tools::Rm {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  RmMock(const int argc, char **argv) : Rm(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  RmMock(const RmMock &) = delete;
+  RmMock(RmMock &&) = delete;
+  RmMock &operator=(const RmMock &) = delete;
+  RmMock &operator=(RmMock &&) = delete;
+  ~RmMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<RmMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const bool, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc
new file mode 100644
index 0000000000000..d33f49b6aec67
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.cc
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-setrep-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+SetrepMock::~SetrepMock() = default;
+
+void SetrepMock::SetExpectations(
+    std::function<std::unique_ptr<SetrepMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<SetrepMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<SetrepMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassPermissionsAndAPath<SetrepMock>) {
+    const auto number = args[0];
+    const auto path = args[1];
+    EXPECT_CALL(*this, HandlePath(path, number))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h
new file mode 100644
index 0000000000000..db1e0960ae0a7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-setrep-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_SETREP_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_SETREP_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-setrep.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class SetrepMock} is an {@class Setrep} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class SetrepMock : public hdfs::tools::Setrep {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  SetrepMock(const int argc, char **argv) : Setrep(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  SetrepMock(const SetrepMock &) = delete;
+  SetrepMock(SetrepMock &&) = delete;
+  SetrepMock &operator=(const SetrepMock &) = delete;
+  SetrepMock &operator=(SetrepMock &&) = delete;
+  ~SetrepMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<SetrepMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc
new file mode 100644
index 0000000000000..efa773ce70fdc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.cc
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-stat-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+StatMock::~StatMock() = default;
+
+void StatMock::SetExpectations(
+    std::function<std::unique_ptr<StatMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<StatMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<StatMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<StatMock>) {
+    const auto path = args[0];
+    EXPECT_CALL(*this, HandlePath(path))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h
new file mode 100644
index 0000000000000..01781f5f1751c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-stat-mock.h
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_STAT_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_STAT_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-stat.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class StatMock} is an {@class Stat} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class StatMock : public hdfs::tools::Stat {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  StatMock(const int argc, char **argv) : Stat(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  StatMock(const StatMock &) = delete;
+  StatMock(StatMock &&) = delete;
+  StatMock &operator=(const StatMock &) = delete;
+  StatMock &operator=(StatMock &&) = delete;
+  ~StatMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<StatMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &), (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc
new file mode 100644
index 0000000000000..59c9f3a23b371
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.cc
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-tail-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+TailMock::~TailMock() = default;
+
+void TailMock::SetExpectations(
+    std::function<std::unique_ptr<TailMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<TailMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<TailMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &PassAPath<TailMock>) {
+    const auto path = args[0];
+    EXPECT_CALL(*this, HandlePath(path, false))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+
+  if (*test_case_func == &PassFOptAndAPath<TailMock>) {
+    const auto f_opt = args[0];
+    const auto path = args[1];
+    ASSERT_EQ(f_opt, "-f");
+    EXPECT_CALL(*this, HandlePath(path, true))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h
new file mode 100644
index 0000000000000..e9fb9b7521bfa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tail-mock.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_TAIL_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_TAIL_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-tail.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class TailMock} is an {@class Tail} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class TailMock : public hdfs::tools::Tail {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  TailMock(const int argc, char **argv) : Tail(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  TailMock(const TailMock &) = delete;
+  TailMock(TailMock &&) = delete;
+  TailMock &operator=(const TailMock &) = delete;
+  TailMock &operator=(TailMock &&) = delete;
+  ~TailMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void SetExpectations(std::function<std::unique_ptr<TailMock>()> test_case,
+                       const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const bool),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
index 8ffce3b3f6a40..2a16c3c0d97f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
@@ -20,9 +20,27 @@
 #include <gtest/gtest.h>
 
 #include "hdfs-allow-snapshot-mock.h"
-#include "hdfs-df-mock.h"
 #include "hdfs-cat-mock.h"
+#include "hdfs-chgrp-mock.h"
+#include "hdfs-chmod-mock.h"
+#include "hdfs-chown-mock.h"
+#include "hdfs-copy-to-local-mock.h"
+#include "hdfs-count-mock.h"
+#include "hdfs-create-snapshot-mock.h"
 #include "hdfs-delete-snapshot-mock.h"
+#include "hdfs-df-mock.h"
+#include "hdfs-disallow-snapshot-mock.h"
+#include "hdfs-du-mock.h"
+#include "hdfs-find-mock.h"
+#include "hdfs-get-mock.h"
+#include "hdfs-ls-mock.h"
+#include "hdfs-mkdir-mock.h"
+#include "hdfs-move-to-local-mock.h"
+#include "hdfs-rename-snapshot-mock.h"
+#include "hdfs-rm-mock.h"
+#include "hdfs-setrep-mock.h"
+#include "hdfs-stat-mock.h"
+#include "hdfs-tail-mock.h"
 #include "hdfs-tool-test-fixtures.h"
 #include "hdfs-tool-tests.h"
 
@@ -32,17 +50,27 @@
  * hdfs-tool-test.h} to yield the test suite.
  */
 
-/**
- * This file combines the test fixtures defined in {@file
- * hdfs-tool-test-fixtures.h} and the test cases defined in {@file
- * hdfs-tool-test.h} to yield the test structure.
- */
-
+// Basic tests
 INSTANTIATE_TEST_SUITE_P(
     HdfsAllowSnapshot, HdfsToolBasicTest,
     testing::Values(PassAPath<hdfs::tools::test::AllowSnapshotMock>,
                     CallHelp<hdfs::tools::test::AllowSnapshotMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsDisallowSnapshot, HdfsToolBasicTest,
+    testing::Values(PassAPath<hdfs::tools::test::DisallowSnapshotMock>,
+                    CallHelp<hdfs::tools::test::DisallowSnapshotMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsRenameSnapshot, HdfsToolBasicTest,
+    testing::Values(Pass3Paths<hdfs::tools::test::RenameSnapshotMock>,
+                    CallHelp<hdfs::tools::test::RenameSnapshotMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCreateSnapshot, HdfsToolBasicTest,
+    testing::Values(PassNOptAndAPath<hdfs::tools::test::CreateSnapshotMock>,
+                    CallHelp<hdfs::tools::test::CreateSnapshotMock>));
+
 INSTANTIATE_TEST_SUITE_P(HdfsCat, HdfsToolBasicTest,
                          testing::Values(PassAPath<hdfs::tools::test::CatMock>,
                                          CallHelp<hdfs::tools::test::CatMock>));
@@ -51,23 +79,290 @@ INSTANTIATE_TEST_SUITE_P(HdfsDf, HdfsToolBasicTest,
                          testing::Values(PassAPath<hdfs::tools::test::DfMock>,
                                          CallHelp<hdfs::tools::test::DfMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsDu, HdfsToolBasicTest,
+    testing::Values(PassAPath<hdfs::tools::test::DuMock>,
+                    CallHelp<hdfs::tools::test::DuMock>,
+                    PassRecursivePath<hdfs::tools::test::DuMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsLs, HdfsToolBasicTest,
+    testing::Values(PassAPath<hdfs::tools::test::LsMock>,
+                    CallHelp<hdfs::tools::test::LsMock>,
+                    PassRecursivePath<hdfs::tools::test::LsMock>));
+
 INSTANTIATE_TEST_SUITE_P(
     HdfsDeleteSnapshot, HdfsToolBasicTest,
     testing::Values(CallHelp<hdfs::tools::test::DeleteSnapshotMock>,
                     Pass2Paths<hdfs::tools::test::DeleteSnapshotMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChown, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::ChownMock>,
+                    PassOwnerAndAPath<hdfs::tools::test::ChownMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::ChownMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChmod, HdfsToolBasicTest,
+    testing::Values(
+        CallHelp<hdfs::tools::test::ChmodMock>,
+        PassPermissionsAndAPath<hdfs::tools::test::ChmodMock>,
+        PassRecursivePermissionsAndAPath<hdfs::tools::test::ChmodMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChgrp, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::ChgrpMock>,
+                    PassOwnerAndAPath<hdfs::tools::test::ChgrpMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::ChgrpMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::CopyToLocalMock>,
+                    Pass2Paths<hdfs::tools::test::CopyToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsGet, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::GetMock>,
+                    Pass2Paths<hdfs::tools::test::GetMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::MoveToLocalMock>,
+                    Pass2Paths<hdfs::tools::test::MoveToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCount, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::CountMock>,
+                    PassAPath<hdfs::tools::test::CountMock>,
+                    PassQOptAndPath<hdfs::tools::test::CountMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMkdir, HdfsToolBasicTest,
+    testing::Values(
+        CallHelp<hdfs::tools::test::MkdirMock>,
+        PassAPath<hdfs::tools::test::MkdirMock>,
+        PassPOptAndPath<hdfs::tools::test::MkdirMock>,
+        PassMOptPermissionsAndAPath<hdfs::tools::test::MkdirMock>,
+        PassMPOptsPermissionsAndAPath<hdfs::tools::test::MkdirMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsRm, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::RmMock>,
+                    PassAPath<hdfs::tools::test::RmMock>,
+                    PassRecursivePath<hdfs::tools::test::RmMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsFind, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::FindMock>,
+                    PassAPath<hdfs::tools::test::FindMock>,
+                    PassNStrMNumAndAPath<hdfs::tools::test::FindMock>,
+                    PassMOptPermissionsAndAPath<hdfs::tools::test::FindMock>,
+                    PassNOptAndAPath<hdfs::tools::test::FindMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsSetrep, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::SetrepMock>,
+                    PassPermissionsAndAPath<hdfs::tools::test::SetrepMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsStat, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::StatMock>,
+                    PassAPath<hdfs::tools::test::StatMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsTail, HdfsToolBasicTest,
+    testing::Values(PassAPath<hdfs::tools::test::TailMock>,
+                    CallHelp<hdfs::tools::test::TailMock>,
+                    PassFOptAndAPath<hdfs::tools::test::TailMock>));
+
+// Negative tests
 INSTANTIATE_TEST_SUITE_P(
     HdfsAllowSnapshot, HdfsToolNegativeTestThrows,
     testing::Values(Pass2Paths<hdfs::tools::test::AllowSnapshotMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsRenameSnapshot, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::RenameSnapshotMock>,
+                    Pass2Paths<hdfs::tools::test::RenameSnapshotMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCreateSnapshot, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::CreateSnapshotMock>,
+                    Pass3Paths<hdfs::tools::test::CreateSnapshotMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsDisallowSnapshot, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::DisallowSnapshotMock>));
+
 INSTANTIATE_TEST_SUITE_P(
     HdfsDf, HdfsToolNegativeTestThrows,
     testing::Values(Pass2Paths<hdfs::tools::test::DfMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsDu, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::DuMock>,
+                    Pass3Paths<hdfs::tools::test::DuMock>,
+                    PassNOptAndAPath<hdfs::tools::test::DuMock>,
+                    PassOwnerAndAPath<hdfs::tools::test::DuMock>,
+                    PassPermissionsAndAPath<hdfs::tools::test::DuMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsLs, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::LsMock>,
+                    Pass3Paths<hdfs::tools::test::LsMock>,
+                    PassNOptAndAPath<hdfs::tools::test::LsMock>,
+                    PassOwnerAndAPath<hdfs::tools::test::LsMock>,
+                    PassPermissionsAndAPath<hdfs::tools::test::LsMock>));
+
 INSTANTIATE_TEST_SUITE_P(
     HdfsCat, HdfsToolNegativeTestThrows,
     testing::Values(Pass2Paths<hdfs::tools::test::CatMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolNegativeTestThrows,
+    testing::Values(Pass3Paths<hdfs::tools::test::CopyToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsGet, HdfsToolNegativeTestThrows,
+    testing::Values(Pass3Paths<hdfs::tools::test::GetMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolNegativeTestThrows,
+    testing::Values(Pass3Paths<hdfs::tools::test::MoveToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCount, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::CountMock>,
+                    Pass3Paths<hdfs::tools::test::CountMock>,
+                    PassNOptAndAPath<hdfs::tools::test::CountMock>,
+                    PassRecursive<hdfs::tools::test::CountMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMkdir, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::MkdirMock>,
+                    Pass3Paths<hdfs::tools::test::MkdirMock>,
+                    PassNOptAndAPath<hdfs::tools::test::MkdirMock>,
+                    PassRecursive<hdfs::tools::test::MkdirMock>,
+                    PassMOpt<hdfs::tools::test::MkdirMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsRm, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::RmMock>,
+                    Pass3Paths<hdfs::tools::test::RmMock>,
+                    PassNOptAndAPath<hdfs::tools::test::RmMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::RmMock>,
+                    PassMOpt<hdfs::tools::test::RmMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsFind, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::FindMock>,
+                    Pass3Paths<hdfs::tools::test::FindMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::FindMock>,
+                    PassRecursive<hdfs::tools::test::FindMock>,
+                    PassRecursivePath<hdfs::tools::test::FindMock>,
+                    PassMPOptsPermissionsAndAPath<hdfs::tools::test::FindMock>,
+                    PassMOpt<hdfs::tools::test::FindMock>,
+                    PassNOpt<hdfs::tools::test::FindMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChgrp, HdfsToolNegativeTestThrows,
+    testing::Values(PassNOptAndAPath<hdfs::tools::test::ChgrpMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsSetrep, HdfsToolNegativeTestThrows,
+    testing::Values(
+        Pass3Paths<hdfs::tools::test::SetrepMock>,
+        PassRecursiveOwnerAndAPath<hdfs::tools::test::SetrepMock>,
+        PassRecursive<hdfs::tools::test::SetrepMock>,
+        PassMPOptsPermissionsAndAPath<hdfs::tools::test::SetrepMock>,
+        PassMOpt<hdfs::tools::test::SetrepMock>,
+        PassNOpt<hdfs::tools::test::SetrepMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsStat, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::StatMock>,
+                    Pass3Paths<hdfs::tools::test::StatMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::StatMock>,
+                    PassRecursive<hdfs::tools::test::StatMock>,
+                    PassRecursivePath<hdfs::tools::test::StatMock>,
+                    PassMPOptsPermissionsAndAPath<hdfs::tools::test::StatMock>,
+                    PassMOpt<hdfs::tools::test::StatMock>,
+                    PassNOpt<hdfs::tools::test::StatMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsTail, HdfsToolNegativeTestThrows,
+    testing::Values(Pass2Paths<hdfs::tools::test::TailMock>,
+                    Pass3Paths<hdfs::tools::test::TailMock>,
+                    PassNOptAndAPath<hdfs::tools::test::TailMock>,
+                    PassRecursiveOwnerAndAPath<hdfs::tools::test::TailMock>,
+                    PassMOpt<hdfs::tools::test::TailMock>,
+                    PassRecursive<hdfs::tools::test::TailMock>,
+                    PassRecursivePath<hdfs::tools::test::TailMock>,
+                    PassNOpt<hdfs::tools::test::TailMock>,
+                    PassOwnerAndAPath<hdfs::tools::test::TailMock>,
+                    PassMPOptsPermissionsAndAPath<hdfs::tools::test::TailMock>,
+                    PassPermissionsAndAPath<hdfs::tools::test::TailMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsRm, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassRecursive<hdfs::tools::test::RmMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMkdir, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassPOpt<hdfs::tools::test::MkdirMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCount, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassQOpt<hdfs::tools::test::CountMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::MoveToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::CopyToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsGet, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::GetMock>));
+
 INSTANTIATE_TEST_SUITE_P(
     HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow,
     testing::Values(PassAPath<hdfs::tools::test::DeleteSnapshotMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsDu, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassRecursive<hdfs::tools::test::DuMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsLs, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassRecursive<hdfs::tools::test::LsMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChown, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::ChownMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChown, HdfsToolNegativeTestThrows,
+    testing::Values(PassNOptAndAPath<hdfs::tools::test::ChownMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChmod, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::ChmodMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChmod, HdfsToolNegativeTestThrows,
+    testing::Values(PassNOptAndAPath<hdfs::tools::test::ChmodMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsChgrp, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::ChgrpMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsSetrep, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::SetrepMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsTail, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassFOpt<hdfs::tools::test::TailMock>));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h
index f7fd6a706a646..8a1ae8cf0b772 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h
@@ -17,8 +17,8 @@
   under the License.
 */
 
-#ifndef LIBHDFSPP_TOOLS_HDFS_TOOL_TEST
-#define LIBHDFSPP_TOOLS_HDFS_TOOL_TEST
+#ifndef LIBHDFSPP_TOOLS_HDFS_TOOL_TESTS
+#define LIBHDFSPP_TOOLS_HDFS_TOOL_TESTS
 
 #include <memory>
 #include <string>
@@ -44,6 +44,44 @@ template <class T> std::unique_ptr<T> PassAPath() {
   return hdfs_tool;
 }
 
+template <class T> std::unique_ptr<T> PassRecursive() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-R");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassRecursive<T>, {arg1});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassRecursivePath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-R");
+  static std::string arg2("a/b/c");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassRecursivePath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassFOptAndAPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-f");
+  static std::string arg2("a/b/c");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassFOptAndAPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
 template <class T> std::unique_ptr<T> CallHelp() {
   constexpr auto argc = 2;
   static std::string exe("hdfs_tool_name");
@@ -69,4 +107,236 @@ template <class T> std::unique_ptr<T> Pass2Paths() {
   return hdfs_tool;
 }
 
+template <class T> std::unique_ptr<T> Pass3Paths() {
+  constexpr auto argc = 4;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("a/b/c");
+  static std::string arg2("d/e/f");
+  static std::string arg3("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(Pass3Paths<T>, {arg1, arg2, arg3});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassNOptAndAPath() {
+  constexpr auto argc = 4;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-n");
+  static std::string arg2("some_name");
+  static std::string arg3("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassNOptAndAPath<T>, {arg1, arg2, arg3});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassOwnerAndAPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("new_owner:new_group");
+  static std::string arg2("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassOwnerAndAPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassRecursiveOwnerAndAPath() {
+  constexpr auto argc = 4;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-R");
+  static std::string arg2("new_owner:new_group");
+  static std::string arg3("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassRecursiveOwnerAndAPath<T>, {arg1, arg2, arg3});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassPermissionsAndAPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("757");
+  static std::string arg2("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassPermissionsAndAPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassInvalidPermissionsAndAPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("123456789123456789123456789");
+  static std::string arg2("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassInvalidPermissionsAndAPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassRecursivePermissionsAndAPath() {
+  constexpr auto argc = 4;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-R");
+  static std::string arg2("757");
+  static std::string arg3("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassRecursivePermissionsAndAPath<T>,
+                             {arg1, arg2, arg3});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassQOpt() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-q");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassQOpt<T>, {arg1});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassQOptAndPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-q");
+  static std::string arg2("a/b/c");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassQOptAndPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassPOpt() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-p");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassPOpt<T>, {arg1});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassMOpt() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-m");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassMOpt<T>, {arg1});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassFOpt() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-f");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassFOpt<T>, {arg1});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassPOptAndPath() {
+  constexpr auto argc = 3;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-p");
+  static std::string arg2("a/b/c");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassPOptAndPath<T>, {arg1, arg2});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassMOptPermissionsAndAPath() {
+  constexpr auto argc = 4;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-m");
+  static std::string arg2("757");
+  static std::string arg3("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassMOptPermissionsAndAPath<T>,
+                             {arg1, arg2, arg3});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassMPOptsPermissionsAndAPath() {
+  constexpr auto argc = 5;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-m");
+  static std::string arg2("757");
+  static std::string arg3("-p");
+  static std::string arg4("g/h/i");
+
+  static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data(),
+                         arg4.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassMPOptsPermissionsAndAPath<T>,
+                             {arg1, arg2, arg3, arg4});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassNStrMNumAndAPath() {
+  constexpr auto argc = 6;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-n");
+  static std::string arg2("some_str");
+  static std::string arg3("-m");
+  static std::string arg4("757");
+  static std::string arg5("some/path");
+
+  static char *argv[] = {exe.data(),  arg1.data(), arg2.data(),
+                         arg3.data(), arg4.data(), arg5.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassNStrMNumAndAPath<T>,
+                             {arg1, arg2, arg3, arg4, arg5});
+  return hdfs_tool;
+}
+
+template <class T> std::unique_ptr<T> PassNOpt() {
+  constexpr auto argc = 2;
+  static std::string exe("hdfs_tool_name");
+  static std::string arg1("-n");
+
+  static char *argv[] = {exe.data(), arg1.data()};
+
+  auto hdfs_tool = std::make_unique<T>(argc, argv);
+  hdfs_tool->SetExpectations(PassNOpt<T>, {arg1});
+  return hdfs_tool;
+}
+
 #endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
index 6a7d0bec37ed4..565346e088376 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
@@ -31,3 +31,20 @@ add_test(x_platform_utils_test x_platform_utils_test)
 target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
 target_link_libraries(x_platform_syscall_test gmock_main)
 add_test(x_platform_syscall_test x_platform_syscall_test)
+
+add_executable(x_platform_types_test types_test.cc)
+target_include_directories(x_platform_types_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_types_test gtest_main)
+add_test(x_platform_types_test x_platform_types_test)
+
+add_library(x_platform_dirent_test_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> dirent_test.cc)
+add_executable(x_platform_dirent_test $<TARGET_OBJECTS:x_platform_dirent_test_obj> $<TARGET_OBJECTS:x_platform_obj>)
+target_include_directories(x_platform_dirent_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_dirent_test PRIVATE gtest_main)
+add_test(x_platform_dirent_test x_platform_dirent_test)
+
+add_executable(x_platform_dirent_c_test $<TARGET_OBJECTS:x_platform_dirent_test_obj> $<TARGET_OBJECTS:x_platform_obj> $<TARGET_OBJECTS:x_platform_obj_c_api> c-api/dirent_test.cc)
+target_compile_definitions(x_platform_dirent_c_test PRIVATE USE_X_PLATFORM_DIRENT)
+target_include_directories(x_platform_dirent_c_test PRIVATE ${LIBHDFSPP_LIB_DIR} ../)
+target_link_libraries(x_platform_dirent_c_test PRIVATE gtest_main)
+add_test(x_platform_dirent_c_test x_platform_dirent_c_test)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.cc
new file mode 100644
index 0000000000000..6456b02e84d09
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.cc
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cassert>
+#include <cerrno>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <unordered_set>
+
+#include "x-platform/c-api/dirent.h"
+#include "x-platform/c-api/dirent_test.h"
+
+std::unordered_set<std::string>
+DirentCApiTest::ListDirAndFiles(const std::string &path) const {
+  std::stringstream err_msg;
+  std::unordered_set<std::string> paths;
+
+  DIR *dir = opendir(path.c_str());
+  if (dir == nullptr) {
+    err_msg << "Unable to open directory " << path;
+    throw std::runtime_error(err_msg.str());
+  }
+
+  errno = 0;
+  for (struct dirent *file; (file = readdir(dir)) != nullptr; errno = 0) {
+    std::filesystem::path absolute_path{path};
+    absolute_path = absolute_path / file->d_name;
+    paths.emplace(absolute_path.string());
+  }
+
+  if (errno != 0) {
+    err_msg << "Expected errno to be 0, instead it is " << errno;
+    throw std::runtime_error(err_msg.str());
+  }
+
+  if (const auto result = closedir(dir); result != 0) {
+    err_msg << "Expected the return value of closedir() to be 0, instead it is "
+            << result;
+    throw std::runtime_error(err_msg.str());
+  }
+  return paths;
+}
+
+TEST_F(DirentCApiTest, TestEmptyFolder) {
+  const auto expected = CreateTempDirAndFiles(0, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestOneFolder) {
+  const auto expected = CreateTempDirAndFiles(1, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestOneFile) {
+  const auto expected = CreateTempDirAndFiles(0, 1);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestMultipleFolders) {
+  const auto expected = CreateTempDirAndFiles(10, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestMultipleFiles) {
+  const auto expected = CreateTempDirAndFiles(0, 10);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestOneFileAndFolder) {
+  const auto expected = CreateTempDirAndFiles(1, 1);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentCApiTest, TestMultipleFilesAndFolders) {
+  const auto expected = CreateTempDirAndFiles(10, 10);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionException.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.h
similarity index 65%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionException.java
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.h
index 74607b8915a55..6e7054c78cb28 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/c-api/dirent_test.h
@@ -16,20 +16,22 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.swift.exceptions;
+#ifndef LIBHDFSPP_CROSS_PLATFORM_C_API_DIRENT_TEST
+#define LIBHDFSPP_CROSS_PLATFORM_C_API_DIRENT_TEST
+
+#include "x-platform/dirent.h"
+#include "x-platform/dirent_test.h"
 
 /**
- * Thrown to indicate that connection is lost or failed to be made
+ * Test fixture for testing {@link XPlatform::Dirent}'s C APIs.
  */
-public class SwiftConnectionException extends SwiftException {
-  public SwiftConnectionException() {
-  }
-
-  public SwiftConnectionException(String message) {
-    super(message);
-  }
+class DirentCApiTest : public DirentTest {
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::unordered_set<std::string>
+  ListDirAndFiles(const std::string &path) const override;
+};
 
-  public SwiftConnectionException(String message, Throwable cause) {
-    super(message, cause);
-  }
-}
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.cc
new file mode 100644
index 0000000000000..d0c9f98b0ec1e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.cc
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <unordered_set>
+#include <variant>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "dirent_test.h"
+#include "x-platform/dirent.h"
+#include "x-platform/syscall.h"
+
+void DirentTest::SetUp() {
+  tmp_root_ = std::filesystem::temp_directory_path() / GetTempName();
+  if (!std::filesystem::create_directories(tmp_root_)) {
+    std::stringstream err_msg;
+    err_msg << "Unable to create temp directory " << tmp_root_.string();
+    throw std::runtime_error(err_msg.str());
+  }
+}
+
+void DirentTest::TearDown() { std::filesystem::remove_all(tmp_root_); }
+
+std::string DirentTest::GetTempName(const std::string &pattern) const {
+  std::vector pattern_raw(pattern.begin(), pattern.end());
+  if (!XPlatform::Syscall::CreateTempDir(pattern_raw)) {
+    std::stringstream err_msg;
+    err_msg << "Creating temp dir failed" << std::endl;
+    throw std::runtime_error(err_msg.str());
+  }
+
+  std::string tmp_dir_path(pattern_raw.data());
+  std::filesystem::remove(tmp_dir_path);
+  return tmp_dir_path;
+}
+
+std::unordered_set<std::string>
+DirentTest::CreateTempDirAndFiles(std::size_t num_dirs,
+                                  std::size_t num_files) const {
+  std::unordered_set<std::string> paths;
+  for (std::size_t i = 0; i < num_dirs; ++i) {
+    const auto tmp_dir_absolute_path = tmp_root_ / std::to_string(i);
+    if (std::error_code err;
+        !std::filesystem::create_directories(tmp_dir_absolute_path, err)) {
+      std::stringstream err_msg;
+      err_msg << "Unable to create the temp dir "
+              << tmp_dir_absolute_path.string() << " reason: " << err.message();
+      throw std::runtime_error(err_msg.str());
+    }
+    paths.emplace(tmp_dir_absolute_path.string());
+  }
+
+  for (std::size_t i = 0; i < num_files; ++i) {
+    std::stringstream tmp_filename;
+    tmp_filename << i << ".txt";
+    const auto tmp_file_absolute_path = tmp_root_ / tmp_filename.str();
+    std::ofstream{tmp_file_absolute_path};
+    paths.emplace(tmp_file_absolute_path.string());
+  }
+  return paths;
+}
+
+std::unordered_set<std::string>
+DirentTest::ListDirAndFiles(const std::string &path) const {
+  std::unordered_set<std::string> paths;
+
+  XPlatform::Dirent dirent(path);
+  for (auto dir_entry = dirent.NextFile();
+       !std::holds_alternative<std::monostate>(dir_entry);
+       dir_entry = dirent.NextFile()) {
+    if (std::holds_alternative<std::error_code>(dir_entry)) {
+      std::stringstream err_msg;
+      const auto err = std::get<std::error_code>(dir_entry);
+      err_msg << "Error in listing directory " << path
+              << " cause: " << err.message();
+      throw std::runtime_error(err_msg.str());
+    }
+
+    if (std::holds_alternative<std::filesystem::directory_entry>(dir_entry)) {
+      const auto entry = std::get<std::filesystem::directory_entry>(dir_entry);
+      paths.emplace(entry.path().string());
+    }
+  }
+  return paths;
+}
+
+TEST_F(DirentTest, TestEmptyFolder) {
+  const auto expected = CreateTempDirAndFiles(0, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestOneFolder) {
+  const auto expected = CreateTempDirAndFiles(1, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestOneFile) {
+  const auto expected = CreateTempDirAndFiles(0, 1);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestMultipleFolders) {
+  const auto expected = CreateTempDirAndFiles(10, 0);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestMultipleFiles) {
+  const auto expected = CreateTempDirAndFiles(0, 10);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestOneFileAndFolder) {
+  const auto expected = CreateTempDirAndFiles(1, 1);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(DirentTest, TestMultipleFilesAndFolders) {
+  const auto expected = CreateTempDirAndFiles(10, 10);
+  const auto actual = ListDirAndFiles(tmp_root_.string());
+  EXPECT_EQ(expected, actual);
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.h
new file mode 100644
index 0000000000000..4b79c30eea4e7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/dirent_test.h
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_CROSS_PLATFORM_DIRENT_TEST
+#define LIBHDFSPP_CROSS_PLATFORM_DIRENT_TEST
+
+#include <filesystem>
+#include <string>
+#include <unordered_set>
+
+#include <gtest/gtest.h>
+
+#include "x-platform/dirent.h"
+
+/**
+ * Test fixture for testing {@link XPlatform::Dirent}.
+ */
+class DirentTest : public ::testing::Test {
+protected:
+  void SetUp() override;
+  void TearDown() override;
+
+  /**
+   * Gets a name for creating temporary file or folder. This also ensures that
+   * the temporary file or folder does not exist.
+   *
+   * @param pattern The pattern to use for naming the temporary directory.
+   * @return The temporary file or folder name that can be used for creating the
+   * same.
+   */
+  [[nodiscard]] std::string
+  GetTempName(const std::string &pattern = "test_XXXXXX") const;
+
+  /**
+   * Creates the given number of temporary files and directories under the
+   * {@link DirentTest#tmp_root_}.
+   *
+   * @param num_dirs The number of temporary directories to create.
+   * @param num_files The number of temporary files to create.
+   * @return An {@link std::unordered_set> of the absolute paths of all the
+   * temporary files and folders that were created.
+   */
+  [[nodiscard]] std::unordered_set<std::string>
+  CreateTempDirAndFiles(std::size_t num_dirs, std::size_t num_files) const;
+
+  /**
+   * Lists all the children of the given path.
+   *
+   * @param path The path whose children must be listed.
+   * @return An {@link std::unordered_set} containing the absolute paths of all
+   * the children of the given path.
+   */
+  [[nodiscard]] virtual std::unordered_set<std::string>
+  ListDirAndFiles(const std::string &path) const;
+
+  /**
+   * The root in temp folder under which the rest of all the temporary files and
+   * folders will be created for the purpose of testing.
+   */
+  std::filesystem::path tmp_root_;
+};
+
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc
new file mode 100644
index 0000000000000..b234fa2f107eb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "types_test.h"
+#include "x-platform/types.h"
+
+INSTANTIATE_TYPED_TEST_SUITE_P(SSizeTTest, XPlatformTypesTest, ssize_t);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h
new file mode 100644
index 0000000000000..b24f657880615
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST
+#define LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST
+
+#include <limits>
+
+#include <gtest/gtest.h>
+
+/**
+ * {@class XPlatformTypesTest} tests the types defined in the XPlatform library.
+ */
+template <class T> class XPlatformTypesTest : public testing::Test {
+public:
+  XPlatformTypesTest() = default;
+  XPlatformTypesTest(const XPlatformTypesTest &) = delete;
+  XPlatformTypesTest(XPlatformTypesTest &&) = delete;
+  XPlatformTypesTest &operator=(const XPlatformTypesTest &) = delete;
+  XPlatformTypesTest &operator=(XPlatformTypesTest &&) = delete;
+  ~XPlatformTypesTest() override;
+};
+
+template <class T> XPlatformTypesTest<T>::~XPlatformTypesTest() = default;
+
+TYPED_TEST_SUITE_P(XPlatformTypesTest);
+
+/**
+ * Tests whether ssize_t can hold -1.
+ */
+TYPED_TEST_P(XPlatformTypesTest, SSizeTMinusOne) {
+  constexpr TypeParam value = -1;
+  ASSERT_EQ(value, -1);
+}
+
+/**
+ * Tests whether ssize_t can hold at least an int.
+ */
+TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldInts) {
+  constexpr auto actual = std::numeric_limits<TypeParam>::max();
+  constexpr auto expected = std::numeric_limits<int>::max();
+  ASSERT_GE(actual, expected);
+}
+
+// For 64-bit systems.
+#if _WIN64 || __x86_64__ || __ppc64__
+/**
+ * Tests whether ssize_t can hold at least a long int.
+ */
+TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldLongInts) {
+  constexpr auto actual = std::numeric_limits<TypeParam>::max();
+  constexpr auto expected = std::numeric_limits<long int>::max();
+  ASSERT_GE(actual, expected);
+}
+
+REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne,
+                            SSizeTCanHoldInts, SSizeTCanHoldLongInts);
+#else
+REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne,
+                            SSizeTCanHoldInts);
+#endif
+
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c
index ab6209c864a31..3036acfd353c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c
@@ -71,9 +71,11 @@ static const char *copy_path(const UriPathSegmentA *ps, char **buffer) {
 static int parse_int(const char *first, const char *after_last) {
 	const int size = after_last - first;
 	if (size) {
-		char buffer[size + 1];
+                char* buffer = (char*) malloc(size + 1);
 		memcpyz(buffer, first, size);
-		return atoi(buffer);
+                const int value = atoi(buffer);
+                free(buffer);
+                return value;
 	}
 	return 0;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
index c1222502547cc..52caf01915653 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
@@ -28,67 +28,30 @@ link_directories( ${LIBHDFSPP_DIR}/lib )
 add_library(tools_common_obj OBJECT tools_common.cc)
 add_library(tools_common $<TARGET_OBJECTS:tools_common_obj>)
 
+add_subdirectory(internal)
+
 add_library(hdfs_tool_obj OBJECT hdfs-tool.cc)
 target_include_directories(hdfs_tool_obj PRIVATE ../tools)
 
 add_subdirectory(hdfs-cat)
-
-add_executable(hdfs_chgrp hdfs_chgrp.cc)
-target_link_libraries(hdfs_chgrp tools_common hdfspp_static)
-
-add_executable(hdfs_chown hdfs_chown.cc)
-target_link_libraries(hdfs_chown tools_common hdfspp_static)
-
-add_executable(hdfs_chmod hdfs_chmod.cc)
-target_link_libraries(hdfs_chmod tools_common hdfspp_static)
-
-add_executable(hdfs_find hdfs_find.cc)
-target_link_libraries(hdfs_find tools_common hdfspp_static)
-
-add_executable(hdfs_mkdir hdfs_mkdir.cc)
-target_link_libraries(hdfs_mkdir tools_common hdfspp_static)
-
-add_executable(hdfs_rm hdfs_rm.cc)
-target_link_libraries(hdfs_rm tools_common hdfspp_static)
-
-add_executable(hdfs_ls hdfs_ls.cc)
-target_link_libraries(hdfs_ls tools_common hdfspp_static)
-
-add_executable(hdfs_stat hdfs_stat.cc)
-target_link_libraries(hdfs_stat tools_common hdfspp_static)
-
-add_executable(hdfs_count hdfs_count.cc)
-target_link_libraries(hdfs_count tools_common hdfspp_static)
-
+add_subdirectory(hdfs-chgrp)
+add_subdirectory(hdfs-chown)
+add_subdirectory(hdfs-chmod)
+add_subdirectory(hdfs-find)
+add_subdirectory(hdfs-mkdir)
+add_subdirectory(hdfs-rm)
+add_subdirectory(hdfs-ls)
+add_subdirectory(hdfs-stat)
+add_subdirectory(hdfs-count)
 add_subdirectory(hdfs-df)
-
-add_executable(hdfs_du hdfs_du.cc)
-target_link_libraries(hdfs_du tools_common hdfspp_static)
-
-add_executable(hdfs_get hdfs_get.cc)
-target_link_libraries(hdfs_get tools_common hdfspp_static)
-
-add_executable(hdfs_copyToLocal hdfs_copyToLocal.cc)
-target_link_libraries(hdfs_copyToLocal tools_common hdfspp_static)
-
-add_executable(hdfs_moveToLocal hdfs_moveToLocal.cc)
-target_link_libraries(hdfs_moveToLocal tools_common hdfspp_static)
-
-add_executable(hdfs_setrep hdfs_setrep.cc)
-target_link_libraries(hdfs_setrep tools_common hdfspp_static)
-
+add_subdirectory(hdfs-du)
+add_subdirectory(hdfs-get)
+add_subdirectory(hdfs-copy-to-local)
+add_subdirectory(hdfs-move-to-local)
+add_subdirectory(hdfs-setrep)
 add_subdirectory(hdfs-allow-snapshot)
-
-add_executable(hdfs_disallowSnapshot hdfs_disallowSnapshot.cc)
-target_link_libraries(hdfs_disallowSnapshot tools_common hdfspp_static)
-
-add_executable(hdfs_createSnapshot hdfs_createSnapshot.cc)
-target_link_libraries(hdfs_createSnapshot tools_common hdfspp_static)
-
-add_executable(hdfs_renameSnapshot hdfs_renameSnapshot.cc)
-target_link_libraries(hdfs_renameSnapshot tools_common hdfspp_static)
-
+add_subdirectory(hdfs-disallow-snapshot)
+add_subdirectory(hdfs-create-snapshot)
+add_subdirectory(hdfs-rename-snapshot)
 add_subdirectory(hdfs-delete-snapshot)
-
-add_executable(hdfs_tail hdfs_tail.cc)
-target_link_libraries(hdfs_tail tools_common hdfspp_static)
+add_subdirectory(hdfs-tail)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt
new file mode 100644
index 0000000000000..101365900a404
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_chgrp_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> $<TARGET_OBJECTS:hdfs_ownership_obj> hdfs-chgrp.cc)
+target_include_directories(hdfs_chgrp_lib PRIVATE ../../tools hdfs-chgrp ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_chgrp_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_chgrp main.cc)
+target_include_directories(hdfs_chgrp PRIVATE ../../tools)
+target_link_libraries(hdfs_chgrp PRIVATE hdfs_chgrp_lib)
+
+install(TARGETS hdfs_chgrp RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc
new file mode 100644
index 0000000000000..cbb3b4514b13d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "hdfs-chgrp.h"
+#include "internal/hdfs-ownership.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Chgrp::Chgrp(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Chgrp::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Change the group association of each FILE to GROUP.");
+  add_options("file", po::value<std::string>(),
+              "The path to the file whose group needs to be modified");
+  add_options("recursive,R", "Operate on files and directories recursively");
+  add_options(
+      "group", po::value<std::string>(),
+      "The group to which the file's group association needs to be changed to");
+
+  // An exception is thrown if these arguments are missing or if the arguments'
+  // count doesn't tally.
+  pos_opt_desc_.add("group", 1);
+  pos_opt_desc_.add("file", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool Chgrp::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help");
+  }
+
+  // Rest of the cases must contain more than 2 arguments on the command line
+  return argc_ > 2;
+}
+
+std::string Chgrp ::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_chgrp [OPTION] GROUP FILE" << std::endl
+       << std::endl
+       << "Change the group association of each FILE to GROUP." << std::endl
+       << "The user must be the owner of files. Additional information is in "
+          "the Permissions Guide:"
+       << std::endl
+       << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/"
+          "hadoop-hdfs/HdfsPermissionsGuide.html"
+       << std::endl
+       << std::endl
+       << "  -R  operate on files and directories recursively" << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file"
+       << std::endl
+       << "hdfs_chgrp new_group /dir/file" << std::endl;
+  return desc.str();
+}
+
+bool Chgrp::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS chgrp tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("file") > 0 && opt_val_.count("group") > 0) {
+    const auto file = opt_val_["file"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    const auto group = opt_val_["group"].as<std::string>();
+    return HandlePath(group, recursive, file);
+  }
+
+  return true;
+}
+
+bool Chgrp::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Chgrp::HandlePath(const std::string &group, const bool recursive,
+                       const std::string &file) const {
+  // Building a URI object from the given file
+  auto uri = hdfs::parse_path_or_exit(file);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  // Wrap async FileSystem::SetOwner with promise to make it a blocking call
+  const auto promise = std::make_shared<std::promise<hdfs::Status>>();
+  auto future(promise->get_future());
+  auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); };
+
+  if (!recursive) {
+    fs->SetOwner(uri.get_path(), "", group, handler);
+  } else {
+    /*
+     * Allocating shared state, which includes: username and groupname to be
+     * set, handler to be called, request counter, and a boolean to keep track
+     * if find is done
+     */
+    const auto state =
+        std::make_shared<OwnerState>("", group, handler, 0, false);
+
+    /*
+     * Keep requesting more from Find until we process the entire listing. Call
+     * handler when Find is done and request counter is 0. Find guarantees that
+     * the handler will only be called once at a time so we do not need locking
+     * in handler_find.
+     */
+    auto handler_find = [fs,
+                         state](const hdfs::Status &status_find,
+                                const std::vector<hdfs::StatInfo> &stat_infos,
+                                const bool has_more_results) -> bool {
+      /*
+       * For each result returned by Find we call async SetOwner with the
+       * handler below. SetOwner DOES NOT guarantee that the handler will only
+       * be called once at a time, so we DO need locking in handler_set_owner.
+       */
+      auto handler_set_owner = [state](const hdfs::Status &status_set_owner) {
+        std::lock_guard guard(state->lock);
+
+        // Decrement the counter once since we are done with this async call
+        if (!status_set_owner.ok() && state->status.ok()) {
+          // We make sure we set state->status only on the first error.
+          state->status = status_set_owner;
+        }
+
+        state->request_counter--;
+        if (state->request_counter == 0 && state->find_is_done) {
+          state->handler(state->status); // exit
+        }
+      };
+
+      if (!stat_infos.empty() && state->status.ok()) {
+        for (const auto &s : stat_infos) {
+          // Launch an asynchronous call to SetOwner for every returned result
+          state->request_counter++;
+          fs->SetOwner(s.full_path, state->user, state->group,
+                       handler_set_owner);
+        }
+      }
+
+      /*
+       * Lock this section because handler_set_owner might be accessing the same
+       * shared variables simultaneously.
+       */
+      std::lock_guard guard(state->lock);
+      if (!status_find.ok() && state->status.ok()) {
+        // We make sure we set state->status only on the first error.
+        state->status = status_find;
+      }
+
+      if (!has_more_results) {
+        state->find_is_done = true;
+        if (state->request_counter == 0) {
+          state->handler(state->status); // exit
+        }
+        return false;
+      }
+      return true;
+    };
+
+    // Asynchronous call to Find
+    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+             handler_find);
+  }
+
+  // Block until promise is set
+  const auto status = future.get();
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h
new file mode 100644
index 0000000000000..bd1920ff07492
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP
+#define LIBHDFSPP_TOOLS_HDFS_CHGRP
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Chgrp} is an {@class HdfsTool} that changes the owner and/or group of
+ * each file to owner and/or group.
+ */
+class Chgrp : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Chgrp(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Chgrp(const Chgrp &) = default;
+  Chgrp(Chgrp &&) = default;
+  Chgrp &operator=(const Chgrp &) = delete;
+  Chgrp &operator=(Chgrp &&) = delete;
+  ~Chgrp() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path to the file argument that's passed to this tool.
+   *
+   * @param group The name of the group to which to change to.
+   * @param recursive Whether this operation needs to be performed recursively
+   * on all the files in the given path's sub-directory.
+   * @param file The path to the file whose group needs to be changed.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &group,
+                                        bool recursive,
+                                        const std::string &file) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc
new file mode 100644
index 0000000000000..52f36748b5e46
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-chgrp.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chgrp "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Chgrp chgrp(argc, argv);
+  auto success = false;
+
+  try {
+    success = chgrp.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt
new file mode 100644
index 0000000000000..a1e17f87ebac5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_chmod_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> $<TARGET_OBJECTS:hdfs_ownership_obj> hdfs-chmod.cc)
+target_include_directories(hdfs_chmod_lib PRIVATE ../../tools hdfs-chmod ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_chmod_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_chmod main.cc)
+target_include_directories(hdfs_chmod PRIVATE ../../tools)
+target_link_libraries(hdfs_chmod PRIVATE hdfs_chmod_lib)
+
+install(TARGETS hdfs_chmod RUNTIME DESTINATION bin)
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc
new file mode 100644
index 0000000000000..cd5aefabfcf4b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "hdfs-chmod.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Chmod::Chmod(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Chmod::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Change the permissions of each FILE to MODE.");
+  add_options("file", po::value<std::string>(),
+              "The path to the file whose permissions needs to be modified");
+  add_options("recursive,R", "Operate on files and directories recursively");
+  add_options("permissions", po::value<std::string>(),
+              "Octal representation of the permission bits");
+
+  // An exception is thrown if these arguments are missing or if the arguments'
+  // count doesn't tally.
+  pos_opt_desc_.add("permissions", 1);
+  pos_opt_desc_.add("file", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool Chmod::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help");
+  }
+
+  // Rest of the cases must contain more than 2 arguments on the command line
+  return argc_ > 2;
+}
+
+std::string Chmod ::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_chmod [OPTION] <MODE[,MODE]... | OCTALMODE> FILE"
+       << std::endl
+       << std::endl
+       << "Change the permissions of each FILE to MODE." << std::endl
+       << "The user must be the owner of the file, or else a super-user."
+       << std::endl
+       << "Additional information is in the Permissions Guide:" << std::endl
+       << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/"
+          "hadoop-hdfs/HdfsPermissionsGuide.html"
+       << std::endl
+       << std::endl
+       << "  -R  operate on files and directories recursively" << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file"
+       << std::endl
+       << "hdfs_chmod 777 /dir/file" << std::endl;
+  return desc.str();
+}
+
+bool Chmod::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS chmod tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("file") > 0 && opt_val_.count("permissions") > 0) {
+    const auto file = opt_val_["file"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    const auto permissions = opt_val_["permissions"].as<std::string>();
+    return HandlePath(permissions, recursive, file);
+  }
+
+  return true;
+}
+
+bool Chmod::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Chmod::HandlePath(const std::string &permissions, const bool recursive,
+                       const std::string &file) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(file);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  /*
+   * Wrap async FileSystem::SetPermission with promise to make it a blocking
+   * call.
+   */
+  const auto promise = std::make_shared<std::promise<hdfs::Status>>();
+  auto future(promise->get_future());
+  auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); };
+
+  /*
+   * strtol is reading the value with base 8, NULL because we are reading in
+   * just one value.
+   *
+   * The strtol function may result in errors so check for that before
+   * typecasting.
+   */
+  errno = 0;
+  long result = strtol(permissions.c_str(), nullptr, 8);
+  bool all_0_in_permission = std::all_of(permissions.begin(), permissions.end(),
+                                         [](char c) { return c == '0'; });
+  /*
+   * The errno is set to ERANGE incase the string doesn't fit in long
+   * Also, the result is set to 0, in case conversion is not possible
+   */
+  if ((errno == ERANGE) || (!all_0_in_permission && result == 0))
+    return false;
+  auto perm = static_cast<uint16_t>(result);
+  if (!recursive) {
+    fs->SetPermission(uri.get_path(), perm, handler);
+  } else {
+    /*
+     * Allocating shared state, which includes -
+     * 1. Permissions to be set
+     * 2. Handler to be called
+     * 3. Request counter
+     * 4. A boolean to keep track if find is done
+     */
+    auto state = std::make_shared<PermissionState>(perm, handler, 0, false);
+
+    /*
+     * Keep requesting more from Find until we process the entire listing. Call
+     * handler when Find is done and request counter is 0. Find guarantees that
+     * the handler will only be called once at a time so we do not need locking
+     * in handler_find.
+     */
+    auto handler_find = [fs,
+                         state](const hdfs::Status &status_find,
+                                const std::vector<hdfs::StatInfo> &stat_infos,
+                                const bool has_more_results) -> bool {
+      /*
+       * For each result returned by Find we call async SetPermission with the
+       * handler below. SetPermission DOES NOT guarantee that the handler will
+       * only be called once at a time, so we DO need locking in
+       * handler_set_permission.
+       */
+      auto handler_set_permission =
+          [state](const hdfs::Status &status_set_permission) {
+            std::lock_guard guard(state->lock);
+
+            // Decrement the counter once since we are done with this async call
+            if (!status_set_permission.ok() && state->status.ok()) {
+              // We make sure we set state->status only on the first error.
+              state->status = status_set_permission;
+            }
+            state->request_counter--;
+            if (state->request_counter == 0 && state->find_is_done) {
+              state->handler(state->status); // exit
+            }
+          };
+
+      if (!stat_infos.empty() && state->status.ok()) {
+        for (const auto &s : stat_infos) {
+          /*
+           * Launch an asynchronous call to SetPermission for every returned
+           * result
+           */
+          state->request_counter++;
+          fs->SetPermission(s.full_path, state->permissions,
+                            handler_set_permission);
+        }
+      }
+
+      /*
+       * Lock this section because handler_set_permission might be accessing the
+       * same shared variables simultaneously
+       */
+      std::lock_guard guard(state->lock);
+      if (!status_find.ok() && state->status.ok()) {
+        // We make sure we set state->status only on the first error.
+        state->status = status_find;
+      }
+      if (!has_more_results) {
+        state->find_is_done = true;
+        if (state->request_counter == 0) {
+          state->handler(state->status); // exit
+        }
+        return false;
+      }
+      return true;
+    };
+
+    // Asynchronous call to Find
+    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+             handler_find);
+  }
+
+  // Block until promise is set
+  const auto status = future.get();
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h
new file mode 100644
index 0000000000000..4400625a3b947
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD
+#define LIBHDFSPP_TOOLS_HDFS_CHMOD
+
+#include <functional>
+#include <mutex>
+#include <string>
+#include <utility>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+#include "hdfspp/status.h"
+
+namespace hdfs::tools {
+struct PermissionState {
+  PermissionState(const uint16_t permissions,
+                  std::function<void(const hdfs::Status &)> handler,
+                  const uint64_t request_counter, const bool find_is_done)
+      : permissions(permissions), handler(std::move(handler)),
+        request_counter(request_counter), find_is_done(find_is_done) {}
+
+  const uint16_t permissions;
+  const std::function<void(const hdfs::Status &)> handler;
+
+  /**
+   * The request counter is incremented once every time SetOwner async call is
+   * made
+   */
+  uint64_t request_counter;
+
+  /**
+   * This boolean will be set when find returns the last result
+   */
+  bool find_is_done{false};
+
+  /**
+   * Final status to be returned
+   */
+  hdfs::Status status;
+
+  /**
+   * Shared variables will need protection with a lock
+   */
+  std::mutex lock;
+};
+
+/**
+ * {@class Chmod} is an {@class HdfsTool} that changes the permissions to a
+ * file or folder.
+ */
+class Chmod : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Chmod(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Chmod(const Chmod &) = default;
+  Chmod(Chmod &&) = default;
+  Chmod &operator=(const Chmod &) = delete;
+  Chmod &operator=(Chmod &&) = delete;
+  ~Chmod() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the file to the file argument that's passed to this tool.
+   *
+   * @param permissions An octal representation of the new permissions to be
+   * assigned.
+   * @param recursive Whether this operation needs to be performed recursively
+   * on all the files in the given path's sub-directory.
+   * @param file The path to the file whose ownership needs to be changed.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &permissions,
+                                        bool recursive,
+                                        const std::string &file) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc
new file mode 100644
index 0000000000000..f79be66ccc6aa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-chmod.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chmod "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Chmod chmod(argc, argv);
+  auto success = false;
+
+  try {
+    success = chmod.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt
new file mode 100644
index 0000000000000..6773d8f88683a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_chown_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> $<TARGET_OBJECTS:hdfs_ownership_obj> hdfs-chown.cc)
+target_include_directories(hdfs_chown_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_chown_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_chown main.cc)
+target_include_directories(hdfs_chown PRIVATE ../../tools)
+target_link_libraries(hdfs_chown PRIVATE hdfs_chown_lib)
+
+install(TARGETS hdfs_chown RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc
new file mode 100644
index 0000000000000..41e4ca91bf92e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "hdfs-chown.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Chown::Chown(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Chown::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options(
+      "help,h",
+      "Change the owner and/or group of each FILE to OWNER and/or GROUP.");
+  add_options("file", po::value<std::string>(),
+              "The path to the file whose ownership needs to be modified");
+  add_options("recursive,R", "Operate on files and directories recursively");
+  add_options(
+      "user-group", po::value<std::string>(),
+      "The user:group to which the file's ownership needs to be changed to");
+
+  // An exception is thrown if these arguments are missing or if the arguments'
+  // count doesn't tally.
+  pos_opt_desc_.add("user-group", 1);
+  pos_opt_desc_.add("file", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool Chown::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help");
+  }
+
+  // Rest of the cases must contain more than 2 arguments on the command line
+  return argc_ > 2;
+}
+
+std::string Chown ::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE" << std::endl
+       << std::endl
+       << "Change the owner and/or group of each FILE to OWNER and/or GROUP."
+       << std::endl
+       << "The user must be a super-user. Additional information is in the "
+          "Permissions Guide:"
+       << std::endl
+       << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/"
+          "hadoop-hdfs/HdfsPermissionsGuide.html"
+       << std::endl
+       << std::endl
+       << "  -R  operate on files and directories recursively" << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Owner is unchanged if missing.  Group is unchanged if missing."
+       << std::endl
+       << "OWNER and GROUP may be numeric as well as symbolic." << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_chown -R new_owner:new_group "
+          "hdfs://localhost.localdomain:8020/dir/file"
+       << std::endl
+       << "hdfs_chown new_owner /dir/file" << std::endl;
+  return desc.str();
+}
+
+bool Chown::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS chown tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("file") > 0 && opt_val_.count("user-group") > 0) {
+    const auto file = opt_val_["file"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    const Ownership ownership(opt_val_["user-group"].as<std::string>());
+    return HandlePath(ownership, recursive, file);
+  }
+
+  return true;
+}
+
+bool Chown::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Chown::HandlePath(const Ownership &ownership, const bool recursive,
+                       const std::string &file) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(file);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  // Wrap async FileSystem::SetOwner with promise to make it a blocking call
+  auto promise = std::make_shared<std::promise<hdfs::Status>>();
+  auto future(promise->get_future());
+  auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); };
+
+  if (!recursive) {
+    fs->SetOwner(uri.get_path(), ownership.GetUser(),
+                 ownership.GetGroup().value_or(""), handler);
+  } else {
+    /*
+     * Allocating shared state, which includes: username and groupname to be
+     * set, handler to be called, request counter, and a boolean to keep track
+     * if find is done
+     */
+    auto state = std::make_shared<OwnerState>(ownership.GetUser(),
+                                              ownership.GetGroup().value_or(""),
+                                              handler, 0, false);
+
+    /*
+     * Keep requesting more from Find until we process the entire listing. Call
+     * handler when Find is done and request counter is 0. Find guarantees that
+     * the handler will only be called once at a time so we do not need locking
+     * in handler_find.
+     */
+    auto handler_find = [fs,
+                         state](const hdfs::Status &status_find,
+                                const std::vector<hdfs::StatInfo> &stat_infos,
+                                const bool has_more_results) -> bool {
+      /*
+       * For each result returned by Find we call async SetOwner with the
+       * handler below. SetOwner DOES NOT guarantee that the handler will only
+       * be called once at a time, so we DO need locking in handler_set_owner.
+       */
+      auto handler_set_owner = [state](const hdfs::Status &status_set_owner) {
+        std::lock_guard guard(state->lock);
+
+        // Decrement the counter once since we are done with this async call
+        if (!status_set_owner.ok() && state->status.ok()) {
+          // We make sure we set state->status only on the first error.
+          state->status = status_set_owner;
+        }
+
+        state->request_counter--;
+        if (state->request_counter == 0 && state->find_is_done) {
+          state->handler(state->status); // exit
+        }
+      };
+
+      if (!stat_infos.empty() && state->status.ok()) {
+        for (const auto &s : stat_infos) {
+          // Launch an asynchronous call to SetOwner for every returned result
+          state->request_counter++;
+          fs->SetOwner(s.full_path, state->user, state->group,
+                       handler_set_owner);
+        }
+      }
+
+      /*
+       * Lock this section because handler_set_owner might be accessing the same
+       * shared variables simultaneously.
+       */
+      std::lock_guard guard(state->lock);
+      if (!status_find.ok() && state->status.ok()) {
+        // We make sure we set state->status only on the first error.
+        state->status = status_find;
+      }
+
+      if (!has_more_results) {
+        state->find_is_done = true;
+        if (state->request_counter == 0) {
+          state->handler(state->status); // exit
+        }
+        return false;
+      }
+      return true;
+    };
+
+    // Asynchronous call to Find
+    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+             handler_find);
+  }
+
+  // Block until promise is set
+  const auto status = future.get();
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h
new file mode 100644
index 0000000000000..25774be25ec6e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN
+#define LIBHDFSPP_TOOLS_HDFS_CHOWN
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+#include "internal/hdfs-ownership.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Chown} is an {@class HdfsTool} that changes the owner and/or group of
+ * each file to owner and/or group.
+ */
+class Chown : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Chown(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Chown(const Chown &) = default;
+  Chown(Chown &&) = default;
+  Chown &operator=(const Chown &) = delete;
+  Chown &operator=(Chown &&) = delete;
+  ~Chown() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the file to the file argument that's passed to this tool.
+   *
+   * @param ownership The owner's user and group names.
+   * @param recursive Whether this operation needs to be performed recursively
+   * on all the files in the given path's sub-directory.
+   * @param file The path to the file whose ownership needs to be changed.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const Ownership &ownership,
+                                        bool recursive,
+                                        const std::string &file) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc
new file mode 100644
index 0000000000000..26492c7679ef2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-chown.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chown "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Chown chown(argc, argv);
+  auto success = false;
+
+  try {
+    success = chown.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt
new file mode 100644
index 0000000000000..3546f9b67a74c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_copyToLocal_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-copy-to-local.cc)
+target_include_directories(hdfs_copyToLocal_lib PRIVATE ../../tools hdfs-copyToLocal ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_copyToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_copyToLocal main.cc)
+target_include_directories(hdfs_copyToLocal PRIVATE ../../tools)
+target_link_libraries(hdfs_copyToLocal PRIVATE hdfs_copyToLocal_lib)
+
+install(TARGETS hdfs_copyToLocal RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
new file mode 100644
index 0000000000000..7affa1fbdc92a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-copy-to-local.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+CopyToLocal::CopyToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool CopyToLocal::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Copies the file from the given HDFS source path to "
+                        "the destination path on the local machine");
+
+  add_options("source", po::value<std::string>(), "The HDFS source file path");
+  add_options("target", po::value<std::string>(),
+              "The target local system file path");
+
+  // We allow only two arguments to be passed to this tool. An exception is
+  // thrown if more arguments are passed.
+  pos_opt_desc_.add("source", 1);
+  pos_opt_desc_.add("target", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool CopyToLocal::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help") > 0;
+  }
+
+  // Rest of the cases must contain exactly 2 arguments
+  return argc_ == 3;
+}
+
+std::string CopyToLocal::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_" << GetToolName() << " [OPTION] SRC_FILE DST_FILE"
+       << std::endl
+       << std::endl
+       << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
+       << std::endl
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_" << GetToolName()
+       << " hdfs://localhost.localdomain:8020/dir/file "
+          "/home/usr/myfile"
+       << std::endl
+       << "hdfs_" << GetToolName() << " /dir/file /home/usr/dir/file"
+       << std::endl;
+  return desc.str();
+}
+
+bool CopyToLocal::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS " << GetToolName() << " tool"
+              << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) {
+    const auto source = opt_val_["source"].as<std::string>();
+    const auto target = opt_val_["target"].as<std::string>();
+    return HandlePath(source, target);
+  }
+
+  return false;
+}
+
+bool CopyToLocal::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool CopyToLocal::HandlePath(const std::string &source,
+                             const std::string &target) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(source);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  auto dst_file = std::fopen(target.c_str(), "wb");
+  if (!dst_file) {
+    std::cerr << "Unable to open the destination file: " << target << std::endl;
+    return false;
+  }
+
+  readFile(fs, uri.get_path(), 0, dst_file, false);
+  std::fclose(dst_file);
+  return true;
+}
+
+std::string CopyToLocal::GetToolName() const { return "copyToLocal"; }
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
new file mode 100644
index 0000000000000..0137f1e614d1c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL
+#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class CopyToLocal} is an {@class HdfsTool} that copies the file from the
+ * given HDFS source path to the destination path on the local machine.
+ */
+class CopyToLocal : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CopyToLocal(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  CopyToLocal(const CopyToLocal &) = default;
+  CopyToLocal(CopyToLocal &&) = default;
+  CopyToLocal &operator=(const CopyToLocal &) = delete;
+  CopyToLocal &operator=(CopyToLocal &&) = delete;
+  ~CopyToLocal() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param source The source file path in HDFS.
+   * @param target The target file path on the local machine.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &source,
+                                        const std::string &target) const;
+
+  /**
+   * @return The name of the tool.
+   */
+  [[nodiscard]] virtual std::string GetToolName() const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc
new file mode 100644
index 0000000000000..4dfc4d1c59165
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-copy-to-local.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr
+        << "Error: Unable to schedule clean-up tasks for HDFS copy_to_local tool, exiting"
+        << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::CopyToLocal copy_to_local(argc, argv);
+  auto success = false;
+
+  try {
+    success = copy_to_local.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt
new file mode 100644
index 0000000000000..fbf21fecbe26a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_count_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-count.cc)
+target_include_directories(hdfs_count_lib PRIVATE ../../tools hdfs-count ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_count_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_count main.cc)
+target_include_directories(hdfs_count PRIVATE ../../tools)
+target_link_libraries(hdfs_count PRIVATE hdfs_count_lib)
+
+install(TARGETS hdfs_count RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc
new file mode 100644
index 0000000000000..fca5c9713b6d3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-count.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Count::Count(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Count::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options(
+      "help,h",
+      "Count the number of directories, files and bytes under the given path");
+  add_options("show-quota,q", "Output additional columns before the rest: "
+                              "QUOTA, SPACE_QUOTA, SPACE_CONSUMED");
+  add_options("path", po::value<std::string>(),
+              "The path to the file that needs to be count-ed");
+
+  // We allow only one argument to be passed to this tool. An exception is
+  // thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Count::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_count [OPTION] FILE" << std::endl
+       << std::endl
+       << "Count the number of directories, files and bytes under the path "
+          "that match the specified FILE pattern."
+       << std::endl
+       << "The output columns with -count are: DIR_COUNT, FILE_COUNT, "
+          "CONTENT_SIZE, PATHNAME"
+       << std::endl
+       << std::endl
+       << "  -q    output additional columns before the rest: QUOTA, "
+          "SPACE_QUOTA, SPACE_CONSUMED"
+       << std::endl
+       << "  -h    display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_count hdfs://localhost.localdomain:8020/dir" << std::endl
+       << "hdfs_count -q /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool Count::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS count tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto show_quota = opt_val_.count("show-quota") > 0;
+    return HandlePath(show_quota, path);
+  }
+
+  return false;
+}
+
+bool Count::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Count::HandlePath(const bool show_quota, const std::string &path) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (fs == nullptr) {
+    std::cerr << "Could not connect the file system." << std::endl;
+    return false;
+  }
+
+  hdfs::ContentSummary content_summary;
+  const auto status = fs->GetContentSummary(uri.get_path(), content_summary);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+
+  std::cout << content_summary.str(show_quota) << std::endl;
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h
new file mode 100644
index 0000000000000..473c60967fead
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT
+#define LIBHDFSPP_TOOLS_HDFS_COUNT
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Count} is an {@class HdfsTool} that counts the number of directories,
+ * files and bytes under the given path.
+ */
+class Count : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Count(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Count(const Count &) = default;
+  Count(Count &&) = default;
+  Count &operator=(const Count &) = delete;
+  Count &operator=(Count &&) = delete;
+  ~Count() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param show_quota Output additional columns before the rest: QUOTA,
+   * SPACE_QUOTA, SPACE_CONSUMED.
+   * @param path The path to the directory for which the files, directories and
+   * bytes need to be counted.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(bool show_quota,
+                                        const std::string &path) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc
new file mode 100644
index 0000000000000..807ad3b2c39d0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-count.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS count "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Count count(argc, argv);
+  auto success = false;
+
+  try {
+    success = count.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt
new file mode 100644
index 0000000000000..871fcd8d732c0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_createSnapshot_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-create-snapshot.cc)
+target_include_directories(hdfs_createSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_createSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_createSnapshot main.cc)
+target_include_directories(hdfs_createSnapshot PRIVATE ../../tools)
+target_link_libraries(hdfs_createSnapshot PRIVATE hdfs_createSnapshot_lib)
+
+install(TARGETS hdfs_createSnapshot RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc
new file mode 100644
index 0000000000000..6036e3a76f9f3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc
@@ -0,0 +1,138 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-create-snapshot.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+CreateSnapshot::CreateSnapshot(const int argc, char **argv)
+    : HdfsTool(argc, argv) {}
+
+bool CreateSnapshot::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Show the help for hdfs_createSnapshot");
+  add_options("path", po::value<std::string>(),
+              "The path to the directory for creating the snapshot");
+  add_options("name,n", po::value<std::string>(),
+              "The snapshot name, a default name is selected if omitted");
+
+  // We allow only one argument to be passed to path option. An exception is
+  // thrown if multiple arguments are passed to this.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool CreateSnapshot::ValidateConstraints() const {
+  // If the name option is specified, there will be 4 arguments in total
+  if (argc_ == 4) {
+    return opt_val_.count("name") > 0;
+  }
+
+  // Rest of the cases must contain more than 1 argument on the command line
+  return argc_ > 1;
+}
+
+std::string CreateSnapshot::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_createSnapshot [OPTION] PATH" << std::endl
+       << std::endl
+       << "Create a snapshot of a snapshot-able directory." << std::endl
+       << "This operation requires owner privilege of the snapshot-able "
+          "directory."
+       << std::endl
+       << std::endl
+       << "  -n NAME   The snapshot name. When it is omitted, a default name "
+          "is generated"
+       << std::endl
+       << "             using a timestamp with the format:" << std::endl
+       << R"(             "'s'yyyyMMdd-HHmmss.SSS", e.g. s20130412-151029.033)"
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_createSnapshot hdfs://localhost.localdomain:8020/dir"
+       << std::endl
+       << "hdfs_createSnapshot -n MySnapshot /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool CreateSnapshot::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS create snapshot tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto name = opt_val_.count("name") > 0
+                          ? std::optional{opt_val_["name"].as<std::string>()}
+                          : std::nullopt;
+    return HandleSnapshot(path, name);
+  }
+
+  return true;
+}
+
+bool CreateSnapshot::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool CreateSnapshot::HandleSnapshot(
+    const std::string &path, const std::optional<std::string> &name) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  const auto status = fs->CreateSnapshot(uri.get_path(), name.value_or(""));
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h
new file mode 100644
index 0000000000000..9f1fc09ff2cc3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT
+#define LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT
+
+#include <optional>
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class CreateSnapshot} is an {@class HdfsTool} that facilitates the creation
+ * of the snapshot of a snapshot-able directory located at PATH.
+ */
+class CreateSnapshot : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CreateSnapshot(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  CreateSnapshot(const CreateSnapshot &) = default;
+  CreateSnapshot(CreateSnapshot &&) = default;
+  CreateSnapshot &operator=(const CreateSnapshot &) = delete;
+  CreateSnapshot &operator=(CreateSnapshot &&) = delete;
+  ~CreateSnapshot() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the snapshot-able directory for which the snapshot
+   * needs to be created.
+   * @param name The name to assign to the snapshot after creating it.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool
+  HandleSnapshot(const std::string &path,
+                 const std::optional<std::string> &name = std::nullopt) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc
new file mode 100644
index 0000000000000..7801bb4c60475
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc
@@ -0,0 +1,54 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-create-snapshot.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS create "
+                 "snapshot tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::CreateSnapshot create_snapshot(argc, argv);
+  auto success = false;
+
+  try {
+    success = create_snapshot.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+    success = false;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt
new file mode 100644
index 0000000000000..263fc3adb9683
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_disallowSnapshot_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-disallow-snapshot.cc)
+target_include_directories(hdfs_disallowSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_disallowSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_disallowSnapshot main.cc)
+target_include_directories(hdfs_disallowSnapshot PRIVATE ../../tools)
+target_link_libraries(hdfs_disallowSnapshot PRIVATE hdfs_disallowSnapshot_lib)
+
+install(TARGETS hdfs_disallowSnapshot RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc
new file mode 100644
index 0000000000000..576c1bec30fa7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc
@@ -0,0 +1,117 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-disallow-snapshot.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+DisallowSnapshot::DisallowSnapshot(const int argc, char **argv)
+    : HdfsTool(argc, argv) {}
+
+bool DisallowSnapshot::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Show the help for hdfs_disallowSnapshot");
+  add_options(
+      "path", po::value<std::string>(),
+      "The path to the directory for which snapshot-ing must be disallowed");
+
+  // We allow only one argument to be passed to this tool. An exception is
+  // thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string DisallowSnapshot::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_disallowSnapshot [OPTION] PATH" << std::endl
+       << std::endl
+       << "Disallowing snapshots of a directory at PATH to be created."
+       << std::endl
+       << "All snapshots of the directory must be deleted before disallowing "
+          "snapshots."
+       << std::endl
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_disallowSnapshot hdfs://localhost.localdomain:8020/dir"
+       << std::endl
+       << "hdfs_disallowSnapshot /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool DisallowSnapshot::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS allow snapshot tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    return HandleSnapshot(path);
+  }
+
+  return true;
+}
+
+bool DisallowSnapshot::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool DisallowSnapshot::HandleSnapshot(const std::string &path) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  const auto status = fs->DisallowSnapshot(uri.get_path());
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h
new file mode 100644
index 0000000000000..b4174873cff1c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT
+#define LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class DisallowSnapshot} is an {@class HdfsTool} that disallows the
+ * snapshots from being created for a directory. All the existing snapshots must
+ * be deleted first.
+ */
+class DisallowSnapshot : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  DisallowSnapshot(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  DisallowSnapshot(const DisallowSnapshot &) = default;
+  DisallowSnapshot(DisallowSnapshot &&) = default;
+  DisallowSnapshot &operator=(const DisallowSnapshot &) = delete;
+  DisallowSnapshot &operator=(DisallowSnapshot &&) = delete;
+  ~DisallowSnapshot() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory for which snapshot-ing must be
+   * disallowed.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandleSnapshot(const std::string &path) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc
new file mode 100644
index 0000000000000..07009200f3994
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc
@@ -0,0 +1,54 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-disallow-snapshot.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS disallow "
+                 "snapshot tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::DisallowSnapshot disallow_snapshot(argc, argv);
+  auto success = false;
+
+  try {
+    success = disallow_snapshot.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+    success = false;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists-gtest.txt.in b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt
similarity index 63%
rename from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists-gtest.txt.in
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt
index 10a414588d59f..7164136fa10d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists-gtest.txt.in
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/CMakeLists.txt
@@ -16,18 +16,12 @@
 # limitations under the License.
 #
 
-cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+add_library(hdfs_du_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-du.cc)
+target_include_directories(hdfs_du_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_du_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
 
-project(googletest-download NONE)
+add_executable(hdfs_du main.cc)
+target_include_directories(hdfs_du PRIVATE ../../tools)
+target_link_libraries(hdfs_du PRIVATE hdfs_du_lib)
 
-include(ExternalProject)
-ExternalProject_Add(googletest
-  GIT_REPOSITORY    https://github.com/google/googletest.git
-  GIT_TAG           release-1.10.0
-  SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/googletest-src"
-  BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/googletest-build"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND     ""
-  INSTALL_COMMAND   ""
-  TEST_COMMAND      ""
-)
+install(TARGETS hdfs_du RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc
new file mode 100644
index 0000000000000..5b5cab67d060b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.cc
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-du.h"
+#include "internal/get-content-summary-state.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Du::Du(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Du::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h",
+              "Displays sizes of files and directories contained in the given "
+              "PATH or the length of a file in case PATH is just a file");
+  add_options("recursive,R", "Operate on files and directories recursively");
+  add_options("path", po::value<std::string>(),
+              "The path indicating the filesystem that needs to be du-ed");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Du::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_du [OPTION] PATH" << std::endl
+       << std::endl
+       << "Displays sizes of files and directories contained in the given PATH"
+       << std::endl
+       << "or the length of a file in case PATH is just a file" << std::endl
+       << std::endl
+       << "  -R        operate on files and directories recursively"
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_du hdfs://localhost.localdomain:8020/dir/file" << std::endl
+       << "hdfs_du -R /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool Du::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS du tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    return HandlePath(path, recursive);
+  }
+
+  return false;
+}
+
+bool Du::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Du::HandlePath(const std::string &path, const bool recursive) const {
+  // Building a URI object from the given path.
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect to the file system." << std::endl;
+    return false;
+  }
+
+  /*
+   * Wrap async FileSystem::GetContentSummary with promise to make it a blocking
+   * call.
+   */
+  const auto promise = std::make_shared<std::promise<hdfs::Status>>();
+  std::future future(promise->get_future());
+  auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); };
+
+  /*
+   * Allocating shared state, which includes: handler to be called, request
+   * counter, and a boolean to keep track if find is done.
+   */
+  const auto state =
+      std::make_shared<GetContentSummaryState>(handler, 0, false);
+
+  /*
+   * Keep requesting more from Find until we process the entire listing. Call
+   * handler when Find is done and request counter is 0. Find guarantees that
+   * the handler will only be called once at a time so we do not need locking in
+   * handler_find.
+   */
+  auto handler_find = [fs, state](const hdfs::Status &status_find,
+                                  const std::vector<hdfs::StatInfo> &stat_infos,
+                                  const bool has_more_results) -> bool {
+    /*
+     * For each result returned by Find we call async GetContentSummary with the
+     * handler below. GetContentSummary DOES NOT guarantee that the handler will
+     * only be called once at a time, so we DO need locking in
+     * handler_get_content_summary.
+     */
+    auto handler_get_content_summary =
+        [state](const hdfs::Status &status_get_summary,
+                const hdfs::ContentSummary &si) {
+          std::lock_guard guard(state->lock);
+          std::cout << si.str_du() << std::endl;
+          // Decrement the counter once since we are done with this async call.
+          if (!status_get_summary.ok() && state->status.ok()) {
+            // We make sure we set state->status only on the first error.
+            state->status = status_get_summary;
+          }
+          state->request_counter--;
+          if (state->request_counter == 0 && state->find_is_done) {
+            state->handler(state->status); // exit
+          }
+        };
+
+    if (!stat_infos.empty() && state->status.ok()) {
+      for (hdfs::StatInfo const &s : stat_infos) {
+        /*
+         * Launch an asynchronous call to GetContentSummary for every returned
+         * result.
+         */
+        state->request_counter++;
+        fs->GetContentSummary(s.full_path, handler_get_content_summary);
+      }
+    }
+
+    /*
+     * Lock this section because handler_get_content_summary might be accessing
+     * the same shared variables simultaneously.
+     */
+    std::lock_guard guard(state->lock);
+    if (!status_find.ok() && state->status.ok()) {
+      // We make sure we set state->status only on the first error.
+      state->status = status_find;
+    }
+
+    if (!has_more_results) {
+      state->find_is_done = true;
+      if (state->request_counter == 0) {
+        state->handler(state->status); // exit
+      }
+      return false;
+    }
+    return true;
+  };
+
+  // Asynchronous call to Find.
+  if (!recursive) {
+    fs->GetListing(uri.get_path(), handler_find);
+  } else {
+    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+             handler_find);
+  }
+
+  // Block until promise is set.
+  const auto status = future.get();
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h
new file mode 100644
index 0000000000000..494270868eec6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/hdfs-du.h
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_DU
+#define LIBHDFSPP_TOOLS_HDFS_DU
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Du} is an {@class HdfsTool} that displays the size of the directories
+ * and files.
+ */
+class Du : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Du(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Du(const Du &) = default;
+  Du(Du &&) = default;
+  Du &operator=(const Du &) = delete;
+  Du &operator=(Du &&) = delete;
+  ~Du() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory for which we need du info.
+   * @param recursive A boolean indicating whether du needs to be
+   * performed recursively for the given path.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &path,
+                                        bool recursive) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc
new file mode 100644
index 0000000000000..0d8738fe3d1f7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-du/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-du.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr
+        << "Error: Unable to schedule clean-up tasks for HDFS df tool, exiting"
+        << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Du du(argc, argv);
+  auto success = false;
+
+  try {
+    success = du.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt
new file mode 100644
index 0000000000000..c6ce02132875a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_find_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-find.cc)
+target_include_directories(hdfs_find_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_find_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_find main.cc)
+target_include_directories(hdfs_find PRIVATE ../../tools)
+target_link_libraries(hdfs_find PRIVATE hdfs_find_lib)
+
+install(TARGETS hdfs_find RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc
new file mode 100644
index 0000000000000..ee1a04019f295
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.cc
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdint>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-find.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Find::Find(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Find::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options(
+      "help,h",
+      "Finds all files recursively starting from the specified PATH and prints "
+      "their file paths. This hdfs_find tool mimics the POSIX find.");
+  add_options(
+      "name,n", po::value<std::string>(),
+      "If provided, all results will be matching the NAME pattern otherwise, "
+      "the implicit '*' will be used NAME allows wild-cards");
+  add_options(
+      "max-depth,m", po::value<uint32_t>(),
+      "If provided, the maximum depth to recurse after the end of the path is "
+      "reached will be limited by MAX_DEPTH otherwise, the maximum depth to "
+      "recurse is unbound MAX_DEPTH can be set to 0 for pure globbing and "
+      "ignoring the NAME option (no recursion after the end of the path)");
+  add_options("path", po::value<std::string>(),
+              "The path where we want to start the find operation");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Find::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_find [OPTION] PATH" << std::endl
+       << std::endl
+       << "Finds all files recursively starting from the" << std::endl
+       << "specified PATH and prints their file paths." << std::endl
+       << "This hdfs_find tool mimics the POSIX find." << std::endl
+       << std::endl
+       << "Both PATH and NAME can have wild-cards." << std::endl
+       << std::endl
+       << "  -n NAME       if provided all results will be matching the NAME "
+          "pattern"
+       << std::endl
+       << "                otherwise, the implicit '*' will be used"
+       << std::endl
+       << "                NAME allows wild-cards" << std::endl
+       << std::endl
+       << "  -m MAX_DEPTH  if provided the maximum depth to recurse after the "
+          "end of"
+       << std::endl
+       << "                the path is reached will be limited by MAX_DEPTH"
+       << std::endl
+       << "                otherwise, the maximum depth to recurse is unbound"
+       << std::endl
+       << "                MAX_DEPTH can be set to 0 for pure globbing and "
+          "ignoring"
+       << std::endl
+       << "                the NAME option (no recursion after the end of the "
+          "path)"
+       << std::endl
+       << std::endl
+       << "  -h            display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_find hdfs://localhost.localdomain:8020/dir?/tree* -n "
+          "some?file*name"
+       << std::endl
+       << "hdfs_find / -n file_name -m 3" << std::endl;
+  return desc.str();
+}
+
+bool Find::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS find tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto name =
+        opt_val_.count("name") > 0 ? opt_val_["name"].as<std::string>() : "*";
+    const auto max_depth = opt_val_.count("max-depth") <= 0
+                               ? hdfs::FileSystem::GetDefaultFindMaxDepth()
+                               : opt_val_["max-depth"].as<uint32_t>();
+    return HandlePath(path, name, max_depth);
+  }
+
+  return false;
+}
+
+bool Find::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Find::HandlePath(const std::string &path, const std::string &name,
+                      const uint32_t max_depth) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system." << std::endl;
+    return false;
+  }
+
+  const auto promise = std::make_shared<std::promise<void>>();
+  std::future<void> future(promise->get_future());
+  auto final_status = hdfs::Status::OK();
+
+  /**
+   * Keep requesting more until we get the entire listing. Set the promise
+   * when we have the entire listing to stop.
+   *
+   * Find guarantees that the handler will only be called once at a time,
+   * so we do not need any locking here. It also guarantees that the handler
+   * will be only called once with has_more_results set to false.
+   */
+  auto handler = [promise,
+                  &final_status](const hdfs::Status &status,
+                                 const std::vector<hdfs::StatInfo> &stat_info,
+                                 const bool has_more_results) -> bool {
+    // Print result chunks as they arrive
+    if (!stat_info.empty()) {
+      for (hdfs::StatInfo const &info : stat_info) {
+        std::cout << info.str() << std::endl;
+      }
+    }
+    if (!status.ok() && final_status.ok()) {
+      // We make sure we set 'status' only on the first error
+      final_status = status;
+    }
+    if (!has_more_results) {
+      promise->set_value(); // Set promise
+      return false;         // Request stop sending results
+    }
+    return true; // request more results
+  };
+
+  // Asynchronous call to Find
+  fs->Find(uri.get_path(), name, max_depth, handler);
+
+  // Block until promise is set
+  future.get();
+  if (!final_status.ok()) {
+    std::cerr << "Error: " << final_status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h
new file mode 100644
index 0000000000000..9adde3c622bf1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/hdfs-find.h
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_FIND
+#define LIBHDFSPP_TOOLS_HDFS_FIND
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Find} is an {@class HdfsTool} finds all files recursively starting
+ * from the specified PATH and prints their file paths. This tool mimics the
+ * POSIX find.
+ */
+class Find : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Find(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Find(const Find &) = default;
+  Find(Find &&) = default;
+  Find &operator=(const Find &) = delete;
+  Find &operator=(Find &&) = delete;
+  ~Find() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory to begin the find.
+   * @param name The pattern name of the search term.
+   * @param max_depth The maximum depth of the traversal while searching through
+   * the folders.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &path,
+                                        const std::string &name,
+                                        uint32_t max_depth) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc
new file mode 100644
index 0000000000000..1f63aa7a2d8e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-find/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-find.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr
+        << "Error: Unable to schedule clean-up tasks for HDFS find tool, exiting"
+        << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Find find(argc, argv);
+  auto success = false;
+
+  try {
+    success = find.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt
new file mode 100644
index 0000000000000..367bca6a0279c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_get_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-get.cc)
+target_include_directories(hdfs_get_lib PRIVATE ../../tools hdfs-get ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_get_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static hdfs_copyToLocal_lib)
+
+add_executable(hdfs_get main.cc)
+target_include_directories(hdfs_get PRIVATE ../../tools)
+target_link_libraries(hdfs_get PRIVATE hdfs_get_lib)
+
+install(TARGETS hdfs_get RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc
new file mode 100644
index 0000000000000..e45c1d0215b26
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs-get.h"
+
+namespace hdfs::tools {
+Get::Get(const int argc, char **argv) : CopyToLocal(argc, argv) {}
+
+std::string Get::GetToolName() const { return "get"; }
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h
new file mode 100644
index 0000000000000..8153264a72a3e
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_GET
+#define LIBHDFSPP_TOOLS_HDFS_GET
+
+#include "hdfs-copy-to-local/hdfs-copy-to-local.h"
+
+namespace hdfs::tools {
+class Get : public CopyToLocal {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Get(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Get(const Get &) = default;
+  Get(Get &&) = default;
+  Get &operator=(const Get &) = delete;
+  Get &operator=(Get &&) = delete;
+  ~Get() override = default;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetToolName() const override;
+};
+} // namespace hdfs::tools
+
+#endif
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc
similarity index 52%
rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc
index eda1d1fee402e..916e60a4631c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpRequestLogAppender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc
@@ -15,48 +15,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.http;
 
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.log4j.AppenderSkeleton;
+#include <cstdlib>
+#include <exception>
+#include <iostream>
 
-/**
- * Log4j Appender adapter for HttpRequestLog
- */
-public class HttpRequestLogAppender extends AppenderSkeleton {
-
-  private String filename;
-  private int retainDays;
+#include <google/protobuf/stubs/common.h>
 
-  public HttpRequestLogAppender() {
-  }
-
-  public void setRetainDays(int retainDays) {
-    this.retainDays = retainDays;
-  }
+#include "hdfs-get.h"
 
-  public int getRetainDays() {
-    return retainDays;
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS "
+                 "get tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
   }
 
-  public void setFilename(String filename) {
-    this.filename = filename;
-  }
-
-  public String getFilename() {
-    return filename;
-  }
-
-  @Override
-  public void append(LoggingEvent event) {
-  }
+  hdfs::tools::Get get(argc, argv);
+  auto success = false;
 
-  @Override
-  public void close() {
+  try {
+    success = get.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
   }
 
-  @Override
-  public boolean requiresLayout() {
-    return false;
+  if (!success) {
+    std::exit(EXIT_FAILURE);
   }
+  return 0;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt
new file mode 100644
index 0000000000000..196d6356e2ee2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_ls_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-ls.cc)
+target_include_directories(hdfs_ls_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_ls_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_ls main.cc)
+target_include_directories(hdfs_ls PRIVATE ../../tools)
+target_link_libraries(hdfs_ls PRIVATE hdfs_ls_lib)
+
+install(TARGETS hdfs_ls RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc
new file mode 100644
index 0000000000000..8367bbc71feee
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.cc
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-ls.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Ls::Ls(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Ls::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "List information about the files");
+  add_options("recursive,R", "Operate on files and directories recursively");
+  add_options("path", po::value<std::string>(),
+              "The path for which we need to do ls");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Ls::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_ls [OPTION] FILE" << std::endl
+       << std::endl
+       << "List information about the FILEs." << std::endl
+       << std::endl
+       << "  -R        list subdirectories recursively" << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_ls hdfs://localhost.localdomain:8020/dir" << std::endl
+       << "hdfs_ls -R /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool Ls::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS ls tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    return HandlePath(path, recursive);
+  }
+
+  return false;
+}
+
+bool Ls::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Ls::HandlePath(const std::string &path, const bool recursive) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  const auto promise = std::make_shared<std::promise<void>>();
+  auto future(promise->get_future());
+  auto result = hdfs::Status::OK();
+
+  /*
+   * Keep requesting more until we get the entire listing. Set the promise
+   * when we have the entire listing to stop.
+   *
+   * Find and GetListing guarantee that the handler will only be called once at
+   * a time, so we do not need any locking here. They also guarantee that the
+   * handler will be only called once with has_more_results set to false.
+   */
+  auto handler = [promise,
+                  &result](const hdfs::Status &status,
+                           const std::vector<hdfs::StatInfo> &stat_info,
+                           const bool has_more_results) -> bool {
+    // Print result chunks as they arrive
+    if (!stat_info.empty()) {
+      for (const auto &info : stat_info) {
+        std::cout << info.str() << std::endl;
+      }
+    }
+    if (!status.ok() && result.ok()) {
+      // We make sure we set the result only on the first error
+      result = status;
+    }
+    if (!has_more_results) {
+      promise->set_value(); // Set promise
+      return false;         // Request to stop sending results
+    }
+    return true; // Request more results
+  };
+
+  if (!recursive) {
+    // Asynchronous call to GetListing
+    fs->GetListing(uri.get_path(), handler);
+  } else {
+    // Asynchronous call to Find
+    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+             handler);
+  }
+
+  // Block until promise is set
+  future.get();
+  if (!result.ok()) {
+    std::cerr << "Error: " << result.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h
new file mode 100644
index 0000000000000..c38731b67a5fe
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/hdfs-ls.h
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_LS
+#define LIBHDFSPP_TOOLS_HDFS_LS
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Ls} is an {@class HdfsTool} that lists information about the files.
+ */
+class Ls : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Ls(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Ls(const Ls &) = default;
+  Ls(Ls &&) = default;
+  Ls &operator=(const Ls &) = delete;
+  Ls &operator=(Ls &&) = delete;
+  ~Ls() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory for which we need to ls.
+   * @param recursive A boolean indicating whether ls needs to be
+   * performed recursively for the given path.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &path,
+                                        bool recursive) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc
similarity index 52%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java
rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc
index b471f218e57e1..5d5312f492d45 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-ls/main.cc
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,31 +15,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.fs.swift.http;
 
-/**
- * Response tuple from GET operations; combines the input stream with the content length
- */
-public class HttpBodyContent {
-  private final long contentLength;
-  private final HttpInputStreamWithRelease inputStream;
+#include <cstdlib>
+#include <exception>
+#include <iostream>
 
-  /**
-   * build a body response
-   * @param inputStream input stream from the operation
-   * @param contentLength length of content; may be -1 for "don't know"
-   */
-  public HttpBodyContent(HttpInputStreamWithRelease inputStream,
-                         long contentLength) {
-    this.contentLength = contentLength;
-    this.inputStream = inputStream;
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-ls.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS ls tool, "
+                 "exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
   }
 
-  public long getContentLength() {
-    return contentLength;
+  hdfs::tools::Ls ls(argc, argv);
+  auto success = false;
+
+  try {
+    success = ls.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
   }
 
-  public HttpInputStreamWithRelease getInputStream() {
-    return inputStream;
+  if (!success) {
+    std::exit(EXIT_FAILURE);
   }
+  return 0;
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt
new file mode 100644
index 0000000000000..5efce8de86be0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_mkdir_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-mkdir.cc)
+target_include_directories(hdfs_mkdir_lib PRIVATE ../../tools hdfs-mkdir ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_mkdir_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_mkdir main.cc)
+target_include_directories(hdfs_mkdir PRIVATE ../../tools)
+target_link_libraries(hdfs_mkdir PRIVATE hdfs_mkdir_lib)
+
+install(TARGETS hdfs_mkdir RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc
new file mode 100644
index 0000000000000..9c406fb959fc0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-mkdir.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Mkdir::Mkdir(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Mkdir::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Create directory if it does not exist");
+  add_options("create-parents,p", "Create parent directories as needed");
+  add_options(
+      "mode,m", po::value<std::string>(),
+      "Set the permissions for the new directory (and newly created parents if "
+      "any). The permissions are specified in octal representation");
+  add_options("path", po::value<std::string>(),
+              "The path to the directory that needs to be created");
+
+  // We allow only one argument to be passed to this tool. An exception is
+  // thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Mkdir::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_mkdir [OPTION] DIRECTORY" << std::endl
+       << std::endl
+       << "Create the DIRECTORY(ies), if they do not already exist."
+       << std::endl
+       << std::endl
+       << "  -p        make parent directories as needed" << std::endl
+       << "  -m  MODE  set file mode (octal permissions) for the new "
+          "DIRECTORY(ies)"
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2" << std::endl
+       << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir"
+       << std::endl;
+  return desc.str();
+}
+
+bool Mkdir::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS mkdir tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto create_parents = opt_val_.count("create-parents") > 0;
+    const auto permissions =
+        opt_val_.count("mode") > 0
+            ? std::optional(opt_val_["mode"].as<std::string>())
+            : std::nullopt;
+    return HandlePath(create_parents, permissions, path);
+  }
+
+  return false;
+}
+
+bool Mkdir::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Mkdir::HandlePath(const bool create_parents,
+                       const std::optional<std::string> &permissions,
+                       const std::string &path) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (fs == nullptr) {
+    std::cerr << "Could not connect the file system." << std::endl;
+    return false;
+  }
+
+  const auto status =
+      fs->Mkdirs(uri.get_path(), GetPermissions(permissions), create_parents);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+
+  return true;
+}
+
+uint16_t Mkdir::GetPermissions(const std::optional<std::string> &permissions) {
+  if (permissions) {
+    // TODO : Handle the error returned by std::strtol.
+    return static_cast<uint16_t>(
+        std::strtol(permissions.value().c_str(), nullptr, 8));
+  }
+
+  return hdfs::FileSystem::GetDefaultPermissionMask();
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h
new file mode 100644
index 0000000000000..60875f3c7af7c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR
+#define LIBHDFSPP_TOOLS_HDFS_MKDIR
+
+#include <optional>
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Mkdir} is an {@class HdfsTool} that creates directory if it does not
+ * exist.
+ */
+class Mkdir : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Mkdir(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Mkdir(const Mkdir &) = default;
+  Mkdir(Mkdir &&) = default;
+  Mkdir &operator=(const Mkdir &) = delete;
+  Mkdir &operator=(Mkdir &&) = delete;
+  ~Mkdir() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param create_parents Creates parent directories as needed if this boolean
+   * is set to true.
+   * @param permissions An octal representation of the permissions to be stamped
+   * to each directory that gets created.
+   * @param path The path in the filesystem where the directory must be created.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool
+  HandlePath(bool create_parents, const std::optional<std::string> &permissions,
+             const std::string &path) const;
+
+  /**
+   * @param permissions The permissions string to convert to octal value.
+   * @return The octal representation of the permissions supplied as parameter
+   * to this tool.
+   */
+  [[nodiscard]] static uint16_t
+  GetPermissions(const std::optional<std::string> &permissions);
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc
new file mode 100644
index 0000000000000..3f70ef6dd5772
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-mkdir.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS mkdir "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Mkdir mkdir(argc, argv);
+  auto success = false;
+
+  try {
+    success = mkdir.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt
new file mode 100644
index 0000000000000..bd3486d472524
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_moveToLocal_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-move-to-local.cc)
+target_include_directories(hdfs_moveToLocal_lib PRIVATE ../../tools hdfs-moveToLocal ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_moveToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_moveToLocal main.cc)
+target_include_directories(hdfs_moveToLocal PRIVATE ../../tools)
+target_link_libraries(hdfs_moveToLocal PRIVATE hdfs_moveToLocal_lib)
+
+install(TARGETS hdfs_moveToLocal RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc
new file mode 100644
index 0000000000000..9aeecea7b6e87
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-move-to-local.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+MoveToLocal::MoveToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool MoveToLocal::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Moves the file from the given HDFS source path to "
+                        "the destination path on the local machine");
+
+  add_options("source", po::value<std::string>(), "The HDFS source file path");
+  add_options("target", po::value<std::string>(),
+              "The target local system file path");
+
+  // We allow only two arguments to be passed to this tool. An exception is
+  // thrown if more arguments are passed.
+  pos_opt_desc_.add("source", 1);
+  pos_opt_desc_.add("target", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool MoveToLocal::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help") > 0;
+  }
+
+  // Rest of the cases must contain exactly 2 arguments
+  return argc_ == 3;
+}
+
+std::string MoveToLocal::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE" << std::endl
+       << std::endl
+       << "Move SRC_FILE from hdfs to DST_FILE on the local file system."
+       << std::endl
+       << "Moving is done by copying SRC_FILE to DST_FILE, and then"
+       << std::endl
+       << "deleting DST_FILE if copy succeeded." << std::endl
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file "
+          "/home/usr/myfile"
+       << std::endl
+       << "hdfs_moveToLocal /dir/file /home/usr/dir/file" << std::endl;
+  return desc.str();
+}
+
+bool MoveToLocal::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS moveToLocal tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) {
+    const auto source = opt_val_["source"].as<std::string>();
+    const auto target = opt_val_["target"].as<std::string>();
+    return HandlePath(source, target);
+  }
+
+  return false;
+}
+
+bool MoveToLocal::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool MoveToLocal::HandlePath(const std::string &source,
+                             const std::string &target) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(source);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  auto dst_file = std::fopen(target.c_str(), "wb");
+  if (!dst_file) {
+    std::cerr << "Unable to open the destination file: " << target << std::endl;
+    return false;
+  }
+
+  readFile(fs, uri.get_path(), 0, dst_file, true);
+  std::fclose(dst_file);
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h
new file mode 100644
index 0000000000000..982a67914fa8b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL
+#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class MoveToLocal} is an {@class HdfsTool} that copies the file from the
+ * given HDFS source path to the destination path on the local machine.
+ */
+class MoveToLocal : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  MoveToLocal(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  MoveToLocal(const MoveToLocal &) = default;
+  MoveToLocal(MoveToLocal &&) = default;
+  MoveToLocal &operator=(const MoveToLocal &) = delete;
+  MoveToLocal &operator=(MoveToLocal &&) = delete;
+  ~MoveToLocal() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param source The source file path in HDFS.
+   * @param target The target file path on the local machine.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &source,
+                                        const std::string &target) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc
new file mode 100644
index 0000000000000..2d13464185934
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-move-to-local.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS "
+                 "move_to_local tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::MoveToLocal move_to_local(argc, argv);
+  auto success = false;
+
+  try {
+    success = move_to_local.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt
new file mode 100644
index 0000000000000..44d9447eac791
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_renameSnapshot_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-rename-snapshot.cc)
+target_include_directories(hdfs_renameSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_renameSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_renameSnapshot main.cc)
+target_include_directories(hdfs_renameSnapshot PRIVATE ../../tools)
+target_link_libraries(hdfs_renameSnapshot PRIVATE hdfs_renameSnapshot_lib)
+
+install(TARGETS hdfs_renameSnapshot RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc
new file mode 100644
index 0000000000000..80858c5471b4f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc
@@ -0,0 +1,139 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-rename-snapshot.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+RenameSnapshot::RenameSnapshot(const int argc, char **argv)
+    : HdfsTool(argc, argv) {}
+
+bool RenameSnapshot::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Show the help for hdfs_renameSnapshot");
+  add_options("path", po::value<std::string>(),
+              "The path to the directory that is snapshot-able");
+  add_options("old-name", po::value<std::string>(),
+              "The old/current name of the snapshot");
+  add_options("new-name", po::value<std::string>(),
+              "The new name for the snapshot");
+
+  // Register "path", "old-name" and "new-name" as positional arguments.
+  // We allow only three arguments to be passed to this tool. An exception is
+  // thrown if more than three arguments are passed.
+  pos_opt_desc_.add("path", 1);
+  pos_opt_desc_.add("old-name", 1);
+  pos_opt_desc_.add("new-name", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool RenameSnapshot::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help") > 0;
+  }
+
+  // Rest of the cases must contain more than 2 argument on the command line
+  return argc_ > 3;
+}
+
+std::string RenameSnapshot::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_renameSnapshot [OPTION] PATH OLD_NAME NEW_NAME"
+       << std::endl
+       << std::endl
+       << "Rename a snapshot from OLD_NAME to NEW_NAME." << std::endl
+       << "This operation requires owner privilege of the snapshot-able "
+          "directory."
+       << std::endl
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_renameSnapshot hdfs://localhost.localdomain:8020/dir oldDir "
+          "newDir"
+       << std::endl
+       << "hdfs_renameSnapshot /dir1/dir2 oldSnap newSnap" << std::endl;
+  return desc.str();
+}
+
+bool RenameSnapshot::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS rename snapshot tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0 && opt_val_.count("old-name") > 0 &&
+      opt_val_.count("new-name") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto old_name = opt_val_["old-name"].as<std::string>();
+    const auto new_name = opt_val_["new-name"].as<std::string>();
+    return HandleSnapshot(path, old_name, new_name);
+  }
+
+  return true;
+}
+
+bool RenameSnapshot::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool RenameSnapshot::HandleSnapshot(const std::string &path,
+                                    const std::string &old_name,
+                                    const std::string &new_name) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  const auto status = fs->RenameSnapshot(uri.get_path(), old_name, new_name);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h
new file mode 100644
index 0000000000000..5133999b87f14
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT
+#define LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class RenameSnapshot} is an {@class HdfsTool} that facilitates the
+ * renaming of the snapshots.
+ */
+class RenameSnapshot : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  RenameSnapshot(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  RenameSnapshot(const RenameSnapshot &) = default;
+  RenameSnapshot(RenameSnapshot &&) = default;
+  RenameSnapshot &operator=(const RenameSnapshot &) = delete;
+  RenameSnapshot &operator=(RenameSnapshot &&) = delete;
+  ~RenameSnapshot() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the arguments that are passed to this tool.
+   *
+   * @param path The path to the directory that is snapshot-able.
+   * @param old_name The name of the current/older snapshot.
+   * @param new_name The new name for the old snapshot.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandleSnapshot(const std::string &path,
+                                            const std::string &old_name,
+                                            const std::string &new_name) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc
new file mode 100644
index 0000000000000..c52425411d382
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc
@@ -0,0 +1,54 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+*/
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-rename-snapshot.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS rename "
+                 "snapshot tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::RenameSnapshot rename_snapshot(argc, argv);
+  auto success = false;
+
+  try {
+    success = rename_snapshot.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+    success = false;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt
new file mode 100644
index 0000000000000..70a7db5167463
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_rm_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-rm.cc)
+target_include_directories(hdfs_rm_lib PRIVATE ../../tools hdfs-rm ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_rm_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_rm main.cc)
+target_include_directories(hdfs_rm PRIVATE ../../tools)
+target_link_libraries(hdfs_rm PRIVATE hdfs_rm_lib)
+
+install(TARGETS hdfs_rm RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc
new file mode 100644
index 0000000000000..ef1a85649d257
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-rm.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Rm::Rm(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Rm::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Remove/unlink the files or directories.");
+  add_options("recursive,R",
+              "Remove the directories and their contents recursively.");
+  add_options("path", po::value<std::string>(),
+              "The path to the file that needs to be removed.");
+
+  // We allow only one argument to be passed to this tool. An exception is
+  // thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Rm::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_rm [OPTION] FILE" << std::endl
+       << std::endl
+       << "Remove (unlink) the FILE(s) or directory(ies)." << std::endl
+       << std::endl
+       << "  -R        remove directories and their contents recursively"
+       << std::endl
+       << "  -h        display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" << std::endl
+       << "hdfs_rm -R /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool Rm::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS rm tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto recursive = opt_val_.count("recursive") > 0;
+    return HandlePath(recursive, path);
+  }
+
+  return false;
+}
+
+bool Rm::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Rm::HandlePath(const bool recursive, const std::string &path) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (fs == nullptr) {
+    std::cerr << "Could not connect the file system." << std::endl;
+    return false;
+  }
+
+  const auto status = fs->Delete(uri.get_path(), recursive);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h
new file mode 100644
index 0000000000000..af5faf65be489
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_RM
+#define LIBHDFSPP_TOOLS_HDFS_RM
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Rm} is an {@class HdfsTool} that removes/unlinks the files or
+ * directories.
+ */
+class Rm : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Rm(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Rm(const Rm &) = default;
+  Rm(Rm &&) = default;
+  Rm &operator=(const Rm &) = delete;
+  Rm &operator=(Rm &&) = delete;
+  ~Rm() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param recursive Perform this operation recursively on the sub-directories.
+   * @param path The path to the file/directory that needs to be removed.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(bool recursive,
+                                        const std::string &path) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc
new file mode 100644
index 0000000000000..16577f52a849a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-rm.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS rm "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Rm rm(argc, argv);
+  auto success = false;
+
+  try {
+    success = rm.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt
new file mode 100644
index 0000000000000..a0d8bafa63080
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_setrep_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-setrep.cc)
+target_include_directories(hdfs_setrep_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_setrep_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_setrep main.cc)
+target_include_directories(hdfs_setrep PRIVATE ../../tools)
+target_link_libraries(hdfs_setrep PRIVATE hdfs_setrep_lib)
+
+install(TARGETS hdfs_setrep RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc
new file mode 100644
index 0000000000000..542659b29f141
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.cc
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-setrep.h"
+#include "internal/set-replication-state.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Setrep::Setrep(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Setrep::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h",
+              "Changes the replication factor of a file at PATH. If PATH is a "
+              "directory then the command recursively changes the replication "
+              "factor of all files under the directory tree rooted at PATH.");
+  add_options(
+      "replication-factor", po::value<std::string>(),
+      "The replication factor to set for the given path and its children.");
+  add_options("path", po::value<std::string>(),
+              "The path for which the replication factor needs to be set.");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("replication-factor", 1);
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool Setrep::ValidateConstraints() const {
+  // Only "help" is allowed as single argument.
+  if (argc_ == 2) {
+    return opt_val_.count("help");
+  }
+
+  // Rest of the cases must contain more than 2 arguments on the command line.
+  return argc_ > 2;
+}
+
+std::string Setrep::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_setrep [OPTION] NUM_REPLICAS PATH" << std::endl
+       << std::endl
+       << "Changes the replication factor of a file at PATH. If PATH is a "
+          "directory then the command"
+       << std::endl
+       << "recursively changes the replication factor of all files under the "
+          "directory tree rooted at PATH."
+       << std::endl
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_setrep 5 hdfs://localhost.localdomain:8020/dir/file"
+       << std::endl
+       << "hdfs_setrep 3 /dir1/dir2" << std::endl;
+  return desc.str();
+}
+
+bool Setrep::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS setrep tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0 && opt_val_.count("replication-factor") > 0) {
+    const auto replication_factor =
+        opt_val_["replication-factor"].as<std::string>();
+    const auto path = opt_val_["path"].as<std::string>();
+    return HandlePath(path, replication_factor);
+  }
+
+  return false;
+}
+
+bool Setrep::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Setrep::HandlePath(const std::string &path,
+                        const std::string &replication_factor) const {
+  // Building a URI object from the given path.
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect to the file system." << std::endl;
+    return false;
+  }
+
+  /*
+   * Wrap async FileSystem::SetReplication with promise to make it a blocking
+   * call.
+   */
+  auto promise = std::make_shared<std::promise<hdfs::Status>>();
+  std::future future(promise->get_future());
+  auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); };
+
+  const auto replication = static_cast<uint16_t>(
+      std::strtol(replication_factor.c_str(), nullptr, 8));
+  /*
+   * Allocating shared state, which includes:
+   * replication to be set, handler to be called, request counter, and a boolean
+   * to keep track if find is done
+   */
+  auto state =
+      std::make_shared<SetReplicationState>(replication, handler, 0, false);
+
+  /*
+   * Keep requesting more from Find until we process the entire listing. Call
+   * handler when Find is done and request counter is 0. Find guarantees that
+   * the handler will only be called once at a time so we do not need locking in
+   * handler_find.
+   */
+  auto handler_find = [fs, state](const hdfs::Status &status_find,
+                                  const std::vector<hdfs::StatInfo> &stat_infos,
+                                  const bool has_more_results) -> bool {
+    /*
+     * For each result returned by Find we call async SetReplication with the
+     * handler below. SetReplication DOES NOT guarantee that the handler will
+     * only be called once at a time, so we DO need locking in
+     * handler_set_replication.
+     */
+    auto handler_set_replication =
+        [state](const hdfs::Status &status_set_replication) {
+          std::lock_guard guard(state->lock);
+
+          // Decrement the counter once since we are done with this async call.
+          if (!status_set_replication.ok() && state->status.ok()) {
+            // We make sure we set state->status only on the first error.
+            state->status = status_set_replication;
+          }
+          state->request_counter--;
+          if (state->request_counter == 0 && state->find_is_done) {
+            state->handler(state->status); // Exit.
+          }
+        };
+    if (!stat_infos.empty() && state->status.ok()) {
+      for (hdfs::StatInfo const &stat_info : stat_infos) {
+        // Launch an asynchronous call to SetReplication for every returned
+        // file.
+        if (stat_info.file_type == hdfs::StatInfo::IS_FILE) {
+          state->request_counter++;
+          fs->SetReplication(stat_info.full_path, state->replication,
+                             handler_set_replication);
+        }
+      }
+    }
+
+    /*
+     * Lock this section because handlerSetReplication might be accessing the
+     * same shared variables simultaneously.
+     */
+    std::lock_guard guard(state->lock);
+    if (!status_find.ok() && state->status.ok()) {
+      // We make sure we set state->status only on the first error.
+      state->status = status_find;
+    }
+    if (!has_more_results) {
+      state->find_is_done = true;
+      if (state->request_counter == 0) {
+        state->handler(state->status); // Exit.
+      }
+      return false;
+    }
+    return true;
+  };
+
+  // Asynchronous call to Find.
+  fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(),
+           handler_find);
+
+  // Block until promise is set.
+  const auto status = future.get();
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h
new file mode 100644
index 0000000000000..20ee7405b6de9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/hdfs-setrep.h
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_SETREP
+#define LIBHDFSPP_TOOLS_HDFS_SETREP
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Setrep} is an {@class HdfsTool} that changes the replication factor
+ * of a file at a given path. If the path is a directory, then it recursively
+ * changes the replication factor of all files under the directory tree rooted
+ * at the given path.
+ */
+class Setrep : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Setrep(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Setrep(const Setrep &) = default;
+  Setrep(Setrep &&) = default;
+  Setrep &operator=(const Setrep &) = delete;
+  Setrep &operator=(Setrep &&) = delete;
+  ~Setrep() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory for which we need setrep info.
+   * @param replication_factor The replication factor to set to given path and
+   * its children.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool
+  HandlePath(const std::string &path,
+             const std::string &replication_factor) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc
new file mode 100644
index 0000000000000..a3d8399c575ab
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-setrep/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-setrep.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS setrep "
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Setrep setrep(argc, argv);
+  auto success = false;
+
+  try {
+    success = setrep.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt
new file mode 100644
index 0000000000000..3a390cc2ed20a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_stat_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-stat.cc)
+target_include_directories(hdfs_stat_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_stat_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_stat main.cc)
+target_include_directories(hdfs_stat PRIVATE ../../tools)
+target_link_libraries(hdfs_stat PRIVATE hdfs_stat_lib)
+
+install(TARGETS hdfs_stat RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc
new file mode 100644
index 0000000000000..e20fc4576d31c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.cc
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-stat.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Stat::Stat(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Stat::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Displays the stat information for the given path. The "
+                        "path can be a file or a directory.");
+  add_options("path", po::value<std::string>(),
+              "The path in the filesystem for which to display the "
+              "stat information.");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Stat::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_stat PATH" << std::endl
+       << std::endl
+       << "Displays the stat information for the given path." << std::endl
+       << "The path can be a file or a directory." << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_stat hdfs://localhost.localdomain:8020/dir/file" << std::endl;
+  return desc.str();
+}
+
+bool Stat::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS stat tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    return HandlePath(path);
+  }
+
+  return false;
+}
+
+bool Stat::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Stat::HandlePath(const std::string &path) const {
+  // Building a URI object from the given uri_path
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, false);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  hdfs::StatInfo stat_info;
+  const auto status = fs->GetFileInfo(uri.get_path(), stat_info);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+  std::cout << stat_info.str() << std::endl;
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h
new file mode 100644
index 0000000000000..a23affc1c461c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/hdfs-stat.h
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_STAT
+#define LIBHDFSPP_TOOLS_HDFS_STAT
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Stat} is an {@class HdfsTool} that displays the stat information for
+ * the given path. The path can be a file or a directory.
+ */
+class Stat : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Stat(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Stat(const Stat &) = default;
+  Stat(Stat &&) = default;
+  Stat &operator=(const Stat &) = delete;
+  Stat &operator=(Stat &&) = delete;
+  ~Stat() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the directory for which we need the stat info.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &path) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc
new file mode 100644
index 0000000000000..4437419259ca0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-stat/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-stat.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule the clean-up tasks for HDFS stat"
+                 "tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Stat stat(argc, argv);
+  auto success = false;
+
+  try {
+    success = stat.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt
new file mode 100644
index 0000000000000..5169e00f1996f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/CMakeLists.txt
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_tail_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-tail.cc)
+target_include_directories(hdfs_tail_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_tail_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_tail main.cc)
+target_include_directories(hdfs_tail PRIVATE ../../tools)
+target_link_libraries(hdfs_tail PRIVATE hdfs_tail_lib)
+
+install(TARGETS hdfs_tail RUNTIME DESTINATION bin)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc
new file mode 100644
index 0000000000000..7eaa4602c894f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.cc
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <chrono>
+#include <future>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <thread>
+
+#include "hdfs-tail.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+Tail::Tail(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool Tail::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Displays last kilobyte of the file to stdout.");
+  add_options("follow,f",
+              "Append data to the output as the file grows, as in Unix.");
+  add_options("path", po::value<std::string>(),
+              "The path indicating the filesystem that needs to be tailed.");
+
+  // We allow only one positional argument to be passed to this tool. An
+  // exception is thrown if multiple arguments are passed.
+  pos_opt_desc_.add("path", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+std::string Tail::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_tail [OPTION] FILE" << std::endl
+       << std::endl
+       << "Displays last kilobyte of the file to stdout." << std::endl
+       << std::endl
+       << "  -f  append data to the output as the file grows, as in Unix"
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_tail hdfs://localhost.localdomain:8020/dir/file" << std::endl
+       << "hdfs_tail /dir/file" << std::endl;
+  return desc.str();
+}
+
+bool Tail::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS tail tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("path") > 0) {
+    const auto path = opt_val_["path"].as<std::string>();
+    const auto follow = opt_val_.count("follow") > 0;
+    return HandlePath(path, follow);
+  }
+
+  return false;
+}
+
+bool Tail::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool Tail::HandlePath(const std::string &path, const bool follow) const {
+  // Building a URI object from the given path.
+  auto uri = hdfs::parse_path_or_exit(path);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect to the file system." << std::endl;
+    return false;
+  }
+
+  // We need to get the size of the file using stat.
+  hdfs::StatInfo stat_info;
+  auto status = fs->GetFileInfo(uri.get_path(), stat_info);
+  if (!status.ok()) {
+    std::cerr << "Error: " << status.ToString() << std::endl;
+    return false;
+  }
+
+  // Determine where to start reading.
+  off_t offset{0};
+  if (stat_info.length > tail_size_in_bytes) {
+    offset = static_cast<off_t>(stat_info.length - tail_size_in_bytes);
+  }
+
+  do {
+    const auto current_length = static_cast<off_t>(stat_info.length);
+    readFile(fs, uri.get_path(), offset, stdout, false);
+
+    // Exit if -f flag was not set.
+    if (!follow) {
+      break;
+    }
+
+    do {
+      // Sleep for the refresh rate.
+      std::this_thread::sleep_for(std::chrono::seconds(refresh_rate_in_sec));
+
+      // Use stat to check the new file size.
+      status = fs->GetFileInfo(uri.get_path(), stat_info);
+      if (!status.ok()) {
+        std::cerr << "Error: " << status.ToString() << std::endl;
+        return false;
+      }
+
+      // If file became longer, loop back and print the difference.
+    } while (static_cast<off_t>(stat_info.length) <= current_length);
+  } while (true);
+
+  return true;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h
new file mode 100644
index 0000000000000..f103d0d5b9470
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/hdfs-tail.h
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_TAIL
+#define LIBHDFSPP_TOOLS_HDFS_TAIL
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Tail} is an {@class HdfsTool} displays last kilobyte of the file to
+ * stdout.
+ */
+class Tail : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  Tail(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  Tail(const Tail &) = default;
+  Tail(Tail &&) = default;
+  Tail &operator=(const Tail &) = delete;
+  Tail &operator=(Tail &&) = delete;
+  ~Tail() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; }
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param path The path to the file which needs to be tailed.
+   * @param follow Append data to the output as the file grows, as in Unix.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &path,
+                                        bool follow) const;
+
+  /**
+   * The tail size in bytes.
+   */
+  static constexpr uint64_t tail_size_in_bytes{1024};
+
+  /**
+   * The refresh rate for {@link hdfs::tools::Tail} in seconds.
+   */
+  static constexpr int refresh_rate_in_sec{1};
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc
new file mode 100644
index 0000000000000..fb85915e1d3f2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-tail/main.cc
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-tail.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS tail tool, "
+                 "exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::Tail tail(argc, argv);
+  auto success = false;
+
+  try {
+    success = tail.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
deleted file mode 100644
index d64affeb14467..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE"
-      << std::endl
-      << std::endl << "Change the group association of each FILE to GROUP."
-      << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:"
-      << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
-      << std::endl
-      << std::endl << "  -R  operate on files and directories recursively"
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_chgrp new_group /dir/file"
-      << std::endl;
-}
-
-struct SetOwnerState {
-  const std::string username;
-  const std::string groupname;
-  const std::function<void(const hdfs::Status &)> handler;
-  //The request counter is incremented once every time SetOwner async call is made
-  uint64_t request_counter;
-  //This boolean will be set when find returns the last result
-  bool find_is_done;
-  //Final status to be returned
-  hdfs::Status status;
-  //Shared variables will need protection with a lock
-  std::mutex lock;
-  SetOwnerState(const std::string & username_, const std::string & groupname_,
-                const std::function<void(const hdfs::Status &)> & handler_,
-              uint64_t request_counter_, bool find_is_done_)
-      : username(username_),
-        groupname(groupname_),
-        handler(handler_),
-        request_counter(request_counter_),
-        find_is_done(find_is_done_),
-        status(),
-        lock() {
-  }
-};
-
-int main(int argc, char *argv[]) {
-  //We should have 3 or 4 parameters
-  if (argc != 3 && argc != 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = 1;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string group = argv[optind];
-  //Owner stays the same, just group association changes.
-  std::string owner = "";
-  std::string uri_path = argv[optind + 1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
-  std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
-  std::future<hdfs::Status> future(promise->get_future());
-  auto handler = [promise](const hdfs::Status &s) {
-    promise->set_value(s);
-  };
-
-  if(!recursive){
-    fs->SetOwner(uri.get_path(), owner, group, handler);
-  }
-  else {
-    //Allocating shared state, which includes:
-    //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
-    std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
-
-    // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
-    // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
-    auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
-      //For each result returned by Find we call async SetOwner with the handler below.
-      //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
-      auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
-        std::lock_guard<std::mutex> guard(state->lock);
-
-        //Decrement the counter once since we are done with this async call
-        if (!status_set_owner.ok() && state->status.ok()){
-          //We make sure we set state->status only on the first error.
-          state->status = status_set_owner;
-        }
-        state->request_counter--;
-        if(state->request_counter == 0 && state->find_is_done){
-          state->handler(state->status); //exit
-        }
-      };
-      if(!stat_infos.empty() && state->status.ok()) {
-        for (hdfs::StatInfo const& s : stat_infos) {
-          //Launch an asynchronous call to SetOwner for every returned result
-          state->request_counter++;
-          fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
-        }
-      }
-
-      //Lock this section because handlerSetOwner might be accessing the same
-      //shared variables simultaneously
-      std::lock_guard<std::mutex> guard(state->lock);
-      if (!status_find.ok() && state->status.ok()){
-        //We make sure we set state->status only on the first error.
-        state->status = status_find;
-      }
-      if(!has_more_results){
-        state->find_is_done = true;
-        if(state->request_counter == 0){
-          state->handler(state->status); //exit
-        }
-        return false;
-      }
-      return true;
-    };
-
-    //Asynchronous call to Find
-    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
-  }
-
-  /* block until promise is set */
-  hdfs::Status status = future.get();
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
deleted file mode 100644
index 091f18f904ecc..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_chmod [OPTION] <MODE[,MODE]... | OCTALMODE> FILE"
-      << std::endl
-      << std::endl << "Change the permissions of each FILE to MODE."
-      << std::endl << "The user must be the owner of the file, or else a super-user."
-      << std::endl << "Additional information is in the Permissions Guide:"
-      << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
-      << std::endl
-      << std::endl << "  -R  operate on files and directories recursively"
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_chmod 777 /dir/file"
-      << std::endl;
-}
-
-struct SetPermissionState {
-  const uint16_t permissions;
-  const std::function<void(const hdfs::Status &)> handler;
-  //The request counter is incremented once every time SetOwner async call is made
-  uint64_t request_counter;
-  //This boolean will be set when find returns the last result
-  bool find_is_done;
-  //Final status to be returned
-  hdfs::Status status;
-  //Shared variables will need protection with a lock
-  std::mutex lock;
-  SetPermissionState(const uint16_t permissions_, const std::function<void(const hdfs::Status &)> & handler_,
-              uint64_t request_counter_, bool find_is_done_)
-      : permissions(permissions_),
-        handler(handler_),
-        request_counter(request_counter_),
-        find_is_done(find_is_done_),
-        status(),
-        lock() {
-  }
-};
-
-int main(int argc, char *argv[]) {
-  //We should have 3 or 4 parameters
-  if (argc != 3 && argc != 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = 1;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string permissions = argv[optind];
-  std::string uri_path = argv[optind + 1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  /* wrap async FileSystem::SetPermission with promise to make it a blocking call */
-  std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
-  std::future<hdfs::Status> future(promise->get_future());
-  auto handler = [promise](const hdfs::Status &s) {
-    promise->set_value(s);
-  };
-
-  //strtol() is reading the value with base 8, NULL because we are reading in just one value.
-  uint16_t perm = strtol(permissions.c_str(), NULL, 8);
-  if(!recursive){
-    fs->SetPermission(uri.get_path(), perm, handler);
-  }
-  else {
-    //Allocating shared state, which includes:
-    //permissions to be set, handler to be called, request counter, and a boolean to keep track if find is done
-    std::shared_ptr<SetPermissionState> state = std::make_shared<SetPermissionState>(perm, handler, 0, false);
-
-    // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
-    // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
-    auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
-      //For each result returned by Find we call async SetPermission with the handler below.
-      //SetPermission DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetPermission.
-      auto handlerSetPermission = [state](const hdfs::Status &status_set_permission) {
-        std::lock_guard<std::mutex> guard(state->lock);
-
-        //Decrement the counter once since we are done with this async call
-        if (!status_set_permission.ok() && state->status.ok()){
-          //We make sure we set state->status only on the first error.
-          state->status = status_set_permission;
-        }
-        state->request_counter--;
-        if(state->request_counter == 0 && state->find_is_done){
-          state->handler(state->status); //exit
-        }
-      };
-      if(!stat_infos.empty() && state->status.ok()) {
-        for (hdfs::StatInfo const& s : stat_infos) {
-          //Launch an asynchronous call to SetPermission for every returned result
-          state->request_counter++;
-          fs->SetPermission(s.full_path, state->permissions, handlerSetPermission);
-        }
-      }
-
-      //Lock this section because handlerSetPermission might be accessing the same
-      //shared variables simultaneously
-      std::lock_guard<std::mutex> guard(state->lock);
-      if (!status_find.ok() && state->status.ok()){
-        //We make sure we set state->status only on the first error.
-        state->status = status_find;
-      }
-      if(!has_more_results){
-        state->find_is_done = true;
-        if(state->request_counter == 0){
-          state->handler(state->status); //exit
-        }
-        return false;
-      }
-      return true;
-    };
-
-    //Asynchronous call to Find
-    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
-  }
-
-  /* block until promise is set */
-  hdfs::Status status = future.get();
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
deleted file mode 100644
index 24f7470b4163e..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE"
-      << std::endl
-      << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP."
-      << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:"
-      << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html"
-      << std::endl
-      << std::endl << "  -R  operate on files and directories recursively"
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Owner is unchanged if missing.  Group is unchanged if missing."
-      << std::endl << "OWNER and GROUP may be numeric as well as symbolic."
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_chown new_owner /dir/file"
-      << std::endl;
-}
-
-struct SetOwnerState {
-  const std::string username;
-  const std::string groupname;
-  const std::function<void(const hdfs::Status &)> handler;
-  //The request counter is incremented once every time SetOwner async call is made
-  uint64_t request_counter;
-  //This boolean will be set when find returns the last result
-  bool find_is_done;
-  //Final status to be returned
-  hdfs::Status status;
-  //Shared variables will need protection with a lock
-  std::mutex lock;
-  SetOwnerState(const std::string & username_, const std::string & groupname_,
-                const std::function<void(const hdfs::Status &)> & handler_,
-              uint64_t request_counter_, bool find_is_done_)
-      : username(username_),
-        groupname(groupname_),
-        handler(handler_),
-        request_counter(request_counter_),
-        find_is_done(find_is_done_),
-        status(),
-        lock() {
-  }
-};
-
-int main(int argc, char *argv[]) {
-  //We should have 3 or 4 parameters
-  if (argc != 3 && argc != 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = 1;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string owner_and_group = argv[optind];
-  std::string uri_path = argv[optind + 1];
-
-  std::string owner, group;
-  size_t owner_end = owner_and_group.find(":");
-  if(owner_end == std::string::npos) {
-    owner = owner_and_group;
-  } else {
-    owner = owner_and_group.substr(0, owner_end);
-    group = owner_and_group.substr(owner_end + 1);
-  }
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  /* wrap async FileSystem::SetOwner with promise to make it a blocking call */
-  std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
-  std::future<hdfs::Status> future(promise->get_future());
-  auto handler = [promise](const hdfs::Status &s) {
-    promise->set_value(s);
-  };
-
-  if(!recursive){
-    fs->SetOwner(uri.get_path(), owner, group, handler);
-  }
-  else {
-    //Allocating shared state, which includes:
-    //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done
-    std::shared_ptr<SetOwnerState> state = std::make_shared<SetOwnerState>(owner, group, handler, 0, false);
-
-    // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
-    // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
-    auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
-      //For each result returned by Find we call async SetOwner with the handler below.
-      //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner.
-      auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) {
-        std::lock_guard<std::mutex> guard(state->lock);
-
-        //Decrement the counter once since we are done with this async call
-        if (!status_set_owner.ok() && state->status.ok()){
-          //We make sure we set state->status only on the first error.
-          state->status = status_set_owner;
-        }
-        state->request_counter--;
-        if(state->request_counter == 0 && state->find_is_done){
-          state->handler(state->status); //exit
-        }
-      };
-      if(!stat_infos.empty() && state->status.ok()) {
-        for (hdfs::StatInfo const& s : stat_infos) {
-          //Launch an asynchronous call to SetOwner for every returned result
-          state->request_counter++;
-          fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner);
-        }
-      }
-
-      //Lock this section because handlerSetOwner might be accessing the same
-      //shared variables simultaneously
-      std::lock_guard<std::mutex> guard(state->lock);
-      if (!status_find.ok() && state->status.ok()){
-        //We make sure we set state->status only on the first error.
-        state->status = status_find;
-      }
-      if(!has_more_results){
-        state->find_is_done = true;
-        if(state->request_counter == 0){
-          state->handler(state->status); //exit
-        }
-        return false;
-      }
-      return true;
-    };
-
-    //Asynchronous call to Find
-    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
-  }
-
-  /* block until promise is set */
-  hdfs::Status status = future.get();
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
deleted file mode 100644
index c9eb193e1baa0..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE"
-      << std::endl
-      << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
-      << std::endl << "hdfs_copyToLocal /dir/file /home/usr/dir/file"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc > 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-  std::string dest = argv[optind+1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
-  if(!dst_file){
-    std::cerr << "Unable to open the destination file: " << dest << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  readFile(fs, uri.get_path(), 0, dst_file, false);
-  std::fclose(dst_file);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
deleted file mode 100644
index 345ccc6e27959..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_count [OPTION] FILE"
-      << std::endl
-      << std::endl << "Count the number of directories, files and bytes under the path that match the specified FILE pattern."
-      << std::endl << "The output columns with -count are: DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME"
-      << std::endl
-      << std::endl << "  -q    output additional columns before the rest: QUOTA, SPACE_QUOTA, SPACE_CONSUMED"
-      << std::endl << "  -h    display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_count hdfs://localhost.localdomain:8020/dir"
-      << std::endl << "hdfs_count -q /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool quota = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "qh")) != -1) {
-    switch (input)
-    {
-    case 'q':
-      quota = true;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::ContentSummary content_summary;
-  hdfs::Status status = fs->GetContentSummary(uri.get_path(), content_summary);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  std::cout << content_summary.str(quota) << std::endl;
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
deleted file mode 100644
index 1236dd61130e6..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_createSnapshot [OPTION] PATH"
-      << std::endl
-      << std::endl << "Create a snapshot of a snapshottable directory."
-      << std::endl << "This operation requires owner privilege of the snapshottable directory."
-      << std::endl
-      << std::endl << "  -n NAME   The snapshot name. When it is omitted, a default name is generated"
-      << std::endl << "             using a timestamp with the format:"
-      << std::endl << "             \"'s'yyyyMMdd-HHmmss.SSS\", e.g. s20130412-151029.033"
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_createSnapshot hdfs://localhost.localdomain:8020/dir"
-      << std::endl << "hdfs_createSnapshot -n MySnapshot /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-  std::string name;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "hn:")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case 'n':
-      name = optarg;
-      break;
-    case '?':
-      if (optopt == 'n')
-        std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
-      else if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::Status status = fs->CreateSnapshot(uri.get_path(), name);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
deleted file mode 100644
index b5c97ab9033f3..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_disallowSnapshot [OPTION] PATH"
-      << std::endl
-      << std::endl << "Disallowing snapshots of a directory at PATH to be created."
-      << std::endl << "All snapshots of the directory must be deleted before disallowing snapshots."
-      << std::endl
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_disallowSnapshot hdfs://localhost.localdomain:8020/dir"
-      << std::endl << "hdfs_disallowSnapshot /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::Status status = fs->DisallowSnapshot(uri.get_path());
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
deleted file mode 100644
index f6b6e73f09a18..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_du.cc
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_du [OPTION] PATH"
-      << std::endl
-      << std::endl << "Displays sizes of files and directories contained in the given PATH"
-      << std::endl << "or the length of a file in case PATH is just a file"
-      << std::endl
-      << std::endl << "  -R        operate on files and directories recursively"
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_du hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_du -R /dir1/dir2"
-      << std::endl;
-}
-
-struct GetContentSummaryState {
-  const std::function<void(const hdfs::Status &)> handler;
-  //The request counter is incremented once every time GetContentSummary async call is made
-  uint64_t request_counter;
-  //This boolean will be set when find returns the last result
-  bool find_is_done;
-  //Final status to be returned
-  hdfs::Status status;
-  //Shared variables will need protection with a lock
-  std::mutex lock;
-  GetContentSummaryState(const std::function<void(const hdfs::Status &)> & handler_,
-              uint64_t request_counter_, bool find_is_done_)
-      : handler(handler_),
-        request_counter(request_counter_),
-        find_is_done(find_is_done_),
-        status(),
-        lock() {
-  }
-};
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = true;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  /* wrap async FileSystem::GetContentSummary with promise to make it a blocking call */
-  std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
-  std::future<hdfs::Status> future(promise->get_future());
-  auto handler = [promise](const hdfs::Status &s) {
-    promise->set_value(s);
-  };
-
-  //Allocating shared state, which includes:
-  //handler to be called, request counter, and a boolean to keep track if find is done
-  std::shared_ptr<GetContentSummaryState> state = std::make_shared<GetContentSummaryState>(handler, 0, false);
-
-  // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
-  // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
-  auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
-    //For each result returned by Find we call async GetContentSummary with the handler below.
-    //GetContentSummary DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerGetContentSummary.
-    auto handlerGetContentSummary = [state](const hdfs::Status &status_get_summary, const hdfs::ContentSummary &si) {
-      std::lock_guard<std::mutex> guard(state->lock);
-      std::cout << si.str_du() << std::endl;
-      //Decrement the counter once since we are done with this async call
-      if (!status_get_summary.ok() && state->status.ok()){
-        //We make sure we set state->status only on the first error.
-        state->status = status_get_summary;
-      }
-      state->request_counter--;
-      if(state->request_counter == 0 && state->find_is_done){
-        state->handler(state->status); //exit
-      }
-    };
-    if(!stat_infos.empty() && state->status.ok()) {
-      for (hdfs::StatInfo const& s : stat_infos) {
-        //Launch an asynchronous call to GetContentSummary for every returned result
-        state->request_counter++;
-        fs->GetContentSummary(s.full_path, handlerGetContentSummary);
-      }
-    }
-
-    //Lock this section because handlerGetContentSummary might be accessing the same
-    //shared variables simultaneously
-    std::lock_guard<std::mutex> guard(state->lock);
-    if (!status_find.ok() && state->status.ok()){
-      //We make sure we set state->status only on the first error.
-      state->status = status_find;
-    }
-    if(!has_more_results){
-      state->find_is_done = true;
-      if(state->request_counter == 0){
-        state->handler(state->status); //exit
-      }
-      return false;
-    }
-    return true;
-  };
-
-  if(!recursive){
-    //Asynchronous call to Find
-    fs->GetListing(uri.get_path(), handlerFind);
-  } else {
-    //Asynchronous call to Find
-    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
-  }
-
-  /* block until promise is set */
-  hdfs::Status status = future.get();
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
deleted file mode 100644
index 348f851ad3835..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_find.cc
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_find [OPTION] PATH"
-      << std::endl
-      << std::endl << "Finds all files recursively starting from the"
-      << std::endl << "specified PATH and prints their file paths."
-      << std::endl << "This hdfs_find tool mimics the POSIX find."
-      << std::endl
-      << std::endl << "Both PATH and NAME can have wild-cards."
-      << std::endl
-      << std::endl << "  -n NAME       if provided all results will be matching the NAME pattern"
-      << std::endl << "                otherwise, the implicit '*' will be used"
-      << std::endl << "                NAME allows wild-cards"
-      << std::endl
-      << std::endl << "  -m MAX_DEPTH  if provided the maximum depth to recurse after the end of"
-      << std::endl << "                the path is reached will be limited by MAX_DEPTH"
-      << std::endl << "                otherwise, the maximum depth to recurse is unbound"
-      << std::endl << "                MAX_DEPTH can be set to 0 for pure globbing and ignoring"
-      << std::endl << "                the NAME option (no recursion after the end of the path)"
-      << std::endl
-      << std::endl << "  -h            display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_find hdfs://localhost.localdomain:8020/dir?/tree* -n some?file*name"
-      << std::endl << "hdfs_find / -n file_name -m 3"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-  //If NAME is not specified we use implicit "*"
-  std::string name = "*";
-  //If MAX_DEPTH is not specified we use the max value of uint_32_t
-  uint32_t max_depth = hdfs::FileSystem::GetDefaultFindMaxDepth();
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "hn:m:")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case 'n':
-      name = optarg;
-      break;
-    case 'm':
-      max_depth = std::stoi(optarg);
-      break;
-    case '?':
-      if (optopt == 'n' || optopt == 'm')
-        std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
-      else if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::shared_ptr<std::promise<void>> promise = std::make_shared<std::promise<void>>();
-  std::future<void> future(promise->get_future());
-  hdfs::Status status = hdfs::Status::OK();
-
-  /**
-    * Keep requesting more until we get the entire listing. Set the promise
-    * when we have the entire listing to stop.
-    *
-    * Find guarantees that the handler will only be called once at a time,
-    * so we do not need any locking here. It also guarantees that the handler will be
-    * only called once with has_more_results set to false.
-    */
-  auto handler = [promise, &status]
-                  (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
-    //Print result chunks as they arrive
-    if(!si.empty()) {
-      for (hdfs::StatInfo const& s : si) {
-        std::cout << s.str() << std::endl;
-      }
-    }
-    if(!s.ok() && status.ok()){
-      //We make sure we set 'status' only on the first error.
-      status = s;
-    }
-    if (!has_more_results) {
-      promise->set_value();  //set promise
-      return false;         //request stop sending results
-    }
-    return true;  //request more results
-  };
-
-  //Asynchronous call to Find
-  fs->Find(uri.get_path(), name, max_depth, handler);
-
-  //block until promise is set
-  future.get();
-  if(!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
deleted file mode 100644
index 16dd72d24bf72..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE"
-      << std::endl
-      << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
-      << std::endl << "hdfs_get /dir/file /home/usr/dir/file"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc > 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-  std::string dest = argv[optind+1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
-  if(!dst_file){
-    std::cerr << "Unable to open the destination file: " << dest << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  readFile(fs, uri.get_path(), 0, dst_file, false);
-  std::fclose(dst_file);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
deleted file mode 100644
index e35e51da1d32c..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_ls.cc
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_ls [OPTION] FILE"
-      << std::endl
-      << std::endl << "List information about the FILEs."
-      << std::endl
-      << std::endl << "  -R        list subdirectories recursively"
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_ls hdfs://localhost.localdomain:8020/dir"
-      << std::endl << "hdfs_ls -R /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = true;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::shared_ptr<std::promise<void>> promise = std::make_shared<std::promise<void>>();
-  std::future<void> future(promise->get_future());
-  hdfs::Status status = hdfs::Status::OK();
-
-  /**
-    * Keep requesting more until we get the entire listing. Set the promise
-    * when we have the entire listing to stop.
-    *
-    * Find and GetListing guarantee that the handler will only be called once at a time,
-    * so we do not need any locking here. They also guarantee that the handler will be
-    * only called once with has_more_results set to false.
-    */
-  auto handler = [promise, &status]
-                  (const hdfs::Status &s, const std::vector<hdfs::StatInfo> & si, bool has_more_results) -> bool {
-    //Print result chunks as they arrive
-    if(!si.empty()) {
-      for (hdfs::StatInfo const& s : si) {
-        std::cout << s.str() << std::endl;
-      }
-    }
-    if(!s.ok() && status.ok()){
-      //We make sure we set 'status' only on the first error.
-      status = s;
-    }
-    if (!has_more_results) {
-      promise->set_value();  //set promise
-      return false;         //request stop sending results
-    }
-    return true;  //request more results
-  };
-
-  if(!recursive){
-    //Asynchronous call to GetListing
-    fs->GetListing(uri.get_path(), handler);
-  } else {
-    //Asynchronous call to Find
-    fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handler);
-  }
-
-  //block until promise is set
-  future.get();
-  if(!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
deleted file mode 100644
index 3ccc6017b58e5..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_mkdir [OPTION] DIRECTORY"
-      << std::endl
-      << std::endl << "Create the DIRECTORY(ies), if they do not already exist."
-      << std::endl
-      << std::endl << "  -p        make parent directories as needed"
-      << std::endl << "  -m  MODE  set file mode (octal permissions) for the new DIRECTORY(ies)"
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2"
-      << std::endl << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool create_parents = false;
-  uint16_t permissions = hdfs::FileSystem::GetDefaultPermissionMask();
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "pm:h")) != -1) {
-    switch (input)
-    {
-    case 'p':
-      create_parents = true;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case 'm':
-      //Get octal permissions for the new DIRECTORY(ies)
-      permissions = strtol(optarg, NULL, 8);
-      break;
-    case '?':
-      if (optopt == 'm')
-        std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl;
-      else if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::Status status = fs->Mkdirs(uri.get_path(), permissions, create_parents);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
deleted file mode 100644
index 0b8393747ab11..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE"
-      << std::endl
-      << std::endl << "Move SRC_FILE from hdfs to DST_FILE on the local file system."
-      << std::endl << "Moving is done by copying SRC_FILE to DST_FILE, and then"
-      << std::endl << "deleting DST_FILE if copy succeeded."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
-      << std::endl << "hdfs_moveToLocal /dir/file /home/usr/dir/file"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc > 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-  std::string dest = argv[optind+1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
-  if(!dst_file){
-    std::cerr << "Unable to open the destination file: " << dest << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  readFile(fs, uri.get_path(), 0, dst_file, true);
-  std::fclose(dst_file);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
deleted file mode 100644
index 8d38fc2292d1e..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_renameSnapshot [OPTION] PATH OLD_NAME NEW_NAME"
-      << std::endl
-      << std::endl << "Rename a snapshot from OLD_NAME to NEW_NAME."
-      << std::endl << "This operation requires owner privilege of the snapshottable directory."
-      << std::endl
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_renameSnapshot hdfs://localhost.localdomain:8020/dir oldDir newDir"
-      << std::endl << "hdfs_renameSnapshot /dir1/dir2 oldSnap newSnap"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-  std::string old_name = argv[optind+1];
-  std::string new_name = argv[optind+2];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::Status status = fs->RenameSnapshot(uri.get_path(), old_name, new_name);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
deleted file mode 100644
index 7056cf9a674d5..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_rm [OPTION] FILE"
-      << std::endl
-      << std::endl << "Remove (unlink) the FILE(s) or directory(ies)."
-      << std::endl
-      << std::endl << "  -R        remove directories and their contents recursively"
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_rm -R /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool recursive = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "Rh")) != -1) {
-    switch (input)
-    {
-    case 'R':
-      recursive = true;
-      break;
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::Status status = fs->Delete(uri.get_path(), recursive);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
deleted file mode 100644
index 019e24d63fea0..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_setrep.cc
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include <future>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_setrep [OPTION] NUM_REPLICAS PATH"
-      << std::endl
-      << std::endl << "Changes the replication factor of a file at PATH. If PATH is a directory then the command"
-      << std::endl << "recursively changes the replication factor of all files under the directory tree rooted at PATH."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_setrep 5 hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_setrep 3 /dir1/dir2"
-      << std::endl;
-}
-
-struct SetReplicationState {
-  const uint16_t replication;
-  const std::function<void(const hdfs::Status &)> handler;
-  //The request counter is incremented once every time SetReplication async call is made
-  uint64_t request_counter;
-  //This boolean will be set when find returns the last result
-  bool find_is_done;
-  //Final status to be returned
-  hdfs::Status status;
-  //Shared variables will need protection with a lock
-  std::mutex lock;
-  SetReplicationState(const uint16_t replication_, const std::function<void(const hdfs::Status &)> & handler_,
-              uint64_t request_counter_, bool find_is_done_)
-      : replication(replication_),
-        handler(handler_),
-        request_counter(request_counter_),
-        find_is_done(find_is_done_),
-        status(),
-        lock() {
-  }
-};
-
-int main(int argc, char *argv[]) {
-  //We should have 3 or 4 parameters
-  if (argc < 3) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string repl = argv[optind];
-  std::string uri_path = argv[optind + 1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, true);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  /* wrap async FileSystem::SetReplication with promise to make it a blocking call */
-  std::shared_ptr<std::promise<hdfs::Status>> promise = std::make_shared<std::promise<hdfs::Status>>();
-  std::future<hdfs::Status> future(promise->get_future());
-  auto handler = [promise](const hdfs::Status &s) {
-    promise->set_value(s);
-  };
-
-  uint16_t replication = std::stoi(repl.c_str(), NULL, 8);
-  //Allocating shared state, which includes:
-  //replication to be set, handler to be called, request counter, and a boolean to keep track if find is done
-  std::shared_ptr<SetReplicationState> state = std::make_shared<SetReplicationState>(replication, handler, 0, false);
-
-  // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0.
-  // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind.
-  auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector<hdfs::StatInfo> & stat_infos, bool has_more_results) -> bool {
-
-    //For each result returned by Find we call async SetReplication with the handler below.
-    //SetReplication DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetReplication.
-    auto handlerSetReplication = [state](const hdfs::Status &status_set_replication) {
-      std::lock_guard<std::mutex> guard(state->lock);
-
-      //Decrement the counter once since we are done with this async call
-      if (!status_set_replication.ok() && state->status.ok()){
-        //We make sure we set state->status only on the first error.
-        state->status = status_set_replication;
-      }
-      state->request_counter--;
-      if(state->request_counter == 0 && state->find_is_done){
-        state->handler(state->status); //exit
-      }
-    };
-    if(!stat_infos.empty() && state->status.ok()) {
-      for (hdfs::StatInfo const& s : stat_infos) {
-        //Launch an asynchronous call to SetReplication for every returned file
-        if(s.file_type == hdfs::StatInfo::IS_FILE){
-          state->request_counter++;
-          fs->SetReplication(s.full_path, state->replication, handlerSetReplication);
-        }
-      }
-    }
-
-    //Lock this section because handlerSetReplication might be accessing the same
-    //shared variables simultaneously
-    std::lock_guard<std::mutex> guard(state->lock);
-    if (!status_find.ok() && state->status.ok()){
-      //We make sure we set state->status only on the first error.
-      state->status = status_find;
-    }
-    if(!has_more_results){
-      state->find_is_done = true;
-      if(state->request_counter == 0){
-        state->handler(state->status); //exit
-      }
-      return false;
-    }
-    return true;
-  };
-
-  //Asynchronous call to Find
-  fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind);
-
-  /* block until promise is set */
-  hdfs::Status status = future.get();
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
deleted file mode 100644
index 59d513b21afd3..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_stat.cc
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_rm [OPTION] FILE"
-      << std::endl
-      << std::endl << "Display FILE status."
-      << std::endl
-      << std::endl << "  -h        display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_rm -R /dir1/dir2"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  //We should have at least 2 arguments
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  hdfs::StatInfo stat_info;
-  hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  std::cout << stat_info.str() << std::endl;
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
deleted file mode 100644
index 8125bdcfecfeb..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_tail.cc
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_tail [OPTION] FILE"
-      << std::endl
-      << std::endl << "Displays last kilobyte of the file to stdout."
-      << std::endl
-      << std::endl << "  -f  output appended data as the file grows, as in Unix"
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_tail hdfs://localhost.localdomain:8020/dir/file"
-      << std::endl << "hdfs_tail /dir/file"
-      << std::endl;
-}
-
-#define TAIL_SIZE 1024
-#define REFRESH_RATE 1 //seconds
-
-int main(int argc, char *argv[]) {
-  if (argc < 2) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  bool follow = false;
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "hf")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case 'f':
-      follow = true;
-      break;
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  //We need to get the size of the file using stat
-  hdfs::StatInfo stat_info;
-  hdfs::Status status = fs->GetFileInfo(uri.get_path(), stat_info);
-  if (!status.ok()) {
-    std::cerr << "Error: " << status.ToString() << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  //Determine where to start reading
-  off_t offset = 0;
-  if(stat_info.length > TAIL_SIZE){
-    offset = stat_info.length - TAIL_SIZE;
-  }
-
-  do {
-    off_t current_length = (off_t) stat_info.length;
-    readFile(fs, uri.get_path(), offset, stdout, false);
-
-    //Exit if -f flag was not set
-    if(!follow){
-      break;
-    }
-
-    do{
-      //Sleep for the REFRESH_RATE
-      sleep(REFRESH_RATE);
-      //Use stat to check the new filesize.
-      status = fs->GetFileInfo(uri.get_path(), stat_info);
-      if (!status.ok()) {
-        std::cerr << "Error: " << status.ToString() << std::endl;
-        exit(EXIT_FAILURE);
-      }
-      //If file became longer, loop back and print the difference
-    }
-    while((off_t) stat_info.length <= current_length);
-  } while (true);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt
new file mode 100644
index 0000000000000..b223905c9cc99
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_ownership_obj OBJECT hdfs-ownership.cc)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h
new file mode 100644
index 0000000000000..f35b55b3cfa6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/get-content-summary-state.h
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_DU_GET_CONTENT_SUMMARY_STATE
+#define LIBHDFSPP_TOOLS_HDFS_DU_GET_CONTENT_SUMMARY_STATE
+
+#include <functional>
+#include <mutex>
+#include <utility>
+
+#include "hdfspp/hdfspp.h"
+
+namespace hdfs::tools {
+/**
+ * The {@class GetContentSummaryState} is used to hold intermediate information
+ * during the execution of {@link hdfs::FileSystem#GetContentSummary}.
+ */
+struct GetContentSummaryState {
+  GetContentSummaryState(std::function<void(const hdfs::Status &)> handler,
+                         const uint64_t request_counter,
+                         const bool find_is_done)
+      : handler{std::move(handler)}, request_counter{request_counter},
+        find_is_done{find_is_done} {}
+
+  /**
+   * The handler that is used to update the status asynchronously.
+   */
+  const std::function<void(const hdfs::Status &)> handler;
+
+  /**
+   * The request counter is incremented once every time GetContentSummary async
+   * call is made.
+   */
+  uint64_t request_counter;
+
+  /**
+   * This boolean will be set when find returns the last result.
+   */
+  bool find_is_done;
+
+  /**
+   * Final status to be returned.
+   */
+  hdfs::Status status;
+
+  /**
+   * Shared variables will need protection with a lock.
+   */
+  std::mutex lock;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc
new file mode 100644
index 0000000000000..e9dc74e541424
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfs-ownership.h"
+
+namespace hdfs::tools {
+Ownership::Ownership(const std::string &user_and_group) {
+  const auto owner_end = user_and_group.find(':');
+  if (owner_end == std::string::npos) {
+    user_ = user_and_group;
+    return;
+  }
+
+  user_ = user_and_group.substr(0, owner_end);
+  group_ = user_and_group.substr(owner_end + 1);
+}
+
+bool Ownership::operator==(const Ownership &other) const {
+  const auto same_user = user_ == other.user_;
+  if (group_.has_value() && other.group_.has_value()) {
+    return same_user && group_.value() == other.group_.value();
+  }
+
+  if (!group_.has_value() && !other.group_.has_value()) {
+    return same_user;
+  }
+  return false;
+}
+} // namespace hdfs::tools
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h
new file mode 100644
index 0000000000000..037ab61e58cda
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_OWNERSHIP
+#define LIBHDFSPP_TOOLS_HDFS_OWNERSHIP
+
+#include <functional>
+#include <mutex>
+#include <optional>
+#include <utility>
+
+#include "hdfspp/status.h"
+
+namespace hdfs::tools {
+/**
+ * {@class Ownership} contains the user and group ownership information.
+ */
+struct Ownership {
+  explicit Ownership(const std::string &user_and_group);
+
+  [[nodiscard]] const std::string &GetUser() const { return user_; }
+
+  [[nodiscard]] const std::optional<std::string> &GetGroup() const {
+    return group_;
+  }
+
+  bool operator==(const Ownership &other) const;
+
+private:
+  std::string user_;
+  std::optional<std::string> group_;
+};
+
+/**
+ * {@class OwnerState} holds information needed for recursive traversal of some
+ * of the HDFS APIs.
+ */
+struct OwnerState {
+  OwnerState(std::string username, std::string group,
+             std::function<void(const hdfs::Status &)> handler,
+             const uint64_t request_counter, const bool find_is_done)
+      : user{std::move(username)}, group{std::move(group)}, handler{std::move(
+                                                                handler)},
+        request_counter{request_counter}, find_is_done{find_is_done} {}
+
+  const std::string user;
+  const std::string group;
+  const std::function<void(const hdfs::Status &)> handler;
+
+  /**
+   * The request counter is incremented once every time SetOwner async call is
+   * made.
+   */
+  uint64_t request_counter;
+
+  /**
+   * This boolean will be set when find returns the last result.
+   */
+  bool find_is_done{false};
+
+  /**
+   * Final status to be returned.
+   */
+  hdfs::Status status{};
+
+  /**
+   * Shared variables will need protection with a lock.
+   */
+  std::mutex lock;
+};
+} // namespace hdfs::tools
+
+#endif
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h
new file mode 100644
index 0000000000000..5d432eddbf711
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/set-replication-state.h
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_SET_REPLICATION_STATE
+#define LIBHDFSPP_TOOLS_HDFS_SET_REPLICATION_STATE
+
+#include <functional>
+#include <mutex>
+
+#include "hdfspp/hdfspp.h"
+
+namespace hdfs::tools {
+/**
+ * {@class SetReplicationState} helps in handling the intermediate results while
+ * running {@link Setrep}.
+ */
+struct SetReplicationState {
+  SetReplicationState(const uint16_t replication,
+                      std::function<void(const hdfs::Status &)> handler,
+                      const uint64_t request_counter, const bool find_is_done)
+      : replication{replication}, handler{std::move(handler)},
+        request_counter{request_counter}, find_is_done{find_is_done} {}
+
+  /**
+   * The replication factor.
+   */
+  const uint16_t replication;
+
+  /**
+   * Handle the given {@link hdfs::Status}.
+   */
+  const std::function<void(const hdfs::Status &)> handler;
+
+  /**
+   * The request counter is incremented once every time SetReplication async
+   * call is made.
+   */
+  uint64_t request_counter;
+
+  /**
+   * This boolean will be set when find returns the last result.
+   */
+  bool find_is_done;
+
+  /**
+   * Final status to be returned.
+   */
+  hdfs::Status status;
+
+  /**
+   * Shared variables will need protection with a lock.
+   */
+  std::mutex lock;
+};
+} // namespace hdfs::tools
+
+#endif
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
index 6cc5a5b692290..ec59dfbcd24c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
@@ -85,7 +85,6 @@ namespace hdfs {
   static char input_buffer[BUF_SIZE];
 
   void readFile(std::shared_ptr<hdfs::FileSystem> fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete) {
-    ssize_t total_bytes_read = 0;
     size_t last_bytes_read = 0;
 
     hdfs::FileHandle *file_raw = nullptr;
@@ -103,7 +102,6 @@ namespace hdfs {
       if(status.ok()) {
         //Writing file chunks to stdout
         fwrite(input_buffer, last_bytes_read, 1, dst_file);
-        total_bytes_read += last_bytes_read;
         offset += last_bytes_read;
       } else {
         if(status.is_invalid_offset()){
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index a6cc3f1ed39aa..24599a2b244a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -219,38 +219,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
               </execution>
             </executions>
           </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-enforcer-plugin</artifactId>
-            <dependencies>
-              <dependency>
-                <groupId>de.skuzzle.enforcer</groupId>
-                <artifactId>restrict-imports-enforcer-rule</artifactId>
-                <version>${restrict-imports.enforcer.version}</version>
-              </dependency>
-            </dependencies>
-            <executions>
-              <execution>
-                <id>banned-illegal-imports</id>
-                <phase>process-sources</phase>
-                <goals>
-                  <goal>enforce</goal>
-                </goals>
-                <configuration>
-                  <rules>
-                    <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                      <includeTestCode>true</includeTestCode>
-                      <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                      <bannedImports>
-                        <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                        <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                      </bannedImports>
-                    </restrictImports>
-                  </rules>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
         </plugins>
       </build>
     </profile>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
index 75c29d30314b7..dc1eb8746964e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java
@@ -29,7 +29,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DFSClient;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java
index c37a21e7d83f5..ad410d4d388f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
 import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
-import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.net.NetUtils;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java
index 3995fa5566bb0..5fb1f1a01200b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java
@@ -19,7 +19,7 @@
 
 import java.util.Comparator;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * OffsetRange is the range of read/write request. A single point (e.g.,[5,5])
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
index 0e07b1bfb62aa..8e4026926c83b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
@@ -58,7 +58,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
index feb681d27a811..70ae4b29e9f96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
index 882e9cda86aa0..ed1856c9a851c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * WriteCtx saves the context of one write request, such as request, channel,
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java
index 77646af2c5ed9..8380c3c9bb3ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java
@@ -18,8 +18,6 @@
 package org.apache.hadoop.hdfs.nfs.nfs3;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import org.junit.Test;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
index a5997b46a9154..4899d9bd4606c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestViewfsWithNfs3.java
@@ -154,8 +154,6 @@ public static void setup() throws Exception {
     DFSTestUtil.createFile(viewFs, new Path("/hdfs2/write2"), 0, (short) 1, 0);
     DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameMultiNN"),
         0, (short) 1, 0);
-    DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameSingleNN"),
-        0, (short) 1, 0);
   }
 
   @AfterClass
@@ -307,6 +305,8 @@ public void testNfsRenameMultiNN() throws Exception {
 
   @Test (timeout = 60000)
   public void testNfsRenameSingleNN() throws Exception {
+    DFSTestUtil.createFile(viewFs, new Path("/hdfs1/renameSingleNN"),
+            0, (short) 1, 0);
     HdfsFileStatus fromFileStatus = nn1.getRpcServer().getFileInfo("/user1");
     int fromNNId = Nfs3Utils.getNamenodeId(config, hdfs1.getUri());
     FileHandle fromHandle =
@@ -316,6 +316,8 @@ public void testNfsRenameSingleNN() throws Exception {
         nn1.getRpcServer().getFileInfo("/user1/renameSingleNN");
     Assert.assertEquals(statusBeforeRename.isDirectory(), false);
 
+    Path successFilePath = new Path("/user1/renameSingleNNSucess");
+    hdfs1.delete(successFilePath, false);
     testNfsRename(fromHandle, "renameSingleNN",
         fromHandle, "renameSingleNNSucess", Nfs3Status.NFS3_OK);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 2959fc54b7d97..d4d5c1eb33983 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -63,6 +63,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -305,38 +311,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           </filesets>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
   <profiles>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/rbfbalance/RouterFedBalance.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/rbfbalance/RouterFedBalance.java
index 4161ab503fe77..0cb4b54bfc431 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/rbfbalance/RouterFedBalance.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/rbfbalance/RouterFedBalance.java
@@ -63,7 +63,7 @@
  * dst sub-namespace with distcp.
  *
  * 1. Move data from the source path to the destination path with distcp.
- * 2. Update the the mount entry.
+ * 2. Update the mount entry.
  * 3. Delete the source path to trash.
  */
 public class RouterFedBalance extends Configured implements Tool {
@@ -77,7 +77,7 @@ public class RouterFedBalance extends Configured implements Tool {
   private static final String TRASH_PROCEDURE = "trash-procedure";
 
   /**
-   * This class helps building the balance job.
+   * This class helps to build the balance job.
    */
   private class Builder {
     /* Force close all open files while there is no diff. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java
index 629f21e565181..db917be71285c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java
@@ -23,10 +23,15 @@
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
 import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT;
+
 /**
  * Base fairness policy that implements @RouterRpcFairnessPolicyController.
  * Internally a map of nameservice to Semaphore is used to control permits.
@@ -34,21 +39,32 @@
 public class AbstractRouterRpcFairnessPolicyController
     implements RouterRpcFairnessPolicyController {
 
-  private static final Logger LOG =
+  public static final Logger LOG =
       LoggerFactory.getLogger(AbstractRouterRpcFairnessPolicyController.class);
 
   /** Hash table to hold semaphore for each configured name service. */
   private Map<String, Semaphore> permits;
 
+  private long acquireTimeoutMs = DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT;
+
   public void init(Configuration conf) {
     this.permits = new HashMap<>();
+    long timeoutMs = conf.getTimeDuration(DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT,
+        DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+    if (timeoutMs >= 0) {
+      acquireTimeoutMs = timeoutMs;
+    } else {
+      LOG.warn("Invalid value {} configured for {} should be greater than or equal to 0. " +
+          "Using default value of : {}ms instead.", timeoutMs,
+          DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT, DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT);
+    }
   }
 
   @Override
   public boolean acquirePermit(String nsId) {
     try {
       LOG.debug("Taking lock for nameservice {}", nsId);
-      return this.permits.get(nsId).tryAcquire(1, TimeUnit.SECONDS);
+      return this.permits.get(nsId).tryAcquire(acquireTimeoutMs, TimeUnit.MILLISECONDS);
     } catch (InterruptedException e) {
       LOG.debug("Cannot get a permit for nameservice {}", nsId);
     }
@@ -62,6 +78,7 @@ public void releasePermit(String nsId) {
 
   @Override
   public void shutdown() {
+    LOG.debug("Shutting down router fairness policy controller");
     // drain all semaphores
     for (Semaphore sema: this.permits.values()) {
       sema.drainPermits();
@@ -75,4 +92,17 @@ protected void insertNameServiceWithPermits(String nsId, int maxPermits) {
   protected int getAvailablePermits(String nsId) {
     return this.permits.get(nsId).availablePermits();
   }
+
+  @Override
+  public String getAvailableHandlerOnPerNs() {
+    JSONObject json = new JSONObject();
+    permits.forEach((k, v) -> {
+      try {
+        json.put(k, v.availablePermits());
+      } catch (JSONException e) {
+        LOG.warn("Cannot put {} into JSONObject", k, e);
+      }
+    });
+    return json.toString();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java
index b6a7df48431a8..3b85da59e1f52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java
@@ -46,4 +46,9 @@ public void releasePermit(String nsId) {
   public void shutdown() {
     // Nothing for now.
   }
+
+  @Override
+  public String getAvailableHandlerOnPerNs(){
+    return "N/A";
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java
new file mode 100644
index 0000000000000..f7bc0e8f5a6e0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RefreshFairnessPolicyControllerHandler.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.fairness;
+
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.apache.hadoop.ipc.RefreshHandler;
+import org.apache.hadoop.ipc.RefreshResponse;
+
+public class RefreshFairnessPolicyControllerHandler implements RefreshHandler {
+
+  final static public String HANDLER_IDENTIFIER = "RefreshFairnessPolicyController";
+  private final Router router;
+
+  public RefreshFairnessPolicyControllerHandler(Router router) {
+    this.router = router;
+  }
+
+  @Override
+  public RefreshResponse handleRefresh(String identifier, String[] args) {
+    if (HANDLER_IDENTIFIER.equals(identifier)) {
+      return new RefreshResponse(0, router.getRpcServer().refreshFairnessPolicyController());
+    }
+    return new RefreshResponse(-1, "Failed");
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java
index 80e7d218b2a7d..354383a168f4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java
@@ -62,4 +62,9 @@ public interface RouterRpcFairnessPolicyController {
    * Shutdown steps to stop accepting new permission requests and clean-up.
    */
   void shutdown();
+
+  /**
+   * Returns the JSON string of the available handler for each Ns.
+   */
+  String getAvailableHandlerOnPerNs();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java
index b4e3dc36b0e37..35045bdca8e27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java
@@ -50,13 +50,10 @@ public StaticRouterRpcFairnessPolicyController(Configuration conf) {
     init(conf);
   }
 
-  public void init(Configuration conf)
-      throws IllegalArgumentException {
+  public void init(Configuration conf) throws IllegalArgumentException {
     super.init(conf);
     // Total handlers configured to process all incoming Rpc.
-    int handlerCount = conf.getInt(
-        DFS_ROUTER_HANDLER_COUNT_KEY,
-        DFS_ROUTER_HANDLER_COUNT_DEFAULT);
+    int handlerCount = conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY, DFS_ROUTER_HANDLER_COUNT_DEFAULT);
 
     LOG.info("Handlers available for fairness assignment {} ", handlerCount);
 
@@ -71,8 +68,7 @@ public void init(Configuration conf)
     allConfiguredNS.add(CONCURRENT_NS);
     validateHandlersCount(conf, handlerCount, allConfiguredNS);
     for (String nsId : allConfiguredNS) {
-      int dedicatedHandlers =
-          conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0);
+      int dedicatedHandlers = conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0);
       LOG.info("Dedicated handlers {} for ns {} ", dedicatedHandlers, nsId);
       if (dedicatedHandlers > 0) {
         handlerCount -= dedicatedHandlers;
@@ -86,7 +82,7 @@ public void init(Configuration conf)
     // Assign remaining handlers equally to remaining name services and
     // general pool if applicable.
     if (!unassignedNS.isEmpty()) {
-      LOG.info("Unassigned ns {}", unassignedNS.toString());
+      LOG.info("Unassigned ns {}", unassignedNS);
       int handlersPerNS = handlerCount / unassignedNS.size();
       LOG.info("Handlers available per ns {}", handlersPerNS);
       for (String nsId : unassignedNS) {
@@ -96,28 +92,25 @@ public void init(Configuration conf)
     }
 
     // Assign remaining handlers if any to fan out calls.
-    int leftOverHandlers = handlerCount % unassignedNS.size();
+    int leftOverHandlers = unassignedNS.isEmpty() ? handlerCount :
+        handlerCount % unassignedNS.size();
     int existingPermits = getAvailablePermits(CONCURRENT_NS);
     if (leftOverHandlers > 0) {
       LOG.info("Assigned extra {} handlers to commons pool", leftOverHandlers);
-      insertNameServiceWithPermits(CONCURRENT_NS,
-          existingPermits + leftOverHandlers);
+      insertNameServiceWithPermits(CONCURRENT_NS, existingPermits + leftOverHandlers);
     }
-    LOG.info("Final permit allocation for concurrent ns: {}",
-        getAvailablePermits(CONCURRENT_NS));
+    LOG.info("Final permit allocation for concurrent ns: {}", getAvailablePermits(CONCURRENT_NS));
   }
 
   private static void logAssignment(String nsId, int count) {
-    LOG.info("Assigned {} handlers to nsId {} ",
-        count, nsId);
+    LOG.info("Assigned {} handlers to nsId {} ", count, nsId);
   }
 
-  private void validateHandlersCount(Configuration conf, int handlerCount,
-                                     Set<String> allConfiguredNS) {
+  private void validateHandlersCount(Configuration conf,
+      int handlerCount, Set<String> allConfiguredNS) {
     int totalDedicatedHandlers = 0;
     for (String nsId : allConfiguredNS) {
-      int dedicatedHandlers =
-              conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0);
+      int dedicatedHandlers = conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0);
       if (dedicatedHandlers > 0) {
         // Total handlers should not be less than sum of dedicated handlers.
         totalDedicatedHandlers += dedicatedHandlers;
@@ -127,8 +120,7 @@ private void validateHandlersCount(Configuration conf, int handlerCount,
       }
     }
     if (totalDedicatedHandlers > handlerCount) {
-      String msg = String.format(ERROR_MSG, handlerCount,
-          totalDedicatedHandlers);
+      String msg = String.format(ERROR_MSG, handlerCount, totalDedicatedHandlers);
       LOG.error(msg);
       throw new IllegalArgumentException(msg);
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
index b9ea8709e90f9..ed3069af83633 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java
@@ -50,7 +50,7 @@ public interface FederationMBean {
 
   /**
    * Get the latest state of all routers.
-   * @return JSON with all of the known routers or null if failure.
+   * @return JSON with all the known routers or null if failure.
    */
   String getRouters();
 
@@ -343,4 +343,11 @@ public interface FederationMBean {
    * with the highest risk of loss.
    */
   long getHighestPriorityLowRedundancyECBlocks();
+
+  /**
+   * Returns the number of paths to be processed by storage policy satisfier.
+   *
+   * @return The number of paths to be processed by sps.
+   */
+  int getPendingSPSPaths();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
index a4469a3025a11..65c6c34eb2ff6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java
@@ -30,6 +30,10 @@ public interface FederationRPCMBean {
 
   long getProxyOps();
 
+  long getActiveProxyOps();
+
+  long getObserverProxyOps();
+
   double getProxyAvg();
 
   long getProcessingOps();
@@ -109,6 +113,12 @@ public interface FederationRPCMBean {
    */
   String getRpcClientConnections();
 
+  /**
+   * JSON representation of the available handler per Ns.
+   * @return JSON string representation.
+   */
+  String getAvailableHandlerOnPerNs();
+
   /**
    * Get the JSON representation of the async caller thread pool.
    * @return JSON string representation of the async caller thread pool.
@@ -120,4 +130,16 @@ public interface FederationRPCMBean {
    * @return Number of operations rejected due to lack of permits.
    */
   long getProxyOpPermitRejected();
+
+  /**
+   * Get the number of operations rejected due to lack of permits of each namespace.
+   * @return Number of operations rejected due to lack of permits of each namespace.
+   */
+  String getProxyOpPermitRejectedPerNs();
+
+  /**
+   * Get the number of operations accepted of each namespace.
+   * @return Number of operations accepted of each namespace.
+   */
+  String getProxyOpPermitAcceptedPerNs();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
index 1e6aa8050d92e..5d5f9fb8aa12a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java
@@ -21,6 +21,7 @@
 import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
 import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
@@ -49,7 +50,10 @@ public class FederationRPCMetrics implements FederationRPCMBean {
   private MutableRate proxy;
   @Metric("Number of operations the Router proxied to a Namenode")
   private MutableCounterLong proxyOp;
-
+  @Metric("Number of operations the Router proxied to a Active Namenode")
+  private MutableCounterLong activeProxyOp;
+  @Metric("Number of operations the Router proxied to a Observer Namenode")
+  private MutableCounterLong observerProxyOp;
   @Metric("Number of operations to hit a standby NN")
   private MutableCounterLong proxyOpFailureStandby;
   @Metric("Number of operations to fail to reach NN")
@@ -189,41 +193,49 @@ public long getRouterFailureLockedOps() {
   }
 
   @Override
+  @Metric({"RpcServerCallQueue", "Length of the rpc server call queue"})
   public int getRpcServerCallQueue() {
     return rpcServer.getServer().getCallQueueLen();
   }
 
   @Override
+  @Metric({"RpcServerNumOpenConnections", "Number of the rpc server open connections"})
   public int getRpcServerNumOpenConnections() {
     return rpcServer.getServer().getNumOpenConnections();
   }
 
   @Override
+  @Metric({"RpcClientNumConnections", "Number of the rpc client open connections"})
   public int getRpcClientNumConnections() {
     return rpcServer.getRPCClient().getNumConnections();
   }
 
   @Override
+  @Metric({"RpcClientNumActiveConnections", "Number of the rpc client active connections"})
   public int getRpcClientNumActiveConnections() {
     return rpcServer.getRPCClient().getNumActiveConnections();
   }
 
   @Override
+  @Metric({"RpcClientNumIdleConnections", "Number of the rpc client idle connections"})
   public int getRpcClientNumIdleConnections() {
     return rpcServer.getRPCClient().getNumIdleConnections();
   }
 
   @Override
+  @Metric({"RpcClientNumActiveConnectionsRecently", "Number of the rpc client active connections recently"})
   public int getRpcClientNumActiveConnectionsRecently() {
     return rpcServer.getRPCClient().getNumActiveConnectionsRecently();
   }
 
   @Override
+  @Metric({"RpcClientNumCreatingConnections", "Number of the rpc client creating connections"})
   public int getRpcClientNumCreatingConnections() {
     return rpcServer.getRPCClient().getNumCreatingConnections();
   }
 
   @Override
+  @Metric({"RpcClientNumConnectionPools", "Number of the rpc client connection pools"})
   public int getRpcClientNumConnectionPools() {
     return rpcServer.getRPCClient().getNumConnectionPools();
   }
@@ -233,6 +245,12 @@ public String getRpcClientConnections() {
     return rpcServer.getRPCClient().getJSON();
   }
 
+  @Override
+  public String getAvailableHandlerOnPerNs() {
+    return rpcServer.getRPCClient().
+        getRouterRpcFairnessPolicyController().getAvailableHandlerOnPerNs();
+  }
+
   @Override
   public String getAsyncCallerPool() {
     return rpcServer.getRPCClient().getAsyncCallerPoolJson();
@@ -242,9 +260,15 @@ public String getAsyncCallerPool() {
    * Add the time to proxy an operation from the moment the Router sends it to
    * the Namenode until it replied.
    * @param time Proxy time of an operation in nanoseconds.
+   * @param state NameNode state. Maybe null
    */
-  public void addProxyTime(long time) {
+  public void addProxyTime(long time, FederationNamenodeServiceState state) {
     proxy.add(time);
+    if(FederationNamenodeServiceState.ACTIVE == state) {
+      activeProxyOp.incr();
+    } else if (FederationNamenodeServiceState.OBSERVER == state) {
+      observerProxyOp.incr();
+    }
     proxyOp.incr();
   }
 
@@ -258,6 +282,16 @@ public long getProxyOps() {
     return proxyOp.value();
   }
 
+  @Override
+  public long getActiveProxyOps() {
+    return activeProxyOp.value();
+  }
+
+  @Override
+  public long getObserverProxyOps() {
+    return observerProxyOp.value();
+  }
+
   /**
    * Add the time to process a request in the Router from the time we receive
    * the call until we send it to the Namenode.
@@ -286,4 +320,14 @@ public void incrProxyOpPermitRejected() {
   public long getProxyOpPermitRejected() {
     return proxyOpPermitRejected.value();
   }
+
+  @Override
+  public String getProxyOpPermitRejectedPerNs() {
+    return rpcServer.getRPCClient().getRejectedPermitsPerNsJSON();
+  }
+
+  @Override
+  public String getProxyOpPermitAcceptedPerNs() {
+    return rpcServer.getRPCClient().getAcceptedPermitsPerNsJSON();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
index dc53f8a24e67c..b57fa070546e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.federation.metrics;
 
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
@@ -26,6 +28,8 @@
 import javax.management.StandardMBean;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
 import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor;
 import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
 import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
@@ -49,7 +53,7 @@ public class FederationRPCPerformanceMonitor implements RouterRpcMonitor {
 
   /** Time for an operation to be received in the Router. */
   private static final ThreadLocal<Long> START_TIME = new ThreadLocal<>();
-  /** Time for an operation to be send to the Namenode. */
+  /** Time for an operation to be sent to the Namenode. */
   private static final ThreadLocal<Long> PROXY_TIME = new ThreadLocal<>();
 
   /** Configuration for the performance monitor. */
@@ -61,6 +65,9 @@ public class FederationRPCPerformanceMonitor implements RouterRpcMonitor {
 
   /** JMX interface to monitor the RPC metrics. */
   private FederationRPCMetrics metrics;
+  /** JMX interface to monitor each Nameservice RPC metrics. */
+  private Map<String, NameserviceRPCMetrics> nameserviceRPCMetricsMap =
+      new ConcurrentHashMap<>();
   private ObjectName registeredBean;
 
   /** Thread pool for logging stats. */
@@ -77,6 +84,11 @@ public void init(Configuration configuration, RouterRpcServer rpcServer,
 
     // Create metrics
     this.metrics = FederationRPCMetrics.create(conf, server);
+    for (String nameservice : FederationUtil.getAllConfiguredNS(conf)) {
+      LOG.info("Create Nameservice RPC Metrics for " + nameservice);
+      this.nameserviceRPCMetricsMap.computeIfAbsent(nameservice,
+          k -> NameserviceRPCMetrics.create(conf, k));
+    }
 
     // Create thread pool
     ThreadFactory threadFactory = new ThreadFactoryBuilder()
@@ -136,27 +148,42 @@ public long proxyOp() {
   }
 
   @Override
-  public void proxyOpComplete(boolean success) {
+  public void proxyOpComplete(boolean success, String nsId,
+      FederationNamenodeServiceState state) {
     if (success) {
       long proxyTime = getProxyTime();
-      if (metrics != null && proxyTime >= 0) {
-        metrics.addProxyTime(proxyTime);
+      if (proxyTime >= 0) {
+        if (metrics != null) {
+          metrics.addProxyTime(proxyTime, state);
+        }
+        if (nameserviceRPCMetricsMap != null &&
+            nameserviceRPCMetricsMap.containsKey(nsId)) {
+          nameserviceRPCMetricsMap.get(nsId).addProxyTime(proxyTime);
+        }
       }
     }
   }
 
   @Override
-  public void proxyOpFailureStandby() {
+  public void proxyOpFailureStandby(String nsId) {
     if (metrics != null) {
       metrics.incrProxyOpFailureStandby();
     }
+    if (nameserviceRPCMetricsMap != null &&
+        nameserviceRPCMetricsMap.containsKey(nsId)) {
+      nameserviceRPCMetricsMap.get(nsId).incrProxyOpFailureStandby();
+    }
   }
 
   @Override
-  public void proxyOpFailureCommunicate() {
+  public void proxyOpFailureCommunicate(String nsId) {
     if (metrics != null) {
       metrics.incrProxyOpFailureCommunicate();
     }
+    if (nameserviceRPCMetricsMap != null &&
+        nameserviceRPCMetricsMap.containsKey(nsId)) {
+      nameserviceRPCMetricsMap.get(nsId).incrProxyOpFailureCommunicate();
+    }
   }
 
   @Override
@@ -181,10 +208,14 @@ public void proxyOpRetries() {
   }
 
   @Override
-  public void proxyOpNoNamenodes() {
+  public void proxyOpNoNamenodes(String nsId) {
     if (metrics != null) {
       metrics.incrProxyOpNoNamenodes();
     }
+    if (nameserviceRPCMetricsMap != null &&
+        nameserviceRPCMetricsMap.containsKey(nsId)) {
+      nameserviceRPCMetricsMap.get(nsId).incrProxyOpNoNamenodes();
+    }
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index c48728a923c0d..0c62922146311 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -874,6 +874,16 @@ public long getCurrentTokensCount() {
     return 0;
   }
 
+  @Override
+  public int getPendingSPSPaths() {
+    try {
+      return getRBFMetrics().getPendingSPSPaths();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of paths to be processed by sps", e);
+    }
+    return 0;
+  }
+
   private Router getRouter() throws IOException {
     if (this.router == null) {
       throw new IOException("Router is not initialized");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMBean.java
similarity index 64%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java
rename to hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMBean.java
index 7b4af0c3e094d..54a40bdd631d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/DisableEventTypeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMBean.java
@@ -15,28 +15,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.yarn.metrics;
+package org.apache.hadoop.hdfs.server.federation.metrics;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.metrics2.MetricsCollector;
-import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.classification.InterfaceStability;
 
+/**
+ * JMX interface for the RPC server of Nameservice.
+ */
 @InterfaceAudience.Private
-@Metrics(context="yarn")
-public class DisableEventTypeMetrics implements EventTypeMetrics {
-  @Override
-  public void increment(Enum type, long processingTimeUs) {
-    //nop
-    return;
-  }
-  @Override
-  public void getMetrics(MetricsCollector collector, boolean all) {
-    //nop
-    return;
-  }
-
-  @Override
-  public long get(Enum type) {
-    return 0;
-  }
+@InterfaceStability.Evolving
+public interface NameserviceRPCMBean {
+
+  long getProxyOps();
+
+  double getProxyAvg();
+
+  long getProxyOpFailureCommunicate();
+
+  long getProxyOpFailureStandby();
+
+  long getProxyOpNoNamenodes();
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMetrics.java
new file mode 100644
index 0000000000000..1cc0234780f8d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NameserviceRPCMetrics.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Implementation of the Nameservice RPC metrics collector.
+ */
+@Metrics(name = "NameserviceRPCActivity", about = "Nameservice RPC Activity",
+    context = "dfs")
+public class NameserviceRPCMetrics implements NameserviceRPCMBean {
+
+  public final static String NAMESERVICE_RPC_METRICS_PREFIX = "NameserviceActivity-";
+
+  private final String nsId;
+
+  @Metric("Time for the Router to proxy an operation to the Nameservice")
+  private MutableRate proxy;
+  @Metric("Number of operations the Router proxied to a NameService")
+  private MutableCounterLong proxyOp;
+
+  @Metric("Number of operations to hit a standby NN")
+  private MutableCounterLong proxyOpFailureStandby;
+  @Metric("Number of operations to fail to reach NN")
+  private MutableCounterLong proxyOpFailureCommunicate;
+  @Metric("Number of operations to hit no namenodes available")
+  private MutableCounterLong proxyOpNoNamenodes;
+
+  public NameserviceRPCMetrics(Configuration conf, String nsId) {
+    this.nsId = nsId;
+  }
+
+  public static NameserviceRPCMetrics create(Configuration conf,
+      String nameService) {
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    String name = NAMESERVICE_RPC_METRICS_PREFIX + (nameService.isEmpty()
+        ? "UndefinedNameService"+ ThreadLocalRandom.current().nextInt()
+        : nameService);
+    return ms.register(name, "HDFS Federation NameService RPC Metrics",
+        new NameserviceRPCMetrics(conf, name));
+  }
+
+  public void incrProxyOpFailureStandby() {
+    proxyOpFailureStandby.incr();
+  }
+
+  @Override
+  public long getProxyOpFailureStandby() {
+    return proxyOpFailureStandby.value();
+  }
+
+  public void incrProxyOpFailureCommunicate() {
+    proxyOpFailureCommunicate.incr();
+  }
+
+  @Override
+  public long getProxyOpFailureCommunicate() {
+    return proxyOpFailureCommunicate.value();
+  }
+
+  public void incrProxyOpNoNamenodes() {
+    proxyOpNoNamenodes.incr();
+  }
+
+  @Override
+  public long getProxyOpNoNamenodes() {
+    return proxyOpNoNamenodes.value();
+  }
+
+
+  /**
+   * Add the time to proxy an operation from the moment the Router sends it to
+   * the Namenode until it replied.
+   * @param time Proxy time of an operation in nanoseconds.
+   */
+  public void addProxyTime(long time) {
+    proxy.add(time);
+    proxyOp.incr();
+  }
+
+  @Override
+  public double getProxyAvg() {
+    return proxy.lastStat().mean();
+  }
+
+  @Override
+  public long getProxyOps() {
+    return proxyOp.value();
+  }
+
+  public String getNsId() {
+    return this.nsId;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java
index 5bba819fd0756..e1068394f6f42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.federation.metrics;
 
+import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
 import static org.apache.hadoop.util.Time.now;
 
 import java.io.IOException;
@@ -49,6 +50,7 @@
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -79,7 +81,11 @@
 import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.apache.hadoop.hdfs.server.federation.store.records.RouterState;
 import org.apache.hadoop.hdfs.server.federation.store.records.StateStoreVersion;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
@@ -100,12 +106,16 @@ public class RBFMetrics implements RouterMBean, FederationMBean {
   private static final Logger LOG =
       LoggerFactory.getLogger(RBFMetrics.class);
 
+  private final MetricsRegistry registry = new MetricsRegistry("RBFMetrics");
+
   /** Format for a date. */
   private static final String DATE_FORMAT = "yyyy/MM/dd HH:mm:ss";
 
   /** Prevent holding the page from load too long. */
   private final long timeOut;
 
+  /** Enable/Disable getNodeUsage. **/
+  private boolean enableGetDNUsage;
 
   /** Router interface. */
   private final Router router;
@@ -168,9 +178,20 @@ public RBFMetrics(Router router) throws IOException {
     Configuration conf = router.getConfig();
     this.timeOut = conf.getTimeDuration(RBFConfigKeys.DN_REPORT_TIME_OUT,
         RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS);
+    this.enableGetDNUsage = conf.getBoolean(RBFConfigKeys.DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY,
+        RBFConfigKeys.DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT);
     this.topTokenRealOwners = conf.getInt(
         RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY,
         RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY_DEFAULT);
+
+    registry.tag(ProcessName, "Router");
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    ms.register(RBFMetrics.class.getName(), "RBFActivity Metrics", this);
+  }
+
+  @VisibleForTesting
+  public void setEnableGetDNUsage(boolean enableGetDNUsage) {
+    this.enableGetDNUsage = enableGetDNUsage;
   }
 
   /**
@@ -183,6 +204,8 @@ public void close() {
     if (this.federationBeanName != null) {
       MBeans.unregister(federationBeanName);
     }
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    ms.unregisterSource(RBFMetrics.class.getName());
   }
 
   @Override
@@ -277,7 +300,7 @@ public String getMountTable() {
 
       // Dump mount table entries information into JSON
       for (MountTable entry : orderedMounts) {
-        // Sumarize destinations
+        // Summarize destinations
         Set<String> nameservices = new LinkedHashSet<>();
         Set<String> paths = new LinkedHashSet<>();
         for (RemoteLocation location : entry.getDestinations()) {
@@ -458,53 +481,65 @@ public int getNumExpiredNamenodes() {
   }
 
   @Override
+  @Metric({"NumLiveNodes", "Number of live data nodes"})
   public int getNumLiveNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfActiveDatanodes);
   }
 
   @Override
+  @Metric({"NumDeadNodes", "Number of dead data nodes"})
   public int getNumDeadNodes() {
     return getNameserviceAggregatedInt(MembershipStats::getNumOfDeadDatanodes);
   }
 
   @Override
+  @Metric({"NumStaleNodes", "Number of stale data nodes"})
   public int getNumStaleNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfStaleDatanodes);
   }
 
   @Override
+  @Metric({"NumDecommissioningNodes", "Number of Decommissioning data nodes"})
   public int getNumDecommissioningNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfDecommissioningDatanodes);
   }
 
   @Override
+  @Metric({"NumDecomLiveNodes", "Number of decommissioned Live data nodes"})
   public int getNumDecomLiveNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfDecomActiveDatanodes);
   }
 
   @Override
+  @Metric({"NumDecomDeadNodes", "Number of decommissioned dead data nodes"})
   public int getNumDecomDeadNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfDecomDeadDatanodes);
   }
 
   @Override
+  @Metric({"NumInMaintenanceLiveDataNodes",
+      "Number of IN_MAINTENANCE live data nodes"})
   public int getNumInMaintenanceLiveDataNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfInMaintenanceLiveDataNodes);
   }
 
   @Override
+  @Metric({"NumInMaintenanceDeadDataNodes",
+      "Number of IN_MAINTENANCE dead data nodes"})
   public int getNumInMaintenanceDeadDataNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfInMaintenanceDeadDataNodes);
   }
 
   @Override
+  @Metric({"NumEnteringMaintenanceDataNodes",
+      "Number of ENTERING_MAINTENANCE data nodes"})
   public int getNumEnteringMaintenanceDataNodes() {
     return getNameserviceAggregatedInt(
         MembershipStats::getNumOfEnteringMaintenanceDataNodes);
@@ -512,35 +547,34 @@ public int getNumEnteringMaintenanceDataNodes() {
 
   @Override // NameNodeMXBean
   public String getNodeUsage() {
-    float median = 0;
-    float max = 0;
-    float min = 0;
-    float dev = 0;
+    double median = 0;
+    double max = 0;
+    double min = 0;
+    double dev = 0;
 
     final Map<String, Map<String, Object>> info = new HashMap<>();
     try {
-      RouterRpcServer rpcServer = this.router.getRpcServer();
-      DatanodeInfo[] live = rpcServer.getDatanodeReport(
-          DatanodeReportType.LIVE, false, timeOut);
+      DatanodeInfo[] live = null;
+      if (this.enableGetDNUsage) {
+        RouterRpcServer rpcServer = this.router.getRpcServer();
+        live = rpcServer.getDatanodeReport(DatanodeReportType.LIVE, false, timeOut);
+      } else {
+        LOG.debug("Getting node usage is disabled.");
+      }
 
-      if (live.length > 0) {
-        float totalDfsUsed = 0;
-        float[] usages = new float[live.length];
+      if (live != null && live.length > 0) {
+        double[] usages = new double[live.length];
         int i = 0;
         for (DatanodeInfo dn : live) {
           usages[i++] = dn.getDfsUsedPercent();
-          totalDfsUsed += dn.getDfsUsedPercent();
         }
-        totalDfsUsed /= live.length;
         Arrays.sort(usages);
         median = usages[usages.length / 2];
         max = usages[usages.length - 1];
         min = usages[0];
 
-        for (i = 0; i < usages.length; i++) {
-          dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed);
-        }
-        dev = (float) Math.sqrt(dev / usages.length);
+        StandardDeviation deviation = new StandardDeviation();
+        dev = deviation.evaluate(usages);
       }
     } catch (IOException e) {
       LOG.error("Cannot get the live nodes: {}", e.getMessage());
@@ -557,34 +591,41 @@ public String getNodeUsage() {
   }
 
   @Override
+  @Metric({"NumBlocks", "Total number of blocks"})
   public long getNumBlocks() {
     return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocks);
   }
 
   @Override
+  @Metric({"NumOfMissingBlocks", "Number of missing blocks"})
   public long getNumOfMissingBlocks() {
     return getNameserviceAggregatedLong(MembershipStats::getNumOfBlocksMissing);
   }
 
   @Override
+  @Metric({"NumOfBlocksPendingReplication",
+      "Number of blocks pending replication"})
   public long getNumOfBlocksPendingReplication() {
     return getNameserviceAggregatedLong(
         MembershipStats::getNumOfBlocksPendingReplication);
   }
 
   @Override
+  @Metric({"NumOfBlocksUnderReplicated", "Number of blocks under replication"})
   public long getNumOfBlocksUnderReplicated() {
     return getNameserviceAggregatedLong(
         MembershipStats::getNumOfBlocksUnderReplicated);
   }
 
   @Override
+  @Metric({"NumOfBlocksPendingDeletion", "Number of blocks pending deletion"})
   public long getNumOfBlocksPendingDeletion() {
     return getNameserviceAggregatedLong(
         MembershipStats::getNumOfBlocksPendingDeletion);
   }
 
   @Override
+  @Metric({"NumFiles", "Number of files"})
   public long getNumFiles() {
     return getNameserviceAggregatedLong(MembershipStats::getNumOfFiles);
   }
@@ -659,6 +700,7 @@ public String getRouterStatus() {
   }
 
   @Override
+  @Metric({"CurrentTokensCount", "Number of router's current tokens"})
   public long getCurrentTokensCount() {
     RouterSecurityManager mgr =
         this.router.getRpcServer().getRouterSecurityManager();
@@ -714,11 +756,19 @@ public long getHighestPriorityLowRedundancyECBlocks() {
   }
 
   @Override
+  public int getPendingSPSPaths() {
+    return getNameserviceAggregatedInt(
+        MembershipStats::getPendingSPSPaths);
+  }
+
+  @Override
+  @Metric({"RouterFederationRenameCount", "Number of federation rename"})
   public int getRouterFederationRenameCount() {
     return this.router.getRpcServer().getRouterFederationRenameCount();
   }
 
   @Override
+  @Metric({"SchedulerJobCount", "Number of scheduler job"})
   public int getSchedulerJobCount() {
     return this.router.getRpcServer().getSchedulerJobCount();
   }
@@ -836,7 +886,7 @@ private List<MembershipState> getActiveNamenodeRegistrations()
       // Fetch the most recent namenode registration
       String nsId = nsInfo.getNameserviceId();
       List<? extends FederationNamenodeContext> nns =
-          namenodeResolver.getNamenodesForNameserviceId(nsId);
+          namenodeResolver.getNamenodesForNameserviceId(nsId, false);
       if (nns != null) {
         FederationNamenodeContext nn = nns.get(0);
         if (nn instanceof MembershipState) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
index f06df70b517cf..cae1f478604d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
@@ -43,6 +43,17 @@
 @InterfaceStability.Evolving
 public interface ActiveNamenodeResolver {
 
+  /**
+   * Report a failed, unavailable NN address for a nameservice or blockPool.
+   *
+   * @param ns Nameservice identifier.
+   * @param failedAddress The address the failed responded to the command.
+   *
+   * @throws IOException If the state store cannot be accessed.
+   */
+  void updateUnavailableNamenode(
+      String ns, InetSocketAddress failedAddress) throws IOException;
+
   /**
    * Report a successful, active NN address for a nameservice or blockPool.
    *
@@ -56,20 +67,30 @@ void updateActiveNamenode(
 
   /**
    * Returns a prioritized list of the most recent cached registration entries
-   * for a single nameservice ID.
-   * Returns an empty list if none are found. Returns entries in preference of:
+   * for a single nameservice ID. Returns an empty list if none are found.
+   * In the case of not observerRead Returns entries in preference of :
    * <ul>
    * <li>The most recent ACTIVE NN
+   * <li>The most recent OBSERVER NN
+   * <li>The most recent STANDBY NN
+   * <li>The most recent UNAVAILABLE NN
+   * </ul>
+   *
+   * In the case of observerRead Returns entries in preference of :
+   * <ul>
+   * <li>The most recent OBSERVER NN
+   * <li>The most recent ACTIVE NN
    * <li>The most recent STANDBY NN
    * <li>The most recent UNAVAILABLE NN
    * </ul>
    *
    * @param nameserviceId Nameservice identifier.
+   * @param listObserversFirst Observer read case, observer NN will be ranked first
    * @return Prioritized list of namenode contexts.
    * @throws IOException If the state store cannot be accessed.
    */
-  List<? extends FederationNamenodeContext>
-      getNamenodesForNameserviceId(String nameserviceId) throws IOException;
+  List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
+      String nameserviceId, boolean listObserversFirst) throws IOException;
 
   /**
    * Returns a prioritized list of the most recent cached registration entries
@@ -77,6 +98,7 @@ void updateActiveNamenode(
    * Returns an empty list if none are found. Returns entries in preference of:
    * <ul>
    * <li>The most recent ACTIVE NN
+   * <li>The most recent OBSERVER NN
    * <li>The most recent STANDBY NN
    * <li>The most recent UNAVAILABLE NN
    * </ul>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
index 13593e694a80e..862f851fb59bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java
@@ -19,6 +19,7 @@
 
 import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE;
 import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED;
+import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.OBSERVER;
 import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE;
 
 import java.io.IOException;
@@ -32,6 +33,7 @@
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore;
 import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
@@ -73,8 +75,11 @@ public class MembershipNamenodeResolver
   /** Parent router ID. */
   private String routerId;
 
-  /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */
-  private Map<String, List<? extends FederationNamenodeContext>> cacheNS;
+  /** Cached lookup of namenodes for nameservice. The keys are a pair of the nameservice
+   * name and a boolean indicating if observer namenodes should be listed first.
+   * If true, observer namenodes are listed first. If false, active namenodes are listed first.
+   *  Invalidated on cache refresh. */
+  private Map<Pair<String,Boolean>, List<? extends FederationNamenodeContext>> cacheNS;
   /** Cached lookup of NN for block pool. Invalidated on cache refresh. */
   private Map<String, List<? extends FederationNamenodeContext>> cacheBP;
 
@@ -123,9 +128,13 @@ public boolean loadCache(boolean force) {
     // Our cache depends on the store, update it first
     try {
       MembershipStore membership = getMembershipStore();
-      membership.loadCache(force);
+      if (!membership.loadCache(force)) {
+        return false;
+      }
       DisabledNameserviceStore disabled = getDisabledNameserviceStore();
-      disabled.loadCache(force);
+      if (!disabled.loadCache(force)) {
+        return false;
+      }
     } catch (IOException e) {
       LOG.error("Cannot update membership from the State Store", e);
     }
@@ -136,11 +145,21 @@ public boolean loadCache(boolean force) {
     return true;
   }
 
+  @Override public void updateUnavailableNamenode(String nsId,
+      InetSocketAddress address) throws IOException {
+    updateNameNodeState(nsId, address, UNAVAILABLE);
+  }
+
   @Override
   public void updateActiveNamenode(
       final String nsId, final InetSocketAddress address) throws IOException {
+    updateNameNodeState(nsId, address, ACTIVE);
+  }
+
 
-    // Called when we have an RPC miss and successful hit on an alternate NN.
+  private void updateNameNodeState(final String nsId,
+      final InetSocketAddress address, FederationNamenodeServiceState state)
+      throws IOException {
     // Temporarily update our cache, it will be overwritten on the next update.
     try {
       MembershipState partial = MembershipState.newInstance();
@@ -160,10 +179,11 @@ public void updateActiveNamenode(
         MembershipState record = records.get(0);
         UpdateNamenodeRegistrationRequest updateRequest =
             UpdateNamenodeRegistrationRequest.newInstance(
-                record.getNameserviceId(), record.getNamenodeId(), ACTIVE);
+                record.getNameserviceId(), record.getNamenodeId(), state);
         membership.updateNamenodeRegistration(updateRequest);
 
-        cacheNS.remove(nsId);
+        cacheNS.remove(Pair.of(nsId, Boolean.TRUE));
+        cacheNS.remove(Pair.of(nsId, Boolean.FALSE));
         // Invalidating the full cacheBp since getting the blockpool id from
         // namespace id is quite costly.
         cacheBP.clear();
@@ -175,9 +195,9 @@ public void updateActiveNamenode(
 
   @Override
   public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
-      final String nsId) throws IOException {
+      final String nsId, boolean listObserversFirst) throws IOException {
 
-    List<? extends FederationNamenodeContext> ret = cacheNS.get(nsId);
+    List<? extends FederationNamenodeContext> ret = cacheNS.get(Pair.of(nsId, listObserversFirst));
     if (ret != null) {
       return ret;
     }
@@ -189,7 +209,8 @@ public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
       partial.setNameserviceId(nsId);
       GetNamenodeRegistrationsRequest request =
           GetNamenodeRegistrationsRequest.newInstance(partial);
-      result = getRecentRegistrationForQuery(request, true, false);
+      result = getRecentRegistrationForQuery(request, true,
+          false, listObserversFirst);
     } catch (StateStoreUnavailableException e) {
       LOG.error("Cannot get active NN for {}, State Store unavailable", nsId);
       return null;
@@ -218,7 +239,7 @@ public List<? extends FederationNamenodeContext> getNamenodesForNameserviceId(
 
     // Cache the response
     ret = Collections.unmodifiableList(result);
-    cacheNS.put(nsId, result);
+    cacheNS.put(Pair.of(nsId, listObserversFirst), result);
     return ret;
   }
 
@@ -235,7 +256,7 @@ public List<? extends FederationNamenodeContext> getNamenodesForBlockPoolId(
             GetNamenodeRegistrationsRequest.newInstance(partial);
 
         final List<MembershipState> result =
-            getRecentRegistrationForQuery(request, true, false);
+            getRecentRegistrationForQuery(request, true, false, false);
         if (result == null || result.isEmpty()) {
           LOG.error("Cannot locate eligible NNs for {}", bpId);
         } else {
@@ -306,6 +327,7 @@ public boolean registerNamenode(NamenodeStatusReport report)
           report.getHighestPriorityLowRedundancyReplicatedBlocks());
       stats.setHighestPriorityLowRedundancyECBlocks(
           report.getHighestPriorityLowRedundancyECBlocks());
+      stats.setPendingSPSPaths(report.getPendingSPSPaths());
       record.setStats(stats);
     }
 
@@ -345,22 +367,34 @@ public Set<String> getDisabledNamespaces() throws IOException {
   }
 
   /**
-   * Picks the most relevant record registration that matches the query. Return
-   * registrations matching the query in this preference: 1) Most recently
-   * updated ACTIVE registration 2) Most recently updated STANDBY registration
-   * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if
-   * showUnavailable). EXPIRED registrations are ignored.
+   * Picks the most relevant record registration that matches the query.
+   * If not observer read,
+   * return registrations matching the query in this preference:
+   * 1) Most recently updated ACTIVE registration
+   * 2) Most recently updated Observer registration
+   * 3) Most recently updated STANDBY registration (if showStandby)
+   * 4) Most recently updated UNAVAILABLE registration (if showUnavailable).
+   *
+   * If observer read,
+   * return registrations matching the query in this preference:
+   * 1) Observer registrations, shuffled to disperse queries.
+   * 2) Most recently updated ACTIVE registration
+   * 3) Most recently updated STANDBY registration (if showStandby)
+   * 4) Most recently updated UNAVAILABLE registration (if showUnavailable).
+   *
+   * EXPIRED registrations are ignored.
    *
    * @param request The select query for NN registrations.
    * @param addUnavailable include UNAVAILABLE registrations.
    * @param addExpired include EXPIRED registrations.
+   * @param observerRead  Observer read case, observer NN will be ranked first
    * @return List of memberships or null if no registrations that
    *         both match the query AND the selected states.
    * @throws IOException
    */
   private List<MembershipState> getRecentRegistrationForQuery(
       GetNamenodeRegistrationsRequest request, boolean addUnavailable,
-      boolean addExpired) throws IOException {
+      boolean addExpired, boolean observerRead) throws IOException {
 
     // Retrieve a list of all registrations that match this query.
     // This may include all NN records for a namespace/blockpool, including
@@ -370,24 +404,34 @@ private List<MembershipState> getRecentRegistrationForQuery(
         membershipStore.getNamenodeRegistrations(request);
 
     List<MembershipState> memberships = response.getNamenodeMemberships();
-    if (!addExpired || !addUnavailable) {
-      Iterator<MembershipState> iterator = memberships.iterator();
-      while (iterator.hasNext()) {
-        MembershipState membership = iterator.next();
-        if (membership.getState() == EXPIRED && !addExpired) {
-          iterator.remove();
-        } else if (membership.getState() == UNAVAILABLE && !addUnavailable) {
-          iterator.remove();
-        }
+    List<MembershipState> observerMemberships = new ArrayList<>();
+    Iterator<MembershipState> iterator = memberships.iterator();
+    while (iterator.hasNext()) {
+      MembershipState membership = iterator.next();
+      if (membership.getState() == EXPIRED && !addExpired) {
+        iterator.remove();
+      } else if (membership.getState() == UNAVAILABLE && !addUnavailable) {
+        iterator.remove();
+      } else if (membership.getState() == OBSERVER && observerRead) {
+        iterator.remove();
+        observerMemberships.add(membership);
       }
     }
 
-    List<MembershipState> priorityList = new ArrayList<>();
-    priorityList.addAll(memberships);
-    Collections.sort(priorityList, new NamenodePriorityComparator());
+    memberships.sort(new NamenodePriorityComparator());
+    if(observerRead) {
+      List<MembershipState> ret = new ArrayList<>(
+          memberships.size() + observerMemberships.size());
+      if(observerMemberships.size() > 1) {
+        Collections.shuffle(observerMemberships);
+      }
+      ret.addAll(observerMemberships);
+      ret.addAll(memberships);
+      memberships = ret;
+    }
 
-    LOG.debug("Selected most recent NN {} for query", priorityList);
-    return priorityList;
+    LOG.debug("Selected most recent NN {} for query", memberships);
+    return memberships;
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
index 9c0f33763f208..1fdd4cdfba8f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
@@ -359,7 +359,7 @@ public static boolean isTrashPath(String path) throws IOException {
   public static String getTrashRoot() throws IOException {
     // Gets the Trash directory for the current user.
     return FileSystem.USER_HOME_PREFIX + "/" +
-        RouterRpcServer.getRemoteUser().getUserName() + "/" +
+        RouterRpcServer.getRemoteUser().getShortUserName() + "/" +
         FileSystem.TRASH_PREFIX;
   }
 
@@ -398,7 +398,9 @@ public boolean loadCache(boolean force) {
     try {
       // Our cache depends on the store, update it first
       MountTableStore mountTable = this.getMountTableStore();
-      mountTable.loadCache(force);
+      if (!mountTable.loadCache(force)) {
+        return false;
+      }
 
       GetMountTableEntriesRequest request =
           GetMountTableEntriesRequest.newInstance("/");
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
index feb5a86dba83b..d7da11e6420ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java
@@ -75,6 +75,7 @@ public class NamenodeStatusReport {
   private long numberOfMissingBlocksWithReplicationFactorOne = -1;
   private long highestPriorityLowRedundancyReplicatedBlocks = -1;
   private long highestPriorityLowRedundancyECBlocks = -1;
+  private int pendingSPSPaths = -1;
 
   /** If the fields are valid. */
   private boolean registrationValid = false;
@@ -367,12 +368,13 @@ public int getNumEnteringMaintenanceDataNodes() {
    * @param numBlocksPendingReplication Number of blocks pending replication.
    * @param numBlocksUnderReplicated Number of blocks under replication.
    * @param numBlocksPendingDeletion Number of blocks pending deletion.
-   * @param providedSpace Space in provided storage.
+   * @param providedStorageSpace Space in provided storage.
+   * @param numPendingSPSPaths The number of paths to be processed by storage policy satisfier.
    */
   public void setNamesystemInfo(long available, long total,
       long numFiles, long numBlocks, long numBlocksMissing,
       long numBlocksPendingReplication, long numBlocksUnderReplicated,
-      long numBlocksPendingDeletion, long providedSpace) {
+      long numBlocksPendingDeletion, long providedStorageSpace, int numPendingSPSPaths) {
     this.totalSpace = total;
     this.availableSpace = available;
     this.numOfBlocks = numBlocks;
@@ -382,7 +384,8 @@ public void setNamesystemInfo(long available, long total,
     this.numOfBlocksPendingDeletion = numBlocksPendingDeletion;
     this.numOfFiles = numFiles;
     this.statsValid = true;
-    this.providedSpace = providedSpace;
+    this.providedSpace = providedStorageSpace;
+    this.pendingSPSPaths = numPendingSPSPaths;
   }
 
   /**
@@ -460,6 +463,15 @@ public long getHighestPriorityLowRedundancyECBlocks() {
     return this.highestPriorityLowRedundancyECBlocks;
   }
 
+  /**
+   * Returns the number of paths to be processed by storage policy satisfier.
+   *
+   * @return The number of paths to be processed by sps.
+   */
+  public int getPendingSPSPaths() {
+    return this.pendingSPSPaths;
+  }
+
   /**
    * Get the number of blocks.
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java
index 88e20649506dc..591ac5b3c37ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java
@@ -39,7 +39,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Order the destinations based on available space. This resolver uses a
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
index 9a5434b91ce2f..df026bb66efd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java
@@ -20,6 +20,7 @@
 import java.net.InetSocketAddress;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.util.Time;
@@ -53,9 +54,14 @@ public class ConnectionContext {
   private long lastActiveTs = 0;
   /** The connection's active status would expire after this window. */
   private final static long ACTIVE_WINDOW_TIME = TimeUnit.SECONDS.toMillis(30);
+  /** The maximum number of requests that this connection can handle concurrently. **/
+  private final int maxConcurrencyPerConn;
 
-  public ConnectionContext(ProxyAndInfo<?> connection) {
+  public ConnectionContext(ProxyAndInfo<?> connection, Configuration conf) {
     this.client = connection;
+    this.maxConcurrencyPerConn = conf.getInt(
+        RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY,
+        RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_DEFAULT);
   }
 
   /**
@@ -93,6 +99,23 @@ public synchronized boolean isClosed() {
    * @return True if the connection can be used.
    */
   public synchronized boolean isUsable() {
+    return hasAvailableConcurrency() && !isClosed();
+  }
+
+  /**
+   * Return true if this connection context still has available concurrency,
+   * else return false.
+   */
+  private synchronized boolean hasAvailableConcurrency() {
+    return this.numThreads < maxConcurrencyPerConn;
+  }
+
+  /**
+   *  Check if the connection is idle. It checks if the connection is not used
+   *  by another thread.
+   * @return True if the connection is not used by another thread.
+   */
+  public synchronized boolean isIdle() {
     return !isActive() && !isClosed();
   }
 
@@ -124,7 +147,7 @@ public synchronized void release() {
    */
   public synchronized void close(boolean force) {
     if (!force && this.numThreads > 0) {
-      // this is an erroneous case but we have to close the connection
+      // this is an erroneous case, but we have to close the connection
       // anyway since there will be connection leak if we don't do so
       // the connection has been moved out of the pool
       LOG.error("Active connection with {} handlers will be closed",
@@ -132,7 +155,7 @@ public synchronized void close(boolean force) {
     }
     this.closed = true;
     Object proxy = this.client.getProxy();
-    // Nobody should be using this anymore so it should close right away
+    // Nobody should be using this anymore, so it should close right away
     RPC.stopProxy(proxy);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
index aad272f5831fc..eeaa9cd4b34eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java
@@ -73,6 +73,14 @@ public class ConnectionManager {
 
   /** Queue for creating new connections. */
   private final BlockingQueue<ConnectionPool> creatorQueue;
+  /**
+   * Global federated namespace context for router.
+   */
+  private final RouterStateIdContext routerStateIdContext;
+  /**
+   * Map from connection pool ID to namespace.
+   */
+  private final Map<ConnectionPoolId, String> connectionPoolToNamespaceMap;
   /** Max size of queue for creating new connections. */
   private final int creatorQueueMaxSize;
 
@@ -85,15 +93,19 @@ public class ConnectionManager {
   /** If the connection manager is running. */
   private boolean running = false;
 
+  public ConnectionManager(Configuration config) {
+    this(config, new RouterStateIdContext(config));
+  }
 
   /**
    * Creates a proxy client connection pool manager.
    *
    * @param config Configuration for the connections.
    */
-  public ConnectionManager(Configuration config) {
+  public ConnectionManager(Configuration config, RouterStateIdContext routerStateIdContext) {
     this.conf = config;
-
+    this.routerStateIdContext = routerStateIdContext;
+    this.connectionPoolToNamespaceMap = new HashMap<>();
     // Configure minimum, maximum and active connection pools
     this.maxSize = this.conf.getInt(
         RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE,
@@ -135,12 +147,12 @@ public void start() {
     this.creator.start();
 
     // Schedule a task to remove stale connection pools and sockets
-    long recyleTimeMs = Math.min(
+    long recycleTimeMs = Math.min(
         poolCleanupPeriodMs, connectionCleanupPeriodMs);
     LOG.info("Cleaning every {} seconds",
-        TimeUnit.MILLISECONDS.toSeconds(recyleTimeMs));
+        TimeUnit.MILLISECONDS.toSeconds(recycleTimeMs));
     this.cleaner.scheduleAtFixedRate(
-        new CleanupTask(), 0, recyleTimeMs, TimeUnit.MILLISECONDS);
+        new CleanupTask(), 0, recycleTimeMs, TimeUnit.MILLISECONDS);
 
     // Mark the manager as running
     this.running = true;
@@ -160,6 +172,10 @@ public void close() {
         pool.close();
       }
       this.pools.clear();
+      for (String nsID: connectionPoolToNamespaceMap.values()) {
+        routerStateIdContext.removeNamespaceStateId(nsID);
+      }
+      connectionPoolToNamespaceMap.clear();
     } finally {
       writeLock.unlock();
     }
@@ -172,12 +188,12 @@ public void close() {
    * @param ugi User group information.
    * @param nnAddress Namenode address for the connection.
    * @param protocol Protocol for the connection.
+   * @param nsId Nameservice identity.
    * @return Proxy client to connect to nnId as UGI.
    * @throws IOException If the connection cannot be obtained.
    */
   public ConnectionContext getConnection(UserGroupInformation ugi,
-      String nnAddress, Class<?> protocol) throws IOException {
-
+      String nnAddress, Class<?> protocol, String nsId) throws IOException {
     // Check if the manager is shutdown
     if (!this.running) {
       LOG.error(
@@ -205,14 +221,19 @@ public ConnectionContext getConnection(UserGroupInformation ugi,
         if (pool == null) {
           pool = new ConnectionPool(
               this.conf, nnAddress, ugi, this.minSize, this.maxSize,
-              this.minActiveRatio, protocol);
+              this.minActiveRatio, protocol,
+              new PoolAlignmentContext(this.routerStateIdContext, nsId));
           this.pools.put(connectionId, pool);
+          this.connectionPoolToNamespaceMap.put(connectionId, nsId);
         }
       } finally {
         writeLock.unlock();
       }
     }
 
+    long clientStateId = RouterStateIdContext.getClientStateIdFromCurrentCall(nsId);
+    pool.getPoolAlignmentContext().advanceClientStateId(clientStateId);
+
     ConnectionContext conn = pool.getConnection();
 
     // Add a new connection to the pool if it wasn't usable
@@ -364,9 +385,9 @@ void cleanup(ConnectionPool pool) {
       long timeSinceLastActive = Time.now() - pool.getLastActiveTime();
       int total = pool.getNumConnections();
       // Active is a transient status in many cases for a connection since
-      // the handler thread uses the connection very quickly. Thus the number
+      // the handler thread uses the connection very quickly. Thus, the number
       // of connections with handlers using at the call time is constantly low.
-      // Recently active is more lasting status and it shows how many
+      // Recently active is more lasting status, and it shows how many
       // connections have been used with a recent time period. (i.e. 30 seconds)
       int active = pool.getNumActiveConnectionsRecently();
       float poolMinActiveRatio = pool.getMinActiveRatio();
@@ -376,9 +397,9 @@ void cleanup(ConnectionPool pool) {
         // The number should at least be 1
         int targetConnectionsCount = Math.max(1,
             (int)(poolMinActiveRatio * total) - active);
-        List<ConnectionContext> conns =
+        List<ConnectionContext> connections =
             pool.removeConnections(targetConnectionsCount);
-        for (ConnectionContext conn : conns) {
+        for (ConnectionContext conn : connections) {
           conn.close();
         }
         LOG.debug("Removed connection {} used {} seconds ago. " +
@@ -430,6 +451,11 @@ public void run() {
         try {
           for (ConnectionPoolId poolId : toRemove) {
             pools.remove(poolId);
+            String nsID = connectionPoolToNamespaceMap.get(poolId);
+            connectionPoolToNamespaceMap.remove(poolId);
+            if (!connectionPoolToNamespaceMap.values().contains(nsID)) {
+              routerStateIdContext.removeNamespaceStateId(nsID);
+            }
           }
         } finally {
           writeLock.unlock();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
index 293a4b64d2031..ef3580b35d6f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java
@@ -32,6 +32,7 @@
 import javax.net.SocketFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.ipc.AlignmentContext;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -77,7 +78,6 @@ public class ConnectionPool {
   private static final Logger LOG =
       LoggerFactory.getLogger(ConnectionPool.class);
 
-
   /** Configuration settings for the connection pool. */
   private final Configuration conf;
 
@@ -94,6 +94,8 @@ public class ConnectionPool {
   private volatile List<ConnectionContext> connections = new ArrayList<>();
   /** Connection index for round-robin. */
   private final AtomicInteger clientIndex = new AtomicInteger(0);
+  /** Underlying socket index. **/
+  private final AtomicInteger socketIndex = new AtomicInteger(0);
 
   /** Min number of connections per user. */
   private final int minSize;
@@ -105,6 +107,11 @@ public class ConnectionPool {
   /** The last time a connection was active. */
   private volatile long lastActiveTime = 0;
 
+  /** Enable using multiple physical socket or not. **/
+  private final boolean enableMultiSocket;
+  /** StateID alignment context. */
+  private final PoolAlignmentContext alignmentContext;
+
   /** Map for the protocols and their protobuf implementations. */
   private final static Map<Class<?>, ProtoImpl> PROTO_MAP = new HashMap<>();
   static {
@@ -134,7 +141,8 @@ private static class ProtoImpl {
 
   protected ConnectionPool(Configuration config, String address,
       UserGroupInformation user, int minPoolSize, int maxPoolSize,
-      float minActiveRatio, Class<?> proto) throws IOException {
+      float minActiveRatio, Class<?> proto, PoolAlignmentContext alignmentContext)
+      throws IOException {
 
     this.conf = config;
 
@@ -149,9 +157,14 @@ protected ConnectionPool(Configuration config, String address,
     this.minSize = minPoolSize;
     this.maxSize = maxPoolSize;
     this.minActiveRatio = minActiveRatio;
+    this.enableMultiSocket = conf.getBoolean(
+        RBFConfigKeys.DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_KEY,
+        RBFConfigKeys.DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_DEFAULT);
+
+    this.alignmentContext = alignmentContext;
 
     // Add minimum connections to the pool
-    for (int i=0; i<this.minSize; i++) {
+    for (int i = 0; i < this.minSize; i++) {
       ConnectionContext newConnection = newConnection();
       this.connections.add(newConnection);
     }
@@ -204,30 +217,37 @@ public AtomicInteger getClientIndex() {
     return this.clientIndex;
   }
 
+  /**
+   * Get the alignment context for this pool.
+   * @return Alignment context
+   */
+  public PoolAlignmentContext getPoolAlignmentContext() {
+    return this.alignmentContext;
+  }
+
   /**
    * Return the next connection round-robin.
    *
    * @return Connection context.
    */
   protected ConnectionContext getConnection() {
-
     this.lastActiveTime = Time.now();
-
-    // Get a connection from the pool following round-robin
-    ConnectionContext conn = null;
     List<ConnectionContext> tmpConnections = this.connections;
-    int size = tmpConnections.size();
-    // Inc and mask off sign bit, lookup index should be non-negative int
-    int threadIndex = this.clientIndex.getAndIncrement() & 0x7FFFFFFF;
-    for (int i=0; i<size; i++) {
-      int index = (threadIndex + i) % size;
-      conn = tmpConnections.get(index);
-      if (conn != null && conn.isUsable()) {
-        return conn;
+    for (ConnectionContext tmpConnection : tmpConnections) {
+      if (tmpConnection != null && tmpConnection.isUsable()) {
+        return tmpConnection;
       }
     }
 
-    // We return a connection even if it's active
+    ConnectionContext conn = null;
+    // We return a connection even if it's busy
+    int size = tmpConnections.size();
+    if (size > 0) {
+      // Get a connection from the pool following round-robin
+      // Inc and mask off sign bit, lookup index should be non-negative int
+      int threadIndex = this.clientIndex.getAndIncrement() & 0x7FFFFFFF;
+      conn = tmpConnections.get(threadIndex % size);
+    }
     return conn;
   }
 
@@ -256,10 +276,9 @@ public synchronized List<ConnectionContext> removeConnections(int num) {
       int targetCount = Math.min(num, this.connections.size() - this.minSize);
       // Remove and close targetCount of connections
       List<ConnectionContext> tmpConnections = new ArrayList<>();
-      for (int i = 0; i < this.connections.size(); i++) {
-        ConnectionContext conn = this.connections.get(i);
+      for (ConnectionContext conn : this.connections) {
         // Only pick idle connections to close
-        if (removed.size() < targetCount && conn.isUsable()) {
+        if (removed.size() < targetCount && conn.isIdle()) {
           removed.add(conn);
         } else {
           tmpConnections.add(conn);
@@ -267,8 +286,8 @@ public synchronized List<ConnectionContext> removeConnections(int num) {
       }
       this.connections = tmpConnections;
     }
-    LOG.debug("Expected to remove {} connection " +
-        "and actually removed {} connections", num, removed.size());
+    LOG.debug("Expected to remove {} connection and actually removed {} connections",
+        num, removed.size());
     return removed;
   }
 
@@ -303,7 +322,6 @@ protected int getNumConnections() {
    */
   protected int getNumActiveConnections() {
     int ret = 0;
-
     List<ConnectionContext> tmpConnections = this.connections;
     for (ConnectionContext conn : tmpConnections) {
       if (conn.isActive()) {
@@ -320,10 +338,9 @@ protected int getNumActiveConnections() {
    */
   protected int getNumIdleConnections() {
     int ret = 0;
-
     List<ConnectionContext> tmpConnections = this.connections;
     for (ConnectionContext conn : tmpConnections) {
-      if (conn.isUsable()) {
+      if (conn.isIdle()) {
         ret++;
       }
     }
@@ -393,8 +410,9 @@ public String getJSON() {
    * @throws IOException If it cannot get a new connection.
    */
   public ConnectionContext newConnection() throws IOException {
-    return newConnection(
-        this.conf, this.namenodeAddress, this.ugi, this.protocol);
+    return newConnection(this.conf, this.namenodeAddress,
+        this.ugi, this.protocol, this.enableMultiSocket,
+        this.socketIndex.incrementAndGet(), alignmentContext);
   }
 
   /**
@@ -402,19 +420,22 @@ public ConnectionContext newConnection() throws IOException {
    * context for a single user/security context. To maximize throughput it is
    * recommended to use multiple connection per user+server, allowing multiple
    * writes and reads to be dispatched in parallel.
-   * @param <T>
+   * @param <T> Input type T.
    *
    * @param conf Configuration for the connection.
    * @param nnAddress Address of server supporting the ClientProtocol.
    * @param ugi User context.
    * @param proto Interface of the protocol.
+   * @param enableMultiSocket Enable multiple socket or not.
+   * @param alignmentContext Client alignment context.
    * @return proto for the target ClientProtocol that contains the user's
    *         security context.
    * @throws IOException If it cannot be created.
    */
   protected static <T> ConnectionContext newConnection(Configuration conf,
-      String nnAddress, UserGroupInformation ugi, Class<T> proto)
-      throws IOException {
+      String nnAddress, UserGroupInformation ugi, Class<T> proto,
+      boolean enableMultiSocket, int socketIndex,
+      AlignmentContext alignmentContext) throws IOException {
     if (!PROTO_MAP.containsKey(proto)) {
       String msg = "Unsupported protocol for connection to NameNode: "
           + ((proto != null) ? proto.getName() : "null");
@@ -437,15 +458,24 @@ protected static <T> ConnectionContext newConnection(Configuration conf,
     }
     InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress);
     final long version = RPC.getProtocolVersion(classes.protoPb);
-    Object proxy = RPC.getProtocolProxy(classes.protoPb, version, socket, ugi,
-        conf, factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy();
+    Object proxy;
+    if (enableMultiSocket) {
+      FederationConnectionId connectionId = new FederationConnectionId(
+          socket, classes.protoPb, ugi, RPC.getRpcTimeout(conf),
+          defaultPolicy, conf, socketIndex);
+      proxy = RPC.getProtocolProxy(classes.protoPb, version, connectionId,
+          conf, factory, alignmentContext).getProxy();
+    } else {
+      proxy = RPC.getProtocolProxy(classes.protoPb, version, socket, ugi,
+          conf, factory, RPC.getRpcTimeout(conf), defaultPolicy, null,
+          alignmentContext).getProxy();
+    }
+
     T client = newProtoClient(proto, classes, proxy);
     Text dtService = SecurityUtil.buildTokenService(socket);
 
-    ProxyAndInfo<T> clientProxy =
-        new ProxyAndInfo<T>(client, dtService, socket);
-    ConnectionContext connection = new ConnectionContext(clientProxy);
-    return connection;
+    ProxyAndInfo<T> clientProxy = new ProxyAndInfo<T>(client, dtService, socket);
+    return new ConnectionContext(clientProxy, conf);
   }
 
   private static <T> T newProtoClient(Class<T> proto, ProtoImpl classes,
@@ -453,7 +483,7 @@ private static <T> T newProtoClient(Class<T> proto, ProtoImpl classes,
     try {
       Constructor<?> constructor =
           classes.protoClientPb.getConstructor(classes.protoPb);
-      Object o = constructor.newInstance(new Object[] {proxy});
+      Object o = constructor.newInstance(proxy);
       if (proto.isAssignableFrom(o.getClass())) {
         @SuppressWarnings("unchecked")
         T client = (T) o;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java
new file mode 100644
index 0000000000000..0be1f8b1be2a5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.net.InetSocketAddress;
+
+public class FederationConnectionId extends Client.ConnectionId {
+  private final int index;
+
+  public FederationConnectionId(InetSocketAddress address, Class<?> protocol,
+      UserGroupInformation ticket, int rpcTimeout,
+      RetryPolicy connectionRetryPolicy, Configuration conf, int index) {
+    super(address, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf);
+    this.index = index;
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder()
+        .append(super.hashCode())
+        .append(this.index)
+        .toHashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!super.equals(obj)) {
+      return false;
+    }
+    if (obj instanceof FederationConnectionId) {
+      FederationConnectionId other = (FederationConnectionId)obj;
+      return new EqualsBuilder()
+          .append(this.index, other.index)
+          .isEquals();
+    }
+    return false;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
index 7ff853946d700..e593e888c9ac4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
@@ -231,7 +231,7 @@ public static ActiveNamenodeResolver newActiveNamenodeResolver(
   }
 
   /**
-   * Add the the number of children for an existing HdfsFileStatus object.
+   * Add the number of children for an existing HdfsFileStatus object.
    * @param dirStatus HdfsfileStatus object.
    * @param children number of children to be added.
    * @return HdfsFileStatus with the number of children specified.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherThread.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherThread.java
index a077c4b3f45a4..40ff843fa1dfe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherThread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/MountTableRefresherThread.java
@@ -50,14 +50,14 @@ public MountTableRefresherThread(MountTableManager manager,
 
   /**
    * Refresh mount table cache of local and remote routers. Local and remote
-   * routers will be refreshed differently. Lets understand what are the
+   * routers will be refreshed differently. Let's understand what are the
    * local and remote routers and refresh will be done differently on these
    * routers. Suppose there are three routers R1, R2 and R3. User want to add
    * new mount table entry. He will connect to only one router, not all the
    * routers. Suppose He connects to R1 and calls add mount table entry through
    * API or CLI. Now in this context R1 is local router, R2 and R3 are remote
    * routers. Because add mount table entry is invoked on R1, R1 will update the
-   * cache locally it need not to make RPC call. But R1 will make RPC calls to
+   * cache locally it need not make RPC call. But R1 will make RPC calls to
    * update cache on R2 and R3.
    */
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
index 1afce83a7e55d..b2f60d931499c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.federation.router;
 
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEALTH_MONITOR_TIMEOUT;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT;
 import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS;
 import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT;
 
@@ -25,6 +27,7 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.HAServiceProtocol;
@@ -48,6 +51,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.classification.VisibleForTesting;
+
 /**
  * The {@link Router} periodically checks the state of a Namenode (usually on
  * the same server) and reports their high availability (HA) state and
@@ -83,6 +88,10 @@ public class NamenodeHeartbeatService extends PeriodicService {
   private NNHAServiceTarget localTarget;
   /** Cache HA protocol. */
   private HAServiceProtocol localTargetHAProtocol;
+  /** Cache NN protocol. */
+  private NamenodeProtocol namenodeProtocol;
+  /** Cache Client protocol. */
+  private ClientProtocol clientProtocol;
   /** RPC address for the namenode. */
   private String rpcAddress;
   /** Service RPC address for the namenode. */
@@ -98,6 +107,9 @@ public class NamenodeHeartbeatService extends PeriodicService {
 
   private String resolvedHost;
   private String originalNnId;
+
+  private int healthMonitorTimeoutMs = (int) DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT;
+
   /**
    * Create a new Namenode status updater.
    * @param resolver Namenode resolver service to handle NN registration.
@@ -209,6 +221,15 @@ protected void serviceInit(Configuration configuration) throws Exception {
         DFS_ROUTER_HEARTBEAT_INTERVAL_MS,
         DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT));
 
+    long timeoutMs = conf.getTimeDuration(DFS_ROUTER_HEALTH_MONITOR_TIMEOUT,
+        DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+    if (timeoutMs < 0) {
+      LOG.warn("Invalid value {} configured for {} should be greater than or equal to 0. " +
+          "Using value of : 0ms instead.", timeoutMs, DFS_ROUTER_HEALTH_MONITOR_TIMEOUT);
+      this.healthMonitorTimeoutMs = 0;
+    } else {
+      this.healthMonitorTimeoutMs = (int) timeoutMs;
+    }
 
     super.serviceInit(configuration);
   }
@@ -267,12 +288,16 @@ private void updateState() {
       LOG.error("Namenode is not operational: {}", getNamenodeDesc());
     } else if (report.haStateValid()) {
       // block and HA status available
-      LOG.debug("Received service state: {} from HA namenode: {}",
-          report.getState(), getNamenodeDesc());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Received service state: {} from HA namenode: {}",
+            report.getState(), getNamenodeDesc());
+      }
     } else if (localTarget == null) {
       // block info available, HA status not expected
-      LOG.debug(
-          "Reporting non-HA namenode as operational: " + getNamenodeDesc());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(
+            "Reporting non-HA namenode as operational: {}", getNamenodeDesc());
+      }
     } else {
       // block info available, HA status should be available, but was not
       // fetched do nothing and let the current state stand
@@ -303,65 +328,26 @@ protected NamenodeStatusReport getNamenodeStatusReport() {
       LOG.debug("Probing NN at service address: {}", serviceAddress);
 
       URI serviceURI = new URI("hdfs://" + serviceAddress);
-      // Read the filesystem info from RPC (required)
-      NamenodeProtocol nn = NameNodeProxies
-          .createProxy(this.conf, serviceURI, NamenodeProtocol.class)
-          .getProxy();
 
-      if (nn != null) {
-        NamespaceInfo info = nn.versionRequest();
-        if (info != null) {
-          report.setNamespaceInfo(info);
-        }
-      }
+      // Read the filesystem info from RPC (required)
+      updateNameSpaceInfoParameters(serviceURI, report);
       if (!report.registrationValid()) {
         return report;
       }
 
       // Check for safemode from the client protocol. Currently optional, but
       // should be required at some point for QoS
-      try {
-        ClientProtocol client = NameNodeProxies
-            .createProxy(this.conf, serviceURI, ClientProtocol.class)
-            .getProxy();
-        if (client != null) {
-          boolean isSafeMode = client.setSafeMode(
-              SafeModeAction.SAFEMODE_GET, false);
-          report.setSafeMode(isSafeMode);
-        }
-      } catch (Exception e) {
-        LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e);
-      }
+      updateSafeModeParameters(serviceURI, report);
 
       // Read the stats from JMX (optional)
       updateJMXParameters(webAddress, report);
 
-      if (localTarget != null) {
-        // Try to get the HA status
-        try {
-          // Determine if NN is active
-          // TODO: dynamic timeout
-          if (localTargetHAProtocol == null) {
-            localTargetHAProtocol = localTarget.getProxy(conf, 30*1000);
-          }
-          HAServiceStatus status = localTargetHAProtocol.getServiceStatus();
-          report.setHAServiceState(status.getState());
-        } catch (Throwable e) {
-          if (e.getMessage().startsWith("HA for namenode is not enabled")) {
-            LOG.error("HA for {} is not enabled", getNamenodeDesc());
-            localTarget = null;
-          } else {
-            // Failed to fetch HA status, ignoring failure
-            LOG.error("Cannot fetch HA status for {}: {}",
-                getNamenodeDesc(), e.getMessage(), e);
-          }
-          localTargetHAProtocol = null;
-        }
-      }
-    } catch(IOException e) {
+      // Try to get the HA status
+      updateHAStatusParameters(report);
+    } catch (IOException e) {
       LOG.error("Cannot communicate with {}: {}",
           getNamenodeDesc(), e.getMessage());
-    } catch(Throwable e) {
+    } catch (Throwable e) {
       // Generic error that we don't know about
       LOG.error("Unexpected exception while communicating with {}: {}",
           getNamenodeDesc(), e.getMessage(), e);
@@ -369,6 +355,11 @@ protected NamenodeStatusReport getNamenodeStatusReport() {
     return report;
   }
 
+  @VisibleForTesting
+  NNHAServiceTarget getLocalTarget(){
+    return this.localTarget;
+  }
+
   /**
    * Get the description of the Namenode to monitor.
    * @return Description of the Namenode to monitor.
@@ -387,6 +378,59 @@ private static String getNnHeartBeatServiceName(String nsId, String nnId) {
         (nnId == null ? "" : " " + nnId);
   }
 
+  /**
+   * Get the namespace information for a Namenode via RPC and add them to the report.
+   * @param serviceURI Server address of the Namenode to monitor.
+   * @param report Namenode status report updating with namespace information data.
+   * @throws IOException This method will throw IOException up, because RBF need
+   *                     use Namespace Info to identify this NS. If there are some IOExceptions,
+   *                     RBF doesn't need to get other information from NameNode,
+   *                     so throw IOException up.
+   */
+  private void updateNameSpaceInfoParameters(URI serviceURI,
+      NamenodeStatusReport report) throws IOException {
+    try {
+      if (this.namenodeProtocol == null) {
+        this.namenodeProtocol = NameNodeProxies.createProxy(this.conf, serviceURI,
+            NamenodeProtocol.class).getProxy();
+      }
+      if (namenodeProtocol != null) {
+        NamespaceInfo info = namenodeProtocol.versionRequest();
+        if (info != null) {
+          report.setNamespaceInfo(info);
+        }
+      }
+    } catch (IOException e) {
+      this.namenodeProtocol = null;
+      throw e;
+    }
+  }
+
+  /**
+   * Get the safemode information for a Namenode via RPC and add them to the report.
+   * Safemode is only one status of NameNode and is useless for RBF identify one NameNode.
+   * So If there are some IOExceptions, RBF can just ignore it and try to collect
+   * other information form namenode continue.
+   * @param serviceURI Server address of the Namenode to monitor.
+   * @param report Namenode status report updating with safemode information data.
+   */
+  private void updateSafeModeParameters(URI serviceURI, NamenodeStatusReport report) {
+    try {
+      if (this.clientProtocol == null) {
+        this.clientProtocol = NameNodeProxies
+            .createProxy(this.conf, serviceURI, ClientProtocol.class)
+            .getProxy();
+      }
+      if (clientProtocol != null) {
+        boolean isSafeMode = clientProtocol.setSafeMode(SafeModeAction.SAFEMODE_GET, false);
+        report.setSafeMode(isSafeMode);
+      }
+    } catch (Exception e) {
+      LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e);
+      this.clientProtocol = null;
+    }
+  }
+
   /**
    * Get the parameters for a Namenode from JMX and add them to the report.
    * @param address Web interface of the Namenode to monitor.
@@ -403,6 +447,34 @@ private void updateJMXParameters(
     }
   }
 
+  /**
+   * Get the HA status for a Namenode via RPC and add them to the report.
+   * @param report Namenode status report updating with HA status information data.
+   */
+  private void updateHAStatusParameters(NamenodeStatusReport report) {
+    if (localTarget != null) {
+      try {
+        // Determine if NN is active
+        if (localTargetHAProtocol == null) {
+          localTargetHAProtocol = localTarget.getHealthMonitorProxy(
+              conf, this.healthMonitorTimeoutMs);
+          LOG.debug("Get HA status with address {}", lifelineAddress);
+        }
+        HAServiceStatus status = localTargetHAProtocol.getServiceStatus();
+        report.setHAServiceState(status.getState());
+      } catch (Throwable e) {
+        if (e.getMessage().startsWith("HA for namenode is not enabled")) {
+          LOG.error("HA for {} is not enabled", getNamenodeDesc());
+          localTarget = null;
+        } else {
+          // Failed to fetch HA status, ignoring failure
+          LOG.error("Cannot fetch HA status for {}", getNamenodeDesc(), e);
+        }
+        localTargetHAProtocol = null;
+      }
+    }
+  }
+
   /**
    * Fetches NamenodeInfo metrics from namenode.
    * @param address Web interface of the Namenode to monitor.
@@ -466,7 +538,8 @@ private void getFsNamesystemMetrics(String address,
               jsonObject.getLong("PendingReplicationBlocks"),
               jsonObject.getLong("UnderReplicatedBlocks"),
               jsonObject.getLong("PendingDeletionBlocks"),
-              jsonObject.optLong("ProvidedCapacityTotal"));
+              jsonObject.optLong("ProvidedCapacityTotal"),
+              jsonObject.getInt("PendingSPSPaths"));
         }
       }
     }
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftException.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java
similarity index 65%
rename from hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftException.java
rename to hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java
index eba674fee5d55..1a010ac546cc5 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java
@@ -15,29 +15,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-package org.apache.hadoop.fs.swift.exceptions;
+package org.apache.hadoop.hdfs.server.federation.router;
 
 import java.io.IOException;
 
 /**
- * A Swift-specific exception -subclasses exist
- * for various specific problems.
+ * This exception is thrown when can not get any mount point for the input path.
+ * RBF cannot forward any requests for the path.
  */
-public class SwiftException extends IOException {
-  public SwiftException() {
-    super();
-  }
+public class NoLocationException extends IOException {
 
-  public SwiftException(String message) {
-    super(message);
-  }
-
-  public SwiftException(String message, Throwable cause) {
-    super(message, cause);
-  }
+  private static final long serialVersionUID = 1L;
 
-  public SwiftException(Throwable cause) {
-    super(cause);
+  public NoLocationException(String path, Class<?> t) {
+    super("Cannot find locations for " + path + " in " + t.getSimpleName());
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
index b690b8685c0a4..0883ba3a3dbb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java
@@ -167,21 +167,18 @@ protected synchronized void startPeriodic() {
     stopPeriodic();
 
     // Create the runnable service
-    Runnable updateRunnable = new Runnable() {
-      @Override
-      public void run() {
-        LOG.debug("Running {} update task", serviceName);
-        try {
-          if (!isRunning) {
-            return;
-          }
-          periodicInvoke();
-          runCount++;
-          lastRun = Time.now();
-        } catch (Exception ex) {
-          errorCount++;
-          LOG.warn(serviceName + " service threw an exception", ex);
+    Runnable updateRunnable = () -> {
+      LOG.debug("Running {} update task", serviceName);
+      try {
+        if (!isRunning) {
+          return;
         }
+        periodicInvoke();
+        runCount++;
+        lastRun = Time.now();
+      } catch (Exception ex) {
+        errorCount++;
+        LOG.warn("{} service threw an exception", serviceName, ex);
       }
     };
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java
new file mode 100644
index 0000000000000..1f2b12d445f32
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.LongAccumulator;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.ipc.AlignmentContext;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos;
+
+
+/**
+ * An alignment context shared by all connections in a {@link ConnectionPool}.
+ * There is a distinct connection pool for each [namespace,UGI] pairing.
+ * <p>
+ * {@link #sharedGlobalStateId} is a reference to a
+ * shared {@link LongAccumulator} object in the {@link RouterStateIdContext}.
+ * {@link #poolLocalStateId} is specific to each PoolAlignmentContext.
+ * <p>
+ * The shared {@link #sharedGlobalStateId} is updated only using
+ * responses from NameNodes, so clients cannot poison it.
+ * {@link #poolLocalStateId} is used to propagate client observed
+ * state into NameNode requests. A misbehaving client can poison this but the effect is only
+ * visible to other clients with the same UGI and accessing the same namespace.
+ */
+public class PoolAlignmentContext implements AlignmentContext {
+  private LongAccumulator sharedGlobalStateId;
+  private LongAccumulator poolLocalStateId;
+
+  PoolAlignmentContext(RouterStateIdContext routerStateIdContext, String namespaceId) {
+    sharedGlobalStateId = routerStateIdContext.getNamespaceStateId(namespaceId);
+    poolLocalStateId = new LongAccumulator(Math::max, Long.MIN_VALUE);
+  }
+
+  /**
+   * Client side implementation only receives state alignment info.
+   * It does not provide state alignment info therefore this does nothing.
+   */
+  @Override
+  public void updateResponseState(RpcHeaderProtos.RpcResponseHeaderProto.Builder header) {
+    // Do nothing.
+  }
+
+  /**
+   * Router updates a globally shared value using response from
+   * namenodes.
+   */
+  @Override
+  public void receiveResponseState(RpcHeaderProtos.RpcResponseHeaderProto header) {
+    sharedGlobalStateId.accumulate(header.getStateId());
+  }
+
+  /**
+   * Client side implementation for routers to provide state info in requests to
+   * namenodes.
+   */
+  @Override
+  public void updateRequestState(RpcHeaderProtos.RpcRequestHeaderProto.Builder header) {
+    header.setStateId(poolLocalStateId.get());
+  }
+
+  /**
+   * Client side implementation only provides state alignment info in requests.
+   * Client does not receive RPC requests therefore this does nothing.
+   */
+  @Override
+  public long receiveRequestState(RpcHeaderProtos.RpcRequestHeaderProto header, long threshold)
+      throws IOException {
+    // Do nothing.
+    return 0;
+  }
+
+  @Override
+  public long getLastSeenStateId() {
+    return sharedGlobalStateId.get();
+  }
+
+  @Override
+  public boolean isCoordinatedCall(String protocolName, String method) {
+    throw new UnsupportedOperationException(
+        "Client should not be checking uncoordinated call");
+  }
+
+  public void advanceClientStateId(Long clientStateId) {
+    poolLocalStateId.accumulate(clientStateId);
+  }
+
+  @VisibleForTesting
+  public long getPoolLocalStateId() {
+    return this.poolLocalStateId.get();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
index 741e470c6fc3f..c598076f636e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
@@ -96,6 +96,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       FEDERATION_ROUTER_PREFIX + "heartbeat.interval";
   public static final long DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT =
       TimeUnit.SECONDS.toMillis(5);
+  public static final String DFS_ROUTER_HEALTH_MONITOR_TIMEOUT =
+      FEDERATION_ROUTER_PREFIX + "health.monitor.timeout";
+  public static final long DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT =
+      TimeUnit.SECONDS.toMillis(30);
   public static final String DFS_ROUTER_MONITOR_NAMENODE =
       FEDERATION_ROUTER_PREFIX + "monitor.namenode";
   public static final String DFS_ROUTER_MONITOR_NAMENODE_RESOLUTION_ENABLED =
@@ -135,6 +139,12 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       FEDERATION_ROUTER_PREFIX + "connection.clean.ms";
   public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT =
       TimeUnit.SECONDS.toMillis(10);
+  public static final String DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_KEY =
+      FEDERATION_ROUTER_PREFIX + "enable.multiple.socket";
+  public static final boolean DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_DEFAULT = false;
+  public static final String DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY =
+      FEDERATION_ROUTER_PREFIX + "max.concurrency.per.connection";
+  public static final int DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_DEFAULT = 1;
 
   // HDFS Router RPC client
   public static final String DFS_ROUTER_CLIENT_THREADS_SIZE =
@@ -181,6 +191,16 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       FEDERATION_STORE_PREFIX + "enable";
   public static final boolean DFS_ROUTER_STORE_ENABLE_DEFAULT = true;
 
+  public static final String DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY =
+      FEDERATION_ROUTER_PREFIX + "observer.read.default";
+  public static final boolean DFS_ROUTER_OBSERVER_READ_DEFAULT_VALUE = false;
+  public static final String DFS_ROUTER_OBSERVER_READ_OVERRIDES =
+      FEDERATION_ROUTER_PREFIX + "observer.read.overrides";
+
+  public static final String DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE =
+      FEDERATION_ROUTER_PREFIX + "observer.federated.state.propagation.maxsize";
+  public static final int DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE_DEFAULT = 5;
+
   public static final String FEDERATION_STORE_SERIALIZER_CLASS =
       FEDERATION_STORE_PREFIX + "serializer";
   public static final Class<StateStoreSerializerPBImpl>
@@ -255,7 +275,7 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       TimeUnit.MINUTES.toMillis(1);
   /**
    * Remote router mount table cache is updated through RouterClient(RPC call).
-   * To improve performance, RouterClient connections are cached but it should
+   * To improve performance, RouterClient connections are cached, but it should
    * not be kept in cache forever. This property defines the max time a
    * connection can be cached.
    */
@@ -315,6 +335,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
   public static final long DN_REPORT_CACHE_EXPIRE_MS_DEFAULT =
       TimeUnit.SECONDS.toMillis(10);
+  public static final String DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY =
+      FEDERATION_ROUTER_PREFIX + "enable.get.dn.usage";
+  public static final boolean DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT = true;
 
   // HDFS Router-based federation quota
   public static final String DFS_ROUTER_QUOTA_ENABLE =
@@ -354,6 +377,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
       NoRouterRpcFairnessPolicyController.class;
   public static final String DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX =
       FEDERATION_ROUTER_FAIRNESS_PREFIX + "handler.count.";
+  public static final String DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT =
+      FEDERATION_ROUTER_FAIRNESS_PREFIX + "acquire.timeout";
+  public static final long   DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT =
+      TimeUnit.SECONDS.toMillis(1);
 
   // HDFS Router Federation Rename.
   public static final String DFS_ROUTER_FEDERATION_RENAME_PREFIX =
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
index 6f1121ef9fd66..ecaa97b93308d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RemoteMethod.java
@@ -36,7 +36,7 @@ public class RemoteMethod {
   private static final Logger LOG = LoggerFactory.getLogger(RemoteMethod.class);
 
 
-  /** List of parameters: static and dynamic values, matchings types. */
+  /** List of parameters: static and dynamic values, matching types. */
   private final Object[] params;
   /** List of method parameters types, matches parameters. */
   private final Class<?>[] types;
@@ -131,7 +131,7 @@ public Class<?> getProtocol() {
   /**
    * Get the represented java method.
    *
-   * @return Method
+   * @return {@link Method}
    * @throws IOException If the method cannot be found.
    */
   public Method getMethod() throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index dbfed250156a7..7cca21dccf196 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -196,6 +196,8 @@ protected void serviceInit(Configuration configuration) throws Exception {
       this.setRpcServerAddress(rpcServer.getRpcAddress());
     }
 
+    checkRouterId();
+
     if (conf.getBoolean(
         RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE,
         RBFConfigKeys.DFS_ROUTER_ADMIN_ENABLE_DEFAULT)) {
@@ -308,6 +310,21 @@ protected void serviceInit(Configuration configuration) throws Exception {
     }
   }
 
+  /**
+   * Set the router id if not set to prevent RouterHeartbeatService
+   * update state store with a null router id.
+   */
+  private void checkRouterId() {
+    if (this.routerId == null) {
+      InetSocketAddress confRpcAddress = conf.getSocketAddr(
+          RBFConfigKeys.DFS_ROUTER_RPC_BIND_HOST_KEY,
+          RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_KEY,
+          RBFConfigKeys.DFS_ROUTER_RPC_ADDRESS_DEFAULT,
+          RBFConfigKeys.DFS_ROUTER_RPC_PORT_DEFAULT);
+      setRpcServerAddress(confRpcAddress);
+    }
+  }
+
   private String getDisabledDependentServices() {
     if (this.stateStore == null && this.adminServer == null) {
       return StateStoreService.class.getSimpleName() + ","
@@ -740,7 +757,7 @@ public ActiveNamenodeResolver getNamenodeResolver() {
   /**
    * Get the state store interface for the router heartbeats.
    *
-   * @return FederationRouterStateStore state store API handle.
+   * @return RouterStore state store API handle.
    */
   public RouterStore getRouterStateManager() {
     if (this.routerStateManager == null && this.stateStore != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
index 0d322ca501674..db1922ad80811 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java
@@ -20,6 +20,7 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.server.federation.fairness.RefreshFairnessPolicyControllerHandler.HANDLER_IDENTIFIER;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -30,7 +31,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB;
 import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB;
 import org.apache.hadoop.hdfs.protocolPB.RouterPolicyProvider;
+import org.apache.hadoop.hdfs.server.federation.fairness.RefreshFairnessPolicyControllerHandler;
 import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
 import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
@@ -102,7 +104,7 @@
 import org.apache.hadoop.thirdparty.protobuf.BlockingService;
 
 /**
- * This class is responsible for handling all of the Admin calls to the HDFS
+ * This class is responsible for handling all the Admin calls to the HDFS
  * router. It is created, started, and stopped by {@link Router}.
  */
 public class RouterAdminServer extends AbstractService
@@ -211,6 +213,8 @@ public RouterAdminServer(Configuration conf, Router router)
         genericRefreshService, adminServer);
     DFSUtil.addPBProtocol(conf, RefreshCallQueueProtocolPB.class,
         refreshCallQueueService, adminServer);
+
+    registerRefreshFairnessPolicyControllerHandler();
   }
 
   /**
@@ -784,4 +788,9 @@ public void refreshCallQueue() throws IOException {
     Configuration configuration = new Configuration();
     router.getRpcServer().getServer().refreshCallQueue(configuration);
   }
+
+  private void registerRefreshFairnessPolicyControllerHandler() {
+    RefreshRegistry.defaultRegistry()
+        .register(HANDLER_IDENTIFIER, new RefreshFairnessPolicyControllerHandler(router));
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
index d5bbdf046c2f5..a5f83c95b7baf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
@@ -115,6 +115,7 @@
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 /**
  * Module that implements all the RPC calls in {@link ClientProtocol} in the
@@ -291,8 +292,10 @@ public HdfsFileStatus create(String src, FsPermission masked,
     RemoteLocation createLocation = null;
     try {
       createLocation = rpcServer.getCreateLocation(src, locations);
-      return rpcClient.invokeSingle(createLocation, method,
+      HdfsFileStatus status = rpcClient.invokeSingle(createLocation, method,
           HdfsFileStatus.class);
+      status.setNamespace(createLocation.getNameserviceId());
+      return status;
     } catch (IOException ioe) {
       final List<RemoteLocation> newLocations = checkFaultTolerantRetry(
           method, src, ioe, createLocation, locations);
@@ -305,7 +308,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
    * Check if an exception is caused by an unavailable subcluster or not. It
    * also checks the causes.
    * @param ioe IOException to check.
-   * @return If caused by an unavailable subcluster. False if the should not be
+   * @return If caused by an unavailable subcluster. False if they should not be
    *         retried (e.g., NSQuotaExceededException).
    */
   protected static boolean isUnavailableSubclusterException(
@@ -377,8 +380,11 @@ public LastBlockWithStatus append(String src, final String clientName,
     RemoteMethod method = new RemoteMethod("append",
         new Class<?>[] {String.class, String.class, EnumSetWritable.class},
         new RemoteParam(), clientName, flag);
-    return rpcClient.invokeSequential(
-        locations, method, LastBlockWithStatus.class, null);
+    RemoteResult result = rpcClient.invokeSequential(
+        method, locations, LastBlockWithStatus.class, null);
+    LastBlockWithStatus lbws = (LastBlockWithStatus) result.getResult();
+    lbws.getFileStatus().setNamespace(result.getLocation().getNameserviceId());
+    return lbws;
   }
 
   @Override
@@ -759,14 +765,49 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
     }
   }
 
+  private Map<String, FederationNamespaceInfo> getAvailableNamespaces()
+      throws IOException {
+    Map<String, FederationNamespaceInfo> allAvailableNamespaces =
+        new HashMap<>();
+    namenodeResolver.getNamespaces().forEach(
+        k -> allAvailableNamespaces.put(k.getNameserviceId(), k));
+    return allAvailableNamespaces;
+  }
+
+  /**
+   * Try to get a list of FederationNamespaceInfo for renewLease RPC.
+   */
+  private List<FederationNamespaceInfo> getRenewLeaseNSs(List<String> namespaces)
+      throws IOException {
+    if (namespaces == null || namespaces.isEmpty()) {
+      return new ArrayList<>(namenodeResolver.getNamespaces());
+    }
+    List<FederationNamespaceInfo> result = new ArrayList<>();
+    Map<String, FederationNamespaceInfo> allAvailableNamespaces =
+        getAvailableNamespaces();
+    for (String namespace : namespaces) {
+      if (!allAvailableNamespaces.containsKey(namespace)) {
+        return new ArrayList<>(namenodeResolver.getNamespaces());
+      } else {
+        result.add(allAvailableNamespaces.get(namespace));
+      }
+    }
+    return result;
+  }
+
   @Override
-  public void renewLease(String clientName) throws IOException {
+  public void renewLease(String clientName, List<String> namespaces)
+      throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
 
     RemoteMethod method = new RemoteMethod("renewLease",
-        new Class<?>[] {String.class}, clientName);
-    Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
-    rpcClient.invokeConcurrent(nss, method, false, false);
+        new Class<?>[] {String.class, List.class}, clientName, null);
+    List<FederationNamespaceInfo> nss = getRenewLeaseNSs(namespaces);
+    if (nss.size() == 1) {
+      rpcClient.invokeSingle(nss.get(0).getNameserviceId(), method);
+    } else {
+      rpcClient.invokeConcurrent(nss, method, false, false);
+    }
   }
 
   @Override
@@ -895,19 +936,22 @@ public BatchedDirectoryListing getBatchedListing(String[] srcs,
   public HdfsFileStatus getFileInfo(String src) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
-    final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(src, false, false);
-    RemoteMethod method = new RemoteMethod("getFileInfo",
-        new Class<?>[] {String.class}, new RemoteParam());
-
     HdfsFileStatus ret = null;
-    // If it's a directory, we check in all locations
-    if (rpcServer.isPathAll(src)) {
-      ret = getFileInfoAll(locations, method);
-    } else {
-      // Check for file information sequentially
-      ret = rpcClient.invokeSequential(
-          locations, method, HdfsFileStatus.class, null);
+    IOException noLocationException = null;
+    try {
+      final List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, false, false);
+      RemoteMethod method = new RemoteMethod("getFileInfo",
+          new Class<?>[] {String.class}, new RemoteParam());
+
+      // If it's a directory, we check in all locations
+      if (rpcServer.isPathAll(src)) {
+        ret = getFileInfoAll(locations, method);
+      } else {
+        // Check for file information sequentially
+        ret = rpcClient.invokeSequential(locations, method, HdfsFileStatus.class, null);
+      }
+    } catch (NoLocationException | RouterResolveException e) {
+      noLocationException = e;
     }
 
     // If there is no real path, check mount points
@@ -926,6 +970,12 @@ public HdfsFileStatus getFileInfo(String src) throws IOException {
       }
     }
 
+    // Can't find mount point for path and the path didn't contain any sub monit points,
+    // throw the NoLocationException to client.
+    if (ret == null && noLocationException != null) {
+      throw noLocationException;
+    }
+
     return ret;
   }
 
@@ -1202,14 +1252,93 @@ public void setBalancerBandwidth(long bandwidth) throws IOException {
     rpcClient.invokeConcurrent(nss, method, true, false);
   }
 
+  /**
+   * Recursively get all the locations for the path.
+   * For example, there are some mount points:
+   *   /a -> ns0 -> /a
+   *   /a/b -> ns1 -> /a/b
+   *   /a/b/c -> ns2 -> /a/b/c
+   * When the path is '/a', the result of locations should be
+   * {ns0 -> [RemoteLocation(/a)], ns1 -> [RemoteLocation(/a/b)], ns2 -> [RemoteLocation(/a/b/c)]}
+   * @param path the path to get the locations.
+   * @return a map to store all the locations and key is namespace id.
+   * @throws IOException
+   */
+  @VisibleForTesting
+  Map<String, List<RemoteLocation>> getAllLocations(String path) throws IOException {
+    Map<String, List<RemoteLocation>> locations = new HashMap<>();
+    try {
+      List<RemoteLocation> parentLocations = rpcServer.getLocationsForPath(path, false, false);
+      parentLocations.forEach(
+          l -> locations.computeIfAbsent(l.getNameserviceId(), k -> new ArrayList<>()).add(l));
+    } catch (NoLocationException | RouterResolveException e) {
+      LOG.debug("Cannot find locations for {}.", path);
+    }
+
+    final List<String> children = subclusterResolver.getMountPoints(path);
+    if (children != null) {
+      for (String child : children) {
+        String childPath = new Path(path, child).toUri().getPath();
+        Map<String, List<RemoteLocation>> childLocations = getAllLocations(childPath);
+        childLocations.forEach(
+            (k, v) -> locations.computeIfAbsent(k, l -> new ArrayList<>()).addAll(v));
+      }
+    }
+    return locations;
+  }
+
+  /**
+   * Get all the locations of the path for {@link this#getContentSummary(String)}.
+   * For example, there are some mount points:
+   *   /a -> ns0 -> /a
+   *   /a/b -> ns0 -> /a/b
+   *   /a/b/c -> ns1 -> /a/b/c
+   * When the path is '/a', the result of locations should be
+   * [RemoteLocation('/a', ns0, '/a'), RemoteLocation('/a/b/c', ns1, '/a/b/c')]
+   * When the path is '/b', will throw NoLocationException.
+   * @param path the path to get content summary
+   * @return one list contains all the remote location
+   * @throws IOException
+   */
+  @VisibleForTesting
+  List<RemoteLocation> getLocationsForContentSummary(String path) throws IOException {
+    // Try to get all the locations of the path.
+    final Map<String, List<RemoteLocation>> ns2Locations = getAllLocations(path);
+    if (ns2Locations.isEmpty()) {
+      throw new NoLocationException(path, subclusterResolver.getClass());
+    }
+
+    final List<RemoteLocation> locations = new ArrayList<>();
+    // remove the redundancy remoteLocation order by destination.
+    ns2Locations.forEach((k, v) -> {
+      List<RemoteLocation> sortedList = v.stream().sorted().collect(Collectors.toList());
+      int size = sortedList.size();
+      for (int i = size - 1; i > -1; i--) {
+        RemoteLocation currentLocation = sortedList.get(i);
+        if (i == 0) {
+          locations.add(currentLocation);
+        } else {
+          RemoteLocation preLocation = sortedList.get(i - 1);
+          if (!currentLocation.getDest().startsWith(preLocation.getDest() + Path.SEPARATOR)) {
+            locations.add(currentLocation);
+          } else {
+            LOG.debug("Ignore redundant location {}, because there is an ancestor location {}",
+                currentLocation, preLocation);
+          }
+        }
+      }
+    });
+
+    return locations;
+  }
+
   @Override
   public ContentSummary getContentSummary(String path) throws IOException {
     rpcServer.checkOperation(NameNode.OperationCategory.READ);
 
     // Get the summaries from regular files
     final Collection<ContentSummary> summaries = new ArrayList<>();
-    final List<RemoteLocation> locations =
-        rpcServer.getLocationsForPath(path, false, false);
+    final List<RemoteLocation> locations = getLocationsForContentSummary(path);
     final RemoteMethod method = new RemoteMethod("getContentSummary",
         new Class<?>[] {String.class}, new RemoteParam());
     final List<RemoteResult<RemoteLocation, ContentSummary>> results =
@@ -1229,24 +1358,6 @@ public ContentSummary getContentSummary(String path) throws IOException {
       }
     }
 
-    // Add mount points at this level in the tree
-    final List<String> children = subclusterResolver.getMountPoints(path);
-    if (children != null) {
-      for (String child : children) {
-        Path childPath = new Path(path, child);
-        try {
-          ContentSummary mountSummary = getContentSummary(
-              childPath.toString());
-          if (mountSummary != null) {
-            summaries.add(mountSummary);
-          }
-        } catch (Exception e) {
-          LOG.error("Cannot get content summary for mount {}: {}",
-              childPath, e.getMessage());
-        }
-      }
-    }
-
     // Throw original exception if no original nor mount points
     if (summaries.isEmpty() && notFoundException != null) {
       throw notFoundException;
@@ -1807,7 +1918,10 @@ public BatchedEntries<OpenFileEntry> listOpenFiles(long prevId,
 
   @Override
   public void msync() throws IOException {
-    rpcServer.checkOperation(NameNode.OperationCategory.READ, false);
+    rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
+    Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+    RemoteMethod method = new RemoteMethod("msync");
+    rpcClient.invokeConcurrent(nss, method);
   }
 
   @Override
@@ -1815,6 +1929,12 @@ public void satisfyStoragePolicy(String path) throws IOException {
     storagePolicy.satisfyStoragePolicy(path);
   }
 
+  @Override
+  public DatanodeInfo[] getSlowDatanodeReport() throws IOException {
+    rpcServer.checkOperation(NameNode.OperationCategory.UNCHECKED);
+    return rpcServer.getSlowDatanodeReport(true, 0);
+  }
+
   @Override
   public HAServiceProtocol.HAServiceState getHAServiceState() {
     if (rpcServer.isSafeMode()) {
@@ -1897,6 +2017,8 @@ private ContentSummary aggregateContentSummary(
     long quota = 0;
     long spaceConsumed = 0;
     long spaceQuota = 0;
+    long snapshotDirectoryCount = 0;
+    long snapshotFileCount = 0;
     String ecPolicy = "";
 
     for (ContentSummary summary : summaries) {
@@ -1906,6 +2028,8 @@ private ContentSummary aggregateContentSummary(
       quota = summary.getQuota();
       spaceConsumed += summary.getSpaceConsumed();
       spaceQuota = summary.getSpaceQuota();
+      snapshotDirectoryCount += summary.getSnapshotDirectoryCount();
+      snapshotFileCount += summary.getSnapshotFileCount();
       // We return from the first response as we assume that the EC policy
       // of each sub-cluster is same.
       if (ecPolicy.isEmpty()) {
@@ -1921,6 +2045,8 @@ private ContentSummary aggregateContentSummary(
         .spaceConsumed(spaceConsumed)
         .spaceQuota(spaceQuota)
         .erasureCodingPolicy(ecPolicy)
+        .snapshotDirectoryCount(snapshotDirectoryCount)
+        .snapshotFileCount(snapshotFileCount)
         .build();
     return ret;
   }
@@ -2153,7 +2279,7 @@ private List<RemoteResult<RemoteLocation, DirectoryListing>> getListingInt(
           .invokeConcurrent(locations, method, false, -1,
               DirectoryListing.class);
       return listings;
-    } catch (RouterResolveException e) {
+    } catch (NoLocationException | RouterResolveException e) {
       LOG.debug("Cannot get locations for {}, {}.", src, e.getMessage());
       return new ArrayList<>();
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java
index 41216fce702ba..a439e5c0ce84e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java
@@ -25,20 +25,19 @@
 import java.util.Map;
 
 import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.DfsServlet;
 import org.apache.hadoop.security.UserGroupInformation;
 
 /**
  * This class is used in Namesystem's web server to do fsck on namenode.
  */
 @InterfaceAudience.Private
-public class RouterFsckServlet extends HttpServlet {
+public class RouterFsckServlet extends DfsServlet {
   /** for java.io.Serializable. */
   private static final long serialVersionUID = 1L;
 
@@ -67,15 +66,4 @@ public void doGet(HttpServletRequest request, HttpServletResponse response)
     }
   }
 
-  /**
-   * Copy from {@link org.apache.hadoop.hdfs.server.namenode.DfsServlet}.
-   * @param request Http request from the user
-   * @param conf configuration
-   * @return ugi of the requested user
-   * @throws IOException failed to get ugi
-   */
-  protected UserGroupInformation getUGI(HttpServletRequest request,
-      Configuration conf) throws IOException {
-    return JspHelper.getUGI(getServletContext(), request, conf);
-  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
index 1316cf7184988..19d7442acb6ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java
@@ -63,12 +63,7 @@ public RouterHeartbeatService(Router router) {
    * Trigger the update of the Router state asynchronously.
    */
   protected void updateStateAsync() {
-    Thread thread = new Thread(new Runnable() {
-      @Override
-      public void run() {
-        updateStateStore();
-      }
-    }, "Router Heartbeat Async");
+    Thread thread = new Thread(this::updateStateStore, "Router Heartbeat Async");
     thread.setDaemon(true);
     thread.start();
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
index a4a7d9e9ddc2d..e9b780d5bca29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
@@ -200,7 +200,7 @@ private List<MountTable> getMountTableEntries() throws IOException {
    * During this time, the quota usage cache will also be updated by
    * quota manager:
    * 1. Stale paths (entries) will be removed.
-   * 2. Existing entries will be override and updated.
+   * 2. Existing entries will be overridden and updated.
    * @return List of mount tables which quota was set.
    * @throws IOException
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index ea65245e71974..62ae4b0b95de7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -22,6 +22,7 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
 import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessConstants.CONCURRENT_NS;
 
 import java.io.EOFException;
@@ -37,6 +38,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -47,6 +49,7 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -54,6 +57,7 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -62,23 +66,26 @@
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
-import org.apache.hadoop.hdfs.server.federation.fairness.AbstractRouterRpcFairnessPolicyController;
 import org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessPolicyController;
 import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
 import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
 import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly;
 import org.apache.hadoop.io.retry.RetryPolicies;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
 import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.ipc.ObserverRetryOnActiveException;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.RetriableException;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.ipc.Server.Call;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.net.ConnectTimeoutException;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
 import org.eclipse.jetty.util.ajax.JSON;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -125,26 +132,33 @@ public class RouterRpcClient {
   private final RouterRpcMonitor rpcMonitor;
   /** Field separator of CallerContext. */
   private final String contextFieldSeparator;
+  /** Observer read enabled. Default for all nameservices. */
+  private final boolean observerReadEnabledDefault;
+  /** Nameservice specific overrides of the default setting for enabling observer reads. */
+  private HashSet<String> observerReadEnabledOverrides = new HashSet<>();
 
   /** Pattern to parse a stack trace line. */
   private static final Pattern STACK_TRACE_PATTERN =
       Pattern.compile("\\tat (.*)\\.(.*)\\((.*):(\\d*)\\)");
 
-  private static final String CLIENT_IP_STR = "clientIp";
-
   /** Fairness manager to control handlers assigned per NS. */
-  private RouterRpcFairnessPolicyController routerRpcFairnessPolicyController;
+  private volatile RouterRpcFairnessPolicyController routerRpcFairnessPolicyController;
+  private Map<String, LongAdder> rejectedPermitsPerNs = new ConcurrentHashMap<>();
+  private Map<String, LongAdder> acceptedPermitsPerNs = new ConcurrentHashMap<>();
+
+  private final boolean enableProxyUser;
 
   /**
    * Create a router RPC client to manage remote procedure calls to NNs.
    *
-   * @param conf Hdfs Configuation.
+   * @param conf Hdfs Configuration.
    * @param router A router using this RPC client.
    * @param resolver A NN resolver to determine the currently active NN in HA.
    * @param monitor Optional performance monitor.
    */
   public RouterRpcClient(Configuration conf, Router router,
-      ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) {
+      ActiveNamenodeResolver resolver, RouterRpcMonitor monitor,
+      RouterStateIdContext routerStateIdContext) {
     this.router = router;
 
     this.namenodeResolver = resolver;
@@ -153,7 +167,7 @@ public RouterRpcClient(Configuration conf, Router router,
     this.contextFieldSeparator =
         clientConf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
             HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
-    this.connectionManager = new ConnectionManager(clientConf);
+    this.connectionManager = new ConnectionManager(clientConf, routerStateIdContext);
     this.connectionManager.start();
     this.routerRpcFairnessPolicyController =
         FederationUtil.newFairnessPolicyController(conf);
@@ -192,6 +206,18 @@ public RouterRpcClient(Configuration conf, Router router,
     this.retryPolicy = RetryPolicies.failoverOnNetworkException(
         RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts,
         failoverSleepBaseMillis, failoverSleepMaxMillis);
+    String[] ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS);
+    this.enableProxyUser = ipProxyUsers != null && ipProxyUsers.length > 0;
+    this.observerReadEnabledDefault = conf.getBoolean(
+        RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY,
+        RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_VALUE);
+    String[] observerReadOverrides = conf.getStrings(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_OVERRIDES);
+    if (observerReadOverrides != null) {
+      observerReadEnabledOverrides.addAll(Arrays.asList(observerReadOverrides));
+    }
+    if (this.observerReadEnabledDefault) {
+      LOG.info("Observer read is enabled for router.");
+    }
   }
 
   /**
@@ -319,6 +345,23 @@ public String getAsyncCallerPoolJson() {
     return JSON.toString(info);
   }
 
+  /**
+   * JSON representation of the rejected permits for each nameservice.
+   *
+   * @return String representation of the rejected permits for each nameservice.
+   */
+  public String getRejectedPermitsPerNsJSON() {
+    return JSON.toString(rejectedPermitsPerNs);
+  }
+
+  /**
+   * JSON representation of the accepted permits for each nameservice.
+   *
+   * @return String representation of the accepted permits for each nameservice.
+   */
+  public String getAcceptedPermitsPerNsJSON() {
+    return JSON.toString(acceptedPermitsPerNs);
+  }
   /**
    * Get ClientProtocol proxy client for a NameNode. Each combination of user +
    * NN must use a unique proxy client. Previously created clients are cached
@@ -344,13 +387,13 @@ private ConnectionContext getConnection(UserGroupInformation ugi, String nsId,
 
       // TODO Add tokens from the federated UGI
       UserGroupInformation connUGI = ugi;
-      if (UserGroupInformation.isSecurityEnabled()) {
+      if (UserGroupInformation.isSecurityEnabled() || this.enableProxyUser) {
         UserGroupInformation routerUser = UserGroupInformation.getLoginUser();
         connUGI = UserGroupInformation.createProxyUser(
             ugi.getUserName(), routerUser);
       }
       connection = this.connectionManager.getConnection(
-          connUGI, rpcAddress, proto);
+          connUGI, rpcAddress, proto, nsId);
       LOG.debug("User {} NN {} is using connection {}",
           ugi.getUserName(), rpcAddress, connection);
     } catch (Exception ex) {
@@ -426,16 +469,19 @@ private RetryDecision shouldRetry(final IOException ioe, final int retryCount,
    * @param ugi User group information.
    * @param namenodes A prioritized list of namenodes within the same
    *                  nameservice.
-   * @param method Remote ClientProtcol method to invoke.
+   * @param useObserver Whether to use observer namenodes.
+   * @param method Remote ClientProtocol method to invoke.
    * @param params Variable list of parameters matching the method.
    * @return The result of invoking the method.
    * @throws ConnectException If it cannot connect to any Namenode.
    * @throws StandbyException If all Namenodes are in Standby.
    * @throws IOException If it cannot invoke the method.
    */
-  private Object invokeMethod(
+  @VisibleForTesting
+  public Object invokeMethod(
       final UserGroupInformation ugi,
       final List<? extends FederationNamenodeContext> namenodes,
+      boolean useObserver,
       final Class<?> protocol, final Method method, final Object... params)
           throws ConnectException, StandbyException, IOException {
 
@@ -445,15 +491,19 @@ private Object invokeMethod(
           + router.getRouterId());
     }
 
-    appendClientIpToCallerContextIfAbsent();
+    addClientInfoToCallerContext();
 
     Object ret = null;
     if (rpcMonitor != null) {
       rpcMonitor.proxyOp();
     }
     boolean failover = false;
+    boolean shouldUseObserver = useObserver;
     Map<FederationNamenodeContext, IOException> ioes = new LinkedHashMap<>();
     for (FederationNamenodeContext namenode : namenodes) {
+      if (!shouldUseObserver && (namenode.getState() == FederationNamenodeServiceState.OBSERVER)) {
+        continue;
+      }
       ConnectionContext connection = null;
       String nsId = namenode.getNameserviceId();
       String rpcAddress = namenode.getRpcAddress();
@@ -463,13 +513,14 @@ private Object invokeMethod(
         final Object proxy = client.getProxy();
 
         ret = invoke(nsId, 0, method, proxy, params);
-        if (failover) {
+        if (failover &&
+            FederationNamenodeServiceState.OBSERVER != namenode.getState()) {
           // Success on alternate server, update
           InetSocketAddress address = client.getAddress();
           namenodeResolver.updateActiveNamenode(nsId, address);
         }
         if (this.rpcMonitor != null) {
-          this.rpcMonitor.proxyOpComplete(true);
+          this.rpcMonitor.proxyOpComplete(true, nsId, namenode.getState());
         }
         if (this.router.getRouterClientMetrics() != null) {
           this.router.getRouterClientMetrics().incInvokedMethod(method);
@@ -477,20 +528,29 @@ private Object invokeMethod(
         return ret;
       } catch (IOException ioe) {
         ioes.put(namenode, ioe);
-        if (ioe instanceof StandbyException) {
+        if (ioe instanceof ObserverRetryOnActiveException) {
+          LOG.info("Encountered ObserverRetryOnActiveException from {}."
+                  + " Retry active namenode directly.", namenode);
+          shouldUseObserver = false;
+        } else if (ioe instanceof StandbyException) {
           // Fail over indicated by retry policy and/or NN
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpFailureStandby();
+            this.rpcMonitor.proxyOpFailureStandby(nsId);
           }
           failover = true;
         } else if (isUnavailableException(ioe)) {
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpFailureCommunicate();
+            this.rpcMonitor.proxyOpFailureCommunicate(nsId);
+          }
+          if (FederationNamenodeServiceState.OBSERVER == namenode.getState()) {
+            namenodeResolver.updateUnavailableNamenode(nsId,
+                NetUtils.createSocketAddr(namenode.getRpcAddress()));
+          } else {
+            failover = true;
           }
-          failover = true;
         } else if (ioe instanceof RemoteException) {
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpComplete(true);
+            this.rpcMonitor.proxyOpComplete(true, nsId, namenode.getState());
           }
           RemoteException re = (RemoteException) ioe;
           ioe = re.unwrapRemoteException();
@@ -499,7 +559,7 @@ private Object invokeMethod(
           throw ioe;
         } else if (ioe instanceof ConnectionNullException) {
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpFailureCommunicate();
+            this.rpcMonitor.proxyOpFailureCommunicate(nsId);
           }
           LOG.error("Get connection for {} {} error: {}", nsId, rpcAddress,
               ioe.getMessage());
@@ -509,7 +569,7 @@ private Object invokeMethod(
           throw se;
         } else if (ioe instanceof NoNamenodesAvailableException) {
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpNoNamenodes();
+            this.rpcMonitor.proxyOpNoNamenodes(nsId);
           }
           LOG.error("Cannot get available namenode for {} {} error: {}",
               nsId, rpcAddress, ioe.getMessage());
@@ -519,8 +579,8 @@ private Object invokeMethod(
           // Other communication error, this is a failure
           // Communication retries are handled by the retry policy
           if (this.rpcMonitor != null) {
-            this.rpcMonitor.proxyOpFailureCommunicate();
-            this.rpcMonitor.proxyOpComplete(false);
+            this.rpcMonitor.proxyOpFailureCommunicate(nsId);
+            this.rpcMonitor.proxyOpComplete(false, nsId, namenode.getState());
           }
           throw ioe;
         }
@@ -531,7 +591,7 @@ private Object invokeMethod(
       }
     }
     if (this.rpcMonitor != null) {
-      this.rpcMonitor.proxyOpComplete(false);
+      this.rpcMonitor.proxyOpComplete(false, null, null);
     }
 
     // All namenodes were unavailable or in standby
@@ -565,26 +625,39 @@ private Object invokeMethod(
   }
 
   /**
-   * For tracking which is the actual client address.
-   * It adds trace info "clientIp:ip" to caller context if it's absent.
+   * For tracking some information about the actual client.
+   * It adds trace info "clientIp:ip", "clientPort:port",
+   * "clientId:id" and "clientCallId:callId"
+   * in the caller context, removing the old values if they were
+   * already present.
    */
-  private void appendClientIpToCallerContextIfAbsent() {
-    String clientIpInfo = CLIENT_IP_STR + ":" + Server.getRemoteAddress();
-    final CallerContext ctx = CallerContext.getCurrent();
-    if (isClientIpInfoAbsent(clientIpInfo, ctx)) {
-      String origContext = ctx == null ? null : ctx.getContext();
-      byte[] origSignature = ctx == null ? null : ctx.getSignature();
-      CallerContext.setCurrent(
-          new CallerContext.Builder(origContext, contextFieldSeparator)
-              .append(clientIpInfo)
-              .setSignature(origSignature)
-              .build());
+  private void addClientInfoToCallerContext() {
+    CallerContext ctx = CallerContext.getCurrent();
+    String origContext = ctx == null ? null : ctx.getContext();
+    byte[] origSignature = ctx == null ? null : ctx.getSignature();
+    CallerContext.Builder builder =
+        new CallerContext.Builder("", contextFieldSeparator)
+            .append(CallerContext.CLIENT_IP_STR, Server.getRemoteAddress())
+            .append(CallerContext.CLIENT_PORT_STR,
+                Integer.toString(Server.getRemotePort()))
+            .append(CallerContext.CLIENT_ID_STR,
+                StringUtils.byteToHexString(Server.getClientId()))
+            .append(CallerContext.CLIENT_CALL_ID_STR,
+                Integer.toString(Server.getCallId()))
+            .setSignature(origSignature);
+    // Append the original caller context
+    if (origContext != null) {
+      for (String part : origContext.split(contextFieldSeparator)) {
+        String[] keyValue =
+            part.split(CallerContext.Builder.KEY_VALUE_SEPARATOR, 2);
+        if (keyValue.length == 2) {
+          builder.appendIfAbsent(keyValue[0], keyValue[1]);
+        } else if (keyValue.length == 1) {
+          builder.append(keyValue[0]);
+        }
+      }
     }
-  }
-
-  private boolean isClientIpInfoAbsent(String clientIpInfo, CallerContext ctx){
-    return ctx == null || ctx.getContext() == null
-        || !ctx.getContext().contains(clientIpInfo);
+    CallerContext.setCurrent(builder.build());
   }
 
   /**
@@ -601,16 +674,12 @@ private boolean isClientIpInfoAbsent(String clientIpInfo, CallerContext ctx){
    * @param params Variable parameters
    * @return Response from the remote server
    * @throws IOException
-   * @throws InterruptedException
    */
   private Object invoke(String nsId, int retryCount, final Method method,
       final Object obj, final Object... params) throws IOException {
     try {
       return method.invoke(obj, params);
-    } catch (IllegalAccessException e) {
-      LOG.error("Unexpected exception while proxying API", e);
-      return null;
-    } catch (IllegalArgumentException e) {
+    } catch (IllegalAccessException | IllegalArgumentException e) {
       LOG.error("Unexpected exception while proxying API", e);
       return null;
     } catch (InvocationTargetException e) {
@@ -674,7 +743,7 @@ public static boolean isUnavailableException(IOException ioe) {
    */
   private boolean isClusterUnAvailable(String nsId) throws IOException {
     List<? extends FederationNamenodeContext> nnState = this.namenodeResolver
-        .getNamenodesForNameserviceId(nsId);
+        .getNamenodesForNameserviceId(nsId, false);
 
     if (nnState != null) {
       for (FederationNamenodeContext nnContext : nnState) {
@@ -802,17 +871,18 @@ public Object invokeSingleBlockPool(final String bpId, RemoteMethod method)
   public Object invokeSingle(final String nsId, RemoteMethod method)
       throws IOException {
     UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
-    acquirePermit(nsId, ugi, method);
+    RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController();
+    acquirePermit(nsId, ugi, method, controller);
     try {
-      List<? extends FederationNamenodeContext> nns =
-          getNamenodesForNameservice(nsId);
+      boolean isObserverRead = isObserverReadEligible(nsId, method.getMethod());
+      List<? extends FederationNamenodeContext> nns = getOrderedNamenodes(nsId, isObserverRead);
       RemoteLocationContext loc = new RemoteLocation(nsId, "/", "/");
       Class<?> proto = method.getProtocol();
       Method m = method.getMethod();
       Object[] params = method.getParams(loc);
-      return invokeMethod(ugi, nns, proto, m, params);
+      return invokeMethod(ugi, nns, isObserverRead, proto, m, params);
     } finally {
-      releasePermit(nsId, ugi, method);
+      releasePermit(nsId, ugi, method, controller);
     }
   }
 
@@ -887,7 +957,7 @@ public <T> T invokeSingle(final RemoteLocationContext location,
    * @throws IOException if the success condition is not met and one of the RPC
    *           calls generated a remote exception.
    */
-  public Object invokeSequential(
+  public <T> T invokeSequential(
       final List<? extends RemoteLocationContext> locations,
       final RemoteMethod remoteMethod) throws IOException {
     return invokeSequential(locations, remoteMethod, null, null);
@@ -963,6 +1033,7 @@ public <R extends RemoteLocationContext, T> RemoteResult invokeSequential(
       Class<T> expectedResultClass, Object expectedResultValue)
       throws IOException {
 
+    RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController();
     final UserGroupInformation ugi = RouterRpcServer.getRemoteUser();
     final Method m = remoteMethod.getMethod();
     List<IOException> thrownExceptions = new ArrayList<>();
@@ -970,13 +1041,15 @@ public <R extends RemoteLocationContext, T> RemoteResult invokeSequential(
     // Invoke in priority order
     for (final RemoteLocationContext loc : locations) {
       String ns = loc.getNameserviceId();
-      acquirePermit(ns, ugi, remoteMethod);
+      acquirePermit(ns, ugi, remoteMethod, controller);
+      boolean isObserverRead = isObserverReadEligible(ns, m);
       List<? extends FederationNamenodeContext> namenodes =
-          getNamenodesForNameservice(ns);
+          getOrderedNamenodes(ns, isObserverRead);
       try {
         Class<?> proto = remoteMethod.getProtocol();
         Object[] params = remoteMethod.getParams(loc);
-        Object result = invokeMethod(ugi, namenodes, proto, m, params);
+        Object result = invokeMethod(
+            ugi, namenodes, isObserverRead, proto, m, params);
         // Check if the result is what we expected
         if (isExpectedClass(expectedResultClass, result) &&
             isExpectedValue(expectedResultValue, result)) {
@@ -998,14 +1071,14 @@ public <R extends RemoteLocationContext, T> RemoteResult invokeSequential(
       } catch (Exception e) {
         // Unusual error, ClientProtocol calls always use IOException (or
         // RemoteException). Re-wrap in IOException for compatibility with
-        // ClientProtcol.
+        // ClientProtocol.
         LOG.error("Unexpected exception {} proxying {} to {}",
             e.getClass(), m.getName(), ns, e);
         IOException ioe = new IOException(
             "Unexpected exception proxying API " + e.getMessage(), e);
         thrownExceptions.add(ioe);
       } finally {
-        releasePermit(ns, ugi, remoteMethod);
+        releasePermit(ns, ugi, remoteMethod, controller);
       }
     }
 
@@ -1330,20 +1403,23 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
       // Shortcut, just one call
       T location = locations.iterator().next();
       String ns = location.getNameserviceId();
-      acquirePermit(ns, ugi, method);
+      RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController();
+      acquirePermit(ns, ugi, method, controller);
+      boolean isObserverRead = isObserverReadEligible(ns, m);
       final List<? extends FederationNamenodeContext> namenodes =
-          getNamenodesForNameservice(ns);
+          getOrderedNamenodes(ns, isObserverRead);
       try {
         Class<?> proto = method.getProtocol();
         Object[] paramList = method.getParams(location);
-        R result = (R) invokeMethod(ugi, namenodes, proto, m, paramList);
+        R result = (R) invokeMethod(
+            ugi, namenodes, isObserverRead, proto, m, paramList);
         RemoteResult<T, R> remoteResult = new RemoteResult<>(location, result);
         return Collections.singletonList(remoteResult);
       } catch (IOException ioe) {
         // Localize the exception
         throw processException(ioe, location);
       } finally {
-        releasePermit(ns, ugi, method);
+        releasePermit(ns, ugi, method, controller);
       }
     }
 
@@ -1354,8 +1430,9 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
     final CallerContext originContext = CallerContext.getCurrent();
     for (final T location : locations) {
       String nsId = location.getNameserviceId();
+      boolean isObserverRead = isObserverReadEligible(nsId, m);
       final List<? extends FederationNamenodeContext> namenodes =
-          getNamenodesForNameservice(nsId);
+          getOrderedNamenodes(nsId, isObserverRead);
       final Class<?> proto = method.getProtocol();
       final Object[] paramList = method.getParams(location);
       if (standby) {
@@ -1372,7 +1449,8 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
           callables.add(
               () -> {
                 transferThreadLocalContext(originCall, originContext);
-                return invokeMethod(ugi, nnList, proto, m, paramList);
+                return invokeMethod(
+                    ugi, nnList, isObserverRead, proto, m, paramList);
               });
         }
       } else {
@@ -1381,7 +1459,8 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
         callables.add(
             () -> {
               transferThreadLocalContext(originCall, originContext);
-              return invokeMethod(ugi, namenodes, proto, m, paramList);
+              return invokeMethod(
+                  ugi, namenodes, isObserverRead, proto, m, paramList);
             });
       }
     }
@@ -1393,7 +1472,8 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
       this.router.getRouterClientMetrics().incInvokedConcurrent(m);
     }
 
-    acquirePermit(CONCURRENT_NS, ugi, method);
+    RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController();
+    acquirePermit(CONCURRENT_NS, ugi, method, controller);
     try {
       List<Future<Object>> futures = null;
       if (timeOutMs > 0) {
@@ -1418,7 +1498,7 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
           results.add(new RemoteResult<>(location, ioe));
         } catch (ExecutionException ex) {
           Throwable cause = ex.getCause();
-          LOG.debug("Canot execute {} in {}: {}",
+          LOG.debug("Cannot execute {} in {}: {}",
               m.getName(), location, cause.getMessage());
 
           // Convert into IOException if needed
@@ -1451,7 +1531,7 @@ public <T extends RemoteLocationContext, R> Map<T, R> invokeConcurrent(
       throw new IOException(
           "Unexpected error while invoking API " + ex.getMessage(), ex);
     } finally {
-      releasePermit(CONCURRENT_NS, ugi, method);
+      releasePermit(CONCURRENT_NS, ugi, method, controller);
     }
   }
 
@@ -1469,27 +1549,6 @@ private void transferThreadLocalContext(
     CallerContext.setCurrent(originContext);
   }
 
-  /**
-   * Get a prioritized list of NNs that share the same nameservice ID (in the
-   * same namespace). NNs that are reported as ACTIVE will be first in the list.
-   *
-   * @param nsId The nameservice ID for the namespace.
-   * @return A prioritized list of NNs to use for communication.
-   * @throws IOException If a NN cannot be located for the nameservice ID.
-   */
-  private List<? extends FederationNamenodeContext> getNamenodesForNameservice(
-      final String nsId) throws IOException {
-
-    final List<? extends FederationNamenodeContext> namenodes =
-        namenodeResolver.getNamenodesForNameserviceId(nsId);
-
-    if (namenodes == null || namenodes.isEmpty()) {
-      throw new IOException("Cannot locate a registered namenode for " + nsId +
-          " from " + router.getRouterId());
-    }
-    return namenodes;
-  }
-
   /**
    * Get a prioritized list of NNs that share the same block pool ID (in the
    * same namespace). NNs that are reported as ACTIVE will be first in the list.
@@ -1532,48 +1591,143 @@ private String getNameserviceForBlockPoolId(final String bpId)
    * @param nsId Identifier of the block pool.
    * @param ugi UserGroupIdentifier associated with the user.
    * @param m Remote method that needs to be invoked.
+   * @param controller fairness policy controller to acquire permit from
    * @throws IOException If permit could not be acquired for the nsId.
    */
-  private void acquirePermit(
-      final String nsId, final UserGroupInformation ugi, final RemoteMethod m)
+  private void acquirePermit(final String nsId, final UserGroupInformation ugi,
+      final RemoteMethod m, RouterRpcFairnessPolicyController controller)
       throws IOException {
-    if (routerRpcFairnessPolicyController != null
-        && !routerRpcFairnessPolicyController.acquirePermit(nsId)) {
-      // Throw StandByException,
-      // Clients could fail over and try another router.
-      if (rpcMonitor != null) {
-        rpcMonitor.getRPCMetrics().incrProxyOpPermitRejected();
+    if (controller != null) {
+      if (!controller.acquirePermit(nsId)) {
+        // Throw StandByException,
+        // Clients could fail over and try another router.
+        if (rpcMonitor != null) {
+          rpcMonitor.getRPCMetrics().incrProxyOpPermitRejected();
+        }
+        incrRejectedPermitForNs(nsId);
+        LOG.debug("Permit denied for ugi: {} for method: {}",
+            ugi, m.getMethodName());
+        String msg =
+            "Router " + router.getRouterId() +
+                " is overloaded for NS: " + nsId;
+        throw new StandbyException(msg);
       }
-      LOG.debug("Permit denied for ugi: {} for method: {}",
-          ugi, m.getMethodName());
-      String msg =
-          "Router " + router.getRouterId() +
-              " is overloaded for NS: " + nsId;
-      throw new StandbyException(msg);
+      incrAcceptedPermitForNs(nsId);
     }
   }
 
   /**
    * Release permit for specific nsId after processing against downstream
    * nsId is completed.
-   *
-   * @param nsId Identifier of the block pool.
+   *  @param nsId Identifier of the block pool.
    * @param ugi UserGroupIdentifier associated with the user.
    * @param m Remote method that needs to be invoked.
+   * @param controller fairness policy controller to release permit from
    */
-  private void releasePermit(
-      final String nsId, final UserGroupInformation ugi, final RemoteMethod m) {
-    if (routerRpcFairnessPolicyController != null) {
-      routerRpcFairnessPolicyController.releasePermit(nsId);
+  private void releasePermit(final String nsId, final UserGroupInformation ugi,
+      final RemoteMethod m, RouterRpcFairnessPolicyController controller) {
+    if (controller != null) {
+      controller.releasePermit(nsId);
       LOG.trace("Permit released for ugi: {} for method: {}", ugi,
           m.getMethodName());
     }
   }
 
-  @VisibleForTesting
-  public AbstractRouterRpcFairnessPolicyController
+  public RouterRpcFairnessPolicyController
       getRouterRpcFairnessPolicyController() {
-    return (AbstractRouterRpcFairnessPolicyController
-          )routerRpcFairnessPolicyController;
+    return routerRpcFairnessPolicyController;
+  }
+
+  private void incrRejectedPermitForNs(String ns) {
+    rejectedPermitsPerNs.computeIfAbsent(ns, k -> new LongAdder()).increment();
+  }
+
+  public Long getRejectedPermitForNs(String ns) {
+    return rejectedPermitsPerNs.containsKey(ns) ?
+        rejectedPermitsPerNs.get(ns).longValue() : 0L;
+  }
+
+  private void incrAcceptedPermitForNs(String ns) {
+    acceptedPermitsPerNs.computeIfAbsent(ns, k -> new LongAdder()).increment();
+  }
+
+  public Long getAcceptedPermitForNs(String ns) {
+    return acceptedPermitsPerNs.containsKey(ns) ?
+        acceptedPermitsPerNs.get(ns).longValue() : 0L;
+  }
+
+  /**
+   * Refreshes/changes the fairness policy controller implementation if possible
+   * and returns the controller class name.
+   * @param conf Configuration
+   * @return New controller class name if successfully refreshed, else old controller class name
+   */
+  public synchronized String refreshFairnessPolicyController(Configuration conf) {
+    RouterRpcFairnessPolicyController newController;
+    try {
+      newController = FederationUtil.newFairnessPolicyController(conf);
+    } catch (RuntimeException e) {
+      LOG.error("Failed to create router fairness policy controller", e);
+      return getCurrentFairnessPolicyControllerClassName();
+    }
+
+    if (newController != null) {
+      if (routerRpcFairnessPolicyController != null) {
+        routerRpcFairnessPolicyController.shutdown();
+      }
+      routerRpcFairnessPolicyController = newController;
+    }
+    return getCurrentFairnessPolicyControllerClassName();
+  }
+
+  private String getCurrentFairnessPolicyControllerClassName() {
+    if (routerRpcFairnessPolicyController != null) {
+      return routerRpcFairnessPolicyController.getClass().getCanonicalName();
+    }
+    return null;
+  }
+
+  /**
+   * Get a prioritized list of NNs that share the same nameservice ID (in the
+   * same namespace).
+   * In observer read case, OBSERVER NNs will be first in the list.
+   * Otherwise, ACTIVE NNs will be first in the list.
+   *
+   * @param nsId The nameservice ID for the namespace.
+   * @param isObserverRead Read on observer namenode.
+   * @return A prioritized list of NNs to use for communication.
+   * @throws IOException If a NN cannot be located for the nameservice ID.
+   */
+  private List<? extends FederationNamenodeContext> getOrderedNamenodes(String nsId,
+      boolean isObserverRead) throws IOException {
+    final List<? extends FederationNamenodeContext> namenodes;
+
+    if (RouterStateIdContext.getClientStateIdFromCurrentCall(nsId) > Long.MIN_VALUE) {
+      namenodes = namenodeResolver.getNamenodesForNameserviceId(nsId, isObserverRead);
+    } else {
+      namenodes = namenodeResolver.getNamenodesForNameserviceId(nsId, false);
+    }
+
+    if (namenodes == null || namenodes.isEmpty()) {
+      throw new IOException("Cannot locate a registered namenode for " + nsId +
+          " from " + router.getRouterId());
+    }
+    return namenodes;
+  }
+
+  private boolean isObserverReadEligible(String nsId, Method method) {
+    boolean isReadEnabledForNamespace = observerReadEnabledDefault != observerReadEnabledOverrides.contains(nsId);
+    return isReadEnabledForNamespace && isReadCall(method);
+  }
+
+  /**
+   * Check if a method is read-only.
+   * @return whether the 'method' is a read-only operation.
+   */
+  private static boolean isReadCall(Method method) {
+    if (!method.isAnnotationPresent(ReadOnly.class)) {
+      return false;
+    }
+    return !method.getAnnotationsByType(ReadOnly.class)[0].activeOnly();
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
index 5a2adb9e54e41..256f03f12ff38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
 import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
 
 /**
@@ -54,26 +55,27 @@ void init(
 
   /**
    * Start proxying an operation to the Namenode.
-   * @return Id of the thread doing the proxying.
+   * @return id of the thread doing the proxying.
    */
   long proxyOp();
 
   /**
    * Mark a proxy operation as completed.
    * @param success If the operation was successful.
+   * @param state proxy namenode state.
    */
-  void proxyOpComplete(boolean success);
+  void proxyOpComplete(boolean success, String nsId, FederationNamenodeServiceState state);
 
   /**
    * Failed to proxy an operation to a Namenode because it was in standby.
    */
-  void proxyOpFailureStandby();
+  void proxyOpFailureStandby(String nsId);
 
   /**
    * Failed to proxy an operation to a Namenode because of an unexpected
    * exception.
    */
-  void proxyOpFailureCommunicate();
+  void proxyOpFailureCommunicate(String nsId);
 
   /**
    * Failed to proxy an operation to a Namenode because the client was
@@ -95,7 +97,7 @@ void init(
   /**
    * Failed to proxy an operation because of no namenodes available.
    */
-  void proxyOpNoNamenodes();
+  void proxyOpNoNamenodes(String nsId);
 
   /**
    * If the Router cannot contact the State Store in an operation.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index b8405daca5406..c4173163436ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -43,14 +43,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumSet;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
@@ -254,18 +252,18 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol,
   /**
    * Construct a router RPC server.
    *
-   * @param configuration HDFS Configuration.
+   * @param conf HDFS Configuration.
    * @param router A router using this RPC server.
    * @param nnResolver The NN resolver instance to determine active NNs in HA.
    * @param fileResolver File resolver to resolve file paths to subclusters.
    * @throws IOException If the RPC server could not be created.
    */
-  public RouterRpcServer(Configuration configuration, Router router,
+  public RouterRpcServer(Configuration conf, Router router,
       ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver)
           throws IOException {
     super(RouterRpcServer.class.getName());
 
-    this.conf = configuration;
+    this.conf = conf;
     this.router = router;
     this.namenodeResolver = nnResolver;
     this.subclusterResolver = fileResolver;
@@ -323,6 +321,7 @@ public RouterRpcServer(Configuration configuration, Router router,
 
     // Create security manager
     this.securityManager = new RouterSecurityManager(this.conf);
+    RouterStateIdContext routerStateIdContext = new RouterStateIdContext(conf);
 
     this.rpcServer = new RPC.Builder(this.conf)
         .setProtocol(ClientNamenodeProtocolPB.class)
@@ -333,6 +332,7 @@ public RouterRpcServer(Configuration configuration, Router router,
         .setnumReaders(readerCount)
         .setQueueSizePerHandler(handlerQueueSize)
         .setVerbose(false)
+        .setAlignmentContext(routerStateIdContext)
         .setSecretManager(this.securityManager.getSecretManager())
         .build();
 
@@ -386,7 +386,7 @@ public RouterRpcServer(Configuration configuration, Router router,
 
     // Create the client
     this.rpcClient = new RouterRpcClient(this.conf, this.router,
-        this.namenodeResolver, this.rpcMonitor);
+        this.namenodeResolver, this.rpcMonitor, routerStateIdContext);
 
     // Initialize modules
     this.quotaCall = new Quota(this.router, this);
@@ -407,7 +407,7 @@ public RouterRpcServer(Configuration configuration, Router router,
                 .asMap()
                 .keySet()
                 .parallelStream()
-                .forEach((key) -> this.dnCache.refresh(key)),
+                .forEach(this.dnCache::refresh),
             0,
             dnCacheExpire, TimeUnit.MILLISECONDS);
     initRouterFedRename();
@@ -538,7 +538,7 @@ public FileSubclusterResolver getSubclusterResolver() {
   }
 
   /**
-   * Get the active namenode resolver
+   * Get the active namenode resolver.
    *
    * @return Active namenode resolver.
    */
@@ -583,7 +583,7 @@ public InetSocketAddress getRpcAddress() {
    * @param op Category of the operation to check.
    * @param supported If the operation is supported or not. If not, it will
    *                  throw an UnsupportedOperationException.
-   * @throws SafeModeException If the Router is in safe mode and cannot serve
+   * @throws StandbyException If the Router is in safe mode and cannot serve
    *                           client requests.
    * @throws UnsupportedOperationException If the operation is not supported.
    */
@@ -606,7 +606,7 @@ void checkOperation(OperationCategory op, boolean supported)
    * UNCHECKED. This function should be called by all ClientProtocol functions.
    *
    * @param op Category of the operation to check.
-   * @throws SafeModeException If the Router is in safe mode and cannot serve
+   * @throws StandbyException If the Router is in safe mode and cannot serve
    *                           client requests.
    */
   void checkOperation(OperationCategory op)
@@ -671,8 +671,8 @@ static String getMethodName() {
 
   /**
    * Invokes the method at default namespace, if default namespace is not
-   * available then at the first available namespace.
-   * If the namespace is unavailable, retry once with other namespace.
+   * available then at the other available namespaces.
+   * If the namespace is unavailable, retry with other namespaces.
    * @param <T> expected return type.
    * @param method the remote method.
    * @return the response received after invoking method.
@@ -681,28 +681,29 @@ static String getMethodName() {
   <T> T invokeAtAvailableNs(RemoteMethod method, Class<T> clazz)
       throws IOException {
     String nsId = subclusterResolver.getDefaultNamespace();
-    // If default Ns is not present return result from first namespace.
     Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
-    try {
-      if (!nsId.isEmpty()) {
+    // If no namespace is available, then throw this IOException.
+    IOException io = new IOException("No namespace available.");
+    // If default Ns is present return result from that namespace.
+    if (!nsId.isEmpty()) {
+      try {
         return rpcClient.invokeSingle(nsId, method, clazz);
+      } catch (IOException ioe) {
+        if (!clientProto.isUnavailableSubclusterException(ioe)) {
+          LOG.debug("{} exception cannot be retried",
+              ioe.getClass().getSimpleName());
+          throw ioe;
+        }
+        // Remove the already tried namespace.
+        nss.removeIf(n -> n.getNameserviceId().equals(nsId));
+        return invokeOnNs(method, clazz, io, nss);
       }
-      // If no namespace is available, throw IOException.
-      IOException io = new IOException("No namespace available.");
-      return invokeOnNs(method, clazz, io, nss);
-    } catch (IOException ioe) {
-      if (!clientProto.isUnavailableSubclusterException(ioe)) {
-        LOG.debug("{} exception cannot be retried",
-            ioe.getClass().getSimpleName());
-        throw ioe;
-      }
-      Set<FederationNamespaceInfo> nssWithoutFailed = getNameSpaceInfo(nsId);
-      return invokeOnNs(method, clazz, ioe, nssWithoutFailed);
     }
+    return invokeOnNs(method, clazz, io, nss);
   }
 
   /**
-   * Invoke the method on first available namespace,
+   * Invoke the method sequentially on available namespaces,
    * throw no namespace available exception, if no namespaces are available.
    * @param method the remote method.
    * @param clazz  Class for the return type.
@@ -716,24 +717,22 @@ <T> T invokeOnNs(RemoteMethod method, Class<T> clazz, IOException ioe,
     if (nss.isEmpty()) {
       throw ioe;
     }
-    String nsId = nss.iterator().next().getNameserviceId();
-    return rpcClient.invokeSingle(nsId, method, clazz);
-  }
-
-  /**
-   * Get set of namespace info's removing the already invoked namespaceinfo.
-   * @param nsId already invoked namespace id
-   * @return List of name spaces in the federation on
-   * removing the already invoked namespaceinfo.
-   */
-  private Set<FederationNamespaceInfo> getNameSpaceInfo(String nsId) {
-    Set<FederationNamespaceInfo> namespaceInfos = new HashSet<>();
-    for (FederationNamespaceInfo ns : namespaceInfos) {
-      if (!nsId.equals(ns.getNameserviceId())) {
-        namespaceInfos.add(ns);
+    for (FederationNamespaceInfo fnInfo : nss) {
+      String nsId = fnInfo.getNameserviceId();
+      LOG.debug("Invoking {} on namespace {}", method, nsId);
+      try {
+        return rpcClient.invokeSingle(nsId, method, clazz);
+      } catch (IOException e) {
+        LOG.debug("Failed to invoke {} on namespace {}", method, nsId, e);
+        // Ignore the exception and try on other namespace, if the tried
+        // namespace is unavailable, else throw the received exception.
+        if (!clientProto.isUnavailableSubclusterException(e)) {
+          throw e;
+        }
       }
     }
-    return namespaceInfos;
+    // Couldn't get a response from any of the namespace, throw ioe.
+    throw ioe;
   }
 
   @Override // ClientProtocol
@@ -983,8 +982,9 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
   }
 
   @Override // ClientProtocol
-  public void renewLease(String clientName) throws IOException {
-    clientProto.renewLease(clientName);
+  public void renewLease(String clientName, List<String> namespaces)
+      throws IOException {
+    clientProto.renewLease(clientName, namespaces);
   }
 
   @Override // ClientProtocol
@@ -1097,24 +1097,7 @@ public DatanodeInfo[] getDatanodeReport(
     Map<FederationNamespaceInfo, DatanodeInfo[]> results =
         rpcClient.invokeConcurrent(nss, method, requireResponse, false,
             timeOutMs, DatanodeInfo[].class);
-    for (Entry<FederationNamespaceInfo, DatanodeInfo[]> entry :
-        results.entrySet()) {
-      FederationNamespaceInfo ns = entry.getKey();
-      DatanodeInfo[] result = entry.getValue();
-      for (DatanodeInfo node : result) {
-        String nodeId = node.getXferAddr();
-        DatanodeInfo dn = datanodesMap.get(nodeId);
-        if (dn == null || node.getLastUpdate() > dn.getLastUpdate()) {
-          // Add the subcluster as a suffix to the network location
-          node.setNetworkLocation(
-              NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
-              node.getNetworkLocation());
-          datanodesMap.put(nodeId, node);
-        } else {
-          LOG.debug("{} is in multiple subclusters", nodeId);
-        }
-      }
-    }
+    updateDnMap(results, datanodesMap);
     // Map -> Array
     Collection<DatanodeInfo> datanodes = datanodesMap.values();
     return toArray(datanodes, DatanodeInfo.class);
@@ -1348,7 +1331,7 @@ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
     clientProto.modifyAclEntries(src, aclSpec);
   }
 
-  @Override // ClienProtocol
+  @Override // ClientProtocol
   public void removeAclEntries(String src, List<AclEntry> aclSpec)
       throws IOException {
     clientProto.removeAclEntries(src, aclSpec);
@@ -1580,6 +1563,11 @@ public void satisfyStoragePolicy(String path) throws IOException {
     clientProto.satisfyStoragePolicy(path);
   }
 
+  @Override // ClientProtocol
+  public DatanodeInfo[] getSlowDatanodeReport() throws IOException {
+    return clientProto.getSlowDatanodeReport();
+  }
+
   @Override // NamenodeProtocol
   public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size,
       long minBlockSize, long hotBlockTimeInterval) throws IOException {
@@ -1745,8 +1733,7 @@ protected List<RemoteLocation> getLocationsForPath(String path,
       final PathLocation location =
           this.subclusterResolver.getDestinationForPath(path);
       if (location == null) {
-        throw new IOException("Cannot find locations for " + path + " in " +
-            this.subclusterResolver.getClass().getSimpleName());
+        throw new NoLocationException(path, this.subclusterResolver.getClass());
       }
 
       // We may block some write operations
@@ -1779,6 +1766,9 @@ protected List<RemoteLocation> getLocationsForPath(String path,
           locs.add(loc);
         }
       }
+      if (locs.isEmpty()) {
+        throw new NoLocationException(path, this.subclusterResolver.getClass());
+      }
       return locs;
     } catch (IOException ioe) {
       if (this.rpcMonitor != null) {
@@ -1992,6 +1982,57 @@ public int getSchedulerJobCount() {
     return fedRenameScheduler.getAllJobs().size();
   }
 
+  public String refreshFairnessPolicyController() {
+    return rpcClient.refreshFairnessPolicyController(new Configuration());
+  }
+
+  /**
+   * Get the slow running datanodes report with a timeout.
+   *
+   * @param requireResponse If we require all the namespaces to report.
+   * @param timeOutMs Time out for the reply in milliseconds.
+   * @return List of datanodes.
+   * @throws IOException If it cannot get the report.
+   */
+  public DatanodeInfo[] getSlowDatanodeReport(boolean requireResponse, long timeOutMs)
+      throws IOException {
+    checkOperation(OperationCategory.UNCHECKED);
+
+    Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
+    RemoteMethod method = new RemoteMethod("getSlowDatanodeReport");
+
+    Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
+    Map<FederationNamespaceInfo, DatanodeInfo[]> results =
+        rpcClient.invokeConcurrent(nss, method, requireResponse, false,
+            timeOutMs, DatanodeInfo[].class);
+    updateDnMap(results, datanodesMap);
+    // Map -> Array
+    Collection<DatanodeInfo> datanodes = datanodesMap.values();
+    return toArray(datanodes, DatanodeInfo.class);
+  }
+
+  private void updateDnMap(Map<FederationNamespaceInfo, DatanodeInfo[]> results,
+      Map<String, DatanodeInfo> datanodesMap) {
+    for (Entry<FederationNamespaceInfo, DatanodeInfo[]> entry :
+        results.entrySet()) {
+      FederationNamespaceInfo ns = entry.getKey();
+      DatanodeInfo[] result = entry.getValue();
+      for (DatanodeInfo node : result) {
+        String nodeId = node.getXferAddr();
+        DatanodeInfo dn = datanodesMap.get(nodeId);
+        if (dn == null || node.getLastUpdate() > dn.getLastUpdate()) {
+          // Add the subcluster as a suffix to the network location
+          node.setNetworkLocation(
+              NodeBase.PATH_SEPARATOR_STR + ns.getNameserviceId() +
+                  node.getNetworkLocation());
+          datanodesMap.put(nodeId, node);
+        } else {
+          LOG.debug("{} is in multiple subclusters", nodeId);
+        }
+      }
+    }
+  }
+
   /**
    * Deals with loading datanode report into the cache and refresh.
    */
@@ -2024,12 +2065,7 @@ public DatanodeInfo[] load(DatanodeReportType type) throws Exception {
     public ListenableFuture<DatanodeInfo[]> reload(
         final DatanodeReportType type, DatanodeInfo[] oldValue)
         throws Exception {
-      return executorService.submit(new Callable<DatanodeInfo[]>() {
-        @Override
-        public DatanodeInfo[] call() throws Exception {
-          return load(type);
-        }
-      });
+      return executorService.submit(() -> load(type));
     }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java
new file mode 100644
index 0000000000000..a15a0001e535a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.HashSet;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.LongAccumulator;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RouterFederatedStateProto;
+import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly;
+import org.apache.hadoop.ipc.AlignmentContext;
+import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
+
+
+/**
+ * This is the router implementation to hold the state Ids for all
+ * namespaces. This object is only updated by responses from NameNodes.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+class RouterStateIdContext implements AlignmentContext {
+
+  private final HashSet<String> coordinatedMethods;
+  /**
+   * Collection of last-seen namespace state Ids for a set of namespaces.
+   * Each value is globally shared by all outgoing connections to a particular namespace,
+   * so updates should only be performed using reliable responses from NameNodes.
+   */
+  private final ConcurrentHashMap<String, LongAccumulator> namespaceIdMap;
+  // Size limit for the map of state Ids to send to clients.
+  private final int maxSizeOfFederatedStateToPropagate;
+
+  RouterStateIdContext(Configuration conf) {
+    this.coordinatedMethods = new HashSet<>();
+    // For now, only ClientProtocol methods can be coordinated, so only checking
+    // against ClientProtocol.
+    for (Method method : ClientProtocol.class.getDeclaredMethods()) {
+      if (method.isAnnotationPresent(ReadOnly.class)
+          && method.getAnnotationsByType(ReadOnly.class)[0].isCoordinated()) {
+        coordinatedMethods.add(method.getName());
+      }
+    }
+
+    namespaceIdMap = new ConcurrentHashMap<>();
+
+    maxSizeOfFederatedStateToPropagate =
+        conf.getInt(RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE,
+        RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE_DEFAULT);
+  }
+
+  /**
+   * Adds the {@link #namespaceIdMap} to the response header that will be sent to a client.
+   */
+  public void setResponseHeaderState(RpcResponseHeaderProto.Builder headerBuilder) {
+    if (namespaceIdMap.isEmpty()) {
+      return;
+    }
+    RouterFederatedStateProto.Builder builder = RouterFederatedStateProto.newBuilder();
+    namespaceIdMap.forEach((k, v) -> builder.putNamespaceStateIds(k, v.get()));
+    headerBuilder.setRouterFederatedState(builder.build().toByteString());
+  }
+
+  public LongAccumulator getNamespaceStateId(String nsId) {
+    return namespaceIdMap.computeIfAbsent(nsId, key -> new LongAccumulator(Math::max, Long.MIN_VALUE));
+  }
+
+  public void removeNamespaceStateId(String nsId) {
+    namespaceIdMap.remove(nsId);
+  }
+
+  /**
+   * Utility function to parse routerFederatedState field in RPC headers.
+   */
+  public static Map<String, Long> getRouterFederatedStateMap(ByteString byteString) {
+    if (byteString != null) {
+      RouterFederatedStateProto federatedState;
+      try {
+        federatedState = RouterFederatedStateProto.parseFrom(byteString);
+      } catch (InvalidProtocolBufferException e) {
+        throw new RuntimeException(e);
+      }
+      return federatedState.getNamespaceStateIdsMap();
+    } else {
+      return Collections.emptyMap();
+    }
+  }
+
+  public static long getClientStateIdFromCurrentCall(String nsId) {
+    Long clientStateID = Long.MIN_VALUE;
+    Server.Call call = Server.getCurCall().get();
+    if (call != null) {
+      ByteString callFederatedNamespaceState = call.getFederatedNamespaceState();
+      if (callFederatedNamespaceState != null) {
+        Map<String, Long> clientFederatedStateIds = getRouterFederatedStateMap(callFederatedNamespaceState);
+        clientStateID = clientFederatedStateIds.getOrDefault(nsId, Long.MIN_VALUE);
+      }
+    }
+    return clientStateID;
+  }
+
+  @Override
+  public void updateResponseState(RpcResponseHeaderProto.Builder header) {
+    if (namespaceIdMap.size() <= maxSizeOfFederatedStateToPropagate) {
+      setResponseHeaderState(header);
+    }
+  }
+
+  @Override
+  public void receiveResponseState(RpcResponseHeaderProto header) {
+    // Do nothing.
+  }
+
+  @Override
+  public void updateRequestState(RpcRequestHeaderProto.Builder header) {
+    // Do nothing.
+  }
+
+  /**
+   * Routers do not update their state using information from clients
+   * to avoid clients interfering with one another.
+   */
+  @Override
+  public long receiveRequestState(RpcRequestHeaderProto header,
+      long clientWaitTime) throws RetriableException {
+    // Do nothing.
+    return 0;
+  }
+
+  @Override
+  public long getLastSeenStateId() {
+    return 0;
+  }
+
+  @Override
+  public boolean isCoordinatedCall(String protocolName, String methodName) {
+    return protocolName.equals(ClientProtocol.class.getCanonicalName())
+        && coordinatedMethods.contains(methodName);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index cd233e0bed7a2..a66953b1bd752 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -19,6 +19,8 @@
 
 import static org.apache.hadoop.util.StringUtils.getTrimmedStringCollection;
 
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -72,6 +74,8 @@
 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
 import org.apache.hadoop.hdfs.web.resources.SnapshotNameParam;
+import org.apache.hadoop.hdfs.web.resources.SnapshotDiffStartPathParam;
+import org.apache.hadoop.hdfs.web.resources.SnapshotDiffIndexParam;
 import org.apache.hadoop.hdfs.web.resources.StartAfterParam;
 import org.apache.hadoop.hdfs.web.resources.StoragePolicyParam;
 import org.apache.hadoop.hdfs.web.resources.StorageSpaceQuotaParam;
@@ -335,6 +339,8 @@ protected Response get(
       final FsActionParam fsAction,
       final SnapshotNameParam snapshotName,
       final OldSnapshotNameParam oldSnapshotName,
+      final SnapshotDiffStartPathParam snapshotDiffStartPath,
+      final SnapshotDiffIndexParam snapshotDiffIndex,
       final TokenKindParam tokenKind,
       final TokenServiceParam tokenService,
       final NoRedirectParam noredirectParam,
@@ -383,6 +389,7 @@ protected Response get(
         return super.get(ugi, delegation, username, doAsUser, fullpath, op,
             offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
             excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
+            snapshotDiffStartPath, snapshotDiffIndex,
             tokenKind, tokenService, noredirectParam, startAfter);
       }
       default:
@@ -403,7 +410,7 @@ protected Response get(
    * @param path Path to check.
    * @param op Operation to perform.
    * @param openOffset Offset for opening a file.
-   * @param excludeDatanodes Blocks to excluded.
+   * @param excludeDatanodes Blocks to exclude.
    * @param parameters Other parameters.
    * @return Redirection URI.
    * @throws URISyntaxException If it cannot parse the URI.
@@ -414,6 +421,9 @@ private URI redirectURI(final Router router, final UserGroupInformation ugi,
       final DoAsParam doAsUser, final String path, final HttpOpParam.Op op,
       final long openOffset, final String excludeDatanodes,
       final Param<?, ?>... parameters) throws URISyntaxException, IOException {
+    if (!DFSUtil.isValidName(path)) {
+      throw new InvalidPathException(path);
+    }
     final DatanodeInfo dn =
         chooseDatanode(router, path, op, openOffset, excludeDatanodes);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
index 7b28c03a529ad..6fea9b9946d4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/CachedRecordStore.java
@@ -85,7 +85,7 @@ protected CachedRecordStore(Class<R> clazz, StateStoreDriver driver) {
    *
    * @param clazz Class of the record to store.
    * @param driver State Store driver.
-   * @param over If the entries should be override if they expire
+   * @param over If the entries should be overridden if they expire
    */
   protected CachedRecordStore(
       Class<R> clazz, StateStoreDriver driver, boolean over) {
@@ -100,7 +100,7 @@ protected CachedRecordStore(
    * @throws StateStoreUnavailableException If the cache is not initialized.
    */
   private void checkCacheAvailable() throws StateStoreUnavailableException {
-    if (!this.initialized) {
+    if (!getDriver().isDriverReady() || !this.initialized) {
       throw new StateStoreUnavailableException(
           "Cached State Store not initialized, " +
           getRecordClass().getSimpleName() + " records not valid");
@@ -125,7 +125,6 @@ public boolean loadCache(boolean force) throws IOException {
       } catch (IOException e) {
         LOG.error("Cannot get \"{}\" records from the State Store",
             getRecordClass().getSimpleName());
-        this.initialized = false;
         return false;
       }
 
@@ -153,7 +152,7 @@ public boolean loadCache(boolean force) throws IOException {
   }
 
   /**
-   * Check if it's time to update the cache. Update it it was never updated.
+   * Check if it's time to update the cache. Update it was never updated.
    *
    * @return If it's time to update this cache.
    */
@@ -206,7 +205,7 @@ public void overrideExpiredRecords(QueryResult<R> query) throws IOException {
    * Updates the state store with any record overrides we detected, such as an
    * expired state.
    *
-   * @param record Record record to be updated.
+   * @param record record to be updated.
    * @throws IOException If the values cannot be updated.
    */
   public void overrideExpiredRecord(R record) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
index 4352ae19bde18..c6545d7425c92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java
@@ -48,7 +48,7 @@
  * StateStoreDriver}, NameNode registrations are cached until the next query.
  * The fetched registration data is aggregated using a quorum to determine the
  * best/most accurate state for each NameNode. The cache is periodically updated
- * by the @{link StateStoreCacheUpdateService}.
+ * by the {@link StateStoreCacheUpdateService}.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
index 507c83786a8f4..77939799e72b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java
@@ -272,6 +272,15 @@ public <T extends RecordStore<?>> T getRegisteredRecordStore(
     return null;
   }
 
+  /**
+   * Get the list of all RecordStores.
+   * @return a list of each RecordStore.
+   */
+  @SuppressWarnings("unchecked")
+  public <T extends RecordStore<? extends BaseRecord>> List<T> getRecordStores() {
+    return new ArrayList<>((Collection<T>) recordStores.values());
+  }
+
   /**
    * List of records supported by this State Store.
    *
@@ -331,7 +340,7 @@ public StateStoreDriver getDriver() {
   }
 
   /**
-   * Fetch a unique identifier for this state store instance. Typically it is
+   * Fetch a unique identifier for this state store instance. Typically, it is
    * the address of the router.
    *
    * @return Unique identifier for this store.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
index d595a97d01717..a4e9c1ce82bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreDriver.java
@@ -97,7 +97,7 @@ protected Configuration getConf() {
   }
 
   /**
-   * Gets a unique identifier for the running task/process. Typically the
+   * Gets a unique identifier for the running task/process. Typically, the
    * router address.
    *
    * @return Unique identifier for the running task.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index 871919594f57d..c93d919aea0a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
@@ -85,7 +85,8 @@ protected abstract <T extends BaseRecord> BufferedReader getReader(
    * @param path Path of the record to write.
    * @return Writer for the record.
    */
-  protected abstract <T extends BaseRecord> BufferedWriter getWriter(
+  @VisibleForTesting
+  public abstract <T extends BaseRecord> BufferedWriter getWriter(
       String path);
 
   /**
@@ -348,25 +349,18 @@ public <T extends BaseRecord> boolean putAll(
     for (Entry<String, T> entry : toWrite.entrySet()) {
       String recordPath = entry.getKey();
       String recordPathTemp = recordPath + "." + now() + TMP_MARK;
-      BufferedWriter writer = getWriter(recordPathTemp);
-      try {
+      boolean recordWrittenSuccessfully = true;
+      try (BufferedWriter writer = getWriter(recordPathTemp)) {
         T record = entry.getValue();
         String line = serializeString(record);
         writer.write(line);
       } catch (IOException e) {
         LOG.error("Cannot write {}", recordPathTemp, e);
+        recordWrittenSuccessfully = false;
         success = false;
-      } finally {
-        if (writer != null) {
-          try {
-            writer.close();
-          } catch (IOException e) {
-            LOG.error("Cannot close the writer for {}", recordPathTemp, e);
-          }
-        }
       }
       // Commit
-      if (!rename(recordPathTemp, recordPath)) {
+      if (recordWrittenSuccessfully && !rename(recordPathTemp, recordPath)) {
         LOG.error("Failed committing record into {}", recordPath);
         success = false;
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
index 9d2b1ab2fb73a..6ca2663716162 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileImpl.java
@@ -31,6 +31,7 @@
 import java.util.List;
 
 import org.apache.commons.lang3.ArrayUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
 import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
 import org.slf4j.Logger;
@@ -125,7 +126,8 @@ protected <T extends BaseRecord> BufferedReader getReader(String filename) {
   }
 
   @Override
-  protected <T extends BaseRecord> BufferedWriter getWriter(String filename) {
+  @VisibleForTesting
+  public <T extends BaseRecord> BufferedWriter getWriter(String filename) {
     BufferedWriter writer = null;
     try {
       LOG.debug("Writing file: {}", filename);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
index e6bf159e2f597..ee34d8a4cabbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileSystemImpl.java
@@ -28,13 +28,14 @@
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
 import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
 import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
@@ -82,17 +83,8 @@ protected boolean mkdir(String path) {
   @Override
   protected boolean rename(String src, String dst) {
     try {
-      if (fs instanceof DistributedFileSystem) {
-        DistributedFileSystem dfs = (DistributedFileSystem)fs;
-        dfs.rename(new Path(src), new Path(dst), Options.Rename.OVERWRITE);
-        return true;
-      } else {
-        // Replace should be atomic but not available
-        if (fs.exists(new Path(dst))) {
-          fs.delete(new Path(dst), true);
-        }
-        return fs.rename(new Path(src), new Path(dst));
-      }
+      FileUtil.rename(fs, new Path(src), new Path(dst), Options.Rename.OVERWRITE);
+      return true;
     } catch (Exception e) {
       LOG.error("Cannot rename {} to {}", src, dst, e);
       return false;
@@ -148,7 +140,8 @@ protected <T extends BaseRecord> BufferedReader getReader(String pathName) {
   }
 
   @Override
-  protected <T extends BaseRecord> BufferedWriter getWriter(String pathName) {
+  @VisibleForTesting
+  public <T extends BaseRecord> BufferedWriter getWriter(String pathName) {
     BufferedWriter writer = null;
     Path path = new Path(pathName);
     try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
index c6441caf82119..45442da0ab570 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java
@@ -50,6 +50,7 @@
  * |--- MEMBERSHIP
  * |--- REBALANCER
  * |--- ROUTERS
+ * |--- DISABLE_NAMESERVICE
  */
 public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
index a63a0f3b3ab0e..5d22b77afe2fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MembershipStoreImpl.java
@@ -185,7 +185,9 @@ public NamenodeHeartbeatResponse namenodeHeartbeat(
 
   @Override
   public boolean loadCache(boolean force) throws IOException {
-    super.loadCache(force);
+    if (!super.loadCache(force)) {
+      return false;
+    }
 
     // Update local cache atomically
     cacheWriteLock.lock();
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
index 6b3e55f6d98fd..9b45f28396de6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java
@@ -31,19 +31,21 @@
  * StateStoreDriver} to handle querying, updating and deleting data records. The
  * data storage driver is initialized and maintained by the {@link
  * org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}. The state store
+ * StateStoreService}. The state store
  * supports fetching all records of a type, filtering by column values or
  * fetching a single record by its primary key.
  * <p>
  * The state store contains several API interfaces, one for each data records
  * type.
  * <ul>
- * <li>FederationMembershipStateStore: state of all Namenodes in the federation.
+ * <li>MembershipStore: state of all Namenodes in the federation.
  * Uses the MembershipState record.
- * <li>FederationMountTableStore: Mount table mapping paths in the global
+ * <li>MountTableStore: Mount table mapping paths in the global
  * namespace to individual subcluster paths. Uses the MountTable record.
- * <li>RouterStateStore: State of all routers in the federation. Uses the
+ * <li>RouterStore: State of all routers in the federation. Uses the
  * RouterState record.
+ * <li>DisabledNameserviceStore: state of all disabled nameservice in the federation.
+ * Uses the DisabledNameservice record.
  * </ul>
  * Each API is defined in a separate interface. The implementations of these
  * interfaces are responsible for accessing the {@link
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
index b70cccf593ae9..10b172eb50a21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/GetRouterRegistrationsRequest.java
@@ -22,7 +22,7 @@
 import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
 
 /**
- * API request for retrieving a all non-expired router registrations present in
+ * API request for retrieving an all non-expired router registrations present in
  * the state store.
  */
 public abstract class GetRouterRegistrationsRequest {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
index 8422a8c4b6d15..98988f8bb2d86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/FederationProtocolPBTranslator.java
@@ -92,7 +92,7 @@ public B getBuilder() {
 
   /**
    * Get the serialized proto object. If the translator was created from a byte
-   * stream, returns the intitial byte stream. Otherwise creates a new byte
+   * stream, returns the initial byte stream. Otherwise, creates a new byte
    * stream from the cached builder.
    *
    * @return Protobuf message object
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
index 4add8fa99c6eb..80889b3d4aa4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java
@@ -32,7 +32,7 @@
 /**
  * Data schema for storing NN registration information in the
  * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}.
+ * StateStoreService}.
  */
 public abstract class MembershipState extends BaseRecord
     implements FederationNamenodeContext {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
index 21c8c2f79fce4..3e05a12cd9b9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java
@@ -133,6 +133,10 @@ public abstract void setHighestPriorityLowRedundancyECBlocks(
 
   public abstract long getHighestPriorityLowRedundancyECBlocks();
 
+  public abstract void setPendingSPSPaths(int pendingSPSPaths);
+
+  public abstract int getPendingSPSPaths();
+
   @Override
   public SortedMap<String, String> getPrimaryKeys() {
     // This record is not stored directly, no key needed
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
index bedf37b64c8e8..a80866780129d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
@@ -42,9 +42,9 @@
 /**
  * Data schema for {@link
  * org.apache.hadoop.hdfs.server.federation.store.MountTableStore
- * FederationMountTableStore} data stored in the {@link
+ * MountTableStore} data stored in the {@link
  * org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}. Supports string serialization.
+ * StateStoreService}. Supports string serialization.
  */
 public abstract class MountTable extends BaseRecord {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
index 761e2a4872e9a..337a58c359812 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java
@@ -31,7 +31,7 @@
  * Entry to log the state of a
  * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} in the
  * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService
- * FederationStateStoreService}.
+ * StateStoreService}.
  */
 public abstract class RouterState extends BaseRecord {
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
index 2caa59dfca7e5..9dff84befa4ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MembershipStatsPBImpl.java
@@ -297,4 +297,14 @@ public long getHighestPriorityLowRedundancyECBlocks() {
     return this.translator.getProtoOrBuilder()
         .getHighestPriorityLowRedundancyECBlocks();
   }
+
+  @Override
+  public void setPendingSPSPaths(int pendingSPSPaths) {
+    this.translator.getBuilder().setPendingSPSPaths(pendingSPSPaths);
+  }
+
+  @Override
+  public int getPendingSPSPaths() {
+    return this.translator.getProtoOrBuilder().getPendingSPSPaths();
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index b8e7c796a147d..3ecb4c2caba70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.tools.federation;
 
 import java.io.IOException;
+import java.io.PrintStream;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collection;
@@ -26,6 +27,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.TreeMap;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -46,6 +48,10 @@
 import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
 import org.apache.hadoop.hdfs.server.federation.router.RouterQuotaUsage;
 import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager;
+import org.apache.hadoop.hdfs.server.federation.store.CachedRecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.RecordStore;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest;
@@ -70,7 +76,9 @@
 import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.BaseRecord;
 import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord;
 import org.apache.hadoop.ipc.ProtobufRpcEngine2;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RefreshResponse;
@@ -97,6 +105,7 @@
 public class RouterAdmin extends Configured implements Tool {
 
   private static final Logger LOG = LoggerFactory.getLogger(RouterAdmin.class);
+  private static final String DUMP_COMMAND = "-dumpState";
 
   private RouterClient client;
 
@@ -133,7 +142,7 @@ private String getUsage(String cmd) {
       String[] commands =
           {"-add", "-update", "-rm", "-ls", "-getDestination", "-setQuota",
               "-setStorageTypeQuota", "-clrQuota", "-clrStorageTypeQuota",
-              "-safemode", "-nameservice", "-getDisabledNameservices",
+              DUMP_COMMAND, "-safemode", "-nameservice", "-getDisabledNameservices",
               "-refresh", "-refreshRouterArgs",
               "-refreshSuperUserGroupsConfiguration", "-refreshCallQueue"};
       StringBuilder usage = new StringBuilder();
@@ -187,13 +196,15 @@ private String getUsage(String cmd) {
       return "\t[-refreshSuperUserGroupsConfiguration]";
     } else if (cmd.equals("-refreshCallQueue")) {
       return "\t[-refreshCallQueue]";
+    } else if (cmd.equals(DUMP_COMMAND)) {
+      return "\t[" + DUMP_COMMAND + "]";
     }
     return getUsage(null);
   }
 
   /**
    * Usage: validates the maximum number of arguments for a command.
-   * @param arg List of of command line parameters.
+   * @param arg List of command line parameters.
    */
   private void validateMax(String[] arg) {
     if (arg[0].equals("-ls")) {
@@ -224,7 +235,8 @@ private void validateMax(String[] arg) {
       if (arg.length > 1) {
         throw new IllegalArgumentException("No arguments allowed");
       }
-    } else if (arg[0].equals("-refreshCallQueue")) {
+    } else if (arg[0].equals("-refreshCallQueue") ||
+        arg[0].equals(DUMP_COMMAND)) {
       if (arg.length > 1) {
         throw new IllegalArgumentException("No arguments allowed");
       }
@@ -286,6 +298,15 @@ private boolean validateMin(String[] argv) {
     return true;
   }
 
+  /**
+   * Does this command run in the local process?
+   * @param cmd the string of the command
+   * @return is this a local command?
+   */
+  boolean isLocalCommand(String cmd) {
+    return DUMP_COMMAND.equals(cmd);
+  }
+
   @Override
   public int run(String[] argv) throws Exception {
     if (argv.length < 1) {
@@ -303,6 +324,10 @@ public int run(String[] argv) throws Exception {
       System.err.println("Not enough parameters specificed for cmd " + cmd);
       printUsage(cmd);
       return exitCode;
+    } else if (isLocalCommand(argv[0])) {
+      if (DUMP_COMMAND.equals(argv[0])) {
+        return dumpStateStore(getConf(), System.out) ? 0 : -1;
+      }
     }
     String address = null;
     // Initialize RouterClient
@@ -407,7 +432,7 @@ public int run(String[] argv) throws Exception {
       System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
       printUsage(cmd);
     } catch (RemoteException e) {
-      // This is a error returned by the server.
+      // This is an error returned by the server.
       // Print out the first line of the error message, ignore the stack trace.
       exitCode = -1;
       debugException = e;
@@ -807,7 +832,7 @@ public void listMounts(String[] argv, int i) throws IOException {
     } else if (argv[i].equals("-d")) { // Check if -d parameter is specified.
       detail = true;
       if (argv.length == 2) {
-        path = "/"; // If no path is provide with -ls -d.
+        path = "/"; // If no path is provided with -ls -d.
       } else {
         path = argv[++i];
       }
@@ -1301,6 +1326,49 @@ private int refreshCallQueue() throws IOException {
     return returnCode;
   }
 
+  /**
+   * Dumps the contents of the StateStore to stdout.
+   * @return true if it was successful
+   */
+  public static boolean dumpStateStore(Configuration conf,
+                                PrintStream output) throws IOException {
+    StateStoreService service = new StateStoreService();
+    conf.setBoolean(RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE, false);
+    service.init(conf);
+    service.loadDriver();
+    if (!service.isDriverReady()) {
+      System.err.println("Can't initialize driver");
+      return false;
+    }
+    // Get the stores sorted by name
+    Map<String, RecordStore<? extends BaseRecord>> stores = new TreeMap<>();
+    for(RecordStore<? extends BaseRecord> store: service.getRecordStores()) {
+      String recordName = StateStoreUtils.getRecordName(store.getRecordClass());
+      stores.put(recordName, store);
+    }
+    for (Entry<String, RecordStore<? extends BaseRecord>> pair: stores.entrySet()) {
+      String recordName = pair.getKey();
+      RecordStore<? extends BaseRecord> store = pair.getValue();
+      output.println("---- " + recordName + " ----");
+      if (store instanceof CachedRecordStore) {
+        for (Object record: ((CachedRecordStore) store).getCachedRecords()) {
+          if (record instanceof BaseRecord && record instanceof PBRecord) {
+            BaseRecord baseRecord = (BaseRecord) record;
+            // Generate the pseudo-json format of the protobuf record
+            String recordString = ((PBRecord) record).getProto().toString();
+            // Indent each line
+            recordString = "    " + recordString.replaceAll("\n", "\n    ");
+            output.println(String.format("  %s:", baseRecord.getPrimaryKey()));
+            output.println(recordString);
+          }
+        }
+        output.println();
+      }
+    }
+    service.stop();
+    return true;
+  }
+
   /**
    * Normalize a path for that filesystem.
    *
@@ -1341,4 +1409,4 @@ public FsPermission getMode() {
       return mode;
     }
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
index 4a83ebc6ca267..c8636826c3cc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto
@@ -54,6 +54,7 @@ message NamenodeMembershipStatsRecordProto {
   optional uint64 numberOfMissingBlocksWithReplicationFactorOne = 31;
   optional uint64 highestPriorityLowRedundancyReplicatedBlocks = 32;
   optional uint64 HighestPriorityLowRedundancyECBlocks = 33;
+  optional uint32 pendingSPSPaths = 34;
 }
 
 message NamenodeMembershipRecordProto {
@@ -310,4 +311,4 @@ message GetDisabledNameservicesRequestProto {
 
 message GetDisabledNameservicesResponseProto {
   repeated string nameServiceIds = 1;
-}
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
index fcf6a28475fbd..52a1e3a3bd1e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
@@ -134,6 +134,33 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.federation.router.enable.multiple.socket</name>
+    <value>false</value>
+    <description>
+      If enable multiple downstream socket or not. If true, ConnectionPool
+      will use a new socket when creating a new connection for the same user,
+      and RouterRPCClient will get a better throughput. It's best used with
+      dfs.federation.router.max.concurrency.per.connection together to get
+      a better throughput with fewer sockets. Such as enable
+      dfs.federation.router.enable.multiple.socket and
+      set dfs.federation.router.max.concurrency.per.connection = 20.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.federation.router.max.concurrency.per.connection</name>
+    <value>1</value>
+    <description>
+      The maximum number of requests that a connection can handle concurrently.
+      When the number of requests being processed by a socket is less than this value,
+      new request will be processed by this socket. When enable
+      dfs.federation.router.enable.multiple.socket, it's best
+      set this value greater than 1, such as 20, to avoid frequent
+      creation and idle sockets in the case of a NS with jitter requests.
+    </description>
+  </property>
+
   <property>
     <name>dfs.federation.router.connection.pool.clean.ms</name>
     <value>60000</value>
@@ -168,6 +195,16 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.federation.router.enable.get.dn.usage</name>
+    <value>true</value>
+    <description>
+      If true, the getNodeUsage method in RBFMetrics will return an up-to-date
+      result collecting from downstream nameservices. But it will take a long
+      time and take up thread resources. If false, it will return a mock result with all 0.
+    </description>
+  </property>
+
   <property>
     <name>dfs.federation.router.metrics.class</name>
     <value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
@@ -385,6 +422,14 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.federation.router.health.monitor.timeout</name>
+    <value>30s</value>
+    <description>
+      Time out for Router to obtain HAServiceStatus from NameNode.
+    </description>
+  </property>
+
   <property>
     <name>dfs.federation.router.heartbeat-state.interval</name>
     <value>5s</value>
@@ -723,6 +768,14 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.federation.router.fairness.acquire.timeout</name>
+    <value>1s</value>
+    <description>
+      The maximum time to wait for a permit.
+    </description>
+  </property>
+
   <property>
     <name>dfs.federation.router.federation.rename.bandwidth</name>
     <value>10</value>
@@ -781,4 +834,34 @@
       (delete the source path directly) and skip (skip both trash and deletion).
     </description>
   </property>
+
+  <property>
+    <name>dfs.federation.router.observer.read.default</name>
+    <value>false</value>
+    <description>
+      Whether observer reads are enabled. This is a default for all nameservices.
+      The default can be inverted for individual namespace by adding them to
+      dfs.federation.router.observer.read.overrides.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.federation.router.observer.read.overrides</name>
+    <value/>
+    <description>
+      Commas separated list of namespaces for which to invert the default configuration,
+      dfs.federation.router.observer.read.default, for whether to enable observer reads.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.federation.router.observer.federated.state.propagation.maxsize</name>
+    <value>5</value>
+    <description>
+      The maximum size of the federated state to send in the RPC header. Sending the federated
+      state removes the need to msync on every read call, but at the expense of having a larger
+      header. The cost tradeoff between the larger header and always msync'ing depends on the number
+      of namespaces in use and the latency of the msync requests.
+    </description>
+  </property>
 </configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html
index 49c3e6606accc..15f7d7feb3848 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html
@@ -311,7 +311,7 @@ <h4 class="modal-title" id="delete-modal-title">Delete</h4>
         </ul>
       </p>
     </script>
-    <script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+    <script type="text/javascript" src="/static/jquery-3.6.0.min.js">
     </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap-editable.min.js">
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
index 5d9a9521f523c..9fa43a308584b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -458,7 +458,7 @@
   </ul>
 </div>
 <small>
-<table class="table">
+<table class="table" id="table-mounttable">
   <thead>
     <tr>
       <th>Global path</th>
@@ -478,9 +478,9 @@
   <tbody>
     {#MountTable}
     <tr>
-      <td>{sourcePath}</td>
-      <td>{nameserviceId}</td>
-      <td>{path}</td>
+      <td ng-value="{sourcePath}">{sourcePath}</td>
+      <td ng-value="{nameserviceId}">{nameserviceId}</td>
+      <td ng-value="{path}">{path}</td>
       <td>{order}</td>
       <td align="center" class="federationhealth-mounttable-icon federationhealth-mounttable-{readonly}" title="{status}"/>
       <td align="center" class="mount-table-icon mount-table-fault-tolerant-{faulttolerant}" title="{ftStatus}"></td>
@@ -499,7 +499,7 @@
 
 
 
-<script type="text/javascript" src="static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="static/jquery.dataTables.min.js"></script>
 <script type="text/javascript" src="static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="static/dataTables.bootstrap.js"></script>
@@ -509,5 +509,6 @@
 <script type="text/javascript" src="static/dfs-dust.js"></script>
 <script type="text/javascript" src="/static/d3-v4.1.1.min.js"></script>
 <script type="text/javascript" src="federationhealth.js"></script>
+<script type="text/javascript" src="static/histogram-hostip.js"></script>
 </body>
 </html>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
index dc24a322d770b..6220ca14a8cbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
@@ -475,6 +475,22 @@
         var base = dust.makeBase(HELPERS);
         dust.render('mounttable', base.push(data), function(err, out) {
           $('#tab-mounttable').html(out);
+          $('#table-mounttable').dataTable( {
+            'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ],
+            'columns': [
+              { 'orderDataType': 'ng-value', 'searchable': true },
+              { 'orderDataType': 'ng-value', 'searchable': true },
+              { 'orderDataType': 'ng-value', 'searchable': true },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" },
+              { 'type': 'string' , "defaultContent": "" }
+            ]});
           $('#ui-tabs a[href="#tab-mounttable"]').tab('show');
         });
       })).fail(ajax_error_handler);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
index 151289fa7f7d8..098c73a3b71dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -46,7 +46,6 @@ This approach has the same architecture as [YARN federation](../../hadoop-yarn/h
 ### Example flow
 The simplest configuration deploys a Router on each NameNode machine.
 The Router monitors the local NameNode and its state and heartbeats to the State Store.
-The Router monitors the local NameNode and heartbeats the state to the State Store.
 When a regular DFS client contacts any of the Routers to access a file in the federated filesystem, the Router checks the Mount Table in the State Store (i.e., the local cache) to find out which subcluster contains the file.
 Then it checks the Membership table in the State Store (i.e., the local cache) for the NameNode responsible for the subcluster.
 After it has identified the correct NameNode, the Router proxies the request.
@@ -329,6 +328,17 @@ To trigger a runtime-refresh of the resource specified by \<key\> on \<host:ipc\
 
     [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -refreshRouterArgs <host:ipc_port> <key> [arg1..argn]
 
+### Router state dump
+
+To diagnose the current state of the routers, you can use the
+dumpState command. It generates a text dump of the records in the
+State Store. Since it uses the configuration to find and read the
+state store, it is often easiest to use the machine where the routers
+run. The command runs locally, so the routers do not have to be up to
+use this command.
+
+    [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -dumpState
+
 Client configuration
 --------------------
 
@@ -375,6 +385,20 @@ With this setting a user can interact with `ns-fed` as a regular namespace:
 This federated namespace can also be set as the default one at **core-site.xml** using `fs.defaultFS`.
 
 
+NameNode configuration
+--------------------
+
+In order for the system to support data-locality, you must configure your NameNodes so that they will trust the routers to supply the user's client IP address. `dfs.namenode.ip-proxy-users` defines a comma separated list of users that are allowed to provide the client ip address via the caller context.
+
+```xml
+<configuration>
+  <property>
+    <name>dfs.namenode.ip-proxy-users</name>
+    <value>hdfs</value>
+  </property>
+</configuration>
+```
+
 Router configuration
 --------------------
 
@@ -402,11 +426,13 @@ The RPC server to receive connections from the clients.
 The Router forwards the client requests to the NameNodes.
 It uses a pool of connections to reduce the latency of creating them.
 
-| Property | Default | Description|
+| Property | Default | Description |
 |:---- |:---- |:---- |
 | dfs.federation.router.connection.pool-size | 1 | Size of the pool of connections from the router to namenodes. |
 | dfs.federation.router.connection.clean.ms | 10000 | Time interval, in milliseconds, to check if the connection pool should remove unused connections. |
 | dfs.federation.router.connection.pool.clean.ms | 60000 | Time interval, in milliseconds, to check if the connection manager should remove unused connection pools. |
+| dfs.federation.router.enable.multiple.socket | false | If true, ConnectionPool will use a new socket when creating a new connection for the same user. And it's best used with dfs.federation.router.max.concurrency.per.connection together. |
+| dfs.federation.router.max.concurrency.per.connection | 1 | The maximum number of requests that a connection can handle concurrently. |
 
 ### Admin server
 
@@ -500,7 +526,6 @@ Isolation and dedicated assignment of RPC handlers across all configured downstr
 
 | Property | Default | Description|
 |:---- |:---- |:---- |
-| dfs.federation.router.fairness.enable | `false` | If `true`, dedicated RPC handlers will be assigned to each nameservice based on the fairness assignment policy configured. |
 | dfs.federation.router.fairness.policy.controller.class | `org.apache.hadoop.hdfs.server.federation.fairness.NoRouterRpcFairnessPolicyController` | Default handler allocation model to be used if isolation feature is enabled. Recommend to use `org.apache.hadoop.hdfs.server.federation.fairness.StaticRouterRpcFairnessPolicyController` to fully use the feature. |
 | dfs.federation.router.fairness.handler.count.*EXAMPLENAMESERVICE* | | Dedicated handler assigned to a specific nameservice. If none is specified equal allocation is done across all nameservices. |
 | dfs.federation.router.fairness.handler.count.concurrent | | Dedicated handler assigned to fan out calls such as `renewLease`. |
@@ -514,7 +539,7 @@ More metrics info can see [RBF Metrics](../../hadoop-project-dist/hadoop-common/
 Router Federation Rename
 -------
 
-Enable Router to rename across namespaces. Currently it is implemented based on [HDFS Federation Balance](../../../hadoop-federation-balance/HDFSFederationBalance.md) and has some limits comparing with normal rename.
+Enable Router to rename across namespaces. Currently it is implemented based on [HDFS Federation Balance](../../hadoop-federation-balance/HDFSFederationBalance.md) and has some limits comparing with normal rename.
 1. It is much slower than the normal rename so need a longer RPC timeout configuration. See `ipc.client.rpc-timeout.ms` and its description for more information about RPC timeout.
 2. It doesn't support snapshot path.
 3. It doesn't support to rename path with multiple destinations.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
index e758eee4fda7c..b0a897d9f4bb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -92,8 +92,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.function.Supplier;
-
 /**
  * Helper utilities for testing HDFS Federation.
  */
@@ -174,26 +172,23 @@ public static void waitNamenodeRegistered(
       final String nsId, final String nnId,
       final FederationNamenodeServiceState state) throws Exception {
 
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-      @Override
-      public Boolean get() {
-        try {
-          List<? extends FederationNamenodeContext> namenodes =
-              resolver.getNamenodesForNameserviceId(nsId);
-          if (namenodes != null) {
-            for (FederationNamenodeContext namenode : namenodes) {
-              // Check if this is the Namenode we are checking
-              if (namenode.getNamenodeId() == nnId  ||
-                  namenode.getNamenodeId().equals(nnId)) {
-                return state == null || namenode.getState().equals(state);
-              }
+    GenericTestUtils.waitFor(() -> {
+      try {
+        List<? extends FederationNamenodeContext> namenodes =
+            resolver.getNamenodesForNameserviceId(nsId, false);
+        if (namenodes != null) {
+          for (FederationNamenodeContext namenode : namenodes) {
+            // Check if this is the Namenode we are checking
+            if (namenode.getNamenodeId() == nnId  ||
+                namenode.getNamenodeId().equals(nnId)) {
+              return state == null || namenode.getState().equals(state);
             }
           }
-        } catch (IOException e) {
-          // Ignore
         }
-        return false;
+      } catch (IOException e) {
+        // Ignore
       }
+      return false;
     }, 1000, 60 * 1000);
   }
 
@@ -209,22 +204,19 @@ public static void waitNamenodeRegistered(
       final ActiveNamenodeResolver resolver, final String nsId,
       final FederationNamenodeServiceState state) throws Exception {
 
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-      @Override
-      public Boolean get() {
-        try {
-          List<? extends FederationNamenodeContext> nns =
-              resolver.getNamenodesForNameserviceId(nsId);
-          for (FederationNamenodeContext nn : nns) {
-            if (nn.getState().equals(state)) {
-              return true;
-            }
+    GenericTestUtils.waitFor(() -> {
+      try {
+        List<? extends FederationNamenodeContext> nns =
+            resolver.getNamenodesForNameserviceId(nsId, false);
+        for (FederationNamenodeContext nn : nns) {
+          if (nn.getState().equals(state)) {
+            return true;
           }
-        } catch (IOException e) {
-          // Ignore
         }
-        return false;
+      } catch (IOException e) {
+        // Ignore
       }
+      return false;
     }, 1000, 20 * 1000);
   }
 
@@ -361,19 +353,16 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
    */
   public static void waitRouterRegistered(RouterStore stateManager,
       long routerCount, int timeout) throws Exception {
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-      @Override
-      public Boolean get() {
-        try {
-          List<RouterState> cachedRecords = stateManager.getCachedRecords();
-          if (cachedRecords.size() == routerCount) {
-            return true;
-          }
-        } catch (IOException e) {
-          // Ignore
+    GenericTestUtils.waitFor(() -> {
+      try {
+        List<RouterState> cachedRecords = stateManager.getCachedRecords();
+        if (cachedRecords.size() == routerCount) {
+          return true;
         }
-        return false;
+      } catch (IOException e) {
+        // Ignore
       }
+      return false;
     }, 100, timeout);
   }
 
@@ -390,15 +379,13 @@ public static void simulateThrowExceptionRouterRpcServer(
     ConnectionManager connectionManager =
         new ConnectionManager(server.getConfig());
     ConnectionManager spyConnectionManager = spy(connectionManager);
-    doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocation) throws Throwable {
-        LOG.info("Simulating connectionManager throw IOException {}",
-            invocation.getMock());
-        throw new IOException("Simulate connectionManager throw IOException");
-      }
+    doAnswer(invocation -> {
+      LOG.info("Simulating connectionManager throw IOException {}",
+          invocation.getMock());
+      throw new IOException("Simulate connectionManager throw IOException");
     }).when(spyConnectionManager).getConnection(
-        any(UserGroupInformation.class), any(String.class), any(Class.class));
+        any(UserGroupInformation.class), any(String.class), any(Class.class),
+        any(String.class));
 
     Whitebox.setInternalState(rpcClient, "connectionManager",
         spyConnectionManager);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
index 8a7a03e018b95..2c70395870496 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java
@@ -86,9 +86,12 @@
 import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
 import org.apache.hadoop.hdfs.server.federation.router.Router;
 import org.apache.hadoop.hdfs.server.federation.router.RouterClient;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcClient;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
 import org.apache.hadoop.hdfs.server.namenode.FSImage;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
+import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.net.NetUtils;
@@ -231,16 +234,31 @@ public FileSystem getFileSystem() throws IOException {
       return DistributedFileSystem.get(conf);
     }
 
+    public FileSystem getFileSystem(Configuration configuration) throws  IOException {
+      configuration.addResource(conf);
+      return DistributedFileSystem.get(configuration);
+    }
+
+    public FileSystem getFileSystemWithObserverReadProxyProvider() throws IOException {
+      Configuration observerReadConf = new Configuration(conf);
+      observerReadConf.set(DFS_NAMESERVICES,
+          observerReadConf.get(DFS_NAMESERVICES)+ ",router-service");
+      observerReadConf.set(DFS_HA_NAMENODES_KEY_PREFIX + ".router-service", "router1");
+      observerReadConf.set(DFS_NAMENODE_RPC_ADDRESS_KEY+ ".router-service.router1",
+          getFileSystemURI().toString());
+      observerReadConf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX
+          + "." + "router-service", ObserverReadProxyProvider.class.getName());
+      DistributedFileSystem.setDefaultUri(observerReadConf, "hdfs://router-service");
+
+      return DistributedFileSystem.get(observerReadConf);
+    }
+
     public DFSClient getClient(UserGroupInformation user)
         throws IOException, URISyntaxException, InterruptedException {
 
       LOG.info("Connecting to router at {}", fileSystemUri);
-      return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
-        @Override
-        public DFSClient run() throws IOException {
-          return new DFSClient(fileSystemUri, conf);
-        }
-      });
+      return user.doAs((PrivilegedExceptionAction<DFSClient>)
+          () -> new DFSClient(fileSystemUri, conf));
     }
 
     public RouterClient getAdminClient() throws IOException {
@@ -267,6 +285,14 @@ public DFSClient getClient() throws IOException, URISyntaxException {
     public Configuration getConf() {
       return conf;
     }
+
+    public RouterRpcServer getRouterRpcServer() {
+      return router.getRpcServer();
+    }
+
+    public RouterRpcClient getRouterRpcClient() {
+      return getRouterRpcServer().getRPCClient();
+    }
   }
 
   /**
@@ -374,12 +400,8 @@ public DFSClient getClient(UserGroupInformation user)
         throws IOException, URISyntaxException, InterruptedException {
 
       LOG.info("Connecting to namenode at {}", fileSystemUri);
-      return user.doAs(new PrivilegedExceptionAction<DFSClient>() {
-        @Override
-        public DFSClient run() throws IOException {
-          return new DFSClient(fileSystemUri, conf);
-        }
-      });
+      return user.doAs((PrivilegedExceptionAction<DFSClient>)
+          () -> new DFSClient(fileSystemUri, conf));
     }
 
     public DFSClient getClient() throws IOException, URISyntaxException {
@@ -779,7 +801,7 @@ public void startCluster(Configuration overrideConf) {
       Configuration nnConf = generateNamenodeConfiguration(ns0);
       if (overrideConf != null) {
         nnConf.addResource(overrideConf);
-        // Router also uses this configurations as initial values.
+        // Router also uses these configurations as initial values.
         routerConf = new Configuration(overrideConf);
       }
 
@@ -804,6 +826,7 @@ public void startCluster(Configuration overrideConf) {
           .numDataNodes(numDNs)
           .nnTopology(topology)
           .dataNodeConfOverlays(dnConfs)
+          .checkExitOnShutdown(false)
           .storageTypes(storageTypes)
           .racks(racks)
           .build();
@@ -1036,6 +1059,27 @@ public void switchToStandby(String nsId, String nnId) {
     }
   }
 
+  /**
+   * Switch a namenode in a nameservice to be the observer.
+   * @param nsId Nameservice identifier.
+   * @param nnId Namenode identifier.
+   */
+  public void switchToObserver(String nsId, String nnId) {
+    try {
+      int total = cluster.getNumNameNodes();
+      NameNodeInfo[] nns = cluster.getNameNodeInfos();
+      for (int i = 0; i < total; i++) {
+        NameNodeInfo nn = nns[i];
+        if (nn.getNameserviceId().equals(nsId) &&
+            nn.getNamenodeId().equals(nnId)) {
+          cluster.transitionToObserver(i);
+        }
+      }
+    } catch (Throwable e) {
+      LOG.error("Cannot transition to active", e);
+    }
+  }
+
   /**
    * Stop the federated HDFS cluster.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java
index a4755c20fcae4..27fcf8726b6c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java
@@ -156,15 +156,12 @@ protected void setupMock() throws IOException {
     NamespaceInfo nsInfo = new NamespaceInfo(1, this.nsId, this.nsId, 1);
     when(mockNn.versionRequest()).thenReturn(nsInfo);
 
-    when(mockNn.getServiceStatus()).thenAnswer(new Answer<HAServiceStatus>() {
-      @Override
-      public HAServiceStatus answer(InvocationOnMock invocation)
-          throws Throwable {
-        HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
-        haStatus.setNotReadyToBecomeActive("");
-        return haStatus;
-      }
-    });
+    when(mockNn.getServiceStatus()).
+        thenAnswer((Answer<HAServiceStatus>) invocation -> {
+          HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState());
+          haStatus.setNotReadyToBecomeActive("");
+          return haStatus;
+        });
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index 43efd85228d72..4aaa8e7569e88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -23,6 +23,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -59,6 +60,7 @@ public class MockResolver
   private String defaultNamespace = null;
   private boolean disableDefaultNamespace = false;
   private volatile boolean disableRegistration = false;
+  private TreeSet<String> disableNamespaces = new TreeSet<>();
 
   public MockResolver() {
     this.cleanRegistrations();
@@ -118,12 +120,24 @@ public void setDisableRegistration(boolean isDisable) {
     disableRegistration = isDisable;
   }
 
+  @Override public void updateUnavailableNamenode(String ns,
+      InetSocketAddress failedAddress) throws IOException {
+    updateNameNodeState(ns, failedAddress,
+        FederationNamenodeServiceState.UNAVAILABLE);
+  }
+
   @Override
   public void updateActiveNamenode(
       String nsId, InetSocketAddress successfulAddress) {
+    updateNameNodeState(nsId, successfulAddress,
+        FederationNamenodeServiceState.ACTIVE);
+  }
 
-    String address = successfulAddress.getHostName() + ":" +
-        successfulAddress.getPort();
+  private void updateNameNodeState(String nsId,
+      InetSocketAddress iAddr,
+      FederationNamenodeServiceState state) {
+    String sAddress = iAddr.getHostName() + ":" +
+        iAddr.getPort();
     String key = nsId;
     if (key != null) {
       // Update the active entry
@@ -131,13 +145,13 @@ public void updateActiveNamenode(
       List<FederationNamenodeContext> namenodes =
           (List<FederationNamenodeContext>) this.resolver.get(key);
       for (FederationNamenodeContext namenode : namenodes) {
-        if (namenode.getRpcAddress().equals(address)) {
+        if (namenode.getRpcAddress().equals(sAddress)) {
           MockNamenodeContext nn = (MockNamenodeContext) namenode;
-          nn.setState(FederationNamenodeServiceState.ACTIVE);
+          nn.setState(state);
           break;
         }
       }
-      // This operation modifies the list so we need to be careful
+      // This operation modifies the list, so we need to be careful
       synchronized(namenodes) {
         Collections.sort(namenodes, new NamenodePriorityComparator());
       }
@@ -146,14 +160,39 @@ public void updateActiveNamenode(
 
   @Override
   public synchronized List<? extends FederationNamenodeContext>
-      getNamenodesForNameserviceId(String nameserviceId) {
+      getNamenodesForNameserviceId(String nameserviceId, boolean observerRead) {
     // Return a copy of the list because it is updated periodically
     List<? extends FederationNamenodeContext> namenodes =
         this.resolver.get(nameserviceId);
     if (namenodes == null) {
       namenodes = new ArrayList<>();
     }
-    return Collections.unmodifiableList(new ArrayList<>(namenodes));
+
+    List<FederationNamenodeContext> ret = new ArrayList<>();
+
+    if (observerRead) {
+      Iterator<? extends FederationNamenodeContext> iterator = namenodes
+          .iterator();
+      List<FederationNamenodeContext> observerNN = new ArrayList<>();
+      List<FederationNamenodeContext> nonObserverNN = new ArrayList<>();
+      while (iterator.hasNext()) {
+        FederationNamenodeContext membership = iterator.next();
+        if (membership.getState() == FederationNamenodeServiceState.OBSERVER) {
+          observerNN.add(membership);
+        } else {
+          nonObserverNN.add(membership);
+        }
+      }
+      Collections.shuffle(observerNN);
+      Collections.sort(nonObserverNN, new NamenodePriorityComparator());
+      ret.addAll(observerNN);
+      ret.addAll(nonObserverNN);
+    } else {
+      ret.addAll(namenodes);
+      Collections.sort(ret, new NamenodePriorityComparator());
+    }
+
+    return Collections.unmodifiableList(ret);
   }
 
   @Override
@@ -300,9 +339,17 @@ public synchronized Set<FederationNamespaceInfo> getNamespaces()
     return Collections.unmodifiableSet(this.namespaces);
   }
 
+  public void clearDisableNamespaces() {
+    this.disableNamespaces.clear();
+  }
+
+  public void disableNamespace(String nsId) {
+    this.disableNamespaces.add(nsId);
+  }
+
   @Override
   public Set<String> getDisabledNamespaces() throws IOException {
-    return new TreeSet<>();
+    return this.disableNamespaces;
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java
index c3fc324255c08..8fc9de0cb261f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java
@@ -41,8 +41,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Test the Router handlers fairness control rejects
- * requests when the handlers are overloaded.
+ * Test the Router handlers fairness control rejects and accepts requests.
  */
 public class TestRouterHandlersFairness {
 
@@ -126,6 +125,12 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness)
       throws Exception {
 
     RouterContext routerContext = cluster.getRandomRouter();
+    URI address = routerContext.getFileSystemURI();
+    Configuration conf = new HdfsConfiguration();
+    final int numOps = 10;
+    AtomicInteger overloadException = new AtomicInteger();
+
+    // Test when handlers are overloaded
     if (fairness) {
       if (isConcurrent) {
         LOG.info("Taking fanout lock first");
@@ -142,11 +147,80 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness)
         }
       }
     }
-    URI address = routerContext.getFileSystemURI();
-    Configuration conf = new HdfsConfiguration();
-    final int numOps = 10;
-    final AtomicInteger overloadException = new AtomicInteger();
+    int originalRejectedPermits = getTotalRejectedPermits(routerContext);
+
+    // |- All calls should fail since permits not released
+    innerCalls(address, numOps, isConcurrent, conf, overloadException);
+
+    int latestRejectedPermits = getTotalRejectedPermits(routerContext);
+    assertEquals(latestRejectedPermits - originalRejectedPermits,
+        overloadException.get());
+
+    if (fairness) {
+      assertTrue(overloadException.get() > 0);
+      if (isConcurrent) {
+        LOG.info("Release fanout lock that was taken before test");
+        // take the lock for concurrent NS to block fanout calls
+        routerContext.getRouter().getRpcServer()
+            .getRPCClient().getRouterRpcFairnessPolicyController()
+            .releasePermit(RouterRpcFairnessConstants.CONCURRENT_NS);
+      } else {
+        for (String ns : cluster.getNameservices()) {
+          routerContext.getRouter().getRpcServer()
+              .getRPCClient().getRouterRpcFairnessPolicyController()
+              .releasePermit(ns);
+        }
+      }
+    } else {
+      assertEquals("Number of failed RPCs without fairness configured",
+          0, overloadException.get());
+    }
+
+    // Test when handlers are not overloaded
+    int originalAcceptedPermits = getTotalAcceptedPermits(routerContext);
+    overloadException = new AtomicInteger();
+
+    // |- All calls should succeed since permits not acquired
+    innerCalls(address, numOps, isConcurrent, conf, overloadException);
+
+    int latestAcceptedPermits = getTotalAcceptedPermits(routerContext);
+    assertEquals(latestAcceptedPermits - originalAcceptedPermits, numOps);
+    assertEquals(overloadException.get(), 0);
+  }
+
+  private void invokeSequential(ClientProtocol routerProto) throws IOException {
+    routerProto.getFileInfo("/test.txt");
+  }
+
+  private void invokeConcurrent(ClientProtocol routerProto, String clientName)
+      throws IOException {
+    routerProto.renewLease(clientName, null);
+  }
+
+  private int getTotalRejectedPermits(RouterContext routerContext) {
+    int totalRejectedPermits = 0;
+    for (String ns : cluster.getNameservices()) {
+      totalRejectedPermits += routerContext.getRouterRpcClient()
+          .getRejectedPermitForNs(ns);
+    }
+    totalRejectedPermits += routerContext.getRouterRpcClient()
+        .getRejectedPermitForNs(RouterRpcFairnessConstants.CONCURRENT_NS);
+    return totalRejectedPermits;
+  }
 
+  private int getTotalAcceptedPermits(RouterContext routerContext) {
+    int totalAcceptedPermits = 0;
+    for (String ns : cluster.getNameservices()) {
+      totalAcceptedPermits += routerContext.getRouterRpcClient()
+          .getAcceptedPermitForNs(ns);
+    }
+    totalAcceptedPermits += routerContext.getRouterRpcClient()
+        .getAcceptedPermitForNs(RouterRpcFairnessConstants.CONCURRENT_NS);
+    return totalAcceptedPermits;
+  }
+
+  private void innerCalls(URI address, int numOps, boolean isConcurrent,
+      Configuration conf, AtomicInteger overloadException) throws IOException {
     for (int i = 0; i < numOps; i++) {
       DFSClient routerClient = null;
       try {
@@ -177,35 +251,5 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness)
       }
       overloadException.get();
     }
-
-    if (fairness) {
-      assertTrue(overloadException.get() > 0);
-      if (isConcurrent) {
-        LOG.info("Release fanout lock that was taken before test");
-        // take the lock for concurrent NS to block fanout calls
-        routerContext.getRouter().getRpcServer()
-            .getRPCClient().getRouterRpcFairnessPolicyController()
-            .releasePermit(RouterRpcFairnessConstants.CONCURRENT_NS);
-      } else {
-        for (String ns : cluster.getNameservices()) {
-          routerContext.getRouter().getRpcServer()
-              .getRPCClient().getRouterRpcFairnessPolicyController()
-              .releasePermit(ns);
-        }
-      }
-    } else {
-      assertEquals("Number of failed RPCs without fairness configured",
-          0, overloadException.get());
-    }
   }
-
-  private void invokeSequential(ClientProtocol routerProto) throws IOException {
-    routerProto.getFileInfo("/test.txt");
-  }
-
-  private void invokeConcurrent(ClientProtocol routerProto, String clientName)
-      throws IOException {
-    routerProto.renewLease(clientName);
-  }
-
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java
new file mode 100644
index 0000000000000..0741f1aed441a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.federation.fairness;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.apache.hadoop.hdfs.server.federation.router.RemoteMethod;
+import org.apache.hadoop.hdfs.server.federation.router.RouterRpcClient;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX;
+import static org.junit.Assert.assertEquals;
+
+public class TestRouterRefreshFairnessPolicyController {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRouterRefreshFairnessPolicyController.class);
+  private final GenericTestUtils.LogCapturer controllerLog =
+      GenericTestUtils.LogCapturer.captureLogs(AbstractRouterRpcFairnessPolicyController.LOG);
+
+  private StateStoreDFSCluster cluster;
+
+  @BeforeClass
+  public static void setLogLevel() {
+    GenericTestUtils.setLogLevel(AbstractRouterRpcFairnessPolicyController.LOG, Level.DEBUG);
+  }
+
+  @After
+  public void cleanup() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Before
+  public void setupCluster() throws Exception {
+    cluster = new StateStoreDFSCluster(false, 2);
+    Configuration conf = new RouterConfigBuilder().stateStore().rpc().build();
+
+    // Handlers concurrent:ns0 = 3:3
+    conf.setClass(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS,
+        StaticRouterRpcFairnessPolicyController.class, RouterRpcFairnessPolicyController.class);
+    conf.setInt(RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY, 9);
+    // Allow metrics
+    conf.setBoolean(RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE, true);
+
+    // Datanodes not needed for this test.
+    cluster.setNumDatanodesPerNameservice(0);
+
+    cluster.addRouterOverrides(conf);
+    cluster.startCluster();
+    cluster.startRouters();
+    cluster.waitClusterUp();
+  }
+
+  @Test
+  public void testRefreshNonexistentHandlerClass() {
+    MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter();
+    routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS,
+        "org.apache.hadoop.hdfs.server.federation.fairness.ThisControllerDoesNotExist");
+    assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(),
+        routerContext.getRouterRpcClient()
+            .refreshFairnessPolicyController(routerContext.getConf()));
+  }
+
+  @Test
+  public void testRefreshClassDoesNotImplementControllerInterface() {
+    MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter();
+    routerContext.getConf()
+        .set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS, "java.lang.String");
+    assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(),
+        routerContext.getRouterRpcClient()
+            .refreshFairnessPolicyController(routerContext.getConf()));
+  }
+
+  @Test
+  public void testRefreshSuccessful() {
+    MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter();
+
+    routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS,
+        StaticRouterRpcFairnessPolicyController.class.getCanonicalName());
+    assertEquals(StaticRouterRpcFairnessPolicyController.class.getCanonicalName(),
+        routerContext.getRouterRpcClient()
+            .refreshFairnessPolicyController(routerContext.getConf()));
+
+    routerContext.getConf().set(RBFConfigKeys.DFS_ROUTER_FAIRNESS_POLICY_CONTROLLER_CLASS,
+        NoRouterRpcFairnessPolicyController.class.getCanonicalName());
+    assertEquals(NoRouterRpcFairnessPolicyController.class.getCanonicalName(),
+        routerContext.getRouterRpcClient()
+            .refreshFairnessPolicyController(routerContext.getConf()));
+  }
+
+  @Test
+  public void testConcurrentRefreshRequests() throws InterruptedException {
+    MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter();
+    RouterRpcClient client = Mockito.spy(routerContext.getRouterRpcClient());
+    controllerLog.clearOutput();
+
+    // Spawn 100 concurrent refresh requests
+    Thread[] threads = new Thread[100];
+    for (int i = 0; i < 100; i++) {
+      threads[i] = new Thread(() ->
+          client.refreshFairnessPolicyController(routerContext.getConf()));
+    }
+
+    for (Thread thread : threads) {
+      thread.start();
+    }
+
+    for (Thread thread : threads) {
+      thread.join();
+    }
+
+    // There should be 100 controller shutdowns. All controllers created should be shut down.
+    assertEquals(100, StringUtils.countMatches(controllerLog.getOutput(),
+        "Shutting down router fairness policy controller"));
+    controllerLog.clearOutput();
+  }
+
+  @Test
+  public void testRefreshStaticChangeHandlers() throws Exception {
+    // Setup and mock
+    MiniRouterDFSCluster.RouterContext routerContext = cluster.getRandomRouter();
+    RouterRpcClient client = Mockito.spy(routerContext.getRouterRpcClient());
+    final long sleepTime = 3000;
+    Mockito.doAnswer(invocationOnMock -> {
+      Thread.sleep(sleepTime);
+      return null;
+    }).when(client)
+        .invokeMethod(Mockito.any(), Mockito.any(), Mockito.anyBoolean(),
+            Mockito.any(), Mockito.any(), Mockito.any());
+
+    // No calls yet
+    assertEquals("{}",
+        routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs());
+    List<Thread> preRefreshInvocations = makeDummyInvocations(client, 4, "ns0");
+
+    Thread.sleep(2000);
+    // 3 permits acquired, calls will take 3s to finish and release permits
+    // 1 invocation rejected
+    assertEquals("{\"ns0\":3}",
+        routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs());
+    assertEquals("{\"ns0\":1}",
+        routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitRejectedPerNs());
+
+    Configuration conf = routerContext.getConf();
+    final int newNs0Permits = 2;
+    final int newNs1Permits = 4;
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits);
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits);
+    Thread threadRefreshController = new Thread(() -> client.
+        refreshFairnessPolicyController(routerContext.getConf()));
+    threadRefreshController.start();
+    threadRefreshController.join();
+
+    // Wait for all dummy invocation threads to finish
+    for (Thread thread : preRefreshInvocations) {
+      thread.join();
+    }
+
+    // Controller should now have 2:4 handlers for ns0:ns1
+    // Make 4 calls to ns0 and 6 calls to ns1 so that each will fail twice
+    StaticRouterRpcFairnessPolicyController controller =
+        (StaticRouterRpcFairnessPolicyController) client.getRouterRpcFairnessPolicyController();
+    System.out.println(controller.getAvailableHandlerOnPerNs());
+    List<Thread> ns0Invocations = makeDummyInvocations(client, newNs0Permits + 2, "ns0");
+    List<Thread> ns1Invocations = makeDummyInvocations(client, newNs1Permits + 2, "ns1");
+
+    // Wait for these threads to finish
+    for (Thread thread : ns0Invocations) {
+      thread.join();
+    }
+    for (Thread thread : ns1Invocations) {
+      thread.join();
+    }
+    assertEquals("{\"ns0\":5,\"ns1\":4}",
+        routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitAcceptedPerNs());
+    assertEquals("{\"ns0\":3,\"ns1\":2}",
+        routerContext.getRouterRpcServer().getRPCMetrics().getProxyOpPermitRejectedPerNs());
+  }
+
+  private List<Thread> makeDummyInvocations(RouterRpcClient client, final int nThreads,
+      final String namespace) {
+    RemoteMethod dummyMethod = Mockito.mock(RemoteMethod.class);
+    List<Thread> threadAcquirePermits = new ArrayList<>();
+    for (int i = 0; i < nThreads; i++) {
+      Thread threadAcquirePermit = new Thread(() -> {
+        try {
+          client.invokeSingle(namespace, dummyMethod);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      });
+      threadAcquirePermits.add(threadAcquirePermit);
+      threadAcquirePermit.start();
+    }
+    return threadAcquirePermits;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java
index 8e816643ac5bb..1f5770b1ddaa3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java
@@ -23,13 +23,18 @@
 import org.apache.hadoop.hdfs.server.federation.router.FederationUtil;
 import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.junit.Test;
 import org.slf4j.LoggerFactory;
 
+import java.util.concurrent.TimeUnit;
+
 import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessConstants.CONCURRENT_NS;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT;
 import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
 import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -82,6 +87,26 @@ public void testHandlerAllocationPreconfigured() {
     assertFalse(routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS));
   }
 
+  @Test
+  public void testAcquireTimeout() {
+    Configuration conf = createConf(40);
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", 30);
+    conf.setTimeDuration(DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT, 100, TimeUnit.MILLISECONDS);
+    RouterRpcFairnessPolicyController routerRpcFairnessPolicyController =
+        FederationUtil.newFairnessPolicyController(conf);
+
+    // ns1 should have 30 permits allocated
+    for (int i = 0; i < 30; i++) {
+      assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns1"));
+    }
+    long acquireBeginTimeMs = Time.monotonicNow();
+    assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns1"));
+    long acquireTimeMs = Time.monotonicNow() - acquireBeginTimeMs;
+
+    // There are some other operations, so acquireTimeMs >= 100ms.
+    assertTrue(acquireTimeMs >= 100);
+  }
+
   @Test
   public void testAllocationErrorWithZeroHandlers() {
     Configuration conf = createConf(0);
@@ -101,6 +126,26 @@ public void testAllocationErrorForLowDefaultHandlersPerNS() {
     verifyInstantiationError(conf, 1, 3);
   }
 
+  @Test
+  public void testGetAvailableHandlerOnPerNs() {
+    RouterRpcFairnessPolicyController routerRpcFairnessPolicyController
+        = getFairnessPolicyController(30);
+    assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":10}",
+        routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
+    routerRpcFairnessPolicyController.acquirePermit("ns1");
+    assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":9}",
+        routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
+  }
+
+  @Test
+  public void testGetAvailableHandlerOnPerNsForNoFairness() {
+    Configuration conf = new Configuration();
+    RouterRpcFairnessPolicyController routerRpcFairnessPolicyController =
+        FederationUtil.newFairnessPolicyController(conf);
+    assertEquals("N/A",
+        routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs());
+  }
+
   @Test
   public void testAllocationErrorForLowPreconfiguredHandlers() {
     Configuration conf = createConf(1);
@@ -108,6 +153,30 @@ public void testAllocationErrorForLowPreconfiguredHandlers() {
     verifyInstantiationError(conf, 1, 4);
   }
 
+  @Test
+  public void testHandlerAllocationConcurrentConfigured() {
+    Configuration conf = createConf(5);
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", 1);
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns2", 1);
+    conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "concurrent", 1);
+    RouterRpcFairnessPolicyController routerRpcFairnessPolicyController =
+        FederationUtil.newFairnessPolicyController(conf);
+
+    // ns1, ns2 should have 1 permit each
+    assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns1"));
+    assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns2"));
+    assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns1"));
+    assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns2"));
+
+    // concurrent should have 3 permits
+    for (int i=0; i<3; i++) {
+      assertTrue(
+          routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS));
+    }
+    assertFalse(routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS));
+  }
+
+
   private void verifyInstantiationError(Configuration conf, int handlerCount,
       int totalDedicatedHandlers) {
     GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestNameserviceRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestNameserviceRPCMetrics.java
new file mode 100644
index 0000000000000..7b6bcb2143737
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestNameserviceRPCMetrics.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdfs.server.federation.metrics.NameserviceRPCMetrics.NAMESERVICE_RPC_METRICS_PREFIX;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+
+/**
+ * Test case for RouterClientMetrics.
+ */
+public class TestNameserviceRPCMetrics {
+  private static final Configuration CONF = new HdfsConfiguration();
+  static {
+    CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
+    CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
+    CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+    CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+  }
+
+  private static final int NUM_SUBCLUSTERS = 2;
+  private static final int NUM_DNS = 3;
+
+  /** Federated HDFS cluster. */
+  private static MiniRouterDFSCluster cluster;
+
+  /** The first Router Context for this federated cluster. */
+  private MiniRouterDFSCluster.RouterContext routerContext;
+
+  /** The first Router for this federated cluster. */
+  private Router router;
+
+  /** Filesystem interface to the Router. */
+  private FileSystem routerFS;
+  /** Filesystem interface to the Namenode. */
+  private FileSystem nnFS;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+    cluster = new MiniRouterDFSCluster(false, NUM_SUBCLUSTERS);
+    cluster.setNumDatanodesPerNameservice(NUM_DNS);
+    cluster.startCluster();
+
+    Configuration routerConf = new RouterConfigBuilder()
+        .metrics()
+        .rpc()
+        .quota()
+        .build();
+    cluster.addRouterOverrides(routerConf);
+    cluster.startRouters();
+
+    // Register and verify all NNs with all routers
+    cluster.registerNamenodes();
+    cluster.waitNamenodeRegistration();
+
+  }
+
+  @Before
+  public void testSetup() throws Exception {
+    // Create mock locations
+    cluster.installMockLocations();
+
+    // Delete all files via the NNs and verify
+    cluster.deleteAllFiles();
+
+    // Create test fixtures on NN
+    cluster.createTestDirectoriesNamenode();
+
+    // Wait to ensure NN has fully created its test directories
+    Thread.sleep(100);
+
+    routerContext = cluster.getRouters().get(0);
+    this.routerFS = routerContext.getFileSystem();
+
+    // Add extra location to the root mount / such that the root mount points:
+    // /
+    //   ns0 -> /target-ns0
+    //   ns1 -> /target-ns1
+    router = routerContext.getRouter();
+    MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+    resolver.addLocation("/target-ns0", cluster.getNameservices().get(0), "/target-ns0");
+    resolver.addLocation("/target-ns1", cluster.getNameservices().get(1), "/target-ns1");
+
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    cluster.shutdown();
+  }
+
+  @Test
+  public void testProxyOp() throws IOException {
+    routerFS.listStatus(new Path("/target-ns0"));
+    assertCounter("ProxyOp", 1L,
+        getMetrics(NAMESERVICE_RPC_METRICS_PREFIX + "ns0"));
+    assertCounter("ProxyOp", 0L,
+        getMetrics(NAMESERVICE_RPC_METRICS_PREFIX + "ns1"));
+
+    routerFS.listStatus(new Path("/target-ns1"));
+    assertCounter("ProxyOp", 1L,
+        getMetrics(NAMESERVICE_RPC_METRICS_PREFIX + "ns0"));
+    assertCounter("ProxyOp", 1L,
+        getMetrics(NAMESERVICE_RPC_METRICS_PREFIX + "ns1"));
+  }
+
+}
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java
index 25473f8df9233..c86397b511de6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java
@@ -219,6 +219,8 @@ public void testNameserviceStatsDataSource()
           json.getLong("numOfEnteringMaintenanceDataNodes"));
       assertEquals(stats.getProvidedSpace(),
           json.getLong("providedSpace"));
+      assertEquals(stats.getPendingSPSPaths(),
+          json.getInt("pendingSPSPaths"));
       nameservicesFound++;
     }
     assertEquals(getNameservices().size(), nameservicesFound);
@@ -296,6 +298,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) {
     long highestPriorityLowRedundancyReplicatedBlocks = 0;
     long highestPriorityLowRedundancyECBlocks = 0;
     long numFiles = 0;
+    int pendingSPSPaths = 0;
     for (MembershipState mock : getActiveMemberships()) {
       MembershipStats stats = mock.getStats();
       numBlocks += stats.getNumOfBlocks();
@@ -316,6 +319,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) {
           stats.getHighestPriorityLowRedundancyReplicatedBlocks();
       highestPriorityLowRedundancyECBlocks +=
           stats.getHighestPriorityLowRedundancyECBlocks();
+      pendingSPSPaths += stats.getPendingSPSPaths();
     }
 
     assertEquals(numBlocks, bean.getNumBlocks());
@@ -342,6 +346,7 @@ private void validateClusterStatsFederationBean(FederationMBean bean) {
         bean.getHighestPriorityLowRedundancyReplicatedBlocks());
     assertEquals(highestPriorityLowRedundancyECBlocks,
         bean.getHighestPriorityLowRedundancyECBlocks());
+    assertEquals(pendingSPSPaths, bean.getPendingSPSPaths());
   }
 
   private void validateClusterStatsRouterBean(RouterMBean bean) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
index da16c05910785..3397718745ffb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
@@ -156,7 +156,7 @@ public void testGetQuota() throws Exception {
 
   @Test
   public void testRenewLease() throws Exception {
-    router.getRpcServer().renewLease("test");
+    router.getRpcServer().renewLease("test", null);
     assertCounter("RenewLeaseOps", 2L, getMetrics(ROUTER_METRICS));
     assertCounter("ConcurrentRenewLeaseOps", 1L, getMetrics(ROUTER_METRICS));
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
index df80037c69917..b602a27c95f60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java
@@ -129,7 +129,7 @@ private void verifyFirstRegistration(String nsId, String nnId,
       int resultsCount, FederationNamenodeServiceState state)
           throws IOException {
     List<? extends FederationNamenodeContext> namenodes =
-        namenodeResolver.getNamenodesForNameserviceId(nsId);
+        namenodeResolver.getNamenodesForNameserviceId(nsId, false);
     if (resultsCount == 0) {
       assertNull(namenodes);
     } else {
@@ -291,18 +291,18 @@ public void testCacheUpdateOnNamenodeStateUpdate() throws IOException {
             HAServiceState.STANDBY)));
     stateStore.refreshCaches(true);
     // Check whether the namenpde state is reported correct as standby.
-    FederationNamenodeContext namenode =
-        namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0);
+    FederationNamenodeContext namenode = namenodeResolver
+        .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0);
     assertEquals(FederationNamenodeServiceState.STANDBY, namenode.getState());
     String rpcAddr = namenode.getRpcAddress();
     InetSocketAddress inetAddr = getInetSocketAddress(rpcAddr);
 
-    // If the namenode state changes and it serves request,
+    // If the namenode state changes, and it serves request,
     // RouterRpcClient calls updateActiveNamenode to update the state to active,
     // Check whether correct updated state is returned post update.
     namenodeResolver.updateActiveNamenode(NAMESERVICES[0], inetAddr);
-    FederationNamenodeContext namenode1 =
-        namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0);
+    FederationNamenodeContext namenode1 = namenodeResolver
+        .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0);
     assertEquals("The namenode state should be ACTIVE post update.",
         FederationNamenodeServiceState.ACTIVE, namenode1.getState());
   }
@@ -318,8 +318,8 @@ public void testCacheUpdateOnNamenodeStateUpdateWithIp()
 
     InetSocketAddress inetAddr = getInetSocketAddress(rpcAddress);
     namenodeResolver.updateActiveNamenode(NAMESERVICES[0], inetAddr);
-    FederationNamenodeContext namenode =
-        namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0);
+    FederationNamenodeContext namenode = namenodeResolver
+        .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0);
     assertEquals("The namenode state should be ACTIVE post update.",
         FederationNamenodeServiceState.ACTIVE, namenode.getState());
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
index 08e75b2d30918..0625bbadf7d42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java
@@ -43,7 +43,6 @@
 import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
 import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
 /**
@@ -78,12 +77,8 @@ public void testLocalResolver() throws IOException {
     StringBuilder sb = new StringBuilder("clientX");
     LocalResolver localResolver = new LocalResolver(conf, router);
     LocalResolver spyLocalResolver = spy(localResolver);
-    doAnswer(new Answer<String>() {
-      @Override
-      public String answer(InvocationOnMock invocation) throws Throwable {
-        return sb.toString();
-      }
-    }).when(spyLocalResolver).getClientAddr();
+    doAnswer((Answer<String>) invocation -> sb.toString()
+        ).when(spyLocalResolver).getClientAddr();
 
     // Add the mocks to the resolver
     MultipleDestinationMountTableResolver resolver =
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
index e397692e9a86d..920c9c4e519cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
@@ -19,7 +19,10 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RouterFederatedStateProto;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -31,11 +34,13 @@
 import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertNotNull;
@@ -80,15 +85,15 @@ public void shutdown() {
   public void testCleanup() throws Exception {
     Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
 
-    ConnectionPool pool1 = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, ClientProtocol.class);
+    ConnectionPool pool1 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1,
+        0, 10, 0.5f, ClientProtocol.class, null);
     addConnectionsToPool(pool1, 9, 4);
     poolMap.put(
         new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class),
         pool1);
 
-    ConnectionPool pool2 = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10, 0.5f, ClientProtocol.class);
+    ConnectionPool pool2 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER2,
+        0, 10, 0.5f, ClientProtocol.class, null);
     addConnectionsToPool(pool2, 10, 10);
     poolMap.put(
         new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS, ClientProtocol.class),
@@ -110,8 +115,8 @@ public void testCleanup() throws Exception {
     checkPoolConnections(TEST_USER2, 10, 10);
 
     // Make sure the number of connections doesn't go below minSize
-    ConnectionPool pool3 = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10, 0.5f, ClientProtocol.class);
+    ConnectionPool pool3 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER3,
+        2, 10, 0.5f, ClientProtocol.class, null);
     addConnectionsToPool(pool3, 8, 0);
     poolMap.put(
         new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS, ClientProtocol.class),
@@ -131,12 +136,50 @@ public void testCleanup() throws Exception {
     checkPoolConnections(TEST_USER3, 4, 2);
   }
 
+  @Test
+  public void testGetConnectionWithConcurrency() throws Exception {
+    Map<ConnectionPoolId, ConnectionPool> poolMap = connManager.getPools();
+    Configuration copyConf = new Configuration(conf);
+    copyConf.setInt(RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY, 20);
+
+    ConnectionPool pool = new ConnectionPool(
+        copyConf, TEST_NN_ADDRESS, TEST_USER1, 1, 10, 0.5f,
+        ClientProtocol.class, null);
+    poolMap.put(
+        new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class),
+        pool);
+    assertEquals(1, pool.getNumConnections());
+    // one connection can process the maximum number of requests concurrently.
+    for (int i = 0; i < 20; i++) {
+      ConnectionContext cc = pool.getConnection();
+      assertTrue(cc.isUsable());
+      cc.getClient();
+    }
+    assertEquals(1, pool.getNumConnections());
+
+    // Ask for more and this returns an unusable connection
+    ConnectionContext cc1 = pool.getConnection();
+    assertTrue(cc1.isActive());
+    assertFalse(cc1.isUsable());
+
+    // add a new connection into pool
+    pool.addConnection(pool.newConnection());
+    // will return the new connection
+    ConnectionContext cc2 = pool.getConnection();
+    assertTrue(cc2.isUsable());
+    cc2.getClient();
+
+    assertEquals(2, pool.getNumConnections());
+
+    checkPoolConnections(TEST_USER1, 2, 2);
+  }
+
   @Test
   public void testConnectionCreatorWithException() throws Exception {
     // Create a bad connection pool pointing to unresolvable namenode address.
     ConnectionPool badPool = new ConnectionPool(
-            conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f,
-            ClientProtocol.class);
+        conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f,
+        ClientProtocol.class, null);
     BlockingQueue<ConnectionPool> queue = new ArrayBlockingQueue<>(1);
     queue.add(badPool);
     ConnectionManager.ConnectionCreator connectionCreator =
@@ -144,7 +187,7 @@ public void testConnectionCreatorWithException() throws Exception {
     connectionCreator.setDaemon(true);
     connectionCreator.start();
     // Wait to make sure async thread is scheduled and picks
-    GenericTestUtils.waitFor(()->queue.isEmpty(), 50, 5000);
+    GenericTestUtils.waitFor(queue::isEmpty, 50, 5000);
     // At this point connection creation task should be definitely picked up.
     assertTrue(queue.isEmpty());
     // At this point connection thread should still be alive.
@@ -162,7 +205,7 @@ public void testGetConnectionWithException() throws Exception {
     // Create a bad connection pool pointing to unresolvable namenode address.
     ConnectionPool badPool = new ConnectionPool(
         conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 1, 10, 0.5f,
-        ClientProtocol.class);
+        ClientProtocol.class, null);
   }
 
   @Test
@@ -171,8 +214,8 @@ public void testGetConnection() throws Exception {
     final int totalConns = 10;
     int activeConns = 5;
 
-    ConnectionPool pool = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, ClientProtocol.class);
+    ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1,
+        0, 10, 0.5f, ClientProtocol.class, null);
     addConnectionsToPool(pool, totalConns, activeConns);
     poolMap.put(
         new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class),
@@ -196,8 +239,8 @@ public void testGetConnection() throws Exception {
 
   @Test
   public void testValidClientIndex() throws Exception {
-    ConnectionPool pool = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER1, 2, 2, 0.5f, ClientProtocol.class);
+    ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1,
+        2, 2, 0.5f, ClientProtocol.class, null);
     for(int i = -3; i <= 3; i++) {
       pool.getClientIndex().set(i);
       ConnectionContext conn = pool.getConnection();
@@ -212,8 +255,8 @@ public void getGetConnectionNamenodeProtocol() throws Exception {
     final int totalConns = 10;
     int activeConns = 5;
 
-    ConnectionPool pool = new ConnectionPool(
-        conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, NamenodeProtocol.class);
+    ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1,
+        0, 10, 0.5f, NamenodeProtocol.class, null);
     addConnectionsToPool(pool, totalConns, activeConns);
     poolMap.put(
         new ConnectionPoolId(
@@ -266,6 +309,51 @@ private void checkPoolConnections(UserGroupInformation ugi,
     }
   }
 
+  @Test
+  public void testAdvanceClientStateId() throws IOException {
+    // Start one ConnectionManager
+    Configuration tmpConf = new Configuration();
+    ConnectionManager tmpConnManager = new ConnectionManager(tmpConf);
+    tmpConnManager.start();
+    Map<ConnectionPoolId, ConnectionPool> poolMap = tmpConnManager.getPools();
+
+    // Mock one Server.Call with FederatedNamespaceState that ns0 = 1L.
+    Server.Call mockCall1 = new Server.Call(1, 1, null, null,
+        RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3});
+    Map<String, Long> nsStateId = new HashMap<>();
+    nsStateId.put("ns0", 1L);
+    RouterFederatedStateProto.Builder stateBuilder = RouterFederatedStateProto.newBuilder();
+    nsStateId.forEach(stateBuilder::putNamespaceStateIds);
+    mockCall1.setFederatedNamespaceState(stateBuilder.build().toByteString());
+
+    Server.getCurCall().set(mockCall1);
+
+    // Create one new connection pool
+    tmpConnManager.getConnection(TEST_USER1, TEST_NN_ADDRESS, NamenodeProtocol.class, "ns0");
+    assertEquals(1, poolMap.size());
+    ConnectionPoolId connectionPoolId = new ConnectionPoolId(TEST_USER1,
+        TEST_NN_ADDRESS, NamenodeProtocol.class);
+    ConnectionPool pool = poolMap.get(connectionPoolId);
+    assertEquals(1L, pool.getPoolAlignmentContext().getPoolLocalStateId());
+
+    // Mock one Server.Call with FederatedNamespaceState that ns0 = 2L.
+    Server.Call mockCall2 = new Server.Call(2, 1, null, null,
+        RPC.RpcKind.RPC_BUILTIN, new byte[] {1, 2, 3});
+    nsStateId.clear();
+    nsStateId.put("ns0", 2L);
+    stateBuilder = RouterFederatedStateProto.newBuilder();
+    nsStateId.forEach(stateBuilder::putNamespaceStateIds);
+    mockCall2.setFederatedNamespaceState(stateBuilder.build().toByteString());
+
+    Server.getCurCall().set(mockCall2);
+
+    // Get one existed connection for ns0
+    tmpConnManager.getConnection(TEST_USER1, TEST_NN_ADDRESS, NamenodeProtocol.class, "ns0");
+    assertEquals(1, poolMap.size());
+    pool = poolMap.get(connectionPoolId);
+    assertEquals(2L, pool.getPoolAlignmentContext().getPoolLocalStateId());
+  }
+
   @Test
   public void testConfigureConnectionActiveRatio() throws IOException {
     // test 1 conn below the threshold and these conns are closed
@@ -286,7 +374,7 @@ private void testConnectionCleanup(float ratio, int totalConns,
 
     // Create one new connection pool
     tmpConnManager.getConnection(TEST_USER1, TEST_NN_ADDRESS,
-        NamenodeProtocol.class);
+        NamenodeProtocol.class, "ns0");
 
     Map<ConnectionPoolId, ConnectionPool> poolMap = tmpConnManager.getPools();
     ConnectionPoolId connectionPoolId = new ConnectionPoolId(TEST_USER1,
@@ -317,6 +405,6 @@ public void testUnsupportedProtoExceptionMsg() throws Exception {
         "Unsupported protocol for connection to NameNode: "
             + TestConnectionManager.class.getName(),
         () -> ConnectionPool.newConnection(conf, TEST_NN_ADDRESS, TEST_USER1,
-            TestConnectionManager.class));
+            TestConnectionManager.class, false, 0, null));
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
index ae04150d70fa9..78f41c5d92a2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java
@@ -159,7 +159,7 @@ public void cleanup() throws IOException {
   public void testWithoutDisabling() throws IOException {
     // ns0 is slow and renewLease should take a long time
     long t0 = monotonicNow();
-    routerProtocol.renewLease("client0");
+    routerProtocol.renewLease("client0", null);
     long t = monotonicNow() - t0;
     assertTrue("It took too little: " + t + "ms",
         t > TimeUnit.SECONDS.toMillis(1));
@@ -178,7 +178,7 @@ public void testDisabling() throws Exception {
 
     // renewLease should be fast as we are skipping ns0
     long t0 = monotonicNow();
-    routerProtocol.renewLease("client0");
+    routerProtocol.renewLease("client0", null);
     long t = monotonicNow() - t0;
     assertTrue("It took too long: " + t + "ms",
         t < TimeUnit.SECONDS.toMillis(1));
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java
new file mode 100644
index 0000000000000..23e72546aac55
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java
@@ -0,0 +1,548 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertThrows;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.ClientGSIContext;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RouterFederatedStateProto;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestInfo;
+
+
+public class TestObserverWithRouter {
+  private static final String SKIP_BEFORE_EACH_CLUSTER_STARTUP = "SkipBeforeEachClusterStartup";
+  private MiniRouterDFSCluster cluster;
+  private RouterContext routerContext;
+  private FileSystem fileSystem;
+
+  @BeforeEach
+  void init(TestInfo info) throws Exception {
+    if (info.getTags().contains(SKIP_BEFORE_EACH_CLUSTER_STARTUP)) {
+      return;
+    }
+    startUpCluster(2, null);
+  }
+
+  @AfterEach
+  public void teardown() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+
+    routerContext = null;
+
+    if (fileSystem != null) {
+      fileSystem.close();
+      fileSystem = null;
+    }
+  }
+
+  public void startUpCluster(int numberOfObserver, Configuration confOverrides) throws Exception {
+    int numberOfNamenode = 2 + numberOfObserver;
+    Configuration conf = new Configuration(false);
+    conf.setBoolean(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY, true);
+    conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
+    conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "0ms");
+    if (confOverrides != null) {
+      conf.addResource(confOverrides);
+    }
+    cluster = new MiniRouterDFSCluster(true, 2, numberOfNamenode);
+    cluster.addNamenodeOverrides(conf);
+    // Start NNs and DNs and wait until ready
+    cluster.startCluster();
+
+    // Making one Namenode active per nameservice
+    if (cluster.isHighAvailability()) {
+      for (String ns : cluster.getNameservices()) {
+        cluster.switchToActive(ns, NAMENODES[0]);
+        cluster.switchToStandby(ns, NAMENODES[1]);
+        for (int i = 2; i < numberOfNamenode; i++) {
+          cluster.switchToObserver(ns, NAMENODES[i]);
+        }
+      }
+    }
+
+    Configuration routerConf = new RouterConfigBuilder()
+        .metrics()
+        .rpc()
+        .build();
+
+    cluster.addRouterOverrides(conf);
+    cluster.addRouterOverrides(routerConf);
+
+    // Start routers with only an RPC service
+    cluster.startRouters();
+
+    // Register and verify all NNs with all routers
+    cluster.registerNamenodes();
+    cluster.waitNamenodeRegistration();
+    // Setup the mount table
+    cluster.installMockLocations();
+
+    cluster.waitActiveNamespaces();
+    routerContext  = cluster.getRandomRouter();
+  }
+
+  private static Configuration getConfToEnableObserverReads() {
+    Configuration conf = new Configuration();
+    conf.setBoolean(HdfsClientConfigKeys.DFS_RBF_OBSERVER_READ_ENABLE, true);
+    return conf;
+  }
+
+  @Test
+  public void testObserverRead() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    internalTestObserverRead();
+  }
+
+  /**
+   * Tests that without adding config to use ObserverProxyProvider, the client shouldn't
+   * have reads served by Observers.
+   * Fixes regression in HDFS-13522.
+   */
+  @Test
+  public void testReadWithoutObserverClientConfigurations() throws Exception {
+    fileSystem = routerContext.getFileSystem();
+    assertThrows(AssertionError.class, this::internalTestObserverRead);
+  }
+
+  public void internalTestObserverRead()
+      throws Exception {
+    List<? extends FederationNamenodeContext> namenodes = routerContext
+        .getRouter().getNamenodeResolver()
+        .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true);
+    assertEquals("First namenode should be observer", namenodes.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    Path path = new Path("/testFile");
+    // Send create call
+    fileSystem.create(path).close();
+
+    // Send read request
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Create and complete calls should be sent to active
+    assertEquals("Two calls should be sent to active", 2, rpcCountForActive);
+
+    long rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    // getBlockLocations should be sent to observer
+    assertEquals("One call should be sent to observer", 1, rpcCountForObserver);
+  }
+
+  @Test
+  @Tag(SKIP_BEFORE_EACH_CLUSTER_STARTUP)
+  public void testObserverReadWithoutFederatedStatePropagation() throws Exception {
+    Configuration confOverrides = new Configuration(false);
+    confOverrides.setInt(RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE, 0);
+    startUpCluster(2, confOverrides);
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    List<? extends FederationNamenodeContext> namenodes = routerContext
+        .getRouter().getNamenodeResolver()
+        .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true);
+    assertEquals("First namenode should be observer", namenodes.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    Path path = new Path("/testFile");
+    // Send Create call to active
+    fileSystem.create(path).close();
+
+    // Send read request to observer. The router will msync to the active namenode.
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Create, complete and getBlockLocations calls should be sent to active
+    assertEquals("Three calls should be sent to active", 3, rpcCountForActive);
+
+    long rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    assertEquals("No call should be sent to observer", 0, rpcCountForObserver);
+  }
+
+  @Test
+  @Tag(SKIP_BEFORE_EACH_CLUSTER_STARTUP)
+  public void testDisablingObserverReadUsingNameserviceOverride() throws Exception {
+    // Disable observer reads using per-nameservice override
+    Configuration confOverrides = new Configuration(false);
+    confOverrides.set(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_OVERRIDES, "ns0");
+    startUpCluster(2, confOverrides);
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+
+    Path path = new Path("/testFile");
+    fileSystem.create(path).close();
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Create, complete and read calls should be sent to active
+    assertEquals("Three calls should be sent to active", 3, rpcCountForActive);
+
+    long rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    assertEquals("Zero calls should be sent to observer", 0, rpcCountForObserver);
+  }
+
+  @Test
+  public void testReadWhenObserverIsDown() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    Path path = new Path("/testFile1");
+    // Send Create call to active
+    fileSystem.create(path).close();
+
+    // Stop observer NN
+    int nnIndex = stopObserver(1);
+    assertNotEquals("No observer found", 3, nnIndex);
+    nnIndex = stopObserver(1);
+    assertNotEquals("No observer found", 4, nnIndex);
+
+    // Send read request
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Create, complete and getBlockLocation calls should be sent to active
+    assertEquals("Three calls should be sent to active", 3,
+        rpcCountForActive);
+
+    long rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    assertEquals("No call should send to observer", 0,
+        rpcCountForObserver);
+  }
+
+  @Test
+  public void testMultipleObserver() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    Path path = new Path("/testFile1");
+    // Send Create call to active
+    fileSystem.create(path).close();
+
+    // Stop one observer NN
+    stopObserver(1);
+
+    // Send read request
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+
+    long expectedActiveRpc = 2;
+    long expectedObserverRpc = 1;
+
+    // Create and complete calls should be sent to active
+    assertEquals("Two calls should be sent to active",
+        expectedActiveRpc, rpcCountForActive);
+
+    long rpcCountForObserver = routerContext.getRouter()
+        .getRpcServer().getRPCMetrics().getObserverProxyOps();
+    // getBlockLocation call should send to observer
+    assertEquals("Read should be success with another observer",
+        expectedObserverRpc, rpcCountForObserver);
+
+    // Stop one observer NN
+    stopObserver(1);
+
+    // Send read request
+    fileSystem.open(path).close();
+
+    rpcCountForActive = routerContext.getRouter()
+        .getRpcServer().getRPCMetrics().getActiveProxyOps();
+
+    // getBlockLocation call should be sent to active
+    expectedActiveRpc += 1;
+    assertEquals("One call should be sent to active", expectedActiveRpc,
+        rpcCountForActive);
+    expectedObserverRpc += 0;
+    rpcCountForObserver = routerContext.getRouter()
+        .getRpcServer().getRPCMetrics().getObserverProxyOps();
+    assertEquals("No call should send to observer",
+        expectedObserverRpc, rpcCountForObserver);
+  }
+
+  private int stopObserver(int num) {
+    int nnIndex;
+    for (nnIndex = 0; nnIndex < cluster.getNamenodes().size(); nnIndex++) {
+      NameNode nameNode = cluster.getCluster().getNameNode(nnIndex);
+      if (nameNode != null && nameNode.isObserverState()) {
+        cluster.getCluster().shutdownNameNode(nnIndex);
+        num--;
+        if (num == 0) {
+          break;
+        }
+      }
+    }
+    return nnIndex;
+  }
+
+  // test router observer with multiple to know which observer NN received
+  // requests
+  @Test
+  @Tag(SKIP_BEFORE_EACH_CLUSTER_STARTUP)
+  public void testMultipleObserverRouter() throws Exception {
+    StateStoreDFSCluster innerCluster;
+    MembershipNamenodeResolver resolver;
+
+    String ns0;
+    String ns1;
+    //create 4NN, One Active One Standby and Two Observers
+    innerCluster = new StateStoreDFSCluster(true, 4, 4, TimeUnit.SECONDS.toMillis(5),
+        TimeUnit.SECONDS.toMillis(5));
+    Configuration routerConf =
+        new RouterConfigBuilder().stateStore().admin().rpc()
+            .enableLocalHeartbeat(true).heartbeat().build();
+
+    StringBuilder sb = new StringBuilder();
+    ns0 = innerCluster.getNameservices().get(0);
+    MiniRouterDFSCluster.NamenodeContext context =
+        innerCluster.getNamenodes(ns0).get(1);
+    routerConf.set(DFS_NAMESERVICE_ID, ns0);
+    routerConf.set(DFS_HA_NAMENODE_ID_KEY, context.getNamenodeId());
+
+    // Specify namenodes (ns1.nn0,ns1.nn1) to monitor
+    ns1 = innerCluster.getNameservices().get(1);
+    for (MiniRouterDFSCluster.NamenodeContext ctx : innerCluster.getNamenodes(ns1)) {
+      String suffix = ctx.getConfSuffix();
+      if (sb.length() != 0) {
+        sb.append(",");
+      }
+      sb.append(suffix);
+    }
+    routerConf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString());
+    routerConf.setBoolean(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY, true);
+    routerConf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
+    routerConf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "0ms");
+
+    innerCluster.addNamenodeOverrides(routerConf);
+    innerCluster.addRouterOverrides(routerConf);
+    innerCluster.startCluster();
+
+    if (innerCluster.isHighAvailability()) {
+      for (String ns : innerCluster.getNameservices()) {
+        innerCluster.switchToActive(ns, NAMENODES[0]);
+        innerCluster.switchToStandby(ns, NAMENODES[1]);
+        for (int i = 2; i < 4; i++) {
+          innerCluster.switchToObserver(ns, NAMENODES[i]);
+        }
+      }
+    }
+    innerCluster.startRouters();
+    innerCluster.waitClusterUp();
+
+    routerContext = innerCluster.getRandomRouter();
+    resolver = (MembershipNamenodeResolver) routerContext.getRouter()
+        .getNamenodeResolver();
+
+    resolver.loadCache(true);
+    List<? extends FederationNamenodeContext> namespaceInfo0 =
+        resolver.getNamenodesForNameserviceId(ns0, true);
+    List<? extends FederationNamenodeContext> namespaceInfo1 =
+        resolver.getNamenodesForNameserviceId(ns1, true);
+    assertEquals(namespaceInfo0.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    assertEquals(namespaceInfo0.get(1).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    assertNotEquals(namespaceInfo0.get(0).getNamenodeId(),
+        namespaceInfo0.get(1).getNamenodeId());
+    assertEquals(namespaceInfo1.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+
+    innerCluster.shutdown();
+  }
+
+  @Test
+  public void testUnavailableObserverNN() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    stopObserver(2);
+
+    Path path = new Path("/testFile");
+    // Send Create call to active
+    fileSystem.create(path).close();
+
+    // Send read request.
+    fileSystem.open(path).close();
+
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+
+    // Create, complete and getBlockLocations
+    // calls should be sent to active.
+    assertEquals("Three calls should be send to active",
+        3, rpcCountForActive);
+
+
+    boolean hasUnavailable = false;
+    for(String ns : cluster.getNameservices()) {
+      List<? extends FederationNamenodeContext> nns = routerContext.getRouter()
+          .getNamenodeResolver().getNamenodesForNameserviceId(ns, false);
+      for(FederationNamenodeContext nn : nns) {
+        if(FederationNamenodeServiceState.UNAVAILABLE == nn.getState()) {
+          hasUnavailable = true;
+        }
+      }
+    }
+    // After attempting to communicate with unavailable observer namenode,
+    // its state is updated to unavailable.
+    assertTrue("There must be unavailable namenodes", hasUnavailable);
+  }
+
+  @Test
+  public void testRouterMsync() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    Path path = new Path("/testFile");
+
+    // Send Create call to active
+    fileSystem.create(path).close();
+    long rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Create and complete calls should be sent to active
+    assertEquals("Two calls should be sent to active", 2,
+        rpcCountForActive);
+
+    // Send msync
+    fileSystem.msync();
+    rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // 2 msync calls should be sent. One to each active namenode in the two namespaces.
+    assertEquals("Four calls should be sent to active", 4,
+        rpcCountForActive);
+  }
+
+  @Test
+  public void testSingleRead() throws Exception {
+    fileSystem = routerContext.getFileSystem(getConfToEnableObserverReads());
+    List<? extends FederationNamenodeContext> namenodes = routerContext
+        .getRouter().getNamenodeResolver()
+        .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true);
+    assertEquals("First namenode should be observer", namenodes.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    Path path = new Path("/");
+
+    long rpcCountForActive;
+    long rpcCountForObserver;
+
+    // Send read request
+    fileSystem.listFiles(path, false);
+    fileSystem.close();
+
+    rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // getListingCall sent to active.
+    assertEquals("Only one call should be sent to active", 1, rpcCountForActive);
+
+    rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    // getList call should be sent to observer
+    assertEquals("No calls should be sent to observer", 0, rpcCountForObserver);
+  }
+
+  @Test
+  public void testSingleReadUsingObserverReadProxyProvider() throws Exception {
+    fileSystem = routerContext.getFileSystemWithObserverReadProxyProvider();
+    List<? extends FederationNamenodeContext> namenodes = routerContext
+        .getRouter().getNamenodeResolver()
+        .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true);
+    assertEquals("First namenode should be observer", namenodes.get(0).getState(),
+        FederationNamenodeServiceState.OBSERVER);
+    Path path = new Path("/");
+
+    long rpcCountForActive;
+    long rpcCountForObserver;
+
+    // Send read request
+    fileSystem.listFiles(path, false);
+    fileSystem.close();
+
+    rpcCountForActive = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getActiveProxyOps();
+    // Two msync calls to the active namenodes.
+    assertEquals("Two calls should be sent to active", 2, rpcCountForActive);
+
+    rpcCountForObserver = routerContext.getRouter().getRpcServer()
+        .getRPCMetrics().getObserverProxyOps();
+    // getList call should be sent to observer
+    assertEquals("One call should be sent to observer", 1, rpcCountForObserver);
+  }
+
+  @Test
+  @Tag(SKIP_BEFORE_EACH_CLUSTER_STARTUP)
+  public void testClientReceiveResponseState() {
+    ClientGSIContext clientGSIContext = new ClientGSIContext();
+
+    Map<String, Long> mockMapping = new HashMap<>();
+    mockMapping.put("ns0", 10L);
+    RouterFederatedStateProto.Builder builder = RouterFederatedStateProto.newBuilder();
+    mockMapping.forEach(builder::putNamespaceStateIds);
+    RpcHeaderProtos.RpcResponseHeaderProto header = RpcHeaderProtos.RpcResponseHeaderProto
+        .newBuilder()
+        .setCallId(1)
+        .setStatus(RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto.SUCCESS)
+        .setRouterFederatedState(builder.build().toByteString())
+        .build();
+    clientGSIContext.receiveResponseState(header);
+
+    Map<String, Long> mockLowerMapping = new HashMap<>();
+    mockLowerMapping.put("ns0", 8L);
+    builder = RouterFederatedStateProto.newBuilder();
+    mockLowerMapping.forEach(builder::putNamespaceStateIds);
+    header = RpcHeaderProtos.RpcResponseHeaderProto.newBuilder()
+        .setRouterFederatedState(builder.build().toByteString())
+        .setCallId(2)
+        .setStatus(RpcHeaderProtos.RpcResponseHeaderProto.RpcStatusProto.SUCCESS)
+        .build();
+    clientGSIContext.receiveResponseState(header);
+
+    Map<String, Long> latestFederateState = ClientGSIContext.getRouterFederatedStateMap(
+        clientGSIContext.getRouterFederatedState());
+    Assertions.assertEquals(1, latestFederateState.size());
+    Assertions.assertEquals(10L, latestFederateState.get("ns0"));
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestPoolAlignmentContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestPoolAlignmentContext.java
new file mode 100644
index 0000000000000..ef6745654cf22
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestPoolAlignmentContext.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+public class TestPoolAlignmentContext {
+  @Test
+  public void testNamenodeRequestsOnlyUsePoolLocalStateID() {
+    RouterStateIdContext routerStateIdContext = new RouterStateIdContext(new Configuration());
+    String namespaceId = "namespace1";
+    routerStateIdContext.getNamespaceStateId(namespaceId).accumulate(20L);
+    PoolAlignmentContext poolContext1 = new PoolAlignmentContext(routerStateIdContext, namespaceId);
+    PoolAlignmentContext poolContext2 = new PoolAlignmentContext(routerStateIdContext, namespaceId);
+
+    assertRequestHeaderStateId(poolContext1, Long.MIN_VALUE);
+    assertRequestHeaderStateId(poolContext2, Long.MIN_VALUE);
+    Assertions.assertEquals(20L, poolContext1.getLastSeenStateId());
+    Assertions.assertEquals(20L, poolContext2.getLastSeenStateId());
+
+    poolContext1.advanceClientStateId(30L);
+    assertRequestHeaderStateId(poolContext1, 30L);
+    assertRequestHeaderStateId(poolContext2, Long.MIN_VALUE);
+    Assertions.assertEquals(20L, poolContext1.getLastSeenStateId());
+    Assertions.assertEquals(20L, poolContext2.getLastSeenStateId());
+  }
+
+  private void assertRequestHeaderStateId(PoolAlignmentContext poolAlignmentContext,
+      Long expectedValue) {
+    RpcRequestHeaderProto.Builder builder = RpcRequestHeaderProto.newBuilder();
+    poolAlignmentContext.updateRequestState(builder);
+    Assertions.assertEquals(expectedValue, builder.getStateId());
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java
new file mode 100644
index 0000000000000..76d64e4eb5583
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Testing DFSClient renewLease with same INodeId.
+ */
+public class TestRenewLeaseWithSameINodeId {
+
+  /** Federated HDFS cluster. */
+  private static MiniRouterDFSCluster cluster;
+
+  /** The first Router Context for this federated cluster. */
+  private static MiniRouterDFSCluster.RouterContext routerContext;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+    cluster = new MiniRouterDFSCluster(false, 2);
+    cluster.setNumDatanodesPerNameservice(3);
+    cluster.startCluster();
+
+    Configuration routerConf = new RouterConfigBuilder()
+        .metrics()
+        .rpc()
+        .quota()
+        .build();
+    cluster.addRouterOverrides(routerConf);
+    cluster.startRouters();
+
+    // Register and verify all NNs with all routers
+    cluster.registerNamenodes();
+    cluster.waitNamenodeRegistration();
+
+    routerContext = cluster.getRouters().get(0);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    cluster.shutdown();
+  }
+
+  /**
+   * Testing case:
+   * 1. One Router DFSClient writing multi files from different namespace with same iNodeId.
+   * 2. DFSClient Lease Renewer should work well.
+   */
+  @Test
+  public void testRenewLeaseWithSameINodeId() throws IOException {
+    // Add mount point "/ns0" and "/ns1"
+    Router router = cluster.getRouters().get(0).getRouter();
+    MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+    resolver.addLocation("/ns0", cluster.getNameservices().get(0), "/ns0");
+    resolver.addLocation("/ns1", cluster.getNameservices().get(1), "/ns1");
+
+    DistributedFileSystem fs = (DistributedFileSystem) routerContext.getFileSystem();
+
+    Path path1 = new Path("/ns0/file");
+    Path path2 = new Path("/ns1/file");
+
+    try (FSDataOutputStream ignored1 = fs.create(path1);
+         FSDataOutputStream ignored2 = fs.create(path2)) {
+      HdfsFileStatus fileStatus1 = fs.getClient().getFileInfo(path1.toUri().getPath());
+      HdfsFileStatus fileStatus2 = fs.getClient().getFileInfo(path2.toUri().getPath());
+
+      // The fileId of the files from different new namespaces should be same.
+      assertEquals(fileStatus2.getFileId(), fileStatus1.getFileId());
+
+      // The number of fileBeingWritten of this DFSClient should be two.
+      assertEquals(2, fs.getClient().getNumOfFilesBeingWritten());
+    }
+
+    // The number of fileBeingWritten of this DFSClient should be zero.
+    assertEquals(0, fs.getClient().getNumOfFilesBeingWritten());
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
index 44c0fc7ed3095..9b8fb67e68122 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouter.java
@@ -45,7 +45,7 @@
 import org.junit.Test;
 
 /**
- * The the safe mode for the {@link Router} controlled by
+ * The safe mode for the {@link Router} controlled by
  * {@link SafeModeTimer}.
  */
 public class TestRouter {
@@ -223,10 +223,14 @@ public void testRouterMetricsWhenDisabled() throws Exception {
 
   @Test
   public void testSwitchRouter() throws IOException {
-    assertRouterHeartbeater(true, true);
-    assertRouterHeartbeater(true, false);
-    assertRouterHeartbeater(false, true);
-    assertRouterHeartbeater(false, false);
+    assertRouterHeartbeater(true, true, true);
+    assertRouterHeartbeater(true, true, false);
+    assertRouterHeartbeater(true, false, true);
+    assertRouterHeartbeater(true, false, false);
+    assertRouterHeartbeater(false, true, true);
+    assertRouterHeartbeater(false, true, false);
+    assertRouterHeartbeater(false, false, true);
+    assertRouterHeartbeater(false, false, false);
   }
 
   /**
@@ -235,15 +239,19 @@ public void testSwitchRouter() throws IOException {
    * @param expectedRouterHeartbeat expect the routerHeartbeat enable state.
    * @param expectedNNHeartbeat expect the nnHeartbeat enable state.
    */
-  private void assertRouterHeartbeater(boolean expectedRouterHeartbeat,
+  private void assertRouterHeartbeater(boolean enableRpcServer, boolean expectedRouterHeartbeat,
       boolean expectedNNHeartbeat) throws IOException {
     final Router router = new Router();
-    Configuration baseCfg = new RouterConfigBuilder(conf).rpc().build();
+    Configuration baseCfg = new RouterConfigBuilder(conf).rpc(enableRpcServer).build();
     baseCfg.setBoolean(RBFConfigKeys.DFS_ROUTER_HEARTBEAT_ENABLE,
         expectedRouterHeartbeat);
     baseCfg.setBoolean(RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_ENABLE,
         expectedNNHeartbeat);
     router.init(baseCfg);
+
+    // RouterId can not be null , used by RouterHeartbeatService.updateStateStore()
+    assertNotNull(router.getRouterId());
+
     RouterHeartbeatService routerHeartbeatService =
         router.getRouterHeartbeatService();
     if (expectedRouterHeartbeat) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index b9dff65b28a09..761fad2fb7ae4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -42,16 +42,20 @@
 import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
 import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics;
 import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
 import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
 import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
 import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
 import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
 import org.apache.hadoop.hdfs.server.federation.store.impl.DisabledNameserviceStoreImpl;
 import org.apache.hadoop.hdfs.server.federation.store.impl.MountTableStoreImpl;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
 import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
+import org.apache.hadoop.hdfs.server.federation.store.records.MockStateStoreDriver;
 import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -742,7 +746,7 @@ public void testMountTablePermissions() throws Exception {
    * @param mount
    *          target mount table
    * @param canRead
-   *          whether can list mount tables under specified mount
+   *          whether you can list mount tables under specified mount
    * @param addCommandCode
    *          expected return code of add command executed for specified mount
    * @param rmCommandCode
@@ -852,6 +856,7 @@ public void testInvalidArgumentMessage() throws Exception {
         + " <quota in bytes or quota size string>]\n"
         + "\t[-clrQuota <path>]\n"
         + "\t[-clrStorageTypeQuota <path>]\n"
+        + "\t[-dumpState]\n"
         + "\t[-safemode enter | leave | get]\n"
         + "\t[-nameservice enable | disable <nameservice>]\n"
         + "\t[-getDisabledNameservices]\n"
@@ -1467,7 +1472,7 @@ public void testUpdateErrorCase() throws Exception {
         err.toString().contains("update: /noMount doesn't exist."));
     err.reset();
 
-    // Check update if non true/false value is passed for readonly.
+    // Check update if no true/false value is passed for readonly.
     argv = new String[] {"-update", src, "-readonly", "check"};
     assertEquals(-1, ToolRunner.run(admin, argv));
     assertTrue(err.toString(), err.toString().contains("update: "
@@ -1759,6 +1764,72 @@ public void testRefreshCallQueue() throws Exception {
     assertTrue(err.toString().contains("No arguments allowed"));
   }
 
+  @Test
+  public void testDumpState() throws Exception {
+    MockStateStoreDriver driver = new MockStateStoreDriver();
+    driver.clearAll();
+    // Add two records for block1
+    driver.put(MembershipState.newInstance("routerId", "ns1",
+        "ns1-ha1", "cluster1", "block1", "rpc1",
+        "service1", "lifeline1", "https", "nn01",
+        FederationNamenodeServiceState.ACTIVE, false), false, false);
+    driver.put(MembershipState.newInstance("routerId", "ns1",
+        "ns1-ha2", "cluster1", "block1", "rpc2",
+        "service2", "lifeline2", "https", "nn02",
+        FederationNamenodeServiceState.STANDBY, false), false, false);
+    Configuration conf = new Configuration();
+    conf.setClass(RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+        MockStateStoreDriver.class,
+        StateStoreDriver.class);
+    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+    try (PrintStream stream = new PrintStream(buffer)) {
+      RouterAdmin.dumpStateStore(conf, stream);
+    }
+    final String expected =
+        "---- DisabledNameservice ----\n" +
+            "\n" +
+            "---- MembershipState ----\n" +
+            "  ns1-ha1-ns1-routerId:\n" +
+            "    dateCreated: XXX\n" +
+            "    dateModified: XXX\n" +
+            "    routerId: \"routerId\"\n" +
+            "    nameserviceId: \"ns1\"\n" +
+            "    namenodeId: \"ns1-ha1\"\n" +
+            "    clusterId: \"cluster1\"\n" +
+            "    blockPoolId: \"block1\"\n" +
+            "    webAddress: \"nn01\"\n" +
+            "    rpcAddress: \"rpc1\"\n" +
+            "    serviceAddress: \"service1\"\n" +
+            "    lifelineAddress: \"lifeline1\"\n" +
+            "    state: \"ACTIVE\"\n" +
+            "    isSafeMode: false\n" +
+            "    webScheme: \"https\"\n" +
+            "    \n" +
+            "  ns1-ha2-ns1-routerId:\n" +
+            "    dateCreated: XXX\n" +
+            "    dateModified: XXX\n" +
+            "    routerId: \"routerId\"\n" +
+            "    nameserviceId: \"ns1\"\n" +
+            "    namenodeId: \"ns1-ha2\"\n" +
+            "    clusterId: \"cluster1\"\n" +
+            "    blockPoolId: \"block1\"\n" +
+            "    webAddress: \"nn02\"\n" +
+            "    rpcAddress: \"rpc2\"\n" +
+            "    serviceAddress: \"service2\"\n" +
+            "    lifelineAddress: \"lifeline2\"\n" +
+            "    state: \"STANDBY\"\n" +
+            "    isSafeMode: false\n" +
+            "    webScheme: \"https\"\n" +
+            "    \n" +
+            "\n" +
+            "---- MountTable ----\n" +
+            "\n" +
+            "---- RouterState ----";
+    // Replace the time values with XXX
+    assertEquals(expected,
+        buffer.toString().trim().replaceAll("[0-9]{4,}+", "XXX"));
+  }
+
   private void addMountTable(String src, String nsId, String dst)
       throws Exception {
     String[] argv = new String[] {"-add", src, nsId, dst};
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java
index cc7f5a61a22ff..8d776546801ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java
@@ -50,7 +50,8 @@
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.codehaus.jackson.map.ObjectMapper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
@@ -205,33 +206,29 @@ private void testOverloaded(int expOverloadMin, int expOverloadMax,
     for (int i = 0; i < numOps; i++) {
       // Stagger the operations a little (50ms)
       final int sleepTime = i * 50;
-      Future<?> future = exec.submit(new Runnable() {
-        @Override
-        public void run() {
-          DFSClient routerClient = null;
-          try {
-            Thread.sleep(sleepTime);
-            routerClient = new DFSClient(address, conf);
-            String clientName = routerClient.getClientName();
-            ClientProtocol routerProto = routerClient.getNamenode();
-            routerProto.renewLease(clientName);
-          } catch (RemoteException re) {
-            IOException ioe = re.unwrapRemoteException();
-            assertTrue("Wrong exception: " + ioe,
-                ioe instanceof StandbyException);
-            assertExceptionContains("is overloaded", ioe);
-            overloadException.incrementAndGet();
-          } catch (IOException e) {
-            fail("Unexpected exception: " + e);
-          } catch (InterruptedException e) {
-            fail("Cannot sleep: " + e);
-          } finally {
-            if (routerClient != null) {
-              try {
-                routerClient.close();
-              } catch (IOException e) {
-                LOG.error("Cannot close the client");
-              }
+      Future<?> future = exec.submit(() -> {
+        DFSClient routerClient = null;
+        try {
+          Thread.sleep(sleepTime);
+          routerClient = new DFSClient(address, conf);
+          String clientName = routerClient.getClientName();
+          ClientProtocol routerProto = routerClient.getNamenode();
+          routerProto.renewLease(clientName, null);
+        } catch (RemoteException re) {
+          IOException ioe = re.unwrapRemoteException();
+          assertTrue("Wrong exception: " + ioe, ioe instanceof StandbyException);
+          assertExceptionContains("is overloaded", ioe);
+          overloadException.incrementAndGet();
+        } catch (IOException e) {
+          fail("Unexpected exception: " + e);
+        } catch (InterruptedException e) {
+          fail("Cannot sleep: " + e);
+        } finally {
+          if (routerClient != null) {
+            try {
+              routerClient.close();
+            } catch (IOException e) {
+              LOG.error("Cannot close the client");
             }
           }
         }
@@ -389,7 +386,7 @@ public void testAsyncCallerPoolMetrics() throws Exception {
               cluster.getRouterClientConf());
           String clientName = routerClient.getClientName();
           ClientProtocol routerProto = routerClient.getNamenode();
-          routerProto.renewLease(clientName);
+          routerProto.renewLease(clientName, null);
         } catch (Exception e) {
           fail("Client request failed: " + e);
         } finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java
index bf571e2ff790c..34d50937b2bf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java
@@ -168,7 +168,7 @@ public void cleanup() throws Exception {
     }
     namenodes.clear();
 
-    routers.forEach(router ->  router.stop());
+    routers.forEach(Router::stop);
     routers.clear();
 
     if (service != null) {
@@ -582,7 +582,7 @@ public String toString() {
   }
 
   /**
-   * Asserts that the results are the expected amount and it has both success
+   * Asserts that the results are the expected amount, and it has both success
    * and failure.
    * @param msg Message to show when the assertion fails.
    * @param expected Expected number of results.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java
new file mode 100644
index 0000000000000..be8fcf682bd1a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RouterFederatedStateProto;
+import org.apache.hadoop.ipc.AlignmentContext;
+import org.apache.hadoop.ipc.ClientId;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcConstants;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos;
+import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.util.ProtoUtil;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+
+public class TestRouterFederatedState {
+
+  @Test
+  public void testRpcRouterFederatedState() throws InvalidProtocolBufferException {
+    byte[] uuid = ClientId.getClientId();
+    Map<String, Long> expectedStateIds = new HashMap<String, Long>() {
+      {
+        put("namespace1", 11L);
+        put("namespace2", 22L);
+      }
+    };
+
+    AlignmentContext alignmentContext = new AlignmentContextWithRouterState(expectedStateIds);
+
+    RpcHeaderProtos.RpcRequestHeaderProto header = ProtoUtil.makeRpcRequestHeader(
+        RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+        RpcHeaderProtos.RpcRequestHeaderProto.OperationProto.RPC_FINAL_PACKET,
+        0,
+        RpcConstants.INVALID_RETRY_COUNT,
+        uuid,
+        alignmentContext);
+
+    Map<String, Long> stateIdsFromHeader =
+        RouterFederatedStateProto.parseFrom(
+            header.getRouterFederatedState().toByteArray()
+        ).getNamespaceStateIdsMap();
+
+    assertEquals(expectedStateIds, stateIdsFromHeader);
+  }
+
+  private static class AlignmentContextWithRouterState implements AlignmentContext {
+
+    private Map<String, Long> routerFederatedState;
+
+    AlignmentContextWithRouterState(Map<String, Long> namespaceStates) {
+      this.routerFederatedState = namespaceStates;
+    }
+
+    @Override
+    public void updateRequestState(RpcHeaderProtos.RpcRequestHeaderProto.Builder header) {
+      RouterFederatedStateProto fedState = RouterFederatedStateProto
+          .newBuilder()
+          .putAllNamespaceStateIds(routerFederatedState)
+          .build();
+
+      header.setRouterFederatedState(fedState.toByteString());
+    }
+
+    @Override
+    public void updateResponseState(RpcHeaderProtos.RpcResponseHeaderProto.Builder header) {}
+
+    @Override
+    public void receiveResponseState(RpcHeaderProtos.RpcResponseHeaderProto header) {}
+
+    @Override
+    public long receiveRequestState(RpcHeaderProtos.RpcRequestHeaderProto header, long threshold) {
+      return 0;
+    }
+
+    @Override
+    public long getLastSeenStateId() {
+      return 0;
+    }
+
+    @Override
+    public boolean isCoordinatedCall(String protocolName, String method) {
+      return false;
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableCacheRefresh.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableCacheRefresh.java
index c90e614a5cd40..82bc7d905e606 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableCacheRefresh.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableCacheRefresh.java
@@ -308,9 +308,9 @@ public void run() {
         TimeUnit.SECONDS);
     mountTableRefresherService.init(config);
     // One router is not responding for 1 minute, still refresh should
-    // finished in 5 second as cache update timeout is set 5 second.
+    // finish in 5 second as cache update timeout is set 5 second.
     mountTableRefresherService.refresh();
-    // Test case timeout is assert for this test case.
+    // Test case timeout is asserted for this test case.
   }
 
   /**
@@ -349,7 +349,7 @@ protected RouterClient createRouterClient(
     mountTableRefresherService.refresh();
     assertNotEquals("No RouterClient is created.", 0, createCounter.get());
     /*
-     * Wait for clients to expire. Lets wait triple the cache eviction period.
+     * Wait for clients to expire. Let's wait triple the cache eviction period.
      * After cache eviction period all created client must be removed and
      * closed.
      */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java
new file mode 100644
index 0000000000000..57d4c69db698f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.resolver.RouterResolveException;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test a router end-to-end including the MountTable without default nameservice.
+ */
+public class TestRouterMountTableWithoutDefaultNS {
+  private static StateStoreDFSCluster cluster;
+  private static RouterContext routerContext;
+  private static MountTableResolver mountTable;
+  private static ClientProtocol routerProtocol;
+  private static FileSystem nnFs0;
+  private static FileSystem nnFs1;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+    // Build and start a federated cluster
+    cluster = new StateStoreDFSCluster(false, 2);
+    Configuration conf = new RouterConfigBuilder()
+        .stateStore()
+        .admin()
+        .rpc()
+        .build();
+    conf.setInt(RBFConfigKeys.DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_KEY, 20);
+    conf.setBoolean(RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE, false);
+    cluster.addRouterOverrides(conf);
+    cluster.startCluster();
+    cluster.startRouters();
+    cluster.waitClusterUp();
+
+    // Get the end points
+    nnFs0 = cluster.getNamenode("ns0", null).getFileSystem();
+    nnFs1 = cluster.getNamenode("ns1", null).getFileSystem();
+    routerContext = cluster.getRandomRouter();
+    Router router = routerContext.getRouter();
+    routerProtocol = routerContext.getClient().getNamenode();
+    mountTable = (MountTableResolver) router.getSubclusterResolver();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (cluster != null) {
+      cluster.stopRouter(routerContext);
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @After
+  public void clearMountTable() throws IOException {
+    RouterClient client = routerContext.getAdminClient();
+    MountTableManager mountTableManager = client.getMountTableManager();
+    GetMountTableEntriesRequest req1 = GetMountTableEntriesRequest.newInstance("/");
+    GetMountTableEntriesResponse response = mountTableManager.getMountTableEntries(req1);
+    for (MountTable entry : response.getEntries()) {
+      RemoveMountTableEntryRequest req2 =
+          RemoveMountTableEntryRequest.newInstance(entry.getSourcePath());
+      mountTableManager.removeMountTableEntry(req2);
+    }
+  }
+
+  /**
+   * Add a mount table entry to the mount table through the admin API.
+   * @param entry Mount table entry to add.
+   * @return If it was succesfully added.
+   * @throws IOException Problems adding entries.
+   */
+  private boolean addMountTable(final MountTable entry) throws IOException {
+    RouterClient client = routerContext.getAdminClient();
+    MountTableManager mountTableManager = client.getMountTableManager();
+    AddMountTableEntryRequest addRequest = AddMountTableEntryRequest.newInstance(entry);
+    AddMountTableEntryResponse addResponse = mountTableManager.addMountTableEntry(addRequest);
+
+    // Reload the Router cache
+    mountTable.loadCache(true);
+
+    return addResponse.getStatus();
+  }
+
+  /**
+   * Verify that RBF that disable default nameservice should support
+   * get information about ancestor mount points.
+   */
+  @Test
+  public void testGetFileInfoWithSubMountPoint() throws IOException {
+    MountTable addEntry = MountTable.newInstance("/testdir/1",
+        Collections.singletonMap("ns0", "/testdir/1"));
+    assertTrue(addMountTable(addEntry));
+    HdfsFileStatus finfo = routerProtocol.getFileInfo("/testdir");
+    assertNotNull(finfo);
+    assertEquals("supergroup", finfo.getGroup());
+    assertTrue(finfo.isDirectory());
+  }
+
+  /**
+   * Verify that RBF doesn't support get the file information
+   * with no location and sub mount points.
+   */
+  @Test
+  public void testGetFileInfoWithoutSubMountPoint() throws Exception {
+    MountTable addEntry = MountTable.newInstance("/testdir/1",
+        Collections.singletonMap("ns0", "/testdir/1"));
+    assertTrue(addMountTable(addEntry));
+    LambdaTestUtils.intercept(RouterResolveException.class,
+        () -> routerContext.getRouter().getRpcServer().getFileInfo("/testdir2"));
+  }
+
+  /**
+   * Verify that RBF that disable default nameservice should support
+   * get information about ancestor mount points.
+   */
+  @Test
+  public void testGetContentSummaryWithSubMountPoint() throws IOException {
+    MountTable addEntry = MountTable.newInstance("/testdir/1/2",
+        Collections.singletonMap("ns0", "/testdir/1/2"));
+    assertTrue(addMountTable(addEntry));
+
+    try {
+      writeData(nnFs0, new Path("/testdir/1/2/3"), 10 * 1024 * 1024);
+
+      RouterRpcServer routerRpcServer = routerContext.getRouterRpcServer();
+      ContentSummary summaryFromRBF = routerRpcServer.getContentSummary("/testdir");
+      assertNotNull(summaryFromRBF);
+      assertEquals(1, summaryFromRBF.getFileCount());
+      assertEquals(10 * 1024 * 1024, summaryFromRBF.getLength());
+    } finally {
+      nnFs0.delete(new Path("/testdir"), true);
+    }
+  }
+
+  @Test
+  public void testGetAllLocations() throws IOException {
+    // Add mount table entry.
+    MountTable addEntry = MountTable.newInstance("/testA",
+        Collections.singletonMap("ns0", "/testA"));
+    assertTrue(addMountTable(addEntry));
+    addEntry = MountTable.newInstance("/testA/testB",
+        Collections.singletonMap("ns1", "/testA/testB"));
+    assertTrue(addMountTable(addEntry));
+    addEntry = MountTable.newInstance("/testA/testB/testC",
+        Collections.singletonMap("ns2", "/testA/testB/testC"));
+    assertTrue(addMountTable(addEntry));
+
+    RouterClientProtocol protocol = routerContext.getRouterRpcServer().getClientProtocolModule();
+    Map<String, List<RemoteLocation>> locations = protocol.getAllLocations("/testA");
+    assertEquals(3, locations.size());
+  }
+
+  @Test
+  public void testGetLocationsForContentSummary() throws Exception {
+    // Add mount table entry.
+    MountTable addEntry = MountTable.newInstance("/testA/testB",
+        Collections.singletonMap("ns0", "/testA/testB"));
+    assertTrue(addMountTable(addEntry));
+    addEntry = MountTable.newInstance("/testA/testB/testC",
+        Collections.singletonMap("ns1", "/testA/testB/testC"));
+    assertTrue(addMountTable(addEntry));
+
+    RouterClientProtocol protocol = routerContext.getRouterRpcServer().getClientProtocolModule();
+    List<RemoteLocation> locations = protocol.getLocationsForContentSummary("/testA");
+    assertEquals(2, locations.size());
+
+    for (RemoteLocation location : locations) {
+      String nsId = location.getNameserviceId();
+      if ("ns0".equals(nsId)) {
+        assertEquals("/testA/testB", location.getDest());
+      } else if ("ns1".equals(nsId)) {
+        assertEquals("/testA/testB/testC", location.getDest());
+      } else {
+        fail("Unexpected NS " + nsId);
+      }
+    }
+
+    LambdaTestUtils.intercept(NoLocationException.class,
+        () -> protocol.getLocationsForContentSummary("/testB"));
+  }
+
+  @Test
+  public void testGetContentSummary() throws Exception {
+    try {
+      // Add mount table entry.
+      MountTable addEntry = MountTable.newInstance("/testA",
+          Collections.singletonMap("ns0", "/testA"));
+      assertTrue(addMountTable(addEntry));
+      addEntry = MountTable.newInstance("/testA/testB",
+          Collections.singletonMap("ns0", "/testA/testB"));
+      assertTrue(addMountTable(addEntry));
+      addEntry = MountTable.newInstance("/testA/testB/testC",
+          Collections.singletonMap("ns1", "/testA/testB/testC"));
+      assertTrue(addMountTable(addEntry));
+
+      writeData(nnFs0, new Path("/testA/testB/file1"), 1024 * 1024);
+      writeData(nnFs1, new Path("/testA/testB/testC/file2"), 1024 * 1024);
+      writeData(nnFs1, new Path("/testA/testB/testC/file3"), 1024 * 1024);
+
+      RouterRpcServer routerRpcServer = routerContext.getRouterRpcServer();
+      ContentSummary summary = routerRpcServer.getContentSummary("/testA");
+      assertEquals(3, summary.getFileCount());
+      assertEquals(1024 * 1024 * 3, summary.getLength());
+
+      LambdaTestUtils.intercept(NoLocationException.class,
+          () -> routerRpcServer.getContentSummary("/testB"));
+    } finally {
+      nnFs0.delete(new Path("/testA"), true);
+      nnFs1.delete(new Path("/testA"), true);
+    }
+  }
+
+  void writeData(FileSystem fs, Path path, int fileLength) throws IOException {
+    try (FSDataOutputStream outputStream = fs.create(path)) {
+      for (int writeSize = 0; writeSize < fileLength; writeSize++) {
+        outputStream.write(writeSize);
+      }
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
index 9bf149c6490d7..04b4b58bcb6e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
@@ -167,7 +167,7 @@ public void testHearbeat() throws InterruptedException, IOException {
     // Verify the locator has matching NN entries for each NS
     for (String ns : cluster.getNameservices()) {
       List<? extends FederationNamenodeContext> nns =
-          namenodeResolver.getNamenodesForNameserviceId(ns);
+          namenodeResolver.getNamenodesForNameserviceId(ns, false);
 
       // Active
       FederationNamenodeContext active = nns.get(0);
@@ -191,7 +191,7 @@ public void testHearbeat() throws InterruptedException, IOException {
 
     // Verify the locator has recorded the failover for the failover NS
     List<? extends FederationNamenodeContext> failoverNSs =
-        namenodeResolver.getNamenodesForNameserviceId(failoverNS);
+        namenodeResolver.getNamenodesForNameserviceId(failoverNS, false);
     // Active
     FederationNamenodeContext active = failoverNSs.get(0);
     assertEquals(NAMENODES[1], active.getNamenodeId());
@@ -202,7 +202,7 @@ public void testHearbeat() throws InterruptedException, IOException {
 
     // Verify the locator has the same records for the other ns
     List<? extends FederationNamenodeContext> normalNss =
-        namenodeResolver.getNamenodesForNameserviceId(normalNs);
+        namenodeResolver.getNamenodesForNameserviceId(normalNs, false);
     // Active
     active = normalNss.get(0);
     assertEquals(NAMENODES[0], active.getNamenodeId());
@@ -211,6 +211,50 @@ public void testHearbeat() throws InterruptedException, IOException {
     assertEquals(NAMENODES[1], standby.getNamenodeId());
   }
 
+  @Test
+  public void testNamenodeHeartbeatServiceHAServiceProtocolProxy(){
+    testNamenodeHeartbeatServiceHAServiceProtocol(
+        "test-ns", "nn", 1000, -1, -1, 1003,
+        "host01.test:1000", "host02.test:1000");
+    testNamenodeHeartbeatServiceHAServiceProtocol(
+        "test-ns", "nn", 1000, 1001, -1, 1003,
+        "host01.test:1001", "host02.test:1001");
+    testNamenodeHeartbeatServiceHAServiceProtocol(
+        "test-ns", "nn", 1000, -1, 1002, 1003,
+        "host01.test:1002", "host02.test:1002");
+    testNamenodeHeartbeatServiceHAServiceProtocol(
+        "test-ns", "nn", 1000, 1001, 1002, 1003,
+        "host01.test:1002", "host02.test:1002");
+  }
+
+  private void testNamenodeHeartbeatServiceHAServiceProtocol(
+      String nsId, String nnId,
+      int rpcPort, int servicePort,
+      int lifelinePort, int webAddressPort,
+      String expected0, String expected1) {
+    Configuration conf = generateNamenodeConfiguration(nsId, nnId,
+        rpcPort, servicePort, lifelinePort, webAddressPort);
+
+    Router testRouter = new Router();
+    testRouter.setConf(conf);
+
+    Collection<NamenodeHeartbeatService> heartbeatServices =
+        testRouter.createNamenodeHeartbeatServices();
+
+    assertEquals(2, heartbeatServices.size());
+
+    Iterator<NamenodeHeartbeatService> iterator = heartbeatServices.iterator();
+    NamenodeHeartbeatService service0 = iterator.next();
+    service0.init(conf);
+    assertNotNull(service0.getLocalTarget());
+    assertEquals(expected0, service0.getLocalTarget().getHealthMonitorAddress().toString());
+
+    NamenodeHeartbeatService service1 = iterator.next();
+    service1.init(conf);
+    assertNotNull(service1.getLocalTarget());
+    assertEquals(expected1, service1.getLocalTarget().getHealthMonitorAddress().toString());
+  }
+
   @Test
   public void testNamenodeHeartbeatServiceNNResolution() {
     String nsId = "test-ns";
@@ -261,10 +305,14 @@ private Configuration generateNamenodeConfiguration(
 
     conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY + "." + suffix,
         MockDomainNameResolver.DOMAIN + ":" + rpcPort);
-    conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
-        MockDomainNameResolver.DOMAIN + ":" + servicePort);
-    conf.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY + "." + suffix,
-        MockDomainNameResolver.DOMAIN + ":" + lifelinePort);
+    if (servicePort >= 0){
+      conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + "." + suffix,
+          MockDomainNameResolver.DOMAIN + ":" + servicePort);
+    }
+    if (lifelinePort >= 0){
+      conf.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY + "." + suffix,
+          MockDomainNameResolver.DOMAIN + ":" + lifelinePort);
+    }
     conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY + "." + suffix,
         MockDomainNameResolver.DOMAIN + ":" + webAddressPort);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
index 4fae86b01d399..bae2dea3ceabf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java
@@ -204,7 +204,7 @@ public void testNamenodeMonitoring() throws Exception {
     final List<FederationNamenodeContext> namespaceInfo = new ArrayList<>();
     for (String nsId : nns.keySet()) {
       List<? extends FederationNamenodeContext> nnReports =
-          resolver.getNamenodesForNameserviceId(nsId);
+          resolver.getNamenodesForNameserviceId(nsId, false);
       namespaceInfo.addAll(nnReports);
     }
     for (FederationNamenodeContext nnInfo : namespaceInfo) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java
index ab507aaf9ecd4..f23b02092a299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java
@@ -194,7 +194,7 @@ private void testWebScheme(HttpConfig.Policy httpPolicy,
     final List<FederationNamenodeContext> namespaceInfo = new ArrayList<>();
     for (String nsId : nns.keySet()) {
       List<? extends FederationNamenodeContext> nnReports =
-          resolver.getNamenodesForNameserviceId(nsId);
+          resolver.getNamenodesForNameserviceId(nsId, false);
       namespaceInfo.addAll(nnReports);
     }
     for (FederationNamenodeContext nnInfo : namespaceInfo) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
index b69004198eb48..aa3d547056134 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
@@ -74,8 +74,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.function.Supplier;
-
 /**
  * Tests quota behaviors in Router-based Federation.
  */
@@ -210,21 +208,17 @@ public void testStorageSpaceQuotaExceed() throws Exception {
     routerClient.create("/ssquota/file", true).close();
     routerClient.create("/ssquota/subdir/file", true).close();
 
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-
-      @Override
-      public Boolean get() {
-        boolean isDsQuotaViolated = false;
-        try {
-          // append data to trigger NSQuotaExceededException
-          appendData("/ssquota/file", routerClient, BLOCK_SIZE);
-          appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
-        } catch (DSQuotaExceededException e) {
-          isDsQuotaViolated = true;
-        } catch (IOException ignored) {
-        }
-        return isDsQuotaViolated;
+    GenericTestUtils.waitFor(() -> {
+      boolean isDsQuotaViolated = false;
+      try {
+        // append data to trigger NSQuotaExceededException
+        appendData("/ssquota/file", routerClient, BLOCK_SIZE);
+        appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE);
+      } catch (DSQuotaExceededException e) {
+        isDsQuotaViolated = true;
+      } catch (IOException ignored) {
       }
+      return isDsQuotaViolated;
     }, 5000, 60000);
 
     // append data to destination path in real FileSystem should be okay
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
index 4a1dd2e091bc7..d24ef8ebc3c4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
@@ -75,11 +75,11 @@ public void testGetChildrenPaths() {
   public void testGetQuotaUsage() {
     RouterQuotaUsage quotaGet;
 
-    // test case1: get quota with an non-exist path
+    // test case1: get quota with a non-exist path
     quotaGet = manager.getQuotaUsage("/non-exist-path");
     assertNull(quotaGet);
 
-    // test case2: get quota from an no-quota set path
+    // test case2: get quota from a no-quota set path
     RouterQuotaUsage.Builder quota = new RouterQuotaUsage.Builder()
         .quota(HdfsConstants.QUOTA_RESET)
         .spaceQuota(HdfsConstants.QUOTA_RESET);
@@ -88,7 +88,7 @@ public void testGetQuotaUsage() {
     // it should return null
     assertNull(quotaGet);
 
-    // test case3: get quota from an quota-set path
+    // test case3: get quota from a quota-set path
     quota.quota(1);
     quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet", quota.build());
@@ -96,24 +96,24 @@ public void testGetQuotaUsage() {
     assertEquals(1, quotaGet.getQuota());
     assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
-    // test case4: get quota with an non-exist child path
+    // test case4: get quota with a non-exist child path
     quotaGet = manager.getQuotaUsage("/hasQuotaSet/file");
     // it will return the nearest ancestor which quota was set
     assertEquals(1, quotaGet.getQuota());
     assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
-    // test case5: get quota with an child path which its parent
+    // test case5: get quota with a child path which its parent
     // wasn't quota set
     quota.quota(HdfsConstants.QUOTA_RESET);
     quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet/noQuotaSet", quota.build());
-    // here should returns the quota of path /hasQuotaSet
+    // here should return the quota of path /hasQuotaSet
     // (the nearest ancestor which quota was set)
     quotaGet = manager.getQuotaUsage("/hasQuotaSet/noQuotaSet/file");
     assertEquals(1, quotaGet.getQuota());
     assertEquals(HdfsConstants.QUOTA_RESET, quotaGet.getSpaceQuota());
 
-    // test case6: get quota with an child path which its parent was quota set
+    // test case6: get quota with a child path which its parent was quota set
     quota.quota(2);
     quota.spaceQuota(HdfsConstants.QUOTA_RESET);
     manager.put("/hasQuotaSet/hasQuotaSet", quota.build());
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
index a16f9d10c85fc..1054e5ac8cf97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
@@ -54,8 +54,6 @@
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
-import java.util.function.Supplier;
-
 /**
  * Test retry behavior of the Router RPC Client.
  */
@@ -68,7 +66,7 @@ public class TestRouterRPCClientRetries {
   private static ClientProtocol routerProtocol;
 
   @Rule
-  public final Timeout testTimeout = new Timeout(100000);
+  public final Timeout testTimeout = new Timeout(100000L, TimeUnit.MILLISECONDS);
 
   @Before
   public void setUp() throws Exception {
@@ -153,7 +151,7 @@ public void testRetryWhenOneNameServiceDown() throws Exception {
 
     DFSClient client = nnContext1.getClient();
     // Renew lease for the DFS client, it will succeed.
-    routerProtocol.renewLease(client.getClientName());
+    routerProtocol.renewLease(client.getClientName(), null);
 
     // Verify the retry times, it will retry one time for ns0.
     FederationRPCMetrics rpcMetrics = routerContext.getRouter()
@@ -168,7 +166,7 @@ public void testRetryWhenOneNameServiceDown() throws Exception {
   private void registerInvalidNameReport() throws IOException {
     String ns0 = cluster.getNameservices().get(0);
     List<? extends FederationNamenodeContext> origin = resolver
-        .getNamenodesForNameserviceId(ns0);
+        .getNamenodesForNameserviceId(ns0, false);
     FederationNamenodeContext nnInfo = origin.get(0);
     NamenodeStatusReport report = new NamenodeStatusReport(ns0,
         nnInfo.getNamenodeId(), nnInfo.getRpcAddress(),
@@ -237,11 +235,6 @@ private static int getNumDatanodes(final String jsonString)
   private static void waitUpdateLiveNodes(
       final String oldValue, final NamenodeBeanMetrics metrics)
           throws Exception {
-    waitFor(new Supplier<Boolean>() {
-      @Override
-      public Boolean get() {
-        return !oldValue.equals(metrics.getLiveNodes());
-      }
-    }, 500, 5 * 1000);
+    waitFor(() -> !oldValue.equals(metrics.getLiveNodes()), 500, 5 * 1000);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
index 2887c08f15a11..b05337443f659 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
@@ -20,6 +20,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -39,6 +40,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.AclEntry;
@@ -74,13 +76,14 @@
  * Tests router rpc with multiple destination mount table resolver.
  */
 public class TestRouterRPCMultipleDestinationMountTableResolver {
-  private static final List<String> NS_IDS = Arrays.asList("ns0", "ns1");
+  private static final List<String> NS_IDS = Arrays.asList("ns0", "ns1", "ns2");
 
   private static StateStoreDFSCluster cluster;
   private static RouterContext routerContext;
   private static MountTableResolver resolver;
   private static DistributedFileSystem nnFs0;
   private static DistributedFileSystem nnFs1;
+  private static DistributedFileSystem nnFs2;
   private static DistributedFileSystem routerFs;
   private static RouterRpcServer rpcServer;
 
@@ -88,7 +91,7 @@ public class TestRouterRPCMultipleDestinationMountTableResolver {
   public static void setUp() throws Exception {
 
     // Build and start a federated cluster
-    cluster = new StateStoreDFSCluster(false, 2,
+    cluster = new StateStoreDFSCluster(false, 3,
         MultipleDestinationMountTableResolver.class);
     Configuration routerConf =
         new RouterConfigBuilder().stateStore().admin().quota().rpc().build();
@@ -109,6 +112,8 @@ public static void setUp() throws Exception {
         .getNamenode(cluster.getNameservices().get(0), null).getFileSystem();
     nnFs1 = (DistributedFileSystem) cluster
         .getNamenode(cluster.getNameservices().get(1), null).getFileSystem();
+    nnFs2 = (DistributedFileSystem) cluster
+        .getNamenode(cluster.getNameservices().get(2), null).getFileSystem();
     routerFs = (DistributedFileSystem) routerContext.getFileSystem();
     rpcServer =routerContext.getRouter().getRpcServer();
   }
@@ -506,7 +511,7 @@ public void testIsMultiDestDir() throws Exception {
    */
   @Test
   public void testSnapshotPathResolution() throws Exception {
-    // Create a mount entry with non isPathAll order, so as to call
+    // Create a mount entry with non isPathAll order, to call
     // invokeSequential.
     Map<String, String> destMap = new HashMap<>();
     destMap.put("ns0", "/tmp_ns0");
@@ -643,6 +648,42 @@ public void testContentSummaryMultipleDestWithMaxValue()
     assertEquals(ssQuota, cs.getSpaceQuota());
   }
 
+  /**
+   * Test RouterRpcServer#invokeAtAvailableNs on mount point with multiple destinations
+   * and making a one of the destination's subcluster unavailable.
+   */
+  @Test
+  public void testInvokeAtAvailableNs() throws IOException {
+    // Create a mount point with multiple destinations.
+    Path path = new Path("/testInvokeAtAvailableNs");
+    Map<String, String> destMap = new HashMap<>();
+    destMap.put("ns0", "/testInvokeAtAvailableNs");
+    destMap.put("ns1", "/testInvokeAtAvailableNs");
+    nnFs0.mkdirs(path);
+    nnFs1.mkdirs(path);
+    MountTable addEntry =
+        MountTable.newInstance("/testInvokeAtAvailableNs", destMap);
+    addEntry.setQuota(new RouterQuotaUsage.Builder().build());
+    addEntry.setDestOrder(DestinationOrder.RANDOM);
+    addEntry.setFaultTolerant(true);
+    assertTrue(addMountTable(addEntry));
+
+    // Make one subcluster unavailable.
+    MiniDFSCluster dfsCluster = cluster.getCluster();
+    dfsCluster.shutdownNameNode(0);
+    dfsCluster.shutdownNameNode(1);
+    try {
+      // Verify that #invokeAtAvailableNs works by calling #getServerDefaults.
+      RemoteMethod method = new RemoteMethod("getServerDefaults");
+      FsServerDefaults serverDefaults =
+          rpcServer.invokeAtAvailableNs(method, FsServerDefaults.class);
+      assertNotNull(serverDefaults);
+    } finally {
+      dfsCluster.restartNameNode(0);
+      dfsCluster.restartNameNode(1);
+    }
+  }
+
   /**
    * Test write on mount point with multiple destinations
    * and making a one of the destination's subcluster unavailable.
@@ -857,6 +898,9 @@ private static FileSystem getFileSystem(final String nsId) {
     if (nsId.equals("ns1")) {
       return nnFs1;
     }
+    if (nsId.equals("ns2")) {
+      return nnFs2;
+    }
     return null;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java
new file mode 100644
index 0000000000000..e498874812a9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.io.retry.RetryInvocationHandler;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class TestRouterRetryCache {
+  /** Federated HDFS cluster. */
+  private MiniRouterDFSCluster cluster;
+
+  @Before
+  public  void setup() throws Exception {
+    UserGroupInformation routerUser = UserGroupInformation.getLoginUser();
+    Configuration conf = new Configuration();
+    String adminUser = routerUser.getUserName();
+    conf.set("hadoop.proxyuser." + adminUser + ".hosts", "*");
+    conf.set("hadoop.proxyuser." + adminUser + ".groups", "*");
+    conf.set("hadoop.proxyuser.fake_joe.hosts", "*");
+    conf.set("hadoop.proxyuser.fake_joe.groups", "*");
+    conf.set(DFS_NAMENODE_IP_PROXY_USERS, routerUser.getShortUserName());
+    cluster = new MiniRouterDFSCluster(true, 1, conf);
+    cluster.addNamenodeOverrides(conf);
+
+    // Start NNs and DNs and wait until ready
+    cluster.startCluster();
+
+    // Start routers with only an RPC service
+    cluster.startRouters();
+
+    // Register and verify all NNs with all routers
+    cluster.registerNamenodes();
+    cluster.waitNamenodeRegistration();
+
+    // Setup the mount table
+    cluster.installMockLocations();
+
+    // Making one Namenodes active per nameservice
+    if (cluster.isHighAvailability()) {
+      for (String ns : cluster.getNameservices()) {
+        cluster.switchToActive(ns, NAMENODES[0]);
+        cluster.switchToStandby(ns, NAMENODES[1]);
+      }
+    }
+    cluster.waitActiveNamespaces();
+  }
+
+  @After
+  public void teardown() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Test
+  public void testRetryCacheWithOneLevelProxyUser() throws Exception {
+    internalTestRetryCache(false);
+  }
+
+  @Test
+  public void testRetryCacheWithTwoLevelProxyUser() throws Exception {
+    internalTestRetryCache(true);
+  }
+
+  /**
+   * Test RetryCache through RBF with proxyUser and non-ProxyUser respectively.
+   *
+   * 1. Start cluster with current user.
+   * 2. Create one test directory by the admin user.
+   * 3. Create one Router FileSystem with one mocked user, one proxyUser or non-ProxyUser.
+   * 4. Try to create one test directory by the router fileSystem.
+   * 5. Try to rename the new test directory to one test destination directory
+   * 6. Then failover the active to the standby
+   * 7. Try to rename the source directory to the destination directory again with the same callId
+   * 8. Try to
+   */
+  private void internalTestRetryCache(boolean twoLevelProxyUGI) throws Exception {
+    RetryInvocationHandler.SET_CALL_ID_FOR_TEST.set(false);
+    FileSystem routerFS = cluster.getRandomRouter().getFileSystem();
+    Path testDir = new Path("/target-ns0/testdir");
+    routerFS.mkdirs(testDir);
+    routerFS.setPermission(testDir, FsPermission.getDefault());
+
+    // Run as fake joe to authorize the test
+    UserGroupInformation joe = UserGroupInformation.createUserForTesting("fake_joe",
+        new String[] {"fake_group"});
+    if (twoLevelProxyUGI) {
+      joe = UserGroupInformation.createProxyUser("fake_proxy_joe", joe);
+    }
+    FileSystem joeFS = joe.doAs((PrivilegedExceptionAction<FileSystem>) () ->
+        FileSystem.newInstance(routerFS.getUri(), routerFS.getConf()));
+
+    Path renameSrc = new Path(testDir, "renameSrc");
+    Path renameDst = new Path(testDir, "renameDst");
+    joeFS.mkdirs(renameSrc);
+
+    assertEquals(HAServiceProtocol.HAServiceState.ACTIVE,
+        cluster.getCluster().getNamesystem(0).getState());
+
+    int callId = Client.nextCallId();
+    Client.setCallIdAndRetryCount(callId, 0, null);
+    assertTrue(joeFS.rename(renameSrc, renameDst));
+
+    Client.setCallIdAndRetryCount(callId, 0, null);
+    assertTrue(joeFS.rename(renameSrc, renameDst));
+
+    String ns0 = cluster.getNameservices().get(0);
+    cluster.switchToStandby(ns0, NAMENODES[0]);
+    cluster.switchToActive(ns0, NAMENODES[1]);
+
+    assertEquals(HAServiceProtocol.HAServiceState.ACTIVE,
+        cluster.getCluster().getNamesystem(1).getState());
+
+    Client.setCallIdAndRetryCount(callId, 0, null);
+    assertTrue(joeFS.rename(renameSrc, renameDst));
+
+    FileStatus fileStatus = joeFS.getFileStatus(renameDst);
+    if (twoLevelProxyUGI) {
+      assertEquals("fake_proxy_joe", fileStatus.getOwner());
+    } else {
+      assertEquals("fake_joe", fileStatus.getOwner());
+    }
+
+    joeFS.delete(renameDst, true);
+  }
+
+  @Test
+  public void testParseSpecialValue() {
+    String mockContent = "mockContent,clientIp:127.0.0.1," +
+        "clientCallId:12345,clientId:mockClientId";
+    String clientIp = NameNode.parseSpecialValue(mockContent, "clientIp:");
+    assertEquals("127.0.0.1", clientIp);
+
+    String clientCallId = NameNode.parseSpecialValue(
+        mockContent, "clientCallId:");
+    assertEquals("12345", clientCallId);
+
+    String clientId = NameNode.parseSpecialValue(mockContent, "clientId:");
+    assertEquals("mockClientId", clientId);
+
+    String clientRetryNum = NameNode.parseSpecialValue(
+        mockContent, "clientRetryNum:");
+    assertNull(clientRetryNum);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 21329c8142331..35b640d40678a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -56,7 +56,9 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -106,6 +108,7 @@
 import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
 import org.apache.hadoop.hdfs.server.federation.MockResolver;
 import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics;
 import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
@@ -128,6 +131,7 @@
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -704,6 +708,7 @@ public void testProxyGetDatanodeReport() throws Exception {
 
     DatanodeInfo[] combinedData =
         routerProtocol.getDatanodeReport(DatanodeReportType.ALL);
+    assertEquals(0, routerProtocol.getSlowDatanodeReport().length);
     final Map<Integer, String> routerDNMap = new TreeMap<>();
     for (DatanodeInfo dn : combinedData) {
       String subcluster = dn.getNetworkLocation().split("/")[1];
@@ -1448,6 +1453,119 @@ public void testProxyRestoreFailedStorage() throws Exception {
     assertEquals(nnSuccess, routerSuccess);
   }
 
+  private void testRenewLeaseInternal(DistributedFileSystem dfs,
+      FederationRPCMetrics rpcMetrics, Path testPath, boolean createFlag)
+      throws Exception {
+    FSDataOutputStream outputStream = null;
+    try {
+      if (createFlag) {
+        outputStream = dfs.create(testPath);
+      } else {
+        outputStream = dfs.append(testPath);
+      }
+      outputStream.write("hello world. \n".getBytes());
+      long proxyOpBeforeRenewLease = rpcMetrics.getProxyOps();
+      assertTrue(dfs.getClient().renewLease());
+      long proxyOpAfterRenewLease = rpcMetrics.getProxyOps();
+      assertEquals((proxyOpBeforeRenewLease + 1), proxyOpAfterRenewLease);
+    } finally {
+      if (outputStream != null) {
+        outputStream.close();
+      }
+    }
+  }
+
+  @Test
+  public void testRenewLeaseForECFile() throws Exception {
+    String ecName = "RS-6-3-1024k";
+    FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics();
+    // Install a mount point to a different path to check
+    MockResolver resolver =
+        (MockResolver)router.getRouter().getSubclusterResolver();
+    String ns0 = cluster.getNameservices().get(0);
+    resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0");
+
+    // Stop LeaseRenewer
+    DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS;
+    routerDFS.getClient().getLeaseRenewer().interruptAndJoin();
+
+    Path testECPath = new Path("/testRenewLease0/ecDirectory/test_ec.txt");
+    routerDFS.mkdirs(testECPath.getParent());
+    routerDFS.setErasureCodingPolicy(
+        testECPath.getParent(), ecName);
+    testRenewLeaseInternal(routerDFS, metrics, testECPath, true);
+
+    ErasureCodingPolicy ecPolicy = routerDFS.getErasureCodingPolicy(testECPath);
+    assertNotNull(ecPolicy);
+    assertEquals(ecName, ecPolicy.getName());
+  }
+
+
+  @Test
+  public void testRenewLeaseForReplicaFile() throws Exception {
+    FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics();
+    // Install a mount point to a different path to check
+    MockResolver resolver =
+        (MockResolver)router.getRouter().getSubclusterResolver();
+    String ns0 = cluster.getNameservices().get(0);
+    resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0");
+
+    // Stop LeaseRenewer
+    DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS;
+    routerDFS.getClient().getLeaseRenewer().interruptAndJoin();
+
+    // Test Replica File
+    Path testPath = new Path("/testRenewLease0/test_replica.txt");
+    testRenewLeaseInternal(routerDFS, metrics, testPath, true);
+    testRenewLeaseInternal(routerDFS, metrics, testPath, false);
+  }
+
+  @Test
+  public void testRenewLeaseWithMultiStream() throws Exception {
+    FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics();
+    // Install a mount point to a different path to check
+    MockResolver resolver =
+        (MockResolver)router.getRouter().getSubclusterResolver();
+    String ns0 = cluster.getNameservices().get(0);
+    String ns1 = cluster.getNameservices().get(1);
+    resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0");
+    resolver.addLocation("/testRenewLease1", ns1, "/testRenewLease1");
+
+    // Stop LeaseRenewer
+    DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS;
+    routerDFS.getClient().getLeaseRenewer().interruptAndJoin();
+
+    Path newTestPath0 = new Path("/testRenewLease0/test1.txt");
+    Path newTestPath1 = new Path("/testRenewLease1/test1.txt");
+    try (FSDataOutputStream outStream1 = routerDFS.create(newTestPath0);
+         FSDataOutputStream outStream2 = routerDFS.create(newTestPath1)) {
+      outStream1.write("hello world \n".getBytes());
+      outStream2.write("hello world \n".getBytes());
+      long proxyOpBeforeRenewLease2 = metrics.getProxyOps();
+      assertTrue(routerDFS.getClient().renewLease());
+      long proxyOpAfterRenewLease2 = metrics.getProxyOps();
+      assertEquals((proxyOpBeforeRenewLease2 + 2), proxyOpAfterRenewLease2);
+    }
+  }
+
+  @Test
+  public void testMkdirWithDisableNameService() throws Exception {
+    MockResolver resolver = (MockResolver)router.getRouter().getSubclusterResolver();
+    String ns0 = cluster.getNameservices().get(0);
+    resolver.addLocation("/mnt", ns0, "/");
+    MockResolver activeNamenodeResolver = (MockResolver)router.getRouter().getNamenodeResolver();
+    activeNamenodeResolver.disableNamespace(ns0);
+
+    try {
+      FsPermission permission = new FsPermission("777");
+      RouterRpcServer rpcServer = router.getRouter().getRpcServer();
+      LambdaTestUtils.intercept(NoLocationException.class,
+          () -> rpcServer.mkdirs("/mnt/folder0/folder1", permission, true));
+    } finally {
+      activeNamenodeResolver.clearDisableNamespaces();
+    }
+  }
+
   @Test
   public void testProxyExceptionMessages() throws IOException {
 
@@ -1950,9 +2068,12 @@ public void testMkdirsWithCallerContext() throws IOException {
     FsPermission permission = new FsPermission("755");
     routerProtocol.mkdirs(dirPath, permission, false);
 
-    // The audit log should contains "callerContext=clientContext,clientIp:"
-    assertTrue(auditlog.getOutput()
-        .contains("callerContext=clientContext,clientIp:"));
+    // The audit log should contains "callerContext=clientIp:...,clientContext"
+    final String logOutput = auditlog.getOutput();
+    assertTrue(logOutput.contains("callerContext=clientIp:"));
+    assertTrue(logOutput.contains(",clientContext"));
+    assertTrue(logOutput.contains(",clientId"));
+    assertTrue(logOutput.contains(",clientCallId"));
     assertTrue(verifyFileExists(routerFS, dirPath));
   }
 
@@ -1967,4 +2088,146 @@ public void testSetBalancerBandwidth() throws Exception {
       return datanodes.get(0).getBalancerBandwidth() == newBandwidth;
     }, 100, 60 * 1000);
   }
+
+  @Test
+  public void testAddClientIpPortToCallerContext() throws IOException {
+    GenericTestUtils.LogCapturer auditLog =
+        GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+    // 1. ClientIp and ClientPort are not set on the client.
+    // Set client context.
+    CallerContext.setCurrent(
+        new CallerContext.Builder("clientContext").build());
+
+    // Create a directory via the router.
+    String dirPath = "/test";
+    routerProtocol.mkdirs(dirPath, new FsPermission("755"), false);
+
+    // The audit log should contains "clientIp:" and "clientPort:".
+    assertTrue(auditLog.getOutput().contains("clientIp:"));
+    assertTrue(auditLog.getOutput().contains("clientPort:"));
+    assertTrue(verifyFileExists(routerFS, dirPath));
+    auditLog.clearOutput();
+
+    // 2. ClientIp and ClientPort are set on the client.
+    // Reset client context.
+    CallerContext.setCurrent(
+        new CallerContext.Builder(
+            "clientContext,clientIp:1.1.1.1,clientPort:1234").build());
+
+    // Create a directory via the router.
+    routerProtocol.getFileInfo(dirPath);
+
+    // The audit log should not contain the original clientIp and clientPort
+    // set by client.
+    assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1"));
+    assertFalse(auditLog.getOutput().contains("clientPort:1234"));
+  }
+
+  @Test
+  public void testAddClientIdAndCallIdToCallerContext() throws IOException {
+    GenericTestUtils.LogCapturer auditLog =
+        GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog);
+
+    // 1. ClientId and ClientCallId are not set on the client.
+    // Set client context.
+    CallerContext.setCurrent(
+        new CallerContext.Builder("clientContext").build());
+
+    // Create a directory via the router.
+    String dirPath = "/test";
+    routerProtocol.mkdirs(dirPath, new FsPermission("755"), false);
+
+    // The audit log should contains "clientId:" and "clientCallId:".
+    assertTrue(auditLog.getOutput().contains("clientId:"));
+    assertTrue(auditLog.getOutput().contains("clientCallId:"));
+    assertTrue(verifyFileExists(routerFS, dirPath));
+    auditLog.clearOutput();
+
+    // 2. ClientId and ClientCallId are set on the client.
+    // Reset client context.
+    CallerContext.setCurrent(
+        new CallerContext.Builder(
+            "clientContext,clientId:mockClientId,clientCallId:4321").build());
+
+    // Create a directory via the router.
+    routerProtocol.getFileInfo(dirPath);
+
+    // The audit log should not contain the original clientId and clientCallId
+    // set by client.
+    assertFalse(auditLog.getOutput().contains("clientId:mockClientId"));
+    assertFalse(auditLog.getOutput().contains("clientCallId:4321"));
+  }
+
+  @Test
+  public void testContentSummaryWithSnapshot() throws Exception {
+    DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS;
+    Path dirPath = new Path("/testdir");
+    Path subdirPath = new Path(dirPath, "subdir");
+    Path filePath1 = new Path(dirPath, "file");
+    Path filePath2 = new Path(subdirPath, "file2");
+
+    // Create directories.
+    routerDFS.mkdirs(dirPath);
+    routerDFS.mkdirs(subdirPath);
+
+    // Create files.
+    createFile(routerDFS, filePath1.toString(), 32);
+    createFile(routerDFS, filePath2.toString(), 16);
+
+    // Allow & Create snapshot.
+    routerDFS.allowSnapshot(dirPath);
+    routerDFS.createSnapshot(dirPath, "s1");
+
+    try {
+      // Check content summary, snapshot count should be 0
+      ContentSummary contentSummary = routerDFS.getContentSummary(dirPath);
+      assertEquals(0, contentSummary.getSnapshotDirectoryCount());
+      assertEquals(0, contentSummary.getSnapshotFileCount());
+
+      // Delete the file & subdir(Total 2 files deleted & 1 directory)
+      routerDFS.delete(filePath1, true);
+      routerDFS.delete(subdirPath, true);
+
+      // Get the Content Summary
+      contentSummary = routerDFS.getContentSummary(dirPath);
+      assertEquals(1, contentSummary.getSnapshotDirectoryCount());
+      assertEquals(2, contentSummary.getSnapshotFileCount());
+    } finally {
+      // Cleanup
+      routerDFS.deleteSnapshot(dirPath, "s1");
+      routerDFS.disallowSnapshot(dirPath);
+      routerDFS.delete(dirPath, true);
+    }
+  }
+
+  @Test
+  public void testDisableNodeUsageInRBFMetrics() throws JSONException {
+    RBFMetrics rbfMetrics = router.getRouter().getMetrics();
+    FederationRPCMetrics federationRPCMetrics = router.getRouter().getRpcServer().getRPCMetrics();
+
+    long proxyOpBefore = federationRPCMetrics.getProxyOps();
+    String nodeUsageEnable = router.getRouter().getMetrics().getNodeUsage();
+    assertNotNull(nodeUsageEnable);
+    long proxyOpAfterWithEnable = federationRPCMetrics.getProxyOps();
+    assertEquals(proxyOpBefore + 2, proxyOpAfterWithEnable);
+
+    rbfMetrics.setEnableGetDNUsage(false);
+    String nodeUsageDisable = rbfMetrics.getNodeUsage();
+    assertNotNull(nodeUsageDisable);
+    long proxyOpAfterWithDisable = federationRPCMetrics.getProxyOps();
+    assertEquals(proxyOpAfterWithEnable, proxyOpAfterWithDisable);
+    JSONObject jsonObject = new JSONObject(nodeUsageDisable);
+    JSONObject json = jsonObject.getJSONObject("nodeUsage");
+    assertEquals("0.00%", json.get("min"));
+    assertEquals("0.00%", json.get("median"));
+    assertEquals("0.00%", json.get("max"));
+    assertEquals("0.00%", json.get("stdDev"));
+
+    rbfMetrics.setEnableGetDNUsage(true);
+    String nodeUsageWithReEnable = rbfMetrics.getNodeUsage();
+    assertNotNull(nodeUsageWithReEnable);
+    long proxyOpAfterWithReEnable = federationRPCMetrics.getProxyOps();
+    assertEquals(proxyOpAfterWithDisable + 2, proxyOpAfterWithReEnable);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
index e50464c0be7b5..6ade57326e316 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
@@ -75,7 +75,7 @@
 import org.junit.Test;
 
 /**
- * The the RPC interface of the {@link getRouter()} implemented by
+ * The RPC interface of the {@link getRouter()} implemented by
  * {@link RouterRpcServer}.
  */
 public class TestRouterRpcMultiDestination extends TestRouterRpc {
@@ -464,9 +464,8 @@ public void testCallerContextWithMultiDestinations() throws IOException {
     for (String line : auditLog.getOutput().split("\n")) {
       if (line.contains(auditFlag)) {
         // assert origin caller context exist in audit log
-        assertTrue(line.contains("callerContext=clientContext"));
-        String callerContext = line.substring(
-            line.indexOf("callerContext=clientContext"));
+        String callerContext = line.substring(line.indexOf("callerContext="));
+        assertTrue(callerContext.contains("clientContext"));
         // assert client ip info exist in caller context
         assertTrue(callerContext.contains(clientIpInfo));
         // assert client ip info appears only once in caller context
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
index fa1079a4edeb4..57518c811197f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcStoragePolicySatisfier.java
@@ -31,7 +31,6 @@
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
 import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
-import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
 import org.apache.hadoop.hdfs.server.namenode.sps.Context;
 import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
 import org.apache.hadoop.hdfs.server.sps.ExternalSPSContext;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterTrash.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterTrash.java
index acd7b87a14b4f..dfb8c33c72d4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterTrash.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterTrash.java
@@ -189,6 +189,44 @@ public void testMoveToTrashNoMountPoint() throws IOException,
     assertEquals(2, fileStatuses.length);
   }
 
+  @Test
+  public void testMoveToTrashWithKerberosUser() throws IOException,
+      URISyntaxException, InterruptedException {
+    //Constructs the structure of the KerBoers user name
+    String kerberosUser = "randomUser/dev@HADOOP.COM";
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(kerberosUser);
+    MountTable addEntry = MountTable.newInstance(MOUNT_POINT,
+        Collections.singletonMap(ns1, MOUNT_POINT));
+    assertTrue(addMountTable(addEntry));
+    // current user client
+    MiniRouterDFSCluster.NamenodeContext nn1Context = cluster.getNamenode(ns1, null);
+    DFSClient currentUserClientNs0 = nnContext.getClient();
+    DFSClient currentUserClientNs1 = nn1Context.getClient();
+
+    currentUserClientNs0.setOwner("/", ugi.getShortUserName(), ugi.getShortUserName());
+    currentUserClientNs1.setOwner("/", ugi.getShortUserName(), ugi.getShortUserName());
+
+    // test user client
+    DFSClient testUserClientNs1 = nn1Context.getClient(ugi);
+    testUserClientNs1.mkdirs(MOUNT_POINT, new FsPermission("777"), true);
+    assertTrue(testUserClientNs1.exists(MOUNT_POINT));
+    // create test file
+    testUserClientNs1.create(FILE, true);
+    Path filePath = new Path(FILE);
+
+    FileStatus[] fileStatuses = routerFs.listStatus(filePath);
+    assertEquals(1, fileStatuses.length);
+    assertEquals(ugi.getShortUserName(), fileStatuses[0].getOwner());
+    // move to Trash
+    Configuration routerConf = routerContext.getConf();
+    FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, routerConf);
+    Trash trash = new Trash(fs, routerConf);
+    assertTrue(trash.moveToTrash(filePath));
+    fileStatuses = fs.listStatus(
+        new Path("/user/" + ugi.getShortUserName() + "/.Trash/Current" + MOUNT_POINT));
+    assertEquals(1, fileStatuses.length);
+  }
+
   @Test
   public void testDeleteToTrashExistMountPoint() throws IOException,
       URISyntaxException, InterruptedException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
index 707a2f7baa348..2be7b7b1f49c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterUserMappings.java
@@ -39,7 +39,6 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.tools.GetUserMappingsProtocol;
-import org.apache.hadoop.util.Sets;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -57,6 +56,8 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -120,7 +121,7 @@ public Set<String> getGroupsSet(String user) throws IOException {
       LOG.info("Getting groups in MockUnixGroupsMapping");
       String g1 = user + (10 * i + 1);
       String g2 = user + (10 * i + 2);
-      Set<String> s = Sets.newHashSet(g1, g2);
+      Set<String> s = new HashSet<>(Arrays.asList(g1, g2));
       i++;
       return s;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
index 8e82d44c4ddfe..7c3643f5a511e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
@@ -23,9 +23,12 @@
 import static org.junit.Assert.fail;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +37,7 @@
 import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
 import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
 import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -145,4 +149,24 @@ public void testGetNsFromDataNodeNetworkLocation() {
     assertEquals("", RouterWebHdfsMethods
         .getNsFromDataNodeNetworkLocation("whatever-rack-info1"));
   }
+
+  @Test
+  public void testWebHdfsCreateWithInvalidPath() throws Exception {
+    // A path name include duplicated slashes.
+    String path = "//tmp//file";
+    assertResponse(path);
+  }
+
+  private void assertResponse(String path) throws IOException {
+    URL url = new URL(getUri(path));
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    // Assert response code.
+    assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, conn.getResponseCode());
+    // Assert exception.
+    Map<?, ?> response = WebHdfsFileSystem.jsonParse(conn, true);
+    assertEquals("InvalidPathException",
+        ((LinkedHashMap) response.get("RemoteException")).get("exception"));
+    conn.disconnect();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/token/TestZKDelegationTokenSecretManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/token/TestZKDelegationTokenSecretManagerImpl.java
index 3c7f8e88a91d1..e4c293091b045 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/token/TestZKDelegationTokenSecretManagerImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/token/TestZKDelegationTokenSecretManagerImpl.java
@@ -110,7 +110,7 @@ public void testMultiNodeTokenRemovalShortSyncWithoutWatch()
     conf.setBoolean(ZK_DTSM_TOKEN_WATCHER_ENABLED, false);
     // make sync quick
     conf.setInt(ZK_DTSM_ROUTER_TOKEN_SYNC_INTERVAL, 3);
-    // set the renew window and removal interval to be a
+    // set the renewal window and removal interval to be a
     // short time to trigger the background cleanup
     conf.setInt(RENEW_INTERVAL, 10);
     conf.setInt(REMOVAL_SCAN_INTERVAL, 10);
@@ -170,7 +170,7 @@ public void testMultiNodeTokenRemovalLongSyncWithoutWatch()
     conf.setBoolean(ZK_DTSM_TOKEN_WATCHER_ENABLED, false);
     // make sync quick
     conf.setInt(ZK_DTSM_ROUTER_TOKEN_SYNC_INTERVAL, 20);
-    // set the renew window and removal interval to be a
+    // set the renewal window and removal interval to be a
     // short time to trigger the background cleanup
     conf.setInt(RENEW_INTERVAL, 10);
     conf.setInt(REMOVAL_SCAN_INTERVAL, 10);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
index 0fad76de050bd..50840460a3943 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/FederationStateStoreTestUtils.java
@@ -269,6 +269,7 @@ public static MembershipState createMockRegistrationForNamenode(
     stats.setNumOfDecomActiveDatanodes(15);
     stats.setNumOfDecomDeadDatanodes(5);
     stats.setNumOfBlocks(10);
+    stats.setPendingSPSPaths(10);
     entry.setStats(stats);
     return entry;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
index 63bc6235a6116..533a1d97daf7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java
@@ -186,7 +186,7 @@ public void testRegistrationMajorityQuorum()
     // 1) ns0:nn0 - Standby (newest)
     // 2) ns0:nn0 - Active (oldest)
     // 3) ns0:nn0 - Active (2nd oldest)
-    // 4) ns0:nn0 - Active (3nd oldest element, newest active element)
+    // 4) ns0:nn0 - Active (3rd oldest element, newest active element)
     // Verify the selected entry is the newest majority opinion (4)
     String ns = "ns0";
     String nn = "nn0";
@@ -586,8 +586,6 @@ private MembershipState getExpiredNamenodeRegistration(
   /**
    * Register a namenode heartbeat with the state store.
    *
-   * @param store FederationMembershipStateStore instance to retrieve the
-   *          membership data records.
    * @param namenode A fully populated namenode membership record to be
    *          committed to the data store.
    * @return True if successful, false otherwise.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
index 6e5bd9ca85ffb..d4dfcc4863fdb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java
@@ -227,7 +227,6 @@ private MountTable getMountTableEntry(String mount) throws IOException {
   /**
    * Fetch all mount table records beneath a root path.
    *
-   * @param store FederationMountTableStore instance to commit the data.
    * @param mount The root search path, enter "/" to return all mount table
    *          records.
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
index fe1b9a5bfa04c..5ad01dce8e729 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreDriverBase.java
@@ -234,6 +234,25 @@ public <T extends BaseRecord> void testInsert(
     assertEquals(11, records2.size());
   }
 
+  public <T extends BaseRecord> void testInsertWithErrorDuringWrite(
+      StateStoreDriver driver, Class<T> recordClass)
+      throws IllegalArgumentException, IllegalAccessException, IOException {
+
+    assertTrue(driver.removeAll(recordClass));
+    QueryResult<T> queryResult0 = driver.get(recordClass);
+    List<T> records0 = queryResult0.getRecords();
+    assertTrue(records0.isEmpty());
+
+    // Insert single
+    BaseRecord record = generateFakeRecord(recordClass);
+    driver.put(record, true, false);
+
+    // Verify that no record was inserted.
+    QueryResult<T> queryResult1 = driver.get(recordClass);
+    List<T> records1 = queryResult1.getRecords();
+    assertEquals(0, records1.size());
+  }
+
   public <T extends BaseRecord> void testFetchErrors(StateStoreDriver driver,
       Class<T> clazz) throws IllegalAccessException, IOException {
 
@@ -556,7 +575,7 @@ private static Map<String, Class<?>> getFields(BaseRecord record) {
   }
 
   /**
-   * Get the type of a field.
+   * Get the type of field.
    *
    * @param fieldName
    * @return Field type
@@ -601,7 +620,7 @@ private static Method locateGetter(BaseRecord record, String fieldName) {
   }
 
   /**
-   * Expands a data object from the store into an record object. Default store
+   * Expands a data object from the store into a record object. Default store
    * data type is a String. Override if additional serialization is required.
    *
    * @param data Object containing the serialized data. Only string is
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
index 8c4b188cc47e3..dbd4b9bdae2ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/driver/TestStateStoreFileSystem.java
@@ -17,16 +17,26 @@
  */
 package org.apache.hadoop.hdfs.server.federation.store.driver;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileBaseImpl;
 import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl;
+import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+
 
 /**
  * Test the FileSystem (e.g., HDFS) implementation of the State Store driver.
@@ -91,4 +101,18 @@ public void testMetrics()
       throws IllegalArgumentException, IllegalAccessException, IOException {
     testMetrics(getStateStoreDriver());
   }
+
+  @Test
+  public void testInsertWithErrorDuringWrite()
+      throws IllegalArgumentException, IllegalAccessException, IOException {
+    StateStoreFileBaseImpl driver = spy((StateStoreFileBaseImpl)getStateStoreDriver());
+    doAnswer((Answer<BufferedWriter>) a -> {
+      BufferedWriter writer = (BufferedWriter) a.callRealMethod();
+      BufferedWriter spyWriter = spy(writer);
+      doThrow(IOException.class).when(spyWriter).write(any(String.class));
+      return spyWriter;
+    }).when(driver).getWriter(any());
+
+    testInsertWithErrorDuringWrite(driver, MembershipState.class);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/MockStateStoreDriver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/MockStateStoreDriver.java
new file mode 100644
index 0000000000000..9f600cb6f3fde
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/MockStateStoreDriver.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.records;
+
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils;
+import org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreBaseImpl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A mock StateStoreDriver that runs in memory that can force IOExceptions
+ * upon demand.
+ */
+public class MockStateStoreDriver extends StateStoreBaseImpl {
+  private boolean giveErrors = false;
+  private boolean initialized = false;
+  private static final Map<String, Map<String, BaseRecord>> VALUE_MAP = new HashMap<>();
+
+  @Override
+  public boolean initDriver() {
+    initialized = true;
+    return true;
+  }
+
+  @Override
+  public <T extends BaseRecord> boolean initRecordStorage(String className,
+                                                          Class<T> clazz) {
+    return true;
+  }
+
+  @Override
+  public boolean isDriverReady() {
+    return initialized;
+  }
+
+  @Override
+  public void close() throws Exception {
+    VALUE_MAP.clear();
+    initialized = false;
+  }
+
+  /**
+   * Should this object throw an IOException on each following call?
+   * @param value should we throw errors?
+   */
+  public void setGiveErrors(boolean value) {
+    giveErrors = value;
+  }
+
+  /**
+   * Check to see if this StateStore should throw IOException on each call.
+   * @throws IOException thrown if giveErrors has been set
+   */
+  private void checkErrors() throws IOException {
+    if (giveErrors) {
+      throw new IOException("Induced errors");
+    }
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T extends BaseRecord> QueryResult<T> get(Class<T> clazz) throws IOException {
+    checkErrors();
+    Map<String, BaseRecord> map = VALUE_MAP.get(StateStoreUtils.getRecordName(clazz));
+    List<T> results =
+        map != null ? new ArrayList<>((Collection<T>) map.values()) : new ArrayList<>();
+    return new QueryResult<>(results, System.currentTimeMillis());
+  }
+
+  @Override
+  public <T extends BaseRecord> boolean putAll(List<T> records,
+                                               boolean allowUpdate,
+                                               boolean errorIfExists)
+      throws IOException {
+    checkErrors();
+    for (T record : records) {
+      Map<String, BaseRecord> map =
+          VALUE_MAP.computeIfAbsent(StateStoreUtils.getRecordName(record.getClass()),
+              k -> new HashMap<>());
+      String key = record.getPrimaryKey();
+      BaseRecord oldRecord = map.get(key);
+      if (oldRecord == null || allowUpdate) {
+        map.put(key, record);
+      } else if (errorIfExists) {
+        throw new IOException("Record already exists for " + record.getClass()
+            + ": " + key);
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Clear all records from the store.
+   */
+  public void clearAll() {
+    VALUE_MAP.clear();
+  }
+
+  @Override
+  public <T extends BaseRecord> boolean removeAll(Class<T> clazz) throws IOException {
+    checkErrors();
+    return VALUE_MAP.remove(StateStoreUtils.getRecordName(clazz)) != null;
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public <T extends BaseRecord> int remove(Class<T> clazz,
+                                           Query<T> query)
+      throws IOException {
+    checkErrors();
+    int result = 0;
+    Map<String, BaseRecord> map =
+        VALUE_MAP.get(StateStoreUtils.getRecordName(clazz));
+    if (map != null) {
+      for (Iterator<BaseRecord> itr = map.values().iterator(); itr.hasNext();) {
+        BaseRecord record = itr.next();
+        if (query.matches((T) record)) {
+          itr.remove();
+          result += 1;
+        }
+      }
+    }
+    return result;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
index dfe2bc98bf40a..8226178fe7691 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/records/TestRouterState.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -20,8 +20,16 @@
 import static org.junit.Assert.assertEquals;
 
 import java.io.IOException;
+import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState;
+import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
 import org.apache.hadoop.hdfs.server.federation.router.RouterServiceState;
+import org.apache.hadoop.hdfs.server.federation.store.StateStoreService;
+import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreDriver;
 import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
 import org.junit.Test;
 
@@ -40,7 +48,7 @@ public class TestRouterState {
   private static final RouterServiceState STATE = RouterServiceState.RUNNING;
 
 
-  private RouterState generateRecord() throws IOException {
+  private RouterState generateRecord() {
     RouterState record = RouterState.newInstance(ADDRESS, START_TIME, STATE);
     record.setVersion(VERSION);
     record.setCompileInfo(COMPILE_INFO);
@@ -82,4 +90,46 @@ public void testSerialization() throws IOException {
 
     validateRecord(newRecord);
   }
+
+  @Test
+  public void testStateStoreResilience() throws Exception {
+    StateStoreService service = new StateStoreService();
+    Configuration conf = new Configuration();
+    conf.setClass(RBFConfigKeys.FEDERATION_STORE_DRIVER_CLASS,
+        MockStateStoreDriver.class,
+        StateStoreDriver.class);
+    conf.setBoolean(RBFConfigKeys.DFS_ROUTER_METRICS_ENABLE, false);
+    service.init(conf);
+    MockStateStoreDriver driver = (MockStateStoreDriver) service.getDriver();
+    driver.clearAll();
+    // Add two records for block1
+    driver.put(MembershipState.newInstance("routerId", "ns1",
+        "ns1-ha1", "cluster1", "block1", "rpc1",
+        "service1", "lifeline1", "https", "nn01",
+        FederationNamenodeServiceState.ACTIVE, false), false, false);
+    driver.put(MembershipState.newInstance("routerId", "ns1",
+        "ns1-ha2", "cluster1", "block1", "rpc2",
+        "service2", "lifeline2", "https", "nn02",
+        FederationNamenodeServiceState.STANDBY, false), false, false);
+    // load the cache
+    service.loadDriver();
+    MembershipNamenodeResolver resolver = new MembershipNamenodeResolver(conf, service);
+    service.refreshCaches(true);
+
+    // look up block1
+    List<? extends FederationNamenodeContext> result =
+        resolver.getNamenodesForBlockPoolId("block1");
+    assertEquals(2, result.size());
+
+    // cause io errors and then reload the cache
+    driver.setGiveErrors(true);
+    long previousUpdate = service.getCacheUpdateTime();
+    service.refreshCaches(true);
+    assertEquals(previousUpdate, service.getCacheUpdateTime());
+
+    // make sure the old cache is still there
+    result = resolver.getNamenodesForBlockPoolId("block1");
+    assertEquals(2, result.size());
+    service.stop();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.10.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.10.2.xml
new file mode 100644
index 0000000000000..8ab450692ddf3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.10.2.xml
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue May 24 22:54:50 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 2.10.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.10.2.jar:/usr/lib/jvm/zulu-7-amd64/lib/tools.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.10.2.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/7.9/nimbus-jose-jwt-7.9.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.3/json-smart-1.3.3.jar:/maven/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/jline/jline/0.9.94/jline-0.9.94.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.10.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-2.10.2.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.5/commons-io-2.5.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.50.Final/netty-all-4.1.50.Final.jar:/maven/xerces/xercesImpl/2.12.0/xercesImpl-2.12.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.9.10/jackson-core-2.9.10.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 2.10.2 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.server.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml
new file mode 100644
index 0000000000000..5454f53be9122
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.3.xml
@@ -0,0 +1,674 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Sun Mar 20 02:08:45 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 3.2.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.3.jar:/maven/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.3.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.40.v20210413/jetty-servlet-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-security/9.4.40.v20210413/jetty-security-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.40.v20210413/jetty-webapp-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.40.v20210413/jetty-xml-9.4.40.v20210413.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/eclipse/jetty/jetty-server/9.4.40.v20210413/jetty-server-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-http/9.4.40.v20210413/jetty-http-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-io/9.4.40.v20210413/jetty-io-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-util/9.4.40.v20210413/jetty-util-9.4.40.v20210413.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.40.v20210413/jetty-util-ajax-9.4.40.v20210413.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/log4j/log4j/1.2.17/log4j-1.2.17.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.68.Final/netty-all-4.1.68.Final.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 3.2.3 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.aliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+  <class name="InMemoryAliasMap" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="init" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="list" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol.IterationResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="marker" type="java.util.Optional"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="java.util.Optional"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getBlockPoolId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="fromProvidedStorageLocationBytes" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocationDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="com.google.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="fromBlockBytes" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="com.google.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InMemoryAliasMap is an implementation of the InMemoryAliasMapProtocol for
+ use with LevelDB.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+  <!-- start interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <interface name="BlockAlias"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface used to load provided blocks.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.FileRegion -->
+  <class name="FileRegion" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.common.BlockAlias"/>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="org.apache.hadoop.hdfs.protocol.Block, org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProvidedStorageLocation" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent provided blocks that are file regions,
+ i.e., can be described using (path, offset, length).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.FileRegion -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+  <class name="BlockAliasMap" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a reader to the alias map.
+ @param opts reader options
+ @param blockPoolID block pool id to use
+ @return {@link Reader} to the alias map. If a Reader for the blockPoolID
+ cannot be created, this will return null.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the writer for the alias map.
+ @param opts writer options.
+ @param blockPoolID block pool id to use
+ @return {@link Writer} to the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="refresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract class used to read and write block maps for provided blocks.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap.impl">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <class name="LevelDBFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="LevelDBFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A LevelDB based implementation of {@link BlockAliasMap}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+  <class name="TextFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TextFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="blockPoolIDFromFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="fileNameFromBlockPoolID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockPoolID" type="java.lang.String"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class is used for block maps stored as text files,
+ with a specified delimiter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.command">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.connectors">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.datamodel">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.planner">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml
new file mode 100644
index 0000000000000..2aa6ef1cdb5be
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml
@@ -0,0 +1,674 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:52:30 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.4.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/slf4j/slf4j-reload4j/1.7.35/slf4j-reload4j-1.7.35.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.68.Final/netty-all-4.1.68.Final.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 3.2.4 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.aliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+  <class name="InMemoryAliasMap" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="init" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="list" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol.IterationResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="marker" type="java.util.Optional"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="java.util.Optional"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getBlockPoolId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="fromProvidedStorageLocationBytes" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocationDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="com.google.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="fromBlockBytes" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="com.google.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InMemoryAliasMap is an implementation of the InMemoryAliasMapProtocol for
+ use with LevelDB.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+  <!-- start interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <interface name="BlockAlias"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface used to load provided blocks.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.FileRegion -->
+  <class name="FileRegion" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.common.BlockAlias"/>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="org.apache.hadoop.hdfs.protocol.Block, org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProvidedStorageLocation" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent provided blocks that are file regions,
+ i.e., can be described using (path, offset, length).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.FileRegion -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+  <class name="BlockAliasMap" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a reader to the alias map.
+ @param opts reader options
+ @param blockPoolID block pool id to use
+ @return {@link Reader} to the alias map. If a Reader for the blockPoolID
+ cannot be created, this will return null.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the writer for the alias map.
+ @param opts writer options.
+ @param blockPoolID block pool id to use
+ @return {@link Writer} to the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="refresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract class used to read and write block maps for provided blocks.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap.impl">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <class name="LevelDBFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="LevelDBFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A LevelDB based implementation of {@link BlockAliasMap}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+  <class name="TextFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TextFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="blockPoolIDFromFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="fileNameFromBlockPoolID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockPoolID" type="java.lang.String"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class is used for block maps stored as text files,
+ with a specified delimiter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.command">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.connectors">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.datamodel">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.planner">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml
new file mode 100644
index 0000000000000..b4d954cb53ebd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.2.xml
@@ -0,0 +1,835 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Feb 21 21:15:43 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 3.3.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.2.jar:/maven/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.2.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.2.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/log4j/log4j/1.2.17/log4j-1.2.17.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/com/google/code/findbugs/findbugs/3.0.1/findbugs-3.0.1.jar:/maven/net/jcip/jcip-annotations/1.0/jcip-annotations-1.0.jar:/maven/com/google/code/findbugs/bcel-findbugs/6.0/bcel-findbugs-6.0.jar:/maven/com/google/code/findbugs/jFormatString/2.0.1/jFormatString-2.0.1.jar:/maven/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar:/maven/org/ow2/asm/asm-debug-all/5.0.2/asm-debug-all-5.0.2.jar:/maven/org/ow2/asm/asm-commons/5.0.2/asm-commons-5.0.2.jar:/maven/org/ow2/asm/asm-tree/5.0.2/asm-tree-5.0.2.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/com/apple/AppleJavaExtensions/1.4/AppleJavaExtensions-1.4.jar:/maven/jaxen/jaxen/1.1.6/jaxen-1.1.6.jar:/maven/org/slf4j/slf4j-log4j12/1.7.30/slf4j-log4j12-1.7.30.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.68.Final/netty-all-4.1.68.Final.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.2.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 3.3.2 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <method name="getHostAndPort" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get host and port of JournalNode.
+
+ @return colon separated host and port.]]>
+      </doc>
+    </method>
+    <method name="getClusterIds" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of the clusters of JournalNode's journals
+ as one JournalNode may support multiple clusters.
+
+ @return list of clusters.]]>
+      </doc>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the version of Hadoop.
+
+ @return the version of Hadoop.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.aliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+  <class name="InMemoryAliasMap" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="init" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="list" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol.IterationResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="marker" type="java.util.Optional"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="java.util.Optional"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getBlockPoolId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="fromProvidedStorageLocationBytes" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocationDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="fromBlockBytes" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transferForBootstrap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="response" type="javax.servlet.http.HttpServletResponse"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="aliasMap" type="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Transfer this aliasmap for bootstrapping standby Namenodes. The map is
+ transferred as a tar.gz archive. This archive needs to be extracted on the
+ standby Namenode.
+
+ @param response http response.
+ @param conf configuration to use.
+ @param aliasMap aliasmap to transfer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="completeBootstrapTransfer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aliasMap" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Extract the aliasmap archive to complete the bootstrap process. This method
+ has to be called after the aliasmap archive is transfered from the primary
+ Namenode.
+
+ @param aliasMap location of the aliasmap.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[InMemoryAliasMap is an implementation of the InMemoryAliasMapProtocol for
+ use with LevelDB.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+  <!-- start interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <interface name="BlockAlias"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface used to load provided blocks.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.FileRegion -->
+  <class name="FileRegion" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.common.BlockAlias"/>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="org.apache.hadoop.hdfs.protocol.Block, org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProvidedStorageLocation" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent provided blocks that are file regions,
+ i.e., can be described using (path, offset, length).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.FileRegion -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+  <class name="BlockAliasMap" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a reader to the alias map.
+ @param opts reader options
+ @param blockPoolID block pool id to use
+ @return {@link Reader} to the alias map. If a Reader for the blockPoolID
+ cannot be created, this will return null.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the writer for the alias map.
+ @param opts writer options.
+ @param blockPoolID block pool id to use
+ @return {@link Writer} to the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="refresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract class used to read and write block maps for provided blocks.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap.impl">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <class name="LevelDBFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="LevelDBFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A LevelDB based implementation of {@link BlockAliasMap}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+  <class name="TextFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TextFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="blockPoolIDFromFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="fileNameFromBlockPoolID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockPoolID" type="java.lang.String"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class is used for block maps stored as text files,
+ with a specified delimiter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.command">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.connectors">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.datamodel">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.planner">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <class name="DefaultAuditLogger" extends="org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DefaultAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logAuditMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <field name="STRING_BUILDER" type="java.lang.ThreadLocal"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="isCallerContextEnabled" type="boolean"
+      transient="false" volatile="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callerContextMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum bytes a caller context string can have.]]>
+      </doc>
+    </field>
+    <field name="callerSignatureMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logTokenTrackingId" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[adds a tracking ID for all audit log events.]]>
+      </doc>
+    </field>
+    <field name="debugCmdSet" type="java.util.Set"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of commands to provide debug messages.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for Namenode and Router to Audit events
+ information. This class can be extended and can be used when no access logger
+ is defined in the config file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.3.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.3.xml
new file mode 100644
index 0000000000000..0eaf5019aa6d2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.3.xml
@@ -0,0 +1,835 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:03:54 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/com/google/code/findbugs/findbugs/3.0.1/findbugs-3.0.1.jar:/maven/net/jcip/jcip-annotations/1.0/jcip-annotations-1.0.jar:/maven/com/google/code/findbugs/bcel-findbugs/6.0/bcel-findbugs-6.0.jar:/maven/com/google/code/findbugs/jFormatString/2.0.1/jFormatString-2.0.1.jar:/maven/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar:/maven/org/ow2/asm/asm-debug-all/5.0.2/asm-debug-all-5.0.2.jar:/maven/org/ow2/asm/asm-commons/5.0.2/asm-commons-5.0.2.jar:/maven/org/ow2/asm/asm-tree/5.0.2/asm-tree-5.0.2.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/com/apple/AppleJavaExtensions/1.4/AppleJavaExtensions-1.4.jar:/maven/jaxen/jaxen/1.1.6/jaxen-1.1.6.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.68.Final/netty-all-4.1.68.Final.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 3.3.3 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <method name="getHostAndPort" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get host and port of JournalNode.
+
+ @return colon separated host and port.]]>
+      </doc>
+    </method>
+    <method name="getClusterIds" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of the clusters of JournalNode's journals
+ as one JournalNode may support multiple clusters.
+
+ @return list of clusters.]]>
+      </doc>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the version of Hadoop.
+
+ @return the version of Hadoop.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.aliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+  <class name="InMemoryAliasMap" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="init" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="list" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol.IterationResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="marker" type="java.util.Optional"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="java.util.Optional"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getBlockPoolId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="fromProvidedStorageLocationBytes" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocationDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="fromBlockBytes" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transferForBootstrap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="response" type="javax.servlet.http.HttpServletResponse"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="aliasMap" type="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Transfer this aliasmap for bootstrapping standby Namenodes. The map is
+ transferred as a tar.gz archive. This archive needs to be extracted on the
+ standby Namenode.
+
+ @param response http response.
+ @param conf configuration to use.
+ @param aliasMap aliasmap to transfer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="completeBootstrapTransfer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aliasMap" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Extract the aliasmap archive to complete the bootstrap process. This method
+ has to be called after the aliasmap archive is transfered from the primary
+ Namenode.
+
+ @param aliasMap location of the aliasmap.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[InMemoryAliasMap is an implementation of the InMemoryAliasMapProtocol for
+ use with LevelDB.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+  <!-- start interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <interface name="BlockAlias"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface used to load provided blocks.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.FileRegion -->
+  <class name="FileRegion" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.common.BlockAlias"/>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="org.apache.hadoop.hdfs.protocol.Block, org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProvidedStorageLocation" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent provided blocks that are file regions,
+ i.e., can be described using (path, offset, length).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.FileRegion -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+  <class name="BlockAliasMap" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a reader to the alias map.
+ @param opts reader options
+ @param blockPoolID block pool id to use
+ @return {@link Reader} to the alias map. If a Reader for the blockPoolID
+ cannot be created, this will return null.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the writer for the alias map.
+ @param opts writer options.
+ @param blockPoolID block pool id to use
+ @return {@link Writer} to the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="refresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract class used to read and write block maps for provided blocks.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap.impl">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <class name="LevelDBFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="LevelDBFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A LevelDB based implementation of {@link BlockAliasMap}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+  <class name="TextFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TextFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="blockPoolIDFromFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="fileNameFromBlockPoolID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockPoolID" type="java.lang.String"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class is used for block maps stored as text files,
+ with a specified delimiter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.command">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.connectors">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.datamodel">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.planner">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <class name="DefaultAuditLogger" extends="org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DefaultAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logAuditMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <field name="STRING_BUILDER" type="java.lang.ThreadLocal"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="isCallerContextEnabled" type="boolean"
+      transient="false" volatile="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callerContextMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum bytes a caller context string can have.]]>
+      </doc>
+    </field>
+    <field name="callerSignatureMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logTokenTrackingId" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[adds a tracking ID for all audit log events.]]>
+      </doc>
+    </field>
+    <field name="debugCmdSet" type="java.util.Set"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of commands to provide debug messages.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for Namenode and Router to Audit events
+ information. This class can be extended and can be used when no access logger
+ is defined in the config file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml
new file mode 100644
index 0000000000000..b3978b01a6994
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml
@@ -0,0 +1,835 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:04:11 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.4.jar:/maven/com/squareup/okhttp3/okhttp/4.9.3/okhttp-4.9.3.jar:/maven/com/squareup/okio/okio/2.8.0/okio-2.8.0.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib/1.4.10/kotlin-stdlib-1.4.10.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib-common/1.4.10/kotlin-stdlib-common-1.4.10.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/com/google/code/findbugs/findbugs/3.0.1/findbugs-3.0.1.jar:/maven/net/jcip/jcip-annotations/1.0/jcip-annotations-1.0.jar:/maven/com/google/code/findbugs/bcel-findbugs/6.0/bcel-findbugs-6.0.jar:/maven/com/google/code/findbugs/jFormatString/2.0.1/jFormatString-2.0.1.jar:/maven/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar:/maven/org/ow2/asm/asm-debug-all/5.0.2/asm-debug-all-5.0.2.jar:/maven/org/ow2/asm/asm-commons/5.0.2/asm-commons-5.0.2.jar:/maven/org/ow2/asm/asm-tree/5.0.2/asm-tree-5.0.2.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/com/apple/AppleJavaExtensions/1.4/AppleJavaExtensions-1.4.jar:/maven/jaxen/jaxen/1.1.6/jaxen-1.1.6.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/io/netty/netty-all/4.1.77.Final/netty-all-4.1.77.Final.jar:/maven/io/netty/netty-buffer/4.1.77.Final/netty-buffer-4.1.77.Final.jar:/maven/io/netty/netty-codec/4.1.77.Final/netty-codec-4.1.77.Final.jar:/maven/io/netty/netty-codec-dns/4.1.77.Final/netty-codec-dns-4.1.77.Final.jar:/maven/io/netty/netty-codec-haproxy/4.1.77.Final/netty-codec-haproxy-4.1.77.Final.jar:/maven/io/netty/netty-codec-http/4.1.77.Final/netty-codec-http-4.1.77.Final.jar:/maven/io/netty/netty-codec-http2/4.1.77.Final/netty-codec-http2-4.1.77.Final.jar:/maven/io/netty/netty-codec-memcache/4.1.77.Final/netty-codec-memcache-4.1.77.Final.jar:/maven/io/netty/netty-codec-mqtt/4.1.77.Final/netty-codec-mqtt-4.1.77.Final.jar:/maven/io/netty/netty-codec-redis/4.1.77.Final/netty-codec-redis-4.1.77.Final.jar:/maven/io/netty/netty-codec-smtp/4.1.77.Final/netty-codec-smtp-4.1.77.Final.jar:/maven/io/netty/netty-codec-socks/4.1.77.Final/netty-codec-socks-4.1.77.Final.jar:/maven/io/netty/netty-codec-stomp/4.1.77.Final/netty-codec-stomp-4.1.77.Final.jar:/maven/io/netty/netty-codec-xml/4.1.77.Final/netty-codec-xml-4.1.77.Final.jar:/maven/io/netty/netty-common/4.1.77.Final/netty-common-4.1.77.Final.jar:/maven/io/netty/netty-handler/4.1.77.Final/netty-handler-4.1.77.Final.jar:/maven/io/netty/netty-handler-proxy/4.1.77.Final/netty-handler-proxy-4.1.77.Final.jar:/maven/io/netty/netty-resolver/4.1.77.Final/netty-resolver-4.1.77.Final.jar:/maven/io/netty/netty-resolver-dns/4.1.77.Final/netty-resolver-dns-4.1.77.Final.jar:/maven/io/netty/netty-transport/4.1.77.Final/netty-transport-4.1.77.Final.jar:/maven/io/netty/netty-transport-rxtx/4.1.77.Final/netty-transport-rxtx-4.1.77.Final.jar:/maven/io/netty/netty-transport-sctp/4.1.77.Final/netty-transport-sctp-4.1.77.Final.jar:/maven/io/netty/netty-transport-udt/4.1.77.Final/netty-transport-udt-4.1.77.Final.jar:/maven/io/netty/netty-transport-classes-epoll/4.1.77.Final/netty-transport-classes-epoll-4.1.77.Final.jar:/maven/io/netty/netty-transport-native-unix-common/4.1.77.Final/netty-transport-native-unix-common-4.1.77.Final.jar:/maven/io/netty/netty-transport-classes-kqueue/4.1.77.Final/netty-transport-classes-kqueue-4.1.77.Final.jar:/maven/io/netty/netty-resolver-dns-classes-macos/4.1.77.Final/netty-resolver-dns-classes-macos-4.1.77.Final.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 3.3.4 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <method name="getHostAndPort" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get host and port of JournalNode.
+
+ @return colon separated host and port.]]>
+      </doc>
+    </method>
+    <method name="getClusterIds" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of the clusters of JournalNode's journals
+ as one JournalNode may support multiple clusters.
+
+ @return list of clusters.]]>
+      </doc>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the version of Hadoop.
+
+ @return the version of Hadoop.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.aliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+  <class name="InMemoryAliasMap" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="init" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="list" return="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol.IterationResult"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="marker" type="java.util.Optional"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="read" return="java.util.Optional"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getBlockPoolId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="fromProvidedStorageLocationBytes" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocationDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="fromBlockBytes" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockDbFormat" type="byte[]"/>
+      <exception name="InvalidProtocolBufferException" type="org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="providedStorageLocation" type="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toProtoBufBytes" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="block" type="org.apache.hadoop.hdfs.protocol.Block"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="transferForBootstrap"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="response" type="javax.servlet.http.HttpServletResponse"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="aliasMap" type="org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Transfer this aliasmap for bootstrapping standby Namenodes. The map is
+ transferred as a tar.gz archive. This archive needs to be extracted on the
+ standby Namenode.
+
+ @param response http response.
+ @param conf configuration to use.
+ @param aliasMap aliasmap to transfer.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="completeBootstrapTransfer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aliasMap" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Extract the aliasmap archive to complete the bootstrap process. This method
+ has to be called after the aliasmap archive is transfered from the primary
+ Namenode.
+
+ @param aliasMap location of the aliasmap.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[InMemoryAliasMap is an implementation of the InMemoryAliasMapProtocol for
+ use with LevelDB.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+  <!-- start interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <interface name="BlockAlias"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface used to load provided blocks.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.common.BlockAlias -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.FileRegion -->
+  <class name="FileRegion" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.common.BlockAlias"/>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long, long, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="long, org.apache.hadoop.fs.Path, long, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileRegion" type="org.apache.hadoop.hdfs.protocol.Block, org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBlock" return="org.apache.hadoop.hdfs.protocol.Block"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProvidedStorageLocation" return="org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class is used to represent provided blocks that are file regions,
+ i.e., can be described using (path, offset, length).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.FileRegion -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+  <class name="BlockAliasMap" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BlockAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a reader to the alias map.
+ @param opts reader options
+ @param blockPoolID block pool id to use
+ @return {@link Reader} to the alias map. If a Reader for the blockPoolID
+ cannot be created, this will return null.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the writer for the alias map.
+ @param opts writer options.
+ @param blockPoolID block pool id to use
+ @return {@link Writer} to the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="refresh"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Refresh the alias map.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract class used to read and write block maps for provided blocks.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.common.blockaliasmap.impl">
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <class name="LevelDBFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="LevelDBFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A LevelDB based implementation of {@link BlockAliasMap}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap -->
+  <!-- start class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+  <class name="TextFileRegionAliasMap" extends="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TextFileRegionAliasMap"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReader" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Reader.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getWriter" return="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="opts" type="org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap.Writer.Options"/>
+      <param name="blockPoolID" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="refresh"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="blockPoolIDFromFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="fileNameFromBlockPoolID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blockPoolID" type="java.lang.String"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class is used for block maps stored as text files,
+ with a specified delimiter.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.command">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.connectors">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.datamodel">
+</package>
+<package name="org.apache.hadoop.hdfs.server.diskbalancer.planner">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <class name="DefaultAuditLogger" extends="org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DefaultAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logAuditMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+    </method>
+    <field name="STRING_BUILDER" type="java.lang.ThreadLocal"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="isCallerContextEnabled" type="boolean"
+      transient="false" volatile="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callerContextMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum bytes a caller context string can have.]]>
+      </doc>
+    </field>
+    <field name="callerSignatureMaxLen" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logTokenTrackingId" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[adds a tracking ID for all audit log events.]]>
+      </doc>
+    </field>
+    <field name="debugCmdSet" type="java.util.Set"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of commands to provide debug messages.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class provides an interface for Namenode and Router to Audit events
+ information. This class can be extended and can be used when no access logger
+ is defined in the config file.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.DefaultAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index ff4a8dc228aa6..b51c7154f7b92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -422,7 +422,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/main/webapps/static/moment.min.js</exclude>
             <exclude>src/main/webapps/static/dust-full-2.0.0.min.js</exclude>
             <exclude>src/main/webapps/static/dust-helpers-1.1.1.min.js</exclude>
-            <exclude>src/main/webapps/static/jquery-3.5.1.min.js</exclude>
+            <exclude>src/main/webapps/static/jquery-3.6.0.min.js</exclude>
             <exclude>src/main/webapps/static/jquery.dataTables.min.js</exclude>
             <exclude>src/main/webapps/static/json-bignum.js</exclude>
             <exclude>src/main/webapps/static/dataTables.bootstrap.css</exclude>
@@ -447,38 +447,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
           </filesets>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
@@ -503,7 +471,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
               <forkCount>${testsThreadCount}</forkCount>
               <reuseForks>false</reuseForks>
               <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 942929ac285e9..0e10bc61c99a1 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -123,6 +123,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.datanode.data.write.bandwidthPerSec";
   // A value of zero indicates no limit
   public static final long DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_DEFAULT = 0;
+  public static final String DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_KEY =
+      "dfs.datanode.ec.reconstruct.read.bandwidthPerSec";
+  public static final long DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_DEFAULT =
+      0; // A value of zero indicates no limit
+  public static final String DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_KEY =
+      "dfs.datanode.ec.reconstruct.write.bandwidthPerSec";
+  public static final long DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_DEFAULT =
+      0; // A value of zero indicates no limit
   @Deprecated
   public static final String  DFS_DATANODE_READAHEAD_BYTES_KEY =
       HdfsClientConfigKeys.DFS_DATANODE_READAHEAD_BYTES_KEY;
@@ -143,6 +151,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final long    DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
   public static final String  DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume";
   public static final int     DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+  public static final String  DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+      "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+  public static final int     DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
   public static final String  DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec";
   public static final int     DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60;
   public static final String  DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = "dfs.datanode.ram.disk.replica.tracker";
@@ -473,10 +484,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec";
   public static final long    DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L;
 
-  /** Block deletion increment. */
-  public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY =
-      "dfs.namenode.block.deletion.increment";
-  public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
+  /** The limit of single lock holding duration.*/
+  public static final String DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS =
+      "dfs.namenode.block.deletion.lock.threshold.ms";
+  public static final int DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT =
+      50;
+  /** The sleep interval for releasing lock.*/
+  public static final String DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS =
+      "dfs.namenode.block.deletion.unlock.interval.ms";
+  public static final int DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT =
+      10;
 
   public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
       HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
@@ -540,7 +557,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   // This value uses the times of heartbeat interval to define the minimum value for stale interval.  
   public static final String DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY = "dfs.namenode.stale.datanode.minimum.interval";
   public static final int DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_DEFAULT = 3; // i.e. min_interval is 3 * heartbeat_interval = 9s
-  
+  public static final String DFS_NAMENODE_REMOVE_DEAD_DATANODE_BATCHNUM_KEY
+      = "dfs.namenode.remove.dead.datanode.batchnum";
+  public static final int DFS_NAMENODE_REMOVE_BAD_BATCH_NUM_DEFAULT = 10;
   // When the percentage of stale datanodes reaches this ratio,
   // allow writing to stale nodes to prevent hotspots.
   public static final String DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY = "dfs.namenode.write.stale.datanode.ratio";
@@ -617,14 +636,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String DFS_DATANODE_LOCK_FAIR_KEY =
       "dfs.datanode.lock.fair";
   public static final boolean DFS_DATANODE_LOCK_FAIR_DEFAULT = true;
-  public static final String DFS_DATANODE_LOCK_READ_WRITE_ENABLED_KEY =
-      "dfs.datanode.lock.read.write.enabled";
-  public static final Boolean DFS_DATANODE_LOCK_READ_WRITE_ENABLED_DEFAULT =
-      true;
-  public static final String  DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_KEY =
-      "dfs.datanode.lock-reporting-threshold-ms";
-  public static final long
-      DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 300L;
 
   public static final String  DFS_UPGRADE_DOMAIN_FACTOR = "dfs.namenode.upgrade.domain.factor";
   public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = DFS_REPLICATION_DEFAULT;
@@ -698,6 +709,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.datanode.max.disks.to.report";
   public static final int DFS_DATANODE_MAX_DISKS_TO_REPORT_DEFAULT =
       5;
+  public static final String DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY =
+      "dfs.datanode.max.slowdisks.to.exclude";
+  public static final int DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT =
+      0;
   public static final String  DFS_DATANODE_HOST_NAME_KEY =
       HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY;
   public static final String  DFS_NAMENODE_CHECKPOINT_DIR_KEY =
@@ -727,6 +742,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.datanode.metrics.logger.period.seconds";
   public static final int DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT =
       600;
+  public static final String  DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY =
+      "dfs.namenode.audit.log.with.remote.port";
+  public static final boolean DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT =
+      false;
   /**
    * The maximum number of getBlocks RPCs data movement utilities can make to
    * a NameNode per second. Values &lt;= 0 disable throttling. This affects
@@ -735,6 +754,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
    */
   public static final String  DFS_NAMENODE_GETBLOCKS_MAX_QPS_KEY = "dfs.namenode.get-blocks.max-qps";
   public static final int     DFS_NAMENODE_GETBLOCKS_MAX_QPS_DEFAULT = 20;
+  public static final String  DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY
+      = "dfs.namenode.get-blocks.check.operation";
+  public static final boolean  DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT
+      = true;
 
   public static final String  DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth";
   public static final long    DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;
@@ -812,6 +835,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.storage.policy.satisfier.retry.max.attempts";
   public static final int DFS_STORAGE_POLICY_SATISFIER_MAX_RETRY_ATTEMPTS_DEFAULT =
       3;
+  public static final String DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_KEY =
+      "dfs.storage.policy.satisfier.move.task.retry.max.attempts";
+  public static final int DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT =
+      3;
   public static final String DFS_STORAGE_DEFAULT_POLICY =
       "dfs.storage.default.policy";
   public static final HdfsConstants.StoragePolicy
@@ -976,6 +1003,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.namenode.lifeline.handler.count";
   public static final String  DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY = "dfs.namenode.service.handler.count";
   public static final int     DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT = 10;
+  // List of users that can override their client ip
+  public static final String  DFS_NAMENODE_IP_PROXY_USERS = "dfs.namenode.ip-proxy-users";
   public static final String  DFS_HTTP_POLICY_KEY = "dfs.http.policy";
   public static final String  DFS_HTTP_POLICY_DEFAULT =  HttpConfig.Policy.HTTP_ONLY.name();
   public static final String  DFS_DATANODE_HTTPSERVER_FILTER_HANDLERS = "dfs.datanode.httpserver.filter.handlers";
@@ -1195,6 +1224,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction";
   public static final float   DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
       0.6f;
+  public static final String
+          DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY =
+          "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance";
+  public static final int
+      DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT =
+      5;
   public static final String
       DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY =
       "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy"
@@ -1202,6 +1237,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final float
       DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
       0.6f;
+  public static final String
+      DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY =
+      "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy"
+          + ".balanced-space-tolerance";
+  public static final int
+      DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT =
+      5;
   public static final String
       DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY =
       "dfs.namenode.available-space-block-placement-policy.balance-local-node";
@@ -1356,6 +1398,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       "dfs.journalnode.edits.dir.perm";
   public static final String DFS_JOURNAL_EDITS_DIR_PERMISSION_DEFAULT =
       "700";
+  public static final String  DFS_JOURNALNODE_HANDLER_COUNT_KEY =
+      "dfs.journalnode.handler.count";
+  public static final int     DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT = 5;
+
 
   public static final String  DFS_JOURNALNODE_HTTP_ADDRESS_KEY = "dfs.journalnode.http-address";
   public static final int     DFS_JOURNALNODE_HTTP_PORT_DEFAULT = 8480;
@@ -1378,7 +1424,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final long DFS_JOURNALNODE_SYNC_INTERVAL_DEFAULT = 2*60*1000L;
   public static final String DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY =
       "dfs.journalnode.edit-cache-size.bytes";
-  public static final int DFS_JOURNALNODE_EDIT_CACHE_SIZE_DEFAULT = 1024 * 1024;
+
+  public static final String DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY =
+          "dfs.journalnode.edit-cache-size.fraction";
+  public static final float DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_DEFAULT = 0.5f;
 
   // Journal-node related configs for the client side.
   public static final String  DFS_QJOURNAL_QUEUE_SIZE_LIMIT_KEY = "dfs.qjournal.queued-edits.limit.mb";
@@ -1506,6 +1555,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String[] NNTOP_WINDOWS_MINUTES_DEFAULT = {"1", "5", "25"};
   public static final String DFS_PIPELINE_ECN_ENABLED = "dfs.pipeline.ecn";
   public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false;
+  public static final String DFS_PIPELINE_SLOWNODE_ENABLED = "dfs.pipeline.slownode";
+  public static final boolean DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT = false;
 
   // Key Provider Cache Expiry
   public static final String DFS_DATANODE_BLOCK_PINNING_ENABLED =
@@ -1632,6 +1683,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
       DFS_NAMESERVICES_RESOLVER_IMPL =
       "dfs.datanode.nameservices.resolver.impl";
 
+  public static final String
+      DFS_DATANODE_LOCKMANAGER_TRACE =
+      "dfs.datanode.lockmanager.trace";
+
+  public static final boolean
+      DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT = false;
+
   // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
   @Deprecated
   public static final String  DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 968bad3bbe5e8..7237489e7bfef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -76,7 +76,6 @@
 import org.apache.hadoop.net.DomainNameResolverFactory;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -110,7 +109,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.BlockingService;
 
 @InterfaceAudience.Private
@@ -310,7 +309,11 @@ public static String byteArray2PathString(final byte[][] components,
     // specifically not using StringBuilder to more efficiently build
     // string w/o excessive byte[] copies and charset conversions.
     final int range = offset + length;
-    Preconditions.checkPositionIndexes(offset, range, components.length);
+    if (offset < 0 || range < offset || range > components.length) {
+      throw new IndexOutOfBoundsException(
+          "Incorrect index [offset, range, size] ["
+              + offset + ", " + range + ", " + components.length + "]");
+    }
     if (length == 0) {
       return "";
     }
@@ -721,8 +724,9 @@ private static Collection<String> getParentNameServices(Configuration conf)
     } else {
       // Ensure that the internal service is indeed in the list of all available
       // nameservices.
-      Set<String> availableNameServices = Sets.newHashSet(conf
-          .getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES));
+      Collection<String> namespaces = conf
+          .getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
+      Set<String> availableNameServices = new HashSet<>(namespaces);
       for (String nsId : parentNameServices) {
         if (!availableNameServices.contains(nsId)) {
           throw new IOException("Unknown nameservice: " + nsId);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
index 58034ddad4705..e1c2fcec7f285 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
@@ -57,7 +57,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Lists;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
index 35d81201ad1d4..77e610434b49b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.net;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
index 0d0d209d977f3..5f23a5b6b3e48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.net;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.net.InnerNode;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index eb7caead4bc0e..3e30a6309d903 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.hdfs.server.datanode.Replica;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 import org.apache.hadoop.thirdparty.protobuf.CodedInputStream;
 import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
index aef009a03a1dc..d03ad3d4d7ce6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-
 import java.util.Date;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -28,7 +26,7 @@
 import org.apache.hadoop.util.IntrusiveCollection;
 import org.apache.hadoop.util.IntrusiveCollection.Element;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Namenode class that tracks state related to a cached path.
@@ -63,7 +61,7 @@ public CacheDirective(long id, String path,
       short replication, long expiryTime) {
     Preconditions.checkArgument(id > 0);
     this.id = id;
-    this.path = checkNotNull(path);
+    this.path = Preconditions.checkNotNull(path);
     Preconditions.checkArgument(replication > 0);
     this.replication = replication;
     this.expiryTime = expiryTime;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
index ef547788f1ac7..626bf2678883c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 
 /**
  * SnapshotInfo maintains information for a snapshot
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index ab767e3b93810..8bcfb199ff5a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BaseHeaderProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.CachingStrategyProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ClientOperationHeaderProto;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DataTransferTraceInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockGroupChecksumProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpCopyBlockProto;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
index 1e10fbbd3dd03..fcb6b7d7bc575 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
@@ -52,10 +52,12 @@
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DataTransferEncryptorMessageProto.DataTransferEncryptorStatus;
 import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
 import org.apache.hadoop.hdfs.server.datanode.DNConf;
 import org.apache.hadoop.security.SaslPropertiesResolver;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
 import org.apache.hadoop.util.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -441,6 +443,14 @@ private IOStreamPair doSaslHandshake(Peer peer, OutputStream underlyingOut,
         // error, the client will get a new encryption key from the NN and retry
         // connecting to this DN.
         sendInvalidKeySaslErrorMessage(out, ioe.getCause().getMessage());
+      } else if (ioe instanceof SaslException &&
+          ioe.getCause() != null &&
+          (ioe.getCause() instanceof InvalidBlockTokenException ||
+              ioe.getCause() instanceof SecretManager.InvalidToken)) {
+        // This could be because the client is long-lived and block token is expired
+        // The client will get new block token from the NN, upon receiving this error
+        // and retry connecting to this DN
+        sendInvalidTokenSaslErrorMessage(out, ioe.getCause().getMessage());
       } else {
         sendGenericSaslErrorMessage(out, ioe.getMessage());
       }
@@ -460,4 +470,16 @@ private static void sendInvalidKeySaslErrorMessage(DataOutputStream out,
     sendSaslMessage(out, DataTransferEncryptorStatus.ERROR_UNKNOWN_KEY, null,
         message);
   }
+
+  /**
+   * Sends a SASL negotiation message indicating an invalid token error.
+   *
+   * @param out     stream to receive message
+   * @param message to send
+   * @throws IOException for any error
+   */
+  private static void sendInvalidTokenSaslErrorMessage(DataOutputStream out,
+      String message) throws IOException {
+    sendSaslMessage(out, DataTransferEncryptorStatus.ERROR, null, message, null, true);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 5132afaa4b15c..79c122cf5bae0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -156,6 +156,8 @@
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSlowDatanodeReportResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetSnapshotDiffReportListingRequestProto;
@@ -816,7 +818,7 @@ public GetBatchedListingResponseProto getBatchedListing(
   public RenewLeaseResponseProto renewLease(RpcController controller,
       RenewLeaseRequestProto req) throws ServiceException {
     try {
-      server.renewLease(req.getClientName());
+      server.renewLease(req.getClientName(), req.getNamespacesList());
       return VOID_RENEWLEASE_RESPONSE;
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -2058,4 +2060,18 @@ public HAServiceStateResponseProto getHAServiceState(
       throw new ServiceException(e);
     }
   }
+
+  @Override
+  public GetSlowDatanodeReportResponseProto getSlowDatanodeReport(RpcController controller,
+      GetSlowDatanodeReportRequestProto request) throws ServiceException {
+    try {
+      List<? extends DatanodeInfoProto> result =
+          PBHelperClient.convert(server.getSlowDatanodeReport());
+      return GetSlowDatanodeReportResponseProto.newBuilder()
+          .addAllDatanodeInfoProto(result)
+          .build();
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
index 695a945ec712c..fd58c0c7ca289 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
@@ -183,7 +183,8 @@ public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration,
       rollingUpdateStatus = PBHelperClient.convert(resp.getRollingUpgradeStatus());
     }
     return new HeartbeatResponse(cmds, PBHelper.convert(resp.getHaStatus()),
-        rollingUpdateStatus, resp.getFullBlockReportLeaseId());
+        rollingUpdateStatus, resp.getFullBlockReportLeaseId(),
+        resp.getIsSlownode());
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index 9244b9fef8571..08a777b9a5181 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -61,7 +61,7 @@
 import org.apache.hadoop.hdfs.server.protocol.StorageReport;
 import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.RpcController;
 import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 
@@ -152,6 +152,7 @@ public HeartbeatResponseProto sendHeartbeat(RpcController controller,
     }
 
     builder.setFullBlockReportLeaseId(response.getFullBlockReportLeaseId());
+    builder.setIsSlownode(response.getIsSlownode());
     return builder.build();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 690ad0c279019..b5f7b9c80f25f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -112,6 +112,7 @@
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
 import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
 import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
@@ -853,11 +854,15 @@ public static List<SlowPeerReportProto> convertSlowPeerInfo(
 
     List<SlowPeerReportProto> slowPeerInfoProtos =
         new ArrayList<>(slowPeers.getSlowPeers().size());
-    for (Map.Entry<String, Double> entry :
-        slowPeers.getSlowPeers().entrySet()) {
-      slowPeerInfoProtos.add(SlowPeerReportProto.newBuilder()
+    for (Map.Entry<String, OutlierMetrics> entry : slowPeers.getSlowPeers().entrySet()) {
+      OutlierMetrics outlierMetrics = entry.getValue();
+      slowPeerInfoProtos.add(
+          SlowPeerReportProto.newBuilder()
               .setDataNodeId(entry.getKey())
-              .setAggregateLatency(entry.getValue())
+              .setAggregateLatency(outlierMetrics.getActualLatency())
+              .setMedian(outlierMetrics.getMedian())
+              .setMad(outlierMetrics.getMad())
+              .setUpperLimitLatency(outlierMetrics.getUpperLimitLatency())
               .build());
     }
     return slowPeerInfoProtos;
@@ -871,15 +876,19 @@ public static SlowPeerReports convertSlowPeerInfo(
       return SlowPeerReports.EMPTY_REPORT;
     }
 
-    Map<String, Double> slowPeersMap = new HashMap<>(slowPeerProtos.size());
+    Map<String, OutlierMetrics> slowPeersMap = new HashMap<>(slowPeerProtos.size());
     for (SlowPeerReportProto proto : slowPeerProtos) {
       if (!proto.hasDataNodeId()) {
         // The DataNodeId should be reported.
         continue;
       }
-      slowPeersMap.put(
-          proto.getDataNodeId(),
-          proto.hasAggregateLatency() ? proto.getAggregateLatency() : 0.0);
+      Double aggregateLatency = proto.hasAggregateLatency() ? proto.getAggregateLatency() : 0.0;
+      Double medianLatency = proto.hasMedian() ? proto.getMedian() : 0.0;
+      Double madLatency = proto.hasMad() ? proto.getMad() : 0.0;
+      Double upperLimitLatency = proto.hasUpperLimitLatency() ? proto.getUpperLimitLatency() : 0.0;
+      OutlierMetrics outlierMetrics =
+          new OutlierMetrics(medianLatency, madLatency, upperLimitLatency, aggregateLatency);
+      slowPeersMap.put(proto.getDataNodeId(), outlierMetrics);
     }
     return SlowPeerReports.create(slowPeersMap);
   }
@@ -1040,11 +1049,17 @@ public static BlockECReconstructionInfo convertBlockECReconstructionInfo(
 
     byte[] liveBlkIndices = blockEcReconstructionInfoProto.getLiveBlockIndices()
         .toByteArray();
+    byte[] excludeReconstructedIndices =
+        blockEcReconstructionInfoProto.hasExcludeReconstructedIndices() ?
+            blockEcReconstructionInfoProto.getExcludeReconstructedIndices()
+                .toByteArray() : new byte[0];
     ErasureCodingPolicy ecPolicy =
         PBHelperClient.convertErasureCodingPolicy(
             blockEcReconstructionInfoProto.getEcPolicy());
-    return new BlockECReconstructionInfo(block, sourceDnInfos, targetDnInfos,
-        targetStorageUuids, convertStorageTypes, liveBlkIndices, ecPolicy);
+    return new BlockECReconstructionInfo(
+        block, sourceDnInfos, targetDnInfos,
+        targetStorageUuids, convertStorageTypes, liveBlkIndices,
+        excludeReconstructedIndices, ecPolicy);
   }
 
   public static BlockECReconstructionInfoProto convertBlockECRecoveryInfo(
@@ -1070,6 +1085,10 @@ public static BlockECReconstructionInfoProto convertBlockECRecoveryInfo(
     byte[] liveBlockIndices = blockEcRecoveryInfo.getLiveBlockIndices();
     builder.setLiveBlockIndices(PBHelperClient.getByteString(liveBlockIndices));
 
+    byte[] excludeReconstructedIndices = blockEcRecoveryInfo.getExcludeReconstructedIndices();
+    builder.setExcludeReconstructedIndices(
+        PBHelperClient.getByteString(excludeReconstructedIndices));
+
     builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(
         blockEcRecoveryInfo.getErasureCodingPolicy()));
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
index c1771a68c07cd..a65120e36102c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
@@ -36,7 +36,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
@@ -87,7 +87,7 @@ boolean isEpochEstablished() {
   
   /**
    * @return the epoch number for this writer. This may only be called after
-   * a successful call to {@link #createNewUniqueEpoch(NamespaceInfo)}.
+   * a successful call to {@link QuorumJournalManager#createNewUniqueEpoch()}.
    */
   long getEpoch() {
     Preconditions.checkState(myEpoch != INVALID_EPOCH,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index 4688f8db9c74f..4b7e59c51f13e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -23,7 +23,6 @@
 import java.net.URI;
 import java.net.URL;
 import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -58,7 +57,7 @@
 import org.apache.hadoop.util.StopWatch;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
@@ -67,6 +66,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncaughtExceptionHandlers;
+import org.apache.hadoop.util.Time;
 
 /**
  * Channel to a remote JournalNode using Hadoop IPC.
@@ -154,26 +154,15 @@ public class IPCLoggerChannel implements AsyncLogger {
 
   private static final long WARN_JOURNAL_MILLIS_THRESHOLD = 1000;
   
-  static final Factory FACTORY = new AsyncLogger.Factory() {
-    @Override
-    public AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo,
-        String journalId, String nameServiceId, InetSocketAddress addr) {
-      return new IPCLoggerChannel(conf, nsInfo, journalId, nameServiceId, addr);
-    }
-  };
+  static final Factory FACTORY = IPCLoggerChannel::new;
 
-  public IPCLoggerChannel(Configuration conf,
-      NamespaceInfo nsInfo,
-      String journalId,
-      InetSocketAddress addr) {
+  public IPCLoggerChannel(Configuration conf, NamespaceInfo nsInfo,
+      String journalId, InetSocketAddress addr) {
     this(conf, nsInfo, journalId, null, addr);
   }
 
-  public IPCLoggerChannel(Configuration conf,
-                          NamespaceInfo nsInfo,
-                          String journalId,
-                          String nameServiceId,
-                          InetSocketAddress addr) {
+  public IPCLoggerChannel(Configuration conf, NamespaceInfo nsInfo,
+      String journalId, String nameServiceId, InetSocketAddress addr) {
     this.conf = conf;
     this.nsInfo = nsInfo;
     this.journalId = journalId;
@@ -202,7 +191,7 @@ public synchronized void setCommittedTxId(long txid) {
         "Trying to move committed txid backwards in client " +
          "old: %s new: %s", committedTxId, txid);
     this.committedTxId = txid;
-    this.lastCommitNanos = System.nanoTime();
+    this.lastCommitNanos = Time.monotonicNowNanos();
   }
   
   @Override
@@ -229,25 +218,19 @@ protected QJournalProtocol createProxy() throws IOException {
     final Configuration confCopy = new Configuration(conf);
     
     // Need to set NODELAY or else batches larger than MTU can trigger 
-    // 40ms nagling delays.
-    confCopy.setBoolean(
-        CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY,
-        true);
-    
+    // 40ms nailing delays.
+    confCopy.setBoolean(CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY, true);
     RPC.setProtocolEngine(confCopy,
         QJournalProtocolPB.class, ProtobufRpcEngine2.class);
     return SecurityUtil.doAsLoginUser(
-        new PrivilegedExceptionAction<QJournalProtocol>() {
-          @Override
-          public QJournalProtocol run() throws IOException {
-            RPC.setProtocolEngine(confCopy,
-                QJournalProtocolPB.class, ProtobufRpcEngine2.class);
-            QJournalProtocolPB pbproxy = RPC.getProxy(
-                QJournalProtocolPB.class,
-                RPC.getProtocolVersion(QJournalProtocolPB.class),
-                addr, confCopy);
-            return new QJournalProtocolTranslatorPB(pbproxy);
-          }
+        (PrivilegedExceptionAction<QJournalProtocol>) () -> {
+          RPC.setProtocolEngine(confCopy,
+              QJournalProtocolPB.class, ProtobufRpcEngine2.class);
+          QJournalProtocolPB pbproxy = RPC.getProxy(
+              QJournalProtocolPB.class,
+              RPC.getProtocolVersion(QJournalProtocolPB.class),
+              addr, confCopy);
+          return new QJournalProtocolTranslatorPB(pbproxy);
         });
   }
   
@@ -260,10 +243,8 @@ protected ExecutorService createSingleThreadExecutor() {
     return Executors.newSingleThreadExecutor(
         new ThreadFactoryBuilder()
           .setDaemon(true)
-          .setNameFormat("Logger channel (from single-thread executor) to " +
-              addr)
-          .setUncaughtExceptionHandler(
-              UncaughtExceptionHandlers.systemExit())
+          .setNameFormat("Logger channel (from single-thread executor) to " + addr)
+          .setUncaughtExceptionHandler(UncaughtExceptionHandlers.systemExit())
           .build());
   }
 
@@ -308,11 +289,6 @@ private synchronized RequestInfo createReqInfo() {
         epoch, ipcSerial++, committedTxId);
   }
 
-  @VisibleForTesting
-  synchronized long getNextIpcSerial() {
-    return ipcSerial;
-  }
-
   public synchronized int getQueuedEditsSize() {
     return queuedEditsSizeBytes;
   }
@@ -333,11 +309,7 @@ public synchronized boolean isOutOfSync() {
   @VisibleForTesting
   void waitForAllPendingCalls() throws InterruptedException {
     try {
-      singleThreadExecutor.submit(new Runnable() {
-        @Override
-        public void run() {
-        }
-      }).get();
+      singleThreadExecutor.submit(() -> {}).get();
     } catch (ExecutionException e) {
       // This can't happen!
       throw new AssertionError(e);
@@ -346,36 +318,23 @@ public void run() {
 
   @Override
   public ListenableFuture<Boolean> isFormatted() {
-    return singleThreadExecutor.submit(new Callable<Boolean>() {
-      @Override
-      public Boolean call() throws IOException {
-        return getProxy().isFormatted(journalId, nameServiceId);
-      }
-    });
+    return singleThreadExecutor.submit(() -> getProxy().isFormatted(journalId, nameServiceId));
   }
 
   @Override
   public ListenableFuture<GetJournalStateResponseProto> getJournalState() {
-    return singleThreadExecutor.submit(new Callable<GetJournalStateResponseProto>() {
-      @Override
-      public GetJournalStateResponseProto call() throws IOException {
-        GetJournalStateResponseProto ret =
-            getProxy().getJournalState(journalId, nameServiceId);
-        constructHttpServerURI(ret);
-        return ret;
-      }
+    return singleThreadExecutor.submit(() -> {
+      GetJournalStateResponseProto ret = getProxy().getJournalState(journalId, nameServiceId);
+      constructHttpServerURI(ret);
+      return ret;
     });
   }
 
   @Override
   public ListenableFuture<NewEpochResponseProto> newEpoch(
       final long epoch) {
-    return singleThreadExecutor.submit(new Callable<NewEpochResponseProto>() {
-      @Override
-      public NewEpochResponseProto call() throws IOException {
-        return getProxy().newEpoch(journalId, nameServiceId, nsInfo, epoch);
-      }
-    });
+    return singleThreadExecutor.submit(
+        () -> getProxy().newEpoch(journalId, nameServiceId, nsInfo, epoch));
   }
   
   @Override
@@ -390,50 +349,43 @@ public ListenableFuture<Void> sendEdits(
     
     // When this batch is acked, we use its submission time in order
     // to calculate how far we are lagging.
-    final long submitNanos = System.nanoTime();
+    final long submitNanos = Time.monotonicNowNanos();
     
     ListenableFuture<Void> ret = null;
     try {
-      ret = singleThreadExecutor.submit(new Callable<Void>() {
-        @Override
-        public Void call() throws IOException {
-          throwIfOutOfSync();
-
-          long rpcSendTimeNanos = System.nanoTime();
-          try {
-            getProxy().journal(createReqInfo(),
-                segmentTxId, firstTxnId, numTxns, data);
-          } catch (IOException e) {
-            QuorumJournalManager.LOG.warn(
-                "Remote journal " + IPCLoggerChannel.this + " failed to " +
-                "write txns " + firstTxnId + "-" + (firstTxnId + numTxns - 1) +
-                ". Will try to write to this JN again after the next " +
-                "log roll.", e); 
-            synchronized (IPCLoggerChannel.this) {
-              outOfSync = true;
-            }
-            throw e;
-          } finally {
-            long now = System.nanoTime();
-            long rpcTime = TimeUnit.MICROSECONDS.convert(
-                now - rpcSendTimeNanos, TimeUnit.NANOSECONDS);
-            long endToEndTime = TimeUnit.MICROSECONDS.convert(
-                now - submitNanos, TimeUnit.NANOSECONDS);
-            metrics.addWriteEndToEndLatency(endToEndTime);
-            metrics.addWriteRpcLatency(rpcTime);
-            if (rpcTime / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) {
-              QuorumJournalManager.LOG.warn(
-                  "Took " + (rpcTime / 1000) + "ms to send a batch of " +
-                  numTxns + " edits (" + data.length + " bytes) to " +
-                  "remote journal " + IPCLoggerChannel.this);
-            }
-          }
+      ret = singleThreadExecutor.submit(() -> {
+        throwIfOutOfSync();
+
+        final long rpcSendTimeNanos = Time.monotonicNowNanos();
+        try {
+          getProxy().journal(createReqInfo(), segmentTxId, firstTxnId, numTxns, data);
+        } catch (IOException e) {
+          QuorumJournalManager.LOG.warn("Remote journal {} failed to write txns {}-{}."
+                  + " Will try to write to this JN again after the next log roll.",
+              IPCLoggerChannel.this, firstTxnId, (firstTxnId + numTxns - 1), e);
           synchronized (IPCLoggerChannel.this) {
-            highestAckedTxId = firstTxnId + numTxns - 1;
-            lastAckNanos = submitNanos;
+            outOfSync = true;
           }
-          return null;
+          throw e;
+        } finally {
+          final long nowNanos = Time.monotonicNowNanos();
+          final long rpcTimeMicros = TimeUnit.MICROSECONDS.convert(
+              (nowNanos - rpcSendTimeNanos), TimeUnit.NANOSECONDS);
+          final long endToEndTimeMicros = TimeUnit.MICROSECONDS.convert(
+              (nowNanos - submitNanos), TimeUnit.NANOSECONDS);
+          metrics.addWriteEndToEndLatency(endToEndTimeMicros);
+          metrics.addWriteRpcLatency(rpcTimeMicros);
+          if (rpcTimeMicros / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) {
+            QuorumJournalManager.LOG.warn(
+                "Took {}ms to send a batch of {} edits ({} bytes) to remote journal {}.",
+                rpcTimeMicros / 1000, numTxns, data.length, IPCLoggerChannel.this);
+          }
+        }
+        synchronized (IPCLoggerChannel.this) {
+          highestAckedTxId = firstTxnId + numTxns - 1;
+          lastAckNanos = submitNanos;
         }
+        return null;
       });
     } finally {
       if (ret == null) {
@@ -460,14 +412,12 @@ public void onSuccess(Void t) {
     return ret;
   }
 
-  private void throwIfOutOfSync()
-      throws JournalOutOfSyncException, IOException {
+  private void throwIfOutOfSync() throws IOException {
     if (isOutOfSync()) {
       // Even if we're out of sync, it's useful to send an RPC
       // to the remote node in order to update its lag metrics, etc.
       heartbeatIfNecessary();
-      throw new JournalOutOfSyncException(
-          "Journal disabled until next roll");
+      throw new JournalOutOfSyncException("Journal disabled until next roll");
     }
   }
 
@@ -497,12 +447,10 @@ private void heartbeatIfNecessary() throws IOException {
   private synchronized void reserveQueueSpace(int size)
       throws LoggerTooFarBehindException {
     Preconditions.checkArgument(size >= 0);
-    if (queuedEditsSizeBytes + size > queueSizeLimitBytes &&
-        queuedEditsSizeBytes > 0) {
-      QuorumJournalManager.LOG.warn("Pending edits to " + IPCLoggerChannel.this
-          + " is going to exceed limit size: " + queueSizeLimitBytes
-          + ", current queued edits size: " + queuedEditsSizeBytes
-          + ", will silently drop " + size + " bytes of edits!");
+    if (queuedEditsSizeBytes + size > queueSizeLimitBytes && queuedEditsSizeBytes > 0) {
+      QuorumJournalManager.LOG.warn("Pending edits to {} is going to exceed limit size: {}"
+          + ", current queued edits size: {}, will silently drop {} bytes of edits!",
+          IPCLoggerChannel.class, queueSizeLimitBytes, queuedEditsSizeBytes, size);
       throw new LoggerTooFarBehindException();
     }
     queuedEditsSizeBytes += size;
@@ -514,203 +462,144 @@ private synchronized void unreserveQueueSpace(int size) {
   }
 
   @Override
-  public ListenableFuture<Void> format(final NamespaceInfo nsInfo,
-      final boolean force) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws Exception {
-        getProxy().format(journalId, nameServiceId, nsInfo, force);
-        return null;
-      }
+  public ListenableFuture<Void> format(final NamespaceInfo nsInfo, final boolean force) {
+    return singleThreadExecutor.submit(() -> {
+      getProxy().format(journalId, nameServiceId, nsInfo, force);
+      return null;
     });
   }
   
   @Override
-  public ListenableFuture<Void> startLogSegment(final long txid,
-      final int layoutVersion) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().startLogSegment(createReqInfo(), txid, layoutVersion);
-        synchronized (IPCLoggerChannel.this) {
-          if (outOfSync) {
-            outOfSync = false;
-            QuorumJournalManager.LOG.info(
-                "Restarting previously-stopped writes to " +
-                IPCLoggerChannel.this + " in segment starting at txid " +
-                txid);
-          }
+  public ListenableFuture<Void> startLogSegment(final long txid, final int layoutVersion) {
+    return singleThreadExecutor.submit(() -> {
+      getProxy().startLogSegment(createReqInfo(), txid, layoutVersion);
+      synchronized (IPCLoggerChannel.this) {
+        if (outOfSync) {
+          outOfSync = false;
+          QuorumJournalManager.LOG.info(
+              "Restarting previously-stopped writes to {} in segment starting at txid {}.",
+                  IPCLoggerChannel.class, txid);
         }
-        return null;
       }
+      return null;
     });
   }
   
   @Override
-  public ListenableFuture<Void> finalizeLogSegment(
-      final long startTxId, final long endTxId) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        throwIfOutOfSync();
-        
-        getProxy().finalizeLogSegment(createReqInfo(), startTxId, endTxId);
-        return null;
-      }
+  public ListenableFuture<Void> finalizeLogSegment(final long startTxId, final long endTxId) {
+    return singleThreadExecutor.submit(() -> {
+      throwIfOutOfSync();
+      getProxy().finalizeLogSegment(createReqInfo(), startTxId, endTxId);
+      return null;
     });
   }
   
   @Override
   public ListenableFuture<Void> purgeLogsOlderThan(final long minTxIdToKeep) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws Exception {
-        getProxy().purgeLogsOlderThan(createReqInfo(), minTxIdToKeep);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().purgeLogsOlderThan(createReqInfo(), minTxIdToKeep);
+      return null;
     });
   }
 
   @Override
   public ListenableFuture<GetJournaledEditsResponseProto> getJournaledEdits(
       long fromTxnId, int maxTransactions) {
-    return parallelExecutor.submit(
-        new Callable<GetJournaledEditsResponseProto>() {
-          @Override
-          public GetJournaledEditsResponseProto call() throws IOException {
-            return getProxy().getJournaledEdits(journalId, nameServiceId,
-                fromTxnId, maxTransactions);
-          }
-        });
+    return parallelExecutor.submit(() -> getProxy().getJournaledEdits(
+        journalId, nameServiceId, fromTxnId, maxTransactions));
   }
 
   @Override
   public ListenableFuture<RemoteEditLogManifest> getEditLogManifest(
       final long fromTxnId, final boolean inProgressOk) {
-    return parallelExecutor.submit(new Callable<RemoteEditLogManifest>() {
-      @Override
-      public RemoteEditLogManifest call() throws IOException {
-        GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest(
-            journalId, nameServiceId, fromTxnId, inProgressOk);
-        // Update the http port, since we need this to build URLs to any of the
-        // returned logs.
-        constructHttpServerURI(ret);
-        return PBHelper.convert(ret.getManifest());
-      }
+    return parallelExecutor.submit(() -> {
+      GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest(
+          journalId, nameServiceId, fromTxnId, inProgressOk);
+      // Update the http port, since we need this to build URLs to any of the
+      // returned logs.
+      constructHttpServerURI(ret);
+      return PBHelper.convert(ret.getManifest());
     });
   }
 
   @Override
-  public ListenableFuture<PrepareRecoveryResponseProto> prepareRecovery(
-      final long segmentTxId) {
-    return singleThreadExecutor.submit(new Callable<PrepareRecoveryResponseProto>() {
-      @Override
-      public PrepareRecoveryResponseProto call() throws IOException {
-        if (!hasHttpServerEndPoint()) {
-          // force an RPC call so we know what the HTTP port should be if it
-          // haven't done so.
-          GetJournalStateResponseProto ret = getProxy().getJournalState(
-              journalId, nameServiceId);
-          constructHttpServerURI(ret);
-        }
-        return getProxy().prepareRecovery(createReqInfo(), segmentTxId);
+  public ListenableFuture<PrepareRecoveryResponseProto> prepareRecovery(final long segmentTxId) {
+    return singleThreadExecutor.submit(() -> {
+      if (!hasHttpServerEndPoint()) {
+        // force an RPC call, so we know what the HTTP port should be if it
+        // hasn't done so.
+        GetJournalStateResponseProto ret = getProxy().getJournalState(
+            journalId, nameServiceId);
+        constructHttpServerURI(ret);
       }
+      return getProxy().prepareRecovery(createReqInfo(), segmentTxId);
     });
   }
 
   @Override
-  public ListenableFuture<Void> acceptRecovery(
-      final SegmentStateProto log, final URL url) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().acceptRecovery(createReqInfo(), log, url);
-        return null;
-      }
+  public ListenableFuture<Void> acceptRecovery(final SegmentStateProto log, final URL url) {
+    return singleThreadExecutor.submit(() -> {
+      getProxy().acceptRecovery(createReqInfo(), log, url);
+      return null;
     });
   }
   
   @Override
   public ListenableFuture<Void> doPreUpgrade() {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().doPreUpgrade(journalId);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().doPreUpgrade(journalId);
+      return null;
     });
   }
   
   @Override
   public ListenableFuture<Void> doUpgrade(final StorageInfo sInfo) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().doUpgrade(journalId, sInfo);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().doUpgrade(journalId, sInfo);
+      return null;
     });
   }
   
   @Override
   public ListenableFuture<Void> doFinalize() {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().doFinalize(journalId, nameServiceId);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().doFinalize(journalId, nameServiceId);
+      return null;
     });
   }
   
   @Override
   public ListenableFuture<Boolean> canRollBack(final StorageInfo storage,
       final StorageInfo prevStorage, final int targetLayoutVersion) {
-    return singleThreadExecutor.submit(new Callable<Boolean>() {
-      @Override
-      public Boolean call() throws IOException {
-        return getProxy().canRollBack(journalId, nameServiceId,
-            storage, prevStorage, targetLayoutVersion);
-      }
-    });
+    return singleThreadExecutor.submit(
+        () -> getProxy().canRollBack(journalId, nameServiceId,
+            storage, prevStorage, targetLayoutVersion));
   }
 
   @Override
   public ListenableFuture<Void> doRollback() {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().doRollback(journalId, nameServiceId);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().doRollback(journalId, nameServiceId);
+      return null;
     });
   }
 
   @Override
   public ListenableFuture<Void> discardSegments(final long startTxId) {
-    return singleThreadExecutor.submit(new Callable<Void>() {
-      @Override
-      public Void call() throws IOException {
-        getProxy().discardSegments(journalId, nameServiceId, startTxId);
-        return null;
-      }
+    return singleThreadExecutor.submit(() -> {
+      getProxy().discardSegments(journalId, nameServiceId, startTxId);
+      return null;
     });
   }
 
   @Override
   public ListenableFuture<Long> getJournalCTime() {
-    return singleThreadExecutor.submit(new Callable<Long>() {
-      @Override
-      public Long call() throws IOException {
-        return getProxy().getJournalCTime(journalId, nameServiceId);
-      }
-    });
+    return singleThreadExecutor.submit(() -> getProxy().getJournalCTime(journalId, nameServiceId));
   }
 
   @Override
   public String toString() {
-    return InetAddresses.toAddrString(addr.getAddress()) + ':' +
-        addr.getPort();
+    return InetAddresses.toAddrString(addr.getAddress()) + ':' + addr.getPort();
   }
 
   @Override
@@ -745,7 +634,7 @@ private void constructHttpServerURI(GetEditLogManifestResponseProto ret) {
       URI uri = URI.create(ret.getFromURL());
       httpServerURL = getHttpServerURI(uri.getScheme(), uri.getPort());
     } else {
-      httpServerURL = getHttpServerURI("http", ret.getHttpPort());;
+      httpServerURL = getHttpServerURI("http", ret.getHttpPort());
     }
   }
 
@@ -754,7 +643,7 @@ private void constructHttpServerURI(GetJournalStateResponseProto ret) {
       URI uri = URI.create(ret.getFromURL());
       httpServerURL = getHttpServerURI(uri.getScheme(), uri.getPort());
     } else {
-      httpServerURL = getHttpServerURI("http", ret.getHttpPort());;
+      httpServerURL = getHttpServerURI("http", ret.getHttpPort());
     }
   }
 
@@ -778,5 +667,4 @@ private URL getHttpServerURI(String scheme, int port) {
   private boolean hasHttpServerEndPoint() {
    return httpServerURL != null;
   }
-
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java
index e2a169aeb3c5f..c3ad872f30d0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.util.Timer;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java
index 1f60e3d468821..65b227ccea8d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Exception thrown when too many exceptions occur while gathering
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 4f8893952bd61..faf71a7b545d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -61,7 +61,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 /**
@@ -479,10 +479,9 @@ public void recoverUnfinalizedSegments() throws IOException {
     LOG.info("Successfully started new epoch " + loggers.getEpoch());
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("newEpoch(" + loggers.getEpoch() + ") responses:\n" +
-        QuorumCall.mapToString(resps));
+      LOG.debug("newEpoch({}) responses:\n{}", loggers.getEpoch(), QuorumCall.mapToString(resps));
     }
-    
+
     long mostRecentSegmentTxId = Long.MIN_VALUE;
     for (NewEpochResponseProto r : resps.values()) {
       if (r.hasLastSegmentTxId()) {
@@ -518,10 +517,7 @@ public void selectInputStreams(Collection<EditLogInputStream> streams,
     // the cache used for RPC calls is not enabled; fall back to using the
     // streaming mechanism to serve such requests
     if (inProgressOk && inProgressTailingEnabled) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Tailing edits starting from txn ID " + fromTxnId +
-            " via RPC mechanism");
-      }
+      LOG.debug("Tailing edits starting from txn ID {} via RPC mechanism", fromTxnId);
       try {
         Collection<EditLogInputStream> rpcStreams = new ArrayList<>();
         selectRpcInputStreams(rpcStreams, fromTxnId, onlyDurableTxns);
@@ -585,8 +581,8 @@ private void selectRpcInputStreams(Collection<EditLogInputStream> streams,
     int maxAllowedTxns = !onlyDurableTxns ? highestTxnCount :
         responseCounts.get(responseCounts.size() - loggers.getMajoritySize());
     if (maxAllowedTxns == 0) {
-      LOG.debug("No new edits available in logs; requested starting from " +
-          "ID {}", fromTxnId);
+      LOG.debug("No new edits available in logs; requested starting from ID {}",
+          fromTxnId);
       return;
     }
     LogAction logAction = selectInputStreamLogHelper.record(fromTxnId);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java
index 4b2a518ac0dff..38006a499d97e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Booleans;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
index 81b3f8c1a1f1f..f726ff8f84de6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java
@@ -27,11 +27,11 @@
 
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.text.StringEscapeUtils;
+import org.apache.hadoop.hdfs.server.namenode.DfsServlet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -64,7 +64,7 @@
  * </ul>
  */
 @InterfaceAudience.Private
-public class GetJournalEditServlet extends HttpServlet {
+public class GetJournalEditServlet extends DfsServlet {
 
   private static final long serialVersionUID = -4635891628211723009L;
   private static final Logger LOG =
@@ -77,17 +77,11 @@ public class GetJournalEditServlet extends HttpServlet {
 
   protected boolean isValidRequestor(HttpServletRequest request, Configuration conf)
       throws IOException {
-    String remotePrincipal = request.getUserPrincipal().getName();
-    String remoteShortName = request.getRemoteUser();
-    if (remotePrincipal == null) { // This really shouldn't happen...
-      LOG.warn("Received null remoteUser while authorizing access to " +
-          "GetJournalEditServlet");
-      return false;
-    }
+    UserGroupInformation ugi = getUGI(request, conf);
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Validating request made by " + remotePrincipal +
-          " / " + remoteShortName + ". This user is: " +
+      LOG.debug("Validating request made by " + ugi.getUserName() +
+          " / " + ugi.getShortUserName() + ". This user is: " +
           UserGroupInformation.getLoginUser());
     }
 
@@ -115,9 +109,9 @@ protected boolean isValidRequestor(HttpServletRequest request, Configuration con
     for (String v : validRequestors) {
       if (LOG.isDebugEnabled())
         LOG.debug("isValidRequestor is comparing to valid requestor: " + v);
-      if (v != null && v.equals(remotePrincipal)) {
+      if (v != null && v.equals(ugi.getUserName())) {
         if (LOG.isDebugEnabled())
-          LOG.debug("isValidRequestor is allowing: " + remotePrincipal);
+          LOG.debug("isValidRequestor is allowing: " + ugi.getUserName());
         return true;
       }
     }
@@ -125,16 +119,16 @@ protected boolean isValidRequestor(HttpServletRequest request, Configuration con
     // Additionally, we compare the short name of the requestor to this JN's
     // username, because we want to allow requests from other JNs during
     // recovery, but we can't enumerate the full list of JNs.
-    if (remoteShortName.equals(
+    if (ugi.getShortUserName().equals(
           UserGroupInformation.getLoginUser().getShortUserName())) {
       if (LOG.isDebugEnabled())
         LOG.debug("isValidRequestor is allowing other JN principal: " +
-            remotePrincipal);
+            ugi.getUserName());
       return true;
     }
 
     if (LOG.isDebugEnabled())
-      LOG.debug("isValidRequestor is rejecting: " + remotePrincipal);
+      LOG.debug("isValidRequestor is rejecting: " + ugi.getUserName());
     return false;
   }
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
index 7f82bff3376f0..78ef6acc9af4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JNStorage.java
@@ -123,10 +123,6 @@ File getSyncLogTemporaryFile(long segmentTxId, long epoch) {
     return new File(sd.getCurrentDir(), name);
   }
 
-  File getCurrentDir() {
-    return sd.getCurrentDir();
-  }
-
   /**
    * Directory {@code edits.sync} temporarily holds the log segments
    * downloaded through {@link JournalNodeSyncer} before they are moved to
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index 296d4af59a496..ffa613018c6a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -73,7 +73,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
@@ -750,10 +750,23 @@ public GetJournaledEditsResponseProto getJournaledEdits(long sinceTxId,
           "is a requirement to fetch journaled edits via RPC. Please enable " +
           "it via " + DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY);
     }
-    if (sinceTxId > getHighestWrittenTxId()) {
-      // Requested edits that don't exist yet; short-circuit the cache here
+    long highestTxId = getHighestWrittenTxId();
+    if (sinceTxId == highestTxId + 1) {
+      // Requested edits that don't exist yet, but this is expected,
+      // because namenode always get the journaled edits with the sinceTxId
+      // equal to image.getLastAppliedTxId() + 1. Short-circuiting the cache here
+      // and returning a response with a count of 0.
       metrics.rpcEmptyResponses.incr();
       return GetJournaledEditsResponseProto.newBuilder().setTxnCount(0).build();
+    } else if (sinceTxId > highestTxId + 1) {
+      // Requested edits that don't exist yet and this is unexpected. Means that there is a lag
+      // in this journal that does not contain some edits that should exist.
+      // Throw one NewerTxnIdException to make namenode treat this response as an exception.
+      // More detailed info please refer to: HDFS-16659 and HDFS-16771.
+      metrics.rpcEmptyResponses.incr();
+      throw new NewerTxnIdException(
+          "Highest txn ID available in the journal is %d, but requested txns starting at %d.",
+          highestTxId, sinceTxId);
     }
     try {
       List<ByteBuffer> buffers = new ArrayList<>();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
index a20cd218521f9..3e8831d8087c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.qjournal.server;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.util.Lists;
@@ -45,6 +45,7 @@
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HTTP_BIND_HOST_KEY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
+import static org.apache.hadoop.util.Time.now;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -83,6 +84,7 @@ public class JournalNode implements Tool, Configurable, JournalNodeMXBean {
   private String httpServerURI;
   private final ArrayList<File> localDir = Lists.newArrayList();
   Tracer tracer;
+  private long startTime = 0;
 
   static {
     HdfsConfiguration.init();
@@ -122,6 +124,11 @@ synchronized Journal getOrCreateJournal(String jid,
     return journal;
   }
 
+  @VisibleForTesting
+  public JournalNodeSyncer getJournalSyncer(String jid) {
+    return journalSyncersById.get(jid);
+  }
+
   @VisibleForTesting
   public boolean getJournalSyncerStatus(String jid) {
     if (journalSyncersById.get(jid) != null) {
@@ -241,6 +248,7 @@ public void start() throws IOException {
 
       rpcServer = new JournalNodeRpcServer(conf, this);
       rpcServer.start();
+      startTime = now();
     } catch (IOException ioe) {
       //Shutdown JournalNode of JournalNodeRpcServer fails to start
       LOG.error("Failed to start JournalNode.", ioe);
@@ -415,6 +423,19 @@ public String getVersion() {
     return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
   }
 
+  @Override // JournalNodeMXBean
+  public long getJNStartedTimeInMillis() {
+    return this.startTime;
+  }
+
+  @Override
+  // JournalNodeMXBean
+  public List<String> getStorageInfos() {
+    return journalsById.values().stream()
+        .map(journal -> journal.getStorage().toMapString())
+        .collect(Collectors.toList());
+  }
+
   /**
    * Register JournalNodeMXBean
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java
index f265c31a34aea..0cce05f5820d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeMXBean.java
@@ -57,4 +57,20 @@ public interface JournalNodeMXBean {
    * @return the version of Hadoop.
    */
   String getVersion();
+
+  /**
+   * Get the start time of the JournalNode.
+   *
+   * @return the start time of the JournalNode.
+   */
+  long getJNStartedTimeInMillis();
+
+  /**
+   * Get the list of the storage infos of JournalNode's journals. Storage infos
+   * include layout version, namespace id, cluster id and creation time of the
+   * File system state.
+   *
+   * @return the list of storage infos associated with journals.
+   */
+  List<String> getStorageInfos();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
index 46820d38662be..ab909aef2ecd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
@@ -55,6 +55,8 @@
 import java.net.InetSocketAddress;
 import java.net.URL;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_JOURNALNODE_RPC_BIND_HOST_KEY;
 
 
@@ -63,9 +65,9 @@
 public class JournalNodeRpcServer implements QJournalProtocol,
     InterQJournalProtocol {
   private static final Logger LOG = JournalNode.LOG;
-  private static final int HANDLER_COUNT = 5;
   private final JournalNode jn;
   private Server server;
+  private final int handlerCount;
 
   JournalNodeRpcServer(Configuration conf, JournalNode jn) throws IOException {
     this.jn = jn;
@@ -90,16 +92,30 @@ public class JournalNodeRpcServer implements QJournalProtocol,
         new QJournalProtocolServerSideTranslatorPB(this);
     BlockingService service = QJournalProtocolService
         .newReflectiveBlockingService(translator);
+    int confHandlerCount = conf.getInt(DFS_JOURNALNODE_HANDLER_COUNT_KEY,
+        DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT);
+    if (confHandlerCount <= 0) {
+      LOG.warn("Invalid value for: {} = {}, Should be > 0,"
+              + " will use default value of: {}.",
+          DFS_JOURNALNODE_HANDLER_COUNT_KEY, confHandlerCount,
+          DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT);
+      confHandlerCount = DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT;
+    }
+    this.handlerCount = confHandlerCount;
+    LOG.info("The number of JournalNodeRpcServer handlers is {}.",
+        this.handlerCount);
     
     this.server = new RPC.Builder(confCopy)
         .setProtocol(QJournalProtocolPB.class)
         .setInstance(service)
         .setBindAddress(bindHost)
         .setPort(addr.getPort())
-        .setNumHandlers(HANDLER_COUNT)
+        .setNumHandlers(this.handlerCount)
         .setVerbose(false)
         .build();
 
+    this.server.addTerseExceptions(NewerTxnIdException.class);
+    this.server.addTerseExceptions(JournaledEditsCache.CacheMissException.class);
 
     //Adding InterQJournalProtocolPB to server
     InterQJournalProtocolServerSideTranslatorPB
@@ -121,6 +137,11 @@ public class JournalNodeRpcServer implements QJournalProtocol,
     this.server.setTracer(jn.tracer);
   }
 
+  @VisibleForTesting
+  protected int getHandlerCount() {
+    return this.handlerCount;
+  }
+
   void start() {
     this.server.start();
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
index ff46aa751d7ba..f451b46de7b37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.qjournal.server;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -54,6 +55,7 @@
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * A Journal Sync thread runs through the lifetime of the JN. It periodically
@@ -153,6 +155,9 @@ private boolean getOtherJournalNodeProxies() {
         LOG.warn("Could not add proxy for Journal at addresss " + addr, e);
       }
     }
+    // Check if there are any other JournalNodes before starting the sync.  Although some proxies
+    // may be unresolved now, the act of attempting to sync will instigate resolution when the
+    // servers become available.
     if (otherJNProxies.isEmpty()) {
       LOG.error("Cannot sync as there is no other JN available for sync.");
       return false;
@@ -273,7 +278,7 @@ private List<InetSocketAddress> getOtherJournalNodeAddrs() {
       }
 
       if (uriStr == null || uriStr.isEmpty()) {
-        HashSet<String> sharedEditsUri = Sets.newHashSet();
+        HashSet<String> sharedEditsUri = new HashSet<>();
         if (nameServiceId != null) {
           Collection<String> nnIds = DFSUtilClient.getNameNodeIds(
               conf, nameServiceId);
@@ -310,12 +315,24 @@ private List<InetSocketAddress> getOtherJournalNodeAddrs() {
     return null;
   }
 
-  private List<InetSocketAddress> getJournalAddrList(String uriStr) throws
+  @VisibleForTesting
+  protected List<InetSocketAddress> getJournalAddrList(String uriStr) throws
       URISyntaxException,
       IOException {
     URI uri = new URI(uriStr);
-    return Util.getLoggerAddresses(uri,
-        Sets.newHashSet(jn.getBoundIpcAddress()), conf);
+
+    InetSocketAddress boundIpcAddress = jn.getBoundIpcAddress();
+    Set<InetSocketAddress> excluded = Sets.newHashSet(boundIpcAddress);
+    List<InetSocketAddress> addrList = Util.getLoggerAddresses(uri, excluded, conf);
+
+    // Exclude the current JournalNode instance (a local address and the same port).  If the address
+    // is bound to a local address on the same port, then remove it to handle scenarios where a
+    // wildcard address (e.g. "0.0.0.0") is used.   We can't simply exclude all local addresses
+    // since we may be running multiple servers on the same host.
+    addrList.removeIf(addr -> !addr.isUnresolved() &&  addr.getAddress().isAnyLocalAddress()
+          && boundIpcAddress.getPort() == addr.getPort());
+
+    return addrList;
   }
 
   private void getMissingLogSegments(List<RemoteEditLog> thisJournalEditLogs,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
index 65f54609ef3f5..339b7fa7b680c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
 import org.apache.hadoop.util.AutoCloseableLock;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * An in-memory cache of edits in their serialized form. This is used to serve
@@ -121,12 +122,18 @@ class JournaledEditsCache {
   // ** End lock-protected fields **
 
   JournaledEditsCache(Configuration conf) {
+    float fraction = conf.getFloat(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY,
+        DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_DEFAULT);
+    Preconditions.checkArgument((fraction > 0 && fraction < 1.0f),
+        String.format("Cache config %s is set at %f, it should be a positive float value, " +
+            "less than 1.0. The recommended value is less than 0.9.",
+            DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, fraction));
     capacity = conf.getInt(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
-        DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_DEFAULT);
+        (int) (Runtime.getRuntime().maxMemory() * fraction));
     if (capacity > 0.9 * Runtime.getRuntime().maxMemory()) {
       Journal.LOG.warn(String.format("Cache capacity is set at %d bytes but " +
           "maximum JVM memory is only %d bytes. It is recommended that you " +
-          "decrease the cache size or increase the heap size.",
+          "decrease the cache size/fraction or increase the heap size.",
           capacity, Runtime.getRuntime().maxMemory()));
     }
     Journal.LOG.info("Enabling the journaled edits cache with a capacity " +
@@ -277,11 +284,12 @@ void storeEdits(byte[] inputData, long newStartTxn, long newEndTxn,
         initialize(INVALID_TXN_ID);
         Journal.LOG.warn(String.format("A single batch of edits was too " +
                 "large to fit into the cache: startTxn = %d, endTxn = %d, " +
-                "input length = %d. The capacity of the cache (%s) must be " +
+                "input length = %d. The cache size (%s) or cache fraction (%s) must be " +
                 "increased for it to work properly (current capacity %d)." +
                 "Cache is now empty.",
             newStartTxn, newEndTxn, inputData.length,
-            DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity));
+            DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
+            DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, capacity));
         return;
       }
       if (dataMap.isEmpty()) {
@@ -388,10 +396,11 @@ private CacheMissException getCacheMissException(long requestedTxnId) {
     } else {
       return new CacheMissException(lowestTxnId - requestedTxnId,
           "Oldest txn ID available in the cache is %d, but requested txns " +
-              "starting at %d. The cache size (%s) may need to be increased " +
-              "to hold more transactions (currently %d bytes containing %d " +
+              "starting at %d. The cache size (%s) or cache fraction (%s) may need to be " +
+              "increased to hold more transactions (currently %d bytes containing %d " +
               "transactions)", lowestTxnId, requestedTxnId,
-          DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity,
+              DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
+              DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, capacity,
           highestTxnId - lowestTxnId + 1);
     }
   }
@@ -414,4 +423,9 @@ long getCacheMissAmount() {
 
   }
 
+  @VisibleForTesting
+  int getCapacity() {
+    return capacity;
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java
new file mode 100644
index 0000000000000..ec691402719d7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.qjournal.server;
+
+import java.io.IOException;
+
+/**
+ * Exception when no edits are available.
+ */
+public class NewerTxnIdException extends IOException {
+  private static final long serialVersionUID = 0L;
+
+  public NewerTxnIdException(String msgFormat, Object... msgArgs) {
+    super(String.format(msgFormat, msgArgs));
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
index 7517ebb3f1853..18cdeaebfda82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
@@ -46,7 +46,7 @@
 import org.apache.hadoop.util.Timer;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultiset;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Multiset;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
index aef2306a55c15..b89998d989586 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
@@ -50,7 +50,7 @@
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 
 /**
@@ -383,7 +383,7 @@ protected void logUpdateMasterKey(DelegationKey key)
           namesystem.logUpdateMasterKey(key);
         }
       } finally {
-        namesystem.readUnlock();
+        namesystem.readUnlock("logUpdateMasterKey");
       }
     } catch (InterruptedException ie) {
       // AbstractDelegationTokenManager may crash if an exception is thrown.
@@ -412,7 +412,7 @@ protected void logExpireToken(final DelegationTokenIdentifier dtId)
           namesystem.logExpireDelegationToken(dtId);
         }
       } finally {
-        namesystem.readUnlock();
+        namesystem.readUnlock("logExpireToken");
       }
     } catch (InterruptedException ie) {
       // AbstractDelegationTokenManager may crash if an exception is thrown.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 664ac62aeaef9..02004f337c10a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.balancer;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
 import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS;
 
 import java.io.IOException;
@@ -75,7 +74,7 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /** <p>The balancer is a tool that balances disk space usage on an HDFS cluster
  * when some datanodes become full or when new empty nodes join the cluster.
@@ -998,7 +997,7 @@ static BalancerParameters parse(String[] args) {
         try {
           for(int i = 0; i < args.length; i++) {
             if ("-threshold".equalsIgnoreCase(args[i])) {
-              checkArgument(++i < args.length,
+              Preconditions.checkArgument(++i < args.length,
                 "Threshold value is missing: args = " + Arrays.toString(args));
               try {
                 double threshold = Double.parseDouble(args[i]);
@@ -1015,7 +1014,7 @@ static BalancerParameters parse(String[] args) {
                 throw e;
               }
             } else if ("-policy".equalsIgnoreCase(args[i])) {
-              checkArgument(++i < args.length,
+              Preconditions.checkArgument(++i < args.length,
                 "Policy value is missing: args = " + Arrays.toString(args));
               try {
                 b.setBalancingPolicy(BalancingPolicy.parse(args[i]));
@@ -1036,7 +1035,7 @@ static BalancerParameters parse(String[] args) {
               i = processHostList(args, i, "source", sourceNodes);
               b.setSourceNodes(sourceNodes);
             } else if ("-blockpools".equalsIgnoreCase(args[i])) {
-              checkArgument(
+              Preconditions.checkArgument(
                   ++i < args.length,
                   "blockpools value is missing: args = "
                       + Arrays.toString(args));
@@ -1045,7 +1044,7 @@ static BalancerParameters parse(String[] args) {
                   + blockpools.toString());
               b.setBlockpools(blockpools);
             } else if ("-idleiterations".equalsIgnoreCase(args[i])) {
-              checkArgument(++i < args.length,
+              Preconditions.checkArgument(++i < args.length,
                   "idleiterations value is missing: args = " + Arrays
                       .toString(args));
               int maxIdleIteration = Integer.parseInt(args[i]);
@@ -1061,7 +1060,7 @@ static BalancerParameters parse(String[] args) {
               b.setRunAsService(true);
               LOG.info("Balancer will run as a long running service");
             } else if ("-hotBlockTimeInterval".equalsIgnoreCase(args[i])) {
-              checkArgument(++i < args.length,
+              Preconditions.checkArgument(++i < args.length,
                   "hotBlockTimeInterval value is missing: args = "
                   + Arrays.toString(args));
               long hotBlockTimeInterval = Long.parseLong(args[i]);
@@ -1077,7 +1076,7 @@ static BalancerParameters parse(String[] args) {
                   + Arrays.toString(args));
             }
           }
-          checkArgument(excludedNodes == null || includedNodes == null,
+          Preconditions.checkArgument(excludedNodes == null || includedNodes == null,
               "-exclude and -include options cannot be specified together.");
         } catch(RuntimeException e) {
           printUsage(System.err);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 90fd7e6fa1218..5c66d66912072 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -85,7 +85,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /** Dispatching block replica moves between datanodes. */
 @InterfaceAudience.Private
@@ -241,6 +241,7 @@ public class PendingMove {
     private DDatanode proxySource;
     private StorageGroup target;
 
+    @VisibleForTesting
     PendingMove(Source source, StorageGroup target) {
       this.source = source;
       this.target = target;
@@ -282,6 +283,7 @@ private boolean chooseBlockAndProxy() {
     /**
      * @return true if the given block is good for the tentative move.
      */
+    @VisibleForTesting
     boolean markMovedIfGoodBlock(DBlock block, StorageType targetStorageType) {
       synchronized (block) {
         synchronized (movedBlocks) {
@@ -495,7 +497,7 @@ public long getNumBytes(StorageGroup storage) {
 
   public static class DBlockStriped extends DBlock {
 
-    final byte[] indices;
+    private byte[] indices;
     final short dataBlockNum;
     final int cellSize;
 
@@ -532,6 +534,29 @@ public long getNumBytes(StorageGroup storage) {
       }
       return block.getNumBytes();
     }
+
+    public void setIndices(byte[] indices) {
+      this.indices = indices;
+    }
+
+    /**
+     * Adjust EC block indices，it will remove the element of adjustList from indices.
+     * @param adjustList the list will be removed from indices
+     */
+    public void adjustIndices(List<Integer> adjustList) {
+      if (adjustList.isEmpty()) {
+        return;
+      }
+
+      byte[] newIndices = new byte[indices.length - adjustList.size()];
+      for (int i = 0, j = 0; i < indices.length; ++i) {
+        if (!adjustList.contains(i)) {
+          newIndices[j] = indices[i];
+          ++j;
+        }
+      }
+      this.indices = newIndices;
+    }
   }
 
   /** The class represents a desired move. */
@@ -810,7 +835,7 @@ Iterator<DBlock> getBlockIterator() {
      * 
      * @return the total size of the received blocks in the number of bytes.
      */
-    private long getBlockList() throws IOException {
+    private long getBlockList() throws IOException, IllegalArgumentException {
       final long size = Math.min(getBlocksSize, blocksToReceive);
       final BlocksWithLocations newBlksLocs =
           nnc.getBlocks(getDatanodeInfo(), size, getBlocksMinBlockSize,
@@ -848,7 +873,14 @@ private long getBlockList() throws IOException {
           synchronized (block) {
             block.clearLocations();
 
+            if (blkLocs instanceof StripedBlockWithLocations) {
+              // EC block may adjust indices before, avoid repeated adjustments
+              ((DBlockStriped) block).setIndices(
+                  ((StripedBlockWithLocations) blkLocs).getIndices());
+            }
+
             // update locations
+            List<Integer> adjustList = new ArrayList<>();
             final String[] datanodeUuids = blkLocs.getDatanodeUuids();
             final StorageType[] storageTypes = blkLocs.getStorageTypes();
             for (int i = 0; i < datanodeUuids.length; i++) {
@@ -856,8 +888,20 @@ private long getBlockList() throws IOException {
                   datanodeUuids[i], storageTypes[i]);
               if (g != null) { // not unknown
                 block.addLocation(g);
+              } else if (blkLocs instanceof StripedBlockWithLocations) {
+                // some datanode may not in storageGroupMap due to decommission operation
+                // or balancer cli with "-exclude" parameter
+                adjustList.add(i);
               }
             }
+
+            if (!adjustList.isEmpty()) {
+              // block.locations mismatch with block.indices
+              // adjust indices to get correct internalBlock for Datanode in #getInternalBlock
+              ((DBlockStriped) block).adjustIndices(adjustList);
+              Preconditions.checkArgument(((DBlockStriped) block).indices.length
+                  == block.locations.size());
+            }
           }
           if (!srcBlocks.contains(block) && isGoodBlockCandidate(block)) {
             if (LOG.isTraceEnabled()) {
@@ -977,7 +1021,7 @@ private void dispatchBlocks() {
             }
             blocksToReceive -= received;
             continue;
-          } catch (IOException e) {
+          } catch (IOException | IllegalArgumentException e) {
             LOG.warn("Exception while getting reportedBlock list", e);
             return;
           }
@@ -1319,6 +1363,7 @@ public static boolean checkForSuccess(
    * 2. the block does not have a replica/internalBlock on the target;
    * 3. doing the move does not reduce the number of racks that the block has
    */
+  @VisibleForTesting
   boolean isGoodBlockCandidate(StorageGroup source, StorageGroup target,
       StorageType targetStorageType, DBlock block) {
     if (source.equals(target)) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index 750c351e81d6a..34be025203d47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -31,7 +32,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter;
 import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -147,11 +148,11 @@ public static void checkOtherInstanceRunning(boolean toCheck) {
 
   private final BalancerProtocols namenode;
   /**
-   * If set requestToStandby true, Balancer will getBlocks from
+   * If set getBlocksToStandby true, Balancer will getBlocks from
    * Standby NameNode only and it can reduce the performance impact of Active
    * NameNode, especially in a busy HA mode cluster.
    */
-  private boolean requestToStandby;
+  private boolean getBlocksToStandby;
   private String nsId;
   private Configuration config;
   private final KeyManager keyManager;
@@ -191,9 +192,9 @@ public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
 
     this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
         BalancerProtocols.class, fallbackToSimpleAuth).getProxy();
-    this.requestToStandby = conf.getBoolean(
-        DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_KEY,
-        DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_DEFAULT);
+    this.getBlocksToStandby = !conf.getBoolean(
+        DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY,
+        DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT);
     this.config = conf;
 
     this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf);
@@ -260,13 +261,15 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long
     }
     boolean isRequestStandby = false;
     NamenodeProtocol nnProxy = null;
+    InetSocketAddress standbyAddress = null;
     try {
       ProxyPair proxyPair = getProxy();
       isRequestStandby = proxyPair.isRequestStandby;
       ClientProtocol proxy = proxyPair.clientProtocol;
       if (isRequestStandby) {
+        standbyAddress = RPC.getServerAddress(proxy);
         nnProxy = NameNodeProxies.createNonHAProxy(
-            config, RPC.getServerAddress(proxy), NamenodeProtocol.class,
+            config, standbyAddress, NamenodeProtocol.class,
             UserGroupInformation.getCurrentUser(), false).getProxy();
       } else {
         nnProxy = namenode;
@@ -274,7 +277,8 @@ public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size, long
       return nnProxy.getBlocks(datanode, size, minBlockSize, timeInterval);
     } finally {
       if (isRequestStandby) {
-        LOG.info("Request #getBlocks to Standby NameNode success.");
+        LOG.info("Request #getBlocks to Standby NameNode success. " +
+            "remoteAddress: {}", standbyAddress.getHostString());
       }
     }
   }
@@ -297,15 +301,19 @@ public boolean isUpgrading() throws IOException {
   public DatanodeStorageReport[] getLiveDatanodeStorageReport()
       throws IOException {
     boolean isRequestStandby = false;
+    InetSocketAddress standbyAddress = null;
     try {
       ProxyPair proxyPair = getProxy();
       isRequestStandby = proxyPair.isRequestStandby;
       ClientProtocol proxy = proxyPair.clientProtocol;
+      if (isRequestStandby) {
+        standbyAddress = RPC.getServerAddress(proxy);
+      }
       return proxy.getDatanodeStorageReport(DatanodeReportType.LIVE);
     } finally {
       if (isRequestStandby) {
         LOG.info("Request #getLiveDatanodeStorageReport to Standby " +
-            "NameNode success.");
+            "NameNode success. remoteAddress: {}", standbyAddress.getHostString());
       }
     }
   }
@@ -318,7 +326,7 @@ public DatanodeStorageReport[] getLiveDatanodeStorageReport()
   private ProxyPair getProxy() throws IOException {
     boolean isRequestStandby = false;
     ClientProtocol clientProtocol = null;
-    if (requestToStandby && nsId != null
+    if (getBlocksToStandby && nsId != null
         && HAUtil.isHAEnabled(config, nsId)) {
       List<ClientProtocol> namenodes =
           HAUtil.getProxiesForAllNameNodesInNameservice(config, nsId);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
index 6926c3e01c448..9e349423daa77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -20,6 +20,8 @@
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY;
 
 import java.util.Collection;
 import java.util.EnumMap;
@@ -27,7 +29,7 @@
 import java.util.Random;
 import java.util.Set;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -47,6 +49,8 @@ public class AvailableSpaceBlockPlacementPolicy extends
   private static final Random RAND = new Random();
   private int balancedPreference =
       (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
+  private int balancedSpaceTolerance =
+          DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
   private boolean optimizeLocal;
 
   @Override
@@ -59,9 +63,14 @@ public void initialize(Configuration conf, FSClusterStats stats,
           DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
 
     LOG.info("Available space block placement policy initialized: "
-        + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+        + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
         + " = " + balancedPreferencePercent);
 
+    balancedSpaceTolerance =
+        conf.getInt(
+        DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY,
+        DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT);
+
     optimizeLocal = conf.getBoolean(
         DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY,
         DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_DEFAULT);
@@ -77,6 +86,16 @@ public void initialize(Configuration conf, FSClusterStats stats,
           + " is less than 0.5 so datanodes with more used percent will"
           + " receive  more block allocations.");
     }
+
+    if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) {
+      LOG.warn("The value of "
+          + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY
+          + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " +
+            DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT
+          + " will be used instead.");
+      balancedSpaceTolerance =
+              DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
+    }
     balancedPreference = (int) (100 * balancedPreferencePercent);
   }
 
@@ -183,7 +202,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b,
   protected int compareDataNode(final DatanodeDescriptor a,
       final DatanodeDescriptor b, boolean isBalanceLocal) {
     if (a.equals(b)
-        || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5 || ((
+        || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance || ((
         isBalanceLocal && a.getDfsUsedPercent() < 50))) {
       return 0;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java
index e673347de630d..85de2fee3e2f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -32,7 +32,9 @@
 import java.util.Random;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY;
 
 /**
  * Space balanced rack fault tolerant block placement policy.
@@ -45,7 +47,8 @@ public class AvailableSpaceRackFaultTolerantBlockPlacementPolicy
   private static final Random RAND = new Random();
   private int balancedPreference = (int) (100
       * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
-
+  private int balancedSpaceTolerance =
+        DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
   @Override
   public void initialize(Configuration conf, FSClusterStats stats,
       NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) {
@@ -54,6 +57,10 @@ public void initialize(Configuration conf, FSClusterStats stats,
         DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
         DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
 
+    balancedSpaceTolerance = conf.getInt(
+            DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY,
+            DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT);
+
     LOG.info("Available space rack fault tolerant block placement policy "
         + "initialized: "
         + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
@@ -70,6 +77,18 @@ public void initialize(Configuration conf, FSClusterStats stats,
           + " is less than 0.5 so datanodes with more used percent will"
           + " receive  more block allocations.");
     }
+
+
+    if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) {
+      LOG.warn("The value of "
+          + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY
+          + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " +
+            DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT
+          + " will be used instead.");
+      balancedSpaceTolerance =
+            DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;
+    }
+
     balancedPreference = (int) (100 * balancedPreferencePercent);
   }
 
@@ -118,7 +137,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a,
   protected int compareDataNode(final DatanodeDescriptor a,
       final DatanodeDescriptor b) {
     if (a.equals(b)
-        || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) {
+        || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance) {
       return 0;
     }
     return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index df14107c322dd..a79e4c594ac2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index b8047a8d08015..659f218437539 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -22,7 +22,7 @@
 import java.util.LinkedList;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -86,7 +86,7 @@ public BlockInfo(short size) {
 
   public BlockInfo(Block blk, short size) {
     super(blk);
-    this.triplets = new Object[3*size];
+    this.triplets = new Object[3 * size];
     this.bcId = INVALID_INODE_ID;
     this.replication = isStriped() ? 0 : size;
   }
@@ -126,34 +126,34 @@ public DatanodeDescriptor getDatanode(int index) {
 
   DatanodeStorageInfo getStorageInfo(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
-    return (DatanodeStorageInfo)triplets[index*3];
+    assert index >= 0 && index * 3 < triplets.length : "Index is out of bound";
+    return (DatanodeStorageInfo)triplets[index * 3];
   }
 
   BlockInfo getPrevious(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+1];
+    assert index >= 0 && index * 3 + 1 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index * 3 + 1];
     assert info == null ||
         info.getClass().getName().startsWith(BlockInfo.class.getName()) :
-        "BlockInfo is expected at " + index*3;
+        "BlockInfo is expected at " + (index * 3 + 1);
     return info;
   }
 
   BlockInfo getNext(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo)triplets[index*3+2];
+    assert index >= 0 && index * 3 + 2 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index * 3 + 2];
     assert info == null || info.getClass().getName().startsWith(
         BlockInfo.class.getName()) :
-        "BlockInfo is expected at " + index*3;
+        "BlockInfo is expected at " + (index * 3 + 2);
     return info;
   }
 
   void setStorageInfo(int index, DatanodeStorageInfo storage) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
-    triplets[index*3] = storage;
+    assert index >= 0 && index * 3 < triplets.length : "Index is out of bound";
+    triplets[index * 3] = storage;
   }
 
   /**
@@ -166,9 +166,9 @@ void setStorageInfo(int index, DatanodeStorageInfo storage) {
    */
   BlockInfo setPrevious(int index, BlockInfo to) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo) triplets[index*3+1];
-    triplets[index*3+1] = to;
+    assert index >= 0 && index * 3 + 1 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo) triplets[index * 3 + 1];
+    triplets[index * 3 + 1] = to;
     return info;
   }
 
@@ -182,9 +182,9 @@ BlockInfo setPrevious(int index, BlockInfo to) {
    */
   BlockInfo setNext(int index, BlockInfo to) {
     assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfo info = (BlockInfo) triplets[index*3+2];
-    triplets[index*3+2] = to;
+    assert index >= 0 && index * 3 + 2 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo) triplets[index * 3 + 2];
+    triplets[index * 3 + 2] = to;
     return info;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index d68b72d6c6277..651f39f2b69a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.Block;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 53368c98a4820..4b8d092935a00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 69e9ca8e7f397..3b0bbe847bfb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -47,6 +47,7 @@
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import java.util.concurrent.atomic.AtomicLong;
 import javax.management.ObjectName;
@@ -126,7 +127,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -191,6 +192,9 @@ public class BlockManager implements BlockStatsMXBean {
   private volatile long lowRedundancyBlocksCount = 0L;
   private volatile long scheduledReplicationBlocksCount = 0L;
 
+  private final long deleteBlockLockTimeMs;
+  private final long deleteBlockUnlockIntervalTimeMs;
+
   /** flag indicating whether replication queues have been initialized */
   private boolean initializedReplQueues;
 
@@ -294,6 +298,14 @@ public long getTotalECBlockGroups() {
     return blocksMap.getECBlockGroups();
   }
 
+  /** Used by metrics. */
+  public int getPendingSPSPaths() {
+    if (spsManager != null) {
+      return spsManager.getPendingSPSPaths();
+    }
+    return 0;
+  }
+
   /**
    * redundancyRecheckInterval is how often namenode checks for new
    * reconstruction work.
@@ -324,6 +336,12 @@ public long getTotalECBlockGroups() {
    * {@link #redundancyThread} has run at least one full iteration.
    */
   private final AtomicLong lastRedundancyCycleTS = new AtomicLong(-1);
+  /**
+   * markedDeleteBlockScrubber thread for handling async delete blocks.
+   */
+  private final Daemon markedDeleteBlockScrubberThread =
+      new Daemon(new MarkedDeleteBlockScrubber());
+
   /** Block report thread for handling async reports. */
   private final BlockReportProcessingThread blockReportThread;
 
@@ -422,6 +440,12 @@ public long getTotalECBlockGroups() {
    */
   private int numBlocksPerIteration;
 
+  /**
+   * The blocks of deleted files are put into the queue,
+   * and the cleanup thread processes these blocks periodically.
+   */
+  private final ConcurrentLinkedQueue<List<BlockInfo>> markedDeleteQueue;
+
   /**
    * Progress of the Reconstruction queues initialisation.
    */
@@ -461,66 +485,54 @@ public long getTotalECBlockGroups() {
   public BlockManager(final Namesystem namesystem, boolean haEnabled,
       final Configuration conf) throws IOException {
     this.namesystem = namesystem;
-    datanodeManager = new DatanodeManager(this, namesystem, conf);
-    heartbeatManager = datanodeManager.getHeartbeatManager();
+    this.datanodeManager = new DatanodeManager(this, namesystem, conf);
+    this.heartbeatManager = datanodeManager.getHeartbeatManager();
     this.blockIdManager = new BlockIdManager(this);
-    blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
+    this.blocksPerPostpondedRescan = (int)Math.min(Integer.MAX_VALUE,
         datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan());
-    rescannedMisreplicatedBlocks =
+    this.rescannedMisreplicatedBlocks =
         new ArrayList<Block>(blocksPerPostpondedRescan);
-    startupDelayBlockDeletionInMs = conf.getLong(
+    this.startupDelayBlockDeletionInMs = conf.getLong(
         DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
-        DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L;
-    invalidateBlocks = new InvalidateBlocks(
+        DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT)
+        * 1000L;
+    this.deleteBlockLockTimeMs = conf.getLong(
+        DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS,
+        DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT);
+    this.deleteBlockUnlockIntervalTimeMs = conf.getLong(
+        DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS,
+        DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT);
+    this.invalidateBlocks = new InvalidateBlocks(
         datanodeManager.getBlockInvalidateLimit(),
         startupDelayBlockDeletionInMs,
         blockIdManager);
-
+    this.markedDeleteQueue = new ConcurrentLinkedQueue<>();
     // Compute the map capacity by allocating 2% of total memory
-    blocksMap = new BlocksMap(
+    this.blocksMap = new BlocksMap(
         LightWeightGSet.computeCapacity(2.0, "BlocksMap"));
-    placementPolicies = new BlockPlacementPolicies(
-      conf, datanodeManager.getFSClusterStats(),
-      datanodeManager.getNetworkTopology(),
-      datanodeManager.getHost2DatanodeMap());
-    storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(conf);
-    pendingReconstruction = new PendingReconstructionBlocks(conf.getInt(
+    this.placementPolicies = new BlockPlacementPolicies(
+        conf, datanodeManager.getFSClusterStats(),
+        datanodeManager.getNetworkTopology(),
+        datanodeManager.getHost2DatanodeMap());
+    this.storagePolicySuite = BlockStoragePolicySuite.createDefaultSuite(conf);
+    this.pendingReconstruction = new PendingReconstructionBlocks(conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
         DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT)
         * 1000L);
 
     createSPSManager(conf);
 
-    blockTokenSecretManager = createBlockTokenSecretManager(conf);
-
-    providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
+    this.blockTokenSecretManager = createBlockTokenSecretManager(conf);
+    this.providedStorageMap = new ProvidedStorageMap(namesystem, this, conf);
 
     this.maxCorruptFilesReturned = conf.getInt(
-      DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY,
-      DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED);
+        DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY,
+        DFSConfigKeys.DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED);
     this.defaultReplication = conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
         DFSConfigKeys.DFS_REPLICATION_DEFAULT);
 
-    final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY,
-        DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT);
-    final int minR = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
-        DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
-    if (minR <= 0)
-      throw new IOException("Unexpected configuration parameters: "
-          + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
-          + " = " + minR + " <= 0");
-    if (maxR > Short.MAX_VALUE)
-      throw new IOException("Unexpected configuration parameters: "
-          + DFSConfigKeys.DFS_REPLICATION_MAX_KEY
-          + " = " + maxR + " > " + Short.MAX_VALUE);
-    if (minR > maxR)
-      throw new IOException("Unexpected configuration parameters: "
-          + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
-          + " = " + minR + " > "
-          + DFSConfigKeys.DFS_REPLICATION_MAX_KEY
-          + " = " + maxR);
-    this.minReplication = (short)minR;
-    this.maxReplication = (short)maxR;
+    this.minReplication = (short) initMinReplication(conf);
+    this.maxReplication = (short) initMaxReplication(conf);
 
     this.maxReplicationStreams =
         conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY,
@@ -552,6 +564,66 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
         DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
         DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
 
+    this.minReplicationToBeInMaintenance =
+        (short) initMinReplicationToBeInMaintenance(conf);
+    this.replQueueResetToHeadThreshold =
+        initReplQueueResetToHeadThreshold(conf);
+
+    long heartbeatIntervalSecs = conf.getTimeDuration(
+        DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+        DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
+    long blockRecoveryTimeout = getBlockRecoveryTimeout(heartbeatIntervalSecs);
+    this.pendingRecoveryBlocks = new PendingRecoveryBlocks(blockRecoveryTimeout);
+
+    this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
+
+    this.bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled,
+        conf);
+
+    int queueSize = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY,
+        DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT);
+    this.blockReportThread = new BlockReportProcessingThread(queueSize);
+
+    this.deleteCorruptReplicaImmediately =
+        conf.getBoolean(DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED,
+            DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED_DEFAULT);
+
+    printInitialConfigs();
+  }
+
+  private int initMinReplication(Configuration conf) throws IOException {
+    final int minR = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
+        DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
+    if (minR <= 0) {
+      throw new IOException("Unexpected configuration parameters: "
+          + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
+          + " = " + minR + " <= 0");
+    }
+    return minR;
+  }
+
+  private int initMaxReplication(Configuration conf) throws IOException {
+    final int maxR = conf.getInt(DFSConfigKeys.DFS_REPLICATION_MAX_KEY,
+        DFSConfigKeys.DFS_REPLICATION_MAX_DEFAULT);
+    if (maxR > Short.MAX_VALUE) {
+      throw new IOException("Unexpected configuration parameters: "
+          + DFSConfigKeys.DFS_REPLICATION_MAX_KEY
+          + " = " + maxR + " > " + Short.MAX_VALUE);
+    }
+    if (minReplication > maxR) {
+      throw new IOException("Unexpected configuration parameters: "
+          + DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY
+          + " = " + minReplication + " > "
+          + DFSConfigKeys.DFS_REPLICATION_MAX_KEY
+          + " = " + maxR);
+    }
+    return maxR;
+  }
+
+  private int initMinReplicationToBeInMaintenance(Configuration conf)
+      throws IOException {
     final int minMaintenanceR = conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY,
         DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_DEFAULT);
@@ -568,39 +640,25 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
           + DFSConfigKeys.DFS_REPLICATION_KEY
           + " = " + defaultReplication);
     }
-    this.minReplicationToBeInMaintenance = (short)minMaintenanceR;
+    return minMaintenanceR;
+  }
 
-    replQueueResetToHeadThreshold = conf.getInt(
+  private int initReplQueueResetToHeadThreshold(Configuration conf) {
+    int threshold = conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS,
         DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT);
-    if (replQueueResetToHeadThreshold < 0) {
+    if (threshold < 0) {
       LOG.warn("{} is set to {} and it must be >= 0. Resetting to default {}",
           DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS,
-          replQueueResetToHeadThreshold, DFSConfigKeys.
+          threshold, DFSConfigKeys.
               DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT);
-      replQueueResetToHeadThreshold = DFSConfigKeys.
+      threshold = DFSConfigKeys.
           DFS_NAMENODE_REDUNDANCY_QUEUE_RESTART_ITERATIONS_DEFAULT;
     }
+    return threshold;
+  }
 
-    long heartbeatIntervalSecs = conf.getTimeDuration(
-        DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
-        DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
-    long blockRecoveryTimeout = getBlockRecoveryTimeout(heartbeatIntervalSecs);
-    pendingRecoveryBlocks = new PendingRecoveryBlocks(blockRecoveryTimeout);
-
-    this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
-
-    bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
-
-    int queueSize = conf.getInt(
-        DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY,
-        DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT);
-    blockReportThread = new BlockReportProcessingThread(queueSize);
-
-    this.deleteCorruptReplicaImmediately =
-        conf.getBoolean(DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED,
-            DFS_NAMENODE_CORRUPT_BLOCK_DELETE_IMMEDIATELY_ENABLED_DEFAULT);
-
+  private void printInitialConfigs() {
     LOG.info("defaultReplication         = {}", defaultReplication);
     LOG.info("maxReplication             = {}", maxReplication);
     LOG.info("minReplication             = {}", minReplication);
@@ -725,6 +783,9 @@ public void activate(Configuration conf, long blockTotal) {
     datanodeManager.activate(conf);
     this.redundancyThread.setName("RedundancyMonitor");
     this.redundancyThread.start();
+    this.markedDeleteBlockScrubberThread.
+        setName("MarkedDeleteBlockScrubberThread");
+    this.markedDeleteBlockScrubberThread.start();
     this.blockReportThread.start();
     mxBeanName = MBeans.register("NameNode", "BlockStats", this);
     bmSafeMode.activate(blockTotal);
@@ -738,8 +799,10 @@ public void close() {
     try {
       redundancyThread.interrupt();
       blockReportThread.interrupt();
+      markedDeleteBlockScrubberThread.interrupt();
       redundancyThread.join(3000);
       blockReportThread.join(3000);
+      markedDeleteBlockScrubberThread.join(3000);
     } catch (InterruptedException ie) {
     }
     datanodeManager.close();
@@ -757,6 +820,11 @@ public BlockPlacementPolicy getBlockPlacementPolicy() {
     return placementPolicies.getPolicy(CONTIGUOUS);
   }
 
+  @VisibleForTesting
+  public BlockPlacementPolicy getStriptedBlockPlacementPolicy() {
+    return placementPolicies.getPolicy(STRIPED);
+  }
+
   public void refreshBlockPlacementPolicy(Configuration conf) {
     BlockPlacementPolicies bpp =
         new BlockPlacementPolicies(conf, datanodeManager.getFSClusterStats(),
@@ -881,7 +949,7 @@ private void dumpBlockMeta(Block block, PrintWriter out) {
     // source node returned is not used
     chooseSourceDatanodes(blockInfo, containingNodes,
         containingLiveReplicasNodes, numReplicas, new ArrayList<Byte>(),
-        new ArrayList<Byte>(), LowRedundancyBlocks.LEVEL);
+        new ArrayList<Byte>(), new ArrayList<Byte>(), LowRedundancyBlocks.LEVEL);
     
     // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are 
     // not included in the numReplicas.liveReplicas() count
@@ -999,6 +1067,26 @@ public void setBlocksReplWorkMultiplier(int newVal) {
     blocksReplWorkMultiplier = newVal;
   }
 
+  /**
+   * Updates the value used for pendingReconstruction timeout, which is set by
+   * {@code DFSConfigKeys.
+   *     DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY} initially.
+   *
+   * @param newVal - Must be a positive non-zero integer.
+   */
+  public void setReconstructionPendingTimeout(int newVal) {
+    ensurePositiveInt(newVal,
+        DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY);
+    pendingReconstruction.setTimeout(newVal * 1000L);
+  }
+
+  /** Returns the current setting for pendingReconstruction timeout, set by
+   * {@code DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY}.
+   */
+  public int getReconstructionPendingTimeout() {
+    return (int)(pendingReconstruction.getTimeout() / 1000L);
+  }
+
   public int getDefaultStorageNum(BlockInfo block) {
     switch (block.getBlockType()) {
     case STRIPED: return ((BlockInfoStriped) block).getRealTotalBlockNum();
@@ -1029,7 +1117,7 @@ public short getMinReplicationToBeInMaintenance() {
     return minReplicationToBeInMaintenance;
   }
 
-  private short getMinMaintenanceStorageNum(BlockInfo block) {
+  short getMinMaintenanceStorageNum(BlockInfo block) {
     if (block.isStriped()) {
       return ((BlockInfoStriped) block).getRealDataBlockNum();
     } else {
@@ -1473,6 +1561,7 @@ public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
       if (LOG.isDebugEnabled()) {
         LOG.debug("blocks = {}", java.util.Arrays.asList(blocks));
       }
+
       final AccessMode mode = needBlockToken? BlockTokenIdentifier.AccessMode.READ: null;
 
       LocatedBlockBuilder locatedBlocks = providedStorageMap
@@ -1638,9 +1727,16 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
     if(numBlocks == 0) {
       return new BlocksWithLocations(new BlockWithLocations[0]);
     }
+
+    // skip stale storage
+    DatanodeStorageInfo[] storageInfos = Arrays
+        .stream(node.getStorageInfos())
+        .filter(s -> !s.areBlockContentsStale())
+        .toArray(DatanodeStorageInfo[]::new);
+
     // starting from a random block
     int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
-    Iterator<BlockInfo> iter = node.getBlockIterator(startBlock);
+    Iterator<BlockInfo> iter = node.getBlockIterator(startBlock, storageInfos);
     List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
     List<BlockInfo> pending = new ArrayList<BlockInfo>();
     long totalSize = 0;
@@ -1659,8 +1755,8 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
       }
     }
     if(totalSize<size) {
-      iter = node.getBlockIterator(); // start from the beginning
-      for(int i=0; i<startBlock&&totalSize<size; i++) {
+      iter = node.getBlockIterator(0, storageInfos); // start from the beginning
+      for(int i = 0; i < startBlock && totalSize < size && iter.hasNext(); i++) {
         curBlock = iter.next();
         if(!curBlock.isComplete())  continue;
         if (curBlock.getNumBytes() < minBlockSize) {
@@ -1798,8 +1894,7 @@ public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
     }
 
     if (storage == null) {
-      blockLog.debug("BLOCK* findAndMarkBlockAsCorrupt: {} not found on {}",
-          blk, dn);
+      blockLog.debug("BLOCK* findAndMarkBlockAsCorrupt: {} not found on {}", blk, dn);
       return;
     }
     markBlockAsCorrupt(new BlockToMarkCorrupt(reportedBlock, storedBlock,
@@ -1818,8 +1913,10 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
       DatanodeStorageInfo storageInfo,
       DatanodeDescriptor node) throws IOException {
     if (b.getStored().isDeleted()) {
-      blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
-          " corrupt as it does not belong to any file", b);
+      if (blockLog.isDebugEnabled()) {
+        blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
+            " corrupt as it does not belong to any file", b);
+      }
       addToInvalidates(b.getCorrupted(), node);
       return;
     }
@@ -1841,23 +1938,29 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
         b.getReasonCode(), b.getStored().isStriped());
 
     NumberReplicas numberOfReplicas = countNodes(b.getStored());
-    boolean hasEnoughLiveReplicas = numberOfReplicas.liveReplicas() >=
+    final int numUsableReplicas = numberOfReplicas.liveReplicas() +
+        numberOfReplicas.decommissioning() +
+        numberOfReplicas.liveEnteringMaintenanceReplicas();
+    boolean hasEnoughLiveReplicas = numUsableReplicas >=
         expectedRedundancies;
 
     boolean minReplicationSatisfied = hasMinStorage(b.getStored(),
-        numberOfReplicas.liveReplicas());
+        numUsableReplicas);
 
     boolean hasMoreCorruptReplicas = minReplicationSatisfied &&
         (numberOfReplicas.liveReplicas() + numberOfReplicas.corruptReplicas()) >
         expectedRedundancies;
     boolean corruptedDuringWrite = minReplicationSatisfied &&
         b.isCorruptedDuringWrite();
-    // case 1: have enough number of live replicas
-    // case 2: corrupted replicas + live replicas > Replication factor
+    // case 1: have enough number of usable replicas
+    // case 2: corrupted replicas + usable replicas > Replication factor
     // case 3: Block is marked corrupt due to failure while writing. In this
     //         case genstamp will be different than that of valid block.
     // In all these cases we can delete the replica.
-    // In case of 3, rbw block will be deleted and valid block can be replicated
+    // In case 3, rbw block will be deleted and valid block can be replicated.
+    // Note NN only becomes aware of corrupt blocks when the block report is sent,
+    // this means that by default it can take up to 6 hours for a corrupt block to
+    // be invalidated, after which the valid block can be replicated.
     if (hasEnoughLiveReplicas || hasMoreCorruptReplicas
         || corruptedDuringWrite) {
       if (b.getStored().isStriped()) {
@@ -1894,10 +1997,12 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn,
 
     // Check how many copies we have of the block
     if (nr.replicasOnStaleNodes() > 0 && !deleteCorruptReplicaImmediately) {
-      blockLog.debug("BLOCK* invalidateBlocks: postponing " +
-          "invalidation of {} on {} because {} replica(s) are located on " +
-          "nodes with potentially out-of-date block reports", b, dn,
-          nr.replicasOnStaleNodes());
+      if (blockLog.isDebugEnabled()) {
+        blockLog.debug("BLOCK* invalidateBlocks: postponing " +
+            "invalidation of {} on {} because {} replica(s) are located on " +
+            "nodes with potentially out-of-date block reports", b, dn,
+            nr.replicasOnStaleNodes());
+      }
       postponeBlock(b.getCorrupted());
       return false;
     } else {
@@ -1905,8 +2010,7 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn,
       // function and know there are enough live replicas, so we can delete it.
       addToInvalidates(b.getCorrupted(), dn);
       removeStoredBlock(b.getStored(), node);
-      blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.",
-          b, dn);
+      blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.", b, dn);
       return true;
     }
   }
@@ -1984,7 +2088,7 @@ int computeBlockReconstructionWork(int blocksToProcess) {
       blocksToReconstruct = neededReconstruction
           .chooseLowRedundancyBlocks(blocksToProcess, reset);
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("computeBlockReconstructionWork");
     }
     return computeReconstructionWorkForBlocks(blocksToReconstruct);
   }
@@ -2018,7 +2122,7 @@ int computeReconstructionWorkForBlocks(
         }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("computeReconstructionWorkForBlocks");
     }
 
     // Step 2: choose target nodes for each reconstruction task
@@ -2059,7 +2163,7 @@ int computeReconstructionWorkForBlocks(
         }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("computeReconstructionWorkForBlocks");
     }
 
     if (blockLog.isDebugEnabled()) {
@@ -2071,13 +2175,11 @@ int computeReconstructionWorkForBlocks(
           for (DatanodeStorageInfo target : targets) {
             targetList.append(' ').append(target.getDatanodeDescriptor());
           }
-          blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.getSrcNodes(),
-              rw.getBlock(), targetList);
+          blockLog.debug("BLOCK* ask {} to replicate {} to {}",
+              rw.getSrcNodes(), rw.getBlock(), targetList);
         }
       }
-
-      blockLog.debug(
-          "BLOCK* neededReconstruction = {} pendingReconstruction = {}",
+      blockLog.debug("BLOCK* neededReconstruction = {} pendingReconstruction = {}",
           neededReconstruction.size(), pendingReconstruction.size());
     }
 
@@ -2110,18 +2212,29 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block,
     NumberReplicas numReplicas = new NumberReplicas();
     List<Byte> liveBlockIndices = new ArrayList<>();
     List<Byte> liveBusyBlockIndices = new ArrayList<>();
+    List<Byte> excludeReconstructed = new ArrayList<>();
     final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block,
         containingNodes, liveReplicaNodes, numReplicas,
-        liveBlockIndices, liveBusyBlockIndices, priority);
+        liveBlockIndices, liveBusyBlockIndices, excludeReconstructed, priority);
     short requiredRedundancy = getExpectedLiveRedundancyNum(block,
         numReplicas);
-    if(srcNodes == null || srcNodes.length == 0) {
+    if (srcNodes == null || srcNodes.length == 0) {
       // block can not be reconstructed from any node
       LOG.debug("Block {} cannot be reconstructed from any node", block);
       NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled();
       return null;
     }
 
+    // skip if source datanodes for reconstructing ec block are not enough
+    if (block.isStriped()) {
+      BlockInfoStriped stripedBlock = (BlockInfoStriped) block;
+      if (stripedBlock.getRealDataBlockNum() > srcNodes.length) {
+        LOG.debug("Block {} cannot be reconstructed due to shortage of source datanodes ", block);
+        NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled();
+        return null;
+      }
+    }
+
     // liveReplicaNodes can include READ_ONLY_SHARED replicas which are
     // not included in the numReplicas.liveReplicas() count
     assert liveReplicaNodes.size() >= numReplicas.liveReplicas();
@@ -2129,8 +2242,8 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block,
     int pendingNum = pendingReconstruction.getNumReplicas(block);
     if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) {
       neededReconstruction.remove(block, priority);
-      blockLog.debug("BLOCK* Removing {} from neededReconstruction as" +
-          " it has enough replicas", block);
+      blockLog.debug("BLOCK* Removing {} from neededReconstruction as it has enough replicas",
+          block);
       NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled();
       return null;
     }
@@ -2170,9 +2283,13 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block,
       for (int i = 0; i < liveBusyBlockIndices.size(); i++) {
         busyIndices[i] = liveBusyBlockIndices.get(i);
       }
+      byte[] excludeReconstructedIndices = new byte[excludeReconstructed.size()];
+      for (int i = 0; i < excludeReconstructed.size(); i++) {
+        excludeReconstructedIndices[i] = excludeReconstructed.get(i);
+      }
       return new ErasureCodingWork(getBlockPoolId(), block, bc, newSrcNodes,
           containingNodes, liveReplicaNodes, additionalReplRequired,
-          priority, newIndices, busyIndices);
+          priority, newIndices, busyIndices, excludeReconstructedIndices);
     } else {
       return new ReplicationWork(block, bc, srcNodes,
           containingNodes, liveReplicaNodes, additionalReplRequired,
@@ -2226,8 +2343,8 @@ boolean validateReconstructionWork(BlockReconstructionWork rw) {
     if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) {
       neededReconstruction.remove(block, priority);
       rw.resetTargets();
-      blockLog.debug("BLOCK* Removing {} from neededReconstruction as" +
-          " it has enough replicas", block);
+      blockLog.debug("BLOCK* Removing {} from neededReconstruction as it has enough replicas",
+          block);
       return false;
     }
 
@@ -2258,8 +2375,8 @@ boolean validateReconstructionWork(BlockReconstructionWork rw) {
     // The reason we use 'pending' is so we can retry
     // reconstructions that fail after an appropriate amount of time.
     pendingReconstruction.increment(block, targets);
-    blockLog.debug("BLOCK* block {} is moved from neededReconstruction to "
-        + "pendingReconstruction", block);
+    blockLog.debug("BLOCK* block {} is moved from neededReconstruction to pendingReconstruction",
+        block);
 
     int numEffectiveReplicas = numReplicas.liveReplicas() + pendingNum;
     // remove from neededReconstruction
@@ -2402,6 +2519,10 @@ private DatanodeDescriptor getDatanodeDescriptorFromStorage(
    *                    replicas of the given block.
    * @param liveBlockIndices List to be populated with indices of healthy
    *                         blocks in a striped block group
+   * @param liveBusyBlockIndices List to be populated with indices of healthy
+   *                             blocks in a striped block group in busy DN,
+   *                             which the recovery work have reached their
+   *                             replication limits
    * @param priority integer representing replication priority of the given
    *                 block
    * @return the array of DatanodeDescriptor of the chosen nodes from which to
@@ -2412,7 +2533,7 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block,
       List<DatanodeDescriptor> containingNodes,
       List<DatanodeStorageInfo> nodesContainingLiveReplicas,
       NumberReplicas numReplicas, List<Byte> liveBlockIndices,
-      List<Byte> liveBusyBlockIndices, int priority) {
+      List<Byte> liveBusyBlockIndices, List<Byte> excludeReconstructed, int priority) {
     containingNodes.clear();
     nodesContainingLiveReplicas.clear();
     List<DatanodeDescriptor> srcNodes = new ArrayList<>();
@@ -2478,18 +2599,24 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block,
 
       if (priority != LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY
           && (!node.isDecommissionInProgress() && !node.isEnteringMaintenance())
-          && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams) {
+          && node.getNumberOfBlocksToBeReplicated() +
+          node.getNumberOfBlocksToBeErasureCoded() >= maxReplicationStreams) {
         if (isStriped && (state == StoredReplicaState.LIVE
             || state == StoredReplicaState.DECOMMISSIONING)) {
           liveBusyBlockIndices.add(blockIndex);
+          //HDFS-16566 ExcludeReconstructed won't be reconstructed.
+          excludeReconstructed.add(blockIndex);
         }
         continue; // already reached replication limit
       }
 
-      if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit) {
+      if (node.getNumberOfBlocksToBeReplicated() +
+          node.getNumberOfBlocksToBeErasureCoded() >= replicationStreamsHardLimit) {
         if (isStriped && (state == StoredReplicaState.LIVE
             || state == StoredReplicaState.DECOMMISSIONING)) {
           liveBusyBlockIndices.add(blockIndex);
+          //HDFS-16566 ExcludeReconstructed won't be reconstructed.
+          excludeReconstructed.add(blockIndex);
         }
         continue;
       }
@@ -2544,7 +2671,7 @@ void processPendingReconstructions() {
           }
         }
       } finally {
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("processPendingReconstructions");
       }
       /* If we know the target datanodes where the replication timedout,
        * we could invoke decBlocksScheduled() on it. Its ok for now.
@@ -2644,9 +2771,11 @@ public void removeBlocksAndUpdateSafemodeTotal(BlocksMapUpdateInfo blocks) {
       removeBlock(b);
     }
     if (trackBlockCounts) {
-      LOG.debug("Adjusting safe-mode totals for deletion."
-          + "decreasing safeBlocks by {}, totalBlocks by {}",
-          numRemovedSafe, numRemovedComplete);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Adjusting safe-mode totals for deletion."
+                + "decreasing safeBlocks by {}, totalBlocks by {}",
+            numRemovedSafe, numRemovedComplete);
+      }
       bmSafeMode.adjustBlockTotals(-numRemovedSafe, -numRemovedComplete);
     }
   }
@@ -2658,7 +2787,7 @@ public long getProvidedCapacity() {
   void updateHeartbeat(DatanodeDescriptor node, StorageReport[] reports,
       long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary) {
-
+    BlockManagerFaultInjector.getInstance().mockAnException();
     for (StorageReport report: reports) {
       providedStorageMap.updateStorage(node, report.getStorage());
     }
@@ -2670,6 +2799,7 @@ void updateHeartbeatState(DatanodeDescriptor node,
       StorageReport[] reports, long cacheCapacity, long cacheUsed,
       int xceiverCount, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary) {
+    BlockManagerFaultInjector.getInstance().mockAnException();
     for (StorageReport report: reports) {
       providedStorageMap.updateStorage(node, report.getStorage());
     }
@@ -2717,6 +2847,9 @@ public boolean checkBlockReportLease(BlockReportContext context,
       return true;
     }
     DatanodeDescriptor node = datanodeManager.getDatanode(nodeID);
+    if (node == null) {
+      throw new UnregisteredNodeException(nodeID, null);
+    }
     final long startTime = Time.monotonicNow();
     return blockReportLeaseManager.checkLease(node, startTime,
         context.getLeaseId());
@@ -2741,6 +2874,8 @@ public boolean processReport(final DatanodeID nodeID,
     Collection<Block> invalidatedBlocks = Collections.emptyList();
     String strBlockReportId =
         context != null ? Long.toHexString(context.getReportId()) : "";
+    String fullBrLeaseId =
+        context != null ? Long.toHexString(context.getLeaseId()) : "";
 
     try {
       node = datanodeManager.getDatanode(nodeID);
@@ -2763,10 +2898,10 @@ public boolean processReport(final DatanodeID nodeID,
       if (namesystem.isInStartupSafeMode()
           && !StorageType.PROVIDED.equals(storageInfo.getStorageType())
           && storageInfo.getBlockReportCount() > 0) {
-        blockLog.info("BLOCK* processReport 0x{}: "
+        blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: "
             + "discarded non-initial block report from {}"
             + " because namenode still in startup phase",
-            strBlockReportId, nodeID);
+            strBlockReportId, fullBrLeaseId, nodeID);
         blockReportLeaseManager.removeLease(node);
         return !node.hasStaleStorages();
       }
@@ -2774,11 +2909,11 @@ public boolean processReport(final DatanodeID nodeID,
       if (storageInfo.getBlockReportCount() == 0) {
         // The first block report can be processed a lot more efficiently than
         // ordinary block reports.  This shortens restart times.
-        blockLog.info("BLOCK* processReport 0x{}: Processing first "
+        blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: Processing first "
             + "storage report for {} from datanode {}",
-            strBlockReportId,
+            strBlockReportId, fullBrLeaseId,
             storageInfo.getStorageID(),
-            nodeID.getDatanodeUuid());
+            nodeID);
         processFirstBlockReport(storageInfo, newReport);
       } else {
         // Block reports for provided storage are not
@@ -2790,14 +2925,14 @@ public boolean processReport(final DatanodeID nodeID,
       storageInfo.receivedBlockReport();
     } finally {
       endTime = Time.monotonicNow();
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("processReport");
     }
 
-    if(blockLog.isDebugEnabled()) {
+    if (blockLog.isDebugEnabled()) {
       for (Block b : invalidatedBlocks) {
-        blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} " +
-                        "does not belong to any file.", strBlockReportId, b,
-                         node, b.getNumBytes());
+        blockLog.debug("BLOCK* processReport 0x{} with lease ID 0x{}: {} on node {} size {} " +
+                "does not belong to any file.", strBlockReportId, fullBrLeaseId, b,
+            node, b.getNumBytes());
       }
     }
 
@@ -2806,9 +2941,9 @@ public boolean processReport(final DatanodeID nodeID,
     if (metrics != null) {
       metrics.addStorageBlockReport((int) (endTime - startTime));
     }
-    blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " +
+    blockLog.info("BLOCK* processReport 0x{} with lease ID 0x{}: from storage {} node {}, " +
         "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " +
-        "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(),
+        "invalidatedBlocks: {}", strBlockReportId, fullBrLeaseId, storage.getStorageID(),
         nodeID, newReport.getNumberOfBlocks(),
         node.hasStaleStorages(), (endTime - startTime),
         invalidatedBlocks.size());
@@ -2829,12 +2964,13 @@ public void removeBRLeaseIfNeeded(final DatanodeID nodeID,
           node.setLastBlockReportTime(now());
           node.setLastBlockReportMonotonic(Time.monotonicNow());
         }
-        LOG.debug("Processing RPC with index {} out of total {} RPCs in "
-                + "processReport 0x{}", context.getCurRpc(),
-            context.getTotalRpcs(), Long.toHexString(context.getReportId()));
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Processing RPC with index {} out of total {} RPCs in processReport 0x{}",
+              context.getCurRpc(), context.getTotalRpcs(), Long.toHexString(context.getReportId()));
+        }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("removeBRLeaseIfNeeded");
     }
   }
 
@@ -2856,14 +2992,16 @@ void rescanPostponedMisreplicatedBlocks() {
 
         BlockInfo bi = getStoredBlock(b);
         if (bi == null) {
-          LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
-              "Postponed mis-replicated block {} no longer found " +
-              "in block map.", b);
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
+                "Postponed mis-replicated block {} no longer found " +
+                "in block map.", b);
+          }
           continue;
         }
         MisReplicationResult res = processMisReplicatedBlock(bi);
-        LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
-            "Re-scanned block {}, result is {}", b, res);
+        LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: Re-scanned block {}, result is {}",
+            b, res);
         if (res == MisReplicationResult.POSTPONE) {
           rescannedMisreplicatedBlocks.add(b);
         }
@@ -2872,7 +3010,7 @@ void rescanPostponedMisreplicatedBlocks() {
       postponedMisreplicatedBlocks.addAll(rescannedMisreplicatedBlocks);
       rescannedMisreplicatedBlocks.clear();
       long endSize = postponedMisreplicatedBlocks.size();
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("rescanPostponedMisreplicatedBlocks");
       LOG.info("Rescan of postponedMisreplicatedBlocks completed in {}" +
           " msecs. {} blocks are left. {} blocks were removed.",
           (Time.monotonicNow() - startTime), endSize, (startSize - endSize));
@@ -2955,9 +3093,11 @@ public void markBlockReplicasAsCorrupt(Block oldBlock,
         }
       }
       if (isCorrupt) {
-        blockLog.debug("BLOCK* markBlockReplicasAsCorrupt: mark block replica" +
-            " {} on {} as corrupt because the dn is not in the new committed " +
-            "storage list.", b, storage.getDatanodeDescriptor());
+        if (blockLog.isDebugEnabled()) {
+          blockLog.debug("BLOCK* markBlockReplicasAsCorrupt: mark block replica" +
+              " {} on {} as corrupt because the dn is not in the new committed " +
+              "storage list.", b, storage.getDatanodeDescriptor());
+        }
         markBlockAsCorrupt(b, storage, storage.getDatanodeDescriptor());
       }
     }
@@ -2989,6 +3129,7 @@ void processFirstBlockReport(
             iblk.getBlockName(), storageInfo.getDatanodeDescriptor(),
             iblk.getNumBytes(), reportedState);
       }
+
       if (shouldPostponeBlocksFromFuture && isGenStampInFuture(iblk)) {
         queueReportedBlock(storageInfo, iblk, reportedState,
             QUEUE_REASON_FUTURE_GENSTAMP);
@@ -3205,9 +3346,11 @@ private void queueReportedBlock(DatanodeStorageInfo storageInfo, Block block,
       ReplicaState reportedState, String reason) {
     assert shouldPostponeBlocksFromFuture;
 
-    LOG.debug("Queueing reported block {} in state {}" +
-            " from datanode {} for later processing because {}.",
-        block, reportedState, storageInfo.getDatanodeDescriptor(), reason);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Queueing reported block {} in state {}" +
+              " from datanode {} for later processing because {}.",
+          block, reportedState, storageInfo.getDatanodeDescriptor(), reason);
+    }
     pendingDNMessages.enqueueReportedBlock(storageInfo, block, reportedState);
   }
 
@@ -3474,9 +3617,8 @@ private Block addStoredBlock(final BlockInfo block,
     }
     if (storedBlock == null || storedBlock.isDeleted()) {
       // If this block does not belong to anyfile, then we are done.
-      blockLog.debug("BLOCK* addStoredBlock: {} on {} size {} but it does not" +
-          " belong to any file", block, node, block.getNumBytes());
-
+      blockLog.debug("BLOCK* addStoredBlock: {} on {} size {} but it does not belong to any file",
+          reportedBlock, node, reportedBlock.getNumBytes());
       // we could add this block to invalidate set of this datanode.
       // it will happen in next block report otherwise.
       return block;
@@ -3490,13 +3632,13 @@ private Block addStoredBlock(final BlockInfo block,
       curReplicaDelta =
           (node.isDecommissioned() || node.isDecommissionInProgress()) ? 0 : 1;
       if (logEveryBlock) {
-        blockLog.debug("BLOCK* addStoredBlock: {} is added to {} (size={})",
-            node, storedBlock, storedBlock.getNumBytes());
+        blockLog.info("BLOCK* addStoredBlock: {} is added to {} (size={})",
+            node, reportedBlock, reportedBlock.getNumBytes());
       }
     } else if (result == AddBlockResult.REPLACED) {
       curReplicaDelta = 0;
       blockLog.warn("BLOCK* addStoredBlock: block {} moved to storageType " +
-          "{} on node {}", storedBlock, storageInfo.getStorageType(), node);
+          "{} on node {}", reportedBlock, storageInfo.getStorageType(), node);
     } else {
       // if the same block is added again and the replica was corrupt
       // previously because of a wrong gen stamp, remove it from the
@@ -3504,9 +3646,11 @@ private Block addStoredBlock(final BlockInfo block,
       corruptReplicas.removeFromCorruptReplicasMap(block, node,
           Reason.GENSTAMP_MISMATCH);
       curReplicaDelta = 0;
-      blockLog.debug("BLOCK* addStoredBlock: Redundant addStoredBlock request"
-              + " received for {} on node {} size {}", storedBlock, node,
-          storedBlock.getNumBytes());
+      if (blockLog.isDebugEnabled()) {
+        blockLog.debug("BLOCK* addStoredBlock: Redundant addStoredBlock request"
+                + " received for {} on node {} size {}", reportedBlock, node,
+            reportedBlock.getNumBytes());
+      }
     }
 
     // Now check for completion of blocks and safe block count
@@ -3561,7 +3705,7 @@ private Block addStoredBlock(final BlockInfo block,
           ". blockMap has {} but corrupt replicas map has {}",
           storedBlock, numCorruptNodes, corruptReplicasCount);
     }
-    if ((corruptReplicasCount > 0) && (numLiveReplicas >= fileRedundancy)) {
+    if ((corruptReplicasCount > 0) && (numUsableReplicas >= fileRedundancy)) {
       invalidateCorruptReplicas(storedBlock, reportedBlock, num);
     }
     return storedBlock;
@@ -3607,8 +3751,8 @@ private void invalidateCorruptReplicas(BlockInfo blk, Block reported,
           removedFromBlocksMap = false;
         }
       } catch (IOException e) {
-        blockLog.debug("invalidateCorruptReplicas error in deleting bad block"
-            + " {} on {}", blk, node, e);
+        blockLog.debug("invalidateCorruptReplicas error in deleting bad block {} on {}",
+            blk, node, e);
         removedFromBlocksMap = false;
       }
     }
@@ -3739,7 +3883,7 @@ private void processMisReplicatesAsync() throws InterruptedException {
           break;
         }
       } finally {
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("processMisReplicatesAsync");
         // Make sure it is out of the write lock for sufficiently long time.
         Thread.sleep(sleepDuration);
       }
@@ -3790,11 +3934,11 @@ public int processMisReplicatedBlocks(List<BlockInfo> blocks) {
             BlockInfo blk = iter.next();
             MisReplicationResult r = processMisReplicatedBlock(blk);
             processed++;
-            LOG.debug("BLOCK* processMisReplicatedBlocks: " +
-                    "Re-scanned block {}, result is {}", blk, r);
+            LOG.debug("BLOCK* processMisReplicatedBlocks: Re-scanned block {}, result is {}",
+                blk, r);
           }
         } finally {
-          namesystem.writeUnlock();
+          namesystem.writeUnlock("processMisReplicatedBlocks");
         }
       }
     } catch (InterruptedException ex) {
@@ -4024,6 +4168,14 @@ private void chooseExcessRedundancyStriped(BlockCollection bc,
         List<DatanodeStorageInfo> replicasToDelete = placementPolicy
             .chooseReplicasToDelete(nonExcess, candidates, (short) 1,
                 excessTypes, null, null);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Choose redundant EC replicas to delete from blk_{} which is located in {}",
+              sblk.getBlockId(), storage2index);
+          LOG.debug("Storages with candidate blocks to be deleted: {}", candidates);
+          LOG.debug("Storages with blocks to be deleted: {}", replicasToDelete);
+        }
+        Preconditions.checkArgument(candidates.containsAll(replicasToDelete),
+            "The EC replicas to be deleted are not in the candidate list");
         for (DatanodeStorageInfo chosen : replicasToDelete) {
           processChosenExcessRedundancy(nonExcess, chosen, storedBlock);
           candidates.remove(chosen);
@@ -4049,8 +4201,8 @@ private void processChosenExcessRedundancy(
     //
     final Block blockToInvalidate = getBlockOnStorage(storedBlock, chosen);
     addToInvalidates(blockToInvalidate, chosen.getDatanodeDescriptor());
-    blockLog.debug("BLOCK* chooseExcessRedundancies: "
-        + "({}, {}) is added to invalidated blocks set", chosen, storedBlock);
+    blockLog.debug("BLOCK* chooseExcessRedundancies: ({}, {}) is added to invalidated blocks set",
+        chosen, storedBlock);
   }
 
   private void removeStoredBlock(DatanodeStorageInfo storageInfo, Block block,
@@ -4072,8 +4224,8 @@ public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
     assert (namesystem.hasWriteLock());
     {
       if (storedBlock == null || !blocksMap.removeNode(storedBlock, node)) {
-        blockLog.debug("BLOCK* removeStoredBlock: {} has already been" +
-            " removed from node {}", storedBlock, node);
+        blockLog.debug("BLOCK* removeStoredBlock: {} has already been removed from node {}",
+            storedBlock, node);
         return;
       }
 
@@ -4085,8 +4237,10 @@ public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
         removed |= node.getCached().remove(cblock);
         removed |= node.getPendingUncached().remove(cblock);
         if (removed) {
-          blockLog.debug("BLOCK* removeStoredBlock: {} removed from caching "
-              + "related lists on node {}", storedBlock, node);
+          if (blockLog.isDebugEnabled()) {
+            blockLog.debug("BLOCK* removeStoredBlock: {} removed from caching "
+                + "related lists on node {}", storedBlock, node);
+          }
         }
       }
 
@@ -4111,9 +4265,9 @@ private void removeStaleReplicas(List<ReplicaUnderConstruction> staleReplicas,
     for (ReplicaUnderConstruction r : staleReplicas) {
       removeStoredBlock(block,
           r.getExpectedStorageLocation().getDatanodeDescriptor());
-      NameNode.blockStateChangeLog
-          .debug("BLOCK* Removing stale replica {}" + " of {}", r,
-              Block.toString(r));
+      if (blockLog.isDebugEnabled()) {
+        blockLog.debug("BLOCK* Removing stale replica {} of {}", r, Block.toString(r));
+      }
     }
   }
   /**
@@ -4241,8 +4395,8 @@ private boolean processAndHandleReportedBlock(
           maxNumBlocksToLog, numBlocksLogged);
     }
     for (Block b : toInvalidate) {
-      blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not " +
-          "belong to any file", b, node, b.getNumBytes());
+      blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not belong to any file",
+          b, node, b.getNumBytes());
       addToInvalidates(b, node);
     }
     for (BlockToMarkCorrupt b : toCorrupt) {
@@ -4323,9 +4477,11 @@ private void processIncrementalBlockReport(final DatanodeDescriptor node,
       blockLog.debug("BLOCK* block {}: {} is received from {}",
           rdbi.getStatus(), rdbi.getBlock(), node);
     }
-    blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from "
-            + "{} receiving: {}, received: {}, deleted: {}", node, receiving,
-        received, deleted);
+    if (blockLog.isDebugEnabled()) {
+      blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from "
+              + "{} receiving: {}, received: {}, deleted: {}", node, receiving,
+          received, deleted);
+    }
   }
 
   /**
@@ -4509,7 +4665,7 @@ void processExtraRedundancyBlocksOnInService(
       // testPlacementWithLocalRackNodesDecommissioned, it is not protected by
       // lock, only when called by DatanodeManager.refreshNodes have writeLock
       if (namesystem.hasWriteLock()) {
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("processExtraRedundancyBlocksOnInService");
         try {
           Thread.sleep(1);
         } catch (InterruptedException e) {
@@ -4541,7 +4697,7 @@ boolean isNodeHealthyForDecommissionOrMaintenance(DatanodeDescriptor node) {
     if (pendingReconstructionBlocksCount == 0 &&
         lowRedundancyBlocksCount == 0) {
       LOG.info("Node {} is dead and there are no low redundancy" +
-          " blocks or blocks pending reconstruction. Safe to decommission or",
+          " blocks or blocks pending reconstruction. Safe to decommission or" +
           " put in maintenance.", node);
       return true;
     }
@@ -4641,7 +4797,7 @@ private void updateNeededReconstructions(final BlockInfo block,
             repl.outOfServiceReplicas(), oldExpectedReplicas);
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("updateNeededReconstructions");
     }
   }
 
@@ -4698,10 +4854,12 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) {
         return 0;
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("invalidateWorkForOneNode");
+    }
+    if (blockLog.isDebugEnabled()) {
+      blockLog.debug("BLOCK* {}: ask {} to delete {}",
+          getClass().getSimpleName(), dn, toInvalidate);
     }
-    blockLog.debug("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(),
-        dn, toInvalidate);
     return toInvalidate.size();
   }
 
@@ -4909,6 +5067,77 @@ public long getLastRedundancyMonitorTS() {
     return lastRedundancyCycleTS.get();
   }
 
+  /**
+   * Periodically deletes the marked block.
+   */
+  private class MarkedDeleteBlockScrubber implements Runnable {
+    private Iterator<BlockInfo> toDeleteIterator = null;
+    private boolean isSleep;
+    private NameNodeMetrics metrics;
+
+    private void remove(long time) {
+      if (checkToDeleteIterator()) {
+        namesystem.writeLock();
+        try {
+          while (toDeleteIterator.hasNext()) {
+            removeBlock(toDeleteIterator.next());
+            metrics.decrPendingDeleteBlocksCount();
+            if (Time.monotonicNow() - time > deleteBlockLockTimeMs) {
+              isSleep = true;
+              break;
+            }
+          }
+        } finally {
+          namesystem.writeUnlock("markedDeleteBlockScrubberThread");
+        }
+      }
+    }
+
+    private boolean checkToDeleteIterator() {
+      return toDeleteIterator != null && toDeleteIterator.hasNext();
+    }
+
+    @Override
+    public void run() {
+      LOG.info("Start MarkedDeleteBlockScrubber thread");
+      while (namesystem.isRunning() &&
+          !Thread.currentThread().isInterrupted()) {
+        if (!markedDeleteQueue.isEmpty() || checkToDeleteIterator()) {
+          try {
+            metrics = NameNode.getNameNodeMetrics();
+            metrics.setDeleteBlocksQueued(markedDeleteQueue.size());
+            isSleep = false;
+            long startTime = Time.monotonicNow();
+            remove(startTime);
+            while (!isSleep && !markedDeleteQueue.isEmpty() &&
+                !Thread.currentThread().isInterrupted()) {
+              List<BlockInfo> markedDeleteList = markedDeleteQueue.poll();
+              if (markedDeleteList != null) {
+                toDeleteIterator = markedDeleteList.listIterator();
+              }
+              remove(startTime);
+            }
+          } catch (Exception e){
+            LOG.warn("MarkedDeleteBlockScrubber encountered an exception" +
+                " during the block deletion process, " +
+                " the deletion of the block will retry in {} millisecond.",
+                deleteBlockUnlockIntervalTimeMs, e);
+          }
+        }
+        if (isSleep) {
+          LOG.debug("Clear markedDeleteQueue over {} millisecond to release the write lock",
+              deleteBlockLockTimeMs);
+        }
+        try {
+          Thread.sleep(deleteBlockUnlockIntervalTimeMs);
+        } catch (InterruptedException e) {
+          LOG.info("Stopping MarkedDeleteBlockScrubber.");
+          break;
+        }
+      }
+    }
+  }
+
   /**
    * Periodically calls computeBlockRecoveryWork().
    */
@@ -4977,7 +5206,7 @@ int computeDatanodeWork() {
       this.updateState();
       this.scheduledReplicationBlocksCount = workFound;
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("computeDatanodeWork");
     }
     workFound += this.computeInvalidateWork(nodesToProcess);
     return workFound;
@@ -5217,7 +5446,7 @@ private void processQueue() {
               action = queue.poll();
             } while (action != null);
           } finally {
-            namesystem.writeUnlock();
+            namesystem.writeUnlock("processQueue");
             metrics.addBlockOpsBatched(processed - 1);
           }
         } catch (InterruptedException e) {
@@ -5257,6 +5486,17 @@ public BlockIdManager getBlockIdManager() {
     return blockIdManager;
   }
 
+  @VisibleForTesting
+  public ConcurrentLinkedQueue<List<BlockInfo>> getMarkedDeleteQueue() {
+    return markedDeleteQueue;
+  }
+
+  public void addBLocksToMarkedDeleteQueue(List<BlockInfo> blockInfos) {
+    markedDeleteQueue.add(blockInfos);
+    NameNode.getNameNodeMetrics().
+        incrPendingDeleteBlocksCount(blockInfos.size());
+  }
+
   public long nextGenerationStamp(boolean legacyBlock) throws IOException {
     return blockIdManager.nextGenerationStamp(legacyBlock);
   }
@@ -5360,4 +5600,14 @@ public void disableSPS() {
   public StoragePolicySatisfyManager getSPSManager() {
     return spsManager;
   }
+
+  public void setExcludeSlowNodesEnabled(boolean enable) {
+    placementPolicies.getPolicy(CONTIGUOUS).setExcludeSlowNodesEnabled(enable);
+    placementPolicies.getPolicy(STRIPED).setExcludeSlowNodesEnabled(enable);
+  }
+
+  @VisibleForTesting
+  public boolean getExcludeSlowNodesEnabled(BlockType blockType) {
+    return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java
index 7b1f04a64db9c..652cb0439a56b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerFaultInjector.java
@@ -49,4 +49,8 @@ public void requestBlockReportLease(DatanodeDescriptor node, long leaseId) {
   @VisibleForTesting
   public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) {
   }
+
+  @VisibleForTesting
+  public void mockAnException() {
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
index 6fac6dc2abec2..4349ba01401d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.util.Daemon;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -649,8 +649,8 @@ private SafeModeMonitor(Configuration conf) {
           DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT);
       if (recheckInterval < 1) {
         LOG.warn("Invalid value for " +
-            DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT +
-            ".Should be greater than 0, but is {}", recheckInterval);
+            DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY +
+            ". Should be greater than 0, but is {}", recheckInterval);
         recheckInterval = DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT;
       }
       LOG.info("Using {} as SafeModeMonitor Interval", recheckInterval);
@@ -670,7 +670,7 @@ public void run() {
             break;
           }
         } finally {
-          namesystem.writeUnlock();
+          namesystem.writeUnlock("leaveSafeMode");
         }
 
         try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 07df43b83dcf8..85468a57bde47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -25,7 +25,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.AddBlockFlag;
@@ -196,8 +196,9 @@ public void adjustSetsWithChosenReplica(
     if (moreThanOne.remove(cur)) {
       if (storages.size() == 1) {
         final DatanodeStorageInfo remaining = storages.get(0);
-        moreThanOne.remove(remaining);
-        exactlyOne.add(remaining);
+        if (moreThanOne.remove(remaining)) {
+          exactlyOne.add(remaining);
+        }
       }
     } else {
       exactlyOne.remove(cur);
@@ -261,4 +262,16 @@ public <T> void splitNodesWithRack(
       }
     }
   }
+
+  /**
+   * Updates the value used for excludeSlowNodesEnabled, which is set by
+   * {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY}
+   * initially.
+   *
+   * @param enable true, we will filter out slow nodes
+   * when choosing targets for blocks, otherwise false not filter.
+   */
+  public abstract void setExcludeSlowNodesEnabled(boolean enable);
+
+  public abstract boolean getExcludeSlowNodesEnabled();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index c16fe59f596e6..1fef3db69d0fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -26,7 +26,7 @@
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.AddBlockFlag;
@@ -103,7 +103,7 @@ private String getText() {
   protected double considerLoadFactor;
   private boolean preferLocalNode;
   private boolean dataNodePeerStatsEnabled;
-  private boolean excludeSlowNodesEnabled;
+  private volatile boolean excludeSlowNodesEnabled;
   protected NetworkTopology clusterMap;
   protected Host2NodesMap host2datanodeMap;
   private FSClusterStats stats;
@@ -238,8 +238,8 @@ DatanodeStorageInfo[] chooseTarget(String src,
       return getPipeline(writer,
           results.toArray(new DatanodeStorageInfo[results.size()]));
     } catch (NotEnoughReplicasException nr) {
-      LOG.debug("Failed to choose with favored nodes (={}), disregard favored"
-          + " nodes hint and retry.", favoredNodes, nr);
+      LOG.debug("Failed to choose with favored nodes (={}), disregard favored nodes hint and retry",
+          favoredNodes, nr);
       // Fall back to regular block placement disregarding favored nodes hint
       return chooseTarget(src, numOfReplicas, writer, 
           new ArrayList<DatanodeStorageInfo>(numOfReplicas), false, 
@@ -304,7 +304,7 @@ private DatanodeStorageInfo[] chooseTarget(int numOfReplicas,
         && stats.isAvoidingStaleDataNodesForWrite());
     boolean avoidLocalRack = (addBlockFlags != null
         && addBlockFlags.contains(AddBlockFlag.NO_LOCAL_RACK) && writer != null
-        && clusterMap.getNumOfRacks() > 2);
+        && clusterMap.getNumOfNonEmptyRacks() > 2);
     boolean avoidLocalNode = (addBlockFlags != null
         && addBlockFlags.contains(AddBlockFlag.NO_LOCAL_WRITE)
         && writer != null
@@ -385,7 +385,7 @@ protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) {
       totalNumOfReplicas = clusterSize;
     }
     // No calculation needed when there is only one rack or picking one node.
-    int numOfRacks = clusterMap.getNumOfRacks();
+    int numOfRacks = clusterMap.getNumOfNonEmptyRacks();
     // HDFS-14527 return default when numOfRacks = 0 to avoid
     // ArithmeticException when calc maxNodesPerRack at following logic.
     if (numOfRacks <= 1 || totalNumOfReplicas <= 1) {
@@ -435,7 +435,7 @@ private EnumMap<StorageType, Integer> getRequiredStorageTypes(
    * @param storageTypes storage type to be considered for target
    * @return local node of writer (not chosen node)
    */
-  private Node chooseTarget(int numOfReplicas,
+  private Node chooseTarget(final int numOfReplicas,
                             Node writer,
                             final Set<Node> excludedNodes,
                             final long blocksize,
@@ -469,7 +469,7 @@ private Node chooseTarget(int numOfReplicas,
     LOG.trace("storageTypes={}", storageTypes);
 
     try {
-      if ((numOfReplicas = requiredStorageTypes.size()) == 0) {
+      if (requiredStorageTypes.size() == 0) {
         throw new NotEnoughReplicasException(
             "All required storage types are unavailable: "
             + " unavailableStorages=" + unavailableStorages
@@ -498,10 +498,10 @@ private Node chooseTarget(int numOfReplicas,
         for (DatanodeStorageInfo resultStorage : results) {
           addToExcludedNodes(resultStorage.getDatanodeDescriptor(), oldExcludedNodes);
         }
-        // Set numOfReplicas, since it can get out of sync with the result list
+        // Set newNumOfReplicas, since it can get out of sync with the result list
         // if the NotEnoughReplicasException was thrown in chooseRandom().
-        numOfReplicas = totalReplicasExpected - results.size();
-        return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
+        int newNumOfReplicas = totalReplicasExpected - results.size();
+        return chooseTarget(newNumOfReplicas, writer, oldExcludedNodes, blocksize,
             maxNodesPerRack, results, false, storagePolicy, unavailableStorages,
             newBlock, null);
       }
@@ -520,8 +520,8 @@ private Node chooseTarget(int numOfReplicas,
           addToExcludedNodes(resultStorage.getDatanodeDescriptor(),
               oldExcludedNodes);
         }
-        numOfReplicas = totalReplicasExpected - results.size();
-        return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize,
+        int newNumOfReplicas = totalReplicasExpected - results.size();
+        return chooseTarget(newNumOfReplicas, writer, oldExcludedNodes, blocksize,
             maxNodesPerRack, results, false, storagePolicy, unavailableStorages,
             newBlock, null);
       }
@@ -715,17 +715,19 @@ protected DatanodeStorageInfo chooseLocalRack(Node localMachine,
         DatanodeDescriptor nextNode = resultStorage.getDatanodeDescriptor();
         if (nextNode != localMachine) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Failed to choose from local rack (location = " + localRack
-                + "), retry with the rack of the next replica (location = "
-                + nextNode.getNetworkLocation() + ")", e);
+            LOG.debug("Failed to choose from local rack (location = {}), retry with the rack "
+                + "of the next replica (location = {})", localRack,
+                nextNode.getNetworkLocation(), e);
           }
           return chooseFromNextRack(nextNode, excludedNodes, blocksize,
               maxNodesPerRack, results, avoidStaleNodes, storageTypes);
         }
       }
 
-      LOG.debug("Failed to choose from local rack (location = {}); the second"
-          + " replica is not found, retry choosing randomly", localRack, e);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Failed to choose from local rack (location = {}); the second"
+            + " replica is not found, retry choosing randomly", localRack, e);
+      }
 
       //the second replica is not found, randomly choose one from the network
       return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
@@ -745,9 +747,10 @@ private DatanodeStorageInfo chooseFromNextRack(Node next,
       return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack,
           results, avoidStaleNodes, storageTypes);
     } catch (NotEnoughReplicasException e) {
-      LOG.debug("Failed to choose from the next rack (location = {}), "
-          + "retry choosing randomly", nextRack, e);
-        // otherwise randomly choose one from the network
+      LOG.debug("Failed to choose from the next rack (location = {}), retry choosing randomly",
+          nextRack, e);
+
+      // otherwise randomly choose one from the network
       return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
           maxNodesPerRack, results, avoidStaleNodes, storageTypes);
     }
@@ -775,10 +778,8 @@ protected void chooseRemoteRack(int numOfReplicas,
           excludedNodes, blocksize, maxReplicasPerRack, results,
           avoidStaleNodes, storageTypes);
     } catch (NotEnoughReplicasException e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Failed to choose remote rack (location = ~"
-            + localMachine.getNetworkLocation() + "), fallback to local rack", e);
-      }
+      LOG.debug("Failed to choose remote rack (location = ~{}), fallback to local rack",
+          localMachine.getNetworkLocation(), e);
       chooseRandom(numOfReplicas-(results.size()-oldNumOfReplicas),
                    localMachine.getNetworkLocation(), excludedNodes, blocksize, 
                    maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
@@ -1173,7 +1174,7 @@ public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
         .map(dn -> dn.getNetworkLocation()).distinct().count();
 
     return new BlockPlacementStatusDefault(Math.toIntExact(rackCount),
-        minRacks, clusterMap.getNumOfRacks());
+        minRacks, clusterMap.getNumOfNonEmptyRacks());
   }
 
   /**
@@ -1276,8 +1277,7 @@ public List<DatanodeStorageInfo> chooseReplicasToDelete(
       firstOne = false;
       if (cur == null) {
         LOG.debug(
-            "No excess replica can be found. excessTypes: {}. "
-                + "moreThanOne: {}. exactlyOne: {}.",
+            "No excess replica can be found. excessTypes: {}. moreThanOne: {}. exactlyOne: {}.",
             excessTypes, moreThanOne, exactlyOne);
         break;
       }
@@ -1359,5 +1359,14 @@ protected Collection<DatanodeStorageInfo> pickupReplicaSet(
   void setPreferLocalNode(boolean prefer) {
     this.preferLocalNode = prefer;
   }
-}
 
+  @Override
+  public void setExcludeSlowNodesEnabled(boolean enable) {
+    this.excludeSlowNodesEnabled = enable;
+  }
+
+  @Override
+  public boolean getExcludeSlowNodesEnabled() {
+    return excludeSlowNodesEnabled;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
index dad877fdc76fe..a3b3f482e8c23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
@@ -42,7 +42,7 @@ protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) {
       totalNumOfReplicas = clusterSize;
     }
     // No calculation needed when there is only one rack or picking one node.
-    int numOfRacks = clusterMap.getNumOfRacks();
+    int numOfRacks = clusterMap.getNumOfNonEmptyRacks();
     // HDFS-14527 return default when numOfRacks = 0 to avoid
     // ArithmeticException when calc maxNodesPerRack at following logic.
     if (numOfRacks <= 1 || totalNumOfReplicas <= 1) {
@@ -90,38 +90,39 @@ protected Node chooseTargetInOrder(int numOfReplicas,
                                  EnumMap<StorageType, Integer> storageTypes)
                                  throws NotEnoughReplicasException {
     int totalReplicaExpected = results.size() + numOfReplicas;
-    int numOfRacks = clusterMap.getNumOfRacks();
-    if (totalReplicaExpected < numOfRacks ||
-        totalReplicaExpected % numOfRacks == 0) {
-      writer = chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
-          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
-      return writer;
-    }
+    int numOfRacks = clusterMap.getNumOfNonEmptyRacks();
 
-    assert totalReplicaExpected > (maxNodesPerRack -1) * numOfRacks;
+    try {
+      if (totalReplicaExpected < numOfRacks ||
+          totalReplicaExpected % numOfRacks == 0) {
+        writer = chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
+            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+        return writer;
+      }
 
-    // Calculate numOfReplicas for filling each rack exactly (maxNodesPerRack-1)
-    // replicas.
-    HashMap<String, Integer> rackCounts = new HashMap<>();
-    for (DatanodeStorageInfo dsInfo : results) {
-      String rack = dsInfo.getDatanodeDescriptor().getNetworkLocation();
-      Integer count = rackCounts.get(rack);
-      if (count != null) {
-        rackCounts.put(rack, count + 1);
-      } else {
-        rackCounts.put(rack, 1);
+      assert totalReplicaExpected > (maxNodesPerRack -1) * numOfRacks;
+
+      // Calculate numOfReplicas for filling each rack exactly (maxNodesPerRack-1)
+      // replicas.
+      HashMap<String, Integer> rackCounts = new HashMap<>();
+      for (DatanodeStorageInfo dsInfo : results) {
+        String rack = dsInfo.getDatanodeDescriptor().getNetworkLocation();
+        Integer count = rackCounts.get(rack);
+        if (count != null) {
+          rackCounts.put(rack, count + 1);
+        } else {
+          rackCounts.put(rack, 1);
+        }
       }
-    }
-    int excess = 0; // Sum of the above (maxNodesPerRack-1) part of nodes in results
-    for (int count : rackCounts.values()) {
-      if (count > maxNodesPerRack -1) {
-        excess += count - (maxNodesPerRack -1);
+      int excess = 0; // Sum of the above (maxNodesPerRack-1) part of nodes in results
+      for (int count : rackCounts.values()) {
+        if (count > maxNodesPerRack -1) {
+          excess += count - (maxNodesPerRack -1);
+        }
       }
-    }
-    numOfReplicas = Math.min(totalReplicaExpected - results.size(),
-        (maxNodesPerRack -1) * numOfRacks - (results.size() - excess));
+      numOfReplicas = Math.min(totalReplicaExpected - results.size(),
+          (maxNodesPerRack -1) * numOfRacks - (results.size() - excess));
 
-    try {
       // Try to spread the replicas as evenly as possible across racks.
       // This is done by first placing with (maxNodesPerRack-1), then spreading
       // the remainder by calling again with maxNodesPerRack.
@@ -243,7 +244,7 @@ public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
       racks.add(dn.getNetworkLocation());
     }
     return new BlockPlacementStatusDefault(racks.size(), numberOfReplicas,
-        clusterMap.getNumOfRacks());
+        clusterMap.getNumOfNonEmptyRacks());
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
index 3560a36dde1bc..2e7e78d14e2c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.util.Time;
@@ -190,8 +190,8 @@ public synchronized void register(DatanodeDescriptor dn) {
 
   private synchronized NodeData registerNode(DatanodeDescriptor dn) {
     if (nodes.containsKey(dn.getDatanodeUuid())) {
-      LOG.info("Can't register DN {} because it is already registered.",
-          dn.getDatanodeUuid());
+      LOG.info("Can't register DN {} ({}) because it is already registered.",
+          dn.getDatanodeUuid(), dn.getXferAddr());
       return null;
     }
     NodeData node = new NodeData(dn.getDatanodeUuid());
@@ -213,8 +213,8 @@ private synchronized void remove(NodeData node) {
   public synchronized void unregister(DatanodeDescriptor dn) {
     NodeData node = nodes.remove(dn.getDatanodeUuid());
     if (node == null) {
-      LOG.info("Can't unregister DN {} because it is not currently " +
-          "registered.", dn.getDatanodeUuid());
+      LOG.info("Can't unregister DN {} ({}) because it is not currently " +
+          "registered.", dn.getDatanodeUuid(), dn.getXferAddr());
       return;
     }
     remove(node);
@@ -224,7 +224,7 @@ public synchronized long requestLease(DatanodeDescriptor dn) {
     NodeData node = nodes.get(dn.getDatanodeUuid());
     if (node == null) {
       LOG.warn("DN {} ({}) requested a lease even though it wasn't yet " +
-          "registered.  Registering now.", dn.getDatanodeUuid(),
+          "registered. Registering now.", dn.getDatanodeUuid(),
           dn.getXferAddr());
       node = registerNode(dn);
     }
@@ -232,9 +232,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) {
       // The DataNode wants a new lease, even though it already has one.
       // This can happen if the DataNode is restarted in between requesting
       // a lease and using it.
-      LOG.debug("Removing existing BR lease 0x{} for DN {} in order to " +
+      LOG.debug("Removing existing BR lease 0x{} for DN {} ({}) in order to " +
                "issue a new one.", Long.toHexString(node.leaseId),
-               dn.getDatanodeUuid());
+               dn.getDatanodeUuid(), dn.getXferAddr());
     }
     remove(node);
     long monotonicNowMs = Time.monotonicNow();
@@ -248,9 +248,9 @@ public synchronized long requestLease(DatanodeDescriptor dn) {
           allLeases.append(prefix).append(cur.datanodeUuid);
           prefix = ", ";
         }
-        LOG.debug("Can't create a new BR lease for DN {}, because " +
-              "numPending equals maxPending at {}.  Current leases: {}",
-              dn.getDatanodeUuid(), numPending, allLeases.toString());
+        LOG.debug("Can't create a new BR lease for DN {} ({}), because " +
+              "numPending equals maxPending at {}. Current leases: {}",
+              dn.getDatanodeUuid(), dn.getXferAddr(), numPending, allLeases);
       }
       return 0;
     }
@@ -259,8 +259,8 @@ public synchronized long requestLease(DatanodeDescriptor dn) {
     node.leaseTimeMs = monotonicNowMs;
     pendingHead.addToEnd(node);
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Created a new BR lease 0x{} for DN {}.  numPending = {}",
-          Long.toHexString(node.leaseId), dn.getDatanodeUuid(), numPending);
+      LOG.debug("Created a new BR lease 0x{} for DN {} ({}). numPending = {}",
+          Long.toHexString(node.leaseId), dn.getDatanodeUuid(), dn.getXferAddr(), numPending);
     }
     return node.leaseId;
   }
@@ -293,36 +293,36 @@ private synchronized void pruneExpiredPending(long monotonicNowMs) {
   public synchronized boolean checkLease(DatanodeDescriptor dn,
                                          long monotonicNowMs, long id) {
     if (id == 0) {
-      LOG.debug("Datanode {} is using BR lease id 0x0 to bypass " +
-          "rate-limiting.", dn.getDatanodeUuid());
+      LOG.debug("Datanode {} ({}) is using BR lease id 0x0 to bypass " +
+          "rate-limiting.", dn.getDatanodeUuid(), dn.getXferAddr());
       return true;
     }
     NodeData node = nodes.get(dn.getDatanodeUuid());
     if (node == null) {
-      LOG.info("BR lease 0x{} is not valid for unknown datanode {}",
-          Long.toHexString(id), dn.getDatanodeUuid());
+      LOG.info("BR lease 0x{} is not valid for unknown datanode {} ({})",
+          Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr());
       return false;
     }
     if (node.leaseId == 0) {
-      LOG.warn("BR lease 0x{} is not valid for DN {}, because the DN " +
+      LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the DN " +
                "is not in the pending set.",
-               Long.toHexString(id), dn.getDatanodeUuid());
+               Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr());
       return false;
     }
     if (pruneIfExpired(monotonicNowMs, node)) {
-      LOG.warn("BR lease 0x{} is not valid for DN {}, because the lease " +
-               "has expired.", Long.toHexString(id), dn.getDatanodeUuid());
+      LOG.warn("BR lease 0x{} is not valid for DN {} ({}), because the lease " +
+          "has expired.", Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr());
       return false;
     }
     if (id != node.leaseId) {
-      LOG.warn("BR lease 0x{} is not valid for DN {}.  Expected BR lease 0x{}.",
-          Long.toHexString(id), dn.getDatanodeUuid(),
+      LOG.warn("BR lease 0x{} is not valid for DN {} ({}). Expected BR lease 0x{}.",
+          Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(),
           Long.toHexString(node.leaseId));
       return false;
     }
     if (LOG.isTraceEnabled()) {
-      LOG.trace("BR lease 0x{} is valid for DN {}.",
-          Long.toHexString(id), dn.getDatanodeUuid());
+      LOG.trace("BR lease 0x{} is valid for DN {} ({}).",
+          Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr());
     }
     return true;
   }
@@ -330,20 +330,20 @@ public synchronized boolean checkLease(DatanodeDescriptor dn,
   public synchronized long removeLease(DatanodeDescriptor dn) {
     NodeData node = nodes.get(dn.getDatanodeUuid());
     if (node == null) {
-      LOG.info("Can't remove lease for unknown datanode {}",
-               dn.getDatanodeUuid());
+      LOG.info("Can't remove lease for unknown datanode {} ({})",
+          dn.getDatanodeUuid(), dn.getXferAddr());
       return 0;
     }
     long id = node.leaseId;
     if (id == 0) {
-      LOG.debug("DN {} has no lease to remove.", dn.getDatanodeUuid());
+      LOG.debug("DN {} ({}) has no lease to remove.", dn.getDatanodeUuid(), dn.getXferAddr());
       return 0;
     }
     remove(node);
     deferredHead.addToEnd(node);
     if (LOG.isTraceEnabled()) {
-      LOG.trace("Removed BR lease 0x{} for DN {}.  numPending = {}",
-                Long.toHexString(id), dn.getDatanodeUuid(), numPending);
+      LOG.trace("Removed BR lease 0x{} for DN {} ({}). numPending = {}",
+          Long.toHexString(id), dn.getDatanodeUuid(), dn.getXferAddr(), numPending);
     }
     return id;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index 419dcfcddf1b9..35d3b0a20f8fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java
index 3ce5ef07acdca..bcbd94cd3cc7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java
@@ -19,7 +19,7 @@
 
 import static org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.Block;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index eab58124cb11c..1e5f952040d53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -55,7 +55,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Scans the namesystem, scheduling blocks to be cached as appropriate.
@@ -302,7 +302,7 @@ private void rescan() throws InterruptedException {
       rescanCachedBlockMap();
       blockManager.getDatanodeManager().resetLastCachingDirectiveSentTime();
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("cacheReplicationMonitorRescan");
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
index c04f3daabf70e..79d5a065b08a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
 import org.apache.hadoop.hdfs.util.LightWeightHashSet;
 import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.util.HashMap;
@@ -32,8 +33,7 @@
 import java.util.List;
 import java.util.Iterator;
 import java.util.LinkedList;
-import java.util.ArrayDeque;
-import java.util.Queue;
+import java.util.stream.Collectors;
 
 /**
  * This class implements the logic to track decommissioning and entering
@@ -71,16 +71,10 @@ public class DatanodeAdminBackoffMonitor extends DatanodeAdminMonitorBase
       outOfServiceNodeBlocks = new HashMap<>();
 
   /**
-   * Any nodes where decommission or maintenance has been cancelled are added
-   * to this queue for later processing.
-   */
-  private final Queue<DatanodeDescriptor> cancelledNodes = new ArrayDeque<>();
-
-  /**
-   * The numbe of blocks to process when moving blocks to pendingReplication
+   * The number of blocks to process when moving blocks to pendingReplication
    * before releasing and reclaiming the namenode lock.
    */
-  private int blocksPerLock;
+  private volatile int blocksPerLock;
 
   /**
    * The number of blocks that have been checked on this tick.
@@ -89,7 +83,7 @@ public class DatanodeAdminBackoffMonitor extends DatanodeAdminMonitorBase
   /**
    * The maximum number of blocks to hold in PendingRep at any time.
    */
-  private int pendingRepLimit;
+  private volatile int pendingRepLimit;
 
   /**
    * The list of blocks which have been placed onto the replication queue
@@ -149,8 +143,8 @@ protected void processConf() {
    */
   @Override
   public void stopTrackingNode(DatanodeDescriptor dn) {
-    pendingNodes.remove(dn);
-    cancelledNodes.add(dn);
+    getPendingNodes().remove(dn);
+    getCancelledNodes().add(dn);
   }
 
   @Override
@@ -189,9 +183,32 @@ public void run() {
          * node will be removed from tracking by the pending cancel.
          */
         processCancelledNodes();
+
+        // Having more nodes decommissioning than can be tracked will impact decommissioning
+        // performance due to queueing delay
+        int numTrackedNodes = outOfServiceNodeBlocks.size();
+        int numQueuedNodes = getPendingNodes().size();
+        int numDecommissioningNodes = numTrackedNodes + numQueuedNodes;
+        if (numDecommissioningNodes > maxConcurrentTrackedNodes) {
+          LOG.warn(
+              "{} nodes are decommissioning but only {} nodes will be tracked at a time. "
+                  + "{} nodes are currently queued waiting to be decommissioned.",
+              numDecommissioningNodes, maxConcurrentTrackedNodes, numQueuedNodes);
+
+          // Re-queue unhealthy nodes to make space for decommissioning healthy nodes
+          final List<DatanodeDescriptor> unhealthyDns = outOfServiceNodeBlocks.keySet().stream()
+              .filter(dn -> !blockManager.isNodeHealthyForDecommissionOrMaintenance(dn))
+              .collect(Collectors.toList());
+          getUnhealthyNodesToRequeue(unhealthyDns, numDecommissioningNodes).forEach(dn -> {
+            getPendingNodes().add(dn);
+            outOfServiceNodeBlocks.remove(dn);
+            pendingRep.remove(dn);
+          });
+        }
+
         processPendingNodes();
       } finally {
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("DatanodeAdminMonitorV2Thread");
       }
       // After processing the above, various parts of the check() method will
       // take and drop the read / write lock as needed. Aside from the
@@ -207,8 +224,8 @@ public void run() {
       LOG.info("Checked {} blocks this tick. {} nodes are now " +
           "in maintenance or transitioning state. {} nodes pending. {} " +
           "nodes waiting to be cancelled.",
-          numBlocksChecked, outOfServiceNodeBlocks.size(), pendingNodes.size(),
-          cancelledNodes.size());
+          numBlocksChecked, outOfServiceNodeBlocks.size(), getPendingNodes().size(),
+          getCancelledNodes().size());
     }
   }
 
@@ -220,10 +237,10 @@ public void run() {
    * the pendingNodes list from being modified externally.
    */
   private void processPendingNodes() {
-    while (!pendingNodes.isEmpty() &&
+    while (!getPendingNodes().isEmpty() &&
         (maxConcurrentTrackedNodes == 0 ||
             outOfServiceNodeBlocks.size() < maxConcurrentTrackedNodes)) {
-      outOfServiceNodeBlocks.put(pendingNodes.poll(), null);
+      outOfServiceNodeBlocks.put(getPendingNodes().poll(), null);
     }
   }
 
@@ -235,8 +252,8 @@ private void processPendingNodes() {
    * write lock to prevent the cancelledNodes list being modified externally.
    */
   private void processCancelledNodes() {
-    while(!cancelledNodes.isEmpty()) {
-      DatanodeDescriptor dn = cancelledNodes.poll();
+    while(!getCancelledNodes().isEmpty()) {
+      DatanodeDescriptor dn = getCancelledNodes().poll();
       outOfServiceNodeBlocks.remove(dn);
       pendingRep.remove(dn);
     }
@@ -321,12 +338,12 @@ private void processMaintenanceNodes() {
           // which added the node to the cancelled list. Therefore expired
           // maintenance nodes do not need to be added to the toRemove list.
           dnAdmin.stopMaintenance(dn);
-          namesystem.writeUnlock();
+          namesystem.writeUnlock("processMaintenanceNodes");
           namesystem.writeLock();
         }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("processMaintenanceNodes");
     }
   }
 
@@ -385,7 +402,7 @@ private void processCompletedNodes(List<DatanodeDescriptor> toRemove) {
         }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("processCompletedNodes");
     }
   }
 
@@ -507,7 +524,7 @@ private void moveBlocksToPending() {
           // replication
           if (blocksProcessed >= blocksPerLock) {
             blocksProcessed = 0;
-            namesystem.writeUnlock();
+            namesystem.writeUnlock("moveBlocksToPending");
             namesystem.writeLock();
           }
           blocksProcessed++;
@@ -529,7 +546,7 @@ private void moveBlocksToPending() {
         }
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("moveBlocksToPending");
     }
     LOG.debug("{} blocks are now pending replication", pendingCount);
   }
@@ -613,7 +630,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn,
     try {
       storage = dn.getStorageInfos();
     } finally {
-      namesystem.readUnlock();
+      namesystem.readUnlock("scanDatanodeStorage");
     }
 
     for (DatanodeStorageInfo s : storage) {
@@ -643,7 +660,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn,
           numBlocksChecked++;
         }
       } finally {
-        namesystem.readUnlock();
+        namesystem.readUnlock("scanDatanodeStorage");
       }
     }
   }
@@ -698,7 +715,7 @@ private void processPendingReplication() {
             suspectBlocks.getOutOfServiceBlockCount());
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("processPendingReplication");
     }
   }
 
@@ -785,6 +802,26 @@ private boolean isBlockReplicatedOk(DatanodeDescriptor datanode,
     return false;
   }
 
+  @VisibleForTesting
+  @Override
+  public int getPendingRepLimit() {
+    return pendingRepLimit;
+  }
+
+  public void setPendingRepLimit(int pendingRepLimit) {
+    this.pendingRepLimit = pendingRepLimit;
+  }
+
+  @VisibleForTesting
+  @Override
+  public int getBlocksPerLock() {
+    return blocksPerLock;
+  }
+
+  public void setBlocksPerLock(int blocksPerLock) {
+    this.blocksPerLock = blocksPerLock;
+  }
+
   static class BlockStats {
     private LightWeightHashSet<Long> openFiles =
         new LightWeightLinkedSet<>();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java
index dfa7bf4e8381b..94049b35dc480 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hdfs.util.LightWeightHashSet;
 import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
 import org.apache.hadoop.util.ChunkedArrayList;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -113,6 +114,15 @@ protected void processConf() {
       numBlocksPerCheck =
           DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_DEFAULT;
     }
+
+    final String deprecatedKey = "dfs.namenode.decommission.nodes.per.interval";
+    final String strNodes = conf.get(deprecatedKey);
+    if (strNodes != null) {
+      LOG.warn("Deprecated configuration key {} will be ignored.", deprecatedKey);
+      LOG.warn("Please update your configuration to use {} instead.",
+          DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY);
+    }
+
     LOG.info("Initialized the Default Decommission and Maintenance monitor");
   }
 
@@ -123,8 +133,8 @@ private boolean exceededNumBlocksPerCheck() {
 
   @Override
   public void stopTrackingNode(DatanodeDescriptor dn) {
-    pendingNodes.remove(dn);
-    outOfServiceNodeBlocks.remove(dn);
+    getPendingNodes().remove(dn);
+    getCancelledNodes().add(dn);
   }
 
   @Override
@@ -137,6 +147,28 @@ public int getNumNodesChecked() {
     return numNodesChecked;
   }
 
+  @VisibleForTesting
+  @Override
+  public int getPendingRepLimit() {
+    return 0;
+  }
+
+  @Override
+  public void setPendingRepLimit(int pendingRepLimit) {
+    // nothing.
+  }
+
+  @VisibleForTesting
+  @Override
+  public int getBlocksPerLock() {
+    return 0;
+  }
+
+  @Override
+  public void setBlocksPerLock(int blocksPerLock) {
+    // nothing.
+  }
+
   @Override
   public void run() {
     LOG.debug("DatanodeAdminMonitor is running.");
@@ -152,31 +184,46 @@ public void run() {
     // Check decommission or maintenance progress.
     namesystem.writeLock();
     try {
+      processCancelledNodes();
       processPendingNodes();
       check();
     } catch (Exception e) {
       LOG.warn("DatanodeAdminMonitor caught exception when processing node.",
           e);
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("DatanodeAdminMonitorThread");
     }
     if (numBlocksChecked + numNodesChecked > 0) {
       LOG.info("Checked {} blocks and {} nodes this tick. {} nodes are now " +
               "in maintenance or transitioning state. {} nodes pending.",
           numBlocksChecked, numNodesChecked, outOfServiceNodeBlocks.size(),
-          pendingNodes.size());
+          getPendingNodes().size());
     }
   }
 
   /**
-   * Pop datanodes off the pending list and into decomNodeBlocks,
+   * Pop datanodes off the pending priority queue and into decomNodeBlocks,
    * subject to the maxConcurrentTrackedNodes limit.
    */
   private void processPendingNodes() {
-    while (!pendingNodes.isEmpty() &&
+    while (!getPendingNodes().isEmpty() &&
         (maxConcurrentTrackedNodes == 0 ||
             outOfServiceNodeBlocks.size() < maxConcurrentTrackedNodes)) {
-      outOfServiceNodeBlocks.put(pendingNodes.poll(), null);
+      outOfServiceNodeBlocks.put(getPendingNodes().poll(), null);
+    }
+  }
+
+  /**
+   * Process any nodes which have had their decommission or maintenance mode
+   * cancelled by an administrator.
+   *
+   * This method must be executed under the write lock to prevent the
+   * internal structures being modified concurrently.
+   */
+  private void processCancelledNodes() {
+    while(!getCancelledNodes().isEmpty()) {
+      DatanodeDescriptor dn = getCancelledNodes().poll();
+      outOfServiceNodeBlocks.remove(dn);
     }
   }
 
@@ -185,6 +232,8 @@ private void check() {
         it = new CyclicIteration<>(outOfServiceNodeBlocks,
         iterkey).iterator();
     final List<DatanodeDescriptor> toRemove = new ArrayList<>();
+    final List<DatanodeDescriptor> unhealthyDns = new ArrayList<>();
+    boolean isValidState = true;
 
     while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem
         .isRunning()) {
@@ -221,6 +270,10 @@ private void check() {
           LOG.debug("Processing {} node {}", dn.getAdminState(), dn);
           pruneReliableBlocks(dn, blocks);
         }
+        final boolean isHealthy = blockManager.isNodeHealthyForDecommissionOrMaintenance(dn);
+        if (!isHealthy) {
+          unhealthyDns.add(dn);
+        }
         if (blocks.size() == 0) {
           if (!fullScan) {
             // If we didn't just do a full scan, need to re-check with the
@@ -236,8 +289,6 @@ private void check() {
           }
           // If the full scan is clean AND the node liveness is okay,
           // we can finally mark as DECOMMISSIONED or IN_MAINTENANCE.
-          final boolean isHealthy =
-              blockManager.isNodeHealthyForDecommissionOrMaintenance(dn);
           if (blocks.size() == 0 && isHealthy) {
             if (dn.isDecommissionInProgress()) {
               dnAdmin.setDecommissioned(dn);
@@ -247,6 +298,7 @@ private void check() {
               // to track maintenance expiration.
               dnAdmin.setInMaintenance(dn);
             } else {
+              isValidState  = false;
               Preconditions.checkState(false,
                   "Node %s is in an invalid state! "
                       + "Invalid state: %s %s blocks are on this dn.",
@@ -270,12 +322,36 @@ private void check() {
         // an invalid state.
         LOG.warn("DatanodeAdminMonitor caught exception when processing node "
             + "{}.", dn, e);
-        pendingNodes.add(dn);
+        if(isValidState){
+          getPendingNodes().add(dn);
+        } else {
+          LOG.warn("Ignoring the node {} which is in invalid state", dn);
+        }
         toRemove.add(dn);
+        unhealthyDns.remove(dn);
       } finally {
         iterkey = dn;
       }
     }
+
+    // Having more nodes decommissioning than can be tracked will impact decommissioning
+    // performance due to queueing delay
+    int numTrackedNodes = outOfServiceNodeBlocks.size() - toRemove.size();
+    int numQueuedNodes = getPendingNodes().size();
+    int numDecommissioningNodes = numTrackedNodes + numQueuedNodes;
+    if (numDecommissioningNodes > maxConcurrentTrackedNodes) {
+      LOG.warn(
+          "{} nodes are decommissioning but only {} nodes will be tracked at a time. "
+              + "{} nodes are currently queued waiting to be decommissioned.",
+          numDecommissioningNodes, maxConcurrentTrackedNodes, numQueuedNodes);
+
+      // Re-queue unhealthy nodes to make space for decommissioning healthy nodes
+      getUnhealthyNodesToRequeue(unhealthyDns, numDecommissioningNodes).forEach(dn -> {
+        getPendingNodes().add(dn);
+        outOfServiceNodeBlocks.remove(dn);
+      });
+    }
+
     // Remove the datanodes that are DECOMMISSIONED or in service after
     // maintenance expiration.
     for (DatanodeDescriptor dn : toRemove) {
@@ -350,7 +426,7 @@ private void processBlocksInternal(
         // lock.
         // Yielding is required in case of block number is greater than the
         // configured per-iteration-limit.
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("processBlocksInternal");
         try {
           LOG.debug("Yielded lock during decommission/maintenance check");
           Thread.sleep(0, 500);
@@ -367,8 +443,10 @@ private void processBlocksInternal(
       // Remove the block from the list if it's no longer in the block map,
       // e.g. the containing file has been deleted
       if (blockManager.blocksMap.getStoredBlock(block) == null) {
-        LOG.trace("Removing unknown block {}", block);
-        it.remove();
+        if (pruneReliableBlocks) {
+          LOG.trace("Removing unknown block {}", block);
+          it.remove();
+        }
         continue;
       }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
index fecec597f5a5f..af207a843fd27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
 import static org.apache.hadoop.util.Time.monotonicNow;
 
 import java.util.Queue;
@@ -30,6 +29,7 @@
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.Namesystem;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -105,35 +105,9 @@ void activate(Configuration conf) {
         DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
         DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT,
         TimeUnit.SECONDS);
-    checkArgument(intervalSecs >= 0, "Cannot set a negative " +
+    Preconditions.checkArgument(intervalSecs >= 0, "Cannot set a negative " +
         "value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY);
 
-    int blocksPerInterval = conf.getInt(
-        DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY,
-        DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_DEFAULT);
-
-    final String deprecatedKey =
-        "dfs.namenode.decommission.nodes.per.interval";
-    final String strNodes = conf.get(deprecatedKey);
-    if (strNodes != null) {
-      LOG.warn("Deprecated configuration key {} will be ignored.",
-          deprecatedKey);
-      LOG.warn("Please update your configuration to use {} instead.",
-          DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY);
-    }
-
-    checkArgument(blocksPerInterval > 0,
-        "Must set a positive value for "
-        + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY);
-
-    final int maxConcurrentTrackedNodes = conf.getInt(
-        DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES,
-        DFSConfigKeys
-            .DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES_DEFAULT);
-    checkArgument(maxConcurrentTrackedNodes >= 0, "Cannot set a negative " +
-        "value for "
-        + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES);
-
     Class cls = null;
     try {
       cls = conf.getClass(
@@ -152,10 +126,7 @@ void activate(Configuration conf) {
     executor.scheduleWithFixedDelay(monitor, intervalSecs, intervalSecs,
         TimeUnit.SECONDS);
 
-    LOG.debug("Activating DatanodeAdminManager with interval {} seconds, " +
-            "{} max blocks per interval, " +
-            "{} max concurrently tracked nodes.", intervalSecs,
-        blocksPerInterval, maxConcurrentTrackedNodes);
+    LOG.debug("Activating DatanodeAdminManager with interval {} seconds.", intervalSecs);
   }
 
   /**
@@ -177,6 +148,8 @@ public void startDecommission(DatanodeDescriptor node) {
     if (!node.isDecommissionInProgress() && !node.isDecommissioned()) {
       // Update DN stats maintained by HeartbeatManager
       hbManager.startDecommission(node);
+      // Update cluster's emptyRack
+      blockManager.getDatanodeManager().getNetworkTopology().decommissionNode(node);
       // hbManager.startDecommission will set dead node to decommissioned.
       if (node.isDecommissionInProgress()) {
         for (DatanodeStorageInfo storage : node.getStorageInfos()) {
@@ -201,6 +174,8 @@ public void stopDecommission(DatanodeDescriptor node) {
     if (node.isDecommissionInProgress() || node.isDecommissioned()) {
       // Update DN stats maintained by HeartbeatManager
       hbManager.stopDecommission(node);
+      // Update cluster's emptyRack
+      blockManager.getDatanodeManager().getNetworkTopology().recommissionNode(node);
       // extra redundancy blocks will be detected and processed when
       // the dead node comes back and send in its full block report.
       if (node.isAlive()) {
@@ -346,8 +321,7 @@ protected boolean isSufficient(BlockInfo block, BlockCollection bc,
         }
       }
     }
-    if (isMaintenance
-      && numLive >= blockManager.getMinReplicationToBeInMaintenance()) {
+    if (isMaintenance && numLive >= blockManager.getMinMaintenanceStorageNum(block)) {
       return true;
     }
     return false;
@@ -413,4 +387,30 @@ void runMonitorForTest() throws ExecutionException, InterruptedException {
     executor.submit(monitor).get();
   }
 
-}
\ No newline at end of file
+  public void refreshPendingRepLimit(int pendingRepLimit, String key) {
+    ensurePositiveInt(pendingRepLimit, key);
+    this.monitor.setPendingRepLimit(pendingRepLimit);
+  }
+
+  @VisibleForTesting
+  public int getPendingRepLimit() {
+    return this.monitor.getPendingRepLimit();
+  }
+
+  public void refreshBlocksPerLock(int blocksPerLock, String key) {
+    ensurePositiveInt(blocksPerLock, key);
+    this.monitor.setBlocksPerLock(blocksPerLock);
+  }
+
+  @VisibleForTesting
+  public int getBlocksPerLock() {
+    return this.monitor.getBlocksPerLock();
+  }
+
+  private void ensurePositiveInt(int val, String key) {
+    Preconditions.checkArgument(
+        (val > 0),
+        key + " = '" + val + "' is invalid. " +
+            "It should be a positive, non-zero integer value.");
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java
index 9eee241edddf8..1403161a0f5c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java
@@ -25,7 +25,11 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayDeque;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
 import java.util.Queue;
+import java.util.stream.Stream;
 
 /**
  * This abstract class provides some base methods which are inherited by
@@ -35,12 +39,26 @@
 public abstract class DatanodeAdminMonitorBase
     implements DatanodeAdminMonitorInterface, Configurable {
 
+  /**
+   * Sort by lastUpdate time descending order, such that unhealthy
+   * nodes are de-prioritized given they cannot be decommissioned.
+   */
+  static final Comparator<DatanodeDescriptor> PENDING_NODES_QUEUE_COMPARATOR =
+      (dn1, dn2) -> Long.compare(dn2.getLastUpdate(), dn1.getLastUpdate());
+
   protected BlockManager blockManager;
   protected Namesystem namesystem;
   protected DatanodeAdminManager dnAdmin;
   protected Configuration conf;
 
-  protected final Queue<DatanodeDescriptor> pendingNodes = new ArrayDeque<>();
+  private final PriorityQueue<DatanodeDescriptor> pendingNodes = new PriorityQueue<>(
+      PENDING_NODES_QUEUE_COMPARATOR);
+
+  /**
+   * Any nodes where decommission or maintenance has been cancelled are added
+   * to this queue for later processing.
+   */
+  private final Queue<DatanodeDescriptor> cancelledNodes = new ArrayDeque<>();
 
   /**
    * The maximum number of nodes to track in outOfServiceNodeBlocks.
@@ -105,6 +123,10 @@ public void setConf(Configuration conf) {
           DFSConfigKeys
               .DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES_DEFAULT;
     }
+
+    LOG.debug("Activating DatanodeAdminMonitor with {} max concurrently tracked nodes.",
+        maxConcurrentTrackedNodes);
+
     processConf();
   }
 
@@ -151,4 +173,39 @@ public int getPendingNodeCount() {
   public Queue<DatanodeDescriptor> getPendingNodes() {
     return pendingNodes;
   }
+
+  @Override
+  public Queue<DatanodeDescriptor> getCancelledNodes() {
+    return cancelledNodes;
+  }
+
+  /**
+   * If node "is dead while in Decommission In Progress", it cannot be decommissioned
+   * until it becomes healthy again. If there are more pendingNodes than can be tracked
+   * & some unhealthy tracked nodes, then re-queue the unhealthy tracked nodes
+   * to avoid blocking decommissioning of healthy nodes.
+   *
+   * @param unhealthyDns The unhealthy datanodes which may be re-queued
+   * @param numDecommissioningNodes The total number of nodes being decommissioned
+   * @return Stream of unhealthy nodes to be re-queued
+   */
+  Stream<DatanodeDescriptor> getUnhealthyNodesToRequeue(
+      final List<DatanodeDescriptor> unhealthyDns, int numDecommissioningNodes) {
+    if (!unhealthyDns.isEmpty()) {
+      // Compute the number of unhealthy nodes to re-queue
+      final int numUnhealthyNodesToRequeue =
+          Math.min(numDecommissioningNodes - maxConcurrentTrackedNodes, unhealthyDns.size());
+
+      LOG.warn("{} limit has been reached, re-queueing {} "
+              + "nodes which are dead while in Decommission In Progress.",
+          DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES,
+          numUnhealthyNodesToRequeue);
+
+      // Order unhealthy nodes by lastUpdate descending such that nodes
+      // which have been unhealthy the longest are preferred to be re-queued
+      return unhealthyDns.stream().sorted(PENDING_NODES_QUEUE_COMPARATOR.reversed())
+          .limit(numUnhealthyNodesToRequeue);
+    }
+    return Stream.empty();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorInterface.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorInterface.java
index f34c00587c78d..a4774742108fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorInterface.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorInterface.java
@@ -32,8 +32,17 @@ public interface DatanodeAdminMonitorInterface extends Runnable {
   int getTrackedNodeCount();
   int getNumNodesChecked();
   Queue<DatanodeDescriptor> getPendingNodes();
+  Queue<DatanodeDescriptor> getCancelledNodes();
 
   void setBlockManager(BlockManager bm);
   void setDatanodeAdminManager(DatanodeAdminManager dnm);
   void setNameSystem(Namesystem ns);
+
+  int getPendingRepLimit();
+
+  void setPendingRepLimit(int pendingRepLimit);
+
+  int getBlocksPerLock();
+
+  void setBlocksPerLock(int blocksPerLock);
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index f4ee01d4271a8..c77d54591a9f7 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -197,8 +197,10 @@ public Type getType() {
   /** A queue of blocks to be replicated by this datanode */
   private final BlockQueue<BlockTargetPair> replicateBlocks =
       new BlockQueue<>();
-  /** A queue of blocks to be erasure coded by this datanode */
-  private final BlockQueue<BlockECReconstructionInfo> erasurecodeBlocks =
+  /** A queue of ec blocks to be replicated by this datanode. */
+  private final BlockQueue<BlockTargetPair> ecBlocksToBeReplicated = new BlockQueue<>();
+  /** A queue of ec blocks to be erasure coded by this datanode. */
+  private final BlockQueue<BlockECReconstructionInfo> ecBlocksToBeErasureCoded =
       new BlockQueue<>();
   /** A queue of blocks to be recovered by this datanode */
   private final BlockQueue<BlockInfo> recoverBlocks = new BlockQueue<>();
@@ -358,7 +360,8 @@ public void clearBlockQueues() {
     }
     this.recoverBlocks.clear();
     this.replicateBlocks.clear();
-    this.erasurecodeBlocks.clear();
+    this.ecBlocksToBeReplicated.clear();
+    this.ecBlocksToBeErasureCoded.clear();
     // pendingCached, cached, and pendingUncached are protected by the
     // FSN lock.
     this.pendingCached.clear();
@@ -647,6 +650,17 @@ Iterator<BlockInfo> getBlockIterator(final int startBlock) {
     return new BlockIterator(startBlock, getStorageInfos());
   }
 
+  /**
+   * Get iterator, which starts iterating from the specified block and storages.
+   *
+   * @param startBlock on which blocks are start iterating
+   * @param storageInfos specified storages
+   */
+  Iterator<BlockInfo> getBlockIterator(
+      final int startBlock, final DatanodeStorageInfo[] storageInfos) {
+    return new BlockIterator(startBlock, storageInfos);
+  }
+
   @VisibleForTesting
   public void incrementPendingReplicationWithoutTargets() {
     pendingReplicationWithoutTargets++;
@@ -667,18 +681,27 @@ public void addBlockToBeReplicated(Block block,
     replicateBlocks.offer(new BlockTargetPair(block, targets));
   }
 
+  /**
+   * Store ec block to be replicated work.
+   */
+  @VisibleForTesting
+  public void addECBlockToBeReplicated(Block block, DatanodeStorageInfo[] targets) {
+    assert (block != null && targets != null && targets.length > 0);
+    ecBlocksToBeReplicated.offer(new BlockTargetPair(block, targets));
+  }
+
   /**
    * Store block erasure coding work.
    */
   void addBlockToBeErasureCoded(ExtendedBlock block,
       DatanodeDescriptor[] sources, DatanodeStorageInfo[] targets,
-      byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) {
+      byte[] liveBlockIndices, byte[] excludeReconstrutedIndices, ErasureCodingPolicy ecPolicy) {
     assert (block != null && sources != null && sources.length > 0);
     BlockECReconstructionInfo task = new BlockECReconstructionInfo(block,
-        sources, targets, liveBlockIndices, ecPolicy);
-    erasurecodeBlocks.offer(task);
+        sources, targets, liveBlockIndices, excludeReconstrutedIndices, ecPolicy);
+    ecBlocksToBeErasureCoded.offer(task);
     BlockManager.LOG.debug("Adding block reconstruction task " + task + "to "
-        + getName() + ", current queue size is " + erasurecodeBlocks.size());
+        + getName() + ", current queue size is " + ecBlocksToBeErasureCoded.size());
   }
 
   /**
@@ -709,7 +732,8 @@ void addBlocksToBeInvalidated(List<Block> blocklist) {
    * The number of work items that are pending to be replicated.
    */
   int getNumberOfBlocksToBeReplicated() {
-    return pendingReplicationWithoutTargets + replicateBlocks.size();
+    return pendingReplicationWithoutTargets + replicateBlocks.size()
+        + ecBlocksToBeReplicated.size();
   }
 
   /**
@@ -717,7 +741,15 @@ int getNumberOfBlocksToBeReplicated() {
    */
   @VisibleForTesting
   public int getNumberOfBlocksToBeErasureCoded() {
-    return erasurecodeBlocks.size();
+    return ecBlocksToBeErasureCoded.size();
+  }
+
+  /**
+   * The number of ec work items that are pending to be replicated.
+   */
+  @VisibleForTesting
+  public int getNumberOfECBlocksToBeReplicated() {
+    return ecBlocksToBeReplicated.size();
   }
 
   @VisibleForTesting
@@ -729,9 +761,13 @@ List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
     return replicateBlocks.poll(maxTransfers);
   }
 
+  List<BlockTargetPair> getECReplicatedCommand(int maxTransfers) {
+    return ecBlocksToBeReplicated.poll(maxTransfers);
+  }
+
   public List<BlockECReconstructionInfo> getErasureCodeCommand(
       int maxTransfers) {
-    return erasurecodeBlocks.poll(maxTransfers);
+    return ecBlocksToBeErasureCoded.poll(maxTransfers);
   }
 
   public BlockInfo[] getLeaseRecoveryCommand(int maxTransfers) {
@@ -983,7 +1019,11 @@ public String dumpDatanode() {
     if (repl > 0) {
       sb.append(" ").append(repl).append(" blocks to be replicated;");
     }
-    int ec = erasurecodeBlocks.size();
+    int ecRepl = ecBlocksToBeReplicated.size();
+    if (ecRepl > 0) {
+      sb.append(" ").append(ecRepl).append(" ec blocks to be replicated;");
+    }
+    int ec = ecBlocksToBeErasureCoded.size();
     if(ec > 0) {
       sb.append(" ").append(ec).append(" blocks to be erasure coded;");
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 23bcc98b7688f..88f3ac4e7c4cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -18,13 +18,13 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import static org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol.DNA_ERASURE_CODING_RECONSTRUCTION;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT;
 import static org.apache.hadoop.util.Time.monotonicNow;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
 
 import org.apache.hadoop.fs.StorageType;
@@ -141,7 +141,7 @@ public class DatanodeManager {
   private final boolean avoidStaleDataNodesForRead;
 
   /** Whether or not to avoid using slow DataNodes for reading. */
-  private final boolean avoidSlowDataNodesForRead;
+  private volatile boolean avoidSlowDataNodesForRead;
 
   /** Whether or not to consider lad for reading. */
   private final boolean readConsiderLoad;
@@ -195,10 +195,6 @@ public class DatanodeManager {
   private final HashMap<String, Integer> datanodesSoftwareVersions =
     new HashMap<>(4, 0.75f);
 
-  /**
-   * True if we should process latency metrics from downstream peers.
-   */
-  private final boolean dataNodePeerStatsEnabled;
   /**
    *  True if we should process latency metrics from individual DN disks.
    */
@@ -212,12 +208,11 @@ public class DatanodeManager {
   private static final String IP_PORT_SEPARATOR = ":";
 
   @Nullable
-  private final SlowPeerTracker slowPeerTracker;
+  private SlowPeerTracker slowPeerTracker;
   private static Set<String> slowNodesUuidSet = Sets.newConcurrentHashSet();
   private Daemon slowPeerCollectorDaemon;
   private final long slowPeerCollectionInterval;
-  private final int maxSlowPeerReportNodes;
-  private boolean excludeSlowNodesEnabled;
+  private volatile int maxSlowPeerReportNodes;
 
   @Nullable
   private final SlowDiskTracker slowDiskTracker;
@@ -250,19 +245,15 @@ public class DatanodeManager {
     this.datanodeAdminManager = new DatanodeAdminManager(namesystem,
         blockManager, heartbeatManager);
     this.fsClusterStats = newFSClusterStats();
-    this.dataNodePeerStatsEnabled = conf.getBoolean(
-        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
-        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
     this.dataNodeDiskStatsEnabled = Util.isDiskStatsEnabled(conf.getInt(
         DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
         DFSConfigKeys.
             DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT));
     final Timer timer = new Timer();
-    this.slowPeerTracker = dataNodePeerStatsEnabled ?
-        new SlowPeerTracker(conf, timer) : null;
-    this.excludeSlowNodesEnabled = conf.getBoolean(
-        DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY,
-        DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT);
+    final boolean dataNodePeerStatsEnabledVal =
+        conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
+            DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
+    initSlowPeerTracker(conf, timer, dataNodePeerStatsEnabledVal);
     this.maxSlowPeerReportNodes = conf.getInt(
         DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
         DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT);
@@ -270,7 +261,7 @@ public class DatanodeManager {
         DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_KEY,
         DFSConfigKeys.DFS_NAMENODE_SLOWPEER_COLLECT_INTERVAL_DEFAULT,
         TimeUnit.MILLISECONDS);
-    if (slowPeerTracker != null && excludeSlowNodesEnabled) {
+    if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
       startSlowPeerCollector();
     }
     this.slowDiskTracker = dataNodeDiskStatsEnabled ?
@@ -320,18 +311,12 @@ public class DatanodeManager {
         DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
     this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval
         + 10 * 1000 * heartbeatIntervalSeconds;
-    // Effected block invalidate limit is the bigger value between
-    // value configured in hdfs-site.xml, and 20 * HB interval.
     final int configuredBlockInvalidateLimit = conf.getInt(
         DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY,
         DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
-    final int countedBlockInvalidateLimit = 20*(int)(heartbeatIntervalSeconds);
-    this.blockInvalidateLimit = Math.max(countedBlockInvalidateLimit,
-        configuredBlockInvalidateLimit);
-    LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY
-        + ": configured=" + configuredBlockInvalidateLimit
-        + ", counted=" + countedBlockInvalidateLimit
-        + ", effected=" + blockInvalidateLimit);
+    // Block invalidate limit also has some dependency on heartbeat interval.
+    // Check setBlockInvalidateLimit().
+    setBlockInvalidateLimit(configuredBlockInvalidateLimit);
     this.checkIpHostnameInRegistration = conf.getBoolean(
         DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY,
         DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT);
@@ -378,6 +363,21 @@ public class DatanodeManager {
         DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT);
   }
 
+  /**
+   * Determines whether slow peer tracker should be enabled. If dataNodePeerStatsEnabledVal is
+   * true, slow peer tracker is initialized.
+   *
+   * @param conf The configuration to use while initializing slowPeerTracker.
+   * @param timer Timer object for slowPeerTracker.
+   * @param dataNodePeerStatsEnabled To determine whether slow peer tracking should be enabled.
+   */
+  public void initSlowPeerTracker(Configuration conf, Timer timer,
+      boolean dataNodePeerStatsEnabled) {
+    this.slowPeerTracker = dataNodePeerStatsEnabled ?
+        new SlowPeerTracker(conf, timer) :
+        new SlowPeerDisabledTracker(conf, timer);
+  }
+
   private void startSlowPeerCollector() {
     if (slowPeerCollectorDaemon != null) {
       return;
@@ -511,7 +511,25 @@ private boolean isInactive(DatanodeInfo datanode) {
   private boolean isSlowNode(String dnUuid) {
     return avoidSlowDataNodesForRead && slowNodesUuidSet.contains(dnUuid);
   }
-  
+
+  public void setAvoidSlowDataNodesForReadEnabled(boolean enable) {
+    this.avoidSlowDataNodesForRead = enable;
+  }
+
+  @VisibleForTesting
+  public boolean getEnableAvoidSlowDataNodesForRead() {
+    return this.avoidSlowDataNodesForRead;
+  }
+
+  public void setMaxSlowpeerCollectNodes(int maxNodes) {
+    this.maxSlowPeerReportNodes = maxNodes;
+  }
+
+  @VisibleForTesting
+  public int getMaxSlowpeerCollectNodes() {
+    return this.maxSlowPeerReportNodes;
+  }
+
   /**
    * Sort the non-striped located blocks by the distance to the target host.
    *
@@ -824,9 +842,7 @@ private void removeDatanode(DatanodeDescriptor nodeInfo,
     decrementVersionCount(nodeInfo.getSoftwareVersion());
     blockManager.getBlockReportLeaseManager().unregister(nodeInfo);
 
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("remove datanode " + nodeInfo);
-    }
+    LOG.debug("remove datanode {}.", nodeInfo);
     blockManager.checkSafeMode();
   }
 
@@ -846,7 +862,7 @@ public void removeDatanode(final DatanodeID node)
                                      + node + " does not exist");
       }
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("removeDatanode");
     }
   }
 
@@ -888,8 +904,8 @@ void addDatanode(final DatanodeDescriptor node) {
     resolveUpgradeDomain(node);
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug(getClass().getSimpleName() + ".addDatanode: "
-          + "node " + node + " is added to datanodeMap.");
+      LOG.debug("{}.addDatanode: node {} is added to datanodeMap.",
+          getClass().getSimpleName(), node);
     }
   }
 
@@ -900,9 +916,8 @@ private void wipeDatanode(final DatanodeID node) {
       host2DatanodeMap.remove(datanodeMap.remove(key));
     }
     if (LOG.isDebugEnabled()) {
-      LOG.debug(getClass().getSimpleName() + ".wipeDatanode("
-          + node + "): storage " + key 
-          + " is removed from datanodeMap.");
+      LOG.debug("{}.wipeDatanode({}): storage {} is removed from datanodeMap.",
+          getClass().getSimpleName(), node, key);
     }
   }
 
@@ -1165,15 +1180,13 @@ public void registerDatanode(DatanodeRegistration nodeReg)
         nodeN = null;
       }
   
+      boolean updateHost2DatanodeMap = false;
       if (nodeS != null) {
         if (nodeN == nodeS) {
           // The same datanode has been just restarted to serve the same data 
           // storage. We do not need to remove old data blocks, the delta will
           // be calculated on the next block report from the datanode
-          if(NameNode.stateChangeLog.isDebugEnabled()) {
-            NameNode.stateChangeLog.debug("BLOCK* registerDatanode: "
-                + "node restarted.");
-          }
+          NameNode.stateChangeLog.debug("BLOCK* registerDatanode: node restarted.");
         } else {
           // nodeS is found
           /* The registering datanode is a replacement node for the existing 
@@ -1183,7 +1196,11 @@ public void registerDatanode(DatanodeRegistration nodeReg)
             nodes with its data cleared (or user can just remove the StorageID
             value in "VERSION" file under the data directory of the datanode,
             but this is might not work if VERSION file format has changed 
-         */        
+         */
+          // Check if nodeS's host information is same as nodeReg's, if not,
+          // it needs to update host2DatanodeMap accordringly.
+          updateHost2DatanodeMap = !nodeS.getXferAddr().equals(nodeReg.getXferAddr());
+
           NameNode.stateChangeLog.info("BLOCK* registerDatanode: " + nodeS
               + " is replaced by " + nodeReg + " with the same storageID "
               + nodeReg.getDatanodeUuid());
@@ -1193,6 +1210,11 @@ nodes with its data cleared (or user can just remove the StorageID
         try {
           // update cluster map
           getNetworkTopology().remove(nodeS);
+
+          // Update Host2DatanodeMap
+          if (updateHost2DatanodeMap) {
+            getHost2DatanodeMap().remove(nodeS);
+          }
           if(shouldCountVersion(nodeS)) {
             decrementVersionCount(nodeS.getSoftwareVersion());
           }
@@ -1211,6 +1233,11 @@ nodes with its data cleared (or user can just remove the StorageID
             nodeS.setDependentHostNames(
                 getNetworkDependenciesWithDefault(nodeS));
           }
+
+          if (updateHost2DatanodeMap) {
+            getHost2DatanodeMap().add(nodeS);
+          }
+
           getNetworkTopology().add(nodeS);
           resolveUpgradeDomain(nodeS);
 
@@ -1290,7 +1317,7 @@ public void refreshNodes(final Configuration conf) throws IOException {
       refreshDatanodes();
       countSoftwareVersions();
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("refreshNodes");
     }
   }
 
@@ -1300,8 +1327,8 @@ private void refreshHostsReader(Configuration conf) throws IOException {
     // Update the file names and refresh internal includes and excludes list.
     if (conf == null) {
       conf = new HdfsConfiguration();
-      this.hostConfigManager.setConf(conf);
     }
+    this.hostConfigManager.setConf(conf);
     this.hostConfigManager.refresh();
   }
   
@@ -1502,9 +1529,11 @@ void checkIfClusterIsNowMultiRack(DatanodeDescriptor node) {
             "now be replicated cross-rack";
         LOG.info(message);
       } else {
-        message += "Not checking for mis-replicated blocks because this NN is " +
-            "not yet processing repl queues.";
-        LOG.debug(message);
+        if (LOG.isDebugEnabled()) {
+          message += "Not checking for mis-replicated blocks because this NN "
+              + "is not yet processing repl queues.";
+          LOG.debug(message);
+        }
       }
       hasClusterEverBeenMultiRack = true;
       if (blockManager.isPopulatingReplQueues()) {
@@ -1626,15 +1655,23 @@ public List<DatanodeDescriptor> getDatanodeListForReport(
     }
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("getDatanodeListForReport with " +
-          "includedNodes = " + hostConfigManager.getIncludes() +
-          ", excludedNodes = " + hostConfigManager.getExcludes() +
-          ", foundNodes = " + foundNodes +
-          ", nodes = " + nodes);
+      LOG.debug("getDatanodeListForReport with includedNodes = {}, excludedNodes = {}"
+              + ", foundNodes = {}, nodes = {}.", hostConfigManager.getIncludes(),
+          hostConfigManager.getExcludes(), foundNodes, nodes);
     }
     return nodes;
   }
-  
+
+  public List<DatanodeDescriptor> getAllSlowDataNodes() {
+    if (slowPeerTracker == null) {
+      LOG.debug("{} is disabled. Try enabling it first to capture slow peer outliers.",
+          DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY);
+      return ImmutableList.of();
+    }
+    List<String> slowNodes = slowPeerTracker.getSlowNodes(getNumOfDataNodes());
+    return getDnDescriptorsFromIpAddr(slowNodes);
+  }
+
   /**
    * Checks if name resolution was successful for the given address.  If IP
    * address and host name are the same, then it means name resolution has
@@ -1747,8 +1784,8 @@ private void addCacheCommands(String blockPoolId, DatanodeDescriptor nodeinfo,
   /** Handle heartbeat from datanodes. */
   public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
       StorageReport[] reports, final String blockPoolId,
-      long cacheCapacity, long cacheUsed, int xceiverCount, 
-      int maxTransfers, int failedVolumes,
+      long cacheCapacity, long cacheUsed, int xceiverCount,
+      int xmitsInProgress, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary,
       @Nonnull SlowPeerReports slowPeers,
       @Nonnull SlowDiskReports slowDisks) throws IOException {
@@ -1788,22 +1825,41 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
     // Allocate _approximately_ maxTransfers pending tasks to DataNode.
     // NN chooses pending tasks based on the ratio between the lengths of
     // replication and erasure-coded block queues.
-    int totalReplicateBlocks = nodeinfo.getNumberOfReplicateBlocks();
-    int totalECBlocks = nodeinfo.getNumberOfBlocksToBeErasureCoded();
-    int totalBlocks = totalReplicateBlocks + totalECBlocks;
+    int replicationBlocks = nodeinfo.getNumberOfReplicateBlocks();
+    int ecBlocksToBeReplicated = nodeinfo.getNumberOfECBlocksToBeReplicated();
+    int ecBlocksToBeErasureCoded = nodeinfo.getNumberOfBlocksToBeErasureCoded();
+    int totalBlocks = replicationBlocks + ecBlocksToBeReplicated + ecBlocksToBeErasureCoded;
     if (totalBlocks > 0) {
-      int numReplicationTasks = (int) Math.ceil(
-          (double) (totalReplicateBlocks * maxTransfers) / totalBlocks);
-      int numECTasks = (int) Math.ceil(
-          (double) (totalECBlocks * maxTransfers) / totalBlocks);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Pending replication tasks: " + numReplicationTasks
-            + " erasure-coded tasks: " + numECTasks);
+      int maxTransfers = blockManager.getMaxReplicationStreams() - xmitsInProgress;
+      int maxECReplicatedTransfers;
+      if (nodeinfo.isDecommissionInProgress()) {
+        maxECReplicatedTransfers = blockManager.getReplicationStreamsHardLimit()
+            - xmitsInProgress;
+      } else {
+        maxECReplicatedTransfers = maxTransfers;
       }
+      int numReplicationTasks = (int) Math.ceil(
+          (double) (replicationBlocks * maxTransfers) / totalBlocks);
+      int numEcReplicatedTasks = (int) Math.ceil(
+              (double) (ecBlocksToBeReplicated * maxECReplicatedTransfers) / totalBlocks);
+      int numECReconstructedTasks = (int) Math.ceil(
+          (double) (ecBlocksToBeErasureCoded * maxTransfers) / totalBlocks);
+      LOG.debug("Pending replication tasks: {} ec to be replicated tasks: {} " +
+                      "ec reconstruction tasks: {}.",
+          numReplicationTasks, numEcReplicatedTasks, numECReconstructedTasks);
       // check pending replication tasks
-      List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
+      List<BlockTargetPair> pendingReplicationList = nodeinfo.getReplicationCommand(
           numReplicationTasks);
-      if (pendingList != null && !pendingList.isEmpty()) {
+      List<BlockTargetPair> pendingECReplicatedList = nodeinfo.getECReplicatedCommand(
+              numEcReplicatedTasks);
+      List<BlockTargetPair> pendingList = new ArrayList<BlockTargetPair>();
+      if(pendingReplicationList != null && !pendingReplicationList.isEmpty()) {
+        pendingList.addAll(pendingReplicationList);
+      }
+      if(pendingECReplicatedList != null && !pendingECReplicatedList.isEmpty()) {
+        pendingList.addAll(pendingECReplicatedList);
+      }
+      if (!pendingList.isEmpty()) {
         // If the block is deleted, the block size will become
         // BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't
         // need
@@ -1825,7 +1881,7 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
       }
       // check pending erasure coding tasks
       List<BlockECReconstructionInfo> pendingECList = nodeinfo
-          .getErasureCodeCommand(numECTasks);
+          .getErasureCodeCommand(numECReconstructedTasks);
       if (pendingECList != null && !pendingECList.isEmpty()) {
         cmds.add(new BlockECReconstructionCommand(
             DNA_ERASURE_CODING_RECONSTRUCTION, pendingECList));
@@ -1850,25 +1906,22 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
       nodeinfo.setBalancerBandwidth(0);
     }
 
-    if (slowPeerTracker != null) {
-      final Map<String, Double> slowPeersMap = slowPeers.getSlowPeers();
+    Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
+
+    if (slowPeerTracker.isSlowPeerTrackerEnabled()) {
+      final Map<String, OutlierMetrics> slowPeersMap = slowPeers.getSlowPeers();
       if (!slowPeersMap.isEmpty()) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("DataNode " + nodeReg + " reported slow peers: " +
-              slowPeersMap);
-        }
-        for (String slowNodeId : slowPeersMap.keySet()) {
-          slowPeerTracker.addReport(slowNodeId, nodeReg.getIpcAddr(false));
+        LOG.debug("DataNode {} reported slow peers: {}.", nodeReg, slowPeersMap);
+        for (Map.Entry<String, OutlierMetrics> slowNodeEntry : slowPeersMap.entrySet()) {
+          slowPeerTracker.addReport(slowNodeEntry.getKey(), nodeReg.getIpcAddr(false),
+              slowNodeEntry.getValue());
         }
       }
     }
 
     if (slowDiskTracker != null) {
       if (!slowDisks.getSlowDisks().isEmpty()) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("DataNode " + nodeReg + " reported slow disks: " +
-              slowDisks.getSlowDisks());
-        }
+        LOG.debug("DataNode {} reported slow disks: {}.", nodeReg, slowDisks.getSlowDisks());
         slowDiskTracker.addSlowDiskReport(nodeReg.getIpcAddr(false), slowDisks);
       }
       slowDiskTracker.checkAndUpdateReportIfNecessary();
@@ -1897,9 +1950,7 @@ public void handleLifeline(DatanodeRegistration nodeReg,
       StorageReport[] reports, long cacheCapacity,
       long cacheUsed, int xceiverCount, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Received handleLifeline from nodeReg = " + nodeReg);
-    }
+    LOG.debug("Received handleLifeline from nodeReg = {}.", nodeReg);
     DatanodeDescriptor nodeinfo = getDatanode(nodeReg);
     if (nodeinfo == null || !nodeinfo.isRegistered()) {
       // This can happen if the lifeline message comes when DataNode is either
@@ -2076,8 +2127,25 @@ private void setHeartbeatInterval(long intervalSeconds,
     this.heartbeatRecheckInterval = recheckInterval;
     this.heartbeatExpireInterval = 2L * recheckInterval + 10 * 1000
         * intervalSeconds;
-    this.blockInvalidateLimit = Math.max(20 * (int) (intervalSeconds),
-        blockInvalidateLimit);
+    this.blockInvalidateLimit = getBlockInvalidateLimit(blockInvalidateLimit);
+  }
+
+  private int getBlockInvalidateLimitFromHBInterval() {
+    return 20 * (int) heartbeatIntervalSeconds;
+  }
+
+  private int getBlockInvalidateLimit(int configuredBlockInvalidateLimit) {
+    return Math.max(getBlockInvalidateLimitFromHBInterval(), configuredBlockInvalidateLimit);
+  }
+
+  public void setBlockInvalidateLimit(int configuredBlockInvalidateLimit) {
+    final int countedBlockInvalidateLimit = getBlockInvalidateLimitFromHBInterval();
+    // Effected block invalidate limit is the bigger value between
+    // value configured in hdfs-site.xml, and 20 * HB interval.
+    this.blockInvalidateLimit = getBlockInvalidateLimit(configuredBlockInvalidateLimit);
+    LOG.info("{} : configured={}, counted={}, effected={}",
+        DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, configuredBlockInvalidateLimit,
+        countedBlockInvalidateLimit, this.blockInvalidateLimit);
   }
 
   /**
@@ -2086,7 +2154,8 @@ private void setHeartbeatInterval(long intervalSeconds,
    * @return
    */
   public String getSlowPeersReport() {
-    return slowPeerTracker != null ? slowPeerTracker.getJson() : null;
+    Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
+    return slowPeerTracker.getJson();
   }
 
   /**
@@ -2095,24 +2164,29 @@ public String getSlowPeersReport() {
    */
   public Set<String> getSlowPeersUuidSet() {
     Set<String> slowPeersUuidSet = Sets.newConcurrentHashSet();
-    if (slowPeerTracker == null) {
-      return slowPeersUuidSet;
-    }
-    ArrayList<String> slowNodes =
-        slowPeerTracker.getSlowNodes(maxSlowPeerReportNodes);
-    for (String slowNode : slowNodes) {
-      if (StringUtils.isBlank(slowNode)
-              || !slowNode.contains(IP_PORT_SEPARATOR)) {
+    List<String> slowNodes;
+    Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
+    slowNodes = slowPeerTracker.getSlowNodes(maxSlowPeerReportNodes);
+    List<DatanodeDescriptor> datanodeDescriptors = getDnDescriptorsFromIpAddr(slowNodes);
+    datanodeDescriptors.forEach(
+        datanodeDescriptor -> slowPeersUuidSet.add(datanodeDescriptor.getDatanodeUuid()));
+    return slowPeersUuidSet;
+  }
+
+  private List<DatanodeDescriptor> getDnDescriptorsFromIpAddr(List<String> nodes) {
+    List<DatanodeDescriptor> datanodeDescriptors = new ArrayList<>();
+    for (String node : nodes) {
+      if (StringUtils.isBlank(node) || !node.contains(IP_PORT_SEPARATOR)) {
         continue;
       }
-      String ipAddr = slowNode.split(IP_PORT_SEPARATOR)[0];
+      String ipAddr = node.split(IP_PORT_SEPARATOR)[0];
       DatanodeDescriptor datanodeByHost =
           host2DatanodeMap.getDatanodeByHost(ipAddr);
       if (datanodeByHost != null) {
-        slowPeersUuidSet.add(datanodeByHost.getDatanodeUuid());
+        datanodeDescriptors.add(datanodeByHost);
       }
     }
-    return slowPeersUuidSet;
+    return datanodeDescriptors;
   }
 
   /**
@@ -2170,9 +2244,19 @@ public DatanodeStorageReport[] getDatanodeStorageReport(
     for (int i = 0; i < reports.length; i++) {
       final DatanodeDescriptor d = datanodes.get(i);
       reports[i] = new DatanodeStorageReport(
-          new DatanodeInfoBuilder().setFrom(d).build(), d.getStorageReports());
+          new DatanodeInfoBuilder().setFrom(d).setNumBlocks(d.numBlocks()).build(),
+          d.getStorageReports());
     }
     return reports;
   }
-}
 
+  @VisibleForTesting
+  public Map<String, DatanodeDescriptor> getDatanodeMap() {
+    return datanodeMap;
+  }
+
+  public void setMaxSlowPeersToReport(int maxSlowPeersToReport) {
+    Preconditions.checkNotNull(slowPeerTracker, "slowPeerTracker should not be un-assigned");
+    slowPeerTracker.setMaxSlowPeersToReport(maxSlowPeersToReport);
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 01ef5dca4e6b6..df9f6e00a39df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -168,6 +168,11 @@ public boolean areBlockContentsStale() {
     return blockContentsStale;
   }
 
+  @VisibleForTesting
+  public void setBlockContentsStale(boolean value) {
+    blockContentsStale = value;
+  }
+
   void markStaleAfterFailover() {
     heartbeatedSinceFailover = false;
     blockContentsStale = true;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
index 8de3f381ddffe..147f4c3fd624a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
@@ -30,8 +30,9 @@
 import java.util.Set;
 
 class ErasureCodingWork extends BlockReconstructionWork {
-  private final byte[] liveBlockIndicies;
-  private final byte[] liveBusyBlockIndicies;
+  private final byte[] liveBlockIndices;
+  private final byte[] liveBusyBlockIndices;
+  private final byte[] excludeReconstructedIndices;
   private final String blockPoolId;
 
   public ErasureCodingWork(String blockPoolId, BlockInfo block,
@@ -40,18 +41,20 @@ public ErasureCodingWork(String blockPoolId, BlockInfo block,
       List<DatanodeDescriptor> containingNodes,
       List<DatanodeStorageInfo> liveReplicaStorages,
       int additionalReplRequired, int priority,
-      byte[] liveBlockIndicies, byte[] liveBusyBlockIndicies) {
+      byte[] liveBlockIndices, byte[] liveBusyBlockIndices,
+      byte[] excludeReconstrutedIndices) {
     super(block, bc, srcNodes, containingNodes,
         liveReplicaStorages, additionalReplRequired, priority);
     this.blockPoolId = blockPoolId;
-    this.liveBlockIndicies = liveBlockIndicies;
-    this.liveBusyBlockIndicies = liveBusyBlockIndicies;
+    this.liveBlockIndices = liveBlockIndices;
+    this.liveBusyBlockIndices = liveBusyBlockIndices;
+    this.excludeReconstructedIndices=excludeReconstrutedIndices;
     LOG.debug("Creating an ErasureCodingWork to {} reconstruct ",
         block);
   }
 
-  byte[] getLiveBlockIndicies() {
-    return liveBlockIndicies;
+  byte[] getLiveBlockIndices() {
+    return liveBlockIndices;
   }
 
   @Override
@@ -72,15 +75,15 @@ void chooseTargets(BlockPlacementPolicy blockplacement,
    */
   private boolean hasAllInternalBlocks() {
     final BlockInfoStriped block = (BlockInfoStriped) getBlock();
-    if (liveBlockIndicies.length
-        + liveBusyBlockIndicies.length < block.getRealTotalBlockNum()) {
+    if (liveBlockIndices.length
+        + liveBusyBlockIndices.length < block.getRealTotalBlockNum()) {
       return false;
     }
     BitSet bitSet = new BitSet(block.getTotalBlockNum());
-    for (byte index : liveBlockIndicies) {
+    for (byte index : liveBlockIndices) {
       bitSet.set(index);
     }
-    for (byte busyIndex: liveBusyBlockIndicies) {
+    for (byte busyIndex: liveBusyBlockIndices) {
       bitSet.set(busyIndex);
     }
     for (int i = 0; i < block.getRealDataBlockNum(); i++) {
@@ -147,21 +150,21 @@ void addTaskToDatanode(NumberReplicas numberReplicas) {
     } else {
       targets[0].getDatanodeDescriptor().addBlockToBeErasureCoded(
           new ExtendedBlock(blockPoolId, stripedBlk), getSrcNodes(), targets,
-          getLiveBlockIndicies(), stripedBlk.getErasureCodingPolicy());
+          liveBlockIndices, excludeReconstructedIndices, stripedBlk.getErasureCodingPolicy());
     }
   }
 
   private void createReplicationWork(int sourceIndex,
       DatanodeStorageInfo target) {
     BlockInfoStriped stripedBlk = (BlockInfoStriped) getBlock();
-    final byte blockIndex = liveBlockIndicies[sourceIndex];
+    final byte blockIndex = liveBlockIndices[sourceIndex];
     final DatanodeDescriptor source = getSrcNodes()[sourceIndex];
     final long internBlkLen = StripedBlockUtil.getInternalBlockLength(
         stripedBlk.getNumBytes(), stripedBlk.getCellSize(),
         stripedBlk.getDataBlockNum(), blockIndex);
     final Block targetBlk = new Block(stripedBlk.getBlockId() + blockIndex,
         internBlkLen, stripedBlk.getGenerationStamp());
-    source.addBlockToBeReplicated(targetBlk,
+    source.addECBlockToBeReplicated(targetBlk,
         new DatanodeStorageInfo[] {target});
     LOG.debug("Add replication task from source {} to "
         + "target {} for EC block {}", source, target, targetBlk);
@@ -173,7 +176,7 @@ private List<Integer> findLeavingServiceSources() {
     BitSet bitSet = new BitSet(block.getRealTotalBlockNum());
     for (int i = 0; i < getSrcNodes().length; i++) {
       if (getSrcNodes()[i].isInService()) {
-        bitSet.set(liveBlockIndicies[i]);
+        bitSet.set(liveBlockIndices[i]);
       }
     }
     // If the block is on the node which is decommissioning or
@@ -184,7 +187,7 @@ private List<Integer> findLeavingServiceSources() {
       if ((getSrcNodes()[i].isDecommissionInProgress() ||
           (getSrcNodes()[i].isEnteringMaintenance() &&
           getSrcNodes()[i].isAlive())) &&
-          !bitSet.get(liveBlockIndicies[i])) {
+          !bitSet.get(liveBlockIndices[i])) {
         srcIndices.add(i);
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 96d83f29ae271..01e1b6392a09d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -71,6 +71,7 @@ class HeartbeatManager implements DatanodeStatistics {
   /** Heartbeat monitor thread. */
   private final Daemon heartbeatThread = new Daemon(new Monitor());
   private final StopWatch heartbeatStopWatch = new StopWatch();
+  private final int numOfDeadDatanodesRemove;
 
   final Namesystem namesystem;
   final BlockManager blockManager;
@@ -96,6 +97,9 @@ class HeartbeatManager implements DatanodeStatistics {
     enableLogStaleNodes = conf.getBoolean(
         DFSConfigKeys.DFS_NAMENODE_ENABLE_LOG_STALE_DATANODE_KEY,
         DFSConfigKeys.DFS_NAMENODE_ENABLE_LOG_STALE_DATANODE_DEFAULT);
+    this.numOfDeadDatanodesRemove = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_REMOVE_DEAD_DATANODE_BATCHNUM_KEY,
+        DFSConfigKeys.DFS_NAMENODE_REMOVE_BAD_BATCH_NUM_DEFAULT);
 
     if (avoidStaleDataNodesForWrite && staleInterval < recheckInterval) {
       this.heartbeatRecheckInterval = staleInterval;
@@ -256,9 +260,12 @@ synchronized void updateHeartbeat(final DatanodeDescriptor node,
       int xceiverCount, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary) {
     stats.subtract(node);
-    blockManager.updateHeartbeat(node, reports, cacheCapacity, cacheUsed,
-        xceiverCount, failedVolumes, volumeFailureSummary);
-    stats.add(node);
+    try {
+      blockManager.updateHeartbeat(node, reports, cacheCapacity, cacheUsed,
+          xceiverCount, failedVolumes, volumeFailureSummary);
+    } finally {
+      stats.add(node);
+    }
   }
 
   synchronized void updateLifeline(final DatanodeDescriptor node,
@@ -266,13 +273,16 @@ synchronized void updateLifeline(final DatanodeDescriptor node,
       int xceiverCount, int failedVolumes,
       VolumeFailureSummary volumeFailureSummary) {
     stats.subtract(node);
-    // This intentionally calls updateHeartbeatState instead of
-    // updateHeartbeat, because we don't want to modify the
-    // heartbeatedSinceRegistration flag.  Arrival of a lifeline message does
-    // not count as arrival of the first heartbeat.
-    blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed,
-        xceiverCount, failedVolumes, volumeFailureSummary);
-    stats.add(node);
+    try {
+      // This intentionally calls updateHeartbeatState instead of
+      // updateHeartbeat, because we don't want to modify the
+      // heartbeatedSinceRegistration flag.  Arrival of a lifeline message does
+      // not count as arrival of the first heartbeat.
+      blockManager.updateHeartbeatState(node, reports, cacheCapacity, cacheUsed,
+          xceiverCount, failedVolumes, volumeFailureSummary);
+    } finally {
+      stats.add(node);
+    }
   }
 
   synchronized void startDecommission(final DatanodeDescriptor node) {
@@ -398,7 +408,7 @@ private void dumpStaleNodes(List<DatanodeDescriptor> staleNodes) {
   /**
    * Check if there are any expired heartbeats, and if so,
    * whether any blocks have to be re-replicated.
-   * While removing dead datanodes, make sure that only one datanode is marked
+   * While removing dead datanodes, make sure that limited datanodes is marked
    * dead at a time within the synchronized section. Otherwise, a cascading
    * effect causes more datanodes to be declared dead.
    * Check if there are any failed storage and if so,
@@ -430,12 +440,17 @@ void heartbeatCheck() {
       return;
     }
     boolean allAlive = false;
+    // Locate limited dead nodes.
+    List<DatanodeDescriptor> deadDatanodes = new ArrayList<>(
+        numOfDeadDatanodesRemove);
+    // Locate limited failed storages that isn't on a dead node.
+    List<DatanodeStorageInfo> failedStorages = new ArrayList<>(
+        numOfDeadDatanodesRemove);
+
     while (!allAlive) {
-      // locate the first dead node.
-      DatanodeDescriptor dead = null;
 
-      // locate the first failed storage that isn't on a dead node.
-      DatanodeStorageInfo failedStorage = null;
+      deadDatanodes.clear();
+      failedStorages.clear();
 
       // check the number of stale storages
       int numOfStaleStorages = 0;
@@ -446,9 +461,10 @@ void heartbeatCheck() {
           if (shouldAbortHeartbeatCheck(0)) {
             return;
           }
-          if (dead == null && dm.isDatanodeDead(d)) {
+          if (deadDatanodes.size() < numOfDeadDatanodesRemove &&
+              dm.isDatanodeDead(d)) {
             stats.incrExpiredHeartbeats();
-            dead = d;
+            deadDatanodes.add(d);
             // remove the node from stale list to adjust the stale list size
             // before setting the stale count of the DatanodeManager
             removeNodeFromStaleList(d);
@@ -470,10 +486,10 @@ void heartbeatCheck() {
               numOfStaleStorages++;
             }
 
-            if (failedStorage == null &&
+            if (failedStorages.size() < numOfDeadDatanodesRemove &&
                 storageInfo.areBlocksOnFailedStorage() &&
-                d != dead) {
-              failedStorage = storageInfo;
+                !deadDatanodes.contains(d)) {
+              failedStorages.add(storageInfo);
             }
           }
         }
@@ -486,27 +502,27 @@ void heartbeatCheck() {
       // log nodes detected as stale since last heartBeat
       dumpStaleNodes(staleNodes);
 
-      allAlive = dead == null && failedStorage == null;
+      allAlive = deadDatanodes.isEmpty() && failedStorages.isEmpty();
       if (!allAlive && namesystem.isInStartupSafeMode()) {
         return;
       }
 
-      if (dead != null) {
+      for (DatanodeDescriptor dead : deadDatanodes) {
         // acquire the fsnamesystem lock, and then remove the dead node.
         namesystem.writeLock();
         try {
           dm.removeDeadDatanode(dead, !dead.isMaintenance());
         } finally {
-          namesystem.writeUnlock();
+          namesystem.writeUnlock("removeDeadDatanode");
         }
       }
-      if (failedStorage != null) {
+      for (DatanodeStorageInfo failedStorage : failedStorages) {
         // acquire the fsnamesystem lock, and remove blocks on the storage.
         namesystem.writeLock();
         try {
           blockManager.removeBlocksAssociatedTo(failedStorage);
         } finally {
-          namesystem.writeUnlock();
+          namesystem.writeUnlock("removeBlocksAssociatedTo");
         }
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
index af0c92df5e272..d12e5fbae1362 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java
index a73266795ea6e..ddaa2fec5ded3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java
@@ -248,8 +248,8 @@ private int getPriorityContiguous(int curReplicas, int readOnlyReplicas,
       // highest priority
       return QUEUE_HIGHEST_PRIORITY;
     } else if ((curReplicas * 3) < expectedReplicas) {
-      //can only afford one replica loss
-      //this is considered very insufficiently redundant blocks.
+      //there is less than a third as many blocks as requested;
+      //this is considered very under-replicated.
       return QUEUE_VERY_LOW_REDUNDANCY;
     } else {
       //add to the normal queue for insufficiently redundant blocks
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
index 3e56606197bb4..6c3b4c97bed37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java
@@ -76,6 +76,14 @@ void start() {
     timerThread.start();
   }
 
+  public void setTimeout(long timeoutPeriod) {
+    this.timeout = timeoutPeriod;
+  }
+
+  public long getTimeout() {
+    return this.timeout;
+  }
+
   /**
    * Add a block to the list of pending reconstructions
    * @param block The corresponding block
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java
new file mode 100644
index 0000000000000..ac109e0c90ae3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerDisabledTracker.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.Timer;
+
+/**
+ * Disabled tracker for slow peers. To be used when dfs.datanode.peer.stats.enabled is disabled.
+ */
+@InterfaceAudience.Private
+public class SlowPeerDisabledTracker extends SlowPeerTracker {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SlowPeerDisabledTracker.class);
+
+  public SlowPeerDisabledTracker(Configuration conf, Timer timer) {
+    super(conf, timer);
+    final boolean dataNodePeerStatsEnabledVal =
+        conf.getBoolean(DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY,
+            DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT);
+    Preconditions.checkArgument(!dataNodePeerStatsEnabledVal,
+        "SlowPeerDisabledTracker should only be used for disabled slow peer stats.");
+  }
+
+  @Override
+  public boolean isSlowPeerTrackerEnabled() {
+    return false;
+  }
+
+  @Override
+  public void addReport(String slowNode, String reportingNode, OutlierMetrics slowNodeMetrics) {
+    LOG.trace("Adding slow peer report is disabled. To enable it, please enable config {}.",
+        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY);
+  }
+
+  @Override
+  public Set<SlowPeerLatencyWithReportingNode> getReportsForNode(String slowNode) {
+    LOG.trace("Retrieval of slow peer report is disabled. To enable it, please enable config {}.",
+        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY);
+    return ImmutableSet.of();
+  }
+
+  @Override
+  public Map<String, SortedSet<SlowPeerLatencyWithReportingNode>> getReportsForAllDataNodes() {
+    LOG.trace("Retrieval of slow peer report for all nodes is disabled. "
+            + "To enable it, please enable config {}.",
+        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY);
+    return ImmutableMap.of();
+  }
+
+  @Override
+  public String getJson() {
+    LOG.trace("Retrieval of slow peer reports as json string is disabled. "
+            + "To enable it, please enable config {}.",
+        DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY);
+    return null;
+  }
+
+  @Override
+  public List<String> getSlowNodes(int numNodes) {
+    return ImmutableList.of();
+  }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerJsonReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerJsonReport.java
new file mode 100644
index 0000000000000..b9b741e9d611c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerJsonReport.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.SortedSet;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * This structure is a thin wrapper over slow peer reports to make Json
+ * [de]serialization easy.
+ */
+@InterfaceAudience.Private
+final class SlowPeerJsonReport {
+
+  @JsonProperty("SlowNode")
+  private final String slowNode;
+
+  @JsonProperty("SlowPeerLatencyWithReportingNodes")
+  private final SortedSet<SlowPeerLatencyWithReportingNode> slowPeerLatencyWithReportingNodes;
+
+  SlowPeerJsonReport(
+      @JsonProperty("SlowNode")
+          String slowNode,
+      @JsonProperty("SlowPeerLatencyWithReportingNodes")
+          SortedSet<SlowPeerLatencyWithReportingNode> slowPeerLatencyWithReportingNodes) {
+    this.slowNode = slowNode;
+    this.slowPeerLatencyWithReportingNodes = slowPeerLatencyWithReportingNodes;
+  }
+
+  public String getSlowNode() {
+    return slowNode;
+  }
+
+  public SortedSet<SlowPeerLatencyWithReportingNode> getSlowPeerLatencyWithReportingNodes() {
+    return slowPeerLatencyWithReportingNodes;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    SlowPeerJsonReport that = (SlowPeerJsonReport) o;
+
+    return new EqualsBuilder()
+        .append(slowNode, that.slowNode)
+        .append(slowPeerLatencyWithReportingNodes, that.slowPeerLatencyWithReportingNodes)
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(17, 37)
+        .append(slowNode)
+        .append(slowPeerLatencyWithReportingNodes)
+        .toHashCode();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerLatencyWithReportingNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerLatencyWithReportingNode.java
new file mode 100644
index 0000000000000..b90f809f1ed8c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerLatencyWithReportingNode.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * This class represents the reporting node and the slow node's latency as observed by the
+ * reporting node. This class is used by SlowPeerJsonReport class.
+ */
+@InterfaceAudience.Private
+final class SlowPeerLatencyWithReportingNode
+    implements Comparable<SlowPeerLatencyWithReportingNode> {
+
+  @JsonProperty("ReportingNode")
+  private final String reportingNode;
+
+  @JsonProperty("ReportedLatency")
+  private final Double reportedLatency;
+
+  @JsonProperty("MedianLatency")
+  private final Double medianLatency;
+
+  @JsonProperty("MadLatency")
+  private final Double madLatency;
+
+  @JsonProperty("UpperLimitLatency")
+  private final Double upperLimitLatency;
+
+  SlowPeerLatencyWithReportingNode(
+      @JsonProperty("ReportingNode")
+          String reportingNode,
+      @JsonProperty("ReportedLatency")
+          Double reportedLatency,
+      @JsonProperty("MedianLatency")
+          Double medianLatency,
+      @JsonProperty("MadLatency")
+          Double madLatency,
+      @JsonProperty("UpperLimitLatency")
+          Double upperLimitLatency) {
+    this.reportingNode = reportingNode;
+    this.reportedLatency = reportedLatency;
+    this.medianLatency = medianLatency;
+    this.madLatency = madLatency;
+    this.upperLimitLatency = upperLimitLatency;
+  }
+
+  public String getReportingNode() {
+    return reportingNode;
+  }
+
+  public Double getReportedLatency() {
+    return reportedLatency;
+  }
+
+  public Double getMedianLatency() {
+    return medianLatency;
+  }
+
+  public Double getMadLatency() {
+    return madLatency;
+  }
+
+  public Double getUpperLimitLatency() {
+    return upperLimitLatency;
+  }
+
+  @Override
+  public int compareTo(SlowPeerLatencyWithReportingNode o) {
+    return this.reportingNode.compareTo(o.getReportingNode());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    SlowPeerLatencyWithReportingNode that = (SlowPeerLatencyWithReportingNode) o;
+
+    return new EqualsBuilder()
+        .append(reportingNode, that.reportingNode)
+        .append(reportedLatency, that.reportedLatency)
+        .append(medianLatency, that.medianLatency)
+        .append(madLatency, that.madLatency)
+        .append(upperLimitLatency, that.upperLimitLatency)
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(17, 37)
+        .append(reportingNode)
+        .append(reportedLatency)
+        .append(medianLatency)
+        .append(madLatency)
+        .append(upperLimitLatency)
+        .toHashCode();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
index e54934e8813a7..e4feb4815eee4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
@@ -29,6 +28,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
 import org.apache.hadoop.util.Timer;
 import org.slf4j.Logger;
@@ -37,8 +37,8 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
@@ -80,7 +80,7 @@ public class SlowPeerTracker {
    * Number of nodes to include in JSON report. We will return nodes with
    * the highest number of votes from peers.
    */
-  private final int maxNodesToReport;
+  private volatile int maxNodesToReport;
 
   /**
    * Information about peers that have reported a node as being slow.
@@ -94,7 +94,7 @@ public class SlowPeerTracker {
    * Stale reports are not evicted proactively and can potentially
    * hang around forever.
    */
-  private final ConcurrentMap<String, ConcurrentMap<String, Long>>
+  private final ConcurrentMap<String, ConcurrentMap<String, LatencyWithLastReportTime>>
       allReports;
 
   public SlowPeerTracker(Configuration conf, Timer timer) {
@@ -104,21 +104,30 @@ public SlowPeerTracker(Configuration conf, Timer timer) {
         DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
         DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT,
         TimeUnit.MILLISECONDS) * 3;
-    this.maxNodesToReport = conf.getInt(
-        DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
-        DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT);
+    this.setMaxSlowPeersToReport(conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
+        DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT));
+  }
+
+  /**
+   * If SlowPeerTracker is enabled, return true, else returns false.
+   *
+   * @return true if slow peer tracking is enabled, else false.
+   */
+  public boolean isSlowPeerTrackerEnabled() {
+    return true;
   }
 
   /**
    * Add a new report. DatanodeIds can be the DataNodeIds or addresses
    * We don't care as long as the caller is consistent.
    *
-   * @param reportingNode DataNodeId of the node reporting on its peer.
    * @param slowNode DataNodeId of the peer suspected to be slow.
+   * @param reportingNode DataNodeId of the node reporting on its peer.
+   * @param slowNodeMetrics Aggregate latency metrics of slownode as reported by the
+   *     reporting node.
    */
-  public void addReport(String slowNode,
-                        String reportingNode) {
-    ConcurrentMap<String, Long> nodeEntries = allReports.get(slowNode);
+  public void addReport(String slowNode, String reportingNode, OutlierMetrics slowNodeMetrics) {
+    ConcurrentMap<String, LatencyWithLastReportTime> nodeEntries = allReports.get(slowNode);
 
     if (nodeEntries == null) {
       // putIfAbsent guards against multiple writers.
@@ -127,7 +136,8 @@ public void addReport(String slowNode,
     }
 
     // Replace the existing entry from this node, if any.
-    nodeEntries.put(reportingNode, timer.monotonicNow());
+    nodeEntries.put(reportingNode,
+        new LatencyWithLastReportTime(timer.monotonicNow(), slowNodeMetrics));
   }
 
   /**
@@ -137,8 +147,8 @@ public void addReport(String slowNode,
    * @param slowNode target node Id.
    * @return set of reports which implicate the target node as being slow.
    */
-  public Set<String> getReportsForNode(String slowNode) {
-    final ConcurrentMap<String, Long> nodeEntries =
+  public Set<SlowPeerLatencyWithReportingNode> getReportsForNode(String slowNode) {
+    final ConcurrentMap<String, LatencyWithLastReportTime> nodeEntries =
         allReports.get(slowNode);
 
     if (nodeEntries == null || nodeEntries.isEmpty()) {
@@ -153,17 +163,19 @@ public Set<String> getReportsForNode(String slowNode) {
    *
    * @return map from SlowNodeId {@literal ->} (set of nodes reporting peers).
    */
-  public Map<String, SortedSet<String>> getReportsForAllDataNodes() {
+  public Map<String, SortedSet<SlowPeerLatencyWithReportingNode>> getReportsForAllDataNodes() {
     if (allReports.isEmpty()) {
       return ImmutableMap.of();
     }
 
-    final Map<String, SortedSet<String>> allNodesValidReports = new HashMap<>();
+    final Map<String, SortedSet<SlowPeerLatencyWithReportingNode>> allNodesValidReports =
+        new HashMap<>();
     final long now = timer.monotonicNow();
 
-    for (Map.Entry<String, ConcurrentMap<String, Long>> entry :
-        allReports.entrySet()) {
-      SortedSet<String> validReports = filterNodeReports(entry.getValue(), now);
+    for (Map.Entry<String, ConcurrentMap<String, LatencyWithLastReportTime>> entry
+        : allReports.entrySet()) {
+      SortedSet<SlowPeerLatencyWithReportingNode> validReports =
+          filterNodeReports(entry.getValue(), now);
       if (!validReports.isEmpty()) {
         allNodesValidReports.put(entry.getKey(), validReports);
       }
@@ -174,17 +186,21 @@ public Map<String, SortedSet<String>> getReportsForAllDataNodes() {
   /**
    * Filter the given reports to return just the valid ones.
    *
-   * @param reports
-   * @param now
-   * @return
+   * @param reports Current set of reports.
+   * @param now Current time.
+   * @return Set of valid reports that were created within last reportValidityMs millis.
    */
-  private SortedSet<String> filterNodeReports(
-      ConcurrentMap<String, Long> reports, long now) {
-    final SortedSet<String> validReports = new TreeSet<>();
-
-    for (Map.Entry<String, Long> entry : reports.entrySet()) {
-      if (now - entry.getValue() < reportValidityMs) {
-        validReports.add(entry.getKey());
+  private SortedSet<SlowPeerLatencyWithReportingNode> filterNodeReports(
+      ConcurrentMap<String, LatencyWithLastReportTime> reports, long now) {
+    final SortedSet<SlowPeerLatencyWithReportingNode> validReports = new TreeSet<>();
+
+    for (Map.Entry<String, LatencyWithLastReportTime> entry : reports.entrySet()) {
+      if (now - entry.getValue().getTime() < reportValidityMs) {
+        OutlierMetrics outlierMetrics = entry.getValue().getLatency();
+        validReports.add(
+            new SlowPeerLatencyWithReportingNode(entry.getKey(), outlierMetrics.getActualLatency(),
+                outlierMetrics.getMedian(), outlierMetrics.getMad(),
+                outlierMetrics.getUpperLimitLatency()));
       }
     }
     return validReports;
@@ -196,7 +212,7 @@ private SortedSet<String> filterNodeReports(
    *         serialization failed.
    */
   public String getJson() {
-    Collection<ReportForJson> validReports = getJsonReports(
+    Collection<SlowPeerJsonReport> validReports = getJsonReports(
         maxNodesToReport);
     try {
       return WRITER.writeValueAsString(validReports);
@@ -207,42 +223,15 @@ public String getJson() {
     }
   }
 
-  /**
-   * This structure is a thin wrapper over reports to make Json
-   * [de]serialization easy.
-   */
-  public static class ReportForJson {
-    @JsonProperty("SlowNode")
-    final private String slowNode;
-
-    @JsonProperty("ReportingNodes")
-    final private SortedSet<String> reportingNodes;
-
-    public ReportForJson(
-        @JsonProperty("SlowNode") String slowNode,
-        @JsonProperty("ReportingNodes") SortedSet<String> reportingNodes) {
-      this.slowNode = slowNode;
-      this.reportingNodes = reportingNodes;
-    }
-
-    public String getSlowNode() {
-      return slowNode;
-    }
-
-    public SortedSet<String> getReportingNodes() {
-      return reportingNodes;
-    }
-  }
-
   /**
    * Returns all tracking slow peers.
    * @param numNodes
    * @return
    */
-  public ArrayList<String> getSlowNodes(int numNodes) {
-    Collection<ReportForJson> jsonReports = getJsonReports(numNodes);
+  public List<String> getSlowNodes(int numNodes) {
+    Collection<SlowPeerJsonReport> jsonReports = getJsonReports(numNodes);
     ArrayList<String> slowNodes = new ArrayList<>();
-    for (ReportForJson jsonReport : jsonReports) {
+    for (SlowPeerJsonReport jsonReport : jsonReports) {
       slowNodes.add(jsonReport.getSlowNode());
     }
     if (!slowNodes.isEmpty()) {
@@ -257,35 +246,30 @@ public ArrayList<String> getSlowNodes(int numNodes) {
    * @param numNodes number of nodes to return. This is to limit the
    *                 size of the generated JSON.
    */
-  private Collection<ReportForJson> getJsonReports(int numNodes) {
+  private Collection<SlowPeerJsonReport> getJsonReports(int numNodes) {
     if (allReports.isEmpty()) {
       return Collections.emptyList();
     }
 
-    final PriorityQueue<ReportForJson> topNReports =
-        new PriorityQueue<>(allReports.size(),
-            new Comparator<ReportForJson>() {
-          @Override
-          public int compare(ReportForJson o1, ReportForJson o2) {
-            return Ints.compare(o1.reportingNodes.size(),
-                o2.reportingNodes.size());
-          }
-        });
+    final PriorityQueue<SlowPeerJsonReport> topNReports = new PriorityQueue<>(allReports.size(),
+        (o1, o2) -> Ints.compare(o1.getSlowPeerLatencyWithReportingNodes().size(),
+            o2.getSlowPeerLatencyWithReportingNodes().size()));
 
     final long now = timer.monotonicNow();
 
-    for (Map.Entry<String, ConcurrentMap<String, Long>> entry :
-        allReports.entrySet()) {
-      SortedSet<String> validReports = filterNodeReports(
-          entry.getValue(), now);
+    for (Map.Entry<String, ConcurrentMap<String, LatencyWithLastReportTime>> entry
+        : allReports.entrySet()) {
+      SortedSet<SlowPeerLatencyWithReportingNode> validReports =
+          filterNodeReports(entry.getValue(), now);
       if (!validReports.isEmpty()) {
         if (topNReports.size() < numNodes) {
-          topNReports.add(new ReportForJson(entry.getKey(), validReports));
-        } else if (topNReports.peek().getReportingNodes().size() <
-            validReports.size()){
+          topNReports.add(new SlowPeerJsonReport(entry.getKey(), validReports));
+        } else if (topNReports.peek() != null
+            && topNReports.peek().getSlowPeerLatencyWithReportingNodes().size()
+            < validReports.size()) {
           // Remove the lowest element
           topNReports.poll();
-          topNReports.add(new ReportForJson(entry.getKey(), validReports));
+          topNReports.add(new SlowPeerJsonReport(entry.getKey(), validReports));
         }
       }
     }
@@ -296,4 +280,27 @@ public int compare(ReportForJson o1, ReportForJson o2) {
   long getReportValidityMs() {
     return reportValidityMs;
   }
+
+  public synchronized void setMaxSlowPeersToReport(int maxSlowPeersToReport) {
+    this.maxNodesToReport = maxSlowPeersToReport;
+  }
+
+  private static class LatencyWithLastReportTime {
+    private final Long time;
+    private final OutlierMetrics latency;
+
+    LatencyWithLastReportTime(Long time, OutlierMetrics latency) {
+      this.time = time;
+      this.latency = latency;
+    }
+
+    public Long getTime() {
+      return time;
+    }
+
+    public OutlierMetrics getLatency() {
+      return latency;
+    }
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java
new file mode 100644
index 0000000000000..bf6edda7abd06
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/AutoCloseDataSetLock.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.common;
+
+import org.apache.hadoop.util.AutoCloseableLock;
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.concurrent.locks.Lock;
+
+import static org.apache.hadoop.hdfs.server.datanode.DataSetLockManager.LOG;
+
+/**
+ * Extending AutoCloseableLock such that the users can
+ * use a try-with-resource syntax.
+ */
+public class AutoCloseDataSetLock extends AutoCloseableLock {
+  private Lock lock;
+  private AutoCloseDataSetLock parentLock;
+  private DataNodeLockManager<AutoCloseDataSetLock> dataNodeLockManager;
+
+  public AutoCloseDataSetLock(Lock lock) {
+    this.lock = lock;
+  }
+
+  @Override
+  public void close() {
+    if (lock != null) {
+      lock.unlock();
+      if (dataNodeLockManager != null) {
+        dataNodeLockManager.hook();
+      }
+    } else {
+      LOG.error("Try to unlock null lock" +
+          StringUtils.getStackTrace(Thread.currentThread()));
+    }
+    if (parentLock != null) {
+      parentLock.close();
+    }
+  }
+
+  /**
+   * Actually acquire the lock.
+   */
+  public void lock() {
+    if (lock != null) {
+      lock.lock();
+      return;
+    }
+    LOG.error("Try to lock null lock" +
+        StringUtils.getStackTrace(Thread.currentThread()));
+  }
+
+  public void setParentLock(AutoCloseDataSetLock parent) {
+    if (parentLock == null) {
+      this.parentLock = parent;
+    }
+  }
+
+  public void setDataNodeLockManager(DataNodeLockManager<AutoCloseDataSetLock>
+      dataNodeLockManager) {
+    this.dataNodeLockManager = dataNodeLockManager;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
new file mode 100644
index 0000000000000..e7a3b38357ac9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.common;
+
+/**
+ * Use for manage a set of lock for datanode.
+ */
+public interface DataNodeLockManager<T extends AutoCloseDataSetLock> {
+
+  /**
+   * Acquire block pool level first if you want to Acquire volume lock.
+   * Or only acquire block pool level lock.
+   */
+  enum LockLevel {
+    BLOCK_POOl,
+    VOLUME
+  }
+
+  /**
+   * Acquire readLock and then lock.
+   */
+  T readLock(LockLevel level, String... resources);
+
+  /**
+   * Acquire writeLock and then lock.
+   */
+  T writeLock(LockLevel level, String... resources);
+
+  /**
+   * Add a lock to LockManager.
+   */
+  void addLock(LockLevel level, String... resources);
+
+  /**
+   * Remove a lock from LockManager.
+   */
+  void removeLock(LockLevel level, String... resources);
+
+  /**
+   * LockManager may need to back hook.
+   */
+  void hook();
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 58e9d5fdd9557..8e343f998a626 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
 import org.apache.hadoop.util.StringUtils;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java
index e9f1cf09e86d4..afed1e9e6e72d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java
@@ -117,17 +117,12 @@ private boolean matchRule(String user, String remoteIp, String path) {
       String rulePath = rule.getPath();
       LOG.trace("Evaluating rule, subnet: {}, path: {}",
           subnet != null ? subnet.getCidrSignature() : "*", rulePath);
-      try {
-        if ((subnet == null || subnet.isInRange(remoteIp))
-            && FilenameUtils.directoryContains(rulePath, path)) {
-          LOG.debug("Found matching rule, subnet: {}, path: {}; returned true",
-              rule.getSubnet() != null ? subnet.getCidrSignature() : null,
-              rulePath);
-          return true;
-        }
-      } catch (IOException e) {
-        LOG.warn("Got IOException {}; returned false", e);
-        return false;
+      if ((subnet == null || subnet.isInRange(remoteIp))
+          && FilenameUtils.directoryContains(rulePath, path)) {
+        LOG.debug("Found matching rule, subnet: {}, path: {}; returned true",
+            rule.getSubnet() != null ? subnet.getCidrSignature() : null,
+            rulePath);
+        return true;
       }
     }
 
@@ -231,9 +226,8 @@ public void handleInteraction(HttpInteraction interaction)
     final String query = interaction.getQueryString();
     final String uri = interaction.getRequestURI();
     if (!uri.startsWith(WebHdfsFileSystem.PATH_PREFIX)) {
-      LOG.trace("Rejecting interaction; wrong URI: {}", uri);
-      interaction.sendError(HttpServletResponse.SC_NOT_FOUND,
-          "The request URI must start with " + WebHdfsFileSystem.PATH_PREFIX);
+      LOG.trace("Proceeding with interaction since the request doesn't access WebHDFS API");
+      interaction.proceed();
       return;
     }
     final String path = uri.substring(WebHdfsFileSystem.PATH_PREFIX.length());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java
index 43b8c3c05d102..7a14e38f5505f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/IncorrectVersionException.java
@@ -21,7 +21,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 
 /**
  * The exception is thrown when external version does not match 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
index 2f249655eed8e..4265c288e88d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
@@ -198,6 +198,9 @@ public static String getRemoteAddr(HttpServletRequest request) {
     return remoteAddr;
   }
 
+  public static int getRemotePort(HttpServletRequest request) {
+    return request.getRemotePort();
+  }
 
   /**
    * Expected user name should be a short name.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java
new file mode 100644
index 0000000000000..848495cc4e018
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/NoLockManager.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.common;
+
+import java.util.concurrent.locks.Lock;
+
+/**
+ * Some ut or temp replicaMap not need to lock with DataSetLockManager.
+ */
+public class NoLockManager implements DataNodeLockManager<AutoCloseDataSetLock> {
+  private final NoDataSetLock lock = new NoDataSetLock(null);
+
+  private static final class NoDataSetLock extends AutoCloseDataSetLock {
+
+    private NoDataSetLock(Lock lock) {
+      super(lock);
+    }
+
+    @Override
+    public void lock() {
+    }
+
+    @Override
+    public void close() {
+    }
+  }
+
+  public NoLockManager() {
+  }
+
+  @Override
+  public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
+    return lock;
+  }
+
+  @Override
+  public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) {
+    return lock;
+  }
+
+  @Override
+  public void addLock(LockLevel level, String... resources) {
+  }
+
+  @Override
+  public void removeLock(LockLevel level, String... resources) {
+  }
+
+  @Override
+  public void hook() {
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 83a82566f6c3e..7a4535f33898c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -54,7 +54,7 @@
 import org.apache.hadoop.util.VersionInfo;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
index 8643821c86993..4a06588258abd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
@@ -21,6 +21,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
 import java.util.SortedSet;
@@ -112,6 +113,20 @@ public String toString() {
     .append(";nsid=").append(namespaceID).append(";c=").append(cTime);
     return sb.toString();
   }
+
+  /**
+   * Returns string representation of Storage info attributes stored in Map.
+   *
+   * @return string representation of storage info attributes.
+   */
+  public String toMapString() {
+    Map<String, Object> storageInfo = new HashMap<>();
+    storageInfo.put("LayoutVersion", layoutVersion);
+    storageInfo.put("ClusterId", clusterID);
+    storageInfo.put("NamespaceId", namespaceID);
+    storageInfo.put("CreationTime", cTime);
+    return storageInfo.toString();
+  }
   
   public String toColonSeparatedString() {
     return Joiner.on(":").join(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
index 816e278ccf56b..5039db6ceb488 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java
@@ -54,7 +54,7 @@
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
index 183853739aa29..e94e48ed0f7b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
@@ -26,7 +26,6 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Iterator;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java
index f87fcaef054c0..f7756c74851a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/sps/BlockDispatcher.java
@@ -101,7 +101,7 @@ public BlockDispatcher(int sockTimeout, int ioFileBuffSize,
    */
   public BlockMovementStatus moveBlock(BlockMovingInfo blkMovingInfo,
       SaslDataTransferClient saslClient, ExtendedBlock eb, Socket sock,
-      DataEncryptionKeyFactory km, Token<BlockTokenIdentifier> accessToken) {
+      DataEncryptionKeyFactory km, Token<BlockTokenIdentifier> accessToken) throws IOException {
     LOG.info("Start moving block:{} from src:{} to destin:{} to satisfy "
         + "storageType, sourceStoragetype:{} and destinStoragetype:{}",
         blkMovingInfo.getBlock(), blkMovingInfo.getSource(),
@@ -149,14 +149,6 @@ public BlockMovementStatus moveBlock(BlockMovingInfo blkMovingInfo,
       LOG.debug("Pinned block can't be moved, so skipping block:{}",
           blkMovingInfo.getBlock(), e);
       return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_SUCCESS;
-    } catch (IOException e) {
-      // TODO: handle failure retries
-      LOG.warn(
-          "Failed to move block:{} from src:{} to destin:{} to satisfy "
-              + "storageType:{}",
-          blkMovingInfo.getBlock(), blkMovingInfo.getSource(),
-          blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), e);
-      return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE;
     } finally {
       IOUtils.closeStream(out);
       IOUtils.closeStream(in);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 0e9a1f755855e..d660970c72542 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.StorageType;
@@ -40,6 +40,7 @@
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -143,11 +144,11 @@ void writeUnlock() {
   void refreshNNList(String serviceId, List<String> nnIds,
       ArrayList<InetSocketAddress> addrs,
       ArrayList<InetSocketAddress> lifelineAddrs) throws IOException {
-    Set<InetSocketAddress> oldAddrs = Sets.newHashSet();
+    Set<InetSocketAddress> oldAddrs = new HashSet<>();
     for (BPServiceActor actor : bpServices) {
       oldAddrs.add(actor.getNNSocketAddress());
     }
-    Set<InetSocketAddress> newAddrs = Sets.newHashSet(addrs);
+    Set<InetSocketAddress> newAddrs = new HashSet<>(addrs);
     
     // Process added NNs
     Set<InetSocketAddress> addedNNs = Sets.difference(newAddrs, oldAddrs);
@@ -378,7 +379,7 @@ void verifyAndSetNamespaceInfo(BPServiceActor actor, NamespaceInfo nsInfo)
 
     if(nsInfo.getState() == HAServiceState.ACTIVE
         && bpServiceToActive == null) {
-      LOG.info("Acknowledging ACTIVE Namenode during handshake" + actor);
+      LOG.info("Acknowledging ACTIVE Namenode during handshake {}", actor);
       bpServiceToActive = actor;
     }
 
@@ -740,7 +741,6 @@ private boolean processCommandFromActive(DatanodeCommand cmd,
         // Exceptions caught here are not expected to be disk-related.
         throw e;
       }
-      dn.metrics.incrBlocksRemoved(toDelete.length);
       break;
     case DatanodeProtocol.DNA_CACHE:
       LOG.info("DatanodeCommand action: DNA_CACHE for " +
@@ -853,4 +853,12 @@ boolean shouldRetryInit() {
     return isAlive();
   }
 
+  boolean isSlownode() {
+    for (BPServiceActor actor : bpServices) {
+      if (actor.isSlownode()) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 718520f7273de..844b67ce1a877 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.datanode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
 import static org.apache.hadoop.util.Time.monotonicNow;
 
 import java.io.Closeable;
@@ -53,6 +55,7 @@
 import org.apache.hadoop.hdfs.protocolPB.DatanodeLifelineProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
@@ -70,6 +73,7 @@
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.util.VersionUtil;
@@ -112,6 +116,7 @@ enum RunningState {
   private String nnId = null;
   private volatile RunningState runningState = RunningState.CONNECTING;
   private volatile boolean shouldServiceRun = true;
+  private volatile boolean isSlownode = false;
   private final DataNode dn;
   private final DNConf dnConf;
   private long prevBlockReportId;
@@ -205,6 +210,7 @@ Map<String, String> getActorInfoMap() {
         String.valueOf(getScheduler().getLastBlockReportTime()));
     info.put("maxBlockReportSize", String.valueOf(getMaxBlockReportSize()));
     info.put("maxDataLength", String.valueOf(maxDataLength));
+    info.put("isSlownode", String.valueOf(isSlownode));
     return info;
   }
 
@@ -304,6 +310,10 @@ private void connectToNNAndHandshake() throws IOException {
     // info.
     NamespaceInfo nsInfo = retrieveNamespaceInfo();
 
+    // init block pool lock when init.
+    dn.getDataSetLockManager().addLock(LockLevel.BLOCK_POOl,
+        nsInfo.getBlockPoolID());
+
     // Verify that this matches the other NN in this HA pair.
     // This also initializes our block pool in the DN if we are
     // the first NN connection for this BP.
@@ -452,8 +462,8 @@ List<DatanodeCommand> blockReport(long fullBrLeaseId) throws IOException {
       dn.getMetrics().addBlockReport(brSendCost, getRpcMetricSuffix());
       final int nCmds = cmds.size();
       LOG.info((success ? "S" : "Uns") +
-          "uccessfully sent block report 0x" +
-          Long.toHexString(reportId) + " to namenode: " + nnAddr +
+          "uccessfully sent block report 0x" + Long.toHexString(reportId) +
+          " with lease ID 0x" + Long.toHexString(fullBrLeaseId) + " to namenode: " + nnAddr +
           ",  containing " + reports.length +
           " storage report(s), of which we sent " + numReportsSent + "." +
           " The reports had " + totalBlockCount +
@@ -549,11 +559,11 @@ HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease)
         volumeFailureSummary.getFailedStorageLocations().length : 0;
     final boolean outliersReportDue = scheduler.isOutliersReportDue(now);
     final SlowPeerReports slowPeers =
-        outliersReportDue && dn.getPeerMetrics() != null ?
+        outliersReportDue && dnConf.peerStatsEnabled && dn.getPeerMetrics() != null ?
             SlowPeerReports.create(dn.getPeerMetrics().getOutliers()) :
             SlowPeerReports.EMPTY_REPORT;
     final SlowDiskReports slowDisks =
-        outliersReportDue && dn.getDiskMetrics() != null ?
+        outliersReportDue && dnConf.diskStatsEnabled && dn.getDiskMetrics() != null ?
             SlowDiskReports.create(dn.getDiskMetrics().getDiskOutliersStats()) :
             SlowDiskReports.EMPTY_REPORT;
 
@@ -729,6 +739,7 @@ private void offerService() throws Exception {
               handleRollingUpgradeStatus(resp);
             }
             commandProcessingThread.enqueue(resp.getCommands());
+            isSlownode = resp.getIsSlownode();
           }
         }
 
@@ -805,7 +816,7 @@ void register(NamespaceInfo nsInfo) throws IOException {
     // off disk - so update the bpRegistration object from that info
     DatanodeRegistration newBpRegistration = bpos.createRegistration();
 
-    LOG.info(this + " beginning handshake with NN");
+    LOG.info("{} beginning handshake with NN: {}.", this, nnAddr);
 
     while (shouldRun()) {
       try {
@@ -815,15 +826,14 @@ void register(NamespaceInfo nsInfo) throws IOException {
         bpRegistration = newBpRegistration;
         break;
       } catch(EOFException e) {  // namenode might have just restarted
-        LOG.info("Problem connecting to server: " + nnAddr + " :"
-            + e.getLocalizedMessage());
+        LOG.info("Problem connecting to server: {} : {}.", nnAddr, e.getLocalizedMessage());
       } catch(SocketTimeoutException e) {  // namenode is busy
-        LOG.info("Problem connecting to server: " + nnAddr);
+        LOG.info("Problem connecting to server: {}.", nnAddr);
       } catch(RemoteException e) {
-        LOG.warn("RemoteException in register", e);
+        LOG.warn("RemoteException in register to server: {}.", nnAddr, e);
         throw e;
       } catch(IOException e) {
-        LOG.warn("Problem connecting to server: " + nnAddr);
+        LOG.warn("Problem connecting to server: {}.", nnAddr);
       }
       // Try again in a second
       sleepAndLogInterrupts(1000, "connecting to server");
@@ -833,7 +843,7 @@ void register(NamespaceInfo nsInfo) throws IOException {
       throw new IOException("DN shut down before block pool registered");
     }
 
-    LOG.info(this + " successfully registered with NN");
+    LOG.info("{} successfully registered with NN: {}.", this, nnAddr);
     bpos.registrationSucceeded(this, bpRegistration);
 
     // reset lease id whenever registered to NN.
@@ -1190,8 +1200,8 @@ static class Scheduler {
 
     private final long heartbeatIntervalMs;
     private final long lifelineIntervalMs;
-    private final long blockReportIntervalMs;
-    private final long outliersReportIntervalMs;
+    private volatile long blockReportIntervalMs;
+    private volatile long outliersReportIntervalMs;
 
     Scheduler(long heartbeatIntervalMs, long lifelineIntervalMs,
               long blockReportIntervalMs, long outliersReportIntervalMs) {
@@ -1267,6 +1277,7 @@ boolean isOutliersReportDue(long curTime) {
 
     void forceFullBlockReportNow() {
       forceFullBlockReport.set(true);
+      resetBlockReportTime = true;
     }
 
     /**
@@ -1333,7 +1344,8 @@ long getHeartbeatWaitTime() {
     }
 
     long getLifelineWaitTime() {
-      return nextLifelineTime - monotonicNow();
+      long waitTime = nextLifelineTime - monotonicNow();
+      return waitTime > 0 ? waitTime : 0;
     }
 
     @VisibleForTesting
@@ -1346,6 +1358,27 @@ void setNextBlockReportTime(long nextBlockReportTime) {
       this.nextBlockReportTime.getAndSet(nextBlockReportTime);
     }
 
+    long getBlockReportIntervalMs() {
+      return this.blockReportIntervalMs;
+    }
+
+    void setBlockReportIntervalMs(long intervalMs) {
+      Preconditions.checkArgument(intervalMs > 0,
+          DFS_BLOCKREPORT_INTERVAL_MSEC_KEY + " should be larger than 0");
+      this.blockReportIntervalMs = intervalMs;
+    }
+
+    void setOutliersReportIntervalMs(long intervalMs) {
+      Preconditions.checkArgument(intervalMs > 0,
+          DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY + " should be larger than 0");
+      this.outliersReportIntervalMs = intervalMs;
+    }
+
+    @VisibleForTesting
+    long getOutliersReportIntervalMs() {
+      return this.outliersReportIntervalMs;
+    }
+
     /**
      * Wrapped for testing.
      * @return
@@ -1474,4 +1507,8 @@ void stopCommandProcessingThread() {
       commandProcessingThread.interrupt();
     }
   }
+
+  boolean isSlownode() {
+    return isSlownode;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
index 265267da887d2..eb2bfeb78698f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.DFSUtilClient;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
index e34c7ec8bae5d..073576546c790 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java
@@ -33,7 +33,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.slf4j.Logger;
 
@@ -196,8 +196,8 @@ private void doRefreshNamenodes(
       
       // Step 2. Any nameservices we currently have but are no longer present
       // need to be removed.
-      toRemove = Sets.newHashSet(Sets.difference(
-          bpByNameserviceId.keySet(), addrMap.keySet()));
+      toRemove = Sets.difference(
+          bpByNameserviceId.keySet(), addrMap.keySet());
       
       assert toRefresh.size() + toAdd.size() ==
         addrMap.size() :
@@ -307,4 +307,27 @@ protected BPOfferService createBPOS(
   Map<String, BPOfferService> getBpByNameserviceId() {
     return bpByNameserviceId;
   }
+
+  boolean isSlownodeByNameserviceId(String nsId) {
+    if (bpByNameserviceId.containsKey(nsId)) {
+      return bpByNameserviceId.get(nsId).isSlownode();
+    }
+    return false;
+  }
+
+  boolean isSlownodeByBlockPoolId(String bpId) {
+    if (bpByBlockPoolId.containsKey(bpId)) {
+      return bpByBlockPoolId.get(bpId).isSlownode();
+    }
+    return false;
+  }
+
+  boolean isSlownode() {
+    for (BPOfferService bpOfferService : bpByBlockPoolId.values()) {
+      if (bpOfferService.isSlownode()) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
index 8dad8009297cd..a82ed0045a6a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
@@ -48,7 +48,7 @@
 import org.apache.hadoop.util.Lists;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Manages storage for the set of BlockPoolSlices which share a particular 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index cf1f688c11d44..77e0be6c7b32e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -307,6 +307,17 @@ Replica getReplica() {
     return replicaInfo;
   }
 
+  public void releaseAnyRemainingReservedSpace() {
+    if (replicaInfo != null) {
+      if (replicaInfo.getReplicaInfo().getBytesReserved() > 0) {
+        LOG.warn("Block {} has not released the reserved bytes. "
+                + "Releasing {} bytes as part of close.", replicaInfo.getBlockId(),
+            replicaInfo.getReplicaInfo().getBytesReserved());
+        replicaInfo.releaseAllBytesReserved();
+      }
+    }
+  }
+
   /**
    * close files and release volume reference.
    */
@@ -412,6 +423,7 @@ boolean packetSentInTime() {
   void flushOrSync(boolean isSync, long seqno) throws IOException {
     long flushTotalNanos = 0;
     long begin = Time.monotonicNow();
+    DataNodeFaultInjector.get().delay();
     if (checksumOut != null) {
       long flushStartNanos = System.nanoTime();
       checksumOut.flush();
@@ -445,6 +457,7 @@ void flushOrSync(boolean isSync, long seqno) throws IOException {
     }
     long duration = Time.monotonicNow() - begin;
     if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
+      datanode.metrics.incrSlowFlushOrSyncCount();
       LOG.warn("Slow flushOrSync took " + duration + "ms (threshold="
           + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos="
           + flushTotalNanos + "ns, volume=" + getVolumeBaseUri()
@@ -887,7 +900,7 @@ duration, datanodeSlowLogThresholdMs, getVolumeBaseUri(),
    */
   private void trackSendPacketToLastNodeInPipeline(final long elapsedMs) {
     final DataNodePeerMetrics peerMetrics = datanode.getPeerMetrics();
-    if (peerMetrics != null && isPenultimateNode) {
+    if (datanode.getDnConf().peerStatsEnabled && peerMetrics != null && isPenultimateNode) {
       peerMetrics.addSendPacketDownstream(mirrorNameForMetrics, elapsedMs);
     }
   }
@@ -1107,7 +1120,7 @@ private void initPerfMonitoring(DatanodeInfo[] downstreams) {
     if (downstreams != null && downstreams.length > 0) {
       downstreamDNs = downstreams;
       isPenultimateNode = (downstreams.length == 1);
-      if (isPenultimateNode && datanode.getPeerMetrics() != null) {
+      if (isPenultimateNode && datanode.getDnConf().peerStatsEnabled) {
         mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ?
             downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr());
         LOG.debug("Will collect peer metrics for downstream node {}",
@@ -1327,7 +1340,8 @@ void sendOOBResponse(final Status ackStatus) throws IOException,
       LOG.info("Sending an out of band ack of type " + ackStatus);
       try {
         sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L,
-            PipelineAck.combineHeader(datanode.getECN(), ackStatus));
+            PipelineAck.combineHeader(datanode.getECN(), ackStatus,
+                datanode.getSLOWByBlockPoolId(block.getBlockPoolId())));
       } finally {
         // Let others send ack. Unless there are miltiple OOB send
         // calls, there can be only one waiter, the responder thread.
@@ -1409,7 +1423,8 @@ public void run() {
                 LOG.info("Relaying an out of band ack of type " + oobStatus);
                 sendAckUpstream(ack, PipelineAck.UNKOWN_SEQNO, 0L, 0L,
                     PipelineAck.combineHeader(datanode.getECN(),
-                      Status.SUCCESS));
+                      Status.SUCCESS,
+                      datanode.getSLOWByBlockPoolId(block.getBlockPoolId())));
                 continue;
               }
               seqno = ack.getSeqno();
@@ -1499,7 +1514,8 @@ public void run() {
           Status myStatus = pkt != null ? pkt.ackStatus : Status.SUCCESS;
           sendAckUpstream(ack, expected, totalAckTimeNanos,
             (pkt != null ? pkt.offsetInBlock : 0),
-            PipelineAck.combineHeader(datanode.getECN(), myStatus));
+              PipelineAck.combineHeader(datanode.getECN(), myStatus,
+                  datanode.getSLOWByBlockPoolId(block.getBlockPoolId())));
           if (pkt != null) {
             // remove the packet from the ack queue
             removeAckHead();
@@ -1620,8 +1636,10 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
         // downstream nodes, reply should contain one reply.
         replies = new int[] { myHeader };
       } else if (mirrorError) { // ack read error
-        int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS);
-        int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR);
+        int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS,
+            datanode.getSLOWByBlockPoolId(block.getBlockPoolId()));
+        int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR,
+            datanode.getSLOWByBlockPoolId(block.getBlockPoolId()));
         replies = new int[] {h, h1};
       } else {
         short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack
@@ -1631,6 +1649,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
         for (int i = 0; i < ackLen; ++i) {
           replies[i + 1] = ack.getHeaderFlag(i);
         }
+        DataNodeFaultInjector.get().markSlow(mirrorAddr, replies);
         // If the mirror has reported that it received a corrupt packet,
         // do self-destruct to mark myself bad, instead of making the
         // mirror node bad. The mirror is guaranteed to be good without
@@ -1650,6 +1669,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
       }
       // send my ack back to upstream datanode
       long begin = Time.monotonicNow();
+      DataNodeFaultInjector.get().delay();
       /* for test only, no-op in production system */
       DataNodeFaultInjector.get().delaySendingAckToUpstream(inAddr);
       replyAck.write(upstreamOut);
@@ -1659,6 +1679,7 @@ private void sendAckUpstreamUnprotected(PipelineAck ack, long seqno,
           inAddr,
           duration);
       if (duration > datanodeSlowLogThresholdMs) {
+        datanode.metrics.incrSlowAckToUpstreamCount();
         LOG.warn("Slow PacketResponder send ack to upstream took " + duration
             + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString
             + ", replyAck=" + replyAck
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
index d575b592bd443..e4861f9774870 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java
@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 53da12f774293..69e3db11bdf73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.ScanResultHandler;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index d3cdee11381ca..5c4212fea537f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -41,6 +41,7 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
@@ -57,7 +58,7 @@
 import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_SEQUENTIAL;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 
 /**
@@ -256,7 +257,8 @@ class BlockSender implements java.io.Closeable {
       // the append write.
       ChunkChecksum chunkChecksum = null;
       final long replicaVisibleLength;
-      try(AutoCloseableLock lock = datanode.data.acquireDatasetReadLock()) {
+      try (AutoCloseableLock lock = datanode.getDataSetLockManager().readLock(
+          LockLevel.BLOCK_POOl, block.getBlockPoolId())) {
         replica = getReplica(block, datanode);
         replicaVisibleLength = replica.getVisibleLength();
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 7f86668fbb93d..9b5343321d30b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -74,6 +74,7 @@
 import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
 import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.util.Preconditions;
 
 import java.util.concurrent.TimeUnit;
 
@@ -105,14 +106,14 @@ public class DNConf {
   final long readaheadLength;
   final long heartBeatInterval;
   private final long lifelineIntervalMs;
-  final long blockReportInterval;
-  final long blockReportSplitThreshold;
-  final boolean peerStatsEnabled;
-  final boolean diskStatsEnabled;
-  final long outliersReportIntervalMs;
+  volatile long blockReportInterval;
+  volatile long blockReportSplitThreshold;
+  volatile boolean peerStatsEnabled;
+  volatile boolean diskStatsEnabled;
+  volatile long outliersReportIntervalMs;
   final long ibrInterval;
-  final long initialBlockReportDelayMs;
-  final long cacheReportInterval;
+  volatile long initialBlockReportDelayMs;
+  volatile long cacheReportInterval;
   final long datanodeSlowIoWarningThresholdMs;
 
   final String minimumNameNodeVersion;
@@ -214,19 +215,7 @@ public DNConf(final Configurable dn) {
     this.datanodeSlowIoWarningThresholdMs = getConf().getLong(
         DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY,
         DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
-
-    long initBRDelay = getConf().getTimeDuration(
-        DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
-        DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT,
-        TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
-    if (initBRDelay >= blockReportInterval) {
-      initBRDelay = 0;
-      DataNode.LOG.info(DFS_BLOCKREPORT_INITIAL_DELAY_KEY + " is "
-          + "greater than or equal to" + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY
-          + ".  Setting initial delay to 0 msec:");
-    }
-    initialBlockReportDelayMs = initBRDelay;
-    
+    initBlockReportDelay();
     heartBeatInterval = getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
         DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS,
         TimeUnit.MILLISECONDS);
@@ -310,6 +299,19 @@ public DNConf(final Configurable dn) {
     );
   }
 
+  private void initBlockReportDelay() {
+    long initBRDelay = getConf().getTimeDuration(
+        DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
+        DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
+    if (initBRDelay >= blockReportInterval || initBRDelay < 0) {
+      initBRDelay = 0;
+      DataNode.LOG.info(DFS_BLOCKREPORT_INITIAL_DELAY_KEY +
+          " is greater than or equal to " + DFS_BLOCKREPORT_INTERVAL_MSEC_KEY +
+          ". Setting initial delay to 0 msec.");
+    }
+    initialBlockReportDelayMs = initBRDelay;
+  }
+
   // We get minimumNameNodeVersion via a method so it can be mocked out in tests.
   String getMinimumNameNodeVersion() {
     return this.minimumNameNodeVersion;
@@ -475,7 +477,49 @@ public long getProcessCommandsThresholdMs() {
     return processCommandsThresholdMs;
   }
 
+  void setBlockReportInterval(long intervalMs) {
+    Preconditions.checkArgument(intervalMs > 0,
+        DFS_BLOCKREPORT_INTERVAL_MSEC_KEY + " should be larger than 0");
+    blockReportInterval = intervalMs;
+  }
+
   public long getBlockReportInterval() {
     return blockReportInterval;
   }
+
+  void setCacheReportInterval(long intervalMs) {
+    Preconditions.checkArgument(intervalMs > 0,
+        DFS_CACHEREPORT_INTERVAL_MSEC_KEY + " should be larger than 0");
+    cacheReportInterval = intervalMs;
+  }
+
+  public long getCacheReportInterval() {
+    return cacheReportInterval;
+  }
+
+  void setBlockReportSplitThreshold(long threshold) {
+    Preconditions.checkArgument(threshold >= 0,
+        DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY + " should be larger than or equal to 0");
+    blockReportSplitThreshold = threshold;
+  }
+
+  void setInitBRDelayMs(String delayMs) {
+    dn.getConf().set(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, delayMs);
+    initBlockReportDelay();
+  }
+
+  void setPeerStatsEnabled(boolean enablePeerStats) {
+    peerStatsEnabled = enablePeerStats;
+  }
+
+  public void setFileIoProfilingSamplingPercentage(int samplingPercentage) {
+    diskStatsEnabled = Util.isDiskStatsEnabled(samplingPercentage);
+  }
+
+  public void setOutliersReportIntervalMs(String reportIntervalMs) {
+    dn.getConf().set(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, reportIntervalMs);
+    outliersReportIntervalMs = getConf().getTimeDuration(
+        DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+        DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 4625c67873d66..58334bf5c0750 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -18,6 +18,19 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ALLOW_SAME_DISK_TIERING;
@@ -27,6 +40,8 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
@@ -36,11 +51,27 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NETWORK_COUNTS_CACHE_MAX_SIZE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_NETWORK_COUNTS_CACHE_MAX_SIZE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OOB_TIMEOUT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
@@ -52,9 +83,14 @@
 import static org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage.PIPELINE_SETUP_CREATE;
 import static org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage.PIPELINE_SETUP_STREAMING_RECOVERY;
 import static org.apache.hadoop.util.ExitUtil.terminate;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Time.now;
 
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.DF;
+import org.apache.hadoop.fs.DU;
+import org.apache.hadoop.fs.GetSpaceUsed;
+import org.apache.hadoop.fs.WindowsGetSpaceUsed;
 import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ReconfigurationProtocolService;
 
 import java.io.BufferedOutputStream;
@@ -118,9 +154,13 @@
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.datanode.checker.DatasetVolumeChecker;
 import org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.util.*;
 import org.apache.hadoop.hdfs.client.BlockReportOptions;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -215,7 +255,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
 import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
@@ -299,7 +339,23 @@ public class DataNode extends ReconfigurableBase
       Collections.unmodifiableList(
           Arrays.asList(
               DFS_DATANODE_DATA_DIR_KEY,
-              DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY));
+              DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
+              DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+              DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY,
+              DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
+              DFS_DATANODE_MAX_RECEIVER_THREADS_KEY,
+              DFS_CACHEREPORT_INTERVAL_MSEC_KEY,
+              DFS_DATANODE_PEER_STATS_ENABLED_KEY,
+              DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
+              DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
+              DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY,
+              DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
+              DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY,
+              DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
+              DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
+              FS_DU_INTERVAL_KEY,
+              FS_GETSPACEUSED_JITTER_KEY,
+              FS_GETSPACEUSED_CLASSNAME));
 
   public static final Log METRICS_LOG = LogFactory.getLog("DataNodeMetricsLog");
 
@@ -341,8 +397,8 @@ public static InetSocketAddress createSocketAddr(String target) {
 
   DataNodeMetrics metrics;
   @Nullable
-  private DataNodePeerMetrics peerMetrics;
-  private DataNodeDiskMetrics diskMetrics;
+  private volatile DataNodePeerMetrics peerMetrics;
+  private volatile DataNodeDiskMetrics diskMetrics;
   private InetSocketAddress streamingAddr;
 
   private LoadingCache<String, Map<String, Long>> datanodeNetworkCounts;
@@ -372,6 +428,7 @@ public static InetSocketAddress createSocketAddr(String target) {
   private final String confVersion;
   private final long maxNumberOfBlocksToLog;
   private final boolean pipelineSupportECN;
+  private final boolean pipelineSupportSlownode;
 
   private final List<String> usersWithLocalPathAccess;
   private final boolean connectToDnViaHostname;
@@ -391,6 +448,7 @@ public static InetSocketAddress createSocketAddr(String target) {
       .availableProcessors();
   private static final double CONGESTION_RATIO = 1.5;
   private DiskBalancer diskBalancer;
+  private DataSetLockManager dataSetLockManager;
 
   private final ExecutorService xferService;
 
@@ -411,6 +469,11 @@ private static Tracer createTracer(Configuration conf) {
 
   private ScheduledThreadPoolExecutor metricsLoggerTimer;
 
+  private long startTime = 0;
+
+  private DataTransferThrottler ecReconstuctReadThrottler;
+  private DataTransferThrottler ecReconstuctWriteThrottler;
+
   /**
    * Creates a dummy DataNode for testing purpose.
    */
@@ -427,8 +490,10 @@ private static Tracer createTracer(Configuration conf) {
     this.connectToDnViaHostname = false;
     this.blockScanner = new BlockScanner(this, this.getConf());
     this.pipelineSupportECN = false;
+    this.pipelineSupportSlownode = false;
     this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
     this.dnConf = new DNConf(this);
+    this.dataSetLockManager = new DataSetLockManager(conf);
     initOOBTimeout();
     storageLocationChecker = null;
     volumeChecker = new DatasetVolumeChecker(conf, new Timer());
@@ -447,6 +512,7 @@ private static Tracer createTracer(Configuration conf) {
     super(conf);
     this.tracer = createTracer(conf);
     this.fileIoProvider = new FileIoProvider(conf, this);
+    this.dataSetLockManager = new DataSetLockManager(conf);
     this.blockScanner = new BlockScanner(this);
     this.lastDiskErrorCheck = 0;
     this.maxNumberOfBlocksToLog = conf.getLong(DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
@@ -465,6 +531,9 @@ private static Tracer createTracer(Configuration conf) {
     this.pipelineSupportECN = conf.getBoolean(
         DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED,
         DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT);
+    this.pipelineSupportSlownode = conf.getBoolean(
+        DFSConfigKeys.DFS_PIPELINE_SLOWNODE_ENABLED,
+        DFSConfigKeys.DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT);
 
     confVersion = "core-" +
         conf.get("hadoop.common.configuration.version", "UNSPECIFIED") +
@@ -519,6 +588,16 @@ public Map<String, Long> load(String key) {
 
     initOOBTimeout();
     this.storageLocationChecker = storageLocationChecker;
+    long ecReconstuctReadBandwidth = conf.getLongBytes(
+        DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_KEY,
+        DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_READ_BANDWIDTHPERSEC_DEFAULT);
+    long ecReconstuctWriteBandwidth = conf.getLongBytes(
+        DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_KEY,
+        DFSConfigKeys.DFS_DATANODE_EC_RECONSTRUCT_WRITE_BANDWIDTHPERSEC_DEFAULT);
+    this.ecReconstuctReadThrottler = ecReconstuctReadBandwidth > 0 ?
+        new DataTransferThrottler(100, ecReconstuctReadBandwidth) : null;
+    this.ecReconstuctWriteThrottler = ecReconstuctWriteBandwidth > 0 ?
+        new DataTransferThrottler(100, ecReconstuctWriteBandwidth) : null;
   }
 
   @Override  // ReconfigurableBase
@@ -533,83 +612,337 @@ protected Configuration getNewConf() {
   public String reconfigurePropertyImpl(String property, String newVal)
       throws ReconfigurationException {
     switch (property) {
-      case DFS_DATANODE_DATA_DIR_KEY: {
-        IOException rootException = null;
+    case DFS_DATANODE_DATA_DIR_KEY: {
+      IOException rootException = null;
+      try {
+        LOG.info("Reconfiguring {} to {}", property, newVal);
+        this.refreshVolumes(newVal);
+        return getConf().get(DFS_DATANODE_DATA_DIR_KEY);
+      } catch (IOException e) {
+        rootException = e;
+      } finally {
+        // Send a full block report to let NN acknowledge the volume changes.
         try {
-          LOG.info("Reconfiguring {} to {}", property, newVal);
-          this.refreshVolumes(newVal);
-          return getConf().get(DFS_DATANODE_DATA_DIR_KEY);
+          triggerBlockReport(
+              new BlockReportOptions.Factory().setIncremental(false).build());
         } catch (IOException e) {
-          rootException = e;
+          LOG.warn("Exception while sending the block report after refreshing"
+              + " volumes {} to {}", property, newVal, e);
+          if (rootException == null) {
+            rootException = e;
+          }
         } finally {
-          // Send a full block report to let NN acknowledge the volume changes.
-          try {
-            triggerBlockReport(
-                new BlockReportOptions.Factory().setIncremental(false).build());
-          } catch (IOException e) {
-            LOG.warn("Exception while sending the block report after refreshing"
-                + " volumes {} to {}", property, newVal, e);
-            if (rootException == null) {
-              rootException = e;
-            }
-          } finally {
-            if (rootException != null) {
-              throw new ReconfigurationException(property, newVal,
-                  getConf().get(property), rootException);
-            }
+          if (rootException != null) {
+            throw new ReconfigurationException(property, newVal,
+                getConf().get(property), rootException);
           }
         }
-        break;
       }
-      case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: {
-        ReconfigurationException rootException = null;
-        try {
-          LOG.info("Reconfiguring {} to {}", property, newVal);
-          int movers;
-          if (newVal == null) {
-            // set to default
-            movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
-          } else {
-            movers = Integer.parseInt(newVal);
-            if (movers <= 0) {
-              rootException = new ReconfigurationException(
-                  property,
-                  newVal,
-                  getConf().get(property),
-                  new IllegalArgumentException(
-                      "balancer max concurrent movers must be larger than 0"));
-            }
-          }
-          boolean success = xserver.updateBalancerMaxConcurrentMovers(movers);
-          if (!success) {
+      break;
+    }
+    case DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY: {
+      ReconfigurationException rootException = null;
+      try {
+        LOG.info("Reconfiguring {} to {}", property, newVal);
+        int movers;
+        if (newVal == null) {
+          // set to default
+          movers = DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
+        } else {
+          movers = Integer.parseInt(newVal);
+          if (movers <= 0) {
             rootException = new ReconfigurationException(
                 property,
                 newVal,
                 getConf().get(property),
                 new IllegalArgumentException(
-                    "Could not modify concurrent moves thread count"));
+                    "balancer max concurrent movers must be larger than 0"));
           }
-          return Integer.toString(movers);
-        } catch (NumberFormatException nfe) {
+        }
+        boolean success = xserver.updateBalancerMaxConcurrentMovers(movers);
+        if (!success) {
           rootException = new ReconfigurationException(
-              property, newVal, getConf().get(property), nfe);
-        } finally {
-          if (rootException != null) {
-            LOG.warn(String.format(
-                "Exception in updating balancer max concurrent movers %s to %s",
-                property, newVal), rootException);
-            throw rootException;
-          }
+              property,
+              newVal,
+              getConf().get(property),
+              new IllegalArgumentException(
+                  "Could not modify concurrent moves thread count"));
+        }
+        return Integer.toString(movers);
+      } catch (NumberFormatException nfe) {
+        rootException = new ReconfigurationException(
+            property, newVal, getConf().get(property), nfe);
+      } finally {
+        if (rootException != null) {
+          LOG.warn(String.format(
+              "Exception in updating balancer max concurrent movers %s to %s",
+              property, newVal), rootException);
+          throw rootException;
         }
-        break;
       }
-      default:
-        break;
+      break;
+    }
+    case DFS_BLOCKREPORT_INTERVAL_MSEC_KEY:
+    case DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY:
+    case DFS_BLOCKREPORT_INITIAL_DELAY_KEY:
+      return reconfBlockReportParameters(property, newVal);
+    case DFS_DATANODE_MAX_RECEIVER_THREADS_KEY:
+      return reconfDataXceiverParameters(property, newVal);
+    case DFS_CACHEREPORT_INTERVAL_MSEC_KEY:
+      return reconfCacheReportParameters(property, newVal);
+    case DFS_DATANODE_PEER_STATS_ENABLED_KEY:
+    case DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY:
+    case DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY:
+    case DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY:
+      return reconfSlowPeerParameters(property, newVal);
+    case DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY:
+    case DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY:
+    case DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY:
+    case DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY:
+      return reconfSlowDiskParameters(property, newVal);
+    case FS_DU_INTERVAL_KEY:
+    case FS_GETSPACEUSED_JITTER_KEY:
+    case FS_GETSPACEUSED_CLASSNAME:
+      return reconfDfsUsageParameters(property, newVal);
+    default:
+      break;
     }
     throw new ReconfigurationException(
         property, newVal, getConf().get(property));
   }
 
+  private String reconfDataXceiverParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      Preconditions.checkNotNull(getXferServer(), "DataXceiverServer has not been initialized.");
+      int threads = (newVal == null ? DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT :
+          Integer.parseInt(newVal));
+      result = Integer.toString(threads);
+      getXferServer().setMaxXceiverCount(threads);
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
+  private String reconfCacheReportParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      Preconditions.checkNotNull(dnConf, "DNConf has not been initialized.");
+      long reportInterval = (newVal == null ? DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT :
+          Long.parseLong(newVal));
+      result = Long.toString(reportInterval);
+      dnConf.setCacheReportInterval(reportInterval);
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
+  private String reconfBlockReportParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result = null;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      if (property.equals(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY)) {
+        Preconditions.checkNotNull(dnConf, "DNConf has not been initialized.");
+        long intervalMs = newVal == null ? DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT :
+            Long.parseLong(newVal);
+        result = Long.toString(intervalMs);
+        dnConf.setBlockReportInterval(intervalMs);
+        for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+          if (bpos != null) {
+            for (BPServiceActor actor : bpos.getBPServiceActors()) {
+              actor.getScheduler().setBlockReportIntervalMs(intervalMs);
+            }
+          }
+        }
+      } else if (property.equals(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY)) {
+        Preconditions.checkNotNull(dnConf, "DNConf has not been initialized.");
+        long threshold = newVal == null ? DFS_BLOCKREPORT_SPLIT_THRESHOLD_DEFAULT :
+            Long.parseLong(newVal);
+        result = Long.toString(threshold);
+        dnConf.setBlockReportSplitThreshold(threshold);
+      } else if (property.equals(DFS_BLOCKREPORT_INITIAL_DELAY_KEY)) {
+        Preconditions.checkNotNull(dnConf, "DNConf has not been initialized.");
+        int initialDelay = newVal == null ? DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT :
+            Integer.parseInt(newVal);
+        result = Integer.toString(initialDelay);
+        dnConf.setInitBRDelayMs(result);
+      }
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
+  private String reconfSlowPeerParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result = null;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      if (property.equals(DFS_DATANODE_PEER_STATS_ENABLED_KEY)) {
+        Preconditions.checkNotNull(dnConf, "DNConf has not been initialized.");
+        if (newVal != null && !newVal.equalsIgnoreCase("true")
+            && !newVal.equalsIgnoreCase("false")) {
+          throw new IllegalArgumentException("Not a valid Boolean value for " + property +
+              " in reconfSlowPeerParameters");
+        }
+        boolean enable = (newVal == null ? DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT :
+            Boolean.parseBoolean(newVal));
+        result = Boolean.toString(enable);
+        dnConf.setPeerStatsEnabled(enable);
+        if (enable) {
+          // Create if it doesn't exist, overwrite if it does.
+          peerMetrics = DataNodePeerMetrics.create(getDisplayName(), getConf());
+        }
+      } else if (property.equals(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY)) {
+        Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled.");
+        long minNodes = (newVal == null ? DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(minNodes);
+        peerMetrics.setMinOutlierDetectionNodes(minNodes);
+      } else if (property.equals(DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY)) {
+        Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled.");
+        long threshold = (newVal == null ? DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(threshold);
+        peerMetrics.setLowThresholdMs(threshold);
+      } else if (property.equals(DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY)) {
+        Preconditions.checkNotNull(peerMetrics, "DataNode peer stats may be disabled.");
+        long minSamples = (newVal == null ?
+            DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(minSamples);
+        peerMetrics.setMinOutlierDetectionSamples(minSamples);
+      }
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
+  private String reconfSlowDiskParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result = null;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      if (property.equals(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY)) {
+        checkNotNull(dnConf, "DNConf has not been initialized.");
+        String reportInterval = (newVal == null ? DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT :
+            newVal);
+        result = reportInterval;
+        dnConf.setOutliersReportIntervalMs(reportInterval);
+        for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+          if (bpos != null) {
+            for (BPServiceActor actor : bpos.getBPServiceActors()) {
+              actor.getScheduler().setOutliersReportIntervalMs(
+                  dnConf.outliersReportIntervalMs);
+            }
+          }
+        }
+      } else if (property.equals(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY)) {
+        checkNotNull(dnConf, "DNConf has not been initialized.");
+        int samplingPercentage = (newVal == null ?
+            DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT :
+            Integer.parseInt(newVal));
+        result = Integer.toString(samplingPercentage);
+        dnConf.setFileIoProfilingSamplingPercentage(samplingPercentage);
+        if (fileIoProvider != null) {
+          fileIoProvider.getProfilingEventHook().setSampleRangeMax(samplingPercentage);
+        }
+        if (samplingPercentage > 0 && diskMetrics == null) {
+          diskMetrics = new DataNodeDiskMetrics(this,
+              dnConf.outliersReportIntervalMs, getConf());
+        } else if (samplingPercentage <= 0 && diskMetrics != null) {
+          diskMetrics.shutdownAndWait();
+        }
+      } else if (property.equals(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY)) {
+        checkNotNull(diskMetrics, "DataNode disk stats may be disabled.");
+        long minDisks = (newVal == null ? DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(minDisks);
+        diskMetrics.setMinOutlierDetectionDisks(minDisks);
+      } else if (property.equals(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY)) {
+        checkNotNull(diskMetrics, "DataNode disk stats may be disabled.");
+        long threshold = (newVal == null ? DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(threshold);
+        diskMetrics.setLowThresholdMs(threshold);
+      }
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
+  private String reconfDfsUsageParameters(String property, String newVal)
+      throws ReconfigurationException {
+    String result = null;
+    try {
+      LOG.info("Reconfiguring {} to {}", property, newVal);
+      if (data == null) {
+        LOG.debug("FsDatasetSpi has not been initialized.");
+        throw new IOException("FsDatasetSpi has not been initialized");
+      }
+      if (property.equals(FS_DU_INTERVAL_KEY)) {
+        long interval = (newVal == null ? FS_DU_INTERVAL_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(interval);
+        List<FsVolumeImpl> volumeList = data.getVolumeList();
+        for (FsVolumeImpl fsVolume : volumeList) {
+          Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
+          for (BlockPoolSlice value : blockPoolSlices.values()) {
+            value.updateDfsUsageConfig(interval, null, null);
+          }
+        }
+      } else if (property.equals(FS_GETSPACEUSED_JITTER_KEY)) {
+        long jitter = (newVal == null ? FS_GETSPACEUSED_JITTER_DEFAULT :
+            Long.parseLong(newVal));
+        result = Long.toString(jitter);
+        List<FsVolumeImpl> volumeList = data.getVolumeList();
+        for (FsVolumeImpl fsVolume : volumeList) {
+          Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
+          for (BlockPoolSlice value : blockPoolSlices.values()) {
+            value.updateDfsUsageConfig(null, jitter, null);
+          }
+        }
+      } else if (property.equals(FS_GETSPACEUSED_CLASSNAME)) {
+        Class<? extends GetSpaceUsed> klass;
+        if (newVal == null) {
+          if (Shell.WINDOWS) {
+            klass = DU.class;
+          } else {
+            klass = WindowsGetSpaceUsed.class;
+          }
+        } else {
+          klass = Class.forName(newVal).asSubclass(GetSpaceUsed.class);
+        }
+        result = klass.getName();
+        List<FsVolumeImpl> volumeList = data.getVolumeList();
+        for (FsVolumeImpl fsVolume : volumeList) {
+          Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
+          for (BlockPoolSlice value : blockPoolSlices.values()) {
+            value.updateDfsUsageConfig(null, null, klass);
+          }
+        }
+      }
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException | IOException | ClassNotFoundException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
   /**
    * Get a list of the keys of the re-configurable properties in configuration.
    */
@@ -636,6 +969,23 @@ public PipelineAck.ECN getECN() {
         PipelineAck.ECN.SUPPORTED;
   }
 
+  /**
+   * The SLOW bit for the DataNode of the specific BlockPool.
+   * The DataNode should return:
+   * <ul>
+   *   <li>SLOW.DISABLED when SLOW is disabled
+   *   <li>SLOW.NORMAL when SLOW is enabled and DN is not slownode.</li>
+   *   <li>SLOW.SLOW when SLOW is enabled and DN is slownode.</li>
+   * </ul>
+   */
+  public PipelineAck.SLOW getSLOWByBlockPoolId(String bpId) {
+    if (!pipelineSupportSlownode) {
+      return PipelineAck.SLOW.DISABLED;
+    }
+    return isSlownodeByBlockPoolId(bpId) ? PipelineAck.SLOW.SLOW :
+        PipelineAck.SLOW.NORMAL;
+  }
+
   public FileIoProvider getFileIoProvider() {
     return fileIoProvider;
   }
@@ -826,6 +1176,7 @@ private void refreshVolumes(String newVolumes) throws IOException {
               .newFixedThreadPool(changedVolumes.newLocations.size());
           List<Future<IOException>> exceptions = Lists.newArrayList();
 
+          checkStorageState("refreshVolumes");
           for (final StorageLocation location : changedVolumes.newLocations) {
             exceptions.add(service.submit(new Callable<IOException>() {
               @Override
@@ -925,6 +1276,7 @@ private synchronized void removeVolumes(
         clearFailure, Joiner.on(",").join(storageLocations)));
 
     IOException ioe = null;
+    checkStorageState("removeVolumes");
     // Remove volumes and block infos from FsDataset.
     data.removeVolumes(storageLocations, clearFailure);
 
@@ -1088,7 +1440,7 @@ private void checkSuperuserPrivilege() throws IOException, AccessControlExceptio
       return;
     }
     // Try to get the ugi in the RPC call.
-    UserGroupInformation callerUgi = ipcServer.getRemoteUser();
+    UserGroupInformation callerUgi = Server.getRemoteUser();
     if (callerUgi == null) {
       // This is not from RPC.
       callerUgi = UserGroupInformation.getCurrentUser();
@@ -1383,7 +1735,7 @@ boolean areCacheReportsDisabledForTests() {
   /**
    * This method starts the data node with the specified conf.
    * 
-   * If conf's CONFIG_PROPERTY_SIMULATED property is set
+   * If conf's DFS_DATANODE_FSDATASET_FACTORY_KEY property is set
    * then a simulated storage based data node is created.
    * 
    * @param dataDirectories - only for a non-simulated storage data node
@@ -1951,7 +2303,7 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block,
       Token<BlockTokenIdentifier> token) throws IOException {
     checkBlockLocalPathAccess();
     checkBlockToken(block, token, BlockTokenIdentifier.AccessMode.READ);
-    Preconditions.checkNotNull(data, "Storage not yet initialized");
+    checkStorageState("getBlockLocalPathInfo");
     BlockLocalPathInfo info = data.getBlockLocalPathInfo(block);
     if (info != null) {
       LOG.trace("getBlockLocalPathInfo successful " +
@@ -2001,6 +2353,7 @@ FileInputStream[] requestShortCircuitFdsForRead(final ExtendedBlock blk,
     FileInputStream fis[] = new FileInputStream[2];
     
     try {
+      checkStorageState("requestShortCircuitFdsForRead");
       fis[0] = (FileInputStream)data.getBlockInputStream(blk, 0);
       fis[1] = DatanodeUtil.getMetaDataInputStream(blk, data);
     } catch (ClassCastException e) {
@@ -2185,7 +2538,7 @@ public void shutdown() {
     if (metrics != null) {
       metrics.shutdown();
     }
-    if (diskMetrics != null) {
+    if (dnConf.diskStatsEnabled && diskMetrics != null) {
       diskMetrics.shutdownAndWait();
     }
     if (dataNodeInfoBeanName != null) {
@@ -2201,6 +2554,7 @@ public void shutdown() {
       notifyAll();
     }
     tracer.close();
+    dataSetLockManager.lockLeakCheck();
   }
 
   /**
@@ -2718,6 +3072,7 @@ public void runDatanodeDaemon() throws IOException {
     }
     ipcServer.setTracer(tracer);
     ipcServer.start();
+    startTime = now();
     startPlugins(getConf());
   }
 
@@ -3029,6 +3384,7 @@ public static void main(String args[]) {
   @Override // InterDatanodeProtocol
   public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
       throws IOException {
+    checkStorageState("initReplicaRecovery");
     return data.initReplicaRecovery(rBlock);
   }
 
@@ -3039,6 +3395,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
   public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock,
       final long recoveryId, final long newBlockId, final long newLength)
       throws IOException {
+    checkStorageState("updateReplicaUnderRecovery");
     final Replica r = data.updateReplicaUnderRecovery(oldBlock,
         recoveryId, newBlockId, newLength);
     // Notify the namenode of the updated block info. This is important
@@ -3104,7 +3461,8 @@ void transferReplicaForPipelineRecovery(final ExtendedBlock b,
     final BlockConstructionStage stage;
 
     //get replica information
-    try(AutoCloseableLock lock = data.acquireDatasetReadLock()) {
+    try (AutoCloseableLock lock = dataSetLockManager.readLock(
+        LockLevel.BLOCK_POOl, b.getBlockPoolId())) {
       Block storedBlock = data.getStoredBlock(b.getBlockPoolId(),
           b.getBlockId());
       if (null == storedBlock) {
@@ -3195,6 +3553,11 @@ public String getHttpPort(){
     return this.getConf().get("dfs.datanode.info.port");
   }
 
+  @Override // DataNodeMXBean
+  public long getDNStartedTimeInMillis() {
+    return this.startTime;
+  }
+
   public String getRevision() {
     return VersionInfo.getRevision();
   }
@@ -3267,7 +3630,10 @@ public String getBPServiceActorInfo() {
    */
   @Override // DataNodeMXBean
   public String getVolumeInfo() {
-    Preconditions.checkNotNull(data, "Storage not yet initialized");
+    if (data == null) {
+      LOG.debug("Storage not yet initialized.");
+      return "";
+    }
     return JSON.toString(data.getVolumeInfoMap());
   }
   
@@ -3281,7 +3647,7 @@ public String getDiskBalancerStatus() {
     try {
       return getDiskBalancer().queryWorkStatus().toJsonString();
     } catch (IOException ex) {
-      LOG.debug("Reading diskbalancer Status failed. ex:{}", ex);
+      LOG.debug("Reading diskbalancer Status failed.", ex);
       return "";
     }
   }
@@ -3315,10 +3681,23 @@ public void deleteBlockPool(String blockPoolId, boolean force)
           "The block pool is still running. First do a refreshNamenodes to " +
           "shutdown the block pool service");
     }
-   
+    checkStorageState("deleteBlockPool");
     data.deleteBlockPool(blockPoolId, force);
   }
 
+  /**
+   * Check if storage has been initialized.
+   * @param methodName caller name
+   * @throws IOException throw IOException if not yet initialized.
+   */
+  private void checkStorageState(String methodName) throws IOException {
+    if (data == null) {
+      String message = "Storage not yet initialized for " + methodName;
+      LOG.debug(message);
+      throw new IOException(message);
+    }
+  }
+
   @Override // ClientDatanodeProtocol
   public synchronized void shutdownDatanode(boolean forUpgrade) throws IOException {
     checkSuperuserPrivilege();
@@ -3482,6 +3861,14 @@ public ShortCircuitRegistry getShortCircuitRegistry() {
     return shortCircuitRegistry;
   }
 
+  public DataTransferThrottler getEcReconstuctReadThrottler() {
+    return ecReconstuctReadThrottler;
+  }
+
+  public DataTransferThrottler getEcReconstuctWriteThrottler() {
+    return ecReconstuctWriteThrottler;
+  }
+
   /**
    * Check the disk error synchronously.
    */
@@ -3741,13 +4128,13 @@ void setBlockScanner(BlockScanner blockScanner) {
 
   @Override // DataNodeMXBean
   public String getSendPacketDownstreamAvgInfo() {
-    return peerMetrics != null ?
+    return dnConf.peerStatsEnabled && peerMetrics != null ?
         peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson() : null;
   }
 
   @Override // DataNodeMXBean
   public String getSlowDisks() {
-    if (diskMetrics == null) {
+    if (!dnConf.diskStatsEnabled || diskMetrics == null) {
       //Disk Stats not enabled
       return null;
     }
@@ -3759,6 +4146,7 @@ public String getSlowDisks() {
   @Override
   public List<DatanodeVolumeInfo> getVolumeReport() throws IOException {
     checkSuperuserPrivilege();
+    checkStorageState("getVolumeReport");
     Map<String, Object> volumeInfoMap = data.getVolumeInfoMap();
     if (volumeInfoMap == null) {
       LOG.warn("DataNode volume info not available.");
@@ -3814,4 +4202,25 @@ private static boolean isWrite(BlockConstructionStage stage) {
     return (stage == PIPELINE_SETUP_STREAMING_RECOVERY
         || stage == PIPELINE_SETUP_APPEND_RECOVERY);
   }
+
+  public DataSetLockManager getDataSetLockManager() {
+    return dataSetLockManager;
+  }
+
+  boolean isSlownodeByNameserviceId(String nsId) {
+    return blockPoolManager.isSlownodeByNameserviceId(nsId);
+  }
+
+  boolean isSlownodeByBlockPoolId(String bpId) {
+    return blockPoolManager.isSlownodeByBlockPoolId(bpId);
+  }
+
+  boolean isSlownode() {
+    return blockPoolManager.isSlownode();
+  }
+
+  @VisibleForTesting
+  public BlockPoolManager getBlockPoolManager() {
+    return blockPoolManager;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
index 1d7bcf1d50f9c..7b116d9e566f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java
@@ -155,4 +155,11 @@ public void delay() {}
    * into an erasure coding reconstruction.
    */
   public void badDecoding(ByteBuffer[] outputs) {}
+
+  public void markSlow(String dnAddr, int[] replies) {}
+
+  /**
+   * Just delay delete replica a while.
+   */
+  public void delayDeleteReplica() {}
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
index 7a8f59bb667a5..65537754741cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java
@@ -157,4 +157,11 @@ public interface DataNodeMXBean {
    * @return true, if security is enabled.
    */
   boolean isSecurityEnabled();
+
+  /**
+   * Get the start time of the DataNode.
+   *
+   * @return Start time of the DataNode.
+   */
+  long getDNStartedTimeInMillis();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
new file mode 100644
index 0000000000000..eac1259fb84fd
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
@@ -0,0 +1,296 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+
+import java.util.HashMap;
+import java.util.Stack;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Class for maintain a set of lock for fsDataSetImpl.
+ */
+public class DataSetLockManager implements DataNodeLockManager<AutoCloseDataSetLock> {
+  public static final Log LOG = LogFactory.getLog(DataSetLockManager.class);
+  private final HashMap<String, TrackLog> threadCountMap = new HashMap<>();
+  private final LockMap lockMap = new LockMap();
+  private boolean isFair = true;
+  private final boolean openLockTrace;
+  private Exception lastException;
+
+  /**
+   * Class for maintain lockMap and is thread safe.
+   */
+  private class LockMap {
+    private final HashMap<String, AutoCloseDataSetLock> readlockMap = new HashMap<>();
+    private final HashMap<String, AutoCloseDataSetLock> writeLockMap = new HashMap<>();
+
+    public synchronized void addLock(String name, ReentrantReadWriteLock lock) {
+      AutoCloseDataSetLock readLock = new AutoCloseDataSetLock(lock.readLock());
+      AutoCloseDataSetLock writeLock = new AutoCloseDataSetLock(lock.writeLock());
+      if (openLockTrace) {
+        readLock.setDataNodeLockManager(DataSetLockManager.this);
+        writeLock.setDataNodeLockManager(DataSetLockManager.this);
+      }
+      readlockMap.putIfAbsent(name, readLock);
+      writeLockMap.putIfAbsent(name, writeLock);
+    }
+
+    public synchronized void removeLock(String name) {
+      if (!readlockMap.containsKey(name) || !writeLockMap.containsKey(name)) {
+        LOG.error("The lock " + name + " is not in LockMap");
+      }
+      readlockMap.remove(name);
+      writeLockMap.remove(name);
+    }
+
+    public synchronized AutoCloseDataSetLock getReadLock(String name) {
+      return readlockMap.get(name);
+    }
+
+    public synchronized AutoCloseDataSetLock getWriteLock(String name) {
+      return writeLockMap.get(name);
+    }
+  }
+
+  /**
+   * Generate lock order string concatenates with lock name.
+   * @param level which level lock want to acquire.
+   * @param resources lock name by lock order.
+   * @return lock order string concatenates with lock name.
+   */
+  private String generateLockName(LockLevel level, String... resources) {
+    if (resources.length == 1 && level == LockLevel.BLOCK_POOl) {
+      if (resources[0] == null) {
+        throw new IllegalArgumentException("acquire a null block pool lock");
+      }
+      return resources[0];
+    } else if (resources.length == 2 && level == LockLevel.VOLUME) {
+      if (resources[0] == null || resources[1] == null) {
+        throw new IllegalArgumentException("acquire a null bp lock : "
+            + resources[0] + "volume lock :" + resources[1]);
+      }
+      return resources[0] + resources[1];
+    } else {
+      throw new IllegalArgumentException("lock level do not match resource");
+    }
+  }
+
+  /**
+   * Class for record thread acquire lock stack trace and count.
+   */
+  private static class TrackLog {
+    private final Stack<Exception> logStack = new Stack<>();
+    private int lockCount = 0;
+    private final String threadName;
+
+    TrackLog(String threadName) {
+      this.threadName = threadName;
+      incrLockCount();
+    }
+
+    public void incrLockCount() {
+      logStack.push(new Exception("lock stack trace"));
+      lockCount += 1;
+    }
+
+    public void decrLockCount() {
+      logStack.pop();
+      lockCount -= 1;
+    }
+
+    public void showLockMessage() {
+      LOG.error("hold lock thread name is:" + threadName +
+          " hold count is:" + lockCount);
+      while (!logStack.isEmpty()) {
+        Exception e = logStack.pop();
+        LOG.error("lock stack ", e);
+      }
+    }
+
+    public boolean shouldClear() {
+      return lockCount == 1;
+    }
+  }
+
+  public DataSetLockManager(Configuration conf) {
+    this.isFair = conf.getBoolean(
+        DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_KEY,
+        DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_DEFAULT);
+    this.openLockTrace = conf.getBoolean(
+        DFSConfigKeys.DFS_DATANODE_LOCKMANAGER_TRACE,
+        DFSConfigKeys.DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT);
+  }
+
+  public DataSetLockManager() {
+    this.openLockTrace = true;
+  }
+
+  @Override
+  public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
+    if (level == LockLevel.BLOCK_POOl) {
+      return getReadLock(level, resources[0]);
+    } else {
+      AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
+      AutoCloseDataSetLock volLock = getReadLock(level, resources);
+      volLock.setParentLock(bpLock);
+      if (openLockTrace) {
+        LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " +
+            resources[0]);
+      }
+      return volLock;
+    }
+  }
+
+  @Override
+  public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) {
+    if (level == LockLevel.BLOCK_POOl) {
+      return getWriteLock(level, resources[0]);
+    } else {
+      AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
+      AutoCloseDataSetLock volLock = getWriteLock(level, resources);
+      volLock.setParentLock(bpLock);
+      if (openLockTrace) {
+        LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " +
+            resources[0]);
+      }
+      return volLock;
+    }
+  }
+
+  /**
+   * Return a not null ReadLock.
+   */
+  private AutoCloseDataSetLock getReadLock(LockLevel level, String... resources) {
+    String lockName = generateLockName(level, resources);
+    AutoCloseDataSetLock lock = lockMap.getReadLock(lockName);
+    if (lock == null) {
+      LOG.warn("Ignore this error during dn restart: Not existing readLock "
+          + lockName);
+      lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
+      lock = lockMap.getReadLock(lockName);
+    }
+    lock.lock();
+    if (openLockTrace) {
+      putThreadName(getThreadName());
+    }
+    return lock;
+  }
+
+  /**
+   * Return a not null WriteLock.
+   */
+  private AutoCloseDataSetLock getWriteLock(LockLevel level, String... resources) {
+    String lockName = generateLockName(level, resources);
+    AutoCloseDataSetLock lock = lockMap.getWriteLock(lockName);
+    if (lock == null) {
+      LOG.warn("Ignore this error during dn restart: Not existing writeLock"
+          + lockName);
+      lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
+      lock = lockMap.getWriteLock(lockName);
+    }
+    lock.lock();
+    if (openLockTrace) {
+      putThreadName(getThreadName());
+    }
+    return lock;
+  }
+
+  @Override
+  public void addLock(LockLevel level, String... resources) {
+    String lockName = generateLockName(level, resources);
+    if (level == LockLevel.BLOCK_POOl) {
+      lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
+    } else {
+      lockMap.addLock(resources[0], new ReentrantReadWriteLock(isFair));
+      lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
+    }
+  }
+
+  @Override
+  public void removeLock(LockLevel level, String... resources) {
+    String lockName = generateLockName(level, resources);
+    try (AutoCloseDataSetLock lock = writeLock(level, resources)) {
+      lockMap.removeLock(lockName);
+    }
+  }
+
+  @Override
+  public void hook() {
+    if (openLockTrace) {
+      removeThreadName(getThreadName());
+    }
+  }
+
+  /**
+   * Add thread name when lock a lock.
+   */
+  private synchronized void putThreadName(String thread) {
+    if (threadCountMap.containsKey(thread)) {
+      TrackLog trackLog = threadCountMap.get(thread);
+      trackLog.incrLockCount();
+    }
+    threadCountMap.putIfAbsent(thread, new TrackLog(thread));
+  }
+
+  public void lockLeakCheck() {
+    if (!openLockTrace) {
+      LOG.warn("not open lock leak check func");
+      return;
+    }
+    if (threadCountMap.isEmpty()) {
+      LOG.warn("all lock has release");
+      return;
+    }
+    setLastException(new Exception("lock Leak"));
+    threadCountMap.forEach((name, trackLog) -> trackLog.showLockMessage());
+  }
+
+  /**
+   * Remove thread name when unlock a lock.
+   */
+  private synchronized void removeThreadName(String thread) {
+    if (threadCountMap.containsKey(thread)) {
+      TrackLog trackLog = threadCountMap.get(thread);
+      if (trackLog.shouldClear()) {
+        threadCountMap.remove(thread);
+        return;
+      }
+      trackLog.decrLockCount();
+    }
+  }
+
+  private void setLastException(Exception e) {
+    this.lastException = e;
+  }
+
+  public Exception getLastException() {
+    return lastException;
+  }
+
+  private String getThreadName() {
+    return Thread.currentThread().getName() + Thread.currentThread().getId();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 782f2f36cc198..5cf13f698e9b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -1403,7 +1403,7 @@ public boolean accept(File dir, String name) {
   }
 
   /**
-   * Get the BlockPoolSliceStorage from {@link bpStorageMap}.
+   * Get the BlockPoolSliceStorage from {@link #bpStorageMap}.
    * If the object is not found, create a new object and put it to the map.
    */
   synchronized BlockPoolSliceStorage getBlockPoolSliceStorage(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 1554ace5e801b..770410230162a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 import javax.crypto.SecretKey;
 import org.apache.commons.logging.Log;
@@ -341,7 +341,7 @@ public void run() {
    * the thread dies away.
    */
   private void collectThreadLocalStates() {
-    if (datanode.getPeerMetrics() != null) {
+    if (datanode.getDnConf().peerStatsEnabled && datanode.getPeerMetrics() != null) {
       datanode.getPeerMetrics().collectThreadLocalStates();
     }
   }
@@ -415,6 +415,9 @@ public void requestShortCircuitFds(final ExtendedBlock blk,
                     "Not verifying {}", slotId);
         }
         success = true;
+        // update metrics
+        datanode.metrics.addReadBlockOp(elapsed());
+        datanode.metrics.incrReadsFromClient(true, blk.getNumBytes());
       }
     } finally {
       if ((!success) && (registeredSlotId != null)) {
@@ -948,6 +951,9 @@ public void writeBlock(final ExtendedBlock block,
       IOUtils.closeStream(mirrorIn);
       IOUtils.closeStream(replyOut);
       IOUtils.closeSocket(mirrorSock);
+      if (blockReceiver != null) {
+        blockReceiver.releaseAnyRemainingReservedSpace();
+      }
       IOUtils.closeStream(blockReceiver);
       setCurrentBlockReceiver(null);
     }
@@ -1093,7 +1099,7 @@ public void copyBlock(final ExtendedBlock block,
     if (!dataXceiverServer.balanceThrottler.acquire()) { // not able to start
       String msg = "Not able to copy block " + block.getBlockId() + " " +
           "to " + peer.getRemoteAddressString() + " because threads " +
-          "quota is exceeded.";
+          "quota=" + dataXceiverServer.balanceThrottler.getMaxConcurrentMovers() + " is exceeded.";
       LOG.info(msg);
       sendResponse(ERROR, msg);
       return;
@@ -1167,7 +1173,7 @@ public void replaceBlock(final ExtendedBlock block,
     if (!dataXceiverServer.balanceThrottler.acquire()) { // not able to start
       String msg = "Not able to receive block " + block.getBlockId() +
           " from " + peer.getRemoteAddressString() + " because threads " +
-          "quota is exceeded.";
+          "quota=" + dataXceiverServer.balanceThrottler.getMaxConcurrentMovers() + " is exceeded.";
       LOG.warn(msg);
       sendResponse(ERROR, msg);
       return;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
index 3eabb0102ae03..3a67b76e43467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
@@ -36,7 +36,7 @@
 import org.apache.hadoop.util.Daemon;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 
@@ -68,8 +68,7 @@ class DataXceiverServer implements Runnable {
    * Enforcing the limit is required in order to avoid data-node
    * running out of memory.
    */
-  int maxXceiverCount =
-    DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT;
+  volatile int maxXceiverCount;
 
   /**
    * A manager to make sure that cluster balancing does not take too much
@@ -514,4 +513,15 @@ public boolean updateBalancerMaxConcurrentMovers(final int movers) {
   void setMaxReconfigureWaitTime(int max) {
     this.maxReconfigureWaitTime = max;
   }
+
+  public void setMaxXceiverCount(int xceiverCount) {
+    Preconditions.checkArgument(xceiverCount > 0,
+        "dfs.datanode.max.transfer.threads should be larger than 0");
+    maxXceiverCount = xceiverCount;
+  }
+
+  @VisibleForTesting
+  public int getMaxXceiverCount() {
+    return maxXceiverCount;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 58276e5ccc00e..f97dbbfcd6e8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -41,6 +41,7 @@
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -141,7 +142,8 @@ public String toString() {
           + ", missing metadata files: " + missingMetaFile
           + ", missing block files: " + missingBlockFile
           + ", missing blocks in memory: " + missingMemoryBlocks
-          + ", mismatched blocks: " + mismatchBlocks;
+          + ", mismatched blocks: " + mismatchBlocks
+          + ", duplicated blocks: " + duplicateBlocks;
     }
   }
 
@@ -391,7 +393,7 @@ private void clear() {
   }
 
   /**
-   * Main program loop for DirectoryScanner. Runs {@link reconcile()} and
+   * Main program loop for DirectoryScanner. Runs {@link #reconcile()} and
    * handles any exceptions.
    */
   @Override
@@ -539,21 +541,30 @@ private void scan() {
           m++;
           continue;
         }
-        // Block file and/or metadata file exists on the disk
-        // Block exists in memory
-        if (info.getBlockFile() == null) {
-          // Block metadata file exits and block file is missing
-          addDifference(diffRecord, statsRecord, info);
-        } else if (info.getGenStamp() != memBlock.getGenerationStamp()
-            || info.getBlockLength() != memBlock.getNumBytes()) {
-          // Block metadata file is missing or has wrong generation stamp,
-          // or block file length is different than expected
+
+        // Block and meta must be regular file
+        boolean isRegular = FileUtil.isRegularFile(info.getBlockFile(), false) &&
+                FileUtil.isRegularFile(info.getMetaFile(), false);
+        if (!isRegular) {
           statsRecord.mismatchBlocks++;
           addDifference(diffRecord, statsRecord, info);
-        } else if (memBlock.compareWith(info) != 0) {
-          // volumeMap record and on-disk files do not match.
-          statsRecord.duplicateBlocks++;
-          addDifference(diffRecord, statsRecord, info);
+        } else {
+          // Block file and/or metadata file exists on the disk
+          // Block exists in memory
+          if (info.getBlockFile() == null) {
+            // Block metadata file exits and block file is missing
+            addDifference(diffRecord, statsRecord, info);
+          } else if (info.getGenStamp() != memBlock.getGenerationStamp()
+                  || info.getBlockLength() != memBlock.getNumBytes()) {
+            // Block metadata file is missing or has wrong generation stamp,
+            // or block file length is different than expected
+            statsRecord.mismatchBlocks++;
+            addDifference(diffRecord, statsRecord, info);
+          } else if (memBlock.compareWith(info) != 0) {
+            // volumeMap record and on-disk files do not match.
+            statsRecord.duplicateBlocks++;
+            addDifference(diffRecord, statsRecord, info);
+          }
         }
         d++;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index d46448324ecc2..4126140678759 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -19,13 +19,12 @@
 package org.apache.hadoop.hdfs.server.datanode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi
     .FsVolumeReferences;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.datanode.DiskBalancerWorkStatus
@@ -502,16 +501,13 @@ private void createWorkPlan(NodePlan plan) throws DiskBalancerException {
   private Map<String, String> getStorageIDToVolumeBasePathMap()
       throws DiskBalancerException {
     Map<String, String> storageIDToVolBasePathMap = new HashMap<>();
-    FsDatasetSpi.FsVolumeReferences references;
-    try {
-      try(AutoCloseableLock lock = this.dataset.acquireDatasetReadLock()) {
-        references = this.dataset.getFsVolumeReferences();
-        for (int ndx = 0; ndx < references.size(); ndx++) {
-          FsVolumeSpi vol = references.get(ndx);
-          storageIDToVolBasePathMap.put(vol.getStorageID(),
-              vol.getBaseURI().getPath());
-        }
-        references.close();
+    // Get volumes snapshot so no need to acquire dataset lock.
+    try (FsDatasetSpi.FsVolumeReferences references = dataset.
+        getFsVolumeReferences()) {
+      for (int ndx = 0; ndx < references.size(); ndx++) {
+        FsVolumeSpi vol = references.get(ndx);
+        storageIDToVolBasePathMap.put(vol.getStorageID(),
+            vol.getBaseURI().getPath());
       }
     } catch (IOException ex) {
       LOG.error("Disk Balancer - Internal Error.", ex);
@@ -808,7 +804,7 @@ private boolean isLessThanNeeded(long blockSize,
       long bytesToCopy = item.getBytesToCopy() - item.getBytesCopied();
       bytesToCopy = bytesToCopy +
           ((bytesToCopy * getBlockTolerancePercentage(item)) / 100);
-      return (blockSize <= bytesToCopy) ? true : false;
+      return blockSize <= bytesToCopy;
     }
 
     /**
@@ -833,7 +829,7 @@ private long getBlockTolerancePercentage(DiskBalancerWorkItem item) {
     private boolean isCloseEnough(DiskBalancerWorkItem item) {
       long temp = item.getBytesCopied() +
           ((item.getBytesCopied() * getBlockTolerancePercentage(item)) / 100);
-      return (item.getBytesToCopy() >= temp) ? false : true;
+      return item.getBytesToCopy() < temp;
     }
 
     /**
@@ -989,7 +985,7 @@ private void closePoolIters(List<FsVolumeSpi.BlockIterator> poolIters) {
         try {
           iter.close();
         } catch (IOException ex) {
-          LOG.error("Error closing a block pool iter. ex: {}", ex);
+          LOG.error("Error closing a block pool iter. ex: ", ex);
         }
       }
     }
@@ -1124,7 +1120,7 @@ public void copyBlocks(VolumePair pair, DiskBalancerWorkItem item) {
                 startTime);
             item.setSecondsElapsed(secondsElapsed);
           } catch (IOException ex) {
-            LOG.error("Exception while trying to copy blocks. error: {}", ex);
+            LOG.error("Exception while trying to copy blocks. error: ", ex);
             item.incErrorCount();
           } catch (InterruptedException e) {
             LOG.error("Copy Block Thread interrupted, exiting the copy.");
@@ -1133,7 +1129,7 @@ public void copyBlocks(VolumePair pair, DiskBalancerWorkItem item) {
             this.setExitFlag();
           } catch (RuntimeException ex) {
             // Exiting if any run time exceptions.
-            LOG.error("Got an unexpected Runtime Exception {}", ex);
+            LOG.error("Got an unexpected Runtime Exception ", ex);
             item.incErrorCount();
             this.setExitFlag();
           }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
index 7d6435ea3f2c7..552f5199f1d3a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java
@@ -1071,4 +1071,8 @@ private void onFailure(@Nullable FsVolumeSpi volume, long begin) {
     }
     profilingEventHook.onFailure(volume, begin);
   }
+
+  public ProfilingFileIoEvents getProfilingEventHook() {
+    return profilingEventHook;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
index 99d2fc8e04ea8..24b6bd550e7b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/LocalReplicaInPipeline.java
@@ -174,6 +174,10 @@ public void releaseAllBytesReserved() {
     getVolume().releaseLockedMemory(bytesReserved);
     bytesReserved = 0;
   }
+  @Override
+  public void releaseReplicaInfoBytesReserved() {
+    bytesReserved = 0;
+  }
 
   @Override
   public void setLastChecksumAndDataLen(long dataLength, byte[] checksum) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
index 2da3b1e8f9844..c22401b645f14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -40,8 +41,8 @@ class ProfilingFileIoEvents {
   static final Logger LOG =
       LoggerFactory.getLogger(ProfilingFileIoEvents.class);
 
-  private final boolean isEnabled;
-  private final int sampleRangeMax;
+  private volatile boolean isEnabled;
+  private volatile int sampleRangeMax;
 
   public ProfilingFileIoEvents(@Nullable Configuration conf) {
     if (conf != null) {
@@ -49,15 +50,7 @@ public ProfilingFileIoEvents(@Nullable Configuration conf) {
           DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
           DFSConfigKeys
               .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_DEFAULT);
-      isEnabled = Util.isDiskStatsEnabled(fileIOSamplingPercentage);
-      if (fileIOSamplingPercentage > 100) {
-        LOG.warn(DFSConfigKeys
-            .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY +
-            " value cannot be more than 100. Setting value to 100");
-        fileIOSamplingPercentage = 100;
-      }
-      sampleRangeMax = (int) ((double) fileIOSamplingPercentage / 100 *
-          Integer.MAX_VALUE);
+      setSampleRangeMax(fileIOSamplingPercentage);
     } else {
       isEnabled = false;
       sampleRangeMax = 0;
@@ -80,7 +73,7 @@ public void afterMetadataOp(@Nullable FsVolumeSpi volume,
     if (isEnabled) {
       DataNodeVolumeMetrics metrics = getVolumeMetrics(volume);
       if (metrics != null) {
-        metrics.addMetadastaOperationLatency(Time.monotonicNow() - begin);
+        metrics.addMetadataOperationLatency(Time.monotonicNow() - begin);
       }
     }
   }
@@ -116,6 +109,12 @@ public void afterFileIo(@Nullable FsVolumeSpi volume,
         case WRITE:
           metrics.addWriteIoLatency(latency);
           break;
+        case TRANSFER:
+          metrics.addTransferIoLatency(latency);
+          break;
+        case NATIVE_COPY:
+          metrics.addNativeCopyIoLatency(latency);
+          break;
         default:
         }
       }
@@ -139,4 +138,26 @@ private DataNodeVolumeMetrics getVolumeMetrics(final FsVolumeSpi volume) {
     }
     return null;
   }
+
+  public void setSampleRangeMax(int fileIOSamplingPercentage) {
+    isEnabled = Util.isDiskStatsEnabled(fileIOSamplingPercentage);
+    if (fileIOSamplingPercentage > 100) {
+      LOG.warn(DFSConfigKeys
+          .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY +
+          " value cannot be more than 100. Setting value to 100");
+      fileIOSamplingPercentage = 100;
+    }
+    sampleRangeMax = (int) ((double) fileIOSamplingPercentage / 100 *
+        Integer.MAX_VALUE);
+  }
+
+  @VisibleForTesting
+  public boolean getDiskStatsEnabled() {
+    return isEnabled;
+  }
+
+  @VisibleForTesting
+  public int getSampleRangeMax() {
+    return sampleRangeMax;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
index 174827b5a20eb..65da42d3a205a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
@@ -51,6 +51,11 @@ public interface ReplicaInPipeline extends Replica {
    */
   public void releaseAllBytesReserved();
 
+  /**
+   * Release the reserved space from the ReplicaInfo.
+   */
+  void releaseReplicaInfoBytesReserved();
+
   /**
    * store the checksum for the last chunk along with the data length
    * @param dataLength number of bytes on disk
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
index 960811a2e0769..0728ac4b4e55c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java
@@ -47,7 +47,7 @@
 import org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 5212bc6cbdf92..d8f1e23ec379b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -31,7 +31,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -293,7 +293,7 @@ public void handle(ExtendedBlock block, IOException e) {
             volume, block);
         return;
       }
-      LOG.warn("Reporting bad {} on {}", block, volume);
+      LOG.warn("Reporting bad {} on {}", block, volume, e);
       scanner.datanode.handleBadBlock(block, e, true);
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
index 6ae4f01299ceb..1cd97800e33cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
@@ -22,14 +22,13 @@
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.Beta;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.GwtCompatible;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.apache.hadoop.util.Preconditions;
+
 import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater
     .newUpdater;
 
@@ -290,7 +289,7 @@ public synchronized Throwable fillInStackTrace() {
     final Throwable exception;
 
     Failure(Throwable exception) {
-      this.exception = checkNotNull(exception);
+      this.exception = Preconditions.checkNotNull(exception);
     }
   }
 
@@ -679,8 +678,8 @@ protected final boolean wasInterrupted() {
    */
   @Override
   public void addListener(Runnable listener, Executor executor) {
-    checkNotNull(listener, "Runnable was null.");
-    checkNotNull(executor, "Executor was null.");
+    Preconditions.checkNotNull(listener, "Runnable was null.");
+    Preconditions.checkNotNull(executor, "Executor was null.");
     Listener oldHead = listeners;
     if (oldHead != Listener.TOMBSTONE) {
       Listener newNode = new Listener(listener, executor);
@@ -737,7 +736,7 @@ protected boolean set(@Nullable V value) {
    * @return true if the attempt was accepted, completing the {@code Future}
    */
   protected boolean setException(Throwable throwable) {
-    Object valueToSet = new Failure(checkNotNull(throwable));
+    Object valueToSet = new Failure(Preconditions.checkNotNull(throwable));
     if (ATOMIC_HELPER.casValue(this, null, valueToSet)) {
       complete(this);
       return true;
@@ -771,7 +770,7 @@ protected boolean setException(Throwable throwable) {
    */
   @Beta
   protected boolean setFuture(ListenableFuture<? extends V> future) {
-    checkNotNull(future);
+    Preconditions.checkNotNull(future);
     Object localValue = value;
     if (localValue == null) {
       if (future.isDone()) {
@@ -1097,7 +1096,7 @@ public sun.misc.Unsafe run() throws Exception {
     }
 
     public static void throwIfUnchecked(Throwable throwable) {
-      checkNotNull(throwable);
+      Preconditions.checkNotNull(throwable);
       if (throwable instanceof RuntimeException) {
         throw (RuntimeException) throwable;
       }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
index a58c536117d46..da6ae95918d9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.hdfs.server.datanode.checker;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
@@ -77,7 +77,6 @@ public class DatasetVolumeChecker {
 
   private final AtomicLong numVolumeChecks = new AtomicLong(0);
   private final AtomicLong numSyncDatasetChecks = new AtomicLong(0);
-  private final AtomicLong numAsyncDatasetChecks = new AtomicLong(0);
   private final AtomicLong numSkippedChecks = new AtomicLong(0);
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java
index f969c7ade288b..af62835c4a69a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java
@@ -44,14 +44,14 @@
 
 /**
  * An implementation of {@link AsyncChecker} that skips checking recently
- * checked objects. It will enforce at least {@link minMsBetweenChecks}
+ * checked objects. It will enforce at least {@link #minMsBetweenChecks}
  * milliseconds between two successive checks of any one object.
  *
  * It is assumed that the total number of Checkable objects in the system
  * is small, (not more than a few dozen) since the checker uses O(Checkables)
  * storage and also potentially O(Checkables) threads.
  *
- * {@link minMsBetweenChecks} should be configured reasonably
+ * {@link #minMsBetweenChecks} should be configured reasonably
  * by the caller to avoid spinning up too many threads frequently.
  */
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
index d014e499f912e..6f80780e140b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
@@ -20,7 +20,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.checker;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
 import org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture;
 import org.slf4j.Logger;
@@ -31,6 +31,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.ExecutionException;
 
 /**
  * Implementation of {@code Futures#withTimeout}.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 9fb5d9e708ca2..74c4cf1bd5f34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.erasurecode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -127,7 +127,7 @@ public void processErasureCodingTasks(
             reconInfo.getExtendedBlock(), reconInfo.getErasureCodingPolicy(),
             reconInfo.getLiveBlockIndices(), reconInfo.getSourceDnInfos(),
             reconInfo.getTargetDnInfos(), reconInfo.getTargetStorageTypes(),
-            reconInfo.getTargetStorageIDs());
+            reconInfo.getTargetStorageIDs(), reconInfo.getExcludeReconstructedIndices());
         // It may throw IllegalArgumentException from task#stripedReader
         // constructor.
         final StripedBlockReconstructor task =
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index 3ead793542c7b..ecd6351b46f64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -95,6 +95,10 @@ void reconstruct() throws IOException {
           (int) Math.min(getStripedReader().getBufferSize(), remaining);
 
       long start = Time.monotonicNow();
+      long bytesToRead = (long) toReconstructLen * getStripedReader().getMinRequiredSources();
+      if (getDatanode().getEcReconstuctReadThrottler() != null) {
+        getDatanode().getEcReconstuctReadThrottler().throttle(bytesToRead);
+      }
       // step1: read from minimum source DNs required for reconstruction.
       // The returned success list is the source DNs we do real read from
       getStripedReader().readMinimumSources(toReconstructLen);
@@ -105,6 +109,10 @@ void reconstruct() throws IOException {
       long decodeEnd = Time.monotonicNow();
 
       // step3: transfer data
+      long bytesToWrite = (long) toReconstructLen * stripedWriter.getTargets();
+      if (getDatanode().getEcReconstuctWriteThrottler() != null) {
+        getDatanode().getEcReconstuctWriteThrottler().throttle(bytesToWrite);
+      }
       if (stripedWriter.transferData2Targets() == 0) {
         String error = "Transfer failed for all targets.";
         throw new IOException(error);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
index f8998849baf84..20d5c6f44fb41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.datanode.erasurecode;
 
 import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -508,4 +508,9 @@ CachingStrategy getCachingStrategy() {
   int getXmits() {
     return xmits;
   }
+
+  public int getMinRequiredSources() {
+    return minRequiredSources;
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
index c166f5ec03125..caf8dfa950446 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java
@@ -41,26 +41,28 @@ public class StripedReconstructionInfo {
   private final DatanodeInfo[] targets;
   private final StorageType[] targetStorageTypes;
   private final String[] targetStorageIds;
+  private final byte[] excludeReconstructedIndices;
 
   public StripedReconstructionInfo(ExtendedBlock blockGroup,
       ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
       byte[] targetIndices) {
     this(blockGroup, ecPolicy, liveIndices, sources, targetIndices, null,
-        null, null);
+        null, null, new byte[0]);
   }
 
   StripedReconstructionInfo(ExtendedBlock blockGroup,
       ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
       DatanodeInfo[] targets, StorageType[] targetStorageTypes,
-      String[] targetStorageIds) {
+      String[] targetStorageIds, byte[] excludeReconstructedIndices) {
     this(blockGroup, ecPolicy, liveIndices, sources, null, targets,
-        targetStorageTypes, targetStorageIds);
+        targetStorageTypes, targetStorageIds, excludeReconstructedIndices);
   }
 
   private StripedReconstructionInfo(ExtendedBlock blockGroup,
       ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources,
       byte[] targetIndices, DatanodeInfo[] targets,
-      StorageType[] targetStorageTypes, String[] targetStorageIds) {
+      StorageType[] targetStorageTypes, String[] targetStorageIds,
+      byte[] excludeReconstructedIndices) {
 
     this.blockGroup = blockGroup;
     this.ecPolicy = ecPolicy;
@@ -70,6 +72,7 @@ private StripedReconstructionInfo(ExtendedBlock blockGroup,
     this.targets = targets;
     this.targetStorageTypes = targetStorageTypes;
     this.targetStorageIds = targetStorageIds;
+    this.excludeReconstructedIndices = excludeReconstructedIndices;
   }
 
   ExtendedBlock getBlockGroup() {
@@ -104,5 +107,9 @@ String[] getTargetStorageIds() {
     return targetStorageIds;
   }
 
+  byte[] getExcludeReconstructedIndices() {
+    return excludeReconstructedIndices;
+  }
+
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index 2caa872d99b54..7acb679200118 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -120,6 +120,7 @@ abstract class StripedReconstructor {
   private final CachingStrategy cachingStrategy;
   private long maxTargetLength = 0L;
   private final BitSet liveBitSet;
+  private final BitSet excludeBitSet;
 
   // metrics
   private AtomicLong bytesRead = new AtomicLong(0);
@@ -137,6 +138,12 @@ abstract class StripedReconstructor {
     for (int i = 0; i < stripedReconInfo.getLiveIndices().length; i++) {
       liveBitSet.set(stripedReconInfo.getLiveIndices()[i]);
     }
+    excludeBitSet = new BitSet(
+            ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
+    for (int i = 0; i < stripedReconInfo.getExcludeReconstructedIndices().length; i++) {
+      excludeBitSet.set(stripedReconInfo.getExcludeReconstructedIndices()[i]);
+    }
+
     blockGroup = stripedReconInfo.getBlockGroup();
     stripedReader = new StripedReader(this, datanode, conf, stripedReconInfo);
     cachingStrategy = CachingStrategy.newDefaultStrategy();
@@ -261,6 +268,10 @@ BitSet getLiveBitSet() {
     return liveBitSet;
   }
 
+  BitSet getExcludeBitSet(){
+    return excludeBitSet;
+  }
+
   long getMaxTargetLength() {
     return maxTargetLength;
   }
@@ -277,6 +288,10 @@ RawErasureDecoder getDecoder() {
     return decoder;
   }
 
+  int getNumLiveBlocks(){
+    return liveBitSet.cardinality();
+  }
+
   void cleanup() {
     if (decoder != null) {
       decoder.release();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
index 683b63fa149af..00be1279c8179 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.erasurecode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
@@ -82,8 +82,10 @@ class StripedWriter {
     assert targetStorageIds != null;
 
     writers = new StripedBlockWriter[targets.length];
-
     targetIndices = new short[targets.length];
+    Preconditions.checkArgument(
+            targetIndices.length <= dataBlkNum + parityBlkNum - reconstructor.getNumLiveBlocks(),
+            "Reconstruction work gets too much targets.");
     Preconditions.checkArgument(targetIndices.length <= parityBlkNum,
         "Too much missed striped blocks.");
     initTargetIndices();
@@ -123,13 +125,14 @@ void init() throws IOException {
 
   private void initTargetIndices() {
     BitSet bitset = reconstructor.getLiveBitSet();
+    BitSet excludebitset=reconstructor.getExcludeBitSet();
 
     int m = 0;
     hasValidTargets = false;
     for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
       if (!bitset.get(i)) {
         if (reconstructor.getBlockLen(i) > 0) {
-          if (m < targets.length) {
+          if (m < targets.length && !excludebitset.get(i)) {
             targetIndices[m++] = (short)i;
             hasValidTargets = true;
           }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
index 72ed47c67fa93..5d12fa72bb165 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/AvailableSpaceVolumeChoosingPolicy.java
@@ -63,7 +63,6 @@ public class AvailableSpaceVolumeChoosingPolicy<V extends FsVolumeSpi>
 
   public AvailableSpaceVolumeChoosingPolicy() {
     this(new Random());
-    initLocks();
   }
 
   private void initLocks() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/DataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/DataNodeVolumeMetrics.java
index 87509e5b92167..0ce57efd59549 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/DataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/DataNodeVolumeMetrics.java
@@ -71,6 +71,14 @@ public class DataNodeVolumeMetrics {
   private MutableRate writeIoRate;
   private MutableQuantiles[] writeIoLatencyQuantiles;
 
+  @Metric("file io transfer rate")
+  private MutableRate transferIoRate;
+  private MutableQuantiles[] transferIoLatencyQuantiles;
+
+  @Metric("file io nativeCopy rate")
+  private MutableRate nativeCopyIoRate;
+  private MutableQuantiles[] nativeCopyIoLatencyQuantiles;
+
   @Metric("number of file io errors")
   private MutableCounterLong totalFileIoErrors;
   @Metric("file io error rate")
@@ -162,6 +170,40 @@ public double getWriteIoStdDev() {
     return writeIoRate.lastStat().stddev();
   }
 
+  // Based on transferIoRate
+  public long getTransferIoSampleCount() {
+    return transferIoRate.lastStat().numSamples();
+  }
+
+  public double getTransferIoMean() {
+    return transferIoRate.lastStat().mean();
+  }
+
+  public double getTransferIoStdDev() {
+    return transferIoRate.lastStat().stddev();
+  }
+
+  public MutableQuantiles[] getTransferIoQuantiles() {
+    return transferIoLatencyQuantiles;
+  }
+
+  // Based on nativeCopyIoRate
+  public long getNativeCopyIoSampleCount() {
+    return nativeCopyIoRate.lastStat().numSamples();
+  }
+
+  public double getNativeCopyIoMean() {
+    return nativeCopyIoRate.lastStat().mean();
+  }
+
+  public double getNativeCopyIoStdDev() {
+    return nativeCopyIoRate.lastStat().stddev();
+  }
+
+  public MutableQuantiles[] getNativeCopyIoQuantiles() {
+    return nativeCopyIoLatencyQuantiles;
+  }
+
   public long getTotalFileIoErrors() {
     return totalFileIoErrors.value();
   }
@@ -193,11 +235,13 @@ public DataNodeVolumeMetrics(final MetricsSystem metricsSystem,
     syncIoLatencyQuantiles = new MutableQuantiles[len];
     readIoLatencyQuantiles = new MutableQuantiles[len];
     writeIoLatencyQuantiles = new MutableQuantiles[len];
+    transferIoLatencyQuantiles = new MutableQuantiles[len];
+    nativeCopyIoLatencyQuantiles = new MutableQuantiles[len];
     for (int i = 0; i < len; i++) {
       int interval = intervals[i];
       metadataOperationLatencyQuantiles[i] = registry.newQuantiles(
           "metadataOperationLatency" + interval + "s",
-          "Meatadata Operation Latency in ms", "ops", "latency", interval);
+          "Metadata Operation Latency in ms", "ops", "latency", interval);
       dataFileIoLatencyQuantiles[i] = registry.newQuantiles(
           "dataFileIoLatency" + interval + "s",
           "Data File Io Latency in ms", "ops", "latency", interval);
@@ -213,6 +257,12 @@ public DataNodeVolumeMetrics(final MetricsSystem metricsSystem,
       writeIoLatencyQuantiles[i] = registry.newQuantiles(
           "writeIoLatency" + interval + "s",
           "Data write Io Latency in ms", "ops", "latency", interval);
+      transferIoLatencyQuantiles[i] = registry.newQuantiles(
+          "transferIoLatency" + interval + "s",
+          "Data transfer Io Latency in ms", "ops", "latency", interval);
+      nativeCopyIoLatencyQuantiles[i] = registry.newQuantiles(
+          "nativeCopyIoLatency" + interval + "s",
+          "Data nativeCopy Io Latency in ms", "ops", "latency", interval);
     }
   }
 
@@ -238,7 +288,7 @@ public void unRegister() {
     ms.unregisterSource(name);
   }
 
-  public void addMetadastaOperationLatency(final long latency) {
+  public void addMetadataOperationLatency(final long latency) {
     totalMetadataOperations.incr();
     metadataOperationRate.add(latency);
     for (MutableQuantiles q : metadataOperationLatencyQuantiles) {
@@ -282,8 +332,22 @@ public void addWriteIoLatency(final long latency) {
     }
   }
 
+  public void addTransferIoLatency(final long latency) {
+    transferIoRate.add(latency);
+    for (MutableQuantiles q: transferIoLatencyQuantiles) {
+      q.add(latency);
+    }
+  }
+
+  public void addNativeCopyIoLatency(final long latency) {
+    nativeCopyIoRate.add(latency);
+    for (MutableQuantiles q: nativeCopyIoLatencyQuantiles) {
+      q.add(latency);
+    }
+  }
+
   public void addFileIoError(final long latency) {
     totalFileIoErrors.incr();
-    metadataOperationRate.add(latency);
+    fileIoErrorRate.add(latency);
   }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index f162ea1b3ae15..8d1d10bccd2fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -35,8 +35,10 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.MountVolumeMap;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -241,7 +243,7 @@ StorageReport[] getStorageReports(String bpid)
    * Gets a list of references to the finalized blocks for the given block pool.
    * <p>
    * Callers of this function should call
-   * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being
+   * {@link FsDatasetSpi#acquireDatasetLockManager} to avoid blocks' status being
    * changed during list iteration.
    * </p>
    * @return a list of references to the finalized blocks for the given block
@@ -657,21 +659,12 @@ ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block,
   ReplicaInfo moveBlockAcrossVolumes(final ExtendedBlock block,
       FsVolumeSpi destination) throws IOException;
 
-  /**
-   * Acquire the lock of the data set. This prevents other threads from
-   * modifying the volume map structure inside the datanode, but other changes
-   * are still possible. For example modifying the genStamp of a block instance.
-   */
-  AutoCloseableLock acquireDatasetLock();
-
   /***
-   * Acquire the read lock of the data set. This prevents other threads from
-   * modifying the volume map structure inside the datanode, but other changes
-   * are still possible. For example modifying the genStamp of a block instance.
+   * Acquire lock Manager for the data set. This prevents other threads from
+   * modifying the volume map structure inside the datanode.
    * @return The AutoClosable read lock instance.
    */
-  AutoCloseableLock acquireDatasetReadLock();
-
+  DataNodeLockManager<? extends AutoCloseDataSetLock> acquireDatasetLockManager();
 
   /**
    * Deep copy the replica info belonging to given block pool.
@@ -687,4 +680,9 @@ ReplicaInfo moveBlockAcrossVolumes(final ExtendedBlock block,
    * @throws IOException
    */
   MountVolumeMap getMountVolumeMap() throws IOException;
+
+  /**
+   * Get the volume list.
+   */
+  List<FsVolumeImpl> getVolumeList();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index f2d4e08a45bdb..4611df765f990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -43,10 +43,10 @@
 import java.util.concurrent.ForkJoinTask;
 import java.util.concurrent.RecursiveAction;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.hdfs.server.datanode.FSCachingGetSpaceUsed;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -78,6 +78,10 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
+
 /**
  * A block pool slice represents a portion of a block pool stored on a volume.
  * Taken together, all BlockPoolSlices sharing a block pool ID across a
@@ -85,7 +89,7 @@
  *
  * This class is synchronized by {@link FsVolumeImpl}.
  */
-class BlockPoolSlice {
+public class BlockPoolSlice {
   static final Logger LOG = LoggerFactory.getLogger(BlockPoolSlice.class);
 
   private final String bpid;
@@ -116,6 +120,8 @@ class BlockPoolSlice {
   private final Timer timer;
   private final int maxDataLength;
   private final FileIoProvider fileIoProvider;
+  private final Configuration config;
+  private final File bpDir;
 
   private static ForkJoinPool addReplicaThreadPool = null;
   private static final int VOLUMES_REPLICA_ADD_THREADPOOL_SIZE = Runtime
@@ -129,7 +135,7 @@ public int compare(File f1, File f2) {
   };
 
   // TODO:FEDERATION scalability issue - a thread per DU is needed
-  private final GetSpaceUsed dfsUsage;
+  private volatile GetSpaceUsed dfsUsage;
 
   /**
    * Create a blook pool slice
@@ -142,6 +148,8 @@ public int compare(File f1, File f2) {
    */
   BlockPoolSlice(String bpid, FsVolumeImpl volume, File bpDir,
       Configuration conf, Timer timer) throws IOException {
+    this.config = conf;
+    this.bpDir = bpDir;
     this.bpid = bpid;
     this.volume = volume;
     this.fileIoProvider = volume.getFileIoProvider();
@@ -233,6 +241,40 @@ public void run() {
         SHUTDOWN_HOOK_PRIORITY);
   }
 
+  public void updateDfsUsageConfig(Long interval, Long jitter, Class<? extends GetSpaceUsed> klass)
+          throws IOException {
+    // Close the old dfsUsage if it is CachingGetSpaceUsed.
+    if (dfsUsage instanceof CachingGetSpaceUsed) {
+      ((CachingGetSpaceUsed) dfsUsage).close();
+    }
+    if (interval != null) {
+      Preconditions.checkArgument(interval > 0,
+          FS_DU_INTERVAL_KEY + " should be larger than 0");
+      config.setLong(FS_DU_INTERVAL_KEY, interval);
+    }
+    if (jitter != null) {
+      Preconditions.checkArgument(jitter >= 0,
+          FS_GETSPACEUSED_JITTER_KEY + " should be larger than or equal to 0");
+      config.setLong(FS_GETSPACEUSED_JITTER_KEY, jitter);
+    }
+
+    if (klass != null) {
+      config.setClass(FS_GETSPACEUSED_CLASSNAME, klass, CachingGetSpaceUsed.class);
+    }
+    // Start new dfsUsage.
+    this.dfsUsage = new FSCachingGetSpaceUsed.Builder().setBpid(bpid)
+        .setVolume(volume)
+        .setPath(bpDir)
+        .setConf(config)
+        .setInitialUsed(loadDfsUsed())
+        .build();
+  }
+
+  @VisibleForTesting
+  public GetSpaceUsed getDfsUsage() {
+    return dfsUsage;
+  }
+
   private synchronized static void initializeAddReplicaPool(Configuration conf,
       FsDatasetImpl dataset) {
     if (addReplicaThreadPool == null) {
@@ -298,9 +340,13 @@ long loadDfsUsed() {
     long mtime;
     Scanner sc;
 
+    File duCacheFile = new File(currentDir, DU_CACHE_FILE);
     try {
-      sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
+      sc = new Scanner(duCacheFile, "UTF-8");
     } catch (FileNotFoundException fnfe) {
+      FsDatasetImpl.LOG.warn("{} file missing in {}, will proceed with Du " +
+              "for space computation calculation, ",
+              DU_CACHE_FILE, currentDir);
       return -1;
     }
 
@@ -309,21 +355,31 @@ long loadDfsUsed() {
       if (sc.hasNextLong()) {
         cachedDfsUsed = sc.nextLong();
       } else {
+        FsDatasetImpl.LOG.warn("cachedDfsUsed not found in file:{}, will " +
+                "proceed with Du for space computation calculation, ",
+                duCacheFile);
         return -1;
       }
       // Get the recorded mtime from the file.
       if (sc.hasNextLong()) {
         mtime = sc.nextLong();
       } else {
+        FsDatasetImpl.LOG.warn("mtime not found in file:{}, will proceed" +
+                " with Du for space computation calculation, ", duCacheFile);
         return -1;
       }
 
+      long elapsedTime = timer.now() - mtime;
       // Return the cached value if mtime is okay.
-      if (mtime > 0 && (timer.now() - mtime < cachedDfsUsedCheckTime)) {
+      if (mtime > 0 && (elapsedTime < cachedDfsUsedCheckTime)) {
         FsDatasetImpl.LOG.info("Cached dfsUsed found for " + currentDir + ": " +
             cachedDfsUsed);
         return cachedDfsUsed;
       }
+      FsDatasetImpl.LOG.warn("elapsed time:{} is greater than threshold:{}," +
+                      " mtime:{} in file:{}, will proceed with Du for space" +
+                      " computation calculation",
+              elapsedTime, cachedDfsUsedCheckTime, mtime, duCacheFile);
       return -1;
     } finally {
       sc.close();
@@ -900,7 +956,7 @@ void shutdown(BlockListAsLongs blocksListToPersist) {
 
   private boolean readReplicasFromCache(ReplicaMap volumeMap,
       final RamDiskReplicaTracker lazyWriteReplicaMap) {
-    ReplicaMap tmpReplicaMap = new ReplicaMap(new ReentrantReadWriteLock());
+    ReplicaMap tmpReplicaMap = new ReplicaMap();
     File replicaFile = new File(replicaCacheDir, REPLICA_CACHE_FILE);
     // Check whether the file exists or not.
     if (!replicaFile.exists()) {
@@ -1082,9 +1138,11 @@ public ForkJoinPool getAddReplicaThreadPool() {
   }
 
   @VisibleForTesting
-  public static void reInitializeAddReplicaThreadPool() {
-    addReplicaThreadPool.shutdown();
-    addReplicaThreadPool = null;
+  public synchronized static void reInitializeAddReplicaThreadPool() {
+    if (addReplicaThreadPool != null) {
+      addReplicaThreadPool.shutdown();
+      addReplicaThreadPool = null;
+    }
   }
 
   @VisibleForTesting
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 7d5f33b8b8cf8..e5eef8e2e23f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -30,6 +30,9 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -65,13 +68,12 @@ class FsDatasetAsyncDiskService {
   // ThreadPool core pool size
   private static final int CORE_THREADS_PER_VOLUME = 1;
   // ThreadPool maximum pool size
-  private static final int MAXIMUM_THREADS_PER_VOLUME = 4;
+  private final int maxNumThreadsPerVolume;
   // ThreadPool keep-alive time for threads over core pool size
   private static final long THREADS_KEEP_ALIVE_SECONDS = 60; 
   
   private final DataNode datanode;
   private final FsDatasetImpl fsdatasetImpl;
-  private final ThreadGroup threadGroup;
   private Map<String, ThreadPoolExecutor> executors
       = new HashMap<String, ThreadPoolExecutor>();
   private Map<String, Set<Long>> deletedBlockIds 
@@ -89,7 +91,12 @@ class FsDatasetAsyncDiskService {
   FsDatasetAsyncDiskService(DataNode datanode, FsDatasetImpl fsdatasetImpl) {
     this.datanode = datanode;
     this.fsdatasetImpl = fsdatasetImpl;
-    this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+    maxNumThreadsPerVolume = datanode.getConf().getInt(
+      DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY,
+          DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT);
+    Preconditions.checkArgument(maxNumThreadsPerVolume > 0,
+        DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY +
+          " must be a positive integer.");
   }
 
   private void addExecutorForVolume(final FsVolumeImpl volume) {
@@ -102,7 +109,7 @@ public Thread newThread(Runnable r) {
         synchronized (this) {
           thisIndex = counter++;
         }
-        Thread t = new Thread(threadGroup, r);
+        Thread t = new Thread(r);
         t.setName("Async disk worker #" + thisIndex +
             " for volume " + volume);
         return t;
@@ -110,7 +117,7 @@ public Thread newThread(Runnable r) {
     };
 
     ThreadPoolExecutor executor = new ThreadPoolExecutor(
-        CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+        CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume,
         THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
         new LinkedBlockingQueue<Runnable>(), threadFactory);
 
@@ -210,16 +217,20 @@ synchronized void shutdown() {
     }
   }
 
-  public void submitSyncFileRangeRequest(FsVolumeImpl volume,
-      final ReplicaOutputStreams streams, final long offset, final long nbytes,
-      final int flags) {
-    execute(volume, new Runnable() {
-      @Override
-      public void run() {
+  public void submitSyncFileRangeRequest(FsVolumeImpl volume, final ReplicaOutputStreams streams,
+      final long offset, final long nbytes, final int flags) {
+    execute(volume, () -> {
+      try {
+        streams.syncFileRangeIfPossible(offset, nbytes, flags);
+      } catch (NativeIOException e) {
         try {
-          streams.syncFileRangeIfPossible(offset, nbytes, flags);
-        } catch (NativeIOException e) {
-          LOG.warn("sync_file_range error", e);
+          LOG.warn("sync_file_range error. Volume: {}, Capacity: {}, Available space: {}, "
+                  + "File range offset: {}, length: {}, flags: {}", volume, volume.getCapacity(),
+              volume.getAvailable(), offset, nbytes, flags, e);
+        } catch (IOException ioe) {
+          LOG.warn("sync_file_range error. Volume: {}, Capacity: {}, "
+                  + "File range offset: {}, length: {}, flags: {}", volume, volume.getCapacity(),
+              offset, nbytes, flags, e);
         }
       }
     });
@@ -324,6 +335,13 @@ private boolean moveFiles() {
     @Override
     public void run() {
       try {
+        // For testing, simulate the case asynchronously deletion of the
+        // replica task stacked pending.
+        DataNodeFaultInjector.get().delayDeleteReplica();
+        if (!fsdatasetImpl.removeReplicaFromMem(block, volume)) {
+          return;
+        }
+
         final long blockLength = replicaToDelete.getBlockDataLength();
         final long metaLength = replicaToDelete.getMetadataLength();
         boolean result;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
index 417e26647466c..5ca755954fced 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
@@ -24,7 +24,7 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_CACHE_REVOCATION_POLLING_MS_DEFAULT;
 
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import java.io.FileInputStream;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 081d755eb603d..2f2bd99a2758e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -32,7 +32,6 @@
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -41,14 +40,12 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import javax.management.NotCompliantMBeanException;
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.HardLink;
 import org.apache.hadoop.classification.VisibleForTesting;
 
@@ -63,6 +60,10 @@
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
+import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
@@ -118,14 +119,12 @@
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
-import org.apache.hadoop.util.InstrumentedReadWriteLock;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Timer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -187,7 +186,8 @@ public StorageReport[] getStorageReports(String bpid)
 
   @Override
   public FsVolumeImpl getVolume(final ExtendedBlock b) {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        b.getBlockPoolId())) {
       final ReplicaInfo r =
           volumeMap.get(b.getBlockPoolId(), b.getLocalBlock());
       return r != null ? (FsVolumeImpl) r.getVolume() : null;
@@ -197,7 +197,8 @@ public FsVolumeImpl getVolume(final ExtendedBlock b) {
   @Override // FsDatasetSpi
   public Block getStoredBlock(String bpid, long blkid)
       throws IOException {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        bpid)) {
       ReplicaInfo r = volumeMap.get(bpid, blkid);
       if (r == null) {
         return null;
@@ -209,12 +210,14 @@ public Block getStoredBlock(String bpid, long blkid)
   @Override
   public Set<? extends Replica> deepCopyReplica(String bpid)
       throws IOException {
-    Set<? extends Replica> replicas = null;
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
-      replicas = new HashSet<>(volumeMap.replicas(bpid) == null ? Collections.
-          EMPTY_SET : volumeMap.replicas(bpid));
-    }
-    return Collections.unmodifiableSet(replicas);
+    Set<ReplicaInfo> replicas = new HashSet<>();
+    volumeMap.replicas(bpid, (iterator) -> {
+      while (iterator.hasNext()) {
+        ReplicaInfo b = iterator.next();
+        replicas.add(b);
+      }
+    });
+    return replicas;
   }
 
   /**
@@ -274,13 +277,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
   private boolean blockPinningEnabled;
   private final int maxDataLength;
 
-  @VisibleForTesting
-  final AutoCloseableLock datasetWriteLock;
-  @VisibleForTesting
-  final AutoCloseableLock datasetReadLock;
-  @VisibleForTesting
-  final InstrumentedReadWriteLock datasetRWLock;
-  private final Condition datasetWriteLockCondition;
+  private final DataSetLockManager lockManager;
   private static String blockPoolId = "";
 
   // Make limited notify times from DirectoryScanner to NameNode.
@@ -299,33 +296,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
     this.dataStorage = storage;
     this.conf = conf;
     this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
-    this.datasetRWLock = new InstrumentedReadWriteLock(
-        conf.getBoolean(DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_KEY,
-            DFSConfigKeys.DFS_DATANODE_LOCK_FAIR_DEFAULT),
-        "FsDatasetRWLock", LOG, conf.getTimeDuration(
-        DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY,
-        DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS),
-        conf.getTimeDuration(
-            DFSConfigKeys.DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_KEY,
-            DFSConfigKeys.DFS_DATANODE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT,
-            TimeUnit.MILLISECONDS));
-    this.datasetWriteLock = new AutoCloseableLock(datasetRWLock.writeLock());
-    boolean enableRL = conf.getBoolean(
-        DFSConfigKeys.DFS_DATANODE_LOCK_READ_WRITE_ENABLED_KEY,
-        DFSConfigKeys.DFS_DATANODE_LOCK_READ_WRITE_ENABLED_DEFAULT);
-    // The read lock can be disabled by the above config key. If it is disabled
-    // then we simply make the both the read and write lock variables hold
-    // the write lock. All accesses to the lock are via these variables, so that
-    // effectively disables the read lock.
-    if (enableRL) {
-      LOG.info("The datanode lock is a read write lock");
-      this.datasetReadLock = new AutoCloseableLock(datasetRWLock.readLock());
-    } else {
-      LOG.info("The datanode lock is an exclusive write lock");
-      this.datasetReadLock = this.datasetWriteLock;
-    }
-    this.datasetWriteLockCondition = datasetWriteLock.newCondition();
+    this.lockManager = datanode.getDataSetLockManager();
 
     // The number of volumes required for operation is the total number
     // of volumes minus the number of failed volumes we can tolerate.
@@ -364,7 +335,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
     }
 
     storageMap = new ConcurrentHashMap<String, DatanodeStorage>();
-    volumeMap = new ReplicaMap(datasetReadLock, datasetWriteLock);
+    volumeMap = new ReplicaMap(lockManager);
     ramDiskReplicaTracker = RamDiskReplicaTracker.getInstance(conf, this);
 
     @SuppressWarnings("unchecked")
@@ -374,7 +345,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
             RoundRobinVolumeChoosingPolicy.class,
             VolumeChoosingPolicy.class), conf);
     volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(),
-        blockChooserImpl, conf);
+        blockChooserImpl, conf, datanode.getDiskMetrics());
     asyncDiskService = new FsDatasetAsyncDiskService(datanode, this);
     asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode, conf);
     deletingBlock = new HashMap<String, Set<Long>>();
@@ -420,16 +391,6 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
     lastDirScannerNotifyTime = System.currentTimeMillis();
   }
 
-  @Override
-  public AutoCloseableLock acquireDatasetLock() {
-    return datasetWriteLock.acquire();
-  }
-
-  @Override
-  public AutoCloseableLock acquireDatasetReadLock() {
-    return datasetReadLock.acquire();
-  }
-
   /**
    * Gets initial volume failure information for all volumes that failed
    * immediately at startup.  The method works by determining the set difference
@@ -440,11 +401,7 @@ public AutoCloseableLock acquireDatasetReadLock() {
    */
   private static List<VolumeFailureInfo> getInitialVolumeFailureInfos(
       Collection<StorageLocation> dataLocations, DataStorage storage) {
-    Set<StorageLocation> failedLocationSet = Sets.newHashSetWithExpectedSize(
-        dataLocations.size());
-    for (StorageLocation sl: dataLocations) {
-      failedLocationSet.add(sl);
-    }
+    Set<StorageLocation> failedLocationSet = new HashSet<>(dataLocations);
     for (Iterator<Storage.StorageDirectory> it = storage.dirIterator();
          it.hasNext(); ) {
       Storage.StorageDirectory sd = it.next();
@@ -464,42 +421,43 @@ private static List<VolumeFailureInfo> getInitialVolumeFailureInfos(
    * Activate a volume to serve requests.
    * @throws IOException if the storage UUID already exists.
    */
-  private void activateVolume(
+  private synchronized void activateVolume(
       ReplicaMap replicaMap,
       Storage.StorageDirectory sd, StorageType storageType,
       FsVolumeReference ref) throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-      DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid());
-      if (dnStorage != null) {
-        final String errorMsg = String.format(
-            "Found duplicated storage UUID: %s in %s.",
-            sd.getStorageUuid(), sd.getVersionFile());
-        LOG.error(errorMsg);
-        throw new IOException(errorMsg);
-      }
-      // Check if there is same storage type on the mount.
-      // Only useful when same disk tiering is turned on.
-      FsVolumeImpl volumeImpl = (FsVolumeImpl) ref.getVolume();
-      FsVolumeReference checkRef = volumes
-          .getMountVolumeMap()
-          .getVolumeRefByMountAndStorageType(
-              volumeImpl.getMount(), volumeImpl.getStorageType());
-      if (checkRef != null) {
-        final String errorMsg = String.format(
-            "Storage type %s already exists on same mount: %s.",
-            volumeImpl.getStorageType(), volumeImpl.getMount());
-        checkRef.close();
-        LOG.error(errorMsg);
-        throw new IOException(errorMsg);
-      }
-      volumeMap.mergeAll(replicaMap);
-      storageMap.put(sd.getStorageUuid(),
-          new DatanodeStorage(sd.getStorageUuid(),
-              DatanodeStorage.State.NORMAL,
-              storageType));
-      asyncDiskService.addVolume(volumeImpl);
-      volumes.addVolume(ref);
-    }
+    for (String bp : volumeMap.getBlockPoolList()) {
+      lockManager.addLock(LockLevel.VOLUME, bp, ref.getVolume().getStorageID());
+    }
+    DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid());
+    if (dnStorage != null) {
+      final String errorMsg = String.format(
+          "Found duplicated storage UUID: %s in %s.",
+          sd.getStorageUuid(), sd.getVersionFile());
+      LOG.error(errorMsg);
+      throw new IOException(errorMsg);
+    }
+    // Check if there is same storage type on the mount.
+    // Only useful when same disk tiering is turned on.
+    FsVolumeImpl volumeImpl = (FsVolumeImpl) ref.getVolume();
+    FsVolumeReference checkRef = volumes
+        .getMountVolumeMap()
+        .getVolumeRefByMountAndStorageType(
+            volumeImpl.getMount(), volumeImpl.getStorageType());
+    if (checkRef != null) {
+      final String errorMsg = String.format(
+          "Storage type %s already exists on same mount: %s.",
+          volumeImpl.getStorageType(), volumeImpl.getMount());
+      checkRef.close();
+      LOG.error(errorMsg);
+      throw new IOException(errorMsg);
+    }
+    volumeMap.mergeAll(replicaMap);
+    storageMap.put(sd.getStorageUuid(),
+        new DatanodeStorage(sd.getStorageUuid(),
+            DatanodeStorage.State.NORMAL,
+            storageType));
+    asyncDiskService.addVolume(volumeImpl);
+    volumes.addVolume(ref);
   }
 
   private void addVolume(Storage.StorageDirectory sd) throws IOException {
@@ -516,8 +474,8 @@ private void addVolume(Storage.StorageDirectory sd) throws IOException {
                               .setConf(this.conf)
                               .build();
     FsVolumeReference ref = fsVolume.obtainReference();
-    ReplicaMap tempVolumeMap =
-        new ReplicaMap(datasetReadLock, datasetWriteLock);
+    // no need to acquire lock.
+    ReplicaMap tempVolumeMap = new ReplicaMap();
     fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker);
 
     activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref);
@@ -556,8 +514,8 @@ public void addVolume(final StorageLocation location,
     StorageType storageType = location.getStorageType();
     final FsVolumeImpl fsVolume =
         createFsVolume(sd.getStorageUuid(), sd, location);
-    final ReplicaMap tempVolumeMap =
-        new ReplicaMap(new ReentrantReadWriteLock());
+    // no need to add lock
+    final ReplicaMap tempVolumeMap = new ReplicaMap();
     ArrayList<IOException> exceptions = Lists.newArrayList();
 
     for (final NamespaceInfo nsInfo : nsInfos) {
@@ -602,7 +560,9 @@ public void removeVolumes(
         new ArrayList<>(storageLocsToRemove);
     Map<String, List<ReplicaInfo>> blkToInvalidate = new HashMap<>();
     List<String> storageToRemove = new ArrayList<>();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    // This object lock is protect data structure related volumes like add and
+    // remove.This will obtain volumeMap lock again if access replicaInfo.
+    synchronized (this) {
       for (int idx = 0; idx < dataStorage.getNumStorageDirs(); idx++) {
         Storage.StorageDirectory sd = dataStorage.getStorageDir(idx);
         final StorageLocation sdLocation = sd.getStorageLocation();
@@ -614,7 +574,7 @@ public void removeVolumes(
           // Disable the volume from the service.
           asyncDiskService.removeVolume(sd.getStorageUuid());
           volumes.removeVolume(sdLocation, clearFailure);
-          volumes.waitVolumeRemoved(5000, datasetWriteLockCondition);
+          volumes.waitVolumeRemoved(5000, this);
 
           // Removed all replica information for the blocks on the volume.
           // Unlike updating the volumeMap in addVolume(), this operation does
@@ -622,18 +582,19 @@ public void removeVolumes(
           for (String bpid : volumeMap.getBlockPoolList()) {
             List<ReplicaInfo> blocks = blkToInvalidate
                 .computeIfAbsent(bpid, (k) -> new ArrayList<>());
-            for (Iterator<ReplicaInfo> it =
-                  volumeMap.replicas(bpid).iterator(); it.hasNext();) {
-              ReplicaInfo block = it.next();
-              final StorageLocation blockStorageLocation =
-                  block.getVolume().getStorageLocation();
-              LOG.trace("checking for block " + block.getBlockId() +
-                  " with storageLocation " + blockStorageLocation);
-              if (blockStorageLocation.equals(sdLocation)) {
-                blocks.add(block);
-                it.remove();
+            volumeMap.replicas(bpid, (iterator) -> {
+              while (iterator.hasNext()) {
+                ReplicaInfo block = iterator.next();
+                final StorageLocation blockStorageLocation =
+                    block.getVolume().getStorageLocation();
+                LOG.trace("checking for block " + block.getBlockId() +
+                    " with storageLocation " + blockStorageLocation);
+                if (blockStorageLocation.equals(sdLocation)) {
+                  blocks.add(block);
+                  iterator.remove();
+                }
               }
-            }
+            });
           }
           storageToRemove.add(sd.getStorageUuid());
           storageLocationsToRemove.remove(sdLocation);
@@ -661,9 +622,12 @@ public void removeVolumes(
       }
     }
 
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-      for(String storageUuid : storageToRemove) {
+    synchronized (this) {
+      for (String storageUuid : storageToRemove) {
         storageMap.remove(storageUuid);
+        for (String bp : volumeMap.getBlockPoolList()) {
+          lockManager.removeLock(LockLevel.VOLUME, bp, storageUuid);
+        }
       }
     }
   }
@@ -852,7 +816,8 @@ public InputStream getBlockInputStream(ExtendedBlock b,
       long seekOffset) throws IOException {
 
     ReplicaInfo info;
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        b.getBlockPoolId())) {
       info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock());
     }
 
@@ -934,13 +899,20 @@ ReplicaInfo getReplicaInfo(String bpid, long blkid)
     return info;
   }
 
+  String getStorageUuidForLock(ExtendedBlock b)
+      throws ReplicaNotFoundException {
+    return getReplicaInfo(b.getBlockPoolId(), b.getBlockId())
+        .getStorageUuid();
+  }
+
   /**
    * Returns handles to the block file and its metadata file
    */
   @Override // FsDatasetSpi
   public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b,
       long blkOffset, long metaOffset) throws IOException {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       ReplicaInfo info = getReplicaInfo(b);
       FsVolumeReference ref = info.getVolume().obtainReference();
       try {
@@ -1116,7 +1088,8 @@ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
             targetStorageType, targetStorageId);
     boolean useVolumeOnSameMount = false;
 
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        block.getBlockPoolId())) {
       if (shouldConsiderSameMountVolume) {
         volumeRef = volumes.getVolumeByMount(targetStorageType,
             ((FsVolumeImpl) replicaInfo.getVolume()).getMount(),
@@ -1310,7 +1283,8 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block, FsVolumeSpi
 
     FsVolumeReference volumeRef = null;
 
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        block.getBlockPoolId())) {
       volumeRef = destination.obtainReference();
     }
 
@@ -1324,6 +1298,11 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block, FsVolumeSpi
     return replicaInfo;
   }
 
+  @Override
+  public DataNodeLockManager<AutoCloseDataSetLock> acquireDatasetLockManager() {
+    return lockManager;
+  }
+
   /**
    * Compute and store the checksum for a block file that does not already have
    * its checksum computed.
@@ -1398,7 +1377,8 @@ static void computeChecksum(ReplicaInfo srcReplica, File dstMeta,
   @Override  // FsDatasetSpi
   public ReplicaHandler append(ExtendedBlock b,
       long newGS, long expectedBlockLen) throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       // If the block was successfully finalized because all packets
       // were successfully processed at the Datanode but the ack for
       // some of the packets were not received by the client. The client
@@ -1450,7 +1430,8 @@ public ReplicaHandler append(ExtendedBlock b,
   private ReplicaInPipeline append(String bpid,
       ReplicaInfo replicaInfo, long newGS, long estimateBlockLen)
       throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        bpid, replicaInfo.getStorageUuid())) {
       // If the block is cached, start uncaching it.
       if (replicaInfo.getState() != ReplicaState.FINALIZED) {
         throw new IOException("Only a Finalized replica can be appended to; "
@@ -1546,7 +1527,8 @@ public ReplicaHandler recoverAppend(
 
     while (true) {
       try {
-        try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl,
+            b.getBlockPoolId())) {
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
           ReplicaInPipeline replica;
@@ -1578,7 +1560,8 @@ public Replica recoverClose(ExtendedBlock b, long newGS,
     LOG.info("Recover failed close " + b);
     while (true) {
       try {
-        try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+            b.getBlockPoolId(), getStorageUuidForLock(b))) {
           // check replica's state
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           // bump the replica's GS
@@ -1601,7 +1584,8 @@ public ReplicaHandler createRbw(
       StorageType storageType, String storageId, ExtendedBlock b,
       boolean allowLazyPersist) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        b.getBlockPoolId())) {
       ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
           b.getBlockId());
       if (replicaInfo != null) {
@@ -1648,20 +1632,20 @@ public ReplicaHandler createRbw(
       }
 
       ReplicaInPipeline newReplicaInfo;
-      try {
+      try (AutoCloseableLock l = lockManager.writeLock(LockLevel.VOLUME,
+          b.getBlockPoolId(), v.getStorageID())) {
         newReplicaInfo = v.createRbw(b);
         if (newReplicaInfo.getReplicaInfo().getState() != ReplicaState.RBW) {
           throw new IOException("CreateRBW returned a replica of state "
               + newReplicaInfo.getReplicaInfo().getState()
               + " for block " + b.getBlockId());
         }
+        volumeMap.add(b.getBlockPoolId(), newReplicaInfo.getReplicaInfo());
+        return new ReplicaHandler(newReplicaInfo, ref);
       } catch (IOException e) {
         IOUtils.cleanupWithLogger(null, ref);
         throw e;
       }
-
-      volumeMap.add(b.getBlockPoolId(), newReplicaInfo.getReplicaInfo());
-      return new ReplicaHandler(newReplicaInfo, ref);
     } finally {
       if (dataNodeMetrics != null) {
         long createRbwMs = Time.monotonicNow() - startTimeMs;
@@ -1679,7 +1663,8 @@ public ReplicaHandler recoverRbw(
     try {
       while (true) {
         try {
-          try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+          try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+              b.getBlockPoolId(), getStorageUuidForLock(b))) {
             ReplicaInfo replicaInfo =
                 getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
             // check the replica's state
@@ -1710,7 +1695,8 @@ public ReplicaHandler recoverRbw(
   private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw,
       ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
       throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       // check generation stamp
       long replicaGenerationStamp = rbw.getGenerationStamp();
       if (replicaGenerationStamp < b.getGenerationStamp() ||
@@ -1771,7 +1757,8 @@ private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw,
   public ReplicaInPipeline convertTemporaryToRbw(
       final ExtendedBlock b) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       final long blockId = b.getBlockId();
       final long expectedGs = b.getGenerationStamp();
       final long visible = b.getNumBytes();
@@ -1850,7 +1837,8 @@ public ReplicaHandler createTemporary(StorageType storageType,
     ReplicaInfo lastFoundReplicaInfo = null;
     boolean isInPipeline = false;
     do {
-      try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+      try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl,
+          b.getBlockPoolId())) {
         ReplicaInfo currentReplicaInfo =
             volumeMap.get(b.getBlockPoolId(), b.getBlockId());
         if (currentReplicaInfo == lastFoundReplicaInfo) {
@@ -1905,11 +1893,12 @@ public ReplicaHandler createTemporary(StorageType storageType,
           false);
     }
     long startHoldLockTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-      FsVolumeReference ref = volumes.getNextVolume(storageType, storageId, b
-          .getNumBytes());
-      FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
-      ReplicaInPipeline newReplicaInfo;
+    FsVolumeReference ref = volumes.getNextVolume(storageType, storageId, b
+        .getNumBytes());
+    FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
+    ReplicaInPipeline newReplicaInfo;
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), v.getStorageID())) {
       try {
         newReplicaInfo = v.createTemporary(b);
         LOG.debug("creating temporary for block: {} on volume: {}",
@@ -1966,7 +1955,8 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
     ReplicaInfo replicaInfo = null;
     ReplicaInfo finalizedReplicaInfo = null;
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       if (Thread.interrupted()) {
         // Don't allow data modifications from interrupted threads
         throw new IOException("Cannot finalize block from Interrupted Thread");
@@ -2002,7 +1992,8 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
 
   private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
       throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        bpid, replicaInfo.getStorageUuid())) {
       // Compare generation stamp of old and new replica before finalizing
       if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp()
           > replicaInfo.getGenerationStamp()) {
@@ -2024,6 +2015,9 @@ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
 
         newReplicaInfo = v.addFinalizedBlock(
             bpid, replicaInfo, replicaInfo, replicaInfo.getBytesReserved());
+        if (replicaInfo instanceof ReplicaInPipeline) {
+          ((ReplicaInPipeline) replicaInfo).releaseReplicaInfoBytesReserved();
+        }
         if (v.isTransientStorage()) {
           releaseLockedMemory(
               replicaInfo.getOriginalBytesReserved()
@@ -2048,7 +2042,8 @@ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
   @Override // FsDatasetSpi
   public void unfinalizeBlock(ExtendedBlock b) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        b.getBlockPoolId(), getStorageUuidForLock(b))) {
       ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
           b.getLocalBlock());
       if (replicaInfo != null &&
@@ -2106,47 +2101,50 @@ public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
         new HashMap<String, BlockListAsLongs.Builder>();
 
     List<FsVolumeImpl> curVolumes = null;
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       curVolumes = volumes.getVolumes();
       for (FsVolumeSpi v : curVolumes) {
         builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength));
       }
 
       Set<String> missingVolumesReported = new HashSet<>();
-      for (ReplicaInfo b : volumeMap.replicas(bpid)) {
-        // skip PROVIDED replicas.
-        if (b.getVolume().getStorageType() == StorageType.PROVIDED) {
-          continue;
-        }
-        String volStorageID = b.getVolume().getStorageID();
-        switch(b.getState()) {
-        case FINALIZED:
-        case RBW:
-        case RWR:
-          break;
-        case RUR:
-          // use the original replica.
-          b = b.getOriginalReplica();
-          break;
-        case TEMPORARY:
-          continue;
-        default:
-          assert false : "Illegal ReplicaInfo state.";
-          continue;
-        }
-        BlockListAsLongs.Builder storageBuilder = builders.get(volStorageID);
-        // a storage in the process of failing will not be in the volumes list
-        // but will be in the replica map.
-        if (storageBuilder != null) {
-          storageBuilder.add(b);
-        } else {
-          if (!missingVolumesReported.contains(volStorageID)) {
-            LOG.warn("Storage volume: " + volStorageID + " missing for the"
-                + " replica block: " + b + ". Probably being removed!");
-            missingVolumesReported.add(volStorageID);
+      volumeMap.replicas(bpid, (iterator) -> {
+        while (iterator.hasNext()) {
+          ReplicaInfo b = iterator.next();
+          // skip PROVIDED replicas.
+          if (b.getVolume().getStorageType() == StorageType.PROVIDED) {
+            continue;
+          }
+          String volStorageID = b.getVolume().getStorageID();
+          switch(b.getState()) {
+          case FINALIZED:
+          case RBW:
+          case RWR:
+            break;
+          case RUR:
+            // use the original replica.
+            b = b.getOriginalReplica();
+            break;
+          case TEMPORARY:
+            continue;
+          default:
+            assert false : "Illegal ReplicaInfo state.";
+            continue;
+          }
+          BlockListAsLongs.Builder storageBuilder = builders.get(volStorageID);
+          // a storage in the process of failing will not be in the volumes list
+          // but will be in the replica map.
+          if (storageBuilder != null) {
+            storageBuilder.add(b);
+          } else {
+            if (!missingVolumesReported.contains(volStorageID)) {
+              LOG.warn("Storage volume: " + volStorageID + " missing for the"
+                  + " replica block: " + b + ". Probably being removed!");
+              missingVolumesReported.add(volStorageID);
+            }
           }
         }
-      }
+      });
     }
 
     for (FsVolumeImpl v : curVolumes) {
@@ -2161,7 +2159,7 @@ public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
    * Gets a list of references to the finalized blocks for the given block pool.
    * <p>
    * Callers of this function should call
-   * {@link FsDatasetSpi#acquireDatasetLock()} to avoid blocks' status being
+   * {@link FsDatasetSpi#acquireDatasetLockManager()} ()} to avoid blocks' status being
    * changed during list iteration.
    * </p>
    * @return a list of references to the finalized blocks for the given block
@@ -2169,16 +2167,16 @@ public Map<DatanodeStorage, BlockListAsLongs> getBlockReports(String bpid) {
    */
   @Override
   public List<ReplicaInfo> getFinalizedBlocks(String bpid) {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
-      final List<ReplicaInfo> finalized = new ArrayList<ReplicaInfo>(
-          volumeMap.size(bpid));
-      for (ReplicaInfo b : volumeMap.replicas(bpid)) {
+    ArrayList<ReplicaInfo> finalized = new ArrayList<>();
+    volumeMap.replicas(bpid, (iterator) -> {
+      while (iterator.hasNext()) {
+        ReplicaInfo b = iterator.next();
         if (b.getState() == ReplicaState.FINALIZED) {
-          finalized.add(b);
+          finalized.add(new FinalizedReplica((FinalizedReplica)b));
         }
       }
-      return finalized;
-    }
+    });
+    return finalized;
   }
 
   /**
@@ -2307,10 +2305,10 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async)
       throws IOException {
     final List<String> errors = new ArrayList<String>();
     for (int i = 0; i < invalidBlks.length; i++) {
-      final ReplicaInfo removing;
+      final ReplicaInfo info;
       final FsVolumeImpl v;
-      try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-        final ReplicaInfo info = volumeMap.get(bpid, invalidBlks[i]);
+      try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
+        info = volumeMap.get(bpid, invalidBlks[i]);
         if (info == null) {
           ReplicaInfo infoByBlockId =
               volumeMap.get(bpid, invalidBlks[i].getBlockId());
@@ -2344,47 +2342,21 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async)
           LOG.warn("Parent directory check failed; replica {} is " +
               "not backed by a local file", info);
         }
-        removing = volumeMap.remove(bpid, invalidBlks[i]);
-        addDeletingBlock(bpid, removing.getBlockId());
-        LOG.debug("Block file {} is to be deleted", removing.getBlockURI());
-        if (removing instanceof ReplicaInPipeline) {
-          ((ReplicaInPipeline) removing).releaseAllBytesReserved();
-        }
-      }
-
-      if (v.isTransientStorage()) {
-        RamDiskReplica replicaInfo =
-          ramDiskReplicaTracker.getReplica(bpid, invalidBlks[i].getBlockId());
-        if (replicaInfo != null) {
-          if (!replicaInfo.getIsPersisted()) {
-            datanode.getMetrics().incrRamDiskBlocksDeletedBeforeLazyPersisted();
-          }
-          ramDiskReplicaTracker.discardReplica(replicaInfo.getBlockPoolId(),
-            replicaInfo.getBlockId(), true);
-        }
       }
 
-      // If a DFSClient has the replica in its cache of short-circuit file
-      // descriptors (and the client is using ShortCircuitShm), invalidate it.
-      datanode.getShortCircuitRegistry().processBlockInvalidation(
-                new ExtendedBlockId(invalidBlks[i].getBlockId(), bpid));
-
-      // If the block is cached, start uncaching it.
-      cacheManager.uncacheBlock(bpid, invalidBlks[i].getBlockId());
-
       try {
         if (async) {
           // Delete the block asynchronously to make sure we can do it fast
           // enough.
           // It's ok to unlink the block file before the uncache operation
           // finishes.
-          asyncDiskService.deleteAsync(v.obtainReference(), removing,
+          asyncDiskService.deleteAsync(v.obtainReference(), info,
               new ExtendedBlock(bpid, invalidBlks[i]),
-              dataStorage.getTrashDirectoryForReplica(bpid, removing));
+              dataStorage.getTrashDirectoryForReplica(bpid, info));
         } else {
-          asyncDiskService.deleteSync(v.obtainReference(), removing,
+          asyncDiskService.deleteSync(v.obtainReference(), info,
               new ExtendedBlock(bpid, invalidBlks[i]),
-              dataStorage.getTrashDirectoryForReplica(bpid, removing));
+              dataStorage.getTrashDirectoryForReplica(bpid, info));
         }
       } catch (ClosedChannelException e) {
         LOG.warn("Volume {} is closed, ignore the deletion task for " +
@@ -2423,6 +2395,91 @@ public void invalidate(String bpid, ReplicaInfo block) {
         block.getStorageUuid());
   }
 
+  /**
+   * Remove Replica from ReplicaMap.
+   *
+   * @param block
+   * @param volume
+   * @return
+   */
+  boolean removeReplicaFromMem(final ExtendedBlock block, final FsVolumeImpl volume) {
+    final String bpid = block.getBlockPoolId();
+    final Block localBlock = block.getLocalBlock();
+    final long blockId = localBlock.getBlockId();
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
+      final ReplicaInfo info = volumeMap.get(bpid, localBlock);
+      if (info == null) {
+        ReplicaInfo infoByBlockId = volumeMap.get(bpid, blockId);
+        if (infoByBlockId == null) {
+          // It is okay if the block is not found -- it
+          // may be deleted earlier.
+          LOG.info("Failed to delete replica {}: ReplicaInfo not found " +
+              "in removeReplicaFromMem.", localBlock);
+        } else {
+          LOG.error("Failed to delete replica {}: GenerationStamp not matched, " +
+              "existing replica is {} in removeReplicaFromMem.",
+              localBlock, Block.toString(infoByBlockId));
+        }
+        return false;
+      }
+
+      FsVolumeImpl v = (FsVolumeImpl) info.getVolume();
+      if (v == null) {
+        LOG.error("Failed to delete replica {}. No volume for this replica {} " +
+            "in removeReplicaFromMem.", localBlock, info);
+        return false;
+      }
+
+      try {
+        File blockFile = new File(info.getBlockURI());
+        if (blockFile.getParentFile() == null) {
+          LOG.error("Failed to delete replica {}. Parent not found for block file: {} " +
+              "in removeReplicaFromMem.", localBlock, blockFile);
+          return false;
+        }
+      } catch(IllegalArgumentException e) {
+        LOG.warn("Parent directory check failed; replica {} is " +
+            "not backed by a local file in removeReplicaFromMem.", info);
+      }
+
+      if (!volume.getStorageID().equals(v.getStorageID())) {
+        LOG.error("Failed to delete replica {}. Appear different volumes, oldVolume: {} " +
+            "and newVolume: {} for this replica in removeReplicaFromMem.",
+            localBlock, volume, v);
+        return false;
+      }
+
+      ReplicaInfo removing = volumeMap.remove(bpid, localBlock);
+      addDeletingBlock(bpid, removing.getBlockId());
+      LOG.debug("Block file {} is to be deleted", removing.getBlockURI());
+      datanode.getMetrics().incrBlocksRemoved(1);
+      if (removing instanceof ReplicaInPipeline) {
+        ((ReplicaInPipeline) removing).releaseAllBytesReserved();
+      }
+    }
+
+    if (volume.isTransientStorage()) {
+      RamDiskReplicaTracker.RamDiskReplica replicaInfo = ramDiskReplicaTracker.
+          getReplica(bpid, blockId);
+      if (replicaInfo != null) {
+        if (!replicaInfo.getIsPersisted()) {
+          datanode.getMetrics().incrRamDiskBlocksDeletedBeforeLazyPersisted();
+        }
+        ramDiskReplicaTracker.discardReplica(replicaInfo.getBlockPoolId(),
+            replicaInfo.getBlockId(), true);
+      }
+    }
+
+    // If a DFSClient has the replica in its cache of short-circuit file
+    // descriptors (and the client is using ShortCircuitShm), invalidate it.
+    datanode.getShortCircuitRegistry().processBlockInvalidation(
+        ExtendedBlockId.fromExtendedBlock(block));
+
+    // If the block is cached, start uncaching it.
+    cacheManager.uncacheBlock(bpid, blockId);
+    return true;
+  }
+
   /**
    * Asynchronously attempts to cache a single block via {@link FsDatasetCache}.
    */
@@ -2432,10 +2489,17 @@ private void cacheBlock(String bpid, long blockId) {
     long length, genstamp;
     Executor volumeExecutor;
 
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-      ReplicaInfo info = volumeMap.get(bpid, blockId);
+    ReplicaInfo info = volumeMap.get(bpid, blockId);
+    if (info == null) {
+      LOG.warn("Failed to cache block with id " + blockId + ", pool " +
+          bpid + ": ReplicaInfo not found.");
+      return;
+    }
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid,
+        info.getStorageUuid())) {
       boolean success = false;
       try {
+        info = volumeMap.get(bpid, blockId);
         if (info == null) {
           LOG.warn("Failed to cache block with id " + blockId + ", pool " +
               bpid + ": ReplicaInfo not found.");
@@ -2500,7 +2564,8 @@ public boolean isCached(String bpid, long blockId) {
 
   @Override // FsDatasetSpi
   public boolean contains(final ExtendedBlock block) {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        block.getBlockPoolId())) {
       final long blockId = block.getLocalBlock().getBlockId();
       final String bpid = block.getBlockPoolId();
       final ReplicaInfo r = volumeMap.get(bpid, blockId);
@@ -2627,7 +2692,8 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo)
       curDirScannerNotifyCount = 0;
       lastDirScannerNotifyTime = startTimeMs;
     }
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid,
+        vol.getStorageID())) {
       memBlockInfo = volumeMap.get(bpid, blockId);
       if (memBlockInfo != null &&
           memBlockInfo.getState() != ReplicaState.FINALIZED) {
@@ -2645,6 +2711,9 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo)
           Block.getGenerationStamp(diskMetaFile.getName()) :
           HdfsConstants.GRANDFATHER_GENERATION_STAMP;
 
+      final boolean isRegular = FileUtil.isRegularFile(diskMetaFile, false) &&
+          FileUtil.isRegularFile(diskFile, false);
+
       if (vol.getStorageType() == StorageType.PROVIDED) {
         if (memBlockInfo == null) {
           // replica exists on provided store but not in memory
@@ -2812,6 +2881,9 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo)
             + memBlockInfo.getNumBytes() + " to "
             + memBlockInfo.getBlockDataLength());
         memBlockInfo.setNumBytes(memBlockInfo.getBlockDataLength());
+      } else if (!isRegular) {
+        corruptBlock = new Block(memBlockInfo);
+        LOG.warn("Block:{} is not a regular file.", corruptBlock.getBlockId());
       }
     } finally {
       if (dataNodeMetrics != null) {
@@ -2844,7 +2916,7 @@ public ReplicaInfo getReplica(String bpid, long blockId) {
 
   @Override 
   public String getReplicaString(String bpid, long blockId) {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       final Replica r = volumeMap.get(bpid, blockId);
       return r == null ? "null" : r.toString();
     }
@@ -2858,12 +2930,40 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
         datanode.getDnConf().getXceiverStopTimeout());
   }
 
+  ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
+      Block block, long recoveryId, long xceiverStopTimeout) throws IOException {
+    while (true) {
+      try {
+        ReplicaInfo replica = map.get(bpid, block.getBlockId());
+        if (replica == null) {
+          return null;
+        }
+        LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
+            + ", replica=" + replica);
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
+            replica.getStorageUuid())) {
+          return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
+        }
+      } catch (MustStopExistingWriter e) {
+        e.getReplicaInPipeline().stopWriter(xceiverStopTimeout);
+      }
+    }
+  }
+
   /** static version of {@link #initReplicaRecovery(RecoveringBlock)}. */
   static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
-      Block block, long recoveryId, long xceiverStopTimeout) throws IOException {
+       Block block, long recoveryId, long xceiverStopTimeout, DataSetLockManager
+       lockManager) throws IOException {
     while (true) {
       try {
-        try (AutoCloseableLock lock = map.getLock().acquire()) {
+        ReplicaInfo replica = map.get(bpid, block.getBlockId());
+        if (replica == null) {
+          return null;
+        }
+        LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
+            + ", replica=" + replica);
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
+            replica.getStorageUuid())) {
           return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
         }
       } catch (MustStopExistingWriter e) {
@@ -2876,9 +2976,6 @@ static ReplicaRecoveryInfo initReplicaRecoveryImpl(String bpid, ReplicaMap map,
       Block block, long recoveryId)
           throws IOException, MustStopExistingWriter {
     final ReplicaInfo replica = map.get(bpid, block.getBlockId());
-    LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
-        + ", replica=" + replica);
-
     //check replica
     if (replica == null) {
       return null;
@@ -2952,7 +3049,8 @@ public Replica updateReplicaUnderRecovery(
                                     final long newBlockId,
                                     final long newlength) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
+        oldBlock.getBlockPoolId(), getStorageUuidForLock(oldBlock))) {
       //get replica
       final String bpid = oldBlock.getBlockPoolId();
       final ReplicaInfo replica = volumeMap.get(bpid, oldBlock.getBlockId());
@@ -3071,7 +3169,8 @@ private ReplicaInfo updateReplicaUnderRecovery(
   @Override // FsDatasetSpi
   public long getReplicaVisibleLength(final ExtendedBlock block)
   throws IOException {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        block.getBlockPoolId())) {
       final Replica replica = getReplicaInfo(block.getBlockPoolId(),
           block.getBlockId());
       if (replica.getGenerationStamp() < block.getGenerationStamp()) {
@@ -3088,13 +3187,17 @@ public void addBlockPool(String bpid, Configuration conf)
       throws IOException {
     LOG.info("Adding block pool " + bpid);
     AddBlockPoolException volumeExceptions = new AddBlockPoolException();
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
       try {
         volumes.addBlockPool(bpid, conf);
       } catch (AddBlockPoolException e) {
         volumeExceptions.mergeException(e);
       }
       volumeMap.initBlockPool(bpid);
+      Set<String> vols = storageMap.keySet();
+      for (String v : vols) {
+        lockManager.addLock(LockLevel.VOLUME, bpid, v);
+      }
     }
     try {
       volumes.getAllVolumesMap(bpid, volumeMap, ramDiskReplicaTracker);
@@ -3118,7 +3221,7 @@ public static void setBlockPoolId(String bpid) {
 
   @Override
   public void shutdownBlockPool(String bpid) {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
       LOG.info("Removing block pool " + bpid);
       Map<DatanodeStorage, BlockListAsLongs> blocksPerVolume
           = getBlockReports(bpid);
@@ -3192,7 +3295,7 @@ public Map<String, Object> getVolumeInfoMap() {
   @Override //FsDatasetSpi
   public void deleteBlockPool(String bpid, boolean force)
       throws IOException {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
       List<FsVolumeImpl> curVolumes = volumes.getVolumes();
       if (!force) {
         for (FsVolumeImpl volume : curVolumes) {
@@ -3221,7 +3324,8 @@ public void deleteBlockPool(String bpid, boolean force)
   @Override // FsDatasetSpi
   public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
       throws IOException {
-    try (AutoCloseableLock lock = datasetReadLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
+        block.getBlockPoolId())) {
       final Replica replica = volumeMap.get(block.getBlockPoolId(),
           block.getBlockId());
       if (replica == null) {
@@ -3275,7 +3379,7 @@ public void clearRollingUpgradeMarker(String bpid) throws IOException {
   @Override
   public void onCompleteLazyPersist(String bpId, long blockId,
       long creationTime, File[] savedFiles, FsVolumeImpl targetVolume) {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpId)) {
       ramDiskReplicaTracker.recordEndLazyPersist(bpId, blockId, savedFiles);
 
       targetVolume.incDfsUsedAndNumBlocks(bpId, savedFiles[0].length()
@@ -3409,7 +3513,8 @@ private boolean saveNextReplica() {
       try {
         block = ramDiskReplicaTracker.dequeueNextReplicaToPersist();
         if (block != null) {
-          try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+          try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl,
+              block.getBlockPoolId())) {
             replicaInfo = volumeMap.get(block.getBlockPoolId(), block.getBlockId());
 
             // If replicaInfo is null, the block was either deleted before
@@ -3475,28 +3580,32 @@ public void evictBlocks(long bytesNeeded) throws IOException {
 
         ReplicaInfo replicaInfo, newReplicaInfo;
         final String bpid = replicaState.getBlockPoolId();
+        final FsVolumeImpl lazyPersistVolume = replicaState.getLazyPersistVolume();
 
-        try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
+        try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
           replicaInfo = getReplicaInfo(replicaState.getBlockPoolId(),
                                        replicaState.getBlockId());
           Preconditions.checkState(replicaInfo.getVolume().isTransientStorage());
           ramDiskReplicaTracker.discardReplica(replicaState.getBlockPoolId(),
               replicaState.getBlockId(), false);
 
-          // Move the replica from lazyPersist/ to finalized/ on
-          // the target volume
-          newReplicaInfo =
-              replicaState.getLazyPersistVolume().activateSavedReplica(bpid,
-                  replicaInfo, replicaState);
-          // Update the volumeMap entry.
-          volumeMap.add(bpid, newReplicaInfo);
-
-          // Update metrics
-          datanode.getMetrics().incrRamDiskBlocksEvicted();
-          datanode.getMetrics().addRamDiskBlocksEvictionWindowMs(
-              Time.monotonicNow() - replicaState.getCreationTime());
-          if (replicaState.getNumReads() == 0) {
-            datanode.getMetrics().incrRamDiskBlocksEvictedWithoutRead();
+          try (AutoCloseableLock lock1 = lockManager.writeLock(LockLevel.VOLUME,
+              bpid, lazyPersistVolume.getStorageID())) {
+            // Move the replica from lazyPersist/ to finalized/ on
+            // the target volume
+            newReplicaInfo =
+                replicaState.getLazyPersistVolume().activateSavedReplica(bpid,
+                    replicaInfo, replicaState);
+            // Update the volumeMap entry.
+            volumeMap.add(bpid, newReplicaInfo);
+
+            // Update metrics
+            datanode.getMetrics().incrRamDiskBlocksEvicted();
+            datanode.getMetrics().addRamDiskBlocksEvictionWindowMs(
+                Time.monotonicNow() - replicaState.getCreationTime());
+            if (replicaState.getNumReads() == 0) {
+              datanode.getMetrics().incrRamDiskBlocksEvictedWithoutRead();
+            }
           }
 
           // Delete the block+meta files from RAM disk and release locked
@@ -3577,8 +3686,8 @@ public void removeDeletedBlocks(String bpid, Set<Long> blockIds) {
       }
     }
   }
-  
-  private void addDeletingBlock(String bpid, Long blockId) {
+
+  protected void addDeletingBlock(String bpid, Long blockId) {
     synchronized(deletingBlock) {
       Set<Long> s = deletingBlock.get(bpid);
       if (s == null) {
@@ -3654,20 +3763,28 @@ public int getVolumeCount() {
   }
 
   void stopAllDataxceiverThreads(FsVolumeImpl volume) {
-    try (AutoCloseableLock lock = datasetWriteLock.acquire()) {
-      for (String bpid : volumeMap.getBlockPoolList()) {
-        Collection<ReplicaInfo> replicas = volumeMap.replicas(bpid);
-        for (ReplicaInfo replicaInfo : replicas) {
-          if ((replicaInfo.getState() == ReplicaState.TEMPORARY
-              || replicaInfo.getState() == ReplicaState.RBW)
-              && replicaInfo.getVolume().equals(volume)) {
-            ReplicaInPipeline replicaInPipeline =
-                (ReplicaInPipeline) replicaInfo;
-            replicaInPipeline.interruptThread();
+    for (String bpid : volumeMap.getBlockPoolList()) {
+      try (AutoCloseDataSetLock lock = lockManager
+          .writeLock(LockLevel.BLOCK_POOl, bpid)) {
+        volumeMap.replicas(bpid, (iterator) -> {
+          while (iterator.hasNext()) {
+            ReplicaInfo replicaInfo = iterator.next();
+            if ((replicaInfo.getState() == ReplicaState.TEMPORARY
+                || replicaInfo.getState() == ReplicaState.RBW)
+                && replicaInfo.getVolume().equals(volume)) {
+              ReplicaInPipeline replicaInPipeline =
+                  (ReplicaInPipeline) replicaInfo;
+              replicaInPipeline.interruptThread();
+            }
           }
-        }
+        });
       }
     }
   }
+
+  @Override
+  public List<FsVolumeImpl> getVolumeList() {
+    return volumes.getVolumes();
+  }
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
index 39e47db7174a2..b7686c512963a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java
@@ -36,7 +36,7 @@
 import java.util.Arrays;
 
 import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.Block;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 459137ea3c293..806afbdb2d115 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -89,7 +89,7 @@
 import com.fasterxml.jackson.databind.ObjectWriter;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
@@ -548,6 +548,10 @@ long getRecentReserved() {
     return recentReserved;
   }
 
+  public Map<String, BlockPoolSlice> getBlockPoolSlices() {
+    return bpSlices;
+  }
+
   long getReserved(){
     return reserved != null ? reserved.getReserved() : 0;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 9400c7c7f4ca1..262a24bd3aa45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -33,6 +33,7 @@
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Condition;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
@@ -43,6 +44,7 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
 import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeDiskMetrics;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.AutoCloseableLock;
@@ -67,15 +69,17 @@ class FsVolumeList {
   private final boolean enableSameDiskTiering;
   private final MountVolumeMap mountVolumeMap;
   private Map<URI, Double> capacityRatioMap;
+  private final DataNodeDiskMetrics diskMetrics;
 
   FsVolumeList(List<VolumeFailureInfo> initialVolumeFailureInfos,
       BlockScanner blockScanner,
       VolumeChoosingPolicy<FsVolumeImpl> blockChooser,
-      Configuration config) {
+      Configuration config, DataNodeDiskMetrics dataNodeDiskMetrics) {
     this.blockChooser = blockChooser;
     this.blockScanner = blockScanner;
     this.checkDirsLock = new AutoCloseableLock();
     this.checkDirsLockCondition = checkDirsLock.newCondition();
+    this.diskMetrics = dataNodeDiskMetrics;
     for (VolumeFailureInfo volumeFailureInfo: initialVolumeFailureInfos) {
       volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(),
           volumeFailureInfo);
@@ -100,6 +104,15 @@ List<FsVolumeImpl> getVolumes() {
 
   private FsVolumeReference chooseVolume(List<FsVolumeImpl> list,
       long blockSize, String storageId) throws IOException {
+
+    // Exclude slow disks when choosing volume.
+    if (diskMetrics != null) {
+      List<String> slowDisksToExclude = diskMetrics.getSlowDisksToExclude();
+      list = list.stream()
+          .filter(volume -> !slowDisksToExclude.contains(volume.getBaseURI().getPath()))
+          .collect(Collectors.toList());
+    }
+
     while (true) {
       FsVolumeImpl volume = blockChooser.chooseVolume(list, blockSize,
           storageId);
@@ -332,6 +345,28 @@ void waitVolumeRemoved(int sleepMillis, Condition condition) {
     FsDatasetImpl.LOG.info("Volume reference is released.");
   }
 
+  /**
+   * Wait for the reference of the volume removed from a previous
+   * {@link #removeVolume(FsVolumeImpl)} call to be released.
+   *
+   * @param sleepMillis interval to recheck.
+   */
+  void waitVolumeRemoved(int sleepMillis, Object condition) {
+    while (!checkVolumesRemoved()) {
+      if (FsDatasetImpl.LOG.isDebugEnabled()) {
+        FsDatasetImpl.LOG.debug("Waiting for volume reference to be released.");
+      }
+      try {
+        condition.wait(sleepMillis);
+      } catch (InterruptedException e) {
+        FsDatasetImpl.LOG.info("Thread interrupted when waiting for "
+            + "volume reference to be released.");
+        Thread.currentThread().interrupt();
+      }
+    }
+    FsDatasetImpl.LOG.info("Volume reference is released.");
+  }
+
   @Override
   public String toString() {
     return volumes.toString();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java
index a35c6f2a31bb4..5d1b21f4a0b93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.ExtendedBlockId;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java
index 6a5810349ea48..526eb55593d4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.ExtendedBlockId;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java
index 7c6562f58d16e..69a46257317bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ProvidedVolumeImpl.java
@@ -30,7 +30,10 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -60,10 +63,6 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Timer;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.ObjectReader;
-import org.codehaus.jackson.map.ObjectWriter;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
@@ -134,7 +133,7 @@ static class ProvidedBlockPoolSlice {
     ProvidedBlockPoolSlice(String bpid, ProvidedVolumeImpl volume,
         Configuration conf) {
       this.providedVolume = volume;
-      bpVolumeMap = new ReplicaMap(new ReentrantReadWriteLock());
+      bpVolumeMap = new ReplicaMap();
       Class<? extends BlockAliasMap> fmt =
           conf.getClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS,
               TextFileRegionAliasMap.class, BlockAliasMap.class);
@@ -219,7 +218,7 @@ private void incrNumBlocks() {
     }
 
     public boolean isEmpty() {
-      return bpVolumeMap.replicas(bpid).size() == 0;
+      return bpVolumeMap.size(bpid) == 0;
     }
 
     public void shutdown(BlockListAsLongs blocksListsAsLongs) {
@@ -371,14 +370,11 @@ public void releaseReservedSpace(long bytesToRelease) {
 
   private static final ObjectWriter WRITER =
       new ObjectMapper().writerWithDefaultPrettyPrinter();
-  private static final ObjectReader READER =
-      new ObjectMapper().reader(ProvidedBlockIteratorState.class);
 
   private static class ProvidedBlockIteratorState {
     ProvidedBlockIteratorState() {
       iterStartMs = Time.now();
       lastSavedMs = iterStartMs;
-      atEnd = false;
       lastBlockId = -1L;
     }
 
@@ -390,9 +386,6 @@ private static class ProvidedBlockIteratorState {
     @JsonProperty
     private long iterStartMs;
 
-    @JsonProperty
-    private boolean atEnd;
-
     // The id of the last block read when the state of the iterator is saved.
     // This implementation assumes that provided blocks are returned
     // in sorted order of the block ids.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
index f7b12ff179941..15bd9dec60478 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -184,7 +184,7 @@ public long getLockedBytesReserved() {
    * {@link org.apache.hadoop.hdfs.DFSConfigKeys#DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY}.
    *
    * @param conf the configuration to be used
-   * @param dataset the FsDataset object.
+   * @param fsDataset the FsDataset object.
    * @return an instance of RamDiskReplicaTracker
    */
   static RamDiskReplicaTracker getInstance(final Configuration conf,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
index c1d103ed50dba..6ecc48a95fd2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java
@@ -18,46 +18,49 @@
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
 import java.util.Collection;
-import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
-import java.util.concurrent.locks.ReadWriteLock;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
+import org.apache.hadoop.hdfs.server.common.NoLockManager;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.util.LightWeightResizableGSet;
-import org.apache.hadoop.util.AutoCloseableLock;
 
 /**
  * Maintains the replica map. 
  */
 class ReplicaMap {
   // Lock object to synchronize this instance.
-  private final AutoCloseableLock readLock;
-  private final AutoCloseableLock writeLock;
-  
+  private DataNodeLockManager<AutoCloseDataSetLock> lockManager;
+
   // Map of block pool Id to another map of block Id to ReplicaInfo.
   private final Map<String, LightWeightResizableGSet<Block, ReplicaInfo>> map =
-      new HashMap<>();
+      new ConcurrentHashMap<>();
 
-  ReplicaMap(AutoCloseableLock readLock, AutoCloseableLock writeLock) {
-    if (readLock == null || writeLock == null) {
+  ReplicaMap(DataNodeLockManager<AutoCloseDataSetLock> manager) {
+    if (manager == null) {
       throw new HadoopIllegalArgumentException(
-          "Lock to synchronize on cannot be null");
+          "Object to synchronize on cannot be null");
     }
-    this.readLock = readLock;
-    this.writeLock = writeLock;
+    this.lockManager = manager;
   }
 
-  ReplicaMap(ReadWriteLock lock) {
-    this(new AutoCloseableLock(lock.readLock()),
-        new AutoCloseableLock(lock.writeLock()));
+  // Used for ut or temp replicaMap that no need to protected by lock.
+  ReplicaMap() {
+    this.lockManager = new NoLockManager();
   }
   
   String[] getBlockPoolList() {
-    try (AutoCloseableLock l = readLock.acquire()) {
-      return map.keySet().toArray(new String[map.keySet().size()]);   
-    }
+    Set<String> bpset = map.keySet();
+    return bpset.toArray(new String[bpset.size()]);
   }
   
   private void checkBlockPool(String bpid) {
@@ -100,7 +103,7 @@ ReplicaInfo get(String bpid, Block block) {
    */
   ReplicaInfo get(String bpid, long blockId) {
     checkBlockPool(bpid);
-    try (AutoCloseableLock l = readLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       return m != null ? m.get(new Block(blockId)) : null;
     }
@@ -117,12 +120,12 @@ ReplicaInfo get(String bpid, long blockId) {
   ReplicaInfo add(String bpid, ReplicaInfo replicaInfo) {
     checkBlockPool(bpid);
     checkBlock(replicaInfo);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       if (m == null) {
         // Add an entry for block pool if it does not exist already
-        m = new LightWeightResizableGSet<Block, ReplicaInfo>();
-        map.put(bpid, m);
+        map.putIfAbsent(bpid, new LightWeightResizableGSet<Block, ReplicaInfo>());
+        m = map.get(bpid);
       }
       return  m.put(replicaInfo);
     }
@@ -135,12 +138,12 @@ ReplicaInfo add(String bpid, ReplicaInfo replicaInfo) {
   ReplicaInfo addAndGet(String bpid, ReplicaInfo replicaInfo) {
     checkBlockPool(bpid);
     checkBlock(replicaInfo);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       if (m == null) {
         // Add an entry for block pool if it does not exist already
-        m = new LightWeightResizableGSet<Block, ReplicaInfo>();
-        map.put(bpid, m);
+        map.putIfAbsent(bpid, new LightWeightResizableGSet<Block, ReplicaInfo>());
+        m = map.get(bpid);
       }
       ReplicaInfo oldReplicaInfo = m.get(replicaInfo);
       if (oldReplicaInfo != null) {
@@ -164,13 +167,28 @@ void addAll(ReplicaMap other) {
    * Merge all entries from the given replica map into the local replica map.
    */
   void mergeAll(ReplicaMap other) {
-    other.map.forEach(
-        (bp, replicaInfos) -> {
-          replicaInfos.forEach(
-              replicaInfo -> add(bp, replicaInfo)
-          );
+    Set<String> bplist = other.map.keySet();
+    for (String bp : bplist) {
+      checkBlockPool(bp);
+      try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bp)) {
+        LightWeightResizableGSet<Block, ReplicaInfo> replicaInfos = other.map.get(bp);
+        LightWeightResizableGSet<Block, ReplicaInfo> curSet = map.get(bp);
+        HashSet<ReplicaInfo> replicaSet = new HashSet<>();
+        //Can't add to GSet while in another GSet iterator may cause endlessLoop
+        for (ReplicaInfo replicaInfo : replicaInfos) {
+          replicaSet.add(replicaInfo);
         }
-    );
+        for (ReplicaInfo replicaInfo : replicaSet) {
+          checkBlock(replicaInfo);
+          if (curSet == null) {
+            // Add an entry for block pool if it does not exist already
+            curSet = new LightWeightResizableGSet<>();
+            map.put(bp, curSet);
+          }
+          curSet.put(replicaInfo);
+        }
+      }
+    }
   }
   
   /**
@@ -184,7 +202,7 @@ void mergeAll(ReplicaMap other) {
   ReplicaInfo remove(String bpid, Block block) {
     checkBlockPool(bpid);
     checkBlock(block);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       if (m != null) {
         ReplicaInfo replicaInfo = m.get(block);
@@ -206,7 +224,7 @@ ReplicaInfo remove(String bpid, Block block) {
    */
   ReplicaInfo remove(String bpid, long blockId) {
     checkBlockPool(bpid);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       if (m != null) {
         return m.remove(new Block(blockId));
@@ -221,7 +239,7 @@ ReplicaInfo remove(String bpid, long blockId) {
    * @return the number of replicas in the map
    */
   int size(String bpid) {
-    try (AutoCloseableLock l = readLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       return m != null ? m.size() : 0;
     }
@@ -229,11 +247,9 @@ int size(String bpid) {
   
   /**
    * Get a collection of the replicas for given block pool
-   * This method is <b>not synchronized</b>. It needs to be synchronized
-   * externally using the lock, both for getting the replicas
-   * values from the map and iterating over it. Mutex can be accessed using
-   * {@link #getLock()} method.
-   * 
+   * This method is <b>not synchronized</b>. If you want to keep thread safe
+   * Use method {@link #replicas(String, Consumer<Iterator<ReplicaInfo>>)}.
+   *
    * @param bpid block pool id
    * @return a collection of the replicas belonging to the block pool
    */
@@ -243,9 +259,25 @@ Collection<ReplicaInfo> replicas(String bpid) {
     return m != null ? m.values() : null;
   }
 
+  /**
+   * execute function for one block pool and protect by LockManager.
+   * This method is <b>synchronized</b>.
+   *
+   * @param bpid block pool id
+   */
+  void replicas(String bpid, Consumer<Iterator<ReplicaInfo>> consumer) {
+    LightWeightResizableGSet<Block, ReplicaInfo> m = null;
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.BLOCK_POOl, bpid)) {
+      m = map.get(bpid);
+      if (m !=null) {
+        m.getIterator(consumer);
+      }
+    }
+  }
+
   void initBlockPool(String bpid) {
     checkBlockPool(bpid);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
       LightWeightResizableGSet<Block, ReplicaInfo> m = map.get(bpid);
       if (m == null) {
         // Add an entry for block pool if it does not exist already
@@ -257,26 +289,8 @@ void initBlockPool(String bpid) {
   
   void cleanUpBlockPool(String bpid) {
     checkBlockPool(bpid);
-    try (AutoCloseableLock l = writeLock.acquire()) {
+    try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
       map.remove(bpid);
     }
   }
-  
-  /**
-   * Get the lock object used for synchronizing ReplicasMap
-   * @return lock object
-   */
-  AutoCloseableLock getLock() {
-    return writeLock;
-  }
-
-  /**
-   * Get the lock object used for synchronizing the ReplicasMap for read only
-   * operations.
-   * @return The read lock object
-   */
-  AutoCloseableLock getReadLock() {
-    return readLock;
-  }
-
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
index cae464bfae700..409084cfe8be8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
@@ -30,13 +30,21 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp;
 import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY;
 
 /**
  * This class detects and maintains DataNode disk outliers and their
@@ -64,11 +72,19 @@ public class DataNodeDiskMetrics {
   /**
    * Minimum number of disks to run outlier detection.
    */
-  private final long minOutlierDetectionDisks;
+  private volatile long minOutlierDetectionDisks;
   /**
    * Threshold in milliseconds below which a disk is definitely not slow.
    */
-  private final long lowThresholdMs;
+  private volatile long lowThresholdMs;
+  /**
+   * The number of slow disks that needs to be excluded.
+   */
+  private int maxSlowDisksToExclude;
+  /**
+   * List of slow disks that need to be excluded.
+   */
+  private List<String> slowDisksToExclude = new ArrayList<>();
 
   public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs,
       Configuration conf) {
@@ -80,6 +96,9 @@ public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs,
     lowThresholdMs =
         conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
             DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT);
+    maxSlowDisksToExclude =
+        conf.getInt(DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY,
+            DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_DEFAULT);
     slowDiskDetector =
         new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs);
     shouldRun = true;
@@ -127,6 +146,21 @@ public void run() {
 
             detectAndUpdateDiskOutliers(metadataOpStats, readIoStats,
                 writeIoStats);
+
+            // Sort the slow disks by latency and extract the top n by maxSlowDisksToExclude.
+            if (maxSlowDisksToExclude > 0) {
+              ArrayList<DiskLatency> diskLatencies = new ArrayList<>();
+              for (Map.Entry<String, Map<DiskOp, Double>> diskStats :
+                  diskOutliersStats.entrySet()) {
+                diskLatencies.add(new DiskLatency(diskStats.getKey(), diskStats.getValue()));
+              }
+
+              Collections.sort(diskLatencies, (o1, o2)
+                  -> Double.compare(o2.getMaxLatency(), o1.getMaxLatency()));
+
+              slowDisksToExclude = diskLatencies.stream().limit(maxSlowDisksToExclude)
+                  .map(DiskLatency::getSlowDisk).collect(Collectors.toList());
+            }
           }
 
           try {
@@ -171,6 +205,35 @@ private void detectAndUpdateDiskOutliers(Map<String, Double> metadataOpStats,
     }
   }
 
+  /**
+   * This structure is a wrapper over disk latencies.
+   */
+  public static class DiskLatency {
+    final private String slowDisk;
+    final private Map<DiskOp, Double> latencyMap;
+
+    public DiskLatency(
+        String slowDiskID,
+        Map<DiskOp, Double> latencyMap) {
+      this.slowDisk = slowDiskID;
+      this.latencyMap = latencyMap;
+    }
+
+    double getMaxLatency() {
+      double maxLatency = 0;
+      for (double latency : latencyMap.values()) {
+        if (latency > maxLatency) {
+          maxLatency = latency;
+        }
+      }
+      return maxLatency;
+    }
+
+    public String getSlowDisk() {
+      return slowDisk;
+    }
+  }
+
   private void addDiskStat(Map<String, Map<DiskOp, Double>> diskStats,
       String disk, DiskOp diskOp, double latency) {
     if (!diskStats.containsKey(disk)) {
@@ -206,4 +269,35 @@ public void addSlowDiskForTesting(String slowDiskPath,
       diskOutliersStats.put(slowDiskPath, latencies);
     }
   }
+
+  public List<String> getSlowDisksToExclude() {
+    return slowDisksToExclude;
+  }
+
+  public void setLowThresholdMs(long thresholdMs) {
+    Preconditions.checkArgument(thresholdMs > 0,
+        DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY + " should be larger than 0");
+    lowThresholdMs = thresholdMs;
+    this.slowDiskDetector.setLowThresholdMs(thresholdMs);
+  }
+
+  public long getLowThresholdMs() {
+    return lowThresholdMs;
+  }
+
+  public void setMinOutlierDetectionDisks(long minDisks) {
+    Preconditions.checkArgument(minDisks > 0,
+        DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY + " should be larger than 0");
+    minOutlierDetectionDisks = minDisks;
+    this.slowDiskDetector.setMinNumResources(minDisks);
+  }
+
+  public long getMinOutlierDetectionDisks() {
+    return minOutlierDetectionDisks;
+  }
+
+  @VisibleForTesting
+  public OutlierDetector getSlowDiskDetector() {
+    return this.slowDiskDetector;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 5203d7bf87f89..649d30e91e034 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -191,6 +191,8 @@ public class DataNodeMetrics {
   @Metric MutableCounterLong packetsSlowWriteToMirror;
   @Metric MutableCounterLong packetsSlowWriteToDisk;
   @Metric MutableCounterLong packetsSlowWriteToOsCache;
+  @Metric private MutableCounterLong slowFlushOrSyncCount;
+  @Metric private MutableCounterLong slowAckToUpstreamCount;
 
   @Metric("Number of replaceBlock ops between" +
       " storage types on same host with local copy")
@@ -440,6 +442,14 @@ public void incrVolumeFailures(int size) {
     volumeFailures.incr(size);
   }
 
+  public void incrSlowFlushOrSyncCount() {
+    slowFlushOrSyncCount.incr();
+  }
+
+  public void incrSlowAckToUpstreamCount() {
+    slowAckToUpstreamCount.incr();
+  }
+
   public void incrDatanodeNetworkErrors() {
     datanodeNetworkErrors.incr();
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
index 750e53db13bf7..a77c3ba0643a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodePeerMetrics.java
@@ -21,18 +21,24 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.metrics2.MetricsJsonBuilder;
 import org.apache.hadoop.metrics2.lib.MutableRollingAverages;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY;
 
 /**
  * This class maintains DataNode peer metrics (e.g. numOps, AvgTime, etc.) for
@@ -49,6 +55,8 @@ public class DataNodePeerMetrics {
 
   private final String name;
 
+  // Strictly to be used by test code only. Source code is not supposed to use this.
+  private Map<String, OutlierMetrics> testOutlier = null;
 
   private final OutlierDetector slowNodeDetector;
 
@@ -57,15 +65,15 @@ public class DataNodePeerMetrics {
    * for outlier detection. If the number of samples is below this then
    * outlier detection is skipped.
    */
-  private final long minOutlierDetectionSamples;
+  private volatile long minOutlierDetectionSamples;
   /**
    * Threshold in milliseconds below which a DataNode is definitely not slow.
    */
-  private final long lowThresholdMs;
+  private volatile long lowThresholdMs;
   /**
    * Minimum number of nodes to run outlier detection.
    */
-  private final long minOutlierDetectionNodes;
+  private volatile long minOutlierDetectionNodes;
 
   public DataNodePeerMetrics(final String name, Configuration conf) {
     this.name = name;
@@ -73,11 +81,11 @@ public DataNodePeerMetrics(final String name, Configuration conf) {
         DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY,
         DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT);
     lowThresholdMs =
-        conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
-            DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT);
+        conf.getLong(DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
+            DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT);
     minOutlierDetectionNodes =
-        conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
-            DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT);
+        conf.getLong(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
+            DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT);
     this.slowNodeDetector =
         new OutlierDetector(minOutlierDetectionNodes, lowThresholdMs);
     sendPacketDownstreamRollingAverages = new MutableRollingAverages("Time");
@@ -87,7 +95,7 @@ public String name() {
     return name;
   }
 
-  long getMinOutlierDetectionSamples() {
+  public long getMinOutlierDetectionSamples() {
     return minOutlierDetectionSamples;
   }
 
@@ -136,18 +144,66 @@ public void collectThreadLocalStates() {
    * Retrieve the set of dataNodes that look significantly slower
    * than their peers.
    */
-  public Map<String, Double> getOutliers() {
-    // This maps the metric name to the aggregate latency.
-    // The metric name is the datanode ID.
-    final Map<String, Double> stats =
-        sendPacketDownstreamRollingAverages.getStats(
-            minOutlierDetectionSamples);
-    LOG.trace("DataNodePeerMetrics: Got stats: {}", stats);
+  public Map<String, OutlierMetrics> getOutliers() {
+    // outlier must be null for source code.
+    if (testOutlier == null) {
+      // This maps the metric name to the aggregate latency.
+      // The metric name is the datanode ID.
+      final Map<String, Double> stats =
+          sendPacketDownstreamRollingAverages.getStats(minOutlierDetectionSamples);
+      LOG.trace("DataNodePeerMetrics: Got stats: {}", stats);
+      return slowNodeDetector.getOutlierMetrics(stats);
+    } else {
+      // this happens only for test code.
+      return testOutlier;
+    }
+  }
 
-    return slowNodeDetector.getOutliers(stats);
+  /**
+   * Strictly to be used by test code only. Source code is not supposed to use this. This method
+   * directly sets outlier mapping so that aggregate latency metrics are not calculated for tests.
+   *
+   * @param outlier outlier directly set by tests.
+   */
+  public void setTestOutliers(Map<String, OutlierMetrics> outlier) {
+    this.testOutlier = outlier;
   }
 
   public MutableRollingAverages getSendPacketDownstreamRollingAverages() {
     return sendPacketDownstreamRollingAverages;
   }
+
+  public void setMinOutlierDetectionNodes(long minNodes) {
+    Preconditions.checkArgument(minNodes > 0,
+        DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY + " should be larger than 0");
+    minOutlierDetectionNodes = minNodes;
+    this.slowNodeDetector.setMinNumResources(minNodes);
+  }
+
+  public long getMinOutlierDetectionNodes() {
+    return minOutlierDetectionNodes;
+  }
+
+  public void setLowThresholdMs(long thresholdMs) {
+    Preconditions.checkArgument(thresholdMs > 0,
+        DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY + " should be larger than 0");
+    lowThresholdMs = thresholdMs;
+    this.slowNodeDetector.setLowThresholdMs(thresholdMs);
+  }
+
+  public long getLowThresholdMs() {
+    return lowThresholdMs;
+  }
+
+  public void setMinOutlierDetectionSamples(long minSamples) {
+    Preconditions.checkArgument(minSamples > 0,
+        DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY +
+            " should be larger than 0");
+    minOutlierDetectionSamples = minSamples;
+  }
+
+  @VisibleForTesting
+  public OutlierDetector getSlowNodeDetector() {
+    return this.slowNodeDetector;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
index 1e20e38fdc7be..e13cf27517852 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/OutlierDetector.java
@@ -22,6 +22,8 @@
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -60,7 +62,7 @@ public class OutlierDetector {
   /**
    * Minimum number of resources to run outlier detection.
    */
-  private final long minNumResources;
+  private volatile long minNumResources;
 
   /**
    * The multiplier is from Leys, C. et al.
@@ -70,7 +72,7 @@ public class OutlierDetector {
   /**
    * Threshold in milliseconds below which a node/ disk is definitely not slow.
    */
-  private final long lowThresholdMs;
+  private volatile long lowThresholdMs;
 
   /**
    * Deviation multiplier. A sample is considered to be an outlier if it
@@ -108,6 +110,26 @@ public OutlierDetector(long minNumResources, long lowThresholdMs) {
    * @return
    */
   public Map<String, Double> getOutliers(Map<String, Double> stats) {
+    final Map<String, Double> slowResources = new HashMap<>();
+    Map<String, OutlierMetrics> slowResourceMetrics = getOutlierMetrics(stats);
+    slowResourceMetrics.forEach(
+        (node, outlierMetrics) -> slowResources.put(node, outlierMetrics.getActualLatency()));
+    return slowResources;
+  }
+
+  /**
+   * Return a set of nodes whose latency is much higher than
+   * their counterparts. The input is a map of (resource {@literal ->} aggregate
+   * latency) entries.
+   *
+   * The aggregate may be an arithmetic mean or a percentile e.g.
+   * 90th percentile. Percentiles are a better choice than median
+   * since latency is usually not a normal distribution.
+   *
+   * @param stats map of aggregate latency entries.
+   * @return map of outlier nodes to outlier metrics.
+   */
+  public Map<String, OutlierMetrics> getOutlierMetrics(Map<String, Double> stats) {
     if (stats.size() < minNumResources) {
       LOG.debug("Skipping statistical outlier detection as we don't have " +
               "latency data for enough resources. Have {}, need at least {}",
@@ -124,19 +146,20 @@ public Map<String, Double> getOutliers(Map<String, Double> stats) {
     upperLimitLatency = Math.max(
         upperLimitLatency, median + (DEVIATION_MULTIPLIER * mad));
 
-    final Map<String, Double> slowResources = new HashMap<>();
+    final Map<String, OutlierMetrics> slowResources = new HashMap<>();
 
-    LOG.trace("getOutliers: List={}, MedianLatency={}, " +
-        "MedianAbsoluteDeviation={}, upperLimitLatency={}",
-        sorted, median, mad, upperLimitLatency);
+    LOG.trace("getOutliers: List={}, MedianLatency={}, "
+            + "MedianAbsoluteDeviation={}, upperLimitLatency={}", sorted, median, mad,
+        upperLimitLatency);
 
     // Find resources whose latency exceeds the threshold.
     for (Map.Entry<String, Double> entry : stats.entrySet()) {
       if (entry.getValue() > upperLimitLatency) {
-        slowResources.put(entry.getKey(), entry.getValue());
+        OutlierMetrics outlierMetrics =
+            new OutlierMetrics(median, mad, upperLimitLatency, entry.getValue());
+        slowResources.put(entry.getKey(), outlierMetrics);
       }
     }
-
     return slowResources;
   }
 
@@ -180,4 +203,20 @@ public static Double computeMedian(List<Double> sortedValues) {
     }
     return median;
   }
+
+  public void setMinNumResources(long minNodes) {
+    minNumResources = minNodes;
+  }
+
+  public long getMinOutlierDetectionNodes() {
+    return minNumResources;
+  }
+
+  public void setLowThresholdMs(long thresholdMs) {
+    lowThresholdMs = thresholdMs;
+  }
+
+  public long getLowThresholdMs() {
+    return lowThresholdMs;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index e4a0c209762be..bb2757287fbe0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web;
 
-import io.netty.bootstrap.ChannelFactory;
 import io.netty.bootstrap.ServerBootstrap;
+import io.netty.channel.ChannelFactory;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelInitializer;
@@ -28,6 +28,7 @@
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.channel.WriteBufferWaterMark;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.ssl.SslHandler;
@@ -64,7 +65,6 @@
 import java.security.GeneralSecurityException;
 import java.util.Enumeration;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT;
@@ -77,8 +77,6 @@
  */
 public class DatanodeHttpServer implements Closeable {
   static final Logger LOG = LoggerFactory.getLogger(DatanodeHttpServer.class);
-  private static final ConcurrentHashMap<Class<?>, Object> HANDLER_STATE
-      = new ConcurrentHashMap<Class<?>, Object>() {};
   // HttpServer threads are only used for the web UI and basic servlets, so
   // set them to the minimum possible
   private static final int HTTP_SELECTOR_THREADS = 1;
@@ -171,15 +169,13 @@ protected void initChannel(SocketChannel ch) throws Exception {
             });
 
       this.httpServer.childOption(
-          ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK,
-          conf.getInt(
-              DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK,
-              DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK_DEFAULT));
-      this.httpServer.childOption(
-          ChannelOption.WRITE_BUFFER_LOW_WATER_MARK,
-          conf.getInt(
-              DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK,
-              DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK_DEFAULT));
+          ChannelOption.WRITE_BUFFER_WATER_MARK,
+          new WriteBufferWaterMark(conf.getInt(
+               DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK,
+               DFSConfigKeys.DFS_WEBHDFS_NETTY_LOW_WATERMARK_DEFAULT),
+               conf.getInt(
+                   DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK,
+                   DFSConfigKeys.DFS_WEBHDFS_NETTY_HIGH_WATERMARK_DEFAULT)));
 
       if (externalHttpChannel == null) {
         httpServer.channel(NioServerSocketChannel.class);
@@ -281,11 +277,10 @@ private ChannelHandler[] getFilterHandlers(Configuration configuration) {
       try {
         Method initializeState = classes[i].getDeclaredMethod("initializeState",
             Configuration.class);
-        Constructor constructor =
+        Constructor<?> constructor =
             classes[i].getDeclaredConstructor(initializeState.getReturnType());
         handlers[i] = (ChannelHandler) constructor.newInstance(
-            HANDLER_STATE.getOrDefault(classes[i],
-            initializeState.invoke(null, configuration)));
+            initializeState.invoke(null, configuration));
       } catch (NoSuchMethodException | InvocationTargetException
           | IllegalAccessException | InstantiationException
           | IllegalArgumentException e) {
@@ -307,18 +302,7 @@ public InetSocketAddress getHttpsAddress() {
   public void start() throws IOException {
     if (httpServer != null) {
       InetSocketAddress infoAddr = DataNode.getInfoAddr(conf);
-      ChannelFuture f = httpServer.bind(infoAddr);
-      try {
-        f.syncUninterruptibly();
-      } catch (Throwable e) {
-        if (e instanceof BindException) {
-          throw NetUtils.wrapException(null, 0, infoAddr.getHostName(),
-              infoAddr.getPort(), (SocketException) e);
-        } else {
-          throw e;
-        }
-      }
-      httpAddress = (InetSocketAddress) f.channel().localAddress();
+      httpAddress = getChannelLocalAddress(httpServer, infoAddr);
       LOG.info("Listening HTTP traffic on " + httpAddress);
     }
 
@@ -327,21 +311,25 @@ public void start() throws IOException {
           NetUtils.createSocketAddr(conf.getTrimmed(
               DFS_DATANODE_HTTPS_ADDRESS_KEY,
               DFS_DATANODE_HTTPS_ADDRESS_DEFAULT));
-      ChannelFuture f = httpsServer.bind(secInfoSocAddr);
+      httpsAddress = getChannelLocalAddress(httpsServer, secInfoSocAddr);
+      LOG.info("Listening HTTPS traffic on " + httpsAddress);
+    }
+  }
 
-      try {
-        f.syncUninterruptibly();
-      } catch (Throwable e) {
-        if (e instanceof BindException) {
-          throw NetUtils.wrapException(null, 0, secInfoSocAddr.getHostName(),
-              secInfoSocAddr.getPort(), (SocketException) e);
-        } else {
-          throw e;
-        }
+  private InetSocketAddress getChannelLocalAddress(
+      ServerBootstrap server, InetSocketAddress address) throws IOException {
+    ChannelFuture f = server.bind(address);
+    try {
+      f.syncUninterruptibly();
+    } catch (Throwable e) {
+      if (e instanceof BindException) {
+        throw NetUtils.wrapException(null, 0, address.getHostName(),
+                address.getPort(), (SocketException) e);
+      } else {
+        throw e;
       }
-      httpsAddress = (InetSocketAddress) f.channel().localAddress();
-      LOG.info("Listening HTTPS traffic on " + httpsAddress);
     }
+    return (InetSocketAddress) f.channel().localAddress();
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java
index 798def0c716b7..c76d93469196b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java
@@ -43,8 +43,8 @@
 import java.util.List;
 import java.util.Map;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 
@@ -197,7 +197,7 @@ public String getRemoteAddr() {
     @Override
     public String getQueryString() {
       try {
-        return (new URI(req.getUri()).getQuery());
+        return (new URI(req.uri()).getQuery());
       } catch (URISyntaxException e) {
         return null;
       }
@@ -205,7 +205,7 @@ public String getQueryString() {
 
     @Override
     public String getRequestURI() {
-      String uri = req.getUri();
+      String uri = req.uri();
       // Netty's getUri includes the query string, while Servlet's does not
       return (uri.substring(0, uri.indexOf("?") >= 0 ? uri.indexOf("?") :
           uri.length()));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
index a2c2d370439ac..4e3c2d1ae7bda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
index 9a2e0b71a4482..ef8d90a4c4e6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
@@ -34,12 +34,12 @@
 import io.netty.handler.codec.http.HttpRequest;
 import io.netty.handler.codec.http.HttpRequestEncoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import org.slf4j.Logger;
 
 import java.net.InetSocketAddress;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Values;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
 import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 
@@ -98,7 +98,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
   @Override
   public void channelRead0
     (final ChannelHandlerContext ctx, final HttpRequest req) {
-    uri = req.getUri();
+    uri = req.uri();
     final Channel client = ctx.channel();
     Bootstrap proxiedServer = new Bootstrap()
       .group(client.eventLoop())
@@ -117,15 +117,14 @@ protected void initChannel(SocketChannel ch) throws Exception {
       public void operationComplete(ChannelFuture future) throws Exception {
         if (future.isSuccess()) {
           ctx.channel().pipeline().remove(HttpResponseEncoder.class);
-          HttpRequest newReq = new DefaultFullHttpRequest(HTTP_1_1,
-            req.getMethod(), req.getUri());
+          HttpRequest newReq = new DefaultFullHttpRequest(HTTP_1_1, req.method(), req.uri());
           newReq.headers().add(req.headers());
-          newReq.headers().set(CONNECTION, Values.CLOSE);
+          newReq.headers().set(CONNECTION, HttpHeaderValues.CLOSE);
           future.channel().writeAndFlush(newReq);
         } else {
           DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
             INTERNAL_SERVER_ERROR);
-          resp.headers().set(CONNECTION, Values.CLOSE);
+          resp.headers().set(CONNECTION, HttpHeaderValues.CLOSE);
           LOG.info("Proxy " + uri + " failed. Cause: ", future.cause());
           ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
           client.close();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
index 8ec5bf6f64c84..4545f733a8cf3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
@@ -43,7 +43,7 @@ class URLDispatcher extends SimpleChannelInboundHandler<HttpRequest> {
   @Override
   protected void channelRead0(ChannelHandlerContext ctx, HttpRequest req)
       throws Exception {
-    String uri = req.getUri();
+    String uri = req.uri();
     ChannelPipeline p = ctx.pipeline();
     if (uri.startsWith(WEBHDFS_PREFIX)) {
       WebHdfsHandler h = new WebHdfsHandler(conf, confForCreate);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index 02ec25c13c874..6fe0851bb3f70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -33,8 +33,8 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
 import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
index 3d928ff3eed64..f01479a9b92e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
@@ -30,8 +30,8 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
 
 class HdfsWriter extends SimpleChannelInboundHandler<HttpContent> {
   private final DFSClient client;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index 2f901da165ff4..8de736ac0e237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
@@ -62,17 +62,17 @@
 import java.security.PrivilegedExceptionAction;
 import java.util.EnumSet;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.ACCEPT;
-import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_HEADERS;
-import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS;
-import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN;
-import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
-import static io.netty.handler.codec.http.HttpHeaders.Names.LOCATION;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaderNames.LOCATION;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE;
 import static io.netty.handler.codec.http.HttpMethod.GET;
 import static io.netty.handler.codec.http.HttpMethod.OPTIONS;
 import static io.netty.handler.codec.http.HttpMethod.POST;
@@ -122,8 +122,8 @@ public WebHdfsHandler(Configuration conf, Configuration confForCreate)
   @Override
   public void channelRead0(final ChannelHandlerContext ctx,
                            final HttpRequest req) throws Exception {
-    Preconditions.checkArgument(req.getUri().startsWith(WEBHDFS_PREFIX));
-    QueryStringDecoder queryString = new QueryStringDecoder(req.getUri());
+    Preconditions.checkArgument(req.uri().startsWith(WEBHDFS_PREFIX));
+    QueryStringDecoder queryString = new QueryStringDecoder(req.uri());
     params = new ParameterParser(queryString, conf);
     DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
     ugi = ugiProvider.ugi();
@@ -144,7 +144,7 @@ public Void run() throws Exception {
             LOG.warn("Error retrieving hostname: ", e);
             host = "unknown";
           }
-          REQLOG.info(host + " " + req.getMethod() + " "  + req.getUri() + " " +
+          REQLOG.info(host + " " + req.method() + " "  + req.uri() + " " +
               getResponseCode());
         }
         return null;
@@ -154,13 +154,13 @@ public Void run() throws Exception {
 
   int getResponseCode() {
     return (resp == null) ? INTERNAL_SERVER_ERROR.code() :
-        resp.getStatus().code();
+        resp.status().code();
   }
 
   public void handle(ChannelHandlerContext ctx, HttpRequest req)
     throws IOException, URISyntaxException {
     String op = params.op();
-    HttpMethod method = req.getMethod();
+    HttpMethod method = req.method();
     if (PutOpParam.Op.CREATE.name().equalsIgnoreCase(op)
       && method == PUT) {
       onCreate(ctx);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
index 6889f5f92e389..2707b9412bc3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.command;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.codec.digest.DigestUtils;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index b78a86bf499e3..c90b77e98d2e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -52,7 +52,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.Closeable;
 import java.io.IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
index 78b16e999f84e..ff5389b43d002 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.command;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.codec.digest.DigestUtils;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
index e36628edf0eb2..ec180538a9843 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.command;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index e9f9f33e71535..3732de86aa0fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.diskbalancer.command;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
 
 import org.apache.commons.cli.CommandLine;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
index 520e80f3974c7..f5d4cfacd8265 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.command;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.hadoop.conf.Configuration;
@@ -84,7 +84,7 @@ public void execute(CommandLine cmd) throws Exception {
         System.out.printf("%s", workStatus.currentStateString());
       }
     } catch (DiskBalancerException ex) {
-      LOG.error("Query plan failed. ex: {}", ex);
+      LOG.error("Query plan failed.", ex);
       throw ex;
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
index f93a909418332..e26f0a113b33f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -34,7 +34,7 @@
 import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
 import org.apache.hadoop.util.Lists;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java
index b7bb3f02dce9d..f9bcd5e018065 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
index 268c055a354ac..1cc82253f9885 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
@@ -19,7 +19,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
index 0e405ff7bd3c5..c801f36ea5205 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
index fce858aaca01b..fe9edf6678ac5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java
index bcce012ff84b9..f59f4fc9e3fe3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
index 0ed56afb39a5a..59b908671791a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer.planner;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java
index 72df5abe6bcaa..39a7c57bca2cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java
@@ -21,7 +21,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.ObjectWriter;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 import java.util.LinkedList;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
index a610aee48496c..a6e7d00f848d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
@@ -26,8 +26,6 @@
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.util.LongBitFormat;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
-
 /**
  * Class to pack an AclEntry into an integer. <br>
  * An ACL entry is represented by a 32-bit integer in Big Endian format. <br>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index 2dfe50742dac2..9c67d3da43ab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 
 /**
@@ -56,8 +55,9 @@ int getEntriesSize() {
    * @throws IndexOutOfBoundsException if pos out of bound
    */
   int getEntryAt(int pos) {
-    Preconditions.checkPositionIndex(pos, entries.length,
-        "Invalid position for AclEntry");
+    if (pos < 0 || pos > entries.length) {
+      throw new IndexOutOfBoundsException("Invalid position for AclEntry");
+    }
     return entries[pos];
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
index 614eb63d055a7..49dcb8c0a3757 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java
@@ -17,14 +17,13 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import java.net.InetAddress;
-import java.security.Principal;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 
+import java.net.InetAddress;
+
 /**
  * Interface defining an audit logger.
  */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index 48a3416922baf..7bde21dcb69e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.StringUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Extension of FSImage for the backup node.
@@ -222,7 +222,7 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data)
       try {
         getNamesystem().dir.updateCountForQuota();
       } finally {
-        getNamesystem().writeUnlock();
+        getNamesystem().writeUnlock("applyEdits");
       }
     } finally {
       backupInputStream.clear();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
index 03dab29a54a5d..e71b057595952 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java
@@ -976,9 +976,8 @@ private void setCachedLocations(LocatedBlock block) {
   public final void processCacheReport(final DatanodeID datanodeID,
       final List<Long> blockIds) throws IOException {
     if (!enabled) {
-      LOG.debug("Ignoring cache report from {} because {} = false. " +
-              "number of blocks: {}", datanodeID,
-              DFS_NAMENODE_CACHING_ENABLED_KEY, blockIds.size());
+      LOG.debug("Ignoring cache report from {} because {} = false. number of blocks: {}",
+          datanodeID, DFS_NAMENODE_CACHING_ENABLED_KEY, blockIds.size());
       return;
     }
     namesystem.writeLock();
@@ -1003,9 +1002,8 @@ public final void processCacheReport(final DatanodeID datanodeID,
     if (metrics != null) {
       metrics.addCacheBlockReport((int) (endTime - startTime));
     }
-    LOG.debug("Processed cache report from {}, blocks: {}, " +
-        "processing time: {} msecs", datanodeID, blockIds.size(), 
-        (endTime - startTime));
+    LOG.debug("Processed cache report from {}, blocks: {}, processing time: {} msecs",
+        datanodeID, blockIds.size(), (endTime - startTime));
   }
 
   private void processCacheReportImpl(final DatanodeDescriptor datanode,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java
index dda4789b74e49..8016a1ec11a30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java
@@ -32,7 +32,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.IntrusiveCollection;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * A CachePool describes a set of cache resources being managed by the NameNode.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
index d18d448ab2500..ab657fb672490 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
@@ -250,7 +250,7 @@ void doCheckpoint() throws IOException {
               sig.mostRecentCheckpointTxId);
           bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
         } finally {
-          backupNode.namesystem.writeUnlock();
+          backupNode.namesystem.writeUnlock("doCheckpointByBackupNode");
         }
       }
       rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index e304baf652843..61a34828f14dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.FsAction;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
index d17fd06bc882c..a77075fa4f2e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
@@ -21,7 +21,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
index 459cb5be5db67..ba4f32fd2154d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
@@ -46,7 +46,7 @@
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
index ccc233efcbafe..9577a528923ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
@@ -32,7 +32,7 @@
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.IOUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * A double-buffer for edits. New edits are written into the first buffer
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index de6e49d747aa5..7bf5879971615 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -53,7 +53,7 @@
 import org.apache.hadoop.util.Lists;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 
@@ -192,7 +192,7 @@ public void pauseForTestingAfterNthCheckpoint(final String zone,
     try {
       iip = dir.resolvePath(pc, zone, DirOp.READ);
     } finally {
-      dir.getFSNamesystem().readUnlock();
+      dir.getFSNamesystem().readUnlock("pauseForTestingAfterNthCheckpoint");
     }
     reencryptionHandler
         .pauseForTestingAfterNthCheckpoint(iip.getLastINode().getId(), count);
@@ -224,7 +224,7 @@ public ZoneReencryptionStatus getZoneStatus(final String zone)
       return getReencryptionStatus().getZoneStatus(inode.getId());
     } finally {
       dir.readUnlock();
-      dir.getFSNamesystem().readUnlock();
+      dir.getFSNamesystem().readUnlock("getZoneStatus");
     }
   }
 
@@ -285,7 +285,7 @@ void stopReencryptThread() {
     try {
       reencryptionHandler.stopThreads();
     } finally {
-      dir.getFSNamesystem().writeUnlock();
+      dir.getFSNamesystem().writeUnlock("stopReencryptThread");
     }
     if (reencryptHandlerExecutor != null) {
       reencryptHandlerExecutor.shutdownNow();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 7dddfeda21c42..1a7a65a0d5562 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -18,7 +18,8 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -304,6 +305,12 @@ public synchronized ErasureCodingPolicy addPolicy(
           + policy.getCodecName() + " is not supported");
     }
 
+    int blocksInGroup = policy.getNumDataUnits() + policy.getNumParityUnits();
+    if (blocksInGroup > HdfsServerConstants.MAX_BLOCKS_IN_GROUP) {
+      throw new HadoopIllegalArgumentException("Number of data and parity blocks in an EC group " +
+          blocksInGroup + " should not exceed maximum " + HdfsServerConstants.MAX_BLOCKS_IN_GROUP);
+    }
+
     if (policy.getCellSize() > maxCellSize) {
       throw new HadoopIllegalArgumentException("Cell size " +
           policy.getCellSize() + " should not exceed maximum " +
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index 6ced588e96882..b17c459cc993a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclEntryScope;
@@ -228,10 +226,12 @@ private static void unprotectedRemoveAcl(FSDirectory fsd, INodesInPath iip)
       int groupEntryIndex = Collections.binarySearch(
           featureEntries, groupEntryKey,
           AclTransformation.ACL_ENTRY_COMPARATOR);
-      Preconditions.checkPositionIndex(groupEntryIndex, featureEntries.size(),
-          "Invalid group entry index after binary-searching inode: " +
-              inode.getFullPathName() + "(" + inode.getId() + ") "
-              + "with featureEntries:" + featureEntries);
+      if (groupEntryIndex < 0 || groupEntryIndex > featureEntries.size()) {
+        throw new IndexOutOfBoundsException(
+            "Invalid group entry index after binary-searching inode: "
+                + inode.getFullPathName() + "(" + inode.getId() + ") "
+                + "with featureEntries:" + featureEntries);
+      }
       FsAction groupPerm = featureEntries.get(groupEntryIndex).getPermission();
       FsPermission newPerm = new FsPermission(perm.getUserAction(), groupPerm,
           perm.getOtherAction(), perm.getStickyBit());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 7e90d4bafa8bb..ba00e8ae936a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -40,7 +40,7 @@
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion.Feature;
 import org.apache.hadoop.ipc.RetriableException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Helper class to perform append operation.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index db1baab66b3dc..04913d1a7cee0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -415,7 +415,7 @@ static BlockInfo[] unprotectedSetReplication(
       bm.setReplication(oldBR, targetReplication, b);
     }
 
-    if (oldBR != -1) {
+    if (oldBR != -1 && FSDirectory.LOG.isDebugEnabled()) {
       if (oldBR > targetReplication) {
         FSDirectory.LOG.debug("Decreasing replication from {} to {} for {}",
                              oldBR, targetReplication, iip.getPath());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index 04ae358c67afb..ea5ac38aa8659 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.fs.permission.FsAction;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index 0969679be175c..2110a408b0877 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -54,7 +54,7 @@
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.Time;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 
 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
@@ -656,7 +656,7 @@ static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
     Preconditions.checkNotNull(ezKeyName);
 
     // Generate EDEK while not holding the fsn lock.
-    fsn.writeUnlock();
+    fsn.writeUnlock("getEncryptionKeyInfo");
     try {
       EncryptionFaultInjector.getInstance().startFileBeforeGenerateKey();
       return new EncryptionKeyInfo(protocolVersion, suite, ezKeyName,
@@ -733,7 +733,7 @@ static String getKeyNameForZone(final FSDirectory dir,
       dir.ezManager.checkEncryptionZoneRoot(iip.getLastINode(), zone);
       return dir.ezManager.getKeyName(iip);
     } finally {
-      dir.getFSNamesystem().readUnlock();
+      dir.getFSNamesystem().readUnlock("getKeyNameForZone");
     }
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 4815dfce986ac..6628b56a132e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -34,7 +34,7 @@
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.Lists;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index da324fb46738a..862880d95b2d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import org.apache.hadoop.fs.permission.FsCreateModes;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index 8f6fcdc1d1f02..64bc46d901622 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.InvalidPathException;
@@ -81,8 +81,12 @@ private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src,
     // Assume dstParent existence check done by callers.
     INode dstParent = dst.getINode(-2);
     // Use the destination parent's storage policy for quota delta verify.
+    final boolean isSrcSetSp = src.getLastINode().isSetStoragePolicy();
+    final byte storagePolicyID = isSrcSetSp ?
+        src.getLastINode().getLocalStoragePolicyID() :
+        dstParent.getStoragePolicyID();
     final QuotaCounts delta = src.getLastINode()
-        .computeQuotaUsage(bsps, dstParent.getStoragePolicyID(), false,
+        .computeQuotaUsage(bsps, storagePolicyID, false,
             Snapshot.CURRENT_STATE_ID);
 
     // Reduce the required quota by dst that is being removed
@@ -148,8 +152,8 @@ private static INodesInPath dstForRenameTo(
    * @param srcIIP source path
    * @param dstIIP destination path
    * @return true INodesInPath if rename succeeds; null otherwise
-   * @deprecated See {@link #renameToInt(FSDirectory, String, String,
-   * boolean, Options.Rename...)}
+   * @deprecated See {@link #renameToInt(FSDirectory, FSPermissionChecker,
+   * String, String, boolean, Options.Rename...)}
    */
   @Deprecated
   static INodesInPath unprotectedRenameTo(FSDirectory fsd,
@@ -244,8 +248,8 @@ static RenameResult renameToInt(
     String src = srcArg;
     String dst = dstArg;
     if (NameNode.stateChangeLog.isDebugEnabled()) {
-      NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: with options -" +
-          " " + src + " to " + dst);
+      NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: with options={} {} to {}",
+          Arrays.toString(options), src, dst);
     }
 
     BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
@@ -254,8 +258,8 @@ static RenameResult renameToInt(
   }
 
   /**
-   * @see {@link #unprotectedRenameTo(FSDirectory, String, String, INodesInPath,
-   * INodesInPath, long, BlocksMapUpdateInfo, Options.Rename...)}
+   * @see {@link #unprotectedRenameTo(FSDirectory, INodesInPath, INodesInPath,
+   * long, BlocksMapUpdateInfo, Options.Rename...)}
    */
   static RenameResult renameTo(FSDirectory fsd, FSPermissionChecker pc,
       String src, String dst, BlocksMapUpdateInfo collectedBlocks,
@@ -478,8 +482,8 @@ static RenameResult unprotectedRenameTo(FSDirectory fsd,
   }
 
   /**
-   * @deprecated Use {@link #renameToInt(FSDirectory, String, String,
-   * boolean, Options.Rename...)}
+   * @deprecated Use {@link #renameToInt(FSDirectory, FSPermissionChecker,
+   * String, String, boolean, Options.Rename...)}
    */
   @Deprecated
   private static RenameResult renameTo(FSDirectory fsd, FSPermissionChecker pc,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 8aff179358896..236e308f4101e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 0d9c6aeeb9c45..7b7f4a0f9c070 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.hdfs.AddBlockFlag;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 4151e016f4261..6e6ade291ce27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.util.Lists;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index d89b087c46061..d4fed21e98e17 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -22,7 +22,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1372,9 +1372,13 @@ public INodesInPath addLastINode(INodesInPath existing, INode inode,
     // always verify inode name
     verifyINodeName(inode.getLocalNameBytes());
 
+    final boolean isSrcSetSp = inode.isSetStoragePolicy();
+    final byte storagePolicyID = isSrcSetSp ?
+        inode.getLocalStoragePolicyID() :
+        parent.getStoragePolicyID();
     final QuotaCounts counts = inode
         .computeQuotaUsage(getBlockStoragePolicySuite(),
-            parent.getStoragePolicyID(), false, Snapshot.CURRENT_STATE_ID);
+            storagePolicyID, false, Snapshot.CURRENT_STATE_ID);
     updateCount(existing, pos, counts, checkQuota);
 
     boolean isRename = (inode.getParent() != null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index aa6a0a001bad1..5bb6872e5889d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
 import static org.apache.hadoop.util.ExitUtil.terminate;
 import static org.apache.hadoop.util.Time.monotonicNow;
 
@@ -30,6 +31,7 @@
 import java.util.List;
 import java.util.concurrent.atomic.LongAdder;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -107,12 +109,11 @@
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
 import org.apache.hadoop.util.Lists;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -195,6 +196,9 @@ private enum State {
 
   protected final OpInstanceCache cache = new OpInstanceCache();
 
+  // Users who can override the client ip
+  private final String[] ipProxyUsers;
+
   /**
    * The edit directories that are shared between primary and secondary.
    */
@@ -246,6 +250,7 @@ static FSEditLog newInstance(Configuration conf, NNStorage storage,
    * @param editsDirs List of journals to use
    */
   FSEditLog(Configuration conf, NNStorage storage, List<URI> editsDirs) {
+    ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS);
     isSyncRunning = false;
     this.conf = conf;
     this.storage = storage;
@@ -799,8 +804,10 @@ private void printStatistics(boolean force) {
   /** Record the RPC IDs if necessary */
   private void logRpcIds(FSEditLogOp op, boolean toLogRpcIds) {
     if (toLogRpcIds) {
-      op.setRpcClientId(Server.getClientId());
-      op.setRpcCallId(Server.getCallId());
+      Pair<byte[], Integer> clientIdAndCallId =
+          NameNode.getClientIdAndCallId(this.ipProxyUsers);
+      op.setRpcClientId(clientIdAndCallId.getLeft());
+      op.setRpcCallId(clientIdAndCallId.getRight());
     }
   }
 
@@ -1515,11 +1522,12 @@ public synchronized void purgeLogsOlderThan(final long minTxIdToKeep) {
     if (!isOpenForWrite()) {
       return;
     }
-    
-    assert curSegmentTxId == HdfsServerConstants.INVALID_TXID || // on format this is no-op
-      minTxIdToKeep <= curSegmentTxId :
-      "cannot purge logs older than txid " + minTxIdToKeep +
-      " when current segment starts at " + curSegmentTxId;
+
+    Preconditions.checkArgument(
+        curSegmentTxId == HdfsServerConstants.INVALID_TXID || // on format this is no-op
+        minTxIdToKeep <= curSegmentTxId,
+        "cannot purge logs older than txid " + minTxIdToKeep +
+        " when current segment starts at " + curSegmentTxId);
     if (minTxIdToKeep == 0) {
       return;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
index ad03a85636d7e..115e9485fa0a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -37,7 +37,7 @@
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 class FSEditLogAsync extends FSEditLog implements Runnable {
   static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index aea928d4cf63a..efbc3b20c6032 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -118,7 +118,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.log.LogThrottlingHelper.LogAction;
 
@@ -912,6 +912,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
       fsNamesys.getFSImage().updateStorageVersion();
       fsNamesys.getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK,
           NameNodeFile.IMAGE);
+      fsNamesys.setNeedRollbackFsImage(false);
       break;
     }
     case OP_ADD_CACHE_DIRECTIVE: {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 3ab0a9facd3df..c8e0cd224e1af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -140,7 +140,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 
 /**
@@ -823,7 +823,7 @@ static class AddOp extends AddCloseOp {
     }
 
     static AddOp getInstance(OpInstanceCache cache) {
-      return (AddOp) cache.get(OP_ADD);
+      return cache.get(OP_ADD);
     }
 
     @Override
@@ -851,7 +851,7 @@ static class CloseOp extends AddCloseOp {
     }
 
     static CloseOp getInstance(OpInstanceCache cache) {
-      return (CloseOp)cache.get(OP_CLOSE);
+      return cache.get(OP_CLOSE);
     }
 
     @Override
@@ -969,7 +969,7 @@ static class AddBlockOp extends FSEditLogOp {
     }
     
     static AddBlockOp getInstance(OpInstanceCache cache) {
-      return (AddBlockOp) cache.get(OP_ADD_BLOCK);
+      return cache.get(OP_ADD_BLOCK);
     }
 
     @Override
@@ -1081,7 +1081,7 @@ static class UpdateBlocksOp extends FSEditLogOp implements BlockListUpdatingOp {
     }
     
     static UpdateBlocksOp getInstance(OpInstanceCache cache) {
-      return (UpdateBlocksOp)cache.get(OP_UPDATE_BLOCKS);
+      return cache.get(OP_UPDATE_BLOCKS);
     }
 
     @Override
@@ -1175,7 +1175,7 @@ static class SetReplicationOp extends FSEditLogOp {
     }
 
     static SetReplicationOp getInstance(OpInstanceCache cache) {
-      return (SetReplicationOp)cache.get(OP_SET_REPLICATION);
+      return cache.get(OP_SET_REPLICATION);
     }
 
     @Override
@@ -1254,7 +1254,7 @@ static class ConcatDeleteOp extends FSEditLogOp {
     }
 
     static ConcatDeleteOp getInstance(OpInstanceCache cache) {
-      return (ConcatDeleteOp)cache.get(OP_CONCAT_DELETE);
+      return cache.get(OP_CONCAT_DELETE);
     }
 
     @Override
@@ -1412,7 +1412,7 @@ static class RenameOldOp extends FSEditLogOp {
     }
 
     static RenameOldOp getInstance(OpInstanceCache cache) {
-      return (RenameOldOp)cache.get(OP_RENAME_OLD);
+      return cache.get(OP_RENAME_OLD);
     }
 
     @Override
@@ -1524,7 +1524,7 @@ static class DeleteOp extends FSEditLogOp {
     }
 
     static DeleteOp getInstance(OpInstanceCache cache) {
-      return (DeleteOp)cache.get(OP_DELETE);
+      return cache.get(OP_DELETE);
     }
 
     @Override
@@ -1625,7 +1625,7 @@ static class MkdirOp extends FSEditLogOp {
     }
     
     static MkdirOp getInstance(OpInstanceCache cache) {
-      return (MkdirOp)cache.get(OP_MKDIR);
+      return cache.get(OP_MKDIR);
     }
 
     @Override
@@ -1798,7 +1798,7 @@ static class SetGenstampV1Op extends FSEditLogOp {
     }
 
     static SetGenstampV1Op getInstance(OpInstanceCache cache) {
-      return (SetGenstampV1Op)cache.get(OP_SET_GENSTAMP_V1);
+      return cache.get(OP_SET_GENSTAMP_V1);
     }
 
     @Override
@@ -1864,7 +1864,7 @@ static class SetGenstampV2Op extends FSEditLogOp {
     }
 
     static SetGenstampV2Op getInstance(OpInstanceCache cache) {
-      return (SetGenstampV2Op)cache.get(OP_SET_GENSTAMP_V2);
+      return cache.get(OP_SET_GENSTAMP_V2);
     }
 
     @Override
@@ -1922,7 +1922,7 @@ static class AllocateBlockIdOp extends FSEditLogOp {
     }
 
     static AllocateBlockIdOp getInstance(OpInstanceCache cache) {
-      return (AllocateBlockIdOp)cache.get(OP_ALLOCATE_BLOCK_ID);
+      return cache.get(OP_ALLOCATE_BLOCK_ID);
     }
 
     @Override
@@ -1981,7 +1981,7 @@ static class SetPermissionsOp extends FSEditLogOp {
     }
 
     static SetPermissionsOp getInstance(OpInstanceCache cache) {
-      return (SetPermissionsOp)cache.get(OP_SET_PERMISSIONS);
+      return cache.get(OP_SET_PERMISSIONS);
     }
 
     @Override
@@ -2054,7 +2054,7 @@ static class SetOwnerOp extends FSEditLogOp {
     }
 
     static SetOwnerOp getInstance(OpInstanceCache cache) {
-      return (SetOwnerOp)cache.get(OP_SET_OWNER);
+      return cache.get(OP_SET_OWNER);
     }
 
     @Override
@@ -2141,7 +2141,7 @@ static class SetNSQuotaOp extends FSEditLogOp {
     }
 
     static SetNSQuotaOp getInstance(OpInstanceCache cache) {
-      return (SetNSQuotaOp)cache.get(OP_SET_NS_QUOTA);
+      return cache.get(OP_SET_NS_QUOTA);
     }
 
     @Override
@@ -2199,7 +2199,7 @@ static class ClearNSQuotaOp extends FSEditLogOp {
     }
 
     static ClearNSQuotaOp getInstance(OpInstanceCache cache) {
-      return (ClearNSQuotaOp)cache.get(OP_CLEAR_NS_QUOTA);
+      return cache.get(OP_CLEAR_NS_QUOTA);
     }
 
     @Override
@@ -2253,7 +2253,7 @@ static class SetQuotaOp extends FSEditLogOp {
     }
 
     static SetQuotaOp getInstance(OpInstanceCache cache) {
-      return (SetQuotaOp)cache.get(OP_SET_QUOTA);
+      return cache.get(OP_SET_QUOTA);
     }
 
     @Override
@@ -2338,7 +2338,7 @@ static class SetQuotaByStorageTypeOp extends FSEditLogOp {
     }
 
     static SetQuotaByStorageTypeOp getInstance(OpInstanceCache cache) {
-      return (SetQuotaByStorageTypeOp)cache.get(OP_SET_QUOTA_BY_STORAGETYPE);
+      return cache.get(OP_SET_QUOTA_BY_STORAGETYPE);
     }
 
     @Override
@@ -2421,7 +2421,7 @@ static class TimesOp extends FSEditLogOp {
     }
 
     static TimesOp getInstance(OpInstanceCache cache) {
-      return (TimesOp)cache.get(OP_TIMES);
+      return cache.get(OP_TIMES);
     }
 
     @Override
@@ -2530,7 +2530,7 @@ static class SymlinkOp extends FSEditLogOp {
     }
 
     static SymlinkOp getInstance(OpInstanceCache cache) {
-      return (SymlinkOp)cache.get(OP_SYMLINK);
+      return cache.get(OP_SYMLINK);
     }
 
     @Override
@@ -2689,7 +2689,7 @@ static class RenameOp extends FSEditLogOp {
     }
 
     static RenameOp getInstance(OpInstanceCache cache) {
-      return (RenameOp)cache.get(OP_RENAME);
+      return cache.get(OP_RENAME);
     }
 
     @Override
@@ -2854,7 +2854,7 @@ static class TruncateOp extends FSEditLogOp {
     }
 
     static TruncateOp getInstance(OpInstanceCache cache) {
-      return (TruncateOp)cache.get(OP_TRUNCATE);
+      return cache.get(OP_TRUNCATE);
     }
 
     @Override
@@ -2988,7 +2988,7 @@ static class ReassignLeaseOp extends FSEditLogOp {
     }
 
     static ReassignLeaseOp getInstance(OpInstanceCache cache) {
-      return (ReassignLeaseOp)cache.get(OP_REASSIGN_LEASE);
+      return cache.get(OP_REASSIGN_LEASE);
     }
 
     @Override
@@ -3070,7 +3070,7 @@ static class GetDelegationTokenOp extends FSEditLogOp {
     }
 
     static GetDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (GetDelegationTokenOp)cache.get(OP_GET_DELEGATION_TOKEN);
+      return cache.get(OP_GET_DELEGATION_TOKEN);
     }
 
     @Override
@@ -3149,7 +3149,7 @@ static class RenewDelegationTokenOp extends FSEditLogOp {
     }
 
     static RenewDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (RenewDelegationTokenOp)cache.get(OP_RENEW_DELEGATION_TOKEN);
+      return cache.get(OP_RENEW_DELEGATION_TOKEN);
     }
 
     @Override
@@ -3227,7 +3227,7 @@ static class CancelDelegationTokenOp extends FSEditLogOp {
     }
 
     static CancelDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (CancelDelegationTokenOp)cache.get(OP_CANCEL_DELEGATION_TOKEN);
+      return cache.get(OP_CANCEL_DELEGATION_TOKEN);
     }
 
     @Override
@@ -3286,7 +3286,7 @@ static class UpdateMasterKeyOp extends FSEditLogOp {
     }
 
     static UpdateMasterKeyOp getInstance(OpInstanceCache cache) {
-      return (UpdateMasterKeyOp)cache.get(OP_UPDATE_MASTER_KEY);
+      return cache.get(OP_UPDATE_MASTER_KEY);
     }
 
     @Override
@@ -3345,7 +3345,7 @@ private LogSegmentOp(FSEditLogOpCodes code) {
 
     static LogSegmentOp getInstance(OpInstanceCache cache,
         FSEditLogOpCodes code) {
-      return (LogSegmentOp)cache.get(code);
+      return cache.get(code);
     }
 
     @Override
@@ -3404,7 +3404,7 @@ static class InvalidOp extends FSEditLogOp {
     }
 
     static InvalidOp getInstance(OpInstanceCache cache) {
-      return (InvalidOp)cache.get(OP_INVALID);
+      return cache.get(OP_INVALID);
     }
 
     @Override
@@ -3457,7 +3457,7 @@ public CreateSnapshotOp() {
     }
     
     static CreateSnapshotOp getInstance(OpInstanceCache cache) {
-      return (CreateSnapshotOp)cache.get(OP_CREATE_SNAPSHOT);
+      return cache.get(OP_CREATE_SNAPSHOT);
     }
 
     @Override
@@ -3562,7 +3562,7 @@ static class DeleteSnapshotOp extends FSEditLogOp {
     }
     
     static DeleteSnapshotOp getInstance(OpInstanceCache cache) {
-      return (DeleteSnapshotOp)cache.get(OP_DELETE_SNAPSHOT);
+      return cache.get(OP_DELETE_SNAPSHOT);
     }
 
     @Override
@@ -3669,7 +3669,7 @@ static class RenameSnapshotOp extends FSEditLogOp {
     }
     
     static RenameSnapshotOp getInstance(OpInstanceCache cache) {
-      return (RenameSnapshotOp) cache.get(OP_RENAME_SNAPSHOT);
+      return cache.get(OP_RENAME_SNAPSHOT);
     }
 
     @Override
@@ -3788,7 +3788,7 @@ public AllowSnapshotOp(String snapRoot) {
     }
 
     static AllowSnapshotOp getInstance(OpInstanceCache cache) {
-      return (AllowSnapshotOp) cache.get(OP_ALLOW_SNAPSHOT);
+      return cache.get(OP_ALLOW_SNAPSHOT);
     }
 
     @Override
@@ -3847,7 +3847,7 @@ public DisallowSnapshotOp(String snapRoot) {
     }
 
     static DisallowSnapshotOp getInstance(OpInstanceCache cache) {
-      return (DisallowSnapshotOp) cache.get(OP_DISALLOW_SNAPSHOT);
+      return cache.get(OP_DISALLOW_SNAPSHOT);
     }
 
     void resetSubFields() {
@@ -3901,7 +3901,7 @@ public AddCacheDirectiveInfoOp() {
     }
 
     static AddCacheDirectiveInfoOp getInstance(OpInstanceCache cache) {
-      return (AddCacheDirectiveInfoOp) cache.get(OP_ADD_CACHE_DIRECTIVE);
+      return cache.get(OP_ADD_CACHE_DIRECTIVE);
     }
 
     @Override
@@ -3971,7 +3971,7 @@ public ModifyCacheDirectiveInfoOp() {
     }
 
     static ModifyCacheDirectiveInfoOp getInstance(OpInstanceCache cache) {
-      return (ModifyCacheDirectiveInfoOp) cache.get(OP_MODIFY_CACHE_DIRECTIVE);
+      return cache.get(OP_MODIFY_CACHE_DIRECTIVE);
     }
 
     @Override
@@ -4047,7 +4047,7 @@ public RemoveCacheDirectiveInfoOp() {
     }
 
     static RemoveCacheDirectiveInfoOp getInstance(OpInstanceCache cache) {
-      return (RemoveCacheDirectiveInfoOp) cache.get(OP_REMOVE_CACHE_DIRECTIVE);
+      return cache.get(OP_REMOVE_CACHE_DIRECTIVE);
     }
 
     @Override
@@ -4104,7 +4104,7 @@ public AddCachePoolOp() {
     }
 
     static AddCachePoolOp getInstance(OpInstanceCache cache) {
-      return (AddCachePoolOp) cache.get(OP_ADD_CACHE_POOL);
+      return cache.get(OP_ADD_CACHE_POOL);
     }
 
     @Override
@@ -4170,7 +4170,7 @@ public ModifyCachePoolOp() {
     }
 
     static ModifyCachePoolOp getInstance(OpInstanceCache cache) {
-      return (ModifyCachePoolOp) cache.get(OP_MODIFY_CACHE_POOL);
+      return cache.get(OP_MODIFY_CACHE_POOL);
     }
 
     @Override
@@ -4243,7 +4243,7 @@ public RemoveCachePoolOp() {
     }
 
     static RemoveCachePoolOp getInstance(OpInstanceCache cache) {
-      return (RemoveCachePoolOp) cache.get(OP_REMOVE_CACHE_POOL);
+      return cache.get(OP_REMOVE_CACHE_POOL);
     }
 
     @Override
@@ -4300,7 +4300,7 @@ static class RemoveXAttrOp extends FSEditLogOp {
     }
     
     static RemoveXAttrOp getInstance(OpInstanceCache cache) {
-      return (RemoveXAttrOp) cache.get(OP_REMOVE_XATTR);
+      return cache.get(OP_REMOVE_XATTR);
     }
 
     @Override
@@ -4353,7 +4353,7 @@ static class SetXAttrOp extends FSEditLogOp {
     }
     
     static SetXAttrOp getInstance(OpInstanceCache cache) {
-      return (SetXAttrOp) cache.get(OP_SET_XATTR);
+      return cache.get(OP_SET_XATTR);
     }
 
     @Override
@@ -4406,7 +4406,7 @@ static class SetAclOp extends FSEditLogOp {
     }
 
     static SetAclOp getInstance(OpInstanceCache cache) {
-      return (SetAclOp) cache.get(OP_SET_ACL);
+      return cache.get(OP_SET_ACL);
     }
 
     @Override
@@ -4507,8 +4507,7 @@ static class AddErasureCodingPolicyOp extends FSEditLogOp {
     }
 
     static AddErasureCodingPolicyOp getInstance(OpInstanceCache cache) {
-      return (AddErasureCodingPolicyOp) cache
-          .get(OP_ADD_ERASURE_CODING_POLICY);
+      return cache.get(OP_ADD_ERASURE_CODING_POLICY);
     }
 
     @Override
@@ -4620,8 +4619,7 @@ static class EnableErasureCodingPolicyOp extends FSEditLogOp {
     }
 
     static EnableErasureCodingPolicyOp getInstance(OpInstanceCache cache) {
-      return (EnableErasureCodingPolicyOp) cache
-          .get(OP_ENABLE_ERASURE_CODING_POLICY);
+      return cache.get(OP_ENABLE_ERASURE_CODING_POLICY);
     }
 
     @Override
@@ -4689,8 +4687,7 @@ static class DisableErasureCodingPolicyOp extends FSEditLogOp {
     }
 
     static DisableErasureCodingPolicyOp getInstance(OpInstanceCache cache) {
-      return (DisableErasureCodingPolicyOp) cache
-          .get(OP_DISABLE_ERASURE_CODING_POLICY);
+      return cache.get(OP_DISABLE_ERASURE_CODING_POLICY);
     }
 
     @Override
@@ -4756,8 +4753,7 @@ static class RemoveErasureCodingPolicyOp extends FSEditLogOp {
     }
 
     static RemoveErasureCodingPolicyOp getInstance(OpInstanceCache cache) {
-      return (RemoveErasureCodingPolicyOp) cache
-          .get(OP_REMOVE_ERASURE_CODING_POLICY);
+      return cache.get(OP_REMOVE_ERASURE_CODING_POLICY);
     }
 
     @Override
@@ -4879,7 +4875,7 @@ static class SetStoragePolicyOp extends FSEditLogOp {
     }
 
     static SetStoragePolicyOp getInstance(OpInstanceCache cache) {
-      return (SetStoragePolicyOp) cache.get(OP_SET_STORAGE_POLICY);
+      return cache.get(OP_SET_STORAGE_POLICY);
     }
 
     @Override
@@ -4946,7 +4942,7 @@ static class RollingUpgradeStartOp extends RollingUpgradeOp {
     }
 
     static RollingUpgradeStartOp getInstance(OpInstanceCache cache) {
-      return (RollingUpgradeStartOp) cache.get(OP_ROLLING_UPGRADE_START);
+      return cache.get(OP_ROLLING_UPGRADE_START);
     }
   }
 
@@ -4956,7 +4952,7 @@ static class RollingUpgradeFinalizeOp extends RollingUpgradeOp {
     }
 
     static RollingUpgradeFinalizeOp getInstance(OpInstanceCache cache) {
-      return (RollingUpgradeFinalizeOp) cache.get(OP_ROLLING_UPGRADE_FINALIZE);
+      return cache.get(OP_ROLLING_UPGRADE_FINALIZE);
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index f7749ce7e231b..c149d6021bbb8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -77,7 +77,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * FSImage handles checkpointing and logging of the namespace edits.
@@ -582,12 +582,12 @@ void doImportCheckpoint(FSNamesystem target) throws IOException {
 
     if (checkpointDirs == null || checkpointDirs.isEmpty()) {
       throw new IOException("Cannot import image from a checkpoint. "
-                            + "\"dfs.namenode.checkpoint.dir\" is not set." );
+                            + "\"dfs.namenode.checkpoint.dir\" is not set.");
     }
     
     if (checkpointEditsDirs == null || checkpointEditsDirs.isEmpty()) {
       throw new IOException("Cannot import image from a checkpoint. "
-                            + "\"dfs.namenode.checkpoint.dir\" is not set." );
+                            + "\"dfs.namenode.checkpoint.edits.dir\" is not set.");
     }
 
     FSImage realImage = target.getFSImage();
@@ -649,7 +649,7 @@ void openEditLogForWrite(int layoutVersion) throws IOException {
    */
   void reloadFromImageFile(File file, FSNamesystem target) throws IOException {
     target.clear();
-    LOG.debug("Reloading namespace from " + file);
+    LOG.debug("Reloading namespace from {}.", file);
     loadFSImage(file, target, null, false);
   }
 
@@ -728,7 +728,7 @@ LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) {
     }
  
     for (EditLogInputStream l : editStreams) {
-      LOG.debug("Planning to load edit log stream: " + l);
+      LOG.debug("Planning to load edit log stream: {}.", l);
     }
     if (!editStreams.iterator().hasNext()) {
       LOG.info("No edit log streams selected.");
@@ -892,8 +892,10 @@ public long loadEdits(Iterable<EditLogInputStream> editStreams,
       FSNamesystem target, long maxTxnsToRead,
       StartupOption startOpt, MetaRecoveryContext recovery)
       throws IOException {
-    LOG.debug("About to load edits:\n  " + Joiner.on("\n  ").join(editStreams));
-    
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("About to load edits:\n  {}.", Joiner.on("\n  ").join(editStreams));
+    }
+
     long prevLastAppliedTxId = lastAppliedTxId;
     long remainingReadTxns = maxTxnsToRead;
     try {    
@@ -1348,10 +1350,10 @@ private void renameImageFileInDir(StorageDirectory sd, NameNodeFile fromNnf,
     final File fromFile = NNStorage.getStorageFile(sd, fromNnf, txid);
     final File toFile = NNStorage.getStorageFile(sd, toNnf, txid);
     // renameTo fails on Windows if the destination file already exists.
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("renaming  " + fromFile.getAbsolutePath() 
-                + " to " + toFile.getAbsolutePath());
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("renaming  {} to {}", fromFile.getAbsoluteFile(), toFile.getAbsolutePath());
     }
+
     if (!fromFile.renameTo(toFile)) {
       if (!toFile.delete() || !fromFile.renameTo(toFile)) {
         throw new IOException("renaming  " + fromFile.getAbsolutePath() + " to "  + 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 2ce2c645e1511..7e679296e25c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -76,7 +76,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Contains inner classes for reading or writing the on-disk format for
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 0a69c99cab810..3f0c9faa97c9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -72,7 +72,7 @@
 import org.apache.hadoop.hdfs.util.EnumCounters;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 3d75cebf729d3..404f2c73ad3a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -53,7 +53,7 @@
 import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Static utility functions for serializing various pieces of data in the correct
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7ad90ac1a93ca..5b8bc0ac61cfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -24,12 +24,16 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_KEY;
@@ -99,6 +103,7 @@
 import org.apache.commons.text.CaseUtils;
 import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
@@ -343,7 +348,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
@@ -397,6 +402,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics =
       registry.newRatesWithAggregation("detailedLockHoldTimeMetrics");
 
+  private final String contextFieldSeparator;
+
   boolean isAuditEnabled() {
     return (!isDefaultAuditLogger || auditLog.isInfoEnabled())
         && !auditLoggers.isEmpty();
@@ -410,8 +417,8 @@ void logAuditEvent(boolean succeeded, String cmd, String src)
   private void logAuditEvent(boolean succeeded, String cmd, String src,
       String dst, FileStatus stat) throws IOException {
     if (isAuditEnabled() && isExternalInvocation()) {
-      logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(),
-                    cmd, src, dst, stat);
+      logAuditEvent(succeeded, NameNode.getRemoteUser(), Server.getRemoteIp(),
+          cmd, src, dst, stat);
     }
   }
 
@@ -442,6 +449,9 @@ private void logAuditEvent(boolean succeeded,
     for (AuditLogger logger : auditLoggers) {
       if (logger instanceof HdfsAuditLogger) {
         HdfsAuditLogger hdfsLogger = (HdfsAuditLogger) logger;
+        if (auditLogWithRemotePort) {
+          appendClientPortToCallerContextIfAbsent();
+        }
         hdfsLogger.logAuditEvent(succeeded, ugiStr, addr, cmd, src, dst,
             status, CallerContext.getCurrent(), ugi, dtSecretManager);
       } else {
@@ -450,6 +460,24 @@ private void logAuditEvent(boolean succeeded,
     }
   }
 
+  private void appendClientPortToCallerContextIfAbsent() {
+    final CallerContext ctx = CallerContext.getCurrent();
+    if (isClientPortInfoAbsent(ctx)) {
+      String origContext = ctx == null ? null : ctx.getContext();
+      byte[] origSignature = ctx == null ? null : ctx.getSignature();
+      CallerContext.setCurrent(
+          new CallerContext.Builder(origContext, contextFieldSeparator)
+              .append(CallerContext.CLIENT_PORT_STR, String.valueOf(Server.getRemotePort()))
+              .setSignature(origSignature)
+              .build());
+    }
+  }
+
+  private boolean isClientPortInfoAbsent(CallerContext ctx){
+    return ctx == null || ctx.getContext() == null
+        || !ctx.getContext().contains(CallerContext.CLIENT_PORT_STR);
+  }
+
   /**
    * Logger for audit events, noting successful FSNamesystem operations. Emits
    * to FSNamesystem.audit at INFO. Each event causes a set of tab-separated
@@ -475,7 +503,12 @@ private void logAuditEvent(boolean succeeded,
   private final boolean standbyShouldCheckpoint;
   private final boolean isSnapshotTrashRootEnabled;
   private final int snapshotDiffReportLimit;
-  private final int blockDeletionIncrement;
+
+  /**
+   * Whether enable checkOperation when call getBlocks.
+   * It is enabled  by default.
+   */
+  private final boolean isGetBlocksCheckOperationEnabled;
 
   /** Interval between each check of lease to release. */
   private final long leaseRecheckIntervalMs;
@@ -501,6 +534,7 @@ private void logAuditEvent(boolean succeeded,
   // underlying logger is disabled, and avoid some unnecessary work.
   private final boolean isDefaultAuditLogger;
   private final List<AuditLogger> auditLoggers;
+  private final boolean auditLogWithRemotePort;
 
   /** The namespace tree. */
   FSDirectory dir;
@@ -833,6 +867,12 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
       LOG.info("Enabling async auditlog");
       enableAsyncAuditLog(conf);
     }
+    auditLogWithRemotePort =
+        conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY,
+            DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT);
+    this.contextFieldSeparator =
+        conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY,
+            HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT);
     fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics);
     cond = fsLock.newWriteLockCondition();
     cpLock = new ReentrantLock();
@@ -979,6 +1019,10 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
       this.leaseRecheckIntervalMs = conf.getLong(
           DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY,
           DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT);
+      Preconditions.checkArgument(
+              leaseRecheckIntervalMs > 0,
+              DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY +
+                      " must be greater than zero");
       this.maxLockHoldToReleaseLeaseMs = conf.getLong(
           DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY,
           DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT);
@@ -1021,12 +1065,10 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
       this.allowOwnerSetQuota = conf.getBoolean(
           DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_KEY,
           DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_DEFAULT);
-      this.blockDeletionIncrement = conf.getInt(
-          DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY,
-          DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT);
-      Preconditions.checkArgument(blockDeletionIncrement > 0,
-          DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY +
-              " must be a positive integer.");
+      this.isGetBlocksCheckOperationEnabled = conf.getBoolean(
+          DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY,
+          DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT);
+
     } catch(IOException e) {
       LOG.error(getClass().getSimpleName() + " initialization failed.", e);
       close();
@@ -1366,8 +1408,8 @@ void startActiveServices() throws IOException {
         }
 
         if (LOG.isDebugEnabled()) {
-          LOG.debug("NameNode metadata after re-processing " +
-              "replication and invalidation queues during failover:\n" +
+          LOG.debug("NameNode metadata after re-processing {}"
+              + "replication and invalidation queues during failover:\n",
               metaSaveAsString());
         }
 
@@ -1750,6 +1792,7 @@ public void readUnlock() {
     this.fsLock.readUnlock();
   }
 
+  @Override
   public void readUnlock(String opName) {
     this.fsLock.readUnlock(opName);
   }
@@ -1774,6 +1817,7 @@ public void writeUnlock() {
     this.fsLock.writeUnlock();
   }
 
+  @Override
   public void writeUnlock(String opName) {
     this.fsLock.writeUnlock(opName);
   }
@@ -1899,10 +1943,13 @@ public boolean isInStandbyState() {
    */
   public BlocksWithLocations getBlocks(DatanodeID datanode, long size, long
       minimumBlockSize, long timeInterval) throws IOException {
-    checkOperation(OperationCategory.READ);
+    OperationCategory checkOp =
+        isGetBlocksCheckOperationEnabled ? OperationCategory.READ :
+            OperationCategory.UNCHECKED;
+    checkOperation(checkOp);
     readLock();
     try {
-      checkOperation(OperationCategory.READ);
+      checkOperation(checkOp);
       return getBlockManager().getBlocksWithLocations(datanode, size,
           minimumBlockSize, timeInterval);
     } finally {
@@ -2156,14 +2203,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String srcArg,
               }
             }
           }
-        } else if (haEnabled && haContext != null &&
-            haContext.getState().getServiceState() == OBSERVER) {
-          for (LocatedBlock b : res.blocks.getLocatedBlocks()) {
-            if (b.getLocations() == null || b.getLocations().length == 0) {
-              throw new ObserverRetryOnActiveException("Zero blocklocations "
-                  + "for " + srcArg);
-            }
-          }
+        } else if (isObserver()) {
+          checkBlockLocationsWhenObserver(res.blocks, srcArg);
         }
       } finally {
         readUnlock(operationName, getLockReportInfoSupplier(srcArg));
@@ -2334,8 +2375,8 @@ boolean truncate(String src, long newLength, String clientName,
       }
       getEditLog().logSync();
       if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
-        removeBlocks(toRemoveBlocks);
-        toRemoveBlocks.clear();
+        blockManager.addBLocksToMarkedDeleteQueue(
+            toRemoveBlocks.getToDeleteList());
       }
       logAuditEvent(true, operationName, src, null, status);
     } catch (AccessControlException e) {
@@ -2629,8 +2670,8 @@ CryptoProtocolVersion chooseProtocolVersion(
 
     for (CryptoProtocolVersion c : supportedVersions) {
       if (c.equals(CryptoProtocolVersion.UNKNOWN)) {
-        LOG.debug("Ignoring unknown CryptoProtocolVersion provided by " +
-            "client: {}", c.getUnknownValue());
+        LOG.debug("Ignoring unknown CryptoProtocolVersion provided by client: {}",
+            c.getUnknownValue());
         continue;
       }
       if (c.equals(required)) {
@@ -2782,8 +2823,8 @@ private HdfsFileStatus startFileInt(String src,
       if (!skipSync) {
         getEditLog().logSync();
         if (toRemoveBlocks != null) {
-          removeBlocks(toRemoveBlocks);
-          toRemoveBlocks.clear();
+          blockManager.addBLocksToMarkedDeleteQueue(
+              toRemoveBlocks.getToDeleteList());
         }
       }
     }
@@ -2941,8 +2982,7 @@ LastBlockWithStatus appendFile(String srcArg, String holder,
       requireEffectiveLayoutVersionForFeature(Feature.APPEND_NEW_BLOCK);
     }
 
-    NameNode.stateChangeLog.debug(
-        "DIR* NameSystem.appendFile: src={}, holder={}, clientMachine={}",
+    NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: src={}, holder={}, clientMachine={}",
         srcArg, holder, clientMachine);
     try {
       boolean skipSync = false;
@@ -2999,8 +3039,8 @@ LocatedBlock getAdditionalBlock(
       DatanodeInfo[] excludedNodes, String[] favoredNodes,
       EnumSet<AddBlockFlag> flags) throws IOException {
     final String operationName = "getAdditionalBlock";
-    NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: {}  inodeId {}" +
-        " for {}", src, fileId, clientName);
+    NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: {}  inodeId {} for {}",
+        src, fileId, clientName);
 
     LocatedBlock[] onRetryBlock = new LocatedBlock[1];
     FSDirWriteFileOp.ValidateAddBlockResult r;
@@ -3102,8 +3142,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId,
    */
   void abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
       throws IOException {
-    NameNode.stateChangeLog.debug(
-        "BLOCK* NameSystem.abandonBlock: {} of file {}", b, src);
+    NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} of file {}", b, src);
     checkOperation(OperationCategory.WRITE);
     final FSPermissionChecker pc = getPermissionChecker();
     FSPermissionChecker.setOperationType(null);
@@ -3112,8 +3151,8 @@ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot abandon block " + b + " for file" + src);
       FSDirWriteFileOp.abandonBlock(dir, pc, b, fileId, src, holder);
-      NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} is " +
-          "removed from pendingCreates", b);
+      NameNode.stateChangeLog.debug(
+          "BLOCK* NameSystem.abandonBlock: {} is removed from pendingCreates", b);
     } finally {
       writeUnlock("abandonBlock");
     }
@@ -3306,8 +3345,8 @@ void renameTo(final String src, final String dst,
     assert res != null;
     BlocksMapUpdateInfo collectedBlocks = res.collectedBlocks;
     if (!collectedBlocks.getToDeleteList().isEmpty()) {
-      removeBlocks(collectedBlocks);
-      collectedBlocks.clear();
+      blockManager.addBLocksToMarkedDeleteQueue(
+          collectedBlocks.getToDeleteList());
     }
 
     logAuditEvent(true, operationName + " (options=" +
@@ -3346,7 +3385,8 @@ boolean delete(String src, boolean recursive, boolean logRetryCache)
     getEditLog().logSync();
     logAuditEvent(ret, operationName, src);
     if (toRemovedBlocks != null) {
-      removeBlocks(toRemovedBlocks); // Incremental deletion of blocks
+      blockManager.addBLocksToMarkedDeleteQueue(
+          toRemovedBlocks.getToDeleteList());
     }
     return ret;
   }
@@ -3356,30 +3396,6 @@ FSPermissionChecker getPermissionChecker()
     return dir.getPermissionChecker();
   }
 
-  /**
-   * From the given list, incrementally remove the blocks from blockManager
-   * Writelock is dropped and reacquired every blockDeletionIncrement to
-   * ensure that other waiters on the lock can get in. See HDFS-2938
-   * 
-   * @param blocks
-   *          An instance of {@link BlocksMapUpdateInfo} which contains a list
-   *          of blocks that need to be removed from blocksMap
-   */
-  void removeBlocks(BlocksMapUpdateInfo blocks) {
-    List<BlockInfo> toDeleteList = blocks.getToDeleteList();
-    Iterator<BlockInfo> iter = toDeleteList.iterator();
-    while (iter.hasNext()) {
-      writeLock();
-      try {
-        for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) {
-          blockManager.removeBlock(iter.next());
-        }
-      } finally {
-        writeUnlock("removeBlocks");
-      }
-    }
-  }
-  
   /**
    * Remove leases and inodes related to a given path
    * @param removedUCFiles INodes whose leases need to be released
@@ -3449,6 +3465,10 @@ HdfsFileStatus getFileInfo(final String src, boolean resolveLink,
       logAuditEvent(false, operationName, src);
       throw e;
     }
+    if (needLocation && isObserver() && stat instanceof HdfsLocatedFileStatus) {
+      LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks();
+      checkBlockLocationsWhenObserver(lbs, src);
+    }
     logAuditEvent(true, operationName, src);
     return stat;
   }
@@ -3993,9 +4013,10 @@ void commitBlockSynchronization(ExtendedBlock oldBlock,
       if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) &&
           iFile.getLastBlock().isComplete()) {
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Unexpected block (={}) since the file (={}) is not "
-                + "under construction", oldBlock, iFile.getLocalName());
+          LOG.debug("Unexpected block (={}) since the file (={}) is not under construction",
+              oldBlock, iFile.getLocalName());
         }
+
         return;
       }
 
@@ -4153,6 +4174,14 @@ DirectoryListing getListing(String src, byte[] startAfter,
       logAuditEvent(false, operationName, src);
       throw e;
     }
+    if (needLocation && isObserver()) {
+      for (HdfsFileStatus fs : dl.getPartialListing()) {
+        if (fs instanceof HdfsLocatedFileStatus) {
+          LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks();
+          checkBlockLocationsWhenObserver(lbs, fs.toString());
+        }
+      }
+    }
     logAuditEvent(true, operationName, src);
     return dl;
   }
@@ -4370,11 +4399,9 @@ HeartbeatResponse handleHeartbeat(DatanodeRegistration nodeReg,
     readLock();
     try {
       //get datanode commands
-      final int maxTransfer = blockManager.getMaxReplicationStreams()
-          - xmitsInProgress;
       DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat(
           nodeReg, reports, getBlockPoolId(), cacheCapacity, cacheUsed,
-          xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary,
+          xceiverCount, xmitsInProgress, failedVolumes, volumeFailureSummary,
           slowPeers, slowDisks);
       long blockReportLeaseId = 0;
       if (requestFullBlockReportLease) {
@@ -4386,8 +4413,11 @@ nodeReg, reports, getBlockPoolId(), cacheCapacity, cacheUsed,
           haContext.getState().getServiceState(),
           getFSImage().getCorrectLastAppliedOrWrittenTxId());
 
+      Set<String> slownodes = DatanodeManager.getSlowNodesUuidSet();
+      boolean isSlownode = slownodes.contains(nodeReg.getDatanodeUuid());
+
       return new HeartbeatResponse(cmds, haState, rollingUpgradeInfo,
-          blockReportLeaseId);
+          blockReportLeaseId, isSlownode);
     } finally {
       readUnlock("handleHeartbeat");
     }
@@ -4454,8 +4484,8 @@ private void closeFile(String path, INodeFile file) {
     assert hasWriteLock();
     // file is closed
     getEditLog().logCloseFile(path, file);
-    NameNode.stateChangeLog.debug("closeFile: {} with {} blocks is persisted" +
-        " to the file system", path, file.getBlocks().length);
+    NameNode.stateChangeLog.debug("closeFile: {} with {} blocks is persisted to the file system",
+        path, file.getBlocks().length);
   }
 
   /**
@@ -4585,7 +4615,8 @@ private void clearCorruptLazyPersistFiles()
                   INodesInPath.fromINode((INodeFile) bc), false);
           changed |= toRemoveBlocks != null;
           if (toRemoveBlocks != null) {
-            removeBlocks(toRemoveBlocks); // Incremental deletion of blocks
+            blockManager.addBLocksToMarkedDeleteQueue(
+                toRemoveBlocks.getToDeleteList());
           }
         }
       } finally {
@@ -4841,6 +4872,12 @@ public long getCurrentTokensCount() {
         dtSecretManager.getCurrentTokensSize() : -1;
   }
 
+  @Override
+  @Metric({"PendingSPSPaths", "The number of paths to be processed by storage policy satisfier"})
+  public int getPendingSPSPaths() {
+    return blockManager.getPendingSPSPaths();
+  }
+
   /**
    * Returns the length of the wait Queue for the FSNameSystemLock.
    *
@@ -4880,6 +4917,32 @@ int getNumberOfDatanodes(DatanodeReportType type) {
     }
   }
 
+  DatanodeInfo[] slowDataNodesReport() throws IOException {
+    String operationName = "slowDataNodesReport";
+    DatanodeInfo[] datanodeInfos;
+    checkOperation(OperationCategory.UNCHECKED);
+    readLock();
+    try {
+      checkOperation(OperationCategory.UNCHECKED);
+      final DatanodeManager dm = getBlockManager().getDatanodeManager();
+      final List<DatanodeDescriptor> results = dm.getAllSlowDataNodes();
+      datanodeInfos = getDatanodeInfoFromDescriptors(results);
+    } finally {
+      readUnlock(operationName, getLockReportInfoSupplier(null));
+    }
+    logAuditEvent(true, operationName, null);
+    return datanodeInfos;
+  }
+
+  private DatanodeInfo[] getDatanodeInfoFromDescriptors(List<DatanodeDescriptor> results) {
+    DatanodeInfo[] datanodeInfos = new DatanodeInfo[results.size()];
+    for (int i = 0; i < datanodeInfos.length; i++) {
+      datanodeInfos[i] = new DatanodeInfoBuilder().setFrom(results.get(i)).build();
+      datanodeInfos[i].setNumBlocks(results.get(i).numBlocks());
+    }
+    return datanodeInfos;
+  }
+
   DatanodeInfo[] datanodeReport(final DatanodeReportType type)
       throws IOException {
     String operationName = "datanodeReport";
@@ -4891,12 +4954,7 @@ DatanodeInfo[] datanodeReport(final DatanodeReportType type)
       checkOperation(OperationCategory.UNCHECKED);
       final DatanodeManager dm = getBlockManager().getDatanodeManager();      
       final List<DatanodeDescriptor> results = dm.getDatanodeListForReport(type);
-      arr = new DatanodeInfo[results.size()];
-      for (int i=0; i<arr.length; i++) {
-        arr[i] = new DatanodeInfoBuilder().setFrom(results.get(i))
-            .build();
-        arr[i].setNumBlocks(results.get(i).numBlocks());
-      }
+      arr = getDatanodeInfoFromDescriptors(results);
     } finally {
       readUnlock(operationName, getLockReportInfoSupplier(null));
     }
@@ -6055,9 +6113,7 @@ Collection<CorruptFileBlockInfo> listCorruptFileBlocks(String path,
       if (cookieTab[0] == null) {
         cookieTab[0] = String.valueOf(getIntCookie(cookieTab[0]));
       }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("there are no corrupt file blocks.");
-      }
+      LOG.debug("there are no corrupt file blocks.");
       return corruptFiles;
     }
 
@@ -7296,7 +7352,8 @@ void deleteSnapshot(String snapshotRoot, String snapshotName,
     // Breaking the pattern as removing blocks have to happen outside of the
     // global lock
     if (blocksToBeDeleted != null) {
-      removeBlocks(blocksToBeDeleted);
+      blockManager.addBLocksToMarkedDeleteQueue(
+          blocksToBeDeleted.getToDeleteList());
     }
     logAuditEvent(true, operationName, rootPath, null, null);
   }
@@ -7322,7 +7379,8 @@ public void gcDeletedSnapshot(String snapshotRoot, String snapshotName)
     } finally {
       writeUnlock(operationName, getLockReportInfoSupplier(rootPath));
     }
-    removeBlocks(blocksToBeDeleted);
+    blockManager.addBLocksToMarkedDeleteQueue(
+        blocksToBeDeleted.getToDeleteList());
   }
 
   /**
@@ -8363,7 +8421,7 @@ public ECTopologyVerifierResult getECTopologyResultForPolicies(
             getBlockManager().getDatanodeManager().getNumOfDataNodes();
         int numOfRacks =
             getBlockManager().getDatanodeManager().getNetworkTopology()
-                .getNumOfRacks();
+                .getNumOfNonEmptyRacks();
         result = ECTopologyVerifier
             .getECTopologyVerifierResult(numOfRacks, numOfDataNodes, policies);
       }
@@ -8574,7 +8632,7 @@ void checkAccess(String src, FsAction mode) throws IOException {
         src = iip.getPath();
         INode inode = iip.getLastINode();
         if (inode == null) {
-          throw new FileNotFoundException("Path not found");
+          throw new FileNotFoundException("Path not found: " + src);
         }
         if (isPermissionEnabled) {
           dir.checkPathAccess(pc, iip, mode);
@@ -8764,18 +8822,18 @@ public void logAuditEvent(boolean succeeded, String userName,
             callerContext != null &&
             callerContext.isContextValid()) {
           sb.append("\t").append("callerContext=");
-          if (callerContext.getContext().length() > callerContextMaxLen) {
-            sb.append(callerContext.getContext().substring(0,
-                callerContextMaxLen));
+          String context = escapeJava(callerContext.getContext());
+          if (context.length() > callerContextMaxLen) {
+            sb.append(context, 0, callerContextMaxLen);
           } else {
-            sb.append(callerContext.getContext());
+            sb.append(context);
           }
           if (callerContext.getSignature() != null &&
               callerContext.getSignature().length > 0 &&
               callerContext.getSignature().length <= callerSignatureMaxLen) {
             sb.append(":")
-                .append(new String(callerContext.getSignature(),
-                CallerContext.SIGNATURE_ENCODING));
+                .append(escapeJava(new String(callerContext.getSignature(),
+                CallerContext.SIGNATURE_ENCODING)));
           }
         }
         logAuditMessage(sb.toString());
@@ -8908,7 +8966,7 @@ private ECTopologyVerifierResult getEcTopologyVerifierResultForEnabledPolicies()
     int numOfDataNodes =
         getBlockManager().getDatanodeManager().getNumOfDataNodes();
     int numOfRacks = getBlockManager().getDatanodeManager().getNetworkTopology()
-        .getNumOfRacks();
+        .getNumOfNonEmptyRacks();
     ErasureCodingPolicy[] enabledEcPolicies =
         getErasureCodingPolicyManager().getCopyOfEnabledPolicies();
     return ECTopologyVerifier
@@ -8969,5 +9027,23 @@ public void checkErasureCodingSupported(String operationName)
       throw new UnsupportedActionException(operationName + " not supported.");
     }
   }
-}
 
+  private boolean isObserver() {
+    return haEnabled && haContext != null && haContext.getState().getServiceState() == OBSERVER;
+  }
+
+  private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src)
+      throws ObserverRetryOnActiveException {
+    if (blocks == null) {
+      return;
+    }
+    List<LocatedBlock> locatedBlockList = blocks.getLocatedBlocks();
+    if (locatedBlockList != null) {
+      for (LocatedBlock b : locatedBlockList) {
+        if (b.getLocations() == null || b.getLocations().length == 0) {
+          throw new ObserverRetryOnActiveException("Zero blocklocations for " + src);
+        }
+      }
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 324cd5d441201..c7430e38cd07b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -23,7 +23,7 @@
 import java.util.List;
 import java.util.Stack;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.ipc.CallerContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -273,31 +273,41 @@ void checkPermission(INodesInPath inodesInPath, boolean doCheckOwner,
     AccessControlEnforcer enforcer = getAccessControlEnforcer();
 
     String opType = operationType.get();
-    if (this.authorizeWithContext && opType != null) {
-      INodeAttributeProvider.AuthorizationContext.Builder builder =
-          new INodeAttributeProvider.AuthorizationContext.Builder();
-      builder.fsOwner(fsOwner).
-          supergroup(supergroup).
-          callerUgi(callerUgi).
-          inodeAttrs(inodeAttrs).
-          inodes(inodes).
-          pathByNameArr(components).
-          snapshotId(snapshotId).
-          path(path).
-          ancestorIndex(ancestorIndex).
-          doCheckOwner(doCheckOwner).
-          ancestorAccess(ancestorAccess).
-          parentAccess(parentAccess).
-          access(access).
-          subAccess(subAccess).
-          ignoreEmptyDir(ignoreEmptyDir).
-          operationName(opType).
-          callerContext(CallerContext.getCurrent());
-      enforcer.checkPermissionWithContext(builder.build());
-    } else {
-      enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
-          inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
-          ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+    try {
+      if (this.authorizeWithContext && opType != null) {
+        INodeAttributeProvider.AuthorizationContext.Builder builder =
+            new INodeAttributeProvider.AuthorizationContext.Builder();
+        builder.fsOwner(fsOwner).
+            supergroup(supergroup).
+            callerUgi(callerUgi).
+            inodeAttrs(inodeAttrs).
+            inodes(inodes).
+            pathByNameArr(components).
+            snapshotId(snapshotId).
+            path(path).
+            ancestorIndex(ancestorIndex).
+            doCheckOwner(doCheckOwner).
+            ancestorAccess(ancestorAccess).
+            parentAccess(parentAccess).
+            access(access).
+            subAccess(subAccess).
+            ignoreEmptyDir(ignoreEmptyDir).
+            operationName(opType).
+            callerContext(CallerContext.getCurrent());
+        enforcer.checkPermissionWithContext(builder.build());
+      } else {
+        enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs,
+            inodes, components, snapshotId, path, ancestorIndex, doCheckOwner,
+            ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
+      }
+    } catch (AccessControlException ace) {
+      Class<?> exceptionClass = ace.getClass();
+      if (exceptionClass.equals(AccessControlException.class)
+          || exceptionClass.equals(TraverseAccessControlException.class)) {
+        throw ace;
+      }
+      // Only form a new ACE for subclasses which come from external enforcers
+      throw new AccessControlException(ace);
     }
 
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
index a90dc27a54fa8..c1d26f40c0826 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java
@@ -34,7 +34,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * FSTreeTraverser traverse directory recursively and process files
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
index fab8f7e43531a..42a5d2d7385bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
@@ -48,7 +48,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
index 3b8c33165395b..3325222267d5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsImageValidation.java
@@ -243,7 +243,7 @@ public void run() {
         loader.load(fsImageFile, false);
       } finally {
         namesystem.getFSDirectory().writeUnlock();
-        namesystem.writeUnlock();
+        namesystem.writeUnlock("loadImage");
       }
     }
     t.cancel();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
index e5c02e81dccfc..059b6531242e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
@@ -48,9 +48,9 @@ public void doGet(HttpServletRequest request, HttpServletResponse response
     @SuppressWarnings("unchecked")
     final Map<String,String[]> pmap = request.getParameterMap();
     final PrintWriter out = response.getWriter();
-    final InetAddress remoteAddress = 
+    final InetAddress remoteAddress =
       InetAddress.getByName(request.getRemoteAddr());
-    final ServletContext context = getServletContext();    
+    final ServletContext context = getServletContext();
     final Configuration conf = NameNodeHttpServer.getConfFromContext(context);
 
     final UserGroupInformation ugi = getUGI(request, conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
index 0a355d0ec3e77..5c0a34ce76582 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import java.net.InetAddress;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileStatus;
@@ -26,6 +24,8 @@
 import org.apache.hadoop.ipc.CallerContext;
 import org.apache.hadoop.security.UserGroupInformation;
 
+import java.net.InetAddress;
+
 /**
  * Extension of {@link AuditLogger}.
  */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index dacde02a8a9ad..102ca72f2b8d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -339,6 +339,16 @@ public boolean isFile() {
     return false;
   }
 
+  /**
+   * Check if this inode itself has a storage policy set.
+   */
+  public boolean isSetStoragePolicy() {
+    if (isSymlink()) {
+      return false;
+    }
+    return getLocalStoragePolicyID() != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
+  }
+
   /** Cast this inode to an {@link INodeFile}.  */
   public INodeFile asFile() {
     throw new IllegalStateException("Current inode is not a file: "
@@ -994,14 +1004,14 @@ public static class ReclaimContext {
 
     /**
      * @param bsps
- *          block storage policy suite to calculate intended storage type
- *          usage
+     *      block storage policy suite to calculate intended storage type
+     *      usage
      * @param collectedBlocks
-*          blocks collected from the descents for further block
-*          deletion/update will be added to the given map.
+     *     blocks collected from the descents for further block
+     *     deletion/update will be added to the given map.
      * @param removedINodes
-*          INodes collected from the descents for further cleaning up of
-     * @param removedUCFiles
+     *     INodes collected from the descents for further cleaning up of
+     * @param removedUCFiles INodes whose leases need to be released
      */
     public ReclaimContext(
         BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index c0a54e596366d..e375bc1e25ddf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -43,7 +43,7 @@
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.security.AccessControlException;
 
 import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
index 5e5c4b4b81fb7..05b78ccc6f53d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
@@ -22,7 +22,7 @@
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.util.EnumCounters;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * The attributes of an inode.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 2b6bc061ab6c3..aa2b95d2ea608 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -59,7 +59,7 @@
 import static org.apache.hadoop.io.erasurecode.ErasureCodeConstants.REPLICATION_POLICY_ID;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /** I-node for closed file. */
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index f35949fdcdbed..01709832e9a3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.util.GSet;
 import org.apache.hadoop.util.LightWeightGSet;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Storing all the {@link INode}s and maintaining the mapping between INode ID
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
index ccb4b87a0e8de..6e655f7a13408 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor;
 import org.apache.hadoop.security.AccessControlException;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java
index 9241644dcb9c2..b7fff697bd841 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.namenode.FsImageValidation.Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
index a2c7804a83521..febd6f48ef4f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
@@ -24,7 +24,7 @@
 import org.apache.hadoop.hdfs.util.LongBitFormat;
 import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * {@link INode} with additional fields including id, name, permission,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index c2cdd48d4952a..832d84a50f28f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
 import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.ID_INTEGER_COMPARATOR;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
index f85f434e799e5..442c1aba95b1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java
@@ -65,7 +65,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * This class is used in Namesystem's jetty to retrieve/upload a file 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
index 8457f4186f394..19efc0d631120 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.slf4j.Logger;
@@ -42,10 +43,9 @@
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.util.Sets;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Manages a collection of Journals. None of the methods are synchronized, it is
@@ -677,7 +677,7 @@ public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) {
         // storage directory with ancient logs. Clear out any logs we've
         // accumulated so far, and then skip to the next segment of logs
         // after the gap.
-        SortedSet<Long> startTxIds = Sets.newTreeSet(logsByStartTxId.keySet());
+        SortedSet<Long> startTxIds = new TreeSet<>(logsByStartTxId.keySet());
         startTxIds = startTxIds.tailSet(curStartTxId);
         if (startTxIds.isEmpty()) {
           break;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 4caacc35c2fda..b21a34a932af9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -51,7 +51,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index ece11a360d003..ca8b6fc752065 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -59,7 +59,7 @@
 import org.eclipse.jetty.util.ajax.JSON;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * NNStorage is responsible for management of the StorageDirectories used by
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
index a4758bf695f94..b81cfccd75734 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
@@ -38,7 +38,7 @@
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.util.Lists;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
index 8086b60637dae..4e9ebec1d8529 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
@@ -34,7 +34,7 @@
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 public abstract class NNUpgradeUtil {
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 5e601cdd9a803..36ea5c2f646cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.ipc.CallerContext;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.util.Set;
 import org.apache.commons.logging.Log;
@@ -98,6 +101,8 @@
 import org.apache.hadoop.util.GcTimeMonitor;
 import org.apache.hadoop.util.GcTimeMonitor.Builder;
 import org.apache.hadoop.tracing.Tracer;
+import org.apache.hadoop.util.Timer;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -121,6 +126,11 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_DEFAULT;
@@ -187,6 +197,18 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_GC_TIME_MONITOR_ENABLE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK_DEFAULT;
 
 import static org.apache.hadoop.util.ExitUtil.terminate;
 import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
@@ -328,7 +350,16 @@ public enum OperationCategory {
           DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
           DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
           DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY,
-          DFS_IMAGE_PARALLEL_LOAD_KEY));
+          DFS_IMAGE_PARALLEL_LOAD_KEY,
+          DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY,
+          DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY,
+          DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
+          DFS_BLOCK_INVALIDATE_LIMIT_KEY,
+          DFS_DATANODE_PEER_STATS_ENABLED_KEY,
+          DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
+          DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
+          DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
+          DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK));
 
   private static final String USAGE = "Usage: hdfs namenode ["
       + StartupOption.BACKUP.getName() + "] | \n\t["
@@ -384,7 +415,6 @@ public long getProtocolVersion(String protocol,
    */
   @Deprecated
   public static final int DEFAULT_PORT = DFS_NAMENODE_RPC_PORT_DEFAULT;
-  public static final String FS_HDFS_IMPL_KEY = "fs.hdfs.impl";
   public static final Logger LOG =
       LoggerFactory.getLogger(NameNode.class.getName());
   public static final Logger stateChangeLog =
@@ -476,6 +506,97 @@ public static NameNodeMetrics getNameNodeMetrics() {
     return metrics;
   }
 
+  /**
+   * Try to obtain the actual client info according to the current user.
+   * @param ipProxyUsers Users who can override client infos
+   */
+  private static String clientInfoFromContext(
+      final String[] ipProxyUsers) {
+    if (ipProxyUsers != null) {
+      UserGroupInformation user =
+          UserGroupInformation.getRealUserOrSelf(Server.getRemoteUser());
+      if (user != null &&
+          ArrayUtils.contains(ipProxyUsers, user.getShortUserName())) {
+        CallerContext context = CallerContext.getCurrent();
+        if (context != null && context.isContextValid()) {
+          return context.getContext();
+        }
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Try to obtain the value corresponding to the key by parsing the content.
+   * @param content the full content to be parsed.
+   * @param key trying to obtain the value of the key.
+   * @return the value corresponding to the key.
+   */
+  @VisibleForTesting
+  public static String parseSpecialValue(String content, String key) {
+    int posn = content.indexOf(key);
+    if (posn != -1) {
+      posn += key.length();
+      int end = content.indexOf(",", posn);
+      return end == -1 ? content.substring(posn)
+          : content.substring(posn, end);
+    }
+    return null;
+  }
+
+  /**
+   * Try to obtain the actual client's machine according to the current user.
+   * @param ipProxyUsers Users who can override client infos.
+   * @return The actual client's machine.
+   */
+  public static String getClientMachine(final String[] ipProxyUsers) {
+    String clientMachine = null;
+    String cc = clientInfoFromContext(ipProxyUsers);
+    if (cc != null) {
+      // if the rpc has a caller context of "clientIp:1.2.3.4,CLI",
+      // return "1.2.3.4" as the client machine.
+      String key = CallerContext.CLIENT_IP_STR +
+          CallerContext.Builder.KEY_VALUE_SEPARATOR;
+      clientMachine = parseSpecialValue(cc, key);
+    }
+
+    if (clientMachine == null) {
+      clientMachine = Server.getRemoteAddress();
+      if (clientMachine == null) { //not a RPC client
+        clientMachine = "";
+      }
+    }
+    return clientMachine;
+  }
+
+  /**
+   * Try to obtain the actual client's id and call id
+   * according to the current user.
+   * @param ipProxyUsers Users who can override client infos
+   * @return The actual client's id and call id.
+   */
+  public static Pair<byte[], Integer> getClientIdAndCallId(
+      final String[] ipProxyUsers) {
+    byte[] clientId = Server.getClientId();
+    int callId = Server.getCallId();
+    String cc = clientInfoFromContext(ipProxyUsers);
+    if (cc != null) {
+      String clientIdKey = CallerContext.CLIENT_ID_STR +
+          CallerContext.Builder.KEY_VALUE_SEPARATOR;
+      String clientIdStr = parseSpecialValue(cc, clientIdKey);
+      if (clientIdStr != null) {
+        clientId = StringUtils.hexStringToByte(clientIdStr);
+      }
+      String callIdKey = CallerContext.CLIENT_CALL_ID_STR +
+          CallerContext.Builder.KEY_VALUE_SEPARATOR;
+      String callIdStr = parseSpecialValue(cc, callIdKey);
+      if (callIdStr != null) {
+        callId = Integer.parseInt(callIdStr);
+      }
+    }
+    return Pair.of(clientId, callId);
+  }
+
   /**
    * Returns object used for reporting namenode startup progress.
    * 
@@ -1888,6 +2009,9 @@ synchronized void transitionToStandby() throws IOException {
   synchronized void transitionToObserver() throws IOException {
     String operationName = "transitionToObserver";
     namesystem.checkSuperuserPrivilege(operationName);
+    if (notBecomeActiveInSafemode && isInSafeMode()) {
+      throw new ServiceFailedException(getRole() + " still not leave safemode");
+    }
     if (!haEnabled) {
       throw new ServiceFailedException("HA for namenode is not enabled");
     }
@@ -2079,7 +2203,7 @@ public void writeLock() {
     @Override
     public void writeUnlock() {
       namesystem.unlockRetryCache();
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("HAState");
     }
     
     /** Check if an operation of given category is allowed */
@@ -2189,7 +2313,8 @@ protected String reconfigurePropertyImpl(String property, String newVal)
     } else if (property.equals(DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY)
         || property.equals(DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY)
         || property.equals(
-            DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION)) {
+            DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION)
+        || property.equals(DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) {
       return reconfReplicationParameters(newVal, property);
     } else if (property.equals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY) || property
         .equals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY)) {
@@ -2197,6 +2322,18 @@ protected String reconfigurePropertyImpl(String property, String newVal)
       return newVal;
     } else if (property.equals(DFS_IMAGE_PARALLEL_LOAD_KEY)) {
       return reconfigureParallelLoad(newVal);
+    } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY) || (property.equals(
+        DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) || (property.equals(
+        DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY)) || (property.equals(
+        DFS_DATANODE_PEER_STATS_ENABLED_KEY)) || property.equals(
+        DFS_DATANODE_MAX_NODES_TO_REPORT_KEY)) {
+      return reconfigureSlowNodesParameters(datanodeManager, property, newVal);
+    } else if (property.equals(DFS_BLOCK_INVALIDATE_LIMIT_KEY)) {
+      return reconfigureBlockInvalidateLimit(datanodeManager, property, newVal);
+    } else if (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT) ||
+        (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK))) {
+      return reconfigureDecommissionBackoffMonitorParameters(datanodeManager, property,
+          newVal);
     } else {
       throw new ReconfigurationException(property, newVal, getConf().get(
           property));
@@ -2227,9 +2364,17 @@ private String reconfReplicationParameters(final String newVal,
                 DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT,
                 newVal));
         newSetting = bm.getBlocksReplWorkMultiplier();
+      } else if (
+          property.equals(
+              DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) {
+        bm.setReconstructionPendingTimeout(
+            adjustNewVal(
+                DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT,
+                newVal));
+        newSetting = bm.getReconstructionPendingTimeout();
       } else {
         throw new IllegalArgumentException("Unexpected property " +
-            property + "in reconfReplicationParameters");
+            property + " in reconfReplicationParameters");
       }
       LOG.info("RECONFIGURE* changed {} to {}", property, newSetting);
       return String.valueOf(newSetting);
@@ -2237,7 +2382,7 @@ private String reconfReplicationParameters(final String newVal,
       throw new ReconfigurationException(property, newVal, getConf().get(
           property), e);
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("reconfReplicationParameters");
     }
   }
 
@@ -2274,7 +2419,7 @@ private String reconfHeartbeatInterval(final DatanodeManager datanodeManager,
       throw new ReconfigurationException(property, newVal, getConf().get(
           property), nfe);
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("reconfHeartbeatInterval");
       LOG.info("RECONFIGURE* changed heartbeatInterval to "
           + datanodeManager.getHeartbeatInterval());
     }
@@ -2298,7 +2443,7 @@ private String reconfHeartbeatRecheckInterval(
       throw new ReconfigurationException(property, newVal, getConf().get(
           property), nfe);
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("reconfHeartbeatRecheckInterval");
       LOG.info("RECONFIGURE* changed heartbeatRecheckInterval to "
           + datanodeManager.getHeartbeatRecheckInterval());
     }
@@ -2383,6 +2528,121 @@ String reconfigureParallelLoad(String newVal) {
     return Boolean.toString(enableParallelLoad);
   }
 
+  String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager,
+      final String property, final String newVal) throws ReconfigurationException {
+    BlockManager bm = namesystem.getBlockManager();
+    namesystem.writeLock();
+    String result;
+    try {
+      switch (property) {
+      case DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY: {
+        boolean enable = (newVal == null ?
+            DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT :
+            Boolean.parseBoolean(newVal));
+        result = Boolean.toString(enable);
+        datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable);
+        break;
+      }
+      case DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY: {
+        boolean enable = (newVal == null ?
+            DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT :
+            Boolean.parseBoolean(newVal));
+        result = Boolean.toString(enable);
+        bm.setExcludeSlowNodesEnabled(enable);
+        break;
+      }
+      case DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY: {
+        int maxSlowpeerCollectNodes = (newVal == null ?
+            DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT :
+            Integer.parseInt(newVal));
+        result = Integer.toString(maxSlowpeerCollectNodes);
+        datanodeManager.setMaxSlowpeerCollectNodes(maxSlowpeerCollectNodes);
+        break;
+      }
+      case DFS_DATANODE_PEER_STATS_ENABLED_KEY: {
+        Timer timer = new Timer();
+        if (newVal != null && !newVal.equalsIgnoreCase("true") && !newVal.equalsIgnoreCase(
+            "false")) {
+          throw new IllegalArgumentException(newVal + " is not boolean value");
+        }
+        final boolean peerStatsEnabled = newVal == null ?
+            DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT :
+            Boolean.parseBoolean(newVal);
+        result = Boolean.toString(peerStatsEnabled);
+        datanodeManager.initSlowPeerTracker(getConf(), timer, peerStatsEnabled);
+        break;
+      }
+      case DFS_DATANODE_MAX_NODES_TO_REPORT_KEY: {
+        int maxSlowPeersToReport = (newVal == null
+            ? DFS_DATANODE_MAX_NODES_TO_REPORT_DEFAULT : Integer.parseInt(newVal));
+        result = Integer.toString(maxSlowPeersToReport);
+        datanodeManager.setMaxSlowPeersToReport(maxSlowPeersToReport);
+        break;
+      }
+      default: {
+        throw new IllegalArgumentException(
+            "Unexpected property " + property + " in reconfigureSlowNodesParameters");
+      }
+      }
+      LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
+      return result;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(
+          property), e);
+    } finally {
+      namesystem.writeUnlock("reconfigureSlowNodesParameters");
+    }
+  }
+
+  private String reconfigureBlockInvalidateLimit(final DatanodeManager datanodeManager,
+      final String property, final String newVal) throws ReconfigurationException {
+    namesystem.writeLock();
+    try {
+      if (newVal == null) {
+        datanodeManager.setBlockInvalidateLimit(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
+      } else {
+        datanodeManager.setBlockInvalidateLimit(Integer.parseInt(newVal));
+      }
+      final String updatedBlockInvalidateLimit =
+          String.valueOf(datanodeManager.getBlockInvalidateLimit());
+      LOG.info("RECONFIGURE* changed blockInvalidateLimit to {}", updatedBlockInvalidateLimit);
+      return updatedBlockInvalidateLimit;
+    } catch (NumberFormatException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    } finally {
+      namesystem.writeUnlock("reconfigureBlockInvalidateLimit");
+    }
+  }
+
+  private String reconfigureDecommissionBackoffMonitorParameters(
+      final DatanodeManager datanodeManager, final String property, final String newVal)
+      throws ReconfigurationException {
+    String newSetting = null;
+    try {
+      if (property.equals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT)) {
+        int pendingRepLimit = (newVal == null ?
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT_DEFAULT :
+            Integer.parseInt(newVal));
+        datanodeManager.getDatanodeAdminManager().refreshPendingRepLimit(pendingRepLimit,
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT);
+        newSetting = String.valueOf(datanodeManager.getDatanodeAdminManager().getPendingRepLimit());
+      } else if (property.equals(
+          DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK)) {
+        int blocksPerLock = (newVal == null ?
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK_DEFAULT :
+            Integer.parseInt(newVal));
+        datanodeManager.getDatanodeAdminManager().refreshBlocksPerLock(blocksPerLock,
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK);
+        newSetting = String.valueOf(datanodeManager.getDatanodeAdminManager().getBlocksPerLock());
+      }
+      LOG.info("RECONFIGURE* changed reconfigureDecommissionBackoffMonitorParameters {} to {}",
+          property, newSetting);
+      return newSetting;
+    } catch (IllegalArgumentException e) {
+      throw new ReconfigurationException(property, newVal, getConf().get(property), e);
+    }
+  }
+
   @Override  // ReconfigurableBase
   protected Configuration getNewConf() {
     return new HdfsConfiguration();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index f5e261dc78aae..8dfc3111a6ba0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -66,10 +66,10 @@ public static boolean supports(final LayoutFeature f, final int lv) {
    * layout version is the earliest prior version to which a downgrade is
    * possible after initiating rolling upgrade.  If the feature cannot satisfy
    * compatibility with any prior version, then set its minimum compatible
-   * lqyout version to itself to indicate that downgrade is impossible.
+   * layout version to itself to indicate that downgrade is impossible.
    * Satisfying compatibility might require adding logic to the new feature to
    * reject operations or handle them differently while rolling upgrade is in
-   * progress.  In general, it's possible to satisfy compatiblity for downgrade
+   * progress.  In general, it's possible to satisfy compatibility for downgrade
    * if the new feature just involves adding new edit log ops.  Deeper
    * structural changes, such as changing the way we place files in the metadata
    * directories, might be incompatible.  Feature implementations should strive
@@ -98,7 +98,7 @@ public enum Feature implements LayoutFeature {
     /**
      * Feature that is added at layout version {@code lv} - 1. 
      * @param lv new layout version with the addition of this feature
-     * @param minCompatLV minimium compatible layout version
+     * @param minCompatLV minimum compatible layout version
      * @param description description of the feature
      */
     Feature(final int lv, int minCompatLV, final String description) {
@@ -106,7 +106,7 @@ public enum Feature implements LayoutFeature {
     }
 
     /**
-     * NameNode feature that is added at layout version {@code ancestoryLV}.
+     * NameNode feature that is added at layout version {@code ancestorLV}.
      * @param lv new layout version with the addition of this feature
      * @param ancestorLV layout version from which the new lv is derived from.
      * @param minCompatLV minimum compatible layout version
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 474b5e29c33ac..b19bfc13acf65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -21,6 +21,7 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HANDLER_COUNT_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_HANDLER_RATIO_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_HANDLER_RATIO_KEY;
@@ -45,6 +46,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -267,6 +270,9 @@ public class NameNodeRpcServer implements NamenodeProtocols {
 
   private final String defaultECPolicyName;
 
+  // Users who can override the client info
+  private final String[] ipProxyUsers;
+
   public NameNodeRpcServer(Configuration conf, NameNode nn)
       throws IOException {
     this.nn = nn;
@@ -277,6 +283,7 @@ public NameNodeRpcServer(Configuration conf, NameNode nn)
     int handlerCount = 
       conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY, 
                   DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
+    ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS);
 
     RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class,
         ProtobufRpcEngine2.class);
@@ -450,7 +457,7 @@ public NameNodeRpcServer(Configuration conf, NameNode nn)
 
     GlobalStateIdContext stateIdContext = null;
     if (enableStateContext) {
-      stateIdContext = new GlobalStateIdContext((namesystem));
+      stateIdContext = new GlobalStateIdContext(namesystem);
     }
 
     clientRpcServer = new RPC.Builder(conf)
@@ -703,8 +710,7 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
     if(!nn.isRole(NamenodeRole.NAMENODE))
       throw new IOException("Only an ACTIVE node can invoke startCheckpoint.");
 
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
-      null);
+    CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (NamenodeCommand) cacheEntry.getPayload();
     }
@@ -717,13 +723,33 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration)
     return ret;
   }
 
+  /**
+   * Return the current CacheEntry.
+   */
+  private CacheEntry getCacheEntry() {
+    Pair<byte[], Integer> clientInfo =
+        NameNode.getClientIdAndCallId(this.ipProxyUsers);
+    return RetryCache.waitForCompletion(
+        retryCache, clientInfo.getLeft(), clientInfo.getRight());
+  }
+
+  /**
+   * Return the current CacheEntryWithPayload.
+   */
+  private CacheEntryWithPayload getCacheEntryWithPayload(Object payload) {
+    Pair<byte[], Integer> clientInfo =
+        NameNode.getClientIdAndCallId(this.ipProxyUsers);
+    return RetryCache.waitForCompletion(retryCache, payload,
+        clientInfo.getLeft(), clientInfo.getRight());
+  }
+
   @Override // NamenodeProtocol
   public void endCheckpoint(NamenodeRegistration registration,
                             CheckpointSignature sig) throws IOException {
     String operationName = "endCheckpoint";
     checkNNStartup();
     namesystem.checkSuperuserPrivilege(operationName);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -784,16 +810,14 @@ public HdfsFileStatus create(String src, FsPermission masked,
       throws IOException {
     checkNNStartup();
     String clientMachine = getClientMachine();
-    if (stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* NameNode.create: file "
-          +src+" for "+clientName+" at "+clientMachine);
-    }
+    stateChangeLog.debug("*DIR* NameNode.create: file {} for {} at {}.",
+        src, clientName, clientMachine);
     if (!checkPathLength(src)) {
       throw new IOException("create: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null);
+    CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (HdfsFileStatus) cacheEntry.getPayload();
     }
@@ -819,13 +843,10 @@ public LastBlockWithStatus append(String src, String clientName,
       EnumSetWritable<CreateFlag> flag) throws IOException {
     checkNNStartup();
     String clientMachine = getClientMachine();
-    if (stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* NameNode.append: file "
-          +src+" for "+clientName+" at "+clientMachine);
-    }
+    stateChangeLog.debug("*DIR* NameNode.append: file {} for {} at {}.",
+        src, clientName, clientMachine);
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
-        null);
+    CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (LastBlockWithStatus) cacheEntry.getPayload();
     }
@@ -869,8 +890,8 @@ public void unsetStoragePolicy(String src)
   public void setStoragePolicy(String src, String policyName)
       throws IOException {
     checkNNStartup();
-    stateChangeLog.debug("*DIR* NameNode.setStoragePolicy for path: {}, " +
-            "policyName: {}", src, policyName);
+    stateChangeLog.debug("*DIR* NameNode.setStoragePolicy for path: {}, policyName: {}",
+        src, policyName);
     namesystem.setStoragePolicy(src, policyName);
   }
 
@@ -924,13 +945,9 @@ public LocatedBlock getAdditionalDatanode(final String src,
       ) throws IOException {
     checkNNStartup();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("getAdditionalDatanode: src=" + src
-          + ", fileId=" + fileId
-          + ", blk=" + blk
-          + ", existings=" + Arrays.asList(existings)
-          + ", excludes=" + Arrays.asList(excludes)
-          + ", numAdditionalNodes=" + numAdditionalNodes
-          + ", clientName=" + clientName);
+      LOG.debug("getAdditionalDatanode: src={}, fileId={}, blk={}, existings={}, excludes={}"
+          + ", numAdditionalNodes={}, clientName={}", src, fileId, blk, Arrays.asList(existings),
+          Arrays.asList(excludes), numAdditionalNodes, clientName);
     }
 
     metrics.incrGetAdditionalDatanodeOps();
@@ -991,7 +1008,7 @@ public void updatePipeline(String clientName, ExtendedBlock oldBlock,
       throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1028,15 +1045,13 @@ public long getPreferredBlockSize(String filename)
   @Override // ClientProtocol
   public boolean rename(String src, String dst) throws IOException {
     checkNNStartup();
-    if(stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
-    }
+    stateChangeLog.debug("*DIR* NameNode.rename: {} to {}.", src, dst);
     if (!checkPathLength(dst)) {
       throw new IOException("rename: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return true; // Return previous response
     }
@@ -1056,10 +1071,12 @@ public boolean rename(String src, String dst) throws IOException {
   @Override // ClientProtocol
   public void concat(String trg, String[] src) throws IOException {
     checkNNStartup();
-    stateChangeLog.debug("*DIR* NameNode.concat: src path {} to" +
-        " target path {}", Arrays.toString(src), trg);
+    if (stateChangeLog.isDebugEnabled()) {
+      stateChangeLog.debug("*DIR* NameNode.concat: src path {} to target path {}",
+          Arrays.toString(src), trg);
+    }
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1077,15 +1094,13 @@ public void concat(String trg, String[] src) throws IOException {
   public void rename2(String src, String dst, Options.Rename... options)
       throws IOException {
     checkNNStartup();
-    if(stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst);
-    }
+    stateChangeLog.debug("*DIR* NameNode.rename: {} to {}.", src, dst);
     if (!checkPathLength(dst)) {
       throw new IOException("rename: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1103,8 +1118,7 @@ public void rename2(String src, String dst, Options.Rename... options)
   public boolean truncate(String src, long newLength, String clientName)
       throws IOException {
     checkNNStartup();
-    stateChangeLog
-        .debug("*DIR* NameNode.truncate: " + src + " to " + newLength);
+    stateChangeLog.debug("*DIR* NameNode.truncate: {} to {}", src, newLength);
     String clientMachine = getClientMachine();
     try {
       return namesystem.truncate(
@@ -1117,12 +1131,9 @@ public boolean truncate(String src, long newLength, String clientName)
   @Override // ClientProtocol
   public boolean delete(String src, boolean recursive) throws IOException {
     checkNNStartup();
-    if (stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
-          + ", recursive=" + recursive);
-    }
+    stateChangeLog.debug("*DIR* Namenode.delete: src={}, recursive={}.", src, recursive);
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return true; // Return previous response
     }
@@ -1153,9 +1164,7 @@ private boolean checkPathLength(String src) {
   public boolean mkdirs(String src, FsPermission masked, boolean createParent)
       throws IOException {
     checkNNStartup();
-    if(stateChangeLog.isDebugEnabled()) {
-      stateChangeLog.debug("*DIR* NameNode.mkdirs: " + src);
-    }
+    stateChangeLog.debug("*DIR* NameNode.mkdirs: {}.", src);
     if (!checkPathLength(src)) {
       throw new IOException("mkdirs: Pathname too long.  Limit " 
                             + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
@@ -1166,7 +1175,14 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
   }
 
   @Override // ClientProtocol
-  public void renewLease(String clientName) throws IOException {
+  public void renewLease(String clientName, List<String> namespaces)
+      throws IOException {
+    if (namespaces != null && namespaces.size() > 0) {
+      LOG.warn("namespaces({}) should be null or empty "
+          + "on NameNode side, please check it.", namespaces);
+      throw new IOException("namespaces(" + namespaces
+          + ") should be null or empty");
+    }
     checkNNStartup();
     namesystem.renewLease(clientName);        
   }
@@ -1300,7 +1316,7 @@ public boolean restoreFailedStorage(String arg) throws IOException {
   @Override // ClientProtocol
   public boolean saveNamespace(long timeWindow, long txGap) throws IOException {
     checkNNStartup();
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return true; // Return previous response
     }
@@ -1488,7 +1504,7 @@ public QuotaUsage getQuotaUsage(String path) throws IOException {
   public void satisfyStoragePolicy(String src) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1501,6 +1517,12 @@ public void satisfyStoragePolicy(String src) throws IOException {
     }
   }
 
+  @Override
+  public DatanodeInfo[] getSlowDatanodeReport() throws IOException {
+    checkNNStartup();
+    return namesystem.slowDataNodesReport();
+  }
+
   @Override // ClientProtocol
   public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
                        StorageType type)
@@ -1529,7 +1551,7 @@ public void createSymlink(String target, String link, FsPermission dirPerms,
       boolean createParent) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1611,10 +1633,8 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg,
         final BlockReportContext context) throws IOException {
     checkNNStartup();
     verifyRequest(nodeReg);
-    if(blockStateChangeLog.isDebugEnabled()) {
-      blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: "
-           + "from " + nodeReg + ", reports.length=" + reports.length);
-    }
+    blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: from {}, reports.length={}.",
+        nodeReg, reports.length);
     final BlockManager bm = namesystem.getBlockManager(); 
     boolean noStaleStorages = false;
     try {
@@ -1633,7 +1653,7 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg,
         }
       }
     } catch (UnregisteredNodeException une) {
-      LOG.debug("Datanode {} is attempting to report but not register yet.",
+      LOG.warn("Datanode {} is attempting to report but not register yet.",
           nodeReg);
       return RegisterCommand.REGISTER;
     }
@@ -1657,10 +1677,8 @@ public DatanodeCommand cacheReport(DatanodeRegistration nodeReg,
       String poolId, List<Long> blockIds) throws IOException {
     checkNNStartup();
     verifyRequest(nodeReg);
-    if (blockStateChangeLog.isDebugEnabled()) {
-      blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: "
-           + "from " + nodeReg + " " + blockIds.size() + " blocks");
-    }
+    blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: from {} {} blocks",
+        nodeReg, blockIds.size());
     namesystem.getCacheManager().processCacheReport(nodeReg, blockIds);
     return null;
   }
@@ -1672,11 +1690,8 @@ public void blockReceivedAndDeleted(final DatanodeRegistration nodeReg,
     checkNNStartup();
     verifyRequest(nodeReg);
     metrics.incrBlockReceivedAndDeletedOps();
-    if(blockStateChangeLog.isDebugEnabled()) {
-      blockStateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: "
-          +"from "+nodeReg+" "+receivedAndDeletedBlocks.length
-          +" blocks.");
-    }
+    blockStateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: from {} {} blocks.",
+        nodeReg, receivedAndDeletedBlocks.length);
     final BlockManager bm = namesystem.getBlockManager();
     for (final StorageReceivedDeletedBlocks r : receivedAndDeletedBlocks) {
       bm.enqueueBlockOp(new Runnable() {
@@ -1804,9 +1819,7 @@ public Collection<RefreshResponse> refresh(String identifier, String[] args) {
   
   @Override // GetUserMappingsProtocol
   public String[] getGroupsForUser(String user) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Getting groups for user " + user);
-    }
+    LOG.debug("Getting groups for user {}", user);
     return UserGroupInformation.createRemoteUser(user).getGroupNames();
   }
 
@@ -1899,12 +1912,11 @@ private void verifySoftwareVersion(DatanodeRegistration dnReg)
     }
   }
 
-  private static String getClientMachine() {
-    String clientMachine = Server.getRemoteAddress();
-    if (clientMachine == null) { //not a RPC client
-      clientMachine = "";
-    }
-    return clientMachine;
+  /**
+   * Get the actual client's machine.
+   */
+  private String getClientMachine() {
+    return NameNode.getClientMachine(this.ipProxyUsers);
   }
 
   @Override
@@ -1924,8 +1936,7 @@ public String createSnapshot(String snapshotRoot, String snapshotName)
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
-        null);
+    CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (String) cacheEntry.getPayload();
     }
@@ -1952,7 +1963,7 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName)
     }
     namesystem.checkOperation(OperationCategory.WRITE);
     metrics.incrDeleteSnapshotOps();
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -1986,15 +1997,15 @@ public void disallowSnapshot(String snapshot) throws IOException {
   public void renameSnapshot(String snapshotRoot, String snapshotOldName,
       String snapshotNewName) throws IOException {
     checkNNStartup();
-    LOG.debug("*DIR* NameNode.renameSnapshot: Snapshot Path {}, " +
-        "snapshotOldName {}, snapshotNewName {}", snapshotRoot,
-        snapshotOldName, snapshotNewName);
+    LOG.debug(
+        "*DIR* NameNode.renameSnapshot: Snapshot Path {},snapshotOldName {}, snapshotNewName {}",
+        snapshotRoot, snapshotOldName, snapshotNewName);
     if (snapshotNewName == null || snapshotNewName.isEmpty()) {
       throw new IOException("The new snapshot name is null or empty.");
     }
     namesystem.checkOperation(OperationCategory.WRITE);
     metrics.incrRenameSnapshotOps();
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -2055,8 +2066,7 @@ public long addCacheDirective(
       CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
-      (retryCache, null);
+    CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (Long) cacheEntry.getPayload();
     }
@@ -2077,7 +2087,7 @@ public void modifyCacheDirective(
       CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2095,7 +2105,7 @@ public void modifyCacheDirective(
   public void removeCacheDirective(long id) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2122,7 +2132,7 @@ public BatchedEntries<CacheDirectiveEntry> listCacheDirectives(long prevId,
   public void addCachePool(CachePoolInfo info) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -2139,7 +2149,7 @@ public void addCachePool(CachePoolInfo info) throws IOException {
   public void modifyCachePool(CachePoolInfo info) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -2156,7 +2166,7 @@ public void modifyCachePool(CachePoolInfo info) throws IOException {
   public void removeCachePool(String cachePoolName) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2219,7 +2229,7 @@ public void createEncryptionZone(String src, String keyName)
     throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2251,7 +2261,7 @@ public void reencryptEncryptionZone(final String zone,
       final ReencryptAction action) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2275,7 +2285,7 @@ public BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(
   public void setErasureCodingPolicy(String src, String ecPolicyName)
       throws IOException {
     checkNNStartup();
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2283,8 +2293,7 @@ public void setErasureCodingPolicy(String src, String ecPolicyName)
     try {
       if (ecPolicyName == null) {
         ecPolicyName = defaultECPolicyName;
-        LOG.debug("No policy name is specified, " +
-            "set the default policy name instead");
+        LOG.debug("No policy name is specified, set the default policy name instead");
       }
       LOG.debug("Set erasure coding policy {} on {}", ecPolicyName, src);
       namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null);
@@ -2299,7 +2308,7 @@ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
       throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -2329,7 +2338,7 @@ public List<XAttr> listXAttrs(String src) throws IOException {
   public void removeXAttr(String src, XAttr xAttr) throws IOException {
     checkNNStartup();
     namesystem.checkOperation(OperationCategory.WRITE);
-    CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
     }
@@ -2512,7 +2521,7 @@ public ErasureCodingPolicy getErasureCodingPolicy(String src) throws IOException
   @Override // ClientProtocol
   public void unsetErasureCodingPolicy(String src) throws IOException {
     checkNNStartup();
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2538,8 +2547,7 @@ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
     String operationName = "addErasureCodingPolicies";
     checkNNStartup();
     namesystem.checkSuperuserPrivilege(operationName);
-    final CacheEntryWithPayload cacheEntry =
-        RetryCache.waitForCompletion(retryCache, null);
+    final CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (AddErasureCodingPolicyResponse[]) cacheEntry.getPayload();
     }
@@ -2562,7 +2570,7 @@ public void removeErasureCodingPolicy(String ecPolicyName)
     String operationName = "removeErasureCodingPolicy";
     checkNNStartup();
     namesystem.checkSuperuserPrivilege(operationName);
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2581,7 +2589,7 @@ public void enableErasureCodingPolicy(String ecPolicyName)
     String operationName = "enableErasureCodingPolicy";
     checkNNStartup();
     namesystem.checkSuperuserPrivilege(operationName);
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
@@ -2600,7 +2608,7 @@ public void disableErasureCodingPolicy(String ecPolicyName)
     String operationName = "disableErasureCodingPolicy";
     checkNNStartup();
     namesystem.checkSuperuserPrivilege(operationName);
-    final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+    final CacheEntry cacheEntry = getCacheEntry();
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
index 15f799ab215ba..5c046cdce0ad3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.io.IOUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs;
 import org.apache.hadoop.log.LogThrottlingHelper;
 import org.apache.hadoop.log.LogThrottlingHelper.LogAction;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
index da5413ab5e4e8..2a7002b5cd2f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -350,7 +350,7 @@ public void run() {
         getReencryptionStatus().markZoneStarted(zoneId);
         resetSubmissionTracker(zoneId);
       } finally {
-        dir.getFSNamesystem().readUnlock();
+        dir.getFSNamesystem().readUnlock("reEncryptThread");
       }
 
       try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
index 2650141718776..43684a7accccd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
@@ -32,7 +32,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 import java.util.Arrays;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java
index 35a7cd2f643cc..ae6bef783cacc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.util.Canceler;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Context for an ongoing SaveNamespace operation. This class
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index 0163e7311d809..e95200b35aaa5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -78,7 +78,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.VersionInfo;
 
 import javax.management.ObjectName;
@@ -803,7 +803,7 @@ enum Command {
       geteditsizeOpt = new Option("geteditsize",
         "return the number of uncheckpointed transactions on the NameNode");
       checkpointOpt = OptionBuilder.withArgName("force")
-        .hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");;
+        .hasOptionalArg().withDescription("checkpoint on startup").create("checkpoint");
       formatOpt = new Option("format", "format the local storage during startup");
       helpOpt = new Option("h", "help", false, "get help information");
       
@@ -1098,7 +1098,7 @@ void doMerge(
       try {
         dstImage.reloadFromImageFile(file, dstNamesystem);
       } finally {
-        dstNamesystem.writeUnlock();
+        dstNamesystem.writeUnlock("reloadFromImageFile");
       }
       dstNamesystem.imageLoadComplete();
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
index ee6b8eb80ce04..ff116b511889e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -69,6 +68,7 @@ public int get(T t) {
       }
       Integer old = t2i.putIfAbsent(t, sn);
       if (old != null) {
+        current.getAndDecrement();
         return old;
       }
       i2t.put(sn, t);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
index af1025ab457ec..4d46e691df217 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.hdfs.XAttrHelper;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 import org.apache.hadoop.hdfs.util.LongBitFormat;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
index da760e9ddd020..22c616067a6cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.Lists;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index 3f273cb5e75e6..7a9ce46b1159f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -71,7 +71,7 @@
 import org.apache.hadoop.util.ToolRunner;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Tool which allows the standby node's storage directories to be bootstrapped
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index cdea5eac04d33..6921e204ae235 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -59,7 +59,7 @@
 import static org.apache.hadoop.util.ExitUtil.terminate;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 
 /**
@@ -123,7 +123,7 @@ public class EditLogTailer {
 
   /**
    * The timeout in milliseconds of calling rollEdits RPC to Active NN.
-   * @see HDFS-4176.
+   * See HDFS-4176.
    */
   private final long rollEditsTimeoutMs;
 
@@ -262,9 +262,7 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
     nnCount = nns.size();
     // setup the iterator to endlessly loop the nns
     this.nnLookup = Iterators.cycle(nns);
-
-    LOG.debug("logRollPeriodMs=" + logRollPeriodMs +
-        " sleepTime=" + sleepTimeMs);
+    LOG.debug("logRollPeriodMs={} sleepTime={}.", logRollPeriodMs, sleepTimeMs);
   }
 
   public void start() {
@@ -328,38 +326,39 @@ public Void run() throws Exception {
   
   @VisibleForTesting
   public long doTailEdits() throws IOException, InterruptedException {
+    Collection<EditLogInputStream> streams;
+    FSImage image = namesystem.getFSImage();
+
+    long lastTxnId = image.getLastAppliedTxId();
+    LOG.debug("lastTxnId: {}", lastTxnId);
+    long startTime = timer.monotonicNow();
+    try {
+      streams = editLog.selectInputStreams(lastTxnId + 1, 0,
+          null, inProgressOk, true);
+    } catch (IOException ioe) {
+      // This is acceptable. If we try to tail edits in the middle of an edits
+      // log roll, i.e. the last one has been finalized but the new inprogress
+      // edits file hasn't been started yet.
+      LOG.warn("Edits tailer failed to find any streams. Will try again " +
+          "later.", ioe);
+      return 0;
+    } finally {
+      NameNode.getNameNodeMetrics().addEditLogFetchTime(
+          timer.monotonicNow() - startTime);
+    }
     // Write lock needs to be interruptible here because the 
     // transitionToActive RPC takes the write lock before calling
     // tailer.stop() -- so if we're not interruptible, it will
     // deadlock.
     namesystem.writeLockInterruptibly();
     try {
-      FSImage image = namesystem.getFSImage();
-
-      long lastTxnId = image.getLastAppliedTxId();
-      
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("lastTxnId: " + lastTxnId);
-      }
-      Collection<EditLogInputStream> streams;
-      long startTime = timer.monotonicNow();
-      try {
-        streams = editLog.selectInputStreams(lastTxnId + 1, 0,
-            null, inProgressOk, true);
-      } catch (IOException ioe) {
-        // This is acceptable. If we try to tail edits in the middle of an edits
-        // log roll, i.e. the last one has been finalized but the new inprogress
-        // edits file hasn't been started yet.
-        LOG.warn("Edits tailer failed to find any streams. Will try again " +
-            "later.", ioe);
+      long currentLastTxnId = image.getLastAppliedTxId();
+      if (lastTxnId != currentLastTxnId) {
+        LOG.warn("The currentLastTxnId({}) is different from preLastTxtId({})",
+            currentLastTxnId, lastTxnId);
         return 0;
-      } finally {
-        NameNode.getNameNodeMetrics().addEditLogFetchTime(
-            timer.monotonicNow() - startTime);
-      }
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("edit streams to load from: " + streams.size());
       }
+      LOG.debug("edit streams to load from: {}.", streams.size());
       
       // Once we have streams to load, errors encountered are legitimate cause
       // for concern, so we don't catch them here. Simple errors reading from
@@ -372,10 +371,7 @@ public long doTailEdits() throws IOException, InterruptedException {
         editsLoaded = elie.getNumEditsLoaded();
         throw elie;
       } finally {
-        if (editsLoaded > 0 || LOG.isDebugEnabled()) {
-          LOG.debug(String.format("Loaded %d edits starting from txid %d ",
-              editsLoaded, lastTxnId));
-        }
+        LOG.debug("Loaded {} edits starting from txid {}.", editsLoaded, lastTxnId);
         NameNode.getNameNodeMetrics().addNumEditLogLoaded(editsLoaded);
       }
 
@@ -385,7 +381,7 @@ public long doTailEdits() throws IOException, InterruptedException {
       lastLoadedTxnId = image.getLastAppliedTxId();
       return editsLoaded;
     } finally {
-      namesystem.writeUnlock();
+      namesystem.writeUnlock("doTailEdits");
     }
   }
 
@@ -413,6 +409,8 @@ Callable<Void> getNameNodeProxy() {
     return new MultipleNameNodeProxy<Void>() {
       @Override
       protected Void doWork() throws IOException {
+        LOG.info("Triggering log rolling to the remote NameNode, " +
+            "active NameNode = {}", currentNN.getIpcAddress());
         cachedActiveProxy.rollEditLog();
         return null;
       }
@@ -424,7 +422,6 @@ protected Void doWork() throws IOException {
    */
   @VisibleForTesting
   void triggerActiveLogRoll() {
-    LOG.info("Triggering log roll on remote NameNode");
     Future<Void> future = null;
     try {
       future = rollEditsRpcExecutor.submit(getNameNodeProxy());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java
index 1ff251d808c8c..784d19660f104 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java
@@ -24,14 +24,11 @@
 import java.util.Collections;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Objects;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 /**
  * Information about a single remote NameNode
  */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
index 89df652f23e51..ec848668d2561 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java
@@ -49,7 +49,7 @@
 import org.apache.hadoop.util.Lists;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import org.slf4j.Logger;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
index 7e5f108167ccc..59bb01f0063cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
@@ -254,4 +254,11 @@ public interface FSNamesystemMBean {
    * @return number of DTs
    */
   long getCurrentTokensCount();
+
+  /**
+   * Returns the number of paths to be processed by storage policy satisfier.
+   *
+   * @return The number of paths to be processed by sps.
+   */
+  int getPendingSPSPaths();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
index fd1fab7a7fa32..f0cf00238b5ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
@@ -89,6 +89,10 @@ public class NameNodeMetrics {
   MutableCounterLong blockOpsBatched;
   @Metric("Number of pending edits")
   MutableGaugeInt pendingEditsCount;
+  @Metric("Number of delete blocks Queued")
+  MutableGaugeInt deleteBlocksQueued;
+  @Metric("Number of pending deletion blocks")
+  MutableGaugeInt pendingDeleteBlocksCount;
 
   @Metric("Number of file system operations")
   public long totalFileOps(){
@@ -341,6 +345,18 @@ public void setBlockOpsQueued(int size) {
     blockOpsQueued.set(size);
   }
 
+  public void setDeleteBlocksQueued(int size) {
+    deleteBlocksQueued.set(size);
+  }
+
+  public void incrPendingDeleteBlocksCount(int size) {
+    pendingDeleteBlocksCount.incr(size);
+  }
+
+  public void decrPendingDeleteBlocksCount() {
+    pendingDeleteBlocksCount.decr();
+  }
+
   public void addBlockOpsBatched(int count) {
     blockOpsBatched.incr(count);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
index df052f171afa8..b1f15dbd1b331 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
@@ -24,7 +24,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * The difference of an inode between in two snapshots.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
index dedc1e49d341f..ecdad34360937 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.snapshot;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.
     DirectoryWithSnapshotFeature.DirectoryDiff;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 1d6b0e5a33159..c181c05d0811b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -49,7 +49,7 @@
 import org.apache.hadoop.util.Time;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * A directory with this feature is a snapshottable directory, where snapshots
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index 2c0c8146f4044..3893f42fde2de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.snapshot;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.namenode.*;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index a6e2d5debb13b..f1a21cce45a4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -82,7 +82,7 @@
 import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
 import org.apache.hadoop.hdfs.util.EnumCounters;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.ByteString;
 
 @InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
index d57da2204aa38..c62455c724feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java
@@ -79,7 +79,7 @@ private void gcDeletedSnapshot(String name) {
       LOG.error("Failed to chooseDeletedSnapshot", e);
       throw e;
     } finally {
-      namesystem.readUnlock();
+      namesystem.readUnlock("gcDeletedSnapshot");
     }
     if (deleted == null) {
       LOG.trace("{}: no snapshots are marked as deleted.", name);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
index 58dd2cf0a3f04..74d06c883b8d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
@@ -33,7 +33,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes;
 import org.apache.hadoop.util.ChunkedArrayList;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
index 4b03c4f0d53a1..e20a46e1b75f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java
@@ -29,7 +29,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeReference;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.ChunkedArrayList;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
index b43c45854bbfd..527d767b09a00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
@@ -40,7 +40,7 @@
 import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * A helper class defining static methods for reading/writing snapshot related
@@ -70,7 +70,7 @@ public static void saveSnapshots(INodeDirectory current, DataOutput out)
 
   /**
    * Save SnapshotDiff list for an INodeDirectoryWithSnapshot.
-   * @param sNode The directory that the SnapshotDiff list belongs to.
+   * @param diffs The directory that the SnapshotDiff list belongs to.
    * @param out The {@link DataOutput} to write.
    */
   private static <N extends INode, A extends INodeAttributes, D extends AbstractINodeDiff<N, A, D>>
@@ -186,7 +186,7 @@ private static List<INode> loadCreatedList(INodeDirectory parent,
    * @param createdList The created list associated with the deleted list in 
    *                    the same Diff.
    * @param in The {@link DataInput} to read.
-   * @param loader The {@link Loader} instance.
+   * @param loader The {@link FSImageFormat.Loader} instance.
    * @return The deleted list.
    */
   private static List<INode> loadDeletedList(INodeDirectory parent,
@@ -260,7 +260,7 @@ public static void loadDirectoryDiffList(INodeDirectory dir,
    * Load the snapshotINode field of {@link AbstractINodeDiff}.
    * @param snapshot The Snapshot associated with the {@link AbstractINodeDiff}.
    * @param in The {@link DataInput} to read.
-   * @param loader The {@link Loader} instance that this loading procedure is
+   * @param loader The {@link FSImageFormat.Loader} instance that this loading procedure is
    *               using.
    * @return The snapshotINode.
    */
@@ -281,7 +281,7 @@ private static INodeDirectoryAttributes loadSnapshotINodeInDirectoryDiff(
    * Load {@link DirectoryDiff} from fsimage.
    * @param parent The directory that the SnapshotDiff belongs to.
    * @param in The {@link DataInput} instance to read.
-   * @param loader The {@link Loader} instance that this loading procedure is 
+   * @param loader The {@link FSImageFormat.Loader} instance that this loading procedure is
    *               using.
    * @return A {@link DirectoryDiff}.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index d6c2263e5766e..9e6fc36377cf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -65,7 +65,7 @@
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.util.Lists;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -391,8 +391,8 @@ public INodeDirectory checkAndGetSnapshottableAncestorDir(
       final INodesInPath iip) throws IOException {
     final INodeDirectory dir = getSnapshottableAncestorDir(iip);
     if (dir == null) {
-      throw new SnapshotException("Directory is neither snapshottable nor" +
-          " under a snap root!");
+      throw new SnapshotException("The path " + iip.getPath()
+          + " is neither snapshottable nor under a snapshot root!");
     }
     return dir;
   }
@@ -402,7 +402,7 @@ public INodeDirectory getSnapshottableAncestorDir(final INodesInPath iip)
     final String path = iip.getPath();
     final INode inode = iip.getLastINode();
     final INodeDirectory dir;
-    if (inode instanceof INodeDirectory) {
+    if (inode != null && inode.isDirectory()) {
       dir = INodeDirectory.valueOf(inode, path);
     } else {
       dir = INodeDirectory.valueOf(iip.getINode(-2), iip.getParentPath());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
index 1a136e571d652..174f2be7a7172 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
@@ -301,6 +301,16 @@ public int getAttemptedItemsCount() {
     }
   }
 
+  @VisibleForTesting
+  public List<AttemptedItemInfo> getStorageMovementAttemptedItems() {
+    return storageMovementAttemptedItems;
+  }
+
+  @VisibleForTesting
+  public BlockingQueue<Block> getMovementFinishedBlocks() {
+    return movementFinishedBlocks;
+  }
+
   public void clearQueues() {
     movementFinishedBlocks.clear();
     synchronized (storageMovementAttemptedItems) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
index 9a1faed952595..b6fc15d2104a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementNeeded.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.sps;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -227,15 +228,18 @@ public synchronized void clearQueuesWithNotification() {
    * ID's to process for satisfy the policy.
    */
   private class SPSPathIdProcessor implements Runnable {
+    private static final int MAX_RETRY_COUNT = 3;
 
     @Override
     public void run() {
       LOG.info("Starting SPSPathIdProcessor!.");
       Long startINode = null;
+      int retryCount = 0;
       while (ctxt.isRunning()) {
         try {
           if (!ctxt.isInSafeMode()) {
             if (startINode == null) {
+              retryCount = 0;
               startINode = ctxt.getNextSPSPath();
             } // else same id will be retried
             if (startINode == null) {
@@ -248,7 +252,12 @@ public void run() {
                   pendingWorkForDirectory.get(startINode);
               if (dirPendingWorkInfo != null
                   && dirPendingWorkInfo.isDirWorkDone()) {
-                ctxt.removeSPSHint(startINode);
+                try {
+                  ctxt.removeSPSHint(startINode);
+                } catch (FileNotFoundException e) {
+                  // ignore if the file doesn't already exist
+                  startINode = null;
+                }
                 pendingWorkForDirectory.remove(startINode);
               }
             }
@@ -268,6 +277,11 @@ public void run() {
             LOG.info("Interrupted while waiting in SPSPathIdProcessor", t);
             break;
           }
+          retryCount++;
+          if (retryCount >= MAX_RETRY_COUNT) {
+            LOG.warn("Skipping this inode {} due to too many retries.", startINode);
+            startINode = null;
+          }
         }
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
index 686644f3ec019..3efe6b1cd6d86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
@@ -58,7 +58,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Setting storagePolicy on a file after the file write will only update the new
@@ -1077,7 +1077,7 @@ public void clearQueues() {
    * attempted or reported time stamp. This is used by
    * {@link BlockStorageMovementAttemptedItems#storageMovementAttemptedItems}.
    */
-  final static class AttemptedItemInfo extends ItemInfo {
+  public final static class AttemptedItemInfo extends ItemInfo {
     private long lastAttemptedOrReportedTime;
     private final Set<Block> blocks;
 
@@ -1095,7 +1095,7 @@ final static class AttemptedItemInfo extends ItemInfo {
      * @param retryCount
      *          file retry count
      */
-    AttemptedItemInfo(long rootId, long trackId,
+    public AttemptedItemInfo(long rootId, long trackId,
         long lastAttemptedOrReportedTime,
         Set<Block> blocks, int retryCount) {
       super(rootId, trackId, retryCount);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
index 40e3faa5550e5..2c7f36a690bd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfyManager.java
@@ -60,7 +60,7 @@ public class StoragePolicySatisfyManager {
   private final StoragePolicySatisfier spsService;
   private final boolean storagePolicyEnabled;
   private volatile StoragePolicySatisfierMode mode;
-  private final Queue<Long> pathsToBeTraveresed;
+  private final Queue<Long> pathsToBeTraversed;
   private final int outstandingPathsLimit;
   private final Namesystem namesystem;
 
@@ -77,7 +77,7 @@ public StoragePolicySatisfyManager(Configuration conf,
         DFSConfigKeys.DFS_SPS_MAX_OUTSTANDING_PATHS_KEY,
         DFSConfigKeys.DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT);
     mode = StoragePolicySatisfierMode.fromString(modeVal);
-    pathsToBeTraveresed = new LinkedList<Long>();
+    pathsToBeTraversed = new LinkedList<Long>();
     this.namesystem = namesystem;
     // instantiate SPS service by just keeps config reference and not starting
     // any supporting threads.
@@ -218,8 +218,8 @@ public boolean isSatisfierRunning() {
    *         storages.
    */
   public Long getNextPathId() {
-    synchronized (pathsToBeTraveresed) {
-      return pathsToBeTraveresed.poll();
+    synchronized (pathsToBeTraversed) {
+      return pathsToBeTraversed.poll();
     }
   }
 
@@ -228,7 +228,7 @@ public Long getNextPathId() {
    * @throws IOException
    */
   public void verifyOutstandingPathQLimit() throws IOException {
-    long size = pathsToBeTraveresed.size();
+    long size = pathsToBeTraversed.size();
     // Checking that the SPS call Q exceeds the allowed limit.
     if (outstandingPathsLimit - size <= 0) {
       LOG.debug("Satisifer Q - outstanding limit:{}, current size:{}",
@@ -244,15 +244,15 @@ public void verifyOutstandingPathQLimit() throws IOException {
    * @throws IOException
    */
   private void clearPathIds(){
-    synchronized (pathsToBeTraveresed) {
-      Iterator<Long> iterator = pathsToBeTraveresed.iterator();
+    synchronized (pathsToBeTraversed) {
+      Iterator<Long> iterator = pathsToBeTraversed.iterator();
       while (iterator.hasNext()) {
         Long trackId = iterator.next();
         try {
           namesystem.removeXattr(trackId,
               HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY);
         } catch (IOException e) {
-          LOG.debug("Failed to remove sps xatttr!", e);
+          LOG.debug("Failed to remove sps xattr!", e);
         }
         iterator.remove();
       }
@@ -263,8 +263,8 @@ private void clearPathIds(){
    * Clean up all sps path ids.
    */
   public void removeAllPathIds() {
-    synchronized (pathsToBeTraveresed) {
-      pathsToBeTraveresed.clear();
+    synchronized (pathsToBeTraversed) {
+      pathsToBeTraversed.clear();
     }
   }
 
@@ -273,8 +273,8 @@ public void removeAllPathIds() {
    * @param id
    */
   public void addPathId(long id) {
-    synchronized (pathsToBeTraveresed) {
-      pathsToBeTraveresed.add(id);
+    synchronized (pathsToBeTraversed) {
+      pathsToBeTraversed.add(id);
     }
   }
 
@@ -292,4 +292,11 @@ public boolean isEnabled() {
   public StoragePolicySatisfierMode getMode() {
     return mode;
   }
+
+  /**
+   * @return the number of paths to be processed by storage policy satisfier.
+   */
+  public int getPendingSPSPaths() {
+    return pathsToBeTraversed.size();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java
index e6a8b97fb3faa..1ba0b8332d183 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/package-info.java
@@ -17,7 +17,7 @@
  */
 
 /**
- * This package provides a mechanism for tracking {@link NameNode} startup
+ * This package provides a mechanism for tracking NameNode startup
  * progress.  The package models NameNode startup as a series of {@link Phase}s,
  * with each phase further sub-divided into multiple {@link Step}s.  All phases
  * are coarse-grained and typically known in advance, implied by the structure of
@@ -40,4 +40,3 @@
 package org.apache.hadoop.hdfs.server.namenode.startupprogress;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
index 93eea6068c2fb..a43eb62d72b41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
@@ -19,7 +19,7 @@
 
 import java.net.InetAddress;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
index e78e41957d107..d0c0b6958cc27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * This class is a common place for NNTop configuration.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
index 3edc29e47d393..8015641e1f1bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
@@ -29,7 +29,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java
index ad7a6d9b6b68d..3dcc9e628dc07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.visitor;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 3f2028e7c000c..a3250c213caf0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -55,6 +55,7 @@
 import javax.ws.rs.core.Response.ResponseBuilder;
 import javax.ws.rs.core.Response.Status;
 
+import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -85,6 +86,7 @@
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
 import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
@@ -132,6 +134,7 @@ public class NamenodeWebHdfsMethods {
   private String scheme;
   private Principal userPrincipal;
   private String remoteAddr;
+  private int remotePort;
 
   private @Context ServletContext context;
   private @Context HttpServletResponse response;
@@ -146,6 +149,7 @@ public NamenodeWebHdfsMethods(@Context HttpServletRequest request) {
     // get the remote address, if coming in via a trusted proxy server then
     // the address with be that of the proxied client
     remoteAddr = JspHelper.getRemoteAddr(request);
+    remotePort = JspHelper.getRemotePort(request);
     supportEZ =
         Boolean.valueOf(request.getHeader(WebHdfsFileSystem.EZ_HEADER));
   }
@@ -224,6 +228,10 @@ public String getHostAddress() {
         return getRemoteAddr();
       }
       @Override
+      public int getRemotePort() {
+        return getRemotePortFromJSPHelper();
+      }
+      @Override
       public InetAddress getHostInetAddress() {
         try {
           return InetAddress.getByName(getHostAddress());
@@ -254,6 +262,10 @@ protected String getRemoteAddr() {
     return remoteAddr;
   }
 
+  protected int getRemotePortFromJSPHelper() {
+    return remotePort;
+  }
+
   protected void queueExternalCall(ExternalCall call)
       throws IOException, InterruptedException {
     final NameNode namenode = (NameNode)context.getAttribute("name.node");
@@ -390,6 +402,9 @@ private URI redirectURI(ResponseBuilder rb, final NameNode namenode,
       final String path, final HttpOpParam.Op op, final long openOffset,
       final long blocksize, final String excludeDatanodes,
       final Param<?, ?>... parameters) throws URISyntaxException, IOException {
+    if (!DFSUtil.isValidName(path)) {
+      throw new InvalidPathException(path);
+    }
     final DatanodeInfo dn;
     final NamenodeProtocols np = getRPCServer(namenode);
     HdfsFileStatus status = null;
@@ -1037,6 +1052,10 @@ public Response getRoot(
           final SnapshotNameParam snapshotName,
       @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
           final OldSnapshotNameParam oldSnapshotName,
+      @QueryParam(SnapshotDiffStartPathParam.NAME) @DefaultValue(SnapshotDiffStartPathParam.DEFAULT)
+          final SnapshotDiffStartPathParam snapshotDiffStartPath,
+      @QueryParam(SnapshotDiffIndexParam.NAME) @DefaultValue(SnapshotDiffIndexParam.DEFAULT)
+          final SnapshotDiffIndexParam snapshotDiffIndex,
       @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT)
           final TokenKindParam tokenKind,
       @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
@@ -1048,7 +1067,9 @@ public Response getRoot(
       ) throws IOException, InterruptedException {
     return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
         renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes,
-        fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService,
+        fsAction, snapshotName, oldSnapshotName,
+        snapshotDiffStartPath, snapshotDiffIndex,
+        tokenKind, tokenService,
         noredirect, startAfter);
   }
 
@@ -1088,6 +1109,10 @@ public Response get(
           final SnapshotNameParam snapshotName,
       @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
           final OldSnapshotNameParam oldSnapshotName,
+      @QueryParam(SnapshotDiffStartPathParam.NAME) @DefaultValue(SnapshotDiffStartPathParam.DEFAULT)
+          final SnapshotDiffStartPathParam snapshotDiffStartPath,
+      @QueryParam(SnapshotDiffIndexParam.NAME) @DefaultValue(SnapshotDiffIndexParam.DEFAULT)
+          final SnapshotDiffIndexParam snapshotDiffIndex,
       @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT)
           final TokenKindParam tokenKind,
       @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
@@ -1108,6 +1133,7 @@ public Response run() throws IOException, URISyntaxException {
         return get(ugi, delegation, username, doAsUser, path.getAbsolutePath(),
             op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
             excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
+            snapshotDiffStartPath, snapshotDiffIndex,
             tokenKind, tokenService, noredirect, startAfter);
       }
     });
@@ -1137,6 +1163,8 @@ protected Response get(
       final FsActionParam fsAction,
       final SnapshotNameParam snapshotName,
       final OldSnapshotNameParam oldSnapshotName,
+      final SnapshotDiffStartPathParam snapshotDiffStartPath,
+      final SnapshotDiffIndexParam snapshotDiffIndex,
       final TokenKindParam tokenKind,
       final TokenServiceParam tokenService,
       final NoRedirectParam noredirectParam,
@@ -1335,6 +1363,14 @@ protected Response get(
       final String js = JsonUtil.toJsonString(diffReport);
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
     }
+    case GETSNAPSHOTDIFFLISTING: {
+      SnapshotDiffReportListing diffReport = cp.getSnapshotDiffReportListing(
+          fullpath, oldSnapshotName.getValue(), snapshotName.getValue(),
+          DFSUtilClient.string2Bytes(snapshotDiffStartPath.getValue()),
+          snapshotDiffIndex.getValue());
+      final String js = JsonUtil.toJsonString(diffReport);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
     case GETSNAPSHOTTABLEDIRECTORYLIST: {
       SnapshottableDirectoryStatus[] snapshottableDirectoryList =
           cp.getSnapshottableDirListing();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java
index b2495c8d6d9d8..3b1e2d608465a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java
@@ -78,21 +78,23 @@ public static class BlockECReconstructionInfo {
     private String[] targetStorageIDs;
     private StorageType[] targetStorageTypes;
     private final byte[] liveBlockIndices;
+    private final byte[] excludeReconstructedIndices;
     private final ErasureCodingPolicy ecPolicy;
 
     public BlockECReconstructionInfo(ExtendedBlock block,
         DatanodeInfo[] sources, DatanodeStorageInfo[] targetDnStorageInfo,
-        byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) {
+        byte[] liveBlockIndices, byte[] excludeReconstructedIndices, ErasureCodingPolicy ecPolicy) {
       this(block, sources, DatanodeStorageInfo
           .toDatanodeInfos(targetDnStorageInfo), DatanodeStorageInfo
           .toStorageIDs(targetDnStorageInfo), DatanodeStorageInfo
-          .toStorageTypes(targetDnStorageInfo), liveBlockIndices, ecPolicy);
+          .toStorageTypes(targetDnStorageInfo), liveBlockIndices,
+          excludeReconstructedIndices, ecPolicy);
     }
 
     public BlockECReconstructionInfo(ExtendedBlock block,
         DatanodeInfo[] sources, DatanodeInfo[] targets,
         String[] targetStorageIDs, StorageType[] targetStorageTypes,
-        byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) {
+        byte[] liveBlockIndices, byte[] excludeReconstructedIndices, ErasureCodingPolicy ecPolicy) {
       this.block = block;
       this.sources = sources;
       this.targets = targets;
@@ -100,6 +102,7 @@ public BlockECReconstructionInfo(ExtendedBlock block,
       this.targetStorageTypes = targetStorageTypes;
       this.liveBlockIndices = liveBlockIndices == null ?
           new byte[]{} : liveBlockIndices;
+      this.excludeReconstructedIndices = excludeReconstructedIndices;
       this.ecPolicy = ecPolicy;
     }
 
@@ -127,6 +130,10 @@ public byte[] getLiveBlockIndices() {
       return liveBlockIndices;
     }
 
+    public byte[] getExcludeReconstructedIndices() {
+      return excludeReconstructedIndices;
+    }
+
     public ErasureCodingPolicy getErasureCodingPolicy() {
       return ecPolicy;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
index f60d748dc9f56..21fc09b183b49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.StorageType;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java
index 8d6384e700d5e..4ee930e67b724 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java
@@ -36,14 +36,23 @@ public class HeartbeatResponse {
   private final RollingUpgradeStatus rollingUpdateStatus;
 
   private final long fullBlockReportLeaseId;
-  
+
+  private final boolean isSlownode;
+
   public HeartbeatResponse(DatanodeCommand[] cmds,
       NNHAStatusHeartbeat haStatus, RollingUpgradeStatus rollingUpdateStatus,
       long fullBlockReportLeaseId) {
+    this(cmds, haStatus, rollingUpdateStatus, fullBlockReportLeaseId, false);
+  }
+
+  public HeartbeatResponse(DatanodeCommand[] cmds,
+      NNHAStatusHeartbeat haStatus, RollingUpgradeStatus rollingUpdateStatus,
+      long fullBlockReportLeaseId, boolean isSlownode) {
     commands = cmds;
     this.haStatus = haStatus;
     this.rollingUpdateStatus = rollingUpdateStatus;
     this.fullBlockReportLeaseId = fullBlockReportLeaseId;
+    this.isSlownode = isSlownode;
   }
   
   public DatanodeCommand[] getCommands() {
@@ -61,4 +70,8 @@ public RollingUpgradeStatus getRollingUpdateStatus() {
   public long getFullBlockReportLeaseId() {
     return fullBlockReportLeaseId;
   }
+
+  public boolean getIsSlownode() {
+    return isSlownode;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
index 8b94a11594146..93125a22dbd24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
@@ -31,7 +31,7 @@
 import org.apache.hadoop.util.VersionInfo;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * NamespaceInfo is returned by the name-node in reply 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
index 391078f558509..8cd7ebd1cefac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
@@ -21,7 +21,7 @@
 import java.util.List;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
index 64dec8bbc5c3c..ec3837424cc20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSBlockMoveTaskHandler.java
@@ -80,11 +80,15 @@ public class ExternalSPSBlockMoveTaskHandler implements BlockMoveTaskHandler {
   private Daemon movementTrackerThread;
   private final SPSService service;
   private final BlockDispatcher blkDispatcher;
+  private final int maxRetry;
 
   public ExternalSPSBlockMoveTaskHandler(Configuration conf,
       NameNodeConnector nnc, SPSService spsService) {
     int moverThreads = conf.getInt(DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY,
         DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT);
+    maxRetry = conf.getInt(
+        DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_KEY,
+        DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT);
     moveExecutor = initializeBlockMoverThreadPool(moverThreads);
     mCompletionServ = new ExecutorCompletionService<>(moveExecutor);
     this.nnc = nnc;
@@ -151,7 +155,7 @@ public void submitMoveTask(BlockMovingInfo blkMovingInfo) throws IOException {
     // during block movement assignment logic. In the internal movement,
     // remaining space is bookkeeping at the DatanodeDescriptor, please refer
     // IntraSPSNameNodeBlockMoveTaskHandler#submitMoveTask implementation and
-    // updating via the funcation call -
+    // updating via the function call -
     // dn.incrementBlocksScheduled(blkMovingInfo.getTargetStorageType());
     LOG.debug("Received BlockMovingTask {}", blkMovingInfo);
     BlockMovingTask blockMovingTask = new BlockMovingTask(blkMovingInfo);
@@ -195,21 +199,25 @@ private BlockMovementStatus moveBlock() {
 
       final KeyManager km = nnc.getKeyManager();
       Token<BlockTokenIdentifier> accessToken;
-      try {
-        accessToken = km.getAccessToken(eb,
-            new StorageType[]{blkMovingInfo.getTargetStorageType()},
-            new String[0]);
-      } catch (IOException e) {
-        // TODO: handle failure retries
-        LOG.warn(
-            "Failed to move block:{} from src:{} to destin:{} to satisfy "
-                + "storageType:{}",
-            blkMovingInfo.getBlock(), blkMovingInfo.getSource(),
-            blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), e);
-        return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE;
+      int retry = 0;
+      while (retry <= maxRetry) {
+        try {
+          ExternalSPSFaultInjector.getInstance().mockAnException(retry);
+          accessToken = km.getAccessToken(eb,
+              new StorageType[]{blkMovingInfo.getTargetStorageType()},
+              new String[0]);
+          return blkDispatcher.moveBlock(blkMovingInfo, saslClient, eb,
+              new Socket(), km, accessToken);
+        } catch (IOException e) {
+          LOG.warn(
+              "Failed to move block:{} from src:{} to dest:{} to satisfy "
+                  + "storageType:{}, retry: {}",
+              blkMovingInfo.getBlock(), blkMovingInfo.getSource(),
+              blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), retry, e);
+          retry++;
+        }
       }
-      return blkDispatcher.moveBlock(blkMovingInfo, saslClient, eb,
-          new Socket(), km, accessToken);
+      return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE;
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
index 8427e93a70972..78f1dc6bf3e75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
@@ -24,6 +24,7 @@
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -39,10 +40,12 @@
 import org.apache.hadoop.hdfs.server.namenode.sps.Context;
 import org.apache.hadoop.hdfs.server.namenode.sps.FileCollector;
 import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
+import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
 import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeMap;
 import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeWithStorage;
 import org.apache.hadoop.hdfs.server.protocol.BlockStorageMovementCommand.BlockMovingInfo;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.hdfs.server.sps.metrics.ExternalSPSBeanMetrics;
 import org.apache.hadoop.net.NetworkTopology;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,6 +65,7 @@ public class ExternalSPSContext implements Context {
   private final FileCollector fileCollector;
   private final BlockMoveTaskHandler externalHandler;
   private final BlockMovementListener blkMovementListener;
+  private ExternalSPSBeanMetrics spsBeanMetrics;
 
   public ExternalSPSContext(SPSService service, NameNodeConnector nnc) {
     this.service = service;
@@ -208,4 +212,17 @@ public void notifyMovementTriedBlocks(Block[] moveAttemptFinishedBlks) {
       LOG.info("Movement attempted blocks", actualBlockMovements);
     }
   }
+
+  public void initMetrics(StoragePolicySatisfier sps) {
+    spsBeanMetrics = new ExternalSPSBeanMetrics(sps);
+  }
+
+  public void closeMetrics() {
+    spsBeanMetrics.close();
+  }
+
+  @VisibleForTesting
+  public ExternalSPSBeanMetrics getSpsBeanMetrics() {
+    return spsBeanMetrics;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java
new file mode 100644
index 0000000000000..5ddf1ee3c0f6b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSFaultInjector.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.sps;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+
+import java.io.IOException;
+
+/**
+ * Used to inject certain faults for testing.
+ */
+public class ExternalSPSFaultInjector {
+  @VisibleForTesting
+  private static ExternalSPSFaultInjector instance =
+      new ExternalSPSFaultInjector();
+
+  @VisibleForTesting
+  public static ExternalSPSFaultInjector getInstance() {
+    return instance;
+  }
+
+  @VisibleForTesting
+  public static void setInstance(ExternalSPSFaultInjector instance) {
+    ExternalSPSFaultInjector.instance = instance;
+  }
+
+  @VisibleForTesting
+  public void mockAnException(int retry) throws IOException {
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
index 15cdc6eb47bcf..40d01bda4a3f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
@@ -38,6 +38,7 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -47,8 +48,7 @@
  */
 @InterfaceAudience.Private
 public final class ExternalStoragePolicySatisfier {
-  public static final Logger LOG = LoggerFactory
-      .getLogger(ExternalStoragePolicySatisfier.class);
+  public static final Logger LOG = LoggerFactory.getLogger(ExternalStoragePolicySatisfier.class);
 
   private ExternalStoragePolicySatisfier() {
     // This is just a class to start and run external sps.
@@ -59,6 +59,7 @@ private ExternalStoragePolicySatisfier() {
    */
   public static void main(String[] args) throws Exception {
     NameNodeConnector nnc = null;
+    ExternalSPSContext context = null;
     try {
       StringUtils.startupShutdownMessage(StoragePolicySatisfier.class, args,
           LOG);
@@ -68,9 +69,10 @@ public static void main(String[] args) throws Exception {
       StoragePolicySatisfier sps = new StoragePolicySatisfier(spsConf);
       nnc = getNameNodeConnector(spsConf);
 
-      ExternalSPSContext context = new ExternalSPSContext(sps, nnc);
+      context = new ExternalSPSContext(sps, nnc);
       sps.init(context);
       sps.start(StoragePolicySatisfierMode.EXTERNAL);
+      context.initMetrics(sps);
       if (sps != null) {
         sps.join();
       }
@@ -81,6 +83,11 @@ public static void main(String[] args) throws Exception {
       if (nnc != null) {
         nnc.close();
       }
+      if (context!= null) {
+        if (context.getSpsBeanMetrics() != null) {
+          context.closeMetrics();
+        }
+      }
     }
   }
 
@@ -96,20 +103,25 @@ private static void secureLogin(Configuration conf)
         socAddr.getHostName());
   }
 
-  private static NameNodeConnector getNameNodeConnector(Configuration conf)
-      throws IOException, InterruptedException {
+  public static NameNodeConnector getNameNodeConnector(Configuration conf)
+      throws InterruptedException {
     final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
     final Path externalSPSPathId = HdfsServerConstants.MOVER_ID_PATH;
+    String serverName = ExternalStoragePolicySatisfier.class.getSimpleName();
     while (true) {
       try {
         final List<NameNodeConnector> nncs = NameNodeConnector
             .newNameNodeConnectors(namenodes,
-                ExternalStoragePolicySatisfier.class.getSimpleName(),
+                serverName,
                 externalSPSPathId, conf,
                 NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS);
         return nncs.get(0);
       } catch (IOException e) {
         LOG.warn("Failed to connect with namenode", e);
+        if (e.getMessage().equals("Another " + serverName + " is running.")) {
+          ExitUtil.terminate(-1,
+              "Exit immediately because another " + serverName + " is running");
+        }
         Thread.sleep(3000); // retry the connection after few secs
       }
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java
new file mode 100644
index 0000000000000..adab0e40328a9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.sps.metrics;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.namenode.sps.ItemInfo;
+import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
+import org.apache.hadoop.metrics2.util.MBeans;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import javax.management.StandardMBean;
+import java.util.HashSet;
+
+/**
+ * Expose the ExternalSPS metrics.
+ */
+public class ExternalSPSBeanMetrics implements ExternalSPSMXBean {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ExternalSPSBeanMetrics.class);
+
+  /**
+   * ExternalSPS bean.
+   */
+  private ObjectName externalSPSBeanName;
+  private StoragePolicySatisfier storagePolicySatisfier;
+
+  public ExternalSPSBeanMetrics(StoragePolicySatisfier sps) {
+    try {
+      this.storagePolicySatisfier = sps;
+      StandardMBean bean = new StandardMBean(this, ExternalSPSMXBean.class);
+      this.externalSPSBeanName = MBeans.register("ExternalSPS", "ExternalSPS", bean);
+      LOG.info("Registered ExternalSPS MBean: {}", this.externalSPSBeanName);
+    } catch (NotCompliantMBeanException e) {
+      throw new RuntimeException("Bad externalSPS MBean setup", e);
+    }
+  }
+
+  /**
+   * Unregister the JMX interfaces.
+   */
+  public void close() {
+    if (externalSPSBeanName != null) {
+      MBeans.unregister(externalSPSBeanName);
+      externalSPSBeanName = null;
+    }
+  }
+
+  @Override
+  public int getProcessingQueueSize() {
+    return storagePolicySatisfier.processingQueueSize();
+  }
+
+  @VisibleForTesting
+  public void updateProcessingQueueSize() {
+    storagePolicySatisfier.getStorageMovementQueue()
+        .add(new ItemInfo(0, 1, 1));
+  }
+
+  @Override
+  public int getMovementFinishedBlocksCount() {
+    return storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocksCount();
+  }
+
+  @VisibleForTesting
+  public void updateMovementFinishedBlocksCount() {
+    storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocks()
+        .add(new Block(1));
+  }
+
+  @Override
+  public int getAttemptedItemsCount() {
+    return storagePolicySatisfier.getAttemptedItemsMonitor().getAttemptedItemsCount();
+  }
+
+  @VisibleForTesting
+  public void updateAttemptedItemsCount() {
+    storagePolicySatisfier.getAttemptedItemsMonitor().getStorageMovementAttemptedItems()
+        .add(new StoragePolicySatisfier.AttemptedItemInfo(0, 1,
+            1, new HashSet<>(), 1));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersionWatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java
similarity index 56%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersionWatcher.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java
index b00f13a0aba61..12492fbbf0d9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineVersionWatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java
@@ -15,33 +15,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-package org.apache.hadoop.yarn.server.timeline;
+package org.apache.hadoop.hdfs.server.sps.metrics;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
 
+/**
+ * This is the JMX management interface for ExternalSPS information.
+ * End users shouldn't be implementing these interfaces, and instead
+ * access this information through the JMX APIs.
+ */
 @InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TimelineVersionWatcher extends TestWatcher {
-  static final float DEFAULT_TIMELINE_VERSION = 1.0f;
-  private TimelineVersion version;
+@InterfaceStability.Stable
+public interface ExternalSPSMXBean {
+
+  /**
+   * Gets the queue size of StorageMovementNeeded.
+   *
+   * @return the queue size of StorageMovementNeeded.
+   */
+  int getProcessingQueueSize();
 
-  @Override
-  protected void starting(Description description) {
-    version = description.getAnnotation(TimelineVersion.class);
-  }
+  /**
+   * Gets the count of movement finished blocks.
+   *
+   * @return the count of movement finished blocks.
+   */
+  int getMovementFinishedBlocksCount();
 
   /**
-   * @return the version number of timeline server for the current test (using
-   * timeline server v1.0 by default)
+   * Gets the count of attempted items.
+   *
+   * @return the count of attempted items.
    */
-  public float getTimelineVersion() {
-    if(version == null) {
-      return DEFAULT_TIMELINE_VERSION;
-    }
-    return version.value();
-  }
-}
+  int getAttemptedItemsCount();
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java
new file mode 100644
index 0000000000000..a6d26f69d37f9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package provides the ability to expose external SPS metrics to JMX.
+ */
+package org.apache.hadoop.hdfs.server.sps.metrics;
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
index f619fb40f55d9..36f4f026ce5f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java
@@ -18,7 +18,7 @@
  */
 package org.apache.hadoop.hdfs.tools;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 4f57040867588..1d3e8da77a3a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -35,8 +35,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
@@ -105,7 +107,7 @@
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * This class provides some DFS administrative access shell commands.
@@ -433,7 +435,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep
    */
   private static final String commonUsageSummary =
     "\t[-report [-live] [-dead] [-decommissioning] " +
-    "[-enteringmaintenance] [-inmaintenance]]\n" +
+      "[-enteringmaintenance] [-inmaintenance] [-slownodes]]\n" +
     "\t[-safemode <enter | leave | get | wait | forceExit>]\n" +
     "\t[-saveNamespace [-beforeShutdown]]\n" +
     "\t[-rollEdits]\n" +
@@ -451,8 +453,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep
     "\t[-refreshSuperUserGroupsConfiguration]\n" +
     "\t[-refreshCallQueue]\n" +
     "\t[-refresh <host:ipc_port> <key> [arg1..argn]\n" +
-    "\t[-reconfig <namenode|datanode> <host:ipc_port> " +
-      "<start|status|properties>]\n" +
+      "\t[-reconfig <namenode|datanode> <host:ipc_port|livenodes> <start|status|properties>]\n" +
     "\t[-printTopology]\n" +
       "\t[-refreshNamenodes datanode_host:ipc_port]\n" +
       "\t[-getVolumeReport datanode_host:ipc_port]\n" +
@@ -587,11 +588,13 @@ public void report(String[] argv, int i) throws IOException {
         StringUtils.popOption("-enteringmaintenance", args);
     final boolean listInMaintenance =
         StringUtils.popOption("-inmaintenance", args);
+    final boolean listSlowNodes =
+        StringUtils.popOption("-slownodes", args);
 
 
     // If no filter flags are found, then list all DN types
     boolean listAll = (!listLive && !listDead && !listDecommissioning
-        && !listEnteringMaintenance && !listInMaintenance);
+        && !listEnteringMaintenance && !listInMaintenance && !listSlowNodes);
 
     if (listAll || listLive) {
       printDataNodeReports(dfs, DatanodeReportType.LIVE, listLive, "Live");
@@ -615,6 +618,10 @@ public void report(String[] argv, int i) throws IOException {
       printDataNodeReports(dfs, DatanodeReportType.IN_MAINTENANCE,
           listInMaintenance, "In maintenance");
     }
+
+    if (listAll || listSlowNodes) {
+      printSlowDataNodeReports(dfs, listSlowNodes, "Slow");
+    }
   }
 
   private static void printDataNodeReports(DistributedFileSystem dfs,
@@ -632,6 +639,20 @@ private static void printDataNodeReports(DistributedFileSystem dfs,
     }
   }
 
+  private static void printSlowDataNodeReports(DistributedFileSystem dfs, boolean listNodes,
+      String nodeState) throws IOException {
+    DatanodeInfo[] nodes = dfs.getSlowDatanodeStats();
+    if (nodes.length > 0 || listNodes) {
+      System.out.println(nodeState + " datanodes (" + nodes.length + "):\n");
+    }
+    if (nodes.length > 0) {
+      for (DatanodeInfo dn : nodes) {
+        System.out.println(dn.getDatanodeReport());
+        System.out.println();
+      }
+    }
+  }
+
   /**
    * Safe mode maintenance command.
    * Usage: hdfs dfsadmin -safemode [enter | leave | get | wait | forceExit]
@@ -882,7 +903,7 @@ public int saveNamespace(String[] argv) throws IOException {
                 + proxy.getAddress());
           }
         }catch (IOException ioe){
-          System.out.println("Save namespace failed for " +
+          System.err.println("Save namespace failed for " +
               proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -939,7 +960,7 @@ public int restoreFailedStorage(String arg) throws IOException {
           System.out.println("restoreFailedStorage is set to " + res + " for "
               + proxy.getAddress());
         } catch (IOException ioe){
-          System.out.println("restoreFailedStorage failed for "
+          System.err.println("restoreFailedStorage failed for "
               + proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -982,7 +1003,7 @@ public int refreshNodes() throws IOException {
           System.out.println("Refresh nodes successful for " +
               proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Refresh nodes failed for " +
+          System.err.println("Refresh nodes failed for " +
               proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1038,7 +1059,7 @@ public int listOpenFiles(String[] argv) throws IOException {
       openFilesRemoteIterator = dfs.listOpenFiles(openFilesTypes, path);
       printOpenFiles(openFilesRemoteIterator);
     } catch (IOException ioe){
-      System.out.println("List open files failed.");
+      System.err.println("List open files failed.");
       throw ioe;
     }
     return 0;
@@ -1148,7 +1169,7 @@ private void printHelp(String cmd) {
       commonUsageSummary;
 
     String report ="-report [-live] [-dead] [-decommissioning] "
-        + "[-enteringmaintenance] [-inmaintenance]:\n" +
+        + "[-enteringmaintenance] [-inmaintenance] [-slownodes]:\n" +
         "\tReports basic filesystem information and statistics. \n" +
         "\tThe dfs usage can be different from \"du\" usage, because it\n" +
         "\tmeasures raw space used by replication, checksums, snapshots\n" +
@@ -1224,12 +1245,14 @@ private void printHelp(String cmd) {
 
     String refreshCallQueue = "-refreshCallQueue: Reload the call queue from config\n";
 
-    String reconfig = "-reconfig <namenode|datanode> <host:ipc_port> " +
+    String reconfig = "-reconfig <namenode|datanode> <host:ipc_port|livenodes> " +
         "<start|status|properties>:\n" +
         "\tStarts or gets the status of a reconfiguration operation, \n" +
         "\tor gets a list of reconfigurable properties.\n" +
-
-        "\tThe second parameter specifies the node type\n";
+        "\tThe second parameter specifies the node type\n" +
+        "\tThe third parameter specifies host address. For start or status, \n" +
+        "\tdatanode supports livenodes as third parameter, which will start \n" +
+        "\tor retrieve reconfiguration on all live datanodes.";
     String genericRefresh = "-refresh: Arguments are <hostname:ipc_port>" +
             " <resource_identifier> [arg1..argn]\n" +
             "\tTriggers a runtime-refresh of the resource specified by " +
@@ -1470,7 +1493,7 @@ public int finalizeUpgrade() throws IOException {
           System.out.println("Finalize upgrade successful for " +
               proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Finalize upgrade failed for " +
+          System.err.println("Finalize upgrade failed for " +
               proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1599,7 +1622,7 @@ public int metaSave(String[] argv, int idx) throws IOException {
             throw re;
           }
         } catch (IOException ioe) {
-          System.out.println("Created metasave file " + pathname
+          System.err.println("Created metasave file " + pathname
               + " in the log directory of namenode " + proxy.getAddress()
               + " failed");
           exceptions.add(ioe);
@@ -1624,40 +1647,45 @@ public int metaSave(String[] argv, int idx) throws IOException {
    * @throws IOException If an error while getting datanode report
    */
   public int printTopology() throws IOException {
-      DistributedFileSystem dfs = getDFS();
-      final DatanodeInfo[] report = dfs.getDataNodeStats();
-
-      // Build a map of rack -> nodes from the datanode report
-      HashMap<String, TreeSet<String> > tree = new HashMap<String, TreeSet<String>>();
-      for(DatanodeInfo dni : report) {
-        String location = dni.getNetworkLocation();
-        String name = dni.getName();
-        
-        if(!tree.containsKey(location)) {
-          tree.put(location, new TreeSet<String>());
-        }
+    DistributedFileSystem dfs = getDFS();
+    final DatanodeInfo[] report = dfs.getDataNodeStats();
+
+    // Build a map of rack -> nodes from the datanode report
+    Map<String, HashMap<String, String>> map = new HashMap<>();
+    for(DatanodeInfo dni : report) {
+      String location = dni.getNetworkLocation();
+      String name = dni.getName();
+      String dnState = dni.getAdminState().toString();
         
-        tree.get(location).add(name);
+      if(!map.containsKey(location)) {
+        map.put(location, new HashMap<>());
       }
+
+      Map<String, String> node = map.get(location);
+      node.put(name, dnState);
+    }
       
-      // Sort the racks (and nodes) alphabetically, display in order
-      ArrayList<String> racks = new ArrayList<String>(tree.keySet());
-      Collections.sort(racks);
+    // Sort the racks (and nodes) alphabetically, display in order
+    List<String> racks = new ArrayList<>(map.keySet());
+    Collections.sort(racks);
       
-      for(String r : racks) {
-        System.out.println("Rack: " + r);
-        TreeSet<String> nodes = tree.get(r);
-
-        for(String n : nodes) {
-          System.out.print("   " + n);
-          String hostname = NetUtils.getHostNameOfIP(n);
-          if(hostname != null)
-            System.out.print(" (" + hostname + ")");
-          System.out.println();
+    for(String r : racks) {
+      System.out.println("Rack: " + r);
+      Map<String, String> nodes = map.get(r);
+
+      for(Map.Entry<String, String> entry : nodes.entrySet()) {
+        String n = entry.getKey();
+        System.out.print("   " + n);
+        String hostname = NetUtils.getHostNameOfIP(n);
+        if(hostname != null) {
+          System.out.print(" (" + hostname + ")");
         }
-
+        System.out.print(" " + entry.getValue());
         System.out.println();
       }
+
+      System.out.println();
+    }
     return 0;
   }
   
@@ -1698,7 +1726,7 @@ public int refreshServiceAcl() throws IOException {
           System.out.println("Refresh service acl successful for "
               + proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Refresh service acl failed for "
+          System.err.println("Refresh service acl failed for "
               + proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1751,7 +1779,7 @@ public int refreshUserToGroupsMappings() throws IOException {
           System.out.println("Refresh user to groups mapping successful for "
               + proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Refresh user to groups mapping failed for "
+          System.err.println("Refresh user to groups mapping failed for "
               + proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1806,7 +1834,7 @@ public int refreshSuperUserGroupsConfiguration() throws IOException {
           System.out.println("Refresh super user groups configuration " +
               "successful for " + proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Refresh super user groups configuration " +
+          System.err.println("Refresh super user groups configuration " +
               "failed for " + proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1855,7 +1883,7 @@ public int refreshCallQueue() throws IOException {
           System.out.println("Refresh call queue successful for "
               + proxy.getAddress());
         }catch (IOException ioe){
-          System.out.println("Refresh call queue failed for "
+          System.err.println("Refresh call queue failed for "
               + proxy.getAddress());
           exceptions.add(ioe);
         }
@@ -1877,15 +1905,15 @@ public int refreshCallQueue() throws IOException {
     return 0;
   }
 
-  public int reconfig(String[] argv, int i) throws IOException {
+  public int reconfig(String[] argv, int i) throws IOException, InterruptedException {
     String nodeType = argv[i];
     String address = argv[i + 1];
     String op = argv[i + 2];
 
     if ("start".equals(op)) {
-      return startReconfiguration(nodeType, address, System.out, System.err);
+      return startReconfigurationUtil(nodeType, address, System.out, System.err);
     } else if ("status".equals(op)) {
-      return getReconfigurationStatus(nodeType, address, System.out, System.err);
+      return getReconfigurationStatusUtil(nodeType, address, System.out, System.err);
     } else if ("properties".equals(op)) {
       return getReconfigurableProperties(nodeType, address, System.out,
           System.err);
@@ -1895,12 +1923,57 @@ public int reconfig(String[] argv, int i) throws IOException {
   }
 
   int startReconfiguration(final String nodeThpe, final String address)
-      throws IOException {
-    return startReconfiguration(nodeThpe, address, System.out, System.err);
+      throws IOException, InterruptedException {
+    return startReconfigurationUtil(nodeThpe, address, System.out, System.err);
+  }
+
+  int startReconfigurationUtil(final String nodeType, final String address, final PrintStream out,
+      final PrintStream err) throws IOException, InterruptedException {
+    if (!"livenodes".equals(address)) {
+      return startReconfiguration(nodeType, address, out, err);
+    }
+    if (!"datanode".equals(nodeType)) {
+      err.println("Only datanode type supports reconfiguration in bulk.");
+      return 1;
+    }
+    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    DistributedFileSystem dfs = getDFS();
+    DatanodeInfo[] nodes = dfs.getDataNodeStats(DatanodeReportType.LIVE);
+    AtomicInteger successCount = new AtomicInteger();
+    AtomicInteger failCount = new AtomicInteger();
+    if (nodes != null) {
+      for (DatanodeInfo node : nodes) {
+        executorService.submit(() -> {
+          int status = startReconfiguration(nodeType, node.getIpcAddr(false), out, err);
+          if (status == 0) {
+            successCount.incrementAndGet();
+          } else {
+            failCount.incrementAndGet();
+          }
+        });
+      }
+      while ((successCount.get() + failCount.get()) < nodes.length) {
+        Thread.sleep(1000);
+      }
+      executorService.shutdown();
+      if (!executorService.awaitTermination(1, TimeUnit.MINUTES)) {
+        err.println("Executor service could not be terminated in 60s. Please wait for"
+            + " sometime before the system cools down.");
+      }
+      out.println("Starting of reconfiguration task successful on " + successCount.get()
+          + " nodes, failed on " + failCount.get() + " nodes.");
+      if (failCount.get() == 0) {
+        return 0;
+      } else {
+        return 1;
+      }
+    }
+    err.println("DFS datanode stats could not be retrieved.");
+    return 1;
   }
 
   int startReconfiguration(final String nodeType, final String address,
-      final PrintStream out, final PrintStream err) throws IOException {
+      final PrintStream out, final PrintStream err) {
     String outMsg = null;
     String errMsg = null;
     int ret = 0;
@@ -1941,8 +2014,53 @@ int startReconfigurationDispatch(final String nodeType,
     }
   }
 
-  int getReconfigurationStatus(final String nodeType, final String address,
-      final PrintStream out, final PrintStream err) throws IOException {
+  int getReconfigurationStatusUtil(final String nodeType, final String address,
+      final PrintStream out, final PrintStream err) throws IOException, InterruptedException {
+    if (!"livenodes".equals(address)) {
+      return getReconfigurationStatus(nodeType, address, out, err);
+    }
+    if (!"datanode".equals(nodeType)) {
+      err.println("Only datanode type supports reconfiguration in bulk.");
+      return 1;
+    }
+    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    DistributedFileSystem dfs = getDFS();
+    DatanodeInfo[] nodes = dfs.getDataNodeStats(DatanodeReportType.LIVE);
+    AtomicInteger successCount = new AtomicInteger();
+    AtomicInteger failCount = new AtomicInteger();
+    if (nodes != null) {
+      for (DatanodeInfo node : nodes) {
+        executorService.submit(() -> {
+          int status = getReconfigurationStatus(nodeType, node.getIpcAddr(false), out, err);
+          if (status == 0) {
+            successCount.incrementAndGet();
+          } else {
+            failCount.incrementAndGet();
+          }
+        });
+      }
+      while ((successCount.get() + failCount.get()) < nodes.length) {
+        Thread.sleep(1000);
+      }
+      executorService.shutdown();
+      if (!executorService.awaitTermination(1, TimeUnit.MINUTES)) {
+        err.println("Executor service could not be terminated in 60s. Please wait for"
+            + " sometime before the system cools down.");
+      }
+      out.println("Retrieval of reconfiguration status successful on " + successCount.get()
+          + " nodes, failed on " + failCount.get() + " nodes.");
+      if (failCount.get() == 0) {
+        return 0;
+      } else {
+        return 1;
+      }
+    }
+    err.println("DFS datanode stats could not be retrieved.");
+    return 1;
+  }
+
+  int getReconfigurationStatus(final String nodeType, final String address, final PrintStream out,
+      final PrintStream err) {
     String outMsg = null;
     String errMsg = null;
     ReconfigurationTaskStatus status = null;
@@ -2126,7 +2244,7 @@ private static void printUsage(String cmd) {
     if ("-report".equals(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
           + " [-report] [-live] [-dead] [-decommissioning]"
-          + " [-enteringmaintenance] [-inmaintenance]");
+          + " [-enteringmaintenance] [-inmaintenance] [-slownodes]");
     } else if ("-safemode".equals(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
           + " [-safemode enter | leave | get | wait | forceExit]");
@@ -2188,7 +2306,7 @@ private static void printUsage(String cmd) {
                          + " [-refreshCallQueue]");
     } else if ("-reconfig".equals(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
-          + " [-reconfig <namenode|datanode> <host:ipc_port> "
+          + " [-reconfig <namenode|datanode> <host:ipc_port|livenodes> "
           + "<start|status|properties>]");
     } else if ("-refresh".equals(cmd)) {
       System.err.println("Usage: hdfs dfsadmin"
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
index 15c63732f7a69..a462cafb458c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
@@ -23,7 +23,7 @@
 import java.util.Collection;
 import java.util.Map;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSortedMap;
 import org.apache.commons.cli.CommandLine;
@@ -247,7 +247,7 @@ private boolean checkSupportObserver(HAServiceTarget target) {
   }
 
   private int transitionToObserver(final CommandLine cmd)
-      throws IOException, ServiceFailedException {
+      throws IOException {
     String[] argv = cmd.getArgs();
     if (argv.length != 1) {
       errOut.println("transitionToObserver: incorrect number of arguments");
@@ -262,8 +262,13 @@ private int transitionToObserver(final CommandLine cmd)
     if (!checkManualStateManagementOK(target)) {
       return -1;
     }
-    HAServiceProtocol proto = target.getProxy(getConf(), 0);
-    HAServiceProtocolHelper.transitionToObserver(proto, createReqInfo());
+    try {
+      HAServiceProtocol proto = target.getProxy(getConf(), 0);
+      HAServiceProtocolHelper.transitionToObserver(proto, createReqInfo());
+    } catch (ServiceFailedException e) {
+      errOut.println("transitionToObserver failed! " + e.getLocalizedMessage());
+      return -1;
+    }
     return 0;
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index db30133d0c028..acb370e421a30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -27,6 +27,7 @@
 import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -137,8 +139,17 @@ public DFSck(Configuration conf, PrintStream out) throws IOException {
     super(conf);
     this.ugi = UserGroupInformation.getCurrentUser();
     this.out = out;
+    int connectTimeout = (int) conf.getTimeDuration(
+        HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
+        HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
+        TimeUnit.MILLISECONDS);
+    int readTimeout = (int) conf.getTimeDuration(
+        HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
+        HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
+        TimeUnit.MILLISECONDS);
+
     this.connectionFactory = URLConnectionFactory
-        .newDefaultURLConnectionFactory(conf);
+        .newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
     this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index f5967e15a63d0..7116c2578ca59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -24,15 +24,41 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -69,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool {
       new VerifyMetaCommand(),
       new ComputeMetaCommand(),
       new RecoverLeaseCommand(),
+      new VerifyECCommand(),
       new HelpCommand()
   };
 
@@ -387,6 +414,232 @@ int run(List<String> args) throws IOException {
     }
   }
 
+  /**
+   * The command for verifying the correctness of erasure coding on an erasure coded file.
+   */
+  private class VerifyECCommand extends DebugCommand {
+    private DFSClient client;
+    private int dataBlkNum;
+    private int parityBlkNum;
+    private int cellSize;
+    private boolean useDNHostname;
+    private CachingStrategy cachingStrategy;
+    private int stripedReadBufferSize;
+    private CompletionService<Integer> readService;
+    private RawErasureEncoder encoder;
+    private BlockReader[] blockReaders;
+
+
+    VerifyECCommand() {
+      super("verifyEC",
+          "verifyEC -file <file> [-blockId <blk_Id>] [-skipFailureBlocks]",
+          "  -file Verify HDFS erasure coding on all block groups of the file." +
+              System.lineSeparator() +
+          "  -skipFailureBlocks specify will skip any block group failures during verify," +
+          "  and continues verify all block groups of the file," + System.lineSeparator() +
+          "  the default is not to skip failure blocks." + System.lineSeparator() +
+          "  -blockId specify blk_Id to verify for a specific one block group.");
+    }
+
+    int run(List<String> args) throws IOException {
+      if (args.size() < 2) {
+        System.out.println(usageText);
+        System.out.println(helpText + System.lineSeparator());
+        return 1;
+      }
+      String file = StringUtils.popOptionWithArgument("-file", args);
+      Path path = new Path(file);
+      DistributedFileSystem dfs = AdminHelper.getDFS(getConf());
+      this.client = dfs.getClient();
+
+      FileStatus fileStatus;
+      try {
+        fileStatus = dfs.getFileStatus(path);
+      } catch (FileNotFoundException e) {
+        System.err.println("File " + file + " does not exist.");
+        return 1;
+      }
+
+      if (!fileStatus.isFile()) {
+        System.err.println("File " + file + " is not a regular file.");
+        return 1;
+      }
+      if (!dfs.isFileClosed(path)) {
+        System.err.println("File " + file + " is not closed.");
+        return 1;
+      }
+      this.useDNHostname = getConf().getBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
+          DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
+      this.cachingStrategy = CachingStrategy.newDefaultStrategy();
+      this.stripedReadBufferSize = getConf().getInt(
+          DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
+          DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT);
+
+      LocatedBlocks locatedBlocks = client.getLocatedBlocks(file, 0, fileStatus.getLen());
+      if (locatedBlocks.getErasureCodingPolicy() == null) {
+        System.err.println("File " + file + " is not erasure coded.");
+        return 1;
+      }
+      ErasureCodingPolicy ecPolicy = locatedBlocks.getErasureCodingPolicy();
+      this.dataBlkNum = ecPolicy.getNumDataUnits();
+      this.parityBlkNum = ecPolicy.getNumParityUnits();
+      this.cellSize = ecPolicy.getCellSize();
+      this.encoder = CodecUtil.createRawEncoder(getConf(), ecPolicy.getCodecName(),
+          new ErasureCoderOptions(dataBlkNum, parityBlkNum));
+      int blockNum = dataBlkNum + parityBlkNum;
+      this.readService = new ExecutorCompletionService<>(
+          DFSUtilClient.getThreadPoolExecutor(blockNum, blockNum, 60,
+              new LinkedBlockingQueue<>(), "read-", false));
+      this.blockReaders = new BlockReader[blockNum];
+
+      String needToVerifyBlockId = StringUtils.popOptionWithArgument("-blockId", args);
+      boolean skipFailureBlocks = StringUtils.popOption("-skipFailureBlocks", args);
+      boolean isHealthy = true;
+
+      for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
+        String blockName = locatedBlock.getBlock().getBlockName();
+        if (needToVerifyBlockId == null || needToVerifyBlockId.equals(blockName)) {
+          System.out.println("Checking EC block group: " + blockName);
+          LocatedStripedBlock blockGroup = (LocatedStripedBlock) locatedBlock;
+
+          try {
+            verifyBlockGroup(blockGroup);
+            System.out.println("Status: OK");
+          } catch (Exception e) {
+            System.err.println("Status: ERROR, message: " + e.getMessage());
+            isHealthy = false;
+            if (!skipFailureBlocks) {
+              break;
+            }
+          } finally {
+            closeBlockReaders();
+          }
+
+          if (needToVerifyBlockId != null) {
+            break;
+          }
+        }
+      }
+      if (isHealthy) {
+        if (needToVerifyBlockId == null) {
+          System.out.println("\nAll EC block group status: OK");
+        }
+        return 0;
+      }
+      return 1;
+    }
+
+    private void verifyBlockGroup(LocatedStripedBlock blockGroup) throws Exception {
+      final LocatedBlock[] indexedBlocks = StripedBlockUtil.parseStripedBlockGroup(blockGroup,
+          cellSize, dataBlkNum, parityBlkNum);
+
+      int blockNumExpected = Math.min(dataBlkNum,
+          (int) ((blockGroup.getBlockSize() - 1) / cellSize + 1)) + parityBlkNum;
+      if (blockGroup.getBlockIndices().length < blockNumExpected) {
+        throw new Exception("Block group is under-erasure-coded.");
+      }
+
+      long maxBlockLen = 0L;
+      DataChecksum checksum = null;
+      for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
+        LocatedBlock block = indexedBlocks[i];
+        if (block == null) {
+          blockReaders[i] = null;
+          continue;
+        }
+        if (block.getBlockSize() > maxBlockLen) {
+          maxBlockLen = block.getBlockSize();
+        }
+        BlockReader blockReader = createBlockReader(block.getBlock(),
+            block.getLocations()[0], block.getBlockToken());
+        if (checksum == null) {
+          checksum = blockReader.getDataChecksum();
+        } else {
+          assert checksum.equals(blockReader.getDataChecksum());
+        }
+        blockReaders[i] = blockReader;
+      }
+      assert checksum != null;
+      int bytesPerChecksum = checksum.getBytesPerChecksum();
+      int bufferSize = stripedReadBufferSize < bytesPerChecksum ? bytesPerChecksum :
+          stripedReadBufferSize - stripedReadBufferSize % bytesPerChecksum;
+      final ByteBuffer[] buffers = new ByteBuffer[dataBlkNum + parityBlkNum];
+      final ByteBuffer[] outputs = new ByteBuffer[parityBlkNum];
+      for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
+        buffers[i] = ByteBuffer.allocate(bufferSize);
+      }
+      for (int i = 0; i < parityBlkNum; i++) {
+        outputs[i] = ByteBuffer.allocate(bufferSize);
+      }
+      long positionInBlock = 0L;
+      while (positionInBlock < maxBlockLen) {
+        final int toVerifyLen = (int) Math.min(bufferSize, maxBlockLen - positionInBlock);
+        List<Future<Integer>> futures = new ArrayList<>(dataBlkNum + parityBlkNum);
+        for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
+          final int fi = i;
+          futures.add(this.readService.submit(() -> {
+            BlockReader blockReader = blockReaders[fi];
+            ByteBuffer buffer = buffers[fi];
+            buffer.clear();
+            buffer.limit(toVerifyLen);
+            int readLen = 0;
+            if (blockReader != null) {
+              int toRead = buffer.remaining();
+              while (readLen < toRead) {
+                int nread = blockReader.read(buffer);
+                if (nread <= 0) {
+                  break;
+                }
+                readLen += nread;
+              }
+            }
+            while (buffer.hasRemaining()) {
+              buffer.put((byte) 0);
+            }
+            buffer.flip();
+            return readLen;
+          }));
+        }
+        for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
+          futures.get(i).get(1, TimeUnit.MINUTES);
+        }
+        ByteBuffer[] inputs = new ByteBuffer[dataBlkNum];
+        System.arraycopy(buffers, 0, inputs, 0, dataBlkNum);
+        for (int i = 0; i < parityBlkNum; i++) {
+          outputs[i].clear();
+          outputs[i].limit(toVerifyLen);
+        }
+        this.encoder.encode(inputs, outputs);
+        for (int i = 0; i < parityBlkNum; i++) {
+          if (!buffers[dataBlkNum + i].equals(outputs[i])) {
+            throw new Exception("EC compute result not match.");
+          }
+        }
+        positionInBlock += toVerifyLen;
+      }
+    }
+
+    private BlockReader createBlockReader(ExtendedBlock block, DatanodeInfo dnInfo,
+                                          Token<BlockTokenIdentifier> token) throws IOException {
+      InetSocketAddress dnAddress = NetUtils.createSocketAddr(dnInfo.getXferAddr(useDNHostname));
+      Peer peer = client.newConnectedPeer(dnAddress, token, dnInfo);
+      return BlockReaderRemote.newBlockReader(
+          "dummy", block, token, 0,
+          block.getNumBytes(), true, "", peer, dnInfo,
+          null, cachingStrategy, -1, getConf());
+    }
+
+    private void closeBlockReaders() {
+      for (int i = 0; i < blockReaders.length; i++) {
+        if (blockReaders[i] != null) {
+          IOUtils.closeStream(blockReaders[i]);
+          blockReaders[i] = null;
+        }
+      }
+    }
+
+  }
+
   /**
    * The command for getting help about other commands.
    */
@@ -459,9 +712,9 @@ private void printUsage() {
       if (!command.name.equals("help")) {
         System.out.println(command.usageText);
       }
-      System.out.println();
-      ToolRunner.printGenericCommandUsage(System.out);
     }
+    System.out.println();
+    ToolRunner.printGenericCommandUsage(System.out);
   }
 
   public static void main(String[] argsArray) throws Exception {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
index 78a1bc08f1471..cb3a1e4a83bfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java
@@ -33,7 +33,7 @@
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.net.NetUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java
index 7238c58cb579b..fc5f30e883001 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java
@@ -86,6 +86,10 @@ public OfflineEditsXmlLoader(OfflineEditsVisitor visitor,
   public void loadEdits() throws IOException {
     try {
       XMLReader xr = XMLReaderFactory.createXMLReader();
+      xr.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+      xr.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+      xr.setFeature("http://xml.org/sax/features/external-general-entities", false);
+      xr.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
       xr.setContentHandler(this);
       xr.setErrorHandler(this);
       xr.setDTDHandler(null);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
index ddf7933f0322c..9fabd1887ce48 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
@@ -60,8 +60,8 @@ public class XmlEditsVisitor implements OfflineEditsVisitor {
   public XmlEditsVisitor(OutputStream out)
       throws IOException {
     this.out = out;
-    factory =(SAXTransformerFactory)SAXTransformerFactory.newInstance();
     try {
+      factory = org.apache.hadoop.util.XMLUtils.newSecureSAXTransformerFactory();
       TransformerHandler handler = factory.newTransformerHandler();
       handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
       handler.getTransformer().setOutputProperty(OutputKeys.ENCODING, "UTF-8");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
index f14ee5f930fe0..963406771b18a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
@@ -40,10 +40,10 @@
 import java.util.List;
 import java.util.Map;
 
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
 import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
 import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
@@ -76,7 +76,7 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception {
   @Override
   public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
       throws Exception {
-    if (request.getMethod() != HttpMethod.GET) {
+    if (request.method() != HttpMethod.GET) {
       DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
           METHOD_NOT_ALLOWED);
       resp.headers().set(CONNECTION, CLOSE);
@@ -84,7 +84,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
       return;
     }
 
-    QueryStringDecoder decoder = new QueryStringDecoder(request.getUri());
+    QueryStringDecoder decoder = new QueryStringDecoder(request.uri());
     // check path. throw exception if path doesn't start with WEBHDFS_PREFIX
     String path = getPath(decoder);
     final String op = getOp(decoder);
@@ -140,7 +140,7 @@ public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
   public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
           throws Exception {
     Exception e = cause instanceof Exception ? (Exception) cause : new
-      Exception(cause);
+        Exception(cause);
     final String output = JsonUtil.toJsonString(e);
     ByteBuf content = Unpooled.wrappedBuffer(output.getBytes(Charsets.UTF_8));
     final DefaultFullHttpResponse resp = new DefaultFullHttpResponse(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index af459617667cc..26d8c0ff949b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -52,7 +52,7 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.thirdparty.protobuf.CodedInputStream;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
index 54b183b7b6965..fbeea0f673c0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
@@ -33,7 +33,7 @@
 import org.apache.hadoop.util.LimitInputStream;
 import org.apache.hadoop.util.StringUtils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * This is the tool for analyzing file sizes in the namespace image. In order to
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
index 9ad4b090649b2..6a2049acb4b54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET;
 import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET;
@@ -56,6 +56,7 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.AclEntry;
@@ -147,6 +148,8 @@ private OfflineImageReconstructor(CountingOutputStream out,
       InputStreamReader reader) throws XMLStreamException {
     this.out = out;
     XMLInputFactory factory = XMLInputFactory.newInstance();
+    factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+    factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
     this.events = factory.createXMLEventReader(reader);
     this.sections = new HashMap<>();
     this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor());
@@ -1761,6 +1764,10 @@ private void processXml() throws Exception {
       XMLEvent ev = expectTag("[section header]", true);
       if (ev.getEventType() == XMLStreamConstants.END_ELEMENT) {
         if (ev.asEndElement().getName().getLocalPart().equals("fsimage")) {
+          if(unprocessedSections.size() == 1 && unprocessedSections.contains
+                  (SnapshotDiffSectionProcessor.NAME)){
+            break;
+          }
           throw new IOException("FSImage XML ended prematurely, without " +
               "including section(s) " + StringUtils.join(", ",
               unprocessedSections));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index dbcb452e166aa..fea8a8fdfbb8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -80,6 +80,9 @@ public class OfflineImageViewerPB {
       + "    delimiter. The default delimiter is \\t, though this may be\n"
       + "    changed via the -delimiter argument.\n"
       + "    -sp print storage policy, used by delimiter only.\n"
+      + "    -ec print erasure coding policy, used by delimiter only.\n"
+      + "    -m  defines multiThread to process sub-sections, \n"
+      + "    used by delimiter only.\n"
       + "  * DetectCorruption: Detect potential corruption of the image by\n"
       + "    selectively loading parts of it and actively searching for\n"
       + "    inconsistencies. Outputs a summary of the found corruptions\n"
@@ -100,13 +103,30 @@ public class OfflineImageViewerPB {
       + "                       against image file. (XML|FileDistribution|\n"
       + "                       ReverseXML|Web|Delimited|DetectCorruption)\n"
       + "                       The default is Web.\n"
+      + "-addr <arg>            Specify the address(host:port) to listen.\n"
+      + "                       (localhost:5978 by default). This option is\n"
+      + "                       used with Web processor.\n"
+      + "-maxSize <arg>         Specify the range [0, maxSize] of file sizes\n"
+      + "                       to be analyzed in bytes (128GB by default).\n"
+      + "                       This option is used with FileDistribution processor.\n"
+      + "-step <arg>            Specify the granularity of the distribution in bytes\n"
+      + "                       (2MB by default). This option is used\n"
+      + "                       with FileDistribution processor.\n"
+      + "-format                Format the output result in a human-readable fashion rather\n"
+      + "                       than a number of bytes. (false by default).\n"
+      + "                       This option is used with FileDistribution processor.\n"
       + "-delimiter <arg>       Delimiting string to use with Delimited or \n"
       + "                       DetectCorruption processor. \n"
+      + "-sp                    Whether to print storage policy (default is false). \n"
+      + "                       Is used by Delimited processor only. \n"
+      + "-ec                    Whether to print erasure coding policy (default is false). \n"
+      + "                       Is used by Delimited processor only. \n"
       + "-t,--temp <arg>        Use temporary dir to cache intermediate\n"
       + "                       result to generate DetectCorruption or\n"
       + "                       Delimited outputs. If not set, the processor\n"
       + "                       constructs the namespace in memory \n"
       + "                       before outputting text.\n"
+      + "-m,--multiThread <arg> Use multiThread to process sub-sections.\n"
       + "-h,--help              Display usage information and exit\n";
 
   /**
@@ -131,7 +151,9 @@ private static Options buildOptions() {
     options.addOption("addr", true, "");
     options.addOption("delimiter", true, "");
     options.addOption("sp", false, "");
+    options.addOption("ec", false, "");
     options.addOption("t", "temp", true, "");
+    options.addOption("m", "multiThread", true, "");
 
     return options;
   }
@@ -185,6 +207,7 @@ public static int run(String[] args) throws Exception {
     String delimiter = cmd.getOptionValue("delimiter",
         PBImageTextWriter.DEFAULT_DELIMITER);
     String tempPath = cmd.getOptionValue("t", "");
+    int threads = Integer.parseInt(cmd.getOptionValue("m", "1"));
 
     Configuration conf = new Configuration();
     PrintStream out = null;
@@ -225,17 +248,18 @@ public static int run(String[] args) throws Exception {
         break;
       case "DELIMITED":
         boolean printStoragePolicy = cmd.hasOption("sp");
+        boolean printECPolicy = cmd.hasOption("ec");
         try (PBImageDelimitedTextWriter writer =
             new PBImageDelimitedTextWriter(out, delimiter,
-                tempPath, printStoragePolicy);
-            RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
-          writer.visit(r);
+                tempPath, printStoragePolicy, printECPolicy, threads,
+                outputFile, conf)) {
+          writer.visit(inputFile);
         }
         break;
       case "DETECTCORRUPTION":
         try (PBImageCorruptionDetector detector =
             new PBImageCorruptionDetector(out, delimiter, tempPath)) {
-          detector.visit(new RandomAccessFile(inputFile, "r"));
+          detector.visit(inputFile);
         }
         break;
       default:
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
index 737e7384b9a7c..17593867bd642 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -337,7 +337,7 @@ public void afterOutput() throws IOException {
         if (parentId != -1) {
           entryBuilder.setParentId(parentId);
         }
-        printIfNotEmpty(entryBuilder.build());
+        printIfNotEmpty(serialOutStream(), entryBuilder.build());
       }
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
index 45d42f0396b1a..39fd7658ef54d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
@@ -17,9 +17,12 @@
  */
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeFile;
@@ -46,6 +49,8 @@
 public class PBImageDelimitedTextWriter extends PBImageTextWriter {
   private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm";
   private boolean printStoragePolicy;
+  private boolean printECPolicy;
+  private ErasureCodingPolicyManager ecManager;
 
   static class OutputEntryBuilder {
     private final SimpleDateFormat dateFormatter =
@@ -62,6 +67,7 @@ static class OutputEntryBuilder {
     private long nsQuota = 0;
     private long dsQuota = 0;
     private int storagePolicy = 0;
+    private String ecPolicy = "-";
 
     private String dirPermission = "-";
     private PermissionStatus permissionStatus;
@@ -83,6 +89,13 @@ static class OutputEntryBuilder {
           aclPermission = "+";
         }
         storagePolicy = file.getStoragePolicyID();
+        if (writer.printECPolicy && file.hasErasureCodingPolicyID()) {
+          ErasureCodingPolicy policy = writer.ecManager.
+              getByID((byte) file.getErasureCodingPolicyID());
+          if (policy != null) {
+            ecPolicy = policy.getName();
+          }
+        }
         break;
       case DIRECTORY:
         INodeDirectory dir = inode.getDirectory();
@@ -95,6 +108,12 @@ static class OutputEntryBuilder {
           aclPermission = "+";
         }
         storagePolicy = writer.getStoragePolicy(dir.getXAttrs());
+        if (writer.printECPolicy) {
+          String name= writer.getErasureCodingPolicyName(dir.getXAttrs());
+          if (name != null) {
+            ecPolicy = name;
+          }
+        }
         break;
       case SYMLINK:
         INodeSymlink s = inode.getSymlink();
@@ -134,6 +153,9 @@ public String build() {
       if (writer.printStoragePolicy) {
         writer.append(buffer, storagePolicy);
       }
+      if (writer.printECPolicy) {
+        writer.append(buffer, ecPolicy);
+      }
       return buffer.substring(1);
     }
   }
@@ -146,8 +168,21 @@ public String build() {
   PBImageDelimitedTextWriter(PrintStream out, String delimiter,
                              String tempPath, boolean printStoragePolicy)
       throws IOException {
-    super(out, delimiter, tempPath);
+    this(out, delimiter, tempPath, printStoragePolicy, false, 1, "-", null);
+  }
+
+  PBImageDelimitedTextWriter(PrintStream out, String delimiter,
+                             String tempPath, boolean printStoragePolicy,
+                             boolean printECPolicy, int threads,
+                             String parallelOut, Configuration conf)
+      throws IOException {
+    super(out, delimiter, tempPath, threads, parallelOut);
     this.printStoragePolicy = printStoragePolicy;
+    if (printECPolicy && conf != null) {
+      this.printECPolicy = true;
+      ecManager = ErasureCodingPolicyManager.getInstance();
+      ecManager.init(conf);
+    }
   }
 
   @Override
@@ -181,6 +216,9 @@ public String getHeader() {
     if (printStoragePolicy) {
       append(buffer, "StoragePolicyId");
     }
+    if (printECPolicy) {
+      append(buffer, "ErasureCodingPolicy");
+    }
     return buffer.toString();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
index ef0b168658c68..bd6c860ccf071 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -21,17 +21,27 @@
 import java.io.Closeable;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.io.UnsupportedEncodingException;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -55,6 +65,7 @@
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
 import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.Time;
@@ -66,13 +77,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY;
+
 /**
  * This class reads the protobuf-based fsimage and generates text output
  * for each inode to {@link PBImageTextWriter#out}. The sub-class can override
- * {@link getEntry()} to generate formatted string for each inode.
+ * {@link #getEntry(String, INode)} to generate formatted string for each inode.
  *
  * Since protobuf-based fsimage does not guarantee the order of inodes and
  * directories, PBImageTextWriter runs two-phase scans:
@@ -455,20 +468,22 @@ public String getParentPath(long inode) throws IOException {
         return "/";
       }
       long parent = getFromDirChildMap(inode);
-      if (!dirPathCache.containsKey(parent)) {
-        byte[] bytes = dirMap.get(toBytes(parent));
-        if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
-          // The parent is an INodeReference, which is generated from snapshot.
-          // For delimited oiv tool, no need to print out metadata in snapshots.
-          throw PBImageTextWriter.createIgnoredSnapshotException(inode);
+      byte[] bytes = dirMap.get(toBytes(parent));
+      synchronized (this) {
+        if (!dirPathCache.containsKey(parent)) {
+          if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
+            // The parent is an INodeReference, which is generated from snapshot.
+            // For delimited oiv tool, no need to print out metadata in snapshots.
+            throw PBImageTextWriter.createIgnoredSnapshotException(inode);
+          }
+          String parentName = toString(bytes);
+          String parentPath =
+              new Path(getParentPath(parent),
+                  parentName.isEmpty() ? "/" : parentName).toString();
+          dirPathCache.put(parent, parentPath);
         }
-        String parentName = toString(bytes);
-        String parentPath =
-            new Path(getParentPath(parent),
-                parentName.isEmpty() ? "/" : parentName).toString();
-        dirPathCache.put(parent, parentPath);
+        return dirPathCache.get(parent);
       }
-      return dirPathCache.get(parent);
     }
 
     @Override
@@ -493,9 +508,12 @@ public long getParentId(long id) throws IOException {
   }
 
   private SerialNumberManager.StringTable stringTable;
-  private PrintStream out;
+  private final PrintStream out;
   private MetadataMap metadataMap = null;
   private String delimiter;
+  private File filename;
+  private int numThreads;
+  private String parallelOutputFile;
 
   /**
    * Construct a PB FsImage writer to generate text file.
@@ -503,8 +521,8 @@ public long getParentId(long id) throws IOException {
    * @param tempPath the path to store metadata. If it is empty, store metadata
    *                 in memory instead.
    */
-  PBImageTextWriter(PrintStream out, String delimiter, String tempPath)
-      throws IOException {
+  PBImageTextWriter(PrintStream out, String delimiter, String tempPath,
+      int numThreads, String parallelOutputFile) throws IOException {
     this.out = out;
     this.delimiter = delimiter;
     if (tempPath.isEmpty()) {
@@ -512,6 +530,17 @@ public long getParentId(long id) throws IOException {
     } else {
       metadataMap = new LevelDBMetadataMap(tempPath);
     }
+    this.numThreads = numThreads;
+    this.parallelOutputFile = parallelOutputFile;
+  }
+
+  PBImageTextWriter(PrintStream out, String delimiter, String tempPath)
+      throws IOException {
+    this(out, delimiter, tempPath, 1, "-");
+  }
+
+  protected PrintStream serialOutStream() {
+    return out;
   }
 
   @Override
@@ -562,7 +591,9 @@ void append(StringBuffer buffer, String field) {
    */
   abstract protected void afterOutput() throws IOException;
 
-  public void visit(RandomAccessFile file) throws IOException {
+  public void visit(String filePath) throws IOException {
+    filename = new File(filePath);
+    RandomAccessFile file = new RandomAccessFile(filePath, "r");
     Configuration conf = new Configuration();
     if (!FSImageUtil.checkFileFormat(file)) {
       throw new IOException("Unrecognized FSImage");
@@ -642,21 +673,122 @@ long getParentId(long id) throws IOException {
   private void output(Configuration conf, FileSummary summary,
       FileInputStream fin, ArrayList<FileSummary.Section> sections)
       throws IOException {
+    ArrayList<FileSummary.Section> allINodeSubSections =
+        getINodeSubSections(sections);
+    if (numThreads > 1 && !parallelOutputFile.equals("-") &&
+        allINodeSubSections.size() > 1) {
+      outputInParallel(conf, summary, allINodeSubSections);
+    } else {
+      LOG.info("Serial output due to threads num: {}, parallel output file: {}, " +
+          "subSections: {}.", numThreads, parallelOutputFile, allINodeSubSections.size());
+      outputInSerial(conf, summary, fin, sections);
+    }
+  }
+
+  private void outputInSerial(Configuration conf, FileSummary summary,
+      FileInputStream fin, ArrayList<FileSummary.Section> sections)
+      throws IOException {
     InputStream is;
     long startTime = Time.monotonicNow();
-    out.println(getHeader());
+    serialOutStream().println(getHeader());
     for (FileSummary.Section section : sections) {
       if (SectionName.fromString(section.getName()) == SectionName.INODE) {
         fin.getChannel().position(section.getOffset());
         is = FSImageUtil.wrapInputStreamForCompression(conf,
             summary.getCodec(), new BufferedInputStream(new LimitInputStream(
                 fin, section.getLength())));
-        outputINodes(is);
+        INodeSection s = INodeSection.parseDelimitedFrom(is);
+        LOG.info("Found {} INodes in the INode section", s.getNumInodes());
+        int count = outputINodes(is, serialOutStream());
+        LOG.info("Outputted {} INodes.", count);
       }
     }
     afterOutput();
     long timeTaken = Time.monotonicNow() - startTime;
-    LOG.debug("Time to output inodes: {}ms", timeTaken);
+    LOG.debug("Time to output inodes: {} ms", timeTaken);
+  }
+
+  /**
+   * STEP1: Multi-threaded process sub-sections.
+   * Given n (n>1) threads to process k (k>=n) sections,
+   * output parsed results of each section to tmp file in order.
+   * STEP2: Merge tmp files.
+   */
+  private void outputInParallel(Configuration conf, FileSummary summary,
+      ArrayList<FileSummary.Section> subSections)
+      throws IOException {
+    int nThreads = Integer.min(numThreads, subSections.size());
+    LOG.info("Outputting in parallel with {} sub-sections using {} threads",
+        subSections.size(), nThreads);
+    final CopyOnWriteArrayList<IOException> exceptions = new CopyOnWriteArrayList<>();
+    CountDownLatch latch = new CountDownLatch(subSections.size());
+    ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
+    AtomicLong expectedINodes = new AtomicLong(0);
+    AtomicLong totalParsed = new AtomicLong(0);
+    String codec = summary.getCodec();
+    String[] paths = new String[subSections.size()];
+
+    for (int i = 0; i < subSections.size(); i++) {
+      paths[i] = parallelOutputFile + ".tmp." + i;
+      int index = i;
+      executorService.submit(() -> {
+        LOG.info("Output iNodes of section-{}", index);
+        InputStream is = null;
+        try (PrintStream outStream = new PrintStream(paths[index], "UTF-8")) {
+          long startTime = Time.monotonicNow();
+          is = getInputStreamForSection(subSections.get(index), codec, conf);
+          if (index == 0) {
+            // The first iNode section has a header which must be processed first
+            INodeSection s = INodeSection.parseDelimitedFrom(is);
+            expectedINodes.set(s.getNumInodes());
+          }
+          totalParsed.addAndGet(outputINodes(is, outStream));
+          long timeTaken = Time.monotonicNow() - startTime;
+          LOG.info("Time to output iNodes of section-{}: {} ms", index, timeTaken);
+        } catch (Exception e) {
+          exceptions.add(new IOException(e));
+        } finally {
+          latch.countDown();
+          try {
+            if (is != null) {
+              is.close();
+            }
+          } catch (IOException ioe) {
+            LOG.warn("Failed to close the input stream, ignoring", ioe);
+          }
+        }
+      });
+    }
+
+    try {
+      latch.await();
+    } catch (InterruptedException e) {
+      LOG.error("Interrupted waiting for countdown latch", e);
+      throw new IOException(e);
+    }
+
+    executorService.shutdown();
+    if (exceptions.size() != 0) {
+      LOG.error("Failed to output INode sub-sections, {} exception(s) occurred.",
+          exceptions.size());
+      throw exceptions.get(0);
+    }
+    if (totalParsed.get() != expectedINodes.get()) {
+      throw new IOException("Expected to parse " + expectedINodes + " in parallel, " +
+          "but parsed " + totalParsed.get() + ". The image may be corrupt.");
+    }
+    LOG.info("Completed outputting all INode sub-sections to {} tmp files.",
+        subSections.size());
+
+    try (PrintStream ps = new PrintStream(parallelOutputFile, "UTF-8")) {
+      ps.println(getHeader());
+    }
+
+    // merge tmp files
+    long startTime = Time.monotonicNow();
+    mergeFiles(paths, parallelOutputFile);
+    long timeTaken = Time.monotonicNow() - startTime;
+    LOG.info("Completed all stages. Time to merge files: {} ms", timeTaken);
   }
 
   protected PermissionStatus getPermission(long perm) {
@@ -763,22 +895,27 @@ protected void buildNamespace(InputStream in, List<Long> refIdList)
     LOG.info("Scanned {} INode directories to build namespace.", count);
   }
 
-  void printIfNotEmpty(String line) {
+  void printIfNotEmpty(PrintStream outStream, String line) {
     if (!line.isEmpty()) {
-      out.println(line);
+      outStream.println(line);
     }
   }
 
-  private void outputINodes(InputStream in) throws IOException {
-    INodeSection s = INodeSection.parseDelimitedFrom(in);
-    LOG.info("Found {} INodes in the INode section", s.getNumInodes());
+  private int outputINodes(InputStream in, PrintStream outStream)
+      throws IOException {
     long ignored = 0;
     long ignoredSnapshots = 0;
-    for (int i = 0; i < s.getNumInodes(); ++i) {
+    // As the input stream is a LimitInputStream, the reading will stop when
+    // EOF is encountered at the end of the stream.
+    int count = 0;
+    while (true) {
       INode p = INode.parseDelimitedFrom(in);
+      if (p == null) {
+        break;
+      }
       try {
         String parentPath = metadataMap.getParentPath(p.getId());
-        printIfNotEmpty(getEntry(parentPath, p));
+        printIfNotEmpty(outStream, getEntry(parentPath, p));
       } catch (IOException ioe) {
         ignored++;
         if (!(ioe instanceof IgnoreSnapshotException)) {
@@ -790,16 +927,16 @@ private void outputINodes(InputStream in) throws IOException {
           }
         }
       }
-
-      if (LOG.isDebugEnabled() && i % 100000 == 0) {
-        LOG.debug("Outputted {} INodes.", i);
+      count++;
+      if (LOG.isDebugEnabled() && count % 100000 == 0) {
+        LOG.debug("Outputted {} INodes.", count);
       }
     }
     if (ignored > 0) {
       LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on"
               + " debug log for details", ignored, ignoredSnapshots);
     }
-    LOG.info("Outputted {} INodes.", s.getNumInodes());
+    return count;
   }
 
   private static IgnoreSnapshotException createIgnoredSnapshotException(
@@ -822,4 +959,98 @@ public int getStoragePolicy(
     }
     return HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
   }
+
+  private ArrayList<FileSummary.Section> getINodeSubSections(
+      ArrayList<FileSummary.Section> sections) {
+    ArrayList<FileSummary.Section> subSections = new ArrayList<>();
+    Iterator<FileSummary.Section> iter = sections.iterator();
+    while (iter.hasNext()) {
+      FileSummary.Section s = iter.next();
+      if (SectionName.fromString(s.getName()) == SectionName.INODE_SUB) {
+        subSections.add(s);
+      }
+    }
+    return subSections;
+  }
+
+  /**
+   * Given a FSImage FileSummary.section, return a LimitInput stream set to
+   * the starting position of the section and limited to the section length.
+   * @param section The FileSummary.Section containing the offset and length
+   * @param compressionCodec The compression codec in use, if any
+   * @return An InputStream for the given section
+   * @throws IOException
+   */
+  private InputStream getInputStreamForSection(FileSummary.Section section,
+      String compressionCodec, Configuration conf)
+      throws IOException {
+    // channel of RandomAccessFile is not thread safe, use File
+    FileInputStream fin = new FileInputStream(filename);
+    try {
+      FileChannel channel = fin.getChannel();
+      channel.position(section.getOffset());
+      InputStream in = new BufferedInputStream(new LimitInputStream(fin,
+          section.getLength()));
+
+      in = FSImageUtil.wrapInputStreamForCompression(conf,
+          compressionCodec, in);
+      return in;
+    } catch (IOException e) {
+      fin.close();
+      throw e;
+    }
+  }
+
+  /**
+   * @param srcPaths Source files of contents to be merged
+   * @param resultPath Merged file path
+   * @throws IOException
+   */
+  public static void mergeFiles(String[] srcPaths, String resultPath)
+      throws IOException {
+    if (srcPaths == null || srcPaths.length < 1) {
+      LOG.warn("no source files to merge.");
+      return;
+    }
+
+    File[] files = new File[srcPaths.length];
+    for (int i = 0; i < srcPaths.length; i++) {
+      files[i] = new File(srcPaths[i]);
+    }
+
+    File resultFile = new File(resultPath);
+    try (FileChannel resultChannel =
+             new FileOutputStream(resultFile, true).getChannel()) {
+      for (File file : files) {
+        try (FileChannel src = new FileInputStream(file).getChannel()) {
+          resultChannel.transferFrom(src, resultChannel.size(), src.size());
+        }
+      }
+    }
+
+    for (File file : files) {
+      if (!file.delete() && file.exists()) {
+        LOG.warn("delete tmp file: {} returned false", file);
+      }
+    }
+  }
+
+  public String getErasureCodingPolicyName
+      (INodeSection.XAttrFeatureProto xattrFeatureProto) {
+    List<XAttr> xattrs =
+        FSImageFormatPBINode.Loader.loadXAttrs(xattrFeatureProto, stringTable);
+    for (XAttr xattr : xattrs) {
+      if (XATTR_ERASURECODING_POLICY.contains(xattr.getName())){
+        try{
+          ByteArrayInputStream bIn = new ByteArrayInputStream(xattr.getValue());
+          DataInputStream dIn = new DataInputStream(bIn);
+          return WritableUtils.readString(dIn);
+        } catch (IOException ioException){
+          return null;
+        }
+      }
+    }
+    return null;
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
index 2f030b4734ce9..640065bb5c663 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
@@ -20,7 +20,6 @@
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
index 21a7bb58750b8..298e645d43224 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.util;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
index fc3039af70ec5..9ebfa665e1c1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.util;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.ArrayUtils;
 
 import java.util.Arrays;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java
index 9012ceab3e198..dde7297f56a6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java
@@ -19,7 +19,7 @@
 
 import java.util.Arrays;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Similar to {@link EnumCounters} except that the value type is double.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java
index 2bc63ec77eb29..77ec789058831 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/MD5FileUtils.java
@@ -68,7 +68,7 @@ public static void verifySavedMD5(File dataFile, MD5Hash expectedMD5)
   /**
    * Read the md5 file stored alongside the given data file
    * and match the md5 file content.
-   * @param dataFile the file containing data
+   * @param md5File the file containing md5 data
    * @return a matcher with two matched groups
    *   where group(1) is the md5 string and group(2) is the data file path.
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
index deaeaa43247fd..05c1a06abda23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/RwLock.java
@@ -28,6 +28,12 @@ public interface RwLock {
   /** Release read lock. */
   public void readUnlock();
 
+  /**
+   * Release read lock with operation name.
+   * @param opName Option name.
+   */
+  public void readUnlock(String opName);
+
   /** Check if the current thread holds read lock. */
   public boolean hasReadLock();
 
@@ -40,6 +46,12 @@ public interface RwLock {
   /** Release write lock. */
   public void writeUnlock();
 
+  /**
+   * Release write lock with operation name.
+   * @param opName Option name.
+   */
+  public void writeUnlock(String opName);
+
   /** Check if the current thread holds write lock. */
   public boolean hasWriteLock();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 307b0e8440fe2..b91399e399e41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.web;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileChecksum;
@@ -335,6 +336,17 @@ public static String toJsonString(final LocatedBlocks locatedblocks
       return null;
     }
 
+    final Map<String, Object> m = toJsonMap(locatedblocks);
+    return toJsonString(LocatedBlocks.class, m);
+  }
+
+  /** Convert LocatedBlocks to a Map. */
+  public static Map<String, Object> toJsonMap(final LocatedBlocks locatedblocks)
+      throws IOException {
+    if (locatedblocks == null) {
+      return null;
+    }
+
     final Map<String, Object> m = new TreeMap<String, Object>();
     m.put("fileLength", locatedblocks.getFileLength());
     m.put("isUnderConstruction", locatedblocks.isUnderConstruction());
@@ -342,7 +354,7 @@ public static String toJsonString(final LocatedBlocks locatedblocks
     m.put("locatedBlocks", toJsonArray(locatedblocks.getLocatedBlocks()));
     m.put("lastLocatedBlock", toJsonMap(locatedblocks.getLastLocatedBlock()));
     m.put("isLastBlockComplete", locatedblocks.isLastBlockComplete());
-    return toJsonString(LocatedBlocks.class, m);
+    return m;
   }
 
   /** Convert a ContentSummary to a Json string. */
@@ -577,6 +589,58 @@ private static Object toJsonMap(
     return m;
   }
 
+  public static String toJsonString(SnapshotDiffReportListing diffReport) {
+    return toJsonString(SnapshotDiffReportListing.class.getSimpleName(),
+        toJsonMap(diffReport));
+  }
+
+  private static Object toJsonMap(SnapshotDiffReportListing diffReport) {
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put("lastPath", DFSUtilClient.bytes2String(diffReport.getLastPath()));
+    m.put("lastIndex", diffReport.getLastIndex());
+    m.put("isFromEarlier", diffReport.getIsFromEarlier());
+
+    Object[] modifyList = new Object[diffReport.getModifyList().size()];
+    for (int i = 0; i < diffReport.getModifyList().size(); i++) {
+      modifyList[i] = toJsonMap(diffReport.getModifyList().get(i));
+    }
+    m.put("modifyList", modifyList);
+
+    Object[] createList = new Object[diffReport.getCreateList().size()];
+    for (int i = 0; i < diffReport.getCreateList().size(); i++) {
+      createList[i] = toJsonMap(diffReport.getCreateList().get(i));
+    }
+    m.put("createList", createList);
+
+    Object[] deleteList = new Object[diffReport.getDeleteList().size()];
+    for (int i = 0; i < diffReport.getDeleteList().size(); i++) {
+      deleteList[i] = toJsonMap(diffReport.getDeleteList().get(i));
+    }
+    m.put("deleteList", deleteList);
+
+    return m;
+  }
+
+  private static Object toJsonMap(
+      SnapshotDiffReportListing.DiffReportListingEntry diffReportEntry) {
+    final Map<String, Object> m = new TreeMap<String, Object>();
+    m.put("dirId", diffReportEntry.getDirId());
+    m.put("fileId", diffReportEntry.getFileId());
+
+    if (diffReportEntry.getSourcePath() != null) {
+      m.put("sourcePath",
+          DFSUtilClient.byteArray2String(diffReportEntry.getSourcePath()));
+    }
+
+    if (diffReportEntry.getTargetPath() != null) {
+      m.put("targetPath",
+          DFSUtilClient.byteArray2String(diffReportEntry.getTargetPath()));
+    }
+
+    m.put("isReference", diffReportEntry.isReference());
+    return m;
+  }
+
   public static String toJsonString(
       SnapshottableDirectoryStatus[] snapshottableDirectoryList) {
     if (snapshottableDirectoryList == null) {
@@ -624,7 +688,8 @@ private static Object toJsonMap(
     return m;
   }
 
-  private static Map<String, Object> toJsonMap(
+  @VisibleForTesting
+  static Map<String, Object> toJsonMap(
       final BlockLocation blockLocation) throws IOException {
     if (blockLocation == null) {
       return null;
@@ -644,15 +709,20 @@ private static Map<String, Object> toJsonMap(
 
   public static String toJsonString(BlockLocation[] locations)
       throws IOException {
+    return toJsonString("BlockLocations", JsonUtil.toJsonMap(locations));
+  }
+
+  public static Map<String, Object> toJsonMap(BlockLocation[] locations)
+      throws IOException {
     if (locations == null) {
       return null;
     }
     final Map<String, Object> m = new HashMap<>();
     Object[] blockLocations = new Object[locations.length];
-    for(int i=0; i<locations.length; i++) {
+    for (int i = 0; i < locations.length; i++) {
       blockLocations[i] = toJsonMap(locations[i]);
     }
     m.put(BlockLocation.class.getSimpleName(), blockLocations);
-    return toJsonString("BlockLocations", m);
+    return m;
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto
index e10a8861e6153..9e436ea9b4d6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeLifelineProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
index 4a98f2d01e9e9..c537ce3ae494c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
@@ -223,6 +223,7 @@ message HeartbeatResponseProto {
   optional RollingUpgradeStatusProto rollingUpgradeStatus = 3;
   optional RollingUpgradeStatusProto rollingUpgradeStatusV2 = 4;
   optional uint64 fullBlockReportLeaseId = 5 [ default = 0 ];
+  optional bool isSlownode = 6 [ default = false ];
 }
 
 /**
@@ -407,6 +408,9 @@ message CommitBlockSynchronizationResponseProto {
 message SlowPeerReportProto {
   optional string dataNodeId = 1;
   optional double aggregateLatency = 2;
+  optional double median = 3;
+  optional double mad = 4;
+  optional double upperLimitLatency = 5;
 }
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto
index 6d45a935ee4fd..ecff0e8bd8743 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HAZKInfo.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
index 78607efddab30..e1488258692de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
index 47332a8817bdd..742fd82643df0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto
index e73ca23e92f94..1c78423b40990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterQJournalProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
index 35c401e33e5e6..bfbfc6fd3367b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
index 88d9fbc2e04d0..32cdade055eee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
index e366d1fb8d74b..b0a5a19f8ec7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 syntax = "proto2";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index a49137bf582cc..bf7f99aa1fa6a 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -565,6 +565,17 @@
    </description>
 </property>
 
+<property>
+   <name>dfs.namenode.ip-proxy-users</name>
+   <value></value>
+   <description>A comma separated list of user names that are allowed by the
+     NameNode to specify a different client IP address in the caller context.
+     This is used by Router-Based Federation (RBF) to provide the actual client's
+     IP address to the NameNode, which is critical to preserve data locality when
+     using RBF. If you are using RBF, add the user that runs the routers.
+   </description>
+</property>
+
 <property>
   <name>dfs.namenode.acls.enabled</name>
   <value>true</value>
@@ -1640,7 +1651,7 @@
   <name>dfs.block.scanner.volume.bytes.per.second</name>
   <value>1048576</value>
   <description>
-        If this is 0, the DataNode's block scanner will be disabled.  If this
+        If this is configured less than or equal to zero, the DataNode's block scanner will be disabled.  If this
         is positive, this is the number of bytes per second that the DataNode's
         block scanner will try to scan from each volume.
   </description>
@@ -2483,6 +2494,15 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.max.slowdisks.to.exclude</name>
+  <value>0</value>
+  <description>
+    The number of slow disks that needs to be excluded. By default, this parameter is set to 0,
+    which disables excluding slow disk when choosing volume.
+  </description>
+</property>
+
 <property>
   <name>hadoop.user.group.metrics.percentiles.intervals</name>
   <value></value>
@@ -2982,6 +3002,16 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume</name>
+  <value>4</value>
+  <description>
+    The maximum number of threads per volume used to process async disk
+    operations on the datanode. These threads consume I/O and CPU at the
+    same time. This will affect normal data node operations.
+  </description>
+</property>
+
 <property>
   <name>dfs.cachereport.intervalMsec</name>
   <value>10000</value>
@@ -3348,6 +3378,25 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.client.refresh.read-block-locations.register-automatically</name>
+    <value>true</value>
+    <description>
+      Whether to auto-register all DFSInputStreams for background LocatedBlock refreshes.
+      If false, user must manually register using DFSClient#addLocatedBlocksRefresh(DFSInputStream)
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.client.refresh.read-block-locations.threads</name>
+    <value>5</value>
+    <description>
+      Number of threads to use for refreshing LocatedBlocks of registered
+      DFSInputStreams. If a DFSClient opens many DFSInputStreams, increasing
+      this may help refresh them all in a timely manner.
+    </description>
+  </property>
+
 <property>
   <name>dfs.namenode.lease-recheck-interval-ms</name>
   <value>2000</value>
@@ -3416,30 +3465,6 @@
   </description>
 </property>
 
-<property>
-  <name>dfs.datanode.lock.read.write.enabled</name>
-  <value>true</value>
-  <description>If this is true, the FsDataset lock will be a read write lock. If
-    it is false, all locks will be a write lock.
-    Enabling this should give better datanode throughput, as many read only
-    functions can run concurrently under the read lock, when they would
-    previously have required the exclusive write lock. As the feature is
-    experimental, this switch can be used to disable the shared read lock, and
-    cause all lock acquisitions to use the exclusive write lock.
-  </description>
-</property>
-
-<property>
-  <name>dfs.datanode.lock-reporting-threshold-ms</name>
-  <value>300</value>
-  <description>When thread waits to obtain a lock, or a thread holds a lock for
-    more than the threshold, a log message will be written. Note that
-    dfs.lock.suppress.warning.interval ensures a single log message is
-    emitted per interval for waiting threads and a single message for holding
-    threads to avoid excessive logging.
-  </description>
-</property>
-
 <property>
   <name>dfs.namenode.startup.delay.block.deletion.sec</name>
   <value>0</value>
@@ -3700,7 +3725,7 @@
   <name>dfs.ha.nn.not-become-active-in-safemode</name>
   <value>false</value>
   <description>
-    This will prevent safe mode namenodes to become active while other standby
+    This will prevent safe mode namenodes to become active or observer while other standby
     namenodes might be ready to serve requests when it is set to true.
   </description>
 </property>
@@ -4081,6 +4106,14 @@
     Mover, and StoragePolicySatisfier.
   </description>
 </property>
+<property>
+  <name>dfs.namenode.get-blocks.check.operation</name>
+  <value>true</value>
+  <description>
+    Set false to disable checkOperation and getBlocks for Balancer
+    will route to Standby NameNode for HA mode setup.
+  </description>
+</property>
 <property>
   <name>dfs.balancer.dispatcherThreads</name>
   <value>200</value>
@@ -4220,14 +4253,18 @@
   <value>org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault</value>
   <description>
     Class representing block placement policy for non-striped files.
-    There are four block placement policies currently being supported:
+    There are six block placement policies currently being supported:
     BlockPlacementPolicyDefault, BlockPlacementPolicyWithNodeGroup,
-    BlockPlacementPolicyRackFaultTolerant and BlockPlacementPolicyWithUpgradeDomain.
+    BlockPlacementPolicyRackFaultTolerant, BlockPlacementPolicyWithUpgradeDomain,
+    AvailableSpaceBlockPlacementPolicy and AvailableSpaceRackFaultTolerantBlockPlacementPolicy.
     BlockPlacementPolicyDefault chooses the desired number of targets
     for placing block replicas in a default way. BlockPlacementPolicyWithNodeGroup
     places block replicas on environment with node-group layer. BlockPlacementPolicyRackFaultTolerant
     places the replicas to more racks.
     BlockPlacementPolicyWithUpgradeDomain places block replicas that honors upgrade domain policy.
+    AvailableSpaceBlockPlacementPolicy places block replicas based on space balanced policy.
+    AvailableSpaceRackFaultTolerantBlockPlacementPolicy places block replicas based on
+    space balanced rack fault tolerant policy.
     The details of placing replicas are documented in the javadoc of the corresponding policy classes.
     The default policy is BlockPlacementPolicyDefault, and the corresponding class is
     org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault.
@@ -4667,6 +4704,24 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.datanode.ec.reconstruct.read.bandwidthPerSec</name>
+    <value>0</value>
+    <description>
+      Specifies the maximum amount of bandwidth that the EC reconstruction can utilize for reading.
+      When the bandwidth value is zero, there is no limit.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.datanode.ec.reconstruct.write.bandwidthPerSec</name>
+    <value>0</value>
+    <description>
+      Specifies the maximum amount of bandwidth that the EC reconstruction can utilize for writing.
+      When the bandwidth value is zero, there is no limit.
+    </description>
+  </property>
+
 <property>
   <name>dfs.datanode.fsdataset.factory</name>
   <value></value>
@@ -4890,7 +4945,7 @@
 
 <property>
   <name>dfs.journalnode.edit-cache-size.bytes</name>
-  <value>1048576</value>
+  <value></value>
   <description>
     The size, in bytes, of the in-memory cache of edits to keep on the
     JournalNode. This cache is used to serve edits for tailing via the RPC-based
@@ -4900,6 +4955,22 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.journalnode.edit-cache-size.fraction</name>
+  <value>0.5f</value>
+  <description>
+    This ratio refers to the proportion of the maximum memory of the JVM.
+    Used to calculate the size of the edits cache that is kept in the JournalNode's memory.
+    This config is an alternative to the dfs.journalnode.edit-cache-size.bytes.
+    And it is used to serve edits for tailing via the RPC-based mechanism, and is only
+    enabled when dfs.ha.tail-edits.in-progress is true. Transactions range in size but
+    are around 200 bytes on average, so the default of 1MB can store around 5000 transactions.
+    So we can configure a reasonable value based on the maximum memory. The recommended value
+    is less than 0.9. If we set dfs.journalnode.edit-cache-size.bytes, this parameter will
+    not take effect.
+  </description>
+</property>
+
 <property>
   <name>dfs.journalnode.kerberos.internal.spnego.principal</name>
   <value></value>
@@ -5056,6 +5127,14 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.namenode.audit.log.with.remote.port</name>
+  <value>false</value>
+  <description>
+    If true, adds a port of RPC call to callerContext for all audit log events.
+  </description>
+</property>
+
 <property>
   <name>dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction</name>
   <value>0.6</value>
@@ -5067,6 +5146,18 @@
   </description>
 </property>
 
+  <property>
+    <name>dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance</name>
+    <value>5</value>
+    <description>
+      Only used when the dfs.block.replicator.classname is set to
+      org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy.
+      Special value between 0 and 20, inclusive. if the value is set beyond the scope,
+      this value will be set as 5 by default, Increases tolerance of
+      placing blocks on Datanodes with similar disk space used.
+    </description>
+  </property>
+
 <property>
   <name>
     dfs.namenode.available-space-block-placement-policy.balance-local-node
@@ -5092,7 +5183,17 @@
     high load increases as the value reaches near 0.
   </description>
 </property>
-
+<property>
+  <name>dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance</name>
+  <value>5</value>
+  <description>
+    Only used when the dfs.block.replicator.classname is set to
+    org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy.
+    Special value between 0 and 20, inclusive. if the value is set beyond the scope,
+    this value will be set as 5 by default, Increases tolerance of
+    placing blocks on Datanodes with similar disk space used.
+  </description>
+</property>
 <property>
   <name>dfs.namenode.backup.dnrpc-address</name>
   <value></value>
@@ -5260,6 +5361,14 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.namenode.remove.dead.datanode.batchnum</name>
+  <value>10</value>
+  <description>
+    Maximum number of datanodes removed by HeartbeatManager per scan.
+  </description>
+</property>
+
 <property>
   <name>dfs.namenode.snapshot.capture.openfiles</name>
   <value>false</value>
@@ -5420,6 +5529,14 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.storage.policy.satisfier.move.task.retry.max.attempts</name>
+  <value>3</value>
+  <description>
+    Max retries for moving task to satisfy the block storage policy.
+  </description>
+</property>
+
 <property>
   <name>dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms</name>
   <value>300000</value>
@@ -5481,6 +5598,14 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.pipeline.slownode</name>
+  <value>false</value>
+  <description>
+    If true, allows slownode information to be replied to Client via PipelineAck.
+  </description>
+</property>
+
 <property>
   <name>dfs.qjournal.accept-recovery.timeout.ms</name>
   <value>120000</value>
@@ -6024,12 +6149,21 @@
   </property>
 
   <property>
-    <name>dfs.namenode.block.deletion.increment</name>
-    <value>1000</value>
+    <name>dfs.namenode.block.deletion.lock.threshold.ms</name>
+    <value>50</value>
     <description>
-      The number of block deletion increment.
-      This setting will control the block increment deletion rate to
-      ensure that other waiters on the lock can get in.
+      The limit of single time lock holding duration for the block asynchronous
+      deletion thread.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.namenode.block.deletion.unlock.interval.ms</name>
+    <value>10</value>
+    <description>
+      The sleep interval for yield lock.
+      When the single time lock holding duration of the block asynchronous deletion
+      thread exceeds limit, sleeping to yield lock.
     </description>
   </property>
 
@@ -6123,6 +6257,15 @@
     </description>
   </property>
 
+  <property>
+    <name>dfs.journalnode.handler.count</name>
+    <value>5</value>
+    <description>
+      The number of JournalNode RPC server threads that listen to
+      requests from clients.
+    </description>
+  </property>
+
   <property>
     <name>dfs.namenode.lease-hard-limit-sec</name>
     <value>1200</value>
@@ -6270,4 +6413,56 @@
       Effective with dfs.nameservices.resolution-enabled on.
     </description>
   </property>
+
+  <property>
+    <name>dfs.client.mark.slownode.as.badnode.threshold</name>
+    <value>10</value>
+    <description>
+      The threshold to mark a slownode as a badnode. If we get PipelineAck from
+      a slownode continuously for ${dfs.client.treat.slownode.as.badnode.threshold}
+      times, we should mark it as a badnode.
+    </description>
+  </property>
+
+  <property>
+    <name>dfs.datanode.lockmanager.trace</name>
+    <value>false</value>
+    <description>
+      If this is true, after shut down datanode lock Manager will print all leak
+      thread that not release by lock Manager. Only used for test or trace dead lock
+      problem. In produce default set false, because it's have little performance loss.
+    </description>
+  </property>
+  <property>
+    <name>dfs.client.fsck.connect.timeout</name>
+    <value>60000ms</value>
+    <description>
+      The amount of time the fsck client will wait to connect to the namenode
+      before timing out.
+    </description>
+  </property>
+  <property>
+    <name>dfs.client.fsck.read.timeout</name>
+    <value>60000ms</value>
+    <description>
+      The amount of time the fsck client will wait to read from the namenode
+      before timing out. If the namenode does not report progress more
+      frequently than this time, the client will give up waiting.
+    </description>
+  </property>
+  <property>
+    <name>dfs.client.output.stream.uniq.default.key</name>
+    <value>DEFAULT</value>
+    <description>
+      The default prefix key to construct the uniqKey for one DFSOutputStream.
+      If the namespace is DEFAULT, it's best to change this conf to other value.
+    </description>
+  </property>
+  <property>
+    <name>dfs.client.rbf.observer.read.enable</name>
+    <value>false</value>
+    <description>
+      Enables observer reads for clients. This should only be enabled when clients are using routers.
+    </description>
+  </property>
 </configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
index 7301064651e2b..caab81ef686b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
@@ -71,6 +71,7 @@
 <div class="page-header"><h1>DataNode on <small>{HostName}:{DataPort}</small></h1></div>
 <table class="table table-bordered table-striped">
   <tr><th>Cluster ID:</th><td>{ClusterId}</td></tr>
+  <tr><th>Started:</th><td>{DNStartedTimeInMillis|date_tostring}</td></tr>
   <tr><th>Version:</th><td>{Version}</td></tr>
 </table>
 {/dn}
@@ -125,7 +126,7 @@
   {/dn.VolumeInfo}
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="/static/moment.min.js"></script>
 <script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 72952edd4d71d..4bf2e2bd7f5fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -479,7 +479,7 @@
 </table>
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js">
 </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
 </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
 </script><script type="text/javascript" src="/static/dataTables.bootstrap.js">
@@ -489,6 +489,7 @@
 </script><script type="text/javascript" src="/static/dfs-dust.js">
 </script><script type="text/javascript" src="/static/d3-v4.1.1.min.js">
 </script><script type="text/javascript" src="dfshealth.js">
+</script><script type="text/javascript" src="/static/histogram-hostip.js">
 </script>
 </body>
 </html>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
index 86502ddc132d9..42fb059f09ed1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
@@ -580,45 +580,3 @@
     load_page();
   });
 })();
-
-function open_hostip_list(x0, x1) {
-  close_hostip_list();
-  var ips = new Array();
-  for (var i = 0; i < liveNodes.length; i++) {
-    var dn = liveNodes[i];
-    var index = (dn.usedSpace / dn.capacity) * 100.0;
-    if (index == 0) {
-      index = 1;
-    }
-    //More than 100% do not care,so not record in 95%-100% bar
-    if (index > x0 && index <= x1) {
-      ips.push(dn.infoAddr.split(":")[0]);
-    }
-  }
-  var ipsText = '';
-  for (var i = 0; i < ips.length; i++) {
-    ipsText += ips[i] + '\n';
-  }
-  var histogram_div = document.getElementById('datanode-usage-histogram');
-  histogram_div.setAttribute('style', 'position: relative');
-  var ips_div = document.createElement("textarea");
-  ips_div.setAttribute('id', 'datanode_ips');
-  ips_div.setAttribute('rows', '8');
-  ips_div.setAttribute('cols', '14');
-  ips_div.setAttribute('style', 'position: absolute;top: 0px;right: -38px;');
-  ips_div.setAttribute('readonly', 'readonly');
-  histogram_div.appendChild(ips_div);
-
-  var close_div = document.createElement("div");
-  histogram_div.appendChild(close_div);
-  close_div.setAttribute('id', 'close_ips');
-  close_div.setAttribute('style', 'position: absolute;top: 0px;right: -62px;width:20px;height;20px');
-  close_div.setAttribute('onclick', 'close_hostip_list()');
-  close_div.innerHTML = "X";
-  ips_div.innerHTML = ipsText;
-}
-
-function close_hostip_list() {
-  $("#datanode_ips").remove();
-  $("#close_ips").remove();
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index 3f0509a229700..b51a0ddfd1749 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -311,7 +311,7 @@ <h4 class="modal-title" id="delete-modal-title">Delete</h4>
         </ul>
       </p>
     </script>
-    <script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+    <script type="text/javascript" src="/static/jquery-3.6.0.min.js">
     </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap-editable.min.js">
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalnode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalnode.html
index f661c588e3b89..e12560af2afff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalnode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalnode.html
@@ -71,6 +71,7 @@
   <div class="page-header"><h1>JournalNode on <small>{HostAndPort}</small></h1></div>
   <table class="table table-bordered table-striped">
     <tr><th>Cluster ID:</th><td>{ClusterIds}</td></tr>
+    <tr><th>Started:</th><td>{JNStartedTimeInMillis|date_tostring}</td></tr>
     <tr><th>Version:</th><td>{Version}</td></tr>
   </table>
   {/jn}
@@ -96,7 +97,7 @@
   </table>
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="/static/moment.min.js"></script>
 <script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
index 41a468d4cfd80..a3484fbcb6a88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
@@ -86,7 +86,7 @@
 {/snn}
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js">
 </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
 </script><script type="text/javascript" src="/static/moment.min.js">
 </script><script type="text/javascript" src="/static/dust-full-2.0.0.min.js">
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js
new file mode 100644
index 0000000000000..eff8e506aa845
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+function open_hostip_list(x0, x1) {
+  close_hostip_list();
+  var ips = new Array();
+  for (var i = 0; i < liveNodes.length; i++) {
+    var dn = liveNodes[i];
+    var index = (dn.usedSpace / dn.capacity) * 100.0;
+    if (index == 0) {
+      index = 1;
+    }
+    //More than 100% do not care,so not record in 95%-100% bar
+    if (index > x0 && index <= x1) {
+      ips.push(dn.infoAddr.split(":")[0]);
+    }
+  }
+  var ipsText = '';
+  for (var i = 0; i < ips.length; i++) {
+    ipsText += ips[i] + '\n';
+  }
+  var histogram_div = document.getElementById('datanode-usage-histogram');
+  histogram_div.setAttribute('style', 'position: relative');
+  var ips_div = document.createElement("textarea");
+  ips_div.setAttribute('id', 'datanode_ips');
+  ips_div.setAttribute('rows', '8');
+  ips_div.setAttribute('cols', '14');
+  ips_div.setAttribute('style', 'position: absolute;top: 0px;right: -38px;');
+  ips_div.setAttribute('readonly', 'readonly');
+  histogram_div.appendChild(ips_div);
+
+  var close_div = document.createElement("div");
+  histogram_div.appendChild(close_div);
+  close_div.setAttribute('id', 'close_ips');
+  close_div.setAttribute('style', 'position: absolute;top: 0px;right: -62px;width:20px;height;20px');
+  close_div.setAttribute('onclick', 'close_hostip_list()');
+  close_div.innerHTML = "X";
+  ips_div.innerHTML = ipsText;
+}
+
+function close_hostip_list() {
+  $("#datanode_ips").remove();
+  $("#close_ips").remove();
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
deleted file mode 100644
index b0614034ad3a9..0000000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v3.5.1 | (c) JS Foundation and other contributors | jquery.org/license */
-!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(D).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(D(this,e||[],!1))},not:function(e){return this.pushStack(D(this,e||[],!0))},is:function(e){return!!D(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var j,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^key/,we=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Te=/^([^.]*)(?:\.(.+)|)/;function Ce(){return!0}function Ee(){return!1}function Se(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function ke(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)ke(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Ee;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Ae(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,Ce)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=Te.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=Te.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click",Ce),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ce:Ee,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Ee,isPropagationStopped:Ee,isImmediatePropagationStopped:Ee,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ce,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ce,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ce,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&be.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&we.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Ae(this,e,Se),!1},trigger:function(){return Ae(this,e),!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return ke(this,e,t,n,r)},one:function(e,t,n,r){return ke(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Ee),this.each(function(){S.event.remove(this,e,n,t)})}});var Ne=/<script|<style|<link/i,De=/checked\s*(?:[^=]|=\s*.checked.)/i,je=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function Pe(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&De.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Pe(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),Le)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,He),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(je,""),u,l))}return n}function Re(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Oe(o[r],a[r]);else Oe(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Re(this,e,!0)},remove:function(e){return Re(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Pe(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||qe(this,e).appendChild(e)})},prepend:function(){return Pe(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=qe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ne.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Pe(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Me=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Ie=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},We=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Fe=new RegExp(ne.join("|"),"i");function Be(e,t,n){var r,i,o,a,s=e.style;return(n=n||Ie(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Me.test(a)&&Fe.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function $e(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px",t.style.height="1px",n.style.height="9px",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=3<parseInt(r.height),re.removeChild(e)),a}}))}();var _e=["Webkit","Moz","ms"],ze=E.createElement("div").style,Ue={};function Xe(e){var t=S.cssProps[e]||Ue[e];return t||(e in ze?e:Ue[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=_e.length;while(n--)if((e=_e[n]+t)in ze)return e}(e)||e)}var Ve=/^(none|table(?!-c[ea]).+)/,Ge=/^--/,Ye={position:"absolute",visibility:"hidden",display:"block"},Qe={letterSpacing:"0",fontWeight:"400"};function Je(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Ke(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Ze(e,t,n){var r=Ie(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=Be(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Me.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Ke(e,t,n||(i?"border":"content"),o,r,a)+"px"}function et(e,t,n,r,i){return new et.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Be(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Ge.test(t),l=e.style;if(u||(t=Xe(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Ge.test(t)||(t=Xe(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Be(e,t,r)),"normal"===i&&t in Qe&&(i=Qe[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ve.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Ze(e,u,n):We(e,Ye,function(){return Ze(e,u,n)})},set:function(e,t,n){var r,i=Ie(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Ke(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Ke(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Je(0,t,s)}}}),S.cssHooks.marginLeft=$e(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Be(e,"marginLeft"))||e.getBoundingClientRect().left-We(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Je)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Ie(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=et).prototype={constructor:et,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=et.propHooks[this.prop];return e&&e.get?e.get(this):et.propHooks._default.get(this)},run:function(e){var t,n=et.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):et.propHooks._default.set(this),this}}).init.prototype=et.prototype,(et.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[Xe(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=et.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=et.prototype.init,S.fx.step={};var tt,nt,rt,it,ot=/^(?:toggle|show|hide)$/,at=/queueHooks$/;function st(){nt&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(st):C.setTimeout(st,S.fx.interval),S.fx.tick())}function ut(){return C.setTimeout(function(){tt=void 0}),tt=Date.now()}function lt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ct(e,t,n){for(var r,i=(ft.tweeners[t]||[]).concat(ft.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function ft(o,e,t){var n,a,r=0,i=ft.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=tt||ut(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:tt||ut(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=ft.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ct,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(ft,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],ft.tweeners[n]=ft.tweeners[n]||[],ft.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ot.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ct(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?ft.prefilters.unshift(e):ft.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=ft(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&at.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(lt(r,!0),e,t,n)}}),S.each({slideDown:lt("show"),slideUp:lt("hide"),slideToggle:lt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(tt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),tt=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){nt||(nt=!0,st())},S.fx.stop=function(){nt=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},rt=E.createElement("input"),it=E.createElement("select").appendChild(E.createElement("option")),rt.type="checkbox",y.checkOn=""!==rt.value,y.optSelected=it.selected,(rt=E.createElement("input")).value="t",rt.type="radio",y.radioValue="t"===rt.value;var pt,dt=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?pt:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),pt={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=dt[t]||S.find.attr;dt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=dt[o],dt[o]=r,r=null!=a(e,t,n)?o:null,dt[o]=i),r}});var ht=/^(?:input|select|textarea|button)$/i,gt=/^(?:a|area)$/i;function vt(e){return(e.match(P)||[]).join(" ")}function yt(e){return e.getAttribute&&e.getAttribute("class")||""}function mt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):ht.test(e.nodeName)||gt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,yt(this)))});if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,yt(this)))});if(!arguments.length)return this.attr("class","");if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,yt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=mt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=yt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+vt(yt(n))+" ").indexOf(t))return!0;return!1}});var xt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(xt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:vt(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var bt=/^(?:focusinfocus|focusoutblur)$/,wt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!bt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,bt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,wt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,wt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var Tt=C.location,Ct={guid:Date.now()},Et=/\?/;S.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||S.error("Invalid XML: "+e),t};var St=/\[\]$/,kt=/\r?\n/g,At=/^(?:submit|button|image|reset|file)$/i,Nt=/^(?:input|select|textarea|keygen)/i;function Dt(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||St.test(n)?i(n,t):Dt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)Dt(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)Dt(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&Nt.test(this.nodeName)&&!At.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(kt,"\r\n")}}):{name:t.name,value:n.replace(kt,"\r\n")}}).get()}});var jt=/%20/g,qt=/#.*$/,Lt=/([?&])_=[^&]*/,Ht=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ot=/^(?:GET|HEAD)$/,Pt=/^\/\//,Rt={},Mt={},It="*/".concat("*"),Wt=E.createElement("a");function Ft(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Bt(t,i,o,a){var s={},u=t===Mt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function $t(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Wt.href=Tt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Tt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Tt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":It,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?$t($t(e,S.ajaxSettings),t):$t(S.ajaxSettings,e)},ajaxPrefilter:Ft(Rt),ajaxTransport:Ft(Mt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Ht.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Tt.href)+"").replace(Pt,Tt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Wt.protocol+"//"+Wt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Bt(Rt,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ot.test(v.type),f=v.url.replace(qt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(jt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Et.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Lt,"$1"),o=(Et.test(f)?"&":"?")+"_="+Ct.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+It+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Bt(Mt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var _t={0:200,1223:204},zt=S.ajaxSettings.xhr();y.cors=!!zt&&"withCredentials"in zt,y.ajax=zt=!!zt,S.ajaxTransport(function(i){var o,a;if(y.cors||zt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(_t[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=vt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Gt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Gt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Yt=C.jQuery,Qt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Qt),e&&C.jQuery===S&&(C.jQuery=Yt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
new file mode 100644
index 0000000000000..200b54e470a5d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),j=function(e,t){return e===t&&(l=!0),0},D={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&D.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(j),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(j).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var D,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^([^.]*)(?:\.(.+)|)/;function we(){return!0}function Te(){return!1}function Ce(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ee(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ee(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Te;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Se(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n&&n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,we)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=be.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=be.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click",we),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?we:Te,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Te,isPropagationStopped:Te,isImmediatePropagationStopped:Te,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=we,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=we,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=we,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Se(this,e,Ce),!1},trigger:function(){return Se(this,e),!0},_default:function(){return!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return Ee(this,e,t,n,r)},one:function(e,t,n,r){return Ee(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Te),this.each(function(){S.event.remove(this,e,n,t)})}});var ke=/<script|<style|<link/i,Ae=/checked\s*(?:[^=]|=\s*.checked.)/i,Ne=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function He(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Ae.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),He(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),De)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,qe),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(Ne,""),u,l))}return n}function Oe(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Le(o[r],a[r]);else Le(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Oe(this,e,!0)},remove:function(e){return Oe(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return He(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||je(this,e).appendChild(e)})},prepend:function(){return He(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=je(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!ke.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return He(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Pe=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Re=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Me=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ie=new RegExp(ne.join("|"),"i");function We(e,t,n){var r,i,o,a,s=e.style;return(n=n||Re(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Pe.test(a)&&Ie.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function Fe(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,re.removeChild(e)),a}}))}();var Be=["Webkit","Moz","ms"],$e=E.createElement("div").style,_e={};function ze(e){var t=S.cssProps[e]||_e[e];return t||(e in $e?e:_e[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Be.length;while(n--)if((e=Be[n]+t)in $e)return e}(e)||e)}var Ue=/^(none|table(?!-c[ea]).+)/,Xe=/^--/,Ve={position:"absolute",visibility:"hidden",display:"block"},Ge={letterSpacing:"0",fontWeight:"400"};function Ye(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Qe(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Je(e,t,n){var r=Re(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=We(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Pe.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Qe(e,t,n||(i?"border":"content"),o,r,a)+"px"}function Ke(e,t,n,r,i){return new Ke.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=We(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Xe.test(t),l=e.style;if(u||(t=ze(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Xe.test(t)||(t=ze(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=We(e,t,r)),"normal"===i&&t in Ge&&(i=Ge[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ue.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Je(e,u,n):Me(e,Ve,function(){return Je(e,u,n)})},set:function(e,t,n){var r,i=Re(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Qe(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Qe(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Ye(0,t,s)}}}),S.cssHooks.marginLeft=Fe(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(We(e,"marginLeft"))||e.getBoundingClientRect().left-Me(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Ye)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Re(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=Ke).prototype={constructor:Ke,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=Ke.propHooks[this.prop];return e&&e.get?e.get(this):Ke.propHooks._default.get(this)},run:function(e){var t,n=Ke.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):Ke.propHooks._default.set(this),this}}).init.prototype=Ke.prototype,(Ke.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[ze(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=Ke.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=Ke.prototype.init,S.fx.step={};var Ze,et,tt,nt,rt=/^(?:toggle|show|hide)$/,it=/queueHooks$/;function ot(){et&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(ot):C.setTimeout(ot,S.fx.interval),S.fx.tick())}function at(){return C.setTimeout(function(){Ze=void 0}),Ze=Date.now()}function st(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ut(e,t,n){for(var r,i=(lt.tweeners[t]||[]).concat(lt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function lt(o,e,t){var n,a,r=0,i=lt.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=Ze||at(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:Ze||at(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=lt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ut,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(lt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],lt.tweeners[n]=lt.tweeners[n]||[],lt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],rt.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ut(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?lt.prefilters.unshift(e):lt.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=lt(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&it.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(st(r,!0),e,t,n)}}),S.each({slideDown:st("show"),slideUp:st("hide"),slideToggle:st("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(Ze=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),Ze=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){et||(et=!0,ot())},S.fx.stop=function(){et=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},tt=E.createElement("input"),nt=E.createElement("select").appendChild(E.createElement("option")),tt.type="checkbox",y.checkOn=""!==tt.value,y.optSelected=nt.selected,(tt=E.createElement("input")).value="t",tt.type="radio",y.radioValue="t"===tt.value;var ct,ft=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?ct:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ct={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=ft[t]||S.find.attr;ft[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=ft[o],ft[o]=r,r=null!=a(e,t,n)?o:null,ft[o]=i),r}});var pt=/^(?:input|select|textarea|button)$/i,dt=/^(?:a|area)$/i;function ht(e){return(e.match(P)||[]).join(" ")}function gt(e){return e.getAttribute&&e.getAttribute("class")||""}function vt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):pt.test(e.nodeName)||dt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,gt(this)))});if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,gt(this)))});if(!arguments.length)return this.attr("class","");if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,gt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=vt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=gt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+ht(gt(n))+" ").indexOf(t))return!0;return!1}});var yt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(yt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:ht(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var mt=/^(?:focusinfocus|focusoutblur)$/,xt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!mt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,mt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,xt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,xt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var bt=C.location,wt={guid:Date.now()},Tt=/\?/;S.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||S.error("Invalid XML: "+(n?S.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Ct=/\[\]$/,Et=/\r?\n/g,St=/^(?:submit|button|image|reset|file)$/i,kt=/^(?:input|select|textarea|keygen)/i;function At(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||Ct.test(n)?i(n,t):At(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)At(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)At(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&kt.test(this.nodeName)&&!St.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(Et,"\r\n")}}):{name:t.name,value:n.replace(Et,"\r\n")}}).get()}});var Nt=/%20/g,jt=/#.*$/,Dt=/([?&])_=[^&]*/,qt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Lt=/^(?:GET|HEAD)$/,Ht=/^\/\//,Ot={},Pt={},Rt="*/".concat("*"),Mt=E.createElement("a");function It(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Wt(t,i,o,a){var s={},u=t===Pt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Ft(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Mt.href=bt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:bt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(bt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Rt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Ft(Ft(e,S.ajaxSettings),t):Ft(S.ajaxSettings,e)},ajaxPrefilter:It(Ot),ajaxTransport:It(Pt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=qt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||bt.href)+"").replace(Ht,bt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Mt.protocol+"//"+Mt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Wt(Ot,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Lt.test(v.type),f=v.url.replace(jt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Nt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Tt.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Dt,"$1"),o=(Tt.test(f)?"&":"?")+"_="+wt.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+Rt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Wt(Pt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&S.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Bt={0:200,1223:204},$t=S.ajaxSettings.xhr();y.cors=!!$t&&"withCredentials"in $t,y.ajax=$t=!!$t,S.ajaxTransport(function(i){var o,a;if(y.cors||$t&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Bt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=ht(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Xt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Xt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Vt=C.jQuery,Gt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Gt),e&&C.jQuery===S&&(C.jQuery=Vt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
index 85dd817e13257..b3fd698155643 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
@@ -1,160 +1,4 @@
-/*! DataTables 1.10.7
- * ©2008-2015 SpryMedia Ltd - datatables.net/license
- */
-(function(Ea,Q,k){var P=function(h){function W(a){var b,c,e={};h.each(a,function(d){if((b=d.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=d.replace(b[0],b[2].toLowerCase()),e[c]=d,"o"===b[1]&&W(a[d])});a._hungarianMap=e}function H(a,b,c){a._hungarianMap||W(a);var e;h.each(b,function(d){e=a._hungarianMap[d];if(e!==k&&(c||b[e]===k))"o"===e.charAt(0)?(b[e]||(b[e]={}),h.extend(!0,b[e],b[d]),H(a[e],b[e],c)):b[e]=b[d]})}function P(a){var b=m.defaults.oLanguage,c=a.sZeroRecords;
-!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&E(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&E(a,a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&db(a)}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");
-A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&H(m.models.oSearch,a[b])}function fb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;b&&!h.isArray(b)&&(a.aDataSort=[b])}function gb(a){var a=a.oBrowser,b=h("<div/>").css({position:"absolute",top:0,left:0,height:1,width:1,overflow:"hidden"}).append(h("<div/>").css({position:"absolute",
-top:1,left:1,width:100,overflow:"scroll"}).append(h('<div class="test"/>').css({width:"100%",height:10}))).appendTo("body"),c=b.find(".test");a.bScrollOversize=100===c[0].offsetWidth;a.bScrollbarLeft=1!==Math.round(c.offset().left);b.remove()}function hb(a,b,c,e,d,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;e!==d;)a.hasOwnProperty(e)&&(g=j?b(g,a[e],e,a):a[e],j=!0,e+=f);return g}function Fa(a,b){var c=m.defaults.column,e=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:Q.createElement("th"),sTitle:c.sTitle?
-c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[e],mData:c.mData?c.mData:e,idx:e});a.aoColumns.push(c);c=a.aoPreSearchCols;c[e]=h.extend({},m.models.oSearch,c[e]);ka(a,e,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b],e=a.oClasses,d=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=d.attr("width")||null;var f=(d.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),H(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&
-(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),E(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),E(b,c,"aDataSort"));var g=b.mData,j=R(g),i=b.mRender?R(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b.fnGetData=function(a,b,c){var e=j(a,b,k,c);return i&&b?i(e,b,a,c):e};b.fnSetData=function(a,b,c){return S(g)(a,b,c)};"number"!==typeof g&&
-(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,d.addClass(e.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=e.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=e.sSortableAsc,b.sSortingClassJUI=e.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=e.sSortableDesc,b.sSortingClassJUI=e.sSortJUIDescAllowed):(b.sSortingClass=e.sSortable,b.sSortingClassJUI=e.sSortJUI)}function X(a){if(!1!==a.oFeatures.bAutoWidth){var b=
-a.aoColumns;Ga(a);for(var c=0,e=b.length;c<e;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&Y(a);w(a,null,"column-sizing",[a])}function la(a,b){var c=Z(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function $(a,b){var c=Z(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function aa(a){return Z(a,"bVisible").length}function Z(a,b){var c=[];h.map(a.aoColumns,function(a,d){a[b]&&c.push(d)});return c}function Ha(a){var b=a.aoColumns,c=a.aoData,e=m.ext.type.detect,d,
-f,g,j,i,h,l,q,n;d=0;for(f=b.length;d<f;d++)if(l=b[d],n=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=e.length;g<j;g++){i=0;for(h=c.length;i<h;i++){n[i]===k&&(n[i]=x(a,i,d,"type"));q=e[g](n[i],a);if(!q&&g!==e.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function ib(a,b,c,e){var d,f,g,j,i,o,l=a.aoColumns;if(b)for(d=b.length-1;0<=d;d--){o=b[d];var q=o.targets!==k?o.targets:o.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<
-g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Fa(a);e(q[f],o)}else if("number"===typeof q[f]&&0>q[f])e(l.length+q[f],o);else if("string"===typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&e(j,o)}}if(c){d=0;for(a=c.length;d<a;d++)e(d,c[d])}}function K(a,b,c,e){var d=a.aoData.length,f=h.extend(!0,{},m.models.oRow,{src:c?"dom":"data"});f._aData=b;a.aoData.push(f);for(var b=a.aoColumns,f=0,g=b.length;f<g;f++)c&&Ia(a,d,f,x(a,d,f)),b[f].sType=null;a.aiDisplayMaster.push(d);
-(c||!a.oFeatures.bDeferRender)&&Ja(a,d,c,e);return d}function ma(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,d){c=na(a,d);return K(a,c.data,d,c.cells)})}function x(a,b,c,e){var d=a.iDraw,f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,c=f.fnGetData(g,e,{settings:a,row:b,col:c});if(c===k)return a.iDrawError!=d&&null===j&&(I(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b,4),a.iDrawError=d),j;if((c===g||null===c)&&
-null!==j)c=j;else if("function"===typeof c)return c.call(g);return null===c&&"display"==e?"":c}function Ia(a,b,c,e){a.aoColumns[c].fnSetData(a.aoData[b]._aData,e,{settings:a,row:b,col:c})}function Ka(a){return h.map(a.match(/(\\.|[^\.])+/g),function(a){return a.replace(/\\./g,".")})}function R(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=R(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,
-c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=Ka(f);for(var i=0,h=j.length;i<h;i++){f=j[i].match(ba);g=j[i].match(T);if(f){j[i]=j[i].replace(ba,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");i=0;for(h=a.length;i<h;i++)g.push(c(a[i],b,j));a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(T,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===
-k)return k;a=a[j[i]]}}return a};return function(b,d){return c(b,d,a)}}return function(b){return b[a]}}function S(a){if(h.isPlainObject(a))return S(a._);if(null===a)return function(){};if("function"===typeof a)return function(b,e,d){a(b,"set",e,d)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,e,d){var d=Ka(d),f;f=d[d.length-1];for(var g,j,i=0,h=d.length-1;i<h;i++){g=d[i].match(ba);j=d[i].match(T);if(g){d[i]=d[i].replace(ba,"");a[d[i]]=[];
-f=d.slice();f.splice(0,i+1);g=f.join(".");j=0;for(h=e.length;j<h;j++)f={},b(f,e[j],g),a[d[i]].push(f);return}j&&(d[i]=d[i].replace(T,""),a=a[d[i]](e));if(null===a[d[i]]||a[d[i]]===k)a[d[i]]={};a=a[d[i]]}if(f.match(T))a[f.replace(T,"")](e);else a[f.replace(ba,"")]=e};return function(c,e){return b(c,e,a)}}return function(b,e){b[a]=e}}function La(a){return D(a.aoData,"_aData")}function oa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0}function pa(a,b,c){for(var e=-1,d=0,f=a.length;d<
-f;d++)a[d]==b?e=d:a[d]>b&&a[d]--; -1!=e&&c===k&&a.splice(e,1)}function ca(a,b,c,e){var d=a.aoData[b],f,g=function(c,f){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=x(a,b,f,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===d.src)d._aData=na(a,d,e,e===k?k:d._aData).data;else{var j=d.anCells;if(j)if(e!==k)g(j[e],e);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}d._aSortData=null;d._aFilterData=null;g=a.aoColumns;if(e!==k)g[e].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;
-Ma(d)}}function na(a,b,c,e){var d=[],f=b.firstChild,g,j=0,i,o=a.aoColumns,l=a._rowReadObject,e=e||l?{}:[],q=function(a,b){if("string"===typeof a){var c=a.indexOf("@");-1!==c&&(c=a.substring(c+1),S(a)(e,b.getAttribute(c)))}},a=function(a){if(c===k||c===j)g=o[j],i=h.trim(a.innerHTML),g&&g._bAttrSrc?(S(g.mData._)(e,i),q(g.mData.sort,a),q(g.mData.type,a),q(g.mData.filter,a)):l?(g._setter||(g._setter=S(g.mData)),g._setter(e,i)):e[j]=i;j++};if(f)for(;f;){b=f.nodeName.toUpperCase();if("TD"==b||"TH"==b)a(f),
-d.push(f);f=f.nextSibling}else{d=b.anCells;f=0;for(b=d.length;f<b;f++)a(d[f])}return{data:e,cells:d}}function Ja(a,b,c,e){var d=a.aoData[b],f=d._aData,g=[],j,i,h,l,q;if(null===d.nTr){j=c||Q.createElement("tr");d.nTr=j;d.anCells=g;j._DT_RowIndex=b;Ma(d);l=0;for(q=a.aoColumns.length;l<q;l++){h=a.aoColumns[l];i=c?e[l]:Q.createElement(h.sCellType);g.push(i);if(!c||h.mRender||h.mData!==l)i.innerHTML=x(a,b,l,"display");h.sClass&&(i.className+=" "+h.sClass);h.bVisible&&!c?j.appendChild(i):!h.bVisible&&c&&
-i.parentNode.removeChild(i);h.fnCreatedCell&&h.fnCreatedCell.call(a.oInstance,i,x(a,b,l),f,b,l)}w(a,"aoRowCreatedCallback",null,[j,f,b])}d.nTr.setAttribute("role","row")}function Ma(a){var b=a.nTr,c=a._aData;if(b){c.DT_RowId&&(b.id=c.DT_RowId);if(c.DT_RowClass){var e=c.DT_RowClass.split(" ");a.__rowc=a.__rowc?Na(a.__rowc.concat(e)):e;h(b).removeClass(a.__rowc.join(" ")).addClass(c.DT_RowClass)}c.DT_RowAttr&&h(b).attr(c.DT_RowAttr);c.DT_RowData&&h(b).data(c.DT_RowData)}}function jb(a){var b,c,e,d,
-f,g=a.nTHead,j=a.nTFoot,i=0===h("th, td",g).length,o=a.oClasses,l=a.aoColumns;i&&(d=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],e=h(f.nTh).addClass(f.sClass),i&&e.appendTo(d),a.oFeatures.bSort&&(e.addClass(f.sSortingClass),!1!==f.bSortable&&(e.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Oa(a,f.nTh,b))),f.sTitle!=e.html()&&e.html(f.sTitle),Pa(a,"header")(a,e,f,o);i&&da(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(o.sHeaderTH);
-h(j).find(">tr>th, >tr>td").addClass(o.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function ea(a,b,c){var e,d,f,g=[],j=[],i=a.aoColumns.length,o;if(b){c===k&&(c=!1);e=0;for(d=b.length;e<d;e++){g[e]=b[e].slice();g[e].nTr=b[e].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[e].splice(f,1);j.push([])}e=0;for(d=g.length;e<d;e++){if(a=g[e].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[e].length;f<b;f++)if(o=
-i=1,j[e][f]===k){a.appendChild(g[e][f].cell);for(j[e][f]=1;g[e+i]!==k&&g[e][f].cell==g[e+i][f].cell;)j[e+i][f]=1,i++;for(;g[e][f+o]!==k&&g[e][f].cell==g[e][f+o].cell;){for(c=0;c<i;c++)j[e+c][f+o]=1;o++}h(g[e][f].cell).attr("rowspan",i).attr("colspan",o)}}}}function M(a){var b=w(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,e=a.asStripeClasses,d=e.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==B(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=
-j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,o=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!kb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:o;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ja(a,l);l=q.nTr;if(0!==d){var n=e[c%d];q._sRowStripe!=n&&(h(l).removeClass(q._sRowStripe).addClass(n),q._sRowStripe=n)}w(a,"aoRowCallback",null,[l,q._aData,c,j]);b.push(l);c++}}else c=f.sZeroRecords,
-1==a.iDraw&&"ajax"==B(a)?c=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":d?e[0]:""}).append(h("<td />",{valign:"top",colSpan:aa(a),"class":a.oClasses.sRowEmpty}).html(c))[0];w(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],La(a),g,o,i]);w(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],La(a),g,o,i]);e=h(a.nTBody);e.children().detach();e.append(h(b));w(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=
-!1}}function N(a,b){var c=a.oFeatures,e=c.bFilter;c.bSort&&lb(a);e?fa(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;M(a);a._drawHold=!1}function mb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),e=a.oFeatures,d=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=d[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,o,l,q,n=0;n<f.length;n++){g=
-null;j=f[n];if("<"==j){i=h("<div/>")[0];o=f[n+1];if("'"==o||'"'==o){l="";for(q=2;f[n+q]!=o;)l+=f[n+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(o=l.split("."),i.id=o[0].substr(1,o[0].length-1),i.className=o[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;n+=q}d.append(i);d=h(i)}else if(">"==j)d=d.parent();else if("l"==j&&e.bPaginate&&e.bLengthChange)g=nb(a);else if("f"==j&&e.bFilter)g=ob(a);else if("r"==j&&e.bProcessing)g=pb(a);else if("t"==j)g=qb(a);else if("i"==
-j&&e.bInfo)g=rb(a);else if("p"==j&&e.bPaginate)g=sb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(o=i.length;q<o;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),d.append(g))}c.replaceWith(d)}function da(a,b){var c=h(b).children("tr"),e,d,f,g,j,i,o,l,q,n;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<i;f++){e=c[f];for(d=e.firstChild;d;){if("TD"==d.nodeName.toUpperCase()||"TH"==d.nodeName.toUpperCase()){l=
-1*d.getAttribute("colspan");q=1*d.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;o=g;n=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][o+j]={cell:d,unique:n},a[f+g].nTr=e}d=d.nextSibling}}}function qa(a,b,c){var e=[];c||(c=a.aoHeader,b&&(c=[],da(c,b)));for(var b=0,d=c.length;b<d;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!e[f]||!a.bSortCellsTop))e[f]=c[b][f].cell;return e}function ra(a,b,c){w(a,"aoServerParams","serverParams",[b]);
-if(b&&h.isArray(b)){var e={},d=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(d);c?(c=c[0],e[c]||(e[c]=[]),e[c].push(b.value)):e[b.name]=b.value});b=e}var f,g=a.ajax,j=a.oInstance,i=function(b){w(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var o=h.isFunction(f)?f(b,a):f,b=h.isFunction(f)&&o?o:h.extend(!0,b,o);delete g.data}o={data:b,success:function(b){var c=b.error||b.sError;c&&I(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,
-c){var f=w(a,null,"xhr",[a,null,a.jqXHR]);-1===h.inArray(!0,f)&&("parsererror"==c?I(a,0,"Invalid JSON response",1):4===b.readyState&&I(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;w(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(o,{url:g||a.sAjaxSource})):h.isFunction(g)?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(o,g)),g.data=f)}function kb(a){return a.bAjaxDataGet?
-(a.iDraw++,C(a,!0),ra(a,tb(a),function(b){ub(a,b)}),!1):!0}function tb(a){var b=a.aoColumns,c=b.length,e=a.oFeatures,d=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,o,l,q=U(a);g=a._iDisplayStart;i=!1!==e.bPaginate?a._iDisplayLength:-1;var n=function(a,b){j.push({name:a,value:b})};n("sEcho",a.iDraw);n("iColumns",c);n("sColumns",D(b,"sName").join(","));n("iDisplayStart",g);n("iDisplayLength",i);var k={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:d.sSearch,regex:d.bRegex}};for(g=
-0;g<c;g++)o=b[g],l=f[g],i="function"==typeof o.mData?"function":o.mData,k.columns.push({data:i,name:o.sName,searchable:o.bSearchable,orderable:o.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),n("mDataProp_"+g,i),e.bFilter&&(n("sSearch_"+g,l.sSearch),n("bRegex_"+g,l.bRegex),n("bSearchable_"+g,o.bSearchable)),e.bSort&&n("bSortable_"+g,o.bSortable);e.bFilter&&(n("sSearch",d.sSearch),n("bRegex",d.bRegex));e.bSort&&(h.each(q,function(a,b){k.order.push({column:b.col,dir:b.dir});n("iSortCol_"+a,b.col);
-n("sSortDir_"+a,b.dir)}),n("iSortingCols",q.length));b=m.ext.legacy.ajax;return null===b?a.sAjaxSource?j:k:b?j:k}function ub(a,b){var c=sa(a,b),e=b.sEcho!==k?b.sEcho:b.draw,d=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(e){if(1*e<a.iDraw)return;a.iDraw=1*e}oa(a);a._iRecordsTotal=parseInt(d,10);a._iRecordsDisplay=parseInt(f,10);e=0;for(d=c.length;e<d;e++)K(a,c[e]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;
-M(a);a._bInitComplete||ta(a,b);a.bAjaxDataGet=!0;C(a,!1)}function sa(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?R(c)(b):b}function ob(a){var b=a.oClasses,c=a.sTableId,e=a.oLanguage,d=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=e.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),
-f=function(){var b=!this.value?"":this.value;b!=d.sSearch&&(fa(a,{sSearch:b,bRegex:d.bRegex,bSmart:d.bSmart,bCaseInsensitive:d.bCaseInsensitive}),a._iDisplayStart=0,M(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===B(a)?400:0,i=h("input",b).val(d.sSearch).attr("placeholder",e.sSearchPlaceholder).bind("keyup.DT search.DT input.DT paste.DT cut.DT",g?ua(f,g):f).bind("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==
-Q.activeElement&&i.val(d.sSearch)}catch(f){}});return b[0]}function fa(a,b,c){var e=a.oPreviousSearch,d=a.aoPreSearchCols,f=function(a){e.sSearch=a.sSearch;e.bRegex=a.bRegex;e.bSmart=a.bSmart;e.bCaseInsensitive=a.bCaseInsensitive};Ha(a);if("ssp"!=B(a)){vb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<d.length;b++)wb(a,d[b].sSearch,b,d[b].bEscapeRegex!==k?!d[b].bEscapeRegex:d[b].bRegex,d[b].bSmart,d[b].bCaseInsensitive);xb(a)}else f(b);a.bFiltered=
-!0;w(a,null,"search",[a])}function xb(a){for(var b=m.ext.search,c=a.aiDisplay,e,d,f=0,g=b.length;f<g;f++){for(var j=[],i=0,h=c.length;i<h;i++)d=c[i],e=a.aoData[d],b[f](a,e._aFilterData,d,e._aData,i)&&j.push(d);c.length=0;c.push.apply(c,j)}}function wb(a,b,c,e,d,f){if(""!==b)for(var g=a.aiDisplay,e=Qa(b,e,d,f),d=g.length-1;0<=d;d--)b=a.aoData[g[d]]._aFilterData[c],e.test(b)||g.splice(d,1)}function vb(a,b,c,e,d,f){var e=Qa(b,e,d,f),d=a.oPreviousSearch.sSearch,f=a.aiDisplayMaster,g;0!==m.ext.search.length&&
-(c=!0);g=yb(a);if(0>=b.length)a.aiDisplay=f.slice();else{if(g||c||d.length>b.length||0!==b.indexOf(d)||a.bSorted)a.aiDisplay=f.slice();b=a.aiDisplay;for(c=b.length-1;0<=c;c--)e.test(a.aoData[b[c]]._sFilterRow)||b.splice(c,1)}}function Qa(a,b,c,e){a=b?a:va(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',"")}).join(")(?=.*?")+").*$");return RegExp(a,e?"i":"")}function va(a){return a.replace(Yb,"\\$1")}
-function yb(a){var b=a.aoColumns,c,e,d,f,g,j,i,h,l=m.ext.type.search;c=!1;e=0;for(f=a.aoData.length;e<f;e++)if(h=a.aoData[e],!h._aFilterData){j=[];d=0;for(g=b.length;d<g;d++)c=b[d],c.bSearchable?(i=x(a,e,d,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(wa.innerHTML=i,i=Zb?wa.textContent:wa.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);h._aFilterData=j;h._sFilterRow=j.join("  ");c=!0}return c}
-function zb(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Ab(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function rb(a){var b=a.sTableId,c=a.aanFeatures.i,e=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Bb,sName:"information"}),e.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",b+"_info"));return e[0]}function Bb(a){var b=
-a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,e=a._iDisplayStart+1,d=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Cb(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,e,d,f,g,j));h(b).html(j)}}function Cb(a,b){var c=a.fnFormatNumber,e=a._iDisplayStart+1,d=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===d;return b.replace(/_START_/g,c.call(a,e)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,
-c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(e/d))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/d)))}function ga(a){var b,c,e=a.iInitDisplayStart,d=a.aoColumns,f;c=a.oFeatures;if(a.bInitialised){mb(a);jb(a);ea(a,a.aoHeader);ea(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Ga(a);b=0;for(c=d.length;b<c;b++)f=d[b],f.sWidth&&(f.nTh.style.width=s(f.sWidth));N(a);d=B(a);"ssp"!=d&&("ajax"==d?ra(a,[],function(c){var f=sa(a,c);for(b=0;b<f.length;b++)K(a,f[b]);
-a.iInitDisplayStart=e;N(a);C(a,!1);ta(a,c)},a):(C(a,!1),ta(a)))}else setTimeout(function(){ga(a)},200)}function ta(a,b){a._bInitComplete=!0;b&&X(a);w(a,"aoInitComplete","init",[a,b])}function Ra(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Sa(a);w(a,null,"length",[a,c])}function nb(a){for(var b=a.oClasses,c=a.sTableId,e=a.aLengthMenu,d=h.isArray(e[0]),f=d?e[0]:e,e=d?e[1]:e,d=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)d[0][g]=new Option(e[g],
-f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",d[0].outerHTML));h("select",i).val(a._iDisplayLength).bind("change.DT",function(){Ra(a,h(this).val());M(a)});h(a.nTable).bind("length.dt.DT",function(b,c,f){a===c&&h("select",i).val(f)});return i[0]}function sb(a){var b=a.sPaginationType,c=m.ext.pager[b],e="function"===typeof c,d=function(a){M(a)},b=h("<div/>").addClass(a.oClasses.sPaging+b)[0],
-f=a.aanFeatures;e||c.fnInit(a,b,d);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(e){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),q,l=0;for(q=f.p.length;l<q;l++)Pa(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,d)},sName:"pagination"}));return b}function Ta(a,b,c){var e=a._iDisplayStart,d=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===d?e=0:"number"===typeof b?(e=b*d,e>f&&(e=0)):
-"first"==b?e=0:"previous"==b?(e=0<=d?e-d:0,0>e&&(e=0)):"next"==b?e+d<f&&(e+=d):"last"==b?e=Math.floor((f-1)/d)*d:I(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==e;a._iDisplayStart=e;b&&(w(a,null,"page",[a]),c&&M(a));return b}function pb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");w(a,
-null,"processing",[a,b])}function qb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var e=c.sX,d=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),o=h(b[0].cloneNode(!1)),l=b.children("tfoot");c.sX&&"100%"===b.attr("width")&&b.removeAttr("width");l.length||(l=null);c=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,
-width:e?!e?null:s(e):"100%"}).append(h("<div/>",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({overflow:"auto",height:!d?null:s(d),width:!e?null:s(e)}).append(b));l&&c.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:e?!e?null:s(e):"100%"}).append(h("<div/>",{"class":f.sScrollFootInner}).append(o.removeAttr("id").css("margin-left",
-0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=c.children(),q=b[0],f=b[1],n=l?b[2]:null;if(e)h(f).on("scroll.DT",function(){var a=this.scrollLeft;q.scrollLeft=a;l&&(n.scrollLeft=a)});a.nScrollHead=q;a.nScrollBody=f;a.nScrollFoot=n;a.aoDrawCallback.push({fn:Y,sName:"scrolling"});return c[0]}function Y(a){var b=a.oScroll,c=b.sX,e=b.sXInner,d=b.sY,f=b.iBarWidth,g=h(a.nScrollHead),j=g[0].style,i=g.children("div"),o=i[0].style,l=i.children("table"),i=a.nScrollBody,q=h(i),n=i.style,
-k=h(a.nScrollFoot).children("div"),p=k.children("table"),m=h(a.nTHead),r=h(a.nTable),t=r[0],O=t.style,L=a.nTFoot?h(a.nTFoot):null,ha=a.oBrowser,w=ha.bScrollOversize,v,u,y,x,z,A=[],B=[],C=[],D,E=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};r.children("thead, tfoot").remove();z=m.clone().prependTo(r);v=m.find("tr");y=z.find("tr");z.find("th, td").removeAttr("tabindex");L&&(x=L.clone().prependTo(r),u=L.find("tr"),x=x.find("tr"));
-c||(n.width="100%",g[0].style.width="100%");h.each(qa(a,z),function(b,c){D=la(a,b);c.style.width=a.aoColumns[D].sWidth});L&&G(function(a){a.style.width=""},x);b.bCollapse&&""!==d&&(n.height=q[0].offsetHeight+m[0].offsetHeight+"px");g=r.outerWidth();if(""===c){if(O.width="100%",w&&(r.find("tbody").height()>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(r.outerWidth()-f)}else""!==e?O.width=s(e):g==q.width()&&q.height()<r.height()?(O.width=s(g-f),r.outerWidth()>g-f&&(O.width=s(g))):O.width=
-s(g);g=r.outerWidth();G(E,y);G(function(a){C.push(a.innerHTML);A.push(s(h(a).css("width")))},y);G(function(a,b){a.style.width=A[b]},v);h(y).height(0);L&&(G(E,x),G(function(a){B.push(s(h(a).css("width")))},x),G(function(a,b){a.style.width=B[b]},u),h(x).height(0));G(function(a,b){a.innerHTML='<div class="dataTables_sizing" style="height:0;overflow:hidden;">'+C[b]+"</div>";a.style.width=A[b]},y);L&&G(function(a,b){a.innerHTML="";a.style.width=B[b]},x);if(r.outerWidth()<g){u=i.scrollHeight>i.offsetHeight||
-"scroll"==q.css("overflow-y")?g+f:g;if(w&&(i.scrollHeight>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(u-f);(""===c||""!==e)&&I(a,1,"Possible column misalignment",6)}else u="100%";n.width=s(u);j.width=s(u);L&&(a.nScrollFoot.style.width=s(u));!d&&w&&(n.height=s(t.offsetHeight+f));d&&b.bCollapse&&(n.height=s(d),b=c&&t.offsetWidth>i.offsetWidth?f:0,t.offsetHeight<i.offsetHeight&&(n.height=s(t.offsetHeight+b)));b=r.outerWidth();l[0].style.width=s(b);o.width=s(b);l=r.height()>i.clientHeight||
-"scroll"==q.css("overflow-y");ha="padding"+(ha.bScrollbarLeft?"Left":"Right");o[ha]=l?f+"px":"0px";L&&(p[0].style.width=s(b),k[0].style.width=s(b),k[0].style[ha]=l?f+"px":"0px");q.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)i.scrollTop=0}function G(a,b,c){for(var e=0,d=0,f=b.length,g,j;d<f;){g=b[d].firstChild;for(j=c?c[d].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,e):a(g,e),e++),g=g.nextSibling,j=c?j.nextSibling:null;d++}}function Ga(a){var b=a.nTable,c=a.aoColumns,e=a.oScroll,d=e.sY,f=e.sX,
-g=e.sXInner,j=c.length,e=Z(a,"bVisible"),i=h("th",a.nTHead),o=b.getAttribute("width"),l=b.parentNode,k=!1,n,m;(n=b.style.width)&&-1!==n.indexOf("%")&&(o=n);for(n=0;n<e.length;n++)m=c[e[n]],null!==m.sWidth&&(m.sWidth=Db(m.sWidthOrig,l),k=!0);if(!k&&!f&&!d&&j==aa(a)&&j==i.length)for(n=0;n<j;n++)c[n].sWidth=s(i.eq(n).width());else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var p=h("<tr/>").appendTo(j.find("tbody"));j.find("tfoot th, tfoot td").css("width",
-"");i=qa(a,j.find("thead")[0]);for(n=0;n<e.length;n++)m=c[e[n]],i[n].style.width=null!==m.sWidthOrig&&""!==m.sWidthOrig?s(m.sWidthOrig):"";if(a.aoData.length)for(n=0;n<e.length;n++)k=e[n],m=c[k],h(Eb(a,k)).clone(!1).append(m.sContentPadding).appendTo(p);j.appendTo(l);f&&g?j.width(g):f?(j.css("width","auto"),j.width()<l.offsetWidth&&j.width(l.offsetWidth)):d?j.width(l.offsetWidth):o&&j.width(o);Fb(a,j[0]);if(f){for(n=g=0;n<e.length;n++)m=c[e[n]],d=h(i[n]).outerWidth(),g+=null===m.sWidthOrig?d:parseInt(m.sWidth,
-10)+d-h(i[n]).width();j.width(s(g));b.style.width=s(g)}for(n=0;n<e.length;n++)if(m=c[e[n]],d=h(i[n]).width())m.sWidth=s(d);b.style.width=s(j.css("width"));j.remove()}o&&(b.style.width=s(o));if((o||f)&&!a._reszEvt)b=function(){h(Ea).bind("resize.DT-"+a.sInstance,ua(function(){X(a)}))},a.oBrowser.bScrollOversize?setTimeout(b,1E3):b(),a._reszEvt=!0}function ua(a,b){var c=b!==k?b:200,e,d;return function(){var b=this,g=+new Date,j=arguments;e&&g<e+c?(clearTimeout(d),d=setTimeout(function(){e=k;a.apply(b,
-j)},c)):(e=g,a.apply(b,j))}}function Db(a,b){if(!a)return 0;var c=h("<div/>").css("width",s(a)).appendTo(b||Q.body),e=c[0].offsetWidth;c.remove();return e}function Fb(a,b){var c=a.oScroll;if(c.sX||c.sY)c=!c.sX?c.iBarWidth:0,b.style.width=s(h(b).outerWidth()-c)}function Eb(a,b){var c=Gb(a,b);if(0>c)return null;var e=a.aoData[c];return!e.nTr?h("<td/>").html(x(a,c,b,"display"))[0]:e.anCells[b]}function Gb(a,b){for(var c,e=-1,d=-1,f=0,g=a.aoData.length;f<g;f++)c=x(a,f,b,"display")+"",c=c.replace($b,""),
-c.length>e&&(e=c.length,d=f);return d}function s(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function Hb(){var a=m.__scrollbarWidth;if(a===k){var b=h("<p/>").css({position:"absolute",top:0,left:0,width:"100%",height:150,padding:0,overflow:"scroll",visibility:"hidden"}).appendTo("body"),a=b[0].offsetWidth-b[0].clientWidth;m.__scrollbarWidth=a;b.remove()}return a}function U(a){var b,c,e=[],d=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var o=[];
-f=function(a){a.length&&!h.isArray(a[0])?o.push(a):o.push.apply(o,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<o.length;a++){i=o[a][0];f=d[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=d[g].sType||"string",o[a]._idx===k&&(o[a]._idx=h.inArray(o[a][1],d[g].asSorting)),e.push({src:i,col:g,dir:o[a][1],index:o[a]._idx,type:j,formatter:m.ext.type.order[j+"-pre"]})}return e}function lb(a){var b,c,e=[],d=m.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;
-Ha(a);h=U(a);b=0;for(c=h.length;b<c;b++)j=h[b],j.formatter&&g++,Ib(a,j.col);if("ssp"!=B(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)e[i[b]]=b;g===h.length?i.sort(function(a,b){var c,d,g,j,i=h.length,k=f[a]._aSortData,m=f[b]._aSortData;for(g=0;g<i;g++)if(j=h[g],c=k[j.col],d=m[j.col],c=c<d?-1:c>d?1:0,0!==c)return"asc"===j.dir?c:-c;c=e[a];d=e[b];return c<d?-1:c>d?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,r=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=m[i.col],g=r[i.col],i=d[i.type+
-"-"+i.dir]||d["string-"+i.dir],c=i(c,g),0!==c)return c;c=e[a];g=e[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,e=a.aoColumns,d=U(a),a=a.oLanguage.oAria,f=0,g=e.length;f<g;f++){c=e[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<d.length&&d[0].col==f?(i.setAttribute("aria-sort","asc"==d[0].dir?"ascending":"descending"),c=j[d[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",
-b)}}function Ua(a,b,c,e){var d=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof d[0]&&(d=a.aaSorting=[d]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,D(d,"0")),-1!==c?(b=g(d[c],!0),null===b&&1===d.length&&(b=0),null===b?d.splice(c,1):(d[c][1]=f[b],d[c]._idx=b)):(d.push([b,f[0],0]),d[d.length-1]._idx=0)):d.length&&d[0][0]==b?(b=g(d[0]),d.length=1,d[0][1]=f[b],d[0]._idx=b):(d.length=0,d.push([b,f[0]]),d[0]._idx=
-0);N(a);"function"==typeof e&&e(a)}function Oa(a,b,c,e){var d=a.aoColumns[c];Va(b,{},function(b){!1!==d.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Ua(a,c,b.shiftKey,e);"ssp"!==B(a)&&C(a,!1)},0)):Ua(a,c,b.shiftKey,e))})}function xa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,e=U(a),d=a.oFeatures,f,g;if(d.bSort&&d.bSortClasses){d=0;for(f=b.length;d<f;d++)g=b[d].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>d?d+1:3));d=0;for(f=e.length;d<f;d++)g=e[d].src,h(D(a.aoData,"anCells",
-g)).addClass(c+(2>d?d+1:3))}a.aLastSort=e}function Ib(a,b){var c=a.aoColumns[b],e=m.ext.order[c.sSortDataType],d;e&&(d=e.call(a.oInstance,a,b,$(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||e)f=e?d[j]:x(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function ya(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),
-search:zb(a.oPreviousSearch),columns:h.map(a.aoColumns,function(b,e){return{visible:b.bVisible,search:zb(a.aoPreSearchCols[e])}})};w(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,b)}}function Kb(a){var b,c,e=a.aoColumns;if(a.oFeatures.bStateSave){var d=a.fnStateLoadCallback.call(a.oInstance,a);if(d&&d.time&&(b=w(a,"aoStateLoadParams","stateLoadParams",[a,d]),-1===h.inArray(!1,b)&&(b=a.iStateDuration,!(0<b&&d.time<+new Date-1E3*b)&&e.length===
-d.columns.length))){a.oLoadedState=h.extend(!0,{},d);d.start!==k&&(a._iDisplayStart=d.start,a.iInitDisplayStart=d.start);d.length!==k&&(a._iDisplayLength=d.length);d.order!==k&&(a.aaSorting=[],h.each(d.order,function(b,c){a.aaSorting.push(c[0]>=e.length?[0,c[1]]:c)}));d.search!==k&&h.extend(a.oPreviousSearch,Ab(d.search));b=0;for(c=d.columns.length;b<c;b++){var f=d.columns[b];f.visible!==k&&(e[b].bVisible=f.visible);f.search!==k&&h.extend(a.aoPreSearchCols[b],Ab(f.search))}w(a,"aoStateLoaded","stateLoaded",
-[a,d])}}}function za(a){var b=m.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function I(a,b,c,e){c="DataTables warning: "+(null!==a?"table id="+a.sTableId+" - ":"")+c;e&&(c+=". For more information about this error, please see http://datatables.net/tn/"+e);if(b)Ea.console&&console.log&&console.log(c);else if(b=m.ext,b=b.sErrMode||b.errMode,w(a,null,"error",[a,e,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,e,c)}}function E(a,b,c,e){h.isArray(c)?
-h.each(c,function(c,f){h.isArray(f)?E(a,b,f[0],f[1]):E(a,b,f)}):(e===k&&(e=c),b[c]!==k&&(a[e]=b[c]))}function Lb(a,b,c){var e,d;for(d in b)b.hasOwnProperty(d)&&(e=b[d],h.isPlainObject(e)?(h.isPlainObject(a[d])||(a[d]={}),h.extend(!0,a[d],e)):a[d]=c&&"data"!==d&&"aaData"!==d&&h.isArray(e)?e.slice():e);return a}function Va(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).bind("selectstart.DT",function(){return!1})}function z(a,
-b,c,e){c&&a[b].push({fn:c,sName:e})}function w(a,b,c,e){var d=[];b&&(d=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,e)}));null!==c&&(b=h.Event(c+".dt"),h(a.nTable).trigger(b,e),d.push(b.result));return d}function Sa(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),e=a._iDisplayLength;b>=c&&(b=c-e);b-=b%e;if(-1===e||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=a.renderer,e=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?e[c[b]]||e._:"string"===typeof c?e[c]||e._:e._}function B(a){return a.oFeatures.bServerSide?
-"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function Wa(a,b){var c=[],c=Mb.numbers_length,e=Math.floor(c/2);b<=c?c=V(0,b):a<=e?(c=V(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-e?c=V(b-(c-2),b):(c=V(a-e+2,a+e-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function db(a){h.each({num:function(b){return Aa(b,a)},"num-fmt":function(b){return Aa(b,a,Xa)},"html-num":function(b){return Aa(b,a,Ba)},"html-num-fmt":function(b){return Aa(b,a,Ba,Xa)}},function(b,
-c){u.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(u.type.search[b+a]=u.type.search.html)})}function Nb(a){return function(){var b=[za(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m,u,t,r,v,Ya={},Ob=/[\r\n]/g,Ba=/<.*?>/g,ac=/^[\w\+\-]/,bc=/[\w\+\-]$/,Yb=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Xa=/[',$\u00a3\u20ac\u00a5%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,J=function(a){return!a||!0===a||
-"-"===a?!0:!1},Pb=function(a){var b=parseInt(a,10);return!isNaN(b)&&isFinite(a)?b:null},Qb=function(a,b){Ya[b]||(Ya[b]=RegExp(va(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Ya[b],"."):a},Za=function(a,b,c){var e="string"===typeof a;if(J(a))return!0;b&&e&&(a=Qb(a,b));c&&e&&(a=a.replace(Xa,""));return!isNaN(parseFloat(a))&&isFinite(a)},Rb=function(a,b,c){return J(a)?!0:!(J(a)||"string"===typeof a)?null:Za(a.replace(Ba,""),b,c)?!0:null},D=function(a,b,c){var e=[],d=0,f=a.length;
-if(c!==k)for(;d<f;d++)a[d]&&a[d][b]&&e.push(a[d][b][c]);else for(;d<f;d++)a[d]&&e.push(a[d][b]);return e},ia=function(a,b,c,e){var d=[],f=0,g=b.length;if(e!==k)for(;f<g;f++)a[b[f]][c]&&d.push(a[b[f]][c][e]);else for(;f<g;f++)d.push(a[b[f]][c]);return d},V=function(a,b){var c=[],e;b===k?(b=0,e=a):(e=b,b=a);for(var d=b;d<e;d++)c.push(d);return c},Sb=function(a){for(var b=[],c=0,e=a.length;c<e;c++)a[c]&&b.push(a[c]);return b},Na=function(a){var b=[],c,e,d=a.length,f,g=0;e=0;a:for(;e<d;e++){c=a[e];for(f=
-0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b},A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ba=/\[.*?\]$/,T=/\(\)$/,wa=h("<div>")[0],Zb=wa.textContent!==k,$b=/<.*?>/g;m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(za(this[u.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),e=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===
-k||b)&&c.draw();return e.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],e=c.oScroll;a===k||a?b.draw(!1):(""!==e.sX||""!==e.sY)&&Y(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var e=this.api(!0),a=e.rows(a),d=a.settings()[0],h=d.aoData[a[0][0]];a.remove();b&&b.call(this,d,h);(c===k||c)&&e.draw();return h};
-this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,e,d,h){d=this.api(!0);null===b||b===k?d.search(a,c,e,h):d.column(b).search(a,c,e,h);d.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var e=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==e||"th"==e?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};
-this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===
-k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return za(this[u.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,e,d){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(d===k||d)&&h.columns.adjust();(e===k||e)&&h.draw();return 0};this.fnVersionCheck=u.fnVersionCheck;var b=this,c=a===k,e=this.length;c&&(a={});this.oApi=this.internal=u.internal;for(var d in m.ext.internal)d&&
-(this[d]=Nb(d));this.each(function(){var d={},d=1<e?Lb(d,a,!0):a,g=0,j,i=this.getAttribute("id"),o=!1,l=m.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())I(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{eb(l);fb(l.column);H(l,l,!0);H(l.column,l.column,!0);H(l,h.extend(d,q.data()));var n=m.settings,g=0;for(j=n.length;g<j;g++){var r=n[g];if(r.nTable==this||r.nTHead.parentNode==this||r.nTFoot&&r.nTFoot.parentNode==this){g=d.bRetrieve!==k?d.bRetrieve:l.bRetrieve;if(c||g)return r.oInstance;
-if(d.bDestroy!==k?d.bDestroy:l.bDestroy){r.oInstance.fnDestroy();break}else{I(r,0,"Cannot reinitialise DataTable",3);return}}if(r.sTableId==this.id){n.splice(g,1);break}}if(null===i||""===i)this.id=i="DataTables_Table_"+m.ext._unique++;var p=h.extend(!0,{},m.models.oSettings,{sDestroyWidth:q[0].style.width,sInstance:i,sTableId:i});p.nTable=this;p.oApi=b.internal;p.oInit=d;n.push(p);p.oInstance=1===b.length?b:q.dataTable();eb(d);d.oLanguage&&P(d.oLanguage);d.aLengthMenu&&!d.iDisplayLength&&(d.iDisplayLength=
-h.isArray(d.aLengthMenu[0])?d.aLengthMenu[0][0]:d.aLengthMenu[0]);d=Lb(h.extend(!0,{},l),d);E(p.oFeatures,d,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));E(p,d,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback",
-"renderer","searchDelay",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"],["bJQueryUI","bJUI"]]);E(p.oScroll,d,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);E(p.oLanguage,d,"fnInfoCallback");z(p,"aoDrawCallback",d.fnDrawCallback,"user");z(p,"aoServerParams",d.fnServerParams,"user");z(p,"aoStateSaveParams",d.fnStateSaveParams,"user");z(p,"aoStateLoadParams",
-d.fnStateLoadParams,"user");z(p,"aoStateLoaded",d.fnStateLoaded,"user");z(p,"aoRowCallback",d.fnRowCallback,"user");z(p,"aoRowCreatedCallback",d.fnCreatedRow,"user");z(p,"aoHeaderCallback",d.fnHeaderCallback,"user");z(p,"aoFooterCallback",d.fnFooterCallback,"user");z(p,"aoInitComplete",d.fnInitComplete,"user");z(p,"aoPreDrawCallback",d.fnPreDrawCallback,"user");i=p.oClasses;d.bJQueryUI?(h.extend(i,m.ext.oJUIClasses,d.oClasses),d.sDom===l.sDom&&"lfrtip"===l.sDom&&(p.sDom='<"H"lfr>t<"F"ip>'),p.renderer)?
-h.isPlainObject(p.renderer)&&!p.renderer.header&&(p.renderer.header="jqueryui"):p.renderer="jqueryui":h.extend(i,m.ext.classes,d.oClasses);q.addClass(i.sTable);if(""!==p.oScroll.sX||""!==p.oScroll.sY)p.oScroll.iBarWidth=Hb();!0===p.oScroll.sX&&(p.oScroll.sX="100%");p.iInitDisplayStart===k&&(p.iInitDisplayStart=d.iDisplayStart,p._iDisplayStart=d.iDisplayStart);null!==d.iDeferLoading&&(p.bDeferLoading=!0,g=h.isArray(d.iDeferLoading),p._iRecordsDisplay=g?d.iDeferLoading[0]:d.iDeferLoading,p._iRecordsTotal=
-g?d.iDeferLoading[1]:d.iDeferLoading);var t=p.oLanguage;h.extend(!0,t,d.oLanguage);""!==t.sUrl&&(h.ajax({dataType:"json",url:t.sUrl,success:function(a){P(a);H(l.oLanguage,a);h.extend(true,t,a);ga(p)},error:function(){ga(p)}}),o=!0);null===d.asStripeClasses&&(p.asStripeClasses=[i.sStripeOdd,i.sStripeEven]);var g=p.asStripeClasses,s=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(g,function(a){return s.hasClass(a)}))&&(h("tbody tr",this).removeClass(g.join(" ")),p.asDestroyStripes=g.slice());
-n=[];g=this.getElementsByTagName("thead");0!==g.length&&(da(p.aoHeader,g[0]),n=qa(p));if(null===d.aoColumns){r=[];g=0;for(j=n.length;g<j;g++)r.push(null)}else r=d.aoColumns;g=0;for(j=r.length;g<j;g++)Fa(p,n?n[g]:null);ib(p,d.aoColumnDefs,r,function(a,b){ka(p,a,b)});if(s.length){var u=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h.each(na(p,s[0]).cells,function(a,b){var c=p.aoColumns[a];if(c.mData===a){var d=u(b,"sort")||u(b,"order"),e=u(b,"filter")||u(b,"search");if(d!==null||e!==
-null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};ka(p,a)}}})}var v=p.oFeatures;d.bStateSave&&(v.bStateSave=!0,Kb(p,d),z(p,"aoDrawCallback",ya,"state_save"));if(d.aaSorting===k){n=p.aaSorting;g=0;for(j=n.length;g<j;g++)n[g][1]=p.aoColumns[g].asSorting[0]}xa(p);v.bSort&&z(p,"aoDrawCallback",function(){if(p.bSorted){var a=U(p),b={};h.each(a,function(a,c){b[c.src]=c.dir});w(p,null,"order",[p,a,b]);Jb(p)}});z(p,"aoDrawCallback",
-function(){(p.bSorted||B(p)==="ssp"||v.bDeferRender)&&xa(p)},"sc");gb(p);g=q.children("caption").each(function(){this._captionSide=q.css("caption-side")});j=q.children("thead");0===j.length&&(j=h("<thead/>").appendTo(this));p.nTHead=j[0];j=q.children("tbody");0===j.length&&(j=h("<tbody/>").appendTo(this));p.nTBody=j[0];j=q.children("tfoot");if(0===j.length&&0<g.length&&(""!==p.oScroll.sX||""!==p.oScroll.sY))j=h("<tfoot/>").appendTo(this);0===j.length||0===j.children().length?q.addClass(i.sNoFooter):
-0<j.length&&(p.nTFoot=j[0],da(p.aoFooter,p.nTFoot));if(d.aaData)for(g=0;g<d.aaData.length;g++)K(p,d.aaData[g]);else(p.bDeferLoading||"dom"==B(p))&&ma(p,h(p.nTBody).children("tr"));p.aiDisplay=p.aiDisplayMaster.slice();p.bInitialised=!0;!1===o&&ga(p)}});b=null;return this};var Tb=[],y=Array.prototype,cc=function(a){var b,c,e=m.settings,d=h.map(e,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,d),-1!==b?[e[b]]:
-null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,d);return-1!==b?e[b]:null}).toArray()};t=function(a,b){if(!(this instanceof t))return new t(a,b);var c=[],e=function(a){(a=cc(a))&&c.push.apply(c,a)};if(h.isArray(a))for(var d=0,f=a.length;d<f;d++)e(a[d]);else e(a);this.context=Na(c);b&&this.push.apply(this,b.toArray?b.toArray():b);this.selector={rows:null,cols:null,opts:null};
-t.extend(this,this,Tb)};m.Api=t;t.prototype={any:function(){return 0!==this.flatten().length},concat:y.concat,context:[],each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=this.context;return b.length>a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(y.filter)b=y.filter.call(this,a,this);else for(var c=0,e=this.length;c<e;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new t(this.context,b)},flatten:function(){var a=[];
-return new t(this.context,a.concat.apply(a,this.toArray()))},join:y.join,indexOf:y.indexOf||function(a,b){for(var c=b||0,e=this.length;c<e;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,e){var d=[],f,g,h,i,o,l=this.context,q,n,m=this.selector;"string"===typeof a&&(e=c,c=b,b=a,a=!1);g=0;for(h=l.length;g<h;g++){var p=new t(l[g]);if("table"===b)f=c.call(p,l[g],g),f!==k&&d.push(f);else if("columns"===b||"rows"===b)f=c.call(p,l[g],this[g],g),f!==k&&d.push(f);else if("column"===b||"column-rows"===
-b||"row"===b||"cell"===b){n=this[g];"column-rows"===b&&(q=Ca(l[g],m.opts));i=0;for(o=n.length;i<o;i++)f=n[i],f="cell"===b?c.call(p,l[g],f.row,f.column,g,i):c.call(p,l[g],f,g,i,q),f!==k&&d.push(f)}}return d.length||e?(a=new t(l,a?d.concat.apply([],d):d),b=a.selector,b.rows=m.rows,b.cols=m.cols,b.opts=m.opts,a):this},lastIndexOf:y.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(y.map)b=y.map.call(this,a,this);else for(var c=
-0,e=this.length;c<e;c++)b.push(a.call(this,this[c],c));return new t(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:y.pop,push:y.push,reduce:y.reduce||function(a,b){return hb(this,a,b,0,this.length,1)},reduceRight:y.reduceRight||function(a,b){return hb(this,a,b,this.length-1,-1,-1)},reverse:y.reverse,selector:null,shift:y.shift,sort:y.sort,splice:y.splice,toArray:function(){return y.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},
-unique:function(){return new t(this.context,Na(this))},unshift:y.unshift};t.extend=function(a,b,c){if(c.length&&b&&(b instanceof t||b.__dt_wrapper)){var e,d,f,g=function(a,b,c){return function(){var d=b.apply(a,arguments);t.extend(d,d,c.methodExt);return d}};e=0;for(d=c.length;e<d;e++)f=c[e],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,t.extend(a,b[f.name],f.propExt)}};t.register=r=function(a,b){if(h.isArray(a))for(var c=0,e=a.length;c<
-e;c++)t.register(a[c],b);else for(var d=a.split("."),f=Tb,g,j,c=0,e=d.length;c<e;c++){g=(j=-1!==d[c].indexOf("()"))?d[c].replace("()",""):d[c];var i;a:{i=0;for(var o=f.length;i<o;i++)if(f[i].name===g){i=f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===e-1?i.val=b:f=j?i.methodExt:i.propExt}};t.registerPlural=v=function(a,b,c){t.register(a,c);t.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof t?a.length?h.isArray(a[0])?new t(a.context,
-a[0]):a[0]:k:a})};r("tables()",function(a){var b;if(a){b=t;var c=this.context;if("number"===typeof a)a=[c[a]];else var e=h.map(c,function(a){return a.nTable}),a=h(e).filter(a).map(function(){var a=h.inArray(this,e);return c[a]}).toArray();b=new b(a)}else b=this;return b});r("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new t(b[0]):a});v("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});v("tables().body()","table().body()",
-function(){return this.iterator("table",function(a){return a.nTBody},1)});v("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});v("tables().footer()","table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});v("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});r("draw()",function(a){return this.iterator("table",function(b){N(b,
-!1===a)})});r("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Ta(b,a)})});r("page.info()",function(){if(0===this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a._iDisplayLength,e=a.fnRecordsDisplay(),d=-1===c;return{page:d?0:Math.floor(b/c),pages:d?1:Math.ceil(e/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:e}});r("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:
-k:this.iterator("table",function(b){Ra(b,a)})});var Ub=function(a,b,c){if(c){var e=new t(a);e.one("draw",function(){c(e.ajax.json())})}"ssp"==B(a)?N(a,b):(C(a,!0),ra(a,[],function(c){oa(a);for(var c=sa(a,c),e=0,g=c.length;e<g;e++)K(a,c[e]);N(a,b);C(a,!1)}))};r("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});r("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});r("ajax.reload()",function(a,b){return this.iterator("table",function(c){Ub(c,
-!1===b,a)})});r("ajax.url()",function(a){var b=this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});r("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Ub(c,!1===b,a)})});var $a=function(a,b,c,e,d){var f=[],g,j,i,o,l,q;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(o=b.length;i<o;i++){j=
-b[i]&&b[i].split?b[i].split(","):[b[i]];l=0;for(q=j.length;l<q;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&f.push.apply(f,g)}a=u.selector[a];if(a.length){i=0;for(o=a.length;i<o;i++)f=a[i](e,d,f)}return f},ab=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},bb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},
-Ca=function(a,b){var c,e,d,f=[],g=a.aiDisplay;c=a.aiDisplayMaster;var j=b.search;e=b.order;d=b.page;if("ssp"==B(a))return"removed"===j?[]:V(0,c.length);if("current"==d){c=a._iDisplayStart;for(e=a.fnDisplayEnd();c<e;c++)f.push(g[c])}else if("current"==e||"applied"==e)f="none"==j?c.slice():"applied"==j?g.slice():h.map(c,function(a){return-1===h.inArray(a,g)?a:null});else if("index"==e||"original"==e){c=0;for(e=a.aoData.length;c<e;c++)"none"==j?f.push(c):(d=h.inArray(c,g),(-1===d&&"removed"==j||0<=d&&
-"applied"==j)&&f.push(c))}return f};r("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var d=b;return $a("row",a,function(a){var b=Pb(a);if(b!==null&&!d)return[b];var j=Ca(c,d);if(b!==null&&h.inArray(b,j)!==-1)return[b];if(!a)return j;if(typeof a==="function")return h.map(j,function(b){var d=c.aoData[b];return a(b,d._aData,d.nTr)?b:null});b=Sb(ia(c.aoData,j,"nTr"));return a.nodeName&&h.inArray(a,b)!==-1?[a._DT_RowIndex]:h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},
-c,d)},1);c.selector.rows=a;c.selector.opts=b;return c});r("rows().nodes()",function(){return this.iterator("row",function(a,b){return a.aoData[b].nTr||k},1)});r("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ia(a.aoData,b,"_aData")},1)});v("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var e=b.aoData[c];return"search"===a?e._aFilterData:e._aSortData},1)});v("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",
-function(b,c){ca(b,c,a)})});v("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,b){return b},1)});v("rows().remove()","row().remove()",function(){var a=this;return this.iterator("row",function(b,c,e){var d=b.aoData;d.splice(c,1);for(var f=0,g=d.length;f<g;f++)null!==d[f].nTr&&(d[f].nTr._DT_RowIndex=f);h.inArray(c,b.aiDisplay);pa(b.aiDisplayMaster,c);pa(b.aiDisplay,c);pa(a[e],c,!1);Sa(b)})});r("rows.add()",function(a){var b=this.iterator("table",function(b){var c,
-f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(ma(b,c)[0]):h.push(K(b,c));return h},1),c=this.rows(-1);c.pop();c.push.apply(c,b.toArray());return c});r("row()",function(a,b){return bb(this.rows(a,b))});r("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:k;b[0].aoData[this[0]]._aData=a;ca(b[0],this[0],"data");return this});r("row().node()",function(){var a=this.context;return a.length&&this.length?
-a[0].aoData[this[0]].nTr||null:null});r("row.add()",function(a){a instanceof h&&a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?ma(b,a)[0]:K(b,a)});return this.row(b[0])});var cb=function(a,b){var c=a.context;c.length&&(c=c[0].aoData[b!==k?b:a[0]],c._details&&(c._details.remove(),c._detailsShow=k,c._details=k))},Vb=function(a,b){var c=a.context;if(c.length&&a.length){var e=c[0].aoData[a[0]];if(e._details){(e._detailsShow=b)?e._details.insertAfter(e.nTr):
-e._details.detach();var d=c[0],f=new t(d),g=d.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){d===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",function(a,b){if(d===b)for(var c,e=aa(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",e)}),f.on("destroy.dt.DT_details",
-function(a,b){if(d===b)for(var c=0,e=g.length;c<e;c++)g[c]._details&&cb(f,c)}))}}};r("row().child()",function(a,b){var c=this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)cb(this);else if(c.length&&this.length){var e=c[0],c=c[0].aoData[this[0]],d=[],f=function(a,b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?d.push(a):(c=h("<tr><td/></tr>").addClass(b),
-h("td",c).addClass(b).html(a)[0].colSpan=aa(e),d.push(c[0]))};f(a,b);c._details&&c._details.remove();c._details=h(d);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});r(["row().child.show()","row().child().show()"],function(){Vb(this,!0);return this});r(["row().child.hide()","row().child().hide()"],function(){Vb(this,!1);return this});r(["row().child.remove()","row().child().remove()"],function(){cb(this);return this});r("row().child.isShown()",function(){var a=this.context;return a.length&&
-this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var dc=/^(.+):(name|visIdx|visible)$/,Wb=function(a,b,c,e,d){for(var c=[],e=0,f=d.length;e<f;e++)c.push(x(a,d[e],b));return c};r("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=ab(b),c=this.iterator("table",function(c){var d=a,f=b,g=c.aoColumns,j=D(g,"sName"),i=D(g,"nTh");return $a("column",d,function(a){var b=Pb(a);if(a==="")return V(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var d=Ca(c,
-f);return h.map(g,function(b,f){return a(f,Wb(c,f,0,0,d),i[f])?f:null})}var k=typeof a==="string"?a.match(dc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});return[m[m.length+b]]}return[la(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null})}else return h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray()},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});v("columns().header()",
-"column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});v("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});v("columns().data()","column().data()",function(){return this.iterator("column-rows",Wb,1)});v("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});v("columns().cache()","column().cache()",
-function(a){return this.iterator("column-rows",function(b,c,e,d,f){return ia(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});v("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,e,d){return ia(a.aoData,d,"anCells",b)},1)});v("columns().visible()","column().visible()",function(a,b){return this.iterator("column",function(c,e){if(a===k)return c.aoColumns[e].bVisible;var d=c.aoColumns,f=d[e],g=c.aoData,j,i,m;if(a!==k&&f.bVisible!==a){if(a){var l=
-h.inArray(!0,D(d,"bVisible"),e+1);j=0;for(i=g.length;j<i;j++)m=g[j].nTr,d=g[j].anCells,m&&m.insertBefore(d[e],d[l]||null)}else h(D(c.aoData,"anCells",e)).detach();f.bVisible=a;ea(c,c.aoHeader);ea(c,c.aoFooter);if(b===k||b)X(c),(c.oScroll.sX||c.oScroll.sY)&&Y(c);w(c,null,"column-visibility",[c,e,a]);ya(c)}})});v("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?$(b,c):c},1)});r("columns.adjust()",function(){return this.iterator("table",
-function(a){X(a)},1)});r("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return la(c,b);if("fromData"===a||"toVisible"===a)return $(c,b)}});r("column()",function(a,b){return bb(this.columns(a,b))});r("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=ab(c),f=b.aoData,g=Ca(b,e),i=Sb(ia(f,g,"anCells")),
-j=h([].concat.apply([],i)),l,m=b.aoColumns.length,o,r,t,s,u,v;return $a("cell",d,function(a){var c=typeof a==="function";if(a===null||a===k||c){o=[];r=0;for(t=g.length;r<t;r++){l=g[r];for(s=0;s<m;s++){u={row:l,column:s};if(c){v=b.aoData[l];a(u,x(b,l,s),v.anCells?v.anCells[s]:null)&&o.push(u)}else o.push(u)}}return o}return h.isPlainObject(a)?[a]:j.filter(a).map(function(a,b){l=b.parentNode._DT_RowIndex;return{row:l,column:h.inArray(b,f[l].anCells)}}).toArray()},b,e)});var e=this.columns(b,c),d=this.rows(a,
-c),f,g,j,i,m,l=this.iterator("table",function(a,b){f=[];g=0;for(j=d[b].length;g<j;g++){i=0;for(m=e[b].length;i<m;i++)f.push({row:d[b][g],column:e[b][i]})}return f},1);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});v("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b].anCells)?a[c]:k},1)});r("cells().data()",function(){return this.iterator("cell",function(a,b,c){return x(a,b,c)},1)});v("cells().cache()","cell().cache()",function(a){a=
-"search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,e){return b.aoData[c][a][e]},1)});v("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,e){return x(b,c,e,a)},1)});v("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:$(a,c)}},1)});v("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,e){ca(b,c,a,e)})});r("cell()",
-function(a,b,c){return bb(this.cells(a,b,c))});r("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?x(b[0],c[0].row,c[0].column):k;Ia(b[0],c[0].row,c[0].column,a);ca(b[0],c[0].row,"data",c[0].column);return this});r("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:h.isArray(a[0])||(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});
-r("order.listener()",function(a,b,c){return this.iterator("table",function(e){Oa(e,a,b,c)})});r(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,e){var d=[];h.each(b[e],function(b,c){d.push([c,a])});c.aaSorting=d})});r("search()",function(a,b,c,e){var d=this.context;return a===k?0!==d.length?d[0].oPreviousSearch.sSearch:k:this.iterator("table",function(d){d.oFeatures.bFilter&&fa(d,h.extend({},d.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:
-b,bSmart:null===c?!0:c,bCaseInsensitive:null===e?!0:e}),1)})});v("columns().search()","column().search()",function(a,b,c,e){return this.iterator("column",function(d,f){var g=d.aoPreSearchCols;if(a===k)return g[f].sSearch;d.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===e?!0:e}),fa(d,d.oPreviousSearch,1))})});r("state()",function(){return this.context.length?this.context[0].oSavedState:null});r("state.clear()",function(){return this.iterator("table",
-function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});r("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});r("state.save()",function(){return this.iterator("table",function(a){ya(a)})});m.versionCheck=m.fnVersionCheck=function(a){for(var b=m.version.split("."),a=a.split("."),c,e,d=0,f=a.length;d<f;d++)if(c=parseInt(b[d],10)||0,e=parseInt(a[d],10)||0,c!==e)return c>e;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;h.each(m.settings,
-function(a,d){var f=d.nScrollHead?h("table",d.nScrollHead)[0]:null,g=d.nScrollFoot?h("table",d.nScrollFoot)[0]:null;if(d.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){return h.map(m.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable})};m.util={throttle:ua,escapeRegex:va};m.camelToHungarian=H;r("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,
-b){r(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0].match(/\.dt\b/)||(a[0]+=".dt");var e=h(this.tables().nodes());e[b].apply(e,a);return this})});r("clear()",function(){return this.iterator("table",function(a){oa(a)})});r("settings()",function(){return new t(this.context,this.context)});r("init()",function(){var a=this.context;return a.length?a[0].oInit:null});r("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});r("destroy()",
-function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,e=b.oClasses,d=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(d),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),q;b.bDestroying=!0;w(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.unbind(".DT").find(":not(tbody *)").unbind(".DT");h(Ea).unbind(".DT-"+b.sInstance);d!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&d!=j.parentNode&&(i.children("tfoot").detach(),
-i.append(j));i.detach();k.detach();b.aaSorting=[];b.aaSortingFixed=[];xa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(e.sSortable+" "+e.sSortableAsc+" "+e.sSortableDesc+" "+e.sSortableNone);b.bJUI&&(h("th span."+e.sSortIcon+", td span."+e.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+e.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));!a&&c&&c.insertBefore(d,b.nTableReinsertBefore);f.children().detach();f.append(l);i.css("width",b.sDestroyWidth).removeClass(e.sTable);
-(q=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%q])});c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){r(b+"s().every()",function(a){return this.iterator(b,function(e,d,f){a.call((new t(e))[b](d,f))})})});r("i18n()",function(a,b,c){var e=this.context[0],a=R(a)(e.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);return a.replace("%d",c)});m.version="1.10.7";m.settings=
-[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",
-sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,
-fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,
-fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},
-sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,
-sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null};W(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};W(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,
-bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],
-sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,
-bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==B(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==B(this)?1*this._iRecordsDisplay:
-this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,e=this.aiDisplay.length,d=this.oFeatures,f=d.bPaginate;return d.bServerSide?!1===f||-1===a?b+e:Math.min(b+a,this._iRecordsDisplay):!f||c>e||-1===a?e:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{}};m.ext=u={buttons:{},classes:{},errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},
-header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(u,{afnFiltering:u.search,aTypes:u.type.detect,ofnSearch:u.type.search,oSort:u.type.order,afnSortData:u.order,aoFeatures:u.feature,oApi:u.internal,oStdClasses:u.classes,oPagination:u.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",
-sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",
-sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var Da="",Da="",F=Da+"ui-state-default",ja=Da+"css_right ui-icon ui-icon-",Xb=Da+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses,
-m.ext.classes,{sPageButton:"fg-button ui-button "+F,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:F+" sorting_asc",sSortDesc:F+" sorting_desc",sSortable:F+" sorting",sSortableAsc:F+" sorting_asc_disabled",sSortableDesc:F+" sorting_desc_disabled",sSortableNone:F+" sorting_disabled",sSortJUIAsc:ja+"triangle-1-n",sSortJUIDesc:ja+"triangle-1-s",sSortJUI:ja+"carat-2-n-s",
-sSortJUIAscAllowed:ja+"carat-1-n",sSortJUIDescAllowed:ja+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+F,sScrollFoot:"dataTables_scrollFoot "+F,sHeaderTH:F,sFooterTH:F,sJUIHeader:Xb+" ui-corner-tl ui-corner-tr",sJUIFooter:Xb+" ui-corner-bl ui-corner-br"});var Mb=m.ext.pager;h.extend(Mb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},simple_numbers:function(a,b){return["previous",
-Wa(a,b),"next"]},full_numbers:function(a,b){return["first","previous",Wa(a,b),"next","last"]},_numbers:Wa,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,e,d,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i,k,l=0,m=function(b,e){var n,r,t,s,u=function(b){Ta(a,b.data.action,true)};n=0;for(r=e.length;n<r;n++){s=e[n];if(h.isArray(s)){t=h("<"+(s.DT_el||"div")+"/>").appendTo(b);m(t,s)}else{k=i="";switch(s){case "ellipsis":b.append('<span class="ellipsis">&#x2026;</span>');break;
-case "first":i=j.sFirst;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "previous":i=j.sPrevious;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "next":i=j.sNext;k=s+(d<f-1?"":" "+g.sPageButtonDisabled);break;case "last":i=j.sLast;k=s+(d<f-1?"":" "+g.sPageButtonDisabled);break;default:i=s+1;k=d===s?g.sPageButtonActive:""}if(i){t=h("<a>",{"class":g.sPageButton+" "+k,"aria-controls":a.sTableId,"data-dt-idx":l,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(i).appendTo(b);
-Va(t,{action:s},u);l++}}}},n;try{n=h(Q.activeElement).data("dt-idx")}catch(r){}m(h(b).empty(),e);n&&h(b).find("[data-dt-idx="+n+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&(!ac.test(a)||!bc.test(a)))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||J(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;
-return Rb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Rb(a,c,!0)?"html-num-fmt"+c:null},function(a){return J(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(m.ext.type.search,{html:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," ").replace(Ba,""):""},string:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," "):a}});var Aa=function(a,b,c,e){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Qb(a,b));a.replace&&(c&&(a=a.replace(c,"")),
-e&&(a=a.replace(e,"")));return 1*a};h.extend(u.type.order,{"date-pre":function(a){return Date.parse(a)||0},"html-pre":function(a){return J(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return J(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});db("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,e){h(a.nTable).on("order.dt.DT",function(d,
-f,g,h){if(a===f){d=c.idx;b.removeClass(c.sSortingClass+" "+e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,b,c,e){h("<div/>").addClass(e.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(e.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(d,f,g,h){if(a===f){d=c.idx;b.removeClass(e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass);
-b.find("span."+e.sSortIcon).removeClass(e.sSortJUIAsc+" "+e.sSortJUIDesc+" "+e.sSortJUI+" "+e.sSortJUIAscAllowed+" "+e.sSortJUIDescAllowed).addClass(h[d]=="asc"?e.sSortJUIAsc:h[d]=="desc"?e.sSortJUIDesc:c.sSortingClassJUI)}})}}});m.render={number:function(a,b,c,e){return{display:function(d){if("number"!==typeof d&&"string"!==typeof d)return d;var f=0>d?"-":"",d=Math.abs(parseFloat(d)),g=parseInt(d,10),d=c?b+(d-g).toFixed(c).substring(2):"";return f+(e||"")+g.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
-a)+d}}}};h.extend(m.ext.internal,{_fnExternApiFunc:Nb,_fnBuildAjax:ra,_fnAjaxUpdate:kb,_fnAjaxParameters:tb,_fnAjaxUpdateDraw:ub,_fnAjaxDataSrc:sa,_fnAddColumn:Fa,_fnColumnOptions:ka,_fnAdjustColumnSizing:X,_fnVisibleToColumnIndex:la,_fnColumnIndexToVisible:$,_fnVisbleColumns:aa,_fnGetColumns:Z,_fnColumnTypes:Ha,_fnApplyColumnDefs:ib,_fnHungarianMap:W,_fnCamelToHungarian:H,_fnLanguageCompat:P,_fnBrowserDetect:gb,_fnAddData:K,_fnAddTr:ma,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex:
-null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:x,_fnSetCellData:Ia,_fnSplitObjNotation:Ka,_fnGetObjectDataFn:R,_fnSetObjectDataFn:S,_fnGetDataMaster:La,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:ca,_fnGetRowElements:na,_fnCreateTr:Ja,_fnBuildHead:jb,_fnDrawHead:ea,_fnDraw:M,_fnReDraw:N,_fnAddOptionsHtml:mb,_fnDetectHeader:da,_fnGetUniqueThs:qa,_fnFeatureHtmlFilter:ob,_fnFilterComplete:fa,_fnFilterCustom:xb,_fnFilterColumn:wb,_fnFilter:vb,_fnFilterCreateSearch:Qa,
-_fnEscapeRegex:va,_fnFilterData:yb,_fnFeatureHtmlInfo:rb,_fnUpdateInfo:Bb,_fnInfoMacros:Cb,_fnInitialise:ga,_fnInitComplete:ta,_fnLengthChange:Ra,_fnFeatureHtmlLength:nb,_fnFeatureHtmlPaginate:sb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:pb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:qb,_fnScrollDraw:Y,_fnApplyToChildren:G,_fnCalculateColumnWidths:Ga,_fnThrottle:ua,_fnConvertToWidth:Db,_fnScrollingWidthAdjust:Fb,_fnGetWidestNode:Eb,_fnGetMaxLenString:Gb,_fnStringToCss:s,_fnScrollBarWidth:Hb,_fnSortFlatten:U,
-_fnSort:lb,_fnSortAria:Jb,_fnSortListener:Ua,_fnSortAttachListener:Oa,_fnSortingClasses:xa,_fnSortData:Ib,_fnSaveState:ya,_fnLoadState:Kb,_fnSettingsFromNode:za,_fnLog:I,_fnMap:E,_fnBindAction:Va,_fnCallbackReg:z,_fnCallbackFire:w,_fnLengthOverflow:Sa,_fnRenderer:Pa,_fnDataSource:B,_fnRowAttributes:Ma,_fnCalculateEnd:function(){}});h.fn.dataTable=m;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]=
-b});return h.fn.dataTable};"function"===typeof define&&define.amd?define("datatables",["jquery"],P):"object"===typeof exports?module.exports=P(require("jquery")):jQuery&&!jQuery.fn.dataTable&&P(jQuery)})(window,document);
+/*!
+ DataTables 1.11.5
+ ©2008-2021 SpryMedia Ltd - datatables.net/license
+*/ var $jscomp=$jscomp||{};$jscomp.scope={},$jscomp.findInternal=function(t,e,n){t instanceof String&&(t=String(t));for(var a=t.length,r=0;r<a;r++){var o=t[r];if(e.call(n,o,r,t))return{i:r,v:o}}return{i:-1,v:void 0}},$jscomp.ASSUME_ES5=!1,$jscomp.ASSUME_NO_NATIVE_MAP=!1,$jscomp.ASSUME_NO_NATIVE_SET=!1,$jscomp.SIMPLE_FROUND_POLYFILL=!1,$jscomp.ISOLATE_POLYFILLS=!1,$jscomp.defineProperty=$jscomp.ASSUME_ES5||"function"==typeof Object.defineProperties?Object.defineProperty:function(t,e,n){return t==Array.prototype||t==Object.prototype||(t[e]=n.value),t},$jscomp.getGlobal=function(t){t=["object"==typeof globalThis&&globalThis,t,"object"==typeof window&&window,"object"==typeof self&&self,"object"==typeof global&&global];for(var e=0;e<t.length;++e){var n=t[e];if(n&&n.Math==Math)return n}throw Error("Cannot find global object")},$jscomp.global=$jscomp.getGlobal(this),$jscomp.IS_SYMBOL_NATIVE="function"==typeof Symbol&&"symbol"==typeof Symbol("x"),$jscomp.TRUST_ES6_POLYFILLS=!$jscomp.ISOLATE_POLYFILLS||$jscomp.IS_SYMBOL_NATIVE,$jscomp.polyfills={},$jscomp.propertyToPolyfillSymbol={},$jscomp.POLYFILL_PREFIX="$jscp$";var $jscomp$lookupPolyfilledValue=function(t,e){var n=$jscomp.propertyToPolyfillSymbol[e];return null==n?t[e]:void 0!==(n=t[n])?n:t[e]};$jscomp.polyfill=function(t,e,n,a){e&&($jscomp.ISOLATE_POLYFILLS?$jscomp.polyfillIsolated(t,e,n,a):$jscomp.polyfillUnisolated(t,e,n,a))},$jscomp.polyfillUnisolated=function(t,e,n,a){for(a=0,n=$jscomp.global,t=t.split(".");a<t.length-1;a++){var r=t[a];if(!(r in n))return;n=n[r]}(e=e(a=n[t=t[t.length-1]]))!=a&&null!=e&&$jscomp.defineProperty(n,t,{configurable:!0,writable:!0,value:e})},$jscomp.polyfillIsolated=function(t,e,n,a){var r=t.split(".");t=1===r.length,a=r[0],a=!t&&a in $jscomp.polyfills?$jscomp.polyfills:$jscomp.global;for(var o=0;o<r.length-1;o++){var i=r[o];if(!(i in a))return;a=a[i]}r=r[r.length-1],null!=(e=e(n=$jscomp.IS_SYMBOL_NATIVE&&"es6"===n?a[r]:null))&&(t?$jscomp.defineProperty($jscomp.polyfills,r,{configurable:!0,writable:!0,value:e}):e!==n&&($jscomp.propertyToPolyfillSymbol[r]=$jscomp.IS_SYMBOL_NATIVE?$jscomp.global.Symbol(r):$jscomp.POLYFILL_PREFIX+r,r=$jscomp.propertyToPolyfillSymbol[r],$jscomp.defineProperty(a,r,{configurable:!0,writable:!0,value:e})))},$jscomp.polyfill("Array.prototype.find",function(t){return t||function(t,e){return $jscomp.findInternal(this,t,e).v}},"es6","es3"),function(t){"function"==typeof define&&define.amd?define(["jquery"],function(e){return t(e,window,document)}):"object"==typeof exports?module.exports=function(e,n){return e||(e=window),n||(n="undefined"!=typeof window?require("jquery"):require("jquery")(e)),t(n,e,e.document)}:window.DataTable=t(jQuery,window,document)}(function(t,e,n,a){function r(e){var n,a,o={};t.each(e,function(t,i){(n=t.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(n[1]+" ")&&(o[a=t.replace(n[0],n[2].toLowerCase())]=t,"o"===n[1]&&r(e[t]))}),e._hungarianMap=o}function o(e,n,i){var l;e._hungarianMap||r(e),t.each(n,function(r,s){(l=e._hungarianMap[r])!==a&&(i||n[l]===a)&&("o"===l.charAt(0)?(n[l]||(n[l]={}),t.extend(!0,n[l],n[r]),o(e[l],n[l],i)):n[l]=n[r])})}function i(t){var e=tB.defaults.oLanguage,n=e.sDecimal;if(n&&tk(n),t){var a=t.sZeroRecords;!t.sEmptyTable&&a&&"No data available in table"===e.sEmptyTable&&tA(t,t,"sZeroRecords","sEmptyTable"),!t.sLoadingRecords&&a&&"Loading..."===e.sLoadingRecords&&tA(t,t,"sZeroRecords","sLoadingRecords"),t.sInfoThousands&&(t.sThousands=t.sInfoThousands),(t=t.sDecimal)&&n!==t&&tk(t)}}function l(t){if(tQ(t,"ordering","bSort"),tQ(t,"orderMulti","bSortMulti"),tQ(t,"orderClasses","bSortClasses"),tQ(t,"orderCellsTop","bSortCellsTop"),tQ(t,"order","aaSorting"),tQ(t,"orderFixed","aaSortingFixed"),tQ(t,"paging","bPaginate"),tQ(t,"pagingType","sPaginationType"),tQ(t,"pageLength","iDisplayLength"),tQ(t,"searching","bFilter"),"boolean"==typeof t.sScrollX&&(t.sScrollX=t.sScrollX?"100%":""),"boolean"==typeof t.scrollX&&(t.scrollX=t.scrollX?"100%":""),t=t.aoSearchCols)for(var e=0,n=t.length;e<n;e++)t[e]&&o(tB.models.oSearch,t[e])}function s(t){tQ(t,"orderable","bSortable"),tQ(t,"orderData","aDataSort"),tQ(t,"orderSequence","asSorting"),tQ(t,"orderDataType","sortDataType");var e=t.aDataSort;"number"!=typeof e||Array.isArray(e)||(t.aDataSort=[e])}function u(n){if(!tB.__browser){var a={};tB.__browser=a;var r=t("<div/>").css({position:"fixed",top:0,left:-1*t(e).scrollLeft(),height:1,width:1,overflow:"hidden"}).append(t("<div/>").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(t("<div/>").css({width:"100%",height:10}))).appendTo("body"),o=r.children(),i=o.children();a.barWidth=o[0].offsetWidth-o[0].clientWidth,a.bScrollOversize=100===i[0].offsetWidth&&100!==o[0].clientWidth,a.bScrollbarLeft=1!==Math.round(i.offset().left),a.bBounding=!!r[0].getBoundingClientRect().width,r.remove()}t.extend(n.oBrowser,tB.__browser),n.oScroll.iBarWidth=tB.__browser.barWidth}function c(t,e,n,r,o,i){var l=!1;if(n!==a){var s=n;l=!0}for(;r!==o;)t.hasOwnProperty(r)&&(s=l?e(s,t[r],r,t):t[r],l=!0,r+=i);return s}function f(e,a){var r=tB.defaults.column,o=e.aoColumns.length;r=t.extend({},tB.models.oColumn,r,{nTh:a||n.createElement("th"),sTitle:r.sTitle?r.sTitle:a?a.innerHTML:"",aDataSort:r.aDataSort?r.aDataSort:[o],mData:r.mData?r.mData:o,idx:o}),e.aoColumns.push(r),(r=e.aoPreSearchCols)[o]=t.extend({},tB.models.oSearch,r[o]),d(e,o,t(a).data())}function d(e,n,r){n=e.aoColumns[n];var i=e.oClasses,l=t(n.nTh);if(!n.sWidthOrig){n.sWidthOrig=l.attr("width")||null;var u=(l.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);u&&(n.sWidthOrig=u[1])}r!==a&&null!==r&&(s(r),o(tB.defaults.column,r,!0),r.mDataProp===a||r.mData||(r.mData=r.mDataProp),r.sType&&(n._sManualType=r.sType),r.className&&!r.sClass&&(r.sClass=r.className),r.sClass&&l.addClass(r.sClass),t.extend(n,r),tA(n,r,"sWidth","sWidthOrig"),r.iDataSort!==a&&(n.aDataSort=[r.iDataSort]),tA(n,r,"aDataSort"));var c=n.mData,f=en(c),d=n.mRender?en(n.mRender):null;r=function(t){return"string"==typeof t&&-1!==t.indexOf("@")},n._bAttrSrc=t.isPlainObject(c)&&(r(c.sort)||r(c.type)||r(c.filter)),n._setter=null,n.fnGetData=function(t,e,n){var r=f(t,e,a,n);return d&&e?d(r,e,t,n):r},n.fnSetData=function(t,e,n){return ea(c)(t,e,n)},"number"!=typeof c&&(e._rowReadObject=!0),e.oFeatures.bSort||(n.bSortable=!1,l.addClass(i.sSortableNone)),e=-1!==t.inArray("asc",n.asSorting),r=-1!==t.inArray("desc",n.asSorting),n.bSortable&&(e||r)?e&&!r?(n.sSortingClass=i.sSortableAsc,n.sSortingClassJUI=i.sSortJUIAscAllowed):!e&&r?(n.sSortingClass=i.sSortableDesc,n.sSortingClassJUI=i.sSortJUIDescAllowed):(n.sSortingClass=i.sSortable,n.sSortingClassJUI=i.sSortJUI):(n.sSortingClass=i.sSortableNone,n.sSortingClassJUI="")}function h(t){if(!1!==t.oFeatures.bAutoWidth){var e=t.aoColumns;th(t);for(var n=0,a=e.length;n<a;n++)e[n].nTh.style.width=e[n].sWidth}""===(e=t.oScroll).sY&&""===e.sX||tf(t),tP(t,null,"column-sizing",[t])}function p(t,e){return"number"==typeof(t=$(t,"bVisible"))[e]?t[e]:null}function g(e,n){return e=$(e,"bVisible"),-1!==(n=t.inArray(n,e))?n:null}function b(e){var n=0;return t.each(e.aoColumns,function(e,a){a.bVisible&&"none"!==t(a.nTh).css("display")&&n++}),n}function $(e,n){var a=[];return t.map(e.aoColumns,function(t,e){t[n]&&a.push(e)}),a}function m(t){var e,n,r,o=t.aoColumns,i=t.aoData,l=tB.ext.type.detect,s=0;for(e=o.length;s<e;s++){var u=o[s],c=[];if(!u.sType&&u._sManualType)u.sType=u._sManualType;else if(!u.sType){var f=0;for(n=l.length;f<n;f++){var d=0;for(r=i.length;d<r;d++){c[d]===a&&(c[d]=D(t,d,s,"type"));var h=l[f](c[d],t);if(!h&&f!==l.length-1||"html"===h&&!tY(c[d]))break}if(h){u.sType=h;break}}u.sType||(u.sType="string")}}}function v(e,n,r,o){var i,l,s,u=e.aoColumns;if(n)for(i=n.length-1;0<=i;i--){var c=n[i],d=c.targets!==a?c.targets:c.aTargets;Array.isArray(d)||(d=[d]);var h=0;for(l=d.length;h<l;h++)if("number"==typeof d[h]&&0<=d[h]){for(;u.length<=d[h];)f(e);o(d[h],c)}else if("number"==typeof d[h]&&0>d[h])o(u.length+d[h],c);else if("string"==typeof d[h]){var p=0;for(s=u.length;p<s;p++)("_all"==d[h]||t(u[p].nTh).hasClass(d[h]))&&o(p,c)}}if(r)for(i=0,e=r.length;i<e;i++)o(i,r[i])}function S(e,n,r,o){var i=e.aoData.length,l=t.extend(!0,{},tB.models.oRow,{src:r?"dom":"data",idx:i});l._aData=n,e.aoData.push(l);for(var s=e.aoColumns,u=0,c=s.length;u<c;u++)s[u].sType=null;return e.aiDisplayMaster.push(i),(n=e.rowIdFn(n))!==a&&(e.aIds[n]=l),!r&&e.oFeatures.bDeferRender||L(e,i,r,o),i}function y(e,n){var a;return n instanceof t||(n=t(n)),n.map(function(t,n){return a=A(e,n),S(e,a.data,n,a.cells)})}function D(t,e,n,r){"search"===r?r="filter":"order"===r&&(r="sort");var o=t.iDraw,i=t.aoColumns[n],l=t.aoData[e]._aData,s=i.sDefaultContent,u=i.fnGetData(l,r,{settings:t,row:e,col:n});if(u===a)return t.iDrawError!=o&&null===s&&(tI(t,0,"Requested unknown parameter "+("function"==typeof i.mData?"{function}":"'"+i.mData+"'")+" for row "+e+", column "+n,4),t.iDrawError=o),s;if((u===l||null===u)&&null!==s&&r!==a)u=s;else if("function"==typeof u)return u.call(l);return null===u&&"display"===r?"":("filter"===r&&(t=tB.ext.type.search)[i.sType]&&(u=t[i.sType](u)),u)}function T(t,e,n,a){t.aoColumns[n].fnSetData(t.aoData[e]._aData,a,{settings:t,row:e,col:n})}function C(e){return t.map(e.match(/(\\.|[^\.])+/g)||[""],function(t){return t.replace(/\\\./g,".")})}function w(t){return t3(t.aoData,"_aData")}function x(t){t.aoData.length=0,t.aiDisplayMaster.length=0,t.aiDisplay.length=0,t.aIds={}}function _(t,e,n){for(var r=-1,o=0,i=t.length;o<i;o++)t[o]==e?r=o:t[o]>e&&t[o]--;-1!=r&&n===a&&t.splice(r,1)}function I(t,e,n,r){var o,i=t.aoData[e],l=function(n,a){for(;n.childNodes.length;)n.removeChild(n.firstChild);n.innerHTML=D(t,e,a,"display")};if("dom"!==n&&(n&&"auto"!==n||"dom"!==i.src)){var s=i.anCells;if(s){if(r!==a)l(s[r],r);else for(n=0,o=s.length;n<o;n++)l(s[n],n)}}else i._aData=A(t,i,r,r===a?a:i._aData).data;if(i._aSortData=null,i._aFilterData=null,l=t.aoColumns,r!==a)l[r].sType=null;else{for(n=0,o=l.length;n<o;n++)l[n].sType=null;F(t,i)}}function A(t,e,n,r){var o,i,l=[],s=e.firstChild,u=0,c=t.aoColumns,f=t._rowReadObject;r=r!==a?r:f?{}:[];var d=function(t,e){if("string"==typeof t){var n=t.indexOf("@");-1!==n&&(n=t.substring(n+1),ea(t)(r,e.getAttribute(n)))}},h=function(t){(n===a||n===u)&&(o=c[u],i=t.innerHTML.trim(),o&&o._bAttrSrc?(ea(o.mData._)(r,i),d(o.mData.sort,t),d(o.mData.type,t),d(o.mData.filter,t)):f?(o._setter||(o._setter=ea(o.mData)),o._setter(r,i)):r[u]=i),u++};if(s)for(;s;){var p=s.nodeName.toUpperCase();("TD"==p||"TH"==p)&&(h(s),l.push(s)),s=s.nextSibling}else for(l=e.anCells,s=0,p=l.length;s<p;s++)h(l[s]);return(e=e.firstChild?e:e.nTr)&&(e=e.getAttribute("id"))&&ea(t.rowId)(r,e),{data:r,cells:l}}function L(e,a,r,o){var i,l,s=e.aoData[a],u=s._aData,c=[];if(null===s.nTr){var f=r||n.createElement("tr");s.nTr=f,s.anCells=c,f._DT_RowIndex=a,F(e,s);var d=0;for(i=e.aoColumns.length;d<i;d++){var h=e.aoColumns[d];(s=(l=!r)?n.createElement(h.sCellType):o[d])._DT_CellIndex={row:a,column:d},c.push(s),(l||!(!h.mRender&&h.mData===d||t.isPlainObject(h.mData)&&h.mData._===d+".display"))&&(s.innerHTML=D(e,a,d,"display")),h.sClass&&(s.className+=" "+h.sClass),h.bVisible&&!r?f.appendChild(s):!h.bVisible&&r&&s.parentNode.removeChild(s),h.fnCreatedCell&&h.fnCreatedCell.call(e.oInstance,s,D(e,a,d),u,a,d)}tP(e,"aoRowCreatedCallback",null,[f,u,a,c])}}function F(e,n){var a=n.nTr,r=n._aData;a&&((e=e.rowIdFn(r))&&(a.id=e),r.DT_RowClass&&(e=r.DT_RowClass.split(" "),n.__rowc=n.__rowc?t5(n.__rowc.concat(e)):e,t(a).removeClass(n.__rowc.join(" ")).addClass(r.DT_RowClass)),r.DT_RowAttr&&t(a).attr(r.DT_RowAttr),r.DT_RowData&&t(a).data(r.DT_RowData))}function j(e){var n,a,r=e.nTHead,o=e.nTFoot,i=0===t("th, td",r).length,l=e.oClasses,s=e.aoColumns;i&&(a=t("<tr/>").appendTo(r));var u=0;for(n=s.length;u<n;u++){var c=s[u],f=t(c.nTh).addClass(c.sClass);i&&f.appendTo(a),e.oFeatures.bSort&&(f.addClass(c.sSortingClass),!1!==c.bSortable&&(f.attr("tabindex",e.iTabIndex).attr("aria-controls",e.sTableId),tD(e,c.nTh,u))),c.sTitle!=f[0].innerHTML&&f.html(c.sTitle),tO(e,"header")(e,f,c,l)}if(i&&N(e.aoHeader,r),t(r).children("tr").children("th, td").addClass(l.sHeaderTH),t(o).children("tr").children("th, td").addClass(l.sFooterTH),null!==o)for(e=e.aoFooter[0],u=0,n=e.length;u<n;u++)(c=s[u]).nTf=e[u].cell,c.sClass&&t(c.nTf).addClass(c.sClass)}function P(e,n,r){var o,i,l=[],s=[],u=e.aoColumns.length;if(n){r===a&&(r=!1);var c=0;for(o=n.length;c<o;c++){for(l[c]=n[c].slice(),l[c].nTr=n[c].nTr,i=u-1;0<=i;i--)e.aoColumns[i].bVisible||r||l[c].splice(i,1);s.push([])}for(c=0,o=l.length;c<o;c++){if(e=l[c].nTr)for(;i=e.firstChild;)e.removeChild(i);for(i=0,n=l[c].length;i<n;i++){var f=u=1;if(s[c][i]===a){for(e.appendChild(l[c][i].cell),s[c][i]=1;l[c+u]!==a&&l[c][i].cell==l[c+u][i].cell;)s[c+u][i]=1,u++;for(;l[c][i+f]!==a&&l[c][i].cell==l[c][i+f].cell;){for(r=0;r<u;r++)s[c+r][i+f]=1;f++}t(l[c][i].cell).attr("rowspan",u).attr("colspan",f)}}}}}function R(e,n){var r="ssp"==tH(e),o=e.iInitDisplayStart;if(o!==a&&-1!==o&&(e._iDisplayStart=r?o:o>=e.fnRecordsDisplay()?0:o,e.iInitDisplayStart=-1),r=tP(e,"aoPreDrawCallback","preDraw",[e]),-1!==t.inArray(!1,r))tu(e,!1);else{r=[];var i=0,l=(o=e.asStripeClasses).length,s=e.oLanguage,u="ssp"==tH(e),c=e.aiDisplay,f=e._iDisplayStart,d=e.fnDisplayEnd();if(e.bDrawing=!0,e.bDeferLoading)e.bDeferLoading=!1,e.iDraw++,tu(e,!1);else if(u){if(!e.bDestroying&&!n){M(e);return}}else e.iDraw++;if(0!==c.length)for(n=u?e.aoData.length:d,s=u?0:f;s<n;s++){u=c[s];var h=e.aoData[u];null===h.nTr&&L(e,u);var p=h.nTr;if(0!==l){var g=o[i%l];h._sRowStripe!=g&&(t(p).removeClass(h._sRowStripe).addClass(g),h._sRowStripe=g)}tP(e,"aoRowCallback",null,[p,h._aData,i,s,u]),r.push(p),i++}else i=s.sZeroRecords,1==e.iDraw&&"ajax"==tH(e)?i=s.sLoadingRecords:s.sEmptyTable&&0===e.fnRecordsTotal()&&(i=s.sEmptyTable),r[0]=t("<tr/>",{class:l?o[0]:""}).append(t("<td />",{valign:"top",colSpan:b(e),class:e.oClasses.sRowEmpty}).html(i))[0];tP(e,"aoHeaderCallback","header",[t(e.nTHead).children("tr")[0],w(e),f,d,c]),tP(e,"aoFooterCallback","footer",[t(e.nTFoot).children("tr")[0],w(e),f,d,c]),(o=t(e.nTBody)).children().detach(),o.append(t(r)),tP(e,"aoDrawCallback","draw",[e]),e.bSorted=!1,e.bFiltered=!1,e.bDrawing=!1}}function O(t,e){var n=t.oFeatures,a=n.bFilter;n.bSort&&tv(t),a?X(t,t.oPreviousSearch):t.aiDisplay=t.aiDisplayMaster.slice(),!0!==e&&(t._iDisplayStart=0),t._drawHold=e,R(t),t._drawHold=!1}function H(e){var n=e.oClasses,a=t(e.nTable);a=t("<div/>").insertBefore(a);var r=e.oFeatures,o=t("<div/>",{id:e.sTableId+"_wrapper",class:n.sWrapper+(e.nTFoot?"":" "+n.sNoFooter)});e.nHolding=a[0],e.nTableWrapper=o[0],e.nTableReinsertBefore=e.nTable.nextSibling;for(var i,l,s,u,c,f,d=e.sDom.split(""),h=0;h<d.length;h++){if(i=null,"<"==(l=d[h])){if(s=t("<div/>")[0],"'"==(u=d[h+1])||'"'==u){for(f=2,c="";d[h+f]!=u;)c+=d[h+f],f++;"H"==c?c=n.sJUIHeader:"F"==c&&(c=n.sJUIFooter),-1!=c.indexOf(".")?(u=c.split("."),s.id=u[0].substr(1,u[0].length-1),s.className=u[1]):"#"==c.charAt(0)?s.id=c.substr(1,c.length-1):s.className=c,h+=f}o.append(s),o=t(s)}else if(">"==l)o=o.parent();else if("l"==l&&r.bPaginate&&r.bLengthChange)i=to(e);else if("f"==l&&r.bFilter)i=V(e);else if("r"==l&&r.bProcessing)i=ts(e);else if("t"==l)i=tc(e);else if("i"==l&&r.bInfo)i=Q(e);else if("p"==l&&r.bPaginate)i=ti(e);else if(0!==tB.ext.feature.length){for(s=tB.ext.feature,f=0,u=s.length;f<u;f++)if(l==s[f].cFeature){i=s[f].fnInit(e);break}}i&&((s=e.aanFeatures)[l]||(s[l]=[]),s[l].push(i),o.append(i))}a.replaceWith(o),e.nHolding=null}function N(e,n){n=t(n).children("tr"),e.splice(0,e.length);var a,r,o,i=0;for(o=n.length;i<o;i++)e.push([]);for(i=0,o=n.length;i<o;i++){var l=n[i];for(a=l.firstChild;a;){if("TD"==a.nodeName.toUpperCase()||"TH"==a.nodeName.toUpperCase()){var s=1*a.getAttribute("colspan"),u=1*a.getAttribute("rowspan");s=s&&0!==s&&1!==s?s:1,u=u&&0!==u&&1!==u?u:1;var c=0;for(r=e[i];r[c];)c++;var f=c,d=1===s;for(r=0;r<s;r++)for(c=0;c<u;c++)e[i+c][f+r]={cell:a,unique:d},e[i+c].nTr=l}a=a.nextSibling}}}function k(t,e,n){var a=[];n||(n=t.aoHeader,e&&N(n=[],e)),e=0;for(var r=n.length;e<r;e++)for(var o=0,i=n[e].length;o<i;o++)!n[e][o].unique||a[o]&&t.bSortCellsTop||(a[o]=n[e][o].cell);return a}function E(e,n,a){if(tP(e,"aoServerParams","serverParams",[n]),n&&Array.isArray(n)){var r={},o=/(.*?)\[\]$/;t.each(n,function(t,e){(t=e.name.match(o))?(r[t=t[0]]||(r[t]=[]),r[t].push(e.value)):r[e.name]=e.value}),n=r}var i=e.ajax,l=e.oInstance,s=function(t){var n=e.jqXHR?e.jqXHR.status:null;(null===t||"number"==typeof n&&204==n)&&U(e,t={},[]),(n=t.error||t.sError)&&tI(e,0,n),e.json=t,tP(e,null,"xhr",[e,t,e.jqXHR]),a(t)};if(t.isPlainObject(i)&&i.data){var u=i.data,c="function"==typeof u?u(n,e):u;n="function"==typeof u&&c?c:t.extend(!0,n,c),delete i.data}c={data:n,success:s,dataType:"json",cache:!1,type:e.sServerMethod,error:function(n,a,r){r=tP(e,null,"xhr",[e,null,e.jqXHR]),-1===t.inArray(!0,r)&&("parsererror"==a?tI(e,0,"Invalid JSON response",1):4===n.readyState&&tI(e,0,"Ajax error",7)),tu(e,!1)}},e.oAjaxData=n,tP(e,null,"preXhr",[e,n]),e.fnServerData?e.fnServerData.call(l,e.sAjaxSource,t.map(n,function(t,e){return{name:e,value:t}}),s,e):e.sAjaxSource||"string"==typeof i?e.jqXHR=t.ajax(t.extend(c,{url:i||e.sAjaxSource})):"function"==typeof i?e.jqXHR=i.call(l,n,s,e):(e.jqXHR=t.ajax(t.extend(c,i)),i.data=u)}function M(t){t.iDraw++,tu(t,!0),E(t,W(t),function(e){B(t,e)})}function W(e){var n=e.aoColumns,a=n.length,r=e.oFeatures,o=e.oPreviousSearch,i=e.aoPreSearchCols,l=[],s=tm(e),u=e._iDisplayStart,c=!1!==r.bPaginate?e._iDisplayLength:-1,f=function(t,e){l.push({name:t,value:e})};f("sEcho",e.iDraw),f("iColumns",a),f("sColumns",t3(n,"sName").join(",")),f("iDisplayStart",u),f("iDisplayLength",c);var d={draw:e.iDraw,columns:[],order:[],start:u,length:c,search:{value:o.sSearch,regex:o.bRegex}};for(u=0;u<a;u++){var h=n[u],p=i[u];c="function"==typeof h.mData?"function":h.mData,d.columns.push({data:c,name:h.sName,searchable:h.bSearchable,orderable:h.bSortable,search:{value:p.sSearch,regex:p.bRegex}}),f("mDataProp_"+u,c),r.bFilter&&(f("sSearch_"+u,p.sSearch),f("bRegex_"+u,p.bRegex),f("bSearchable_"+u,h.bSearchable)),r.bSort&&f("bSortable_"+u,h.bSortable)}return r.bFilter&&(f("sSearch",o.sSearch),f("bRegex",o.bRegex)),r.bSort&&(t.each(s,function(t,e){d.order.push({column:e.col,dir:e.dir}),f("iSortCol_"+t,e.col),f("sSortDir_"+t,e.dir)}),f("iSortingCols",s.length)),null===(n=tB.ext.legacy.ajax)?e.sAjaxSource?l:d:n?l:d}function B(t,e){var n=function(t,n){return e[t]!==a?e[t]:e[n]},r=U(t,e),o=n("sEcho","draw"),i=n("iTotalRecords","recordsTotal");if(n=n("iTotalDisplayRecords","recordsFiltered"),o!==a){if(1*o<t.iDraw)return;t.iDraw=1*o}for(r||(r=[]),x(t),t._iRecordsTotal=parseInt(i,10),t._iRecordsDisplay=parseInt(n,10),o=0,i=r.length;o<i;o++)S(t,r[o]);t.aiDisplay=t.aiDisplayMaster.slice(),R(t,!0),t._bInitComplete||ta(t,e),tu(t,!1)}function U(e,n,r){if(e=t.isPlainObject(e.ajax)&&e.ajax.dataSrc!==a?e.ajax.dataSrc:e.sAjaxDataProp,!r)return"data"===e?n.aaData||n[e]:""!==e?en(e)(n):n;ea(e)(n,r)}function V(e){var a=e.oClasses,r=e.sTableId,o=e.oLanguage,i=e.oPreviousSearch,l=e.aanFeatures,s='<input type="search" class="'+a.sFilterInput+'"/>',u=o.sSearch;u=u.match(/_INPUT_/)?u.replace("_INPUT_",s):u+s,a=t("<div/>",{id:l.f?null:r+"_filter",class:a.sFilter}).append(t("<label/>").append(u));var c=function(t){var n=this.value?this.value:"";i.return&&"Enter"!==t.key||n==i.sSearch||(X(e,{sSearch:n,bRegex:i.bRegex,bSmart:i.bSmart,bCaseInsensitive:i.bCaseInsensitive,return:i.return}),e._iDisplayStart=0,R(e))};l=null!==e.searchDelay?e.searchDelay:"ssp"===tH(e)?400:0;var f=t("input",a).val(i.sSearch).attr("placeholder",o.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",l?es(c,l):c).on("mouseup",function(t){setTimeout(function(){c.call(f[0],t)},10)}).on("keypress.DT",function(t){if(13==t.keyCode)return!1}).attr("aria-controls",r);return t(e.nTable).on("search.dt.DT",function(t,a){if(e===a)try{f[0]!==n.activeElement&&f.val(i.sSearch)}catch(r){}}),a[0]}function X(t,e,n){var r=t.oPreviousSearch,o=t.aoPreSearchCols,i=function(t){r.sSearch=t.sSearch,r.bRegex=t.bRegex,r.bSmart=t.bSmart,r.bCaseInsensitive=t.bCaseInsensitive,r.return=t.return},l=function(t){return t.bEscapeRegex!==a?!t.bEscapeRegex:t.bRegex};if(m(t),"ssp"!=tH(t)){for(Y(t,e.sSearch,n,l(e),e.bSmart,e.bCaseInsensitive,e.return),i(e),e=0;e<o.length;e++)q(t,o[e].sSearch,e,l(o[e]),o[e].bSmart,o[e].bCaseInsensitive);J(t)}else i(e);t.bFiltered=!0,tP(t,null,"search",[t])}function J(e){for(var n,a,r=tB.ext.search,o=e.aiDisplay,i=0,l=r.length;i<l;i++){for(var s=[],u=0,c=o.length;u<c;u++)a=o[u],n=e.aoData[a],r[i](e,n._aFilterData,a,n._aData,u)&&s.push(a);o.length=0,t.merge(o,s)}}function q(t,e,n,a,r,o){if(""!==e){var i=[],l=t.aiDisplay;for(a=G(e,a,r,o),r=0;r<l.length;r++)e=t.aoData[l[r]]._aFilterData[n],a.test(e)&&i.push(l[r]);t.aiDisplay=i}}function Y(t,e,n,a,r,o){r=G(e,a,r,o);var i=t.oPreviousSearch.sSearch,l=t.aiDisplayMaster;o=[],0!==tB.ext.search.length&&(n=!0);var s=z(t);if(0>=e.length)t.aiDisplay=l.slice();else{for((s||n||a||i.length>e.length||0!==e.indexOf(i)||t.bSorted)&&(t.aiDisplay=l.slice()),e=t.aiDisplay,n=0;n<e.length;n++)r.test(t.aoData[e[n]]._sFilterRow)&&o.push(e[n]);t.aiDisplay=o}}function G(e,n,a,r){return e=n?e:er(e),a&&(e="^(?=.*?"+t.map(e.match(/"[^"]+"|[^ ]+/g)||[""],function(t){if('"'===t.charAt(0)){var e=t.match(/^"(.*)"$/);t=e?e[1]:t}return t.replace('"',"")}).join(")(?=.*?")+").*$"),RegExp(e,r?"i":"")}function z(t){var e,n,a=t.aoColumns,r=!1,o=0;for(e=t.aoData.length;o<e;o++){var i=t.aoData[o];if(!i._aFilterData){var l=[];for(r=0,n=a.length;r<n;r++){var s=a[r];s.bSearchable?(null===(s=D(t,o,r,"filter"))&&(s=""),"string"!=typeof s&&s.toString&&(s=s.toString())):s="",s.indexOf&&-1!==s.indexOf("&")&&(eo.innerHTML=s,s=ei?eo.textContent:eo.innerText),s.replace&&(s=s.replace(/[\r\n\u2028]/g,"")),l.push(s)}i._aFilterData=l,i._sFilterRow=l.join("  "),r=!0}}return r}function Z(t){return{search:t.sSearch,smart:t.bSmart,regex:t.bRegex,caseInsensitive:t.bCaseInsensitive}}function K(t){return{sSearch:t.search,bSmart:t.smart,bRegex:t.regex,bCaseInsensitive:t.caseInsensitive}}function Q(e){var n=e.sTableId,a=e.aanFeatures.i,r=t("<div/>",{class:e.oClasses.sInfo,id:a?null:n+"_info"});return a||(e.aoDrawCallback.push({fn:tt,sName:"information"}),r.attr("role","status").attr("aria-live","polite"),t(e.nTable).attr("aria-describedby",n+"_info")),r[0]}function tt(e){var n=e.aanFeatures.i;if(0!==n.length){var a=e.oLanguage,r=e._iDisplayStart+1,o=e.fnDisplayEnd(),i=e.fnRecordsTotal(),l=e.fnRecordsDisplay(),s=l?a.sInfo:a.sInfoEmpty;l!==i&&(s+=" "+a.sInfoFiltered),s+=a.sInfoPostFix,s=te(e,s),null!==(a=a.fnInfoCallback)&&(s=a.call(e.oInstance,e,r,o,i,l,s)),t(n).html(s)}}function te(t,e){var n=t.fnFormatNumber,a=t._iDisplayStart+1,r=t._iDisplayLength,o=t.fnRecordsDisplay(),i=-1===r;return e.replace(/_START_/g,n.call(t,a)).replace(/_END_/g,n.call(t,t.fnDisplayEnd())).replace(/_MAX_/g,n.call(t,t.fnRecordsTotal())).replace(/_TOTAL_/g,n.call(t,o)).replace(/_PAGE_/g,n.call(t,i?1:Math.ceil(a/r))).replace(/_PAGES_/g,n.call(t,i?1:Math.ceil(o/r)))}function tn(t){var e=t.iInitDisplayStart,n=t.aoColumns,a=t.oFeatures,r=t.bDeferLoading;if(t.bInitialised){H(t),j(t),P(t,t.aoHeader),P(t,t.aoFooter),tu(t,!0),a.bAutoWidth&&th(t);var o=0;for(a=n.length;o<a;o++){var i=n[o];i.sWidth&&(i.nTh.style.width=t$(i.sWidth))}tP(t,null,"preInit",[t]),O(t),("ssp"!=(n=tH(t))||r)&&("ajax"==n?E(t,[],function(n){var a=U(t,n);for(o=0;o<a.length;o++)S(t,a[o]);t.iInitDisplayStart=e,O(t),tu(t,!1),ta(t,n)},t):(tu(t,!1),ta(t)))}else setTimeout(function(){tn(t)},200)}function ta(t,e){t._bInitComplete=!0,(e||t.oInit.aaData)&&h(t),tP(t,null,"plugin-init",[t,e]),tP(t,"aoInitComplete","init",[t,e])}function tr(t,e){e=parseInt(e,10),t._iDisplayLength=e,tR(t),tP(t,null,"length",[t,e])}function to(e){var n=e.oClasses,a=e.sTableId,r=e.aLengthMenu,o=Array.isArray(r[0]),i=o?r[0]:r;r=o?r[1]:r,o=t("<select/>",{name:a+"_length","aria-controls":a,class:n.sLengthSelect});for(var l=0,s=i.length;l<s;l++)o[0][l]=new Option("number"==typeof r[l]?e.fnFormatNumber(r[l]):r[l],i[l]);var u=t("<div><label/></div>").addClass(n.sLength);return e.aanFeatures.l||(u[0].id=a+"_length"),u.children().append(e.oLanguage.sLengthMenu.replace("_MENU_",o[0].outerHTML)),t("select",u).val(e._iDisplayLength).on("change.DT",function(n){tr(e,t(this).val()),R(e)}),t(e.nTable).on("length.dt.DT",function(n,a,r){e===a&&t("select",u).val(r)}),u[0]}function ti(e){var n=e.sPaginationType,a=tB.ext.pager[n],r="function"==typeof a,o=function(t){R(t)};n=t("<div/>").addClass(e.oClasses.sPaging+n)[0];var i=e.aanFeatures;return r||a.fnInit(e,n,o),i.p||(n.id=e.sTableId+"_paginate",e.aoDrawCallback.push({fn:function(t){if(r){var e,n=t._iDisplayStart,l=t._iDisplayLength,s=t.fnRecordsDisplay(),u=-1===l;for(s=a(n=u?0:Math.ceil(n/l),l=u?1:Math.ceil(s/l)),u=0,e=i.p.length;u<e;u++)tO(t,"pageButton")(t,i.p[u],u,s,n,l)}else a.fnUpdate(t,o)},sName:"pagination"})),n}function tl(t,e,n){var a=t._iDisplayStart,r=t._iDisplayLength,o=t.fnRecordsDisplay();return 0===o||-1===r?a=0:"number"==typeof e?(a=e*r)>o&&(a=0):"first"==e?a=0:"previous"==e?0>(a=0<=r?a-r:0)&&(a=0):"next"==e?a+r<o&&(a+=r):"last"==e?a=Math.floor((o-1)/r)*r:tI(t,0,"Unknown paging action: "+e,5),e=t._iDisplayStart!==a,t._iDisplayStart=a,e&&(tP(t,null,"page",[t]),n&&R(t)),e}function ts(e){return t("<div/>",{id:e.aanFeatures.r?null:e.sTableId+"_processing",class:e.oClasses.sProcessing}).html(e.oLanguage.sProcessing).insertBefore(e.nTable)[0]}function tu(e,n){e.oFeatures.bProcessing&&t(e.aanFeatures.r).css("display",n?"block":"none"),tP(e,null,"processing",[e,n])}function tc(e){var n=t(e.nTable),a=e.oScroll;if(""===a.sX&&""===a.sY)return e.nTable;var r=a.sX,o=a.sY,i=e.oClasses,l=n.children("caption"),s=l.length?l[0]._captionSide:null,u=t(n[0].cloneNode(!1)),c=t(n[0].cloneNode(!1)),f=n.children("tfoot");f.length||(f=null),u=t("<div/>",{class:i.sScrollWrapper}).append(t("<div/>",{class:i.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:r?r?t$(r):null:"100%"}).append(t("<div/>",{class:i.sScrollHeadInner}).css({"box-sizing":"content-box",width:a.sXInner||"100%"}).append(u.removeAttr("id").css("margin-left",0).append("top"===s?l:null).append(n.children("thead"))))).append(t("<div/>",{class:i.sScrollBody}).css({position:"relative",overflow:"auto",width:r?t$(r):null}).append(n)),f&&u.append(t("<div/>",{class:i.sScrollFoot}).css({overflow:"hidden",border:0,width:r?r?t$(r):null:"100%"}).append(t("<div/>",{class:i.sScrollFootInner}).append(c.removeAttr("id").css("margin-left",0).append("bottom"===s?l:null).append(n.children("tfoot")))));var d=(n=u.children())[0];i=n[1];var h=f?n[2]:null;return r&&t(i).on("scroll.DT",function(t){t=this.scrollLeft,d.scrollLeft=t,f&&(h.scrollLeft=t)}),t(i).css("max-height",o),a.bCollapse||t(i).css("height",o),e.nScrollHead=d,e.nScrollBody=i,e.nScrollFoot=h,e.aoDrawCallback.push({fn:tf,sName:"scrolling"}),u[0]}function tf(n){var r=n.oScroll,o=r.sX,i=r.sXInner,l=r.sY;r=r.iBarWidth;var s=t(n.nScrollHead),u=s[0].style,c=s.children("div"),f=c[0].style,d=c.children("table"),g=t(c=n.nScrollBody),b=c.style,$=t(n.nScrollFoot).children("div"),m=$.children("table"),v=t(n.nTHead),S=t(n.nTable),y=S[0],D=y.style,T=n.nTFoot?t(n.nTFoot):null,C=n.oBrowser,w=C.bScrollOversize;t3(n.aoColumns,"nTh");var x,_=[],I=[],A=[],L=[],F=function(t){(t=t.style).paddingTop="0",t.paddingBottom="0",t.borderTopWidth="0",t.borderBottomWidth="0",t.height=0},j=c.scrollHeight>c.clientHeight;if(n.scrollBarVis!==j&&n.scrollBarVis!==a)n.scrollBarVis=j,h(n);else{if(n.scrollBarVis=j,S.children("thead, tfoot").remove(),T){var P=T.clone().prependTo(S),R=T.find("tr");P=P.find("tr")}var O=v.clone().prependTo(S);v=v.find("tr"),j=O.find("tr"),O.find("th, td").removeAttr("tabindex"),o||(b.width="100%",s[0].style.width="100%"),t.each(k(n,O),function(t,e){x=p(n,t),e.style.width=n.aoColumns[x].sWidth}),T&&td(function(t){t.style.width=""},P),s=S.outerWidth(),""===o?(D.width="100%",w&&(S.find("tbody").height()>c.offsetHeight||"scroll"==g.css("overflow-y"))&&(D.width=t$(S.outerWidth()-r)),s=S.outerWidth()):""!==i&&(D.width=t$(i),s=S.outerWidth()),td(F,j),td(function(n){var a=e.getComputedStyle?e.getComputedStyle(n).width:t$(t(n).width());A.push(n.innerHTML),_.push(a)},j),td(function(t,e){t.style.width=_[e]},v),t(j).css("height",0),T&&(td(F,P),td(function(e){L.push(e.innerHTML),I.push(t$(t(e).css("width")))},P),td(function(t,e){t.style.width=I[e]},R),t(P).height(0)),td(function(t,e){t.innerHTML='<div class="dataTables_sizing">'+A[e]+"</div>",t.childNodes[0].style.height="0",t.childNodes[0].style.overflow="hidden",t.style.width=_[e]},j),T&&td(function(t,e){t.innerHTML='<div class="dataTables_sizing">'+L[e]+"</div>",t.childNodes[0].style.height="0",t.childNodes[0].style.overflow="hidden",t.style.width=I[e]},P),Math.round(S.outerWidth())<Math.round(s)?(R=c.scrollHeight>c.offsetHeight||"scroll"==g.css("overflow-y")?s+r:s,w&&(c.scrollHeight>c.offsetHeight||"scroll"==g.css("overflow-y"))&&(D.width=t$(R-r)),""!==o&&""===i||tI(n,1,"Possible column misalignment",6)):R="100%",b.width=t$(R),u.width=t$(R),T&&(n.nScrollFoot.style.width=t$(R)),!l&&w&&(b.height=t$(y.offsetHeight+r)),o=S.outerWidth(),d[0].style.width=t$(o),f.width=t$(o),i=S.height()>c.clientHeight||"scroll"==g.css("overflow-y"),f[l="padding"+(C.bScrollbarLeft?"Left":"Right")]=i?r+"px":"0px",T&&(m[0].style.width=t$(o),$[0].style.width=t$(o),$[0].style[l]=i?r+"px":"0px"),S.children("colgroup").insertBefore(S.children("thead")),g.trigger("scroll"),(n.bSorted||n.bFiltered)&&!n._drawHold&&(c.scrollTop=0)}}function td(t,e,n){for(var a,r,o=0,i=0,l=e.length;i<l;){for(a=e[i].firstChild,r=n?n[i].firstChild:null;a;)1===a.nodeType&&(n?t(a,r,o):t(a,o),o++),a=a.nextSibling,r=n?r.nextSibling:null;i++}}function th(n){var a,r=n.nTable,o=n.aoColumns,i=n.oScroll,l=i.sY,s=i.sX,u=i.sXInner,c=o.length,f=$(n,"bVisible"),d=t("th",n.nTHead),g=r.getAttribute("width"),m=r.parentNode,v=!1,S=n.oBrowser;for(i=S.bScrollOversize,(a=r.style.width)&&-1!==a.indexOf("%")&&(g=a),a=0;a<f.length;a++){var y=o[f[a]];null!==y.sWidth&&(y.sWidth=tp(y.sWidthOrig,m),v=!0)}if(!i&&(v||s||l||c!=b(n)||c!=d.length)){(c=t(r).clone().css("visibility","hidden").removeAttr("id")).find("tbody tr").remove();var D=t("<tr/>").appendTo(c.find("tbody"));for(c.find("thead, tfoot").remove(),c.append(t(n.nTHead).clone()).append(t(n.nTFoot).clone()),c.find("tfoot th, tfoot td").css("width",""),d=k(n,c.find("thead")[0]),a=0;a<f.length;a++)y=o[f[a]],d[a].style.width=null!==y.sWidthOrig&&""!==y.sWidthOrig?t$(y.sWidthOrig):"",y.sWidthOrig&&s&&t(d[a]).append(t("<div/>").css({width:y.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(n.aoData.length)for(a=0;a<f.length;a++)y=o[v=f[a]],t(tg(n,v)).clone(!1).append(y.sContentPadding).appendTo(D);for(t("[name]",c).removeAttr("name"),y=t("<div/>").css(s||l?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(c).appendTo(m),s&&u?c.width(u):s?(c.css("width","auto"),c.removeAttr("width"),c.width()<m.clientWidth&&g&&c.width(m.clientWidth)):l?c.width(m.clientWidth):g&&c.width(g),a=l=0;a<f.length;a++)u=(m=t(d[a])).outerWidth()-m.width(),l+=m=S.bBounding?Math.ceil(d[a].getBoundingClientRect().width):m.outerWidth(),o[f[a]].sWidth=t$(m-u);r.style.width=t$(l),y.remove()}else for(a=0;a<c;a++)null!==(f=p(n,a))&&(o[f].sWidth=t$(d.eq(a).width()));g&&(r.style.width=t$(g)),(g||s)&&!n._reszEvt&&(r=function(){t(e).on("resize.DT-"+n.sInstance,es(function(){h(n)}))},i?setTimeout(r,1e3):r(),n._reszEvt=!0)}function tp(e,a){return e?(a=(e=t("<div/>").css("width",t$(e)).appendTo(a||n.body))[0].offsetWidth,e.remove(),a):0}function tg(e,n){var a=tb(e,n);if(0>a)return null;var r=e.aoData[a];return r.nTr?r.anCells[n]:t("<td/>").html(D(e,a,n,"display"))[0]}function tb(t,e){for(var n,a=-1,r=-1,o=0,i=t.aoData.length;o<i;o++)(n=(n=(n=D(t,o,e,"display")+"").replace(el,"")).replace(/&nbsp;/g," ")).length>a&&(a=n.length,r=o);return r}function t$(t){return null===t?"0px":"number"==typeof t?0>t?"0px":t+"px":t.match(/\d$/)?t+"px":t}function tm(e){var n=[],r=e.aoColumns,o=e.aaSortingFixed,i=t.isPlainObject(o),l=[],s=function(e){e.length&&!Array.isArray(e[0])?l.push(e):t.merge(l,e)};for(Array.isArray(o)&&s(o),i&&o.pre&&s(o.pre),s(e.aaSorting),i&&o.post&&s(o.post),e=0;e<l.length;e++){var u=l[e][0];for(s=r[u].aDataSort,o=0,i=s.length;o<i;o++){var c=s[o],f=r[c].sType||"string";l[e]._idx===a&&(l[e]._idx=t.inArray(l[e][1],r[c].asSorting)),n.push({src:u,col:c,dir:l[e][1],index:l[e]._idx,type:f,formatter:tB.ext.type.order[f+"-pre"]})}}return n}function tv(t){var e,n=[],a=tB.ext.type.order,r=t.aoData,o=0,i=t.aiDisplayMaster;m(t);var l=tm(t),s=0;for(e=l.length;s<e;s++){var u=l[s];u.formatter&&o++,tT(t,u.col)}if("ssp"!=tH(t)&&0!==l.length){for(s=0,e=i.length;s<e;s++)n[i[s]]=s;o===l.length?i.sort(function(t,e){var a,o=l.length,i=r[t]._aSortData,s=r[e]._aSortData;for(a=0;a<o;a++){var u=l[a],c=i[u.col],f=s[u.col];if(0!=(c=c<f?-1:c>f?1:0))return"asc"===u.dir?c:-c}return(c=n[t])<(f=n[e])?-1:c>f?1:0}):i.sort(function(t,e){var o,i=l.length,s=r[t]._aSortData,u=r[e]._aSortData;for(o=0;o<i;o++){var c=l[o],f=s[c.col],d=u[c.col];if(0!==(f=(c=a[c.type+"-"+c.dir]||a["string-"+c.dir])(f,d)))return f}return(f=n[t])<(d=n[e])?-1:f>d?1:0})}t.bSorted=!0}function tS(t){var e=t.aoColumns,n=tm(t);t=t.oLanguage.oAria;for(var a=0,r=e.length;a<r;a++){var o=e[a],i=o.asSorting,l=o.ariaTitle||o.sTitle.replace(/<.*?>/g,""),s=o.nTh;s.removeAttribute("aria-sort"),o.bSortable&&(0<n.length&&n[0].col==a?(s.setAttribute("aria-sort","asc"==n[0].dir?"ascending":"descending"),o=i[n[0].index+1]||i[0]):o=i[0],l+="asc"===o?t.sSortAscending:t.sSortDescending),s.setAttribute("aria-label",l)}}function ty(e,n,r,o){var i=e.aaSorting,l=e.aoColumns[n].asSorting,s=function(e,n){var r=e._idx;return r===a&&(r=t.inArray(e[1],l)),r+1<l.length?r+1:n?null:0};"number"==typeof i[0]&&(i=e.aaSorting=[i]),r&&e.oFeatures.bSortMulti?-1!==(r=t.inArray(n,t3(i,"0")))?(null===(n=s(i[r],!0))&&1===i.length&&(n=0),null===n?i.splice(r,1):(i[r][1]=l[n],i[r]._idx=n)):(i.push([n,l[0],0]),i[i.length-1]._idx=0):i.length&&i[0][0]==n?(n=s(i[0]),i.length=1,i[0][1]=l[n],i[0]._idx=n):(i.length=0,i.push([n,l[0]]),i[0]._idx=0),O(e),"function"==typeof o&&o(e)}function tD(t,e,n,a){var r=t.aoColumns[n];tF(e,{},function(e){!1!==r.bSortable&&(t.oFeatures.bProcessing?(tu(t,!0),setTimeout(function(){ty(t,n,e.shiftKey,a),"ssp"!==tH(t)&&tu(t,!1)},0)):ty(t,n,e.shiftKey,a))})}function t8(e){var n,a=e.aLastSort,r=e.oClasses.sSortColumn,o=tm(e),i=e.oFeatures;if(i.bSort&&i.bSortClasses){for(i=0,n=a.length;i<n;i++){var l=a[i].src;t(t3(e.aoData,"anCells",l)).removeClass(r+(2>i?i+1:3))}for(i=0,n=o.length;i<n;i++)l=o[i].src,t(t3(e.aoData,"anCells",l)).addClass(r+(2>i?i+1:3))}e.aLastSort=o}function tT(t,e){var n,a=t.aoColumns[e],r=tB.ext.order[a.sSortDataType];r&&(n=r.call(t.oInstance,t,e,g(t,e)));for(var o,i=tB.ext.type.order[a.sType+"-pre"],l=0,s=t.aoData.length;l<s;l++)(a=t.aoData[l])._aSortData||(a._aSortData=[]),(!a._aSortData[e]||r)&&(o=r?n[l]:D(t,l,e,"sort"),a._aSortData[e]=i?i(o):o)}function tC(e){if(!e._bLoadingState){var n={time:+new Date,start:e._iDisplayStart,length:e._iDisplayLength,order:t.extend(!0,[],e.aaSorting),search:Z(e.oPreviousSearch),columns:t.map(e.aoColumns,function(t,n){return{visible:t.bVisible,search:Z(e.aoPreSearchCols[n])}})};e.oSavedState=n,tP(e,"aoStateSaveParams","stateSaveParams",[e,n]),e.oFeatures.bStateSave&&!e.bDestroying&&e.fnStateSaveCallback.call(e.oInstance,e,n)}}function tw(t,e,n){if(t.oFeatures.bStateSave)return(e=t.fnStateLoadCallback.call(t.oInstance,t,function(e){tx(t,e,n)}))!==a&&tx(t,e,n),!0;n()}function tx(e,n,r){var o,i=e.aoColumns;e._bLoadingState=!0;var l=e._bInitComplete?new tB.Api(e):null;if(n&&n.time){var s=tP(e,"aoStateLoadParams","stateLoadParams",[e,n]);if(-1!==t.inArray(!1,s))e._bLoadingState=!1;else if(0<(s=e.iStateDuration)&&n.time<+new Date-1e3*s)e._bLoadingState=!1;else if(n.columns&&i.length!==n.columns.length)e._bLoadingState=!1;else{if(e.oLoadedState=t.extend(!0,{},n),n.start!==a&&(null===l?(e._iDisplayStart=n.start,e.iInitDisplayStart=n.start):tl(e,n.start/n.length)),n.length!==a&&(e._iDisplayLength=n.length),n.order!==a&&(e.aaSorting=[],t.each(n.order,function(t,n){e.aaSorting.push(n[0]>=i.length?[0,n[1]]:n)})),n.search!==a&&t.extend(e.oPreviousSearch,K(n.search)),n.columns){for(s=0,o=n.columns.length;s<o;s++){var u=n.columns[s];u.visible!==a&&(l?l.column(s).visible(u.visible,!1):i[s].bVisible=u.visible),u.search!==a&&t.extend(e.aoPreSearchCols[s],K(u.search))}l&&l.columns.adjust()}e._bLoadingState=!1,tP(e,"aoStateLoaded","stateLoaded",[e,n])}}else e._bLoadingState=!1;r()}function t_(e){var n=tB.settings;return -1!==(e=t.inArray(e,t3(n,"nTable")))?n[e]:null}function tI(t,n,a,r){if(a="DataTables warning: "+(t?"table id="+t.sTableId+" - ":"")+a,r&&(a+=". For more information about this error, please see http://datatables.net/tn/"+r),n)e.console&&console.log&&console.log(a);else if(n=(n=tB.ext).sErrMode||n.errMode,t&&tP(t,null,"error",[t,r,a]),"alert"==n)alert(a);else{if("throw"==n)throw Error(a);"function"==typeof n&&n(t,r,a)}}function tA(e,n,r,o){Array.isArray(r)?t.each(r,function(t,a){Array.isArray(a)?tA(e,n,a[0],a[1]):tA(e,n,a)}):(o===a&&(o=r),n[r]!==a&&(e[o]=n[r]))}function tL(e,n,a){var r;for(r in n)if(n.hasOwnProperty(r)){var o=n[r];t.isPlainObject(o)?(t.isPlainObject(e[r])||(e[r]={}),t.extend(!0,e[r],o)):a&&"data"!==r&&"aaData"!==r&&Array.isArray(o)?e[r]=o.slice():e[r]=o}return e}function tF(e,n,a){t(e).on("click.DT",n,function(n){t(e).trigger("blur"),a(n)}).on("keypress.DT",n,function(t){13===t.which&&(t.preventDefault(),a(t))}).on("selectstart.DT",function(){return!1})}function tj(t,e,n,a){n&&t[e].push({fn:n,sName:a})}function tP(e,n,a,r){var o=[];return n&&(o=t.map(e[n].slice().reverse(),function(t,n){return t.fn.apply(e.oInstance,r)})),null!==a&&(n=t.Event(a+".dt"),t(e.nTable).trigger(n,r),o.push(n.result)),o}function tR(t){var e=t._iDisplayStart,n=t.fnDisplayEnd(),a=t._iDisplayLength;e>=n&&(e=n-a),e-=e%a,(-1===a||0>e)&&(e=0),t._iDisplayStart=e}function tO(e,n){e=e.renderer;var a=tB.ext.renderer[n];return t.isPlainObject(e)&&e[n]?a[e[n]]||a._:"string"==typeof e&&a[e]||a._}function tH(t){return t.oFeatures.bServerSide?"ssp":t.ajax||t.sAjaxSource?"ajax":"dom"}function tN(t,e){var n=eI.numbers_length,a=Math.floor(n/2);return e<=n?t=t4(0,e):t<=a?((t=t4(0,n-2)).push("ellipsis"),t.push(e-1)):(t>=e-1-a?t=t4(e-(n-2),e):((t=t4(t-a+2,t+a-1)).push("ellipsis"),t.push(e-1)),t.splice(0,0,"ellipsis"),t.splice(0,0,0)),t.DT_el="span",t}function tk(e){t.each({num:function(t){return eA(t,e)},"num-fmt":function(t){return eA(t,e,tq)},"html-num":function(t){return eA(t,e,tX)},"html-num-fmt":function(t){return eA(t,e,tX,tq)}},function(t,n){tM.type.order[t+e+"-pre"]=n,t.match(/^html\-/)&&(tM.type.search[t+e]=tM.type.search.html)})}function tE(t){return function(){var e=[t_(this[tB.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return tB.ext.internal[t].apply(this,e)}}var tM,tW,t9,tB=function(e,n){if(this instanceof tB)return t(e).DataTable(n);n=e,this.$=function(t,e){return this.api(!0).$(t,e)},this._=function(t,e){return this.api(!0).rows(t,e).data()},this.api=function(t){return new ed(t?t_(this[tM.iApiIndex]):this)},this.fnAddData=function(e,n){var r=this.api(!0);return e=Array.isArray(e)&&(Array.isArray(e[0])||t.isPlainObject(e[0]))?r.rows.add(e):r.row.add(e),(n===a||n)&&r.draw(),e.flatten().toArray()},this.fnAdjustColumnSizing=function(t){var e=this.api(!0).columns.adjust(),n=e.settings()[0],r=n.oScroll;t===a||t?e.draw(!1):(""!==r.sX||""!==r.sY)&&tf(n)},this.fnClearTable=function(t){var e=this.api(!0).clear();(t===a||t)&&e.draw()},this.fnClose=function(t){this.api(!0).row(t).child.hide()},this.fnDeleteRow=function(t,e,n){var r=this.api(!0),o=(t=r.rows(t)).settings()[0],i=o.aoData[t[0][0]];return t.remove(),e&&e.call(this,o,i),(n===a||n)&&r.draw(),i},this.fnDestroy=function(t){this.api(!0).destroy(t)},this.fnDraw=function(t){this.api(!0).draw(t)},this.fnFilter=function(t,e,n,r,o,i){o=this.api(!0),null===e||e===a?o.search(t,n,r,i):o.column(e).search(t,n,r,i),o.draw()},this.fnGetData=function(t,e){var n=this.api(!0);if(t!==a){var r=t.nodeName?t.nodeName.toLowerCase():"";return e!==a||"td"==r||"th"==r?n.cell(t,e).data():n.row(t).data()||null}return n.data().toArray()},this.fnGetNodes=function(t){var e=this.api(!0);return t!==a?e.row(t).node():e.rows().nodes().flatten().toArray()},this.fnGetPosition=function(t){var e=this.api(!0),n=t.nodeName.toUpperCase();return"TR"==n?e.row(t).index():"TD"==n||"TH"==n?[(t=e.cell(t).index()).row,t.columnVisible,t.column]:null},this.fnIsOpen=function(t){return this.api(!0).row(t).child.isShown()},this.fnOpen=function(t,e,n){return this.api(!0).row(t).child(e,n).show().child()[0]},this.fnPageChange=function(t,e){t=this.api(!0).page(t),(e===a||e)&&t.draw(!1)},this.fnSetColumnVis=function(t,e,n){t=this.api(!0).column(t).visible(e),(n===a||n)&&t.columns.adjust().draw()},this.fnSettings=function(){return t_(this[tM.iApiIndex])},this.fnSort=function(t){this.api(!0).order(t).draw()},this.fnSortListener=function(t,e,n){this.api(!0).order.listener(t,e,n)},this.fnUpdate=function(t,e,n,r,o){var i=this.api(!0);return n===a||null===n?i.row(e).data(t):i.cell(e,n).data(t),(o===a||o)&&i.columns.adjust(),(r===a||r)&&i.draw(),0},this.fnVersionCheck=tM.fnVersionCheck;var r=this,c=n===a,h=this.length;for(var p in c&&(n={}),this.oApi=this.internal=tM.internal,tB.ext.internal)p&&(this[p]=tE(p));return this.each(function(){var e,p={},g=1<h?tL(p,n,!0):n,b=0;p=this.getAttribute("id");var $=!1,m=tB.defaults,D=t(this);if("table"!=this.nodeName.toLowerCase())tI(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{l(m),s(m.column),o(m,m,!0),o(m.column,m.column,!0),o(m,t.extend(g,D.data()),!0);var T=tB.settings;for(b=0,e=T.length;b<e;b++){var C=T[b];if(C.nTable==this||C.nTHead&&C.nTHead.parentNode==this||C.nTFoot&&C.nTFoot.parentNode==this){var w=g.bRetrieve!==a?g.bRetrieve:m.bRetrieve;if(c||w)return C.oInstance;if(g.bDestroy!==a?g.bDestroy:m.bDestroy){C.oInstance.fnDestroy();break}tI(C,0,"Cannot reinitialise DataTable",3);return}if(C.sTableId==this.id){T.splice(b,1);break}}(null===p||""===p)&&(this.id=p="DataTables_Table_"+tB.ext._unique++);var x=t.extend(!0,{},tB.models.oSettings,{sDestroyWidth:D[0].style.width,sInstance:p,sTableId:p});x.nTable=this,x.oApi=r.internal,x.oInit=g,T.push(x),x.oInstance=1===r.length?r:D.dataTable(),l(g),i(g.oLanguage),g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=Array.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]),g=tL(t.extend(!0,{},m),g),tA(x.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" ")),tA(x,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"]]),tA(x.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]),tA(x.oLanguage,g,"fnInfoCallback"),tj(x,"aoDrawCallback",g.fnDrawCallback,"user"),tj(x,"aoServerParams",g.fnServerParams,"user"),tj(x,"aoStateSaveParams",g.fnStateSaveParams,"user"),tj(x,"aoStateLoadParams",g.fnStateLoadParams,"user"),tj(x,"aoStateLoaded",g.fnStateLoaded,"user"),tj(x,"aoRowCallback",g.fnRowCallback,"user"),tj(x,"aoRowCreatedCallback",g.fnCreatedRow,"user"),tj(x,"aoHeaderCallback",g.fnHeaderCallback,"user"),tj(x,"aoFooterCallback",g.fnFooterCallback,"user"),tj(x,"aoInitComplete",g.fnInitComplete,"user"),tj(x,"aoPreDrawCallback",g.fnPreDrawCallback,"user"),x.rowIdFn=en(g.rowId),u(x);var _=x.oClasses;t.extend(_,tB.ext.classes,g.oClasses),D.addClass(_.sTable),x.iInitDisplayStart===a&&(x.iInitDisplayStart=g.iDisplayStart,x._iDisplayStart=g.iDisplayStart),null!==g.iDeferLoading&&(x.bDeferLoading=!0,p=Array.isArray(g.iDeferLoading),x._iRecordsDisplay=p?g.iDeferLoading[0]:g.iDeferLoading,x._iRecordsTotal=p?g.iDeferLoading[1]:g.iDeferLoading);var I=x.oLanguage;t.extend(!0,I,g.oLanguage),I.sUrl?(t.ajax({dataType:"json",url:I.sUrl,success:function(e){o(m.oLanguage,e),i(e),t.extend(!0,I,e),tP(x,null,"i18n",[x]),tn(x)},error:function(){tn(x)}}),$=!0):tP(x,null,"i18n",[x]),null===g.asStripeClasses&&(x.asStripeClasses=[_.sStripeOdd,_.sStripeEven]),p=x.asStripeClasses;var A=D.children("tbody").find("tr").eq(0);if(-1!==t.inArray(!0,t.map(p,function(t,e){return A.hasClass(t)}))&&(t("tbody tr",this).removeClass(p.join(" ")),x.asDestroyStripes=p.slice()),p=[],0!==(T=this.getElementsByTagName("thead")).length&&(N(x.aoHeader,T[0]),p=k(x)),null===g.aoColumns)for(T=[],b=0,e=p.length;b<e;b++)T.push(null);else T=g.aoColumns;for(b=0,e=T.length;b<e;b++)f(x,p?p[b]:null);if(v(x,g.aoColumnDefs,T,function(t,e){d(x,t,e)}),A.length){var L=function(t,e){return null!==t.getAttribute("data-"+e)?e:null};t(A[0]).children("th, td").each(function(t,e){var n=x.aoColumns[t];if(n.mData===t){var r=L(e,"sort")||L(e,"order");e=L(e,"filter")||L(e,"search"),(null!==r||null!==e)&&(n.mData={_:t+".display",sort:null!==r?t+".@data-"+r:a,type:null!==r?t+".@data-"+r:a,filter:null!==e?t+".@data-"+e:a},d(x,t))}})}var F=x.oFeatures;p=function(){if(g.aaSorting===a){var n=x.aaSorting;for(b=0,e=n.length;b<e;b++)n[b][1]=x.aoColumns[b].asSorting[0]}t8(x),F.bSort&&tj(x,"aoDrawCallback",function(){if(x.bSorted){var e=tm(x),n={};t.each(e,function(t,e){n[e.src]=e.dir}),tP(x,null,"order",[x,e,n]),tS(x)}}),tj(x,"aoDrawCallback",function(){(x.bSorted||"ssp"===tH(x)||F.bDeferRender)&&t8(x)},"sc"),n=D.children("caption").each(function(){this._captionSide=t(this).css("caption-side")});var r=D.children("thead");0===r.length&&(r=t("<thead/>").appendTo(D)),x.nTHead=r[0];var o=D.children("tbody");if(0===o.length&&(o=t("<tbody/>").insertAfter(r)),x.nTBody=o[0],0===(r=D.children("tfoot")).length&&0<n.length&&(""!==x.oScroll.sX||""!==x.oScroll.sY)&&(r=t("<tfoot/>").appendTo(D)),0===r.length||0===r.children().length?D.addClass(_.sNoFooter):0<r.length&&(x.nTFoot=r[0],N(x.aoFooter,x.nTFoot)),g.aaData)for(b=0;b<g.aaData.length;b++)S(x,g.aaData[b]);else(x.bDeferLoading||"dom"==tH(x))&&y(x,t(x.nTBody).children("tr"));x.aiDisplay=x.aiDisplayMaster.slice(),x.bInitialised=!0,!1===$&&tn(x)},tj(x,"aoDrawCallback",tC,"state_save"),g.bStateSave?(F.bStateSave=!0,tw(x,g,p)):p()}}),r=null,this},tU={},tV=/[\r\n\u2028]/g,tX=/<.*?>/g,t0=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,tJ=/(\/|\.|\*|\+|\?|\||\(|\)|\[|\]|\{|\}|\\|\$|\^|\-)/g,tq=/['\u00A0,$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,tY=function(t){return!t||!0===t||"-"===t},tG=function(t){var e=parseInt(t,10);return!isNaN(e)&&isFinite(t)?e:null},tz=function(t,e){return tU[e]||(tU[e]=RegExp(er(e),"g")),"string"==typeof t&&"."!==e?t.replace(/\./g,"").replace(tU[e],"."):t},t1=function(t,e,n){var a="string"==typeof t;return!!tY(t)||(e&&a&&(t=tz(t,e)),n&&a&&(t=t.replace(tq,"")),!isNaN(parseFloat(t))&&isFinite(t))},tZ=function(t,e,n){return!!tY(t)||(tY(t)||"string"==typeof t)&&!!t1(t.replace(tX,""),e,n)||null},t3=function(t,e,n){var r=[],o=0,i=t.length;if(n!==a)for(;o<i;o++)t[o]&&t[o][e]&&r.push(t[o][e][n]);else for(;o<i;o++)t[o]&&r.push(t[o][e]);return r},t2=function(t,e,n,r){var o=[],i=0,l=e.length;if(r!==a)for(;i<l;i++)t[e[i]][n]&&o.push(t[e[i]][n][r]);else for(;i<l;i++)o.push(t[e[i]][n]);return o},t4=function(t,e){var n=[];if(e===a){e=0;var r=t}else r=e,e=t;for(t=e;t<r;t++)n.push(t);return n},t6=function(t){for(var e=[],n=0,a=t.length;n<a;n++)t[n]&&e.push(t[n]);return e},t5=function(t){a:{if(!(2>t.length))for(var e=t.slice().sort(),n=e[0],a=1,r=e.length;a<r;a++){if(e[a]===n){e=!1;break a}n=e[a]}e=!0}if(e)return t.slice();e=[],r=t.length;var o,i=0;a=0;a:for(;a<r;a++){for(o=0,n=t[a];o<i;o++)if(e[o]===n)continue a;e.push(n),i++}return e},tK=function(t,e){if(Array.isArray(e))for(var n=0;n<e.length;n++)tK(t,e[n]);else t.push(e);return t},t7=function(t,e){return e===a&&(e=0),-1!==this.indexOf(t,e)};Array.isArray||(Array.isArray=function(t){return"[object Array]"===Object.prototype.toString.call(t)}),Array.prototype.includes||(Array.prototype.includes=t7),String.prototype.trim||(String.prototype.trim=function(){return this.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,"")}),String.prototype.includes||(String.prototype.includes=t7),tB.util={throttle:function(t,e){var n,r,o=e!==a?e:200;return function(){var e=this,i=+new Date,l=arguments;n&&i<n+o?(clearTimeout(r),r=setTimeout(function(){n=a,t.apply(e,l)},o)):(n=i,t.apply(e,l))}},escapeRegex:function(t){return t.replace(tJ,"\\$1")},set:function(e){if(t.isPlainObject(e))return tB.util.set(e._);if(null===e)return function(){};if("function"==typeof e)return function(t,n,a){e(t,"set",n,a)};if("string"!=typeof e||-1===e.indexOf(".")&&-1===e.indexOf("[")&&-1===e.indexOf("("))return function(t,n){t[e]=n};var n=function(t,e,r){for(var o,i,l=(r=C(r))[r.length-1],s=0,u=r.length-1;s<u;s++){if("__proto__"===r[s]||"constructor"===r[s])throw Error("Cannot set prototype values");if(o=r[s].match(et),i=r[s].match(ee),o){if(r[s]=r[s].replace(et,""),t[r[s]]=[],(l=r.slice()).splice(0,s+1),o=l.join("."),Array.isArray(e))for(i=0,u=e.length;i<u;i++)n(l={},e[i],o),t[r[s]].push(l);else t[r[s]]=e;return}i&&(r[s]=r[s].replace(ee,""),t=t[r[s]](e)),(null===t[r[s]]||t[r[s]]===a)&&(t[r[s]]={}),t=t[r[s]]}l.match(ee)?t[l.replace(ee,"")](e):t[l.replace(et,"")]=e};return function(t,a){return n(t,a,e)}},get:function(e){if(t.isPlainObject(e)){var n={};return t.each(e,function(t,e){e&&(n[t]=tB.util.get(e))}),function(t,e,r,o){var i=n[e]||n._;return i!==a?i(t,e,r,o):t}}if(null===e)return function(t){return t};if("function"==typeof e)return function(t,n,a,r){return e(t,n,a,r)};if("string"!=typeof e||-1===e.indexOf(".")&&-1===e.indexOf("[")&&-1===e.indexOf("("))return function(t,n){return t[e]};var r=function(t,e,n){if(""!==n)for(var o=C(n),i=0,l=o.length;i<l;i++){n=o[i].match(et);var s=o[i].match(ee);if(n){if(o[i]=o[i].replace(et,""),""!==o[i]&&(t=t[o[i]]),s=[],o.splice(0,i+1),o=o.join("."),Array.isArray(t))for(i=0,l=t.length;i<l;i++)s.push(r(t[i],e,o));t=""===(t=n[0].substring(1,n[0].length-1))?s:s.join(t);break}if(s){o[i]=o[i].replace(ee,""),t=t[o[i]]();continue}if(null===t||t[o[i]]===a)return a;t=t[o[i]]}return t};return function(t,n){return r(t,n,e)}}};var tQ=function(t,e,n){t[e]!==a&&(t[n]=t[e])},et=/\[.*?\]$/,ee=/\(\)$/,en=tB.util.get,ea=tB.util.set,er=tB.util.escapeRegex,eo=t("<div>")[0],ei=eo.textContent!==a,el=/<.*?>/g,es=tB.util.throttle,eu=[],ec=Array.prototype,ef=function(e){var n,a=tB.settings,r=t.map(a,function(t,e){return t.nTable});if(!e)return[];if(e.nTable&&e.oApi)return[e];if(e.nodeName&&"table"===e.nodeName.toLowerCase()){var o=t.inArray(e,r);return -1!==o?[a[o]]:null}return e&&"function"==typeof e.settings?e.settings().toArray():("string"==typeof e?n=t(e):e instanceof t&&(n=e),n)?n.map(function(e){return -1!==(o=t.inArray(this,r))?a[o]:null}).toArray():void 0},ed=function(e,n){if(!(this instanceof ed))return new ed(e,n);var a=[],r=function(t){(t=ef(t))&&a.push.apply(a,t)};if(Array.isArray(e))for(var o=0,i=e.length;o<i;o++)r(e[o]);else r(e);this.context=t5(a),n&&t.merge(this,n),this.selector={rows:null,cols:null,opts:null},ed.extend(this,this,eu)};tB.Api=ed,t.extend(ed.prototype,{any:function(){return 0!==this.count()},concat:ec.concat,context:[],count:function(){return this.flatten().length},each:function(t){for(var e=0,n=this.length;e<n;e++)t.call(this,this[e],e,this);return this},eq:function(t){var e=this.context;return e.length>t?new ed(e[t],this[t]):null},filter:function(t){var e=[];if(ec.filter)e=ec.filter.call(this,t,this);else for(var n=0,a=this.length;n<a;n++)t.call(this,this[n],n,this)&&e.push(this[n]);return new ed(this.context,e)},flatten:function(){var t=[];return new ed(this.context,t.concat.apply(t,this.toArray()))},join:ec.join,indexOf:ec.indexOf||function(t,e){e=e||0;for(var n=this.length;e<n;e++)if(this[e]===t)return e;return -1},iterator:function(t,e,n,r){var o,i,l,s=[],u=this.context,c=this.selector;"string"==typeof t&&(r=n,n=e,e=t,t=!1);var f=0;for(o=u.length;f<o;f++){var d=new ed(u[f]);if("table"===e){var h=n.call(d,u[f],f);h!==a&&s.push(h)}else if("columns"===e||"rows"===e)(h=n.call(d,u[f],this[f],f))!==a&&s.push(h);else if("column"===e||"column-rows"===e||"row"===e||"cell"===e){var p=this[f];"column-rows"===e&&(l=em(u[f],c.opts));var g=0;for(i=p.length;g<i;g++)h=p[g],(h="cell"===e?n.call(d,u[f],h.row,h.column,f,g):n.call(d,u[f],h,f,g,l))!==a&&s.push(h)}}return s.length||r?((e=(t=new ed(u,t?s.concat.apply([],s):s)).selector).rows=c.rows,e.cols=c.cols,e.opts=c.opts,t):this},lastIndexOf:ec.lastIndexOf||function(t,e){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(t){var e=[];if(ec.map)e=ec.map.call(this,t,this);else for(var n=0,a=this.length;n<a;n++)e.push(t.call(this,this[n],n));return new ed(this.context,e)},pluck:function(t){return this.map(function(e){return e[t]})},pop:ec.pop,push:ec.push,reduce:ec.reduce||function(t,e){return c(this,t,e,0,this.length,1)},reduceRight:ec.reduceRight||function(t,e){return c(this,t,e,this.length-1,-1,-1)},reverse:ec.reverse,selector:null,shift:ec.shift,slice:function(){return new ed(this.context,this)},sort:ec.sort,splice:ec.splice,toArray:function(){return ec.slice.call(this)},to$:function(){return t(this)},toJQuery:function(){return t(this)},unique:function(){return new ed(this.context,t5(this))},unshift:ec.unshift}),ed.extend=function(t,e,n){if(n.length&&e&&(e instanceof ed||e.__dt_wrapper)){var a,r=function(t,e,n){return function(){var a=e.apply(t,arguments);return ed.extend(a,a,n.methodExt),a}},o=0;for(a=n.length;o<a;o++){var i=n[o];e[i.name]="function"===i.type?r(t,i.val,i):"object"===i.type?{}:i.val,e[i.name].__dt_wrapper=!0,ed.extend(t,e[i.name],i.propExt)}}},ed.register=tW=function(e,n){if(Array.isArray(e))for(var a=0,r=e.length;a<r;a++)ed.register(e[a],n);else{r=e.split(".");var o,i=eu;for(e=0,a=r.length;e<a;e++){var l=(o=-1!==r[e].indexOf("()"))?r[e].replace("()",""):r[e];a:{for(var s=0,u=i.length;s<u;s++)if(i[s].name===l){s=i[s];break a}s=null}s||(s={name:l,val:{},methodExt:[],propExt:[],type:"object"},i.push(s)),e===a-1?(s.val=n,s.type="function"==typeof n?"function":t.isPlainObject(n)?"object":"other"):i=o?s.methodExt:s.propExt}}},ed.registerPlural=t9=function(t,e,n){ed.register(t,n),ed.register(e,function(){var t=n.apply(this,arguments);return t===this?this:t instanceof ed?t.length?Array.isArray(t[0])?new ed(t.context,t[0]):t[0]:a:t})};var eh=function(e,n){if(Array.isArray(e))return t.map(e,function(t){return eh(t,n)});if("number"==typeof e)return[n[e]];var a=t.map(n,function(t,e){return t.nTable});return t(a).filter(e).map(function(e){return n[e=t.inArray(this,a)]}).toArray()};tW("tables()",function(t){return t!==a&&null!==t?new ed(eh(t,this.context)):this}),tW("table()",function(t){var e=(t=this.tables(t)).context;return e.length?new ed(e[0]):t}),t9("tables().nodes()","table().node()",function(){return this.iterator("table",function(t){return t.nTable},1)}),t9("tables().body()","table().body()",function(){return this.iterator("table",function(t){return t.nTBody},1)}),t9("tables().header()","table().header()",function(){return this.iterator("table",function(t){return t.nTHead},1)}),t9("tables().footer()","table().footer()",function(){return this.iterator("table",function(t){return t.nTFoot},1)}),t9("tables().containers()","table().container()",function(){return this.iterator("table",function(t){return t.nTableWrapper},1)}),tW("draw()",function(t){return this.iterator("table",function(e){"page"===t?R(e):("string"==typeof t&&(t="full-hold"!==t),O(e,!1===t))})}),tW("page()",function(t){return t===a?this.page.info().page:this.iterator("table",function(e){tl(e,t)})}),tW("page.info()",function(t){if(0===this.context.length)return a;var e=(t=this.context[0])._iDisplayStart,n=t.oFeatures.bPaginate?t._iDisplayLength:-1,r=t.fnRecordsDisplay(),o=-1===n;return{page:o?0:Math.floor(e/n),pages:o?1:Math.ceil(r/n),start:e,end:t.fnDisplayEnd(),length:n,recordsTotal:t.fnRecordsTotal(),recordsDisplay:r,serverSide:"ssp"===tH(t)}}),tW("page.len()",function(t){return t===a?0!==this.context.length?this.context[0]._iDisplayLength:a:this.iterator("table",function(e){tr(e,t)})});var ep=function(t,e,n){if(n){var a=new ed(t);a.one("draw",function(){n(a.ajax.json())})}if("ssp"==tH(t))O(t,e);else{tu(t,!0);var r=t.jqXHR;r&&4!==r.readyState&&r.abort(),E(t,[],function(n){x(t),n=U(t,n);for(var a=0,r=n.length;a<r;a++)S(t,n[a]);O(t,e),tu(t,!1)})}};tW("ajax.json()",function(){var t=this.context;if(0<t.length)return t[0].json}),tW("ajax.params()",function(){var t=this.context;if(0<t.length)return t[0].oAjaxData}),tW("ajax.reload()",function(t,e){return this.iterator("table",function(n){ep(n,!1===e,t)})}),tW("ajax.url()",function(e){var n=this.context;return e===a?0===n.length?a:(n=n[0]).ajax?t.isPlainObject(n.ajax)?n.ajax.url:n.ajax:n.sAjaxSource:this.iterator("table",function(n){t.isPlainObject(n.ajax)?n.ajax.url=e:n.ajax=e})}),tW("ajax.url().load()",function(t,e){return this.iterator("table",function(n){ep(n,!1===e,t)})});var eg=function(t,e,n,r,o){var i,l,s,u=[],c=typeof e;for(e&&"string"!==c&&"function"!==c&&e.length!==a||(e=[e]),c=0,l=e.length;c<l;c++){var f=e[c]&&e[c].split&&!e[c].match(/[\[\(:]/)?e[c].split(","):[e[c]],d=0;for(s=f.length;d<s;d++)(i=n("string"==typeof f[d]?f[d].trim():f[d]))&&i.length&&(u=u.concat(i))}if((t=tM.selector[t]).length)for(c=0,l=t.length;c<l;c++)u=t[c](r,o,u);return t5(u)},eb=function(e){return e||(e={}),e.filter&&e.search===a&&(e.search=e.filter),t.extend({search:"none",order:"current",page:"all"},e)},e$=function(t){for(var e=0,n=t.length;e<n;e++)if(0<t[e].length)return t[0]=t[e],t[0].length=1,t.length=1,t.context=[t.context[e]],t;return t.length=0,t},em=function(e,n){var a=[],r=e.aiDisplay,o=e.aiDisplayMaster,i=n.search,l=n.order;if(n=n.page,"ssp"==tH(e))return"removed"===i?[]:t4(0,o.length);if("current"==n)for(l=e._iDisplayStart,e=e.fnDisplayEnd();l<e;l++)a.push(r[l]);else if("current"==l||"applied"==l){if("none"==i)a=o.slice();else if("applied"==i)a=r.slice();else if("removed"==i){var s={};for(l=0,e=r.length;l<e;l++)s[r[l]]=null;a=t.map(o,function(t){return s.hasOwnProperty(t)?null:t})}}else if("index"==l||"original"==l)for(l=0,e=e.aoData.length;l<e;l++)"none"==i?a.push(l):(-1===(o=t.inArray(l,r))&&"removed"==i||0<=o&&"applied"==i)&&a.push(l);return a},ev=function(e,n,r){var o;return eg("row",n,function(n){var i=tG(n),l=e.aoData;if(null!==i&&!r||(o||(o=em(e,r)),null!==i&&-1!==t.inArray(i,o)))return[i];if(null===n||n===a||""===n)return o;if("function"==typeof n)return t.map(o,function(t){var e=l[t];return n(t,e._aData,e.nTr)?t:null});if(n.nodeName){i=n._DT_RowIndex;var s=n._DT_CellIndex;return i!==a?l[i]&&l[i].nTr===n?[i]:[]:s?l[s.row]&&l[s.row].nTr===n.parentNode?[s.row]:[]:(i=t(n).closest("*[data-dt-row]")).length?[i.data("dt-row")]:[]}return"string"==typeof n&&"#"===n.charAt(0)&&(i=e.aIds[n.replace(/^#/,"")])!==a?[i.idx]:(i=t6(t2(e.aoData,o,"nTr")),t(i).filter(n).map(function(){return this._DT_RowIndex}).toArray())},e,r)};tW("rows()",function(e,n){e===a?e="":t.isPlainObject(e)&&(n=e,e=""),n=eb(n);var r=this.iterator("table",function(t){return ev(t,e,n)},1);return r.selector.rows=e,r.selector.opts=n,r}),tW("rows().nodes()",function(){return this.iterator("row",function(t,e){return t.aoData[e].nTr||a},1)}),tW("rows().data()",function(){return this.iterator(!0,"rows",function(t,e){return t2(t.aoData,e,"_aData")},1)}),t9("rows().cache()","row().cache()",function(t){return this.iterator("row",function(e,n){return e=e.aoData[n],"search"===t?e._aFilterData:e._aSortData},1)}),t9("rows().invalidate()","row().invalidate()",function(t){return this.iterator("row",function(e,n){I(e,n,t)})}),t9("rows().indexes()","row().index()",function(){return this.iterator("row",function(t,e){return e},1)}),t9("rows().ids()","row().id()",function(t){for(var e=[],n=this.context,a=0,r=n.length;a<r;a++)for(var o=0,i=this[a].length;o<i;o++){var l=n[a].rowIdFn(n[a].aoData[this[a][o]]._aData);e.push((!0===t?"#":"")+l)}return new ed(n,e)}),t9("rows().remove()","row().remove()",function(){var t=this;return this.iterator("row",function(e,n,r){var o,i,l=e.aoData,s=l[n];l.splice(n,1);var u=0;for(o=l.length;u<o;u++){var c=l[u],f=c.anCells;if(null!==c.nTr&&(c.nTr._DT_RowIndex=u),null!==f)for(c=0,i=f.length;c<i;c++)f[c]._DT_CellIndex.row=u}_(e.aiDisplayMaster,n),_(e.aiDisplay,n),_(t[r],n,!1),0<e._iRecordsDisplay&&e._iRecordsDisplay--,tR(e),(n=e.rowIdFn(s._aData))!==a&&delete e.aIds[n]}),this.iterator("table",function(t){for(var e=0,n=t.aoData.length;e<n;e++)t.aoData[e].idx=e}),this}),tW("rows.add()",function(e){var n=this.iterator("table",function(t){var n,a=[],r=0;for(n=e.length;r<n;r++){var o=e[r];o.nodeName&&"TR"===o.nodeName.toUpperCase()?a.push(y(t,o)[0]):a.push(S(t,o))}return a},1),a=this.rows(-1);return a.pop(),t.merge(a,n),a}),tW("row()",function(t,e){return e$(this.rows(t,e))}),tW("row().data()",function(t){var e=this.context;if(t===a)return e.length&&this.length?e[0].aoData[this[0]]._aData:a;var n=e[0].aoData[this[0]];return n._aData=t,Array.isArray(t)&&n.nTr&&n.nTr.id&&ea(e[0].rowId)(t,n.nTr.id),I(e[0],this[0],"data"),this}),tW("row().node()",function(){var t=this.context;return t.length&&this.length&&t[0].aoData[this[0]].nTr||null}),tW("row.add()",function(e){e instanceof t&&e.length&&(e=e[0]);var n=this.iterator("table",function(t){return e.nodeName&&"TR"===e.nodeName.toUpperCase()?y(t,e)[0]:S(t,e)});return this.row(n[0])}),t(n).on("plugin-init.dt",function(e,n){(e=new ed(n)).on("stateSaveParams",function(t,e,n){t=e.rowIdFn,e=e.aoData;for(var a=[],r=0;r<e.length;r++)e[r]._detailsShow&&a.push("#"+t(e[r]._aData));n.childRows=a});var a=e.state.loaded();a&&a.childRows&&e.rows(t.map(a.childRows,function(t){return t.replace(/:/g,"\\:")})).every(function(){tP(n,null,"requestChild",[this])})});var eS=function(e,n,a,r){var o=[],i=function(n,a){if(Array.isArray(n)||n instanceof t)for(var r=0,l=n.length;r<l;r++)i(n[r],a);else n.nodeName&&"tr"===n.nodeName.toLowerCase()?o.push(n):(r=t("<tr><td></td></tr>").addClass(a),t("td",r).addClass(a).html(n)[0].colSpan=b(e),o.push(r[0]))};i(a,r),n._details&&n._details.detach(),n._details=t(o),n._detailsShow&&n._details.insertAfter(n.nTr)},ey=tB.util.throttle(function(t){tC(t[0])},500),eD=function(e,n){var r=e.context;r.length&&(e=r[0].aoData[n!==a?n:e[0]])&&e._details&&(e._details.remove(),e._detailsShow=a,e._details=a,t(e.nTr).removeClass("dt-hasChild"),ey(r))},e8=function(e,n){var a=e.context;if(a.length&&e.length){var r=a[0].aoData[e[0]];r._details&&((r._detailsShow=n)?(r._details.insertAfter(r.nTr),t(r.nTr).addClass("dt-hasChild")):(r._details.detach(),t(r.nTr).removeClass("dt-hasChild")),tP(a[0],null,"childRow",[n,e.row(e[0])]),eT(a[0]),ey(a))}},eT=function(t){var e=new ed(t),n=t.aoData;e.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details"),0<t3(n,"_details").length&&(e.on("draw.dt.DT_details",function(a,r){t===r&&e.rows({page:"current"}).eq(0).each(function(t){(t=n[t])._detailsShow&&t._details.insertAfter(t.nTr)})}),e.on("column-visibility.dt.DT_details",function(e,a,r,o){if(t===a)for(a=b(a),r=0,o=n.length;r<o;r++)(e=n[r])._details&&e._details.children("td[colspan]").attr("colspan",a)}),e.on("destroy.dt.DT_details",function(a,r){if(t===r)for(a=0,r=n.length;a<r;a++)n[a]._details&&eD(e,a)}))};tW("row().child()",function(t,e){var n=this.context;return t===a?n.length&&this.length?n[0].aoData[this[0]]._details:a:(!0===t?this.child.show():!1===t?eD(this):n.length&&this.length&&eS(n[0],n[0].aoData[this[0]],t,e),this)}),tW(["row().child.show()","row().child().show()"],function(t){return e8(this,!0),this}),tW(["row().child.hide()","row().child().hide()"],function(){return e8(this,!1),this}),tW(["row().child.remove()","row().child().remove()"],function(){return eD(this),this}),tW("row().child.isShown()",function(){var t=this.context;return!!t.length&&!!this.length&&!!t[0].aoData[this[0]]._detailsShow});var eC=/^([^:]+):(name|visIdx|visible)$/,ew=function(t,e,n,a,r){n=[],a=0;for(var o=r.length;a<o;a++)n.push(D(t,r[a],e));return n},ex=function(e,n,a){var r=e.aoColumns,o=t3(r,"sName"),i=t3(r,"nTh");return eg("column",n,function(n){var l=tG(n);if(""===n)return t4(r.length);if(null!==l)return[0<=l?l:r.length+l];if("function"==typeof n){var s=em(e,a);return t.map(r,function(t,a){return n(a,ew(e,a,0,0,s),i[a])?a:null})}var u="string"==typeof n?n.match(eC):"";if(u)switch(u[2]){case"visIdx":case"visible":if(0>(l=parseInt(u[1],10))){var c=t.map(r,function(t,e){return t.bVisible?e:null});return[c[c.length+l]]}return[p(e,l)];case"name":return t.map(o,function(t,e){return t===u[1]?e:null});default:return[]}return n.nodeName&&n._DT_CellIndex?[n._DT_CellIndex.column]:(l=t(i).filter(n).map(function(){return t.inArray(this,i)}).toArray()).length||!n.nodeName?l:(l=t(n).closest("*[data-dt-column]")).length?[l.data("dt-column")]:[]},e,a)};tW("columns()",function(e,n){e===a?e="":t.isPlainObject(e)&&(n=e,e=""),n=eb(n);var r=this.iterator("table",function(t){return ex(t,e,n)},1);return r.selector.cols=e,r.selector.opts=n,r}),t9("columns().header()","column().header()",function(t,e){return this.iterator("column",function(t,e){return t.aoColumns[e].nTh},1)}),t9("columns().footer()","column().footer()",function(t,e){return this.iterator("column",function(t,e){return t.aoColumns[e].nTf},1)}),t9("columns().data()","column().data()",function(){return this.iterator("column-rows",ew,1)}),t9("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(t,e){return t.aoColumns[e].mData},1)}),t9("columns().cache()","column().cache()",function(t){return this.iterator("column-rows",function(e,n,a,r,o){return t2(e.aoData,o,"search"===t?"_aFilterData":"_aSortData",n)},1)}),t9("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(t,e,n,a,r){return t2(t.aoData,r,"anCells",e)},1)}),t9("columns().visible()","column().visible()",function(e,n){var r=this,o=this.iterator("column",function(n,r){if(e===a)return n.aoColumns[r].bVisible;var o,i=n.aoColumns,l=i[r],s=n.aoData;if(e!==a&&l.bVisible!==e){if(e){var u=t.inArray(!0,t3(i,"bVisible"),r+1);for(i=0,o=s.length;i<o;i++){var c=s[i].nTr;n=s[i].anCells,c&&c.insertBefore(n[r],n[u]||null)}}else t(t3(n.aoData,"anCells",r)).detach();l.bVisible=e}});return e!==a&&this.iterator("table",function(o){P(o,o.aoHeader),P(o,o.aoFooter),o.aiDisplay.length||t(o.nTBody).find("td[colspan]").attr("colspan",b(o)),tC(o),r.iterator("column",function(t,a){tP(t,null,"column-visibility",[t,a,e,n])}),(n===a||n)&&r.columns.adjust()}),o}),t9("columns().indexes()","column().index()",function(t){return this.iterator("column",function(e,n){return"visible"===t?g(e,n):n},1)}),tW("columns.adjust()",function(){return this.iterator("table",function(t){h(t)},1)}),tW("column.index()",function(t,e){if(0!==this.context.length){var n=this.context[0];if("fromVisible"===t||"toData"===t)return p(n,e);if("fromData"===t||"toVisible"===t)return g(n,e)}}),tW("column()",function(t,e){return e$(this.columns(t,e))});var e_=function(e,n,r){var o,i,l,s,u,c,f,d=e.aoData,h=em(e,r),p=t6(t2(d,h,"anCells")),g=t(tK([],p)),b=e.aoColumns.length;return eg("cell",n,function(n){var r="function"==typeof n;if(null===n||n===a||r){for(i=[],l=0,s=h.length;l<s;l++)for(o=h[l],u=0;u<b;u++)c={row:o,column:u},r?(f=d[o],n(c,D(e,o,u),f.anCells?f.anCells[u]:null)&&i.push(c)):i.push(c);return i}return t.isPlainObject(n)?n.column!==a&&n.row!==a&&-1!==t.inArray(n.row,h)?[n]:[]:(r=g.filter(n).map(function(t,e){return{row:e._DT_CellIndex.row,column:e._DT_CellIndex.column}}).toArray()).length||!n.nodeName?r:(f=t(n).closest("*[data-dt-row]")).length?[{row:f.data("dt-row"),column:f.data("dt-column")}]:[]},e,r)};tW("cells()",function(e,n,r){if(t.isPlainObject(e)&&(e.row===a?(r=e,e=null):(r=n,n=null)),t.isPlainObject(n)&&(r=n,n=null),null===n||n===a)return this.iterator("table",function(t){return e_(t,e,eb(r))});var o,i,l,s,u=r?{page:r.page,order:r.order,search:r.search}:{},c=this.columns(n,u),f=this.rows(e,u);return u=this.iterator("table",function(t,e){for(t=[],o=0,i=f[e].length;o<i;o++)for(l=0,s=c[e].length;l<s;l++)t.push({row:f[e][o],column:c[e][l]});return t},1),u=r&&r.selected?this.cells(u,r):u,t.extend(u.selector,{cols:n,rows:e,opts:r}),u}),t9("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(t,e,n){return(t=t.aoData[e])&&t.anCells?t.anCells[n]:a},1)}),tW("cells().data()",function(){return this.iterator("cell",function(t,e,n){return D(t,e,n)},1)}),t9("cells().cache()","cell().cache()",function(t){return t="search"===t?"_aFilterData":"_aSortData",this.iterator("cell",function(e,n,a){return e.aoData[n][t][a]},1)}),t9("cells().render()","cell().render()",function(t){return this.iterator("cell",function(e,n,a){return D(e,n,a,t)},1)}),t9("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(t,e,n){return{row:e,column:n,columnVisible:g(t,n)}},1)}),t9("cells().invalidate()","cell().invalidate()",function(t){return this.iterator("cell",function(e,n,a){I(e,n,t,a)})}),tW("cell()",function(t,e,n){return e$(this.cells(t,e,n))}),tW("cell().data()",function(t){var e=this.context,n=this[0];return t===a?e.length&&n.length?D(e[0],n[0].row,n[0].column):a:(T(e[0],n[0].row,n[0].column,t),I(e[0],n[0].row,"data",n[0].column),this)}),tW("order()",function(t,e){var n=this.context;return t===a?0!==n.length?n[0].aaSorting:a:("number"==typeof t?t=[[t,e]]:t.length&&!Array.isArray(t[0])&&(t=Array.prototype.slice.call(arguments)),this.iterator("table",function(e){e.aaSorting=t.slice()}))}),tW("order.listener()",function(t,e,n){return this.iterator("table",function(a){tD(a,t,e,n)})}),tW("order.fixed()",function(e){if(!e){var n=this.context;return Array.isArray(n=n.length?n[0].aaSortingFixed:a)?{pre:n}:n}return this.iterator("table",function(n){n.aaSortingFixed=t.extend(!0,{},e)})}),tW(["columns().order()","column().order()"],function(e){var n=this;return this.iterator("table",function(a,r){var o=[];t.each(n[r],function(t,n){o.push([n,e])}),a.aaSorting=o})}),tW("search()",function(e,n,r,o){var i=this.context;return e===a?0!==i.length?i[0].oPreviousSearch.sSearch:a:this.iterator("table",function(a){a.oFeatures.bFilter&&X(a,t.extend({},a.oPreviousSearch,{sSearch:e+"",bRegex:null!==n&&n,bSmart:null===r||r,bCaseInsensitive:null===o||o}),1)})}),t9("columns().search()","column().search()",function(e,n,r,o){return this.iterator("column",function(i,l){var s=i.aoPreSearchCols;if(e===a)return s[l].sSearch;i.oFeatures.bFilter&&(t.extend(s[l],{sSearch:e+"",bRegex:null!==n&&n,bSmart:null===r||r,bCaseInsensitive:null===o||o}),X(i,i.oPreviousSearch,1))})}),tW("state()",function(){return this.context.length?this.context[0].oSavedState:null}),tW("state.clear()",function(){return this.iterator("table",function(t){t.fnStateSaveCallback.call(t.oInstance,t,{})})}),tW("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null}),tW("state.save()",function(){return this.iterator("table",function(t){tC(t)})}),tB.versionCheck=tB.fnVersionCheck=function(t){var e=tB.version.split(".");t=t.split(".");for(var n,a,r=0,o=t.length;r<o;r++)if(n=parseInt(e[r],10)||0,a=parseInt(t[r],10)||0,n!==a)return n>a;return!0},tB.isDataTable=tB.fnIsDataTable=function(e){var n=t(e).get(0),a=!1;return e instanceof tB.Api||(t.each(tB.settings,function(e,r){e=r.nScrollHead?t("table",r.nScrollHead)[0]:null;var o=r.nScrollFoot?t("table",r.nScrollFoot)[0]:null;(r.nTable===n||e===n||o===n)&&(a=!0)}),a)},tB.tables=tB.fnTables=function(e){var n=!1;t.isPlainObject(e)&&(n=e.api,e=e.visible);var a=t.map(tB.settings,function(n){if(!e||e&&t(n.nTable).is(":visible"))return n.nTable});return n?new ed(a):a},tB.camelToHungarian=o,tW("$()",function(e,n){return n=t(n=this.rows(n).nodes()),t([].concat(n.filter(e).toArray(),n.find(e).toArray()))}),t.each(["on","one","off"],function(e,n){tW(n+"()",function(){var e=Array.prototype.slice.call(arguments);e[0]=t.map(e[0].split(/\s/),function(t){return t.match(/\.dt\b/)?t:t+".dt"}).join(" ");var a=t(this.tables().nodes());return a[n].apply(a,e),this})}),tW("clear()",function(){return this.iterator("table",function(t){x(t)})}),tW("settings()",function(){return new ed(this.context,this.context)}),tW("init()",function(){var t=this.context;return t.length?t[0].oInit:null}),tW("data()",function(){return this.iterator("table",function(t){return t3(t.aoData,"_aData")}).flatten()}),tW("destroy()",function(n){return n=n||!1,this.iterator("table",function(a){var r=a.nTableWrapper.parentNode,o=a.oClasses,i=a.nTable,l=a.nTBody,s=a.nTHead,u=a.nTFoot,c=t(i);l=t(l);var f,d=t(a.nTableWrapper),h=t.map(a.aoData,function(t){return t.nTr});a.bDestroying=!0,tP(a,"aoDestroyCallback","destroy",[a]),n||new ed(a).columns().visible(!0),d.off(".DT").find(":not(tbody *)").off(".DT"),t(e).off(".DT-"+a.sInstance),i!=s.parentNode&&(c.children("thead").detach(),c.append(s)),u&&i!=u.parentNode&&(c.children("tfoot").detach(),c.append(u)),a.aaSorting=[],a.aaSortingFixed=[],t8(a),t(h).removeClass(a.asStripeClasses.join(" ")),t("th, td",s).removeClass(o.sSortable+" "+o.sSortableAsc+" "+o.sSortableDesc+" "+o.sSortableNone),l.children().detach(),l.append(h),c[s=n?"remove":"detach"](),d[s](),!n&&r&&(r.insertBefore(i,a.nTableReinsertBefore),c.css("width",a.sDestroyWidth).removeClass(o.sTable),(f=a.asDestroyStripes.length)&&l.children().each(function(e){t(this).addClass(a.asDestroyStripes[e%f])})),-1!==(r=t.inArray(a,tB.settings))&&tB.settings.splice(r,1)})}),t.each(["column","row","cell"],function(t,e){tW(e+"s().every()",function(t){var n=this.selector.opts,r=this;return this.iterator(e,function(o,i,l,s,u){t.call(r[e](i,"cell"===e?l:n,"cell"===e?n:a),i,l,s,u)})})}),tW("i18n()",function(e,n,r){var o=this.context[0];return(e=en(e)(o.oLanguage))===a&&(e=n),r!==a&&t.isPlainObject(e)&&(e=e[r]!==a?e[r]:e._),e.replace("%d",r)}),tB.version="1.11.5",tB.settings=[],tB.models={},tB.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0,return:!1},tB.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1},tB.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null},tB.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(t){return t.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(t){try{return JSON.parse((-1===t.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+t.sInstance+"_"+location.pathname))}catch(e){return{}}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(t,e){try{(-1===t.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+t.sInstance+"_"+location.pathname,JSON.stringify(e))}catch(n){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:t.extend({},tB.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"},r(tB.defaults),tB.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null},r(tB.defaults.column),tB.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,jqXHR:null,json:a,oAjaxData:a,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==tH(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==tH(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var t=this._iDisplayLength,e=this._iDisplayStart,n=e+t,a=this.aiDisplay.length,r=this.oFeatures,o=r.bPaginate;return r.bServerSide?!1===o||-1===t?e+a:Math.min(e+t,this._iRecordsDisplay):!o||n>a||-1===t?a:n},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null},tB.ext=tM={buttons:{},classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:tB.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:tB.version},t.extend(tM,{afnFiltering:tM.search,aTypes:tM.type.detect,ofnSearch:tM.type.search,oSort:tM.type.order,afnSortData:tM.order,aoFeatures:tM.feature,oApi:tM.internal,oStdClasses:tM.classes,oPagination:tM.pager}),t.extend(tB.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_desc_disabled",sSortableDesc:"sorting_asc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var eI=tB.ext.pager;t.extend(eI,{simple:function(t,e){return["previous","next"]},full:function(t,e){return["first","previous","next","last"]},numbers:function(t,e){return[tN(t,e)]},simple_numbers:function(t,e){return["previous",tN(t,e),"next"]},full_numbers:function(t,e){return["first","previous",tN(t,e),"next","last"]},first_last_numbers:function(t,e){return["first",tN(t,e),"last"]},_numbers:tN,numbers_length:7}),t.extend(!0,tB.ext.renderer,{pageButton:{_:function(e,r,o,i,l,s){var u,c,f=e.oClasses,d=e.oLanguage.oPaginate,h=e.oLanguage.oAria.paginate||{},p=0,g=function(n,a){var r,i=f.sPageButtonDisabled,b=function(t){tl(e,t.data.action,!0)},$=0;for(r=a.length;$<r;$++){var m=a[$];if(Array.isArray(m)){var v=t("<"+(m.DT_el||"div")+"/>").appendTo(n);g(v,m)}else{switch(u=null,c=m,v=e.iTabIndex,m){case"ellipsis":n.append('<span class="ellipsis">&#x2026;</span>');break;case"first":u=d.sFirst,0===l&&(v=-1,c+=" "+i);break;case"previous":u=d.sPrevious,0===l&&(v=-1,c+=" "+i);break;case"next":u=d.sNext,(0===s||l===s-1)&&(v=-1,c+=" "+i);break;case"last":u=d.sLast,(0===s||l===s-1)&&(v=-1,c+=" "+i);break;default:u=e.fnFormatNumber(m+1),c=l===m?f.sPageButtonActive:""}null!==u&&(v=t("<a>",{class:f.sPageButton+" "+c,"aria-controls":e.sTableId,"aria-label":h[m],"data-dt-idx":p,tabindex:v,id:0===o&&"string"==typeof m?e.sTableId+"_"+m:null}).html(u).appendTo(n),tF(v,{action:m},b),p++)}}};try{var b=t(r).find(n.activeElement).data("dt-idx")}catch($){}g(t(r).empty(),i),b!==a&&t(r).find("[data-dt-idx="+b+"]").trigger("focus")}}}),t.extend(tB.ext.type.detect,[function(t,e){return t1(t,e=e.oLanguage.sDecimal)?"num"+e:null},function(t,e){return(!t||t instanceof Date||t0.test(t))&&(null!==(e=Date.parse(t))&&!isNaN(e)||tY(t))?"date":null},function(t,e){return t1(t,e=e.oLanguage.sDecimal,!0)?"num-fmt"+e:null},function(t,e){return tZ(t,e=e.oLanguage.sDecimal)?"html-num"+e:null},function(t,e){return tZ(t,e=e.oLanguage.sDecimal,!0)?"html-num-fmt"+e:null},function(t,e){return tY(t)||"string"==typeof t&&-1!==t.indexOf("<")?"html":null}]),t.extend(tB.ext.type.search,{html:function(t){return tY(t)?t:"string"==typeof t?t.replace(tV," ").replace(tX,""):""},string:function(t){return tY(t)?t:"string"==typeof t?t.replace(tV," "):t}});var eA=function(t,e,n,a){return 0===t||t&&"-"!==t?(e&&(t=tz(t,e)),t.replace&&(n&&(t=t.replace(n,"")),a&&(t=t.replace(a,""))),1*t):-1/0};t.extend(tM.type.order,{"date-pre":function(t){return isNaN(t=Date.parse(t))?-1/0:t},"html-pre":function(t){return tY(t)?"":t.replace?t.replace(/<.*?>/g,"").toLowerCase():t+""},"string-pre":function(t){return tY(t)?"":"string"==typeof t?t.toLowerCase():t.toString?t.toString():""},"string-asc":function(t,e){return t<e?-1:t>e?1:0},"string-desc":function(t,e){return t<e?1:t>e?-1:0}}),tk(""),t.extend(!0,tB.ext.renderer,{header:{_:function(e,n,a,r){t(e.nTable).on("order.dt.DT",function(t,o,i,l){e===o&&(t=a.idx,n.removeClass(r.sSortAsc+" "+r.sSortDesc).addClass("asc"==l[t]?r.sSortAsc:"desc"==l[t]?r.sSortDesc:a.sSortingClass))})},jqueryui:function(e,n,a,r){t("<div/>").addClass(r.sSortJUIWrapper).append(n.contents()).append(t("<span/>").addClass(r.sSortIcon+" "+a.sSortingClassJUI)).appendTo(n),t(e.nTable).on("order.dt.DT",function(t,o,i,l){e===o&&(t=a.idx,n.removeClass(r.sSortAsc+" "+r.sSortDesc).addClass("asc"==l[t]?r.sSortAsc:"desc"==l[t]?r.sSortDesc:a.sSortingClass),n.find("span."+r.sSortIcon).removeClass(r.sSortJUIAsc+" "+r.sSortJUIDesc+" "+r.sSortJUI+" "+r.sSortJUIAscAllowed+" "+r.sSortJUIDescAllowed).addClass("asc"==l[t]?r.sSortJUIAsc:"desc"==l[t]?r.sSortJUIDesc:a.sSortingClassJUI))})}}});var eL=function(t){return Array.isArray(t)&&(t=t.join(",")),"string"==typeof t?t.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;"):t};return tB.render={number:function(t,e,n,a,r){return{display:function(o){if("number"!=typeof o&&"string"!=typeof o)return o;var i=0>o?"-":"",l=parseFloat(o);return isNaN(l)?eL(o):(o=Math.abs(l=l.toFixed(n)),l=parseInt(o,10),o=n?e+(o-l).toFixed(n).substring(2):"",0===l&&0===parseFloat(o)&&(i=""),i+(a||"")+l.toString().replace(/\B(?=(\d{3})+(?!\d))/g,t)+o+(r||""))}}},text:function(){return{display:eL,filter:eL}}},t.extend(tB.ext.internal,{_fnExternApiFunc:tE,_fnBuildAjax:E,_fnAjaxUpdate:M,_fnAjaxParameters:W,_fnAjaxUpdateDraw:B,_fnAjaxDataSrc:U,_fnAddColumn:f,_fnColumnOptions:d,_fnAdjustColumnSizing:h,_fnVisibleToColumnIndex:p,_fnColumnIndexToVisible:g,_fnVisbleColumns:b,_fnGetColumns:$,_fnColumnTypes:m,_fnApplyColumnDefs:v,_fnHungarianMap:r,_fnCamelToHungarian:o,_fnLanguageCompat:i,_fnBrowserDetect:u,_fnAddData:S,_fnAddTr:y,_fnNodeToDataIndex:function(t,e){return e._DT_RowIndex!==a?e._DT_RowIndex:null},_fnNodeToColumnIndex:function(e,n,a){return t.inArray(a,e.aoData[n].anCells)},_fnGetCellData:D,_fnSetCellData:T,_fnSplitObjNotation:C,_fnGetObjectDataFn:en,_fnSetObjectDataFn:ea,_fnGetDataMaster:w,_fnClearTable:x,_fnDeleteIndex:_,_fnInvalidate:I,_fnGetRowElements:A,_fnCreateTr:L,_fnBuildHead:j,_fnDrawHead:P,_fnDraw:R,_fnReDraw:O,_fnAddOptionsHtml:H,_fnDetectHeader:N,_fnGetUniqueThs:k,_fnFeatureHtmlFilter:V,_fnFilterComplete:X,_fnFilterCustom:J,_fnFilterColumn:q,_fnFilter:Y,_fnFilterCreateSearch:G,_fnEscapeRegex:er,_fnFilterData:z,_fnFeatureHtmlInfo:Q,_fnUpdateInfo:tt,_fnInfoMacros:te,_fnInitialise:tn,_fnInitComplete:ta,_fnLengthChange:tr,_fnFeatureHtmlLength:to,_fnFeatureHtmlPaginate:ti,_fnPageChange:tl,_fnFeatureHtmlProcessing:ts,_fnProcessingDisplay:tu,_fnFeatureHtmlTable:tc,_fnScrollDraw:tf,_fnApplyToChildren:td,_fnCalculateColumnWidths:th,_fnThrottle:es,_fnConvertToWidth:tp,_fnGetWidestNode:tg,_fnGetMaxLenString:tb,_fnStringToCss:t$,_fnSortFlatten:tm,_fnSort:tv,_fnSortAria:tS,_fnSortListener:ty,_fnSortAttachListener:tD,_fnSortingClasses:t8,_fnSortData:tT,_fnSaveState:tC,_fnLoadState:tw,_fnImplementState:tx,_fnSettingsFromNode:t_,_fnLog:tI,_fnMap:tA,_fnBindAction:tF,_fnCallbackReg:tj,_fnCallbackFire:tP,_fnLengthOverflow:tR,_fnRenderer:tO,_fnDataSource:tH,_fnRowAttributes:F,_fnExtend:tL,_fnCalculateEnd:function(){}}),t.fn.dataTable=tB,tB.$=t,t.fn.dataTableSettings=tB.settings,t.fn.dataTableExt=tB.ext,t.fn.DataTable=function(e){return t(this).dataTable(e).api()},t.each(tB,function(e,n){t.fn.DataTable[e]=n}),tB});
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
index 906899290b698..d1204ccedd277 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
@@ -1,5 +1,5 @@
 //! moment.js
-//! version : 2.22.1
+//! version : 2.29.2
 //! license : MIT
 //! momentjs.com
-!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):e.moment=t()}(this,function(){"use strict";var e,i;function c(){return e.apply(null,arguments)}function o(e){return e instanceof Array||"[object Array]"===Object.prototype.toString.call(e)}function u(e){return null!=e&&"[object Object]"===Object.prototype.toString.call(e)}function l(e){return void 0===e}function d(e){return"number"==typeof e||"[object Number]"===Object.prototype.toString.call(e)}function h(e){return e instanceof Date||"[object Date]"===Object.prototype.toString.call(e)}function f(e,t){var n,s=[];for(n=0;n<e.length;++n)s.push(t(e[n],n));return s}function m(e,t){return Object.prototype.hasOwnProperty.call(e,t)}function _(e,t){for(var n in t)m(t,n)&&(e[n]=t[n]);return m(t,"toString")&&(e.toString=t.toString),m(t,"valueOf")&&(e.valueOf=t.valueOf),e}function y(e,t,n,s){return Ot(e,t,n,s,!0).utc()}function g(e){return null==e._pf&&(e._pf={empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}),e._pf}function p(e){if(null==e._isValid){var t=g(e),n=i.call(t.parsedDateParts,function(e){return null!=e}),s=!isNaN(e._d.getTime())&&t.overflow<0&&!t.empty&&!t.invalidMonth&&!t.invalidWeekday&&!t.weekdayMismatch&&!t.nullInput&&!t.invalidFormat&&!t.userInvalidated&&(!t.meridiem||t.meridiem&&n);if(e._strict&&(s=s&&0===t.charsLeftOver&&0===t.unusedTokens.length&&void 0===t.bigHour),null!=Object.isFrozen&&Object.isFrozen(e))return s;e._isValid=s}return e._isValid}function v(e){var t=y(NaN);return null!=e?_(g(t),e):g(t).userInvalidated=!0,t}i=Array.prototype.some?Array.prototype.some:function(e){for(var t=Object(this),n=t.length>>>0,s=0;s<n;s++)if(s in t&&e.call(this,t[s],s,t))return!0;return!1};var r=c.momentProperties=[];function w(e,t){var n,s,i;if(l(t._isAMomentObject)||(e._isAMomentObject=t._isAMomentObject),l(t._i)||(e._i=t._i),l(t._f)||(e._f=t._f),l(t._l)||(e._l=t._l),l(t._strict)||(e._strict=t._strict),l(t._tzm)||(e._tzm=t._tzm),l(t._isUTC)||(e._isUTC=t._isUTC),l(t._offset)||(e._offset=t._offset),l(t._pf)||(e._pf=g(t)),l(t._locale)||(e._locale=t._locale),0<r.length)for(n=0;n<r.length;n++)l(i=t[s=r[n]])||(e[s]=i);return e}var t=!1;function M(e){w(this,e),this._d=new Date(null!=e._d?e._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===t&&(t=!0,c.updateOffset(this),t=!1)}function S(e){return e instanceof M||null!=e&&null!=e._isAMomentObject}function D(e){return e<0?Math.ceil(e)||0:Math.floor(e)}function k(e){var t=+e,n=0;return 0!==t&&isFinite(t)&&(n=D(t)),n}function a(e,t,n){var s,i=Math.min(e.length,t.length),r=Math.abs(e.length-t.length),a=0;for(s=0;s<i;s++)(n&&e[s]!==t[s]||!n&&k(e[s])!==k(t[s]))&&a++;return a+r}function Y(e){!1===c.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+e)}function n(i,r){var a=!0;return _(function(){if(null!=c.deprecationHandler&&c.deprecationHandler(null,i),a){for(var e,t=[],n=0;n<arguments.length;n++){if(e="","object"==typeof arguments[n]){for(var s in e+="\n["+n+"] ",arguments[0])e+=s+": "+arguments[0][s]+", ";e=e.slice(0,-2)}else e=arguments[n];t.push(e)}Y(i+"\nArguments: "+Array.prototype.slice.call(t).join("")+"\n"+(new Error).stack),a=!1}return r.apply(this,arguments)},r)}var s,O={};function T(e,t){null!=c.deprecationHandler&&c.deprecationHandler(e,t),O[e]||(Y(t),O[e]=!0)}function x(e){return e instanceof Function||"[object Function]"===Object.prototype.toString.call(e)}function b(e,t){var n,s=_({},e);for(n in t)m(t,n)&&(u(e[n])&&u(t[n])?(s[n]={},_(s[n],e[n]),_(s[n],t[n])):null!=t[n]?s[n]=t[n]:delete s[n]);for(n in e)m(e,n)&&!m(t,n)&&u(e[n])&&(s[n]=_({},s[n]));return s}function P(e){null!=e&&this.set(e)}c.suppressDeprecationWarnings=!1,c.deprecationHandler=null,s=Object.keys?Object.keys:function(e){var t,n=[];for(t in e)m(e,t)&&n.push(t);return n};var W={};function H(e,t){var n=e.toLowerCase();W[n]=W[n+"s"]=W[t]=e}function R(e){return"string"==typeof e?W[e]||W[e.toLowerCase()]:void 0}function C(e){var t,n,s={};for(n in e)m(e,n)&&(t=R(n))&&(s[t]=e[n]);return s}var F={};function L(e,t){F[e]=t}function U(e,t,n){var s=""+Math.abs(e),i=t-s.length;return(0<=e?n?"+":"":"-")+Math.pow(10,Math.max(0,i)).toString().substr(1)+s}var N=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,G=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,V={},E={};function I(e,t,n,s){var i=s;"string"==typeof s&&(i=function(){return this[s]()}),e&&(E[e]=i),t&&(E[t[0]]=function(){return U(i.apply(this,arguments),t[1],t[2])}),n&&(E[n]=function(){return this.localeData().ordinal(i.apply(this,arguments),e)})}function A(e,t){return e.isValid()?(t=j(t,e.localeData()),V[t]=V[t]||function(s){var e,i,t,r=s.match(N);for(e=0,i=r.length;e<i;e++)E[r[e]]?r[e]=E[r[e]]:r[e]=(t=r[e]).match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"");return function(e){var t,n="";for(t=0;t<i;t++)n+=x(r[t])?r[t].call(e,s):r[t];return n}}(t),V[t](e)):e.localeData().invalidDate()}function j(e,t){var n=5;function s(e){return t.longDateFormat(e)||e}for(G.lastIndex=0;0<=n&&G.test(e);)e=e.replace(G,s),G.lastIndex=0,n-=1;return e}var Z=/\d/,z=/\d\d/,$=/\d{3}/,q=/\d{4}/,J=/[+-]?\d{6}/,B=/\d\d?/,Q=/\d\d\d\d?/,X=/\d\d\d\d\d\d?/,K=/\d{1,3}/,ee=/\d{1,4}/,te=/[+-]?\d{1,6}/,ne=/\d+/,se=/[+-]?\d+/,ie=/Z|[+-]\d\d:?\d\d/gi,re=/Z|[+-]\d\d(?::?\d\d)?/gi,ae=/[0-9]{0,256}['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFF07\uFF10-\uFFEF]{1,256}|[\u0600-\u06FF\/]{1,256}(\s*?[\u0600-\u06FF]{1,256}){1,2}/i,oe={};function ue(e,n,s){oe[e]=x(n)?n:function(e,t){return e&&s?s:n}}function le(e,t){return m(oe,e)?oe[e](t._strict,t._locale):new RegExp(de(e.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(e,t,n,s,i){return t||n||s||i})))}function de(e){return e.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}var he={};function ce(e,n){var t,s=n;for("string"==typeof e&&(e=[e]),d(n)&&(s=function(e,t){t[n]=k(e)}),t=0;t<e.length;t++)he[e[t]]=s}function fe(e,i){ce(e,function(e,t,n,s){n._w=n._w||{},i(e,n._w,n,s)})}var me=0,_e=1,ye=2,ge=3,pe=4,ve=5,we=6,Me=7,Se=8;function De(e){return ke(e)?366:365}function ke(e){return e%4==0&&e%100!=0||e%400==0}I("Y",0,0,function(){var e=this.year();return e<=9999?""+e:"+"+e}),I(0,["YY",2],0,function(){return this.year()%100}),I(0,["YYYY",4],0,"year"),I(0,["YYYYY",5],0,"year"),I(0,["YYYYYY",6,!0],0,"year"),H("year","y"),L("year",1),ue("Y",se),ue("YY",B,z),ue("YYYY",ee,q),ue("YYYYY",te,J),ue("YYYYYY",te,J),ce(["YYYYY","YYYYYY"],me),ce("YYYY",function(e,t){t[me]=2===e.length?c.parseTwoDigitYear(e):k(e)}),ce("YY",function(e,t){t[me]=c.parseTwoDigitYear(e)}),ce("Y",function(e,t){t[me]=parseInt(e,10)}),c.parseTwoDigitYear=function(e){return k(e)+(68<k(e)?1900:2e3)};var Ye,Oe=Te("FullYear",!0);function Te(t,n){return function(e){return null!=e?(be(this,t,e),c.updateOffset(this,n),this):xe(this,t)}}function xe(e,t){return e.isValid()?e._d["get"+(e._isUTC?"UTC":"")+t]():NaN}function be(e,t,n){e.isValid()&&!isNaN(n)&&("FullYear"===t&&ke(e.year())&&1===e.month()&&29===e.date()?e._d["set"+(e._isUTC?"UTC":"")+t](n,e.month(),Pe(n,e.month())):e._d["set"+(e._isUTC?"UTC":"")+t](n))}function Pe(e,t){if(isNaN(e)||isNaN(t))return NaN;var n,s=(t%(n=12)+n)%n;return e+=(t-s)/12,1===s?ke(e)?29:28:31-s%7%2}Ye=Array.prototype.indexOf?Array.prototype.indexOf:function(e){var t;for(t=0;t<this.length;++t)if(this[t]===e)return t;return-1},I("M",["MM",2],"Mo",function(){return this.month()+1}),I("MMM",0,0,function(e){return this.localeData().monthsShort(this,e)}),I("MMMM",0,0,function(e){return this.localeData().months(this,e)}),H("month","M"),L("month",8),ue("M",B),ue("MM",B,z),ue("MMM",function(e,t){return t.monthsShortRegex(e)}),ue("MMMM",function(e,t){return t.monthsRegex(e)}),ce(["M","MM"],function(e,t){t[_e]=k(e)-1}),ce(["MMM","MMMM"],function(e,t,n,s){var i=n._locale.monthsParse(e,s,n._strict);null!=i?t[_e]=i:g(n).invalidMonth=e});var We=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,He="January_February_March_April_May_June_July_August_September_October_November_December".split("_");var Re="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_");function Ce(e,t){var n;if(!e.isValid())return e;if("string"==typeof t)if(/^\d+$/.test(t))t=k(t);else if(!d(t=e.localeData().monthsParse(t)))return e;return n=Math.min(e.date(),Pe(e.year(),t)),e._d["set"+(e._isUTC?"UTC":"")+"Month"](t,n),e}function Fe(e){return null!=e?(Ce(this,e),c.updateOffset(this,!0),this):xe(this,"Month")}var Le=ae;var Ue=ae;function Ne(){function e(e,t){return t.length-e.length}var t,n,s=[],i=[],r=[];for(t=0;t<12;t++)n=y([2e3,t]),s.push(this.monthsShort(n,"")),i.push(this.months(n,"")),r.push(this.months(n,"")),r.push(this.monthsShort(n,""));for(s.sort(e),i.sort(e),r.sort(e),t=0;t<12;t++)s[t]=de(s[t]),i[t]=de(i[t]);for(t=0;t<24;t++)r[t]=de(r[t]);this._monthsRegex=new RegExp("^("+r.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+i.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+s.join("|")+")","i")}function Ge(e){var t=new Date(Date.UTC.apply(null,arguments));return e<100&&0<=e&&isFinite(t.getUTCFullYear())&&t.setUTCFullYear(e),t}function Ve(e,t,n){var s=7+t-n;return-((7+Ge(e,0,s).getUTCDay()-t)%7)+s-1}function Ee(e,t,n,s,i){var r,a,o=1+7*(t-1)+(7+n-s)%7+Ve(e,s,i);return o<=0?a=De(r=e-1)+o:o>De(e)?(r=e+1,a=o-De(e)):(r=e,a=o),{year:r,dayOfYear:a}}function Ie(e,t,n){var s,i,r=Ve(e.year(),t,n),a=Math.floor((e.dayOfYear()-r-1)/7)+1;return a<1?s=a+Ae(i=e.year()-1,t,n):a>Ae(e.year(),t,n)?(s=a-Ae(e.year(),t,n),i=e.year()+1):(i=e.year(),s=a),{week:s,year:i}}function Ae(e,t,n){var s=Ve(e,t,n),i=Ve(e+1,t,n);return(De(e)-s+i)/7}I("w",["ww",2],"wo","week"),I("W",["WW",2],"Wo","isoWeek"),H("week","w"),H("isoWeek","W"),L("week",5),L("isoWeek",5),ue("w",B),ue("ww",B,z),ue("W",B),ue("WW",B,z),fe(["w","ww","W","WW"],function(e,t,n,s){t[s.substr(0,1)]=k(e)});I("d",0,"do","day"),I("dd",0,0,function(e){return this.localeData().weekdaysMin(this,e)}),I("ddd",0,0,function(e){return this.localeData().weekdaysShort(this,e)}),I("dddd",0,0,function(e){return this.localeData().weekdays(this,e)}),I("e",0,0,"weekday"),I("E",0,0,"isoWeekday"),H("day","d"),H("weekday","e"),H("isoWeekday","E"),L("day",11),L("weekday",11),L("isoWeekday",11),ue("d",B),ue("e",B),ue("E",B),ue("dd",function(e,t){return t.weekdaysMinRegex(e)}),ue("ddd",function(e,t){return t.weekdaysShortRegex(e)}),ue("dddd",function(e,t){return t.weekdaysRegex(e)}),fe(["dd","ddd","dddd"],function(e,t,n,s){var i=n._locale.weekdaysParse(e,s,n._strict);null!=i?t.d=i:g(n).invalidWeekday=e}),fe(["d","e","E"],function(e,t,n,s){t[s]=k(e)});var je="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_");var Ze="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_");var ze="Su_Mo_Tu_We_Th_Fr_Sa".split("_");var $e=ae;var qe=ae;var Je=ae;function Be(){function e(e,t){return t.length-e.length}var t,n,s,i,r,a=[],o=[],u=[],l=[];for(t=0;t<7;t++)n=y([2e3,1]).day(t),s=this.weekdaysMin(n,""),i=this.weekdaysShort(n,""),r=this.weekdays(n,""),a.push(s),o.push(i),u.push(r),l.push(s),l.push(i),l.push(r);for(a.sort(e),o.sort(e),u.sort(e),l.sort(e),t=0;t<7;t++)o[t]=de(o[t]),u[t]=de(u[t]),l[t]=de(l[t]);this._weekdaysRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+u.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+o.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+a.join("|")+")","i")}function Qe(){return this.hours()%12||12}function Xe(e,t){I(e,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),t)})}function Ke(e,t){return t._meridiemParse}I("H",["HH",2],0,"hour"),I("h",["hh",2],0,Qe),I("k",["kk",2],0,function(){return this.hours()||24}),I("hmm",0,0,function(){return""+Qe.apply(this)+U(this.minutes(),2)}),I("hmmss",0,0,function(){return""+Qe.apply(this)+U(this.minutes(),2)+U(this.seconds(),2)}),I("Hmm",0,0,function(){return""+this.hours()+U(this.minutes(),2)}),I("Hmmss",0,0,function(){return""+this.hours()+U(this.minutes(),2)+U(this.seconds(),2)}),Xe("a",!0),Xe("A",!1),H("hour","h"),L("hour",13),ue("a",Ke),ue("A",Ke),ue("H",B),ue("h",B),ue("k",B),ue("HH",B,z),ue("hh",B,z),ue("kk",B,z),ue("hmm",Q),ue("hmmss",X),ue("Hmm",Q),ue("Hmmss",X),ce(["H","HH"],ge),ce(["k","kk"],function(e,t,n){var s=k(e);t[ge]=24===s?0:s}),ce(["a","A"],function(e,t,n){n._isPm=n._locale.isPM(e),n._meridiem=e}),ce(["h","hh"],function(e,t,n){t[ge]=k(e),g(n).bigHour=!0}),ce("hmm",function(e,t,n){var s=e.length-2;t[ge]=k(e.substr(0,s)),t[pe]=k(e.substr(s)),g(n).bigHour=!0}),ce("hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[ge]=k(e.substr(0,s)),t[pe]=k(e.substr(s,2)),t[ve]=k(e.substr(i)),g(n).bigHour=!0}),ce("Hmm",function(e,t,n){var s=e.length-2;t[ge]=k(e.substr(0,s)),t[pe]=k(e.substr(s))}),ce("Hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[ge]=k(e.substr(0,s)),t[pe]=k(e.substr(s,2)),t[ve]=k(e.substr(i))});var et,tt=Te("Hours",!0),nt={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:He,monthsShort:Re,week:{dow:0,doy:6},weekdays:je,weekdaysMin:ze,weekdaysShort:Ze,meridiemParse:/[ap]\.?m?\.?/i},st={},it={};function rt(e){return e?e.toLowerCase().replace("_","-"):e}function at(e){var t=null;if(!st[e]&&"undefined"!=typeof module&&module&&module.exports)try{t=et._abbr,require("./locale/"+e),ot(t)}catch(e){}return st[e]}function ot(e,t){var n;return e&&((n=l(t)?lt(e):ut(e,t))?et=n:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+e+" not found. Did you forget to load it?")),et._abbr}function ut(e,t){if(null!==t){var n,s=nt;if(t.abbr=e,null!=st[e])T("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),s=st[e]._config;else if(null!=t.parentLocale)if(null!=st[t.parentLocale])s=st[t.parentLocale]._config;else{if(null==(n=at(t.parentLocale)))return it[t.parentLocale]||(it[t.parentLocale]=[]),it[t.parentLocale].push({name:e,config:t}),null;s=n._config}return st[e]=new P(b(s,t)),it[e]&&it[e].forEach(function(e){ut(e.name,e.config)}),ot(e),st[e]}return delete st[e],null}function lt(e){var t;if(e&&e._locale&&e._locale._abbr&&(e=e._locale._abbr),!e)return et;if(!o(e)){if(t=at(e))return t;e=[e]}return function(e){for(var t,n,s,i,r=0;r<e.length;){for(t=(i=rt(e[r]).split("-")).length,n=(n=rt(e[r+1]))?n.split("-"):null;0<t;){if(s=at(i.slice(0,t).join("-")))return s;if(n&&n.length>=t&&a(i,n,!0)>=t-1)break;t--}r++}return et}(e)}function dt(e){var t,n=e._a;return n&&-2===g(e).overflow&&(t=n[_e]<0||11<n[_e]?_e:n[ye]<1||n[ye]>Pe(n[me],n[_e])?ye:n[ge]<0||24<n[ge]||24===n[ge]&&(0!==n[pe]||0!==n[ve]||0!==n[we])?ge:n[pe]<0||59<n[pe]?pe:n[ve]<0||59<n[ve]?ve:n[we]<0||999<n[we]?we:-1,g(e)._overflowDayOfYear&&(t<me||ye<t)&&(t=ye),g(e)._overflowWeeks&&-1===t&&(t=Me),g(e)._overflowWeekday&&-1===t&&(t=Se),g(e).overflow=t),e}function ht(e,t,n){return null!=e?e:null!=t?t:n}function ct(e){var t,n,s,i,r,a=[];if(!e._d){var o,u;for(o=e,u=new Date(c.now()),s=o._useUTC?[u.getUTCFullYear(),u.getUTCMonth(),u.getUTCDate()]:[u.getFullYear(),u.getMonth(),u.getDate()],e._w&&null==e._a[ye]&&null==e._a[_e]&&function(e){var t,n,s,i,r,a,o,u;if(null!=(t=e._w).GG||null!=t.W||null!=t.E)r=1,a=4,n=ht(t.GG,e._a[me],Ie(Tt(),1,4).year),s=ht(t.W,1),((i=ht(t.E,1))<1||7<i)&&(u=!0);else{r=e._locale._week.dow,a=e._locale._week.doy;var l=Ie(Tt(),r,a);n=ht(t.gg,e._a[me],l.year),s=ht(t.w,l.week),null!=t.d?((i=t.d)<0||6<i)&&(u=!0):null!=t.e?(i=t.e+r,(t.e<0||6<t.e)&&(u=!0)):i=r}s<1||s>Ae(n,r,a)?g(e)._overflowWeeks=!0:null!=u?g(e)._overflowWeekday=!0:(o=Ee(n,s,i,r,a),e._a[me]=o.year,e._dayOfYear=o.dayOfYear)}(e),null!=e._dayOfYear&&(r=ht(e._a[me],s[me]),(e._dayOfYear>De(r)||0===e._dayOfYear)&&(g(e)._overflowDayOfYear=!0),n=Ge(r,0,e._dayOfYear),e._a[_e]=n.getUTCMonth(),e._a[ye]=n.getUTCDate()),t=0;t<3&&null==e._a[t];++t)e._a[t]=a[t]=s[t];for(;t<7;t++)e._a[t]=a[t]=null==e._a[t]?2===t?1:0:e._a[t];24===e._a[ge]&&0===e._a[pe]&&0===e._a[ve]&&0===e._a[we]&&(e._nextDay=!0,e._a[ge]=0),e._d=(e._useUTC?Ge:function(e,t,n,s,i,r,a){var o=new Date(e,t,n,s,i,r,a);return e<100&&0<=e&&isFinite(o.getFullYear())&&o.setFullYear(e),o}).apply(null,a),i=e._useUTC?e._d.getUTCDay():e._d.getDay(),null!=e._tzm&&e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),e._nextDay&&(e._a[ge]=24),e._w&&void 0!==e._w.d&&e._w.d!==i&&(g(e).weekdayMismatch=!0)}}var ft=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,mt=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,_t=/Z|[+-]\d\d(?::?\d\d)?/,yt=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],gt=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],pt=/^\/?Date\((\-?\d+)/i;function vt(e){var t,n,s,i,r,a,o=e._i,u=ft.exec(o)||mt.exec(o);if(u){for(g(e).iso=!0,t=0,n=yt.length;t<n;t++)if(yt[t][1].exec(u[1])){i=yt[t][0],s=!1!==yt[t][2];break}if(null==i)return void(e._isValid=!1);if(u[3]){for(t=0,n=gt.length;t<n;t++)if(gt[t][1].exec(u[3])){r=(u[2]||" ")+gt[t][0];break}if(null==r)return void(e._isValid=!1)}if(!s&&null!=r)return void(e._isValid=!1);if(u[4]){if(!_t.exec(u[4]))return void(e._isValid=!1);a="Z"}e._f=i+(r||"")+(a||""),kt(e)}else e._isValid=!1}var wt=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/;function Mt(e,t,n,s,i,r){var a=[function(e){var t=parseInt(e,10);{if(t<=49)return 2e3+t;if(t<=999)return 1900+t}return t}(e),Re.indexOf(t),parseInt(n,10),parseInt(s,10),parseInt(i,10)];return r&&a.push(parseInt(r,10)),a}var St={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};function Dt(e){var t,n,s,i=wt.exec(e._i.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").trim());if(i){var r=Mt(i[4],i[3],i[2],i[5],i[6],i[7]);if(t=i[1],n=r,s=e,t&&Ze.indexOf(t)!==new Date(n[0],n[1],n[2]).getDay()&&(g(s).weekdayMismatch=!0,!(s._isValid=!1)))return;e._a=r,e._tzm=function(e,t,n){if(e)return St[e];if(t)return 0;var s=parseInt(n,10),i=s%100;return(s-i)/100*60+i}(i[8],i[9],i[10]),e._d=Ge.apply(null,e._a),e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),g(e).rfc2822=!0}else e._isValid=!1}function kt(e){if(e._f!==c.ISO_8601)if(e._f!==c.RFC_2822){e._a=[],g(e).empty=!0;var t,n,s,i,r,a,o,u,l=""+e._i,d=l.length,h=0;for(s=j(e._f,e._locale).match(N)||[],t=0;t<s.length;t++)i=s[t],(n=(l.match(le(i,e))||[])[0])&&(0<(r=l.substr(0,l.indexOf(n))).length&&g(e).unusedInput.push(r),l=l.slice(l.indexOf(n)+n.length),h+=n.length),E[i]?(n?g(e).empty=!1:g(e).unusedTokens.push(i),a=i,u=e,null!=(o=n)&&m(he,a)&&he[a](o,u._a,u,a)):e._strict&&!n&&g(e).unusedTokens.push(i);g(e).charsLeftOver=d-h,0<l.length&&g(e).unusedInput.push(l),e._a[ge]<=12&&!0===g(e).bigHour&&0<e._a[ge]&&(g(e).bigHour=void 0),g(e).parsedDateParts=e._a.slice(0),g(e).meridiem=e._meridiem,e._a[ge]=function(e,t,n){var s;if(null==n)return t;return null!=e.meridiemHour?e.meridiemHour(t,n):(null!=e.isPM&&((s=e.isPM(n))&&t<12&&(t+=12),s||12!==t||(t=0)),t)}(e._locale,e._a[ge],e._meridiem),ct(e),dt(e)}else Dt(e);else vt(e)}function Yt(e){var t,n,s,i,r=e._i,a=e._f;return e._locale=e._locale||lt(e._l),null===r||void 0===a&&""===r?v({nullInput:!0}):("string"==typeof r&&(e._i=r=e._locale.preparse(r)),S(r)?new M(dt(r)):(h(r)?e._d=r:o(a)?function(e){var t,n,s,i,r;if(0===e._f.length)return g(e).invalidFormat=!0,e._d=new Date(NaN);for(i=0;i<e._f.length;i++)r=0,t=w({},e),null!=e._useUTC&&(t._useUTC=e._useUTC),t._f=e._f[i],kt(t),p(t)&&(r+=g(t).charsLeftOver,r+=10*g(t).unusedTokens.length,g(t).score=r,(null==s||r<s)&&(s=r,n=t));_(e,n||t)}(e):a?kt(e):l(n=(t=e)._i)?t._d=new Date(c.now()):h(n)?t._d=new Date(n.valueOf()):"string"==typeof n?(s=t,null===(i=pt.exec(s._i))?(vt(s),!1===s._isValid&&(delete s._isValid,Dt(s),!1===s._isValid&&(delete s._isValid,c.createFromInputFallback(s)))):s._d=new Date(+i[1])):o(n)?(t._a=f(n.slice(0),function(e){return parseInt(e,10)}),ct(t)):u(n)?function(e){if(!e._d){var t=C(e._i);e._a=f([t.year,t.month,t.day||t.date,t.hour,t.minute,t.second,t.millisecond],function(e){return e&&parseInt(e,10)}),ct(e)}}(t):d(n)?t._d=new Date(n):c.createFromInputFallback(t),p(e)||(e._d=null),e))}function Ot(e,t,n,s,i){var r,a={};return!0!==n&&!1!==n||(s=n,n=void 0),(u(e)&&function(e){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(e).length;var t;for(t in e)if(e.hasOwnProperty(t))return!1;return!0}(e)||o(e)&&0===e.length)&&(e=void 0),a._isAMomentObject=!0,a._useUTC=a._isUTC=i,a._l=n,a._i=e,a._f=t,a._strict=s,(r=new M(dt(Yt(a))))._nextDay&&(r.add(1,"d"),r._nextDay=void 0),r}function Tt(e,t,n,s){return Ot(e,t,n,s,!1)}c.createFromInputFallback=n("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",function(e){e._d=new Date(e._i+(e._useUTC?" UTC":""))}),c.ISO_8601=function(){},c.RFC_2822=function(){};var xt=n("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var e=Tt.apply(null,arguments);return this.isValid()&&e.isValid()?e<this?this:e:v()}),bt=n("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var e=Tt.apply(null,arguments);return this.isValid()&&e.isValid()?this<e?this:e:v()});function Pt(e,t){var n,s;if(1===t.length&&o(t[0])&&(t=t[0]),!t.length)return Tt();for(n=t[0],s=1;s<t.length;++s)t[s].isValid()&&!t[s][e](n)||(n=t[s]);return n}var Wt=["year","quarter","month","week","day","hour","minute","second","millisecond"];function Ht(e){var t=C(e),n=t.year||0,s=t.quarter||0,i=t.month||0,r=t.week||0,a=t.day||0,o=t.hour||0,u=t.minute||0,l=t.second||0,d=t.millisecond||0;this._isValid=function(e){for(var t in e)if(-1===Ye.call(Wt,t)||null!=e[t]&&isNaN(e[t]))return!1;for(var n=!1,s=0;s<Wt.length;++s)if(e[Wt[s]]){if(n)return!1;parseFloat(e[Wt[s]])!==k(e[Wt[s]])&&(n=!0)}return!0}(t),this._milliseconds=+d+1e3*l+6e4*u+1e3*o*60*60,this._days=+a+7*r,this._months=+i+3*s+12*n,this._data={},this._locale=lt(),this._bubble()}function Rt(e){return e instanceof Ht}function Ct(e){return e<0?-1*Math.round(-1*e):Math.round(e)}function Ft(e,n){I(e,0,0,function(){var e=this.utcOffset(),t="+";return e<0&&(e=-e,t="-"),t+U(~~(e/60),2)+n+U(~~e%60,2)})}Ft("Z",":"),Ft("ZZ",""),ue("Z",re),ue("ZZ",re),ce(["Z","ZZ"],function(e,t,n){n._useUTC=!0,n._tzm=Ut(re,e)});var Lt=/([\+\-]|\d\d)/gi;function Ut(e,t){var n=(t||"").match(e);if(null===n)return null;var s=((n[n.length-1]||[])+"").match(Lt)||["-",0,0],i=60*s[1]+k(s[2]);return 0===i?0:"+"===s[0]?i:-i}function Nt(e,t){var n,s;return t._isUTC?(n=t.clone(),s=(S(e)||h(e)?e.valueOf():Tt(e).valueOf())-n.valueOf(),n._d.setTime(n._d.valueOf()+s),c.updateOffset(n,!1),n):Tt(e).local()}function Gt(e){return 15*-Math.round(e._d.getTimezoneOffset()/15)}function Vt(){return!!this.isValid()&&(this._isUTC&&0===this._offset)}c.updateOffset=function(){};var Et=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,It=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;function At(e,t){var n,s,i,r=e,a=null;return Rt(e)?r={ms:e._milliseconds,d:e._days,M:e._months}:d(e)?(r={},t?r[t]=e:r.milliseconds=e):(a=Et.exec(e))?(n="-"===a[1]?-1:1,r={y:0,d:k(a[ye])*n,h:k(a[ge])*n,m:k(a[pe])*n,s:k(a[ve])*n,ms:k(Ct(1e3*a[we]))*n}):(a=It.exec(e))?(n="-"===a[1]?-1:(a[1],1),r={y:jt(a[2],n),M:jt(a[3],n),w:jt(a[4],n),d:jt(a[5],n),h:jt(a[6],n),m:jt(a[7],n),s:jt(a[8],n)}):null==r?r={}:"object"==typeof r&&("from"in r||"to"in r)&&(i=function(e,t){var n;if(!e.isValid()||!t.isValid())return{milliseconds:0,months:0};t=Nt(t,e),e.isBefore(t)?n=Zt(e,t):((n=Zt(t,e)).milliseconds=-n.milliseconds,n.months=-n.months);return n}(Tt(r.from),Tt(r.to)),(r={}).ms=i.milliseconds,r.M=i.months),s=new Ht(r),Rt(e)&&m(e,"_locale")&&(s._locale=e._locale),s}function jt(e,t){var n=e&&parseFloat(e.replace(",","."));return(isNaN(n)?0:n)*t}function Zt(e,t){var n={milliseconds:0,months:0};return n.months=t.month()-e.month()+12*(t.year()-e.year()),e.clone().add(n.months,"M").isAfter(t)&&--n.months,n.milliseconds=+t-+e.clone().add(n.months,"M"),n}function zt(s,i){return function(e,t){var n;return null===t||isNaN(+t)||(T(i,"moment()."+i+"(period, number) is deprecated. Please use moment()."+i+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),n=e,e=t,t=n),$t(this,At(e="string"==typeof e?+e:e,t),s),this}}function $t(e,t,n,s){var i=t._milliseconds,r=Ct(t._days),a=Ct(t._months);e.isValid()&&(s=null==s||s,a&&Ce(e,xe(e,"Month")+a*n),r&&be(e,"Date",xe(e,"Date")+r*n),i&&e._d.setTime(e._d.valueOf()+i*n),s&&c.updateOffset(e,r||a))}At.fn=Ht.prototype,At.invalid=function(){return At(NaN)};var qt=zt(1,"add"),Jt=zt(-1,"subtract");function Bt(e,t){var n=12*(t.year()-e.year())+(t.month()-e.month()),s=e.clone().add(n,"months");return-(n+(t-s<0?(t-s)/(s-e.clone().add(n-1,"months")):(t-s)/(e.clone().add(n+1,"months")-s)))||0}function Qt(e){var t;return void 0===e?this._locale._abbr:(null!=(t=lt(e))&&(this._locale=t),this)}c.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",c.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var Xt=n("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(e){return void 0===e?this.localeData():this.locale(e)});function Kt(){return this._locale}function en(e,t){I(0,[e,e.length],0,t)}function tn(e,t,n,s,i){var r;return null==e?Ie(this,s,i).year:((r=Ae(e,s,i))<t&&(t=r),function(e,t,n,s,i){var r=Ee(e,t,n,s,i),a=Ge(r.year,0,r.dayOfYear);return this.year(a.getUTCFullYear()),this.month(a.getUTCMonth()),this.date(a.getUTCDate()),this}.call(this,e,t,n,s,i))}I(0,["gg",2],0,function(){return this.weekYear()%100}),I(0,["GG",2],0,function(){return this.isoWeekYear()%100}),en("gggg","weekYear"),en("ggggg","weekYear"),en("GGGG","isoWeekYear"),en("GGGGG","isoWeekYear"),H("weekYear","gg"),H("isoWeekYear","GG"),L("weekYear",1),L("isoWeekYear",1),ue("G",se),ue("g",se),ue("GG",B,z),ue("gg",B,z),ue("GGGG",ee,q),ue("gggg",ee,q),ue("GGGGG",te,J),ue("ggggg",te,J),fe(["gggg","ggggg","GGGG","GGGGG"],function(e,t,n,s){t[s.substr(0,2)]=k(e)}),fe(["gg","GG"],function(e,t,n,s){t[s]=c.parseTwoDigitYear(e)}),I("Q",0,"Qo","quarter"),H("quarter","Q"),L("quarter",7),ue("Q",Z),ce("Q",function(e,t){t[_e]=3*(k(e)-1)}),I("D",["DD",2],"Do","date"),H("date","D"),L("date",9),ue("D",B),ue("DD",B,z),ue("Do",function(e,t){return e?t._dayOfMonthOrdinalParse||t._ordinalParse:t._dayOfMonthOrdinalParseLenient}),ce(["D","DD"],ye),ce("Do",function(e,t){t[ye]=k(e.match(B)[0])});var nn=Te("Date",!0);I("DDD",["DDDD",3],"DDDo","dayOfYear"),H("dayOfYear","DDD"),L("dayOfYear",4),ue("DDD",K),ue("DDDD",$),ce(["DDD","DDDD"],function(e,t,n){n._dayOfYear=k(e)}),I("m",["mm",2],0,"minute"),H("minute","m"),L("minute",14),ue("m",B),ue("mm",B,z),ce(["m","mm"],pe);var sn=Te("Minutes",!1);I("s",["ss",2],0,"second"),H("second","s"),L("second",15),ue("s",B),ue("ss",B,z),ce(["s","ss"],ve);var rn,an=Te("Seconds",!1);for(I("S",0,0,function(){return~~(this.millisecond()/100)}),I(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),I(0,["SSS",3],0,"millisecond"),I(0,["SSSS",4],0,function(){return 10*this.millisecond()}),I(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),I(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),I(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),I(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),I(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),H("millisecond","ms"),L("millisecond",16),ue("S",K,Z),ue("SS",K,z),ue("SSS",K,$),rn="SSSS";rn.length<=9;rn+="S")ue(rn,ne);function on(e,t){t[we]=k(1e3*("0."+e))}for(rn="S";rn.length<=9;rn+="S")ce(rn,on);var un=Te("Milliseconds",!1);I("z",0,0,"zoneAbbr"),I("zz",0,0,"zoneName");var ln=M.prototype;function dn(e){return e}ln.add=qt,ln.calendar=function(e,t){var n=e||Tt(),s=Nt(n,this).startOf("day"),i=c.calendarFormat(this,s)||"sameElse",r=t&&(x(t[i])?t[i].call(this,n):t[i]);return this.format(r||this.localeData().calendar(i,this,Tt(n)))},ln.clone=function(){return new M(this)},ln.diff=function(e,t,n){var s,i,r;if(!this.isValid())return NaN;if(!(s=Nt(e,this)).isValid())return NaN;switch(i=6e4*(s.utcOffset()-this.utcOffset()),t=R(t)){case"year":r=Bt(this,s)/12;break;case"month":r=Bt(this,s);break;case"quarter":r=Bt(this,s)/3;break;case"second":r=(this-s)/1e3;break;case"minute":r=(this-s)/6e4;break;case"hour":r=(this-s)/36e5;break;case"day":r=(this-s-i)/864e5;break;case"week":r=(this-s-i)/6048e5;break;default:r=this-s}return n?r:D(r)},ln.endOf=function(e){return void 0===(e=R(e))||"millisecond"===e?this:("date"===e&&(e="day"),this.startOf(e).add(1,"isoWeek"===e?"week":e).subtract(1,"ms"))},ln.format=function(e){e||(e=this.isUtc()?c.defaultFormatUtc:c.defaultFormat);var t=A(this,e);return this.localeData().postformat(t)},ln.from=function(e,t){return this.isValid()&&(S(e)&&e.isValid()||Tt(e).isValid())?At({to:this,from:e}).locale(this.locale()).humanize(!t):this.localeData().invalidDate()},ln.fromNow=function(e){return this.from(Tt(),e)},ln.to=function(e,t){return this.isValid()&&(S(e)&&e.isValid()||Tt(e).isValid())?At({from:this,to:e}).locale(this.locale()).humanize(!t):this.localeData().invalidDate()},ln.toNow=function(e){return this.to(Tt(),e)},ln.get=function(e){return x(this[e=R(e)])?this[e]():this},ln.invalidAt=function(){return g(this).overflow},ln.isAfter=function(e,t){var n=S(e)?e:Tt(e);return!(!this.isValid()||!n.isValid())&&("millisecond"===(t=R(l(t)?"millisecond":t))?this.valueOf()>n.valueOf():n.valueOf()<this.clone().startOf(t).valueOf())},ln.isBefore=function(e,t){var n=S(e)?e:Tt(e);return!(!this.isValid()||!n.isValid())&&("millisecond"===(t=R(l(t)?"millisecond":t))?this.valueOf()<n.valueOf():this.clone().endOf(t).valueOf()<n.valueOf())},ln.isBetween=function(e,t,n,s){return("("===(s=s||"()")[0]?this.isAfter(e,n):!this.isBefore(e,n))&&(")"===s[1]?this.isBefore(t,n):!this.isAfter(t,n))},ln.isSame=function(e,t){var n,s=S(e)?e:Tt(e);return!(!this.isValid()||!s.isValid())&&("millisecond"===(t=R(t||"millisecond"))?this.valueOf()===s.valueOf():(n=s.valueOf(),this.clone().startOf(t).valueOf()<=n&&n<=this.clone().endOf(t).valueOf()))},ln.isSameOrAfter=function(e,t){return this.isSame(e,t)||this.isAfter(e,t)},ln.isSameOrBefore=function(e,t){return this.isSame(e,t)||this.isBefore(e,t)},ln.isValid=function(){return p(this)},ln.lang=Xt,ln.locale=Qt,ln.localeData=Kt,ln.max=bt,ln.min=xt,ln.parsingFlags=function(){return _({},g(this))},ln.set=function(e,t){if("object"==typeof e)for(var n=function(e){var t=[];for(var n in e)t.push({unit:n,priority:F[n]});return t.sort(function(e,t){return e.priority-t.priority}),t}(e=C(e)),s=0;s<n.length;s++)this[n[s].unit](e[n[s].unit]);else if(x(this[e=R(e)]))return this[e](t);return this},ln.startOf=function(e){switch(e=R(e)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===e&&this.weekday(0),"isoWeek"===e&&this.isoWeekday(1),"quarter"===e&&this.month(3*Math.floor(this.month()/3)),this},ln.subtract=Jt,ln.toArray=function(){var e=this;return[e.year(),e.month(),e.date(),e.hour(),e.minute(),e.second(),e.millisecond()]},ln.toObject=function(){var e=this;return{years:e.year(),months:e.month(),date:e.date(),hours:e.hours(),minutes:e.minutes(),seconds:e.seconds(),milliseconds:e.milliseconds()}},ln.toDate=function(){return new Date(this.valueOf())},ln.toISOString=function(e){if(!this.isValid())return null;var t=!0!==e,n=t?this.clone().utc():this;return n.year()<0||9999<n.year()?A(n,t?"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYYYY-MM-DD[T]HH:mm:ss.SSSZ"):x(Date.prototype.toISOString)?t?this.toDate().toISOString():new Date(this.valueOf()+60*this.utcOffset()*1e3).toISOString().replace("Z",A(n,"Z")):A(n,t?"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYY-MM-DD[T]HH:mm:ss.SSSZ")},ln.inspect=function(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var e="moment",t="";this.isLocal()||(e=0===this.utcOffset()?"moment.utc":"moment.parseZone",t="Z");var n="["+e+'("]',s=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",i=t+'[")]';return this.format(n+s+"-MM-DD[T]HH:mm:ss.SSS"+i)},ln.toJSON=function(){return this.isValid()?this.toISOString():null},ln.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},ln.unix=function(){return Math.floor(this.valueOf()/1e3)},ln.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},ln.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},ln.year=Oe,ln.isLeapYear=function(){return ke(this.year())},ln.weekYear=function(e){return tn.call(this,e,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)},ln.isoWeekYear=function(e){return tn.call(this,e,this.isoWeek(),this.isoWeekday(),1,4)},ln.quarter=ln.quarters=function(e){return null==e?Math.ceil((this.month()+1)/3):this.month(3*(e-1)+this.month()%3)},ln.month=Fe,ln.daysInMonth=function(){return Pe(this.year(),this.month())},ln.week=ln.weeks=function(e){var t=this.localeData().week(this);return null==e?t:this.add(7*(e-t),"d")},ln.isoWeek=ln.isoWeeks=function(e){var t=Ie(this,1,4).week;return null==e?t:this.add(7*(e-t),"d")},ln.weeksInYear=function(){var e=this.localeData()._week;return Ae(this.year(),e.dow,e.doy)},ln.isoWeeksInYear=function(){return Ae(this.year(),1,4)},ln.date=nn,ln.day=ln.days=function(e){if(!this.isValid())return null!=e?this:NaN;var t,n,s=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=e?(t=e,n=this.localeData(),e="string"!=typeof t?t:isNaN(t)?"number"==typeof(t=n.weekdaysParse(t))?t:null:parseInt(t,10),this.add(e-s,"d")):s},ln.weekday=function(e){if(!this.isValid())return null!=e?this:NaN;var t=(this.day()+7-this.localeData()._week.dow)%7;return null==e?t:this.add(e-t,"d")},ln.isoWeekday=function(e){if(!this.isValid())return null!=e?this:NaN;if(null!=e){var t=(n=e,s=this.localeData(),"string"==typeof n?s.weekdaysParse(n)%7||7:isNaN(n)?null:n);return this.day(this.day()%7?t:t-7)}return this.day()||7;var n,s},ln.dayOfYear=function(e){var t=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==e?t:this.add(e-t,"d")},ln.hour=ln.hours=tt,ln.minute=ln.minutes=sn,ln.second=ln.seconds=an,ln.millisecond=ln.milliseconds=un,ln.utcOffset=function(e,t,n){var s,i=this._offset||0;if(!this.isValid())return null!=e?this:NaN;if(null!=e){if("string"==typeof e){if(null===(e=Ut(re,e)))return this}else Math.abs(e)<16&&!n&&(e*=60);return!this._isUTC&&t&&(s=Gt(this)),this._offset=e,this._isUTC=!0,null!=s&&this.add(s,"m"),i!==e&&(!t||this._changeInProgress?$t(this,At(e-i,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,c.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?i:Gt(this)},ln.utc=function(e){return this.utcOffset(0,e)},ln.local=function(e){return this._isUTC&&(this.utcOffset(0,e),this._isUTC=!1,e&&this.subtract(Gt(this),"m")),this},ln.parseZone=function(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var e=Ut(ie,this._i);null!=e?this.utcOffset(e):this.utcOffset(0,!0)}return this},ln.hasAlignedHourOffset=function(e){return!!this.isValid()&&(e=e?Tt(e).utcOffset():0,(this.utcOffset()-e)%60==0)},ln.isDST=function(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},ln.isLocal=function(){return!!this.isValid()&&!this._isUTC},ln.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},ln.isUtc=Vt,ln.isUTC=Vt,ln.zoneAbbr=function(){return this._isUTC?"UTC":""},ln.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},ln.dates=n("dates accessor is deprecated. Use date instead.",nn),ln.months=n("months accessor is deprecated. Use month instead",Fe),ln.years=n("years accessor is deprecated. Use year instead",Oe),ln.zone=n("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",function(e,t){return null!=e?("string"!=typeof e&&(e=-e),this.utcOffset(e,t),this):-this.utcOffset()}),ln.isDSTShifted=n("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",function(){if(!l(this._isDSTShifted))return this._isDSTShifted;var e={};if(w(e,this),(e=Yt(e))._a){var t=e._isUTC?y(e._a):Tt(e._a);this._isDSTShifted=this.isValid()&&0<a(e._a,t.toArray())}else this._isDSTShifted=!1;return this._isDSTShifted});var hn=P.prototype;function cn(e,t,n,s){var i=lt(),r=y().set(s,t);return i[n](r,e)}function fn(e,t,n){if(d(e)&&(t=e,e=void 0),e=e||"",null!=t)return cn(e,t,n,"month");var s,i=[];for(s=0;s<12;s++)i[s]=cn(e,s,n,"month");return i}function mn(e,t,n,s){"boolean"==typeof e?d(t)&&(n=t,t=void 0):(t=e,e=!1,d(n=t)&&(n=t,t=void 0)),t=t||"";var i,r=lt(),a=e?r._week.dow:0;if(null!=n)return cn(t,(n+a)%7,s,"day");var o=[];for(i=0;i<7;i++)o[i]=cn(t,(i+a)%7,s,"day");return o}hn.calendar=function(e,t,n){var s=this._calendar[e]||this._calendar.sameElse;return x(s)?s.call(t,n):s},hn.longDateFormat=function(e){var t=this._longDateFormat[e],n=this._longDateFormat[e.toUpperCase()];return t||!n?t:(this._longDateFormat[e]=n.replace(/MMMM|MM|DD|dddd/g,function(e){return e.slice(1)}),this._longDateFormat[e])},hn.invalidDate=function(){return this._invalidDate},hn.ordinal=function(e){return this._ordinal.replace("%d",e)},hn.preparse=dn,hn.postformat=dn,hn.relativeTime=function(e,t,n,s){var i=this._relativeTime[n];return x(i)?i(e,t,n,s):i.replace(/%d/i,e)},hn.pastFuture=function(e,t){var n=this._relativeTime[0<e?"future":"past"];return x(n)?n(t):n.replace(/%s/i,t)},hn.set=function(e){var t,n;for(n in e)x(t=e[n])?this[n]=t:this["_"+n]=t;this._config=e,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)},hn.months=function(e,t){return e?o(this._months)?this._months[e.month()]:this._months[(this._months.isFormat||We).test(t)?"format":"standalone"][e.month()]:o(this._months)?this._months:this._months.standalone},hn.monthsShort=function(e,t){return e?o(this._monthsShort)?this._monthsShort[e.month()]:this._monthsShort[We.test(t)?"format":"standalone"][e.month()]:o(this._monthsShort)?this._monthsShort:this._monthsShort.standalone},hn.monthsParse=function(e,t,n){var s,i,r;if(this._monthsParseExact)return function(e,t,n){var s,i,r,a=e.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],s=0;s<12;++s)r=y([2e3,s]),this._shortMonthsParse[s]=this.monthsShort(r,"").toLocaleLowerCase(),this._longMonthsParse[s]=this.months(r,"").toLocaleLowerCase();return n?"MMM"===t?-1!==(i=Ye.call(this._shortMonthsParse,a))?i:null:-1!==(i=Ye.call(this._longMonthsParse,a))?i:null:"MMM"===t?-1!==(i=Ye.call(this._shortMonthsParse,a))?i:-1!==(i=Ye.call(this._longMonthsParse,a))?i:null:-1!==(i=Ye.call(this._longMonthsParse,a))?i:-1!==(i=Ye.call(this._shortMonthsParse,a))?i:null}.call(this,e,t,n);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),s=0;s<12;s++){if(i=y([2e3,s]),n&&!this._longMonthsParse[s]&&(this._longMonthsParse[s]=new RegExp("^"+this.months(i,"").replace(".","")+"$","i"),this._shortMonthsParse[s]=new RegExp("^"+this.monthsShort(i,"").replace(".","")+"$","i")),n||this._monthsParse[s]||(r="^"+this.months(i,"")+"|^"+this.monthsShort(i,""),this._monthsParse[s]=new RegExp(r.replace(".",""),"i")),n&&"MMMM"===t&&this._longMonthsParse[s].test(e))return s;if(n&&"MMM"===t&&this._shortMonthsParse[s].test(e))return s;if(!n&&this._monthsParse[s].test(e))return s}},hn.monthsRegex=function(e){return this._monthsParseExact?(m(this,"_monthsRegex")||Ne.call(this),e?this._monthsStrictRegex:this._monthsRegex):(m(this,"_monthsRegex")||(this._monthsRegex=Ue),this._monthsStrictRegex&&e?this._monthsStrictRegex:this._monthsRegex)},hn.monthsShortRegex=function(e){return this._monthsParseExact?(m(this,"_monthsRegex")||Ne.call(this),e?this._monthsShortStrictRegex:this._monthsShortRegex):(m(this,"_monthsShortRegex")||(this._monthsShortRegex=Le),this._monthsShortStrictRegex&&e?this._monthsShortStrictRegex:this._monthsShortRegex)},hn.week=function(e){return Ie(e,this._week.dow,this._week.doy).week},hn.firstDayOfYear=function(){return this._week.doy},hn.firstDayOfWeek=function(){return this._week.dow},hn.weekdays=function(e,t){return e?o(this._weekdays)?this._weekdays[e.day()]:this._weekdays[this._weekdays.isFormat.test(t)?"format":"standalone"][e.day()]:o(this._weekdays)?this._weekdays:this._weekdays.standalone},hn.weekdaysMin=function(e){return e?this._weekdaysMin[e.day()]:this._weekdaysMin},hn.weekdaysShort=function(e){return e?this._weekdaysShort[e.day()]:this._weekdaysShort},hn.weekdaysParse=function(e,t,n){var s,i,r;if(this._weekdaysParseExact)return function(e,t,n){var s,i,r,a=e.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],s=0;s<7;++s)r=y([2e3,1]).day(s),this._minWeekdaysParse[s]=this.weekdaysMin(r,"").toLocaleLowerCase(),this._shortWeekdaysParse[s]=this.weekdaysShort(r,"").toLocaleLowerCase(),this._weekdaysParse[s]=this.weekdays(r,"").toLocaleLowerCase();return n?"dddd"===t?-1!==(i=Ye.call(this._weekdaysParse,a))?i:null:"ddd"===t?-1!==(i=Ye.call(this._shortWeekdaysParse,a))?i:null:-1!==(i=Ye.call(this._minWeekdaysParse,a))?i:null:"dddd"===t?-1!==(i=Ye.call(this._weekdaysParse,a))?i:-1!==(i=Ye.call(this._shortWeekdaysParse,a))?i:-1!==(i=Ye.call(this._minWeekdaysParse,a))?i:null:"ddd"===t?-1!==(i=Ye.call(this._shortWeekdaysParse,a))?i:-1!==(i=Ye.call(this._weekdaysParse,a))?i:-1!==(i=Ye.call(this._minWeekdaysParse,a))?i:null:-1!==(i=Ye.call(this._minWeekdaysParse,a))?i:-1!==(i=Ye.call(this._weekdaysParse,a))?i:-1!==(i=Ye.call(this._shortWeekdaysParse,a))?i:null}.call(this,e,t,n);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),s=0;s<7;s++){if(i=y([2e3,1]).day(s),n&&!this._fullWeekdaysParse[s]&&(this._fullWeekdaysParse[s]=new RegExp("^"+this.weekdays(i,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[s]=new RegExp("^"+this.weekdaysShort(i,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[s]=new RegExp("^"+this.weekdaysMin(i,"").replace(".",".?")+"$","i")),this._weekdaysParse[s]||(r="^"+this.weekdays(i,"")+"|^"+this.weekdaysShort(i,"")+"|^"+this.weekdaysMin(i,""),this._weekdaysParse[s]=new RegExp(r.replace(".",""),"i")),n&&"dddd"===t&&this._fullWeekdaysParse[s].test(e))return s;if(n&&"ddd"===t&&this._shortWeekdaysParse[s].test(e))return s;if(n&&"dd"===t&&this._minWeekdaysParse[s].test(e))return s;if(!n&&this._weekdaysParse[s].test(e))return s}},hn.weekdaysRegex=function(e){return this._weekdaysParseExact?(m(this,"_weekdaysRegex")||Be.call(this),e?this._weekdaysStrictRegex:this._weekdaysRegex):(m(this,"_weekdaysRegex")||(this._weekdaysRegex=$e),this._weekdaysStrictRegex&&e?this._weekdaysStrictRegex:this._weekdaysRegex)},hn.weekdaysShortRegex=function(e){return this._weekdaysParseExact?(m(this,"_weekdaysRegex")||Be.call(this),e?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(m(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=qe),this._weekdaysShortStrictRegex&&e?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)},hn.weekdaysMinRegex=function(e){return this._weekdaysParseExact?(m(this,"_weekdaysRegex")||Be.call(this),e?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(m(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=Je),this._weekdaysMinStrictRegex&&e?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)},hn.isPM=function(e){return"p"===(e+"").toLowerCase().charAt(0)},hn.meridiem=function(e,t,n){return 11<e?n?"pm":"PM":n?"am":"AM"},ot("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(e){var t=e%10;return e+(1===k(e%100/10)?"th":1===t?"st":2===t?"nd":3===t?"rd":"th")}}),c.lang=n("moment.lang is deprecated. Use moment.locale instead.",ot),c.langData=n("moment.langData is deprecated. Use moment.localeData instead.",lt);var _n=Math.abs;function yn(e,t,n,s){var i=At(t,n);return e._milliseconds+=s*i._milliseconds,e._days+=s*i._days,e._months+=s*i._months,e._bubble()}function gn(e){return e<0?Math.floor(e):Math.ceil(e)}function pn(e){return 4800*e/146097}function vn(e){return 146097*e/4800}function wn(e){return function(){return this.as(e)}}var Mn=wn("ms"),Sn=wn("s"),Dn=wn("m"),kn=wn("h"),Yn=wn("d"),On=wn("w"),Tn=wn("M"),xn=wn("y");function bn(e){return function(){return this.isValid()?this._data[e]:NaN}}var Pn=bn("milliseconds"),Wn=bn("seconds"),Hn=bn("minutes"),Rn=bn("hours"),Cn=bn("days"),Fn=bn("months"),Ln=bn("years");var Un=Math.round,Nn={ss:44,s:45,m:45,h:22,d:26,M:11};var Gn=Math.abs;function Vn(e){return(0<e)-(e<0)||+e}function En(){if(!this.isValid())return this.localeData().invalidDate();var e,t,n=Gn(this._milliseconds)/1e3,s=Gn(this._days),i=Gn(this._months);t=D((e=D(n/60))/60),n%=60,e%=60;var r=D(i/12),a=i%=12,o=s,u=t,l=e,d=n?n.toFixed(3).replace(/\.?0+$/,""):"",h=this.asSeconds();if(!h)return"P0D";var c=h<0?"-":"",f=Vn(this._months)!==Vn(h)?"-":"",m=Vn(this._days)!==Vn(h)?"-":"",_=Vn(this._milliseconds)!==Vn(h)?"-":"";return c+"P"+(r?f+r+"Y":"")+(a?f+a+"M":"")+(o?m+o+"D":"")+(u||l||d?"T":"")+(u?_+u+"H":"")+(l?_+l+"M":"")+(d?_+d+"S":"")}var In=Ht.prototype;return In.isValid=function(){return this._isValid},In.abs=function(){var e=this._data;return this._milliseconds=_n(this._milliseconds),this._days=_n(this._days),this._months=_n(this._months),e.milliseconds=_n(e.milliseconds),e.seconds=_n(e.seconds),e.minutes=_n(e.minutes),e.hours=_n(e.hours),e.months=_n(e.months),e.years=_n(e.years),this},In.add=function(e,t){return yn(this,e,t,1)},In.subtract=function(e,t){return yn(this,e,t,-1)},In.as=function(e){if(!this.isValid())return NaN;var t,n,s=this._milliseconds;if("month"===(e=R(e))||"year"===e)return t=this._days+s/864e5,n=this._months+pn(t),"month"===e?n:n/12;switch(t=this._days+Math.round(vn(this._months)),e){case"week":return t/7+s/6048e5;case"day":return t+s/864e5;case"hour":return 24*t+s/36e5;case"minute":return 1440*t+s/6e4;case"second":return 86400*t+s/1e3;case"millisecond":return Math.floor(864e5*t)+s;default:throw new Error("Unknown unit "+e)}},In.asMilliseconds=Mn,In.asSeconds=Sn,In.asMinutes=Dn,In.asHours=kn,In.asDays=Yn,In.asWeeks=On,In.asMonths=Tn,In.asYears=xn,In.valueOf=function(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*k(this._months/12):NaN},In._bubble=function(){var e,t,n,s,i,r=this._milliseconds,a=this._days,o=this._months,u=this._data;return 0<=r&&0<=a&&0<=o||r<=0&&a<=0&&o<=0||(r+=864e5*gn(vn(o)+a),o=a=0),u.milliseconds=r%1e3,e=D(r/1e3),u.seconds=e%60,t=D(e/60),u.minutes=t%60,n=D(t/60),u.hours=n%24,o+=i=D(pn(a+=D(n/24))),a-=gn(vn(i)),s=D(o/12),o%=12,u.days=a,u.months=o,u.years=s,this},In.clone=function(){return At(this)},In.get=function(e){return e=R(e),this.isValid()?this[e+"s"]():NaN},In.milliseconds=Pn,In.seconds=Wn,In.minutes=Hn,In.hours=Rn,In.days=Cn,In.weeks=function(){return D(this.days()/7)},In.months=Fn,In.years=Ln,In.humanize=function(e){if(!this.isValid())return this.localeData().invalidDate();var t,n,s,i,r,a,o,u,l,d,h,c=this.localeData(),f=(n=!e,s=c,i=At(t=this).abs(),r=Un(i.as("s")),a=Un(i.as("m")),o=Un(i.as("h")),u=Un(i.as("d")),l=Un(i.as("M")),d=Un(i.as("y")),(h=r<=Nn.ss&&["s",r]||r<Nn.s&&["ss",r]||a<=1&&["m"]||a<Nn.m&&["mm",a]||o<=1&&["h"]||o<Nn.h&&["hh",o]||u<=1&&["d"]||u<Nn.d&&["dd",u]||l<=1&&["M"]||l<Nn.M&&["MM",l]||d<=1&&["y"]||["yy",d])[2]=n,h[3]=0<+t,h[4]=s,function(e,t,n,s,i){return i.relativeTime(t||1,!!n,e,s)}.apply(null,h));return e&&(f=c.pastFuture(+this,f)),c.postformat(f)},In.toISOString=En,In.toString=En,In.toJSON=En,In.locale=Qt,In.localeData=Kt,In.toIsoString=n("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",En),In.lang=Xt,I("X",0,0,"unix"),I("x",0,0,"valueOf"),ue("x",se),ue("X",/[+-]?\d+(\.\d{1,3})?/),ce("X",function(e,t,n){n._d=new Date(1e3*parseFloat(e,10))}),ce("x",function(e,t,n){n._d=new Date(k(e))}),c.version="2.22.1",e=Tt,c.fn=ln,c.min=function(){return Pt("isBefore",[].slice.call(arguments,0))},c.max=function(){return Pt("isAfter",[].slice.call(arguments,0))},c.now=function(){return Date.now?Date.now():+new Date},c.utc=y,c.unix=function(e){return Tt(1e3*e)},c.months=function(e,t){return fn(e,t,"months")},c.isDate=h,c.locale=ot,c.invalid=v,c.duration=At,c.isMoment=S,c.weekdays=function(e,t,n){return mn(e,t,n,"weekdays")},c.parseZone=function(){return Tt.apply(null,arguments).parseZone()},c.localeData=lt,c.isDuration=Rt,c.monthsShort=function(e,t){return fn(e,t,"monthsShort")},c.weekdaysMin=function(e,t,n){return mn(e,t,n,"weekdaysMin")},c.defineLocale=ut,c.updateLocale=function(e,t){if(null!=t){var n,s,i=nt;null!=(s=at(e))&&(i=s._config),(n=new P(t=b(i,t))).parentLocale=st[e],st[e]=n,ot(e)}else null!=st[e]&&(null!=st[e].parentLocale?st[e]=st[e].parentLocale:null!=st[e]&&delete st[e]);return st[e]},c.locales=function(){return s(st)},c.weekdaysShort=function(e,t,n){return mn(e,t,n,"weekdaysShort")},c.normalizeUnits=R,c.relativeTimeRounding=function(e){return void 0===e?Un:"function"==typeof e&&(Un=e,!0)},c.relativeTimeThreshold=function(e,t){return void 0!==Nn[e]&&(void 0===t?Nn[e]:(Nn[e]=t,"s"===e&&(Nn.ss=t-1),!0))},c.calendarFormat=function(e,t){var n=e.diff(t,"days",!0);return n<-6?"sameElse":n<-1?"lastWeek":n<0?"lastDay":n<1?"sameDay":n<2?"nextDay":n<7?"nextWeek":"sameElse"},c.prototype=ln,c.HTML5_FMT={DATETIME_LOCAL:"YYYY-MM-DDTHH:mm",DATETIME_LOCAL_SECONDS:"YYYY-MM-DDTHH:mm:ss",DATETIME_LOCAL_MS:"YYYY-MM-DDTHH:mm:ss.SSS",DATE:"YYYY-MM-DD",TIME:"HH:mm",TIME_SECONDS:"HH:mm:ss",TIME_MS:"HH:mm:ss.SSS",WEEK:"YYYY-[W]WW",MONTH:"YYYY-MM"},c});
+!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):e.moment=t()}(this,function(){"use strict";var e;function c(){return e.apply(null,arguments)}function a(e){return e instanceof Array||"[object Array]"===Object.prototype.toString.call(e)}function o(e){return null!=e&&"[object Object]"===Object.prototype.toString.call(e)}function f(e,t){return Object.prototype.hasOwnProperty.call(e,t)}function u(e){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(e).length;for(var t in e)if(f(e,t))return;return 1}function l(e){return void 0===e}function h(e){return"number"==typeof e||"[object Number]"===Object.prototype.toString.call(e)}function s(e){return e instanceof Date||"[object Date]"===Object.prototype.toString.call(e)}function i(e,t){for(var n=[],s=e.length,i=0;i<s;++i)n.push(t(e[i],i));return n}function d(e,t){for(var n in t)f(t,n)&&(e[n]=t[n]);return f(t,"toString")&&(e.toString=t.toString),f(t,"valueOf")&&(e.valueOf=t.valueOf),e}function m(e,t,n,s){return Tt(e,t,n,s,!0).utc()}function _(e){return null==e._pf&&(e._pf={empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidEra:null,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],era:null,meridiem:null,rfc2822:!1,weekdayMismatch:!1}),e._pf}function y(e){if(null==e._isValid){var t=_(e),n=g.call(t.parsedDateParts,function(e){return null!=e}),n=!isNaN(e._d.getTime())&&t.overflow<0&&!t.empty&&!t.invalidEra&&!t.invalidMonth&&!t.invalidWeekday&&!t.weekdayMismatch&&!t.nullInput&&!t.invalidFormat&&!t.userInvalidated&&(!t.meridiem||t.meridiem&&n);if(e._strict&&(n=n&&0===t.charsLeftOver&&0===t.unusedTokens.length&&void 0===t.bigHour),null!=Object.isFrozen&&Object.isFrozen(e))return n;e._isValid=n}return e._isValid}function r(e){var t=m(NaN);return null!=e?d(_(t),e):_(t).userInvalidated=!0,t}var g=Array.prototype.some||function(e){for(var t=Object(this),n=t.length>>>0,s=0;s<n;s++)if(s in t&&e.call(this,t[s],s,t))return!0;return!1},w=c.momentProperties=[],t=!1;function p(e,t){var n,s,i,r=w.length;if(l(t._isAMomentObject)||(e._isAMomentObject=t._isAMomentObject),l(t._i)||(e._i=t._i),l(t._f)||(e._f=t._f),l(t._l)||(e._l=t._l),l(t._strict)||(e._strict=t._strict),l(t._tzm)||(e._tzm=t._tzm),l(t._isUTC)||(e._isUTC=t._isUTC),l(t._offset)||(e._offset=t._offset),l(t._pf)||(e._pf=_(t)),l(t._locale)||(e._locale=t._locale),0<r)for(n=0;n<r;n++)l(i=t[s=w[n]])||(e[s]=i);return e}function v(e){p(this,e),this._d=new Date(null!=e._d?e._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===t&&(t=!0,c.updateOffset(this),t=!1)}function k(e){return e instanceof v||null!=e&&null!=e._isAMomentObject}function M(e){!1===c.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+e)}function n(r,a){var o=!0;return d(function(){if(null!=c.deprecationHandler&&c.deprecationHandler(null,r),o){for(var e,t,n=[],s=arguments.length,i=0;i<s;i++){if(e="","object"==typeof arguments[i]){for(t in e+="\n["+i+"] ",arguments[0])f(arguments[0],t)&&(e+=t+": "+arguments[0][t]+", ");e=e.slice(0,-2)}else e=arguments[i];n.push(e)}M(r+"\nArguments: "+Array.prototype.slice.call(n).join("")+"\n"+(new Error).stack),o=!1}return a.apply(this,arguments)},a)}var D={};function S(e,t){null!=c.deprecationHandler&&c.deprecationHandler(e,t),D[e]||(M(t),D[e]=!0)}function Y(e){return"undefined"!=typeof Function&&e instanceof Function||"[object Function]"===Object.prototype.toString.call(e)}function O(e,t){var n,s=d({},e);for(n in t)f(t,n)&&(o(e[n])&&o(t[n])?(s[n]={},d(s[n],e[n]),d(s[n],t[n])):null!=t[n]?s[n]=t[n]:delete s[n]);for(n in e)f(e,n)&&!f(t,n)&&o(e[n])&&(s[n]=d({},s[n]));return s}function b(e){null!=e&&this.set(e)}c.suppressDeprecationWarnings=!1,c.deprecationHandler=null;var x=Object.keys||function(e){var t,n=[];for(t in e)f(e,t)&&n.push(t);return n};function T(e,t,n){var s=""+Math.abs(e);return(0<=e?n?"+":"":"-")+Math.pow(10,Math.max(0,t-s.length)).toString().substr(1)+s}var N=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|N{1,5}|YYYYYY|YYYYY|YYYY|YY|y{2,4}|yo?|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,P=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,R={},W={};function C(e,t,n,s){var i="string"==typeof s?function(){return this[s]()}:s;e&&(W[e]=i),t&&(W[t[0]]=function(){return T(i.apply(this,arguments),t[1],t[2])}),n&&(W[n]=function(){return this.localeData().ordinal(i.apply(this,arguments),e)})}function U(e,t){return e.isValid()?(t=H(t,e.localeData()),R[t]=R[t]||function(s){for(var e,i=s.match(N),t=0,r=i.length;t<r;t++)W[i[t]]?i[t]=W[i[t]]:i[t]=(e=i[t]).match(/\[[\s\S]/)?e.replace(/^\[|\]$/g,""):e.replace(/\\/g,"");return function(e){for(var t="",n=0;n<r;n++)t+=Y(i[n])?i[n].call(e,s):i[n];return t}}(t),R[t](e)):e.localeData().invalidDate()}function H(e,t){var n=5;function s(e){return t.longDateFormat(e)||e}for(P.lastIndex=0;0<=n&&P.test(e);)e=e.replace(P,s),P.lastIndex=0,--n;return e}var F={};function L(e,t){var n=e.toLowerCase();F[n]=F[n+"s"]=F[t]=e}function V(e){return"string"==typeof e?F[e]||F[e.toLowerCase()]:void 0}function G(e){var t,n,s={};for(n in e)f(e,n)&&(t=V(n))&&(s[t]=e[n]);return s}var E={};function A(e,t){E[e]=t}function I(e){return e%4==0&&e%100!=0||e%400==0}function j(e){return e<0?Math.ceil(e)||0:Math.floor(e)}function Z(e){var t=+e,e=0;return e=0!=t&&isFinite(t)?j(t):e}function z(t,n){return function(e){return null!=e?(q(this,t,e),c.updateOffset(this,n),this):$(this,t)}}function $(e,t){return e.isValid()?e._d["get"+(e._isUTC?"UTC":"")+t]():NaN}function q(e,t,n){e.isValid()&&!isNaN(n)&&("FullYear"===t&&I(e.year())&&1===e.month()&&29===e.date()?(n=Z(n),e._d["set"+(e._isUTC?"UTC":"")+t](n,e.month(),xe(n,e.month()))):e._d["set"+(e._isUTC?"UTC":"")+t](n))}var B=/\d/,J=/\d\d/,Q=/\d{3}/,X=/\d{4}/,K=/[+-]?\d{6}/,ee=/\d\d?/,te=/\d\d\d\d?/,ne=/\d\d\d\d\d\d?/,se=/\d{1,3}/,ie=/\d{1,4}/,re=/[+-]?\d{1,6}/,ae=/\d+/,oe=/[+-]?\d+/,ue=/Z|[+-]\d\d:?\d\d/gi,le=/Z|[+-]\d\d(?::?\d\d)?/gi,he=/[0-9]{0,256}['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFF07\uFF10-\uFFEF]{1,256}|[\u0600-\u06FF\/]{1,256}(\s*?[\u0600-\u06FF]{1,256}){1,2}/i;function de(e,n,s){me[e]=Y(n)?n:function(e,t){return e&&s?s:n}}function ce(e,t){return f(me,e)?me[e](t._strict,t._locale):new RegExp(fe(e.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(e,t,n,s,i){return t||n||s||i})))}function fe(e){return e.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}var me={},_e={};function ye(e,n){var t,s,i=n;for("string"==typeof e&&(e=[e]),h(n)&&(i=function(e,t){t[n]=Z(e)}),s=e.length,t=0;t<s;t++)_e[e[t]]=i}function ge(e,i){ye(e,function(e,t,n,s){n._w=n._w||{},i(e,n._w,n,s)})}var we,pe=0,ve=1,ke=2,Me=3,De=4,Se=5,Ye=6,Oe=7,be=8;function xe(e,t){if(isNaN(e)||isNaN(t))return NaN;var n,n=(t%(n=12)+n)%n;return e+=(t-n)/12,1==n?I(e)?29:28:31-n%7%2}we=Array.prototype.indexOf||function(e){for(var t=0;t<this.length;++t)if(this[t]===e)return t;return-1},C("M",["MM",2],"Mo",function(){return this.month()+1}),C("MMM",0,0,function(e){return this.localeData().monthsShort(this,e)}),C("MMMM",0,0,function(e){return this.localeData().months(this,e)}),L("month","M"),A("month",8),de("M",ee),de("MM",ee,J),de("MMM",function(e,t){return t.monthsShortRegex(e)}),de("MMMM",function(e,t){return t.monthsRegex(e)}),ye(["M","MM"],function(e,t){t[ve]=Z(e)-1}),ye(["MMM","MMMM"],function(e,t,n,s){s=n._locale.monthsParse(e,s,n._strict);null!=s?t[ve]=s:_(n).invalidMonth=e});var Te="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Ne="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Pe=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,Re=he,We=he;function Ce(e,t){var n;if(!e.isValid())return e;if("string"==typeof t)if(/^\d+$/.test(t))t=Z(t);else if(!h(t=e.localeData().monthsParse(t)))return e;return n=Math.min(e.date(),xe(e.year(),t)),e._d["set"+(e._isUTC?"UTC":"")+"Month"](t,n),e}function Ue(e){return null!=e?(Ce(this,e),c.updateOffset(this,!0),this):$(this,"Month")}function He(){function e(e,t){return t.length-e.length}for(var t,n=[],s=[],i=[],r=0;r<12;r++)t=m([2e3,r]),n.push(this.monthsShort(t,"")),s.push(this.months(t,"")),i.push(this.months(t,"")),i.push(this.monthsShort(t,""));for(n.sort(e),s.sort(e),i.sort(e),r=0;r<12;r++)n[r]=fe(n[r]),s[r]=fe(s[r]);for(r=0;r<24;r++)i[r]=fe(i[r]);this._monthsRegex=new RegExp("^("+i.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+n.join("|")+")","i")}function Fe(e){return I(e)?366:365}C("Y",0,0,function(){var e=this.year();return e<=9999?T(e,4):"+"+e}),C(0,["YY",2],0,function(){return this.year()%100}),C(0,["YYYY",4],0,"year"),C(0,["YYYYY",5],0,"year"),C(0,["YYYYYY",6,!0],0,"year"),L("year","y"),A("year",1),de("Y",oe),de("YY",ee,J),de("YYYY",ie,X),de("YYYYY",re,K),de("YYYYYY",re,K),ye(["YYYYY","YYYYYY"],pe),ye("YYYY",function(e,t){t[pe]=2===e.length?c.parseTwoDigitYear(e):Z(e)}),ye("YY",function(e,t){t[pe]=c.parseTwoDigitYear(e)}),ye("Y",function(e,t){t[pe]=parseInt(e,10)}),c.parseTwoDigitYear=function(e){return Z(e)+(68<Z(e)?1900:2e3)};var Le=z("FullYear",!0);function Ve(e,t,n,s,i,r,a){var o;return e<100&&0<=e?(o=new Date(e+400,t,n,s,i,r,a),isFinite(o.getFullYear())&&o.setFullYear(e)):o=new Date(e,t,n,s,i,r,a),o}function Ge(e){var t;return e<100&&0<=e?((t=Array.prototype.slice.call(arguments))[0]=e+400,t=new Date(Date.UTC.apply(null,t)),isFinite(t.getUTCFullYear())&&t.setUTCFullYear(e)):t=new Date(Date.UTC.apply(null,arguments)),t}function Ee(e,t,n){n=7+t-n;return n-(7+Ge(e,0,n).getUTCDay()-t)%7-1}function Ae(e,t,n,s,i){var r,i=1+7*(t-1)+(7+n-s)%7+Ee(e,s,i),i=i<=0?Fe(r=e-1)+i:i>Fe(e)?(r=e+1,i-Fe(e)):(r=e,i);return{year:r,dayOfYear:i}}function Ie(e,t,n){var s,i,r=Ee(e.year(),t,n),r=Math.floor((e.dayOfYear()-r-1)/7)+1;return r<1?s=r+je(i=e.year()-1,t,n):r>je(e.year(),t,n)?(s=r-je(e.year(),t,n),i=e.year()+1):(i=e.year(),s=r),{week:s,year:i}}function je(e,t,n){var s=Ee(e,t,n),n=Ee(e+1,t,n);return(Fe(e)-s+n)/7}C("w",["ww",2],"wo","week"),C("W",["WW",2],"Wo","isoWeek"),L("week","w"),L("isoWeek","W"),A("week",5),A("isoWeek",5),de("w",ee),de("ww",ee,J),de("W",ee),de("WW",ee,J),ge(["w","ww","W","WW"],function(e,t,n,s){t[s.substr(0,1)]=Z(e)});function Ze(e,t){return e.slice(t,7).concat(e.slice(0,t))}C("d",0,"do","day"),C("dd",0,0,function(e){return this.localeData().weekdaysMin(this,e)}),C("ddd",0,0,function(e){return this.localeData().weekdaysShort(this,e)}),C("dddd",0,0,function(e){return this.localeData().weekdays(this,e)}),C("e",0,0,"weekday"),C("E",0,0,"isoWeekday"),L("day","d"),L("weekday","e"),L("isoWeekday","E"),A("day",11),A("weekday",11),A("isoWeekday",11),de("d",ee),de("e",ee),de("E",ee),de("dd",function(e,t){return t.weekdaysMinRegex(e)}),de("ddd",function(e,t){return t.weekdaysShortRegex(e)}),de("dddd",function(e,t){return t.weekdaysRegex(e)}),ge(["dd","ddd","dddd"],function(e,t,n,s){s=n._locale.weekdaysParse(e,s,n._strict);null!=s?t.d=s:_(n).invalidWeekday=e}),ge(["d","e","E"],function(e,t,n,s){t[s]=Z(e)});var ze="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),$e="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),qe="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),Be=he,Je=he,Qe=he;function Xe(){function e(e,t){return t.length-e.length}for(var t,n,s,i=[],r=[],a=[],o=[],u=0;u<7;u++)s=m([2e3,1]).day(u),t=fe(this.weekdaysMin(s,"")),n=fe(this.weekdaysShort(s,"")),s=fe(this.weekdays(s,"")),i.push(t),r.push(n),a.push(s),o.push(t),o.push(n),o.push(s);i.sort(e),r.sort(e),a.sort(e),o.sort(e),this._weekdaysRegex=new RegExp("^("+o.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+r.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+i.join("|")+")","i")}function Ke(){return this.hours()%12||12}function et(e,t){C(e,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),t)})}function tt(e,t){return t._meridiemParse}C("H",["HH",2],0,"hour"),C("h",["hh",2],0,Ke),C("k",["kk",2],0,function(){return this.hours()||24}),C("hmm",0,0,function(){return""+Ke.apply(this)+T(this.minutes(),2)}),C("hmmss",0,0,function(){return""+Ke.apply(this)+T(this.minutes(),2)+T(this.seconds(),2)}),C("Hmm",0,0,function(){return""+this.hours()+T(this.minutes(),2)}),C("Hmmss",0,0,function(){return""+this.hours()+T(this.minutes(),2)+T(this.seconds(),2)}),et("a",!0),et("A",!1),L("hour","h"),A("hour",13),de("a",tt),de("A",tt),de("H",ee),de("h",ee),de("k",ee),de("HH",ee,J),de("hh",ee,J),de("kk",ee,J),de("hmm",te),de("hmmss",ne),de("Hmm",te),de("Hmmss",ne),ye(["H","HH"],Me),ye(["k","kk"],function(e,t,n){e=Z(e);t[Me]=24===e?0:e}),ye(["a","A"],function(e,t,n){n._isPm=n._locale.isPM(e),n._meridiem=e}),ye(["h","hh"],function(e,t,n){t[Me]=Z(e),_(n).bigHour=!0}),ye("hmm",function(e,t,n){var s=e.length-2;t[Me]=Z(e.substr(0,s)),t[De]=Z(e.substr(s)),_(n).bigHour=!0}),ye("hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[Me]=Z(e.substr(0,s)),t[De]=Z(e.substr(s,2)),t[Se]=Z(e.substr(i)),_(n).bigHour=!0}),ye("Hmm",function(e,t,n){var s=e.length-2;t[Me]=Z(e.substr(0,s)),t[De]=Z(e.substr(s))}),ye("Hmmss",function(e,t,n){var s=e.length-4,i=e.length-2;t[Me]=Z(e.substr(0,s)),t[De]=Z(e.substr(s,2)),t[Se]=Z(e.substr(i))});var nt=z("Hours",!0);var st,it={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",w:"a week",ww:"%d weeks",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Te,monthsShort:Ne,week:{dow:0,doy:6},weekdays:ze,weekdaysMin:qe,weekdaysShort:$e,meridiemParse:/[ap]\.?m?\.?/i},rt={},at={};function ot(e){return e&&e.toLowerCase().replace("_","-")}function ut(e){for(var t,n,s,i,r=0;r<e.length;){for(t=(i=ot(e[r]).split("-")).length,n=(n=ot(e[r+1]))?n.split("-"):null;0<t;){if(s=lt(i.slice(0,t).join("-")))return s;if(n&&n.length>=t&&function(e,t){for(var n=Math.min(e.length,t.length),s=0;s<n;s+=1)if(e[s]!==t[s])return s;return n}(i,n)>=t-1)break;t--}r++}return st}function lt(t){var e;if(void 0===rt[t]&&"undefined"!=typeof module&&module&&module.exports&&null!=t.match("^[^/\\\\]*$"))try{e=st._abbr,require("./locale/"+t),ht(e)}catch(e){rt[t]=null}return rt[t]}function ht(e,t){return e&&((t=l(t)?ct(e):dt(e,t))?st=t:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+e+" not found. Did you forget to load it?")),st._abbr}function dt(e,t){if(null===t)return delete rt[e],null;var n,s=it;if(t.abbr=e,null!=rt[e])S("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),s=rt[e]._config;else if(null!=t.parentLocale)if(null!=rt[t.parentLocale])s=rt[t.parentLocale]._config;else{if(null==(n=lt(t.parentLocale)))return at[t.parentLocale]||(at[t.parentLocale]=[]),at[t.parentLocale].push({name:e,config:t}),null;s=n._config}return rt[e]=new b(O(s,t)),at[e]&&at[e].forEach(function(e){dt(e.name,e.config)}),ht(e),rt[e]}function ct(e){var t;if(!(e=e&&e._locale&&e._locale._abbr?e._locale._abbr:e))return st;if(!a(e)){if(t=lt(e))return t;e=[e]}return ut(e)}function ft(e){var t=e._a;return t&&-2===_(e).overflow&&(t=t[ve]<0||11<t[ve]?ve:t[ke]<1||t[ke]>xe(t[pe],t[ve])?ke:t[Me]<0||24<t[Me]||24===t[Me]&&(0!==t[De]||0!==t[Se]||0!==t[Ye])?Me:t[De]<0||59<t[De]?De:t[Se]<0||59<t[Se]?Se:t[Ye]<0||999<t[Ye]?Ye:-1,_(e)._overflowDayOfYear&&(t<pe||ke<t)&&(t=ke),_(e)._overflowWeeks&&-1===t&&(t=Oe),_(e)._overflowWeekday&&-1===t&&(t=be),_(e).overflow=t),e}var mt=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([+-]\d\d(?::?\d\d)?|\s*Z)?)?$/,_t=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d|))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([+-]\d\d(?::?\d\d)?|\s*Z)?)?$/,yt=/Z|[+-]\d\d(?::?\d\d)?/,gt=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/],["YYYYMM",/\d{6}/,!1],["YYYY",/\d{4}/,!1]],wt=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],pt=/^\/?Date\((-?\d+)/i,vt=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/,kt={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};function Mt(e){var t,n,s,i,r,a,o=e._i,u=mt.exec(o)||_t.exec(o),l=gt.length,o=wt.length;if(u){for(_(e).iso=!0,t=0,n=l;t<n;t++)if(gt[t][1].exec(u[1])){i=gt[t][0],s=!1!==gt[t][2];break}if(null!=i){if(u[3]){for(t=0,n=o;t<n;t++)if(wt[t][1].exec(u[3])){r=(u[2]||" ")+wt[t][0];break}if(null==r)return void(e._isValid=!1)}if(s||null==r){if(u[4]){if(!yt.exec(u[4]))return void(e._isValid=!1);a="Z"}e._f=i+(r||"")+(a||""),bt(e)}else e._isValid=!1}else e._isValid=!1}else e._isValid=!1}function Dt(e,t,n,s,i,r){i=[function(e){e=parseInt(e,10);{if(e<=49)return 2e3+e;if(e<=999)return 1900+e}return e}(e),Ne.indexOf(t),parseInt(n,10),parseInt(s,10),parseInt(i,10)];return r&&i.push(parseInt(r,10)),i}function St(e){var t,n,s,i,r=vt.exec(e._i.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").replace(/^\s\s*/,"").replace(/\s\s*$/,""));r?(t=Dt(r[4],r[3],r[2],r[5],r[6],r[7]),n=r[1],s=t,i=e,n&&$e.indexOf(n)!==new Date(s[0],s[1],s[2]).getDay()?(_(i).weekdayMismatch=!0,i._isValid=!1):(e._a=t,e._tzm=(i=r[8],t=r[9],r=r[10],i?kt[i]:t?0:60*(((t=parseInt(r,10))-(r=t%100))/100)+r),e._d=Ge.apply(null,e._a),e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),_(e).rfc2822=!0)):e._isValid=!1}function Yt(e,t,n){return null!=e?e:null!=t?t:n}function Ot(e){var t,n,s,i,r,a=[];if(!e._d){for(i=e,r=new Date(c.now()),n=i._useUTC?[r.getUTCFullYear(),r.getUTCMonth(),r.getUTCDate()]:[r.getFullYear(),r.getMonth(),r.getDate()],e._w&&null==e._a[ke]&&null==e._a[ve]&&function(e){var t,n,s,i,r,a,o,u;null!=(t=e._w).GG||null!=t.W||null!=t.E?(r=1,a=4,n=Yt(t.GG,e._a[pe],Ie(Nt(),1,4).year),s=Yt(t.W,1),((i=Yt(t.E,1))<1||7<i)&&(o=!0)):(r=e._locale._week.dow,a=e._locale._week.doy,u=Ie(Nt(),r,a),n=Yt(t.gg,e._a[pe],u.year),s=Yt(t.w,u.week),null!=t.d?((i=t.d)<0||6<i)&&(o=!0):null!=t.e?(i=t.e+r,(t.e<0||6<t.e)&&(o=!0)):i=r);s<1||s>je(n,r,a)?_(e)._overflowWeeks=!0:null!=o?_(e)._overflowWeekday=!0:(a=Ae(n,s,i,r,a),e._a[pe]=a.year,e._dayOfYear=a.dayOfYear)}(e),null!=e._dayOfYear&&(s=Yt(e._a[pe],n[pe]),(e._dayOfYear>Fe(s)||0===e._dayOfYear)&&(_(e)._overflowDayOfYear=!0),s=Ge(s,0,e._dayOfYear),e._a[ve]=s.getUTCMonth(),e._a[ke]=s.getUTCDate()),t=0;t<3&&null==e._a[t];++t)e._a[t]=a[t]=n[t];for(;t<7;t++)e._a[t]=a[t]=null==e._a[t]?2===t?1:0:e._a[t];24===e._a[Me]&&0===e._a[De]&&0===e._a[Se]&&0===e._a[Ye]&&(e._nextDay=!0,e._a[Me]=0),e._d=(e._useUTC?Ge:Ve).apply(null,a),s=e._useUTC?e._d.getUTCDay():e._d.getDay(),null!=e._tzm&&e._d.setUTCMinutes(e._d.getUTCMinutes()-e._tzm),e._nextDay&&(e._a[Me]=24),e._w&&void 0!==e._w.d&&e._w.d!==s&&(_(e).weekdayMismatch=!0)}}function bt(e){if(e._f!==c.ISO_8601)if(e._f!==c.RFC_2822){e._a=[],_(e).empty=!0;for(var t,n,s,i,r,a=""+e._i,o=a.length,u=0,l=H(e._f,e._locale).match(N)||[],h=l.length,d=0;d<h;d++)n=l[d],(t=(a.match(ce(n,e))||[])[0])&&(0<(i=a.substr(0,a.indexOf(t))).length&&_(e).unusedInput.push(i),a=a.slice(a.indexOf(t)+t.length),u+=t.length),W[n]?(t?_(e).empty=!1:_(e).unusedTokens.push(n),s=n,r=e,null!=(i=t)&&f(_e,s)&&_e[s](i,r._a,r,s)):e._strict&&!t&&_(e).unusedTokens.push(n);_(e).charsLeftOver=o-u,0<a.length&&_(e).unusedInput.push(a),e._a[Me]<=12&&!0===_(e).bigHour&&0<e._a[Me]&&(_(e).bigHour=void 0),_(e).parsedDateParts=e._a.slice(0),_(e).meridiem=e._meridiem,e._a[Me]=function(e,t,n){if(null==n)return t;return null!=e.meridiemHour?e.meridiemHour(t,n):null!=e.isPM?((n=e.isPM(n))&&t<12&&(t+=12),t=n||12!==t?t:0):t}(e._locale,e._a[Me],e._meridiem),null!==(o=_(e).era)&&(e._a[pe]=e._locale.erasConvertYear(o,e._a[pe])),Ot(e),ft(e)}else St(e);else Mt(e)}function xt(e){var t=e._i,n=e._f;return e._locale=e._locale||ct(e._l),null===t||void 0===n&&""===t?r({nullInput:!0}):("string"==typeof t&&(e._i=t=e._locale.preparse(t)),k(t)?new v(ft(t)):(s(t)?e._d=t:a(n)?function(e){var t,n,s,i,r,a,o=!1,u=e._f.length;if(0===u)return _(e).invalidFormat=!0,e._d=new Date(NaN);for(i=0;i<u;i++)r=0,a=!1,t=p({},e),null!=e._useUTC&&(t._useUTC=e._useUTC),t._f=e._f[i],bt(t),y(t)&&(a=!0),r+=_(t).charsLeftOver,r+=10*_(t).unusedTokens.length,_(t).score=r,o?r<s&&(s=r,n=t):(null==s||r<s||a)&&(s=r,n=t,a&&(o=!0));d(e,n||t)}(e):n?bt(e):l(n=(t=e)._i)?t._d=new Date(c.now()):s(n)?t._d=new Date(n.valueOf()):"string"==typeof n?function(e){var t=pt.exec(e._i);null===t?(Mt(e),!1===e._isValid&&(delete e._isValid,St(e),!1===e._isValid&&(delete e._isValid,e._strict?e._isValid=!1:c.createFromInputFallback(e)))):e._d=new Date(+t[1])}(t):a(n)?(t._a=i(n.slice(0),function(e){return parseInt(e,10)}),Ot(t)):o(n)?function(e){var t,n;e._d||(n=void 0===(t=G(e._i)).day?t.date:t.day,e._a=i([t.year,t.month,n,t.hour,t.minute,t.second,t.millisecond],function(e){return e&&parseInt(e,10)}),Ot(e))}(t):h(n)?t._d=new Date(n):c.createFromInputFallback(t),y(e)||(e._d=null),e))}function Tt(e,t,n,s,i){var r={};return!0!==t&&!1!==t||(s=t,t=void 0),!0!==n&&!1!==n||(s=n,n=void 0),(o(e)&&u(e)||a(e)&&0===e.length)&&(e=void 0),r._isAMomentObject=!0,r._useUTC=r._isUTC=i,r._l=n,r._i=e,r._f=t,r._strict=s,(r=new v(ft(xt(r=r))))._nextDay&&(r.add(1,"d"),r._nextDay=void 0),r}function Nt(e,t,n,s){return Tt(e,t,n,s,!1)}c.createFromInputFallback=n("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",function(e){e._d=new Date(e._i+(e._useUTC?" UTC":""))}),c.ISO_8601=function(){},c.RFC_2822=function(){};var Pt=n("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var e=Nt.apply(null,arguments);return this.isValid()&&e.isValid()?e<this?this:e:r()}),Rt=n("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var e=Nt.apply(null,arguments);return this.isValid()&&e.isValid()?this<e?this:e:r()});function Wt(e,t){var n,s;if(!(t=1===t.length&&a(t[0])?t[0]:t).length)return Nt();for(n=t[0],s=1;s<t.length;++s)t[s].isValid()&&!t[s][e](n)||(n=t[s]);return n}var Ct=["year","quarter","month","week","day","hour","minute","second","millisecond"];function Ut(e){var t=G(e),n=t.year||0,s=t.quarter||0,i=t.month||0,r=t.week||t.isoWeek||0,a=t.day||0,o=t.hour||0,u=t.minute||0,l=t.second||0,e=t.millisecond||0;this._isValid=function(e){var t,n,s=!1,i=Ct.length;for(t in e)if(f(e,t)&&(-1===we.call(Ct,t)||null!=e[t]&&isNaN(e[t])))return!1;for(n=0;n<i;++n)if(e[Ct[n]]){if(s)return!1;parseFloat(e[Ct[n]])!==Z(e[Ct[n]])&&(s=!0)}return!0}(t),this._milliseconds=+e+1e3*l+6e4*u+1e3*o*60*60,this._days=+a+7*r,this._months=+i+3*s+12*n,this._data={},this._locale=ct(),this._bubble()}function Ht(e){return e instanceof Ut}function Ft(e){return e<0?-1*Math.round(-1*e):Math.round(e)}function Lt(e,n){C(e,0,0,function(){var e=this.utcOffset(),t="+";return e<0&&(e=-e,t="-"),t+T(~~(e/60),2)+n+T(~~e%60,2)})}Lt("Z",":"),Lt("ZZ",""),de("Z",le),de("ZZ",le),ye(["Z","ZZ"],function(e,t,n){n._useUTC=!0,n._tzm=Gt(le,e)});var Vt=/([\+\-]|\d\d)/gi;function Gt(e,t){t=(t||"").match(e);return null===t?null:0===(t=60*(e=((t[t.length-1]||[])+"").match(Vt)||["-",0,0])[1]+Z(e[2]))?0:"+"===e[0]?t:-t}function Et(e,t){var n;return t._isUTC?(n=t.clone(),t=(k(e)||s(e)?e:Nt(e)).valueOf()-n.valueOf(),n._d.setTime(n._d.valueOf()+t),c.updateOffset(n,!1),n):Nt(e).local()}function At(e){return-Math.round(e._d.getTimezoneOffset())}function It(){return!!this.isValid()&&(this._isUTC&&0===this._offset)}c.updateOffset=function(){};var jt=/^(-|\+)?(?:(\d*)[. ])?(\d+):(\d+)(?::(\d+)(\.\d*)?)?$/,Zt=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;function zt(e,t){var n,s=e,i=null;return Ht(e)?s={ms:e._milliseconds,d:e._days,M:e._months}:h(e)||!isNaN(+e)?(s={},t?s[t]=+e:s.milliseconds=+e):(i=jt.exec(e))?(n="-"===i[1]?-1:1,s={y:0,d:Z(i[ke])*n,h:Z(i[Me])*n,m:Z(i[De])*n,s:Z(i[Se])*n,ms:Z(Ft(1e3*i[Ye]))*n}):(i=Zt.exec(e))?(n="-"===i[1]?-1:1,s={y:$t(i[2],n),M:$t(i[3],n),w:$t(i[4],n),d:$t(i[5],n),h:$t(i[6],n),m:$t(i[7],n),s:$t(i[8],n)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=function(e,t){var n;if(!e.isValid()||!t.isValid())return{milliseconds:0,months:0};t=Et(t,e),e.isBefore(t)?n=qt(e,t):((n=qt(t,e)).milliseconds=-n.milliseconds,n.months=-n.months);return n}(Nt(s.from),Nt(s.to)),(s={}).ms=n.milliseconds,s.M=n.months),s=new Ut(s),Ht(e)&&f(e,"_locale")&&(s._locale=e._locale),Ht(e)&&f(e,"_isValid")&&(s._isValid=e._isValid),s}function $t(e,t){e=e&&parseFloat(e.replace(",","."));return(isNaN(e)?0:e)*t}function qt(e,t){var n={};return n.months=t.month()-e.month()+12*(t.year()-e.year()),e.clone().add(n.months,"M").isAfter(t)&&--n.months,n.milliseconds=+t-+e.clone().add(n.months,"M"),n}function Bt(s,i){return function(e,t){var n;return null===t||isNaN(+t)||(S(i,"moment()."+i+"(period, number) is deprecated. Please use moment()."+i+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),n=e,e=t,t=n),Jt(this,zt(e,t),s),this}}function Jt(e,t,n,s){var i=t._milliseconds,r=Ft(t._days),t=Ft(t._months);e.isValid()&&(s=null==s||s,t&&Ce(e,$(e,"Month")+t*n),r&&q(e,"Date",$(e,"Date")+r*n),i&&e._d.setTime(e._d.valueOf()+i*n),s&&c.updateOffset(e,r||t))}zt.fn=Ut.prototype,zt.invalid=function(){return zt(NaN)};var Qt=Bt(1,"add"),he=Bt(-1,"subtract");function Xt(e){return"string"==typeof e||e instanceof String}function Kt(e){return k(e)||s(e)||Xt(e)||h(e)||function(t){var e=a(t),n=!1;e&&(n=0===t.filter(function(e){return!h(e)&&Xt(t)}).length);return e&&n}(e)||function(e){var t,n,s=o(e)&&!u(e),i=!1,r=["years","year","y","months","month","M","days","day","d","dates","date","D","hours","hour","h","minutes","minute","m","seconds","second","s","milliseconds","millisecond","ms"],a=r.length;for(t=0;t<a;t+=1)n=r[t],i=i||f(e,n);return s&&i}(e)||null==e}function en(e,t){if(e.date()<t.date())return-en(t,e);var n=12*(t.year()-e.year())+(t.month()-e.month()),s=e.clone().add(n,"months"),s=t-s<0?(t-s)/(s-e.clone().add(n-1,"months")):(t-s)/(e.clone().add(1+n,"months")-s);return-(n+s)||0}function tn(e){return void 0===e?this._locale._abbr:(null!=(e=ct(e))&&(this._locale=e),this)}c.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",c.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";te=n("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(e){return void 0===e?this.localeData():this.locale(e)});function nn(){return this._locale}var sn=126227808e5;function rn(e,t){return(e%t+t)%t}function an(e,t,n){return e<100&&0<=e?new Date(e+400,t,n)-sn:new Date(e,t,n).valueOf()}function on(e,t,n){return e<100&&0<=e?Date.UTC(e+400,t,n)-sn:Date.UTC(e,t,n)}function un(e,t){return t.erasAbbrRegex(e)}function ln(){for(var e=[],t=[],n=[],s=[],i=this.eras(),r=0,a=i.length;r<a;++r)t.push(fe(i[r].name)),e.push(fe(i[r].abbr)),n.push(fe(i[r].narrow)),s.push(fe(i[r].name)),s.push(fe(i[r].abbr)),s.push(fe(i[r].narrow));this._erasRegex=new RegExp("^("+s.join("|")+")","i"),this._erasNameRegex=new RegExp("^("+t.join("|")+")","i"),this._erasAbbrRegex=new RegExp("^("+e.join("|")+")","i"),this._erasNarrowRegex=new RegExp("^("+n.join("|")+")","i")}function hn(e,t){C(0,[e,e.length],0,t)}function dn(e,t,n,s,i){var r;return null==e?Ie(this,s,i).year:(r=je(e,s,i),function(e,t,n,s,i){i=Ae(e,t,n,s,i),i=Ge(i.year,0,i.dayOfYear);return this.year(i.getUTCFullYear()),this.month(i.getUTCMonth()),this.date(i.getUTCDate()),this}.call(this,e,t=r<t?r:t,n,s,i))}C("N",0,0,"eraAbbr"),C("NN",0,0,"eraAbbr"),C("NNN",0,0,"eraAbbr"),C("NNNN",0,0,"eraName"),C("NNNNN",0,0,"eraNarrow"),C("y",["y",1],"yo","eraYear"),C("y",["yy",2],0,"eraYear"),C("y",["yyy",3],0,"eraYear"),C("y",["yyyy",4],0,"eraYear"),de("N",un),de("NN",un),de("NNN",un),de("NNNN",function(e,t){return t.erasNameRegex(e)}),de("NNNNN",function(e,t){return t.erasNarrowRegex(e)}),ye(["N","NN","NNN","NNNN","NNNNN"],function(e,t,n,s){s=n._locale.erasParse(e,s,n._strict);s?_(n).era=s:_(n).invalidEra=e}),de("y",ae),de("yy",ae),de("yyy",ae),de("yyyy",ae),de("yo",function(e,t){return t._eraYearOrdinalRegex||ae}),ye(["y","yy","yyy","yyyy"],pe),ye(["yo"],function(e,t,n,s){var i;n._locale._eraYearOrdinalRegex&&(i=e.match(n._locale._eraYearOrdinalRegex)),n._locale.eraYearOrdinalParse?t[pe]=n._locale.eraYearOrdinalParse(e,i):t[pe]=parseInt(e,10)}),C(0,["gg",2],0,function(){return this.weekYear()%100}),C(0,["GG",2],0,function(){return this.isoWeekYear()%100}),hn("gggg","weekYear"),hn("ggggg","weekYear"),hn("GGGG","isoWeekYear"),hn("GGGGG","isoWeekYear"),L("weekYear","gg"),L("isoWeekYear","GG"),A("weekYear",1),A("isoWeekYear",1),de("G",oe),de("g",oe),de("GG",ee,J),de("gg",ee,J),de("GGGG",ie,X),de("gggg",ie,X),de("GGGGG",re,K),de("ggggg",re,K),ge(["gggg","ggggg","GGGG","GGGGG"],function(e,t,n,s){t[s.substr(0,2)]=Z(e)}),ge(["gg","GG"],function(e,t,n,s){t[s]=c.parseTwoDigitYear(e)}),C("Q",0,"Qo","quarter"),L("quarter","Q"),A("quarter",7),de("Q",B),ye("Q",function(e,t){t[ve]=3*(Z(e)-1)}),C("D",["DD",2],"Do","date"),L("date","D"),A("date",9),de("D",ee),de("DD",ee,J),de("Do",function(e,t){return e?t._dayOfMonthOrdinalParse||t._ordinalParse:t._dayOfMonthOrdinalParseLenient}),ye(["D","DD"],ke),ye("Do",function(e,t){t[ke]=Z(e.match(ee)[0])});ne=z("Date",!0);C("DDD",["DDDD",3],"DDDo","dayOfYear"),L("dayOfYear","DDD"),A("dayOfYear",4),de("DDD",se),de("DDDD",Q),ye(["DDD","DDDD"],function(e,t,n){n._dayOfYear=Z(e)}),C("m",["mm",2],0,"minute"),L("minute","m"),A("minute",14),de("m",ee),de("mm",ee,J),ye(["m","mm"],De);Te=z("Minutes",!1);C("s",["ss",2],0,"second"),L("second","s"),A("second",15),de("s",ee),de("ss",ee,J),ye(["s","ss"],Se);var cn,ze=z("Seconds",!1);for(C("S",0,0,function(){return~~(this.millisecond()/100)}),C(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),C(0,["SSS",3],0,"millisecond"),C(0,["SSSS",4],0,function(){return 10*this.millisecond()}),C(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),C(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),C(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),C(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),C(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),L("millisecond","ms"),A("millisecond",16),de("S",se,B),de("SS",se,J),de("SSS",se,Q),cn="SSSS";cn.length<=9;cn+="S")de(cn,ae);function fn(e,t){t[Ye]=Z(1e3*("0."+e))}for(cn="S";cn.length<=9;cn+="S")ye(cn,fn);qe=z("Milliseconds",!1),C("z",0,0,"zoneAbbr"),C("zz",0,0,"zoneName");ie=v.prototype;function mn(e){return e}ie.add=Qt,ie.calendar=function(e,t){1===arguments.length&&(arguments[0]?Kt(arguments[0])?(e=arguments[0],t=void 0):function(e){for(var t=o(e)&&!u(e),n=!1,s=["sameDay","nextDay","lastDay","nextWeek","lastWeek","sameElse"],i=0;i<s.length;i+=1)n=n||f(e,s[i]);return t&&n}(arguments[0])&&(t=arguments[0],e=void 0):t=e=void 0);var n=e||Nt(),e=Et(n,this).startOf("day"),e=c.calendarFormat(this,e)||"sameElse",t=t&&(Y(t[e])?t[e].call(this,n):t[e]);return this.format(t||this.localeData().calendar(e,this,Nt(n)))},ie.clone=function(){return new v(this)},ie.diff=function(e,t,n){var s,i,r;if(!this.isValid())return NaN;if(!(s=Et(e,this)).isValid())return NaN;switch(i=6e4*(s.utcOffset()-this.utcOffset()),t=V(t)){case"year":r=en(this,s)/12;break;case"month":r=en(this,s);break;case"quarter":r=en(this,s)/3;break;case"second":r=(this-s)/1e3;break;case"minute":r=(this-s)/6e4;break;case"hour":r=(this-s)/36e5;break;case"day":r=(this-s-i)/864e5;break;case"week":r=(this-s-i)/6048e5;break;default:r=this-s}return n?r:j(r)},ie.endOf=function(e){var t,n;if(void 0===(e=V(e))||"millisecond"===e||!this.isValid())return this;switch(n=this._isUTC?on:an,e){case"year":t=n(this.year()+1,0,1)-1;break;case"quarter":t=n(this.year(),this.month()-this.month()%3+3,1)-1;break;case"month":t=n(this.year(),this.month()+1,1)-1;break;case"week":t=n(this.year(),this.month(),this.date()-this.weekday()+7)-1;break;case"isoWeek":t=n(this.year(),this.month(),this.date()-(this.isoWeekday()-1)+7)-1;break;case"day":case"date":t=n(this.year(),this.month(),this.date()+1)-1;break;case"hour":t=this._d.valueOf(),t+=36e5-rn(t+(this._isUTC?0:6e4*this.utcOffset()),36e5)-1;break;case"minute":t=this._d.valueOf(),t+=6e4-rn(t,6e4)-1;break;case"second":t=this._d.valueOf(),t+=1e3-rn(t,1e3)-1}return this._d.setTime(t),c.updateOffset(this,!0),this},ie.format=function(e){return e=e||(this.isUtc()?c.defaultFormatUtc:c.defaultFormat),e=U(this,e),this.localeData().postformat(e)},ie.from=function(e,t){return this.isValid()&&(k(e)&&e.isValid()||Nt(e).isValid())?zt({to:this,from:e}).locale(this.locale()).humanize(!t):this.localeData().invalidDate()},ie.fromNow=function(e){return this.from(Nt(),e)},ie.to=function(e,t){return this.isValid()&&(k(e)&&e.isValid()||Nt(e).isValid())?zt({from:this,to:e}).locale(this.locale()).humanize(!t):this.localeData().invalidDate()},ie.toNow=function(e){return this.to(Nt(),e)},ie.get=function(e){return Y(this[e=V(e)])?this[e]():this},ie.invalidAt=function(){return _(this).overflow},ie.isAfter=function(e,t){return e=k(e)?e:Nt(e),!(!this.isValid()||!e.isValid())&&("millisecond"===(t=V(t)||"millisecond")?this.valueOf()>e.valueOf():e.valueOf()<this.clone().startOf(t).valueOf())},ie.isBefore=function(e,t){return e=k(e)?e:Nt(e),!(!this.isValid()||!e.isValid())&&("millisecond"===(t=V(t)||"millisecond")?this.valueOf()<e.valueOf():this.clone().endOf(t).valueOf()<e.valueOf())},ie.isBetween=function(e,t,n,s){return e=k(e)?e:Nt(e),t=k(t)?t:Nt(t),!!(this.isValid()&&e.isValid()&&t.isValid())&&(("("===(s=s||"()")[0]?this.isAfter(e,n):!this.isBefore(e,n))&&(")"===s[1]?this.isBefore(t,n):!this.isAfter(t,n)))},ie.isSame=function(e,t){return e=k(e)?e:Nt(e),!(!this.isValid()||!e.isValid())&&("millisecond"===(t=V(t)||"millisecond")?this.valueOf()===e.valueOf():(e=e.valueOf(),this.clone().startOf(t).valueOf()<=e&&e<=this.clone().endOf(t).valueOf()))},ie.isSameOrAfter=function(e,t){return this.isSame(e,t)||this.isAfter(e,t)},ie.isSameOrBefore=function(e,t){return this.isSame(e,t)||this.isBefore(e,t)},ie.isValid=function(){return y(this)},ie.lang=te,ie.locale=tn,ie.localeData=nn,ie.max=Rt,ie.min=Pt,ie.parsingFlags=function(){return d({},_(this))},ie.set=function(e,t){if("object"==typeof e)for(var n=function(e){var t,n=[];for(t in e)f(e,t)&&n.push({unit:t,priority:E[t]});return n.sort(function(e,t){return e.priority-t.priority}),n}(e=G(e)),s=n.length,i=0;i<s;i++)this[n[i].unit](e[n[i].unit]);else if(Y(this[e=V(e)]))return this[e](t);return this},ie.startOf=function(e){var t,n;if(void 0===(e=V(e))||"millisecond"===e||!this.isValid())return this;switch(n=this._isUTC?on:an,e){case"year":t=n(this.year(),0,1);break;case"quarter":t=n(this.year(),this.month()-this.month()%3,1);break;case"month":t=n(this.year(),this.month(),1);break;case"week":t=n(this.year(),this.month(),this.date()-this.weekday());break;case"isoWeek":t=n(this.year(),this.month(),this.date()-(this.isoWeekday()-1));break;case"day":case"date":t=n(this.year(),this.month(),this.date());break;case"hour":t=this._d.valueOf(),t-=rn(t+(this._isUTC?0:6e4*this.utcOffset()),36e5);break;case"minute":t=this._d.valueOf(),t-=rn(t,6e4);break;case"second":t=this._d.valueOf(),t-=rn(t,1e3)}return this._d.setTime(t),c.updateOffset(this,!0),this},ie.subtract=he,ie.toArray=function(){var e=this;return[e.year(),e.month(),e.date(),e.hour(),e.minute(),e.second(),e.millisecond()]},ie.toObject=function(){var e=this;return{years:e.year(),months:e.month(),date:e.date(),hours:e.hours(),minutes:e.minutes(),seconds:e.seconds(),milliseconds:e.milliseconds()}},ie.toDate=function(){return new Date(this.valueOf())},ie.toISOString=function(e){if(!this.isValid())return null;var t=!0!==e;return(e=t?this.clone().utc():this).year()<0||9999<e.year()?U(e,t?"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYYYY-MM-DD[T]HH:mm:ss.SSSZ"):Y(Date.prototype.toISOString)?t?this.toDate().toISOString():new Date(this.valueOf()+60*this.utcOffset()*1e3).toISOString().replace("Z",U(e,"Z")):U(e,t?"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYY-MM-DD[T]HH:mm:ss.SSSZ")},ie.inspect=function(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var e,t="moment",n="";return this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",n="Z"),e="["+t+'("]',t=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",this.format(e+t+"-MM-DD[T]HH:mm:ss.SSS"+(n+'[")]'))},"undefined"!=typeof Symbol&&null!=Symbol.for&&(ie[Symbol.for("nodejs.util.inspect.custom")]=function(){return"Moment<"+this.format()+">"}),ie.toJSON=function(){return this.isValid()?this.toISOString():null},ie.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},ie.unix=function(){return Math.floor(this.valueOf()/1e3)},ie.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},ie.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},ie.eraName=function(){for(var e,t=this.localeData().eras(),n=0,s=t.length;n<s;++n){if(e=this.clone().startOf("day").valueOf(),t[n].since<=e&&e<=t[n].until)return t[n].name;if(t[n].until<=e&&e<=t[n].since)return t[n].name}return""},ie.eraNarrow=function(){for(var e,t=this.localeData().eras(),n=0,s=t.length;n<s;++n){if(e=this.clone().startOf("day").valueOf(),t[n].since<=e&&e<=t[n].until)return t[n].narrow;if(t[n].until<=e&&e<=t[n].since)return t[n].narrow}return""},ie.eraAbbr=function(){for(var e,t=this.localeData().eras(),n=0,s=t.length;n<s;++n){if(e=this.clone().startOf("day").valueOf(),t[n].since<=e&&e<=t[n].until)return t[n].abbr;if(t[n].until<=e&&e<=t[n].since)return t[n].abbr}return""},ie.eraYear=function(){for(var e,t,n=this.localeData().eras(),s=0,i=n.length;s<i;++s)if(e=n[s].since<=n[s].until?1:-1,t=this.clone().startOf("day").valueOf(),n[s].since<=t&&t<=n[s].until||n[s].until<=t&&t<=n[s].since)return(this.year()-c(n[s].since).year())*e+n[s].offset;return this.year()},ie.year=Le,ie.isLeapYear=function(){return I(this.year())},ie.weekYear=function(e){return dn.call(this,e,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)},ie.isoWeekYear=function(e){return dn.call(this,e,this.isoWeek(),this.isoWeekday(),1,4)},ie.quarter=ie.quarters=function(e){return null==e?Math.ceil((this.month()+1)/3):this.month(3*(e-1)+this.month()%3)},ie.month=Ue,ie.daysInMonth=function(){return xe(this.year(),this.month())},ie.week=ie.weeks=function(e){var t=this.localeData().week(this);return null==e?t:this.add(7*(e-t),"d")},ie.isoWeek=ie.isoWeeks=function(e){var t=Ie(this,1,4).week;return null==e?t:this.add(7*(e-t),"d")},ie.weeksInYear=function(){var e=this.localeData()._week;return je(this.year(),e.dow,e.doy)},ie.weeksInWeekYear=function(){var e=this.localeData()._week;return je(this.weekYear(),e.dow,e.doy)},ie.isoWeeksInYear=function(){return je(this.year(),1,4)},ie.isoWeeksInISOWeekYear=function(){return je(this.isoWeekYear(),1,4)},ie.date=ne,ie.day=ie.days=function(e){if(!this.isValid())return null!=e?this:NaN;var t,n,s=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=e?(t=e,n=this.localeData(),e="string"!=typeof t?t:isNaN(t)?"number"==typeof(t=n.weekdaysParse(t))?t:null:parseInt(t,10),this.add(e-s,"d")):s},ie.weekday=function(e){if(!this.isValid())return null!=e?this:NaN;var t=(this.day()+7-this.localeData()._week.dow)%7;return null==e?t:this.add(e-t,"d")},ie.isoWeekday=function(e){if(!this.isValid())return null!=e?this:NaN;if(null==e)return this.day()||7;var t=(t=e,e=this.localeData(),"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t);return this.day(this.day()%7?t:t-7)},ie.dayOfYear=function(e){var t=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==e?t:this.add(e-t,"d")},ie.hour=ie.hours=nt,ie.minute=ie.minutes=Te,ie.second=ie.seconds=ze,ie.millisecond=ie.milliseconds=qe,ie.utcOffset=function(e,t,n){var s,i=this._offset||0;if(!this.isValid())return null!=e?this:NaN;if(null==e)return this._isUTC?i:At(this);if("string"==typeof e){if(null===(e=Gt(le,e)))return this}else Math.abs(e)<16&&!n&&(e*=60);return!this._isUTC&&t&&(s=At(this)),this._offset=e,this._isUTC=!0,null!=s&&this.add(s,"m"),i!==e&&(!t||this._changeInProgress?Jt(this,zt(e-i,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,c.updateOffset(this,!0),this._changeInProgress=null)),this},ie.utc=function(e){return this.utcOffset(0,e)},ie.local=function(e){return this._isUTC&&(this.utcOffset(0,e),this._isUTC=!1,e&&this.subtract(At(this),"m")),this},ie.parseZone=function(){var e;return null!=this._tzm?this.utcOffset(this._tzm,!1,!0):"string"==typeof this._i&&(null!=(e=Gt(ue,this._i))?this.utcOffset(e):this.utcOffset(0,!0)),this},ie.hasAlignedHourOffset=function(e){return!!this.isValid()&&(e=e?Nt(e).utcOffset():0,(this.utcOffset()-e)%60==0)},ie.isDST=function(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},ie.isLocal=function(){return!!this.isValid()&&!this._isUTC},ie.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},ie.isUtc=It,ie.isUTC=It,ie.zoneAbbr=function(){return this._isUTC?"UTC":""},ie.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},ie.dates=n("dates accessor is deprecated. Use date instead.",ne),ie.months=n("months accessor is deprecated. Use month instead",Ue),ie.years=n("years accessor is deprecated. Use year instead",Le),ie.zone=n("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",function(e,t){return null!=e?(this.utcOffset(e="string"!=typeof e?-e:e,t),this):-this.utcOffset()}),ie.isDSTShifted=n("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",function(){if(!l(this._isDSTShifted))return this._isDSTShifted;var e,t={};return p(t,this),(t=xt(t))._a?(e=(t._isUTC?m:Nt)(t._a),this._isDSTShifted=this.isValid()&&0<function(e,t,n){for(var s=Math.min(e.length,t.length),i=Math.abs(e.length-t.length),r=0,a=0;a<s;a++)(n&&e[a]!==t[a]||!n&&Z(e[a])!==Z(t[a]))&&r++;return r+i}(t._a,e.toArray())):this._isDSTShifted=!1,this._isDSTShifted});X=b.prototype;function _n(e,t,n,s){var i=ct(),t=m().set(s,t);return i[n](t,e)}function yn(e,t,n){if(h(e)&&(t=e,e=void 0),e=e||"",null!=t)return _n(e,t,n,"month");for(var s=[],i=0;i<12;i++)s[i]=_n(e,i,n,"month");return s}function gn(e,t,n,s){t=("boolean"==typeof e?h(t)&&(n=t,t=void 0):(t=e,e=!1,h(n=t)&&(n=t,t=void 0)),t||"");var i,r=ct(),a=e?r._week.dow:0,o=[];if(null!=n)return _n(t,(n+a)%7,s,"day");for(i=0;i<7;i++)o[i]=_n(t,(i+a)%7,s,"day");return o}X.calendar=function(e,t,n){return Y(e=this._calendar[e]||this._calendar.sameElse)?e.call(t,n):e},X.longDateFormat=function(e){var t=this._longDateFormat[e],n=this._longDateFormat[e.toUpperCase()];return t||!n?t:(this._longDateFormat[e]=n.match(N).map(function(e){return"MMMM"===e||"MM"===e||"DD"===e||"dddd"===e?e.slice(1):e}).join(""),this._longDateFormat[e])},X.invalidDate=function(){return this._invalidDate},X.ordinal=function(e){return this._ordinal.replace("%d",e)},X.preparse=mn,X.postformat=mn,X.relativeTime=function(e,t,n,s){var i=this._relativeTime[n];return Y(i)?i(e,t,n,s):i.replace(/%d/i,e)},X.pastFuture=function(e,t){return Y(e=this._relativeTime[0<e?"future":"past"])?e(t):e.replace(/%s/i,t)},X.set=function(e){var t,n;for(n in e)f(e,n)&&(Y(t=e[n])?this[n]=t:this["_"+n]=t);this._config=e,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)},X.eras=function(e,t){for(var n,s=this._eras||ct("en")._eras,i=0,r=s.length;i<r;++i)switch("string"==typeof s[i].since&&(n=c(s[i].since).startOf("day"),s[i].since=n.valueOf()),typeof s[i].until){case"undefined":s[i].until=1/0;break;case"string":n=c(s[i].until).startOf("day").valueOf(),s[i].until=n.valueOf()}return s},X.erasParse=function(e,t,n){var s,i,r,a,o,u=this.eras();for(e=e.toUpperCase(),s=0,i=u.length;s<i;++s)if(r=u[s].name.toUpperCase(),a=u[s].abbr.toUpperCase(),o=u[s].narrow.toUpperCase(),n)switch(t){case"N":case"NN":case"NNN":if(a===e)return u[s];break;case"NNNN":if(r===e)return u[s];break;case"NNNNN":if(o===e)return u[s]}else if(0<=[r,a,o].indexOf(e))return u[s]},X.erasConvertYear=function(e,t){var n=e.since<=e.until?1:-1;return void 0===t?c(e.since).year():c(e.since).year()+(t-e.offset)*n},X.erasAbbrRegex=function(e){return f(this,"_erasAbbrRegex")||ln.call(this),e?this._erasAbbrRegex:this._erasRegex},X.erasNameRegex=function(e){return f(this,"_erasNameRegex")||ln.call(this),e?this._erasNameRegex:this._erasRegex},X.erasNarrowRegex=function(e){return f(this,"_erasNarrowRegex")||ln.call(this),e?this._erasNarrowRegex:this._erasRegex},X.months=function(e,t){return e?(a(this._months)?this._months:this._months[(this._months.isFormat||Pe).test(t)?"format":"standalone"])[e.month()]:a(this._months)?this._months:this._months.standalone},X.monthsShort=function(e,t){return e?(a(this._monthsShort)?this._monthsShort:this._monthsShort[Pe.test(t)?"format":"standalone"])[e.month()]:a(this._monthsShort)?this._monthsShort:this._monthsShort.standalone},X.monthsParse=function(e,t,n){var s,i;if(this._monthsParseExact)return function(e,t,n){var s,i,r,e=e.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],s=0;s<12;++s)r=m([2e3,s]),this._shortMonthsParse[s]=this.monthsShort(r,"").toLocaleLowerCase(),this._longMonthsParse[s]=this.months(r,"").toLocaleLowerCase();return n?"MMM"===t?-1!==(i=we.call(this._shortMonthsParse,e))?i:null:-1!==(i=we.call(this._longMonthsParse,e))?i:null:"MMM"===t?-1!==(i=we.call(this._shortMonthsParse,e))||-1!==(i=we.call(this._longMonthsParse,e))?i:null:-1!==(i=we.call(this._longMonthsParse,e))||-1!==(i=we.call(this._shortMonthsParse,e))?i:null}.call(this,e,t,n);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),s=0;s<12;s++){if(i=m([2e3,s]),n&&!this._longMonthsParse[s]&&(this._longMonthsParse[s]=new RegExp("^"+this.months(i,"").replace(".","")+"$","i"),this._shortMonthsParse[s]=new RegExp("^"+this.monthsShort(i,"").replace(".","")+"$","i")),n||this._monthsParse[s]||(i="^"+this.months(i,"")+"|^"+this.monthsShort(i,""),this._monthsParse[s]=new RegExp(i.replace(".",""),"i")),n&&"MMMM"===t&&this._longMonthsParse[s].test(e))return s;if(n&&"MMM"===t&&this._shortMonthsParse[s].test(e))return s;if(!n&&this._monthsParse[s].test(e))return s}},X.monthsRegex=function(e){return this._monthsParseExact?(f(this,"_monthsRegex")||He.call(this),e?this._monthsStrictRegex:this._monthsRegex):(f(this,"_monthsRegex")||(this._monthsRegex=We),this._monthsStrictRegex&&e?this._monthsStrictRegex:this._monthsRegex)},X.monthsShortRegex=function(e){return this._monthsParseExact?(f(this,"_monthsRegex")||He.call(this),e?this._monthsShortStrictRegex:this._monthsShortRegex):(f(this,"_monthsShortRegex")||(this._monthsShortRegex=Re),this._monthsShortStrictRegex&&e?this._monthsShortStrictRegex:this._monthsShortRegex)},X.week=function(e){return Ie(e,this._week.dow,this._week.doy).week},X.firstDayOfYear=function(){return this._week.doy},X.firstDayOfWeek=function(){return this._week.dow},X.weekdays=function(e,t){return t=a(this._weekdays)?this._weekdays:this._weekdays[e&&!0!==e&&this._weekdays.isFormat.test(t)?"format":"standalone"],!0===e?Ze(t,this._week.dow):e?t[e.day()]:t},X.weekdaysMin=function(e){return!0===e?Ze(this._weekdaysMin,this._week.dow):e?this._weekdaysMin[e.day()]:this._weekdaysMin},X.weekdaysShort=function(e){return!0===e?Ze(this._weekdaysShort,this._week.dow):e?this._weekdaysShort[e.day()]:this._weekdaysShort},X.weekdaysParse=function(e,t,n){var s,i;if(this._weekdaysParseExact)return function(e,t,n){var s,i,r,e=e.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],s=0;s<7;++s)r=m([2e3,1]).day(s),this._minWeekdaysParse[s]=this.weekdaysMin(r,"").toLocaleLowerCase(),this._shortWeekdaysParse[s]=this.weekdaysShort(r,"").toLocaleLowerCase(),this._weekdaysParse[s]=this.weekdays(r,"").toLocaleLowerCase();return n?"dddd"===t?-1!==(i=we.call(this._weekdaysParse,e))?i:null:"ddd"===t?-1!==(i=we.call(this._shortWeekdaysParse,e))?i:null:-1!==(i=we.call(this._minWeekdaysParse,e))?i:null:"dddd"===t?-1!==(i=we.call(this._weekdaysParse,e))||-1!==(i=we.call(this._shortWeekdaysParse,e))||-1!==(i=we.call(this._minWeekdaysParse,e))?i:null:"ddd"===t?-1!==(i=we.call(this._shortWeekdaysParse,e))||-1!==(i=we.call(this._weekdaysParse,e))||-1!==(i=we.call(this._minWeekdaysParse,e))?i:null:-1!==(i=we.call(this._minWeekdaysParse,e))||-1!==(i=we.call(this._weekdaysParse,e))||-1!==(i=we.call(this._shortWeekdaysParse,e))?i:null}.call(this,e,t,n);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),s=0;s<7;s++){if(i=m([2e3,1]).day(s),n&&!this._fullWeekdaysParse[s]&&(this._fullWeekdaysParse[s]=new RegExp("^"+this.weekdays(i,"").replace(".","\\.?")+"$","i"),this._shortWeekdaysParse[s]=new RegExp("^"+this.weekdaysShort(i,"").replace(".","\\.?")+"$","i"),this._minWeekdaysParse[s]=new RegExp("^"+this.weekdaysMin(i,"").replace(".","\\.?")+"$","i")),this._weekdaysParse[s]||(i="^"+this.weekdays(i,"")+"|^"+this.weekdaysShort(i,"")+"|^"+this.weekdaysMin(i,""),this._weekdaysParse[s]=new RegExp(i.replace(".",""),"i")),n&&"dddd"===t&&this._fullWeekdaysParse[s].test(e))return s;if(n&&"ddd"===t&&this._shortWeekdaysParse[s].test(e))return s;if(n&&"dd"===t&&this._minWeekdaysParse[s].test(e))return s;if(!n&&this._weekdaysParse[s].test(e))return s}},X.weekdaysRegex=function(e){return this._weekdaysParseExact?(f(this,"_weekdaysRegex")||Xe.call(this),e?this._weekdaysStrictRegex:this._weekdaysRegex):(f(this,"_weekdaysRegex")||(this._weekdaysRegex=Be),this._weekdaysStrictRegex&&e?this._weekdaysStrictRegex:this._weekdaysRegex)},X.weekdaysShortRegex=function(e){return this._weekdaysParseExact?(f(this,"_weekdaysRegex")||Xe.call(this),e?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(f(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=Je),this._weekdaysShortStrictRegex&&e?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)},X.weekdaysMinRegex=function(e){return this._weekdaysParseExact?(f(this,"_weekdaysRegex")||Xe.call(this),e?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(f(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=Qe),this._weekdaysMinStrictRegex&&e?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)},X.isPM=function(e){return"p"===(e+"").toLowerCase().charAt(0)},X.meridiem=function(e,t,n){return 11<e?n?"pm":"PM":n?"am":"AM"},ht("en",{eras:[{since:"0001-01-01",until:1/0,offset:1,name:"Anno Domini",narrow:"AD",abbr:"AD"},{since:"0000-12-31",until:-1/0,offset:1,name:"Before Christ",narrow:"BC",abbr:"BC"}],dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(e){var t=e%10;return e+(1===Z(e%100/10)?"th":1==t?"st":2==t?"nd":3==t?"rd":"th")}}),c.lang=n("moment.lang is deprecated. Use moment.locale instead.",ht),c.langData=n("moment.langData is deprecated. Use moment.localeData instead.",ct);var wn=Math.abs;function pn(e,t,n,s){n=zt(t,n);return e._milliseconds+=s*n._milliseconds,e._days+=s*n._days,e._months+=s*n._months,e._bubble()}function vn(e){return e<0?Math.floor(e):Math.ceil(e)}function kn(e){return 4800*e/146097}function Mn(e){return 146097*e/4800}function Dn(e){return function(){return this.as(e)}}re=Dn("ms"),K=Dn("s"),B=Dn("m"),J=Dn("h"),se=Dn("d"),Q=Dn("w"),Qt=Dn("M"),Rt=Dn("Q"),Pt=Dn("y");function Sn(e){return function(){return this.isValid()?this._data[e]:NaN}}he=Sn("milliseconds"),nt=Sn("seconds"),Te=Sn("minutes"),ze=Sn("hours"),qe=Sn("days"),ne=Sn("months"),Le=Sn("years");var Yn=Math.round,On={ss:44,s:45,m:45,h:22,d:26,w:null,M:11};function bn(e,t,n,s){var i=zt(e).abs(),r=Yn(i.as("s")),a=Yn(i.as("m")),o=Yn(i.as("h")),u=Yn(i.as("d")),l=Yn(i.as("M")),h=Yn(i.as("w")),i=Yn(i.as("y")),u=(r<=n.ss?["s",r]:r<n.s&&["ss",r])||a<=1&&["m"]||a<n.m&&["mm",a]||o<=1&&["h"]||o<n.h&&["hh",o]||u<=1&&["d"]||u<n.d&&["dd",u];return(u=(u=null!=n.w?u||h<=1&&["w"]||h<n.w&&["ww",h]:u)||l<=1&&["M"]||l<n.M&&["MM",l]||i<=1&&["y"]||["yy",i])[2]=t,u[3]=0<+e,u[4]=s,function(e,t,n,s,i){return i.relativeTime(t||1,!!n,e,s)}.apply(null,u)}var xn=Math.abs;function Tn(e){return(0<e)-(e<0)||+e}function Nn(){if(!this.isValid())return this.localeData().invalidDate();var e,t,n,s,i,r,a,o=xn(this._milliseconds)/1e3,u=xn(this._days),l=xn(this._months),h=this.asSeconds();return h?(e=j(o/60),t=j(e/60),o%=60,e%=60,n=j(l/12),l%=12,s=o?o.toFixed(3).replace(/\.?0+$/,""):"",i=Tn(this._months)!==Tn(h)?"-":"",r=Tn(this._days)!==Tn(h)?"-":"",a=Tn(this._milliseconds)!==Tn(h)?"-":"",(h<0?"-":"")+"P"+(n?i+n+"Y":"")+(l?i+l+"M":"")+(u?r+u+"D":"")+(t||e||o?"T":"")+(t?a+t+"H":"")+(e?a+e+"M":"")+(o?a+s+"S":"")):"P0D"}X=Ut.prototype;return X.isValid=function(){return this._isValid},X.abs=function(){var e=this._data;return this._milliseconds=wn(this._milliseconds),this._days=wn(this._days),this._months=wn(this._months),e.milliseconds=wn(e.milliseconds),e.seconds=wn(e.seconds),e.minutes=wn(e.minutes),e.hours=wn(e.hours),e.months=wn(e.months),e.years=wn(e.years),this},X.add=function(e,t){return pn(this,e,t,1)},X.subtract=function(e,t){return pn(this,e,t,-1)},X.as=function(e){if(!this.isValid())return NaN;var t,n,s=this._milliseconds;if("month"===(e=V(e))||"quarter"===e||"year"===e)switch(t=this._days+s/864e5,n=this._months+kn(t),e){case"month":return n;case"quarter":return n/3;case"year":return n/12}else switch(t=this._days+Math.round(Mn(this._months)),e){case"week":return t/7+s/6048e5;case"day":return t+s/864e5;case"hour":return 24*t+s/36e5;case"minute":return 1440*t+s/6e4;case"second":return 86400*t+s/1e3;case"millisecond":return Math.floor(864e5*t)+s;default:throw new Error("Unknown unit "+e)}},X.asMilliseconds=re,X.asSeconds=K,X.asMinutes=B,X.asHours=J,X.asDays=se,X.asWeeks=Q,X.asMonths=Qt,X.asQuarters=Rt,X.asYears=Pt,X.valueOf=function(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*Z(this._months/12):NaN},X._bubble=function(){var e=this._milliseconds,t=this._days,n=this._months,s=this._data;return 0<=e&&0<=t&&0<=n||e<=0&&t<=0&&n<=0||(e+=864e5*vn(Mn(n)+t),n=t=0),s.milliseconds=e%1e3,e=j(e/1e3),s.seconds=e%60,e=j(e/60),s.minutes=e%60,e=j(e/60),s.hours=e%24,t+=j(e/24),n+=e=j(kn(t)),t-=vn(Mn(e)),e=j(n/12),n%=12,s.days=t,s.months=n,s.years=e,this},X.clone=function(){return zt(this)},X.get=function(e){return e=V(e),this.isValid()?this[e+"s"]():NaN},X.milliseconds=he,X.seconds=nt,X.minutes=Te,X.hours=ze,X.days=qe,X.weeks=function(){return j(this.days()/7)},X.months=ne,X.years=Le,X.humanize=function(e,t){if(!this.isValid())return this.localeData().invalidDate();var n=!1,s=On;return"object"==typeof e&&(t=e,e=!1),"boolean"==typeof e&&(n=e),"object"==typeof t&&(s=Object.assign({},On,t),null!=t.s&&null==t.ss&&(s.ss=t.s-1)),t=this.localeData(),s=bn(this,!n,s,t),n&&(s=t.pastFuture(+this,s)),t.postformat(s)},X.toISOString=Nn,X.toString=Nn,X.toJSON=Nn,X.locale=tn,X.localeData=nn,X.toIsoString=n("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",Nn),X.lang=te,C("X",0,0,"unix"),C("x",0,0,"valueOf"),de("x",oe),de("X",/[+-]?\d+(\.\d{1,3})?/),ye("X",function(e,t,n){n._d=new Date(1e3*parseFloat(e))}),ye("x",function(e,t,n){n._d=new Date(Z(e))}),c.version="2.29.2",e=Nt,c.fn=ie,c.min=function(){return Wt("isBefore",[].slice.call(arguments,0))},c.max=function(){return Wt("isAfter",[].slice.call(arguments,0))},c.now=function(){return Date.now?Date.now():+new Date},c.utc=m,c.unix=function(e){return Nt(1e3*e)},c.months=function(e,t){return yn(e,t,"months")},c.isDate=s,c.locale=ht,c.invalid=r,c.duration=zt,c.isMoment=k,c.weekdays=function(e,t,n){return gn(e,t,n,"weekdays")},c.parseZone=function(){return Nt.apply(null,arguments).parseZone()},c.localeData=ct,c.isDuration=Ht,c.monthsShort=function(e,t){return yn(e,t,"monthsShort")},c.weekdaysMin=function(e,t,n){return gn(e,t,n,"weekdaysMin")},c.defineLocale=dt,c.updateLocale=function(e,t){var n,s;return null!=t?(s=it,null!=rt[e]&&null!=rt[e].parentLocale?rt[e].set(O(rt[e]._config,t)):(t=O(s=null!=(n=lt(e))?n._config:s,t),null==n&&(t.abbr=e),(t=new b(t)).parentLocale=rt[e],rt[e]=t),ht(e)):null!=rt[e]&&(null!=rt[e].parentLocale?(rt[e]=rt[e].parentLocale,e===ht()&&ht(e)):null!=rt[e]&&delete rt[e]),rt[e]},c.locales=function(){return x(rt)},c.weekdaysShort=function(e,t,n){return gn(e,t,n,"weekdaysShort")},c.normalizeUnits=V,c.relativeTimeRounding=function(e){return void 0===e?Yn:"function"==typeof e&&(Yn=e,!0)},c.relativeTimeThreshold=function(e,t){return void 0!==On[e]&&(void 0===t?On[e]:(On[e]=t,"s"===e&&(On.ss=t-1),!0))},c.calendarFormat=function(e,t){return(t=e.diff(t,"days",!0))<-6?"sameElse":t<-1?"lastWeek":t<0?"lastDay":t<1?"sameDay":t<2?"nextDay":t<7?"nextWeek":"sameElse"},c.prototype=ie,c.HTML5_FMT={DATETIME_LOCAL:"YYYY-MM-DDTHH:mm",DATETIME_LOCAL_SECONDS:"YYYY-MM-DDTHH:mm:ss",DATETIME_LOCAL_MS:"YYYY-MM-DDTHH:mm:ss.SSS",DATE:"YYYY-MM-DD",TIME:"HH:mm",TIME_SECONDS:"HH:mm:ss",TIME_MS:"HH:mm:ss.SSS",WEEK:"GGGG-[W]WW",MONTH:"YYYY-MM"},c});
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
index da3d45c2060bf..09ef0370f7f09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
@@ -32,7 +32,8 @@ Centralized cache management in HDFS has many significant advantages.
 
 4.  Centralized caching can improve overall cluster memory utilization. When relying on the OS buffer cache at each DataNode, repeated reads of a block will result in all *n* replicas of the block being pulled into buffer cache. With centralized cache management, a user can explicitly pin only *m* of the *n* replicas, saving *n-m* memory.
 
-5.  HDFS supports non-volatile storage class memory (SCM, also known as persistent memory) cache in Linux platform. User can enable either memory cache or SCM cache for a DataNode. Memory cache and SCM cache can coexist among DataNodes. In the current implementation, the cache data in SCM will be cleaned up when DataNode restarts. Persistent HDFS cache support on SCM will be considered in the future.
+5.  HDFS supports non-volatile storage class memory (SCM, also known as persistent memory) cache in Linux platform. User can enable either DRAM cache or SCM cache for a DataNode. DRAM cache and SCM cache can coexist among DataNodes. In addition, cache persistence is supported by SCM cache. The status of cache persisted in SCM will be recovered
+during the start of DataNode if `dfs.datanode.pmem.cache.recovery` is set to true. Otherwise, previously persisted cache will be dropped and data need to be re-cached.
 
 Use Cases
 ---------
@@ -260,7 +261,7 @@ The following properties are not required, but may be specified for tuning:
 
 *   dfs.datanode.pmem.cache.recovery
 
-    This parameter is used to determine whether to recover the status for previous cache on persistent memory during the start of DataNode. If it is enabled, DataNode will recover the status for previously cached data on persistent memory. Thus, re-caching data will be avoided. If this property is not enabled, DataNode will clean up the previous cache, if any, on persistent memory. This property can only work when persistent memory is enabled, i.e., `dfs.datanode.pmem.cache.dirs` is configured.
+    This parameter is used to determine whether to recover the status for previous cache on persistent memory during the start of DataNode. If it is enabled, DataNode will recover the status for previously cached data on persistent memory. Thus, re-caching is avoided. If this property is not enabled, DataNode will drop cache, if any, on persistent memory. This property can only work when persistent memory cache is enabled, i.e., `dfs.datanode.pmem.cache.dirs` is configured.
 
 ### OS Limits
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index a1509d2d01d1d..0b91bdeb11cf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -227,6 +227,9 @@ Usage: `hdfs oiv [OPTIONS] -i INPUT_FILE`
 | `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
 | `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
 | `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
+| `-sp` | Whether to print Storage policy(default is false). This option is used with Delimited processor only. |
+| `-ec` | Whether to print Erasure coding policy(default is false). This option is used with Delimited processor only. |
+| `-m`,`--multiThread` *arg* | Specify multiThread to process sub-sections. This option is used with Delimited processor only. |
 | `-t`,`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
 | `-h`,`--help` | Display the tool usage and help information and exit. |
 
@@ -306,11 +309,11 @@ Usage:
 
 Runs a cluster balancing utility. An administrator can simply press Ctrl-C to stop the rebalancing process. See [Balancer](./HdfsUserGuide.html#Balancer) for more details.
 
-Note that the `blockpool` policy is more strict than the `datanode` policy.
+Note that the `blockpool` policy is stricter than the `datanode` policy.
 
 Besides the above command options, a pinning feature is introduced starting from 2.7.0 to prevent certain replicas from getting moved by balancer/mover. This pinning feature is disabled by default, and can be enabled by configuration property "dfs.datanode.block-pinning.enabled". When enabled, this feature only affects blocks that are written to favored nodes specified in the create() call. This feature is useful when we want to maintain the data locality, for applications such as HBase regionserver.
 
-If you want to run Balancer as a long running service, please start Balancer using `-asService` parameter with daemon-mode. You can do this by using the following command: `hdfs --daemon start balancer -asService`, or just use sbin/start-balancer.sh script with parameter `-asService`.
+If you want to run Balancer as a long-running service, please start Balancer using `-asService` parameter with daemon-mode. You can do this by using the following command: `hdfs --daemon start balancer -asService`, or just use sbin/start-balancer.sh script with parameter `-asService`.
 
 ### `cacheadmin`
 
@@ -356,7 +359,7 @@ Runs a HDFS datanode.
 
 Usage:
 
-        hdfs dfsadmin [-report [-live] [-dead] [-decommissioning] [-enteringmaintenance] [-inmaintenance]]
+        hdfs dfsadmin [-report [-live] [-dead] [-decommissioning] [-enteringmaintenance] [-inmaintenance] [-slownodes]]
         hdfs dfsadmin [-safemode enter | leave | get | wait | forceExit]
         hdfs dfsadmin [-saveNamespace [-beforeShutdown]]
         hdfs dfsadmin [-rollEdits]
@@ -374,7 +377,7 @@ Usage:
         hdfs dfsadmin [-refreshSuperUserGroupsConfiguration]
         hdfs dfsadmin [-refreshCallQueue]
         hdfs dfsadmin [-refresh <host:ipc_port> <key> [arg1..argn]]
-        hdfs dfsadmin [-reconfig <namenode|datanode> <host:ipc_port> <start |status |properties>]
+        hdfs dfsadmin [-reconfig <namenode|datanode> <host:ipc_port|livenodes> <start |status |properties>]
         hdfs dfsadmin [-printTopology]
         hdfs dfsadmin [-refreshNamenodes datanodehost:port]
         hdfs dfsadmin [-getVolumeReport datanodehost:port]
@@ -394,7 +397,7 @@ Usage:
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
-| `-report` `[-live]` `[-dead]` `[-decommissioning]` `[-enteringmaintenance]` `[-inmaintenance]` | Reports basic filesystem information and statistics, The dfs usage can be different from "du" usage, because it measures raw space used by replication, checksums, snapshots and etc. on all the DNs. Optional flags may be used to filter the list of displayed DataNodes. |
+| `-report` `[-live]` `[-dead]` `[-decommissioning]` `[-enteringmaintenance]` `[-inmaintenance]` `[-slownodes]` | Reports basic filesystem information and statistics, The dfs usage can be different from "du" usage, because it measures raw space used by replication, checksums, snapshots and etc. on all the DNs. Optional flags may be used to filter the list of displayed DataNodes. Filters are either based on the DN state (e.g. live, dead, decommissioning) or the nature of the DN (e.g. slow nodes - nodes with higher latency than their peers). |
 | `-safemode` enter\|leave\|get\|wait\|forceExit | Safe mode maintenance command. Safe mode is a Namenode state in which it <br/>1. does not accept changes to the name space (read-only) <br/>2. does not replicate or delete blocks. <br/>Safe mode is entered automatically at Namenode startup, and leaves safe mode automatically when the configured minimum percentage of blocks satisfies the minimum replication condition. If Namenode detects any anomaly then it will linger in safe mode till that issue is resolved. If that anomaly is the consequence of a deliberate action, then administrator can use -safemode forceExit to exit safe mode. The cases where forceExit may be required are<br/> 1. Namenode metadata is not consistent. If Namenode detects that metadata has been modified out of band and can cause data loss, then Namenode will enter forceExit state. At that point user can either restart Namenode with correct metadata files or forceExit (if data loss is acceptable).<br/>2. Rollback causes metadata to be replaced and rarely it can trigger safe mode forceExit state in Namenode. In that case you may proceed by issuing -safemode forceExit.<br/> Safe mode can also be entered manually, but then it can only be turned off manually as well. |
 | `-saveNamespace` `[-beforeShutdown]` | Save current namespace into storage directories and reset edits log. Requires safe mode. If the "beforeShutdown" option is given, the NameNode does a checkpoint if and only if no checkpoint has been done during a time window (a configurable number of checkpoint periods). This is usually used before shutting down the NameNode to prevent potential fsimage/editlog corruption. |
 | `-rollEdits` | Rolls the edit log on the active NameNode. |
@@ -412,7 +415,7 @@ Usage:
 | `-refreshSuperUserGroupsConfiguration` | Refresh superuser proxy groups mappings |
 | `-refreshCallQueue` | Reload the call queue from config. |
 | `-refresh` \<host:ipc\_port\> \<key\> [arg1..argn] | Triggers a runtime-refresh of the resource specified by \<key\> on \<host:ipc\_port\>. All other args after are sent to the host. |
-| `-reconfig` \<datanode \|namenode\> \<host:ipc\_port\> \<start\|status\|properties\> | Starts reconfiguration or gets the status of an ongoing reconfiguration, or gets a list of reconfigurable properties. The second parameter specifies the node type. |
+| `-reconfig` \<datanode \|namenode\> \<host:ipc\_port\|livenodes> \<start\|status\|properties\> | Starts reconfiguration or gets the status of an ongoing reconfiguration, or gets a list of reconfigurable properties. The second parameter specifies the node type. The third parameter specifies host address. For start or status, datanode supports livenodes as third parameter, which will start or retrieve reconfiguration on all live datanodes. |
 | `-printTopology` | Print a tree of the racks and their nodes as reported by the Namenode |
 | `-refreshNamenodes` datanodehost:port | For the given datanode, reloads the configuration files, stops serving the removed block-pools and starts serving new block-pools. |
 | `-getVolumeReport` datanodehost:port | For the given datanode, get the volume report. |
@@ -463,7 +466,7 @@ Usage:
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
 | `-add` *source* *nameservices* *destination* | Add a mount table entry or update if it exists. |
-| `-update` *source* *nameservices* *destination* | Update a mount table entry attribures. |
+| `-update` *source* *nameservices* *destination* | Update a mount table entry attributes. |
 | `-rm` *source* | Remove mount point of specified path. |
 | `-ls` `[-d]` *path* | List mount points under specified path. Specify -d parameter to get detailed listing.|
 | `-getDestination` *path* | Get the subcluster where a file is or should be created. |
@@ -496,7 +499,7 @@ Usage:
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
-|-plan| Creates a disbalancer plan|
+|-plan| Creates a diskbalancer plan|
 |-execute| Executes a given plan on a datanode|
 |-query| Gets the current diskbalancer status from a datanode|
 |-cancel| Cancels a running plan|
@@ -567,7 +570,7 @@ See [HDFS HA with NFS](./HDFSHighAvailabilityWithNFS.html#Administrative_command
 
 Usage: `hdfs journalnode`
 
-This comamnd starts a journalnode for use with [HDFS HA with QJM](./HDFSHighAvailabilityWithQJM.html#Administrative_commands).
+This command starts a journalnode for use with [HDFS HA with QJM](./HDFSHighAvailabilityWithQJM.html#Administrative_commands).
 
 ### `mover`
 
@@ -622,13 +625,13 @@ Runs the namenode. More info about the upgrade and rollback is at [Upgrade Rollb
 
 Usage: `hdfs nfs3`
 
-This comamnd starts the NFS3 gateway for use with the [HDFS NFS3 Service](./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service).
+This command starts the NFS3 gateway for use with the [HDFS NFS3 Service](./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service).
 
 ### `portmap`
 
 Usage: `hdfs portmap`
 
-This comamnd starts the RPC portmap for use with the [HDFS NFS3 Service](./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service).
+This command starts the RPC portmap for use with the [HDFS NFS3 Service](./HdfsNfsGateway.html#Start_and_stop_NFS_gateway_service).
 
 ### `secondarynamenode`
 
@@ -666,7 +669,7 @@ Usage: `hdfs zkfc [-formatZK [-force] [-nonInteractive]]`
 | `-formatZK` | Format the Zookeeper instance. -force: formats the znode if the znode exists. -nonInteractive: formats the znode aborts if the znode exists, unless -force option is specified. |
 | `-h` | Display help |
 
-This comamnd starts a Zookeeper Failover Controller process for use with [HDFS HA with QJM](./HDFSHighAvailabilityWithQJM.html#Administrative_commands).
+This command starts a Zookeeper Failover Controller process for use with [HDFS HA with QJM](./HDFSHighAvailabilityWithQJM.html#Administrative_commands).
 
 Debug Commands
 --------------
@@ -708,6 +711,16 @@ Usage: `hdfs debug recoverLease -path <path> [-retries <num-retries>]`
 
 Recover the lease on the specified path. The path must reside on an HDFS file system. The default number of retries is 1.
 
+### `verifyEC`
+
+Usage: `hdfs debug verifyEC -file <file>`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| [`-file` *EC-file*] | HDFS EC file to be verified. |
+
+Verify the correctness of erasure coding on an erasure coded file.
+
 dfsadmin with ViewFsOverloadScheme
 ----------------------------------
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index 955f179be003f..d3b1ca8f2e9f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -37,8 +37,7 @@ A plan is a set of statements that describe how much data should move between tw
 A plan is composed of multiple move steps. A move step has source disk, destination
 disk and number of bytes to move. A plan can be executed against an operational data node. Disk balancer should not
 interfere with other processes since it throttles how much data is copied
-every second. Please note that disk balancer is not enabled by default on a cluster.
-To enable diskbalancer `dfs.disk.balancer.enabled` must be set to `true` in hdfs-site.xml.
+every second. Please note that disk balancer is enabled by default on a cluster.
 
 
 Commands
@@ -123,7 +122,7 @@ There is a set of diskbalancer settings that can be controlled via hdfs-site.xml
 
 | Setting | Description |
 |:---- |:---- |
-|`dfs.disk.balancer.enabled`| This parameter controls if diskbalancer is enabled for a cluster. if this is not enabled, any execute command will be rejected by the datanode.The default value is false.|
+|`dfs.disk.balancer.enabled`| This parameter controls if diskbalancer is enabled for a cluster. if this is not enabled, any execute command will be rejected by the datanode.The default value is true.|
 |`dfs.disk.balancer.max.disk.throughputInMBperSec` | This controls the maximum disk bandwidth consumed by diskbalancer while copying data. If a value like 10MB is specified then diskbalancer on the average will only copy 10MB/S. The default value is 10MB/S.|
 |`dfs.disk.balancer.max.disk.errors`| sets the value of maximum number of errors we can ignore for a specific move between two disks before it is abandoned. For example, if a plan has 3 pair of disks to copy between , and the first disk set encounters more than 5 errors, then we abandon the first copy and start the second copy in the plan. The default value of max errors is set to 5.|
 |`dfs.disk.balancer.block.tolerance.percent`| The tolerance percent specifies when we have reached a good enough value for any copy step. For example, if you specify 10% then getting close to 10% of the target value is good enough.|
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 7d26883161873..d28aefde03040 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -234,7 +234,7 @@ Certain HDFS operations, i.e., `hflush`, `hsync`, `concat`, `setReplication`, `t
 are not supported on erasure coded files due to substantial technical
 challenges.
 
-* `append()` and `truncate()` on an erasure coded file will throw `IOException`.
+* `append()` and `truncate()` on an erasure coded file will throw `IOException`. However, with `NEW_BLOCK` flag enabled, append to a closed striped file is supported.
 * `concat()` will throw `IOException` if files are mixed with different erasure
 coding policies or with replicated files.
 * `setReplication()` is no-op on erasure coded files.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
index 06cda83651e56..aec1534b03b5b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
@@ -192,8 +192,8 @@ The order in which you set these configurations is unimportant, but the values y
     terminated, before transitioning another NameNode to the Active state. In
     order to do this, you must configure at least one **fencing method.** These are
     configured as a carriage-return-separated list, which will be attempted in order
-    until one indicates that fencing has succeeded. There are two methods which
-    ship with Hadoop: *shell* and *sshfence*. For information on implementing
+    until one indicates that fencing has succeeded. There are three methods which
+    ship with Hadoop: *shell*, *sshfence* and *powershell*. For information on implementing
     your own custom fencing method, see the *org.apache.hadoop.ha.NodeFencer* class.
 
     - - -
@@ -283,7 +283,27 @@ The order in which you set these configurations is unimportant, but the values y
     a subshell to kill its parent in some number of seconds).
 
     - - -
-  
+
+    **powershell** - use PowerShell to remotely connect to a machine and kill
+    the required process
+
+    The *powershell* fencing method uses PowerShell command. It may be
+    configured like so:
+
+            <property>
+              <name>dfs.ha.fencing.methods</name>
+              <value>powershell(NameNode)</value>
+            </property>
+
+    The argument passed to this fencer should be a unique string in the "CommandLine"
+    attribute for the "java.exe" process. For example, the full path for the Namenode:
+    "org.apache.hadoop.hdfs.server.namenode.NameNode".
+    The administrator can also shorten the name to "Namenode" if it's unique.
+
+    **Note:** This only works in Windows.
+
+    - - -
+
 *   **fs.defaultFS** - the default path prefix used by the Hadoop FS client when none is given
 
     Optionally, you may now configure the default path for Hadoop clients to use
@@ -296,12 +316,14 @@ The order in which you set these configurations is unimportant, but the values y
           <value>hdfs://mycluster</value>
         </property>
 
-*   **dfs.ha.nn.not-become-active-in-safemode** - if prevent safe mode namenodes to become active
+*   **dfs.ha.nn.not-become-active-in-safemode** - if prevent safe mode namenodes to become active or observer
 
     Whether allow namenode to become active when it is in safemode, when it is
     set to true, namenode in safemode will report SERVICE_UNHEALTHY to ZKFC if
     auto failover is on, or will throw exception to fail the transition to
-    active if auto failover is off. For example:
+    active if auto failover is off. If you transition namenode to observer state
+    when it is in safemode, when this configuration is set to true, namenode will throw exception
+    to fail the transition to observer. For example:
 
         <property>
           <name>dfs.ha.nn.not-become-active-in-safemode</name>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 4bb5971107eb7..b6b408db8b4ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -239,8 +239,8 @@ The order in which you set these configurations is unimportant, but the values y
 
     The fencing methods used during a failover are configured as a
     carriage-return-separated list, which will be attempted in order until
-    one indicates that fencing has succeeded. There are two methods which ship with
-    Hadoop: *shell* and *sshfence*. For information on implementing your own custom
+    one indicates that fencing has succeeded. There are three methods which ship with
+    Hadoop: *shell*, *sshfence* and *powershell*. For information on implementing your own custom
     fencing method, see the *org.apache.hadoop.ha.NodeFencer* class.
 
     - - -
@@ -331,6 +331,26 @@ The order in which you set these configurations is unimportant, but the values y
 
     - - -
 
+    **powershell** - use PowerShell to remotely connect to a machine and kill
+    the required process
+
+    The *powershell* fencing method uses PowerShell command. It may be
+    configured like so:
+
+            <property>
+              <name>dfs.ha.fencing.methods</name>
+              <value>powershell(NameNode)</value>
+            </property>
+
+    The argument passed to this fencer should be a unique string in the "CommandLine"
+    attribute for the "java.exe" process. For example, the full path for the Namenode:
+    "org.apache.hadoop.hdfs.server.namenode.NameNode".
+    The administrator can also shorten the name to "Namenode" if it's unique.
+
+    **Note:** This only works in Windows.
+
+    - - -
+
 *   **fs.defaultFS** - the default path prefix used by the Hadoop FS client when none is given
 
     Optionally, you may now configure the default path for Hadoop clients to use
@@ -356,12 +376,14 @@ The order in which you set these configurations is unimportant, but the values y
           <value>/path/to/journal/node/local/data</value>
         </property>
 
-*   **dfs.ha.nn.not-become-active-in-safemode** - if prevent safe mode namenodes to become active
+*   **dfs.ha.nn.not-become-active-in-safemode** - if prevent safe mode namenodes to become active or observer
 
     Whether allow namenode to become active when it is in safemode, when it is
     set to true, namenode in safemode will report SERVICE_UNHEALTHY to ZKFC if
     auto failover is on, or will throw exception to fail the transition to
-    active if auto failover is off. For example:
+    active if auto failover is off. If you transition namenode to observer state
+    when it is in safemode, when this configuration is set to true, namenode will throw exception
+    to fail the transition to observer. For example:
 
         <property>
           <name>dfs.ha.nn.not-become-active-in-safemode</name>
@@ -480,6 +502,16 @@ lag time will be much longer. The relevant configurations are:
     the oldest data in the cache was at transaction ID 20, a value of 10 would be added to the
     average.
 
+*   **dfs.journalnode.edit-cache-size.fraction** - This fraction refers to the proportion of
+    the maximum memory of the JVM. Used to calculate the size of the edits cache that is
+    kept in the JournalNode's memory. This config is an alternative to the
+    dfs.journalnode.edit-cache-size.bytes. And it is used to serve edits for tailing via
+    the RPC-based mechanism, and is only enabled when dfs.ha.tail-edits.in-progress is true.
+    Transactions range in size but are around 200 bytes on average, so the default of 1MB
+    can store around 5000 transactions. So we can configure a reasonable value based on
+    the maximum memory. The recommended value is less than 0.9. If we set
+    dfs.journalnode.edit-cache-size.bytes, this parameter will not take effect.
+
 This feature is primarily useful in conjunction with the Standby/Observer Read feature. Using this
 feature, read requests can be serviced from non-active NameNodes; thus tailing in-progress edits
 provides these nodes with the ability to serve requests with data which is much more fresh. See the
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md
index 4550f0441b62e..102c31f021d0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md
@@ -22,7 +22,7 @@ By default HDFS supports BlockPlacementPolicyDefault. Where one block on local a
 
 ## BlockPlacementPolicyRackFaultTolerant
 
-BlockPlacementPolicyRackFaultTolerant can be used to split the placement of blocks across multiple rack.By default with replication of 3 BlockPlacementPolicyDefault will  put one replica on the local machine if the writer is on a datanode, otherwise on a random datanode in the same rack as that of the writer, another replica on a node in a different (remote) rack, and the last on a different node in the same remote rack. So totally 2 racks will be used, in sceneraio like 2 racks going down at the same time will cause data inavailability where using BlockPlacementPolicyRackFaultTolerant will helop in placing 3 blocks on 3 different racks.
+BlockPlacementPolicyRackFaultTolerant can be used to split the placement of blocks across multiple rack. By default, with replication of 3 BlockPlacementPolicyDefault will  put one replica on the local machine if the writer is on a datanode, otherwise on a random datanode in the same rack as that of the writer, another replica on a node in a different (remote) rack, and the last on a different node in the same remote rack. So totally 2 racks will be used, in scenario like 2 racks going down at the same time will cause data unavailability where using BlockPlacementPolicyRackFaultTolerant will help in placing 3 blocks on 3 different racks.
 
 For more details check [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-7891)
 
@@ -42,7 +42,7 @@ For more details check [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-78
 
 ## BlockPlacementPolicyWithNodeGroup
 
-With new 3 layer hierarchical topology, a node group level got introduced, which maps well onto a infrastructure that is based on a virtulized environment. In Virtualized environment multiple vm's will be hosted on same physical machine. Vm's on the same physical host are affected by the same hardware failure. So mapping the physical host a node groups this block placement gurantees that it will never place more than one replica on the same node group (physical host), in case of node group failure, only one replica will be lost at the maximum.
+With new 3 layer hierarchical topology, a node group level got introduced, which maps well onto an infrastructure that is based on a virtualized environment. In Virtualized environment multiple vm's will be hosted on same physical machine. Vm's on the same physical host are affected by the same hardware failure. So mapping the physical host a node groups this block placement guarantees that it will never place more than one replica on the same node group (physical host), in case of node group failure, only one replica will be lost at the maximum.
 
   **Configurations :**
 
@@ -119,6 +119,16 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic
   </description>
 </property>
 
+<property>
+<name>dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance</name>
+<value>5</value>
+<description>
+    Special value between 0 and 20, inclusive. if the value is set beyond the scope,
+    this value will be set as 5 by default, Increases tolerance of
+    placing blocks on Datanodes with similar disk space used.
+</description>
+</property>
+
 <property>
   <name>
     dfs.namenode.available-space-block-placement-policy.balance-local-node
@@ -160,6 +170,18 @@ amongst maximum number of racks possible and at the same time will try to choose
     high load increases as the value reaches near 0.
   </description>
 </property>
+
+<property>
+  <name>dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance</name>
+  <value>5</value>
+  <description>
+    Only used when the dfs.block.replicator.classname is set to
+    org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy.
+    Special value between 0 and 20, inclusive. if the value is set beyond the scope,
+    this value will be set as 5 by default, Increases tolerance of
+    placing blocks on Datanodes with similar disk space used.
+  </description>
+</property>
 ```
 
 For more details check [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288)
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index 3c284c98f3d18..b9f7bbfc70033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -182,7 +182,7 @@ ACLs (Access Control Lists)
 
 In addition to the traditional POSIX permissions model, HDFS also supports POSIX ACLs (Access Control Lists). ACLs are useful for implementing permission requirements that differ from the natural organizational hierarchy of users and groups. An ACL provides a way to set different permissions for specific named users or named groups, not only the file's owner and the file's group.
 
-By default, support for ACLs is disabled, and the NameNode disallows creation of ACLs. To enable support for ACLs, set `dfs.namenode.acls.enabled` to true in the NameNode configuration.
+By default, support for ACLs is enabled, and the NameNode allows creation of ACLs. To disable support for ACLs, set `dfs.namenode.acls.enabled` to false in the NameNode configuration.
 
 An ACL consists of a set of ACL entries. Each ACL entry names a specific user or group and grants or denies read, write and execute permissions for that specific user or group. For example:
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 54a8056068ba0..16970730c925e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -351,7 +351,13 @@ Datanode supports hot swappable drives. The user can add or replace HDFS data vo
 * The user runs `dfsadmin -reconfig datanode HOST:PORT start` to start
   the reconfiguration process. The user can use
   `dfsadmin -reconfig datanode HOST:PORT status`
-  to query the running status of the reconfiguration task.
+  to query the running status of the reconfiguration task. In place of
+  HOST:PORT, we can also specify livenodes for datanode. It would allow
+  start or query reconfiguration on all live datanodes, whereas specifying
+  HOST:PORT would only allow start or query of reconfiguration on the
+  particular datanode represented by HOST:PORT. The examples for livenodes
+  queries are `dfsadmin -reconfig datanode livenodes start` and
+  `dfsadmin -reconfig datanode livenodes status`.
 
 * Once the reconfiguration task has completed, the user can safely `umount`
   the removed data volume directories and physically remove the disks.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
index 00aeb5bd2e082..74026ec86250f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ObserverNameNode.md
@@ -194,6 +194,24 @@ few configurations to your **hdfs-site.xml**:
           <value>1048576</value>
         </property>
 
+*  **dfs.journalnode.edit-cache-size.fraction** - the fraction refers to
+   the proportion of the maximum memory of the JVM.
+
+   Used to calculate the size of the edits cache that
+   is kept in the JournalNode's memory.
+   This config is an alternative to the dfs.journalnode.edit-cache-size.bytes.
+   And it is used to serve edits for tailing via the RPC-based mechanism, and is only
+   enabled when dfs.ha.tail-edits.in-progress is true. Transactions range in size but
+   are around 200 bytes on average, so the default of 1MB can store around 5000 transactions.
+   So we can configure a reasonable value based on the maximum memory. The recommended value
+   is less than 0.9. If we set dfs.journalnode.edit-cache-size.bytes, this parameter will
+   not take effect.
+
+        <property>
+          <name>dfs.journalnode.edit-cache-size.fraction</name>
+          <value>0.5f</value>
+        </property>
+
 *  **dfs.namenode.accesstime.precision** -- whether to enable access
    time for HDFS file.
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
index b29a888475941..5201e0ee921eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
@@ -109,34 +109,34 @@ In the below mount table configuration, namespace `/data` is linked to the files
 ```xml
 <configuration>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./data</name>
+    <name>fs.viewfs.mounttable.clusterX.link./data</name>
     <value>hdfs://nn1-clusterx.example.com:8020/data</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./project</name>
+    <name>fs.viewfs.mounttable.clusterX.link./project</name>
     <value>hdfs://nn2-clusterx.example.com:8020/project</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./user</name>
+    <name>fs.viewfs.mounttable.clusterX.link./user</name>
     <value>hdfs://nn3-clusterx.example.com:8020/user</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./tmp</name>
+    <name>fs.viewfs.mounttable.clusterX.link./tmp</name>
     <value>hdfs://nn4-clusterx.example.com:8020/tmp</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.linkFallback</name>
+    <name>fs.viewfs.mounttable.clusterX.linkFallback</name>
     <value>hdfs://nn5-clusterx.example.com:8020/home</value>
   </property>
 </configuration>
 ```
 
-Alternatively we can have the mount table's root merged with the root of another filesystem via `linkMergeSlash`. In the below mount table configuration, ClusterY's root is merged with the root filesystem at `hdfs://nn1-clustery.example.com:8020`.
+Alternatively we can have the mount table's root merged with the root of another filesystem via `linkMergeSlash`. In the below mount table configuration, clusterY's root is merged with the root filesystem at `hdfs://nn1-clustery.example.com:8020`.
 
 ```xml
 <configuration>
   <property>
-    <name>fs.viewfs.mounttable.ClusterY.linkMergeSlash</name>
+    <name>fs.viewfs.mounttable.clusterY.linkMergeSlash</name>
     <value>hdfs://nn1-clustery.example.com:8020/</value>
   </property>
 </configuration>
@@ -443,7 +443,7 @@ The mount tables can be described in `core-site.xml` but it is better to use ind
 </configuration> 
 ```
 
-In the file `mountTable.xml`, there is a definition of the mount table "ClusterX" for the hypothetical cluster that is a federation of the three namespace volumes managed by the three namenodes
+In the file `mountTable.xml`, there is a definition of the mount table "clusterX" for the hypothetical cluster that is a federation of the three namespace volumes managed by the three namenodes
 
 1.  nn1-clusterx.example.com:8020,
 2.  nn2-clusterx.example.com:8020, and
@@ -454,23 +454,23 @@ Here `/home` and `/tmp` are in the namespace managed by namenode nn1-clusterx.ex
 ```xml
 <configuration>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.homedir</name>
+    <name>fs.viewfs.mounttable.clusterX.homedir</name>
     <value>/home</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./home</name>
+    <name>fs.viewfs.mounttable.clusterX.link./home</name>
     <value>hdfs://nn1-clusterx.example.com:8020/home</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./tmp</name>
+    <name>fs.viewfs.mounttable.clusterX.link./tmp</name>
     <value>hdfs://nn1-clusterx.example.com:8020/tmp</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./projects/foo</name>
+    <name>fs.viewfs.mounttable.clusterX.link./projects/foo</name>
     <value>hdfs://nn2-clusterx.example.com:8020/projects/foo</value>
   </property>
   <property>
-    <name>fs.viewfs.mounttable.ClusterX.link./projects/bar</name>
+    <name>fs.viewfs.mounttable.clusterX.link./projects/bar</name>
     <value>hdfs://nn3-clusterx.example.com:8020/projects/bar</value>
   </property>
 </configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 3fb68d40e929b..46b5613fe726c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -52,6 +52,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
     * [`GETALLSTORAGEPOLICY`](#Get_all_Storage_Policies) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getAllStoragePolicies)
     * [`GETSTORAGEPOLICY`](#Get_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy)
     * [`GETSNAPSHOTDIFF`](#Get_Snapshot_Diff)
+    * [`GETSNAPSHOTDIFFLISTING`](#Get_Snapshot_Diff_Iteratively)
     * [`GETSNAPSHOTTABLEDIRECTORYLIST`](#Get_Snapshottable_Directory_List)
     * [`GETSNAPSHOTLIST`](#Get_Snapshot_List)
     * [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations)
@@ -516,6 +517,7 @@ See also: [`newlength`](#New_Length), [FileSystem](../../api/org/apache/hadoop/f
             "replication"     : 0,
             "snapshotEnabled" : true
             "type"            : "DIRECTORY"    //enum {FILE, DIRECTORY, SYMLINK}
+            "ecPolicy"        : "RS-6-3-1024k"
           }
         }
 
@@ -1604,6 +1606,27 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renameSna
 
         {"SnapshotDiffReport":{"diffList":[],"fromSnapshot":"s3","snapshotRoot":"/foo","toSnapshot":"s4"}}
 
+### Get Snapshot Diff Iteratively
+
+* Submit a HTTP GET request.
+
+        curl -i -X GET "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETSNAPSHOTDIFFLISTING
+                           &oldsnapshotname=<SNAPSHOTNAME>&snapshotname=<SNAPSHOTNAME>&snapshotdiffstartpath=<STARTPATH>&snapshotdiffindex=<STARTINDEX>
+
+    If `snapshotdiffstartpath` and `snapshotdiffindex` are not given,
+    `""` (empty string) and `-1` are used respectively implying the first iteration.
+
+    The client receives a response with a
+    [`SnapshotDiffReportListing` JSON object](#SnapshotDiffReportListing_JSON_Schema).
+    The value of `lastPath` and `lastIndex` must be specified as
+    the value of `snapshotdiffstartpath` and `snapshotdiffindex` respectively on next iteration.
+
+        HTTP/1.1 200 OK
+        Content-Type: application/json
+        Transfer-Encoding: chunked
+
+        {"SnapshotDiffReportListing":{"createList":[],"deleteList":[],"isFromEarlier":true,"lastIndex":-1,"lastPath":"","modifyList":[]}}
+
 ### Get Snapshottable Directory List
 
 * Submit a HTTP GET request.
@@ -2289,6 +2312,26 @@ var fileStatusProperties =
       "description": "The type of the path object.",
       "enum"       : ["FILE", "DIRECTORY", "SYMLINK"],
       "required"   : true
+    },
+    "aclBit":
+    {
+       "description": "Has ACLs set or not.",
+       "type"       : "boolean",
+    },
+    "encBit":
+    {
+       "description": "Is Encrypted or not.",
+       "type"       : "boolean",
+    },
+    "ecBit":
+    {
+       "description": "Is ErasureCoded or not.",
+       "type"       : "boolean",
+    },
+    "ecPolicy":
+    {
+       "description": "The namenode of ErasureCodePolicy.",
+       "type"       : "String",
     }
   }
 };
@@ -2665,6 +2708,109 @@ var diffReportEntries =
 }
 ```
 
+### SnapshotDiffReportListing JSON Schema
+
+```json
+{
+  "name": "SnapshotDiffReportListing",
+  "type": "object",
+  "properties":
+  {
+    "SnapshotDiffReportListing":
+    {
+      "type"        : "object",
+      "properties"  :
+      {
+        "isFromEarlier":
+        {
+          "description" : "the diff is calculated from older to newer snapshot or not",
+          "type"        : "boolean",
+          "required"    : true
+        },
+        "lastIndex":
+        {
+          "description" : "the last index of listing iteration",
+          "type"        : "integer",
+          "required"    : true
+        },
+        "lastPath":
+        {
+          "description" : "String representation of the last path of the listing iteration",
+          "type"        : "string",
+          "required"    : true
+        },
+        "modifyList":
+        {
+          "description": "An array of DiffReportListingEntry",
+          "type"        : "array",
+          "items"       : diffReportListingEntries,
+          "required"    : true
+        },
+        "createList":
+        {
+          "description": "An array of DiffReportListingEntry",
+          "type"        : "array",
+          "items"       : diffReportListingEntries,
+          "required"    : true
+        },
+        "deleteList":
+        {
+          "description": "An array of DiffReportListingEntry",
+          "type"        : "array",
+          "items"       : diffReportListingEntries,
+          "required"    : true
+        }
+      }
+    }
+  }
+}
+```
+
+#### DiffReportListing Entries
+
+JavaScript syntax is used to define `diffReportEntries` so that it can be referred in `SnapshotDiffReport` JSON schema.
+
+```javascript
+var diffReportListingEntries =
+{
+  "type": "object",
+  "properties":
+  {
+    "dirId":
+    {
+      "description" : "inode id of the directory",
+      "type"        : "integer",
+      "required"    : true
+    },
+    "fileId":
+    {
+      "description" : "inode id of the file",
+      "type"        : "integer",
+      "required"    : true
+    },
+    "isRereference":
+    {
+      "description" : "this is reference or not",
+      "type"        : "boolean",
+      "required"    : true
+    },
+    "sourcePath":
+    {
+      "description" : "string representation of path where changes have happened",
+      "type"        : "string",
+      "required"    : true
+    },
+    "targetPath":
+    {
+      "description" : "string representation of target path of rename op",
+      "type"        : "string",
+      "required"    : false
+    }
+  }
+}
+```
+
+
 ### SnapshottableDirectoryList JSON Schema
 
 ```json
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java
index dcd91c7d848bb..70f9af9841708 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestGenericRefresh.java
@@ -19,9 +19,7 @@
 package org.apache.hadoop;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
index 99b1ddbbc1130..7cf216b45508d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java
@@ -71,7 +71,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.function.Supplier;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java
index 05060af4a90d8..aefb4835b14d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.test.PathUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java
index 84bd98bf67c78..061858c3d2863 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.fs;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
index fba9c42858d61..cd0155389c001 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsFileSystem.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.fs;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index fcb52577d9991..62dc3076d00a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
+import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
 
 import org.junit.After;
 import org.junit.AfterClass;
@@ -181,6 +182,13 @@ int getExpectedDelegationTokenCountWithCredentials() {
     return 2;
   }
 
+  @Override
+  Path getTrashRootInFallBackFS() throws IOException {
+    return new Path(
+        "/" + TRASH_PREFIX + "/" + UserGroupInformation.getCurrentUser()
+            .getShortUserName());
+  }
+
   @Test
   public void testTrashRootsAfterEncryptionZoneDeletion() throws Exception {
     final Path zone = new Path("/EZ");
@@ -479,4 +487,23 @@ public Object run() throws IOException {
     assertEquals("The owner did not match ", owner, userUgi.getShortUserName());
     otherfs.delete(user1Path, false);
   }
+
+  @Test
+  public void testInternalDirectoryPermissions() throws IOException {
+    LOG.info("Starting testInternalDirectoryPermissions!");
+    Configuration localConf = new Configuration(conf);
+    ConfigUtil.addLinkFallback(
+        localConf, new Path(targetTestRoot, "fallbackDir").toUri());
+    FileSystem fs = FileSystem.get(FsConstants.VIEWFS_URI, localConf);
+    // check that the default permissions on a sub-folder of an internal
+    // directory are the same as those created on non-internal directories.
+    Path subDirOfInternalDir = new Path("/internalDir/dir1");
+    fs.mkdirs(subDirOfInternalDir);
+
+    Path subDirOfRealDir = new Path("/internalDir/linkToDir2/dir1");
+    fs.mkdirs(subDirOfRealDir);
+
+    assertEquals(fs.getFileStatus(subDirOfInternalDir).getPermission(),
+        fs.getFileStatus(subDirOfRealDir).getPermission());
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
index dcfa051c3902d..3d0e705e6a98c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java
@@ -103,7 +103,7 @@ protected String getDefaultWorkingDirectory() {
   }
 
   @Override
-  @Test
+  @Test(timeout = 60000)
   public void testAppend() throws IOException {
     AppendTestUtil.testAppend(fs, new Path("/append/f"));
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
index 650a4722798e8..76eee6d727c06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
@@ -496,7 +496,7 @@ private boolean isFallBackExist(Configuration config) {
    * When InnerCache disabled, all matching ViewFileSystemOverloadScheme
    * initialized scheme file systems would not use FileSystem cache.
    */
-  @Test(timeout = 3000)
+  @Test(timeout = 30000)
   public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets()
       throws Exception {
     final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER);
@@ -523,7 +523,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets()
    * initialized scheme file systems should continue to take advantage of
    * FileSystem cache.
    */
-  @Test(timeout = 3000)
+  @Test(timeout = 30000)
   public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets()
       throws Exception {
     final Path localTragetPath = new Path(localTargetDir.toURI());
@@ -545,7 +545,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets()
   /**
    * Tests the rename with nfly mount link.
    */
-  @Test(timeout = 3000)
+  @Test(timeout = 30000)
   public void testNflyRename() throws Exception {
     final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER);
     final Path hdfsTargetPath2 = new Path(defaultFSURI + HDFS_USER_FOLDER + 1);
@@ -577,7 +577,7 @@ public void testNflyRename() throws Exception {
   /**
    * Tests the write and read contents with nfly mount link.
    */
-  @Test(timeout = 3000)
+  @Test(timeout = 30000)
   public void testNflyWriteRead() throws Exception {
     final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER);
     final Path hdfsTargetPath2 = new Path(defaultFSURI + HDFS_USER_FOLDER + 1);
@@ -604,7 +604,7 @@ public void testNflyWriteRead() throws Exception {
    * target file. 3. Tests the read works with repairOnRead flag. 4. Tests that
    * previously deleted file fully recovered and exists.
    */
-  @Test(timeout = 3000)
+  @Test(timeout = 30000)
   public void testNflyRepair() throws Exception {
     final NflyFSystem.NflyKey repairKey = NflyFSystem.NflyKey.repairOnRead;
     final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsLinkFallback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsLinkFallback.java
index 09e02be640e5e..ba4a80ca11549 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsLinkFallback.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsLinkFallback.java
@@ -29,6 +29,7 @@
 import java.net.URISyntaxException;
 import java.util.EnumSet;
 
+import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.AbstractFileSystem;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -182,6 +184,26 @@ public void testMkdirOfNewDirWithOutMatchingToMountOrFallbackDirTree()
     assertTrue(fsTarget.exists(test));
   }
 
+  /**
+   * Test getDelegationToken when fallback is configured.
+   */
+  @Test
+  public void testGetDelegationToken() throws IOException {
+    Configuration conf = new Configuration();
+    conf.setBoolean(Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS, false);
+    ConfigUtil.addLink(conf, "/user",
+        new Path(targetTestRoot.toString(), "user").toUri());
+    ConfigUtil.addLink(conf, "/data",
+        new Path(targetTestRoot.toString(), "data").toUri());
+    ConfigUtil.addLinkFallback(conf, targetTestRoot.toUri());
+
+    FileContext fcView =
+        FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
+    List<Token<?>> tokens = fcView.getDelegationTokens(new Path("/"), "tester");
+    // Two tokens from the two mount points and one token from fallback
+    assertEquals(3, tokens.size());
+  }
+
   /**
    * Tests that when the parent dirs does not exist in fallback but the parent
    * dir is same as mount internal directory, then we create parent structure
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
index 10b18032e13e3..82a983004dfe7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
@@ -449,7 +449,7 @@ protected void refreshNodes(final int nnIndex) throws IOException {
         refreshNodes(conf);
   }
 
-  static private DatanodeDescriptor getDatanodeDesriptor(
+  static DatanodeDescriptor getDatanodeDesriptor(
       final FSNamesystem ns, final String datanodeUuid) {
     return ns.getBlockManager().getDatanodeManager().getDatanode(datanodeUuid);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BenchmarkThroughput.java
index 515a0a07192f4..e22e07e802eb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BenchmarkThroughput.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BenchmarkThroughput.java
@@ -187,7 +187,7 @@ public int run(String[] args) throws IOException {
     }
     Configuration conf = getConf();
     // the size of the file to write
-    long SIZE = conf.getLong("dfsthroughput.file.size",
+    long fileSize = conf.getLong("dfsthroughput.file.size",
         10L * 1024 * 1024 * 1024);
     BUFFER_SIZE = conf.getInt("dfsthroughput.buffer.size", 4 * 1024);
 
@@ -203,9 +203,9 @@ public int run(String[] args) throws IOException {
     ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
     FileSystem rawLocal = checkedLocal.getRawFileSystem();
     for(int i=0; i < reps; ++i) {
-      writeAndReadLocalFile("local", conf, SIZE);
-      writeAndReadFile(rawLocal, "raw", conf, SIZE);
-      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
+      writeAndReadLocalFile("local", conf, fileSize);
+      writeAndReadFile(rawLocal, "raw", conf, fileSize);
+      writeAndReadFile(checkedLocal, "checked", conf, fileSize);
     }
     MiniDFSCluster cluster = null;
     try {
@@ -214,7 +214,7 @@ public int run(String[] args) throws IOException {
       cluster.waitActive();
       FileSystem dfs = cluster.getFileSystem();
       for(int i=0; i < reps; ++i) {
-        writeAndReadFile(dfs, "dfs", conf, SIZE);
+        writeAndReadFile(dfs, "dfs", conf, fileSize);
       }
     } finally {
       if (cluster != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
index 6a211ef51c194..ccf45f6afc9a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
@@ -54,8 +54,8 @@ public static DFSClient getClient(DistributedFileSystem dfs)
     return dfs.dfs;
   }
 
-  public static ExtendedBlock getPreviousBlock(DFSClient client, long fileId) {
-    return client.getPreviousBlock(fileId);
+  public static ExtendedBlock getPreviousBlock(DFSClient client, String renewLeaseKey) {
+    return client.getPreviousBlock(renewLeaseKey);
   }
 
   public static long getFileId(DFSOutputStream out) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index f27f9bee9913a..594b46f6cacb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -72,7 +72,7 @@
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 import java.util.function.Supplier;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
index ba5a451d8c2f6..b2ef1b4ec3065 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataInputStream;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 04a7611032826..dd8bb2043828b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -71,6 +71,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -87,7 +88,6 @@
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.util.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -148,7 +148,7 @@
 import org.apache.hadoop.util.ToolRunner;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * This class creates a single-process DFS cluster for junit testing.
@@ -203,6 +203,8 @@ public static class Builder {
     private int nameNodeHttpPort = 0;
     private final Configuration conf;
     private int numDataNodes = 1;
+    private int[] dnHttpPorts = null;
+    private int[] dnIpcPorts = null;
     private StorageType[][] storageTypes = null;
     private StorageType[] storageTypes1D = null;
     private int storagesPerDatanode = DEFAULT_STORAGES_PER_DATANODE;
@@ -277,6 +279,16 @@ public Builder numDataNodes(int val) {
       return this;
     }
 
+    public Builder setDnHttpPorts(int... ports) {
+      this.dnHttpPorts = ports;
+      return this;
+    }
+
+    public Builder setDnIpcPorts(int... ports) {
+      this.dnIpcPorts = ports;
+      return this;
+    }
+
     /**
      * Default: DEFAULT_STORAGES_PER_DATANODE
      */
@@ -599,7 +611,9 @@ protected MiniDFSCluster(Builder builder) throws IOException {
                        builder.checkDataNodeHostConfig,
                        builder.dnConfOverlays,
                        builder.skipFsyncForTesting,
-                       builder.useConfiguredTopologyMappingClass);
+                       builder.useConfiguredTopologyMappingClass,
+                       builder.dnHttpPorts,
+                       builder.dnIpcPorts);
   }
   
   public static class DataNodeProperties {
@@ -639,7 +653,7 @@ public DataNode getDatanode() {
   private boolean federation;
   private boolean checkExitOnShutdown = true;
   protected final int storagesPerDatanode;
-  private Set<FileSystem> fileSystems = Sets.newHashSet();
+  private Set<FileSystem> fileSystems = new HashSet<>();
 
   private List<long[]> storageCap = Lists.newLinkedList();
 
@@ -873,7 +887,7 @@ public MiniDFSCluster(int nameNodePort,
                        operation, null, racks, hosts,
                        null, simulatedCapacities, null, true, false,
                        MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
-                       true, false, false, null, true, false);
+                       true, false, false, null, true, false, null, null);
   }
 
   private void initMiniDFSCluster(
@@ -891,7 +905,9 @@ private void initMiniDFSCluster(
       boolean checkDataNodeHostConfig,
       Configuration[] dnConfOverlays,
       boolean skipFsyncForTesting,
-      boolean useConfiguredTopologyMappingClass)
+      boolean useConfiguredTopologyMappingClass,
+      int[] dnHttpPorts,
+      int[] dnIpcPorts)
   throws IOException {
     boolean success = false;
     try {
@@ -974,9 +990,9 @@ private void initMiniDFSCluster(
 
       // Start the DataNodes
       startDataNodes(conf, numDataNodes, storageTypes, manageDataDfsDirs,
-          dnStartOpt != null ? dnStartOpt : startOpt,
-          racks, hosts, storageCapacities, simulatedCapacities, setupHostsFile,
-          checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
+          dnStartOpt != null ? dnStartOpt : startOpt, racks, hosts, storageCapacities,
+          simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, checkDataNodeHostConfig,
+          dnConfOverlays, dnHttpPorts, dnIpcPorts);
       waitClusterUp();
       //make sure ProxyUsers uses the latest conf
       ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
@@ -1068,7 +1084,7 @@ private static void copyKey(Configuration srcConf, Configuration destConf,
    * Do the rest of the NN configuration for things like shared edits,
    * as well as directory formatting, etc. for a single nameservice
    * @param nnCounter the count of the number of namenodes already configured/started. Also,
-   *                  acts as the <i>index</i> to the next NN to start (since indicies start at 0).
+   *                  acts as the <i>index</i> to the next NN to start (since indices start at 0).
    * @throws IOException
    */
   private void configureNameService(MiniDFSNNTopology.NSConf nameservice, int nsCounter,
@@ -1598,8 +1614,8 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
                              String[] racks, String[] hosts,
                              long[] simulatedCapacities,
                              boolean setupHostsFile) throws IOException {
-    startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts,
-        null, simulatedCapacities, setupHostsFile, false, false, null);
+    startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, null,
+        simulatedCapacities, setupHostsFile, false, false, null, null, null);
   }
 
   public synchronized void startDataNodes(Configuration conf, int numDataNodes,
@@ -1608,14 +1624,14 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
       long[] simulatedCapacities,
       boolean setupHostsFile,
       boolean checkDataNodeAddrConfig) throws IOException {
-    startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts,
-        null, simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null);
+    startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts, null,
+        simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null, null, null);
   }
 
   /**
    * Modify the config and start up additional DataNodes.  The info port for
    * DataNodes is guaranteed to use a free port.
-   *  
+   *
    *  Data nodes can run with the name node in the mini cluster or
    *  a real name node. For example, running with a real name node is useful
    *  when running simulated data nodes with a real name node.
@@ -1625,6 +1641,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
    * @param conf the base configuration to use in starting the DataNodes.  This
    *          will be modified as necessary.
    * @param numDataNodes Number of DataNodes to start; may be zero
+   * @param storageTypes Storage Types for DataNodes.
    * @param manageDfsDirs if true, the data directories for DataNodes will be
    *          created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be
    *          set in the conf
@@ -1632,13 +1649,16 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
    *          or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
    * @param racks array of strings indicating the rack that each DataNode is on
    * @param hosts array of strings indicating the hostnames for each DataNode
+   * @param storageCapacities array of Storage Capacities to be used while testing.
    * @param simulatedCapacities array of capacities of the simulated data nodes
    * @param setupHostsFile add new nodes to dfs hosts files
    * @param checkDataNodeAddrConfig if true, only set DataNode port addresses if not already set in config
    * @param checkDataNodeHostConfig if true, only set DataNode hostname key if not already set in config
    * @param dnConfOverlays An array of {@link Configuration} objects that will overlay the
    *              global MiniDFSCluster Configuration for the corresponding DataNode.
-   * @throws IllegalStateException if NameNode has been shutdown
+   * @param dnHttpPorts An array of Http ports if present, to be used for DataNodes.
+   * @param dnIpcPorts An array of Ipc ports if present, to be used for DataNodes.
+   * @throws IOException If the DFS daemons experience some issues.
    */
   public synchronized void startDataNodes(Configuration conf, int numDataNodes,
       StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation,
@@ -1648,7 +1668,9 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
       boolean setupHostsFile,
       boolean checkDataNodeAddrConfig,
       boolean checkDataNodeHostConfig,
-      Configuration[] dnConfOverlays) throws IOException {
+      Configuration[] dnConfOverlays,
+      int[] dnHttpPorts,
+      int[] dnIpcPorts) throws IOException {
     assert storageCapacities == null || simulatedCapacities == null;
     assert storageTypes == null || storageTypes.length == numDataNodes;
     assert storageCapacities == null || storageCapacities.length == numDataNodes;
@@ -1656,6 +1678,19 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
     if (operation == StartupOption.RECOVER) {
       return;
     }
+
+    if (dnHttpPorts != null && dnHttpPorts.length != numDataNodes) {
+      throw new IllegalArgumentException(
+          "Num of http ports (" + dnHttpPorts.length + ") should match num of DataNodes ("
+              + numDataNodes + ")");
+    }
+
+    if (dnIpcPorts != null && dnIpcPorts.length != numDataNodes) {
+      throw new IllegalArgumentException(
+          "Num of ipc ports (" + dnIpcPorts.length + ") should match num of DataNodes ("
+              + numDataNodes + ")");
+    }
+
     if (checkDataNodeHostConfig) {
       conf.setIfUnset(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1");
     } else {
@@ -1711,7 +1746,15 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
         dnConf.addResource(dnConfOverlays[i]);
       }
       // Set up datanode address
-      setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
+      int httpPort = 0;
+      int ipcPort = 0;
+      if(dnHttpPorts != null) {
+        httpPort = dnHttpPorts[i - curDatanodesNum];
+      }
+      if(dnIpcPorts != null) {
+        ipcPort = dnIpcPorts[i - curDatanodesNum];
+      }
+      setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig, httpPort, ipcPort);
       if (manageDfsDirs) {
         String dirs = makeDataNodeDirs(i, storageTypes == null ?
           null : storageTypes[i - curDatanodesNum]);
@@ -2116,10 +2159,11 @@ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
     LOG.info("Shutting down the Mini HDFS Cluster");
     if (checkExitOnShutdown)  {
       if (ExitUtil.terminateCalled()) {
-        LOG.error("Test resulted in an unexpected exit",
-            ExitUtil.getFirstExitException());
+        Exception cause = ExitUtil.getFirstExitException();
+        LOG.error("Test resulted in an unexpected exit", cause);
         ExitUtil.resetFirstExitException();
-        throw new AssertionError("Test resulted in an unexpected exit");
+        throw new AssertionError("Test resulted in an unexpected exit: " +
+            cause.toString(), cause);
       }
     }
     if (closeFileSystem) {
@@ -2265,11 +2309,15 @@ public synchronized void restartNameNode(int nnIndex, boolean waitActive,
     nn.getHttpServer()
         .setAttribute(ImageServlet.RECENT_IMAGE_CHECK_ENABLED, false);
     info.nameNode = nn;
+    info.nameserviceId = info.conf.get(DFS_NAMESERVICE_ID);
+    info.nnId = info.conf.get(DFS_HA_NAMENODE_ID_KEY);
     info.setStartOpt(startOpt);
     if (waitActive) {
-      waitClusterUp();
+      if (numDataNodes > 0) {
+        waitNameNodeUp(nnIndex);
+      }
       LOG.info("Restarted the namenode");
-      waitActive();
+      waitActive(nnIndex);
     }
   }
 
@@ -2775,11 +2823,25 @@ public void waitActive(int nnIndex) throws IOException {
     DFSClient client = new DFSClient(addr, conf);
 
     // ensure all datanodes have registered and sent heartbeat to the namenode
-    while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+    int failedCount = 0;
+    while (true) {
       try {
-        LOG.info("Waiting for cluster to become active");
-        Thread.sleep(100);
+        while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) {
+          LOG.info("Waiting for cluster to become active");
+          Thread.sleep(100);
+        }
+        break;
+      } catch (IOException e) {
+        failedCount++;
+        // Cached RPC connection to namenode, if any, is expected to fail once
+        if (failedCount > 1) {
+          LOG.warn("Tried waitActive() " + failedCount
+              + " time(s) and failed, giving up.  " + StringUtils
+              .stringifyException(e));
+          throw e;
+        }
       } catch (InterruptedException e) {
+        throw new IOException(e);
       }
     }
 
@@ -2815,22 +2877,7 @@ public Boolean get() {
    */
   public void waitActive() throws IOException {
     for (int index = 0; index < namenodes.size(); index++) {
-      int failedCount = 0;
-      while (true) {
-        try {
-          waitActive(index);
-          break;
-        } catch (IOException e) {
-          failedCount++;
-          // Cached RPC connection to namenode, if any, is expected to fail once
-          if (failedCount > 1) {
-            LOG.warn("Tried waitActive() " + failedCount
-                + " time(s) and failed, giving up.  "
-                + StringUtils.stringifyException(e));
-            throw e;
-          }
-        }
-      }
+      waitActive(index);
     }
     LOG.info("Cluster is active");
   }
@@ -3362,9 +3409,9 @@ public void setBlockRecoveryTimeout(long timeout) {
           timeout);
     }
   }
-  
+
   protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile,
-                           boolean checkDataNodeAddrConfig) throws IOException {
+      boolean checkDataNodeAddrConfig, int httpPort, int ipcPort) throws IOException {
     if (setupHostsFile) {
       String hostsFile = conf.get(DFS_HOSTS, "").trim();
       if (hostsFile.length() == 0) {
@@ -3387,11 +3434,11 @@ protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile,
       }
     }
     if (checkDataNodeAddrConfig) {
-      conf.setIfUnset(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0");
-      conf.setIfUnset(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0");
+      conf.setIfUnset(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:" + httpPort);
+      conf.setIfUnset(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:" + ipcPort);
     } else {
-      conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:0");
+      conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "127.0.0.1:" + httpPort);
+      conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:" + ipcPort);
     }
   }
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
index 8b4e9e5ef7379..f7d1cb10e8f60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
@@ -118,7 +118,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
     for (int i = curDatanodesNum; i < curDatanodesNum+numDataNodes; i++) {
       Configuration dnConf = new HdfsConfiguration(conf);
       // Set up datanode address
-      setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
+      setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig, 0, 0);
       if (manageDfsDirs) {
         String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]);
         dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs);
@@ -235,7 +235,9 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
       boolean setupHostsFile,
       boolean checkDataNodeAddrConfig,
       boolean checkDataNodeHostConfig,
-      Configuration[] dnConfOverlays) throws IOException {
+      Configuration[] dnConfOverlays,
+      int[] dnHttpPorts,
+      int[] dnIpcPorts) throws IOException {
     startDataNodes(conf, numDataNodes, storageTypes, manageDfsDirs, operation, racks,
         NODE_GROUPS, hosts, storageCapacities, simulatedCapacities, setupHostsFile,
         checkDataNodeAddrConfig, checkDataNodeHostConfig);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
index 31707d672c39a..0a8161c9c2efa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
@@ -22,7 +22,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Lists;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAbandonBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAbandonBlock.java
index e7d8b38aed9ce..02298165402e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAbandonBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAbandonBlock.java
@@ -68,7 +68,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  /** Abandon a block while creating a file */
+  /* Abandon a block while creating a file */
   public void testAbandonBlock() throws IOException {
     String src = FILE_NAME_PREFIX + "foo";
 
@@ -105,7 +105,7 @@ public void testAbandonBlock() throws IOException {
   }
 
   @Test
-  /** Make sure that the quota is decremented correctly when a block is abandoned */
+  /* Make sure that the quota is decremented correctly when a block is abandoned */
   public void testQuotaUpdatedWhenBlockAbandoned() throws IOException {
     // Setting diskspace quota to 3MB
     fs.setQuota(new Path("/"), HdfsConstants.QUOTA_DONT_SET, 3 * 1024 * 1024);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
index b4e9550e11828..8a702d571644f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
@@ -50,7 +50,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.event.Level;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 54c3eda4b86b9..0bb0336c4dd9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1274,23 +1274,29 @@ public void testChooseTargetWithTopology() throws Exception {
     DFSTestUtil.formatNameNode(conf);
     NameNode namenode = new NameNode(conf);
 
-    final BlockManager bm = namenode.getNamesystem().getBlockManager();
-    BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
-    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
-    for (DatanodeDescriptor datanode : dataNodes) {
-      cluster.add(datanode);
-    }
-
-    DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3,
-        dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
-        new HashSet<Node>(), 0, policy1, null);
-    System.out.println(Arrays.asList(targets));
-    Assert.assertEquals(3, targets.length);
-    targets = replicator.chooseTarget("/foo", 3,
-        dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
-        new HashSet<Node>(), 0, policy2, null);
-    System.out.println(Arrays.asList(targets));
-    Assert.assertEquals(3, targets.length);
+    try {
+      final BlockManager bm = namenode.getNamesystem().getBlockManager();
+      BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
+      NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
+      for (DatanodeDescriptor datanode : dataNodes) {
+        cluster.add(datanode);
+      }
+
+      DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3,
+              dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
+              new HashSet<Node>(), 0, policy1, null);
+      System.out.println(Arrays.asList(targets));
+      Assert.assertEquals(3, targets.length);
+      targets = replicator.chooseTarget("/foo", 3,
+              dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
+              new HashSet<Node>(), 0, policy2, null);
+      System.out.println(Arrays.asList(targets));
+      Assert.assertEquals(3, targets.length);
+    } finally {
+      if (namenode != null) {
+        namenode.stop();
+      }
+    }
   }
 
   @Test
@@ -1321,20 +1327,85 @@ public void testChooseSsdOverDisk() throws Exception {
     DFSTestUtil.formatNameNode(conf);
     NameNode namenode = new NameNode(conf);
 
-    final BlockManager bm = namenode.getNamesystem().getBlockManager();
-    BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
-    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
-    for (DatanodeDescriptor datanode : dataNodes) {
-      cluster.add(datanode);
+    try {
+      final BlockManager bm = namenode.getNamesystem().getBlockManager();
+      BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
+      NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
+      for (DatanodeDescriptor datanode : dataNodes) {
+        cluster.add(datanode);
+      }
+
+      DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3,
+              dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
+              new HashSet<Node>(), 0, policy, null);
+      System.out.println(policy.getName() + ": " + Arrays.asList(targets));
+      Assert.assertEquals(2, targets.length);
+      Assert.assertEquals(StorageType.SSD, targets[0].getStorageType());
+      Assert.assertEquals(StorageType.DISK, targets[1].getStorageType());
+    } finally {
+      if (namenode != null) {
+        namenode.stop();
+      }
+    }
+  }
+
+  /**
+   * Consider a File with All_SSD storage policy.
+   * 1. Choose 3 DISK DNs for pipeline because SSD DNs no enough at
+   * the beginning.
+   * 2. One of DISK DNs fails And it need choose one new DN for existing.
+   * pipeline {@link DataStreamer addDatanode2ExistingPipeline()}.
+   * Make sure the number of target DNs are 3.
+   * see HDFS-16182.
+   */
+  @Test
+  public void testAddDatanode2ExistingPipelineInSsd() throws Exception {
+    BlockStoragePolicy policy = POLICY_SUITE.getPolicy(ALLSSD);
+
+    final String[] racks = {"/d1/r1", "/d2/r2", "/d3/r3", "/d4/r4", "/d5/r5",
+        "/d6/r6", "/d7/r7"};
+    final String[] hosts = {"host1", "host2", "host3", "host4", "host5",
+        "host6", "host7"};
+    final StorageType[] disks = {StorageType.DISK, StorageType.DISK, StorageType.DISK};
+
+    final DatanodeStorageInfo[] diskStorages
+        = DFSTestUtil.createDatanodeStorageInfos(7, racks, hosts, disks);
+    final DatanodeDescriptor[] dataNodes
+        = DFSTestUtil.toDatanodeDescriptor(diskStorages);
+    for (int i = 0; i < dataNodes.length; i++) {
+      BlockManagerTestUtil.updateStorage(dataNodes[i],
+          new DatanodeStorage("ssd" + i + 1, DatanodeStorage.State.NORMAL,
+              StorageType.SSD));
     }
 
-    DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3,
-        dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false,
-        new HashSet<Node>(), 0, policy, null);
-    System.out.println(policy.getName() + ": " + Arrays.asList(targets));
-    Assert.assertEquals(2, targets.length);
-    Assert.assertEquals(StorageType.SSD, targets[0].getStorageType());
-    Assert.assertEquals(StorageType.DISK, targets[1].getStorageType());
+    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+    File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+        new File(baseDir, "name").getPath());
+    DFSTestUtil.formatNameNode(conf);
+    NameNode namenode = new NameNode(conf);
+
+    try {
+      final BlockManager bm = namenode.getNamesystem().getBlockManager();
+      BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
+      NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
+      for (DatanodeDescriptor datanode : dataNodes) {
+        cluster.add(datanode);
+      }
+      // chsenDs are DISK StorageType to simulate not enough SDD Storage
+      List<DatanodeStorageInfo> chsenDs = new ArrayList<>();
+      chsenDs.add(diskStorages[0]);
+      chsenDs.add(diskStorages[1]);
+      DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 1,
+              null, chsenDs, true,
+              new HashSet<Node>(), 0, policy, null);
+      Assert.assertEquals(3, targets.length);
+    } finally {
+      if (namenode != null) {
+        namenode.stop();
+      }
+    }
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
index 95d6825d29740..d86700b39b110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlocksScheduledCounter.java
@@ -190,6 +190,8 @@ public void testScheduledBlocksCounterDecrementOnDeletedBlock()
 
       // 4. delete the file
       dfs.delete(filePath, true);
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
       int blocksScheduled = 0;
       for (DatanodeDescriptor descriptor : dnList) {
         if (descriptor.getBlocksScheduled() != 0) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index 3b766f930a335..70aa9d7d635a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
+import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
 import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
@@ -447,7 +448,7 @@ public void testPipelineRecoveryOnRestartFailure() throws Exception {
    *  rolling upgrades. The client should be able to retry pipeline recovery
    *  more times than the default.
    *  (in a row for the same packet, including the heartbeat packet)
-   *  (See{@link DataStreamer#pipelineRecoveryCount})
+   *  (See{@link DataStreamer#getPipelineRecoveryCount})
    */
   @Test(timeout = 60000)
   public void testPipelineRecoveryOnDatanodeUpgrade() throws Exception {
@@ -893,4 +894,56 @@ public Boolean get() {
       DataNodeFaultInjector.set(oldDnInjector);
     }
   }
+
+  @Test
+  public void testPipelineRecoveryWithSlowNode() throws Exception {
+    final int oneWriteSize = 5000;
+
+    final int threshold = 3;
+    Configuration conf = new HdfsConfiguration();
+    conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_MARK_SLOWNODE_AS_BADNODE_THRESHOLD_KEY, threshold);
+
+    // Need 4 datanodes to verify the replaceDatanode during pipeline recovery
+    final MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+    DataNodeFaultInjector old = DataNodeFaultInjector.get();
+
+    try {
+      DistributedFileSystem fs = cluster.getFileSystem();
+      Path fileName = new Path("/f");
+      FSDataOutputStream o = fs.create(fileName);
+      // Flush to get the pipeline created.
+      o.writeBytes("hello");
+      o.hflush();
+      DFSOutputStream dfsO = (DFSOutputStream) o.getWrappedStream();
+      final DatanodeInfo[] pipeline = dfsO.getStreamer().getNodes();
+      final String lastDn = pipeline[2].getXferAddr(false);
+
+      DataNodeFaultInjector.set(new DataNodeFaultInjector() {
+        @Override
+        public void markSlow(String mirrorAddr, int[] replies) {
+          if (!lastDn.equals(mirrorAddr)) {
+            // Only fail for last DN
+            return;
+          }
+          assert(replies.length == 2);
+          replies[1] = PipelineAck.setSLOWForHeader(replies[1], PipelineAck.SLOW.SLOW);
+        }
+      });
+
+      int count = 0;
+      Random r = new Random();
+      byte[] b = new byte[oneWriteSize];
+      while (count < threshold) {
+        r.nextBytes(b);
+        o.write(b);
+        count++;
+        o.hflush();
+      }
+      Assert.assertNotEquals(lastDn, dfsO.getStreamer().getNodes()[2].getXferAddr(false));
+    } finally {
+      DataNodeFaultInjector.set(old);
+      cluster.shutdown();
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 970003b0e58cc..c335d38f73311 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -384,7 +384,7 @@ public void testLeaseRenewSocketTimeout() throws Exception
       cluster.waitActive();
       NamenodeProtocols spyNN = spy(cluster.getNameNodeRpc());
       Mockito.doThrow(new SocketTimeoutException()).when(spyNN).renewLease(
-          Mockito.anyString());
+          Mockito.anyString(), any());
       DFSClient client = new DFSClient(null, spyNN, conf, null);
       // Get hold of the lease renewer instance used by the client
       final LeaseRenewer leaseRenewer1 = client.getLeaseRenewer();
@@ -392,7 +392,7 @@ public void testLeaseRenewSocketTimeout() throws Exception
       OutputStream out1 = client.create(file1, false);
 
       Mockito.verify(spyNN, timeout(10000).times(1)).renewLease(
-          Mockito.anyString());
+          Mockito.anyString(), any());
       verifyEmptyLease(leaseRenewer1);
       GenericTestUtils.waitFor(() -> !(leaseRenewer1.isRunning()), 100, 10000);
       try {
@@ -406,12 +406,12 @@ public void testLeaseRenewSocketTimeout() throws Exception
       // Verify DFSClient can do write operation after renewLease no longer
       // throws SocketTimeoutException.
       Mockito.doNothing().when(spyNN).renewLease(
-          Mockito.anyString());
+          Mockito.anyString(), any());
       final LeaseRenewer leaseRenewer2 = client.getLeaseRenewer();
       leaseRenewer2.setRenewalTime(100);
       OutputStream out2 = client.create(file2, false);
       Mockito.verify(spyNN, timeout(10000).times(2)).renewLease(
-          Mockito.anyString());
+          Mockito.anyString(), any());
       out2.write(new byte[256]);
       out2.close();
       verifyEmptyLease(leaseRenewer2);
@@ -1309,7 +1309,7 @@ public void delayWhenRenewLeaseTimeout() {
           try {
             //1. trigger get LeaseRenewer lock
             Mockito.doThrow(new SocketTimeoutException()).when(spyNN)
-                .renewLease(Mockito.anyString());
+                .renewLease(Mockito.anyString(), any());
           } catch (IOException e) {
             e.printStackTrace();
           }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStreamBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStreamBlockLocations.java
index 9fed914c8d6fc..50378f6038161 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStreamBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStreamBlockLocations.java
@@ -21,28 +21,25 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.util.Time;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -137,154 +134,111 @@ public void teardown() throws IOException {
   }
 
   @Test
-  public void testRead() throws Exception {
+  public void testRefreshBlockLocations() throws IOException {
+    final String fileName = "/test_cache_locations";
+    filePath = createFile(fileName);
+
+    try (DFSInputStream fin = dfsClient.open(fileName)) {
+      LocatedBlocks existing = fin.locatedBlocks;
+      long lastRefreshedAt = fin.getLastRefreshedBlocksAtForTesting();
+
+      assertFalse("should not have attempted refresh",
+          fin.refreshBlockLocations(null));
+      assertEquals("should not have updated lastRefreshedAt",
+          lastRefreshedAt, fin.getLastRefreshedBlocksAtForTesting());
+      assertSame("should not have modified locatedBlocks",
+          existing, fin.locatedBlocks);
+
+      // fake a dead node to force refresh
+      // refreshBlockLocations should return true, indicating we attempted a refresh
+      // nothing should be changed, because locations have not changed
+      fin.addToLocalDeadNodes(dfsClient.datanodeReport(DatanodeReportType.LIVE)[0]);
+      assertTrue("should have attempted refresh",
+          fin.refreshBlockLocations(null));
+      verifyChanged(fin, existing, lastRefreshedAt);
+
+      // reset
+      lastRefreshedAt = fin.getLastRefreshedBlocksAtForTesting();
+      existing = fin.locatedBlocks;
+
+      // It's hard to test explicitly for non-local nodes, but we can fake it
+      // because we also treat unresolved as non-local. Pass in a cache where all the datanodes
+      // are unresolved hosts.
+      Map<String, InetSocketAddress> mockAddressCache = new HashMap<>();
+      InetSocketAddress unresolved = InetSocketAddress.createUnresolved("www.google.com", 80);
+      for (DataNode dataNode : dfsCluster.getDataNodes()) {
+        mockAddressCache.put(dataNode.getDatanodeUuid(), unresolved);
+      }
+
+      assertTrue("should have attempted refresh",
+          fin.refreshBlockLocations(mockAddressCache));
+      verifyChanged(fin, existing, lastRefreshedAt);
+    }
+  }
+
+  private void verifyChanged(DFSInputStream fin, LocatedBlocks existing, long lastRefreshedAt) {
+    assertTrue("lastRefreshedAt should have incremented",
+        fin.getLastRefreshedBlocksAtForTesting() > lastRefreshedAt);
+    assertNotSame("located blocks should have changed",
+        existing, fin.locatedBlocks);
+    assertTrue("deadNodes should be empty",
+        fin.getLocalDeadNodes().isEmpty());
+  }
+
+  @Test
+  public void testDeferredRegistrationStatefulRead() throws IOException {
+    testWithRegistrationMethod(DFSInputStream::read);
+  }
+
+  @Test
+  public void testDeferredRegistrationPositionalRead() throws IOException {
+    testWithRegistrationMethod(fin -> fin.readFully(0, new byte[1]));
+  }
+
+  @Test
+  public void testDeferredRegistrationGetAllBlocks() throws IOException {
+    testWithRegistrationMethod(DFSInputStream::getAllBlocks);
+  }
+
+  @FunctionalInterface
+  interface ThrowingConsumer {
+    void accept(DFSInputStream fin) throws IOException;
+  }
+
+  private void testWithRegistrationMethod(ThrowingConsumer registrationMethod) throws IOException {
     final String fileName = "/test_cache_locations";
-    filePath = new Path(fileName);
+    filePath = createFile(fileName);
+
     DFSInputStream fin = null;
-    FSDataOutputStream fout = null;
     try {
-      // create a file and write for testing
-      fout = fs.create(filePath, REPLICATION_FACTOR);
-      fout.write(new byte[(fileLength)]);
-      // finalize the file by closing the output stream
-      fout.close();
-      fout = null;
-      // get the located blocks
-      LocatedBlocks referenceLocatedBlocks =
-          dfsClient.getLocatedBlocks(fileName, 0, fileLength);
-      assertEquals(numOfBlocks, referenceLocatedBlocks.locatedBlockCount());
-      String poolId = dfsCluster.getNamesystem().getBlockPoolId();
       fin = dfsClient.open(fileName);
-      // get the located blocks from fin
-      LocatedBlocks finLocatedBlocks = fin.locatedBlocks;
-      assertEquals(dfsClientPrefetchSize / BLOCK_SIZE,
-          finLocatedBlocks.locatedBlockCount());
-      final int chunkReadSize = BLOCK_SIZE / 4;
-      byte[] readBuffer = new byte[chunkReadSize];
-      // read the first block
-      DatanodeInfo prevDNInfo = null;
-      DatanodeInfo currDNInfo = null;
-      int bytesRead = 0;
-      int firstBlockMark = BLOCK_SIZE;
-      // get the second block locations
-      LocatedBlock firstLocatedBlk =
-          fin.locatedBlocks.getLocatedBlocks().get(0);
-      DatanodeInfo[] firstBlkDNInfos = firstLocatedBlk.getLocations();
-      while (fin.getPos() < firstBlockMark) {
-        bytesRead = fin.read(readBuffer);
-        Assert.assertTrue("Unexpected number of read bytes",
-            chunkReadSize >= bytesRead);
-        if (currDNInfo == null) {
-          currDNInfo = fin.getCurrentDatanode();
-          assertNotNull("current FIS datanode is null", currDNInfo);
-          continue;
-        }
-        prevDNInfo = currDNInfo;
-        currDNInfo = fin.getCurrentDatanode();
-        assertEquals("the DFSInput stream does not read from same node",
-            prevDNInfo, currDNInfo);
-      }
+      assertFalse("should not be tracking input stream on open",
+          dfsClient.getLocatedBlockRefresher().isInputStreamTracked(fin));
 
-      assertEquals("InputStream exceeds expected position",
-          firstBlockMark, fin.getPos());
-      // get the second block locations
-      LocatedBlock secondLocatedBlk =
-          fin.locatedBlocks.getLocatedBlocks().get(1);
-      // get the nodeinfo for that block
-      DatanodeInfo[] secondBlkDNInfos = secondLocatedBlk.getLocations();
-      DatanodeInfo deadNodeInfo = secondBlkDNInfos[0];
-      // stop the datanode in the list of the
-      DataNode deadNode = getdataNodeFromHostName(dfsCluster,
-          deadNodeInfo.getHostName());
-      // Shutdown and wait for datanode to be marked dead
-      DatanodeRegistration reg = InternalDataNodeTestUtils.
-          getDNRegistrationForBP(dfsCluster.getDataNodes().get(0), poolId);
-      DataNodeProperties stoppedDNProps =
-          dfsCluster.stopDataNode(deadNodeInfo.getName());
+      // still not registered because it hasn't been an hour by the time we call this
+      registrationMethod.accept(fin);
+      assertFalse("should not be tracking input stream after first read",
+          dfsClient.getLocatedBlockRefresher().isInputStreamTracked(fin));
 
-      List<DataNode> datanodesPostStoppage = dfsCluster.getDataNodes();
-      assertEquals(NUM_DATA_NODES - 1, datanodesPostStoppage.size());
-      // get the located blocks
-      LocatedBlocks afterStoppageLocatedBlocks =
-          dfsClient.getLocatedBlocks(fileName, 0, fileLength);
-      // read second block
-      int secondBlockMark =  (int) (1.5 * BLOCK_SIZE);
-      boolean firstIteration = true;
-      if (this.enableBlkExpiration) {
-        // set the time stamps to make sure that we do not refresh locations yet
-        fin.setReadTimeStampsForTesting(Time.monotonicNow());
-      }
-      while (fin.getPos() < secondBlockMark) {
-        bytesRead = fin.read(readBuffer);
-        assertTrue("dead node used to read at position: " + fin.getPos(),
-            fin.deadNodesContain(deadNodeInfo));
-        Assert.assertTrue("Unexpected number of read bytes",
-            chunkReadSize >= bytesRead);
-        prevDNInfo = currDNInfo;
-        currDNInfo = fin.getCurrentDatanode();
-        assertNotEquals(deadNodeInfo, currDNInfo);
-        if (firstIteration) {
-          // currDNInfo has to be different unless first block locs is different
-          assertFalse("FSInputStream should pick a different DN",
-              firstBlkDNInfos[0].equals(deadNodeInfo)
-                  && prevDNInfo.equals(currDNInfo));
-          firstIteration = false;
-        }
-      }
-      assertEquals("InputStream exceeds expected position",
-          secondBlockMark, fin.getPos());
-      // restart the dead node with the same port
-      assertTrue(dfsCluster.restartDataNode(stoppedDNProps, true));
-      dfsCluster.waitActive();
-      List<DataNode> datanodesPostRestart = dfsCluster.getDataNodes();
-      assertEquals(NUM_DATA_NODES, datanodesPostRestart.size());
-      // continue reading from block 2 again. We should read from deadNode
-      int thirdBlockMark =  2 * BLOCK_SIZE;
-      firstIteration = true;
-      while (fin.getPos() < thirdBlockMark) {
-        bytesRead = fin.read(readBuffer);
-        if (this.enableBlkExpiration) {
-          assertEquals("node is removed from deadNodes after 1st iteration",
-              firstIteration, fin.deadNodesContain(deadNodeInfo));
-        } else {
-          assertTrue(fin.deadNodesContain(deadNodeInfo));
-        }
-        Assert.assertTrue("Unexpected number of read bytes",
-            chunkReadSize >= bytesRead);
-        prevDNInfo = currDNInfo;
-        currDNInfo = fin.getCurrentDatanode();
-        if (!this.enableBlkExpiration) {
-          assertNotEquals(deadNodeInfo, currDNInfo);
-        }
-        if (firstIteration) {
-          assertEquals(prevDNInfo, currDNInfo);
-          firstIteration = false;
-          if (this.enableBlkExpiration) {
-            // reset the time stamps of located blocks to force cache expiration
-            fin.setReadTimeStampsForTesting(
-                Time.monotonicNow() - (dfsInputLocationsTimeout + 1));
-          }
-        }
-      }
-      assertEquals("InputStream exceeds expected position",
-          thirdBlockMark, fin.getPos());
+      // artificially make it have been an hour
+      fin.setLastRefreshedBlocksAtForTesting(Time.monotonicNow() - (dfsInputLocationsTimeout + 1));
+      registrationMethod.accept(fin);
+      assertEquals("SHOULD be tracking input stream on read after interval, only if enabled",
+          enableBlkExpiration, dfsClient.getLocatedBlockRefresher().isInputStreamTracked(fin));
     } finally {
-      if (fout != null) {
-        fout.close();
-      }
       if (fin != null) {
         fin.close();
+        assertFalse(dfsClient.getLocatedBlockRefresher().isInputStreamTracked(fin));
       }
+      fs.delete(filePath, true);
     }
   }
 
-  private DataNode getdataNodeFromHostName(MiniDFSCluster cluster,
-      String hostName) {
-    for (DataNode dn : cluster.getDataNodes()) {
-      if (dn.getDatanodeId().getHostName().equals(hostName)) {
-        return dn;
-      }
+  private Path createFile(String fileName) throws IOException {
+    Path path = new Path(fileName);
+    try (FSDataOutputStream fout = fs.create(path, REPLICATION_FACTOR)) {
+      fout.write(new byte[(fileLength)]);
     }
-    return null;
+    return path;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
index 432ac8e9a2e0c..8b90287e82fe1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CreateFlag;
@@ -67,8 +68,9 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyLong;
 import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
@@ -300,6 +302,86 @@ public void testCongestionBackoff() throws IOException {
     Assert.assertTrue(congestedNodes.isEmpty());
   }
 
+  @Test(timeout=60000)
+  public void testCongestionAckDelay() {
+    DfsClientConf dfsClientConf = mock(DfsClientConf.class);
+    DFSClient client = mock(DFSClient.class);
+    when(client.getConf()).thenReturn(dfsClientConf);
+    when(client.getTracer()).thenReturn(FsTracer.get(new Configuration()));
+    client.clientRunning = true;
+    DataStreamer stream = new DataStreamer(
+            mock(HdfsFileStatus.class),
+            mock(ExtendedBlock.class),
+            client,
+            "foo", null, null, null, null, null, null);
+    DataOutputStream blockStream = mock(DataOutputStream.class);
+    Whitebox.setInternalState(stream, "blockStream", blockStream);
+    Whitebox.setInternalState(stream, "stage",
+            BlockConstructionStage.PIPELINE_CLOSE);
+    @SuppressWarnings("unchecked")
+    LinkedList<DFSPacket> dataQueue = (LinkedList<DFSPacket>)
+            Whitebox.getInternalState(stream, "dataQueue");
+    @SuppressWarnings("unchecked")
+    ArrayList<DatanodeInfo> congestedNodes = (ArrayList<DatanodeInfo>)
+            Whitebox.getInternalState(stream, "congestedNodes");
+    int backOffMaxTime = (int)
+            Whitebox.getInternalState(stream, "CONGESTION_BACK_OFF_MAX_TIME_IN_MS");
+    DFSPacket[] packet = new DFSPacket[100];
+    AtomicBoolean isDelay = new AtomicBoolean(true);
+
+    // ResponseProcessor needs the dataQueue for the next step.
+    new Thread(() -> {
+      for (int i = 0; i < 10; i++) {
+        // In order to ensure that other threads run for a period of time to prevent affecting
+        // the results.
+        try {
+          Thread.sleep(backOffMaxTime / 50);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        synchronized (dataQueue) {
+          congestedNodes.add(mock(DatanodeInfo.class));
+          // The DataStreamer releases the dataQueue before sleeping, and the ResponseProcessor
+          // has time to hold the dataQueue to continuously accept ACKs and add congestedNodes
+          // to the list. Therefore, congestedNodes.size() is greater than 1.
+          if (congestedNodes.size() > 1){
+            isDelay.set(false);
+            try {
+              doThrow(new IOException()).when(blockStream).flush();
+            } catch (Exception e) {
+              e.printStackTrace();
+            }
+          }
+        }
+      }
+      try {
+        doThrow(new IOException()).when(blockStream).flush();
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+      // Prevent the DataStreamer from always waiting because the
+      // dataQueue may be empty, so that the unit test cannot exit.
+      DFSPacket endPacket = mock(DFSPacket.class);
+      dataQueue.add(endPacket);
+    }).start();
+
+    // The purpose of adding packets to the dataQueue is to make the DataStreamer run
+    // normally and judge whether to enter the sleep state according to the congestion.
+    new Thread(() -> {
+      for (int i = 0; i < 100; i++) {
+        packet[i] = mock(DFSPacket.class);
+        dataQueue.add(packet[i]);
+        try {
+          Thread.sleep(backOffMaxTime / 100);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+      }
+    }).start();
+    stream.run();
+    Assert.assertFalse(isDelay.get());
+  }
+
   @Test
   public void testNoLocalWriteFlag() throws IOException {
     DistributedFileSystem fs = cluster.getFileSystem();
@@ -352,7 +434,7 @@ public void testEndLeaseCall() throws Exception {
         EnumSet.of(CreateFlag.CREATE), (short) 3, 1024, null , 1024, null);
     DFSOutputStream spyDFSOutputStream = Mockito.spy(dfsOutputStream);
     spyDFSOutputStream.closeThreads(anyBoolean());
-    verify(spyClient, times(1)).endFileLease(anyLong());
+    verify(spyClient, times(1)).endFileLease(anyString());
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
index 15ce06b69fb6f..818f7568e9e1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.Time;
 import org.junit.After;
 import org.junit.Before;
@@ -260,6 +261,17 @@ private void createAndCheckPermission(OpType op, Path name, short umask,
     checkPermission(name, expectedPermission, delete);
   }
 
+  @Test
+  public void testFSNamesystemCheckAccess() throws Exception{
+    Path testInvalidPath = new Path("/test2");
+    fs = FileSystem.get(conf);
+
+    LambdaTestUtils.intercept(
+        FileNotFoundException.class,
+        "Path not found: " + testInvalidPath,
+        () -> fs.access(testInvalidPath, FsAction.READ));
+  }
+
   /* Check if the permission of a file/directory is the same as the
    * expected permission; If the delete flag is true, delete the
    * file/directory afterwards.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
index fe2eee28b751e..427dc43d3bb15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -161,6 +162,8 @@ public void testRenameWithOverwrite() throws Exception {
       assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock().
           getLocalBlock()) != null);
       dfs.rename(srcPath, dstPath, Rename.OVERWRITE);
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
       assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock().
           getLocalBlock()) == null);
       
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 44c06e39ea904..e54b7332b1ef0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -37,6 +37,7 @@
 import java.util.function.Supplier;
 
 import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.XAttrNotFoundException;
 import org.apache.hadoop.util.Lists;
 import org.slf4j.Logger;
@@ -1410,7 +1411,7 @@ private void confirmOwner(String owner, String group,
     }
   }
 
-  @Test (timeout = 30000)
+  @Test (timeout = 60000)
   public void testFilePermissions() throws IOException {
     Configuration conf = new HdfsConfiguration();
 
@@ -3043,6 +3044,96 @@ public void testAppendToFileBadArgs() throws Exception {
     assertThat(res, not(0));
   }
 
+  @Test (timeout = 300000)
+  public void testAppendToFileWithOptionN() throws Exception {
+    final int inputFileLength = 1024 * 1024;
+    File testRoot = new File(TEST_ROOT_DIR, "testAppendToFileWithOptionN");
+    testRoot.mkdirs();
+
+    File file1 = new File(testRoot, "file1");
+    createLocalFileWithRandomData(inputFileLength, file1);
+
+    Configuration conf = new HdfsConfiguration();
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(6).build()) {
+      cluster.waitActive();
+      FileSystem hdfs = cluster.getFileSystem();
+      assertTrue("Not a HDFS: " + hdfs.getUri(),
+          hdfs instanceof DistributedFileSystem);
+
+      // Run appendToFile with option n by replica policy once, make sure that the target file is
+      // created and is of the right size and block number is correct.
+      String dir = "/replica";
+      boolean mkdirs = hdfs.mkdirs(new Path(dir));
+      assertTrue("Mkdir fail", mkdirs);
+      Path remoteFile = new Path(dir + "/remoteFile");
+      FsShell shell = new FsShell();
+      shell.setConf(conf);
+      String[] argv = new String[] {
+          "-appendToFile", "-n", file1.toString(), remoteFile.toString() };
+      int res = ToolRunner.run(shell, argv);
+      assertEquals("Run appendToFile command fail", 0, res);
+      FileStatus fileStatus = hdfs.getFileStatus(remoteFile);
+      assertEquals("File size should be " + inputFileLength,
+          inputFileLength, fileStatus.getLen());
+      BlockLocation[] fileBlockLocations =
+          hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+      assertEquals("Block Num should be 1", 1, fileBlockLocations.length);
+
+      // Run appendToFile with option n by replica policy again and
+      // make sure that the target file size has been doubled and block number has been doubled.
+      res = ToolRunner.run(shell, argv);
+      assertEquals("Run appendToFile command fail", 0, res);
+      fileStatus = hdfs.getFileStatus(remoteFile);
+      assertEquals("File size should be " + inputFileLength * 2,
+          inputFileLength * 2, fileStatus.getLen());
+      fileBlockLocations = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+      assertEquals("Block Num should be 2", 2, fileBlockLocations.length);
+
+      // Before run appendToFile with option n by ec policy, set ec policy for the dir.
+      dir = "/ecPolicy";
+      final String ecPolicyName = "RS-6-3-1024k";
+      mkdirs = hdfs.mkdirs(new Path(dir));
+      assertTrue("Mkdir fail", mkdirs);
+      ((DistributedFileSystem) hdfs).setErasureCodingPolicy(new Path(dir), ecPolicyName);
+      ErasureCodingPolicy erasureCodingPolicy =
+          ((DistributedFileSystem) hdfs).getErasureCodingPolicy(new Path(dir));
+      assertEquals("Set ec policy fail", ecPolicyName, erasureCodingPolicy.getName());
+
+      // Run appendToFile with option n by ec policy once, make sure that the target file is
+      // created and is of the right size and block group number is correct.
+      remoteFile = new Path(dir + "/remoteFile");
+      argv = new String[] {
+          "-appendToFile", "-n", file1.toString(), remoteFile.toString() };
+      res = ToolRunner.run(shell, argv);
+      assertEquals("Run appendToFile command fail", 0, res);
+      fileStatus = hdfs.getFileStatus(remoteFile);
+      assertEquals("File size should be " + inputFileLength,
+          inputFileLength, fileStatus.getLen());
+      fileBlockLocations = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+      assertEquals("Block Group Num should be 1", 1, fileBlockLocations.length);
+
+      // Run appendToFile without option n by ec policy again and make sure that
+      // append on EC file without new block must fail.
+      argv = new String[] {
+          "-appendToFile", file1.toString(), remoteFile.toString() };
+      res = ToolRunner.run(shell, argv);
+      assertTrue("Run appendToFile command must fail", res != 0);
+
+      // Run appendToFile with option n by ec policy again and
+      // make sure that the target file size has been doubled
+      // and block group number has been doubled.
+      argv = new String[] {
+          "-appendToFile", "-n", file1.toString(), remoteFile.toString() };
+      res = ToolRunner.run(shell, argv);
+      assertEquals("Run appendToFile command fail", 0, res);
+      fileStatus = hdfs.getFileStatus(remoteFile);
+      assertEquals("File size should be " + inputFileLength * 2,
+          inputFileLength * 2, fileStatus.getLen());
+      fileBlockLocations = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+      assertEquals("Block Group Num should be 2", 2, fileBlockLocations.length);
+    }
+  }
+
   @Test (timeout = 30000)
   public void testSetXAttrPermission() throws Exception {
     UserGroupInformation user = UserGroupInformation.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShellTouch.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShellTouch.java
new file mode 100644
index 0000000000000..d0334cbf42747
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShellTouch.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Date;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.TouchCommands;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests to perform Touch operations on DFS.
+ */
+public class TestDFSShellTouch {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestDFSShellTouch.class);
+
+  private static MiniDFSCluster miniCluster;
+  private static DistributedFileSystem dfs;
+  private static FsShell shell;
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    final Configuration conf = new Configuration();
+    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,
+        GenericTestUtils.getTestDir("TestDFSShellTouch").getAbsolutePath());
+
+    miniCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+    miniCluster.waitActive();
+    dfs = miniCluster.getFileSystem();
+    shell = new FsShell(dfs.getConf());
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (miniCluster != null) {
+      miniCluster.shutdown(true, true);
+    }
+  }
+
+  @Test
+  public void testTouch() throws Exception {
+    final String newFileName = "newFile1";
+    final Path newFile = new Path(newFileName);
+    dfs.delete(newFile, true);
+    assertThat(dfs.exists(newFile)).isFalse();
+    dfs.create(newFile);
+    assertThat(dfs.exists(newFile)).isTrue();
+
+    String strTime = formatTimestamp(System.currentTimeMillis());
+    Date dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-t", strTime, newFileName)).as(
+        "Expected successful touch on a new file" + " with a specified timestamp").isEqualTo(0);
+    FileStatus newStatus = dfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    FileStatus fileStatus = dfs.getFileStatus(newFile);
+    Thread.sleep(500);
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-a", "-t", strTime, newFileName)).as(
+        "Expected successful touch with a specified access time").isEqualTo(0);
+    newStatus = dfs.getFileStatus(newFile);
+    // Verify if access time is recorded correctly (and modification time
+    // remains unchanged).
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(fileStatus.getModificationTime());
+
+    fileStatus = dfs.getFileStatus(newFile);
+    Thread.sleep(500);
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-m", "-t", strTime, newFileName)).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+    // Verify if modification time is recorded correctly (and access time
+    // remains unchanged).
+    newStatus = dfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(fileStatus.getAccessTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-t", strTime, newFileName)).as(
+        "Expected successful touch with a specified timestamp").isEqualTo(0);
+
+    // Verify if both modification and access times are recorded correctly
+    newStatus = dfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-a", "-m", "-t", strTime, newFileName)).as(
+        "Expected successful touch with a specified timestamp").isEqualTo(0);
+
+    // Verify if both modification and access times are recorded correctly
+    newStatus = dfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    assertThat(shellRun("-touch", "-t", newFileName)).as(
+        "Expected failed touch with a missing timestamp").isNotEqualTo(0);
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+    assertThat(shellRun("-touch", "-c", "-t", strTime, newFileName)).as(
+        "Expected successful touch on a non-existent file with -c option").isEqualTo(0);
+    fileStatus = dfs.getFileStatus(newFile);
+    assertThat(fileStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(fileStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    dfs.delete(newFile, true);
+    assertThat(dfs.exists(newFile)).isFalse();
+
+  }
+
+  @Test
+  public void testTouchDirs() throws IOException, ParseException, InterruptedException {
+    final String newFileName = "dir2/newFile2";
+    final Path newFile = new Path(newFileName);
+    FileStatus newStatus;
+    FileStatus fileStatus;
+    String strTime;
+    Date dateObj;
+    Path dirPath = new Path("dir2");
+    dfs.mkdirs(dirPath);
+    dfs.delete(newFile, true);
+    assertThat(dfs.exists(newFile)).isFalse();
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-t", strTime, newFileName)).as(
+        "Expected successful touch on a new file with a specified timestamp").isEqualTo(0);
+    newStatus = dfs.getFileStatus(newFile);
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    Thread.sleep(500);
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-m", "-a", "-t", strTime, "dir2")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = dfs.getFileStatus(dirPath);
+    // Verify if both modification and access times are recorded correctly
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    fileStatus = dfs.getFileStatus(dirPath);
+    Thread.sleep(500);
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-m", "-t", strTime, "dir2")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = dfs.getFileStatus(dirPath);
+    // Verify if modification time is recorded correctly (and access time
+    // remains unchanged).
+    assertThat(newStatus.getAccessTime()).isEqualTo(fileStatus.getAccessTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(dateObj.getTime());
+
+    fileStatus = dfs.getFileStatus(dirPath);
+    Thread.sleep(500);
+
+    strTime = formatTimestamp(System.currentTimeMillis());
+    dateObj = parseTimestamp(strTime);
+
+    assertThat(shellRun("-touch", "-a", "-t", strTime, "dir2")).as(
+        "Expected successful touch with a specified modification time").isEqualTo(0);
+
+    newStatus = dfs.getFileStatus(dirPath);
+    // Verify if access time is recorded correctly (and modification time
+    // remains unchanged).
+    assertThat(newStatus.getAccessTime()).isEqualTo(dateObj.getTime());
+    assertThat(newStatus.getModificationTime()).isEqualTo(fileStatus.getModificationTime());
+
+    dfs.delete(newFile, true);
+    dfs.delete(dirPath, true);
+    assertThat(dfs.exists(newFile)).isFalse();
+    assertThat(dfs.exists(dirPath)).isFalse();
+  }
+
+  private int shellRun(String... args) {
+    int exitCode = shell.run(args);
+    LOG.info("exit " + exitCode + " - " + StringUtils.join(" ", args));
+    return exitCode;
+  }
+
+  private String formatTimestamp(long timeInMillis) {
+    return (new TouchCommands.Touch()).getDateFormat().format(new Date(timeInMillis));
+  }
+
+  private Date parseTimestamp(String tstamp) throws ParseException {
+    return (new TouchCommands.Touch()).getDateFormat().parse(tstamp);
+  }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index aedea3c8acde4..2369d42482a41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -49,7 +49,9 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
@@ -664,4 +666,75 @@ public void testUnbuffer() throws Exception {
       assertNull(in.parityBuf);
       in.close();
   }
+
+  @Test
+  public void testBlockReader() throws Exception {
+    ErasureCodingPolicy targetPolicy = StripedFileTestUtil.getDefaultECPolicy(); // RS-6-3-1024k
+    if (!ecPolicy.equals(targetPolicy)) {
+      // Be sure not affected by random EC policy from
+      // TestDFSStripedInputStreamWithRandomECPolicy.
+      return;
+    }
+    int fileSize = 19 * cellSize + 100;
+    long stripeSize = (long) dataBlocks * cellSize;
+    byte[] bytes = StripedFileTestUtil.generateBytes(fileSize);
+    DFSTestUtil.writeFile(fs, filePath, new String(bytes));
+
+    try (DFSStripedInputStream in =
+             (DFSStripedInputStream) fs.getClient().open(filePath.toString())) {
+      // Verify pread:
+      verifyPreadRanges(in, 0, 2 * cellSize,
+          2 * cellSize, Arrays.asList("0_0_1048576", "1_0_1048576"));
+      verifyPreadRanges(in, 0, 5 * cellSize + 9527,
+          5 * cellSize + 9527, Arrays.asList("0_0_1048576", "1_0_1048576",
+              "2_0_1048576", "3_0_1048576", "4_0_1048576", "5_0_1048576"));
+      verifyPreadRanges(in, 100, 5 * cellSize + 9527,
+          5 * cellSize + 9527, Arrays.asList("0_100_1048476", "1_0_1048576",
+              "2_0_1048576", "3_0_1048576", "4_0_1048576", "5_0_1048576"));
+      verifyPreadRanges(in, stripeSize * 3, 2 * cellSize,
+          cellSize + 100, Arrays.asList("0_1048576_1048576", "1_1048576_100"));
+
+      // Verify sread:
+      verifySreadRanges(in, 0, Arrays.asList("0_0_2097152", "1_0_2097152",
+          "2_0_2097152", "3_0_2097152", "4_0_2097152", "5_0_2097152"));
+      verifySreadRanges(in, stripeSize * 2, Arrays.asList("0_0_2097152", "1_0_1048676",
+          "2_0_1048576", "3_0_1048576", "4_0_1048576", "5_0_1048576"));
+    }
+  }
+
+  private void verifyPreadRanges(DFSStripedInputStream in, long position,
+                                 int length, int lengthExpected,
+                                 List<String> rangesExpected) throws Exception {
+    List<String> ranges = new ArrayList<>(); // range format: chunkIndex_offset_len
+    DFSClientFaultInjector.set(new DFSClientFaultInjector() {
+      @Override
+      public void onCreateBlockReader(LocatedBlock block, int chunkIndex,
+                                      long offset, long length) {
+        ranges.add(String.format("%s_%s_%s", chunkIndex, offset, length));
+      }
+    });
+    assertEquals(lengthExpected, in.read(position, new byte[length], 0, length));
+    Collections.sort(ranges);
+    Collections.sort(rangesExpected);
+    assertEquals(rangesExpected, ranges);
+  }
+
+  private void verifySreadRanges(DFSStripedInputStream in, long position,
+                                 List<String> rangesExpected) throws Exception {
+    List<String> ranges = new ArrayList<>(); // range format: chunkIndex_offset_len
+    DFSClientFaultInjector.set(new DFSClientFaultInjector() {
+      @Override
+      public void onCreateBlockReader(LocatedBlock block, int chunkIndex,
+                                      long offset, long length) {
+        ranges.add(String.format("%s_%s_%s", chunkIndex, offset, length));
+      }
+    });
+    in.seek(position);
+    int length = in.read(new byte[1024]);
+    assertEquals(1024, length);
+    Collections.sort(ranges);
+    Collections.sort(rangesExpected);
+    assertEquals(rangesExpected, ranges);
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java
index bbe991dacc781..7c2775aa8fdc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 9a024c3084586..e6ce29316c5b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -53,6 +53,7 @@
 import java.net.URISyntaxException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -80,7 +81,6 @@
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.Sets;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -1042,10 +1042,10 @@ public void testGetNNServiceRpcAddressesForNsIds() throws IOException {
 
     {
       Collection<String> internal = DFSUtil.getInternalNameServices(conf);
-      assertEquals(Sets.newHashSet("nn1"), internal);
+      assertEquals(new HashSet<>(Arrays.asList("nn1")), internal);
 
       Collection<String> all = DFSUtilClient.getNameServiceIds(conf);
-      assertEquals(Sets.newHashSet("nn1", "nn2"), all);
+      assertEquals(new HashSet<>(Arrays.asList("nn1", "nn2")), all);
     }
 
     Map<String, Map<String, InetSocketAddress>> nnMap = DFSUtil
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index b1a675c77b62a..98556c4fd15ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -551,6 +551,39 @@ public void TestPipeLineAckCompatibility() throws IOException {
         .CHECKSUM_OK), newAck.getHeaderFlag(0));
   }
 
+  @Test
+  public void testPipeLineAckCompatibilityWithSLOW() throws IOException {
+    DataTransferProtos.PipelineAckProto proto = DataTransferProtos
+        .PipelineAckProto.newBuilder()
+        .setSeqno(0)
+        .addReply(Status.CHECKSUM_OK)
+        .addFlag(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED,
+            Status.CHECKSUM_OK))
+        .build();
+
+    DataTransferProtos.PipelineAckProto newProto = DataTransferProtos
+        .PipelineAckProto.newBuilder()
+        .setSeqno(0)
+        .addReply(Status.CHECKSUM_OK)
+        .addFlag(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED,
+            Status.CHECKSUM_OK, PipelineAck.SLOW.SLOW))
+        .build();
+
+    ByteArrayOutputStream oldAckBytes = new ByteArrayOutputStream();
+    proto.writeDelimitedTo(oldAckBytes);
+    PipelineAck oldAck = new PipelineAck();
+    oldAck.readFields(new ByteArrayInputStream(oldAckBytes.toByteArray()));
+    assertEquals(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED, Status
+        .CHECKSUM_OK, PipelineAck.SLOW.DISABLED), oldAck.getHeaderFlag(0));
+
+    PipelineAck newAck = new PipelineAck();
+    ByteArrayOutputStream newAckBytes = new ByteArrayOutputStream();
+    newProto.writeDelimitedTo(newAckBytes);
+    newAck.readFields(new ByteArrayInputStream(newAckBytes.toByteArray()));
+    assertEquals(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED, Status
+        .CHECKSUM_OK, PipelineAck.SLOW.SLOW), newAck.getHeaderFlag(0));
+  }
+
   void writeBlock(String poolId, long blockId, DataChecksum checksum) throws IOException {
     writeBlock(new ExtendedBlock(poolId, blockId),
         BlockConstructionStage.PIPELINE_SETUP_CREATE, 0L, checksum);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
index de738eef177a3..9638f71ef8286 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
@@ -81,21 +81,21 @@ public void testDatanodeReportWithUpgradeDomain() throws Exception {
       datanode.setUpgradeDomain(ud1);
       hostsFileWriter.initIncludeHosts(
           new DatanodeAdminProperties[]{datanode});
-      client.refreshNodes();
+      cluster.getNamesystem().getBlockManager().getDatanodeManager().refreshNodes(conf);
       DatanodeInfo[] all = client.datanodeReport(DatanodeReportType.ALL);
       assertEquals(all[0].getUpgradeDomain(), ud1);
 
       datanode.setUpgradeDomain(null);
       hostsFileWriter.initIncludeHosts(
           new DatanodeAdminProperties[]{datanode});
-      client.refreshNodes();
+      cluster.getNamesystem().getBlockManager().getDatanodeManager().refreshNodes(conf);
       all = client.datanodeReport(DatanodeReportType.ALL);
       assertEquals(all[0].getUpgradeDomain(), null);
 
       datanode.setUpgradeDomain(ud2);
       hostsFileWriter.initIncludeHosts(
           new DatanodeAdminProperties[]{datanode});
-      client.refreshNodes();
+      cluster.getNamesystem().getBlockManager().getDatanodeManager().refreshNodes(conf);
       all = client.datanodeReport(DatanodeReportType.ALL);
       assertEquals(all[0].getUpgradeDomain(), ud2);
     } finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index f7e6dce003311..0133d3aec37b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -26,6 +27,8 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -40,6 +43,8 @@
 import java.util.EnumSet;
 
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
 import org.apache.commons.text.TextStringBuilder;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -47,8 +52,10 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
@@ -59,6 +66,7 @@
 import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
 import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -77,6 +85,7 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.ToolRunner;
+
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -667,6 +676,21 @@ public void testDecommissionWithOpenfile()
     fdos.close();
   }
 
+  @Test(timeout = 20000)
+  public void testDecommissionWithUnknownBlock() throws IOException {
+    startCluster(1, 3);
+
+    FSNamesystem ns = getCluster().getNamesystem(0);
+    DatanodeManager datanodeManager = ns.getBlockManager().getDatanodeManager();
+
+    BlockInfo blk = new BlockInfoContiguous(new Block(1L), (short) 1);
+    DatanodeDescriptor dn = datanodeManager.getDatanodes().iterator().next();
+    dn.getStorageInfos()[0].addBlock(blk, blk);
+
+    datanodeManager.getDatanodeAdminManager().startDecommission(dn);
+    waitNodeState(dn, DatanodeInfo.AdminStates.DECOMMISSIONED);
+  }
+
   private static String scanIntoString(final ByteArrayOutputStream baos) {
     final TextStringBuilder sb = new TextStringBuilder();
     final Scanner scanner = new Scanner(baos.toString());
@@ -1299,7 +1323,7 @@ public void testBlocksPerInterval() throws Exception {
         DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY,
         3);
     // Disable the normal monitor runs
-    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+    getConf().setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY,
         Integer.MAX_VALUE);
     startCluster(1, 3);
     final FileSystem fs = getCluster().getFileSystem();
@@ -1352,7 +1376,7 @@ public void testPendingNodeButDecommissioned() throws Exception {
         DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES,
         1);
     // Disable the normal monitor runs
-    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+    getConf().setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY,
         Integer.MAX_VALUE);
     startCluster(1, 2);
     final DatanodeManager datanodeManager =
@@ -1401,7 +1425,7 @@ public void testPendingNodes() throws Exception {
         DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES,
         1);
     // Disable the normal monitor runs
-    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+    getConf().setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY,
         Integer.MAX_VALUE);
     startCluster(1, 3);
     final FileSystem fs = getCluster().getFileSystem();
@@ -1654,4 +1678,510 @@ public Boolean get() {
 
     cleanupFile(fileSys, file);
   }
+
+  /**
+   * Test DatanodeAdminManager logic to re-queue unhealthy decommissioning nodes
+   * which are blocking the decommissioning of healthy nodes.
+   * Force the tracked nodes set to be filled with nodes lost while decommissioning,
+   * then decommission healthy nodes & validate they are decommissioned eventually.
+   */
+  @Test(timeout = 120000)
+  public void testRequeueUnhealthyDecommissioningNodes() throws Exception {
+    // Create a MiniDFSCluster with 3 live datanode in AdminState=NORMAL and
+    // 2 dead datanodes in AdminState=DECOMMISSION_INPROGRESS and a file
+    // with replication factor of 5.
+    final int numLiveNodes = 3;
+    final int numDeadNodes = 2;
+    final int numNodes = numLiveNodes + numDeadNodes;
+    final List<DatanodeDescriptor> liveNodes = new ArrayList<>();
+    final Map<DatanodeDescriptor, MiniDFSCluster.DataNodeProperties> deadNodeProps =
+        new HashMap<>();
+    final ArrayList<DatanodeInfo> decommissionedNodes = new ArrayList<>();
+    final Path filePath = new Path("/tmp/test");
+    createClusterWithDeadNodesDecommissionInProgress(numLiveNodes, liveNodes, numDeadNodes,
+        deadNodeProps, decommissionedNodes, filePath);
+    final FSNamesystem namesystem = getCluster().getNamesystem();
+    final BlockManager blockManager = namesystem.getBlockManager();
+    final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
+    final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager();
+
+    // Validate the 2 "dead" nodes are not removed from the tracked nodes set
+    // after several seconds of operation
+    final Duration checkDuration = Duration.ofSeconds(5);
+    Instant checkUntil = Instant.now().plus(checkDuration);
+    while (Instant.now().isBefore(checkUntil)) {
+      BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      assertEquals(
+          "Unexpected number of decommissioning nodes queued in DatanodeAdminManager.",
+          0, decomManager.getNumPendingNodes());
+      assertEquals(
+          "Unexpected number of decommissioning nodes tracked in DatanodeAdminManager.",
+          numDeadNodes, decomManager.getNumTrackedNodes());
+      assertTrue(
+          "Dead decommissioning nodes unexpectedly transitioned out of DECOMMISSION_INPROGRESS.",
+          deadNodeProps.keySet().stream()
+              .allMatch(node -> node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)));
+      Thread.sleep(500);
+    }
+
+    // Delete the file such that its no longer a factor blocking decommissioning of live nodes
+    // which have block replicas for that file
+    getCluster().getFileSystem().delete(filePath, true);
+
+    // Start decommissioning 2 "live" datanodes
+    int numLiveDecommNodes = 2;
+    final List<DatanodeDescriptor> liveDecommNodes = liveNodes.subList(0, numLiveDecommNodes);
+    for (final DatanodeDescriptor liveNode : liveDecommNodes) {
+      takeNodeOutofService(0, liveNode.getDatanodeUuid(), 0, decommissionedNodes,
+          AdminStates.DECOMMISSION_INPROGRESS);
+      decommissionedNodes.add(liveNode);
+    }
+
+    // Write a new file such that there are under-replicated blocks preventing decommissioning
+    // of dead nodes
+    writeFile(getCluster().getFileSystem(), filePath, numNodes, 10);
+
+    // Validate that the live datanodes are put into the pending decommissioning queue
+    GenericTestUtils.waitFor(() -> decomManager.getNumTrackedNodes() == numDeadNodes
+            && decomManager.getNumPendingNodes() == numLiveDecommNodes
+            && liveDecommNodes.stream().allMatch(
+                node -> node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)),
+        500, 30000);
+    assertThat(liveDecommNodes)
+        .as("Check all live decommissioning nodes queued in DatanodeAdminManager")
+        .containsAll(decomManager.getPendingNodes());
+
+    // Run DatanodeAdminManager.Monitor, then validate the dead nodes are re-queued & the
+    // live nodes are decommissioned
+    if (this instanceof TestDecommissionWithBackoffMonitor) {
+      // For TestDecommissionWithBackoffMonitor a single tick/execution of the
+      // DatanodeAdminBackoffMonitor will re-queue the dead nodes, then call
+      // "processPendingNodes" to de-queue the live nodes & decommission them
+      BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      assertEquals(
+          "DatanodeAdminBackoffMonitor did not re-queue dead decommissioning nodes as expected.",
+          2, decomManager.getNumPendingNodes());
+      assertEquals(
+          "DatanodeAdminBackoffMonitor did not re-queue dead decommissioning nodes as expected.",
+          0, decomManager.getNumTrackedNodes());
+    } else {
+      // For TestDecommission a single tick/execution of the DatanodeAdminDefaultMonitor
+      // will re-queue the dead nodes. A seconds tick is needed to de-queue the live nodes
+      // & decommission them
+      BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      assertEquals(
+          "DatanodeAdminDefaultMonitor did not re-queue dead decommissioning nodes as expected.",
+          4, decomManager.getNumPendingNodes());
+      assertEquals(
+          "DatanodeAdminDefaultMonitor did not re-queue dead decommissioning nodes as expected.",
+          0, decomManager.getNumTrackedNodes());
+      BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      assertEquals(
+          "DatanodeAdminDefaultMonitor did not decommission live nodes as expected.",
+          2, decomManager.getNumPendingNodes());
+      assertEquals(
+          "DatanodeAdminDefaultMonitor did not decommission live nodes as expected.",
+          0, decomManager.getNumTrackedNodes());
+    }
+    assertTrue("Live nodes not DECOMMISSIONED as expected.", liveDecommNodes.stream()
+        .allMatch(node -> node.getAdminState().equals(AdminStates.DECOMMISSIONED)));
+    assertTrue("Dead nodes not DECOMMISSION_INPROGRESS as expected.",
+        deadNodeProps.keySet().stream()
+            .allMatch(node -> node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)));
+    assertThat(deadNodeProps.keySet())
+        .as("Check all dead decommissioning nodes queued in DatanodeAdminManager")
+        .containsAll(decomManager.getPendingNodes());
+
+    // Validate the 2 "dead" nodes are not removed from the tracked nodes set
+    // after several seconds of operation
+    checkUntil = Instant.now().plus(checkDuration);
+    while (Instant.now().isBefore(checkUntil)) {
+      BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      assertEquals(
+          "Unexpected number of decommissioning nodes queued in DatanodeAdminManager.",
+          0, decomManager.getNumPendingNodes());
+      assertEquals(
+          "Unexpected number of decommissioning nodes tracked in DatanodeAdminManager.",
+          numDeadNodes, decomManager.getNumTrackedNodes());
+      assertTrue(
+          "Dead decommissioning nodes unexpectedly transitioned out of DECOMMISSION_INPROGRESS.",
+          deadNodeProps.keySet().stream()
+              .allMatch(node -> node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)));
+      Thread.sleep(500);
+    }
+
+    // Delete the file such that there are no more under-replicated blocks
+    // allowing the dead nodes to be decommissioned
+    getCluster().getFileSystem().delete(filePath, true);
+
+    // Validate the dead nodes are eventually decommissioned
+    GenericTestUtils.waitFor(() -> {
+      try {
+        BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+      } catch (ExecutionException | InterruptedException e) {
+        LOG.warn("Exception running DatanodeAdminMonitor", e);
+        return false;
+      }
+      return decomManager.getNumTrackedNodes() == 0 && decomManager.getNumPendingNodes() == 0
+          && deadNodeProps.keySet().stream().allMatch(
+              node -> node.getAdminState().equals(AdminStates.DECOMMISSIONED));
+    }, 500, 30000);
+  }
+
+  /**
+   * Create a MiniDFSCluster with "numLiveNodes" live datanodes in AdminState=NORMAL and
+   * "numDeadNodes" dead datanodes in AdminState=DECOMMISSION_INPROGRESS. Create a file
+   * replicated to all datanodes.
+   *
+   * @param numLiveNodes  - number of live nodes in cluster
+   * @param liveNodes     - list which will be loaded with references to 3 live datanodes
+   * @param numDeadNodes  - number of live nodes in cluster
+   * @param deadNodeProps - map which will be loaded with references to 2 dead datanodes
+   * @param decommissionedNodes - list which will be loaded with references to decommissioning nodes
+   * @param filePath      - path used to create HDFS file
+   */
+  private void createClusterWithDeadNodesDecommissionInProgress(final int numLiveNodes,
+      final List<DatanodeDescriptor> liveNodes, final int numDeadNodes,
+      final Map<DatanodeDescriptor, MiniDFSCluster.DataNodeProperties> deadNodeProps,
+      final ArrayList<DatanodeInfo> decommissionedNodes, final Path filePath) throws Exception {
+    assertTrue("Must have numLiveNode > 0", numLiveNodes > 0);
+    assertTrue("Must have numDeadNode > 0", numDeadNodes > 0);
+    int numNodes = numLiveNodes + numDeadNodes;
+
+    // Allow "numDeadNodes" datanodes to be decommissioned at a time
+    getConf()
+        .setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES, numDeadNodes);
+    // Disable the normal monitor runs
+    getConf()
+        .setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, Integer.MAX_VALUE);
+
+    // Start cluster with "numNodes" datanodes
+    startCluster(1, numNodes);
+    final FSNamesystem namesystem = getCluster().getNamesystem();
+    final BlockManager blockManager = namesystem.getBlockManager();
+    final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
+    final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager();
+    assertEquals(numNodes, getCluster().getDataNodes().size());
+    getCluster().waitActive();
+
+    // "numLiveNodes" datanodes will remain "live"
+    for (final DataNode node : getCluster().getDataNodes().subList(0, numLiveNodes)) {
+      liveNodes.add(getDatanodeDesriptor(namesystem, node.getDatanodeUuid()));
+    }
+    assertEquals(numLiveNodes, liveNodes.size());
+
+    // "numDeadNodes" datanodes will be "dead" while decommissioning
+    final List<DatanodeDescriptor> deadNodes =
+        getCluster().getDataNodes().subList(numLiveNodes, numNodes).stream()
+            .map(dn -> getDatanodeDesriptor(namesystem, dn.getDatanodeUuid()))
+            .collect(Collectors.toList());
+    assertEquals(numDeadNodes, deadNodes.size());
+
+    // Create file with block replicas on all nodes
+    writeFile(getCluster().getFileSystem(), filePath, numNodes, 10);
+
+    // Cause the "dead" nodes to be lost while in state decommissioning
+    // and fill the tracked nodes set with those "dead" nodes
+    for (final DatanodeDescriptor deadNode : deadNodes) {
+      // Start decommissioning the node, it will not be able to complete due to the
+      // under-replicated file
+      takeNodeOutofService(0, deadNode.getDatanodeUuid(), 0, decommissionedNodes,
+          AdminStates.DECOMMISSION_INPROGRESS);
+      decommissionedNodes.add(deadNode);
+
+      // Stop the datanode so that it is lost while decommissioning
+      MiniDFSCluster.DataNodeProperties dn = getCluster().stopDataNode(deadNode.getXferAddr());
+      deadNodeProps.put(deadNode, dn);
+      deadNode.setLastUpdate(213); // Set last heartbeat to be in the past
+    }
+    assertEquals(numDeadNodes, deadNodeProps.size());
+
+    // Wait for the decommissioning nodes to become dead & to be added to "pendingNodes"
+    GenericTestUtils.waitFor(() -> decomManager.getNumTrackedNodes() == 0
+        && decomManager.getNumPendingNodes() == numDeadNodes
+        && deadNodes.stream().allMatch(node ->
+            !BlockManagerTestUtil.isNodeHealthyForDecommissionOrMaintenance(blockManager, node)
+                && !node.isAlive()), 500, 20000);
+  }
+
+  /*
+  This test reproduces a scenario where an under-replicated block on a decommissioning node
+  cannot be replicated to some datanodes because they have a corrupt replica of the block.
+  The test ensures that the corrupt replicas are eventually invalidated so that the
+  under-replicated block can be replicated to sufficient datanodes & the decommissioning
+  node can be decommissioned.
+   */
+  @Test(timeout = 60000)
+  public void testDeleteCorruptReplicaForUnderReplicatedBlock() throws Exception {
+    // Constants
+    final Path file = new Path("/test-file");
+    final int numDatanode = 3;
+    final short replicationFactor = 2;
+    final int numStoppedNodes = 2;
+    final int numDecommNodes = 1;
+    assertEquals(numDatanode, numStoppedNodes + numDecommNodes);
+
+    // Run monitor every 5 seconds to speed up decommissioning & make the test faster
+    final int datanodeAdminMonitorFixedRateSeconds = 5;
+    getConf().setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY,
+        datanodeAdminMonitorFixedRateSeconds);
+    // Set block report interval to 6 hours to avoid unexpected block reports.
+    // The default block report interval is different for a MiniDFSCluster
+    getConf().setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+        DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT);
+    // Run the BlockManager RedundancyMonitor every 3 seconds such that the Namenode
+    // sends under-replication blocks for replication frequently
+    getConf().setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+        DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT);
+    // Ensure that the DataStreamer client will replace the bad datanode on append failure
+    getConf().set(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY, "ALWAYS");
+    // Avoid having the DataStreamer client fail the append operation if datanode replacement fails
+    getConf()
+        .setBoolean(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_KEY, true);
+
+    // References to datanodes in the cluster
+    // - 2 datanode will be stopped to generate corrupt block replicas & then
+    //     restarted later to validate the corrupt replicas are invalidated
+    // - 1 datanode will start decommissioning to make the block under replicated
+    final List<DatanodeDescriptor> allNodes = new ArrayList<>();
+    final List<DatanodeDescriptor> stoppedNodes = new ArrayList<>();
+    final DatanodeDescriptor decommNode;
+
+    // Create MiniDFSCluster
+    startCluster(1, numDatanode);
+    getCluster().waitActive();
+    final FSNamesystem namesystem = getCluster().getNamesystem();
+    final BlockManager blockManager = namesystem.getBlockManager();
+    final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
+    final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager();
+    final FileSystem fs = getCluster().getFileSystem();
+
+    // Get DatanodeDescriptors
+    for (final DataNode node : getCluster().getDataNodes()) {
+      allNodes.add(getDatanodeDesriptor(namesystem, node.getDatanodeUuid()));
+    }
+
+    // Create block with 2 FINALIZED replicas
+    // Note that:
+    // - calling hflush leaves block in state ReplicaBeingWritten
+    // - calling close leaves the block in state FINALIZED
+    // - amount of data is kept small because flush is not synchronous
+    LOG.info("Creating Initial Block with {} FINALIZED replicas", replicationFactor);
+    FSDataOutputStream out = fs.create(file, replicationFactor);
+    for (int i = 0; i < 512; i++) {
+      out.write(i);
+    }
+    out.close();
+
+    // Validate the block exists with expected number of replicas
+    assertEquals(1, blockManager.getTotalBlocks());
+    BlockLocation[] blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+    assertEquals(1, blocksInFile.length);
+    List<String> replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+    assertEquals(replicationFactor, replicasInBlock.size());
+
+    // Identify the DatanodeDescriptors associated with the 2 nodes with replicas.
+    // Each of nodes with a replica will be stopped later to corrupt the replica
+    DatanodeDescriptor decommNodeTmp = null;
+    for (DatanodeDescriptor node : allNodes) {
+      if (replicasInBlock.contains(node.getName())) {
+        stoppedNodes.add(node);
+      } else {
+        decommNodeTmp = node;
+      }
+    }
+    assertEquals(numStoppedNodes, stoppedNodes.size());
+    assertNotNull(decommNodeTmp);
+    decommNode = decommNodeTmp;
+    final DatanodeDescriptor firstStoppedNode = stoppedNodes.get(0);
+    final DatanodeDescriptor secondStoppedNode = stoppedNodes.get(1);
+    LOG.info("Detected 2 nodes with replicas : {} , {}", firstStoppedNode.getXferAddr(),
+        secondStoppedNode.getXferAddr());
+    LOG.info("Detected 1 node without replica : {}", decommNode.getXferAddr());
+
+    // Stop firstStoppedNode & the append to the block pipeline such that DataStreamer client:
+    // - detects firstStoppedNode as bad link in block pipeline
+    // - replaces the firstStoppedNode with decommNode in block pipeline
+    // The result is that:
+    // - secondStoppedNode & decommNode have a live block replica
+    // - firstStoppedNode has a corrupt replica (corrupt because of old GenStamp)
+    LOG.info("Stopping first node with replica {}", firstStoppedNode.getXferAddr());
+    final List<MiniDFSCluster.DataNodeProperties> stoppedNodeProps = new ArrayList<>();
+    MiniDFSCluster.DataNodeProperties stoppedNodeProp =
+        getCluster().stopDataNode(firstStoppedNode.getXferAddr());
+    stoppedNodeProps.add(stoppedNodeProp);
+    firstStoppedNode.setLastUpdate(213); // Set last heartbeat to be in the past
+    // Wait for NN to detect the datanode as dead
+    GenericTestUtils.waitFor(
+        () -> 2 == datanodeManager.getNumLiveDataNodes() && 1 == datanodeManager
+            .getNumDeadDataNodes(), 500, 30000);
+    // Append to block pipeline
+    appendBlock(fs, file, 2);
+
+    // Stop secondStoppedNode & the append to the block pipeline such that DataStreamer client:
+    // - detects secondStoppedNode as bad link in block pipeline
+    // - attempts to replace secondStoppedNode but cannot because there are no more live nodes
+    // - appends to the block pipeline containing just decommNode
+    // The result is that:
+    // - decommNode has a live block replica
+    // - firstStoppedNode & secondStoppedNode both have a corrupt replica
+    LOG.info("Stopping second node with replica {}", secondStoppedNode.getXferAddr());
+    stoppedNodeProp = getCluster().stopDataNode(secondStoppedNode.getXferAddr());
+    stoppedNodeProps.add(stoppedNodeProp);
+    secondStoppedNode.setLastUpdate(213); // Set last heartbeat to be in the past
+    // Wait for NN to detect the datanode as dead
+    GenericTestUtils.waitFor(() -> numDecommNodes == datanodeManager.getNumLiveDataNodes()
+        && numStoppedNodes == datanodeManager.getNumDeadDataNodes(), 500, 30000);
+    // Append to block pipeline
+    appendBlock(fs, file, 1);
+
+    // Validate block replica locations
+    blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+    assertEquals(1, blocksInFile.length);
+    replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+    assertEquals(numDecommNodes, replicasInBlock.size());
+    assertTrue(replicasInBlock.contains(decommNode.getName()));
+    LOG.info("Block now has 2 corrupt replicas on [{} , {}] and 1 live replica on {}",
+        firstStoppedNode.getXferAddr(), secondStoppedNode.getXferAddr(), decommNode.getXferAddr());
+
+    LOG.info("Decommission node {} with the live replica", decommNode.getXferAddr());
+    final ArrayList<DatanodeInfo> decommissionedNodes = new ArrayList<>();
+    takeNodeOutofService(0, decommNode.getDatanodeUuid(), 0, decommissionedNodes,
+        AdminStates.DECOMMISSION_INPROGRESS);
+
+    // Wait for the datanode to start decommissioning
+    try {
+      GenericTestUtils.waitFor(() -> decomManager.getNumTrackedNodes() == 0
+          && decomManager.getNumPendingNodes() == numDecommNodes && decommNode.getAdminState()
+          .equals(AdminStates.DECOMMISSION_INPROGRESS), 500, 30000);
+    } catch (Exception e) {
+      blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+      assertEquals(1, blocksInFile.length);
+      replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+      String errMsg = String.format("Node %s failed to start decommissioning."
+              + " numTrackedNodes=%d , numPendingNodes=%d , adminState=%s , nodesWithReplica=[%s]",
+          decommNode.getXferAddr(), decomManager.getNumTrackedNodes(),
+          decomManager.getNumPendingNodes(), decommNode.getAdminState(),
+          String.join(", ", replicasInBlock));
+      LOG.error(errMsg); // Do not log generic timeout exception
+      fail(errMsg);
+    }
+
+    // Validate block replica locations
+    blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+    assertEquals(1, blocksInFile.length);
+    replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+    assertEquals(numDecommNodes, replicasInBlock.size());
+    assertEquals(replicasInBlock.get(0), decommNode.getName());
+    LOG.info("Block now has 2 corrupt replicas on [{} , {}] and 1 decommissioning replica on {}",
+        firstStoppedNode.getXferAddr(), secondStoppedNode.getXferAddr(), decommNode.getXferAddr());
+
+    // Restart the 2 stopped datanodes
+    LOG.info("Restarting stopped nodes {} , {}", firstStoppedNode.getXferAddr(),
+        secondStoppedNode.getXferAddr());
+    for (final MiniDFSCluster.DataNodeProperties stoppedNode : stoppedNodeProps) {
+      assertTrue(getCluster().restartDataNode(stoppedNode));
+    }
+    for (final MiniDFSCluster.DataNodeProperties stoppedNode : stoppedNodeProps) {
+      try {
+        getCluster().waitDatanodeFullyStarted(stoppedNode.getDatanode(), 30000);
+        LOG.info("Node {} Restarted", stoppedNode.getDatanode().getXferAddress());
+      } catch (Exception e) {
+        String errMsg = String.format("Node %s Failed to Restart within 30 seconds",
+            stoppedNode.getDatanode().getXferAddress());
+        LOG.error(errMsg); // Do not log generic timeout exception
+        fail(errMsg);
+      }
+    }
+
+    // Trigger block reports for the 2 restarted nodes to ensure their corrupt
+    // block replicas are identified by the namenode
+    for (MiniDFSCluster.DataNodeProperties dnProps : stoppedNodeProps) {
+      DataNodeTestUtils.triggerBlockReport(dnProps.getDatanode());
+    }
+
+    // Validate the datanode is eventually decommissioned
+    // Some changes are needed to ensure replication/decommissioning occur in a timely manner:
+    // - if the namenode sends a DNA_TRANSFER before sending the DNA_INVALIDATE's then:
+    //   - the block will enter the pendingReconstruction queue
+    //   - this prevent the block from being sent for transfer again for some time
+    // - solution is to call "clearQueues" so that DNA_TRANSFER is sent again after DNA_INVALIDATE
+    // - need to run the check less frequently than DatanodeAdminMonitor
+    //       such that in between "clearQueues" calls 2 things can occur:
+    //   - DatanodeAdminMonitor runs which sets the block as neededReplication
+    //   - datanode heartbeat is received which sends the DNA_TRANSFER to the node
+    final int checkEveryMillis = datanodeAdminMonitorFixedRateSeconds * 2 * 1000;
+    try {
+      GenericTestUtils.waitFor(() -> {
+        blockManager.clearQueues(); // Clear pendingReconstruction queue
+        return decomManager.getNumTrackedNodes() == 0 && decomManager.getNumPendingNodes() == 0
+            && decommNode.getAdminState().equals(AdminStates.DECOMMISSIONED);
+      }, checkEveryMillis, 40000);
+    } catch (Exception e) {
+      blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+      assertEquals(1, blocksInFile.length);
+      replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+      String errMsg = String.format("Node %s failed to complete decommissioning."
+              + " numTrackedNodes=%d , numPendingNodes=%d , adminState=%s , nodesWithReplica=[%s]",
+          decommNode.getXferAddr(), decomManager.getNumTrackedNodes(),
+          decomManager.getNumPendingNodes(), decommNode.getAdminState(),
+          String.join(", ", replicasInBlock));
+      LOG.error(errMsg); // Do not log generic timeout exception
+      fail(errMsg);
+    }
+
+    // Validate block replica locations.
+    // Note that in order for decommissioning to complete the block must be
+    // replicated to both of the restarted datanodes; this implies that the
+    // corrupt replicas were invalidated on both of the restarted datanodes.
+    blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+    assertEquals(1, blocksInFile.length);
+    replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+    assertEquals(numDatanode, replicasInBlock.size());
+    assertTrue(replicasInBlock.contains(decommNode.getName()));
+    for (final DatanodeDescriptor node : stoppedNodes) {
+      assertTrue(replicasInBlock.contains(node.getName()));
+    }
+    LOG.info("Block now has 2 live replicas on [{} , {}] and 1 decommissioned replica on {}",
+        firstStoppedNode.getXferAddr(), secondStoppedNode.getXferAddr(), decommNode.getXferAddr());
+  }
+
+  void appendBlock(final FileSystem fs, final Path file, int expectedReplicas) throws IOException {
+    LOG.info("Appending to the block pipeline");
+    boolean failed = false;
+    Exception failedReason = null;
+    try {
+      FSDataOutputStream out = fs.append(file);
+      for (int i = 0; i < 512; i++) {
+        out.write(i);
+      }
+      out.close();
+    } catch (Exception e) {
+      failed = true;
+      failedReason = e;
+    } finally {
+      BlockLocation[] blocksInFile = fs.getFileBlockLocations(file, 0, 0);
+      assertEquals(1, blocksInFile.length);
+      List<String> replicasInBlock = Arrays.asList(blocksInFile[0].getNames());
+      if (failed) {
+        String errMsg = String.format(
+            "Unexpected exception appending to the block pipeline."
+                + " nodesWithReplica=[%s]", String.join(", ", replicasInBlock));
+        LOG.error(errMsg, failedReason); // Do not swallow the exception
+        fail(errMsg);
+      } else if (expectedReplicas != replicasInBlock.size()) {
+        String errMsg = String.format("Expecting %d replicas in block pipeline,"
+                + " unexpectedly found %d replicas. nodesWithReplica=[%s]", expectedReplicas,
+            replicasInBlock.size(), String.join(", ", replicasInBlock));
+        LOG.error(errMsg);
+        fail(errMsg);
+      } else {
+        String infoMsg = String.format(
+            "Successfully appended block pipeline with %d replicas."
+                + " nodesWithReplica=[%s]",
+            replicasInBlock.size(), String.join(", ", replicasInBlock));
+        LOG.info(infoMsg);
+      }
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
index c68cb1707c2fe..206f75eae7035 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java
@@ -759,7 +759,7 @@ public void testDecommissionWithFailedReplicating() throws Exception {
     DatanodeInfo extraDn = getDatanodeOutOfTheBlock(blk);
     DatanodeDescriptor target = bm.getDatanodeManager()
         .getDatanode(extraDn.getDatanodeUuid());
-    dn0.addBlockToBeReplicated(targetBlk,
+    dn0.addECBlockToBeReplicated(targetBlk,
         new DatanodeStorageInfo[] {target.getStorageInfos()[0]});
 
     // dn0 replicates in success
@@ -883,7 +883,7 @@ public void testDecommissionWithMissingBlock() throws Exception {
         .getDatanode(extraDn.getDatanodeUuid());
     DatanodeDescriptor dnStartIndexDecommission = bm.getDatanodeManager()
         .getDatanode(dnLocs[decommNodeIndex].getDatanodeUuid());
-    dnStartIndexDecommission.addBlockToBeReplicated(targetBlk,
+    dnStartIndexDecommission.addECBlockToBeReplicated(targetBlk,
         new DatanodeStorageInfo[] {target.getStorageInfos()[0]});
 
     // Wait for replication success.
@@ -972,7 +972,7 @@ public void testCountNodes() throws Exception{
     DatanodeInfo extraDn = getDatanodeOutOfTheBlock(blk);
     DatanodeDescriptor target = bm.getDatanodeManager()
         .getDatanode(extraDn.getDatanodeUuid());
-    dn0.addBlockToBeReplicated(targetBlk,
+    dn0.addECBlockToBeReplicated(targetBlk,
         new DatanodeStorageInfo[] {target.getStorageInfos()[0]});
 
     // dn0 replicates in success
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 1900160ed912f..12bc75a9f78a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -105,6 +105,7 @@
 import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension;
 import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.XMLUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -153,7 +154,6 @@
 import org.xml.sax.helpers.DefaultHandler;
 
 import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
 
 public class TestEncryptionZones {
   static final Logger LOG = LoggerFactory.getLogger(TestEncryptionZones.class);
@@ -1734,7 +1734,7 @@ public void testOfflineImageViewerOnEncryptionZones() throws Exception {
     PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), pw);
     v.visit(new RandomAccessFile(originalFsimage, "r"));
     final String xml = output.toString();
-    SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+    SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
     parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 835d18f3a08ae..1fd909c949a12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -747,6 +747,15 @@ public void testAddErasureCodingPolicies() throws Exception {
     assertEquals(1, responses.length);
     assertFalse(responses[0].isSucceed());
 
+    // Test numDataUnits + numParityUnits > 16
+    toAddSchema = new ECSchema("rs", 14, 4);
+    newPolicy =
+        new ErasureCodingPolicy(toAddSchema, 128 * 1024 * 1024);
+    policyArray = new ErasureCodingPolicy[]{newPolicy};
+    responses = fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertFalse(responses[0].isSucceed());
+
     // Test too big cell size
     toAddSchema = new ECSchema("rs", 3, 2);
     newPolicy =
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
index 3e9adcac7e7e0..a7663b0ba9aa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend3.java
@@ -26,7 +26,6 @@
 import java.util.List;
 
 import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils.MaterializedReplica;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.mockito.invocation.InvocationOnMock;
 import static org.mockito.Mockito.spy;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
index c19d8c3e4a5e2..29ab1baa705ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java
@@ -215,6 +215,39 @@ public void testStripedAndReplicatedFileChecksum() throws Exception {
     }
   }
 
+  /**
+   * Test the corner case of the COMPOSITE_CRC.
+   * For Stripe File, last block size in the file is (int)(blockSize * 0.5),
+   *    but the last block size in the check length is (int)(blockSize * 0.6).
+   * For Replicate File, the last block size in the file is (int)(blockSize * 0.5),
+   *    but the last block size in the check length is ((dataBlocks - 1) * blockSize
+   *    + (int) (blockSize * 0.6))
+   */
+  @Test(timeout = 90000)
+  public void testStripedAndReplicatedFileChecksum2() throws Exception {
+    final int lastBlockSize = (int) (blockSize * 0.5);
+    final int fullStripeLength = dataBlocks * blockSize;
+    final int testFileSize = fullStripeLength + lastBlockSize;
+    prepareTestFiles(testFileSize, new String[] {stripedFile1, replicatedFile});
+
+    final int specialLength = (dataBlocks - 1) * blockSize
+        + (int) (blockSize * 0.6);
+
+    Assert.assertTrue(specialLength % blockSize > lastBlockSize);
+    Assert.assertTrue(specialLength % fullStripeLength > lastBlockSize);
+
+    FileChecksum stripedFileChecksum = getFileChecksum(stripedFile1,
+        specialLength, false);
+    FileChecksum replicatedFileChecksum = getFileChecksum(replicatedFile,
+        specialLength, false);
+
+    if (checksumCombineMode.equals(ChecksumCombineMode.COMPOSITE_CRC.name())) {
+      Assert.assertEquals(replicatedFileChecksum, stripedFileChecksum);
+    } else {
+      Assert.assertNotEquals(replicatedFileChecksum, stripedFileChecksum);
+    }
+  }
+
   @Test(timeout = 90000)
   public void testDifferentBlockSizeReplicatedFileChecksum() throws Exception {
     byte[] fileData = StripedFileTestUtil.generateBytes(fileSize);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index a7cf68b10168f..2c41c410ca3f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -75,6 +75,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -272,7 +273,7 @@ public void testServerDefaultsWithMinimalCaching() throws Exception  {
               defaults.getDefaultStoragePolicyId());
       doReturn(newDefaults).when(spyNamesystem).getServerDefaults();
 
-      // Verify that the value is updated correctly. Wait for 3 seconds.
+      // Verify that the value is updated correctly. Wait for 6 seconds.
       GenericTestUtils.waitFor(()->{
         try {
           FsServerDefaults currDef = dfsClient.getServerDefaults();
@@ -281,7 +282,7 @@ public void testServerDefaultsWithMinimalCaching() throws Exception  {
           // do nothing;
           return false;
         }
-      }, 1, 3000);
+      }, 1, 6000);
 
     } finally {
       cluster.shutdown();
@@ -1356,6 +1357,8 @@ public void testFileCreationWithOverwrite() throws Exception {
       assertBlocks(bm, oldBlocks, true);
       
       out = dfs.create(filePath, true);
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
       byte[] newData = AppendTestUtil.randomBytes(seed, fileSize);
       try {
         out.write(newData);
@@ -1363,6 +1366,8 @@ public void testFileCreationWithOverwrite() throws Exception {
         out.close();
       }
       dfs.deleteOnExit(filePath);
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
       
       LocatedBlocks newBlocks = NameNodeAdapter.getBlockLocations(
           nn, file, 0, fileSize);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 1ee166e6cd3f1..2c366e2934c34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -468,4 +469,68 @@ public void testGetBlocksWithHotBlockTimeInterval() throws Exception {
       cluster.shutdown();
     }
   }
+
+  @Test
+  public void testReadSkipStaleStorage() throws Exception {
+    final short repFactor = (short) 1;
+    final int blockNum = 64;
+    final int storageNum = 2;
+    final int fileLen = BLOCK_SIZE * blockNum;
+    final Path path = new Path("testReadSkipStaleStorage");
+    final Configuration conf = new HdfsConfiguration();
+
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(1)
+        .storagesPerDatanode(storageNum)
+        .build();
+    cluster.waitActive();
+
+    FileSystem fs = cluster.getFileSystem();
+    DFSTestUtil.createFile(fs, path, false, 1024, fileLen,
+        BLOCK_SIZE, repFactor, 0, true);
+
+    // get datanode info
+    ClientProtocol client = NameNodeProxies.createProxy(conf,
+        cluster.getFileSystem(0).getUri(),
+        ClientProtocol.class).getProxy();
+    DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL);
+
+    // get storage info
+    BlockManager bm0 = cluster.getNamesystem(0).getBlockManager();
+    DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager()
+        .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos();
+
+    InetSocketAddress addr = new InetSocketAddress("localhost",
+        cluster.getNameNodePort());
+    NamenodeProtocol namenode = NameNodeProxies.createProxy(conf,
+        DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy();
+
+    // check blocks count equals to blockNum
+    BlockWithLocations[] blocks = namenode.getBlocks(
+        dataNodes[0], fileLen*2, 0, 0).getBlocks();
+    assertEquals(blockNum, blocks.length);
+
+    // calculate the block count on storage[0]
+    int count = 0;
+    for (BlockWithLocations b : blocks) {
+      for (String s : b.getStorageIDs()) {
+        if (s.equals(storageInfos[0].getStorageID())) {
+          count++;
+        }
+      }
+    }
+
+    // set storage[0] stale
+    storageInfos[0].setBlockContentsStale(true);
+    blocks = namenode.getBlocks(
+        dataNodes[0], fileLen*2, 0, 0).getBlocks();
+    assertEquals(blockNum - count, blocks.length);
+
+    // set all storage stale
+    bm0.getDatanodeManager().markAllDatanodesStale();
+    blocks = namenode.getBlocks(
+        dataNodes[0], fileLen*2, 0, 0).getBlocks();
+    assertEquals(0, blocks.length);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
index 6da46dec378d2..9de5813872f1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
@@ -63,6 +63,11 @@ protected String getDefaultWorkingDirectory() {
     return defaultWorkingDirectory;
   }
 
+  @Override
+  protected int getGlobalTimeout() {
+    return 60 * 1000;
+  }
+
   @Test
   public void testAppend() throws IOException {
     AppendTestUtil.testAppend(fs, new Path("/testAppend/f"));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
index 0a74a2d025659..6f43f858c1d96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
@@ -19,9 +19,7 @@
 
 import static org.junit.Assert.*;
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
index 9fc6b3894940c..58011d7f15fce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
@@ -32,6 +32,8 @@ public class TestKeyProviderCache {
 
   public static class DummyKeyProvider extends KeyProvider {
 
+    public static int CLOSE_CALL_COUNT = 0;
+
     public DummyKeyProvider(Configuration conf) {
       super(conf);
     }
@@ -76,6 +78,10 @@ public KeyVersion rollNewVersion(String name, byte[] material)
     public void flush() throws IOException {
     }
 
+    @Override
+    public void close() {
+      CLOSE_CALL_COUNT += 1;
+    }
   }
 
   public static class Factory extends KeyProviderFactory {
@@ -124,6 +130,9 @@ public void testCache() throws Exception {
     Assert.assertFalse("Same KeyProviders returned !!",
         keyProvider1 == keyProvider4);
 
+    kpCache.invalidateCache();
+    Assert.assertEquals("Expected number of closing calls doesn't match",
+        3, DummyKeyProvider.CLOSE_CALL_COUNT);
   }
 
   private URI getKeyProviderUriFromConf(Configuration conf) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
index 5d7b62a42846a..8b527d07a29b8 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -90,7 +90,8 @@ public void testLeaseAbort() throws Exception {
 
       // stub the renew method.
       doThrow(new RemoteException(InvalidToken.class.getName(),
-          "Your token is worthless")).when(spyNN).renewLease(anyString());
+          "Your token is worthless")).when(spyNN).renewLease(
+              anyString(), any());
 
       // We don't need to wait the lease renewer thread to act.
       // call renewLease() manually.
@@ -131,7 +132,7 @@ public void testLeaseAbort() throws Exception {
       Assert.assertTrue(originalRenewer.isEmpty());
 
       // unstub
-      doNothing().when(spyNN).renewLease(anyString());
+      doNothing().when(spyNN).renewLease(anyString(), any());
 
       // existing input streams should work
       try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
index 5aa1f6383704f..cc3e5ed2c7328 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.function.Supplier;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.hadoop.conf.Configuration;
@@ -188,6 +188,24 @@ public void testLeaseRecovery() throws Exception {
     }
   }
 
+  @Test
+  public void testSafeLength() {
+    checkSafeLength(0, 0); // Length of: 0
+    checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
+    checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
+    checkSafeLength(189729792, 1132462080L); // Length of: 189729792
+    checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
+    checkSafeLength(517399040, 3101687808L); // Length of: 517399040
+    checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
+  }
+
+  private void checkSafeLength(int blockLength, long expectedSafeLength) {
+    int[] blockLengths = new int[]{blockLength, blockLength, blockLength, blockLength,
+        blockLength, blockLength};
+    long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
+    Assert.assertEquals(expectedSafeLength, safeLength);
+  }
+
   private void runTest(int[] blockLengths, long safeLength) throws Exception {
     writePartialBlocks(blockLengths);
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocatedBlocksRefresher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocatedBlocksRefresher.java
new file mode 100644
index 0000000000000..5d8c479f2f46d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocatedBlocksRefresher.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestLocatedBlocksRefresher {
+  private static final Logger LOG = LoggerFactory.getLogger(TestLocatedBlocksRefresher.class);
+
+  private static final int BLOCK_SIZE = 1024 * 1024;
+  private static final short REPLICATION_FACTOR = (short) 4;
+  private static final String[] RACKS = new String[] {
+      "/d1/r1", "/d1/r1", "/d1/r2", "/d1/r2", "/d1/r2", "/d2/r3", "/d2/r3" };
+  private static final int NUM_DATA_NODES = RACKS.length;
+
+  private final int numOfBlocks = 24;
+  private final int fileLength = numOfBlocks * BLOCK_SIZE;
+  private final int dfsClientPrefetchSize = fileLength / 2;
+
+  private MiniDFSCluster cluster;
+  private Configuration conf;
+
+  @Before
+  public void setUp() throws Exception {
+    cluster = null;
+    conf = new HdfsConfiguration();
+
+    // disable shortcircuit reading
+    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, false);
+    // set replication factor
+    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION_FACTOR);
+    // set block size and other sizes
+    conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setLong(HdfsClientConfigKeys.Read.PREFETCH_SIZE_KEY,
+        dfsClientPrefetchSize);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (cluster != null) {
+      cluster.shutdown(true, true);
+    }
+  }
+
+  private void setupTest(long refreshInterval) throws IOException {
+    conf.setLong(DFS_CLIENT_REFRESH_READ_BLOCK_LOCATIONS_MS_KEY, refreshInterval);
+
+    // this is necessary to ensure no caching between runs
+    conf.set("dfs.client.context", UUID.randomUUID().toString());
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).racks(RACKS).build();
+    cluster.waitActive();
+  }
+
+  @Test
+  public void testDisabledOnZeroInterval() throws IOException {
+    setupTest(0);
+    assertNull(cluster.getFileSystem().getClient().getLocatedBlockRefresher());
+  }
+
+  @Test
+  public void testEnabledOnNonZeroInterval() throws Exception {
+    setupTest(1000);
+    LocatedBlocksRefresher refresher =
+        cluster.getFileSystem().getClient().getLocatedBlockRefresher();
+    assertNotNull(refresher);
+    assertNoMoreRefreshes(refresher);
+  }
+
+  @Test
+  public void testRefreshOnDeadNodes() throws Exception {
+    setupTest(1000);
+    DistributedFileSystem fs = cluster.getFileSystem();
+    DFSClient client = fs.getClient();
+    LocatedBlocksRefresher refresher = client.getLocatedBlockRefresher();
+
+    String fileName = createTestFile(fs);
+
+    try (DFSInputStream fin = client.open(fileName)) {
+      LocatedBlocks locatedBlocks = fin.locatedBlocks;
+      assertEquals(dfsClientPrefetchSize / BLOCK_SIZE,
+          locatedBlocks.locatedBlockCount());
+
+      // should not be tracked yet
+      assertFalse(refresher.isInputStreamTracked(fin));
+
+      // track and verify
+      refresher.addInputStream(fin);
+      assertTrue(refresher.isInputStreamTracked(fin));
+
+      // no refreshes yet, as nothing has happened
+      assertNoMoreRefreshes(refresher);
+      synchronized (fin.infoLock) {
+        assertSame(locatedBlocks, fin.locatedBlocks);
+      }
+
+      stopNodeHostingBlocks(fin, NUM_DATA_NODES - 1);
+
+      // read blocks, which should trigger dead node for the one we stopped
+      int chunkReadSize = BLOCK_SIZE / 4;
+      byte[] readBuffer = new byte[chunkReadSize];
+      fin.read(0, readBuffer, 0, readBuffer.length);
+
+      assertEquals(1, fin.getLocalDeadNodes().size());
+
+      // we should get a refresh now
+      assertRefreshes(refresher, 1);
+
+      // verify that it actually changed things
+      synchronized (fin.infoLock) {
+        assertNotSame(locatedBlocks, fin.locatedBlocks);
+        assertTrue(fin.getLocalDeadNodes().isEmpty());
+      }
+
+      // no more refreshes because everything is happy again
+      assertNoMoreRefreshes(refresher);
+
+      // stop another node, and try to trigger a new deadNode
+      stopNodeHostingBlocks(fin, NUM_DATA_NODES - 2);
+      readBuffer = new byte[chunkReadSize];
+      fin.read(0, readBuffer, 0, readBuffer.length);
+
+      // we should refresh again now, and verify
+      // may actually be more than 1, since the first dead node
+      // may still be listed in the replicas for the bock
+      assertTrue(fin.getLocalDeadNodes().size() > 0);
+
+      assertRefreshes(refresher, 1);
+
+      synchronized (fin.infoLock) {
+        assertNotSame(locatedBlocks, fin.locatedBlocks);
+        assertTrue(fin.getLocalDeadNodes().isEmpty());
+      }
+
+      // de-register, and expect no more refreshes below
+      refresher.removeInputStream(fin);
+    }
+
+    assertNoMoreRefreshes(refresher);
+  }
+
+  private void stopNodeHostingBlocks(DFSInputStream fin, int expectedNodes) {
+    synchronized (fin.infoLock) {
+      int idx = fin.locatedBlocks.findBlock(0);
+      for (int i = 0; i < REPLICATION_FACTOR; i++) {
+        String deadNodeAddr = fin.locatedBlocks.get(idx).getLocations()[i].getXferAddr();
+
+        DataNodeProperties dataNodeProperties = cluster.stopDataNode(deadNodeAddr);
+        if (dataNodeProperties != null) {
+          List<DataNode> datanodesPostStoppage = cluster.getDataNodes();
+          assertEquals(expectedNodes, datanodesPostStoppage.size());
+          return;
+        }
+      }
+
+      throw new RuntimeException("Could not find a datanode to stop");
+    }
+  }
+
+  private void assertNoMoreRefreshes(LocatedBlocksRefresher refresher) throws InterruptedException {
+    long interval = refresher.getInterval();
+    int runCount = refresher.getRunCount();
+    int refreshCount = refresher.getRefreshCount();
+
+    LOG.info("Waiting for at least {} runs, from current {}, expecting no refreshes",
+        runCount + 3, runCount);
+    // wait for it to run 3 times, with some buffer
+    awaitWithTimeout(() -> refresher.getRunCount() > runCount + 3, 5 * interval);
+
+    // it should not have refreshed anything, because no DFSInputStreams registered anymore
+    assertEquals(refreshCount, refresher.getRefreshCount());
+  }
+
+  private void assertRefreshes(LocatedBlocksRefresher refresher, int expectedRefreshes)
+      throws InterruptedException {
+    int runCount = refresher.getRunCount();
+    int refreshCount = refresher.getRefreshCount();
+    int expectedRuns = 3;
+
+    if (expectedRefreshes < 0) {
+      expectedRefreshes = expectedRuns;
+    }
+
+    LOG.info(
+        "Waiting for at least {} runs, from current {}. Expecting {} refreshes, from current {}",
+        runCount + expectedRuns, runCount, refreshCount + expectedRefreshes, refreshCount
+    );
+
+    // wait for it to run 3 times
+    awaitWithTimeout(() -> refresher.getRunCount() >= runCount + expectedRuns, 10_000);
+
+    // the values may not be identical due to any refreshes that occurred before we opened
+    // the DFSInputStream but the difference should be identical since we are refreshing
+    // every time
+    assertEquals(expectedRefreshes, refresher.getRefreshCount() - refreshCount);
+  }
+
+  private void awaitWithTimeout(Supplier<Boolean> test, long timeoutMillis)
+      throws InterruptedException {
+    long now = Time.monotonicNow();
+
+    while(!test.get()) {
+      if (Time.monotonicNow() - now > timeoutMillis) {
+        fail("Timed out waiting for true condition");
+        return;
+      }
+
+      Thread.sleep(50);
+    }
+  }
+
+  private String createTestFile(FileSystem fs) throws IOException {
+    String fileName = "/located_blocks_" + UUID.randomUUID().toString();
+    Path filePath = new Path(fileName);
+    try (FSDataOutputStream fout = fs.create(filePath, REPLICATION_FACTOR)) {
+      fout.write(new byte[(fileLength)]);
+    }
+    fs.deleteOnExit(filePath);
+
+    return fileName;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceWithStriped.java
new file mode 100644
index 0000000000000..2e17b9681b790
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceWithStriped.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.CombinedHostFileManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.HostConfigManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.util.HostsFileWriter;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class tests the in maintenance of datanode with striped blocks.
+ */
+public class TestMaintenanceWithStriped {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMaintenanceWithStriped.class);
+
+  // heartbeat interval in seconds
+  private static final int HEARTBEAT_INTERVAL = 1;
+  // block report in msec
+  private static final int BLOCKREPORT_INTERVAL_MSEC = 1000;
+  // replication interval
+  private static final int NAMENODE_REPLICATION_INTERVAL = 1;
+
+  private Configuration conf;
+  private MiniDFSCluster cluster;
+  private DistributedFileSystem dfs;
+  private final ErasureCodingPolicy ecPolicy =
+      StripedFileTestUtil.getDefaultECPolicy();
+  private int numDNs;
+  private final int cellSize = ecPolicy.getCellSize();
+  private final int dataBlocks = ecPolicy.getNumDataUnits();
+  private final int parityBlocks = ecPolicy.getNumParityUnits();
+  private final int blockSize = cellSize * 4;
+  private final int blockGroupSize = blockSize * dataBlocks;
+  private final Path ecDir = new Path("/" + this.getClass().getSimpleName());
+  private HostsFileWriter hostsFileWriter;
+  private boolean useCombinedHostFileManager = true;
+
+  private FSNamesystem fsn;
+  private BlockManager bm;
+
+  protected Configuration createConfiguration() {
+    return new HdfsConfiguration();
+  }
+
+  @Before
+  public void setup() throws IOException {
+    // Set up the hosts/exclude files.
+    hostsFileWriter = new HostsFileWriter();
+    conf = createConfiguration();
+    if (useCombinedHostFileManager) {
+      conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY,
+          CombinedHostFileManager.class, HostConfigManager.class);
+    }
+    hostsFileWriter.initialize(conf, "temp/admin");
+
+
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+        2000);
+    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+    conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+        BLOCKREPORT_INTERVAL_MSEC);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
+        4);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+        NAMENODE_REPLICATION_INTERVAL);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    conf.setInt(
+        DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
+        cellSize - 1);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
+        false);
+
+    numDNs = dataBlocks + parityBlocks + 5;
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+    cluster.waitActive();
+    dfs = cluster.getFileSystem(0);
+    fsn = cluster.getNamesystem();
+    bm = fsn.getBlockManager();
+
+    dfs.enableErasureCodingPolicy(
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+    dfs.mkdirs(ecDir);
+    dfs.setErasureCodingPolicy(ecDir,
+        StripedFileTestUtil.getDefaultECPolicy().getName());
+  }
+
+  @After
+  public void teardown() throws IOException {
+    hostsFileWriter.cleanup();
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  /**
+   * test DN maintenance with striped blocks.
+   * @throws Exception
+   */
+  @Test(timeout = 120000)
+  public void testInMaintenance() throws Exception {
+    //1. create EC file
+    // d0 d1 d2 d3 d4 d5 d6 d7 d8
+    final Path ecFile = new Path(ecDir, "testInMaintenance");
+    int writeBytes = cellSize * dataBlocks;
+    writeStripedFile(dfs, ecFile, writeBytes);
+    Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
+    FileChecksum fileChecksum1 = dfs.getFileChecksum(ecFile, writeBytes);
+
+    final INodeFile fileNode = cluster.getNamesystem().getFSDirectory()
+        .getINode4Write(ecFile.toString()).asFile();
+    BlockInfo firstBlock = fileNode.getBlocks()[0];
+    DatanodeStorageInfo[] dnStorageInfos = bm.getStorages(firstBlock);
+
+    //2. maintenance node
+    // d4 d5 d6 d7 d8
+    int maintenanceDNIndex = 4;
+    int numMaintenance= 5;
+    List<DatanodeInfo> maintenanceNodes = new ArrayList<>();
+
+    for (int i = maintenanceDNIndex; i < numMaintenance + maintenanceDNIndex; ++i) {
+      maintenanceNodes.add(dnStorageInfos[i].getDatanodeDescriptor());
+    }
+
+    maintenanceNode(0, maintenanceNodes, AdminStates.IN_MAINTENANCE, Long.MAX_VALUE);
+
+    //3. wait for maintenance block to replicate
+    GenericTestUtils.waitFor(
+        () -> maintenanceNodes.size() == fsn.getNumInMaintenanceLiveDataNodes(),
+            100, 60000);
+
+    //4. check DN status, it should be reconstructed again
+    LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
+        ecFile.toString(), 0, writeBytes);
+    LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+
+    BlockInfoStriped blockInfo =
+        (BlockInfoStriped)bm.getStoredBlock(
+            new Block(bg.getBlock().getBlockId()));
+
+    // So far, there are 11 total internal blocks, 6 live (d0 d1 d2 d3 d4' d5')
+    // and 5 in maintenance (d4 d5 d6 d7 d8) internal blocks.
+
+    assertEquals(6, bm.countNodes(blockInfo).liveReplicas());
+    assertEquals(5, bm.countNodes(blockInfo).maintenanceNotForReadReplicas());
+
+    FileChecksum fileChecksum2 = dfs.getFileChecksum(ecFile, writeBytes);
+    Assert.assertEquals("Checksum mismatches!", fileChecksum1, fileChecksum2);
+  }
+
+
+  /* Get DFSClient to the namenode */
+  private static DFSClient getDfsClient(NameNode nn, Configuration conf)
+      throws IOException {
+    return new DFSClient(nn.getNameNodeAddress(), conf);
+  }
+
+  private byte[] writeStripedFile(DistributedFileSystem fs, Path ecFile,
+      int writeBytes) throws Exception {
+    byte[] bytes = StripedFileTestUtil.generateBytes(writeBytes);
+    DFSTestUtil.writeFile(fs, ecFile, new String(bytes));
+    StripedFileTestUtil.waitBlockGroupsReported(fs, ecFile.toString());
+
+    StripedFileTestUtil.checkData(fs, ecFile, writeBytes,
+        new ArrayList<DatanodeInfo>(), null, blockGroupSize);
+    return bytes;
+  }
+
+  /*
+   * maintenance the DN at index dnIndex or one random node if dnIndex is set
+   * to -1 and wait for the node to reach the given {@code waitForState}.
+   */
+  private void maintenanceNode(int nnIndex, List<DatanodeInfo> maintenancedNodes,
+      AdminStates waitForState, long maintenanceExpirationInMS)
+          throws IOException, TimeoutException, InterruptedException {
+    DFSClient client = getDfsClient(cluster.getNameNode(nnIndex), conf);
+    DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE);
+
+    // write nodename into the exclude file.
+    Map<String, Long> maintenanceNodes = new HashMap<>();
+
+    for (DatanodeInfo dn : maintenancedNodes) {
+      boolean nodeExists = false;
+      for (DatanodeInfo dninfo : info) {
+        if (dninfo.getDatanodeUuid().equals(dn.getDatanodeUuid())) {
+          nodeExists = true;
+          break;
+        }
+      }
+      assertTrue("Datanode: " + dn + " is not LIVE", nodeExists);
+      maintenanceNodes.put(dn.getName(), maintenanceExpirationInMS);
+      LOG.info("Maintenance node: " + dn.getName());
+    }
+    // write node names into the json host file.
+    hostsFileWriter.initOutOfServiceHosts(null, maintenanceNodes);
+
+    refreshNodes(cluster.getNamesystem(nnIndex), conf);
+    for (DatanodeInfo dn : maintenancedNodes) {
+      DatanodeInfo ret = NameNodeAdapter
+          .getDatanode(cluster.getNamesystem(nnIndex), dn);
+      LOG.info("Waiting for node " + ret + " to change state to " + waitForState
+          + " current state: " + ret.getAdminState());
+      GenericTestUtils.waitFor(
+          () -> ret.getAdminState() == waitForState,
+              100, 60000);
+      LOG.info("node " + ret + " reached the state " + waitForState);
+    }
+  }
+
+  private static void refreshNodes(final FSNamesystem ns,
+      final Configuration conf) throws IOException {
+    ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
+  }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index 74a8e44bf7b0b..0b6d07bc2f6f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdfs;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeTrue;
 
@@ -26,6 +27,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Random;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
 
@@ -38,11 +40,15 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.test.PathUtils;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Tests MiniDFS cluster setup/teardown and isolation.
@@ -52,6 +58,8 @@
  */
 public class TestMiniDFSCluster {
 
+  private static final Logger LOG = LoggerFactory.getLogger(TestMiniDFSCluster.class);
+
   private static final String CLUSTER_1 = "cluster1";
   private static final String CLUSTER_2 = "cluster2";
   private static final String CLUSTER_3 = "cluster3";
@@ -309,6 +317,98 @@ public void testSetUpFederatedCluster() throws Exception {
             DFSUtil.addKeySuffixes(
             DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
       }
+
+      // Shutdown namenodes individually.
+      cluster.shutdownNameNode(0);
+      cluster.shutdownNameNode(1);
+
+      // Restart namenodes individually with wait active, both should be successful.
+      cluster.restartNameNode(0);
+      cluster.restartNameNode(1);
     }
   }
+
+  // There is a possibility that this test might fail if any other concurrently running
+  // test could bind same port as one of the ports returned by NetUtils.getFreeSocketPorts(6)
+  // before datanodes are started.
+  @Test
+  public void testStartStopWithPorts() throws Exception {
+    Configuration conf = new Configuration();
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of http ports (1) should match num of DataNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniDFSCluster.Builder(conf).numDataNodes(3).setDnHttpPorts(8481).build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of ipc ports (2) should match num of DataNodes (1)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniDFSCluster.Builder(conf).setDnIpcPorts(8481, 8482).build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of ipc ports (1) should match num of DataNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniDFSCluster.Builder(conf).numDataNodes(3).setDnHttpPorts(800, 9000, 10000)
+              .setDnIpcPorts(8481).build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of http ports (4) should match num of DataNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniDFSCluster.Builder(conf).setDnHttpPorts(800, 9000, 1000, 2000)
+              .setDnIpcPorts(8481, 8482, 8483).numDataNodes(3).build();
+        });
+
+    final Set<Integer> httpAndIpcPorts = NetUtils.getFreeSocketPorts(6);
+    LOG.info("Free socket ports: {}", httpAndIpcPorts);
+
+    assertThat(httpAndIpcPorts).doesNotContain(0);
+
+    final int[] httpPorts = new int[3];
+    final int[] ipcPorts = new int[3];
+    int httpPortIdx = 0;
+    int ipcPortIdx = 0;
+    for (Integer httpAndIpcPort : httpAndIpcPorts) {
+      if (httpPortIdx < 3) {
+        httpPorts[httpPortIdx++] = httpAndIpcPort;
+      } else {
+        ipcPorts[ipcPortIdx++] = httpAndIpcPort;
+      }
+    }
+
+    LOG.info("Http ports selected: {}", httpPorts);
+    LOG.info("Ipc ports selected: {}", ipcPorts);
+
+    try (MiniDFSCluster miniDfsCluster = new MiniDFSCluster.Builder(conf)
+        .setDnHttpPorts(httpPorts)
+        .setDnIpcPorts(ipcPorts)
+        .numDataNodes(3).build()) {
+      miniDfsCluster.waitActive();
+
+      assertEquals(httpPorts[0],
+          miniDfsCluster.getDataNode(ipcPorts[0]).getInfoPort());
+      assertEquals(httpPorts[1],
+          miniDfsCluster.getDataNode(ipcPorts[1]).getInfoPort());
+      assertEquals(httpPorts[2],
+          miniDfsCluster.getDataNode(ipcPorts[2]).getInfoPort());
+
+      assertEquals(ipcPorts[0],
+          miniDfsCluster.getDataNode(ipcPorts[0]).getIpcPort());
+      assertEquals(ipcPorts[1],
+          miniDfsCluster.getDataNode(ipcPorts[1]).getIpcPort());
+      assertEquals(ipcPorts[2],
+          miniDfsCluster.getDataNode(ipcPorts[2]).getIpcPort());
+    }
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
index 3978444c6a217..0b73157e71898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestModTime.java
@@ -26,8 +26,6 @@
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index 2e36b131fb3c3..a11af7220481a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -958,6 +959,44 @@ public void testQuotaByStorageType() throws Exception {
         6 * fileSpace);
   }
 
+  @Test
+  public void testRenameInodeWithStorageType() throws IOException {
+    final int size = 64;
+    final short repl = 1;
+    final Path foo = new Path("/foo");
+    final Path bs1 = new Path(foo, "bs1");
+    final Path wow = new Path(bs1, "wow");
+    final Path bs2 = new Path(foo, "bs2");
+    final Path wow2 = new Path(bs2, "wow2");
+    final Path wow3 = new Path(bs2, "wow3");
+
+    dfs.mkdirs(bs1, FsPermission.getDirDefault());
+    dfs.mkdirs(bs2, FsPermission.getDirDefault());
+    dfs.setQuota(bs1, 1000, 434217728);
+    dfs.setQuota(bs2, 1000, 434217728);
+    // file wow3 without storage policy
+    DFSTestUtil.createFile(dfs, wow3, size, repl, 0);
+
+    dfs.setStoragePolicy(bs2, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+    DFSTestUtil.createFile(dfs, wow, size, repl, 0);
+    DFSTestUtil.createFile(dfs, wow2, size, repl, 0);
+    assertTrue("Without storage policy, typeConsumed should be 0.",
+        dfs.getQuotaUsage(bs1).getTypeConsumed(StorageType.SSD) == 0);
+    assertTrue("With storage policy, typeConsumed should not be 0.",
+        dfs.getQuotaUsage(bs2).getTypeConsumed(StorageType.SSD) != 0);
+    // wow3 without storage policy , rename will not change typeConsumed
+    dfs.rename(wow3, bs1);
+    assertTrue("Rename src without storagePolicy, dst typeConsumed should not be changed.",
+        dfs.getQuotaUsage(bs2).getTypeConsumed(StorageType.SSD) == 0);
+
+    long srcTypeQuota = dfs.getQuotaUsage(bs2).getTypeQuota(StorageType.SSD);
+    dfs.rename(bs2, bs1);
+    long dstTypeQuota = dfs.getQuotaUsage(bs1).getTypeConsumed(StorageType.SSD);
+    assertTrue("Rename with storage policy, typeConsumed should not be 0.",
+        dstTypeQuota != srcTypeQuota);
+  }
+
   private static void checkContentSummary(final ContentSummary expected,
       final ContentSummary computed) {
     assertEquals(expected.toString(), computed.toString());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
index 2fb9212f35429..f80cb01bab8a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -19,10 +19,13 @@
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
@@ -43,6 +46,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
+import static org.apache.hadoop.hdfs.ReadStripedFileWithDecodingHelper.BLOCK_SIZE;
 import static org.apache.hadoop.hdfs.ReadStripedFileWithDecodingHelper.CELL_SIZE;
 import static org.apache.hadoop.hdfs.ReadStripedFileWithDecodingHelper.NUM_DATA_UNITS;
 import static org.apache.hadoop.hdfs.ReadStripedFileWithDecodingHelper.NUM_PARITY_UNITS;
@@ -127,7 +131,7 @@ public void testReportBadBlock() throws IOException {
   }
 
   @Test
-  public void testInvalidateBlock() throws IOException {
+  public void testInvalidateBlock() throws IOException, InterruptedException {
     final Path file = new Path("/invalidate");
     final int length = 10;
     final byte[] bytes = StripedFileTestUtil.generateBytes(length);
@@ -151,6 +155,8 @@ public void testInvalidateBlock() throws IOException {
     try {
       // delete the file
       dfs.delete(file, true);
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem().getBlockManager());
       // check the block is added to invalidateBlocks
       final FSNamesystem fsn = cluster.getNamesystem();
       final BlockManager bm = fsn.getBlockManager();
@@ -162,4 +168,57 @@ public void testInvalidateBlock() throws IOException {
       DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
     }
   }
+
+  @Test
+  public void testMoreThanOneCorruptedBlock() throws IOException {
+    final Path file = new Path("/corrupted");
+    final int length = BLOCK_SIZE * NUM_DATA_UNITS;
+    final byte[] bytes = StripedFileTestUtil.generateBytes(length);
+    DFSTestUtil.writeFile(dfs, file, bytes);
+
+    // read the file with more than one corrupted data block
+    byte[] buffer = new byte[length + 100];
+    for (int count = 2; count < NUM_PARITY_UNITS; ++count) {
+      ReadStripedFileWithDecodingHelper.corruptBlocks(cluster, dfs, file, count, 0,
+          false);
+      StripedFileTestUtil.verifyStatefulRead(dfs, file, length, bytes,
+          buffer);
+    }
+  }
+
+  @Test
+  public void testReadWithCorruptedDataBlockAndParityBlock() throws IOException {
+    final Path file = new Path("/corruptedDataBlockAndParityBlock");
+    final int length = BLOCK_SIZE * NUM_DATA_UNITS;
+    final byte[] bytes = StripedFileTestUtil.generateBytes(length);
+    DFSTestUtil.writeFile(dfs, file, bytes);
+
+    // set one dataBlock and the first parityBlock corrupted
+    int dataBlkDelNum = 1;
+    int parityBlkDelNum = 1;
+    int recoverBlkNum = dataBlkDelNum + parityBlkDelNum;
+    int[] dataBlkIndices = {0};
+    int[] parityBlkIndices = {6};
+
+    LocatedBlocks locatedBlocks = ReadStripedFileWithDecodingHelper.getLocatedBlocks(dfs, file);
+    LocatedStripedBlock lastBlock =
+        (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+
+    int[] delBlkIndices = new int[recoverBlkNum];
+    System.arraycopy(dataBlkIndices, 0,
+        delBlkIndices, 0, dataBlkIndices.length);
+    System.arraycopy(parityBlkIndices, 0,
+        delBlkIndices, dataBlkIndices.length, parityBlkIndices.length);
+    ExtendedBlock[] delBlocks = new ExtendedBlock[recoverBlkNum];
+    for (int i = 0; i < recoverBlkNum; i++) {
+      delBlocks[i] = StripedBlockUtil
+          .constructInternalBlock(lastBlock.getBlock(),
+              CELL_SIZE, NUM_DATA_UNITS, delBlkIndices[i]);
+      cluster.corruptBlockOnDataNodes(delBlocks[i]);
+    }
+
+    byte[] buffer = new byte[length + 100];
+    StripedFileTestUtil.verifyStatefulRead(dfs, file, length, bytes,
+        buffer);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index fa3c1aa68130f..cda714b2cb3fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -472,7 +472,7 @@ public void testProcessErasureCodingTasksSubmitionShouldSucceed()
 
     BlockECReconstructionInfo invalidECInfo = new BlockECReconstructionInfo(
         new ExtendedBlock("bp-id", 123456), dataDNs, dnStorageInfo, liveIndices,
-        ecPolicy);
+        new byte[0], ecPolicy);
     List<BlockECReconstructionInfo> ecTasks = new ArrayList<>();
     ecTasks.add(invalidECInfo);
     dataNode.getErasureCodingWorker().processErasureCodingTasks(ecTasks);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
index a716335d1aa0b..bb5da24a682f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
@@ -22,6 +22,7 @@
 import java.lang.management.ManagementFactory;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeoutException;
 
 import javax.management.AttributeNotFoundException;
 import javax.management.InstanceNotFoundException;
@@ -32,6 +33,11 @@
 import javax.management.ReflectionException;
 import javax.management.openmbean.CompositeDataSupport;
 
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -79,16 +85,38 @@ public static void runCmd(DFSAdmin dfsadmin, boolean success,
     }
   }
 
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
+  /**
+   * Create a default HDFS configuration which has test-specific data directories.  This is
+   * intended to protect against interactions between test runs that might corrupt results.  Each
+   * test run's data is automatically cleaned-up by JUnit.
+   *
+   * @return a default configuration with test-specific data directories
+   */
+  public Configuration getHdfsConfiguration() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+
+    // Override the file system locations with test-specific temporary folders
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
+        folder.newFolder("dfs/name").toString());
+    conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY,
+        folder.newFolder("dfs/namesecondary").toString());
+    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+        folder.newFolder("dfs/data").toString());
+
+    return conf;
+  }
+
   /**
    * Test DFSAdmin Upgrade Command.
    */
   @Test
   public void testDFSAdminRollingUpgradeCommands() throws Exception {
     // start a cluster
-    final Configuration conf = new HdfsConfiguration();
-    MiniDFSCluster cluster = null;
-    try {
-      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) {
       cluster.waitActive();
 
       final Path foo = new Path("/foo");
@@ -148,8 +176,6 @@ public void testDFSAdminRollingUpgradeCommands() throws Exception {
         Assert.assertTrue(dfs.exists(bar));
         Assert.assertTrue(dfs.exists(baz));
       }
-    } finally {
-      if(cluster != null) cluster.shutdown();
     }
   }
 
@@ -171,115 +197,116 @@ public void testRollingUpgradeWithQJM() throws Exception {
     LOG.info("nn1Dir=" + nn1Dir);
     LOG.info("nn2Dir=" + nn2Dir);
 
-    final Configuration conf = new HdfsConfiguration();
-    final MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build();
-    mjc.waitActive();
-    setConf(conf, nn1Dir, mjc);
-
-    {
-      // Start the cluster once to generate the dfs dirs
-      final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(0)
-        .manageNameDfsDirs(false)
-        .checkExitOnShutdown(false)
-        .build();
-      // Shutdown the cluster before making a copy of the namenode dir to release
-      // all file locks, otherwise, the copy will fail on some platforms.
-      cluster.shutdown();
-    }
-
-    MiniDFSCluster cluster2 = null;
-    try {
-      // Start a second NN pointed to the same quorum.
-      // We need to copy the image dir from the first NN -- or else
-      // the new NN will just be rejected because of Namespace mismatch.
-      FileUtil.fullyDelete(nn2Dir);
-      FileUtil.copy(nn1Dir, FileSystem.getLocal(conf).getRaw(),
-          new Path(nn2Dir.getAbsolutePath()), false, conf);
-
-      // Start the cluster again
-      final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(0)
-        .format(false)
-        .manageNameDfsDirs(false)
-        .checkExitOnShutdown(false)
-        .build();
-
-      final Path foo = new Path("/foo");
-      final Path bar = new Path("/bar");
-      final Path baz = new Path("/baz");
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build()) {
+      mjc.waitActive();
+      setConf(conf, nn1Dir, mjc);
 
-      final RollingUpgradeInfo info1;
       {
-        final DistributedFileSystem dfs = cluster.getFileSystem();
-        dfs.mkdirs(foo);
-
-        //start rolling upgrade
-        dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
-        info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
-        dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
-        LOG.info("START\n" + info1);
-
-        //query rolling upgrade
-        assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
-
-        dfs.mkdirs(bar);
+        // Start the cluster once to generate the dfs dirs
+        final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(0)
+            .manageNameDfsDirs(false)
+            .checkExitOnShutdown(false)
+            .build();
+        // Shutdown the cluster before making a copy of the namenode dir to release
+        // all file locks, otherwise, the copy will fail on some platforms.
         cluster.shutdown();
       }
 
-      // cluster2 takes over QJM
-      final Configuration conf2 = setConf(new Configuration(), nn2Dir, mjc);
-      cluster2 = new MiniDFSCluster.Builder(conf2)
-        .numDataNodes(0)
-        .format(false)
-        .manageNameDfsDirs(false)
-        .build();
-      final DistributedFileSystem dfs2 = cluster2.getFileSystem();
-
-      // Check that cluster2 sees the edits made on cluster1
-      Assert.assertTrue(dfs2.exists(foo));
-      Assert.assertTrue(dfs2.exists(bar));
-      Assert.assertFalse(dfs2.exists(baz));
-
-      //query rolling upgrade in cluster2
-      assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
+      MiniDFSCluster cluster2 = null;
+      try {
+        // Start a second NN pointed to the same quorum.
+        // We need to copy the image dir from the first NN -- or else
+        // the new NN will just be rejected because of Namespace mismatch.
+        FileUtil.fullyDelete(nn2Dir);
+        FileUtil.copy(nn1Dir, FileSystem.getLocal(conf).getRaw(),
+            new Path(nn2Dir.getAbsolutePath()), false, conf);
+
+        // Start the cluster again
+        final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+            .numDataNodes(0)
+            .format(false)
+            .manageNameDfsDirs(false)
+            .checkExitOnShutdown(false)
+            .build();
+
+        final Path foo = new Path("/foo");
+        final Path bar = new Path("/bar");
+        final Path baz = new Path("/baz");
+
+        final RollingUpgradeInfo info1;
+        {
+          final DistributedFileSystem dfs = cluster.getFileSystem();
+          dfs.mkdirs(foo);
+
+          //start rolling upgrade
+          dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+          info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
+          dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+          LOG.info("START\n" + info1);
+
+          //query rolling upgrade
+          assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
+
+          dfs.mkdirs(bar);
+          cluster.shutdown();
+        }
 
-      dfs2.mkdirs(baz);
+        // cluster2 takes over QJM
+        final Configuration conf2 = setConf(new Configuration(), nn2Dir, mjc);
+        cluster2 = new MiniDFSCluster.Builder(conf2)
+            .numDataNodes(0)
+            .format(false)
+            .manageNameDfsDirs(false)
+            .build();
+        final DistributedFileSystem dfs2 = cluster2.getFileSystem();
+
+        // Check that cluster2 sees the edits made on cluster1
+        Assert.assertTrue(dfs2.exists(foo));
+        Assert.assertTrue(dfs2.exists(bar));
+        Assert.assertFalse(dfs2.exists(baz));
+
+        //query rolling upgrade in cluster2
+        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
+
+        dfs2.mkdirs(baz);
+
+        LOG.info("RESTART cluster 2");
+        cluster2.restartNameNode();
+        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
+        Assert.assertTrue(dfs2.exists(foo));
+        Assert.assertTrue(dfs2.exists(bar));
+        Assert.assertTrue(dfs2.exists(baz));
+
+        //restart cluster with -upgrade should fail.
+        try {
+          cluster2.restartNameNode("-upgrade");
+        } catch (IOException e) {
+          LOG.info("The exception is expected.", e);
+        }
 
-      LOG.info("RESTART cluster 2");
-      cluster2.restartNameNode();
-      assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
-      Assert.assertTrue(dfs2.exists(foo));
-      Assert.assertTrue(dfs2.exists(bar));
-      Assert.assertTrue(dfs2.exists(baz));
+        LOG.info("RESTART cluster 2 again");
+        cluster2.restartNameNode();
+        assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
+        Assert.assertTrue(dfs2.exists(foo));
+        Assert.assertTrue(dfs2.exists(bar));
+        Assert.assertTrue(dfs2.exists(baz));
 
-      //restart cluster with -upgrade should fail.
-      try {
-        cluster2.restartNameNode("-upgrade");
-      } catch(IOException e) {
-        LOG.info("The exception is expected.", e);
+        //finalize rolling upgrade
+        final RollingUpgradeInfo finalize = dfs2.rollingUpgrade(
+            RollingUpgradeAction.FINALIZE);
+        Assert.assertTrue(finalize.isFinalized());
+
+        LOG.info("RESTART cluster 2 with regular startup option");
+        cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
+        cluster2.restartNameNode();
+        Assert.assertTrue(dfs2.exists(foo));
+        Assert.assertTrue(dfs2.exists(bar));
+        Assert.assertTrue(dfs2.exists(baz));
+      } finally {
+        if (cluster2 != null) cluster2.shutdown();
       }
-
-      LOG.info("RESTART cluster 2 again");
-      cluster2.restartNameNode();
-      assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
-      Assert.assertTrue(dfs2.exists(foo));
-      Assert.assertTrue(dfs2.exists(bar));
-      Assert.assertTrue(dfs2.exists(baz));
-
-      //finalize rolling upgrade
-      final RollingUpgradeInfo finalize = dfs2.rollingUpgrade(
-          RollingUpgradeAction.FINALIZE);
-      Assert.assertTrue(finalize.isFinalized());
-
-      LOG.info("RESTART cluster 2 with regular startup option");
-      cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
-      cluster2.restartNameNode();
-      Assert.assertTrue(dfs2.exists(foo));
-      Assert.assertTrue(dfs2.exists(bar));
-      Assert.assertTrue(dfs2.exists(baz));
-    } finally {
-      if (cluster2 != null) cluster2.shutdown();
     }
   }
 
@@ -308,10 +335,8 @@ private static void checkMxBean() throws Exception {
   @Test
   public void testRollback() throws Exception {
     // start a cluster
-    final Configuration conf = new HdfsConfiguration();
-    MiniDFSCluster cluster = null;
-    try {
-      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniDFSCluster cluster  = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) {
       cluster.waitActive();
 
       final Path foo = new Path("/foo");
@@ -325,7 +350,7 @@ public void testRollback() throws Exception {
       out.write(data, 0, data.length);
       out.close();
 
-      checkMxBeanIsNull();
+      waitForNullMxBean();
       startRollingUpgrade(foo, bar, file, data, cluster);
       checkMxBean();
       cluster.getFileSystem().rollEdits();
@@ -351,11 +376,21 @@ public void testRollback() throws Exception {
 
       startRollingUpgrade(foo, bar, file, data, cluster);
       rollbackRollingUpgrade(foo, bar, file, data, cluster);
-    } finally {
-      if(cluster != null) cluster.shutdown();
     }
   }
 
+  private void waitForNullMxBean() throws TimeoutException, InterruptedException {
+    GenericTestUtils.waitFor(() -> {
+      try {
+        checkMxBeanIsNull();
+        return true;
+      } catch (Throwable t) {
+        LOG.error("Something went wrong.", t);
+        return false;
+      }
+    }, 100, 8000, "RollingUpgradeStatus is already set");
+  }
+
   private static void startRollingUpgrade(Path foo, Path bar,
       Path file, byte[] data,
       MiniDFSCluster cluster) throws IOException {
@@ -394,10 +429,8 @@ private static void rollbackRollingUpgrade(Path foo, Path bar,
   @Test
   public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception {
     // start a cluster
-    final Configuration conf = new HdfsConfiguration();
-    MiniDFSCluster cluster = null;
-    try {
-      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) {
       cluster.waitActive();
       final DFSAdmin dfsadmin = new DFSAdmin(conf);
       DataNode dn = cluster.getDataNodes().get(0);
@@ -418,8 +451,6 @@ public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception {
 
       // ping should fail.
       assertEquals(-1, dfsadmin.run(args1));
-    } finally {
-      if (cluster != null) cluster.shutdown();
     }
   }
 
@@ -449,7 +480,7 @@ private void testFinalize(int nnCount) throws Exception {
 
   private void testFinalize(int nnCount, boolean skipImageDeltaCheck)
       throws Exception {
-    final Configuration conf = new HdfsConfiguration();
+    final Configuration conf = getHdfsConfiguration();
     MiniQJMHACluster cluster = null;
     final Path foo = new Path("/foo");
     final Path bar = new Path("/bar");
@@ -515,10 +546,8 @@ public void testQueryWithMultipleNN() throws Exception {
   }
 
   private void testQuery(int nnCount) throws Exception{
-    final Configuration conf = new Configuration();
-    MiniQJMHACluster cluster = null;
-    try {
-      cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build()) {
       MiniDFSCluster dfsCluster = cluster.getDfsCluster();
       dfsCluster.waitActive();
 
@@ -548,19 +577,13 @@ private void testQuery(int nnCount) throws Exception{
       // The NN should have a copy of the fsimage in case of rollbacks.
       Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage()
               .hasRollbackFSImage());
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
     }
   }
 
   @Test (timeout = 300000)
   public void testQueryAfterRestart() throws IOException, InterruptedException {
-    final Configuration conf = new Configuration();
-    MiniDFSCluster cluster = null;
-    try {
-      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+    final Configuration conf = getHdfsConfiguration();
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) {
       cluster.waitActive();
       DistributedFileSystem dfs = cluster.getFileSystem();
 
@@ -574,10 +597,6 @@ public void testQueryAfterRestart() throws IOException, InterruptedException {
 
       cluster.restartNameNodes();
       dfs.rollingUpgrade(RollingUpgradeAction.QUERY);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
     }
   }
 
@@ -593,7 +612,7 @@ public void testCheckpointWithMultipleNN() throws IOException, InterruptedExcept
 
   @Test(timeout = 60000)
   public void testRollBackImage() throws Exception {
-    final Configuration conf = new Configuration();
+    final Configuration conf = getHdfsConfiguration();
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 10);
     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 2);
@@ -638,15 +657,14 @@ public void duringUploadInProgess()
   }
 
   public void testCheckpoint(int nnCount) throws IOException, InterruptedException {
-    final Configuration conf = new Configuration();
+    final Configuration conf = getHdfsConfiguration();
     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
 
-    MiniQJMHACluster cluster = null;
     final Path foo = new Path("/foo");
 
-    try {
-      cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();
+    try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount)
+        .build()) {
       MiniDFSCluster dfsCluster = cluster.getDfsCluster();
       dfsCluster.waitActive();
 
@@ -668,17 +686,14 @@ public void testCheckpoint(int nnCount) throws IOException, InterruptedException
         verifyNNCheckpoint(dfsCluster, txid, i);
       }
 
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
     }
   }
 
   /**
    * Verify that the namenode at the given index has an FSImage with a TxId up to txid-1
    */
-  private void verifyNNCheckpoint(MiniDFSCluster dfsCluster, long txid, int nnIndex) throws InterruptedException {
+  private void verifyNNCheckpoint(MiniDFSCluster dfsCluster, long txid, int nnIndex)
+      throws InterruptedException {
     int retries = 0;
     while (++retries < 5) {
       NNStorage storage = dfsCluster.getNamesystem(nnIndex).getFSImage()
@@ -708,6 +723,39 @@ static void queryForPreparation(DistributedFileSystem dfs) throws IOException,
     }
   }
 
+  @Test
+  public void testEditLogTailerRollingUpgrade() throws IOException, InterruptedException {
+    Configuration conf = new Configuration();
+    conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
+
+    HAUtil.setAllowStandbyReads(conf, true);
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .nnTopology(MiniDFSNNTopology.simpleHATopology())
+        .numDataNodes(0)
+        .build();
+    cluster.waitActive();
+
+    cluster.transitionToActive(0);
+
+    NameNode nn1 = cluster.getNameNode(0);
+    NameNode nn2 = cluster.getNameNode(1);
+    try {
+      // RU start should trigger rollback image in standbycheckpointer
+      nn1.getRpcServer().rollingUpgrade(HdfsConstants.RollingUpgradeAction.PREPARE);
+      HATestUtil.waitForStandbyToCatchUp(nn1, nn2);
+      Assert.assertTrue(nn2.getNamesystem().isNeedRollbackFsImage());
+
+      // RU finalize should reset rollback image flag in standbycheckpointer
+      nn1.getRpcServer().rollingUpgrade(HdfsConstants.RollingUpgradeAction.FINALIZE);
+      HATestUtil.waitForStandbyToCatchUp(nn1, nn2);
+      Assert.assertFalse(nn2.getNamesystem().isNeedRollbackFsImage());
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   /**
    * In non-HA setup, after rolling upgrade prepare, the Secondary NN should
    * still be able to do checkpoint
@@ -719,7 +767,7 @@ public void testCheckpointWithSNN() throws Exception {
     SecondaryNameNode snn = null;
 
     try {
-      Configuration conf = new HdfsConfiguration();
+      Configuration conf = getHdfsConfiguration();
       cluster = new MiniDFSCluster.Builder(conf).build();
       cluster.waitActive();
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java
index 189b5f575a48f..ff17c2f61740d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeDowngrade.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
 import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
-import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
 import org.junit.Assert;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java
index 9dd2987f55f64..c7a7e7a8d11c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSeekBug.java
@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.Random;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetTimes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetTimes.java
index 74cde2a2a95ec..7039a6ba692f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetTimes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetTimes.java
@@ -42,7 +42,6 @@
 import org.apache.hadoop.util.Time;
 import org.junit.Assert;
 import org.junit.Test;
-import org.mockito.Mockito;
 
 /**
  * This class tests the access time on files.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSlowDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSlowDatanodeReport.java
new file mode 100644
index 0000000000000..187919c39b9aa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSlowDatanodeReport.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys
+    .DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
+
+/**
+ * Tests to report slow running datanodes.
+ */
+public class TestSlowDatanodeReport {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestSlowDatanodeReport.class);
+
+  private MiniDFSCluster cluster;
+
+  @Before
+  public void testSetup() throws Exception {
+    Configuration conf = new Configuration();
+
+    conf.set(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, "1000");
+    conf.set(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "true");
+    conf.set(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY, "1");
+    conf.set(DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY, "1");
+
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+    cluster.waitActive();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    cluster.shutdown();
+  }
+
+  @Test
+  public void testSingleNodeReport() throws Exception {
+    List<DataNode> dataNodes = cluster.getDataNodes();
+    DataNode slowNode = dataNodes.get(1);
+    OutlierMetrics outlierMetrics = new OutlierMetrics(1.245, 2.69375, 4.5667, 15.5);
+    dataNodes.get(0).getPeerMetrics().setTestOutliers(
+        ImmutableMap.of(slowNode.getDatanodeHostname() + ":" + slowNode.getIpcPort(),
+            outlierMetrics));
+    DistributedFileSystem distributedFileSystem = cluster.getFileSystem();
+    Assert.assertEquals(3, distributedFileSystem.getDataNodeStats().length);
+    GenericTestUtils.waitFor(() -> {
+      try {
+        DatanodeInfo[] slowNodeInfo = distributedFileSystem.getSlowDatanodeStats();
+        LOG.info("Slow Datanode report: {}", Arrays.asList(slowNodeInfo));
+        return slowNodeInfo.length == 1;
+      } catch (IOException e) {
+        LOG.error("Failed to retrieve slownode report", e);
+        return false;
+      }
+    }, 2000, 180000, "Slow nodes could not be detected");
+    LOG.info("Slow peer report: {}", cluster.getNameNode().getSlowPeersReport());
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().length() > 0);
+    Assert.assertTrue(
+        cluster.getNameNode().getSlowPeersReport().contains(slowNode.getDatanodeHostname()));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("15.5"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("1.245"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("2.69375"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("4.5667"));
+  }
+
+  @Test
+  public void testMultiNodesReport() throws Exception {
+    List<DataNode> dataNodes = cluster.getDataNodes();
+    OutlierMetrics outlierMetrics1 = new OutlierMetrics(2.498237, 19.2495, 23.568204, 14.5);
+    OutlierMetrics outlierMetrics2 = new OutlierMetrics(3.2535, 22.4945, 44.5667, 18.7);
+    dataNodes.get(0).getPeerMetrics().setTestOutliers(ImmutableMap.of(
+        dataNodes.get(1).getDatanodeHostname() + ":" + dataNodes.get(1).getIpcPort(),
+        outlierMetrics1));
+    dataNodes.get(1).getPeerMetrics().setTestOutliers(ImmutableMap.of(
+        dataNodes.get(2).getDatanodeHostname() + ":" + dataNodes.get(2).getIpcPort(),
+        outlierMetrics2));
+    DistributedFileSystem distributedFileSystem = cluster.getFileSystem();
+    Assert.assertEquals(3, distributedFileSystem.getDataNodeStats().length);
+    GenericTestUtils.waitFor(() -> {
+      try {
+        DatanodeInfo[] slowNodeInfo = distributedFileSystem.getSlowDatanodeStats();
+        LOG.info("Slow Datanode report: {}", Arrays.asList(slowNodeInfo));
+        return slowNodeInfo.length == 2;
+      } catch (IOException e) {
+        LOG.error("Failed to retrieve slownode report", e);
+        return false;
+      }
+    }, 2000, 200000, "Slow nodes could not be detected");
+    LOG.info("Slow peer report: {}", cluster.getNameNode().getSlowPeersReport());
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().length() > 0);
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport()
+        .contains(dataNodes.get(1).getDatanodeHostname()));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport()
+        .contains(dataNodes.get(2).getDatanodeHostname()));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("14.5"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("18.7"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("23.568204"));
+    Assert.assertTrue(cluster.getNameNode().getSlowPeersReport().contains("22.4945"));
+  }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
index 32ac2980cd590..97256b6524e9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
-import org.apache.hadoop.util.ChunkedArrayList;
+
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -224,7 +224,7 @@ public void testSnapshotCommandsWithURI()throws Exception {
     fs.delete(new Path("/Fully/QPath"), true);
   }
 
-  @Test (timeout=60000)
+  @Test (timeout=120000)
   public void testSnapshotDiff()throws Exception {
     Configuration config = new HdfsConfiguration();
     Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
index 2b90c92388f32..7ebf55f571cd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
@@ -51,7 +51,7 @@
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.io.Files;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
index 1d0024d44472e..6681b2e627d78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.net.Node;
-import org.apache.hadoop.util.Sets;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -34,6 +33,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -309,11 +309,11 @@ public void testChooseRandomWithStorageType() throws Exception {
     // test the choose random can return desired storage type nodes without
     // exclude
     Set<String> diskUnderL1 =
-        Sets.newHashSet("host2", "host4", "host5", "host6");
-    Set<String> archiveUnderL1 = Sets.newHashSet("host1", "host3");
-    Set<String> ramdiskUnderL1 = Sets.newHashSet("host7");
-    Set<String> ssdUnderL1 = Sets.newHashSet("host8");
-    Set<String> nvdimmUnderL1 = Sets.newHashSet("host9");
+        new HashSet<>(Arrays.asList("host2", "host4", "host5", "host6"));
+    Set<String> archiveUnderL1 = new HashSet<>(Arrays.asList("host1", "host3"));
+    Set<String> ramdiskUnderL1 = new HashSet<>(Arrays.asList("host7"));
+    Set<String> ssdUnderL1 = new HashSet<>(Arrays.asList("host8"));
+    Set<String> nvdimmUnderL1 = new HashSet<>(Arrays.asList("host9"));
     for (int i = 0; i < 10; i++) {
       n = CLUSTER.chooseRandomWithStorageType("/l1", null, null,
           StorageType.DISK);
@@ -396,7 +396,7 @@ public void testChooseRandomWithStorageTypeWithExcluded() throws Exception {
     assertEquals("host6", dd.getHostName());
     // exclude the host on r4 (since there is only one host, no randomness here)
     excluded.add(n);
-    Set<String> expectedSet = Sets.newHashSet("host4", "host5");
+    Set<String> expectedSet = new HashSet<>(Arrays.asList("host4", "host5"));
     for (int i = 0; i < 10; i++) {
       // under l1, there are four hosts with DISK:
       // /l1/d1/r1/host2, /l1/d1/r2/host4, /l1/d1/r2/host5 and /l1/d2/r3/host6
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransferExpiredBlockToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransferExpiredBlockToken.java
new file mode 100644
index 0000000000000..838ad7cc97f22
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestSaslDataTransferExpiredBlockToken.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol.datatransfer.sasl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.HedgedRead;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Retry;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.SecurityTestUtil;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+public class TestSaslDataTransferExpiredBlockToken extends SaslDataTransferTestCase {
+  private static final int BLOCK_SIZE = 4096;
+  private static final int FILE_SIZE = 2 * BLOCK_SIZE;
+  private static final Path PATH = new Path("/file1");
+
+  private final byte[] rawData = new byte[FILE_SIZE];
+  private MiniDFSCluster cluster;
+
+  @Rule
+  public Timeout timeout = new Timeout(60, TimeUnit.SECONDS);
+
+  @Before
+  public void before() throws Exception {
+    Random r = new Random();
+    r.nextBytes(rawData);
+
+    HdfsConfiguration conf = createSecureConfig("authentication,integrity,privacy");
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+    cluster.waitActive();
+
+    try (FileSystem fs = cluster.getFileSystem()) {
+      createFile(fs);
+    }
+
+    // set a short token lifetime (1 second) initially
+    SecurityTestUtil.setBlockTokenLifetime(
+        cluster.getNameNode().getNamesystem().getBlockManager().getBlockTokenSecretManager(),
+        1000L);
+  }
+
+  @After
+  public void shutdown() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  private void createFile(FileSystem fs) throws IOException {
+    try (FSDataOutputStream out = fs.create(PATH)) {
+      out.write(rawData);
+    }
+  }
+
+  // read a file using blockSeekTo()
+  private boolean checkFile1(FSDataInputStream in) {
+    byte[] toRead = new byte[FILE_SIZE];
+    int totalRead = 0;
+    int nRead = 0;
+    try {
+      while ((nRead = in.read(toRead, totalRead, toRead.length - totalRead)) > 0) {
+        totalRead += nRead;
+      }
+    } catch (IOException e) {
+      return false;
+    }
+    assertEquals("Cannot read file.", toRead.length, totalRead);
+    return checkFile(toRead);
+  }
+
+  // read a file using fetchBlockByteRange()/hedgedFetchBlockByteRange()
+  private boolean checkFile2(FSDataInputStream in) {
+    byte[] toRead = new byte[FILE_SIZE];
+    try {
+      assertEquals("Cannot read file", toRead.length, in.read(0, toRead, 0, toRead.length));
+    } catch (IOException e) {
+      return false;
+    }
+    return checkFile(toRead);
+  }
+
+  private boolean checkFile(byte[] fileToCheck) {
+    if (fileToCheck.length != rawData.length) {
+      return false;
+    }
+    for (int i = 0; i < fileToCheck.length; i++) {
+      if (fileToCheck[i] != rawData[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private FileSystem newFileSystem() throws IOException {
+    Configuration clientConf = new Configuration(cluster.getConfiguration(0));
+
+    clientConf.setInt(Retry.WINDOW_BASE_KEY, Integer.MAX_VALUE);
+
+    return FileSystem.newInstance(cluster.getURI(), clientConf);
+  }
+
+  private FileSystem newFileSystemHedgedRead() throws IOException {
+    Configuration clientConf = new Configuration(cluster.getConfiguration(0));
+
+    clientConf.setInt(Retry.WINDOW_BASE_KEY, 3000);
+    clientConf.setInt(HedgedRead.THREADPOOL_SIZE_KEY, 5);
+
+    return FileSystem.newInstance(cluster.getURI(), clientConf);
+  }
+
+  @Test
+  public void testBlockSeekToWithExpiredToken() throws Exception {
+    // read using blockSeekTo(). Acquired tokens are cached in in
+    try (FileSystem fs = newFileSystem(); FSDataInputStream in = fs.open(PATH)) {
+      waitBlockTokenExpired(in);
+      assertTrue(checkFile1(in));
+    }
+  }
+
+  @Test
+  public void testFetchBlockByteRangeWithExpiredToken() throws Exception {
+    // read using fetchBlockByteRange(). Acquired tokens are cached in in
+    try (FileSystem fs = newFileSystem(); FSDataInputStream in = fs.open(PATH)) {
+      waitBlockTokenExpired(in);
+      assertTrue(checkFile2(in));
+    }
+  }
+
+  @Test
+  public void testHedgedFetchBlockByteRangeWithExpiredToken() throws Exception {
+    // read using hedgedFetchBlockByteRange(). Acquired tokens are cached in in
+    try (FileSystem fs = newFileSystemHedgedRead(); FSDataInputStream in = fs.open(PATH)) {
+      waitBlockTokenExpired(in);
+      assertTrue(checkFile2(in));
+    }
+  }
+
+  private void waitBlockTokenExpired(FSDataInputStream in1) throws Exception {
+    DFSInputStream innerStream = (DFSInputStream) in1.getWrappedStream();
+    for (LocatedBlock block : innerStream.getAllBlocks()) {
+      while (!SecurityTestUtil.isBlockTokenExpired(block.getBlockToken())) {
+        Thread.sleep(100);
+      }
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index e9bcef527085d..3b0b8d1564b84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -21,6 +21,7 @@
 import org.apache.hadoop.thirdparty.protobuf.UninitializedMessageException;
 import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 
 import static org.hamcrest.CoreMatchers.is;
@@ -748,9 +749,10 @@ public void testBlockECRecoveryCommand() {
     DatanodeStorageInfo[] targetDnInfos0 = new DatanodeStorageInfo[] {
         targetDnInfos_0, targetDnInfos_1 };
     byte[] liveBlkIndices0 = new byte[2];
+    byte[] excludeReconstructedIndices0=new byte[2];
     BlockECReconstructionInfo blkECRecoveryInfo0 = new BlockECReconstructionInfo(
         new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0,
-        liveBlkIndices0, StripedFileTestUtil.getDefaultECPolicy());
+        liveBlkIndices0, excludeReconstructedIndices0, StripedFileTestUtil.getDefaultECPolicy());
     DatanodeInfo[] dnInfos1 = new DatanodeInfo[] {
         DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() };
     DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil
@@ -762,9 +764,10 @@ public void testBlockECRecoveryCommand() {
     DatanodeStorageInfo[] targetDnInfos1 = new DatanodeStorageInfo[] {
         targetDnInfos_2, targetDnInfos_3 };
     byte[] liveBlkIndices1 = new byte[2];
+    byte[] excludeReconstructedIndices = new byte[2];
     BlockECReconstructionInfo blkECRecoveryInfo1 = new BlockECReconstructionInfo(
         new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1,
-        liveBlkIndices1, StripedFileTestUtil.getDefaultECPolicy());
+        liveBlkIndices1, excludeReconstructedIndices, StripedFileTestUtil.getDefaultECPolicy());
     List<BlockECReconstructionInfo> blkRecoveryInfosList = new ArrayList<BlockECReconstructionInfo>();
     blkRecoveryInfosList.add(blkECRecoveryInfo0);
     blkRecoveryInfosList.add(blkECRecoveryInfo1);
@@ -805,8 +808,14 @@ public void testDataNodeInfoPBHelper() {
   @Test
   public void testSlowPeerInfoPBHelper() {
     // Test with a map that has a few slow peer entries.
+    OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 0.0);
+    OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 1.0);
+    OutlierMetrics outlierMetrics3 = new OutlierMetrics(0.0, 0.0, 0.0, 2.0);
     final SlowPeerReports slowPeers = SlowPeerReports.create(
-        ImmutableMap.of("peer1", 0.0, "peer2", 1.0, "peer3", 2.0));
+        ImmutableMap.of(
+            "peer1", outlierMetrics1,
+            "peer2", outlierMetrics2,
+            "peer3", outlierMetrics3));
     SlowPeerReports slowPeersConverted1 = PBHelper.convertSlowPeerInfo(
         PBHelper.convertSlowPeerInfo(slowPeers));
     assertTrue(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
index d0bbd44f1afbb..1c43b39159a99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
@@ -20,6 +20,7 @@
 import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.FAKE_NSINFO;
 import static org.junit.Assert.fail;
 
+import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -45,13 +46,16 @@
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
 import org.apache.hadoop.test.GenericTestUtils;
 
-public class MiniJournalCluster {
+public final class MiniJournalCluster implements Closeable {
+
   public static final String CLUSTER_WAITACTIVE_URI = "waitactive";
   public static class Builder {
     private String baseDir;
     private int numJournalNodes = 3;
     private boolean format = true;
     private final Configuration conf;
+    private int[] httpPorts = null;
+    private int[] rpcPorts = null;
 
     static {
       DefaultMetricsSystem.setMiniClusterMode(true);
@@ -76,6 +80,16 @@ public Builder format(boolean f) {
       return this;
     }
 
+    public Builder setHttpPorts(int... ports) {
+      this.httpPorts = ports;
+      return this;
+    }
+
+    public Builder setRpcPorts(int... ports) {
+      this.rpcPorts = ports;
+      return this;
+    }
+
     public MiniJournalCluster build() throws IOException {
       return new MiniJournalCluster(this);
     }
@@ -99,6 +113,19 @@ private JNInfo(JournalNode node) {
   private final JNInfo[] nodes;
   
   private MiniJournalCluster(Builder b) throws IOException {
+
+    if (b.httpPorts != null && b.httpPorts.length != b.numJournalNodes) {
+      throw new IllegalArgumentException(
+          "Num of http ports (" + b.httpPorts.length + ") should match num of JournalNodes ("
+              + b.numJournalNodes + ")");
+    }
+
+    if (b.rpcPorts != null && b.rpcPorts.length != b.numJournalNodes) {
+      throw new IllegalArgumentException(
+          "Num of rpc ports (" + b.rpcPorts.length + ") should match num of JournalNodes ("
+              + b.numJournalNodes + ")");
+    }
+
     LOG.info("Starting MiniJournalCluster with " +
         b.numJournalNodes + " journal nodes");
     
@@ -173,8 +200,10 @@ private Configuration createConfForNode(Builder b, int idx) {
     Configuration conf = new Configuration(b.conf);
     File logDir = getStorageDir(idx);
     conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, logDir.toString());
-    conf.set(DFSConfigKeys.DFS_JOURNALNODE_RPC_ADDRESS_KEY, "localhost:0");
-    conf.set(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, "localhost:0");
+    int httpPort = b.httpPorts != null ? b.httpPorts[idx] : 0;
+    int rpcPort = b.rpcPorts != null ? b.rpcPorts[idx] : 0;
+    conf.set(DFSConfigKeys.DFS_JOURNALNODE_RPC_ADDRESS_KEY, "localhost:" + rpcPort);
+    conf.set(DFSConfigKeys.DFS_JOURNALNODE_HTTP_ADDRESS_KEY, "localhost:" + httpPort);
     return conf;
   }
 
@@ -274,4 +303,10 @@ public void setNamenodeSharedEditsConf(String jid) {
           .DFS_NAMENODE_SHARED_EDITS_DIR_KEY, quorumJournalURI.toString());
     }
   }
+
+  @Override
+  public void close() throws IOException {
+    this.shutdown();
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
index 3ece3d7e47a68..0791e0ace1c0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java
@@ -35,7 +35,7 @@
 import java.util.List;
 import java.util.Random;
 
-public class MiniQJMHACluster {
+public class MiniQJMHACluster implements AutoCloseable {
   private MiniDFSCluster cluster;
   private MiniJournalCluster journalCluster;
   private final Configuration conf;
@@ -189,4 +189,15 @@ public void shutdown() throws IOException {
     cluster.shutdown();
     journalCluster.shutdown();
   }
+
+  @Override
+  public void close() {
+    try {
+      shutdown();
+    } catch (IOException shutdownFailure) {
+      LOG.warn("Exception while closing journal cluster", shutdownFailure);
+    }
+
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
index cace7c92891ab..ccbbc94c99ede 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java
@@ -22,15 +22,23 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
+import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.qjournal.server.JournalNode;
-import org.junit.Test;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
 
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMiniJournalCluster {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestMiniJournalCluster.class);
+
   @Test
   public void testStartStop() throws IOException {
     Configuration conf = new Configuration();
@@ -52,4 +60,92 @@ public void testStartStop() throws IOException {
       c.shutdown();
     }
   }
+
+  @Test
+  public void testStartStopWithPorts() throws Exception {
+    Configuration conf = new Configuration();
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of http ports (1) should match num of JournalNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniJournalCluster.Builder(conf).setHttpPorts(8481).build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of rpc ports (2) should match num of JournalNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniJournalCluster.Builder(conf).setRpcPorts(8481, 8482).build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of rpc ports (1) should match num of JournalNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniJournalCluster.Builder(conf).setHttpPorts(800, 9000, 10000).setRpcPorts(8481)
+              .build();
+        });
+
+    LambdaTestUtils.intercept(
+        IllegalArgumentException.class,
+        "Num of http ports (4) should match num of JournalNodes (3)",
+        "MiniJournalCluster port validation failed",
+        () -> {
+          new MiniJournalCluster.Builder(conf).setHttpPorts(800, 9000, 1000, 2000)
+              .setRpcPorts(8481, 8482, 8483).build();
+        });
+
+    final Set<Integer> httpAndRpcPorts = NetUtils.getFreeSocketPorts(6);
+    LOG.info("Free socket ports: {}", httpAndRpcPorts);
+
+    for (Integer httpAndRpcPort : httpAndRpcPorts) {
+      assertNotEquals("None of the acquired socket port should not be zero", 0,
+          httpAndRpcPort.intValue());
+    }
+
+    final int[] httpPorts = new int[3];
+    final int[] rpcPorts = new int[3];
+    int httpPortIdx = 0;
+    int rpcPortIdx = 0;
+    for (Integer httpAndRpcPort : httpAndRpcPorts) {
+      if (httpPortIdx < 3) {
+        httpPorts[httpPortIdx++] = httpAndRpcPort;
+      } else {
+        rpcPorts[rpcPortIdx++] = httpAndRpcPort;
+      }
+    }
+
+    LOG.info("Http ports selected: {}", httpPorts);
+    LOG.info("Rpc ports selected: {}", rpcPorts);
+
+    try (MiniJournalCluster miniJournalCluster = new MiniJournalCluster.Builder(conf)
+        .setHttpPorts(httpPorts)
+        .setRpcPorts(rpcPorts).build()) {
+      miniJournalCluster.waitActive();
+      URI uri = miniJournalCluster.getQuorumJournalURI("myjournal");
+      String[] addrs = uri.getAuthority().split(";");
+      assertEquals(3, addrs.length);
+
+      assertEquals(httpPorts[0], miniJournalCluster.getJournalNode(0).getHttpAddress().getPort());
+      assertEquals(httpPorts[1], miniJournalCluster.getJournalNode(1).getHttpAddress().getPort());
+      assertEquals(httpPorts[2], miniJournalCluster.getJournalNode(2).getHttpAddress().getPort());
+
+      assertEquals(rpcPorts[0],
+          miniJournalCluster.getJournalNode(0).getRpcServer().getAddress().getPort());
+      assertEquals(rpcPorts[1],
+          miniJournalCluster.getJournalNode(1).getRpcServer().getAddress().getPort());
+      assertEquals(rpcPorts[2],
+          miniJournalCluster.getJournalNode(2).getRpcServer().getAddress().getPort());
+
+      JournalNode node = miniJournalCluster.getJournalNode(0);
+      String dir = node.getConf().get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY);
+      assertEquals(new File(MiniDFSCluster.getBaseDirectory() + "journalnode-0").getAbsolutePath(),
+          dir);
+    }
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
index 6cdbe2d31d9f3..e9f46e335c6e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
@@ -34,6 +34,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 
@@ -51,7 +52,6 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.ProtobufRpcEngine2;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Sets;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -59,7 +59,7 @@
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -108,7 +108,7 @@ private static long determineMaxIpcNumber() throws Exception {
       qjm.format(FAKE_NSINFO, false);
       doWorkload(cluster, qjm);
       
-      SortedSet<Integer> ipcCounts = Sets.newTreeSet();
+      SortedSet<Integer> ipcCounts = new TreeSet<>();
       for (AsyncLogger l : qjm.getLoggerSetForTests().getLoggersForTests()) {
         InvocationCountingChannel ch = (InvocationCountingChannel)l;
         ch.waitForAllPendingCalls();
@@ -198,7 +198,7 @@ public void testRecoverAfterDoubleFailures() throws Exception {
   public void testUnresolvableHostName() throws Exception {
     expectedException.expect(UnknownHostException.class);
     new QuorumJournalManager(conf,
-        new URI("qjournal://" + "bogus:12345" + "/" + JID), FAKE_NSINFO);
+        new URI("qjournal://" + "bogus.invalid:12345" + "/" + JID), FAKE_NSINFO);
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index 5666ae58b0f41..e2ee2e365d9ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -28,7 +28,7 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-
+import static org.mockito.ArgumentMatchers.eq;
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
@@ -40,11 +40,17 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.hdfs.server.common.Util;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 import org.apache.hadoop.net.MockDomainNameResolver;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
 import org.apache.hadoop.util.Lists;
+import org.mockito.stubbing.Answer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -53,6 +59,7 @@
 import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster;
 import org.apache.hadoop.hdfs.qjournal.QJMTestUtil;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
+import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto;
 import org.apache.hadoop.hdfs.qjournal.server.JournalFaultInjector;
 import org.apache.hadoop.hdfs.qjournal.server.JournalNode;
 import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
@@ -1101,6 +1108,56 @@ public void testSelectViaRpcTwoJNsError() throws Exception {
     }
   }
 
+  /**
+   * Test selecting EditLogInputStream after some journalNode jitter.
+   * Suppose there are 3 journalNodes, JN0 ~ JN2.
+   *  1. JN0 has some abnormal cases when Active Namenode is syncing 10 Edits with first txid 11.
+   *  2. NameNode just ignore the abnormal JN0 and continue to sync Edits to Journal 1 and 2.
+   *  3. JN0 backed to health.
+   *  4. NameNode continue sync 10 Edits with first txid 21.
+   *  5. At this point, there are no Edits 11 ~ 30 in the cache of JN0.
+   *  6. Observer NameNode try to select EditLogInputStream through
+   *     getJournaledEdits with since txId 21.
+   *  7. JN2 has some abnormal cases and caused a slow response.
+   */
+  @Test
+  public void testSelectViaRPCAfterJNJitter() throws Exception {
+    EditLogOutputStream stm = qjm.startLogSegment(
+        1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+    SettableFuture<Void> slowLog = SettableFuture.create();
+    Mockito.doReturn(slowLog).when(spies.get(0))
+        .sendEdits(eq(1L), eq(11L), eq(10), Mockito.any());
+    // Successfully write these edits to JN0 ~ JN2
+    writeTxns(stm, 1, 10);
+    // Failed write these edits to JN0, but successfully write them to JN1 ~ JN2
+    writeTxns(stm, 11, 10);
+    // Successfully write these edits to JN1 ~ JN2
+    writeTxns(stm, 21, 20);
+
+    Semaphore semaphore = new Semaphore(0);
+    spyGetJournaledEdits(0, 21, () -> semaphore.release(1));
+    spyGetJournaledEdits(1, 21, () -> semaphore.release(1));
+    spyGetJournaledEdits(2, 21, () -> semaphore.acquireUninterruptibly(2));
+
+    List<EditLogInputStream> streams = new ArrayList<>();
+    qjm.selectInputStreams(streams, 21, true, true);
+
+    assertEquals(1, streams.size());
+    assertEquals(21, streams.get(0).getFirstTxId());
+    assertEquals(40, streams.get(0).getLastTxId());
+  }
+
+  private void spyGetJournaledEdits(int jnSpyIdx, long fromTxId, Runnable preHook) {
+    Mockito.doAnswer((Answer<ListenableFuture<GetJournaledEditsResponseProto>>) invocation -> {
+      preHook.run();
+      @SuppressWarnings("unchecked")
+      ListenableFuture<GetJournaledEditsResponseProto> result =
+          (ListenableFuture<GetJournaledEditsResponseProto>) invocation.callRealMethod();
+      return result;
+    }).when(spies.get(jnSpyIdx)).getJournaledEdits(fromTxId,
+        QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT);
+  }
+
   @Test
   public void testSelectViaRpcAfterJNRestart() throws Exception {
     EditLogOutputStream stm =
@@ -1208,4 +1265,39 @@ private void checkRecovery(MiniJournalCluster cluster,
           segmentTxId);
     }
   }
+
+  @Test
+  public void testSelectLatestEditsWithoutStreaming() throws Exception {
+    EditLogOutputStream stm = qjm.startLogSegment(
+        1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+    // Successfully write these edits to JN0 ~ JN2
+    writeTxns(stm, 1, 10);
+
+    AtomicInteger atomicInteger = new AtomicInteger(0);
+    spyGetEditLogManifest(0, 11, true, atomicInteger::incrementAndGet);
+    spyGetEditLogManifest(1, 11, true, atomicInteger::incrementAndGet);
+    spyGetEditLogManifest(2, 11, true, atomicInteger::incrementAndGet);
+
+    List<EditLogInputStream> streams = new ArrayList<>();
+    qjm.selectInputStreams(streams, 1, true, true);
+    assertEquals(1, streams.size());
+    assertEquals(1, streams.get(0).getFirstTxId());
+    assertEquals(10, streams.get(0).getLastTxId());
+
+    streams.clear();
+    qjm.selectInputStreams(streams, 11, true, true);
+    assertEquals(0, streams.size());
+    assertEquals(0, atomicInteger.get());
+  }
+
+  private void spyGetEditLogManifest(int jnSpyIdx, long fromTxId,
+      boolean inProgressOk, Runnable preHook) {
+    Mockito.doAnswer((Answer<ListenableFuture<RemoteEditLogManifest>>) invocation -> {
+      preHook.run();
+      @SuppressWarnings("unchecked")
+      ListenableFuture<RemoteEditLogManifest> result =
+          (ListenableFuture<RemoteEditLogManifest>) invocation.callRealMethod();
+      return result;
+    }).when(spies.get(jnSpyIdx)).getEditLogManifest(fromTxId, inProgressOk);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java
new file mode 100644
index 0000000000000..7d9dea0351651
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.qjournal.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestGetJournalEditServlet {
+
+  private final static Configuration CONF = new HdfsConfiguration();
+
+  private final static GetJournalEditServlet SERVLET = new GetJournalEditServlet();
+
+  @BeforeClass
+  public static void setUp() throws ServletException {
+    // Configure Hadoop
+    CONF.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/");
+    CONF.set(DFSConfigKeys.HADOOP_SECURITY_AUTH_TO_LOCAL,
+        "RULE:[2:$1/$2@$0]([nsdj]n/.*@REALM\\.TLD)s/.*/hdfs/\nDEFAULT");
+    CONF.set(DFSConfigKeys.DFS_NAMESERVICES, "ns");
+    CONF.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "nn/_HOST@REALM.TLD");
+
+    // Configure Kerberos UGI
+    UserGroupInformation.setConfiguration(CONF);
+    UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(
+        "jn/somehost@REALM.TLD"));
+
+    // Initialize the servlet
+    ServletConfig config = mock(ServletConfig.class);
+    SERVLET.init(config);
+  }
+
+  /**
+   * Unauthenticated user should be rejected.
+   *
+   * @throws IOException for unexpected validation failures
+   */
+  @Test
+  public void testWithoutUser() throws IOException {
+    // Test: Make a request without specifying a user
+    HttpServletRequest request = mock(HttpServletRequest.class);
+    boolean isValid = SERVLET.isValidRequestor(request, CONF);
+
+    // Verify: The request is invalid
+    assertThat(isValid).isFalse();
+  }
+
+  /**
+   * Namenode requests should be authorized, since it will match the configured namenode.
+   *
+   * @throws IOException for unexpected validation failures
+   */
+  @Test
+  public void testRequestNameNode() throws IOException, ServletException {
+    // Test: Make a request from a namenode
+    HttpServletRequest request = mock(HttpServletRequest.class);
+    when(request.getParameter(UserParam.NAME)).thenReturn("nn/localhost@REALM.TLD");
+    boolean isValid = SERVLET.isValidRequestor(request, CONF);
+
+    assertThat(isValid).isTrue();
+  }
+
+  /**
+   * There is a fallback using the short name, which is used by journalnodes.
+   *
+   * @throws IOException for unexpected validation failures
+   */
+  @Test
+  public void testRequestShortName() throws IOException {
+    // Test: Make a request from a namenode
+    HttpServletRequest request = mock(HttpServletRequest.class);
+    when(request.getParameter(UserParam.NAME)).thenReturn("jn/localhost@REALM.TLD");
+    boolean isValid = SERVLET.isValidRequestor(request, CONF);
+
+    assertThat(isValid).isTrue();
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
index a8d4442bdff28..064dd9e5dd86e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
@@ -142,6 +142,10 @@ public void setup() throws Exception {
           "qjournal://journalnode0:9900;journalnode1:9901/" + journalId);
       conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn2",
           "qjournal://journalnode0:9902;journalnode1:9903/" + journalId);
+    } else if (testName.getMethodName().equals("testConfAbnormalHandlerNumber")) {
+      conf.setInt(DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_KEY, -1);
+    } else if (testName.getMethodName().equals("testConfNormalHandlerNumber")) {
+      conf.setInt(DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_KEY, 10);
     }
     jn = new JournalNode();
     jn.setConf(conf);
@@ -672,4 +676,26 @@ private void setupStaticHostResolution(int journalNodeCount,
     }
   }
 
+  @Test
+  public void testConfNormalHandlerNumber() {
+    int confHandlerNumber = jn.getConf().getInt(
+        DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_KEY,
+        DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT);
+    assertTrue(confHandlerNumber > 0);
+    int handlerCount = jn.getRpcServer().getHandlerCount();
+    assertEquals(confHandlerNumber, handlerCount);
+    assertEquals(10, handlerCount);
+  }
+
+  @Test
+  public void testConfAbnormalHandlerNumber() {
+    int confHandlerCount = jn.getConf().getInt(
+        DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_KEY,
+        DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT);
+    assertTrue(confHandlerCount <= 0);
+    int handlerCount = jn.getRpcServer().getHandlerCount();
+    assertEquals(
+        DFSConfigKeys.DFS_JOURNALNODE_HANDLER_COUNT_DEFAULT,
+        handlerCount);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java
index e571fbce3a805..179e5a9ed6292 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java
@@ -44,6 +44,7 @@ public class TestJournalNodeMXBean {
   
   private static final String NAMESERVICE = "ns1";
   private static final int NUM_JN = 1;
+  private static final int NS_ID = 12345;
   
   private MiniJournalCluster jCluster;
   private JournalNode jn;
@@ -80,9 +81,8 @@ public void testJournalNodeMXBean() throws Exception {
     assertFalse(journalStatus.contains(NAMESERVICE));
 
     // format the journal ns1
-    final NamespaceInfo FAKE_NSINFO = new NamespaceInfo(12345, "mycluster",
-        "my-bp", 0L);
-    jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO, false);
+    final NamespaceInfo fakeNsInfo = new NamespaceInfo(NS_ID, "mycluster", "my-bp", 0L);
+    jn.getOrCreateJournal(NAMESERVICE).format(fakeNsInfo, false);
 
     // check again after format
     // getJournalsStatus
@@ -104,8 +104,16 @@ public void testJournalNodeMXBean() throws Exception {
     String[] clusterId = (String[]) mbs.getAttribute(mxbeanName, "ClusterIds");
     assertEquals(jn.getClusterIds().size(), clusterId.length);
     assertEquals("mycluster", clusterId[0]);
+    long startTime = (long) mbs.getAttribute(mxbeanName, "JNStartedTimeInMillis");
+    assertTrue("JournalNode start time should not be 0", startTime > 0);
+    assertEquals(jn.getJNStartedTimeInMillis(), startTime);
     String version = (String) mbs.getAttribute(mxbeanName, "Version");
     assertEquals(jn.getVersion(), version);
+    String[] journalStorageInfos = (String[]) mbs.getAttribute(mxbeanName, "StorageInfos");
+    assertEquals(jn.getStorageInfos().size(), journalStorageInfos.length);
+    assertTrue(journalStorageInfos[1].contains("ClusterId=mycluster"));
+    assertTrue(journalStorageInfos[1].contains("CreationTime=0"));
+    assertTrue(journalStorageInfos[1].contains("NamespaceId=" + NS_ID));
 
     // restart journal node without formatting
     jCluster = new MiniJournalCluster.Builder(new Configuration()).format(false)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java
index bc4cf3a6ee7b1..28e36e03bfaa5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.qjournal.server;
 
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
 import java.util.function.Supplier;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -33,6 +35,9 @@
 import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile;
 import static org.apache.hadoop.hdfs.server.namenode.FileJournalManager
     .getLogFile;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Lists;
@@ -94,18 +99,53 @@ public void shutDownMiniCluster() throws IOException {
     }
   }
 
+  /**
+   * Test that the "self exclusion" works when there are multiple JournalNode instances running on
+   * the same server, but on different ports.
+   */
+  @Test
+  public void testJournalNodeExcludesSelfMultilpePorts() throws URISyntaxException, IOException {
+    String uri = qjmhaCluster.getJournalCluster().getQuorumJournalURI("ns1").toString();
+    JournalNodeSyncer syncer = jCluster.getJournalNode(0).getJournalSyncer("ns1");
+
+    // Test: Get the Journal address list for the default configuration
+    List<InetSocketAddress> addrList = syncer.getJournalAddrList(uri);
+
+    // Verify: One of the addresses should be excluded so that the node isn't syncing with itself
+    assertEquals(2, addrList.size());
+  }
+
+  /**
+   * Test that the "self exclusion" works when there a host uses a wildcard address.
+   */
+  @Test
+  public void testJournalNodeExcludesSelfWildCard() throws URISyntaxException, IOException {
+    String uri = qjmhaCluster.getJournalCluster().getQuorumJournalURI("ns1").toString();
+    JournalNodeSyncer syncer = jCluster.getJournalNode(0).getJournalSyncer("ns1");
+
+    // Test: Request the same Journal address list, but using the IPv4 "0.0.0.0" which is commonly
+    // used as a bind host.
+    String boundHostUri = uri.replaceAll("127.0.0.1", "0.0.0.0");
+    List<InetSocketAddress> boundHostAddrList = syncer.getJournalAddrList(boundHostUri);
+
+    // Verify: One of the address should be excluded so that the node isn't syncing with itself
+    assertEquals(2, boundHostAddrList.size());
+  }
+
   @Test(timeout=30000)
   public void testJournalNodeSync() throws Exception {
 
     //As by default 3 journal nodes are started;
     for(int i=0; i<3; i++) {
-      Assert.assertEquals(true,
+      assertEquals(true,
           jCluster.getJournalNode(i).getJournalSyncerStatus("ns1"));
     }
 
     File firstJournalDir = jCluster.getJournalDir(0, jid);
     File firstJournalCurrentDir = new StorageDirectory(firstJournalDir)
         .getCurrentDir();
+    assertThat(jCluster.getJournalNode(0).getRpcServer().getRpcServer().getRpcMetrics()
+        .getTotalRequests()).isGreaterThan(20);
 
     // Generate some edit logs and delete one.
     long firstTxId = generateEditLog();
@@ -382,13 +422,13 @@ public void testSyncDuringRollingUpgrade() throws Exception {
           HdfsConstants.RollingUpgradeAction.PREPARE);
 
     //query rolling upgrade
-    Assert.assertEquals(info, dfsActive.rollingUpgrade(
+    assertEquals(info, dfsActive.rollingUpgrade(
         HdfsConstants.RollingUpgradeAction.QUERY));
 
     // Restart the Standby NN with rollingUpgrade option
     dfsCluster.restartNameNode(standbyNNindex, true,
         "-rollingUpgrade", "started");
-    Assert.assertEquals(info, dfsActive.rollingUpgrade(
+    assertEquals(info, dfsActive.rollingUpgrade(
         HdfsConstants.RollingUpgradeAction.QUERY));
 
     // Do some edits and delete some edit logs
@@ -416,7 +456,7 @@ public void testSyncDuringRollingUpgrade() throws Exception {
     // Restart the current standby NN (previously active)
     dfsCluster.restartNameNode(standbyNNindex, true,
         "-rollingUpgrade", "started");
-    Assert.assertEquals(info, dfsActive.rollingUpgrade(
+    assertEquals(info, dfsActive.rollingUpgrade(
         HdfsConstants.RollingUpgradeAction.QUERY));
     dfsCluster.waitActive();
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournaledEditsCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournaledEditsCache.java
index 2a178a1547ec4..82b8b587694b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournaledEditsCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournaledEditsCache.java
@@ -221,6 +221,27 @@ public void testCacheMalformedInput() throws Exception {
     cache.retrieveEdits(-1, 10, new ArrayList<>());
   }
 
+  @Test
+  public void testCacheSizeConfigs() {
+    // Assert the default configs.
+    Configuration config = new Configuration();
+    cache = new JournaledEditsCache(config);
+    assertEquals((int) (Runtime.getRuntime().maxMemory() * 0.5f), cache.getCapacity());
+
+    // Set dfs.journalnode.edit-cache-size.bytes.
+    Configuration config1 = new Configuration();
+    config1.setInt(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, 1);
+    config1.setFloat(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, 0.1f);
+    cache = new JournaledEditsCache(config1);
+    assertEquals(1, cache.getCapacity());
+
+    // Don't set dfs.journalnode.edit-cache-size.bytes.
+    Configuration config2 = new Configuration();
+    config2.setFloat(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_FRACTION_KEY, 0.1f);
+    cache = new JournaledEditsCache(config2);
+    assertEquals((int) (Runtime.getRuntime().maxMemory() * 0.1f), cache.getCapacity());
+  }
+
   private void storeEdits(int startTxn, int endTxn) throws Exception {
     cache.storeEdits(createTxnData(startTxn, endTxn - startTxn + 1), startTxn,
         endTxn, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 8ec968d36c226..49d7a7ea97b33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -470,6 +470,19 @@ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
   static void waitForBalancer(long totalUsedSpace, long totalCapacity,
       ClientProtocol client, MiniDFSCluster cluster, BalancerParameters p,
       int expectedExcludedNodes) throws IOException, TimeoutException {
+    waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, expectedExcludedNodes, true);
+  }
+
+  /**
+   * Wait until balanced: each datanode gives utilization within.
+   * BALANCE_ALLOWED_VARIANCE of average
+   * @throws IOException
+   * @throws TimeoutException
+   */
+  static void waitForBalancer(long totalUsedSpace, long totalCapacity,
+      ClientProtocol client, MiniDFSCluster cluster, BalancerParameters p,
+      int expectedExcludedNodes, boolean checkExcludeNodesUtilization)
+      throws IOException, TimeoutException {
     long timeout = TIMEOUT;
     long failtime = (timeout <= 0L) ? Long.MAX_VALUE
         : Time.monotonicNow() + timeout;
@@ -491,7 +504,9 @@ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
         double nodeUtilization = ((double)datanode.getDfsUsed())
             / datanode.getCapacity();
         if (Dispatcher.Util.isExcluded(p.getExcludedNodes(), datanode)) {
-          assertTrue(nodeUtilization == 0);
+          if (checkExcludeNodesUtilization) {
+            assertTrue(nodeUtilization == 0);
+          }
           actualExcludedNodeCount++;
           continue;
         }
@@ -778,6 +793,12 @@ private void runBalancer(Configuration conf, long totalUsedSpace,
   private void runBalancer(Configuration conf, long totalUsedSpace,
       long totalCapacity, BalancerParameters p, int excludedNodes)
       throws Exception {
+    runBalancer(conf, totalUsedSpace, totalCapacity, p, excludedNodes, true);
+  }
+
+  private void runBalancer(Configuration conf, long totalUsedSpace,
+      long totalCapacity, BalancerParameters p, int excludedNodes,
+      boolean checkExcludeNodesUtilization) throws Exception {
     waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
 
     int retry = 5;
@@ -798,7 +819,7 @@ private void runBalancer(Configuration conf, long totalUsedSpace,
       LOG.info("  .");
       try {
         waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p,
-            excludedNodes);
+            excludedNodes, checkExcludeNodesUtilization);
       } catch (TimeoutException e) {
         // See HDFS-11682. NN may not get heartbeat to reflect the newest
         // block changes.
@@ -1666,6 +1687,11 @@ private void doTestBalancerWithStripedFile(Configuration conf) throws Exception
       BalancerParameters p = BalancerParameters.DEFAULT;
       runBalancer(conf, totalUsedSpace, totalCapacity, p, 0);
 
+      // namenode will ask datanode to delete replicas in heartbeat response
+      cluster.triggerHeartbeats();
+      // namenode will update block locations according to the report
+      cluster.triggerDeletionReports();
+
       // verify locations of striped blocks
       locatedBlocks = client.getBlockLocations(fileName, 0, fileLen);
       StripedFileTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
@@ -1678,6 +1704,103 @@ private void doTestBalancerWithStripedFile(Configuration conf) throws Exception
     }
   }
 
+  @Test
+  public void testBalancerWithExcludeListWithStripedFile() throws Exception {
+    Configuration conf = new Configuration();
+    initConfWithStripe(conf);
+    NameNodeConnector.setWrite2IdFile(true);
+    doTestBalancerWithExcludeListWithStripedFile(conf);
+    NameNodeConnector.setWrite2IdFile(false);
+  }
+
+  private void doTestBalancerWithExcludeListWithStripedFile(Configuration conf) throws Exception {
+    int numOfDatanodes = dataBlocks + parityBlocks + 5;
+    int numOfRacks = dataBlocks;
+    long capacity = 20 * defaultBlockSize;
+    long[] capacities = new long[numOfDatanodes];
+    Arrays.fill(capacities, capacity);
+    String[] racks = new String[numOfDatanodes];
+    for (int i = 0; i < numOfDatanodes; i++) {
+      racks[i] = "/rack" + (i % numOfRacks);
+    }
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(numOfDatanodes)
+        .racks(racks)
+        .simulatedCapacities(capacities)
+        .build();
+
+    try {
+      cluster.waitActive();
+      client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+          ClientProtocol.class).getProxy();
+      client.enableErasureCodingPolicy(
+          StripedFileTestUtil.getDefaultECPolicy().getName());
+      client.setErasureCodingPolicy("/",
+          StripedFileTestUtil.getDefaultECPolicy().getName());
+
+      long totalCapacity = sum(capacities);
+
+      // fill up the cluster with 30% data. It'll be 45% full plus parity.
+      long fileLen = totalCapacity * 3 / 10;
+      long totalUsedSpace = fileLen * (dataBlocks + parityBlocks) / dataBlocks;
+      FileSystem fs = cluster.getFileSystem(0);
+      DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, r.nextLong());
+
+      // verify locations of striped blocks
+      LocatedBlocks locatedBlocks = client.getBlockLocations(fileName, 0, fileLen);
+      StripedFileTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
+
+      // get datanode report
+      DatanodeInfo[] datanodeReport = client.getDatanodeReport(DatanodeReportType.ALL);
+      long totalBlocks = 0;
+      for (DatanodeInfo dn : datanodeReport) {
+        totalBlocks += dn.getNumBlocks();
+      }
+
+      // add datanode in new rack
+      String newRack = "/rack" + (++numOfRacks);
+      cluster.startDataNodes(conf, 2, true, null,
+          new String[]{newRack, newRack}, null,
+          new long[]{capacity, capacity});
+      totalCapacity += capacity*2;
+      cluster.triggerHeartbeats();
+
+      // add datanode to exclude list
+      Set<String> excludedList = new HashSet<>();
+      excludedList.add(datanodeReport[0].getXferAddr());
+      BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+      pBuilder.setExcludedNodes(excludedList);
+
+      // start balancer and check the failed num of moving task
+      runBalancer(conf, totalUsedSpace, totalCapacity, pBuilder.build(),
+          excludedList.size(), false);
+
+      // check total blocks, max wait time 60s
+      final long blocksBeforeBalancer = totalBlocks;
+      GenericTestUtils.waitFor(() -> {
+        DatanodeInfo[] datanodeInfos = null;
+        try {
+          cluster.triggerHeartbeats();
+          datanodeInfos = client.getDatanodeReport(DatanodeReportType.ALL);
+        } catch (IOException e) {
+          Assert.fail(e.getMessage());
+        }
+        long blocksAfterBalancer = 0;
+        for (DatanodeInfo dn : datanodeInfos) {
+          blocksAfterBalancer += dn.getNumBlocks();
+        }
+        return blocksBeforeBalancer == blocksAfterBalancer;
+      }, 3000, 60000);
+
+      // verify locations of striped blocks
+      locatedBlocks = client.getBlockLocations(fileName, 0, fileLen);
+      StripedFileTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
+
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   private void testNullStripedBlocks(Configuration conf) throws IOException {
     NameNodeConnector nnc = NameNodeConnector.newNameNodeConnectors(
         DFSUtil.getInternalNsRpcUris(conf),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java
index 0d6300cf05e91..ba36a42e0c3db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerLongRunningTasks.java
@@ -262,7 +262,7 @@ public void testBalancerWithRamDisk() throws Exception {
     long[][] storageCapacities = new long[][]{{ramDiskStorageLimit,
             diskStorageLimit}};
     cluster.startDataNodes(conf, replicationFactor, storageTypes, true, null,
-        null, null, storageCapacities, null, false, false, false, null);
+        null, null, storageCapacities, null, false, false, false, null, null, null);
 
     cluster.triggerHeartbeats();
     Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerService.java
index 0ec0431555d46..a75b5092df768 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerService.java
@@ -126,6 +126,7 @@ public void testBalancerServiceBalanceTwice() throws Exception {
     TestBalancer.initConf(conf);
     try {
       setupCluster(conf);
+      TestBalancerWithHANameNodes.waitStoragesNoStale(cluster, client, 0);
       long totalCapacity = addOneDataNode(conf); // make cluster imbalanced
 
       Thread balancerThread =
@@ -193,6 +194,7 @@ public void testBalancerServiceOnError() throws Exception {
       cluster.transitionToActive(0);
       cluster.waitActive();
 
+      TestBalancerWithHANameNodes.waitStoragesNoStale(cluster, client, 0);
       long totalCapacity = addOneDataNode(conf);
       TestBalancer.waitForBalancer(totalUsedSpace, totalCapacity, client,
           cluster, BalancerParameters.DEFAULT);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index 9b76da0dca22e..d69051c8d7af7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -18,9 +18,7 @@
 package org.apache.hadoop.hdfs.server.balancer;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -47,12 +45,17 @@
 import org.apache.hadoop.hdfs.NameNodeProxies;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
 import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
 import org.junit.Test;
 import org.slf4j.LoggerFactory;
@@ -75,6 +78,26 @@ public class TestBalancerWithHANameNodes {
     TestBalancer.initTestSetup();
   }
 
+  public static void waitStoragesNoStale(MiniDFSCluster cluster,
+      ClientProtocol client, int nnIndex) throws Exception {
+    // trigger a full block report and wait all storages out of stale
+    cluster.triggerBlockReports();
+    DatanodeInfo[] dataNodes = client.getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
+    GenericTestUtils.waitFor(() -> {
+      BlockManager bm = cluster.getNamesystem(nnIndex).getBlockManager();
+      for (DatanodeInfo dn : dataNodes) {
+        DatanodeStorageInfo[] storageInfos = bm.getDatanodeManager()
+            .getDatanode(dn.getDatanodeUuid()).getStorageInfos();
+        for (DatanodeStorageInfo s : storageInfos) {
+          if (s.areBlockContentsStale()) {
+            return false;
+          }
+        }
+      }
+      return true;
+    }, 300, 60000);
+  }
+
   /**
    * Test a cluster with even distribution, then a new empty node is added to
    * the cluster. Test start a cluster with specified number of nodes, and fills
@@ -103,13 +126,17 @@ public void testBalancerWithHANameNodes() throws Exception {
       client = NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
           ClientProtocol.class).getProxy();
 
-      doTest(conf);
+      doTest(conf, true);
     } finally {
       cluster.shutdown();
     }
   }
 
   void doTest(Configuration conf) throws Exception {
+    doTest(conf, false);
+  }
+
+  void doTest(Configuration conf, boolean withHA) throws Exception {
     int numOfDatanodes = TEST_CAPACITIES.length;
     long totalCapacity = TestBalancer.sum(TEST_CAPACITIES);
     // fill up the cluster to be 30% full
@@ -117,12 +144,19 @@ void doTest(Configuration conf) throws Exception {
     TestBalancer.createFile(cluster, TestBalancer.filePath, totalUsedSpace
         / numOfDatanodes, (short) numOfDatanodes, 0);
 
-    boolean isRequestStandby = conf.getBoolean(
-        DFS_HA_ALLOW_STALE_READ_KEY, DFS_HA_ALLOW_STALE_READ_DEFAULT);
+    boolean isRequestStandby = !conf.getBoolean(
+        DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY,
+        DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT);
     if (isRequestStandby) {
       HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
           cluster.getNameNode(1));
     }
+
+    // all storages are stale after HA
+    if (withHA) {
+      waitStoragesNoStale(cluster, client, 0);
+    }
+
     // start up an empty node with the same capacity and on the same rack
     long newNodeCapacity = TestBalancer.CAPACITY; // new node's capacity
     String newNodeRack = TestBalancer.RACK2; // new node's rack
@@ -147,7 +181,7 @@ void doTest(Configuration conf) throws Exception {
   @Test(timeout = 60000)
   public void testBalancerRequestSBNWithHA() throws Exception {
     Configuration conf = new HdfsConfiguration();
-    conf.setBoolean(DFS_HA_ALLOW_STALE_READ_KEY, true);
+    conf.setBoolean(DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, false);
     conf.setLong(DFS_HA_TAILEDITS_PERIOD_KEY, 1);
     //conf.setBoolean(DFS_HA_BALANCER_REQUEST_STANDBY_KEY, true);
     TestBalancer.initConf(conf);
@@ -168,15 +202,19 @@ public void testBalancerRequestSBNWithHA() throws Exception {
     try {
       cluster.waitActive();
       cluster.transitionToActive(0);
+      String standbyNameNode = cluster.getNameNode(1).
+          getNameNodeAddress().getHostString();
       Thread.sleep(500);
       client = NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
           ClientProtocol.class).getProxy();
       doTest(conf);
       // Check getBlocks request to Standby NameNode.
       assertTrue(log.getOutput().contains(
-          "Request #getBlocks to Standby NameNode success."));
+          "Request #getBlocks to Standby NameNode success. remoteAddress: " +
+            standbyNameNode));
       assertTrue(log.getOutput().contains(
-          "Request #getLiveDatanodeStorageReport to Standby NameNode success"));
+          "Request #getLiveDatanodeStorageReport to Standby NameNode success. " +
+            "remoteAddress: " + standbyNameNode));
     } finally {
       cluster.shutdown();
     }
@@ -294,8 +332,8 @@ nsId, new Path("/test"),
       nncActive.close();
 
       // Request to standby namenode.
-      conf.setBoolean(DFSConfigKeys.DFS_HA_ALLOW_STALE_READ_KEY,
-          true);
+      conf.setBoolean(DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY,
+          false);
       NameNodeConnector nncStandby = new NameNodeConnector(
           "nncStandby", namenode,
           nsId, new Path("/test"),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java
index ec86093ad54a2..1e75452d3d86b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java
@@ -50,6 +50,7 @@ abstract public class BaseReplicationPolicyTest {
   protected NameNode namenode;
   protected DatanodeManager dnManager;
   protected BlockPlacementPolicy replicator;
+  private BlockPlacementPolicy striptedPolicy;
   protected final String filename = "/dummyfile.txt";
   protected DatanodeStorageInfo[] storages;
   protected String blockPlacementPolicy;
@@ -90,6 +91,7 @@ public void setupCluster() throws Exception {
 
     final BlockManager bm = namenode.getNamesystem().getBlockManager();
     replicator = bm.getBlockPlacementPolicy();
+    striptedPolicy = bm.getStriptedBlockPlacementPolicy();
     cluster = bm.getDatanodeManager().getNetworkTopology();
     dnManager = bm.getDatanodeManager();
     // construct network topology
@@ -111,6 +113,10 @@ void updateHeartbeatWithUsage() {
     }
   }
 
+  public BlockPlacementPolicy getStriptedPolicy() {
+    return striptedPolicy;
+  }
+
   @After
   public void tearDown() throws Exception {
     namenode.stop();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 2b8804c12cc6a..4fa320ac29e6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -36,9 +36,12 @@
 import org.apache.hadoop.test.Whitebox;
 import org.junit.Assert;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 public class BlockManagerTestUtil {
+
+  static final long SLEEP_TIME = 1000;
+
   public static void setNodeReplicationLimit(final BlockManager blockManager,
       final int limit) {
     blockManager.maxReplicationStreams = limit;
@@ -178,7 +181,20 @@ public static HeartbeatManager getHeartbeatManager(
    */
   public static  CorruptReplicasMap getCorruptReplicas(final BlockManager blockManager){
     return blockManager.corruptReplicas;
-    
+
+  }
+
+  /**
+   * Wait for the processing of the marked deleted block to complete.
+   */
+  public static void waitForMarkedDeleteQueueIsEmpty(
+      BlockManager blockManager) throws InterruptedException {
+    while (true) {
+      if (blockManager.getMarkedDeleteQueue().isEmpty()) {
+        return;
+      }
+      Thread.sleep(SLEEP_TIME);
+    }
   }
 
   /**
@@ -367,6 +383,16 @@ public static void recheckDecommissionState(DatanodeManager dm)
     dm.getDatanodeAdminManager().runMonitorForTest();
   }
 
+  /**
+   * Have BlockManager check isNodeHealthyForDecommissionOrMaintenance for a given datanode.
+   * @param blockManager the BlockManager to check against
+   * @param dn the datanode to check
+   */
+  public static boolean isNodeHealthyForDecommissionOrMaintenance(BlockManager blockManager,
+      DatanodeDescriptor dn) {
+    return blockManager.isNodeHealthyForDecommissionOrMaintenance(dn);
+  }
+
   /**
    * add block to the replicateBlocks queue of the Datanode
    */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
index f58961eb9d072..af7690d681e49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
@@ -173,6 +173,52 @@ public void testChooseDataNode() {
     }
   }
 
+  @Test
+  public void testChooseSimilarDataNode() {
+    DatanodeDescriptor[] tolerateDataNodes;
+    DatanodeStorageInfo[] tolerateStorages;
+    int capacity  = 3;
+    Collection<Node> allTolerateNodes = new ArrayList<>(capacity);
+    String[] ownerRackOfTolerateNodes = new String[capacity];
+    for (int i = 0; i < capacity; i++) {
+      ownerRackOfTolerateNodes[i] = "rack"+i;
+    }
+    tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes);
+    tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages);
+
+    Collections.addAll(allTolerateNodes, tolerateDataNodes);
+    final BlockManager bm = namenode.getNamesystem().getBlockManager();
+    AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy =
+            (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy();
+
+    updateHeartbeatWithUsage(tolerateDataNodes[0],
+            20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * blockSize, 0L, 0L, 0L, 0, 0);
+
+    updateHeartbeatWithUsage(tolerateDataNodes[1],
+            11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * blockSize, 0L, 0L, 0L, 0, 0);
+
+    updateHeartbeatWithUsage(tolerateDataNodes[2],
+            10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * blockSize, 0L, 0L, 0L, 0, 0);
+
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0],
+            tolerateDataNodes[1], false) == 0);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1],
+            tolerateDataNodes[0], false) == 0);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0],
+            tolerateDataNodes[2], false) == -1);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2],
+            tolerateDataNodes[0], false) == 1);
+  }
+
   @AfterClass
   public static void teardownCluster() {
     if (namenode != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java
index 179c6c6b6931c..7acf132be75eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java
@@ -41,6 +41,7 @@
 import java.util.HashSet;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests AvailableSpaceRackFaultTolerant block placement policy.
@@ -64,7 +65,7 @@ public class TestAvailableSpaceRackFaultTolerantBPP {
   public static void setupCluster() throws Exception {
     conf = new HdfsConfiguration();
     conf.setFloat(
-        DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
+        DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
         0.6f);
     String[] racks = new String[NUM_RACKS];
     for (int i = 0; i < NUM_RACKS; i++) {
@@ -206,6 +207,52 @@ public void testMaxRackAllocation() {
     assertEquals(REPLICA, racks.size());
   }
 
+  @Test
+  public void testChooseSimilarDataNode() {
+    DatanodeDescriptor[] tolerateDataNodes;
+    DatanodeStorageInfo[] tolerateStorages;
+    int capacity  = 3;
+    Collection<Node> allTolerateNodes = new ArrayList<>(capacity);
+    String[] ownerRackOfTolerateNodes = new String[capacity];
+    for (int i = 0; i < capacity; i++) {
+      ownerRackOfTolerateNodes[i] = "rack"+i;
+    }
+    tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes);
+    tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages);
+
+    Collections.addAll(allTolerateNodes, tolerateDataNodes);
+    final BlockManager bm = namenode.getNamesystem().getBlockManager();
+    AvailableSpaceRackFaultTolerantBlockPlacementPolicy toleratePlacementPolicy =
+            (AvailableSpaceRackFaultTolerantBlockPlacementPolicy)bm.getBlockPlacementPolicy();
+
+    updateHeartbeatWithUsage(tolerateDataNodes[0],
+            20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
+
+    updateHeartbeatWithUsage(tolerateDataNodes[1],
+            11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
+
+    updateHeartbeatWithUsage(tolerateDataNodes[2],
+            10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,
+            HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+                    * BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
+
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0],
+            tolerateDataNodes[1]) == 0);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1],
+            tolerateDataNodes[0]) == 0);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0],
+            tolerateDataNodes[2]) == -1);
+    assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2],
+            tolerateDataNodes[0]) == 1);
+  }
+
   @AfterClass
   public static void teardownCluster() {
     if (namenode != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index d5e0a99fe789b..04f474f67ef90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -700,6 +700,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception {
             new NumberReplicas(),
             new ArrayList<Byte>(),
             new ArrayList<Byte>(),
+            new ArrayList<Byte>(),
             LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY)[0]);
 
     assertEquals("Does not choose a source node for a less-than-highest-priority"
@@ -712,6 +713,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception {
             new NumberReplicas(),
             new ArrayList<Byte>(),
             new ArrayList<Byte>(),
+            new ArrayList<Byte>(),
             LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY).length);
 
     // Increase the replication count to test replication count > hard limit
@@ -727,6 +729,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception {
             new NumberReplicas(),
             new ArrayList<Byte>(),
             new ArrayList<Byte>(),
+            new ArrayList<Byte>(),
             LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY).length);
   }
 
@@ -773,6 +776,7 @@ public void testChooseSrcDatanodesWithDupEC() throws Exception {
     NumberReplicas numReplicas = new NumberReplicas();
     List<Byte> liveBlockIndices = new ArrayList<>();
     List<Byte> liveBusyBlockIndices = new ArrayList<>();
+    List<Byte> excludeReconstructedIndices = new ArrayList<>();
 
     bm.chooseSourceDatanodes(
             aBlockInfoStriped,
@@ -780,6 +784,7 @@ public void testChooseSrcDatanodesWithDupEC() throws Exception {
             liveNodes,
             numReplicas, liveBlockIndices,
             liveBusyBlockIndices,
+            excludeReconstructedIndices,
             LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY);
 
     assertEquals("Choose the source node for reconstruction with one node reach"
@@ -836,6 +841,7 @@ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception {
     NumberReplicas numReplicas = new NumberReplicas();
     List<Byte> liveBlockIndices = new ArrayList<>();
     List<Byte> liveBusyBlockIndices = new ArrayList<>();
+    List<Byte> excludeReconstructedIndices = new ArrayList<>();
 
     bm.chooseSourceDatanodes(
         aBlockInfoStriped,
@@ -843,6 +849,7 @@ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception {
         nodesContainingLiveReplicas,
         numReplicas, liveBlockIndices,
         liveBusyBlockIndices,
+        excludeReconstructedIndices,
         LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY);
     assertEquals("There are 5 live replicas in " +
             "[ds2, ds3, ds4, ds5, ds6] datanodes ",
@@ -852,6 +859,156 @@ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception {
         0, numReplicas.redundantInternalBlocks());
   }
 
+  @Test
+  public void testSkipReconstructionWithManyBusyNodes() {
+    NameNode.initMetrics(new Configuration(), HdfsServerConstants.NamenodeRole.NAMENODE);
+    long blockId = -9223372036854775776L; // real ec block id
+    // RS-3-2 EC policy
+    ErasureCodingPolicy ecPolicy =
+        SystemErasureCodingPolicies.getPolicies().get(1);
+
+    // create an EC block group: 3 data blocks + 2 parity blocks
+    Block aBlockGroup = new Block(blockId, ecPolicy.getCellSize() * ecPolicy.getNumDataUnits(), 0);
+    BlockInfoStriped aBlockInfoStriped = new BlockInfoStriped(aBlockGroup, ecPolicy);
+
+    // create 4 storageInfo, which means 1 block is missing
+    DatanodeStorageInfo ds1 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage1", "1.1.1.1", "rack1", "host1");
+    DatanodeStorageInfo ds2 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage2", "2.2.2.2", "rack2", "host2");
+    DatanodeStorageInfo ds3 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage3", "3.3.3.3", "rack3", "host3");
+    DatanodeStorageInfo ds4 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage4", "4.4.4.4", "rack4", "host4");
+
+    // link block with storage
+    aBlockInfoStriped.addStorage(ds1, aBlockGroup);
+    aBlockInfoStriped.addStorage(ds2, new Block(blockId + 1, 0, 0));
+    aBlockInfoStriped.addStorage(ds3, new Block(blockId + 2, 0, 0));
+    aBlockInfoStriped.addStorage(ds4, new Block(blockId + 3, 0, 0));
+
+    addEcBlockToBM(blockId, ecPolicy);
+    aBlockInfoStriped.setBlockCollectionId(mockINodeId);
+
+    // reconstruction should be scheduled
+    BlockReconstructionWork work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNotNull(work);
+
+    // simulate the 2 nodes reach maxReplicationStreams
+    for(int i = 0; i < bm.maxReplicationStreams; i++){
+      ds3.getDatanodeDescriptor().incrementPendingReplicationWithoutTargets();
+      ds4.getDatanodeDescriptor().incrementPendingReplicationWithoutTargets();
+    }
+
+    // reconstruction should be skipped since the number of non-busy nodes are not enough
+    work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNull(work);
+  }
+
+  @Test
+  public void testSkipReconstructionWithManyBusyNodes2() {
+    NameNode.initMetrics(new Configuration(), HdfsServerConstants.NamenodeRole.NAMENODE);
+    long blockId = -9223372036854775776L; // real ec block id
+    // RS-3-2 EC policy
+    ErasureCodingPolicy ecPolicy =
+        SystemErasureCodingPolicies.getPolicies().get(1);
+
+    // create an EC block group: 2 data blocks + 2 parity blocks
+    Block aBlockGroup = new Block(blockId,
+        ecPolicy.getCellSize() * (ecPolicy.getNumDataUnits() - 1), 0);
+    BlockInfoStriped aBlockInfoStriped = new BlockInfoStriped(aBlockGroup, ecPolicy);
+
+    // create 3 storageInfo, which means 1 block is missing
+    DatanodeStorageInfo ds1 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage1", "1.1.1.1", "rack1", "host1");
+    DatanodeStorageInfo ds2 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage2", "2.2.2.2", "rack2", "host2");
+    DatanodeStorageInfo ds3 = DFSTestUtil.createDatanodeStorageInfo(
+        "storage3", "3.3.3.3", "rack3", "host3");
+
+    // link block with storage
+    aBlockInfoStriped.addStorage(ds1, aBlockGroup);
+    aBlockInfoStriped.addStorage(ds2, new Block(blockId + 1, 0, 0));
+    aBlockInfoStriped.addStorage(ds3, new Block(blockId + 2, 0, 0));
+
+    addEcBlockToBM(blockId, ecPolicy);
+    aBlockInfoStriped.setBlockCollectionId(mockINodeId);
+
+    // reconstruction should be scheduled
+    BlockReconstructionWork work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNotNull(work);
+
+    // simulate the 1 node reaches maxReplicationStreams
+    for(int i = 0; i < bm.maxReplicationStreams; i++){
+      ds2.getDatanodeDescriptor().incrementPendingReplicationWithoutTargets();
+    }
+
+    // reconstruction should still be scheduled since there are 2 source nodes to create 2 blocks
+    work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNotNull(work);
+
+    // simulate the 1 more node reaches maxReplicationStreams
+    for(int i = 0; i < bm.maxReplicationStreams; i++){
+      ds3.getDatanodeDescriptor().incrementPendingReplicationWithoutTargets();
+    }
+
+    // reconstruction should be skipped since the number of non-busy nodes are not enough
+    work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNull(work);
+  }
+
+  @Test
+  public void testSkipReconstructionWithManyBusyNodes3() {
+    NameNode.initMetrics(new Configuration(), HdfsServerConstants.NamenodeRole.NAMENODE);
+    long blockId = -9223372036854775776L; // Real ec block id
+    // RS-3-2 EC policy
+    ErasureCodingPolicy ecPolicy =
+            SystemErasureCodingPolicies.getPolicies().get(1);
+
+    // Create an EC block group: 3 data blocks + 2 parity blocks.
+    Block aBlockGroup = new Block(blockId, ecPolicy.getCellSize() * ecPolicy.getNumDataUnits(), 0);
+    BlockInfoStriped aBlockInfoStriped = new BlockInfoStriped(aBlockGroup, ecPolicy);
+
+    // Create 4 storageInfo, which means 1 block is missing.
+    DatanodeStorageInfo ds1 = DFSTestUtil.createDatanodeStorageInfo(
+            "storage1", "1.1.1.1", "rack1", "host1");
+    DatanodeStorageInfo ds2 = DFSTestUtil.createDatanodeStorageInfo(
+            "storage2", "2.2.2.2", "rack2", "host2");
+    DatanodeStorageInfo ds3 = DFSTestUtil.createDatanodeStorageInfo(
+            "storage3", "3.3.3.3", "rack3", "host3");
+    DatanodeStorageInfo ds4 = DFSTestUtil.createDatanodeStorageInfo(
+            "storage4", "4.4.4.4", "rack4", "host4");
+
+    // Link block with storage.
+    aBlockInfoStriped.addStorage(ds1, aBlockGroup);
+    aBlockInfoStriped.addStorage(ds2, new Block(blockId + 1, 0, 0));
+    aBlockInfoStriped.addStorage(ds3, new Block(blockId + 2, 0, 0));
+    aBlockInfoStriped.addStorage(ds4, new Block(blockId + 3, 0, 0));
+
+    addEcBlockToBM(blockId, ecPolicy);
+    aBlockInfoStriped.setBlockCollectionId(mockINodeId);
+
+    // Reconstruction should be scheduled.
+    BlockReconstructionWork work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNotNull(work);
+
+    ExtendedBlock dummyBlock = new ExtendedBlock("bpid", 1, 1, 1);
+    DatanodeDescriptor dummyDD = ds1.getDatanodeDescriptor();
+    DatanodeDescriptor[] dummyDDArray = new DatanodeDescriptor[]{dummyDD};
+    DatanodeStorageInfo[] dummyDSArray = new DatanodeStorageInfo[]{ds1};
+    // Simulate the 2 nodes reach maxReplicationStreams.
+    for(int i = 0; i < bm.maxReplicationStreams; i++){ //Add some dummy EC reconstruction task.
+      ds3.getDatanodeDescriptor().addBlockToBeErasureCoded(dummyBlock, dummyDDArray,
+              dummyDSArray, new byte[0], new byte[0], ecPolicy);
+      ds4.getDatanodeDescriptor().addBlockToBeErasureCoded(dummyBlock, dummyDDArray,
+              dummyDSArray, new byte[0], new byte[0], ecPolicy);
+    }
+
+    // Reconstruction should be skipped since the number of non-busy nodes are not enough.
+    work = bm.scheduleReconstruction(aBlockInfoStriped, 3);
+    assertNull(work);
+  }
+
   @Test
   public void testFavorDecomUntilHardLimit() throws Exception {
     bm.maxReplicationStreams = 0;
@@ -877,6 +1034,7 @@ public void testFavorDecomUntilHardLimit() throws Exception {
             new NumberReplicas(),
             new LinkedList<Byte>(),
             new ArrayList<Byte>(),
+            new ArrayList<Byte>(),
             LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY)[0]);
 
 
@@ -893,6 +1051,7 @@ public void testFavorDecomUntilHardLimit() throws Exception {
             new NumberReplicas(),
             new LinkedList<Byte>(),
             new ArrayList<Byte>(),
+            new ArrayList<Byte>(),
             LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY).length);
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
index 291b7c62b8ed8..d32cde834736e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.Whitebox;
 
+import org.assertj.core.api.Assertions;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -242,6 +243,20 @@ public void testCheckSafeMode9() throws Exception {
     assertTrue(content.contains("Using 3000 as SafeModeMonitor Interval"));
   }
 
+  @Test(timeout = 20000)
+  public void testCheckSafeMode10(){
+    Configuration conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY, -1);
+    GenericTestUtils.LogCapturer logs =
+            GenericTestUtils.LogCapturer.captureLogs(BlockManagerSafeMode.LOG);
+    BlockManagerSafeMode blockManagerSafeMode = new BlockManagerSafeMode(bm,
+            fsn, true, conf);
+    String content = logs.getOutput();
+    Assertions.assertThat(content).contains("Invalid value for " +
+            DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY +
+            ". Should be greater than 0, but is -1");
+  }
+
   /**
    * Test that the block safe increases up to block threshold.
    *
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java
index a5acc14edd935..d1ae0b600fcfa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportLease.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
 import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
@@ -136,6 +137,50 @@ public void testCheckBlockReportLease() throws Exception {
     }
   }
 
+  @Test
+  public void testCheckBlockReportLeaseWhenDnUnregister() throws Exception {
+    HdfsConfiguration conf = new HdfsConfiguration();
+    Random rand = new Random();
+
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+      FSNamesystem fsn = cluster.getNamesystem();
+      BlockManager blockManager = fsn.getBlockManager();
+      String poolId = cluster.getNamesystem().getBlockPoolId();
+      NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
+
+      // Remove the unique DataNode to simulate the unregistered situation.
+      // This is similar to starting NameNode, and DataNodes are not registered yet.
+      DataNode dn = cluster.getDataNodes().get(0);
+      blockManager.getDatanodeManager().getDatanodeMap().remove(dn.getDatanodeUuid());
+
+      // Trigger BlockReport.
+      DatanodeRegistration dnRegistration = dn.getDNRegistrationForBP(poolId);
+      StorageReport[] storages = dn.getFSDataset().getStorageReports(poolId);
+      ExecutorService pool = Executors.newFixedThreadPool(1);
+      BlockReportContext brContext = new BlockReportContext(1, 0,
+          rand.nextLong(), 1);
+      Future<DatanodeCommand> sendBRFuture = pool.submit(() -> {
+        // Build every storage with 100 blocks for sending report.
+        DatanodeStorage[] datanodeStorages
+            = new DatanodeStorage[storages.length];
+        for (int i = 0; i < storages.length; i++) {
+          datanodeStorages[i] = storages[i].getStorage();
+        }
+        StorageBlockReport[] reports = createReports(datanodeStorages, 100);
+
+        // Send blockReport.
+        return rpcServer.blockReport(dnRegistration, poolId, reports,
+            brContext);
+      });
+
+      // When unregistered DataNode triggering the block report, will throw an
+      // UnregisteredNodeException. After NameNode processing, RegisterCommand
+      // is returned to the DataNode.
+      DatanodeCommand datanodeCommand = sendBRFuture.get();
+      assertTrue(datanodeCommand instanceof RegisterCommand);
+    }
+  }
+
   private StorageBlockReport[] createReports(DatanodeStorage[] dnStorages,
       int numBlocks) {
     int longsPerBlock = 3;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java
index d7920a75c1353..4ae0316fa7a22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java
@@ -253,6 +253,8 @@ public void testDatanodeReRegistration() throws Exception {
     }
     dfs.delete(path, false);
     dfs.delete(ecFile, false);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     namesystem.writeLock();
     InvalidateBlocks invalidateBlocks;
     int totalStripedDataBlocks = totalBlockGroups * (ecPolicy.getNumDataUnits()
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeAdminMonitorBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeAdminMonitorBase.java
new file mode 100644
index 0000000000000..451c63be131ea
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeAdminMonitorBase.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+
+public class TestDatanodeAdminMonitorBase {
+  public static final Logger LOG = LoggerFactory.getLogger(TestDatanodeAdminMonitorBase.class);
+
+  // Sort by lastUpdate time descending order, such that unhealthy
+  // nodes are de-prioritized given they cannot be decommissioned.
+  private static final int NUM_DATANODE = 10;
+  private static final int[] UNORDERED_LAST_UPDATE_TIMES =
+      new int[] {0, 5, 2, 11, 0, 3, 1001, 5, 1, 103};
+  private static final int[] ORDERED_LAST_UPDATE_TIMES =
+      new int[] {1001, 103, 11, 5, 5, 3, 2, 1, 0, 0};
+  private static final int[] REVERSE_ORDER_LAST_UPDATE_TIMES =
+      new int[] {0, 0, 1, 2, 3, 5, 5, 11, 103, 1001};
+
+  private static final DatanodeDescriptor[] NODES;
+
+  static {
+    NODES = new DatanodeDescriptor[NUM_DATANODE];
+    for (int i = 0; i < NUM_DATANODE; i++) {
+      NODES[i] = new DatanodeDescriptor(DatanodeID.EMPTY_DATANODE_ID);
+      NODES[i].setLastUpdate(UNORDERED_LAST_UPDATE_TIMES[i]);
+      NODES[i].setLastUpdateMonotonic(UNORDERED_LAST_UPDATE_TIMES[i]);
+    }
+  }
+
+  /**
+   * Verify that DatanodeAdminManager pendingNodes priority queue
+   * correctly orders the nodes by lastUpdate time descending.
+   */
+  @Test
+  public void testPendingNodesQueueOrdering() {
+    final PriorityQueue<DatanodeDescriptor> pendingNodes =
+        new PriorityQueue<>(DatanodeAdminMonitorBase.PENDING_NODES_QUEUE_COMPARATOR);
+
+    pendingNodes.addAll(Arrays.asList(NODES));
+
+    for (int i = 0; i < NUM_DATANODE; i++) {
+      final DatanodeDescriptor dn = pendingNodes.poll();
+      Assert.assertNotNull(dn);
+      Assert.assertEquals(ORDERED_LAST_UPDATE_TIMES[i], dn.getLastUpdate());
+    }
+  }
+
+  /**
+   * Verify that DatanodeAdminManager logic to sort unhealthy nodes
+   * correctly orders the nodes by lastUpdate time ascending.
+   */
+  @Test
+  public void testPendingNodesQueueReverseOrdering() {
+    final List<DatanodeDescriptor> nodes = Arrays.asList(NODES);
+    final List<DatanodeDescriptor> reverseOrderNodes =
+        nodes.stream().sorted(DatanodeAdminMonitorBase.PENDING_NODES_QUEUE_COMPARATOR.reversed())
+            .collect(Collectors.toList());
+
+    Assert.assertEquals(NUM_DATANODE, reverseOrderNodes.size());
+    for (int i = 0; i < NUM_DATANODE; i++) {
+      Assert.assertEquals(REVERSE_ORDER_LAST_UPDATE_TIMES[i],
+          reverseOrderNodes.get(i).getLastUpdate());
+    }
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index 5f5452ac16d59..015a0385a735d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -86,6 +86,10 @@ public class TestDatanodeManager {
   private static DatanodeManager mockDatanodeManager(
       FSNamesystem fsn, Configuration conf) throws IOException {
     BlockManager bm = Mockito.mock(BlockManager.class);
+    Mockito.when(bm.getMaxReplicationStreams()).thenReturn(
+        conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 2));
+    Mockito.when(bm.getReplicationStreamsHardLimit()).thenReturn(
+        conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 2));
     BlockReportLeaseManager blm = new BlockReportLeaseManager(conf);
     Mockito.when(bm.getBlockReportLeaseManager()).thenReturn(blm);
     DatanodeManager dm = new DatanodeManager(bm, fsn, conf);
@@ -138,6 +142,35 @@ public void testNumVersionsCorrectAfterReregister()
         mapToCheck.get("version1").intValue(), 1);
   }
 
+  /**
+   * This test checks that if a node is re-registered with a different ip, its
+   * host2DatanodeMap is correctly updated with the new ip.
+   */
+  @Test
+  public void testHost2NodeMapCorrectAfterReregister()
+          throws IOException, InterruptedException {
+    //Create the DatanodeManager which will be tested
+    FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+    Mockito.when(fsn.hasWriteLock()).thenReturn(true);
+    Configuration conf = new Configuration();
+    DatanodeManager dm = mockDatanodeManager(fsn, conf);
+
+    String storageID = "someStorageID1";
+    String ipOld = "someIPOld" + storageID;
+    String ipNew = "someIPNew" + storageID;
+
+    dm.registerDatanode(new DatanodeRegistration(
+            new DatanodeID(ipOld, "", storageID, 9000, 0, 0, 0),
+            null, null, "version"));
+
+    dm.registerDatanode(new DatanodeRegistration(
+            new DatanodeID(ipNew, "", storageID, 9000, 0, 0, 0),
+            null, null, "version"));
+
+    assertNull("should be no node with old ip", dm.getDatanodeByHost(ipOld));
+    assertNotNull("should be a node with new ip", dm.getDatanodeByHost(ipNew));
+  }
+
   /**
    * This test sends a random sequence of node registrations and node removals
    * to the DatanodeManager (of nodes with different IDs and versions), and
@@ -934,27 +967,37 @@ public void testRemoveIncludedNode() throws IOException {
    * Verify the correctness of pending recovery process.
    *
    * @param numReplicationBlocks the number of replication blocks in the queue.
-   * @param numECBlocks number of EC blocks in the queue.
+   * @param numEcBlocksToBeReplicated the number of EC blocks to be replicated in the queue.
+   * @param numBlocksToBeErasureCoded number of EC blocks to be erasure coded in the queue.
    * @param maxTransfers the maxTransfer value.
-   * @param numReplicationTasks the number of replication tasks polled from
-   *                            the queue.
-   * @param numECTasks the number of EC tasks polled from the queue.
+   * @param maxTransfersHardLimit the maxTransfer hard limit value.
+   * @param numReplicationTasks the number of replication tasks polled from the queue.
+   * @param numECTasksToBeReplicated the number of EC tasks to be replicated polled from the queue.
+   * @param numECTasksToBeErasureCoded the number of EC tasks to be erasure coded polled from
+   *                                   the queue.
+   * @param isDecommissioning if the node is in the decommissioning process.
    *
    * @throws IOException
    */
   private void verifyPendingRecoveryTasks(
-      int numReplicationBlocks, int numECBlocks,
-      int maxTransfers, int numReplicationTasks, int numECTasks)
+      int numReplicationBlocks, int numEcBlocksToBeReplicated, int numBlocksToBeErasureCoded,
+      int maxTransfers, int maxTransfersHardLimit, int numReplicationTasks,
+      int numECTasksToBeReplicated, int numECTasksToBeErasureCoded, boolean isDecommissioning)
       throws IOException {
     FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
     Mockito.when(fsn.hasWriteLock()).thenReturn(true);
     Configuration conf = new Configuration();
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, maxTransfers);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY,
+        maxTransfersHardLimit);
     DatanodeManager dm = Mockito.spy(mockDatanodeManager(fsn, conf));
 
     DatanodeDescriptor nodeInfo = Mockito.mock(DatanodeDescriptor.class);
     Mockito.when(nodeInfo.isRegistered()).thenReturn(true);
     Mockito.when(nodeInfo.getStorageInfos())
         .thenReturn(new DatanodeStorageInfo[0]);
+    Mockito.when(nodeInfo.isDecommissionInProgress())
+        .thenReturn(isDecommissioning);
 
     if (numReplicationBlocks > 0) {
       Mockito.when(nodeInfo.getNumberOfReplicateBlocks())
@@ -968,56 +1011,72 @@ private void verifyPendingRecoveryTasks(
           .thenReturn(tasks);
     }
 
-    if (numECBlocks > 0) {
+    if (numEcBlocksToBeReplicated > 0) {
+      Mockito.when(nodeInfo.getNumberOfECBlocksToBeReplicated())
+              .thenReturn(numEcBlocksToBeReplicated);
+
+      List<BlockTargetPair> ecReplicatedTasks =
+              Collections.nCopies(
+                      Math.min(numECTasksToBeReplicated, numEcBlocksToBeReplicated),
+                      new BlockTargetPair(null, null));
+      Mockito.when(nodeInfo.getECReplicatedCommand(numECTasksToBeReplicated))
+              .thenReturn(ecReplicatedTasks);
+    }
+
+    if (numBlocksToBeErasureCoded > 0) {
       Mockito.when(nodeInfo.getNumberOfBlocksToBeErasureCoded())
-          .thenReturn(numECBlocks);
+          .thenReturn(numBlocksToBeErasureCoded);
 
       List<BlockECReconstructionInfo> tasks =
-          Collections.nCopies(numECTasks, null);
-      Mockito.when(nodeInfo.getErasureCodeCommand(numECTasks))
+          Collections.nCopies(numECTasksToBeErasureCoded, null);
+      Mockito.when(nodeInfo.getErasureCodeCommand(numECTasksToBeErasureCoded))
           .thenReturn(tasks);
     }
 
     DatanodeRegistration dnReg = Mockito.mock(DatanodeRegistration.class);
     Mockito.when(dm.getDatanode(dnReg)).thenReturn(nodeInfo);
     DatanodeCommand[] cmds = dm.handleHeartbeat(
-        dnReg, new StorageReport[1], "bp-123", 0, 0, 10, maxTransfers, 0, null,
+        dnReg, new StorageReport[1], "bp-123", 0, 0, 10, 0, 0, null,
         SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT);
 
     long expectedNumCmds = Arrays.stream(
-        new int[]{numReplicationTasks, numECTasks})
+        new int[]{numReplicationTasks + numECTasksToBeReplicated, numECTasksToBeErasureCoded})
         .filter(x -> x > 0)
         .count();
     assertEquals(expectedNumCmds, cmds.length);
 
     int idx = 0;
-    if (numReplicationTasks > 0) {
+    if (numReplicationTasks > 0 || numECTasksToBeReplicated > 0) {
       assertTrue(cmds[idx] instanceof BlockCommand);
       BlockCommand cmd = (BlockCommand) cmds[0];
-      assertEquals(numReplicationTasks, cmd.getBlocks().length);
-      assertEquals(numReplicationTasks, cmd.getTargets().length);
+      assertEquals(numReplicationTasks + numECTasksToBeReplicated, cmd.getBlocks().length);
+      assertEquals(numReplicationTasks + numECTasksToBeReplicated, cmd.getTargets().length);
       idx++;
     }
 
-    if (numECTasks > 0) {
+    if (numECTasksToBeErasureCoded > 0) {
       assertTrue(cmds[idx] instanceof BlockECReconstructionCommand);
       BlockECReconstructionCommand cmd =
           (BlockECReconstructionCommand) cmds[idx];
-      assertEquals(numECTasks, cmd.getECTasks().size());
+      assertEquals(numECTasksToBeErasureCoded, cmd.getECTasks().size());
     }
 
     Mockito.verify(nodeInfo).getReplicationCommand(numReplicationTasks);
-    Mockito.verify(nodeInfo).getErasureCodeCommand(numECTasks);
+    Mockito.verify(nodeInfo).getECReplicatedCommand(numECTasksToBeReplicated);
+    Mockito.verify(nodeInfo).getErasureCodeCommand(numECTasksToBeErasureCoded);
   }
 
   @Test
   public void testPendingRecoveryTasks() throws IOException {
     // Tasks are slitted according to the ratio between queue lengths.
-    verifyPendingRecoveryTasks(20, 20, 20, 10, 10);
-    verifyPendingRecoveryTasks(40, 10, 20, 16, 4);
+    verifyPendingRecoveryTasks(20, 0, 20, 20, 30, 10, 0, 10, false);
+    verifyPendingRecoveryTasks(40, 0, 10, 20, 30, 16, 0, 4, false);
 
     // Approximately load tasks if the ratio between queue length is large.
-    verifyPendingRecoveryTasks(400, 1, 20, 20, 1);
+    verifyPendingRecoveryTasks(400, 0, 1, 20, 30, 20, 0, 1, false);
+
+    // Tasks use dfs.namenode.replication.max-streams-hard-limit for decommissioning node
+    verifyPendingRecoveryTasks(20, 10, 10, 20, 40, 10, 10, 5, true);
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 57f61b41fef2b..20163cc5fa531 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1018,6 +1018,64 @@ public void testChooseReplicaToDelete() throws Exception {
     assertEquals(chosen, storages[1]);
   }
 
+  /**
+   * Test for the chooseReplicaToDelete are processed based on
+   * EC and STRIPED Policy.
+   */
+  @Test
+  public void testStripedChooseReplicaToDelete() throws Exception {
+    List<DatanodeStorageInfo> replicaList = new ArrayList<>();
+    List<DatanodeStorageInfo> candidate = new ArrayList<>();
+    final Map<String, List<DatanodeStorageInfo>> rackMap
+        = new HashMap<String, List<DatanodeStorageInfo>>();
+
+    replicaList.add(storages[0]);
+    replicaList.add(storages[1]);
+    replicaList.add(storages[2]);
+    replicaList.add(storages[4]);
+
+    candidate.add(storages[0]);
+    candidate.add(storages[2]);
+    candidate.add(storages[4]);
+
+    // Refresh the last update time for all the datanodes
+    for (int i = 0; i < dataNodes.length; i++) {
+      DFSTestUtil.resetLastUpdatesWithOffset(dataNodes[i], 0);
+    }
+
+    List<DatanodeStorageInfo> first = new ArrayList<>();
+    List<DatanodeStorageInfo> second = new ArrayList<>();
+    BlockPlacementPolicy policy = getStriptedPolicy();
+    policy.splitNodesWithRack(replicaList, candidate, rackMap, first,
+        second);
+    // storages[0] is in first set as its rack has two replica nodes,
+    // while storages[2] and dataNodes[4] are in second set.
+    assertEquals(1, first.size());
+    assertEquals(2, second.size());
+    List<StorageType> excessTypes = new ArrayList<>();
+    excessTypes.add(StorageType.DEFAULT);
+    DatanodeStorageInfo chosen = ((BlockPlacementPolicyDefault) policy)
+        .chooseReplicaToDelete(first, second, excessTypes, rackMap);
+    // Within all storages, storages[0] is in the rack that has two replica blocks
+    assertEquals(chosen, storages[0]);
+    policy.adjustSetsWithChosenReplica(rackMap, first, second, chosen);
+    assertEquals(0, first.size());
+    assertEquals(2, second.size());
+
+    // Within second set, storages[2] should be next to be deleted in order.
+    excessTypes.add(StorageType.DEFAULT);
+    chosen = ((BlockPlacementPolicyDefault) policy).chooseReplicaToDelete(
+        first, second, excessTypes, rackMap);
+    assertEquals(chosen, storages[2]);
+    policy.adjustSetsWithChosenReplica(rackMap, first, second, chosen);
+    assertEquals(0, first.size());
+    assertEquals(1, second.size());
+
+    chosen = ((BlockPlacementPolicyDefault) policy).chooseReplicaToDelete(
+        first, second, excessTypes, rackMap);
+    assertEquals(chosen, null);
+  }
+
   private long calculateRemaining(DatanodeDescriptor dataNode) {
     long sum = 0;
     for (DatanodeStorageInfo storageInfo: dataNode.getStorageInfos()){
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java
index f2c24a646b84a..05af3f9bb56cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyExcludeSlowNodes.java
@@ -22,6 +22,8 @@
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
+
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -88,12 +90,18 @@ public void testChooseTargetExcludeSlowNodes() throws Exception {
 
       // mock slow nodes
       SlowPeerTracker tracker = dnManager.getSlowPeerTracker();
-      tracker.addReport(dataNodes[0].getInfoAddr(), dataNodes[3].getInfoAddr());
-      tracker.addReport(dataNodes[0].getInfoAddr(), dataNodes[4].getInfoAddr());
-      tracker.addReport(dataNodes[1].getInfoAddr(), dataNodes[4].getInfoAddr());
-      tracker.addReport(dataNodes[1].getInfoAddr(), dataNodes[5].getInfoAddr());
-      tracker.addReport(dataNodes[2].getInfoAddr(), dataNodes[3].getInfoAddr());
-      tracker.addReport(dataNodes[2].getInfoAddr(), dataNodes[5].getInfoAddr());
+      OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 1.29463);
+      tracker.addReport(dataNodes[0].getInfoAddr(), dataNodes[3].getInfoAddr(), outlierMetrics1);
+      OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 2.9576);
+      tracker.addReport(dataNodes[0].getInfoAddr(), dataNodes[4].getInfoAddr(), outlierMetrics2);
+      OutlierMetrics outlierMetrics3 = new OutlierMetrics(0.0, 0.0, 0.0, 3.59674);
+      tracker.addReport(dataNodes[1].getInfoAddr(), dataNodes[4].getInfoAddr(), outlierMetrics3);
+      OutlierMetrics outlierMetrics4 = new OutlierMetrics(0.0, 0.0, 0.0, 4.238456);
+      tracker.addReport(dataNodes[1].getInfoAddr(), dataNodes[5].getInfoAddr(), outlierMetrics4);
+      OutlierMetrics outlierMetrics5 = new OutlierMetrics(0.0, 0.0, 0.0, 5.18375);
+      tracker.addReport(dataNodes[2].getInfoAddr(), dataNodes[3].getInfoAddr(), outlierMetrics5);
+      OutlierMetrics outlierMetrics6 = new OutlierMetrics(0.0, 0.0, 0.0, 6.39576);
+      tracker.addReport(dataNodes[2].getInfoAddr(), dataNodes[5].getInfoAddr(), outlierMetrics6);
 
       // waiting for slow nodes collector run
       Thread.sleep(3000);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java
index fb2928cc4861e..67fb5a6a48c04 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java
@@ -23,8 +23,7 @@
 import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.server.blockmanagement.SlowPeerTracker
-    .ReportForJson;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.util.FakeTimer;
 import org.junit.Before;
 import org.junit.Rule;
@@ -37,6 +36,7 @@
 import java.util.Set;
 
 import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -45,8 +45,7 @@
  * Tests for {@link SlowPeerTracker}.
  */
 public class TestSlowPeerTracker {
-  public static final Logger LOG = LoggerFactory.getLogger(
-      TestSlowPeerTracker.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestSlowPeerTracker.class);
 
   /**
    * Set a timeout for every test case.
@@ -59,7 +58,7 @@ public class TestSlowPeerTracker {
   private FakeTimer timer;
   private long reportValidityMs;
   private static final ObjectReader READER =
-      new ObjectMapper().readerFor(new TypeReference<Set<ReportForJson>>() {});
+      new ObjectMapper().readerFor(new TypeReference<Set<SlowPeerJsonReport>>() {});
 
   @Before
   public void setup() {
@@ -80,9 +79,9 @@ public void testEmptyReports() {
 
   @Test
   public void testReportsAreRetrieved() {
-    tracker.addReport("node2", "node1");
-    tracker.addReport("node3", "node1");
-    tracker.addReport("node3", "node2");
+    tracker.addReport("node2", "node1", new OutlierMetrics(0.0, 0.0, 0.0, 1.2));
+    tracker.addReport("node3", "node1", new OutlierMetrics(0.0, 0.0, 0.0, 2.1));
+    tracker.addReport("node3", "node2", new OutlierMetrics(0.0, 0.0, 0.0, 1.22));
 
     assertThat(tracker.getReportsForAllDataNodes().size(), is(2));
     assertThat(tracker.getReportsForNode("node2").size(), is(1));
@@ -95,9 +94,9 @@ public void testReportsAreRetrieved() {
    */
   @Test
   public void testAllReportsAreExpired() {
-    tracker.addReport("node2", "node1");
-    tracker.addReport("node3", "node2");
-    tracker.addReport("node1", "node3");
+    tracker.addReport("node2", "node1", new OutlierMetrics(0.0, 0.0, 0.0, 0.123));
+    tracker.addReport("node3", "node2", new OutlierMetrics(0.0, 0.0, 0.0, 0.2334));
+    tracker.addReport("node1", "node3", new OutlierMetrics(0.0, 0.0, 0.0, 1.234));
 
     // No reports should expire after 1ms.
     timer.advance(1);
@@ -117,13 +116,14 @@ public void testAllReportsAreExpired() {
    */
   @Test
   public void testSomeReportsAreExpired() {
-    tracker.addReport("node3", "node1");
-    tracker.addReport("node3", "node2");
+    tracker.addReport("node3", "node1", new OutlierMetrics(0.0, 0.0, 0.0, 1.234));
+    tracker.addReport("node3", "node2", new OutlierMetrics(0.0, 0.0, 0.0, 1.222));
     timer.advance(reportValidityMs);
-    tracker.addReport("node3", "node4");
+    tracker.addReport("node3", "node4", new OutlierMetrics(0.0, 0.0, 0.0, 1.20));
     assertThat(tracker.getReportsForAllDataNodes().size(), is(1));
     assertThat(tracker.getReportsForNode("node3").size(), is(1));
-    assertTrue(tracker.getReportsForNode("node3").contains("node4"));
+    assertEquals(1, tracker.getReportsForNode("node3").stream()
+        .filter(e -> e.getReportingNode().equals("node4")).count());
   }
 
   /**
@@ -131,24 +131,30 @@ public void testSomeReportsAreExpired() {
    */
   @Test
   public void testReplacement() {
-    tracker.addReport("node2", "node1");
+    OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 2.1);
+    tracker.addReport("node2", "node1", outlierMetrics1);
     timer.advance(reportValidityMs); // Expire the report.
     assertThat(tracker.getReportsForAllDataNodes().size(), is(0));
 
     // This should replace the expired report with a newer valid one.
-    tracker.addReport("node2", "node1");
+    OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 0.001);
+    tracker.addReport("node2", "node1", outlierMetrics2);
     assertThat(tracker.getReportsForAllDataNodes().size(), is(1));
     assertThat(tracker.getReportsForNode("node2").size(), is(1));
   }
 
   @Test
   public void testGetJson() throws IOException {
-    tracker.addReport("node1", "node2");
-    tracker.addReport("node2", "node3");
-    tracker.addReport("node2", "node1");
-    tracker.addReport("node4", "node1");
+    OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 1.1);
+    tracker.addReport("node1", "node2", outlierMetrics1);
+    OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 1.23);
+    tracker.addReport("node2", "node3", outlierMetrics2);
+    OutlierMetrics outlierMetrics3 = new OutlierMetrics(0.0, 0.0, 0.0, 2.13);
+    tracker.addReport("node2", "node1", outlierMetrics3);
+    OutlierMetrics outlierMetrics4 = new OutlierMetrics(0.0, 0.0, 0.0, 1.244);
+    tracker.addReport("node4", "node1", outlierMetrics4);
 
-    final Set<ReportForJson> reports = getAndDeserializeJson();
+    final Set<SlowPeerJsonReport> reports = getAndDeserializeJson();
 
     // And ensure its contents are what we expect.
     assertThat(reports.size(), is(3));
@@ -161,19 +167,19 @@ public void testGetJson() throws IOException {
 
   @Test
   public void testGetJsonSizeIsLimited() throws IOException {
-    tracker.addReport("node1", "node2");
-    tracker.addReport("node1", "node3");
-    tracker.addReport("node2", "node3");
-    tracker.addReport("node2", "node4");
-    tracker.addReport("node3", "node4");
-    tracker.addReport("node3", "node5");
-    tracker.addReport("node4", "node6");
-    tracker.addReport("node5", "node6");
-    tracker.addReport("node5", "node7");
-    tracker.addReport("node6", "node7");
-    tracker.addReport("node6", "node8");
-
-    final Set<ReportForJson> reports = getAndDeserializeJson();
+    tracker.addReport("node1", "node2", new OutlierMetrics(0.0, 0.0, 0.0, 1.634));
+    tracker.addReport("node1", "node3", new OutlierMetrics(0.0, 0.0, 0.0, 2.3566));
+    tracker.addReport("node2", "node3", new OutlierMetrics(0.0, 0.0, 0.0, 3.869));
+    tracker.addReport("node2", "node4", new OutlierMetrics(0.0, 0.0, 0.0, 4.1356));
+    tracker.addReport("node3", "node4", new OutlierMetrics(0.0, 0.0, 0.0, 1.73057));
+    tracker.addReport("node3", "node5", new OutlierMetrics(0.0, 0.0, 0.0, 2.4956730));
+    tracker.addReport("node4", "node6", new OutlierMetrics(0.0, 0.0, 0.0, 3.29847));
+    tracker.addReport("node5", "node6", new OutlierMetrics(0.0, 0.0, 0.0, 4.13444));
+    tracker.addReport("node5", "node7", new OutlierMetrics(0.0, 0.0, 0.0, 5.10845));
+    tracker.addReport("node6", "node8", new OutlierMetrics(0.0, 0.0, 0.0, 2.37464));
+    tracker.addReport("node6", "node7", new OutlierMetrics(0.0, 0.0, 0.0, 1.29475656));
+
+    final Set<SlowPeerJsonReport> reports = getAndDeserializeJson();
 
     // Ensure that node4 is not in the list since it was
     // tagged by just one peer and we already have 5 other nodes.
@@ -185,22 +191,49 @@ public void testGetJsonSizeIsLimited() throws IOException {
     assertTrue(isNodeInReports(reports, "node3"));
     assertTrue(isNodeInReports(reports, "node5"));
     assertTrue(isNodeInReports(reports, "node6"));
+
+    assertEquals(1, reports.stream().filter(
+        e -> e.getSlowNode().equals("node1") && e.getSlowPeerLatencyWithReportingNodes().size() == 2
+            && e.getSlowPeerLatencyWithReportingNodes().first().getReportedLatency().equals(1.634)
+            && e.getSlowPeerLatencyWithReportingNodes().last().getReportedLatency().equals(2.3566))
+        .count());
+
+    assertEquals(1, reports.stream().filter(
+        e -> e.getSlowNode().equals("node2") && e.getSlowPeerLatencyWithReportingNodes().size() == 2
+            && e.getSlowPeerLatencyWithReportingNodes().first().getReportedLatency().equals(3.869)
+            && e.getSlowPeerLatencyWithReportingNodes().last().getReportedLatency().equals(4.1356))
+        .count());
+
+    assertEquals(1, reports.stream().filter(
+        e -> e.getSlowNode().equals("node3") && e.getSlowPeerLatencyWithReportingNodes().size() == 2
+            && e.getSlowPeerLatencyWithReportingNodes().first().getReportedLatency().equals(1.73057)
+            && e.getSlowPeerLatencyWithReportingNodes().last().getReportedLatency()
+            .equals(2.4956730)).count());
+
+    assertEquals(1, reports.stream().filter(
+        e -> e.getSlowNode().equals("node6") && e.getSlowPeerLatencyWithReportingNodes().size() == 2
+            && e.getSlowPeerLatencyWithReportingNodes().first().getReportedLatency()
+            .equals(1.29475656) && e.getSlowPeerLatencyWithReportingNodes().last()
+            .getReportedLatency().equals(2.37464)).count());
   }
 
   @Test
   public void testLowRankedElementsIgnored() throws IOException {
     // Insert 5 nodes with 2 peer reports each.
     for (int i = 0; i < 5; ++i) {
-      tracker.addReport("node" + i, "reporter1");
-      tracker.addReport("node" + i, "reporter2");
+      OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 1.295673);
+      tracker.addReport("node" + i, "reporter1", outlierMetrics1);
+      OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 2.38560);
+      tracker.addReport("node" + i, "reporter2", outlierMetrics2);
     }
 
     // Insert 10 nodes with 1 peer report each.
     for (int i = 10; i < 20; ++i) {
-      tracker.addReport("node" + i, "reporter1");
+      OutlierMetrics outlierMetrics = new OutlierMetrics(0.0, 0.0, 0.0, 3.4957);
+      tracker.addReport("node" + i, "reporter1", outlierMetrics);
     }
 
-    final Set<ReportForJson> reports = getAndDeserializeJson();
+    final Set<SlowPeerJsonReport> reports = getAndDeserializeJson();
 
     // Ensure that only the first 5 nodes with two reports each were
     // included in the JSON.
@@ -210,8 +243,8 @@ public void testLowRankedElementsIgnored() throws IOException {
   }
 
   private boolean isNodeInReports(
-      Set<ReportForJson> reports, String node) {
-    for (ReportForJson report : reports) {
+      Set<SlowPeerJsonReport> reports, String node) {
+    for (SlowPeerJsonReport report : reports) {
       if (report.getSlowNode().equalsIgnoreCase(node)) {
         return true;
       }
@@ -219,7 +252,7 @@ private boolean isNodeInReports(
     return false;
   }
 
-  private Set<ReportForJson> getAndDeserializeJson()
+  private Set<SlowPeerJsonReport> getAndDeserializeJson()
       throws IOException {
     final String json = tracker.getJson();
     LOG.info("Got JSON: {}", json);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestHostRestrictingAuthorizationFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestHostRestrictingAuthorizationFilter.java
index 34bc616e540b8..503c4170c4620 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestHostRestrictingAuthorizationFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestHostRestrictingAuthorizationFilter.java
@@ -244,14 +244,13 @@ public void doFilter(ServletRequest servletRequest,
   }
 
   /**
-   * Test acceptable behavior to malformed requests
-   * Case: the request URI does not start with "/webhdfs/v1"
+   * A request that don't access WebHDFS API should pass through.
    */
   @Test
-  public void testInvalidURI() throws Exception {
+  public void testNotWebhdfsAPIRequest() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
     Mockito.when(request.getMethod()).thenReturn("GET");
-    Mockito.when(request.getRequestURI()).thenReturn("/InvalidURI");
+    Mockito.when(request.getRequestURI()).thenReturn("/conf");
     HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
 
     Filter filter = new HostRestrictingAuthorizationFilter();
@@ -260,11 +259,7 @@ public void testInvalidURI() throws Exception {
     FilterConfig fc = new DummyFilterConfig(configs);
 
     filter.init(fc);
-    filter.doFilter(request, response,
-        (servletRequest, servletResponse) -> {});
-    Mockito.verify(response, Mockito.times(1))
-        .sendError(Mockito.eq(HttpServletResponse.SC_NOT_FOUND),
-                   Mockito.anyString());
+    filter.doFilter(request, response, (servletRequest, servletResponse) -> {});
     filter.destroy();
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
index 987b74df47b8b..0f710f9a3c395 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
index 30fee2fddd99b..2b04e2707a034 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
@@ -46,7 +46,7 @@
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 73e8bf7cb618e..e66b62e4e51fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -40,6 +40,9 @@
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.MountVolumeMap;
 import org.apache.hadoop.thirdparty.com.google.common.math.LongMath;
 import org.apache.commons.lang3.ArrayUtils;
@@ -48,7 +51,6 @@
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImplTestUtils;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -162,7 +164,7 @@ public static byte simulatedByte(Block b, long offsetInBlk) {
 
   static final byte[] nullCrcFileData;
 
-  private final AutoCloseableLock datasetLock;
+  private final DataNodeLockManager datasetLockManager;
   private final FileIoProvider fileIoProvider;
 
   static {
@@ -354,6 +356,10 @@ synchronized public void setBytesAcked(long bytesAcked) {
     public void releaseAllBytesReserved() {
     }
 
+    @Override
+    public void releaseReplicaInfoBytesReserved() {
+    }
+
     @Override
     synchronized public long getBytesOnDisk() {
       if (finalized) {
@@ -416,7 +422,6 @@ public void waitForMinLength(long minLength, long time, TimeUnit unit)
       } while (deadLine > System.currentTimeMillis());
       throw new IOException("Minimum length was not achieved within timeout");
     }
-
     @Override
     public FsVolumeSpi getVolume() {
       return getStorage(theBlock).getVolume();
@@ -707,6 +712,8 @@ public SimulatedFSDataset(DataStorage storage, Configuration conf) {
 
   public SimulatedFSDataset(DataNode datanode, DataStorage storage, Configuration conf) {
     this.datanode = datanode;
+    this.datasetLockManager = datanode == null ? new DataSetLockManager() :
+        datanode.getDataSetLockManager();
     int storageCount;
     if (storage != null && storage.getNumStorageDirs() > 0) {
       storageCount = storage.getNumStorageDirs();
@@ -721,9 +728,6 @@ public SimulatedFSDataset(DataNode datanode, DataStorage storage, Configuration
 
     registerMBean(datanodeUuid);
     this.fileIoProvider = new FileIoProvider(conf, datanode);
-
-    this.datasetLock = new AutoCloseableLock();
-
     this.storages = new ArrayList<>();
     for (int i = 0; i < storageCount; i++) {
       this.storages.add(new SimulatedStorage(
@@ -1587,14 +1591,8 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block,
   }
 
   @Override
-  public AutoCloseableLock acquireDatasetLock() {
-    return datasetLock.acquire();
-  }
-
-  @Override
-  public AutoCloseableLock acquireDatasetReadLock() {
-    // No RW lock implementation in simulated dataset currently.
-    return datasetLock.acquire();
+  public DataNodeLockManager<AutoCloseDataSetLock> acquireDatasetLockManager() {
+    return null;
   }
 
   @Override
@@ -1611,5 +1609,10 @@ public Set<? extends Replica> deepCopyReplica(String bpid)
   public MountVolumeMap getMountVolumeMap() {
     return null;
   }
+
+  @Override
+  public List<FsVolumeImpl> getVolumeList() {
+    return null;
+  }
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index fc2a998acb3d8..d300eac4b69cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -30,6 +30,7 @@
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertSame;
@@ -87,6 +88,7 @@
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.Time;
 import org.junit.Before;
+import org.junit.After;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
@@ -127,7 +129,9 @@ public class TestBPOfferService {
   private final int[] heartbeatCounts = new int[3];
   private DataNode mockDn;
   private FsDatasetSpi<?> mockFSDataset;
-  
+  private DataSetLockManager dataSetLockManager = new DataSetLockManager();
+  private boolean isSlownode;
+
   @Before
   public void setupMocks() throws Exception {
     mockNN1 = setupNNMock(0);
@@ -151,6 +155,14 @@ public void setupMocks() throws Exception {
 
     // Wire the dataset to the DN.
     Mockito.doReturn(mockFSDataset).when(mockDn).getFSDataset();
+    Mockito.doReturn(dataSetLockManager).when(mockDn).getDataSetLockManager();
+  }
+
+  @After
+  public void checkDataSetLockManager() {
+    dataSetLockManager.lockLeakCheck();
+    // make sure no lock Leak.
+    assertNull(dataSetLockManager.getLastException());
   }
 
   /**
@@ -216,6 +228,23 @@ public HeartbeatResponse answer(InvocationOnMock invocation)
     }
   }
 
+  private class HeartbeatIsSlownodeAnswer implements Answer<HeartbeatResponse> {
+    private final int nnIdx;
+
+    HeartbeatIsSlownodeAnswer(int nnIdx) {
+      this.nnIdx = nnIdx;
+    }
+
+    @Override
+    public HeartbeatResponse answer(InvocationOnMock invocation)
+        throws Throwable {
+      HeartbeatResponse heartbeatResponse = new HeartbeatResponse(
+          datanodeCommands[nnIdx], mockHaStatuses[nnIdx], null,
+          0, isSlownode);
+
+      return heartbeatResponse;
+    }
+  }
 
   private class HeartbeatRegisterAnswer implements Answer<HeartbeatResponse> {
     private final int nnIdx;
@@ -489,6 +518,7 @@ public void testPickActiveNameNode() throws Exception {
   public void testBPInitErrorHandling() throws Exception {
     final DataNode mockDn = Mockito.mock(DataNode.class);
     Mockito.doReturn(true).when(mockDn).shouldRun();
+    Mockito.doReturn(dataSetLockManager).when(mockDn).getDataSetLockManager();
     Configuration conf = new Configuration();
     File dnDataDir = new File(
       new File(TEST_BUILD_DATA, "testBPInitErrorHandling"), "data");
@@ -1182,6 +1212,44 @@ public Object answer(InvocationOnMock invocation)
     }
   }
 
+  @Test(timeout = 15000)
+  public void testSetIsSlownode() throws Exception {
+    assertEquals(mockDn.isSlownode(), false);
+    Mockito.when(mockNN1.sendHeartbeat(
+            Mockito.any(DatanodeRegistration.class),
+            Mockito.any(StorageReport[].class),
+            Mockito.anyLong(),
+            Mockito.anyLong(),
+            Mockito.anyInt(),
+            Mockito.anyInt(),
+            Mockito.anyInt(),
+            Mockito.any(VolumeFailureSummary.class),
+            Mockito.anyBoolean(),
+            Mockito.any(SlowPeerReports.class),
+            Mockito.any(SlowDiskReports.class)))
+        .thenAnswer(new HeartbeatIsSlownodeAnswer(0));
+
+    BPOfferService bpos = setupBPOSForNNs(mockNN1);
+    bpos.start();
+
+    try {
+      waitForInitialization(bpos);
+
+      bpos.triggerHeartbeatForTests();
+      assertFalse(bpos.isSlownode());
+
+      isSlownode = true;
+      bpos.triggerHeartbeatForTests();
+      assertTrue(bpos.isSlownode());
+
+      isSlownode = false;
+      bpos.triggerHeartbeatForTests();
+      assertFalse(bpos.isSlownode());
+    } finally {
+      bpos.stop();
+    }
+  }
+
   @Test(timeout = 15000)
   public void testCommandProcessingThread() throws Exception {
     Configuration conf = new HdfsConfiguration();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
index 8d2df18711256..9d7c820249c65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery2.java
@@ -46,7 +46,6 @@
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -252,10 +251,8 @@ public void testRaceBetweenReplicaRecoveryAndFinalizeBlock()
           final BlockRecoveryCommand.RecoveringBlock recoveringBlock =
               new BlockRecoveryCommand.RecoveringBlock(block.getBlock(),
                   locations, block.getBlock().getGenerationStamp() + 1);
-          try(AutoCloseableLock lock = dataNode.data.acquireDatasetLock()) {
-            Thread.sleep(2000);
-            dataNode.initReplicaRecovery(recoveringBlock);
-          }
+          Thread.sleep(2000);
+          dataNode.initReplicaRecovery(recoveringBlock);
         } catch (Exception e) {
           LOG.error("Something went wrong.", e);
           recoveryInitResult.set(false);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
index e1473fb7e9803..4251c15718545 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java
@@ -282,11 +282,17 @@ public void testVolumeIteratorWithCaching() throws Exception {
   public void testDisableVolumeScanner() throws Exception {
     Configuration conf = new Configuration();
     disableBlockScanner(conf);
-    TestContext ctx = new TestContext(conf, 1);
-    try {
-      Assert.assertFalse(ctx.datanode.getBlockScanner().isEnabled());
-    } finally {
-      ctx.close();
+    try(TestContext ctx = new TestContext(conf, 1)) {
+      assertFalse(ctx.datanode.getBlockScanner().isEnabled());
+    }
+  }
+
+  @Test(timeout=60000)
+  public void testDisableVolumeScanner2() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setLong(DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND, -1L);
+    try(TestContext ctx = new TestContext(conf, 1)) {
+      assertFalse(ctx.datanode.getBlockScanner().isEnabled());
     }
   }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
index b07e9e461f19b..166ae118f784e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
+import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hdfs.server.datanode.BPServiceActor.Scheduler;
@@ -141,6 +142,23 @@ public void testScheduleNextBlockReport3() {
     }
   }
 
+  /**
+   * force trigger full block report multi times, the next block report
+   * must be scheduled in the range (now + BLOCK_REPORT_INTERVAL_SEC).
+   */
+  @Test
+  public void testScheduleNextBlockReport4() {
+    for (final long now : getTimestamps()) {
+      Scheduler scheduler = makeMockScheduler(now);
+      for (int i = 0; i < getTimestamps().size(); ++i) {
+        scheduler.forceFullBlockReportNow();
+        scheduler.scheduleNextBlockReport();
+      }
+      assertTrue(scheduler.getNextBlockReportTime() - now >= 0);
+      assertTrue(scheduler.getNextBlockReportTime() - now <= BLOCK_REPORT_INTERVAL_MS);
+    }
+  }
+
   @Test
   public void testScheduleHeartbeat() {
     for (final long now : getTimestamps()) {
@@ -187,6 +205,18 @@ public void testScheduleLifeline() {
     }
   }
 
+  @Test
+  public void testScheduleLifelineScheduleTime() {
+    Scheduler mockScheduler = spy(new Scheduler(
+        HEARTBEAT_INTERVAL_MS, LIFELINE_INTERVAL_MS,
+        BLOCK_REPORT_INTERVAL_MS, OUTLIER_REPORT_INTERVAL_MS));
+    long now = Time.monotonicNow();
+    mockScheduler.scheduleNextLifeline(now);
+    long mockMonotonicNow = now + LIFELINE_INTERVAL_MS * 2;
+    doReturn(mockMonotonicNow).when(mockScheduler).monotonicNow();
+    assertTrue(mockScheduler.getLifelineWaitTime() >= 0);
+  }
+
   @Test
   public void testOutlierReportScheduling() {
     for (final long now : getTimestamps()) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
index df8be2220693d..9d6958ea2ab94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
@@ -29,10 +29,6 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.junit.Test;
 
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.*;
-
-
 /**
  * Test to verify that the DataNode Uuid is correctly initialized before
  * FsDataSet initialization.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
index 44f90690e34a4..ef6cf05dcdbda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java
@@ -25,6 +25,9 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerFaultInjector;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@@ -95,6 +98,12 @@ public class TestDataNodeLifeline {
   private FSNamesystem namesystem;
   private DataNode dn;
   private BPServiceActor bpsa;
+  private final BlockManagerFaultInjector injector = new BlockManagerFaultInjector() {
+    @Override
+    public void mockAnException() {
+      throw new UnknownError("Unknown exception");
+    }
+  };
 
   @Before
   public void setup() throws Exception {
@@ -336,4 +345,49 @@ public T answer(InvocationOnMock invocation)
       return result;
     }
   }
+
+  /**
+   * Mock an exception in HeartbeatManager#updateHeartbeat and HeartbeatManager#updateLifeline
+   * respectively, and trigger the heartbeat and lifeline in sequence. The capacityTotal obtained
+   * before and after this operation should be the same.
+   * @throws Exception
+   */
+  @Test
+  public void testHeartbeatAndLifelineOnError() throws Exception {
+    final Configuration config = new HdfsConfiguration();
+    config.set(DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY, "0.0.0.0:0");
+
+    try(MiniDFSCluster cluster =
+            new MiniDFSCluster.Builder(config).numDataNodes(1).build()) {
+      cluster.waitActive();
+      final FSNamesystem fsNamesystem = cluster.getNamesystem();
+
+      // Get capacityTotal before triggering heartbeat and lifeline.
+      DatanodeStatistics datanodeStatistics =
+          fsNamesystem.getBlockManager().getDatanodeManager().getDatanodeStatistics();
+      long capacityTotalBefore = datanodeStatistics.getCapacityTotal();
+
+      // Mock an exception in HeartbeatManager#updateHeartbeat and HeartbeatManager#updateLifeline.
+      BlockManagerFaultInjector.instance = injector;
+      DataNode dataNode = cluster.getDataNodes().get(0);
+      BlockPoolManager blockPoolManager = dataNode.getBlockPoolManager();
+      for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+        if (bpos != null) {
+          for (BPServiceActor actor : bpos.getBPServiceActors()) {
+            try {
+              actor.triggerHeartbeatForTests();
+              actor.sendLifelineForTests();
+            } catch (Throwable e) {
+              assertTrue(e.getMessage().contains("Unknown exception"));
+            }
+          }
+        }
+      }
+
+      // Get capacityTotal after triggering heartbeat and lifeline.
+      long capacityTotalAfter = datanodeStatistics.getCapacityTotal();
+      // The capacityTotal should be same.
+      assertEquals(capacityTotalBefore, capacityTotalAfter);
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
index 73f664b98036e..ea43cccbb1835 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
@@ -75,6 +75,10 @@ public void testDataNodeMXBean() throws Exception {
       // get attribute "Version"
       String version = (String)mbs.getAttribute(mxbeanName, "Version");
       Assert.assertEquals(datanode.getVersion(),version);
+      // get attribute "DNStartedTimeInMillis"
+      long startTime = (long) mbs.getAttribute(mxbeanName, "DNStartedTimeInMillis");
+      Assert.assertTrue("Datanode start time should not be 0", startTime > 0);
+      Assert.assertEquals(datanode.getDNStartedTimeInMillis(), startTime);
       // get attribute "SotfwareVersion"
       String softwareVersion =
           (String)mbs.getAttribute(mxbeanName, "SoftwareVersion");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index e1501fadce938..2bf7861287a2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -25,6 +25,7 @@
 import static org.junit.Assert.*;
 
 import java.io.Closeable;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -38,7 +39,10 @@
 import net.jcip.annotations.NotThreadSafe;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.net.unix.DomainSocket;
+import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.util.Lists;
+import org.junit.Assume;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -599,6 +603,60 @@ public void testNNRpcMetricsWithNonHA() throws IOException {
     MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
     assertCounter("HeartbeatsNumOps", 1L, rb);
   }
+  @Test(timeout = 60000)
+  public void testSlowMetrics() throws Exception {
+    DataNodeFaultInjector dnFaultInjector = new DataNodeFaultInjector() {
+      @Override public void delay() {
+        try {
+          Thread.sleep(310);
+        } catch (InterruptedException e) {
+        }
+      }
+    };
+    DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.get();
+    DataNodeFaultInjector.set(dnFaultInjector);
+
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+      final FileSystem fs = cluster.getFileSystem();
+      List<DataNode> datanodes = cluster.getDataNodes();
+      assertEquals(datanodes.size(), 3);
+      final DataNode datanode = datanodes.get(0);
+      MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
+      final long longFileLen = 10;
+      final long startFlushOrSyncValue =
+          getLongCounter("SlowFlushOrSyncCount", rb);
+      final long startAckToUpstreamValue =
+          getLongCounter("SlowAckToUpstreamCount", rb);
+      final AtomicInteger x = new AtomicInteger(0);
+
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override public Boolean get() {
+          x.getAndIncrement();
+          try {
+            DFSTestUtil
+                .createFile(fs, new Path("/time.txt." + x.get()), longFileLen,
+                    (short) 3, Time.monotonicNow());
+          } catch (IOException ioe) {
+            LOG.error("Caught IOException while ingesting DN metrics", ioe);
+            return false;
+          }
+          MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
+          final long endFlushOrSyncValue = getLongCounter("SlowFlushOrSyncCount", rbNew);
+          final long endAckToUpstreamValue = getLongCounter("SlowAckToUpstreamCount", rbNew);
+          return endFlushOrSyncValue > startFlushOrSyncValue
+              && endAckToUpstreamValue > startAckToUpstreamValue;
+        }
+      }, 30, 30000);
+    } finally {
+      DataNodeFaultInjector.set(oldDnInjector);
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
 
   @Test
   public void testNNRpcMetricsWithHA() throws IOException {
@@ -654,4 +712,38 @@ public void testNNRpcMetricsWithFederationAndHA() throws IOException {
     assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb);
     assertCounter("HeartbeatsNumOps", 4L, rb);
   }
+
+  @Test
+  public void testNodeLocalMetrics() throws Exception {
+    Assume.assumeTrue(null == DomainSocket.getLoadingFailureReason());
+    Configuration conf = new HdfsConfiguration();
+    conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
+    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+    DomainSocket.disableBindPathValidation();
+    conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
+        new File(sockDir.getDir(),
+            "testNodeLocalMetrics._PORT.sock").getAbsolutePath());
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    try {
+      cluster.waitActive();
+      FileSystem fs = cluster.getFileSystem();
+      Path testFile = new Path("/testNodeLocalMetrics.txt");
+      DFSTestUtil.createFile(fs, testFile, 10L, (short)1, 1L);
+      DFSTestUtil.readFile(fs, testFile);
+      List<DataNode> datanodes = cluster.getDataNodes();
+      assertEquals(1, datanodes.size());
+
+      DataNode datanode = datanodes.get(0);
+      MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
+
+      // Write related metrics
+      assertCounter("WritesFromLocalClient", 1L, rb);
+      // Read related metrics
+      assertCounter("ReadsFromLocalClient", 1L, rb);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodePeerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodePeerMetrics.java
index 41fb41f48005a..9ea8a08a7629c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodePeerMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodePeerMetrics.java
@@ -122,10 +122,16 @@ public void testRemoveStaleRecord() throws Exception {
     GenericTestUtils.waitFor(
         () -> rollingAverages.getStats(numSamples).size() > 0, 500, 5000);
     assertEquals(3, rollingAverages.getStats(numSamples).size());
+    String json = peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson();
+    for (String peerAddr : peerAddrList) {
+      assertThat(json, containsString(peerAddr));
+    }
     /* wait for stale report to be removed */
     GenericTestUtils.waitFor(
         () -> rollingAverages.getStats(numSamples).isEmpty(), 500, 10000);
     assertEquals(0, rollingAverages.getStats(numSamples).size());
+    json = peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson();
+    assertEquals("{}", json);
 
     /* dn can report peer metrics normally when it added back to cluster */
     for (String peerAddr : peerAddrList) {
@@ -138,6 +144,10 @@ public void testRemoveStaleRecord() throws Exception {
     GenericTestUtils.waitFor(
         () -> rollingAverages.getStats(numSamples).size() > 0, 500, 10000);
     assertEquals(3, rollingAverages.getStats(numSamples).size());
+    json = peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson();
+    for (String peerAddr : peerAddrList) {
+      assertThat(json, containsString(peerAddr));
+    }
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
index 8cbd38bc601dc..14e3f63691bd5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
@@ -18,24 +18,58 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_JITTER_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CACHEREPORT_INTERVAL_MSEC_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.ReconfigurationException;
+import org.apache.hadoop.fs.CachingGetSpaceUsed;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GetSpaceUsed;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -53,6 +87,7 @@ public class TestDataNodeReconfiguration {
   private final int NUM_NAME_NODE = 1;
   private final int NUM_DATA_NODE = 10;
   private MiniDFSCluster cluster;
+  private static long counter = 0;
 
   @Before
   public void Setup() throws IOException {
@@ -75,6 +110,7 @@ public void tearDown() throws Exception {
   private void startDFSCluster(int numNameNodes, int numDataNodes)
       throws IOException {
     Configuration conf = new Configuration();
+    conf.setBoolean(DFS_DATANODE_PEER_STATS_ENABLED_KEY, true);
 
     MiniDFSNNTopology nnTopology = MiniDFSNNTopology
         .simpleFederatedTopology(numNameNodes);
@@ -293,4 +329,462 @@ private void testAcquireOnMaxConcurrentMoversReconfiguration(
     assertEquals("should not be able to get thread quota", false,
         dataNode.xserver.balanceThrottler.acquire());
   }
+
+  @Test
+  public void testBlockReportIntervalReconfiguration()
+      throws ReconfigurationException {
+    int blockReportInterval = 300 * 1000;
+    String[] blockReportParameters = {
+        DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+        DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY,
+        DFS_BLOCKREPORT_INITIAL_DELAY_KEY};
+
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+      BlockPoolManager blockPoolManager = dn.getBlockPoolManager();
+
+      // Try invalid values.
+      for (String blockReportParameter : blockReportParameters) {
+        try {
+          dn.reconfigureProperty(blockReportParameter, "text");
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting NumberFormatException",
+              expected.getCause() instanceof NumberFormatException);
+        }
+      }
+
+      try {
+        dn.reconfigureProperty(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, String.valueOf(-1));
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting IllegalArgumentException",
+            expected.getCause() instanceof IllegalArgumentException);
+      }
+      try {
+        dn.reconfigureProperty(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY, String.valueOf(-1));
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting IllegalArgumentException",
+            expected.getCause() instanceof IllegalArgumentException);
+      }
+      dn.reconfigureProperty(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, String.valueOf(-1));
+      assertEquals(0, dn.getDnConf().initialBlockReportDelayMs);
+
+      // Change properties and verify the change.
+      dn.reconfigureProperty(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+          String.valueOf(blockReportInterval));
+      for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+        if (bpos != null) {
+          for (BPServiceActor actor : bpos.getBPServiceActors()) {
+            assertEquals(String.format("%s has wrong value",
+                DFS_BLOCKREPORT_INTERVAL_MSEC_KEY),
+                blockReportInterval,
+                actor.getScheduler().getBlockReportIntervalMs());
+          }
+        }
+      }
+
+      dn.reconfigureProperty(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY, String.valueOf(123));
+      assertEquals(123, dn.getDnConf().blockReportSplitThreshold);
+
+      dn.reconfigureProperty(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, "123");
+      assertEquals(123000, dn.getDnConf().initialBlockReportDelayMs);
+
+      // Revert to default and verify default.
+      dn.reconfigureProperty(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, null);
+      for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+        if (bpos != null) {
+          for (BPServiceActor actor : bpos.getBPServiceActors()) {
+            assertEquals(String.format("%s has wrong value",
+                DFS_BLOCKREPORT_INTERVAL_MSEC_KEY),
+                DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT,
+                actor.getScheduler().getBlockReportIntervalMs());
+          }
+        }
+      }
+      assertNull(String.format("expect %s is not configured", DFS_BLOCKREPORT_INTERVAL_MSEC_KEY),
+          dn.getConf().get(DFS_BLOCKREPORT_INTERVAL_MSEC_KEY));
+
+      dn.reconfigureProperty(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY, null);
+      assertNull(String.format("expect %s is not configured", DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY),
+          dn.getConf().get(DFS_BLOCKREPORT_SPLIT_THRESHOLD_KEY));
+
+      dn.reconfigureProperty(DFS_BLOCKREPORT_INITIAL_DELAY_KEY, null);
+      assertNull(String.format("expect %s is not configured", DFS_BLOCKREPORT_INITIAL_DELAY_KEY),
+          dn.getConf().get(DFS_BLOCKREPORT_INITIAL_DELAY_KEY));
+    }
+  }
+
+  @Test
+  public void testDataXceiverReconfiguration()
+      throws ReconfigurationException {
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+
+      // Try invalid values.
+      try {
+        dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, "text");
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting NumberFormatException",
+            expected.getCause() instanceof NumberFormatException);
+      }
+      try {
+        dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, String.valueOf(-1));
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting IllegalArgumentException",
+            expected.getCause() instanceof IllegalArgumentException);
+      }
+
+      // Change properties and verify change.
+      dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, String.valueOf(123));
+      assertEquals(String.format("%s has wrong value", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY),
+          123, dn.getXferServer().getMaxXceiverCount());
+
+      // Revert to default.
+      dn.reconfigureProperty(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY, null);
+      assertEquals(String.format("%s has wrong value", DFS_DATANODE_MAX_RECEIVER_THREADS_KEY),
+          DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT, dn.getXferServer().getMaxXceiverCount());
+
+      assertNull(String.format("expect %s is not configured",
+          DFS_DATANODE_MAX_RECEIVER_THREADS_KEY),
+          dn.getConf().get(DFS_DATANODE_MAX_RECEIVER_THREADS_KEY));
+    }
+  }
+
+  @Test
+  public void testCacheReportReconfiguration()
+      throws ReconfigurationException {
+    int cacheReportInterval = 300 * 1000;
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+
+      // Try invalid values.
+      try {
+        dn.reconfigureProperty(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, "text");
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting NumberFormatException",
+            expected.getCause() instanceof NumberFormatException);
+      }
+      try {
+        dn.reconfigureProperty(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, String.valueOf(-1));
+        fail("ReconfigurationException expected");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting IllegalArgumentException",
+            expected.getCause() instanceof IllegalArgumentException);
+      }
+
+      // Change properties.
+      dn.reconfigureProperty(DFS_CACHEREPORT_INTERVAL_MSEC_KEY,
+          String.valueOf(cacheReportInterval));
+
+      // Verify change.
+      assertEquals(String.format("%s has wrong value", DFS_CACHEREPORT_INTERVAL_MSEC_KEY),
+          cacheReportInterval, dn.getDnConf().getCacheReportInterval());
+
+      // Revert to default.
+      dn.reconfigureProperty(DFS_CACHEREPORT_INTERVAL_MSEC_KEY, null);
+      assertEquals(String.format("%s has wrong value", DFS_CACHEREPORT_INTERVAL_MSEC_KEY),
+          DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT, dn.getDnConf().getCacheReportInterval());
+
+      assertNull(String.format("expect %s is not configured", DFS_CACHEREPORT_INTERVAL_MSEC_KEY),
+          dn.getConf().get(DFS_CACHEREPORT_INTERVAL_MSEC_KEY));
+    }
+  }
+
+  @Test
+  public void testSlowPeerParameters() throws Exception {
+    String[] slowPeersParameters = {
+        DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY,
+        DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY,
+        DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY};
+
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+
+      // Try invalid values.
+      LambdaTestUtils.intercept(ReconfigurationException.class,
+          "Could not change property dfs.datanode.peer.stats.enabled from 'true' to 'text'",
+          () -> dn.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "text"));
+
+      for (String parameter : slowPeersParameters) {
+        try {
+          dn.reconfigureProperty(parameter, "text");
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting NumberFormatException",
+              expected.getCause() instanceof NumberFormatException);
+        }
+
+        try {
+          dn.reconfigureProperty(parameter, String.valueOf(-1));
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting IllegalArgumentException",
+              expected.getCause() instanceof IllegalArgumentException);
+        }
+      }
+
+      // Change and verify properties.
+      dn.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "false");
+      assertFalse(dn.getDnConf().peerStatsEnabled);
+
+      // Reset DFS_DATANODE_PEER_STATS_ENABLED_KEY to true.
+      dn.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "true");
+      for (String parameter : slowPeersParameters) {
+        dn.reconfigureProperty(parameter, "123");
+      }
+      assertEquals(123, dn.getPeerMetrics().getMinOutlierDetectionNodes());
+      assertEquals(123, dn.getPeerMetrics().getLowThresholdMs());
+      assertEquals(123, dn.getPeerMetrics().getMinOutlierDetectionSamples());
+      assertEquals(123,
+          dn.getPeerMetrics().getSlowNodeDetector().getMinOutlierDetectionNodes());
+      assertEquals(123,
+          dn.getPeerMetrics().getSlowNodeDetector().getLowThresholdMs());
+
+      // Revert to default and verify.
+      dn.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY, null);
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_PEER_STATS_ENABLED_KEY), null,
+          dn.getConf().get(DFS_DATANODE_PEER_STATS_ENABLED_KEY));
+
+      // Reset DFS_DATANODE_PEER_STATS_ENABLED_KEY to true.
+      dn.reconfigureProperty(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "true");
+
+      for (String parameter : slowPeersParameters) {
+        dn.reconfigureProperty(parameter, null);
+      }
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY), null,
+          dn.getConf().get(DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_KEY));
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY), null,
+          dn.getConf().get(DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_KEY));
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY), null,
+          dn.getConf().get(DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_KEY));
+      assertEquals(dn.getPeerMetrics().getSlowNodeDetector().getMinOutlierDetectionNodes(),
+          DFS_DATANODE_MIN_OUTLIER_DETECTION_NODES_DEFAULT);
+      assertEquals(dn.getPeerMetrics().getSlowNodeDetector().getLowThresholdMs(),
+          DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT);
+    }
+  }
+
+  @Test
+  public void testSlowDiskParameters() throws ReconfigurationException, IOException {
+    String[] slowDisksParameters1 = {
+        DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
+        DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY};
+
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+
+      // Try invalid values.
+      try {
+        dn.reconfigureProperty(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, "text");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting NumberFormatException",
+            expected.getCause() instanceof NumberFormatException);
+      }
+
+      try {
+        dn.reconfigureProperty(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, "text");
+      } catch (ReconfigurationException expected) {
+        assertTrue("expecting NumberFormatException",
+            expected.getCause() instanceof NumberFormatException);
+      }
+
+      // Enable disk stats, make DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY > 0.
+      dn.reconfigureProperty(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, "1");
+      for (String parameter : slowDisksParameters1) {
+        try {
+          dn.reconfigureProperty(parameter, "text");
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting NumberFormatException",
+              expected.getCause() instanceof NumberFormatException);
+        }
+
+        try {
+          dn.reconfigureProperty(parameter, String.valueOf(-1));
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting IllegalArgumentException",
+              expected.getCause() instanceof IllegalArgumentException);
+        }
+      }
+
+      // Change and verify properties.
+      dn.reconfigureProperty(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, "1ms");
+      assertEquals(1, dn.getDnConf().outliersReportIntervalMs);
+
+      BlockPoolManager blockPoolManager = new BlockPoolManager(dn);
+      blockPoolManager.refreshNamenodes(dn.getConf());
+      for (BPOfferService bpos : blockPoolManager.getAllNamenodeThreads()) {
+        if (bpos != null) {
+          for (BPServiceActor actor : bpos.getBPServiceActors()) {
+            assertEquals(String.format("%s has wrong value",
+                DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY),
+                1, actor.getScheduler().getOutliersReportIntervalMs());
+          }
+        }
+      }
+
+      String[] slowDisksParameters2 = {
+          DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
+          DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
+          DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY};
+      for (String parameter : slowDisksParameters2) {
+        dn.reconfigureProperty(parameter, "99");
+      }
+      // Assert diskMetrics.
+      assertEquals(99, dn.getDiskMetrics().getMinOutlierDetectionDisks());
+      assertEquals(99, dn.getDiskMetrics().getLowThresholdMs());
+      // Assert dnConf.
+      assertTrue(dn.getDnConf().diskStatsEnabled);
+      // Assert profilingEventHook.
+      assertTrue(dn.getFileIoProvider().getProfilingEventHook().getDiskStatsEnabled());
+      assertEquals((int) ((double) 99 / 100 * Integer.MAX_VALUE),
+          dn.getFileIoProvider().getProfilingEventHook().getSampleRangeMax());
+      // Assert slowDiskDetector.
+      assertEquals(99,
+          dn.getDiskMetrics().getSlowDiskDetector().getMinOutlierDetectionNodes());
+      assertEquals(99,
+          dn.getDiskMetrics().getSlowDiskDetector().getLowThresholdMs());
+
+      // Revert to default and verify.
+      dn.reconfigureProperty(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, null);
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY), null,
+          dn.getConf().get(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY));
+
+      dn.reconfigureProperty(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, null);
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY), null,
+          dn.getConf().get(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY));
+      assertFalse(dn.getFileIoProvider().getProfilingEventHook().getDiskStatsEnabled());
+      assertEquals(0,
+          dn.getFileIoProvider().getProfilingEventHook().getSampleRangeMax());
+
+      // Enable disk stats, make DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY > 0.
+      dn.reconfigureProperty(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, "1");
+      dn.reconfigureProperty(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY, null);
+      dn.reconfigureProperty(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY, null);
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY), null,
+          dn.getConf().get(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY));
+      assertEquals(String.format("expect %s is not configured",
+          DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY), null,
+          dn.getConf().get(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY));
+      assertEquals(DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT,
+          dn.getDiskMetrics().getSlowDiskDetector().getMinOutlierDetectionNodes());
+      assertEquals(DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT,
+          dn.getDiskMetrics().getSlowDiskDetector().getLowThresholdMs());
+    }
+  }
+
+  @Test
+  public void testDfsUsageParameters() throws ReconfigurationException {
+    String[] dfsUsageParameters = {
+        FS_DU_INTERVAL_KEY,
+        FS_GETSPACEUSED_JITTER_KEY};
+
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+
+      // Try invalid values.
+      for (String parameter : dfsUsageParameters) {
+        try {
+          dn.reconfigureProperty(parameter, "text");
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting NumberFormatException",
+              expected.getCause() instanceof NumberFormatException);
+        }
+
+        try {
+          dn.reconfigureProperty(parameter, String.valueOf(-1));
+          fail("ReconfigurationException expected");
+        } catch (ReconfigurationException expected) {
+          assertTrue("expecting IllegalArgumentException",
+              expected.getCause() instanceof IllegalArgumentException);
+        }
+      }
+
+      // Change and verify properties.
+      for (String parameter : dfsUsageParameters) {
+        dn.reconfigureProperty(parameter, "99");
+      }
+      List<FsVolumeImpl> volumeList = dn.data.getVolumeList();
+      for (FsVolumeImpl fsVolume : volumeList) {
+        Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
+        for (Map.Entry<String, BlockPoolSlice> entry : blockPoolSlices.entrySet()) {
+          GetSpaceUsed dfsUsage = entry.getValue().getDfsUsage();
+          if (dfsUsage instanceof CachingGetSpaceUsed) {
+            assertEquals(99,
+                ((CachingGetSpaceUsed) entry.getValue().getDfsUsage()).getRefreshInterval());
+            assertEquals(99,
+                ((CachingGetSpaceUsed) entry.getValue().getDfsUsage()).getJitter());
+          }
+        }
+      }
+
+      // Revert to default and verify.
+      for (String parameter : dfsUsageParameters) {
+        dn.reconfigureProperty(parameter, null);
+      }
+      for (FsVolumeImpl fsVolume : volumeList) {
+        Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
+        for (Map.Entry<String, BlockPoolSlice> entry : blockPoolSlices.entrySet()) {
+          GetSpaceUsed dfsUsage = entry.getValue().getDfsUsage();
+          if (dfsUsage instanceof CachingGetSpaceUsed) {
+            assertEquals(String.format("expect %s is not configured",
+                FS_DU_INTERVAL_KEY), FS_DU_INTERVAL_DEFAULT,
+                ((CachingGetSpaceUsed) entry.getValue().getDfsUsage()).getRefreshInterval());
+            assertEquals(String.format("expect %s is not configured",
+                FS_GETSPACEUSED_JITTER_KEY), FS_GETSPACEUSED_JITTER_DEFAULT,
+                ((CachingGetSpaceUsed) entry.getValue().getDfsUsage()).getJitter());
+          }
+          assertEquals(String.format("expect %s is not configured",
+              FS_DU_INTERVAL_KEY), null,
+              dn.getConf().get(FS_DU_INTERVAL_KEY));
+          assertEquals(String.format("expect %s is not configured",
+              FS_GETSPACEUSED_JITTER_KEY), null,
+              dn.getConf().get(FS_GETSPACEUSED_JITTER_KEY));
+        }
+      }
+    }
+  }
+
+  public static class DummyCachingGetSpaceUsed extends CachingGetSpaceUsed {
+    public DummyCachingGetSpaceUsed(Builder builder) throws IOException {
+      super(builder.setInterval(1000).setJitter(0L));
+    }
+
+    @Override
+    protected void refresh() {
+      counter++;
+    }
+  }
+
+  @Test
+  public void testDfsUsageKlass() throws ReconfigurationException, InterruptedException {
+
+    long lastCounter = counter;
+    Thread.sleep(5000);
+    assertEquals(lastCounter, counter);
+
+    for (int i = 0; i < NUM_DATA_NODE; i++) {
+      DataNode dn = cluster.getDataNodes().get(i);
+      dn.reconfigurePropertyImpl(FS_GETSPACEUSED_CLASSNAME,
+              DummyCachingGetSpaceUsed.class.getName());
+    }
+
+    lastCounter = counter;
+    Thread.sleep(5000);
+    assertTrue(counter > lastCounter);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
index 85054be630bdd..7a62f8da6a1cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java
@@ -151,7 +151,7 @@ private void verifyDataNodeVolumeMetrics(final FileSystem fs,
     LOG.info("MetadataOperationSampleCount : " +
         metrics.getMetadataOperationSampleCount());
     LOG.info("MetadataOperationMean : " + metrics.getMetadataOperationMean());
-    LOG.info("MetadataFileIoStdDev : " +
+    LOG.info("MetadataOperationStdDev : " +
         metrics.getMetadataOperationStdDev());
 
     LOG.info("DataFileIoSampleCount : " + metrics.getDataFileIoSampleCount());
@@ -166,7 +166,7 @@ private void verifyDataNodeVolumeMetrics(final FileSystem fs,
     LOG.info("syncIoMean : " + metrics.getSyncIoMean());
     LOG.info("syncIoStdDev : " + metrics.getSyncIoStdDev());
 
-    LOG.info("readIoSampleCount : " + metrics.getReadIoMean());
+    LOG.info("readIoSampleCount : " + metrics.getReadIoSampleCount());
     LOG.info("readIoMean : " + metrics.getReadIoMean());
     LOG.info("readIoStdDev : " + metrics.getReadIoStdDev());
 
@@ -174,6 +174,15 @@ private void verifyDataNodeVolumeMetrics(final FileSystem fs,
     LOG.info("writeIoMean : " + metrics.getWriteIoMean());
     LOG.info("writeIoStdDev : " + metrics.getWriteIoStdDev());
 
+    LOG.info("transferIoSampleCount : " + metrics.getTransferIoSampleCount());
+    LOG.info("transferIoMean : " + metrics.getTransferIoMean());
+    LOG.info("transferIoStdDev : " + metrics.getTransferIoStdDev());
+
+    LOG.info("nativeCopyIoSampleCount : " +
+        metrics.getNativeCopyIoSampleCount());
+    LOG.info("nativeCopyIoMean : " + metrics.getNativeCopyIoMean());
+    LOG.info("nativeCopyIoStdDev : " + metrics.getNativeCopyIoStdDev());
+
     LOG.info("fileIoErrorSampleCount : "
         + metrics.getFileIoErrorSampleCount());
     LOG.info("fileIoErrorMean : " + metrics.getFileIoErrorMean());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
new file mode 100644
index 0000000000000..b514accdf16e9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestDataSetLockManager {
+  private DataSetLockManager manager;
+
+  @Before
+  public void init() {
+    manager = new DataSetLockManager();
+  }
+
+  @Test(timeout = 5000)
+  public void testBaseFunc() {
+    manager.addLock(LockLevel.BLOCK_POOl, "BPtest");
+    manager.addLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+
+    AutoCloseDataSetLock lock = manager.writeLock(LockLevel.BLOCK_POOl, "BPtest");
+    AutoCloseDataSetLock lock1 = manager.readLock(LockLevel.BLOCK_POOl, "BPtest");
+    lock1.close();
+    lock.close();
+
+    manager.lockLeakCheck();
+    assertNull(manager.getLastException());
+
+    AutoCloseDataSetLock lock2 = manager.writeLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    AutoCloseDataSetLock lock3 = manager.readLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    lock3.close();
+    lock2.close();
+
+    manager.lockLeakCheck();
+    assertNull(manager.getLastException());
+
+    AutoCloseDataSetLock lock4 = manager.writeLock(LockLevel.BLOCK_POOl, "BPtest");
+    AutoCloseDataSetLock lock5 = manager.readLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    lock5.close();
+    lock4.close();
+
+    manager.lockLeakCheck();
+    assertNull(manager.getLastException());
+
+    manager.writeLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    manager.lockLeakCheck();
+
+    Exception lastException = manager.getLastException();
+    assertEquals(lastException.getMessage(), "lock Leak");
+  }
+
+  @Test(timeout = 5000)
+  public void testAcquireWriteLockError() throws InterruptedException {
+    Thread t = new Thread(() -> {
+      manager.readLock(LockLevel.BLOCK_POOl, "test");
+      manager.writeLock(LockLevel.BLOCK_POOl, "test");
+    });
+    t.start();
+    Thread.sleep(1000);
+    manager.lockLeakCheck();
+    Exception lastException = manager.getLastException();
+    assertEquals(lastException.getMessage(), "lock Leak");
+  }
+
+  @Test(timeout = 5000)
+  public void testLockLeakCheck() {
+    manager.writeLock(LockLevel.BLOCK_POOl, "test");
+    manager.lockLeakCheck();
+    Exception lastException = manager.getLastException();
+    assertEquals(lastException.getMessage(), "lock Leak");
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java
index 7c1c27b4dd4ed..772452decc0c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeRegister.java
@@ -76,7 +76,7 @@ public void setUp() throws IOException {
     actor = new BPServiceActor("test", "test", INVALID_ADDR, null, mockBPOS);
 
     fakeNsInfo = mock(NamespaceInfo.class);
-    // Return a a good software version.
+    // Return a good software version.
     doReturn(VersionInfo.getVersion()).when(fakeNsInfo).getSoftwareVersion();
     // Return a good layout version for now.
     doReturn(HdfsServerConstants.NAMENODE_LAYOUT_VERSION).when(fakeNsInfo)
@@ -144,7 +144,7 @@ public void testDNShutdwonBeforeRegister() throws Exception {
     DataNode dn = new DataNode(conf, locations, null, null);
     BPOfferService bpos = new BPOfferService("test_ns",
         Lists.newArrayList("nn0"), Lists.newArrayList(nnADDR),
-        Collections.<InetSocketAddress>nCopies(1, null), dn);
+        Collections.nCopies(1, null), dn);
     DatanodeProtocolClientSideTranslatorPB fakeDnProt =
         mock(DatanodeProtocolClientSideTranslatorPB.class);
     when(fakeDnProt.versionRequest()).thenReturn(fakeNsInfo);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index 7f79778842780..74c70cec76967 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -49,6 +49,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSClient;
@@ -58,9 +59,11 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ReportCompiler;
 import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
@@ -71,6 +74,7 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.AutoCloseableLock;
@@ -130,7 +134,8 @@ private List<LocatedBlock> createFile(String fileNamePrefix, long fileLen,
 
   /** Truncate a block file. */
   private long truncateBlockFile() throws IOException {
-    try (AutoCloseableLock lock = fds.acquireDatasetLock()) {
+    try (AutoCloseableLock lock = fds.acquireDatasetLockManager().writeLock(
+        LockLevel.BLOCK_POOl, bpid)) {
       for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) {
         File f = new File(b.getBlockURI());
         File mf = new File(b.getMetadataURI());
@@ -155,7 +160,8 @@ private long truncateBlockFile() throws IOException {
 
   /** Delete a block file */
   private long deleteBlockFile() {
-    try (AutoCloseableLock lock = fds.acquireDatasetLock()) {
+    try (AutoCloseableLock lock = fds.acquireDatasetLockManager().
+        writeLock(LockLevel.BLOCK_POOl, bpid)) {
       for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) {
         File f = new File(b.getBlockURI());
         File mf = new File(b.getMetadataURI());
@@ -171,7 +177,8 @@ private long deleteBlockFile() {
 
   /** Delete block meta file */
   private long deleteMetaFile() {
-    try (AutoCloseableLock lock = fds.acquireDatasetLock()) {
+    try (AutoCloseableLock lock = fds.acquireDatasetLockManager().
+        writeLock(LockLevel.BLOCK_POOl, bpid)) {
       for (ReplicaInfo b : FsDatasetTestUtil.getReplicas(fds, bpid)) {
         // Delete a metadata file
         if (b.metadataExists() && b.deleteMetadata()) {
@@ -190,7 +197,8 @@ private long deleteMetaFile() {
    * @throws IOException
    */
   private void duplicateBlock(long blockId) throws IOException {
-    try (AutoCloseableLock lock = fds.acquireDatasetLock()) {
+    try (AutoCloseableLock lock = fds.acquireDatasetLockManager().
+        writeLock(LockLevel.BLOCK_POOl, bpid)) {
       ReplicaInfo b = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId);
       try (FsDatasetSpi.FsVolumeReferences volumes =
           fds.getFsVolumeReferences()) {
@@ -507,6 +515,53 @@ public void testDeleteBlockOnTransientStorage() throws Exception {
     }
   }
 
+  @Test(timeout = 600000)
+  public void testRegularBlock() throws Exception {
+    Configuration conf = getConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf).build();
+    try {
+      cluster.waitActive();
+      bpid = cluster.getNamesystem().getBlockPoolId();
+      fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0));
+      client = cluster.getFileSystem().getClient();
+      conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, 1);
+      // log trace
+      GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.
+          captureLogs(NameNode.stateChangeLog);
+      // Add files with 5 blocks
+      createFile(GenericTestUtils.getMethodName(), BLOCK_LENGTH * 5, false);
+
+      List<ReplicaInfo> infos = new ArrayList<>(FsDatasetTestUtil.getReplicas(fds, bpid));
+      ReplicaInfo lastReplica = infos.get(infos.size() - 1);
+      ReplicaInfo penultimateReplica = infos.get(infos.size() - 2);
+
+      String blockParent = new File(lastReplica.getBlockURI().getPath()).getParent();
+      File lastBlockFile = new File(blockParent, getBlockFile(lastReplica.getBlockId()));
+      File penultimateBlockFile = new File(blockParent,
+          getBlockFile(penultimateReplica.getBlockId()));
+      FileUtil.symLink(lastBlockFile.toString(), penultimateBlockFile.toString());
+      ExtendedBlock block = new ExtendedBlock(bpid, penultimateReplica.getBlockId());
+
+      scanner = new DirectoryScanner(fds, conf);
+      scanner.setRetainDiffs(true);
+      scanner.reconcile();
+      DirectoryScanner.Stats stats = scanner.stats.get(bpid);
+      assertNotNull(stats);
+      assertEquals(1, stats.mismatchBlocks);
+
+      // check nn log
+      String msg = "*DIR* reportBadBlocks for block: " + bpid + ":" +
+          getBlockFile(block.getBlockId());
+      assertTrue(logCapturer.getOutput().contains(msg));
+    } finally {
+      if (scanner != null) {
+        scanner.shutdown();
+        scanner = null;
+      }
+      cluster.shutdown();
+    }
+  }
+
   @Test(timeout = 600000)
   public void testDirectoryScanner() throws Exception {
     // Run the test with and without parallel scanning
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java
index 60d4cca059ac9..1f18b7bde0f4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestRefreshNamenodes.java
@@ -23,6 +23,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
@@ -73,13 +74,13 @@ public void testRefreshNamenodes() throws IOException {
 
       // Ensure a BPOfferService in the datanodes corresponds to
       // a namenode in the cluster
-      Set<InetSocketAddress> nnAddrsFromCluster = Sets.newHashSet();
+      Set<InetSocketAddress> nnAddrsFromCluster = new HashSet<>();
       for (int i = 0; i < 4; i++) {
         assertTrue(nnAddrsFromCluster.add(
             cluster.getNameNode(i).getNameNodeAddress()));
       }
       
-      Set<InetSocketAddress> nnAddrsFromDN = Sets.newHashSet();
+      Set<InetSocketAddress> nnAddrsFromDN = new HashSet<>();
       for (BPOfferService bpos : dn.getAllBpOs()) {
         for (BPServiceActor bpsa : bpos.getBPServiceActors()) {
           assertTrue(nnAddrsFromDN.add(bpsa.getNNSocketAddress()));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index 3fbd4de721260..77e2e2077d1f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -23,8 +23,10 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.server.common.AutoCloseDataSetLock;
+import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.MountVolumeMap;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
 import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
@@ -452,12 +454,7 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block,
   }
 
   @Override
-  public AutoCloseableLock acquireDatasetLock() {
-    return null;
-  }
-
-  @Override
-  public AutoCloseableLock acquireDatasetReadLock() {
+  public DataNodeLockManager<AutoCloseDataSetLock> acquireDatasetLockManager() {
     return null;
   }
 
@@ -471,4 +468,9 @@ public Set<? extends Replica> deepCopyReplica(String bpid)
   public MountVolumeMap getMountVolumeMap() {
     return null;
   }
+
+  @Override
+  public List<FsVolumeImpl> getVolumeList() {
+    return null;
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
index 084caf038c3db..460d1c1eb7d4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java
@@ -45,6 +45,10 @@ public long getBytesAcked() {
   public void setBytesAcked(long bytesAcked) {
   }
 
+  @Override
+  public void releaseReplicaInfoBytesReserved() {
+  }
+
   @Override
   public void releaseAllBytesReserved() {
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
index 6c8e828f3689e..81cfa3e08b87b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 
 public class ExternalVolumeImpl implements FsVolumeSpi {
+  private final String defaultStroageId = "test";
   @Override
   public FsVolumeReference obtainReference() throws ClosedChannelException {
     return null;
@@ -54,7 +55,7 @@ public long getAvailable() throws IOException {
 
   @Override
   public String getStorageID() {
-    return null;
+    return defaultStroageId;
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
index c4280bc97b4ea..7e8497e1ebdbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.io.FileExistsException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -434,7 +434,7 @@ public void changeStoredGenerationStamp(
   @Override
   public Iterator<Replica> getStoredReplicas(String bpid) throws IOException {
     // Reload replicas from the disk.
-    ReplicaMap replicaMap = new ReplicaMap(dataset.datasetRWLock);
+    ReplicaMap replicaMap = new ReplicaMap(dataset.acquireDatasetLockManager());
     try (FsVolumeReferences refs = dataset.getFsVolumeReferences()) {
       for (FsVolumeSpi vol : refs) {
         FsVolumeImpl volume = (FsVolumeImpl) vol;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index 9774fea1c9a50..eaaa7b0c89227 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -46,7 +46,7 @@
 import java.util.concurrent.TimeoutException;
 
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestAddBlockPoolException.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestAddBlockPoolException.java
index 79529eab42fac..c01dbc3eecd0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestAddBlockPoolException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestAddBlockPoolException.java
@@ -34,7 +34,7 @@
 public class TestAddBlockPoolException {
 
   @Test
-  public void testHasExeceptionsReturnsCorrectValue() {
+  public void testHasExceptionsReturnsCorrectValue() {
     AddBlockPoolException e = new AddBlockPoolException();
     assertFalse(e.hasExceptions());
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestCacheByPmemMappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestCacheByPmemMappableBlockLoader.java
index 9575028944997..baea8f5de2bbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestCacheByPmemMappableBlockLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestCacheByPmemMappableBlockLoader.java
@@ -66,6 +66,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY;
+import static org.junit.Assume.assumeTrue;
 
 /**
  * Tests HDFS persistent memory cache by PmemMappableBlockLoader.
@@ -104,6 +105,8 @@ public class TestCacheByPmemMappableBlockLoader {
 
   @BeforeClass
   public static void setUpClass() throws Exception {
+    assumeTrue("Requires PMDK", NativeIO.POSIX.isPmdkAvailable());
+
     oldInjector = DataNodeFaultInjector.get();
     DataNodeFaultInjector.set(new DataNodeFaultInjector() {
       @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestDatanodeRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestDatanodeRestart.java
index 0afee5fa57f27..ac9587e54f7ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestDatanodeRestart.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestDatanodeRestart.java
@@ -18,12 +18,8 @@
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.util.Collection;
-import java.util.Iterator;
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
@@ -40,7 +36,6 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
-import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 1ae7582a3279f..9f2425d9374fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -21,11 +21,18 @@
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 
 import org.apache.hadoop.fs.DF;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
+import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner;
 import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
 
@@ -62,20 +69,22 @@
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.io.MultipleIOException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.FakeTimer;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -93,7 +102,6 @@
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DN_CACHED_DFSUSED_CHECK_INTERVAL_MS;
@@ -139,6 +147,7 @@ public class TestFsDatasetImpl {
   private DataNode datanode;
   private DataStorage storage;
   private FsDatasetImpl dataset;
+  private DataSetLockManager manager = new DataSetLockManager();
   
   private final static String BLOCKPOOL = "BP-TEST";
 
@@ -212,6 +221,7 @@ public void setUp() throws IOException {
     this.conf.set(DFSConfigKeys.DFS_DATANODE_REPLICA_CACHE_ROOT_DIR_KEY,
         replicaCacheRootDir);
 
+    when(datanode.getDataSetLockManager()).thenReturn(manager);
     when(datanode.getConf()).thenReturn(conf);
     final DNConf dnConf = new DNConf(datanode);
     when(datanode.getDnConf()).thenReturn(dnConf);
@@ -220,6 +230,8 @@ public void setUp() throws IOException {
     final ShortCircuitRegistry shortCircuitRegistry =
         new ShortCircuitRegistry(conf);
     when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry);
+    DataNodeMetrics dataNodeMetrics = DataNodeMetrics.create(conf, "mockName");
+    when(datanode.getMetrics()).thenReturn(dataNodeMetrics);
 
     createStorageDirs(storage, conf, NUM_INIT_VOLUMES);
     dataset = new FsDatasetImpl(datanode, storage, conf);
@@ -231,116 +243,11 @@ public void setUp() throws IOException {
     assertEquals(0, dataset.getNumFailedVolumes());
   }
 
-  @Test(timeout=10000)
-  public void testReadLockEnabledByDefault()
-      throws Exception {
-    final FsDatasetSpi ds = dataset;
-    AtomicBoolean accessed = new AtomicBoolean(false);
-    CountDownLatch latch = new CountDownLatch(1);
-    CountDownLatch waiterLatch = new CountDownLatch(1);
-
-    Thread holder = new Thread() {
-      public void run() {
-        try (AutoCloseableLock l = ds.acquireDatasetReadLock()) {
-          latch.countDown();
-          // wait for the waiter thread to access the lock.
-          waiterLatch.await();
-        } catch (Exception e) {
-        }
-      }
-    };
-
-    Thread waiter = new Thread() {
-      public void run() {
-        try {
-          latch.await();
-        } catch (InterruptedException e) {
-          waiterLatch.countDown();
-          return;
-        }
-        try (AutoCloseableLock l = ds.acquireDatasetReadLock()) {
-          accessed.getAndSet(true);
-          // signal the holder thread.
-          waiterLatch.countDown();
-        } catch (Exception e) {
-        }
-      }
-    };
-    waiter.start();
-    holder.start();
-    holder.join();
-    waiter.join();
-    // The holder thread is still holding the lock, but the waiter can still
-    // run as the lock is a shared read lock.
-    // Otherwise test will timeout with deadlock.
-    assertEquals(true, accessed.get());
-    holder.interrupt();
-  }
-
-  @Test(timeout=20000)
-  public void testReadLockCanBeDisabledByConfig()
-      throws Exception {
-    HdfsConfiguration conf = new HdfsConfiguration();
-    conf.setBoolean(
-        DFSConfigKeys.DFS_DATANODE_LOCK_READ_WRITE_ENABLED_KEY, false);
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
-        .numDataNodes(1).build();
-    try {
-      AtomicBoolean accessed = new AtomicBoolean(false);
-      cluster.waitActive();
-      DataNode dn = cluster.getDataNodes().get(0);
-      final FsDatasetSpi<?> ds = DataNodeTestUtils.getFSDataset(dn);
-
-      CountDownLatch latch = new CountDownLatch(1);
-      CountDownLatch waiterLatch = new CountDownLatch(1);
-      Thread holder = new Thread() {
-        public void run() {
-          try (AutoCloseableLock l = ds.acquireDatasetReadLock()) {
-            latch.countDown();
-            // wait for the waiter thread to access the lock.
-            waiterLatch.await();
-          } catch (Exception e) {
-          }
-        }
-      };
-
-      Thread waiter = new Thread() {
-        public void run() {
-          try {
-            // Wait for holder to get ds read lock.
-            latch.await();
-          } catch (InterruptedException e) {
-            waiterLatch.countDown();
-            return;
-          }
-          try (AutoCloseableLock l = ds.acquireDatasetReadLock()) {
-            accessed.getAndSet(true);
-            // signal the holder thread.
-            waiterLatch.countDown();
-          } catch (Exception e) {
-          }
-        }
-      };
-      waiter.start();
-      holder.start();
-      // Wait for sometime to make sure we are in deadlock,
-      try {
-        GenericTestUtils.waitFor(() ->
-                accessed.get(),
-            100, 10000);
-        fail("Waiter thread should not execute.");
-      } catch (TimeoutException e) {
-      }
-      // Release waiterLatch to exit deadlock.
-      waiterLatch.countDown();
-      holder.join();
-      waiter.join();
-      // After releasing waiterLatch water
-      // thread will be able to execute.
-      assertTrue(accessed.get());
-    } finally {
-      cluster.shutdown();
-    }
+  @After
+  public void checkDataSetLockManager() {
+    manager.lockLeakCheck();
+    // make sure no lock Leak.
+    assertNull(manager.getLastException());
   }
 
   @Test
@@ -412,6 +319,7 @@ public void testAddVolumeWithSameDiskTiering() throws IOException {
     final ShortCircuitRegistry shortCircuitRegistry =
         new ShortCircuitRegistry(conf);
     when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry);
+    when(datanode.getDataSetLockManager()).thenReturn(manager);
 
     createStorageDirs(storage, conf, 1);
     dataset = new FsDatasetImpl(datanode, storage, conf);
@@ -479,6 +387,8 @@ public void testAddVolumeWithCustomizedCapacityRatio()
     final ShortCircuitRegistry shortCircuitRegistry =
         new ShortCircuitRegistry(conf);
     when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry);
+    when(datanode.getDataSetLockManager()).thenReturn(manager);
+
 
     createStorageDirs(storage, conf, 0);
 
@@ -710,6 +620,53 @@ public void run() {}
         + "volumeMap.", 0, totalNumReplicas);
   }
 
+  @Test(timeout = 30000)
+  public void testConcurrentWriteAndDeleteBlock() throws Exception {
+    // Feed FsDataset with block metadata.
+    final int numBlocks = 1000;
+    final int threadCount = 10;
+    // Generate data blocks.
+    ExecutorService pool = Executors.newFixedThreadPool(threadCount);
+    List<Future<?>> futureList = new ArrayList<>();
+    Random random = new Random();
+    // Random write block and delete half of them.
+    for (int i = 0; i < threadCount; i++) {
+      Thread thread = new Thread() {
+        @Override
+        public void run() {
+          try {
+            String bpid = BLOCK_POOL_IDS[random.nextInt(BLOCK_POOL_IDS.length)];
+            for (int blockId = 0; blockId < numBlocks; blockId++) {
+              ExtendedBlock eb = new ExtendedBlock(bpid, blockId);
+              ReplicaHandler replica = null;
+              try {
+                replica = dataset.createRbw(StorageType.DEFAULT, null, eb,
+                    false);
+                if (blockId % 2 > 0) {
+                  dataset.invalidate(bpid, new Block[]{eb.getLocalBlock()});
+                }
+              } finally {
+                if (replica != null) {
+                  replica.close();
+                }
+              }
+            }
+            // Just keep final consistency no need to care exception.
+          } catch (Exception ignore) {}
+        }
+      };
+      thread.setName("AddBlock" + i);
+      futureList.add(pool.submit(thread));
+    }
+    // Wait for data generation
+    for (Future<?> f : futureList) {
+      f.get();
+    }
+    for (String bpid : dataset.volumeMap.getBlockPoolList()) {
+      assertEquals(numBlocks / 2, dataset.volumeMap.size(bpid));
+    }
+  }
+
   @Test(timeout = 5000)
   public void testRemoveNewlyAddedVolume() throws IOException {
     final int numExistingVolumes = getNumVolumes();
@@ -744,6 +701,7 @@ public void testAddVolumeFailureReleasesInUseLock() throws IOException {
     FsDatasetImpl spyDataset = spy(dataset);
     FsVolumeImpl mockVolume = mock(FsVolumeImpl.class);
     File badDir = new File(BASE_DIR, "bad");
+    when(mockVolume.getStorageID()).thenReturn("test");
     badDir.mkdirs();
     doReturn(mockVolume).when(spyDataset)
         .createFsVolume(anyString(), any(StorageDirectory.class),
@@ -1830,4 +1788,138 @@ public void testReleaseVolumeRefIfExceptionThrown() throws IOException {
       cluster.shutdown();
     }
   }
+
+  @Test(timeout = 30000)
+  public void testTransferAndNativeCopyMetrics() throws IOException {
+    Configuration config = new HdfsConfiguration();
+    config.setInt(
+        DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY,
+        100);
+    config.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
+        "60,300,1500");
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+        .numDataNodes(1)
+        .storageTypes(new StorageType[]{StorageType.DISK, StorageType.DISK})
+        .storagesPerDatanode(2)
+        .build()) {
+      FileSystem fs = cluster.getFileSystem();
+      DataNode dataNode = cluster.getDataNodes().get(0);
+
+      // Create file that has one block with one replica.
+      Path filePath = new Path(name.getMethodName());
+      DFSTestUtil.createFile(fs, filePath, 100, (short) 1, 0);
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
+
+      // Copy a new replica to other volume.
+      FsDatasetImpl fsDataSetImpl = (FsDatasetImpl) dataNode.getFSDataset();
+      ReplicaInfo newReplicaInfo = createNewReplicaObj(block, fsDataSetImpl);
+      fsDataSetImpl.finalizeNewReplica(newReplicaInfo, block);
+
+      // Get the volume where the original replica resides.
+      FsVolumeSpi volume = null;
+      for (FsVolumeSpi fsVolumeReference :
+          fsDataSetImpl.getFsVolumeReferences()) {
+        if (!fsVolumeReference.getStorageID()
+            .equals(newReplicaInfo.getStorageUuid())) {
+          volume = fsVolumeReference;
+        }
+      }
+
+      // Assert metrics.
+      DataNodeVolumeMetrics metrics = volume.getMetrics();
+      assertEquals(2, metrics.getTransferIoSampleCount());
+      assertEquals(3, metrics.getTransferIoQuantiles().length);
+      assertEquals(2, metrics.getNativeCopyIoSampleCount());
+      assertEquals(3, metrics.getNativeCopyIoQuantiles().length);
+    }
+  }
+
+  /**
+   * The block should be in the replicaMap if the async deletion task is pending.
+   */
+  @Test
+  public void testAysncDiskServiceDeleteReplica()
+      throws IOException, InterruptedException, TimeoutException {
+    HdfsConfiguration config = new HdfsConfiguration();
+    // Bump up replication interval.
+    config.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 10);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).numDataNodes(3).build();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    String bpid = cluster.getNamesystem().getBlockPoolId();
+    DataNodeFaultInjector oldInjector = DataNodeFaultInjector.get();
+    final Semaphore semaphore = new Semaphore(0);
+    try {
+      cluster.waitActive();
+      final DataNodeFaultInjector injector = new DataNodeFaultInjector() {
+        @Override
+        public void delayDeleteReplica() {
+          // Lets wait for the remove replica process.
+          try {
+            semaphore.acquire(1);
+          } catch (InterruptedException e) {
+            // ignore.
+          }
+        }
+      };
+      DataNodeFaultInjector.set(injector);
+
+      // Create file.
+      Path path = new Path("/testfile");
+      DFSTestUtil.createFile(fs, path, 1024, (short) 3, 0);
+      DFSTestUtil.waitReplication(fs, path, (short) 3);
+      LocatedBlock lb = DFSTestUtil.getAllBlocks(fs, path).get(0);
+      ExtendedBlock extendedBlock = lb.getBlock();
+      DatanodeInfo[] loc = lb.getLocations();
+      assertEquals(3, loc.length);
+
+      // DN side.
+      DataNode dn = cluster.getDataNode(loc[0].getIpcPort());
+      final FsDatasetImpl ds = (FsDatasetImpl) DataNodeTestUtils.getFSDataset(dn);
+      List<Block> blockList = Lists.newArrayList(extendedBlock.getLocalBlock());
+      assertNotNull(ds.getStoredBlock(bpid, extendedBlock.getBlockId()));
+      ds.invalidate(bpid, blockList.toArray(new Block[0]));
+
+      // Test get blocks and datanodes.
+      loc = DFSTestUtil.getAllBlocks(fs, path).get(0).getLocations();
+      assertEquals(3, loc.length);
+      List<String> uuids = Lists.newArrayList();
+      for (DatanodeInfo datanodeInfo : loc) {
+        uuids.add(datanodeInfo.getDatanodeUuid());
+      }
+      assertTrue(uuids.contains(dn.getDatanodeUuid()));
+
+      // Do verification that the first replication shouldn't be deleted from the memory first.
+      // Because the namenode still contains this replica, so client will try to read it.
+      // If this replica is deleted from memory, the client would got an ReplicaNotFoundException.
+      assertNotNull(ds.getStoredBlock(bpid, extendedBlock.getBlockId()));
+
+      // Make it resume the removeReplicaFromMem method.
+      semaphore.release(1);
+
+      // Waiting for the async deletion task finish.
+      GenericTestUtils.waitFor(() ->
+          ds.asyncDiskService.countPendingDeletions() == 0, 100, 1000);
+
+      // Sleep for two heartbeat times.
+      Thread.sleep(config.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+          DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+          TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2);
+
+      // Test get blocks and datanodes again.
+      loc = DFSTestUtil.getAllBlocks(fs, path).get(0).getLocations();
+      assertEquals(2, loc.length);
+      uuids = Lists.newArrayList();
+      for (DatanodeInfo datanodeInfo : loc) {
+        uuids.add(datanodeInfo.getDatanodeUuid());
+      }
+      // The namenode does not contain this replica.
+      assertFalse(uuids.contains(dn.getDatanodeUuid()));
+
+      // This replica has deleted from datanode memory.
+      assertNull(ds.getStoredBlock(bpid, extendedBlock.getBlockId()));
+    } finally {
+      cluster.shutdown();
+      DataNodeFaultInjector.set(oldInjector);
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 15495dfd59fc7..1b638e28fa7de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -21,6 +21,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.DF;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
@@ -32,12 +33,16 @@
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy;
+import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.util.StringUtils;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -52,7 +57,7 @@
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY;
 import static org.junit.Assert.assertEquals;
@@ -73,6 +78,10 @@ public class TestFsVolumeList {
   private FsDatasetImpl dataset = null;
   private String baseDir;
   private BlockScanner blockScanner;
+  private final static int NUM_DATANODES = 3;
+  private final static int STORAGES_PER_DATANODE = 3;
+  private final static int DEFAULT_BLOCK_SIZE = 102400;
+  private final static int BUFFER_LENGTH = 1024;
 
   @Before
   public void setUp() {
@@ -89,7 +98,7 @@ public void setUp() {
   public void testGetNextVolumeWithClosedVolume() throws IOException {
     FsVolumeList volumeList = new FsVolumeList(
         Collections.<VolumeFailureInfo>emptyList(),
-        blockScanner, blockChooser, conf);
+        blockScanner, blockChooser, conf, null);
     final List<FsVolumeImpl> volumes = new ArrayList<>();
     for (int i = 0; i < 3; i++) {
       File curDir = new File(baseDir, "nextvolume-" + i);
@@ -132,7 +141,7 @@ public Boolean get() {
   @Test(timeout=30000)
   public void testReleaseVolumeRefIfNoBlockScanner() throws IOException {
     FsVolumeList volumeList = new FsVolumeList(
-        Collections.<VolumeFailureInfo>emptyList(), null, blockChooser, conf);
+        Collections.<VolumeFailureInfo>emptyList(), null, blockChooser, conf, null);
     File volDir = new File(baseDir, "volume-0");
     volDir.mkdirs();
     FsVolumeImpl volume = new FsVolumeImplBuilder()
@@ -397,7 +406,7 @@ public void run() {
     fs.close();
     FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0)
         .getFSDataset();
-    ReplicaMap volumeMap = new ReplicaMap(new ReentrantReadWriteLock());
+    ReplicaMap volumeMap = new ReplicaMap(fsDataset.acquireDatasetLockManager());
     RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker
         .getInstance(conf, fsDataset);
     FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0);
@@ -511,7 +520,7 @@ public void testGetVolumeWithSameDiskArchival() throws Exception {
         .build();
     FsVolumeList volumeList = new FsVolumeList(
         Collections.<VolumeFailureInfo>emptyList(),
-        blockScanner, blockChooser, conf);
+        blockScanner, blockChooser, conf, null);
     volumeList.addVolume(archivalVolume.obtainReference());
     volumeList.addVolume(diskVolume.obtainReference());
 
@@ -620,4 +629,99 @@ public void testDfsUsageStatWithSameDiskArchival() throws Exception {
     mountVolumeMap.removeVolume(spyArchivalVolume);
     assertEquals(dfCapacity - duReserved, spyDiskVolume.getCapacity());
   }
+
+  @Test
+  public void testExcludeSlowDiskWhenChoosingVolume() throws Exception {
+    conf = new HdfsConfiguration();
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+    // Set datanode outliers report interval to 1s.
+    conf.setStrings(DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, "1s");
+    // Enable datanode disk metrics collector.
+    conf.setInt(DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_PERCENTAGE_KEY, 30);
+    // Enable excluding slow disks when choosing volume.
+    conf.setInt(DFSConfigKeys.DFS_DATANODE_MAX_SLOWDISKS_TO_EXCLUDE_KEY, 1);
+    // Ensure that each volume capacity is larger than the DEFAULT_BLOCK_SIZE.
+    long capacity = 10 * DEFAULT_BLOCK_SIZE;
+    long[][] capacities = new long[NUM_DATANODES][STORAGES_PER_DATANODE];
+    String[] hostnames = new String[NUM_DATANODES];
+    for (int i = 0; i < NUM_DATANODES; i++) {
+      hostnames[i] = i + "." + i + "." + i + "." + i;
+      for(int j = 0; j < STORAGES_PER_DATANODE; j++){
+        capacities[i][j]=capacity;
+      }
+    }
+
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+        .hosts(hostnames)
+        .numDataNodes(NUM_DATANODES)
+        .storagesPerDatanode(STORAGES_PER_DATANODE)
+        .storageCapacities(capacities).build();
+    cluster.waitActive();
+    FileSystem fs = cluster.getFileSystem();
+
+    // Create file for each datanode.
+    ArrayList<DataNode> dataNodes = cluster.getDataNodes();
+    DataNode dn0 = dataNodes.get(0);
+    DataNode dn1 = dataNodes.get(1);
+    DataNode dn2 = dataNodes.get(2);
+
+    // Mock the first disk of each datanode is a slowest disk.
+    String slowDisk0OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(0)
+        .getVolume().getBaseURI().getPath();
+    String slowDisk0OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(0)
+        .getVolume().getBaseURI().getPath();
+    String slowDisk0OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(0)
+        .getVolume().getBaseURI().getPath();
+
+    String slowDisk1OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(1)
+        .getVolume().getBaseURI().getPath();
+    String slowDisk1OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(1)
+        .getVolume().getBaseURI().getPath();
+    String slowDisk1OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(1)
+        .getVolume().getBaseURI().getPath();
+
+    dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn0, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+        SlowDiskReports.DiskOp.METADATA, 2.0));
+    dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn1, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+        SlowDiskReports.DiskOp.METADATA, 2.0));
+    dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn2, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+        SlowDiskReports.DiskOp.METADATA, 2.0));
+
+    dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn0, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+        SlowDiskReports.DiskOp.METADATA, 1.0));
+    dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn1, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+        SlowDiskReports.DiskOp.METADATA, 1.0));
+    dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn2, ImmutableMap.of(
+        SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+        SlowDiskReports.DiskOp.METADATA, 1.0));
+
+    // Wait until the data on the slow disk is collected successfully.
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override public Boolean get() {
+        return dn0.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
+            dn1.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
+            dn2.getDiskMetrics().getSlowDisksToExclude().size() == 1;
+      }
+    }, 1000, 5000);
+
+    // Create a file with 3 replica.
+    DFSTestUtil.createFile(fs, new Path("/file0"), false, BUFFER_LENGTH, 1000,
+        DEFAULT_BLOCK_SIZE, (short) 3, 0, false, null);
+
+    // Asserts that the number of blocks created on a slow disk is 0.
+    Assert.assertEquals(0, dn0.getVolumeReport().stream()
+        .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn0)).collect(Collectors.toList()).get(0)
+        .getNumBlocks());
+    Assert.assertEquals(0, dn1.getVolumeReport().stream()
+        .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn1)).collect(Collectors.toList()).get(0)
+        .getNumBlocks());
+    Assert.assertEquals(0, dn2.getVolumeReport().stream()
+        .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn2)).collect(Collectors.toList()).get(0)
+        .getNumBlocks());
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
index b72b1cd1bb8bd..659d53eda9b6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
@@ -25,7 +25,6 @@
 import java.net.InetSocketAddress;
 import java.net.SocketTimeoutException;
 import java.util.List;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -47,10 +46,12 @@
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.Replica;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaUnderRecovery;
+import org.apache.hadoop.hdfs.server.datanode.extdataset.ExternalVolumeImpl;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
 import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
@@ -218,7 +219,7 @@ private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception {
   }
 
   private static ReplicaInfo createReplicaInfo(Block b) {
-    return new FinalizedReplica(b, null, null);
+    return new FinalizedReplica(b, new ExternalVolumeImpl(), null);
   }
 
   private static void assertEquals(ReplicaInfo originalInfo, ReplicaRecoveryInfo recoveryInfo) {
@@ -236,7 +237,8 @@ public void testInitReplicaRecovery() throws IOException {
     final long firstblockid = 10000L;
     final long gs = 7777L;
     final long length = 22L;
-    final ReplicaMap map = new ReplicaMap(new ReentrantReadWriteLock());
+    DataSetLockManager manager = new DataSetLockManager();
+    final ReplicaMap map = new ReplicaMap(manager);
     String bpid = "BP-TEST";
     final Block[] blocks = new Block[5];
     for(int i = 0; i < blocks.length; i++) {
@@ -252,7 +254,7 @@ public void testInitReplicaRecovery() throws IOException {
       final long recoveryid = gs + 1;
       final ReplicaRecoveryInfo recoveryInfo = FsDatasetImpl
           .initReplicaRecovery(bpid, map, blocks[0], recoveryid,
-              DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+              DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
       assertEquals(originalInfo, recoveryInfo);
 
       final ReplicaUnderRecovery updatedInfo = (ReplicaUnderRecovery)map.get(bpid, b);
@@ -263,7 +265,7 @@ public void testInitReplicaRecovery() throws IOException {
       final long recoveryid2 = gs + 2;
       final ReplicaRecoveryInfo recoveryInfo2 = FsDatasetImpl
           .initReplicaRecovery(bpid, map, blocks[0], recoveryid2,
-              DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+              DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
       assertEquals(originalInfo, recoveryInfo2);
 
       final ReplicaUnderRecovery updatedInfo2 = (ReplicaUnderRecovery)map.get(bpid, b);
@@ -273,7 +275,7 @@ public void testInitReplicaRecovery() throws IOException {
       //case RecoveryInProgressException
       try {
         FsDatasetImpl.initReplicaRecovery(bpid, map, b, recoveryid,
-            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
         Assert.fail();
       }
       catch(RecoveryInProgressException ripe) {
@@ -286,7 +288,7 @@ public void testInitReplicaRecovery() throws IOException {
       final Block b = new Block(firstblockid - 1, length, gs);
       ReplicaRecoveryInfo r = FsDatasetImpl.initReplicaRecovery(bpid, map, b,
           recoveryid,
-          DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+          DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
       Assert.assertNull("Data-node should not have this replica.", r);
     }
     
@@ -295,7 +297,7 @@ public void testInitReplicaRecovery() throws IOException {
       final Block b = new Block(firstblockid + 1, length, gs);
       try {
         FsDatasetImpl.initReplicaRecovery(bpid, map, b, recoveryid,
-            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
         Assert.fail();
       }
       catch(IOException ioe) {
@@ -309,7 +311,7 @@ public void testInitReplicaRecovery() throws IOException {
       final Block b = new Block(firstblockid, length, gs+1);
       try {
         FsDatasetImpl.initReplicaRecovery(bpid, map, b, recoveryid,
-            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
+            DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT, manager);
         fail("InitReplicaRecovery should fail because replica's " +
         		"gs is less than the block's gs");
       } catch (IOException e) {
@@ -317,6 +319,10 @@ public void testInitReplicaRecovery() throws IOException {
            "replica.getGenerationStamp() < block.getGenerationStamp(), block=");
       }
     }
+
+    manager.lockLeakCheck();
+    // make sure no lock Leak.
+    assertNull(manager.getLastException());
   }
 
   /** 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
index 2d54c480461c5..699854cd1759c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSOutputStream;
 import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -175,6 +176,8 @@ public void testWritePipelineFailure()
 
     // Delete the file and ensure locked RAM goes to zero.
     fs.delete(path, false);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
     waitForLockedBytesUsed(fsd, 0);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestPmemCacheRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestPmemCacheRecovery.java
index d3232c8362949..6ce420adeb197 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestPmemCacheRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestPmemCacheRecovery.java
@@ -27,6 +27,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
 import java.io.IOException;
@@ -103,6 +104,8 @@ public class TestPmemCacheRecovery {
 
   @BeforeClass
   public static void setUpClass() throws Exception {
+    assumeTrue("Requires PMDK", NativeIO.POSIX.isPmdkAvailable());
+
     oldInjector = DataNodeFaultInjector.get();
     DataNodeFaultInjector.set(new DataNodeFaultInjector() {
       @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
index afd816864a1d7..f8d66c2f2cccf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
@@ -44,7 +44,6 @@
 import java.util.Optional;
 import java.util.Random;
 import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -66,6 +65,7 @@
 import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
 import org.apache.hadoop.hdfs.server.datanode.DNConf;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner;
 import org.apache.hadoop.hdfs.server.datanode.DirectoryScanner.ScanInfoVolumeReport;
@@ -109,6 +109,7 @@ public class TestProvidedImpl {
   private DataNode datanode;
   private DataStorage storage;
   private FsDatasetImpl dataset;
+  private DataSetLockManager manager = new DataSetLockManager();
   private static Map<Long, String> blkToPathMap;
   private static List<FsVolumeImpl> providedVolumes;
   private static long spaceUsed = 0;
@@ -319,6 +320,7 @@ public void setUp() throws IOException {
     conf.setLong(DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, 0);
 
     when(datanode.getConf()).thenReturn(conf);
+    when(datanode.getDataSetLockManager()).thenReturn(manager);
     final DNConf dnConf = new DNConf(datanode);
     when(datanode.getDnConf()).thenReturn(dnConf);
     // reset the space used
@@ -400,7 +402,7 @@ public void testProvidedVolumeImpl() throws IOException {
   public void testBlockLoad() throws IOException {
     for (int i = 0; i < providedVolumes.size(); i++) {
       FsVolumeImpl vol = providedVolumes.get(i);
-      ReplicaMap volumeMap = new ReplicaMap(new ReentrantReadWriteLock());
+      ReplicaMap volumeMap = new ReplicaMap();
       vol.getVolumeMap(volumeMap, null);
 
       assertEquals(vol.getBlockPoolList().length, BLOCK_POOL_IDS.length);
@@ -476,7 +478,7 @@ private int getBlocksInProvidedVolumes(String basePath, int numBlocks,
       vol.setFileRegionProvider(BLOCK_POOL_IDS[CHOSEN_BP_ID],
           new TestFileRegionBlockAliasMap(fileRegionIterator, minBlockId,
               numBlocks));
-      ReplicaMap volumeMap = new ReplicaMap(new ReentrantReadWriteLock());
+      ReplicaMap volumeMap = new ReplicaMap();
       vol.getVolumeMap(BLOCK_POOL_IDS[CHOSEN_BP_ID], volumeMap, null);
       totalBlocks += volumeMap.size(BLOCK_POOL_IDS[CHOSEN_BP_ID]);
     }
@@ -586,7 +588,7 @@ public void testProvidedReplicaSuffixExtraction() {
   public void testProvidedReplicaPrefix() throws Exception {
     for (int i = 0; i < providedVolumes.size(); i++) {
       FsVolumeImpl vol = providedVolumes.get(i);
-      ReplicaMap volumeMap = new ReplicaMap(new ReentrantReadWriteLock());
+      ReplicaMap volumeMap = new ReplicaMap();
       vol.getVolumeMap(volumeMap, null);
 
       Path expectedPrefix = new Path(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java
index 59203bb7d3468..3058598bb6c6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReplicaMap.java
@@ -26,13 +26,12 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * Unit test for ReplicasMap class
  */
 public class TestReplicaMap {
-  private final ReplicaMap map = new ReplicaMap(new ReentrantReadWriteLock());
+  private final ReplicaMap map = new ReplicaMap();
   private final String bpid = "BP-TEST";
   private final  Block block = new Block(1234, 1234, 1234);
   
@@ -112,7 +111,7 @@ public void testRemove() {
 
   @Test
   public void testMergeAll() {
-    ReplicaMap temReplicaMap = new ReplicaMap(new ReentrantReadWriteLock());
+    ReplicaMap temReplicaMap = new ReplicaMap();
     Block tmpBlock = new Block(5678, 5678, 5678);
     temReplicaMap.add(bpid, new FinalizedReplica(tmpBlock, null, null));
 
@@ -123,7 +122,7 @@ public void testMergeAll() {
 
   @Test
   public void testAddAll() {
-    ReplicaMap temReplicaMap = new ReplicaMap(new ReentrantReadWriteLock());
+    ReplicaMap temReplicaMap = new ReplicaMap();
     Block tmpBlock = new Block(5678, 5678, 5678);
     temReplicaMap.add(bpid, new FinalizedReplica(tmpBlock, null, null));
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
index efc203b73000b..f5e5be65607a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.ChecksumException;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.ClientContext;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
index a702cec7cb0ca..de4d236617374 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestSpaceReservation.java
@@ -18,8 +18,14 @@
 
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
+import java.util.Collection;
+import java.util.EnumSet;
 import java.util.function.Supplier;
 
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -746,4 +752,48 @@ public Boolean get() {
     }, 500, 30000);
     checkReservedSpace(0);
   }
+
+  /**
+   * Ensure that bytes reserved of ReplicaInfo gets cleared
+   * during finalize.
+   *
+   * @throws IOException
+   */
+  @Test(timeout = 300000)
+  public void testReplicaInfoBytesReservedReleasedOnFinalize() throws IOException {
+    short replication = 3;
+    int bufferLength = 4096;
+    startCluster(BLOCK_SIZE, replication, -1);
+
+    String methodName = GenericTestUtils.getMethodName();
+    Path path = new Path("/" + methodName + ".01.dat");
+
+    FSDataOutputStream fos =
+        fs.create(path, FsPermission.getFileDefault(), EnumSet.of(CreateFlag.CREATE), bufferLength,
+            replication, BLOCK_SIZE, null);
+    // Allocate a block.
+    fos.write(new byte[bufferLength]);
+    fos.hsync();
+
+    DataNode dataNode = cluster.getDataNodes().get(0);
+    FsDatasetImpl fsDataSetImpl = (FsDatasetImpl) dataNode.getFSDataset();
+    long expectedReservedSpace = BLOCK_SIZE - bufferLength;
+
+    String bpid = cluster.getNamesystem().getBlockPoolId();
+    Collection<ReplicaInfo> replicas = FsDatasetTestUtil.getReplicas(fsDataSetImpl, bpid);
+    ReplicaInfo r = replicas.iterator().next();
+
+    // Verify Initial Bytes Reserved for Replica and Volume are correct
+    assertEquals(fsDataSetImpl.getVolumeList().get(0).getReservedForReplicas(),
+        expectedReservedSpace);
+    assertEquals(r.getBytesReserved(), expectedReservedSpace);
+
+    // Verify Bytes Reserved for Replica and Volume are correct after finalize
+    fsDataSetImpl.finalizeNewReplica(r, new ExtendedBlock(bpid, r));
+
+    assertEquals(fsDataSetImpl.getVolumeList().get(0).getReservedForReplicas(), 0L);
+    assertEquals(r.getBytesReserved(), 0L);
+
+    fos.close();
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index e939389624130..202fb190f3d64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -27,7 +27,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageType;
@@ -246,8 +245,10 @@ private void testAppend(String bpid, FsDatasetSpi<?> dataSet,
       Assert.fail("Should not have appended to a non-existent replica " + 
           blocks[NON_EXISTENT]);
     } catch (ReplicaNotFoundException e) {
-      Assert.assertEquals(ReplicaNotFoundException.NON_EXISTENT_REPLICA + 
-          blocks[NON_EXISTENT], e.getMessage());
+      String expectMessage = ReplicaNotFoundException.NON_EXISTENT_REPLICA
+          + blocks[NON_EXISTENT].getBlockPoolId() + ":"
+          + blocks[NON_EXISTENT].getBlockId();
+      Assert.assertEquals(expectMessage, e.getMessage());
     }
     
     newGS = blocks[FINALIZED].getGenerationStamp()+1;
@@ -550,7 +551,7 @@ public  void testReplicaMapAfterDatanodeRestart() throws Exception {
           bpList.size() == 2);
       
       createReplicas(bpList, volumes, cluster.getFsDatasetTestUtils(dn));
-      ReplicaMap oldReplicaMap = new ReplicaMap(new ReentrantReadWriteLock());
+      ReplicaMap oldReplicaMap = new ReplicaMap();
       oldReplicaMap.addAll(dataSet.volumeMap);
 
       cluster.restartDataNode(0);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
index 1faddb362ecd1..0042bcb042646 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/metrics/TestDataNodeOutlierDetectionViaMetrics.java
@@ -21,6 +21,7 @@
 import java.util.function.Supplier;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.metrics2.lib.MetricsTestHelper;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Before;
@@ -100,7 +101,7 @@ public Boolean get() {
       }
     }, 500, 100_000);
 
-    final Map<String, Double> outliers = peerMetrics.getOutliers();
+    final Map<String, OutlierMetrics> outliers = peerMetrics.getOutliers();
     LOG.info("Got back outlier nodes: {}", outliers);
     assertThat(outliers.size(), is(1));
     assertTrue(outliers.containsKey(slowNodeName));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java
index 62827a2e5d3bc..b6ad5a9a5d6b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java
@@ -57,8 +57,7 @@ public void testDataNodeXFrameOptionsEnabled() throws Exception {
     cluster = createCluster(xFrameEnabled, null);
     HttpURLConnection conn = getConn(cluster);
     String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS");
-    Assert.assertTrue("X-FRAME-OPTIONS is absent in the header",
-        xfoHeader != null);
+    Assert.assertNotNull("X-FRAME-OPTIONS is absent in the header", xfoHeader);
     Assert.assertTrue(xfoHeader.endsWith(HttpServer2.XFrameOption
         .SAMEORIGIN.toString()));
   }
@@ -69,7 +68,7 @@ public void testNameNodeXFrameOptionsDisabled() throws Exception {
     cluster = createCluster(xFrameEnabled, null);
     HttpURLConnection conn = getConn(cluster);
     String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS");
-    Assert.assertTrue("unexpected X-FRAME-OPTION in header", xfoHeader == null);
+    Assert.assertNull("unexpected X-FRAME-OPTION in header", xfoHeader);
   }
 
   @Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestHostRestrictingAuthorizationFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestHostRestrictingAuthorizationFilterHandler.java
index 031ac0aa29d35..a918a081e7f6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestHostRestrictingAuthorizationFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestHostRestrictingAuthorizationFilterHandler.java
@@ -48,7 +48,7 @@ public class TestHostRestrictingAuthorizationFilterHandler {
    * Test running in with no ACL rules (restrict all)
    */
   @Test
-  public void testRejectAll() throws Exception {
+  public void testRejectAll() {
     EmbeddedChannel channel = new CustomEmbeddedChannel("127.0.0.1", 1006,
         new HostRestrictingAuthorizationFilterHandler());
     FullHttpRequest httpRequest =
@@ -61,7 +61,7 @@ public void testRejectAll() throws Exception {
     DefaultHttpResponse channelResponse =
         (DefaultHttpResponse) channel.outboundMessages().poll();
     assertNotNull("Expected response to exist.", channelResponse);
-    assertEquals(HttpResponseStatus.FORBIDDEN, channelResponse.getStatus());
+    assertEquals(HttpResponseStatus.FORBIDDEN, channelResponse.status());
     assertFalse(channel.isOpen());
   }
 
@@ -70,7 +70,7 @@ public void testRejectAll() throws Exception {
    * reused
    */
   @Test
-  public void testMultipleAcceptedGETsOneChannel() throws Exception {
+  public void testMultipleAcceptedGETsOneChannel() {
     Configuration conf = new Configuration();
     conf.set(CONFNAME, "*,*,/allowed");
     HostRestrictingAuthorizationFilter filter =
@@ -102,7 +102,7 @@ public void testMultipleAcceptedGETsOneChannel() throws Exception {
    * single filter instance
    */
   @Test
-  public void testMultipleChannels() throws Exception {
+  public void testMultipleChannels() {
     Configuration conf = new Configuration();
     conf.set(CONFNAME, "*,*,/allowed");
     HostRestrictingAuthorizationFilter filter =
@@ -140,7 +140,7 @@ public void testMultipleChannels() throws Exception {
    * Test accepting a GET request for the file checksum
    */
   @Test
-  public void testAcceptGETFILECHECKSUM() throws Exception {
+  public void testAcceptGETFILECHECKSUM() {
     EmbeddedChannel channel = new CustomEmbeddedChannel("127.0.0.1", 1006,
         new HostRestrictingAuthorizationFilterHandler());
     FullHttpRequest httpRequest =
@@ -158,7 +158,7 @@ public void testAcceptGETFILECHECKSUM() throws Exception {
    */
   protected static class CustomEmbeddedChannel extends EmbeddedChannel {
 
-    private InetSocketAddress socketAddress;
+    private final InetSocketAddress socketAddress;
 
     /*
      * A normal @{EmbeddedChannel} constructor which takes the remote client
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
index 2a496fb0ec222..aa9b6f1c19c5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
index aa2c5a7c8e3f9..073bb532ddf6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
@@ -16,7 +16,7 @@
  */
 package org.apache.hadoop.hdfs.server.diskbalancer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.function.Supplier;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
index e9c74aedd08b5..2fab1410d02df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
@@ -19,7 +19,7 @@
 
 package org.apache.hadoop.hdfs.server.diskbalancer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.function.Supplier;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index 1d6331d83c558..d0107b507ade9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -158,7 +158,7 @@ public void testSubmitPlanInNonRegularStatus() throws Exception {
    * Tests running multiple commands under on setup. This mainly covers
    * {@link org.apache.hadoop.hdfs.server.diskbalancer.command.Command#close}
    */
-  @Test(timeout = 60000)
+  @Test(timeout = 120000)
   public void testRunMultipleCommandsUnderOneSetup() throws Exception {
 
     final int numDatanodes = 1;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index d78b598bd3f5c..90353c352ea41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -949,7 +949,7 @@ public void testMoverWithStripedFile() throws Exception {
               {StorageType.ARCHIVE, StorageType.ARCHIVE},
               {StorageType.ARCHIVE, StorageType.ARCHIVE},
               {StorageType.ARCHIVE, StorageType.ARCHIVE}},
-          true, null, null, null,capacities, null, false, false, false, null);
+          true, null, null, null, capacities, null, false, false, false, null, null, null);
       cluster.triggerHeartbeats();
 
       // move file to ARCHIVE
@@ -982,7 +982,7 @@ public void testMoverWithStripedFile() throws Exception {
               { StorageType.SSD, StorageType.DISK },
               { StorageType.SSD, StorageType.DISK },
               { StorageType.SSD, StorageType.DISK } },
-          true, null, null, null, capacities, null, false, false, false, null);
+          true, null, null, null, capacities, null, false, false, false, null, null, null);
       cluster.triggerHeartbeats();
 
       // move file blocks to ONE_SSD policy
@@ -1372,7 +1372,7 @@ private void startAdditionalDNs(final Configuration conf,
       final MiniDFSCluster cluster) throws IOException {
 
     cluster.startDataNodes(conf, newNodesRequired, newTypes, true, null, null,
-        null, null, null, false, false, false, null);
+        null, null, null, false, false, false, null, null, null);
     cluster.triggerHeartbeats();
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index d95e76fc85294..94f7ef69c9bb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -65,7 +65,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 
 /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
index 7f4a0ce54197d..0334cb8a45f37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
@@ -416,7 +416,7 @@ public static void assertFileContentsSame(File... files) throws Exception {
     if (files.length < 2) return;
     
     Map<File, String> md5s = getFileMD5s(files);
-    if (Sets.newHashSet(md5s.values()).size() > 1) {
+    if (new HashSet<>(md5s.values()).size() > 1) {
       fail("File contents differed:\n  " +
           Joiner.on("\n  ")
             .withKeyValueSeparator("=")
@@ -433,7 +433,8 @@ public static void assertFileContentsDifferent(
       File... files) throws Exception
   {
     Map<File, String> md5s = getFileMD5s(files);
-    if (Sets.newHashSet(md5s.values()).size() != expectedUniqueHashes) {
+    int uniqueHashes = new HashSet<>(md5s.values()).size();
+    if (uniqueHashes != expectedUniqueHashes) {
       fail("Expected " + expectedUniqueHashes + " different hashes, got:\n  " +
           Joiner.on("\n  ")
             .withKeyValueSeparator("=")
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
index 6f7087b921510..ed18696c2325e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
@@ -77,6 +77,10 @@ synchronized String getNextFileName(String fileNamePrefix) {
     return fn;
   }
 
+  public long getFileCount() {
+    return fileCount;
+  }
+
   private synchronized void reset() {
     Arrays.fill(pathIndecies, -1);
     fileCount = 0L;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 7f6d572fceb39..847f7dc0c12c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -27,7 +27,7 @@
 import java.util.EnumSet;
 import java.util.List;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -569,20 +569,23 @@ void generateInputs(int[] opsPerThread) throws IOException {
       // int generatedFileIdx = 0;
       LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
       fileNames = new String[numThreads][];
-      for(int idx=0; idx < numThreads; idx++) {
-        int threadOps = opsPerThread[idx];
-        fileNames[idx] = new String[threadOps];
-        for(int jdx=0; jdx < threadOps; jdx++)
-          fileNames[idx][jdx] = nameGenerator.
-                                  getNextFileName("ThroughputBench");
+      try {
+        for(int idx=0; idx < numThreads; idx++) {
+          int threadOps = opsPerThread[idx];
+          fileNames[idx] = new String[threadOps];
+          for(int jdx=0; jdx < threadOps; jdx++) {
+            fileNames[idx][jdx] = nameGenerator.
+                    getNextFileName("ThroughputBench");
+          }
+        }
+      } catch (ArrayIndexOutOfBoundsException e) {
+        LOG.error("The current environment allows {} files to be created. " +
+            "If you want to test more files, please update the -filesPerDir parameter.",
+                nameGenerator.getFileCount());
+        throw e;
       }
     }
 
-    void dummyActionNoSynch(int daemonId, int fileIdx) {
-      for(int i=0; i < 2000; i++)
-        fileNames[daemonId][fileIdx].contains(""+i);
-    }
-
     /**
      * returns client name
      */
@@ -598,7 +601,6 @@ String getExecutionArgument(int daemonId) {
     long executeOp(int daemonId, int inputIdx, String clientName) 
     throws IOException {
       long start = Time.now();
-      // dummyActionNoSynch(fileIdx);
       clientProto.create(fileNames[daemonId][inputIdx],
           FsPermission.getDefault(), clientName,
           new EnumSetWritable<CreateFlag>(EnumSet
@@ -675,12 +677,20 @@ void generateInputs(int[] opsPerThread) throws IOException {
           false);
       LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
       dirPaths = new String[numThreads][];
-      for(int idx=0; idx < numThreads; idx++) {
-        int threadOps = opsPerThread[idx];
-        dirPaths[idx] = new String[threadOps];
-        for(int jdx=0; jdx < threadOps; jdx++)
-          dirPaths[idx][jdx] = nameGenerator.
-              getNextFileName("ThroughputBench");
+      try {
+        for(int idx=0; idx < numThreads; idx++) {
+          int threadOps = opsPerThread[idx];
+          dirPaths[idx] = new String[threadOps];
+          for(int jdx=0; jdx < threadOps; jdx++) {
+            dirPaths[idx][jdx] = nameGenerator.
+                    getNextFileName("ThroughputBench");
+          }
+        }
+      } catch (ArrayIndexOutOfBoundsException e) {
+        LOG.error("The current environment allows {} directories to be created. " +
+            "If you want to test more directories, please update the -dirsPerDir parameter.",
+                nameGenerator.getFileCount());
+        throw e;
       }
     }
 
@@ -1219,10 +1229,28 @@ void generateInputs(int[] ignore) throws IOException {
 
     private ExtendedBlock addBlocks(String fileName, String clientName)
     throws IOException {
+      DatanodeInfo[] excludeNodes = null;
+      DatanodeInfo[] dnInfos = clientProto.getDatanodeReport(
+          HdfsConstants.DatanodeReportType.LIVE);
+      if (dnInfos != null && dnInfos.length > 0) {
+        List<DatanodeInfo> tmpNodes = new ArrayList<>();
+        String localHost = DNS.getDefaultHost("default", "default");
+        for (DatanodeInfo dnInfo : dnInfos) {
+          if (!localHost.equals(dnInfo.getHostName()) ||
+              (dnInfo.getXferPort() > datanodes.length)) {
+            tmpNodes.add(dnInfo);
+          }
+        }
+
+        if (tmpNodes.size() > 0) {
+          excludeNodes = tmpNodes.toArray(new DatanodeInfo[tmpNodes.size()]);
+        }
+      }
+
       ExtendedBlock prevBlock = null;
       for(int jdx = 0; jdx < blocksPerFile; jdx++) {
         LocatedBlock loc = addBlock(fileName, clientName,
-            prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
+            prevBlock, excludeNodes, HdfsConstants.GRANDFATHER_INODE_ID, null);
         prevBlock = loc.getBlock();
         for(DatanodeInfo dnInfo : loc.getLocations()) {
           int dnIdx = dnInfo.getXferPort() - 1;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
index 088a47e893650..1a763b5bae3bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
@@ -99,7 +99,7 @@ public void testRetryAddBlockWhileInChooseTarget() throws Exception {
                                             HdfsConstants.GRANDFATHER_INODE_ID,
                                             "clientName", null, onRetryBlock);
     } finally {
-      ns.readUnlock();;
+      ns.readUnlock();
     }
     DatanodeStorageInfo targets[] = FSDirWriteFileOp.chooseTargetForNewBlock(
         ns.getBlockManager(), src, null, null, null, r);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java
index da4e71e3c5d4c..cfed8d23ce381 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAllowFormat.java
@@ -165,7 +165,8 @@ public void testFormatShouldBeIgnoredForNonFileBasedDirs() throws Exception {
     String localhost = "127.0.0.1";
     InetSocketAddress nnAddr1 = new InetSocketAddress(localhost, 8020);
     InetSocketAddress nnAddr2 = new InetSocketAddress(localhost, 8020);
-    HATestUtil.setFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
+    HATestUtil.setFailoverConfigurations(conf, logicalName, null,
+        nnAddr1, nnAddr2);
 
     conf.set(DFS_NAMENODE_NAME_DIR_KEY,
         new File(DFS_BASE_DIR, "name").getAbsolutePath());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index aa2c7f689766f..1cc950723be34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -55,6 +55,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY;
@@ -69,6 +70,7 @@
 import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.NNTOP_ENABLED_KEY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -89,6 +91,20 @@ public class TestAuditLogger {
   }
 
   private static final short TEST_PERMISSION = (short) 0654;
+  private static final Pattern AUDIT_PATTERN = Pattern.compile(
+      ".*allowed=.*?\\s" +
+      "ugi=.*?\\s" +
+      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+      "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
+      "perm=.*?");
+  private static final Pattern AUDIT_WITH_PORT_PATTERN = Pattern.compile(
+      ".*allowed=.*?\\s" +
+      "ugi=.*?\\s" +
+      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+      "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
+      "perm=.*?" +
+      "proto=.*?" +
+      "callerContext=.*?clientPort\\:(\\d{0,9}).*?");
 
   @Before
   public void setup() {
@@ -240,10 +256,9 @@ public void testAuditLoggerWithCallContext() throws IOException {
     conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
     conf.setInt(HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY, 128);
     conf.setInt(HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY, 40);
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
-    LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
 
-    try {
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+      LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
       cluster.waitClusterUp();
       final FileSystem fs = cluster.getFileSystem();
       final long time = System.currentTimeMillis();
@@ -398,8 +413,8 @@ public void run() {
       assertFalse(auditlog.getOutput().contains("callerContext="));
       auditlog.clearOutput();
 
-    } finally {
-      cluster.shutdown();
+      // clear client context
+      CallerContext.setCurrent(null);
     }
   }
 
@@ -544,6 +559,75 @@ public void testBrokenLogger() throws IOException {
     }
   }
 
+  /**
+   * Test adding remote port to audit log.
+   */
+  @Test
+  public void testAuditLogWithRemotePort() throws Exception {
+    // Audit log without remote port by default.
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster1 = new MiniDFSCluster.Builder(conf).build();
+    try {
+      LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+      cluster1.waitClusterUp();
+      FileSystem fs = cluster1.getFileSystem();
+      long time = System.currentTimeMillis();
+      fs.setTimes(new Path("/"), time, time);
+      assertTrue(AUDIT_PATTERN.matcher(auditLog.getOutput().trim()).matches());
+      assertFalse(auditLog.getOutput().contains("clientPort"));
+      auditLog.clearOutput();
+    } finally {
+      cluster1.shutdown();
+    }
+
+    // Audit log with remote port.
+    conf.setBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY, true);
+    conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
+    MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf).build();
+    try {
+      LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+      cluster2.waitClusterUp();
+      FileSystem fs = cluster2.getFileSystem();
+      long time = System.currentTimeMillis();
+      fs.setTimes(new Path("/"), time, time);
+      assertTrue(AUDIT_WITH_PORT_PATTERN.matcher(
+          auditLog.getOutput().trim()).matches());
+      auditLog.clearOutput();
+    } finally {
+      cluster2.shutdown();
+    }
+  }
+
+  @Test
+  public void testCallerContextCharacterEscape() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true);
+    conf.setInt(HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY, 128);
+    conf.setInt(HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY, 40);
+
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+      LogCapturer auditlog = LogCapturer.captureLogs(FSNamesystem.auditLog);
+      cluster.waitClusterUp();
+      final FileSystem fs = cluster.getFileSystem();
+      final long time = System.currentTimeMillis();
+      final Path p = new Path("/");
+
+      assertNull(CallerContext.getCurrent());
+
+      CallerContext context = new CallerContext.Builder("c1\nc2").append("c3\tc4")
+          .setSignature("s1\ns2".getBytes(CallerContext.SIGNATURE_ENCODING)).build();
+      CallerContext.setCurrent(context);
+      LOG.info("Set current caller context as {}", CallerContext.getCurrent());
+      fs.setTimes(p, time, time);
+      assertTrue(auditlog.getOutput().endsWith(
+          String.format("callerContext=c1\\nc2,c3\\tc4:s1\\ns2%n")));
+      auditlog.clearOutput();
+
+      // clear client context
+      CallerContext.setCurrent(null);
+    }
+  }
+
   public static class DummyAuditLogger implements AuditLogger {
 
     static boolean initialized;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
index 9fe7404eed8eb..2832782caf918 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
@@ -92,9 +92,9 @@ public TestAuditLogs(boolean useAsyncLog, boolean useAsyncEdits) {
   // allowed=(true|false) ugi=name ip=/address cmd={cmd} src={path} dst=null perm=null
   static final Pattern auditPattern = Pattern.compile(
       "allowed=.*?\\s" +
-      "ugi=.*?\\s" + 
-      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + 
-      "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + 
+      "ugi=.*?\\s" +
+      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
+      "cmd=.*?\\ssrc=.*?\\sdst=null\\s" +
       "perm=.*?");
   static final Pattern successPattern = Pattern.compile(
       ".*allowed=true.*");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
index 88b7d2bf7f23d..3beea47800d7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
@@ -19,24 +19,35 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.net.StaticMapping;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.*;
+import java.util.function.Supplier;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -167,6 +178,108 @@ private void doTestChooseTargetSpecialCase() throws Exception {
     }
   }
 
+  /**
+   * Verify decommission a dn which is an only node in its rack.
+   */
+  @Test
+  public void testPlacementWithOnlyOneNodeInRackDecommission() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    final String[] racks = {"/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK4", "/RACK5", "/RACK2"};
+    final String[] hosts = {"/host0", "/host1", "/host2", "/host3", "/host4", "/host5", "/host6"};
+
+    // enables DFSNetworkTopology
+    conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+        BlockPlacementPolicyRackFaultTolerant.class,
+        BlockPlacementPolicy.class);
+    conf.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
+        DEFAULT_BLOCK_SIZE / 2);
+
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(7).racks(racks)
+        .hosts(hosts).build();
+    cluster.waitActive();
+    nameNodeRpc = cluster.getNameNodeRpc();
+    namesystem = cluster.getNamesystem();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    fs.enableErasureCodingPolicy("RS-3-2-1024k");
+    fs.setErasureCodingPolicy(new Path("/"), "RS-3-2-1024k");
+
+    final BlockManager bm = cluster.getNamesystem().getBlockManager();
+    final DatanodeManager dm = bm.getDatanodeManager();
+    assertTrue(dm.getNetworkTopology() instanceof DFSNetworkTopology);
+
+    String clientMachine = "/host4";
+    String clientRack = "/RACK4";
+    String src = "/test";
+
+    final DatanodeManager dnm = namesystem.getBlockManager().getDatanodeManager();
+    DatanodeDescriptor dnd4 = dnm.getDatanode(cluster.getDataNodes().get(4).getDatanodeId());
+    assertEquals(dnd4.getNetworkLocation(), clientRack);
+    dnm.getDatanodeAdminManager().startDecommission(dnd4);
+    short replication = 5;
+    short additionalReplication = 1;
+
+    try {
+      // Create the file with client machine
+      HdfsFileStatus fileStatus = namesystem.startFile(src, perm,
+          clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true,
+          replication, DEFAULT_BLOCK_SIZE * 1024 * 10, null, null, null, false);
+
+      //test chooseTarget for new file
+      LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine,
+          null, null, fileStatus.getFileId(), null, null);
+      HashMap<String, Integer> racksCount = new HashMap<String, Integer>();
+      doTestLocatedBlockRacks(racksCount, replication, 4, locatedBlock);
+
+      //test chooseTarget for existing file.
+      LocatedBlock additionalLocatedBlock =
+          nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(),
+              locatedBlock.getBlock(), locatedBlock.getLocations(),
+              locatedBlock.getStorageIDs(), DatanodeInfo.EMPTY_ARRAY,
+              additionalReplication, clientMachine);
+
+      racksCount.clear();
+      doTestLocatedBlockRacks(racksCount, additionalReplication + replication,
+          4, additionalLocatedBlock);
+      assertEquals(racksCount.get("/RACK0"), (Integer)2);
+      assertEquals(racksCount.get("/RACK2"), (Integer)2);
+    } finally {
+      dnm.getDatanodeAdminManager().stopDecommission(dnd4);
+    }
+
+    //test if decommission succeeded
+    DatanodeDescriptor dnd3 = dnm.getDatanode(cluster.getDataNodes().get(3).getDatanodeId());
+    cluster.getNamesystem().writeLock();
+    try {
+      dm.getDatanodeAdminManager().startDecommission(dnd3);
+    } finally {
+      cluster.getNamesystem().writeUnlock();
+    }
+
+    // make sure the decommission finishes and the block in on 4 racks
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        return dnd3.isDecommissioned();
+      }
+    }, 1000, 10 * 1000);
+
+    LocatedBlocks locatedBlocks =
+        cluster.getFileSystem().getClient().getLocatedBlocks(
+            src, 0, DEFAULT_BLOCK_SIZE);
+    assertEquals(4, bm.getDatanodeManager().
+        getNetworkTopology().getNumOfNonEmptyRacks());
+    for (LocatedBlock block : locatedBlocks.getLocatedBlocks()) {
+      BlockPlacementStatus status = bm.getStriptedBlockPlacementPolicy()
+              .verifyBlockPlacement(block.getLocations(), 5);
+      Assert.assertTrue(status.isPlacementPolicySatisfied());
+    }
+  }
+
   private void shuffle(DatanodeInfo[] locs, String[] storageIDs) {
     int length = locs.length;
     Object[][] pairs = new Object[length][];
@@ -198,6 +311,17 @@ private void doTestLocatedBlock(int replication, LocatedBlock locatedBlock) {
     assertTrue(maxCount - minCount <= 1);
   }
 
+  private void doTestLocatedBlockRacks(HashMap<String, Integer> racksCount, int replication,
+                                       int validracknum, LocatedBlock locatedBlock) {
+    assertEquals(replication, locatedBlock.getLocations().length);
+
+    for (DatanodeInfo node :
+        locatedBlock.getLocations()) {
+      addToRacksCount(node.getNetworkLocation(), racksCount);
+    }
+    assertEquals(validracknum, racksCount.size());
+  }
+
   private void addToRacksCount(String rack, HashMap<String, Integer> racksCount) {
     Integer count = racksCount.get(rack);
     if (count == null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
index 49e79da674e5d..cab87e014a756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
@@ -45,9 +45,6 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-
 public class TestBlockUnderConstruction {
   static final String BASE_DIR = "/test/TestBlockUnderConstruction";
   static final int BLOCK_SIZE = 8192; // same as TestFileCreation.blocksize
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
index cff4e1f32f5b6..33fefca5fda8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckPointForSecurityTokens.java
@@ -91,7 +91,7 @@ public void testSaveNamespace() throws IOException {
         log.scanLog(Long.MAX_VALUE, true);
         long numTransactions = (log.getLastTxId() - log.getFirstTxId()) + 1;
         assertEquals("In-progress log " + log + " should have 5 transactions",
-                     5, numTransactions);;
+                     5, numTransactions);
       }
 
       // Saving image in safe mode should succeed
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
index 064d5ae83957a..1265863e11515 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
@@ -2613,7 +2613,7 @@ private static List<File> getCheckpointCurrentDirs(SecondaryNameNode secondary)
   }
 
   private static CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) {
-    CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());;
+    CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());
     secondary1.setFSImage(spy);
     return spy;
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockWithInvalidGenStamp.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockWithInvalidGenStamp.java
index f7b1ea5bac351..d57e00ef3967e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockWithInvalidGenStamp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockWithInvalidGenStamp.java
@@ -20,14 +20,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.io.IOUtils;
 import org.junit.After;
 import org.junit.Assert;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
index e8bd8377a3c85..420aa8c1af79c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
@@ -445,6 +445,8 @@ public void testDecommissionStatusAfterDNRestart() throws Exception {
     // Delete the under-replicated file, which should let the 
     // DECOMMISSION_IN_PROGRESS node become DECOMMISSIONED
     AdminStatesBaseTest.cleanupFile(fileSys, f);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     BlockManagerTestUtil.recheckDecommissionState(dm);
     // Block until the admin's monitor updates the number of tracked nodes.
     waitForDecommissionedNodes(dm.getDatanodeAdminManager(), 0);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
index 083d9e5070dd7..9dc2cc9182bba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
@@ -581,8 +581,8 @@ public void run() {
   }
 
   static SetOwnerOp getSetOwnerOp(OpInstanceCache cache, String group) {
-    return ((SetOwnerOp)cache.get(OP_SET_OWNER))
-        .setSource("/").setUser("u").setGroup(group);
+    SetOwnerOp setOwnerOp = cache.get(OP_SET_OWNER);
+    return setOwnerOp.setSource("/").setUser("u").setGroup(group);
   }
 
   static class BlockingOpMatcher implements ArgumentMatcher<FSEditLogOp> {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 185db6916ab0d..0445d211fee89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -80,6 +80,7 @@
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.Time;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -275,6 +276,22 @@ public void testSaveAndLoadStripedINodeFile() throws IOException{
     }
   }
 
+  @Test
+  public void testImportCheckpoint() throws Exception{
+    Configuration conf = new Configuration();
+    conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY, "");
+    try(MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()){
+      cluster.waitActive();
+      FSNamesystem fsn = cluster.getNamesystem();
+      FSImage fsImage= new FSImage(conf);
+      LambdaTestUtils.intercept(
+          IOException.class,
+          "Cannot import image from a checkpoint. "
+                  + "\"dfs.namenode.checkpoint.edits.dir\" is not set.",
+          () -> fsImage.doImportCheckpoint(fsn));
+    }
+  }
+
   /**
    * Test if a INodeFileUnderConstruction with BlockInfoStriped can be
    * saved and loaded by FSImageSerialization
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
index f5a112c7acd54..b8cc32e43e2d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java
@@ -150,7 +150,7 @@ public void testWhenSomeNodesAreNotGood() throws Exception {
     d.stopDecommission();
 
     BlockLocation[] locations = getBlockLocations(p);
-    Assert.assertEquals(replication, locations[0].getNames().length);;
+    Assert.assertEquals(replication, locations[0].getNames().length);
     //also make sure that the datanode[0] is not in the list of hosts
     for (int i = 0; i < replication; i++) {
       final String loc = locations[0].getNames()[i];
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java
index da09298b777dd..3ae6f4f7baccd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileContextXAttr.java
@@ -29,9 +29,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.junit.BeforeClass;
 
 /**
  * Tests of XAttr operations using FileContext APIs.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
index d5a04c8e4ffe5..02818c952dabf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileLimit.java
@@ -30,7 +30,6 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 57f5ea33eb0ee..13bc00f0944e3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -33,6 +33,7 @@
 import java.io.IOException;
 import java.util.concurrent.ThreadLocalRandom;
 
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.test.LambdaTestUtils;
@@ -319,7 +320,8 @@ public void testTruncateWithOtherOperations() throws IOException {
   }
 
   @Test
-  public void testSnapshotWithAppendTruncate() throws IOException {
+  public void testSnapshotWithAppendTruncate()
+      throws IOException, InterruptedException {
     testSnapshotWithAppendTruncate(0, 1, 2);
     testSnapshotWithAppendTruncate(0, 2, 1);
     testSnapshotWithAppendTruncate(1, 0, 2);
@@ -333,7 +335,8 @@ public void testSnapshotWithAppendTruncate() throws IOException {
    * Delete snapshots in the specified order and verify that
    * remaining snapshots are still readable.
    */
-  void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
+  void testSnapshotWithAppendTruncate(int... deleteOrder)
+      throws IOException, InterruptedException {
     FSDirectory fsDir = cluster.getNamesystem().getFSDirectory();
     fs.mkdirs(parent);
     fs.setQuota(parent, 100, 1000);
@@ -381,16 +384,16 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     // Truncate to block boundary
     int newLength = length[0] + BLOCK_SIZE / 2;
     boolean isReady = fs.truncate(src, newLength);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertTrue("Recovery is not expected.", isReady);
     assertFileLength(snapshotFiles[2], length[2]);
     assertFileLength(snapshotFiles[1], length[1]);
     assertFileLength(snapshotFiles[0], length[0]);
     assertBlockNotPresent(appendedBlk);
-
     // Diskspace consumed should be 16 bytes * 3. [blk 1,2,3 SS:4]
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(48L));
-
     // Truncate full block again
     newLength = length[0] - BLOCK_SIZE / 2;
     isReady = fs.truncate(src, newLength);
@@ -398,11 +401,9 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     assertFileLength(snapshotFiles[2], length[2]);
     assertFileLength(snapshotFiles[1], length[1]);
     assertFileLength(snapshotFiles[0], length[0]);
-
     // Diskspace consumed should be 16 bytes * 3. [blk 1,2 SS:3,4]
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(48L));
-
     // Truncate half of the last block
     newLength -= BLOCK_SIZE / 2;
     isReady = fs.truncate(src, newLength);
@@ -413,15 +414,12 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     assertFileLength(snapshotFiles[0], length[0]);
     Block replacedBlk = getLocatedBlocks(src).getLastLocatedBlock()
         .getBlock().getLocalBlock();
-
     // Diskspace consumed should be 16 bytes * 3. [blk 1,6 SS:2,3,4]
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(54L));
-
     snapshotDir = fs.createSnapshot(parent, ss[3]);
     snapshotFiles[3] = new Path(snapshotDir, truncateFile);
     length[3] = newLength;
-
     // Delete file. Should still be able to read snapshots
     int numINodes = fsDir.getInodeMapSize();
     isReady = fs.delete(src, false);
@@ -432,17 +430,15 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     assertFileLength(snapshotFiles[0], length[0]);
     assertEquals("Number of INodes should not change",
         numINodes, fsDir.getInodeMapSize());
-
     fs.deleteSnapshot(parent, ss[3]);
-
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertBlockExists(firstBlk);
     assertBlockExists(lastBlk);
     assertBlockNotPresent(replacedBlk);
-
     // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(48L));
-
     // delete snapshots in the specified order
     fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
     assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
@@ -451,12 +447,12 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     assertBlockExists(lastBlk);
     assertEquals("Number of INodes should not change",
         numINodes, fsDir.getInodeMapSize());
-
     // Diskspace consumed should be 16 bytes * 3. [SS:1,2,3,4]
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(48L));
-
     fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
     assertBlockExists(firstBlk);
     contentSummary = fs.getContentSummary(parent);
@@ -470,11 +466,11 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
     }
     assertEquals("Number of INodes should not change",
         numINodes, fsDir .getInodeMapSize());
-
     fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertBlockNotPresent(firstBlk);
     assertBlockNotPresent(lastBlk);
-
     // Diskspace consumed should be 0 bytes * 3. []
     contentSummary = fs.getContentSummary(parent);
     assertThat(contentSummary.getSpaceConsumed(), is(0L));
@@ -488,7 +484,8 @@ void testSnapshotWithAppendTruncate(int ... deleteOrder) throws IOException {
    * remaining snapshots are still readable.
    */
   @Test
-  public void testSnapshotWithTruncates() throws IOException {
+  public void testSnapshotWithTruncates()
+      throws IOException, InterruptedException {
     testSnapshotWithTruncates(0, 1, 2);
     testSnapshotWithTruncates(0, 2, 1);
     testSnapshotWithTruncates(1, 0, 2);
@@ -497,7 +494,8 @@ public void testSnapshotWithTruncates() throws IOException {
     testSnapshotWithTruncates(2, 1, 0);
   }
 
-  void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
+  void testSnapshotWithTruncates(int... deleteOrder)
+      throws IOException, InterruptedException {
     fs.mkdirs(parent);
     fs.setQuota(parent, 100, 1000);
     fs.allowSnapshot(parent);
@@ -544,6 +542,8 @@ void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
     assertThat(contentSummary.getSpaceConsumed(), is(42L));
 
     fs.deleteSnapshot(parent, ss[deleteOrder[0]]);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertFileLength(snapshotFiles[deleteOrder[1]], length[deleteOrder[1]]);
     assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
     assertFileLength(src, length[2]);
@@ -561,6 +561,8 @@ void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
     }
 
     fs.deleteSnapshot(parent, ss[deleteOrder[1]]);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertFileLength(snapshotFiles[deleteOrder[2]], length[deleteOrder[2]]);
     assertFileLength(src, length[2]);
     assertBlockExists(firstBlk);
@@ -581,6 +583,8 @@ void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
     }
 
     fs.deleteSnapshot(parent, ss[deleteOrder[2]]);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     assertFileLength(src, length[2]);
     assertBlockExists(firstBlk);
 
@@ -590,6 +594,8 @@ void testSnapshotWithTruncates(int ... deleteOrder) throws IOException {
     assertThat(contentSummary.getLength(), is(6L));
 
     fs.delete(src, false);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem().getBlockManager());
     assertBlockNotPresent(firstBlk);
 
     // Diskspace consumed should be 0 bytes * 3. []
@@ -1256,7 +1262,8 @@ public void testTruncateWithRollingUpgrade() throws Exception {
         cluster.getNamesystem().getFSDirectory().getBlockManager()
             .getTotalBlocks());
     fs.delete(p, true);
-
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem().getBlockManager());
     assertEquals("block num should 0", 0,
         cluster.getNamesystem().getFSDirectory().getBlockManager()
             .getTotalBlocks());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 049a785481616..420635e012ac7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -50,6 +50,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -118,7 +119,6 @@
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -144,14 +144,14 @@ public class TestFsck {
   // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null
   static final Pattern FSCK_PATTERN = Pattern.compile(
       "allowed=.*?\\s" +
-      "ugi=.*?\\s" + 
-      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + 
+      "ugi=.*?\\s" +
+      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
       "cmd=fsck\\ssrc=\\/\\sdst=null\\s" + 
       "perm=null\\s" + "proto=.*");
   static final Pattern GET_FILE_INFO_PATTERN = Pattern.compile(
       "allowed=.*?\\s" +
-      "ugi=.*?\\s" + 
-      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + 
+      "ugi=.*?\\s" +
+      "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
       "cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" + 
       "perm=null\\s" + "proto=.*");
 
@@ -386,15 +386,15 @@ public void testFsckMove() throws Exception {
                                         cluster.getNameNodePort()), conf);
     String[] fileNames = util.getFileNames(topDir);
     CorruptedTestFile[] ctFiles = new CorruptedTestFile[]{
-        new CorruptedTestFile(fileNames[0], Sets.newHashSet(0),
+        new CorruptedTestFile(fileNames[0], new HashSet<>(Arrays.asList(0)),
             dfsClient, numDatanodes, dfsBlockSize),
-        new CorruptedTestFile(fileNames[1], Sets.newHashSet(2, 3),
+        new CorruptedTestFile(fileNames[1], new HashSet<>(Arrays.asList(2, 3)),
             dfsClient, numDatanodes, dfsBlockSize),
-        new CorruptedTestFile(fileNames[2], Sets.newHashSet(4),
+        new CorruptedTestFile(fileNames[2], new HashSet<>(Arrays.asList(4)),
             dfsClient, numDatanodes, dfsBlockSize),
-        new CorruptedTestFile(fileNames[3], Sets.newHashSet(0, 1, 2, 3),
+        new CorruptedTestFile(fileNames[3], new HashSet<>(Arrays.asList(0, 1, 2, 3)),
             dfsClient, numDatanodes, dfsBlockSize),
-        new CorruptedTestFile(fileNames[4], Sets.newHashSet(1, 2, 3, 4),
+        new CorruptedTestFile(fileNames[4], new HashSet<>(Arrays.asList(1, 2, 3, 4)),
             dfsClient, numDatanodes, dfsBlockSize)
     };
     int totalMissingBlocks = 0;
@@ -2215,7 +2215,7 @@ public void testFsckMoveAfterCorruption() throws Exception {
         new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
     final String blockFileToCorrupt = fileNames[0];
     final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt,
-        Sets.newHashSet(0), dfsClient, numDatanodes, dfsBlockSize);
+        new HashSet<>(Arrays.asList(0)), dfsClient, numDatanodes, dfsBlockSize);
     ctf.corruptBlocks(cluster);
 
     // Wait for fsck to discover all the missing blocks
@@ -2310,7 +2310,7 @@ private void testUpgradeDomain(boolean defineUpgradeDomain,
       dnProp.setPort(datanodeID.getXferPort());
       dnProp.setUpgradeDomain(upgradeDomain);
       hostsFileWriter.initIncludeHosts(new DatanodeAdminProperties[]{dnProp});
-      cluster.getFileSystem().refreshNodes();
+      cluster.getNamesystem(0).getBlockManager().getDatanodeManager().refreshNodes(conf);
     }
 
     // create files
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java
index 327c51ca765fa..5870fbb549884 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.viewfs.ConfigUtil;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
index e86413d8bf3dc..78ebcb4e2dfd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.lang.management.ManagementFactory;
@@ -175,4 +176,38 @@ public void testHostsIncludeForDeadCount() throws Exception {
       hostsFileWriter.cleanup();
     }
   }
+
+  @Test
+  public void testNewHostAndExcludeFile() throws Exception {
+    Configuration conf = getConf();
+
+    HostsFileWriter writer1 = new HostsFileWriter();
+    writer1.initialize(conf, "old_temp/decommission");
+    writer1.initIncludeHosts(new String[]{"localhost:52", "127.0.0.1:7777"});
+
+    // Write all hosts to a new dfs.hosts file.
+    HostsFileWriter writer2 = new HostsFileWriter();
+    Configuration newConf = new Configuration(getConf());
+    writer2.initialize(newConf, "new_temp/decommission");
+    writer2.initIncludeHosts(new String[]{
+        "localhost:52", "127.0.0.1:7777", "localhost:100"});
+
+    MiniDFSCluster cluster = null;
+    try {
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+      final FSNamesystem ns = cluster.getNameNode().getNamesystem();
+      assertEquals(2, ns.getNumDeadDataNodes());
+      assertEquals(0, ns.getNumLiveDataNodes());
+
+      ns.getBlockManager().getDatanodeManager().refreshNodes(newConf);
+      assertEquals(3, ns.getNumDeadDataNodes());
+      assertEquals(0, ns.getNumLiveDataNodes());
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+      writer1.cleanup();
+      writer2.cleanup();
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
index 776a1981ce4e5..700b32f2895c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java
@@ -48,6 +48,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.junit.Assert.fail;
+
 public class TestINodeAttributeProvider {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestINodeAttributeProvider.class);
@@ -57,6 +59,15 @@ public class TestINodeAttributeProvider {
   private static final short HDFS_PERMISSION = 0777;
   private static final short PROVIDER_PERMISSION = 0770;
   private static boolean runPermissionCheck = false;
+  private static boolean shouldThrowAccessException = false;
+
+  public static class MyAuthorizationProviderAccessException
+      extends AccessControlException {
+
+    public MyAuthorizationProviderAccessException() {
+      super();
+    }
+  };
 
   public static class MyAuthorizationProvider extends INodeAttributeProvider {
 
@@ -82,6 +93,9 @@ public void checkPermission(String fsOwner, String supergroup,
               ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir);
         }
         CALLED.add("checkPermission|" + ancestorAccess + "|" + parentAccess + "|" + access);
+        if (shouldThrowAccessException) {
+          throw new MyAuthorizationProviderAccessException();
+        }
       }
 
       @Override
@@ -96,6 +110,9 @@ public void checkPermissionWithContext(
         CALLED.add("checkPermission|" + authzContext.getAncestorAccess()
             + "|" + authzContext.getParentAccess() + "|" + authzContext
             .getAccess());
+        if (shouldThrowAccessException) {
+          throw new MyAuthorizationProviderAccessException();
+        }
       }
     }
 
@@ -238,6 +255,7 @@ public void cleanUp() throws IOException {
       miniDFS = null;
     }
     runPermissionCheck = false;
+    shouldThrowAccessException = false;
     Assert.assertTrue(CALLED.contains("stop"));
   }
 
@@ -457,6 +475,44 @@ public Void run() throws Exception {
     });
   }
 
+  @Test
+  // HDFS-16529 - Ensure enforcer AccessControlException subclass are caught
+  // and re-thrown as plain ACE exceptions.
+  public void testSubClassedAccessControlExceptions() throws Exception {
+    FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
+    shouldThrowAccessException = true;
+    final Path userPath = new Path("/user");
+    final Path authz = new Path("/user/authz");
+    final Path authzChild = new Path("/user/authz/child2");
+
+    fs.mkdirs(userPath);
+    fs.setPermission(userPath, new FsPermission(HDFS_PERMISSION));
+    fs.mkdirs(authz);
+    fs.setPermission(authz, new FsPermission(HDFS_PERMISSION));
+    fs.mkdirs(authzChild);
+    fs.setPermission(authzChild, new FsPermission(HDFS_PERMISSION));
+    UserGroupInformation ugi = UserGroupInformation.createUserForTesting("u1",
+        new String[]{"g1"});
+    ugi.doAs(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
+        try {
+          fs.access(authzChild, FsAction.ALL);
+          fail("Exception should be thrown");
+          // The DFS Client will get a RemoteException containing an
+          // AccessControlException (ACE). If the ACE is a subclass of ACE then
+          // the client does not unwrap it correctly. The change in HDFS-16529
+          // is to ensure ACE is always thrown rather than a sub class to avoid
+          // this issue.
+        } catch (AccessControlException ace) {
+          Assert.assertEquals(AccessControlException.class, ace.getClass());
+        }
+        return null;
+      }
+    });
+  }
+
 
   @Test
   // HDFS-15165 - ContentSummary calls should use the provider permissions(if
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
index df36322e9f77a..42f054006c9a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.util.Random;
 
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -50,7 +51,6 @@ public class TestLargeDirectoryDelete {
   static {
     CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
     CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
-    CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1);
   }
   
   /** create a file with a length of <code>filelen</code> */
@@ -140,6 +140,8 @@ protected void execute() throws Throwable {
     
     final long start = Time.now();
     mc.getFileSystem().delete(new Path("/root"), true); // recursive delete
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        mc.getNamesystem(0).getBlockManager());
     final long end = Time.now();
     threads[0].endThread();
     threads[1].endThread();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
index c88570b56e0e0..4387f7679bce3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java
@@ -139,6 +139,8 @@ public void testMetasaveAfterDelete()
     nnRpc.delete("/filestatus0", true);
     nnRpc.delete("/filestatus1", true);
 
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem().getBlockManager());
     nnRpc.metaSave("metasaveAfterDelete.out.txt");
 
     // Verification
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
index 44bf5b78784e0..f7a8d92864fa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
@@ -166,4 +166,25 @@ public void testNNThroughputForAppendOp() throws Exception {
       }
     }
   }
+
+  /**
+   * This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
+   * for block report operation.
+   */
+  @Test(timeout = 120000)
+  public void testNNThroughputForBlockReportOp() throws Exception {
+    final Configuration conf = new HdfsConfiguration();
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
+    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).
+        numDataNodes(3).build()) {
+      cluster.waitActive();
+      final Configuration benchConf = new HdfsConfiguration();
+      benchConf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
+      benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
+      NNThroughputBenchmark.runBenchmark(benchConf,
+          new String[]{"-fs", cluster.getURI().toString(), "-op",
+              "blockReport", "-datanodes", "3", "-reports", "2"});
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 81c9cb8670066..d670025bf5069 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -1031,7 +1031,8 @@ public Boolean get() {
   @Test
   public void testTotalBlocksMetrics() throws Exception {
     MiniDFSCluster cluster = null;
-    FSNamesystem namesystem = null;
+    FSNamesystem activeNn = null;
+    FSNamesystem standbyNn = null;
     DistributedFileSystem fs = null;
     try {
       Configuration conf = new HdfsConfiguration();
@@ -1046,12 +1047,16 @@ public void testTotalBlocksMetrics() throws Exception {
       conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
 
       cluster = new MiniDFSCluster.Builder(conf)
-          .numDataNodes(totalSize).build();
-      namesystem = cluster.getNamesystem();
-      fs = cluster.getFileSystem();
+          .nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(1)).
+              numDataNodes(totalSize).build();
+      cluster.waitActive();
+      cluster.transitionToActive(0);
+      activeNn = cluster.getNamesystem(0);
+      standbyNn = cluster.getNamesystem(1);
+      fs = cluster.getFileSystem(0);
       fs.enableErasureCodingPolicy(
           StripedFileTestUtil.getDefaultECPolicy().getName());
-      verifyTotalBlocksMetrics(0L, 0L, namesystem.getTotalBlocks());
+      verifyTotalBlocksMetrics(0L, 0L, activeNn.getTotalBlocks());
 
       // create small file
       Path replDirPath = new Path("/replicated");
@@ -1068,7 +1073,7 @@ public void testTotalBlocksMetrics() throws Exception {
       final int smallLength = cellSize * dataBlocks;
       final byte[] smallBytes = StripedFileTestUtil.generateBytes(smallLength);
       DFSTestUtil.writeFile(fs, ecFileSmall, smallBytes);
-      verifyTotalBlocksMetrics(1L, 1L, namesystem.getTotalBlocks());
+      verifyTotalBlocksMetrics(1L, 1L, activeNn.getTotalBlocks());
 
       // create learge file
       Path replFileLarge = new Path(replDirPath, "replfile_large");
@@ -1079,15 +1084,20 @@ public void testTotalBlocksMetrics() throws Exception {
       final int largeLength = blockSize * totalSize + smallLength;
       final byte[] largeBytes = StripedFileTestUtil.generateBytes(largeLength);
       DFSTestUtil.writeFile(fs, ecFileLarge, largeBytes);
-      verifyTotalBlocksMetrics(3L, 3L, namesystem.getTotalBlocks());
+      verifyTotalBlocksMetrics(3L, 3L, activeNn.getTotalBlocks());
 
       // delete replicated files
       fs.delete(replDirPath, true);
-      verifyTotalBlocksMetrics(0L, 3L, namesystem.getTotalBlocks());
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
+      verifyTotalBlocksMetrics(0L, 3L, activeNn.getTotalBlocks());
 
       // delete ec files
       fs.delete(ecDirPath, true);
-      verifyTotalBlocksMetrics(0L, 0L, namesystem.getTotalBlocks());
+      BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+          cluster.getNamesystem(0).getBlockManager());
+      verifyTotalBlocksMetrics(0L, 0L, activeNn.getTotalBlocks());
+      verifyTotalBlocksMetrics(0L, 0L, standbyNn.getTotalBlocks());
     } finally {
       if (fs != null) {
         try {
@@ -1096,9 +1106,16 @@ public void testTotalBlocksMetrics() throws Exception {
           throw e;
         }
       }
-      if (namesystem != null) {
+      if (activeNn != null) {
+        try {
+          activeNn.close();
+        } catch (Exception e) {
+          throw e;
+        }
+      }
+      if (standbyNn != null) {
         try {
-          namesystem.close();
+          standbyNn.close();
         } catch (Exception e) {
           throw e;
         }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index ada7c82150a12..5573b1fa107fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -19,11 +19,16 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminBackoffMonitor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeAdminMonitorInterface;
 import org.junit.Test;
 import org.junit.Before;
 import org.junit.After;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY;
 import static org.junit.Assert.*;
 
@@ -34,16 +39,21 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode;
+import org.apache.hadoop.hdfs.protocol.BlockType;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.SlowPeerTracker;
 import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfyManager;
+import org.apache.hadoop.hdfs.server.protocol.OutlierMetrics;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.test.GenericTestUtils;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
@@ -51,6 +61,11 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
 import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT;
 
 public class TestNameNodeReconfigure {
@@ -302,6 +317,8 @@ public void testReconfigureStoragePolicySatisfierEnabled()
         StoragePolicySatisfierMode.EXTERNAL.toString(),
         nameNode.getConf().get(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
             DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT));
+    assertNotNull("SPS Manager should be created",
+        nameNode.getNamesystem().getBlockManager().getSPSManager());
   }
 
   /**
@@ -317,6 +334,8 @@ public void testSatisfyStoragePolicyAfterSatisfierDisabled()
         StoragePolicySatisfierMode.NONE.toString());
     verifySPSEnabled(nameNode, DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
         StoragePolicySatisfierMode.NONE, false);
+    assertNull("SPS Manager should be null",
+        nameNode.getNamesystem().getBlockManager().getSPSManager());
 
     Path filePath = new Path("/testSPS");
     DistributedFileSystem fileSystem = cluster.getFileSystem();
@@ -340,7 +359,7 @@ void verifySPSEnabled(final NameNode nameNode, String property,
             .getNamesystem().getBlockManager().getSPSManager();
     boolean isSPSRunning = spsMgr != null ? spsMgr.isSatisfierRunning()
         : false;
-    assertEquals(property + " has wrong value", isSPSRunning, isSPSRunning);
+    assertEquals(property + " has wrong value", isSatisfierRunning, isSPSRunning);
     String actual = nameNode.getConf().get(property,
         DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT);
     assertEquals(property + " has wrong value", expected,
@@ -394,6 +413,245 @@ public void testEnableParallelLoadAfterReconfigured()
     assertEquals(true, FSImageFormatProtobuf.getEnableParallelLoad());
   }
 
+  @Test
+  public void testEnableSlowNodesParametersAfterReconfigured()
+      throws ReconfigurationException {
+    final NameNode nameNode = cluster.getNameNode();
+    final BlockManager blockManager = nameNode.namesystem.getBlockManager();
+    final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
+
+    // By default, avoidSlowDataNodesForRead is false.
+    assertEquals(false, datanodeManager.getEnableAvoidSlowDataNodesForRead());
+
+    nameNode.reconfigureProperty(
+        DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, Boolean.toString(true));
+
+    // After reconfigured, avoidSlowDataNodesForRead is true.
+    assertEquals(true, datanodeManager.getEnableAvoidSlowDataNodesForRead());
+
+    // By default, excludeSlowNodesEnabled is false.
+    assertEquals(false, blockManager.
+        getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
+    assertEquals(false, blockManager.
+        getExcludeSlowNodesEnabled(BlockType.STRIPED));
+
+    nameNode.reconfigureProperty(
+        DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, Boolean.toString(true));
+
+    // After reconfigured, excludeSlowNodesEnabled is true.
+    assertEquals(true, blockManager.
+        getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS));
+    assertEquals(true, blockManager.
+        getExcludeSlowNodesEnabled(BlockType.STRIPED));
+  }
+
+  @Test
+  public void testReconfigureMaxSlowpeerCollectNodes()
+      throws ReconfigurationException {
+    final NameNode nameNode = cluster.getNameNode();
+    final DatanodeManager datanodeManager = nameNode.namesystem
+        .getBlockManager().getDatanodeManager();
+
+    // By default, DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY is 5.
+    assertEquals(5, datanodeManager.getMaxSlowpeerCollectNodes());
+
+    // Reconfigure.
+    nameNode.reconfigureProperty(
+        DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, Integer.toString(10));
+
+    // Assert DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY is 10.
+    assertEquals(10, datanodeManager.getMaxSlowpeerCollectNodes());
+  }
+
+  @Test
+  public void testBlockInvalidateLimit() throws ReconfigurationException {
+    final NameNode nameNode = cluster.getNameNode();
+    final DatanodeManager datanodeManager = nameNode.namesystem
+        .getBlockManager().getDatanodeManager();
+
+    assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY + " is not correctly set",
+        customizedBlockInvalidateLimit, datanodeManager.getBlockInvalidateLimit());
+
+    try {
+      nameNode.reconfigureProperty(DFS_BLOCK_INVALIDATE_LIMIT_KEY, "non-numeric");
+      fail("Should not reach here");
+    } catch (ReconfigurationException e) {
+      assertEquals(
+          "Could not change property dfs.block.invalidate.limit from '500' to 'non-numeric'",
+          e.getMessage());
+    }
+
+    nameNode.reconfigureProperty(DFS_BLOCK_INVALIDATE_LIMIT_KEY, "2500");
+
+    assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY + " is not honored after reconfiguration", 2500,
+        datanodeManager.getBlockInvalidateLimit());
+
+    nameNode.reconfigureProperty(DFS_HEARTBEAT_INTERVAL_KEY, "500");
+
+    // 20 * 500 (10000) > 2500
+    // Hence, invalid block limit should be reset to 10000
+    assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY + " is not reconfigured correctly", 10000,
+        datanodeManager.getBlockInvalidateLimit());
+  }
+
+  @Test
+  public void testSlowPeerTrackerEnabled() throws Exception {
+    final NameNode nameNode = cluster.getNameNode();
+    final DatanodeManager datanodeManager = nameNode.namesystem.getBlockManager()
+        .getDatanodeManager();
+
+    assertFalse("SlowNode tracker is already enabled. It should be disabled by default",
+        datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
+
+    try {
+      nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "non-boolean");
+      fail("should not reach here");
+    } catch (ReconfigurationException e) {
+      assertEquals(
+          "Could not change property dfs.datanode.peer.stats.enabled from 'false' to 'non-boolean'",
+          e.getMessage());
+    }
+
+    nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "True");
+    assertTrue("SlowNode tracker is still disabled. Reconfiguration could not be successful",
+        datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
+
+    nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, null);
+    assertFalse("SlowNode tracker is still enabled. Reconfiguration could not be successful",
+        datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
+
+  }
+
+  @Test
+  public void testSlowPeerMaxNodesToReportReconf() throws Exception {
+    final NameNode nameNode = cluster.getNameNode();
+    final DatanodeManager datanodeManager = nameNode.namesystem.getBlockManager()
+        .getDatanodeManager();
+    nameNode.reconfigurePropertyImpl(DFS_DATANODE_PEER_STATS_ENABLED_KEY, "true");
+    assertTrue("SlowNode tracker is still disabled. Reconfiguration could not be successful",
+        datanodeManager.getSlowPeerTracker().isSlowPeerTrackerEnabled());
+
+    SlowPeerTracker tracker = datanodeManager.getSlowPeerTracker();
+
+    OutlierMetrics outlierMetrics1 = new OutlierMetrics(0.0, 0.0, 0.0, 1.1);
+    tracker.addReport("node1", "node70", outlierMetrics1);
+    OutlierMetrics outlierMetrics2 = new OutlierMetrics(0.0, 0.0, 0.0, 1.23);
+    tracker.addReport("node2", "node71", outlierMetrics2);
+    OutlierMetrics outlierMetrics3 = new OutlierMetrics(0.0, 0.0, 0.0, 2.13);
+    tracker.addReport("node3", "node72", outlierMetrics3);
+    OutlierMetrics outlierMetrics4 = new OutlierMetrics(0.0, 0.0, 0.0, 1.244);
+    tracker.addReport("node4", "node73", outlierMetrics4);
+    OutlierMetrics outlierMetrics5 = new OutlierMetrics(0.0, 0.0, 0.0, 0.2);
+    tracker.addReport("node5", "node74", outlierMetrics4);
+    OutlierMetrics outlierMetrics6 = new OutlierMetrics(0.0, 0.0, 0.0, 1.244);
+    tracker.addReport("node6", "node75", outlierMetrics4);
+
+    String jsonReport = tracker.getJson();
+    LOG.info("Retrieved slow peer json report: {}", jsonReport);
+
+    List<Boolean> containReport = validatePeerReport(jsonReport);
+    assertEquals(1, containReport.stream().filter(reportVal -> !reportVal).count());
+
+    nameNode.reconfigurePropertyImpl(DFS_DATANODE_MAX_NODES_TO_REPORT_KEY, "2");
+    jsonReport = tracker.getJson();
+    LOG.info("Retrieved slow peer json report: {}", jsonReport);
+
+    containReport = validatePeerReport(jsonReport);
+    assertEquals(4, containReport.stream().filter(reportVal -> !reportVal).count());
+  }
+
+  private List<Boolean> validatePeerReport(String jsonReport) {
+    List<Boolean> containReport = new ArrayList<>();
+    containReport.add(jsonReport.contains("node1"));
+    containReport.add(jsonReport.contains("node2"));
+    containReport.add(jsonReport.contains("node3"));
+    containReport.add(jsonReport.contains("node4"));
+    containReport.add(jsonReport.contains("node5"));
+    containReport.add(jsonReport.contains("node6"));
+    return containReport;
+  }
+
+  @Test
+  public void testReconfigureDecommissionBackoffMonitorParameters()
+      throws ReconfigurationException, IOException {
+    Configuration conf = new HdfsConfiguration();
+    conf.setClass(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MONITOR_CLASS,
+        DatanodeAdminBackoffMonitor.class, DatanodeAdminMonitorInterface.class);
+    int defaultPendingRepLimit = 1000;
+    conf.setInt(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, defaultPendingRepLimit);
+    int defaultBlocksPerLock = 1000;
+    conf.setInt(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK,
+        defaultBlocksPerLock);
+
+    try (MiniDFSCluster newCluster = new MiniDFSCluster.Builder(conf).build()) {
+      newCluster.waitActive();
+      final NameNode nameNode = newCluster.getNameNode();
+      final DatanodeManager datanodeManager = nameNode.namesystem
+          .getBlockManager().getDatanodeManager();
+
+      // verify defaultPendingRepLimit.
+      assertEquals(datanodeManager.getDatanodeAdminManager().getPendingRepLimit(),
+          defaultPendingRepLimit);
+
+      // try invalid pendingRepLimit.
+      try {
+        nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
+            "non-numeric");
+        fail("Should not reach here");
+      } catch (ReconfigurationException e) {
+        assertEquals("Could not change property " +
+            "dfs.namenode.decommission.backoff.monitor.pending.limit from '" +
+            defaultPendingRepLimit + "' to 'non-numeric'", e.getMessage());
+      }
+
+      try {
+        nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
+            "-1");
+        fail("Should not reach here");
+      } catch (ReconfigurationException e) {
+        assertEquals("Could not change property " +
+            "dfs.namenode.decommission.backoff.monitor.pending.limit from '" +
+            defaultPendingRepLimit + "' to '-1'", e.getMessage());
+      }
+
+      // try correct pendingRepLimit.
+      nameNode.reconfigureProperty(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT,
+          "20000");
+      assertEquals(datanodeManager.getDatanodeAdminManager().getPendingRepLimit(), 20000);
+
+      // verify defaultBlocksPerLock.
+      assertEquals(datanodeManager.getDatanodeAdminManager().getBlocksPerLock(),
+          defaultBlocksPerLock);
+
+      // try invalid blocksPerLock.
+      try {
+        nameNode.reconfigureProperty(
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK,
+            "non-numeric");
+        fail("Should not reach here");
+      } catch (ReconfigurationException e) {
+        assertEquals("Could not change property " +
+            "dfs.namenode.decommission.backoff.monitor.pending.blocks.per.lock from '" +
+            defaultBlocksPerLock + "' to 'non-numeric'", e.getMessage());
+      }
+
+      try {
+        nameNode.reconfigureProperty(
+            DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, "-1");
+        fail("Should not reach here");
+      } catch (ReconfigurationException e) {
+        assertEquals("Could not change property " +
+            "dfs.namenode.decommission.backoff.monitor.pending.blocks.per.lock from '" +
+            defaultBlocksPerLock + "' to '-1'", e.getMessage());
+      }
+
+      // try correct blocksPerLock.
+      nameNode.reconfigureProperty(
+          DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, "10000");
+      assertEquals(datanodeManager.getDatanodeAdminManager().getBlocksPerLock(), 10000);
+    }
+  }
+
   @After
   public void shutDown() throws IOException {
     if (cluster != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
index 51389c8336f16..8a701a31b9fb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
@@ -49,7 +50,6 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
-import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.util.StringUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -299,7 +299,7 @@ public long getLastValidTxId() {
 
     @Override
     public Set<Long> getValidTxIds() {
-      return Sets.newHashSet(0L);
+      return new HashSet<>(Arrays.asList(0L));
     } 
     
     public int getMaxOpSize() {
@@ -341,7 +341,7 @@ public long getLastValidTxId() {
 
     @Override
     public Set<Long> getValidTxIds() {
-      return Sets.newHashSet(0L);
+      return new HashSet<>(Arrays.asList(0L));
     } 
   }
 
@@ -387,7 +387,7 @@ public long getLastValidTxId() {
 
     @Override
     public Set<Long> getValidTxIds() {
-      return Sets.newHashSet(1L , 2L, 3L, 5L, 6L, 7L, 8L, 9L, 10L);
+      return new HashSet<>(Arrays.asList(1L, 2L, 3L, 5L, 6L, 7L, 8L, 9L, 10L));
     }
   }
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
index ada93e84f0ead..2960a7ee6d4bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,15 +24,33 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.PrivilegedExceptionAction;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
+import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
+import org.junit.jupiter.api.Timeout;
 
 public class TestNameNodeRpcServer {
 
@@ -59,5 +77,148 @@ public void testNamenodeRpcBindAny() throws IOException {
       conf.unset(DFS_NAMENODE_RPC_BIND_HOST_KEY);
     }
   }
+
+  /**
+   * Get the preferred DataNode location for the first block of the
+   * given file.
+   * @param fs The file system to use
+   * @param p The path to use
+   * @return the preferred host to get the data
+   */
+  private static String getPreferredLocation(DistributedFileSystem fs,
+                                             Path p) throws IOException{
+    // Use getLocatedBlocks because it is the basis for HDFS open,
+    // but provides visibility into which host will be used.
+    LocatedBlocks blocks = fs.getClient()
+        .getLocatedBlocks(p.toUri().getPath(), 0);
+    return blocks.get(0).getLocations()[0].getHostName();
+  }
+
+  // Because of the randomness of the NN assigning DN, we run multiple
+  // trials. 1/3^20=3e-10, so that should be good enough.
+  static final int ITERATIONS_TO_USE = 20;
+
+  @Test
+  @Timeout(30000)
+  public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception {
+    // Make 3 nodes & racks so that we have a decent shot of detecting when
+    // our change overrides the random choice of datanode.
+    Configuration conf = new HdfsConfiguration();
+    conf.set(DFS_NAMENODE_IP_PROXY_USERS, "fake_joe");
+    final CallerContext original = CallerContext.getCurrent();
+
+    MiniQJMHACluster qjmhaCluster = null;
+    try {
+      String baseDir = GenericTestUtils.getRandomizedTempPath();
+      MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
+      builder.getDfsBuilder().numDataNodes(3);
+      qjmhaCluster = builder.baseDir(baseDir).build();
+      MiniDFSCluster dfsCluster = qjmhaCluster.getDfsCluster();
+      dfsCluster.waitActive();
+      dfsCluster.transitionToActive(0);
+
+      // Set the caller context to set the ip address
+      CallerContext.setCurrent(
+          new CallerContext.Builder("test", conf)
+              .build());
+
+      dfsCluster.getFileSystem(0).setPermission(
+          new Path("/"), FsPermission.getDirDefault());
+
+      // Run as fake joe to authorize the test
+      UserGroupInformation joe =
+          UserGroupInformation.createUserForTesting("fake_joe",
+              new String[]{"fake_group"});
+
+      FileSystem joeFs = joe.doAs((PrivilegedExceptionAction<FileSystem>) () ->
+          FileSystem.get(dfsCluster.getURI(0), conf));
+
+      Path testPath = new Path("/foo");
+      // Write a sample file
+      FSDataOutputStream stream = joeFs.create(testPath);
+      stream.write("Hello world!\n".getBytes(StandardCharsets.UTF_8));
+      stream.close();
+
+      qjmhaCluster.getDfsCluster().transitionToStandby(0);
+      qjmhaCluster.getDfsCluster().transitionToActive(1);
+
+      DistributedFileSystem nn1 = dfsCluster.getFileSystem(1);
+      assertNotNull(nn1.getFileStatus(testPath));
+    } finally {
+      CallerContext.setCurrent(original);
+      if (qjmhaCluster != null) {
+        try {
+          qjmhaCluster.shutdown();
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+      }
+      // Reset the config
+      conf.unset(DFS_NAMENODE_IP_PROXY_USERS);
+    }
+  }
+
+  /**
+   * A test to make sure that if an authorized user adds "clientIp:" to their
+   * caller context, it will be used to make locality decisions on the NN.
+   */
+  @Test
+  public void testNamenodeRpcClientIpProxy() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+
+    conf.set(DFS_NAMENODE_IP_PROXY_USERS, "fake_joe");
+    // Make 3 nodes & racks so that we have a decent shot of detecting when
+    // our change overrides the random choice of datanode.
+    final String[] racks = new String[]{"/rack1", "/rack2", "/rack3"};
+    final String[] hosts = new String[]{"node1", "node2", "node3"};
+    MiniDFSCluster cluster = null;
+    final CallerContext original = CallerContext.getCurrent();
+
+    try {
+      cluster = new MiniDFSCluster.Builder(conf)
+          .racks(racks).hosts(hosts).numDataNodes(hosts.length)
+          .build();
+      cluster.waitActive();
+      DistributedFileSystem fs = cluster.getFileSystem();
+      // Write a sample file
+      final Path fooName = fs.makeQualified(new Path("/foo"));
+      FSDataOutputStream stream = fs.create(fooName);
+      stream.write("Hello world!\n".getBytes(StandardCharsets.UTF_8));
+      stream.close();
+      // Set the caller context to set the ip address
+      CallerContext.setCurrent(
+          new CallerContext.Builder("test", conf)
+              .append(CallerContext.CLIENT_IP_STR, hosts[0])
+              .build());
+      // Should get a random mix of DataNodes since we aren't joe.
+      for (int trial = 0; trial < ITERATIONS_TO_USE; ++trial) {
+        String host = getPreferredLocation(fs, fooName);
+        if (!hosts[0].equals(host)) {
+          // found some other host, so things are good
+          break;
+        } else if (trial == ITERATIONS_TO_USE - 1) {
+          assertNotEquals("Failed to get non-node1", hosts[0], host);
+        }
+      }
+      // Run as fake joe to authorize the test
+      UserGroupInformation joe =
+          UserGroupInformation.createUserForTesting("fake_joe",
+              new String[]{"fake_group"});
+      DistributedFileSystem joeFs =
+          (DistributedFileSystem) DFSTestUtil.getFileSystemAs(joe, conf);
+      // As joe, we should get all node1.
+      for (int trial = 0; trial < ITERATIONS_TO_USE; ++trial) {
+        String host = getPreferredLocation(joeFs, fooName);
+        assertEquals("Trial " + trial + " failed", hosts[0], host);
+      }
+    } finally {
+      CallerContext.setCurrent(original);
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+      // Reset the config
+      conf.unset(DFS_NAMENODE_IP_PROXY_USERS);
+    }
+  }
 }
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
index a32e2188e8822..50740bd06e831 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServerMethods.java
@@ -20,9 +20,14 @@
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -31,6 +36,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+
 public class TestNameNodeRpcServerMethods {
   private static NamenodeProtocols nnRpc;
   private static Configuration conf;
@@ -83,4 +90,27 @@ public void testDeleteSnapshotWhenSnapshotNameIsEmpty() throws Exception {
 
   }
 
+  @Test
+  public void testGetDatanodeStorageReportWithNumBLocksNotZero() throws Exception {
+    int buffSize = 1024;
+    long blockSize = 1024 * 1024;
+    String file = "/testFile";
+    DistributedFileSystem dfs = cluster.getFileSystem();
+    FSDataOutputStream outputStream = dfs.create(
+        new Path(file), true, buffSize, (short)1, blockSize);
+    byte[] outBuffer = new byte[buffSize];
+    for (int i = 0; i < buffSize; i++) {
+      outBuffer[i] = (byte) (i & 0x00ff);
+    }
+    outputStream.write(outBuffer);
+    outputStream.close();
+
+    int numBlocks = 0;
+    DatanodeStorageReport[] reports
+        = nnRpc.getDatanodeStorageReport(HdfsConstants.DatanodeReportType.ALL);
+    for (DatanodeStorageReport r : reports) {
+      numBlocks += r.getDatanodeInfo().getNumBlocks();
+    }
+    assertEquals(1, numBlocks);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
index 5bcb3a571a75c..0469b91988451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java
@@ -192,7 +192,7 @@ private void testStorageTypes(StorageType[][] storageTypes,
    * Types.
    * @throws IOException
    */
-  @Test(timeout=60000)
+  @Test(timeout=120000)
   public void testTargetStorageTypes() throws ReconfigurationException,
       InterruptedException, TimeoutException, IOException {
     // DISK and not anything else.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index fc307bf84d86f..0a7874fc7a808 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo;
 
 import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -84,6 +86,7 @@ private void initConf(Configuration conf) {
     // chooseUnderReplicatedBlocks at once.
     conf.setInt(
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 5);
+
   }
 
   @Test
@@ -430,4 +433,86 @@ public void testReconstructionWork() throws Exception {
       dfsCluster.shutdown();
     }
   }
+  private byte[] writeStripedFile(DistributedFileSystem fs, Path ecFile,
+                                  int writeBytes) throws Exception {
+    byte[] bytes = StripedFileTestUtil.generateBytes(writeBytes);
+    DFSTestUtil.writeFile(fs, ecFile, new String(bytes));
+    StripedFileTestUtil.waitBlockGroupsReported(fs, ecFile.toString());
+
+    return bytes;
+  }
+  @Test
+  public void testReconstrutionWithBusyBlock1() throws Exception {
+    //When the index of busy block is smaller than the missing block
+    //[0(busy),1(busy),3,4,5,6,7,8]
+    int busyNodeIndex1 = 0;
+    int busyNodeIndex2 = 1;
+    int deadNodeIndex = 2;
+    final Path ecDir = new Path(GenericTestUtils.getRandomizedTempPath());
+    final Path ecFile = new Path(ecDir, "testReconstrutionWithBusyBlock1");
+    int writeBytes = cellSize * dataBlocks;
+    HdfsConfiguration conf = new HdfsConfiguration();
+    initConf(conf);
+    conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, false);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
+            2000);
+    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+    conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
+            1000);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
+            4);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
+            1);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 5)
+            .build();
+    cluster.waitActive();
+    DistributedFileSystem dfs = cluster.getFileSystem(0);
+    dfs.enableErasureCodingPolicy(
+            StripedFileTestUtil.getDefaultECPolicy().getName());
+    dfs.mkdirs(ecDir);
+    dfs.setErasureCodingPolicy(ecDir,
+            StripedFileTestUtil.getDefaultECPolicy().getName());
+    byte[] originBytesArray = writeStripedFile(dfs, ecFile, writeBytes);
+    List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile))
+            .getAllBlocks();
+    LocatedStripedBlock lsb = (LocatedStripedBlock) lbs.get(0);
+    DatanodeInfo[] dnList = lsb.getLocations();
+    BlockManager bm = cluster.getNamesystem().getBlockManager();
+    BlockInfoStriped blockInfo =
+            (BlockInfoStriped) bm.getStoredBlock(
+                    new Block(lsb.getBlock().getBlockId()));
+
+    //1.Make nodes busy
+    DatanodeDescriptor busyNode = bm.getDatanodeManager()
+            .getDatanode(dnList[busyNodeIndex1].getDatanodeUuid());
+    for (int j = 0; j < maxReplicationStreams; j++) {
+      busyNode.incrementPendingReplicationWithoutTargets();
+    }
+    DatanodeDescriptor busyNode2 = bm.getDatanodeManager()
+            .getDatanode(dnList[busyNodeIndex2].getDatanodeUuid());
+    for (int j = 0; j < maxReplicationStreams; j++) {
+      busyNode2.incrementPendingReplicationWithoutTargets();
+    }
+
+    //2.Make a node missing
+    DataNode dn = cluster.getDataNode(dnList[deadNodeIndex].getIpcPort());
+    cluster.stopDataNode(dnList[deadNodeIndex].getXferAddr());
+    cluster.setDataNodeDead(dn.getDatanodeId());
+
+    //3.Whether there is excess replicas or not during the recovery?
+    assertEquals(8, bm.countNodes(blockInfo).liveReplicas());
+
+    GenericTestUtils.waitFor(
+        () -> {
+          return bm.countNodes(blockInfo).liveReplicas() == 9||
+              bm.countNodes(blockInfo).excessReplicas() >= 1||
+              bm.countNodes(blockInfo).redundantInternalBlocks() >= 1;
+        },
+        10, 100000);
+
+    assertEquals(0, bm.countNodes(blockInfo).excessReplicas());
+    assertEquals(9, bm.countNodes(blockInfo).liveReplicas());
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java
index 8dc81f8c1a21d..8336a432b9a41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java
@@ -49,6 +49,9 @@ public void setup() throws IOException {
     config.setInt(
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
         12);
+    config.setInt(
+        DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
+        300);
 
     cluster = new MiniDFSCluster.Builder(config)
         .nnTopology(MiniDFSNNTopology.simpleSingleNN(0, 0))
@@ -72,6 +75,7 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException {
     assertEquals(8, bm.getMaxReplicationStreams());
     assertEquals(10, bm.getReplicationStreamsHardLimit());
     assertEquals(12, bm.getBlocksReplWorkMultiplier());
+    assertEquals(300, bm.getReconstructionPendingTimeout());
 
     cluster.getNameNode().reconfigurePropertyImpl(
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, "20");
@@ -81,10 +85,14 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException {
     cluster.getNameNode().reconfigurePropertyImpl(
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
         "24");
+    cluster.getNameNode().reconfigurePropertyImpl(
+        DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
+        "180");
 
     assertEquals(20, bm.getMaxReplicationStreams());
     assertEquals(22, bm.getReplicationStreamsHardLimit());
     assertEquals(24, bm.getBlocksReplWorkMultiplier());
+    assertEquals(180, bm.getReconstructionPendingTimeout());
   }
 
   /**
@@ -96,7 +104,8 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
     String[] keys = new String[]{
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY,
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY,
-        DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
+        DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
+        DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY
     };
 
     // Ensure we cannot set any of the parameters negative
@@ -112,6 +121,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
     assertEquals(8, bm.getMaxReplicationStreams());
     assertEquals(10, bm.getReplicationStreamsHardLimit());
     assertEquals(12, bm.getBlocksReplWorkMultiplier());
+    assertEquals(300, bm.getReconstructionPendingTimeout());
 
     for (String key : keys) {
       ReconfigurationException e =
@@ -126,6 +136,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
     assertEquals(8, bm.getMaxReplicationStreams());
     assertEquals(10, bm.getReplicationStreamsHardLimit());
     assertEquals(12, bm.getBlocksReplWorkMultiplier());
+    assertEquals(300, bm.getReconstructionPendingTimeout());
 
     // Ensure none of the parameters can be set to a string value
     for (String key : keys) {
@@ -139,5 +150,6 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
     assertEquals(8, bm.getMaxReplicationStreams());
     assertEquals(10, bm.getReplicationStreamsHardLimit());
     assertEquals(12, bm.getBlocksReplWorkMultiplier());
+    assertEquals(300, bm.getReconstructionPendingTimeout());
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java
index 9473cb3ed950d..1d3187dffe0b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecondaryWebUi.java
@@ -21,7 +21,6 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -31,7 +30,6 @@
 import javax.management.*;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.net.URL;
 
 public class TestSecondaryWebUi {
   
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNode.java
index c90a91c67d4c0..e67e5cf720c4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNode.java
@@ -19,7 +19,6 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java
index d36b9c997e2b9..60601a6ab9133 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNodeWithExternalKdc.java
@@ -19,7 +19,6 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestUpgradeDomainBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestUpgradeDomainBlockPlacementPolicy.java
index 0421941f3f4c5..abd26d4f5a4f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestUpgradeDomainBlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestUpgradeDomainBlockPlacementPolicy.java
@@ -69,11 +69,11 @@ public class TestUpgradeDomainBlockPlacementPolicy {
   static final Set<DatanodeID> expectedDatanodeIDs = new HashSet<>();
   private MiniDFSCluster cluster = null;
   private HostsFileWriter hostsFileWriter = new HostsFileWriter();
+  private Configuration conf = new HdfsConfiguration();
 
   @Before
   public void setup() throws IOException {
     StaticMapping.resetMap();
-    Configuration conf = new HdfsConfiguration();
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
     conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE / 2);
     conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
@@ -130,7 +130,7 @@ private void refreshDatanodeAdminProperties()
     datanodes[0].setAdminState(DatanodeInfo.AdminStates.DECOMMISSIONED);
     datanodes[5].setAdminState(DatanodeInfo.AdminStates.DECOMMISSIONED);
     hostsFileWriter.initIncludeHosts(datanodes);
-    cluster.getFileSystem().refreshNodes();
+    cluster.getNamesystem(0).getBlockManager().getDatanodeManager().refreshNodes(conf);
 
     expectedDatanodeIDs.clear();
     expectedDatanodeIDs.add(cluster.getDataNodes().get(2).getDatanodeId());
@@ -169,7 +169,7 @@ private void refreshDatanodeAdminProperties2()
     datanodes[2].setAdminState(DatanodeInfo.AdminStates.DECOMMISSIONED);
     datanodes[3].setAdminState(DatanodeInfo.AdminStates.DECOMMISSIONED);
     hostsFileWriter.initIncludeHosts(datanodes);
-    cluster.getFileSystem().refreshNodes();
+    cluster.getNamesystem(0).getBlockManager().getDatanodeManager().refreshNodes(conf);
 
     expectedDatanodeIDs.clear();
     expectedDatanodeIDs.add(cluster.getDataNodes().get(0).getDatanodeId());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
index 307fe04618ba6..ca772e18167f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
@@ -17,13 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSUtil.createUri;
-
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Proxy;
@@ -36,11 +29,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.LongAccumulator;
+import java.util.function.Supplier;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.ClientGSIContext;
@@ -49,6 +43,7 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Failover;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -59,47 +54,52 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
 
-import java.util.function.Supplier;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
+import static org.apache.hadoop.hdfs.DFSUtil.createUri;
 
 /**
  * Static utility functions useful for testing HA.
  */
 public abstract class HATestUtil {
   private static final Logger LOG = LoggerFactory.getLogger(HATestUtil.class);
-  
+
   private static final String LOGICAL_HOSTNAME = "ha-nn-uri-%d";
-  
+
   /**
    * Trigger an edits log roll on the active and then wait for the standby to
    * catch up to all the edits done by the active. This method will check
    * repeatedly for up to NN_LAG_TIMEOUT milliseconds, and then fail throwing
    * {@link CouldNotCatchUpException}
-   * 
+   *
    * @param active active NN
    * @param standby standby NN which should catch up to active
    * @throws IOException if an error occurs rolling the edit log
    * @throws CouldNotCatchUpException if the standby doesn't catch up to the
    *         active in NN_LAG_TIMEOUT milliseconds
    */
-  public static void waitForStandbyToCatchUp(NameNode active,
-      NameNode standby) throws InterruptedException, IOException, CouldNotCatchUpException {
-    long activeTxId = active.getNamesystem().getFSImage().getEditLog()
-      .getLastWrittenTxId();
+  public static void waitForStandbyToCatchUp(NameNode active, NameNode standby)
+      throws InterruptedException, IOException, CouldNotCatchUpException {
+    long activeTxId =
+        active.getNamesystem().getFSImage().getEditLog().getLastWrittenTxId();
 
     active.getRpcServer().rollEditLog();
 
     long start = Time.now();
     while (Time.now() - start < TestEditLogTailer.NN_LAG_TIMEOUT) {
-      long nn2HighestTxId = standby.getNamesystem().getFSImage()
-        .getLastAppliedTxId();
+      long nn2HighestTxId =
+          standby.getNamesystem().getFSImage().getLastAppliedTxId();
       if (nn2HighestTxId >= activeTxId) {
         return;
       }
       Thread.sleep(TestEditLogTailer.SLEEP_TIME);
     }
-    throw new CouldNotCatchUpException("Standby did not catch up to txid " +
-        activeTxId + " (currently at " +
-        standby.getNamesystem().getFSImage().getLastAppliedTxId() + ")");
+    throw new CouldNotCatchUpException(
+        "Standby did not catch up to txid " + activeTxId + " (currently at "
+            + standby.getNamesystem().getFSImage().getLastAppliedTxId() + ")");
   }
 
   /**
@@ -119,7 +119,7 @@ public Boolean get() {
         return true;
       }
     }, 1000, 10000);
-    
+
   }
 
   /**
@@ -144,16 +144,18 @@ public CouldNotCatchUpException(String message) {
       super(message);
     }
   }
-  
-  /** Gets the filesystem instance by setting the failover configurations */
+
+  /**
+   * Gets the filesystem instance by setting the failover configurations.
+   */
   public static DistributedFileSystem configureFailoverFs(
       MiniDFSCluster cluster, Configuration conf)
       throws IOException, URISyntaxException {
     return configureFailoverFs(cluster, conf, 0);
   }
 
-  /** 
-   * Gets the filesystem instance by setting the failover configurations
+  /**
+   * Gets the filesystem instance by setting the failover configurations.
    * @param cluster the single process DFS cluster
    * @param conf cluster configuration
    * @param nsIndex namespace index starting with zero
@@ -164,13 +166,13 @@ public static DistributedFileSystem configureFailoverFs(
       int nsIndex) throws IOException, URISyntaxException {
     conf = new Configuration(conf);
     String logicalName = getLogicalHostname(cluster);
-    setFailoverConfigurations(cluster, conf, logicalName, nsIndex);
+    setFailoverConfigurations(cluster, conf, logicalName, null, nsIndex);
     FileSystem fs = FileSystem.get(new URI("hdfs://" + logicalName), conf);
     return (DistributedFileSystem)fs;
   }
 
   public static <P extends ObserverReadProxyProvider<?>>
-  DistributedFileSystem configureObserverReadFs(
+      DistributedFileSystem configureObserverReadFs(
       MiniDFSCluster cluster, Configuration conf,
       Class<P> classFPP, boolean isObserverReadEnabled)
           throws IOException, URISyntaxException {
@@ -246,8 +248,8 @@ public static MiniQJMHACluster setUpObserverCluster(
     return qjmhaCluster;
   }
 
-  public static <P extends FailoverProxyProvider<?>>
-  void setupHAConfiguration(MiniDFSCluster cluster,
+  public static <P extends FailoverProxyProvider<?>> void
+      setupHAConfiguration(MiniDFSCluster cluster,
       Configuration conf, int nsIndex, Class<P> classFPP) {
     MiniDFSCluster.NameNodeInfo[] nns = cluster.getNameNodeInfos(nsIndex);
     List<String> nnAddresses = new ArrayList<String>();
@@ -264,18 +266,23 @@ public static void setFailoverConfigurations(MiniDFSCluster cluster,
       Configuration conf) {
     setFailoverConfigurations(cluster, conf, getLogicalHostname(cluster));
   }
-  
+
   /** Sets the required configurations for performing failover of default namespace. */
   public static void setFailoverConfigurations(MiniDFSCluster cluster,
       Configuration conf, String logicalName) {
-    setFailoverConfigurations(cluster, conf, logicalName, 0);
+    setFailoverConfigurations(cluster, conf, logicalName, null, 0);
   }
-  
+
   /** Sets the required configurations for performing failover.  */
   public static void setFailoverConfigurations(MiniDFSCluster cluster,
-      Configuration conf, String logicalName, int nsIndex) {
-    setFailoverConfigurations(cluster, conf, logicalName, nsIndex,
-        ConfiguredFailoverProxyProvider.class);
+      Configuration conf, String logicalName, String proxyProvider,
+      int nsIndex) {
+    MiniDFSCluster.NameNodeInfo[] nns = cluster.getNameNodeInfos(nsIndex);
+    List<InetSocketAddress> nnAddresses = new ArrayList<InetSocketAddress>(3);
+    for (MiniDFSCluster.NameNodeInfo nn : nns) {
+      nnAddresses.add(nn.nameNode.getNameNodeAddress());
+    }
+    setFailoverConfigurations(conf, logicalName, proxyProvider, nnAddresses);
   }
 
   /** Sets the required configurations for performing failover.  */
@@ -290,19 +297,56 @@ public static void setFailoverConfigurations(MiniDFSCluster cluster,
     setFailoverConfigurations(conf, logicalName, nnAddresses, classFPP);
   }
 
-  public static void setFailoverConfigurations(Configuration conf, String logicalName,
-      InetSocketAddress ... nnAddresses){
-    setFailoverConfigurations(conf, logicalName, Arrays.asList(nnAddresses),
-        ConfiguredFailoverProxyProvider.class);
+  public static void setFailoverConfigurations(Configuration conf,
+      String logicalName, String proxyProvider,
+      InetSocketAddress... nnAddresses) {
+    setFailoverConfigurations(conf, logicalName, proxyProvider,
+        Arrays.asList(nnAddresses));
+  }
+
+  /**
+   * Sets the required configurations for performing failover.
+   */
+  public static void setFailoverConfigurations(
+      Configuration conf, String logicalName,
+      String proxyProvider, List<InetSocketAddress> nnAddresses) {
+    final List<String> addresses = new ArrayList<>();
+    nnAddresses.forEach(addr ->
+        addresses.add("hdfs://" + addr.getHostName() + ":" + addr.getPort()));
+    setFailoverConfigurations(conf, logicalName, proxyProvider, addresses);
+  }
+
+  public static void setFailoverConfigurations(
+      Configuration conf, String logicalName,
+      String proxyProvider, Iterable<String> nnAddresses) {
+    List<String> nnids = new ArrayList<String>();
+    int i = 0;
+    for (String address : nnAddresses) {
+      String nnId = "nn" + (i + 1);
+      nnids.add(nnId);
+      conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, logicalName, nnId), address);
+      i++;
+    }
+    conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
+    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName),
+        Joiner.on(',').join(nnids));
+    if (proxyProvider == null) {
+      conf.set(Failover.PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
+          ConfiguredFailoverProxyProvider.class.getName());
+    } else {
+      conf.set(Failover.PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
+          proxyProvider);
+    }
+    conf.set("fs.defaultFS", "hdfs://" + logicalName);
   }
 
   /**
-   * Sets the required configurations for performing failover
+   * Sets the required configurations for performing failover.
    */
   public static <P extends FailoverProxyProvider<?>> void
       setFailoverConfigurations(Configuration conf, String logicalName,
       List<InetSocketAddress> nnAddresses, Class<P> classFPP) {
-    final List<String> addresses = new ArrayList();
+    final List<String> addresses = new ArrayList<>();
     nnAddresses.forEach(
         addr -> addresses.add(
             "hdfs://" + addr.getHostName() + ":" + addr.getPort()));
@@ -310,7 +354,7 @@ public static void setFailoverConfigurations(Configuration conf, String logicalN
   }
 
   public static <P extends FailoverProxyProvider<?>>
-  void setFailoverConfigurations(
+      void setFailoverConfigurations(
       Configuration conf, String logicalName,
       Iterable<String> nnAddresses, Class<P> classFPP) {
     List<String> nnids = new ArrayList<String>();
@@ -332,13 +376,13 @@ void setFailoverConfigurations(
   public static String getLogicalHostname(MiniDFSCluster cluster) {
     return String.format(LOGICAL_HOSTNAME, cluster.getInstanceId());
   }
-  
+
   public static URI getLogicalUri(MiniDFSCluster cluster)
       throws URISyntaxException {
     return new URI(HdfsConstants.HDFS_URI_SCHEME + "://" +
         getLogicalHostname(cluster));
   }
-  
+
   public static void waitForCheckpoint(MiniDFSCluster cluster, int nnIdx,
       List<Integer> txids) throws InterruptedException {
     long start = Time.now();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
index 18f987db97481..ff6c2288b538b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConsistentReadsObserver.java
@@ -20,6 +20,7 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY;
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -108,8 +109,6 @@ public static void shutDownCluster() throws IOException {
 
   @Test
   public void testRequeueCall() throws Exception {
-    setObserverRead(true);
-
     // Update the configuration just for the observer, by enabling
     // IPC backoff and using the test scheduler class, which starts to backoff
     // after certain number of calls.
@@ -125,6 +124,7 @@ public void testRequeueCall() throws Exception {
         + CommonConfigurationKeys.IPC_BACKOFF_ENABLE, true);
 
     NameNodeAdapter.getRpcServer(nn).refreshCallQueue(configuration);
+    assertThat(NameNodeAdapter.getRpcServer(nn).getTotalRequests()).isGreaterThan(0);
 
     dfs.create(testPath, (short)1).close();
     assertSentTo(0);
@@ -134,6 +134,7 @@ public void testRequeueCall() throws Exception {
     // be triggered and client should retry active NN.
     dfs.getFileStatus(testPath);
     assertSentTo(0);
+    assertThat(NameNodeAdapter.getRpcServer(nn).getTotalRequests()).isGreaterThan(1);
     // reset the original call queue
     NameNodeAdapter.getRpcServer(nn).refreshCallQueue(originalConf);
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
index 2ef48a31f9dc3..168273117b50f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
@@ -95,7 +95,7 @@ public void setupCluster() throws Exception {
     cluster.waitActive();
     
     String logicalName = HATestUtil.getLogicalHostname(cluster);
-    HATestUtil.setFailoverConfigurations(cluster, conf, logicalName, 0);
+    HATestUtil.setFailoverConfigurations(cluster, conf, logicalName, null, 0);
 
     nn0 = cluster.getNameNode(0);
     nn1 = cluster.getNameNode(1);
@@ -317,7 +317,7 @@ public Void run() throws Exception {
     longUgi.doAs(new PrivilegedExceptionAction<Void>() {
       @Override
       public Void run() throws Exception {
-        token.cancel(conf);;
+        token.cancel(conf);
         return null;
       }
     });
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java
index 46ebb8f104225..c6e3d139e1723 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAFsck.java
@@ -17,11 +17,14 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.ha;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
+import java.util.Arrays;
+
+import org.junit.Test;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.slf4j.event.Level;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -32,15 +35,30 @@
 import org.apache.hadoop.hdfs.tools.DFSck;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.event.Level;
-import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class TestHAFsck {
   
   static {
     GenericTestUtils.setLogLevel(DFSUtil.LOG, Level.TRACE);
   }
-  
+
+  @Parameter
+  private String proxyProvider;
+
+  public String getProxyProvider() {
+    return proxyProvider;
+  }
+
+  @Parameterized.Parameters(name = "ProxyProvider: {0}")
+  public static Iterable<Object[]> data() {
+    return Arrays.asList(new Object[][]
+        {{ConfiguredFailoverProxyProvider.class.getName()},
+        {RequestHedgingProxyProvider.class.getName()}});
+  }
+
   /**
    * Test that fsck still works with HA enabled.
    */
@@ -65,9 +83,9 @@ public void testHaFsck() throws Exception {
       cluster.transitionToActive(0);
       
       // Make sure conf has the relevant HA configs.
-      HATestUtil.setFailoverConfigurations(cluster, conf, "ha-nn-uri-0", 0);
+      HATestUtil.setFailoverConfigurations(cluster, conf, "ha-nn-uri-0", getProxyProvider(), 0);
       
-      fs = HATestUtil.configureFailoverFs(cluster, conf);
+      fs = FileSystem.get(conf);
       fs.mkdirs(new Path("/test1"));
       fs.mkdirs(new Path("/test2"));
       
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index 537e6a34bd0b1..4766c4cecc9b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
@@ -347,6 +347,8 @@ public void testBlocksRemovedBeforeStandbyRestart() throws Exception {
     // once it starts up
     banner("Removing the blocks without rolling the edit log");
     fs.delete(new Path("/test"), true);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     BlockManagerTestUtil.computeAllPendingWork(
         nn0.getNamesystem().getBlockManager());
     cluster.triggerHeartbeats();
@@ -386,6 +388,8 @@ public void testBlocksRemovedWhileInSafeMode() throws Exception {
     // ACKed when due to block removals.
     banner("Removing the blocks without rolling the edit log");
     fs.delete(new Path("/test"), true);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem(0).getBlockManager());
     BlockManagerTestUtil.computeAllPendingWork(
         nn0.getNamesystem().getBlockManager());
     
@@ -848,13 +852,13 @@ public void testOpenFileWhenNNAndClientCrashAfterAddBlock() throws Exception {
           null);
       create.write(testData.getBytes());
       create.hflush();
-      long fileId = ((DFSOutputStream)create.
-          getWrappedStream()).getFileId();
+      String renewLeaseKey = ((DFSOutputStream)create.
+          getWrappedStream()).getUniqKey();
       FileStatus fileStatus = dfs.getFileStatus(filePath);
       DFSClient client = DFSClientAdapter.getClient(dfs);
       // add one dummy block at NN, but not write to DataNode
       ExtendedBlock previousBlock =
-          DFSClientAdapter.getPreviousBlock(client, fileId);
+          DFSClientAdapter.getPreviousBlock(client, renewLeaseKey);
       DFSClientAdapter.getNamenode(client).addBlock(
           pathString,
           client.getClientName(),
@@ -973,4 +977,26 @@ public void testTransitionToActiveWhenSafeMode() throws Exception {
           () -> miniCluster.transitionToActive(0));
     }
   }
+
+  @Test
+  public void testTransitionToObserverWhenSafeMode() throws Exception {
+    Configuration config = new Configuration();
+    config.setBoolean(DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE, true);
+    try (MiniDFSCluster miniCluster = new MiniDFSCluster.Builder(config,
+        new File(GenericTestUtils.getRandomizedTempPath()))
+        .nnTopology(MiniDFSNNTopology.simpleHATopology())
+        .numDataNodes(1)
+        .build()) {
+      miniCluster.waitActive();
+      miniCluster.transitionToStandby(0);
+      miniCluster.transitionToStandby(1);
+      NameNode namenode0 = miniCluster.getNameNode(0);
+      NameNode namenode1 = miniCluster.getNameNode(1);
+      NameNodeAdapter.enterSafeMode(namenode0, false);
+      NameNodeAdapter.enterSafeMode(namenode1, false);
+      LambdaTestUtils.intercept(ServiceFailedException.class,
+          "NameNode still not leave safemode",
+          () -> miniCluster.transitionToObserver(0));
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
index 29cae6f13adbc..8b691a11725a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java
@@ -21,6 +21,7 @@
 import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState;
 import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.*;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
@@ -36,7 +37,10 @@
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
@@ -61,9 +65,12 @@
 import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
 import org.apache.hadoop.hdfs.server.namenode.TestFsck;
 import org.apache.hadoop.hdfs.tools.GetGroups;
 import org.apache.hadoop.ipc.ObserverRetryOnActiveException;
+import org.apache.hadoop.ipc.metrics.RpcMetrics;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.junit.After;
@@ -124,6 +131,43 @@ public static void shutDownCluster() throws IOException {
     }
   }
 
+  @Test
+  public void testObserverRequeue() throws Exception {
+    ScheduledExecutorService interruptor =
+        Executors.newScheduledThreadPool(1);
+
+    FSNamesystem observerFsNS = dfsCluster.getNamesystem(2);
+    RpcMetrics obRpcMetrics = ((NameNodeRpcServer)dfsCluster
+        .getNameNodeRpc(2)).getClientRpcServer().getRpcMetrics();
+    try {
+      // Stop EditlogTailer of Observer NameNode.
+      observerFsNS.getEditLogTailer().stop();
+      long oldRequeueNum = obRpcMetrics.getRpcRequeueCalls();
+      ScheduledFuture<FileStatus> scheduledFuture = interruptor.schedule(
+          () -> {
+            Path tmpTestPath = new Path("/TestObserverRequeue");
+            dfs.create(tmpTestPath, (short)1).close();
+            assertSentTo(0);
+            // This operation will be blocked in ObserverNameNode
+            // until EditlogTailer tailed edits from journalNode.
+            FileStatus fileStatus = dfs.getFileStatus(tmpTestPath);
+            assertSentTo(2);
+            return fileStatus;
+          }, 0, TimeUnit.SECONDS);
+
+      GenericTestUtils.waitFor(() -> obRpcMetrics.getRpcRequeueCalls() > oldRequeueNum,
+          50, 10000);
+
+      observerFsNS.getEditLogTailer().doTailEdits();
+      FileStatus fileStatus = scheduledFuture.get(10000, TimeUnit.MILLISECONDS);
+      assertNotNull(fileStatus);
+    } finally {
+      EditLogTailer editLogTailer = new EditLogTailer(observerFsNS, conf);
+      observerFsNS.setEditLogTailerForTests(editLogTailer);
+      editLogTailer.start();
+    }
+  }
+
   @Test
   public void testNoActiveToObserver() throws Exception {
     try {
@@ -369,6 +413,12 @@ public void testObserverNodeBlockMissingRetry() throws Exception {
     dfs.open(testPath);
     assertSentTo(0);
 
+    dfs.getClient().listPaths("/", new byte[0], true);
+    assertSentTo(0);
+
+    dfs.getClient().getLocatedFileInfo(testPath.toString(), false);
+    assertSentTo(0);
+
     Mockito.reset(bmSpy);
   }
 
@@ -602,6 +652,29 @@ public void run() {
     }
   }
 
+  @Test
+  public void testSimpleReadEmptyDirOrFile() throws IOException {
+    // read empty dir
+    dfs.mkdirs(new Path("/emptyDir"));
+    assertSentTo(0);
+
+    dfs.getClient().listPaths("/", new byte[0], true);
+    assertSentTo(2);
+
+    dfs.getClient().getLocatedFileInfo("/emptyDir", true);
+    assertSentTo(2);
+
+    // read empty file
+    dfs.create(new Path("/emptyFile"), (short)1);
+    assertSentTo(0);
+
+    dfs.getClient().getLocatedFileInfo("/emptyFile", true);
+    assertSentTo(2);
+
+    dfs.getClient().getBlockLocations("/emptyFile", 0, 1);
+    assertSentTo(2);
+  }
+
   private static void assertSentTo(DistributedFileSystem fs, int nnIdx)
       throws IOException {
     assertTrue("Request was not sent to the expected namenode " + nnIdx,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index 349b7ac24112d..4d472d74b46a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -48,6 +48,9 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Random;
+
+import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
+import org.apache.hadoop.ipc.metrics.RpcDetailedMetrics;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 
 import org.slf4j.Logger;
@@ -661,6 +664,8 @@ public void testExcessBlocks() throws Exception {
     // verify ExcessBlocks metric is decremented and
     // excessReplicateMap is cleared after deleting a file
     fs.delete(file, true);
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem().getBlockManager());
     rb = getMetrics(NS_METRICS);
     assertGauge("ExcessBlocks", 0L, rb);
     assertEquals(0L, bm.getExcessBlocksCount());
@@ -1126,4 +1131,14 @@ public void testEditLogTailing() throws Exception {
     }
 
   }
+
+  @Test
+  public void testNNRPCMetricIntegrity() {
+    RpcDetailedMetrics metrics =
+        ((NameNodeRpcServer) cluster.getNameNode()
+            .getRpcServer()).getClientRpcServer().getRpcDetailedMetrics();
+    MetricsRecordBuilder rb = getMetrics(metrics.name());
+    // CommitBlockSynchronizationNumOps should exist.
+    assertCounter("CommitBlockSynchronizationNumOps", 0L, rb);
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index b85530c3cbee6..0eb094c48094a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -1128,7 +1129,8 @@ public void testHANNRestartAfterSnapshotDeletion() throws Exception {
   }
 
   @Test
-  public void testCorrectNumberOfBlocksAfterRestart() throws IOException {
+  public void testCorrectNumberOfBlocksAfterRestart()
+      throws IOException, InterruptedException {
     final Path foo = new Path("/foo");
     final Path bar = new Path(foo, "bar");
     final Path file = new Path(foo, "file");
@@ -1149,9 +1151,10 @@ public void testCorrectNumberOfBlocksAfterRestart() throws IOException {
     hdfs.delete(bar, true);
     hdfs.delete(foo, true);
 
-    long numberOfBlocks = cluster.getNamesystem().getBlocksTotal();
     cluster.restartNameNode(0);
-    assertEquals(numberOfBlocks, cluster.getNamesystem().getBlocksTotal());
+    BlockManagerTestUtil.waitForMarkedDeleteQueueIsEmpty(
+        cluster.getNamesystem().getBlockManager());
+    assertEquals(0, cluster.getNamesystem().getBlocksTotal());
   }
 
   /*
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
index efe59ca0d943d..1d50e50d426f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
@@ -345,7 +345,8 @@ public void testSnapRootDescendantDiffReport() throws Exception {
           + nonSnapDir.getName() + "'should fail!");
     } catch (SnapshotException e) {
       GenericTestUtils.assertExceptionContains(
-          "Directory is neither snapshottable nor under a snap root!", e);
+          "The path " + nonSnapDir +
+              " is neither snapshottable nor under a snapshot root!", e);
     }
 
     final String invalidName = "invalid";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
index badff6821243e..03931d6c6630d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
@@ -34,7 +34,6 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
index 664f459ebae01..8e0364078357d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
@@ -186,7 +186,7 @@ public void testMoverWithFullStripe() throws Exception {
               {StorageType.ARCHIVE, StorageType.ARCHIVE},
               {StorageType.ARCHIVE, StorageType.ARCHIVE},
               {StorageType.ARCHIVE, StorageType.ARCHIVE}},
-          true, null, null, null, capacities, null, false, false, false, null);
+          true, null, null, null, capacities, null, false, false, false, null, null, null);
       cluster.triggerHeartbeats();
 
       // move file to ARCHIVE
@@ -294,7 +294,7 @@ public void testWhenOnlyFewTargetNodesAreAvailableToSatisfyStoragePolicy()
           new StorageType[][]{
               {StorageType.ARCHIVE, StorageType.ARCHIVE},
               {StorageType.ARCHIVE, StorageType.ARCHIVE}},
-          true, null, null, null, capacities, null, false, false, false, null);
+          true, null, null, null, capacities, null, false, false, false, null, null, null);
       cluster.triggerHeartbeats();
 
       // Move file to ARCHIVE. Only 5 datanodes are available with ARCHIVE
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java
index 4fe3d15c4a82b..ef2004f6eb6e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/TestStartupProgressMetrics.java
@@ -17,9 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.namenode.startupprogress;
 
-import static org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase.*;
 import static org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressTestHelper.*;
-import static org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType.*;
 import static org.apache.hadoop.test.MetricsAsserts.*;
 import static org.junit.Assert.*;
 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
index 77922a09912e2..58d72f14d73ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
@@ -33,15 +33,18 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SPS_KEYTAB_FILE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SPS_MAX_OUTSTANDING_PATHS_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY;
 import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_SATISFY_STORAGE_POLICY;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
@@ -82,6 +85,7 @@
 import org.apache.hadoop.hdfs.server.namenode.sps.BlockMovementListener;
 import org.apache.hadoop.hdfs.server.namenode.sps.BlockStorageMovementAttemptedItems;
 import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
+import org.apache.hadoop.hdfs.server.sps.metrics.ExternalSPSBeanMetrics;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.security.SecurityUtil;
@@ -90,6 +94,8 @@
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.ExitUtil;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -98,6 +104,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
 import java.util.function.Supplier;
 
 /**
@@ -127,6 +135,14 @@ public class TestExternalStoragePolicySatisfier {
   private static final int DEFAULT_BLOCK_SIZE = 1024;
   private static final Logger LOG =
       LoggerFactory.getLogger(TestExternalStoragePolicySatisfier.class);
+  private final ExternalSPSFaultInjector injector = new ExternalSPSFaultInjector() {
+    @Override
+    public void mockAnException(int retry) throws IOException {
+      if (retry < DFS_STORAGE_POLICY_SATISFIER_MOVE_TASK_MAX_RETRY_ATTEMPTS_DEFAULT) {
+        throw new IOException("IO exception");
+      }
+    }
+  };
 
   @Before
   public void setUp() {
@@ -189,6 +205,20 @@ private void shutdownCluster() {
     }
   }
 
+  private void stopExternalSps() {
+    if (externalSps != null) {
+      externalSps.stopGracefully();
+    }
+  }
+
+  private void startExternalSps() {
+    externalSps = new StoragePolicySatisfier(getConf());
+    externalCtxt = new ExternalSPSContext(externalSps, nnc);
+
+    externalSps.init(externalCtxt);
+    externalSps.start(StoragePolicySatisfierMode.EXTERNAL);
+  }
+
   private void createCluster() throws IOException {
     getConf().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
     setCluster(startCluster(getConf(), allDiskTypes, NUM_OF_DATANODES,
@@ -197,9 +227,32 @@ private void createCluster() throws IOException {
     writeContent(FILE);
   }
 
+  private void createCluster(boolean createMoverPath) throws IOException {
+    getConf().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
+    setCluster(startCluster(getConf(), allDiskTypes, NUM_OF_DATANODES,
+        STORAGES_PER_DATANODE, CAPACITY, createMoverPath, true));
+    getFS();
+    writeContent(FILE);
+  }
+
+  private void createClusterDoNotStartSPS() throws IOException {
+    getConf().setLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
+    setCluster(startCluster(getConf(), allDiskTypes, NUM_OF_DATANODES,
+        STORAGES_PER_DATANODE, CAPACITY, true, false));
+    getFS();
+    writeContent(FILE);
+  }
+
   private MiniDFSCluster startCluster(final Configuration conf,
       StorageType[][] storageTypes, int numberOfDatanodes, int storagesPerDn,
       long nodeCapacity) throws IOException {
+    return startCluster(conf, storageTypes, numberOfDatanodes, storagesPerDn,
+        nodeCapacity, false, true);
+  }
+
+  private MiniDFSCluster startCluster(final Configuration conf,
+      StorageType[][] storageTypes, int numberOfDatanodes, int storagesPerDn,
+      long nodeCapacity, boolean createMoverPath, boolean startSPS) throws IOException {
     long[][] capacities = new long[numberOfDatanodes][storagesPerDn];
     for (int i = 0; i < numberOfDatanodes; i++) {
       for (int j = 0; j < storagesPerDn; j++) {
@@ -211,14 +264,16 @@ private MiniDFSCluster startCluster(final Configuration conf,
         .storageTypes(storageTypes).storageCapacities(capacities).build();
     cluster.waitActive();
 
-    nnc = DFSTestUtil.getNameNodeConnector(getConf(),
-        HdfsServerConstants.MOVER_ID_PATH, 1, false);
+    if (startSPS) {
+      nnc = DFSTestUtil.getNameNodeConnector(getConf(),
+          HdfsServerConstants.MOVER_ID_PATH, 1, createMoverPath);
 
-    externalSps = new StoragePolicySatisfier(getConf());
-    externalCtxt = new ExternalSPSContext(externalSps, nnc);
+      externalSps = new StoragePolicySatisfier(getConf());
+      externalCtxt = new ExternalSPSContext(externalSps, nnc);
 
-    externalSps.init(externalCtxt);
-    externalSps.start(StoragePolicySatisfierMode.EXTERNAL);
+      externalSps.init(externalCtxt);
+      externalSps.start(StoragePolicySatisfierMode.EXTERNAL);
+    }
     return cluster;
   }
 
@@ -428,6 +483,44 @@ public void testWhenStoragePolicySetToCOLD()
     }
   }
 
+  @Test(timeout = 300000)
+  public void testInfiniteStartWhenAnotherSPSRunning()
+      throws Exception {
+
+    try {
+      // Create cluster and create mover path when get NameNodeConnector.
+      createCluster(true);
+
+      // Disable system exit for assert.
+      ExitUtil.disableSystemExit();
+
+      // Get NameNodeConnector one more time to simulate starting other sps process.
+      // It should exit immediately when another sps is running.
+      LambdaTestUtils.intercept(ExitUtil.ExitException.class,
+          "Exit immediately because another ExternalStoragePolicySatisfier is running",
+          () -> ExternalStoragePolicySatisfier.getNameNodeConnector(config));
+    } finally {
+      // Reset first exit exception to avoid AssertionError in MiniDFSCluster#shutdown.
+      // This has no effect on functionality.
+      ExitUtil.resetFirstExitException();
+      shutdownCluster();
+    }
+  }
+
+  @Test(timeout = 300000)
+  public void testWhenStoragePolicySetToCOLDWithException()
+      throws Exception {
+
+    try {
+      createCluster();
+      // Mock an IOException 3 times, and moving tasks should succeed finally.
+      ExternalSPSFaultInjector.setInstance(injector);
+      doTestWhenStoragePolicySetToCOLD();
+    } finally {
+      shutdownCluster();
+    }
+  }
+
   private void doTestWhenStoragePolicySetToCOLD() throws Exception {
     // Change policy to COLD
     dfs.setStoragePolicy(new Path(FILE), COLD);
@@ -1258,7 +1351,7 @@ public void testSPSWhenFileHasLowRedundancyBlocks() throws Exception {
    * 4. Set policy and call satisfyStoragePolicy for file.
    * 5. Block should be moved successfully.
    */
-  @Test(timeout = 300000)
+  @Test(timeout = 600000)
   public void testSPSWhenFileHasExcessRedundancyBlocks() throws Exception {
     try {
       config.set(DFSConfigKeys
@@ -1370,6 +1463,45 @@ public void testSPSWithDirectoryTreeWithoutFile() throws Exception {
     }
   }
 
+  /**
+   * Test SPS that satisfy the files and then delete the files before start SPS.
+   */
+  @Test(timeout = 300000)
+  public void testSPSSatisfyAndThenDeleteFileBeforeStartSPS() throws Exception {
+    try {
+      createCluster();
+      HdfsAdmin hdfsAdmin =
+          new HdfsAdmin(FileSystem.getDefaultUri(config), config);
+
+      StorageType[][] newtypes =
+          new StorageType[][]{{StorageType.DISK, StorageType.ARCHIVE},
+              {StorageType.DISK, StorageType.ARCHIVE},
+              {StorageType.DISK, StorageType.ARCHIVE}};
+      startAdditionalDNs(config, 3, NUM_OF_DATANODES, newtypes,
+          STORAGES_PER_DATANODE, CAPACITY, hdfsCluster);
+
+      stopExternalSps();
+
+      dfs.setStoragePolicy(new Path(FILE), COLD);
+      hdfsAdmin.satisfyStoragePolicy(new Path(FILE));
+      dfs.delete(new Path(FILE), true);
+
+      startExternalSps();
+
+      String file1 = "/testMoveToSatisfyStoragePolicy_1";
+      writeContent(file1);
+      dfs.setStoragePolicy(new Path(file1), COLD);
+      hdfsAdmin.satisfyStoragePolicy(new Path(file1));
+
+      hdfsCluster.triggerHeartbeats();
+      DFSTestUtil.waitExpectedStorageType(file1, StorageType.ARCHIVE, 3, 30000,
+          dfs);
+    } finally {
+      shutdownCluster();
+    }
+  }
+
+
   /**
    * Test SPS for directory which has multilevel directories.
    */
@@ -1474,6 +1606,20 @@ public void testMoveBlocksWithUnderReplicatedBlocks() throws Exception {
     }
   }
 
+  @Test(timeout = 300000)
+  public void testExternalSPSMetrics()
+      throws Exception {
+
+    try {
+      createClusterDoNotStartSPS();
+      dfs.satisfyStoragePolicy(new Path(FILE));
+      // Assert metrics.
+      assertEquals(1, hdfsCluster.getNamesystem().getPendingSPSPaths());
+    } finally {
+      shutdownCluster();
+    }
+  }
+
   private static void createDirectoryTree(DistributedFileSystem dfs)
       throws Exception {
     // tree structure
@@ -1646,7 +1792,7 @@ private void startAdditionalDNs(final Configuration conf,
     }
 
     cluster.startDataNodes(conf, newNodesRequired, newTypes, true, null, null,
-        null, capacities, null, false, false, false, null);
+        null, capacities, null, false, false, false, null, null, null);
     cluster.triggerHeartbeats();
   }
 
@@ -1675,4 +1821,34 @@ public void clear() {
       actualBlockMovements.clear();
     }
   }
+
+  @Test(timeout = 300000)
+  public void testExternalSPSMetricsExposedToJMX() throws Exception {
+    try {
+      createCluster();
+      // Start JMX but stop SPS thread to prevent mock data from being consumed.
+      externalSps.stop(true);
+      externalCtxt.initMetrics(externalSps);
+
+      ExternalSPSBeanMetrics spsBeanMetrics = externalCtxt.getSpsBeanMetrics();
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxBeanName = new ObjectName("Hadoop:service=ExternalSPS,name=ExternalSPS");
+      // Assert metrics before update.
+      assertEquals(0, mbs.getAttribute(mxBeanName, "AttemptedItemsCount"));
+      assertEquals(0, mbs.getAttribute(mxBeanName, "ProcessingQueueSize"));
+      assertEquals(0, mbs.getAttribute(mxBeanName, "MovementFinishedBlocksCount"));
+
+      // Update metrics.
+      spsBeanMetrics.updateAttemptedItemsCount();
+      spsBeanMetrics.updateProcessingQueueSize();
+      spsBeanMetrics.updateMovementFinishedBlocksCount();
+
+      // Assert metrics after update.
+      assertEquals(1, mbs.getAttribute(mxBeanName, "AttemptedItemsCount"));
+      assertEquals(1, mbs.getAttribute(mxBeanName, "ProcessingQueueSize"));
+      assertEquals(1, mbs.getAttribute(mxBeanName, "MovementFinishedBlocksCount"));
+    } finally {
+      shutdownCluster();
+    }
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index aac1bf5059bc3..7b11ec30b43d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -21,7 +21,6 @@
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY;
-import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 import java.io.DataOutputStream;
@@ -63,6 +62,7 @@
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry;
 import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry.RegisteredShm;
@@ -89,7 +89,7 @@
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.function.Supplier;
 import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap;
 
@@ -957,6 +957,83 @@ public void testDomainSocketClosedByDN() throws Exception {
     }
   }
 
+  // Regression test for HDFS-16535
+  @Test(timeout = 60000)
+  public void testDomainSocketClosedByMultipleDNs() throws Exception {
+    TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+    String testName = "testDomainSocketClosedByMultipleDNs";
+    Configuration conf = createShortCircuitConf(testName, sockDir);
+    conf.set(DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(),
+        testName + "._PORT").getAbsolutePath());
+    MiniDFSCluster cluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+
+    try {
+      cluster.waitActive();
+      DistributedFileSystem fs = cluster.getFileSystem();
+      final ShortCircuitCache cache =
+          fs.getClient().getClientContext().getShortCircuitCache();
+
+      ExtendedBlockId blockId0 = new ExtendedBlockId(123, "xyz");
+      ExtendedBlockId blockId1 = new ExtendedBlockId(456, "xyz");
+
+      DataNode dn0 = cluster.getDataNodes().get(0);
+      DataNode dn1 = cluster.getDataNodes().get(1);
+
+      DomainPeer peer0 = new DomainPeer(DomainSocket.connect(new File(
+          sockDir.getDir(), testName + "." + dn0.getXferPort()).getAbsolutePath()));
+      DomainPeer peer1 = new DomainPeer(DomainSocket.connect(new File(
+          sockDir.getDir(), testName + "." + dn1.getXferPort()).getAbsolutePath()));
+
+      final DatanodeInfo dnInfo0 = new DatanodeInfo.DatanodeInfoBuilder()
+          .setNodeID(dn0.getDatanodeId()).build();
+      final DatanodeInfo dnInfo1 = new DatanodeInfo.DatanodeInfoBuilder()
+          .setNodeID(dn1.getDatanodeId()).build();
+
+      // Allocate 2 shm slots from DataNode-0
+      MutableBoolean usedPeer = new MutableBoolean(false);
+      Slot slot1 = cache.allocShmSlot(dnInfo0, peer0, usedPeer, blockId0,
+          "testDomainSocketClosedByMultipleDNs_client");
+      dn0.getShortCircuitRegistry()
+          .registerSlot(blockId0, slot1.getSlotId(), false);
+
+      Slot slot2 = cache.allocShmSlot(dnInfo0, peer0, usedPeer, blockId0,
+          "testDomainSocketClosedByMultipleDNs_client");
+      dn0.getShortCircuitRegistry()
+          .registerSlot(blockId0, slot2.getSlotId(), false);
+
+      // Allocate 1 shm slot from DataNode-1
+      Slot slot3 = cache.allocShmSlot(dnInfo1, peer1, usedPeer, blockId1,
+          "testDomainSocketClosedByMultipleDNs_client");
+      dn1.getShortCircuitRegistry()
+          .registerSlot(blockId1, slot3.getSlotId(), false);
+
+      Assert.assertEquals(2, cache.getDfsClientShmManager().getShmNum());
+      Assert.assertEquals(1, dn0.getShortCircuitRegistry().getShmNum());
+      Assert.assertEquals(1, dn1.getShortCircuitRegistry().getShmNum());
+
+      // Release the slot of DataNode-1 first.
+      cache.scheduleSlotReleaser(slot3);
+      Thread.sleep(2000);
+      Assert.assertEquals(1, cache.getDfsClientShmManager().getShmNum());
+
+      // Release the slots of DataNode-0.
+      cache.scheduleSlotReleaser(slot1);
+      Thread.sleep(2000);
+      Assert.assertEquals("0 ShmNum means the shm of DataNode-0 is shutdown" +
+              " due to slot release failures.",
+          1, cache.getDfsClientShmManager().getShmNum());
+      cache.scheduleSlotReleaser(slot2);
+      Thread.sleep(2000);
+
+      Assert.assertEquals(0, dn0.getShortCircuitRegistry().getShmNum());
+      Assert.assertEquals(0, dn1.getShortCircuitRegistry().getShmNum());
+      Assert.assertEquals(0, cache.getDfsClientShmManager().getShmNum());
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
   @Test(timeout = 60000)
   public void testDNRestart() throws Exception {
     TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index e9e1cad25a922..59491206dcbbe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -31,13 +31,20 @@
 import java.util.function.Supplier;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_MAX_NODES_TO_REPORT_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.text.TextStringBuilder;
@@ -66,6 +73,8 @@
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
@@ -88,6 +97,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY;
 import static org.apache.hadoop.hdfs.client.HdfsAdmin.TRASH_PERMISSION;
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.anyOf;
@@ -180,20 +190,20 @@ private void restartCluster() throws IOException {
   }
 
   private void getReconfigurableProperties(String nodeType, String address,
-      final List<String> outs, final List<String> errs) throws IOException {
+      final List<String> outs, final List<String> errs) throws IOException, InterruptedException {
     reconfigurationOutErrFormatter("getReconfigurableProperties", nodeType,
         address, outs, errs);
   }
 
   private void getReconfigurationStatus(String nodeType, String address,
-      final List<String> outs, final List<String> errs) throws IOException {
+      final List<String> outs, final List<String> errs) throws IOException, InterruptedException {
     reconfigurationOutErrFormatter("getReconfigurationStatus", nodeType,
         address, outs, errs);
   }
 
   private void reconfigurationOutErrFormatter(String methodName,
       String nodeType, String address, final List<String> outs,
-      final List<String> errs) throws IOException {
+      final List<String> errs) throws IOException, InterruptedException {
     ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
     PrintStream outStream = new PrintStream(bufOut);
     ByteArrayOutputStream bufErr = new ByteArrayOutputStream();
@@ -206,9 +216,9 @@ private void reconfigurationOutErrFormatter(String methodName,
           outStream,
           errStream);
     } else if (methodName.equals("getReconfigurationStatus")) {
-      admin.getReconfigurationStatus(nodeType, address, outStream, errStream);
+      admin.getReconfigurationStatusUtil(nodeType, address, outStream, errStream);
     } else if (methodName.equals("startReconfiguration")) {
-      admin.startReconfiguration(nodeType, address, outStream, errStream);
+      admin.startReconfigurationUtil(nodeType, address, outStream, errStream);
     }
 
     scanIntoList(bufOut, outs);
@@ -329,13 +339,13 @@ public void testDFSAdminUnreachableDatanode() throws Exception {
   }
 
   @Test(timeout = 30000)
-  public void testDataNodeGetReconfigurableProperties() throws IOException {
+  public void testDataNodeGetReconfigurableProperties() throws IOException, InterruptedException {
     final int port = datanode.getIpcPort();
     final String address = "localhost:" + port;
     final List<String> outs = Lists.newArrayList();
     final List<String> errs = Lists.newArrayList();
     getReconfigurableProperties("datanode", address, outs, errs);
-    assertEquals(3, outs.size());
+    assertEquals(19, outs.size());
     assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1));
   }
 
@@ -425,17 +435,26 @@ public void testDataNodeGetReconfigurationStatus() throws IOException,
   }
 
   @Test(timeout = 30000)
-  public void testNameNodeGetReconfigurableProperties() throws IOException {
+  public void testNameNodeGetReconfigurableProperties() throws IOException, InterruptedException {
     final String address = namenode.getHostAndPort();
     final List<String> outs = Lists.newArrayList();
     final List<String> errs = Lists.newArrayList();
     getReconfigurableProperties("namenode", address, outs, errs);
-    assertEquals(13, outs.size());
-    assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(1));
-    assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(2));
-    assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(3));
-    assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(4));
-    assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(5));
+    assertEquals(22, outs.size());
+    assertTrue(outs.get(0).contains("Reconfigurable properties:"));
+    assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1));
+    assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2));
+    assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(3));
+    assertEquals(DFS_DATANODE_MAX_NODES_TO_REPORT_KEY, outs.get(4));
+    assertEquals(DFS_DATANODE_PEER_STATS_ENABLED_KEY, outs.get(5));
+    assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(6));
+    assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(7));
+    assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(8));
+    assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(9));
+    assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_BLOCKS_PER_LOCK, outs.get(10));
+    assertEquals(DFS_NAMENODE_DECOMMISSION_BACKOFF_MONITOR_PENDING_LIMIT, outs.get(11));
+    assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(12));
+    assertEquals(DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, outs.get(13));
     assertEquals(errs.size(), 0);
   }
 
@@ -449,7 +468,7 @@ public Boolean get() {
         errs.clear();
         try {
           getReconfigurationStatus(nodeType, address, outs, errs);
-        } catch (IOException e) {
+        } catch (IOException | InterruptedException e) {
           LOG.error(String.format(
               "call getReconfigurationStatus on %s[%s] failed.", nodeType,
               address), e);
@@ -509,6 +528,52 @@ public void testPrintTopology() throws Exception {
     }
   }
 
+  @Test(timeout = 30000)
+  public void testPrintTopologyWithStatus() throws Exception {
+    redirectStream();
+    final Configuration dfsConf = new HdfsConfiguration();
+    final File baseDir = new File(
+            PathUtils.getTestDir(getClass()),
+            GenericTestUtils.getMethodName());
+    dfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+
+    final int numDn = 4;
+    final String[] racks = {
+        "/d1/r1", "/d1/r2",
+        "/d2/r1", "/d2/r2"};
+
+    try (MiniDFSCluster miniCluster = new MiniDFSCluster.Builder(dfsConf)
+            .numDataNodes(numDn).racks(racks).build()) {
+      miniCluster.waitActive();
+      assertEquals(numDn, miniCluster.getDataNodes().size());
+
+      DatanodeManager dm = miniCluster.getNameNode().getNamesystem().
+          getBlockManager().getDatanodeManager();
+      DatanodeDescriptor maintenanceNode = dm.getDatanode(
+          miniCluster.getDataNodes().get(1).getDatanodeId());
+      maintenanceNode.setInMaintenance();
+      DatanodeDescriptor demissionNode = dm.getDatanode(
+          miniCluster.getDataNodes().get(2).getDatanodeId());
+      demissionNode.setDecommissioned();
+
+      final DFSAdmin dfsAdmin = new DFSAdmin(dfsConf);
+
+      resetStream();
+      final int ret = ToolRunner.run(dfsAdmin, new String[] {"-printTopology"});
+
+      /* collect outputs */
+      final List<String> outs = Lists.newArrayList();
+      scanIntoList(out, outs);
+
+      /* verify results */
+      assertEquals(0, ret);
+      assertTrue(outs.get(1).contains(DatanodeInfo.AdminStates.NORMAL.toString()));
+      assertTrue(outs.get(4).contains(DatanodeInfo.AdminStates.IN_MAINTENANCE.toString()));
+      assertTrue(outs.get(7).contains(DatanodeInfo.AdminStates.DECOMMISSIONED.toString()));
+      assertTrue(outs.get(10).contains(DatanodeInfo.AdminStates.NORMAL.toString()));
+    }
+  }
+
   @Test(timeout = 30000)
   public void testNameNodeGetReconfigurationStatus() throws IOException,
       InterruptedException, TimeoutException {
@@ -1158,4 +1223,51 @@ public Integer run() throws Exception {
       }
     });
   }
-}
\ No newline at end of file
+
+  @Test
+  public void testAllDatanodesReconfig()
+      throws IOException, InterruptedException, TimeoutException {
+    ReconfigurationUtil reconfigurationUtil = mock(ReconfigurationUtil.class);
+    cluster.getDataNodes().get(0).setReconfigurationUtil(reconfigurationUtil);
+    cluster.getDataNodes().get(1).setReconfigurationUtil(reconfigurationUtil);
+
+    List<ReconfigurationUtil.PropertyChange> changes = new ArrayList<>();
+    changes.add(new ReconfigurationUtil.PropertyChange(
+        DFS_DATANODE_PEER_STATS_ENABLED_KEY, "true",
+        datanode.getConf().get(DFS_DATANODE_PEER_STATS_ENABLED_KEY)));
+    when(reconfigurationUtil.parseChangedProperties(any(Configuration.class),
+        any(Configuration.class))).thenReturn(changes);
+
+    assertEquals(0, admin.startReconfiguration("datanode", "livenodes"));
+    final List<String> outsForStartReconf = new ArrayList<>();
+    final List<String> errsForStartReconf = new ArrayList<>();
+    reconfigurationOutErrFormatter("startReconfiguration", "datanode",
+        "livenodes", outsForStartReconf, errsForStartReconf);
+    assertEquals(3, outsForStartReconf.size());
+    assertEquals(0, errsForStartReconf.size());
+    assertTrue(outsForStartReconf.get(0).startsWith("Started reconfiguration task on node"));
+    assertTrue(outsForStartReconf.get(1).startsWith("Started reconfiguration task on node"));
+    assertEquals("Starting of reconfiguration task successful on 2 nodes, failed on 0 nodes.",
+        outsForStartReconf.get(2));
+
+    Thread.sleep(1000);
+    final List<String> outs = new ArrayList<>();
+    final List<String> errs = new ArrayList<>();
+    awaitReconfigurationFinished("datanode", "livenodes", outs, errs);
+    assertEquals(9, outs.size());
+    assertEquals(0, errs.size());
+    LOG.info("dfsadmin -status -livenodes output:");
+    outs.forEach(s -> LOG.info("{}", s));
+    assertTrue(outs.get(0).startsWith("Reconfiguring status for node"));
+    assertTrue("SUCCESS: Changed property dfs.datanode.peer.stats.enabled".equals(outs.get(2))
+        || "SUCCESS: Changed property dfs.datanode.peer.stats.enabled".equals(outs.get(1)));
+    assertTrue("\tFrom: \"false\"".equals(outs.get(3)) || "\tFrom: \"false\"".equals(outs.get(2)));
+    assertTrue("\tTo: \"true\"".equals(outs.get(4)) || "\tTo: \"true\"".equals(outs.get(3)));
+    assertEquals("SUCCESS: Changed property dfs.datanode.peer.stats.enabled", outs.get(5));
+    assertEquals("\tFrom: \"false\"", outs.get(6));
+    assertEquals("\tTo: \"true\"", outs.get(7));
+    assertEquals("Retrieval of reconfiguration status successful on 2 nodes, failed on 0 nodes.",
+        outs.get(8));
+  }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
index 6b8657ccce3b4..5681f3bdb25eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
@@ -69,6 +69,20 @@ private void assertOutputMatches(String string) {
     err.reset();
   }
 
+  private void assertOutputMatches(String outMessage, String errMessage) {
+    String errOutput = new String(err.toByteArray(), Charsets.UTF_8);
+    String output = new String(out.toByteArray(), Charsets.UTF_8);
+
+    if (!errOutput.matches(errMessage) || !output.matches(outMessage)) {
+      fail("Expected output to match '" + outMessage + " and " + errMessage +
+              "' but err_output was:\n" + errOutput + "\n and output was: \n" +
+              output);
+    }
+
+    out.reset();
+    err.reset();
+  }
+
   private void setHAConf(Configuration conf, String nn1Addr, String nn2Addr) {
     conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
         "hdfs://" + NSID);
@@ -186,9 +200,9 @@ public void testSaveNamespaceNN1UpNN2Down() throws Exception {
 //
     exitCode = admin.run(new String[] {"-saveNamespace"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "Save namespace successful for.*" + newLine
-        + "Save namespace failed for.*" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Save namespace successful for.*" + newLine;
+    String errMessage = "Save namespace failed for ([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -204,9 +218,9 @@ public void testSaveNamespaceNN1DownNN2Up() throws Exception {
 
     exitCode = admin.run(new String[] {"-saveNamespace"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "Save namespace failed for.*" + newLine
-        + "Save namespace successful for.*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Save namespace failed for ([\\s\\S]*)" + newLine;
+    String outMessage = "Save namespace successful for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -223,7 +237,7 @@ public void testSaveNamespaceNN1DownNN2Down() throws Exception {
 
     exitCode = admin.run(new String[] {"-saveNamespace"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "Save namespace failed for.*";
+    message = "Save namespace failed for ([\\s\\S]*)";
     assertOutputMatches(message + newLine + message + newLine);
   }
 
@@ -253,22 +267,22 @@ public void testRestoreFailedStorageNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-restoreFailedStorage", "check"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "restoreFailedStorage is set to false for.*" + newLine
-        + "restoreFailedStorage failed for.*" + newLine;
+    String outMessage = "restoreFailedStorage is set to false for.*" + newLine;
+    String errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
     // Default is false
-    assertOutputMatches(message);
+    assertOutputMatches(outMessage, errMessage);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "true"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage is set to true for.*" + newLine
-        + "restoreFailedStorage failed for.*" + newLine;
-    assertOutputMatches(message);
+    outMessage = "restoreFailedStorage is set to true for.*" + newLine;
+    errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "false"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage is set to false for.*" + newLine
-        + "restoreFailedStorage failed for.*" + newLine;
-    assertOutputMatches(message);
+    outMessage = "restoreFailedStorage is set to false for.*" + newLine;
+    errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -277,22 +291,22 @@ public void testRestoreFailedStorageNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-restoreFailedStorage", "check"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "restoreFailedStorage failed for.*" + newLine
-        + "restoreFailedStorage is set to false for.*" + newLine;
+    String errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
+    String outMessage = "restoreFailedStorage is set to false for.*" + newLine;
     // Default is false
-    assertOutputMatches(message);
+    assertOutputMatches(outMessage, errMessage);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "true"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage failed for.*" + newLine
-        + "restoreFailedStorage is set to true for.*" + newLine;
-    assertOutputMatches(message);
+    errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
+    outMessage = "restoreFailedStorage is set to true for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "false"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage failed for.*" + newLine
-        + "restoreFailedStorage is set to false for.*" + newLine;
-    assertOutputMatches(message);
+    errMessage = "restoreFailedStorage failed for ([\\s\\S]*)" + newLine;
+    outMessage = "restoreFailedStorage is set to false for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -302,18 +316,18 @@ public void testRestoreFailedStorageNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-restoreFailedStorage", "check"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "restoreFailedStorage failed for.*";
+    String message = "restoreFailedStorage failed for ([\\s\\S]*)";
     // Default is false
     assertOutputMatches(message + newLine + message + newLine);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "true"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage failed for.*";
+    message = "restoreFailedStorage failed for ([\\s\\S]*)";
     assertOutputMatches(message + newLine + message + newLine);
 
     exitCode = admin.run(new String[] {"-restoreFailedStorage", "false"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    message = "restoreFailedStorage failed for.*";
+    message = "restoreFailedStorage failed for ([\\s\\S]*)";
     assertOutputMatches(message + newLine + message + newLine);
   }
 
@@ -332,9 +346,9 @@ public void testRefreshNodesNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshNodes"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh nodes successful for.*" + newLine
-        + "Refresh nodes failed for.*" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Refresh nodes successful for .*" + newLine;
+    String errMessage = "Refresh nodes failed for ([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -343,9 +357,9 @@ public void testRefreshNodesNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-refreshNodes"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh nodes failed for.*" + newLine
-        + "Refresh nodes successful for.*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Refresh nodes failed for ([\\s\\S]*)" + newLine;
+    String outMessage = "Refresh nodes successful for .*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -355,7 +369,7 @@ public void testRefreshNodesNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshNodes"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh nodes failed for.*";
+    String message = "Refresh nodes failed for ([\\s\\S]*)";
     assertOutputMatches(message + newLine + message + newLine);
   }
 
@@ -432,11 +446,11 @@ public void testMetaSaveNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Created metasave file dfs.meta in the log directory"
-        + " of namenode.*" + newLine
-        + "Created metasave file dfs.meta in the log directory"
-        + " of namenode.*failed" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Created metasave file dfs.meta in the log " +
+            "directory of namenode.*" + newLine;
+    String errMessage = "Created metasave file dfs.meta in the log " +
+            "directory of namenode.*failed" + newLine + ".*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -446,11 +460,11 @@ public void testMetaSaveNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Created metasave file dfs.meta in the log directory"
-        + " of namenode.*failed" + newLine
-        + "Created metasave file dfs.meta in the log directory"
-        + " of namenode.*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Created metasave file dfs.meta in the log " +
+            "directory of namenode.*failed" + newLine + ".*" + newLine;
+    String outMessage = "Created metasave file dfs.meta in the log " +
+            "directory of namenode.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -460,9 +474,8 @@ public void testMetaSaveNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Created metasave file dfs.meta in the log directory"
-        + " of namenode.*failed";
-    assertOutputMatches(message + newLine + message + newLine);
+    String message = "([\\s\\S]*)2 exceptions([\\s\\S]*)";
+    assertOutputMatches(message + newLine);
   }
 
   @Test (timeout = 30000)
@@ -480,9 +493,9 @@ public void testRefreshServiceAclNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshServiceAcl"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh service acl successful for.*" + newLine
-        + "Refresh service acl failed for.*" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Refresh service acl successful for.*" + newLine;
+    String errMessage = "Refresh service acl failed for([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -491,9 +504,9 @@ public void testRefreshServiceAclNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-refreshServiceAcl"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh service acl failed for.*" + newLine
-        + "Refresh service acl successful for.*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Refresh service acl failed for([\\s\\S]*)" + newLine;
+    String outMessage = "Refresh service acl successful for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -503,8 +516,8 @@ public void testRefreshServiceAclNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshServiceAcl"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh service acl failed for.*";
-    assertOutputMatches(message + newLine + message + newLine);
+    String message = "([\\s\\S]*)2 exceptions([\\s\\S]*)";
+    assertOutputMatches(message + newLine);
   }
 
 
@@ -523,11 +536,9 @@ public void testRefreshUserToGroupsMappingsNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshUserToGroupsMappings"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh user to groups mapping successful for.*"
-        + newLine
-        + "Refresh user to groups mapping failed for.*"
-        + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Refresh user to groups mapping successful for.*" + newLine;
+    String errMessage = "Refresh user to groups mapping failed for([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -536,11 +547,9 @@ public void testRefreshUserToGroupsMappingsNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-refreshUserToGroupsMappings"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh user to groups mapping failed for.*"
-        + newLine
-        + "Refresh user to groups mapping successful for.*"
-        + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Refresh user to groups mapping failed for([\\s\\S]*)" + newLine;
+    String outMessage = "Refresh user to groups mapping successful for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -550,8 +559,8 @@ public void testRefreshUserToGroupsMappingsNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshUserToGroupsMappings"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh user to groups mapping failed for.*";
-    assertOutputMatches(message + newLine + message + newLine);
+    String message = "([\\s\\S]*)2 exceptions([\\s\\S]*)";
+    assertOutputMatches(message + newLine);
   }
 
   @Test (timeout = 30000)
@@ -572,11 +581,11 @@ public void testRefreshSuperUserGroupsConfigurationNN1UpNN2Down()
     int exitCode = admin.run(
         new String[] {"-refreshSuperUserGroupsConfiguration"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh super user groups configuration successful for.*"
-        + newLine
-        + "Refresh super user groups configuration failed for.*"
-        + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Refresh super user groups configuration successful for.*"
+            + newLine;
+    String errMessage = "Refresh super user groups configuration failed for([\\s\\S]*)"
+            + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -587,11 +596,11 @@ public void testRefreshSuperUserGroupsConfigurationNN1DownNN2Up()
     int exitCode = admin.run(
         new String[] {"-refreshSuperUserGroupsConfiguration"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh super user groups configuration failed for.*"
-        + newLine
-        + "Refresh super user groups configuration successful for.*"
-        + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Refresh super user groups configuration failed for([\\s\\S]*)"
+            + newLine;
+    String outMessage = "Refresh super user groups configuration successful for.*"
+            + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -603,8 +612,8 @@ public void testRefreshSuperUserGroupsConfigurationNN1DownNN2Down()
     int exitCode = admin.run(
         new String[] {"-refreshSuperUserGroupsConfiguration"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh super user groups configuration failed for.*";
-    assertOutputMatches(message + newLine + message + newLine);
+    String message = "([\\s\\S]*)2 exceptions([\\s\\S]*)";
+    assertOutputMatches(message + newLine);
   }
 
   @Test (timeout = 30000)
@@ -622,9 +631,9 @@ public void testRefreshCallQueueNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshCallQueue"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh call queue successful for.*" + newLine
-        + "Refresh call queue failed for.*" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Refresh call queue successful for.*" + newLine;
+    String errMessage = "Refresh call queue failed for([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -633,9 +642,9 @@ public void testRefreshCallQueueNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(0);
     int exitCode = admin.run(new String[] {"-refreshCallQueue"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh call queue failed for.*" + newLine
-        + "Refresh call queue successful for.*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Refresh call queue failed for([\\s\\S]*)" + newLine;
+    String outMessage = "Refresh call queue successful for.*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -645,8 +654,8 @@ public void testRefreshCallQueueNN1DownNN2Down() throws Exception {
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-refreshCallQueue"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Refresh call queue failed for.*";
-    assertOutputMatches(message + newLine + message + newLine);
+    String message = "([\\s\\S]*)2 exceptions([\\s\\S]*)";
+    assertOutputMatches(message + newLine);
   }
 
   @Test (timeout = 30000)
@@ -671,9 +680,9 @@ public void testFinalizeUpgradeNN1UpNN2Down() throws Exception {
     cluster.getDfsCluster().transitionToActive(0);
     int exitCode = admin.run(new String[] {"-finalizeUpgrade"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Finalize upgrade successful for .*" + newLine
-        + "Finalize upgrade failed for .*" + newLine;
-    assertOutputMatches(message);
+    String outMessage = "Finalize upgrade successful for .*" + newLine;
+    String errMessage = "Finalize upgrade failed for ([\\s\\S]*)" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -683,9 +692,9 @@ public void testFinalizeUpgradeNN1DownNN2Up() throws Exception {
     cluster.getDfsCluster().transitionToActive(1);
     int exitCode = admin.run(new String[] {"-finalizeUpgrade"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = "Finalize upgrade failed for .*" + newLine
-        + "Finalize upgrade successful for .*" + newLine;
-    assertOutputMatches(message);
+    String errMessage = "Finalize upgrade failed for ([\\s\\S]*)" + newLine;
+    String outMessage = "Finalize upgrade successful for .*" + newLine;
+    assertOutputMatches(outMessage, errMessage);
   }
 
   @Test (timeout = 30000)
@@ -788,7 +797,8 @@ public void testListOpenFilesNN1DownNN2Down() throws Exception{
     cluster.getDfsCluster().shutdownNameNode(1);
     int exitCode = admin.run(new String[] {"-listOpenFiles"});
     assertNotEquals(err.toString().trim(), 0, exitCode);
-    String message = ".*" + newLine + "List open files failed." + newLine;
+    String message = "List open files failed." + newLine
+            + ".*" + newLine;
     assertOutputMatches(message);
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
index aa048f865c2de..d0edd175ec1ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.hdfs.tools;
 
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -70,6 +72,7 @@ public class TestDFSHAAdminMiniCluster {
   @Before
   public void setup() throws IOException {
     conf = new Configuration();
+    conf.setBoolean(DFS_HA_NN_NOT_BECOME_ACTIVE_IN_SAFEMODE, true);
     cluster = new MiniDFSCluster.Builder(conf)
         .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0)
         .build();
@@ -161,7 +164,28 @@ public void testObserverIllegalTransition() throws Exception {
     assertEquals(-1, runTool("-transitionToActive", "nn1"));
     assertFalse(nnode1.isActiveState());
   }
-  
+
+  /**
+   * Tests that a Namenode in safe mode should not be transfer to observer.
+   */
+  @Test
+  public void testObserverTransitionInSafeMode() throws Exception {
+    NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
+    DFSHAAdmin admin = new DFSHAAdmin();
+    admin.setConf(conf);
+    System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
+    int result = admin.run(
+        new String[]{"-transitionToObserver", "-forcemanual", "nn1"});
+    assertEquals("State transition returned: " + result, -1, result);
+
+    NameNodeAdapter.leaveSafeMode(cluster.getNameNode(0));
+    System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
+    int result1 = admin.run(
+        new String[]{"-transitionToObserver", "-forcemanual", "nn1"});
+    assertEquals("State transition returned: " + result1, 0, result1);
+    assertFalse(cluster.getNameNode(0).isInSafeMode());
+  }
+
   @Test
   public void testTryFailoverToSafeMode() throws Exception {
     conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, 
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
index 5c890a5481415..37cd38eedb843 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
@@ -17,15 +17,22 @@
  */
 package org.apache.hadoop.hdfs.tools;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -34,6 +41,8 @@
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
+import java.util.List;
+import java.util.Random;
 
 import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil.*;
 import static org.junit.Assert.assertEquals;
@@ -44,23 +53,16 @@ public class TestDebugAdmin {
   static private final String TEST_ROOT_DIR =
       new File(System.getProperty("test.build.data", "/tmp"),
           TestDebugAdmin.class.getSimpleName()).getAbsolutePath();
-
+  private Configuration conf = new Configuration();
   private MiniDFSCluster cluster;
-  private DistributedFileSystem fs;
   private DebugAdmin admin;
-  private DataNode datanode;
 
   @Before
   public void setUp() throws Exception {
     final File testRoot = new File(TEST_ROOT_DIR);
     testRoot.delete();
     testRoot.mkdirs();
-    Configuration conf = new Configuration();
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-    cluster.waitActive();
-    fs = cluster.getFileSystem();
     admin = new DebugAdmin(conf);
-    datanode = cluster.getDataNodes().get(0);
   }
 
   @After
@@ -92,8 +94,11 @@ private String runCmd(String[] cmd) throws Exception {
 
   @Test(timeout = 60000)
   public void testRecoverLease() throws Exception {
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
     assertEquals("ret: 1, You must supply a -path argument to recoverLease.",
         runCmd(new String[]{"recoverLease", "-retries", "1"}));
+    DistributedFileSystem fs = cluster.getFileSystem();
     FSDataOutputStream out = fs.create(new Path("/foo"));
     out.write(123);
     out.close();
@@ -103,6 +108,10 @@ public void testRecoverLease() throws Exception {
 
   @Test(timeout = 60000)
   public void testVerifyMetaCommand() throws Exception {
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    DataNode datanode = cluster.getDataNodes().get(0);
     DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef);
     FsDatasetSpi<?> fsd = datanode.getFSDataset();
     ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar"));
@@ -128,6 +137,10 @@ public void testVerifyMetaCommand() throws Exception {
 
   @Test(timeout = 60000)
   public void testComputeMetaCommand() throws Exception {
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
+    DistributedFileSystem fs = cluster.getFileSystem();
+    DataNode datanode = cluster.getDataNodes().get(0);
     DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef);
     FsDatasetSpi<?> fsd = datanode.getFSDataset();
     ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar"));
@@ -166,8 +179,137 @@ public void testComputeMetaCommand() throws Exception {
 
   @Test(timeout = 60000)
   public void testRecoverLeaseforFileNotFound() throws Exception {
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+    cluster.waitActive();
     assertTrue(runCmd(new String[] {
         "recoverLease", "-path", "/foo", "-retries", "2" }).contains(
         "Giving up on recoverLease for /foo after 1 try"));
   }
+
+  @Test(timeout = 60000)
+  public void testVerifyECCommand() throws Exception {
+    final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies.getByID(
+        SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
+    cluster = DFSTestUtil.setupCluster(conf, 6, 5, 0);
+    cluster.waitActive();
+    DistributedFileSystem fs = cluster.getFileSystem();
+
+    assertEquals("ret: 1, verifyEC -file <file> [-blockId <blk_Id>] " +
+        "[-skipFailureBlocks]  -file Verify HDFS erasure coding on all block groups of the file." +
+        "  -skipFailureBlocks specify will skip any block group failures during verify," +
+        "  and continues verify all block groups of the file," +
+        "  the default is not to skip failure blocks." +
+        "  -blockId specify blk_Id to verify for a specific one block group.",
+        runCmd(new String[]{"verifyEC"}));
+
+    assertEquals("ret: 1, File /bar does not exist.",
+        runCmd(new String[]{"verifyEC", "-file", "/bar"}));
+
+    fs.create(new Path("/bar")).close();
+    assertEquals("ret: 1, File /bar is not erasure coded.",
+        runCmd(new String[]{"verifyEC", "-file", "/bar"}));
+
+
+    final Path ecDir = new Path("/ec");
+    fs.mkdir(ecDir, FsPermission.getDirDefault());
+    fs.enableErasureCodingPolicy(ecPolicy.getName());
+    fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
+
+    assertEquals("ret: 1, File /ec is not a regular file.",
+        runCmd(new String[]{"verifyEC", "-file", "/ec"}));
+
+    fs.create(new Path(ecDir, "foo"));
+    assertEquals("ret: 1, File /ec/foo is not closed.",
+        runCmd(new String[]{"verifyEC", "-file", "/ec/foo"}));
+
+    final short repl = 1;
+    final long k = 1024;
+    final long m = k * k;
+    final long seed = 0x1234567L;
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_65535"), 65535, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_65535"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_256k"), 256 * k, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_256k"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_1m"), m, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_1m"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_2m"), 2 * m, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_2m"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_3m"), 3 * m, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_3m"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_5m"), 5 * m, repl, seed);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_5m"})
+        .contains("All EC block group status: OK"));
+    DFSTestUtil.createFile(fs, new Path(ecDir, "foo_6m"), (int) k, 6 * m, m, repl, seed);
+    assertEquals("ret: 0, Checking EC block group: blk_x;Status: OK" +
+            "Checking EC block group: blk_x;Status: OK" +
+            "All EC block group status: OK",
+        runCmd(new String[]{"verifyEC", "-file", "/ec/foo_6m"})
+            .replaceAll("blk_-[0-9]+", "blk_x;"));
+
+    Path corruptFile = new Path(ecDir, "foo_corrupt");
+    DFSTestUtil.createFile(fs, corruptFile, 5841961, repl, seed);
+    List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs, corruptFile);
+    assertEquals(1, blocks.size());
+    LocatedStripedBlock blockGroup = (LocatedStripedBlock) blocks.get(0);
+    LocatedBlock[] indexedBlocks = StripedBlockUtil.parseStripedBlockGroup(blockGroup,
+        ecPolicy.getCellSize(), ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits());
+    // Try corrupt block 0 in block group.
+    LocatedBlock toCorruptLocatedBlock = indexedBlocks[0];
+    ExtendedBlock toCorruptBlock = toCorruptLocatedBlock.getBlock();
+    DataNode datanode = cluster.getDataNode(toCorruptLocatedBlock.getLocations()[0].getIpcPort());
+    File blockFile = getBlockFile(datanode.getFSDataset(),
+        toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock());
+    File metaFile = getMetaFile(datanode.getFSDataset(),
+        toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock());
+    // Write error bytes to block file and re-generate meta checksum.
+    byte[] errorBytes = new byte[2097152];
+    new Random(seed).nextBytes(errorBytes);
+    FileUtils.writeByteArrayToFile(blockFile, errorBytes);
+    metaFile.delete();
+    runCmd(new String[]{"computeMeta", "-block", blockFile.getAbsolutePath(),
+        "-out", metaFile.getAbsolutePath()});
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_corrupt"})
+        .contains("Status: ERROR, message: EC compute result not match."));
+
+    // Specify -blockId.
+    Path newFile = new Path(ecDir, "foo_new");
+    DFSTestUtil.createFile(fs, newFile, (int) k, 6 * m, m, repl, seed);
+    blocks = DFSTestUtil.getAllBlocks(fs, newFile);
+    assertEquals(2, blocks.size());
+    blockGroup = (LocatedStripedBlock) blocks.get(0);
+    String blockName = blockGroup.getBlock().getBlockName();
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_new", "-blockId", blockName})
+        .contains("ret: 0, Checking EC block group: " + blockName + "Status: OK"));
+
+    // Specify -verifyAllFailures.
+    indexedBlocks = StripedBlockUtil.parseStripedBlockGroup(blockGroup,
+        ecPolicy.getCellSize(), ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits());
+    // Try corrupt block 0 in block group.
+    toCorruptLocatedBlock = indexedBlocks[0];
+    toCorruptBlock = toCorruptLocatedBlock.getBlock();
+    datanode = cluster.getDataNode(toCorruptLocatedBlock.getLocations()[0].getIpcPort());
+    blockFile = getBlockFile(datanode.getFSDataset(),
+        toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock());
+    metaFile = getMetaFile(datanode.getFSDataset(),
+        toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock());
+    metaFile.delete();
+    // Write error bytes to block file and re-generate meta checksum.
+    errorBytes = new byte[1048576];
+    new Random(0x12345678L).nextBytes(errorBytes);
+    FileUtils.writeByteArrayToFile(blockFile, errorBytes);
+    runCmd(new String[]{"computeMeta", "-block", blockFile.getAbsolutePath(),
+        "-out", metaFile.getAbsolutePath()});
+    // VerifyEC and set skipFailureBlocks.
+    LocatedStripedBlock blockGroup2 = (LocatedStripedBlock) blocks.get(1);
+    assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_new", "-skipFailureBlocks"})
+        .contains("ret: 1, Checking EC block group: " + blockGroup.getBlock().getBlockName() +
+            "Status: ERROR, message: EC compute result not match." +
+            "Checking EC block group: " + blockGroup2.getBlock().getBlockName() + "Status: OK"));
+  }
+
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
index 226e4861f45f5..6f30416ed60e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
@@ -164,7 +164,7 @@ public void testRecoveryMode() throws IOException {
   public void testStored() throws IOException {
     // reference edits stored with source code (see build.xml)
     final String cacheDir = System.getProperty("test.cache.data",
-        "build/test/cache");
+        "target/test-classes");
     // binary, XML, reparsed binary
     String editsStored = cacheDir + "/editsStored";
     String editsStoredParsedXml = cacheDir + "/editsStoredParsed.xml";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 5c91530bd9d58..047750de2250b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -83,8 +83,10 @@
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.net.NetUtils;
@@ -93,6 +95,7 @@
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.XMLUtils;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@@ -122,6 +125,7 @@
 import static org.apache.hadoop.fs.permission.FsAction.ALL;
 import static org.apache.hadoop.fs.permission.FsAction.EXECUTE;
 import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
+import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR;
 import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry;
 import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_NAME;
 import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY;
@@ -186,6 +190,12 @@ public static void createOriginalFSImage() throws IOException {
       conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
       conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
           "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
+      // fsimage with sub-section conf
+      conf.set(DFSConfigKeys.DFS_IMAGE_PARALLEL_LOAD_KEY, "true");
+      conf.set(DFSConfigKeys.DFS_IMAGE_PARALLEL_INODE_THRESHOLD_KEY, "1");
+      conf.set(DFSConfigKeys.DFS_IMAGE_PARALLEL_TARGET_SECTIONS_KEY, "4");
+      conf.set(DFSConfigKeys.DFS_IMAGE_PARALLEL_THREADS_KEY, "4");
+
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
       cluster.waitActive();
       DistributedFileSystem hdfs = cluster.getFileSystem();
@@ -556,7 +566,7 @@ public void testPBImageXmlWriter() throws IOException, SAXException,
     try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
       v.visit(r);
     }
-    SAXParserFactory spf = SAXParserFactory.newInstance();
+    SAXParserFactory spf = XMLUtils.newSecureSAXParserFactory();
     SAXParser parser = spf.newSAXParser();
     final String xml = output.toString();
     ECXMLHandler ecxmlHandler = new ECXMLHandler();
@@ -791,6 +801,13 @@ public void testPBDelimitedWriter() throws IOException, InterruptedException {
         new FileSystemTestHelper().getTestRootDir() + "/delimited.db");
   }
 
+  @Test
+  public void testParallelPBDelimitedWriter() throws Exception {
+    testParallelPBDelimitedWriter("");  // Test in memory db.
+    testParallelPBDelimitedWriter(new FileSystemTestHelper().getTestRootDir()
+        + "/parallel-delimited.db");
+  }
+
   @Test
   public void testCorruptionOutputEntryBuilder() throws IOException {
     PBImageCorruptionDetector corrDetector =
@@ -882,11 +899,10 @@ private void testPBDelimitedWriter(String db)
     final String DELIMITER = "\t";
     ByteArrayOutputStream output = new ByteArrayOutputStream();
 
-    try (PrintStream o = new PrintStream(output);
-        RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
+    try (PrintStream o = new PrintStream(output)) {
       PBImageDelimitedTextWriter v =
           new PBImageDelimitedTextWriter(o, DELIMITER, db);
-      v.visit(r);
+      v.visit(originalFsimage.getAbsolutePath());
     }
 
     Set<String> fileNames = new HashSet<>();
@@ -920,6 +936,37 @@ private void testPBDelimitedWriter(String db)
     assertEquals(writtenFiles.keySet(), fileNames);
   }
 
+  private void testParallelPBDelimitedWriter(String db) throws Exception{
+    String delimiter = "\t";
+    int numThreads = 4;
+
+    File parallelDelimitedOut = new File(tempDir, "parallelDelimitedOut");
+    if (OfflineImageViewerPB.run(new String[] {"-p", "Delimited",
+        "-i", originalFsimage.getAbsolutePath(),
+        "-o", parallelDelimitedOut.getAbsolutePath(),
+        "-delimiter", delimiter,
+        "-t", db,
+        "-m", String.valueOf(numThreads)}) != 0) {
+      throw new IOException("oiv returned failure outputting in parallel.");
+    }
+    MD5Hash parallelMd5 = MD5FileUtils.computeMd5ForFile(parallelDelimitedOut);
+
+    File serialDelimitedOut = new File(tempDir, "serialDelimitedOut");
+    if (db != "") {
+      db = db + "/../serial.db";
+    }
+    if (OfflineImageViewerPB.run(new String[] {"-p", "Delimited",
+        "-i", originalFsimage.getAbsolutePath(),
+        "-o", serialDelimitedOut.getAbsolutePath(),
+        "-t", db,
+        "-delimiter", delimiter}) != 0) {
+      throw new IOException("oiv returned failure outputting in serial.");
+    }
+    MD5Hash serialMd5 = MD5FileUtils.computeMd5ForFile(serialDelimitedOut);
+
+    assertEquals(parallelMd5, serialMd5);
+  }
+
   private void testPBCorruptionDetector(String db)
       throws IOException, InterruptedException {
     final String delimiter = "\t";
@@ -928,7 +975,7 @@ private void testPBCorruptionDetector(String db)
     try (PrintStream o = new PrintStream(output)) {
       PBImageCorruptionDetector v =
           new PBImageCorruptionDetector(o, delimiter, db);
-      v.visit(new RandomAccessFile(originalFsimage, "r"));
+      v.visit(originalFsimage.getAbsolutePath());
     }
 
     try (
@@ -982,13 +1029,13 @@ private void properINodeDelete(List<Long> idsToDelete, Document doc)
 
   private void deleteINodeFromXML(File inputFile, File outputFile,
       List<Long> corruptibleIds) throws Exception {
-    DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory docFactory = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
     Document doc = docBuilder.parse(inputFile);
 
     properINodeDelete(corruptibleIds, doc);
 
-    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    TransformerFactory transformerFactory = XMLUtils.newSecureTransformerFactory();
     Transformer transformer = transformerFactory.newTransformer();
     DOMSource source = new DOMSource(doc);
     StreamResult result = new StreamResult(outputFile);
@@ -1024,7 +1071,7 @@ private String testCorruptionDetectorRun(int runNumber,
     try (PrintStream o = new PrintStream(output)) {
       PBImageCorruptionDetector v =
           new PBImageCorruptionDetector(o, ",", db);
-      v.visit(new RandomAccessFile(corruptedImage, "r"));
+      v.visit(corruptedImage.getAbsolutePath());
     }
     return output.toString();
   }
@@ -1120,17 +1167,17 @@ public void testReverseXmlRoundTrip() throws Throwable {
     LOG.info("Creating reverseImage.xml=" + reverseImageXml.getAbsolutePath() +
         ", reverseImage=" + reverseImage.getAbsolutePath() +
         ", reverseImage2Xml=" + reverseImage2Xml.getAbsolutePath());
-    if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+    if (OfflineImageViewerPB.run(new String[] {"-p", "XML",
          "-i", originalFsimage.getAbsolutePath(),
          "-o", reverseImageXml.getAbsolutePath() }) != 0) {
       throw new IOException("oiv returned failure creating first XML file.");
     }
-    if (OfflineImageViewerPB.run(new String[] { "-p", "ReverseXML",
+    if (OfflineImageViewerPB.run(new String[] {"-p", "ReverseXML",
           "-i", reverseImageXml.getAbsolutePath(),
           "-o", reverseImage.getAbsolutePath() }) != 0) {
       throw new IOException("oiv returned failure recreating fsimage file.");
     }
-    if (OfflineImageViewerPB.run(new String[] { "-p", "XML",
+    if (OfflineImageViewerPB.run(new String[] {"-p", "XML",
         "-i", reverseImage.getAbsolutePath(),
         "-o", reverseImage2Xml.getAbsolutePath() }) != 0) {
       throw new IOException("oiv returned failure creating second " +
@@ -1139,7 +1186,7 @@ public void testReverseXmlRoundTrip() throws Throwable {
     // The XML file we wrote based on the re-created fsimage should be the
     // same as the one we dumped from the original fsimage.
     Assert.assertEquals("",
-      GenericTestUtils.getFilesDiff(reverseImageXml, reverseImage2Xml));
+        GenericTestUtils.getFilesDiff(reverseImageXml, reverseImage2Xml));
   }
 
   /**
@@ -1174,10 +1221,47 @@ public void testReverseXmlWrongLayoutVersion() throws Throwable {
     }
   }
 
+  /**
+   * Tests that the ReverseXML processor doesn't accept XML files without the SnapshotDiffSection.
+   */
+  @Test
+  public void testReverseXmlWithoutSnapshotDiffSection() throws Throwable {
+    File imageWSDS = new File(tempDir, "imageWithoutSnapshotDiffSection.xml");
+    try(PrintWriter writer = new PrintWriter(imageWSDS, "UTF-8")) {
+      writer.println("<?xml version=\"1.0\"?>");
+      writer.println("<fsimage>");
+      writer.println("<version>");
+      writer.println("<layoutVersion>-67</layoutVersion>");
+      writer.println("<onDiskVersion>1</onDiskVersion>");
+      writer.println("<oivRevision>545bbef596c06af1c3c8dca1ce29096a64608478</oivRevision>");
+      writer.println("</version>");
+      writer.println("<FileUnderConstructionSection></FileUnderConstructionSection>");
+      writer.println("<ErasureCodingSection></ErasureCodingSection>");
+      writer.println("<INodeSection><lastInodeId>91488</lastInodeId><numInodes>0</numInodes>" +
+              "</INodeSection>");
+      writer.println("<SecretManagerSection><currentId>90</currentId><tokenSequenceNumber>35" +
+              "</tokenSequenceNumber><numDelegationKeys>0</numDelegationKeys><numTokens>0" +
+              "</numTokens></SecretManagerSection>");
+      writer.println("<INodeReferenceSection></INodeReferenceSection>");
+      writer.println("<SnapshotSection><snapshotCounter>0</snapshotCounter><numSnapshots>0" +
+              "</numSnapshots></SnapshotSection>");
+      writer.println("<NameSection><namespaceId>326384987</namespaceId></NameSection>");
+      writer.println("<CacheManagerSection><nextDirectiveId>1</nextDirectiveId><numPools>0" +
+              "</numPools><numDirectives>0</numDirectives></CacheManagerSection>");
+      writer.println("<INodeDirectorySection></INodeDirectorySection>");
+      writer.println("</fsimage>");
+    }
+      OfflineImageReconstructor.run(imageWSDS.getAbsolutePath(),
+              imageWSDS.getAbsolutePath() + ".out");
+  }
+
   @Test
   public void testFileDistributionCalculatorForException() throws Exception {
     File fsimageFile = null;
     Configuration conf = new Configuration();
+    // Avoid using the same cluster dir to cause the global originalFsimage
+    // file to be cleared.
+    conf.set(HDFS_MINIDFS_BASEDIR, GenericTestUtils.getRandomizedTempPath());
     HashMap<String, FileStatus> files = Maps.newHashMap();
 
     // Create a initial fsimage file
@@ -1287,10 +1371,9 @@ public void testOfflineImageViewerForECPolicies() throws Exception {
     v.visit(new RandomAccessFile(originalFsimage, "r"));
     final String xml = output.toString();
 
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
-    InputSource is = new InputSource();
-    is.setCharacterStream(new StringReader(xml));
+    InputSource is = new InputSource(new StringReader(xml));
     Document dom = db.parse(is);
     NodeList ecSection = dom.getElementsByTagName(ERASURE_CODING_SECTION_NAME);
     assertEquals(1, ecSection.getLength());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java
index 4955846432a60..3af8e03d898e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java
@@ -21,7 +21,6 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
@@ -29,11 +28,9 @@
 import java.io.StringReader;
 import java.net.HttpURLConnection;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.HashMap;
 
-import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
 
@@ -50,11 +47,12 @@
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.XMLUtils;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@@ -225,7 +223,7 @@ public void testPBImageXmlWriterForAcl() throws Exception{
     PrintStream o = new PrintStream(output);
     PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
     v.visit(new RandomAccessFile(originalFsimage, "r"));
-    SAXParserFactory spf = SAXParserFactory.newInstance();
+    SAXParserFactory spf = XMLUtils.newSecureSAXParserFactory();
     SAXParser parser = spf.newSAXParser();
     final String xml = output.toString();
     parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
@@ -239,7 +237,7 @@ public void testPBDelimitedWriterForAcl() throws Exception {
     try (PrintStream o = new PrintStream(output)) {
       PBImageDelimitedTextWriter v =
           new PBImageDelimitedTextWriter(o, DELIMITER, "");  // run in memory.
-      v.visit(new RandomAccessFile(originalFsimage, "r"));
+      v.visit(originalFsimage.getAbsolutePath());
     }
 
     try (
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java
new file mode 100644
index 0000000000000..c4e5622f41326
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests OfflineImageViewer if the input fsimage has HDFS ErasureCodingPolicy
+ * entries.
+ */
+public class TestOfflineImageViewerForErasureCodingPolicy {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestOfflineImageViewerForErasureCodingPolicy.class);
+
+  private static File originalFsimage = null;
+  private static File tempDir;
+
+  /**
+   * Create a populated namespace for later testing. Save its contents to a
+   * data structure and store its fsimage location.
+   */
+  @BeforeClass
+  public static void createOriginalFSImage() throws IOException {
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+      conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, true);
+
+      File[] nnDirs = MiniDFSCluster.getNameNodeDirectory(
+          MiniDFSCluster.getBaseDirectory(), 0, 0);
+      tempDir = nnDirs[0];
+
+      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(10).build();
+      cluster.waitActive();
+      DistributedFileSystem hdfs = cluster.getFileSystem();
+
+      hdfs.enableErasureCodingPolicy("RS-6-3-1024k");
+      hdfs.enableErasureCodingPolicy("RS-3-2-1024k");
+
+      Path dir = new Path("/dir_wo_ec_rs63");
+      hdfs.mkdirs(dir);
+      hdfs.setErasureCodingPolicy(dir, "RS-6-3-1024k");
+
+      dir = new Path("/dir_wo_ec_rs63/sub_dir_1");
+      hdfs.mkdirs(dir);
+
+      dir = new Path("/dir_wo_ec_rs63/sub_dir_2");
+      hdfs.mkdirs(dir);
+
+      Path file = new Path("/dir_wo_ec_rs63/file_wo_ec_1");
+      try (FSDataOutputStream  o = hdfs.create(file)) {
+        o.write(123);
+      }
+
+      file = new Path("/dir_wo_ec_rs63/file_wo_ec_2");
+      try (FSDataOutputStream  o = hdfs.create(file)) {
+        o.write(123);
+      }
+
+      dir = new Path("/dir_wo_ec_rs32");
+      hdfs.mkdirs(dir);
+      hdfs.setErasureCodingPolicy(dir, "RS-3-2-1024k");
+
+      dir = new Path("/dir_wo_ec_rs32/sub_dir_1");
+      hdfs.mkdirs(dir);
+
+      file = new Path("/dir_wo_ec_rs32/file_wo_ec");
+      try (FSDataOutputStream  o = hdfs.create(file)) {
+        o.write(123);
+      }
+
+      dir = new Path("/dir_wo_rep");
+      hdfs.mkdirs(dir);
+
+      dir = new Path("/dir_wo_rep/sub_dir_1");
+      hdfs.mkdirs(dir);
+
+      file = new Path("/dir_wo_rep/file_rep");
+      try (FSDataOutputStream  o = hdfs.create(file)) {
+        o.write(123);
+      }
+
+      // Write results to the fsimage file
+      hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false);
+      hdfs.saveNamespace();
+
+      // Determine the location of the fsimage file
+      originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+          .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
+      if (originalFsimage == null) {
+        throw new RuntimeException("Didn't generate or can't find fsimage");
+      }
+      LOG.debug("original FS image file is " + originalFsimage);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+      }
+    }
+  }
+
+  @AfterClass
+  public static void deleteOriginalFSImage() throws IOException {
+    if (originalFsimage != null && originalFsimage.exists()) {
+      originalFsimage.delete();
+    }
+  }
+
+  @Test
+  public void testPBDelimitedWriterForErasureCodingPolicy() throws Exception {
+    String expected = DFSTestUtil.readResoucePlainFile(
+        "testErasureCodingPolicy.csv");
+    String result = readECPolicyFromFsimageFile();
+    assertEquals(expected, result);
+  }
+
+  private String readECPolicyFromFsimageFile() throws Exception {
+    StringBuilder builder = new StringBuilder();
+    String delemiter = "\t";
+
+    File delimitedOutput = new File(tempDir, "delimitedOutput");
+
+    if (OfflineImageViewerPB.run(new String[] {"-p", "Delimited",
+        "-i", originalFsimage.getAbsolutePath(),
+        "-o", delimitedOutput.getAbsolutePath(),
+        "-ec"}) != 0) {
+      throw new IOException("oiv returned failure creating " +
+          "delimited output with ec.");
+    }
+
+    try (InputStream input = new FileInputStream(delimitedOutput);
+         BufferedReader reader =
+             new BufferedReader(new InputStreamReader(input))) {
+      String line;
+      boolean header = true;
+      while ((line = reader.readLine()) != null) {
+        String[] fields = line.split(delemiter);
+        if (!header) {
+          String path = fields[0];
+          String ecPolicy = fields[12];
+          builder.append(path).append(",").append(ecPolicy).append("\n");
+        }
+        header = false;
+      }
+    }
+    return builder.toString();
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
index 9d8c82c63708a..d1922dff8216e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hdfs.util;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.StripedFileTestUtil;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
index 839f8946be06b..d27aecb54c083 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
@@ -21,8 +21,6 @@
 import static org.apache.hadoop.fs.permission.AclEntryType.*;
 import static org.apache.hadoop.fs.permission.FsAction.*;
 import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -48,8 +46,11 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus.Flags;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry;
 import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.ChunkedArrayList;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.Time;
 
@@ -128,7 +129,7 @@ public void testHdfsFileStatus() throws Exception {
         .isdir(true)
         .build();
 
-    assertFalse(hdfsFileStatus.isSymlink());
+    Assert.assertFalse(hdfsFileStatus.isSymlink());
     LambdaTestUtils.intercept(IOException.class,
         "Path " + hdfsFileStatus.getPath() + " is not a symbolic link",
         () -> hdfsFileStatus.getSymlink());
@@ -150,7 +151,7 @@ public void testHdfsFileStatus() throws Exception {
         .append("}")
         .toString();
 
-    assertEquals(expectString, hdfsFileStatus.toString());
+    Assert.assertEquals(expectString, hdfsFileStatus.toString());
   }
 
   @Test
@@ -391,6 +392,83 @@ public void testGetXAttrFromJson() throws IOException {
     Assert.assertArrayEquals(XAttrCodec.decodeValue("0x313131"), value);
   }
 
+  @Test
+  public void testSnapshotDiffReportListingEmptyReport() throws IOException {
+    SnapshotDiffReportListing report = new SnapshotDiffReportListing();
+    String jsonString = JsonUtil.toJsonString(report);
+    Map<?, ?> json = READER.readValue(jsonString);
+    SnapshotDiffReportListing parsed =
+        JsonUtilClient.toSnapshotDiffReportListing(json);
+
+    assertEquals(report, parsed);
+  }
+
+  @Test
+  public void testSnapshotDiffReportListing() throws IOException {
+    List<DiffReportListingEntry> mlist = new ChunkedArrayList<>();
+    List<DiffReportListingEntry> clist = new ChunkedArrayList<>();
+    List<DiffReportListingEntry> dlist = new ChunkedArrayList<>();
+    clist.add(new DiffReportListingEntry(
+        1L, 2L, DFSUtilClient.string2Bytes("dir1/file2"), false, null));
+    clist.add(new DiffReportListingEntry(
+        1L, 3L, DFSUtilClient.string2Bytes("dir1/file3"), false, null));
+    dlist.add(new DiffReportListingEntry(
+        1L, 4L, DFSUtilClient.string2Bytes("dir1/file4"), false, null));
+    dlist.add(new DiffReportListingEntry(
+        1L, 5L,
+        DFSUtilClient.string2Bytes("dir1/file5"),
+        true,
+        DFSUtilClient.string2Bytes("dir1/file6")));
+
+    SnapshotDiffReportListing report =
+        new SnapshotDiffReportListing(
+            DFSUtilClient.string2Bytes("dir1/file2"), mlist, clist, dlist, 3, true);
+    String jsonString = JsonUtil.toJsonString(report);
+    Map<?, ?> json = READER.readValue(jsonString);
+    SnapshotDiffReportListing parsed =
+        JsonUtilClient.toSnapshotDiffReportListing(json);
+
+    assertEquals(report, parsed);
+  }
+
+  private void assertEquals(
+      SnapshotDiffReportListing expected, SnapshotDiffReportListing actual) {
+    Assert.assertEquals(expected.getLastIndex(), actual.getLastIndex());
+    Assert.assertEquals(expected.getIsFromEarlier(), actual.getIsFromEarlier());
+    assertEquals(expected.getModifyList(), actual.getModifyList());
+    assertEquals(expected.getCreateList(), actual.getCreateList());
+    assertEquals(expected.getDeleteList(), actual.getDeleteList());
+    Assert.assertArrayEquals(expected.getLastPath(), actual.getLastPath());
+  }
+
+  private void assertEquals(
+      List<DiffReportListingEntry> expected, List<DiffReportListingEntry> actual) {
+    Assert.assertEquals(expected.size(), actual.size());
+
+    for (int i = 0; i < expected.size(); i++) {
+      DiffReportListingEntry a = expected.get(i);
+      DiffReportListingEntry b = actual.get(i);
+
+      Assert.assertEquals(a.getFileId(), b.getFileId());
+      Assert.assertEquals(a.getDirId(), b.getDirId());
+      Assert.assertEquals(a.isReference(), b.isReference());
+      if (a.getSourcePath() != null) {
+        Assert.assertArrayEquals(
+            DFSUtilClient.byteArray2bytes(a.getSourcePath()),
+            DFSUtilClient.byteArray2bytes(b.getSourcePath()));
+      } else {
+        Assert.assertArrayEquals(a.getSourcePath(), b.getSourcePath());
+      }
+      if (a.getTargetPath() != null) {
+        Assert.assertArrayEquals(
+            DFSUtilClient.byteArray2bytes(a.getTargetPath()),
+            DFSUtilClient.byteArray2bytes(b.getTargetPath()));
+      } else {
+        Assert.assertArrayEquals(a.getTargetPath(), b.getTargetPath());
+      }
+    }
+  }
+
   private void checkDecodeFailure(Map<String, Object> map) {
     try {
       JsonUtilClient.toDatanodeInfo(map);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtilClient.java
new file mode 100644
index 0000000000000..85bd1f857e987
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtilClient.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.util.JsonSerialization;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TestJsonUtilClient {
+  @Test
+  public void testToStringArray() {
+    List<String> strList = new ArrayList<String>(Arrays.asList("aaa", "bbb", "ccc"));
+
+    String[] strArr = JsonUtilClient.toStringArray(strList);
+    assertEquals("Expected 3 items in the array", 3, strArr.length);
+    assertEquals("aaa", strArr[0]);
+    assertEquals("bbb", strArr[1]);
+    assertEquals("ccc", strArr[2]);
+  }
+
+  @Test
+  public void testToBlockLocationArray() throws Exception {
+    BlockLocation blockLocation = new BlockLocation(
+        new String[] {"127.0.0.1:62870"},
+        new String[] {"127.0.0.1"},
+        null,
+        new String[] {"/default-rack/127.0.0.1:62870"},
+        null,
+        new StorageType[] {StorageType.DISK},
+        0, 1, false);
+
+    Map<String, Object> blockLocationsMap =
+        JsonUtil.toJsonMap(new BlockLocation[] {blockLocation});
+    String json = JsonUtil.toJsonString("BlockLocations", blockLocationsMap);
+    assertNotNull(json);
+    Map<?, ?> jsonMap = JsonSerialization.mapReader().readValue(json);
+
+    BlockLocation[] deserializedBlockLocations =
+        JsonUtilClient.toBlockLocationArray(jsonMap);
+    assertEquals(1, deserializedBlockLocations.length);
+    assertEquals(blockLocation.toString(),
+        deserializedBlockLocations[0].toString());
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index f43e541aca44a..c4f53b0561573 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -23,6 +23,7 @@
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
 import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkOpStatistics;
 import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkStatistics;
@@ -54,6 +55,7 @@
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Random;
@@ -521,6 +523,38 @@ public void testCreateWithNoDN() throws Exception {
     }
   }
 
+  @Test
+  public void testWebHdfsCreateWithInvalidPath() throws Exception {
+    final Configuration conf = WebHdfsTestUtil.createConf();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+    // A path name include duplicated slashes.
+    String path = "//tmp//file";
+    assertResponse(path);
+  }
+
+  private String getUri(String path) {
+    final String user = System.getProperty("user.name");
+    final StringBuilder uri = new StringBuilder(cluster.getHttpUri(0));
+    uri.append("/webhdfs/v1").
+        append(path).
+        append("?op=CREATE").
+        append("&user.name=" + user);
+    return uri.toString();
+  }
+
+  private void assertResponse(String path) throws IOException {
+    URL url = new URL(getUri(path));
+    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    conn.setRequestMethod("PUT");
+    // Assert response code.
+    assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, conn.getResponseCode());
+    // Assert exception.
+    Map<?, ?> response = WebHdfsFileSystem.jsonParse(conn, true);
+    assertEquals("InvalidPathException",
+        ((LinkedHashMap) response.get("RemoteException")).get("exception"));
+    conn.disconnect();
+  }
+
   /**
    * Test allow and disallow snapshot through WebHdfs. Verifying webhdfs with
    * Distributed filesystem methods.
@@ -723,6 +757,7 @@ public void testWebHdfsDeleteSnapshot() throws Exception {
   @Test
   public void testWebHdfsSnapshotDiff() throws Exception {
     final Configuration conf = WebHdfsTestUtil.createConf();
+    conf.setInt(DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT, 2);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
     cluster.waitActive();
     final DistributedFileSystem dfs = cluster.getFileSystem();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index 722ebad72d239..c7df1f7c1d2d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -91,6 +91,11 @@ protected String getDefaultWorkingDirectory() {
     return defaultWorkingDirectory;
   }
 
+  @Override
+  protected int getGlobalTimeout() {
+    return 60 * 1000;
+  }
+
   /** HDFS throws AccessControlException
    * when calling exist(..) on a path /foo/bar/file
    * but /foo/bar is indeed a file in HDFS.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index 5758fe7986099..531f1eb0df021 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -633,4 +633,20 @@ public void testCountNumOfAvailableNodes() {
     numNodes = cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes);
     assertEquals(12, numNodes);
   }
+
+  @Test
+  public void testAddAndRemoveNodeWithEmptyRack() {
+    DatanodeDescriptor n1 = DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d2/r3");
+    DatanodeDescriptor n2 = DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3");
+    DatanodeDescriptor n3 = DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3");
+
+    cluster.decommissionNode(n1);
+    assertEquals(6, cluster.getNumOfNonEmptyRacks());
+    cluster.decommissionNode(n2);
+    cluster.decommissionNode(n3);
+    assertEquals(5, cluster.getNumOfNonEmptyRacks());
+
+    cluster.recommissionNode(n1);
+    assertEquals(6, cluster.getNumOfNonEmptyRacks());
+  }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
index d410d3b045500..6806b6715b159 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
@@ -34,6 +34,8 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -47,7 +49,6 @@
 import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
 import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.event.Level;
@@ -93,7 +94,7 @@ public Set<String> getGroupsSet(String user) {
       LOG.info("Getting groups in MockUnixGroupsMapping");
       String g1 = user + (10 * i + 1);
       String g2 = user + (10 * i + 2);
-      Set<String> s = Sets.newHashSet(g1, g2);
+      Set<String> s = new HashSet<>(Arrays.asList(g1, g2));
       i++;
       return s;
     }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
index 2fe6102dee123..acd29655071cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
@@ -29,7 +29,6 @@
 import java.io.PipedOutputStream;
 import java.io.PrintStream;
 import java.lang.management.ManagementFactory;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.TimeoutException;
 
@@ -37,9 +36,6 @@
 import javax.management.ObjectName;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSTestUtil;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv
new file mode 100644
index 0000000000000..862a19cec8e67
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#dir,erasure coding policy
+/,-
+/dir_wo_ec_rs63,RS-6-3-1024k
+/dir_wo_ec_rs63/sub_dir_1,-
+/dir_wo_ec_rs63/sub_dir_2,-
+/dir_wo_ec_rs63/file_wo_ec_1,RS-6-3-1024k
+/dir_wo_ec_rs63/file_wo_ec_2,RS-6-3-1024k
+/dir_wo_ec_rs32,RS-3-2-1024k
+/dir_wo_ec_rs32/sub_dir_1,-
+/dir_wo_ec_rs32/file_wo_ec,RS-3-2-1024k
+/dir_wo_rep,-
+/dir_wo_rep/sub_dir_1,-
+/dir_wo_rep/file_rep,-
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 6142a1a566a94..e6bf314a23d9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -15788,7 +15788,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-report \[-live\] \[-dead\] \[-decommissioning\] \[-enteringmaintenance\] \[-inmaintenance\]:(.)*</expected-output>
+          <expected-output>^-report \[-live\] \[-dead\] \[-decommissioning\] \[-enteringmaintenance\] \[-inmaintenance\] \[-slownodes\]:(.)*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>
@@ -16723,19 +16723,19 @@
         <!-- miniDFS cluster started in CLITestHelper is set to match this output -->
         <comparator>
           <type>RegexpAcrossOutputComparator</type>
-          <expected-output>^Rack: \/rack1\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)</expected-output>
+          <expected-output>^Rack: \/rack1\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service</expected-output>
         </comparator>
         <comparator>
           <type>RegexpAcrossOutputComparator</type>
-          <expected-output>Rack: \/rack2\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)</expected-output>
+          <expected-output>Rack: \/rack2\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service</expected-output>
         </comparator>
         <comparator>
           <type>RegexpAcrossOutputComparator</type>
-          <expected-output>Rack: \/rack3\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)</expected-output>
+          <expected-output>Rack: \/rack3\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service</expected-output>
         </comparator>
         <comparator>
           <type>RegexpAcrossOutputComparator</type>
-          <expected-output>Rack: \/rack4\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)</expected-output>
+          <expected-output>Rack: \/rack4\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service\s*127\.0\.0\.1:\d+\s\([-.a-zA-Z0-9]+\)\sIn Service</expected-output>
         </comparator>
       </comparators>
     </test>
diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
index 4e459b652b29c..8631ebba575c2 100644
--- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
+++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
@@ -291,6 +291,14 @@
       <Bug pattern="EQ_COMPARETO_USE_OBJECT_EQUALS" />
      </Match>
 
+    <!--
+       Ignore untidy code generated by Avro
+    -->
+     <Match>
+       <Class name="org.apache.hadoop.mapreduce.jobhistory.JobSubmitted" />
+       <Bug pattern="NP_NULL_INSTANCEOF" />
+     </Match>
+
      <Match>
        <Class name="org.apache.hadoop.mapred.Task" />
        <Method name="reportFatalError" />
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml
new file mode 100644
index 0000000000000..ff74bb7afb254
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:57:31 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Common 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.2.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.2.4.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.2.4.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/slf4j/slf4j-reload4j/1.7.35/slf4j-reload4j-1.7.35.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Common 3.2.4 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+<package name="org.apache.hadoop.mapreduce">
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.api.protocolrecords">
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <interface name="CancelDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to the {@code ResourceManager} to cancel a
+ delegation token.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <interface name="CancelDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[The response from the {@code ResourceManager} to a cancelDelegationToken
+ request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <interface name="GetDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <interface name="RenewDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to renew a delegation token from
+ the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+  <interface name="RenewDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNextExpirationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNextExpirationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expTime" type="long"/>
+    </method>
+    <doc>
+    <![CDATA[The response to a renewDelegationToken call to the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.security">
+</package>
+<package name="org.apache.hadoop.yarn.proto">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.3.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.3.xml
new file mode 100644
index 0000000000000..9236c16299dea
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.3.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:14:03 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Common 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.3.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.3.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.3.3.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Common 3.3.3 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+<package name="org.apache.hadoop.mapreduce">
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.api.protocolrecords">
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <interface name="CancelDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to the {@code ResourceManager} to cancel a
+ delegation token.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <interface name="CancelDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[The response from the {@code ResourceManager} to a cancelDelegationToken
+ request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <interface name="GetDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <interface name="RenewDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to renew a delegation token from
+ the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+  <interface name="RenewDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNextExpirationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNextExpirationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expTime" type="long"/>
+    </method>
+    <doc>
+    <![CDATA[The response to a renewDelegationToken call to the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.security">
+</package>
+<package name="org.apache.hadoop.yarn.proto">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml
new file mode 100644
index 0000000000000..b8737e922030e
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:15:00 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Common 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.4.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.3.4.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Common 3.3.4 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+<package name="org.apache.hadoop.mapreduce">
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.api.protocolrecords">
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <interface name="CancelDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to the {@code ResourceManager} to cancel a
+ delegation token.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <interface name="CancelDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <doc>
+    <![CDATA[The response from the {@code ResourceManager} to a cancelDelegationToken
+ request.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <interface name="GetDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <interface name="RenewDelegationTokenRequest"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to renew a delegation token from
+ the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+  <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+  <interface name="RenewDelegationTokenResponse"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNextExpirationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNextExpirationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expTime" type="long"/>
+    </method>
+    <doc>
+    <![CDATA[The response to a renewDelegationToken call to the {@code ResourceManager}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.security">
+</package>
+<package name="org.apache.hadoop.yarn.proto">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml
new file mode 100644
index 0000000000000..239b01b46126d
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml
@@ -0,0 +1,28073 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:57:12 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Core 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.2.4.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.4.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/slf4j/slf4j-reload4j/1.7.35/slf4j-reload4j-1.7.35.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Core 3.2.4 -->
+<package name="org.apache.hadoop.filecache">
+  <!-- start class org.apache.hadoop.filecache.DistributedCache -->
+  <class name="DistributedCache" extends="org.apache.hadoop.mapreduce.filecache.DistributedCache"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DistributedCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a archive that has been localized to the conf.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="addLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a file that has been localized to the conf..  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <method name="createAllSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobCacheDir" type="java.io.File"/>
+      <param name="workDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method create symlinks for all files in a given dir in another
+ directory. Currently symlinks cannot be disabled. This is a NO-OP.
+
+ @param conf the configuration
+ @param jobCacheDir the target directory for creating symlinks
+ @param workDir the directory in which the symlinks are created
+ @throws IOException
+ @deprecated Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns {@link FileStatus} of a given cache file on hdfs. Internal to
+ MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return <code>FileStatus</code> of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns mtime of a given cache file on hdfs. Internal to MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return mtime of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setArchiveTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the archives to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of archives.
+ The order should be the same as the order in which the archives are added.]]>
+      </doc>
+    </method>
+    <method name="setFileTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the files to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of files.
+ The order should be the same as the order in which the files are added.]]>
+      </doc>
+    </method>
+    <method name="setLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized archives.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="setLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized files.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <field name="CACHE_FILES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALFILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALFILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALFILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_SYMLINK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_SYMLINK} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_SYMLINK}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Distribute application-specific large, read-only files efficiently.
+
+ <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+ </p>
+
+ <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ <code>DistributedCache</code> assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.</p>
+
+ <p>The framework will copy the necessary files on to the worker node before
+ any tasks for the job are executed on that node. Its efficiency stems from
+ the fact that the files are only copied once per job and the ability to
+ cache archives which are un-archived on the workers.</p>
+
+ <p><code>DistributedCache</code> can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism.  Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task.  In the current
+ version symlinks are always created.  If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used.  All
+ others will not even be downloaded.</p>
+
+ <p><code>DistributedCache</code> tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.</p>
+
+ <p>Here is an illustrative example on how to use the
+ <code>DistributedCache</code>:</p>
+ <p><blockquote><pre>
+     // Setting up the cache for the application
+
+     1. Copy the requisite files to the <code>FileSystem</code>:
+
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+     2. Setup the application's <code>JobConf</code>:
+
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job);
+
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+
+     public static class MapClass extends MapReduceBase
+     implements Mapper&lt;K, V, K, V&gt; {
+
+       private Path[] localArchives;
+       private Path[] localFiles;
+
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+
+       public void map(K key, V value,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+
+ </pre></blockquote>
+
+ It is also very common to use the DistributedCache by using
+ {@link org.apache.hadoop.util.GenericOptionsParser}.
+
+ This class includes methods that should be used by users
+ (specifically those mentioned in the example above, as well
+ as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
+ as well as methods intended for use by the MapReduce framework
+ (e.g., {@link org.apache.hadoop.mapred.JobClient}).
+
+ @see org.apache.hadoop.mapred.JobConf
+ @see org.apache.hadoop.mapred.JobClient
+ @see org.apache.hadoop.mapreduce.Job]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.filecache.DistributedCache -->
+</package>
+<package name="org.apache.hadoop.mapred">
+  <!-- start class org.apache.hadoop.mapred.ClusterStatus -->
+  <class name="ClusterStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getTaskTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of task trackers in the cluster.
+ 
+ @return the number of task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getActiveTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the active task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return an empty graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return 0 graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted task trackers in the cluster.
+ 
+ @return the number of blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getNumExcludedNodes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of excluded hosts in the cluster.
+ @return the number of excluded hosts in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getTTExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running map tasks in the cluster.
+ 
+ @return the number of currently running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running reduce tasks in the cluster.
+ 
+ @return the number of currently running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running map tasks in the cluster.
+ 
+ @return the maximum capacity for running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running reduce tasks in the cluster.
+ 
+ @return the maximum capacity for running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker]]>
+      </doc>
+    </method>
+    <method name="getMaxMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getUsedMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTrackersInfo" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the list of blacklisted trackers along with reasons for blacklisting.
+ 
+ @return the collection of {@link BlackListInfo} objects.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerState" return="org.apache.hadoop.mapred.JobTracker.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current state of the <code>JobTracker</code>,
+ as {@link JobTracker.State}
+
+ {@link JobTracker.State} should no longer be used on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return the invalid state of the <code>JobTracker</code>.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="UNINITIALIZED_MEMORY_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterStatus</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster. 
+   </li>
+   <li>
+   Name of the trackers. 
+   </li>
+   <li>
+   Task capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   State of the <code>JobTracker</code>.
+   </li>
+   <li>
+   Details regarding black listed trackers.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterStatus</code>, via 
+ {@link JobClient#getClusterStatus()}.</p>
+ 
+ @see JobClient]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ClusterStatus -->
+  <!-- start class org.apache.hadoop.mapred.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getGroup" return="org.apache.hadoop.mapred.Counters.Group"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+    </method>
+    <method name="getGroupNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="makeCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String, String)} instead">
+      <param name="group" type="java.lang.String"/>
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter by using strings
+ @param group the name of the group
+ @param id the id of the counter within the group (0 to N-1)
+ @param name the internal name of the counter
+ @return the counter for that name
+ @deprecated use {@link #findCounter(String, String)} instead]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param key identifies a counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param group the name of the group
+ @param counter the internal name of the counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Returns current value of the specified counter, or 0 if the counter
+ does not exist.
+ @param key the counter enum to lookup
+ @return the counter value or 0 if counter not found]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #countCounters()} instead">
+      <doc>
+      <![CDATA[@return the total number of counters
+ @deprecated use {@link #countCounters()} instead]]>
+      </doc>
+    </method>
+    <method name="sum" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.mapred.Counters"/>
+      <param name="b" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Convenience method for computing the sum of two sets of counters.
+ @param a the first counters
+ @param b the second counters
+ @return a new summed counters object]]>
+      </doc>
+    </method>
+    <method name="log"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <doc>
+      <![CDATA[Logs the current counter values.
+ @param log The log to use.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Represent the counter in a textual format that can be converted back to
+ its object form
+ @return the string in the following format
+ {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*]]>
+      </doc>
+    </method>
+    <method name="fromEscapedCompactString" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compactString" type="java.lang.String"/>
+      <exception name="ParseException" type="java.text.ParseException"/>
+      <doc>
+      <![CDATA[Convert a stringified (by {@link #makeEscapedCompactString()} counter
+ representation into a counter object.
+ @param compactString to parse
+ @return a new counters object
+ @throws ParseException]]>
+      </doc>
+    </method>
+    <field name="MAX_COUNTER_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_GROUP_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A set of named counters.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Counter -->
+  <class name="Counters.Counter" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.Counter"/>
+    <constructor name="Counter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="increment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the compact stringified version of the counter in the format
+ [(actual-name)(display-name)(value)]
+ @return the stringified result]]>
+      </doc>
+    </method>
+    <method name="contentEquals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+      <doc>
+      <![CDATA[Checks for (content) equality of two (basic) counters
+ @param counter to compare
+ @return true if content equals
+ @deprecated]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the value of the counter]]>
+      </doc>
+    </method>
+    <method name="getUnderlyingCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A counter record, comprising its name and value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Counter -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Group -->
+  <class name="Counters.Group" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <constructor name="Group"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param counterName the name of the counter
+ @return the value of the specified counter, or 0 if the counter does
+ not exist.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the compact stringified version of the group in the format
+ {(actual-name)(display-name)(value)[][][]} where [] are compact strings
+ for the counters within.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String)} instead">
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given id and create it if it doesn't exist.
+ @param id the numeric id of the counter within the group
+ @param name the internal counter name
+ @return the counter
+ @deprecated use {@link #findCounter(String)} instead]]>
+      </doc>
+    </method>
+    <method name="getCounterForName" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given name and create it if it doesn't exist.
+ @param name the internal counter name
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="addCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    </method>
+    <method name="getUnderlyingGroup" return="org.apache.hadoop.mapreduce.counters.CounterGroupBase"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>Group</code> of counters, comprising of counters from a particular
+  counter {@link Enum} class.
+
+  <p><code>Group</code>handles localization of the class name and the
+  counter names.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Group -->
+  <!-- start class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.mapred.FileInputFormat -->
+  <class name="FileInputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setMinSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSize" type="long"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Splits files returned by {@link #listStatus(JobConf)} when
+ they're too big.]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="goalSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="blockSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job. 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getSplitHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+      <param name="splitSize" type="long"/>
+      <param name="clusterMap" type="org.apache.hadoop.net.NetworkTopology"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This function identifies and returns the hosts that contribute 
+ most for a given split. For calculating the contribution, rack
+ locality is treated on par with host locality, so hosts from racks
+ that contribute the most are preferred over hosts on racks that 
+ contribute less
+ @param blkLocations The list of block locations
+ @param offset 
+ @param splitSize 
+ @return an array of hosts that contribute most to this split
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}.
+ 
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(FileSystem, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapred.OutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <param name="outputPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="runState" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param conf the {@link JobConf} to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param conf the {@link JobConf} to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param conf the {@link JobConf} to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param conf the {@link JobConf} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(JobConf)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p><i>Note:</i> The following is valid only if the {@link OutputCommitter}
+  is {@link FileOutputCommitter}. If <code>OutputCommitter</code> is not 
+  a <code>FileOutputCommitter</code>, the task's temporary output
+  directory is same as {@link #getOutputPath(JobConf)} i.e.
+  <tt>${mapreduce.output.fileoutputformat.outputdir}$</tt></p>
+  
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in <tt>${mapreduce.task.output.dir}</tt> during execution 
+ of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the 
+ framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p><i>Note</i>: the value of <tt>${mapreduce.task.output.dir}</tt> during 
+ execution of a particular task-attempt is actually 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}</tt>, and this value is 
+ set by the map-reduce framework. So, just create any side-files in the 
+ path  returned by {@link #getWorkOutputPath(JobConf)} from map/reduce 
+ task to take advantage of this feature.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Helper function to create the task's temporary output directory and 
+ return the path to the task's output file.
+ 
+ @param conf job-configuration
+ @param name temporary task-output filename
+ @return path to the task's temporary output file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getUniqueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a name that is unique for the task.
+
+ <p>The generated name can be used to create custom files from within the
+ different tasks for the job, the names for different tasks will not collide
+ with each other.</p>
+
+ <p>The given name is postfixed with the task type, 'm' for maps, 'r' for
+ reduces and the task partition number. For example, give a name 'test'
+ running on the first map o the job the generated name will be
+ 'test-m-00000'.</p>
+
+ @param conf the configuration for the job.
+ @param name the name to make unique.
+ @return a unique name accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getPathForCustomFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueName} method to make the file name
+ unique for the task.</p>
+
+ @param conf the configuration for the job.
+ @param name the name for the file.
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplitWithLocationInfo"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[Constructs a split.
+ @deprecated
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobConf, int)} and passed to
+ {@link InputFormat#getRecordReader(InputSplit,JobConf,Reporter)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileSplit -->
+  <!-- start class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.ID -->
+  <class name="ID" extends="org.apache.hadoop.mapreduce.ID"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ID -->
+  <!-- start interface org.apache.hadoop.mapred.InputFormat -->
+  <interface name="InputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple.
+ 
+ @param job job configuration.
+ @param numSplits the desired number of splits, a hint.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordReader} for the given {@link InputSplit}.
+
+ <p>It is the responsibility of the <code>RecordReader</code> to respect
+ record boundaries while processing the logical split to present a 
+ record-oriented view to the individual task.</p>
+ 
+ @param split the {@link InputSplit}
+ @param job the job that this split belongs to
+ @return a {@link RecordReader}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to be respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibilty to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see JobClient
+ @see FileInputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplit -->
+  <interface name="InputSplit"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the total number of bytes in the data of the <code>InputSplit</code>.
+ 
+ @return the number of bytes in the input split.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hostnames where the input split is located.
+ 
+ @return list of hostnames where data of the <code>InputSplit</code> is
+         located as an array of <code>String</code>s.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplit -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <interface name="InputSplitWithLocationInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <!-- start class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <class name="InvalidFileTypeException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidFileTypeException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidFileTypeException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when file type differs from the desired file type. like 
+ getting a file when a directory is expected. Or a wrong file type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <class name="InvalidJobConfException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidJobConfException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when jobconf misses some mendatory attributes
+ or value of some attributes is invalid.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <!-- start class org.apache.hadoop.mapred.JobClient -->
+  <class name="JobClient" extends="org.apache.hadoop.mapreduce.tools.CLI"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="JobClient"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job client.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link JobConf}, and connect to the 
+ default cluster
+ 
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link Configuration}, 
+ and connect to the default cluster
+ 
+ @param conf the configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client, connect to the indicated job tracker.
+ 
+ @param jobTrackAddr the job tracker to connect to.
+ @param conf configuration.]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Connect to the default cluster
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>JobClient</code>.]]>
+      </doc>
+    </method>
+    <method name="getFs" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a filesystem handle.  We need this to prepare jobs
+ for submission to the MapReduce system.
+ 
+ @return the filesystem handle.]]>
+      </doc>
+    </method>
+    <method name="getClusterHandle" return="org.apache.hadoop.mapreduce.Cluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a handle to the Cluster]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobFile" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ 
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param jobFile the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws InvalidJobConfException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobInner" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an {@link RunningJob} object to track an ongoing job.  Returns
+ null if the id does not correspond to any known job.
+
+ @param jobid the jobid of the job.
+ @return the {@link RunningJob} handle to track the job, null if the
+         <code>jobid</code> doesn't correspond to any known job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getJob(JobID)}.">
+      <param name="jobid" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getJob(JobID)}.]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the map tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getMapTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the reduce tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the reduce tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCleanupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the cleanup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the cleanup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSetupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the setup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the setup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getReduceTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param jobId the ID of the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="detailed" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @param  detailed if true then get a detailed status including the
+         tracker names
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobsToComplete" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are not completed and not failed.
+ 
+ @return array of {@link JobStatus} for the running/to-be-run jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are submitted.
+ 
+ @return array of {@link JobStatus} for the submitted jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility that submits a job, then polls for progress until the job is
+ complete.
+ 
+ @param job the job configuration.
+ @throws IOException if the job fails]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="job" type="org.apache.hadoop.mapred.RunningJob"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @param conf the job's configuration
+ @param job the job to track
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Sets the output filter for tasks. only those tasks are printed whose
+ output matches the filter. 
+ @param newValue task filter.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the task output filter out of the JobConf.
+ 
+ @param job the JobConf to examine.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the JobConf to set the task output filter.
+ 
+ @param job the JobConf to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task output filter.
+ @return task filter.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cntrs" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Maps in the cluster.
+  
+ @return the max available Maps in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDefaultReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Reduces in the cluster.
+  
+ @return the max available Reduces in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where job-specific files are to be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="isJobDirValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobDirPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if the job directory is clean and has all the required components
+ for (re) starting the job]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fetch the staging area directory for the application
+ 
+ @return path to staging area directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about root level queues
+ configured
+
+ @return the array of root level JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about immediate children
+ of queue queueName.
+ 
+ @param queueName
+ @return the array of immediate children JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of queue information objects about all the Job Queues
+ configured.
+ 
+ @return Array of JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobsFromQueue" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets all the jobs which were added to particular Job Queue
+ 
+ @param queueName name of the Job Queue
+ @return Array of jobs present in the job queue
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.mapred.JobQueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the queue information associated to a particular Job Queue
+ 
+ @param queueName name of the job queue.
+ @return Queue information associated to particular queue.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapred.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return true if the renewal went well
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>JobClient</code> is the primary interface for the user-job to interact
+ with the cluster.
+ 
+ <code>JobClient</code> provides facilities to submit jobs, track their 
+ progress, access component-tasks' reports/logs, get the Map-Reduce cluster
+ status information etc.
+ 
+ <p>The job submission process involves:
+ <ol>
+   <li>
+   Checking the input and output specifications of the job.
+   </li>
+   <li>
+   Computing the {@link InputSplit}s for the job.
+   </li>
+   <li>
+   Setup the requisite accounting information for the {@link DistributedCache} 
+   of the job, if necessary.
+   </li>
+   <li>
+   Copying the job's jar and configuration to the map-reduce system directory 
+   on the distributed file-system. 
+   </li>
+   <li>
+   Submitting the job to the cluster and optionally monitoring
+   it's status.
+   </li>
+ </ol>
+  
+ Normally the user creates the application, describes various facets of the
+ job via {@link JobConf} and then uses the <code>JobClient</code> to submit 
+ the job and monitor its progress.
+ 
+ <p>Here is an example on how to use <code>JobClient</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+ </pre></blockquote>
+ 
+ <b id="JobControl">Job Control</b>
+ 
+ <p>At times clients would chain map-reduce jobs to accomplish complex tasks 
+ which cannot be done via a single map-reduce job. This is fairly easy since 
+ the output of the job, typically, goes to distributed file-system and that 
+ can be used as the input for the next job.</p>
+ 
+ <p>However, this also means that the onus on ensuring jobs are complete 
+ (success/failure) lies squarely on the clients. In such situations the 
+ various job-control options are:
+ <ol>
+   <li>
+   {@link #runJob(JobConf)} : submits the job and returns only after 
+   the job has completed.
+   </li>
+   <li>
+   {@link #submitJob(JobConf)} : only submits the job, then poll the 
+   returned handle to the {@link RunningJob} to query status and make 
+   scheduling decisions.
+   </li>
+   <li>
+   {@link JobConf#setJobEndNotificationURI(String)} : setup a notification
+   on job-completion, thus avoiding polling.
+   </li>
+ </ol>
+ 
+ @see JobConf
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobClient -->
+  <!-- start class org.apache.hadoop.mapred.JobConf -->
+  <class name="JobConf" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new map/reduce configuration where the behavior of reading from the
+ default resources can be turned off.
+ <p>
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user jar for the map-reduce job.
+ 
+ @return the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user jar for the map-reduce job.
+ 
+ @param jar the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getJarUnpackPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pattern for jar contents to unpack on the tasktracker]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the job's jar file by finding an example class location.
+ 
+ @param cls the example class.]]>
+      </doc>
+    </method>
+    <method name="getLocalDirs" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use MRAsyncDiskService.moveAndDeleteAllVolumes instead.]]>
+      </doc>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subdir" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs a local file name. Files are distributed among configured
+ local directories.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setKeepFailedTaskFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should keep the intermediate files for 
+ failed tasks.
+ 
+ @param keep <code>true</code> if framework should keep the intermediate files 
+             for failed tasks, <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="getKeepFailedTaskFiles" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the temporary files for failed tasks be kept?
+ 
+ @return should the files be kept?]]>
+      </doc>
+    </method>
+    <method name="setKeepTaskFilesPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set a regular expression for task names that should be kept. 
+ The regular expression ".*_m_000123_0" would keep the files
+ for the first instance of map 123 that ran.
+ 
+ @param pattern the java.util.regex.Pattern to match against the 
+        task names.]]>
+      </doc>
+    </method>
+    <method name="getKeepTaskFilesPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the regular expression that is matched against the task names
+ to see if we need to keep the files.
+ 
+ @return the pattern as a string, if it was set, othewise null.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="setNumTasksToExecutePerJvm"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numTasks" type="int"/>
+      <doc>
+      <![CDATA[Sets the number of tasks that a spawned task JVM should run
+ before it exits
+ @param numTasks the number of tasks to execute; defaults to 1;
+ -1 signifies no limit]]>
+      </doc>
+    </method>
+    <method name="getNumTasksToExecutePerJvm" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of tasks that a spawned JVM should execute]]>
+      </doc>
+    </method>
+    <method name="getInputFormat" return="org.apache.hadoop.mapred.InputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link InputFormat} implementation for the map-reduce job,
+ defaults to {@link TextInputFormat} if not specified explicity.
+ 
+ @return the {@link InputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link InputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormat" return="org.apache.hadoop.mapred.OutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} implementation for the map-reduce job,
+ defaults to {@link TextOutputFormat} if not specified explicity.
+ 
+ @return the {@link OutputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapred.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} implementation for the map-reduce job,
+ defaults to {@link FileOutputCommitter} if not specified explicitly.
+ 
+ @return the {@link OutputCommitter} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputCommitter} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputCommitter} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setOutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setCompressMapOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Should the map outputs be compressed before transfer?
+ 
+ @param compress should the map outputs be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressMapOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are the outputs of the maps be compressed?
+ 
+ @return <code>true</code> if the outputs of the maps are to be compressed,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the given class as the  {@link CompressionCodec} for the map outputs.
+ 
+ @param codecClass the {@link CompressionCodec} class that will compress  
+                   the map outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the map outputs.
+ 
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} class that should be used to compress the 
+         map outputs.
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+  
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ 
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link RawComparator} comparator used to compare keys.
+ 
+ @param theClass the {@link RawComparator} comparator used to 
+                 compare keys.
+ @see #setOutputValueGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see #setCombinerKeyGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for 
+ grouping keys of inputs to the reduce.
+ 
+ @return comparator set by the user for grouping values.
+ @see #setOutputValueGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for
+ grouping keys in the input to the combiner.
+
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control
+ how keys are sorted, this can be used in conjunction to simulate
+ <i>secondary sort on values</i>.</p>
+
+ <p><i>Note</i>: This is not a guarantee of the combiner sort being
+ <i>stable</i> in any sense. (In any case, with the order of available
+ map-outputs to the combiner being non-deterministic, it wouldn't make
+ that much sense.)</p>
+
+ @param theClass the comparator class to be used for grouping keys for the
+ combiner. It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setOutputValueGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for 
+ grouping keys in the input to the reduce.
+ 
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to 
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+  
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+ 
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control 
+ how keys are sorted, this can be used in conjunction to simulate 
+ <i>secondary sort on values</i>.</p>
+  
+ <p><i>Note</i>: This is not a guarantee of the reduce sort being 
+ <i>stable</i> in any sense. (In any case, with the order of available 
+ map-outputs to the reduce being non-deterministic, it wouldn't make 
+ that much sense.)</p>
+ 
+ @param theClass the comparator class to be used for grouping keys. 
+                 It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)
+ @see #setCombinerKeyGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="getUseNewMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the mapper?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewMapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the mapper.
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getUseNewReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the reducer?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewReducer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the reducer. 
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ 
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} class for the job.
+ 
+ @param theClass the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapRunnerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link MapRunnable} class for the job.
+ 
+ @return the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapRunnerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Expert: Set the {@link MapRunnable} class for the job.
+ 
+ Typically used to exert greater control on {@link Mapper}s.
+ 
+ @param theClass the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Partitioner} used to partition {@link Mapper}-outputs 
+ to be sent to the {@link Reducer}s.
+ 
+ @return the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} class used to partition 
+ {@link Mapper}-outputs to be sent to the {@link Reducer}s.
+ 
+ @param theClass the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} class for the job.
+ 
+ @param theClass the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. Typically the combiner is same as the
+ the {@link Reducer} for the job i.e. {@link #getReducerClass()}.
+ 
+ @return the user-defined combiner class used to combine map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. 
+ 
+ <p>The combiner is an application-specified aggregation operation, which
+ can help cut down the amount of data transferred between the 
+ {@link Mapper} and the {@link Reducer}, leading to better performance.</p>
+ 
+ <p>The framework may invoke the combiner 0, 1, or multiple times, in both
+ the mapper and reducer tasks. In general, the combiner is called as the
+ sort/merge result is written to disk. The combiner must:
+ <ul>
+   <li> be side-effect free</li>
+   <li> have the same input and output key types and the same input and 
+        output value types</li>
+ </ul>
+ 
+ <p>Typically the combiner is same as the <code>Reducer</code> for the  
+ job i.e. {@link #setReducerClass(Class)}.</p>
+ 
+ @param theClass the user-defined combiner class used to combine 
+                 map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getMapSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for map tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be 
+                           used for this job for map tasks,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getReduceSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for reduce tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used 
+                           for reduce tasks for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getNumMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of map tasks for this job.
+ Defaults to <code>1</code>.
+ 
+ @return the number of map tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumMapTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the number of map tasks for this job.
+ 
+ <p><i>Note</i>: This is only a <i>hint</i> to the framework. The actual 
+ number of spawned map tasks depends on the number of {@link InputSplit}s 
+ generated by the job's {@link InputFormat#getSplits(JobConf, int)}.
+  
+ A custom {@link InputFormat} is typically used to accurately control 
+ the number of map tasks for the job.</p>
+ 
+ <b id="NoOfMaps">How many maps?</b>
+ 
+ <p>The number of maps is usually driven by the total size of the inputs 
+ i.e. total number of blocks of the input files.</p>
+  
+ <p>The right level of parallelism for maps seems to be around 10-100 maps 
+ per-node, although it has been set up to 300 or so for very cpu-light map 
+ tasks. Task setup takes awhile, so it is best if the maps take at least a 
+ minute to execute.</p>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of input files. However, the {@link FileSystem} blocksize of the 
+ input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+  
+ <p>Thus, if you expect 10TB of input data and have a blocksize of 128MB, 
+ you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is 
+ used to set it even higher.</p>
+ 
+ @param n the number of map tasks for this job.
+ @see InputFormat#getSplits(JobConf, int)
+ @see FileInputFormat
+ @see FileSystem#getDefaultBlockSize()
+ @see FileStatus#getBlockSize()]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of reduce tasks for this job. Defaults to
+ <code>1</code>.
+
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the requisite number of reduce tasks for this job.
+ 
+ <b id="NoOfReduces">How many reduces?</b>
+ 
+ <p>The right number of reduces seems to be <code>0.95</code> or 
+ <code>1.75</code> multiplied by (
+ <i>available memory for reduce tasks</i>
+ (The value of this should be smaller than
+ numNodes * yarn.nodemanager.resource.memory-mb
+ since the resource of memory is shared by map tasks and other
+ applications) /
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.reduce.memory.mb">
+ mapreduce.reduce.memory.mb</a>).
+ </p>
+ 
+ <p>With <code>0.95</code> all of the reduces can launch immediately and 
+ start transfering map outputs as the maps finish. With <code>1.75</code> 
+ the faster nodes will finish their first round of reduces and launch a 
+ second wave of reduces doing a much better job of load balancing.</p>
+ 
+ <p>Increasing the number of reduces increases the framework overhead, but 
+ increases load balancing and lowers the cost of failures.</p>
+ 
+ <p>The scaling factors above are slightly less than whole numbers to 
+ reserve a few reduce slots in the framework for speculative-tasks, failures
+ etc.</p> 
+
+ <b id="ReducerNone">Reducer NONE</b>
+ 
+ <p>It is legal to set the number of reduce-tasks to <code>zero</code>.</p>
+ 
+ <p>In this case the output of the map-tasks directly go to distributed 
+ file-system, to the path set by 
+ {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the 
+ framework doesn't sort the map-outputs before writing it out to HDFS.</p>
+ 
+ @param n the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapreduce.map.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapreduce.reduce.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.]]>
+      </doc>
+    </method>
+    <method name="getSessionId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified session identifier. The default is the empty string.
+
+ The session identifier is used to tag metric data that is reported to some
+ performance metrics system via the org.apache.hadoop.metrics API.  The 
+ session identifier is intended, in particular, for use by Hadoop-On-Demand 
+ (HOD) which allocates a virtual Hadoop cluster dynamically and transiently. 
+ HOD will set the session identifier by modifying the mapred-site.xml file 
+ before starting the cluster.
+
+ When not running under HOD, this identifer is expected to remain set to 
+ the empty string.
+
+ @return the session identifier, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setSessionId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sessionId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified session identifier.  
+
+ @param sessionId the new session id.]]>
+      </doc>
+    </method>
+    <method name="setMaxTaskFailuresPerTracker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="noFailures" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds <code>noFailures</code>, the 
+ tasktracker is <i>blacklisted</i> for this job. 
+ 
+ @param noFailures maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxTaskFailuresPerTracker" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Expert: Get the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds this, the tasktracker is
+ <i>blacklisted</i> for this job. 
+ 
+ @return the maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of map tasks that can fail without 
+ the job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts()} 
+ attempts before being declared as <i>failed</i>.
+  
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed map-task results in
+ the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of map tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the maximum percentage of map tasks that can fail without the
+ job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts} attempts 
+ before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of map tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of reduce tasks that can fail without 
+ the job being aborted. 
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed reduce-task results 
+ in the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of reduce tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum percentage of reduce tasks that can fail without the job
+ being aborted.
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of reduce tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="setJobPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="int"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link JobPriority} for this job.
+
+ @return the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriorityAsInteger" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this job.
+
+ @return the priority for this job.]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setMapDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the map tasks fail.
+ 
+ <p>The debug script can aid debugging of failed map tasks. The script is 
+ given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script needs to be symlinked. </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param mDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getMapDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the map task's debug script.
+ 
+ @return the debug Script for the mapred job for failed map tasks.
+ @see #setMapDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="setReduceDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the reduce tasks fail.
+ 
+ <p>The debug script can aid debugging of failed reduce tasks. The script
+ is given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script file needs to be symlinked </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param rDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getReduceDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reduce task's debug Script
+ 
+ @return the debug script for the mapred job for failed reduce tasks.
+ @see #setReduceDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationURI" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the uri to be invoked in-order to send a notification after the job 
+ has completed (success/failure). 
+ 
+ @return the job end notification uri, <code>null</code> if it hasn't
+         been set.
+ @see #setJobEndNotificationURI(String)]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationURI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the uri to be invoked in-order to send a notification after the job
+ has completed (success/failure).
+ 
+ <p>The uri can contain 2 special parameters: <tt>$jobId</tt> and 
+ <tt>$jobStatus</tt>. Those, if present, are replaced by the job's 
+ identifier and completion-status respectively.</p>
+ 
+ <p>This is typically used by application-writers to implement chaining of 
+ Map-Reduce jobs in an <i>asynchronous manner</i>.</p>
+ 
+ @param uri the job end notification uri
+ @see JobStatus]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationCustomNotifierClass" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class to be invoked in order to send a notification
+ after the job has completed (success/failure).
+
+ @return the fully-qualified name of the class which implements
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier} set through the
+ {@link org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS}
+ property
+
+ @see JobConf#setJobEndNotificationCustomNotifierClass(java.lang.String)
+ @see org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationCustomNotifierClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="customNotifierClassName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the class to be invoked in order to send a notification after the job
+ has completed (success/failure).
+
+ A notification url still has to be set which will be passed to
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier#notifyOnce(
+ java.net.URL, org.apache.hadoop.conf.Configuration)}
+ along with the Job's conf.
+
+ If this is set instead of using a simple HttpURLConnection
+ we'll create a new instance of this class
+ which should be an implementation of
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier},
+ and we'll invoke that.
+
+ @param customNotifierClassName the fully-qualified name of the class
+     which implements
+     {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier}
+
+ @see JobConf#setJobEndNotificationURI(java.lang.String)
+ @see
+ org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="getJobLocalDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get job-specific shared directory for use as scratch space
+ 
+ <p>
+ When a job starts, a shared directory is created at location
+ <code>
+ ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ </code>.
+ This directory is exposed to the users through 
+ <code>mapreduce.job.local.dir </code>.
+ So, the tasks can use this space 
+ as scratch space and share files among them. </p>
+ This value is available as System property also.
+ 
+ @return The localized job specific shared directory]]>
+      </doc>
+    </method>
+    <method name="getMemoryForMapTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a map task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_MAP_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a map task of the job, in MB,]]>
+      </doc>
+    </method>
+    <method name="setMemoryForMapTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getMemoryForReduceTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a reduce task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_REDUCE_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a reduce task of the job, in MB.]]>
+      </doc>
+    </method>
+    <method name="setMemoryForReduceTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the queue to which this job is submitted.
+ Defaults to 'default'.
+ 
+ @return name of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the queue to which this job should be submitted.
+ 
+ @param queueName Name of the queue]]>
+      </doc>
+    </method>
+    <method name="normalizeMemoryConfigValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[Normalize the negative values in configuration
+ 
+ @param val
+ @return normalized value]]>
+      </doc>
+    </method>
+    <method name="findContainingJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="my_class" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Find a jar that contains a class of the same name, if any.
+ It will return a jar file, even if that is not the first thing
+ on the class path that has a class with the same name.
+ 
+ @param my_class the class to find.
+ @return a jar file that contains the class, or null.]]>
+      </doc>
+    </method>
+    <method name="getMaxVirtualMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}">
+      <doc>
+      <![CDATA[Get the memory required to run a task of this job, in bytes. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ This method is deprecated. Now, different memory limits can be
+ set for map and reduce tasks of a job, in MB. 
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. 
+ Otherwise, this method will return the larger of the values returned by 
+ {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()}
+ after converting them into bytes.
+
+ @return Memory required to run a task of this job, in bytes.
+ @see #setMaxVirtualMemoryForTask(long)
+ @deprecated Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}]]>
+      </doc>
+    </method>
+    <method name="setMaxVirtualMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}">
+      <param name="vmem" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum amount of memory any task of this job can use. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ mapred.task.maxvmem is split into
+ mapreduce.map.memory.mb
+ and mapreduce.map.memory.mb,mapred
+ each of the new key are set
+ as mapred.task.maxvmem / 1024
+ as new values are in MB
+
+ @param vmem Maximum amount of virtual memory in bytes any task of this job
+             can use.
+ @see #getMaxVirtualMemoryForTask()
+ @deprecated
+  Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}]]>
+      </doc>
+    </method>
+    <method name="getMaxPhysicalMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="this variable is deprecated and nolonger in use.">
+      <doc>
+      <![CDATA[@deprecated this variable is deprecated and nolonger in use.]]>
+      </doc>
+    </method>
+    <method name="setMaxPhysicalMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPRED_TASK_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UPPER_LIMIT_ON_TASK_VMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_MAXPMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="DISABLED_MEMORY_LIMIT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A value which if set for memory related configuration options,
+ indicates that the options are turned off.
+ Deprecated because it makes no sense in the context of MR2.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_LOCAL_DIR_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Property name for the configuration property mapreduce.cluster.local.dir]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of the queue to which jobs will be submitted, if no queue
+ name is mentioned.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_MAP_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UNPACK_JAR_PATTERN_DEFAULT" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Pattern for the default unpacking behavior for job jars]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the child
+ map and reduce tasks.
+ 
+ Java opts for the task tracker child processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass 
+ other environment variables to the child processes.
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the map tasks.
+ 
+ Java opts for the task tracker child map processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass 
+ other environment variables to the map processes.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the reduce tasks.
+ 
+ Java opts for the task tracker child reduce processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to 
+ pass process environment variables to the reduce processes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPRED_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map/reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.map.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.reduce.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the map task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the reduce task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default logging level for map/reduce tasks.]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ID} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_STRING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_TAGS} instead]]>
+      </doc>
+    </field>
+    <field name="MAPREDUCE_RECOVER_JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPREDUCE_RECOVER_JOB" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A map/reduce job configuration.
+ 
+ <p><code>JobConf</code> is the primary interface for a user to describe a 
+ map-reduce job to the Hadoop framework for execution. The framework tries to
+ faithfully execute the job as-is described by <code>JobConf</code>, however:
+ <ol>
+   <li>
+   Some configuration parameters might have been marked as 
+   <a href="{@docRoot}/org/apache/hadoop/conf/Configuration.html#FinalParams">
+   final</a> by administrators and hence cannot be altered.
+   </li>
+   <li>
+   While some job parameters are straight-forward to set 
+   (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly
+   with the rest of the framework and/or job-configuration and is relatively
+   more complex for the user to control finely
+   (e.g. {@link #setNumMapTasks(int)}).
+   </li>
+ </ol>
+ 
+ <p><code>JobConf</code> typically specifies the {@link Mapper}, combiner 
+ (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and 
+ {@link OutputFormat} implementations to be used etc.
+
+ <p>Optionally <code>JobConf</code> is used to specify other advanced facets 
+ of the job such as <code>Comparator</code>s to be used, files to be put in  
+ the {@link DistributedCache}, whether or not intermediate and/or job outputs 
+ are to be compressed (and how), debugability via user-provided scripts 
+ ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}),
+ for doing post-processing on task logs, task's stdout, stderr, syslog. 
+ and etc.</p>
+ 
+ <p>Here is an example on how to configure a job via <code>JobConf</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+ </pre></blockquote>
+ 
+ @see JobClient
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobConf -->
+  <!-- start interface org.apache.hadoop.mapred.JobConfigurable -->
+  <interface name="JobConfigurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Initializes a new instance from a {@link JobConf}.
+
+ @param job the configuration]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[That what may be configured.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobConfigurable -->
+  <!-- start interface org.apache.hadoop.mapred.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job Configuration
+ 
+ @return JobConf]]>
+      </doc>
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress mechanism for reporting progress.
+ 
+ @return progress mechanism]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobContext -->
+  <!-- start class org.apache.hadoop.mapred.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapreduce.JobID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.JobID"/>
+      <doc>
+      <![CDATA[Downgrade a new JobID to an old one
+ @param old a new or old JobID
+ @return either old or a new JobID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getJobIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>any job</i> 
+ run on the jobtracker started at <i>200707121733</i>, we would use :
+ <pre> 
+ JobID.getTaskIDsPattern("200707121733", null);
+ </pre>
+ which will return :
+ <pre> "job_200707121733_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @return a regex pattern matching JobIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local".
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobID -->
+  <!-- start class org.apache.hadoop.mapred.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobPriority -->
+  <!-- start class org.apache.hadoop.mapred.JobQueueInfo -->
+  <class name="JobQueueInfo" extends="org.apache.hadoop.mapreduce.QueueInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobQueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Job Queue Info.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobQueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new JobQueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <method name="getQueueState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use getState() instead]]>
+      </doc>
+    </method>
+    <method name="getChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobQueueInfo -->
+  <!-- start class org.apache.hadoop.mapred.JobStatus -->
+  <class name="JobStatus" extends="org.apache.hadoop.mapreduce.JobStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on cleanup
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="getJobRunState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[Helper method to get human-readable state of the job.
+ @param state job state
+ @return human-readable state of the job]]>
+      </doc>
+    </method>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.mapreduce.JobStatus"/>
+    </method>
+    <method name="getJobId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use getJobID instead">
+      <doc>
+      <![CDATA[@deprecated use getJobID instead]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="getRunState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCEEDED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KILLED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Describes the current status of a job.  This is
+ not intended to be a comprehensive piece of data.
+ For that, look at JobProfile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobStatus -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line. Each line
+ is divided into key and value parts by a separator byte. If no such a byte
+ exists, the key will be the entire line and value will be empty.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapred.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.Mapper -->
+  <interface name="Mapper"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Maps a single input key/value pair into an intermediate key/value pair.
+ 
+ <p>Output pairs need not be of the same types as input pairs.  A given 
+ input pair may map to zero or many output pairs.  Output pairs are 
+ collected with calls to 
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes significant amount of time to process individual key/value
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the input key.
+ @param value the input value.
+ @param output collects mapped keys and values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link JobConf} for the 
+ job via the {@link JobConfigurable#configure(JobConf)} and initialize
+ themselves. Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+ 
+ <p>The framework then calls 
+ {@link #map(Object, Object, OutputCollector, Reporter)} 
+ for each key/value pair in the <code>InputSplit</code> for that task.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the grouping by specifying
+ a <code>Comparator</code> via 
+ {@link JobConf#setOutputKeyComparatorClass(Class)}.</p>
+
+ <p>The grouped <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>The intermediate, grouped outputs are always stored in 
+ {@link SequenceFile}s. Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>JobConf</code>.</p>
+  
+ <p>If the job has 
+ <a href="{@docRoot}/org/apache/hadoop/mapred/JobConf.html#ReducerNone">zero
+ reduces</a> then the output of the <code>Mapper</code> is directly written
+ to the {@link FileSystem} without grouping by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyMapper&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Mapper&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Process the &lt;key, value&gt; pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of &lt;key, value&gt; pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+ </pre></blockquote>
+
+ <p>Applications may write a custom {@link MapRunnable} to exert greater
+ control on map processing e.g. multi-threaded <code>Mapper</code>s etc.</p>
+ 
+ @see JobConf
+ @see InputFormat
+ @see Partitioner  
+ @see Reducer
+ @see MapReduceBase
+ @see MapRunnable
+ @see SequenceFile]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Mapper -->
+  <!-- start class org.apache.hadoop.mapred.MapReduceBase -->
+  <class name="MapReduceBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="MapReduceBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for {@link Mapper} and {@link Reducer} implementations.
+ 
+ <p>Provides default no-op implementations for a few methods, most non-trivial
+ applications need to override some of them.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapReduceBase -->
+  <!-- start interface org.apache.hadoop.mapred.MapRunnable -->
+  <interface name="MapRunnable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="run"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Start mapping input <tt>&lt;key, value&gt;</tt> pairs.
+  
+ <p>Mapping of input records to output records is complete when this method 
+ returns.</p>
+ 
+ @param input the {@link RecordReader} to read the input records.
+ @param output the {@link OutputCollector} to collect the outputrecords.
+ @param reporter {@link Reporter} to report progress, status-updates etc.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Expert: Generic interface for {@link Mapper}s.
+ 
+ <p>Custom implementations of <code>MapRunnable</code> can exert greater 
+ control on map processing e.g. multi-threaded, asynchronous mappers etc.</p>
+ 
+ @see Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.MapRunnable -->
+  <!-- start class org.apache.hadoop.mapred.MapRunner -->
+  <class name="MapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMapper" return="org.apache.hadoop.mapred.Mapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Default {@link MapRunnable} implementation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapRunner -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <class name="MultiFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link MultiFileSplit}'s
+ in {@link #getSplits(JobConf, int)} method. Splits are constructed from 
+ the files under the input paths. Each split returned contains <i>nearly</i>
+ equal content length. <br>  
+ Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>MultiFileSplit</code>'s.
+ @see MultiFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileSplit -->
+  <class name="MultiFileSplit" extends="org.apache.hadoop.mapred.lib.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. Unlike {@link FileSplit}, MultiFileSplit 
+ class does not represent a split of a file, but a split of input files 
+ into smaller sets. The atomic unit of split is a file. <br> 
+ MultiFileSplit can be used to implement {@link RecordReader}'s, with 
+ reading one record per file.
+ @see FileSplit
+ @see MultiFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileSplit -->
+  <!-- start interface org.apache.hadoop.mapred.OutputCollector -->
+  <interface name="OutputCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="collect"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a key/value pair to the output.
+
+ @param key the key to collect.
+ @param value to value to collect.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Collects the <code>&lt;key, value&gt;</code> pairs output by {@link Mapper}s
+ and {@link Reducer}s.
+  
+ <p><code>OutputCollector</code> is the generalization of the facility 
+ provided by the Map-Reduce framework to collect data output by either the 
+ <code>Mapper</code> or the <code>Reducer</code> i.e. intermediate outputs 
+ or the output of the job.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputCollector -->
+  <!-- start class org.apache.hadoop.mapred.OutputCommitter -->
+  <class name="OutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="status" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus#FAILED} or 
+ {@link JobStatus#KILLED}. This is called from the application
+ master process for the entire job. This may be called multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @param status final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task. This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.
+ 
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRConstants#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>. This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.
+ @deprecated Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="runState" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapred.OutputFormat -->
+  <interface name="OutputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given job.
+
+ @param ignored
+ @param job configuration for the job whose output is being written.
+ @param name the unique name for this part of the output.
+ @param progress mechanism for reporting progress while writing to file.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ @param ignored
+ @param job job configuration.
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter
+ @see JobConf]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.OutputLogFilter -->
+  <class name="OutputLogFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="OutputLogFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[This class filters log files from directory given
+ It doesnt accept paths having _logs.
+ This can be used to list paths of output directory as follows:
+   Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
+                                   new OutputLogFilter()));]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputLogFilter -->
+  <!-- start interface org.apache.hadoop.mapred.Partitioner -->
+  <interface name="Partitioner"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the paritition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be paritioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Partitioner -->
+  <!-- start interface org.apache.hadoop.mapred.RecordReader -->
+  <interface name="RecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads the next key/value pair from the input for processing.
+
+ @param key the key to read data into
+ @param value the value to read data into
+ @return true iff a key/value was read, false if at EOF]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a key.
+ 
+ @return a new key object.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a value.
+ 
+ @return a new value object.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current position in the input.
+ 
+ @return the current position in the input.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this {@link InputSplit} to future operations.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[How much of the input has the {@link RecordReader} consumed i.e.
+ has been processed by?
+ 
+ @return progress from <code>0.0</code> to <code>1.0</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordReader</code> reads &lt;key, value&gt; pairs from an 
+ {@link InputSplit}.
+   
+ <p><code>RecordReader</code>, typically, converts the byte-oriented view of 
+ the input, provided by the <code>InputSplit</code>, and presents a 
+ record-oriented view for the {@link Mapper} and {@link Reducer} tasks for
+ processing. It thus assumes the responsibility of processing record 
+ boundaries and presenting the tasks with keys and values.</p>
+ 
+ @see InputSplit
+ @see InputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordReader -->
+  <!-- start interface org.apache.hadoop.mapred.RecordWriter -->
+  <interface name="RecordWriter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param reporter facility to report progress.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapred.Reducer -->
+  <interface name="Reducer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="reduce"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<i>Reduces</i> values for a given key.  
+ 
+ <p>The framework calls this method for each 
+ <code>&lt;key, (list of values)&gt;</code> pair in the grouped inputs.
+ Output values must be of the same type as input values.  Input keys must 
+ not be altered. The framework will <b>reuse</b> the key and value objects
+ that are passed into the reduce, therefore the application should clone
+ the objects they want to keep a copy of. In many cases, all values are 
+ combined into zero or one value.
+ </p>
+   
+ <p>Output pairs are collected with calls to  
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes a significant amount of time to process individual key/value 
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the key.
+ @param values the list of values to reduce.
+ @param output to collect keys and combined values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p>The number of <code>Reducer</code>s for the job is set by the user via 
+ {@link JobConf#setNumReduceTasks(int)}. <code>Reducer</code> implementations 
+ can access the {@link JobConf} for the job via the 
+ {@link JobConfigurable#configure(JobConf)} method and initialize themselves. 
+ Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p><code>Reducer</code> is input the grouped output of a {@link Mapper}.
+   In the phase the framework, for each <code>Reducer</code>, fetches the 
+   relevant partition of the output of all the <code>Mapper</code>s, via HTTP. 
+   </p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework groups <code>Reducer</code> inputs by <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key) in this
+   stage.</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>If equivalence rules for keys while grouping the intermediates are 
+   different from those for grouping keys before reduction, then one may 
+   specify a <code>Comparator</code> via 
+   {@link JobConf#setOutputValueGroupingComparator(Class)}.Since 
+   {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to 
+   control how intermediate keys are grouped, these can be used in conjunction 
+   to simulate <i>secondary sort on values</i>.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterator, OutputCollector, Reporter)}
+   method is called for each <code>&lt;key, (list of values)&gt;</code> pair in
+   the grouped inputs.</p>
+   <p>The output of the reduce task is typically written to the 
+   {@link FileSystem} via 
+   {@link OutputCollector#collect(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyReducer&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Reducer&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator&lt;V&gt; values,
+                          OutputCollector&lt;K, V&gt; output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the &lt;key, value&gt; pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the &lt;key, value&gt; 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of &lt;key, list of values&gt; pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner
+ @see Reporter
+ @see MapReduceBase]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reducer -->
+  <!-- start interface org.apache.hadoop.mapred.Reporter -->
+  <interface name="Reporter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the status description for the task.
+ 
+ @param status brief description of the current status.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param group counter group
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the key, which can be of
+ any {@link Enum} type, by the specified amount.
+ 
+ @param key key to identify the counter to be incremented. The key can be
+            be any <code>Enum</code>. 
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the group and counter name
+ by the specified amount.
+ 
+ @param group name to identify the group of the counter to be incremented.
+ @param counter name to identify the counter within the group.
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="getInputSplit" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Get the {@link InputSplit} object for a map.
+ 
+ @return the <code>InputSplit</code> that the map is reading from.
+ @throws UnsupportedOperationException if called outside a mapper]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress of the task. Progress is represented as a number between
+ 0 and 1 (inclusive).]]>
+      </doc>
+    </method>
+    <field name="NULL" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A constant of Reporter type that does nothing.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A facility for Map-Reduce applications to report progress and update 
+ counters, status information etc.
+ 
+ <p>{@link Mapper} and {@link Reducer} can use the <code>Reporter</code>
+ provided to report progress or just indicate that they are alive. In 
+ scenarios where the application takes significant amount of time to
+ process individual key/value pairs, this is crucial since the framework 
+ might assume that the task has timed-out and kill that task.
+
+ <p>Applications can also update {@link Counters} via the provided 
+ <code>Reporter</code> .</p>
+ 
+ @see Progressable
+ @see Counters]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reporter -->
+  <!-- start interface org.apache.hadoop.mapred.RunningJob -->
+  <interface name="RunningJob"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying job configuration
+
+ @return the configuration of the job.]]>
+      </doc>
+    </method>
+    <method name="getID" return="org.apache.hadoop.mapred.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job identifier.
+ 
+ @return the job identifier.]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.">
+      <doc>
+      <![CDATA[@deprecated This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the job.
+ 
+ @return the name of the job.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Blocks until the job is complete.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ {@link JobStatus}
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobStatus" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a snapshot of the current status, {@link JobStatus}, of the Job.
+ Need to call again for latest information.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job. Blocks until all job tasks have been killed as well.
+ If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param priority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @param shouldFail if true the task is failed and added to failed tasks 
+                   list, otherwise it is just killed, w/o affecting 
+                   job failure status.  
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #killTask(TaskAttemptID, boolean)}">
+      <param name="taskId" type="java.lang.String"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job.
+ 
+ @return the counters for this job or null if the job has been retired.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the url where history file is archived. Returns empty string if 
+ history file is not available yet. 
+ 
+ @return the url where history file is archived
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether the job has been removed from JobTracker memory and retired.
+ On retire, the job history file is copied to a location known by 
+ {@link #getHistoryUrl()}
+ @return <code>true</code> if the job retired, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get failure info for the job.
+ @return the failure info for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RunningJob</code> is the user-interface to query for details on a 
+ running Map-Reduce job.
+ 
+ <p>Clients can get hold of <code>RunningJob</code> via the {@link JobClient}
+ and then query the running-job for details such as name, configuration, 
+ progress etc.</p> 
+ 
+ @see JobClient]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RunningJob -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapred.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes keys, values to 
+ {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, 
+ except it generates SequenceFileAsTextRecordReader 
+ which converts the input keys and values to their 
+ String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileAsTextRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by calling toString()
+ method. This class to SequenceFileAsTextInputFormat class is as LineRecordReader
+ class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param job job configuration
+ @param reporter reporter who sends report to task tracker
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param conf application configuration
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.SequenceFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of key that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of value that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCurrentValue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SkipBadRecords -->
+  <class name="SkipBadRecords" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SkipBadRecords"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAttemptsToStartSkipping" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @return attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="setAttemptsToStartSkipping"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="attemptsToStartSkipping" type="int"/>
+      <doc>
+      <![CDATA[Set the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @param attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrMapperProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                       {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrMapperProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrReducerProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                    {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrReducerProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.]]>
+      </doc>
+    </method>
+    <method name="getSkipOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @return path skip output directory. Null is returned if this is not set 
+ and output directory is also not set.]]>
+      </doc>
+    </method>
+    <method name="setSkipOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @param path skip output directory path]]>
+      </doc>
+    </method>
+    <method name="getMapperMaxSkipRecords" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="setMapperMaxSkipRecords"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipRecs" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="getReducerMaxSkipGroups" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <method name="setReducerMaxSkipGroups"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipGrps" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <field name="COUNTER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Special counters which are written by the application and are 
+ used by the framework for detecting bad records. For detecting bad records 
+ these counters must be incremented by the application.]]>
+      </doc>
+    </field>
+    <field name="COUNTER_MAP_PROCESSED_RECORDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed map records.
+ @see SkipBadRecords#getAutoIncrMapperProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <field name="COUNTER_REDUCE_PROCESSED_GROUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed reduce groups.
+ @see SkipBadRecords#getAutoIncrReducerProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Utility class for skip bad records functionality. It contains various 
+ settings related to skipping of bad records.
+ 
+ <p>Hadoop provides an optional mode of execution in which the bad records
+ are detected and skipped in further attempts.
+ 
+ <p>This feature can be used when map/reduce tasks crashes deterministically on 
+ certain input. This happens due to bugs in the map/reduce function. The usual
+ course would be to fix these bugs. But sometimes this is not possible; 
+ perhaps the bug is in third party libraries for which the source code is 
+ not available. Due to this, the task never reaches to completion even with 
+ multiple attempts and complete data for that task is lost.</p>
+  
+ <p>With this feature, only a small portion of data is lost surrounding 
+ the bad record, which may be acceptable for some user applications.
+ see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}</p>
+ 
+ <p>The skipping mode gets kicked off after certain no of failures 
+ see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}</p>
+  
+ <p>In the skipping mode, the map/reduce task maintains the record range which 
+ is getting processed at all times. Before giving the input to the
+ map/reduce function, it sends this record range to the Task tracker.
+ If task crashes, the Task tracker knows which one was the last reported
+ range. On further attempts that range get skipped.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SkipBadRecords -->
+  <!-- start class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <class name="SplitLocationInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitLocationInfo" type="java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isOnDisk" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInMemory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <!-- start interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapred.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapreduce.TaskAttemptID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapred.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param taskId taskId number
+ @param id the task attempt number
+ @deprecated Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskAttemptID to an old one
+ @param old the new id
+ @return either old or a new TaskAttemptID constructed to match old]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType} 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="org.apache.hadoop.mapreduce.TaskCompletionEvent"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapred.TaskAttemptID, int, boolean, org.apache.hadoop.mapred.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getTaskAttemptId()} instead.">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id
+ @deprecated use {@link #getTaskAttemptId()} instead.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getTaskStatus" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="setTaskId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id.
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapred.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapreduce.TaskID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(String, int, TaskType, int)}">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(String, int, TaskType, int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskID to an old one
+ @param old a new or old TaskID
+ @return either old or a new TaskID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs
+ @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}]]>
+      </doc>
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType}, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs]]>
+      </doc>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+    </method>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskID -->
+  <!-- start class org.apache.hadoop.mapred.TaskReport -->
+  <class name="TaskReport" extends="org.apache.hadoop.mapreduce.TaskReport"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The string of the task id.]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The id of the task.]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setSuccessfulAttempt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[set successful attempt ID of the task.]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulTaskAttempt" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attempt ID that took this task to completion]]>
+      </doc>
+    </method>
+    <method name="setRunningTaskAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runningAttempts" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[set running attempt(s) of the task.]]>
+      </doc>
+    </method>
+    <method name="getRunningTaskAttempts" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the running task attempt IDs for this task]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[set finish time of task. 
+ @param finishTime finish time of task.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[set start time of the task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A report on the state of a task.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskReport -->
+  <!-- start class org.apache.hadoop.mapred.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Utils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A utility class. It provides
+   A path filter utility to filter out output/part files in the output dir]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Utils -->
+</package>
+<package name="org.apache.hadoop.mapred.jobcontrol">
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf, java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param jobConf a mapred job configuration representing a job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getAssignedJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setAssignedJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="setAssignedJobID should not be called.
+ JOBID is set by the framework.">
+      <param name="mapredJobID" type="org.apache.hadoop.mapred.JobID"/>
+      <doc>
+      <![CDATA[@deprecated setAssignedJobID should not be called.
+ JOBID is set by the framework.]]>
+      </doc>
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred job conf of this job]]>
+      </doc>
+    </method>
+    <method name="setJobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Set the mapred job conf for this job.
+ @param jobConf the mapred job conf for this job.]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[This is a no-op function, Its a behavior change from 1.x We no more can
+ change the state from job
+ 
+ @param state
+          the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="getJobClient" return="org.apache.hadoop.mapred.JobClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job client of this job]]>
+      </doc>
+    </method>
+    <method name="getDependingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="getMapredJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setMapredJobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapredJobID" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is no-op method for backward compatibility. It's a behavior change
+ from 1.x, we can not change job ids from job.
+ 
+ @param mapredJobID
+          the mapred job ID for this job.]]>
+      </doc>
+    </method>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="WAITING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="READY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEPENDENT_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+  <class name="JobControl" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJobs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapred.join">
+  <!-- start class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <interface name="ComposableInputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <interface name="ComposableRecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <implements name="java.lang.Comparable"/>
+    <method name="id" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key this RecordReader would supply on a call to next(K,V)]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RecordReader into the object provided.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the stream is not empty, but provides no guarantee that
+ a call to next(K,V) will succeed.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[While key-value pairs from this RecordReader match the given key, register
+ them with the JoinCollector provided.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapred.join.expr</tt> property and
+ user-supplied join types from <tt>mapred.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapred.join.define.&lt;ident&gt;</tt> to a classname. In the expression
+ <tt>mapred.join.expr</tt>, the identifier will be assumed to be a
+ ComposableRecordReader.
+ <tt>mapred.join.keycomparator</tt> can be a classname used to compare keys
+ in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapred.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for faliing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key value common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createInternalValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unsupported (returns zero in all cases).]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapred.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapred.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Node -->
+  <class name="Parser.Node" extends="java.lang.Object"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapred.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.TupleWritable -->
+  <class name="TupleWritable" extends="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="U"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write key-value pair at the head of this stream to the objects provided;
+ get next key-value pair from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new value from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request position from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapred.lib">
+  <!-- start class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}. 
+ 
+ @see org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <class name="ChainMapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ <p>
+
+ @param job              job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="java.lang.Object"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>map(...)</code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ <p>
+ ChainMapper usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <class name="ChainReducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="reducerConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Sets the Reducer class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Reducer leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Reducer does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+
+ @param job              job's JobConf to add the Reducer class.
+ @param klass            the Reducer class to add.
+ @param inputKeyClass    reducer input key class.
+ @param inputValueClass  reducer input value class.
+ @param outputKeyClass   reducer output key class.
+ @param outputValueClass reducer output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param reducerConf      a JobConf with the configuration for the Reducer
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ .
+
+ @param job              chain job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>reduce(...)</code> method of the Reducer with the
+ <code>map(...) </code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion, the output of the first becomes the input of the
+ second, and so on until the last Mapper, the output of the last Mapper will
+ be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain.
+ This enables having reusable specialized Mappers that can be combined to
+ perform composite operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ <p>
+ ChainReducer usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(List)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(List)}.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(PathFilter...)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(PathFilter...)}.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link org.apache.hadoop.mapred.InputFormat} that returns {@link CombineFileSplit}'s
+ in {@link org.apache.hadoop.mapred.InputFormat#getSplits(JobConf, int)} method. 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default spliting behaviour in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement {@link org.apache.hadoop.mapred.InputFormat#getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>CombineFileSplit</code>'s.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.mapred.Reporter, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return the amount of data processed]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapred.JobConf"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="reporter" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapred.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapred.FileInputFormat, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.Reporter, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <class name="FieldSelectionMapReduce" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="FieldSelectionMapReduce"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <!-- start class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapred.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the supplied output format.
+ @param out the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapred.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <class name="HashPartitioner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <class name="IdentityMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="IdentityMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identity function.  Input key/value pair is written directly to
+ output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the identity function, mapping inputs directly to outputs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <class name="IdentityReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="IdentityReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all keys and values directly to output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Performs no reduction, writing all input values directly to the output.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.mapreduce.lib.partition.InputSampler"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSampler" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="sampler" type="org.apache.hadoop.mapred.lib.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InputSampler -->
+  <!-- start class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapred.lib.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link JobConf} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} that sums long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <class name="MultipleOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a composite record writer that can write key/value data to different
+ output files
+ 
+ @param fs
+          the file system to use
+ @param job
+          the job conf for the job
+ @param name
+          the leaf file name for the output file (such as part-00000")
+ @param arg3
+          a progressable for reporting progress.
+ @return a composite record writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateLeafFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the leaf name for the output file name. The default behavior does
+ not change the leaf file name (such as part-00000)
+ 
+ @param name
+          the leaf file name for the output file
+ @return the given leaf file name]]>
+      </doc>
+    </method>
+    <method name="generateFileNameForKeyValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the file output file name based on the given key and the leaf file
+ name. The default behavior is that the file name does not depend on the
+ key.
+ 
+ @param key
+          the key of the output data
+ @param name
+          the leaf file name
+ @return generated file name]]>
+      </doc>
+    </method>
+    <method name="generateActualKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual key from the given key/value. The default behavior is that
+ the actual key is equal to the given key
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual key derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="generateActualValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual value from the given key and value. The default behavior is that
+ the actual value is equal to the given value
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual value derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="getInputFileBasedOutputFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the outfile name based on a given name and the input file name. If
+ the {@link JobContext#MAP_INPUT_FILE} does not exists (i.e. this is not for a map only job),
+ the given name is returned unchanged. If the config value for
+ "num.of.trailing.legs.to.use" is not set, or set 0 or negative, the given
+ name is returned unchanged. Otherwise, return a file name consisting of the
+ N trailing legs of the input file name where N is the config value for
+ "num.of.trailing.legs.to.use".
+ 
+ @param job
+          the job config
+ @param name
+          the output file name
+ @return the outfile name based on a given name and the input file name.]]>
+      </doc>
+    </method>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param fs
+          the file system to use
+ @param job
+          a job conf object
+ @param name
+          the name of the file over which a record writer object will be
+          constructed
+ @param arg3
+          a progressable object
+ @return A RecordWriter object over the given file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This abstract class extends the FileOutputFormat, allowing to write the
+ output data to different output files. There are three basic use cases for
+ this class.
+ 
+ Case one: This class is used for a map reduce job with at least one reducer.
+ The reducer wants to write data to different files depending on the actual
+ keys. It is assumed that a key (or value) encodes the actual key (value)
+ and the desired location for the actual key (value).
+ 
+ Case two: This class is used for a map only job. The job wants to use an
+ output file name that is either a part of the input file name of the input
+ data, or some derivation of it.
+ 
+ Case three: This class is used for a map only job. The job wants to use an
+ output file name that depends on both the keys and the input file name,]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple named outputs support, it should be
+ instantiated in the Mapper/Reducer configure method.
+
+ @param job the job configuration object]]>
+      </doc>
+    </constructor>
+    <method name="getNamedOutputsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns list of channel names.
+
+ @param conf job conf
+ @return List of channel Names]]>
+      </doc>
+    </method>
+    <method name="isMultiNamedOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns if a named output is multiple.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return <code>true</code> if the name output is multi, <code>false</code>
+         if it is single. If the name output is not defined it returns
+         <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputFormatClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named output OutputFormat.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return namedOutput OutputFormat]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the key class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class for the named output key]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the value class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class of named output value]]>
+      </doc>
+    </method>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="addMultiNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a multi named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+ @param conf    job conf to enableadd the named output.
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+
+ @param conf    job conf to enableadd the named output.
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputs" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns iterator with the defined name outputs.
+
+ @return iterator with the defined named outputs]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a named output.
+
+ @param namedOutput the named output name
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="multiName" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a multi named output.
+
+ @param namedOutput the named output name
+ @param multiName   the multi name part
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes all the opened named outputs.
+ <p>
+ If overriden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>
+
+ @throws java.io.IOException thrown if any of the MultipleOutput files
+                             could not be closed properly.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing to additional outputs other
+ than the job default output via the <code>OutputCollector</code> passed to
+ the <code>map()</code> and <code>reduce()</code> methods of the
+ <code>Mapper</code> and <code>Reducer</code> implementations.
+ <p>
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ <p>
+ A named output can be a single file or a multi file. The later is referred as
+ a multi named output.
+ <p>
+ A multi named output is an unbound set of files all sharing the same
+ <code>OutputFormat</code>, key class and value class configuration.
+ <p>
+ When named outputs are used within a <code>Mapper</code> implementation,
+ key/values written to a name output are not part of the reduce phase, only
+ key/values written to the job <code>OutputCollector</code> are part of the
+ reduce phase.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled. The counters
+ group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+ </pre>
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ public class MOReduce implements
+   Reducer&lt;WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <class name="MultipleSequenceFileOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleSequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output data 
+ to different output files in sequence file output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <class name="MultipleTextOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleTextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output
+ data to different output files in Text output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <class name="MultithreadedMapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MultithreadedMapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Multithreaded implementation for {@link MapRunnable}.
+ <p>
+ It can be used instead of the default implementation,
+ of {@link org.apache.hadoop.mapred.MapRunner}, when the Map
+ operation is not CPU bound in order to improve throughput.
+ <p>
+ Map implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured to use this MapRunnable class (using
+ the JobConf.setMapRunnerClass method) and
+ the number of threads the thread-pool can use with the
+ <code>mapred.map.multithreadedrunner.threads</code> property, its default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <!-- start class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see org.apache.hadoop.mapred.FileInputFormat#getSplits(JobConf, int)]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <class name="TokenCountMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="TokenCountMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that maps text values into &lt;token,freq&gt; pairs.  Uses
+ {@link StringTokenizer} to break text into tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.
+ @deprecated Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #getPartitionFile(Configuration)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(JobConf,Path)
+ @deprecated Use 
+ {@link #getPartitionFile(Configuration)}
+ instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param job a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value aggregator 
+ descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class that may be 
+ dynamically loaded. The other is to delegate invocations of 
+ generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param job a job configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Combiner does not need to configure.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param output to collect combined values]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param job
+          a JobConf object that may contain the information that can be used
+          to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a JobConf object. Its main
+ function is to generate a list of aggregation-id/value pairs. An aggregation
+ id encodes an aggregation type which is used to guide the way to aggregate
+ the value in the reduce/combiner phrase of an Aggregate based job.The mapper in
+ an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @param caller the the caller class.
+ @return a JobConf object ready for submission.
+
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a JobConf object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specilizing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner classes,
+ and a set of built-in value aggregators, and a generic utility class that
+ helps user create map/reduce jobs using the generic class. The built-in
+ aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(JobConfjob); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="org.apache.hadoop.io.Text"/>
+      <param name="arg1" type="java.util.Iterator"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param key
+          the key is expected to be a Text object, whose prefix indicates
+          the type of aggregation to aggregate the values. In effect, data
+          driven computing is achieved. It is assumed that each aggregator's
+          getReport method emits appropriate output for the aggregator. This
+          may be further customiized.
+ @param values
+          the values to be aggregated]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.db">
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="chunks" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, eg. '(updated &gt;
+ 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(JobConf, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns the number of records in
+ the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(JobConf, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBWritable"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+</package>
+<package name="org.apache.hadoop.mapred.pipes">
+  <!-- start class org.apache.hadoop.mapred.pipes.Submitter -->
+  <class name="Submitter" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="Submitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Submitter" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExecutable" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the URI of the application's executable.
+ @param conf
+ @return the URI where the application's executable is located]]>
+      </doc>
+    </method>
+    <method name="setExecutable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="executable" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the URI for the application's executable. Normally this is a hdfs: 
+ location.
+ @param conf
+ @param executable The URI of the application's executable.]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job is using a Java RecordReader.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java RecordReader
+ @param conf the configuration to check
+ @return is it a Java RecordReader?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Mapper is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Mapper.
+ @param conf the configuration to check
+ @return is it a Java Mapper?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Reducer is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Reducer.
+ @param conf the configuration to check
+ @return is it a Java Reducer?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job will use a Java RecordWriter.
+ @param conf the configuration to modify
+ @param value the new value to set]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordWriter" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Will the reduce use a Java RecordWriter?
+ @param conf the configuration to check
+ @return true, if the output of the job will be written by Java]]>
+      </doc>
+    </method>
+    <method name="getKeepCommandFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Does the user want to keep the command file for debugging? If this is
+ true, pipes will write a copy of the command data to a file in the
+ task directory named "downlink.data", which may be used to run the C++
+ program under the debugger. You probably also want to set 
+ JobConf.setKeepFailedTaskFiles(true) to keep the entire directory from
+ being deleted.
+ To run using the data file, set the environment variable 
+ "mapreduce.pipes.commandfile" to point to the file.
+ @param conf the configuration to check
+ @return will the framework save the command file?]]>
+      </doc>
+    </method>
+    <method name="setKeepCommandFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to keep the command file for debugging
+ @param conf the configuration to modify
+ @param keep the new value]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link Submitter#runJob(JobConf)}">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException
+ @deprecated Use {@link Submitter#runJob(JobConf)}]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobSubmit" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the Map-Reduce framework.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Submit a pipes job based on the command line arguments.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PRESERVE_COMMANDFILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXECUTABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INTERPRETOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RW" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_REDUCE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The main entry point and job submitter. It may either be used as a command
+ line-based or API-based method to launch Pipes jobs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.pipes.Submitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce">
+  <!-- start class org.apache.hadoop.mapreduce.Cluster -->
+  <class name="Cluster" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Cluster" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="Cluster" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Cluster</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the file system where job-specific files are stored
+ 
+ @return object of FileSystem
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job corresponding to jobid.
+ 
+ @param jobId
+ @return object of {@link Job}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the queues in cluster.
+ 
+ @return array of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get queue information for the specified name.
+ 
+ @param name queuename
+ @return object of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLogParams" return="org.apache.hadoop.mapreduce.v2.LogParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+      <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get log parameters for the specified jobID or taskAttemptID
+ @param jobID the job id.
+ @param taskAttemptID the task attempt id. Optional.
+ @return the LogParams
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current cluster status.
+ 
+ @return object of {@link ClusterMetrics}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getActiveTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all active trackers in the cluster.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get blacklisted trackers.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapreduce.Job[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getAllJobStatuses()} instead.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the jobs in cluster.
+ 
+ @return array of {@link Job}
+ @throws IOException
+ @throws InterruptedException
+ @deprecated Use {@link #getAllJobStatuses()} instead.]]>
+      </doc>
+    </method>
+    <method name="getAllJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job status for all jobs in the cluster.
+ @return job status for all jobs in cluster
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where 
+ job-specific files will  be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker's view of the staging directory path where 
+ job-specific files will  be placed.
+ 
+ @return the staging directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getJobHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the job history file path for a given job id. The job history file at 
+ this path may or may not be existing depending on the job completion state.
+ The file is present only for the completed jobs.
+ @param jobId the JobID of the job submitted by the current user.
+ @return the file path of the job history file
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the root level queues.
+ @return array of JobQueueInfo object.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns immediate children of queueName.
+ @param queueName
+ @return array of JobQueueInfo which are children of queueName
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return the new expiration time
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides a way to access information about the map/reduce cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Cluster -->
+  <!-- start class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <class name="ClusterMetrics" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRunningMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running map tasks in the cluster.
+ 
+ @return running maps]]>
+      </doc>
+    </method>
+    <method name="getRunningReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running reduce tasks in the cluster.
+ 
+ @return running reduces]]>
+      </doc>
+    </method>
+    <method name="getOccupiedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of occupied map slots in the cluster.
+ 
+ @return occupied map slot count]]>
+      </doc>
+    </method>
+    <method name="getOccupiedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of occupied reduce slots in the cluster.
+ 
+ @return occupied reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of reserved map slots in the cluster.
+ 
+ @return reserved map slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved reduce slots in the cluster.
+ 
+ @return reserved reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getMapSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of map slots in the cluster.
+ 
+ @return map slot capacity]]>
+      </doc>
+    </method>
+    <method name="getReduceSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of reduce slots in the cluster.
+ 
+ @return reduce slot capacity]]>
+      </doc>
+    </method>
+    <method name="getTotalJobSubmissions" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of job submissions in the cluster.
+ 
+ @return total number of job submissions]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active trackers in the cluster.
+ 
+ @return active tracker count.]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted trackers in the cluster.
+ 
+ @return blacklisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getGrayListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted trackers in the cluster.
+ 
+ @return graylisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getDecommissionedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of decommissioned trackers in the cluster.
+ 
+ @return decommissioned tracker count]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterMetrics</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster.  
+   </li>
+   <li>
+   Number of blacklisted and decommissioned trackers.  
+   </li>
+   <li>
+   Slot capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently occupied/reserved map and reduce slots.
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   The number of job submissions.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterMetrics</code>, via 
+ {@link Cluster#getClusterStatus()}.</p>
+ 
+ @see Cluster]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <!-- start interface org.apache.hadoop.mapreduce.Counter -->
+  <interface name="Counter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="(and no-op by default)">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the counter
+ @param displayName of the counter
+ @deprecated (and no-op by default)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the name of the counter]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the counter.
+ @return the user facing name of the counter]]>
+      </doc>
+    </method>
+    <method name="getValue" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[What is the current value of this counter?
+ @return the current value]]>
+      </doc>
+    </method>
+    <method name="setValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set this counter by the given value
+ @param value the value to set]]>
+      </doc>
+    </method>
+    <method name="increment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+      <doc>
+      <![CDATA[Increment this counter by the given value
+ @param incr the value to increase this counter by]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A named counter that tracks the progress of a map/reduce job.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> is named by
+ an {@link Enum} and has a long for the value.</p>
+
+ <p><code>Counters</code> are bunched into Groups, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.Counter -->
+  <!-- start interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <interface name="CounterGroup"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <doc>
+    <![CDATA[A group of {@link Counter}s that logically belong together. Typically,
+ it is an {@link Enum} subclass and the counters are the values.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <!-- start class org.apache.hadoop.mapreduce.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the Counters object from the another counters object
+ @param <C> the type of counter
+ @param <G> the type of counter group
+ @param counters the old counters object]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[<p><code>Counters</code> holds per job/task counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
+ comprising of counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Counters -->
+  <!-- start class org.apache.hadoop.mapreduce.ID -->
+  <class name="ID" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[returns the int which represents the identifier]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare IDs by associated numbers]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SEPARATOR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ID -->
+  <!-- start class org.apache.hadoop.mapreduce.InputFormat -->
+  <class name="InputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple. The InputFormat
+ also creates the {@link RecordReader} to read the {@link InputSplit}.
+ 
+ @param context job configuration.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Create a record reader for a given split. The framework will call
+ {@link RecordReader#initialize(InputSplit, TaskAttemptContext)} before
+ the split is used.
+ @param split the split to be read
+ @param context the information about the task
+ @return a new record reader
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibility to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see FileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.InputSplit -->
+  <class name="InputSplit" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the size of the split, so that the input splits can be sorted by size.
+ @return the number of bytes in the split
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the list of nodes by name where the data for the split would be local.
+ The locations do not need to be serialized.
+ 
+ @return a new array of the node nodes.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.task.JobContextImpl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="Job"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, String)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration, String)}]]>
+      </doc>
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} .
+ A Cluster will be created with a generic {@link Configuration}.
+ 
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a 
+ given {@link Configuration}.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a given jobName.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.JobStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration} and {@link JobStatus}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param status job status
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster}.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ 
+ @return JobStatus#State
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get start time of the job.
+ 
+ @return the start time of the job]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get finish time of the job.
+ 
+ @return the finish time of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the scheduling info of the job]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the priority info of the job]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Dump stats to screen.]]>
+      </doc>
+    </method>
+    <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the tasks of a job.
+ 
+ @param type Type of the task
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job.  Blocks until all job tasks have been
+ killed as well.  If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param jobPriority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+
+ @param jobPriority
+          the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <param name="numEvents" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @param numEvents number of events to fetch
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link org.apache.hadoop.mapred.TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="failTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fail indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapreduce.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job. May return null if the job has been
+ retired and the job is no longer in the completed job store.
+ 
+ @return the counters for this job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tasks" type="int"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the number of reduce tasks for the job.
+ @param tasks the number of reduce tasks
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setInputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} for the job.
+ @param cls the <code>InputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} for the job.
+ @param cls the <code>OutputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} for the job.
+ @param cls the <code>Mapper</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the Jar by finding where a given class came from.
+ @param cls the example class]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job jar]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the combiner class for the job.
+ @param cls the combiner to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} for the job.
+ @param cls the <code>Reducer</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} for the job.
+ @param cls the <code>Partitioner</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to combiner,
+ {@link Reducer#reduce(Object, Iterable,
+ org.apache.hadoop.mapreduce.Reducer.Context)}
+
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSortComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls how the keys are sorted before they
+ are passed to the {@link Reducer}.
+ @param cls the raw comparator
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to 
+ {@link Reducer#reduce(Object, Iterable, 
+                       org.apache.hadoop.mapreduce.Reducer.Context)}
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setJobSetupCleanupNeeded"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="needed" type="boolean"/>
+      <doc>
+      <![CDATA[Specify whether job-setup and job-cleanup is needed for the job 
+ 
+ @param needed If <code>true</code>, job-setup and job-cleanup will be
+               considered from {@link OutputCommitter} 
+               else ignored.]]>
+      </doc>
+    </method>
+    <method name="setCacheArchives"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archives" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of archives
+ @param archives The list of archives that need to be localized]]>
+      </doc>
+    </method>
+    <method name="setCacheFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of files
+ @param files The list of files that need to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheArchive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a archives to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a file to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addFileToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an file path to the current set of classpath entries It adds the file
+ to cache as well.
+ 
+ Files added with this method will not be unpacked while being added to the
+ classpath.
+ To add archives to classpath, use the {@link #addArchiveToClassPath(Path)}
+ method instead.
+
+ @param file Path of the file to be added]]>
+      </doc>
+    </method>
+    <method name="addArchiveToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archive" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an archive path to the current set of classpath entries. It adds the
+ archive to cache as well.
+ 
+ Archive files will be unpacked and added to the classpath
+ when being distributed.
+
+ @param archive Path of the archive to be added]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to enable symlinks, but currently symlinks cannot be
+ disabled.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setCancelDelegationTokenUponJobCompletion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Sets the flag that will allow the JobTracker to cancel the HDFS delegation
+ tokens upon job completion. Defaults to true.]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCacheAndClasspath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Job Submitter will also
+ add the file to classpath. Intended to be used by user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addArchiveToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add an archive to job config for shared cache processing. If shared cache
+ is enabled, it will return true, otherwise, return false. We don't check
+ with SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Intended to be used by
+ user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="setFileSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for files. If the parameter
+ was previously set, this method will replace the old value with the new
+ provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="setArchiveSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for archives. If the
+ parameter was previously set, this method will replace the old value with
+ the new provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getFileSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for files.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getArchiveSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for archives.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and return immediately.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verbose" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and wait for it to finish.
+ @param verbose print the progress to the user
+ @return true if the job succeeded
+ @throws IOException thrown if the communication with the 
+         <code>JobTracker</code> is lost]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="getProgressPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which monitorAndPrintJob() prints status]]>
+      </doc>
+    </method>
+    <method name="getCompletionPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which waitForCompletion() should check.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the task output filter.
+ 
+ @param conf the configuration.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="newValue" type="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the Configuration to set the task output filter.
+ 
+ @param conf the Configuration to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation to which the job is submitted to, if any
+
+ @return the reservationId the identifier of the job's reservation, null if
+         the job does not have any reservation associated with it]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation to which the job is submitted to
+
+ @param reservationId the reservationId to set]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Job</code>.
+ @throws IOException if fail to close.]]>
+      </doc>
+    </method>
+    <field name="OUTPUT_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPLETION_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets completionPollInvervalMillis]]>
+      </doc>
+    </field>
+    <field name="PROGRESS_MONITOR_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets progMonitorPollIntervalMillis]]>
+      </doc>
+    </field>
+    <field name="USED_GENERIC_PARSER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUBMIT_REPLICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SUBMIT_REPLICATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_WILDCARD_FOR_LIBJARS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_USE_WILDCARD_FOR_LIBJARS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The job submitter's view of the Job.
+ 
+ <p>It allows the user to configure the
+ job, submit it, control its execution, and query the state. The set methods
+ only work until the job is submitted, afterwards they will throw an 
+ IllegalStateException. </p>
+ 
+ <p>
+ Normally the user creates the application, describes various facets of the
+ job via {@link Job} and then submits the job and monitor its progress.</p>
+ 
+ <p>Here is an example on how to submit a job:</p>
+ <p><blockquote><pre>
+     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+ </pre></blockquote>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Job -->
+  <!-- start interface org.apache.hadoop.mapreduce.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MRJobConfig"/>
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration for the job.
+ @return the shared configuration object]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique ID for the job.
+ @return the object with the job id]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get configured the number of reduce tasks for this job. Defaults to 
+ <code>1</code>.
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="getInputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link InputFormat} class for the job.
+ 
+ @return the {@link InputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the combiner class for the job.
+ 
+ @return the combiner class for the job.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} class for the job.
+ 
+ @return the {@link OutputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Partitioner} class for the job.
+ 
+ @return the {@link Partitioner} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getSortComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pathname of the job's jar.
+ @return the pathname]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="getGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the reduce.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setGroupingComparatorClass(Class)
+ @see #getCombinerKeyGroupingComparator()]]>
+      </doc>
+    </method>
+    <method name="getJobSetupCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether job-setup and job-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getTaskCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether task-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to check if symlinks should be used, but currently
+ symlinks cannot be disabled.
+ @return true]]>
+      </doc>
+    </method>
+    <method name="getArchiveClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the archive entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getCacheArchives" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache archives set in the Configuration
+ @return A URI array of the caches set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCacheFiles" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache files set in the Configuration
+ @return A URI array of the files set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheArchives" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized caches
+ @return A path array of localized caches
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheFiles" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized files
+ @return A path array of localized files
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.]]>
+      </doc>
+    </method>
+    <method name="getFileClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getArchiveTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the archives.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getFileTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the files.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapred.map.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapred.reduce.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A read-only view of the job that is provided to the tasks while they
+ are running.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.JobContext -->
+  <!-- start class org.apache.hadoop.mapreduce.JobCounter -->
+  <class name="JobCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJtIdentifier" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare JobIds by first jtIdentifiers, then by job numbers]]>
+      </doc>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the stuff after the "job" prefix to the given builder. This is useful,
+ because the sub-ids use this substring at the start of their string.
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOBID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local" and a random number.
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobID -->
+  <!-- start class org.apache.hadoop.mapreduce.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobPriority -->
+  <!-- start class org.apache.hadoop.mapreduce.JobStatus -->
+  <class name="JobStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <doc>
+      <![CDATA[Change the current run state of the job.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Used to set the scheduling information associated to a particular Job.
+ 
+ @param schedulingInfo Scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the job acls.
+ 
+ @param acls {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name
+ @param queue queue name]]>
+      </doc>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set diagnostic information.
+ @param failureInfo diagnostic information]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="getMapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="getCleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="getSetupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="getReduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return start time of the job]]>
+      </doc>
+    </method>
+    <method name="clone" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getUsername" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the username of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the Scheduling information associated to a particular Job.
+ @return the scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="getJobACLs" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job acls.
+ 
+ @return a {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets any available info on the reason of failure of the job.
+ @return diagnostic information on why a job might have failed.]]>
+      </doc>
+    </method>
+    <method name="isJobComplete" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the status is for a completed job.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration file for the job.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the finish time of the job.]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check whether the job has retired.]]>
+      </doc>
+    </method>
+    <method name="getHistoryFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job history file name for a completed job. If job is not 
+ completed or history file not available then return null.]]>
+      </doc>
+    </method>
+    <method name="getNumUsedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="setNumUsedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="getNumReservedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="setNumReservedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="getUsedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the used memory]]>
+      </doc>
+    </method>
+    <method name="setUsedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="m" type="int"/>
+      <doc>
+      <![CDATA[@param m the used memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="int"/>
+      <doc>
+      <![CDATA[@param r the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getNeededMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the needed memory]]>
+      </doc>
+    </method>
+    <method name="setNeededMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the needed memory]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether job running in uber mode
+ @return job in uber-mode]]>
+      </doc>
+    </method>
+    <method name="setUber"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isUber" type="boolean"/>
+      <doc>
+      <![CDATA[Set uber-mode flag 
+ @param isUber Whether job running in uber-mode]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Describes the current status of a job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobStatus -->
+  <!-- start interface org.apache.hadoop.mapreduce.MapContext -->
+  <interface name="MapContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="getInputSplit" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the input split for this map.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context that is given to the {@link Mapper}.
+ @param <KEYIN> the key input type to the Mapper
+ @param <VALUEIN> the value input type to the Mapper
+ @param <KEYOUT> the key output type from the Mapper
+ @param <VALUEOUT> the value output type from the Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.MapContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Mapper -->
+  <class name="Mapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Mapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the beginning of the task.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="value" type="VALUEIN"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once for each key/value pair in the input split. Most applications
+ should override this, but the default is the identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Expert users can override this method for more complete control over the
+ execution of the Mapper.
+ @param context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link Configuration} for 
+ the job via the {@link JobContext#getConfiguration()}.
+ 
+ <p>The framework first calls 
+ {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by
+ {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)}
+ for each key/value pair in the <code>InputSplit</code>. Finally 
+ {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the sorting and grouping by 
+ specifying two key {@link RawComparator} classes.</p>
+
+ <p>The <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link Job#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>Configuration</code>.</p>
+  
+ <p>If the job has zero
+ reduces then the output of the <code>Mapper</code> is directly written
+ to the {@link OutputFormat} without sorting by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class TokenCounterMapper 
+     extends Mapper&lt;Object, Text, Text, IntWritable&gt;{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+ </pre></blockquote>
+
+ <p>Applications may override the
+ {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert
+ greater control on map processing e.g. multi-threaded <code>Mapper</code>s 
+ etc.</p>
+ 
+ @see InputFormat
+ @see JobContext
+ @see Partitioner  
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Mapper -->
+  <!-- start class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <class name="MarkableIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MarkableIteratorInterface"/>
+    <constructor name="MarkableIterator" type="java.util.Iterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new iterator layered on the input iterator
+ @param itr underlying iterator that implements MarkableIteratorInterface]]>
+      </doc>
+    </constructor>
+    <method name="mark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clearMark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="VALUE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>MarkableIterator</code> is a wrapper iterator class that 
+ implements the {@link MarkableIteratorInterface}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <class name="OutputCommitter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.	
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus.State#FAILED} or 
+ {@link JobStatus.State#KILLED}.  This is called from the application
+ master process for the entire job. This may be called multiple times.
+
+ @param jobContext Context of the job whose output is being written.
+ @param state final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task.  This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not successful.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @see #recoverTask(TaskAttemptContext)
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRJobConfig#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>.  This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ This may be called multiple times for the same task.  But from different
+ application attempts.
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputFormat -->
+  <class name="OutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given task.
+
+ @param context the information about the current task.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ @param context information about the job
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the output committer for this output format. This is responsible
+ for ensuring the output is committed correctly.
+ @param context the task context
+ @return an output committer
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.Partitioner -->
+  <class name="Partitioner" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Partitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="value" type="VALUE"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the partition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be partioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+
+ <p>Note: If you require your Partitioner class to obtain the Job's
+ configuration object, implement the {@link Configurable} interface.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Partitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <class name="QueueAclsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueAclsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueAclsInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueAclsInfo" type="java.lang.String, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueAclsInfo object using the queue name and the
+ queue operations array
+ 
+ @param queueName Name of the job queue
+ @param operations]]>
+      </doc>
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name.
+ 
+ @return name]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+    </method>
+    <method name="getOperations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get opearations allowed on queue.
+ 
+ @return array of String]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class to encapsulate Queue ACLs for a particular
+  user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String, org.apache.hadoop.mapreduce.QueueState, org.apache.hadoop.mapreduce.JobStatus[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param queueName
+ @param schedulingInfo
+ @param state
+ @param stats]]>
+      </doc>
+    </constructor>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the queue name of the JobQueueInfo
+ 
+ @param queueName Name of the job queue.]]>
+      </doc>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue name from JobQueueInfo
+ 
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheduling information associated to particular job queue
+ 
+ @param schedulingInfo]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the scheduling information associated to particular job queue.
+ If nothing is set would return <b>"N/A"</b>
+ 
+ @return Scheduling information associated to particular Job Queue]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.QueueState"/>
+      <doc>
+      <![CDATA[Set the state of the queue
+ @param state state of the queue.]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the queue state
+ @return the queue state.]]>
+      </doc>
+    </method>
+    <method name="setJobStatuses"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+    </method>
+    <method name="getQueueChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get immediate children.
+ 
+ @return list of QueueInfo]]>
+      </doc>
+    </method>
+    <method name="setQueueChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.List"/>
+    </method>
+    <method name="getProperties" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get properties.
+ 
+ @return Properties]]>
+      </doc>
+    </method>
+    <method name="setProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="props" type="java.util.Properties"/>
+    </method>
+    <method name="getJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the jobs submitted to queue
+ @return list of JobStatus for the submitted jobs]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getStateName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the stateName]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Enum representing queue state]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueState -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordReader -->
+  <class name="RecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="RecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at initialization.
+ @param split the split that defines the range of records to read
+ @param context the information about the task
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next key, value pair.
+ @return true if a key/value pair was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key
+ @return the current key or null if there is no current key
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the object that was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The current progress of the record reader through its data.
+ @return a number between 0.0 and 1.0 that is the fraction of the data read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the record reader.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The record reader breaks the data into key/value pairs for input to the
+ {@link Mapper}.
+ @param <KEYIN>
+ @param <VALUEIN>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordWriter -->
+  <class name="RecordWriter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RecordWriter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param context the context of the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <interface name="ReduceContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="nextKey" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Start processing next unique key.]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Iterate through the values for the current key, reusing the same value 
+ object, which is stored in the context.
+ @return the series of values associated with the current key. All of the 
+ objects returned directly and indirectly from this method are reused.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context passed to the {@link Reducer}.
+ @param <KEYIN> the class of the input keys
+ @param <VALUEIN> the class of the input values
+ @param <KEYOUT> the class of the output keys
+ @param <VALUEOUT> the class of the output values]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Reducer -->
+  <class name="Reducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Reducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the start of the task.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[This method is called once for each key. Most applications will define
+ their reduce class by overriding this method. The default implementation
+ is an identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advanced application writers can use the 
+ {@link #run(org.apache.hadoop.mapreduce.Reducer.Context)} method to
+ control how the reduce task works.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p><code>Reducer</code> implementations 
+ can access the {@link Configuration} for the job via the 
+ {@link JobContext#getConfiguration()} method.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p>The <code>Reducer</code> copies the sorted output from each 
+   {@link Mapper} using HTTP across the network.</p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework merge sorts <code>Reducer</code> inputs by 
+   <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key).</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>To achieve a secondary sort on the values returned by the value 
+   iterator, the application should extend the key with the secondary
+   key and define a grouping comparator. The keys will be sorted using the
+   entire key, but will be grouped using the grouping comparator to decide
+   which keys and values are sent in the same call to reduce.The grouping 
+   comparator is specified via 
+   {@link Job#setGroupingComparatorClass(Class)}. The sort order is
+   controlled by 
+   {@link Job#setSortComparatorClass(Class)}.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)}
+   method is called for each <code>&lt;key, (collection of values)&gt;</code> in
+   the sorted inputs.</p>
+   <p>The output of the reduce task is typically written to a 
+   {@link RecordWriter} via 
+   {@link Context#write(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class IntSumReducer&lt;Key&gt; extends Reducer&lt;Key,IntWritable,
+                                                 Key,IntWritable&gt; {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable&lt;IntWritable&gt; values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Reducer -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique name for this task attempt.]]>
+      </doc>
+    </method>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="msg" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the current status of the task to the given string.]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the last set status message.
+ @return the current status message]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current progress of the task attempt.
+ @return a number between 0.0 and 1.0 (inclusive) indicating the attempt's
+ progress.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>counterName</code>]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>groupName</code> and 
+ <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>groupName</code> and 
+         <code>counterName</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context for task attempts.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapreduce.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link TaskID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the TaskType of the TaskAttemptID]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the StringBuilder
+ @param builder the builder to append ot
+ @return the builder that was passed in.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskIds by first tipIds, then by task numbers.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="ATTEMPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapreduce.TaskAttemptID, int, boolean, org.apache.hadoop.mapreduce.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getEventId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns event Id. 
+ @return event id]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerHttp" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[http location of the tasktracker where this task ran. 
+ @return http location of tasktracker user logs]]>
+      </doc>
+    </method>
+    <method name="getTaskRunTime" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns time (in millisec) the task took to complete.]]>
+      </doc>
+    </method>
+    <method name="setTaskRunTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskCompletionTime" type="int"/>
+      <doc>
+      <![CDATA[Set the task completion time
+ @param taskCompletionTime time (in millisec) the task took to complete]]>
+      </doc>
+    </method>
+    <method name="setEventId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="eventId" type="int"/>
+      <doc>
+      <![CDATA[set event Id. should be assigned incrementally starting from 0. 
+ @param eventId]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <method name="setTaskStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"/>
+      <doc>
+      <![CDATA[Set task status. 
+ @param status]]>
+      </doc>
+    </method>
+    <method name="setTaskTrackerHttp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskTrackerHttp" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set task tracker http location. 
+ @param taskTrackerHttp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMapTask" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="idWithinJob" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCounter -->
+  <class name="TaskCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} of the task 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable. Sets the task type to
+ {@link TaskType#REDUCE}, the ID to 0, and the job ID to an empty job ID.]]>
+      </doc>
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this tip belongs to.
+
+ @return the JobID object]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID.
+
+ @return whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the task.
+
+ @return the type of the task]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskInProgressIds by first jobIds, then by tip numbers.
+ Reducers are defined as greater than mappers.
+
+ @param o the TaskID against which to compare
+ @return 0 if equal, positive if this TaskID is greater, and negative if
+ this TaskID is less]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the given builder.
+
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskID object from given string.
+
+ @param str the target string
+ @return constructed TaskID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getRepresentingCharacter" return="char"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <doc>
+      <![CDATA[Gets the character representing the {@link TaskType}.
+
+ @param type the TaskType
+ @return the character]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="char"/>
+      <doc>
+      <![CDATA[Gets the {@link TaskType} corresponding to the character.
+
+ @param c the character
+ @return the TaskType]]>
+      </doc>
+    </method>
+    <method name="getAllTaskTypes" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string of characters describing all possible {@link TaskType}
+ values
+
+ @return a string of all task type characters]]>
+      </doc>
+    </method>
+    <field name="TASK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="TASK_ID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="taskIdPattern" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskID -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <interface name="TaskInputOutputContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advance to the next key, value pair, returning null if at end.
+ @return the key object that was read into, or null if no more]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key.
+ @return the current key object or null if there isn't one
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the value object that was read into
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Generate an output key/value pair.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} for the task-attempt.
+ @return the <code>OutputCommitter</code> for the task-attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A context object that allows input and output from the task. It is only
+ supplied to the {@link Mapper} or {@link Reducer}.
+ @param <KEYIN> the input key type for the task
+ @param <VALUEIN> the input value type for the task
+ @param <KEYOUT> the output key type for the task
+ @param <VALUEOUT> the output value type for the task]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <class name="TaskTrackerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskTrackerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskTrackerName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the tasktracker's name.
+ 
+ @return tracker's name.]]>
+      </doc>
+    </method>
+    <method name="isBlacklisted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether tracker is blacklisted
+ @return true if tracker is blacklisted
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="getReasonForBlacklist" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the reason for which the tasktracker was blacklisted.
+ 
+ @return reason which tracker was blacklisted]]>
+      </doc>
+    </method>
+    <method name="getBlacklistReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets a descriptive report about why the tasktracker was blacklisted.
+ 
+ @return report describing why the tasktracker was blacklisted.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Information about TaskTracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskType -->
+  <class name="TaskType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enum for map, reduce, job-setup, job-cleanup, task-cleanup task types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskType -->
+</package>
+<package name="org.apache.hadoop.mapreduce.checkpoint">
+</package>
+<package name="org.apache.hadoop.mapreduce.counters">
+  <!-- start class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <class name="AbstractCounters" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters, org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another counters object.
+ @param <C1> type of the other counter
+ @param <G1> type of the other counter group
+ @param counters the counters object to copy
+ @param groupFactory the factory for new groups]]>
+      </doc>
+    </constructor>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter, create one if necessary
+ @param groupName of the counter
+ @param counterName name of the counter
+ @return the matching counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Find the counter for the given enum. The same enum will always return the
+ same counter.
+ @param key the counter key
+ @return the matching counter object]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of all counter classes.
+ @return Set of counter names.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getGroup" return="G"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named counter group, or an empty group if there is none
+ with the specified name.
+ @param groupName name of the group
+ @return the group]]>
+      </doc>
+    </method>
+    <method name="countCounters" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the total number of counters, by summing the number of counters
+ in each group.
+ @return the total number of counters]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the set of groups.
+ Counters ::= version #fgroups (groupId, group)* #groups (group)*]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return textual representation of the counter values.
+ @return the string]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract class to provide common implementation for the Counters
+ container in both mapred and mapreduce packages.
+
+ @param <C> type of counter inside the counters
+ @param <G> type of group inside the counters]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <!-- start interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+  <interface name="CounterGroupBase"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the internal name of the group
+ @return the internal name]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the group.
+ @return the human readable name]]>
+      </doc>
+    </method>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the group
+ @param displayName of the group]]>
+      </doc>
+    </method>
+    <method name="addCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="T"/>
+      <doc>
+      <![CDATA[Add a counter to this group.
+ @param counter to add]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a counter to this group
+ @param name  of the counter
+ @param displayName of the counter
+ @param value of the counter
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @param displayName the display name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+      <doc>
+      <![CDATA[Find a counter in the group
+ @param counterName the name of the counter
+ @param create create the counter if not found if true
+ @return the counter that was found or added or null if create is false]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of counters in this group.]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+      <doc>
+      <![CDATA[Increment all counters by a group of counters
+ @param rightGroup  the group to be added to this group]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The common counter group interface.
+
+ @param <T> type of the counter for the group]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a double value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="double"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a double value.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getSum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getSum" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <method name="setMaxItems" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <doc>
+      <![CDATA[Set the limit on the number of unique values
+ @param n the desired limit on the number of unique values
+ @return the new limit on the number of unique values]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return the number of unique objects aggregated]]>
+      </doc>
+    </method>
+    <method name="getUniqueItems" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the set of the unique objects]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of the unique objects. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <field name="MAX_NUM_UNIQUE_VALUES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param conf a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for the given 
+   key/value pairs by delegating the invocation to the real object.
+   
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <field name="theAggregatorDescriptor" type="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value 
+ aggregator descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class
+ that may be dynamically loaded. The other is to
+ delegate invocations of generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addNextValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val the value to be added]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the agregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return an array of values as the outputs of the combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="uniqCount" type="long"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param uniqCount the limit in the number of unique values to keep, 
+                  if type is UNIQ_VALUE_COUNT 
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate 1 or 2 aggregation-id/value pairs for the given key/value pair.
+ The first id will be of type LONG_VALUE_SUM, with "record_count" as
+ its aggregation id. If the input is a file split,
+ the second id of the same type will be generated too, with the file name 
+ as its aggregation id. This achieves the behavior of counting the total 
+ number of records in the input data, and the number of records 
+ in each input file.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param conf a configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="inputFile" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param context to collect combined values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for 
+ the given key/value pair.
+ This function is usually called by the mapper of an Aggregate based job.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param conf
+          a Configuration object that may contain the information 
+          that can be used to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a {@link Configuration}
+ object. Its main function is to generate a list of aggregation-id/value 
+ pairs. An aggregation id encodes an aggregation type which is used to 
+ guide the way to aggregate the value in the reduce/combiner phrase of an
+ Aggregate based job. 
+ The mapper in an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param conf The configuration for job
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a Job object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specializing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner 
+ classes, and a set of built-in value aggregators, and a generic utility 
+ class that helps user create map/reduce jobs using the generic class. 
+ The built-in aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(Configuration conf); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getValueAggregatorDescriptor" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spec" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getAggregatorDescriptors" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="DESCRIPTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DESCRIPTOR_NUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[@param key
+        the key is expected to be a Text object, whose prefix indicates
+        the type of aggregation to aggregate the values. In effect, data
+        driven computing is achieved. It is assumed that each aggregator's
+        getReport method emits appropriate output for the aggregator. This
+        may be further customized.
+ @param values the values to be aggregated
+ @param context]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add the given val to the aggregator.
+ 
+ @param val the value to be added. It is expected to be a string
+ in the form of xxxx\tnum, meaning xxxx has num occurrences.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this aggregator.
+ It includes the following basic statistics of the histogram:
+    the number of unique values
+    the minimum value
+    the media value
+    the maximum value
+    the average value
+    the standard deviation]]>
+      </doc>
+    </method>
+    <method name="getReportDetails" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a string representation of the list of value/frequence pairs of 
+ the histogram]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a list value/frequence pairs.
+  The return value is expected to be used by the reducer.]]>
+      </doc>
+    </method>
+    <method name="getReportItems" return="java.util.TreeMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a TreeMap representation of the histogram]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.chain">
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <class name="ChainMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain mapper.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ 
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ </p>
+ ChainMapper usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+  <class name="ChainReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="reducerConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Sets the {@link Reducer} class to the chain job.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's Configuration.
+ This precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ 
+ @param job
+          the job
+ @param klass
+          the Reducer class to add.
+ @param inputKeyClass
+          reducer input key class.
+ @param inputValueClass
+          reducer input value class.
+ @param outputKeyClass
+          reducer output key class.
+ @param outputValueClass
+          reducer output value class.
+ @param reducerConf
+          a configuration for the Reducer class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain reducer.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the
+ chain.
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ 
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion. The output of the reducer becomes the input of
+ the first mapper and output of first becomes the input of the second, and so
+ on until the last Mapper, the output of the last Mapper will be written to
+ the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain. This
+ enables having reusable specialized Mappers that can be combined to perform
+ composite operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p> Using the ChainMapper and the ChainReducer classes is possible to
+ compose Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO. </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ ChainReducer usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.db">
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <class name="BigDecimalSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BigDecimalSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="tryDivide" return="java.math.BigDecimal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="numerator" type="java.math.BigDecimal"/>
+      <param name="denominator" type="java.math.BigDecimal"/>
+      <doc>
+      <![CDATA[Divide numerator by denominator. If impossible in exact mode, use rounding.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over BigDecimal values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <class name="BooleanSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BooleanSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over boolean values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <class name="DataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getBoundingValsQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a query which returns the minimum and maximum values for
+ the order-by column.
+
+ The min value should be in the first column, and the
+ max value should be in the second column of the results.]]>
+      </doc>
+    </method>
+    <method name="setBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="query" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-defined bounding query to use with a user-defined query.
+      This *must* include the substring "$CONDITIONS"
+      (DataDrivenDBInputFormat.SUBSTITUTE_TOKEN) inside the WHERE clause,
+      so that DataDrivenDBInputFormat knows where to insert split clauses.
+      e.g., "SELECT foo FROM mytable WHERE $CONDITIONS"
+      This will be expanded to something like:
+        SELECT foo FROM mytable WHERE (id &gt; 100) AND (id &lt; 250)
+      inside each split.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="splitBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Note that the "orderBy" column is called the "splitBy" in this version.
+ We reuse the same field, but it's not strictly ordering it -- just partitioning
+ the results.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputBoundingQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[setInput() takes a custom query and a separate "bounding query" to use
+      instead of the custom "count query" used by DBInputFormat.]]>
+      </doc>
+    </method>
+    <field name="SUBSTITUTE_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If users are providing their own query, the following string is expected to
+      appear in the WHERE clause, which will be substituted with a pair of conditions
+      on the input to allow input splits to parallelise the import.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ Operates like DBInputFormat, but instead of using LIMIT and OFFSET to demarcate
+ splits, it tries to generate WHERE clauses which separate the data into roughly
+ equivalent shards.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <class name="DataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records,
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table,
+ using data-driven WHERE clause splits.
+ Emits LongWritables containing the record number as
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <class name="DateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.IntegerSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Given a Date 'd', format it as a string for use in a SQL date
+ comparison operation.
+ @param d the date to format.
+ @return the string representing this date in SQL with any appropriate
+ quotation characters, etc.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values.
+ Make use of logic from IntegerSplitter, since date/time are just longs
+ in Java.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the {@link Configuration}.  
+ @param conf the configuration
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Returns a connection object o the DB 
+ @throws ClassNotFoundException 
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getInputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="getInputConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputConditions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conditions" type="java.lang.String"/>
+    </method>
+    <method name="getInputOrderBy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputOrderBy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orderby" type="java.lang.String"/>
+    </method>
+    <method name="getInputQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputCountQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="setInputBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputBoundingQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClass" type="java.lang.Class"/>
+    </method>
+    <method name="getOutputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getOutputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="setOutputFieldCount"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldCount" type="int"/>
+    </method>
+    <method name="getOutputFieldCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_BOUNDING_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the max and min values of the jdbc.input.query]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A container for configuration property names for jobs with DB input/output.
+  
+ The job can be configured using the static methods in this class, 
+ {@link DBInputFormat}, and {@link DBOutputFormat}. 
+ Alternatively, the properties can be set in the configuration with proper
+ values. 
+   
+ @see DBConfiguration#configureDB(Configuration, String, String, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String, String, String...)
+ @see DBOutputFormat#setOutput(Job, String, String...)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBProductName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for getting the total number of rows, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, 
+ eg. '(updated &gt; 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(Job, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns 
+ the number of records in the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(Job, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+    <method name="closeConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="conditions" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="connection" type="java.sql.Connection"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="tableName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fieldNames" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="dbConf" type="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ <p>
+ DBInputFormat emits LongWritables containing the record number as 
+ key and DBWritables as value. 
+ 
+ The SQL query, and input class can be using one of the two 
+ setInput methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="constructQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="table" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Constructs the query used as the prepared statement to insert data.
+ 
+ @param table
+          the table to insert into
+ @param fieldNames
+          the fields to insert into. If field names are unknown, supply an
+          array of nulls.]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with 
+ the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job 
+ with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A OutputFormat that sends the reduce output to a SQL table.
+ <p> 
+ {@link DBOutputFormat} accepts &lt;key,value&gt; pairs, where 
+ key has a type extending DBWritable. Returned {@link RecordWriter} 
+ writes <b>only the key</b> to the database with a batch SQL query.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <class name="DBRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.LongWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #nextKeyValue()}">
+      <param name="key" type="org.apache.hadoop.io.LongWritable"/>
+      <param name="value" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #nextKeyValue()}]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplit" return="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatement" return="java.sql.PreparedStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stmt" type="java.sql.PreparedStatement"/>
+    </method>
+    <field name="statement" type="java.sql.PreparedStatement"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table.
+ Emits LongWritables containing the record number as 
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <interface name="DBSplitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="split" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Given a ResultSet containing one record (and already advanced to that record)
+ with two columns (a low value, and a high value, both of the same type), determine
+ a set of splits that span the given values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DBSplitter will generate DBInputSplits to use with DataDrivenDBInputFormat.
+ DataDrivenDBInputFormat needs to interpolate between two values that
+ represent the lowest and highest valued records to import. Depending
+ on the data-type of the column, this requires different behavior.
+ DBSplitter implementations should perform this for a data type or family
+ of data types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statement" type="java.sql.PreparedStatement"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Sets the fields of the object in the {@link PreparedStatement}.
+ @param statement the statement that the fields are put into.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resultSet" type="java.sql.ResultSet"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Reads the fields of the object from the {@link ResultSet}. 
+ @param resultSet the {@link ResultSet} to get the fields from.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Objects that are read from/written to a database should implement
+ <code>DBWritable</code>. DBWritable, is similar to {@link Writable} 
+ except that the {@link #write(PreparedStatement)} method takes a 
+ {@link PreparedStatement}, and {@link #readFields(ResultSet)} 
+ takes a {@link ResultSet}. 
+ <p>
+ Implementations are responsible for writing the fields of the object 
+ to PreparedStatement, and reading the fields of the object from the 
+ ResultSet. 
+ 
+ <p>Example:</p>
+ If we have the following table in the database :
+ <pre>
+ CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+ </pre>
+ then we can read/write the tuples from/to the table with :
+ <p><pre>
+ public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <class name="FloatSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="FloatSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over floating-point values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <class name="IntegerSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="IntegerSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over integer values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <class name="MySQLDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <class name="MySQLDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <class name="OracleDataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="OracleDataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table in an Oracle db.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <class name="OracleDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a Oracle table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <class name="OracleDateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.DateSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values returned by an Oracle db.
+ Make use of logic from DateSplitter, since this just needs to use
+ some Oracle-specific functions on the formatting end when generating
+ InputSplits.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <class name="OracleDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records from an Oracle DB.]]>
+      </doc>
+    </method>
+    <method name="setSessionTimeZone"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="conn" type="java.sql.Connection"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Set session time zone
+ @param conf The current configuration.
+ We read the 'oracle.sessionTimeZone' property from here.
+ @param conn The connection to alter the timezone properties of.]]>
+      </doc>
+    </method>
+    <field name="SESSION_TIMEZONE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set to a timezone string.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from an Oracle SQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+  <class name="TextSplitter" extends="org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[This method needs to determine the splits between two user-provided strings.
+ In the case where the user's strings are 'A' and 'Z', this is not hard; we 
+ could create two splits from ['A', 'M') and ['M', 'Z'], 26 splits for strings
+ beginning with each letter, etc.
+
+ If a user has provided us with the strings "Ham" and "Haze", however, we need
+ to create splits that differ in the third letter.
+
+ The algorithm used is as follows:
+ Since there are 2**16 unicode characters, we interpret characters as digits in
+ base 65536. Given a string 's' containing characters s_0, s_1 .. s_n, we interpret
+ the string as the number: 0.s_0 s_1 s_2.. s_n in base 65536. Having mapped the
+ low and high strings into floating-point values, we then use the BigDecimalSplitter
+ to establish the even split points, then map the resulting floating point values
+ back into strings.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over text strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.fieldsel">
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <class name="FieldSelectionHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FieldSelectionHelper" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="parseOutputKeyValueSpec" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="specToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldSeparator" type="java.lang.String"/>
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="getKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="extractOutputKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="val" type="java.lang.String"/>
+      <param name="fieldSep" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valFieldList" type="java.util.List"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="ignoreKey" type="boolean"/>
+      <param name="isMap" type="boolean"/>
+    </method>
+    <field name="emptyText" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #DATA_FIELD_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #DATA_FIELD_SEPARATOR}]]>
+      </doc>
+    </field>
+    <field name="MAP_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REDUCE_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <class name="FieldSelectionMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) separated
+ field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec can be a 
+ simple number (e.g. 5) specifying a specific field, or a range (like 2-5)
+ to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value
+ fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+  <class name="FieldSelectionReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a reducer class that can be used to perform field
+ selections in a manner similar to unix cut. 
+ 
+ The input data is treated as fields separated by a user specified
+ separator (the default value is "\t"). The user can specify a list of
+ fields that form the reduce output keys, and a list of fields that form
+ the reduce output values. The fields are the union of those from the key
+ and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) 
+ separated field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec
+ can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) 
+ specifying all the fields starting from field 3. The open range field
+ spec applies value fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.input">
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="setMaxSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="maxSplitSize" type="long"/>
+      <doc>
+      <![CDATA[Specify the maximum size (in bytes) of each split. Each split is
+ approximately equal to the specified size.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeNode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeNode" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per node.
+ This applies to data that is left over after combining data on a single
+ node into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeNode.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeRack"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeRack" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per rack.
+ This applies to data that is left over after combining data on a single
+ rack into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeRack.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SPLIT_MINSIZE_PERNODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE_PERRACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link CombineFileSplit}'s in 
+ {@link InputFormat#getSplits(JobContext)} method. 
+ 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default splitting behavior in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement 
+ {@link InputFormat#createRecordReader(InputSplit, TaskAttemptContext)}
+ to construct <code>RecordReader</code>'s for 
+ <code>CombineFileSplit</code>'s.
+ 
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapreduce.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapreduce.lib.input.FileInputFormat, org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartOffsets" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the start offsets of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getLengths" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the lengths of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the start offset of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the length of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getNumPaths" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns all the Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns all the Paths where this input-split resides]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. 
+ 
+ Unlike {@link FileSplit}, CombineFileSplit class does not represent 
+ a split of a file, but a split of input files into smaller sets. 
+ A split may contain blocks from different file but all 
+ the blocks in the same split are probably local to some rack <br> 
+ CombineFileSplit can be used to implement {@link RecordReader}'s, 
+ with reading one record per file.
+ 
+ @see FileSplit
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <class name="FileInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setInputDirRecursive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputDirRecursive" type="boolean"/>
+      <doc>
+      <![CDATA[@param job
+          the job to modify
+ @param inputDirRecursive]]>
+      </doc>
+    </method>
+    <method name="getInputDirRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[@param job
+          the job to look at.
+ @return should the files to be read recursively?]]>
+      </doc>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower bound on split size imposed by the format.
+ @return the number of bytes of the minimal split for this format]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param context the job context
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+ @param job the job to modify
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="setMinInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the minimum input split size
+ @param job the job to modify
+ @param size the minimum size]]>
+      </doc>
+    </method>
+    <method name="getMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the minimum split size
+ @param job the job
+ @return the minimum number of bytes that can be in a split]]>
+      </doc>
+    </method>
+    <method name="setMaxInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum split size
+ @param job the job to modify
+ @param size the maximum split size]]>
+      </doc>
+    </method>
+    <method name="getMaxSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the maximum split size.
+ @param context the job to look at.
+ @return the maximum number of bytes a split can include]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Generate the list of files and make them into FileSplits.
+ @param job the job context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blockSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="maxSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param job the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param job The job to modify
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param job The job to modify 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job} to modify
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param context The job
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <field name="INPUT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MAXSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PATHFILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LIST_STATUS_NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LIST_STATUS_NUM_THREADS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}s.
+
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobContext)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(JobContext, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <class name="FileInputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host and cached-blocks information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobContext)} and passed to
+ {@link InputFormat#createRecordReader(InputSplit,TaskAttemptContext)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="setKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="line" type="byte[]"/>
+      <param name="lineLen" type="int"/>
+      <param name="pos" type="int"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_VALUE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KEY_VALUE_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #KEY_VALUE_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #KEY_VALUE_SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either line feed or carriage-return are used to signal end of line. 
+ Each line is divided into key and value parts by a separator byte. If no
+ such a byte exists, the key will be the entire line and value will be empty.
+ The separator byte can be specified in config file under the attribute name
+ mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <field name="DIR_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIR_MAPPERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see FileInputFormat#getSplits(JobContext)]]>
+      </doc>
+    </method>
+    <method name="getSplitsForFile" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="numLinesPerSplit" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <method name="setNumLinesPerSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="numLines" type="int"/>
+      <doc>
+      <![CDATA[Set the number of lines per split
+ @param job the job to modify
+ @param numLines the number of lines per split]]>
+      </doc>
+    </method>
+    <method name="getNumLinesPerSplit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the number of lines per split
+ @param job the job
+ @return the number of lines per split]]>
+      </doc>
+    </method>
+    <field name="LINES_PER_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, except it generates
+ SequenceFileAsTextRecordReader which converts the input keys and values
+ to their String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by
+ calling toString() method. This class to SequenceFileAsTextInputFormat
+ class is as LineRecordReader class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param context the task-attempt context
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param job The job
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_FREQUENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.jobcontrol">
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <class name="ControlledJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ControlledJob" type="org.apache.hadoop.mapreduce.Job, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param job a mapreduce job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="ControlledJob" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ 
+ @param conf mapred job configuration representing a job to be executed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job name of this job]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job name for  this job.
+ @param jobName the job name]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job ID of this job assigned by JobControl]]>
+      </doc>
+    </method>
+    <method name="setJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job ID for  this job.
+ @param id the job ID]]>
+      </doc>
+    </method>
+    <method name="getMapredJobId" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapreduce job]]>
+      </doc>
+    </method>
+    <method name="setJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <doc>
+      <![CDATA[Set the mapreduce job
+ @param job the mapreduce job for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setJobState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"/>
+      <doc>
+      <![CDATA[Set the state for this job.
+ @param state the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the message of this job]]>
+      </doc>
+    </method>
+    <method name="setMessage"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the message for this job.
+ @param message the message for this job.]]>
+      </doc>
+    </method>
+    <method name="getDependentJobs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="isCompleted" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in a complete state]]>
+      </doc>
+    </method>
+    <method name="isReady" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in READY state]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="failJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submit this job to mapred. The state becomes RUNNING if submission 
+ is successful, FAILED otherwise.]]>
+      </doc>
+    </method>
+    <field name="CREATE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class encapsulates a MapReduce job and its dependency. It monitors 
+  the states of the depending jobs and updates the state of this job.
+  A job starts in the WAITING state. If it does not have any depending jobs,
+  or all of the depending jobs are in SUCCESS state, then the job state 
+  will become READY. If any depending jobs fail, the job will fail too. 
+  When in READY state, the job can be submitted to Hadoop for execution, with
+  the state changing into RUNNING state. From RUNNING state, the job 
+  can get into SUCCESS or FAILED state, depending 
+  the status of the job execution.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+  <class name="JobControl" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Runnable"/>
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a new controlled job.
+ @param aJob the new controlled job]]>
+      </doc>
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a new job.
+ @param aJob the new job]]>
+      </doc>
+    </method>
+    <method name="addJobCollection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getThreadState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl.ThreadState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[set the thread state to STOPPING so that the 
+ thread will stop when it wakes up.]]>
+      </doc>
+    </method>
+    <method name="suspend"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[suspend the running thread]]>
+      </doc>
+    </method>
+    <method name="resume"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[resume the suspended thread]]>
+      </doc>
+    </method>
+    <method name="allFinished" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The main loop for the thread.
+  The loop does the following:
+  	Check the states of the running jobs
+  	Update the states of waiting jobs
+  	Submit the jobs in ready state]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a set of MapReduce jobs and its dependency.
+   
+  It tracks the states of the jobs by placing them into different tables
+  according to their states. 
+  
+  This class provides APIs for the client app to add a job to the group 
+  and to get the jobs in the group in different states. When a job is 
+  added, an ID unique to the group is assigned to the job. 
+  
+  This class has a thread that submits jobs when they become ready, 
+  monitors the states of the running jobs, and updates the states of jobs
+  based on the state changes of their depending jobs states. The class 
+  provides APIs for suspending/resuming the thread, and 
+  for stopping the thread.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.join">
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <class name="ComposableInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ComposableInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <class name="ComposableRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ComposableRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapreduce.join.expr</tt> property and
+ user-supplied join types from <tt>mapreduce.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <field name="JOIN_EXPR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOIN_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapreduce.join.define.&lt;ident&gt;</tt> to a classname. 
+ In the expression <tt>mapreduce.join.expr</tt>, the identifier will be
+ assumed to be a ComposableRecordReader.
+ <tt>mapreduce.join.keycomparator</tt> can be a classname used to compare 
+ keys in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for failing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createTupleWritable" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="X"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="keyclass" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="key" type="K"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="value" type="X"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <class name="Parser.Node" extends="org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapreduce.lib.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if a call to next may return a value. This is permitted false
+ positives, but not false negatives.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign next value to actual.
+ It is required that elements added to a ResetableIterator be returned in
+ the same order after a call to {@link #reset} (FIFO).
+
+ Note that a call to this may fail for nested joins (i.e. more elements
+ available, but none satisfying the constraints of the join)]]>
+      </doc>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign last value returned to actual.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set iterator to return to the start of its range. Must be called after
+ calling {@link #add} to avoid a ConcurrentModificationException.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an element to the collection of elements to iterate over.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close datasources and release resources. Calling methods on the iterator
+ after calling close has undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Close datasources, but do not release internal resources. Calling this
+ method should permit the object to be reused with a different datasource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <class name="TupleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <method name="has" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Return true if tuple has an element at the position provided.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith Writable from Tuple.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of children in this Tuple.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator over the elements in this tuple.
+ Note that this doesn't flatten the tuple; one may receive tuples
+ from this iterator.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert Tuple to String as in the following.
+ <tt>[&lt;child1&gt;,&lt;child2&gt;,...,&lt;childn&gt;]</tt>]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes each Writable to <code>out</code>.
+ TupleWritable format:
+ {@code
+  <count><type1><type2>...<typen><obj1><obj2>...<objn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <field name="written" type="java.util.BitSet"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedRecordReader" type="int"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current key]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current value]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="empty" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmp" type="org.apache.hadoop.io.WritableComparator"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ 
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.map">
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <class name="MultithreadedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultithreadedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumberOfThreads" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[The number of threads in the thread pool that will run the map function.
+ @param job the job
+ @return the number of threads]]>
+      </doc>
+    </method>
+    <method name="setNumberOfThreads"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="threads" type="int"/>
+      <doc>
+      <![CDATA[Set the number of threads in the pool for running maps.
+ @param job the job to modify
+ @param threads the new number of threads]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the application's mapper class.
+ @param <K1> the map's input key type
+ @param <V1> the map's input value type
+ @param <K2> the map's output key type
+ @param <V2> the map's output value type
+ @param job the job
+ @return the mapper class to run]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the application's mapper class.
+ @param <K1> the map input key type
+ @param <V1> the map input value type
+ @param <K2> the map output key type
+ @param <V2> the map output value type
+ @param job the job to modify
+ @param cls the class to use as the mapper]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the application's maps using a thread pool.]]>
+      </doc>
+    </method>
+    <field name="NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAP_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Multithreaded implementation for @link org.apache.hadoop.mapreduce.Mapper.
+ <p>
+ It can be used instead of the default implementation,
+ {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU
+ bound in order to improve throughput.
+ <p>
+ Mapper implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured with the mapper to use via 
+ {@link #setMapperClass(Job, Class)} and
+ the number of thread the thread-pool can use with the
+ {@link #getNumberOfThreads(JobContext)} method. The default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <class name="TokenCounterMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenCounterMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Tokenize the input values and emit each word with a count of 1.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+  <class name="WrappedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMapContext" return="org.apache.hadoop.mapreduce.Mapper.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapContext" type="org.apache.hadoop.mapreduce.MapContext"/>
+      <doc>
+      <![CDATA[Get a wrapped {@link Mapper.Context} for custom implementations.
+ @param mapContext <code>MapContext</code> to be wrapped
+ @return a wrapped <code>Mapper.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} which wraps a given one to allow custom 
+ {@link Mapper.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.output">
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <class name="BindingPathOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BindingPathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instantiate.
+ @param outputPath output path (may be null)
+ @param context task context
+ @throws IOException on any failure.]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCommitter" return="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the inner committer.
+ @return the bonded committer.]]>
+      </doc>
+    </method>
+    <field name="NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The classname for use in configurations.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is a special committer which creates the factory for the committer and
+ runs off that. Why does it exist? So that you can explicitly instantiate
+ a committer by classname and yet still have the actual implementation
+ driven dynamically by the factory options and destination filesystem.
+ This simplifies integration
+ with existing code which takes the classname of a committer.
+ There's no factory for this, as that would lead to a loop.
+
+ All commit protocol methods and accessors are delegated to the
+ wrapped committer.
+
+ How to use:
+
+ <ol>
+   <li>
+     In applications which take a classname of committer in
+     a configuration option, set it to the canonical name of this class
+     (see {@link #NAME}). When this class is instantiated, it will
+     use the factory mechanism to locate the configured committer for the
+     destination.
+   </li>
+   <li>
+     In code, explicitly create an instance of this committer through
+     its constructor, then invoke commit lifecycle operations on it.
+     The dynamically configured committer will be created in the constructor
+     and have the lifecycle operations relayed to it.
+   </li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the path where final output of the job should be placed.  This
+ could also be considered the committed application attempt path.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @param out the output path to place these in.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param appAttemptId the ID of the application attempt for this job.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @param out The output path to put things in.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until
+ the entire job is committed.
+ @param context the context of the task attempt
+ @return the path where the output of a committed task is stored until
+ the entire job is committed.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until the
+ entire job is committed for a specific application attempt.
+ @param appAttemptId the id of the application attempt to use
+ @param context the context of any task.
+ @return the path where the output of a committed task is stored.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ @return the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the temporary directory that is the root of all of the task 
+ work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do works in commitJobInternal().
+ Could retry on failure if using algorithm 2.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJobInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do following commit job, include:
+ Move all committed tasks to the final output dir (algorithm 1 only).
+ Delete the temporary directory, including all of the work directories.
+ Create a _SUCCESS file to make it as successful.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the temporary directory, including all of the work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[No task setup required.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the files from the work directory to the job output directory
+ @param context the task context]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did this task write any files in the work directory?
+ @param context the task's context]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="PENDING_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of directory where pending data is placed.  Data that has not been
+ committed yet.]]>
+      </doc>
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name 
+
+ The static variable to be compatible with M/R 1.x]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCESSFUL_JOB_OUTPUT_DIR_MARKER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param job the job to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param job the Job to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param job the job to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param job the {@link Job} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param job The job to modify
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(TaskInputOutputContext)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in a work directory during execution 
+ of his task i.e. via 
+ {@link #getWorkOutputPath(TaskInputOutputContext)}, and
+ the framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getPathForWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueFile} method to make the file name
+ unique for the task.</p>
+
+ @param context the context for the task.
+ @param name the name for the file.
+ @param extension the extension for the file
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getUniqueFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a unique filename, based on the task id, name, and extension
+ @param context the task that is calling this
+ @param name the base filename
+ @param extension the filename extension
+ @return a string like $name-[mrsct]-$id$extension]]>
+      </doc>
+    </method>
+    <method name="getDefaultWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the default path and filename for the output format.
+ @param context the task context
+ @param extension an extension to add to the filename
+ @return a full path $output/_temporary/$taskid/part-[mr]-$id
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getOutputName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the base output name for the output file.]]>
+      </doc>
+    </method>
+    <method name="setOutputName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the base output name for output file to be created.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BASE_OUTPUT_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PART" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option: should output be compressed? {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_CODEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If compression is enabled, name of codec: {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Type of compression {@value}: NONE, RECORD, BLOCK.
+ Generally only used in {@code SequenceFileOutputFormat}.]]>
+      </doc>
+    </field>
+    <field name="OUTDIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Destination directory of work: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}s that read from {@link FileSystem}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <class name="FileOutputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapreduce.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the underlying output format.
+ @param baseOut the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapreduce.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link Job} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="OUTPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.
+ Use in conjuction with org.apache.hadoop.mapreduce.lib.output.MultipleOutputs to recreate the
+ behaviour of org.apache.hadoop.mapred.lib.MultipleTextOutputFormat (etc) of the old Hadoop API.
+ See {@link MultipleOutputs} documentation for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapreduce.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes 
+ {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple outputs support,
+ it should be instantiated in the Mapper/Reducer setup method.
+
+ @param context the TaskInputOutputContext object]]>
+      </doc>
+    </constructor>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param job               job to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ 
+ The counters group is the {@link MultipleOutputs} class name.
+ The names of the counters are the same as the named outputs. These
+ counters count the number records written to each output name.
+ By default these counters are disabled.
+
+ @param job    job  to enable counters
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ By default these counters are disabled.
+
+ @param job    the job 
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to the namedOutput.
+
+ Output path is a unique file generated for the namedOutput.
+ For example, {namedOutput}-(m|r)-{part-number}
+ 
+ @param namedOutput the named output name
+ @param key         the key
+ @param value       the value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to baseOutputPath using the namedOutput.
+ 
+ @param namedOutput    the named output name
+ @param key            the key
+ @param value          the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key value to an output file name.
+ 
+ Gets the record writer from job's output format.  
+ Job's output format should be a FileOutputFormat.
+ 
+ @param key       the key
+ @param value     the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Closes all the opened outputs.
+ 
+ This should be called from cleanup method of map/reduce task.
+ If overridden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing output data 
+ to multiple outputs
+ 
+ <p> 
+ Case one: writing to additional outputs other than the job default output.
+
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ </p>
+ 
+ <p>
+ Case two: to write data to different files provided by user
+ </p>
+ 
+ <p>
+ MultipleOutputs supports counters, by default they are disabled. The 
+ counters group is the {@link MultipleOutputs} class name. The names of the 
+ counters are the same as the output name. These count the number records 
+ written to each output name.
+ </p>
+ 
+ Usage pattern for job submission:
+ <pre>
+
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>
+ <p>
+ Usage in Reducer:
+ <pre>
+ &lt;K, V&gt; String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer&lt;WritableComparable, Writable,WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>
+ 
+ <p>
+ When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat,
+ MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat
+ from the old Hadoop API - ie, output can be written from the Reducer to more than one location.
+ </p>
+ 
+ <p>
+ Use <code>MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath)</code> to write key and 
+ value to a path specified by <code>baseOutputPath</code>, with no need to specify a named output.
+ <b>Warning</b>: when the baseOutputPath passed to MultipleOutputs.write
+ is a path that resolves outside of the final job output directory, the
+ directory is created immediately and then persists through subsequent
+ task retries, breaking the concept of output committing:
+ </p>
+ 
+ <pre>
+ private MultipleOutputs&lt;Text, Text&gt; out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs&lt;Text, Text&gt;(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable&lt;Text&gt; values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(&lt;<i>parameter list...</i>&gt;));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+ </pre>
+ 
+ <p>
+ Use your own code in <code>generateFileName()</code> to create a custom path to your results. 
+ '/' characters in <code>baseOutputPath</code> will be translated into directory levels in your file system. 
+ Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. 
+ No call to <code>context.write()</code> is necessary. See example <code>generateFileName()</code> code below. 
+ </p>
+ 
+ <pre>
+ private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+ </pre>
+ 
+ <p>
+ Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000.
+ To prevent this use <code>LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);</code>
+ instead of <code>job.setOutputFormatClass(TextOutputFormat.class);</code> in your Hadoop job configuration.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <class name="PartialFileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter"/>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="cleanUpPartialOutputForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <interface name="PartialOutputCommitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="cleanUpPartialOutputForTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove all previously committed outputs from prior executions of this task.
+ @param context Context for cleaning up previously promoted output.
+ @throws IOException If cleanup fails, then the state of the task my not be
+                     well defined.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for an {@link org.apache.hadoop.mapreduce.OutputCommitter}
+ implementing partial commit of task output, as during preemption.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <class name="PathOutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a task attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a job attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final directory where work will be placed once the job
+ is committed. This may be null, in which case, there is no output
+ path to write data to.
+ @return the path where final output of the job should be placed.]]>
+      </doc>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate: is there an output path?
+ @return true if we have an output path set, else false.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ Warning: there's no guarantee that this work path is on the same
+ FS as the final output, or that it's visible across machines.
+ May be null.
+ @return the work directory
+ @throws IOException IO problem]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A committer which somehow commits data written to a working directory
+ to the final directory during the commit process. The reference
+ implementation of this is the {@link FileOutputCommitter}.
+
+ There are two constructors, both of which do nothing but long and
+ validate their arguments.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes keys, 
+ values to {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSequenceWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.partition">
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOffsets"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="left" type="int"/>
+      <param name="right" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[left:(right+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param left left Python-style offset
+ @param right right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setLeftOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[offset:]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset left Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setRightOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[:(offset+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.BinaryComparable"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Use (the specified slice of the array returned by) 
+ {@link BinaryComparable#getBytes()} to partition.]]>
+      </doc>
+    </method>
+    <field name="LEFT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RIGHT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}.</p>
+ 
+ <p>The subarray to be used for the partitioning can be defined by means
+ of the following properties:
+ <ul>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.left.offset</i>:
+     left offset in array (0 by default)
+   </li>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.right.offset</i>: 
+     right offset in array (-1 by default)
+   </li>
+ </ul>
+ Like in Python, both negative and positive offsets are allowed, but
+ the meaning is slightly different. In case of an array of length 5,
+ for instance, the possible offsets are:
+ <pre><code>
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+ </code></pre>
+ The first row of numbers gives the position of the offsets 0...5 in 
+ the array; the second row gives the corresponding negative offsets. 
+ Contrary to Python, the specified subarray has byte <code>i</code> 
+ and <code>j</code> as first and last element, repectively, when 
+ <code>i</code> and <code>j</code> are the left and right offset.
+ 
+ <p>For Hadoop programs written in Java, it is advisable to use one of 
+ the following static convenience methods for setting the offsets:
+ <ul>
+   <li>{@link #setOffsets}</li>
+   <li>{@link #setLeftOffset}</li>
+   <li>{@link #setRightOffset}</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <class name="HashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="InputSampler" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="sampler" type="org.apache.hadoop.mapreduce.lib.partition.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write a partition file for the given job, using the Sampler provided.
+ Queries the sampler for a sample keyset, sorts by the output key
+ comparator, selects the keys for each rank, and writes to the destination
+ returned from {@link TotalOrderPartitioner#getPartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Driver for InputSampler from the command line.
+ Configures a JobConf instance and calls {@link #writePartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[Utility for collecting samples and writing a partition file for
+ {@link TotalOrderPartitioner}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.io.WritableComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <field name="COMPARATOR_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by 
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="end" type="int"/>
+      <param name="currentHash" type="int"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hash" type="int"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <field name="PARTITIONER_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <class name="RehashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RehashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Rehash {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This partitioner rehashes values returned by {@link Object#hashCode()}
+  to get smoother distribution between partitions which may improve
+  reduce reduce time in some cases and should harm things in no cases.
+  This partitioner is suggested with Integer and Long keys with simple
+  patterns in their distributions.
+  @since 2.0.3]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Read in the partition file and build indexing data structures.
+ If the keytype is {@link org.apache.hadoop.io.BinaryComparable} and
+ <tt>total.order.partitioner.natural.order</tt> is not false, a trie
+ of the first <tt>total.order.partitioner.max.trie.depth</tt>(2) + 1 bytes
+ will be built. Otherwise, keys will be located using a binary search of
+ the partition keyset using the {@link org.apache.hadoop.io.RawComparator}
+ defined for this job. The input file must be sorted with the same
+ comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(Configuration, Path)]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_TRIE_DEPTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NATURAL_ORDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.reduce">
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <class name="IntSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IntSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="Key"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+  <class name="WrappedReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReducerContext" return="org.apache.hadoop.mapreduce.Reducer.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reduceContext" type="org.apache.hadoop.mapreduce.ReduceContext"/>
+      <doc>
+      <![CDATA[A a wrapped {@link Reducer.Context} for custom implementations.
+ @param reduceContext <code>ReduceContext</code> to be wrapped
+ @return a wrapped <code>Reducer.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} which wraps a given one to allow for custom 
+ {@link Reducer.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.server.jobtracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.server.tasktracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.task.annotation">
+  <!-- start class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+  <class name="Checkpointable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Contract representing to the framework that the task can be safely preempted
+ and restarted between invocations of the user-defined function.
+
+ This is often true when the result of a function does not rely on state
+ derived from previous elements in the record stream, but the guarantee is
+ left as an exercise to the implementor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+</package>
+<package name="org.apache.hadoop.mapreduce.tools">
+  <!-- start class org.apache.hadoop.mapreduce.tools.CLI -->
+  <class name="CLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="CLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CLI" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="counters" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTaskLogURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <param name="baseUrl" type="java.lang.String"/>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param job the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws InterruptedException
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="displayJobList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="cluster" type="org.apache.hadoop.mapreduce.Cluster"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="headerPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="dataPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interprets the map reduce cli options]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.tools.CLI -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.3.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.3.xml
new file mode 100644
index 0000000000000..ebe4c64b65fb0
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.3.xml
@@ -0,0 +1,28087 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:13:41 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Core 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.3.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Core 3.3.3 -->
+<package name="org.apache.hadoop.filecache">
+  <!-- start class org.apache.hadoop.filecache.DistributedCache -->
+  <class name="DistributedCache" extends="org.apache.hadoop.mapreduce.filecache.DistributedCache"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DistributedCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a archive that has been localized to the conf.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="addLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a file that has been localized to the conf..  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <method name="createAllSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobCacheDir" type="java.io.File"/>
+      <param name="workDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method create symlinks for all files in a given dir in another
+ directory. Currently symlinks cannot be disabled. This is a NO-OP.
+
+ @param conf the configuration
+ @param jobCacheDir the target directory for creating symlinks
+ @param workDir the directory in which the symlinks are created
+ @throws IOException
+ @deprecated Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns {@link FileStatus} of a given cache file on hdfs. Internal to
+ MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return <code>FileStatus</code> of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns mtime of a given cache file on hdfs. Internal to MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return mtime of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setArchiveTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the archives to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of archives.
+ The order should be the same as the order in which the archives are added.]]>
+      </doc>
+    </method>
+    <method name="setFileTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the files to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of files.
+ The order should be the same as the order in which the files are added.]]>
+      </doc>
+    </method>
+    <method name="setLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized archives.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="setLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized files.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <field name="CACHE_FILES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALFILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALFILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALFILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_SYMLINK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_SYMLINK} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_SYMLINK}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Distribute application-specific large, read-only files efficiently.
+
+ <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+ </p>
+
+ <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ <code>DistributedCache</code> assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.</p>
+
+ <p>The framework will copy the necessary files on to the worker node before
+ any tasks for the job are executed on that node. Its efficiency stems from
+ the fact that the files are only copied once per job and the ability to
+ cache archives which are un-archived on the workers.</p>
+
+ <p><code>DistributedCache</code> can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism.  Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task.  In the current
+ version symlinks are always created.  If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used.  All
+ others will not even be downloaded.</p>
+
+ <p><code>DistributedCache</code> tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.</p>
+
+ <p>Here is an illustrative example on how to use the
+ <code>DistributedCache</code>:</p>
+ <p><blockquote><pre>
+     // Setting up the cache for the application
+
+     1. Copy the requisite files to the <code>FileSystem</code>:
+
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+     2. Setup the application's <code>JobConf</code>:
+
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job);
+
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+
+     public static class MapClass extends MapReduceBase
+     implements Mapper&lt;K, V, K, V&gt; {
+
+       private Path[] localArchives;
+       private Path[] localFiles;
+
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+
+       public void map(K key, V value,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+
+ </pre></blockquote>
+
+ It is also very common to use the DistributedCache by using
+ {@link org.apache.hadoop.util.GenericOptionsParser}.
+
+ This class includes methods that should be used by users
+ (specifically those mentioned in the example above, as well
+ as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
+ as well as methods intended for use by the MapReduce framework
+ (e.g., {@link org.apache.hadoop.mapred.JobClient}).
+
+ @see org.apache.hadoop.mapred.JobConf
+ @see org.apache.hadoop.mapred.JobClient
+ @see org.apache.hadoop.mapreduce.Job]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.filecache.DistributedCache -->
+</package>
+<package name="org.apache.hadoop.mapred">
+  <!-- start class org.apache.hadoop.mapred.ClusterStatus -->
+  <class name="ClusterStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getTaskTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of task trackers in the cluster.
+ 
+ @return the number of task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getActiveTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the active task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return an empty graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return 0 graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted task trackers in the cluster.
+ 
+ @return the number of blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getNumExcludedNodes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of excluded hosts in the cluster.
+ @return the number of excluded hosts in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getTTExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running map tasks in the cluster.
+ 
+ @return the number of currently running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running reduce tasks in the cluster.
+ 
+ @return the number of currently running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running map tasks in the cluster.
+ 
+ @return the maximum capacity for running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running reduce tasks in the cluster.
+ 
+ @return the maximum capacity for running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker]]>
+      </doc>
+    </method>
+    <method name="getMaxMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getUsedMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTrackersInfo" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the list of blacklisted trackers along with reasons for blacklisting.
+ 
+ @return the collection of {@link BlackListInfo} objects.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerState" return="org.apache.hadoop.mapred.JobTracker.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current state of the <code>JobTracker</code>,
+ as {@link JobTracker.State}
+
+ {@link JobTracker.State} should no longer be used on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return the invalid state of the <code>JobTracker</code>.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="UNINITIALIZED_MEMORY_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterStatus</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster. 
+   </li>
+   <li>
+   Name of the trackers. 
+   </li>
+   <li>
+   Task capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   State of the <code>JobTracker</code>.
+   </li>
+   <li>
+   Details regarding black listed trackers.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterStatus</code>, via 
+ {@link JobClient#getClusterStatus()}.</p>
+ 
+ @see JobClient]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ClusterStatus -->
+  <!-- start class org.apache.hadoop.mapred.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getGroup" return="org.apache.hadoop.mapred.Counters.Group"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+    </method>
+    <method name="getGroupNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="makeCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String, String)} instead">
+      <param name="group" type="java.lang.String"/>
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter by using strings
+ @param group the name of the group
+ @param id the id of the counter within the group (0 to N-1)
+ @param name the internal name of the counter
+ @return the counter for that name
+ @deprecated use {@link #findCounter(String, String)} instead]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param key identifies a counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param group the name of the group
+ @param counter the internal name of the counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Returns current value of the specified counter, or 0 if the counter
+ does not exist.
+ @param key the counter enum to lookup
+ @return the counter value or 0 if counter not found]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #countCounters()} instead">
+      <doc>
+      <![CDATA[@return the total number of counters
+ @deprecated use {@link #countCounters()} instead]]>
+      </doc>
+    </method>
+    <method name="sum" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.mapred.Counters"/>
+      <param name="b" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Convenience method for computing the sum of two sets of counters.
+ @param a the first counters
+ @param b the second counters
+ @return a new summed counters object]]>
+      </doc>
+    </method>
+    <method name="log"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <doc>
+      <![CDATA[Logs the current counter values.
+ @param log The log to use.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Represent the counter in a textual format that can be converted back to
+ its object form
+ @return the string in the following format
+ {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*]]>
+      </doc>
+    </method>
+    <method name="fromEscapedCompactString" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compactString" type="java.lang.String"/>
+      <exception name="ParseException" type="java.text.ParseException"/>
+      <doc>
+      <![CDATA[Convert a stringified (by {@link #makeEscapedCompactString()} counter
+ representation into a counter object.
+ @param compactString to parse
+ @return a new counters object
+ @throws ParseException]]>
+      </doc>
+    </method>
+    <field name="MAX_COUNTER_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_GROUP_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A set of named counters.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Counter -->
+  <class name="Counters.Counter" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.Counter"/>
+    <constructor name="Counter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="increment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the compact stringified version of the counter in the format
+ [(actual-name)(display-name)(value)]
+ @return the stringified result]]>
+      </doc>
+    </method>
+    <method name="contentEquals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+      <doc>
+      <![CDATA[Checks for (content) equality of two (basic) counters
+ @param counter to compare
+ @return true if content equals
+ @deprecated]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the value of the counter]]>
+      </doc>
+    </method>
+    <method name="getUnderlyingCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A counter record, comprising its name and value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Counter -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Group -->
+  <class name="Counters.Group" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <constructor name="Group"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param counterName the name of the counter
+ @return the value of the specified counter, or 0 if the counter does
+ not exist.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the compact stringified version of the group in the format
+ {(actual-name)(display-name)(value)[][][]} where [] are compact strings
+ for the counters within.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String)} instead">
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given id and create it if it doesn't exist.
+ @param id the numeric id of the counter within the group
+ @param name the internal counter name
+ @return the counter
+ @deprecated use {@link #findCounter(String)} instead]]>
+      </doc>
+    </method>
+    <method name="getCounterForName" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given name and create it if it doesn't exist.
+ @param name the internal counter name
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="addCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    </method>
+    <method name="getUnderlyingGroup" return="org.apache.hadoop.mapreduce.counters.CounterGroupBase"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>Group</code> of counters, comprising of counters from a particular
+  counter {@link Enum} class.
+
+  <p><code>Group</code>handles localization of the class name and the
+  counter names.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Group -->
+  <!-- start class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.mapred.FileInputFormat -->
+  <class name="FileInputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setMinSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSize" type="long"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ If security is enabled, this method collects
+ delegation tokens from the input paths and adds them to the job's
+ credentials.
+ @param job the job to list input paths for and attach tokens to.
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Splits files returned by {@link #listStatus(JobConf)} when
+ they're too big.]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="goalSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="blockSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job. 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getSplitHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+      <param name="splitSize" type="long"/>
+      <param name="clusterMap" type="org.apache.hadoop.net.NetworkTopology"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This function identifies and returns the hosts that contribute 
+ most for a given split. For calculating the contribution, rack
+ locality is treated on par with host locality, so hosts from racks
+ that contribute the most are preferred over hosts on racks that 
+ contribute less
+ @param blkLocations The list of block locations
+ @param offset 
+ @param splitSize 
+ @return an array of hosts that contribute most to this split
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}.
+ 
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(FileSystem, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapred.OutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <param name="outputPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="runState" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param conf the {@link JobConf} to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param conf the {@link JobConf} to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param conf the {@link JobConf} to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param conf the {@link JobConf} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(JobConf)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p><i>Note:</i> The following is valid only if the {@link OutputCommitter}
+  is {@link FileOutputCommitter}. If <code>OutputCommitter</code> is not 
+  a <code>FileOutputCommitter</code>, the task's temporary output
+  directory is same as {@link #getOutputPath(JobConf)} i.e.
+  <tt>${mapreduce.output.fileoutputformat.outputdir}$</tt></p>
+  
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in <tt>${mapreduce.task.output.dir}</tt> during execution 
+ of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the 
+ framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p><i>Note</i>: the value of <tt>${mapreduce.task.output.dir}</tt> during 
+ execution of a particular task-attempt is actually 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}</tt>, and this value is 
+ set by the map-reduce framework. So, just create any side-files in the 
+ path  returned by {@link #getWorkOutputPath(JobConf)} from map/reduce 
+ task to take advantage of this feature.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Helper function to create the task's temporary output directory and 
+ return the path to the task's output file.
+ 
+ @param conf job-configuration
+ @param name temporary task-output filename
+ @return path to the task's temporary output file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getUniqueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a name that is unique for the task.
+
+ <p>The generated name can be used to create custom files from within the
+ different tasks for the job, the names for different tasks will not collide
+ with each other.</p>
+
+ <p>The given name is postfixed with the task type, 'm' for maps, 'r' for
+ reduces and the task partition number. For example, give a name 'test'
+ running on the first map o the job the generated name will be
+ 'test-m-00000'.</p>
+
+ @param conf the configuration for the job.
+ @param name the name to make unique.
+ @return a unique name accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getPathForCustomFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueName} method to make the file name
+ unique for the task.</p>
+
+ @param conf the configuration for the job.
+ @param name the name for the file.
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplitWithLocationInfo"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[Constructs a split.
+ @deprecated
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobConf, int)} and passed to
+ {@link InputFormat#getRecordReader(InputSplit,JobConf,Reporter)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileSplit -->
+  <!-- start class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.ID -->
+  <class name="ID" extends="org.apache.hadoop.mapreduce.ID"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ID -->
+  <!-- start interface org.apache.hadoop.mapred.InputFormat -->
+  <interface name="InputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple.
+ 
+ @param job job configuration.
+ @param numSplits the desired number of splits, a hint.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordReader} for the given {@link InputSplit}.
+
+ <p>It is the responsibility of the <code>RecordReader</code> to respect
+ record boundaries while processing the logical split to present a 
+ record-oriented view to the individual task.</p>
+ 
+ @param split the {@link InputSplit}
+ @param job the job that this split belongs to
+ @return a {@link RecordReader}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to be respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibilty to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see JobClient
+ @see FileInputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplit -->
+  <interface name="InputSplit"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the total number of bytes in the data of the <code>InputSplit</code>.
+ 
+ @return the number of bytes in the input split.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hostnames where the input split is located.
+ 
+ @return list of hostnames where data of the <code>InputSplit</code> is
+         located as an array of <code>String</code>s.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplit -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <interface name="InputSplitWithLocationInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <!-- start class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <class name="InvalidFileTypeException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidFileTypeException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidFileTypeException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when file type differs from the desired file type. like 
+ getting a file when a directory is expected. Or a wrong file type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ The first element of the list is used as the init cause value.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <class name="InvalidJobConfException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidJobConfException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when jobconf misses some mendatory attributes
+ or value of some attributes is invalid.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <!-- start class org.apache.hadoop.mapred.JobClient -->
+  <class name="JobClient" extends="org.apache.hadoop.mapreduce.tools.CLI"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="JobClient"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job client.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link JobConf}, and connect to the 
+ default cluster
+ 
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link Configuration}, 
+ and connect to the default cluster
+ 
+ @param conf the configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client, connect to the indicated job tracker.
+ 
+ @param jobTrackAddr the job tracker to connect to.
+ @param conf configuration.]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Connect to the default cluster
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>JobClient</code>.]]>
+      </doc>
+    </method>
+    <method name="getFs" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a filesystem handle.  We need this to prepare jobs
+ for submission to the MapReduce system.
+ 
+ @return the filesystem handle.]]>
+      </doc>
+    </method>
+    <method name="getClusterHandle" return="org.apache.hadoop.mapreduce.Cluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a handle to the Cluster]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobFile" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ 
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param jobFile the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws InvalidJobConfException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobInner" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an {@link RunningJob} object to track an ongoing job.  Returns
+ null if the id does not correspond to any known job.
+
+ @param jobid the jobid of the job.
+ @return the {@link RunningJob} handle to track the job, null if the
+         <code>jobid</code> doesn't correspond to any known job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getJob(JobID)}.">
+      <param name="jobid" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getJob(JobID)}.]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the map tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getMapTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the reduce tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the reduce tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCleanupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the cleanup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the cleanup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSetupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the setup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the setup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getReduceTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param jobId the ID of the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="detailed" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @param  detailed if true then get a detailed status including the
+         tracker names
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobsToComplete" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are not completed and not failed.
+ 
+ @return array of {@link JobStatus} for the running/to-be-run jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are submitted.
+ 
+ @return array of {@link JobStatus} for the submitted jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility that submits a job, then polls for progress until the job is
+ complete.
+ 
+ @param job the job configuration.
+ @throws IOException if the job fails]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="job" type="org.apache.hadoop.mapred.RunningJob"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @param conf the job's configuration
+ @param job the job to track
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Sets the output filter for tasks. only those tasks are printed whose
+ output matches the filter. 
+ @param newValue task filter.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the task output filter out of the JobConf.
+ 
+ @param job the JobConf to examine.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the JobConf to set the task output filter.
+ 
+ @param job the JobConf to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task output filter.
+ @return task filter.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cntrs" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Maps in the cluster.
+  
+ @return the max available Maps in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDefaultReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Reduces in the cluster.
+  
+ @return the max available Reduces in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where job-specific files are to be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="isJobDirValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobDirPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if the job directory is clean and has all the required components
+ for (re) starting the job]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fetch the staging area directory for the application
+ 
+ @return path to staging area directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about root level queues
+ configured
+
+ @return the array of root level JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about immediate children
+ of queue queueName.
+ 
+ @param queueName
+ @return the array of immediate children JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of queue information objects about all the Job Queues
+ configured.
+ 
+ @return Array of JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobsFromQueue" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets all the jobs which were added to particular Job Queue
+ 
+ @param queueName name of the Job Queue
+ @return Array of jobs present in the job queue
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.mapred.JobQueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the queue information associated to a particular Job Queue
+ 
+ @param queueName name of the job queue.
+ @return Queue information associated to particular queue.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapred.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return true if the renewal went well
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>JobClient</code> is the primary interface for the user-job to interact
+ with the cluster.
+ 
+ <code>JobClient</code> provides facilities to submit jobs, track their 
+ progress, access component-tasks' reports/logs, get the Map-Reduce cluster
+ status information etc.
+ 
+ <p>The job submission process involves:
+ <ol>
+   <li>
+   Checking the input and output specifications of the job.
+   </li>
+   <li>
+   Computing the {@link InputSplit}s for the job.
+   </li>
+   <li>
+   Setup the requisite accounting information for the {@link DistributedCache} 
+   of the job, if necessary.
+   </li>
+   <li>
+   Copying the job's jar and configuration to the map-reduce system directory 
+   on the distributed file-system. 
+   </li>
+   <li>
+   Submitting the job to the cluster and optionally monitoring
+   it's status.
+   </li>
+ </ol>
+  
+ Normally the user creates the application, describes various facets of the
+ job via {@link JobConf} and then uses the <code>JobClient</code> to submit 
+ the job and monitor its progress.
+ 
+ <p>Here is an example on how to use <code>JobClient</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+ </pre></blockquote>
+ 
+ <b id="JobControl">Job Control</b>
+ 
+ <p>At times clients would chain map-reduce jobs to accomplish complex tasks 
+ which cannot be done via a single map-reduce job. This is fairly easy since 
+ the output of the job, typically, goes to distributed file-system and that 
+ can be used as the input for the next job.</p>
+ 
+ <p>However, this also means that the onus on ensuring jobs are complete 
+ (success/failure) lies squarely on the clients. In such situations the 
+ various job-control options are:
+ <ol>
+   <li>
+   {@link #runJob(JobConf)} : submits the job and returns only after 
+   the job has completed.
+   </li>
+   <li>
+   {@link #submitJob(JobConf)} : only submits the job, then poll the 
+   returned handle to the {@link RunningJob} to query status and make 
+   scheduling decisions.
+   </li>
+   <li>
+   {@link JobConf#setJobEndNotificationURI(String)} : setup a notification
+   on job-completion, thus avoiding polling.
+   </li>
+ </ol>
+ 
+ @see JobConf
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobClient -->
+  <!-- start class org.apache.hadoop.mapred.JobConf -->
+  <class name="JobConf" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new map/reduce configuration where the behavior of reading from the
+ default resources can be turned off.
+ <p>
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user jar for the map-reduce job.
+ 
+ @return the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user jar for the map-reduce job.
+ 
+ @param jar the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getJarUnpackPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pattern for jar contents to unpack on the tasktracker]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the job's jar file by finding an example class location.
+ 
+ @param cls the example class.]]>
+      </doc>
+    </method>
+    <method name="getLocalDirs" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use MRAsyncDiskService.moveAndDeleteAllVolumes instead.]]>
+      </doc>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subdir" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs a local file name. Files are distributed among configured
+ local directories.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setKeepFailedTaskFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should keep the intermediate files for 
+ failed tasks.
+ 
+ @param keep <code>true</code> if framework should keep the intermediate files 
+             for failed tasks, <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="getKeepFailedTaskFiles" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the temporary files for failed tasks be kept?
+ 
+ @return should the files be kept?]]>
+      </doc>
+    </method>
+    <method name="setKeepTaskFilesPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set a regular expression for task names that should be kept. 
+ The regular expression ".*_m_000123_0" would keep the files
+ for the first instance of map 123 that ran.
+ 
+ @param pattern the java.util.regex.Pattern to match against the 
+        task names.]]>
+      </doc>
+    </method>
+    <method name="getKeepTaskFilesPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the regular expression that is matched against the task names
+ to see if we need to keep the files.
+ 
+ @return the pattern as a string, if it was set, othewise null.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="setNumTasksToExecutePerJvm"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numTasks" type="int"/>
+      <doc>
+      <![CDATA[Sets the number of tasks that a spawned task JVM should run
+ before it exits
+ @param numTasks the number of tasks to execute; defaults to 1;
+ -1 signifies no limit]]>
+      </doc>
+    </method>
+    <method name="getNumTasksToExecutePerJvm" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of tasks that a spawned JVM should execute]]>
+      </doc>
+    </method>
+    <method name="getInputFormat" return="org.apache.hadoop.mapred.InputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link InputFormat} implementation for the map-reduce job,
+ defaults to {@link TextInputFormat} if not specified explicity.
+ 
+ @return the {@link InputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link InputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormat" return="org.apache.hadoop.mapred.OutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} implementation for the map-reduce job,
+ defaults to {@link TextOutputFormat} if not specified explicity.
+ 
+ @return the {@link OutputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapred.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} implementation for the map-reduce job,
+ defaults to {@link FileOutputCommitter} if not specified explicitly.
+ 
+ @return the {@link OutputCommitter} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputCommitter} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputCommitter} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setOutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setCompressMapOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Should the map outputs be compressed before transfer?
+ 
+ @param compress should the map outputs be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressMapOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are the outputs of the maps be compressed?
+ 
+ @return <code>true</code> if the outputs of the maps are to be compressed,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the given class as the  {@link CompressionCodec} for the map outputs.
+ 
+ @param codecClass the {@link CompressionCodec} class that will compress  
+                   the map outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the map outputs.
+ 
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} class that should be used to compress the 
+         map outputs.
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+  
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ 
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link RawComparator} comparator used to compare keys.
+ 
+ @param theClass the {@link RawComparator} comparator used to 
+                 compare keys.
+ @see #setOutputValueGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see #setCombinerKeyGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for 
+ grouping keys of inputs to the reduce.
+ 
+ @return comparator set by the user for grouping values.
+ @see #setOutputValueGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for
+ grouping keys in the input to the combiner.
+
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control
+ how keys are sorted, this can be used in conjunction to simulate
+ <i>secondary sort on values</i>.</p>
+
+ <p><i>Note</i>: This is not a guarantee of the combiner sort being
+ <i>stable</i> in any sense. (In any case, with the order of available
+ map-outputs to the combiner being non-deterministic, it wouldn't make
+ that much sense.)</p>
+
+ @param theClass the comparator class to be used for grouping keys for the
+ combiner. It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setOutputValueGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for 
+ grouping keys in the input to the reduce.
+ 
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to 
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+  
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+ 
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control 
+ how keys are sorted, this can be used in conjunction to simulate 
+ <i>secondary sort on values</i>.</p>
+  
+ <p><i>Note</i>: This is not a guarantee of the reduce sort being 
+ <i>stable</i> in any sense. (In any case, with the order of available 
+ map-outputs to the reduce being non-deterministic, it wouldn't make 
+ that much sense.)</p>
+ 
+ @param theClass the comparator class to be used for grouping keys. 
+                 It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)
+ @see #setCombinerKeyGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="getUseNewMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the mapper?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewMapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the mapper.
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getUseNewReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the reducer?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewReducer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the reducer. 
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ 
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} class for the job.
+ 
+ @param theClass the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapRunnerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link MapRunnable} class for the job.
+ 
+ @return the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapRunnerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Expert: Set the {@link MapRunnable} class for the job.
+ 
+ Typically used to exert greater control on {@link Mapper}s.
+ 
+ @param theClass the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Partitioner} used to partition {@link Mapper}-outputs 
+ to be sent to the {@link Reducer}s.
+ 
+ @return the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} class used to partition 
+ {@link Mapper}-outputs to be sent to the {@link Reducer}s.
+ 
+ @param theClass the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} class for the job.
+ 
+ @param theClass the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. Typically the combiner is same as the
+ the {@link Reducer} for the job i.e. {@link #getReducerClass()}.
+ 
+ @return the user-defined combiner class used to combine map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. 
+ 
+ <p>The combiner is an application-specified aggregation operation, which
+ can help cut down the amount of data transferred between the 
+ {@link Mapper} and the {@link Reducer}, leading to better performance.</p>
+ 
+ <p>The framework may invoke the combiner 0, 1, or multiple times, in both
+ the mapper and reducer tasks. In general, the combiner is called as the
+ sort/merge result is written to disk. The combiner must:
+ <ul>
+   <li> be side-effect free</li>
+   <li> have the same input and output key types and the same input and 
+        output value types</li>
+ </ul>
+ 
+ <p>Typically the combiner is same as the <code>Reducer</code> for the  
+ job i.e. {@link #setReducerClass(Class)}.</p>
+ 
+ @param theClass the user-defined combiner class used to combine 
+                 map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getMapSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for map tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be 
+                           used for this job for map tasks,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getReduceSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for reduce tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used 
+                           for reduce tasks for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getNumMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of map tasks for this job.
+ Defaults to <code>1</code>.
+ 
+ @return the number of map tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumMapTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the number of map tasks for this job.
+ 
+ <p><i>Note</i>: This is only a <i>hint</i> to the framework. The actual 
+ number of spawned map tasks depends on the number of {@link InputSplit}s 
+ generated by the job's {@link InputFormat#getSplits(JobConf, int)}.
+  
+ A custom {@link InputFormat} is typically used to accurately control 
+ the number of map tasks for the job.</p>
+ 
+ <b id="NoOfMaps">How many maps?</b>
+ 
+ <p>The number of maps is usually driven by the total size of the inputs 
+ i.e. total number of blocks of the input files.</p>
+  
+ <p>The right level of parallelism for maps seems to be around 10-100 maps 
+ per-node, although it has been set up to 300 or so for very cpu-light map 
+ tasks. Task setup takes awhile, so it is best if the maps take at least a 
+ minute to execute.</p>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of input files. However, the {@link FileSystem} blocksize of the 
+ input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+  
+ <p>Thus, if you expect 10TB of input data and have a blocksize of 128MB, 
+ you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is 
+ used to set it even higher.</p>
+ 
+ @param n the number of map tasks for this job.
+ @see InputFormat#getSplits(JobConf, int)
+ @see FileInputFormat
+ @see FileSystem#getDefaultBlockSize()
+ @see FileStatus#getBlockSize()]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of reduce tasks for this job. Defaults to
+ <code>1</code>.
+
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the requisite number of reduce tasks for this job.
+ 
+ <b id="NoOfReduces">How many reduces?</b>
+ 
+ <p>The right number of reduces seems to be <code>0.95</code> or 
+ <code>1.75</code> multiplied by (
+ <i>available memory for reduce tasks</i>
+ (The value of this should be smaller than
+ numNodes * yarn.nodemanager.resource.memory-mb
+ since the resource of memory is shared by map tasks and other
+ applications) /
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.reduce.memory.mb">
+ mapreduce.reduce.memory.mb</a>).
+ </p>
+ 
+ <p>With <code>0.95</code> all of the reduces can launch immediately and 
+ start transfering map outputs as the maps finish. With <code>1.75</code> 
+ the faster nodes will finish their first round of reduces and launch a 
+ second wave of reduces doing a much better job of load balancing.</p>
+ 
+ <p>Increasing the number of reduces increases the framework overhead, but 
+ increases load balancing and lowers the cost of failures.</p>
+ 
+ <p>The scaling factors above are slightly less than whole numbers to 
+ reserve a few reduce slots in the framework for speculative-tasks, failures
+ etc.</p> 
+
+ <b id="ReducerNone">Reducer NONE</b>
+ 
+ <p>It is legal to set the number of reduce-tasks to <code>zero</code>.</p>
+ 
+ <p>In this case the output of the map-tasks directly go to distributed 
+ file-system, to the path set by 
+ {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the 
+ framework doesn't sort the map-outputs before writing it out to HDFS.</p>
+ 
+ @param n the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapreduce.map.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapreduce.reduce.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.]]>
+      </doc>
+    </method>
+    <method name="getSessionId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified session identifier. The default is the empty string.
+
+ The session identifier is used to tag metric data that is reported to some
+ performance metrics system via the org.apache.hadoop.metrics API.  The 
+ session identifier is intended, in particular, for use by Hadoop-On-Demand 
+ (HOD) which allocates a virtual Hadoop cluster dynamically and transiently. 
+ HOD will set the session identifier by modifying the mapred-site.xml file 
+ before starting the cluster.
+
+ When not running under HOD, this identifer is expected to remain set to 
+ the empty string.
+
+ @return the session identifier, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setSessionId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sessionId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified session identifier.  
+
+ @param sessionId the new session id.]]>
+      </doc>
+    </method>
+    <method name="setMaxTaskFailuresPerTracker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="noFailures" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds <code>noFailures</code>, the 
+ tasktracker is <i>blacklisted</i> for this job. 
+ 
+ @param noFailures maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxTaskFailuresPerTracker" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Expert: Get the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds this, the tasktracker is
+ <i>blacklisted</i> for this job. 
+ 
+ @return the maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of map tasks that can fail without 
+ the job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts()} 
+ attempts before being declared as <i>failed</i>.
+  
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed map-task results in
+ the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of map tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the maximum percentage of map tasks that can fail without the
+ job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts} attempts 
+ before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of map tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of reduce tasks that can fail without 
+ the job being aborted. 
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed reduce-task results 
+ in the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of reduce tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum percentage of reduce tasks that can fail without the job
+ being aborted.
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of reduce tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="setJobPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="int"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link JobPriority} for this job.
+
+ @return the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriorityAsInteger" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this job.
+
+ @return the priority for this job.]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setMapDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the map tasks fail.
+ 
+ <p>The debug script can aid debugging of failed map tasks. The script is 
+ given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script needs to be symlinked. </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param mDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getMapDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the map task's debug script.
+ 
+ @return the debug Script for the mapred job for failed map tasks.
+ @see #setMapDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="setReduceDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the reduce tasks fail.
+ 
+ <p>The debug script can aid debugging of failed reduce tasks. The script
+ is given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script file needs to be symlinked </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param rDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getReduceDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reduce task's debug Script
+ 
+ @return the debug script for the mapred job for failed reduce tasks.
+ @see #setReduceDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationURI" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the uri to be invoked in-order to send a notification after the job 
+ has completed (success/failure). 
+ 
+ @return the job end notification uri, <code>null</code> if it hasn't
+         been set.
+ @see #setJobEndNotificationURI(String)]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationURI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the uri to be invoked in-order to send a notification after the job
+ has completed (success/failure).
+ 
+ <p>The uri can contain 2 special parameters: <tt>$jobId</tt> and 
+ <tt>$jobStatus</tt>. Those, if present, are replaced by the job's 
+ identifier and completion-status respectively.</p>
+ 
+ <p>This is typically used by application-writers to implement chaining of 
+ Map-Reduce jobs in an <i>asynchronous manner</i>.</p>
+ 
+ @param uri the job end notification uri
+ @see JobStatus]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationCustomNotifierClass" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class to be invoked in order to send a notification
+ after the job has completed (success/failure).
+
+ @return the fully-qualified name of the class which implements
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier} set through the
+ {@link org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS}
+ property
+
+ @see JobConf#setJobEndNotificationCustomNotifierClass(java.lang.String)
+ @see org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationCustomNotifierClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="customNotifierClassName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the class to be invoked in order to send a notification after the job
+ has completed (success/failure).
+
+ A notification url still has to be set which will be passed to
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier#notifyOnce(
+ java.net.URL, org.apache.hadoop.conf.Configuration)}
+ along with the Job's conf.
+
+ If this is set instead of using a simple HttpURLConnection
+ we'll create a new instance of this class
+ which should be an implementation of
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier},
+ and we'll invoke that.
+
+ @param customNotifierClassName the fully-qualified name of the class
+     which implements
+     {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier}
+
+ @see JobConf#setJobEndNotificationURI(java.lang.String)
+ @see
+ org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="getJobLocalDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get job-specific shared directory for use as scratch space
+ 
+ <p>
+ When a job starts, a shared directory is created at location
+ <code>
+ ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ </code>.
+ This directory is exposed to the users through 
+ <code>mapreduce.job.local.dir </code>.
+ So, the tasks can use this space 
+ as scratch space and share files among them. </p>
+ This value is available as System property also.
+ 
+ @return The localized job specific shared directory]]>
+      </doc>
+    </method>
+    <method name="getMemoryForMapTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a map task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_MAP_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a map task of the job, in MB,]]>
+      </doc>
+    </method>
+    <method name="setMemoryForMapTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getMemoryForReduceTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a reduce task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_REDUCE_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a reduce task of the job, in MB.]]>
+      </doc>
+    </method>
+    <method name="setMemoryForReduceTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the queue to which this job is submitted.
+ Defaults to 'default'.
+ 
+ @return name of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the queue to which this job should be submitted.
+ 
+ @param queueName Name of the queue]]>
+      </doc>
+    </method>
+    <method name="normalizeMemoryConfigValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[Normalize the negative values in configuration
+ 
+ @param val
+ @return normalized value]]>
+      </doc>
+    </method>
+    <method name="findContainingJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="my_class" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Find a jar that contains a class of the same name, if any.
+ It will return a jar file, even if that is not the first thing
+ on the class path that has a class with the same name.
+ 
+ @param my_class the class to find.
+ @return a jar file that contains the class, or null.]]>
+      </doc>
+    </method>
+    <method name="getMaxVirtualMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}">
+      <doc>
+      <![CDATA[Get the memory required to run a task of this job, in bytes. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ This method is deprecated. Now, different memory limits can be
+ set for map and reduce tasks of a job, in MB. 
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. 
+ Otherwise, this method will return the larger of the values returned by 
+ {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()}
+ after converting them into bytes.
+
+ @return Memory required to run a task of this job, in bytes.
+ @see #setMaxVirtualMemoryForTask(long)
+ @deprecated Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}]]>
+      </doc>
+    </method>
+    <method name="setMaxVirtualMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}">
+      <param name="vmem" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum amount of memory any task of this job can use. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ mapred.task.maxvmem is split into
+ mapreduce.map.memory.mb
+ and mapreduce.map.memory.mb,mapred
+ each of the new key are set
+ as mapred.task.maxvmem / 1024
+ as new values are in MB
+
+ @param vmem Maximum amount of virtual memory in bytes any task of this job
+             can use.
+ @see #getMaxVirtualMemoryForTask()
+ @deprecated
+  Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}]]>
+      </doc>
+    </method>
+    <method name="getMaxPhysicalMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="this variable is deprecated and nolonger in use.">
+      <doc>
+      <![CDATA[@deprecated this variable is deprecated and nolonger in use.]]>
+      </doc>
+    </method>
+    <method name="setMaxPhysicalMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPRED_TASK_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UPPER_LIMIT_ON_TASK_VMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_MAXPMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="DISABLED_MEMORY_LIMIT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A value which if set for memory related configuration options,
+ indicates that the options are turned off.
+ Deprecated because it makes no sense in the context of MR2.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_LOCAL_DIR_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Property name for the configuration property mapreduce.cluster.local.dir]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of the queue to which jobs will be submitted, if no queue
+ name is mentioned.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_MAP_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UNPACK_JAR_PATTERN_DEFAULT" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Pattern for the default unpacking behavior for job jars]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the child
+ map and reduce tasks.
+ 
+ Java opts for the task tracker child processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass 
+ other environment variables to the child processes.
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the map tasks.
+ 
+ Java opts for the task tracker child map processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass 
+ other environment variables to the map processes.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the reduce tasks.
+ 
+ Java opts for the task tracker child reduce processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to 
+ pass process environment variables to the reduce processes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPRED_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map/reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.map.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.reduce.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the map task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the reduce task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default logging level for map/reduce tasks.]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ID} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_STRING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_TAGS} instead]]>
+      </doc>
+    </field>
+    <field name="MAPREDUCE_RECOVER_JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPREDUCE_RECOVER_JOB" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A map/reduce job configuration.
+ 
+ <p><code>JobConf</code> is the primary interface for a user to describe a 
+ map-reduce job to the Hadoop framework for execution. The framework tries to
+ faithfully execute the job as-is described by <code>JobConf</code>, however:
+ <ol>
+   <li>
+   Some configuration parameters might have been marked as 
+   <a href="{@docRoot}/org/apache/hadoop/conf/Configuration.html#FinalParams">
+   final</a> by administrators and hence cannot be altered.
+   </li>
+   <li>
+   While some job parameters are straight-forward to set 
+   (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly
+   with the rest of the framework and/or job-configuration and is relatively
+   more complex for the user to control finely
+   (e.g. {@link #setNumMapTasks(int)}).
+   </li>
+ </ol>
+ 
+ <p><code>JobConf</code> typically specifies the {@link Mapper}, combiner 
+ (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and 
+ {@link OutputFormat} implementations to be used etc.
+
+ <p>Optionally <code>JobConf</code> is used to specify other advanced facets 
+ of the job such as <code>Comparator</code>s to be used, files to be put in  
+ the {@link DistributedCache}, whether or not intermediate and/or job outputs 
+ are to be compressed (and how), debugability via user-provided scripts 
+ ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}),
+ for doing post-processing on task logs, task's stdout, stderr, syslog. 
+ and etc.</p>
+ 
+ <p>Here is an example on how to configure a job via <code>JobConf</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+ </pre></blockquote>
+ 
+ @see JobClient
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobConf -->
+  <!-- start interface org.apache.hadoop.mapred.JobConfigurable -->
+  <interface name="JobConfigurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Initializes a new instance from a {@link JobConf}.
+
+ @param job the configuration]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[That what may be configured.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobConfigurable -->
+  <!-- start interface org.apache.hadoop.mapred.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job Configuration
+ 
+ @return JobConf]]>
+      </doc>
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress mechanism for reporting progress.
+ 
+ @return progress mechanism]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobContext -->
+  <!-- start class org.apache.hadoop.mapred.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapreduce.JobID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.JobID"/>
+      <doc>
+      <![CDATA[Downgrade a new JobID to an old one
+ @param old a new or old JobID
+ @return either old or a new JobID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getJobIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>any job</i> 
+ run on the jobtracker started at <i>200707121733</i>, we would use :
+ <pre> 
+ JobID.getTaskIDsPattern("200707121733", null);
+ </pre>
+ which will return :
+ <pre> "job_200707121733_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @return a regex pattern matching JobIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local".
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobID -->
+  <!-- start class org.apache.hadoop.mapred.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobPriority -->
+  <!-- start class org.apache.hadoop.mapred.JobQueueInfo -->
+  <class name="JobQueueInfo" extends="org.apache.hadoop.mapreduce.QueueInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobQueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Job Queue Info.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobQueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new JobQueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <method name="getQueueState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use getState() instead]]>
+      </doc>
+    </method>
+    <method name="getChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobQueueInfo -->
+  <!-- start class org.apache.hadoop.mapred.JobStatus -->
+  <class name="JobStatus" extends="org.apache.hadoop.mapreduce.JobStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on cleanup
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="getJobRunState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[Helper method to get human-readable state of the job.
+ @param state job state
+ @return human-readable state of the job]]>
+      </doc>
+    </method>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.mapreduce.JobStatus"/>
+    </method>
+    <method name="getJobId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use getJobID instead">
+      <doc>
+      <![CDATA[@deprecated use getJobID instead]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="getRunState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCEEDED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KILLED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Describes the current status of a job.  This is
+ not intended to be a comprehensive piece of data.
+ For that, look at JobProfile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobStatus -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line. Each line
+ is divided into key and value parts by a separator byte. If no such a byte
+ exists, the key will be the entire line and value will be empty.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapred.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.Mapper -->
+  <interface name="Mapper"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Maps a single input key/value pair into an intermediate key/value pair.
+ 
+ <p>Output pairs need not be of the same types as input pairs.  A given 
+ input pair may map to zero or many output pairs.  Output pairs are 
+ collected with calls to 
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes significant amount of time to process individual key/value
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the input key.
+ @param value the input value.
+ @param output collects mapped keys and values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link JobConf} for the 
+ job via the {@link JobConfigurable#configure(JobConf)} and initialize
+ themselves. Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+ 
+ <p>The framework then calls 
+ {@link #map(Object, Object, OutputCollector, Reporter)} 
+ for each key/value pair in the <code>InputSplit</code> for that task.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the grouping by specifying
+ a <code>Comparator</code> via 
+ {@link JobConf#setOutputKeyComparatorClass(Class)}.</p>
+
+ <p>The grouped <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>The intermediate, grouped outputs are always stored in 
+ {@link SequenceFile}s. Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>JobConf</code>.</p>
+  
+ <p>If the job has 
+ <a href="{@docRoot}/org/apache/hadoop/mapred/JobConf.html#ReducerNone">zero
+ reduces</a> then the output of the <code>Mapper</code> is directly written
+ to the {@link FileSystem} without grouping by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyMapper&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Mapper&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Process the &lt;key, value&gt; pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of &lt;key, value&gt; pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+ </pre></blockquote>
+
+ <p>Applications may write a custom {@link MapRunnable} to exert greater
+ control on map processing e.g. multi-threaded <code>Mapper</code>s etc.</p>
+ 
+ @see JobConf
+ @see InputFormat
+ @see Partitioner  
+ @see Reducer
+ @see MapReduceBase
+ @see MapRunnable
+ @see SequenceFile]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Mapper -->
+  <!-- start class org.apache.hadoop.mapred.MapReduceBase -->
+  <class name="MapReduceBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="MapReduceBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for {@link Mapper} and {@link Reducer} implementations.
+ 
+ <p>Provides default no-op implementations for a few methods, most non-trivial
+ applications need to override some of them.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapReduceBase -->
+  <!-- start interface org.apache.hadoop.mapred.MapRunnable -->
+  <interface name="MapRunnable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="run"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Start mapping input <tt>&lt;key, value&gt;</tt> pairs.
+  
+ <p>Mapping of input records to output records is complete when this method 
+ returns.</p>
+ 
+ @param input the {@link RecordReader} to read the input records.
+ @param output the {@link OutputCollector} to collect the outputrecords.
+ @param reporter {@link Reporter} to report progress, status-updates etc.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Expert: Generic interface for {@link Mapper}s.
+ 
+ <p>Custom implementations of <code>MapRunnable</code> can exert greater 
+ control on map processing e.g. multi-threaded, asynchronous mappers etc.</p>
+ 
+ @see Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.MapRunnable -->
+  <!-- start class org.apache.hadoop.mapred.MapRunner -->
+  <class name="MapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMapper" return="org.apache.hadoop.mapred.Mapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Default {@link MapRunnable} implementation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapRunner -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <class name="MultiFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link MultiFileSplit}'s
+ in {@link #getSplits(JobConf, int)} method. Splits are constructed from 
+ the files under the input paths. Each split returned contains <i>nearly</i>
+ equal content length. <br>  
+ Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>MultiFileSplit</code>'s.
+ @see MultiFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileSplit -->
+  <class name="MultiFileSplit" extends="org.apache.hadoop.mapred.lib.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. Unlike {@link FileSplit}, MultiFileSplit 
+ class does not represent a split of a file, but a split of input files 
+ into smaller sets. The atomic unit of split is a file. <br> 
+ MultiFileSplit can be used to implement {@link RecordReader}'s, with 
+ reading one record per file.
+ @see FileSplit
+ @see MultiFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileSplit -->
+  <!-- start interface org.apache.hadoop.mapred.OutputCollector -->
+  <interface name="OutputCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="collect"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a key/value pair to the output.
+
+ @param key the key to collect.
+ @param value to value to collect.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Collects the <code>&lt;key, value&gt;</code> pairs output by {@link Mapper}s
+ and {@link Reducer}s.
+  
+ <p><code>OutputCollector</code> is the generalization of the facility 
+ provided by the Map-Reduce framework to collect data output by either the 
+ <code>Mapper</code> or the <code>Reducer</code> i.e. intermediate outputs 
+ or the output of the job.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputCollector -->
+  <!-- start class org.apache.hadoop.mapred.OutputCommitter -->
+  <class name="OutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="status" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus#FAILED} or 
+ {@link JobStatus#KILLED}. This is called from the application
+ master process for the entire job. This may be called multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @param status final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task. This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.
+ 
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRConstants#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>. This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.
+ @deprecated Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="runState" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapred.OutputFormat -->
+  <interface name="OutputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given job.
+
+ @param ignored
+ @param job configuration for the job whose output is being written.
+ @param name the unique name for this part of the output.
+ @param progress mechanism for reporting progress while writing to file.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ Implementations which write to filesystems which support delegation
+ tokens usually collect the tokens for the destination path(s)
+ and attach them to the job configuration.
+ @param ignored
+ @param job job configuration.
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter
+ @see JobConf]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.OutputLogFilter -->
+  <class name="OutputLogFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="OutputLogFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[This class filters log files from directory given
+ It doesnt accept paths having _logs.
+ This can be used to list paths of output directory as follows:
+   Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
+                                   new OutputLogFilter()));]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputLogFilter -->
+  <!-- start interface org.apache.hadoop.mapred.Partitioner -->
+  <interface name="Partitioner"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the paritition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be paritioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Partitioner -->
+  <!-- start interface org.apache.hadoop.mapred.RecordReader -->
+  <interface name="RecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads the next key/value pair from the input for processing.
+
+ @param key the key to read data into
+ @param value the value to read data into
+ @return true iff a key/value was read, false if at EOF]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a key.
+ 
+ @return a new key object.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a value.
+ 
+ @return a new value object.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current position in the input.
+ 
+ @return the current position in the input.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this {@link InputSplit} to future operations.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[How much of the input has the {@link RecordReader} consumed i.e.
+ has been processed by?
+ 
+ @return progress from <code>0.0</code> to <code>1.0</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordReader</code> reads &lt;key, value&gt; pairs from an 
+ {@link InputSplit}.
+   
+ <p><code>RecordReader</code>, typically, converts the byte-oriented view of 
+ the input, provided by the <code>InputSplit</code>, and presents a 
+ record-oriented view for the {@link Mapper} and {@link Reducer} tasks for
+ processing. It thus assumes the responsibility of processing record 
+ boundaries and presenting the tasks with keys and values.</p>
+ 
+ @see InputSplit
+ @see InputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordReader -->
+  <!-- start interface org.apache.hadoop.mapred.RecordWriter -->
+  <interface name="RecordWriter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param reporter facility to report progress.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapred.Reducer -->
+  <interface name="Reducer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="reduce"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<i>Reduces</i> values for a given key.  
+ 
+ <p>The framework calls this method for each 
+ <code>&lt;key, (list of values)&gt;</code> pair in the grouped inputs.
+ Output values must be of the same type as input values.  Input keys must 
+ not be altered. The framework will <b>reuse</b> the key and value objects
+ that are passed into the reduce, therefore the application should clone
+ the objects they want to keep a copy of. In many cases, all values are 
+ combined into zero or one value.
+ </p>
+   
+ <p>Output pairs are collected with calls to  
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes a significant amount of time to process individual key/value 
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the key.
+ @param values the list of values to reduce.
+ @param output to collect keys and combined values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p>The number of <code>Reducer</code>s for the job is set by the user via 
+ {@link JobConf#setNumReduceTasks(int)}. <code>Reducer</code> implementations 
+ can access the {@link JobConf} for the job via the 
+ {@link JobConfigurable#configure(JobConf)} method and initialize themselves. 
+ Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p><code>Reducer</code> is input the grouped output of a {@link Mapper}.
+   In the phase the framework, for each <code>Reducer</code>, fetches the 
+   relevant partition of the output of all the <code>Mapper</code>s, via HTTP. 
+   </p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework groups <code>Reducer</code> inputs by <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key) in this
+   stage.</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>If equivalence rules for keys while grouping the intermediates are 
+   different from those for grouping keys before reduction, then one may 
+   specify a <code>Comparator</code> via 
+   {@link JobConf#setOutputValueGroupingComparator(Class)}.Since 
+   {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to 
+   control how intermediate keys are grouped, these can be used in conjunction 
+   to simulate <i>secondary sort on values</i>.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterator, OutputCollector, Reporter)}
+   method is called for each <code>&lt;key, (list of values)&gt;</code> pair in
+   the grouped inputs.</p>
+   <p>The output of the reduce task is typically written to the 
+   {@link FileSystem} via 
+   {@link OutputCollector#collect(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyReducer&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Reducer&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator&lt;V&gt; values,
+                          OutputCollector&lt;K, V&gt; output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the &lt;key, value&gt; pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the &lt;key, value&gt; 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of &lt;key, list of values&gt; pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner
+ @see Reporter
+ @see MapReduceBase]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reducer -->
+  <!-- start interface org.apache.hadoop.mapred.Reporter -->
+  <interface name="Reporter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the status description for the task.
+ 
+ @param status brief description of the current status.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param group counter group
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the key, which can be of
+ any {@link Enum} type, by the specified amount.
+ 
+ @param key key to identify the counter to be incremented. The key can be
+            be any <code>Enum</code>. 
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the group and counter name
+ by the specified amount.
+ 
+ @param group name to identify the group of the counter to be incremented.
+ @param counter name to identify the counter within the group.
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="getInputSplit" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Get the {@link InputSplit} object for a map.
+ 
+ @return the <code>InputSplit</code> that the map is reading from.
+ @throws UnsupportedOperationException if called outside a mapper]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress of the task. Progress is represented as a number between
+ 0 and 1 (inclusive).]]>
+      </doc>
+    </method>
+    <field name="NULL" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A constant of Reporter type that does nothing.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A facility for Map-Reduce applications to report progress and update 
+ counters, status information etc.
+ 
+ <p>{@link Mapper} and {@link Reducer} can use the <code>Reporter</code>
+ provided to report progress or just indicate that they are alive. In 
+ scenarios where the application takes significant amount of time to
+ process individual key/value pairs, this is crucial since the framework 
+ might assume that the task has timed-out and kill that task.
+
+ <p>Applications can also update {@link Counters} via the provided 
+ <code>Reporter</code> .</p>
+ 
+ @see Progressable
+ @see Counters]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reporter -->
+  <!-- start interface org.apache.hadoop.mapred.RunningJob -->
+  <interface name="RunningJob"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying job configuration
+
+ @return the configuration of the job.]]>
+      </doc>
+    </method>
+    <method name="getID" return="org.apache.hadoop.mapred.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job identifier.
+ 
+ @return the job identifier.]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.">
+      <doc>
+      <![CDATA[@deprecated This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the job.
+ 
+ @return the name of the job.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Blocks until the job is complete.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ {@link JobStatus}
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobStatus" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a snapshot of the current status, {@link JobStatus}, of the Job.
+ Need to call again for latest information.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job. Blocks until all job tasks have been killed as well.
+ If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param priority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @param shouldFail if true the task is failed and added to failed tasks 
+                   list, otherwise it is just killed, w/o affecting 
+                   job failure status.  
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #killTask(TaskAttemptID, boolean)}">
+      <param name="taskId" type="java.lang.String"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job.
+ 
+ @return the counters for this job or null if the job has been retired.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the url where history file is archived. Returns empty string if 
+ history file is not available yet. 
+ 
+ @return the url where history file is archived
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether the job has been removed from JobTracker memory and retired.
+ On retire, the job history file is copied to a location known by 
+ {@link #getHistoryUrl()}
+ @return <code>true</code> if the job retired, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get failure info for the job.
+ @return the failure info for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RunningJob</code> is the user-interface to query for details on a 
+ running Map-Reduce job.
+ 
+ <p>Clients can get hold of <code>RunningJob</code> via the {@link JobClient}
+ and then query the running-job for details such as name, configuration, 
+ progress etc.</p> 
+ 
+ @see JobClient]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RunningJob -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapred.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes keys, values to 
+ {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, 
+ except it generates SequenceFileAsTextRecordReader 
+ which converts the input keys and values to their 
+ String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileAsTextRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by calling toString()
+ method. This class to SequenceFileAsTextInputFormat class is as LineRecordReader
+ class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param job job configuration
+ @param reporter reporter who sends report to task tracker
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param conf application configuration
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.SequenceFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of key that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of value that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCurrentValue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SkipBadRecords -->
+  <class name="SkipBadRecords" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SkipBadRecords"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAttemptsToStartSkipping" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @return attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="setAttemptsToStartSkipping"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="attemptsToStartSkipping" type="int"/>
+      <doc>
+      <![CDATA[Set the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @param attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrMapperProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                       {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrMapperProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrReducerProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                    {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrReducerProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.]]>
+      </doc>
+    </method>
+    <method name="getSkipOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @return path skip output directory. Null is returned if this is not set 
+ and output directory is also not set.]]>
+      </doc>
+    </method>
+    <method name="setSkipOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @param path skip output directory path]]>
+      </doc>
+    </method>
+    <method name="getMapperMaxSkipRecords" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="setMapperMaxSkipRecords"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipRecs" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="getReducerMaxSkipGroups" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <method name="setReducerMaxSkipGroups"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipGrps" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <field name="COUNTER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Special counters which are written by the application and are 
+ used by the framework for detecting bad records. For detecting bad records 
+ these counters must be incremented by the application.]]>
+      </doc>
+    </field>
+    <field name="COUNTER_MAP_PROCESSED_RECORDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed map records.
+ @see SkipBadRecords#getAutoIncrMapperProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <field name="COUNTER_REDUCE_PROCESSED_GROUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed reduce groups.
+ @see SkipBadRecords#getAutoIncrReducerProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Utility class for skip bad records functionality. It contains various 
+ settings related to skipping of bad records.
+ 
+ <p>Hadoop provides an optional mode of execution in which the bad records
+ are detected and skipped in further attempts.
+ 
+ <p>This feature can be used when map/reduce tasks crashes deterministically on 
+ certain input. This happens due to bugs in the map/reduce function. The usual
+ course would be to fix these bugs. But sometimes this is not possible; 
+ perhaps the bug is in third party libraries for which the source code is 
+ not available. Due to this, the task never reaches to completion even with 
+ multiple attempts and complete data for that task is lost.</p>
+  
+ <p>With this feature, only a small portion of data is lost surrounding 
+ the bad record, which may be acceptable for some user applications.
+ see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}</p>
+ 
+ <p>The skipping mode gets kicked off after certain no of failures 
+ see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}</p>
+  
+ <p>In the skipping mode, the map/reduce task maintains the record range which 
+ is getting processed at all times. Before giving the input to the
+ map/reduce function, it sends this record range to the Task tracker.
+ If task crashes, the Task tracker knows which one was the last reported
+ range. On further attempts that range get skipped.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SkipBadRecords -->
+  <!-- start class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <class name="SplitLocationInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitLocationInfo" type="java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isOnDisk" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInMemory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <!-- start interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapred.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapreduce.TaskAttemptID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapred.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param taskId taskId number
+ @param id the task attempt number
+ @deprecated Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskAttemptID to an old one
+ @param old the new id
+ @return either old or a new TaskAttemptID constructed to match old]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType} 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="org.apache.hadoop.mapreduce.TaskCompletionEvent"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapred.TaskAttemptID, int, boolean, org.apache.hadoop.mapred.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getTaskAttemptId()} instead.">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id
+ @deprecated use {@link #getTaskAttemptId()} instead.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getTaskStatus" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="setTaskId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id.
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapred.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapreduce.TaskID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(String, int, TaskType, int)}">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(String, int, TaskType, int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskID to an old one
+ @param old a new or old TaskID
+ @return either old or a new TaskID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs
+ @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}]]>
+      </doc>
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType}, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs]]>
+      </doc>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+    </method>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskID -->
+  <!-- start class org.apache.hadoop.mapred.TaskReport -->
+  <class name="TaskReport" extends="org.apache.hadoop.mapreduce.TaskReport"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The string of the task id.]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The id of the task.]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setSuccessfulAttempt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[set successful attempt ID of the task.]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulTaskAttempt" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attempt ID that took this task to completion]]>
+      </doc>
+    </method>
+    <method name="setRunningTaskAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runningAttempts" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[set running attempt(s) of the task.]]>
+      </doc>
+    </method>
+    <method name="getRunningTaskAttempts" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the running task attempt IDs for this task]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[set finish time of task. 
+ @param finishTime finish time of task.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[set start time of the task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A report on the state of a task.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskReport -->
+  <!-- start class org.apache.hadoop.mapred.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Utils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A utility class. It provides
+   A path filter utility to filter out output/part files in the output dir]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Utils -->
+</package>
+<package name="org.apache.hadoop.mapred.jobcontrol">
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf, java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param jobConf a mapred job configuration representing a job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getAssignedJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setAssignedJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="setAssignedJobID should not be called.
+ JOBID is set by the framework.">
+      <param name="mapredJobID" type="org.apache.hadoop.mapred.JobID"/>
+      <doc>
+      <![CDATA[@deprecated setAssignedJobID should not be called.
+ JOBID is set by the framework.]]>
+      </doc>
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred job conf of this job]]>
+      </doc>
+    </method>
+    <method name="setJobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Set the mapred job conf for this job.
+ @param jobConf the mapred job conf for this job.]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[This is a no-op function, Its a behavior change from 1.x We no more can
+ change the state from job
+ 
+ @param state
+          the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="getJobClient" return="org.apache.hadoop.mapred.JobClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job client of this job]]>
+      </doc>
+    </method>
+    <method name="getDependingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="getMapredJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setMapredJobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapredJobID" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is no-op method for backward compatibility. It's a behavior change
+ from 1.x, we can not change job ids from job.
+ 
+ @param mapredJobID
+          the mapred job ID for this job.]]>
+      </doc>
+    </method>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="WAITING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="READY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEPENDENT_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+  <class name="JobControl" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJobs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapred.join">
+  <!-- start class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <interface name="ComposableInputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <interface name="ComposableRecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <implements name="java.lang.Comparable"/>
+    <method name="id" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key this RecordReader would supply on a call to next(K,V)]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RecordReader into the object provided.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the stream is not empty, but provides no guarantee that
+ a call to next(K,V) will succeed.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[While key-value pairs from this RecordReader match the given key, register
+ them with the JoinCollector provided.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapred.join.expr</tt> property and
+ user-supplied join types from <tt>mapred.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapred.join.define.&lt;ident&gt;</tt> to a classname. In the expression
+ <tt>mapred.join.expr</tt>, the identifier will be assumed to be a
+ ComposableRecordReader.
+ <tt>mapred.join.keycomparator</tt> can be a classname used to compare keys
+ in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapred.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for faliing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key value common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createInternalValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unsupported (returns zero in all cases).]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapred.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapred.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Node -->
+  <class name="Parser.Node" extends="java.lang.Object"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapred.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.TupleWritable -->
+  <class name="TupleWritable" extends="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="U"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write key-value pair at the head of this stream to the objects provided;
+ get next key-value pair from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new value from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request position from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapred.lib">
+  <!-- start class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}. 
+ 
+ @see org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <class name="ChainMapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ <p>
+
+ @param job              job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="java.lang.Object"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>map(...)</code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ <p>
+ ChainMapper usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <class name="ChainReducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="reducerConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Sets the Reducer class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Reducer leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Reducer does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+
+ @param job              job's JobConf to add the Reducer class.
+ @param klass            the Reducer class to add.
+ @param inputKeyClass    reducer input key class.
+ @param inputValueClass  reducer input value class.
+ @param outputKeyClass   reducer output key class.
+ @param outputValueClass reducer output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param reducerConf      a JobConf with the configuration for the Reducer
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ .
+
+ @param job              chain job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>reduce(...)</code> method of the Reducer with the
+ <code>map(...) </code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion, the output of the first becomes the input of the
+ second, and so on until the last Mapper, the output of the last Mapper will
+ be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain.
+ This enables having reusable specialized Mappers that can be combined to
+ perform composite operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ <p>
+ ChainReducer usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(List)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(List)}.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(PathFilter...)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(PathFilter...)}.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link org.apache.hadoop.mapred.InputFormat} that returns {@link CombineFileSplit}'s
+ in {@link org.apache.hadoop.mapred.InputFormat#getSplits(JobConf, int)} method. 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default spliting behaviour in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement {@link org.apache.hadoop.mapred.InputFormat#getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>CombineFileSplit</code>'s.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.mapred.Reporter, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return the amount of data processed]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapred.JobConf"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="reporter" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapred.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapred.FileInputFormat, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.Reporter, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <class name="FieldSelectionMapReduce" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="FieldSelectionMapReduce"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <!-- start class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapred.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the supplied output format.
+ @param out the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapred.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <class name="HashPartitioner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <class name="IdentityMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="IdentityMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identity function.  Input key/value pair is written directly to
+ output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the identity function, mapping inputs directly to outputs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <class name="IdentityReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="IdentityReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all keys and values directly to output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Performs no reduction, writing all input values directly to the output.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.mapreduce.lib.partition.InputSampler"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSampler" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="sampler" type="org.apache.hadoop.mapred.lib.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InputSampler -->
+  <!-- start class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapred.lib.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link JobConf} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} that sums long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <class name="MultipleOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a composite record writer that can write key/value data to different
+ output files
+ 
+ @param fs
+          the file system to use
+ @param job
+          the job conf for the job
+ @param name
+          the leaf file name for the output file (such as part-00000")
+ @param arg3
+          a progressable for reporting progress.
+ @return a composite record writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateLeafFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the leaf name for the output file name. The default behavior does
+ not change the leaf file name (such as part-00000)
+ 
+ @param name
+          the leaf file name for the output file
+ @return the given leaf file name]]>
+      </doc>
+    </method>
+    <method name="generateFileNameForKeyValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the file output file name based on the given key and the leaf file
+ name. The default behavior is that the file name does not depend on the
+ key.
+ 
+ @param key
+          the key of the output data
+ @param name
+          the leaf file name
+ @return generated file name]]>
+      </doc>
+    </method>
+    <method name="generateActualKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual key from the given key/value. The default behavior is that
+ the actual key is equal to the given key
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual key derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="generateActualValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual value from the given key and value. The default behavior is that
+ the actual value is equal to the given value
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual value derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="getInputFileBasedOutputFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the outfile name based on a given name and the input file name. If
+ the {@link JobContext#MAP_INPUT_FILE} does not exists (i.e. this is not for a map only job),
+ the given name is returned unchanged. If the config value for
+ "num.of.trailing.legs.to.use" is not set, or set 0 or negative, the given
+ name is returned unchanged. Otherwise, return a file name consisting of the
+ N trailing legs of the input file name where N is the config value for
+ "num.of.trailing.legs.to.use".
+ 
+ @param job
+          the job config
+ @param name
+          the output file name
+ @return the outfile name based on a given name and the input file name.]]>
+      </doc>
+    </method>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param fs
+          the file system to use
+ @param job
+          a job conf object
+ @param name
+          the name of the file over which a record writer object will be
+          constructed
+ @param arg3
+          a progressable object
+ @return A RecordWriter object over the given file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This abstract class extends the FileOutputFormat, allowing to write the
+ output data to different output files. There are three basic use cases for
+ this class.
+ 
+ Case one: This class is used for a map reduce job with at least one reducer.
+ The reducer wants to write data to different files depending on the actual
+ keys. It is assumed that a key (or value) encodes the actual key (value)
+ and the desired location for the actual key (value).
+ 
+ Case two: This class is used for a map only job. The job wants to use an
+ output file name that is either a part of the input file name of the input
+ data, or some derivation of it.
+ 
+ Case three: This class is used for a map only job. The job wants to use an
+ output file name that depends on both the keys and the input file name,]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple named outputs support, it should be
+ instantiated in the Mapper/Reducer configure method.
+
+ @param job the job configuration object]]>
+      </doc>
+    </constructor>
+    <method name="getNamedOutputsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns list of channel names.
+
+ @param conf job conf
+ @return List of channel Names]]>
+      </doc>
+    </method>
+    <method name="isMultiNamedOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns if a named output is multiple.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return <code>true</code> if the name output is multi, <code>false</code>
+         if it is single. If the name output is not defined it returns
+         <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputFormatClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named output OutputFormat.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return namedOutput OutputFormat]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the key class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class for the named output key]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the value class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class of named output value]]>
+      </doc>
+    </method>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="addMultiNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a multi named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+ @param conf    job conf to enableadd the named output.
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+
+ @param conf    job conf to enableadd the named output.
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputs" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns iterator with the defined name outputs.
+
+ @return iterator with the defined named outputs]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a named output.
+
+ @param namedOutput the named output name
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="multiName" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a multi named output.
+
+ @param namedOutput the named output name
+ @param multiName   the multi name part
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes all the opened named outputs.
+ <p>
+ If overriden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>
+
+ @throws java.io.IOException thrown if any of the MultipleOutput files
+                             could not be closed properly.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing to additional outputs other
+ than the job default output via the <code>OutputCollector</code> passed to
+ the <code>map()</code> and <code>reduce()</code> methods of the
+ <code>Mapper</code> and <code>Reducer</code> implementations.
+ <p>
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ <p>
+ A named output can be a single file or a multi file. The later is referred as
+ a multi named output.
+ <p>
+ A multi named output is an unbound set of files all sharing the same
+ <code>OutputFormat</code>, key class and value class configuration.
+ <p>
+ When named outputs are used within a <code>Mapper</code> implementation,
+ key/values written to a name output are not part of the reduce phase, only
+ key/values written to the job <code>OutputCollector</code> are part of the
+ reduce phase.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled. The counters
+ group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+ </pre>
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ public class MOReduce implements
+   Reducer&lt;WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <class name="MultipleSequenceFileOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleSequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output data 
+ to different output files in sequence file output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <class name="MultipleTextOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleTextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output
+ data to different output files in Text output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <class name="MultithreadedMapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MultithreadedMapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Multithreaded implementation for {@link MapRunnable}.
+ <p>
+ It can be used instead of the default implementation,
+ of {@link org.apache.hadoop.mapred.MapRunner}, when the Map
+ operation is not CPU bound in order to improve throughput.
+ <p>
+ Map implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured to use this MapRunnable class (using
+ the JobConf.setMapRunnerClass method) and
+ the number of threads the thread-pool can use with the
+ <code>mapred.map.multithreadedrunner.threads</code> property, its default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <!-- start class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see org.apache.hadoop.mapred.FileInputFormat#getSplits(JobConf, int)]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <class name="TokenCountMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="TokenCountMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that maps text values into &lt;token,freq&gt; pairs.  Uses
+ {@link StringTokenizer} to break text into tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.
+ @deprecated Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #getPartitionFile(Configuration)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(JobConf,Path)
+ @deprecated Use 
+ {@link #getPartitionFile(Configuration)}
+ instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param job a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value aggregator 
+ descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class that may be 
+ dynamically loaded. The other is to delegate invocations of 
+ generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param job a job configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Combiner does not need to configure.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param output to collect combined values]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param job
+          a JobConf object that may contain the information that can be used
+          to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a JobConf object. Its main
+ function is to generate a list of aggregation-id/value pairs. An aggregation
+ id encodes an aggregation type which is used to guide the way to aggregate
+ the value in the reduce/combiner phrase of an Aggregate based job.The mapper in
+ an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @param caller the the caller class.
+ @return a JobConf object ready for submission.
+
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a JobConf object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specilizing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner classes,
+ and a set of built-in value aggregators, and a generic utility class that
+ helps user create map/reduce jobs using the generic class. The built-in
+ aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(JobConfjob); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="org.apache.hadoop.io.Text"/>
+      <param name="arg1" type="java.util.Iterator"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param key
+          the key is expected to be a Text object, whose prefix indicates
+          the type of aggregation to aggregate the values. In effect, data
+          driven computing is achieved. It is assumed that each aggregator's
+          getReport method emits appropriate output for the aggregator. This
+          may be further customiized.
+ @param values
+          the values to be aggregated]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.db">
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="chunks" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, eg. '(updated &gt;
+ 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(JobConf, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns the number of records in
+ the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(JobConf, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBWritable"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+</package>
+<package name="org.apache.hadoop.mapred.pipes">
+  <!-- start class org.apache.hadoop.mapred.pipes.Submitter -->
+  <class name="Submitter" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="Submitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Submitter" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExecutable" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the URI of the application's executable.
+ @param conf
+ @return the URI where the application's executable is located]]>
+      </doc>
+    </method>
+    <method name="setExecutable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="executable" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the URI for the application's executable. Normally this is a hdfs: 
+ location.
+ @param conf
+ @param executable The URI of the application's executable.]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job is using a Java RecordReader.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java RecordReader
+ @param conf the configuration to check
+ @return is it a Java RecordReader?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Mapper is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Mapper.
+ @param conf the configuration to check
+ @return is it a Java Mapper?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Reducer is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Reducer.
+ @param conf the configuration to check
+ @return is it a Java Reducer?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job will use a Java RecordWriter.
+ @param conf the configuration to modify
+ @param value the new value to set]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordWriter" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Will the reduce use a Java RecordWriter?
+ @param conf the configuration to check
+ @return true, if the output of the job will be written by Java]]>
+      </doc>
+    </method>
+    <method name="getKeepCommandFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Does the user want to keep the command file for debugging? If this is
+ true, pipes will write a copy of the command data to a file in the
+ task directory named "downlink.data", which may be used to run the C++
+ program under the debugger. You probably also want to set 
+ JobConf.setKeepFailedTaskFiles(true) to keep the entire directory from
+ being deleted.
+ To run using the data file, set the environment variable 
+ "mapreduce.pipes.commandfile" to point to the file.
+ @param conf the configuration to check
+ @return will the framework save the command file?]]>
+      </doc>
+    </method>
+    <method name="setKeepCommandFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to keep the command file for debugging
+ @param conf the configuration to modify
+ @param keep the new value]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link Submitter#runJob(JobConf)}">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException
+ @deprecated Use {@link Submitter#runJob(JobConf)}]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobSubmit" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the Map-Reduce framework.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Submit a pipes job based on the command line arguments.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PRESERVE_COMMANDFILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXECUTABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INTERPRETOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RW" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_REDUCE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The main entry point and job submitter. It may either be used as a command
+ line-based or API-based method to launch Pipes jobs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.pipes.Submitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce">
+  <!-- start class org.apache.hadoop.mapreduce.Cluster -->
+  <class name="Cluster" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Cluster" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="Cluster" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Cluster</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the file system where job-specific files are stored
+ 
+ @return object of FileSystem
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job corresponding to jobid.
+ 
+ @param jobId
+ @return object of {@link Job}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the queues in cluster.
+ 
+ @return array of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get queue information for the specified name.
+ 
+ @param name queuename
+ @return object of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLogParams" return="org.apache.hadoop.mapreduce.v2.LogParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+      <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get log parameters for the specified jobID or taskAttemptID
+ @param jobID the job id.
+ @param taskAttemptID the task attempt id. Optional.
+ @return the LogParams
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current cluster status.
+ 
+ @return object of {@link ClusterMetrics}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getActiveTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all active trackers in the cluster.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get blacklisted trackers.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapreduce.Job[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getAllJobStatuses()} instead.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the jobs in cluster.
+ 
+ @return array of {@link Job}
+ @throws IOException
+ @throws InterruptedException
+ @deprecated Use {@link #getAllJobStatuses()} instead.]]>
+      </doc>
+    </method>
+    <method name="getAllJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job status for all jobs in the cluster.
+ @return job status for all jobs in cluster
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where 
+ job-specific files will  be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker's view of the staging directory path where 
+ job-specific files will  be placed.
+ 
+ @return the staging directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getJobHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the job history file path for a given job id. The job history file at 
+ this path may or may not be existing depending on the job completion state.
+ The file is present only for the completed jobs.
+ @param jobId the JobID of the job submitted by the current user.
+ @return the file path of the job history file
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the root level queues.
+ @return array of JobQueueInfo object.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns immediate children of queueName.
+ @param queueName
+ @return array of JobQueueInfo which are children of queueName
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return the new expiration time
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides a way to access information about the map/reduce cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Cluster -->
+  <!-- start class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <class name="ClusterMetrics" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRunningMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running map tasks in the cluster.
+ 
+ @return running maps]]>
+      </doc>
+    </method>
+    <method name="getRunningReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running reduce tasks in the cluster.
+ 
+ @return running reduces]]>
+      </doc>
+    </method>
+    <method name="getOccupiedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of occupied map slots in the cluster.
+ 
+ @return occupied map slot count]]>
+      </doc>
+    </method>
+    <method name="getOccupiedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of occupied reduce slots in the cluster.
+ 
+ @return occupied reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of reserved map slots in the cluster.
+ 
+ @return reserved map slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved reduce slots in the cluster.
+ 
+ @return reserved reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getMapSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of map slots in the cluster.
+ 
+ @return map slot capacity]]>
+      </doc>
+    </method>
+    <method name="getReduceSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of reduce slots in the cluster.
+ 
+ @return reduce slot capacity]]>
+      </doc>
+    </method>
+    <method name="getTotalJobSubmissions" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of job submissions in the cluster.
+ 
+ @return total number of job submissions]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active trackers in the cluster.
+ 
+ @return active tracker count.]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted trackers in the cluster.
+ 
+ @return blacklisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getGrayListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted trackers in the cluster.
+ 
+ @return graylisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getDecommissionedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of decommissioned trackers in the cluster.
+ 
+ @return decommissioned tracker count]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterMetrics</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster.  
+   </li>
+   <li>
+   Number of blacklisted and decommissioned trackers.  
+   </li>
+   <li>
+   Slot capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently occupied/reserved map and reduce slots.
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   The number of job submissions.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterMetrics</code>, via 
+ {@link Cluster#getClusterStatus()}.</p>
+ 
+ @see Cluster]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <!-- start interface org.apache.hadoop.mapreduce.Counter -->
+  <interface name="Counter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="(and no-op by default)">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the counter
+ @param displayName of the counter
+ @deprecated (and no-op by default)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the name of the counter]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the counter.
+ @return the user facing name of the counter]]>
+      </doc>
+    </method>
+    <method name="getValue" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[What is the current value of this counter?
+ @return the current value]]>
+      </doc>
+    </method>
+    <method name="setValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set this counter by the given value
+ @param value the value to set]]>
+      </doc>
+    </method>
+    <method name="increment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+      <doc>
+      <![CDATA[Increment this counter by the given value
+ @param incr the value to increase this counter by]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A named counter that tracks the progress of a map/reduce job.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> is named by
+ an {@link Enum} and has a long for the value.</p>
+
+ <p><code>Counters</code> are bunched into Groups, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.Counter -->
+  <!-- start interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <interface name="CounterGroup"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <doc>
+    <![CDATA[A group of {@link Counter}s that logically belong together. Typically,
+ it is an {@link Enum} subclass and the counters are the values.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <!-- start class org.apache.hadoop.mapreduce.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the Counters object from the another counters object
+ @param <C> the type of counter
+ @param <G> the type of counter group
+ @param counters the old counters object]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[<p><code>Counters</code> holds per job/task counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
+ comprising of counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Counters -->
+  <!-- start class org.apache.hadoop.mapreduce.ID -->
+  <class name="ID" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[returns the int which represents the identifier]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare IDs by associated numbers]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SEPARATOR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ID -->
+  <!-- start class org.apache.hadoop.mapreduce.InputFormat -->
+  <class name="InputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple. The InputFormat
+ also creates the {@link RecordReader} to read the {@link InputSplit}.
+ 
+ @param context job configuration.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Create a record reader for a given split. The framework will call
+ {@link RecordReader#initialize(InputSplit, TaskAttemptContext)} before
+ the split is used.
+ @param split the split to be read
+ @param context the information about the task
+ @return a new record reader
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibility to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see FileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.InputSplit -->
+  <class name="InputSplit" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the size of the split, so that the input splits can be sorted by size.
+ @return the number of bytes in the split
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the list of nodes by name where the data for the split would be local.
+ The locations do not need to be serialized.
+ 
+ @return a new array of the node nodes.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.task.JobContextImpl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="Job"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, String)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration, String)}]]>
+      </doc>
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} .
+ A Cluster will be created with a generic {@link Configuration}.
+ 
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a 
+ given {@link Configuration}.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a given jobName.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.JobStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration} and {@link JobStatus}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param status job status
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster}.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ 
+ @return JobStatus#State
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get start time of the job.
+ 
+ @return the start time of the job]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get finish time of the job.
+ 
+ @return the finish time of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the scheduling info of the job]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the priority info of the job]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Dump stats to screen.]]>
+      </doc>
+    </method>
+    <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the tasks of a job.
+ 
+ @param type Type of the task
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job.  Blocks until all job tasks have been
+ killed as well.  If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param jobPriority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+
+ @param jobPriority
+          the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <param name="numEvents" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @param numEvents number of events to fetch
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link org.apache.hadoop.mapred.TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="failTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fail indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapreduce.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job. May return null if the job has been
+ retired and the job is no longer in the completed job store.
+ 
+ @return the counters for this job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tasks" type="int"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the number of reduce tasks for the job.
+ @param tasks the number of reduce tasks
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setInputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} for the job.
+ @param cls the <code>InputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} for the job.
+ @param cls the <code>OutputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} for the job.
+ @param cls the <code>Mapper</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the Jar by finding where a given class came from.
+ @param cls the example class]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job jar]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the combiner class for the job.
+ @param cls the combiner to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} for the job.
+ @param cls the <code>Reducer</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} for the job.
+ @param cls the <code>Partitioner</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to combiner,
+ {@link Reducer#reduce(Object, Iterable,
+ org.apache.hadoop.mapreduce.Reducer.Context)}
+
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSortComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls how the keys are sorted before they
+ are passed to the {@link Reducer}.
+ @param cls the raw comparator
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to 
+ {@link Reducer#reduce(Object, Iterable, 
+                       org.apache.hadoop.mapreduce.Reducer.Context)}
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setJobSetupCleanupNeeded"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="needed" type="boolean"/>
+      <doc>
+      <![CDATA[Specify whether job-setup and job-cleanup is needed for the job 
+ 
+ @param needed If <code>true</code>, job-setup and job-cleanup will be
+               considered from {@link OutputCommitter} 
+               else ignored.]]>
+      </doc>
+    </method>
+    <method name="setCacheArchives"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archives" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of archives
+ @param archives The list of archives that need to be localized]]>
+      </doc>
+    </method>
+    <method name="setCacheFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of files
+ @param files The list of files that need to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheArchive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a archives to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a file to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addFileToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an file path to the current set of classpath entries It adds the file
+ to cache as well.
+ 
+ Files added with this method will not be unpacked while being added to the
+ classpath.
+ To add archives to classpath, use the {@link #addArchiveToClassPath(Path)}
+ method instead.
+
+ @param file Path of the file to be added]]>
+      </doc>
+    </method>
+    <method name="addArchiveToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archive" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an archive path to the current set of classpath entries. It adds the
+ archive to cache as well.
+ 
+ Archive files will be unpacked and added to the classpath
+ when being distributed.
+
+ @param archive Path of the archive to be added]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to enable symlinks, but currently symlinks cannot be
+ disabled.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setCancelDelegationTokenUponJobCompletion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Sets the flag that will allow the JobTracker to cancel the HDFS delegation
+ tokens upon job completion. Defaults to true.]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCacheAndClasspath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Job Submitter will also
+ add the file to classpath. Intended to be used by user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addArchiveToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add an archive to job config for shared cache processing. If shared cache
+ is enabled, it will return true, otherwise, return false. We don't check
+ with SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Intended to be used by
+ user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="setFileSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for files. If the parameter
+ was previously set, this method will replace the old value with the new
+ provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="setArchiveSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for archives. If the
+ parameter was previously set, this method will replace the old value with
+ the new provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getFileSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for files.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getArchiveSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for archives.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and return immediately.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verbose" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and wait for it to finish.
+ @param verbose print the progress to the user
+ @return true if the job succeeded
+ @throws IOException thrown if the communication with the 
+         <code>JobTracker</code> is lost]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="getProgressPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which monitorAndPrintJob() prints status]]>
+      </doc>
+    </method>
+    <method name="getCompletionPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which waitForCompletion() should check.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the task output filter.
+ 
+ @param conf the configuration.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="newValue" type="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the Configuration to set the task output filter.
+ 
+ @param conf the Configuration to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation to which the job is submitted to, if any
+
+ @return the reservationId the identifier of the job's reservation, null if
+         the job does not have any reservation associated with it]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation to which the job is submitted to
+
+ @param reservationId the reservationId to set]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Job</code>.
+ @throws IOException if fail to close.]]>
+      </doc>
+    </method>
+    <field name="OUTPUT_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPLETION_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets completionPollInvervalMillis]]>
+      </doc>
+    </field>
+    <field name="PROGRESS_MONITOR_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets progMonitorPollIntervalMillis]]>
+      </doc>
+    </field>
+    <field name="USED_GENERIC_PARSER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUBMIT_REPLICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SUBMIT_REPLICATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_WILDCARD_FOR_LIBJARS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_USE_WILDCARD_FOR_LIBJARS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The job submitter's view of the Job.
+ 
+ <p>It allows the user to configure the
+ job, submit it, control its execution, and query the state. The set methods
+ only work until the job is submitted, afterwards they will throw an 
+ IllegalStateException. </p>
+ 
+ <p>
+ Normally the user creates the application, describes various facets of the
+ job via {@link Job} and then submits the job and monitor its progress.</p>
+ 
+ <p>Here is an example on how to submit a job:</p>
+ <p><blockquote><pre>
+     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+ </pre></blockquote>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Job -->
+  <!-- start interface org.apache.hadoop.mapreduce.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MRJobConfig"/>
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration for the job.
+ @return the shared configuration object]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique ID for the job.
+ @return the object with the job id]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get configured the number of reduce tasks for this job. Defaults to 
+ <code>1</code>.
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="getInputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link InputFormat} class for the job.
+ 
+ @return the {@link InputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the combiner class for the job.
+ 
+ @return the combiner class for the job.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} class for the job.
+ 
+ @return the {@link OutputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Partitioner} class for the job.
+ 
+ @return the {@link Partitioner} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getSortComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pathname of the job's jar.
+ @return the pathname]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="getGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the reduce.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setGroupingComparatorClass(Class)
+ @see #getCombinerKeyGroupingComparator()]]>
+      </doc>
+    </method>
+    <method name="getJobSetupCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether job-setup and job-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getTaskCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether task-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to check if symlinks should be used, but currently
+ symlinks cannot be disabled.
+ @return true]]>
+      </doc>
+    </method>
+    <method name="getArchiveClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the archive entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getCacheArchives" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache archives set in the Configuration
+ @return A URI array of the caches set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCacheFiles" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache files set in the Configuration
+ @return A URI array of the files set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheArchives" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized caches
+ @return A path array of localized caches
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheFiles" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized files
+ @return A path array of localized files
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.]]>
+      </doc>
+    </method>
+    <method name="getFileClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getArchiveTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the archives.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getFileTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the files.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapred.map.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapred.reduce.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A read-only view of the job that is provided to the tasks while they
+ are running.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.JobContext -->
+  <!-- start class org.apache.hadoop.mapreduce.JobCounter -->
+  <class name="JobCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJtIdentifier" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare JobIds by first jtIdentifiers, then by job numbers]]>
+      </doc>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the stuff after the "job" prefix to the given builder. This is useful,
+ because the sub-ids use this substring at the start of their string.
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOBID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local" and a random number.
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobID -->
+  <!-- start class org.apache.hadoop.mapreduce.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobPriority -->
+  <!-- start class org.apache.hadoop.mapreduce.JobStatus -->
+  <class name="JobStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <doc>
+      <![CDATA[Change the current run state of the job.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Used to set the scheduling information associated to a particular Job.
+ 
+ @param schedulingInfo Scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the job acls.
+ 
+ @param acls {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name
+ @param queue queue name]]>
+      </doc>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set diagnostic information.
+ @param failureInfo diagnostic information]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="getMapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="getCleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="getSetupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="getReduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return start time of the job]]>
+      </doc>
+    </method>
+    <method name="clone" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getUsername" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the username of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the Scheduling information associated to a particular Job.
+ @return the scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="getJobACLs" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job acls.
+ 
+ @return a {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets any available info on the reason of failure of the job.
+ @return diagnostic information on why a job might have failed.]]>
+      </doc>
+    </method>
+    <method name="isJobComplete" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the status is for a completed job.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration file for the job.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the finish time of the job.]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check whether the job has retired.]]>
+      </doc>
+    </method>
+    <method name="getHistoryFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job history file name for a completed job. If job is not 
+ completed or history file not available then return null.]]>
+      </doc>
+    </method>
+    <method name="getNumUsedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="setNumUsedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="getNumReservedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="setNumReservedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="getUsedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the used memory]]>
+      </doc>
+    </method>
+    <method name="setUsedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="m" type="int"/>
+      <doc>
+      <![CDATA[@param m the used memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="int"/>
+      <doc>
+      <![CDATA[@param r the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getNeededMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the needed memory]]>
+      </doc>
+    </method>
+    <method name="setNeededMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the needed memory]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether job running in uber mode
+ @return job in uber-mode]]>
+      </doc>
+    </method>
+    <method name="setUber"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isUber" type="boolean"/>
+      <doc>
+      <![CDATA[Set uber-mode flag 
+ @param isUber Whether job running in uber-mode]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Describes the current status of a job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobStatus -->
+  <!-- start interface org.apache.hadoop.mapreduce.MapContext -->
+  <interface name="MapContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="getInputSplit" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the input split for this map.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context that is given to the {@link Mapper}.
+ @param <KEYIN> the key input type to the Mapper
+ @param <VALUEIN> the value input type to the Mapper
+ @param <KEYOUT> the key output type from the Mapper
+ @param <VALUEOUT> the value output type from the Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.MapContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Mapper -->
+  <class name="Mapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Mapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the beginning of the task.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="value" type="VALUEIN"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once for each key/value pair in the input split. Most applications
+ should override this, but the default is the identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Expert users can override this method for more complete control over the
+ execution of the Mapper.
+ @param context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link Configuration} for 
+ the job via the {@link JobContext#getConfiguration()}.
+ 
+ <p>The framework first calls 
+ {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by
+ {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)}
+ for each key/value pair in the <code>InputSplit</code>. Finally 
+ {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the sorting and grouping by 
+ specifying two key {@link RawComparator} classes.</p>
+
+ <p>The <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link Job#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>Configuration</code>.</p>
+  
+ <p>If the job has zero
+ reduces then the output of the <code>Mapper</code> is directly written
+ to the {@link OutputFormat} without sorting by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class TokenCounterMapper 
+     extends Mapper&lt;Object, Text, Text, IntWritable&gt;{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+ </pre></blockquote>
+
+ <p>Applications may override the
+ {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert
+ greater control on map processing e.g. multi-threaded <code>Mapper</code>s 
+ etc.</p>
+ 
+ @see InputFormat
+ @see JobContext
+ @see Partitioner  
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Mapper -->
+  <!-- start class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <class name="MarkableIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MarkableIteratorInterface"/>
+    <constructor name="MarkableIterator" type="java.util.Iterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new iterator layered on the input iterator
+ @param itr underlying iterator that implements MarkableIteratorInterface]]>
+      </doc>
+    </constructor>
+    <method name="mark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clearMark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="VALUE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>MarkableIterator</code> is a wrapper iterator class that 
+ implements the {@link MarkableIteratorInterface}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <class name="OutputCommitter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.	
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus.State#FAILED} or 
+ {@link JobStatus.State#KILLED}.  This is called from the application
+ master process for the entire job. This may be called multiple times.
+
+ @param jobContext Context of the job whose output is being written.
+ @param state final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task.  This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not successful.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @see #recoverTask(TaskAttemptContext)
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRJobConfig#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>.  This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ This may be called multiple times for the same task.  But from different
+ application attempts.
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputFormat -->
+  <class name="OutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given task.
+
+ @param context the information about the current task.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ Implementations which write to filesystems which support delegation
+ tokens usually collect the tokens for the destination path(s)
+ and attach them to the job context's JobConf.
+ @param context information about the job
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the output committer for this output format. This is responsible
+ for ensuring the output is committed correctly.
+ @param context the task context
+ @return an output committer
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.Partitioner -->
+  <class name="Partitioner" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Partitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="value" type="VALUE"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the partition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be partioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+
+ <p>Note: If you require your Partitioner class to obtain the Job's
+ configuration object, implement the {@link Configurable} interface.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Partitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <class name="QueueAclsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueAclsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueAclsInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueAclsInfo" type="java.lang.String, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueAclsInfo object using the queue name and the
+ queue operations array
+ 
+ @param queueName Name of the job queue
+ @param operations]]>
+      </doc>
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name.
+ 
+ @return name]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+    </method>
+    <method name="getOperations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get opearations allowed on queue.
+ 
+ @return array of String]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class to encapsulate Queue ACLs for a particular
+  user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String, org.apache.hadoop.mapreduce.QueueState, org.apache.hadoop.mapreduce.JobStatus[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param queueName
+ @param schedulingInfo
+ @param state
+ @param stats]]>
+      </doc>
+    </constructor>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the queue name of the JobQueueInfo
+ 
+ @param queueName Name of the job queue.]]>
+      </doc>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue name from JobQueueInfo
+ 
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheduling information associated to particular job queue
+ 
+ @param schedulingInfo]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the scheduling information associated to particular job queue.
+ If nothing is set would return <b>"N/A"</b>
+ 
+ @return Scheduling information associated to particular Job Queue]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.QueueState"/>
+      <doc>
+      <![CDATA[Set the state of the queue
+ @param state state of the queue.]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the queue state
+ @return the queue state.]]>
+      </doc>
+    </method>
+    <method name="setJobStatuses"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+    </method>
+    <method name="getQueueChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get immediate children.
+ 
+ @return list of QueueInfo]]>
+      </doc>
+    </method>
+    <method name="setQueueChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.List"/>
+    </method>
+    <method name="getProperties" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get properties.
+ 
+ @return Properties]]>
+      </doc>
+    </method>
+    <method name="setProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="props" type="java.util.Properties"/>
+    </method>
+    <method name="getJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the jobs submitted to queue
+ @return list of JobStatus for the submitted jobs]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getStateName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the stateName]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Enum representing queue state]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueState -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordReader -->
+  <class name="RecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="RecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at initialization.
+ @param split the split that defines the range of records to read
+ @param context the information about the task
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next key, value pair.
+ @return true if a key/value pair was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key
+ @return the current key or null if there is no current key
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the object that was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The current progress of the record reader through its data.
+ @return a number between 0.0 and 1.0 that is the fraction of the data read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the record reader.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The record reader breaks the data into key/value pairs for input to the
+ {@link Mapper}.
+ @param <KEYIN>
+ @param <VALUEIN>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordWriter -->
+  <class name="RecordWriter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RecordWriter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param context the context of the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <interface name="ReduceContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="nextKey" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Start processing next unique key.]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Iterate through the values for the current key, reusing the same value 
+ object, which is stored in the context.
+ @return the series of values associated with the current key. All of the 
+ objects returned directly and indirectly from this method are reused.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context passed to the {@link Reducer}.
+ @param <KEYIN> the class of the input keys
+ @param <VALUEIN> the class of the input values
+ @param <KEYOUT> the class of the output keys
+ @param <VALUEOUT> the class of the output values]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Reducer -->
+  <class name="Reducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Reducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the start of the task.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[This method is called once for each key. Most applications will define
+ their reduce class by overriding this method. The default implementation
+ is an identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advanced application writers can use the 
+ {@link #run(org.apache.hadoop.mapreduce.Reducer.Context)} method to
+ control how the reduce task works.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p><code>Reducer</code> implementations 
+ can access the {@link Configuration} for the job via the 
+ {@link JobContext#getConfiguration()} method.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p>The <code>Reducer</code> copies the sorted output from each 
+   {@link Mapper} using HTTP across the network.</p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework merge sorts <code>Reducer</code> inputs by 
+   <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key).</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>To achieve a secondary sort on the values returned by the value 
+   iterator, the application should extend the key with the secondary
+   key and define a grouping comparator. The keys will be sorted using the
+   entire key, but will be grouped using the grouping comparator to decide
+   which keys and values are sent in the same call to reduce.The grouping 
+   comparator is specified via 
+   {@link Job#setGroupingComparatorClass(Class)}. The sort order is
+   controlled by 
+   {@link Job#setSortComparatorClass(Class)}.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)}
+   method is called for each <code>&lt;key, (collection of values)&gt;</code> in
+   the sorted inputs.</p>
+   <p>The output of the reduce task is typically written to a 
+   {@link RecordWriter} via 
+   {@link Context#write(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class IntSumReducer&lt;Key&gt; extends Reducer&lt;Key,IntWritable,
+                                                 Key,IntWritable&gt; {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable&lt;IntWritable&gt; values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Reducer -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique name for this task attempt.]]>
+      </doc>
+    </method>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="msg" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the current status of the task to the given string.]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the last set status message.
+ @return the current status message]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current progress of the task attempt.
+ @return a number between 0.0 and 1.0 (inclusive) indicating the attempt's
+ progress.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>counterName</code>]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>groupName</code> and 
+ <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>groupName</code> and 
+         <code>counterName</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context for task attempts.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapreduce.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link TaskID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the TaskType of the TaskAttemptID]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the StringBuilder
+ @param builder the builder to append ot
+ @return the builder that was passed in.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskIds by first tipIds, then by task numbers.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="ATTEMPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapreduce.TaskAttemptID, int, boolean, org.apache.hadoop.mapreduce.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getEventId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns event Id. 
+ @return event id]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerHttp" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[http location of the tasktracker where this task ran. 
+ @return http location of tasktracker user logs]]>
+      </doc>
+    </method>
+    <method name="getTaskRunTime" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns time (in millisec) the task took to complete.]]>
+      </doc>
+    </method>
+    <method name="setTaskRunTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskCompletionTime" type="int"/>
+      <doc>
+      <![CDATA[Set the task completion time
+ @param taskCompletionTime time (in millisec) the task took to complete]]>
+      </doc>
+    </method>
+    <method name="setEventId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="eventId" type="int"/>
+      <doc>
+      <![CDATA[set event Id. should be assigned incrementally starting from 0. 
+ @param eventId]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <method name="setTaskStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"/>
+      <doc>
+      <![CDATA[Set task status. 
+ @param status]]>
+      </doc>
+    </method>
+    <method name="setTaskTrackerHttp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskTrackerHttp" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set task tracker http location. 
+ @param taskTrackerHttp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMapTask" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="idWithinJob" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCounter -->
+  <class name="TaskCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} of the task 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable. Sets the task type to
+ {@link TaskType#REDUCE}, the ID to 0, and the job ID to an empty job ID.]]>
+      </doc>
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this tip belongs to.
+
+ @return the JobID object]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID.
+
+ @return whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the task.
+
+ @return the type of the task]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskInProgressIds by first jobIds, then by tip numbers.
+ Reducers are defined as greater than mappers.
+
+ @param o the TaskID against which to compare
+ @return 0 if equal, positive if this TaskID is greater, and negative if
+ this TaskID is less]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the given builder.
+
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskID object from given string.
+
+ @param str the target string
+ @return constructed TaskID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getRepresentingCharacter" return="char"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <doc>
+      <![CDATA[Gets the character representing the {@link TaskType}.
+
+ @param type the TaskType
+ @return the character]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="char"/>
+      <doc>
+      <![CDATA[Gets the {@link TaskType} corresponding to the character.
+
+ @param c the character
+ @return the TaskType]]>
+      </doc>
+    </method>
+    <method name="getAllTaskTypes" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string of characters describing all possible {@link TaskType}
+ values
+
+ @return a string of all task type characters]]>
+      </doc>
+    </method>
+    <field name="TASK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="TASK_ID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="taskIdPattern" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskID -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <interface name="TaskInputOutputContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advance to the next key, value pair, returning null if at end.
+ @return the key object that was read into, or null if no more]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key.
+ @return the current key object or null if there isn't one
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the value object that was read into
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Generate an output key/value pair.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} for the task-attempt.
+ @return the <code>OutputCommitter</code> for the task-attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A context object that allows input and output from the task. It is only
+ supplied to the {@link Mapper} or {@link Reducer}.
+ @param <KEYIN> the input key type for the task
+ @param <VALUEIN> the input value type for the task
+ @param <KEYOUT> the output key type for the task
+ @param <VALUEOUT> the output value type for the task]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <class name="TaskTrackerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskTrackerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskTrackerName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the tasktracker's name.
+ 
+ @return tracker's name.]]>
+      </doc>
+    </method>
+    <method name="isBlacklisted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether tracker is blacklisted
+ @return true if tracker is blacklisted
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="getReasonForBlacklist" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the reason for which the tasktracker was blacklisted.
+ 
+ @return reason which tracker was blacklisted]]>
+      </doc>
+    </method>
+    <method name="getBlacklistReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets a descriptive report about why the tasktracker was blacklisted.
+ 
+ @return report describing why the tasktracker was blacklisted.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Information about TaskTracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskType -->
+  <class name="TaskType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enum for map, reduce, job-setup, job-cleanup, task-cleanup task types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskType -->
+</package>
+<package name="org.apache.hadoop.mapreduce.checkpoint">
+</package>
+<package name="org.apache.hadoop.mapreduce.counters">
+  <!-- start class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <class name="AbstractCounters" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters, org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another counters object.
+ @param <C1> type of the other counter
+ @param <G1> type of the other counter group
+ @param counters the counters object to copy
+ @param groupFactory the factory for new groups]]>
+      </doc>
+    </constructor>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter, create one if necessary
+ @param groupName of the counter
+ @param counterName name of the counter
+ @return the matching counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Find the counter for the given enum. The same enum will always return the
+ same counter.
+ @param key the counter key
+ @return the matching counter object]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of all counter classes.
+ @return Set of counter names.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getGroup" return="G"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named counter group, or an empty group if there is none
+ with the specified name.
+ @param groupName name of the group
+ @return the group]]>
+      </doc>
+    </method>
+    <method name="countCounters" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the total number of counters, by summing the number of counters
+ in each group.
+ @return the total number of counters]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the set of groups.
+ Counters ::= version #fgroups (groupId, group)* #groups (group)*]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return textual representation of the counter values.
+ @return the string]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract class to provide common implementation for the Counters
+ container in both mapred and mapreduce packages.
+
+ @param <C> type of counter inside the counters
+ @param <G> type of group inside the counters]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <!-- start interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+  <interface name="CounterGroupBase"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the internal name of the group
+ @return the internal name]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the group.
+ @return the human readable name]]>
+      </doc>
+    </method>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the group
+ @param displayName of the group]]>
+      </doc>
+    </method>
+    <method name="addCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="T"/>
+      <doc>
+      <![CDATA[Add a counter to this group.
+ @param counter to add]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a counter to this group
+ @param name  of the counter
+ @param displayName of the counter
+ @param value of the counter
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @param displayName the display name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+      <doc>
+      <![CDATA[Find a counter in the group
+ @param counterName the name of the counter
+ @param create create the counter if not found if true
+ @return the counter that was found or added or null if create is false]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of counters in this group.]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+      <doc>
+      <![CDATA[Increment all counters by a group of counters
+ @param rightGroup  the group to be added to this group]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The common counter group interface.
+
+ @param <T> type of the counter for the group]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a double value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="double"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a double value.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getSum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getSum" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <method name="setMaxItems" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <doc>
+      <![CDATA[Set the limit on the number of unique values
+ @param n the desired limit on the number of unique values
+ @return the new limit on the number of unique values]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return the number of unique objects aggregated]]>
+      </doc>
+    </method>
+    <method name="getUniqueItems" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the set of the unique objects]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of the unique objects. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <field name="MAX_NUM_UNIQUE_VALUES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param conf a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for the given 
+   key/value pairs by delegating the invocation to the real object.
+   
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <field name="theAggregatorDescriptor" type="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value 
+ aggregator descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class
+ that may be dynamically loaded. The other is to
+ delegate invocations of generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addNextValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val the value to be added]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the agregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return an array of values as the outputs of the combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="uniqCount" type="long"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param uniqCount the limit in the number of unique values to keep, 
+                  if type is UNIQ_VALUE_COUNT 
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate 1 or 2 aggregation-id/value pairs for the given key/value pair.
+ The first id will be of type LONG_VALUE_SUM, with "record_count" as
+ its aggregation id. If the input is a file split,
+ the second id of the same type will be generated too, with the file name 
+ as its aggregation id. This achieves the behavior of counting the total 
+ number of records in the input data, and the number of records 
+ in each input file.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param conf a configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="inputFile" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param context to collect combined values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for 
+ the given key/value pair.
+ This function is usually called by the mapper of an Aggregate based job.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param conf
+          a Configuration object that may contain the information 
+          that can be used to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a {@link Configuration}
+ object. Its main function is to generate a list of aggregation-id/value 
+ pairs. An aggregation id encodes an aggregation type which is used to 
+ guide the way to aggregate the value in the reduce/combiner phrase of an
+ Aggregate based job. 
+ The mapper in an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param conf The configuration for job
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a Job object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specializing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner 
+ classes, and a set of built-in value aggregators, and a generic utility 
+ class that helps user create map/reduce jobs using the generic class. 
+ The built-in aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(Configuration conf); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getValueAggregatorDescriptor" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spec" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getAggregatorDescriptors" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="DESCRIPTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DESCRIPTOR_NUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[@param key
+        the key is expected to be a Text object, whose prefix indicates
+        the type of aggregation to aggregate the values. In effect, data
+        driven computing is achieved. It is assumed that each aggregator's
+        getReport method emits appropriate output for the aggregator. This
+        may be further customized.
+ @param values the values to be aggregated
+ @param context]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add the given val to the aggregator.
+ 
+ @param val the value to be added. It is expected to be a string
+ in the form of xxxx\tnum, meaning xxxx has num occurrences.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this aggregator.
+ It includes the following basic statistics of the histogram:
+    the number of unique values
+    the minimum value
+    the media value
+    the maximum value
+    the average value
+    the standard deviation]]>
+      </doc>
+    </method>
+    <method name="getReportDetails" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a string representation of the list of value/frequence pairs of 
+ the histogram]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a list value/frequence pairs.
+  The return value is expected to be used by the reducer.]]>
+      </doc>
+    </method>
+    <method name="getReportItems" return="java.util.TreeMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a TreeMap representation of the histogram]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.chain">
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <class name="ChainMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain mapper.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ 
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ </p>
+ ChainMapper usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+  <class name="ChainReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="reducerConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Sets the {@link Reducer} class to the chain job.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's Configuration.
+ This precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ 
+ @param job
+          the job
+ @param klass
+          the Reducer class to add.
+ @param inputKeyClass
+          reducer input key class.
+ @param inputValueClass
+          reducer input value class.
+ @param outputKeyClass
+          reducer output key class.
+ @param outputValueClass
+          reducer output value class.
+ @param reducerConf
+          a configuration for the Reducer class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain reducer.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the
+ chain.
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ 
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion. The output of the reducer becomes the input of
+ the first mapper and output of first becomes the input of the second, and so
+ on until the last Mapper, the output of the last Mapper will be written to
+ the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain. This
+ enables having reusable specialized Mappers that can be combined to perform
+ composite operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p> Using the ChainMapper and the ChainReducer classes is possible to
+ compose Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO. </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ ChainReducer usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.db">
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <class name="BigDecimalSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BigDecimalSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="tryDivide" return="java.math.BigDecimal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="numerator" type="java.math.BigDecimal"/>
+      <param name="denominator" type="java.math.BigDecimal"/>
+      <doc>
+      <![CDATA[Divide numerator by denominator. If impossible in exact mode, use rounding.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over BigDecimal values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <class name="BooleanSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BooleanSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over boolean values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <class name="DataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getBoundingValsQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a query which returns the minimum and maximum values for
+ the order-by column.
+
+ The min value should be in the first column, and the
+ max value should be in the second column of the results.]]>
+      </doc>
+    </method>
+    <method name="setBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="query" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-defined bounding query to use with a user-defined query.
+      This *must* include the substring "$CONDITIONS"
+      (DataDrivenDBInputFormat.SUBSTITUTE_TOKEN) inside the WHERE clause,
+      so that DataDrivenDBInputFormat knows where to insert split clauses.
+      e.g., "SELECT foo FROM mytable WHERE $CONDITIONS"
+      This will be expanded to something like:
+        SELECT foo FROM mytable WHERE (id &gt; 100) AND (id &lt; 250)
+      inside each split.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="splitBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Note that the "orderBy" column is called the "splitBy" in this version.
+ We reuse the same field, but it's not strictly ordering it -- just partitioning
+ the results.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputBoundingQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[setInput() takes a custom query and a separate "bounding query" to use
+      instead of the custom "count query" used by DBInputFormat.]]>
+      </doc>
+    </method>
+    <field name="SUBSTITUTE_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If users are providing their own query, the following string is expected to
+      appear in the WHERE clause, which will be substituted with a pair of conditions
+      on the input to allow input splits to parallelise the import.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ Operates like DBInputFormat, but instead of using LIMIT and OFFSET to demarcate
+ splits, it tries to generate WHERE clauses which separate the data into roughly
+ equivalent shards.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <class name="DataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records,
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table,
+ using data-driven WHERE clause splits.
+ Emits LongWritables containing the record number as
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <class name="DateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.IntegerSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Given a Date 'd', format it as a string for use in a SQL date
+ comparison operation.
+ @param d the date to format.
+ @return the string representing this date in SQL with any appropriate
+ quotation characters, etc.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values.
+ Make use of logic from IntegerSplitter, since date/time are just longs
+ in Java.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the {@link Configuration}.  
+ @param conf the configuration
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Returns a connection object o the DB 
+ @throws ClassNotFoundException 
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getInputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="getInputConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputConditions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conditions" type="java.lang.String"/>
+    </method>
+    <method name="getInputOrderBy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputOrderBy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orderby" type="java.lang.String"/>
+    </method>
+    <method name="getInputQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputCountQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="setInputBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputBoundingQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClass" type="java.lang.Class"/>
+    </method>
+    <method name="getOutputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getOutputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="setOutputFieldCount"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldCount" type="int"/>
+    </method>
+    <method name="getOutputFieldCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_BOUNDING_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the max and min values of the jdbc.input.query]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A container for configuration property names for jobs with DB input/output.
+  
+ The job can be configured using the static methods in this class, 
+ {@link DBInputFormat}, and {@link DBOutputFormat}. 
+ Alternatively, the properties can be set in the configuration with proper
+ values. 
+   
+ @see DBConfiguration#configureDB(Configuration, String, String, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String, String, String...)
+ @see DBOutputFormat#setOutput(Job, String, String...)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBProductName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for getting the total number of rows, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, 
+ eg. '(updated &gt; 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(Job, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns 
+ the number of records in the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(Job, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+    <method name="closeConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="conditions" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="connection" type="java.sql.Connection"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="tableName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fieldNames" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="dbConf" type="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ <p>
+ DBInputFormat emits LongWritables containing the record number as 
+ key and DBWritables as value. 
+ 
+ The SQL query, and input class can be using one of the two 
+ setInput methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="constructQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="table" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Constructs the query used as the prepared statement to insert data.
+ 
+ @param table
+          the table to insert into
+ @param fieldNames
+          the fields to insert into. If field names are unknown, supply an
+          array of nulls.]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with 
+ the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job 
+ with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A OutputFormat that sends the reduce output to a SQL table.
+ <p> 
+ {@link DBOutputFormat} accepts &lt;key,value&gt; pairs, where 
+ key has a type extending DBWritable. Returned {@link RecordWriter} 
+ writes <b>only the key</b> to the database with a batch SQL query.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <class name="DBRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.LongWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #nextKeyValue()}">
+      <param name="key" type="org.apache.hadoop.io.LongWritable"/>
+      <param name="value" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #nextKeyValue()}]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplit" return="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatement" return="java.sql.PreparedStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stmt" type="java.sql.PreparedStatement"/>
+    </method>
+    <field name="statement" type="java.sql.PreparedStatement"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table.
+ Emits LongWritables containing the record number as 
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <interface name="DBSplitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="split" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Given a ResultSet containing one record (and already advanced to that record)
+ with two columns (a low value, and a high value, both of the same type), determine
+ a set of splits that span the given values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DBSplitter will generate DBInputSplits to use with DataDrivenDBInputFormat.
+ DataDrivenDBInputFormat needs to interpolate between two values that
+ represent the lowest and highest valued records to import. Depending
+ on the data-type of the column, this requires different behavior.
+ DBSplitter implementations should perform this for a data type or family
+ of data types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statement" type="java.sql.PreparedStatement"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Sets the fields of the object in the {@link PreparedStatement}.
+ @param statement the statement that the fields are put into.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resultSet" type="java.sql.ResultSet"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Reads the fields of the object from the {@link ResultSet}. 
+ @param resultSet the {@link ResultSet} to get the fields from.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Objects that are read from/written to a database should implement
+ <code>DBWritable</code>. DBWritable, is similar to {@link Writable} 
+ except that the {@link #write(PreparedStatement)} method takes a 
+ {@link PreparedStatement}, and {@link #readFields(ResultSet)} 
+ takes a {@link ResultSet}. 
+ <p>
+ Implementations are responsible for writing the fields of the object 
+ to PreparedStatement, and reading the fields of the object from the 
+ ResultSet. 
+ 
+ <p>Example:</p>
+ If we have the following table in the database :
+ <pre>
+ CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+ </pre>
+ then we can read/write the tuples from/to the table with :
+ <p><pre>
+ public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <class name="FloatSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="FloatSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over floating-point values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <class name="IntegerSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="IntegerSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over integer values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <class name="MySQLDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <class name="MySQLDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <class name="OracleDataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="OracleDataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table in an Oracle db.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <class name="OracleDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a Oracle table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <class name="OracleDateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.DateSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values returned by an Oracle db.
+ Make use of logic from DateSplitter, since this just needs to use
+ some Oracle-specific functions on the formatting end when generating
+ InputSplits.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <class name="OracleDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records from an Oracle DB.]]>
+      </doc>
+    </method>
+    <method name="setSessionTimeZone"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="conn" type="java.sql.Connection"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Set session time zone
+ @param conf The current configuration.
+ We read the 'oracle.sessionTimeZone' property from here.
+ @param conn The connection to alter the timezone properties of.]]>
+      </doc>
+    </method>
+    <field name="SESSION_TIMEZONE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set to a timezone string.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from an Oracle SQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+  <class name="TextSplitter" extends="org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[This method needs to determine the splits between two user-provided strings.
+ In the case where the user's strings are 'A' and 'Z', this is not hard; we 
+ could create two splits from ['A', 'M') and ['M', 'Z'], 26 splits for strings
+ beginning with each letter, etc.
+
+ If a user has provided us with the strings "Ham" and "Haze", however, we need
+ to create splits that differ in the third letter.
+
+ The algorithm used is as follows:
+ Since there are 2**16 unicode characters, we interpret characters as digits in
+ base 65536. Given a string 's' containing characters s_0, s_1 .. s_n, we interpret
+ the string as the number: 0.s_0 s_1 s_2.. s_n in base 65536. Having mapped the
+ low and high strings into floating-point values, we then use the BigDecimalSplitter
+ to establish the even split points, then map the resulting floating point values
+ back into strings.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over text strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.fieldsel">
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <class name="FieldSelectionHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FieldSelectionHelper" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="parseOutputKeyValueSpec" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="specToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldSeparator" type="java.lang.String"/>
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="getKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="extractOutputKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="val" type="java.lang.String"/>
+      <param name="fieldSep" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valFieldList" type="java.util.List"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="ignoreKey" type="boolean"/>
+      <param name="isMap" type="boolean"/>
+    </method>
+    <field name="emptyText" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #DATA_FIELD_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #DATA_FIELD_SEPARATOR}]]>
+      </doc>
+    </field>
+    <field name="MAP_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REDUCE_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <class name="FieldSelectionMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) separated
+ field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec can be a 
+ simple number (e.g. 5) specifying a specific field, or a range (like 2-5)
+ to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value
+ fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+  <class name="FieldSelectionReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a reducer class that can be used to perform field
+ selections in a manner similar to unix cut. 
+ 
+ The input data is treated as fields separated by a user specified
+ separator (the default value is "\t"). The user can specify a list of
+ fields that form the reduce output keys, and a list of fields that form
+ the reduce output values. The fields are the union of those from the key
+ and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) 
+ separated field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec
+ can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) 
+ specifying all the fields starting from field 3. The open range field
+ spec applies value fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.input">
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="setMaxSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="maxSplitSize" type="long"/>
+      <doc>
+      <![CDATA[Specify the maximum size (in bytes) of each split. Each split is
+ approximately equal to the specified size.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeNode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeNode" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per node.
+ This applies to data that is left over after combining data on a single
+ node into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeNode.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeRack"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeRack" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per rack.
+ This applies to data that is left over after combining data on a single
+ rack into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeRack.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SPLIT_MINSIZE_PERNODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE_PERRACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link CombineFileSplit}'s in 
+ {@link InputFormat#getSplits(JobContext)} method. 
+ 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default splitting behavior in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement 
+ {@link InputFormat#createRecordReader(InputSplit, TaskAttemptContext)}
+ to construct <code>RecordReader</code>'s for 
+ <code>CombineFileSplit</code>'s.
+ 
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapreduce.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapreduce.lib.input.FileInputFormat, org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartOffsets" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the start offsets of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getLengths" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the lengths of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the start offset of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the length of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getNumPaths" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns all the Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns all the Paths where this input-split resides]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. 
+ 
+ Unlike {@link FileSplit}, CombineFileSplit class does not represent 
+ a split of a file, but a split of input files into smaller sets. 
+ A split may contain blocks from different file but all 
+ the blocks in the same split are probably local to some rack <br> 
+ CombineFileSplit can be used to implement {@link RecordReader}'s, 
+ with reading one record per file.
+ 
+ @see FileSplit
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <class name="FileInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setInputDirRecursive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputDirRecursive" type="boolean"/>
+      <doc>
+      <![CDATA[@param job
+          the job to modify
+ @param inputDirRecursive]]>
+      </doc>
+    </method>
+    <method name="getInputDirRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[@param job
+          the job to look at.
+ @return should the files to be read recursively?]]>
+      </doc>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower bound on split size imposed by the format.
+ @return the number of bytes of the minimal split for this format]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param context the job context
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+ @param job the job to modify
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="setMinInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the minimum input split size
+ @param job the job to modify
+ @param size the minimum size]]>
+      </doc>
+    </method>
+    <method name="getMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the minimum split size
+ @param job the job
+ @return the minimum number of bytes that can be in a split]]>
+      </doc>
+    </method>
+    <method name="setMaxInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum split size
+ @param job the job to modify
+ @param size the maximum split size]]>
+      </doc>
+    </method>
+    <method name="getMaxSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the maximum split size.
+ @param context the job to look at.
+ @return the maximum number of bytes a split can include]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+
+ If security is enabled, this method collects
+ delegation tokens from the input paths and adds them to the job's
+ credentials.
+ @param job the job to list input paths for and attach tokens to.
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Generate the list of files and make them into FileSplits.
+ @param job the job context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blockSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="maxSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param job the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param job The job to modify
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param job The job to modify 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job} to modify
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param context The job
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <field name="INPUT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MAXSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PATHFILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LIST_STATUS_NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LIST_STATUS_NUM_THREADS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}s.
+
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobContext)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(JobContext, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <class name="FileInputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host and cached-blocks information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobContext)} and passed to
+ {@link InputFormat#createRecordReader(InputSplit,TaskAttemptContext)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ The first element of the list is used as the init cause value.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="setKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="line" type="byte[]"/>
+      <param name="lineLen" type="int"/>
+      <param name="pos" type="int"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_VALUE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KEY_VALUE_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #KEY_VALUE_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #KEY_VALUE_SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either line feed or carriage-return are used to signal end of line. 
+ Each line is divided into key and value parts by a separator byte. If no
+ such a byte exists, the key will be the entire line and value will be empty.
+ The separator byte can be specified in config file under the attribute name
+ mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <field name="DIR_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIR_MAPPERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see FileInputFormat#getSplits(JobContext)]]>
+      </doc>
+    </method>
+    <method name="getSplitsForFile" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="numLinesPerSplit" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <method name="setNumLinesPerSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="numLines" type="int"/>
+      <doc>
+      <![CDATA[Set the number of lines per split
+ @param job the job to modify
+ @param numLines the number of lines per split]]>
+      </doc>
+    </method>
+    <method name="getNumLinesPerSplit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the number of lines per split
+ @param job the job
+ @return the number of lines per split]]>
+      </doc>
+    </method>
+    <field name="LINES_PER_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, except it generates
+ SequenceFileAsTextRecordReader which converts the input keys and values
+ to their String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by
+ calling toString() method. This class to SequenceFileAsTextInputFormat
+ class is as LineRecordReader class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param context the task-attempt context
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param job The job
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_FREQUENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.jobcontrol">
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <class name="ControlledJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ControlledJob" type="org.apache.hadoop.mapreduce.Job, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param job a mapreduce job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="ControlledJob" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ 
+ @param conf mapred job configuration representing a job to be executed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job name of this job]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job name for  this job.
+ @param jobName the job name]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job ID of this job assigned by JobControl]]>
+      </doc>
+    </method>
+    <method name="setJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job ID for  this job.
+ @param id the job ID]]>
+      </doc>
+    </method>
+    <method name="getMapredJobId" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapreduce job]]>
+      </doc>
+    </method>
+    <method name="setJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <doc>
+      <![CDATA[Set the mapreduce job
+ @param job the mapreduce job for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setJobState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"/>
+      <doc>
+      <![CDATA[Set the state for this job.
+ @param state the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the message of this job]]>
+      </doc>
+    </method>
+    <method name="setMessage"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the message for this job.
+ @param message the message for this job.]]>
+      </doc>
+    </method>
+    <method name="getDependentJobs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="isCompleted" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in a complete state]]>
+      </doc>
+    </method>
+    <method name="isReady" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in READY state]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="failJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submit this job to mapred. The state becomes RUNNING if submission 
+ is successful, FAILED otherwise.]]>
+      </doc>
+    </method>
+    <field name="CREATE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class encapsulates a MapReduce job and its dependency. It monitors 
+  the states of the depending jobs and updates the state of this job.
+  A job starts in the WAITING state. If it does not have any depending jobs,
+  or all of the depending jobs are in SUCCESS state, then the job state 
+  will become READY. If any depending jobs fail, the job will fail too. 
+  When in READY state, the job can be submitted to Hadoop for execution, with
+  the state changing into RUNNING state. From RUNNING state, the job 
+  can get into SUCCESS or FAILED state, depending 
+  the status of the job execution.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+  <class name="JobControl" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Runnable"/>
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a new controlled job.
+ @param aJob the new controlled job]]>
+      </doc>
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a new job.
+ @param aJob the new job]]>
+      </doc>
+    </method>
+    <method name="addJobCollection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getThreadState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl.ThreadState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[set the thread state to STOPPING so that the 
+ thread will stop when it wakes up.]]>
+      </doc>
+    </method>
+    <method name="suspend"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[suspend the running thread]]>
+      </doc>
+    </method>
+    <method name="resume"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[resume the suspended thread]]>
+      </doc>
+    </method>
+    <method name="allFinished" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The main loop for the thread.
+  The loop does the following:
+  	Check the states of the running jobs
+  	Update the states of waiting jobs
+  	Submit the jobs in ready state]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a set of MapReduce jobs and its dependency.
+   
+  It tracks the states of the jobs by placing them into different tables
+  according to their states. 
+  
+  This class provides APIs for the client app to add a job to the group 
+  and to get the jobs in the group in different states. When a job is 
+  added, an ID unique to the group is assigned to the job. 
+  
+  This class has a thread that submits jobs when they become ready, 
+  monitors the states of the running jobs, and updates the states of jobs
+  based on the state changes of their depending jobs states. The class 
+  provides APIs for suspending/resuming the thread, and 
+  for stopping the thread.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.join">
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <class name="ComposableInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ComposableInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <class name="ComposableRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ComposableRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapreduce.join.expr</tt> property and
+ user-supplied join types from <tt>mapreduce.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <field name="JOIN_EXPR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOIN_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapreduce.join.define.&lt;ident&gt;</tt> to a classname. 
+ In the expression <tt>mapreduce.join.expr</tt>, the identifier will be
+ assumed to be a ComposableRecordReader.
+ <tt>mapreduce.join.keycomparator</tt> can be a classname used to compare 
+ keys in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for failing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createTupleWritable" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="X"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="keyclass" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="key" type="K"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="value" type="X"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <class name="Parser.Node" extends="org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapreduce.lib.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if a call to next may return a value. This is permitted false
+ positives, but not false negatives.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign next value to actual.
+ It is required that elements added to a ResetableIterator be returned in
+ the same order after a call to {@link #reset} (FIFO).
+
+ Note that a call to this may fail for nested joins (i.e. more elements
+ available, but none satisfying the constraints of the join)]]>
+      </doc>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign last value returned to actual.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set iterator to return to the start of its range. Must be called after
+ calling {@link #add} to avoid a ConcurrentModificationException.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an element to the collection of elements to iterate over.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close datasources and release resources. Calling methods on the iterator
+ after calling close has undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Close datasources, but do not release internal resources. Calling this
+ method should permit the object to be reused with a different datasource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <class name="TupleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <method name="has" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Return true if tuple has an element at the position provided.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith Writable from Tuple.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of children in this Tuple.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator over the elements in this tuple.
+ Note that this doesn't flatten the tuple; one may receive tuples
+ from this iterator.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert Tuple to String as in the following.
+ <tt>[&lt;child1&gt;,&lt;child2&gt;,...,&lt;childn&gt;]</tt>]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes each Writable to <code>out</code>.
+ TupleWritable format:
+ {@code
+  <count><type1><type2>...<typen><obj1><obj2>...<objn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <field name="written" type="java.util.BitSet"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedRecordReader" type="int"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current key]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current value]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="empty" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmp" type="org.apache.hadoop.io.WritableComparator"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ 
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.map">
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <class name="MultithreadedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultithreadedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumberOfThreads" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[The number of threads in the thread pool that will run the map function.
+ @param job the job
+ @return the number of threads]]>
+      </doc>
+    </method>
+    <method name="setNumberOfThreads"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="threads" type="int"/>
+      <doc>
+      <![CDATA[Set the number of threads in the pool for running maps.
+ @param job the job to modify
+ @param threads the new number of threads]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the application's mapper class.
+ @param <K1> the map's input key type
+ @param <V1> the map's input value type
+ @param <K2> the map's output key type
+ @param <V2> the map's output value type
+ @param job the job
+ @return the mapper class to run]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the application's mapper class.
+ @param <K1> the map input key type
+ @param <V1> the map input value type
+ @param <K2> the map output key type
+ @param <V2> the map output value type
+ @param job the job to modify
+ @param cls the class to use as the mapper]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the application's maps using a thread pool.]]>
+      </doc>
+    </method>
+    <field name="NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAP_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Multithreaded implementation for @link org.apache.hadoop.mapreduce.Mapper.
+ <p>
+ It can be used instead of the default implementation,
+ {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU
+ bound in order to improve throughput.
+ <p>
+ Mapper implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured with the mapper to use via 
+ {@link #setMapperClass(Job, Class)} and
+ the number of thread the thread-pool can use with the
+ {@link #getNumberOfThreads(JobContext)} method. The default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <class name="TokenCounterMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenCounterMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Tokenize the input values and emit each word with a count of 1.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+  <class name="WrappedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMapContext" return="org.apache.hadoop.mapreduce.Mapper.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapContext" type="org.apache.hadoop.mapreduce.MapContext"/>
+      <doc>
+      <![CDATA[Get a wrapped {@link Mapper.Context} for custom implementations.
+ @param mapContext <code>MapContext</code> to be wrapped
+ @return a wrapped <code>Mapper.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} which wraps a given one to allow custom 
+ {@link Mapper.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.output">
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <class name="BindingPathOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BindingPathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instantiate.
+ @param outputPath output path (may be null)
+ @param context task context
+ @throws IOException on any failure.]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCommitter" return="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the inner committer.
+ @return the bonded committer.]]>
+      </doc>
+    </method>
+    <field name="NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The classname for use in configurations.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is a special committer which creates the factory for the committer and
+ runs off that. Why does it exist? So that you can explicitly instantiate
+ a committer by classname and yet still have the actual implementation
+ driven dynamically by the factory options and destination filesystem.
+ This simplifies integration
+ with existing code which takes the classname of a committer.
+ There's no factory for this, as that would lead to a loop.
+
+ All commit protocol methods and accessors are delegated to the
+ wrapped committer.
+
+ How to use:
+
+ <ol>
+   <li>
+     In applications which take a classname of committer in
+     a configuration option, set it to the canonical name of this class
+     (see {@link #NAME}). When this class is instantiated, it will
+     use the factory mechanism to locate the configured committer for the
+     destination.
+   </li>
+   <li>
+     In code, explicitly create an instance of this committer through
+     its constructor, then invoke commit lifecycle operations on it.
+     The dynamically configured committer will be created in the constructor
+     and have the lifecycle operations relayed to it.
+   </li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the path where final output of the job should be placed.  This
+ could also be considered the committed application attempt path.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @param out the output path to place these in.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param appAttemptId the ID of the application attempt for this job.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @param out The output path to put things in.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until
+ the entire job is committed.
+ @param context the context of the task attempt
+ @return the path where the output of a committed task is stored until
+ the entire job is committed.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until the
+ entire job is committed for a specific application attempt.
+ @param appAttemptId the id of the application attempt to use
+ @param context the context of any task.
+ @return the path where the output of a committed task is stored.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ @return the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the temporary directory that is the root of all of the task 
+ work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do works in commitJobInternal().
+ Could retry on failure if using algorithm 2.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJobInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do following commit job, include:
+ Move all committed tasks to the final output dir (algorithm 1 only).
+ Delete the temporary directory, including all of the work directories.
+ Create a _SUCCESS file to make it as successful.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the temporary directory, including all of the work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[No task setup required.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the files from the work directory to the job output directory
+ @param context the task context]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did this task write any files in the work directory?
+ @param context the task's context]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="PENDING_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of directory where pending data is placed.  Data that has not been
+ committed yet.]]>
+      </doc>
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name 
+
+ The static variable to be compatible with M/R 1.x]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCESSFUL_JOB_OUTPUT_DIR_MARKER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param job the job to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param job the Job to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param job the job to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param job the {@link Job} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param job The job to modify
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(TaskInputOutputContext)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in a work directory during execution 
+ of his task i.e. via 
+ {@link #getWorkOutputPath(TaskInputOutputContext)}, and
+ the framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getPathForWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueFile} method to make the file name
+ unique for the task.</p>
+
+ @param context the context for the task.
+ @param name the name for the file.
+ @param extension the extension for the file
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getUniqueFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a unique filename, based on the task id, name, and extension
+ @param context the task that is calling this
+ @param name the base filename
+ @param extension the filename extension
+ @return a string like $name-[mrsct]-$id$extension]]>
+      </doc>
+    </method>
+    <method name="getDefaultWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the default path and filename for the output format.
+ @param context the task context
+ @param extension an extension to add to the filename
+ @return a full path $output/_temporary/$taskid/part-[mr]-$id
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getOutputName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the base output name for the output file.]]>
+      </doc>
+    </method>
+    <method name="setOutputName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the base output name for output file to be created.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BASE_OUTPUT_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PART" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option: should output be compressed? {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_CODEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If compression is enabled, name of codec: {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Type of compression {@value}: NONE, RECORD, BLOCK.
+ Generally only used in {@code SequenceFileOutputFormat}.]]>
+      </doc>
+    </field>
+    <field name="OUTDIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Destination directory of work: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}s that read from {@link FileSystem}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <class name="FileOutputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapreduce.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the underlying output format.
+ @param baseOut the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapreduce.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link Job} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="OUTPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.
+ Use in conjuction with org.apache.hadoop.mapreduce.lib.output.MultipleOutputs to recreate the
+ behaviour of org.apache.hadoop.mapred.lib.MultipleTextOutputFormat (etc) of the old Hadoop API.
+ See {@link MultipleOutputs} documentation for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapreduce.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes 
+ {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple outputs support,
+ it should be instantiated in the Mapper/Reducer setup method.
+
+ @param context the TaskInputOutputContext object]]>
+      </doc>
+    </constructor>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param job               job to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ 
+ The counters group is the {@link MultipleOutputs} class name.
+ The names of the counters are the same as the named outputs. These
+ counters count the number records written to each output name.
+ By default these counters are disabled.
+
+ @param job    job  to enable counters
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ By default these counters are disabled.
+
+ @param job    the job 
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to the namedOutput.
+
+ Output path is a unique file generated for the namedOutput.
+ For example, {namedOutput}-(m|r)-{part-number}
+ 
+ @param namedOutput the named output name
+ @param key         the key
+ @param value       the value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to baseOutputPath using the namedOutput.
+ 
+ @param namedOutput    the named output name
+ @param key            the key
+ @param value          the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key value to an output file name.
+ 
+ Gets the record writer from job's output format.  
+ Job's output format should be a FileOutputFormat.
+ 
+ @param key       the key
+ @param value     the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Closes all the opened outputs.
+ 
+ This should be called from cleanup method of map/reduce task.
+ If overridden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing output data 
+ to multiple outputs
+ 
+ <p> 
+ Case one: writing to additional outputs other than the job default output.
+
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ </p>
+ 
+ <p>
+ Case two: to write data to different files provided by user
+ </p>
+ 
+ <p>
+ MultipleOutputs supports counters, by default they are disabled. The 
+ counters group is the {@link MultipleOutputs} class name. The names of the 
+ counters are the same as the output name. These count the number records 
+ written to each output name.
+ </p>
+ 
+ Usage pattern for job submission:
+ <pre>
+
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>
+ <p>
+ Usage in Reducer:
+ <pre>
+ &lt;K, V&gt; String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer&lt;WritableComparable, Writable,WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>
+ 
+ <p>
+ When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat,
+ MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat
+ from the old Hadoop API - ie, output can be written from the Reducer to more than one location.
+ </p>
+ 
+ <p>
+ Use <code>MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath)</code> to write key and 
+ value to a path specified by <code>baseOutputPath</code>, with no need to specify a named output.
+ <b>Warning</b>: when the baseOutputPath passed to MultipleOutputs.write
+ is a path that resolves outside of the final job output directory, the
+ directory is created immediately and then persists through subsequent
+ task retries, breaking the concept of output committing:
+ </p>
+ 
+ <pre>
+ private MultipleOutputs&lt;Text, Text&gt; out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs&lt;Text, Text&gt;(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable&lt;Text&gt; values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(&lt;<i>parameter list...</i>&gt;));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+ </pre>
+ 
+ <p>
+ Use your own code in <code>generateFileName()</code> to create a custom path to your results. 
+ '/' characters in <code>baseOutputPath</code> will be translated into directory levels in your file system. 
+ Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. 
+ No call to <code>context.write()</code> is necessary. See example <code>generateFileName()</code> code below. 
+ </p>
+ 
+ <pre>
+ private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+ </pre>
+ 
+ <p>
+ Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000.
+ To prevent this use <code>LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);</code>
+ instead of <code>job.setOutputFormatClass(TextOutputFormat.class);</code> in your Hadoop job configuration.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <class name="PartialFileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter"/>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="cleanUpPartialOutputForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <interface name="PartialOutputCommitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="cleanUpPartialOutputForTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove all previously committed outputs from prior executions of this task.
+ @param context Context for cleaning up previously promoted output.
+ @throws IOException If cleanup fails, then the state of the task my not be
+                     well defined.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for an {@link org.apache.hadoop.mapreduce.OutputCommitter}
+ implementing partial commit of task output, as during preemption.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <class name="PathOutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a task attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a job attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final directory where work will be placed once the job
+ is committed. This may be null, in which case, there is no output
+ path to write data to.
+ @return the path where final output of the job should be placed.]]>
+      </doc>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate: is there an output path?
+ @return true if we have an output path set, else false.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ Warning: there's no guarantee that this work path is on the same
+ FS as the final output, or that it's visible across machines.
+ May be null.
+ @return the work directory
+ @throws IOException IO problem]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A committer which somehow commits data written to a working directory
+ to the final directory during the commit process. The reference
+ implementation of this is the {@link FileOutputCommitter}.
+
+ There are two constructors, both of which do nothing but long and
+ validate their arguments.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes keys, 
+ values to {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSequenceWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.partition">
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOffsets"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="left" type="int"/>
+      <param name="right" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[left:(right+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param left left Python-style offset
+ @param right right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setLeftOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[offset:]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset left Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setRightOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[:(offset+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.BinaryComparable"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Use (the specified slice of the array returned by) 
+ {@link BinaryComparable#getBytes()} to partition.]]>
+      </doc>
+    </method>
+    <field name="LEFT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RIGHT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}.</p>
+ 
+ <p>The subarray to be used for the partitioning can be defined by means
+ of the following properties:
+ <ul>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.left.offset</i>:
+     left offset in array (0 by default)
+   </li>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.right.offset</i>: 
+     right offset in array (-1 by default)
+   </li>
+ </ul>
+ Like in Python, both negative and positive offsets are allowed, but
+ the meaning is slightly different. In case of an array of length 5,
+ for instance, the possible offsets are:
+ <pre><code>
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+ </code></pre>
+ The first row of numbers gives the position of the offsets 0...5 in 
+ the array; the second row gives the corresponding negative offsets. 
+ Contrary to Python, the specified subarray has byte <code>i</code> 
+ and <code>j</code> as first and last element, repectively, when 
+ <code>i</code> and <code>j</code> are the left and right offset.
+ 
+ <p>For Hadoop programs written in Java, it is advisable to use one of 
+ the following static convenience methods for setting the offsets:
+ <ul>
+   <li>{@link #setOffsets}</li>
+   <li>{@link #setLeftOffset}</li>
+   <li>{@link #setRightOffset}</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <class name="HashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="InputSampler" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="sampler" type="org.apache.hadoop.mapreduce.lib.partition.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write a partition file for the given job, using the Sampler provided.
+ Queries the sampler for a sample keyset, sorts by the output key
+ comparator, selects the keys for each rank, and writes to the destination
+ returned from {@link TotalOrderPartitioner#getPartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Driver for InputSampler from the command line.
+ Configures a JobConf instance and calls {@link #writePartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[Utility for collecting samples and writing a partition file for
+ {@link TotalOrderPartitioner}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.io.WritableComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <field name="COMPARATOR_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by 
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="end" type="int"/>
+      <param name="currentHash" type="int"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hash" type="int"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <field name="PARTITIONER_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <class name="RehashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RehashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Rehash {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This partitioner rehashes values returned by {@link Object#hashCode()}
+  to get smoother distribution between partitions which may improve
+  reduce reduce time in some cases and should harm things in no cases.
+  This partitioner is suggested with Integer and Long keys with simple
+  patterns in their distributions.
+  @since 2.0.3]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Read in the partition file and build indexing data structures.
+ If the keytype is {@link org.apache.hadoop.io.BinaryComparable} and
+ <tt>total.order.partitioner.natural.order</tt> is not false, a trie
+ of the first <tt>total.order.partitioner.max.trie.depth</tt>(2) + 1 bytes
+ will be built. Otherwise, keys will be located using a binary search of
+ the partition keyset using the {@link org.apache.hadoop.io.RawComparator}
+ defined for this job. The input file must be sorted with the same
+ comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(Configuration, Path)]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_TRIE_DEPTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NATURAL_ORDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.reduce">
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <class name="IntSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IntSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="Key"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+  <class name="WrappedReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReducerContext" return="org.apache.hadoop.mapreduce.Reducer.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reduceContext" type="org.apache.hadoop.mapreduce.ReduceContext"/>
+      <doc>
+      <![CDATA[A a wrapped {@link Reducer.Context} for custom implementations.
+ @param reduceContext <code>ReduceContext</code> to be wrapped
+ @return a wrapped <code>Reducer.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} which wraps a given one to allow for custom 
+ {@link Reducer.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.server.jobtracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.server.tasktracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.task.annotation">
+  <!-- start class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+  <class name="Checkpointable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Contract representing to the framework that the task can be safely preempted
+ and restarted between invocations of the user-defined function.
+
+ This is often true when the result of a function does not rely on state
+ derived from previous elements in the record stream, but the guarantee is
+ left as an exercise to the implementor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+</package>
+<package name="org.apache.hadoop.mapreduce.tools">
+  <!-- start class org.apache.hadoop.mapreduce.tools.CLI -->
+  <class name="CLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="CLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CLI" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="counters" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTaskLogURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <param name="baseUrl" type="java.lang.String"/>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param job the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws InterruptedException
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="displayJobList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="cluster" type="org.apache.hadoop.mapreduce.Cluster"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="headerPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="dataPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interprets the map reduce cli options]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.tools.CLI -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml
new file mode 100644
index 0000000000000..dc0a88deac948
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml
@@ -0,0 +1,28087 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:14:39 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce Core 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.4.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.4.jar:/maven/com/squareup/okhttp3/okhttp/4.9.3/okhttp-4.9.3.jar:/maven/com/squareup/okio/okio/2.8.0/okio-2.8.0.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib/1.4.10/kotlin-stdlib-1.4.10.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib-common/1.4.10/kotlin-stdlib-common-1.4.10.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/site/jdiff/xml -apiname Apache Hadoop MapReduce Core 3.3.4 -->
+<package name="org.apache.hadoop.filecache">
+  <!-- start class org.apache.hadoop.filecache.DistributedCache -->
+  <class name="DistributedCache" extends="org.apache.hadoop.mapreduce.filecache.DistributedCache"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DistributedCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a archive that has been localized to the conf.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="addLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add a file that has been localized to the conf..  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <method name="createAllSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobCacheDir" type="java.io.File"/>
+      <param name="workDir" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method create symlinks for all files in a given dir in another
+ directory. Currently symlinks cannot be disabled. This is a NO-OP.
+
+ @param conf the configuration
+ @param jobCacheDir the target directory for creating symlinks
+ @param workDir the directory in which the symlinks are created
+ @throws IOException
+ @deprecated Internal to MapReduce framework.  Use DistributedCacheManager
+ instead.]]>
+      </doc>
+    </method>
+    <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns {@link FileStatus} of a given cache file on hdfs. Internal to
+ MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return <code>FileStatus</code> of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="cache" type="java.net.URI"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns mtime of a given cache file on hdfs. Internal to MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return mtime of a given cache file on hdfs
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setArchiveTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the archives to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of archives.
+ The order should be the same as the order in which the archives are added.]]>
+      </doc>
+    </method>
+    <method name="setFileTimestamps"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="timestamps" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is to check the timestamp of the files to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of files.
+ The order should be the same as the order in which the files are added.]]>
+      </doc>
+    </method>
+    <method name="setLocalArchives"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized archives.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+      </doc>
+    </method>
+    <method name="setLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="str" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the conf to contain the location for localized files.  Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+      </doc>
+    </method>
+    <field name="CACHE_FILES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_SIZES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_SIZES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES_TIMESTAMPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}]]>
+      </doc>
+    </field>
+    <field name="CACHE_ARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_ARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_FILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALARCHIVES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALARCHIVES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_LOCALFILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_LOCALFILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALFILES}]]>
+      </doc>
+    </field>
+    <field name="CACHE_SYMLINK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Warning: {@link #CACHE_SYMLINK} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_SYMLINK}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Distribute application-specific large, read-only files efficiently.
+
+ <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+ </p>
+
+ <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ <code>DistributedCache</code> assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.</p>
+
+ <p>The framework will copy the necessary files on to the worker node before
+ any tasks for the job are executed on that node. Its efficiency stems from
+ the fact that the files are only copied once per job and the ability to
+ cache archives which are un-archived on the workers.</p>
+
+ <p><code>DistributedCache</code> can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism.  Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task.  In the current
+ version symlinks are always created.  If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used.  All
+ others will not even be downloaded.</p>
+
+ <p><code>DistributedCache</code> tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.</p>
+
+ <p>Here is an illustrative example on how to use the
+ <code>DistributedCache</code>:</p>
+ <p><blockquote><pre>
+     // Setting up the cache for the application
+
+     1. Copy the requisite files to the <code>FileSystem</code>:
+
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+     2. Setup the application's <code>JobConf</code>:
+
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job);
+
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+
+     public static class MapClass extends MapReduceBase
+     implements Mapper&lt;K, V, K, V&gt; {
+
+       private Path[] localArchives;
+       private Path[] localFiles;
+
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+
+       public void map(K key, V value,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+
+ </pre></blockquote>
+
+ It is also very common to use the DistributedCache by using
+ {@link org.apache.hadoop.util.GenericOptionsParser}.
+
+ This class includes methods that should be used by users
+ (specifically those mentioned in the example above, as well
+ as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
+ as well as methods intended for use by the MapReduce framework
+ (e.g., {@link org.apache.hadoop.mapred.JobClient}).
+
+ @see org.apache.hadoop.mapred.JobConf
+ @see org.apache.hadoop.mapred.JobClient
+ @see org.apache.hadoop.mapreduce.Job]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.filecache.DistributedCache -->
+</package>
+<package name="org.apache.hadoop.mapred">
+  <!-- start class org.apache.hadoop.mapred.ClusterStatus -->
+  <class name="ClusterStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getTaskTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of task trackers in the cluster.
+ 
+ @return the number of task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getActiveTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the active task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of task trackers in the cluster.
+ 
+ @return the blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackerNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the names of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return an empty graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getGraylistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return 0 graylisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getBlacklistedTrackers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted task trackers in the cluster.
+ 
+ @return the number of blacklisted task trackers in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getNumExcludedNodes" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of excluded hosts in the cluster.
+ @return the number of excluded hosts in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getTTExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running map tasks in the cluster.
+ 
+ @return the number of currently running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of currently running reduce tasks in the cluster.
+ 
+ @return the number of currently running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running map tasks in the cluster.
+ 
+ @return the maximum capacity for running map tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capacity for running reduce tasks in the cluster.
+ 
+ @return the maximum capacity for running reduce tasks in the cluster.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker]]>
+      </doc>
+    </method>
+    <method name="getMaxMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getUsedMemory" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTrackersInfo" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the list of blacklisted trackers along with reasons for blacklisting.
+ 
+ @return the collection of {@link BlackListInfo} objects.]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerState" return="org.apache.hadoop.mapred.JobTracker.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current state of the <code>JobTracker</code>,
+ as {@link JobTracker.State}
+
+ {@link JobTracker.State} should no longer be used on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return the invalid state of the <code>JobTracker</code>.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="UNINITIALIZED_MEMORY_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterStatus</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster. 
+   </li>
+   <li>
+   Name of the trackers. 
+   </li>
+   <li>
+   Task capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   State of the <code>JobTracker</code>.
+   </li>
+   <li>
+   Details regarding black listed trackers.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterStatus</code>, via 
+ {@link JobClient#getClusterStatus()}.</p>
+ 
+ @see JobClient]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ClusterStatus -->
+  <!-- start class org.apache.hadoop.mapred.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getGroup" return="org.apache.hadoop.mapred.Counters.Group"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+    </method>
+    <method name="getGroupNames" return="java.util.Collection"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="makeCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String, String)} instead">
+      <param name="group" type="java.lang.String"/>
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter by using strings
+ @param group the name of the group
+ @param id the id of the counter within the group (0 to N-1)
+ @param name the internal name of the counter
+ @return the counter for that name
+ @deprecated use {@link #findCounter(String, String)} instead]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param key identifies a counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param group the name of the group
+ @param counter the internal name of the counter
+ @param amount amount by which counter is to be incremented]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Returns current value of the specified counter, or 0 if the counter
+ does not exist.
+ @param key the counter enum to lookup
+ @return the counter value or 0 if counter not found]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #countCounters()} instead">
+      <doc>
+      <![CDATA[@return the total number of counters
+ @deprecated use {@link #countCounters()} instead]]>
+      </doc>
+    </method>
+    <method name="sum" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="a" type="org.apache.hadoop.mapred.Counters"/>
+      <param name="b" type="org.apache.hadoop.mapred.Counters"/>
+      <doc>
+      <![CDATA[Convenience method for computing the sum of two sets of counters.
+ @param a the first counters
+ @param b the second counters
+ @return a new summed counters object]]>
+      </doc>
+    </method>
+    <method name="log"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="log" type="org.slf4j.Logger"/>
+      <doc>
+      <![CDATA[Logs the current counter values.
+ @param log The log to use.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Represent the counter in a textual format that can be converted back to
+ its object form
+ @return the string in the following format
+ {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*]]>
+      </doc>
+    </method>
+    <method name="fromEscapedCompactString" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compactString" type="java.lang.String"/>
+      <exception name="ParseException" type="java.text.ParseException"/>
+      <doc>
+      <![CDATA[Convert a stringified (by {@link #makeEscapedCompactString()} counter
+ representation into a counter object.
+ @param compactString to parse
+ @return a new counters object
+ @throws ParseException]]>
+      </doc>
+    </method>
+    <field name="MAX_COUNTER_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_GROUP_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A set of named counters.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Counter -->
+  <class name="Counters.Counter" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.Counter"/>
+    <constructor name="Counter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+    </method>
+    <method name="increment"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the compact stringified version of the counter in the format
+ [(actual-name)(display-name)(value)]
+ @return the stringified result]]>
+      </doc>
+    </method>
+    <method name="contentEquals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+      <doc>
+      <![CDATA[Checks for (content) equality of two (basic) counters
+ @param counter to compare
+ @return true if content equals
+ @deprecated]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the value of the counter]]>
+      </doc>
+    </method>
+    <method name="getUnderlyingCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A counter record, comprising its name and value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Counter -->
+  <!-- start class org.apache.hadoop.mapred.Counters.Group -->
+  <class name="Counters.Group" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <constructor name="Group"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param counterName the name of the counter
+ @return the value of the specified counter, or 0 if the counter does
+ not exist.]]>
+      </doc>
+    </method>
+    <method name="makeEscapedCompactString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the compact stringified version of the group in the format
+ {(actual-name)(display-name)(value)[][][]} where [] are compact strings
+ for the counters within.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #findCounter(String)} instead">
+      <param name="id" type="int"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given id and create it if it doesn't exist.
+ @param id the numeric id of the counter within the group
+ @param name the internal counter name
+ @return the counter
+ @deprecated use {@link #findCounter(String)} instead]]>
+      </doc>
+    </method>
+    <method name="getCounterForName" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the counter for the given name and create it if it doesn't exist.
+ @param name the internal counter name
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDisplayName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="addCounter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+    </method>
+    <method name="addCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+    </method>
+    <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    </method>
+    <method name="getUnderlyingGroup" return="org.apache.hadoop.mapreduce.counters.CounterGroupBase"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>Group</code> of counters, comprising of counters from a particular
+  counter {@link Enum} class.
+
+  <p><code>Group</code>handles localization of the class name and the
+  counter names.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Counters.Group -->
+  <!-- start class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <class name="FileAlreadyExistsException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileAlreadyExistsException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileAlreadyExistsException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when target file already exists for any operation and 
+ is not configured to be overwritten.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+  <!-- start class org.apache.hadoop.mapred.FileInputFormat -->
+  <class name="FileInputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setMinSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSize" type="long"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ If security is enabled, this method collects
+ delegation tokens from the input paths and adds them to the job's
+ credentials.
+ @param job the job to list input paths for and attach tokens to.
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Splits files returned by {@link #listStatus(JobConf)} when
+ they're too big.]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="goalSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="blockSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param conf Configuration of the job. 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param conf The configuration of the job 
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getSplitHosts" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+      <param name="splitSize" type="long"/>
+      <param name="clusterMap" type="org.apache.hadoop.net.NetworkTopology"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This function identifies and returns the hosts that contribute 
+ most for a given split. For calculating the contribution, rack
+ locality is treated on par with host locality, so hosts from racks
+ that contribute the most are preferred over hosts on racks that 
+ contribute less
+ @param blkLocations The list of block locations
+ @param offset 
+ @param splitSize 
+ @return an array of hosts that contribute most to this split
+ @throws IOException]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}.
+ 
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(FileSystem, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapred.OutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <param name="outputPath" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="runState" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapred.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param conf the {@link JobConf} to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param conf the {@link JobConf} to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param conf the {@link JobConf} to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param conf the {@link JobConf} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(JobConf)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p><i>Note:</i> The following is valid only if the {@link OutputCommitter}
+  is {@link FileOutputCommitter}. If <code>OutputCommitter</code> is not 
+  a <code>FileOutputCommitter</code>, the task's temporary output
+  directory is same as {@link #getOutputPath(JobConf)} i.e.
+  <tt>${mapreduce.output.fileoutputformat.outputdir}$</tt></p>
+  
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in <tt>${mapreduce.task.output.dir}</tt> during execution 
+ of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the 
+ framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p><i>Note</i>: the value of <tt>${mapreduce.task.output.dir}</tt> during 
+ execution of a particular task-attempt is actually 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}</tt>, and this value is 
+ set by the map-reduce framework. So, just create any side-files in the 
+ path  returned by {@link #getWorkOutputPath(JobConf)} from map/reduce 
+ task to take advantage of this feature.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Helper function to create the task's temporary output directory and 
+ return the path to the task's output file.
+ 
+ @param conf job-configuration
+ @param name temporary task-output filename
+ @return path to the task's temporary output file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getUniqueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a name that is unique for the task.
+
+ <p>The generated name can be used to create custom files from within the
+ different tasks for the job, the names for different tasks will not collide
+ with each other.</p>
+
+ <p>The given name is postfixed with the task type, 'm' for maps, 'r' for
+ reduces and the task partition number. For example, give a name 'test'
+ running on the first map o the job the generated name will be
+ 'test-m-00000'.</p>
+
+ @param conf the configuration for the job.
+ @param name the name to make unique.
+ @return a unique name accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getPathForCustomFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueName} method to make the file name
+ unique for the task.</p>
+
+ @param conf the configuration for the job.
+ @param name the name for the file.
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplitWithLocationInfo"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[Constructs a split.
+ @deprecated
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobConf, int)} and passed to
+ {@link InputFormat#getRecordReader(InputSplit,JobConf,Reporter)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FileSplit -->
+  <!-- start class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.ID -->
+  <class name="ID" extends="org.apache.hadoop.mapreduce.ID"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.ID -->
+  <!-- start interface org.apache.hadoop.mapred.InputFormat -->
+  <interface name="InputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple.
+ 
+ @param job job configuration.
+ @param numSplits the desired number of splits, a hint.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordReader} for the given {@link InputSplit}.
+
+ <p>It is the responsibility of the <code>RecordReader</code> to respect
+ record boundaries while processing the logical split to present a 
+ record-oriented view to the individual task.</p>
+ 
+ @param split the {@link InputSplit}
+ @param job the job that this split belongs to
+ @return a {@link RecordReader}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to be respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibilty to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see JobClient
+ @see FileInputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplit -->
+  <interface name="InputSplit"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the total number of bytes in the data of the <code>InputSplit</code>.
+ 
+ @return the number of bytes in the input split.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the list of hostnames where the input split is located.
+ 
+ @return list of hostnames where data of the <code>InputSplit</code> is
+         located as an array of <code>String</code>s.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplit -->
+  <!-- start interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <interface name="InputSplitWithLocationInfo"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.InputSplitWithLocationInfo -->
+  <!-- start class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <class name="InvalidFileTypeException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidFileTypeException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidFileTypeException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Used when file type differs from the desired file type. like 
+ getting a file when a directory is expected. Or a wrong file type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidFileTypeException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ The first element of the list is used as the init cause value.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <class name="InvalidJobConfException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidJobConfException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="InvalidJobConfException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when jobconf misses some mendatory attributes
+ or value of some attributes is invalid.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.InvalidJobConfException -->
+  <!-- start class org.apache.hadoop.mapred.JobClient -->
+  <class name="JobClient" extends="org.apache.hadoop.mapreduce.tools.CLI"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="JobClient"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job client.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link JobConf}, and connect to the 
+ default cluster
+ 
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client with the given {@link Configuration}, 
+ and connect to the default cluster
+ 
+ @param conf the configuration.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="JobClient" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a job client, connect to the indicated job tracker.
+ 
+ @param jobTrackAddr the job tracker to connect to.
+ @param conf configuration.]]>
+      </doc>
+    </constructor>
+    <method name="init"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Connect to the default cluster
+ @param conf the job configuration.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>JobClient</code>.]]>
+      </doc>
+    </method>
+    <method name="getFs" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a filesystem handle.  We need this to prepare jobs
+ for submission to the MapReduce system.
+ 
+ @return the filesystem handle.]]>
+      </doc>
+    </method>
+    <method name="getClusterHandle" return="org.apache.hadoop.mapreduce.Cluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a handle to the Cluster]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobFile" type="java.lang.String"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ 
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param jobFile the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws InvalidJobConfException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the MR system.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws FileNotFoundException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobInner" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobid" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an {@link RunningJob} object to track an ongoing job.  Returns
+ null if the id does not correspond to any known job.
+
+ @param jobid the jobid of the job.
+ @return the {@link RunningJob} handle to track the job, null if the
+         <code>jobid</code> doesn't correspond to any known job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getJob(JobID)}.">
+      <param name="jobid" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getJob(JobID)}.]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the map tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getMapTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getMapTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getMapTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the reduce tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the reduce tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCleanupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the cleanup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the cleanup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSetupTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the setup tasks of a job.
+ 
+ @param jobId the job to query.
+ @return the list of all of the setup tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getReduceTaskReports" return="org.apache.hadoop.mapred.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #getReduceTaskReports(JobID)}">
+      <param name="jobId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}]]>
+      </doc>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapred.JobID"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param jobId the ID of the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapred.ClusterStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="detailed" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the Map-Reduce cluster.
+  
+ @param  detailed if true then get a detailed status including the
+         tracker names
+ @return the status information about the Map-Reduce cluster as an object
+         of {@link ClusterStatus}.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobsToComplete" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are not completed and not failed.
+ 
+ @return array of {@link JobStatus} for the running/to-be-run jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the jobs that are submitted.
+ 
+ @return array of {@link JobStatus} for the submitted jobs.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Utility that submits a job, then polls for progress until the job is
+ complete.
+ 
+ @param job the job configuration.
+ @throws IOException if the job fails]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="job" type="org.apache.hadoop.mapred.RunningJob"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @param conf the job's configuration
+ @param job the job to track
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Sets the output filter for tasks. only those tasks are printed whose
+ output matches the filter. 
+ @param newValue task filter.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the task output filter out of the JobConf.
+ 
+ @param job the JobConf to examine.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="newValue" type="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the JobConf to set the task output filter.
+ 
+ @param job the JobConf to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapred.JobClient.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task output filter.
+ @return task filter.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cntrs" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getDefaultMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Maps in the cluster.
+  
+ @return the max available Maps in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDefaultReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get status information about the max available Reduces in the cluster.
+  
+ @return the max available Reduces in the cluster
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where job-specific files are to be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="isJobDirValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobDirPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Checks if the job directory is clean and has all the required components
+ for (re) starting the job]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fetch the staging area directory for the application
+ 
+ @return path to staging area directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about root level queues
+ configured
+
+ @return the array of root level JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns an array of queue information objects about immediate children
+ of queue queueName.
+ 
+ @param queueName
+ @return the array of immediate children JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapred.JobQueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return an array of queue information objects about all the Job Queues
+ configured.
+ 
+ @return Array of JobQueueInfo objects
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobsFromQueue" return="org.apache.hadoop.mapred.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets all the jobs which were added to particular Job Queue
+ 
+ @param queueName name of the Job Queue
+ @return Array of jobs present in the job queue
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.mapred.JobQueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the queue information associated to a particular Job Queue
+ 
+ @param queueName name of the job queue.
+ @return Queue information associated to particular queue.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapred.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return true if the renewal went well
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>JobClient</code> is the primary interface for the user-job to interact
+ with the cluster.
+ 
+ <code>JobClient</code> provides facilities to submit jobs, track their 
+ progress, access component-tasks' reports/logs, get the Map-Reduce cluster
+ status information etc.
+ 
+ <p>The job submission process involves:
+ <ol>
+   <li>
+   Checking the input and output specifications of the job.
+   </li>
+   <li>
+   Computing the {@link InputSplit}s for the job.
+   </li>
+   <li>
+   Setup the requisite accounting information for the {@link DistributedCache} 
+   of the job, if necessary.
+   </li>
+   <li>
+   Copying the job's jar and configuration to the map-reduce system directory 
+   on the distributed file-system. 
+   </li>
+   <li>
+   Submitting the job to the cluster and optionally monitoring
+   it's status.
+   </li>
+ </ol>
+  
+ Normally the user creates the application, describes various facets of the
+ job via {@link JobConf} and then uses the <code>JobClient</code> to submit 
+ the job and monitor its progress.
+ 
+ <p>Here is an example on how to use <code>JobClient</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+ </pre></blockquote>
+ 
+ <b id="JobControl">Job Control</b>
+ 
+ <p>At times clients would chain map-reduce jobs to accomplish complex tasks 
+ which cannot be done via a single map-reduce job. This is fairly easy since 
+ the output of the job, typically, goes to distributed file-system and that 
+ can be used as the input for the next job.</p>
+ 
+ <p>However, this also means that the onus on ensuring jobs are complete 
+ (success/failure) lies squarely on the clients. In such situations the 
+ various job-control options are:
+ <ol>
+   <li>
+   {@link #runJob(JobConf)} : submits the job and returns only after 
+   the job has completed.
+   </li>
+   <li>
+   {@link #submitJob(JobConf)} : only submits the job, then poll the 
+   returned handle to the {@link RunningJob} to query status and make 
+   scheduling decisions.
+   </li>
+   <li>
+   {@link JobConf#setJobEndNotificationURI(String)} : setup a notification
+   on job-completion, thus avoiding polling.
+   </li>
+ </ol>
+ 
+ @see JobConf
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobClient -->
+  <!-- start class org.apache.hadoop.mapred.JobConf -->
+  <class name="JobConf" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.conf.Configuration, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce job configuration.
+ 
+ @param conf a Configuration whose settings will be inherited.
+ @param exampleClass a class whose containing jar is used as the job's jar.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a map/reduce configuration.
+
+ @param config a Configuration-format XML job description file.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobConf" type="boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A new map/reduce configuration where the behavior of reading from the
+ default resources can be turned off.
+ <p>
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+
+ @param loadDefaults specifies whether to load from the default files]]>
+      </doc>
+    </constructor>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user jar for the map-reduce job.
+ 
+ @return the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user jar for the map-reduce job.
+ 
+ @param jar the user jar for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getJarUnpackPattern" return="java.util.regex.Pattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pattern for jar contents to unpack on the tasktracker]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the job's jar file by finding an example class location.
+ 
+ @param cls the example class.]]>
+      </doc>
+    </method>
+    <method name="getLocalDirs" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Use MRAsyncDiskService.moveAndDeleteAllVolumes instead.]]>
+      </doc>
+    </method>
+    <method name="deleteLocalFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="subdir" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathString" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructs a local file name. Files are distributed among configured
+ local directories.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setKeepFailedTaskFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should keep the intermediate files for 
+ failed tasks.
+ 
+ @param keep <code>true</code> if framework should keep the intermediate files 
+             for failed tasks, <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="getKeepFailedTaskFiles" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the temporary files for failed tasks be kept?
+ 
+ @return should the files be kept?]]>
+      </doc>
+    </method>
+    <method name="setKeepTaskFilesPattern"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set a regular expression for task names that should be kept. 
+ The regular expression ".*_m_000123_0" would keep the files
+ for the first instance of map 123 that ran.
+ 
+ @param pattern the java.util.regex.Pattern to match against the 
+        task names.]]>
+      </doc>
+    </method>
+    <method name="getKeepTaskFilesPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the regular expression that is matched against the task names
+ to see if we need to keep the files.
+ 
+ @return the pattern as a string, if it was set, othewise null.]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="setNumTasksToExecutePerJvm"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numTasks" type="int"/>
+      <doc>
+      <![CDATA[Sets the number of tasks that a spawned task JVM should run
+ before it exits
+ @param numTasks the number of tasks to execute; defaults to 1;
+ -1 signifies no limit]]>
+      </doc>
+    </method>
+    <method name="getNumTasksToExecutePerJvm" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of tasks that a spawned JVM should execute]]>
+      </doc>
+    </method>
+    <method name="getInputFormat" return="org.apache.hadoop.mapred.InputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link InputFormat} implementation for the map-reduce job,
+ defaults to {@link TextInputFormat} if not specified explicity.
+ 
+ @return the {@link InputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link InputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormat" return="org.apache.hadoop.mapred.OutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} implementation for the map-reduce job,
+ defaults to {@link TextOutputFormat} if not specified explicity.
+ 
+ @return the {@link OutputFormat} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapred.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} implementation for the map-reduce job,
+ defaults to {@link FileOutputCommitter} if not specified explicitly.
+ 
+ @return the {@link OutputCommitter} implementation for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputCommitter} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputCommitter} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setOutputFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} implementation for the map-reduce job.
+ 
+ @param theClass the {@link OutputFormat} implementation for the map-reduce 
+                 job.]]>
+      </doc>
+    </method>
+    <method name="setCompressMapOutput"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Should the map outputs be compressed before transfer?
+ 
+ @param compress should the map outputs be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressMapOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are the outputs of the maps be compressed?
+ 
+ @return <code>true</code> if the outputs of the maps are to be compressed,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the given class as the  {@link CompressionCodec} for the map outputs.
+ 
+ @param codecClass the {@link CompressionCodec} class that will compress  
+                   the map outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the map outputs.
+ 
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} class that should be used to compress the 
+         map outputs.
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+  
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ 
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link RawComparator} comparator used to compare keys.
+ 
+ @param theClass the {@link RawComparator} comparator used to 
+                 compare keys.
+ @see #setOutputValueGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see #setCombinerKeyGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link WritableComparable} comparator for 
+ grouping keys of inputs to the reduce.
+ 
+ @return comparator set by the user for grouping values.
+ @see #setOutputValueGroupingComparator(Class) for details.]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for
+ grouping keys in the input to the combiner.
+
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control
+ how keys are sorted, this can be used in conjunction to simulate
+ <i>secondary sort on values</i>.</p>
+
+ <p><i>Note</i>: This is not a guarantee of the combiner sort being
+ <i>stable</i> in any sense. (In any case, with the order of available
+ map-outputs to the combiner being non-deterministic, it wouldn't make
+ that much sense.)</p>
+
+ @param theClass the comparator class to be used for grouping keys for the
+ combiner. It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setOutputValueGroupingComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user defined {@link RawComparator} comparator for 
+ grouping keys in the input to the reduce.
+ 
+ <p>This comparator should be provided if the equivalence rules for keys
+ for sorting the intermediates are different from those for grouping keys
+ before each call to 
+ {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.</p>
+  
+ <p>For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed
+ in a single call to the reduce function if K1 and K2 compare as equal.</p>
+ 
+ <p>Since {@link #setOutputKeyComparatorClass(Class)} can be used to control 
+ how keys are sorted, this can be used in conjunction to simulate 
+ <i>secondary sort on values</i>.</p>
+  
+ <p><i>Note</i>: This is not a guarantee of the reduce sort being 
+ <i>stable</i> in any sense. (In any case, with the order of available 
+ map-outputs to the reduce being non-deterministic, it wouldn't make 
+ that much sense.)</p>
+ 
+ @param theClass the comparator class to be used for grouping keys. 
+                 It should implement <code>RawComparator</code>.
+ @see #setOutputKeyComparatorClass(Class)
+ @see #setCombinerKeyGroupingComparator(Class)]]>
+      </doc>
+    </method>
+    <method name="getUseNewMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the mapper?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewMapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the mapper.
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getUseNewReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should the framework use the new context-object code for running
+ the reducer?
+ @return true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="setUseNewReducer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flag" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the framework should use the new api for the reducer. 
+ This is the default for jobs submitted with the new Job api.
+ @param flag true, if the new api should be used]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ 
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} class for the job.
+ 
+ @param theClass the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapRunnerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link MapRunnable} class for the job.
+ 
+ @return the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setMapRunnerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Expert: Set the {@link MapRunnable} class for the job.
+ 
+ Typically used to exert greater control on {@link Mapper}s.
+ 
+ @param theClass the {@link MapRunnable} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Partitioner} used to partition {@link Mapper}-outputs 
+ to be sent to the {@link Reducer}s.
+ 
+ @return the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} class used to partition 
+ {@link Mapper}-outputs to be sent to the {@link Reducer}s.
+ 
+ @param theClass the {@link Partitioner} used to partition map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} class for the job.
+ 
+ @param theClass the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. Typically the combiner is same as the
+ the {@link Reducer} for the job i.e. {@link #getReducerClass()}.
+ 
+ @return the user-defined combiner class used to combine map-outputs.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the user-defined <i>combiner</i> class used to combine map-outputs 
+ before being sent to the reducers. 
+ 
+ <p>The combiner is an application-specified aggregation operation, which
+ can help cut down the amount of data transferred between the 
+ {@link Mapper} and the {@link Reducer}, leading to better performance.</p>
+ 
+ <p>The framework may invoke the combiner 0, 1, or multiple times, in both
+ the mapper and reducer tasks. In general, the combiner is called as the
+ sort/merge result is written to disk. The combiner must:
+ <ul>
+   <li> be side-effect free</li>
+   <li> have the same input and output key types and the same input and 
+        output value types</li>
+ </ul>
+ 
+ <p>Typically the combiner is same as the <code>Reducer</code> for the  
+ job i.e. {@link #setReducerClass(Class)}.</p>
+ 
+ @param theClass the user-defined combiner class used to combine 
+                 map-outputs.]]>
+      </doc>
+    </method>
+    <method name="getSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getMapSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for map tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be 
+                           used for this job for map tasks,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getReduceSpeculativeExecution" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Should speculative execution be used for this job for reduce tasks? 
+ Defaults to <code>true</code>.
+ 
+ @return <code>true</code> if speculative execution be used 
+                           for reduce tasks for this job,
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="getNumMapTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of map tasks for this job.
+ Defaults to <code>1</code>.
+ 
+ @return the number of map tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumMapTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the number of map tasks for this job.
+ 
+ <p><i>Note</i>: This is only a <i>hint</i> to the framework. The actual 
+ number of spawned map tasks depends on the number of {@link InputSplit}s 
+ generated by the job's {@link InputFormat#getSplits(JobConf, int)}.
+  
+ A custom {@link InputFormat} is typically used to accurately control 
+ the number of map tasks for the job.</p>
+ 
+ <b id="NoOfMaps">How many maps?</b>
+ 
+ <p>The number of maps is usually driven by the total size of the inputs 
+ i.e. total number of blocks of the input files.</p>
+  
+ <p>The right level of parallelism for maps seems to be around 10-100 maps 
+ per-node, although it has been set up to 300 or so for very cpu-light map 
+ tasks. Task setup takes awhile, so it is best if the maps take at least a 
+ minute to execute.</p>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of input files. However, the {@link FileSystem} blocksize of the 
+ input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+  
+ <p>Thus, if you expect 10TB of input data and have a blocksize of 128MB, 
+ you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is 
+ used to set it even higher.</p>
+ 
+ @param n the number of map tasks for this job.
+ @see InputFormat#getSplits(JobConf, int)
+ @see FileInputFormat
+ @see FileSystem#getDefaultBlockSize()
+ @see FileStatus#getBlockSize()]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of reduce tasks for this job. Defaults to
+ <code>1</code>.
+
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Set the requisite number of reduce tasks for this job.
+ 
+ <b id="NoOfReduces">How many reduces?</b>
+ 
+ <p>The right number of reduces seems to be <code>0.95</code> or 
+ <code>1.75</code> multiplied by (
+ <i>available memory for reduce tasks</i>
+ (The value of this should be smaller than
+ numNodes * yarn.nodemanager.resource.memory-mb
+ since the resource of memory is shared by map tasks and other
+ applications) /
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.reduce.memory.mb">
+ mapreduce.reduce.memory.mb</a>).
+ </p>
+ 
+ <p>With <code>0.95</code> all of the reduces can launch immediately and 
+ start transfering map outputs as the maps finish. With <code>1.75</code> 
+ the faster nodes will finish their first round of reduces and launch a 
+ second wave of reduces doing a much better job of load balancing.</p>
+ 
+ <p>Increasing the number of reduces increases the framework overhead, but 
+ increases load balancing and lowers the cost of failures.</p>
+ 
+ <p>The scaling factors above are slightly less than whole numbers to 
+ reserve a few reduce slots in the framework for speculative-tasks, failures
+ etc.</p> 
+
+ <b id="ReducerNone">Reducer NONE</b>
+ 
+ <p>It is legal to set the number of reduce-tasks to <code>zero</code>.</p>
+ 
+ <p>In this case the output of the map-tasks directly go to distributed 
+ file-system, to the path set by 
+ {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the 
+ framework doesn't sort the map-outputs before writing it out to HDFS.</p>
+ 
+ @param n the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapreduce.map.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapreduce.reduce.maxattempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.]]>
+      </doc>
+    </method>
+    <method name="getSessionId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified session identifier. The default is the empty string.
+
+ The session identifier is used to tag metric data that is reported to some
+ performance metrics system via the org.apache.hadoop.metrics API.  The 
+ session identifier is intended, in particular, for use by Hadoop-On-Demand 
+ (HOD) which allocates a virtual Hadoop cluster dynamically and transiently. 
+ HOD will set the session identifier by modifying the mapred-site.xml file 
+ before starting the cluster.
+
+ When not running under HOD, this identifer is expected to remain set to 
+ the empty string.
+
+ @return the session identifier, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="setSessionId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sessionId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-specified session identifier.  
+
+ @param sessionId the new session id.]]>
+      </doc>
+    </method>
+    <method name="setMaxTaskFailuresPerTracker"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="noFailures" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds <code>noFailures</code>, the 
+ tasktracker is <i>blacklisted</i> for this job. 
+ 
+ @param noFailures maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxTaskFailuresPerTracker" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Expert: Get the maximum no. of failures of a given job per tasktracker.
+ If the no. of task failures exceeds this, the tasktracker is
+ <i>blacklisted</i> for this job. 
+ 
+ @return the maximum no. of failures of a given job per tasktracker.]]>
+      </doc>
+    </method>
+    <method name="getMaxMapTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of map tasks that can fail without 
+ the job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts()} 
+ attempts before being declared as <i>failed</i>.
+  
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed map-task results in
+ the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of map tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the maximum percentage of map tasks that can fail without the
+ job being aborted. 
+ 
+ Each map task is executed a minimum of {@link #getMaxMapAttempts} attempts 
+ before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of map tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceTaskFailuresPercent" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum percentage of reduce tasks that can fail without 
+ the job being aborted. 
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ Defaults to <code>zero</code>, i.e. <i>any</i> failed reduce-task results 
+ in the job being declared as {@link JobStatus#FAILED}.
+ 
+ @return the maximum percentage of reduce tasks that can fail without
+         the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceTaskFailuresPercent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="percent" type="int"/>
+      <doc>
+      <![CDATA[Set the maximum percentage of reduce tasks that can fail without the job
+ being aborted.
+ 
+ Each reduce task is executed a minimum of {@link #getMaxReduceAttempts()} 
+ attempts before being declared as <i>failed</i>.
+ 
+ @param percent the maximum percentage of reduce tasks that can fail without 
+                the job being aborted.]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="setJobPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="prio" type="int"/>
+      <doc>
+      <![CDATA[Set {@link JobPriority} for this job.
+
+ @param prio the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link JobPriority} for this job.
+
+ @return the {@link JobPriority} for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobPriorityAsInteger" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this job.
+
+ @return the priority for this job.]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setMapDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the map tasks fail.
+ 
+ <p>The debug script can aid debugging of failed map tasks. The script is 
+ given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script needs to be symlinked. </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param mDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getMapDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the map task's debug script.
+ 
+ @return the debug Script for the mapred job for failed map tasks.
+ @see #setMapDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="setReduceDebugScript"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rDbgScript" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the debug script to run when the reduce tasks fail.
+ 
+ <p>The debug script can aid debugging of failed reduce tasks. The script
+ is given task's stdout, stderr, syslog, jobconf files as arguments.</p>
+ 
+ <p>The debug command, run on the node where the map failed, is:</p>
+ <p><blockquote><pre>
+ $script $stdout $stderr $syslog $jobconf.
+ </pre></blockquote>
+ 
+ <p> The script file is distributed through {@link DistributedCache} 
+ APIs. The script file needs to be symlinked </p>
+ 
+ <p>Here is an example on how to submit a script 
+ <p><blockquote><pre>
+ job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ </pre></blockquote>
+ 
+ @param rDbgScript the script name]]>
+      </doc>
+    </method>
+    <method name="getReduceDebugScript" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reduce task's debug Script
+ 
+ @return the debug script for the mapred job for failed reduce tasks.
+ @see #setReduceDebugScript(String)]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationURI" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the uri to be invoked in-order to send a notification after the job 
+ has completed (success/failure). 
+ 
+ @return the job end notification uri, <code>null</code> if it hasn't
+         been set.
+ @see #setJobEndNotificationURI(String)]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationURI"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the uri to be invoked in-order to send a notification after the job
+ has completed (success/failure).
+ 
+ <p>The uri can contain 2 special parameters: <tt>$jobId</tt> and 
+ <tt>$jobStatus</tt>. Those, if present, are replaced by the job's 
+ identifier and completion-status respectively.</p>
+ 
+ <p>This is typically used by application-writers to implement chaining of 
+ Map-Reduce jobs in an <i>asynchronous manner</i>.</p>
+ 
+ @param uri the job end notification uri
+ @see JobStatus]]>
+      </doc>
+    </method>
+    <method name="getJobEndNotificationCustomNotifierClass" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the class to be invoked in order to send a notification
+ after the job has completed (success/failure).
+
+ @return the fully-qualified name of the class which implements
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier} set through the
+ {@link org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS}
+ property
+
+ @see JobConf#setJobEndNotificationCustomNotifierClass(java.lang.String)
+ @see org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="setJobEndNotificationCustomNotifierClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="customNotifierClassName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the class to be invoked in order to send a notification after the job
+ has completed (success/failure).
+
+ A notification url still has to be set which will be passed to
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier#notifyOnce(
+ java.net.URL, org.apache.hadoop.conf.Configuration)}
+ along with the Job's conf.
+
+ If this is set instead of using a simple HttpURLConnection
+ we'll create a new instance of this class
+ which should be an implementation of
+ {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier},
+ and we'll invoke that.
+
+ @param customNotifierClassName the fully-qualified name of the class
+     which implements
+     {@link org.apache.hadoop.mapreduce.CustomJobEndNotifier}
+
+ @see JobConf#setJobEndNotificationURI(java.lang.String)
+ @see
+ org.apache.hadoop.mapreduce.MRJobConfig#MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS]]>
+      </doc>
+    </method>
+    <method name="getJobLocalDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get job-specific shared directory for use as scratch space
+ 
+ <p>
+ When a job starts, a shared directory is created at location
+ <code>
+ ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ </code>.
+ This directory is exposed to the users through 
+ <code>mapreduce.job.local.dir </code>.
+ So, the tasks can use this space 
+ as scratch space and share files among them. </p>
+ This value is available as System property also.
+ 
+ @return The localized job specific shared directory]]>
+      </doc>
+    </method>
+    <method name="getMemoryForMapTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a map task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_MAP_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a map task of the job, in MB,]]>
+      </doc>
+    </method>
+    <method name="setMemoryForMapTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getMemoryForReduceTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get memory required to run a reduce task of the job, in MB.
+ 
+ If a value is specified in the configuration, it is returned.
+ Else, it returns {@link JobContext#DEFAULT_REDUCE_MEMORY_MB}.
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ after converting it from bytes to MB.
+ @return memory required to run a reduce task of the job, in MB.]]>
+      </doc>
+    </method>
+    <method name="setMemoryForReduceTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the name of the queue to which this job is submitted.
+ Defaults to 'default'.
+ 
+ @return name of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the queue to which this job should be submitted.
+ 
+ @param queueName Name of the queue]]>
+      </doc>
+    </method>
+    <method name="normalizeMemoryConfigValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[Normalize the negative values in configuration
+ 
+ @param val
+ @return normalized value]]>
+      </doc>
+    </method>
+    <method name="findContainingJar" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="my_class" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Find a jar that contains a class of the same name, if any.
+ It will return a jar file, even if that is not the first thing
+ on the class path that has a class with the same name.
+ 
+ @param my_class the class to find.
+ @return a jar file that contains the class, or null.]]>
+      </doc>
+    </method>
+    <method name="getMaxVirtualMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}">
+      <doc>
+      <![CDATA[Get the memory required to run a task of this job, in bytes. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ This method is deprecated. Now, different memory limits can be
+ set for map and reduce tasks of a job, in MB. 
+ <p>
+ For backward compatibility, if the job configuration sets the
+ key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. 
+ Otherwise, this method will return the larger of the values returned by 
+ {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()}
+ after converting them into bytes.
+
+ @return Memory required to run a task of this job, in bytes.
+ @see #setMaxVirtualMemoryForTask(long)
+ @deprecated Use {@link #getMemoryForMapTask()} and
+             {@link #getMemoryForReduceTask()}]]>
+      </doc>
+    </method>
+    <method name="setMaxVirtualMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}">
+      <param name="vmem" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum amount of memory any task of this job can use. See
+ {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ <p>
+ mapred.task.maxvmem is split into
+ mapreduce.map.memory.mb
+ and mapreduce.map.memory.mb,mapred
+ each of the new key are set
+ as mapred.task.maxvmem / 1024
+ as new values are in MB
+
+ @param vmem Maximum amount of virtual memory in bytes any task of this job
+             can use.
+ @see #getMaxVirtualMemoryForTask()
+ @deprecated
+  Use {@link #setMemoryForMapTask(long mem)}  and
+  Use {@link #setMemoryForReduceTask(long mem)}]]>
+      </doc>
+    </method>
+    <method name="getMaxPhysicalMemoryForTask" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="this variable is deprecated and nolonger in use.">
+      <doc>
+      <![CDATA[@deprecated this variable is deprecated and nolonger in use.]]>
+      </doc>
+    </method>
+    <method name="setMaxPhysicalMemoryForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mem" type="long"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="MAPRED_TASK_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UPPER_LIMIT_ON_TASK_VMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_MAXPMEM_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </field>
+    <field name="DISABLED_MEMORY_LIMIT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A value which if set for memory related configuration options,
+ indicates that the options are turned off.
+ Deprecated because it makes no sense in the context of MR2.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_LOCAL_DIR_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Property name for the configuration property mapreduce.cluster.local.dir]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of the queue to which jobs will be submitted, if no queue
+ name is mentioned.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_MAP_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}]]>
+      </doc>
+    </field>
+    <field name="UNPACK_JAR_PATTERN_DEFAULT" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Pattern for the default unpacking behavior for job jars]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the child
+ map and reduce tasks.
+ 
+ Java opts for the task tracker child processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass 
+ other environment variables to the child processes.
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_JAVA_OPTS} or 
+                 {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the map tasks.
+ 
+ Java opts for the task tracker child map processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass 
+ other environment variables to the map processes.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the java command line options for the reduce tasks.
+ 
+ Java opts for the task tracker child reduce processes.
+ The following symbol, if present, will be interpolated: @taskid@. 
+ It is replaced by current TaskID. Any other occurrences of '@' will go 
+ unchanged.
+ For example, to enable verbose gc logging to a file named for the taskid in
+ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
+          -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ 
+ The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to 
+ pass process environment variables to the reduce processes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPRED_TASK_JAVA_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAPRED_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the child
+ map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ map tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ULIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.">
+      <doc>
+      <![CDATA[@deprecated
+ Configuration key to set the maximum virtual memory available to the
+ reduce tasks (in kilo-bytes). This has been deprecated and will no
+ longer have any effect.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map/reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+ 
+ @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or 
+                 {@link #MAPRED_REDUCE_TASK_ENV}]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child map tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.map.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the environment of the child reduce tasks.
+ 
+ The format of the value is <code>k1=v1,k2=v2</code>. Further it can 
+ reference existing environment variables via <code>$key</code> on
+ Linux or <code>%key%</code> on Windows.
+ 
+ Example:
+ <ul>
+   <li> A=foo - This will set the env variable A to foo. </li>
+ </ul>
+
+ You can also add environment variables individually by appending
+ <code>.VARNAME</code> to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ <ul>
+   <li>mapreduce.reduce.env.VARNAME=value</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="MAPRED_MAP_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the map task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="MAPRED_REDUCE_TASK_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set the logging level for the reduce task.
+
+ The allowed logging levels are:
+ OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default logging level for map/reduce tasks.]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ID} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_STRING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_ADJACENCY_PREFIX_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead]]>
+      </doc>
+    </field>
+    <field name="WORKFLOW_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#WORKFLOW_TAGS} instead]]>
+      </doc>
+    </field>
+    <field name="MAPREDUCE_RECOVER_JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_MAPREDUCE_RECOVER_JOB" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ not use it]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A map/reduce job configuration.
+ 
+ <p><code>JobConf</code> is the primary interface for a user to describe a 
+ map-reduce job to the Hadoop framework for execution. The framework tries to
+ faithfully execute the job as-is described by <code>JobConf</code>, however:
+ <ol>
+   <li>
+   Some configuration parameters might have been marked as 
+   <a href="{@docRoot}/org/apache/hadoop/conf/Configuration.html#FinalParams">
+   final</a> by administrators and hence cannot be altered.
+   </li>
+   <li>
+   While some job parameters are straight-forward to set 
+   (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly
+   with the rest of the framework and/or job-configuration and is relatively
+   more complex for the user to control finely
+   (e.g. {@link #setNumMapTasks(int)}).
+   </li>
+ </ol>
+ 
+ <p><code>JobConf</code> typically specifies the {@link Mapper}, combiner 
+ (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and 
+ {@link OutputFormat} implementations to be used etc.
+
+ <p>Optionally <code>JobConf</code> is used to specify other advanced facets 
+ of the job such as <code>Comparator</code>s to be used, files to be put in  
+ the {@link DistributedCache}, whether or not intermediate and/or job outputs 
+ are to be compressed (and how), debugability via user-provided scripts 
+ ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}),
+ for doing post-processing on task logs, task's stdout, stderr, syslog. 
+ and etc.</p>
+ 
+ <p>Here is an example on how to configure a job via <code>JobConf</code>:</p>
+ <p><blockquote><pre>
+     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+ </pre></blockquote>
+ 
+ @see JobClient
+ @see ClusterStatus
+ @see Tool
+ @see DistributedCache]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobConf -->
+  <!-- start interface org.apache.hadoop.mapred.JobConfigurable -->
+  <interface name="JobConfigurable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Initializes a new instance from a {@link JobConf}.
+
+ @param job the configuration]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[That what may be configured.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobConfigurable -->
+  <!-- start interface org.apache.hadoop.mapred.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job Configuration
+ 
+ @return JobConf]]>
+      </doc>
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress mechanism for reporting progress.
+ 
+ @return progress mechanism]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.JobContext -->
+  <!-- start class org.apache.hadoop.mapred.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapreduce.JobID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.JobID"/>
+      <doc>
+      <![CDATA[Downgrade a new JobID to an old one
+ @param old a new or old JobID
+ @return either old or a new JobID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getJobIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>any job</i> 
+ run on the jobtracker started at <i>200707121733</i>, we would use :
+ <pre> 
+ JobID.getTaskIDsPattern("200707121733", null);
+ </pre>
+ which will return :
+ <pre> "job_200707121733_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @return a regex pattern matching JobIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local".
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobID -->
+  <!-- start class org.apache.hadoop.mapred.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobPriority -->
+  <!-- start class org.apache.hadoop.mapred.JobQueueInfo -->
+  <class name="JobQueueInfo" extends="org.apache.hadoop.mapreduce.QueueInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobQueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Job Queue Info.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobQueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new JobQueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <method name="getQueueState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use getState() instead]]>
+      </doc>
+    </method>
+    <method name="getChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobQueueInfo -->
+  <!-- start class org.apache.hadoop.mapred.JobStatus -->
+  <class name="JobStatus" extends="org.apache.hadoop.mapreduce.JobStatus"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on cleanup
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, int, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file. 
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapred.JobID, float, float, float, float, int, org.apache.hadoop.mapred.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue job queue name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="getJobRunState" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[Helper method to get human-readable state of the job.
+ @param state job state
+ @return human-readable state of the job]]>
+      </doc>
+    </method>
+    <method name="downgrade" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stat" type="org.apache.hadoop.mapreduce.JobStatus"/>
+    </method>
+    <method name="getJobId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use getJobID instead">
+      <doc>
+      <![CDATA[@deprecated use getJobID instead]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getJobPriority" return="org.apache.hadoop.mapred.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="getRunState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+    </method>
+    <method name="setJobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapred.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCEEDED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KILLED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Describes the current status of a job.  This is
+ not intended to be a comprehensive piece of data.
+ For that, look at JobProfile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.JobStatus -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line. Each line
+ is divided into key and value parts by a separator byte. If no such a byte
+ exists, the key will be the entire line and value will be empty.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapred.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapFileOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.Mapper -->
+  <interface name="Mapper"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Maps a single input key/value pair into an intermediate key/value pair.
+ 
+ <p>Output pairs need not be of the same types as input pairs.  A given 
+ input pair may map to zero or many output pairs.  Output pairs are 
+ collected with calls to 
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes significant amount of time to process individual key/value
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the input key.
+ @param value the input value.
+ @param output collects mapped keys and values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link JobConf} for the 
+ job via the {@link JobConfigurable#configure(JobConf)} and initialize
+ themselves. Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+ 
+ <p>The framework then calls 
+ {@link #map(Object, Object, OutputCollector, Reporter)} 
+ for each key/value pair in the <code>InputSplit</code> for that task.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the grouping by specifying
+ a <code>Comparator</code> via 
+ {@link JobConf#setOutputKeyComparatorClass(Class)}.</p>
+
+ <p>The grouped <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>The intermediate, grouped outputs are always stored in 
+ {@link SequenceFile}s. Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>JobConf</code>.</p>
+  
+ <p>If the job has 
+ <a href="{@docRoot}/org/apache/hadoop/mapred/JobConf.html#ReducerNone">zero
+ reduces</a> then the output of the <code>Mapper</code> is directly written
+ to the {@link FileSystem} without grouping by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyMapper&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Mapper&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector&lt;K, V&gt; output, Reporter reporter)
+       throws IOException {
+         // Process the &lt;key, value&gt; pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of &lt;key, value&gt; pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+ </pre></blockquote>
+
+ <p>Applications may write a custom {@link MapRunnable} to exert greater
+ control on map processing e.g. multi-threaded <code>Mapper</code>s etc.</p>
+ 
+ @see JobConf
+ @see InputFormat
+ @see Partitioner  
+ @see Reducer
+ @see MapReduceBase
+ @see MapRunnable
+ @see SequenceFile]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Mapper -->
+  <!-- start class org.apache.hadoop.mapred.MapReduceBase -->
+  <class name="MapReduceBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="MapReduceBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Default implementation that does nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for {@link Mapper} and {@link Reducer} implementations.
+ 
+ <p>Provides default no-op implementations for a few methods, most non-trivial
+ applications need to override some of them.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapReduceBase -->
+  <!-- start interface org.apache.hadoop.mapred.MapRunnable -->
+  <interface name="MapRunnable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="run"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Start mapping input <tt>&lt;key, value&gt;</tt> pairs.
+  
+ <p>Mapping of input records to output records is complete when this method 
+ returns.</p>
+ 
+ @param input the {@link RecordReader} to read the input records.
+ @param output the {@link OutputCollector} to collect the outputrecords.
+ @param reporter {@link Reporter} to report progress, status-updates etc.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Expert: Generic interface for {@link Mapper}s.
+ 
+ <p>Custom implementations of <code>MapRunnable</code> can exert greater 
+ control on map processing e.g. multi-threaded, asynchronous mappers etc.</p>
+ 
+ @see Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.MapRunnable -->
+  <!-- start class org.apache.hadoop.mapred.MapRunner -->
+  <class name="MapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMapper" return="org.apache.hadoop.mapred.Mapper"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Default {@link MapRunnable} implementation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MapRunner -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <class name="MultiFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link MultiFileSplit}'s
+ in {@link #getSplits(JobConf, int)} method. Splits are constructed from 
+ the files under the input paths. Each split returned contains <i>nearly</i>
+ equal content length. <br>  
+ Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>MultiFileSplit</code>'s.
+ @see MultiFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.MultiFileSplit -->
+  <class name="MultiFileSplit" extends="org.apache.hadoop.mapred.lib.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. Unlike {@link FileSplit}, MultiFileSplit 
+ class does not represent a split of a file, but a split of input files 
+ into smaller sets. The atomic unit of split is a file. <br> 
+ MultiFileSplit can be used to implement {@link RecordReader}'s, with 
+ reading one record per file.
+ @see FileSplit
+ @see MultiFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.MultiFileSplit -->
+  <!-- start interface org.apache.hadoop.mapred.OutputCollector -->
+  <interface name="OutputCollector"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="collect"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a key/value pair to the output.
+
+ @param key the key to collect.
+ @param value to value to collect.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Collects the <code>&lt;key, value&gt;</code> pairs output by {@link Mapper}s
+ and {@link Reducer}s.
+  
+ <p><code>OutputCollector</code> is the generalization of the facility 
+ provided by the Map-Reduce framework to collect data output by either the 
+ <code>Mapper</code> or the <code>Reducer</code> i.e. intermediate outputs 
+ or the output of the job.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputCollector -->
+  <!-- start class org.apache.hadoop.mapred.OutputCommitter -->
+  <class name="OutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} or 
+                 {@link #abortJob(JobContext, int)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <param name="status" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus#FAILED} or 
+ {@link JobStatus#KILLED}. This is called from the application
+ master process for the entire job. This may be called multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @param status final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task. This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.
+ 
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapred.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRConstants#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>. This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.
+ @deprecated Use {@link #commitJob(org.apache.hadoop.mapreduce.JobContext)}
+             or {@link #abortJob(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.mapreduce.JobStatus.State)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="runState" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this
+ is a bridge between the two.]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="true" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This method implements the new interface by calling the old method. Note
+ that the input types are different between the new and old apis and this is
+ a bridge between the two.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapred.OutputFormat -->
+  <interface name="OutputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given job.
+
+ @param ignored
+ @param job configuration for the job whose output is being written.
+ @param name the unique name for this part of the output.
+ @param progress mechanism for reporting progress while writing to file.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ Implementations which write to filesystems which support delegation
+ tokens usually collect the tokens for the destination path(s)
+ and attach them to the job configuration.
+ @param ignored
+ @param job job configuration.
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter
+ @see JobConf]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.OutputLogFilter -->
+  <class name="OutputLogFilter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.fs.PathFilter"/>
+    <constructor name="OutputLogFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="accept" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[This class filters log files from directory given
+ It doesnt accept paths having _logs.
+ This can be used to list paths of output directory as follows:
+   Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
+                                   new OutputLogFilter()));]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.OutputLogFilter -->
+  <!-- start interface org.apache.hadoop.mapred.Partitioner -->
+  <interface name="Partitioner"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the paritition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be paritioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Partitioner -->
+  <!-- start interface org.apache.hadoop.mapred.RecordReader -->
+  <interface name="RecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Reads the next key/value pair from the input for processing.
+
+ @param key the key to read data into
+ @param value the value to read data into
+ @return true iff a key/value was read, false if at EOF]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a key.
+ 
+ @return a new key object.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an object of the appropriate type to be used as a value.
+ 
+ @return a new value object.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current position in the input.
+ 
+ @return the current position in the input.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this {@link InputSplit} to future operations.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[How much of the input has the {@link RecordReader} consumed i.e.
+ has been processed by?
+ 
+ @return progress from <code>0.0</code> to <code>1.0</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordReader</code> reads &lt;key, value&gt; pairs from an 
+ {@link InputSplit}.
+   
+ <p><code>RecordReader</code>, typically, converts the byte-oriented view of 
+ the input, provided by the <code>InputSplit</code>, and presents a 
+ record-oriented view for the {@link Mapper} and {@link Reducer} tasks for
+ processing. It thus assumes the responsibility of processing record 
+ boundaries and presenting the tasks with keys and values.</p>
+ 
+ @see InputSplit
+ @see InputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordReader -->
+  <!-- start interface org.apache.hadoop.mapred.RecordWriter -->
+  <interface name="RecordWriter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param reporter facility to report progress.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapred.Reducer -->
+  <interface name="Reducer"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <implements name="org.apache.hadoop.io.Closeable"/>
+    <method name="reduce"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<i>Reduces</i> values for a given key.  
+ 
+ <p>The framework calls this method for each 
+ <code>&lt;key, (list of values)&gt;</code> pair in the grouped inputs.
+ Output values must be of the same type as input values.  Input keys must 
+ not be altered. The framework will <b>reuse</b> the key and value objects
+ that are passed into the reduce, therefore the application should clone
+ the objects they want to keep a copy of. In many cases, all values are 
+ combined into zero or one value.
+ </p>
+   
+ <p>Output pairs are collected with calls to  
+ {@link OutputCollector#collect(Object,Object)}.</p>
+
+ <p>Applications can use the {@link Reporter} provided to report progress 
+ or just indicate that they are alive. In scenarios where the application 
+ takes a significant amount of time to process individual key/value 
+ pairs, this is crucial since the framework might assume that the task has 
+ timed-out and kill that task. The other way of avoiding this is to set 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.task.timeout">
+ mapreduce.task.timeout</a> to a high-enough value (or even zero for no 
+ time-outs).</p>
+ 
+ @param key the key.
+ @param values the list of values to reduce.
+ @param output to collect keys and combined values.
+ @param reporter facility to report progress.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p>The number of <code>Reducer</code>s for the job is set by the user via 
+ {@link JobConf#setNumReduceTasks(int)}. <code>Reducer</code> implementations 
+ can access the {@link JobConf} for the job via the 
+ {@link JobConfigurable#configure(JobConf)} method and initialize themselves. 
+ Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p><code>Reducer</code> is input the grouped output of a {@link Mapper}.
+   In the phase the framework, for each <code>Reducer</code>, fetches the 
+   relevant partition of the output of all the <code>Mapper</code>s, via HTTP. 
+   </p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework groups <code>Reducer</code> inputs by <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key) in this
+   stage.</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>If equivalence rules for keys while grouping the intermediates are 
+   different from those for grouping keys before reduction, then one may 
+   specify a <code>Comparator</code> via 
+   {@link JobConf#setOutputValueGroupingComparator(Class)}.Since 
+   {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to 
+   control how intermediate keys are grouped, these can be used in conjunction 
+   to simulate <i>secondary sort on values</i>.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterator, OutputCollector, Reporter)}
+   method is called for each <code>&lt;key, (list of values)&gt;</code> pair in
+   the grouped inputs.</p>
+   <p>The output of the reduce task is typically written to the 
+   {@link FileSystem} via 
+   {@link OutputCollector#collect(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+     public class MyReducer&lt;K extends WritableComparable, V extends Writable&gt; 
+     extends MapReduceBase implements Reducer&lt;K, V, K, V&gt; {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator&lt;V&gt; values,
+                          OutputCollector&lt;K, V&gt; output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the &lt;key, value&gt; pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the &lt;key, value&gt; 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of &lt;key, list of values&gt; pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner
+ @see Reporter
+ @see MapReduceBase]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reducer -->
+  <!-- start interface org.apache.hadoop.mapred.Reporter -->
+  <interface name="Reporter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the status description for the task.
+ 
+ @param status brief description of the current status.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} of the given group with the given name.
+ 
+ @param group counter group
+ @param name counter name
+ @return the <code>Counter</code> of the given group/name.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the key, which can be of
+ any {@link Enum} type, by the specified amount.
+ 
+ @param key key to identify the counter to be incremented. The key can be
+            be any <code>Enum</code>. 
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="incrCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="group" type="java.lang.String"/>
+      <param name="counter" type="java.lang.String"/>
+      <param name="amount" type="long"/>
+      <doc>
+      <![CDATA[Increments the counter identified by the group and counter name
+ by the specified amount.
+ 
+ @param group name to identify the group of the counter to be incremented.
+ @param counter name to identify the counter within the group.
+ @param amount A non-negative amount by which the counter is to 
+               be incremented.]]>
+      </doc>
+    </method>
+    <method name="getInputSplit" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="UnsupportedOperationException" type="java.lang.UnsupportedOperationException"/>
+      <doc>
+      <![CDATA[Get the {@link InputSplit} object for a map.
+ 
+ @return the <code>InputSplit</code> that the map is reading from.
+ @throws UnsupportedOperationException if called outside a mapper]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the progress of the task. Progress is represented as a number between
+ 0 and 1 (inclusive).]]>
+      </doc>
+    </method>
+    <field name="NULL" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A constant of Reporter type that does nothing.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A facility for Map-Reduce applications to report progress and update 
+ counters, status information etc.
+ 
+ <p>{@link Mapper} and {@link Reducer} can use the <code>Reporter</code>
+ provided to report progress or just indicate that they are alive. In 
+ scenarios where the application takes significant amount of time to
+ process individual key/value pairs, this is crucial since the framework 
+ might assume that the task has timed-out and kill that task.
+
+ <p>Applications can also update {@link Counters} via the provided 
+ <code>Reporter</code> .</p>
+ 
+ @see Progressable
+ @see Counters]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.Reporter -->
+  <!-- start interface org.apache.hadoop.mapred.RunningJob -->
+  <interface name="RunningJob"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the underlying job configuration
+
+ @return the configuration of the job.]]>
+      </doc>
+    </method>
+    <method name="getID" return="org.apache.hadoop.mapred.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job identifier.
+ 
+ @return the job identifier.]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.">
+      <doc>
+      <![CDATA[@deprecated This method is deprecated and will be removed. Applications should 
+ rather use {@link #getID()}.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the job.
+ 
+ @return the name of the job.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Blocks until the job is complete.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ {@link JobStatus}
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobStatus" return="org.apache.hadoop.mapred.JobStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns a snapshot of the current status, {@link JobStatus}, of the Job.
+ Need to call again for latest information.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job. Blocks until all job tasks have been killed as well.
+ If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setJobPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param priority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @param shouldFail if true the task is failed and added to failed tasks 
+                   list, otherwise it is just killed, w/o affecting 
+                   job failure status.  
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Applications should rather use {@link #killTask(TaskAttemptID, boolean)}">
+      <param name="taskId" type="java.lang.String"/>
+      <param name="shouldFail" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Applications should rather use {@link #killTask(TaskAttemptID, boolean)}]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job.
+ 
+ @return the counters for this job or null if the job has been retired.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the url where history file is archived. Returns empty string if 
+ history file is not available yet. 
+ 
+ @return the url where history file is archived
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether the job has been removed from JobTracker memory and retired.
+ On retire, the job history file is copied to a location known by 
+ {@link #getHistoryUrl()}
+ @return <code>true</code> if the job retired, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get failure info for the job.
+ @return the failure info for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RunningJob</code> is the user-interface to query for details on a 
+ running Map-Reduce job.
+ 
+ <p>Clients can get hold of <code>RunningJob</code> via the {@link JobClient}
+ and then query the running-job for details such as name, configuration, 
+ progress etc.</p> 
+ 
+ @see JobClient]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.RunningJob -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapred.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param conf the {@link JobConf} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes keys, values to 
+ {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, 
+ except it generates SequenceFileAsTextRecordReader 
+ which converts the input keys and values to their 
+ String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileAsTextRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="createKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by calling toString()
+ method. This class to SequenceFileAsTextInputFormat class is as LineRecordReader
+ class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapred.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param job job configuration
+ @param reporter reporter who sends report to task tracker
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param conf application configuration
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.SequenceFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param conf the {@link JobConf} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="SequenceFileRecordReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of key that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="getValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class of value that must be passed to {@link
+ #next(Object, Object)}..]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getCurrentValue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="seek"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="pos" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.SkipBadRecords -->
+  <class name="SkipBadRecords" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SkipBadRecords"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAttemptsToStartSkipping" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @return attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="setAttemptsToStartSkipping"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="attemptsToStartSkipping" type="int"/>
+      <doc>
+      <![CDATA[Set the number of Task attempts AFTER which skip mode 
+ will be kicked off. When skip mode is kicked off, the 
+ tasks reports the range of records which it will process 
+ next to the TaskTracker. So that on failures, TT knows which 
+ ones are possibly the bad records. On further executions, 
+ those are skipped.
+ Default value is 2.
+ 
+ @param conf the configuration
+ @param attemptsToStartSkipping no of task attempts]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrMapperProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                       {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrMapperProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS} is incremented 
+ by MapRunner after invoking the map function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}.]]>
+      </doc>
+    </method>
+    <method name="getAutoIncrReducerProcCount" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @return <code>true</code> if auto increment 
+                    {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.
+         <code>false</code> otherwise.]]>
+      </doc>
+    </method>
+    <method name="setAutoIncrReducerProcCount"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="autoIncr" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which if set to true, 
+ {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS} is incremented 
+ by framework after invoking the reduce function. This value must be set to 
+ false for applications which process the records asynchronously 
+ or buffer the input records. For example streaming. 
+ In such cases applications should increment this counter on their own.
+ Default value is true.
+ 
+ @param conf the configuration
+ @param autoIncr whether to auto increment 
+        {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}.]]>
+      </doc>
+    </method>
+    <method name="getSkipOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @return path skip output directory. Null is returned if this is not set 
+ and output directory is also not set.]]>
+      </doc>
+    </method>
+    <method name="setSkipOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the directory to which skipped records are written. By default it is 
+ the sub directory of the output _logs directory.
+ User can stop writing skipped records by setting the value null.
+ 
+ @param conf the configuration.
+ @param path skip output directory path]]>
+      </doc>
+    </method>
+    <method name="getMapperMaxSkipRecords" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="setMapperMaxSkipRecords"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipRecs" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip records surrounding the bad record PER 
+ bad record in mapper. The number includes the bad record as well.
+ To turn the feature of detection/skipping of bad records off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever records(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipRecs acceptable skip records.]]>
+      </doc>
+    </method>
+    <method name="getReducerMaxSkipGroups" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @return maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <method name="setReducerMaxSkipGroups"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxSkipGrps" type="long"/>
+      <doc>
+      <![CDATA[Set the number of acceptable skip groups surrounding the bad group PER 
+ bad group in reducer. The number includes the bad group as well.
+ To turn the feature of detection/skipping of bad groups off, set the 
+ value to 0.
+ The framework tries to narrow down the skipped range by retrying  
+ until this threshold is met OR all attempts get exhausted for this task. 
+ Set the value to Long.MAX_VALUE to indicate that framework need not try to 
+ narrow down. Whatever groups(depends on application) get skipped are 
+ acceptable.
+ Default value is 0.
+ 
+ @param conf the configuration
+ @param maxSkipGrps acceptable skip groups.]]>
+      </doc>
+    </method>
+    <field name="COUNTER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Special counters which are written by the application and are 
+ used by the framework for detecting bad records. For detecting bad records 
+ these counters must be incremented by the application.]]>
+      </doc>
+    </field>
+    <field name="COUNTER_MAP_PROCESSED_RECORDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed map records.
+ @see SkipBadRecords#getAutoIncrMapperProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <field name="COUNTER_REDUCE_PROCESSED_GROUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of processed reduce groups.
+ @see SkipBadRecords#getAutoIncrReducerProcCount(Configuration)]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Utility class for skip bad records functionality. It contains various 
+ settings related to skipping of bad records.
+ 
+ <p>Hadoop provides an optional mode of execution in which the bad records
+ are detected and skipped in further attempts.
+ 
+ <p>This feature can be used when map/reduce tasks crashes deterministically on 
+ certain input. This happens due to bugs in the map/reduce function. The usual
+ course would be to fix these bugs. But sometimes this is not possible; 
+ perhaps the bug is in third party libraries for which the source code is 
+ not available. Due to this, the task never reaches to completion even with 
+ multiple attempts and complete data for that task is lost.</p>
+  
+ <p>With this feature, only a small portion of data is lost surrounding 
+ the bad record, which may be acceptable for some user applications.
+ see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}</p>
+ 
+ <p>The skipping mode gets kicked off after certain no of failures 
+ see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}</p>
+  
+ <p>In the skipping mode, the map/reduce task maintains the record range which 
+ is getting processed at all times. Before giving the input to the
+ map/reduce function, it sends this record range to the Task tracker.
+ If task crashes, the Task tracker knows which one was the last reported
+ range. On further attempts that range get skipped.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SkipBadRecords -->
+  <!-- start class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <class name="SplitLocationInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SplitLocationInfo" type="java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isOnDisk" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInMemory" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.SplitLocationInfo -->
+  <!-- start interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgressible" return="org.apache.hadoop.util.Progressable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapred.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapreduce.TaskAttemptID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapred.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param taskId taskId number
+ @param id the task attempt number
+ @deprecated Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskAttemptID to an old one
+ @param old the new id
+ @return either old or a new TaskAttemptID constructed to match old]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <param name="attemptId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task attempt IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>all task attempt IDs</i> 
+ of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first 
+ map task</i>, we would use :
+ <pre> 
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+ </pre>
+ which will return :
+ <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType} 
+ @param taskId taskId number, or null
+ @param attemptId the task attempt number, or null
+ @return a regex pattern matching TaskAttemptIDs]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="org.apache.hadoop.mapreduce.TaskCompletionEvent"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapred.TaskAttemptID, int, boolean, org.apache.hadoop.mapred.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #getTaskAttemptId()} instead.">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id
+ @deprecated use {@link #getTaskAttemptId()} instead.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getTaskStatus" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="setTaskId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use {@link #setTaskAttemptId(TaskAttemptID)} instead.">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id.
+ @param taskId
+ @deprecated use {@link #setTaskAttemptId(TaskAttemptID)} instead.]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapred.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapreduce.TaskID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(String, int, TaskType, int)}">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(String, int, TaskType, int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param isMap whether the tip is a map 
+ @param id the tip number
+ @deprecated Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ int)}]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.  
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the {@link TaskType} 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="downgrade" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="old" type="org.apache.hadoop.mapreduce.TaskID"/>
+      <doc>
+      <![CDATA[Downgrade a new TaskID to an old one
+ @param old a new or old TaskID
+ @return either old or a new TaskID build to match old]]>
+      </doc>
+    </method>
+    <method name="read" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="isMap" type="java.lang.Boolean"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param isMap whether the tip is a map, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs
+ @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ Integer)}]]>
+      </doc>
+    </method>
+    <method name="getTaskIDsPattern" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jtIdentifier" type="java.lang.String"/>
+      <param name="jobId" type="java.lang.Integer"/>
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <param name="taskId" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Returns a regex pattern which matches task IDs. Arguments can 
+ be given null, in which case that part of the regex will be generic.  
+ For example to obtain a regex matching <i>the first map task</i> 
+ of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ <pre> 
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+ </pre>
+ which will return :
+ <pre> "task_[^_]*_[0-9]*_m_000001*" </pre> 
+ @param jtIdentifier jobTracker identifier, or null
+ @param jobId job number, or null
+ @param type the {@link TaskType}, or null 
+ @param taskId taskId number, or null
+ @return a regex pattern matching TaskIDs]]>
+      </doc>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+    </method>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskID -->
+  <!-- start class org.apache.hadoop.mapred.TaskReport -->
+  <class name="TaskReport" extends="org.apache.hadoop.mapreduce.TaskReport"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The string of the task id.]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapred.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The id of the task.]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapred.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setSuccessfulAttempt"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+      <doc>
+      <![CDATA[set successful attempt ID of the task.]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulTaskAttempt" return="org.apache.hadoop.mapred.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attempt ID that took this task to completion]]>
+      </doc>
+    </method>
+    <method name="setRunningTaskAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runningAttempts" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[set running attempt(s) of the task.]]>
+      </doc>
+    </method>
+    <method name="getRunningTaskAttempts" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the running task attempt IDs for this task]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[set finish time of task. 
+ @param finishTime finish time of task.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[set start time of the task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A report on the state of a task.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TaskReport -->
+  <!-- start class org.apache.hadoop.mapred.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.TextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.Utils -->
+  <class name="Utils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Utils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A utility class. It provides
+   A path filter utility to filter out output/part files in the output dir]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.Utils -->
+</package>
+<package name="org.apache.hadoop.mapred.jobcontrol">
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf, java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param jobConf a mapred job configuration representing a job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getAssignedJobID" return="org.apache.hadoop.mapred.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setAssignedJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="setAssignedJobID should not be called.
+ JOBID is set by the framework.">
+      <param name="mapredJobID" type="org.apache.hadoop.mapred.JobID"/>
+      <doc>
+      <![CDATA[@deprecated setAssignedJobID should not be called.
+ JOBID is set by the framework.]]>
+      </doc>
+    </method>
+    <method name="getJobConf" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred job conf of this job]]>
+      </doc>
+    </method>
+    <method name="setJobConf"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Set the mapred job conf for this job.
+ @param jobConf the mapred job conf for this job.]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="int"/>
+      <doc>
+      <![CDATA[This is a no-op function, Its a behavior change from 1.x We no more can
+ change the state from job
+ 
+ @param state
+          the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="getJobClient" return="org.apache.hadoop.mapred.JobClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job client of this job]]>
+      </doc>
+    </method>
+    <method name="getDependingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="getMapredJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="setMapredJobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapredJobID" type="java.lang.String"/>
+      <doc>
+      <![CDATA[This is no-op method for backward compatibility. It's a behavior change
+ from 1.x, we can not change job ids from job.
+ 
+ @param mapredJobID
+          the mapred job ID for this job.]]>
+      </doc>
+    </method>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="WAITING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RUNNING" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="READY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEPENDENT_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.Job -->
+  <!-- start class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+  <class name="JobControl" extends="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJobs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getState" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapred.join">
+  <!-- start class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.ArrayListBackedIterator -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <interface name="ComposableInputFormat"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableInputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <interface name="ComposableRecordReader"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <implements name="java.lang.Comparable"/>
+    <method name="id" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key this RecordReader would supply on a call to next(K,V)]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RecordReader into the object provided.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the stream is not empty, but provides no guarantee that
+ a call to next(K,V) will succeed.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[While key-value pairs from this RecordReader match the given key, register
+ them with the JoinCollector provided.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapred.join.expr</tt> property and
+ user-supplied join types from <tt>mapred.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.join.ComposableRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapred.join.define.&lt;ident&gt;</tt> to a classname. In the expression
+ <tt>mapred.join.expr</tt>, the identifier will be assumed to be a
+ ComposableRecordReader.
+ <tt>mapred.join.keycomparator</tt> can be a classname used to compare keys
+ in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapred.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapred.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for faliing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key value common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createInternalValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unsupported (returns zero in all cases).]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="jc" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapred.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapred.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapred.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.mapred.JobConf, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapred.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapred.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapred.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapred.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Node -->
+  <class name="Parser.Node" extends="java.lang.Object"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableInputFormat"/>
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapred.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapred.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapred.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapred.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapred.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapred.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapred.join.TupleWritable -->
+  <class name="TupleWritable" extends="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapred.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="U"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write key-value pair at the head of this stream to the objects provided;
+ get next key-value pair from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new value from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request position from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapred.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapred.lib">
+  <!-- start class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}. 
+ 
+ @see org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <class name="ChainMapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ <p>
+
+ @param job              job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="java.lang.Object"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>map(...)</code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainMapper and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ <p>
+ ChainMapper usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <class name="ChainReducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.]]>
+      </doc>
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="reducerConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Sets the Reducer class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Reducer leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Reducer does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+
+ @param job              job's JobConf to add the Reducer class.
+ @param klass            the Reducer class to add.
+ @param inputKeyClass    reducer input key class.
+ @param inputValueClass  reducer input value class.
+ @param outputKeyClass   reducer output key class.
+ @param outputValueClass reducer output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param reducerConf      a JobConf with the configuration for the Reducer
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="byValue" type="boolean"/>
+      <param name="mapperConf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Adds a Mapper class to the chain job's JobConf.
+ <p>
+ It has to be specified how key and values are passed from one element of
+ the chain to the next, by value or by reference. If a Mapper leverages the
+ assumed semantics that the key and values are not modified by the collector
+ 'by value' must be used. If the Mapper does not expect this semantics, as
+ an optimization to avoid serialization and deserialization 'by reference'
+ can be used.
+ <p>
+ For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ .
+
+ @param job              chain job's JobConf to add the Mapper class.
+ @param klass            the Mapper class to add.
+ @param inputKeyClass    mapper input key class.
+ @param inputValueClass  mapper input value class.
+ @param outputKeyClass   mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue          indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf       a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configures the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.configure(...)</code> should be
+ invoked at the beginning of the overwriter method.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Chains the <code>reduce(...)</code> method of the Reducer with the
+ <code>map(...) </code> methods of the Mappers in the chain.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes  the ChainReducer, the Reducer and all the Mappers in the chain.
+ <p>
+ If this method is overriden <code>super.close()</code> should be
+ invoked at the end of the overwriter method.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion, the output of the first becomes the input of the
+ second, and so on until the last Mapper, the output of the last Mapper will
+ be written to the task's output.
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain.
+ This enables having reusable specialized Mappers that can be combined to
+ perform composite operations within a single task.
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use maching output and input key and
+ value classes as no conversion is done by the chaining code.
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ <p>
+ ChainReducer usage pattern:
+ <p>
+ <pre>
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.ChainReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(List)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(List)}.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #createPool(PathFilter...)}.">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.
+ @deprecated Use {@link #createPool(PathFilter...)}.]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+ 
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An abstract {@link org.apache.hadoop.mapred.InputFormat} that returns {@link CombineFileSplit}'s
+ in {@link org.apache.hadoop.mapred.InputFormat#getSplits(JobConf, int)} method. 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default spliting behaviour in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement {@link org.apache.hadoop.mapred.InputFormat#getRecordReader(InputSplit, JobConf, Reporter)}
+ to construct <code>RecordReader</code>'s for <code>CombineFileSplit</code>'s.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.mapred.Reporter, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return the amount of data processed]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapred.JobConf"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="reporter" type="org.apache.hadoop.mapred.Reporter"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapred.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.RecordReader"/>
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapred.FileInputFormat, org.apache.hadoop.mapred.lib.CombineFileSplit, org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.Reporter, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputSplit"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapred.lib.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapred.lib.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <class name="FieldSelectionMapReduce" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="FieldSelectionMapReduce"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FieldSelectionMapReduce -->
+  <!-- start class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapred.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the supplied output format.
+ @param out the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapred.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <class name="HashPartitioner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <class name="IdentityMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="IdentityMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The identity function.  Input key/value pair is written directly to
+ output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implements the identity function, mapping inputs directly to outputs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <class name="IdentityReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="IdentityReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all keys and values directly to output.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Performs no reduction, writing all input values directly to the output.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.IdentityReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.mapreduce.lib.partition.InputSampler"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSampler" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="sampler" type="org.apache.hadoop.mapred.lib.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InputSampler -->
+  <!-- start class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapred.lib.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link JobConf} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} that sums long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <class name="MultipleOutputFormat" extends="org.apache.hadoop.mapred.FileOutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a composite record writer that can write key/value data to different
+ output files
+ 
+ @param fs
+          the file system to use
+ @param job
+          the job conf for the job
+ @param name
+          the leaf file name for the output file (such as part-00000")
+ @param arg3
+          a progressable for reporting progress.
+ @return a composite record writer
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="generateLeafFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the leaf name for the output file name. The default behavior does
+ not change the leaf file name (such as part-00000)
+ 
+ @param name
+          the leaf file name for the output file
+ @return the given leaf file name]]>
+      </doc>
+    </method>
+    <method name="generateFileNameForKeyValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the file output file name based on the given key and the leaf file
+ name. The default behavior is that the file name does not depend on the
+ key.
+ 
+ @param key
+          the key of the output data
+ @param name
+          the leaf file name
+ @return generated file name]]>
+      </doc>
+    </method>
+    <method name="generateActualKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual key from the given key/value. The default behavior is that
+ the actual key is equal to the given key
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual key derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="generateActualValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <doc>
+      <![CDATA[Generate the actual value from the given key and value. The default behavior is that
+ the actual value is equal to the given value
+ 
+ @param key
+          the key of the output data
+ @param value
+          the value of the output data
+ @return the actual value derived from the given key/value]]>
+      </doc>
+    </method>
+    <method name="getInputFileBasedOutputFileName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate the outfile name based on a given name and the input file name. If
+ the {@link JobContext#MAP_INPUT_FILE} does not exists (i.e. this is not for a map only job),
+ the given name is returned unchanged. If the config value for
+ "num.of.trailing.legs.to.use" is not set, or set 0 or negative, the given
+ name is returned unchanged. Otherwise, return a file name consisting of the
+ N trailing legs of the input file name where N is the config value for
+ "num.of.trailing.legs.to.use".
+ 
+ @param job
+          the job config
+ @param name
+          the output file name
+ @return the outfile name based on a given name and the input file name.]]>
+      </doc>
+    </method>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param fs
+          the file system to use
+ @param job
+          a job conf object
+ @param name
+          the name of the file over which a record writer object will be
+          constructed
+ @param arg3
+          a progressable object
+ @return A RecordWriter object over the given file
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This abstract class extends the FileOutputFormat, allowing to write the
+ output data to different output files. There are three basic use cases for
+ this class.
+ 
+ Case one: This class is used for a map reduce job with at least one reducer.
+ The reducer wants to write data to different files depending on the actual
+ keys. It is assumed that a key (or value) encodes the actual key (value)
+ and the desired location for the actual key (value).
+ 
+ Case two: This class is used for a map only job. The job wants to use an
+ output file name that is either a part of the input file name of the input
+ data, or some derivation of it.
+ 
+ Case three: This class is used for a map only job. The job wants to use an
+ output file name that depends on both the keys and the input file name,]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple named outputs support, it should be
+ instantiated in the Mapper/Reducer configure method.
+
+ @param job the job configuration object]]>
+      </doc>
+    </constructor>
+    <method name="getNamedOutputsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns list of channel names.
+
+ @param conf job conf
+ @return List of channel Names]]>
+      </doc>
+    </method>
+    <method name="isMultiNamedOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns if a named output is multiple.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return <code>true</code> if the name output is multi, <code>false</code>
+         if it is single. If the name output is not defined it returns
+         <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputFormatClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named output OutputFormat.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return namedOutput OutputFormat]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the key class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class for the named output key]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the value class for a named output.
+
+ @param conf        job conf
+ @param namedOutput named output
+ @return class of named output value]]>
+      </doc>
+    </method>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="addMultiNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a multi named output for the job.
+
+ @param conf              job conf to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the
+                          default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+ @param conf    job conf to enableadd the named output.
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ <p>
+ By default these counters are disabled.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled.
+ The counters group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+
+
+ @param conf    job conf to enableadd the named output.
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="getNamedOutputs" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns iterator with the defined name outputs.
+
+ @return iterator with the defined named outputs]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a named output.
+
+ @param namedOutput the named output name
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="getCollector" return="org.apache.hadoop.mapred.OutputCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="multiName" type="java.lang.String"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the output collector for a multi named output.
+
+ @param namedOutput the named output name
+ @param multiName   the multi name part
+ @param reporter    the reporter
+ @return the output collector for the given named output
+ @throws IOException thrown if output collector could not be created]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes all the opened named outputs.
+ <p>
+ If overriden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>
+
+ @throws java.io.IOException thrown if any of the MultipleOutput files
+                             could not be closed properly.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing to additional outputs other
+ than the job default output via the <code>OutputCollector</code> passed to
+ the <code>map()</code> and <code>reduce()</code> methods of the
+ <code>Mapper</code> and <code>Reducer</code> implementations.
+ <p>
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ <p>
+ A named output can be a single file or a multi file. The later is referred as
+ a multi named output.
+ <p>
+ A multi named output is an unbound set of files all sharing the same
+ <code>OutputFormat</code>, key class and value class configuration.
+ <p>
+ When named outputs are used within a <code>Mapper</code> implementation,
+ key/values written to a name output are not part of the reduce phase, only
+ key/values written to the job <code>OutputCollector</code> are part of the
+ reduce phase.
+ <p>
+ MultipleOutputs supports counters, by default the are disabled. The counters
+ group is the {@link MultipleOutputs} class name.
+ </p>
+ The names of the counters are the same as the named outputs. For multi
+ named outputs the name of the counter is the concatenation of the named
+ output, and underscore '_' and the multiname.
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+ </pre>
+ <p>
+ Job configuration usage pattern is:
+ <pre>
+
+ public class MOReduce implements
+   Reducer&lt;WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <class name="MultipleSequenceFileOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleSequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output data 
+ to different output files in sequence file output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleSequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <class name="MultipleTextOutputFormat" extends="org.apache.hadoop.mapred.lib.MultipleOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleTextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getBaseRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="arg3" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class extends the MultipleOutputFormat, allowing to write the output
+ data to different output files in Text output format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultipleTextOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <class name="MultithreadedMapRunner" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.MapRunnable"/>
+    <constructor name="MultithreadedMapRunner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobConf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="input" type="org.apache.hadoop.mapred.RecordReader"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Multithreaded implementation for {@link MapRunnable}.
+ <p>
+ It can be used instead of the default implementation,
+ of {@link org.apache.hadoop.mapred.MapRunner}, when the Map
+ operation is not CPU bound in order to improve throughput.
+ <p>
+ Map implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured to use this MapRunnable class (using
+ the JobConf.setMapRunnerClass method) and
+ the number of threads the thread-pool can use with the
+ <code>mapred.map.multithreadedrunner.threads</code> property, its default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.MultithreadedMapRunner -->
+  <!-- start class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapred.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="numSplits" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see org.apache.hadoop.mapred.FileInputFormat#getSplits(JobConf, int)]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapred.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <class name="TokenCountMapper" extends="org.apache.hadoop.mapred.MapReduceBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <constructor name="TokenCountMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that maps text values into &lt;token,freq&gt; pairs.  Uses
+ {@link StringTokenizer} to break text into tokens.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TokenCountMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Partitioner"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.
+ @deprecated Use 
+ {@link #setPartitionFile(Configuration, Path)}
+ instead]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use 
+ {@link #getPartitionFile(Configuration)}
+ instead">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(JobConf,Path)
+ @deprecated Use 
+ {@link #getPartitionFile(Configuration)}
+ instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.mapred.JobConf"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param job a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value aggregator 
+ descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class that may be 
+ dynamically loaded. The other is to delegate invocations of 
+ generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param job a job configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Combiner does not need to configure.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param output to collect combined values]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param job
+          a JobConf object that may contain the information that can be used
+          to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a JobConf object. Its main
+ function is to generate a list of aggregation-id/value pairs. An aggregation
+ id encodes an aggregation type which is used to guide the way to aggregate
+ the value in the reduce/combiner phrase of an Aggregate based job.The mapper in
+ an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapred.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @param caller the the caller class.
+ @return a JobConf object ready for submission.
+
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a JobConf object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapred.JobConf"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <param name="caller" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specilizing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner classes,
+ and a set of built-in value aggregators, and a generic utility class that
+ helps user create map/reduce jobs using the generic class. The built-in
+ aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(JobConfjob); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.Mapper"/>
+    <implements name="org.apache.hadoop.mapred.Reducer"/>
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="org.apache.hadoop.io.Text"/>
+      <param name="arg1" type="java.util.Iterator"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorJobBase"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.util.Iterator"/>
+      <param name="output" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@param key
+          the key is expected to be a Text object, whose prefix indicates
+          the type of aggregation to aggregate the values. In effect, data
+          driven computing is achieved. It is assumed that each aggregator's
+          getReport method emits appropriate output for the aggregator. This
+          may be further customiized.
+ @param values
+          the values to be aggregated]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arg0" type="K1"/>
+      <param name="arg1" type="V1"/>
+      <param name="arg2" type="org.apache.hadoop.mapred.OutputCollector"/>
+      <param name="arg3" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Do nothing. Should not be called]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapred.lib.db">
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.InputFormat"/>
+    <implements name="org.apache.hadoop.mapred.JobConfigurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="chunks" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, eg. '(updated &gt;
+ 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(JobConf, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns the number of records in
+ the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(JobConf, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="filesystem" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.lib.db.DBOutputFormat -->
+  <!-- start interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBWritable"/>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapred.lib.db.DBWritable -->
+</package>
+<package name="org.apache.hadoop.mapred.pipes">
+  <!-- start class org.apache.hadoop.mapred.pipes.Submitter -->
+  <class name="Submitter" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="Submitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="Submitter" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getExecutable" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Get the URI of the application's executable.
+ @param conf
+ @return the URI where the application's executable is located]]>
+      </doc>
+    </method>
+    <method name="setExecutable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="executable" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the URI for the application's executable. Normally this is a hdfs: 
+ location.
+ @param conf
+ @param executable The URI of the application's executable.]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job is using a Java RecordReader.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java RecordReader
+ @param conf the configuration to check
+ @return is it a Java RecordReader?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Mapper is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaMapper" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Mapper.
+ @param conf the configuration to check
+ @return is it a Java Mapper?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the Reducer is written in Java.
+ @param conf the configuration to modify
+ @param value the new value]]>
+      </doc>
+    </method>
+    <method name="getIsJavaReducer" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Check whether the job is using a Java Reducer.
+ @param conf the configuration to check
+ @return is it a Java Reducer?]]>
+      </doc>
+    </method>
+    <method name="setIsJavaRecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the job will use a Java RecordWriter.
+ @param conf the configuration to modify
+ @param value the new value to set]]>
+      </doc>
+    </method>
+    <method name="getIsJavaRecordWriter" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Will the reduce use a Java RecordWriter?
+ @param conf the configuration to check
+ @return true, if the output of the job will be written by Java]]>
+      </doc>
+    </method>
+    <method name="getKeepCommandFile" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <doc>
+      <![CDATA[Does the user want to keep the command file for debugging? If this is
+ true, pipes will write a copy of the command data to a file in the
+ task directory named "downlink.data", which may be used to run the C++
+ program under the debugger. You probably also want to set 
+ JobConf.setKeepFailedTaskFiles(true) to keep the entire directory from
+ being deleted.
+ To run using the data file, set the environment variable 
+ "mapreduce.pipes.commandfile" to point to the file.
+ @param conf the configuration to check
+ @return will the framework save the command file?]]>
+      </doc>
+    </method>
+    <method name="setKeepCommandFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <param name="keep" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether to keep the command file for debugging
+ @param conf the configuration to modify
+ @param keep the new value]]>
+      </doc>
+    </method>
+    <method name="submitJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link Submitter#runJob(JobConf)}">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException
+ @deprecated Use {@link Submitter#runJob(JobConf)}]]>
+      </doc>
+    </method>
+    <method name="runJob" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the map/reduce cluster. All of the necessary modifications
+ to the job to run under pipes are made to the configuration.
+ @param conf the job to submit to the cluster (MODIFIED)
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="jobSubmit" return="org.apache.hadoop.mapred.RunningJob"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Submit a job to the Map-Reduce framework.
+ This returns a handle to the {@link RunningJob} which can be used to track
+ the running-job.
+ 
+ @param conf the job configuration.
+ @return a handle to the {@link RunningJob} which can be used to track the
+         running-job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Submit a pipes job based on the command line arguments.
+ @param args]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PRESERVE_COMMANDFILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXECUTABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INTERPRETOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_RW" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IS_JAVA_REDUCE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The main entry point and job submitter. It may either be used as a command
+ line-based or API-based method to launch Pipes jobs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapred.pipes.Submitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce">
+  <!-- start class org.apache.hadoop.mapreduce.Cluster -->
+  <class name="Cluster" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Cluster" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="Cluster" type="java.net.InetSocketAddress, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Cluster</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the file system where job-specific files are stored
+ 
+ @return object of FileSystem
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job corresponding to jobid.
+ 
+ @param jobId
+ @return object of {@link Job}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the queues in cluster.
+ 
+ @return array of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get queue information for the specified name.
+ 
+ @param name queuename
+ @return object of {@link QueueInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLogParams" return="org.apache.hadoop.mapreduce.v2.LogParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+      <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get log parameters for the specified jobID or taskAttemptID
+ @param jobID the job id.
+ @param taskAttemptID the task attempt id. Optional.
+ @return the LogParams
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getClusterStatus" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current cluster status.
+ 
+ @return object of {@link ClusterMetrics}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getActiveTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all active trackers in the cluster.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get blacklisted trackers.
+ 
+ @return array of {@link TaskTrackerInfo}
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getAllJobs" return="org.apache.hadoop.mapreduce.Job[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getAllJobStatuses()} instead.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get all the jobs in cluster.
+ 
+ @return array of {@link Job}
+ @throws IOException
+ @throws InterruptedException
+ @deprecated Use {@link #getAllJobStatuses()} instead.]]>
+      </doc>
+    </method>
+    <method name="getAllJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get job status for all jobs in the cluster.
+ @return job status for all jobs in cluster
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getSystemDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker system directory path where 
+ job-specific files will  be placed.
+ 
+ @return the system directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getStagingAreaDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Grab the jobtracker's view of the staging directory path where 
+ job-specific files will  be placed.
+ 
+ @return the staging directory where job-specific files are to be placed.]]>
+      </doc>
+    </method>
+    <method name="getJobHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the job history file path for a given job id. The job history file at 
+ this path may or may not be existing depending on the job completion state.
+ The file is present only for the completed jobs.
+ @param jobId the JobID of the job submitted by the current user.
+ @return the file path of the job history file
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the Queue ACLs for current user
+ @return array of QueueAclsInfo object for current user.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the root level queues.
+ @return array of JobQueueInfo object.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns immediate children of queueName.
+ @param queueName
+ @return array of JobQueueInfo which are children of queueName
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the JobTracker's status.
+ 
+ @return {@link JobTrackerStatus} of the JobTracker
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerExpiryInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get a delegation token for the user from the JobTracker.
+ @param renewer the user who can renew the token
+ @return the new token
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#renew} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Renew a delegation token
+ @param token the token to renew
+ @return the new expiration time
+ @throws InvalidToken
+ @throws IOException
+ @deprecated Use {@link Token#renew} instead]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link Token#cancel} instead">
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Cancel a delegation token from the JobTracker
+ @param token the token to cancel
+ @throws IOException
+ @deprecated Use {@link Token#cancel} instead]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Provides a way to access information about the map/reduce cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Cluster -->
+  <!-- start class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <class name="ClusterMetrics" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="ClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterMetrics" type="int, int, int, int, int, int, int, int, int, int, int, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRunningMaps" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running map tasks in the cluster.
+ 
+ @return running maps]]>
+      </doc>
+    </method>
+    <method name="getRunningReduces" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running reduce tasks in the cluster.
+ 
+ @return running reduces]]>
+      </doc>
+    </method>
+    <method name="getOccupiedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of occupied map slots in the cluster.
+ 
+ @return occupied map slot count]]>
+      </doc>
+    </method>
+    <method name="getOccupiedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of occupied reduce slots in the cluster.
+ 
+ @return occupied reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedMapSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get number of reserved map slots in the cluster.
+ 
+ @return reserved map slot count]]>
+      </doc>
+    </method>
+    <method name="getReservedReduceSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved reduce slots in the cluster.
+ 
+ @return reserved reduce slot count]]>
+      </doc>
+    </method>
+    <method name="getMapSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of map slots in the cluster.
+ 
+ @return map slot capacity]]>
+      </doc>
+    </method>
+    <method name="getReduceSlotCapacity" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of reduce slots in the cluster.
+ 
+ @return reduce slot capacity]]>
+      </doc>
+    </method>
+    <method name="getTotalJobSubmissions" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the total number of job submissions in the cluster.
+ 
+ @return total number of job submissions]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active trackers in the cluster.
+ 
+ @return active tracker count.]]>
+      </doc>
+    </method>
+    <method name="getBlackListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of blacklisted trackers in the cluster.
+ 
+ @return blacklisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getGrayListedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of graylisted trackers in the cluster.
+ 
+ @return graylisted tracker count]]>
+      </doc>
+    </method>
+    <method name="getDecommissionedTaskTrackerCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of decommissioned trackers in the cluster.
+ 
+ @return decommissioned tracker count]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Status information on the current state of the Map-Reduce cluster.
+ 
+ <p><code>ClusterMetrics</code> provides clients with information such as:
+ <ol>
+   <li>
+   Size of the cluster.  
+   </li>
+   <li>
+   Number of blacklisted and decommissioned trackers.  
+   </li>
+   <li>
+   Slot capacity of the cluster. 
+   </li>
+   <li>
+   The number of currently occupied/reserved map and reduce slots.
+   </li>
+   <li>
+   The number of currently running map and reduce tasks.
+   </li>
+   <li>
+   The number of job submissions.
+   </li>
+ </ol>
+ 
+ <p>Clients can query for the latest <code>ClusterMetrics</code>, via 
+ {@link Cluster#getClusterStatus()}.</p>
+ 
+ @see Cluster]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ClusterMetrics -->
+  <!-- start interface org.apache.hadoop.mapreduce.Counter -->
+  <interface name="Counter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="(and no-op by default)">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the counter
+ @param displayName of the counter
+ @deprecated (and no-op by default)]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the name of the counter]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the counter.
+ @return the user facing name of the counter]]>
+      </doc>
+    </method>
+    <method name="getValue" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[What is the current value of this counter?
+ @return the current value]]>
+      </doc>
+    </method>
+    <method name="setValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set this counter by the given value
+ @param value the value to set]]>
+      </doc>
+    </method>
+    <method name="increment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incr" type="long"/>
+      <doc>
+      <![CDATA[Increment this counter by the given value
+ @param incr the value to increase this counter by]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A named counter that tracks the progress of a map/reduce job.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> is named by
+ an {@link Enum} and has a long for the value.</p>
+
+ <p><code>Counters</code> are bunched into Groups, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.Counter -->
+  <!-- start interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <interface name="CounterGroup"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+    <doc>
+    <![CDATA[A group of {@link Counter}s that logically belong together. Typically,
+ it is an {@link Enum} subclass and the counters are the values.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.CounterGroup -->
+  <!-- start class org.apache.hadoop.mapreduce.Counters -->
+  <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Counters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="Counters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct the Counters object from the another counters object
+ @param <C> the type of counter
+ @param <G> the type of counter group
+ @param counters the old counters object]]>
+      </doc>
+    </constructor>
+    <doc>
+    <![CDATA[<p><code>Counters</code> holds per job/task counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link CounterGroup}s, each
+ comprising of counters from a particular <code>Enum</code> class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Counters -->
+  <!-- start class org.apache.hadoop.mapreduce.ID -->
+  <class name="ID" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.WritableComparable"/>
+    <constructor name="ID" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructs an ID object from the given int]]>
+      </doc>
+    </constructor>
+    <constructor name="ID"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[returns the int which represents the identifier]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="that" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare IDs by associated numbers]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SEPARATOR" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A general identifier, which internally stores the id
+ as an integer. This is the super class of {@link JobID}, 
+ {@link TaskID} and {@link TaskAttemptID}.
+ 
+ @see JobID
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.ID -->
+  <!-- start class org.apache.hadoop.mapreduce.InputFormat -->
+  <class name="InputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplits" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Logically split the set of input files for the job.  
+ 
+ <p>Each {@link InputSplit} is then assigned to an individual {@link Mapper}
+ for processing.</p>
+
+ <p><i>Note</i>: The split is a <i>logical</i> split of the inputs and the
+ input files are not physically split into chunks. For e.g. a split could
+ be <i>&lt;input-file-path, start, offset&gt;</i> tuple. The InputFormat
+ also creates the {@link RecordReader} to read the {@link InputSplit}.
+ 
+ @param context job configuration.
+ @return an array of {@link InputSplit}s for the job.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Create a record reader for a given split. The framework will call
+ {@link RecordReader#initialize(InputSplit, TaskAttemptContext)} before
+ the split is used.
+ @param split the split to be read
+ @param context the information about the task
+ @return a new record reader
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputFormat</code> describes the input-specification for a 
+ Map-Reduce job. 
+ 
+ <p>The Map-Reduce framework relies on the <code>InputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the input-specification of the job. 
+   <li>
+   Split-up the input file(s) into logical {@link InputSplit}s, each of 
+   which is then assigned to an individual {@link Mapper}.
+   </li>
+   <li>
+   Provide the {@link RecordReader} implementation to be used to glean
+   input records from the logical <code>InputSplit</code> for processing by 
+   the {@link Mapper}.
+   </li>
+ </ol>
+ 
+ <p>The default behavior of file-based {@link InputFormat}s, typically 
+ sub-classes of {@link FileInputFormat}, is to split the 
+ input into <i>logical</i> {@link InputSplit}s based on the total size, in 
+ bytes, of the input files. However, the {@link FileSystem} blocksize of  
+ the input files is treated as an upper bound for input splits. A lower bound 
+ on the split size can be set via 
+ <a href="{@docRoot}/../hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml#mapreduce.input.fileinputformat.split.minsize">
+ mapreduce.input.fileinputformat.split.minsize</a>.</p>
+ 
+ <p>Clearly, logical splits based on input-size is insufficient for many 
+ applications since record boundaries are to respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibility to respect record-boundaries and present a record-oriented
+ view of the logical <code>InputSplit</code> to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see FileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.InputSplit -->
+  <class name="InputSplit" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the size of the split, so that the input splits can be sorted by size.
+ @return the number of bytes in the split
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the list of nodes by name where the data for the split would be local.
+ The locations do not need to be serialized.
+ 
+ @return a new array of the node nodes.
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets info about which nodes the input split is stored on and how it is
+ stored at each location.
+ 
+ @return list of <code>SplitLocationInfo</code>s describing how the split
+    data is stored at each location. A null value indicates that all the
+    locations have the data stored on disk.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>InputSplit</code> represents the data to be processed by an 
+ individual {@link Mapper}. 
+
+ <p>Typically, it presents a byte-oriented view on the input and is the 
+ responsibility of {@link RecordReader} of the job to process this and present
+ a record-oriented view.
+ 
+ @see InputFormat
+ @see RecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.InputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.Job -->
+  <class name="Job" extends="org.apache.hadoop.mapreduce.task.JobContextImpl"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="java.lang.AutoCloseable"/>
+    <constructor name="Job"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </constructor>
+    <constructor name="Job" type="org.apache.hadoop.conf.Configuration, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration, String)}">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #getInstance(Configuration, String)}]]>
+      </doc>
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} .
+ A Cluster will be created with a generic {@link Configuration}.
+ 
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a 
+ given {@link Configuration}.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="jobName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and a given jobName.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.JobStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration} and {@link JobStatus}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param status job status
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance()}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster}.
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance()}]]>
+      </doc>
+    </method>
+    <method name="getInstance" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #getInstance(Configuration)}">
+      <param name="ignored" type="org.apache.hadoop.mapreduce.Cluster"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Creates a new {@link Job} with no particular {@link Cluster} and given
+ {@link Configuration}.
+ A Cluster will be created from the conf parameter only when it's needed.
+ 
+ The <code>Job</code> makes a copy of the <code>Configuration</code> so 
+ that any necessary internal modifications do not reflect on the incoming 
+ parameter.
+ 
+ @param ignored
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance(Configuration)}]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Returns the current state of the Job.
+ 
+ @return JobStatus#State
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getTrackingURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the URL where some job progress information will be displayed.
+ 
+ @return the URL where some job progress information will be displayed.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path of the submitted job configuration.
+ 
+ @return the path of the submitted job configuration.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get start time of the job.
+ 
+ @return the start time of the job]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get finish time of the job.
+ 
+ @return the finish time of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the scheduling info of the job]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get scheduling info of the job.
+ 
+ @return the priority info of the job]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getHistoryUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Dump stats to screen.]]>
+      </doc>
+    </method>
+    <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the information of the current state of the tasks of a job.
+ 
+ @param type Type of the task
+ @return the list of all of the map tips.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="mapProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 
+ and 1.0.  When all map tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's map-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="reduceProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0 
+ and 1.0.  When all reduce tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's reduce-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="cleanupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0 
+ and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's cleanup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0 
+ and 1.0.  When all setup tasks have completed, the function returns 1.0.
+ 
+ @return the progress of the job's setup-tasks.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isComplete" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job is finished or not. 
+ This is a non-blocking call.
+ 
+ @return <code>true</code> if the job is complete, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isSuccessful" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check if the job completed successfully. 
+ 
+ @return <code>true</code> if the job succeeded, else <code>false</code>.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill the running job.  Blocks until all job tasks have been
+ killed as well.  If the job is no longer running, it simply returns.
+ 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+ @param jobPriority the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setPriorityAsInteger"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobPriority" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Set the priority of a running job.
+
+ @param jobPriority
+          the new priority for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <param name="numEvents" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @param numEvents number of events to fetch
+ @return an array of {@link TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startFrom" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get events indicating completion (success/failure) of component tasks.
+  
+ @param startFrom index to start fetching events from
+ @return an array of {@link org.apache.hadoop.mapred.TaskCompletionEvent}s
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="killTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Kill indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="failTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Fail indicated task attempt.
+ 
+ @param taskId the id of the task to be terminated.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCounters" return="org.apache.hadoop.mapreduce.Counters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets the counters for this job. May return null if the job has been
+ retired and the job is no longer in the completed job store.
+ 
+ @return the counters for this job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getTaskDiagnostics" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskid" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Gets the diagnostic messages for a given task attempt.
+ @param taskid
+ @return the list of diagnostic messages for the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setNumReduceTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tasks" type="int"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the number of reduce tasks for the job.
+ @param tasks the number of reduce tasks
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setWorkingDirectory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the current working directory for the default file system.
+ 
+ @param dir the new current working directory.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setInputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link InputFormat} for the job.
+ @param cls the <code>InputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link OutputFormat} for the job.
+ @param cls the <code>OutputFormat</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Mapper} for the job.
+ @param cls the <code>Mapper</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setJarByClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the Jar by finding where a given class came from.
+ @param cls the example class]]>
+      </doc>
+    </method>
+    <method name="setJar"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jar" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job jar]]>
+      </doc>
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reported username for this job.
+ 
+ @param user the username for this job.]]>
+      </doc>
+    </method>
+    <method name="setCombinerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the combiner class for the job.
+ @param cls the combiner to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setReducerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Reducer} for the job.
+ @param cls the <code>Reducer</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setPartitionerClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the {@link Partitioner} for the job.
+ @param cls the <code>Partitioner</code> to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the map output data. This allows the user to
+ specify the map output key class to be different than the final output
+ value class.
+ 
+ @param theClass the map output key class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setMapOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for the map output data. This allows the user to
+ specify the map output value class to be different than the final output
+ value class.
+ 
+ @param theClass the map output value class.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the key class for the job output data.
+ 
+ @param theClass the key class for the job output data.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="theClass" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the value class for job outputs.
+ 
+ @param theClass the value class for job outputs.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setCombinerKeyGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to combiner,
+ {@link Reducer#reduce(Object, Iterable,
+ org.apache.hadoop.mapreduce.Reducer.Context)}
+
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSortComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls how the keys are sorted before they
+ are passed to the {@link Reducer}.
+ @param cls the raw comparator
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setGroupingComparatorClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cls" type="java.lang.Class"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Define the comparator that controls which keys are grouped together
+ for a single call to 
+ {@link Reducer#reduce(Object, Iterable, 
+                       org.apache.hadoop.mapreduce.Reducer.Context)}
+ @param cls the raw comparator to use
+ @throws IllegalStateException if the job is submitted
+ @see #setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <exception name="IllegalStateException" type="java.lang.IllegalStateException"/>
+      <doc>
+      <![CDATA[Set the user-specified job name.
+ 
+ @param name the job's new name.
+ @throws IllegalStateException if the job is submitted]]>
+      </doc>
+    </method>
+    <method name="setSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on, else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setMapSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for map tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for map tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setReduceSpeculativeExecution"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="speculativeExecution" type="boolean"/>
+      <doc>
+      <![CDATA[Turn speculative execution on or off for this job for reduce tasks. 
+ 
+ @param speculativeExecution <code>true</code> if speculative execution 
+                             should be turned on for reduce tasks,
+                             else <code>false</code>.]]>
+      </doc>
+    </method>
+    <method name="setJobSetupCleanupNeeded"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="needed" type="boolean"/>
+      <doc>
+      <![CDATA[Specify whether job-setup and job-cleanup is needed for the job 
+ 
+ @param needed If <code>true</code>, job-setup and job-cleanup will be
+               considered from {@link OutputCommitter} 
+               else ignored.]]>
+      </doc>
+    </method>
+    <method name="setCacheArchives"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archives" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of archives
+ @param archives The list of archives that need to be localized]]>
+      </doc>
+    </method>
+    <method name="setCacheFiles"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="files" type="java.net.URI[]"/>
+      <doc>
+      <![CDATA[Set the given set of files
+ @param files The list of files that need to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheArchive"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a archives to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addCacheFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+      <doc>
+      <![CDATA[Add a file to be localized
+ @param uri The uri of the cache to be localized]]>
+      </doc>
+    </method>
+    <method name="addFileToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an file path to the current set of classpath entries It adds the file
+ to cache as well.
+ 
+ Files added with this method will not be unpacked while being added to the
+ classpath.
+ To add archives to classpath, use the {@link #addArchiveToClassPath(Path)}
+ method instead.
+
+ @param file Path of the file to be added]]>
+      </doc>
+    </method>
+    <method name="addArchiveToClassPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="archive" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an archive path to the current set of classpath entries. It adds the
+ archive to cache as well.
+ 
+ Archive files will be unpacked and added to the classpath
+ when being distributed.
+
+ @param archive Path of the archive to be added]]>
+      </doc>
+    </method>
+    <method name="createSymlink"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to enable symlinks, but currently symlinks cannot be
+ disabled.]]>
+      </doc>
+    </method>
+    <method name="setMaxMapAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ map task.
+ 
+ @param n the number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="setMaxReduceAttempts"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[Expert: Set the number of maximum attempts that will be made to run a
+ reduce task.
+ 
+ @param n the number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <method name="setProfileEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newValue" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the system should collect profiler information for some of 
+ the tasks in this job? The information is stored in the user log 
+ directory.
+ @param newValue true means it should be gathered]]>
+      </doc>
+    </method>
+    <method name="setProfileParams"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the profiler configuration arguments. If the string contains a '%s' it
+ will be replaced with the name of the profiling output file when the task
+ runs.
+
+ This value is passed to the task child JVM on the command line.
+
+ @param value the configuration string]]>
+      </doc>
+    </method>
+    <method name="setProfileTaskRange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <param name="newValue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ranges of maps or reduces to profile. setProfileEnabled(true) 
+ must also be called.
+ @param newValue a set of integer ranges of the map ids]]>
+      </doc>
+    </method>
+    <method name="setCancelDelegationTokenUponJobCompletion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[Sets the flag that will allow the JobTracker to cancel the HDFS delegation
+ tokens upon job completion. Defaults to true.]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addFileToSharedCacheAndClasspath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add a file to job config for shared cache processing. If shared cache is
+ enabled, it will return true, otherwise, return false. We don't check with
+ SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Job Submitter will also
+ add the file to classpath. Intended to be used by user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="addArchiveToSharedCache" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.net.URI"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Add an archive to job config for shared cache processing. If shared cache
+ is enabled, it will return true, otherwise, return false. We don't check
+ with SCM here given application might not be able to provide the job id;
+ ClientSCMProtocol.use requires the application id. Job Submitter will read
+ the files from job config and take care of things. Intended to be used by
+ user code.
+
+ @param resource The resource that Job Submitter will process later using
+          shared cache.
+ @param conf Configuration to add the resource to
+ @return whether the resource has been added to the configuration]]>
+      </doc>
+    </method>
+    <method name="setFileSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for files. If the parameter
+ was previously set, this method will replace the old value with the new
+ provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="setArchiveSharedCacheUploadPolicies"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="policies" type="java.util.Map"/>
+      <doc>
+      <![CDATA[This is to set the shared cache upload policies for archives. If the
+ parameter was previously set, this method will replace the old value with
+ the new provided map.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @param policies A map containing the shared cache upload policies for a set
+          of resources. The key is the url of the resource and the value is
+          the upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getFileSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for files.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="getArchiveSharedCacheUploadPolicies" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[This is to get the shared cache upload policies for archives.
+
+ @param conf Configuration which stores the shared cache upload policies
+ @return A map containing the shared cache upload policies for a set of
+         resources. The key is the url of the resource and the value is the
+         upload policy. True if it should be uploaded, false otherwise.]]>
+      </doc>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and return immediately.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="waitForCompletion" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="verbose" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Submit the job to the cluster and wait for it to finish.
+ @param verbose print the progress to the user
+ @return true if the job succeeded
+ @throws IOException thrown if the communication with the 
+         <code>JobTracker</code> is lost]]>
+      </doc>
+    </method>
+    <method name="monitorAndPrintJob" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Monitor a job and print status in real-time as progress is made and tasks 
+ fail.
+ @return true if the job succeeded
+ @throws IOException if communication to the JobTracker fails]]>
+      </doc>
+    </method>
+    <method name="getProgressPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which monitorAndPrintJob() prints status]]>
+      </doc>
+    </method>
+    <method name="getCompletionPollInterval" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[The interval at which waitForCompletion() should check.]]>
+      </doc>
+    </method>
+    <method name="getTaskOutputFilter" return="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the task output filter.
+ 
+ @param conf the configuration.
+ @return the filter level.]]>
+      </doc>
+    </method>
+    <method name="setTaskOutputFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="newValue" type="org.apache.hadoop.mapreduce.Job.TaskStatusFilter"/>
+      <doc>
+      <![CDATA[Modify the Configuration to set the task output filter.
+ 
+ @param conf the Configuration to modify.
+ @param newValue the value to set.]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation to which the job is submitted to, if any
+
+ @return the reservationId the identifier of the job's reservation, null if
+         the job does not have any reservation associated with it]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation to which the job is submitted to
+
+ @param reservationId the reservationId to set]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the <code>Job</code>.
+ @throws IOException if fail to close.]]>
+      </doc>
+    </method>
+    <field name="OUTPUT_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPLETION_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets completionPollInvervalMillis]]>
+      </doc>
+    </field>
+    <field name="PROGRESS_MONITOR_POLL_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Key in mapred-*.xml that sets progMonitorPollIntervalMillis]]>
+      </doc>
+    </field>
+    <field name="USED_GENERIC_PARSER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUBMIT_REPLICATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SUBMIT_REPLICATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USE_WILDCARD_FOR_LIBJARS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_USE_WILDCARD_FOR_LIBJARS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The job submitter's view of the Job.
+ 
+ <p>It allows the user to configure the
+ job, submit it, control its execution, and query the state. The set methods
+ only work until the job is submitted, afterwards they will throw an 
+ IllegalStateException. </p>
+ 
+ <p>
+ Normally the user creates the application, describes various facets of the
+ job via {@link Job} and then submits the job and monitor its progress.</p>
+ 
+ <p>Here is an example on how to submit a job:</p>
+ <p><blockquote><pre>
+     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+ </pre></blockquote>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Job -->
+  <!-- start interface org.apache.hadoop.mapreduce.JobContext -->
+  <interface name="JobContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MRJobConfig"/>
+    <method name="getConfiguration" return="org.apache.hadoop.conf.Configuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the configuration for the job.
+ @return the shared configuration object]]>
+      </doc>
+    </method>
+    <method name="getCredentials" return="org.apache.hadoop.security.Credentials"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get credentials for the job.
+ @return credentials for the job]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique ID for the job.
+ @return the object with the job id]]>
+      </doc>
+    </method>
+    <method name="getNumReduceTasks" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get configured the number of reduce tasks for this job. Defaults to 
+ <code>1</code>.
+ @return the number of reduce tasks for this job.]]>
+      </doc>
+    </method>
+    <method name="getWorkingDirectory" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the current working directory for the default file system.
+ 
+ @return the directory name.]]>
+      </doc>
+    </method>
+    <method name="getOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the job output data.
+ @return the key class for the job output data.]]>
+      </doc>
+    </method>
+    <method name="getOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for job outputs.
+ @return the value class for job outputs.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputKeyClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the key class for the map output data. If it is not set, use the
+ (final) output key class. This allows the map output key class to be
+ different than the final output key class.
+ @return the map output key class.]]>
+      </doc>
+    </method>
+    <method name="getMapOutputValueClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the value class for the map output data. If it is not set, use the
+ (final) output value class This allows the map output value class to be
+ different than the final output value class.
+  
+ @return the map output value class.]]>
+      </doc>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name. This is only used to identify the 
+ job to the user.
+ 
+ @return the job's name, defaulting to "".]]>
+      </doc>
+    </method>
+    <method name="getInputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link InputFormat} class for the job.
+ 
+ @return the {@link InputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Mapper} class for the job.
+ 
+ @return the {@link Mapper} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getCombinerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the combiner class for the job.
+ 
+ @return the combiner class for the job.]]>
+      </doc>
+    </method>
+    <method name="getReducerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Reducer} class for the job.
+ 
+ @return the {@link Reducer} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getOutputFormatClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link OutputFormat} class for the job.
+ 
+ @return the {@link OutputFormat} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getPartitionerClass" return="java.lang.Class"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[Get the {@link Partitioner} class for the job.
+ 
+ @return the {@link Partitioner} class for the job.]]>
+      </doc>
+    </method>
+    <method name="getSortComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link RawComparator} comparator used to compare keys.
+ 
+ @return the {@link RawComparator} comparator used to compare keys.]]>
+      </doc>
+    </method>
+    <method name="getJar" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pathname of the job's jar.
+ @return the pathname]]>
+      </doc>
+    </method>
+    <method name="getCombinerKeyGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the combiner.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setCombinerKeyGroupingComparatorClass(Class)]]>
+      </doc>
+    </method>
+    <method name="getGroupingComparator" return="org.apache.hadoop.io.RawComparator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user defined {@link RawComparator} comparator for
+ grouping keys of inputs to the reduce.
+
+ @return comparator set by the user for grouping values.
+ @see Job#setGroupingComparatorClass(Class)
+ @see #getCombinerKeyGroupingComparator()]]>
+      </doc>
+    </method>
+    <method name="getJobSetupCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether job-setup and job-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getTaskCleanupNeeded" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether task-cleanup is needed for the job 
+ 
+ @return boolean]]>
+      </doc>
+    </method>
+    <method name="getProfileEnabled" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether the task profiling is enabled.
+ @return true if some tasks will be profiled]]>
+      </doc>
+    </method>
+    <method name="getProfileParams" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the profiler configuration arguments.
+
+ The default value for this property is
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s"
+ 
+ @return the parameters to pass to the task child to configure profiling]]>
+      </doc>
+    </method>
+    <method name="getProfileTaskRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isMap" type="boolean"/>
+      <doc>
+      <![CDATA[Get the range of maps or reduces to profile.
+ @param isMap is the task a map?
+ @return the task ranges]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reported username for this job.
+ 
+ @return the username]]>
+      </doc>
+    </method>
+    <method name="getSymlink" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Originally intended to check if symlinks should be used, but currently
+ symlinks cannot be disabled.
+ @return true]]>
+      </doc>
+    </method>
+    <method name="getArchiveClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the archive entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getCacheArchives" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache archives set in the Configuration
+ @return A URI array of the caches set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getCacheFiles" return="java.net.URI[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get cache files set in the Configuration
+ @return A URI array of the files set in the Configuration
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheArchives" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized caches
+ @return A path array of localized caches
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheArchives()}.]]>
+      </doc>
+    </method>
+    <method name="getLocalCacheFiles" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the path array of the localized files
+ @return A path array of localized files
+ @throws IOException
+ @deprecated the array returned only includes the items the were 
+ downloaded. There is no way to map this to what is returned by
+ {@link #getCacheFiles()}.]]>
+      </doc>
+    </method>
+    <method name="getFileClassPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file entries in classpath as an array of Path]]>
+      </doc>
+    </method>
+    <method name="getArchiveTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the archives.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getFileTimestamps" return="java.lang.String[]"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamps of the files.  Used by internal
+ DistributedCache and MapReduce code.
+ @return a string array of timestamps]]>
+      </doc>
+    </method>
+    <method name="getMaxMapAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts that will be made to run a
+ map task, as specified by the <code>mapred.map.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+  
+ @return the max number of attempts per map task.]]>
+      </doc>
+    </method>
+    <method name="getMaxReduceAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configured number of maximum attempts  that will be made to run a
+ reduce task, as specified by the <code>mapred.reduce.max.attempts</code>
+ property. If this property is not already set, the default is 4 attempts.
+ 
+ @return the max number of attempts per reduce task.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A read-only view of the job that is provided to the tasks while they
+ are running.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.JobContext -->
+  <!-- start class org.apache.hadoop.mapreduce.JobCounter -->
+  <class name="JobCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.JobID -->
+  <class name="JobID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="JobID" type="java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a JobID object 
+ @param jtIdentifier jobTracker identifier
+ @param id job number]]>
+      </doc>
+    </constructor>
+    <constructor name="JobID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJtIdentifier" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare JobIds by first jtIdentifiers, then by job numbers]]>
+      </doc>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the stuff after the "job" prefix to the given builder. This is useful,
+ because the sub-ids use this substring at the start of their string.
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a JobId object from given string 
+ @return constructed JobId object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="JOB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOBID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[JobID represents the immutable and unique identifier for 
+ the job. JobID consists of two parts. First part 
+ represents the jobtracker identifier, so that jobID to jobtracker map 
+ is defined. For cluster setup this string is the jobtracker 
+ start time, for local setting, it is "local" and a random number.
+ Second part of the JobID is the job number. <br> 
+ An example JobID is : 
+ <code>job_200707121733_0003</code> , which represents the third job 
+ running at the jobtracker started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse JobID strings, but rather 
+ use appropriate constructors or {@link #forName(String)} method. 
+ 
+ @see TaskID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobID -->
+  <!-- start class org.apache.hadoop.mapreduce.JobPriority -->
+  <class name="JobPriority" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.JobPriority[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Used to describe the priority of the running job. 
+ DEFAULT : While submitting a job, if the user is not specifying priority,
+ YARN has the capability to pick the default priority as per its config.
+ Hence MapReduce can indicate such cases with this new enum.
+ UNDEFINED_PRIORITY : YARN supports priority as an integer. Hence other than
+ the five defined enums, YARN can consider other integers also. To generalize
+ such cases, this specific enum is used.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobPriority -->
+  <!-- start class org.apache.hadoop.mapreduce.JobStatus -->
+  <class name="JobStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Cloneable"/>
+    <constructor name="JobStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode]]>
+      </doc>
+    </constructor>
+    <constructor name="JobStatus" type="org.apache.hadoop.mapreduce.JobID, float, float, float, float, org.apache.hadoop.mapreduce.JobStatus.State, org.apache.hadoop.mapreduce.JobPriority, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a job status object for a given jobid.
+ @param jobid The jobid of the job
+ @param setupProgress The progress made on the setup
+ @param mapProgress The progress made on the maps
+ @param reduceProgress The progress made on the reduces
+ @param cleanupProgress The progress made on the cleanup
+ @param runState The current state of the job
+ @param jp Priority of the job.
+ @param user userid of the person who submitted the job.
+ @param jobName user-specified job name.
+ @param queue queue name
+ @param jobFile job configuration file.
+ @param trackingUrl link to the web-ui for details of the job.
+ @param isUber Whether job running in uber mode
+ @param historyFile history file]]>
+      </doc>
+    </constructor>
+    <method name="setMapProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the map progress of this job
+ @param p The value of map progress to set to]]>
+      </doc>
+    </method>
+    <method name="setCleanupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the cleanup progress of this job
+ @param p The value of cleanup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setSetupProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the setup progress of this job
+ @param p The value of setup progress to set to]]>
+      </doc>
+    </method>
+    <method name="setReduceProgress"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="p" type="float"/>
+      <doc>
+      <![CDATA[Sets the reduce progress of this Job
+ @param p The value of reduce progress to set to]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="jp" type="org.apache.hadoop.mapreduce.JobPriority"/>
+      <doc>
+      <![CDATA[Set the priority of the job, defaulting to NORMAL.
+ @param jp new job priority]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+      <doc>
+      <![CDATA[Set the finish time of the job
+ @param finishTime The finishTime of the job]]>
+      </doc>
+    </method>
+    <method name="setHistoryFile"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="historyFile" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job history file url for a completed job]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="setRetired"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the job retire flag to true.]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <doc>
+      <![CDATA[Change the current run state of the job.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time of the job
+ @param startTime The startTime of the job]]>
+      </doc>
+    </method>
+    <method name="setUsername"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="userName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param userName The username of the job]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Used to set the scheduling information associated to a particular Job.
+ 
+ @param schedulingInfo Scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="setJobACLs"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the job acls.
+ 
+ @param acls {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name
+ @param queue queue name]]>
+      </doc>
+    </method>
+    <method name="setFailureInfo"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="failureInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set diagnostic information.
+ @param failureInfo diagnostic information]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="getMapProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in maps]]>
+      </doc>
+    </method>
+    <method name="getCleanupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in cleanup]]>
+      </doc>
+    </method>
+    <method name="getSetupProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in setup]]>
+      </doc>
+    </method>
+    <method name="getReduceProgress" return="float"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Percentage of progress in reduce]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.JobStatus.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return running state of the job]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return start time of the job]]>
+      </doc>
+    </method>
+    <method name="clone" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return The jobid of the Job]]>
+      </doc>
+    </method>
+    <method name="getUsername" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the username of the job]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the Scheduling information associated to a particular Job.
+ @return the scheduling information of the job]]>
+      </doc>
+    </method>
+    <method name="getJobACLs" return="java.util.Map"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the job acls.
+ 
+ @return a {@link Map} from {@link JobACL} to {@link AccessControlList}]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.mapreduce.JobPriority"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the priority of the job
+ @return job priority]]>
+      </doc>
+    </method>
+    <method name="getFailureInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets any available info on the reason of failure of the job.
+ @return diagnostic information on why a job might have failed.]]>
+      </doc>
+    </method>
+    <method name="isJobComplete" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns true if the status is for a completed job.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-specified job name.]]>
+      </doc>
+    </method>
+    <method name="getJobFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration file for the job.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the link to the web-ui for details of the job.]]>
+      </doc>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the finish time of the job.]]>
+      </doc>
+    </method>
+    <method name="isRetired" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check whether the job has retired.]]>
+      </doc>
+    </method>
+    <method name="getHistoryFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job history file name for a completed job. If job is not 
+ completed or history file not available then return null.]]>
+      </doc>
+    </method>
+    <method name="getNumUsedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="setNumUsedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n number of used mapred slots]]>
+      </doc>
+    </method>
+    <method name="getNumReservedSlots" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="setNumReservedSlots"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the number of reserved slots]]>
+      </doc>
+    </method>
+    <method name="getUsedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the used memory]]>
+      </doc>
+    </method>
+    <method name="setUsedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="m" type="int"/>
+      <doc>
+      <![CDATA[@param m the used memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="int"/>
+      <doc>
+      <![CDATA[@param r the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getNeededMem" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the needed memory]]>
+      </doc>
+    </method>
+    <method name="setNeededMem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="int"/>
+      <doc>
+      <![CDATA[@param n the needed memory]]>
+      </doc>
+    </method>
+    <method name="isUber" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether job running in uber mode
+ @return job in uber-mode]]>
+      </doc>
+    </method>
+    <method name="setUber"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isUber" type="boolean"/>
+      <doc>
+      <![CDATA[Set uber-mode flag 
+ @param isUber Whether job running in uber-mode]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Describes the current status of a job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.JobStatus -->
+  <!-- start interface org.apache.hadoop.mapreduce.MapContext -->
+  <interface name="MapContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="getInputSplit" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the input split for this map.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context that is given to the {@link Mapper}.
+ @param <KEYIN> the key input type to the Mapper
+ @param <VALUEIN> the value input type to the Mapper
+ @param <KEYOUT> the key output type from the Mapper
+ @param <VALUEOUT> the value output type from the Mapper]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.MapContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Mapper -->
+  <class name="Mapper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Mapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the beginning of the task.]]>
+      </doc>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="value" type="VALUEIN"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once for each key/value pair in the input split. Most applications
+ should override this, but the default is the identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Expert users can override this method for more complete control over the
+ execution of the Mapper.
+ @param context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Maps input key/value pairs to a set of intermediate key/value pairs.  
+ 
+ <p>Maps are the individual tasks which transform input records into a 
+ intermediate records. The transformed intermediate records need not be of 
+ the same type as the input records. A given input pair may map to zero or 
+ many output pairs.</p> 
+ 
+ <p>The Hadoop Map-Reduce framework spawns one map task for each 
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ <code>Mapper</code> implementations can access the {@link Configuration} for 
+ the job via the {@link JobContext#getConfiguration()}.
+ 
+ <p>The framework first calls 
+ {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by
+ {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)}
+ for each key/value pair in the <code>InputSplit</code>. Finally 
+ {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.</p>
+ 
+ <p>All intermediate values associated with a given output key are 
+ subsequently grouped by the framework, and passed to a {@link Reducer} to  
+ determine the final output. Users can control the sorting and grouping by 
+ specifying two key {@link RawComparator} classes.</p>
+
+ <p>The <code>Mapper</code> outputs are partitioned per 
+ <code>Reducer</code>. Users can control which keys (and hence records) go to 
+ which <code>Reducer</code> by implementing a custom {@link Partitioner}.
+ 
+ <p>Users can optionally specify a <code>combiner</code>, via 
+ {@link Job#setCombinerClass(Class)}, to perform local aggregation of the 
+ intermediate outputs, which helps to cut down the amount of data transferred 
+ from the <code>Mapper</code> to the <code>Reducer</code>.
+ 
+ <p>Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the <code>Configuration</code>.</p>
+  
+ <p>If the job has zero
+ reduces then the output of the <code>Mapper</code> is directly written
+ to the {@link OutputFormat} without sorting by keys.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class TokenCounterMapper 
+     extends Mapper&lt;Object, Text, Text, IntWritable&gt;{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+ </pre></blockquote>
+
+ <p>Applications may override the
+ {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert
+ greater control on map processing e.g. multi-threaded <code>Mapper</code>s 
+ etc.</p>
+ 
+ @see InputFormat
+ @see JobContext
+ @see Partitioner  
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Mapper -->
+  <!-- start class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <class name="MarkableIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.MarkableIteratorInterface"/>
+    <constructor name="MarkableIterator" type="java.util.Iterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new iterator layered on the input iterator
+ @param itr underlying iterator that implements MarkableIteratorInterface]]>
+      </doc>
+    </constructor>
+    <method name="mark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clearMark"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="VALUE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<code>MarkableIterator</code> is a wrapper iterator class that 
+ implements the {@link MarkableIteratorInterface}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.MarkableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <class name="OutputCommitter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputCommitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setupJob"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For the framework to setup the job output during initialization.  This is
+ called from the application master process for the entire job. This will be
+ called multiple times, once per job attempt.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException if temporary output could not be created]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For cleaning up the job's output after job completion.  This is called
+ from the application master process for the entire job. This may be called
+ multiple times.
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException
+ @deprecated Use {@link #commitJob(JobContext)} and
+                 {@link #abortJob(JobContext, JobStatus.State)} instead.]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For committing job's output after successful job completion. Note that this
+ is invoked for jobs with final runstate as SUCCESSFUL.  This is called
+ from the application master process for the entire job. This is guaranteed
+ to only be called once.  If it throws an exception the entire job will
+ fail.	
+ 
+ @param jobContext Context of the job whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For aborting an unsuccessful job's output. Note that this is invoked for 
+ jobs with final runstate as {@link JobStatus.State#FAILED} or 
+ {@link JobStatus.State#KILLED}.  This is called from the application
+ master process for the entire job. This may be called multiple times.
+
+ @param jobContext Context of the job whose output is being written.
+ @param state final runstate of the job
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets up output for the task.  This is called from each individual task's
+ process that will output to HDFS, and it is called just for that task. This
+ may be called multiple times for the same task, but for different task
+ attempts.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Check whether task needs a commit.  This is called from each individual
+ task's process that will output to HDFS, and it is called just for that
+ task.
+ 
+ @param taskContext
+ @return true/false
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[To promote the task's temporary output to final output location.
+ If {@link #needsTaskCommit(TaskAttemptContext)} returns true and this
+ task is the task that the AM determines finished first, this method
+ is called to commit an individual task's output.  This is to mark
+ that tasks output as complete, as {@link #commitJob(JobContext)} will 
+ also be called later on if the entire job finished successfully. This
+ is called from a task's process. This may be called multiple times for the
+ same task, but different task attempts.  It should be very rare for this to
+ be called multiple times and requires odd networking failures to make this
+ happen. In the future the Hadoop framework may eliminate this race.
+ 
+ @param taskContext Context of the task whose output is being written.
+ @throws IOException if commit is not successful.]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Discard the task output. This is called from a task's process to clean 
+ up a single task's output that can not yet been committed. This may be
+ called multiple times for the same task, but for different task attempts.
+ 
+ @param taskContext
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #isRecoverySupported(JobContext)} instead.">
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @see #recoverTask(TaskAttemptContext)
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+      </doc>
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns true if an in-progress job commit can be retried. If the MR AM is
+ re-run then it will check this value to determine if it can retry an
+ in-progress commit that was started by a previous version.
+ Note that in rare scenarios, the previous AM version might still be running
+ at that time, due to system anomalies. Hence if this method returns true
+ then the retry commit operation should be able to run concurrently with
+ the previous operation.
+
+ If repeatable job commit is supported, job restart can tolerate previous
+ AM failures during job commit.
+
+ By default, it is not supported. Extended classes (like:
+ FileOutputCommitter) should explicitly override it if provide support.
+
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> repeatable job commit is supported,
+         <code>false</code> otherwise
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Is task output recovery supported for restarting jobs?
+ 
+ If task output recovery is supported, job restart can be done more
+ efficiently.
+ 
+ @param jobContext
+          Context of the job whose output is being written.
+ @return <code>true</code> if task output recovery is supported,
+         <code>false</code> otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+      </doc>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Recover the task output. 
+ 
+ The retry-count for the job will be passed via the 
+ {@link MRJobConfig#APPLICATION_ATTEMPT_ID} key in  
+ {@link TaskAttemptContext#getConfiguration()} for the 
+ <code>OutputCommitter</code>.  This is called from the application master
+ process, but it is called individually for each task.
+ 
+ If an exception is thrown the task will be attempted again. 
+ 
+ This may be called multiple times for the same task.  But from different
+ application attempts.
+ 
+ @param taskContext Context of the task whose output is being recovered
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputCommitter</code> describes the commit of task output for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputCommitter</code> of 
+ the job to:<p>
+ <ol>
+   <li>
+   Setup the job during initialization. For example, create the temporary 
+   output directory for the job during the initialization of the job.
+   </li>
+   <li>
+   Cleanup the job after the job completion. For example, remove the
+   temporary output directory after the job completion. 
+   </li>
+   <li>
+   Setup the task temporary output.
+   </li> 
+   <li>
+   Check whether a task needs a commit. This is to avoid the commit
+   procedure if a task does not need commit.
+   </li>
+   <li>
+   Commit of the task output.
+   </li>  
+   <li>
+   Discard the task commit.
+   </li>
+ </ol>
+ The methods in this class can be called from several different processes and
+ from several different contexts.  It is important to know which process and
+ which context each is called from.  Each method should be marked accordingly
+ in its documentation.  It is also important to note that not all methods are
+ guaranteed to be called once and only once.  If a method is not guaranteed to
+ have this property the output committer needs to handle this appropriately. 
+ Also note it will only be in rare situations where they may be called 
+ multiple times for the same task.
+ 
+ @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter 
+ @see JobContext
+ @see TaskAttemptContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.OutputFormat -->
+  <class name="OutputFormat" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link RecordWriter} for the given task.
+
+ @param context the information about the current task.
+ @return a {@link RecordWriter} to write the output for the job.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Check for validity of the output-specification for the job.
+  
+ <p>This is to validate the output specification for the job when it is
+ a job is submitted.  Typically checks that it does not already exist,
+ throwing an exception when it already exists, so that output is not
+ overwritten.</p>
+
+ Implementations which write to filesystems which support delegation
+ tokens usually collect the tokens for the destination path(s)
+ and attach them to the job context's JobConf.
+ @param context information about the job
+ @throws IOException when output should not be attempted]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the output committer for this output format. This is responsible
+ for ensuring the output is committed correctly.
+ @param context the task context
+ @return an output committer
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>OutputFormat</code> describes the output-specification for a 
+ Map-Reduce job.
+
+ <p>The Map-Reduce framework relies on the <code>OutputFormat</code> of the
+ job to:<p>
+ <ol>
+   <li>
+   Validate the output-specification of the job. For e.g. check that the 
+   output directory doesn't already exist. 
+   <li>
+   Provide the {@link RecordWriter} implementation to be used to write out
+   the output files of the job. Output files are stored in a 
+   {@link FileSystem}.
+   </li>
+ </ol>
+ 
+ @see RecordWriter]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.OutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.Partitioner -->
+  <class name="Partitioner" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Partitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="value" type="VALUE"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Get the partition number for a given key (hence record) given the total 
+ number of partitions i.e. number of reduce-tasks for the job.
+   
+ <p>Typically a hash function on a all or a subset of the key.</p>
+
+ @param key the key to be partioned.
+ @param value the entry value.
+ @param numPartitions the total number of partitions.
+ @return the partition number for the <code>key</code>.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partitions the key space.
+ 
+ <p><code>Partitioner</code> controls the partitioning of the keys of the 
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the <code>m</code> reduce tasks the intermediate key (and hence the 
+ record) is sent for reduction.</p>
+
+ <p>Note: A <code>Partitioner</code> is created only when there are multiple
+ reducers.</p>
+
+ <p>Note: If you require your Partitioner class to obtain the Job's
+ configuration object, implement the {@link Configurable} interface.</p>
+ 
+ @see Reducer]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Partitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <class name="QueueAclsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueAclsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueAclsInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueAclsInfo" type="java.lang.String, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueAclsInfo object using the queue name and the
+ queue operations array
+ 
+ @param queueName Name of the job queue
+ @param operations]]>
+      </doc>
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name.
+ 
+ @return name]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+    </method>
+    <method name="getOperations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get opearations allowed on queue.
+ 
+ @return array of String]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class to encapsulate Queue ACLs for a particular
+  user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueAclsInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for QueueInfo.]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a new QueueInfo object using the queue name and the
+ scheduling information passed.
+ 
+ @param queueName Name of the job queue
+ @param schedulingInfo Scheduling Information associated with the job
+ queue]]>
+      </doc>
+    </constructor>
+    <constructor name="QueueInfo" type="java.lang.String, java.lang.String, org.apache.hadoop.mapreduce.QueueState, org.apache.hadoop.mapreduce.JobStatus[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param queueName
+ @param schedulingInfo
+ @param state
+ @param stats]]>
+      </doc>
+    </constructor>
+    <method name="setQueueName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the queue name of the JobQueueInfo
+ 
+ @param queueName Name of the job queue.]]>
+      </doc>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue name from JobQueueInfo
+ 
+ @return queue name]]>
+      </doc>
+    </method>
+    <method name="setSchedulingInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="schedulingInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheduling information associated to particular job queue
+ 
+ @param schedulingInfo]]>
+      </doc>
+    </method>
+    <method name="getSchedulingInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the scheduling information associated to particular job queue.
+ If nothing is set would return <b>"N/A"</b>
+ 
+ @return Scheduling information associated to particular Job Queue]]>
+      </doc>
+    </method>
+    <method name="setState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.QueueState"/>
+      <doc>
+      <![CDATA[Set the state of the queue
+ @param state state of the queue.]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the queue state
+ @return the queue state.]]>
+      </doc>
+    </method>
+    <method name="setJobStatuses"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stats" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+    </method>
+    <method name="getQueueChildren" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get immediate children.
+ 
+ @return list of QueueInfo]]>
+      </doc>
+    </method>
+    <method name="setQueueChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.List"/>
+    </method>
+    <method name="getProperties" return="java.util.Properties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get properties.
+ 
+ @return Properties]]>
+      </doc>
+    </method>
+    <method name="setProperties"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="props" type="java.util.Properties"/>
+    </method>
+    <method name="getJobStatuses" return="org.apache.hadoop.mapreduce.JobStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the jobs submitted to queue
+ @return list of JobStatus for the submitted jobs]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Class that contains the information regarding the Job Queues which are 
+ maintained by the Hadoop Map/Reduce framework.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getStateName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the stateName]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.mapreduce.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="state" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Enum representing queue state]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.QueueState -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordReader -->
+  <class name="RecordReader" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Closeable"/>
+    <constructor name="RecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at initialization.
+ @param split the split that defines the range of records to read
+ @param context the information about the task
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next key, value pair.
+ @return true if a key/value pair was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key
+ @return the current key or null if there is no current key
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the object that was read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The current progress of the record reader through its data.
+ @return a number between 0.0 and 1.0 that is the fraction of the data read
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close the record reader.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The record reader breaks the data into key/value pairs for input to the
+ {@link Mapper}.
+ @param <KEYIN>
+ @param <VALUEIN>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.RecordWriter -->
+  <class name="RecordWriter" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RecordWriter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Writes a key/value pair.
+
+ @param key the key to write.
+ @param value the value to write.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Close this <code>RecordWriter</code> to future operations.
+ 
+ @param context the context of the task
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<code>RecordWriter</code> writes the output &lt;key, value&gt; pairs 
+ to an output file.
+ 
+ <p><code>RecordWriter</code> implementations write the job outputs to the
+ {@link FileSystem}.
+ 
+ @see OutputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.RecordWriter -->
+  <!-- start interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <interface name="ReduceContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+    <method name="nextKey" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Start processing next unique key.]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Iterate through the values for the current key, reusing the same value 
+ object, which is stored in the context.
+ @return the series of values associated with the current key. All of the 
+ objects returned directly and indirectly from this method are reused.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context passed to the {@link Reducer}.
+ @param <KEYIN> the class of the input keys
+ @param <VALUEIN> the class of the input values
+ @param <KEYOUT> the class of the output keys
+ @param <VALUEOUT> the class of the output values]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.ReduceContext -->
+  <!-- start class org.apache.hadoop.mapreduce.Reducer -->
+  <class name="Reducer" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Reducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the start of the task.]]>
+      </doc>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="key" type="KEYIN"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[This method is called once for each key. Most applications will define
+ their reduce class by overriding this method. The default implementation
+ is an identity function.]]>
+      </doc>
+    </method>
+    <method name="cleanup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Called once at the end of the task.]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advanced application writers can use the 
+ {@link #run(org.apache.hadoop.mapreduce.Reducer.Context)} method to
+ control how the reduce task works.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Reduces a set of intermediate values which share a key to a smaller set of
+ values.  
+ 
+ <p><code>Reducer</code> implementations 
+ can access the {@link Configuration} for the job via the 
+ {@link JobContext#getConfiguration()} method.</p>
+
+ <p><code>Reducer</code> has 3 primary phases:</p>
+ <ol>
+   <li>
+   
+   <b id="Shuffle">Shuffle</b>
+   
+   <p>The <code>Reducer</code> copies the sorted output from each 
+   {@link Mapper} using HTTP across the network.</p>
+   </li>
+   
+   <li>
+   <b id="Sort">Sort</b>
+   
+   <p>The framework merge sorts <code>Reducer</code> inputs by 
+   <code>key</code>s 
+   (since different <code>Mapper</code>s may have output the same key).</p>
+   
+   <p>The shuffle and sort phases occur simultaneously i.e. while outputs are
+   being fetched they are merged.</p>
+      
+   <b id="SecondarySort">SecondarySort</b>
+   
+   <p>To achieve a secondary sort on the values returned by the value 
+   iterator, the application should extend the key with the secondary
+   key and define a grouping comparator. The keys will be sorted using the
+   entire key, but will be grouped using the grouping comparator to decide
+   which keys and values are sent in the same call to reduce.The grouping 
+   comparator is specified via 
+   {@link Job#setGroupingComparatorClass(Class)}. The sort order is
+   controlled by 
+   {@link Job#setSortComparatorClass(Class)}.</p>
+   
+   
+   For example, say that you want to find duplicate web pages and tag them 
+   all with the url of the "best" known example. You would set up the job 
+   like:
+   <ul>
+     <li>Map Input Key: url</li>
+     <li>Map Input Value: document</li>
+     <li>Map Output Key: document checksum, url pagerank</li>
+     <li>Map Output Value: url</li>
+     <li>Partitioner: by checksum</li>
+     <li>OutputKeyComparator: by checksum and then decreasing pagerank</li>
+     <li>OutputValueGroupingComparator: by checksum</li>
+   </ul>
+   </li>
+   
+   <li>   
+   <b id="Reduce">Reduce</b>
+   
+   <p>In this phase the 
+   {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)}
+   method is called for each <code>&lt;key, (collection of values)&gt;</code> in
+   the sorted inputs.</p>
+   <p>The output of the reduce task is typically written to a 
+   {@link RecordWriter} via 
+   {@link Context#write(Object, Object)}.</p>
+   </li>
+ </ol>
+ 
+ <p>The output of the <code>Reducer</code> is <b>not re-sorted</b>.</p>
+ 
+ <p>Example:</p>
+ <p><blockquote><pre>
+ public class IntSumReducer&lt;Key&gt; extends Reducer&lt;Key,IntWritable,
+                                                 Key,IntWritable&gt; {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable&lt;IntWritable&gt; values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+ </pre></blockquote>
+ 
+ @see Mapper
+ @see Partitioner]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.Reducer -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <interface name="TaskAttemptContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.JobContext"/>
+    <implements name="org.apache.hadoop.util.Progressable"/>
+    <method name="getTaskAttemptID" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the unique name for this task attempt.]]>
+      </doc>
+    </method>
+    <method name="setStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="msg" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the current status of the task to the given string.]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the last set status message.
+ @return the current status message]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The current progress of the task attempt.
+ @return a number between 0.0 and 1.0 (inclusive) indicating the attempt's
+ progress.]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>counterName</code>]]>
+      </doc>
+    </method>
+    <method name="getCounter" return="org.apache.hadoop.mapreduce.Counter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the {@link Counter} for the given <code>groupName</code> and 
+ <code>counterName</code>.
+ @param counterName counter name
+ @return the <code>Counter</code> for the given <code>groupName</code> and 
+         <code>counterName</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The context for task attempts.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskAttemptContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <class name="TaskAttemptID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskAttemptID" type="org.apache.hadoop.mapreduce.TaskID, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskAttemptID object from given {@link TaskID}.  
+ @param taskId TaskID that this task belongs to  
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID" type="java.lang.String, int, boolean, int, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskId object from given parts.
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param taskId taskId number
+ @param id the task attempt number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskAttemptID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="getTaskID" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link TaskID} object that this task attempt belongs to]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the TaskType of the TaskAttemptID]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the StringBuilder
+ @param builder the builder to append ot
+ @return the builder that was passed in.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskIds by first tipIds, then by task numbers.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskAttemptID object from given string 
+ @return constructed TaskAttemptID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <field name="ATTEMPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskAttemptID represents the immutable and unique identifier for 
+ a task attempt. Each task attempt is one particular instance of a Map or
+ Reduce Task identified by its TaskID. 
+ 
+ TaskAttemptID consists of 2 parts. First part is the 
+ {@link TaskID}, that this TaskAttemptID belongs to.
+ Second part is the task attempt number. <br> 
+ An example TaskAttemptID is : 
+ <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ zeroth task attempt for the fifth map task in the third job 
+ running at the jobtracker started at <code>200707121733</code>.
+ <p>
+ Applications should never construct or parse TaskAttemptID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskAttemptID -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <class name="TaskCompletionEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskCompletionEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable.]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskCompletionEvent" type="int, org.apache.hadoop.mapreduce.TaskAttemptID, int, boolean, org.apache.hadoop.mapreduce.TaskCompletionEvent.Status, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor. eventId should be created externally and incremented
+ per event for each job. 
+ @param eventId event id, event id should be unique and assigned in
+  incrementally, starting from 0. 
+ @param taskId task id
+ @param status task's status 
+ @param taskTrackerHttp task tracker's host:port for http.]]>
+      </doc>
+    </constructor>
+    <method name="getEventId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns event Id. 
+ @return event id]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.TaskAttemptID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns task id. 
+ @return task id]]>
+      </doc>
+    </method>
+    <method name="getStatus" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns {@link Status}
+ @return task completion status]]>
+      </doc>
+    </method>
+    <method name="getTaskTrackerHttp" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[http location of the tasktracker where this task ran. 
+ @return http location of tasktracker user logs]]>
+      </doc>
+    </method>
+    <method name="getTaskRunTime" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns time (in millisec) the task took to complete.]]>
+      </doc>
+    </method>
+    <method name="setTaskRunTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskCompletionTime" type="int"/>
+      <doc>
+      <![CDATA[Set the task completion time
+ @param taskCompletionTime time (in millisec) the task took to complete]]>
+      </doc>
+    </method>
+    <method name="setEventId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="eventId" type="int"/>
+      <doc>
+      <![CDATA[set event Id. should be assigned incrementally starting from 0. 
+ @param eventId]]>
+      </doc>
+    </method>
+    <method name="setTaskAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <doc>
+      <![CDATA[Sets task id. 
+ @param taskId]]>
+      </doc>
+    </method>
+    <method name="setTaskStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"/>
+      <doc>
+      <![CDATA[Set task status. 
+ @param status]]>
+      </doc>
+    </method>
+    <method name="setTaskTrackerHttp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskTrackerHttp" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set task tracker http location. 
+ @param taskTrackerHttp]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isMapTask" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="idWithinJob" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="EMPTY_ARRAY" type="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is used to track task completion events on 
+ job tracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <class name="TaskCompletionEvent.Status" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCompletionEvent.Status"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCompletionEvent.Status -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskCounter -->
+  <class name="TaskCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskID -->
+  <class name="TaskID" extends="org.apache.hadoop.mapred.ID"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to 
+ @param type the {@link TaskType} of the task 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, org.apache.hadoop.mapreduce.TaskType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number 
+ @param type the TaskType 
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="org.apache.hadoop.mapreduce.JobID, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskID object from given {@link JobID}.
+
+ @param jobId JobID that this tip belongs to
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID" type="java.lang.String, int, boolean, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a TaskInProgressId object from given parts.
+
+ @param jtIdentifier jobTracker identifier
+ @param jobId job number
+ @param isMap whether the tip is a map
+ @param id the tip number]]>
+      </doc>
+    </constructor>
+    <constructor name="TaskID"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor for Writable. Sets the task type to
+ {@link TaskType#REDUCE}, the ID to 0, and the job ID to an empty job ID.]]>
+      </doc>
+    </constructor>
+    <method name="getJobID" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the {@link JobID} object that this tip belongs to.
+
+ @return the JobID object]]>
+      </doc>
+    </method>
+    <method name="isMap" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns whether this TaskID is a map ID.
+
+ @return whether this TaskID is a map ID]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the type of the task.
+
+ @return the type of the task]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="org.apache.hadoop.mapreduce.ID"/>
+      <doc>
+      <![CDATA[Compare TaskInProgressIds by first jobIds, then by tip numbers.
+ Reducers are defined as greater than mappers.
+
+ @param o the TaskID against which to compare
+ @return 0 if equal, positive if this TaskID is greater, and negative if
+ this TaskID is less]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="appendTo" return="java.lang.StringBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="builder" type="java.lang.StringBuilder"/>
+      <doc>
+      <![CDATA[Add the unique string to the given builder.
+
+ @param builder the builder to append to
+ @return the builder that was passed in]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="forName" return="org.apache.hadoop.mapreduce.TaskID"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="str" type="java.lang.String"/>
+      <exception name="IllegalArgumentException" type="java.lang.IllegalArgumentException"/>
+      <doc>
+      <![CDATA[Construct a TaskID object from given string.
+
+ @param str the target string
+ @return constructed TaskID object or null if the given String is null
+ @throws IllegalArgumentException if the given string is malformed]]>
+      </doc>
+    </method>
+    <method name="getRepresentingCharacter" return="char"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.mapreduce.TaskType"/>
+      <doc>
+      <![CDATA[Gets the character representing the {@link TaskType}.
+
+ @param type the TaskType
+ @return the character]]>
+      </doc>
+    </method>
+    <method name="getTaskType" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="c" type="char"/>
+      <doc>
+      <![CDATA[Gets the {@link TaskType} corresponding to the character.
+
+ @param c the character
+ @return the TaskType]]>
+      </doc>
+    </method>
+    <method name="getAllTaskTypes" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns a string of characters describing all possible {@link TaskType}
+ values
+
+ @return a string of all task type characters]]>
+      </doc>
+    </method>
+    <field name="TASK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idFormat" type="java.text.NumberFormat"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="TASK_ID_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="taskIdPattern" type="java.util.regex.Pattern"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TaskID represents the immutable and unique identifier for 
+ a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ execute the Map or Reduce Task, each of which are uniquely indentified by
+ their TaskAttemptID.
+ 
+ TaskID consists of 3 parts. First part is the {@link JobID}, that this 
+ TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r' 
+ representing whether the task is a map task or a reduce task. 
+ And the third part is the task number. <br> 
+ An example TaskID is : 
+ <code>task_200707121733_0003_m_000005</code> , which represents the
+ fifth map task in the third job running at the jobtracker 
+ started at <code>200707121733</code>. 
+ <p>
+ Applications should never construct or parse TaskID strings
+ , but rather use appropriate constructors or {@link #forName(String)} 
+ method. 
+ 
+ @see JobID
+ @see TaskAttemptID]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskID -->
+  <!-- start interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <interface name="TaskInputOutputContext"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    <method name="nextKeyValue" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Advance to the next key, value pair, returning null if at end.
+ @return the key object that was read into, or null if no more]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="KEYIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current key.
+ @return the current key object or null if there isn't one
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="VALUEIN"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the current value.
+ @return the value object that was read into
+ @throws IOException
+ @throws InterruptedException]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Generate an output key/value pair.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link OutputCommitter} for the task-attempt.
+ @return the <code>OutputCommitter</code> for the task-attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A context object that allows input and output from the task. It is only
+ supplied to the {@link Mapper} or {@link Reducer}.
+ @param <KEYIN> the input key type for the task
+ @param <VALUEIN> the input value type for the task
+ @param <KEYOUT> the output key type for the task
+ @param <VALUEOUT> the output value type for the task]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.TaskInputOutputContext -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <class name="TaskTrackerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="TaskTrackerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TaskTrackerInfo" type="java.lang.String, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTaskTrackerName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the tasktracker's name.
+ 
+ @return tracker's name.]]>
+      </doc>
+    </method>
+    <method name="isBlacklisted" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether tracker is blacklisted
+ @return true if tracker is blacklisted
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="getReasonForBlacklist" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets the reason for which the tasktracker was blacklisted.
+ 
+ @return reason which tracker was blacklisted]]>
+      </doc>
+    </method>
+    <method name="getBlacklistReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Gets a descriptive report about why the tasktracker was blacklisted.
+ 
+ @return report describing why the tasktracker was blacklisted.]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Information about TaskTracker.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskTrackerInfo -->
+  <!-- start class org.apache.hadoop.mapreduce.TaskType -->
+  <class name="TaskType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.TaskType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.TaskType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enum for map, reduce, job-setup, job-cleanup, task-cleanup task types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.TaskType -->
+</package>
+<package name="org.apache.hadoop.mapreduce.checkpoint">
+</package>
+<package name="org.apache.hadoop.mapreduce.counters">
+  <!-- start class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <class name="AbstractCounters" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractCounters" type="org.apache.hadoop.mapreduce.counters.AbstractCounters, org.apache.hadoop.mapreduce.counters.CounterGroupFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct from another counters object.
+ @param <C1> type of the other counter
+ @param <G1> type of the other counter group
+ @param counters the counters object to copy
+ @param groupFactory the factory for new groups]]>
+      </doc>
+    </constructor>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter, create one if necessary
+ @param groupName of the counter
+ @param counterName name of the counter
+ @return the matching counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="C"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Enum"/>
+      <doc>
+      <![CDATA[Find the counter for the given enum. The same enum will always return the
+ same counter.
+ @param key the counter key
+ @return the matching counter object]]>
+      </doc>
+    </method>
+    <method name="getGroupNames" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the names of all counter classes.
+ @return Set of counter names.]]>
+      </doc>
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getGroup" return="G"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="groupName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns the named counter group, or an empty group if there is none
+ with the specified name.
+ @param groupName name of the group
+ @return the group]]>
+      </doc>
+    </method>
+    <method name="countCounters" return="int"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the total number of counters, by summing the number of counters
+ in each group.
+ @return the total number of counters]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the set of groups.
+ Counters ::= version #fgroups (groupId, group)* #groups (group)*]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return textual representation of the counter values.
+ @return the string]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.counters.AbstractCounters"/>
+      <doc>
+      <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericRight" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract class to provide common implementation for the Counters
+ container in both mapred and mapreduce packages.
+
+ @param <C> type of counter inside the counters
+ @param <G> type of group inside the counters]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.counters.AbstractCounters -->
+  <!-- start interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+  <interface name="CounterGroupBase"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the internal name of the group
+ @return the internal name]]>
+      </doc>
+    </method>
+    <method name="getDisplayName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the display name of the group.
+ @return the human readable name]]>
+      </doc>
+    </method>
+    <method name="setDisplayName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the display name of the group
+ @param displayName of the group]]>
+      </doc>
+    </method>
+    <method name="addCounter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counter" type="T"/>
+      <doc>
+      <![CDATA[Add a counter to this group.
+ @param counter to add]]>
+      </doc>
+    </method>
+    <method name="addCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Add a counter to this group
+ @param name  of the counter
+ @param displayName of the counter
+ @param value of the counter
+ @return the counter]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="displayName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @param displayName the display name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <param name="create" type="boolean"/>
+      <doc>
+      <![CDATA[Find a counter in the group
+ @param counterName the name of the counter
+ @param create create the counter if not found if true
+ @return the counter that was found or added or null if create is false]]>
+      </doc>
+    </method>
+    <method name="findCounter" return="T"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="counterName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Find a counter in the group.
+ @param counterName the name of the counter
+ @return the counter that was found or added]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of counters in this group.]]>
+      </doc>
+    </method>
+    <method name="incrAllCounters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+      <doc>
+      <![CDATA[Increment all counters by a group of counters
+ @param rightGroup  the group to be added to this group]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The common counter group interface.
+
+ @param <T> type of the counter for the group]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.counters.CounterGroupBase -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.aggregate">
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <class name="DoubleValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="DoubleValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a double value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="double"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a double value.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getSum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up a sequence of double
+ values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.DoubleValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <class name="LongValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the maximum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <class name="LongValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="newVal" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param newVal
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the minimum of 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <class name="LongValueSum" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="LongValueSum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object whose string representation represents a long value.]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="long"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a long value.]]>
+      </doc>
+    </method>
+    <method name="getSum" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that sums up 
+ a sequence of long values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.LongValueSum -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <class name="StringValueMax" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMax"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the biggest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMax -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <class name="StringValueMin" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="StringValueMin"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          a string.]]>
+      </doc>
+    </method>
+    <method name="getVal" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the aggregated value]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the aggregated value]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of one element. The element is a string
+         representation of the aggregated value. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that maintain the smallest of 
+ a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.StringValueMin -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <class name="UniqValueCount" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="UniqValueCount"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="UniqValueCount" type="long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[constructor
+ @param maxNum the limit in the number of unique values to keep.]]>
+      </doc>
+    </constructor>
+    <method name="setMaxItems" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="n" type="long"/>
+      <doc>
+      <![CDATA[Set the limit on the number of unique values
+ @param n the desired limit on the number of unique values
+ @return the new limit on the number of unique values]]>
+      </doc>
+    </method>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val
+          an object.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return the number of unique objects aggregated]]>
+      </doc>
+    </method>
+    <method name="getUniqueItems" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the set of the unique objects]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return return an array of the unique objects. The return value is
+         expected to be used by the a combiner.]]>
+      </doc>
+    </method>
+    <field name="MAX_NUM_UNIQUE_VALUES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a value aggregator that dedupes a sequence of objects.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UniqValueCount -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <class name="UserDefinedValueAggregatorDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="UserDefinedValueAggregatorDescriptor" type="java.lang.String, org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@param className the class name of the user defined descriptor class
+ @param conf a configure object used for decriptor configuration]]>
+      </doc>
+    </constructor>
+    <method name="createInstance" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create an instance of the given class
+ @param className the name of the class
+ @return a dynamically created instance of the given class]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for the given 
+   key/value pairs by delegating the invocation to the real object.
+   
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this object.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Do nothing.]]>
+      </doc>
+    </method>
+    <field name="theAggregatorDescriptor" type="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a wrapper for a user defined value 
+ aggregator descriptor.
+ It serves two functions: One is to create an object of 
+ ValueAggregatorDescriptor from the name of a user defined class
+ that may be dynamically loaded. The other is to
+ delegate invocations of generateKeyValPairs function to the created object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.UserDefinedValueAggregatorDescriptor -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <interface name="ValueAggregator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="addNextValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add a value to the aggregator
+ 
+ @param val the value to be added]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of the agregator]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return an array of values as the outputs of the combiner.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This interface defines the minimal protocol for value aggregators.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <class name="ValueAggregatorBaseDescriptor" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"/>
+    <constructor name="ValueAggregatorBaseDescriptor"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="generateEntry" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+      <param name="val" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param id the aggregation id
+ @param val the val associated with the id to be aggregated
+ @return an Entry whose key is the aggregation id prefixed with 
+ the aggregation type.]]>
+      </doc>
+    </method>
+    <method name="generateValueAggregator" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="uniqCount" type="long"/>
+      <doc>
+      <![CDATA[@param type the aggregation type
+ @param uniqCount the limit in the number of unique values to keep, 
+                  if type is UNIQ_VALUE_COUNT 
+ @return a value aggregator of the given type.]]>
+      </doc>
+    </method>
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate 1 or 2 aggregation-id/value pairs for the given key/value pair.
+ The first id will be of type LONG_VALUE_SUM, with "record_count" as
+ its aggregation id. If the input is a file split,
+ the second id of the same type will be generated too, with the file name 
+ as its aggregation id. This achieves the behavior of counting the total 
+ number of records in the input data, and the number of records 
+ in each input file.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[get the input file name.
+ 
+ @param conf a configuration object]]>
+      </doc>
+    </method>
+    <field name="UNIQ_VALUE_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DOUBLE_VALUE_SUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_HISTOGRAM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LONG_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MAX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STRING_VALUE_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="inputFile" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements the common functionalities of 
+ the subclasses of ValueAggregatorDescriptor class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <class name="ValueAggregatorCombiner" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorCombiner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Combines values for a given key.  
+ @param key the key is expected to be a Text object, whose prefix indicates
+ the type of aggregation to aggregate the values. 
+ @param values the values to combine
+ @param context to collect combined values]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic combiner of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorCombiner -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <interface name="ValueAggregatorDescriptor"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="generateKeyValPairs" return="java.util.ArrayList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Generate a list of aggregation-id/value pairs for 
+ the given key/value pair.
+ This function is usually called by the mapper of an Aggregate based job.
+ 
+ @param key
+          input key
+ @param val
+          input value
+ @return a list of aggregation id/value pairs. An aggregation id encodes an
+         aggregation type which is used to guide the way to aggregate the
+         value in the reduce/combiner phrase of an Aggregate based job.]]>
+      </doc>
+    </method>
+    <method name="configure"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Configure the object
+ 
+ @param conf
+          a Configuration object that may contain the information 
+          that can be used to configure the object.]]>
+      </doc>
+    </method>
+    <field name="TYPE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ONE" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This interface defines the contract a value aggregator descriptor must
+ support. Such a descriptor can be configured with a {@link Configuration}
+ object. Its main function is to generate a list of aggregation-id/value 
+ pairs. An aggregation id encodes an aggregation type which is used to 
+ guide the way to aggregate the value in the reduce/combiner phrase of an
+ Aggregate based job. 
+ The mapper in an Aggregate based map/reduce job may create one or more of
+ ValueAggregatorDescriptor objects at configuration time. For each input
+ key/value pair, the mapper will use those objects to create aggregation
+ id/value pairs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <class name="ValueAggregatorJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJob"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJobs" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create an Aggregate based map/reduce job.
+ 
+ @param conf The configuration for job
+ @param args the arguments used for job creation. Generic hadoop
+ arguments are accepted.
+ @return a Job object ready for submission.
+ 
+ @throws IOException
+ @see GenericOptionsParser]]>
+      </doc>
+    </method>
+    <method name="createValueAggregatorJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <param name="descriptors" type="java.lang.Class[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setAggregatorDescriptors" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="descriptors" type="java.lang.Class[]"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <doc>
+      <![CDATA[create and run an Aggregate based map/reduce job.
+ 
+ @param args the arguments used for job creation
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the main class for creating a map/reduce job using Aggregate
+ framework. The Aggregate is a specialization of map/reduce framework,
+ specializing for performing various simple aggregations.
+ 
+ Generally speaking, in order to implement an application using Map/Reduce
+ model, the developer is to implement Map and Reduce functions (and possibly
+ combine function). However, a lot of applications related to counting and
+ statistics computing have very similar characteristics. Aggregate abstracts
+ out the general patterns of these functions and implementing those patterns.
+ In particular, the package provides generic mapper/redducer/combiner 
+ classes, and a set of built-in value aggregators, and a generic utility 
+ class that helps user create map/reduce jobs using the generic class. 
+ The built-in aggregators include:
+ 
+ sum over numeric values count the number of distinct values compute the
+ histogram of values compute the minimum, maximum, media,average, standard
+ deviation of numeric values
+ 
+ The developer using Aggregate will need only to provide a plugin class
+ conforming to the following interface:
+ 
+ public interface ValueAggregatorDescriptor { public ArrayList&lt;Entry&gt;
+ generateKeyValPairs(Object key, Object value); public void
+ configure(Configuration conf); }
+ 
+ The package also provides a base class, ValueAggregatorBaseDescriptor,
+ implementing the above interface. The user can extend the base class and
+ implement generateKeyValPairs accordingly.
+ 
+ The primary work of generateKeyValPairs is to emit one or more key/value
+ pairs based on the input key/value pair. The key in an output key/value pair
+ encode two pieces of information: aggregation type and aggregation id. The
+ value will be aggregated onto the aggregation id according the aggregation
+ type.
+ 
+ This class offers a function to generate a map/reduce job using Aggregate
+ framework. The function takes the following parameters: input directory spec
+ input format (text or sequence file) output directory a file specifying the
+ user plugin class]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <class name="ValueAggregatorJobBase" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorJobBase"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getValueAggregatorDescriptor" return="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorDescriptor"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="spec" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getAggregatorDescriptors" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="logSpec"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="DESCRIPTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DESCRIPTOR_NUM" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="aggregatorDescriptorList" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This abstract class implements some common functionalities of the
+ the generic mapper, reducer and combiner classes of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJobBase -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <class name="ValueAggregatorMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K1"/>
+      <param name="value" type="V1"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[the map function. It iterates through the value aggregator descriptor 
+  list to generate aggregation id/value pairs and emit them.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic mapper of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <class name="ValueAggregatorReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ValueAggregatorReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[@param key
+        the key is expected to be a Text object, whose prefix indicates
+        the type of aggregation to aggregate the values. In effect, data
+        driven computing is achieved. It is assumed that each aggregator's
+        getReport method emits appropriate output for the aggregator. This
+        may be further customized.
+ @param values the values to be aggregated
+ @param context]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements the generic reducer of Aggregate.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+  <class name="ValueHistogram" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregator"/>
+    <constructor name="ValueHistogram"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addNextValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[add the given val to the aggregator.
+ 
+ @param val the value to be added. It is expected to be a string
+ in the form of xxxx\tnum, meaning xxxx has num occurrences.]]>
+      </doc>
+    </method>
+    <method name="getReport" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the string representation of this aggregator.
+ It includes the following basic statistics of the histogram:
+    the number of unique values
+    the minimum value
+    the media value
+    the maximum value
+    the average value
+    the standard deviation]]>
+      </doc>
+    </method>
+    <method name="getReportDetails" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a string representation of the list of value/frequence pairs of 
+ the histogram]]>
+      </doc>
+    </method>
+    <method name="getCombinerOutput" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a list value/frequence pairs.
+  The return value is expected to be used by the reducer.]]>
+      </doc>
+    </method>
+    <method name="getReportItems" return="java.util.TreeMap"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a TreeMap representation of the histogram]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[reset the aggregator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class implements a value aggregator that computes the 
+ histogram of a sequence of strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.aggregate.ValueHistogram -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.chain">
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <class name="ChainMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain mapper.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainMapper class allows to use multiple Mapper classes within a single
+ Map task.
+ 
+ <p>
+ The Mapper classes are invoked in a chained (or piped) fashion, the output of
+ the first becomes the input of the second, and so on until the last Mapper,
+ the output of the last Mapper will be written to the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed in a chain. This enables having
+ reusable specialized Mappers that can be combined to perform composite
+ operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p>
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain.
+ </p>
+ ChainMapper usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+  <class name="ChainReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ChainReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setReducer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="reducerConf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Sets the {@link Reducer} class to the chain job.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value. For the added Reducer the configuration given for it,
+ <code>reducerConf</code>, have precedence over the job's Configuration.
+ This precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ 
+ @param job
+          the job
+ @param klass
+          the Reducer class to add.
+ @param inputKeyClass
+          reducer input key class.
+ @param inputValueClass
+          reducer input value class.
+ @param outputKeyClass
+          reducer output key class.
+ @param outputValueClass
+          reducer output value class.
+ @param reducerConf
+          a configuration for the Reducer class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="addMapper"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="klass" type="java.lang.Class"/>
+      <param name="inputKeyClass" type="java.lang.Class"/>
+      <param name="inputValueClass" type="java.lang.Class"/>
+      <param name="outputKeyClass" type="java.lang.Class"/>
+      <param name="outputValueClass" type="java.lang.Class"/>
+      <param name="mapperConf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Adds a {@link Mapper} class to the chain reducer.
+ 
+ <p>
+ The key and values are passed from one element of the chain to the next, by
+ value For the added Mapper the configuration given for it,
+ <code>mapperConf</code>, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+ </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the
+ chain.
+ </p>
+ 
+ @param job
+          The job.
+ @param klass
+          the Mapper class to add.
+ @param inputKeyClass
+          mapper input key class.
+ @param inputValueClass
+          mapper input value class.
+ @param outputKeyClass
+          mapper output key class.
+ @param outputValueClass
+          mapper output value class.
+ @param mapperConf
+          a configuration for the Mapper class. It is recommended to use a
+          Configuration without default values using the
+          <code>Configuration(boolean loadDefaults)</code> constructor with
+          FALSE.]]>
+      </doc>
+    </method>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[The ChainReducer class allows to chain multiple Mapper classes after a
+ Reducer within the Reducer task.
+ 
+ <p>
+ For each record output by the Reducer, the Mapper classes are invoked in a
+ chained (or piped) fashion. The output of the reducer becomes the input of
+ the first mapper and output of first becomes the input of the second, and so
+ on until the last Mapper, the output of the last Mapper will be written to
+ the task's output.
+ </p>
+ <p>
+ The key functionality of this feature is that the Mappers in the chain do not
+ need to be aware that they are executed after the Reducer or in a chain. This
+ enables having reusable specialized Mappers that can be combined to perform
+ composite operations within a single task.
+ </p>
+ <p>
+ Special care has to be taken when creating chains that the key/values output
+ by a Mapper are valid for the following Mapper in the chain. It is assumed
+ all Mappers and the Reduce in the chain use matching output and input key and
+ value classes as no conversion is done by the chaining code.
+ </p>
+ <p> Using the ChainMapper and the ChainReducer classes is possible to
+ compose Map/Reduce jobs that look like <code>[MAP+ / REDUCE MAP*]</code>. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO. </p>
+ <p>
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+ </p>
+ ChainReducer usage pattern:
+ <p>
+ 
+ <pre>
+ ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.chain.ChainReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.db">
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <class name="BigDecimalSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BigDecimalSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="tryDivide" return="java.math.BigDecimal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="numerator" type="java.math.BigDecimal"/>
+      <param name="denominator" type="java.math.BigDecimal"/>
+      <doc>
+      <![CDATA[Divide numerator by denominator. If impossible in exact mode, use rounding.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over BigDecimal values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <class name="BooleanSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="BooleanSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over boolean values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.BooleanSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <class name="DataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getBoundingValsQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a query which returns the minimum and maximum values for
+ the order-by column.
+
+ The min value should be in the first column, and the
+ max value should be in the second column of the results.]]>
+      </doc>
+    </method>
+    <method name="setBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="query" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user-defined bounding query to use with a user-defined query.
+      This *must* include the substring "$CONDITIONS"
+      (DataDrivenDBInputFormat.SUBSTITUTE_TOKEN) inside the WHERE clause,
+      so that DataDrivenDBInputFormat knows where to insert split clauses.
+      e.g., "SELECT foo FROM mytable WHERE $CONDITIONS"
+      This will be expanded to something like:
+        SELECT foo FROM mytable WHERE (id &gt; 100) AND (id &lt; 250)
+      inside each split.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="splitBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Note that the "orderBy" column is called the "splitBy" in this version.
+ We reuse the same field, but it's not strictly ordering it -- just partitioning
+ the results.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputBoundingQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[setInput() takes a custom query and a separate "bounding query" to use
+      instead of the custom "count query" used by DBInputFormat.]]>
+      </doc>
+    </method>
+    <field name="SUBSTITUTE_TOKEN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If users are providing their own query, the following string is expected to
+      appear in the WHERE clause, which will be substituted with a pair of conditions
+      on the input to allow input splits to parallelise the import.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ Operates like DBInputFormat, but instead of using LIMIT and OFFSET to demarcate
+ splits, it tries to generate WHERE clauses which separate the data into roughly
+ equivalent shards.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <class name="DataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records,
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table,
+ using data-driven WHERE clause splits.
+ Emits LongWritables containing the record number as
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <class name="DateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.IntegerSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+      <doc>
+      <![CDATA[Given a Date 'd', format it as a string for use in a SQL date
+ comparison operation.
+ @param d the date to format.
+ @return the string representing this date in SQL with any appropriate
+ quotation characters, etc.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values.
+ Make use of logic from IntegerSplitter, since date/time are just longs
+ in Java.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <class name="DBConfiguration" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="passwd" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the {@link Configuration}.  
+ @param conf the configuration
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL. 
+ @param userName DB access username 
+ @param passwd DB access passwd]]>
+      </doc>
+    </method>
+    <method name="configureDB"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="driverClass" type="java.lang.String"/>
+      <param name="dbUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Sets the DB access related fields in the JobConf.  
+ @param job the job
+ @param driverClass JDBC Driver class name
+ @param dbUrl JDBC DB access URL.]]>
+      </doc>
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Returns a connection object o the DB 
+ @throws ClassNotFoundException 
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getInputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="getInputConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputConditions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conditions" type="java.lang.String"/>
+    </method>
+    <method name="getInputOrderBy" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputOrderBy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="orderby" type="java.lang.String"/>
+    </method>
+    <method name="getInputQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputCountQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="setInputBoundingQuery"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+    </method>
+    <method name="getInputBoundingQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getInputClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInputClass"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inputClass" type="java.lang.Class"/>
+    </method>
+    <method name="getOutputTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputTableName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tableName" type="java.lang.String"/>
+    </method>
+    <method name="getOutputFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setOutputFieldNames"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldNames" type="java.lang.String[]"/>
+    </method>
+    <method name="setOutputFieldCount"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldCount" type="int"/>
+    </method>
+    <method name="getOutputFieldCount" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DRIVER_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JDBC Driver class name]]>
+      </doc>
+    </field>
+    <field name="URL_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[JDBC Database access URL]]>
+      </doc>
+    </field>
+    <field name="USERNAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User name to access the database]]>
+      </doc>
+    </field>
+    <field name="PASSWORD_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Password to access the database]]>
+      </doc>
+    </field>
+    <field name="INPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input table name]]>
+      </doc>
+    </field>
+    <field name="INPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Input table]]>
+      </doc>
+    </field>
+    <field name="INPUT_CONDITIONS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[WHERE clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_ORDER_BY_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ORDER BY clause in the input SELECT statement]]>
+      </doc>
+    </field>
+    <field name="INPUT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whole input query, exluding LIMIT...OFFSET]]>
+      </doc>
+    </field>
+    <field name="INPUT_COUNT_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the count of records]]>
+      </doc>
+    </field>
+    <field name="INPUT_BOUNDING_QUERY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Input query to get the max and min values of the jdbc.input.query]]>
+      </doc>
+    </field>
+    <field name="INPUT_CLASS_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class name implementing DBWritable which will hold input tuples]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_TABLE_NAME_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Output table name]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_NAMES_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Field names in the Output table]]>
+      </doc>
+    </field>
+    <field name="OUTPUT_FIELD_COUNT_PROPERTY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of fields in the Output table]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A container for configuration property names for jobs with DB input/output.
+  
+ The job can be configured using the static methods in this class, 
+ {@link DBInputFormat}, and {@link DBOutputFormat}. 
+ Alternatively, the properties can be set in the configuration with proper
+ values. 
+   
+ @see DBConfiguration#configureDB(Configuration, String, String, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String)
+ @see DBInputFormat#setInput(Job, Class, String, String, String, String...)
+ @see DBOutputFormat#setOutput(Job, String, String...)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBConfiguration -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <class name="DBInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="DBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBProductName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCountQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for getting the total number of rows, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="conditions" type="java.lang.String"/>
+      <param name="orderBy" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param tableName The table to read data from
+ @param conditions The condition which to select data with, 
+ eg. '(updated &gt; 20070101 AND length &gt; 0)'
+ @param orderBy the fieldNames in the orderBy clause.
+ @param fieldNames The field names in the table
+ @see #setInput(Job, Class, String, String)]]>
+      </doc>
+    </method>
+    <method name="setInput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputClass" type="java.lang.Class"/>
+      <param name="inputQuery" type="java.lang.String"/>
+      <param name="inputCountQuery" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initializes the map-part of the job with the appropriate input settings.
+ 
+ @param job The map-reduce job
+ @param inputClass the class object implementing DBWritable, which is the 
+ Java object holding tuple fields.
+ @param inputQuery the input query to select fields. Example : 
+ "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
+ @param inputCountQuery the input query that returns 
+ the number of records in the table. 
+ Example : "SELECT COUNT(f1) FROM Mytable"
+ @see #setInput(Job, Class, String, String, String, String...)]]>
+      </doc>
+    </method>
+    <method name="closeConnection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="conditions" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="connection" type="java.sql.Connection"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="tableName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fieldNames" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="dbConf" type="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table.
+ <p>
+ DBInputFormat emits LongWritables containing the record number as 
+ key and DBWritables as value. 
+ 
+ The SQL query, and input class can be using one of the two 
+ setInput methods.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <class name="DBOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="constructQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="table" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Constructs the query used as the prepared statement to insert data.
+ 
+ @param table
+          the table to insert into
+ @param fieldNames
+          the fields to insert into. If field names are unknown, supply an
+          array of nulls.]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldNames" type="java.lang.String[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job with 
+ the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldNames The field names in the table.]]>
+      </doc>
+    </method>
+    <method name="setOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="tableName" type="java.lang.String"/>
+      <param name="fieldCount" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initializes the reduce-part of the job 
+ with the appropriate output settings
+ 
+ @param job The job
+ @param tableName The table to insert data into
+ @param fieldCount the number of fields in the table.]]>
+      </doc>
+    </method>
+    <field name="dbProductName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A OutputFormat that sends the reduce output to a SQL table.
+ <p> 
+ {@link DBOutputFormat} accepts &lt;key,value&gt; pairs, where 
+ key has a type extending DBWritable. Returned {@link RecordWriter} 
+ writes <b>only the key</b> to the database with a batch SQL query.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <class name="DBRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="DBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[@param split The InputSplit to read data for
+ @throws SQLException]]>
+      </doc>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records, 
+ subclasses can override this for custom behaviour.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.LongWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="createValue" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="getPos" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="deprecated, no comment">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #nextKeyValue()}">
+      <param name="key" type="org.apache.hadoop.io.LongWritable"/>
+      <param name="value" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #nextKeyValue()}]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getSplit" return="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFieldNames" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTableName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConditions" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDBConf" return="org.apache.hadoop.mapreduce.lib.db.DBConfiguration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConnection" return="java.sql.Connection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStatement" return="java.sql.PreparedStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStatement"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="stmt" type="java.sql.PreparedStatement"/>
+    </method>
+    <field name="statement" type="java.sql.PreparedStatement"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a SQL table.
+ Emits LongWritables containing the record number as 
+ key and DBWritables as value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.DBRecordReader -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <interface name="DBSplitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="split" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Given a ResultSet containing one record (and already advanced to that record)
+ with two columns (a low value, and a high value, both of the same type), determine
+ a set of splits that span the given values.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[DBSplitter will generate DBInputSplits to use with DataDrivenDBInputFormat.
+ DataDrivenDBInputFormat needs to interpolate between two values that
+ represent the lowest and highest valued records to import. Depending
+ on the data-type of the column, this requires different behavior.
+ DBSplitter implementations should perform this for a data type or family
+ of data types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBSplitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <interface name="DBWritable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statement" type="java.sql.PreparedStatement"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Sets the fields of the object in the {@link PreparedStatement}.
+ @param statement the statement that the fields are put into.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resultSet" type="java.sql.ResultSet"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Reads the fields of the object from the {@link ResultSet}. 
+ @param resultSet the {@link ResultSet} to get the fields from.
+ @throws SQLException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Objects that are read from/written to a database should implement
+ <code>DBWritable</code>. DBWritable, is similar to {@link Writable} 
+ except that the {@link #write(PreparedStatement)} method takes a 
+ {@link PreparedStatement}, and {@link #readFields(ResultSet)} 
+ takes a {@link ResultSet}. 
+ <p>
+ Implementations are responsible for writing the fields of the object 
+ to PreparedStatement, and reading the fields of the object from the 
+ ResultSet. 
+ 
+ <p>Example:</p>
+ If we have the following table in the database :
+ <pre>
+ CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+ </pre>
+ then we can read/write the tuples from/to the table with :
+ <p><pre>
+ public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+ </pre>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.db.DBWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <class name="FloatSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="FloatSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over floating-point values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.FloatSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <class name="IntegerSplitter" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.db.DBSplitter"/>
+    <constructor name="IntegerSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over integer values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.IntegerSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <class name="MySQLDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <class name="MySQLDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MySQLDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="executeQuery" return="java.sql.ResultSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="query" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </method>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a MySQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.MySQLDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <class name="OracleDataDrivenDBInputFormat" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="OracleDataDrivenDBInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSplitter" return="org.apache.hadoop.mapreduce.lib.db.DBSplitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="sqlDataType" type="int"/>
+      <doc>
+      <![CDATA[@return the DBSplitter implementation to use to divide the table/query into InputSplits.]]>
+      </doc>
+    </method>
+    <method name="createDBRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A InputFormat that reads input data from an SQL table in an Oracle db.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <class name="OracleDataDrivenDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DataDrivenDBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDataDrivenDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[A RecordReader that reads records from a Oracle table via DataDrivenDBRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDataDrivenDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <class name="OracleDateSplitter" extends="org.apache.hadoop.mapreduce.lib.db.DateSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDateSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dateToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="d" type="java.util.Date"/>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over date/time values returned by an Oracle db.
+ Make use of logic from DateSplitter, since this just needs to use
+ some Oracle-specific functions on the formatting end when generating
+ InputSplits.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDateSplitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <class name="OracleDBRecordReader" extends="org.apache.hadoop.mapreduce.lib.db.DBRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="OracleDBRecordReader" type="org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit, java.lang.Class, org.apache.hadoop.conf.Configuration, java.sql.Connection, org.apache.hadoop.mapreduce.lib.db.DBConfiguration, java.lang.String, java.lang.String[], java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="SQLException" type="java.sql.SQLException"/>
+    </constructor>
+    <method name="getSelectQuery" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the query for selecting the records from an Oracle DB.]]>
+      </doc>
+    </method>
+    <method name="setSessionTimeZone"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="conn" type="java.sql.Connection"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[Set session time zone
+ @param conf The current configuration.
+ We read the 'oracle.sessionTimeZone' property from here.
+ @param conn The connection to alter the timezone properties of.]]>
+      </doc>
+    </method>
+    <field name="SESSION_TIMEZONE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration key to set to a timezone string.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that reads records from an Oracle SQL table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.OracleDBRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+  <class name="TextSplitter" extends="org.apache.hadoop.mapreduce.lib.db.BigDecimalSplitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextSplitter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="split" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="results" type="java.sql.ResultSet"/>
+      <param name="colName" type="java.lang.String"/>
+      <exception name="SQLException" type="java.sql.SQLException"/>
+      <doc>
+      <![CDATA[This method needs to determine the splits between two user-provided strings.
+ In the case where the user's strings are 'A' and 'Z', this is not hard; we 
+ could create two splits from ['A', 'M') and ['M', 'Z'], 26 splits for strings
+ beginning with each letter, etc.
+
+ If a user has provided us with the strings "Ham" and "Haze", however, we need
+ to create splits that differ in the third letter.
+
+ The algorithm used is as follows:
+ Since there are 2**16 unicode characters, we interpret characters as digits in
+ base 65536. Given a string 's' containing characters s_0, s_1 .. s_n, we interpret
+ the string as the number: 0.s_0 s_1 s_2.. s_n in base 65536. Having mapped the
+ low and high strings into floating-point values, we then use the BigDecimalSplitter
+ to establish the even split points, then map the resulting floating point values
+ back into strings.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Implement DBSplitter over text strings.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.db.TextSplitter -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.fieldsel">
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <class name="FieldSelectionHelper" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionHelper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FieldSelectionHelper" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="parseOutputKeyValueSpec" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="specToString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fieldSeparator" type="java.lang.String"/>
+      <param name="keyValueSpec" type="java.lang.String"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valueFieldList" type="java.util.List"/>
+    </method>
+    <method name="getKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="extractOutputKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="val" type="java.lang.String"/>
+      <param name="fieldSep" type="java.lang.String"/>
+      <param name="keyFieldList" type="java.util.List"/>
+      <param name="valFieldList" type="java.util.List"/>
+      <param name="allValueFieldsFrom" type="int"/>
+      <param name="ignoreKey" type="boolean"/>
+      <param name="isMap" type="boolean"/>
+    </method>
+    <field name="emptyText" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATA_FIELD_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #DATA_FIELD_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #DATA_FIELD_SEPARATOR}]]>
+      </doc>
+    </field>
+    <field name="MAP_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REDUCE_OUTPUT_KEY_VALUE_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper/reducer class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec".
+ The value is expected to be like "keyFieldsSpec:valueFieldsSpec"
+ key/valueFieldsSpec are comma (,) separated field spec: fieldSpec,fieldSpec,fieldSpec ...
+ Each field spec can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value fields only.
+ They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields 4,3,0 and 1 for keys,
+ and use fields 6,5,1,2,3,7 and above for values.
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec".
+ 
+ The reducer extracts output key/value pairs in a similar manner, except that
+ the key is never ignored.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionHelper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <class name="FieldSelectionMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="val" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The identify function. Input key/value pair is written directly to output.]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a mapper class that can be used to perform
+ field selections in a manner similar to unix cut. The input data is treated
+ as fields separated by a user specified separator (the default value is
+ "\t"). The user can specify a list of fields that form the map output keys,
+ and a list of fields that form the map output values. If the inputformat is
+ TextInputFormat, the mapper will ignore the key to the map function. and the
+ fields are from the value only. Otherwise, the fields are the union of those
+ from the key and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The map output field list spec is under attribute 
+ "mapreduce.fieldsel.map.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) separated
+ field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec can be a 
+ simple number (e.g. 5) specifying a specific field, or a range (like 2-5)
+ to specify a range of fields, or an open range (like 3-) specifying all 
+ the fields starting from field 3. The open range field spec applies value
+ fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+  <class name="FieldSelectionReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FieldSelectionReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class implements a reducer class that can be used to perform field
+ selections in a manner similar to unix cut. 
+ 
+ The input data is treated as fields separated by a user specified
+ separator (the default value is "\t"). The user can specify a list of
+ fields that form the reduce output keys, and a list of fields that form
+ the reduce output values. The fields are the union of those from the key
+ and those from the value.
+ 
+ The field separator is under attribute "mapreduce.fieldsel.data.field.separator"
+ 
+ The reduce output field list spec is under attribute 
+ "mapreduce.fieldsel.reduce.output.key.value.fields.spec". 
+ The value is expected to be like
+ "keyFieldsSpec:valueFieldsSpec" key/valueFieldsSpec are comma (,) 
+ separated field spec: fieldSpec,fieldSpec,fieldSpec ... Each field spec
+ can be a simple number (e.g. 5) specifying a specific field, or a range
+ (like 2-5) to specify a range of fields, or an open range (like 3-) 
+ specifying all the fields starting from field 3. The open range field
+ spec applies value fields only. They have no effect on the key fields.
+ 
+ Here is an example: "4,3,0,1:6,5,1-3,7-". It specifies to use fields
+ 4,3,0 and 1 for keys, and use fields 6,5,1,2,3,7 and above for values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.fieldsel.FieldSelectionReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.input">
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <class name="CombineFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <method name="setMaxSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="maxSplitSize" type="long"/>
+      <doc>
+      <![CDATA[Specify the maximum size (in bytes) of each split. Each split is
+ approximately equal to the specified size.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeNode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeNode" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per node.
+ This applies to data that is left over after combining data on a single
+ node into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeNode.]]>
+      </doc>
+    </method>
+    <method name="setMinSplitSizeRack"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="minSplitSizeRack" type="long"/>
+      <doc>
+      <![CDATA[Specify the minimum size (in bytes) of each split per rack.
+ This applies to data that is left over after combining data on a single
+ rack into splits that are of maximum size specified by maxSplitSize.
+ This leftover data will be combined into its own split if its size
+ exceeds minSplitSizeRack.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="java.util.List"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="createPool"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="filters" type="org.apache.hadoop.fs.PathFilter[]"/>
+      <doc>
+      <![CDATA[Create a new pool and add the filters to it. 
+ A pathname can satisfy any one of the specified filters.
+ A split cannot have files from different pools.]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[This is not implemented yet.]]>
+      </doc>
+    </method>
+    <method name="getFileBlockLocations" return="org.apache.hadoop.fs.BlockLocation[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="SPLIT_MINSIZE_PERNODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE_PERRACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An abstract {@link InputFormat} that returns {@link CombineFileSplit}'s in 
+ {@link InputFormat#getSplits(JobContext)} method. 
+ 
+ Splits are constructed from the files under the input paths. 
+ A split cannot have files from different pools.
+ Each split returned may contain blocks from different files.
+ If a maxSplitSize is specified, then blocks on the same node are
+ combined to form a single split. Blocks that are left over are
+ then combined with other blocks in the same rack. 
+ If maxSplitSize is not specified, then blocks from the same rack
+ are combined in a single split; no attempt is made to create
+ node-local splits.
+ If the maxSplitSize is equal to the block size, then this class
+ is similar to the default splitting behavior in Hadoop: each
+ block is a locally processed split.
+ Subclasses implement 
+ {@link InputFormat#createRecordReader(InputSplit, TaskAttemptContext)}
+ to construct <code>RecordReader</code>'s for 
+ <code>CombineFileSplit</code>'s.
+ 
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <class name="CombineFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReader" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in the CombineFileSplit.]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[return progress based on the amount of data processed so far.]]>
+      </doc>
+    </method>
+    <method name="initNextRecordReader" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the record reader for the next chunk in this CombineFileSplit.]]>
+      </doc>
+    </method>
+    <field name="split" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rrConstructor" type="java.lang.reflect.Constructor"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="idx" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="curReader" type="org.apache.hadoop.mapreduce.RecordReader"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A generic RecordReader that can hand out different recordReaders
+ for each chunk in a {@link CombineFileSplit}.
+ A CombineFileSplit can combine data chunks from multiple files. 
+ This class allows using different RecordReaders for processing
+ these data chunks from different files.
+ @see CombineFileSplit]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <class name="CombineFileRecordReaderWrapper" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineFileRecordReaderWrapper" type="org.apache.hadoop.mapreduce.lib.input.FileInputFormat, org.apache.hadoop.mapreduce.lib.input.CombineFileSplit, org.apache.hadoop.mapreduce.TaskAttemptContext, java.lang.Integer"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A wrapper class for a record reader that handles a single file split. It
+ delegates most of the methods to the wrapped instance. A concrete subclass
+ needs to provide a constructor that calls this parent constructor with the
+ appropriate input format. The subclass constructor must satisfy the specific
+ constructor signature that is required by
+ <code>CombineFileRecordReader</code>.
+
+ Subclassing is needed to get a concrete record reader wrapper because of the
+ constructor requirement.
+
+ @see CombineFileRecordReader
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReaderWrapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <class name="CombineFileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default constructor]]>
+      </doc>
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[], long[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.fs.Path[], long[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CombineFileSplit" type="org.apache.hadoop.mapreduce.lib.input.CombineFileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Copy constructor]]>
+      </doc>
+    </constructor>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartOffsets" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the start offsets of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getLengths" return="long[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns an array containing the lengths of the files in the split]]>
+      </doc>
+    </method>
+    <method name="getOffset" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the start offset of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the length of the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getNumPaths" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the number of Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Returns the i<sup>th</sup> Path]]>
+      </doc>
+    </method>
+    <method name="getPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns all the Paths in the split]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns all the Paths where this input-split resides]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A sub-collection of input files. 
+ 
+ Unlike {@link FileSplit}, CombineFileSplit class does not represent 
+ a split of a file, but a split of input files into smaller sets. 
+ A split may contain blocks from different file but all 
+ the blocks in the same split are probably local to some rack <br> 
+ CombineFileSplit can be used to implement {@link RecordReader}'s, 
+ with reading one record per file.
+ 
+ @see FileSplit
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineFileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <class name="CombineSequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineSequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>SequenceFileInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineSequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <class name="CombineTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CombineTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Input format that is a <code>CombineFileInputFormat</code>-equivalent for
+ <code>TextInputFormat</code>.
+
+ @see CombineFileInputFormat]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <class name="FileInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setInputDirRecursive"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputDirRecursive" type="boolean"/>
+      <doc>
+      <![CDATA[@param job
+          the job to modify
+ @param inputDirRecursive]]>
+      </doc>
+    </method>
+    <method name="getInputDirRecursive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[@param job
+          the job to look at.
+ @return should the files to be read recursively?]]>
+      </doc>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower bound on split size imposed by the format.
+ @return the number of bytes of the minimal split for this format]]>
+      </doc>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="filename" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Is the given filename splittable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ The default implementation in <code>FileInputFormat</code> always returns
+ true. Implementations that may deal with non-splittable files <i>must</i>
+ override this method.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+ 
+ @param context the job context
+ @param filename the file name to check
+ @return is this file splitable?]]>
+      </doc>
+    </method>
+    <method name="setInputPathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filter" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+ @param job the job to modify
+ @param filter the PathFilter class use for filtering the input paths.]]>
+      </doc>
+    </method>
+    <method name="setMinInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the minimum input split size
+ @param job the job to modify
+ @param size the minimum size]]>
+      </doc>
+    </method>
+    <method name="getMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the minimum split size
+ @param job the job
+ @return the minimum number of bytes that can be in a split]]>
+      </doc>
+    </method>
+    <method name="setMaxInputSplitSize"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the maximum split size
+ @param job the job to modify
+ @param size the maximum split size]]>
+      </doc>
+    </method>
+    <method name="getMaxSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the maximum split size.
+ @param context the job to look at.
+ @return the maximum number of bytes a split can include]]>
+      </doc>
+    </method>
+    <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+      </doc>
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression. 
+
+ If security is enabled, this method collects
+ delegation tokens from the input paths and adds them to the job's
+ credentials.
+ @param job the job to list input paths for and attach tokens to.
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+      </doc>
+    </method>
+    <method name="addInputPathRecursively"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="result" type="java.util.List"/>
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add files in the input path recursively into the results.
+ @param result
+          The List to store all files.
+ @param fs
+          The FileSystem.
+ @param path
+          The input path.
+ @param inputFilter
+          The input filter that can be used to filter files/dirs. 
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="makeSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+      <param name="start" type="long"/>
+      <param name="length" type="long"/>
+      <param name="hosts" type="java.lang.String[]"/>
+      <param name="inMemoryHosts" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Generate the list of files and make them into FileSplits.
+ @param job the job context
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="computeSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blockSize" type="long"/>
+      <param name="minSize" type="long"/>
+      <param name="maxSize" type="long"/>
+    </method>
+    <method name="getBlockIndex" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+      <param name="offset" type="long"/>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Sets the given comma separated paths as the list of inputs 
+ for the map-reduce job.
+ 
+ @param job the job
+ @param commaSeparatedPaths Comma separated paths to be set as 
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="commaSeparatedPaths" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add the given comma separated paths to the list of inputs for
+  the map-reduce job.
+ 
+ @param job The job to modify
+ @param commaSeparatedPaths Comma separated paths to be added to
+        the list of inputs for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="setInputPaths"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+ 
+ @param job The job to modify 
+ @param inputPaths the {@link Path}s of the input directories/files 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job} to modify
+ @param path {@link Path} to be added to the list of inputs for 
+            the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+ 
+ @param context The job
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+      </doc>
+    </method>
+    <field name="INPUT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MAXSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SPLIT_MINSIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PATHFILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_INPUT_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INPUT_DIR_NONRECURSIVE_IGNORE_SUBDIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LIST_STATUS_NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LIST_STATUS_NUM_THREADS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A base class for file-based {@link InputFormat}s.
+
+ <p><code>FileInputFormat</code> is the base class for all file-based 
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobContext)}.
+
+ Implementations of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(JobContext, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files <i>must</i> override this method, since
+ the default implementation assumes splitting is always possible.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <class name="FileInputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="FileSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block, possibly null]]>
+      </doc>
+    </constructor>
+    <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, java.lang.String[], java.lang.String[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructs a split with host and cached-blocks information
+
+ @param file the file name
+ @param start the position of the first byte in the file to process
+ @param length the number of bytes in the file to process
+ @param hosts the list of hosts containing the block
+ @param inMemoryHosts the list of hosts containing the block in memory]]>
+      </doc>
+    </constructor>
+    <method name="getPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file containing this split's data.]]>
+      </doc>
+    </method>
+    <method name="getStart" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The position of the first byte in the file to process.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of bytes in the file to process.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getLocationInfo" return="org.apache.hadoop.mapred.SplitLocationInfo[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[A section of an input file.  Returned by {@link
+ InputFormat#getSplits(JobContext)} and passed to
+ {@link InputFormat#createRecordReader(InputSplit,TaskAttemptContext)}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FileSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <class name="FixedLengthInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FixedLengthInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setRecordLength"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="recordLength" type="int"/>
+      <doc>
+      <![CDATA[Set the length of each record
+ @param conf configuration
+ @param recordLength the length of a record]]>
+      </doc>
+    </method>
+    <method name="getRecordLength" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get record length value
+ @param conf configuration
+ @return the record length, zero means none was set]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <field name="FIXED_RECORD_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FixedLengthInputFormat is an input format used to read input files
+ which contain fixed length records.  The content of a record need not be
+ text.  It can be arbitrary binary data.  Users must configure the record
+ length property by calling:
+ FixedLengthInputFormat.setRecordLength(conf, recordLength);<br><br> or
+ conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength);
+ <br><br>
+ @see FixedLengthRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.FixedLengthInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <class name="InvalidInputException" extends="java.io.IOException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidInputException" type="java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create the exception with the given list.
+ The first element of the list is used as the init cause value.
+ @param probs the list of problems to report. this list is not copied.]]>
+      </doc>
+    </constructor>
+    <method name="getProblems" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the complete list of the problems reported.
+ @return the list of problems, which must not be modified]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a summary message of the problems found.
+ @return the concatenated messages from all of the problems.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class wraps a list of problems with the input, so that the user
+ can get a list of problems together instead of finding and fixing them one 
+ by one.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.InvalidInputException -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <class name="KeyValueLineRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueLineRecordReader" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="findSeparator" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="utf" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="length" type="int"/>
+      <param name="sep" type="byte"/>
+    </method>
+    <method name="setKeyValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.Text"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="line" type="byte[]"/>
+      <param name="lineLen" type="int"/>
+      <param name="pos" type="int"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_VALUE_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="KEY_VALUE_SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Use {@link #KEY_VALUE_SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #KEY_VALUE_SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This class treats a line in the input as a key/value pair separated by a 
+ separator character. The separator can be specified in config file 
+ under the attribute name mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ separator is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <class name="KeyValueTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KeyValueTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files. Files are broken into lines.
+ Either line feed or carriage-return are used to signal end of line. 
+ Each line is divided into key and value parts by a separator byte. If no
+ such a byte exists, the key will be the entire line and value will be empty.
+ The separator byte can be specified in config file under the attribute name
+ mapreduce.input.keyvaluelinerecordreader.key.value.separator. The default
+ is the tab character ('\t').]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <class name="MultipleInputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleInputs"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} to the list of
+ inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path]]>
+      </doc>
+    </method>
+    <method name="addInputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="inputFormatClass" type="java.lang.Class"/>
+      <param name="mapperClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Add a {@link Path} with a custom {@link InputFormat} and
+ {@link Mapper} to the list of inputs for the map-reduce job.
+ 
+ @param job The {@link Job}
+ @param path {@link Path} to be added to the list of inputs for the job
+ @param inputFormatClass {@link InputFormat} class to use for this path
+ @param mapperClass {@link Mapper} class to use for this path]]>
+      </doc>
+    </method>
+    <field name="DIR_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIR_MAPPERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class supports MapReduce jobs that have multiple input paths with
+ a different {@link InputFormat} and {@link Mapper} for each path]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.MultipleInputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <class name="NLineInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NLineInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="genericSplit" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Logically splits the set of input files for the job, splits N lines
+ of the input as one split.
+ 
+ @see FileInputFormat#getSplits(JobContext)]]>
+      </doc>
+    </method>
+    <method name="getSplitsForFile" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="numLinesPerSplit" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createFileSplit" return="org.apache.hadoop.mapreduce.lib.input.FileSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="org.apache.hadoop.fs.Path"/>
+      <param name="begin" type="long"/>
+      <param name="length" type="long"/>
+      <doc>
+      <![CDATA[NLineInputFormat uses LineRecordReader, which always reads
+ (and consumes) at least one character out of its upper split
+ boundary. So to make sure that each mapper gets N lines, we
+ move back the upper split limits of each split 
+ by one character here.
+ @param fileName  Path of file
+ @param begin  the position of the first byte in the file to process
+ @param length  number of bytes in InputSplit
+ @return  FileSplit]]>
+      </doc>
+    </method>
+    <method name="setNumLinesPerSplit"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="numLines" type="int"/>
+      <doc>
+      <![CDATA[Set the number of lines per split
+ @param job the job to modify
+ @param numLines the number of lines per split]]>
+      </doc>
+    </method>
+    <method name="getNumLinesPerSplit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the number of lines per split
+ @param job the job
+ @return the number of lines per split]]>
+      </doc>
+    </method>
+    <field name="LINES_PER_MAP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[NLineInputFormat which splits N lines of input as one split.
+
+ In many "pleasantly" parallel applications, each process/mapper 
+ processes the same input file (s), but with computations are 
+ controlled by different parameters.(Referred to as "parameter sweeps").
+ One way to achieve this, is to specify a set of parameters 
+ (one set per line) as input in a control file 
+ (which is the input path to the map-reduce application,
+ where as the input dataset is specified 
+ via a config variable in JobConf.).
+ 
+ The NLineInputFormat can be used in such applications, that splits 
+ the input file such that by default, one line is fed as
+ a value to one map task, and key is the offset.
+ i.e. (k,v) is (LongWritable, Text).
+ The location hints will span the whole mapred cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.NLineInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <class name="SequenceFileAsBinaryInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[InputFormat reading keys, values from SequenceFiles in binary (raw)
+ format.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsBinaryInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <class name="SequenceFileAsTextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class is similar to SequenceFileInputFormat, except it generates
+ SequenceFileAsTextRecordReader which converts the input keys and values
+ to their String forms by calling toString() method.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <class name="SequenceFileAsTextRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsTextRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentValue" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read key/value pair in a line.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[This class converts the input keys and values to their String forms by
+ calling toString() method. This class to SequenceFileAsTextInputFormat
+ class is as LineRecordReader class to TextInputFormat class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <class name="SequenceFileInputFilter" extends="org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFilter"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a record reader for the given split
+ @param split file split
+ @param context the task-attempt context
+ @return RecordReader]]>
+      </doc>
+    </method>
+    <method name="setFilterClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="filterClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[set the filter class
+ 
+ @param job The job
+ @param filterClass filter class]]>
+      </doc>
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_FREQUENCY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_REGEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A class that allows a map/red job to work on a sample of sequence files.
+ The sample is decided by the filter class set by the job.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFilter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <class name="SequenceFileInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getFormatMinSplitSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="listStatus" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <class name="SequenceFileRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the progress within the input split
+ @return 0.0 to 1.0 of the input byte range]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link RecordReader} for {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+  <class name="TextInputFormat" extends="org.apache.hadoop.mapreduce.lib.input.FileInputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="isSplitable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="file" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link InputFormat} for plain text files.  Files are broken into lines.
+ Either linefeed or carriage-return are used to signal end of line.  Keys are
+ the position in the file, and values are the line of text..]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.input.TextInputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.jobcontrol">
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <class name="ControlledJob" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ControlledJob" type="org.apache.hadoop.mapreduce.Job, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ @param job a mapreduce job to be executed.
+ @param dependingJobs an array of jobs the current job depends on]]>
+      </doc>
+    </constructor>
+    <constructor name="ControlledJob" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Construct a job.
+ 
+ @param conf mapred job configuration representing a job to be executed.
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getJobName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job name of this job]]>
+      </doc>
+    </method>
+    <method name="setJobName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job name for  this job.
+ @param jobName the job name]]>
+      </doc>
+    </method>
+    <method name="getJobID" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the job ID of this job assigned by JobControl]]>
+      </doc>
+    </method>
+    <method name="setJobID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the job ID for  this job.
+ @param id the job ID]]>
+      </doc>
+    </method>
+    <method name="getMapredJobId" return="org.apache.hadoop.mapreduce.JobID"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapred ID of this job as assigned by the mapred framework.]]>
+      </doc>
+    </method>
+    <method name="getJob" return="org.apache.hadoop.mapreduce.Job"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the mapreduce job]]>
+      </doc>
+    </method>
+    <method name="setJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <doc>
+      <![CDATA[Set the mapreduce job
+ @param job the mapreduce job for this job.]]>
+      </doc>
+    </method>
+    <method name="getJobState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the state of this job]]>
+      </doc>
+    </method>
+    <method name="setJobState"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="state" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob.State"/>
+      <doc>
+      <![CDATA[Set the state for this job.
+ @param state the new state for this job.]]>
+      </doc>
+    </method>
+    <method name="getMessage" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the message of this job]]>
+      </doc>
+    </method>
+    <method name="setMessage"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the message for this job.
+ @param message the message for this job.]]>
+      </doc>
+    </method>
+    <method name="getDependentJobs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the depending jobs of this job]]>
+      </doc>
+    </method>
+    <method name="addDependingJob" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dependingJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a job to this jobs' dependency list. 
+ Dependent jobs can only be added while a Job 
+ is waiting to run, not during or afterwards.
+ 
+ @param dependingJob Job that this Job depends on.
+ @return <tt>true</tt> if the Job was added.]]>
+      </doc>
+    </method>
+    <method name="isCompleted" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in a complete state]]>
+      </doc>
+    </method>
+    <method name="isReady" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if this job is in READY state]]>
+      </doc>
+    </method>
+    <method name="killJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="failJob"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="message" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="submit"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submit this job to mapred. The state becomes RUNNING if submission 
+ is successful, FAILED otherwise.]]>
+      </doc>
+    </method>
+    <field name="CREATE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class encapsulates a MapReduce job and its dependency. It monitors 
+  the states of the depending jobs and updates the state of this job.
+  A job starts in the WAITING state. If it does not have any depending jobs,
+  or all of the depending jobs are in SUCCESS state, then the job state 
+  will become READY. If any depending jobs fail, the job will fail too. 
+  When in READY state, the job can be submitted to Hadoop for execution, with
+  the state changing into RUNNING state. From RUNNING state, the job 
+  can get into SUCCESS or FAILED state, depending 
+  the status of the job execution.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+  <class name="JobControl" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Runnable"/>
+    <constructor name="JobControl" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Construct a job control for a group of jobs.
+ @param groupName a name identifying this group]]>
+      </doc>
+    </constructor>
+    <method name="getWaitingJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the waiting state]]>
+      </doc>
+    </method>
+    <method name="getRunningJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the running state]]>
+      </doc>
+    </method>
+    <method name="getReadyJobsList" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the ready state]]>
+      </doc>
+    </method>
+    <method name="getSuccessfulJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the jobs in the success state]]>
+      </doc>
+    </method>
+    <method name="getFailedJobList" return="java.util.List"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob"/>
+      <doc>
+      <![CDATA[Add a new controlled job.
+ @param aJob the new controlled job]]>
+      </doc>
+    </method>
+    <method name="addJob" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aJob" type="org.apache.hadoop.mapred.jobcontrol.Job"/>
+      <doc>
+      <![CDATA[Add a new job.
+ @param aJob the new job]]>
+      </doc>
+    </method>
+    <method name="addJobCollection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a collection of jobs
+ 
+ @param jobs]]>
+      </doc>
+    </method>
+    <method name="getThreadState" return="org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl.ThreadState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the thread state]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[set the thread state to STOPPING so that the 
+ thread will stop when it wakes up.]]>
+      </doc>
+    </method>
+    <method name="suspend"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[suspend the running thread]]>
+      </doc>
+    </method>
+    <method name="resume"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[resume the suspended thread]]>
+      </doc>
+    </method>
+    <method name="allFinished" return="boolean"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The main loop for the thread.
+  The loop does the following:
+  	Check the states of the running jobs
+  	Update the states of waiting jobs
+  	Submit the jobs in ready state]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class encapsulates a set of MapReduce jobs and its dependency.
+   
+  It tracks the states of the jobs by placing them into different tables
+  according to their states. 
+  
+  This class provides APIs for the client app to add a job to the group 
+  and to get the jobs in the group in different states. When a job is 
+  added, an ID unique to the group is assigned to the job. 
+  
+  This class has a thread that submits jobs when they become ready, 
+  monitors the states of the running jobs, and updates the states of jobs
+  based on the state changes of their depending jobs states. The class 
+  provides APIs for suspending/resuming the thread, and 
+  for stopping the thread.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.join">
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <class name="ArrayListBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="ArrayListBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ArrayListBackedIterator" type="java.util.ArrayList"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. The
+ implementation uses an {@link java.util.ArrayList} to store elements
+ added to it, replaying them as requested.
+ Prefer {@link StreamBackedIterator}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ArrayListBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <class name="ComposableInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ComposableInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Refinement of InputFormat requiring implementors to provide
+ ComposableRecordReader instead of RecordReader.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <class name="ComposableRecordReader" extends="org.apache.hadoop.mapreduce.RecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ComposableRecordReader"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Additional operations required of a RecordReader to participate in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <class name="CompositeInputFormat" extends="org.apache.hadoop.mapreduce.InputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CompositeInputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setFormat"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Interpret a given string as a composite expression.
+ {@code
+   func  ::= <ident>([<func>,]*<func>)
+   func  ::= tbl(<class>,"<path>")
+   class ::= @see java.lang.Class#forName(java.lang.String)
+   path  ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String)
+ }
+ Reads expression from the <tt>mapreduce.join.expr</tt> property and
+ user-supplied join types from <tt>mapreduce.join.define.&lt;ident&gt;</tt>
+  types. Paths supplied to <tt>tbl</tt> are given as input paths to the
+ InputFormat class listed.
+ @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]>
+      </doc>
+    </method>
+    <method name="addDefaults"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adds the default set of identifiers to the parser.]]>
+      </doc>
+    </method>
+    <method name="getSplits" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Build a CompositeInputSplit from the child InputFormats by assigning the
+ ith split from each child to the ith composite split.]]>
+      </doc>
+    </method>
+    <method name="createRecordReader" return="org.apache.hadoop.mapreduce.RecordReader"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Construct a CompositeRecordReader for the children of this InputFormat
+ as defined in the init expression.
+ The outermost join need only be composable, not necessarily a composite.
+ Mandating TupleWritable isn't strictly correct.]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given InputFormat class (inf), path (p) return:
+ {@code tbl(<inf>, <p>) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <method name="compose" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="java.lang.String"/>
+      <param name="inf" type="java.lang.Class"/>
+      <param name="path" type="org.apache.hadoop.fs.Path[]"/>
+      <doc>
+      <![CDATA[Convenience method for constructing composite formats.
+ Given operation (op), Object class (inf), set of paths (p) return:
+ {@code <op>(tbl(<inf>,<p1>),tbl(<inf>,<p2>),...,tbl(<inf>,<pn>)) }]]>
+      </doc>
+    </method>
+    <field name="JOIN_EXPR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="JOIN_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An InputFormat capable of performing joins over a set of data sources sorted
+ and partitioned the same way.
+
+ A user may define new join types by setting the property
+ <tt>mapreduce.join.define.&lt;ident&gt;</tt> to a classname. 
+ In the expression <tt>mapreduce.join.expr</tt>, the identifier will be
+ assumed to be a ComposableRecordReader.
+ <tt>mapreduce.join.keycomparator</tt> can be a classname used to compare 
+ keys in the join.
+ @see #setFormat
+ @see JoinRecordReader
+ @see MultiFilterRecordReader]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <class name="CompositeInputSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="CompositeInputSplit"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CompositeInputSplit" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="s" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an InputSplit to this collection.
+ @throws IOException If capacity was not specified during construction
+                     or if capacity has been reached.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.mapreduce.InputSplit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return the aggregate length of all child InputSplits currently added.]]>
+      </doc>
+    </method>
+    <method name="getLength" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the length of ith child InputSplit.]]>
+      </doc>
+    </method>
+    <method name="getLocations" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Collect a set of hosts from all child InputSplits.]]>
+      </doc>
+    </method>
+    <method name="getLocation" return="java.lang.String[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[getLocations from ith InputSplit.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write splits in the following format.
+ {@code
+ <count><class1><class2>...<classn><split1><split2>...<splitn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}
+ @throws IOException If the child InputSplit cannot be read, typically
+                     for failing access checks.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This InputSplit contains a set of child InputSplits. Any InputSplit inserted
+ into this collection must have a public default constructor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeInputSplit -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <class name="CompositeRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="CompositeRecordReader" type="int, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a RecordReader with <tt>capacity</tt> children to position
+ <tt>id</tt> in the parent reader.
+ The id of a root CompositeRecordReader is -1 by convention, but relying
+ on this is not recommended.]]>
+      </doc>
+    </constructor>
+    <method name="combine" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="value" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the position in the collector this class occupies.]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="getRecordReaderQueue" return="java.util.PriorityQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return sorted list of RecordReaders for this composite.]]>
+      </doc>
+    </method>
+    <method name="getComparator" return="org.apache.hadoop.io.WritableComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return comparator defining the ordering for RecordReaders in this
+ composite.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add a RecordReader to this collection.
+ The id() of a RecordReader determines where in the Tuple its
+ entry will appear. Adding RecordReaders with the same id has
+ undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key for the current join or the value at the top of the
+ RecordReader heap.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the top of this RR into the given object.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if it is possible that this could emit more values.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Pass skip key to child RRs.]]>
+      </doc>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Obtain an iterator over the child RRs apropos of the value type
+ ultimately emitted from this join.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[If key provided matches that of this Composite, give JoinCollector
+ iterator over values it may emit.]]>
+      </doc>
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[For all child RRs offering the key provided, obtain an iterator
+ at that position in the JoinCollector.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key of join or head of heap
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new key common to all child RRs.
+ @throws ClassCastException if key classes differ.]]>
+      </doc>
+    </method>
+    <method name="createTupleWritable" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a value to be used internally for joins.]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="X"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close all child RRs.]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Report progress as the minimum of all child RR progress.]]>
+      </doc>
+    </method>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="keyclass" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="jc" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="kids" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader[]"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="key" type="K"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="value" type="X"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A RecordReader that can effect joins of RecordReaders sharing a common key
+ type and partitioning.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <class name="InnerJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Return true iff the tuple is full (all data sources contain this key).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full inner join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.InnerJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <class name="JoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="JoinRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Emit the next set of key, value pairs as defined by the child
+ RecordReaders and operation associated with this composite RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="org.apache.hadoop.mapreduce.lib.join.TupleWritable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator wrapping the JoinCollector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite joins returning Tuples of arbitrary Writables.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.JoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <class name="MultiFilterRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultiFilterRecordReader" type="int, org.apache.hadoop.conf.Configuration, int, java.lang.Class"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="emit" return="V"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[For each tuple emitted, return a value (typically one of the values
+ in the tuple).
+ Modifying the Writables in the tuple is permitted and unlikely to affect
+ join behavior in most cases, but it is not recommended. It's safer to
+ clone first.]]>
+      </doc>
+    </method>
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Default implementation offers {@link #emit} every Tuple from the
+ collector (the outer join of child RRs).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getDelegate" return="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator returning a single value from the tuple.
+ @see MultiFilterDelegationIterator]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base class for Composite join returning values derived from multiple
+ sources, but generally not tuples.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <class name="OuterJoinRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.JoinRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="combine" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="srcs" type="java.lang.Object[]"/>
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit everything from the collector.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Full outer join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OuterJoinRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <class name="OverrideRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.MultiFilterRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="emit" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="dst" type="org.apache.hadoop.mapreduce.lib.join.TupleWritable"/>
+      <doc>
+      <![CDATA[Emit the value with the highest position in the tuple.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="V"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fillJoinCollector"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="iterkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Instead of filling the JoinCollector with iterators from all
+ data sources, fill only the rightmost for this key.
+ This not only saves space by discarding the other sources, but
+ it also emits the number of key-value pairs in the preferred
+ RecordReader instead of repeating that stream n times, where
+ n is the cardinality of the cross product of the discarded
+ streams for the given key.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Prefer the &quot;rightmost&quot; data source for this key.
+ For example, <tt>override(S1,S2,S3)</tt> will prefer values
+ from S3 over S2, and values from S2 over S1 for all keys
+ emitted from all sources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.OverrideRecordReader -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <class name="Parser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Parser"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Very simple shift-reduce parser for join expressions.
+
+ This should be sufficient for the user extension permitted now, but ought to
+ be replaced with a parser generator if more complex grammars are supported.
+ In particular, this &quot;shift-reduce&quot; parser has no states. Each set
+ of formals requires a different internal node type, which is responsible for
+ interpreting the list of tokens it receives. This is sufficient for the
+ current grammar, but it has several annoying properties that might inhibit
+ extension. In particular, parenthesis are always function calls; an
+ algebraic or filter grammar would not only require a node type, but must
+ also work around the internals of this parser.
+
+ For most other cases, adding classes to the hierarchy- particularly by
+ extending JoinRecordReader and MultiFilterRecordReader- is fairly
+ straightforward. One need only override the relevant method(s) (usually only
+ {@link CompositeRecordReader#combine}) and include a property to map its
+ value to an identifier in the parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <class name="Parser.Node" extends="org.apache.hadoop.mapreduce.lib.join.ComposableInputFormat"
+    abstract="true"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Node" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="addIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ident" type="java.lang.String"/>
+      <param name="mcstrSig" type="java.lang.Class[]"/>
+      <param name="nodetype" type="java.lang.Class"/>
+      <param name="cl" type="java.lang.Class"/>
+      <exception name="NoSuchMethodException" type="java.lang.NoSuchMethodException"/>
+      <doc>
+      <![CDATA[For a given identifier, add a mapping to the nodetype for the parse
+ tree and to the ComposableRecordReader to be created, including the
+ formals required to invoke the constructor.
+ The nodetype and constructor signature should be filled in from the
+ child node.]]>
+      </doc>
+    </method>
+    <method name="setID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+    </method>
+    <method name="setKeyComparator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="cmpcl" type="java.lang.Class"/>
+    </method>
+    <field name="rrCstrMap" type="java.util.Map"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="ident" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmpcl" type="java.lang.Class"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Node -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <class name="Parser.NodeToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NodeToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <class name="Parser.NumToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NumToken" type="double"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.NumToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <class name="Parser.StrToken" extends="org.apache.hadoop.mapreduce.lib.join.Parser.Token"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrToken" type="org.apache.hadoop.mapreduce.lib.join.Parser.TType, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.StrToken -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <class name="Parser.Token" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNode" return="org.apache.hadoop.mapreduce.lib.join.Parser.Node"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getNum" return="double"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getStr" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Tagged-union type for tokens from the join expression.
+ @see Parser.TType]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.Token -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <class name="Parser.TType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.join.Parser.TType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.Parser.TType -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <interface name="ResetableIterator"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="hasNext" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if a call to next may return a value. This is permitted false
+ positives, but not false negatives.]]>
+      </doc>
+    </method>
+    <method name="next" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign next value to actual.
+ It is required that elements added to a ResetableIterator be returned in
+ the same order after a call to {@link #reset} (FIFO).
+
+ Note that a call to this may fail for nested joins (i.e. more elements
+ available, but none satisfying the constraints of the join)]]>
+      </doc>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Assign last value returned to actual.]]>
+      </doc>
+    </method>
+    <method name="reset"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set iterator to return to the start of its range. Must be called after
+ calling {@link #add} to avoid a ConcurrentModificationException.]]>
+      </doc>
+    </method>
+    <method name="add"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="T"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Add an element to the collection of elements to iterate over.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Close datasources and release resources. Calling methods on the iterator
+ after calling close has undefined behavior.]]>
+      </doc>
+    </method>
+    <method name="clear"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Close datasources, but do not release internal resources. Calling this
+ method should permit the object to be reused with a different datasource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This defines an interface to a stateful Iterator that can replay elements
+ added to it directly.
+ Note that this does not extend {@link java.util.Iterator}.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.join.ResetableIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <class name="StreamBackedIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.join.ResetableIterator"/>
+    <constructor name="StreamBackedIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="next" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="replay" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="val" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="reset"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="item" type="X"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="clear"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This class provides an implementation of ResetableIterator. This
+ implementation uses a byte array to store elements added to it.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.StreamBackedIterator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <class name="TupleWritable" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <implements name="java.lang.Iterable"/>
+    <constructor name="TupleWritable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an empty tuple with no allocated storage for writables.]]>
+      </doc>
+    </constructor>
+    <constructor name="TupleWritable" type="org.apache.hadoop.io.Writable[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize tuple with storage; unknown whether any of them contain
+ &quot;written&quot; values.]]>
+      </doc>
+    </constructor>
+    <method name="has" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Return true if tuple has an element at the position provided.]]>
+      </doc>
+    </method>
+    <method name="get" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="int"/>
+      <doc>
+      <![CDATA[Get ith Writable from Tuple.]]>
+      </doc>
+    </method>
+    <method name="size" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of children in this Tuple.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="iterator" return="java.util.Iterator"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return an iterator over the elements in this tuple.
+ Note that this doesn't flatten the tuple; one may receive tuples
+ from this iterator.]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convert Tuple to String as in the following.
+ <tt>[&lt;child1&gt;,&lt;child2&gt;,...,&lt;childn&gt;]</tt>]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes each Writable to <code>out</code>.
+ TupleWritable format:
+ {@code
+  <count><type1><type2>...<typen><obj1><obj2>...<objn>
+ }]]>
+      </doc>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <field name="written" type="java.util.BitSet"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Writable type storing multiple {@link org.apache.hadoop.io.Writable}s.
+
+ This is *not* a general-purpose tuple type. In almost all cases, users are
+ encouraged to implement their own serializable types, which can perform
+ better validation and provide more efficient encodings than this class is
+ capable. TupleWritable relies on the join framework for type safety and
+ assumes its instances will rarely be persisted, assumptions not only
+ incompatible with, but contrary to the general case.
+
+ @see org.apache.hadoop.io.Writable]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.TupleWritable -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+  <class name="WrappedRecordReader" extends="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedRecordReader" type="int"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="split" type="org.apache.hadoop.mapreduce.InputSplit"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="createKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Request new key from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="createValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="id" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[{@inheritDoc}]]>
+      </doc>
+    </method>
+    <method name="key" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return the key at the head of this RR.]]>
+      </doc>
+    </method>
+    <method name="key"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="qkey" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Clone the key at the head of this RR into the object supplied.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return true if the RR- including the k,v pair stored in this object-
+ is exhausted.]]>
+      </doc>
+    </method>
+    <method name="skip"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Skip key-value pairs with keys less than or equal to the key provided.]]>
+      </doc>
+    </method>
+    <method name="accept"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="i" type="org.apache.hadoop.mapreduce.lib.join.CompositeRecordReader.JoinCollector"/>
+      <param name="key" type="K"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Add an iterator to the collector at the position occupied by this
+ RecordReader over the values in this stream paired with the key
+ provided (ie register a stream of values from this source matching K
+ with a collector).]]>
+      </doc>
+    </method>
+    <method name="nextKeyValue" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Read the next k,v pair into the head of this object; return true iff
+ the RR and this are exhausted.]]>
+      </doc>
+    </method>
+    <method name="getCurrentKey" return="K"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current key]]>
+      </doc>
+    </method>
+    <method name="getCurrentValue" return="U"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get current value]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Request progress from proxied RR.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Forward close request to proxied RR.]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.mapreduce.lib.join.ComposableRecordReader"/>
+      <doc>
+      <![CDATA[Implement Comparable contract (compare key at head of proxied RR
+ with that of another).]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Return true iff compareTo(other) retn true.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="empty" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="cmp" type="org.apache.hadoop.io.WritableComparator"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Proxy class for a RecordReader participating in the join framework.
+ 
+ This class keeps track of the &quot;head&quot; key-value pair for the
+ provided RecordReader and keeps a store of values matching a key when
+ this source is participating in a join.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.join.WrappedRecordReader -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.map">
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <class name="InverseMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InverseMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[The inverse function.  Input keys and values are swapped.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} that swaps keys and values.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.InverseMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <class name="MultithreadedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultithreadedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumberOfThreads" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[The number of threads in the thread pool that will run the map function.
+ @param job the job
+ @return the number of threads]]>
+      </doc>
+    </method>
+    <method name="setNumberOfThreads"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="threads" type="int"/>
+      <doc>
+      <![CDATA[Set the number of threads in the pool for running maps.
+ @param job the job to modify
+ @param threads the new number of threads]]>
+      </doc>
+    </method>
+    <method name="getMapperClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the application's mapper class.
+ @param <K1> the map's input key type
+ @param <V1> the map's input value type
+ @param <K2> the map's output key type
+ @param <V2> the map's output value type
+ @param job the job
+ @return the mapper class to run]]>
+      </doc>
+    </method>
+    <method name="setMapperClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="cls" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the application's mapper class.
+ @param <K1> the map input key type
+ @param <V1> the map input value type
+ @param <K2> the map output key type
+ @param <V2> the map output value type
+ @param job the job to modify
+ @param cls the class to use as the mapper]]>
+      </doc>
+    </method>
+    <method name="run"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Run the application's maps using a thread pool.]]>
+      </doc>
+    </method>
+    <field name="NUM_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAP_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Multithreaded implementation for @link org.apache.hadoop.mapreduce.Mapper.
+ <p>
+ It can be used instead of the default implementation,
+ {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU
+ bound in order to improve throughput.
+ <p>
+ Mapper implementations using this MapRunnable must be thread-safe.
+ <p>
+ The Map-Reduce job has to be configured with the mapper to use via 
+ {@link #setMapperClass(Job, Class)} and
+ the number of thread the thread-pool can use with the
+ {@link #getNumberOfThreads(JobContext)} method. The default
+ value is 10 threads.
+ <p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.MultithreadedMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <class name="RegexMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegexMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setup"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+    </method>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link Mapper} that extracts text matching a regular expression.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.RegexMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <class name="TokenCounterMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TokenCounterMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.Object"/>
+      <param name="value" type="org.apache.hadoop.io.Text"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Mapper.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <doc>
+    <![CDATA[Tokenize the input values and emit each word with a count of 1.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.TokenCounterMapper -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+  <class name="WrappedMapper" extends="org.apache.hadoop.mapreduce.Mapper"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedMapper"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMapContext" return="org.apache.hadoop.mapreduce.Mapper.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="mapContext" type="org.apache.hadoop.mapreduce.MapContext"/>
+      <doc>
+      <![CDATA[Get a wrapped {@link Mapper.Context} for custom implementations.
+ @param mapContext <code>MapContext</code> to be wrapped
+ @return a wrapped <code>Mapper.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Mapper} which wraps a given one to allow custom 
+ {@link Mapper.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.map.WrappedMapper -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.output">
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <class name="BindingPathOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BindingPathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Instantiate.
+ @param outputPath output path (may be null)
+ @param context task context
+ @throws IOException on any failure.]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobContext" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="taskContext" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCommitter" return="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the inner committer.
+ @return the bonded committer.]]>
+      </doc>
+    </method>
+    <field name="NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The classname for use in configurations.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is a special committer which creates the factory for the committer and
+ runs off that. Why does it exist? So that you can explicitly instantiate
+ a committer by classname and yet still have the actual implementation
+ driven dynamically by the factory options and destination filesystem.
+ This simplifies integration
+ with existing code which takes the classname of a committer.
+ There's no factory for this, as that would lead to a loop.
+
+ All commit protocol methods and accessors are delegated to the
+ wrapped committer.
+
+ How to use:
+
+ <ol>
+   <li>
+     In applications which take a classname of committer in
+     a configuration option, set it to the canonical name of this class
+     (see {@link #NAME}). When this class is instantiated, it will
+     use the factory mechanism to locate the configured committer for the
+     destination.
+   </li>
+   <li>
+     In code, explicitly create an instance of this committer through
+     its constructor, then invoke commit lifecycle operations on it.
+     The dynamically configured committer will be created in the constructor
+     and have the lifecycle operations relayed to it.
+   </li>
+ </ol>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <class name="FileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <constructor name="FileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a file output committer
+ @param outputPath the job's output path, or null if you want the output
+ committer to act as a noop.
+ @param context the task's context
+ @throws IOException]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the path where final output of the job should be placed.  This
+ could also be considered the committed application attempt path.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param context the context of the job.  This is used to get the
+ application attempt id.
+ @param out the output path to place these in.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getJobAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a given job attempt will be placed. 
+ @param appAttemptId the ID of the application attempt for this job.
+ @return the path to store job attempt data.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a task attempt is stored until
+ that task is committed.
+ 
+ @param context the context of the task attempt.
+ @param out The output path to put things in.
+ @return the path where a task attempt should be stored.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until
+ the entire job is committed.
+ @param context the context of the task attempt
+ @return the path where the output of a committed task is stored until
+ the entire job is committed.]]>
+      </doc>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="out" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <doc>
+      <![CDATA[Compute the path where the output of a committed task is stored until the
+ entire job is committed for a specific application attempt.
+ @param appAttemptId the id of the application attempt to use
+ @param context the context of any task.
+ @return the path where the output of a committed task is stored.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ @return the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="setupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create the temporary directory that is the root of all of the task 
+ work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do works in commitJobInternal().
+ Could retry on failure if using algorithm 2.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="commitJobInternal"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[The job has completed, so do following commit job, include:
+ Move all committed tasks to the final output dir (algorithm 1 only).
+ Delete the temporary directory, including all of the work directories.
+ Create a _SUCCESS file to make it as successful.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="cleanupJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="abortJob"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="state" type="org.apache.hadoop.mapreduce.JobStatus.State"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the temporary directory, including all of the work directories.
+ @param context the job's context]]>
+      </doc>
+    </method>
+    <method name="setupTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[No task setup required.]]>
+      </doc>
+    </method>
+    <method name="commitTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move the files from the work directory to the job output directory
+ @param context the task context]]>
+      </doc>
+    </method>
+    <method name="abortTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Delete the work directory
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="needsTaskCommit" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Did this task write any files in the work directory?
+ @param context the task's context]]>
+      </doc>
+    </method>
+    <method name="isRecoverySupported" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isCommitJobRepeatable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="recoverTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="PENDING_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Name of directory where pending data is placed.  Data that has not been
+ committed yet.]]>
+      </doc>
+    </field>
+    <field name="TEMP_DIR_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Temporary directory name 
+
+ The static variable to be compatible with M/R 1.x]]>
+      </doc>
+    </field>
+    <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SUCCESSFUL_JOB_OUTPUT_DIR_MARKER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_FAILURE_ATTEMPTS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILEOUTPUTCOMMITTER_TASK_CLEANUP_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified 
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <class name="FileOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setCompressOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="compress" type="boolean"/>
+      <doc>
+      <![CDATA[Set whether the output of the job is compressed.
+ @param job the job to modify
+ @param compress should the output of the job be compressed?]]>
+      </doc>
+    </method>
+    <method name="getCompressOutput" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Is the job output compressed?
+ @param job the Job to look in
+ @return <code>true</code> if the job output should be compressed,
+         <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressorClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="codecClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param job the job to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+                   compress the job outputs]]>
+      </doc>
+    </method>
+    <method name="getOutputCompressorClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="defaultValue" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param job the {@link Job} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the 
+         job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="setOutputPath"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param job The job to modify
+ @param outputDir the {@link Path} of the output directory for 
+ the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+ 
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(TaskInputOutputContext)]]>
+      </doc>
+    </method>
+    <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Get the {@link Path} to the task's temporary output directory 
+  for the map-reduce job
+  
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+ 
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+ 
+ <p>In such cases there could be issues with 2 instances of the same TIP 
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick 
+ unique names per task-attempt (e.g. using the attemptid, say 
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p> 
+ 
+ <p>To get around this the Map-Reduce framework helps the application-writer 
+ out by maintaining a special 
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> 
+ sub-directory for each task-attempt on HDFS where the output of the 
+ task-attempt goes. On successful completion of the task-attempt the files 
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only) 
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the 
+ framework discards the sub-directory of unsuccessful task-attempts. This 
+ is completely transparent to the application.</p>
+ 
+ <p>The application-writer can take advantage of this by creating any 
+ side-files required in a work directory during execution 
+ of his task i.e. via 
+ {@link #getWorkOutputPath(TaskInputOutputContext)}, and
+ the framework will move them out similarly - thus she doesn't have to pick 
+ unique paths per task-attempt.</p>
+ 
+ <p>The entire discussion holds true for maps of jobs with 
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case, 
+ goes directly to HDFS.</p> 
+ 
+ @return the {@link Path} to the task's temporary output directory 
+ for the map-reduce job.]]>
+      </doc>
+    </method>
+    <method name="getPathForWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueFile} method to make the file name
+ unique for the task.</p>
+
+ @param context the context for the task.
+ @param name the name for the file.
+ @param extension the extension for the file
+ @return a unique path accross all tasks of the job.]]>
+      </doc>
+    </method>
+    <method name="getUniqueFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="true"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="extension" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a unique filename, based on the task id, name, and extension
+ @param context the task that is calling this
+ @param name the base filename
+ @param extension the filename extension
+ @return a string like $name-[mrsct]-$id$extension]]>
+      </doc>
+    </method>
+    <method name="getDefaultWorkFile" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="extension" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the default path and filename for the output format.
+ @param context the task context
+ @param extension an extension to add to the filename
+ @return a full path $output/_temporary/$taskid/part-[mr]-$id
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getOutputName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the base output name for the output file.]]>
+      </doc>
+    </method>
+    <method name="setOutputName"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the base output name for output file to be created.]]>
+      </doc>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="BASE_OUTPUT_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="PART" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="COMPRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configuration option: should output be compressed? {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_CODEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If compression is enabled, name of codec: {@value}.]]>
+      </doc>
+    </field>
+    <field name="COMPRESS_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Type of compression {@value}: NONE, RECORD, BLOCK.
+ Generally only used in {@code SequenceFileOutputFormat}.]]>
+      </doc>
+    </field>
+    <field name="OUTDIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Destination directory of work: {@value}.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A base class for {@link OutputFormat}s that read from {@link FileSystem}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <class name="FileOutputFormatCounter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <class name="FilterOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FilterOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FilterOutputFormat" type="org.apache.hadoop.mapreduce.OutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a FilterOutputFormat based on the underlying output format.
+ @param baseOut the underlying OutputFormat]]>
+      </doc>
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="baseOut" type="org.apache.hadoop.mapreduce.OutputFormat"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[FilterOutputFormat is a convenience class that wraps OutputFormat.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <class name="LazyOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FilterOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LazyOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOutputFormatClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the underlying output format for LazyOutputFormat.
+ @param job the {@link Job} to modify
+ @param theClass the underlying class]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="OUTPUT_FORMAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A Convenience class that creates output lazily.
+ Use in conjuction with org.apache.hadoop.mapreduce.lib.output.MultipleOutputs to recreate the
+ behaviour of org.apache.hadoop.mapred.lib.MultipleTextOutputFormat (etc) of the old Hadoop API.
+ See {@link MultipleOutputs} documentation for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <class name="MapFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MapFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getReaders" return="org.apache.hadoop.io.MapFile.Reader[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="dir" type="org.apache.hadoop.fs.Path"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Open the output generated by this format.]]>
+      </doc>
+    </method>
+    <method name="getEntry" return="org.apache.hadoop.io.Writable"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="org.apache.hadoop.io.MapFile.Reader[]"/>
+      <param name="partitioner" type="org.apache.hadoop.mapreduce.Partitioner"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get an entry from output generated by this class.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes 
+ {@link MapFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <class name="MultipleOutputs" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MultipleOutputs" type="org.apache.hadoop.mapreduce.TaskInputOutputContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates and initializes multiple outputs support,
+ it should be instantiated in the Mapper/Reducer setup method.
+
+ @param context the TaskInputOutputContext object]]>
+      </doc>
+    </constructor>
+    <method name="addNamedOutput"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="outputFormatClass" type="java.lang.Class"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Adds a named output for the job.
+
+ @param job               job to add the named output
+ @param namedOutput       named output name, it has to be a word, letters
+                          and numbers only, cannot be the word 'part' as
+                          that is reserved for the default output.
+ @param outputFormatClass OutputFormat class.
+ @param keyClass          key class
+ @param valueClass        value class]]>
+      </doc>
+    </method>
+    <method name="setCountersEnabled"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[Enables or disables counters for the named outputs.
+ 
+ The counters group is the {@link MultipleOutputs} class name.
+ The names of the counters are the same as the named outputs. These
+ counters count the number records written to each output name.
+ By default these counters are disabled.
+
+ @param job    job  to enable counters
+ @param enabled indicates if the counters will be enabled or not.]]>
+      </doc>
+    </method>
+    <method name="getCountersEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Returns if the counters for the named outputs are enabled or not.
+ By default these counters are disabled.
+
+ @param job    the job 
+ @return TRUE if the counters are enabled, FALSE if they are disabled.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to the namedOutput.
+
+ Output path is a unique file generated for the namedOutput.
+ For example, {namedOutput}-(m|r)-{part-number}
+ 
+ @param namedOutput the named output name
+ @param key         the key
+ @param value       the value]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="namedOutput" type="java.lang.String"/>
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key and value to baseOutputPath using the namedOutput.
+ 
+ @param namedOutput    the named output name
+ @param key            the key
+ @param value          the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEYOUT"/>
+      <param name="value" type="VALUEOUT"/>
+      <param name="baseOutputPath" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write key value to an output file name.
+ 
+ Gets the record writer from job's output format.  
+ Job's output format should be a FileOutputFormat.
+ 
+ @param key       the key
+ @param value     the value
+ @param baseOutputPath base-output path to write the record to.
+ Note: Framework will generate unique filename for the baseOutputPath
+ <b>Warning</b>: when the baseOutputPath is a path that resolves
+ outside of the final job output directory, the directory is created
+ immediately and then persists through subsequent task retries, breaking
+ the concept of output committing.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Closes all the opened outputs.
+ 
+ This should be called from cleanup method of map/reduce task.
+ If overridden subclasses must invoke <code>super.close()</code> at the
+ end of their <code>close()</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The MultipleOutputs class simplifies writing output data 
+ to multiple outputs
+ 
+ <p> 
+ Case one: writing to additional outputs other than the job default output.
+
+ Each additional output, or named output, may be configured with its own
+ <code>OutputFormat</code>, with its own key class and with its own value
+ class.
+ </p>
+ 
+ <p>
+ Case two: to write data to different files provided by user
+ </p>
+ 
+ <p>
+ MultipleOutputs supports counters, by default they are disabled. The 
+ counters group is the {@link MultipleOutputs} class name. The names of the 
+ counters are the same as the output name. These count the number records 
+ written to each output name.
+ </p>
+ 
+ Usage pattern for job submission:
+ <pre>
+
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+ </pre>
+ <p>
+ Usage in Reducer:
+ <pre>
+ &lt;K, V&gt; String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer&lt;WritableComparable, Writable,WritableComparable, Writable&gt; {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator&lt;Writable&gt; values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ </pre>
+ 
+ <p>
+ When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat,
+ MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat
+ from the old Hadoop API - ie, output can be written from the Reducer to more than one location.
+ </p>
+ 
+ <p>
+ Use <code>MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath)</code> to write key and 
+ value to a path specified by <code>baseOutputPath</code>, with no need to specify a named output.
+ <b>Warning</b>: when the baseOutputPath passed to MultipleOutputs.write
+ is a path that resolves outside of the final job output directory, the
+ directory is created immediately and then persists through subsequent
+ task retries, breaking the concept of output committing:
+ </p>
+ 
+ <pre>
+ private MultipleOutputs&lt;Text, Text&gt; out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs&lt;Text, Text&gt;(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable&lt;Text&gt; values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(&lt;<i>parameter list...</i>&gt;));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+ </pre>
+ 
+ <p>
+ Use your own code in <code>generateFileName()</code> to create a custom path to your results. 
+ '/' characters in <code>baseOutputPath</code> will be translated into directory levels in your file system. 
+ Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. 
+ No call to <code>context.write()</code> is necessary. See example <code>generateFileName()</code> code below. 
+ </p>
+ 
+ <pre>
+ private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+ </pre>
+ 
+ <p>
+ Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000.
+ To prevent this use <code>LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);</code>
+ instead of <code>job.setOutputFormatClass(TextOutputFormat.class);</code> in your Hadoop job configuration.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.MultipleOutputs -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <class name="NullOutputFormat" extends="org.apache.hadoop.mapreduce.OutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NullOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.JobContext"/>
+    </method>
+    <method name="getOutputCommitter" return="org.apache.hadoop.mapreduce.OutputCommitter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <doc>
+    <![CDATA[Consume all outputs and put them in /dev/null.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.NullOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <class name="PartialFileOutputCommitter" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter"/>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <constructor name="PartialFileOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getCommittedTaskPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="int"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+    </method>
+    <method name="cleanUpPartialOutputForTask"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputCommitter} that commits files specified
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PartialFileOutputCommitter -->
+  <!-- start interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <interface name="PartialOutputCommitter"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="cleanUpPartialOutputForTask"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Remove all previously committed outputs from prior executions of this task.
+ @param context Context for cleaning up previously promoted output.
+ @throws IOException If cleanup fails, then the state of the task my not be
+                     well defined.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface for an {@link org.apache.hadoop.mapreduce.OutputCommitter}
+ implementing partial commit of task output, as during preemption.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.mapreduce.lib.output.PartialOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <class name="PathOutputCommitter" extends="org.apache.hadoop.mapreduce.OutputCommitter"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.TaskAttemptContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a task attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <constructor name="PathOutputCommitter" type="org.apache.hadoop.fs.Path, org.apache.hadoop.mapreduce.JobContext"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Constructor for a job attempt.
+ Subclasses should provide a public constructor with this signature.
+ @param outputPath output path: may be null
+ @param context task context
+ @throws IOException IO problem]]>
+      </doc>
+    </constructor>
+    <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final directory where work will be placed once the job
+ is committed. This may be null, in which case, there is no output
+ path to write data to.
+ @return the path where final output of the job should be placed.]]>
+      </doc>
+    </method>
+    <method name="hasOutputPath" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Predicate: is there an output path?
+ @return true if we have an output path set, else false.]]>
+      </doc>
+    </method>
+    <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the directory that the task should write results into.
+ Warning: there's no guarantee that this work path is on the same
+ FS as the final output, or that it's visible across machines.
+ May be null.
+ @return the work directory
+ @throws IOException IO problem]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A committer which somehow commits data written to a working directory
+ to the final directory during the commit process. The reference
+ implementation of this is the {@link FileOutputCommitter}.
+
+ There are two constructors, both of which do nothing but long and
+ validate their arguments.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <class name="SequenceFileAsBinaryOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileAsBinaryOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setSequenceFileOutputKeyClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the key class for the {@link SequenceFile}
+ <p>This allows the user to specify the key class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="setSequenceFileOutputValueClass"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="theClass" type="java.lang.Class"/>
+      <doc>
+      <![CDATA[Set the value class for the {@link SequenceFile}
+ <p>This allows the user to specify the value class to be different 
+ from the actual class ({@link BytesWritable}) used for writing </p>
+ 
+ @param job the {@link Job} to modify
+ @param theClass the SequenceFile output key class.]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputKeyClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the key class for the {@link SequenceFile}
+ 
+ @return the key class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getSequenceFileOutputValueClass" return="java.lang.Class"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the value class for the {@link SequenceFile}
+ 
+ @return the value class of the {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkOutputSpecs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <field name="KEY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VALUE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[An {@link org.apache.hadoop.mapreduce.OutputFormat} that writes keys, 
+ values to {@link SequenceFile}s in binary(raw) format]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileAsBinaryOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <class name="SequenceFileOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SequenceFileOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSequenceWriter" return="org.apache.hadoop.io.SequenceFile.Writer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <param name="keyClass" type="java.lang.Class"/>
+      <param name="valueClass" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="getOutputCompressionType" return="org.apache.hadoop.io.SequenceFile.CompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job}
+ @return the {@link CompressionType} for the output {@link SequenceFile}, 
+         defaulting to {@link CompressionType#RECORD}]]>
+      </doc>
+    </method>
+    <method name="setOutputCompressionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="style" type="org.apache.hadoop.io.SequenceFile.CompressionType"/>
+      <doc>
+      <![CDATA[Set the {@link CompressionType} for the output {@link SequenceFile}.
+ @param job the {@link Job} to modify
+ @param style the {@link CompressionType} for the output
+              {@link SequenceFile}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes {@link SequenceFile}s.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+  <class name="TextOutputFormat" extends="org.apache.hadoop.mapreduce.lib.output.FileOutputFormat"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TextOutputFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecordWriter" return="org.apache.hadoop.mapreduce.RecordWriter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.TaskAttemptContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <field name="SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SEPERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #SEPARATOR}">
+      <doc>
+      <![CDATA[@deprecated Use {@link #SEPARATOR}]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[An {@link OutputFormat} that writes plain text files.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.partition">
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <class name="BinaryPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="BinaryPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setOffsets"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="left" type="int"/>
+      <param name="right" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[left:(right+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param left left Python-style offset
+ @param right right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setLeftOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[offset:]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset left Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setRightOffset"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="offset" type="int"/>
+      <doc>
+      <![CDATA[Set the subarray to be used for partitioning to 
+ <code>bytes[:(offset+1)]</code> in Python syntax.
+ 
+ @param conf configuration object
+ @param offset right Python-style offset]]>
+      </doc>
+    </method>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.io.BinaryComparable"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+      <doc>
+      <![CDATA[Use (the specified slice of the array returned by) 
+ {@link BinaryComparable#getBytes()} to partition.]]>
+      </doc>
+    </method>
+    <field name="LEFT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RIGHT_OFFSET_PROPERTY_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>Partition {@link BinaryComparable} keys using a configurable part of 
+ the bytes array returned by {@link BinaryComparable#getBytes()}.</p>
+ 
+ <p>The subarray to be used for the partitioning can be defined by means
+ of the following properties:
+ <ul>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.left.offset</i>:
+     left offset in array (0 by default)
+   </li>
+   <li>
+     <i>mapreduce.partition.binarypartitioner.right.offset</i>: 
+     right offset in array (-1 by default)
+   </li>
+ </ul>
+ Like in Python, both negative and positive offsets are allowed, but
+ the meaning is slightly different. In case of an array of length 5,
+ for instance, the possible offsets are:
+ <pre><code>
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+ </code></pre>
+ The first row of numbers gives the position of the offsets 0...5 in 
+ the array; the second row gives the corresponding negative offsets. 
+ Contrary to Python, the specified subarray has byte <code>i</code> 
+ and <code>j</code> as first and last element, repectively, when 
+ <code>i</code> and <code>j</code> are the left and right offset.
+ 
+ <p>For Hadoop programs written in Java, it is advisable to use one of 
+ the following static convenience methods for setting the offsets:
+ <ul>
+   <li>{@link #setOffsets}</li>
+   <li>{@link #setLeftOffset}</li>
+   <li>{@link #setRightOffset}</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.BinaryPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <class name="HashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="HashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Use {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Partition keys by their {@link Object#hashCode()}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <class name="InputSampler" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="InputSampler" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="writePartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="sampler" type="org.apache.hadoop.mapreduce.lib.partition.InputSampler.Sampler"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Write a partition file for the given job, using the Sampler provided.
+ Queries the sampler for a sample keyset, sorts by the output key
+ comparator, selects the keys for each rank, and writes to the destination
+ returned from {@link TotalOrderPartitioner#getPartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Driver for InputSampler from the command line.
+ Configures a JobConf instance and calls {@link #writePartitionFile}.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[Utility for collecting samples and writing a partition file for
+ {@link TotalOrderPartitioner}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.InputSampler -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <class name="KeyFieldBasedComparator" extends="org.apache.hadoop.io.WritableComparator"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="b1" type="byte[]"/>
+      <param name="s1" type="int"/>
+      <param name="l1" type="int"/>
+      <param name="b2" type="byte[]"/>
+      <param name="s2" type="int"/>
+      <param name="l2" type="int"/>
+    </method>
+    <method name="setKeyFieldComparatorOptions"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedComparator} options used to compare keys.
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field). opts are ordering options. The supported options
+  are:
+    -n, (Sort numerically)
+    -r, (Reverse the result of comparison)]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldComparatorOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedComparator} options]]>
+      </doc>
+    </method>
+    <field name="COMPARATOR_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This comparator implementation provides a subset of the features provided
+ by the Unix/GNU Sort. In particular, the supported features are:
+ -n, (Sort numerically)
+ -r, (Reverse the result of comparison)
+ -k pos1[,pos2], where pos is of the form f[.c][opts], where f is the number
+  of the field to use, and c is the number of the first character from the
+  beginning of the field. Fields and character posns are numbered starting
+  with 1; a character position of zero in pos2 indicates the field's last
+  character. If '.c' is omitted from pos1, it defaults to 1 (the beginning
+  of the field); if omitted from pos2, it defaults to 0 (the end of the
+  field). opts are ordering options (any of 'nr' as described above). 
+ We assume that the fields in the key are separated by 
+ {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <class name="KeyFieldBasedPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="KeyFieldBasedPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K2"/>
+      <param name="value" type="V2"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="b" type="byte[]"/>
+      <param name="start" type="int"/>
+      <param name="end" type="int"/>
+      <param name="currentHash" type="int"/>
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="hash" type="int"/>
+      <param name="numReduceTasks" type="int"/>
+    </method>
+    <method name="setKeyFieldPartitionerOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="keySpec" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the {@link KeyFieldBasedPartitioner} options used for 
+ {@link Partitioner}
+ 
+ @param keySpec the key specification of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+      </doc>
+    </method>
+    <method name="getKeyFieldPartitionerOption" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.JobContext"/>
+      <doc>
+      <![CDATA[Get the {@link KeyFieldBasedPartitioner} options]]>
+      </doc>
+    </method>
+    <field name="PARTITIONER_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Defines a way to partition keys based on certain key fields (also see
+  {@link KeyFieldBasedComparator}.
+  The key specification supported is of the form -k pos1[,pos2], where,
+  pos is of the form f[.c][opts], where f is the number
+  of the key field to use, and c is the number of the first character from
+  the beginning of the field. Fields and character posns are numbered 
+  starting with 1; a character position of zero in pos2 indicates the
+  field's last character. If '.c' is omitted from pos1, it defaults to 1
+  (the beginning of the field); if omitted from pos2, it defaults to 0 
+  (the end of the field).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <class name="RehashPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RehashPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numReduceTasks" type="int"/>
+      <doc>
+      <![CDATA[Rehash {@link Object#hashCode()} to partition.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This partitioner rehashes values returned by {@link Object#hashCode()}
+  to get smoother distribution between partitions which may improve
+  reduce reduce time in some cases and should harm things in no cases.
+  This partitioner is suggested with Integer and Long keys with simple
+  patterns in their distributions.
+  @since 2.0.3]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.RehashPartitioner -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+  <class name="TotalOrderPartitioner" extends="org.apache.hadoop.mapreduce.Partitioner"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.conf.Configurable"/>
+    <constructor name="TotalOrderPartitioner"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="setConf"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Read in the partition file and build indexing data structures.
+ If the keytype is {@link org.apache.hadoop.io.BinaryComparable} and
+ <tt>total.order.partitioner.natural.order</tt> is not false, a trie
+ of the first <tt>total.order.partitioner.max.trie.depth</tt>(2) + 1 bytes
+ will be built. Otherwise, keys will be located using a binary search of
+ the partition keyset using the {@link org.apache.hadoop.io.RawComparator}
+ defined for this job. The input file must be sorted with the same
+ comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]>
+      </doc>
+    </method>
+    <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPartition" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="K"/>
+      <param name="value" type="V"/>
+      <param name="numPartitions" type="int"/>
+    </method>
+    <method name="setPartitionFile"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="p" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path to the SequenceFile storing the sorted partition keyset.
+ It must be the case that for <tt>R</tt> reduces, there are <tt>R-1</tt>
+ keys in the SequenceFile.]]>
+      </doc>
+    </method>
+    <method name="getPartitionFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the path to the SequenceFile storing the sorted partition keyset.
+ @see #setPartitionFile(Configuration, Path)]]>
+      </doc>
+    </method>
+    <field name="DEFAULT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PARTITIONER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_TRIE_DEPTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NATURAL_ORDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Partitioner effecting a total order by reading split points from
+ an externally generated source.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner -->
+</package>
+<package name="org.apache.hadoop.mapreduce.lib.reduce">
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <class name="IntSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IntSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="Key"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <class name="LongSumReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LongSumReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="reduce"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="KEY"/>
+      <param name="values" type="java.lang.Iterable"/>
+      <param name="context" type="org.apache.hadoop.mapreduce.Reducer.Context"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer -->
+  <!-- start class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+  <class name="WrappedReducer" extends="org.apache.hadoop.mapreduce.Reducer"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="WrappedReducer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReducerContext" return="org.apache.hadoop.mapreduce.Reducer.Context"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reduceContext" type="org.apache.hadoop.mapreduce.ReduceContext"/>
+      <doc>
+      <![CDATA[A a wrapped {@link Reducer.Context} for custom implementations.
+ @param reduceContext <code>ReduceContext</code> to be wrapped
+ @return a wrapped <code>Reducer.Context</code> for custom implementations]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link Reducer} which wraps a given one to allow for custom 
+ {@link Reducer.Context} implementations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer -->
+</package>
+<package name="org.apache.hadoop.mapreduce.server.jobtracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.server.tasktracker">
+</package>
+<package name="org.apache.hadoop.mapreduce.task.annotation">
+  <!-- start class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+  <class name="Checkpointable"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.annotation.Annotation"/>
+    <doc>
+    <![CDATA[Contract representing to the framework that the task can be safely preempted
+ and restarted between invocations of the user-defined function.
+
+ This is often true when the result of a function does not rely on state
+ derived from previous elements in the record stream, but the guarantee is
+ left as an exercise to the implementor.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.task.annotation.Checkpointable -->
+</package>
+<package name="org.apache.hadoop.mapreduce.tools">
+  <!-- start class org.apache.hadoop.mapreduce.tools.CLI -->
+  <class name="CLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="CLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="CLI" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getCounter" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="counters" type="org.apache.hadoop.mapreduce.Counters"/>
+      <param name="counterGroupName" type="java.lang.String"/>
+      <param name="counterName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getTaskLogURL" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="taskId" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+      <param name="baseUrl" type="java.lang.String"/>
+    </method>
+    <method name="displayTasks"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="job" type="org.apache.hadoop.mapreduce.Job"/>
+      <param name="type" type="java.lang.String"/>
+      <param name="state" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Display the information about a job's tasks, of a particular type and
+ in a particular state
+ 
+ @param job the job
+ @param type the type of the task (map/reduce/setup/cleanup)
+ @param state the state of the task 
+ (pending/running/completed/failed/killed)
+ @throws IOException when there is an error communicating with the master
+ @throws InterruptedException
+ @throws IllegalArgumentException if an invalid type/state is passed]]>
+      </doc>
+    </method>
+    <method name="displayJobList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="jobs" type="org.apache.hadoop.mapreduce.JobStatus[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="argv" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="cluster" type="org.apache.hadoop.mapreduce.Cluster"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="headerPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="dataPattern" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interprets the map reduce cli options]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.mapreduce.tools.CLI -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml
new file mode 100644
index 0000000000000..ed4979ac1fa14
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:58:05 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce JobClient 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/classes:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-mapreduce-client-common-3.2.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.2.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.2.4.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.2.4.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/slf4j/slf4j-reload4j/1.7.35/slf4j-reload4j-1.7.35.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/site/jdiff/xml -apiname Apache Hadoop MapReduce JobClient 3.2.4 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.3.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.3.xml
new file mode 100644
index 0000000000000..3b5eb71c92539
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.3.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:14:59 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce JobClient 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/classes:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-mapreduce-client-common-3.3.3.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.3.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.3.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.3.3.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/site/jdiff/xml -apiname Apache Hadoop MapReduce JobClient 3.3.3 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml
new file mode 100644
index 0000000000000..dd96ac9059277
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:16:00 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop MapReduce JobClient 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -verbose -classpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/classes:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-mapreduce-client-common-3.3.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.4.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-3.3.4.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-3.3.4.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -apidir /build/source/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/site/jdiff/xml -apiname Apache Hadoop MapReduce JobClient 3.3.4 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+
+</api>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index c55cf8c3108a9..e3b3511c0ce17 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -87,6 +87,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 7d151adea7d6c..5dffd735fdafd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -28,6 +28,10 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ipc.ProtocolSignature;
@@ -50,8 +54,8 @@
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
 import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
 import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
 import org.apache.hadoop.mapreduce.v2.app.security.authorize.MRAMPolicyProvider;
@@ -61,10 +65,6 @@
 import org.apache.hadoop.util.StringInterner;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
 
 /**
  * This class is responsible for talking to the task umblical.
@@ -409,6 +409,11 @@ public AMFeedback statusUpdate(TaskAttemptID taskAttemptID,
       if (LOG.isDebugEnabled()) {
         LOG.debug("Ping from " + taskAttemptID.toString());
       }
+      // Consider ping from the tasks for liveliness check
+      if (getConfig().getBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK,
+          MRJobConfig.DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK)) {
+        taskHeartbeatHandler.progressing(yarnAttemptID);
+      }
       return feedback;
     }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index a513c16d2308b..bbf527ebff53a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -59,7 +59,7 @@
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
index ee4ec2c86a1c4..ecae4f2fc061a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryCopyService.java
@@ -35,6 +35,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
 /**
  * Reads in history events from the JobHistoryFile and sends them out again
  * to be recorded.
@@ -118,7 +122,11 @@ public static FSDataInputStream getPreviousJobHistoryFileStream(
         fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
           jobId, (applicationAttemptId.getAttemptId() - 1)));
     LOG.info("History file is at " + historyFile);
-    in = fc.open(historyFile);
+    in = awaitFuture(
+        fc.openFile(historyFile)
+            .opt(FS_OPTION_OPENFILE_READ_POLICY,
+                FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .build());
     return in;
   }
   
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index fc9254af97ab8..9023f6a11ad84 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1481,7 +1481,7 @@ protected void processDoneFiles(JobId jobId) throws IOException {
       summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
       summaryFileOut.close();
       doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
-          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+          JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig())));
     } catch (IOException e) {
       LOG.info("Unable to write out JobSummaryInfo to ["
           + qualifiedSummaryDoneFile + "]", e);
@@ -1738,8 +1738,9 @@ protected boolean moveToDoneNow(Path fromPath, Path toPath)
       boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
           false, getConfig());
 
-      doneDirFS.setPermission(toPath, new FsPermission(
-          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+      doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils.
+          getConfiguredHistoryIntermediateUserDoneDirPermissions(
+          getConfig())));
       if (copied) {
         LOG.info("Copied from: " + fromPath.toString()
             + " to done location: " + toPath.toString());
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 1640ee2f4fe57..fe998bbc3f0cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -146,7 +146,7 @@
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java
index 82360f08d4518..2465811c82a30 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncher.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.mapreduce.v2.app.launcher;
 
 
-import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.yarn.event.EventHandler;
 
 public interface ContainerLauncher 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
index d09b3cb1e56b3..d184d9be64bf8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
@@ -379,27 +379,38 @@ class EventProcessor implements Runnable {
 
     @Override
     public void run() {
-      LOG.info("Processing the event " + event.toString());
+      LOG.info("Processing the event {}", event);
 
       // Load ContainerManager tokens before creating a connection.
       // TODO: Do it only once per NodeManager.
       ContainerId containerID = event.getContainerID();
 
-      Container c = getContainer(event);
       switch(event.getType()) {
 
       case CONTAINER_REMOTE_LAUNCH:
         ContainerRemoteLaunchEvent launchEvent
             = (ContainerRemoteLaunchEvent) event;
-        c.launch(launchEvent);
+        getContainer(event).launch(launchEvent);
         break;
 
       case CONTAINER_REMOTE_CLEANUP:
-        c.kill(event.getDumpContainerThreads());
+        // If the container failed to launch earlier (due to dead node for example),
+        // it has been marked as FAILED and removed from containers during
+        // CONTAINER_REMOTE_LAUNCH event handling.
+        // Skip kill() such container during CONTAINER_REMOTE_CLEANUP as
+        // it is not necessary and could cost 15 minutes delay if the node is dead.
+        if (!containers.containsKey(containerID)) {
+          LOG.info("Skip cleanup of already-removed container {}", containerID);
+          // send killed event to task attempt regardless like in kill().
+          context.getEventHandler().handle(new TaskAttemptEvent(event.getTaskAttemptID(),
+              TaskAttemptEventType.TA_CONTAINER_CLEANED));
+          return;
+        }
+        getContainer(event).kill(event.getDumpContainerThreads());
         break;
 
       case CONTAINER_COMPLETED:
-        c.done();
+        getContainer(event).done();
         break;
 
       }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index 61cc2eb898eba..16ca585e9b460 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -213,12 +213,10 @@ protected AllocateResponse makeRemoteRequest() throws YarnException,
         allocateResponse.getCompletedContainersStatuses().size();
 
     if (ask.size() > 0 || release.size() > 0) {
-      LOG.info("getResources() for " + applicationId + ":" + " ask="
-          + ask.size() + " release= " + release.size() + " newContainers="
-          + allocateResponse.getAllocatedContainers().size()
-          + " finishedContainers=" + numCompletedContainers
-          + " resourcelimit=" + availableResources + " knownNMs="
-          + clusterNmCount);
+      LOG.info("applicationId={}: ask={} release={} newContainers={} finishedContainers={}"
+              + " resourceLimit={} knownNMs={}", applicationId, ask.size(), release.size(),
+          allocateResponse.getAllocatedContainers().size(), numCompletedContainers,
+          availableResources, clusterNmCount);
     }
 
     ask.clear();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java
index 8eda00528f15f..be35b68a56f5f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java
@@ -61,7 +61,7 @@ abstract class StartEndTimesBase implements TaskRuntimeEstimator {
       = new HashMap<Job, DataStatistics>();
 
 
-  private final Map<Job, Float> slowTaskRelativeTresholds
+  private final Map<Job, Float> slowTaskRelativeThresholds
       = new HashMap<Job, Float>();
 
   protected final Set<Task> doneTasks = new HashSet<Task>();
@@ -89,7 +89,7 @@ public void contextualize(Configuration conf, AppContext context) {
       final Job job = entry.getValue();
       mapperStatistics.put(job, new DataStatistics());
       reducerStatistics.put(job, new DataStatistics());
-      slowTaskRelativeTresholds.put
+      slowTaskRelativeThresholds.put
           (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f));
     }
   }
@@ -141,7 +141,7 @@ public long thresholdRuntime(TaskId taskID) {
 
     long result =  statistics == null
         ? Long.MAX_VALUE
-        : (long)statistics.outlier(slowTaskRelativeTresholds.get(job));
+        : (long)statistics.outlier(slowTaskRelativeThresholds.get(job));
     return result;
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
index 72f8047dc33d9..78174afb6f892 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
@@ -78,7 +78,7 @@
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.google.inject.Inject;
 
 @Path("/ws/v1/mapreduce")
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AttemptsPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AttemptsPage.java
index 3b45b168b7dc4..9a6a39555f983 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AttemptsPage.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AttemptsPage.java
@@ -28,7 +28,6 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
-import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
 import org.apache.hadoop.mapreduce.v2.app.job.Task;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
index f8b8c6ccdf1de..b5a7694e4cc6b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
@@ -17,23 +17,30 @@
 */
 package org.apache.hadoop.mapred;
 
-import java.util.function.Supplier;
-import org.apache.hadoop.mapred.Counters.Counter;
-import org.apache.hadoop.mapreduce.checkpoint.EnumCounter;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.Counters.Counter;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.checkpoint.CheckpointID;
+import org.apache.hadoop.mapreduce.checkpoint.EnumCounter;
 import org.apache.hadoop.mapreduce.checkpoint.FSCheckpointID;
 import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
@@ -48,9 +55,9 @@
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
 import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
 import org.apache.hadoop.mapreduce.v2.app.rm.preemption.CheckpointAMPreemptionPolicy;
-import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.event.Dispatcher;
@@ -60,17 +67,22 @@
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.util.ControlledClock;
 import org.apache.hadoop.yarn.util.SystemClock;
-import org.junit.After;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * Tests the behavior of TaskAttemptListenerImpl.
@@ -417,6 +429,23 @@ public void testStatusUpdateProgress()
     verify(hbHandler).progressing(eq(attemptId));
   }
 
+  @Test
+  public void testPingUpdateProgress() throws IOException, InterruptedException {
+    configureMocks();
+    Configuration conf = new Configuration();
+    conf.setBoolean(MRJobConfig.MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK, true);
+    listener.init(conf);
+    listener.start();
+    listener.registerPendingTask(task, wid);
+    listener.registerLaunchedTask(attemptId, wid);
+    verify(hbHandler).register(attemptId);
+
+    // make sure a ping does report progress
+    AMFeedback feedback = listener.statusUpdate(attemptID, null);
+    assertTrue(feedback.getTaskFound());
+    verify(hbHandler, times(1)).progressing(eq(attemptId));
+  }
+
   @Test
   public void testSingleStatusUpdate()
       throws IOException, InterruptedException {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index fbeb94a2a16ac..8159bc2456ca0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -43,6 +44,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.CounterGroup;
@@ -1036,6 +1038,48 @@ public void testDontSetTrackingURLIfHistoryWriteThrows() throws Exception {
       jheh.stop();
     }
   }
+
+  @Test(timeout = 50000)
+  public void testJobHistoryFilePermissions() throws Exception {
+    TestParams t = new TestParams(true);
+    Configuration conf = new Configuration();
+    String setFilePermission = "777";
+    conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission);
+
+    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString());
+
+    JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext,
+        0, false);
+    JHEvenHandlerForTest jheh = spy(realJheh);
+    jheh.init(conf);
+
+    try {
+      jheh.start();
+      handleEvent(jheh, new JobHistoryEvent(t.jobId,
+          new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost",
+              3000, 4000, -1)));
+
+      // Job finishes and successfully writes history
+      handleEvent(jheh, new JobHistoryEvent(t.jobId,
+          new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0,
+              0, 0, 0, 0, 0,
+              new Counters(),
+              new Counters(), new Counters())));
+
+      verify(jheh, times(1)).processDoneFiles(any(JobId.class));
+
+      String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId);
+      String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+      FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0));
+      Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName);
+      FsPermission getIntermediateSummaryFilePermission =
+          fs.getFileStatus(intermediateSummaryFileNamePath).getPermission();
+      assertEquals(setFilePermission,
+          String.valueOf(getIntermediateSummaryFilePermission.toOctal()));
+    } finally {
+      jheh.stop();
+    }
+  }
 }
 
 class JHEvenHandlerForTest extends JobHistoryEventHandler {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
index 44a4760eb6803..3b5cfe221eded 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
@@ -46,9 +46,7 @@
 import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl;
 import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
 import org.junit.Test;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
index 5bf51022327bc..06550378ba939 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
@@ -83,7 +83,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 2057cc80ff01a..88ba8943ceb3f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -209,14 +209,11 @@ public void testHandle() throws Exception {
       ut.waitForPoolToIdle();
       
       verify(mockCM).startContainers(any(StartContainersRequest.class));
-      
+
       LOG.info("inserting cleanup event");
-      ContainerLauncherEvent mockCleanupEvent = 
-        mock(ContainerLauncherEvent.class);
-      when(mockCleanupEvent.getType())
-        .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
-      when(mockCleanupEvent.getContainerID())
-        .thenReturn(contId);
+      ContainerLauncherEvent mockCleanupEvent = mock(ContainerLauncherEvent.class);
+      when(mockCleanupEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
+      when(mockCleanupEvent.getContainerID()).thenReturn(contId);
       when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
       when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
       ut.handle(mockCleanupEvent);
@@ -283,8 +280,21 @@ public void testOutOfOrder() throws Exception {
       ut.handle(mockLaunchEvent);
       
       ut.waitForPoolToIdle();
-      
-      verify(mockCM, never()).startContainers(any(StartContainersRequest.class));
+
+      verify(mockCM).startContainers(any(StartContainersRequest.class));
+
+      LOG.info("inserting cleanup event");
+      ContainerLauncherEvent mockCleanupEvent2 = mock(ContainerLauncherEvent.class);
+      when(mockCleanupEvent2.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
+      when(mockCleanupEvent2.getContainerID()).thenReturn(contId);
+      when(mockCleanupEvent2.getTaskAttemptID()).thenReturn(taskAttemptId);
+      when(mockCleanupEvent2.getContainerMgrAddress()).thenReturn(cmAddress);
+      ut.handle(mockCleanupEvent2);
+
+      ut.waitForPoolToIdle();
+
+      // Verifies stopContainers is called on existing container
+      verify(mockCM).stopContainers(any(StopContainersRequest.class));
     } finally {
       ut.stop();
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
index 5def1d91494c4..29063668b7d67 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.MockAppContext;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -290,7 +291,7 @@ public void verifyAMInfo(JSONObject info, AppContext ctx)
 
   public void verifyAMInfoXML(String xml, AppContext ctx)
       throws JSONException, Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -335,7 +336,7 @@ public void verifyBlacklistedNodesInfo(JSONObject blacklist, AppContext ctx)
   
   public void verifyBlacklistedNodesInfoXML(String xml, AppContext ctx)
       throws JSONException, Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
index f20ac6ff1b88d..28cfb90a17a72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempt.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -185,7 +186,7 @@ public void testGetTaskAttemptIdXMLState() throws Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
@@ -259,7 +260,7 @@ public void testPutTaskAttemptIdXMLState() throws Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
index 32d054ff5c514..aad41966e8f3d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.mapreduce.v2.app.job.Task;
 import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -192,7 +193,7 @@ public void testTaskAttemptsXML() throws JSONException, Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
@@ -316,7 +317,7 @@ public void testTaskAttemptIdXML() throws JSONException, Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
@@ -600,7 +601,7 @@ public void testTaskAttemptIdXMLCounters() throws JSONException, Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
index ee7bb0e3c27dd..6568186648d02 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobConf.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.mapreduce.v2.app.MockAppContext;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -216,7 +217,7 @@ public void testJobConfXML() throws JSONException, Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
index cc57134d236ec..5e4e9f70b35a8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
@@ -44,6 +44,7 @@
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -180,7 +181,7 @@ public void testJobsXML() throws Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -342,7 +343,7 @@ public void testJobIdInvalidXML() throws JSONException, Exception {
           response.getType().toString());
       String msg = response.getEntity(String.class);
       System.out.println(msg);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(msg));
@@ -411,7 +412,7 @@ public void testJobIdXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -702,7 +703,7 @@ public void testJobCountersXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -859,7 +860,7 @@ public void testJobAttemptsXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
index ab4d818f33868..49187a016c32d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.app.job.Task;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -189,7 +190,7 @@ public void testTasksXML() throws JSONException, Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -533,7 +534,7 @@ public void testTaskIdXML() throws JSONException, Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
@@ -713,7 +714,7 @@ public void testJobTaskCountersXML() throws Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
index 8f17ffda422d4..5469d4a9298fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
@@ -45,14 +45,12 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@@ -95,13 +93,13 @@ public synchronized void setup(JobConf conf, JobID jobId) throws IOException {
 
     // Find which resources are to be put on the local classpath
     Map<String, Path> classpaths = new HashMap<String, Path>();
-    Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf);
+    Path[] archiveClassPaths = JobContextImpl.getArchiveClassPaths(conf);
     if (archiveClassPaths != null) {
       for (Path p : archiveClassPaths) {
         classpaths.put(p.toUri().getPath().toString(), p);
       }
     }
-    Path[] fileClassPaths = DistributedCache.getFileClassPaths(conf);
+    Path[] fileClassPaths = JobContextImpl.getFileClassPaths(conf);
     if (fileClassPaths != null) {
       for (Path p : fileClassPaths) {
         classpaths.put(p.toUri().getPath().toString(), p);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index 1c11e7728b24a..15e07ac6abbe7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -20,7 +20,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.http.HttpConfig;
 
 /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
index e22726c303abf..1a1df010dd8c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
@@ -91,9 +91,6 @@ public class JobHistoryUtils {
   public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = 
     FsPermission.createImmutable((short) 01777);
 
-  public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = 
-    FsPermission.createImmutable((short) 0770); // rwx------
-  
   /**
    * Suffix for configuration files.
    */
@@ -206,7 +203,7 @@ public static String getConfiguredHistoryIntermediateDoneDirPrefix(
 
   /**
    * Gets the configured directory permissions for the user directories in the
-   * directory of the intermediate done history files. The user and the group
+   * Gets the configured permissions for the user directories and files in the
    * both need full permissions, this is enforced by this method.
    * @param conf The configuration object
    * @return FsPermission of the user directories
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 010a4ed411ec0..a3ccfd72d8ce8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -51,6 +51,7 @@
 import org.apache.hadoop.mapreduce.TaskID;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
@@ -272,12 +273,12 @@ public static void setClasspath(Map<String, String> environment,
         crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + "*", conf);
     // a * in the classpath will only find a .jar, so we need to filter out
     // all .jars and add everything else
-    addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf),
-        DistributedCache.getCacheFiles(conf),
+    addToClasspathIfNotJar(JobContextImpl.getFileClassPaths(conf),
+        JobContextImpl.getCacheFiles(conf),
         conf,
         environment, classpathEnvVar);
-    addToClasspathIfNotJar(DistributedCache.getArchiveClassPaths(conf),
-        DistributedCache.getCacheArchives(conf),
+    addToClasspathIfNotJar(JobContextImpl.getArchiveClassPaths(conf),
+        JobContextImpl.getCacheArchives(conf),
         conf,
         environment, classpathEnvVar);
     if (userClassesTakesPrecedence) {
@@ -483,8 +484,8 @@ public static void setupDistributedCache(Configuration conf,
 
     // Cache archives
     lrb.setType(LocalResourceType.ARCHIVE);
-    lrb.setUris(DistributedCache.getCacheArchives(conf));
-    lrb.setTimestamps(DistributedCache.getArchiveTimestamps(conf));
+    lrb.setUris(JobContextImpl.getCacheArchives(conf));
+    lrb.setTimestamps(JobContextImpl.getArchiveTimestamps(conf));
     lrb.setSizes(getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES));
     lrb.setVisibilities(DistributedCache.getArchiveVisibilities(conf));
     lrb.setSharedCacheUploadPolicies(
@@ -493,8 +494,8 @@ public static void setupDistributedCache(Configuration conf,
     
     // Cache files
     lrb.setType(LocalResourceType.FILE);
-    lrb.setUris(DistributedCache.getCacheFiles(conf));
-    lrb.setTimestamps(DistributedCache.getFileTimestamps(conf));
+    lrb.setUris(JobContextImpl.getCacheFiles(conf));
+    lrb.setTimestamps(JobContextImpl.getFileTimestamps(conf));
     lrb.setSizes(getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES));
     lrb.setVisibilities(DistributedCache.getFileVisibilities(conf));
     lrb.setSharedCacheUploadPolicies(
@@ -504,9 +505,9 @@ public static void setupDistributedCache(Configuration conf,
 
   /**
    * Set up the DistributedCache related configs to make
-   * {@link DistributedCache#getLocalCacheFiles(Configuration)}
+   * {@link JobContextImpl#getLocalCacheFiles(Configuration)}
    * and
-   * {@link DistributedCache#getLocalCacheArchives(Configuration)}
+   * {@link JobContextImpl#getLocalCacheArchives(Configuration)}
    * working.
    * @param conf
    * @throws java.io.IOException
@@ -518,7 +519,7 @@ public static void setupDistributedCacheLocal(Configuration conf)
     //        ^ ^ all symlinks are created in the current work-dir
 
     // Update the configuration object with localized archives.
-    URI[] cacheArchives = DistributedCache.getCacheArchives(conf);
+    URI[] cacheArchives = JobContextImpl.getCacheArchives(conf);
     if (cacheArchives != null) {
       List<String> localArchives = new ArrayList<String>();
       for (int i = 0; i < cacheArchives.length; ++i) {
@@ -538,7 +539,7 @@ public static void setupDistributedCacheLocal(Configuration conf)
     }
 
     // Update the configuration object with localized files.
-    URI[] cacheFiles = DistributedCache.getCacheFiles(conf);
+    URI[] cacheFiles = JobContextImpl.getCacheFiles(conf);
     if (cacheFiles != null) {
       List<String> localFiles = new ArrayList<String>();
       for (int i = 0; i < cacheFiles.length; ++i) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
index b565d23029b1a..50cc63094bd8c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
@@ -21,7 +21,6 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -31,9 +30,11 @@
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.ArrayList;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -44,23 +45,31 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PositionedReadable;
 import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
+/**
+ * Test the LocalDistributedCacheManager using mocking.
+ * This suite is brittle to changes in the class under test.
+ */
 @SuppressWarnings("deprecation")
 public class TestLocalDistributedCacheManager {
 
+  private static final byte[] TEST_DATA = "This is a test file\n".getBytes();
+
   private static FileSystem mockfs;
 
   public static class MockFileSystem extends FilterFileSystem {
@@ -71,6 +80,14 @@ public MockFileSystem() {
 
   private File localDir;
 
+  /**
+   * Recursive delete of a path.
+   * For safety, paths of length under 5 are rejected.
+   * @param file path to delete.
+   * @throws IOException never, it is just "a dummy in the method signature"
+   * @throws IllegalArgumentException path too short
+   * @throws RuntimeException File.delete() failed.
+   */
   private static void delete(File file) throws IOException {
     if (file.getAbsolutePath().length() < 5) {
       throw new IllegalArgumentException(
@@ -110,9 +127,9 @@ public void cleanup() throws Exception {
    * Mock input stream based on a byte array so that it can be used by a
    * FSDataInputStream.
    */
-  private static class MockInputStream extends ByteArrayInputStream
+  private static final class MockInputStream extends ByteArrayInputStream
       implements Seekable, PositionedReadable {
-    public MockInputStream(byte[] buf) {
+    private MockInputStream(byte[] buf) {
       super(buf);
     }
 
@@ -135,47 +152,45 @@ public void testDownload() throws Exception {
     when(mockfs.getUri()).thenReturn(mockBase);
     Path working = new Path("mock://test-nn1/user/me/");
     when(mockfs.getWorkingDirectory()).thenReturn(working);
-    when(mockfs.resolvePath(any(Path.class))).thenAnswer(new Answer<Path>() {
-      @Override
-      public Path answer(InvocationOnMock args) throws Throwable {
-        return (Path) args.getArguments()[0];
-      }
-    });
+    when(mockfs.resolvePath(any(Path.class))).thenAnswer(
+        (Answer<Path>) args -> (Path) args.getArguments()[0]);
 
     final URI file = new URI("mock://test-nn1/user/me/file.txt#link");
     final Path filePath = new Path(file);
     File link = new File("link");
 
+    // return a filestatus for the file "*/file.txt"; raise FNFE for anything else
     when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
       @Override
       public FileStatus answer(InvocationOnMock args) throws Throwable {
         Path p = (Path)args.getArguments()[0];
         if("file.txt".equals(p.getName())) {
-          return new FileStatus(201, false, 1, 500, 101, 101,
-              FsPermission.getDefault(), "me", "me", filePath);
+          return createMockTestFileStatus(filePath);
         }  else {
-          throw new FileNotFoundException(p+" not supported by mocking");
+          throw notMocked(p);
         }
       }
     });
 
     when(mockfs.getConf()).thenReturn(conf);
     final FSDataInputStream in =
-        new FSDataInputStream(new MockInputStream("This is a test file\n".getBytes()));
-    when(mockfs.open(any(Path.class), anyInt())).thenAnswer(new Answer<FSDataInputStream>() {
-      @Override
-      public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
-        Path src = (Path)args.getArguments()[0];
-        if ("file.txt".equals(src.getName())) {
-          return in;
-        } else {
-          throw new FileNotFoundException(src+" not supported by mocking");
-        }
-      }
-    });
+        new FSDataInputStream(new MockInputStream(TEST_DATA));
+
+    // file.txt: return an openfile builder which will eventually return the data,
+    // anything else: FNFE
+    when(mockfs.openFile(any(Path.class))).thenAnswer(
+        (Answer<FutureDataInputStreamBuilder>) args -> {
+          Path src = (Path)args.getArguments()[0];
+          if ("file.txt".equals(src.getName())) {
+            return new MockOpenFileBuilder(mockfs, src,
+                () -> CompletableFuture.completedFuture(in));
+          } else {
+            throw notMocked(src);
+          }
+        });
 
-    DistributedCache.addCacheFile(file, conf);
-    Map<String, Boolean> policies = new HashMap<String, Boolean>();
+    Job.addCacheFile(file, conf);
+    Map<String, Boolean> policies = new HashMap<>();
     policies.put(file.toString(), true);
     Job.setFileSharedCacheUploadPolicies(conf, policies);
     conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "101");
@@ -192,6 +207,12 @@ public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
     assertFalse(link.exists());
   }
 
+  /**
+   * This test case sets the mock FS to raise FNFE
+   * on any getFileStatus/openFile calls.
+   * If the manager successfully starts up, it means that
+   * no files were probed for/opened.
+   */
   @Test
   public void testEmptyDownload() throws Exception {
     JobID jobId = new JobID();
@@ -202,30 +223,21 @@ public void testEmptyDownload() throws Exception {
     when(mockfs.getUri()).thenReturn(mockBase);
     Path working = new Path("mock://test-nn1/user/me/");
     when(mockfs.getWorkingDirectory()).thenReturn(working);
-    when(mockfs.resolvePath(any(Path.class))).thenAnswer(new Answer<Path>() {
-      @Override
-      public Path answer(InvocationOnMock args) throws Throwable {
-        return (Path) args.getArguments()[0];
-      }
-    });
+    when(mockfs.resolvePath(any(Path.class))).thenAnswer(
+        (Answer<Path>) args -> (Path) args.getArguments()[0]);
 
-    when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
-      @Override
-      public FileStatus answer(InvocationOnMock args) throws Throwable {
-        Path p = (Path)args.getArguments()[0];
-        throw new FileNotFoundException(p+" not supported by mocking");
-      }
-    });
+    when(mockfs.getFileStatus(any(Path.class))).thenAnswer(
+        (Answer<FileStatus>) args -> {
+          Path p = (Path)args.getArguments()[0];
+          throw notMocked(p);
+        });
 
     when(mockfs.getConf()).thenReturn(conf);
-    when(mockfs.open(any(Path.class), anyInt())).thenAnswer(new Answer<FSDataInputStream>() {
-      @Override
-      public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
-        Path src = (Path)args.getArguments()[0];
-        throw new FileNotFoundException(src+" not supported by mocking");
-      }
-    });
-
+    when(mockfs.openFile(any(Path.class))).thenAnswer(
+        (Answer<FutureDataInputStreamBuilder>) args -> {
+          Path src = (Path)args.getArguments()[0];
+            throw notMocked(src);
+        });
     conf.set(MRJobConfig.CACHE_FILES, "");
     conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath());
     LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
@@ -237,6 +249,9 @@ public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
   }
 
 
+  /**
+   * The same file can be added to the cache twice.
+   */
   @Test
   public void testDuplicateDownload() throws Exception {
     JobID jobId = new JobID();
@@ -247,12 +262,8 @@ public void testDuplicateDownload() throws Exception {
     when(mockfs.getUri()).thenReturn(mockBase);
     Path working = new Path("mock://test-nn1/user/me/");
     when(mockfs.getWorkingDirectory()).thenReturn(working);
-    when(mockfs.resolvePath(any(Path.class))).thenAnswer(new Answer<Path>() {
-      @Override
-      public Path answer(InvocationOnMock args) throws Throwable {
-        return (Path) args.getArguments()[0];
-      }
-    });
+    when(mockfs.resolvePath(any(Path.class))).thenAnswer(
+        (Answer<Path>) args -> (Path) args.getArguments()[0]);
 
     final URI file = new URI("mock://test-nn1/user/me/file.txt#link");
     final Path filePath = new Path(file);
@@ -263,32 +274,30 @@ public Path answer(InvocationOnMock args) throws Throwable {
       public FileStatus answer(InvocationOnMock args) throws Throwable {
         Path p = (Path)args.getArguments()[0];
         if("file.txt".equals(p.getName())) {
-          return new FileStatus(201, false, 1, 500, 101, 101,
-              FsPermission.getDefault(), "me", "me", filePath);
+          return createMockTestFileStatus(filePath);
         }  else {
-          throw new FileNotFoundException(p+" not supported by mocking");
+          throw notMocked(p);
         }
       }
     });
 
     when(mockfs.getConf()).thenReturn(conf);
     final FSDataInputStream in =
-        new FSDataInputStream(new MockInputStream("This is a test file\n".getBytes()));
-    when(mockfs.open(any(Path.class), anyInt())).thenAnswer(new Answer<FSDataInputStream>() {
-      @Override
-      public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
-        Path src = (Path)args.getArguments()[0];
-        if ("file.txt".equals(src.getName())) {
-          return in;
-        } else {
-          throw new FileNotFoundException(src+" not supported by mocking");
-        }
-      }
-    });
+        new FSDataInputStream(new MockInputStream(TEST_DATA));
+    when(mockfs.openFile(any(Path.class))).thenAnswer(
+        (Answer<FutureDataInputStreamBuilder>) args -> {
+          Path src = (Path)args.getArguments()[0];
+          if ("file.txt".equals(src.getName())) {
+            return new MockOpenFileBuilder(mockfs, src,
+                () -> CompletableFuture.completedFuture(in));
+          } else {
+            throw notMocked(src);
+          }
+        });
 
-    DistributedCache.addCacheFile(file, conf);
-    DistributedCache.addCacheFile(file, conf);
-    Map<String, Boolean> policies = new HashMap<String, Boolean>();
+    Job.addCacheFile(file, conf);
+    Job.addCacheFile(file, conf);
+    Map<String, Boolean> policies = new HashMap<>();
     policies.put(file.toString(), true);
     Job.setFileSharedCacheUploadPolicies(conf, policies);
     conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "101,101");
@@ -307,7 +316,7 @@ public FSDataInputStream answer(InvocationOnMock args) throws Throwable {
 
   /**
    * This test tries to replicate the issue with the previous version of
-   * {@ref LocalDistributedCacheManager} when the resulting timestamp is
+   * {@link LocalDistributedCacheManager} when the resulting timestamp is
    * identical as that in another process.  Unfortunately, it is difficult
    * to mimic such behavior in a single process unit test.  And mocking
    * the unique id (timestamp previously, UUID otherwise) won't prove the
@@ -322,7 +331,7 @@ public void testMultipleCacheSetup() throws Exception {
     final int threadCount = 10;
     final CyclicBarrier barrier = new CyclicBarrier(threadCount);
 
-    ArrayList<Callable<Void>> setupCallable = new ArrayList<>();
+    List<Callable<Void>> setupCallable = new ArrayList<>();
     for (int i = 0; i < threadCount; ++i) {
       setupCallable.add(() -> {
         barrier.await();
@@ -341,4 +350,58 @@ public void testMultipleCacheSetup() throws Exception {
       manager.close();
     }
   }
+
+  /**
+   * Create test file status using test data as the length.
+   * @param filePath path to the file
+   * @return a file status.
+   */
+  private FileStatus createMockTestFileStatus(final Path filePath) {
+    return new FileStatus(TEST_DATA.length, false, 1, 500, 101, 101,
+        FsPermission.getDefault(), "me", "me", filePath);
+  }
+
+  /**
+   * Exception to throw on a not mocked path.
+   * @return a FileNotFoundException
+   */
+  private FileNotFoundException notMocked(final Path p) {
+    return new FileNotFoundException(p + " not supported by mocking");
+  }
+
+  /**
+   * Openfile builder where the build operation is a l-expression
+   * supplied in the constructor.
+   */
+  private static final class MockOpenFileBuilder extends
+      FutureDataInputStreamBuilderImpl {
+
+    /**
+     * Operation to invoke to build the result.
+     */
+    private final CallableRaisingIOE<CompletableFuture<FSDataInputStream>>
+        buildTheResult;
+
+    /**
+     * Create the builder. the FS and path must be non-null.
+     * FileSystem.getConf() is the only method invoked of the FS by
+     * the superclass.
+     * @param fileSystem fs
+     * @param path path to open
+     * @param buildTheResult builder operation.
+     */
+    private MockOpenFileBuilder(final FileSystem fileSystem, Path path,
+        final CallableRaisingIOE<CompletableFuture<FSDataInputStream>> buildTheResult) {
+      super(fileSystem, path);
+      this.buildTheResult = buildTheResult;
+    }
+
+    @Override
+    public CompletableFuture<FSDataInputStream> build()
+        throws IllegalArgumentException, UnsupportedOperationException,
+               IOException {
+      return buildTheResult.apply();
+    }
+  }
+
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalModeWithNewApis.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalModeWithNewApis.java
index d8ca74aec2f0d..556f7fd716aa7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalModeWithNewApis.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalModeWithNewApis.java
@@ -21,7 +21,6 @@
 
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Random;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
index 2f0a6bdc00974..62b8815e6f1fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
@@ -229,60 +229,60 @@ public void testDeprecatedFunctions() throws Exception {
     Assert.assertEquals("Test Local Archives 1",
         conf.get(DistributedCache.CACHE_LOCALARCHIVES));
     Assert.assertEquals(1,
-        DistributedCache.getLocalCacheArchives(conf).length);
+        JobContextImpl.getLocalCacheArchives(conf).length);
     Assert.assertEquals("Test Local Archives 1",
-        DistributedCache.getLocalCacheArchives(conf)[0].getName());
+        JobContextImpl.getLocalCacheArchives(conf)[0].getName());
     DistributedCache.addLocalArchives(conf, "Test Local Archives 2");
     Assert.assertEquals("Test Local Archives 1,Test Local Archives 2",
         conf.get(DistributedCache.CACHE_LOCALARCHIVES));
     Assert.assertEquals(2,
-        DistributedCache.getLocalCacheArchives(conf).length);
+        JobContextImpl.getLocalCacheArchives(conf).length);
     Assert.assertEquals("Test Local Archives 2",
-        DistributedCache.getLocalCacheArchives(conf)[1].getName());
+        JobContextImpl.getLocalCacheArchives(conf)[1].getName());
     DistributedCache.setLocalArchives(conf, "Test Local Archives 3");
     Assert.assertEquals("Test Local Archives 3",
         conf.get(DistributedCache.CACHE_LOCALARCHIVES));
     Assert.assertEquals(1,
-        DistributedCache.getLocalCacheArchives(conf).length);
+        JobContextImpl.getLocalCacheArchives(conf).length);
     Assert.assertEquals("Test Local Archives 3",
-        DistributedCache.getLocalCacheArchives(conf)[0].getName());
+        JobContextImpl.getLocalCacheArchives(conf)[0].getName());
 
     DistributedCache.addLocalFiles(conf, "Test Local Files 1");
     Assert.assertEquals("Test Local Files 1",
         conf.get(DistributedCache.CACHE_LOCALFILES));
     Assert.assertEquals(1,
-        DistributedCache.getLocalCacheFiles(conf).length);
+        JobContextImpl.getLocalCacheFiles(conf).length);
     Assert.assertEquals("Test Local Files 1",
-        DistributedCache.getLocalCacheFiles(conf)[0].getName());
+        JobContextImpl.getLocalCacheFiles(conf)[0].getName());
     DistributedCache.addLocalFiles(conf, "Test Local Files 2");
     Assert.assertEquals("Test Local Files 1,Test Local Files 2",
         conf.get(DistributedCache.CACHE_LOCALFILES));
     Assert.assertEquals(2,
-        DistributedCache.getLocalCacheFiles(conf).length);
+        JobContextImpl.getLocalCacheFiles(conf).length);
     Assert.assertEquals("Test Local Files 2",
-        DistributedCache.getLocalCacheFiles(conf)[1].getName());
+        JobContextImpl.getLocalCacheFiles(conf)[1].getName());
     DistributedCache.setLocalFiles(conf, "Test Local Files 3");
     Assert.assertEquals("Test Local Files 3",
         conf.get(DistributedCache.CACHE_LOCALFILES));
     Assert.assertEquals(1,
-        DistributedCache.getLocalCacheFiles(conf).length);
+        JobContextImpl.getLocalCacheFiles(conf).length);
     Assert.assertEquals("Test Local Files 3",
-        DistributedCache.getLocalCacheFiles(conf)[0].getName());
+        JobContextImpl.getLocalCacheFiles(conf)[0].getName());
 
     DistributedCache.setArchiveTimestamps(conf, "1234567890");
     Assert.assertEquals(1234567890,
         conf.getLong(DistributedCache.CACHE_ARCHIVES_TIMESTAMPS, 0));
     Assert.assertEquals(1,
-        DistributedCache.getArchiveTimestamps(conf).length);
+        JobContextImpl.getArchiveTimestamps(conf).length);
     Assert.assertEquals(1234567890,
-        DistributedCache.getArchiveTimestamps(conf)[0]);
+        JobContextImpl.getArchiveTimestamps(conf)[0]);
     DistributedCache.setFileTimestamps(conf, "1234567890");
     Assert.assertEquals(1234567890,
         conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0));
     Assert.assertEquals(1,
-        DistributedCache.getFileTimestamps(conf).length);
+        JobContextImpl.getFileTimestamps(conf).length);
     Assert.assertEquals(1234567890,
-        DistributedCache.getFileTimestamps(conf)[0]);
+        JobContextImpl.getFileTimestamps(conf)[0]);
 
     DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"),
         new File("Test Work Dir"));
@@ -297,18 +297,18 @@ public void testDeprecatedFunctions() throws Exception {
         DistributedCache.getTimestamp(conf, symlinkFile.toURI()));
     Assert.assertTrue(symlinkFile.delete());
 
-    DistributedCache.addCacheArchive(symlinkFile.toURI(), conf);
+    Job.addCacheArchive(symlinkFile.toURI(), conf);
     Assert.assertEquals(symlinkFile.toURI().toString(),
         conf.get(DistributedCache.CACHE_ARCHIVES));
-    Assert.assertEquals(1, DistributedCache.getCacheArchives(conf).length);
+    Assert.assertEquals(1, JobContextImpl.getCacheArchives(conf).length);
     Assert.assertEquals(symlinkFile.toURI(),
-        DistributedCache.getCacheArchives(conf)[0]);
+        JobContextImpl.getCacheArchives(conf)[0]);
 
-    DistributedCache.addCacheFile(symlinkFile.toURI(), conf);
+    Job.addCacheFile(symlinkFile.toURI(), conf);
     Assert.assertEquals(symlinkFile.toURI().toString(),
         conf.get(DistributedCache.CACHE_FILES));
-    Assert.assertEquals(1, DistributedCache.getCacheFiles(conf).length);
+    Assert.assertEquals(1, JobContextImpl.getCacheFiles(conf).length);
     Assert.assertEquals(symlinkFile.toURI(),
-        DistributedCache.getCacheFiles(conf)[0]);
+        JobContextImpl.getCacheFiles(conf)[0]);
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index c6a287439da8b..b4952ecc4ddcd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -40,12 +40,10 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.InvalidJobConfException;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
 import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -377,11 +375,11 @@ public void testSetupDistributedCacheConflicts() throws Exception {
     when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
     when(mockFs.resolvePath(filePath)).thenReturn(filePath);
     
-    DistributedCache.addCacheArchive(archive, conf);
+    Job.addCacheArchive(archive, conf);
     conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
     conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
     conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
-    DistributedCache.addCacheFile(file, conf);
+    Job.addCacheFile(file, conf);
     conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
     conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
     conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
@@ -416,8 +414,8 @@ public void testSetupDistributedCacheConflictsFiles() throws Exception {
     when(mockFs.resolvePath(filePath)).thenReturn(filePath);
     when(mockFs.resolvePath(file2Path)).thenReturn(file2Path);
     
-    DistributedCache.addCacheFile(file, conf);
-    DistributedCache.addCacheFile(file2, conf);
+    Job.addCacheFile(file, conf);
+    Job.addCacheFile(file2, conf);
     conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "10,11");
     conf.set(MRJobConfig.CACHE_FILES_SIZES, "10,11");
     conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true,true");
@@ -452,11 +450,11 @@ public void testSetupDistributedCache() throws Exception {
     when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
     when(mockFs.resolvePath(filePath)).thenReturn(filePath);
     
-    DistributedCache.addCacheArchive(archive, conf);
+    Job.addCacheArchive(archive, conf);
     conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
     conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
     conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
-    DistributedCache.addCacheFile(file, conf);
+    Job.addCacheFile(file, conf);
     conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
     conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
     conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 946dbe5e8bc72..7530428d7525d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -45,6 +45,12 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs-client</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.json</groupId>
+          <artifactId>json</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -97,9 +103,14 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-	<configuration>
+        <configuration>
           <systemPropertyVariables>
             <log4j.configuration>file:///${project.basedir}/src/test/resources/log4j.properties</log4j.configuration>
+
+            <!-- project directory   (target/) on a maven build. -->
+            <project.build.directory>${project.build.directory}</project.build.directory>
+            <!-- this property is used in hadoop-azure with separate paths for parallel runs -->
+            <test.build.dir>${project.build.directory}</test.build.dir>
           </systemPropertyVariables>
         </configuration>
       </plugin>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
index 0e24bbe53307d..70328ea00755d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
@@ -23,7 +23,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,7 +43,7 @@ class IndexCache {
   public IndexCache(JobConf conf) {
     this.conf = conf;
     totalMemoryAllowed =
-      conf.getInt(TTConfig.TT_INDEX_CACHE, 10) * 1024 * 1024;
+      conf.getInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 10) * 1024 * 1024;
     LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java
index 1761500ff4a8a..136ac650bff1b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.JobACL;
 import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.slf4j.Logger;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
index 2f8dd63ccede3..f94752fc238e3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
@@ -48,7 +48,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java
index 1ce525490d14b..b2fa79808ca6d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java
@@ -26,10 +26,10 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.params.ClientPNames;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -67,10 +67,13 @@ private static JobEndStatusInfo createNotification(JobConf conf,
 
   private static int httpNotification(String uri, int timeout)
       throws IOException, URISyntaxException {
-    DefaultHttpClient client = new DefaultHttpClient();
-    client.getParams()
-        .setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout)
-        .setLongParameter(ClientPNames.CONN_MANAGER_TIMEOUT, (long) timeout);
+    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+    httpClientBuilder.setDefaultRequestConfig(
+        RequestConfig.custom()
+        .setConnectionRequestTimeout(timeout)
+        .setSocketTimeout(timeout)
+        .build());
+    HttpClient client = httpClientBuilder.build();
     HttpGet httpGet = new HttpGet(new URI(uri));
     httpGet.setHeader("Accept", "*/*");
     return client.execute(httpGet).getStatusLine().getStatusCode();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
index 1fcb118a100fc..5724e72931085 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.CodecPool;
@@ -41,9 +40,13 @@
 import org.apache.hadoop.mapreduce.lib.input.CompressedSplitLineReader;
 import org.apache.hadoop.mapreduce.lib.input.SplitLineReader;
 import org.apache.hadoop.mapreduce.lib.input.UncompressedSplitLineReader;
+import org.apache.hadoop.util.functional.FutureIO;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_END;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_START;
+
 /**
  * Treats keys as offset in file and value as line. 
  */
@@ -109,10 +112,14 @@ public LineRecordReader(Configuration job, FileSplit split,
     // open the file and seek to the start of the split
     final FutureDataInputStreamBuilder builder =
         file.getFileSystem(job).openFile(file);
-    FutureIOSupport.propagateOptions(builder, job,
+    // the start and end of the split may be used to build
+    // an input strategy.
+    builder.opt(FS_OPTION_OPENFILE_SPLIT_START, start)
+        .opt(FS_OPTION_OPENFILE_SPLIT_END, end);
+    FutureIO.propagateOptions(builder, job,
         MRJobConfig.INPUT_FILE_OPTION_PREFIX,
         MRJobConfig.INPUT_FILE_MANDATORY_PREFIX);
-    fileIn = FutureIOSupport.awaitFuture(builder.build());
+    fileIn = FutureIO.awaitFuture(builder.build());
     if (isCompressedInput()) {
       decompressor = CodecPool.getDecompressor(codec);
       if (codec instanceof SplittableCompressionCodec) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index fa4396d77f458..06d9fbbe7a323 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -150,8 +150,8 @@ public void readFields(DataInput in) throws IOException {
    * @param <V>
    */
   class TrackedRecordReader<K, V> 
-      implements RecordReader<K,V> {
-    private RecordReader<K,V> rawIn;
+      implements RecordReader<K, V> {
+    private RecordReader<K, V> rawIn;
     private Counters.Counter fileInputByteCounter;
     private Counters.Counter inputRecordCounter;
     private TaskReporter reporter;
@@ -240,7 +240,7 @@ private long getInputBytes(List<Statistics> stats) {
    * This class skips the records based on the failed ranges from previous 
    * attempts.
    */
-  class SkippingRecordReader<K, V> extends TrackedRecordReader<K,V> {
+  class SkippingRecordReader<K, V> extends TrackedRecordReader<K, V> {
     private SkipRangeIterator skipIt;
     private SequenceFile.Writer skipWriter;
     private boolean toWriteSkipRecs;
@@ -930,7 +930,7 @@ public static class MapOutputBuffer<K extends Object, V extends Object>
     // spill accounting
     private int maxRec;
     private int softLimit;
-    boolean spillInProgress;;
+    boolean spillInProgress;
     int bufferRemaining;
     volatile Throwable sortSpillException = null;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
index cbc8e526f0d99..f6d9ce5916035 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueConfigurationParser.java
@@ -21,7 +21,10 @@
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.QueueState;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.XMLUtils;
+
 import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
+
 import org.xml.sax.SAXException;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -88,7 +91,7 @@ class QueueConfigurationParser {
   static final String VALUE_TAG = "value";
 
   /**
-   * Default constructor for DeperacatedQueueConfigurationParser
+   * Default constructor for QueueConfigurationParser.
    */
   QueueConfigurationParser() {
     
@@ -158,8 +161,9 @@ void setRoot(Queue root) {
    */
   protected Queue loadResource(InputStream resourceInput)
     throws ParserConfigurationException, SAXException, IOException {
-    DocumentBuilderFactory docBuilderFactory
-      = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory docBuilderFactory =
+        XMLUtils.newSecureDocumentBuilderFactory();
+
     //ignore all comments inside the xml file
     docBuilderFactory.setIgnoringComments(true);
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
index 041ab3908046a..fe8767529e47d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskUmbilicalProtocol.java
@@ -24,8 +24,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.mapred.JvmTask;
-import org.apache.hadoop.mapreduce.checkpoint.CheckpointID;
-import org.apache.hadoop.mapreduce.checkpoint.FSCheckpointID;
 import org.apache.hadoop.mapreduce.checkpoint.TaskCheckpointID;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSelector;
 import org.apache.hadoop.security.token.TokenInfo;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputs.java
index 3ef6601fbfe63..a214420df8059 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/MultipleOutputs.java
@@ -17,15 +17,39 @@
  */
 package org.apache.hadoop.mapred.lib;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.util.Progressable;
 
-import java.io.IOException;
-import java.util.*;
-
 /**
  * The MultipleOutputs class simplifies writing to additional outputs other
  * than the job default output via the <code>OutputCollector</code> passed to
@@ -132,6 +156,7 @@ public class MultipleOutputs {
    * Counters group used by the counters of MultipleOutputs.
    */
   private static final String COUNTERS_GROUP = MultipleOutputs.class.getName();
+  private static final Logger LOG = LoggerFactory.getLogger(MultipleOutputs.class);
 
   /**
    * Checks if a named output is alreadyDefined or not.
@@ -381,6 +406,11 @@ public static boolean getCountersEnabled(JobConf conf) {
   private Map<String, RecordWriter> recordWriters;
   private boolean countersEnabled;
 
+  @VisibleForTesting
+  synchronized void setRecordWriters(Map<String, RecordWriter> recordWriters) {
+    this.recordWriters = recordWriters;
+  }
+
   /**
    * Creates and initializes multiple named outputs support, it should be
    * instantiated in the Mapper/Reducer configure method.
@@ -528,8 +558,41 @@ public void collect(Object key, Object value) throws IOException {
    *                             could not be closed properly.
    */
   public void close() throws IOException {
+    int nThreads = conf.getInt(MRConfig.MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT,
+        MRConfig.DEFAULT_MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT);
+    AtomicBoolean encounteredException = new AtomicBoolean(false);
+    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("MultipleOutputs-close")
+        .setUncaughtExceptionHandler(((t, e) -> {
+          LOG.error("Thread " + t + " failed unexpectedly", e);
+          encounteredException.set(true);
+        })).build();
+    ExecutorService executorService = Executors.newFixedThreadPool(nThreads, threadFactory);
+
+    List<Callable<Object>> callableList = new ArrayList<>(recordWriters.size());
+
     for (RecordWriter writer : recordWriters.values()) {
-      writer.close(null);
+      callableList.add(() -> {
+        try {
+          writer.close(null);
+        } catch (IOException e) {
+          LOG.error("Error while closing MultipleOutput file", e);
+          encounteredException.set(true);
+        }
+        return null;
+      });
+    }
+    try {
+      executorService.invokeAll(callableList);
+    } catch (InterruptedException e) {
+      LOG.warn("Closing is Interrupted");
+      Thread.currentThread().interrupt();
+    } finally {
+      executorService.shutdown();
+    }
+
+    if (encounteredException.get()) {
+      throw new IOException(
+          "One or more threads encountered exception during close. See prior errors.");
     }
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
index c8d33841f56ba..767dcd45143ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Application.java
@@ -48,11 +48,11 @@
 import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapred.TaskLog;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -117,7 +117,7 @@ class Application<K1 extends WritableComparable, V1 extends Writable,
     if (interpretor != null) {
       cmd.add(interpretor);
     }
-    String executable = DistributedCache.getLocalCacheFiles(conf)[0].toString();
+    String executable = JobContextImpl.getLocalCacheFiles(conf)[0].toString();
     if (!FileUtil.canExecute(new File(executable))) {
       // LinuxTaskController sets +x permissions on all distcache files already.
       // In case of DefaultTaskController, set permissions here.
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
index 64448c1c46049..a9f42dd1667f8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java
@@ -54,8 +54,9 @@
 import org.apache.hadoop.mapred.lib.HashPartitioner;
 import org.apache.hadoop.mapred.lib.LazyOutputFormat;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Tool;
@@ -319,7 +320,7 @@ private static void setupPipesJob(JobConf conf) throws IOException {
       setIfUnset(conf, MRJobConfig.MAP_DEBUG_SCRIPT,defScript);
       setIfUnset(conf, MRJobConfig.REDUCE_DEBUG_SCRIPT,defScript);
     }
-    URI[] fileCache = DistributedCache.getCacheFiles(conf);
+    URI[] fileCache = JobContextImpl.getCacheFiles(conf);
     if (fileCache == null) {
       fileCache = new URI[1];
     } else {
@@ -334,7 +335,7 @@ private static void setupPipesJob(JobConf conf) throws IOException {
       ie.initCause(e);
       throw ie;
     }
-    DistributedCache.setCacheFiles(fileCache, conf);
+    Job.setCacheFiles(fileCache, conf);
   }
 
   /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 72bc6225e8fea..c73c5bdd839f2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -22,7 +22,6 @@
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -1121,7 +1120,18 @@ public void setJobSetupCleanupNeeded(boolean needed) {
    */
   public void setCacheArchives(URI[] archives) {
     ensureState(JobState.DEFINE);
-    DistributedCache.setCacheArchives(archives, conf);
+    setCacheArchives(archives, conf);
+  }
+
+  /**
+   * Set the configuration with the given set of archives.
+   *
+   * @param archives The list of archives that need to be localized.
+   * @param conf Configuration which will be changed.
+   */
+  public static void setCacheArchives(URI[] archives, Configuration conf) {
+    String cacheArchives = StringUtils.uriToString(archives);
+    conf.set(MRJobConfig.CACHE_ARCHIVES, cacheArchives);
   }
 
   /**
@@ -1130,7 +1140,18 @@ public void setCacheArchives(URI[] archives) {
    */
   public void setCacheFiles(URI[] files) {
     ensureState(JobState.DEFINE);
-    DistributedCache.setCacheFiles(files, conf);
+    setCacheFiles(files, conf);
+  }
+
+  /**
+   * Set the configuration with the given set of files.
+   *
+   * @param files The list of files that need to be localized.
+   * @param conf Configuration which will be changed.
+   */
+  public static void setCacheFiles(URI[] files, Configuration conf) {
+    String cacheFiles = StringUtils.uriToString(files);
+    conf.set(MRJobConfig.CACHE_FILES, cacheFiles);
   }
 
   /**
@@ -1139,16 +1160,53 @@ public void setCacheFiles(URI[] files) {
    */
   public void addCacheArchive(URI uri) {
     ensureState(JobState.DEFINE);
-    DistributedCache.addCacheArchive(uri, conf);
+    addCacheArchive(uri, conf);
   }
-  
+
+  /**
+   * Add an archives to be localized to the conf.
+   *
+   * @param uri  The uri of the cache to be localized.
+   * @param conf Configuration to add the cache to.
+   */
+  public static void addCacheArchive(URI uri, Configuration conf) {
+    String archives = conf.get(MRJobConfig.CACHE_ARCHIVES);
+    conf.set(MRJobConfig.CACHE_ARCHIVES,
+        archives == null ? uri.toString() : archives + "," + uri.toString());
+  }
+
   /**
    * Add a file to be localized
    * @param uri The uri of the cache to be localized
    */
   public void addCacheFile(URI uri) {
     ensureState(JobState.DEFINE);
-    DistributedCache.addCacheFile(uri, conf);
+    addCacheFile(uri, conf);
+  }
+
+  /**
+   * Add a file to be localized to the conf. The localized file will be
+   * downloaded to the execution node(s), and a link will be created to the
+   * file from the job's working directory. If the last part of URI's path name
+   * is "*", then the entire parent directory will be localized and links
+   * will be created from the job's working directory to each file in the
+   * parent directory.
+   * <p>
+   * The access permissions of the file will determine whether the localized
+   * file will be shared across jobs. If the file is not readable by other or
+   * if any of its parent directories is not executable by other, then the
+   * file will not be shared. In the case of a path that ends in "/*",
+   * sharing of the localized files will be determined solely from the
+   * access permissions of the parent directories. The access permissions of
+   * the individual files will be ignored.
+   *
+   * @param uri  The uri of the cache to be localized.
+   * @param conf Configuration to add the cache to.
+   */
+  public static void addCacheFile(URI uri, Configuration conf) {
+    String files = conf.get(MRJobConfig.CACHE_FILES);
+    conf.set(MRJobConfig.CACHE_FILES,
+        files == null ? uri.toString() : files + "," + uri.toString());
   }
 
   /**
@@ -1165,7 +1223,39 @@ public void addCacheFile(URI uri) {
   public void addFileToClassPath(Path file)
     throws IOException {
     ensureState(JobState.DEFINE);
-    DistributedCache.addFileToClassPath(file, conf, file.getFileSystem(conf));
+    addFileToClassPath(file, conf, file.getFileSystem(conf));
+  }
+
+  /**
+   * Add a file path to the current set of classpath entries. The file will
+   * also be added to the cache.
+   *
+   * @param file Path of the file to be added.
+   * @param conf Configuration that contains the classpath setting.
+   * @param fs FileSystem with respect to which {@code file} should be interpreted.
+   */
+  public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) {
+    addFileToClassPath(file, conf, fs, true);
+  }
+
+  /**
+   * Add a file path to the current set of classpath entries. The file will
+   * also be added to the cache if {@code addToCache} is true.
+   *
+   * @param file Path of the file to be added.
+   * @param conf Configuration that contains the classpath setting.
+   * @param fs FileSystem with respect to which {@code file} should be interpreted.
+   * @param addToCache Whether the file should also be added to the cache list.
+   */
+  public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs,
+      boolean addToCache) {
+    String classpath = conf.get(MRJobConfig.CLASSPATH_FILES);
+    conf.set(MRJobConfig.CLASSPATH_FILES,
+        classpath == null ? file.toString() : classpath + "," + file.toString());
+    if (addToCache) {
+      URI uri = fs.makeQualified(file).toUri();
+      Job.addCacheFile(uri, conf);
+    }
   }
 
   /**
@@ -1180,7 +1270,23 @@ public void addFileToClassPath(Path file)
   public void addArchiveToClassPath(Path archive)
     throws IOException {
     ensureState(JobState.DEFINE);
-    DistributedCache.addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
+    addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
+  }
+
+  /**
+   * Add an archive path to the current set of classpath entries. It adds the
+   * archive to cache as well.
+   *
+   * @param archive Path of the archive to be added.
+   * @param conf Configuration that contains the classpath setting.
+   * @param fs FileSystem with respect to which {@code archive} should be interpreted.
+   */
+  public static void addArchiveToClassPath(Path archive, Configuration conf, FileSystem fs) {
+    String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES);
+    conf.set(MRJobConfig.CLASSPATH_ARCHIVES,
+        classpath == null ? archive.toString() : classpath + "," + archive.toString());
+    URI uri = fs.makeQualified(archive).toUri();
+    Job.addCacheArchive(uri, conf);
   }
 
   /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 2825497a42472..6a3cb80a5ed98 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -332,13 +332,12 @@ void uploadLibJars(Job job, Collection<String> libjars, Path submitJobDir,
           // separately.
           foundFragment = (newURI.getFragment() != null) && !fromSharedCache;
         }
-        DistributedCache.addFileToClassPath(new Path(newURI.getPath()), conf,
-            jtFs, false);
+        Job.addFileToClassPath(new Path(newURI.getPath()), conf, jtFs, false);
         if (fromSharedCache) {
           // We simply add this URI to the distributed cache. It will not come
           // from the staging directory (it is in the shared cache), so we
           // must add it to the cache regardless of the wildcard feature.
-          DistributedCache.addCacheFile(newURI, conf);
+          Job.addCacheFile(newURI, conf);
         } else {
           libjarURIs.add(newURI);
         }
@@ -352,10 +351,10 @@ void uploadLibJars(Job job, Collection<String> libjars, Path submitJobDir,
         // Add the whole directory to the cache using a wild card
         Path libJarsDirWildcard =
             jtFs.makeQualified(new Path(libjarsDir, DistributedCache.WILDCARD));
-        DistributedCache.addCacheFile(libJarsDirWildcard.toUri(), conf);
+        Job.addCacheFile(libJarsDirWildcard.toUri(), conf);
       } else {
         for (URI uri : libjarURIs) {
-          DistributedCache.addCacheFile(uri, conf);
+          Job.addCacheFile(uri, conf);
         }
       }
     }
@@ -784,9 +783,11 @@ private URI useSharedCache(URI sourceFile, String resourceName,
   void copyJar(Path originalJarPath, Path submitJarFile,
       short replication) throws IOException {
     jtFs.copyFromLocalFile(originalJarPath, submitJarFile);
-    jtFs.setReplication(submitJarFile, replication);
+    // The operation of setReplication requires certain permissions
+    // so we need to make sure it has enough permissions
     jtFs.setPermission(submitJarFile, new FsPermission(
         JobSubmissionFiles.JOB_FILE_PERMISSION));
+    jtFs.setReplication(submitJarFile, replication);
   }
 
   private void addLog4jToDistributedCache(Job job, Path jobSubmitDir)
@@ -847,8 +848,8 @@ private void copyLog4jPropertyFile(Job job, Path submitJobDir,
       }
       Path tmp = new Path(tmpURI);
       Path newPath = copyRemoteFiles(fileDir, tmp, conf, replication);
-      DistributedCache.addFileToClassPath(new Path(newPath.toUri().getPath()),
-          conf);
+      Path path = new Path(newPath.toUri().getPath());
+      Job.addFileToClassPath(path, conf, path.getFileSystem(conf));
     }
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index dd6c92204640c..232b8bf13bc97 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -71,7 +71,6 @@ class JobSubmitter {
   protected static final Logger LOG =
       LoggerFactory.getLogger(JobSubmitter.class);
   private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
-  private static final int SHUFFLE_KEY_LENGTH = 64;
   private FileSystem jtFs;
   private ClientProtocol submitClient;
   private String submitHostName;
@@ -177,7 +176,9 @@ JobStatus submitJobInternal(Job job, Cluster cluster)
         KeyGenerator keyGen;
         try {
           keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM);
-          keyGen.init(SHUFFLE_KEY_LENGTH);
+          int shuffleKeyLength =
+              conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH);
+          keyGen.init(shuffleKeyLength);
         } catch (NoSuchAlgorithmException e) {
           throw new IOException("Error generating shuffle secret key", e);
         }
@@ -466,7 +467,7 @@ private static void addMRFrameworkToDistributedCache(Configuration conf)
         throw new IllegalArgumentException(e);
       }
 
-      DistributedCache.addCacheArchive(uri, conf);
+      Job.addCacheArchive(uri, conf);
     }
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
index b4d91491e1c5f..8671eb30b993a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
@@ -131,5 +131,7 @@ public interface MRConfig {
   String MASTER_WEBAPP_UI_ACTIONS_ENABLED =
       "mapreduce.webapp.ui-actions.enabled";
   boolean DEFAULT_MASTER_WEBAPP_UI_ACTIONS_ENABLED = true;
+  String MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT = "mapreduce.multiple-outputs-close-threads";
+  int DEFAULT_MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT = 10;
 }
   
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index a90c58dd28b4c..02fa4eca6366f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -577,6 +577,8 @@ public interface MRJobConfig {
   public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures";
   public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5;
 
+  public static final String SHUFFLE_INDEX_CACHE = "mapreduce.reduce.shuffle.indexcache.mb";
+
   public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr";
 
   public static final String REDUCE_SKIP_MAXGROUPS = "mapreduce.reduce.skip.maxgroups";
@@ -919,6 +921,13 @@ public interface MRJobConfig {
     MR_AM_PREFIX + "scheduler.heartbeat.interval-ms";
   public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000;
 
+  /** Whether to consider ping from tasks in liveliness check. */
+  String MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK =
+      "mapreduce.task.ping-for-liveliness-check.enabled";
+  boolean DEFAULT_MR_TASK_ENABLE_PING_FOR_LIVELINESS_CHECK
+      = false;
+
+
   /**
    * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS
    * milliseconds before aborting. During this interval, AM will still try
@@ -1295,4 +1304,6 @@ public interface MRJobConfig {
    */
   @Unstable
   String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must.";
+  String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length";
+  int DEFAULT_SHUFFLE_KEY_LENGTH = 64;
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Mapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Mapper.java
index 6b4147b900bff..8ec6e8e20d4f4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Mapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Mapper.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.mapreduce.task.MapContextImpl;
 
 /** 
  * Maps input key/value pairs to a set of intermediate key/value pairs.  
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
index 97d19a668dec0..3834d4ea88989 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
@@ -28,7 +28,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.util.Preconditions.*;
 import org.apache.hadoop.thirdparty.com.google.common.collect.AbstractIterator;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java
index ca7f1f06514d8..7ef3ec03f383f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.mapreduce.counters;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 import java.io.DataInput;
 import java.io.DataOutput;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
index ada14db944c92..5e20bbe40dcb4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.security.Credentials;
 
 /**
@@ -82,7 +83,7 @@ public static void determineTimestampsAndCacheVisibilities(Configuration job,
    */
   public static void determineTimestamps(Configuration job,
       Map<URI, FileStatus> statCache) throws IOException {
-    URI[] tarchives = DistributedCache.getCacheArchives(job);
+    URI[] tarchives = JobContextImpl.getCacheArchives(job);
     if (tarchives != null) {
       FileStatus status = getFileStatus(job, tarchives[0], statCache);
       StringBuilder archiveFileSizes =
@@ -100,7 +101,7 @@ public static void determineTimestamps(Configuration job,
       setArchiveTimestamps(job, archiveTimestamps.toString());
     }
   
-    URI[] tfiles = DistributedCache.getCacheFiles(job);
+    URI[] tfiles = JobContextImpl.getCacheFiles(job);
     if (tfiles != null) {
       FileStatus status = getFileStatus(job, tfiles[0], statCache);
       StringBuilder fileSizes =
@@ -127,8 +128,8 @@ public static void determineTimestamps(Configuration job,
    */
   public static void getDelegationTokens(Configuration job,
       Credentials credentials) throws IOException {
-    URI[] tarchives = DistributedCache.getCacheArchives(job);
-    URI[] tfiles = DistributedCache.getCacheFiles(job);
+    URI[] tarchives = JobContextImpl.getCacheArchives(job);
+    URI[] tfiles = JobContextImpl.getCacheFiles(job);
     
     int size = (tarchives!=null? tarchives.length : 0) + (tfiles!=null ? tfiles.length :0);
     Path[] ps = new Path[size];
@@ -159,7 +160,7 @@ public static void getDelegationTokens(Configuration job,
    */
   public static void determineCacheVisibilities(Configuration job,
       Map<URI, FileStatus> statCache) throws IOException {
-    URI[] tarchives = DistributedCache.getCacheArchives(job);
+    URI[] tarchives = JobContextImpl.getCacheArchives(job);
     if (tarchives != null) {
       StringBuilder archiveVisibilities =
         new StringBuilder(String.valueOf(isPublic(job, tarchives[0], statCache)));
@@ -169,7 +170,7 @@ public static void determineCacheVisibilities(Configuration job,
       }
       setArchiveVisibilities(job, archiveVisibilities.toString());
     }
-    URI[] tfiles = DistributedCache.getCacheFiles(job);
+    URI[] tfiles = JobContextImpl.getCacheFiles(job);
     if (tfiles != null) {
       StringBuilder fileVisibilities =
         new StringBuilder(String.valueOf(isPublic(job, tfiles[0], statCache)));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
index 532e3ad3d7c0a..dd11d490ab433 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
@@ -23,6 +23,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.*;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.util.*;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.fs.FileSystem;
@@ -145,8 +146,7 @@ public class DistributedCache {
    */
   @Deprecated
   public static void setCacheArchives(URI[] archives, Configuration conf) {
-    String sarchives = StringUtils.uriToString(archives);
-    conf.set(MRJobConfig.CACHE_ARCHIVES, sarchives);
+    Job.setCacheArchives(archives, conf);
   }
 
   /**
@@ -159,8 +159,7 @@ public static void setCacheArchives(URI[] archives, Configuration conf) {
    */
   @Deprecated
   public static void setCacheFiles(URI[] files, Configuration conf) {
-    String sfiles = StringUtils.uriToString(files);
-    conf.set(MRJobConfig.CACHE_FILES, sfiles);
+    Job.setCacheFiles(files, conf);
   }
 
   /**
@@ -174,7 +173,7 @@ public static void setCacheFiles(URI[] files, Configuration conf) {
    */
   @Deprecated
   public static URI[] getCacheArchives(Configuration conf) throws IOException {
-    return StringUtils.stringToURI(conf.getStrings(MRJobConfig.CACHE_ARCHIVES));
+    return JobContextImpl.getCacheArchives(conf);
   }
 
   /**
@@ -188,7 +187,7 @@ public static URI[] getCacheArchives(Configuration conf) throws IOException {
    */
   @Deprecated
   public static URI[] getCacheFiles(Configuration conf) throws IOException {
-    return StringUtils.stringToURI(conf.getStrings(MRJobConfig.CACHE_FILES));
+    return JobContextImpl.getCacheFiles(conf);
   }
 
   /**
@@ -201,10 +200,8 @@ public static URI[] getCacheFiles(Configuration conf) throws IOException {
    * @see JobContext#getLocalCacheArchives()
    */
   @Deprecated
-  public static Path[] getLocalCacheArchives(Configuration conf)
-    throws IOException {
-    return StringUtils.stringToPath(conf
-                                    .getStrings(MRJobConfig.CACHE_LOCALARCHIVES));
+  public static Path[] getLocalCacheArchives(Configuration conf) throws IOException {
+    return JobContextImpl.getLocalCacheArchives(conf);
   }
 
   /**
@@ -219,23 +216,7 @@ public static Path[] getLocalCacheArchives(Configuration conf)
   @Deprecated
   public static Path[] getLocalCacheFiles(Configuration conf)
     throws IOException {
-    return StringUtils.stringToPath(conf.getStrings(MRJobConfig.CACHE_LOCALFILES));
-  }
-
-  /**
-   * Parse a list of strings into longs.
-   * @param strs the list of strings to parse
-   * @return a list of longs that were parsed. same length as strs.
-   */
-  private static long[] parseTimestamps(String[] strs) {
-    if (strs == null) {
-      return null;
-    }
-    long[] result = new long[strs.length];
-    for(int i=0; i < strs.length; ++i) {
-      result[i] = Long.parseLong(strs[i]);
-    }
-    return result;
+    return JobContextImpl.getLocalCacheFiles(conf);
   }
 
   /**
@@ -248,8 +229,7 @@ private static long[] parseTimestamps(String[] strs) {
    */
   @Deprecated
   public static long[] getArchiveTimestamps(Configuration conf) {
-    return parseTimestamps(
-        conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS));
+    return JobContextImpl.getArchiveTimestamps(conf);
   }
 
 
@@ -263,8 +243,7 @@ public static long[] getArchiveTimestamps(Configuration conf) {
    */
   @Deprecated
   public static long[] getFileTimestamps(Configuration conf) {
-    return parseTimestamps(
-        conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS));
+    return JobContextImpl.getFileTimestamps(conf);
   }
 
   /**
@@ -277,9 +256,7 @@ public static long[] getFileTimestamps(Configuration conf) {
    */
   @Deprecated
   public static void addCacheArchive(URI uri, Configuration conf) {
-    String archives = conf.get(MRJobConfig.CACHE_ARCHIVES);
-    conf.set(MRJobConfig.CACHE_ARCHIVES, archives == null ? uri.toString()
-             : archives + "," + uri.toString());
+    Job.addCacheArchive(uri, conf);
   }
 
   /**
@@ -307,9 +284,7 @@ public static void addCacheArchive(URI uri, Configuration conf) {
    */
   @Deprecated
   public static void addCacheFile(URI uri, Configuration conf) {
-    String files = conf.get(MRJobConfig.CACHE_FILES);
-    conf.set(MRJobConfig.CACHE_FILES, files == null ? uri.toString() : files + ","
-             + uri.toString());
+    Job.addCacheFile(uri, conf);
   }
 
   /**
@@ -323,9 +298,8 @@ public static void addCacheFile(URI uri, Configuration conf) {
    * @see Job#addFileToClassPath(Path)
    */
   @Deprecated
-  public static void addFileToClassPath(Path file, Configuration conf)
-    throws IOException {
-	  addFileToClassPath(file, conf, file.getFileSystem(conf));
+  public static void addFileToClassPath(Path file, Configuration conf) throws IOException {
+    Job.addFileToClassPath(file, conf, file.getFileSystem(conf));
   }
 
   /**
@@ -340,7 +314,7 @@ public static void addFileToClassPath(Path file, Configuration conf)
    */
   public static void addFileToClassPath(Path file, Configuration conf,
       FileSystem fs) {
-    addFileToClassPath(file, conf, fs, true);
+    Job.addFileToClassPath(file, conf, fs, true);
   }
 
   /**
@@ -357,14 +331,7 @@ public static void addFileToClassPath(Path file, Configuration conf,
    */
   public static void addFileToClassPath(Path file, Configuration conf,
       FileSystem fs, boolean addToCache) {
-    String classpath = conf.get(MRJobConfig.CLASSPATH_FILES);
-    conf.set(MRJobConfig.CLASSPATH_FILES, classpath == null ? file.toString()
-             : classpath + "," + file.toString());
-
-    if (addToCache) {
-      URI uri = fs.makeQualified(file).toUri();
-      addCacheFile(uri, conf);
-    }
+    Job.addFileToClassPath(file, conf, fs, addToCache);
   }
 
   /**
@@ -377,16 +344,7 @@ public static void addFileToClassPath(Path file, Configuration conf,
    */
   @Deprecated
   public static Path[] getFileClassPaths(Configuration conf) {
-    ArrayList<String> list = (ArrayList<String>)conf.getStringCollection(
-                                MRJobConfig.CLASSPATH_FILES);
-    if (list.size() == 0) {
-      return null;
-    }
-    Path[] paths = new Path[list.size()];
-    for (int i = 0; i < list.size(); i++) {
-      paths[i] = new Path(list.get(i));
-    }
-    return paths;
+    return JobContextImpl.getFileClassPaths(conf);
   }
 
   /**
@@ -401,7 +359,7 @@ public static Path[] getFileClassPaths(Configuration conf) {
   @Deprecated
   public static void addArchiveToClassPath(Path archive, Configuration conf)
     throws IOException {
-    addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
+    Job.addArchiveToClassPath(archive, conf, archive.getFileSystem(conf));
   }
 
   /**
@@ -415,12 +373,7 @@ public static void addArchiveToClassPath(Path archive, Configuration conf)
   public static void addArchiveToClassPath
          (Path archive, Configuration conf, FileSystem fs)
       throws IOException {
-    String classpath = conf.get(MRJobConfig.CLASSPATH_ARCHIVES);
-    conf.set(MRJobConfig.CLASSPATH_ARCHIVES, classpath == null ? archive
-             .toString() : classpath + "," + archive.toString());
-    URI uri = fs.makeQualified(archive).toUri();
-
-    addCacheArchive(uri, conf);
+    Job.addArchiveToClassPath(archive, conf, fs);
   }
 
   /**
@@ -433,16 +386,7 @@ public static void addArchiveToClassPath(Path archive, Configuration conf)
    */
   @Deprecated
   public static Path[] getArchiveClassPaths(Configuration conf) {
-    ArrayList<String> list = (ArrayList<String>)conf.getStringCollection(
-                                MRJobConfig.CLASSPATH_ARCHIVES);
-    if (list.size() == 0) {
-      return null;
-    }
-    Path[] paths = new Path[list.size()];
-    for (int i = 0; i < list.size(); i++) {
-      paths[i] = new Path(list.get(i));
-    }
-    return paths;
+    return JobContextImpl.getArchiveClassPaths(conf);
   }
 
   /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
index a1447c58b4631..a3fa57ee242b7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 /**
  * Event to record start of a task attempt
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
index d09d5ca6e9b8a..275ff1d187122 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
@@ -30,7 +30,6 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 /**
  * Event to record start of a task attempt
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/LongValueMax.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/LongValueMax.java
index 90b4ae0f54442..d574bf694f2de 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/LongValueMax.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/LongValueMax.java
@@ -96,7 +96,7 @@ public void reset() {
    *         expected to be used by the a combiner.
    */
   public ArrayList<String> getCombinerOutput() {
-    ArrayList<String> retv = new ArrayList<String>(1);;
+    ArrayList<String> retv = new ArrayList<String>(1);
     retv.add("" + maxVal);
     return retv;
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
index de25f6490620d..110d4e7f74196 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/aggregate/ValueAggregatorJob.java
@@ -200,7 +200,7 @@ public static Configuration setAggregatorDescriptors(
     conf.setInt(ValueAggregatorJobBase.DESCRIPTOR_NUM, descriptors.length);
     //specify the aggregator descriptors
     for(int i=0; i< descriptors.length; i++) {
-      conf.set(ValueAggregatorJobBase.DESCRIPTOR + i, 
+      conf.set(ValueAggregatorJobBase.DESCRIPTOR + "." + i,
                "UserDefined," + descriptors[i].getName());
     }
     return conf;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
index b6f151ac90205..764f31d6356ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
@@ -122,7 +122,7 @@ public void write(K key, V value) throws IOException {
         key.write(statement);
         statement.addBatch();
       } catch (SQLException e) {
-        e.printStackTrace();
+        throw new IOException("Failed to execute SQL statement for key " + key, e);
       }
     }
   }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
index f780715e7acec..90fc67d2bc6dc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
@@ -18,30 +18,19 @@
 
 package org.apache.hadoop.mapreduce.lib.db;
 
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.IOException;
 import java.sql.Connection;
-import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
index d1c61c4dcb4fd..080c2751c3383 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
@@ -21,9 +21,6 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
@@ -36,15 +33,11 @@
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBRecordReader.java
index 447a8fd55b46a..0682954698011 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBRecordReader.java
@@ -18,30 +18,11 @@
 
 package org.apache.hadoop.mapreduce.lib.db;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DateSplitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DateSplitter.java
index b0866f56cf04c..ba85cb3c8d342 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DateSplitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DateSplitter.java
@@ -20,8 +20,6 @@
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Time;
-import java.sql.Timestamp;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Date;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDBRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDBRecordReader.java
index e71e598e5073b..9e5537926af06 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDBRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDBRecordReader.java
@@ -20,7 +20,6 @@
 
 import java.sql.Connection;
 import java.sql.ResultSet;
-import java.sql.PreparedStatement;
 import java.sql.SQLException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDataDrivenDBRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDataDrivenDBRecordReader.java
index bd8dea6b34691..a55cdc95a733b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDataDrivenDBRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/MySQLDataDrivenDBRecordReader.java
@@ -20,7 +20,6 @@
 
 import java.sql.Connection;
 import java.sql.ResultSet;
-import java.sql.PreparedStatement;
 import java.sql.SQLException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
index 981b040e10af4..f0f67b9ae682d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
@@ -18,29 +18,12 @@
 
 package org.apache.hadoop.mapreduce.lib.db;
 
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.sql.Statement;
 import java.sql.Types;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBRecordReader.java
index 25dcaa0aa5f71..4e5f86b4533f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBRecordReader.java
@@ -19,8 +19,6 @@
 package org.apache.hadoop.mapreduce.lib.db;
 
 import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.PreparedStatement;
 import java.sql.SQLException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/TextSplitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/TextSplitter.java
index a5c6f56072de1..1cc919999946a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/TextSplitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/TextSplitter.java
@@ -22,7 +22,6 @@
 import java.sql.SQLException;
 import java.math.BigDecimal;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 
 import org.slf4j.Logger;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
index 9d0e949a10b81..74b959cf47c3e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
@@ -127,6 +127,8 @@ public CompressedSplitLineReader(SplitCompressionInputStream in,
   @Override
   protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter)
       throws IOException {
+    boolean alreadyReadAfterSplit = didReadAfterSplit();
+
     int bytesRead = in.read(buffer);
 
     // If the split ended in the middle of a record delimiter then we need
@@ -135,7 +137,9 @@ protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter)
     // However if using the default delimiter and the next character is a
     // linefeed then next split will treat it as a delimiter all by itself
     // and the additional record read should not be performed.
-    if (inDelimiter && bytesRead > 0) {
+    boolean justReadAfterSplit = !alreadyReadAfterSplit && didReadAfterSplit();
+
+    if (justReadAfterSplit && inDelimiter && bytesRead > 0) {
       if (usingCRLF) {
         needAdditionalRecord = (buffer[0] != '\n');
       } else {
@@ -152,7 +156,7 @@ public int readLine(Text str, int maxLineLength, int maxBytesToConsume)
     if (!finished) {
       // only allow at most one more record to be read after the stream
       // reports the split ended
-      if (scin.getPos() > scin.getAdjustedEnd()) {
+      if (didReadAfterSplit()) {
         finished = true;
       }
 
@@ -170,4 +174,8 @@ public boolean needAdditionalRecordAfterSplit() {
   protected void unsetNeedAdditionalRecordAfterSplit() {
     needAdditionalRecord = false;
   }
+
+  private boolean didReadAfterSplit() throws IOException {
+    return scin.getPos() > scin.getAdjustedEnd();
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FixedLengthRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FixedLengthRecordReader.java
index c0ae9a5cdac61..6969f61836fbc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FixedLengthRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FixedLengthRecordReader.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.compress.CodecPool;
@@ -40,6 +39,8 @@
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.functional.FutureIO;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -94,10 +95,10 @@ public void initialize(Configuration job, long splitStart, long splitLength,
     // open the file
     final FutureDataInputStreamBuilder builder =
         file.getFileSystem(job).openFile(file);
-    FutureIOSupport.propagateOptions(builder, job,
+    FutureIO.propagateOptions(builder, job,
         MRJobConfig.INPUT_FILE_OPTION_PREFIX,
         MRJobConfig.INPUT_FILE_MANDATORY_PREFIX);
-    fileIn = FutureIOSupport.awaitFuture(builder.build());
+    fileIn = FutureIO.awaitFuture(builder.build());
 
     CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
     if (null != codec) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
index 160c7635658a4..617abaacae065 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.CodecPool;
@@ -40,9 +39,14 @@
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.functional.FutureIO;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_END;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_START;
+
 /**
  * Treats keys as offset in file and value as line. 
  */
@@ -86,10 +90,14 @@ public void initialize(InputSplit genericSplit,
     // open the file and seek to the start of the split
     final FutureDataInputStreamBuilder builder =
         file.getFileSystem(job).openFile(file);
-    FutureIOSupport.propagateOptions(builder, job,
+    // the start and end of the split may be used to build
+    // an input strategy.
+    builder.opt(FS_OPTION_OPENFILE_SPLIT_START, start);
+    builder.opt(FS_OPTION_OPENFILE_SPLIT_END, end);
+    FutureIO.propagateOptions(builder, job,
         MRJobConfig.INPUT_FILE_OPTION_PREFIX,
         MRJobConfig.INPUT_FILE_MANDATORY_PREFIX);
-    fileIn = FutureIOSupport.awaitFuture(builder.build());
+    fileIn = FutureIO.awaitFuture(builder.build());
     
     CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
     if (null!=codec) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
index dfff9ad0d2b73..5161a96c3459d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -39,6 +38,7 @@
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.functional.FutureIO;
 
 /**
  * NLineInputFormat which splits N lines of input as one split.
@@ -99,10 +99,10 @@ public static List<FileSplit> getSplitsForFile(FileStatus status,
     try {
       final FutureDataInputStreamBuilder builder =
           fileName.getFileSystem(conf).openFile(fileName);
-      FutureIOSupport.propagateOptions(builder, conf,
+      FutureIO.propagateOptions(builder, conf,
           MRJobConfig.INPUT_FILE_OPTION_PREFIX,
           MRJobConfig.INPUT_FILE_MANDATORY_PREFIX);
-      FSDataInputStream in  = FutureIOSupport.awaitFuture(builder.build());
+      FSDataInputStream in  = FutureIO.awaitFuture(builder.build());
       lr = new LineReader(in, conf);
       Text line = new Text();
       int numLines = 0;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
index f12678b5562bb..adde981f24723 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
@@ -23,6 +23,10 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.IOStatisticsSupport;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -60,7 +64,8 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Unstable
-public class BindingPathOutputCommitter extends PathOutputCommitter {
+public class BindingPathOutputCommitter extends PathOutputCommitter
+    implements IOStatisticsSource, StreamCapabilities {
 
   /**
    * The classname for use in configurations.
@@ -181,4 +186,22 @@ public String toString() {
   public PathOutputCommitter getCommitter() {
     return committer;
   }
+
+  /**
+   * Pass through if the inner committer supports StreamCapabilities.
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean hasCapability(final String capability) {
+    if (committer instanceof StreamCapabilities) {
+      return ((StreamCapabilities) committer).hasCapability(capability);
+    } else {
+      return false;
+    }
+  }
+
+  @Override
+  public IOStatistics getIOStatistics() {
+    return IOStatisticsSupport.retrieveIOStatistics(committer);
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
index 877d73c5a59de..82b7fcb504622 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
@@ -37,7 +37,7 @@
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
index 15ff3c67c2fb5..2b1f7e37ebe75 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.text.NumberFormat;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
index a3a0e76ab81d1..3c36dfb8bbafd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
@@ -19,16 +19,23 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.*;
-import org.apache.hadoop.mapreduce.Reducer.Context;
-import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import java.io.IOException;
 import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The MultipleOutputs class simplifies writing output data 
@@ -193,6 +200,8 @@ public class MultipleOutputs<KEYOUT, VALUEOUT> {
    * Counters group used by the counters of MultipleOutputs.
    */
   private static final String COUNTERS_GROUP = MultipleOutputs.class.getName();
+  private static final Logger LOG =
+      LoggerFactory.getLogger(org.apache.hadoop.mapred.lib.MultipleOutputs.class);
 
   /**
    * Cache for the taskContexts
@@ -347,6 +356,11 @@ public static boolean getCountersEnabled(JobContext job) {
     return job.getConfiguration().getBoolean(COUNTERS_ENABLED, false);
   }
 
+  @VisibleForTesting
+  synchronized void setRecordWriters(Map<String, RecordWriter<?, ?>> recordWriters) {
+    this.recordWriters = recordWriters;
+  }
+
   /**
    * Wraps RecordWriter to increment counters. 
    */
@@ -570,8 +584,43 @@ public void setStatus(String status) {
    */
   @SuppressWarnings("unchecked")
   public void close() throws IOException, InterruptedException {
+    Configuration conf = context.getConfiguration();
+    int nThreads = conf.getInt(MRConfig.MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT,
+        MRConfig.DEFAULT_MULTIPLE_OUTPUTS_CLOSE_THREAD_COUNT);
+    AtomicBoolean encounteredException = new AtomicBoolean(false);
+    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("MultipleOutputs-close")
+        .setUncaughtExceptionHandler(((t, e) -> {
+          LOG.error("Thread " + t + " failed unexpectedly", e);
+          encounteredException.set(true);
+        })).build();
+    ExecutorService executorService = Executors.newFixedThreadPool(nThreads, threadFactory);
+
+    List<Callable<Object>> callableList = new ArrayList<>(recordWriters.size());
+
     for (RecordWriter writer : recordWriters.values()) {
-      writer.close(context);
+      callableList.add(() -> {
+        try {
+          writer.close(context);
+        } catch (IOException e) {
+          LOG.error("Error while closing MultipleOutput file", e);
+          encounteredException.set(true);
+        }
+        return null;
+      });
+    }
+    try {
+      executorService.invokeAll(callableList);
+    } catch (InterruptedException e) {
+      LOG.warn("Closing is Interrupted");
+      Thread.currentThread().interrupt();
+    } finally {
+      executorService.shutdown();
+    }
+
+    if (encounteredException.get()) {
+      throw new IOException(
+          "One or more threads encountered exception during close. See prior errors.");
     }
   }
 }
+
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
index ddcff646e04fd..3ec2795947c8d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
@@ -22,7 +22,7 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
index e463632fa56ae..c9fbe3b065f44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
index f3f0dabd6551f..cf83bf2d7f4b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
@@ -52,8 +52,8 @@
  * attempt context and a possibly null path.
  *
  */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class PathOutputCommitterFactory extends Configured {
   private static final Logger LOG =
       LoggerFactory.getLogger(PathOutputCommitterFactory.class);
@@ -158,7 +158,7 @@ public static PathOutputCommitterFactory getCommitterFactory(
           scheme);
       if (StringUtils.isNotEmpty(conf.getTrimmed(schemeKey))) {
         // it does, so use that key in the classname lookup
-        LOG.debug("Using schema-specific factory for {}", outputPath);
+        LOG.info("Using schema-specific factory for {}", outputPath);
         key = schemeKey;
       } else {
         LOG.debug("No scheme-specific factory defined in {}", schemeKey);
@@ -171,7 +171,7 @@ public static PathOutputCommitterFactory getCommitterFactory(
     String trimmedValue = conf.getTrimmed(key, "");
     if (StringUtils.isEmpty(trimmedValue)) {
       // empty/null value, use default
-      LOG.debug("No output committer factory defined,"
+      LOG.info("No output committer factory defined,"
           + " defaulting to FileOutputCommitterFactory");
       factory = FileOutputCommitterFactory.class;
     } else {
@@ -179,7 +179,7 @@ public static PathOutputCommitterFactory getCommitterFactory(
       factory = conf.getClass(key,
           FileOutputCommitterFactory.class,
           PathOutputCommitterFactory.class);
-      LOG.debug("Using OutputCommitter factory class {} from key {}",
+      LOG.info("Using OutputCommitter factory class {} from key {}",
           factory, key);
     }
     return ReflectionUtils.newInstance(factory, conf);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java
new file mode 100644
index 0000000000000..024fb3ab34eb2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java
@@ -0,0 +1,774 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.AbortTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageEventCallbacks;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtDebug;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_DIAGNOSTICS_MANIFEST_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_SUMMARY_REPORT_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_COMPLETED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_FAILED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME_RECOVERED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_ABORT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.updateCommonContextOnCommitterExit;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.updateCommonContextOnCommitterEntry;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createIOStatisticsStore;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createJobSummaryFilename;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createManifestOutcome;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestPathForTask;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage.cleanupStageOptionsFromConfig;
+
+/**
+ * This is the Intermediate-Manifest committer.
+ * At every entry point it updates the thread's audit context with
+ * the current stage info; this is a placeholder for
+ * adding audit information to stores other than S3A.
+ *
+ * This is tagged as public/stable. This is mandatory
+ * for the classname and PathOutputCommitter implementation
+ * classes.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ManifestCommitter extends PathOutputCommitter implements
+    IOStatisticsSource, StageEventCallbacks, StreamCapabilities {
+
+  public static final Logger LOG = LoggerFactory.getLogger(
+      ManifestCommitter.class);
+
+  /**
+   * Role: task committer.
+   */
+  public static final String TASK_COMMITTER = "task committer";
+
+  /**
+   * Role: job committer.
+   */
+  public static final String JOB_COMMITTER = "job committer";
+
+  /**
+   * Committer Configuration as extracted from
+   * the job/task context and set in the constructor.
+   */
+  private final ManifestCommitterConfig baseConfig;
+
+  /**
+   * Destination of the job.
+   */
+  private final Path destinationDir;
+
+  /**
+   * For tasks, the attempt directory.
+   * Null for jobs.
+   */
+  private final Path taskAttemptDir;
+
+  /**
+   * IOStatistics to update.
+   */
+  private final IOStatisticsStore iostatistics;
+
+  /**
+   *  The job Manifest Success data; only valid after a job successfully
+   *  commits.
+   */
+  private ManifestSuccessData successReport;
+
+  /**
+   * The active stage; is updated by a callback from within the stages.
+   */
+  private String activeStage;
+
+  /**
+   * The task manifest of the task commit.
+   * Null unless this is a task attempt and the
+   * task has successfully been committed.
+   */
+  private TaskManifest taskAttemptCommittedManifest;
+
+  /**
+   * Create a committer.
+   * @param outputPath output path
+   * @param context job/task context
+   * @throws IOException failure.
+   */
+  public ManifestCommitter(final Path outputPath,
+      final TaskAttemptContext context) throws IOException {
+    super(outputPath, context);
+    this.destinationDir = resolveDestinationDirectory(outputPath,
+        context.getConfiguration());
+    this.iostatistics = createIOStatisticsStore().build();
+    this.baseConfig = enterCommitter(
+        context.getTaskAttemptID() != null,
+        context);
+
+    this.taskAttemptDir = baseConfig.getTaskAttemptDir();
+    LOG.info("Created ManifestCommitter with JobID {},"
+            + " Task Attempt {} and destination {}",
+        context.getJobID(), context.getTaskAttemptID(), outputPath);
+  }
+
+  /**
+   * Committer method invoked; generates a config for it.
+   * Calls {@code #updateCommonContextOnCommitterEntry()}
+   * to update the audit context.
+   * @param isTask is this a task entry point?
+   * @param context context
+   * @return committer config
+   */
+  private ManifestCommitterConfig enterCommitter(boolean isTask,
+      JobContext context) {
+    ManifestCommitterConfig committerConfig =
+        new ManifestCommitterConfig(
+            getOutputPath(),
+            isTask ? TASK_COMMITTER : JOB_COMMITTER,
+            context,
+            iostatistics,
+            this);
+    updateCommonContextOnCommitterEntry(committerConfig);
+    return committerConfig;
+  }
+
+  /**
+   * Set up a job through a {@link SetupJobStage}.
+   * @param jobContext Context of the job whose output is being written.
+   * @throws IOException IO Failure.
+   */
+  @Override
+  public void setupJob(final JobContext jobContext) throws IOException {
+    ManifestCommitterConfig committerConfig = enterCommitter(false,
+        jobContext);
+    StageConfig stageConfig =
+        committerConfig
+            .createStageConfig()
+            .withOperations(createManifestStoreOperations())
+            .build();
+    // set up the job.
+    new SetupJobStage(stageConfig)
+        .apply(committerConfig.getCreateJobMarker());
+    logCommitterStatisticsAtDebug();
+  }
+
+  /**
+   * Set up a task through a {@link SetupTaskStage}.
+   * Classic FileOutputCommitter is a no-op here, relying
+   * on RecordWriters to create the dir implicitly on file
+   * create().
+   * FileOutputCommitter also uses the existence of that
+   * file as a flag to indicate task commit is needed.
+   * @param context task context.
+   * @throws IOException IO Failure.
+   */
+  @Override
+  public void setupTask(final TaskAttemptContext context)
+      throws IOException {
+    ManifestCommitterConfig committerConfig =
+        enterCommitter(true, context);
+    StageConfig stageConfig =
+        committerConfig
+            .createStageConfig()
+            .withOperations(createManifestStoreOperations())
+            .build();
+    // create task attempt dir; delete if present. Or fail?
+    new SetupTaskStage(stageConfig).apply("");
+    logCommitterStatisticsAtDebug();
+  }
+
+  /**
+   * Always return true.
+   * This way, even if there is no output, stats are collected.
+   * @param context task context.
+   * @return true
+   * @throws IOException IO Failure.
+   */
+  @Override
+  public boolean needsTaskCommit(final TaskAttemptContext context)
+      throws IOException {
+    LOG.info("Probe for needsTaskCommit({})",
+        context.getTaskAttemptID());
+    return true;
+  }
+
+  /**
+   * Failure during Job Commit is not recoverable from.
+   *
+   * @param jobContext
+   *          Context of the job whose output is being written.
+   * @return false, always
+   * @throws IOException never
+   */
+  @Override
+  public boolean isCommitJobRepeatable(final JobContext jobContext)
+      throws IOException {
+    LOG.info("Probe for isCommitJobRepeatable({}): returning false",
+        jobContext.getJobID());
+    return false;
+  }
+
+  /**
+   * Declare that task recovery is not supported.
+   * It would be, if someone added the code *and tests*.
+   * @param jobContext
+   *          Context of the job whose output is being written.
+   * @return false, always
+   * @throws IOException never
+   */
+  @Override
+  public boolean isRecoverySupported(final JobContext jobContext)
+      throws IOException {
+    LOG.info("Probe for isRecoverySupported({}): returning false",
+        jobContext.getJobID());
+    return false;
+  }
+
+  /**
+   *
+   * @param taskContext Context of the task whose output is being recovered
+   * @throws IOException always
+   */
+  @Override
+  public void recoverTask(final TaskAttemptContext taskContext)
+      throws IOException {
+    LOG.warn("Rejecting recoverTask({}) call", taskContext.getTaskAttemptID());
+    throw new IOException("Cannot recover task "
+        + taskContext.getTaskAttemptID());
+  }
+
+  /**
+   * Commit the task.
+   * This is where the task attempt tree list takes place.
+   * @param context task context.
+   * @throws IOException IO Failure.
+   */
+  @Override
+  public void commitTask(final TaskAttemptContext context)
+      throws IOException {
+    ManifestCommitterConfig committerConfig = enterCommitter(true,
+        context);
+    try {
+      StageConfig stageConfig = committerConfig.createStageConfig()
+          .withOperations(createManifestStoreOperations())
+          .build();
+      taskAttemptCommittedManifest = new CommitTaskStage(stageConfig)
+          .apply(null).getTaskManifest();
+      iostatistics.incrementCounter(COMMITTER_TASKS_COMPLETED_COUNT, 1);
+    } catch (IOException e) {
+      iostatistics.incrementCounter(COMMITTER_TASKS_FAILED_COUNT, 1);
+      throw e;
+    } finally {
+      logCommitterStatisticsAtDebug();
+      updateCommonContextOnCommitterExit();
+    }
+
+  }
+
+  /**
+   * Abort a task.
+   * @param context task context
+   * @throws IOException failure during the delete
+   */
+  @Override
+  public void abortTask(final TaskAttemptContext context)
+      throws IOException {
+    ManifestCommitterConfig committerConfig = enterCommitter(true,
+        context);
+    try {
+      new AbortTaskStage(
+          committerConfig.createStageConfig()
+              .withOperations(createManifestStoreOperations())
+              .build())
+          .apply(false);
+    } finally {
+      logCommitterStatisticsAtDebug();
+      updateCommonContextOnCommitterExit();
+    }
+  }
+
+  /**
+   * Get the manifest success data for this job; creating on demand if needed.
+   * @param committerConfig source config.
+   * @return the current {@link #successReport} value; never null.
+   */
+  private ManifestSuccessData getOrCreateSuccessData(
+      ManifestCommitterConfig committerConfig) {
+    if (successReport == null) {
+      successReport = createManifestOutcome(
+          committerConfig.createStageConfig(), activeStage);
+    }
+    return successReport;
+  }
+
+  /**
+   * This is the big job commit stage.
+   * Load the manifests, prepare the destination, rename
+   * the files then cleanup the job directory.
+   * @param jobContext Context of the job whose output is being written.
+   * @throws IOException failure.
+   */
+  @Override
+  public void commitJob(final JobContext jobContext) throws IOException {
+
+    ManifestCommitterConfig committerConfig = enterCommitter(false, jobContext);
+
+    // create the initial success data.
+    // this is overwritten by that created during the operation sequence,
+    // but if the sequence fails before that happens, it
+    // will be saved to the report directory.
+    ManifestSuccessData marker = getOrCreateSuccessData(committerConfig);
+    IOException failure = null;
+    try (CloseableTaskPoolSubmitter ioProcs =
+             committerConfig.createSubmitter();
+         ManifestStoreOperations storeOperations = createManifestStoreOperations()) {
+      // the stage config will be shared across all stages.
+      StageConfig stageConfig = committerConfig.createStageConfig()
+          .withOperations(storeOperations)
+          .withIOProcessors(ioProcs)
+          .build();
+
+      // commit the job, including any cleanup and validation.
+      final Configuration conf = jobContext.getConfiguration();
+      CommitJobStage.Result result = new CommitJobStage(stageConfig).apply(
+          new CommitJobStage.Arguments(
+              committerConfig.getCreateJobMarker(),
+              committerConfig.getValidateOutput(),
+              conf.getTrimmed(OPT_DIAGNOSTICS_MANIFEST_DIR, ""),
+              cleanupStageOptionsFromConfig(
+                  OP_STAGE_JOB_CLEANUP, conf)
+          ));
+      marker = result.getJobSuccessData();
+      // update the cached success with the new report.
+      setSuccessReport(marker);
+
+    } catch (IOException e) {
+      // failure. record it for the summary
+      failure = e;
+      // rethrow
+      throw e;
+    } finally {
+      // save the report summary, even on failure
+      maybeSaveSummary(activeStage,
+          committerConfig,
+          marker,
+          failure,
+          true,
+          true);
+      // print job commit stats
+      LOG.info("{}: Job Commit statistics {}",
+          committerConfig.getName(),
+          ioStatisticsToPrettyString(iostatistics));
+      // and warn of rename problems
+      final Long recoveries = iostatistics.counters().get(OP_COMMIT_FILE_RENAME_RECOVERED);
+      if (recoveries != null && recoveries > 0) {
+        LOG.warn("{}: rename failures were recovered from. Number of recoveries: {}",
+            committerConfig.getName(), recoveries);
+      }
+      updateCommonContextOnCommitterExit();
+    }
+  }
+
+  /**
+   * Abort the job.
+   * Invokes
+   * {@link #executeCleanup(String, JobContext, ManifestCommitterConfig)}
+   * then saves the (ongoing) job report data if reporting is enabled.
+   * @param jobContext Context of the job whose output is being written.
+   * @param state final runstate of the job
+   * @throws IOException failure during cleanup; report failure are swallowed
+   */
+  @Override
+  public void abortJob(final JobContext jobContext,
+      final JobStatus.State state)
+      throws IOException {
+    LOG.info("Aborting Job {} in state {}", jobContext.getJobID(), state);
+    ManifestCommitterConfig committerConfig = enterCommitter(false,
+        jobContext);
+    ManifestSuccessData report = getOrCreateSuccessData(
+        committerConfig);
+    IOException failure = null;
+
+    try {
+      executeCleanup(OP_STAGE_JOB_ABORT, jobContext, committerConfig);
+    } catch (IOException e) {
+      // failure.
+      failure = e;
+    }
+    report.setSuccess(false);
+    // job abort does not overwrite any existing report, so a job commit
+    // failure cause will be preserved.
+    maybeSaveSummary(activeStage, committerConfig, report, failure,
+        true, false);
+    // print job stats
+    LOG.info("Job Abort statistics {}",
+        ioStatisticsToPrettyString(iostatistics));
+    updateCommonContextOnCommitterExit();
+  }
+
+  /**
+   * Execute the {@code CleanupJobStage} to remove the job attempt dir.
+   * This does
+   * @param jobContext Context of the job whose output is being written.
+   * @throws IOException failure during cleanup
+   */
+  @SuppressWarnings("deprecation")
+  @Override
+  public void cleanupJob(final JobContext jobContext) throws IOException {
+    ManifestCommitterConfig committerConfig = enterCommitter(false,
+        jobContext);
+    try {
+      executeCleanup(OP_STAGE_JOB_CLEANUP, jobContext, committerConfig);
+    } finally {
+      logCommitterStatisticsAtDebug();
+      updateCommonContextOnCommitterExit();
+    }
+  }
+
+  /**
+   * Perform the cleanup operation for job cleanup or abort.
+   * @param statisticName statistic/stage name
+   * @param jobContext job context
+   * @param committerConfig committer config
+   * @throws IOException failure
+   * @return the outcome
+   */
+  private CleanupJobStage.Result executeCleanup(
+      final String statisticName,
+      final JobContext jobContext,
+      final ManifestCommitterConfig committerConfig) throws IOException {
+    try (CloseableTaskPoolSubmitter ioProcs =
+             committerConfig.createSubmitter()) {
+
+      return new CleanupJobStage(
+          committerConfig.createStageConfig()
+              .withOperations(createManifestStoreOperations())
+              .withIOProcessors(ioProcs)
+              .build())
+          .apply(cleanupStageOptionsFromConfig(
+              statisticName,
+              jobContext.getConfiguration()));
+    }
+  }
+
+  /**
+   * Output path: destination directory of the job.
+   * @return the overall job destination directory.
+   */
+  @Override
+  public Path getOutputPath() {
+    return getDestinationDir();
+  }
+
+  /**
+   * Work path of the current task attempt.
+   * This is null if the task does not have one.
+   * @return a path.
+   */
+  @Override
+  public Path getWorkPath() {
+    return getTaskAttemptDir();
+  }
+
+  /**
+   * Get the job destination dir.
+   * @return dest dir.
+   */
+  private Path getDestinationDir() {
+    return destinationDir;
+  }
+
+  /**
+   * Get the task attempt dir.
+   * May be null.
+   * @return a path or null.
+   */
+  private Path getTaskAttemptDir() {
+    return taskAttemptDir;
+  }
+
+  /**
+   * Callback on stage entry.
+   * Sets {@link #activeStage} and updates the
+   * common context.
+   * @param stage new stage
+   */
+  @Override
+  public void enterStage(String stage) {
+    activeStage = stage;
+    AuditingIntegration.enterStage(stage);
+  }
+
+  /**
+   * Remove stage from common audit context.
+   * @param stage stage exited.
+   */
+  @Override
+  public void exitStage(String stage) {
+    AuditingIntegration.exitStage();
+  }
+
+  /**
+   * Get the unique ID of this job.
+   * @return job ID (yarn, spark)
+   */
+  public String getJobUniqueId() {
+    return baseConfig.getJobUniqueId();
+  }
+
+  /**
+   * Get the config of the task attempt this instance was constructed
+   * with.
+   * @return a configuration.
+   */
+  public Configuration getConf() {
+    return baseConfig.getConf();
+  }
+
+  /**
+   * Get the manifest Success data; only valid after a job.
+   * @return the job _SUCCESS data, or null.
+   */
+  public ManifestSuccessData getSuccessReport() {
+    return successReport;
+  }
+
+  private void setSuccessReport(ManifestSuccessData successReport) {
+    this.successReport = successReport;
+  }
+
+  /**
+   * Get the manifest of the last committed task.
+   * @return a task manifest or null.
+   */
+  @VisibleForTesting
+  TaskManifest getTaskAttemptCommittedManifest() {
+    return taskAttemptCommittedManifest;
+  }
+
+  /**
+   * Compute the path where the output of a task attempt is stored until
+   * that task is committed.
+   * @param context the context of the task attempt.
+   * @return the path where a task attempt should be stored.
+   */
+  @VisibleForTesting
+  public Path getTaskAttemptPath(TaskAttemptContext context) {
+    return enterCommitter(false, context).getTaskAttemptDir();
+  }
+
+  /**
+   * The path to where the manifest file of a task attempt will be
+   * saved when the task is committed.
+   * This path will be the same for all attempts of the same task.
+   * @param context the context of the task attempt.
+   * @return the path where a task attempt should be stored.
+   */
+  @VisibleForTesting
+  public Path getTaskManifestPath(TaskAttemptContext context) {
+    final Path dir = enterCommitter(false, context).getTaskManifestDir();
+
+    return manifestPathForTask(dir,
+        context.getTaskAttemptID().getTaskID().toString());
+  }
+
+  /**
+   * Compute the path where the output of a task attempt is stored until
+   * that task is committed.
+   * @param context the context of the task attempt.
+   * @return the path where a task attempt should be stored.
+   */
+  @VisibleForTesting
+  public Path getJobAttemptPath(JobContext context) {
+
+    return enterCommitter(false, context).getJobAttemptDir();
+  }
+
+  /**
+   * Get the final output path, including resolving any relative path.
+   * @param outputPath output path
+   * @param conf configuration to create any FS with
+   * @return a resolved path.
+   * @throws IOException failure.
+   */
+  private Path resolveDestinationDirectory(Path outputPath,
+      Configuration conf) throws IOException {
+    return FileSystem.get(outputPath.toUri(), conf).makeQualified(outputPath);
+  }
+
+  /**
+   * Create manifest store operations for the destination store.
+   * This MUST NOT be used for the success report operations, as
+   * they may be to a different filesystem.
+   * This is a point which can be overridden during testing.
+   * @return a new store operations instance bonded to the destination fs.
+   * @throws IOException failure to instantiate.
+   */
+  protected ManifestStoreOperations createManifestStoreOperations() throws IOException {
+    return ManifestCommitterSupport.createManifestStoreOperations(
+        baseConfig.getConf(),
+        baseConfig.getDestinationFileSystem(),
+        baseConfig.getDestinationDir());
+  }
+
+  /**
+   * Log IO Statistics at debug.
+   */
+  private void logCommitterStatisticsAtDebug() {
+    logIOStatisticsAtDebug(LOG, "Committer Statistics", this);
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder(
+        "ManifestCommitter{");
+    sb.append(baseConfig);
+    sb.append(", iostatistics=").append(ioStatisticsToPrettyString(iostatistics));
+    sb.append('}');
+    return sb.toString();
+  }
+
+  /**
+   * Save a summary to the report dir if the config option
+   * is set.
+   * The IOStatistics of the summary will be updated to the latest
+   * snapshot of the committer's statistics, so the report is up
+   * to date.
+   * The report will updated with the current active stage,
+   * and if {@code thrown} is non-null, it will be added to the
+   * diagnistics (and the job tagged as a failure).
+   * Static for testability.
+   * @param activeStage active stage
+   * @param config configuration to use.
+   * @param report summary file.
+   * @param thrown any exception indicting failure.
+   * @param quiet should exceptions be swallowed.
+   * @param overwrite should the existing file be overwritten
+   * @return the path of a file, if successfully saved
+   * @throws IOException if a failure occured and quiet==false
+   */
+  private static Path maybeSaveSummary(
+      String activeStage,
+      ManifestCommitterConfig config,
+      ManifestSuccessData report,
+      Throwable thrown,
+      boolean quiet,
+      boolean overwrite) throws IOException {
+    Configuration conf = config.getConf();
+    String reportDir = conf.getTrimmed(OPT_SUMMARY_REPORT_DIR, "");
+    if (reportDir.isEmpty()) {
+      LOG.debug("No summary directory set in " + OPT_SUMMARY_REPORT_DIR);
+      return null;
+    }
+    LOG.debug("Summary directory set in to {}" + OPT_SUMMARY_REPORT_DIR,
+        reportDir);
+
+    // update to the latest statistics
+    report.snapshotIOStatistics(config.getIOStatistics());
+
+    Path reportDirPath = new Path(reportDir);
+    Path path = new Path(reportDirPath,
+        createJobSummaryFilename(config.getJobUniqueId()));
+
+    if (thrown != null) {
+      report.recordJobFailure(thrown);
+    }
+    report.putDiagnostic(STAGE, activeStage);
+    // the store operations here is explicitly created for the FS where
+    // the reports go, which may not be the target FS of the job.
+
+    final FileSystem fs = path.getFileSystem(conf);
+    try (ManifestStoreOperations operations = new ManifestStoreOperationsThroughFileSystem(fs)) {
+      if (!overwrite) {
+        // check for file existence so there is no need to worry about
+        // precisely what exception is raised when overwrite=false and dest file
+        // exists
+        try {
+          FileStatus st = operations.getFileStatus(path);
+          // get here and the file exists
+          LOG.debug("Report already exists: {}", st);
+          return null;
+        } catch (FileNotFoundException ignored) {
+        }
+      }
+      operations.save(report, path, overwrite);
+      LOG.info("Job summary saved to {}", path);
+      return path;
+    } catch (IOException e) {
+      LOG.debug("Failed to save summary to {}", path, e);
+      if (quiet) {
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  @Override
+  public IOStatisticsStore getIOStatistics() {
+    return iostatistics;
+  }
+
+  /**
+   * The committer is compatible with spark's dynamic partitioning
+   * algorithm.
+   * @param capability string to query the stream support for.
+   * @return true if the requested capability is supported.
+   */
+  @Override
+  public boolean hasCapability(final String capability) {
+    return CAPABILITY_DYNAMIC_PARTITIONING.equals(capability);
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java
new file mode 100644
index 0000000000000..40dffee203b8d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageEventCallbacks;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.SUCCESSFUL_JOB_OUTPUT_DIR_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.*;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.buildJobUUID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.getAppAttemptId;
+
+/**
+ * The configuration for the committer as built up from the job configuration
+ * and data passed down from the committer factory.
+ * Isolated for ease of dev/test
+ */
+public final class ManifestCommitterConfig implements IOStatisticsSource {
+
+  /**
+   * Final destination of work.
+   * This is <i>unqualified</i>.
+   */
+  private final Path destinationDir;
+
+  /**
+   * Role: used in log/text messages.
+   */
+  private final String role;
+
+  /**
+   * This is the directory for all intermediate work: where the output
+   * format will write data.
+   * Will be null if built from a job context.
+   */
+  private final Path taskAttemptDir;
+
+  /** Configuration of the job. */
+  private final Configuration conf;
+
+  /** The job context. For a task, this can be cast to a TaskContext. */
+  private final JobContext jobContext;
+
+  /** Should a job marker be created? */
+  private final boolean createJobMarker;
+
+  /**
+   * Job ID Or UUID -without any attempt suffix.
+   * This is expected/required to be unique, though
+   * Spark has had "issues" there until recently
+   * with lack of uniqueness of generated MR Job IDs.
+   */
+  private final String jobUniqueId;
+
+  /**
+   * Where did the job Unique ID come from?
+   */
+  private final String jobUniqueIdSource;
+
+  /**
+   * Number of this attempt; starts at zero.
+   */
+  private final int jobAttemptNumber;
+
+  /**
+   * Job ID + AttemptID.
+   */
+  private final String jobAttemptId;
+
+  /**
+   * Task ID: used as the filename of the manifest.
+   * Will be "" if built from a job context.
+   */
+  private final String taskId;
+
+  /**
+   * Task attempt ID. Determines the working
+   * directory for task attempts to write data into,
+   * and for the task committer to scan.
+   * Will be "" if built from a job context.
+   */
+  private final String taskAttemptId;
+
+  /** Any progressable for progress callbacks. */
+  private final Progressable progressable;
+
+  /**
+   * IOStatistics to update.
+   */
+  private final IOStatisticsStore iostatistics;
+
+
+  /** Should the output be validated after the commit? */
+  private final boolean validateOutput;
+
+  /**
+   * Attempt directory management.
+   */
+  private final ManifestCommitterSupport.AttemptDirectories dirs;
+
+  /**
+   * Callback when a stage is entered.
+   */
+  private final StageEventCallbacks stageEventCallbacks;
+
+  /**
+   * Name for logging.
+   */
+  private final String name;
+
+  /**
+   * Delete target paths on commit? Stricter, but
+   * higher IO cost.
+   */
+  private final boolean deleteTargetPaths;
+
+  /**
+   * Constructor.
+   * @param outputPath destination path of the job.
+   * @param role role for log messages.
+   * @param context job/task context
+   * @param iostatistics IO Statistics
+   * @param stageEventCallbacks stage event callbacks.
+   */
+
+  ManifestCommitterConfig(
+      final Path outputPath,
+      final String role,
+      final JobContext context,
+      final IOStatisticsStore iostatistics,
+      final StageEventCallbacks stageEventCallbacks) {
+    this.role = role;
+    this.jobContext = context;
+    this.conf = context.getConfiguration();
+    this.destinationDir = outputPath;
+    this.iostatistics = iostatistics;
+    this.stageEventCallbacks = stageEventCallbacks;
+
+    Pair<String, String> pair = buildJobUUID(conf, context.getJobID());
+    this.jobUniqueId = pair.getLeft();
+    this.jobUniqueIdSource = pair.getRight();
+    this.jobAttemptNumber = getAppAttemptId(context);
+    this.jobAttemptId = this.jobUniqueId + "_" + jobAttemptNumber;
+
+    // build directories
+    this.dirs = new ManifestCommitterSupport.AttemptDirectories(outputPath,
+        this.jobUniqueId, jobAttemptNumber);
+
+    // read in configuration options
+    this.createJobMarker = conf.getBoolean(
+        SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
+        DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER);
+    this.validateOutput = conf.getBoolean(
+        OPT_VALIDATE_OUTPUT,
+        OPT_VALIDATE_OUTPUT_DEFAULT);
+    this.deleteTargetPaths = conf.getBoolean(
+        OPT_DELETE_TARGET_FILES,
+        OPT_DELETE_TARGET_FILES_DEFAULT);
+
+    // if constructed with a task attempt, build the task ID and path.
+    if (context instanceof TaskAttemptContext) {
+      // it's a task
+      final TaskAttemptContext tac = (TaskAttemptContext) context;
+      TaskAttemptID taskAttempt = Objects.requireNonNull(
+          tac.getTaskAttemptID());
+      taskAttemptId = taskAttempt.toString();
+      taskId = taskAttempt.getTaskID().toString();
+      // Task attempt dir; must be different across instances
+      taskAttemptDir = dirs.getTaskAttemptPath(taskAttemptId);
+      // the context is also the progress callback.
+      progressable = tac;
+      name = String.format(InternalConstants.NAME_FORMAT_TASK_ATTEMPT, taskAttemptId);
+
+    } else {
+      // it's a job
+      taskId = "";
+      taskAttemptId = "";
+      taskAttemptDir = null;
+      progressable = null;
+      name = String.format(InternalConstants.NAME_FORMAT_JOB_ATTEMPT, jobAttemptId);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "ManifestCommitterConfig{" +
+        "name=" + name +
+        ", destinationDir=" + destinationDir +
+        ", role='" + role + '\'' +
+        ", taskAttemptDir=" + taskAttemptDir +
+        ", createJobMarker=" + createJobMarker +
+        ", jobUniqueId='" + jobUniqueId + '\'' +
+        ", jobUniqueIdSource='" + jobUniqueIdSource + '\'' +
+        ", jobAttemptNumber=" + jobAttemptNumber +
+        ", jobAttemptId='" + jobAttemptId + '\'' +
+        ", taskId='" + taskId + '\'' +
+        ", taskAttemptId='" + taskAttemptId + '\'' +
+        '}';
+  }
+
+  /**
+   * Get the destination filesystem.
+   * @return destination FS.
+   * @throws IOException Problems binding to the destination FS.
+   */
+  FileSystem getDestinationFileSystem() throws IOException {
+    return FileSystem.get(destinationDir.toUri(), conf);
+  }
+
+  /**
+   * Create the stage config from the committer
+   * configuration.
+   * This does not bind the store operations
+   * or processors.
+   * @return a stage config with configuration options passed in.
+   */
+  StageConfig createStageConfig() {
+    StageConfig stageConfig = new StageConfig();
+    stageConfig
+        .withIOStatistics(iostatistics)
+        .withJobAttemptNumber(jobAttemptNumber)
+        .withJobDirectories(dirs)
+        .withJobId(jobUniqueId)
+        .withJobIdSource(jobUniqueIdSource)
+        .withName(name)
+        .withProgressable(progressable)
+        .withStageEventCallbacks(stageEventCallbacks)
+        .withTaskAttemptDir(taskAttemptDir)
+        .withTaskAttemptId(taskAttemptId)
+        .withTaskId(taskId)
+        .withDeleteTargetPaths(deleteTargetPaths);
+
+    return stageConfig;
+  }
+
+  public Path getDestinationDir() {
+    return destinationDir;
+  }
+
+  public String getRole() {
+    return role;
+  }
+
+  public Path getTaskAttemptDir() {
+    return taskAttemptDir;
+  }
+
+  public Path getJobAttemptDir() {
+    return dirs.getJobAttemptDir();
+  }
+
+  public Path getTaskManifestDir() {
+    return dirs.getTaskManifestDir();
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  public JobContext getJobContext() {
+    return jobContext;
+  }
+
+  public boolean getCreateJobMarker() {
+    return createJobMarker;
+  }
+
+  public String getJobAttemptId() {
+    return jobAttemptId;
+  }
+
+  public String getTaskAttemptId() {
+    return taskAttemptId;
+  }
+
+  public String getTaskId() {
+    return taskId;
+  }
+
+  public String getJobUniqueId() {
+    return jobUniqueId;
+  }
+
+  public boolean getValidateOutput() {
+    return validateOutput;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public IOStatisticsStore getIOStatistics() {
+    return iostatistics;
+  }
+
+  /**
+   * Create a new submitter task pool from the
+   * {@link ManifestCommitterConstants#OPT_IO_PROCESSORS}
+   * settings.
+   * @return a new thread pool.
+   */
+  public CloseableTaskPoolSubmitter createSubmitter() {
+    return createSubmitter(
+        OPT_IO_PROCESSORS, OPT_IO_PROCESSORS_DEFAULT);
+  }
+
+  /**
+   * Create a new submitter task pool.
+   * @param key config key with pool size.
+   * @param defVal default value.
+   * @return a new task pool.
+   */
+  public CloseableTaskPoolSubmitter createSubmitter(String key, int defVal) {
+    int numThreads = conf.getInt(key, defVal);
+    if (numThreads <= 0) {
+      // ignore the setting if it is too invalid.
+      numThreads = defVal;
+    }
+    return createCloseableTaskSubmitter(numThreads, getJobAttemptId());
+  }
+
+  /**
+   * Create a new submitter task pool.
+   *
+   * @param numThreads thread count.
+   * @param jobAttemptId job ID
+   * @return a new task pool.
+   */
+  public static CloseableTaskPoolSubmitter createCloseableTaskSubmitter(
+      final int numThreads,
+      final String jobAttemptId) {
+    return new CloseableTaskPoolSubmitter(
+        HadoopExecutors.newFixedThreadPool(numThreads,
+            new ThreadFactoryBuilder()
+                .setDaemon(true)
+                .setNameFormat("manifest-committer-" + jobAttemptId + "-%d")
+                .build()));
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java
new file mode 100644
index 0000000000000..fd7b3d816c103
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+
+/**
+ * Public constants for the manifest committer.
+ * This includes all configuration options and their default values.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class ManifestCommitterConstants {
+
+  /**
+   * Suffix to use in manifest files in the manifest subdir.
+   * Value: {@value}.
+   */
+  public static final String MANIFEST_SUFFIX = "-manifest.json";
+
+  /**
+   * Prefix for summary files in the report dir. Call
+   */
+  public static final String SUMMARY_FILENAME_PREFIX = "summary-";
+
+  /**
+   * Format string used to build a summary file from a Job ID.
+   */
+  public static final String SUMMARY_FILENAME_FORMAT =
+      SUMMARY_FILENAME_PREFIX + "%s.json";
+
+  /**
+   * Suffix to use for temp files before renaming them.
+   * Value: {@value}.
+   */
+  public static final String TMP_SUFFIX = ".tmp";
+
+  /**
+   * Initial number of all app attempts.
+   * This is fixed in YARN; for Spark jobs the
+   * same number "0" is used.
+   */
+  public static final int INITIAL_APP_ATTEMPT_ID = 0;
+
+  /**
+   * Format string for building a job dir.
+   * Value: {@value}.
+   */
+  public static final String JOB_DIR_FORMAT_STR = "%s";
+
+  /**
+   * Format string for building a job attempt dir.
+   * This uses the job attempt number so previous versions
+   * can be found trivially.
+   * Value: {@value}.
+   */
+  public static final String JOB_ATTEMPT_DIR_FORMAT_STR = "%02d";
+
+  /**
+   * Name of directory under job attempt dir for manifests.
+   */
+  public static final String JOB_TASK_MANIFEST_SUBDIR = "manifests";
+
+  /**
+   * Name of directory under job attempt dir for task attempts.
+   */
+  public static final String JOB_TASK_ATTEMPT_SUBDIR = "tasks";
+
+
+  /**
+   * Committer classname as recorded in the committer _SUCCESS file.
+   */
+  public static final String MANIFEST_COMMITTER_CLASSNAME =
+      ManifestCommitter.class.getName();
+
+  /**
+   * Marker file to create on success: {@value}.
+   */
+  public static final String SUCCESS_MARKER = "_SUCCESS";
+
+  /** Default job marker option: {@value}. */
+  public static final boolean DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER = true;
+
+  /**
+   * The limit to the number of committed objects tracked during
+   * job commits and saved to the _SUCCESS file.
+   * Value: {@value}.
+   */
+  public static final int SUCCESS_MARKER_FILE_LIMIT = 100;
+
+  /**
+   * The UUID for jobs: {@value}.
+   * This was historically created in Spark 1.x's SQL queries,
+   * but "went away".
+   * It has been restored in recent spark releases.
+   * If found: it is used instead of the MR job attempt ID.
+   */
+  public static final String SPARK_WRITE_UUID = "spark.sql.sources.writeJobUUID";
+
+  /**
+   * String to use as source of the job ID.
+   * This SHOULD be kept in sync with that of
+   * {@code AbstractS3ACommitter.JobUUIDSource}.
+   * Value: {@value}.
+   */
+  public static final String JOB_ID_SOURCE_MAPREDUCE = "JobID";
+
+  /**
+   * Prefix to use for config options: {@value}.
+   */
+  public static final String OPT_PREFIX = "mapreduce.manifest.committer.";
+
+  /**
+   * Should dir cleanup do parallel deletion of task attempt dirs
+   * before trying to delete the toplevel dirs.
+   * For GCS this may deliver speedup, while on ABFS it may avoid
+   * timeouts in certain deployments.
+   * Value: {@value}.
+   */
+  public static final String OPT_CLEANUP_PARALLEL_DELETE =
+      OPT_PREFIX + "cleanup.parallel.delete";
+
+  /**
+   * Default value:  {@value}.
+   */
+  public static final boolean OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT = true;
+
+  /**
+   * Threads to use for IO.
+   */
+  public static final String OPT_IO_PROCESSORS = OPT_PREFIX + "io.threads";
+
+  /**
+   * Default value:  {@value}.
+   */
+  public static final int OPT_IO_PROCESSORS_DEFAULT = 64;
+
+  /**
+   * Directory for saving job summary reports.
+   * These are the _SUCCESS files, but are saved even on
+   * job failures.
+   * Value: {@value}.
+   */
+  public static final String OPT_SUMMARY_REPORT_DIR =
+      OPT_PREFIX + "summary.report.directory";
+
+  /**
+   * Directory for moving manifests under for diagnostics.
+   * Value: {@value}.
+   */
+  public static final String OPT_DIAGNOSTICS_MANIFEST_DIR =
+      OPT_PREFIX + "diagnostics.manifest.directory";
+
+  /**
+   * Should the output be validated?
+   * This will check expected vs actual file lengths, and,
+   * if etags can be obtained, etags.
+   * Value: {@value}.
+   */
+  public static final String OPT_VALIDATE_OUTPUT = OPT_PREFIX + "validate.output";
+
+  /**
+   * Default value: {@value}.
+   */
+  public static final boolean OPT_VALIDATE_OUTPUT_DEFAULT = false;
+
+  /**
+   * Should job commit delete for files/directories at the targets
+   * of renames, and, if found, deleting them?
+   *
+   * This is part of the effective behavior of the FileOutputCommitter,
+   * however it adds an extra delete call per file being committed.
+   *
+   * If a job is writing to a directory which has only just been created
+   * or were unique filenames are being used, there is no need to perform
+   * this preparation.
+   * The recognition of newly created dirs is automatic.
+   *
+   * Value: {@value}.
+   */
+  public static final String OPT_DELETE_TARGET_FILES =
+      OPT_PREFIX + "delete.target.files";
+
+  /**
+   * Default value: {@value}.
+   */
+  public static final boolean OPT_DELETE_TARGET_FILES_DEFAULT = false;
+
+  /**
+   * Name of the factory: {@value}.
+   */
+  public static final String MANIFEST_COMMITTER_FACTORY =
+      ManifestCommitterFactory.class.getName();
+
+  /**
+   * Classname of the store operations; filesystems and tests
+   * may override.
+   * Value: {@value}.
+   */
+  public static final String OPT_STORE_OPERATIONS_CLASS = OPT_PREFIX + "store.operations.classname";
+
+  /**
+   * Default classname of the store operations.
+   * Value: {@value}.
+   */
+  public static final String STORE_OPERATIONS_CLASS_DEFAULT =
+      ManifestStoreOperationsThroughFileSystem.class.getName();
+
+  /**
+   * Stage attribute in audit context: {@value}.
+   */
+  public static final String CONTEXT_ATTR_STAGE = "st";
+
+  /**
+   * Task ID attribute in audit context: {@value}.
+   */
+  public static final String CONTEXT_ATTR_TASK_ATTEMPT_ID = "ta";
+
+  /**
+   * Stream Capabilities probe for spark dynamic partitioning compatibility.
+   */
+  public static final String CAPABILITY_DYNAMIC_PARTITIONING =
+      "mapreduce.job.committer.dynamic.partitioning";
+
+  private ManifestCommitterConstants() {
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java
new file mode 100644
index 0000000000000..b07182cf999ed
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.UNSUPPORTED_FS_SCHEMAS;
+
+/**
+ * This is the committer factory to register as the source of committers
+ * for the job/filesystem schema.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ManifestCommitterFactory extends PathOutputCommitterFactory {
+
+  /**
+   * Name of this factory.
+   */
+  public static final String NAME = ManifestCommitterFactory.class.getName();
+
+  @Override
+  public ManifestCommitter createOutputCommitter(final Path outputPath,
+      final TaskAttemptContext context) throws IOException {
+    // safety check. S3A does not support this, so fail fast.
+    final String scheme = outputPath.toUri().getScheme();
+    if (UNSUPPORTED_FS_SCHEMAS.contains(scheme)) {
+      throw new PathIOException(outputPath.toString(),
+          "This committer does not work with the filesystem of type " + scheme);
+    }
+    return new ManifestCommitter(outputPath, context);
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java
new file mode 100644
index 0000000000000..243fd6087328d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
+
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_RENAME;
+
+/**
+ * Statistic names for committers.
+ * Please keep in sync with org.apache.hadoop.fs.s3a.Statistic
+ * so that S3A and manifest committers are in sync.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class ManifestCommitterStatisticNames {
+
+
+  /** Amount of data committed: {@value}. */
+  public static final String COMMITTER_BYTES_COMMITTED_COUNT =
+      "committer_bytes_committed";
+
+  /** Duration Tracking of time to commit an entire job: {@value}. */
+  public static final String COMMITTER_COMMIT_JOB =
+      "committer_commit_job";
+
+  /** Number of files committed: {@value}. */
+  public static final String COMMITTER_FILES_COMMITTED_COUNT =
+      "committer_files_committed";
+
+  /** "Count of successful tasks:: {@value}. */
+  public static final String COMMITTER_TASKS_COMPLETED_COUNT =
+      "committer_tasks_completed";
+
+  /** Count of failed tasks: {@value}. */
+  public static final String COMMITTER_TASKS_FAILED_COUNT =
+      "committer_tasks_failed";
+
+  /** Count of commits aborted: {@value}. */
+  public static final String COMMITTER_COMMITS_ABORTED_COUNT =
+      "committer_commits_aborted";
+
+  /** Count of commits reverted: {@value}. */
+  public static final String COMMITTER_COMMITS_REVERTED_COUNT =
+      "committer_commits_reverted";
+
+  /** Count of commits failed: {@value}. */
+  public static final String COMMITTER_COMMITS_FAILED =
+      "committer_commits" + StoreStatisticNames.SUFFIX_FAILURES;
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_FILE_COUNT_MEAN =
+      "committer_task_file_count";
+
+  /**
+   * File Size.
+   */
+  public static final String COMMITTER_FILE_SIZE_MEAN =
+      "committer_task_file_size";
+
+  /**
+   * What is a task attempt's directory count.
+   */
+  public static final String COMMITTER_TASK_DIRECTORY_COUNT_MEAN =
+      "committer_task_directory_count";
+
+  /**
+   * What is the depth of a task attempt's directory tree.
+   */
+  public static final String COMMITTER_TASK_DIRECTORY_DEPTH_MEAN =
+      "committer_task_directory_depth";
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_TASK_FILE_COUNT_MEAN =
+      "committer_task_file_count";
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_TASK_FILE_SIZE_MEAN =
+      "committer_task_file_size";
+
+  /**
+   * The size of manifest files. This will be a MeanStatistic.
+   * Useful to highlight whether this size is too large and
+   * effort to optimise for file IO and memory consumption
+   * justified.
+   */
+  public static final String COMMITTER_TASK_MANIFEST_FILE_SIZE =
+      "committer_task_manifest_file_size";
+
+  /**
+   * A file renamed during the commit operation {@value}.
+   */
+  public static final String OP_COMMIT_FILE_RENAME =
+      "commit_file_rename";
+
+  /**
+   * A file rename during the commit operation was recovered from a failure {@value}.
+   */
+  public static final String OP_COMMIT_FILE_RENAME_RECOVERED =
+      "commit_file_rename_recovered";
+
+  /** Directory creation {@value}. */
+  public static final String OP_CREATE_DIRECTORIES = "op_create_directories";
+
+  /** Creating a single directory {@value}. */
+  public static final String OP_CREATE_ONE_DIRECTORY =
+      "op_create_one_directory";
+
+  /**
+   * A file in the destination directory tree has been deleted.
+   *  {@value}.
+   */
+  public static final String OP_DELETE_FILE_UNDER_DESTINATION =
+      "op_delete_file_under_destination";
+
+  /** Directory scan {@value}. */
+  public static final String OP_DIRECTORY_SCAN = "op_directory_scan";
+
+  /**
+   * Overall job commit {@value}.
+   */
+  public static final String OP_STAGE_JOB_COMMIT = COMMITTER_COMMIT_JOB;
+
+  /** {@value}. */
+  public static final String OP_LOAD_ALL_MANIFESTS = "op_load_all_manifests";
+
+  /**
+   * Load a task manifest: {@value}.
+   */
+  public static final String OP_LOAD_MANIFEST = "op_load_manifest";
+
+  /**
+   * mkdir failure statistic: {@value}.
+   * This is incremented whenever mkdir() returns false, e.g because
+   * there was a file at the end of the path.
+   */
+  public static final String OP_MKDIRS_RETURNED_FALSE = "op_mkdir_returned_false";
+
+  /**
+   * msync statistic: {@value}.
+   * This should be the same as StoreStatisticNames.OP_MSYNC; it's
+   * duplicated just to isolate this committer into a single JAR
+   * for ease of testing.
+   */
+  public static final String OP_MSYNC = "op_msync";
+
+  /**
+   * Preparing ancestor dirs: {@value}.
+   * Probing for paths being files, and if true: deleting them.
+   */
+  public static final String OP_PREPARE_DIR_ANCESTORS = "op_prepare_dir_ancestors";
+
+  /** Rename a dir: {@value}. */
+  public static final String OP_RENAME_DIR = OP_RENAME;
+
+
+  /** Rename a file: {@value}. */
+  public static final String OP_RENAME_FILE = OP_RENAME;
+
+  /**
+   * Save a task manifest: {@value}.
+   */
+  public static final String OP_SAVE_TASK_MANIFEST =
+      "task_stage_save_task_manifest";
+
+  /**
+   * Task abort: {@value}.
+   */
+  public static final String OP_STAGE_TASK_ABORT_TASK
+      = "task_stage_abort_task";
+
+  /**
+   * Job abort: {@value}.
+   */
+  public static final String OP_STAGE_JOB_ABORT = "job_stage_abort";
+
+  /**
+   * Job cleanup: {@value}.
+   */
+  public static final String OP_STAGE_JOB_CLEANUP = "job_stage_cleanup";
+
+  /**
+   * Prepare Directories Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_CREATE_TARGET_DIRS =
+      "job_stage_create_target_dirs";
+
+  /**
+   * Load Manifest Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_LOAD_MANIFESTS =
+      "job_stage_load_manifests";
+
+  /**
+   * Rename files stage duration: {@value}.
+   */
+  public static final String OP_STAGE_JOB_RENAME_FILES =
+      "job_stage_rename_files";
+
+
+  /**
+   * Job Setup Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_SETUP = "job_stage_setup";
+
+  /**
+   * Job saving _SUCCESS marker Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_SAVE_SUCCESS =
+      "job_stage_save_success_marker";
+
+  /**
+   * Output Validation (within job commit) Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_VALIDATE_OUTPUT =
+      "job_stage_optional_validate_output";
+
+  /**
+   * Task saving manifest file Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_SAVE_MANIFEST =
+      "task_stage_save_manifest";
+
+  /**
+   * Task Setup Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_SETUP = "task_stage_setup";
+
+  /**
+   * Task Commit Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_COMMIT = "task_stage_commit";
+
+  /** Task Scan directory Stage: {@value}. */
+  public static final String OP_STAGE_TASK_SCAN_DIRECTORY
+      = "task_stage_scan_directory";
+
+  private ManifestCommitterStatisticNames() {
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java
new file mode 100644
index 0000000000000..bf67281785c07
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Class for single/multiple commit data structures.
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class AbstractManifestData<T extends AbstractManifestData>
+    implements Serializable, IOStatisticsSource {
+
+
+  /**
+   * Convert a path to a string which can be included in the JSON.
+   * @param path path
+   * @return a string value, or, if path==null, null.
+   */
+  public static String marshallPath(@Nullable Path path) {
+    return path != null
+        ? path.toUri().toString()
+        : null;
+  }
+
+  /**
+   * Convert a string path to Path type, by way of a URI.
+   * @param path path as a string
+   * @return path value
+   * @throws RuntimeException marshalling failure.
+   */
+  public static Path unmarshallPath(String path) {
+    try {
+      return new Path(new URI(requireNonNull(path, "No path")));
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(
+          "Failed to parse \"" + path + "\" : " + e,
+          e);
+    }
+  }
+
+  /**
+   * Validate the data: those fields which must be non empty, must be set.
+   * @return the validated instance.
+   * @throws IOException if the data is invalid
+   */
+  public abstract T validate() throws IOException;
+
+  /**
+   * Serialize to JSON and then to a byte array, after performing a
+   * preflight validation of the data to be saved.
+   * @return the data in a persistable form.
+   * @throws IOException serialization problem or validation failure.
+   */
+  public abstract byte[] toBytes() throws IOException;
+
+  /**
+   * Save to a hadoop filesystem.
+   * @param fs filesystem
+   * @param path path
+   * @param overwrite should any existing file be overwritten
+   * @throws IOException IO exception
+   */
+  public abstract void save(FileSystem fs, Path path, boolean overwrite)
+      throws IOException;
+
+  /**
+   * Get a (usually shared) JSON serializer.
+   * @return a serializer. Call
+   */
+  public abstract JsonSerialization<T> createSerializer();
+
+  /**
+   * Verify that all instances in a collection are of the given class.
+   * @param it iterator
+   * @param classname classname to require
+   * @throws IOException on a failure
+   */
+  void validateCollectionClass(Iterable it, Class classname)
+      throws IOException {
+    for (Object o : it) {
+      verify(o.getClass().equals(classname),
+          "Collection element is not a %s: %s", classname, o.getClass());
+    }
+  }
+
+  /**
+   * Verify that a condition holds.
+   * @param expression expression which must be true
+   * @param message message to raise on a failure
+   * @param args arguments for the message formatting
+   * @throws IOException on a failure
+   */
+
+  static void verify(boolean expression,
+      String message,
+      Object... args) throws IOException {
+    if (!expression) {
+      throw new IOException(String.format(message, args));
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
similarity index 60%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java
rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
index c14644022463a..cb673383c58be 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreCapabilities.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -16,28 +16,29 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.s3guard;
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * All the capability constants used for the
- * {@link MetadataStore} implementations.
+ * Diagnostic keys in the manifests.
  */
 @InterfaceAudience.Public
-@InterfaceStability.Evolving
-public final class MetadataStoreCapabilities {
-
-  private MetadataStoreCapabilities(){
-  }
-
+@InterfaceStability.Unstable
+public final class DiagnosticKeys {
   /**
-   *  This capability tells if the metadata store supports authoritative
-   *  directories. Used in {@link MetadataStore#getDiagnostics()} as a key
-   *  for this capability. The value can be boolean true or false.
-   *  If the Map.get() returns null for this key, that is interpreted as false.
+   * Attribute added to diagnostics in _SUCCESS file.
    */
-  public static final String PERSISTS_AUTHORITATIVE_BIT =
-      "persist.authoritative.bit";
+  public static final String PRINCIPAL = "principal";
+  public static final String STAGE = "stage";
+  public static final String EXCEPTION = "exception";
+  public static final String STACKTRACE = "stacktrace";
+
+
+  /** Directory where manifests were renamed: {@value}. */
+  public static final String MANIFESTS = "manifests";
+
+  private DiagnosticKeys() {
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java
new file mode 100644
index 0000000000000..15e8cac779592
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.marshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.unmarshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.verify;
+
+/**
+ * A directory entry in the task manifest.
+ * Uses shorter field names for smaller files.
+ * Hash and equals are on dir name only; there's no real expectation
+ * that those operations are needed.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class DirEntry implements Serializable {
+
+  private static final long serialVersionUID = 5658520530209859765L;
+
+  /**
+   * Destination directory.
+   */
+  @JsonProperty("d")
+  private String dir;
+
+  /**
+   * Type of dest entry as found when probed for in task commit.
+   */
+  @JsonProperty("t")
+  private int type;
+
+  /**
+   * Level in the treewalk.
+   */
+  @JsonProperty("l")
+  private int level;
+
+  /**
+   * Constructor only for use by jackson.
+   * Do Not Delete.
+   */
+  private DirEntry() {
+  }
+
+  /**
+   * Construct an entry.
+   *
+   * @param dir destination path.
+   * @param type type of dest entry
+   * @param level Level in the treewalk.
+   *
+   */
+  public DirEntry(
+      final String dir,
+      final int type,
+      final int level) {
+    this.dir = requireNonNull(dir);
+    this.type = type;
+    this.level = level;
+  }
+
+  /**
+   * Construct an entry.
+   *
+   * @param dir destination path.
+   * @param type type of dest entry
+   * @param level Level in the treewalk.
+   *
+   */
+  public DirEntry(
+      final Path dir,
+      final int type,
+      final int level) {
+    this(marshallPath(dir), type, level);
+  }
+
+  public void setDir(final String dir) {
+    this.dir = dir;
+  }
+
+  public String getDir() {
+    return dir;
+  }
+
+  @JsonIgnore
+  public Path getDestPath() {
+    return unmarshallPath(dir);
+  }
+
+  public int getType() {
+    return type;
+  }
+
+  public void setType(final int type) {
+    this.type = type;
+  }
+
+  public void setLevel(final int level) {
+    this.level = level;
+  }
+
+  public int getLevel() {
+    return level;
+  }
+
+  @JsonIgnore
+  public EntryStatus getStatus() {
+    return EntryStatus.toEntryStatus(type);
+  }
+
+  @JsonIgnore
+  public void setStatus(EntryStatus status) {
+    setType(status.ordinal());
+  }
+  public void validate() throws IOException {
+    final String s = toString();
+    verify(dir != null && dir.length() > 0,
+        "destination path is missing from " + s);
+    verify(type >= 0,
+        "Invalid type in " + s);
+    verify(level >= 0,
+        "Invalid level in " + s);
+  }
+
+  @Override
+  public String toString() {
+    return "DirEntry{" +
+        "dir='" + dir + '\'' +
+        ", type=" + type +
+        ", level=" + level +
+        '}';
+  }
+
+  @Override
+  public boolean equals(final Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    DirEntry dirEntry = (DirEntry) o;
+    return dir.equals(dirEntry.dir);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(dir);
+  }
+
+  /**
+   * A directory entry.
+   * @param dest destination path.
+   * @param type type
+   * @param level Level in the treewalk.
+   * @return an entry
+   */
+  public static DirEntry dirEntry(Path dest, int type, int level) {
+    return new DirEntry(dest, type, level);
+  }
+
+  /**
+   * A directory entry.
+   * @param dest destination path.
+   * @param type type
+   * @param level Level in the treewalk.
+   * @return an entry
+   */
+  public static DirEntry dirEntry(Path dest, EntryStatus type, int level) {
+    return dirEntry(dest, type.ordinal(), level);
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java
new file mode 100644
index 0000000000000..73ac5d7f1b179
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * Status of a file or dir entry, designed to be marshalled as
+ * an integer -the ordinal value of the enum is the
+ * wire value.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum EntryStatus {
+
+  unknown,
+  not_found,
+  file,
+  dir,
+  created_dir;
+
+  /**
+   * Go from a marshalled type to a status value.
+   * Any out of range value is converted to unknown.
+   * @param type type
+   * @return the status value.
+   */
+  public static EntryStatus toEntryStatus(int type) {
+    switch (type) {
+    case 1:
+      return not_found;
+    case 2:
+      return file;
+    case 3:
+      return dir;
+    case 4:
+      return created_dir;
+    case 0:
+    default:
+      return unknown;
+    }
+  }
+
+
+  /**
+   * Go from the result of a getFileStatus call or
+   * listing entry to a status.
+   * A null argument is mapped to {@link #not_found}
+   * @param st file status
+   * @return the status enum.
+   */
+  public static EntryStatus toEntryStatus(@Nullable FileStatus st) {
+
+    if (st == null) {
+      return not_found;
+    }
+    if (st.isDirectory()) {
+      return dir;
+    }
+    if (st.isFile()) {
+      return file;
+    }
+    return unknown;
+  }
+
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java
new file mode 100644
index 0000000000000..bc6cdd94b3f26
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.marshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.unmarshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.verify;
+
+/**
+ * A File entry in the task manifest.
+ * Uses shorter field names for smaller files.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public final class FileEntry implements Serializable {
+
+  private static final long serialVersionUID = -550288489009777867L;
+
+  @JsonProperty("s")
+  private String source;
+
+  @JsonProperty("d")
+  private String dest;
+
+  @JsonProperty("z")
+  private long size;
+
+  /**
+   * Etag value if we can extract this.
+   */
+  @JsonProperty("e")
+  private String etag;
+
+  /**
+   * Constructor only for use by jackson.
+   * Do Not Delete.
+   */
+  private FileEntry() {
+  }
+
+  /**
+   * Construct an entry.
+   * @param source source path.
+   * @param dest destination path.
+   * @param size file size.
+   * @param etag optional etag
+   */
+  public FileEntry(
+      final String source,
+      final String dest,
+      final long size,
+      final String etag) {
+    this.source = source;
+    this.dest = dest;
+    this.size = size;
+    this.etag = etag;
+  }
+
+
+  /**
+   * Construct an entry.
+   * @param source source path.
+   * @param dest destination path.
+   * @param size file size.
+   * @param etag optional etag
+   */
+  public FileEntry(
+      final Path source,
+      final Path dest,
+      final long size,
+      final String etag) {
+    this(marshallPath(source), marshallPath(dest), size, etag);
+  }
+
+
+  public void setSource(final String source) {
+    this.source = source;
+  }
+
+  public String getSource() {
+    return source;
+  }
+
+  @JsonIgnore
+  public Path getSourcePath() {
+    return unmarshallPath(source);
+  }
+
+  public void setDest(final String dest) {
+    this.dest = dest;
+  }
+
+  public String getDest() {
+    return dest;
+  }
+
+  @JsonIgnore
+  public Path getDestPath() {
+    return unmarshallPath(dest);
+  }
+
+  public long getSize() {
+    return size;
+  }
+
+  public void setSize(final long size) {
+    this.size = size;
+  }
+
+  public String getEtag() {
+    return etag;
+  }
+
+  public void setEtag(final String etag) {
+    this.etag = etag;
+  }
+
+  public void validate() throws IOException {
+    final String s = toString();
+    verify(source != null && source.length() > 0,
+        "Source is missing from " + s);
+    verify(dest != null && dest.length() > 0,
+        "Source is missing from " + s);
+    verify(size >= 0,
+        "Invalid size in " + s);
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder(
+        "FileOrDirEntry{");
+    sb.append("source='").append(source).append('\'');
+    sb.append(", dest='").append(dest).append('\'');
+    sb.append(", size=").append(size);
+    sb.append(", etag='").append(etag).append('\'');
+    sb.append('}');
+    return sb.toString();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    FileEntry that = (FileEntry) o;
+    return size == that.size && source.equals(that.source) && dest.equals(
+        that.dest) &&
+        Objects.equals(etag, that.etag);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(source, dest);
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java
new file mode 100644
index 0000000000000..c95ec7b11be05
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+
+/**
+ * Tool to print a manifest.
+ */
+public class ManifestPrinter extends Configured implements Tool {
+
+  private static final String USAGE = "ManifestPrinter <success-file>";
+
+  /**
+   * Output for printing.
+   */
+  private final PrintStream out;
+
+  /**
+   * Print to System.out.
+   */
+  public ManifestPrinter() {
+    this(null, System.out);
+  }
+
+  /**
+   * Print to the supplied stream.
+   * @param conf configuration
+   * @param out output
+   */
+  public ManifestPrinter(Configuration conf, PrintStream out) {
+    super(conf);
+    this.out = out;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length != 1) {
+      printUsage();
+      return -1;
+    }
+    Path path = new Path(args[0]);
+    loadAndPrintManifest(path.getFileSystem(getConf()), path);
+    return 0;
+  }
+
+  /**
+   * Load and print a manifest.
+   * @param fs filesystem.
+   * @param path path
+   * @throws IOException failure to load
+   * @return the manifest
+   */
+  public ManifestSuccessData loadAndPrintManifest(FileSystem fs, Path path)
+      throws IOException {
+    // load the manifest
+    println("Manifest file: %s", path);
+    final ManifestSuccessData success = ManifestSuccessData.load(fs, path);
+
+    printManifest(success);
+    return success;
+  }
+
+  private void printManifest(ManifestSuccessData success) {
+    field("succeeded", success.getSuccess());
+    field("created", success.getDate());
+    field("committer", success.getCommitter());
+    field("hostname", success.getHostname());
+    field("description", success.getDescription());
+    field("jobId", success.getJobId());
+    field("jobIdSource", success.getJobIdSource());
+    field("stage", success.getStage());
+    println("Diagnostics\n%s",
+        success.dumpDiagnostics("  ", " = ", "\n"));
+    println("Statistics:\n%s",
+        ioStatisticsToPrettyString(success.getIOStatistics()));
+    out.flush();
+  }
+
+  private void printUsage() {
+    println(USAGE);
+  }
+
+  /**
+   * Print a line to the output stream.
+   * @param format format string
+   * @param args arguments.
+   */
+  private void println(String format, Object... args) {
+    out.format(format, args);
+    out.println();
+  }
+
+  /**
+   * Print a field, if non-null.
+   * @param name field name.
+   * @param value value.
+   */
+  private void field(String name, Object value) {
+    if (value != null) {
+      println("%s: %s", name, value);
+    }
+  }
+
+  /**
+   * Entry point.
+   */
+  public static void main(String[] argv) throws Exception {
+
+    try {
+      int res = ToolRunner.run(new ManifestPrinter(), argv);
+      System.exit(res);
+    } catch (ExitUtil.ExitException e) {
+      ExitUtil.terminate(e);
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java
new file mode 100644
index 0000000000000..73e73075ab3dc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSupport;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Summary data saved into a {@code _SUCCESS} marker file.
+ *
+ * This is a copy of the S3A committer success data format, with
+ * a goal of being/remaining compatible.
+ * This makes it easier for tests in downstream modules to
+ * be able to parse the success files from any of the committers.
+ *
+ * This should be considered public; it is based on the S3A
+ * format, which has proven stable over time.
+ *
+ * The JSON format SHOULD be considered public and evolving
+ * with compatibility across versions.
+ *
+ * All the Java serialization data is different and may change
+ * across versions with no stability guarantees other than
+ * "manifest summaries MAY be serialized between processes with
+ * the exact same version of this binary on their classpaths."
+ * That is sufficient for testing in Spark.
+ *
+ * To aid with Java serialization, the maps and lists are
+ * exclusively those which serialize well.
+ * IOStatisticsSnapshot has a lot of complexity in marshalling
+ * there; this class doesn't worry about concurrent access
+ * so is simpler.
+ *
+ */
+@SuppressWarnings({"unused", "CollectionDeclaredAsConcreteClass"})
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ManifestSuccessData
+    extends AbstractManifestData<ManifestSuccessData> {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ManifestSuccessData.class);
+
+  /**
+   * Supported version value: {@value}.
+   * If this is changed the value of {@link #serialVersionUID} will change,
+   * to avoid deserialization problems.
+   */
+  public static final int VERSION = 1;
+
+  /**
+   * Serialization ID: {@value}.
+   */
+  private static final long serialVersionUID = 4755993198698104084L + VERSION;
+
+  /**
+   * Name to include in persisted data, so as to differentiate from
+   * any other manifests: {@value}.
+   */
+  public static final String NAME
+      = "org.apache.hadoop.fs.s3a.commit.files.SuccessData/" + VERSION;
+
+  /**
+   * Name of file; includes version marker.
+   */
+  private String name;
+
+  /** Timestamp of creation. */
+  private long timestamp;
+
+  /**
+   * Did this succeed?
+   * It is implicitly true in a _SUCCESS file, but if the file
+   * is also saved to a log dir, then it depends on the outcome
+   */
+  private boolean success = true;
+
+  /** Timestamp as date string; no expectation of parseability. */
+  private String date;
+
+  /**
+   * Host which created the file (implicitly: committed the work).
+   */
+  private String hostname;
+
+  /**
+   * Committer name.
+   */
+  private String committer;
+
+  /**
+   * Description text.
+   */
+  private String description;
+
+  /** Job ID, if known. */
+  private String jobId = "";
+
+  /**
+   * Source of the job ID.
+   */
+  private String jobIdSource = "";
+
+  /**
+   * Metrics.
+   * Uses a treemap for serialization.
+   */
+  private TreeMap<String, Long> metrics = new TreeMap<>();
+
+  /**
+   * Diagnostics information.
+   * Uses a treemap for serialization.
+   */
+  private TreeMap<String, String> diagnostics = new TreeMap<>();
+
+  /**
+   * Filenames in the commit.
+   */
+  private ArrayList<String> filenames = new ArrayList<>(0);
+
+  /**
+   * IOStatistics.
+   */
+  @JsonProperty("iostatistics")
+  private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot();
+
+  /**
+   * State (committed, aborted).
+   */
+  private String state;
+
+  /**
+   * Stage: last stage executed.
+   */
+  private String stage;
+
+  @Override
+  public ManifestSuccessData validate() throws IOException {
+    verify(name != null,
+        "Incompatible file format: no 'name' field");
+    verify(NAME.equals(name),
+        "Incompatible file format: " + name);
+    return this;
+  }
+
+  @Override
+  public JsonSerialization<ManifestSuccessData> createSerializer() {
+    return serializer();
+  }
+
+  @Override
+  public byte[] toBytes() throws IOException {
+    return serializer().toBytes(this);
+  }
+
+  /**
+   * To JSON.
+   * @return json string value.
+   * @throws IOException failure
+   */
+  public String toJson() throws IOException {
+    return serializer().toJson(this);
+  }
+
+  @Override
+  public void save(FileSystem fs, Path path, boolean overwrite)
+      throws IOException {
+    // always set the name field before being saved.
+    name = NAME;
+    serializer().save(fs, path, this, overwrite);
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder(
+        "ManifestSuccessData{");
+    sb.append("committer='").append(committer).append('\'');
+    sb.append(", hostname='").append(hostname).append('\'');
+    sb.append(", description='").append(description).append('\'');
+    sb.append(", date='").append(date).append('\'');
+    sb.append(", filenames=[").append(
+        StringUtils.join(filenames, ", "))
+        .append("]");
+    sb.append('}');
+    return sb.toString();
+  }
+
+  /**
+   * Dump the metrics (if any) to a string.
+   * The metrics are sorted for ease of viewing.
+   * @param prefix prefix before every entry
+   * @param middle string between key and value
+   * @param suffix suffix to each entry
+   * @return the dumped string
+   */
+  public String dumpMetrics(String prefix, String middle, String suffix) {
+    return joinMap(metrics, prefix, middle, suffix);
+  }
+
+  /**
+   * Dump the diagnostics (if any) to a string.
+   * @param prefix prefix before every entry
+   * @param middle string between key and value
+   * @param suffix suffix to each entry
+   * @return the dumped string
+   */
+  public String dumpDiagnostics(String prefix, String middle, String suffix) {
+    return joinMap(diagnostics, prefix, middle, suffix);
+  }
+
+  /**
+   * Join any map of string to value into a string, sorting the keys first.
+   * @param map map to join
+   * @param prefix prefix before every entry
+   * @param middle string between key and value
+   * @param suffix suffix to each entry
+   * @return a string for reporting.
+   */
+  protected static String joinMap(Map<String, ?> map,
+      String prefix,
+      String middle, String suffix) {
+    if (map == null) {
+      return "";
+    }
+    List<String> list = new ArrayList<>(map.keySet());
+    Collections.sort(list);
+    StringBuilder sb = new StringBuilder(list.size() * 32);
+    for (String k : list) {
+      sb.append(prefix)
+          .append(k)
+          .append(middle)
+          .append(map.get(k))
+          .append(suffix);
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Load an instance from a file, then validate it.
+   * @param fs filesystem
+   * @param path path
+   * @return the loaded instance
+   * @throws IOException IO failure
+   */
+  public static ManifestSuccessData load(FileSystem fs, Path path)
+      throws IOException {
+    LOG.debug("Reading success data from {}", path);
+    ManifestSuccessData instance = serializer().load(fs, path);
+    instance.validate();
+    return instance;
+  }
+
+  /**
+   * Get a JSON serializer for this class.
+   * @return a serializer.
+   */
+  public static JsonSerialization<ManifestSuccessData> serializer() {
+    return new JsonSerialization<>(ManifestSuccessData.class, false, true);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  /** @return timestamp of creation. */
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public void setTimestamp(long timestamp) {
+    this.timestamp = timestamp;
+  }
+
+  /** @return timestamp as date; no expectation of parseability. */
+  public String getDate() {
+    return date;
+  }
+
+  public void setDate(String date) {
+    this.date = date;
+  }
+
+  /**
+   * @return host which created the file (implicitly: committed the work).
+   */
+  public String getHostname() {
+    return hostname;
+  }
+
+  public void setHostname(String hostname) {
+    this.hostname = hostname;
+  }
+
+  /**
+   * @return committer name.
+   */
+  public String getCommitter() {
+    return committer;
+  }
+
+  public void setCommitter(String committer) {
+    this.committer = committer;
+  }
+
+  /**
+   * @return any description text.
+   */
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  /**
+   * @return any metrics.
+   */
+  public Map<String, Long> getMetrics() {
+    return metrics;
+  }
+
+  public void setMetrics(TreeMap<String, Long> metrics) {
+    this.metrics = metrics;
+  }
+
+  /**
+   * @return a list of filenames in the commit.
+   */
+  public List<String> getFilenames() {
+    return filenames;
+  }
+
+  /**
+   * Get the list of filenames as paths.
+   * @return the paths.
+   */
+  @JsonIgnore
+  public List<Path> getFilenamePaths() {
+    return getFilenames().stream()
+        .map(AbstractManifestData::unmarshallPath)
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Set the list of filename paths.
+   */
+  @JsonIgnore
+  public void setFilenamePaths(List<Path> paths) {
+    setFilenames(new ArrayList<>(
+        paths.stream()
+            .map(AbstractManifestData::marshallPath)
+            .collect(Collectors.toList())));
+  }
+
+  public void setFilenames(ArrayList<String> filenames) {
+    this.filenames = filenames;
+  }
+
+  public Map<String, String> getDiagnostics() {
+    return diagnostics;
+  }
+
+  public void setDiagnostics(TreeMap<String, String> diagnostics) {
+    this.diagnostics = diagnostics;
+  }
+
+  /**
+   * Add a diagnostics entry.
+   * @param key name
+   * @param value value
+   */
+  public void putDiagnostic(String key, String value) {
+    diagnostics.put(key, value);
+  }
+
+  /** @return Job ID, if known. */
+  public String getJobId() {
+    return jobId;
+  }
+
+  public void setJobId(String jobId) {
+    this.jobId = jobId;
+  }
+
+  public String getJobIdSource() {
+    return jobIdSource;
+  }
+
+  public void setJobIdSource(final String jobIdSource) {
+    this.jobIdSource = jobIdSource;
+  }
+
+  @Override
+  public IOStatisticsSnapshot getIOStatistics() {
+    return iostatistics;
+  }
+
+  public void setIOStatistics(final IOStatisticsSnapshot ioStatistics) {
+    this.iostatistics = ioStatistics;
+  }
+
+  /**
+   * Set the IOStatistics to a snapshot of the source.
+   * @param iostats. Statistics; may be null.
+   */
+  public void snapshotIOStatistics(IOStatistics iostats) {
+    setIOStatistics(IOStatisticsSupport.snapshotIOStatistics(iostats));
+  }
+
+  /**
+   * Set the success flag.
+   * @param success did the job succeed?
+   */
+  public void setSuccess(boolean success) {
+    this.success = success;
+  }
+
+  /**
+   * Get the success flag.
+   * @return did the job succeed?
+   */
+  public boolean getSuccess() {
+    return success;
+  }
+
+  public String getState() {
+    return state;
+  }
+
+  public void setState(String state) {
+    this.state = state;
+  }
+
+  public String getStage() {
+    return stage;
+  }
+
+  /**
+   * Note a failure by setting success flag to false,
+   * then add the exception to the diagnostics.
+   * @param thrown throwable
+   */
+  public void recordJobFailure(Throwable thrown) {
+    setSuccess(false);
+    String stacktrace = ExceptionUtils.getStackTrace(thrown);
+    diagnostics.put(DiagnosticKeys.EXCEPTION, thrown.toString());
+    diagnostics.put(DiagnosticKeys.STACKTRACE, stacktrace);
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java
new file mode 100644
index 0000000000000..a06b837aba5dc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * This is the manifest of files which were created by
+ * this task attempt.
+ *
+ * Versioning:
+ * In a rolling cluster update, MR or Spark workers deployed on a newer
+ * node (and/or with a newer version of artifacts in a cluster-FS hosted
+ * tar.gz file) may be a later version of this class than that of
+ * job committer.
+ * If any changes are made to the manifest which are backwards compatible,
+ * this new manifest can still be loaded from JSON and processed.
+ *
+ * If the manifest is no longer compatible, the job output may
+ * be invalid.
+ *
+ * It is CRITICAL that the {@link #VERSION} constant is updated whenever
+ * such an incompatible change is made.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class TaskManifest extends AbstractManifestData<TaskManifest> {
+
+  /**
+   * Supported version value: {@value}.
+   * If this is changed the value of {@code serialVersionUID} will change,
+   * to avoid deserialization problems.
+   */
+  public static final int VERSION = 1;
+
+  /**
+   * Manifest type.
+   */
+  public static final String TYPE =
+      "org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest/"
+      + VERSION;
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TaskManifest.class);
+
+  /**
+   * Serialization version.
+   */
+  private static final long serialVersionUID = 7090285511966046094L + VERSION;
+
+  /**
+   * Manifest type.
+   */
+  @JsonProperty("type")
+  private String type = TYPE;
+
+  /** Version marker. */
+  @JsonProperty("version")
+  private int version = VERSION;
+
+  /**
+   * Job ID; constant over multiple attempts.
+   */
+  @JsonProperty("jobId")
+  private String jobId;
+
+  /**
+   * Number of the job attempt; starts at zero.
+   */
+  @JsonProperty("jobAttemptNumber")
+  private int jobAttemptNumber;
+
+  /**
+   * Task Attempt ID.
+   */
+  @JsonProperty("taskID")
+  private String taskID;
+
+  /**
+   * Task Attempt ID.
+   */
+  @JsonProperty("taskAttemptID")
+  private String taskAttemptID;
+
+  /**
+   * The task attempt directory.
+   */
+  @JsonProperty("taskAttemptDir")
+  private String taskAttemptDir;
+
+  /**
+   * The list of files to commit from this task attempt, including
+   * precalculated destination and size.
+   */
+  @JsonProperty("files")
+  private final List<FileEntry> filesToCommit = new ArrayList<>();
+
+  /**
+   * The list of directories needed by this task attempt, both
+   * source and destination.
+   * All these directories must exist in the destination before any of
+   * the files can be renamed there.
+   */
+  @JsonProperty("directories")
+  private final List<DirEntry> destDirectories = new ArrayList<>();
+
+  /**
+   * Any custom extra data committers may choose to add.
+   */
+  private final Map<String, String> extraData = new HashMap<>(0);
+
+  /**
+   * IOStatistics.
+   */
+  @JsonProperty("iostatistics")
+  private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot();
+
+  /**
+   * Empty constructor; will be used by jackson as well as in application
+   * code.
+   */
+  public TaskManifest() {
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public void setType(String type) {
+    this.type = type;
+  }
+
+  public int getVersion() {
+    return version;
+  }
+
+  public void setVersion(int version) {
+    this.version = version;
+  }
+
+  @Override
+  public IOStatisticsSnapshot getIOStatistics() {
+    return iostatistics;
+  }
+
+  public void setIOStatistics(
+      @Nullable final IOStatisticsSnapshot ioStatistics) {
+    this.iostatistics = ioStatistics;
+  }
+
+  public String getJobId() {
+    return jobId;
+  }
+
+  public void setJobId(final String jobId) {
+    this.jobId = jobId;
+  }
+
+  public int getJobAttemptNumber() {
+    return jobAttemptNumber;
+  }
+
+  public void setJobAttemptNumber(final int jobAttemptNumber) {
+    this.jobAttemptNumber = jobAttemptNumber;
+  }
+
+  public String getTaskID() {
+    return taskID;
+  }
+
+  public void setTaskID(final String taskID) {
+    this.taskID = taskID;
+  }
+
+  public String getTaskAttemptID() {
+    return taskAttemptID;
+  }
+
+  public void setTaskAttemptID(final String taskAttemptID) {
+    this.taskAttemptID = taskAttemptID;
+  }
+
+  public String getTaskAttemptDir() {
+    return taskAttemptDir;
+  }
+
+  public void setTaskAttemptDir(final String taskAttemptDir) {
+    this.taskAttemptDir = taskAttemptDir;
+  }
+
+  /**
+   * Add a file to the list of files to commit.
+   * @param entry entry  to add
+   */
+  public void addFileToCommit(FileEntry entry) {
+    filesToCommit.add(entry);
+  }
+
+  public List<FileEntry> getFilesToCommit() {
+    return filesToCommit;
+  }
+
+  /**
+   * Calculate the total amount of data which will be committed.
+   * @return the sum of sizes of all files to commit.
+   */
+  @JsonIgnore
+  public long getTotalFileSize() {
+    return filesToCommit.stream().mapToLong(FileEntry::getSize).sum();
+  }
+
+  /**
+   * All the destination directories.
+   * @return directory list.
+   */
+  public List<DirEntry> getDestDirectories() {
+    return destDirectories;
+  }
+
+  /**
+   * Add a directory to the list of directories to create.
+   * @param entry entry  to add
+   */
+  public void addDirectory(DirEntry entry) {
+    destDirectories.add(entry);
+  }
+
+  public Map<String, String> getExtraData() {
+    return extraData;
+  }
+
+  @Override
+  public byte[] toBytes() throws IOException {
+    return serializer().toBytes(this);
+  }
+
+  /**
+   * To JSON.
+   * @return json string value.
+   * @throws IOException failure
+   */
+  public String toJson() throws IOException {
+    return serializer().toJson(this);
+  }
+
+  @Override
+  public void save(FileSystem fs, Path path, boolean overwrite)
+      throws IOException {
+    serializer().save(fs, path, this, overwrite);
+  }
+
+  /**
+   * Validate the data: those fields which must be non empty, must be set.
+   * @throws IOException if the data is invalid
+   * @return
+   */
+  public TaskManifest validate() throws IOException {
+    verify(TYPE.equals(type), "Wrong type: %s", type);
+    verify(version == VERSION, "Wrong version: %s", version);
+    validateCollectionClass(extraData.keySet(), String.class);
+    validateCollectionClass(extraData.values(), String.class);
+    Set<String> destinations = new HashSet<>(filesToCommit.size());
+    validateCollectionClass(filesToCommit, FileEntry.class);
+    for (FileEntry c : filesToCommit) {
+      c.validate();
+      verify(!destinations.contains(c.getDest()),
+          "Destination %s is written to by more than one pending commit",
+          c.getDest());
+      destinations.add(c.getDest());
+    }
+    return this;
+  }
+
+  /**
+   * Get a JSON serializer for this class.
+   * @return a serializer.
+   */
+  @Override
+  public JsonSerialization<TaskManifest> createSerializer() {
+    return serializer();
+  }
+
+  /**
+   * Create a JSON serializer for this class.
+   * @return a serializer.
+   */
+  public static JsonSerialization<TaskManifest> serializer() {
+    return new JsonSerialization<>(TaskManifest.class, false, true);
+  }
+
+  /**
+   * Load an instance from a file, then validate it.
+   * @param fs filesystem
+   * @param path path
+   * @return the loaded instance
+   * @throws IOException IO failure/the data is invalid
+   */
+  public static TaskManifest load(FileSystem fs, Path path)
+      throws IOException {
+    LOG.debug("Reading Manifest in file {}", path);
+    return serializer().load(fs, path).validate();
+  }
+
+  /**
+   * Load an instance from a file, then validate it.
+   * If loading through a listing; use this API so that filestatus
+   * hints can be used.
+   * @param serializer serializer.
+   * @param fs filesystem
+   * @param path path to load from
+   * @param status status of file to load
+   * @return the loaded instance
+   * @throws IOException IO failure/the data is invalid
+   */
+  public static TaskManifest load(
+      JsonSerialization<TaskManifest> serializer,
+      FileSystem fs,
+      Path path,
+      FileStatus status)
+      throws IOException {
+    LOG.debug("Reading Manifest in file {}", path);
+    return serializer.load(fs, path, status)
+        .validate();
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java
new file mode 100644
index 0000000000000..e1fbb4ac3325d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Persistence formats.
+ * These are the persistence formats used for passing data from tasks
+ * to the job committer
+ * {@link org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest},
+ * and for a {@code _SUCCESS} file, which is in
+ * {@link org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData}.
+ * The {@code _SUCCESS} file is a copy of the S3A Committer
+ * {@code org.apache.hadoop.fs.s3a.commit.files.ManifestSuccessData},
+ * the intent being that at the JSON-level they are compatible.
+ * This is to aid testing/validation and support calls, with one single
+ * format to load.
+ *
+ * Consult the individual formats for their declarations of access;
+ * the _SUCCESS file is one which tests may use.
+ *
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java
new file mode 100644
index 0000000000000..de67a2a88c7d3
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.audit.CommonAuditContext;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConfig;
+
+import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_JOB_ID;
+import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CONTEXT_ATTR_STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CONTEXT_ATTR_TASK_ATTEMPT_ID;
+
+/**
+ * Helper class to support integration with Hadoop 3.3.2+ Auditing.
+ * This MUST BE the sole place where fs.audit methods are used, so can be replaced
+ * by a stub class on any backport.
+ */
+@InterfaceAudience.Private
+public final class AuditingIntegration {
+  private AuditingIntegration() {
+  }
+
+  /**
+   * Add jobID to current context; also
+   * task attempt ID if set.
+   */
+  public static void updateCommonContextOnCommitterEntry(
+      ManifestCommitterConfig committerConfig) {
+    CommonAuditContext context = currentAuditContext();
+    context.put(PARAM_JOB_ID,
+        committerConfig.getJobUniqueId());
+    // maybe the task attempt ID.
+    if (!committerConfig.getTaskAttemptId().isEmpty()) {
+      context.put(CONTEXT_ATTR_TASK_ATTEMPT_ID,
+          committerConfig.getTaskAttemptId());
+    }
+  }
+
+  /**
+   * Callback on stage entry.
+   * Sets the activeStage and updates the
+   * common context.
+   * @param stage new stage
+   */
+  public static void enterStage(String stage) {
+    currentAuditContext().put(CONTEXT_ATTR_STAGE, stage);
+  }
+
+  /**
+   * Remove stage from common audit context.
+   */
+  public static void exitStage() {
+    currentAuditContext().remove(CONTEXT_ATTR_STAGE);
+  }
+
+  /**
+   * Remove commit info at the end of the task or job.
+   */
+  public static void updateCommonContextOnCommitterExit() {
+    currentAuditContext().remove(PARAM_JOB_ID);
+    currentAuditContext().remove(CONTEXT_ATTR_TASK_ATTEMPT_ID);
+  }
+
+  /**
+   * Update the thread context with the stage name and
+   * job ID.
+   * This MUST be invoked at the start of methods invoked in helper threads,
+   * to ensure that they are all annotated with job and stage.
+   * @param jobId job ID.
+   * @param stage stage name.
+   */
+  public static void enterStageWorker(String jobId, String stage) {
+    CommonAuditContext context = currentAuditContext();
+    context.put(PARAM_JOB_ID, jobId);
+    context.put(CONTEXT_ATTR_STAGE, stage);
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java
new file mode 100644
index 0000000000000..15f9899f3551e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.util.Set;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_DIRECTORY;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_FILE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_LIST_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_MKDIRS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.STORE_IO_RATE_LIMITED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.*;
+
+/**
+ * Constants internal to the manifest committer.
+ */
+@InterfaceAudience.Private
+public final class InternalConstants {
+  private InternalConstants() {
+  }
+
+  /**
+   * Durations.
+   */
+  public static final String[] DURATION_STATISTICS = {
+
+      /* Job stages. */
+      OP_STAGE_JOB_ABORT,
+      OP_STAGE_JOB_CLEANUP,
+      OP_STAGE_JOB_COMMIT,
+      OP_STAGE_JOB_CREATE_TARGET_DIRS,
+      OP_STAGE_JOB_LOAD_MANIFESTS,
+      OP_STAGE_JOB_RENAME_FILES,
+      OP_STAGE_JOB_SAVE_SUCCESS,
+      OP_STAGE_JOB_SETUP,
+      OP_STAGE_JOB_VALIDATE_OUTPUT,
+
+      /* Task stages. */
+
+      OP_STAGE_TASK_ABORT_TASK,
+      OP_STAGE_TASK_COMMIT,
+      OP_STAGE_TASK_SAVE_MANIFEST,
+      OP_STAGE_TASK_SCAN_DIRECTORY,
+      OP_STAGE_TASK_SETUP,
+
+      /* Lower level store/fs operations. */
+      OP_COMMIT_FILE_RENAME,
+      OP_CREATE_DIRECTORIES,
+      OP_CREATE_ONE_DIRECTORY,
+      OP_DIRECTORY_SCAN,
+      OP_DELETE,
+      OP_DELETE_FILE_UNDER_DESTINATION,
+      OP_GET_FILE_STATUS,
+      OP_IS_DIRECTORY,
+      OP_IS_FILE,
+      OP_LIST_STATUS,
+      OP_LOAD_MANIFEST,
+      OP_LOAD_ALL_MANIFESTS,
+      OP_MKDIRS,
+      OP_MKDIRS_RETURNED_FALSE,
+      OP_MSYNC,
+      OP_PREPARE_DIR_ANCESTORS,
+      OP_RENAME_FILE,
+      OP_SAVE_TASK_MANIFEST,
+
+      OBJECT_LIST_REQUEST,
+      OBJECT_CONTINUE_LIST_REQUEST,
+
+      STORE_IO_RATE_LIMITED
+  };
+
+  /**
+   * Counters.
+   */
+  public static final String[] COUNTER_STATISTICS = {
+      COMMITTER_BYTES_COMMITTED_COUNT,
+      COMMITTER_FILES_COMMITTED_COUNT,
+      COMMITTER_TASKS_COMPLETED_COUNT,
+      COMMITTER_TASKS_FAILED_COUNT,
+      COMMITTER_TASK_DIRECTORY_COUNT_MEAN,
+      COMMITTER_TASK_DIRECTORY_DEPTH_MEAN,
+      COMMITTER_TASK_FILE_COUNT_MEAN,
+      COMMITTER_TASK_FILE_SIZE_MEAN,
+      COMMITTER_TASK_MANIFEST_FILE_SIZE,
+      OP_COMMIT_FILE_RENAME_RECOVERED,
+  };
+
+  /**
+   * Error string from ABFS connector on timeout.
+   */
+  public static final String OPERATION_TIMED_OUT = "OperationTimedOut";
+
+  /**
+   * Format string for task attempt names.
+   */
+  public static final String NAME_FORMAT_TASK_ATTEMPT = "[Task-Attempt %s]";
+
+  /**
+   * Format string for job attempt names.
+   */
+  public static final String NAME_FORMAT_JOB_ATTEMPT = "[Job-Attempt %s]";
+
+  /** Schemas of filesystems we know to not work with this committer. */
+  public static final Set<String> UNSUPPORTED_FS_SCHEMAS =
+      ImmutableSet.of("s3a", "wasb");
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java
new file mode 100644
index 0000000000000..f6edde5f460b4
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.io.IOException;
+import java.time.ZonedDateTime;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStoreBuilder;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.PENDING_DIR_NAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.INITIAL_APP_ATTEMPT_ID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ATTEMPT_DIR_FORMAT_STR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_DIR_FORMAT_STR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ID_SOURCE_MAPREDUCE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_TASK_ATTEMPT_SUBDIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_TASK_MANIFEST_SUBDIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_CLASSNAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SPARK_WRITE_UUID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUMMARY_FILENAME_FORMAT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.TMP_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.PRINCIPAL;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.COUNTER_STATISTICS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.DURATION_STATISTICS;
+
+/**
+ * Class for manifest committer support util methods.
+ */
+
+@InterfaceAudience.Private
+public final class ManifestCommitterSupport {
+
+  private ManifestCommitterSupport() {
+  }
+
+  /**
+   * Create an IOStatistics Store with the standard statistics
+   * set up.
+   * @return a store builder preconfigured with the standard stats.
+   */
+  public static IOStatisticsStoreBuilder createIOStatisticsStore() {
+
+    final IOStatisticsStoreBuilder store
+        = iostatisticsStore();
+
+    store.withCounters(COUNTER_STATISTICS);
+    store.withMaximums(COUNTER_STATISTICS);
+    store.withMinimums(COUNTER_STATISTICS);
+    store.withMeanStatistics(COUNTER_STATISTICS);
+    store.withDurationTracking(DURATION_STATISTICS);
+    return store;
+  }
+
+  /**
+   * If the object is an IOStatisticsSource, get and add
+   * its IOStatistics.
+   * @param o source object.
+   */
+  public static void maybeAddIOStatistics(IOStatisticsAggregator ios,
+      Object o) {
+    if (o instanceof IOStatisticsSource) {
+      ios.aggregate(((IOStatisticsSource) o).getIOStatistics());
+    }
+  }
+
+  /**
+   * Build a Job UUID from the job conf (if it is
+   * {@link ManifestCommitterConstants#SPARK_WRITE_UUID}
+   * or the MR job ID.
+   * @param conf job/task configuration
+   * @param jobId job ID from YARN or spark.
+   * @return (a job ID, source)
+   */
+  public static Pair<String, String> buildJobUUID(Configuration conf,
+      JobID jobId) {
+    String jobUUID = conf.getTrimmed(SPARK_WRITE_UUID, "");
+    if (jobUUID.isEmpty()) {
+      jobUUID = jobId.toString();
+      return Pair.of(jobUUID, JOB_ID_SOURCE_MAPREDUCE);
+    } else {
+      return Pair.of(jobUUID, SPARK_WRITE_UUID);
+    }
+  }
+
+  /**
+   * Get the location of pending job attempts.
+   * @param out the base output directory.
+   * @return the location of pending job attempts.
+   */
+  public static Path getPendingJobAttemptsPath(Path out) {
+    return new Path(out, PENDING_DIR_NAME);
+  }
+
+  /**
+   * Get the Application Attempt Id for this job.
+   * @param context the context to look in
+   * @return the Application Attempt Id for a given job.
+   */
+  public static int getAppAttemptId(JobContext context) {
+    return getAppAttemptId(context.getConfiguration());
+  }
+
+  /**
+   * Get the Application Attempt Id for this job
+   * by looking for {@link MRJobConfig#APPLICATION_ATTEMPT_ID}
+   * in the configuration, falling back to 0 if unset.
+   * For spark it will always be 0, for MR it will be set in the AM
+   * to the {@code ApplicationAttemptId} the AM is launched with.
+   * @param conf job configuration.
+   * @return the Application Attempt Id for the job.
+   */
+  public static int getAppAttemptId(Configuration conf) {
+    return conf.getInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+        INITIAL_APP_ATTEMPT_ID);
+  }
+
+  /**
+   * Get the path in the job attempt dir for a manifest for a task.
+   * @param manifestDir manifest directory
+   * @param taskId taskID.
+   * @return the final path to rename the manifest file to
+   */
+  public static Path manifestPathForTask(Path manifestDir, String taskId) {
+
+    return new Path(manifestDir, taskId + MANIFEST_SUFFIX);
+  }
+
+  /**
+   * Get the path in the  manifest subdir for the temp path to save a
+   * task attempt's manifest before renaming it to the
+   * path defined by {@link #manifestPathForTask(Path, String)}.
+   * @param manifestDir manifest directory
+   * @param taskAttemptId task attempt ID.
+   * @return the path to save/load the manifest.
+   */
+  public static Path manifestTempPathForTaskAttempt(Path manifestDir,
+      String taskAttemptId) {
+    return new Path(manifestDir,
+        taskAttemptId + MANIFEST_SUFFIX + TMP_SUFFIX);
+  }
+
+  /**
+   * Create a task attempt dir; stage config must be for a task attempt.
+   * @param stageConfig state config.
+   * @return a manifest with job and task attempt info set up.
+   */
+  public static TaskManifest createTaskManifest(StageConfig stageConfig) {
+    final TaskManifest manifest = new TaskManifest();
+    manifest.setTaskAttemptID(stageConfig.getTaskAttemptId());
+    manifest.setTaskID(stageConfig.getTaskId());
+    manifest.setJobId(stageConfig.getJobId());
+    manifest.setJobAttemptNumber(stageConfig.getJobAttemptNumber());
+    manifest.setTaskAttemptDir(
+        stageConfig.getTaskAttemptDir().toUri().toString());
+    return manifest;
+  }
+
+  /**
+   * Create success/outcome data.
+   * @param stageConfig configuration.
+   * @param stage
+   * @return a _SUCCESS object with some diagnostics.
+   */
+  public static ManifestSuccessData createManifestOutcome(
+      StageConfig stageConfig, String stage) {
+    final ManifestSuccessData outcome = new ManifestSuccessData();
+    outcome.setJobId(stageConfig.getJobId());
+    outcome.setJobIdSource(stageConfig.getJobIdSource());
+    outcome.setCommitter(MANIFEST_COMMITTER_CLASSNAME);
+    // real timestamp
+    outcome.setTimestamp(System.currentTimeMillis());
+    final ZonedDateTime now = ZonedDateTime.now();
+    outcome.setDate(now.toString());
+    outcome.setHostname(NetUtils.getLocalHostname());
+    // add some extra diagnostics which can still be parsed by older
+    // builds of test applications.
+    // Audit Span information can go in here too, in future.
+    try {
+      outcome.putDiagnostic(PRINCIPAL,
+          UserGroupInformation.getCurrentUser().getShortUserName());
+    } catch (IOException ignored) {
+      // don't know who we are? exclude from the diagnostics.
+    }
+    outcome.putDiagnostic(STAGE, stage);
+    return outcome;
+  }
+
+  /**
+   * Create the filename for a report from the jobID.
+   * @param jobId jobId
+   * @return filename for a report.
+   */
+  public static String createJobSummaryFilename(String jobId) {
+    return String.format(SUMMARY_FILENAME_FORMAT, jobId);
+  }
+
+  /**
+   * Get an etag from a FileStatus which MUST BE
+   * an implementation of EtagSource and
+   * whose etag MUST NOT BE null/empty.
+   * @param status the status; may be null.
+   * @return the etag or null if not provided
+   */
+  public static String getEtag(FileStatus status) {
+    if (status instanceof EtagSource) {
+      return ((EtagSource) status).getEtag();
+    } else {
+      return null;
+    }
+  }
+
+  /**
+   * Create the manifest store operations for the given FS.
+   * This supports binding to custom filesystem handlers.
+   * @param conf configuration.
+   * @param filesystem fs.
+   * @param path path under FS.
+   * @return a bonded store operations.
+   * @throws IOException on binding/init problems.
+   */
+  public static ManifestStoreOperations createManifestStoreOperations(
+      final Configuration conf,
+      final FileSystem filesystem,
+      final Path path) throws IOException {
+    try {
+      final Class<? extends ManifestStoreOperations> storeClass = conf.getClass(
+          OPT_STORE_OPERATIONS_CLASS,
+          ManifestStoreOperationsThroughFileSystem.class,
+          ManifestStoreOperations.class);
+      final ManifestStoreOperations operations = storeClass.
+          getDeclaredConstructor().newInstance();
+      operations.bindToFileSystem(filesystem, path);
+      return operations;
+    } catch (Exception e) {
+      throw new PathIOException(path.toString(),
+          "Failed to create Store Operations from configuration option "
+              + OPT_STORE_OPERATIONS_CLASS
+              + ":" + e, e);
+    }
+  }
+
+  /**
+   * Logic to create directory names from job and attempt.
+   * This is self-contained it so it can be used in tests
+   * as well as in the committer.
+   */
+  public static class AttemptDirectories {
+
+    /**
+     * Job output path.
+     */
+    private final Path outputPath;
+
+    /**
+     * Path for the job attempt.
+     */
+    private final Path jobAttemptDir;
+
+    /**
+     * Path for the job.
+     */
+    private final Path jobPath;
+
+    /**
+     * Subdir under the job attempt dir where task
+     * attempts will have subdirectories.
+     */
+    private final Path jobAttemptTaskSubDir;
+
+    /**
+     * temp directory under job dest dir.
+     */
+    private final Path outputTempSubDir;
+
+    /**
+     * Directory to save manifests into.
+     */
+    private final Path taskManifestDir;
+
+    /**
+     * Build the attempt directories.
+     * @param outputPath output path
+     * @param jobUniqueId job ID/UUID
+     * @param jobAttemptNumber job attempt number
+     */
+    public AttemptDirectories(
+        Path outputPath,
+        String jobUniqueId,
+        int jobAttemptNumber) {
+      this.outputPath = requireNonNull(outputPath, "Output path");
+
+      this.outputTempSubDir = new Path(outputPath, PENDING_DIR_NAME);
+      // build the path for the job
+      this.jobPath = new Path(outputTempSubDir,
+          String.format(JOB_DIR_FORMAT_STR, jobUniqueId));
+
+      // then the specific path underneath that for the attempt.
+      this.jobAttemptDir = new Path(jobPath,
+          String.format(JOB_ATTEMPT_DIR_FORMAT_STR, jobAttemptNumber));
+
+      // subdir for task attempts.
+      this.jobAttemptTaskSubDir = new Path(jobAttemptDir, JOB_TASK_ATTEMPT_SUBDIR);
+
+      this.taskManifestDir = new Path(jobAttemptDir, JOB_TASK_MANIFEST_SUBDIR);
+    }
+
+    public Path getOutputPath() {
+      return outputPath;
+    }
+
+    public Path getJobAttemptDir() {
+      return jobAttemptDir;
+    }
+
+    public Path getJobPath() {
+      return jobPath;
+    }
+
+    public Path getJobAttemptTaskSubDir() {
+      return jobAttemptTaskSubDir;
+    }
+
+    public Path getTaskAttemptPath(String taskAttemptId) {
+      return new Path(jobAttemptTaskSubDir, taskAttemptId);
+    }
+
+    public Path getOutputTempSubDir() {
+      return outputTempSubDir;
+    }
+
+    public Path getTaskManifestDir() {
+      return taskManifestDir;
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java
new file mode 100644
index 0000000000000..b81fa9dd32add
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * FileSystem operations which are needed to generate the task manifest.
+ * The specific choice of which implementation to use is configurable.
+ * Object store implementations MAY subclass if they
+ * need to implement resilient commit operations.
+ * However, the actual API MUST NOT be used outside
+ * the manifest committer and its tests.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce, object-stores")
+@InterfaceStability.Unstable
+public abstract class ManifestStoreOperations implements Closeable {
+
+  /**
+   * Bind to the filesystem.
+   * This is called by the manifest committer after the operations
+   * have been instantiated.
+   * @param fileSystem target FS
+   * @param path actual path under FS.
+   * @throws IOException if there are binding problems.
+   */
+  public void bindToFileSystem(FileSystem fileSystem, Path path) throws IOException {
+
+  }
+
+  /**
+   * Forward to {@link FileSystem#getFileStatus(Path)}.
+   * @param path path
+   * @return status
+   * @throws IOException failure.
+   */
+  public abstract FileStatus getFileStatus(Path path) throws IOException;
+
+  /**
+   * Is a path a file? Used during directory creation.
+   * The is a copy & paste of FileSystem.isFile();
+   * {@code StoreOperationsThroughFileSystem} calls into
+   * the FS direct so that stores which optimize their probes
+   * can save on IO.
+   * @param path path to probe
+   * @return true if the path exists and resolves to a file
+   * @throws IOException failure other than FileNotFoundException
+   */
+  public boolean isFile(Path path) throws IOException {
+    try {
+      return getFileStatus(path).isFile();
+    } catch (FileNotFoundException e) {
+      return false;
+    }
+  }
+
+  /**
+   * Forward to {@link FileSystem#delete(Path, boolean)}.
+   * If it returns without an error: there is nothing at
+   * the end of the path.
+   * @param path path
+   * @param recursive recursive delete.
+   * @return true if the path was deleted.
+   * @throws IOException failure.
+   */
+  public abstract boolean delete(Path path, boolean recursive)
+      throws IOException;
+
+  /**
+   * Forward to {@link FileSystem#mkdirs(Path)}.
+   * Usual "what does 'false' mean" ambiguity.
+   * @param path path
+   * @return true if the directory was created.
+   * @throws IOException failure.
+   */
+  public abstract boolean mkdirs(Path path) throws IOException;
+
+  /**
+   * Forward to {@link FileSystem#rename(Path, Path)}.
+   * Usual "what does 'false' mean" ambiguity.
+   * @param source source file
+   * @param dest destination path -which must not exist.
+   * @return the return value of the rename
+   * @throws IOException failure.
+   */
+  public abstract boolean renameFile(Path source, Path dest)
+      throws IOException;
+
+  /**
+   * Rename a dir; defaults to invoking
+   * Forward to {@link #renameFile(Path, Path)}.
+   * Usual "what does 'false' mean?" ambiguity.
+   * @param source source file
+   * @param dest destination path -which must not exist.
+   * @return true if the directory was created.
+   * @throws IOException failure.
+   */
+  public boolean renameDir(Path source, Path dest)
+      throws IOException {
+    return renameFile(source, dest);
+  }
+
+  /**
+   * List the directory.
+   * @param path path to list.
+   * @return an iterator over the results.
+   * @throws IOException any immediate failure.
+   */
+  public abstract RemoteIterator<FileStatus> listStatusIterator(Path path)
+      throws IOException;
+
+  /**
+   * Load a task manifest from the store.
+   * with a real FS, this is done with
+   * {@link TaskManifest#load(JsonSerialization, FileSystem, Path, FileStatus)}
+   *
+   * @param serializer serializer.
+   * @param st status with the path and other data.
+   * @return the manifest
+   * @throws IOException failure to load/parse
+   */
+  public abstract TaskManifest loadTaskManifest(
+      JsonSerialization<TaskManifest> serializer,
+      FileStatus st) throws IOException;
+
+  /**
+   * Save a task manifest by {@code FileSystem.create(path)}.
+   * there's no attempt at renaming anything here.
+   * @param manifestData the manifest/success file
+   * @param path temp path for the initial save
+   * @param overwrite should create(overwrite=true) be used?
+   * @throws IOException failure to load/parse
+   */
+  public abstract <T extends AbstractManifestData<T>> void save(
+      T manifestData,
+      Path path,
+      boolean overwrite) throws IOException;
+
+  /**
+   * Make an msync() call; swallow when unsupported.
+   * @param path path
+   * @throws IOException IO failure
+   */
+  public void msync(Path path) throws IOException {
+
+  }
+
+
+  /**
+   * Extract an etag from a status if the conditions are met.
+   * If the conditions are not met, return null or ""; they will
+   * both be treated as "no etags available"
+   * <pre>
+   *   1. The status is of a type which the implementation recognizes
+   *   as containing an etag.
+   *   2. After casting the etag field can be retrieved
+   *   3. and that value is non-null/non-empty.
+   * </pre>
+   * @param status status, which may be null of any subclass of FileStatus.
+   * @return either a valid etag, or null or "".
+   */
+  public String getEtag(FileStatus status) {
+    return ManifestCommitterSupport.getEtag(status);
+  }
+
+  /**
+   * Does the store preserve etags through renames.
+   * If true, and if the source listing entry has an etag,
+   * it will be used to attempt to validate a failed rename.
+   * @param path path to probe.
+   * @return true if etag comparison is a valid strategy.
+   */
+  public boolean storePreservesEtagsThroughRenames(Path path) {
+    return false;
+  }
+
+  /**
+   * Does the store provide rename resilience through an
+   * implementation of {@link #commitFile(FileEntry)}?
+   * If true then that method will be invoked to commit work
+   * @return true if resilient commit support is available.
+   */
+  public boolean storeSupportsResilientCommit() {
+    return false;
+  }
+
+  /**
+   * Commit one file through any resilient API.
+   * This operation MUST rename source to destination,
+   * else raise an exception.
+   * The result indicates whether or not some
+   * form of recovery took place.
+   *
+   * If etags were collected during task commit, these will be
+   * in the entries passed in here.
+   *
+   * The base implementation always raises
+   * {@code UnsupportedOperationException}
+   * @param entry entry to commit
+   * @return the result of the commit
+   * @throws IOException failure.
+   * @throws UnsupportedOperationException if not available.
+   *
+   */
+  public CommitFileResult commitFile(FileEntry entry) throws IOException {
+    throw new UnsupportedOperationException("Resilient commit not supported");
+  }
+
+  /**
+   * Outcome from the operation {@link #commitFile(FileEntry)}.
+   * As a rename failure MUST raise an exception, this result
+   * only declares whether or not some form of recovery took place.
+   */
+  public static final class CommitFileResult {
+
+    /** Did recovery take place? */
+    private final boolean recovered;
+
+    /** Time waiting for IO capacity, may be null. */
+    @Nullable
+    private final Duration waitTime;
+
+    /**
+     * Full commit result.
+     * @param recovered Did recovery take place?
+     * @param waitTime any time spent waiting for IO capacity.
+     */
+    public static CommitFileResult fromResilientCommit(
+        final boolean recovered,
+        final Duration waitTime) {
+      return new CommitFileResult(recovered, waitTime);
+    }
+
+    /**
+     * Full commit result.
+     * @param recovered Did recovery take place?
+     * @param waitTime any time spent waiting for IO capacity.
+     */
+    public CommitFileResult(final boolean recovered,
+        @Nullable final Duration waitTime) {
+
+      this.recovered = recovered;
+      this.waitTime = waitTime;
+    }
+
+    /**
+     * Did some form of recovery take place?
+     * @return true if the commit succeeded through some form of (etag-based) recovery
+     */
+    public boolean recovered() {
+      return recovered;
+    }
+
+    @Nullable
+    public Duration getWaitTime() {
+      return waitTime;
+    }
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java
new file mode 100644
index 0000000000000..9a0b972bc735b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonPathCapabilities;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Implementation of manifest store operations through the filesystem API.
+ * This class is subclassed in the ABFS module, which does add the resilient
+ * commit method.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce, object-stores")
+@InterfaceStability.Unstable
+public class ManifestStoreOperationsThroughFileSystem extends ManifestStoreOperations {
+
+  /**
+   * Filesystem; set in {@link #bindToFileSystem(FileSystem, Path)}.
+   */
+  private FileSystem fileSystem;
+
+  /**
+   * Has a call to FileSystem.msync() failed as unsupported?
+   * If so, no new attempts will be made when
+   * (@link {@link #msync(Path)} is invoked.
+   */
+  private boolean msyncUnsupported = false;
+
+  /**
+   * Direct Constructor.
+   * @param fileSystem filesystem to write through.
+   */
+  public ManifestStoreOperationsThroughFileSystem(final FileSystem fileSystem) {
+    this.fileSystem = fileSystem;
+  }
+
+  /**
+   * Constructor used for introspection-based binding.
+   */
+  public ManifestStoreOperationsThroughFileSystem() {
+  }
+
+  @Override
+  public void close() throws IOException {
+    /* no-op; FS is assumed to be shared. */
+
+  }
+
+  /**
+   * Get the filesystem.
+   * @return the filesystem; null until bound.
+   */
+  public FileSystem getFileSystem() {
+    return fileSystem;
+  }
+
+  @Override
+  public void bindToFileSystem(FileSystem filesystem, Path path) throws IOException {
+    fileSystem = filesystem;
+  }
+
+  @Override
+  public FileStatus getFileStatus(Path path) throws IOException {
+    return fileSystem.getFileStatus(path);
+  }
+
+  /**
+   * Using FileSystem.isFile to offer stores the option to optimize their probes.
+   * @param path path to probe
+   * @return true if the path resolves to a file.
+   * @throws IOException IO failure.
+   */
+  @SuppressWarnings("deprecation")
+  @Override
+  public boolean isFile(Path path) throws IOException {
+    return fileSystem.isFile(path);
+  }
+
+  @Override
+  public boolean delete(Path path, boolean recursive)
+      throws IOException {
+    return fileSystem.delete(path, recursive);
+  }
+
+  @Override
+  public boolean mkdirs(Path path)
+      throws IOException {
+    return fileSystem.mkdirs(path);
+  }
+
+  @Override
+  public boolean renameFile(Path source, Path dest)
+      throws IOException {
+    return fileSystem.rename(source, dest);
+  }
+
+  @Override
+  public RemoteIterator<FileStatus> listStatusIterator(Path path)
+      throws IOException {
+    return fileSystem.listStatusIterator(path);
+  }
+
+  @Override
+  public TaskManifest loadTaskManifest(
+      JsonSerialization<TaskManifest> serializer,
+      FileStatus st) throws IOException {
+    return TaskManifest.load(serializer, fileSystem, st.getPath(), st);
+  }
+
+  @Override
+  public <T extends AbstractManifestData<T>> void save(
+      final T manifestData,
+      final Path path,
+      final boolean overwrite) throws IOException {
+    manifestData.save(fileSystem, path, overwrite);
+  }
+
+  /**
+   * Probe filesystem capabilities.
+   * @param path path to probe.
+   * @return true if the FS declares its renames work.
+   */
+  @Override
+  public boolean storePreservesEtagsThroughRenames(Path path) {
+    try {
+      return fileSystem.hasPathCapability(path,
+          CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME);
+    } catch (IOException ignored) {
+      return false;
+    }
+  }
+
+  /**
+   * Invokes FileSystem msync(); swallows UnsupportedOperationExceptions.
+   * This ensures client metadata caches are in sync in an HDFS-HA deployment.
+   * No other filesystems support this; in the absence of a hasPathCapability()
+   * probe, after the operation is rejected, an atomic boolean is set
+   * to stop further attempts from even trying.
+   * @param path path
+   * @throws IOException failure to synchronize.
+   */
+  @Override
+  public void msync(Path path) throws IOException {
+    // there's need for atomicity here, as the sole cost of
+    // multiple failures
+    if (msyncUnsupported) {
+      return;
+    }
+    // qualify so we can be confident that the FS being synced
+    // is the one we expect.
+    fileSystem.makeQualified(path);
+    try {
+      fileSystem.msync();
+    } catch (UnsupportedOperationException ignored) {
+      // this exception is the default.
+      // set the unsupported flag so no future attempts are made.
+      msyncUnsupported = true;
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java
new file mode 100644
index 0000000000000..f1dacc25fbcef
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Exception raised during validation.
+ * This can be treated differently from other outcomes.
+ */
+@InterfaceAudience.Private
+public class OutputValidationException extends PathIOException {
+  public OutputValidationException(Path path, String error) {
+    super(path.toUri().toString(), error);
+  }
+
+  public OutputValidationException(Path path,
+      String error,
+      Throwable cause) {
+    super(path.toUri().toString(), error, cause);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/TableDeleteTimeoutException.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
similarity index 67%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/TableDeleteTimeoutException.java
rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
index 7969332139220..eb7dda6cc77fe 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/TableDeleteTimeoutException.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
@@ -16,19 +16,15 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.fs.PathIOException;
-
 /**
- * An exception raised when a table being deleted is still present after
- * the wait time is exceeded.
+ * Implementation classes for the manifest committer.
+ * Nothing outside this module should be using these classes,
+ * except where explicitly stated.
  */
-public class TableDeleteTimeoutException extends PathIOException {
 
-  TableDeleteTimeoutException(final String path,
-      final String error,
-      final Throwable cause) {
-    super(path, error, cause);
-  }
-}
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
new file mode 100644
index 0000000000000..c00ae5ad2249f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Intermediate manifest committer.
+ *
+ * Optimized for object stores where listing is slow, directory renames may not
+ * be atomic, and the output is a deep tree of files intermixed with
+ * the output of (many) other task attempts.
+ *
+ * All classes in this module are private/unstable, except where stated.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java
new file mode 100644
index 0000000000000..c2b44c2a924fd
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_ABORT_TASK;
+
+/**
+ * Abort a task.
+ *
+ * This is done by deleting the task directory.
+ * Exceptions may/may not be suppressed.
+ */
+public class AbortTaskStage extends
+    AbstractJobOrTaskStage<Boolean, Path> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbortTaskStage.class);
+
+  public AbortTaskStage(final StageConfig stageConfig) {
+    super(true, stageConfig, OP_STAGE_TASK_ABORT_TASK, false);
+  }
+
+  /**
+   * Delete the task attempt directory.
+   * @param suppressExceptions should exceptions be ignored?
+   * @return the directory
+   * @throws IOException failure when exceptions were not suppressed
+   */
+  @Override
+  protected Path executeStage(final Boolean suppressExceptions)
+      throws IOException {
+    final Path dir = getTaskAttemptDir();
+    if (dir != null) {
+      LOG.info("{}: Deleting task attempt directory {}", getName(), dir);
+      deleteDir(dir, suppressExceptions);
+    }
+    return dir;
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java
new file mode 100644
index 0000000000000..05ee7a5ac11d7
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.util.OperationDuration;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_FILE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_LIST_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_MKDIRS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.STORE_IO_RATE_LIMITED;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.createTracker;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME_RECOVERED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_LOAD_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_MSYNC;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_RENAME_FILE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_SAVE_TASK_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.enterStageWorker;
+
+/**
+ * A Stage in Task/Job Commit.
+ * A stage can be executed once only, creating the return value of the
+ * {@link #apply(Object)} method, and, potentially, updating the state of the
+ * store via {@link ManifestStoreOperations}.
+ * IOStatistics will also be updated.
+ * Stages are expected to be combined to form the commit protocol.
+ * @param <IN> Type of arguments to the stage.
+ * @param <OUT> Type of result.
+ */
+public abstract class AbstractJobOrTaskStage<IN, OUT>
+    implements JobOrTaskStage<IN, OUT> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbstractJobOrTaskStage.class);
+
+  /**
+   * Error text on rename failure: {@value}.
+   */
+  public static final String FAILED_TO_RENAME_PREFIX = "Failed to ";
+
+  /**
+   * Is this a task stage? If so, toString() includes task
+   * info..
+   */
+  private final boolean isTaskStage;
+
+  /**
+   * Configuration of all the stages in the ongoing committer
+   * operation.
+   */
+  private final StageConfig stageConfig;
+
+  /**
+   * Name of the stage for statistics and logging.
+   */
+  private final String stageStatisticName;
+
+  /**
+   * Callbacks to update store.
+   * This is not made visible to the stages; they must
+   * go through the wrapper classes in this class, which
+   * add statistics and logging.
+   */
+  private final ManifestStoreOperations operations;
+
+  /**
+   * Submitter for doing IO against the store.
+   */
+  private final TaskPool.Submitter ioProcessors;
+
+  /**
+   * Used to stop any re-entrancy of the rename.
+   * This is an execute-once operation.
+   */
+  private final AtomicBoolean executed = new AtomicBoolean(false);
+
+  /**
+   * Tracker of the duration of the execution of the stage.
+   * set after {@link #executeStage(Object)} completes.
+   */
+  private DurationTracker stageExecutionTracker;
+
+  /**
+   * Name for logging.
+   */
+  private final String name;
+
+  /**
+   * Constructor.
+   * @param isTaskStage Is this a task stage?
+   * @param stageConfig stage-independent configuration.
+   * @param stageStatisticName name of the stage for statistics/logging
+   * @param requireIOProcessors are the IO processors required?
+   */
+  protected AbstractJobOrTaskStage(
+      final boolean isTaskStage,
+      final StageConfig stageConfig,
+      final String stageStatisticName,
+      final boolean requireIOProcessors) {
+    this.isTaskStage = isTaskStage;
+    this.stageStatisticName = stageStatisticName;
+    this.stageConfig = stageConfig;
+    requireNonNull(stageConfig.getDestinationDir(), "Destination Directory");
+    requireNonNull(stageConfig.getJobId(), "Job ID");
+    requireNonNull(stageConfig.getJobAttemptDir(), "Job attempt directory");
+    this.operations = requireNonNull(stageConfig.getOperations(),
+        "Operations callbacks");
+    // and the processors of work if required.
+    this.ioProcessors = bindProcessor(
+        requireIOProcessors,
+        stageConfig.getIoProcessors());
+    String stageName;
+    if (isTaskStage) {
+      // force fast failure.
+      getRequiredTaskId();
+      getRequiredTaskAttemptId();
+      getRequiredTaskAttemptDir();
+      stageName = String.format("[Task-Attempt %s]", getRequiredTaskAttemptId());
+    } else  {
+      stageName = String.format("[Job-Attempt %s/%02d]",
+          stageConfig.getJobId(),
+          stageConfig.getJobAttemptNumber());
+    }
+    name = stageName;
+  }
+
+  /**
+   * Bind to the processor if it is required.
+   * @param required is the processor required?
+   * @param processor processor
+   * @return the processor binding
+   * @throws NullPointerException if required == true and processor is null.
+   */
+  private TaskPool.Submitter bindProcessor(
+      final boolean required,
+      final TaskPool.Submitter processor) {
+    return required
+        ? requireNonNull(processor, "required IO processor is null")
+        : null;
+  }
+
+  /**
+   * Stage entry point.
+   * Verifies that this is the first and only time the stage is invoked,
+   * then calls {@link #executeStage(Object)} for the subclass
+   * to perform its part of the commit protocol.
+   * The duration of the stage is collected as a statistic, and its
+   * entry/exit logged at INFO.
+   * @param arguments arguments to the function.
+   * @return the result.
+   * @throws IOException failures.
+   */
+  @Override
+  public final OUT apply(final IN arguments) throws IOException {
+    executeOnlyOnce();
+    progress();
+    String stageName = getStageName(arguments);
+    getStageConfig().enterStage(stageName);
+    String statisticName = getStageStatisticName(arguments);
+    // The tracker here
+    LOG.info("{}: Executing Stage {}", getName(), stageName);
+    stageExecutionTracker = createTracker(getIOStatistics(), statisticName);
+    try {
+      // exec the input function and return its value
+      final OUT out = executeStage(arguments);
+      LOG.info("{}: Stage {} completed after {}",
+          getName(),
+          stageName,
+          OperationDuration.humanTime(
+              stageExecutionTracker.asDuration().toMillis()));
+      return out;
+    } catch (IOException | RuntimeException e) {
+      LOG.error("{}: Stage {} failed: after {}: {}",
+          getName(),
+          stageName,
+          OperationDuration.humanTime(
+              stageExecutionTracker.asDuration().toMillis()),
+          e.toString());
+      LOG.debug("{}: Stage failure:", getName(), e);
+      // input function failed: note it
+      stageExecutionTracker.failed();
+      // and rethrow
+      throw e;
+    } finally {
+      // update the tracker.
+      // this is called after the catch() call would have
+      // set the failed flag.
+      stageExecutionTracker.close();
+      progress();
+      getStageConfig().exitStage(stageName);
+    }
+  }
+
+  /**
+   * The work of a stage.
+   * Executed exactly once.
+   * @param arguments arguments to the function.
+   * @return the result.
+   * @throws IOException failures.
+   */
+  protected abstract OUT executeStage(IN arguments) throws IOException;
+
+  /**
+   * Check that the operation has not been invoked twice.
+   * This is an atomic check.
+   * @throws IllegalStateException on a second invocation.
+   */
+  private void executeOnlyOnce() {
+    Preconditions.checkState(
+        !executed.getAndSet(true),
+        "Stage attempted twice");
+  }
+
+  /**
+   * The stage statistic name.
+   * @param arguments args to the invocation.
+   * @return stage name.
+   */
+  protected String getStageStatisticName(IN arguments) {
+    return stageStatisticName;
+  }
+
+  /**
+   * Stage name for reporting; defaults to
+   * call {@link #getStageStatisticName(IN)}.
+   * @param arguments args to the invocation.
+   * @return name used in updating reports.
+   */
+  protected String getStageName(IN arguments) {
+    return getStageStatisticName(arguments);
+  }
+
+  /**
+   * Get the execution tracker; non-null
+   * after stage execution.
+   * @return a tracker or null.
+   */
+  public DurationTracker getStageExecutionTracker() {
+    return stageExecutionTracker;
+  }
+
+  /**
+   * Adds the duration of the job to an IOStatistics store
+   * (such as the manifest to be saved).
+   * @param iostats store
+   * @param statistic statistic name.
+   */
+  public void addExecutionDurationToStatistics(IOStatisticsStore iostats,
+      String statistic) {
+    iostats.addTimedOperation(
+        statistic,
+        getStageExecutionTracker().asDuration());
+  }
+
+  /**
+   * Note any rate limiting to the given timing statistic.
+   * If the wait was 0, no statistics are updated.
+   * @param statistic statistic key.
+   * @param wait wait duration.
+   */
+  private void noteAnyRateLimiting(String statistic, Duration wait) {
+    if (!wait.isZero()) {
+      // rate limiting took place
+      getIOStatistics().addTimedOperation(
+          statistic,
+          wait.toMillis());
+    }
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder(
+        "AbstractJobOrTaskStage{");
+    sb.append(isTaskStage ? "Task Stage" : "Job Stage");
+    sb.append(" name='").append(name).append('\'');
+    sb.append(" stage='").append(stageStatisticName).append('\'');
+    sb.append('}');
+    return sb.toString();
+  }
+
+  /**
+   * The stage configuration.
+   * @return the stage configuration used by this stage.
+   */
+  protected StageConfig getStageConfig() {
+    return stageConfig;
+  }
+
+  /**
+   * Update the thread context with the stage name and
+   * job ID.
+   * This MUST be invoked at the start of methods invoked in helper threads,
+   * to ensure that they are all annotated with job and stage.
+   * @param stage stage name.
+   */
+  protected void updateAuditContext(final String stage) {
+    enterStageWorker(stageConfig.getJobId(), stage);
+  }
+
+  /**
+   * The IOStatistics are shared across all uses of the
+   * StageConfig.
+   * @return the (possibly shared) IOStatistics.
+   */
+  @Override
+  public final IOStatisticsStore getIOStatistics() {
+    return stageConfig.getIOStatistics();
+  }
+
+  /**
+   * Call progress() on any Progressable passed in.
+   */
+  protected final void progress() {
+    if (stageConfig.getProgressable() != null) {
+      stageConfig.getProgressable().progress();
+    }
+  }
+
+  /**
+   * Get a file status value or, if the path doesn't exist, return null.
+   * @param path path
+   * @return status or null
+   * @throws IOException IO Failure.
+   */
+  protected final FileStatus getFileStatusOrNull(
+      final Path path)
+      throws IOException {
+    try {
+      return getFileStatus(path);
+    } catch (FileNotFoundException e) {
+      return null;
+    }
+  }
+
+  /**
+   * Get a file status value or, if the path doesn't exist, return null.
+   * @param path path
+   * @return status or null
+   * @throws IOException IO Failure.
+   */
+  protected final FileStatus getFileStatus(
+      final Path path)
+      throws IOException {
+    LOG.trace("{}: getFileStatus('{}')", getName(), path);
+    requireNonNull(path,
+        () -> String.format("%s: Null path for getFileStatus() call", getName()));
+    return trackDuration(getIOStatistics(), OP_GET_FILE_STATUS, () ->
+        operations.getFileStatus(path));
+  }
+
+  /**
+   * Get a file status value or, if the path doesn't exist, return null.
+   * @param path path
+   * @return true if the path resolves to a file
+   * @throws IOException IO Failure.
+   */
+  protected final boolean isFile(
+      final Path path)
+      throws IOException {
+    LOG.trace("{}: isFile('{}')", getName(), path);
+    return trackDuration(getIOStatistics(), OP_IS_FILE, () -> {
+      return operations.isFile(path);
+    });
+  }
+
+  /**
+   * Delete a path.
+   * @param path path
+   * @param recursive recursive delete.
+   * @return status or null
+   * @throws IOException IO Failure.
+   */
+  protected final boolean delete(
+      final Path path,
+      final boolean recursive)
+      throws IOException {
+    LOG.trace("{}: delete('{}, {}')", getName(), path, recursive);
+    return delete(path, recursive, OP_DELETE);
+  }
+
+  /**
+   * Delete a path.
+   * @param path path
+   * @param recursive recursive delete.
+   * @param statistic statistic to update
+   * @return status or null
+   * @throws IOException IO Failure.
+   */
+  protected Boolean delete(
+      final Path path,
+      final boolean recursive,
+      final String statistic)
+      throws IOException {
+    return trackDuration(getIOStatistics(), statistic, () -> {
+      return operations.delete(path, recursive);
+    });
+  }
+
+  /**
+   * Create a directory.
+   * @param path path
+   * @param escalateFailure escalate "false" to PathIOE
+   * @return true if the directory was created/exists.
+   * @throws IOException IO Failure.
+   */
+  protected final boolean mkdirs(
+      final Path path,
+      final boolean escalateFailure)
+      throws IOException {
+    LOG.trace("{}: mkdirs('{}')", getName(), path);
+    return trackDuration(getIOStatistics(), OP_MKDIRS, () -> {
+      boolean success = operations.mkdirs(path);
+      if (!success && escalateFailure) {
+        throw new PathIOException(path.toUri().toString(),
+            stageStatisticName + ": mkdirs() returned false");
+      }
+      return success;
+    });
+
+  }
+
+  /**
+   * List all directly files under a path.
+   * Async implementations may under-report their durations.
+   * @param path path
+   * @return iterator over the results.
+   * @throws IOException IO Failure.
+   */
+  protected final RemoteIterator<FileStatus> listStatusIterator(
+      final Path path)
+      throws IOException {
+    LOG.trace("{}: listStatusIterator('{}')", getName(), path);
+    return trackDuration(getIOStatistics(), OP_LIST_STATUS, () ->
+        operations.listStatusIterator(path));
+  }
+
+  /**
+   * Load a manifest file.
+   * @param status source.
+   * @return the manifest.
+   * @throws IOException IO Failure.
+   */
+  protected final TaskManifest loadManifest(
+      final FileStatus status)
+      throws IOException {
+    LOG.trace("{}: loadManifest('{}')", getName(), status);
+    return trackDuration(getIOStatistics(), OP_LOAD_MANIFEST, () ->
+        operations.loadTaskManifest(
+            stageConfig.currentManifestSerializer(),
+            status));
+  }
+
+  /**
+   * List all the manifests in the task manifest dir.
+   * @return a iterator of manifests.
+   * @throws IOException IO Failure.
+   */
+  protected final RemoteIterator<FileStatus> listManifests()
+      throws IOException {
+    return RemoteIterators.filteringRemoteIterator(
+        listStatusIterator(getTaskManifestDir()),
+        st -> st.getPath().toUri().toString().endsWith(MANIFEST_SUFFIX));
+  }
+
+  /**
+   * Make an msync() call; swallow when unsupported.
+   * @param path path
+   * @throws IOException IO failure
+   */
+  protected final void msync(Path path) throws IOException {
+    LOG.trace("{}: msync('{}')", getName(), path);
+    trackDurationOfInvocation(getIOStatistics(), OP_MSYNC, () ->
+        operations.msync(path));
+  }
+
+  /**
+   * Create a directory -failing if it exists or if
+   * mkdirs() failed.
+   * @param operation operation for error reporting.
+   * @param path path path to create.
+   * @return the path.
+   * @throws IOException failure
+   * @throws PathIOException mkdirs failed.
+   * @throws FileAlreadyExistsException destination exists.
+   */
+  protected final Path createNewDirectory(
+      final String operation,
+      final Path path) throws IOException {
+    LOG.trace("{}: {} createNewDirectory('{}')", getName(), operation, path);
+    requireNonNull(path,
+        () -> String.format("%s: Null path for operation %s", getName(), operation));
+    // check for dir existence before trying to create.
+    try {
+      final FileStatus status = getFileStatus(path);
+      // no exception, so the path exists.
+      throw new FileAlreadyExistsException(operation
+          + ": path " + path
+          + " already exists and has status " + status);
+    } catch (FileNotFoundException e) {
+      // the path does not exist, so create it.
+      mkdirs(path, true);
+      return path;
+    }
+  }
+
+  /**
+   * Assert that a path is a directory which must exist.
+   * @param operation operation for error reporting.
+   * @param path path path to create.
+   * @return the path
+   * @throws IOException failure
+   * @throws PathIOException mkdirs failed.
+   * @throws FileAlreadyExistsException destination exists.
+   */
+  protected final Path directoryMustExist(
+      final String operation,
+      final Path path) throws IOException {
+    final FileStatus status = getFileStatus(path);
+    if (!status.isDirectory()) {
+      throw new PathIOException(path.toString(),
+          operation
+              + ": Path is not a directory; its status is :" + status);
+    }
+    return path;
+  }
+
+  /**
+   * Save a task manifest or summary. This will be done by
+   * writing to a temp path and then renaming.
+   * If the destination path exists: Delete it.
+   * @param manifestData the manifest/success file
+   * @param tempPath temp path for the initial save
+   * @param finalPath final path for rename.
+   * @throws IOException failure to load/parse
+   */
+  @SuppressWarnings("unchecked")
+  protected final <T extends AbstractManifestData> void save(T manifestData,
+      final Path tempPath,
+      final Path finalPath) throws IOException {
+    LOG.trace("{}: save('{}, {}, {}')", getName(), manifestData, tempPath, finalPath);
+    trackDurationOfInvocation(getIOStatistics(), OP_SAVE_TASK_MANIFEST, () ->
+        operations.save(manifestData, tempPath, true));
+    renameFile(tempPath, finalPath);
+  }
+
+  /**
+   * Get an etag from a FileStatus which MUST BE
+   * an implementation of EtagSource and
+   * whose etag MUST NOT BE null/empty.
+   * @param status the status; may be null.
+   * @return the etag or null if not provided
+   */
+  public String getEtag(FileStatus status) {
+    return operations.getEtag(status);
+  }
+
+  /**
+   * Rename a file from source to dest; if the underlying FS API call
+   * returned false that's escalated to an IOE.
+   * @param source source file.
+   * @param dest dest file
+   * @throws IOException failure
+   * @throws PathIOException if the rename() call returned false.
+   */
+  protected final void renameFile(final Path source, final Path dest)
+      throws IOException {
+    maybeDeleteDest(true, dest);
+    executeRenamingOperation("renameFile", source, dest,
+        OP_RENAME_FILE, () ->
+            operations.renameFile(source, dest));
+  }
+
+  /**
+   * Rename a file from source to dest; if the underlying FS API call
+   * returned false that's escalated to an IOE.
+   * @param source source file.
+   * @param dest dest file
+   * @throws IOException failure
+   * @throws PathIOException if the rename() call returned false.
+   */
+  protected final void renameDir(final Path source, final Path dest)
+      throws IOException {
+
+    maybeDeleteDest(true, dest);
+    executeRenamingOperation("renameDir", source, dest,
+        OP_RENAME_FILE, () ->
+        operations.renameDir(source, dest)
+    );
+  }
+
+  /**
+   * Commit a file from the manifest using rename or, if available, resilient renaming.
+   * @param entry entry from manifest
+   * @throws PathIOException if the rename() call returned false and was uprated.
+   * @throws IOException failure
+   */
+  protected final CommitOutcome commitFile(FileEntry entry,
+      boolean deleteDest)
+      throws IOException {
+
+    final Path source = entry.getSourcePath();
+    final Path dest = entry.getDestPath();
+
+    maybeDeleteDest(deleteDest, dest);
+    if (storeSupportsResilientCommit()) {
+      // get the commit permits
+      final ManifestStoreOperations.CommitFileResult result = trackDuration(getIOStatistics(),
+          OP_COMMIT_FILE_RENAME, () ->
+              operations.commitFile(entry));
+      if (result.recovered()) {
+        // recovery took place.
+        getIOStatistics().incrementCounter(OP_COMMIT_FILE_RENAME_RECOVERED);
+      }
+      if (result.getWaitTime() != null) {
+        // note any delay which took place
+        noteAnyRateLimiting(STORE_IO_RATE_LIMITED, result.getWaitTime());
+      }
+    } else {
+      // commit with a simple rename; failures will be escalated.
+      executeRenamingOperation("renameFile", source, dest,
+          OP_COMMIT_FILE_RENAME, () ->
+              operations.renameFile(source, dest));
+    }
+    return new CommitOutcome();
+  }
+
+  /**
+   * Does this store support resilient commit.
+   * @return true if resilient commit operations are available.
+   */
+  protected boolean storeSupportsResilientCommit() {
+    return operations.storeSupportsResilientCommit();
+  }
+
+  private void maybeDeleteDest(final boolean deleteDest, final Path dest) throws IOException {
+    if (deleteDest) {
+      // delete the destination, always, knowing that it's a no-op if
+      // the data isn't there. Skipping the change saves one round trip
+      // to actually look for the file/object
+      boolean deleted = delete(dest, true);
+      // log the outcome in case of emergency diagnostics traces
+      // being needed.
+      LOG.debug("{}: delete('{}') returned {}'", getName(), dest, deleted);
+    }
+  }
+
+  /**
+   * Execute an operation to rename a file/dir, commit a manifest entry.
+   * The statistic is tracked; returning false from the operation is considered
+   * a failure from the statistics perspective.
+   * @param operation operation name
+   * @param source source path
+   * @param dest dest path
+   * @param statistic statistic to track
+   * @param action callable of the operation
+   * @throws IOException on any failure
+   */
+  private void executeRenamingOperation(String operation,
+      Path source,
+      Path dest,
+      String statistic,
+      CallableRaisingIOE<Boolean> action) throws IOException {
+
+    LOG.debug("{}: {} '{}' to '{}')", getName(), operation, source, dest);
+    requireNonNull(source, "Null source");
+    requireNonNull(dest, "Null dest");
+
+    // duration tracking is a bit convoluted as it
+    // ensures that rename failures as well as IOEs are
+    // treated as failures from a statistics perspective.
+
+    DurationTracker tracker = createTracker(getIOStatistics(), statistic);
+    boolean success;
+    try {
+      success = action.apply();
+      if (!success) {
+        // record failure in the tracker before closing it
+        tracker.failed();
+      }
+    } catch (IOException | RuntimeException e) {
+      LOG.info("{}: {} raised an exception: {}", getName(), operation, e.toString());
+      LOG.debug("{}: {} stack trace", getName(), operation, e);
+      tracker.failed();
+      throw e;
+    } finally {
+      // success
+      // update the tracker.
+      tracker.close();
+    }
+    // escalate the failure; this is done out of the duration tracker
+    // so its file status probes aren't included.
+    if (!success) {
+      throw escalateRenameFailure(operation, source, dest);
+    }
+  }
+
+  /**
+   * Escalate a rename failure to an exception.
+   * Returns an error exception to throw if one was not
+   * triggered when probing for the source.
+   * @param operation operation name
+   * @param source source path
+   * @param dest dest path
+   * @return an exception to throw
+   * @throws IOException raised probing for source or dest
+   */
+  private PathIOException escalateRenameFailure(String operation,
+      Path source, Path dest) throws IOException {
+    // rename just returned false.
+    // collect information for a meaningful error message
+    // and include in an exception raised.
+
+    // get the source status; this will implicitly raise a FNFE.
+    final FileStatus sourceStatus = getFileStatus(source);
+
+    // and look to see if there is anything at the destination
+    final FileStatus destStatus = getFileStatusOrNull(dest);
+
+    LOG.error("{}: failure to {} {} to {} with" +
+            " source status {} " +
+            " and destination status {}",
+        getName(), operation, source, dest,
+        sourceStatus, destStatus);
+
+    return new PathIOException(source.toString(),
+        FAILED_TO_RENAME_PREFIX + operation + " to " + dest);
+  }
+
+  /**
+   * Outcome from the commit.
+   */
+  public static final class CommitOutcome {
+
+  }
+
+  /**
+   * Job ID: never null.
+   */
+  protected final String getJobId() {
+    return stageConfig.getJobId();
+  }
+
+  /**
+   * Job attempt number.
+   */
+  protected final int getJobAttemptNumber() {
+    return stageConfig.getJobAttemptNumber();
+  }
+
+  /**
+   * ID of the task.
+   */
+  protected final String getTaskId() {
+    return stageConfig.getTaskId();
+  }
+
+  /**
+   * Get the task ID; raise an NPE
+   * if it is null.
+   * @return a non-null task ID.
+   */
+  protected final String getRequiredTaskId() {
+    return requireNonNull(getTaskId(),
+        "No Task ID in stage config");
+  }
+
+  /**
+   * ID of this specific attempt at a task.
+   */
+  protected final String getTaskAttemptId() {
+    return stageConfig.getTaskAttemptId();
+  }
+
+  /**
+   * Get the task attempt ID; raise an NPE
+   * if it is null.
+   * @return a non-null task attempt ID.
+   */
+  protected final String getRequiredTaskAttemptId() {
+    return requireNonNull(getTaskAttemptId(),
+        "No Task Attempt ID in stage config");
+  }
+
+  /**
+   * Job attempt dir.
+   */
+  protected final Path getJobAttemptDir() {
+    return stageConfig.getJobAttemptDir();
+  }
+
+  /**
+   * Directory to put task manifests into.
+   * @return a path under the job attempt dir.
+   */
+  protected final Path getTaskManifestDir() {
+    return stageConfig.getTaskManifestDir();
+  }
+
+
+  /**
+   * Task attempt dir.
+   */
+  protected final Path getTaskAttemptDir() {
+    return stageConfig.getTaskAttemptDir();
+  }
+
+  /**
+   * Get the task attemptDir; raise an NPE
+   * if it is null.
+   * @return a non-null task attempt dir.
+   */
+  protected final Path getRequiredTaskAttemptDir() {
+    return requireNonNull(getTaskAttemptDir(),
+        "No Task Attempt Dir");
+  }
+
+  /**
+   * Destination of job.
+   */
+  protected final Path getDestinationDir() {
+    return stageConfig.getDestinationDir();
+  }
+
+  /**
+   * Stage confog name, for logging.
+   * @return name.
+   */
+  public final String getName() {
+    return name;
+  }
+
+  /**
+   * Submitter for doing IO against the store other than
+   * manifest processing.
+   */
+  protected final TaskPool.Submitter getIOProcessors() {
+    return ioProcessors;
+  }
+
+  /**
+   * Submitter for doing IO against the store other than
+   * manifest processing.
+   * The size parameter is used to select between sequential
+   * and parallel runners.
+   * no data, or one entry: serial.
+   * everything else, parallel.
+   * @param size number of items.
+   * @return a submitter or null
+   */
+  protected final TaskPool.Submitter getIOProcessors(int size) {
+    return size > 1
+        ? getIOProcessors()
+        : null;
+  }
+
+  /**
+   * Delete a directory, possibly suppressing exceptions.
+   * @param dir directory.
+   * @param suppressExceptions should exceptions be suppressed?
+   * @throws IOException exceptions raised in delete if not suppressed.
+   * @return any exception caught and suppressed
+   */
+  protected IOException deleteDir(
+      final Path dir,
+      final Boolean suppressExceptions)
+      throws IOException {
+    try {
+      delete(dir, true);
+      return null;
+    } catch (IOException ex) {
+      LOG.info("Error deleting {}: {}", dir, ex.toString());
+      if (!suppressExceptions) {
+        throw ex;
+      } else {
+        return ex;
+      }
+    }
+  }
+
+  /**
+   * Create an entry for a file to rename under the destination.
+   * If the store operations supports extracting etags from file status
+   * entries, that is included in the entry
+   * @param status source file
+   * @param destDir destination directory
+   * @return an entry which includes the rename path
+   */
+  protected FileEntry fileEntry(FileStatus status, Path destDir) {
+    // generate a new path under the dest dir
+    Path dest = new Path(destDir, status.getPath().getName());
+    return new FileEntry(status.getPath(),
+        dest,
+        status.getLen(),
+        getEtag(status));
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java
new file mode 100644
index 0000000000000..4a5971a5b1cb0
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_CLEANUP_PARALLEL_DELETE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+
+/**
+ * Clean up a job's temporary directory through parallel delete,
+ * base _temporary delete.
+ * Returns: the outcome of the overall operation
+ * The result is detailed purely for the benefit of tests, which need
+ * to make assertions about error handling and fallbacks.
+ *
+ * There's a few known issues with the azure and GCS stores which
+ * this stage tries to address.
+ * - Google GCS directory deletion is O(entries), so is slower for big jobs.
+ * - Azure storage directory delete, when using OAuth authentication or
+ *   when not the store owner triggers a scan down the tree to verify the
+ *   caller has the permission to delete each subdir.
+ *   If this scan takes over 90s, the operation can time out.
+ *
+ * The main solution for both of these is that task attempts are
+ * deleted in parallel, in different threads.
+ * This will speed up GCS cleanup and reduce the risk of
+ * abfs related timeouts.
+ * Exceptions during cleanup can be suppressed,
+ * so that these do not cause the job to fail.
+ *
+ * Also, some users want to be able to run multiple independent jobs
+ * targeting the same output directory simultaneously.
+ * If one job deletes the directory `__temporary` all the others
+ * will fail.
+ *
+ * This can be addressed by disabling cleanup entirely.
+ *
+ */
+public class CleanupJobStage extends
+    AbstractJobOrTaskStage<
+            CleanupJobStage.Arguments,
+            CleanupJobStage.Result> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CleanupJobStage.class);
+
+  /**
+   * Count of deleted directories.
+   */
+  private final AtomicInteger deleteDirCount = new AtomicInteger();
+
+  /**
+   * Count of delete failures.
+   */
+  private final AtomicInteger deleteFailureCount = new AtomicInteger();
+
+  /**
+   * Last delete exception; non null if deleteFailureCount is not zero.
+   */
+  private IOException lastDeleteException;
+
+  /**
+   * Stage name as passed in from arguments.
+   */
+  private String stageName = OP_STAGE_JOB_CLEANUP;
+
+  public CleanupJobStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_CLEANUP, true);
+  }
+
+  /**
+   * Statistic name is extracted from the arguments.
+   * @param arguments args to the invocation.
+   * @return stage name.
+   */
+  @Override
+  protected String getStageStatisticName(Arguments arguments) {
+    return arguments.statisticName;
+  }
+
+  /**
+   * Clean up the job attempt directory tree.
+   * @param args arguments built up.
+   * @return the result.
+   * @throws IOException failure was raised an exceptions weren't surpressed.
+   */
+  @Override
+  protected Result executeStage(
+      final Arguments args)
+      throws IOException {
+    stageName = getStageName(args);
+    // this is $dest/_temporary
+    final Path baseDir = requireNonNull(getStageConfig().getOutputTempSubDir());
+    LOG.debug("{}: Cleaup of directory {} with {}", getName(), baseDir, args);
+    if (!args.enabled) {
+      LOG.info("{}: Cleanup of {} disabled", getName(), baseDir);
+      return new Result(Outcome.DISABLED, baseDir,
+          0, null);
+    }
+    // shortcut of a single existence check before anything else
+    if (getFileStatusOrNull(baseDir) == null) {
+      return new Result(Outcome.NOTHING_TO_CLEAN_UP,
+          baseDir,
+          0, null);
+    }
+
+    Outcome outcome = null;
+    IOException exception;
+
+
+    // to delete.
+    LOG.info("{}: Deleting job directory {}", getName(), baseDir);
+
+    if (args.deleteTaskAttemptDirsInParallel) {
+      // Attempt to do a parallel delete of task attempt dirs;
+      // don't overreact if a delete fails, but stop trying
+      // to delete the others, and fall back to deleting the
+      // job dir.
+      Path taskSubDir
+          = getStageConfig().getJobAttemptTaskSubDir();
+      try (DurationInfo info = new DurationInfo(LOG,
+          "parallel deletion of task attempts in %s",
+          taskSubDir)) {
+        RemoteIterator<FileStatus> dirs =
+            RemoteIterators.filteringRemoteIterator(
+                listStatusIterator(taskSubDir),
+                FileStatus::isDirectory);
+        TaskPool.foreach(dirs)
+            .executeWith(getIOProcessors())
+            .stopOnFailure()
+            .suppressExceptions(false)
+            .run(this::rmTaskAttemptDir);
+        getIOStatistics().aggregate((retrieveIOStatistics(dirs)));
+
+        if (getLastDeleteException() != null) {
+          // one of the task attempts failed.
+          throw getLastDeleteException();
+        }
+        // success: record this as the outcome.
+        outcome = Outcome.PARALLEL_DELETE;
+      } catch (FileNotFoundException ex) {
+        // not a problem if there's no dir to list.
+        LOG.debug("{}: Task attempt dir {} not found", getName(), taskSubDir);
+        outcome = Outcome.DELETED;
+      } catch (IOException ex) {
+        // failure. Log and continue
+        LOG.info(
+            "{}: Exception while listing/deleting task attempts under {}; continuing",
+            getName(),
+            taskSubDir, ex);
+        // not overreacting here as the base delete will still get executing
+        outcome = Outcome.DELETED;
+      }
+    }
+    // Now the top-level deletion; exception gets saved
+    exception = deleteOneDir(baseDir);
+    if (exception != null) {
+      // failure, report and continue
+      // assume failure.
+      outcome = Outcome.FAILURE;
+    } else {
+      // if the outcome isn't already recorded as parallel delete,
+      // mark is a simple delete.
+      if (outcome == null) {
+        outcome = Outcome.DELETED;
+      }
+    }
+
+    Result result = new Result(
+        outcome,
+        baseDir,
+        deleteDirCount.get(),
+        exception);
+    if (!result.succeeded() && !args.suppressExceptions) {
+      result.maybeRethrowException();
+    }
+
+    return result;
+  }
+
+  /**
+   * Delete a single TA dir in a parallel task.
+   * Updates the audit context.
+   * Exceptions are swallowed so that attempts are still made
+   * to delete the others, but the first exception
+   * caught is saved in a field which can be retrieved
+   * via {@link #getLastDeleteException()}.
+   *
+   * @param status dir to be deleted.
+   * @throws IOException delete failure.
+   */
+  private void rmTaskAttemptDir(FileStatus status) throws IOException {
+    // stage name in audit context is the one set in the arguments.
+    updateAuditContext(stageName);
+    // update the progress callback in case delete is really slow.
+    progress();
+    deleteOneDir(status.getPath());
+  }
+
+  /**
+   * Delete a directory.
+   * The {@link #deleteFailureCount} counter.
+   * is incremented on every failure.
+   * @param dir directory
+   * @throws IOException if an IOE was raised
+   * @return any IOE raised.
+   */
+  private IOException deleteOneDir(final Path dir)
+      throws IOException {
+
+    deleteDirCount.incrementAndGet();
+    IOException ex = deleteDir(dir, true);
+    if (ex != null) {
+      deleteFailure(ex);
+    }
+    return ex;
+  }
+
+  /**
+   * Note a failure.
+   * @param ex exception
+   */
+  private synchronized void deleteFailure(IOException ex) {
+    // excaption: add the count
+    deleteFailureCount.incrementAndGet();
+    lastDeleteException = ex;
+  }
+
+  /**
+   * Get the last delete exception; synchronized.
+   * @return the last delete exception or null.
+   */
+  public synchronized IOException getLastDeleteException() {
+    return lastDeleteException;
+  }
+
+  /**
+   * Options to pass down to the cleanup stage.
+   */
+  public static final class Arguments {
+
+    /**
+     * Statistic to update.
+     */
+    private final String statisticName;
+
+    /** Delete is enabled? */
+    private final boolean enabled;
+
+    /** Attempt parallel delete of task attempt dirs? */
+    private final boolean deleteTaskAttemptDirsInParallel;
+
+    /** Ignore failures? */
+    private final boolean suppressExceptions;
+
+    /**
+     * Arguments to the stage.
+     * @param statisticName stage name to report
+     * @param enabled is the stage enabled?
+     * @param deleteTaskAttemptDirsInParallel delete task attempt dirs in
+     *        parallel?
+     * @param suppressExceptions suppress exceptions?
+     */
+    public Arguments(
+        final String statisticName,
+        final boolean enabled,
+        final boolean deleteTaskAttemptDirsInParallel,
+        final boolean suppressExceptions) {
+      this.statisticName = statisticName;
+      this.enabled = enabled;
+      this.deleteTaskAttemptDirsInParallel = deleteTaskAttemptDirsInParallel;
+      this.suppressExceptions = suppressExceptions;
+    }
+
+    public String getStatisticName() {
+      return statisticName;
+    }
+
+    public boolean isEnabled() {
+      return enabled;
+    }
+
+    public boolean isDeleteTaskAttemptDirsInParallel() {
+      return deleteTaskAttemptDirsInParallel;
+    }
+
+    public boolean isSuppressExceptions() {
+      return suppressExceptions;
+    }
+
+    @Override
+    public String toString() {
+      return "Arguments{" +
+          "statisticName='" + statisticName + '\''
+          + ", enabled=" + enabled
+          + ", deleteTaskAttemptDirsInParallel="
+          + deleteTaskAttemptDirsInParallel
+          + ", suppressExceptions=" + suppressExceptions
+          + '}';
+    }
+  }
+
+  /**
+   * Static disabled arguments.
+   */
+  public static final Arguments DISABLED = new Arguments(OP_STAGE_JOB_CLEANUP,
+      false,
+      false,
+      false
+  );
+
+  /**
+   * Build an options argument from a configuration, using the
+   * settings from FileOutputCommitter and manifest committer.
+   * @param statisticName statistic name to use in duration tracking.
+   * @param conf configuration to use.
+   * @return the options to process
+   */
+  public static Arguments cleanupStageOptionsFromConfig(
+      String statisticName, Configuration conf) {
+
+    boolean enabled = !conf.getBoolean(FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED,
+        FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT);
+    boolean suppressExceptions = conf.getBoolean(
+        FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED,
+        FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT);
+    boolean deleteTaskAttemptDirsInParallel = conf.getBoolean(
+        OPT_CLEANUP_PARALLEL_DELETE,
+        OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT);
+    return new Arguments(
+        statisticName,
+        enabled,
+        deleteTaskAttemptDirsInParallel,
+        suppressExceptions
+    );
+  }
+
+  /**
+   * Enum of outcomes.
+   */
+  public enum Outcome {
+    DISABLED("Disabled", false),
+    NOTHING_TO_CLEAN_UP("Nothing to clean up", true),
+    PARALLEL_DELETE("Parallel Delete of Task Attempt Directories", true),
+    DELETED("Delete of job directory", true),
+    FAILURE("Delete failed", false);
+
+    private final String description;
+
+    private final boolean success;
+
+    Outcome(String description, boolean success) {
+      this.description = description;
+      this.success = success;
+    }
+
+    @Override
+    public String toString() {
+      return "Outcome{" + name() +
+          " '" + description + '\'' +
+          "}";
+    }
+
+    /**
+     * description.
+     * @return text for logging
+     */
+    public String getDescription() {
+      return description;
+    }
+
+    /**
+     * Was this a success?
+     * @return true if this outcome is good.
+     */
+    public boolean isSuccess() {
+      return success;
+    }
+  }
+
+  /**
+   * Result of the cleanup.
+   * If the outcome == FAILURE but exceptions were suppressed
+   * (which they are implicitly if an instance of this object
+   * is created and returned), then the exception
+   * MUST NOT be null.
+   */
+  public static final class Result {
+
+    /** Outcome. */
+    private final Outcome outcome;
+
+    /** Directory cleaned up. */
+    private final Path directory;
+
+    /**
+     * Number of delete calls made across all threads.
+     */
+    private final int deleteCalls;
+
+    /**
+     * Any IOE raised.
+     */
+    private final IOException exception;
+
+    public Result(
+        final Outcome outcome,
+        final Path directory,
+        final int deleteCalls,
+        IOException exception) {
+      this.outcome = requireNonNull(outcome, "outcome");
+      this.directory = directory;
+      this.deleteCalls = deleteCalls;
+      this.exception = exception;
+      if (outcome == Outcome.FAILURE) {
+        requireNonNull(exception, "No exception in failure result");
+      }
+    }
+
+    public Path getDirectory() {
+      return directory;
+    }
+
+    public boolean wasExecuted() {
+      return outcome != Outcome.DISABLED;
+    }
+
+    /**
+     * Was the outcome a success?
+     * That is: either the dir wasn't there or through
+     * delete/rename it is no longer there.
+     * @return true if the temporary dir no longer exists.
+     */
+    public boolean succeeded() {
+      return outcome.isSuccess();
+    }
+
+    public Outcome getOutcome() {
+      return outcome;
+    }
+
+    public int getDeleteCalls() {
+      return deleteCalls;
+    }
+
+    public IOException getException() {
+      return exception;
+    }
+
+    /**
+     * If there was an IOE caught, throw it.
+     * For ease of use in (meaningful) lambda expressions
+     * in tests, returns the string value if there
+     * was no exception to throw (for use in tests)
+     * @throws IOException exception.
+     */
+    public String maybeRethrowException() throws IOException {
+      if (exception != null) {
+        throw exception;
+      }
+      return toString();
+    }
+
+    @Override
+    public String toString() {
+      return "CleanupResult{" +
+          "outcome=" + outcome +
+          ", directory=" + directory +
+          ", deleteCalls=" + deleteCalls +
+          ", exception=" + exception +
+          '}';
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java
new file mode 100644
index 0000000000000..a754f2a2da8df
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_BYTES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_FILES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.MANIFESTS;
+
+/**
+ * Commit the Job.
+ * Arguments (save manifest, validate output)
+ * Inputs: saveMarker: boolean, validateOutput: boolean
+ * Outputs: SuccessData
+ */
+public class CommitJobStage extends
+    AbstractJobOrTaskStage<
+            CommitJobStage.Arguments,
+            CommitJobStage.Result> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CommitJobStage.class);
+
+  public CommitJobStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_COMMIT, true);
+  }
+
+  @Override
+  protected CommitJobStage.Result executeStage(
+      final CommitJobStage.Arguments arguments) throws IOException {
+
+    LOG.info("{}: Committing job \"{}\". resilient commit supported = {}",
+        getName(),
+        getJobId(),
+        storeSupportsResilientCommit());
+
+    boolean createMarker = arguments.isCreateMarker();
+
+    // load the manifests
+    final StageConfig stageConfig = getStageConfig();
+    LoadManifestsStage.Result result
+        = new LoadManifestsStage(stageConfig).apply(true);
+    List<TaskManifest> manifests = result.getManifests();
+    LoadManifestsStage.SummaryInfo summary = result.getSummary();
+
+    LOG.debug("{}: Job Summary {}", getName(), summary);
+    LOG.info("{}: Committing job with file count: {}; total size {} bytes",
+        getName(),
+        summary.getFileCount(),
+        byteCountToDisplaySize(summary.getTotalFileSize()));
+
+
+    // add in the manifest statistics to our local IOStatistics for
+    // reporting.
+    IOStatisticsStore iostats = getIOStatistics();
+    iostats.aggregate(summary.getIOStatistics());
+
+    // prepare destination directories.
+    final CreateOutputDirectoriesStage.Result dirStageResults =
+        new CreateOutputDirectoriesStage(stageConfig)
+            .apply(manifests);
+
+    // commit all the tasks.
+    // The success data includes a snapshot of the IO Statistics
+    // and hence all aggregate stats from the tasks.
+    ManifestSuccessData successData;
+    successData = new RenameFilesStage(stageConfig).apply(
+        Pair.of(manifests, dirStageResults.getCreatedDirectories()));
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("{}: _SUCCESS file summary {}", getName(), successData.toJson());
+    }
+    // update the counter of bytes committed and files.
+    // use setCounter so as to ignore any values accumulated when
+    // aggregating tasks.
+    iostats.setCounter(
+        COMMITTER_FILES_COMMITTED_COUNT,
+        summary.getFileCount());
+    iostats.setCounter(
+        COMMITTER_BYTES_COMMITTED_COUNT,
+        summary.getTotalFileSize());
+    successData.snapshotIOStatistics(iostats);
+
+
+    // rename manifests. Only warn on failure here.
+    final String manifestRenameDir = arguments.getManifestRenameDir();
+    if (isNotBlank(manifestRenameDir)) {
+      Path manifestRenamePath = new Path(
+          new Path(manifestRenameDir),
+          getJobId());
+      LOG.info("{}: Renaming manifests to {}", getName(), manifestRenamePath);
+      try {
+        renameDir(getTaskManifestDir(), manifestRenamePath);
+
+        // save this path in the summary diagnostics
+        successData.getDiagnostics().put(MANIFESTS, manifestRenamePath.toUri().toString());
+      } catch (IOException | IllegalArgumentException e) {
+        // rename failure, including path for wrong filesystem
+        LOG.warn("{}: Failed to rename manifests to {}", getName(), manifestRenamePath, e);
+      }
+    }
+
+    // save the _SUCCESS if the option is enabled.
+    Path successPath = null;
+    if (createMarker) {
+      // save a snapshot of the IO Statistics
+
+      successPath = new SaveSuccessFileStage(stageConfig)
+          .apply(successData);
+      LOG.debug("{}: Saving _SUCCESS file to {}", getName(), successPath);
+    }
+
+    // optional cleanup
+    new CleanupJobStage(stageConfig).apply(arguments.getCleanupArguments());
+
+    // and then, after everything else: optionally validate.
+    if (arguments.isValidateOutput()) {
+      // cache and restore the active stage field
+      LOG.info("{}: Validating output.", getName());
+      new ValidateRenamedFilesStage(stageConfig)
+          .apply(result.getManifests());
+    }
+
+    // restore the active stage so that when the report is saved
+    // it is declared as job commit, not cleanup or validate.
+    stageConfig.enterStage(getStageName(arguments));
+
+    // the result
+    return new CommitJobStage.Result(successPath, successData);
+  }
+
+  /**
+   * Arguments for job commit.
+   */
+  public static final class Arguments {
+
+    /** create the _SUCCESS marker? */
+    private final boolean createMarker;
+
+    /** perform validation checks on the files? */
+    private final boolean validateOutput;
+
+    /** optional directory to rename the task manifests to. */
+    private final String manifestRenameDir;
+
+    /** cleanup arguments.. */
+    private final CleanupJobStage.Arguments cleanupArguments;
+
+    /**
+     *
+     * @param createMarker create the _SUCCESS marker?
+     * @param validateOutput perform validation checks on the files?
+     * @param manifestRenameDir optional directory to rename the task manifests to
+     * @param cleanupArguments cleanup arguments.
+     */
+    public Arguments(
+        boolean createMarker,
+        boolean validateOutput,
+        @Nullable String manifestRenameDir,
+        CleanupJobStage.Arguments cleanupArguments) {
+
+      this.createMarker = createMarker;
+      this.validateOutput = validateOutput;
+      this.manifestRenameDir = manifestRenameDir;
+      this.cleanupArguments = requireNonNull(cleanupArguments);
+    }
+
+    public boolean isCreateMarker() {
+      return createMarker;
+    }
+
+    public boolean isValidateOutput() {
+      return validateOutput;
+    }
+
+    public String getManifestRenameDir() {
+      return manifestRenameDir;
+    }
+
+    public CleanupJobStage.Arguments getCleanupArguments() {
+      return cleanupArguments;
+    }
+  }
+
+  /**
+   * Result of the stage.
+   */
+  public static final class Result {
+    /**
+     * Manifest success data.
+     */
+    private final ManifestSuccessData jobSuccessData;
+
+    /**
+     * Success file path. null if not saved.
+     */
+    private final Path successPath;
+
+    public Result(final Path successPath,
+        ManifestSuccessData jobSuccessData) {
+      this.successPath = successPath;
+      this.jobSuccessData = jobSuccessData;
+    }
+
+    public ManifestSuccessData getJobSuccessData() {
+      return jobSuccessData;
+    }
+
+    public Path getSuccessPath() {
+      return successPath;
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java
new file mode 100644
index 0000000000000..bf5ba27ab8ad5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_COMMIT;
+
+/**
+ * Commit a task attempt.
+ * Scan the task attempt directories through
+ * {@link TaskAttemptScanDirectoryStage}
+ * and then save to the task manifest path at
+ * {@link SaveTaskManifestStage}.
+ */
+
+public class CommitTaskStage extends
+    AbstractJobOrTaskStage<Void, CommitTaskStage.Result> {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CommitTaskStage.class);
+
+  public CommitTaskStage(final StageConfig stageConfig) {
+    super(true, stageConfig, OP_STAGE_TASK_COMMIT, false);
+  }
+
+  /**
+   * Scan the task attempt dir then save the manifest.
+   * A snapshot of the IOStats will be included in the manifest;
+   * this includes the scan time.
+   * @param arguments arguments to the function.
+   * @return the path the manifest was saved to, and the manifest.
+   * @throws IOException IO failure.
+   */
+  @Override
+  protected CommitTaskStage.Result executeStage(final Void arguments)
+      throws IOException {
+    LOG.info("{}: Committing task \"{}\"", getName(), getTaskAttemptId());
+
+    // execute the scan
+    final TaskAttemptScanDirectoryStage scanStage =
+        new TaskAttemptScanDirectoryStage(getStageConfig());
+    TaskManifest manifest = scanStage.apply(arguments);
+
+    // add the scan as task commit. It's not quite, as it doesn't include
+    // the saving, but ...
+    scanStage.addExecutionDurationToStatistics(getIOStatistics(), OP_STAGE_TASK_COMMIT);
+
+    // save a snapshot of the IO Statistics
+    final IOStatisticsSnapshot manifestStats = snapshotIOStatistics();
+    manifestStats.aggregate(getIOStatistics());
+    manifest.setIOStatistics(manifestStats);
+
+    // Now save with rename
+    Path manifestPath = new SaveTaskManifestStage(getStageConfig())
+        .apply(manifest);
+    return new CommitTaskStage.Result(manifestPath, manifest);
+  }
+
+  /**
+   * Result of the stage.
+   */
+  public static final class Result {
+    /** The path the manifest was saved to. */
+    private final Path path;
+    /** The manifest. */
+    private final TaskManifest taskManifest;
+
+    public Result(Path path,
+        TaskManifest taskManifest) {
+      this.path = path;
+      this.taskManifest = taskManifest;
+    }
+
+    /**
+     * Get the manifest path.
+     * @return The path the manifest was saved to.
+     */
+    public Path getPath() {
+      return path;
+    }
+
+    /**
+     * Get the manifest.
+     * @return The manifest.
+     */
+    public TaskManifest getTaskManifest() {
+      return taskManifest;
+    }
+
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java
new file mode 100644
index 0000000000000..927004e619742
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.EntryStatus;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.measureDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_CREATE_DIRECTORIES;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_DELETE_FILE_UNDER_DESTINATION;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_MKDIRS_RETURNED_FALSE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_PREPARE_DIR_ANCESTORS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CREATE_TARGET_DIRS;
+import static org.apache.hadoop.util.OperationDuration.humanTime;
+
+/**
+ * Prepare the destination directory tree, as efficiently as possible.
+ * possible -and doing those IO operations in the thread pool.
+ *
+ * The classic FileOutputCommitter does a recursive treewalk and
+ * deletes any files found at paths where directories are to be created.
+ *
+ * Each task manifest's directories are combined with those of the other tasks
+ * to build a set of all directories which are needed, without duplicates.
+ *
+ * This stage requires the aggregate set of manifests to contain
+ * all directories to create, including level,
+ * and expects them to have been probed for existence/state.
+ *
+ * For each level, all dirs are processed in parallel to
+ * be created or, if files, deleted.
+ *
+ * The stage returns the list of directories created, and for testing,
+ * the map of paths to outcomes.
+ *
+ * Directory creation can be surprisingly slow against object stores,
+ * do use benchmarks from real test runs when tuning this algorithm.
+ */
+public class CreateOutputDirectoriesStage extends
+    AbstractJobOrTaskStage<
+        List<TaskManifest>,
+        CreateOutputDirectoriesStage.Result> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CreateOutputDirectoriesStage.class);
+
+  /**
+   * Directories as a map of (path, path).
+   * Using a map rather than any set for efficient concurrency; the
+   * concurrent sets don't do lookups so fast.
+   */
+  private final Map<Path, DirMapState> dirMap = new ConcurrentHashMap<>();
+
+  /**
+   * A list of created paths for the results.
+   */
+  private final List<Path> createdDirectories = new ArrayList<>();
+
+  public CreateOutputDirectoriesStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_CREATE_TARGET_DIRS, true);
+    // add the dest dir to the dir map as we expect the job setup to create it.
+    dirMap.put(getDestinationDir(), DirMapState.dirWasCreated);
+  }
+
+  @Override
+  protected Result executeStage(
+      final List<TaskManifest> taskManifests)
+      throws IOException {
+
+    final List<Path> directories = createAllDirectories(taskManifests);
+    LOG.debug("{}: Created {} directories", getName(), directories.size());
+    return new Result(new HashSet<>(directories), dirMap);
+  }
+
+  /**
+   * For each task, build the list of directories it wants.
+   * @param taskManifests task manifests
+   * @return the list of paths which have been created.
+   */
+  private List<Path> createAllDirectories(final List<TaskManifest> taskManifests)
+      throws IOException {
+
+    // all directories which need to exist across all
+    // tasks.
+    // leaf directories
+    final Map<Path, DirEntry> leaves = new HashMap<>();
+    // parent directories. these do not need to be
+    // explicitly created.
+    final Map<Path, DirEntry> parents = new HashMap<>();
+    // the files which must be deleted as a directory
+    // will be created at that path.
+    final Set<Path> filesToDelete = new HashSet<>();
+
+    // iterate through the task manifests
+    // and all output dirs into the set of dirs to
+    // create.
+    // hopefully there is a lot of overlap, so the
+    // final number of dirs to create is small.
+    for (TaskManifest task: taskManifests) {
+      final List<DirEntry> destDirectories = task.getDestDirectories();
+      Collections.sort(destDirectories, (o1, o2) ->
+          o1.getLevel() - o2.getLevel());
+      for (DirEntry entry: destDirectories) {
+        // add the dest entry
+        final Path path = entry.getDestPath();
+        if (!leaves.containsKey(path)) {
+          leaves.put(path, entry);
+
+          // if it is a file to delete, record this.
+          if (entry.getStatus() == EntryStatus.file) {
+            filesToDelete.add(path);
+          }
+          final Path parent = path.getParent();
+          if (parent != null && leaves.containsKey(parent)) {
+            // there's a parent dir, move it from the leaf list
+            // to parent list
+            parents.put(parent,
+                leaves.remove(parent));
+          }
+        }
+      }
+    }
+
+    // at this point then there is a map of all directories which
+    // are leaf entries and so need to be created if not present,
+    // and the maximum level is known.
+    // we can iterate through all levels deleting any files if there are any.
+
+    // Prepare parent directories.
+    deleteFiles(filesToDelete);
+
+    // Now the real work.
+    final int createCount = leaves.size();
+    LOG.info("Preparing {} directory/directories", createCount);
+    // now probe for and create the leaf dirs, which are those at the
+    // bottom level
+    Duration d = measureDurationOfInvocation(getIOStatistics(), OP_CREATE_DIRECTORIES, () ->
+        TaskPool.foreach(leaves.values())
+            .executeWith(getIOProcessors(createCount))
+            .onFailure(this::reportMkDirFailure)
+            .stopOnFailure()
+            .run(this::createOneDirectory));
+    LOG.info("Time to prepare directories {}", humanTime(d.toMillis()));
+    return createdDirectories;
+  }
+
+  /**
+   * How many failures have been reported.
+   */
+  private final AtomicInteger failureCount = new AtomicInteger();
+
+  /**
+   * report a single directory failure.
+   * @param path path which could not be deleted
+   * @param e exception raised.
+   */
+  private void reportMkDirFailure(DirEntry dirEntry, Exception e) {
+    Path path = dirEntry.getDestPath();
+    final int count = failureCount.incrementAndGet();
+    LOG.warn("{}: mkdir failure #{} Failed to create directory \"{}\": {}",
+        getName(), count, path, e.toString());
+    LOG.debug("{}: Full exception details",
+        getName(), e);
+  }
+
+  /**
+   * Delete all directories where there is a file.
+   * @param filesToDelete set of dirs to where there is a file.
+   * @throws IOException IO problem
+   */
+  private void deleteFiles(final Set<Path> filesToDelete)
+      throws IOException {
+
+    final int size = filesToDelete.size();
+    if (size == 0) {
+      // nothing to delete.
+      return;
+    }
+    LOG.info("{}: Directory entries containing files to delete: {}", getName(), size);
+    Duration d = measureDurationOfInvocation(getIOStatistics(),
+        OP_PREPARE_DIR_ANCESTORS, () ->
+            TaskPool.foreach(filesToDelete)
+                .executeWith(getIOProcessors(size))
+                .stopOnFailure()
+                .run(dir -> {
+                  updateAuditContext(OP_PREPARE_DIR_ANCESTORS);
+                  deleteDirWithFile(dir);
+                }));
+    LOG.info("Time to delete files {}", humanTime(d.toMillis()));
+  }
+
+  /**
+   * Prepare a parent directory.
+   * @param dir directory to probe
+   * @throws IOException failure in probe other than FNFE
+   */
+  private void deleteDirWithFile(Path dir) throws IOException {
+    // report progress back
+    progress();
+    LOG.info("{}: Deleting file {}", getName(), dir);
+    delete(dir, false, OP_DELETE);
+    // note its final state
+    addToDirectoryMap(dir, DirMapState.fileNowDeleted);
+  }
+
+
+  /**
+   * Create a directory is required, updating the directory map
+   * and, if the operation took place, the list of created dirs.
+   * Reports progress on invocation.
+   * @param dirEntry entry
+   * @throws IOException failure.
+   */
+  private void createOneDirectory(final DirEntry dirEntry) throws IOException {
+    // report progress back
+    progress();
+    final Path dir = dirEntry.getDestPath();
+    updateAuditContext(OP_STAGE_JOB_CREATE_TARGET_DIRS);
+    final DirMapState state = maybeCreateOneDirectory(dirEntry);
+    switch (state) {
+    case dirFoundInStore:
+      addToDirectoryMap(dir, state);
+      break;
+    case dirWasCreated:
+    case dirCreatedOnSecondAttempt:
+      addCreatedDirectory(dir);
+      addToDirectoryMap(dir, state);
+      break;
+    default:
+      break;
+    }
+
+  }
+
+
+  /**
+   * Try to efficiently and robustly create a directory in a method which is
+   * expected to be executed in parallel with operations creating
+   * peer directories.
+   * @param path path to create
+   * @return true if dir created/found
+   * @throws IOException IO Failure.
+   */
+  private DirMapState maybeCreateOneDirectory(DirEntry dirEntry) throws IOException {
+    final EntryStatus status = dirEntry.getStatus();
+    if (status == EntryStatus.dir) {
+      return DirMapState.dirFoundInStore;
+    }
+    // present in case directories are ever created in task commits
+    if (status == EntryStatus.created_dir) {
+      return DirMapState.dirWasCreated;
+    }
+
+    // here the dir doesn't exist because
+    // it was a file and has been deleted, or
+    // checks failed. create it.
+    final Path path = dirEntry.getDestPath();
+
+    LOG.info("Creating directory {}", path);
+
+    try {
+      if (mkdirs(path, false)) {
+        // success -return immediately.
+        return DirMapState.dirWasCreated;
+      }
+      getIOStatistics().incrementCounter(OP_MKDIRS_RETURNED_FALSE);
+
+      LOG.info("{}: mkdirs({}) returned false, attempting to recover",
+          getName(), path);
+    } catch (IOException e) {
+      // can be caused by file existing, etc.
+      LOG.info("{}: mkdir({}) raised exception {}", getName(), path, e.toString());
+      LOG.debug("{}: Mkdir stack", getName(), e);
+    }
+
+    // fallback to checking the FS, in case a different process did it.
+    FileStatus st = getFileStatusOrNull(path);
+    if (st != null) {
+      if (!st.isDirectory()) {
+        // is bad: delete a file
+        LOG.info("{}: Deleting file where a directory should go: {}",
+            getName(), st);
+        delete(path, false, OP_DELETE_FILE_UNDER_DESTINATION);
+      } else {
+        // is good.
+        LOG.warn("{}: Even though mkdirs({}) failed, there is now a directory there",
+            getName(), path);
+        return DirMapState.dirFoundInStore;
+      }
+    } else {
+      // nothing found. This should never happen.
+      LOG.warn("{}: Although mkdirs({}) returned false, there's nothing at that path to prevent it",
+          getName(), path);
+
+    }
+
+    // try to create the directory again
+    // if this fails, and IOE is still raised, that
+    // propagate to the caller.
+    if (!mkdirs(path, false)) {
+
+      // mkdirs failed again
+      getIOStatistics().incrementCounter(OP_MKDIRS_RETURNED_FALSE);
+
+      // require the dir to exist, raising an exception if it does not.
+      directoryMustExist("Creating directory ", path);
+    }
+
+    // we only get here if the second attempt recovered
+    return DirMapState.dirCreatedOnSecondAttempt;
+
+  }
+
+  /**
+   * Add a created dir to the list of created dirs.
+   * @param dir new dir.
+   */
+  private void addCreatedDirectory(final Path dir) {
+    synchronized (createdDirectories) {
+      createdDirectories.add(dir);
+    }
+  }
+
+  /**
+   * Add a dir  to the directory map if there is not already an entry there.
+   * @param dir directory.
+   * @param state state of entry
+   */
+  private void addToDirectoryMap(final Path dir,
+      DirMapState state) {
+    if (!dirMap.containsKey(dir)) {
+      dirMap.put(dir, state);
+    }
+  }
+
+
+  /**
+   * Result of the operation.
+   */
+  public static final class Result {
+
+    /** directories created. */
+    private final Set<Path> createdDirectories;
+
+    /**
+     * Map of dirs built up during preparation.
+     */
+    private final Map<Path, DirMapState> dirMap;
+
+    public Result(Set<Path> createdDirectories,
+        Map<Path, DirMapState> dirMap) {
+      this.createdDirectories = requireNonNull(createdDirectories);
+      this.dirMap = requireNonNull(dirMap);
+    }
+
+    public Set<Path> getCreatedDirectories() {
+      return createdDirectories;
+    }
+
+    public Map<Path, DirMapState> getDirMap() {
+      return dirMap;
+    }
+
+    @Override
+    public String toString() {
+      return "Result{" +
+          "directory count=" + createdDirectories.size() +
+          '}';
+    }
+  }
+
+  /**
+   * Enumeration of dir states in the dir map.
+   */
+  public enum DirMapState {
+    dirFoundInStore,
+    dirFoundInMap,
+    dirWasCreated,
+    dirCreatedOnSecondAttempt,
+    fileNowDeleted,
+    ancestorWasDirOrMissing,
+    parentWasNotFile,
+    parentOfCreatedDir
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java
new file mode 100644
index 0000000000000..98975bf7f195c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.util.functional.FunctionRaisingIOE;
+
+/**
+ * Job Stage is a function raising an IOE. and which also
+ * provides IOStats.
+ * @param <IN> Type of arguments to the stage.
+ * @param <OUT> Type of result.
+ */
+public interface JobOrTaskStage<IN, OUT> extends FunctionRaisingIOE<IN, OUT>,
+    IOStatisticsSource {
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java
new file mode 100644
index 0000000000000..3720f549d89c3
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_MANIFEST_FILE_SIZE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_LOAD_ALL_MANIFESTS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_LOAD_MANIFESTS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.maybeAddIOStatistics;
+
+/**
+ * Stage to load all the task manifests in the job attempt directory.
+ * Invoked in Job Commit.
+ * Manifests are loaded in parallel.
+ * The IOStatistics snapshot passed in is built up with the statistics
+ * and the statistics stripped from the manifest if prune == true.
+ * This keeps the memory footprint of each manifest down.
+ */
+public class LoadManifestsStage extends
+    AbstractJobOrTaskStage<
+        Boolean,
+        LoadManifestsStage.Result> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      LoadManifestsStage.class);
+
+  /**
+   * Summary of manifest loading.
+   */
+  private final SummaryInfo summaryInfo = new SummaryInfo();
+
+  /**
+   * Should manifests be pruned of IOStatistics?
+   */
+  private boolean pruneManifests;
+
+  /**
+   * List of loaded manifests.
+   */
+  private final List<TaskManifest> manifests = new ArrayList<>();
+
+  public LoadManifestsStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_LOAD_MANIFESTS, true);
+  }
+
+  /**
+   * Load the manifests.
+   * @param prune should manifests be pruned of IOStatistics?
+   * @return the summary and a list of manifests.
+   * @throws IOException IO failure.
+   */
+  @Override
+  protected LoadManifestsStage.Result executeStage(
+      final Boolean prune) throws IOException {
+
+    final Path manifestDir = getTaskManifestDir();
+    LOG.info("{}: Executing Manifest Job Commit with manifests in {}",
+        getName(),
+        manifestDir);
+    pruneManifests = prune;
+    // build a list of all task manifests successfully committed
+    //
+    msync(manifestDir);
+    final RemoteIterator<FileStatus> manifestFiles = listManifests();
+
+    final List<TaskManifest> manifestList = loadAllManifests(manifestFiles);
+    LOG.info("{}: Summary of {} manifests loaded in {}: {}",
+        getName(),
+        manifestList.size(),
+        manifestDir,
+        summaryInfo);
+
+    // collect any stats
+    maybeAddIOStatistics(getIOStatistics(), manifestFiles);
+    return new LoadManifestsStage.Result(summaryInfo, manifestList);
+  }
+
+  /**
+   * Load all the manifests.
+   * @param manifestFiles list of manifest files.
+   * @return the loaded manifests.
+   * @throws IOException IO Failure.
+   */
+  private List<TaskManifest> loadAllManifests(
+      final RemoteIterator<FileStatus> manifestFiles) throws IOException {
+
+    trackDurationOfInvocation(getIOStatistics(), OP_LOAD_ALL_MANIFESTS, () ->
+        TaskPool.foreach(manifestFiles)
+            .executeWith(getIOProcessors())
+            .stopOnFailure()
+            .run(this::processOneManifest));
+    return manifests;
+  }
+
+  /**
+   * Method invoked to process one manifest.
+   * @param status file to process.
+   * @throws IOException failure to load/parse
+   */
+  private void processOneManifest(FileStatus status)
+      throws IOException {
+    updateAuditContext(OP_LOAD_ALL_MANIFESTS);
+
+    TaskManifest m = fetchTaskManifest(status);
+    progress();
+
+    // update the manifest list in a synchronized block.
+
+    synchronized (manifests) {
+      manifests.add(m);
+      // and the summary info in the same block, to
+      // eliminate the need to acquire a second lock.
+      summaryInfo.add(m);
+    }
+    if (pruneManifests) {
+      m.setIOStatistics(null);
+      m.getExtraData().clear();
+    }
+  }
+
+  /**
+   * Precommit preparation of a single manifest file.
+   * To reduce the memory foot print, the IOStatistics and
+   * extra data of each manifest is cleared.
+   * @param status status of file.
+   * @return number of files.
+   * @throws IOException IO Failure.
+   */
+  private TaskManifest fetchTaskManifest(FileStatus status)
+      throws IOException {
+    if (status.getLen() == 0 || !status.isFile()) {
+      throw new PathIOException(status.getPath().toString(),
+          "Not a valid manifest file; file status = " + status);
+    }
+    // load the manifest, which includes validation.
+    final TaskManifest manifest = loadManifest(status);
+    final String id = manifest.getTaskAttemptID();
+    final int filecount = manifest.getFilesToCommit().size();
+    final long size = manifest.getTotalFileSize();
+    LOG.info("{}: Task Attempt {} file {}: File count: {}; data size={}",
+        getName(), id, status.getPath(), filecount, size);
+    // record file size for tracking of memory consumption.
+    getIOStatistics().addMeanStatisticSample(COMMITTER_TASK_MANIFEST_FILE_SIZE,
+        status.getLen());
+    return manifest;
+  }
+
+  /**
+   * Result of the stage.
+   */
+  public static final class Result {
+    private final SummaryInfo summary;
+
+    private final List<TaskManifest> manifests;
+
+    public Result(SummaryInfo summary,
+        List<TaskManifest> manifests) {
+      this.summary = summary;
+      this.manifests = manifests;
+    }
+
+    public SummaryInfo getSummary() {
+      return summary;
+    }
+
+    public List<TaskManifest> getManifests() {
+      return manifests;
+    }
+
+  }
+
+  /**
+   * Summary information.
+   */
+  public static final class SummaryInfo implements IOStatisticsSource {
+
+    /**
+     * Aggregate IOStatistics.
+     */
+    private IOStatisticsSnapshot iostatistics = snapshotIOStatistics();
+
+    /**
+     * How many manifests were loaded.
+     */
+    private long manifestCount;
+
+    /**
+     * Total number of files to rename.
+     */
+    private long fileCount;
+
+    /**
+     * Total number of directories which may need
+     * to be created.
+     * As there is no dedup, this is likely to be
+     * a (major) overestimate.
+     */
+    private long directoryCount;
+
+    /**
+     * Total amount of data to be committed.
+     */
+    private long totalFileSize;
+
+    /**
+     * Get the IOStatistics.
+     * @return aggregate IOStatistics
+     */
+    @Override
+    public IOStatisticsSnapshot getIOStatistics() {
+      return iostatistics;
+    }
+
+    public long getFileCount() {
+      return fileCount;
+    }
+
+    public long getDirectoryCount() {
+      return directoryCount;
+    }
+
+    public long getTotalFileSize() {
+      return totalFileSize;
+    }
+
+    public long getManifestCount() {
+      return manifestCount;
+    }
+
+    /**
+     * Add all statistics.
+     * @param manifest manifest to add.
+     */
+    public void add(TaskManifest manifest) {
+      manifestCount++;
+      iostatistics.aggregate(manifest.getIOStatistics());
+      fileCount += manifest.getFilesToCommit().size();
+      directoryCount += manifest.getDestDirectories().size();
+      totalFileSize += manifest.getTotalFileSize();
+    }
+
+    /**
+     * To String includes all summary info except statistics.
+     * @return string value
+     */
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder(
+          "SummaryInfo{");
+      sb.append("manifestCount=").append(manifestCount);
+      sb.append(", fileCount=").append(fileCount);
+      sb.append(", directoryCount=").append(directoryCount);
+      sb.append(", totalFileSize=").append(
+          byteCountToDisplaySize(totalFileSize));
+      sb.append('}');
+      return sb.toString();
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java
new file mode 100644
index 0000000000000..e8124c11465e5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER_FILE_LIMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_RENAME_FILES;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createManifestOutcome;
+import static org.apache.hadoop.thirdparty.com.google.common.collect.Iterables.concat;
+
+/**
+ * This stage renames all the files.
+ * Input: the manifests and the set of directories created, as returned by
+ * {@link CreateOutputDirectoriesStage}.
+ * If the job is configured to delete target files, if the parent dir
+ * had to be created, the delete() call can be skipped.
+ * It returns a manifest success data file summarizing the
+ * output, but does not add iostatistics to it.
+ */
+public class RenameFilesStage extends
+    AbstractJobOrTaskStage<
+        Pair<List<TaskManifest>, Set<Path>>,
+        ManifestSuccessData> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      RenameFilesStage.class);
+
+  /**
+   * List of all files committed.
+   */
+  private final List<FileEntry> filesCommitted = new ArrayList<>();
+
+  /**
+   * Total file size.
+   */
+  private long totalFileSize = 0;
+
+  private Set<Path> createdDirectories;
+
+  public RenameFilesStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_RENAME_FILES, true);
+  }
+
+  /**
+   * Get the list of files committed.
+   * Access is not synchronized.
+   * @return direct access to the list of files.
+   */
+  public synchronized  List<FileEntry> getFilesCommitted() {
+    return filesCommitted;
+  }
+
+  /**
+   * Get the total file size of the committed task.
+   * @return a number greater than or equal to zero.
+   */
+  public synchronized long getTotalFileSize() {
+    return totalFileSize;
+  }
+
+  /**
+   * Rename files in job commit.
+   * @param taskManifests a list of task manifests containing files.
+   * @return the job report.
+   * @throws IOException failure
+   */
+  @Override
+  protected ManifestSuccessData executeStage(
+      Pair<List<TaskManifest>, Set<Path>> args)
+      throws IOException {
+
+    final List<TaskManifest> taskManifests = args.getLeft();
+    createdDirectories = args.getRight();
+
+    final ManifestSuccessData success = createManifestOutcome(getStageConfig(),
+        OP_STAGE_JOB_COMMIT);
+    final int manifestCount = taskManifests.size();
+
+    LOG.info("{}: Executing Manifest Job Commit with {} manifests in {}",
+        getName(), manifestCount, getTaskManifestDir());
+
+    // first step is to aggregate the output of all manifests into a single
+    // list of files to commit.
+    // Which Guava can do in a zero-copy concatenated iterator
+
+    final Iterable<FileEntry> filesToCommit = concat(taskManifests.stream()
+        .map(TaskManifest::getFilesToCommit)
+        .collect(Collectors.toList()));
+
+    TaskPool.foreach(filesToCommit)
+        .executeWith(getIOProcessors())
+        .stopOnFailure()
+        .run(this::commitOneFile);
+
+    // synchronized block to keep spotbugs happy.
+    List<FileEntry> committed = getFilesCommitted();
+    LOG.info("{}: Files committed: {}. Total size {}",
+        getName(), committed.size(), getTotalFileSize());
+
+    // Add a subset of the destination files to the success file;
+    // enough for simple testing
+    success.setFilenamePaths(
+        committed
+            .subList(0, Math.min(committed.size(), SUCCESS_MARKER_FILE_LIMIT))
+            .stream().map(FileEntry::getDestPath)
+            .collect(Collectors.toList()));
+
+    success.setSuccess(true);
+
+    return success;
+  }
+
+  /**
+   * Commit one file by rename, then, if that doesn't fail,
+   * add to the files committed list.
+   * @param entry entry to commit.
+   * @throws IOException faiure.
+   */
+  private void commitOneFile(FileEntry entry) throws IOException {
+    updateAuditContext(OP_STAGE_JOB_RENAME_FILES);
+
+    // report progress back
+    progress();
+
+    // if the dest dir is to be deleted,
+    // look to see if the parent dir was created.
+    // if it was. we know that the file doesn't exist.
+    final boolean deleteDest = getStageConfig().getDeleteTargetPaths()
+        && !createdDirectories.contains(entry.getDestPath().getParent());
+    // do the rename
+    commitFile(entry, deleteDest);
+
+    // update the list and IOStats
+    synchronized (this) {
+      filesCommitted.add(entry);
+      totalFileSize += entry.getSize();
+    }
+
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java
new file mode 100644
index 0000000000000..eb9c82f2ae739
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.TMP_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_SAVE_SUCCESS;
+
+/**
+ * Save the _SUCCESS file to the destination directory
+ * via a temp file in the job attempt dir.
+ * Returns the path of the file
+ */
+public class SaveSuccessFileStage extends
+    AbstractJobOrTaskStage<ManifestSuccessData, Path> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      SaveSuccessFileStage.class);
+
+  public SaveSuccessFileStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_SAVE_SUCCESS, false);
+  }
+
+  /**
+   * Stage name is always job commit.
+   * @param arguments args to the invocation.
+   * @return stage name
+   */
+  @Override
+  protected String getStageName(ManifestSuccessData arguments) {
+    // set it to the job commit stage, always.
+    return OP_STAGE_JOB_COMMIT;
+  }
+
+  /**
+   * Execute.
+   * @param successData success data to save
+   * @return path saved to.
+   * @throws IOException failure
+   */
+  @Override
+  protected Path executeStage(final ManifestSuccessData successData)
+      throws IOException {
+    // Save the marker
+    Path successFile = getStageConfig().getJobSuccessMarkerPath();
+    Path successTempFile = new Path(getJobAttemptDir(), SUCCESS_MARKER + TMP_SUFFIX);
+    LOG.debug("{}: Saving _SUCCESS file to {} via {}", successFile,
+        getName(),
+        successTempFile);
+    save(successData, successTempFile, successFile);
+    return successFile;
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java
new file mode 100644
index 0000000000000..fdaf0184cda20
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SAVE_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestPathForTask;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestTempPathForTaskAttempt;
+
+/**
+ * Save a task manifest to the job attempt dir, using the task
+ * ID for the name of the final file.
+ * For atomic writes, the manifest is saved
+ * by writing to a temp file and then renaming it.
+ * Uses both the task ID and task attempt ID to determine the temp filename;
+ * Before the rename of (temp, final-path), any file at the final path
+ * is deleted.
+ * This is so that when this stage is invoked in a task commit, its output
+ * overwrites any of the first commit.
+ * When it succeeds, therefore, unless there is any subsequent commit of
+ * another task, the task manifest at the final path is from this
+ * operation.
+ *
+ * Returns the path where the manifest was saved.
+ */
+public class SaveTaskManifestStage extends
+    AbstractJobOrTaskStage<TaskManifest, Path> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      SaveTaskManifestStage.class);
+
+  public SaveTaskManifestStage(final StageConfig stageConfig) {
+    super(true, stageConfig, OP_STAGE_TASK_SAVE_MANIFEST, false);
+  }
+
+  /**
+   * Save the manifest to a temp file and rename to the final
+   * manifest destination.
+   * @param manifest manifest
+   * @return the path to the final entry
+   * @throws IOException IO failure.
+   */
+  @Override
+  protected Path executeStage(final TaskManifest manifest)
+      throws IOException {
+
+    final Path manifestDir = getTaskManifestDir();
+    // final manifest file is by task ID
+    Path manifestFile = manifestPathForTask(manifestDir,
+        getRequiredTaskId());
+    Path manifestTempFile = manifestTempPathForTaskAttempt(manifestDir,
+        getRequiredTaskAttemptId());
+    LOG.info("{}: Saving manifest file to {}", getName(), manifestFile);
+    save(manifest, manifestTempFile, manifestFile);
+    return manifestFile;
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java
new file mode 100644
index 0000000000000..9b873252df2cb
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_SETUP;
+
+/**
+ * Stage to set up a job by creating the job attempt directory.
+ * The job attempt directory must not exist before the call.
+ */
+public class SetupJobStage extends
+    AbstractJobOrTaskStage<Boolean, Path> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      SetupJobStage.class);
+
+  public SetupJobStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_SETUP, false);
+  }
+
+  /**
+   * Execute the job setup stage.
+   * @param deleteMarker: should any success marker be deleted.
+   * @return the job attempted directory.
+   * @throws IOException failure.
+   */
+  @Override
+  protected Path executeStage(final Boolean deleteMarker) throws IOException {
+    final Path path = getJobAttemptDir();
+    LOG.info("{}: Creating Job Attempt directory {}", getName(), path);
+    createNewDirectory("Job setup", path);
+    createNewDirectory("Creating task manifest dir", getTaskManifestDir());
+    // delete any success marker if so instructed.
+    if (deleteMarker) {
+      delete(getStageConfig().getJobSuccessMarkerPath(), false);
+    }
+    return path;
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java
new file mode 100644
index 0000000000000..e909ee183ec80
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SETUP;
+
+/**
+ * Stage to set up task.
+ * This creates the task attempt directory, after verifying
+ * that the job attempt dir exists (i.e. this is invoked
+ * after the job is started and before any cleanup.
+ * Argument passed in is task name:only for logging.
+ */
+public class SetupTaskStage extends
+    AbstractJobOrTaskStage<String, Path> {
+
+  public SetupTaskStage(final StageConfig stageConfig) {
+    super(true, stageConfig, OP_STAGE_TASK_SETUP, false);
+  }
+
+  /**
+   * Set up a task.
+   * @param name task name (for logging)
+   * @return task attempt directory
+   * @throws IOException IO failure.
+   */
+  @Override
+  protected Path executeStage(final String name) throws IOException {
+    return createNewDirectory("Task setup " + name,
+        requireNonNull(getTaskAttemptDir(), "No task attempt directory"));
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java
new file mode 100644
index 0000000000000..d5bbba5b761e3
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.util.JsonSerialization;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+
+/**
+ * Stage Config.
+ * Everything to configure a stage which is common to all.
+ *
+ * It's isolated from the details of MR datatypes (taskID, taskattempt etc);
+ * at this point it expects parsed values.
+ *
+ * It uses the builder API, but once {@link #build()} is called it goes
+ * read only. This is to ensure that changes cannot
+ * take place when shared across stages.
+ */
+public class StageConfig {
+
+  /**
+   * A flag which freezes the config for
+   * further updates.
+   */
+  private boolean frozen;
+
+  /**
+   * IOStatistics to update.
+   */
+  private IOStatisticsStore iostatistics;
+
+  /**
+   * Job ID; constant over multiple attempts.
+   */
+  private String jobId;
+
+  /**
+   * Where did the job Unique ID come from?
+   */
+  private String jobIdSource = "";
+
+  /**
+   * Number of the job attempt; starts at zero.
+   */
+  private int jobAttemptNumber;
+
+  /**
+   * ID of the task.
+   */
+  private String taskId;
+
+  /**
+   * ID of this specific attempt at a task.
+   */
+  private String taskAttemptId;
+
+  /**
+   * Destination of job.
+   */
+  private Path destinationDir;
+
+  /**
+   * Job attempt dir.
+   */
+  private Path jobAttemptDir;
+
+  /**
+   * temp directory under job dest dir.
+   */
+  private Path outputTempSubDir;
+
+  /**
+   * Task attempt dir.
+   */
+  private Path taskAttemptDir;
+
+  /**
+   * directory where task manifests must go.
+   */
+  private Path taskManifestDir;
+
+  /**
+   * Subdir under the job attempt dir where task
+   * attempts will have subdirectories.
+   */
+  private Path jobAttemptTaskSubDir;
+
+  /**
+   * Callbacks to update store.
+   * This is not made visible to the stages; they must
+   * go through the superclass which
+   * adds statistics and logging.
+   */
+  private ManifestStoreOperations operations;
+
+  /**
+   * Submitter for doing IO against the store other than
+   * manifest processing.
+   */
+  private TaskPool.Submitter ioProcessors;
+
+  /**
+   * Optional progress callback.
+   */
+  private Progressable progressable;
+
+  /**
+   * Callback when a stage is entered.
+   */
+  private StageEventCallbacks enterStageEventHandler;
+
+  /**
+   * Thread local serializer; created on demand
+   * and shareable across a sequence of stages.
+   */
+  private final ThreadLocal<JsonSerialization<TaskManifest>> threadLocalSerializer =
+      ThreadLocal.withInitial(TaskManifest::serializer);
+
+  /**
+   * Delete target paths on commit? Stricter, but
+   * higher IO cost.
+   */
+  private boolean deleteTargetPaths;
+
+  /**
+   * Name for logging.
+   */
+  private String name = "";
+
+  public StageConfig() {
+  }
+
+  /**
+   * Verify that the config is not yet frozen.
+   */
+  private void checkOpen() {
+    Preconditions.checkState(!frozen,
+        "StageConfig is now read-only");
+  }
+
+  /**
+   * The build command makes the config immutable.
+   * Idempotent.
+   * @return the now-frozen config
+   */
+  public StageConfig build() {
+    frozen = true;
+    return this;
+  }
+
+  /**
+   * Set job destination dir.
+   * @param dir new dir
+   * @return this
+   */
+  public StageConfig withDestinationDir(final Path dir) {
+    destinationDir = dir;
+    return this;
+  }
+
+  /**
+   * Set IOStatistics store.
+   * @param store new store
+   * @return this
+   */
+  public StageConfig withIOStatistics(final IOStatisticsStore store) {
+    checkOpen();
+    iostatistics = store;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withIOProcessors(final TaskPool.Submitter value) {
+    checkOpen();
+    ioProcessors = value;
+    return this;
+  }
+
+  /**
+   * Set Job attempt directory.
+   * @param dir new dir
+   * @return this
+   */
+  public StageConfig withJobAttemptDir(final Path dir) {
+    checkOpen();
+    jobAttemptDir = dir;
+    return this;
+  }
+
+  /**
+   * Directory to put task manifests into.
+   * @return a path under the job attempt dir.
+   */
+  public Path getTaskManifestDir() {
+    return taskManifestDir;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public StageConfig withTaskManifestDir(Path value) {
+    checkOpen();
+    taskManifestDir = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public StageConfig withJobAttemptTaskSubDir(Path value) {
+    jobAttemptTaskSubDir = value;
+    return this;
+  }
+
+  /**
+   * Get the path to the subdirectory under $jobID where task
+   * attempts are. List this dir to find all task attempt dirs.
+   * @return a path under the job attempt dir.
+   */
+  public Path getJobAttemptTaskSubDir() {
+    return jobAttemptTaskSubDir;
+  }
+
+  /**
+   * Set the job directories from the attempt directories
+   * information. Does not set task attempt fields.
+   * @param dirs source of directories.
+   * @return this
+   */
+  public StageConfig withJobDirectories(
+      final ManifestCommitterSupport.AttemptDirectories dirs) {
+
+    checkOpen();
+    withJobAttemptDir(dirs.getJobAttemptDir())
+        .withJobAttemptTaskSubDir(dirs.getJobAttemptTaskSubDir())
+        .withDestinationDir(dirs.getOutputPath())
+        .withOutputTempSubDir(dirs.getOutputTempSubDir())
+        .withTaskManifestDir(dirs.getTaskManifestDir());
+
+    return this;
+  }
+
+  /**
+   * Set job ID with no attempt included.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withJobId(final String value) {
+    checkOpen();
+    jobId = value;
+    return this;
+  }
+
+  public Path getOutputTempSubDir() {
+    return outputTempSubDir;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withOutputTempSubDir(final Path value) {
+    checkOpen();
+    outputTempSubDir = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withOperations(final ManifestStoreOperations value) {
+    checkOpen();
+    operations = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withTaskAttemptId(final String value) {
+    checkOpen();
+    taskAttemptId = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withTaskId(final String value) {
+    checkOpen();
+    taskId = value;
+    return this;
+  }
+
+  /**
+   * Set handler for stage entry events..
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withStageEventCallbacks(StageEventCallbacks value) {
+    checkOpen();
+    enterStageEventHandler = value;
+    return this;
+  }
+
+  /**
+   * Optional progress callback.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withProgressable(final Progressable value) {
+    checkOpen();
+    progressable = value;
+    return this;
+  }
+
+  /**
+   * Set the Task attempt directory.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withTaskAttemptDir(final Path value) {
+    checkOpen();
+    taskAttemptDir = value;
+    return this;
+  }
+
+  /**
+   * Set the job attempt number.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withJobAttemptNumber(final int value) {
+    checkOpen();
+    jobAttemptNumber = value;
+    return this;
+  }
+
+  /**
+   * Set the Job ID source.
+   * @param value new value
+   * @return this
+   */
+  public StageConfig withJobIdSource(final String value) {
+    checkOpen();
+    jobIdSource = value;
+    return this;
+  }
+
+  /**
+   * Set name of task/job.
+   * @param value new value
+   * @return the builder
+   */
+  public StageConfig withName(String value) {
+    name = value;
+    return this;
+  }
+
+  /**
+   * Get name of task/job.
+   * @return name for logging.
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * Handler for stage entry events.
+   * @return the handler.
+   */
+  public StageEventCallbacks getEnterStageEventHandler() {
+    return enterStageEventHandler;
+  }
+
+  /**
+   * IOStatistics to update.
+   */
+  public IOStatisticsStore getIOStatistics() {
+    return iostatistics;
+  }
+
+  /**
+   * Job ID.
+   */
+  public String getJobId() {
+    return jobId;
+  }
+
+  /**
+   * ID of the task.
+   */
+  public String getTaskId() {
+    return taskId;
+  }
+
+  /**
+   * ID of this specific attempt at a task.
+   */
+  public String getTaskAttemptId() {
+    return taskAttemptId;
+  }
+
+  /**
+   * Job attempt dir.
+   */
+  public Path getJobAttemptDir() {
+    return jobAttemptDir;
+  }
+
+  /**
+   * Destination of job.
+   */
+  public Path getDestinationDir() {
+    return destinationDir;
+  }
+
+  /**
+   * Get the location of the success marker.
+   * @return a path under the destination directory.
+   */
+  public Path getJobSuccessMarkerPath() {
+    return new Path(destinationDir, SUCCESS_MARKER);
+  }
+
+  /**
+   * Callbacks to update store.
+   * This is not made visible to the stages; they must
+   * go through the wrapper classes in this class, which
+   * add statistics and logging.
+   */
+  public ManifestStoreOperations getOperations() {
+    return operations;
+  }
+
+  /**
+   * Submitter for doing IO against the store other than
+   * manifest processing.
+   */
+  public TaskPool.Submitter getIoProcessors() {
+    return ioProcessors;
+  }
+
+  /**
+   * Get optional progress callback.
+   * @return callback or null
+   */
+  public Progressable getProgressable() {
+    return progressable;
+  }
+
+  /**
+   * Task attempt directory.
+   * @return the task attempt dir.
+   */
+  public Path getTaskAttemptDir() {
+    return taskAttemptDir;
+  }
+
+  /**
+   * Get the job attempt number.
+   * @return the value
+   */
+  public int getJobAttemptNumber() {
+    return jobAttemptNumber;
+  }
+
+  public String getJobIdSource() {
+    return jobIdSource;
+  }
+
+  /**
+   * Get a thread local task manifest serializer.
+   * @return a serializer.
+   */
+  public JsonSerialization<TaskManifest> currentManifestSerializer() {
+    return threadLocalSerializer.get();
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public StageConfig withDeleteTargetPaths(boolean value) {
+    checkOpen();
+    deleteTargetPaths = value;
+    return this;
+  }
+
+  public boolean getDeleteTargetPaths() {
+    return deleteTargetPaths;
+  }
+
+  /**
+   * Enter the stage; calls back to
+   * {@link #enterStageEventHandler} if non-null.
+   * @param stage stage entered
+   */
+  public void enterStage(String stage) {
+    if (enterStageEventHandler != null) {
+      enterStageEventHandler.enterStage(stage);
+    }
+  }
+
+  /**
+   * Exit the stage; calls back to
+   * {@link #enterStageEventHandler} if non-null.
+   * @param stage stage entered
+   */
+  public void exitStage(String stage) {
+    if (enterStageEventHandler != null) {
+      enterStageEventHandler.exitStage(stage);
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java
new file mode 100644
index 0000000000000..e23f72092fb02
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+/**
+ * Callbacks on stage entry/exit.
+ */
+public interface StageEventCallbacks {
+
+  /**
+   * Callback on stage entry.
+   * @param stage new stage
+   */
+  void enterStage(String stage);
+
+  /**
+   * Callback on stage exit.
+   * @param stage stage exited.
+   */
+  void exitStage(String stage);
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java
new file mode 100644
index 0000000000000..4765bcc0865d4
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.LongSummaryStatistics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.EntryStatus;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.DurationInfo;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_DIRECTORY_COUNT_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_DIRECTORY_DEPTH_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_FILE_COUNT_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_FILE_SIZE_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SCAN_DIRECTORY;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createTaskManifest;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.maybeAddIOStatistics;
+
+/**
+ * Stage to scan a directory tree and build a task manifest.
+ * This is executed by the task committer.
+ */
+public final class TaskAttemptScanDirectoryStage
+    extends AbstractJobOrTaskStage<Void, TaskManifest> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TaskAttemptScanDirectoryStage.class);
+
+  public TaskAttemptScanDirectoryStage(
+      final StageConfig stageConfig) {
+    super(true, stageConfig, OP_STAGE_TASK_SCAN_DIRECTORY, false);
+  }
+
+  /**
+   * Build the Manifest.
+   * @return the manifest
+   * @throws IOException failure.
+   */
+  @Override
+  protected TaskManifest executeStage(final Void arguments)
+      throws IOException {
+
+    final Path taskAttemptDir = getRequiredTaskAttemptDir();
+    final TaskManifest manifest = createTaskManifest(getStageConfig());
+
+    LOG.info("{}: scanning directory {}",
+        getName(), taskAttemptDir);
+
+    final int depth = scanDirectoryTree(manifest,
+        taskAttemptDir,
+        getDestinationDir(),
+        0, true);
+    List<FileEntry> filesToCommit = manifest.getFilesToCommit();
+    LongSummaryStatistics fileSummary = filesToCommit.stream()
+        .mapToLong(FileEntry::getSize)
+        .summaryStatistics();
+    long fileDataSize = fileSummary.getSum();
+    long fileCount = fileSummary.getCount();
+    int dirCount = manifest.getDestDirectories().size();
+    LOG.info("{}: directory {} contained {} file(s); data size {}",
+        getName(),
+        taskAttemptDir,
+        fileCount,
+        fileDataSize);
+    LOG.info("{}: Directory count = {}; maximum depth {}",
+        getName(),
+        dirCount,
+        depth);
+    // add statistics about the task output which, when aggregated, provides
+    // insight into structure of job, task skew, etc.
+    IOStatisticsStore iostats = getIOStatistics();
+    iostats.addSample(COMMITTER_TASK_DIRECTORY_COUNT_MEAN, dirCount);
+    iostats.addSample(COMMITTER_TASK_DIRECTORY_DEPTH_MEAN, depth);
+    iostats.addSample(COMMITTER_TASK_FILE_COUNT_MEAN, fileCount);
+    iostats.addSample(COMMITTER_TASK_FILE_SIZE_MEAN, fileDataSize);
+
+    return manifest;
+  }
+
+  /**
+   * Recursively scan a directory tree.
+   * The manifest will contain all files to rename
+   * (source and dest) and directories to create.
+   * All files are processed before any of the subdirs are.
+   * This helps in statistics gathering.
+   * There's some optimizations which could be done with async
+   * fetching of the iterators of those subdirs, but as this
+   * is generally off-critical path then that "enhancement"
+   * can be postponed until data suggests this needs improvement.
+   * @param manifest manifest to update
+   * @param srcDir dir to scan
+   * @param destDir destination directory
+   * @param depth depth from the task attempt dir.
+   * @param parentDirExists does the parent dir exist?
+   * @return the maximum depth of child directories
+   * @throws IOException IO failure.
+   */
+  private int scanDirectoryTree(
+      TaskManifest manifest,
+      Path srcDir,
+      Path destDir,
+      int depth,
+      boolean parentDirExists) throws IOException {
+
+    // generate some task progress in case directory scanning is very slow.
+    progress();
+
+    int maxDepth = 0;
+    int files = 0;
+    boolean dirExists = parentDirExists;
+    List<FileStatus> subdirs = new ArrayList<>();
+    try (DurationInfo ignored = new DurationInfo(LOG, false,
+        "Task Attempt %s source dir %s, dest dir %s",
+        getTaskAttemptId(), srcDir, destDir)) {
+
+      // list the directory. This may block until the listing is complete,
+      // or, if the FS does incremental or asynchronous fetching,
+      // then the next()/hasNext() call will block for the results
+      // unless turned off, ABFS does to this async
+      final RemoteIterator<FileStatus> listing = listStatusIterator(srcDir);
+
+      // when the FS (especially ABFS) does an asyn fetch of the listing,
+      // we can probe for the status of the destination dir while that
+      // page is being fetched.
+      // probe for and add the dest dir entry for all but
+      // the base dir
+
+      if (depth > 0) {
+        final EntryStatus status;
+        if (parentDirExists) {
+          final FileStatus destDirStatus = getFileStatusOrNull(destDir);
+          status = EntryStatus.toEntryStatus(destDirStatus);
+          dirExists = destDirStatus != null;
+        } else {
+          // if there is no parent dir, then there is no need to look
+          // for this directory -report it as missing automatically.
+          status = EntryStatus.not_found;
+        }
+        manifest.addDirectory(DirEntry.dirEntry(
+            destDir,
+            status,
+            depth));
+      }
+
+      // process the listing; this is where abfs will block
+      // to wait the result of the list call.
+      while (listing.hasNext()) {
+        final FileStatus st = listing.next();
+        if (st.isFile()) {
+          // this is a file, so add to the list of files to commit.
+          files++;
+          final FileEntry entry = fileEntry(st, destDir);
+          manifest.addFileToCommit(entry);
+          LOG.debug("To rename: {}", entry);
+        } else {
+          if (st.isDirectory()) {
+            // will need to scan this directory too.
+            subdirs.add(st);
+          } else {
+            // some other object. ignoring
+            LOG.info("Ignoring FS object {}", st);
+          }
+        }
+      }
+      // add any statistics provided by the listing.
+      maybeAddIOStatistics(getIOStatistics(), listing);
+    }
+
+    // now scan the subdirectories
+    LOG.debug("{}: Number of subdirectories under {} found: {}; file count {}",
+        getName(), srcDir, subdirs.size(), files);
+
+    for (FileStatus st : subdirs) {
+      Path destSubDir = new Path(destDir, st.getPath().getName());
+      final int d = scanDirectoryTree(manifest,
+          st.getPath(),
+          destSubDir,
+          depth + 1,
+          dirExists);
+      maxDepth = Math.max(maxDepth, d);
+    }
+
+    return 1 + maxDepth;
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java
new file mode 100644
index 0000000000000..8b72112a3dad8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.OutputValidationException;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_VALIDATE_OUTPUT;
+import static org.apache.hadoop.thirdparty.com.google.common.collect.Iterables.concat;
+
+/**
+ * This stage validates all files by scanning the manifests
+ * and verifying every file in every manifest is of the given size.
+ * Returns a list of all files committed.
+ *
+ * Its cost is one getFileStatus() call (parallelized) per file.
+ * Raises a {@link OutputValidationException} on a validation failure.
+ */
+public class ValidateRenamedFilesStage extends
+    AbstractJobOrTaskStage<
+        List<TaskManifest>,
+        List<FileEntry>> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      ValidateRenamedFilesStage.class);
+
+  /**
+   * Set this to halt all workers.
+   */
+  private final AtomicBoolean halt = new AtomicBoolean();
+
+  /**
+   * List of all files committed.
+   */
+  private List<FileEntry> filesCommitted = new ArrayList<>();
+
+  public ValidateRenamedFilesStage(final StageConfig stageConfig) {
+    super(false, stageConfig, OP_STAGE_JOB_VALIDATE_OUTPUT, true);
+  }
+
+  /**
+   * Get the list of files committed.
+   * @return a possibly empty list.
+   */
+  private synchronized List<FileEntry> getFilesCommitted() {
+    return filesCommitted;
+  }
+
+  /**
+   * Add a file entry to the list of committed files.
+   * @param entry entry
+   */
+  private synchronized void addFileCommitted(FileEntry entry) {
+    filesCommitted.add(entry);
+  }
+
+  /**
+   * Validate the task manifests.
+   * This is done by listing all the directories
+   * and verifying that every file in the source list
+   * has a file in the destination of the same size.
+   * If two tasks have both written the same file or
+   * a source file was changed after the task was committed,
+   * then a mistmatch will be detected -provided the file
+   * length is now different.
+   * @param taskManifests list of manifests.
+   * @return list of files committed.
+   */
+  @Override
+  protected List<FileEntry> executeStage(
+      final List<TaskManifest> taskManifests)
+      throws IOException {
+
+    // set the list of files to be as big as the number of tasks.
+    // synchronized to stop complaints.
+    synchronized (this) {
+      filesCommitted = new ArrayList<>(taskManifests.size());
+    }
+
+    // validate all the files.
+
+    final Iterable<FileEntry> filesToCommit = concat(taskManifests.stream()
+        .map(TaskManifest::getFilesToCommit)
+        .collect(Collectors.toList()));
+
+    TaskPool.foreach(filesToCommit)
+        .executeWith(getIOProcessors())
+        .stopOnFailure()
+        .run(this::validateOneFile);
+
+    return getFilesCommitted();
+  }
+
+  /**
+   * Validate a file.
+   * @param entry entry to probe for
+   * @throws IOException IO problem.
+   * @throws OutputValidationException if the entry is not valid
+   */
+  private void validateOneFile(FileEntry entry) throws IOException {
+    updateAuditContext(OP_STAGE_JOB_VALIDATE_OUTPUT);
+
+    if (halt.get()) {
+      // told to stop
+      return;
+    }
+    // report progress back
+    progress();
+    // look validate the file.
+    // raising an FNFE if the file isn't there.
+    FileStatus destStatus;
+    final Path sourcePath = entry.getSourcePath();
+    Path destPath = entry.getDestPath();
+    try {
+      destStatus = getFileStatus(destPath);
+
+      // it must be a file
+      if (!destStatus.isFile()) {
+        throw new OutputValidationException(destPath,
+            "Expected a file renamed from " + sourcePath
+                + "; found " + destStatus);
+      }
+      final long sourceSize = entry.getSize();
+      final long destSize = destStatus.getLen();
+
+      // etags, if the source had one.
+      final String sourceEtag = entry.getEtag();
+      if (isNotBlank(sourceEtag)) {
+        final String destEtag = ManifestCommitterSupport.getEtag(destStatus);
+        if (!sourceEtag.equals(destEtag)) {
+          LOG.warn("Etag of dest file {}: {} does not match that of manifest entry {}",
+              destPath, destStatus, entry);
+          throw new OutputValidationException(destPath,
+              String.format("Expected the file"
+                      + " renamed from %s"
+                      + " with etag %s and length %s"
+                      + " but found a file with etag %s and length %d",
+                  sourcePath,
+                  sourceEtag,
+                  sourceSize,
+                  destEtag,
+                  destSize));
+
+        }
+      }
+      // check the expected length after any etag validation
+      if (destSize != sourceSize) {
+        LOG.warn("Length of dest file {}: {} does not match that of manifest entry {}",
+            destPath, destStatus, entry);
+        throw new OutputValidationException(destPath,
+            String.format("Expected the file"
+                    + " renamed from %s"
+                    + " with length %d"
+                    + " but found a file of length %d",
+                sourcePath,
+                sourceSize,
+                destSize));
+      }
+
+    } catch (FileNotFoundException e) {
+      // file didn't exist
+      throw new OutputValidationException(destPath,
+          "Expected a file, but it was not found", e);
+    }
+    addFileCommitted(entry);
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java
new file mode 100644
index 0000000000000..8b86046268d1f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Intermediate manifest committer.
+ *
+ * The individual and aggregate stages of the task and job commit process.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
index 68de7a9f5c25d..c19724e842d26 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
@@ -31,10 +31,8 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BinaryComparable;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.RawComparator;
-import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Partitioner;
 import org.apache.hadoop.util.ReflectionUtils;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
index f75ad05d29564..c0e91804dcb46 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
@@ -29,6 +29,12 @@
 @InterfaceStability.Evolving
 public interface TTConfig extends MRConfig {
 
+  /**
+   *  @deprecated Use
+   *  {@link org.apache.hadoop.mapreduce.MRJobConfig#SHUFFLE_INDEX_CACHE}
+   *  instead
+   */
+  @Deprecated
   public static final String TT_INDEX_CACHE = 
     "mapreduce.tasktracker.indexcache.mb";
   public static final String TT_MAP_SLOTS = 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
index 1696246b84321..41d03bb002701 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -43,6 +44,7 @@
 import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * A read-only view of the job that is provided to the tasks while they
@@ -305,7 +307,27 @@ public boolean getSymlink() {
    * Get the archive entries in classpath as an array of Path
    */
   public Path[] getArchiveClassPaths() {
-    return DistributedCache.getArchiveClassPaths(conf);
+    return getArchiveClassPaths(conf);
+  }
+
+  /**
+   * Get the archive entries in classpath as an array of Path.
+   * Used by internal DistributedCache code.
+   *
+   * @param conf Configuration that contains the classpath setting.
+   * @return An array of Path consisting of archive entries in classpath.
+   */
+  public static Path[] getArchiveClassPaths(Configuration conf) {
+    ArrayList<String> list = (ArrayList<String>)conf.getStringCollection(
+        MRJobConfig.CLASSPATH_ARCHIVES);
+    if (list.size() == 0) {
+      return null;
+    }
+    Path[] paths = new Path[list.size()];
+    for (int i = 0; i < list.size(); i++) {
+      paths[i] = new Path(list.get(i));
+    }
+    return paths;
   }
 
   /**
@@ -314,7 +336,18 @@ public Path[] getArchiveClassPaths() {
    * @throws IOException
    */
   public URI[] getCacheArchives() throws IOException {
-    return DistributedCache.getCacheArchives(conf);
+    return getCacheArchives(conf);
+  }
+
+  /**
+   * Get cache archives set in the Configuration. Used by
+   * internal DistributedCache and JobContextImpl code.
+   *
+   * @param conf The configuration which contains the archives.
+   * @return A URI array of the caches set in the Configuration.
+   */
+  public static URI[] getCacheArchives(Configuration conf) {
+    return StringUtils.stringToURI(conf.getStrings(MRJobConfig.CACHE_ARCHIVES));
   }
 
   /**
@@ -324,7 +357,18 @@ public URI[] getCacheArchives() throws IOException {
    */
 
   public URI[] getCacheFiles() throws IOException {
-    return DistributedCache.getCacheFiles(conf);
+    return getCacheFiles(conf);
+  }
+
+  /**
+   * Get cache files set in the Configuration. Used by internal
+   * DistributedCache and MapReduce code.
+   *
+   * @param conf The configuration which contains the files.
+   * @return A URI array of the files set in the Configuration.
+   */
+  public static URI[] getCacheFiles(Configuration conf) {
+    return StringUtils.stringToURI(conf.getStrings(MRJobConfig.CACHE_FILES));
   }
 
   /**
@@ -334,7 +378,17 @@ public URI[] getCacheFiles() throws IOException {
    */
   public Path[] getLocalCacheArchives()
     throws IOException {
-    return DistributedCache.getLocalCacheArchives(conf);
+    return getLocalCacheArchives(conf);
+  }
+
+  /**
+   * Return the path array of the localized caches.
+   *
+   * @param conf Configuration that contains the localized archives.
+   * @return A path array of localized caches.
+   */
+  public static Path[] getLocalCacheArchives(Configuration conf) {
+    return StringUtils.stringToPath(conf.getStrings(MRJobConfig.CACHE_LOCALARCHIVES));
   }
 
   /**
@@ -344,14 +398,82 @@ public Path[] getLocalCacheArchives()
    */
   public Path[] getLocalCacheFiles()
     throws IOException {
-    return DistributedCache.getLocalCacheFiles(conf);
+    return getLocalCacheFiles(conf);
+  }
+
+  /**
+   * Return the path array of the localized files.
+   *
+   * @param conf Configuration that contains the localized files.
+   * @return A path array of localized files.
+   */
+  public static Path[] getLocalCacheFiles(Configuration conf) {
+    return StringUtils.stringToPath(conf.getStrings(MRJobConfig.CACHE_LOCALFILES));
+  }
+
+  /**
+   * Parse a list of strings into longs.
+   * @param strs the list of strings to parse
+   * @return a list of longs that were parsed. same length as strs.
+   */
+  private static long[] parseTimestamps(String[] strs) {
+    if (strs == null) {
+      return null;
+    }
+    long[] result = new long[strs.length];
+    for(int i=0; i < strs.length; ++i) {
+      result[i] = Long.parseLong(strs[i]);
+    }
+    return result;
+  }
+
+  /**
+   * Get the timestamps of the archives. Used by internal
+   * DistributedCache and MapReduce code.
+   *
+   * @param conf The configuration which stored the timestamps.
+   * @return a long array of timestamps.
+   */
+  public static long[] getArchiveTimestamps(Configuration conf) {
+    return parseTimestamps(conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS));
+  }
+
+  /**
+   * Get the timestamps of the files. Used by internal
+   * DistributedCache and MapReduce code.
+   *
+   * @param conf The configuration which stored the timestamps.
+   * @return a long array of timestamps.
+   */
+  public static long[] getFileTimestamps(Configuration conf) {
+    return parseTimestamps(conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS));
   }
 
   /**
    * Get the file entries in classpath as an array of Path
    */
   public Path[] getFileClassPaths() {
-    return DistributedCache.getFileClassPaths(conf);
+    return getFileClassPaths(conf);
+  }
+
+  /**
+   * Get the file entries in classpath as an array of Path.
+   * Used by internal DistributedCache code.
+   *
+   * @param conf Configuration that contains the classpath setting.
+   * @return Array of Path consisting of file entries in the classpath.
+   */
+  public static Path[] getFileClassPaths(Configuration conf) {
+    ArrayList<String> list =
+        (ArrayList<String>) conf.getStringCollection(MRJobConfig.CLASSPATH_FILES);
+    if (list.size() == 0) {
+      return null;
+    }
+    Path[] paths = new Path[list.size()];
+    for (int i = 0; i < list.size(); i++) {
+      paths[i] = new Path(list.get(i));
+    }
+    return paths;
   }
 
   /**
@@ -376,7 +498,7 @@ private static String[] toTimestampStrs(long[] timestamps) {
    * @return a string array of timestamps 
    */
   public String[] getArchiveTimestamps() {
-    return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf));
+    return toTimestampStrs(getArchiveTimestamps(conf));
   }
 
   /**
@@ -385,7 +507,7 @@ public String[] getArchiveTimestamps() {
    * @return a string array of timestamps 
    */
   public String[] getFileTimestamps() {
-    return toTimestampStrs(DistributedCache.getFileTimestamps(conf));
+    return toTimestampStrs(getFileTimestamps(conf));
   }
 
   /** 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 1da5b2f5d3f64..e013d017b1555 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -53,14 +53,15 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
-class Fetcher<K,V> extends Thread {
+@VisibleForTesting
+public class Fetcher<K, V> extends Thread {
   
   private static final Logger LOG = LoggerFactory.getLogger(Fetcher.class);
   
-  /** Number of ms before timing out a copy */
+  /** Number of ms before timing out a copy. */
   private static final int DEFAULT_STALLED_COPY_TIMEOUT = 3 * 60 * 1000;
   
-  /** Basic/unit connection timeout (in milliseconds) */
+  /** Basic/unit connection timeout (in milliseconds). */
   private final static int UNIT_CONNECT_TIMEOUT = 60 * 1000;
   
   /* Default read timeout (in milliseconds) */
@@ -72,10 +73,12 @@ class Fetcher<K,V> extends Thread {
   private static final String FETCH_RETRY_AFTER_HEADER = "Retry-After";
 
   protected final Reporter reporter;
-  private enum ShuffleErrors{IO_ERROR, WRONG_LENGTH, BAD_ID, WRONG_MAP,
+  @VisibleForTesting
+  public enum ShuffleErrors{IO_ERROR, WRONG_LENGTH, BAD_ID, WRONG_MAP,
                                     CONNECTION, WRONG_REDUCE}
-  
-  private final static String SHUFFLE_ERR_GRP_NAME = "Shuffle Errors";
+
+  @VisibleForTesting
+  public final static String SHUFFLE_ERR_GRP_NAME = "Shuffle Errors";
   private final JobConf jobConf;
   private final Counters.Counter connectionErrs;
   private final Counters.Counter ioErrs;
@@ -83,8 +86,8 @@ private enum ShuffleErrors{IO_ERROR, WRONG_LENGTH, BAD_ID, WRONG_MAP,
   private final Counters.Counter badIdErrs;
   private final Counters.Counter wrongMapErrs;
   private final Counters.Counter wrongReduceErrs;
-  protected final MergeManager<K,V> merger;
-  protected final ShuffleSchedulerImpl<K,V> scheduler;
+  protected final MergeManager<K, V> merger;
+  protected final ShuffleSchedulerImpl<K, V> scheduler;
   protected final ShuffleClientMetrics metrics;
   protected final ExceptionReporter exceptionReporter;
   protected final int id;
@@ -111,7 +114,7 @@ private enum ShuffleErrors{IO_ERROR, WRONG_LENGTH, BAD_ID, WRONG_MAP,
   private static SSLFactory sslFactory;
 
   public Fetcher(JobConf job, TaskAttemptID reduceId, 
-                 ShuffleSchedulerImpl<K,V> scheduler, MergeManager<K,V> merger,
+                 ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K, V> merger,
                  Reporter reporter, ShuffleClientMetrics metrics,
                  ExceptionReporter exceptionReporter, SecretKey shuffleKey) {
     this(job, reduceId, scheduler, merger, reporter, metrics,
@@ -120,7 +123,7 @@ public Fetcher(JobConf job, TaskAttemptID reduceId,
 
   @VisibleForTesting
   Fetcher(JobConf job, TaskAttemptID reduceId, 
-                 ShuffleSchedulerImpl<K,V> scheduler, MergeManager<K,V> merger,
+                 ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K, V> merger,
                  Reporter reporter, ShuffleClientMetrics metrics,
                  ExceptionReporter exceptionReporter, SecretKey shuffleKey,
                  int id) {
@@ -315,9 +318,8 @@ protected void copyFromHost(MapHost host) throws IOException {
       return;
     }
     
-    if(LOG.isDebugEnabled()) {
-      LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: "
-        + maps);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + maps);
     }
     
     // List of maps to be fetched yet
@@ -411,8 +413,8 @@ private void openConnectionWithRetry(URL url) throws IOException {
         shouldWait = false;
       } catch (IOException e) {
         if (!fetchRetryEnabled) {
-           // throw exception directly if fetch's retry is not enabled
-           throw e;
+          // throw exception directly if fetch's retry is not enabled
+          throw e;
         }
         if ((Time.monotonicNow() - startTime) >= this.fetchRetryTimeout) {
           LOG.warn("Failed to connect to host: " + url + "after " 
@@ -489,7 +491,7 @@ private TaskAttemptID[] copyMapOutput(MapHost host,
                                 DataInputStream input,
                                 Set<TaskAttemptID> remaining,
                                 boolean canRetry) throws IOException {
-    MapOutput<K,V> mapOutput = null;
+    MapOutput<K, V> mapOutput = null;
     TaskAttemptID mapId = null;
     long decompressedLength = -1;
     long compressedLength = -1;
@@ -611,7 +613,7 @@ private void checkTimeoutOrRetry(MapHost host, IOException ioe)
     // First time to retry.
     long currentTime = Time.monotonicNow();
     if (retryStartTime == 0) {
-       retryStartTime = currentTime;
+      retryStartTime = currentTime;
     }
   
     // Retry is not timeout, let's do retry with throwing an exception.
@@ -628,7 +630,7 @@ private void checkTimeoutOrRetry(MapHost host, IOException ioe)
   }
   
   /**
-   * Do some basic verification on the input received -- Being defensive
+   * Do some basic verification on the input received -- Being defensive.
    * @param compressedLength
    * @param decompressedLength
    * @param forReduce
@@ -695,8 +697,7 @@ private URL getMapOutputURL(MapHost host, Collection<TaskAttemptID> maps
    * only on the last failure. Instead of connecting with a timeout of 
    * X, we try connecting with a timeout of x < X but multiple times. 
    */
-  private void connect(URLConnection connection, int connectionTimeout)
-  throws IOException {
+  private void connect(URLConnection connection, int connectionTimeout) throws IOException {
     int unit = 0;
     if (connectionTimeout < 0) {
       throw new IOException("Invalid timeout "
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
index 2ecc55ecbb1cc..0245f64267b30 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
@@ -20,19 +20,8 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapOutputFile;
 import org.apache.hadoop.mapred.RawKeyValueIterator;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.Task.CombineOutputCollector;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.util.Progress;
 
 import java.io.IOException;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index 18f2a33555ab4..ec6ad158ddc72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -879,4 +879,9 @@ public int compareTo(Path obj) {
       return super.compareTo(obj);
     }
   }
+
+  @VisibleForTesting
+  OnDiskMerger getOnDiskMerger() {
+    return onDiskMerger;
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
index 4a40cb17d4615..c617569da33e8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
@@ -25,6 +25,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -109,4 +110,14 @@ public void run() {
   }
 
   public abstract void merge(List<T> inputs) throws IOException;
+
+  @VisibleForTesting
+  int getMergeFactor() {
+    return mergeFactor;
+  }
+
+  @VisibleForTesting
+  LinkedList<List<T>> getPendingToBeMerged() {
+    return pendingToBeMerged;
+  }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
index cf120342dd1b9..95074de231cf0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
@@ -80,8 +80,6 @@ private static void addDeprecatedKeys()  {
         JTConfig.JT_TASKCACHE_LEVELS),
       new DeprecationDelta("mapred.job.tracker.retire.jobs",
         JTConfig.JT_RETIREJOBS),
-      new DeprecationDelta("mapred.tasktracker.indexcache.mb",
-        TTConfig.TT_INDEX_CACHE),
       new DeprecationDelta("mapred.tasktracker.map.tasks.maximum",
         TTConfig.TT_MAP_SLOTS),
       new DeprecationDelta("mapred.tasktracker.memory_calculator_plugin",
@@ -290,6 +288,10 @@ private static void addDeprecatedKeys()  {
         MRJobConfig.REDUCE_LOG_LEVEL),
       new DeprecationDelta("mapreduce.job.counters.limit",
         MRJobConfig.COUNTERS_MAX_KEY),
+        new DeprecationDelta("mapred.tasktracker.indexcache.mb",
+            MRJobConfig.SHUFFLE_INDEX_CACHE),
+        new DeprecationDelta("mapreduce.tasktracker.indexcache.mb",
+            MRJobConfig.SHUFFLE_INDEX_CACHE),
       new DeprecationDelta("jobclient.completion.poll.interval",
         Job.COMPLETION_POLL_INTERVAL_KEY),
       new DeprecationDelta("jobclient.progress.monitor.poll.interval",
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
index 93cff0f95c534..585a21d568231 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.mapreduce.util;
 
 import java.io.IOException;
-import java.util.Arrays;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 844c91c5ce081..7e1b49c925fe9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -286,6 +286,13 @@
   </description>
 </property>
 
+<property>
+  <name>mapreduce.task.ping-for-liveliness-check.enabled</name>
+  <value>false</value>
+  <description>Whether to consider ping from tasks in liveliness check.
+  </description>
+</property>
+
 <property>
   <name>mapreduce.map.memory.mb</name>
   <value>-1</value>
@@ -1653,14 +1660,18 @@
   <name>yarn.app.mapreduce.client-am.ipc.max-retries</name>
   <value>3</value>
   <description>The number of client retries to the AM - before reconnecting
-    to the RM to fetch Application Status.</description>
+    to the RM to fetch Application Status.
+    In other words, it is the ipc.client.connect.max.retries to be used during
+    reconnecting to the RM and fetching Application Status.</description>
 </property>
 
 <property>
   <name>yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts</name>
   <value>3</value>
   <description>The number of client retries on socket timeouts to the AM - before
-    reconnecting to the RM to fetch Application Status.</description>
+    reconnecting to the RM to fetch Application Status.
+    In other words, it is the ipc.client.connect.max.retries.on.timeouts to be used during
+    reconnecting to the RM and fetching Application Status.</description>
 </property>
 
 <property>
@@ -1715,7 +1726,7 @@
   must specify the appropriate classpath for that archive, and the name of
   the archive must be present in the classpath.
   If mapreduce.app-submission.cross-platform is false, platform-specific
-  environment vairable expansion syntax would be used to construct the default
+  environment variable expansion syntax would be used to construct the default
   CLASSPATH entries.
   For Linux:
   $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*,
@@ -2231,4 +2242,23 @@
   </description>
 </property>
 
+<!-- not yet enabled by default.
+
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+  <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+  <description>
+    The default committer factory for ABFS is for the manifest committer with
+    abfs-specific tuning.
+  </description>
+</property>
+
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.gs</name>
+  <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+  <description>
+    The default committer factory for google cloud storage is for the manifest committer.
+  </description>
+</property>
+-->
 </configuration>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
index 7d942caba7ea5..a0bdf3f714752 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
@@ -48,7 +48,7 @@ When you make a request for the mapreduce application master information, the in
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
-| appId | long | The application id |
+| appId | string | The application id |
 | startedOn | long | The time the application started (in ms since epoch) |
 | name | string | The name of the application |
 | user | string | The user name of the user who started the application |
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md
new file mode 100644
index 0000000000000..12fe1f0b5f2bd
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md
@@ -0,0 +1,677 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# The Manifest Committer for Azure and Google Cloud Storage
+
+This document how to use the _Manifest Committer_.
+
+The _Manifest_ committer is a committer for work which provides
+performance on ABFS for "real world" queries,
+and performance and correctness on GCS.
+It also works with other filesystems, including HDFS.
+However, the design is optimized for object stores where
+listing operatons are slow and expensive.
+
+The architecture and implementation of the committer is covered in
+[Manifest Committer Architecture](manifest_committer_architecture.html).
+
+
+The protocol and its correctness are covered in
+[Manifest Committer Protocol](manifest_committer_protocol.html).
+
+It was added in March 2022, and should be considered unstable
+in early releases.
+
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+## Problem:
+
+The only committer of work from Spark to Azure ADLS Gen 2 "abfs://" storage
+which is safe to use is the "v1 file committer".
+
+This is "correct" in that if a task attempt fails, its output is guaranteed not
+to be included in the final out. The "v2" commit algorithm cannot meet that
+guarantee, which is why it is no longer the default.
+
+But: it is slow, especially on jobs where deep directory trees of output are used.
+Why is it slow? It's hard to point at a particular cause, primarily because of
+the lack of any instrumentation in the `FileOutputCommitter`.
+Stack traces of running jobs generally show `rename()`, though list operations
+do surface too.
+
+On Google GCS, neither the v1 nor v2 algorithm are _safe_ because the google
+filesystem doesn't have the atomic directory rename which the v1 algorithm
+requires.
+
+A further issue is that both Azure and GCS storage may encounter scale issues
+with deleting directories with many descendants.
+This can trigger timeouts because the FileOutputCommitter assumes that
+cleaning up after the job is a fast call to `delete("_temporary", true)`.
+
+## Solution.
+
+The _Intermediate Manifest_ committer is a new committer for
+work which should deliver performance on ABFS
+for "real world" queries, and performance and correctness on GCS.
+
+This committer uses the extension point which came in for the S3A committers.
+Users can declare a new committer factory for abfs:// and gcs:// URLs.
+A suitably configured spark deployment will pick up the new committer.
+
+Directory performance issues in job cleanup can be addressed by two options
+1. The committer will parallelize deletion of task attempt directories before
+   deleting the `_temporary` directory.
+1. Cleanup can be disabled. .
+
+The committer can be used with any filesystem client which has a "real" file rename()
+operation.
+It has been optimised for remote object stores where listing and file probes
+are expensive -the design is less likely to offer such signifcant speedup
+on HDFS -though the parallel renaming operations will speed up jobs
+there compared to the classic v1 algorithm.
+
+# <a name="how"></a> How it works
+
+The full details are covered in [Manifest Committer Architecture](manifest_committer_architecture.html).
+
+# <a name="use"></a> Using the committer
+
+The hooks put in to support the S3A committers were designed to allow every
+filesystem schema to provide their own committer.
+See [Switching To an S3A Committer](../../hadoop-aws/tools/hadoop-aws/committers.html#Switching_to_an_S3A_Committer)
+
+A factory for the abfs schema would be defined in
+`mapreduce.outputcommitter.factory.scheme.abfs` ; and a similar one for `gcs`.
+
+Some matching spark configuration changes, especially for parquet binding, will be required.
+These can be done in `core-site.xml`, if it is not defined in the `mapred-default.xml` JAR.
+
+
+```xml
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+  <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.gs</name>
+  <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+</property>
+```
+
+## Binding to the manifest committer in Spark.
+
+In Apache Spark, the configuration can be done either with command line options (after the '--conf') or by using the `spark-defaults.conf` file. The following is an example of using `spark-defaults.conf` also including the configuration for Parquet with a subclass of the parquet
+committer which uses the factory mechansim internally.
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.abfs org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.gs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+```
+
+
+### <a name="committerinfo"></a> Using the Cloudstore `committerinfo` command to probe committer bindings.
+
+The hadoop committer settings can be validated in a recent build of [cloudstore](https://github.com/steveloughran/cloudstore)
+and its `committerinfo` command.
+This command instantiates a committer for that path through the same factory mechanism as MR and spark jobs use,
+then prints its `toString` value.
+
+```
+hadoop jar cloudstore-1.0.jar committerinfo abfs://testing@ukwest.dfs.core.windows.net/
+
+2021-09-16 19:42:59,731 [main] INFO  commands.CommitterInfo (StoreDurationInfo.java:<init>(53)) - Starting: Create committer
+Committer factory for path abfs://testing@ukwest.dfs.core.windows.net/ is
+ org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory@3315d2d7
+  (classname org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory)
+2021-09-16 19:43:00,897 [main] INFO  manifest.ManifestCommitter (ManifestCommitter.java:<init>(144)) - Created ManifestCommitter with
+   JobID job__0000, Task Attempt attempt__0000_r_000000_1 and destination abfs://testing@ukwest.dfs.core.windows.net/
+Created committer of class org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitter:
+ ManifestCommitter{ManifestCommitterConfig{destinationDir=abfs://testing@ukwest.dfs.core.windows.net/,
+   role='task committer',
+   taskAttemptDir=abfs://testing@ukwest.dfs.core.windows.net/_temporary/manifest_job__0000/0/_temporary/attempt__0000_r_000000_1,
+   createJobMarker=true,
+   jobUniqueId='job__0000',
+   jobUniqueIdSource='JobID',
+   jobAttemptNumber=0,
+   jobAttemptId='job__0000_0',
+   taskId='task__0000_r_000000',
+   taskAttemptId='attempt__0000_r_000000_1'},
+   iostatistics=counters=();
+
+gauges=();
+
+minimums=();
+
+maximums=();
+
+means=();
+}
+
+```
+
+
+## Verifying that the committer was used
+
+The new committer will write a JSON summary of the operation, including statistics, in the `_SUCCESS` file.
+
+If this file exists and is zero bytes long: the classic `FileOutputCommitter` was used.
+
+If this file exists and is greater than zero bytes long, either the manifest committer was used,
+or in the case of S3A filesystems, one of the S3A committers. They all use the same JSON format.
+
+# <a name="configuration"></a> Configuration options
+
+Here are the main configuration options of the committer.
+
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.manifest.committer.delete.target.files` | Delete target files? | `false` |
+| `mapreduce.manifest.committer.io.threads` | Thread count for parallel operations | `64` |
+| `mapreduce.manifest.committer.summary.report.directory` | directory to save reports. | `""` |
+| `mapreduce.manifest.committer.cleanup.parallel.delete` | Delete temporary directories in parallel | `true` |
+| `mapreduce.fileoutputcommitter.cleanup.skipped` | Skip cleanup of `_temporary` directory| `false` |
+| `mapreduce.fileoutputcommitter.cleanup-failures.ignored` | Ignore errors during cleanup | `false` |
+| `mapreduce.fileoutputcommitter.marksuccessfuljobs` | Create a `_SUCCESS` marker file on successful completion. (and delete any existing one in job setup) | `true` |
+
+There are some more, as covered in the (Advanced)[#advanced] section.
+
+
+## <a name="scaling"></a> Scaling jobs `mapreduce.manifest.committer.io.threads`
+
+The core reason that this committer is faster than the classic `FileOutputCommitter`
+is that it tries to parallelize as much file IO as it can during job commit, specifically:
+
+* task manifest loading
+* deletion of files where directories will be created
+* directory creation
+* file-by-file renaming
+* deletion of task attempt directories in job cleanup
+
+These operations are all performed in the same thread pool, whose size is set
+in the option `mapreduce.manifest.committer.io.threads`.
+
+Larger values may be used.
+
+XML
+```xml
+<property>
+  <name>mapreduce.manifest.committer.io.threads</name>
+  <value>200</value>
+</property>
+```
+
+spark-defaults.conf
+```
+spark.hadoop.mapreduce.manifest.committer.io.threads 200
+```
+
+A larger value than that of the number of cores allocated to
+the MapReduce AM or Spark Driver does not directly overload
+the CPUs, as the threads are normally waiting for (slow) IO
+against the object store/filesystem to complete.
+
+Caveats
+* In Spark, multiple jobs may be committed in the same process,
+  each of which will create their own thread pool during job
+  commit or cleanup.
+* Azure rate throttling may be triggered if too many IO requests
+  are made against the store. The rate throttling option
+  `mapreduce.manifest.committer.io.rate` can help avoid this.
+
+
+## <a name="deleting"></a> Optional: deleting target files in Job Commit
+
+The classic `FileOutputCommitter` deletes files at the destination paths
+before renaming the job's files into place.
+
+This is optional in the manifest committers, set in the option
+`mapreduce.manifest.committer.delete.target.files` with a default value of `false`.
+
+This increases performance and is safe to use when all files created by a job
+have unique filenames.
+
+Apache Spark does generate unique filenames for ORC and Parquet since
+[SPARK-8406](https://issues.apache.org/jira/browse/SPARK-8406)
+_Adding UUID to output file name to avoid accidental overwriting_
+
+Avoiding checks for/deleting target files saves one delete call per file being committed, so can
+save a significant amount of store IO.
+
+When appending to existing tables, using formats other than ORC and parquet,
+unless confident that unique identifiers
+are added to each filename, enable deletion of the target files.
+
+```
+spark.hadoop.mapreduce.manifest.committer.delete.target.files true
+```
+
+*Note 1:* the committer will skip deletion operations when it
+created the directory into which a file is to be renamed.
+This makes it slightly more efficient, at least if jobs
+appending data are creating and writing into new partitions.
+
+*Note 2:* the committer still requires tasks within a single
+job to create unique files. This is foundational for
+any job to generate correct data.
+
+# <a name="dynamic"></a> Spark Dynamic Partition overwriting
+
+Spark has a feature called "Dynamic Partition Overwrites",
+
+This can be initiated in SQL
+```SQL
+INSERT OVERWRITE TABLE ...
+```
+Or through DataSet writes where the mode is `overwrite` and the partitioning matches
+that of the existing table
+```scala
+sparkConf.set("spark.sql.sources.partitionOverwriteMode", "dynamic")
+// followed by an overwrite of a Dataset into an existing partitioned table.
+eventData2
+  .write
+  .mode("overwrite")
+  .partitionBy("year", "month")
+  .format("parquet")
+  .save(existingDir)
+```
+
+This feature is implemented in Spark, which
+1. Directs the job to write its new data to a temporary directory
+1. After job commit completes, scans the output to identify the leaf directories "partitions" into which data was written.
+1. Deletes the content of those directories in the destination table
+1. Renames the new files into the partitions.
+
+This is all done in spark, which takes over the tasks of scanning
+the intermediate output tree, deleting partitions and of
+renaming the new files.
+
+This feature also adds the ability for a job to write data entirely outside
+the destination table, which is done by
+1. writing new files into the working directory
+1. spark moving them to the final destination in job commit
+
+
+The manifest committer is compatible with dynamic partition overwrites
+on Azure and Google cloud storage as together they meet the core requirements of
+the extension:
+1. The working directory returned in `getWorkPath()` is in the same filesystem
+  as the final output.
+2. `rename()` is an `O(1)` operation which is safe and fast to use when committing a job.
+
+None of the S3A committers support this. Condition (1) is not met by
+the staging committers, while (2) is not met by S3 itself.
+
+To use the manifest committer with dynamic partition overwrites, the
+spark version must contain
+[SPARK-40034](https://issues.apache.org/jira/browse/SPARK-40034)
+_PathOutputCommitters to work with dynamic partition overwrite_.
+
+Be aware that the rename phase of the operation will be slow
+if many files are renamed -this is done sequentially.
+Parallel renaming would speed this up, *but could trigger the abfs overload
+problems the manifest committer is designed to both minimize the risk
+of and support recovery from*
+
+The spark side of the commit operation will be listing/treewalking
+the temporary output directory (some overhead), followed by
+the file promotion, done with a classic filesystem `rename()`
+call. There will be no explicit rate limiting here.
+
+*What does this mean?*
+
+It means that _dynamic partitioning should not be used on Azure Storage
+for SQL queries/Spark DataSet operations where many thousands of files are created.
+The fact that these will suffer from performance problems before
+throttling scale issues surface, should be considered a warning.
+
+# <a name="SUCCESS"></a> Job Summaries in `_SUCCESS` files
+
+The original hadoop committer creates a zero byte `_SUCCESS` file in the root of the output directory
+unless disabled.
+
+This committer writes a JSON summary which includes
+* The name of the committer.
+* Diagnostics information.
+* A list of some of the files created (for testing; a full list is excluded as it can get big).
+* IO Statistics.
+
+If, after running a query, this `_SUCCESS` file is zero bytes long,
+*the new committer has not been used*
+
+If it is not empty, then it can be examined.
+
+## <a name="printer"></a> Viewing `_SUCCESS` file files through the `ManifestPrinter` tool.
+
+The summary files are JSON, and can be viewed in any text editor.
+
+For a more succinct summary, including better display of statistics, use the `ManifestPrinter` tool.
+
+```bash
+hadoop org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter <path>
+```
+
+This works for the files saved at the base of an output directory, and
+any reports saved to a report directory.
+
+## <a name="summaries"></a> Collecting Job Summaries `mapreduce.manifest.committer.summary.report.directory`
+
+The committer can be configured to save the `_SUCCESS` summary files to a report directory,
+irrespective of whether the job succeed or failed, by setting a fileystem path in
+the option `mapreduce.manifest.committer.summary.report.directory`.
+
+The path does not have to be on the same
+store/filesystem as the destination of work. For example, a local fileystem could be used.
+
+XML
+
+```xml
+<property>
+  <name>mapreduce.manifest.committer.summary.report.directory</name>
+  <value>file:///tmp/reports</value>
+</property>
+```
+
+spark-defaults.conf
+
+```
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory file:///tmp/reports
+```
+
+This allows for the statistics of jobs to be collected irrespective of their outcome, Whether or not
+saving the `_SUCCESS` marker is enabled, and without problems caused by a chain of queries
+overwriting the markers.
+
+
+# <a name="cleanup"></a> Cleanup
+
+Job cleanup is convoluted as it is designed to address a number of issues which
+may surface in cloud storage.
+
+* Slow performance for deletion of directories.
+* Timeout when deleting very deep and wide directory trees.
+* General resilience to cleanup issues escalating to job failures.
+
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.fileoutputcommitter.cleanup.skipped` | Skip cleanup of `_temporary` directory| `false` |
+| `mapreduce.fileoutputcommitter.cleanup-failures.ignored` | Ignore errors during cleanup | `false` |
+| `mapreduce.manifest.committer.cleanup.parallel.delete` | Delete task attempt directories in parallel | `true` |
+
+The algorithm is:
+
+```
+if `mapreduce.fileoutputcommitter.cleanup.skipped`:
+  return
+if `mapreduce.manifest.committer.cleanup.parallel.delete`:
+  attempt parallel delete of task directories; catch any exception
+if not `mapreduce.fileoutputcommitter.cleanup.skipped`:
+  delete(`_temporary`); catch any exception
+if caught-exception and not `mapreduce.fileoutputcommitter.cleanup-failures.ignored`:
+  throw caught-exception
+```
+
+It's a bit complicated, but the goal is to perform a fast/scalable delete and
+throw a meaningful exception if that didn't work.
+
+When working with ABFS and GCS, these settings should normally be left alone.
+If somehow errors surface during cleanup, enabling the option to
+ignore failures will ensure the job still completes.
+Disabling cleanup even avoids the overhead of cleanup, but
+requires a workflow or manual operation to clean up all
+`_temporary` directories on a regular basis.
+
+
+# <a name="abfs"></a> Working with Azure ADLS Gen2 Storage
+
+To switch to the manifest committer, the factory for committers for destinations with `abfs://` URLs must
+be switched to the manifest committer factory, either for the application or
+the entire cluster.
+
+```xml
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+  <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+```
+
+This allows for ADLS Gen2 -specific performance and consistency logic to be used from within the committer.
+In particular:
+* the [Etag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) header
+can be collected in listings and used in the job commit phase.
+* IO rename operations are rate limited
+* recovery is attempted when throttling triggers rename failures.
+
+*Warning* This committer is not compatible with older Azure storage services
+(WASB or ADLS Gen 1).
+
+The core set of Azure-optimized options becomes
+
+```xml
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+  <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+
+<property>
+  <name>spark.hadoop.fs.azure.io.rate.limit</name>
+  <value>10000</value>
+</property>
+```
+
+And optional settings for debugging/performance analysis
+
+```xml
+<property>
+  <name>mapreduce.manifest.committer.summary.report.directory</name>
+  <value>abfs:// Path within same store/separate store</value>
+  <description>Optional: path to where job summaries are saved</description>
+</property>
+```
+
+## <a name="abfs-options"></a> Full set of ABFS options for spark
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.abfs org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory
+spark.hadoop.fs.azure.io.rate.limit 10000
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory  (optional: URI of a directory for job summaries)
+```
+
+## Experimental: ABFS Rename Rate Limiting `fs.azure.io.rate.limit`
+
+To avoid triggering store throttling and backoff delays, as well as other
+throttling-related failure conditions file renames during job commit
+are throttled through a "rate limiter" which limits the number of
+rename operations per second a single instance of the ABFS FileSystem client
+may issue.
+
+| Option | Meaning |
+|--------|---------|
+| `fs.azure.io.rate.limit` | Rate limit in operations/second for IO operations. |
+
+Set the option to `0` remove all rate limiting.
+
+The default value of this is set to 10000, which is the default IO capacity for
+an ADLS storage account.
+
+```xml
+<property>
+  <name>fs.azure.io.rate.limit</name>
+  <value>10000</value>
+  <description>maximum number of renames attempted per second</description>
+</property>
+```
+
+This capacity is set at the level of the filesystem client, and so not
+shared across all processes within a single application, let
+alone other applications sharing the same storage account.
+
+It will be shared with all jobs being committed by the same
+Spark driver, as these do share that filesystem connector.
+
+If rate limiting is imposed, the statistic `store_io_rate_limited` will
+report the time to acquire permits for committing files.
+
+If server-side throttling took place, signs of this can be seen in
+* The store service's logs and their throttling status codes (usually 503 or 500).
+* The job statistic `commit_file_rename_recovered`. This statistic indicates that
+  ADLS throttling manifested as failures in renames, failures which were recovered
+  from in the comitter.
+
+If these are seen -or other applications running at the same time experience
+throttling/throttling-triggered problems, consider reducing the value of
+`fs.azure.io.rate.limit`, and/or requesting a higher IO capacity from Microsoft.
+
+*Important* if you do get extra capacity from Microsoft and you want to use
+it to speed up job commits, increase the value of `fs.azure.io.rate.limit`
+either across the cluster, or specifically for those jobs which you wish
+to allocate extra priority to.
+
+This is still a work in progress; it may be expanded to support
+all IO operations performed by a single filesystem instance.
+
+# <a name="gcs"></a> Working with Google Cloud Storage
+
+The manifest committer is compatible with and tested against Google cloud storage through
+the gcs-connector library from google, which provides a Hadoop filesystem client for the
+schema `gs`.
+
+Google cloud storage has the semantics needed for the commit protocol
+to work safely.
+
+The Spark settings to switch to this committer are
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.gs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory  (optional: URI of a directory for job summaries)
+```
+
+The store's directory delete operations are `O(files)` so the value
+of `mapreduce.manifest.committer.cleanup.parallel.delete`
+SHOULD be left at the default of `true`.
+
+For mapreduce, declare the binding in `core-site.xml`or `mapred-site.xml`
+```xml
+<property>
+  <name>mapreduce.outputcommitter.factory.scheme.gcs</name>
+  <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+</property>
+```
+
+
+# <a name="hdfs"></a> Working with HDFS
+
+This committer _does_ work with HDFS, it has just been targeted at object stores with
+reduced performance on some operations, especially listing and renaming,
+and semantics too reduced for the classic `FileOutputCommitter` to rely on
+(specifically GCS).
+
+To use on HDFS, set the `ManifestCommitterFactory` as the committer factory for `hdfs://` URLs.
+
+Because HDFS does fast directory deletion, there is no need to parallelize deletion
+of task attempt directories during cleanup, so set
+`mapreduce.manifest.committer.cleanup.parallel.delete` to `false`
+
+The final spark bindings becomes
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.hdfs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.hadoop.mapreduce.manifest.committer.cleanup.parallel.delete false
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory  (optional: URI of a directory for job summaries)
+```
+
+# <a name="advanced"></a> Advanced Topics
+
+## Advanced Configuration options
+
+There are some advanced options which are intended for development and testing,
+rather than production use.
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.manifest.committer.store.operations.classname` | Classname for Manifest Store Operations | `""` |
+| `mapreduce.manifest.committer.validate.output` | Perform output validation? | `false` |
+
+
+## Validating output  `mapreduce.manifest.committer.validate.output`
+
+The option `mapreduce.manifest.committer.validate.output` triggers a check of every renamed file to
+verify it has the expected length.
+
+This adds the overhead of a `HEAD` request per file, and so is recommended for testing only.
+
+There is no verification of the actual contents.
+
+## Controlling storage integration `mapreduce.manifest.committer.store.operations.classname`
+
+The manifest committer interacts with filesystems through implementations of the interface
+`ManifestStoreOperations`.
+It is possible to provide custom implementations for store-specific features.
+There is one of these for ABFS; when the abfs-specific committer factory is used this
+is automatically set.
+
+It can be explicitly set.
+```xml
+<property>
+  <name>mapreduce.manifest.committer.store.operations.classname</name>
+  <value>org.apache.hadoop.fs.azurebfs.commit.AbfsManifestStoreOperations</value>
+</property>
+```
+
+The default implementation may also be configured.
+
+```xml
+<property>
+  <name>mapreduce.manifest.committer.store.operations.classname</name>
+  <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem</value>
+</property>
+```
+
+There is no need to alter these values, except when writing new implementations for other stores,
+something which is only needed if the store provides extra integration support for the
+committer.
+
+## <a name="concurrent"></a> Support for concurrent jobs to the same directory
+
+It *may* be possible to run multiple jobs targeting the same directory tree.
+
+For this to work, a number of conditions must be met:
+
+* When using spark, unique job IDs must be set. This meangs the Spark distribution
+  MUST contain the patches for
+  [SPARK-33402](https://issues.apache.org/jira/browse/SPARK-33402)
+  and
+  [SPARK-33230](https://issues.apache.org/jira/browse/SPARK-33230).
+* Cleanup of the `_temporary` directory must be disabled by setting
+  `mapreduce.fileoutputcommitter.cleanup.skipped` to `true`.
+* All jobs/tasks must create files with unique filenames.
+* All jobs must create output with the same directory partition structure.
+* The job/queries MUST NOT be using Spark Dynamic Partitioning "INSERT OVERWRITE TABLE"; data may be lost.
+  This holds for *all* committers, not just the manifest committer.
+* Remember to delete the `_temporary` directory later!
+
+This has *NOT BEEN TESTED*
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md
new file mode 100644
index 0000000000000..d2b4f1ee8b446
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md
@@ -0,0 +1,335 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# Manifest Committer Architecture
+
+This document describes the architecture and other implementation/correctness
+aspects of the [Manifest Committer](manifest_committer.html)
+
+The protocol and its correctness are covered in [Manifest Committer Protocol](manifest_committer_protocol.html).
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+The _Manifest_ committer is a committer for work which provides performance on ABFS for "real world"
+queries, and performance and correctness on GCS.
+
+This committer uses the extension point which came in for the S3A committers.
+Users can declare a new committer factory for `abfs://` and `gcs://` URLs.
+It can be used through Hadoop MapReduce and Apache Spark.
+
+## Background
+
+### Terminology
+
+| Term | Meaning|
+|------|--------|
+| Committer |  A class which can be invoked by MR/Spark to perform the task and job commit operations. |
+| Spark Driver | The spark process scheduling the work and choreographing the commit operation.|
+| Job  | In MapReduce. the entire application. In spark, this is a single stage in a chain of work |
+| Job Attempt | A single attempt at a job. MR supports multiple Job attempts with recovery on partial job failure. Spark says "start again from scratch" |
+| Task | a subsection of a job, such as processing one file, or one part of a file |
+| Task ID |  ID of the task, unique within this job. Usually starts at 0 and is used in filenames (part-0000, part-001, etc.) |
+| Task attempt (TA) | An attempt to perform a task. It may fail, in which case MR/spark will schedule another. |
+| Task Attempt ID | A unique ID for the task attempt. The Task ID + an attempt counter.|
+|  Destination directory | The final destination of work.|
+| Job Attempt Directory | A temporary directory used by the job attempt. This is always _underneath_ the destination directory, so as to ensure it is in the same encryption zone as HDFS, storage volume in other filesystems, etc.|
+| Task Attempt directory | (also known as "Task Attempt Working Directory"). Directory exclusive for each task attempt under which files are written |
+| Task Commit | Taking the output of a Task Attempt and making it the final/exclusive result of that "successful" Task.|
+| Job Commit | aggregating all the outputs of all committed tasks and producing the final results of the job. |
+
+
+
+The purpose of a committer is to ensure that the complete output of
+a job ends up in the destination, even in the presence of failures of tasks.
+
+- _Complete:_ the output includes the work of all successful tasks.
+- _Exclusive:_ the output of unsuccessful tasks is not present.
+- _Concurrent:_ When multiple tasks are committed in parallel the output is the same as when
+  the task commits are serialized. This is not a requirement of Job Commit.
+- _Abortable:_  jobs and tasks may be aborted prior to job commit, after which their output is not visible.
+- _Continuity of correctness:_ once a job is committed, the output of any failed,
+  aborted, or unsuccessful task MUST NO appear at some point in the future.
+
+For Hive's classic hierarchical-directory-structured tables, job committing
+requires the output of all committed tasks to be put into the correct location
+in the directory tree.
+
+The committer built into `hadoop-mapreduce-client-core` module is the `FileOutputCommitter`.
+
+
+
+## The Manifest Committer: A high performance committer for Spark on Azure and Google storage.
+
+The Manifest Committer is a higher performance committer for ABFS and GCS storage
+for jobs which create file across deep directory trees through many tasks.
+
+It will also work on `hdfs://` and indeed, `file://` URLs, but
+it is optimized to address listing and renaming performance and throttling
+issues in cloud storage.
+
+It *will not* work correctly with S3, because it relies on an atomic rename-no-overwrite
+operation to commit the manifest file. It will also have the performance
+problems of copying rather than moving all the generated data.
+
+Although it will work with MapReduce
+there is no handling of multiple job attempts with recovery from previous failed
+attempts.
+
+### The Manifest
+
+A Manifest file is designed which contains (along with IOStatistics and some
+other things)
+
+1. A list of destination directories which must be created if they do not exist.
+1. A list of files to rename, recorded as (absolute source, absolute destination,
+   file-size) entries.
+
+### Task Commit
+
+Task attempts are committed by:
+
+1. Recursively listing the task attempt working dir to build
+   1. A list of directories under which files are renamed.
+   2. A list of files to rename: source, destination, size and optionally, etag.
+2. Saving this information in a manifest file in the job attempt directory with
+   a filename derived from the Task ID.
+   Note: writing to a temp file and then renaming to the final path will be used
+   to ensure the manifest creation is atomic.
+
+
+No renaming takes place —the files are left in their original location.
+
+The directory treewalk is single-threaded, then it is `O(directories)`,
+with each directory listing using one or more paged LIST calls.
+
+This is simple, and for most tasks, the scan is off the critical path of of the job.
+
+Statistics analysis may justify moving to parallel scans in future.
+
+
+### Job Commit
+
+Job Commit consists of:
+
+1. List all manifest files in the job attempt directory.
+1. Load each manifest file, create directories which do not yet exist, then
+   rename each file in the rename list.
+1. Save a JSON `_SUCCESS` file with the same format as the S3A committer (for
+   testing; use write and rename for atomic save)
+
+The job commit phase supports parallelization for many tasks and many files
+per task, specifically:
+
+1. Manifest tasks are loaded and processed in a pool of "manifest processor"
+   threads.
+2. Directory creation and file rename operations are each processed in a pool of "
+   executor" threads: many renames can execute in parallel as they use minimal
+   network IO.
+3. job cleanup can parallelize deletion of task attempt directories. This
+   is relevant as directory deletion is `O(files)` on Google cloud storage,
+   and also on ABFS when OAuth authentication is used.
+
+
+### Ancestor directory preparation
+
+Optional scan of all ancestors ...if any are files, delete.
+
+
+### Parent directory creation
+
+1. Probe shared directory map for directory existing. If found: operation is
+   complete.
+1. if the map is empty, call `getFileStatus()` on the path. Not found: create
+   directory, add entry and those of all parent paths Found and is directory:
+   add entry and those of all parent paths Found and is file: delete. then
+   create as before.
+
+Efficiently handling concurrent creation of directories (or delete+create) is going to be a
+troublespot; some effort is invested there to build the set of directories to
+create.
+
+### File Rename
+
+Files are renamed in parallel.
+
+A pre-rename check for anything being at that path (and deleting it) will be optional.
+With spark creating new UUIDs for each file, this isn't going to happen, and
+saves HTTP requests.
+
+
+### Validation
+
+Optional scan of all committed files and verify length and, if known,
+etag. For testing and diagnostics.
+
+## Benefits
+
+* Pushes the source tree list operations into the task commit phase, which is
+  generally off the critical path of execution
+* Provides an atomic task commit to GCS, as there is no expectation that
+  directory rename is atomic
+* It is possible to pass IOStatistics from workers in manifest.
+* Allows for some pre-rename operations similar to the S3A "Partitioned Staging
+  committer". This can be configured to delete all existing entries in
+  directories scheduled to be created -or fail if those partitions are
+  non-empty.
+  See [Partitioned Staging Committer](../../hadoop-aws/tools/hadoop-aws/committers.html#The_.E2.80.9CPartitioned.E2.80.9D_Staging_Committer)
+* Allows for an optional preflight validation check (verify no duplicate files created by different tasks)
+* Manifests can be viewed, size of output determined, etc, during
+  development/debugging.
+
+### Disadvantages
+
+* Needs a new manifest file format.
+* May makes task commit more complex.
+
+This solution is necessary for GCS and should be beneficial on ABFS as listing
+overheads are paid for in the task committers.
+
+# Implementation Details
+
+### Constraints
+
+A key goal is to keep the manifest committer isolated and neither
+touch the existing committer code nor other parts of the hadoop codebase.
+
+It must plug directly into MR and Spark without needing any changes
+other than already implemented for the S3A Committers
+
+* Self-contained: MUST NOT require changes to hadoop-common, etc.
+* Isolated: MUST NOT make changes to existing committers
+* Integrated: MUST bind via `PathOutputCommitterFactory`.
+
+As a result of this there's a bit of copy and paste from elsewhere,
+e.g. `org.apache.hadoop.util.functional.TaskPool`
+is based on S3ACommitter's `org.apache.hadoop.fs.s3a.commit.Tasks`.
+
+The` _SUCCESS` file MUST be compatible with the S3A JSON file.
+This is to ensure any existing test suites which validate
+S3A committer output can be retargeted at jobs executed
+by the manifest committer without any changes.
+
+
+#### Progress callbacks in job commit.
+
+When? Proposed: heartbeat until renaming finally finishes.
+
+#### Error handling and aborting in job commit.
+
+We would want to stop the entire job commit. Some atomic boolean "abort job"
+would need to be checked in the processing of each task committer thread's
+iteraton through a directory (or processing of each file?)
+Failures in listing or renaming will need to be escalated to halting the entire
+job commit. This implies that any IOE raised in asynchronous rename operation or
+in a task committer thread must:
+
+1. be caught
+1. be stored in a shared field/variable
+1. trigger the abort
+1. be rethrown at the end of the `commitJob()` call
+
+#### Avoiding deadlocks
+
+If a job commit stage is using a thread pool for per-task operations, e.g. loading
+files, that same thread pool MUST NOT be used for parallel operations within
+the per-task stage.
+
+As every `JobStage` is executed in sequence within task or job commit, it
+is safe to share the same thread pool across stages.
+
+In the current implementation, there is no parallel "per manifest" operation
+in job commit other than for actually loading the files.
+The operations to create directories and to rename files are actually
+performed without performing parallel processing of individual manifests.
+
+Directory Preparation: merge the directory lists of all manifests,
+then queue for creation the (hopefully very much smaller) set of unique
+directories.
+
+Rename: iterate through all manifests and queue their renames into a pool for
+renaming.
+
+#### Thread pool lifetimes
+
+The lifespan of thread pools is constrained to that of the stage configuration,
+which will be limited to within each of the `PathOutputCommitter` methods
+to setup, commit, abort and cleanup.
+
+This avoids the thread pool lifecycle problems of the S3A Committers.
+
+#### Scale issues similar to S3A HADOOP-16570.
+
+This was a failure in terasorting where many tasks each generated many files;
+the full list of files to commit (and the etag of every block) was built up in memory and
+validated prior to execution.
+
+The manifest committer assumes that the amount of data being stored in memory is less,
+because there is no longer the need to store an etag for every block of every
+file being committed.
+
+
+#### Duplicate creation of directories in the dest dir
+
+Combine all lists of directories to create and eliminate duplicates.
+
+## Implementation Architecture
+
+The implementation architecture reflects lessons from the S3A Connector.
+
+* Isolate the commit stages from the MR commit class, as that's got a complex lifecycle.
+* Instead, break up into series of _stages_ which can be tested in isolation
+  and chained to provided the final protocol.
+* Don't pass in MR data types (taskID etc) down into the stages -pass down a configuration
+  with general types (string etc).
+* Also pass in a callback for store operations, for ease of implementing a fake store.
+* For each stage: define preconditions and postconditions, failure modes. Test in isolation.
+
+#### Statistics
+
+The committer collects duration statistics on all the operations it performs/invokes
+against filesystems.
+* Those collected during task commit are saved to the manifest (excluding the time to
+save and rename that file)
+* When these manifests are loaded during job commit, these statistics are merged to
+form aggregate statistics of the whole job.
+* Which are saved to the `_SUCCESS` file
+* and to any copy of that file in the directory specified by
+  `mapreduce.manifest.committer.summary.report.directory`, if set.
+  to be saved.
+* The class `org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter`
+  can load and print these.
+
+IO statistics from filsystems and input and output streams used in a query are not
+collected.
+
+
+## Auditing
+
+When invoking the `ManifestCommitter` via the `PathOutputCommitter` API, the following
+attributes are added to the active (thread) context
+
+| Key   | Value           |
+|-------|-----------------|
+| `ji`  | Job ID          |
+| `tai` | Task Attempt ID |
+| `st`  | Stage           |
+
+These are also all set in all the helper threads performing work
+as part of a stage's execution.
+
+Any store/FS which supports auditing is able to collect this data
+and include in their logs.
+
+To ease backporting, all audit integration is in the single class
+`org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration`.
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_protocol.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_protocol.md
new file mode 100644
index 0000000000000..c5af28de70c34
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_protocol.md
@@ -0,0 +1,617 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# Manifest Committer Protocol
+
+This document describes the commit protocol
+ of the [Manifest Committer](manifest_committer.html)
+
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+## Background
+
+### Terminology
+
+| Term | Meaning|
+|------|--------|
+| Committer |  A class which can be invoked by MR Spark to perform the task and job commit operations. |
+| Spark Driver | The spark process scheduling the work and choreographing the commit operation.|
+| Job: in MapReduce | The entire application. In spark, this is a single stage in a chain of work |
+| Job Attempt | A single attempt at a job. MR supports multiple Job attempts with recovery on partial job failure. Spark says "start again from scratch" |
+| Task | a subsection of a job, such as processing one file, or one part of a file |
+| Task ID |  ID of the task, unique within this job. Usually starts at 0 and is used in filenames (part-0000, part-001, etc.) |
+| Task attempt (TA) | An attempt to perform a task. It may fail, in which case MR/spark will schedule another. |
+| Task Attempt ID | A unique ID for the task attempt. The Task ID + an attempt counter.|
+|  Destination directory | The final destination of work.|
+| Job Attempt Directory | A temporary directory used by the job attempt. This is always _underneath_ the destination directory, so as to ensure it is in the same encryption zone as HDFS, storage volume in other filesystems, etc.|
+| Task Attempt directory | Directory under the Job Attempt Directory where task attempts create subdiretories for their own work |
+| Task Attempt Working Directory| Directory exclusive for each task attempt under which files are written  |
+| Task Commit | Taking the output of a Task Attempt and making it the final/exclusive result of that "successful" Task.  |
+| Job Commit | aggregating all the outputs of all committed tasks and producing the final results of the job. |
+
+
+
+The purpose of a committer is to ensure that the complete output of
+a job ends up in the destination, even in the presence of failures of tasks.
+
+- _Complete:_ the output includes the work of all successful tasks.
+- _Exclusive:_ the output of unsuccessful tasks is not present.
+- _Concurrent:_ When multiple tasks are committed in parallel the output is the same as when
+  the task commits are serialized. This is not a requirement of Job Commit.
+- _Abortable:_  jobs and tasks may be aborted prior to job commit, after which their output is not visible.
+- _Continuity of correctness:_ once a job is committed, the output of any failed,
+  aborted, or unsuccessful task MUST NO appear at some point in the future.
+
+For Hive's classic hierarchical-directory-structured tables, job committing
+requires the output of all committed tasks to be put into the correct location
+in the directory tree.
+
+The committer built into `hadoop-mapreduce-client-core` module is the `FileOutputCommitter`.
+
+
+It has two algorithms, v1 and v2.
+
+The v1 algorithm is resilient to all forms of task failure, but slow
+when committing the final aggregate output as it renames each newly created file
+to the correct place in the table one by one.
+
+The v2 algorithm is not considered safe because the output is visible when individual
+tasks commit, rather than being delayed until job commit.
+It is possible for multiple task attempts to get their data into the output
+directory tree, and if a job fails/is aborted before the job is committed,
+thie output is visible.
+
+## File Output Committer V1 and V2
+
+### File Output Committer V1 and V2 Commit algorithms
+
+#### Task attempt execution (V1 and V2)
+
+job attempt directory in `$dest/__temporary/$jobAttemptId/` contains all output
+of the job in progress every task attempt is allocated its own task attempt dir
+`$dest/__temporary/$jobAttemptId/__temporary/$taskAttemptId`
+
+All work for a task is written under the task attempt directory. If the output
+is a deep tree with files at the root, the task attempt dir will end up with a
+similar structure, with the files it has generated and the directories above
+them.
+
+### MapReduce V1 algorithm:
+
+#### v1 Task commit
+
+The task attempt dir is renamed directly underneath the job attempt dir
+
+```
+rename(
+  $dest/__temporary/$jobAttemptId/__temporary/$taskAttemptId
+  $dest/__temporary/$jobAttemptId/$taskId)
+```
+
+#### V1 Job Commit
+
+For each committed task, all files underneath are renamed into the destination
+directory, with a filename relative from the base directory of the task remapped
+to that of the dest dir.
+
+That is, everything under `$dest/__temporary/$jobAttemptId/$taskId` is converted
+to a path under `$dest`.
+
+A recursive treewalk identifies the paths to rename in each TA directory.
+There's some optimisation if the task directory tree contains a subdirectory
+directory which does not exist under the destination: in this case the whole
+directory can be renamed. If the directory already exists, a file-by-file merge
+takes place for that dir, with the action for subdirectories again depending on
+the presence of the destination.
+
+As a result, if the output of each task goes a separate final directory (e.g the
+final partition is unique to a single task), the rename is O(1) for the dir,
+irrespective of children. If the output is to be in the same dir as other
+tasks (or updating existing directories), then the rename performance becomes O(
+files).
+
+Finally, a 0-byte `_SUCCESS` file is written iff `mapreduce.fileoutputcommitter.marksuccessfuljobs` is true.
+
+### MapReduce V2 algorithm:
+
+#### V2 Task commit
+
+The files under the task attempt dir are renamed one by one into the destination
+directory. There's no attempt at optimising directory renaming, because other
+tasks may be committing their work at the same time. It is therefore `O(files)` +
+the cost of listing the directory tree. Again: done with a recursive treewalk,
+not a deep `listFiles(path, recursive=true)` API, which would be faster on HDFS
+and (though not relevant here) S3.
+
+#### V2 Job Commit
+
+A 0-byte `_SUCCESS` file is written iff `mapreduce.fileoutputcommitter.marksuccessfuljobs`
+is true.
+
+
+
+### Why the V2 committer is incorrect/unsafe
+
+If, for a Task T1, Task Attempt 1 (T1A1) fails before committing, the driver
+will schedule a new attempt "T1A2", and commit it. All is good.
+
+But: if T1A1 was given permission to commit and it failed during the commit
+process, some of its output may have been written to the destination directory.
+
+If attempt T1A2 was then told to commit, then if and only if its output had the
+exact set of file names would any already-renamed files be overwritten. If
+different filenames were generated, then the output would contain files of T1A1
+and T1A2.
+
+If T1A1 became partitioned during the commit process, then the job committer
+would schedule another attempt and commit its work. However, if T1A1 still had
+connectivity to the filesystem, it could still be renaming files. The output of
+the two tasks could be intermingled even if the same filenames were used.
+
+## Background: the S3A Committers
+
+The paper, [_A Zero-Rename Committer_](https://github.com/steveloughran/zero-rename-committer/releases/),
+Loughran et. al., covers these committers
+
+It also describes the commit problem, defines correctness, and describes the
+algorithms of the v1 and v2 committers, as well as those of the S3A committers,
+IBM Stocator committer and what we know of EMR's Spark committer.
+
+The `hadoop-aws` JAR contains a pair of committers, "Staging" and "Magic". Both
+of these are implementations of the same problem: safely and rapidly committing
+work to an S3 object store.
+
+The committers take advantage of the fact that S3 offers an atomic way to create
+a file: the PUT request.
+
+Files either exist or they don't. A file can be uploaded direct to its
+destination, and it is only when the upload completes that the file is manifest
+-overwriting any existing copy.
+
+For large files, a multipart upload allows this upload operation to be split
+into a series of POST requests
+
+1 `initiate-upload (path -> upload ID)`
+1. `upload part(path, upload ID, data[]) -> checksum.`
+   This can be parallelised. Up to 10,000 parts can be uploaded to a single
+   object. All but the final part must be >= 5MB.
+1. `complete-upload (path, upload ID, List<checksum>)`
+   this manifests the file, building it from the parts in the sequence of blocks
+   defined by the ordering of the checksums.
+
+The secret for the S3A committers is that the final POST request can be delayed
+until the job commit phase, even though the files are uploaded during task
+attempt execution/commit. The task attempts need to determine the final
+destination of each file, upload the data as part of a multipart operation, then
+save the information needed to complete the upload in a file which is later read
+by the job committer and used in a POST request.
+
+### Staging Committer
+
+The _Staging Committer_ is based on the contribution by Ryan Blue of Netflix.
+it  relies on HDFS to be the consistent store to propagate the `.pendingset` files.
+
+The working directory of each task attempt is in the local filesystem, "the
+staging directory". The information needed to complete the uploads is passed
+from Task Attempts to the Job Committer by using a v1 FileOutputCommitter
+working with the cluster HDFS filesystem. This ensures that the committer has
+the same correctness guarantees as the v1 algorithm.
+
+1. Task commit consists of uploading all files under the local filesystem's task
+   attempt working directory to their final destination path, holding back on
+   the final manifestation POST.
+1. A JSON file containing all information needed to complete the upload of all
+   files in the task attempt is written to the Job Attempt directory of of the
+   wrapped committer working with HDFS.
+1. Job commit: load in all the manifest files in the HDFS job attempt directory,
+   then issued the POST request to complete the uploads. These are parallelised.
+
+
+### The Magic Committer
+
+The _Magic Committer_ is purely-S3A and takes advantage and of
+the fact the authorts could make changes within the file system client itself.
+
+"Magic" paths are defined which, when opened for writing under, initiate a
+multi-party upload to the final destination directory. When the output stream is
+`close()`d, a zero byte marker file is written to the magic path, and a JSON
+.pending file containing all the information needed to complete the upload is
+saved.
+
+Task commit:
+1. List all `.pending` files under each task attempt's magic directory;
+1. Aggregate to a `.pendingset` file
+1. Save to the job attempt directory with the task ID.
+
+Job commit:
+
+1. List `.pendingset` files in the job attempt directory
+1. Complete the uploads with POST requests.
+
+The Magic committer absolutely requires a consistent S3 Store -originally with
+S3Guard. Now that S3 is consistent, raw S3 can be used. It does not need HDFS
+or any other filesystem with `rename()`.
+
+### Correctness
+
+The S3A committer is considered correct because
+
+1. Nothing is materialized until job commit.
+1. Only one task attempt's manifest can be saved to the job attempt directory.
+   Hence: only of the TA's files of the same task ID are exclusively committed.
+1. The staging committer's use of HDFS to pass manifests from TAs to the Job
+   committer ensures that S3's eventual consistency would not cause manifests to
+   be missed.
+1. Until S3 was consistent, the magic committer relied on S3Guard to provide the
+   list consistency needed during both task- and job- commit.
+1. The authors and wider community fixed all the issues related to the committers
+   which have surfaced in production.
+
+Significant issues which were fixed include:
+
+* [HADOOP-15961](https://issues.apache.org/jira/browse/HADOOP-15961).
+  S3A committers: make sure there's regular progress() calls.
+* [HADOOP-16570](https://issues.apache.org/jira/browse/HADOOP-16570).
+  S3A committers encounter scale issues.
+* [HADOOP-16798](https://issues.apache.org/jira/browse/HADOOP-16798).
+  S3A Committer thread pool shutdown problems.
+* [HADOOP-17112](https://issues.apache.org/jira/browse/HADOOP-17112).
+  S3A committers can't handle whitespace in paths.
+* [HADOOP-17318](https://issues.apache.org/jira/browse/HADOOP-17318).
+  Support concurrent S3A commit jobs with same app attempt ID.
+* [HADOOP-17258](https://issues.apache.org/jira/browse/HADOOP-17258).
+  MagicS3GuardCommitter fails with `pendingset` already exists
+* [HADOOP-17414](https://issues.apache.org/jira/browse/HADOOP-17414]).
+  Magic committer files don't have the count of bytes written collected by spark
+* [SPARK-33230](https://issues.apache.org/jira/browse/SPARK-33230)
+  Hadoop committers to get unique job ID in `spark.sql.sources.writeJobUUID`
+* [SPARK-33402](https://issues.apache.org/jira/browse/SPARK-33402)
+  Jobs launched in same second have duplicate MapReduce JobIDs
+* [SPARK-33739](https://issues.apache.org/jira/browse/SPARK-33739]).
+  Jobs committed through the S3A Magic committer don't report
+  the bytes written (depends on HADOOP-17414)
+
+Of those which affected the correctness rather than scale/performance/UX:
+HADOOP-17258 involved the recovery from a failure after TA1 task commit had
+completed —but had failed to report in. SPARK-33402, SPARK-33230 and
+HADOOP-17318 are all related: if two spark jobs/stages started in the
+same second, they had the same job ID. This caused the HDFS directories used by
+the staging committers to be intermingled.
+
+What is notable is this: these are all problems which the minimal integration
+test suites did not discover.
+
+The good news: we now know of these issues and are better placed to avoid
+replicating them again. And know what to write tests for.
+
+## The V1 committer: slow in Azure and slow and unsafe on GCS.
+
+The V1 committer underperforms on ABFS because:
+
+1. Directory listing and file renaming is somewhat slower with ABFS than it is
+   with HDFS.
+1. The v1 committer sequentially commits the output of each task through a
+   listing of each committed task's output, moving directories when none exist
+   in the destination, merging files into extant directories.
+
+The V2 committer is much faster in the job commit because it performs the list
+and rename process in the task commit. Which, because it is non-atomic, is why
+it is considered dangerous to use. What the V2 task commit algorithm does show is
+that it is possible to parallelise committing the output of different tasks by
+using file-by-file rename exclusively.
+
+The V1 committer underperforms on GCS because even the task commit operation,
+—directory rename—, is a non-atomic `O(files)` operation.
+This also means that it is unsafe.
+
+If the task attempt has partitioned and the spark driver schedules/commits another TA, then,
+the task dir may contain 1+ file from the first attempt.
+
+----------------------------------------------------------------------------------------
+
+# The Manifest Committer Protocol
+
+## Requirements of the Store
+
+Stores/filesystems supported by this committer MUST:
+
+* Have consistent listings.
+* Have an atomic `O(1)` file rename operation.
+
+Stores/filesystems supported by this committer SHOULD:
+
+* Rename files successfully, even under load. ABFS does not do this,
+  so special recovery is provided there.
+* Implement the `EtagSource` interface of HADOOP-17979.
+  This is used for ABFS rename recovery, and for optional
+  validation of the final output.
+
+Stores/filesystems supported by this committer MAY:
+
+* Have list operations with high latency.
+* Reject calls under load with throttling responses,
+  which MUST be handled in the filesystem connector.
+
+Stores/filesystems supported by this committer MAY NOT:
+
+* Support atomic directory rename. This is never used except optionally in cleanup.
+* Support `O(1)` directory deletion. The `CleanupJobStage` assumes this is not
+  the case and so deletes task attempt directories in parallel.
+* Support an atomic `create(Path, overwrite=false)` operation.
+  The manifests are committed by writing to a path including the task attempt ID,
+  then renamed to their final path.
+* Support fast `listFiles(path, recursive=true)` calls.
+  This API call is not used.
+
+When compared with the `FileOutputCommitter`, the requirements
+which have been removed are:
+
+* Atomic directory rename.
+* `O(1)` directory deletion.
+* Fast directory listings.
+* The implicit absence of throttling behaviors.
+
+HDFS meets all those requirements, so does not benefit significantly from
+this committer, though it will still work there.
+
+The S3 store does not meet the rename requirements of this committer,
+even now that it is consistent.
+This committer is not safe to use on S3.
+
+### Task and Job IDs
+
+Every job MUST have a unique ID.
+
+The implementation expects the Spark runtime to have the relevant patches to
+ensure this.
+
+The job ID is used to name temporary directories, rather than using the classic
+incrementing natural numbering scheme of `_temporary/0/`.
+That scheme comes from MapReduce where job attempts of attempt ID &gt; 1
+look for tasks committed by predecessors and incorporate that into their
+results.
+
+This committer targets Spark, where there is no attempt at recovery.
+By using the job ID in paths, if jobs are configured to _not_ delete
+all of `_temporary` in job cleanup/abort, then multiple jobs
+MAY be executed using the same table as their destination.
+
+Task IDs and Task Attempt IDs will be derived from Job IDs as usual.
+
+It is expected that filenames of written files SHALL be unique.
+This is done in Spark for ORC and Parquet files, and allows for
+checks for destination files to be omitted by default.
+
+
+## Directory Structure
+
+Given a destination directory `destDir: Path`
+
+A job of id `jobID: String` and attempt number `jobAttemptNumber:int`
+will use the directory:
+
+```
+$destDir/_temporary/manifest_$jobID/$jobAttemptNumber/
+```
+
+For its work (note: it will actually format that final subdir with `%02d`).
+
+This is termed the _Job Attempt Directory_
+
+Under the Job Attempt Directory, a subdirectory `tasks` is
+created. This is termed the _Task Attempt Directory_.
+Every task attempt will have its own subdirectory of this,
+into which its work will be saved.
+
+Under the Job Attempt Directory, a subdirectory `manifests` is created.
+This is termed the _y_.
+
+The manifests of all committed tasks will be saved to this
+directory with the filename of
+`$taskId-manifest.json`
+
+The full path
+
+```
+$destDir/_temporary/manifest_$jobID/$jobAttemptNumber/manifests/$taskId-manifest.json
+```
+
+Is the final location for the manifest of all files created by
+a committed task. It is termed the _Manifest Path of a Committed Task_.
+
+Task attempts will save their manifest into this directory with
+a temporary filename
+`$taskAttemptId-manifest.json.tmp`.
+
+This is termed the _Temporary Path of a Task Attempt's Manifest_.
+
+For the job and task operations then, the following paths are
+defined.
+```
+let jobDirectory = "$destDir/_temporary/manifest_$jobID/"
+let jobAttemptDirectory = jobDirectory + "$jobAttemptNumber/"
+let manifestDirectory = jobAttemptDirectory + "manifests/"
+let taskAttemptDirectory = jobAttemptDirectory + "tasks/"
+```
+
+And for each task attempt, the following paths are also defined
+
+```
+let taskAttemptWorkingDirectory = taskAttemptDirectory + "$taskAttemptId"
+let taskManifestPath = manifestDirectory + "$taskId-manifest.json"
+let taskAttemptTemporaryManifestPath = manifestDirectory + "$taskAttemptId-manifest.json"
+```
+
+## Core Algorithm of the Protocol
+
+1. Each Task attempt writes all its files to a unique directory tree under the
+   Task Attempt Directory.
+2. Task Commit consists of a recursive scan of the directory for that task attempt,
+   creating a list of directories and a list of files.
+3. These lists are saved as a JSON manifest file.
+4. Job commit consists of listing all of the JSON manifest files,
+   loading their contents, creating the aggregate set of destination directories
+   and renaming all files into their final destinations.
+
+
+### The Intermediate Manifest
+
+This is JSON file is designed which contains (along with IOStatistics and some diagnostics)
+
+1. A list of destination directories which must be created if they do not exist.
+1. A list of files to rename as (absolute source, absolute destination,
+   file-size) entries.
+
+### Job Setup
+
+```
+mkdir(jobAttemptDirectory)
+mkdir(manifestDirectory)
+mkdir(taskAttemptDirectory)
+```
+
+### Task Setup
+
+```
+mkdir(taskAttemptWorkingDirectory)
+```
+
+### Task Commit
+
+Task attempts are committed by:
+
+1. Recursively listing the task attempt working dir to build
+   1. A list of destination directories under which files will be renamed,
+      and their status (exists, not_found, file)
+   2. A list of files to rename: source, destination, size and optionally, etag.
+2. These lists populate a JSON file, the _Intermediate Manifest_.
+3. The task attempt saves this file to its _Temporary Path of a Task Attempt's
+   Manifest_.
+4. The task attempt then deletes the _Manifest Path of a Committed Task_ and
+   renames its own manifest file to that path.
+5. If the rename succeeeds, the task commit is considered a success.
+
+No renaming takes place at this point.: the files are left in their original location until
+renamed in job commit.
+
+```
+let (renames, directories) = scan(taskAttemptWorkingDirectory)
+let manifest = new Manifest(renames, directories)
+
+manifest.save(taskAttemptTemporaryManifestPath)
+rename(taskAttemptTemporaryManifestPath, taskManifestPath)
+```
+
+### Task Abort/cleanup
+
+```
+delete(taskAttemptWorkingDirectory)
+```
+
+### Job Commit
+
+Job Commit consists of:
+
+1. List all manifest files in the job attempt directory.
+2. Load each manifest file, create directories which do not yet exist, then
+   rename each file in the rename list.
+3. Optionally save a JSON `_SUCCESS` file with the same format as the S3A committer (for
+   testing; use write and rename for atomic save)
+
+The job commit phase supports parallelization for many tasks and many files
+per task, specifically there is a thread pool for parallel store IO
+
+1. Manifest tasks are loaded and processed in parallel.
+1. Deletion of files where directories are intended to be created.
+1. Creation of leaf directories.
+1. File rename.
+1. In cleanup and abort: deletion of task attempt directories
+1. If validation of output is enabled for testing/debugging: getFileStatus calls
+   to compare file length and, if possible etags.
+
+```
+let manifestPaths = list("$manifestDirectory/*-manifest.json")
+let manifests = manifestPaths.map(p -> loadManifest(p))
+let directoriesToCreate = merge(manifests.directories)
+let filesToRename = concat(manifests.files)
+
+directoriesToCreate.map(p -> mkdirs(p))
+filesToRename.map((src, dest, etag) -> rename(src, dest, etag))
+
+if mapreduce.fileoutputcommitter.marksuccessfuljobs then
+  success.save("$destDir/_SUCCESS")
+
+```
+
+Implementation Note:
+
+To aid debugging and development, the summary be saved to a location
+in the same _or different_ filesystem; the intermediate
+manifests may be renamed to a location in the target filesystem.
+
+```
+if summary.report.directory != "" then
+  success.save("${summary.report.directory}/$jobID.json")
+if diagnostics.manifest.directory != null then
+  rename($manifestDirectory, "${diagnostics.manifest.directory}/$jobID")
+```
+
+The summary report is saved even if job commit fails for any reason
+
+### Job Abort/cleanup
+
+Job cleanup is nominally one of deleting the job directory
+```
+delete(jobDirectory)
+```
+
+To address scale issues with the object stores, this SHALL be preceeded by
+a (parallelized) deletion of all task attempt working directories
+
+```
+let taskAttemptWorkingDirectories = list("taskAttemptDirectory")
+taskAttemptWorkingDirectories.map(p -> delete(p))
+```
+
+
+## Benefits of the new protocol
+
+* Pushes the source tree list operations into the task commit phase, which is
+  generally off the critical path of execution.
+* Reduces the number of directories probed/created to the aggregate set of
+  output directories, with all duplicates eliminated.
+* File rename can be parallelized, with the limits being that of configured
+  thread pool sizes and/or any rate limiting constraints.
+* Provides an atomic task commit to GCS, as there is no expectation that
+  directory rename is atomic.
+* Permits pass IOStatistics from tasks attempts to the job committer via the manifests.
+* Allows for some pre-rename operations in the Job Committer
+  similar to the S3A "Partitioned Staging committer".
+  This can be configured to delete all existing entries in
+  directories scheduled to be created -or fail if those partitions are
+  non-empty.
+  See [Partitioned Staging Committer](../../hadoop-aws/tools/hadoop-aws/committers.html#The_.E2.80.9CPartitioned.E2.80.9D_Staging_Committer)
+* Allows for an optional preflight validation check (verify no duplicate files created by different tasks).
+* Manifests can be viewed, size of output determined, etc., during development/debugging.
+
+## Disadvantages of the new protocol compared to the v1 algorithm
+
+* Needs a new manifest file format.
+* Manifests may get large if tasks create many files and/or subdirectories, or if
+  etags are collected and the length of these tags is significant.
+  The HTTP protocol limits each etag to 8 KiB, so the cost may be 8 KiB per file.
+* Makes task commit more complex than the v1 algorithm.
+* Possibly suboptimal on jobs where individual tasks create unique output directories,
+  as directory rename will never be used to commit a directory.
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java
new file mode 100644
index 0000000000000..fccc01ad7e602
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java
@@ -0,0 +1,54 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.lib.input.BaseLineRecordReaderHelper;
+
+public final class LineRecordReaderHelper extends
+    BaseLineRecordReaderHelper {
+
+  public LineRecordReaderHelper(Path filePath, Configuration conf) {
+    super(filePath, conf);
+  }
+
+  @Override
+  public long countRecords(long start, long length) throws IOException {
+    try (LineRecordReader reader = newReader(start, length)) {
+      LongWritable key = new LongWritable();
+      Text value = new Text();
+
+      long numRecords = 0L;
+      while (reader.next(key, value)) {
+        numRecords++;
+      }
+      return numRecords;
+    }
+  }
+
+  private LineRecordReader newReader(long start, long length)
+      throws IOException {
+    FileSplit split = new FileSplit(getFilePath(), start, length, (String[]) null);
+    return new LineRecordReader(getConf(), split, getRecordDelimiterBytes());
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
index dabce770e820d..2df5cc2dc0690 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -56,7 +56,7 @@ public void testLRCPolicy() throws Exception {
     r.setSeed(seed);
     System.out.println("seed: " + seed);
     fs.delete(p, true);
-    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1);
     final int partsPerMap = 1000;
     final int bytesPerFile = partsPerMap * 24;
     IndexCache cache = new IndexCache(conf);
@@ -127,7 +127,7 @@ public void testLRCPolicy() throws Exception {
   public void testBadIndex() throws Exception {
     final int parts = 30;
     fs.delete(p, true);
-    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1);
     IndexCache cache = new IndexCache(conf);
 
     Path f = new Path(p, "badindex");
@@ -159,7 +159,7 @@ public void testBadIndex() throws Exception {
   @Test
   public void testInvalidReduceNumberOrLength() throws Exception {
     fs.delete(p, true);
-    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1);
     final int partsPerMap = 1000;
     final int bytesPerFile = partsPerMap * 24;
     IndexCache cache = new IndexCache(conf);
@@ -205,7 +205,7 @@ public void testRemoveMap() throws Exception {
     // fails with probability of 100% on code before MAPREDUCE-2541,
     // so it is repeatable in practice.
     fs.delete(p, true);
-    conf.setInt(TTConfig.TT_INDEX_CACHE, 10);
+    conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 10);
     // Make a big file so removeMapThread almost surely runs faster than 
     // getInfoThread 
     final int partsPerMap = 100000;
@@ -251,7 +251,7 @@ public void run() {
   @Test
   public void testCreateRace() throws Exception {
     fs.delete(p, true);
-    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    conf.setInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 1);
     final int partsPerMap = 1000;
     final int bytesPerFile = partsPerMap * 24;
     final IndexCache cache = new IndexCache(conf);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
index e3d1241cdc679..46b581d2d0e22 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java
@@ -139,7 +139,7 @@ public void testLocalJobRunnerUriSubstitution() throws InterruptedException {
         baseUrl + "jobend?jobid=$jobId&status=$jobStatus");
     JobEndNotifier.localRunnerNotification(jobConf, jobStatus);
 
-    // No need to wait for the notification to go thru since calls are
+    // No need to wait for the notification to go through since calls are
     // synchronous
 
     // Validate params
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java
new file mode 100644
index 0000000000000..1588e629c8c1e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java
@@ -0,0 +1,31 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.input.BaseLineRecordReaderHelper;
+import org.apache.hadoop.mapreduce.lib.input.BaseTestLineRecordReaderBZip2;
+
+public final class TestLineRecordReaderBZip2 extends
+    BaseTestLineRecordReaderBZip2 {
+
+  @Override
+  protected BaseLineRecordReaderHelper newReader(Path file) {
+    return new LineRecordReaderHelper(file, getConf());
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java
new file mode 100644
index 0000000000000..d1e21e6e258be
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java
@@ -0,0 +1,58 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapreduce.lib.input;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+public abstract class BaseLineRecordReaderHelper {
+
+  private final Configuration conf;
+  private final Path filePath;
+  private final byte[] recordDelimiterBytes;
+
+
+
+  public BaseLineRecordReaderHelper(Path filePath, Configuration conf) {
+    this.conf = conf;
+    this.filePath = filePath;
+
+    conf.setInt(LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+
+    String delimiter = conf.get("textinputformat.record.delimiter");
+    this.recordDelimiterBytes =
+        null != delimiter ? delimiter.getBytes(StandardCharsets.UTF_8) : null;
+  }
+
+  public abstract long countRecords(long start, long length) throws IOException;
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  public Path getFilePath() {
+    return filePath;
+  }
+
+  public byte[] getRecordDelimiterBytes() {
+    return recordDelimiterBytes;
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java
new file mode 100644
index 0000000000000..1e06f12656930
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java
@@ -0,0 +1,489 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapreduce.lib.input;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.StringJoiner;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter;
+import org.apache.hadoop.io.compress.bzip2.BZip2Utils;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter.BLOCK_SIZE;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.junit.Assert.assertEquals;
+
+public abstract class BaseTestLineRecordReaderBZip2 {
+
+  // LF stands for line feed
+  private static final byte[] LF = new byte[] {'\n'};
+  // CR stands for cartridge return
+  private static final byte[] CR = new byte[] {'\r'};
+  private static final byte[] CR_LF = new byte[] {'\r', '\n'};
+
+  private Configuration conf;
+  private FileSystem fs;
+  private Path tempFile;
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+  public FileSystem getFs() {
+    return fs;
+  }
+
+  public Path getTempFile() {
+    return tempFile;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+
+    Path workDir = new Path(
+        System.getProperty("test.build.data", "target"),
+        "data/" + getClass().getSimpleName());
+
+    fs = workDir.getFileSystem(conf);
+
+    Path inputDir = new Path(workDir, "input");
+    tempFile = new Path(inputDir, "test.txt.bz2");
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    fs.delete(tempFile, /* recursive */ false);
+  }
+
+  @Test
+  public void firstBlockEndsWithLF() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void firstBlockEndsWithLFSecondBlockStartsWithLF() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, LF);
+      // Write 254 empty rows terminating at LF, as those records will get
+      // rolled into the first block record due to run-length encoding, the
+      // 255th LF character will trigger a run to be written to the block. We
+      // only need 254 LF characters since the last byte written by prior
+      // writeManyRecords call is already a LF.
+      writer.writeManyRecords(254, 254, LF);
+
+      // This LF character should be the first byte of the second block, but
+      // if splitting at blocks, the first split will read this record as the
+      // additional record.
+      writer.writeRecord(1, LF);
+
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1255, 2});
+  }
+
+  @Test
+  public void firstBlockEndsWithLFSecondBlockStartsWithCR() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, LF);
+      writer.writeRecord(1, CR);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void firstBlockEndsWithCRLF() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, CR_LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void lastRecordContentSpanAcrossBlocks()
+      throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF);
+      writer.writeRecord(100, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void lastRecordOfBlockHasItsLFInNextBlock() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF);
+      // The LF character is the first byte of the second block
+      writer.writeRecord(51, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void lastRecordOfFirstBlockHasItsCRLFInSecondBlock() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF);
+      // Both CR + LF characters are the first two bytes of second block
+      writer.writeRecord(52, CR_LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void lastRecordOfFirstBlockHasItsCRLFPartlyInSecondBlock()
+      throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF);
+      // The CR character is the last byte of the first block and the LF is
+      // the firs byte of the second block
+      writer.writeRecord(51, CR_LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void lastByteInFirstBlockIsCRFirstByteInSecondBlockIsNotLF()
+      throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, CR);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+      writer.writeRecord(10, LF);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void usingCRDelimiterWithSmallestBufferSize() throws Exception {
+    // Forces calling LineReader#fillBuffer for ever byte read
+    conf.set(IO_FILE_BUFFER_SIZE_KEY, "1");
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, CR);
+      writer.writeRecord(100, CR);
+      writer.writeRecord(10, CR);
+      writer.writeRecord(10, CR);
+      writer.writeRecord(10, CR);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void delimitedByCRSpanningThreeBlocks() throws Exception {
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeRecord(3 * BLOCK_SIZE, CR);
+      writer.writeRecord(3 * BLOCK_SIZE, CR);
+      writer.writeRecord(3 * BLOCK_SIZE, CR);
+    }
+    assertRecordCountsPerSplit(tempFile,
+        new long[] {1, 0, 1, 0, 0, 1, 0, 0, 0});
+  }
+
+  @Test
+  public void customDelimiterLastThreeBytesInBlockAreDelimiter()
+      throws Exception {
+    byte[] delimiter = new byte[] {'e', 'n', 'd'};
+    setDelimiter(delimiter);
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE, 1000, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void customDelimiterDelimiterSpansAcrossBlocks()
+      throws Exception {
+    byte[] delimiter = new byte[] {'e', 'n', 'd'};
+    setDelimiter(delimiter);
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, delimiter);
+      writer.writeRecord(52, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1001, 2});
+  }
+
+  @Test
+  public void customDelimiterLastRecordDelimiterStartsAtNextBlockStart()
+      throws Exception {
+    byte[] delimiter = new byte[] {'e', 'n', 'd'};
+    setDelimiter(delimiter);
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 50, 999, delimiter);
+      writer.writeRecord(53, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  @Test
+  public void customDelimiterLastBlockBytesShareCommonPrefixWithDelimiter()
+      throws Exception {
+    byte[] delimiter = new byte[] {'e', 'n', 'd'};
+    setDelimiter(delimiter);
+
+    try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) {
+      writer.writeManyRecords(BLOCK_SIZE - 4, 999, delimiter);
+      // The first 4 bytes, "an e", will be the last 4 bytes of the first block,
+      // the last byte being 'e' which matches the first character of the
+      // delimiter "end". The first byte of the next block also matches the
+      // second byte of the delimiter "n"; however the next character "c" does
+      // not match the last character of the delimiter. Thus an additional
+      // record should not be read for the split that reads the first block.
+      // The split that reads the second block will just discard
+      // "nchanting tale coming to an end".
+      writer.write("an enchanting tale coming to an end");
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+      writer.writeRecord(10, delimiter);
+    }
+    assertRecordCountsPerSplit(tempFile, new long[] {1000, 3});
+  }
+
+  protected abstract BaseLineRecordReaderHelper newReader(Path file);
+
+  private void assertRecordCountsPerSplit(
+      Path path, long[] countsIfSplitAtBlocks) throws IOException {
+    RecordCountAssert countAssert =
+        new RecordCountAssert(path, countsIfSplitAtBlocks);
+    countAssert.assertSingleSplit();
+    countAssert.assertSplittingAtBlocks();
+    countAssert.assertSplittingJustAfterSecondBlockStarts();
+    countAssert.assertSplittingEachBlockRangeInThreeParts();
+    countAssert.assertSplitsAroundBlockStartOffsets();
+  }
+
+  private class RecordCountAssert {
+
+    private final BaseLineRecordReaderHelper reader;
+    private final long numBlocks;
+    private final long[] countsIfSplitAtBlocks;
+    private final long fileSize;
+    private final long totalRecords;
+    private final List<Long> nextBlockOffsets;
+
+    RecordCountAssert(
+        Path path, long[] countsIfSplitAtBlocks) throws IOException {
+      this.reader = newReader(path);
+      this.countsIfSplitAtBlocks = countsIfSplitAtBlocks;
+      this.fileSize = getFileSize(path);
+      this.totalRecords = Arrays.stream(countsIfSplitAtBlocks).sum();
+      this.numBlocks = countsIfSplitAtBlocks.length;
+      this.nextBlockOffsets = BZip2Utils.getNextBlockMarkerOffsets(path, conf);
+
+      assertEquals(numBlocks, nextBlockOffsets.size() + 1);
+    }
+
+    private void assertSingleSplit() throws IOException {
+      assertEquals(totalRecords, reader.countRecords(0, fileSize));
+    }
+
+    private void assertSplittingAtBlocks() throws IOException {
+      assertSplits(getSplitsAtBlocks());
+    }
+
+    private void assertSplittingJustAfterSecondBlockStarts()
+        throws IOException {
+      if (numBlocks <= 1) {
+        return;
+      }
+      long recordsInFirstTwoBlocks =
+          countsIfSplitAtBlocks[0] + countsIfSplitAtBlocks[1];
+      long remainingRecords = totalRecords - recordsInFirstTwoBlocks;
+
+      long firstSplitSize = nextBlockOffsets.get(0) + 1;
+      assertEquals(
+          recordsInFirstTwoBlocks,
+          reader.countRecords(0, firstSplitSize));
+      assertEquals(
+          remainingRecords,
+          reader.countRecords(firstSplitSize, fileSize - firstSplitSize));
+    }
+
+    private void assertSplittingEachBlockRangeInThreeParts()
+        throws IOException {
+      for (SplitRange splitRange : getSplitsAtBlocks()) {
+        long[] expectedNumRecordsPerPart = new long[] {
+            splitRange.expectedNumRecords, 0, 0
+        };
+        List<SplitRange> parts = splitRange.divide(expectedNumRecordsPerPart);
+        assertSplits(parts);
+      }
+    }
+
+    private void assertSplitsAroundBlockStartOffsets()
+        throws IOException {
+      for (SplitRange split : getSplitsAtBlocks()) {
+        assertSplit(split.withLength(1));
+        if (split.start > 0) {
+          assertSplit(split.moveBy(-2).withLength(3));
+          assertSplit(split.moveBy(-2).withLength(2).withExpectedNumRecords(0));
+          assertSplit(split.moveBy(-1).withLength(2));
+          assertSplit(split.moveBy(-1).withLength(1).withExpectedNumRecords(0));
+        }
+        assertSplit(split.moveBy(1).withLength(1).withExpectedNumRecords(0));
+        assertSplit(split.moveBy(2).withLength(1).withExpectedNumRecords(0));
+      }
+    }
+
+    private List<SplitRange> getSplitsAtBlocks() {
+      List<SplitRange> splits = new ArrayList<>();
+      for (int i = 0; i < numBlocks; i++) {
+        String name = "Block" + i;
+        long start = i == 0 ? 0 : nextBlockOffsets.get(i - 1);
+        long end = i == numBlocks - 1 ? fileSize : nextBlockOffsets.get(i);
+        long length = end - start;
+        long expectedNumRecords = countsIfSplitAtBlocks[i];
+        splits.add(new SplitRange(name, start, length, expectedNumRecords));
+      }
+      return splits;
+    }
+
+    private void assertSplits(Iterable<SplitRange> splitRanges)
+        throws IOException {
+      for (SplitRange splitRange : splitRanges) {
+        assertSplit(splitRange);
+      }
+    }
+
+    private void assertSplit(SplitRange splitRange) throws IOException {
+      String message = splitRange.toString();
+      long actual = reader.countRecords(splitRange.start, splitRange.length);
+      assertEquals(message, splitRange.expectedNumRecords, actual);
+    }
+  }
+
+  private static class SplitRange {
+    final private String name;
+    final private long start;
+    final private long length;
+    final private long expectedNumRecords;
+
+    SplitRange(
+        String name,
+        long start,
+        long length,
+        long expectedNumRecords) {
+      this.name = name;
+      this.start = start;
+      this.length = length;
+      this.expectedNumRecords = expectedNumRecords;
+    }
+
+    @Override
+    public String toString() {
+      return new StringJoiner(", ", SplitRange.class.getSimpleName() + "[", "]")
+          .add("name='" + name + "'")
+          .add("start=" + start)
+          .add("length=" + length)
+          .add("expectedNumRecords=" + expectedNumRecords)
+          .toString();
+    }
+
+    List<SplitRange> divide(long[] expectedNumRecordsPerPart) {
+      int numParts = expectedNumRecordsPerPart.length;
+      checkArgument(numParts > 0);
+
+      long minPartSize = length / numParts;
+      checkArgument(minPartSize > 0);
+      long lastPartExtraSize = length % numParts;
+
+      List<SplitRange> partRanges = new ArrayList<>();
+      long partStart = start;
+      for (int i = 0; i < numParts; i++) {
+        String partName = name + "_Part" + i;
+
+        long extraSize = i == numParts - 1 ? lastPartExtraSize : 0;
+        long partSize = minPartSize + extraSize;
+
+        long partExpectedNumRecords = expectedNumRecordsPerPart[i];
+
+        partRanges.add(new SplitRange(
+            partName, partStart, partSize, partExpectedNumRecords));
+        partStart += partSize;
+      }
+      return partRanges;
+    }
+
+    SplitRange withLength(long newLength) {
+      return new SplitRange(name, start, newLength, expectedNumRecords);
+    }
+
+    SplitRange withExpectedNumRecords(long newExpectedNumRecords) {
+      return new SplitRange(name, start, length, newExpectedNumRecords);
+    }
+
+    SplitRange moveBy(long delta) {
+      return new SplitRange(name, start + delta, length, expectedNumRecords);
+    }
+  }
+
+  private long getFileSize(Path path) throws IOException {
+    return fs.getFileStatus(path).getLen();
+  }
+
+  private void setDelimiter(byte[] delimiter) {
+    conf.set("textinputformat.record.delimiter",
+        new String(delimiter, StandardCharsets.UTF_8));
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java
new file mode 100644
index 0000000000000..56fc7eb4e8662
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java
@@ -0,0 +1,62 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapreduce.lib.input;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+public final class LineRecordReaderHelper extends
+    BaseLineRecordReaderHelper {
+
+  public LineRecordReaderHelper(Path filePath, Configuration conf) {
+    super(filePath, conf);
+  }
+
+  @Override
+  public long countRecords(long start, long length) throws IOException {
+    try (LineRecordReader reader = newReader(start, length)) {
+      long numRecords = 0L;
+      while (reader.nextKeyValue()) {
+        numRecords++;
+      }
+      return numRecords;
+    }
+  }
+
+  private LineRecordReader newReader(long start, long length)
+      throws IOException {
+    FileSplit split = new FileSplit(getFilePath(), start, length, null);
+
+    TaskAttemptContext context =
+        new TaskAttemptContextImpl(getConf(), new TaskAttemptID());
+
+    LineRecordReader reader = new LineRecordReader(getRecordDelimiterBytes());
+    try {
+      reader.initialize(split, context);
+      return reader;
+    } catch (Throwable e) {
+      reader.close();
+      throw e;
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java
new file mode 100644
index 0000000000000..ec01ef332fa90
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java
@@ -0,0 +1,29 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.mapreduce.lib.input;
+
+import org.apache.hadoop.fs.Path;
+
+public final class TestLineRecordReaderBZip2
+    extends BaseTestLineRecordReaderBZip2 {
+
+  @Override
+  protected BaseLineRecordReaderHelper newReader(Path file) {
+    return new LineRecordReaderHelper(file, getConf());
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestJobControl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestJobControl.java
index 41a4eaf64a759..48006f12282c3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestJobControl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestJobControl.java
@@ -20,7 +20,6 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.junit.Test;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/AbstractManifestCommitterTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/AbstractManifestCommitterTest.java
new file mode 100644
index 0000000000000..bcd9d34e263de
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/AbstractManifestCommitterTest.java
@@ -0,0 +1,1085 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.assertj.core.api.Assertions;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.localfs.LocalFSContract;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SaveTaskManifestStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.readDataset;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.SUCCESSFUL_JOB_OUTPUT_DIR_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.COMMITTER_FACTORY_CLASS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConfig.createCloseableTaskSubmitter;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ID_SOURCE_MAPREDUCE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_FACTORY;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_DIAGNOSTICS_MANIFEST_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_SUMMARY_REPORT_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_VALIDATE_OUTPUT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.getProjectBuildDir;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.validateSuccessFile;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.NAME_FORMAT_JOB_ATTEMPT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createIOStatisticsStore;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createTaskManifest;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
+/**
+ * Tests which work with manifest committers.
+ * This is a filesystem contract bound to the local filesystem;
+ * subclasses may change the FS to test against other stores.
+ * Some fields are set up in
+ * in {@link #executeOneTaskAttempt(int, int, int)},
+ * which is why fields are used.
+ * when synchronized access is needed; synchronize on (this) rather
+ * than individual fields
+ */
+public abstract class AbstractManifestCommitterTest
+    extends AbstractFSContractTestBase {
+
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(AbstractManifestCommitterTest.class);
+
+  /**
+   * Some Job and task IDs.
+   */
+  protected static final ManifestCommitterTestSupport.JobAndTaskIDsForTests
+      TASK_IDS = new ManifestCommitterTestSupport.JobAndTaskIDsForTests(2, 2);
+
+  public static final int JOB1 = 1;
+
+  public static final int TASK0 = 0;
+
+  public static final int TASK1 = 1;
+
+  /**
+   * Task attempt 0 index.
+   */
+  public static final int TA0 = 0;
+
+  /**
+   * Task attempt 1 index.
+   */
+  public static final int TA1 = 1;
+
+  /**
+   * Depth of dir tree to generate.
+   */
+  public static final int DEPTH = 3;
+
+  /**
+   * Width of dir tree at every level.
+   */
+  public static final int WIDTH = 2;
+
+  /**
+   * How many files to create in the leaf directories.
+   */
+  public static final int FILES_PER_DIRECTORY = 4;
+
+  /**
+   * Pool size.
+   */
+  public static final int POOL_SIZE = 32;
+
+  /**
+   * FileSystem statistics are collected across every test case.
+   */
+  protected static final IOStatisticsSnapshot FILESYSTEM_IOSTATS =
+      snapshotIOStatistics();
+
+  /**
+   * Counter for creating files. Ensures that across all test suites,
+   * duplicate filenames are never created. Helps assign blame.
+   */
+  private static final AtomicLong CREATE_FILE_COUNTER = new AtomicLong();
+
+  protected static final byte[] NO_DATA = new byte[0];
+
+  /**
+   * The thread leak tracker.
+   */
+  private static final ThreadLeakTracker THREAD_LEAK_TRACKER = new ThreadLeakTracker();
+
+  private static final int MAX_LEN = 64_000;
+
+  /**
+   * Submitter for tasks; may be null.
+   */
+  private CloseableTaskPoolSubmitter submitter;
+
+  /**
+   * Stage statistics. Created in test setup, and in
+   * teardown updates {@link #FILESYSTEM_IOSTATS}.
+   */
+  private IOStatisticsStore stageStatistics;
+
+  /**
+   * Prefer to use these to interact with the FS to
+   * ensure more implicit coverage.
+   */
+  private ManifestStoreOperations storeOperations;
+
+  /**
+   * Progress counter used in all stage configs.
+   */
+  private final ProgressCounter progressCounter = new ProgressCounter();
+
+  /**
+   * Directory for job summary reports.
+   * This should be set up in test suites testing against real object stores.
+   */
+  private File reportDir;
+
+  /**
+   * List of task attempt IDs for those tests which create manifests.
+   */
+  private final List<String> taskAttemptIds = new ArrayList<>();
+
+  /**
+   * List of task IDs for those tests which create manifests.
+   */
+  private final List<String> taskIds = new ArrayList<>();
+
+  /**
+   * any job stage configuration created for operations.
+   */
+  private StageConfig jobStageConfig;
+
+  /**
+   * Destination dir of job.
+   */
+  private Path destDir;
+
+  /**
+   * When creating manifests, total data size.
+   */
+  private final AtomicLong totalDataSize = new AtomicLong();
+
+  /**
+   * Where to move manifests; must be in target FS.
+   */
+  private Path manifestDir;
+
+  /**
+   * Get the contract configuration.
+   * @return the config used to create the FS.
+   */
+  protected Configuration getConfiguration() {
+    return getContract().getConf();
+  }
+
+  /**
+   * Store operations to interact with..
+   * @return store operations.
+   */
+  protected ManifestStoreOperations getStoreOperations() {
+    return storeOperations;
+  }
+
+  /**
+   * Set store operations.
+   * @param storeOperations new value
+   */
+  protected void setStoreOperations(final ManifestStoreOperations storeOperations) {
+    this.storeOperations = storeOperations;
+  }
+
+  public List<String> getTaskAttemptIds() {
+    return taskAttemptIds;
+  }
+
+  public List<String> getTaskIds() {
+    return taskIds;
+  }
+
+  public long getTotalDataSize() {
+    return totalDataSize.get();
+  }
+
+  public Path getManifestDir() {
+    return manifestDir;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public AbstractManifestCommitterTest withManifestDir(Path value) {
+    manifestDir = value;
+    return this;
+  }
+
+  /**
+   * Describe a test in the logs.
+   * @param text text to print
+   * @param args arguments to format in the printing
+   */
+  protected void describe(String text, Object... args) {
+    LOG.info("\n\n{}: {}\n",
+        getMethodName(),
+        String.format(text, args));
+  }
+
+  /**
+   * Local FS unless overridden.
+   * @param conf configuration
+   * @return the FS contract.
+   */
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new LocalFSContract(conf);
+  }
+
+  /** Enable the manifest committer options in the configuration. */
+  @Override
+  protected Configuration createConfiguration() {
+    return enableManifestCommitter(super.createConfiguration());
+  }
+
+  @Override
+  public void setup() throws Exception {
+
+    // set the manifest committer to a localfs path for reports across
+    // all threads.
+    // do this before superclass setup so reportDir is non-null there
+    // and can be used in creating the configuration.
+    reportDir = new File(getProjectBuildDir(), "reports");
+    reportDir.mkdirs();
+
+    // superclass setup includes creating a filesystem instance
+    // for the target store.
+    super.setup();
+
+    manifestDir = path("manifests");
+
+    // destination directory defaults to method path in
+    // target FS
+    setDestDir(methodPath());
+
+    // stage statistics
+    setStageStatistics(createIOStatisticsStore().build());
+    // thread pool for task submission.
+    setSubmitter(createCloseableTaskSubmitter(POOL_SIZE, TASK_IDS.getJobId()));
+    // store operations for the target filesystem.
+    storeOperations = createManifestStoreOperations();
+  }
+
+  /**
+   * Overrride point: create the store operations.
+   * @return store operations for this suite.
+   */
+  protected ManifestStoreOperations createManifestStoreOperations() throws IOException {
+    final FileSystem fs = getFileSystem();
+    return ManifestCommitterSupport.createManifestStoreOperations(fs.getConf(), fs, getTestPath());
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    Thread.currentThread().setName("teardown");
+
+    IOUtils.cleanupWithLogger(LOG, storeOperations, getSubmitter());
+    storeOperations = null;
+    super.teardown();
+    FILESYSTEM_IOSTATS.aggregate(retrieveIOStatistics(getFileSystem()));
+    FILESYSTEM_IOSTATS.aggregate(getStageStatistics());
+  }
+
+  /**
+   * Add a long delay so that you don't get timeouts when working
+   * with object stores or debugging.
+   * @return a longer timeout than the base classes.
+   */
+  @Override
+  protected int getTestTimeoutMillis() {
+    return 600_000;
+  }
+
+  protected Path getTestPath() {
+    return getContract().getTestPath();
+  }
+
+  /**
+   * Get the task submitter.
+   * @return submitter or null
+   */
+  protected CloseableTaskPoolSubmitter getSubmitter() {
+    return submitter;
+  }
+
+  /**
+   * Set the task submitter.
+   * @param submitter new value.
+   */
+  protected void setSubmitter(CloseableTaskPoolSubmitter submitter) {
+    this.submitter = submitter;
+  }
+
+  /**
+   * Get the executor which the submitter also uses.
+   * @return an executor.
+   */
+  protected ExecutorService getExecutorService() {
+    return getSubmitter().getPool();
+  }
+  /**
+   * @return IOStatistics for stage.
+   */
+  protected final IOStatisticsStore getStageStatistics() {
+    return stageStatistics;
+  }
+
+  /**
+   * Set the statistics.
+   * @param stageStatistics statistics.
+   */
+  protected final void setStageStatistics(IOStatisticsStore stageStatistics) {
+    this.stageStatistics = stageStatistics;
+  }
+
+  /**
+   * Get the progress counter invoked during commit operations.
+   * @return progress counter.
+   */
+  protected final ProgressCounter getProgressCounter() {
+    return progressCounter;
+  }
+
+  /**
+   * Get the report directory.
+   * @return report directory.
+   */
+  public final File getReportDir() {
+    return reportDir;
+  }
+
+  /**
+   * Get the report directory as a URI.
+   * @return report directory.
+   */
+  public final URI getReportDirUri() {
+    return getReportDir().toURI();
+  }
+
+  /**
+   * Get the (shared) thread leak tracker.
+   * @return the thread leak tracker.
+   */
+  protected static ThreadLeakTracker getThreadLeakTracker() {
+    return THREAD_LEAK_TRACKER;
+  }
+
+  /**
+   * Make sure there's no thread leakage.
+   */
+  @AfterClass
+  public static void threadLeakage() {
+    THREAD_LEAK_TRACKER.assertNoThreadLeakage();
+  }
+
+  /**
+   * Dump the filesystem statistics after the class.
+   */
+  @AfterClass
+  public static void dumpFileSystemIOStatistics() {
+    LOG.info("Aggregate FileSystem Statistics {}",
+        ioStatisticsToPrettyString(FILESYSTEM_IOSTATS));
+  }
+
+  /**
+   * Create a directory tree through an executor.
+   * dirs created = width^depth;
+   * file count = width^depth * files
+   * If createDirs == true, then directories are created at the bottom,
+   * not files.
+   * @param base base dir
+   * @param prefix prefix for filenames.
+   * @param executor submitter.
+   * @param depth depth of dirs
+   * @param width width of dirs
+   * @param files files to add in each base dir.
+   * @param createDirs create directories rather than files?
+   * @return the list of paths
+   * @throws IOException failure.
+   */
+  public final List<Path> createFilesOrDirs(Path base,
+      String prefix,
+      ExecutorService executor,
+      int depth,
+      int width,
+      int files,
+      boolean createDirs) throws IOException {
+
+    try (DurationInfo ignored = new DurationInfo(LOG, true,
+        "Creating Files %s (%d, %d, %d) under %s",
+        prefix, depth, width, files, base)) {
+
+      assertPathExists("Task attempt dir", base);
+
+      // create the files in the thread pool.
+      List<Future<Path>> futures = createFilesOrDirs(
+          new ArrayList<>(),
+          base, prefix,
+          executor,
+          depth, width, files,
+          createDirs);
+      List<Path> result = new ArrayList<>();
+
+      // now wait for the creations to finish.
+      for (Future<Path> f : futures) {
+        result.add(awaitFuture(f));
+      }
+      return result;
+    }
+  }
+
+  /**
+   * Counter incremented for each file created.
+   */
+  private final AtomicLong fileDataGenerator = new AtomicLong();
+
+  /**
+   * Create files or directories; done in a treewalk and building up
+   * a list of futures to wait for. The list is
+   * build up incrementally rather than through some merging of
+   * lists created down the tree.
+   * If createDirs == true, then directories are created at the bottom,
+   * not files.
+   *
+   * @param futures list of futures to build up.
+   * @param base base dir
+   * @param prefix prefix for filenames.
+   * @param executor submitter.
+   * @param depth depth of dirs
+   * @param width width of dirs
+   * @param files files to add in each base dir.
+   * @param createDirs create directories rather than files?
+   * @return the list of futures
+   */
+  private List<Future<Path>> createFilesOrDirs(
+      List<Future<Path>> futures,
+      Path base,
+      String prefix,
+      ExecutorService executor,
+      int depth,
+      int width,
+      int files,
+      boolean createDirs) {
+
+    if (depth > 0) {
+      // still creating directories
+      for (int i = 0; i < width; i++) {
+        Path child = new Path(base,
+            String.format("dir-%02d-%02d", depth, i));
+        createFilesOrDirs(futures, child, prefix, executor, depth - 1, width, files, false);
+      }
+    } else {
+      // time to create files
+      for (int i = 0; i < files; i++) {
+        Path file = new Path(base,
+            String.format("%s-%04d", prefix,
+                CREATE_FILE_COUNTER.incrementAndGet()));
+        // buld the data. Not actually used in mkdir.
+        long entry = fileDataGenerator.incrementAndGet() & 0xffff;
+        byte[] data = new byte[2];
+        data[0] = (byte) (entry & 0xff);
+        data[1] = (byte) ((entry & 0xff00) >> 8);
+        // the async operation.
+        Future<Path> f = executor.submit(() -> {
+          if (!createDirs) {
+            // create files
+            ContractTestUtils.createFile(getFileSystem(), file, true, data);
+          } else {
+            // create directories
+            mkdirs(file);
+          }
+          return file;
+        });
+        futures.add(f);
+      }
+    }
+    return futures;
+  }
+
+  /**
+   * Create a list of paths under a dir.
+   * @param base base dir
+   * @param count count
+   * @return the list
+   */
+  protected List<Path> subpaths(Path base, int count) {
+    return IntStream.rangeClosed(1, count)
+        .mapToObj(i -> new Path(base, String.format("entry-%02d", i)))
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Submit a mkdir call to the executor pool.
+   * @param path path of dir to create.
+   * @return future
+   */
+  protected CompletableFuture<Path> asyncMkdir(final Path path) {
+    CompletableFuture<Path> f = new CompletableFuture<>();
+    getExecutorService().submit(() -> {
+      try {
+        mkdirs(path);
+        f.complete(path);
+      } catch (IOException e) {
+        f.completeExceptionally(e);
+      }
+    });
+    return f;
+  }
+
+  /**
+   * Given a list of paths, create the dirs async.
+   * @param paths path list
+   * @throws IOException failure
+   */
+  protected void asyncMkdirs(Collection<Path> paths) throws IOException {
+    List<CompletableFuture<Path>> futures = new ArrayList<>();
+    // initiate
+    for (Path path: paths) {
+      futures.add(asyncMkdir(path));
+    }
+    // await
+    for (Future<Path> f : futures) {
+      awaitFuture(f);
+    }
+  }
+
+  /**
+   * Submit an oepration to create a file to the executor pool.
+   * @param path path of file to create.
+   * @return future
+   */
+  protected CompletableFuture<Path> asyncPut(final Path path, byte[] data) {
+    CompletableFuture<Path> f = new CompletableFuture<>();
+    getExecutorService().submit(() -> {
+      try {
+        ContractTestUtils.createFile(getFileSystem(), path, true, data);
+        f.complete(path);
+      } catch (IOException e) {
+        f.completeExceptionally(e);
+      }
+    });
+    return f;
+  }
+
+  /**
+   * Convert the manifest list to a map by task attempt ID.
+   * @param list manifests
+   * @return a map, indexed by task attempt ID.
+   */
+  protected Map<String, TaskManifest> toMap(List<TaskManifest> list) {
+    return list.stream()
+        .collect(Collectors.toMap(TaskManifest::getTaskAttemptID, x -> x));
+  }
+
+  /**
+   * Verify the manifest files match the list of paths.
+   * @param manifest manifest to audit
+   * @param files list of files.
+   */
+  protected void verifyManifestFilesMatch(final TaskManifest manifest,
+      final List<Path> files) {
+    // get the list of source paths
+    Set<Path> filesToRename = manifest.getFilesToCommit()
+        .stream()
+        .map(FileEntry::getSourcePath)
+        .collect(Collectors.toSet());
+    // which must match that of all the files created
+    Assertions.assertThat(filesToRename)
+        .containsExactlyInAnyOrderElementsOf(files);
+  }
+
+  /**
+   * Verify that a task manifest has a given attempt ID.
+   * @param manifest manifest, may be null.
+   * @param attemptId expected attempt ID
+   * @return the manifest, guaranteed to be non-null and of task attempt.
+   */
+  protected TaskManifest verifyManifestTaskAttemptID(
+      final TaskManifest manifest,
+      final String attemptId) {
+    Assertions.assertThat(manifest)
+        .describedAs("Manifest of task %s", attemptId)
+        .isNotNull();
+    Assertions.assertThat(manifest.getTaskAttemptID())
+        .describedAs("Task Attempt ID of manifest %s", manifest)
+        .isEqualTo(attemptId);
+    return manifest;
+  }
+
+  /**
+   * Assert that a path must exist; return the path.
+   * @param message text for error message.
+   * @param path path to validate.
+   * @return the path
+   * @throws IOException IO Failure
+   */
+  Path pathMustExist(final String message,
+      final Path path) throws IOException {
+    assertPathExists(message, path);
+    return path;
+  }
+
+  /**
+   * Assert that a path must exist; return the path.
+   * It must also equal the expected value.
+   * @param message text for error message.
+   * @param expectedPath expected path.
+   * @param actualPath path to validate.
+   * @return the path
+   * @throws IOException IO Failure
+   */
+  Path verifyPath(final String message,
+      final Path expectedPath,
+      final Path actualPath) throws IOException {
+    Assertions.assertThat(actualPath)
+        .describedAs(message)
+        .isEqualTo(expectedPath);
+    return pathMustExist(message, actualPath);
+  }
+
+  /**
+   * Verify that the specified dir has the {@code _SUCCESS} marker
+   * and that it can be loaded.
+   * The contents will be logged and returned.
+   * @param dir directory to scan
+   * @param jobId job ID, only verified if non-empty
+   * @return the loaded success data
+   * @throws IOException IO Failure
+   */
+  protected ManifestSuccessData verifySuccessMarker(Path dir, String jobId)
+      throws IOException {
+    return validateSuccessFile(getFileSystem(), dir, 0, jobId);
+  }
+
+  /**
+   * Read a UTF-8 file.
+   * @param path path to read
+   * @return string value
+   * @throws IOException IO failure
+   */
+  protected String readFile(Path path) throws IOException {
+    return ContractTestUtils.readUTF8(getFileSystem(), path, -1);
+  }
+
+  /**
+   * Modify a (job) config to switch to the manifest committer;
+   * output validation is also enabled.
+   * @param conf config to patch.
+   * @return the updated configuration.
+   */
+  protected Configuration enableManifestCommitter(final Configuration conf) {
+    conf.set(COMMITTER_FACTORY_CLASS, MANIFEST_COMMITTER_FACTORY);
+    // always create a job marker
+    conf.setBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true);
+    // and validate the output, for extra rigorousness
+    conf.setBoolean(OPT_VALIDATE_OUTPUT, true);
+
+    // set the manifest rename dir if non-null
+    if (getManifestDir() != null) {
+      conf.set(OPT_DIAGNOSTICS_MANIFEST_DIR,
+          getManifestDir().toUri().toString());
+    }
+
+    // and bind the report dir
+    conf.set(OPT_SUMMARY_REPORT_DIR, getReportDirUri().toString());
+    return conf;
+  }
+
+  /**
+   * Create the stage config for a job but don't finalize it.
+   * Uses {@link #TASK_IDS} for job/task ID.
+   * @param jobAttemptNumber job attempt number
+   * @param outputPath path where the final output goes
+   * @return the config
+   */
+  protected StageConfig createStageConfigForJob(
+      final int jobAttemptNumber,
+      final Path outputPath) {
+    return createStageConfig(jobAttemptNumber, -1, 0, outputPath);
+  }
+
+  /**
+   * Create the stage config for job or task but don't finalize it.
+   * Uses {@link #TASK_IDS} for job/task ID.
+   * @param jobAttemptNumber job attempt number
+   * @param taskIndex task attempt index; -1 for job attempt only.
+   * @param taskAttemptNumber task attempt number
+   * @param outputPath path where the final output goes
+   * @return the config
+   */
+  protected StageConfig createStageConfig(
+      final int jobAttemptNumber,
+      final int taskIndex,
+      final int taskAttemptNumber,
+      final Path outputPath) {
+    final String jobId = TASK_IDS.getJobId();
+    ManifestCommitterSupport.AttemptDirectories attemptDirs =
+        new ManifestCommitterSupport.AttemptDirectories(outputPath,
+            jobId, jobAttemptNumber);
+    StageConfig config = new StageConfig();
+    config
+        .withIOProcessors(getSubmitter())
+        .withIOStatistics(getStageStatistics())
+        .withJobId(jobId)
+        .withJobIdSource(JOB_ID_SOURCE_MAPREDUCE)
+        .withJobAttemptNumber(jobAttemptNumber)
+        .withJobDirectories(attemptDirs)
+        .withName(String.format(NAME_FORMAT_JOB_ATTEMPT, jobId))
+        .withOperations(getStoreOperations())
+        .withProgressable(getProgressCounter());
+
+    // if there's a task attempt ID set, set up its details
+    if (taskIndex >= 0) {
+      String taskAttempt = TASK_IDS.getTaskAttempt(taskIndex,
+          taskAttemptNumber);
+      config
+          .withTaskAttemptId(taskAttempt)
+          .withTaskId(TASK_IDS.getTaskIdType(taskIndex).toString())
+          .withTaskAttemptDir(
+              attemptDirs.getTaskAttemptPath(taskAttempt));
+    }
+    return config;
+  }
+
+  /**
+   * A job stage config.
+   * @return stage config or null.
+   */
+  protected StageConfig getJobStageConfig() {
+    return jobStageConfig;
+  }
+
+  protected void setJobStageConfig(StageConfig jobStageConfig) {
+    this.jobStageConfig = jobStageConfig;
+  }
+
+  protected Path getDestDir() {
+    return destDir;
+  }
+
+  protected void setDestDir(Path destDir) {
+    this.destDir = destDir;
+  }
+
+  /**
+   * Execute a set of tasks; task ID is a simple count.
+   * task attempt is lowest 2 bits of task ID.
+   * @param taskAttemptCount number of tasks.
+   * @param filesPerTaskAttempt number of files to include in manifest.
+   * @return the manifests.
+   * @throws IOException IO failure.
+   */
+  protected List<TaskManifest> executeTaskAttempts(int taskAttemptCount,
+      int filesPerTaskAttempt) throws IOException {
+
+    try (DurationInfo di = new DurationInfo(LOG, true, "create manifests")) {
+
+      // build a list of the task IDs.
+      // it's really hard to create a list of Integers; the java8
+      // IntStream etc doesn't quite fit as they do their best
+      // keep things unboxed, trying to map(Integer::valueOf) doesn't help.
+      List<Integer> taskIdList = new ArrayList<>(taskAttemptCount);
+      for (int t = 0; t < taskAttemptCount; t++) {
+        taskIdList.add(t);
+      }
+
+      /// execute the tasks
+      List<TaskManifest> manifests = Collections.synchronizedList(
+          new ArrayList<>());
+
+      // then submit their creation/save to the pool.
+      TaskPool.foreach(taskIdList)
+          .executeWith(getSubmitter())
+          .stopOnFailure()
+          .run(i -> {
+            manifests.add(
+                executeOneTaskAttempt(i, i & 0x03, filesPerTaskAttempt));
+          });
+      return manifests;
+
+    }
+  }
+
+  /**
+   * Create at task ID and attempt (adding to taskIDs and taskAttemptIds)
+   * setup the task, create a manifest with fake task entries
+   * and save that manifest to the job attempt dir.
+   * No actual files are created.
+   * @param task task index
+   * @param taskAttempt task attempt value
+   * @param filesPerTaskAttempt number of files to include in manifest.
+   * @return the manifest
+   * @throws IOException failure
+   */
+  protected TaskManifest executeOneTaskAttempt(final int task,
+      int taskAttempt, final int filesPerTaskAttempt) throws IOException {
+
+    String tid = String.format("task_%03d", task);
+    String taskAttemptId = String.format("%s_%02d",
+        tid, taskAttempt);
+    synchronized (this) {
+      taskIds.add(tid);
+      taskAttemptIds.add(taskAttemptId);
+    }
+    // for each task, a job config is created then patched with the task info
+    StageConfig taskStageConfig = createTaskStageConfig(JOB1, tid, taskAttemptId);
+
+    LOG.info("Generating manifest for {}", taskAttemptId);
+
+    // task setup: create dest dir.
+    // This helps generate a realistic
+    // workload for the parallelized job cleanup.
+    new SetupTaskStage(taskStageConfig).apply("task " + taskAttemptId);
+
+    final TaskManifest manifest = createTaskManifest(taskStageConfig);
+
+    Path taDir = taskStageConfig.getTaskAttemptDir();
+    long size = task * 1000_0000L;
+
+    // for each task, 10 dirs, one file per dir.
+    for (int i = 0; i < filesPerTaskAttempt; i++) {
+      Path in = new Path(taDir, "dir-" + i);
+      Path out = new Path(getDestDir(), "dir-" + i);
+      manifest.addDirectory(DirEntry.dirEntry(out, 0, 1));
+      String name = taskStageConfig.getTaskAttemptId() + ".csv";
+      Path src = new Path(in, name);
+      Path dest = new Path(out, name);
+      long fileSize = size + i * 1000L;
+      manifest.addFileToCommit(
+          new FileEntry(src, dest, fileSize, Long.toString(fileSize, 16)));
+      totalDataSize.addAndGet(fileSize);
+    }
+
+    // save the manifest for this stage.
+    new SaveTaskManifestStage(taskStageConfig).apply(manifest);
+    return manifest;
+  }
+
+  public StageConfig createTaskStageConfig(final int jobId, final String tid,
+      final String taskAttemptId) {
+    Path jobAttemptTaskSubDir = getJobStageConfig().getJobAttemptTaskSubDir();
+    StageConfig taskStageConfig = createStageConfigForJob(jobId, getDestDir())
+        .withTaskId(tid)
+        .withTaskAttemptId(taskAttemptId)
+        .withTaskAttemptDir(new Path(jobAttemptTaskSubDir, taskAttemptId));
+    return taskStageConfig;
+  }
+
+  /**
+   * Verify that the job directories have been cleaned up.
+   * @throws IOException IO failure
+   */
+  protected void verifyJobDirsCleanedUp() throws IOException {
+    StageConfig stageConfig = getJobStageConfig();
+    assertPathDoesNotExist("Job attempt dir", stageConfig.getJobAttemptDir());
+    assertPathDoesNotExist("dest temp dir", stageConfig.getOutputTempSubDir());
+  }
+
+  /**
+   * List a directory/directory tree and print files.
+   * @param fileSystem FS
+   * @param path path
+   * @param recursive do a recursive listing?
+   * @return the number of files found.
+   * @throws IOException failure.
+   */
+  public static long lsR(FileSystem fileSystem, Path path, boolean recursive)
+      throws Exception {
+    if (path == null) {
+      // surfaces when someone calls getParent() on something at the top
+      // of the path
+      LOG.info("Empty path");
+      return 0;
+    } else {
+      LOG.info("Listing of {}", path);
+      final long count = RemoteIterators.foreach(
+          fileSystem.listFiles(path, recursive),
+          (status) -> LOG.info("{}", status));
+      LOG.info("Count of entries: {}", count);
+      return count;
+    }
+  }
+
+  /**
+   * Assert that a cleanup stage coursehad a given outcome and
+   * deleted the given number of directories.
+   * @param result result to analyze
+   * @param outcome expected outcome
+   * @param expectedDirsDeleted #of directories deleted. -1 for no checks
+   */
+  protected void assertCleanupResult(
+      CleanupJobStage.Result result,
+      CleanupJobStage.Outcome outcome,
+      int expectedDirsDeleted) {
+    Assertions.assertThat(result.getOutcome())
+        .describedAs("Outcome of cleanup() in %s", result)
+        .isEqualTo(outcome);
+    if (expectedDirsDeleted >= 0) {
+      Assertions.assertThat(result.getDeleteCalls())
+          .describedAs("Number of directories deleted in cleanup %s", result)
+          .isEqualTo(expectedDirsDeleted);
+    }
+  }
+
+  /**
+   * Create and execute a cleanup stage.
+   * @param enabled is the stage enabled?
+   * @param deleteTaskAttemptDirsInParallel delete task attempt dirs in
+   *        parallel?
+   * @param suppressExceptions suppress exceptions?
+   * @param outcome expected outcome.
+   * @param expectedDirsDeleted #of directories deleted. -1 for no checks
+   * @return the result
+   * @throws IOException non-suppressed exception
+   */
+  protected CleanupJobStage.Result cleanup(
+      final boolean enabled,
+      final boolean deleteTaskAttemptDirsInParallel,
+      final boolean suppressExceptions,
+      final CleanupJobStage.Outcome outcome,
+      final int expectedDirsDeleted) throws IOException {
+    StageConfig stageConfig = getJobStageConfig();
+    CleanupJobStage.Result result = new CleanupJobStage(stageConfig)
+        .apply(new CleanupJobStage.Arguments(OP_STAGE_JOB_CLEANUP,
+            enabled, deleteTaskAttemptDirsInParallel, suppressExceptions));
+    assertCleanupResult(result, outcome, expectedDirsDeleted);
+    return result;
+  }
+
+  /**
+   * Read the UTF_8 text in the file.
+   * @param path path to read
+   * @return the string
+   * @throws IOException failure
+   */
+  protected String readText(final Path path) throws IOException {
+
+    final FileSystem fs = getFileSystem();
+    final FileStatus st = fs.getFileStatus(path);
+    Assertions.assertThat(st.getLen())
+        .describedAs("length of file %s", st)
+        .isLessThanOrEqualTo(MAX_LEN);
+
+    return new String(
+        readDataset(fs, path, (int) st.getLen()),
+        StandardCharsets.UTF_8);
+  }
+
+  /**
+   * Counter.
+   */
+  protected static final class ProgressCounter implements Progressable {
+
+    private final AtomicLong counter = new AtomicLong();
+
+    /**
+     * Increment the counter.
+     */
+    @Override
+    public void progress() {
+      counter.incrementAndGet();
+    }
+
+    /**
+     * Get the counter value.
+     * @return the current value.
+     */
+    public long value() {
+      return counter.get();
+    }
+
+    /**
+     * Reset the counter.
+     */
+    public void reset() {
+      counter.set(0);
+    }
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder(
+          "ProgressCounter{");
+      sb.append("counter=").append(counter.get());
+      sb.append('}');
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Get the progress counter of a stage.
+   * @param stageConfig stage
+   * @return its progress counter.
+   */
+  ProgressCounter progressOf(StageConfig stageConfig) {
+    return (ProgressCounter) stageConfig.getProgressable();
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/CommitterTestBinding.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/CommitterTestBinding.java
new file mode 100644
index 0000000000000..c6586ac17599c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/CommitterTestBinding.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.Progressable;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitter.TASK_COMMITTER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.randomJobId;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createIOStatisticsStore;
+
+/**
+ * This class represents a binding to a job in the target dir with TA, JA
+ * and associated paths.
+ * It's self contained so as to be usable in any test suite.
+ */
+class CommitterTestBinding implements
+    IOStatisticsSource {
+
+  /**
+   * IOStatistics counter for progress events.
+   */
+  public static final String PROGRESS_EVENTS = "progress_events";
+
+  /**
+   * IOStatistics to update with progress events.
+   */
+  private final IOStatisticsStore iostatistics;
+
+  /**
+   * Job attempt ID:.
+   */
+  private final String jobAttemptId;
+
+  /**
+   * Job ID.
+   */
+  private final JobID jobId;
+
+  /**
+   * Task Attempt ID, under the job attempt.
+   */
+  private final TaskAttemptID taskAttemptId;
+
+  /**
+   * Task ID.
+   */
+  private final TaskID taskId;
+
+  /**
+   * Task attempt context for the given task Attempt.
+   */
+  private final TaskAttemptContext taskAttemptContext;
+
+  /**
+   * Construct.
+   * @param conf job/task config. This is patched with the app attempt.
+   * @param appAttempt application attempt.
+   * @param taskNumber task number
+   * @param taskAttemptNumber which attempt on this task is it
+   */
+  CommitterTestBinding(
+      Configuration conf,
+      int appAttempt, int taskNumber, int taskAttemptNumber) {
+    iostatistics = createIOStatisticsStore()
+        .withCounters(PROGRESS_EVENTS)
+        .build();
+
+
+    // this is the job ID, with no attempt info.
+    jobId = JobID.forName(randomJobId());
+    jobAttemptId = jobId.toString() + "_ " + appAttempt;
+    taskId = new TaskID(jobId, TaskType.MAP, taskNumber);
+    taskAttemptId = new TaskAttemptID(taskId,
+        taskAttemptNumber);
+    conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttempt);
+    taskAttemptContext = new TaskAttemptContextImpl(conf, taskAttemptId);
+
+  }
+
+  /**
+   * Create a committer config for the given output path.
+   * @param outputPath output path in destFS.
+   * @return a committer for the active task.
+   */
+  ManifestCommitterConfig createCommitterConfig(
+      Path outputPath) {
+    return new ManifestCommitterConfig(outputPath,
+        TASK_COMMITTER,
+        taskAttemptContext,
+        iostatistics,
+        null);
+  }
+
+  /**
+   * Create a stage config from the committer config.
+   * All stats go to the local IOStatisticsStore;
+   * there's a progress callback also set to increment
+   * the counter {@link #PROGRESS_EVENTS}
+   * @return a stage config
+   */
+  StageConfig createStageConfig(Path outputPath) {
+    return createCommitterConfig(outputPath)
+        .createStageConfig()
+        .withProgressable(new ProgressCallback());
+  }
+
+  @Override
+  public IOStatisticsStore getIOStatistics() {
+    return iostatistics;
+  }
+
+  /**
+   * Whenever this progress callback is invoked, the progress_events
+   * counter is incremented. This allows for tests to verify that
+   * callbacks have occurred by asserting on the event counter.
+   */
+  private final class ProgressCallback implements Progressable {
+
+    @Override
+    public void progress() {
+      iostatistics.incrementCounter(PROGRESS_EVENTS, 1);
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterTestSupport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterTestSupport.java
new file mode 100644
index 0000000000000..7297fdd174ec6
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterTestSupport.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_CLASSNAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Support for committer tests.
+ */
+public final class ManifestCommitterTestSupport {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      ManifestCommitterTestSupport.class);
+
+  private static final DateTimeFormatter FORMATTER =
+      DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
+
+  /**
+   * Build directory property.
+   * Value: {@value}.
+   */
+  public static final String PROJECT_BUILD_DIRECTORY_PROPERTY
+      = "project.build.directory";
+
+  /**
+   * default number of task attempts for some tests.
+   * Value: {@value}.
+   */
+  public static final int NUMBER_OF_TASK_ATTEMPTS = 200;
+
+  private ManifestCommitterTestSupport() {
+  }
+
+  /**
+   * Create a random Job ID using the fork ID as part of the number if
+   * set in the current process.
+   * @return fork ID string in a format parseable by Jobs
+   */
+  public static String randomJobId() {
+    String testUniqueForkId = System.getProperty("test.unique.fork.id", "0001");
+    int l = testUniqueForkId.length();
+    String trailingDigits = testUniqueForkId.substring(l - 4, l);
+    int digitValue;
+    try {
+      digitValue = Integer.valueOf(trailingDigits);
+    } catch (NumberFormatException e) {
+      digitValue = 0;
+    }
+
+    return String.format("%s%04d_%04d",
+        FORMATTER.format(LocalDateTime.now()),
+        (long) (Math.random() * 1000),
+        digitValue);
+  }
+
+  public static File getProjectBuildDir() {
+    String propval = System.getProperty(PROJECT_BUILD_DIRECTORY_PROPERTY);
+    if (StringUtils.isEmpty(propval)) {
+      propval = "target";
+    }
+    return new File(propval).getAbsoluteFile();
+  }
+
+  /**
+   * Load a success file; fail if the file is empty/nonexistent.
+   * @param fs filesystem
+   * @param outputPath directory containing the success file.
+   * @return the loaded file.
+   * @throws IOException failure to find/load the file
+   * @throws AssertionError file is 0-bytes long,
+   */
+  public static ManifestSuccessData loadSuccessFile(final FileSystem fs,
+      final Path outputPath) throws IOException {
+    Path success = new Path(outputPath, SUCCESS_MARKER);
+    return ManifestSuccessData.load(fs, success);
+  }
+
+  /**
+   * Load in the success data marker.
+   * @param fs filesystem
+   * @param outputDir ouptu path of job
+   * @param minimumFileCount minimum number of files to have been created
+   * @param jobId job ID, only verified if non-empty
+   * @return the success data
+   * @throws IOException IO failure
+   */
+  public static ManifestSuccessData validateSuccessFile(
+      final FileSystem fs,
+      final Path outputDir,
+      final int minimumFileCount,
+      final String jobId) throws IOException {
+    Path successPath = new Path(outputDir, SUCCESS_MARKER);
+    ManifestSuccessData successData
+        = loadAndPrintSuccessData(fs, successPath);
+    assertThat(successData.getCommitter())
+        .describedAs("Committer field")
+        .isEqualTo(MANIFEST_COMMITTER_CLASSNAME);
+    assertThat(successData.getFilenames())
+        .describedAs("Files committed")
+        .hasSizeGreaterThanOrEqualTo(minimumFileCount);
+    if (isNotEmpty(jobId)) {
+      assertThat(successData.getJobId())
+          .describedAs("JobID")
+          .isEqualTo(jobId);
+    }
+    return successData;
+  }
+
+  /**
+   * Load in and print a success data manifest.
+   * @param fs filesystem
+   * @param successPath full path to success file.
+   * @return the success data
+   * @throws IOException IO failure
+   */
+  public static ManifestSuccessData loadAndPrintSuccessData(
+      FileSystem fs,
+      Path successPath) throws IOException {
+    LOG.info("Manifest {}", successPath);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream ps = new PrintStream(baos);
+    final ManifestPrinter showManifest = new ManifestPrinter(fs.getConf(), ps);
+    ManifestSuccessData successData = showManifest.loadAndPrintManifest(fs, successPath);
+    LOG.info("{}", baos);
+    return successData;
+  }
+
+  /**
+   * Validate all generated files from the manifest.
+   * All files in the manifest must exist.
+   * If the exclusive flag is set, only those must exist
+   * (ignoring all temp files and everything in the _temporary
+   * dir)
+   * @param fs filesystem
+   * @param destDir dest dir to scan
+   * @param successData manifest
+   * @param exclusive expect exclusive and complete data.
+   * @return the files and their status
+   * @throws IOException IO failure.
+   */
+  public static Map<Path, LocatedFileStatus> validateGeneratedFiles(
+      FileSystem fs,
+      Path destDir,
+      ManifestSuccessData successData,
+      boolean exclusive) throws IOException {
+    Map<Path, LocatedFileStatus> map = new HashMap<>();
+    RemoteIterators.foreach(fs.listFiles(destDir, true),
+        e -> {
+          if (!e.getPath().getName().startsWith("_")) {
+            map.put(e.getPath(), e);
+          }
+        });
+    // map has all files other than temp ones and the success marker
+    // what do we expect
+    final List<Path> expected = filesInManifest(successData);
+
+    // all of those must be found
+    Assertions.assertThat(map.keySet())
+        .describedAs("Files in FS compared to manifest")
+        .containsAll(expected);
+
+    // and if exclusive, that too
+    if (exclusive) {
+      Assertions.assertThat(map.keySet())
+          .describedAs("Files in FS compared to manifest")
+          .containsExactlyInAnyOrderElementsOf(expected);
+    }
+    return map;
+  }
+
+  /**
+   * Given a manifest, get the list of filenames
+   * and convert to paths.
+   * @param successData data
+   * @return the paths.
+   */
+  public static List<Path> filesInManifest(ManifestSuccessData successData) {
+    return successData.getFilenames().stream()
+        .map(AbstractManifestData::unmarshallPath)
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * List a directory/directory tree.
+   * @param fileSystem FS
+   * @param path path
+   * @param recursive do a recursive listing?
+   * @return the number of files found.
+   * @throws IOException failure.
+   */
+  public static long lsR(FileSystem fileSystem, Path path, boolean recursive)
+      throws Exception {
+    if (path == null) {
+      // surfaces when someone calls getParent() on something at the top
+      // of the path
+      LOG.info("Empty path");
+      return 0;
+    }
+    return RemoteIterators.foreach(fileSystem.listFiles(path, recursive),
+        (status) -> LOG.info("{}", status));
+  }
+
+  /**
+   * Assert that a file or dir entry matches the given parameters.
+   * Matching on paths, not strings, helps validate marshalling.
+   * @param fileOrDir file or directory
+   * @param src source path
+   * @param dest dest path
+   * @param l length
+   */
+  static void assertFileEntryMatch(
+      final FileEntry fileOrDir,
+      final Path src,
+      final Path dest,
+      final long l) {
+    String entry = fileOrDir.toString();
+    assertThat(fileOrDir.getSourcePath())
+        .describedAs("Source path of " + entry)
+        .isEqualTo(src);
+    assertThat(fileOrDir.getDestPath())
+        .describedAs("Dest path of " + entry)
+        .isEqualTo(dest);
+    assertThat(fileOrDir.getSize())
+        .describedAs("Size of " + entry)
+        .isEqualTo(l);
+  }
+
+  /**
+   * Assert that a dir entry matches the given parameters.
+   * Matching on paths, not strings, helps validate marshalling.
+   * @param fileOrDir file or directory
+   * @param dest dest path
+   * @param type type
+   */
+  static void assertDirEntryMatch(
+      final DirEntry fileOrDir,
+      final Path dest,
+      final long type) {
+    String entry = fileOrDir.toString();
+    assertThat(fileOrDir.getDestPath())
+        .describedAs("Dest path of " + entry)
+        .isEqualTo(dest);
+    assertThat(fileOrDir.getType())
+        .describedAs("type of " + entry)
+        .isEqualTo(type);
+  }
+
+  /**
+   * Closeable which can be used to safely close writers in
+   * a try-with-resources block..
+   */
+  public static final class CloseWriter<K, V> implements AutoCloseable {
+
+    private final RecordWriter<K, V> writer;
+
+    private final TaskAttemptContext context;
+
+    public CloseWriter(RecordWriter<K, V> writer,
+        TaskAttemptContext context) {
+      this.writer = writer;
+      this.context = context;
+    }
+
+    @Override
+    public void close() {
+      try {
+        writer.close(context);
+      } catch (IOException | InterruptedException e) {
+        LOG.error("When closing {} on context {}",
+            writer, context, e);
+      }
+    }
+  }
+
+  public static final String ATTEMPT_STRING =
+      "attempt_%s_m_%06d_%d";
+
+  /**
+   * Creates a random JobID and then as many tasks
+   * with the specific number of task attempts.
+   */
+  public static final class JobAndTaskIDsForTests {
+
+    /** Job ID; will be created uniquely for each instance. */
+    private final String jobId;
+
+    /**
+     * Store the details as strings; generate
+     * IDs on demand.
+     */
+    private final String[][] taskAttempts;
+
+    /**
+     * Constructor.
+     * @param tasks number of tasks.
+     * @param attempts number of attempts.
+     */
+    public JobAndTaskIDsForTests(int tasks, int attempts) {
+      this(randomJobId(), tasks, attempts);
+    }
+
+    public JobAndTaskIDsForTests(final String jobId,
+        int tasks, int attempts) {
+      this.jobId = jobId;
+      this.taskAttempts = new String[tasks][attempts];
+      for (int i = 0; i < tasks; i++) {
+        for (int j = 0; j < attempts; j++) {
+          String a = String.format(ATTEMPT_STRING,
+              jobId, i, j);
+          this.taskAttempts[i][j] = a;
+        }
+      }
+    }
+
+    /**
+     * Get the job ID.
+     * @return job ID string.
+     */
+    public String getJobId() {
+      return jobId;
+    }
+
+    /**
+     * Get the job ID as the MR type.
+     * @return job ID type.
+     */
+    public JobID getJobIdType() {
+      return getTaskIdType(0).getJobID();
+    }
+
+    /**
+     * Get a task attempt ID.
+     * @param task task index
+     * @param attempt attempt number.
+     * @return the task attempt.
+     */
+    public String getTaskAttempt(int task, int attempt) {
+      return taskAttempts[task][attempt];
+    }
+
+    /**
+     * Get task attempt ID as the MR type.
+     * @param task task index
+     * @param attempt attempt number.
+     * @return the task attempt type
+     */
+    public TaskAttemptID getTaskAttemptIdType(int task, int attempt) {
+      return TaskAttemptID.forName(getTaskAttempt(task, attempt));
+    }
+
+    /**
+     * Get task ID as the MR type.
+     * @param task task index
+     * @return the task ID type
+     */
+    public TaskID getTaskIdType(int task) {
+      return TaskAttemptID.forName(getTaskAttempt(task, 0)).getTaskID();
+    }
+
+    /**
+     * Get task ID as a string.
+     * @param task task index
+     * @return the task ID
+     */
+    public String getTaskId(int task) {
+      return getTaskIdType(task).toString();
+    }
+
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/StubStoreOperations.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/StubStoreOperations.java
new file mode 100644
index 0000000000000..d9269767f0761
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/StubStoreOperations.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Stub Store operations.
+ * Everything "works" provided you don't look too close.
+ * Files have etags of their filename; if you move a file without changing its
+ * name, the etag is preserved.
+ */
+public class StubStoreOperations extends ManifestStoreOperations {
+
+  @Override
+  public FileStatus getFileStatus(final Path path) throws IOException {
+    return new TaggedFileStatus(0, false, 1, 1024, 0, path, path.getName());
+  }
+
+  @Override
+  public boolean delete(final Path path, final boolean recursive)
+      throws IOException {
+    return true;
+  }
+
+  @Override
+  public boolean mkdirs(final Path path) throws IOException {
+    return true;
+  }
+
+  @Override
+  public boolean renameFile(final Path source, final Path dest)
+      throws IOException {
+    return true;
+  }
+
+  @Override
+  public RemoteIterator<FileStatus> listStatusIterator(final Path path)
+      throws IOException {
+    return new EmptyRemoteIterator<>();
+  }
+
+  @Override
+  public TaskManifest loadTaskManifest(JsonSerialization<TaskManifest> serializer,
+      final FileStatus st) throws IOException {
+    return new TaskManifest();
+  }
+
+  @Override
+  public <T extends AbstractManifestData<T>> void save(T manifestData,
+      final Path path,
+      final boolean overwrite) throws IOException {
+
+  }
+
+  @Override
+  public void close() throws IOException {
+
+  }
+
+  @Override
+  public boolean storePreservesEtagsThroughRenames(final Path path) {
+    return true;
+  }
+
+  /**
+   * Always empty rempte iterator.
+   * @param <T> type of iterator.
+   */
+  private static final class EmptyRemoteIterator<T>
+      implements RemoteIterator<T> {
+
+    @Override
+    public boolean hasNext() throws IOException {
+      return false;
+    }
+
+    @Override
+    public T next() throws IOException {
+      throw new NoSuchElementException();
+    }
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TaggedFileStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TaggedFileStatus.java
new file mode 100644
index 0000000000000..4a1115b667c1e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TaggedFileStatus.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * File Status with etag derived from the filename, if not explicitly set.
+ */
+public final class TaggedFileStatus extends FileStatus implements EtagSource {
+
+  private final String etag;
+
+  public TaggedFileStatus(final long length,
+      final boolean isdir,
+      final int blockReplication,
+      final long blocksize,
+      final long modificationTime,
+      final Path path,
+      final String etag) {
+    super(length, isdir, blockReplication, blocksize, modificationTime, path);
+    this.etag = etag;
+  }
+
+  public TaggedFileStatus(final FileStatus other, final String etag) throws IOException {
+    super(other);
+    this.etag = etag;
+  }
+
+  @Override
+  public String getEtag() {
+    return etag;
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCleanupStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCleanupStage.java
new file mode 100644
index 0000000000000..8d551c505209c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCleanupStage.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.UnreliableManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+
+
+/**
+ * Test the cleanup stage.
+ */
+public class TestCleanupStage extends AbstractManifestCommitterTest {
+
+  /**
+   * Number of task attempts to create. Manifests are created and written
+   * as well as test dirs, but no actual files.
+   */
+  protected static final int TASK_ATTEMPT_COUNT = 10;
+
+  /**
+   * How many delete calls for the root job delete?
+   */
+  protected static final int ROOT_DELETE_COUNT = 1;
+
+  /**
+   * Tocal invocation count for a successful parallel delete job.
+   */
+  protected static final int PARALLEL_DELETE_COUNT =
+      TASK_ATTEMPT_COUNT + ROOT_DELETE_COUNT;
+
+  /**
+   * Fault Injection.
+   */
+  private UnreliableManifestStoreOperations failures;
+
+  /**
+   * Manifests created.
+   */
+  private List<TaskManifest> manifests;
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    failures = new UnreliableManifestStoreOperations(
+        createManifestStoreOperations());
+    setStoreOperations(failures);
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir);
+    setJobStageConfig(stageConfig);
+    new SetupJobStage(stageConfig).apply(true);
+
+    // lots of tasks, but don't bother creating mock files.
+    manifests = executeTaskAttempts(TASK_ATTEMPT_COUNT, 0);
+  }
+
+  @Test
+  public void testCleanupInParallelHealthy() throws Throwable {
+    describe("parallel cleanup of TA dirs.");
+    cleanup(true, true, false,
+        CleanupJobStage.Outcome.PARALLEL_DELETE,
+        PARALLEL_DELETE_COUNT);
+    verifyJobDirsCleanedUp();
+  }
+
+  @Test
+  public void testCleanupSingletonHealthy() throws Throwable {
+    describe("Cleanup with a single delete. Not the default; would be best on HDFS");
+
+    cleanup(true, false, false,
+        CleanupJobStage.Outcome.DELETED, ROOT_DELETE_COUNT);
+    verifyJobDirsCleanedUp();
+  }
+
+  @Test
+  public void testCleanupNoDir() throws Throwable {
+    describe("parallel cleanup MUST not fail if there's no dir");
+    // first do the cleanup
+    cleanup(true, true, false,
+        CleanupJobStage.Outcome.PARALLEL_DELETE, PARALLEL_DELETE_COUNT);
+
+    // now expect cleanup by single delete still works
+    // the delete count is 0 as pre check skips it
+    cleanup(true, false, false,
+        CleanupJobStage.Outcome.NOTHING_TO_CLEAN_UP, 0);
+
+    // if skipped, that happens first
+    cleanup(false, true, false,
+        CleanupJobStage.Outcome.DISABLED, 0);
+  }
+
+  @Test
+  public void testFailureInParallelDelete() throws Throwable {
+    describe("A parallel delete fails, but the base delete works");
+
+    // pick one of the manifests
+    TaskManifest manifest = manifests.get(4);
+    Path taPath = new Path(manifest.getTaskAttemptDir());
+    failures.addDeletePathToFail(taPath);
+    cleanup(true, true, false,
+        CleanupJobStage.Outcome.DELETED, PARALLEL_DELETE_COUNT);
+  }
+
+  /**
+   * If there's no job task attempt subdir then the list of it will raise
+   * and FNFE; this MUST be caught and the base delete executed.
+   */
+  @Test
+  public void testParallelDeleteNoTaskAttemptDir() throws Throwable {
+    describe("Execute parallel delete where" +
+        " the job task directory does not exist");
+    StageConfig stageConfig = getJobStageConfig();
+    // TA dir doesn't exist, so listing will fail.
+    failures.addPathNotFound(stageConfig.getJobAttemptTaskSubDir());
+    cleanup(true, true, false,
+        CleanupJobStage.Outcome.DELETED, ROOT_DELETE_COUNT);
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCommitTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCommitTaskStage.java
new file mode 100644
index 0000000000000..4f4162d46cb9f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCommitTaskStage.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.FileNotFoundException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test committing a task.
+ */
+public class TestCommitTaskStage extends AbstractManifestCommitterTest {
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir);
+    setJobStageConfig(stageConfig);
+    new SetupJobStage(stageConfig).apply(true);
+  }
+
+  @Test
+  public void testCommitMissingDirectory() throws Throwable {
+
+    String tid = String.format("task_%03d", 1);
+    String taskAttemptId = String.format("%s_%02d",
+        tid, 1);
+    StageConfig taskStageConfig = createTaskStageConfig(JOB1, tid,
+        taskAttemptId);
+
+    // the task attempt dir does not exist
+    Path taDir = taskStageConfig.getTaskAttemptDir();
+    assertPathDoesNotExist("task attempt path", taDir);
+
+    // so the task commit fails
+    intercept(FileNotFoundException.class, () ->
+        new CommitTaskStage(taskStageConfig).apply(null));
+  }
+
+  @Test
+  public void testCommitEmptyDirectory() throws Throwable {
+
+    describe("Commit an empty directory as task then job");
+    String tid = String.format("task_%03d", 2);
+    String taskAttemptId = String.format("%s_%02d",
+        tid, 1);
+    StageConfig taskStageConfig = createTaskStageConfig(JOB1, tid,
+        taskAttemptId);
+
+    // set up the task
+    new SetupTaskStage(taskStageConfig).apply("setup");
+
+    CommitTaskStage.Result result = new CommitTaskStage(taskStageConfig)
+        .apply(null);
+
+    final TaskManifest manifest = result.getTaskManifest();
+    Assertions.assertThat(manifest.getDestDirectories())
+        .as("directories to create")
+        .isEmpty();
+    Assertions.assertThat(manifest.getFilesToCommit())
+        .as("files to commit")
+        .isEmpty();
+
+    final Path path = result.getPath();
+
+    final String manifestBody = readText(path);
+
+    LOG.info("manifest at {} of length {}:\n{}",
+        path, manifestBody.length(), manifestBody);
+
+    // now commit
+    final CommitJobStage.Result outcome = new CommitJobStage(getJobStageConfig())
+        .apply(new CommitJobStage.Arguments(
+            true, true, null,
+            new CleanupJobStage.Arguments(
+                OP_STAGE_JOB_CLEANUP,
+                true,
+                true,
+                false
+            )));
+
+    // review success file
+    final Path successPath = outcome.getSuccessPath();
+    String successBody = readText(successPath);
+    LOG.info("successBody at {} of length {}:\n{}",
+        successPath, successBody.length(), successBody);
+
+    final ManifestSuccessData successData = outcome.getJobSuccessData();
+    Assertions.assertThat(successData.getFilenames())
+        .as("Filenames in _SUCCESS")
+        .isEmpty();
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCreateOutputDirectoriesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCreateOutputDirectoriesStage.java
new file mode 100644
index 0000000000000..4b93ce937f80c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCreateOutputDirectoriesStage.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.EntryStatus;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CreateOutputDirectoriesStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.util.Lists;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupCounterStatistic;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_FILE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_MKDIRS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_FAILURES;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_DELETE_FILE_UNDER_DESTINATION;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_PREPARE_DIR_ANCESTORS;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test directory creation.
+ * As the directory creation phase relies on input from the task manifest to
+ * determine which directories to explicitly create, delete files at
+ * etc, these tests build up manifests and assert that the output
+ * of the directory creation stage matches that of the combination
+ * of the manifest and the filesystem state.
+ */
+public class TestCreateOutputDirectoriesStage extends AbstractManifestCommitterTest {
+
+  /**
+   * Deep tree width, subclasses (including in external projects)
+   * may change.
+   */
+  protected static final int DEEP_TREE_WIDTH = 4;
+
+  /**
+   * The number of directories created in test setup; this must be
+   * added to all assertions of the value of OP_MKDIRS.
+   */
+  private static final int DIRECTORIES_CREATED_IN_SETUP = 2;
+
+  private Path destDir;
+  private CreateOutputDirectoriesStage mkdirStage;
+  private StageConfig stageConfig;
+  private IOStatisticsStore iostats;
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    destDir = methodPath();
+    // clean up dest dir completely
+    destDir.getFileSystem(getConfiguration()).delete(destDir, true);
+    setStoreOperations(createManifestStoreOperations());
+    stageConfig = createStageConfigForJob(JOB1, destDir)
+        .withDeleteTargetPaths(true);
+    setJobStageConfig(stageConfig);
+    // creates the job directories.
+    new SetupJobStage(stageConfig).apply(true);
+    mkdirStage = new CreateOutputDirectoriesStage(stageConfig);
+    iostats = stageConfig.getIOStatistics();
+    // assert original count of dirs created == 2 : job and task manifest
+    verifyStatisticCounterValue(iostats, OP_MKDIRS,
+        DIRECTORIES_CREATED_IN_SETUP);
+    // reset the value to simplify future work
+    iostats.getCounterReference(OP_MKDIRS).set(0);
+  }
+
+  @Test
+  public void testPrepareSomeDirs() throws Throwable {
+
+    final long initialFileStatusCount = lookupCounterStatistic(iostats, OP_GET_FILE_STATUS);
+    final int dirCount = 8;
+    final List<Path> dirs = subpaths(destDir, dirCount);
+    final List<DirEntry> dirEntries = dirEntries(dirs, 1, EntryStatus.not_found);
+
+    // two manifests with duplicate entries
+    final List<TaskManifest> manifests = Lists.newArrayList(
+        manifestWithDirsToCreate(dirEntries),
+        manifestWithDirsToCreate(dirEntries));
+    final CreateOutputDirectoriesStage.Result result = mkdirStage.apply(manifests);
+    Assertions.assertThat(result.getCreatedDirectories())
+        .describedAs("output of %s", mkdirStage)
+        .containsExactlyInAnyOrderElementsOf(dirs);
+
+    LOG.info("Job Statistics\n{}", ioStatisticsToPrettyString(iostats));
+
+    // now dirCount new dirs are added.
+    verifyStatisticCounterValue(iostats, OP_MKDIRS, dirCount);
+
+    // now rerun the same preparation sequence, but this
+    // time declare that the directories exist (as they do)
+    final CreateOutputDirectoriesStage s2 =
+        new CreateOutputDirectoriesStage(stageConfig);
+    final CreateOutputDirectoriesStage.Result r2 = s2.apply(
+        Lists.newArrayList(
+            manifestWithDirsToCreate(dirEntries(dirs, 1, EntryStatus.dir))));
+
+    // no directories are now created.
+    Assertions.assertThat(r2.getCreatedDirectories())
+        .describedAs("output of %s", s2)
+        .isEmpty();
+    LOG.info("Job Statistics after second pass\n{}", ioStatisticsToPrettyString(iostats));
+
+    // second run probed no dest dirs
+    verifyStatisticCounterValue(iostats, OP_GET_FILE_STATUS, initialFileStatusCount);
+    // and no new mkdir calls were made
+    verifyStatisticCounterValue(iostats, OP_MKDIRS, dirCount);
+    verifyStatisticCounterValue(iostats, OP_DELETE_FILE_UNDER_DESTINATION, 0);
+    verifyStatisticCounterValue(iostats, OP_IS_FILE, 0);
+  }
+
+  /**
+   * Given a list of paths, build a list of DirEntry entries.
+   * @param paths list of paths
+   * @param level Level in the treewalk.
+   * @param entryStatus status of dirs
+   * @return list of entries with  the given level and entry status.
+   */
+  protected List<DirEntry> dirEntries(Collection<Path> paths,
+      int level,
+      EntryStatus entryStatus) {
+    return paths.stream()
+        .map(p -> DirEntry.dirEntry(p, entryStatus, level))
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Create a manifest with the list of directory entries added.
+   * Job commit requires the entries to have been probed for, and
+   * for the entire tree under the dest path to be included.
+   * @param dirEntries list of directory entries.
+   * @return the manifest.
+   */
+  protected TaskManifest manifestWithDirsToCreate(List<DirEntry> dirEntries) {
+    final TaskManifest taskManifest = new TaskManifest();
+    taskManifest.getDestDirectories().addAll(dirEntries);
+    return taskManifest;
+  }
+
+  /**
+   * Assert the directory map status of a path.
+   * @param result stage result
+   * @param path path to look up
+   * @param expected expected value.
+   */
+  private static void assertDirMapStatus(
+      CreateOutputDirectoriesStage.Result result,
+      Path path,
+      CreateOutputDirectoriesStage.DirMapState expected) {
+    Assertions.assertThat(result.getDirMap())
+        .describedAs("Directory Map entry for %s", path)
+        .isNotNull()
+        .containsKey(path)
+        .containsEntry(path, expected);
+  }
+
+  /**
+   * Prepare a deep tree {@code c ^ 3} of entries.
+   * Make one of the parent dirs a file.
+   *
+   * From a test-purity perspective, this should
+   * be separate tests. But attempting
+   * operations in the same test cases spreads the
+   * directory setup costs across both, rather than
+   * duplicating it.
+   */
+  @Test
+  public void testPrepareDirtyTree() throws Throwable {
+
+    // build the lists of paths for the different levels
+    final int c = getDeepTreeWidth();
+    final List<Path> level1 = subpaths(destDir, c);
+    final List<Path> level2 = level1.stream().flatMap(p ->
+            subpaths(p, c).stream())
+        .collect(Collectors.toList());
+    final List<Path> level3 = level2.stream().flatMap(p ->
+            subpaths(p, c).stream())
+        .collect(Collectors.toList());
+    // manifest dir entry list contains all levels > 0
+    // adding them out of order verifies sorting takes place
+    // before the merge routine which is intended to strip
+    // out parent dirs
+    final List<DirEntry> directories = new ArrayList<>();
+    final List<DirEntry> l1 = dirEntries(level1, 1, EntryStatus.not_found);
+    directories.addAll(l1);
+    final List<DirEntry> l3 = dirEntries(level3, 3, EntryStatus.not_found);
+    directories.addAll(l3);
+    final List<DirEntry> l2 = dirEntries(level2, 2, EntryStatus.not_found);
+    directories.addAll(l2);
+
+    // one of the level 0 paths is going to be a file
+    final DirEntry parentIsFile = l1.get(1);
+    // one entry has a dir already
+    final DirEntry parentIsDir = l2.get(0);
+    // and one of the dest dirs is a file.
+    final DirEntry leafIsFile = l3.get(0);
+
+    // prepare the output
+    CompletableFuture.allOf(
+        asyncPut(parentIsFile.getDestPath(), NO_DATA),
+        asyncPut(leafIsFile.getDestPath(), NO_DATA),
+        asyncMkdir(parentIsDir.getDestPath()))
+        .join();
+
+    // patch the entries, which, as they are references
+    // into the lists, updates the values there.
+    parentIsFile.setStatus(EntryStatus.file);
+    parentIsDir.setStatus(EntryStatus.dir);
+    leafIsFile.setStatus(EntryStatus.file);
+
+    final List<TaskManifest> manifests = Lists.newArrayList(
+        manifestWithDirsToCreate(directories));
+
+    // first attempt will succeed.
+    final CreateOutputDirectoriesStage.Result result =
+        mkdirStage.apply(manifests);
+
+    LOG.info("Job Statistics\n{}", ioStatisticsToPrettyString(iostats));
+
+    assertDirMapStatus(result, leafIsFile.getDestPath(),
+        CreateOutputDirectoriesStage.DirMapState.fileNowDeleted);
+
+    // for the parent dir, all is good
+    assertDirMapStatus(result, parentIsFile.getDestPath(),
+        CreateOutputDirectoriesStage.DirMapState.fileNowDeleted);
+    Assertions.assertThat(result.getCreatedDirectories())
+        .describedAs("output of %s", mkdirStage)
+        .containsExactlyInAnyOrderElementsOf(level3);
+    verifyStatisticCounterValue(iostats, OP_MKDIRS, level3.size());
+    // do a rerun. where the directory setup will fail because
+    // a directory is present where the manifest says there is
+    // a file.
+    CreateOutputDirectoriesStage attempt2 =
+        new CreateOutputDirectoriesStage(
+            createStageConfigForJob(JOB1, destDir)
+                .withDeleteTargetPaths(true));
+    // attempt will fail because one of the entries marked as
+    // a file to delete is now a non-empty directory
+    LOG.info("Executing failing attempt to create the directories");
+    intercept(IOException.class, () -> attempt2.apply(manifests));
+    verifyStatisticCounterValue(iostats, OP_PREPARE_DIR_ANCESTORS + SUFFIX_FAILURES, 1);
+    verifyStatisticCounterValue(iostats, OP_DELETE + SUFFIX_FAILURES, 1);
+
+    // build a new directory list where everything is declared a directory;
+    // no dirs will be created this time as they all exist.
+    final List<DirEntry> directories3 = new ArrayList<>();
+    directories3.addAll(dirEntries(level1, 1, EntryStatus.dir));
+    directories3.addAll(dirEntries(level2, 2, EntryStatus.dir));
+    directories3.addAll(dirEntries(level3, 3, EntryStatus.dir));
+
+    final List<TaskManifest> manifests3 = Lists.newArrayList(
+        manifestWithDirsToCreate(directories3));
+    CreateOutputDirectoriesStage attempt3 =
+        new CreateOutputDirectoriesStage(
+            createStageConfigForJob(JOB1, destDir)
+                .withDeleteTargetPaths(true));
+    final CreateOutputDirectoriesStage.Result r3 =
+        attempt3.apply(manifests3);
+    assertDirMapStatus(r3, leafIsFile.getDestPath(),
+        CreateOutputDirectoriesStage.DirMapState.dirFoundInStore);
+    Assertions.assertThat(r3.getCreatedDirectories())
+        .describedAs("created directories")
+        .isEmpty();
+  }
+
+  /**
+   * Get the width of the deep tree; subclasses may tune for test performance, though
+   * a wide one is more realistic of real jobs.
+   * @return number of subdirs to create at each level. Must be at least 2
+   */
+  protected int getDeepTreeWidth() {
+    return DEEP_TREE_WIDTH;
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestJobThroughManifestCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestJobThroughManifestCommitter.java
new file mode 100644
index 0000000000000..fa3527f11cc2b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestJobThroughManifestCommitter.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.OutputValidationException;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.AbortTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.LoadManifestsStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.ValidateRenamedFilesStage;
+import org.apache.hadoop.net.NetUtils;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.rm;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ID_SOURCE_MAPREDUCE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_CLASSNAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_BYTES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_FILES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.loadAndPrintSuccessData;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.validateGeneratedFiles;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.PRINCIPAL;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestPathForTask;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage.DISABLED;
+import static org.apache.hadoop.security.UserGroupInformation.getCurrentUser;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test IO through the stages.
+ * This mimics the workflow of a job with two tasks,
+ * the first task has two attempts where the second attempt
+ * is committed after the first attempt (simulating the
+ * failure-during-task-commit which the v2 algorithm cannot
+ * handle).
+ *
+ * The test is ordered and the output dir is not cleaned up
+ * after each test case.
+ * The last test case MUST perform the cleanup.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class TestJobThroughManifestCommitter
+    extends AbstractManifestCommitterTest {
+
+  /** Destination directory. */
+  private Path destDir;
+
+  /** directory names for the tests. */
+  private ManifestCommitterSupport.AttemptDirectories dirs;
+
+  /**
+   * To ensure that the local FS has a shared root path, this is static.
+   */
+  @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+  private static Path sharedTestRoot = null;
+
+  /**
+   * Job ID.
+   */
+  private String jobId;
+
+  /**
+   * Task 0 attempt 0 ID.
+   */
+  private String taskAttempt00;
+
+  /**
+   * Task 0 attempt 1 ID.
+   */
+  private String taskAttempt01;
+
+  /**
+   * Task 1 attempt 0 ID.
+   */
+  private String taskAttempt10;
+
+  /**
+   * Task 1 attempt 1 ID.
+   */
+  private String taskAttempt11;
+
+  /**
+   * Stage config for TA00.
+   */
+  private StageConfig ta00Config;
+
+  /**
+   * Stage config for TA01.
+   */
+  private StageConfig ta01Config;
+
+  /**
+   * Stage config for TA10.
+   */
+  private StageConfig ta10Config;
+
+  /**
+   * Stage config for TA11.
+   */
+  private StageConfig ta11Config;
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    taskAttempt00 = TASK_IDS.getTaskAttempt(TASK0, TA0);
+    taskAttempt01 = TASK_IDS.getTaskAttempt(TASK0, TA1);
+    taskAttempt10 = TASK_IDS.getTaskAttempt(TASK1, TA0);
+    taskAttempt11 = TASK_IDS.getTaskAttempt(TASK1, TA1);
+    setSharedPath(path("TestJobThroughManifestCommitter"));
+    // add a dir with a space in.
+    destDir = new Path(sharedTestRoot, "out put");
+    jobId = TASK_IDS.getJobId();
+    // then the specific path underneath that for the attempt.
+    dirs = new ManifestCommitterSupport.AttemptDirectories(destDir,
+        jobId, 1);
+
+    // config for job attempt 1, task 00
+    setJobStageConfig(createStageConfigForJob(JOB1, destDir).build());
+    ta00Config = createStageConfig(JOB1, TASK0, TA0, destDir).build();
+    ta01Config = createStageConfig(JOB1, TASK0, TA1, destDir).build();
+    ta10Config = createStageConfig(JOB1, TASK1, TA0, destDir).build();
+    ta11Config = createStageConfig(JOB1, TASK1, TA1, destDir).build();
+  }
+
+  /**
+   * Test dir deletion is removed from test case teardown so the
+   * subsequent tests see the output.
+   * @throws IOException failure
+   */
+  @Override
+  protected void deleteTestDirInTeardown() throws IOException {
+    /* no-op */
+  }
+
+  /**
+   * Override point and something to turn on/off when exploring what manifests look like.
+   * Stores where storage is billed MUST enable this.
+   * @return true if, at the end of the run, the test dir should be deleted.
+   */
+  protected boolean shouldDeleteTestRootAtEndOfTestRun() {
+    return false;
+  }
+
+  /**
+   * Invoke this to clean up the test directories.
+   */
+  private void deleteSharedTestRoot() throws IOException {
+    describe("Deleting shared test root %s", sharedTestRoot);
+
+    rm(getFileSystem(), sharedTestRoot, true, false);
+  }
+
+  /**
+   * Set the shared test root if not already set.
+   * @param path path to set.
+   * @return true if the path was set
+   */
+  private static synchronized boolean setSharedPath(final Path path) {
+    if (sharedTestRoot == null) {
+      // set this as needed
+      LOG.info("Set shared path to {}", path);
+      sharedTestRoot = path;
+      return true;
+    }
+    return false;
+  }
+
+  @Test
+  public void test_0000_setupTestDir() throws Throwable {
+    describe("always ensure directory setup is empty");
+    deleteSharedTestRoot();
+  }
+
+  @Test
+  public void test_0100_setupJobStage() throws Throwable {
+    describe("Set up a job");
+    verifyPath("Job attempt dir",
+        dirs.getJobAttemptDir(),
+        new SetupJobStage(getJobStageConfig()).apply(true));
+  }
+
+  /**
+   * And the check that the stage worked.
+   * @throws IOException failure.
+   */
+  private void verifyJobSetupCompleted() throws IOException {
+    assertPathExists("Job attempt dir from test_0100", dirs.getJobAttemptDir());
+  }
+
+  @Test
+  public void test_0110_setupJobOnlyAllowedOnce() throws Throwable {
+    describe("a second creation of a job attempt must fail");
+    verifyJobSetupCompleted();
+    intercept(FileAlreadyExistsException.class, "", () ->
+        new SetupJobStage(getJobStageConfig()).apply(true));
+    // job is still there
+    assertPathExists("Job attempt dir", dirs.getJobAttemptDir());
+  }
+
+  @Test
+  public void test_0120_setupJobNewAttemptNumber() throws Throwable {
+    describe("Creating a new job attempt is supported");
+    verifyJobSetupCompleted();
+    Path path = pathMustExist("Job attempt 2 dir",
+        new SetupJobStage(createStageConfig(2, -1, 0, destDir))
+            .apply(false));
+    Assertions.assertThat(path)
+        .describedAs("Stage created path")
+        .isNotEqualTo(dirs.getJobAttemptDir());
+  }
+
+  @Test
+  public void test_0200_setupTask00() throws Throwable {
+    describe("Set up a task; job must have been set up first");
+    verifyJobSetupCompleted();
+    verifyPath("Task attempt 00",
+        dirs.getTaskAttemptPath(taskAttempt00),
+        new SetupTaskStage(ta00Config).apply("first"));
+  }
+
+  /**
+   * Verify TA00 is set up.
+   */
+  private void verifyTaskAttempt00SetUp() throws IOException {
+    pathMustExist("Dir from taskAttempt00 setup",
+        dirs.getTaskAttemptPath(taskAttempt00));
+  }
+
+  @Test
+  public void test_0210_setupTask00OnlyAllowedOnce() throws Throwable {
+    describe("Second attempt to set up task00 must fail.");
+    verifyTaskAttempt00SetUp();
+    intercept(FileAlreadyExistsException.class, "second", () ->
+        new SetupTaskStage(ta00Config).apply("second"));
+  }
+
+  @Test
+  public void test_0220_setupTask01() throws Throwable {
+    describe("Setup task attempt 01");
+    verifyTaskAttempt00SetUp();
+    verifyPath("Task attempt 01",
+        dirs.getTaskAttemptPath(taskAttempt01),
+        new SetupTaskStage(ta01Config)
+            .apply("01"));
+  }
+
+  @Test
+  public void test_0230_setupTask10() throws Throwable {
+    describe("Setup task attempt 10");
+    verifyJobSetupCompleted();
+    verifyPath("Task attempt 10",
+        dirs.getTaskAttemptPath(taskAttempt10),
+        new SetupTaskStage(ta10Config)
+            .apply("10"));
+  }
+
+  /**
+   * Setup then abort task 11 before creating any files;
+   * verify that commit fails before creating a manifest file.
+   */
+  @Test
+  public void test_0240_setupThenAbortTask11() throws Throwable {
+    describe("Setup then abort task attempt 11");
+    verifyJobSetupCompleted();
+    Path ta11Path = new SetupTaskStage(ta11Config).apply("11");
+    Path deletedDir = new AbortTaskStage(ta11Config).apply(false);
+    Assertions.assertThat(ta11Path)
+        .isEqualTo(deletedDir);
+    assertPathDoesNotExist("aborted directory", ta11Path);
+    // execute will fail as there's no dir to list.
+    intercept(FileNotFoundException.class, () ->
+        new CommitTaskStage(ta11Config).apply(null));
+    assertPathDoesNotExist("task manifest",
+        manifestPathForTask(dirs.getTaskManifestDir(),
+            TASK_IDS.getTaskId(TASK1)));
+  }
+
+  /**
+   * Execute TA01 by generating a lot of files in its directory
+   * then committing the task attempt.
+   * The manifest at the task path (i.e. the record of which attempt's
+   * output is to be used) MUST now have been generated by this TA.
+   */
+  @Test
+  public void test_0300_executeTask00() throws Throwable {
+    describe("Create the files for Task 00, then commit the task");
+    List<Path> files = createFilesOrDirs(dirs.getTaskAttemptPath(taskAttempt00),
+        "part-00", getExecutorService(),
+        DEPTH, WIDTH, FILES_PER_DIRECTORY, false);
+    // saves the task manifest to the job dir
+    CommitTaskStage.Result result = new CommitTaskStage(ta00Config)
+        .apply(null);
+    verifyPathExists(getFileSystem(), "manifest",
+        result.getPath());
+
+    TaskManifest manifest = result.getTaskManifest();
+    manifest.validate();
+    // clear the IOStats to reduce the size of the printed JSON.
+    manifest.setIOStatistics(null);
+    LOG.info("Task Manifest {}", manifest.toJson());
+    validateTaskAttemptManifest(this.taskAttempt00, files, manifest);
+  }
+
+  /**
+   * Validate the manifest of a task attempt.
+   * @param attemptId attempt ID
+   * @param files files which were created.
+   * @param manifest manifest
+   * @throws IOException IO problem
+   */
+  protected void validateTaskAttemptManifest(
+      String attemptId,
+      List<Path> files,
+      TaskManifest manifest) throws IOException {
+
+    verifyManifestTaskAttemptID(manifest, attemptId);
+
+    // validate the manifest
+    verifyManifestFilesMatch(manifest, files);
+  }
+
+  /**
+   * Execute TA01 by generating a lot of files in its directory
+   * then committing the task attempt.
+   * The manifest at the task path (i.e. the record of which attempt's
+   * output is to be used) MUST now have been generated by this TA.
+   * Any existing manifest will have been overwritten.
+   */
+  @Test
+  public void test_0310_executeTask01() throws Throwable {
+    describe("Create the files for Task 01, then commit the task");
+    List<Path> files = createFilesOrDirs(dirs.getTaskAttemptPath(taskAttempt01),
+        "part-00", getExecutorService(),
+        DEPTH, WIDTH, FILES_PER_DIRECTORY, false);
+    // saves the task manifest to the job dir
+    CommitTaskStage.Result result = new CommitTaskStage(ta01Config)
+        .apply(null);
+    Path manifestPath = verifyPathExists(getFileSystem(), "manifest",
+        result.getPath()).getPath();
+
+    // load the manifest from the FS, not the return value,
+    // so we can verify that last task to commit wins.
+    TaskManifest manifest = TaskManifest.load(getFileSystem(), manifestPath);
+    manifest.validate();
+    // clear the IOStats to reduce the size of the printed JSON.
+    manifest.setIOStatistics(null);
+    LOG.info("Task Manifest {}", manifest.toJson());
+
+    validateTaskAttemptManifest(taskAttempt01, files, manifest);
+
+  }
+
+  /**
+   * Second task writes to more directories, but fewer files per dir.
+   * This ensures that there will dirs here which aren't in the first
+   * attempt.
+   */
+  @Test
+  public void test_0320_executeTask10() throws Throwable {
+    describe("Create the files for Task 10, then commit the task");
+    List<Path> files = createFilesOrDirs(
+        dirs.getTaskAttemptPath(ta10Config.getTaskAttemptId()),
+        "part-01", getExecutorService(),
+        DEPTH, WIDTH + 1, FILES_PER_DIRECTORY - 1, false);
+    // saves the task manifest to the job dir
+    CommitTaskStage.Result result = new CommitTaskStage(ta10Config)
+        .apply(null);
+    TaskManifest manifest = result.getTaskManifest();
+    validateTaskAttemptManifest(taskAttempt10, files, manifest);
+  }
+
+  @Test
+  public void test_0340_setupThenAbortTask11() throws Throwable {
+    describe("Setup then abort task attempt 11");
+    Path ta11Path = new SetupTaskStage(ta11Config).apply("11");
+    createFilesOrDirs(
+        ta11Path,
+        "part-01", getExecutorService(),
+        2, 1, 1, false);
+
+    new AbortTaskStage(ta11Config).apply(false);
+    assertPathDoesNotExist("aborted directory", ta11Path);
+    // execute will fail as there's no dir to list.
+    intercept(FileNotFoundException.class, () ->
+        new CommitTaskStage(ta11Config).apply(null));
+
+    // and the manifest MUST be unchanged from the previous stage
+    Path manifestPathForTask1 = manifestPathForTask(dirs.getTaskManifestDir(),
+        TASK_IDS.getTaskId(TASK1));
+    verifyManifestTaskAttemptID(
+        TaskManifest.load(getFileSystem(), manifestPathForTask1),
+        taskAttempt10);
+
+  }
+
+  /**
+   * Load all the committed manifests, which must be TA01 (last of
+   * task 0 to commit) and TA10.
+   */
+  @Test
+  public void test_0400_loadManifests() throws Throwable {
+    describe("Load all manifests; committed must be TA01 and TA10");
+    LoadManifestsStage.Result result
+        = new LoadManifestsStage(getJobStageConfig()).apply(true);
+    String summary = result.getSummary().toString();
+    LOG.info("Manifest summary {}", summary);
+    List<TaskManifest> manifests = result.getManifests();
+    Assertions.assertThat(manifests)
+        .describedAs("Loaded manifests in %s", summary)
+        .hasSize(2);
+    Map<String, TaskManifest> manifestMap = toMap(manifests);
+    verifyManifestTaskAttemptID(
+        manifestMap.get(taskAttempt01), taskAttempt01);
+    verifyManifestTaskAttemptID(
+        manifestMap.get(taskAttempt10), taskAttempt10);
+  }
+
+  @Test
+  public void test_0410_commitJob() throws Throwable {
+    describe("Commit the job");
+    CommitJobStage stage = new CommitJobStage(getJobStageConfig());
+    stage.apply(new CommitJobStage.Arguments(true, false, null, DISABLED));
+  }
+
+  /**
+   * Validate that the job output is good by invoking the
+   * {@link ValidateRenamedFilesStage} stage to
+   * validate all the manifests.
+   */
+  @Test
+  public void test_0420_validateJob() throws Throwable {
+    describe("Validate the output of the job through the validation"
+        + " stage");
+
+
+    // load in the success data.
+    ManifestSuccessData successData = loadAndPrintSuccessData(
+        getFileSystem(),
+        getJobStageConfig().getJobSuccessMarkerPath());
+
+    // load manifests stage will load all the task manifests again
+    List<TaskManifest> manifests = new LoadManifestsStage(getJobStageConfig())
+        .apply(true).getManifests();
+    // Now verify their files exist, returning the list of renamed files.
+    List<String> committedFiles = new ValidateRenamedFilesStage(getJobStageConfig())
+        .apply(manifests)
+        .stream().map(FileEntry::getDest)
+        .collect(Collectors.toList());
+
+    // verify that the list of committed files also matches
+    // that in the _SUCCESS file
+    // note: there's a limit to the #of files in the SUCCESS file
+    // to stop writing it slowing down jobs; therefore we don't
+    // make a simple "all must match check
+    Assertions.assertThat(committedFiles)
+        .containsAll(successData.getFilenames());
+
+    // now patch one of the manifest files by editing an entry
+    FileEntry entry = manifests.get(0).getFilesToCommit().get(0);
+    // no longer exists.
+    String oldName = entry.getDest();
+    String newName = oldName + ".missing";
+    entry.setDest(newName);
+
+    // validation will now fail
+    intercept(OutputValidationException.class, ".missing", () ->
+        new ValidateRenamedFilesStage(getJobStageConfig())
+            .apply(manifests));
+
+    // restore the name, but change the size
+    entry.setDest(oldName);
+    entry.setSize(128_000_000);
+    intercept(OutputValidationException.class, () ->
+        new ValidateRenamedFilesStage(getJobStageConfig())
+            .apply(manifests));
+  }
+
+  @Test
+  public void test_0430_validateStatistics() throws Throwable {
+    // load in the success data.
+    ManifestSuccessData successData = ManifestSuccessData.load(
+        getFileSystem(),
+        getJobStageConfig().getJobSuccessMarkerPath());
+    String json = successData.toJson();
+    LOG.info("Success data is {}", json);
+    Assertions.assertThat(successData)
+        .describedAs("Manifest " + json)
+        .returns(NetUtils.getLocalHostname(),
+            ManifestSuccessData::getHostname)
+        .returns(MANIFEST_COMMITTER_CLASSNAME,
+            ManifestSuccessData::getCommitter)
+        .returns(jobId,
+            ManifestSuccessData::getJobId)
+        .returns(true,
+            ManifestSuccessData::getSuccess)
+        .returns(JOB_ID_SOURCE_MAPREDUCE,
+            ManifestSuccessData::getJobIdSource);
+    // diagnostics
+    Assertions.assertThat(successData.getDiagnostics())
+        .containsEntry(PRINCIPAL,
+            getCurrentUser().getShortUserName())
+        .containsEntry(STAGE, OP_STAGE_JOB_COMMIT);
+
+    // and stats
+    IOStatisticsSnapshot iostats = successData.getIOStatistics();
+
+    int files = successData.getFilenames().size();
+    verifyStatisticCounterValue(iostats,
+        OP_STAGE_JOB_COMMIT, 1);
+    assertThatStatisticCounter(iostats,
+        COMMITTER_FILES_COMMITTED_COUNT)
+        .isGreaterThanOrEqualTo(files);
+    Long totalFiles = iostats.counters().get(COMMITTER_FILES_COMMITTED_COUNT);
+    verifyStatisticCounterValue(iostats,
+        COMMITTER_BYTES_COMMITTED_COUNT, totalFiles * 2);
+  }
+
+  @Test
+  public void test_440_validateSuccessFiles() throws Throwable {
+
+    // load in the success data.
+    final FileSystem fs = getFileSystem();
+    ManifestSuccessData successData = loadAndPrintSuccessData(
+        fs,
+        getJobStageConfig().getJobSuccessMarkerPath());
+    validateGeneratedFiles(fs,
+        getJobStageConfig().getDestinationDir(),
+        successData, false);
+  }
+
+  @Test
+  public void test_0900_cleanupJob() throws Throwable {
+    describe("Cleanup job");
+    CleanupJobStage.Arguments arguments = new CleanupJobStage.Arguments(
+        OP_STAGE_JOB_CLEANUP, true, true, false);
+    // the first run will list the three task attempt dirs and delete each
+    // one before the toplevel dir.
+    CleanupJobStage.Result result = new CleanupJobStage(
+        getJobStageConfig()).apply(arguments);
+    assertCleanupResult(result, CleanupJobStage.Outcome.PARALLEL_DELETE, 1 + 3);
+    assertPathDoesNotExist("Job attempt dir", result.getDirectory());
+
+    // not an error if we retry and the dir isn't there
+    result = new CleanupJobStage(getJobStageConfig()).apply(arguments);
+    assertCleanupResult(result, CleanupJobStage.Outcome.NOTHING_TO_CLEAN_UP, 0);
+  }
+
+  /**
+   * Needed to clean up the shared test root, as test case teardown
+   * does not do it.
+   */
+  //@Test
+  public void test_9999_cleanupTestDir() throws Throwable {
+    if (shouldDeleteTestRootAtEndOfTestRun()) {
+      deleteSharedTestRoot();
+    }
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestLoadManifestsStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestLoadManifestsStage.java
new file mode 100644
index 0000000000000..00372c5941efa
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestLoadManifestsStage.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CreateOutputDirectoriesStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.LoadManifestsStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+
+/**
+ * Test loading manifests from a store.
+ * By not creating files we can simulate a large job just by
+ * creating the manifests.
+ * The SaveTaskManifestStage stage is used for the save operation;
+ * this does a save + rename.
+ * For better test performance against a remote store, a thread
+ * pool is used to save the manifests in parallel.
+ */
+public class TestLoadManifestsStage extends AbstractManifestCommitterTest {
+
+  private int taskAttemptCount;
+
+  /**
+   * How many task attempts to make?
+   * Override point.
+   * @return a number greater than 0.
+   */
+  protected int numberOfTaskAttempts() {
+    return ManifestCommitterTestSupport.NUMBER_OF_TASK_ATTEMPTS;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    taskAttemptCount = numberOfTaskAttempts();
+    Assertions.assertThat(taskAttemptCount)
+        .describedAs("Task attempt count")
+        .isGreaterThan(0);
+  }
+
+  /**
+   * Build a large number of manifests, but without the real files
+   * and directories.
+   * Save the manifests under the job attempt dir, then load
+   * them via the {@link LoadManifestsStage}.
+   * The directory preparation process is then executed after this.
+   * Because we know each task attempt creates the same number of directories,
+   * they will all be merged and so only a limited number of output dirs
+   * will be created.
+   */
+  @Test
+  public void testSaveThenLoadManyManifests() throws Throwable {
+
+    describe("Creating many manifests with fake file/dir entries,"
+        + " load them and prepare the output dirs.");
+
+    int filesPerTaskAttempt = 10;
+    LOG.info("Number of task attempts: {}, files per task attempt {}",
+        taskAttemptCount, filesPerTaskAttempt);
+
+    setJobStageConfig(createStageConfigForJob(JOB1, getDestDir()));
+
+    // set up the job.
+    new SetupJobStage(getJobStageConfig()).apply(false);
+
+    LOG.info("Creating manifest files for {}", taskAttemptCount);
+
+    executeTaskAttempts(taskAttemptCount, filesPerTaskAttempt);
+
+    LOG.info("Loading in the manifests");
+
+    // Load in the manifests
+    LoadManifestsStage stage = new LoadManifestsStage(
+        getJobStageConfig());
+
+    LoadManifestsStage.Result result = stage.apply(true);
+    LoadManifestsStage.SummaryInfo summary = result.getSummary();
+    List<TaskManifest> loadedManifests = result.getManifests();
+
+    Assertions.assertThat(summary.getManifestCount())
+        .describedAs("Manifest count of  %s", summary)
+        .isEqualTo(taskAttemptCount);
+    Assertions.assertThat(summary.getFileCount())
+        .describedAs("File count of  %s", summary)
+        .isEqualTo(taskAttemptCount * (long) filesPerTaskAttempt);
+    Assertions.assertThat(summary.getTotalFileSize())
+        .describedAs("File Size of  %s", summary)
+        .isEqualTo(getTotalDataSize());
+
+    // now that manifest list.
+    List<String> manifestTaskIds = loadedManifests.stream()
+        .map(TaskManifest::getTaskID)
+        .collect(Collectors.toList());
+    Assertions.assertThat(getTaskIds())
+        .describedAs("Task IDs of all tasks")
+        .containsExactlyInAnyOrderElementsOf(manifestTaskIds);
+
+    // now let's see about aggregating a large set of directories
+    Set<Path> createdDirectories = new CreateOutputDirectoriesStage(
+        getJobStageConfig())
+        .apply(loadedManifests)
+        .getCreatedDirectories();
+
+    // but after the merge process, only one per generated file output
+    // dir exists
+    Assertions.assertThat(createdDirectories)
+        .describedAs("Directories created")
+        .hasSize(filesPerTaskAttempt);
+
+    // and skipping the rename stage (which is going to fail),
+    // go straight to cleanup
+    new CleanupJobStage(getJobStageConfig()).apply(
+        new CleanupJobStage.Arguments("", true, true, false));
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java
new file mode 100644
index 0000000000000..3037bf33ad62f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java
@@ -0,0 +1,1819 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.assertj.core.api.Assertions;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSupport;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.listChildren;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.COMMITTER_FACTORY_CLASS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_FACTORY;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SPARK_WRITE_UUID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_BYTES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_FILES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_COMPLETED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_LOAD_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_ABORT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createJobSummaryFilename;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.randomJobId;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.validateSuccessFile;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * This is a contract test for the commit protocol on a target filesystem.
+ * It is subclassed in the ABFS integration tests and elsewhere.
+ * Derived from the S3A protocol suite, which was itself based off
+ * the test suite {@code TestFileOutputCommitter}.
+ *
+ * Some of the methods trigger java warnings about unchecked casts;
+ * it's impossible to remove them, so the checks are suppressed.
+ */
+@SuppressWarnings("unchecked")
+public class TestManifestCommitProtocol
+    extends AbstractManifestCommitterTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestManifestCommitProtocol.class);
+
+  private static final String SUB_DIR = "SUB_DIR";
+
+  /**
+   * Part of the name of the output of task attempt 0.
+   */
+  protected static final String PART_00000 = "part-m-00000";
+
+  private static final Text KEY_1 = new Text("key1");
+
+  private static final Text KEY_2 = new Text("key2");
+
+  private static final Text VAL_1 = new Text("val1");
+
+  private static final Text VAL_2 = new Text("val2");
+
+  /**
+   * Snapshot of stats, which will be collected from
+   * committers.
+   */
+  private static final IOStatisticsSnapshot IOSTATISTICS =
+      IOStatisticsSupport.snapshotIOStatistics();
+
+  /**
+   * Job ID for jobs.
+   */
+  private final String jobId;
+
+  /**
+   * A random task attempt id for testing.
+   */
+  private final String attempt0;
+
+  /**
+   *  Attempt 0's task attempt ID.
+   */
+  private final TaskAttemptID taskAttempt0;
+
+  /**
+   * TA 1.
+   */
+  private final TaskAttemptID taskAttempt1;
+
+  /**
+   * Attempt 1 string value.
+   */
+  private final String attempt1;
+
+
+  /** A job to abort in test case teardown. */
+  private final List<JobData> abortInTeardown = new ArrayList<>(1);
+
+  /**
+   * Output directory.
+   * This is the directory into which output goes;
+   * all the job files go in _temporary underneath.
+   */
+  private Path outputDir;
+
+  /**
+   * Committer factory which calls back into
+   * {@link #createCommitter(Path, TaskAttemptContext)}.
+   */
+  private final LocalCommitterFactory
+      localCommitterFactory = new LocalCommitterFactory();
+
+  /**
+   * Clean up the output dir. No-op if
+   * {@link #outputDir} is null.
+   * @throws IOException failure to delete
+   */
+  private void cleanupOutputDir() throws IOException {
+    if (outputDir != null) {
+      getFileSystem().delete(outputDir, true);
+    }
+  }
+
+  /**
+   * Constructor.
+   */
+  public TestManifestCommitProtocol() {
+    ManifestCommitterTestSupport.JobAndTaskIDsForTests taskIDs
+        = new ManifestCommitterTestSupport.JobAndTaskIDsForTests(2, 2);
+    jobId = taskIDs.getJobId();
+    attempt0 = taskIDs.getTaskAttempt(0, 0);
+    taskAttempt0 = taskIDs.getTaskAttemptIdType(0, 0);
+    attempt1 = taskIDs.getTaskAttempt(0, 1);
+    taskAttempt1 = taskIDs.getTaskAttemptIdType(0, 1);
+  }
+
+  /**
+   * This must return the name of a suite which is unique to the test.
+   * @return a string which must be unique and a valid path.
+   */
+  protected String suitename() {
+    return "TestManifestCommitProtocolLocalFS";
+  }
+
+  /**
+   * Get the log; can be overridden for test case log.
+   * @return a log.
+   */
+  public Logger log() {
+    return LOG;
+  }
+
+  /**
+   * Overridden method returns the suitename as well as the method name,
+   * so if more than one committer test is run in parallel, paths are
+   * isolated.
+   * @return a name for a method, unique across the suites and test cases.
+   */
+  @Override
+  protected String getMethodName() {
+    return suitename() + "-" + super.getMethodName();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    outputDir = path(getMethodName());
+    cleanupOutputDir();
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    describe("teardown");
+    Thread.currentThread().setName("teardown");
+    for (JobData jobData : abortInTeardown) {
+      // stop the job
+      abortJobQuietly(jobData);
+      // and then get its statistics
+      IOSTATISTICS.aggregate(jobData.committer.getIOStatistics());
+    }
+    try {
+      cleanupOutputDir();
+    } catch (IOException e) {
+      log().info("Exception during cleanup", e);
+    }
+    super.teardown();
+  }
+
+  @AfterClass
+  public static void logAggregateIOStatistics() {
+    LOG.info("Final IOStatistics {}",
+        ioStatisticsToPrettyString(IOSTATISTICS));
+  }
+
+  /**
+   * Add the specified job to the current list of jobs to abort in teardown.
+   * @param jobData job data.
+   */
+  protected void abortInTeardown(JobData jobData) {
+    abortInTeardown.add(jobData);
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    bindCommitter(conf);
+    return conf;
+  }
+
+  /***
+   * Set job up to use the manifest committer.
+   * @param conf configuration to set up
+   */
+  protected void bindCommitter(Configuration conf) {
+    conf.set(COMMITTER_FACTORY_CLASS, MANIFEST_COMMITTER_FACTORY);
+  }
+
+  /**
+   * Create a committer for a task.
+   * @param context task context
+   * @return new committer
+   * @throws IOException failure
+   */
+  protected ManifestCommitter createCommitter(
+      TaskAttemptContext context) throws IOException {
+    return createCommitter(getOutputDir(), context);
+  }
+
+  /**
+   * Create a committer for a task and a given output path.
+   * @param outputPath path
+   * @param context task context
+   * @return new committer
+   * @throws IOException failure
+   */
+  protected ManifestCommitter createCommitter(
+      Path outputPath,
+      TaskAttemptContext context) throws IOException {
+    return new ManifestCommitter(outputPath, context);
+  }
+
+  protected Path getOutputDir() {
+    return outputDir;
+  }
+
+  protected String getJobId() {
+    return jobId;
+  }
+
+  protected String getAttempt0() {
+    return attempt0;
+  }
+
+  protected TaskAttemptID getTaskAttempt0() {
+    return taskAttempt0;
+  }
+
+  protected String getAttempt1() {
+    return attempt1;
+  }
+
+  protected TaskAttemptID getTaskAttempt1() {
+    return taskAttempt1;
+  }
+
+  /**
+   * Functional interface for creating committers, designed to allow
+   * different factories to be used to create different failure modes.
+   */
+  @FunctionalInterface
+  public interface CommitterFactory {
+
+    /**
+     * Create a committer for a task.
+     * @param context task context
+     * @return new committer
+     * @throws IOException failure
+     */
+    ManifestCommitter createCommitter(
+        TaskAttemptContext context) throws IOException;
+  }
+
+  /**
+   * The normal committer creation factory, uses the abstract methods
+   * in the class.
+   */
+  protected class LocalCommitterFactory implements CommitterFactory {
+
+    @Override
+    public ManifestCommitter createCommitter(TaskAttemptContext context)
+        throws IOException {
+      return TestManifestCommitProtocol.this
+          .createCommitter(context);
+    }
+  }
+
+  /**
+   * Assert that for a given output, the job context returns a manifest
+   * committer factory. This is what FileOutputFormat does internally,
+   * and is needed to make sure that the relevant settings are being passed
+   * around.
+   * @param context job/task context
+   * @param output destination path.
+   */
+  protected void assertCommitterFactoryIsManifestCommitter(
+      JobContext context, Path output) {
+
+    final Configuration conf = context.getConfiguration();
+    // check one: committer
+    assertConfigurationUsesManifestCommitter(conf);
+    final String factoryName = conf.get(COMMITTER_FACTORY_CLASS, "");
+    final PathOutputCommitterFactory factory
+        = PathOutputCommitterFactory.getCommitterFactory(
+        output,
+        conf);
+    Assertions.assertThat(factory)
+        .describedAs("Committer for output path %s"
+                + " and factory name \"%s\"",
+            output, factoryName)
+        .isInstanceOf(ManifestCommitterFactory.class);
+  }
+
+  /**
+   * This is to debug situations where the test committer factory
+   * on tasks was binding to FileOutputCommitter even when
+   * tests were overriding it.
+   * @param conf configuration to probe.
+   */
+  private void assertConfigurationUsesManifestCommitter(
+      Configuration conf) {
+    final String factoryName = conf.get(COMMITTER_FACTORY_CLASS, null);
+    Assertions.assertThat(factoryName)
+        .describedAs("Value of %s", COMMITTER_FACTORY_CLASS)
+        .isEqualTo(MANIFEST_COMMITTER_FACTORY);
+  }
+
+  /**
+   * Write some text out.
+   * @param context task
+   * @throws IOException IO failure
+   * @throws InterruptedException write interrupted
+   * @return the path written to
+   */
+  protected Path writeTextOutput(TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    describe("write output");
+    try (DurationInfo d = new DurationInfo(LOG,
+        "Writing Text output for task %s", context.getTaskAttemptID())) {
+      TextOutputForTests.LoggingLineRecordWriter<Writable, Object> writer
+          = new TextOutputForTests<Writable, Object>().getRecordWriter(context);
+      writeOutput(writer, context);
+      return writer.getDest();
+    }
+  }
+
+  /**
+   * Write the standard output.
+   * @param writer record writer
+   * @param context task context
+   * @throws IOException IO failure
+   * @throws InterruptedException write interrupted
+   */
+  private void writeOutput(
+      RecordWriter<Writable, Object> writer,
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    NullWritable nullWritable = NullWritable.get();
+    try (ManifestCommitterTestSupport.CloseWriter<Writable, Object> cw =
+             new ManifestCommitterTestSupport.CloseWriter<>(writer, context)) {
+      writer.write(KEY_1, VAL_1);
+      writer.write(null, nullWritable);
+      writer.write(null, VAL_1);
+      writer.write(nullWritable, VAL_2);
+      writer.write(KEY_2, nullWritable);
+      writer.write(KEY_1, null);
+      writer.write(null, null);
+      writer.write(KEY_2, VAL_2);
+      writer.close(context);
+    }
+  }
+
+  /**
+   * Write the output of a map.
+   * @param writer record writer
+   * @param context task context
+   * @throws IOException IO failure
+   * @throws InterruptedException write interrupted
+   */
+  private void writeMapFileOutput(RecordWriter<WritableComparable<?>, Writable> writer,
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    describe("\nWrite map output");
+    try (DurationInfo d = new DurationInfo(LOG,
+        "Writing Text output for task %s", context.getTaskAttemptID());
+         ManifestCommitterTestSupport.CloseWriter<WritableComparable<?>, Writable> cw =
+             new ManifestCommitterTestSupport.CloseWriter<>(writer, context)) {
+      for (int i = 0; i < 10; ++i) {
+        Text val = ((i & 1) == 1) ? VAL_1 : VAL_2;
+        writer.write(new LongWritable(i), val);
+      }
+      LOG.debug("Closing writer {}", writer);
+      writer.close(context);
+    }
+  }
+
+  /**
+   * Details on a job for use in {@code startJob} and elsewhere.
+   */
+  protected static final class JobData {
+
+    private final Job job;
+
+    private final JobContext jContext;
+
+    private final TaskAttemptContext tContext;
+
+    private final ManifestCommitter committer;
+
+    private final Configuration conf;
+
+    private Path writtenTextPath; // null if not written to
+
+    public JobData(Job job,
+        JobContext jContext,
+        TaskAttemptContext tContext,
+        ManifestCommitter committer) {
+      this.job = job;
+      this.jContext = jContext;
+      this.tContext = tContext;
+      this.committer = committer;
+      conf = job.getConfiguration();
+    }
+
+    public String jobId() {
+      return committer.getJobUniqueId();
+    }
+  }
+
+  /**
+   * Create a new job. Sets the task attempt ID,
+   * and output dir; asks for a success marker.
+   * @return the new job
+   * @throws IOException failure
+   */
+  public Job newJob() throws IOException {
+    return newJob(outputDir, getConfiguration(), attempt0);
+  }
+
+  /**
+   * Create a new job. Sets the task attempt ID,
+   * and output dir; asks for a success marker.
+   * Committer factory is set to manifest factory, so is independent
+   * of FS schema.
+   * @param dir dest dir
+   * @param configuration config to get the job from
+   * @param taskAttemptId task attempt
+   * @return the new job
+   * @throws IOException failure
+   */
+  private Job newJob(Path dir, Configuration configuration,
+      String taskAttemptId) throws IOException {
+    Job job = Job.getInstance(configuration);
+    Configuration conf = job.getConfiguration();
+    conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptId);
+    enableManifestCommitter(conf);
+    FileOutputFormat.setOutputPath(job, dir);
+    return job;
+  }
+
+  /**
+   * Start a job with a committer; optionally write the test data.
+   * Always register the job to be aborted (quietly) in teardown.
+   * This is, from an "OO-purity perspective" the wrong kind of method to
+   * do: it's setting things up, mixing functionality, registering for teardown.
+   * Its aim is simple though: a common body of code for starting work
+   * in test cases.
+   * @param writeText should the text be written?
+   * @return the job data 4-tuple
+   * @throws IOException IO problems
+   * @throws InterruptedException interruption during write
+   */
+  protected JobData startJob(boolean writeText)
+      throws IOException, InterruptedException {
+    return startJob(localCommitterFactory, writeText);
+  }
+
+  /**
+   * Start a job with a committer; optionally write the test data.
+   * Always register the job to be aborted (quietly) in teardown.
+   * This is, from an "OO-purity perspective" the wrong kind of method to
+   * do: it's setting things up, mixing functionality, registering for teardown.
+   * Its aim is simple though: a common body of code for starting work
+   * in test cases.
+   * @param factory the committer factory to use
+   * @param writeText should the text be written?
+   * @return the job data 4-tuple
+   * @throws IOException IO problems
+   * @throws InterruptedException interruption during write
+   */
+  protected JobData startJob(CommitterFactory factory, boolean writeText)
+      throws IOException, InterruptedException {
+    Job job = newJob();
+    Configuration conf = job.getConfiguration();
+    assertConfigurationUsesManifestCommitter(conf);
+    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+    conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+    JobContext jContext = new JobContextImpl(conf, taskAttempt0.getJobID());
+    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf,
+        taskAttempt0);
+    ManifestCommitter committer = factory.createCommitter(tContext);
+
+    // setup
+    JobData jobData = new JobData(job, jContext, tContext, committer);
+    setupJob(jobData);
+    abortInTeardown(jobData);
+
+    if (writeText) {
+      // write output
+      jobData.writtenTextPath = writeTextOutput(tContext);
+    }
+    return jobData;
+  }
+
+  /**
+   * Set up the job and task.
+   * @param jobData job data
+   * @throws IOException problems
+   */
+  protected void setupJob(JobData jobData) throws IOException {
+    ManifestCommitter committer = jobData.committer;
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    describe("\nsetup job");
+    try (DurationInfo d = new DurationInfo(LOG,
+        "setup job %s", jContext.getJobID())) {
+      committer.setupJob(jContext);
+    }
+    setupCommitter(committer, tContext);
+    describe("setup complete");
+  }
+
+  private void setupCommitter(
+      final ManifestCommitter committer,
+      final TaskAttemptContext tContext) throws IOException {
+    try (DurationInfo d = new DurationInfo(LOG,
+        "setup task %s", tContext.getTaskAttemptID())) {
+      committer.setupTask(tContext);
+    }
+  }
+
+  /**
+   * Abort a job quietly.
+   * @param jobData job info
+   */
+  protected void abortJobQuietly(JobData jobData) {
+    abortJobQuietly(jobData.committer, jobData.jContext, jobData.tContext);
+  }
+
+  /**
+   * Abort a job quietly: first task, then job.
+   * @param committer committer
+   * @param jContext job context
+   * @param tContext task context
+   */
+  protected void abortJobQuietly(ManifestCommitter committer,
+      JobContext jContext,
+      TaskAttemptContext tContext) {
+    describe("\naborting task");
+    try {
+      committer.abortTask(tContext);
+    } catch (Exception e) {
+      log().warn("Exception aborting task:", e);
+    }
+    describe("\naborting job");
+    try {
+      committer.abortJob(jContext, JobStatus.State.KILLED);
+    } catch (Exception e) {
+      log().warn("Exception aborting job", e);
+    }
+  }
+
+  /**
+   * Commit the task and then the job.
+   * @param committer committer
+   * @param jContext job context
+   * @param tContext task context
+   * @throws IOException problems
+   */
+  protected void commitTaskAndJob(ManifestCommitter committer,
+      JobContext jContext,
+      TaskAttemptContext tContext) throws IOException {
+    try (DurationInfo d = new DurationInfo(LOG,
+        "committing Job %s", jContext.getJobID())) {
+      describe("\ncommitting task");
+      committer.commitTask(tContext);
+      describe("\ncommitting job");
+      committer.commitJob(jContext);
+      describe("commit complete\n");
+    }
+  }
+
+  /**
+   * Execute work as part of a test, after creating the job.
+   * After the execution, {@link #abortJobQuietly(JobData)} is
+   * called for abort/cleanup.
+   * @param name name of work (for logging)
+   * @param action action to execute
+   * @throws Exception failure
+   */
+  protected void executeWork(String name, ActionToTest action)
+      throws Exception {
+    executeWork(name, startJob(false), action);
+  }
+
+  /**
+   * Execute work as part of a test, against the created job.
+   * After the execution, {@link #abortJobQuietly(JobData)} is
+   * called for abort/cleanup.
+   * @param name name of work (for logging)
+   * @param jobData job info
+   * @param action action to execute
+   * @throws Exception failure
+   */
+  public void executeWork(String name,
+      JobData jobData,
+      ActionToTest action) throws Exception {
+    try (DurationInfo d = new DurationInfo(LOG, "Executing %s", name)) {
+      action.exec(jobData.job,
+          jobData.jContext,
+          jobData.tContext,
+          jobData.committer);
+    } finally {
+      abortJobQuietly(jobData);
+    }
+  }
+
+  /**
+   * Load a manifest from the test FS.
+   * @param path path
+   * @return the manifest
+   * @throws IOException failure to load
+   */
+  TaskManifest loadManifest(Path path) throws IOException {
+    return TaskManifest.load(getFileSystem(), path);
+  }
+
+  /**
+   * Verify that recovery doesn't work for these committers.
+   */
+  @Test
+  @SuppressWarnings("deprecation")
+  public void testRecoveryAndCleanup() throws Exception {
+    describe("Test (unsupported) task recovery.");
+    JobData jobData = startJob(true);
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+
+    Assertions.assertThat(committer.getWorkPath())
+        .as("null workPath in committer " + committer)
+        .isNotNull();
+    Assertions.assertThat(committer.getOutputPath())
+        .as("null outputPath in committer " + committer)
+        .isNotNull();
+
+    // Commit the task.
+    commitTask(committer, tContext);
+
+    // load and log the manifest
+    final TaskManifest manifest = loadManifest(
+        committer.getTaskManifestPath(tContext));
+    LOG.info("Manifest {}", manifest);
+
+    Configuration conf2 = jobData.job.getConfiguration();
+    conf2.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+    conf2.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 2);
+    JobContext jContext2 = new JobContextImpl(conf2, taskAttempt0.getJobID());
+    TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2,
+        taskAttempt0);
+    ManifestCommitter committer2 = createCommitter(tContext2);
+    committer2.setupJob(tContext2);
+
+    Assertions.assertThat(committer2.isRecoverySupported())
+        .as("recoverySupported in " + committer2)
+        .isFalse();
+    intercept(IOException.class, "recover",
+        () -> committer2.recoverTask(tContext2));
+
+    // at this point, task attempt 0 has failed to recover
+    // it should be abortable though. This will be a no-op as it already
+    // committed
+    describe("aborting task attempt 2; expect nothing to clean up");
+    committer2.abortTask(tContext2);
+    describe("Aborting job 2; expect pending commits to be aborted");
+    committer2.abortJob(jContext2, JobStatus.State.KILLED);
+  }
+
+  /**
+   * Assert that the task attempt FS Doesn't have a task attempt
+   * directory.
+   * @param committer committer
+   * @param context task context
+   * @throws IOException IO failure.
+   */
+  protected void assertTaskAttemptPathDoesNotExist(
+      ManifestCommitter committer, TaskAttemptContext context)
+      throws IOException {
+    Path attemptPath = committer.getTaskAttemptPath(context);
+    ContractTestUtils.assertPathDoesNotExist(
+        attemptPath.getFileSystem(context.getConfiguration()),
+        "task attempt dir",
+        attemptPath);
+  }
+
+  protected void assertJobAttemptPathDoesNotExist(
+      ManifestCommitter committer, JobContext context)
+      throws IOException {
+    Path attemptPath = committer.getJobAttemptPath(context);
+    ContractTestUtils.assertPathDoesNotExist(
+        attemptPath.getFileSystem(context.getConfiguration()),
+        "job attempt dir",
+        attemptPath);
+  }
+
+  /**
+   * Verify the output of the directory.
+   * That includes the {@code part-m-00000-*}
+   * file existence and contents, as well as optionally, the success marker.
+   * @param dir directory to scan.
+   * @param expectSuccessMarker check the success marker?
+   * @param expectedJobId job ID, verified if non-empty and success data loaded
+   * @throws Exception failure.
+   * @return the success data
+   */
+  private ManifestSuccessData validateContent(Path dir,
+      boolean expectSuccessMarker,
+      String expectedJobId) throws Exception {
+    lsR(getFileSystem(), dir, true);
+    ManifestSuccessData successData;
+    if (expectSuccessMarker) {
+      successData = verifySuccessMarker(dir, expectedJobId);
+    } else {
+      successData = null;
+    }
+    Path expectedFile = getPart0000(dir);
+    log().debug("Validating content in {}", expectedFile);
+    StringBuilder expectedOutput = new StringBuilder();
+    expectedOutput.append(KEY_1).append('\t').append(VAL_1).append("\n");
+    expectedOutput.append(VAL_1).append("\n");
+    expectedOutput.append(VAL_2).append("\n");
+    expectedOutput.append(KEY_2).append("\n");
+    expectedOutput.append(KEY_1).append("\n");
+    expectedOutput.append(KEY_2).append('\t').append(VAL_2).append("\n");
+    String output = readFile(expectedFile);
+    Assertions.assertThat(output)
+        .describedAs("Content of %s", expectedFile)
+        .isEqualTo(expectedOutput.toString());
+    return successData;
+  }
+
+  /**
+   * Identify any path under the directory which begins with the
+   * {@code "part-m-00000"} sequence. There's some compensation for
+   * eventual consistency here.
+   * @param dir directory to scan
+   * @return the full path
+   * @throws FileNotFoundException the path is missing.
+   * @throws Exception failure.
+   */
+  protected Path getPart0000(final Path dir) throws Exception {
+    final FileSystem fs = dir.getFileSystem(getConfiguration());
+    FileStatus[] statuses = fs.listStatus(dir,
+        path -> path.getName().startsWith(PART_00000));
+    if (statuses.length != 1) {
+      // fail, with a listing of the parent dir
+      ContractTestUtils.assertPathExists(fs, "Output file",
+          new Path(dir, PART_00000));
+    }
+    return statuses[0].getPath();
+  }
+
+  /**
+   * Look for the partFile subdir of the output dir
+   * and the ma and data entries.
+   * @param fs filesystem
+   * @param dir output dir
+   * @throws Exception failure.
+   */
+  private void validateMapFileOutputContent(
+      FileSystem fs, Path dir) throws Exception {
+    // map output is a directory with index and data files
+    assertPathExists("Map output", dir);
+    Path expectedMapDir = getPart0000(dir);
+    assertPathExists("Map output", expectedMapDir);
+    assertIsDirectory(expectedMapDir);
+    FileStatus[] files = fs.listStatus(expectedMapDir);
+    Assertions.assertThat(files)
+        .as("No files found in " + expectedMapDir)
+        .isNotEmpty();
+    assertPathExists("index file in " + expectedMapDir,
+        new Path(expectedMapDir, MapFile.INDEX_FILE_NAME));
+    assertPathExists("data file in " + expectedMapDir,
+        new Path(expectedMapDir, MapFile.DATA_FILE_NAME));
+  }
+
+  /**
+   * Full test of the expected lifecycle: start job, task, write, commit task,
+   * commit job.
+   * @throws Exception on a failure
+   */
+  @Test
+  public void testCommitLifecycle() throws Exception {
+    describe("Full test of the expected lifecycle:\n" +
+        " start job, task, write, commit task, commit job.\n" +
+        "Verify:\n" +
+        "* no files are visible after task commit\n" +
+        "* the expected file is visible after job commit\n");
+    JobData jobData = startJob(false);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+    assertCommitterFactoryIsManifestCommitter(tContext,
+        tContext.getWorkingDirectory());
+    validateTaskAttemptWorkingDirectory(committer, tContext);
+
+    // write output
+    describe("1. Writing output");
+    final Path textOutputPath = writeTextOutput(tContext);
+    describe("Output written to %s", textOutputPath);
+
+    describe("2. Committing task");
+    Assertions.assertThat(committer.needsTaskCommit(tContext))
+        .as("No files to commit were found by " + committer)
+        .isTrue();
+    commitTask(committer, tContext);
+    final TaskManifest taskManifest = requireNonNull(
+        committer.getTaskAttemptCommittedManifest(), "committerTaskManifest");
+    final String manifestJSON = taskManifest.toJson();
+    LOG.info("Task manifest {}", manifestJSON);
+    int filesCreated = 1;
+    Assertions.assertThat(taskManifest.getFilesToCommit())
+        .describedAs("Files to commit in task manifest %s", manifestJSON)
+        .hasSize(filesCreated);
+    Assertions.assertThat(taskManifest.getDestDirectories())
+        .describedAs("Directories to create in task manifest %s",
+            manifestJSON)
+        .isEmpty();
+
+    // this is only task commit; there MUST be no part- files in the dest dir
+    try {
+      RemoteIterators.foreach(getFileSystem().listFiles(outputDir, false),
+          (status) ->
+              Assertions.assertThat(status.getPath().toString())
+                  .as("task committed file to dest :" + status)
+                  .contains("part"));
+    } catch (FileNotFoundException ignored) {
+      log().info("Outdir {} is not created by task commit phase ",
+          outputDir);
+    }
+
+    describe("3. Committing job");
+
+    commitJob(committer, jContext);
+
+    // validate output
+    describe("4. Validating content");
+    String jobUniqueId = jobData.jobId();
+    ManifestSuccessData successData = validateContent(outputDir,
+        true,
+        jobUniqueId);
+    // look in the SUMMARY
+    Assertions.assertThat(successData.getDiagnostics())
+        .describedAs("Stage entry in SUCCESS")
+        .containsEntry(STAGE, OP_STAGE_JOB_COMMIT);
+    IOStatisticsSnapshot jobStats = successData.getIOStatistics();
+    // manifest
+    verifyStatisticCounterValue(jobStats,
+        OP_LOAD_MANIFEST, 1);
+    FileStatus st = getFileSystem().getFileStatus(getPart0000(outputDir));
+    verifyStatisticCounterValue(jobStats,
+        COMMITTER_FILES_COMMITTED_COUNT, filesCreated);
+    verifyStatisticCounterValue(jobStats,
+        COMMITTER_BYTES_COMMITTED_COUNT, st.getLen());
+
+    // now load and examine the job report.
+    // this MUST contain all the stats of the summary, plus timings on
+    // job commit itself
+
+    ManifestSuccessData report = loadReport(jobUniqueId, true);
+    Map<String, String> diag = report.getDiagnostics();
+    Assertions.assertThat(diag)
+        .describedAs("Stage entry in report")
+        .containsEntry(STAGE, OP_STAGE_JOB_COMMIT);
+    IOStatisticsSnapshot reportStats = report.getIOStatistics();
+    verifyStatisticCounterValue(reportStats,
+        OP_LOAD_MANIFEST, 1);
+    verifyStatisticCounterValue(reportStats,
+        OP_STAGE_JOB_COMMIT, 1);
+    verifyStatisticCounterValue(reportStats,
+        COMMITTER_FILES_COMMITTED_COUNT, filesCreated);
+    verifyStatisticCounterValue(reportStats,
+        COMMITTER_BYTES_COMMITTED_COUNT, st.getLen());
+
+  }
+
+  /**
+   * Load a summary from the report dir.
+   * @param jobUniqueId job ID
+   * @param expectSuccess is the job expected to have succeeded.
+   * @throws IOException failure to load
+   * @return the report
+   */
+  private ManifestSuccessData loadReport(String jobUniqueId,
+      boolean expectSuccess) throws IOException {
+    File file = new File(getReportDir(),
+        createJobSummaryFilename(jobUniqueId));
+    ContractTestUtils.assertIsFile(FileSystem.getLocal(getConfiguration()),
+        new Path(file.toURI()));
+    ManifestSuccessData report = ManifestSuccessData.serializer().load(file);
+    LOG.info("Report for job {}:\n{}", jobUniqueId, report.toJson());
+    Assertions.assertThat(report.getSuccess())
+        .describedAs("success flag in report")
+        .isEqualTo(expectSuccess);
+    return report;
+  }
+
+  /**
+   * Repeated commit call after job commit.
+   */
+  @Test
+  public void testCommitterWithDuplicatedCommit() throws Exception {
+    describe("Call a task then job commit twice;" +
+        "expect the second task commit to fail.");
+    JobData jobData = startJob(true);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+
+    // do commit
+    describe("committing task");
+    committer.commitTask(tContext);
+
+    // repeated commit while TA dir exists fine/idempotent
+    committer.commitTask(tContext);
+
+    describe("committing job");
+    committer.commitJob(jContext);
+    describe("commit complete\n");
+
+    describe("cleanup");
+    committer.cleanupJob(jContext);
+    // validate output
+    validateContent(outputDir, shouldExpectSuccessMarker(),
+        committer.getJobUniqueId());
+
+    // commit task to fail on retry as task attempt dir doesn't exist
+    describe("Attempting commit of the same task after job commit -expecting failure");
+    expectFNFEonTaskCommit(committer, tContext);
+  }
+
+  /**
+   * HADOOP-17258. If a second task attempt is committed, it
+   * must succeed, and the output of the first TA, even if already
+   * committed, MUST NOT be visible in the final output.
+   * <p></p>
+   * What's important is not just that only one TA must succeed,
+   * but it must be the last one executed.
+   */
+  @Test
+  public void testTwoTaskAttemptsCommit() throws Exception {
+    describe("Commit two task attempts;" +
+        " expect the second attempt to succeed.");
+    JobData jobData = startJob(false);
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+    // do commit
+    describe("\ncommitting task");
+    // write output for TA 1,
+    Path outputTA1 = writeTextOutput(tContext);
+
+    // speculatively execute committer 2.
+
+    // jobconf with a different base to its parts.
+    Configuration conf2 = jobData.conf;
+    conf2.set("mapreduce.output.basename", "attempt2");
+    String attempt2 = "attempt_" + jobId + "_m_000000_1";
+    TaskAttemptID ta2 = TaskAttemptID.forName(attempt2);
+    TaskAttemptContext tContext2 = new TaskAttemptContextImpl(
+        conf2, ta2);
+
+    ManifestCommitter committer2 = localCommitterFactory
+        .createCommitter(tContext2);
+    setupCommitter(committer2, tContext2);
+
+    // verify working dirs are different
+    Assertions.assertThat(committer.getWorkPath())
+        .describedAs("Working dir of %s", committer)
+        .isNotEqualTo(committer2.getWorkPath());
+
+    // write output for TA 2,
+    Path outputTA2 = writeTextOutput(tContext2);
+
+    // verify the names are different.
+    String name1 = outputTA1.getName();
+    String name2 = outputTA2.getName();
+    Assertions.assertThat(name1)
+        .describedAs("name of task attempt output %s", outputTA1)
+        .isNotEqualTo(name2);
+
+    // commit task 1
+    committer.commitTask(tContext);
+
+    // then pretend that task1 didn't respond, so
+    // commit task 2
+    committer2.commitTask(tContext2);
+
+    // and the job
+    committer2.commitJob(tContext);
+
+    // validate output
+    FileSystem fs = getFileSystem();
+    ManifestSuccessData successData = validateSuccessFile(fs, outputDir,
+        1,
+        "");
+    Assertions.assertThat(successData.getFilenames())
+        .describedAs("Files committed")
+        .hasSize(1);
+
+    assertPathExists("attempt2 output", new Path(outputDir, name2));
+    assertPathDoesNotExist("attempt1 output", new Path(outputDir, name1));
+
+  }
+
+  protected boolean shouldExpectSuccessMarker() {
+    return true;
+  }
+
+  /**
+   * Simulate a failure on the first job commit; expect the
+   * second to succeed.
+   */
+  /*@Test
+  public void testCommitterWithFailure() throws Exception {
+    describe("Fail the first job commit then retry");
+    JobData jobData = startJob(new FailingCommitterFactory(), true);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+
+    // do commit
+    committer.commitTask(tContext);
+
+    // now fail job
+    expectSimulatedFailureOnJobCommit(jContext, committer);
+
+    commitJob(committer, jContext);
+
+    // but the data got there, due to the order of operations.
+    validateContent(outDir, shouldExpectSuccessMarker(),
+        committer.getUUID());
+    expectJobCommitToFail(jContext, committer);
+  }
+*/
+
+  /**
+   * Override point: the failure expected on the attempt to commit a failed
+   * job.
+   * @param jContext job context
+   * @param committer committer
+   * @throws Exception any unexpected failure.
+   */
+  protected void expectJobCommitToFail(JobContext jContext,
+      ManifestCommitter committer) throws Exception {
+    // next attempt will fail as there is no longer a directory to commit
+    expectJobCommitFailure(jContext, committer,
+        FileNotFoundException.class);
+  }
+
+  /**
+   * Expect a job commit operation to fail with a specific exception.
+   * @param jContext job context
+   * @param committer committer
+   * @param clazz class of exception
+   * @return the caught exception
+   * @throws Exception any unexpected failure.
+   */
+  protected static <E extends IOException> E expectJobCommitFailure(
+      JobContext jContext,
+      ManifestCommitter committer,
+      Class<E> clazz)
+      throws Exception {
+
+    return intercept(clazz,
+        () -> {
+          committer.commitJob(jContext);
+          return committer.toString();
+        });
+  }
+
+  protected static void expectFNFEonTaskCommit(
+      ManifestCommitter committer,
+      TaskAttemptContext tContext) throws Exception {
+    intercept(FileNotFoundException.class,
+        () -> {
+          committer.commitTask(tContext);
+          return committer.toString();
+        });
+  }
+
+  /**
+   * Commit a task with no output.
+   * Dest dir should exist.
+   */
+  @Test
+  public void testCommitterWithNoOutputs() throws Exception {
+    describe("Have a task and job with no outputs: expect success");
+    JobData jobData = startJob(localCommitterFactory, false);
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+
+    // do commit
+    committer.commitTask(tContext);
+    Path attemptPath = committer.getTaskAttemptPath(tContext);
+    ContractTestUtils.assertPathExists(
+        attemptPath.getFileSystem(tContext.getConfiguration()),
+        "task attempt dir",
+        attemptPath);
+  }
+
+
+  @Test
+  public void testMapFileOutputCommitter() throws Exception {
+    describe("Test that the committer generates map output into a directory\n" +
+        "starting with the prefix part-");
+    JobData jobData = startJob(false);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+    Configuration conf = jobData.conf;
+
+    // write output
+    writeMapFileOutput(new MapFileOutputFormat()
+            .getRecordWriter(tContext), tContext);
+
+    // do commit
+    commitTaskAndJob(committer, jContext, tContext);
+    FileSystem fs = getFileSystem();
+
+    lsR(fs, outputDir, true);
+    String ls = ls(outputDir);
+    describe("\nvalidating");
+
+    // validate output
+    verifySuccessMarker(outputDir, committer.getJobUniqueId());
+
+    describe("validate output of %s", outputDir);
+    validateMapFileOutputContent(fs, outputDir);
+
+    // Ensure getReaders call works and also ignores
+    // hidden filenames (_ or . prefixes)
+    describe("listing");
+    FileStatus[] filtered = fs.listStatus(outputDir, HIDDEN_FILE_FILTER);
+    Assertions.assertThat(filtered)
+        .describedAs("listed children under %s", ls)
+        .hasSize(1);
+    FileStatus fileStatus = filtered[0];
+    Assertions.assertThat(fileStatus.getPath().getName())
+        .as("Not the part file: " + fileStatus)
+        .startsWith(PART_00000);
+
+    describe("getReaders()");
+    Assertions.assertThat(getReaders(fs, outputDir, conf))
+        .describedAs("getReaders() MapFile.Reader entries with shared FS %s %s", outputDir, ls)
+        .hasSize(1);
+
+    describe("getReaders(new FS)");
+    FileSystem fs2 = FileSystem.get(outputDir.toUri(), conf);
+    Assertions.assertThat(getReaders(fs2, outputDir, conf))
+        .describedAs("getReaders(new FS) %s %s", outputDir, ls)
+        .hasSize(1);
+
+    describe("MapFileOutputFormat.getReaders");
+    Assertions.assertThat(MapFileOutputFormat.getReaders(outputDir, conf))
+        .describedAs("MapFileOutputFormat.getReaders(%s) %s", outputDir, ls)
+        .hasSize(1);
+
+  }
+
+  /** Open the output generated by this format. */
+  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+  private static MapFile.Reader[] getReaders(FileSystem fs,
+      Path dir,
+      Configuration conf) throws IOException {
+    Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, HIDDEN_FILE_FILTER));
+
+    // sort names, so that hash partitioning works
+    Arrays.sort(names);
+
+    MapFile.Reader[] parts = new MapFile.Reader[names.length];
+    for (int i = 0; i < names.length; i++) {
+      parts[i] = new MapFile.Reader(names[i], conf);
+    }
+    return parts;
+  }
+
+  public static final PathFilter HIDDEN_FILE_FILTER = (path) ->
+      !path.getName().startsWith("_") && !path.getName().startsWith(".");
+
+  /**
+   * A functional interface which an action to test must implement.
+   */
+  @FunctionalInterface
+  public interface ActionToTest {
+
+    void exec(Job job, JobContext jContext, TaskAttemptContext tContext,
+        ManifestCommitter committer) throws Exception;
+  }
+
+  @Test
+  public void testAbortTaskNoWorkDone() throws Exception {
+    executeWork("abort task no work",
+        (job, jContext, tContext, committer) ->
+            committer.abortTask(tContext));
+  }
+
+  @Test
+  public void testAbortJobNoWorkDone() throws Exception {
+    executeWork("abort task no work",
+        (job, jContext, tContext, committer) ->
+            committer.abortJob(jContext, JobStatus.State.RUNNING));
+  }
+
+  @Test
+  public void testCommitJobButNotTask() throws Exception {
+    executeWork("commit a job while a task's work is pending, " +
+            "expect task writes to be cancelled.",
+        (job, jContext, tContext, committer) -> {
+          // step 1: write the text
+          writeTextOutput(tContext);
+          // step 2: commit the job
+          createCommitter(tContext).commitJob(tContext);
+          // verify that no output can be observed
+          assertPart0000DoesNotExist(outputDir);
+        }
+    );
+  }
+
+  @Test
+  public void testAbortTaskThenJob() throws Exception {
+    JobData jobData = startJob(true);
+    ManifestCommitter committer = jobData.committer;
+
+    // do abort
+    committer.abortTask(jobData.tContext);
+
+    intercept(FileNotFoundException.class, "",
+        () -> getPart0000(committer.getWorkPath()));
+
+    committer.abortJob(jobData.jContext, JobStatus.State.FAILED);
+    assertJobAbortCleanedUp(jobData);
+
+  }
+
+  /**
+   * Extension point: assert that the job was all cleaned up after an abort.
+   * Base assertions
+   * <ul>
+   *   <li>Output dir is absent or, if present, empty</li>
+   * </ul>
+   * @param jobData job data
+   * @throws Exception failure
+   */
+  public void assertJobAbortCleanedUp(JobData jobData) throws Exception {
+    FileSystem fs = getFileSystem();
+    try {
+      FileStatus[] children = listChildren(fs, outputDir);
+      if (children.length != 0) {
+        lsR(fs, outputDir, true);
+      }
+      Assertions.assertThat(children)
+          .as("Output directory not empty " + ls(outputDir))
+          .containsExactly(new FileStatus[0]);
+    } catch (FileNotFoundException e) {
+      // this is a valid state; it means the dest dir doesn't exist yet.
+    }
+
+  }
+
+  @Test
+  public void testFailAbort() throws Exception {
+    describe("Abort the task, then job (failed), abort the job again");
+    JobData jobData = startJob(true);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    ManifestCommitter committer = jobData.committer;
+
+    // do abort
+    committer.abortTask(tContext);
+
+    committer.getJobAttemptPath(jContext);
+    committer.getTaskAttemptPath(tContext);
+    assertPart0000DoesNotExist(outputDir);
+    assertSuccessMarkerDoesNotExist(outputDir);
+    describe("Aborting job into %s", outputDir);
+
+    committer.abortJob(jContext, JobStatus.State.FAILED);
+
+    assertTaskAttemptPathDoesNotExist(committer, tContext);
+    assertJobAttemptPathDoesNotExist(committer, jContext);
+
+    // verify a failure report
+    ManifestSuccessData report = loadReport(jobData.jobId(), false);
+    Map<String, String> diag = report.getDiagnostics();
+    Assertions.assertThat(diag)
+        .describedAs("Stage entry in report")
+        .containsEntry(STAGE, OP_STAGE_JOB_ABORT);
+    IOStatisticsSnapshot reportStats = report.getIOStatistics();
+    verifyStatisticCounterValue(reportStats,
+        OP_STAGE_JOB_ABORT, 1);
+
+    // try again; expect abort to be idempotent.
+    committer.abortJob(jContext, JobStatus.State.FAILED);
+
+  }
+
+  /**
+   * Assert that the given dir does not have the {@code _SUCCESS} marker.
+   * @param dir dir to scan
+   * @throws IOException IO Failure
+   */
+  protected void assertSuccessMarkerDoesNotExist(Path dir) throws IOException {
+    assertPathDoesNotExist("Success marker",
+        new Path(dir, SUCCESS_MARKER));
+  }
+
+  public void assertPart0000DoesNotExist(Path dir) throws Exception {
+    intercept(FileNotFoundException.class,
+        () -> getPart0000(dir));
+    assertPathDoesNotExist("expected output file", new Path(dir, PART_00000));
+  }
+
+  @Test
+  public void testAbortJobNotTask() throws Exception {
+    executeWork("abort task no work",
+        (job, jContext, tContext, committer) -> {
+          // write output
+          writeTextOutput(tContext);
+          committer.abortJob(jContext, JobStatus.State.RUNNING);
+          assertTaskAttemptPathDoesNotExist(
+              committer, tContext);
+          assertJobAttemptPathDoesNotExist(
+              committer, jContext);
+        });
+  }
+
+  /**
+   * This looks at what happens with concurrent commits.
+   * However, the failure condition it looks for (subdir under subdir)
+   * is the kind of failure you see on a rename-based commit.
+   *
+   * What it will not detect is the fact that both tasks will each commit
+   * to the destination directory. That is: whichever commits last wins.
+   *
+   * There's no way to stop this. Instead it is a requirement that the task
+   * commit operation is only executed when the committer is happy to
+   * commit only those tasks which it knows have succeeded, and abort those
+   * which have not.
+   * @throws Exception failure
+   */
+  @Test
+  public void testConcurrentCommitTaskWithSubDir() throws Exception {
+    Job job = newJob();
+    FileOutputFormat.setOutputPath(job, outputDir);
+    final Configuration conf = job.getConfiguration();
+
+    final JobContext jContext =
+        new JobContextImpl(conf, taskAttempt0.getJobID());
+    ManifestCommitter amCommitter = createCommitter(
+        new TaskAttemptContextImpl(conf, taskAttempt0));
+    amCommitter.setupJob(jContext);
+
+    final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
+    taCtx[0] = new TaskAttemptContextImpl(conf, taskAttempt0);
+    taCtx[1] = new TaskAttemptContextImpl(conf, taskAttempt1);
+
+    // IDE/checkstyle complain here about type casting but they
+    // are confused.
+    final TextOutputFormat<Writable, Object>[] tof =
+        new TextOutputForTests[2];
+
+    for (int i = 0; i < tof.length; i++) {
+      tof[i] = new TextOutputForTests<Writable, Object>() {
+        @Override
+        public Path getDefaultWorkFile(
+            TaskAttemptContext context,
+            String extension) throws IOException {
+          final ManifestCommitter foc = (ManifestCommitter)
+              getOutputCommitter(context);
+          return new Path(new Path(foc.getWorkPath(), SUB_DIR),
+              getUniqueFile(context, getOutputName(context), extension));
+        }
+      };
+    }
+
+    final ExecutorService executor = HadoopExecutors.newFixedThreadPool(2);
+    try {
+      for (int i = 0; i < taCtx.length; i++) {
+        final int taskIdx = i;
+        executor.submit(() -> {
+          final OutputCommitter outputCommitter =
+              tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
+          outputCommitter.setupTask(taCtx[taskIdx]);
+          writeOutput(tof[taskIdx].getRecordWriter(taCtx[taskIdx]), taCtx[taskIdx]);
+          describe("Committing Task %d", taskIdx);
+          outputCommitter.commitTask(taCtx[taskIdx]);
+          return null;
+        });
+      }
+    } finally {
+      executor.shutdown();
+      while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
+        log().info("Awaiting thread termination!");
+      }
+    }
+
+    // if we commit here then all tasks will be committed, so there will
+    // be contention for that final directory: both parts will go in.
+
+    describe("\nCommitting Job");
+    amCommitter.commitJob(jContext);
+    assertPathExists("base output directory", outputDir);
+    assertPart0000DoesNotExist(outputDir);
+    Path outSubDir = new Path(outputDir, SUB_DIR);
+    assertPathDoesNotExist("Must not end up with sub_dir/sub_dir",
+        new Path(outSubDir, SUB_DIR));
+
+    // validate output
+    // There's no success marker in the subdirectory
+    validateContent(outSubDir, false, "");
+  }
+
+  @Test
+  public void testUnsupportedSchema() throws Throwable {
+    intercept(PathIOException.class, () ->
+        new ManifestCommitterFactory()
+            .createOutputCommitter(new Path("s3a://unsupported/"), null));
+  }
+
+  /**
+   * Factory for failing committers.
+   */
+
+
+/*
+  protected ManifestCommitter createFailingCommitter(
+  final TaskAttemptContext tContext)
+      throws IOException {
+    //     TODO
+    return null;
+  }
+
+  public class FailingCommitterFactory implements CommitterFactory {
+
+    @Override
+    public ManifestCommitter createCommitter(TaskAttemptContext context)
+        throws IOException {
+      return createFailingCommitter(context);
+    }
+  }*/
+  @Test
+  public void testOutputFormatIntegration() throws Throwable {
+    Configuration conf = getConfiguration();
+    Job job = newJob();
+    assertCommitterFactoryIsManifestCommitter(job, outputDir);
+    job.setOutputFormatClass(TextOutputForTests.class);
+    conf = job.getConfiguration();
+    conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+    conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+    JobContext jContext = new JobContextImpl(conf, taskAttempt0.getJobID());
+    TaskAttemptContext tContext = new TaskAttemptContextImpl(conf,
+        taskAttempt0);
+    TextOutputForTests<IntWritable, IntWritable> outputFormat =
+        (TextOutputForTests<IntWritable, IntWritable>)
+            ReflectionUtils.newInstance(tContext.getOutputFormatClass(), conf);
+    ManifestCommitter committer = (ManifestCommitter)
+        outputFormat.getOutputCommitter(tContext);
+
+    // check path capabilities directly
+    Assertions.assertThat(committer.hasCapability(
+            ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING))
+        .describedAs("dynamic partitioning capability in committer %s",
+            committer)
+        .isTrue();
+    // and through a binding committer -passthrough is critical
+    // for the spark binding.
+    BindingPathOutputCommitter bindingCommitter =
+        new BindingPathOutputCommitter(outputDir, tContext);
+    Assertions.assertThat(bindingCommitter.hasCapability(
+            ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING))
+        .describedAs("dynamic partitioning capability in committer %s",
+            bindingCommitter)
+        .isTrue();
+
+
+    // setup
+    JobData jobData = new JobData(job, jContext, tContext, committer);
+    setupJob(jobData);
+    abortInTeardown(jobData);
+    TextOutputForTests.LoggingLineRecordWriter<IntWritable, IntWritable> recordWriter
+        = outputFormat.getRecordWriter(tContext);
+    IntWritable iw = new IntWritable(1);
+    recordWriter.write(iw, iw);
+    long expectedLength = 4;
+    Path dest = recordWriter.getDest();
+    validateTaskAttemptPathDuringWrite(dest, expectedLength);
+    recordWriter.close(tContext);
+    // at this point
+    validateTaskAttemptPathAfterWrite(dest, expectedLength);
+    Assertions.assertThat(committer.needsTaskCommit(tContext))
+        .as("Committer does not have data to commit " + committer)
+        .isTrue();
+    commitTask(committer, tContext);
+    // at this point the committer tasks stats should be current.
+    IOStatisticsSnapshot snapshot = new IOStatisticsSnapshot(
+        committer.getIOStatistics());
+    String commitsCompleted = COMMITTER_TASKS_COMPLETED_COUNT;
+    LOG.info("after task commit {}", ioStatisticsToPrettyString(snapshot));
+    verifyStatisticCounterValue(snapshot,
+        commitsCompleted, 1);
+    final TaskManifest manifest = loadManifest(
+        committer.getTaskManifestPath(tContext));
+    LOG.info("Manifest {}", manifest.toJson());
+
+    commitJob(committer, jContext);
+    LOG.info("committer iostatistics {}",
+        ioStatisticsSourceToString(committer));
+
+    // validate output
+    ManifestSuccessData successData = verifySuccessMarker(outputDir,
+        committer.getJobUniqueId());
+
+    // the task commit count should get through the job commit
+    IOStatisticsSnapshot successStats = successData.getIOStatistics();
+    LOG.info("loaded statistics {}", successStats);
+    verifyStatisticCounterValue(successStats,
+        commitsCompleted, 1);
+  }
+
+  /**
+   * Create a committer through reflection then use it to abort
+   * a task. This mimics the action of an AM when a container fails and
+   * the AM wants to abort the task attempt.
+   */
+  @Test
+  public void testAMWorkflow() throws Throwable {
+    describe("Create a committer with a null output path & use as an AM");
+    JobData jobData = startJob(true);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+
+    TaskAttemptContext newAttempt = new TaskAttemptContextImpl(
+        jContext.getConfiguration(),
+        taskAttempt0);
+    Configuration conf = jContext.getConfiguration();
+
+    // bind
+    TextOutputForTests.bind(conf);
+
+    OutputFormat<?, ?> outputFormat
+        = ReflectionUtils.newInstance(newAttempt.getOutputFormatClass(), conf);
+    Path outputPath = FileOutputFormat.getOutputPath(newAttempt);
+    Assertions.assertThat(outputPath)
+        .as("null output path in new task attempt")
+        .isNotNull();
+
+    ManifestCommitter committer2 = (ManifestCommitter)
+        outputFormat.getOutputCommitter(newAttempt);
+    committer2.abortTask(tContext);
+
+  }
+
+  /**
+   * Make sure that two jobs in parallel directory trees coexist.
+   * Note: the two jobs are not trying to write to the same
+   * output directory.
+   * That should be possible, but cleanup must be disabled.
+   */
+  @Test
+  public void testParallelJobsToAdjacentPaths() throws Throwable {
+
+    describe("Run two jobs in parallel, assert they both complete");
+    JobData jobData = startJob(true);
+    Job job1 = jobData.job;
+    ManifestCommitter committer1 = jobData.committer;
+    JobContext jContext1 = jobData.jContext;
+    TaskAttemptContext tContext1 = jobData.tContext;
+
+    // now build up a second job
+    String jobId2 = randomJobId();
+    String attempt20 = "attempt_" + jobId2 + "_m_000000_0";
+    TaskAttemptID taskAttempt20 = TaskAttemptID.forName(attempt20);
+    String attempt21 = "attempt_" + jobId2 + "_m_000001_0";
+    TaskAttemptID taskAttempt21 = TaskAttemptID.forName(attempt21);
+
+    Path job1Dest = outputDir;
+    Path job2Dest = new Path(getOutputDir().getParent(),
+        getMethodName() + "job2Dest");
+    // little safety check
+    Assertions.assertThat(job2Dest)
+        .describedAs("Job destinations")
+        .isNotEqualTo(job1Dest);
+
+    // create the second job
+    Job job2 = newJob(job2Dest,
+        unsetUUIDOptions(new JobConf(getConfiguration())),
+        attempt20);
+    Configuration conf2 = job2.getConfiguration();
+    conf2.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+    ManifestCommitter committer2 = null;
+    try {
+      JobContext jContext2 = new JobContextImpl(conf2,
+          taskAttempt20.getJobID());
+      TaskAttemptContext tContext2 =
+          new TaskAttemptContextImpl(conf2, taskAttempt20);
+      committer2 = createCommitter(job2Dest, tContext2);
+      JobData jobData2 = new JobData(job2, jContext2, tContext2, committer2);
+      setupJob(jobData2);
+      abortInTeardown(jobData2);
+      // make sure the directories are different
+      Assertions.assertThat(committer1.getOutputPath())
+          .describedAs("Committer output path of %s and %s", committer1, committer2)
+          .isNotEqualTo(committer2.getOutputPath());
+      // and job IDs
+      Assertions.assertThat(committer1.getJobUniqueId())
+          .describedAs("JobUnique IDs of %s and %s", committer1, committer2)
+          .isNotEqualTo(committer2.getJobUniqueId());
+
+      // job2 setup, write some data there
+      writeTextOutput(tContext2);
+
+      // at this point, job1 and job2 both have uncommitted tasks
+
+      // commit tasks in order task 2, task 1.
+      commitTask(committer2, tContext2);
+      commitTask(committer1, tContext1);
+
+      // commit jobs in order job 1, job 2
+      commitJob(committer1, jContext1);
+
+      getPart0000(job1Dest);
+
+      commitJob(committer2, jContext2);
+      getPart0000(job2Dest);
+
+    } finally {
+      // clean things up in test failures.
+      FileSystem fs = getFileSystem();
+      if (committer1 != null) {
+        fs.delete(committer1.getOutputPath(), true);
+      }
+      if (committer2 != null) {
+        fs.delete(committer2.getOutputPath(), true);
+      }
+    }
+
+  }
+
+  /**
+   * Strip staging/spark UUID options.
+   * @param conf config
+   * @return the patched config
+   */
+  protected Configuration unsetUUIDOptions(final Configuration conf) {
+    conf.unset(SPARK_WRITE_UUID);
+    return conf;
+  }
+
+  /**
+   * Assert that a committer's job attempt path exists.
+   * For the staging committers, this is in the cluster FS.
+   * @param committer committer
+   * @param jobContext job context
+   * @throws IOException failure
+   */
+  protected void assertJobAttemptPathExists(
+      final ManifestCommitter committer,
+      final JobContext jobContext) throws IOException {
+    Path attemptPath = committer.getJobAttemptPath(jobContext);
+    ContractTestUtils.assertIsDirectory(
+        attemptPath.getFileSystem(committer.getConf()),
+        attemptPath);
+  }
+
+  /**
+   * Validate the path of a file being written to during the write
+   * itself.
+   * @param p path
+   * @param expectedLength
+   * @throws IOException IO failure
+   */
+  protected void validateTaskAttemptPathDuringWrite(Path p,
+      final long expectedLength) throws IOException {
+
+  }
+
+  /**
+   * Validate the path of a file being written to after the write
+   * operation has completed.
+   * @param p path
+   * @param expectedLength
+   * @throws IOException IO failure
+   */
+  protected void validateTaskAttemptPathAfterWrite(Path p,
+      final long expectedLength) throws IOException {
+
+  }
+
+  /**
+   * Perform any actions needed to validate the working directory of
+   * a committer.
+   * For example: filesystem, path attributes
+   * @param committer committer instance
+   * @param context task attempt context
+   * @throws IOException IO failure
+   */
+  protected void validateTaskAttemptWorkingDirectory(
+      ManifestCommitter committer,
+      TaskAttemptContext context) throws IOException {
+  }
+
+  /**
+   * Commit a task then validate the state of the committer afterwards.
+   * @param committer committer
+   * @param tContext task context
+   * @throws IOException IO failure
+   */
+  protected void commitTask(final ManifestCommitter committer,
+      final TaskAttemptContext tContext) throws IOException {
+    committer.commitTask(tContext);
+  }
+
+  /**
+   * Commit a job then validate the state of the committer afterwards.
+   * @param committer committer
+   * @param jContext job context
+   * @throws IOException IO failure
+   */
+  protected void commitJob(final ManifestCommitter committer,
+      final JobContext jContext) throws IOException {
+    committer.commitJob(jContext);
+
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java
new file mode 100644
index 0000000000000..11d2beda5d015
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.junit.Test;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.CommonPathCapabilities;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.UnreliableManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.RenameFilesStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyFileContents;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.getEtag;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.UnreliableManifestStoreOperations.SIMULATED_FAILURE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.AbstractJobOrTaskStage.FAILED_TO_RENAME_PREFIX;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test renaming files with fault injection.
+ * This explores etag support and overwrite-on-rename semantics
+ * of the target FS, so some of the tests behave differently
+ * on different stores.
+ */
+public class TestRenameStageFailure extends AbstractManifestCommitterTest {
+
+  /**
+   * Statistic to look for.
+   */
+  public static final String RENAME_FAILURES = OP_COMMIT_FILE_RENAME + ".failures";
+  private static final int FAILING_FILE_INDEX = 5;
+
+  /**
+   * Fault Injection.
+   */
+  private UnreliableManifestStoreOperations failures;
+
+  /** etags returned in listing/file status operations? */
+  private boolean etagsSupported;
+
+  /** etags preserved through rename? */
+  private boolean etagsPreserved;
+
+  /** resilient commit expected? */
+  private boolean resilientCommit;
+
+  protected boolean isResilientCommit() {
+    return resilientCommit;
+  }
+
+  protected boolean isEtagsPreserved() {
+    return etagsPreserved;
+  }
+
+  protected boolean isEtagsSupported() {
+    return etagsSupported;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    final FileSystem fs = getFileSystem();
+    final Path methodPath = methodPath();
+    etagsSupported = fs.hasPathCapability(methodPath,
+        CommonPathCapabilities.ETAGS_AVAILABLE);
+    etagsPreserved = fs.hasPathCapability(methodPath,
+        CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME);
+
+    final ManifestStoreOperations wrappedOperations = getStoreOperations();
+    failures
+        = new UnreliableManifestStoreOperations(wrappedOperations);
+    setStoreOperations(failures);
+    resilientCommit = wrappedOperations.storeSupportsResilientCommit();
+  }
+
+  /**
+   * Does this test suite require rename resilience in the store/FS?
+   * @return true if the store operations are resilient.
+   */
+  protected boolean requireRenameResilience() throws IOException {
+    return false;
+  }
+
+  @Test
+  public void testResilienceAsExpected() throws Throwable {
+    Assertions.assertThat(isResilientCommit())
+        .describedAs("resilient commit support")
+        .isEqualTo(requireRenameResilience());
+  }
+
+  @Test
+  public void testRenameSourceException() throws Throwable {
+    describe("rename fails raising an IOE -expect stage to fail" +
+        " and exception message preserved");
+
+    // destination directory.
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir);
+    Path jobAttemptTaskSubDir = stageConfig.getJobAttemptTaskSubDir();
+
+    // create a manifest with a lot of files, but for
+    // which one of whose renames will fail
+    TaskManifest manifest = new TaskManifest();
+    createFileset(destDir, jobAttemptTaskSubDir, manifest, filesToCreate());
+    final List<FileEntry> filesToCommit = manifest.getFilesToCommit();
+    final FileEntry entry = filesToCommit.get(FAILING_FILE_INDEX);
+    failures.addRenameSourceFilesToFail(entry.getSourcePath());
+
+    // rename MUST fail
+    expectRenameFailure(
+        new RenameFilesStage(stageConfig),
+        manifest,
+        filesToCommit.size(),
+        SIMULATED_FAILURE,
+        PathIOException.class);
+  }
+
+  /**
+   * Number of files to create; must be more than
+   * {@link #FAILING_FILE_INDEX}.
+   */
+  protected int filesToCreate() {
+    return 100;
+  }
+
+  @Test
+  public void testCommitMissingFile() throws Throwable {
+    describe("commit a file which doesn't exist. Expect FNFE always");
+    // destination directory.
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir);
+    Path jobAttemptTaskSubDir = stageConfig.getJobAttemptTaskSubDir();
+    TaskManifest manifest = new TaskManifest();
+    final List<FileEntry> filesToCommit = manifest.getFilesToCommit();
+
+    Path source = new Path(jobAttemptTaskSubDir, "source.parquet");
+    Path dest = new Path(destDir, "destdir.parquet");
+    filesToCommit.add(new FileEntry(source, dest, 0, null));
+    final FileNotFoundException ex = expectRenameFailure(
+        new RenameFilesStage(stageConfig),
+        manifest,
+        0,
+        "",
+        FileNotFoundException.class);
+    LOG.info("Exception raised: {}", ex.toString());
+  }
+
+  /**
+   * Verify that when a job is configured to delete target paths,
+   * renaming will overwrite them.
+   * This test has to use FileSystem contract settings to determine
+   * whether or not the FS will actually permit file-over-file rename.
+   * As POSIX does, local filesystem tests will not fail if the
+   * destination exists.
+   * As ABFS and GCS do reject it, they are required to fail the
+   * first rename sequence, but succeed once delete.target.paths
+   * is true.
+   */
+  @Test
+  public void testDeleteTargetPaths() throws Throwable {
+    describe("Verify that target path deletion works");
+    // destination directory.
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir)
+        .withDeleteTargetPaths(true);
+    Path jobAttemptTaskSubDir = stageConfig.getJobAttemptTaskSubDir();
+    final Path source = new Path(jobAttemptTaskSubDir, "source.txt");
+    final Path dest = new Path(destDir, "source.txt");
+    final byte[] sourceData = "data".getBytes(StandardCharsets.UTF_8);
+    final FileSystem fs = getFileSystem();
+    ContractTestUtils.createFile(fs, source, false, sourceData);
+    touch(fs, dest);
+    TaskManifest manifest = new TaskManifest();
+    final FileEntry entry = createEntryWithEtag(source, dest);
+    manifest.addFileToCommit(entry);
+
+    List<TaskManifest> manifests = new ArrayList<>();
+    manifests.add(manifest);
+
+    // local POSIX filesystems allow rename of file onto file, so
+    // don't fail on the rename.
+    boolean renameOverwritesDest = isSupported(RENAME_OVERWRITES_DEST);
+
+    if (!renameOverwritesDest) {
+      // HDFS, ABFS and GCS do all reject rename of file onto file.
+      // ABFS will use its rename operation so will even raise a
+      // meaningful exception here.
+      final IOException ex = expectRenameFailure(
+          new RenameFilesStage(stageConfig.withDeleteTargetPaths(false)),
+          manifest,
+          0,
+          "",
+          IOException.class);
+      LOG.info("Exception raised: {}", ex.toString());
+    }
+
+    // delete target paths and it works
+    new RenameFilesStage(stageConfig.withDeleteTargetPaths(true))
+        .apply(Pair.of(manifests, Collections.emptySet()));
+
+    // and the new data made it over
+    verifyFileContents(fs, dest, sourceData);
+
+    // lets check the etag too, for completeness
+    if (isEtagsPreserved()) {
+      Assertions.assertThat(getEtag(fs.getFileStatus(dest)))
+          .describedAs("Etag of destination file %s", dest)
+          .isEqualTo(entry.getEtag());
+    }
+
+  }
+
+  @Test
+  public void testRenameReturnsFalse() throws Throwable {
+    describe("commit where rename() returns false for one file." +
+        " Expect failure to be escalated to an IOE");
+
+    Assume.assumeTrue("not used when resilient commits are available",
+        !resilientCommit);
+    // destination directory.
+    Path destDir = methodPath();
+    StageConfig stageConfig = createStageConfigForJob(JOB1, destDir);
+    Path jobAttemptTaskSubDir = stageConfig.getJobAttemptTaskSubDir();
+
+    // create a manifest with a lot of files, but for
+    // which one of whose renames will fail
+    TaskManifest manifest = new TaskManifest();
+    createFileset(destDir, jobAttemptTaskSubDir, manifest, filesToCreate());
+
+    final List<FileEntry> filesToCommit = manifest.getFilesToCommit();
+    final FileEntry entry = filesToCommit.get(FAILING_FILE_INDEX);
+    failures.addRenameSourceFilesToFail(entry.getSourcePath());
+
+    // switch to rename returning false.; again, this must
+    // be escalated to a failure.
+    failures.setRenameToFailWithException(false);
+    expectRenameFailure(
+        new RenameFilesStage(stageConfig),
+        manifest,
+        filesToCommit.size(),
+        FAILED_TO_RENAME_PREFIX,
+        PathIOException.class);
+  }
+
+  /**
+   * Create the source files for a task.
+   * @param destDir destination directory
+   * @param taskAttemptDir directory of the task attempt
+   * @param manifest manifest to update.
+   * @param fileCount how many files.
+   */
+  private void createFileset(
+      final Path destDir,
+      final Path taskAttemptDir,
+      final TaskManifest manifest,
+      final int fileCount) throws IOException {
+    final FileSystem fs = getFileSystem();
+    for (int i = 0; i < fileCount; i++) {
+      String name = String.format("file%04d", i);
+      Path src = new Path(taskAttemptDir, name);
+      Path dest = new Path(destDir, name);
+      touch(fs, src);
+
+      final FileEntry entry = createEntryWithEtag(src, dest);
+      manifest.addFileToCommit(entry);
+    }
+  }
+
+  /**
+   * Create a manifest entry, including size.
+   * If the FS supports etags, one is retrieved.
+   * @param source source
+   * @param dest dest
+   * @return entry
+   * @throws IOException if getFileStatus failed.
+   */
+  private FileEntry createEntryWithEtag(final Path source,
+      final Path dest)
+      throws IOException {
+    final FileStatus st = getFileSystem().getFileStatus(source);
+    final String etag = isEtagsSupported()
+        ? getEtag(st)
+        : null;
+
+    return new FileEntry(source, dest, st.getLen(), etag);
+  }
+
+  /**
+   * Execute rename, expecting a failure.
+   * The number of files renamed MUST be less than the value of {@code files}
+   * @param stage stage
+   * @param manifest task manifests
+   * @param files number of files being renamed.
+   * @param errorText text which must be in the exception string
+   * @param exceptionClass class of the exception
+   * @return the caught exception
+   * @throws Exception if anything else went wrong, or no exception was raised.
+   */
+  private <E extends Throwable> E expectRenameFailure(
+      RenameFilesStage stage,
+      TaskManifest manifest,
+      int files,
+      String errorText,
+      Class<E> exceptionClass) throws Exception {
+
+    List<TaskManifest> manifests = new ArrayList<>();
+    manifests.add(manifest);
+    ProgressCounter progressCounter = getProgressCounter();
+    progressCounter.reset();
+    IOStatisticsStore iostatistics = stage.getIOStatistics();
+    long failures0 = iostatistics.counters().get(RENAME_FAILURES);
+
+    // rename MUST raise an exception.
+    E ex = intercept(exceptionClass, errorText, () ->
+        stage.apply(Pair.of(manifests, Collections.emptySet())));
+
+    LOG.info("Statistics {}", ioStatisticsToPrettyString(iostatistics));
+    // the IOStatistics record the rename as a failure.
+    assertThatStatisticCounter(iostatistics, RENAME_FAILURES)
+        .isEqualTo(failures0 + 1);
+
+    // count of files committed MUST be less than expected.
+    if (files > 0) {
+
+      Assertions.assertThat(stage.getFilesCommitted())
+          .describedAs("Files Committed by stage")
+          .isNotEmpty()
+          .hasSizeLessThan(files);
+
+    }
+
+    // the progress counter will show that the rename did invoke it.
+    // there's no assertion on the actual value as it depends on
+    // execution time of the threads.
+
+    Assertions.assertThat(progressCounter.value())
+        .describedAs("Progress counter %s", progressCounter)
+        .isGreaterThan(0);
+    return ex;
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestTaskManifestFileIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestTaskManifestFileIO.java
new file mode 100644
index 0000000000000..3bd75e5750c75
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestTaskManifestFileIO.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.assertDirEntryMatch;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.assertFileEntryMatch;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.marshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.unmarshallPath;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test Reading/writing manifest files.
+ */
+public class TestTaskManifestFileIO extends AbstractManifestCommitterTest {
+
+  private TaskManifest source;
+
+  private ManifestCommitterTestSupport.JobAndTaskIDsForTests taskIDs;
+
+  private String taskAttempt00;
+
+  private Path testPath;
+
+  private Path taPath;
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    taskIDs = new ManifestCommitterTestSupport.JobAndTaskIDsForTests(2, 2);
+    source = new TaskManifest();
+    taskAttempt00 = taskIDs.getTaskAttempt(0, 0);
+    source.setTaskAttemptID(taskAttempt00);
+    testPath = methodPath();
+    taPath = new Path(testPath, "  " + taskAttempt00);
+    source.setTaskAttemptDir(marshallPath(taPath));
+  }
+
+  /**
+   * Test marshalling, paying attention to paths with spaces in them
+   * as they've been a source of trouble in the S3A committers.
+   */
+  @Test
+  public void testJsonRoundTrip() throws Throwable {
+    describe("Save manifest file to string and back");
+    Path subdirS = new Path(taPath, "subdir");
+    Path subdirD = new Path(testPath, "subdir");
+    source.addDirectory(DirEntry.dirEntry(subdirD, 0, 0));
+
+    // a file
+    Path subfileS = new Path(subdirS, "file");
+    Path subfileD = new Path(subdirD, "file");
+    long len = 256L;
+    FileEntry subFileEntry = new FileEntry(subfileS,
+        subfileD, len, "etag");
+    source.addFileToCommit(subFileEntry);
+
+
+    JsonSerialization<TaskManifest> serializer
+        = TaskManifest.serializer();
+
+    String json = serializer.toJson(source);
+    LOG.info("serialized form\n{}", json);
+    TaskManifest deser = serializer.fromJson(json);
+    deser.validate();
+
+    Assertions.assertThat(deser.getTaskAttemptID())
+        .describedAs("Task attempt ID")
+        .isEqualTo(taskAttempt00);
+
+    Assertions.assertThat(unmarshallPath(deser.getTaskAttemptDir()))
+        .describedAs("Task attempt Dir %s",
+            deser.getTaskAttemptDir())
+        .isEqualTo(taPath);
+
+    Assertions.assertThat(deser.getDestDirectories())
+        .hasSize(1)
+        .allSatisfy(d -> assertDirEntryMatch(d, subdirD, 0));
+    Assertions.assertThat(deser.getFilesToCommit())
+        .hasSize(1)
+        .allSatisfy(d -> assertFileEntryMatch(d, subfileS, subfileD, len));
+    final FileEntry entry = deser.getFilesToCommit().get(0);
+    assertFileEntryMatch(entry, subfileS, subfileD, len);
+    Assertions.assertThat(entry.getEtag())
+        .describedAs("etag of %s", entry)
+        .isEqualTo("etag");
+
+  }
+
+  /**
+   * The manifest validation logic has a safety check that only one
+   * file can rename to the same destination, and that the entries
+   * are valid.
+   */
+  @Test
+  public void testValidateRejectsTwoCommitsToSameDest() throws Throwable {
+
+    Path subdirS = new Path(taPath, "subdir");
+    Path subdirD = new Path(testPath, "subdir");
+    source.addDirectory(DirEntry.dirEntry(subdirD, 0, 0));
+
+    // a file
+    Path subfileS = new Path(subdirS, "file");
+    Path subfileS2 = new Path(subdirS, "file2");
+    Path subfileD = new Path(subdirD, "file");
+    long len = 256L;
+    source.addFileToCommit(
+        new FileEntry(subfileS, subfileD, len, "tag1"));
+    source.addFileToCommit(
+        new FileEntry(subfileS2, subfileD, len, "tag2"));
+    assertValidationFailureOnRoundTrip(source);
+  }
+
+  /**
+   * The manifest validation logic has a safety check that only one
+   * file can rename to the same destination, and that the entries
+   * are valid.
+   */
+  @Test
+  public void testValidateRejectsIncompleteFileEntry() throws Throwable {
+    source.addFileToCommit(
+        new FileEntry(taPath, null, 0, null));
+    assertValidationFailureOnRoundTrip(source);
+  }
+
+  /**
+   * negative lengths are not allowed.
+   */
+  @Test
+  public void testValidateRejectsInvalidFileLength() throws Throwable {
+    source.addFileToCommit(
+        new FileEntry(taPath, testPath, -1, null));
+    assertValidationFailureOnRoundTrip(source);
+  }
+
+  @Test
+  public void testRejectIncompatibleVersion() throws Throwable {
+    source.setVersion(5);
+    assertValidationFailureOnRoundTrip(source);
+  }
+
+  @Test
+  public void testRejectIncompatibleType() throws Throwable {
+    source.setType("Incompatible type");
+    assertValidationFailureOnRoundTrip(source);
+  }
+
+  private void assertValidationFailureOnRoundTrip(
+      final TaskManifest manifest) throws Exception {
+    JsonSerialization<TaskManifest> serializer
+        = TaskManifest.serializer();
+    String json = serializer.toJson(manifest);
+    LOG.info("serialized form\n{}", json);
+    TaskManifest deser = serializer.fromJson(json);
+    intercept(IOException.class, deser::validate);
+  }
+
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TextOutputForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TextOutputForTests.java
new file mode 100644
index 0000000000000..14c57c451ab86
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TextOutputForTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A subclass of {@link TextOutputFormat} which logs what is happening, and
+ * returns a {@link LoggingLineRecordWriter} which allows the caller
+ * to get the destination path.
+ * @param <K> key
+ * @param <V> value
+ */
+public class TextOutputForTests<K, V> extends TextOutputFormat<K, V> {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TextOutputForTests.class);
+
+  public static final String NAME
+      = "org.apache.hadoop.fs.s3a.commit.LoggingTextOutputFormat";
+
+  @Override
+  public LoggingLineRecordWriter<K, V> getRecordWriter(TaskAttemptContext job)
+      throws IOException, InterruptedException {
+    Configuration conf = job.getConfiguration();
+    boolean isCompressed = getCompressOutput(job);
+    String keyValueSeparator = conf.get(SEPARATOR, "\t");
+    CompressionCodec codec = null;
+    String extension = "";
+    if (isCompressed) {
+      Class<? extends CompressionCodec> codecClass =
+          getOutputCompressorClass(job, GzipCodec.class);
+      codec = ReflectionUtils.newInstance(codecClass, conf);
+      extension = codec.getDefaultExtension();
+    }
+    Path file = getDefaultWorkFile(job, extension);
+    FileSystem fs = file.getFileSystem(conf);
+    FSDataOutputStream fileOut = fs.create(file, true);
+    LOG.debug("Creating LineRecordWriter with destination {}", file);
+    if (isCompressed) {
+      return new LoggingLineRecordWriter<>(
+          file, new DataOutputStream(codec.createOutputStream(fileOut)),
+          keyValueSeparator);
+    } else {
+      return new LoggingLineRecordWriter<>(file, fileOut, keyValueSeparator);
+    }
+  }
+
+  /**
+   * Write a line; counts the number of lines written and logs @ debug in the
+   * {@code close()} call.
+   * @param <K> key
+   * @param <V> value
+   */
+  public static class LoggingLineRecordWriter<K, V>
+      extends LineRecordWriter<K, V> {
+
+    private final Path dest;
+
+    private long lines;
+
+    public LoggingLineRecordWriter(Path dest, DataOutputStream out,
+        String keyValueSeparator) {
+      super(out, keyValueSeparator);
+      this.dest = dest;
+    }
+
+    public LoggingLineRecordWriter(DataOutputStream out, Path dest) {
+      super(out);
+      this.dest = dest;
+    }
+
+    @Override
+    public synchronized void write(K key, V value) throws IOException {
+      super.write(key, value);
+      lines++;
+    }
+
+    public synchronized void close(TaskAttemptContext context)
+        throws IOException {
+      LOG.debug("Closing output file {} with {} lines :{}",
+          dest, lines, out);
+      out.close();
+    }
+
+    public Path getDest() {
+      return dest;
+    }
+
+    public long getLines() {
+      return lines;
+    }
+  }
+
+  /**
+   * Bind to a configuration for job submission.
+   * @param conf configuration
+   */
+  public static void bind(Configuration conf) {
+    conf.setClass(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
+        TextOutputForTests.class,
+        OutputFormat.class);
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ThreadLeakTracker.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ThreadLeakTracker.java
new file mode 100644
index 0000000000000..4245ef3aabddb
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ThreadLeakTracker.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import org.assertj.core.api.Assertions;
+
+/**
+ * Thread Leak tracker.
+ * Initialized with the current set of threads and some known-long-life
+ * threads whose presence must not be considered a failure.
+ */
+public class ThreadLeakTracker {
+
+  private final Set<String> trackedThreads =
+      listInitialThreadsForLifecycleChecks();
+
+  public void assertNoThreadLeakage() {
+    Assertions.assertThat(getCurrentThreadNames())
+        .describedAs("The threads at the end of the test run")
+        .isSubsetOf(trackedThreads);
+  }
+
+  public Set<String> getTrackedThreads() {
+    return trackedThreads;
+  }
+
+  /**
+   * Get a set containing the names of all active threads,
+   * stripping out all test runner threads.
+   * @return the current set of threads.
+   */
+  public static Set<String> getCurrentThreadNames() {
+    TreeSet<String> threads = Thread.getAllStackTraces().keySet()
+        .stream()
+        .map(Thread::getName)
+        .filter(n -> n.startsWith("JUnit"))
+        .filter(n -> n.startsWith("surefire"))
+        .collect(Collectors.toCollection(TreeSet::new));
+    return threads;
+  }
+
+  /**
+   * This creates a set containing all current threads and some well-known
+   * thread names whose existence should not fail test runs.
+   * They are generally static cleaner threads created by various classes
+   * on instantiation.
+   * @return a set of threads to use in later assertions.
+   */
+  public static Set<String> listInitialThreadsForLifecycleChecks() {
+    Set<String> threadSet = getCurrentThreadNames();
+    // static filesystem statistics cleaner
+    threadSet.add("org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner");
+
+    // java.lang.UNIXProcess. maybe if chmod is called?
+    threadSet.add("process reaper");
+    // once a quantile has been scheduled, the mutable quantile thread pool
+    // is initialized; it has a minimum thread size of 1.
+    threadSet.add("MutableQuantiles-0");
+    // IDE?
+    threadSet.add("Attach Listener");
+    return threadSet;
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/UnreliableManifestStoreOperations.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/UnreliableManifestStoreOperations.java
new file mode 100644
index 0000000000000..811fc704a2a33
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/UnreliableManifestStoreOperations.java
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.OPERATION_TIMED_OUT;
+
+/**
+ * Wrap an existing {@link ManifestStoreOperations} implementation and fail on
+ * specific paths.
+ * This is for testing. It could be implemented via
+ * Mockito 2 spy code but is not so that:
+ * 1. It can be backported to Hadoop versions using Mockito 1.x.
+ * 2. It can be extended to use in production. This is why it is in
+ * the production module -to allow for downstream tests to adopt it.
+ * 3. You can actually debug what's going on.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class UnreliableManifestStoreOperations extends ManifestStoreOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      UnreliableManifestStoreOperations.class);
+
+  /**
+   * The timeout message ABFS raises.
+   */
+  public static final String E_TIMEOUT
+      = "Operation could not be completed within the specified time";
+
+  /**
+   * Text to use in simulated failure exceptions.
+   */
+  public static final String SIMULATED_FAILURE = "Simulated failure";
+
+  /**
+   * Underlying store operations to wrap.
+   */
+  private final ManifestStoreOperations wrappedOperations;
+
+  /**
+   * Paths of delete operations to fail.
+   */
+  private final Set<Path> deletePathsToFail = new HashSet<>();
+
+  /**
+   * Paths of delete operations to time out, as ABFS may.
+   */
+  private final Set<Path> deletePathsToTimeOut = new HashSet<>();
+
+  /**
+   * Paths of List operations to fail.
+   */
+  private final Set<Path> listToFail = new HashSet<>();
+
+  /**
+   * Paths of mkdirs operations to fail.
+   */
+  private final Set<Path> mkdirsToFail = new HashSet<>();
+
+  /**
+   * Paths which don't exist.
+   */
+  private final Set<Path> pathNotFound = new HashSet<>();
+
+  /**
+   * Source file whose rename/commit will fail.
+   */
+  private final Set<Path> renameSourceFilesToFail = new HashSet<>();
+
+  /**
+   * Dest dir into which all renames/commits will fail.
+   * Subdirectories under this are not checked.
+   */
+  private final Set<Path> renameDestDirsToFail = new HashSet<>();
+
+  /**
+   * Path of save() to fail.
+   */
+  private final Set<Path> saveToFail = new HashSet<>();
+
+  /**
+   * timeout sleep.
+   */
+  private int timeoutSleepTimeMillis;
+
+  /**
+   * Should rename thrown an exception or just return false.
+   */
+  private boolean renameToFailWithException = true;
+
+  /**
+   * Constructor.
+   * @param wrappedOperations operations to wrap.
+   */
+  public UnreliableManifestStoreOperations(final ManifestStoreOperations wrappedOperations) {
+    this.wrappedOperations = wrappedOperations;
+  }
+
+
+  /**
+   * Reset everything.
+   */
+  public void reset() {
+    deletePathsToFail.clear();
+    deletePathsToTimeOut.clear();
+    pathNotFound.clear();
+    renameSourceFilesToFail.clear();
+    renameDestDirsToFail.clear();
+    timeoutSleepTimeMillis = 0;
+  }
+
+  public int getTimeoutSleepTimeMillis() {
+    return timeoutSleepTimeMillis;
+  }
+
+  public void setTimeoutSleepTimeMillis(final int timeoutSleepTimeMillis) {
+    this.timeoutSleepTimeMillis = timeoutSleepTimeMillis;
+  }
+
+  public boolean getRenameToFailWithException() {
+    return renameToFailWithException;
+  }
+
+  public void setRenameToFailWithException(
+      final boolean renameToFailWithException) {
+    this.renameToFailWithException = renameToFailWithException;
+  }
+
+  /**
+   * Add a path to the list of delete paths to fail.
+   * @param path path to add.
+   */
+  public void addDeletePathToFail(Path path) {
+    deletePathsToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of delete paths to time out.
+   * @param path path to add.
+   */
+  public void addDeletePathToTimeOut(Path path) {
+    deletePathsToTimeOut.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of paths where list will fail.
+   * @param path path to add.
+   */
+  public void addListToFail(Path path) {
+    listToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of mkdir calls to fail.
+   * @param path path to add.
+   */
+  public void addMkdirsToFail(Path path) {
+    mkdirsToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path not found.
+   * @param path path
+   */
+  public void addPathNotFound(Path path) {
+    pathNotFound.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of rename source paths to fail.
+   * @param path path to add.
+   */
+  public void addRenameSourceFilesToFail(Path path) {
+    renameSourceFilesToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of dest dirs to fail.
+   * @param path path to add.
+   */
+  public void addRenameDestDirsFail(Path path) {
+    renameDestDirsToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Add a path to the list of paths where save will fail.
+   * @param path path to add.
+   */
+  public void addSaveToFail(Path path) {
+    saveToFail.add(requireNonNull(path));
+  }
+
+  /**
+   * Raise an exception if the path is in the set of target paths.
+   * @param operation operation which failed.
+   * @param path path to check
+   * @param paths paths to probe for {@code path} being in.
+   * @throws IOException simulated failure
+   */
+  private void maybeRaiseIOE(String operation, Path path, Set<Path> paths)
+      throws IOException {
+    if (paths.contains(path)) {
+      LOG.info("Simulating failure of {} with {}", operation, path);
+      throw new PathIOException(path.toString(),
+          SIMULATED_FAILURE + " of " + operation);
+    }
+  }
+
+  /**
+   * Verify that a path is not on the file not found list.
+   * @param path path
+   * @throws FileNotFoundException if configured to fail.
+   */
+  private void verifyExists(Path path) throws FileNotFoundException {
+    if (pathNotFound.contains(path)) {
+      throw new FileNotFoundException(path.toString());
+    }
+  }
+
+  /**
+   * Time out if the path is in the list of timeout paths.
+   * Will sleep first, to help simulate delays.
+   * @param operation operation which failed.
+   * @param path path to check
+   * @param paths paths to probe for {@code path} being in.
+   * @throws IOException simulated timeout
+   */
+  private void maybeTimeout(String operation, Path path, Set<Path> paths)
+      throws IOException {
+    if (paths.contains(path)) {
+      LOG.info("Simulating timeout of {} with {}", operation, path);
+      try {
+        if (timeoutSleepTimeMillis > 0) {
+          Thread.sleep(timeoutSleepTimeMillis);
+        }
+      } catch (InterruptedException e) {
+        throw new InterruptedIOException(e.toString());
+      }
+      throw new PathIOException(path.toString(),
+          "ErrorCode=" + OPERATION_TIMED_OUT
+              + " ErrorMessage=" + E_TIMEOUT);
+    }
+  }
+
+  @Override
+  public FileStatus getFileStatus(final Path path) throws IOException {
+    verifyExists(path);
+    return wrappedOperations.getFileStatus(path);
+  }
+
+  @Override
+  public boolean delete(final Path path, final boolean recursive)
+      throws IOException {
+    String op = "delete";
+    maybeTimeout(op, path, deletePathsToTimeOut);
+    maybeRaiseIOE(op, path, deletePathsToFail);
+    return wrappedOperations.delete(path, recursive);
+  }
+
+  @Override
+  public boolean mkdirs(final Path path) throws IOException {
+    maybeRaiseIOE("mkdirs", path, mkdirsToFail);
+    return wrappedOperations.mkdirs(path);
+  }
+
+  @Override
+  public boolean renameFile(final Path source, final Path dest)
+      throws IOException {
+    String op = "rename";
+    if (renameToFailWithException) {
+      maybeRaiseIOE(op, source, renameSourceFilesToFail);
+      maybeRaiseIOE(op, dest.getParent(), renameDestDirsToFail);
+    } else {
+      if (renameSourceFilesToFail.contains(source)
+          || renameDestDirsToFail.contains(dest.getParent())) {
+        LOG.info("Failing rename({}, {})", source, dest);
+        return false;
+      }
+    }
+    return wrappedOperations.renameFile(source, dest);
+  }
+
+  @Override
+  public RemoteIterator<FileStatus> listStatusIterator(final Path path)
+      throws IOException {
+    verifyExists(path);
+    maybeRaiseIOE("listStatus", path, listToFail);
+    return wrappedOperations.listStatusIterator(path);
+  }
+
+  @Override
+  public TaskManifest loadTaskManifest(JsonSerialization<TaskManifest> serializer,
+      final FileStatus st) throws IOException {
+    verifyExists(st.getPath());
+    return wrappedOperations.loadTaskManifest(serializer, st);
+  }
+
+  @Override
+  public <T extends AbstractManifestData<T>> void save(T manifestData,
+      final Path path,
+      final boolean overwrite) throws IOException {
+    maybeRaiseIOE("save", path, saveToFail);
+    wrappedOperations.save(manifestData, path, overwrite);
+  }
+
+  @Override
+  public void msync(Path path) throws IOException {
+    wrappedOperations.msync(path);
+  }
+
+  @Override
+  public String getEtag(FileStatus status) {
+    return wrappedOperations.getEtag(status);
+  }
+
+  @Override
+  public boolean storeSupportsResilientCommit() {
+    return wrappedOperations.storeSupportsResilientCommit();
+  }
+
+  @Override
+  public CommitFileResult commitFile(final FileEntry entry)
+      throws IOException {
+    if (renameToFailWithException) {
+      maybeRaiseIOE("commitFile",
+          entry.getSourcePath(), renameSourceFilesToFail);
+      maybeRaiseIOE("commitFile",
+          entry.getDestPath().getParent(), renameDestDirsToFail);
+    }
+    return wrappedOperations.commitFile(entry);
+  }
+
+  @Override
+  public boolean storePreservesEtagsThroughRenames(Path path) {
+    return wrappedOperations.storePreservesEtagsThroughRenames(path);
+  }
+
+  @Override
+  public void close() throws IOException {
+    wrappedOperations.close();
+  }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
new file mode 100644
index 0000000000000..61c4073e1bdb6
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These test suites are subclassed in internal modules (hadoop-azure) and
+ * externally for gcs integration testing.
+ * Do test with both before making changes.
+ */
+@InterfaceAudience.LimitedPrivate("integration tests")
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestEventFetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestEventFetcher.java
index 1a0bfc751d380..9df55f126dbf8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestEventFetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestEventFetcher.java
@@ -19,7 +19,6 @@
 
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.anyInt;
-import static org.mockito.Mockito.anyString;
 import static org.mockito.Mockito.eq;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
index 4e718b85a82e2..063daada6d770 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
@@ -23,7 +23,6 @@
 import static org.mockito.Mockito.mock;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -44,7 +43,6 @@
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath;
-import org.apache.hadoop.test.Whitebox;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -217,8 +215,7 @@ public void testIoSortDefaults() {
 
   @SuppressWarnings({ "unchecked", "deprecation" })
   @Test(timeout=10000)
-  public void testOnDiskMerger() throws IOException, URISyntaxException,
-    InterruptedException {
+  public void testOnDiskMerger() throws IOException {
     JobConf jobConf = new JobConf();
     final int SORT_FACTOR = 5;
     jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, SORT_FACTOR);
@@ -229,12 +226,8 @@ public void testOnDiskMerger() throws IOException, URISyntaxException,
       new MergeManagerImpl<IntWritable, IntWritable>(null, jobConf, fs, null
         , null, null, null, null, null, null, null, null, null, mapOutputFile);
 
-    MergeThread<MapOutput<IntWritable, IntWritable>, IntWritable, IntWritable>
-      onDiskMerger = (MergeThread<MapOutput<IntWritable, IntWritable>,
-        IntWritable, IntWritable>) Whitebox.getInternalState(manager,
-          "onDiskMerger");
-    int mergeFactor = (Integer) Whitebox.getInternalState(onDiskMerger,
-      "mergeFactor");
+    MergeThread onDiskMerger = manager.getOnDiskMerger();
+    int mergeFactor = onDiskMerger.getMergeFactor();
 
     // make sure the io.sort.factor is set properly
     assertEquals(mergeFactor, SORT_FACTOR);
@@ -252,9 +245,7 @@ public void testOnDiskMerger() throws IOException, URISyntaxException,
     }
 
     //Check that the files pending to be merged are in sorted order.
-    LinkedList<List<CompressAwarePath>> pendingToBeMerged =
-      (LinkedList<List<CompressAwarePath>>) Whitebox.getInternalState(
-        onDiskMerger, "pendingToBeMerged");
+    LinkedList<List<CompressAwarePath>> pendingToBeMerged = onDiskMerger.getPendingToBeMerged();
     assertTrue("No inputs were added to list pending to merge",
       pendingToBeMerged.size() > 0);
     for(int i = 0; i < pendingToBeMerged.size(); ++i) {
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/contract/localfs.xml
similarity index 67%
rename from hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/contract/localfs.xml
index fbf3a177c91f0..e956a50e8d307 100644
--- a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/contract/localfs.xml
@@ -17,41 +17,58 @@
   -->
 
 <configuration>
-  <!--
-  Openstack Swift is a blobstore, with very different behavior than a
-  classic filesystem.
-  -->
+
+<!--
+This is a copy of the file
+hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml ;
+it is needed for all contract tests which test against the local filesystem.
+
+case sensitivity and permission options are determined at run time from OS type.
+-->
 
   <property>
-    <name>fs.contract.test.root-tests-enabled</name>
+    <name>fs.contract.is-case-sensitive</name>
     <value>true</value>
   </property>
 
+
   <property>
-    <name>fs.contract.test.random-seek-count</name>
-    <value>10</value>
+    <name>fs.contract.supports-unix-permissions</name>
+    <value>true</value>
   </property>
 
+  <!--
+  The remaining options are static
+  -->
+
   <property>
-    <name>fs.contract.is-blobstore</name>
-    <value>true</value>
+    <name>fs.contract.test.root-tests-enabled</name>
+    <value>false</value>
   </property>
 
   <property>
-    <name>fs.contract.create-overwrites-directory</name>
+    <name>fs.contract.test.random-seek-count</name>
+    <value>1000</value>
+  </property>
+
+  <property>
+    <name>fs.contract.rename-creates-dest-dirs</name>
     <value>true</value>
   </property>
 
   <property>
-    <name>fs.contract.create-visibility-delayed</name>
+    <name>fs.contract.rename-overwrites-dest</name>
     <value>true</value>
   </property>
 
   <property>
-    <name>fs.contract.is-case-sensitive</name>
+    <name>fs.contract.rename-remove-dest-if-empty-dir</name>
     <value>true</value>
   </property>
 
+  <!--
+  checksummed filesystems do not support append; see HADOOP-4292
+  -->
   <property>
     <name>fs.contract.supports-append</name>
     <value>false</value>
@@ -59,12 +76,12 @@
 
   <property>
     <name>fs.contract.supports-atomic-directory-delete</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
   <property>
     <name>fs.contract.supports-atomic-rename</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
   <property>
@@ -82,6 +99,12 @@
     <value>true</value>
   </property>
 
+  <property>
+    <name>fs.contract.supports-seek-on-closed-file</name>
+    <value>true</value>
+  </property>
+
+  <!-- checksum FS doesn't allow seeking past EOF -->
   <property>
     <name>fs.contract.rejects-seek-past-eof</name>
     <value>true</value>
@@ -89,16 +112,26 @@
 
   <property>
     <name>fs.contract.supports-strict-exceptions</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-settimes</name>
     <value>true</value>
   </property>
 
   <property>
-    <name>fs.contract.supports-unix-permissions</name>
-    <value>false</value>
+    <name>fs.contract.supports-getfilestatus</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>fs.contract.supports-settimes</name>
+    <value>true</value>
   </property>
 
   <property>
-    <name>fs.contract.rename-returns-false-if-source-missing</name>
+    <name>fs.contract.supports-getfilestatus</name>
     <value>true</value>
   </property>
 
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/core-site.xml
similarity index 51%
rename from hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml
rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/core-site.xml
index 9252e885871e4..9312483e6efba 100644
--- a/hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/core-site.xml
@@ -18,34 +18,16 @@
   ~  limitations under the License.
   -->
 
-<!-- Values used when running unit tests.  This is mostly empty, to -->
-<!-- use of the default values, overriding the potentially -->
-<!-- user-editted core-site.xml in the conf/ directory.  -->
+<!-- Values used when running unit tests.  Specify any values in here that
+     should override the default values. -->
 
 <configuration>
 
-
-    <property>
-        <name>hadoop.tmp.dir</name>
-        <value>target/build/test</value>
-        <description>A base for other temporary directories.</description>
-        <final>true</final>
-    </property>
-
-    <!-- Turn security off for tests by default -->
-    <property>
-        <name>hadoop.security.authentication</name>
-        <value>simple</value>
-    </property>
-
-    <!--
-    To run these tests.
-
-    # Create a file auth-keys.xml  - DO NOT ADD TO REVISION CONTROL
-    # add the property test.fs.swift.name to point to a swift filesystem URL
-    # Add the credentials for the service you are testing against
-    -->
-    <include xmlns="http://www.w3.org/2001/XInclude"
-             href="auth-keys.xml"/>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>target/build/test</value>
+    <description>A base for other temporary directories.</description>
+    <final>false</final>
+  </property>
 
 </configuration>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index baf38d3c84511..79327a780fceb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -39,6 +39,12 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 0136820a39296..6f2a528452c23 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -84,7 +84,6 @@
 import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.webapp.WebApp;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
index fb077dd784991..8b0e741ae6ac9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.v2.app.webapp.App;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.DIV;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
index a00146e680bc0..aa3acfbc54641 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
@@ -123,7 +123,7 @@ public class HsTasksBlock extends HtmlBlock {
       long sortFinishTime = -1;
       long attemptFinishTime = -1;
       long elapsedShuffleTime = -1;
-      long elapsedSortTime = -1;;
+      long elapsedSortTime = -1;
       long elapsedReduceTime = -1;
       long attemptElapsed = -1;
       TaskAttempt successful = info.getSuccessful();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
index b4a4566bb5383..f1dc6260d7481 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
@@ -284,7 +285,7 @@ public void verifyHSInfo(JSONObject info, AppContext ctx)
 
   public void verifyHSInfoXML(String xml, AppContext ctx)
       throws JSONException, Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
index 708a60b821fa5..3ca6db3ab4a21 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java
@@ -47,6 +47,7 @@
 import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
@@ -207,7 +208,7 @@ public void testTaskAttemptsXML() throws JSONException, Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
@@ -331,7 +332,7 @@ public void testTaskAttemptIdXML() throws JSONException, Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
@@ -618,7 +619,7 @@ public void testTaskAttemptIdXMLCounters() throws JSONException, Exception {
           assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
               response.getType().toString());
           String xml = response.getEntity(String.class);
-          DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+          DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
           DocumentBuilder db = dbf.newDocumentBuilder();
           InputSource is = new InputSource();
           is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
index 62a53979e9f8f..21df639473605 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobConf.java
@@ -48,6 +48,7 @@
 import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
@@ -230,7 +231,7 @@ public void testJobConfXML() throws JSONException, Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
index 906b4ad41b203..05ed2775a4bbf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
@@ -190,7 +191,7 @@ public void testJobsXML() throws Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -422,7 +423,7 @@ public void testJobIdInvalidXML() throws JSONException, Exception {
           response.getType().toString());
       String msg = response.getEntity(String.class);
       System.out.println(msg);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(msg));
@@ -489,7 +490,7 @@ public void testJobIdXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -612,7 +613,7 @@ public void testJobCountersXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -769,7 +770,7 @@ public void testJobAttemptsXML() throws Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
index bcef55feda069..47329cc39f85d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java
@@ -46,6 +46,7 @@
 import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
@@ -202,7 +203,7 @@ public void testTasksXML() throws JSONException, Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -549,7 +550,7 @@ public void testTaskIdXML() throws JSONException, Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
@@ -729,7 +730,7 @@ public void testJobTaskCountersXML() throws Exception {
         assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
             response.getType().toString());
         String xml = response.getEntity(String.class);
-        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
         DocumentBuilder db = dbf.newDocumentBuilder();
         InputSource is = new InputSource();
         is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index e98b574d1fe38..17358a37da32d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -93,6 +93,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -104,6 +110,7 @@
       <groupId>org.hsqldb</groupId>
       <artifactId>hsqldb</artifactId>
       <scope>test</scope>
+      <classifier>jdk8</classifier>
     </dependency>
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
index 4262f1f23b5d8..4f1318c044103 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/TestSequenceFileMergeProgress.java
@@ -19,16 +19,11 @@
 package org.apache.hadoop.io;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
 
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator;
-import org.apache.hadoop.io.SequenceFile.Sorter.SegmentDescriptor;
-import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.mapred.*;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
index 35992f5de007f..3aedfbf1ecb2a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
@@ -70,7 +70,7 @@ private static void createBigMapInputFile(Configuration conf, FileSystem fs,
                                 BytesWritable.class, BytesWritable.class,
                                 CompressionType.NONE);
     long numBytesToWrite = fileSizeInMB * 1024 * 1024;
-    int minKeySize = conf.getInt(MIN_KEY, 10);;
+    int minKeySize = conf.getInt(MIN_KEY, 10);
     int keySizeRange = 
       conf.getInt(MAX_KEY, 1000) - minKeySize;
     int minValueSize = conf.getInt(MIN_VALUE, 0);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/EmptyInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/EmptyInputFormat.java
index edb5476febae5..4eaa73566c247 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/EmptyInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/EmptyInputFormat.java
@@ -19,8 +19,6 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.fs.FileSystem;
-
 /**
   * InputFormat which simulates the absence of input data
   * by returning zero split.
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java
index 718dbee0144b6..a71550bce8a44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java
@@ -26,15 +26,8 @@
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.*;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 
 import java.net.URI;
@@ -62,8 +55,8 @@ public static class MapClass extends MapReduceBase
     public void configure(JobConf jconf) {
       conf = jconf;
       try {
-        Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
-        Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
+        Path[] localArchives = JobContextImpl.getLocalCacheArchives(conf);
+        Path[] localFiles = JobContextImpl.getLocalCacheFiles(conf);
         // read the cached files (unzipped, unjarred and text)
         // and put it into a single file TEST_ROOT_DIR/test.txt
         String TEST_ROOT_DIR = jconf.get("test.build.data","/tmp");
@@ -254,7 +247,7 @@ public static TestResult launchMRCache(String indir,
     uris[3] = fs.getUri().resolve(cacheDir + "/test.tgz");
     uris[4] = fs.getUri().resolve(cacheDir + "/test.tar.gz");
     uris[5] = fs.getUri().resolve(cacheDir + "/test.tar");
-    DistributedCache.addCacheFile(uris[0], conf);
+    Job.addCacheFile(uris[0], conf);
 
     // Save expected file sizes
     long[] fileSizes = new long[1];
@@ -262,7 +255,7 @@ public static TestResult launchMRCache(String indir,
 
     long[] archiveSizes = new long[5]; // track last 5
     for (int i = 1; i < 6; i++) {
-      DistributedCache.addCacheArchive(uris[i], conf);
+      Job.addCacheArchive(uris[i], conf);
       archiveSizes[i-1] = // starting with second archive
         fs.getFileStatus(new Path(uris[i].getPath())).getLen();
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
index 5df1af5cce1d1..3372c8f28b6ff 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
@@ -179,8 +179,7 @@ public void testMR() throws Exception {
 
     // Hack for local FS that does not have the concept of a 'mounting point'
     if (isLocalFS()) {
-      String localPathRoot = System.getProperty("test.build.data","/tmp")
-        .toString().replace(' ', '+');;
+      String localPathRoot = System.getProperty("test.build.data", "/tmp").replace(' ', '+');
       inDir = new Path(localPathRoot, inDir);
       outDir = new Path(localPathRoot, outDir);
     }
@@ -217,8 +216,7 @@ private String launchWordCount(JobConf conf,
 
     // Hack for local FS that does not have the concept of a 'mounting point'
     if (isLocalFS()) {
-      String localPathRoot = System.getProperty("test.build.data","/tmp")
-        .toString().replace(' ', '+');;
+      String localPathRoot = System.getProperty("test.build.data", "/tmp").replace(' ', '+');
       inDir = new Path(localPathRoot, inDir);
       outDir = new Path(localPathRoot, outDir);
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
index 04a5127aaf202..ef559d95081db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
@@ -32,8 +32,6 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.lib.LazyOutputFormat;
 import org.junit.Test;
 import static org.junit.Assert.assertTrue;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
index 8984e3740187e..60e32683c3a75 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
@@ -24,8 +24,6 @@
 import java.io.InputStreamReader;
 import java.net.URI;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java
index 26d697a616591..bdfe0f5dc6982 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java
@@ -28,13 +28,14 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
+import org.apache.hadoop.util.XMLUtils;
+
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import static org.junit.Assert.*;
 
 import org.junit.Test;
 
-
 public class TestQueueConfigurationParser {
 /**
  * test xml generation 
@@ -64,7 +65,7 @@ public void testQueueConfigurationParser()
     DOMSource domSource = new DOMSource(e);
     StringWriter writer = new StringWriter();
     StreamResult result = new StreamResult(writer);
-    TransformerFactory tf = TransformerFactory.newInstance();
+    TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
     Transformer transformer = tf.newTransformer();
     transformer.transform(domSource, result);
     String str= writer.toString();
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java
index 9b04f64ac6042..1b99ce0c0aa1d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.hadoop.mapreduce.task.reduce.Fetcher;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -37,6 +38,7 @@
 import java.util.Formatter;
 import java.util.Iterator;
 
+import static org.apache.hadoop.mapreduce.task.reduce.Fetcher.SHUFFLE_ERR_GRP_NAME;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -87,6 +89,9 @@ public void testReduceFromPartialMem() throws Exception {
     final long spill = c.findCounter(TaskCounter.SPILLED_RECORDS).getCounter();
     assertTrue("Expected some records not spilled during reduce" + spill + ")",
         spill < 2 * out); // spilled map records, some records at the reduce
+    long shuffleIoErrors =
+        c.getGroup(SHUFFLE_ERR_GRP_NAME).getCounter(Fetcher.ShuffleErrors.IO_ERROR.toString());
+    assertEquals(0, shuffleIoErrors);
   }
 
   /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
index 68fe1a0198dc6..5698df3128a04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
index f3e58930eacc5..8829a093b13bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.RecordWriter;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.RunningJob;
@@ -46,11 +47,16 @@
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.Iterator;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TestMultipleOutputs extends HadoopTestCase {
 
@@ -70,6 +76,19 @@ public void testWithCounters() throws Exception {
     _testMOWithJavaSerialization(true);
   }
 
+  @SuppressWarnings("unchecked")
+  @Test(expected = IOException.class)
+  public void testParallelCloseIOException() throws IOException {
+    RecordWriter writer = mock(RecordWriter.class);
+    Map<String, RecordWriter> recordWriters = mock(Map.class);
+    when(recordWriters.values()).thenReturn(Arrays.asList(writer, writer));
+    doThrow(new IOException("test IO exception")).when(writer).close(null);
+    JobConf conf = createJobConf();
+    MultipleOutputs mos = new MultipleOutputs(conf);
+    mos.setRecordWriters(recordWriters);
+    mos.close();
+  }
+
   private static final Path ROOT_DIR = new Path("testing/mo");
   private static final Path IN_DIR = new Path(ROOT_DIR, "input");
   private static final Path OUT_DIR = new Path(ROOT_DIR, "output");
@@ -307,6 +326,7 @@ protected void _testMultipleOutputs(boolean withCounters) throws Exception {
 
   }
 
+
   @SuppressWarnings({"unchecked"})
   public static class MOMap implements Mapper<LongWritable, Text, LongWritable,
     Text> {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/FailJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/FailJob.java
index 6f83acf2f5441..105b00295e15f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/FailJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/FailJob.java
@@ -23,10 +23,8 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
index 7c010382eab68..4d4f2276f4ea4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapred.Utils;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
index 4bcacd8ec4858..343e81f163128 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
@@ -437,7 +437,7 @@ private static int test3(IntWritable key,
     int count = 0;
 
     while (values.hasNext()) {
-      i = values.next();;
+      i = values.next();
       LOG.info(key + ":" + i);
       
       if (count == 5) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
index a6f8a725f6991..aca07468b6d91 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
@@ -39,7 +39,6 @@
 import org.junit.Before;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @see TestDelegatingInputFormat
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java
index 1173ea4fa478f..11741d4450a74 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java
@@ -32,9 +32,8 @@
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.junit.After;
+
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
index babd20e66c45a..717163ce243ed 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
@@ -31,7 +31,9 @@
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MapReduceTestUtil;
 import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.Reducer;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -39,10 +41,15 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TestMRMultipleOutputs extends HadoopTestCase {
 
@@ -62,6 +69,20 @@ public void testWithCounters() throws Exception {
     _testMOWithJavaSerialization(true);
   }
 
+  @SuppressWarnings("unchecked")
+  @Test(expected = IOException.class)
+  public void testParallelCloseIOException() throws IOException, InterruptedException {
+    RecordWriter writer = mock(RecordWriter.class);
+    Map recordWriters = mock(Map.class);
+    when(recordWriters.values()).thenReturn(Arrays.asList(writer, writer));
+    Mapper.Context taskInputOutputContext = mock(Mapper.Context.class);
+    when(taskInputOutputContext.getConfiguration()).thenReturn(createJobConf());
+    doThrow(new IOException("test IO exception")).when(writer).close(taskInputOutputContext);
+    MultipleOutputs<Long, String> mos = new MultipleOutputs<Long, String>(taskInputOutputContext);
+    mos.setRecordWriters(recordWriters);
+    mos.close();
+  }
+
   private static String localPathRoot = 
     System.getProperty("test.build.data", "/tmp");
   private static final Path ROOT_DIR = new Path(localPathRoot, "testing/mo");
@@ -85,7 +106,7 @@ public void tearDown() throws Exception {
     fs.delete(ROOT_DIR, true);
     super.tearDown();
   }
-  
+
   protected void _testMOWithJavaSerialization(boolean withCounters) throws Exception {
     String input = "a\nb\nc\nd\ne\nc\nd\ne";
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java
index bdb4ff4794ee9..14dd12ac27713 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java
@@ -36,7 +36,6 @@
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.serializer.JavaSerialization;
 import org.apache.hadoop.io.serializer.JavaSerializationComparator;
-import org.apache.hadoop.io.serializer.Serialization;
 import org.apache.hadoop.io.serializer.WritableSerialization;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.junit.Test;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
index 6115c590d5fd6..9e58d460d1783 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.mapreduce.security;
 
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -26,6 +25,8 @@
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
 
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.junit.Assert;
 
 import org.apache.hadoop.conf.Configuration;
@@ -61,7 +62,7 @@ public class TestJHSSecurity {
       LoggerFactory.getLogger(TestJHSSecurity.class);
   
   @Test
-  public void testDelegationToken() throws IOException, InterruptedException {
+  public void testDelegationToken() throws Exception {
 
     org.apache.log4j.Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
@@ -80,7 +81,7 @@ public void testDelegationToken() throws IOException, InterruptedException {
     final long renewInterval = 10000l;
 
     JobHistoryServer jobHistoryServer = null;
-    MRClientProtocol clientUsingDT = null;
+    MRClientProtocol clientUsingDT;
     long tokenFetchTime;
     try {
       jobHistoryServer = new JobHistoryServer() {
@@ -155,14 +156,11 @@ protected JHSDelegationTokenSecretManager createJHSSecretManager(
       }
       Thread.sleep(50l);
       LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
-      // Token should have expired.      
-      try {
-        clientUsingDT.getJobReport(jobReportRequest);
-        fail("Should not have succeeded with an expired token");
-      } catch (IOException e) {
-        assertTrue(e.getCause().getMessage().contains("is expired"));
-      }
-      
+      // Token should have expired.
+      final MRClientProtocol finalClientUsingDT = clientUsingDT;
+      LambdaTestUtils.intercept(SecretManager.InvalidToken.class, "has expired",
+          () -> finalClientUsingDT.getJobReport(jobReportRequest));
+
       // Test cancellation
       // Stop the existing proxy, start another.
       if (clientUsingDT != null) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java
index a031cd73ac0ed..06b0c20c3f39a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/token/delegation/TestDelegationToken.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
index 0f3916824ae6f..10cb9b85b5bca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
@@ -40,7 +40,7 @@
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import org.apache.hadoop.mapreduce.Job;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -111,7 +111,7 @@ public void testCombinerShouldUpdateTheReporter() throws Exception {
     conf.setCombinerClass(MyCombinerToCheckReporter.class);
     //conf.setJarByClass(MyCombinerToCheckReporter.class);
     conf.setReducerClass(IdentityReducer.class);
-    DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf);
+    Job.addFileToClassPath(TestMRJobs.APP_JAR, conf, TestMRJobs.APP_JAR.getFileSystem(conf));
     conf.setOutputCommitter(CustomOutputCommitter.class);
     conf.setInputFormat(TextInputFormat.class);
     conf.setOutputKeyClass(LongWritable.class);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
index e90c509d7a8af..d1e81b2b11631 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java
@@ -31,7 +31,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.net.InetAddress;
 
 import static org.junit.Assert.assertNull;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/ClassWordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/ClassWordCount.java
index 5f3abd232a18e..17fe2df86f4f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/ClassWordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/ClassWordCount.java
@@ -18,22 +18,11 @@
 
 package testjar;
 
-import java.io.*;
-import java.util.*;
-
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.WordCount;
 
 /**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index ae3b9c6029e57..4c32838afb0b4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -27,6 +27,7 @@ set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-c
 # Add extra compiler and linker flags.
 # -Wno-sign-compare
 hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing -fsigned-char")
+set(CMAKE_CXX_STANDARD 11)
 
 # Source location.
 set(SRC main/native)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 4ec1f7a1c561c..2c8d13fff1922 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.util.QuickSort;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
index da09f59591bee..6fe0143b0120a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataWriter.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * DataOutputStream implementation which buffers data in a fixed-size
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
index 48c32c0239f66..1c77aa850fa6c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestBufferPushPull.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.mapred.nativetask.handlers.IDataLoader;
 import org.apache.hadoop.mapred.nativetask.testutil.TestInput;
 import org.apache.hadoop.mapred.nativetask.testutil.TestInput.KV;
-import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
 import org.apache.hadoop.util.Progress;
 
 import org.junit.Assert;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index 4cf534b9c6f0f..aa581e5a69148 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -22,7 +22,6 @@
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.ByteWritable;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
index 9b46dfa6243bd..2235eb0685fd4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
@@ -19,7 +19,7 @@
 
 import java.util.Random;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs;
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
index 99d4a4cb42616..1f009a4919576 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
@@ -23,6 +23,9 @@
 import java.io.RandomAccessFile;
 
 import org.apache.hadoop.classification.VisibleForTesting;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.handler.stream.ChunkedFile;
 import org.apache.hadoop.io.ReadaheadPool;
 import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
 import org.apache.hadoop.io.nativeio.NativeIO;
@@ -31,8 +34,6 @@
 
 import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
 
-import org.jboss.netty.handler.stream.ChunkedFile;
-
 public class FadvisedChunkedFile extends ChunkedFile {
 
   private static final Logger LOG =
@@ -64,16 +65,16 @@ FileDescriptor getFd() {
   }
 
   @Override
-  public Object nextChunk() throws Exception {
+  public ByteBuf readChunk(ByteBufAllocator allocator) throws Exception {
     synchronized (closeLock) {
       if (fd.valid()) {
         if (manageOsCache && readaheadPool != null) {
           readaheadRequest = readaheadPool
               .readaheadStream(
-                  identifier, fd, getCurrentOffset(), readaheadLength,
-                  getEndOffset(), readaheadRequest);
+                  identifier, fd, currentOffset(), readaheadLength,
+                  endOffset(), readaheadRequest);
         }
-        return super.nextChunk();
+        return super.readChunk(allocator);
       } else {
         return null;
       }
@@ -88,12 +89,12 @@ public void close() throws Exception {
         readaheadRequest = null;
       }
       if (fd.valid() &&
-          manageOsCache && getEndOffset() - getStartOffset() > 0) {
+          manageOsCache && endOffset() - startOffset() > 0) {
         try {
           NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
               identifier,
               fd,
-              getStartOffset(), getEndOffset() - getStartOffset(),
+              startOffset(), endOffset() - startOffset(),
               POSIX_FADV_DONTNEED);
         } catch (Throwable t) {
           LOG.warn("Failed to manage OS cache for " + identifier +
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
index 1d3f162c90193..9290a282e3917 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
@@ -25,6 +25,7 @@
 import java.nio.channels.FileChannel;
 import java.nio.channels.WritableByteChannel;
 
+import io.netty.channel.DefaultFileRegion;
 import org.apache.hadoop.io.ReadaheadPool;
 import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
 import org.apache.hadoop.io.nativeio.NativeIO;
@@ -33,8 +34,6 @@
 
 import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
 
-import org.jboss.netty.channel.DefaultFileRegion;
-
 import org.apache.hadoop.classification.VisibleForTesting;
 
 public class FadvisedFileRegion extends DefaultFileRegion {
@@ -77,8 +76,8 @@ public long transferTo(WritableByteChannel target, long position)
       throws IOException {
     if (readaheadPool != null && readaheadLength > 0) {
       readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
-          getPosition() + position, readaheadLength,
-          getPosition() + getCount(), readaheadRequest);
+          position() + position, readaheadLength,
+          position() + count(), readaheadRequest);
     }
     
     if(this.shuffleTransferToAllowed) {
@@ -147,11 +146,11 @@ long customShuffleTransfer(WritableByteChannel target, long position)
 
   
   @Override
-  public void releaseExternalResources() {
+  protected void deallocate() {
     if (readaheadRequest != null) {
       readaheadRequest.cancel();
     }
-    super.releaseExternalResources();
+    super.deallocate();
   }
   
   /**
@@ -159,10 +158,10 @@ public void releaseExternalResources() {
    * we don't need the region to be cached anymore.
    */
   public void transferSuccessful() {
-    if (manageOsCache && getCount() > 0) {
+    if (manageOsCache && count() > 0) {
       try {
         NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
-            fd, getPosition(), getCount(), POSIX_FADV_DONTNEED);
+            fd, position(), count(), POSIX_FADV_DONTNEED);
       } catch (Throwable t) {
         LOG.warn("Failed to manage OS cache for " + identifier, t);
       }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/LoggingHttpResponseEncoder.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/LoggingHttpResponseEncoder.java
new file mode 100644
index 0000000000000..c7b98ce166ca6
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/LoggingHttpResponseEncoder.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.http.HttpHeaders;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+class LoggingHttpResponseEncoder extends HttpResponseEncoder {
+  private static final Logger LOG = LoggerFactory.getLogger(LoggingHttpResponseEncoder.class);
+  private final boolean logStacktraceOfEncodingMethods;
+
+  LoggingHttpResponseEncoder(boolean logStacktraceOfEncodingMethods) {
+    this.logStacktraceOfEncodingMethods = logStacktraceOfEncodingMethods;
+  }
+
+  @Override
+  public boolean acceptOutboundMessage(Object msg) throws Exception {
+    printExecutingMethod();
+    LOG.info("OUTBOUND MESSAGE: " + msg);
+    return super.acceptOutboundMessage(msg);
+  }
+
+  @Override
+  protected void encodeInitialLine(ByteBuf buf, HttpResponse response) throws Exception {
+    LOG.debug("Executing method: {}, response: {}",
+        getExecutingMethodName(), response);
+    logStacktraceIfRequired();
+    super.encodeInitialLine(buf, response);
+  }
+
+  @Override
+  protected void encode(ChannelHandlerContext ctx, Object msg,
+      List<Object> out) throws Exception {
+    LOG.debug("Encoding to channel {}: {}", ctx.channel(), msg);
+    printExecutingMethod();
+    logStacktraceIfRequired();
+    super.encode(ctx, msg, out);
+  }
+
+  @Override
+  protected void encodeHeaders(HttpHeaders headers, ByteBuf buf) {
+    printExecutingMethod();
+    super.encodeHeaders(headers, buf);
+  }
+
+  @Override
+  public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise
+      promise) throws Exception {
+    LOG.debug("Writing to channel {}: {}", ctx.channel(), msg);
+    printExecutingMethod();
+    super.write(ctx, msg, promise);
+  }
+
+  private void logStacktraceIfRequired() {
+    if (logStacktraceOfEncodingMethods) {
+      LOG.debug("Stacktrace: ", new Throwable());
+    }
+  }
+
+  private void printExecutingMethod() {
+    String methodName = getExecutingMethodName(1);
+    LOG.debug("Executing method: {}", methodName);
+  }
+
+  private String getExecutingMethodName() {
+    return getExecutingMethodName(0);
+  }
+
+  private String getExecutingMethodName(int additionalSkipFrames) {
+    try {
+      StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+      // Array items (indices):
+      // 0: java.lang.Thread.getStackTrace(...)
+      // 1: TestShuffleHandler$LoggingHttpResponseEncoder.getExecutingMethodName(...)
+      int skipFrames = 2 + additionalSkipFrames;
+      String methodName = stackTrace[skipFrames].getMethodName();
+      String className = this.getClass().getSimpleName();
+      return className + "#" + methodName;
+    } catch (Throwable t) {
+      LOG.error("Error while getting execution method name", t);
+      return "unknown";
+    }
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 448082f7fe852..e4a43f85b941a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -18,19 +18,20 @@
 
 package org.apache.hadoop.mapred;
 
+import static io.netty.buffer.Unpooled.wrappedBuffer;
+import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpMethod.GET;
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
+import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static io.netty.handler.codec.http.HttpResponseStatus.OK;
+import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.apache.hadoop.mapred.ShuffleHandler.NettyChannelHelper.*;
 import static org.fusesource.leveldbjni.JniDBFactory.asString;
 import static org.fusesource.leveldbjni.JniDBFactory.bytes;
-import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer;
-import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
-import static org.jboss.netty.handler.codec.http.HttpMethod.GET;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED;
-import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -54,6 +55,44 @@
 
 import javax.crypto.SecretKey;
 
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.ChannelOutboundHandlerAdapter;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.ChannelPromise;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.group.ChannelGroup;
+import io.netty.channel.group.DefaultChannelGroup;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.handler.codec.TooLongFrameException;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.DefaultHttpResponse;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpObjectAggregator;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpRequestDecoder;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.LastHttpContent;
+import io.netty.handler.codec.http.QueryStringDecoder;
+import io.netty.handler.ssl.SslHandler;
+import io.netty.handler.stream.ChunkedWriteHandler;
+import io.netty.handler.timeout.IdleState;
+import io.netty.handler.timeout.IdleStateEvent;
+import io.netty.handler.timeout.IdleStateHandler;
+import io.netty.util.CharsetUtil;
+import io.netty.util.concurrent.DefaultEventExecutorGroup;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataInputByteBuffer;
@@ -79,7 +118,6 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
 import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
@@ -94,42 +132,6 @@
 import org.iq80.leveldb.DB;
 import org.iq80.leveldb.DBException;
 import org.iq80.leveldb.Options;
-import org.jboss.netty.bootstrap.ServerBootstrap;
-import org.jboss.netty.buffer.ChannelBuffers;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-import org.jboss.netty.channel.ChannelHandler;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelPipelineFactory;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
-import org.jboss.netty.channel.group.ChannelGroup;
-import org.jboss.netty.channel.group.DefaultChannelGroup;
-import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.TooLongFrameException;
-import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
-import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
-import org.jboss.netty.handler.codec.http.HttpRequest;
-import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
-import org.jboss.netty.handler.codec.http.HttpResponse;
-import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.jboss.netty.handler.codec.http.QueryStringDecoder;
-import org.jboss.netty.handler.ssl.SslHandler;
-import org.jboss.netty.handler.stream.ChunkedWriteHandler;
-import org.jboss.netty.handler.timeout.IdleState;
-import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
-import org.jboss.netty.handler.timeout.IdleStateEvent;
-import org.jboss.netty.handler.timeout.IdleStateHandler;
-import org.jboss.netty.util.CharsetUtil;
-import org.jboss.netty.util.HashedWheelTimer;
-import org.jboss.netty.util.Timer;
 import org.eclipse.jetty.http.HttpHeader;
 import org.slf4j.LoggerFactory;
 
@@ -182,19 +184,29 @@ public class ShuffleHandler extends AuxiliaryService {
 
   public static final HttpResponseStatus TOO_MANY_REQ_STATUS =
       new HttpResponseStatus(429, "TOO MANY REQUESTS");
-  // This should kept in sync with Fetcher.FETCH_RETRY_DELAY_DEFAULT
+  // This should be kept in sync with Fetcher.FETCH_RETRY_DELAY_DEFAULT
   public static final long FETCH_RETRY_DELAY = 1000L;
   public static final String RETRY_AFTER_HEADER = "Retry-After";
+  static final String ENCODER_HANDLER_NAME = "encoder";
 
   private int port;
-  private ChannelFactory selector;
-  private final ChannelGroup accepted = new DefaultChannelGroup();
+  private EventLoopGroup bossGroup;
+  private EventLoopGroup workerGroup;
+  private ServerBootstrap bootstrap;
+  private Channel ch;
+  private final ChannelGroup accepted =
+      new DefaultChannelGroup(new DefaultEventExecutorGroup(5).next());
+  private final AtomicInteger activeConnections = new AtomicInteger();
   protected HttpPipelineFactory pipelineFact;
   private int sslFileBufferSize;
+
+  //TODO snemeth add a config option for these later, this is temporarily disabled for now.
+  private boolean useOutboundExceptionHandler = false;
+  private boolean useOutboundLogger = false;
   
   /**
    * Should the shuffle use posix_fadvise calls to manage the OS cache during
-   * sendfile
+   * sendfile.
    */
   private boolean manageOsCache;
   private int readaheadLength;
@@ -204,7 +216,7 @@ public class ShuffleHandler extends AuxiliaryService {
   private int maxSessionOpenFiles;
   private ReadaheadPool readaheadPool = ReadaheadPool.getInstance();
 
-  private Map<String,String> userRsrc;
+  private Map<String, String> userRsrc;
   private JobTokenSecretManager secretManager;
 
   private DB stateDb = null;
@@ -235,7 +247,7 @@ public class ShuffleHandler extends AuxiliaryService {
   public static final String CONNECTION_CLOSE = "close";
 
   public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
-    "mapreduce.shuffle.ssl.file.buffer.size";
+      "mapreduce.shuffle.ssl.file.buffer.size";
 
   public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
 
@@ -255,7 +267,7 @@ public class ShuffleHandler extends AuxiliaryService {
   public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
   public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = 
       false;
-  private static final String TIMEOUT_HANDLER = "timeout";
+  static final String TIMEOUT_HANDLER = "timeout";
 
   /* the maximum number of files a single GET request can
    open simultaneously during shuffle
@@ -267,7 +279,6 @@ public class ShuffleHandler extends AuxiliaryService {
   boolean connectionKeepAliveEnabled = false;
   private int connectionKeepAliveTimeOut;
   private int mapOutputMetaInfoCacheSize;
-  private Timer timer;
 
   @Metrics(about="Shuffle output metrics", context="mapred")
   static class ShuffleMetrics implements ChannelFutureListener {
@@ -291,6 +302,49 @@ public void operationComplete(ChannelFuture future) throws Exception {
     }
   }
 
+  static class NettyChannelHelper {
+    static ChannelFuture writeToChannel(Channel ch, Object obj) {
+      LOG.debug("Writing {} to channel: {}", obj.getClass().getSimpleName(), ch.id());
+      return ch.writeAndFlush(obj);
+    }
+
+    static ChannelFuture writeToChannelAndClose(Channel ch, Object obj) {
+      return writeToChannel(ch, obj).addListener(ChannelFutureListener.CLOSE);
+    }
+
+    static ChannelFuture writeToChannelAndAddLastHttpContent(Channel ch, HttpResponse obj) {
+      writeToChannel(ch, obj);
+      return writeLastHttpContentToChannel(ch);
+    }
+
+    static ChannelFuture writeLastHttpContentToChannel(Channel ch) {
+      LOG.debug("Writing LastHttpContent, channel id: {}", ch.id());
+      return ch.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
+    }
+
+    static ChannelFuture closeChannel(Channel ch) {
+      LOG.debug("Closing channel, channel id: {}", ch.id());
+      return ch.close();
+    }
+
+    static void closeChannels(ChannelGroup channelGroup) {
+      channelGroup.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
+    }
+
+    public static ChannelFuture closeAsIdle(Channel ch, int timeout) {
+      LOG.debug("Closing channel as writer was idle for {} seconds", timeout);
+      return closeChannel(ch);
+    }
+
+    public static void channelActive(Channel ch) {
+      LOG.debug("Executing channelActive, channel id: {}", ch.id());
+    }
+
+    public static void channelInactive(Channel ch) {
+      LOG.debug("Executing channelInactive, channel id: {}", ch.id());
+    }
+  }
+
   private final MetricsSystem ms;
   final ShuffleMetrics metrics;
 
@@ -298,29 +352,36 @@ class ReduceMapFileCount implements ChannelFutureListener {
 
     private ReduceContext reduceContext;
 
-    public ReduceMapFileCount(ReduceContext rc) {
+    ReduceMapFileCount(ReduceContext rc) {
       this.reduceContext = rc;
     }
 
     @Override
     public void operationComplete(ChannelFuture future) throws Exception {
+      LOG.trace("operationComplete");
       if (!future.isSuccess()) {
-        future.getChannel().close();
+        LOG.error("Future is unsuccessful. Cause: ", future.cause());
+        closeChannel(future.channel());
         return;
       }
       int waitCount = this.reduceContext.getMapsToWait().decrementAndGet();
       if (waitCount == 0) {
+        LOG.trace("Finished with all map outputs");
+        //HADOOP-15327: Need to send an instance of LastHttpContent to define HTTP
+        //message boundaries. See details in jira.
+        writeLastHttpContentToChannel(future.channel());
         metrics.operationComplete(future);
         // Let the idle timer handler close keep-alive connections
         if (reduceContext.getKeepAlive()) {
-          ChannelPipeline pipeline = future.getChannel().getPipeline();
+          ChannelPipeline pipeline = future.channel().pipeline();
           TimeoutHandler timeoutHandler =
               (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER);
           timeoutHandler.setEnabledTimeout(true);
         } else {
-          future.getChannel().close();
+          closeChannel(future.channel());
         }
       } else {
+        LOG.trace("operationComplete, waitCount > 0, invoking sendMap with reduceContext");
         pipelineFact.getSHUFFLE().sendMap(reduceContext);
       }
     }
@@ -331,7 +392,6 @@ public void operationComplete(ChannelFuture future) throws Exception {
    * Allows sendMapOutput calls from operationComplete()
    */
   private static class ReduceContext {
-
     private List<String> mapIds;
     private AtomicInteger mapsToWait;
     private AtomicInteger mapsToSend;
@@ -342,7 +402,7 @@ private static class ReduceContext {
     private String jobId;
     private final boolean keepAlive;
 
-    public ReduceContext(List<String> mapIds, int rId,
+    ReduceContext(List<String> mapIds, int rId,
                          ChannelHandlerContext context, String usr,
                          Map<String, Shuffle.MapOutputInfo> mapOutputInfoMap,
                          String jobId, boolean keepAlive) {
@@ -448,7 +508,8 @@ public static int deserializeMetaData(ByteBuffer meta) throws IOException {
    * shuffle data requests.
    * @return the serialized version of the jobToken.
    */
-  public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException {
+  public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken)
+      throws IOException {
     //TODO these bytes should be versioned
     DataOutputBuffer jobToken_dob = new DataOutputBuffer();
     jobToken.write(jobToken_dob);
@@ -505,6 +566,11 @@ protected void serviceInit(Configuration conf) throws Exception {
                                         DEFAULT_MAX_SHUFFLE_CONNECTIONS);
     int maxShuffleThreads = conf.getInt(MAX_SHUFFLE_THREADS,
                                         DEFAULT_MAX_SHUFFLE_THREADS);
+    // Since Netty 4.x, the value of 0 threads would default to:
+    // io.netty.channel.MultithreadEventLoopGroup.DEFAULT_EVENT_LOOP_THREADS
+    // by simply passing 0 value to NioEventLoopGroup constructor below.
+    // However, this logic to determinte thread count
+    // was in place so we can keep it for now.
     if (maxShuffleThreads == 0) {
       maxShuffleThreads = 2 * Runtime.getRuntime().availableProcessors();
     }
@@ -520,16 +586,14 @@ protected void serviceInit(Configuration conf) throws Exception {
         DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES);
 
     ThreadFactory bossFactory = new ThreadFactoryBuilder()
-      .setNameFormat("ShuffleHandler Netty Boss #%d")
-      .build();
+        .setNameFormat("ShuffleHandler Netty Boss #%d")
+        .build();
     ThreadFactory workerFactory = new ThreadFactoryBuilder()
-      .setNameFormat("ShuffleHandler Netty Worker #%d")
-      .build();
+        .setNameFormat("ShuffleHandler Netty Worker #%d")
+        .build();
     
-    selector = new NioServerSocketChannelFactory(
-        HadoopExecutors.newCachedThreadPool(bossFactory),
-        HadoopExecutors.newCachedThreadPool(workerFactory),
-        maxShuffleThreads);
+    bossGroup = new NioEventLoopGroup(maxShuffleThreads, bossFactory);
+    workerGroup = new NioEventLoopGroup(maxShuffleThreads, workerFactory);
     super.serviceInit(new Configuration(conf));
   }
 
@@ -540,22 +604,24 @@ protected void serviceStart() throws Exception {
     userRsrc = new ConcurrentHashMap<String,String>();
     secretManager = new JobTokenSecretManager();
     recoverState(conf);
-    ServerBootstrap bootstrap = new ServerBootstrap(selector);
-    // Timer is shared across entire factory and must be released separately
-    timer = new HashedWheelTimer();
     try {
-      pipelineFact = new HttpPipelineFactory(conf, timer);
+      pipelineFact = new HttpPipelineFactory(conf);
     } catch (Exception ex) {
       throw new RuntimeException(ex);
     }
-    bootstrap.setOption("backlog", conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE,
-        DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE));
-    bootstrap.setOption("child.keepAlive", true);
-    bootstrap.setPipelineFactory(pipelineFact);
+
+    bootstrap = new ServerBootstrap();
+    bootstrap.group(bossGroup, workerGroup)
+        .channel(NioServerSocketChannel.class)
+        .option(ChannelOption.SO_BACKLOG,
+            conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE,
+                DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE))
+        .childOption(ChannelOption.SO_KEEPALIVE, true)
+        .childHandler(pipelineFact);
     port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
-    Channel ch = bootstrap.bind(new InetSocketAddress(port));
+    ch = bootstrap.bind(new InetSocketAddress(port)).sync().channel();
     accepted.add(ch);
-    port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
+    port = ((InetSocketAddress)ch.localAddress()).getPort();
     conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
     pipelineFact.SHUFFLE.setPort(port);
     LOG.info(getName() + " listening on port " + port);
@@ -576,18 +642,12 @@ protected void serviceStart() throws Exception {
 
   @Override
   protected void serviceStop() throws Exception {
-    accepted.close().awaitUninterruptibly(10, TimeUnit.SECONDS);
-    if (selector != null) {
-      ServerBootstrap bootstrap = new ServerBootstrap(selector);
-      bootstrap.releaseExternalResources();
-    }
+    closeChannels(accepted);
+
     if (pipelineFact != null) {
       pipelineFact.destroy();
     }
-    if (timer != null) {
-      // Release this shared timer resource
-      timer.stop();
-    }
+
     if (stateDb != null) {
       stateDb.close();
     }
@@ -744,7 +804,7 @@ private void recoverJobShuffleInfo(String jobIdStr, byte[] data)
     JobShuffleInfoProto proto = JobShuffleInfoProto.parseFrom(data);
     String user = proto.getUser();
     TokenProto tokenProto = proto.getJobToken();
-    Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>(
+    Token<JobTokenIdentifier> jobToken = new Token<>(
         tokenProto.getIdentifier().toByteArray(),
         tokenProto.getPassword().toByteArray(),
         new Text(tokenProto.getKind()), new Text(tokenProto.getService()));
@@ -785,29 +845,47 @@ private void removeJobShuffleInfo(JobID jobId) throws IOException {
     }
   }
 
-  static class TimeoutHandler extends IdleStateAwareChannelHandler {
+  @VisibleForTesting
+  public void setUseOutboundExceptionHandler(boolean useHandler) {
+    this.useOutboundExceptionHandler = useHandler;
+  }
 
+  static class TimeoutHandler extends IdleStateHandler {
+    private final int connectionKeepAliveTimeOut;
     private boolean enabledTimeout;
 
+    TimeoutHandler(int connectionKeepAliveTimeOut) {
+      //disable reader timeout
+      //set writer timeout to configured timeout value
+      //disable all idle timeout
+      super(0, connectionKeepAliveTimeOut, 0, TimeUnit.SECONDS);
+      this.connectionKeepAliveTimeOut = connectionKeepAliveTimeOut;
+    }
+
+    @VisibleForTesting
+    public int getConnectionKeepAliveTimeOut() {
+      return connectionKeepAliveTimeOut;
+    }
+
     void setEnabledTimeout(boolean enabledTimeout) {
       this.enabledTimeout = enabledTimeout;
     }
 
     @Override
     public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) {
-      if (e.getState() == IdleState.WRITER_IDLE && enabledTimeout) {
-        e.getChannel().close();
+      if (e.state() == IdleState.WRITER_IDLE && enabledTimeout) {
+        closeAsIdle(ctx.channel(), connectionKeepAliveTimeOut);
       }
     }
   }
 
-  class HttpPipelineFactory implements ChannelPipelineFactory {
+  class HttpPipelineFactory extends ChannelInitializer<SocketChannel> {
+    private static final int MAX_CONTENT_LENGTH = 1 << 16;
 
     final Shuffle SHUFFLE;
     private SSLFactory sslFactory;
-    private final ChannelHandler idleStateHandler;
 
-    public HttpPipelineFactory(Configuration conf, Timer timer) throws Exception {
+    HttpPipelineFactory(Configuration conf) throws Exception {
       SHUFFLE = getShuffle(conf);
       if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY,
                           MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) {
@@ -815,7 +893,6 @@ public HttpPipelineFactory(Configuration conf, Timer timer) throws Exception {
         sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf);
         sslFactory.init();
       }
-      this.idleStateHandler = new IdleStateHandler(timer, 0, connectionKeepAliveTimeOut, 0);
     }
 
     public Shuffle getSHUFFLE() {
@@ -828,30 +905,39 @@ public void destroy() {
       }
     }
 
-    @Override
-    public ChannelPipeline getPipeline() throws Exception {
-      ChannelPipeline pipeline = Channels.pipeline();
+    @Override protected void initChannel(SocketChannel ch) throws Exception {
+      ChannelPipeline pipeline = ch.pipeline();
       if (sslFactory != null) {
         pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
       }
       pipeline.addLast("decoder", new HttpRequestDecoder());
-      pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
-      pipeline.addLast("encoder", new HttpResponseEncoder());
+      pipeline.addLast("aggregator", new HttpObjectAggregator(MAX_CONTENT_LENGTH));
+      pipeline.addLast(ENCODER_HANDLER_NAME, useOutboundLogger ?
+          new LoggingHttpResponseEncoder(false) : new HttpResponseEncoder());
       pipeline.addLast("chunking", new ChunkedWriteHandler());
       pipeline.addLast("shuffle", SHUFFLE);
-      pipeline.addLast("idle", idleStateHandler);
-      pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler());
-      return pipeline;
+      if (useOutboundExceptionHandler) {
+        //https://stackoverflow.com/questions/50612403/catch-all-exception-handling-for-outbound-channelhandler
+        pipeline.addLast("outboundExceptionHandler", new ChannelOutboundHandlerAdapter() {
+          @Override
+          public void write(ChannelHandlerContext ctx, Object msg,
+              ChannelPromise promise) throws Exception {
+            promise.addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE);
+            super.write(ctx, msg, promise);
+          }
+        });
+      }
+      pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler(connectionKeepAliveTimeOut));
       // TODO factor security manager into pipeline
       // TODO factor out encode/decode to permit binary shuffle
       // TODO factor out decode of index to permit alt. models
     }
   }
 
-  class Shuffle extends SimpleChannelUpstreamHandler {
+  @ChannelHandler.Sharable
+  class Shuffle extends ChannelInboundHandlerAdapter {
     private final IndexCache indexCache;
-    private final
-    LoadingCache<AttemptPathIdentifier, AttemptPathInfo> pathCache;
+    private final LoadingCache<AttemptPathIdentifier, AttemptPathInfo> pathCache;
 
     private int port;
 
@@ -904,65 +990,84 @@ private List<String> splitMaps(List<String> mapq) {
     }
 
     @Override
-    public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt) 
+    public void channelActive(ChannelHandlerContext ctx)
         throws Exception {
-      super.channelOpen(ctx, evt);
-
-      if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) {
+      NettyChannelHelper.channelActive(ctx.channel());
+      int numConnections = activeConnections.incrementAndGet();
+      if ((maxShuffleConnections > 0) && (numConnections > maxShuffleConnections)) {
         LOG.info(String.format("Current number of shuffle connections (%d) is " + 
-            "greater than or equal to the max allowed shuffle connections (%d)", 
+            "greater than the max allowed shuffle connections (%d)",
             accepted.size(), maxShuffleConnections));
 
-        Map<String, String> headers = new HashMap<String, String>(1);
+        Map<String, String> headers = new HashMap<>(1);
         // notify fetchers to backoff for a while before closing the connection
         // if the shuffle connection limit is hit. Fetchers are expected to
         // handle this notification gracefully, that is, not treating this as a
         // fetch failure.
         headers.put(RETRY_AFTER_HEADER, String.valueOf(FETCH_RETRY_DELAY));
         sendError(ctx, "", TOO_MANY_REQ_STATUS, headers);
-        return;
+      } else {
+        super.channelActive(ctx);
+        accepted.add(ctx.channel());
+        LOG.debug("Added channel: {}, channel id: {}. Accepted number of connections={}",
+            ctx.channel(), ctx.channel().id(), activeConnections.get());
       }
-      accepted.add(evt.getChannel());
     }
 
     @Override
-    public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
+    public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+      NettyChannelHelper.channelInactive(ctx.channel());
+      super.channelInactive(ctx);
+      int noOfConnections = activeConnections.decrementAndGet();
+      LOG.debug("New value of Accepted number of connections={}", noOfConnections);
+    }
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg)
         throws Exception {
-      HttpRequest request = (HttpRequest) evt.getMessage();
-      if (request.getMethod() != GET) {
-          sendError(ctx, METHOD_NOT_ALLOWED);
-          return;
+      Channel channel = ctx.channel();
+      LOG.trace("Executing channelRead, channel id: {}", channel.id());
+      HttpRequest request = (HttpRequest) msg;
+      LOG.debug("Received HTTP request: {}, channel id: {}", request, channel.id());
+      if (request.method() != GET) {
+        sendError(ctx, METHOD_NOT_ALLOWED);
+        return;
       }
       // Check whether the shuffle version is compatible
-      if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals(
-          request.headers() != null ?
-              request.headers().get(ShuffleHeader.HTTP_HEADER_NAME) : null)
-          || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals(
-              request.headers() != null ?
-                  request.headers()
-                      .get(ShuffleHeader.HTTP_HEADER_VERSION) : null)) {
+      String shuffleVersion = ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION;
+      String httpHeaderName = ShuffleHeader.DEFAULT_HTTP_HEADER_NAME;
+      if (request.headers() != null) {
+        shuffleVersion = request.headers().get(ShuffleHeader.HTTP_HEADER_VERSION);
+        httpHeaderName = request.headers().get(ShuffleHeader.HTTP_HEADER_NAME);
+        LOG.debug("Received from request header: ShuffleVersion={} header name={}, channel id: {}",
+            shuffleVersion, httpHeaderName, channel.id());
+      }
+      if (request.headers() == null ||
+          !ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals(httpHeaderName) ||
+          !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals(shuffleVersion)) {
         sendError(ctx, "Incompatible shuffle request version", BAD_REQUEST);
       }
-      final Map<String,List<String>> q =
-        new QueryStringDecoder(request.getUri()).getParameters();
+      final Map<String, List<String>> q =
+          new QueryStringDecoder(request.uri()).parameters();
       final List<String> keepAliveList = q.get("keepAlive");
       boolean keepAliveParam = false;
       if (keepAliveList != null && keepAliveList.size() == 1) {
         keepAliveParam = Boolean.valueOf(keepAliveList.get(0));
         if (LOG.isDebugEnabled()) {
-          LOG.debug("KeepAliveParam : " + keepAliveList
-            + " : " + keepAliveParam);
+          LOG.debug("KeepAliveParam: {} : {}, channel id: {}",
+              keepAliveList, keepAliveParam, channel.id());
         }
       }
       final List<String> mapIds = splitMaps(q.get("map"));
       final List<String> reduceQ = q.get("reduce");
       final List<String> jobQ = q.get("job");
       if (LOG.isDebugEnabled()) {
-        LOG.debug("RECV: " + request.getUri() +
+        LOG.debug("RECV: " + request.uri() +
             "\n  mapId: " + mapIds +
             "\n  reduceId: " + reduceQ +
             "\n  jobId: " + jobQ +
-            "\n  keepAlive: " + keepAliveParam);
+            "\n  keepAlive: " + keepAliveParam +
+            "\n  channel id: " + channel.id());
       }
 
       if (mapIds == null || reduceQ == null || jobQ == null) {
@@ -986,7 +1091,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
         sendError(ctx, "Bad job parameter", BAD_REQUEST);
         return;
       }
-      final String reqUri = request.getUri();
+      final String reqUri = request.uri();
       if (null == reqUri) {
         // TODO? add upstream?
         sendError(ctx, FORBIDDEN);
@@ -1004,8 +1109,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
 
       Map<String, MapOutputInfo> mapOutputInfoMap =
           new HashMap<String, MapOutputInfo>();
-      Channel ch = evt.getChannel();
-      ChannelPipeline pipeline = ch.getPipeline();
+      ChannelPipeline pipeline = channel.pipeline();
       TimeoutHandler timeoutHandler =
           (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER);
       timeoutHandler.setEnabledTimeout(false);
@@ -1013,16 +1117,29 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
 
       try {
         populateHeaders(mapIds, jobId, user, reduceId, request,
-          response, keepAliveParam, mapOutputInfoMap);
+            response, keepAliveParam, mapOutputInfoMap);
       } catch(IOException e) {
-        ch.write(response);
-        LOG.error("Shuffle error in populating headers :", e);
-        String errorMessage = getErrorMessage(e);
-        sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR);
+        //HADOOP-15327
+        // Need to send an instance of LastHttpContent to define HTTP
+        // message boundaries.
+        //Sending a HTTP 200 OK + HTTP 500 later (sendError)
+        // is quite a non-standard way of crafting HTTP responses,
+        // but we need to keep backward compatibility.
+        // See more details in jira.
+        writeToChannelAndAddLastHttpContent(channel, response);
+        LOG.error("Shuffle error while populating headers. Channel id: " + channel.id(), e);
+        sendError(ctx, getErrorMessage(e), INTERNAL_SERVER_ERROR);
         return;
       }
-      ch.write(response);
-      //Initialize one ReduceContext object per messageReceived call
+      writeToChannel(channel, response).addListener((ChannelFutureListener) future -> {
+        if (future.isSuccess()) {
+          LOG.debug("Written HTTP response object successfully. Channel id: {}", channel.id());
+        } else {
+          LOG.error("Error while writing HTTP response object: {}. " +
+              "Cause: {}, channel id: {}", response, future.cause(), channel.id());
+        }
+      });
+      //Initialize one ReduceContext object per channelRead call
       boolean keepAlive = keepAliveParam || connectionKeepAliveEnabled;
       ReduceContext reduceContext = new ReduceContext(mapIds, reduceId, ctx,
           user, mapOutputInfoMap, jobId, keepAlive);
@@ -1044,9 +1161,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt)
      * @param reduceContext used to call sendMapOutput with correct params.
      * @return the ChannelFuture of the sendMapOutput, can be null.
      */
-    public ChannelFuture sendMap(ReduceContext reduceContext)
-        throws Exception {
-
+    public ChannelFuture sendMap(ReduceContext reduceContext) {
+      LOG.trace("Executing sendMap");
       ChannelFuture nextMap = null;
       if (reduceContext.getMapsToSend().get() <
           reduceContext.getMapIds().size()) {
@@ -1059,21 +1175,24 @@ public ChannelFuture sendMap(ReduceContext reduceContext)
             info = getMapOutputInfo(mapId, reduceContext.getReduceId(),
                 reduceContext.getJobId(), reduceContext.getUser());
           }
+          LOG.trace("Calling sendMapOutput");
           nextMap = sendMapOutput(
               reduceContext.getCtx(),
-              reduceContext.getCtx().getChannel(),
+              reduceContext.getCtx().channel(),
               reduceContext.getUser(), mapId,
               reduceContext.getReduceId(), info);
-          if (null == nextMap) {
+          if (nextMap == null) {
+            //This can only happen if spill file was not found
             sendError(reduceContext.getCtx(), NOT_FOUND);
+            LOG.trace("Returning nextMap: null");
             return null;
           }
           nextMap.addListener(new ReduceMapFileCount(reduceContext));
         } catch (IOException e) {
           if (e instanceof DiskChecker.DiskErrorException) {
-            LOG.error("Shuffle error :" + e);
+            LOG.error("Shuffle error: " + e);
           } else {
-            LOG.error("Shuffle error :", e);
+            LOG.error("Shuffle error: ", e);
           }
           String errorMessage = getErrorMessage(e);
           sendError(reduceContext.getCtx(), errorMessage,
@@ -1125,8 +1244,7 @@ protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
         }
       }
 
-      IndexRecord info =
-        indexCache.getIndexInformation(mapId, reduce, pathInfo.indexPath, user);
+      IndexRecord info = indexCache.getIndexInformation(mapId, reduce, pathInfo.indexPath, user);
 
       if (LOG.isDebugEnabled()) {
         LOG.debug("getMapOutputInfo: jobId=" + jobId + ", mapId=" + mapId +
@@ -1155,7 +1273,6 @@ protected void populateHeaders(List<String> mapIds, String jobId,
             outputInfo.indexRecord.rawLength, reduce);
         DataOutputBuffer dob = new DataOutputBuffer();
         header.write(dob);
-
         contentLength += outputInfo.indexRecord.partLength;
         contentLength += dob.getLength();
       }
@@ -1183,14 +1300,10 @@ protected void populateHeaders(List<String> mapIds, String jobId,
     protected void setResponseHeaders(HttpResponse response,
         boolean keepAliveParam, long contentLength) {
       if (!connectionKeepAliveEnabled && !keepAliveParam) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Setting connection close header...");
-        }
-        response.headers().set(HttpHeader.CONNECTION.asString(),
-            CONNECTION_CLOSE);
+        response.headers().set(HttpHeader.CONNECTION.asString(), CONNECTION_CLOSE);
       } else {
         response.headers().set(HttpHeader.CONTENT_LENGTH.asString(),
-          String.valueOf(contentLength));
+            String.valueOf(contentLength));
         response.headers().set(HttpHeader.CONNECTION.asString(),
             HttpHeader.KEEP_ALIVE.asString());
         response.headers().set(HttpHeader.KEEP_ALIVE.asString(),
@@ -1214,29 +1327,29 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
         throws IOException {
       SecretKey tokenSecret = secretManager.retrieveTokenSecret(appid);
       if (null == tokenSecret) {
-        LOG.info("Request for unknown token " + appid);
-        throw new IOException("could not find jobid");
+        LOG.info("Request for unknown token {}, channel id: {}", appid, ctx.channel().id());
+        throw new IOException("Could not find jobid");
       }
-      // string to encrypt
-      String enc_str = SecureShuffleUtils.buildMsgFrom(requestUri);
+      // encrypting URL
+      String encryptedURL = SecureShuffleUtils.buildMsgFrom(requestUri);
       // hash from the fetcher
       String urlHashStr =
           request.headers().get(SecureShuffleUtils.HTTP_HEADER_URL_HASH);
       if (urlHashStr == null) {
-        LOG.info("Missing header hash for " + appid);
+        LOG.info("Missing header hash for {}, channel id: {}", appid, ctx.channel().id());
         throw new IOException("fetcher cannot be authenticated");
       }
       if (LOG.isDebugEnabled()) {
         int len = urlHashStr.length();
-        LOG.debug("verifying request. enc_str=" + enc_str + "; hash=..." +
-            urlHashStr.substring(len-len/2, len-1));
+        LOG.debug("Verifying request. encryptedURL:{}, hash:{}, channel id: " +
+                "{}", encryptedURL,
+            urlHashStr.substring(len - len / 2, len - 1), ctx.channel().id());
       }
       // verify - throws exception
-      SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret);
+      SecureShuffleUtils.verifyReply(urlHashStr, encryptedURL, tokenSecret);
       // verification passed - encode the reply
-      String reply =
-        SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), 
-            tokenSecret);
+      String reply = SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8),
+          tokenSecret);
       response.headers().set(
           SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply);
       // Put shuffle version into http header
@@ -1246,8 +1359,10 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
           ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
       if (LOG.isDebugEnabled()) {
         int len = reply.length();
-        LOG.debug("Fetcher request verfied. enc_str=" + enc_str + ";reply=" +
-            reply.substring(len-len/2, len-1));
+        LOG.debug("Fetcher request verified. " +
+            "encryptedURL: {}, reply: {}, channel id: {}",
+            encryptedURL, reply.substring(len - len / 2, len - 1),
+            ctx.channel().id());
       }
     }
 
@@ -1255,27 +1370,27 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
         String user, String mapId, int reduce, MapOutputInfo mapOutputInfo)
         throws IOException {
       final IndexRecord info = mapOutputInfo.indexRecord;
-      final ShuffleHeader header =
-        new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce);
+      final ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
+          reduce);
       final DataOutputBuffer dob = new DataOutputBuffer();
       header.write(dob);
-      ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+      writeToChannel(ch, wrappedBuffer(dob.getData(), 0, dob.getLength()));
       final File spillfile =
           new File(mapOutputInfo.mapOutputFileName.toString());
       RandomAccessFile spill;
       try {
         spill = SecureIOUtils.openForRandomRead(spillfile, "r", user, null);
       } catch (FileNotFoundException e) {
-        LOG.info(spillfile + " not found");
+        LOG.info("{} not found. Channel id: {}", spillfile, ctx.channel().id());
         return null;
       }
       ChannelFuture writeFuture;
-      if (ch.getPipeline().get(SslHandler.class) == null) {
+      if (ch.pipeline().get(SslHandler.class) == null) {
         final FadvisedFileRegion partition = new FadvisedFileRegion(spill,
             info.startOffset, info.partLength, manageOsCache, readaheadLength,
             readaheadPool, spillfile.getAbsolutePath(), 
             shuffleBufferSize, shuffleTransferToAllowed);
-        writeFuture = ch.write(partition);
+        writeFuture = writeToChannel(ch, partition);
         writeFuture.addListener(new ChannelFutureListener() {
             // TODO error handling; distinguish IO/connection failures,
             //      attribute to appropriate spill output
@@ -1284,7 +1399,7 @@ public void operationComplete(ChannelFuture future) {
             if (future.isSuccess()) {
               partition.transferSuccessful();
             }
-            partition.releaseExternalResources();
+            partition.deallocate();
           }
         });
       } else {
@@ -1293,7 +1408,7 @@ public void operationComplete(ChannelFuture future) {
             info.startOffset, info.partLength, sslFileBufferSize,
             manageOsCache, readaheadLength, readaheadPool,
             spillfile.getAbsolutePath());
-        writeFuture = ch.write(chunk);
+        writeFuture = writeToChannel(ch, chunk);
       }
       metrics.shuffleConnections.incr();
       metrics.shuffleOutputBytes.incr(info.partLength); // optimistic
@@ -1307,12 +1422,13 @@ protected void sendError(ChannelHandlerContext ctx,
 
     protected void sendError(ChannelHandlerContext ctx, String message,
         HttpResponseStatus status) {
-      sendError(ctx, message, status, Collections.<String, String>emptyMap());
+      sendError(ctx, message, status, Collections.emptyMap());
     }
 
     protected void sendError(ChannelHandlerContext ctx, String msg,
         HttpResponseStatus status, Map<String, String> headers) {
-      HttpResponse response = new DefaultHttpResponse(HTTP_1_1, status);
+      FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, status,
+              Unpooled.copiedBuffer(msg, CharsetUtil.UTF_8));
       response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8");
       // Put shuffle version into http header
       response.headers().set(ShuffleHeader.HTTP_HEADER_NAME,
@@ -1322,48 +1438,45 @@ protected void sendError(ChannelHandlerContext ctx, String msg,
       for (Map.Entry<String, String> header : headers.entrySet()) {
         response.headers().set(header.getKey(), header.getValue());
       }
-      response.setContent(
-          ChannelBuffers.copiedBuffer(msg, CharsetUtil.UTF_8));
 
       // Close the connection as soon as the error message is sent.
-      ctx.getChannel().write(response).addListener(ChannelFutureListener.CLOSE);
+      writeToChannelAndClose(ctx.channel(), response);
     }
 
     @Override
-    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e)
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
         throws Exception {
-      Channel ch = e.getChannel();
-      Throwable cause = e.getCause();
+      Channel ch = ctx.channel();
       if (cause instanceof TooLongFrameException) {
+        LOG.trace("TooLongFrameException, channel id: {}", ch.id());
         sendError(ctx, BAD_REQUEST);
         return;
       } else if (cause instanceof IOException) {
         if (cause instanceof ClosedChannelException) {
-          LOG.debug("Ignoring closed channel error", cause);
+          LOG.debug("Ignoring closed channel error, channel id: " + ch.id(), cause);
           return;
         }
         String message = String.valueOf(cause.getMessage());
         if (IGNORABLE_ERROR_MESSAGE.matcher(message).matches()) {
-          LOG.debug("Ignoring client socket close", cause);
+          LOG.debug("Ignoring client socket close, channel id: " + ch.id(), cause);
           return;
         }
       }
 
-      LOG.error("Shuffle error: ", cause);
-      if (ch.isConnected()) {
-        LOG.error("Shuffle error " + e);
+      LOG.error("Shuffle error. Channel id: " + ch.id(), cause);
+      if (ch.isActive()) {
         sendError(ctx, INTERNAL_SERVER_ERROR);
       }
     }
   }
-  
+
   static class AttemptPathInfo {
     // TODO Change this over to just store local dir indices, instead of the
     // entire path. Far more efficient.
     private final Path indexPath;
     private final Path dataPath;
 
-    public AttemptPathInfo(Path indexPath, Path dataPath) {
+    AttemptPathInfo(Path indexPath, Path dataPath) {
       this.indexPath = indexPath;
       this.dataPath = dataPath;
     }
@@ -1374,7 +1487,7 @@ static class AttemptPathIdentifier {
     private final String user;
     private final String attemptId;
 
-    public AttemptPathIdentifier(String jobId, String user, String attemptId) {
+    AttemptPathIdentifier(String jobId, String user, String attemptId) {
       this.jobId = jobId;
       this.user = user;
       this.attemptId = attemptId;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
index 242382e06a041..ce0c0d6aeafe9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
@@ -104,7 +104,7 @@ public void testCustomShuffleTransfer() throws IOException {
       Assert.assertEquals(count, targetFile.length());
     } finally {
       if (fileRegion != null) {
-        fileRegion.releaseExternalResources();
+        fileRegion.deallocate();
       }
       IOUtils.cleanupWithLogger(LOG, target);
       IOUtils.cleanupWithLogger(LOG, targetFile);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index af3cb87760c62..38500032ef31d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -17,34 +17,65 @@
  */
 package org.apache.hadoop.mapred;
 
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.DefaultFileRegion;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import io.netty.channel.AbstractChannel;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPipeline;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.timeout.IdleStateEvent;
 import org.apache.hadoop.test.GenericTestUtils;
+
+import static io.netty.buffer.Unpooled.wrappedBuffer;
+import static java.util.stream.Collectors.toList;
 import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
 import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
-import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer;
-import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
-import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.Socket;
 import java.net.URL;
 import java.net.SocketAddress;
+import java.net.URLConnection;
 import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
 import java.util.zip.CheckedOutputStream;
 import java.util.zip.Checksum;
 
@@ -71,6 +102,7 @@
 import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.util.DiskChecker;
 import org.apache.hadoop.util.PureJavaCrc32;
+import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -79,22 +111,13 @@
 import org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.records.Version;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.socket.SocketChannel;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.AbstractChannel;
-import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
-import org.jboss.netty.handler.codec.http.HttpRequest;
-import org.jboss.netty.handler.codec.http.HttpResponse;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.jboss.netty.handler.codec.http.HttpMethod;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
+import org.junit.rules.TestName;
 import org.mockito.Mockito;
 import org.eclipse.jetty.http.HttpHeader;
 import org.slf4j.Logger;
@@ -106,10 +129,583 @@ public class TestShuffleHandler {
       LoggerFactory.getLogger(TestShuffleHandler.class);
   private static final File ABS_LOG_DIR = GenericTestUtils.getTestDir(
       TestShuffleHandler.class.getSimpleName() + "LocDir");
+  private static final long ATTEMPT_ID = 12345L;
+  private static final long ATTEMPT_ID_2 = 12346L;
+  private static final HttpResponseStatus OK_STATUS = new HttpResponseStatus(200, "OK");
+
+
+  //Control test execution properties with these flags
+  private static final boolean DEBUG_MODE = false;
+  //WARNING: If this is set to true and proxy server is not running, tests will fail!
+  private static final boolean USE_PROXY = false;
+  private static final int HEADER_WRITE_COUNT = 100000;
+  private static final int ARBITRARY_NEGATIVE_TIMEOUT_SECONDS = -100;
+  private static TestExecution TEST_EXECUTION;
+
+  private static class TestExecution {
+    private static final int DEFAULT_KEEP_ALIVE_TIMEOUT_SECONDS = 1;
+    private static final int DEBUG_KEEP_ALIVE_SECONDS = 1000;
+    private static final int DEFAULT_PORT = 0; //random port
+    private static final int FIXED_PORT = 8088;
+    private static final String PROXY_HOST = "127.0.0.1";
+    private static final int PROXY_PORT = 8888;
+    private static final int CONNECTION_DEBUG_TIMEOUT = 1000000;
+    private final boolean debugMode;
+    private final boolean useProxy;
+
+    TestExecution(boolean debugMode, boolean useProxy) {
+      this.debugMode = debugMode;
+      this.useProxy = useProxy;
+    }
+
+    int getKeepAliveTimeout() {
+      if (debugMode) {
+        return DEBUG_KEEP_ALIVE_SECONDS;
+      }
+      return DEFAULT_KEEP_ALIVE_TIMEOUT_SECONDS;
+    }
+
+    HttpURLConnection openConnection(URL url) throws IOException {
+      HttpURLConnection conn;
+      if (useProxy) {
+        Proxy proxy
+            = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(PROXY_HOST, PROXY_PORT));
+        conn = (HttpURLConnection) url.openConnection(proxy);
+      } else {
+        conn = (HttpURLConnection) url.openConnection();
+      }
+      return conn;
+    }
+
+    int shuffleHandlerPort() {
+      if (debugMode) {
+        return FIXED_PORT;
+      } else {
+        return DEFAULT_PORT;
+      }
+    }
+
+    void parameterizeConnection(URLConnection conn) {
+      if (DEBUG_MODE) {
+        conn.setReadTimeout(CONNECTION_DEBUG_TIMEOUT);
+        conn.setConnectTimeout(CONNECTION_DEBUG_TIMEOUT);
+      }
+    }
+  }
+
+  private static class ResponseConfig {
+    private final int headerWriteCount;
+    private final int mapOutputCount;
+    private final int contentLengthOfOneMapOutput;
+    private long headerSize;
+    public long contentLengthOfResponse;
+
+    ResponseConfig(int headerWriteCount, int mapOutputCount,
+        int contentLengthOfOneMapOutput) {
+      if (mapOutputCount <= 0 && contentLengthOfOneMapOutput > 0) {
+        throw new IllegalStateException("mapOutputCount should be at least 1");
+      }
+      this.headerWriteCount = headerWriteCount;
+      this.mapOutputCount = mapOutputCount;
+      this.contentLengthOfOneMapOutput = contentLengthOfOneMapOutput;
+    }
+
+    private void setHeaderSize(long headerSize) {
+      this.headerSize = headerSize;
+      long contentLengthOfAllHeaders = headerWriteCount * headerSize;
+      this.contentLengthOfResponse = computeContentLengthOfResponse(contentLengthOfAllHeaders);
+      LOG.debug("Content-length of all headers: {}", contentLengthOfAllHeaders);
+      LOG.debug("Content-length of one MapOutput: {}", contentLengthOfOneMapOutput);
+      LOG.debug("Content-length of final HTTP response: {}", contentLengthOfResponse);
+    }
+
+    private long computeContentLengthOfResponse(long contentLengthOfAllHeaders) {
+      int mapOutputCountMultiplier = mapOutputCount;
+      if (mapOutputCount == 0) {
+        mapOutputCountMultiplier = 1;
+      }
+      return (contentLengthOfAllHeaders + contentLengthOfOneMapOutput) * mapOutputCountMultiplier;
+    }
+  }
+
+  private enum ShuffleUrlType {
+    SIMPLE, WITH_KEEPALIVE, WITH_KEEPALIVE_MULTIPLE_MAP_IDS, WITH_KEEPALIVE_NO_MAP_IDS
+  }
+
+  private static class InputStreamReadResult {
+    final String asString;
+    int totalBytesRead;
+
+    InputStreamReadResult(byte[] bytes, int totalBytesRead) {
+      this.asString = new String(bytes, StandardCharsets.UTF_8);
+      this.totalBytesRead = totalBytesRead;
+    }
+  }
+
+  private static abstract class AdditionalMapOutputSenderOperations {
+    public abstract ChannelFuture perform(ChannelHandlerContext ctx, Channel ch) throws IOException;
+  }
+
+  private class ShuffleHandlerForKeepAliveTests extends ShuffleHandler {
+    final LastSocketAddress lastSocketAddress = new LastSocketAddress();
+    final ArrayList<Throwable> failures = new ArrayList<>();
+    final ShuffleHeaderProvider shuffleHeaderProvider;
+    final HeaderPopulator headerPopulator;
+    MapOutputSender mapOutputSender;
+    private Consumer<IdleStateEvent> channelIdleCallback;
+    private CustomTimeoutHandler customTimeoutHandler;
+    private boolean failImmediatelyOnErrors = false;
+    private boolean closeChannelOnError = true;
+    private ResponseConfig responseConfig;
+
+    ShuffleHandlerForKeepAliveTests(long attemptId, ResponseConfig responseConfig,
+        Consumer<IdleStateEvent> channelIdleCallback) throws IOException {
+      this(attemptId, responseConfig);
+      this.channelIdleCallback = channelIdleCallback;
+    }
+
+    ShuffleHandlerForKeepAliveTests(long attemptId, ResponseConfig responseConfig)
+        throws IOException {
+      this.responseConfig = responseConfig;
+      this.shuffleHeaderProvider = new ShuffleHeaderProvider(attemptId);
+      this.responseConfig.setHeaderSize(shuffleHeaderProvider.getShuffleHeaderSize());
+      this.headerPopulator = new HeaderPopulator(this, responseConfig, shuffleHeaderProvider, true);
+      this.mapOutputSender = new MapOutputSender(responseConfig, lastSocketAddress,
+          shuffleHeaderProvider);
+      setUseOutboundExceptionHandler(true);
+    }
+
+    public void setFailImmediatelyOnErrors(boolean failImmediatelyOnErrors) {
+      this.failImmediatelyOnErrors = failImmediatelyOnErrors;
+    }
+
+    public void setCloseChannelOnError(boolean closeChannelOnError) {
+      this.closeChannelOnError = closeChannelOnError;
+    }
+
+    @Override
+    protected Shuffle getShuffle(final Configuration conf) {
+      // replace the shuffle handler with one stubbed for testing
+      return new Shuffle(conf) {
+        @Override
+        protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
+            String jobId, String user) {
+          return null;
+        }
+        @Override
+        protected void verifyRequest(String appid, ChannelHandlerContext ctx,
+            HttpRequest request, HttpResponse response, URL requestUri) {
+        }
+
+        @Override
+        protected void populateHeaders(List<String> mapIds, String jobId,
+            String user, int reduce, HttpRequest request,
+            HttpResponse response, boolean keepAliveParam,
+            Map<String, MapOutputInfo> infoMap) throws IOException {
+          long contentLength = headerPopulator.populateHeaders(
+              keepAliveParam);
+          super.setResponseHeaders(response, keepAliveParam, contentLength);
+        }
+
+        @Override
+        protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
+            Channel ch, String user, String mapId, int reduce,
+            MapOutputInfo info) throws IOException {
+          return mapOutputSender.send(ctx, ch);
+        }
+
+        @Override
+        public void channelActive(ChannelHandlerContext ctx) throws Exception {
+          ctx.pipeline().replace(HttpResponseEncoder.class, ENCODER_HANDLER_NAME,
+              new LoggingHttpResponseEncoder(false));
+          replaceTimeoutHandlerWithCustom(ctx);
+          LOG.debug("Modified pipeline: {}", ctx.pipeline());
+          super.channelActive(ctx);
+        }
+
+        private void replaceTimeoutHandlerWithCustom(ChannelHandlerContext ctx) {
+          TimeoutHandler oldTimeoutHandler =
+              (TimeoutHandler)ctx.pipeline().get(TIMEOUT_HANDLER);
+          int timeoutValue =
+              oldTimeoutHandler.getConnectionKeepAliveTimeOut();
+          customTimeoutHandler = new CustomTimeoutHandler(timeoutValue, channelIdleCallback);
+          ctx.pipeline().replace(TIMEOUT_HANDLER, TIMEOUT_HANDLER, customTimeoutHandler);
+        }
+
+        @Override
+        protected void sendError(ChannelHandlerContext ctx,
+            HttpResponseStatus status) {
+          String message = "Error while processing request. Status: " + status;
+          handleError(ctx, message);
+          if (failImmediatelyOnErrors) {
+            stop();
+          }
+        }
+
+        @Override
+        protected void sendError(ChannelHandlerContext ctx, String message,
+            HttpResponseStatus status) {
+          String errMessage = String.format("Error while processing request. " +
+              "Status: " +
+              "%s, message: %s", status, message);
+          handleError(ctx, errMessage);
+          if (failImmediatelyOnErrors) {
+            stop();
+          }
+        }
+      };
+    }
+
+    private void handleError(ChannelHandlerContext ctx, String message) {
+      LOG.error(message);
+      failures.add(new Error(message));
+      if (closeChannelOnError) {
+        LOG.warn("sendError: Closing channel");
+        ctx.channel().close();
+      }
+    }
+
+    private class CustomTimeoutHandler extends TimeoutHandler {
+      private boolean channelIdle = false;
+      private final Consumer<IdleStateEvent> channelIdleCallback;
+
+      CustomTimeoutHandler(int connectionKeepAliveTimeOut,
+          Consumer<IdleStateEvent> channelIdleCallback) {
+        super(connectionKeepAliveTimeOut);
+        this.channelIdleCallback = channelIdleCallback;
+      }
+
+      @Override
+      public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) {
+        LOG.debug("Channel idle");
+        this.channelIdle = true;
+        if (channelIdleCallback != null) {
+          LOG.debug("Calling channel idle callback..");
+          channelIdleCallback.accept(e);
+        }
+        super.channelIdle(ctx, e);
+      }
+    }
+  }
+
+  private static class MapOutputSender {
+    private final ResponseConfig responseConfig;
+    private final LastSocketAddress lastSocketAddress;
+    private final ShuffleHeaderProvider shuffleHeaderProvider;
+    private AdditionalMapOutputSenderOperations additionalMapOutputSenderOperations;
+
+    MapOutputSender(ResponseConfig responseConfig, LastSocketAddress lastSocketAddress,
+        ShuffleHeaderProvider shuffleHeaderProvider) {
+      this.responseConfig = responseConfig;
+      this.lastSocketAddress = lastSocketAddress;
+      this.shuffleHeaderProvider = shuffleHeaderProvider;
+    }
+
+    public ChannelFuture send(ChannelHandlerContext ctx, Channel ch) throws IOException {
+      LOG.debug("In MapOutputSender#send");
+      lastSocketAddress.setAddress(ch.remoteAddress());
+      ShuffleHeader header = shuffleHeaderProvider.createNewShuffleHeader();
+      ChannelFuture future = writeHeaderNTimes(ch, header, responseConfig.headerWriteCount);
+      // This is the last operation
+      // It's safe to increment ShuffleHeader counter for better identification
+      shuffleHeaderProvider.incrementCounter();
+      if (additionalMapOutputSenderOperations != null) {
+        return additionalMapOutputSenderOperations.perform(ctx, ch);
+      }
+      return future;
+    }
+
+    private ChannelFuture writeHeaderNTimes(Channel ch, ShuffleHeader header, int iterations)
+        throws IOException {
+      DataOutputBuffer dob = new DataOutputBuffer();
+      for (int i = 0; i < iterations; ++i) {
+        header.write(dob);
+      }
+      LOG.debug("MapOutputSender#writeHeaderNTimes WriteAndFlush big chunk of data, " +
+          "outputBufferSize: " + dob.size());
+      return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+    }
+  }
+
+  private static class ShuffleHeaderProvider {
+    private final long attemptId;
+    private int attemptCounter = 0;
+    private int cachedSize = Integer.MIN_VALUE;
+
+    ShuffleHeaderProvider(long attemptId) {
+      this.attemptId = attemptId;
+    }
+
+    ShuffleHeader createNewShuffleHeader() {
+      return new ShuffleHeader(String.format("attempt_%s_1_m_1_0%s", attemptId, attemptCounter),
+          5678, 5678, 1);
+    }
+
+    void incrementCounter() {
+      attemptCounter++;
+    }
+
+    private int getShuffleHeaderSize() throws IOException {
+      if (cachedSize != Integer.MIN_VALUE) {
+        return cachedSize;
+      }
+      DataOutputBuffer dob = new DataOutputBuffer();
+      ShuffleHeader header = createNewShuffleHeader();
+      header.write(dob);
+      cachedSize = dob.size();
+      return cachedSize;
+    }
+  }
+
+  private static class HeaderPopulator {
+    private final ShuffleHandler shuffleHandler;
+    private final boolean disableKeepAliveConfig;
+    private final ShuffleHeaderProvider shuffleHeaderProvider;
+    private final ResponseConfig responseConfig;
+
+    HeaderPopulator(ShuffleHandler shuffleHandler,
+        ResponseConfig responseConfig,
+        ShuffleHeaderProvider shuffleHeaderProvider,
+        boolean disableKeepAliveConfig) {
+      this.shuffleHandler = shuffleHandler;
+      this.responseConfig = responseConfig;
+      this.disableKeepAliveConfig = disableKeepAliveConfig;
+      this.shuffleHeaderProvider = shuffleHeaderProvider;
+    }
+
+    public long populateHeaders(boolean keepAliveParam) throws IOException {
+      // Send some dummy data (populate content length details)
+      DataOutputBuffer dob = new DataOutputBuffer();
+      for (int i = 0; i < responseConfig.headerWriteCount; ++i) {
+        ShuffleHeader header =
+            shuffleHeaderProvider.createNewShuffleHeader();
+        header.write(dob);
+      }
+      // for testing purpose;
+      // disable connectionKeepAliveEnabled if keepAliveParam is available
+      if (keepAliveParam && disableKeepAliveConfig) {
+        shuffleHandler.connectionKeepAliveEnabled = false;
+      }
+      return responseConfig.contentLengthOfResponse;
+    }
+  }
+
+  private static final class HttpConnectionData {
+    private final Map<String, List<String>> headers;
+    private HttpURLConnection conn;
+    private final int payloadLength;
+    private final SocketAddress socket;
+    private int responseCode = -1;
+
+    private HttpConnectionData(HttpURLConnection conn, int payloadLength,
+        SocketAddress socket) {
+      this.headers = conn.getHeaderFields();
+      this.conn = conn;
+      this.payloadLength = payloadLength;
+      this.socket = socket;
+      try {
+        this.responseCode = conn.getResponseCode();
+      } catch (IOException e) {
+        fail("Failed to read response code from connection: " + conn);
+      }
+    }
+
+    static HttpConnectionData create(HttpURLConnection conn, int payloadLength,
+        SocketAddress socket) {
+      return new HttpConnectionData(conn, payloadLength, socket);
+    }
+  }
+
+  private static final class HttpConnectionAssert {
+    private final HttpConnectionData connData;
+
+    private HttpConnectionAssert(HttpConnectionData connData) {
+      this.connData = connData;
+    }
+
+    static HttpConnectionAssert create(HttpConnectionData connData) {
+      return new HttpConnectionAssert(connData);
+    }
+
+    public static void assertKeepAliveConnectionsAreSame(
+        HttpConnectionHelper httpConnectionHelper) {
+      assertTrue("At least two connection data " +
+          "is required to perform this assertion",
+          httpConnectionHelper.connectionData.size() >= 2);
+      SocketAddress firstAddress = httpConnectionHelper.getConnectionData(0).socket;
+      SocketAddress secondAddress = httpConnectionHelper.getConnectionData(1).socket;
+      Assert.assertNotNull("Initial shuffle address should not be null",
+          firstAddress);
+      Assert.assertNotNull("Keep-Alive shuffle address should not be null",
+          secondAddress);
+      assertEquals("Initial shuffle address and keep-alive shuffle "
+          + "address should be the same", firstAddress, secondAddress);
+    }
+
+    public HttpConnectionAssert expectKeepAliveWithTimeout(long timeout) {
+      assertEquals(HttpURLConnection.HTTP_OK, connData.responseCode);
+      assertHeaderValue(HttpHeader.CONNECTION, HttpHeader.KEEP_ALIVE.asString());
+      assertHeaderValue(HttpHeader.KEEP_ALIVE, "timeout=" + timeout);
+      return this;
+    }
+
+    public HttpConnectionAssert expectBadRequest(long timeout) {
+      assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, connData.responseCode);
+      assertHeaderValue(HttpHeader.CONNECTION, HttpHeader.KEEP_ALIVE.asString());
+      assertHeaderValue(HttpHeader.KEEP_ALIVE, "timeout=" + timeout);
+      return this;
+    }
+
+    public HttpConnectionAssert expectResponseContentLength(long size) {
+      assertEquals(size, connData.payloadLength);
+      return this;
+    }
+
+    private void assertHeaderValue(HttpHeader header, String expectedValue) {
+      List<String> headerList = connData.headers.get(header.asString());
+      Assert.assertNotNull("Got null header value for header: " + header, headerList);
+      Assert.assertFalse("Got empty header value for header: " + header, headerList.isEmpty());
+      assertEquals("Unexpected size of header list for header: " + header, 1,
+          headerList.size());
+      assertEquals(expectedValue, headerList.get(0));
+    }
+  }
+
+  private static class HttpConnectionHelper {
+    private final LastSocketAddress lastSocketAddress;
+    List<HttpConnectionData> connectionData = new ArrayList<>();
+
+    HttpConnectionHelper(LastSocketAddress lastSocketAddress) {
+      this.lastSocketAddress = lastSocketAddress;
+    }
+
+    public void connectToUrls(String[] urls, ResponseConfig responseConfig) throws IOException {
+      connectToUrlsInternal(urls, responseConfig, HttpURLConnection.HTTP_OK);
+    }
+
+    public void connectToUrls(String[] urls, ResponseConfig responseConfig, int expectedHttpStatus)
+        throws IOException {
+      connectToUrlsInternal(urls, responseConfig, expectedHttpStatus);
+    }
+
+    private void connectToUrlsInternal(String[] urls, ResponseConfig responseConfig,
+        int expectedHttpStatus) throws IOException {
+      int requests = urls.length;
+      int expectedConnections = urls.length;
+      LOG.debug("Will connect to URLs: {}", Arrays.toString(urls));
+      for (int reqIdx = 0; reqIdx < requests; reqIdx++) {
+        String urlString = urls[reqIdx];
+        LOG.debug("Connecting to URL: {}", urlString);
+        URL url = new URL(urlString);
+        HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
+        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
+            ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
+        conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
+            ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
+        TEST_EXECUTION.parameterizeConnection(conn);
+        conn.connect();
+        if (expectedHttpStatus == HttpURLConnection.HTTP_BAD_REQUEST) {
+          //Catch exception as error are caught with overridden sendError method
+          //Caught errors will be validated later.
+          try {
+            DataInputStream input = new DataInputStream(conn.getInputStream());
+          } catch (Exception e) {
+            expectedConnections--;
+            continue;
+          }
+        }
+        DataInputStream input = new DataInputStream(conn.getInputStream());
+        LOG.debug("Opened DataInputStream for connection: {}/{}", (reqIdx + 1), requests);
+        ShuffleHeader header = new ShuffleHeader();
+        header.readFields(input);
+        InputStreamReadResult result = readDataFromInputStream(input);
+        result.totalBytesRead += responseConfig.headerSize;
+        int expectedContentLength =
+            Integer.parseInt(conn.getHeaderField(HttpHeader.CONTENT_LENGTH.asString()));
+
+        if (result.totalBytesRead != expectedContentLength) {
+          throw new IOException(String.format("Premature EOF InputStream. " +
+              "Expected content-length: %s, " +
+              "Actual content-length: %s", expectedContentLength, result.totalBytesRead));
+        }
+        connectionData.add(HttpConnectionData
+            .create(conn, result.totalBytesRead, lastSocketAddress.getSocketAddres()));
+        input.close();
+        LOG.debug("Finished all interactions with URL: {}. Progress: {}/{}", url, (reqIdx + 1),
+            requests);
+      }
+      assertEquals(expectedConnections, connectionData.size());
+    }
+
+    void validate(Consumer<HttpConnectionData> connDataValidator) {
+      for (int i = 0; i < connectionData.size(); i++) {
+        LOG.debug("Validating connection data #{}", (i + 1));
+        HttpConnectionData connData = connectionData.get(i);
+        connDataValidator.accept(connData);
+      }
+    }
+
+    HttpConnectionData getConnectionData(int i) {
+      return connectionData.get(i);
+    }
+
+    private static InputStreamReadResult readDataFromInputStream(
+        InputStream input) throws IOException {
+      ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
+      byte[] buffer = new byte[1024];
+      int bytesRead;
+      int totalBytesRead = 0;
+      while ((bytesRead = input.read(buffer)) != -1) {
+        dataStream.write(buffer, 0, bytesRead);
+        totalBytesRead += bytesRead;
+      }
+      LOG.debug("Read total bytes: " + totalBytesRead);
+      dataStream.flush();
+      return new InputStreamReadResult(dataStream.toByteArray(), totalBytesRead);
+    }
+  }
+
+  class ShuffleHandlerForTests extends ShuffleHandler {
+    public final ArrayList<Throwable> failures = new ArrayList<>();
+
+    ShuffleHandlerForTests() {
+      setUseOutboundExceptionHandler(true);
+    }
+
+    ShuffleHandlerForTests(MetricsSystem ms) {
+      super(ms);
+      setUseOutboundExceptionHandler(true);
+    }
+
+    @Override
+    protected Shuffle getShuffle(final Configuration conf) {
+      return new Shuffle(conf) {
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx,
+            Throwable cause) throws Exception {
+          LOG.debug("ExceptionCaught");
+          failures.add(cause);
+          super.exceptionCaught(ctx, cause);
+        }
+      };
+    }
+  }
 
   class MockShuffleHandler extends org.apache.hadoop.mapred.ShuffleHandler {
-    private AuxiliaryLocalPathHandler pathHandler =
+    final ArrayList<Throwable> failures = new ArrayList<>();
+
+    private final AuxiliaryLocalPathHandler pathHandler =
         new TestAuxiliaryLocalPathHandler();
+
+    MockShuffleHandler() {
+      setUseOutboundExceptionHandler(true);
+    }
+
+    MockShuffleHandler(MetricsSystem ms) {
+      super(ms);
+      setUseOutboundExceptionHandler(true);
+    }
+
     @Override
     protected Shuffle getShuffle(final Configuration conf) {
       return new Shuffle(conf) {
@@ -120,7 +716,7 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
         }
         @Override
         protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
-            String jobId, String user) throws IOException {
+            String jobId, String user) {
           // Do nothing.
           return null;
         }
@@ -128,7 +724,7 @@ protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
         protected void populateHeaders(List<String> mapIds, String jobId,
             String user, int reduce, HttpRequest request,
             HttpResponse response, boolean keepAliveParam,
-            Map<String, MapOutputInfo> infoMap) throws IOException {
+            Map<String, MapOutputInfo> infoMap) {
           // Do nothing.
         }
         @Override
@@ -140,12 +736,20 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
               new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
           DataOutputBuffer dob = new DataOutputBuffer();
           header.write(dob);
-          ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+          ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
           dob = new DataOutputBuffer();
           for (int i = 0; i < 100; ++i) {
             header.write(dob);
           }
-          return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+          return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+        }
+
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx,
+            Throwable cause) throws Exception {
+          LOG.debug("ExceptionCaught");
+          failures.add(cause);
+          super.exceptionCaught(ctx, cause);
         }
       };
     }
@@ -159,24 +763,22 @@ public AuxiliaryLocalPathHandler getAuxiliaryLocalPathHandler() {
   private class TestAuxiliaryLocalPathHandler
       implements AuxiliaryLocalPathHandler {
     @Override
-    public Path getLocalPathForRead(String path) throws IOException {
+    public Path getLocalPathForRead(String path) {
       return new Path(ABS_LOG_DIR.getAbsolutePath(), path);
     }
 
     @Override
-    public Path getLocalPathForWrite(String path) throws IOException {
+    public Path getLocalPathForWrite(String path) {
       return new Path(ABS_LOG_DIR.getAbsolutePath());
     }
 
     @Override
-    public Path getLocalPathForWrite(String path, long size)
-        throws IOException {
+    public Path getLocalPathForWrite(String path, long size) {
       return new Path(ABS_LOG_DIR.getAbsolutePath());
     }
 
     @Override
-    public Iterable<Path> getAllLocalPathsForRead(String path)
-        throws IOException {
+    public Iterable<Path> getAllLocalPathsForRead(String path) {
       ArrayList<Path> paths = new ArrayList<>();
       paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
       return paths;
@@ -185,16 +787,34 @@ public Iterable<Path> getAllLocalPathsForRead(String path)
 
   private static class MockShuffleHandler2 extends
       org.apache.hadoop.mapred.ShuffleHandler {
+    final ArrayList<Throwable> failures = new ArrayList<>(1);
     boolean socketKeepAlive = false;
+
+    MockShuffleHandler2() {
+      setUseOutboundExceptionHandler(true);
+    }
+
+    MockShuffleHandler2(MetricsSystem ms) {
+      super(ms);
+      setUseOutboundExceptionHandler(true);
+    }
+
     @Override
     protected Shuffle getShuffle(final Configuration conf) {
       return new Shuffle(conf) {
         @Override
         protected void verifyRequest(String appid, ChannelHandlerContext ctx,
-            HttpRequest request, HttpResponse response, URL requestUri)
-            throws IOException {
-          SocketChannel channel = (SocketChannel)(ctx.getChannel());
-          socketKeepAlive = channel.getConfig().isKeepAlive();
+            HttpRequest request, HttpResponse response, URL requestUri) {
+          SocketChannel channel = (SocketChannel)(ctx.channel());
+          socketKeepAlive = channel.config().isKeepAlive();
+        }
+
+        @Override
+        public void exceptionCaught(ChannelHandlerContext ctx,
+            Throwable cause) throws Exception {
+          LOG.debug("ExceptionCaught");
+          failures.add(cause);
+          super.exceptionCaught(ctx, cause);
         }
       };
     }
@@ -204,6 +824,38 @@ protected boolean isSocketKeepAlive() {
     }
   }
 
+  @Rule
+  public TestName name = new TestName();
+
+  @Before
+  public void setup() {
+    TEST_EXECUTION = new TestExecution(DEBUG_MODE, USE_PROXY);
+  }
+
+  @After
+  public void tearDown() {
+    int port = TEST_EXECUTION.shuffleHandlerPort();
+    if (isPortUsed(port)) {
+      String msg = String.format("Port is being used: %d. " +
+          "Current testcase name: %s",
+          port, name.getMethodName());
+      throw new IllegalStateException(msg);
+    }
+  }
+
+  private static boolean isPortUsed(int port) {
+    if (port == 0) {
+      //Don't check if port is 0
+      return false;
+    }
+    try (Socket ignored = new Socket("localhost", port)) {
+      return true;
+    } catch (IOException e) {
+      LOG.error("Port: {}, port check result: {}", port, e.getMessage());
+      return false;
+    }
+  }
+
   /**
    * Test the validation of ShuffleHandler's meta-data's serialization and
    * de-serialization.
@@ -228,21 +880,23 @@ public void testSerializeMeta()  throws Exception {
   @Test (timeout = 10000)
   public void testShuffleMetrics() throws Exception {
     MetricsSystem ms = new MetricsSystemImpl();
-    ShuffleHandler sh = new ShuffleHandler(ms);
+    ShuffleHandler sh = new ShuffleHandlerForTests(ms);
     ChannelFuture cf = mock(ChannelFuture.class);
     when(cf.isSuccess()).thenReturn(true).thenReturn(false);
 
     sh.metrics.shuffleConnections.incr();
-    sh.metrics.shuffleOutputBytes.incr(1*MiB);
+    sh.metrics.shuffleOutputBytes.incr(MiB);
     sh.metrics.shuffleConnections.incr();
     sh.metrics.shuffleOutputBytes.incr(2*MiB);
 
-    checkShuffleMetrics(ms, 3*MiB, 0 , 0, 2);
+    checkShuffleMetrics(ms, 3*MiB, 0, 0, 2);
 
     sh.metrics.operationComplete(cf);
     sh.metrics.operationComplete(cf);
 
     checkShuffleMetrics(ms, 3*MiB, 1, 1, 0);
+
+    sh.stop();
   }
 
   static void checkShuffleMetrics(MetricsSystem ms, long bytes, int failed,
@@ -262,57 +916,54 @@ static void checkShuffleMetrics(MetricsSystem ms, long bytes, int failed,
    */
   @Test (timeout = 10000)
   public void testClientClosesConnection() throws Exception {
-    final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
-    ShuffleHandler shuffleHandler = new ShuffleHandler() {
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    ShuffleHandlerForTests shuffleHandler = new ShuffleHandlerForTests() {
+
       @Override
       protected Shuffle getShuffle(Configuration conf) {
         // replace the shuffle handler with one stubbed for testing
         return new Shuffle(conf) {
           @Override
           protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
-              String jobId, String user) throws IOException {
+              String jobId, String user) {
             return null;
           }
           @Override
           protected void populateHeaders(List<String> mapIds, String jobId,
               String user, int reduce, HttpRequest request,
               HttpResponse response, boolean keepAliveParam,
-              Map<String, MapOutputInfo> infoMap) throws IOException {
+              Map<String, MapOutputInfo> infoMap) {
             // Only set response headers and skip everything else
             // send some dummy value for content-length
             super.setResponseHeaders(response, keepAliveParam, 100);
           }
           @Override
           protected void verifyRequest(String appid, ChannelHandlerContext ctx,
-              HttpRequest request, HttpResponse response, URL requestUri)
-                  throws IOException {
+              HttpRequest request, HttpResponse response, URL requestUri) {
           }
           @Override
           protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
               Channel ch, String user, String mapId, int reduce,
               MapOutputInfo info)
                   throws IOException {
-            // send a shuffle header and a lot of data down the channel
-            // to trigger a broken pipe
             ShuffleHeader header =
                 new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
             DataOutputBuffer dob = new DataOutputBuffer();
             header.write(dob);
-            ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+            ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
             dob = new DataOutputBuffer();
             for (int i = 0; i < 100000; ++i) {
               header.write(dob);
             }
-            return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+            return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
           }
           @Override
           protected void sendError(ChannelHandlerContext ctx,
               HttpResponseStatus status) {
             if (failures.size() == 0) {
               failures.add(new Error());
-              ctx.getChannel().close();
+              ctx.channel().close();
             }
           }
           @Override
@@ -320,7 +971,7 @@ protected void sendError(ChannelHandlerContext ctx, String message,
               HttpResponseStatus status) {
             if (failures.size() == 0) {
               failures.add(new Error());
-              ctx.getChannel().close();
+              ctx.channel().close();
             }
           }
         };
@@ -332,26 +983,30 @@ protected void sendError(ChannelHandlerContext ctx, String message,
     // simulate a reducer that closes early by reading a single shuffle header
     // then closing the connection
     URL url = new URL("http://127.0.0.1:"
-      + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
-      + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_1_0");
-    HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+        + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+        + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_1_0");
+    HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
     conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
         ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
     conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
         ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
     conn.connect();
     DataInputStream input = new DataInputStream(conn.getInputStream());
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
-    Assert.assertEquals("close",
+    assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+    assertEquals("close",
         conn.getHeaderField(HttpHeader.CONNECTION.asString()));
     ShuffleHeader header = new ShuffleHeader();
     header.readFields(input);
     input.close();
 
+    assertEquals("sendError called when client closed connection", 0,
+        shuffleHandler.failures.size());
+    assertEquals("Should have no caught exceptions", Collections.emptyList(),
+        shuffleHandler.failures);
+
     shuffleHandler.stop();
-    Assert.assertTrue("sendError called when client closed connection",
-        failures.size() == 0);
   }
+
   static class LastSocketAddress {
     SocketAddress lastAddress;
     void setAddress(SocketAddress lastAddress) {
@@ -363,161 +1018,180 @@ SocketAddress getSocketAddres() {
   }
 
   @Test(timeout = 10000)
-  public void testKeepAlive() throws Exception {
-    final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
+  public void testKeepAliveInitiallyEnabled() throws Exception {
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
-    // try setting to -ve keep alive timeout.
-    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100);
-    final LastSocketAddress lastSocketAddress = new LastSocketAddress();
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        TEST_EXECUTION.getKeepAliveTimeout());
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT, 0, 0);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig);
+    testKeepAliveWithHttpOk(conf, shuffleHandler, ShuffleUrlType.SIMPLE,
+        ShuffleUrlType.WITH_KEEPALIVE);
+  }
 
-    ShuffleHandler shuffleHandler = new ShuffleHandler() {
-      @Override
-      protected Shuffle getShuffle(final Configuration conf) {
-        // replace the shuffle handler with one stubbed for testing
-        return new Shuffle(conf) {
-          @Override
-          protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
-              String jobId, String user) throws IOException {
-            return null;
-          }
-          @Override
-          protected void verifyRequest(String appid, ChannelHandlerContext ctx,
-              HttpRequest request, HttpResponse response, URL requestUri)
-              throws IOException {
-          }
+  @Test(timeout = 1000000)
+  public void testKeepAliveInitiallyEnabledTwoKeepAliveUrls() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        TEST_EXECUTION.getKeepAliveTimeout());
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT, 0, 0);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig);
+    testKeepAliveWithHttpOk(conf, shuffleHandler, ShuffleUrlType.WITH_KEEPALIVE,
+        ShuffleUrlType.WITH_KEEPALIVE);
+  }
 
-          @Override
-          protected void populateHeaders(List<String> mapIds, String jobId,
-              String user, int reduce, HttpRequest request,
-              HttpResponse response, boolean keepAliveParam,
-              Map<String, MapOutputInfo> infoMap) throws IOException {
-            // Send some dummy data (populate content length details)
-            ShuffleHeader header =
-                new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
-            DataOutputBuffer dob = new DataOutputBuffer();
-            header.write(dob);
-            dob = new DataOutputBuffer();
-            for (int i = 0; i < 100000; ++i) {
-              header.write(dob);
-            }
+  //TODO snemeth implement keepalive test that used properly mocked ShuffleHandler
+  @Test(timeout = 10000)
+  public void testKeepAliveInitiallyDisabled() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, false);
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        TEST_EXECUTION.getKeepAliveTimeout());
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT, 0, 0);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig);
+    testKeepAliveWithHttpOk(conf, shuffleHandler, ShuffleUrlType.WITH_KEEPALIVE,
+        ShuffleUrlType.WITH_KEEPALIVE);
+  }
 
-            long contentLength = dob.getLength();
-            // for testing purpose;
-            // disable connectinKeepAliveEnabled if keepAliveParam is available
-            if (keepAliveParam) {
-              connectionKeepAliveEnabled = false;
-            }
+  @Test(timeout = 10000)
+  public void testKeepAliveMultipleMapAttemptIds() throws Exception {
+    final int mapOutputContentLength = 11;
+    final int mapOutputCount = 2;
 
-            super.setResponseHeaders(response, keepAliveParam, contentLength);
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        TEST_EXECUTION.getKeepAliveTimeout());
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT,
+        mapOutputCount, mapOutputContentLength);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig);
+    shuffleHandler.mapOutputSender.additionalMapOutputSenderOperations =
+        new AdditionalMapOutputSenderOperations() {
+          @Override
+          public ChannelFuture perform(ChannelHandlerContext ctx, Channel ch) throws IOException {
+            File tmpFile = File.createTempFile("test", ".tmp");
+            Files.write(tmpFile.toPath(),
+                "dummytestcontent123456".getBytes(StandardCharsets.UTF_8));
+            final DefaultFileRegion partition = new DefaultFileRegion(tmpFile, 0,
+                mapOutputContentLength);
+            LOG.debug("Writing response partition: {}, channel: {}",
+                partition, ch.id());
+            return ch.writeAndFlush(partition)
+                .addListener((ChannelFutureListener) future ->
+                    LOG.debug("Finished Writing response partition: {}, channel: " +
+                        "{}", partition, ch.id()));
           }
+        };
+    testKeepAliveWithHttpOk(conf, shuffleHandler,
+        ShuffleUrlType.WITH_KEEPALIVE_MULTIPLE_MAP_IDS,
+        ShuffleUrlType.WITH_KEEPALIVE_MULTIPLE_MAP_IDS);
+  }
 
-          @Override
-          protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
-              Channel ch, String user, String mapId, int reduce,
-              MapOutputInfo info) throws IOException {
-            lastSocketAddress.setAddress(ch.getRemoteAddress());
-            HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
+  @Test(timeout = 10000)
+  public void testKeepAliveWithoutMapAttemptIds() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        TEST_EXECUTION.getKeepAliveTimeout());
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT, 0, 0);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig);
+    shuffleHandler.setFailImmediatelyOnErrors(true);
+    //Closing channels caused Netty to open another channel
+    // so 1 request was handled with 2 separate channels,
+    // ultimately generating 2 * HTTP 400 errors.
+    // We'd like to avoid this so disabling closing the channel here.
+    shuffleHandler.setCloseChannelOnError(false);
+    testKeepAliveWithHttpBadRequest(conf, shuffleHandler, ShuffleUrlType.WITH_KEEPALIVE_NO_MAP_IDS);
+  }
 
-            // send a shuffle header and a lot of data down the channel
-            // to trigger a broken pipe
-            ShuffleHeader header =
-                new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
-            DataOutputBuffer dob = new DataOutputBuffer();
-            header.write(dob);
-            ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
-            dob = new DataOutputBuffer();
-            for (int i = 0; i < 100000; ++i) {
-              header.write(dob);
-            }
-            return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
-          }
+  private void testKeepAliveWithHttpOk(
+      Configuration conf,
+      ShuffleHandlerForKeepAliveTests shuffleHandler,
+      ShuffleUrlType... shuffleUrlTypes) throws IOException {
+    testKeepAliveWithHttpStatus(conf, shuffleHandler, shuffleUrlTypes, HttpURLConnection.HTTP_OK);
+  }
 
-          @Override
-          protected void sendError(ChannelHandlerContext ctx,
-              HttpResponseStatus status) {
-            if (failures.size() == 0) {
-              failures.add(new Error());
-              ctx.getChannel().close();
-            }
-          }
+  private void testKeepAliveWithHttpBadRequest(
+      Configuration conf,
+      ShuffleHandlerForKeepAliveTests shuffleHandler,
+      ShuffleUrlType... shuffleUrlTypes) throws IOException {
+    testKeepAliveWithHttpStatus(conf, shuffleHandler, shuffleUrlTypes,
+        HttpURLConnection.HTTP_BAD_REQUEST);
+  }
 
-          @Override
-          protected void sendError(ChannelHandlerContext ctx, String message,
-              HttpResponseStatus status) {
-            if (failures.size() == 0) {
-              failures.add(new Error());
-              ctx.getChannel().close();
-            }
-          }
-        };
-      }
-    };
+  private void testKeepAliveWithHttpStatus(Configuration conf,
+      ShuffleHandlerForKeepAliveTests shuffleHandler,
+      ShuffleUrlType[] shuffleUrlTypes,
+      int expectedHttpStatus) throws IOException {
+    if (expectedHttpStatus != HttpURLConnection.HTTP_BAD_REQUEST) {
+      assertTrue("Expected at least two shuffle URL types ",
+          shuffleUrlTypes.length >= 2);
+    }
     shuffleHandler.init(conf);
     shuffleHandler.start();
 
-    String shuffleBaseURL = "http://127.0.0.1:"
-            + shuffleHandler.getConfig().get(
-              ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
-    URL url =
-        new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&reduce=1&"
-            + "map=attempt_12345_1_m_1_0");
-    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
-        ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
-    conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
-        ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
-    conn.connect();
-    DataInputStream input = new DataInputStream(conn.getInputStream());
-    Assert.assertEquals(HttpHeader.KEEP_ALIVE.asString(),
-        conn.getHeaderField(HttpHeader.CONNECTION.asString()));
-    Assert.assertEquals("timeout=1",
-        conn.getHeaderField(HttpHeader.KEEP_ALIVE.asString()));
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
-    ShuffleHeader header = new ShuffleHeader();
-    header.readFields(input);
-    byte[] buffer = new byte[1024];
-    while (input.read(buffer) != -1) {}
-    SocketAddress firstAddress = lastSocketAddress.getSocketAddres();
-    input.close();
-
-    // For keepAlive via URL
-    url =
-        new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&reduce=1&"
-            + "map=attempt_12345_1_m_1_0&keepAlive=true");
-    conn = (HttpURLConnection) url.openConnection();
-    conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
-        ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
-    conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
-        ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
-    conn.connect();
-    input = new DataInputStream(conn.getInputStream());
-    Assert.assertEquals(HttpHeader.KEEP_ALIVE.asString(),
-        conn.getHeaderField(HttpHeader.CONNECTION.asString()));
-    Assert.assertEquals("timeout=1",
-        conn.getHeaderField(HttpHeader.KEEP_ALIVE.asString()));
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
-    header = new ShuffleHeader();
-    header.readFields(input);
-    input.close();
-    SocketAddress secondAddress = lastSocketAddress.getSocketAddres();
-    Assert.assertNotNull("Initial shuffle address should not be null",
-        firstAddress);
-    Assert.assertNotNull("Keep-Alive shuffle address should not be null",
-        secondAddress);
-    Assert.assertEquals("Initial shuffle address and keep-alive shuffle "
-        + "address should be the same", firstAddress, secondAddress);
+    String[] urls = new String[shuffleUrlTypes.length];
+    for (int i = 0; i < shuffleUrlTypes.length; i++) {
+      ShuffleUrlType url = shuffleUrlTypes[i];
+      if (url == ShuffleUrlType.SIMPLE) {
+        urls[i] = getShuffleUrl(shuffleHandler, ATTEMPT_ID, ATTEMPT_ID);
+      } else if (url == ShuffleUrlType.WITH_KEEPALIVE) {
+        urls[i] = getShuffleUrlWithKeepAlive(shuffleHandler, ATTEMPT_ID, ATTEMPT_ID);
+      } else if (url == ShuffleUrlType.WITH_KEEPALIVE_MULTIPLE_MAP_IDS) {
+        urls[i] = getShuffleUrlWithKeepAlive(shuffleHandler, ATTEMPT_ID, ATTEMPT_ID, ATTEMPT_ID_2);
+      } else if (url == ShuffleUrlType.WITH_KEEPALIVE_NO_MAP_IDS) {
+        urls[i] = getShuffleUrlWithKeepAlive(shuffleHandler, ATTEMPT_ID);
+      }
+    }
+    HttpConnectionHelper connHelper;
+    try {
+      connHelper = new HttpConnectionHelper(shuffleHandler.lastSocketAddress);
+      connHelper.connectToUrls(urls, shuffleHandler.responseConfig, expectedHttpStatus);
+      if (expectedHttpStatus == HttpURLConnection.HTTP_BAD_REQUEST) {
+        assertEquals(1, shuffleHandler.failures.size());
+        assertThat(shuffleHandler.failures.get(0).getMessage(),
+            CoreMatchers.containsString("Status: 400 Bad Request, " +
+                "message: Required param job, map and reduce"));
+      }
+    } finally {
+      shuffleHandler.stop();
+    }
 
+    //Verify expectations
+    int configuredTimeout = TEST_EXECUTION.getKeepAliveTimeout();
+    int expectedTimeout = configuredTimeout < 0 ? 1 : configuredTimeout;
+
+    connHelper.validate(connData -> {
+      HttpConnectionAssert.create(connData)
+          .expectKeepAliveWithTimeout(expectedTimeout)
+          .expectResponseContentLength(shuffleHandler.responseConfig.contentLengthOfResponse);
+    });
+    if (expectedHttpStatus == HttpURLConnection.HTTP_OK) {
+      HttpConnectionAssert.assertKeepAliveConnectionsAreSame(connHelper);
+      assertEquals("Unexpected ShuffleHandler failure", Collections.emptyList(),
+          shuffleHandler.failures);
+    }
   }
 
   @Test(timeout = 10000)
   public void testSocketKeepAlive() throws Exception {
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
-    // try setting to -ve keep alive timeout.
-    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100);
+    // try setting to negative keep alive timeout.
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
+        ARBITRARY_NEGATIVE_TIMEOUT_SECONDS);
     HttpURLConnection conn = null;
     MockShuffleHandler2 shuffleHandler = new MockShuffleHandler2();
     AuxiliaryLocalPathHandler pathHandler =
@@ -535,14 +1209,16 @@ public void testSocketKeepAlive() throws Exception {
       URL url =
           new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&reduce=1&"
               + "map=attempt_12345_1_m_1_0");
-      conn = (HttpURLConnection) url.openConnection();
+      conn = TEST_EXECUTION.openConnection(url);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
           ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
           ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
       conn.connect();
+      int rc = conn.getResponseCode();
       conn.getInputStream();
-      Assert.assertTrue("socket should be set KEEP_ALIVE",
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertTrue("socket should be set KEEP_ALIVE",
           shuffleHandler.isSocketKeepAlive());
     } finally {
       if (conn != null) {
@@ -550,11 +1226,13 @@ public void testSocketKeepAlive() throws Exception {
       }
       shuffleHandler.stop();
     }
+    assertEquals("Should have no caught exceptions",
+        Collections.emptyList(), shuffleHandler.failures);
   }
 
   /**
-   * simulate a reducer that sends an invalid shuffle-header - sometimes a wrong
-   * header_name and sometimes a wrong version
+   * Simulate a reducer that sends an invalid shuffle-header - sometimes a wrong
+   * header_name and sometimes a wrong version.
    * 
    * @throws Exception exception
    */
@@ -562,24 +1240,24 @@ public void testSocketKeepAlive() throws Exception {
   public void testIncompatibleShuffleVersion() throws Exception {
     final int failureNum = 3;
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
-    ShuffleHandler shuffleHandler = new ShuffleHandler();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    ShuffleHandler shuffleHandler = new ShuffleHandlerForTests();
     shuffleHandler.init(conf);
     shuffleHandler.start();
 
     // simulate a reducer that closes early by reading a single shuffle header
     // then closing the connection
     URL url = new URL("http://127.0.0.1:"
-      + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
-      + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_1_0");
+        + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
+        + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_1_0");
     for (int i = 0; i < failureNum; ++i) {
-      HttpURLConnection conn = (HttpURLConnection)url.openConnection();
+      HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
           i == 0 ? "mapreduce" : "other");
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
           i == 1 ? "1.0.0" : "1.0.1");
       conn.connect();
-      Assert.assertEquals(
+      assertEquals(
           HttpURLConnection.HTTP_BAD_REQUEST, conn.getResponseCode());
     }
 
@@ -594,10 +1272,14 @@ public void testIncompatibleShuffleVersion() throws Exception {
    */
   @Test (timeout = 10000)
   public void testMaxConnections() throws Exception {
+    final ArrayList<Throwable> failures = new ArrayList<>();
+    final int maxAllowedConnections = 3;
+    final int notAcceptedConnections = 1;
+    final int connAttempts = maxAllowedConnections + notAcceptedConnections;
     
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
-    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, maxAllowedConnections);
     ShuffleHandler shuffleHandler = new ShuffleHandler() {
       @Override
       protected Shuffle getShuffle(Configuration conf) {
@@ -605,7 +1287,7 @@ protected Shuffle getShuffle(Configuration conf) {
         return new Shuffle(conf) {
           @Override
           protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
-              String jobId, String user) throws IOException {
+              String jobId, String user) {
             // Do nothing.
             return null;
           }
@@ -613,13 +1295,12 @@ protected MapOutputInfo getMapOutputInfo(String mapId, int reduce,
           protected void populateHeaders(List<String> mapIds, String jobId,
               String user, int reduce, HttpRequest request,
               HttpResponse response, boolean keepAliveParam,
-              Map<String, MapOutputInfo> infoMap) throws IOException {
+              Map<String, MapOutputInfo> infoMap) {
             // Do nothing.
           }
           @Override
           protected void verifyRequest(String appid, ChannelHandlerContext ctx,
-              HttpRequest request, HttpResponse response, URL requestUri)
-                  throws IOException {
+              HttpRequest request, HttpResponse response, URL requestUri) {
             // Do nothing.
           }
           @Override
@@ -633,30 +1314,38 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
                 new ShuffleHeader("dummy_header", 5678, 5678, 1);
             DataOutputBuffer dob = new DataOutputBuffer();
             header.write(dob);
-            ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+            ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
             dob = new DataOutputBuffer();
             for (int i=0; i<100000; ++i) {
               header.write(dob);
             }
-            return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+            return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+          }
+
+          @Override
+          public void exceptionCaught(ChannelHandlerContext ctx,
+              Throwable cause) throws Exception {
+            LOG.debug("ExceptionCaught");
+            failures.add(cause);
+            super.exceptionCaught(ctx, cause);
           }
         };
       }
     };
+    shuffleHandler.setUseOutboundExceptionHandler(true);
     shuffleHandler.init(conf);
     shuffleHandler.start();
 
     // setup connections
-    int connAttempts = 3;
-    HttpURLConnection conns[] = new HttpURLConnection[connAttempts];
+    HttpURLConnection[] conns = new HttpURLConnection[connAttempts];
 
     for (int i = 0; i < connAttempts; i++) {
-      String URLstring = "http://127.0.0.1:" 
+      String urlString = "http://127.0.0.1:" 
            + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
            + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_"
            + i + "_0";
-      URL url = new URL(URLstring);
-      conns[i] = (HttpURLConnection)url.openConnection();
+      URL url = new URL(urlString);
+      conns[i] = TEST_EXECUTION.openConnection(url);
       conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
           ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
       conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
@@ -668,34 +1357,61 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
       conns[i].connect();
     }
 
-    //Ensure first connections are okay
-    conns[0].getInputStream();
-    int rc = conns[0].getResponseCode();
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
-    
-    conns[1].getInputStream();
-    rc = conns[1].getResponseCode();
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
-
-    // This connection should be closed because it to above the limit
-    try {
-      rc = conns[2].getResponseCode();
-      Assert.assertEquals("Expected a too-many-requests response code",
-          ShuffleHandler.TOO_MANY_REQ_STATUS.getCode(), rc);
-      long backoff = Long.valueOf(
-          conns[2].getHeaderField(ShuffleHandler.RETRY_AFTER_HEADER));
-      Assert.assertTrue("The backoff value cannot be negative.", backoff > 0);
-      conns[2].getInputStream();
-      Assert.fail("Expected an IOException");
-    } catch (IOException ioe) {
-      LOG.info("Expected - connection should not be open");
-    } catch (NumberFormatException ne) {
-      Assert.fail("Expected a numerical value for RETRY_AFTER header field");
-    } catch (Exception e) {
-      Assert.fail("Expected a IOException");
+    Map<Integer, List<HttpURLConnection>> mapOfConnections = Maps.newHashMap();
+    for (HttpURLConnection conn : conns) {
+      try {
+        conn.getInputStream();
+      } catch (IOException ioe) {
+        LOG.info("Expected - connection should not be open");
+      } catch (NumberFormatException ne) {
+        fail("Expected a numerical value for RETRY_AFTER header field");
+      } catch (Exception e) {
+        fail("Expected a IOException");
+      }
+      int statusCode = conn.getResponseCode();
+      LOG.debug("Connection status code: {}", statusCode);
+      mapOfConnections.putIfAbsent(statusCode, new ArrayList<>());
+      List<HttpURLConnection> connectionList = mapOfConnections.get(statusCode);
+      connectionList.add(conn);
     }
+
+    assertEquals(String.format("Expected only %s and %s response",
+            OK_STATUS, ShuffleHandler.TOO_MANY_REQ_STATUS),
+        Sets.newHashSet(
+            HttpURLConnection.HTTP_OK,
+            ShuffleHandler.TOO_MANY_REQ_STATUS.code()),
+        mapOfConnections.keySet());
     
-    shuffleHandler.stop(); 
+    List<HttpURLConnection> successfulConnections =
+        mapOfConnections.get(HttpURLConnection.HTTP_OK);
+    assertEquals(String.format("Expected exactly %d requests " +
+            "with %s response", maxAllowedConnections, OK_STATUS),
+        maxAllowedConnections, successfulConnections.size());
+
+    //Ensure exactly one connection is HTTP 429 (TOO MANY REQUESTS)
+    List<HttpURLConnection> closedConnections =
+        mapOfConnections.get(ShuffleHandler.TOO_MANY_REQ_STATUS.code());
+    assertEquals(String.format("Expected exactly %d %s response",
+            notAcceptedConnections, ShuffleHandler.TOO_MANY_REQ_STATUS),
+        notAcceptedConnections, closedConnections.size());
+
+    // This connection should be closed because it is above the maximum limit
+    HttpURLConnection conn = closedConnections.get(0);
+    assertEquals(String.format("Expected a %s response",
+            ShuffleHandler.TOO_MANY_REQ_STATUS),
+        ShuffleHandler.TOO_MANY_REQ_STATUS.code(), conn.getResponseCode());
+    long backoff = Long.parseLong(
+        conn.getHeaderField(ShuffleHandler.RETRY_AFTER_HEADER));
+    assertTrue("The backoff value cannot be negative.", backoff > 0);
+
+    shuffleHandler.stop();
+
+    //It's okay to get a ClosedChannelException.
+    //All other kinds of exceptions means something went wrong
+    assertEquals("Should have no caught exceptions",
+        Collections.emptyList(), failures.stream()
+            .filter(f -> !(f instanceof ClosedChannelException))
+            .collect(toList()));
   }
 
   /**
@@ -706,10 +1422,11 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
    */
   @Test(timeout = 100000)
   public void testMapFileAccess() throws IOException {
+    final ArrayList<Throwable> failures = new ArrayList<>();
     // This will run only in NativeIO is enabled as SecureIOUtils need it
     assumeTrue(NativeIO.isAvailable());
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
@@ -720,7 +1437,7 @@ public void testMapFileAccess() throws IOException {
     String appAttemptId = "attempt_12345_1_m_1_0";
     String user = "randomUser";
     String reducerId = "0";
-    List<File> fileMap = new ArrayList<File>();
+    List<File> fileMap = new ArrayList<>();
     createShuffleHandlerFiles(ABS_LOG_DIR, user, appId.toString(), appAttemptId,
         conf, fileMap);
     ShuffleHandler shuffleHandler = new ShuffleHandler() {
@@ -731,15 +1448,31 @@ protected Shuffle getShuffle(Configuration conf) {
 
           @Override
           protected void verifyRequest(String appid, ChannelHandlerContext ctx,
-              HttpRequest request, HttpResponse response, URL requestUri)
-              throws IOException {
+              HttpRequest request, HttpResponse response, URL requestUri) {
             // Do nothing.
           }
 
+          @Override
+          public void exceptionCaught(ChannelHandlerContext ctx,
+              Throwable cause) throws Exception {
+            LOG.debug("ExceptionCaught");
+            failures.add(cause);
+            super.exceptionCaught(ctx, cause);
+          }
+
+          @Override
+          public void channelActive(ChannelHandlerContext ctx) throws Exception {
+            ctx.pipeline().replace(HttpResponseEncoder.class,
+                "loggingResponseEncoder",
+                new LoggingHttpResponseEncoder(false));
+            LOG.debug("Modified pipeline: {}", ctx.pipeline());
+            super.channelActive(ctx);
+          }
         };
       }
     };
     AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
+    shuffleHandler.setUseOutboundExceptionHandler(true);
     shuffleHandler.setAuxiliaryLocalPathHandler(pathHandler);
     shuffleHandler.init(conf);
     try {
@@ -747,13 +1480,13 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
       DataOutputBuffer outputBuffer = new DataOutputBuffer();
       outputBuffer.reset();
       Token<JobTokenIdentifier> jt =
-          new Token<JobTokenIdentifier>("identifier".getBytes(),
+          new Token<>("identifier".getBytes(),
               "password".getBytes(), new Text(user), new Text("shuffleService"));
       jt.write(outputBuffer);
       shuffleHandler
-        .initializeApplication(new ApplicationInitializationContext(user,
-          appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
-            outputBuffer.getLength())));
+          .initializeApplication(new ApplicationInitializationContext(user,
+              appId, ByteBuffer.wrap(outputBuffer.getData(), 0,
+              outputBuffer.getLength())));
       URL url =
           new URL(
               "http://127.0.0.1:"
@@ -761,32 +1494,37 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx,
                       ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
                   + "/mapOutput?job=job_12345_0001&reduce=" + reducerId
                   + "&map=attempt_12345_1_m_1_0");
-      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
           ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
           ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
       conn.connect();
-      byte[] byteArr = new byte[10000];
-      try {
-        DataInputStream is = new DataInputStream(conn.getInputStream());
-        is.readFully(byteArr);
-      } catch (EOFException e) {
-        // ignore
-      }
-      // Retrieve file owner name
-      FileInputStream is = new FileInputStream(fileMap.get(0));
-      String owner = NativeIO.POSIX.getFstat(is.getFD()).getOwner();
-      is.close();
+      DataInputStream is = new DataInputStream(conn.getInputStream());
+      InputStreamReadResult result = HttpConnectionHelper.readDataFromInputStream(is);
+      String receivedString = result.asString;
+
+      //Retrieve file owner name
+      FileInputStream fis = new FileInputStream(fileMap.get(0));
+      String owner = NativeIO.POSIX.getFstat(fis.getFD()).getOwner();
+      fis.close();
 
       String message =
           "Owner '" + owner + "' for path " + fileMap.get(0).getAbsolutePath()
               + " did not match expected owner '" + user + "'";
-      Assert.assertTrue((new String(byteArr)).contains(message));
+      assertTrue(String.format("Received string '%s' should contain " +
+          "message '%s'", receivedString, message),
+          receivedString.contains(message));
+      assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+      LOG.info("received: " + receivedString);
+      assertNotEquals("", receivedString);
     } finally {
       shuffleHandler.stop();
       FileUtil.fullyDelete(ABS_LOG_DIR);
     }
+
+    assertEquals("Should have no caught exceptions",
+        Collections.emptyList(), failures);
   }
 
   private static void createShuffleHandlerFiles(File logDir, String user,
@@ -794,7 +1532,7 @@ private static void createShuffleHandlerFiles(File logDir, String user,
       List<File> fileMap) throws IOException {
     String attemptDir =
         StringUtils.join(Path.SEPARATOR,
-            new String[] { logDir.getAbsolutePath(),
+            new String[] {logDir.getAbsolutePath(),
                 ContainerLocalizer.USERCACHE, user,
                 ContainerLocalizer.APPCACHE, appId, "output", appAttemptId });
     File appAttemptDir = new File(attemptDir);
@@ -808,8 +1546,7 @@ private static void createShuffleHandlerFiles(File logDir, String user,
     createMapOutputFile(mapOutputFile, conf);
   }
 
-  private static void
-    createMapOutputFile(File mapOutputFile, Configuration conf)
+  private static void createMapOutputFile(File mapOutputFile, Configuration conf)
           throws IOException {
     FileOutputStream out = new FileOutputStream(mapOutputFile);
     out.write("Creating new dummy map output file. Used only for testing"
@@ -846,11 +1583,11 @@ public void testRecovery() throws IOException {
     final File tmpDir = new File(System.getProperty("test.build.data",
         System.getProperty("java.io.tmpdir")),
         TestShuffleHandler.class.getName());
-    ShuffleHandler shuffle = new ShuffleHandler();
+    ShuffleHandler shuffle = new ShuffleHandlerForTests();
     AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
     shuffle.setAuxiliaryLocalPathHandler(pathHandler);
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
     conf.set(YarnConfiguration.NM_LOCAL_DIRS,
         ABS_LOG_DIR.getAbsolutePath());
@@ -861,10 +1598,10 @@ public void testRecovery() throws IOException {
       shuffle.init(conf);
       shuffle.start();
 
-      // setup a shuffle token for an application
+      // set up a shuffle token for an application
       DataOutputBuffer outputBuffer = new DataOutputBuffer();
       outputBuffer.reset();
-      Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(
+      Token<JobTokenIdentifier> jt = new Token<>(
           "identifier".getBytes(), "password".getBytes(), new Text(user),
           new Text("shuffleService"));
       jt.write(outputBuffer);
@@ -874,11 +1611,11 @@ public void testRecovery() throws IOException {
 
       // verify we are authorized to shuffle
       int rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
 
       // emulate shuffle handler restart
       shuffle.close();
-      shuffle = new ShuffleHandler();
+      shuffle = new ShuffleHandlerForTests();
       shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
@@ -886,23 +1623,23 @@ public void testRecovery() throws IOException {
 
       // verify we are still authorized to shuffle to the old application
       rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
 
       // shutdown app and verify access is lost
       shuffle.stopApplication(new ApplicationTerminationContext(appId));
       rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);
+      assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);
 
       // emulate shuffle handler restart
       shuffle.close();
-      shuffle = new ShuffleHandler();
+      shuffle = new ShuffleHandlerForTests();
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
       shuffle.start();
 
       // verify we still don't have access
       rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);
+      assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc);
     } finally {
       if (shuffle != null) {
         shuffle.close();
@@ -919,9 +1656,9 @@ public void testRecoveryFromOtherVersions() throws IOException {
         System.getProperty("java.io.tmpdir")),
         TestShuffleHandler.class.getName());
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
-    ShuffleHandler shuffle = new ShuffleHandler();
+    ShuffleHandler shuffle = new ShuffleHandlerForTests();
     AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
     shuffle.setAuxiliaryLocalPathHandler(pathHandler);
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, ABS_LOG_DIR.getAbsolutePath());
@@ -932,10 +1669,10 @@ public void testRecoveryFromOtherVersions() throws IOException {
       shuffle.init(conf);
       shuffle.start();
 
-      // setup a shuffle token for an application
+      // set up a shuffle token for an application
       DataOutputBuffer outputBuffer = new DataOutputBuffer();
       outputBuffer.reset();
-      Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(
+      Token<JobTokenIdentifier> jt = new Token<>(
           "identifier".getBytes(), "password".getBytes(), new Text(user),
           new Text("shuffleService"));
       jt.write(outputBuffer);
@@ -945,11 +1682,11 @@ public void testRecoveryFromOtherVersions() throws IOException {
 
       // verify we are authorized to shuffle
       int rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
 
       // emulate shuffle handler restart
       shuffle.close();
-      shuffle = new ShuffleHandler();
+      shuffle = new ShuffleHandlerForTests();
       shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
@@ -957,44 +1694,44 @@ public void testRecoveryFromOtherVersions() throws IOException {
 
       // verify we are still authorized to shuffle to the old application
       rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
       Version version = Version.newInstance(1, 0);
-      Assert.assertEquals(version, shuffle.getCurrentVersion());
+      assertEquals(version, shuffle.getCurrentVersion());
     
       // emulate shuffle handler restart with compatible version
       Version version11 = Version.newInstance(1, 1);
       // update version info before close shuffle
       shuffle.storeVersion(version11);
-      Assert.assertEquals(version11, shuffle.loadVersion());
+      assertEquals(version11, shuffle.loadVersion());
       shuffle.close();
-      shuffle = new ShuffleHandler();
+      shuffle = new ShuffleHandlerForTests();
       shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
       shuffle.start();
       // shuffle version will be override by CURRENT_VERSION_INFO after restart
       // successfully.
-      Assert.assertEquals(version, shuffle.loadVersion());
+      assertEquals(version, shuffle.loadVersion());
       // verify we are still authorized to shuffle to the old application
       rc = getShuffleResponseCode(shuffle, jt);
-      Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
+      assertEquals(HttpURLConnection.HTTP_OK, rc);
     
       // emulate shuffle handler restart with incompatible version
       Version version21 = Version.newInstance(2, 1);
       shuffle.storeVersion(version21);
-      Assert.assertEquals(version21, shuffle.loadVersion());
+      assertEquals(version21, shuffle.loadVersion());
       shuffle.close();
-      shuffle = new ShuffleHandler();
+      shuffle = new ShuffleHandlerForTests();
       shuffle.setAuxiliaryLocalPathHandler(pathHandler);
       shuffle.setRecoveryPath(new Path(tmpDir.toString()));
       shuffle.init(conf);
     
       try {
         shuffle.start();
-        Assert.fail("Incompatible version, should expect fail here.");
+        fail("Incompatible version, should expect fail here.");
       } catch (ServiceStateException e) {
-        Assert.assertTrue("Exception message mismatch", 
-        e.getMessage().contains("Incompatible version for state DB schema:"));
+        assertTrue("Exception message mismatch",
+            e.getMessage().contains("Incompatible version for state DB schema:"));
       } 
     
     } finally {
@@ -1010,7 +1747,7 @@ private static int getShuffleResponseCode(ShuffleHandler shuffle,
     URL url = new URL("http://127.0.0.1:"
         + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
         + "/mapOutput?job=job_12345_0001&reduce=0&map=attempt_12345_1_m_1_0");
-    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+    HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
     String encHash = SecureShuffleUtils.hashFromString(
         SecureShuffleUtils.buildMsgFrom(url),
         JobTokenSecretManager.createSecretKey(jt.getPassword()));
@@ -1028,9 +1765,9 @@ private static int getShuffleResponseCode(ShuffleHandler shuffle,
 
   @Test(timeout = 100000)
   public void testGetMapOutputInfo() throws Exception {
-    final ArrayList<Throwable> failures = new ArrayList<Throwable>(1);
+    final ArrayList<Throwable> failures = new ArrayList<>(1);
     Configuration conf = new Configuration();
-    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
     conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "simple");
@@ -1040,7 +1777,7 @@ public void testGetMapOutputInfo() throws Exception {
     String appAttemptId = "attempt_12345_1_m_1_0";
     String user = "randomUser";
     String reducerId = "0";
-    List<File> fileMap = new ArrayList<File>();
+    List<File> fileMap = new ArrayList<>();
     createShuffleHandlerFiles(ABS_LOG_DIR, user, appId.toString(), appAttemptId,
         conf, fileMap);
     AuxiliaryLocalPathHandler pathHandler = new TestAuxiliaryLocalPathHandler();
@@ -1062,7 +1799,7 @@ protected void populateHeaders(List<String> mapIds,
           @Override
           protected void verifyRequest(String appid,
               ChannelHandlerContext ctx, HttpRequest request,
-              HttpResponse response, URL requestUri) throws IOException {
+              HttpResponse response, URL requestUri) {
             // Do nothing.
           }
           @Override
@@ -1070,7 +1807,7 @@ protected void sendError(ChannelHandlerContext ctx, String message,
               HttpResponseStatus status) {
             if (failures.size() == 0) {
               failures.add(new Error(message));
-              ctx.getChannel().close();
+              ctx.channel().close();
             }
           }
           @Override
@@ -1082,11 +1819,12 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
                 new ShuffleHeader("attempt_12345_1_m_1_0", 5678, 5678, 1);
             DataOutputBuffer dob = new DataOutputBuffer();
             header.write(dob);
-            return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));
+            return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength()));
           }
         };
       }
     };
+    shuffleHandler.setUseOutboundExceptionHandler(true);
     shuffleHandler.setAuxiliaryLocalPathHandler(pathHandler);
     shuffleHandler.init(conf);
     try {
@@ -1094,8 +1832,8 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
       DataOutputBuffer outputBuffer = new DataOutputBuffer();
       outputBuffer.reset();
       Token<JobTokenIdentifier> jt =
-          new Token<JobTokenIdentifier>("identifier".getBytes(),
-          "password".getBytes(), new Text(user), new Text("shuffleService"));
+          new Token<>("identifier".getBytes(),
+              "password".getBytes(), new Text(user), new Text("shuffleService"));
       jt.write(outputBuffer);
       shuffleHandler
           .initializeApplication(new ApplicationInitializationContext(user,
@@ -1108,7 +1846,7 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
                       ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
                   + "/mapOutput?job=job_12345_0001&reduce=" + reducerId
                   + "&map=attempt_12345_1_m_1_0");
-      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+      HttpURLConnection conn = TEST_EXECUTION.openConnection(url);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
           ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
       conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
@@ -1122,7 +1860,7 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
       } catch (EOFException e) {
         // ignore
       }
-      Assert.assertEquals("sendError called due to shuffle error",
+      assertEquals("sendError called due to shuffle error",
           0, failures.size());
     } finally {
       shuffleHandler.stop();
@@ -1133,11 +1871,10 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx,
   @Test(timeout = 4000)
   public void testSendMapCount() throws Exception {
     final List<ShuffleHandler.ReduceMapFileCount> listenerList =
-        new ArrayList<ShuffleHandler.ReduceMapFileCount>();
-
+        new ArrayList<>();
+    int connectionKeepAliveTimeOut = 5; //arbitrary value
     final ChannelHandlerContext mockCtx =
         mock(ChannelHandlerContext.class);
-    final MessageEvent mockEvt = mock(MessageEvent.class);
     final Channel mockCh = mock(AbstractChannel.class);
     final ChannelPipeline mockPipeline = mock(ChannelPipeline.class);
 
@@ -1146,29 +1883,23 @@ public void testSendMapCount() throws Exception {
     final ChannelFuture mockFuture = createMockChannelFuture(mockCh,
         listenerList);
     final ShuffleHandler.TimeoutHandler timerHandler =
-        new ShuffleHandler.TimeoutHandler();
+        new ShuffleHandler.TimeoutHandler(connectionKeepAliveTimeOut);
 
     // Mock Netty Channel Context and Channel behavior
-    Mockito.doReturn(mockCh).when(mockCtx).getChannel();
-    when(mockCh.getPipeline()).thenReturn(mockPipeline);
+    Mockito.doReturn(mockCh).when(mockCtx).channel();
+    when(mockCh.pipeline()).thenReturn(mockPipeline);
     when(mockPipeline.get(
         Mockito.any(String.class))).thenReturn(timerHandler);
-    when(mockCtx.getChannel()).thenReturn(mockCh);
-    Mockito.doReturn(mockFuture).when(mockCh).write(Mockito.any(Object.class));
-    when(mockCh.write(Object.class)).thenReturn(mockFuture);
+    when(mockCtx.channel()).thenReturn(mockCh);
+    Mockito.doReturn(mockFuture).when(mockCh).writeAndFlush(Mockito.any(Object.class));
 
-    //Mock MessageEvent behavior
-    Mockito.doReturn(mockCh).when(mockEvt).getChannel();
-    when(mockEvt.getChannel()).thenReturn(mockCh);
-    Mockito.doReturn(mockHttpRequest).when(mockEvt).getMessage();
-
-    final ShuffleHandler sh = new MockShuffleHandler();
+    final MockShuffleHandler sh = new MockShuffleHandler();
     Configuration conf = new Configuration();
     sh.init(conf);
     sh.start();
     int maxOpenFiles =conf.getInt(ShuffleHandler.SHUFFLE_MAX_SESSION_OPEN_FILES,
         ShuffleHandler.DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES);
-    sh.getShuffle(conf).messageReceived(mockCtx, mockEvt);
+    sh.getShuffle(conf).channelRead(mockCtx, mockHttpRequest);
     assertTrue("Number of Open files should not exceed the configured " +
             "value!-Not Expected",
         listenerList.size() <= maxOpenFiles);
@@ -1179,23 +1910,97 @@ public void testSendMapCount() throws Exception {
           listenerList.size() <= maxOpenFiles);
     }
     sh.close();
+    sh.stop();
+
+    assertEquals("Should have no caught exceptions",
+        Collections.emptyList(), sh.failures);
+  }
+
+  @Test(timeout = 10000)
+  public void testIdleStateHandlingSpecifiedTimeout() throws Exception {
+    int timeoutSeconds = 4;
+    int expectedTimeoutSeconds = timeoutSeconds;
+    testHandlingIdleState(timeoutSeconds, expectedTimeoutSeconds);
+  }
+
+  @Test(timeout = 10000)
+  public void testIdleStateHandlingNegativeTimeoutDefaultsTo1Second() throws Exception {
+    int expectedTimeoutSeconds = 1; //expected by production code
+    testHandlingIdleState(ARBITRARY_NEGATIVE_TIMEOUT_SECONDS, expectedTimeoutSeconds);
+  }
+
+  private String getShuffleUrlWithKeepAlive(ShuffleHandler shuffleHandler, long jobId,
+      long... attemptIds) {
+    String url = getShuffleUrl(shuffleHandler, jobId, attemptIds);
+    return url + "&keepAlive=true";
+  }
+
+  private String getShuffleUrl(ShuffleHandler shuffleHandler, long jobId, long... attemptIds) {
+    String port = shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY);
+    String shuffleBaseURL = "http://127.0.0.1:" + port;
+
+    StringBuilder mapAttemptIds = new StringBuilder();
+    for (int i = 0; i < attemptIds.length; i++) {
+      if (i == 0) {
+        mapAttemptIds.append("&map=");
+      } else {
+        mapAttemptIds.append(",");
+      }
+      mapAttemptIds.append(String.format("attempt_%s_1_m_1_0", attemptIds[i]));
+    }
+
+    String location = String.format("/mapOutput" +
+        "?job=job_%s_1" +
+        "&reduce=1" +
+        "%s", jobId, mapAttemptIds);
+    return shuffleBaseURL + location;
+  }
+
+  private void testHandlingIdleState(int configuredTimeoutSeconds, int expectedTimeoutSeconds)
+      throws IOException,
+      InterruptedException {
+    Configuration conf = new Configuration();
+    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, TEST_EXECUTION.shuffleHandlerPort());
+    conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true);
+    conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, configuredTimeoutSeconds);
+
+    final CountDownLatch countdownLatch = new CountDownLatch(1);
+    ResponseConfig responseConfig = new ResponseConfig(HEADER_WRITE_COUNT, 0, 0);
+    ShuffleHandlerForKeepAliveTests shuffleHandler = new ShuffleHandlerForKeepAliveTests(
+        ATTEMPT_ID, responseConfig,
+        event -> countdownLatch.countDown());
+    shuffleHandler.init(conf);
+    shuffleHandler.start();
+
+    String shuffleUrl = getShuffleUrl(shuffleHandler, ATTEMPT_ID, ATTEMPT_ID);
+    String[] urls = new String[] {shuffleUrl};
+    HttpConnectionHelper httpConnectionHelper = new HttpConnectionHelper(
+        shuffleHandler.lastSocketAddress);
+    long beforeConnectionTimestamp = System.currentTimeMillis();
+    httpConnectionHelper.connectToUrls(urls, shuffleHandler.responseConfig);
+    countdownLatch.await();
+    long channelClosedTimestamp = System.currentTimeMillis();
+    long secondsPassed =
+        TimeUnit.SECONDS.convert(channelClosedTimestamp - beforeConnectionTimestamp,
+            TimeUnit.MILLISECONDS);
+    assertTrue(String.format("Expected at least %s seconds of timeout. " +
+            "Actual timeout seconds: %s", expectedTimeoutSeconds, secondsPassed),
+        secondsPassed >= expectedTimeoutSeconds);
+    shuffleHandler.stop();
   }
 
   public ChannelFuture createMockChannelFuture(Channel mockCh,
       final List<ShuffleHandler.ReduceMapFileCount> listenerList) {
     final ChannelFuture mockFuture = mock(ChannelFuture.class);
-    when(mockFuture.getChannel()).thenReturn(mockCh);
+    when(mockFuture.channel()).thenReturn(mockCh);
     Mockito.doReturn(true).when(mockFuture).isSuccess();
-    Mockito.doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocation) throws Throwable {
-        //Add ReduceMapFileCount listener to a list
-        if (invocation.getArguments()[0].getClass() ==
-            ShuffleHandler.ReduceMapFileCount.class)
-          listenerList.add((ShuffleHandler.ReduceMapFileCount)
-              invocation.getArguments()[0]);
-        return null;
+    Mockito.doAnswer(invocation -> {
+      //Add ReduceMapFileCount listener to a list
+      if (invocation.getArguments()[0].getClass() == ShuffleHandler.ReduceMapFileCount.class) {
+        listenerList.add((ShuffleHandler.ReduceMapFileCount)
+            invocation.getArguments()[0]);
       }
+      return null;
     }).when(mockFuture).addListener(Mockito.any(
         ShuffleHandler.ReduceMapFileCount.class));
     return mockFuture;
@@ -1203,16 +2008,14 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
 
   public HttpRequest createMockHttpRequest() {
     HttpRequest mockHttpRequest = mock(HttpRequest.class);
-    Mockito.doReturn(HttpMethod.GET).when(mockHttpRequest).getMethod();
-    Mockito.doAnswer(new Answer() {
-      @Override
-      public Object answer(InvocationOnMock invocation) throws Throwable {
-        String uri = "/mapOutput?job=job_12345_1&reduce=1";
-        for (int i = 0; i < 100; i++)
-          uri = uri.concat("&map=attempt_12345_1_m_" + i + "_0");
-        return uri;
+    Mockito.doReturn(HttpMethod.GET).when(mockHttpRequest).method();
+    Mockito.doAnswer(invocation -> {
+      String uri = "/mapOutput?job=job_12345_1&reduce=1";
+      for (int i = 0; i < 100; i++) {
+        uri = uri.concat("&map=attempt_12345_1_m_" + i + "_0");
       }
-    }).when(mockHttpRequest).getUri();
+      return uri;
+    }).when(mockHttpRequest).uri();
     return mockHttpRequest;
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/resources/log4j.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/resources/log4j.properties
index 81a3f6ad5d248..b7d8ad36efc26 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/resources/log4j.properties
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/resources/log4j.properties
@@ -17,3 +17,5 @@ log4j.threshold=ALL
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
+log4j.logger.io.netty=INFO
+log4j.logger.org.apache.hadoop.mapred=INFO
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index 862d68ebc0aaa..52b6dde3794d8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -331,6 +331,8 @@ void buildPackage()
     LOG.info("Compressing tarball");
     try (TarArchiveOutputStream out = new TarArchiveOutputStream(
         targetStream)) {
+      // Workaround for the compress issue present from 1.21: COMPRESS-587
+      out.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
       for (String fullPath : filteredInputFiles) {
         LOG.info("Adding " + fullPath);
         File file = new File(fullPath);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 86994dffc5b64..fdcab2f2ffbef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -68,6 +68,10 @@
           <artifactId>paranamer-ant</artifactId>
           <groupId>com.thoughtworks.paranamer</groupId>
         </exclusion>
+        <exclusion>
+          <groupId>org.xerial.snappy</groupId>
+          <artifactId>snappy-java</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -103,6 +107,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -120,7 +130,7 @@
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>
-      <artifactId>netty</artifactId>
+      <artifactId>netty-all</artifactId>
     </dependency>
     <dependency>
       <groupId>commons-logging</groupId>
@@ -177,38 +187,6 @@
          </properties>
        </configuration>
      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
   <profiles>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index c4a804fabe4b7..e5426a08b3c5b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -70,34 +70,40 @@
       <scope>runtime</scope>
     </dependency>
     <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-hdfs</artifactId>
-       <scope>test</scope>
-       <type>test-jar</type>
-     </dependency>
-     <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-yarn-server-tests</artifactId>
-       <scope>test</scope>
-       <type>test-jar</type>
-     </dependency>
-     <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-mapreduce-client-app</artifactId>
-       <scope>provided</scope>
-     </dependency>
-     <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-mapreduce-client-app</artifactId>
-       <type>test-jar</type>
-       <scope>test</scope>
-     </dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-tests</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.sun.jersey.jersey-test-framework</groupId>
       <artifactId>jersey-test-framework-grizzly2</artifactId>
       <scope>test</scope>
     </dependency>
-     <dependency>
+    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-mapreduce-client-hs</artifactId>
        <scope>test</scope>
@@ -106,12 +112,13 @@
        <groupId>org.hsqldb</groupId>
        <artifactId>hsqldb</artifactId>
        <scope>provided</scope>
+       <classifier>jdk8</classifier>
      </dependency>
      <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
       <artifactId>hadoop-shaded-guava</artifactId>
       <scope>provided</scope>
-     </dependency>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -121,6 +128,21 @@
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   
   <build>
@@ -155,38 +177,6 @@
           </excludes>
         </configuration>
       </plugin>
-       <plugin>
-           <groupId>org.apache.maven.plugins</groupId>
-           <artifactId>maven-enforcer-plugin</artifactId>
-           <dependencies>
-               <dependency>
-                   <groupId>de.skuzzle.enforcer</groupId>
-                   <artifactId>restrict-imports-enforcer-rule</artifactId>
-                   <version>${restrict-imports.enforcer.version}</version>
-               </dependency>
-           </dependencies>
-           <executions>
-               <execution>
-                   <id>banned-illegal-imports</id>
-                   <phase>process-sources</phase>
-                   <goals>
-                       <goal>enforce</goal>
-                   </goals>
-                   <configuration>
-                       <rules>
-                           <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                               <includeTestCode>true</includeTestCode>
-                               <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                               <bannedImports>
-                                   <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                                   <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                               </bannedImports>
-                           </restrictImports>
-                       </rules>
-                   </configuration>
-               </execution>
-           </executions>
-       </plugin>
    </plugins>
    </build>
 </project>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/AggregateWordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/AggregateWordCount.java
index 83f8161c72ee6..3aaf0020cd262 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/AggregateWordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/AggregateWordCount.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor;
 import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob;
+import org.apache.hadoop.util.ExitUtil;
 
 /**
  * This is an example Aggregated Hadoop Map/Reduce application. It reads the
@@ -72,7 +73,7 @@ public static void main(String[] args)
         , new Class[] {WordCountPlugInClass.class});
     job.setJarByClass(AggregateWordCount.class);
     int ret = job.waitForCompletion(true) ? 0 : 1;
-    System.exit(ret);
+    ExitUtil.terminate(ret);
   }
 
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/LongLong.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/LongLong.java
index 8d7e7a0a9713d..8299ba161a80b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/LongLong.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/math/LongLong.java
@@ -84,7 +84,7 @@ static LongLong multiplication(final LongLong r, final long a, final long b) {
     final long v = x1*y1;
 
     final long tmp = (t - u)>>>1;
-    result.d0 = ((t + u)>>>1) - v + ((tmp << MID) & FULL_MASK);;
+    result.d0 = ((t + u)>>>1) - v + ((tmp << MID) & FULL_MASK);
     result.d1 = v + (tmp >> MID);
     return result;
     */
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraChecksum.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraChecksum.java
index 652f32c00b576..12d34ea7d6639 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraChecksum.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraChecksum.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Cluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
index ef20c4c23bd58..161e9715c8b0f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
@@ -33,7 +33,6 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.mapreduce.Cluster;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
index 20ce8ef2b60de..f284a9c380756 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -41,6 +42,12 @@
 import org.apache.hadoop.util.IndexedSortable;
 import org.apache.hadoop.util.QuickSort;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.functional.FutureIO;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_END;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_START;
 
 /**
  * An input format that reads the first 10 characters of each line as the key
@@ -224,12 +231,17 @@ public void initialize(InputSplit split, TaskAttemptContext context)
         throws IOException, InterruptedException {
       Path p = ((FileSplit)split).getPath();
       FileSystem fs = p.getFileSystem(context.getConfiguration());
-      in = fs.open(p);
       long start = ((FileSplit)split).getStart();
       // find the offset to start at a record boundary
       offset = (RECORD_LENGTH - (start % RECORD_LENGTH)) % RECORD_LENGTH;
-      in.seek(start + offset);
       length = ((FileSplit)split).getLength();
+      final FutureDataInputStreamBuilder builder = fs.openFile(p)
+          .opt(FS_OPTION_OPENFILE_SPLIT_START, start)
+          .opt(FS_OPTION_OPENFILE_SPLIT_END, start + length)
+          .opt(FS_OPTION_OPENFILE_READ_POLICY,
+              FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL);
+      in = FutureIO.awaitFuture(builder.build());
+      in.seek(start + offset);
     }
 
     public void close() throws IOException {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraValidate.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraValidate.java
index 6cff25f5def7c..f722fc41341a0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraValidate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraValidate.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Cluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestAggregateWordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestAggregateWordCount.java
new file mode 100644
index 0000000000000..9828b24d98fdf
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestAggregateWordCount.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.examples;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ExitUtil.ExitException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestAggregateWordCount extends HadoopTestCase {
+  public TestAggregateWordCount() throws IOException {
+    super(LOCAL_MR, LOCAL_FS, 1, 1);
+  }
+
+  @BeforeEach
+  public void setUp() throws Exception {
+    super.setUp();
+  }
+
+  @AfterEach
+  public void tearDown() throws Exception {
+    FileSystem fs = getFileSystem();
+    if (fs != null) {
+      fs.delete(TEST_DIR, true);
+    }
+    super.tearDown();
+  }
+
+  // Input/Output paths for sort
+  private static final Path TEST_DIR = new Path(
+      new File(System.getProperty("test.build.data", "/tmp"),
+          "aggregatewordcount").getAbsoluteFile().toURI().toString());
+
+  private static final Path INPUT_PATH = new Path(TEST_DIR, "inPath");
+  private static final Path OUTPUT_PATH = new Path(TEST_DIR, "outPath");
+
+  @Test
+  void testAggregateTestCount()
+      throws IOException, ClassNotFoundException, InterruptedException {
+
+    ExitUtil.disableSystemExit();
+    FileSystem fs = getFileSystem();
+    fs.mkdirs(INPUT_PATH);
+    Path file1 = new Path(INPUT_PATH, "file1");
+    Path file2 = new Path(INPUT_PATH, "file2");
+    FileUtil.write(fs, file1, "Hello World");
+    FileUtil.write(fs, file2, "Hello Hadoop");
+
+    String[] args =
+        new String[]{INPUT_PATH.toString(), OUTPUT_PATH.toString(), "1",
+            "textinputformat"};
+
+    // Run AggregateWordCount Job.
+    try {
+      AggregateWordCount.main(args);
+    } catch (ExitException e) {
+      assertEquals(0, e.status);
+    }
+
+    String allEntries;
+    try (FSDataInputStream stream = fs
+        .open(new Path(OUTPUT_PATH, "part-r-00000"));) {
+      allEntries = IOUtils.toString(stream, Charset.defaultCharset());
+    }
+
+    assertEquals("Hadoop\t1\n" + "Hello\t2\n" + "World\t1\n", allEntries);
+  }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestBaileyBorweinPlouffe.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestBaileyBorweinPlouffe.java
index 2df2df091175d..ffec884795ec1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestBaileyBorweinPlouffe.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestBaileyBorweinPlouffe.java
@@ -18,35 +18,36 @@
 package org.apache.hadoop.examples;
 
 import java.math.BigInteger;
-import org.junit.Test;
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /** Tests for BaileyBorweinPlouffe */
 public class TestBaileyBorweinPlouffe {
 
   @Test
-  public void testMod() {
+  void testMod() {
     final BigInteger TWO = BigInteger.ONE.add(BigInteger.ONE);
-    for(long n = 3; n < 100; n++) {
+    for (long n = 3; n < 100; n++) {
       for (long e = 1; e < 100; e++) {
         final long r = TWO.modPow(
             BigInteger.valueOf(e), BigInteger.valueOf(n)).longValue();
-        Assert.assertEquals("e=" + e + ", n=" + n, r, BaileyBorweinPlouffe
-            .mod(e, n));
+        assertEquals(r, BaileyBorweinPlouffe
+            .mod(e, n), "e=" + e + ", n=" + n);
       }
     }
   }
 
   @Test
-  public void testHexDigit() {
+  void testHexDigit() {
     final long[] answers = {0x43F6, 0xA308, 0x29B7, 0x49F1, 0x8AC8, 0x35EA};
     long d = 1;
-    for(int i = 0; i < answers.length; i++) {
-      Assert.assertEquals("d=" + d, answers[i], BaileyBorweinPlouffe
-          .hexDigits(d));
+    for (int i = 0; i < answers.length; i++) {
+      assertEquals(answers[i], BaileyBorweinPlouffe
+          .hexDigits(d), "d=" + d);
       d *= 10;
     }
 
-    Assert.assertEquals(0x243FL, BaileyBorweinPlouffe.hexDigits(0));
- }
+    assertEquals(0x243FL, BaileyBorweinPlouffe.hexDigits(0));
+  }
 }
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java
index 3f0ffd63f6de0..45aa4102bd10c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java
@@ -17,8 +17,6 @@
 */
 package org.apache.hadoop.examples;
 
-import static org.junit.Assert.assertEquals;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
@@ -31,9 +29,11 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.ToolRunner;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestWordStats {
 
@@ -241,13 +241,14 @@ public static boolean deleteDir(File dir) {
     return dir.delete();
   }
 
-  @Before public void setup() throws Exception {
+  @BeforeEach public void setup() throws Exception {
     deleteDir(new File(MEAN_OUTPUT));
     deleteDir(new File(MEDIAN_OUTPUT));
     deleteDir(new File(STDDEV_OUTPUT));
   }
 
-  @Test public void testGetTheMean() throws Exception {
+  @Test
+  void testGetTheMean() throws Exception {
     String args[] = new String[2];
     args[0] = INPUT;
     args[1] = MEAN_OUTPUT;
@@ -261,7 +262,8 @@ public static boolean deleteDir(File dir) {
     assertEquals(mean, wr.read(INPUT), 0.0);
   }
 
-  @Test public void testGetTheMedian() throws Exception {
+  @Test
+  void testGetTheMedian() throws Exception {
     String args[] = new String[2];
     args[0] = INPUT;
     args[1] = MEDIAN_OUTPUT;
@@ -275,7 +277,8 @@ public static boolean deleteDir(File dir) {
     assertEquals(median, wr.read(INPUT), 0.0);
   }
 
-  @Test public void testGetTheStandardDeviation() throws Exception {
+  @Test
+  void testGetTheStandardDeviation() throws Exception {
     String args[] = new String[2];
     args[0] = INPUT;
     args[1] = STDDEV_OUTPUT;
@@ -289,7 +292,7 @@ public static boolean deleteDir(File dir) {
     assertEquals(stddev, wr.read(INPUT), 0.0);
   }
 
-  @AfterClass public static void cleanup() throws Exception {
+  @AfterAll public static void cleanup() throws Exception {
     deleteDir(new File(MEAN_OUTPUT));
     deleteDir(new File(MEDIAN_OUTPUT));
     deleteDir(new File(STDDEV_OUTPUT));
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java
index 232c53f4d47cc..318306cf9c339 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestLongLong.java
@@ -19,8 +19,9 @@
 
 import java.math.BigInteger;
 import java.util.Random;
-import org.junit.Test;
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestLongLong {
 
@@ -39,12 +40,12 @@ static void verifyMultiplication(long a, long b) {
         "\na = %x\nb = %x\nll= " + ll + "\nbi= " + bi.toString(16) + "\n", a,
         b);
     //System.out.println(s);
-    Assert.assertEquals(s, bi, ll.toBigInteger());
+    assertEquals(bi, ll.toBigInteger(), s);
   }
 
   @Test
-  public void testMultiplication() {
-    for(int i = 0; i < 100; i++) {
+  void testMultiplication() {
+    for (int i = 0; i < 100; i++) {
       final long a = nextPositiveLong();
       final long b = nextPositiveLong();
       verifyMultiplication(a, b);
@@ -54,8 +55,8 @@ public void testMultiplication() {
   }
 
   @Test
-  public void testRightShift() {
-    for(int i = 0; i < 1000; i++) {
+  void testRightShift() {
+    for (int i = 0; i < 1000; i++) {
       final long a = nextPositiveLong();
       final long b = nextPositiveLong();
       verifyRightShift(a, b);
@@ -69,12 +70,12 @@ private static void verifyRightShift(long a, long b) {
     final String s = String.format(
         "\na = %x\nb = %x\nll= " + ll + "\nbi= " + bi.toString(16) + "\n", a,
         b);
-    Assert.assertEquals(s, bi, ll.toBigInteger());
+    assertEquals(bi, ll.toBigInteger(), s);
 
     for (int i = 0; i < LongLong.SIZE >> 1; i++) {
       final long result = ll.shiftRight(i) & MASK;
       final long expected = bi.shiftRight(i).longValue() & MASK;
-      Assert.assertEquals(s, expected, result);
+      assertEquals(expected, result, s);
     }
   }
 }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestModular.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestModular.java
index a75ec29d1c962..96e70efc76b73 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestModular.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestModular.java
@@ -21,8 +21,9 @@
 import java.util.Random;
 
 import org.apache.hadoop.examples.pi.Util.Timer;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestModular{
   private static final Random RANDOM = new Random();
@@ -52,13 +53,13 @@ static long div(long sum, long r, long n) {
   }
 
   @Test
-  public void testDiv() {
-    for(long n = 2; n < 100; n++)
-      for(long r = 1; r < n; r++) {
+  void testDiv() {
+    for (long n = 2; n < 100; n++)
+      for (long r = 1; r < n; r++) {
         final long a = div(0, r, n);
-        final long b = (long)((r*1.0/n) * (1L << DIV_VALID_BIT));
+        final long b = (long) ((r * 1.0 / n) * (1L << DIV_VALID_BIT));
         final String s = String.format("r=%d, n=%d, a=%X, b=%X", r, n, a, b);
-        Assert.assertEquals(s, b, a);
+        assertEquals(b, a, s);
       }
   }
 
@@ -151,9 +152,8 @@ static void squareBenchmarks() {
         final long answer = rn[i][j][1];
         final long s = square_slow(r, n);
         if (s != answer) {
-          Assert.assertEquals(
-              "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -168,9 +168,8 @@ static void squareBenchmarks() {
         final long answer = rn[i][j][1];
         final long s = square(r, n, r2p64);
         if (s != answer) {
-          Assert.assertEquals(
-              "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -185,9 +184,8 @@ static void squareBenchmarks() {
         final BigInteger R = BigInteger.valueOf(r);
         final long s = R.multiply(R).mod(N).longValue();
         if (s != answer) {
-          Assert.assertEquals(
-              "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -202,9 +200,8 @@ static void squareBenchmarks() {
         final BigInteger R = BigInteger.valueOf(r);
         final long s = R.modPow(TWO, N).longValue();
         if (s != answer) {
-          Assert.assertEquals(
-              "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "r=" + r + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -299,9 +296,8 @@ static void modBenchmarks() {
         final long answer = en[i][j][1];
         final long s = Modular.mod(e, n);
         if (s != answer) {
-          Assert.assertEquals(
-              "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -316,9 +312,8 @@ static void modBenchmarks() {
         final long answer = en[i][j][1];
         final long s = m2.mod(e);
         if (s != answer) {
-          Assert.assertEquals(
-              "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -332,9 +327,8 @@ static void modBenchmarks() {
         final long answer = en[i][j][1];
         final long s = m2.mod2(e);
         if (s != answer) {
-          Assert.assertEquals(
-              "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
@@ -348,9 +342,8 @@ static void modBenchmarks() {
         final long answer = en[i][j][1];
         final long s = TWO.modPow(BigInteger.valueOf(e), N).longValue();
         if (s != answer) {
-          Assert.assertEquals(
-              "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s,
-              answer, s);
+          assertEquals(
+              answer, s, "e=" + e + ", n=" + n + ", answer=" + answer + " but s=" + s);
         }
       }
     }
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestSummation.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestSummation.java
index 2741962b329e0..92012ac4d9512 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestSummation.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/pi/math/TestSummation.java
@@ -28,8 +28,9 @@
 import org.apache.hadoop.examples.pi.Util;
 import org.apache.hadoop.examples.pi.Util.Timer;
 import org.apache.hadoop.examples.pi.math.TestModular.Montgomery2;
-import org.junit.Test;
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestSummation {
   static final Random RANDOM = new Random();
@@ -58,12 +59,12 @@ private static void runTestSubtract(Summation sigma, List<Summation> diff) {
 
     final List<Summation> combined = Util.combine(a);
 //    Util.out.println("combined=" + combined);
-    Assert.assertEquals(1, combined.size());
-    Assert.assertEquals(sigma, combined.get(0));
+    assertEquals(1, combined.size());
+    assertEquals(sigma, combined.get(0));
   }
 
   @Test
-  public void testSubtract() {
+  void testSubtract() {
     final Summation sigma = newSummation(3, 10000, 20);
     final int size = 10;
     final List<Summation> parts = Arrays.asList(sigma.partition(size));
@@ -72,10 +73,10 @@ public void testSubtract() {
     runTestSubtract(sigma, new ArrayList<Summation>());
     runTestSubtract(sigma, parts);
 
-    for(int n = 1; n < size; n++) {
-      for(int j = 0; j < 10; j++) {
+    for (int n = 1; n < size; n++) {
+      for (int j = 0; j < 10; j++) {
         final List<Summation> diff = new ArrayList<Summation>(parts);
-        for(int i = 0; i < n; i++)
+        for (int i = 0; i < n; i++)
           diff.remove(RANDOM.nextInt(diff.size()));
 ///        Collections.sort(diff);
         runTestSubtract(sigma, diff);
@@ -132,16 +133,16 @@ private static void computeBenchmarks(final Summation2 sigma) {
     t.tick("sigma=" + sigma);
     final double value = sigma.compute();
     t.tick("compute=" + value);
-    Assert.assertEquals(value, sigma.compute_modular(), DOUBLE_DELTA);
+    assertEquals(value, sigma.compute_modular(), DOUBLE_DELTA);
     t.tick("compute_modular");
-    Assert.assertEquals(value, sigma.compute_montgomery(), DOUBLE_DELTA);
+    assertEquals(value, sigma.compute_montgomery(), DOUBLE_DELTA);
     t.tick("compute_montgomery");
-    Assert.assertEquals(value, sigma.compute_montgomery2(), DOUBLE_DELTA);
+    assertEquals(value, sigma.compute_montgomery2(), DOUBLE_DELTA);
     t.tick("compute_montgomery2");
 
-    Assert.assertEquals(value, sigma.compute_modBigInteger(), DOUBLE_DELTA);
+    assertEquals(value, sigma.compute_modBigInteger(), DOUBLE_DELTA);
     t.tick("compute_modBigInteger");
-    Assert.assertEquals(value, sigma.compute_modPow(), DOUBLE_DELTA);
+    assertEquals(value, sigma.compute_modPow(), DOUBLE_DELTA);
     t.tick("compute_modPow");
   }
 
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
index 1bc9f2f56a2e8..ef1688c09290f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/terasort/TestTeraSort.java
@@ -25,14 +25,15 @@
 import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTeraSort extends HadoopTestCase {
   private static final Logger LOG = LoggerFactory.getLogger(TestTeraSort.class);
@@ -42,7 +43,12 @@ public TestTeraSort()
     super(LOCAL_MR, LOCAL_FS, 1, 1);
   }
 
-  @After
+  @BeforeEach
+  public void setUp() throws Exception {
+    super.setUp();
+  }
+
+  @AfterEach
   public void tearDown() throws Exception {
     getFileSystem().delete(TEST_DIR, true);
     super.tearDown();
@@ -85,7 +91,7 @@ private void runTeraValidator(Configuration job,
   }
 
   @Test
-  public void testTeraSort() throws Exception {
+  void testTeraSort() throws Exception {
     // Run TeraGen to generate input for 'terasort'
     runTeraGen(createJobConf(), SORT_INPUT_PATH);
 
@@ -110,11 +116,11 @@ public void testTeraSort() throws Exception {
 
     // Run tera-validator to check if sort worked correctly
     runTeraValidator(createJobConf(), SORT_OUTPUT_PATH,
-      TERA_OUTPUT_PATH);
+        TERA_OUTPUT_PATH);
   }
 
   @Test
-  public void testTeraSortWithLessThanTwoArgs() throws Exception {
+  void testTeraSortWithLessThanTwoArgs() throws Exception {
     String[] args = new String[1];
     assertThat(new TeraSort().run(args)).isEqualTo(2);
   }
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 09b4d42606118..21554090d7855 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -41,113 +41,51 @@
     <module>hadoop-mapreduce-examples</module>
   </modules>
 
+  <!--
+  Do not add 3rd party dependencies here, add them to the POM of the leaf module.
+  The dependencies in this module are for the assembly plugin, packaging purposes.
+  -->
   <dependencies>
-    <dependency>
-      <groupId>com.google.protobuf</groupId>
-      <artifactId>protobuf-java</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-server</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.ant</groupId>
-          <artifactId>ant</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>io.netty</groupId>
-          <artifactId>netty</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.velocity</groupId>
-          <artifactId>velocity</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <artifactId>paranamer-ant</artifactId>
-          <groupId>com.thoughtworks.paranamer</groupId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-     <groupId>org.slf4j</groupId>
-       <artifactId>slf4j-api</artifactId>
-    </dependency>
-    <dependency>
-     <groupId>org.slf4j</groupId>
-       <artifactId>slf4j-log4j12</artifactId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
-      <scope>test</scope>
+      <artifactId>hadoop-mapreduce-client-common</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <type>test-jar</type>
-      <scope>test</scope>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.google.inject</groupId>
-      <artifactId>guice</artifactId>
+      <artifactId>hadoop-mapreduce-client-hs</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-server</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.sun.jersey.contribs</groupId>
-      <artifactId>jersey-guice</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.google.inject.extensions</groupId>
-      <artifactId>guice-servlet</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.hsqldb</groupId>
-      <artifactId>hsqldb</artifactId>
-      <scope>compile</scope>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-shuffle</artifactId>
+      <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>${leveldbjni.group}</groupId>
-      <artifactId>leveldbjni-all</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-examples</artifactId>
+      <version>${project.version}</version>
     </dependency>
-
   </dependencies>
 
   <build>
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index a5473258c35f6..522c5a9468705 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -39,6 +39,12 @@
       <groupId>org.apache.maven</groupId>
       <artifactId>maven-core</artifactId>
       <version>${maven.dependency.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.sonatype.sisu</groupId>
+          <artifactId>sisu-inject-plexus</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.maven.plugin-tools</groupId>
@@ -62,44 +68,6 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.maven.plugins</groupId>
-      <artifactId>maven-shade-plugin</artifactId>
-      <version>${maven-shade-plugin.version}</version>
-      <scope>provided</scope>
-      <exclusions>
-        <!-- shade conflicts with itself and other maven internals so we simply
-             exclude shade's (older) copies of the dependencies -->
-        <exclusion>
-          <groupId>org.apache.maven</groupId>
-          <artifactId>maven-artifact</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.maven</groupId>
-          <artifactId>maven-compat</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.maven</groupId>
-          <artifactId>maven-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.maven</groupId>
-          <artifactId>maven-model</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.maven</groupId>
-          <artifactId>maven-plugin-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.vafer</groupId>
-          <artifactId>jdependency</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.sonatype.sisu</groupId>
-          <artifactId>sisu-inject-plexus</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
       <artifactId>hadoop-shaded-guava</artifactId>
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java
index e83a8cd957fd1..c84bb1b38cb9f 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java
@@ -30,6 +30,7 @@
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -172,6 +173,7 @@ private long computeChecksum(File file) throws IOException {
 
     public void writeChecksums() throws IOException {
       ObjectMapper mapper = new ObjectMapper();
+      Files.createDirectories(checksumFile.getParentFile().toPath());
       try (BufferedOutputStream out = new BufferedOutputStream(
           new FileOutputStream(checksumFile))) {
         mapper.writeValue(out, computedChecksums);
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
deleted file mode 100644
index 2d5de7abf1750..0000000000000
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.hadoop.maven.plugin.shade.resource;
-
-import java.io.BufferedReader;
-import org.apache.maven.plugins.shade.relocation.Relocator;
-import org.apache.maven.plugins.shade.resource.ResourceTransformer;
-import org.codehaus.plexus.util.IOUtil;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.jar.JarEntry;
-import java.util.jar.JarOutputStream;
-
-/**
- * Resources transformer that appends entries in META-INF/services resources
- * into a single resource. For example, if there are several
- * META-INF/services/org.apache.maven.project.ProjectBuilder resources spread
- * across many JARs the individual entries will all be concatenated into a
- * single META-INF/services/org.apache.maven.project.ProjectBuilder resource
- * packaged into the resultant JAR produced by the shading process.
- *
- * From following sources, only needed until MSHADE-182 gets released
- * * https://s.apache.org/vwjl (source in maven-shade-plugin repo)
- * * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch
- *
- * Has been reformatted according to Hadoop checkstyle rules and modified
- * to meet Hadoop's threshold for Findbugs problems.
- */
-public class ServicesResourceTransformer
-    implements ResourceTransformer {
-
-  private static final String SERVICES_PATH = "META-INF/services";
-
-  private Map<String, ServiceStream> serviceEntries = new HashMap<>();
-
-  private List<Relocator> relocators;
-
-  public boolean canTransformResource(String resource) {
-    if (resource.startsWith(SERVICES_PATH)) {
-      return true;
-    }
-
-    return false;
-  }
-
-  public void processResource(String resource, InputStream is,
-      List<Relocator> relocatorz) throws IOException {
-    ServiceStream out = serviceEntries.get(resource);
-    if (out == null) {
-      out = new ServiceStream();
-      serviceEntries.put(resource, out);
-    }
-
-    out.append(is);
-    is.close();
-
-    if (this.relocators == null) {
-      this.relocators = relocatorz;
-    }
-  }
-
-  public boolean hasTransformedResource() {
-    return serviceEntries.size() > 0;
-  }
-
-  public void modifyOutputStream(JarOutputStream jos)
-      throws IOException {
-    for (Map.Entry<String, ServiceStream> entry : serviceEntries.entrySet()) {
-      String key = entry.getKey();
-      ServiceStream data = entry.getValue();
-
-      if (relocators != null) {
-        key = key.substring(SERVICES_PATH.length() + 1);
-        for (Relocator relocator : relocators) {
-          if (relocator.canRelocateClass(key)) {
-            key = relocator.relocateClass(key);
-            break;
-          }
-        }
-
-        key = SERVICES_PATH + '/' + key;
-      }
-
-      jos.putNextEntry(new JarEntry(key));
-
-      //read the content of service file for candidate classes for relocation
-      //presume everything is UTF8, because Findbugs barfs on default
-      //charset and this seems no worse a choice ¯\_(ツ)_/¯
-      PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos,
-          StandardCharsets.UTF_8));
-      InputStreamReader streamReader =
-          new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8);
-      BufferedReader reader = new BufferedReader(streamReader);
-      String className;
-
-      while ((className = reader.readLine()) != null) {
-
-        if (relocators != null) {
-          for (Relocator relocator : relocators) {
-            //if the class can be relocated then relocate it
-            if (relocator.canRelocateClass(className)) {
-              className = relocator.applyToSourceContent(className);
-              break;
-            }
-          }
-        }
-
-        writer.println(className);
-        writer.flush();
-      }
-
-      reader.close();
-      data.reset();
-    }
-  }
-
-  static class ServiceStream extends ByteArrayOutputStream {
-
-    public ServiceStream() {
-      super(1024);
-    }
-
-    public void append(InputStream is)
-        throws IOException {
-      if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') {
-        write('\n');
-      }
-
-      IOUtil.copy(is, this);
-    }
-
-    public InputStream toInputStream() {
-      return new ByteArrayInputStream(buf, 0, count);
-    }
-
-  }
-
-}
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index cf1c9417004c4..c0334b3fcc178 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -41,6 +41,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>compile</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
diff --git a/hadoop-project-dist/README.txt b/hadoop-project-dist/README.txt
index a4c759720fbbb..922c94aadb3a2 100644
--- a/hadoop-project-dist/README.txt
+++ b/hadoop-project-dist/README.txt
@@ -1,4 +1,4 @@
 DUMMY.
 
 Required for the assembly:single goal not to fail because there
-are not files in the hadoop-project-dist module.
+are no files in the hadoop-project-dist module.
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 024cf055207c5..53e2de63c4a7d 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -134,7 +134,7 @@
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
-        <jdiff.stable.api>3.3.1</jdiff.stable.api>
+        <jdiff.stable.api>3.3.4</jdiff.stable.api>
         <jdiff.stability>-unstable</jdiff.stability>
         <!-- Commented out for HADOOP-11776 -->
         <!-- Uncomment param name="${jdiff.compatibility}" in javadoc doclet if compatibility is not empty -->
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8de03f51c92b6..5c2fad157798e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -31,13 +31,13 @@
 
   <properties>
     <!-- Set the Release year during release -->
-    <release-year>2021</release-year>
+    <release-year>2022</release-year>
 
     <failIfNoTests>false</failIfNoTests>
     <!--Whether to proceed to next module if any test failures exist-->
-    <ignoreTestFailure>true</ignoreTestFailure>
+    <maven.test.failure.ignore>true</maven.test.failure.ignore>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
-    <jetty.version>9.4.43.v20210629</jetty.version>
+    <jetty.version>9.4.48.v20220622</jetty.version>
     <test.exclude>_</test.exclude>
     <test.exclude.pattern>_</test.exclude.pattern>
 
@@ -48,7 +48,7 @@
     <!-- JDIFF generation from embedded ant in the antrun plugin -->
     <jdiff.version>1.0.9</jdiff.version>
     <!-- Version number for xerces used by JDiff -->
-    <xerces.jdiff.version>2.12.1</xerces.jdiff.version>
+    <xerces.jdiff.version>2.12.2</xerces.jdiff.version>
 
     <kafka.version>2.8.1</kafka.version>
 
@@ -63,15 +63,14 @@
     <java.security.egd>file:///dev/urandom</java.security.egd>
 
     <!-- avro version -->
-    <avro.version>1.7.7</avro.version>
+    <avro.version>1.9.2</avro.version>
 
     <!-- jersey version -->
-    <jersey.version>1.19</jersey.version>
+    <jersey.version>1.19.4</jersey.version>
 
     <!-- jackson versions -->
-    <jackson.version>1.9.13</jackson.version>
-    <jackson2.version>2.10.5</jackson2.version>
-    <jackson2.databind.version>2.10.5.1</jackson2.databind.version>
+    <jackson2.version>2.12.7</jackson2.version>
+    <jackson2.databind.version>2.12.7.1</jackson2.databind.version>
 
     <!-- httpcomponents versions -->
     <httpclient.version>4.5.13</httpclient.version>
@@ -80,6 +79,7 @@
     <!-- SLF4J/LOG4J version -->
     <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
+    <log4j2.version>2.17.1</log4j2.version>
 
     <!-- com.google.re2j version -->
     <re2j.version>1.1</re2j.version>
@@ -105,11 +105,11 @@
     <guava.version>27.0-jre</guava.version>
     <guice.version>4.2.3</guice.version>
 
-    <bouncycastle.version>1.60</bouncycastle.version>
+    <bouncycastle.version>1.68</bouncycastle.version>
 
     <!-- Required for testing LDAP integration -->
-    <apacheds.version>2.0.0-M21</apacheds.version>
-    <ldap-api.version>1.0.0-M33</ldap-api.version>
+    <apacheds.version>2.0.0.AM26</apacheds.version>
+    <ldap-api.version>2.0.0</ldap-api.version>
 
     <!-- Apache Commons dependencies -->
     <commons-beanutils.version>1.9.4</commons-beanutils.version>
@@ -117,28 +117,31 @@
     <commons-codec.version>1.15</commons-codec.version>
     <commons-collections.version>3.2.2</commons-collections.version>
     <commons-compress.version>1.21</commons-compress.version>
-    <commons-csv.version>1.0</commons-csv.version>
-    <commons-io.version>2.8.0</commons-io.version>
-    <commons-lang3.version>3.7</commons-lang3.version>
+    <commons-csv.version>1.9.0</commons-csv.version>
+    <commons-io.version>2.11.0</commons-io.version>
+    <commons-lang3.version>3.12.0</commons-lang3.version>
     <commons-logging.version>1.1.3</commons-logging.version>
     <commons-logging-api.version>1.1</commons-logging-api.version>
-    <commons-math3.version>3.1.1</commons-math3.version>
-    <commons-net.version>3.6</commons-net.version>
-    <commons-text.version>1.4</commons-text.version>
+    <commons-math3.version>3.6.1</commons-math3.version>
+    <commons-net.version>3.8.0</commons-net.version>
+    <commons-text.version>1.10.0</commons-text.version>
 
-    <kerby.version>1.0.1</kerby.version>
+    <kerby.version>2.0.2</kerby.version>
     <jcache.version>1.0-alpha-1</jcache.version>
     <ehcache.version>3.3.1</ehcache.version>
     <hikari.version>4.0.3</hikari.version>
     <mssql.version>6.2.1.jre7</mssql.version>
-    <okhttp.version>2.7.5</okhttp.version>
-    <jdom.version>1.1</jdom.version>
+    <okhttp3.version>4.10.0</okhttp3.version>
+    <okio.version>3.2.0</okio.version>
+    <kotlin-stdlib.verion>1.6.20</kotlin-stdlib.verion>
+    <kotlin-stdlib-common.version>1.6.20</kotlin-stdlib-common.version>
+    <jdom2.version>2.0.6.1</jdom2.version>
     <jna.version>5.2.0</jna.version>
     <grizzly.version>2.2.21</grizzly.version>
-    <gson.version>2.2.4</gson.version>
+    <gson.version>2.9.0</gson.version>
     <metrics.version>3.2.4</metrics.version>
     <netty3.version>3.10.6.Final</netty3.version>
-    <netty4.version>4.1.68.Final</netty4.version>
+    <netty4.version>4.1.77.Final</netty4.version>
     <snappy-java.version>1.1.8.2</snappy-java.version>
     <lz4-java.version>1.7.1</lz4-java.version>
 
@@ -148,7 +151,7 @@
 
     <protobuf-compile.version>3.5.1</protobuf-compile.version>
     <grpc.version>1.10.0</grpc.version>
-    <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+    <os-maven-plugin.version>1.7.0</os-maven-plugin.version>
 
     <!-- define the Java language version used by the compiler -->
     <javac.version>1.8</javac.version>
@@ -169,10 +172,9 @@
     <maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
 
     <maven-clean-plugin.version>3.1.0</maven-clean-plugin.version>
-    <maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
     <maven-install-plugin.version>2.5.1</maven-install-plugin.version>
     <maven-resources-plugin.version>2.6</maven-resources-plugin.version>
-    <maven-shade-plugin.version>3.2.1</maven-shade-plugin.version>
+    <maven-shade-plugin.version>3.3.0</maven-shade-plugin.version>
     <maven-jar-plugin.version>2.5</maven-jar-plugin.version>
     <maven-war-plugin.version>3.1.0</maven-war-plugin.version>
     <maven-source-plugin.version>2.3</maven-source-plugin.version>
@@ -182,8 +184,8 @@
     <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
     <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
     <surefire.fork.timeout>900</surefire.fork.timeout>
-    <aws-java-sdk.version>1.11.1026</aws-java-sdk.version>
-    <hsqldb.version>2.3.4</hsqldb.version>
+    <aws-java-sdk.version>1.12.316</aws-java-sdk.version>
+    <hsqldb.version>2.7.1</hsqldb.version>
     <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
     <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
     <phantomjs-maven-plugin.version>0.7</phantomjs-maven-plugin.version>
@@ -195,37 +197,85 @@
     <declared.hadoop.version>${hadoop.version}</declared.hadoop.version>
 
     <swagger-annotations-version>1.5.4</swagger-annotations-version>
-    <snakeyaml.version>1.26</snakeyaml.version>
-    <hbase.one.version>1.4.8</hbase.one.version>
-    <hbase.two.version>2.0.2</hbase.two.version>
+    <snakeyaml.version>1.33</snakeyaml.version>
+    <hbase.one.version>1.7.1</hbase.one.version>
+    <hbase.two.version>2.2.4</hbase.two.version>
     <junit.version>4.13.2</junit.version>
-    <junit.jupiter.version>5.5.1</junit.jupiter.version>
-    <junit.vintage.version>5.5.1</junit.vintage.version>
-    <junit.platform.version>1.5.1</junit.platform.version>
+    <junit.jupiter.version>5.8.2</junit.jupiter.version>
+    <junit.vintage.version>5.8.2</junit.vintage.version>
+    <junit.platform.version>1.8.2</junit.platform.version>
     <assertj.version>3.12.2</assertj.version>
     <jline.version>3.9.0</jline.version>
     <powermock.version>1.5.6</powermock.version>
     <solr.version>8.8.2</solr.version>
     <openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
     <jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
-    <woodstox.version>5.3.0</woodstox.version>
+    <woodstox.version>5.4.0</woodstox.version>
     <json-smart.version>2.4.7</json-smart.version>
     <nimbus-jose-jwt.version>9.8.1</nimbus-jose-jwt.version>
+    <nodejs.version>v12.22.1</nodejs.version>
+    <yarnpkg.version>v1.22.5</yarnpkg.version>
     <apache-ant.version>1.10.11</apache-ant.version>
+    <jmh.version>1.20</jmh.version>
   </properties>
 
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>com.squareup.okhttp</groupId>
+        <groupId>com.squareup.okhttp3</groupId>
         <artifactId>okhttp</artifactId>
-        <version>${okhttp.version}</version>
+        <version>${okhttp3.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.squareup.okio</groupId>
+            <artifactId>okio-jvm</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>com.squareup.okio</groupId>
+        <artifactId>okio-jvm</artifactId>
+        <version>${okio.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <artifactId>kotlin-stdlib</artifactId>
+        <version>${kotlin-stdlib.verion}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.jetbrains</groupId>
+            <artifactId>annotations</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <artifactId>kotlin-stdlib-common</artifactId>
+        <version>${kotlin-stdlib-common.version}</version>
       </dependency>
       <dependency>
         <groupId>com.squareup.okhttp3</groupId>
         <artifactId>mockwebserver</artifactId>
-        <version>3.7.0</version>
+        <version>${okhttp3.version}</version>
         <scope>test</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>com.squareup.okio</groupId>
+            <artifactId>okio-jvm</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib-jdk8</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>jdiff</groupId>
@@ -327,6 +377,11 @@
         <artifactId>hadoop-hdfs-client</artifactId>
         <version>${hadoop.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs-native-client</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs-rbf</artifactId>
@@ -360,6 +415,11 @@
         <artifactId>hadoop-mapreduce-client-common</artifactId>
         <version>${hadoop.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-mapreduce-client-nativetask</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-yarn-api</artifactId>
@@ -378,6 +438,13 @@
         <version>${hadoop.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-mapreduce-client-core</artifactId>
+        <version>${hadoop.version}</version>
+        <type>test-jar</type>
+      </dependency>
+
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
@@ -640,6 +707,7 @@
         <version>${hadoop.version}</version>
       </dependency>
 
+      <!-- This is empty; retained only for downstream app build compatibility. -->
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-openstack</artifactId>
@@ -847,6 +915,12 @@
         <groupId>com.sun.jersey</groupId>
         <artifactId>jersey-core</artifactId>
         <version>${jersey.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.core</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>com.sun.jersey</groupId>
@@ -854,13 +928,21 @@
         <version>${jersey.version}</version>
       </dependency>
       <dependency>
-        <groupId>com.sun.jersey</groupId>
+        <groupId>com.github.pjfanning</groupId>
         <artifactId>jersey-json</artifactId>
-        <version>${jersey.version}</version>
+        <version>1.20</version>
         <exclusions>
           <exclusion>
-            <groupId>stax</groupId>
-            <artifactId>stax-api</artifactId>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.fasterxml.jackson.jaxrs</groupId>
+            <artifactId>jackson-jaxrs-json-provider</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
@@ -931,6 +1013,66 @@
         <version>${netty4.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-codec-socks</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-handler-proxy</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-resolver</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-handler</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-buffer</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-transport</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-common</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-transport-native-unix-common</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-transport-native-epoll</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-codec</artifactId>
+        <version>${netty4.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>commons-io</groupId>
         <artifactId>commons-io</artifactId>
@@ -1007,7 +1149,7 @@
       <dependency>
         <groupId>org.apache.mina</groupId>
         <artifactId>mina-core</artifactId>
-        <version>2.0.16</version>
+        <version>2.0.22</version>
       </dependency>
       <dependency>
         <groupId>org.apache.sshd</groupId>
@@ -1094,7 +1236,7 @@
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-configuration2</artifactId>
-        <version>2.1.1</version>
+        <version>2.8.0</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.commons</groupId>
@@ -1142,26 +1284,6 @@
         <artifactId>woodstox-core</artifactId>
         <version>${woodstox.version}</version>
       </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-mapper-asl</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-core-asl</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-jaxrs</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-xc</artifactId>
-        <version>${jackson.version}</version>
-      </dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-core</artifactId>
@@ -1263,8 +1385,8 @@
       </dependency>
       <dependency>
         <groupId>org.jdom</groupId>
-        <artifactId>jdom</artifactId>
-        <version>${jdom.version}</version>
+        <artifactId>jdom2</artifactId>
+        <version>${jdom2.version}</version>
       </dependency>
       <dependency>
         <groupId>com.googlecode.json-simple</groupId>
@@ -1377,6 +1499,7 @@
         <groupId>org.hsqldb</groupId>
         <artifactId>hsqldb</artifactId>
         <version>${hsqldb.version}</version>
+        <classifier>jdk8</classifier>
       </dependency>
       <dependency>
         <groupId>io.dropwizard.metrics</groupId>
@@ -1406,7 +1529,7 @@
       <dependency>
         <groupId>org.codehaus.jettison</groupId>
         <artifactId>jettison</artifactId>
-        <version>1.1</version>
+        <version>1.5.1</version>
         <exclusions>
           <exclusion>
             <groupId>stax</groupId>
@@ -1441,6 +1564,11 @@
         <artifactId>leveldbjni-all</artifactId>
         <version>1.8</version>
       </dependency>
+      <dependency>
+        <groupId>org.fusesource.hawtjni</groupId>
+        <artifactId>hawtjni-runtime</artifactId>
+        <version>1.11</version>
+      </dependency>
 
       <dependency>
         <groupId>com.microsoft.azure</groupId>
@@ -1469,7 +1597,7 @@
       <dependency>
         <groupId>com.aliyun.oss</groupId>
         <artifactId>aliyun-sdk-oss</artifactId>
-        <version>3.13.0</version>
+        <version>3.13.2</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.httpcomponents</groupId>
@@ -1545,6 +1673,16 @@
          </exclusion>
        </exclusions>
      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-core</artifactId>
+        <version>${jmh.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.openjdk.jmh</groupId>
+        <artifactId>jmh-generator-annprocess</artifactId>
+        <version>${jmh.version}</version>
+      </dependency>
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-test</artifactId>
@@ -1675,6 +1813,12 @@
           <groupId>org.apache.kerby</groupId>
           <artifactId>kerb-simplekdc</artifactId>
           <version>${kerby.version}</version>
+          <exclusions>
+            <exclusion>
+                <groupId>org.jboss.xnio</groupId>
+                <artifactId>xnio-api</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
         <dependency>
           <groupId>org.apache.kerby</groupId>
@@ -1756,12 +1900,43 @@
         <groupId>org.xerial.snappy</groupId>
         <artifactId>snappy-java</artifactId>
         <version>${snappy-java.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.core</artifactId>
+          </exclusion>
+        </exclusions>
       </dependency>
       <dependency>
         <groupId>org.lz4</groupId>
         <artifactId>lz4-java</artifactId>
         <version>${lz4-java.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-1.2-api</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-api</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-core</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-slf4j-impl</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-web</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -1809,7 +1984,6 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>${maven-compiler-plugin.version}</version>
           <configuration>
             <source>${javac.version}</source>
             <target>${javac.version}</target>
@@ -2049,27 +2223,6 @@
                 </replacements>
               </configuration>
             </execution>
-            <execution>
-              <id>replace-guava</id>
-              <phase>process-sources</phase>
-              <goals>
-                <goal>replace</goal>
-              </goals>
-              <configuration>
-                <skip>false</skip><!--This will run for all modules-->
-                <basedir>${basedir}</basedir>
-                <includes>
-                  <include>src/main/java/**/*.java</include>
-                  <include>src/test/java/**/*.java</include>
-                </includes>
-                <replacements>
-                  <replacement>
-                    <token>([^\.])com.google.common</token>
-                    <value>$1${hadoop-thirdparty-shaded-guava-prefix}</value>
-                  </replacement>
-                </replacements>
-              </configuration>
-            </execution>
           </executions>
         </plugin>
       </plugins>
@@ -2118,7 +2271,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <reuseForks>false</reuseForks>
           <forkedProcessTimeoutInSeconds>${surefire.fork.timeout}</forkedProcessTimeoutInSeconds>
           <argLine>${maven-surefire-plugin.argLine}</argLine>
@@ -2190,16 +2342,16 @@
                     <!-- for JDK 8 support -->
                     <include>cglib:cglib:3.2.0</include>
                     <include>com.google.inject:guice:4.0</include>
-                    <include>com.sun.jersey:jersey-core:1.19</include>
-                    <include>com.sun.jersey:jersey-servlet:1.19</include>
-                    <include>com.sun.jersey:jersey-json:1.19</include>
-                    <include>com.sun.jersey:jersey-server:1.19</include>
-                    <include>com.sun.jersey:jersey-client:1.19</include>
-                    <include>com.sun.jersey:jersey-grizzly2:1.19</include>
-                    <include>com.sun.jersey:jersey-grizzly2-servlet:1.19</include>
-                    <include>com.sun.jersey.jersey-test-framework:jersey-test-framework-core:1.19</include>
-                    <include>com.sun.jersey.jersey-test-framework:jersey-test-framework-grizzly2:1.19</include>
-                    <include>com.sun.jersey.contribs:jersey-guice:1.19</include>
+                    <include>com.sun.jersey:jersey-core:1.19.4</include>
+                    <include>com.sun.jersey:jersey-servlet:1.19.4</include>
+                    <include>com.github.pjfanning:jersey-json:1.20</include>
+                    <include>com.sun.jersey:jersey-server:1.19.4</include>
+                    <include>com.sun.jersey:jersey-client:1.19.4</include>
+                    <include>com.sun.jersey:jersey-grizzly2:1.19.4</include>
+                    <include>com.sun.jersey:jersey-grizzly2-servlet:1.19.4</include>
+                    <include>com.sun.jersey.jersey-test-framework:jersey-test-framework-core:1.19.4</include>
+                    <include>com.sun.jersey.jersey-test-framework:jersey-test-framework-grizzly2:1.19.4</include>
+                    <include>com.sun.jersey.contribs:jersey-guice:1.19.4</include>
                     <include>org.ow2.asm:asm:5.0.0</include>
                   </includes>
                 </bannedDependencies>
@@ -2393,8 +2545,9 @@
       </activation>
       <properties>
         <hbase.version>${hbase.one.version}</hbase.version>
-        <hbase-compatible-hadoop.version>2.5.1</hbase-compatible-hadoop.version>
+        <hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
         <hbase-compatible-guava.version>12.0.1</hbase-compatible-guava.version>
+        <hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
         <hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-1</hbase-server-artifactid>
       </properties>
       <dependencyManagement>
@@ -2420,9 +2573,11 @@
       </activation>
       <properties>
         <hbase.version>${hbase.two.version}</hbase.version>
-        <hbase-compatible-hadoop.version>3.0.0</hbase-compatible-hadoop.version>
+        <hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
         <hbase-compatible-guava.version>11.0.2</hbase-compatible-guava.version>
         <hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-2</hbase-server-artifactid>
+        <hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
+        <hbase-compatible-jetty.version>9.3.27.v20190418</hbase-compatible-jetty.version>
       </properties>
       <dependencyManagement>
         <dependencies>
@@ -2459,5 +2614,6 @@
   </profiles>
 
   <repositories>
+
   </repositories>
 </project>
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index 438145a361997..edc38a5286111 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -203,16 +203,6 @@ in both the task configuration and as a Java option.
 Existing configs that already specify both are not affected by this change.
 See the full release notes of MAPREDUCE-5785 for more details.
 
-S3Guard: Consistency and Metadata Caching for the S3A filesystem client
----------------------
-
-[HADOOP-13345](https://issues.apache.org/jira/browse/HADOOP-13345) adds an
-optional feature to the S3A client of Amazon S3 storage: the ability to use
-a DynamoDB table as a fast and consistent store of file and directory
-metadata.
-
-See [S3Guard](./hadoop-aws/tools/hadoop-aws/s3guard.html) for more details.
-
 HDFS Router-Based Federation
 ---------------------
 HDFS Router-Based Federation adds a RPC routing layer that provides a federated
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index a150385048f2f..8e85f379ef7da 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -69,6 +69,7 @@
       <item name="Tracing" href="hadoop-project-dist/hadoop-common/Tracing.html"/>
       <item name="Unix Shell Guide" href="hadoop-project-dist/hadoop-common/UnixShellGuide.html"/>
       <item name="Registry" href="hadoop-project-dist/hadoop-common/registry/index.html"/>
+      <item name="Async Profiler" href="hadoop-project-dist/hadoop-common/AsyncProfilerServlet.html" />
     </menu>
     
     <menu name="HDFS" inherit="top">
@@ -177,7 +178,6 @@
       <item name="Azure Blob Storage" href="hadoop-azure/index.html"/>
       <item name="Azure Data Lake Storage"
             href="hadoop-azure-datalake/index.html"/>
-      <item name="OpenStack Swift" href="hadoop-openstack/index.html"/>
       <item name="Tencent COS" href="hadoop-cos/cloud-storage/index.html"/>
       <item name="Huaweicloud OBS" href="hadoop-huaweicloud/cloud-storage/index.html"/>
     </menu>
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 8a68d9b4f7db6..570032f370b57 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -107,10 +107,22 @@
       <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 759484e4239fa..5f40488bfd684 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -49,7 +49,6 @@
 import org.apache.hadoop.util.Progressable;
 
 import com.aliyun.oss.model.OSSObjectSummary;
-import com.aliyun.oss.model.ObjectListing;
 import com.aliyun.oss.model.ObjectMetadata;
 
 import org.apache.hadoop.util.SemaphoredDelegatingExecutor;
@@ -271,14 +270,15 @@ public FileStatus getFileStatus(Path path) throws IOException {
       meta = store.getObjectMetadata(key);
     }
     if (meta == null) {
-      ObjectListing listing = store.listObjects(key, 1, null, false);
+      OSSListRequest listRequest = store.createListObjectsRequest(key,
+          maxKeys, null, null, false);
+      OSSListResult listing = store.listObjects(listRequest);
       do {
         if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) ||
             CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) {
           return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username);
         } else if (listing.isTruncated()) {
-          listing = store.listObjects(key, 1000, listing.getNextMarker(),
-              false);
+          listing = store.continueListObjects(listRequest, listing);
         } else {
           throw new FileNotFoundException(
               path + ": No such file or directory!");
@@ -416,7 +416,9 @@ public FileStatus[] listStatus(Path path) throws IOException {
         LOG.debug("listStatus: doing listObjects for directory " + key);
       }
 
-      ObjectListing objects = store.listObjects(key, maxKeys, null, false);
+      OSSListRequest listRequest = store.createListObjectsRequest(key,
+          maxKeys, null, null, false);
+      OSSListResult objects = store.listObjects(listRequest);
       while (true) {
         for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
           String objKey = objectSummary.getKey();
@@ -456,8 +458,7 @@ public FileStatus[] listStatus(Path path) throws IOException {
           if (LOG.isDebugEnabled()) {
             LOG.debug("listStatus: list truncated - getting next batch");
           }
-          String nextMarker = objects.getNextMarker();
-          objects = store.listObjects(key, maxKeys, nextMarker, false);
+          objects = store.continueListObjects(listRequest, objects);
         } else {
           break;
         }
@@ -520,7 +521,7 @@ private RemoteIterator<LocatedFileStatus> innerList(final Path f,
           locations);
     } else {
       return store.createLocatedFileStatusIterator(key, maxKeys, this, filter,
-          acceptor, recursive ? null : "/");
+          acceptor, recursive);
     }
   }
 
@@ -707,7 +708,9 @@ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
     ExecutorService executorService = MoreExecutors.listeningDecorator(
         new SemaphoredDelegatingExecutor(boundedCopyThreadPool,
             maxConcurrentCopyTasksPerDir, true));
-    ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true);
+    OSSListRequest listRequest = store.createListObjectsRequest(srcKey,
+        maxKeys, null, null, true);
+    OSSListResult objects = store.listObjects(listRequest);
     // Copy files from src folder to dst
     int copiesToFinish = 0;
     while (true) {
@@ -729,8 +732,7 @@ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
         }
       }
       if (objects.isTruncated()) {
-        String nextMarker = objects.getNextMarker();
-        objects = store.listObjects(srcKey, maxKeys, nextMarker, true);
+        objects = store.continueListObjects(listRequest, objects);
       } else {
         break;
       }
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 3df9a730161d3..e9ac1ddea9ee0 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -36,8 +36,8 @@
 import com.aliyun.oss.model.InitiateMultipartUploadRequest;
 import com.aliyun.oss.model.InitiateMultipartUploadResult;
 import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.ListObjectsV2Request;
 import com.aliyun.oss.model.ObjectMetadata;
-import com.aliyun.oss.model.ObjectListing;
 import com.aliyun.oss.model.OSSObjectSummary;
 import com.aliyun.oss.model.PartETag;
 import com.aliyun.oss.model.PutObjectResult;
@@ -90,6 +90,7 @@ public class AliyunOSSFileSystemStore {
   private long uploadPartSize;
   private int maxKeys;
   private String serverSideEncryptionAlgorithm;
+  private boolean useListV1;
 
   public void initialize(URI uri, Configuration conf, String user,
                          FileSystem.Statistics stat) throws IOException {
@@ -170,6 +171,12 @@ public void initialize(URI uri, Configuration conf, String user,
     }
 
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+    int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
+    if (listVersion < 1 || listVersion > 2) {
+      LOG.warn("Configured fs.oss.list.version {} is invalid, forcing " +
+          "version 2", listVersion);
+    }
+    useListV1 = (listVersion == 1);
   }
 
   /**
@@ -231,14 +238,10 @@ public void deleteObjects(List<String> keysToDelete) throws IOException {
    * @throws IOException if failed to delete directory.
    */
   public void deleteDirs(String key) throws IOException {
-    key = AliyunOSSUtils.maybeAddTrailingSlash(key);
-    ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
-    listRequest.setPrefix(key);
-    listRequest.setDelimiter(null);
-    listRequest.setMaxKeys(maxKeys);
-
+    OSSListRequest listRequest = createListObjectsRequest(key,
+        maxKeys, null, null, true);
     while (true) {
-      ObjectListing objects = ossClient.listObjects(listRequest);
+      OSSListResult objects = listObjects(listRequest);
       statistics.incrementReadOps(1);
       List<String> keysToDelete = new ArrayList<String>();
       for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
@@ -246,7 +249,12 @@ public void deleteDirs(String key) throws IOException {
       }
       deleteObjects(keysToDelete);
       if (objects.isTruncated()) {
-        listRequest.setMarker(objects.getNextMarker());
+        if (objects.isV1()) {
+          listRequest.getV1().setMarker(objects.getV1().getNextMarker());
+        } else {
+          listRequest.getV2().setContinuationToken(
+              objects.getV2().getNextContinuationToken());
+        }
       } else {
         break;
       }
@@ -418,25 +426,76 @@ public void uploadObject(String key, File file) throws IOException {
   /**
    * list objects.
    *
+   * @param listRequest list request.
+   * @return a list of matches.
+   */
+  public OSSListResult listObjects(OSSListRequest listRequest) {
+    OSSListResult listResult;
+    if (listRequest.isV1()) {
+      listResult = OSSListResult.v1(
+          ossClient.listObjects(listRequest.getV1()));
+    } else {
+      listResult = OSSListResult.v2(
+          ossClient.listObjectsV2(listRequest.getV2()));
+    }
+    statistics.incrementReadOps(1);
+    return listResult;
+  }
+
+  /**
+   * continue to list objects depends on previous list result.
+   *
+   * @param listRequest list request.
+   * @param preListResult previous list result.
+   * @return a list of matches.
+   */
+  public OSSListResult continueListObjects(OSSListRequest listRequest,
+      OSSListResult preListResult) {
+    OSSListResult listResult;
+    if (listRequest.isV1()) {
+      listRequest.getV1().setMarker(preListResult.getV1().getNextMarker());
+      listResult = OSSListResult.v1(
+          ossClient.listObjects(listRequest.getV1()));
+    } else {
+      listRequest.getV2().setContinuationToken(
+          preListResult.getV2().getNextContinuationToken());
+      listResult = OSSListResult.v2(
+          ossClient.listObjectsV2(listRequest.getV2()));
+    }
+    statistics.incrementReadOps(1);
+    return listResult;
+  }
+
+  /**
+   * create list objects request.
+   *
    * @param prefix prefix.
    * @param maxListingLength max no. of entries
    * @param marker last key in any previous search.
+   * @param continuationToken list from a specific point.
    * @param recursive whether to list directory recursively.
    * @return a list of matches.
    */
-  public ObjectListing listObjects(String prefix, int maxListingLength,
-      String marker, boolean recursive) {
+  protected OSSListRequest createListObjectsRequest(String prefix,
+      int maxListingLength, String marker,
+      String continuationToken, boolean recursive) {
     String delimiter = recursive ? null : "/";
     prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix);
-    ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
-    listRequest.setPrefix(prefix);
-    listRequest.setDelimiter(delimiter);
-    listRequest.setMaxKeys(maxListingLength);
-    listRequest.setMarker(marker);
-
-    ObjectListing listing = ossClient.listObjects(listRequest);
-    statistics.incrementReadOps(1);
-    return listing;
+    if (useListV1) {
+      ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+      listRequest.setPrefix(prefix);
+      listRequest.setDelimiter(delimiter);
+      listRequest.setMaxKeys(maxListingLength);
+      listRequest.setMarker(marker);
+      return OSSListRequest.v1(listRequest);
+    } else {
+      ListObjectsV2Request listV2Request = new ListObjectsV2Request(bucketName);
+      listV2Request.setPrefix(prefix);
+      listV2Request.setDelimiter(delimiter);
+      listV2Request.setMaxKeys(maxListingLength);
+      listV2Request.setContinuationToken(continuationToken);
+      return OSSListRequest.v2(listV2Request);
+    }
   }
 
   /**
@@ -478,21 +537,7 @@ public void close() {
    * @throws IOException if failed to clean up objects.
    */
   public void purge(String prefix) throws IOException {
-    String key;
-    try {
-      ObjectListing objects = listObjects(prefix, maxKeys, null, true);
-      for (OSSObjectSummary object : objects.getObjectSummaries()) {
-        key = object.getKey();
-        ossClient.deleteObject(bucketName, key);
-        statistics.incrementWriteOps(1);
-      }
-
-      for (String dir: objects.getCommonPrefixes()) {
-        deleteDirs(dir);
-      }
-    } catch (OSSException | ClientException e) {
-      LOG.error("Failed to purge " + prefix);
-    }
+    deleteDirs(prefix);
   }
 
   public RemoteIterator<LocatedFileStatus> singleStatusRemoteIterator(
@@ -520,12 +565,12 @@ public LocatedFileStatus next() throws IOException {
 
   public RemoteIterator<LocatedFileStatus> createLocatedFileStatusIterator(
       final String prefix, final int maxListingLength, FileSystem fs,
-      PathFilter filter, FileStatusAcceptor acceptor, String delimiter) {
+      PathFilter filter, FileStatusAcceptor acceptor, boolean recursive) {
     return new RemoteIterator<LocatedFileStatus>() {
-      private String nextMarker = null;
       private boolean firstListing = true;
       private boolean meetEnd = false;
       private ListIterator<FileStatus> batchIterator;
+      private OSSListRequest listRequest = null;
 
       @Override
       public boolean hasNext() throws IOException {
@@ -550,15 +595,24 @@ public LocatedFileStatus next() throws IOException {
       }
 
       private boolean requestNextBatch() {
+        while (!meetEnd) {
+          if (continueListStatus()) {
+            return true;
+          }
+        }
+
+        return false;
+      }
+
+      private boolean continueListStatus() {
         if (meetEnd) {
           return false;
         }
-        ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
-        listRequest.setPrefix(AliyunOSSUtils.maybeAddTrailingSlash(prefix));
-        listRequest.setMaxKeys(maxListingLength);
-        listRequest.setMarker(nextMarker);
-        listRequest.setDelimiter(delimiter);
-        ObjectListing listing = ossClient.listObjects(listRequest);
+        if (listRequest == null) {
+          listRequest = createListObjectsRequest(prefix,
+              maxListingLength, null, null, recursive);
+        }
+        OSSListResult listing = listObjects(listRequest);
         List<FileStatus> stats = new ArrayList<>(
             listing.getObjectSummaries().size() +
             listing.getCommonPrefixes().size());
@@ -584,7 +638,12 @@ private boolean requestNextBatch() {
 
         batchIterator = stats.listIterator();
         if (listing.isTruncated()) {
-          nextMarker = listing.getNextMarker();
+          if (listing.isV1()) {
+            listRequest.getV1().setMarker(listing.getV1().getNextMarker());
+          } else {
+            listRequest.getV2().setContinuationToken(
+                listing.getV2().getNextContinuationToken());
+          }
         } else {
           meetEnd = true;
         }
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index a6d59c2d0c4ec..1ef7fe72ae246 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -23,7 +23,8 @@
 import java.net.URI;
 
 import com.aliyun.oss.common.auth.CredentialsProvider;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalDirAllocator;
@@ -39,7 +40,7 @@
 final public class AliyunOSSUtils {
   private static final Logger LOG =
       LoggerFactory.getLogger(AliyunOSSUtils.class);
-  private static LocalDirAllocator directoryAllocator;
+  private static volatile LocalDirAllocator directoryAllocator;
 
   private AliyunOSSUtils() {
   }
@@ -171,21 +172,33 @@ public static boolean objectRepresentsDirectory(final String name,
 
   /**
    * Demand create the directory allocator, then create a temporary file.
-   *  @param path prefix for the temporary file
-   *  @param size the size of the file that is going to be written
-   *  @param conf the Configuration object
-   *  @return a unique temporary file
-   *  @throws IOException IO problems
+   * This does not mark the file for deletion when a process exits.
+   * {@link LocalDirAllocator#createTmpFileForWrite(
+   * String, long, Configuration)}.
+   * @param pathStr prefix for the temporary file
+   * @param size the size of the file that is going to be written
+   * @param conf the Configuration object
+   * @return a unique temporary file
+   * @throws IOException IO problems
    */
-  public static File createTmpFileForWrite(String path, long size,
+  public static File createTmpFileForWrite(String pathStr, long size,
       Configuration conf) throws IOException {
     if (conf.get(BUFFER_DIR_KEY) == null) {
       conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
     }
     if (directoryAllocator == null) {
-      directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY);
+      synchronized (AliyunOSSUtils.class) {
+        if (directoryAllocator == null) {
+          directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY);
+        }
+      }
     }
-    return directoryAllocator.createTmpFileForWrite(path, size, conf);
+    Path path = directoryAllocator.getLocalPathForWrite(pathStr,
+        size, conf);
+    File dir = new File(path.getParent().toUri().getPath());
+    String prefix = path.getName();
+    // create a temp file on this directory
+    return File.createTempFile(prefix, null, dir);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 71693d057030f..3421b421813ba 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -154,4 +154,8 @@ private Constants() {
   public static final String UPLOAD_ACTIVE_BLOCKS_KEY =
       "fs.oss.upload.active.blocks";
   public static final int UPLOAD_ACTIVE_BLOCKS_DEFAULT = 4;
+
+  public static final String LIST_VERSION = "fs.oss.list.version";
+
+  public static final int DEFAULT_LIST_VERSION = 2;
 }
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java
new file mode 100644
index 0000000000000..0e42b7d74fdc3
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.ListObjectsV2Request;
+
+/**
+ * API version-independent container for OSS List requests.
+ */
+public class OSSListRequest {
+  /**
+   * Format for the toString() method: {@value}.
+   */
+  private static final String DESCRIPTION
+      = "List %s:/%s delimiter=%s keys=%d";
+
+  private final ListObjectsRequest v1Request;
+  private final ListObjectsV2Request v2Request;
+
+  private OSSListRequest(ListObjectsRequest v1, ListObjectsV2Request v2) {
+    v1Request = v1;
+    v2Request = v2;
+  }
+
+  /**
+   * Restricted constructors to ensure v1 or v2, not both.
+   * @param request v1 request
+   * @return new list request container
+   */
+  public static OSSListRequest v1(ListObjectsRequest request) {
+    return new OSSListRequest(request, null);
+  }
+
+  /**
+   * Restricted constructors to ensure v1 or v2, not both.
+   * @param request v2 request
+   * @return new list request container
+   */
+  public static OSSListRequest v2(ListObjectsV2Request request) {
+    return new OSSListRequest(null, request);
+  }
+
+  /**
+   * Is this a v1 API request or v2?
+   * @return true if v1, false if v2
+   */
+  public boolean isV1() {
+    return v1Request != null;
+  }
+
+  public ListObjectsRequest getV1() {
+    return v1Request;
+  }
+
+  public ListObjectsV2Request getV2() {
+    return v2Request;
+  }
+
+  @Override
+  public String toString() {
+    if (isV1()) {
+      return String.format(DESCRIPTION,
+          v1Request.getBucketName(), v1Request.getPrefix(),
+          v1Request.getDelimiter(), v1Request.getMaxKeys());
+    } else {
+      return String.format(DESCRIPTION,
+          v2Request.getBucketName(), v2Request.getPrefix(),
+          v2Request.getDelimiter(), v2Request.getMaxKeys());
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java
new file mode 100644
index 0000000000000..145bbb036da33
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.model.ListObjectsV2Result;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * API version-independent container for OSS List responses.
+ */
+public final class OSSListResult {
+  private ObjectListing v1Result;
+  private ListObjectsV2Result v2Result;
+
+  protected OSSListResult(ObjectListing v1, ListObjectsV2Result v2) {
+    v1Result = v1;
+    v2Result = v2;
+  }
+
+  /**
+   * Restricted constructors to ensure v1 or v2, not both.
+   * @param result v1 result
+   * @return new list result container
+   */
+  public static OSSListResult v1(ObjectListing result) {
+    return new OSSListResult(result, null);
+  }
+
+  /**
+   * Restricted constructors to ensure v1 or v2, not both.
+   * @param result v2 result
+   * @return new list result container
+   */
+  public static OSSListResult v2(ListObjectsV2Result result) {
+    return new OSSListResult(null, result);
+  }
+
+  /**
+   * Is this a v1 API result or v2?
+   * @return true if v1, false if v2
+   */
+  public boolean isV1() {
+    return v1Result != null;
+  }
+
+  public ObjectListing getV1() {
+    return v1Result;
+  }
+
+  public ListObjectsV2Result getV2() {
+    return v2Result;
+  }
+
+  public List<OSSObjectSummary> getObjectSummaries() {
+    if (isV1()) {
+      return v1Result.getObjectSummaries();
+    } else {
+      return v2Result.getObjectSummaries();
+    }
+  }
+
+  public boolean isTruncated() {
+    if (isV1()) {
+      return v1Result.isTruncated();
+    } else {
+      return v2Result.isTruncated();
+    }
+  }
+
+  public List<String> getCommonPrefixes() {
+    if (isV1()) {
+      return v1Result.getCommonPrefixes();
+    } else {
+      return v2Result.getCommonPrefixes();
+    }
+  }
+
+  /**
+   * Dump the result at debug level.
+   * @param log log to use
+   */
+  public void logAtDebug(Logger log) {
+    Collection<String> prefixes = getCommonPrefixes();
+    Collection<OSSObjectSummary> summaries = getObjectSummaries();
+    log.debug("Prefix count = {}; object count={}",
+        prefixes.size(), summaries.size());
+    for (OSSObjectSummary summary : summaries) {
+      log.debug("Summary: {} {}", summary.getKey(), summary.getSize());
+    }
+    for (String prefix : prefixes) {
+      log.debug("Prefix: {}", prefix);
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 9978bc1fb6931..d48bf1c6b02cb 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -243,6 +243,14 @@ please raise your issues with them.
       <description>Size in bytes in each request from ALiyun OSS.</description>
     </property>
 
+    <property>
+      <name>fs.oss.list.version</name>
+      <value>2</value>
+      <description>Select which version of the OSS SDK's List Objects API to use.
+        Currently support 2(default) and 1(older API).
+      </description>
+    </property>
+
     <property>
       <name>fs.oss.buffer.dir</name>
       <description>Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS</description>
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
index cdf4971b603cf..5e53ef5307a81 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -83,4 +83,14 @@ public static String generateUniqueTestPath() {
     return testUniqueForkId == null ? "/test" :
         "/" + testUniqueForkId + "/test";
   }
+
+  /**
+   * Turn off FS Caching: use if a filesystem with different options from
+   * the default is required.
+   * @param conf configuration to patch
+   */
+  public static void disableFilesystemCaching(Configuration conf) {
+    conf.setBoolean(TestAliyunOSSFileSystemContract.FS_OSS_IMPL_DISABLE_CACHE,
+        true);
+  }
 }
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
index 835e5ffaef5d5..69aa0a5a7956b 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
@@ -23,19 +23,27 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
+import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.BUFFER_DIR_KEY;
 import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_KEY;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.IO_CHUNK_BUFFER_SIZE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Tests regular and multi-part upload functionality for
@@ -43,6 +51,7 @@
  */
 public class TestAliyunOSSBlockOutputStream {
   private FileSystem fs;
+  private static final int PART_SIZE = 1024 * 1024;
   private static String testRootPath =
       AliyunOSSTestUtils.generateUniqueTestPath();
 
@@ -52,7 +61,7 @@ public class TestAliyunOSSBlockOutputStream {
   @Before
   public void setUp() throws Exception {
     Configuration conf = new Configuration();
-    conf.setInt(MULTIPART_UPLOAD_PART_SIZE_KEY, 1024 * 1024);
+    conf.setInt(MULTIPART_UPLOAD_PART_SIZE_KEY, PART_SIZE);
     conf.setInt(IO_CHUNK_BUFFER_SIZE,
         conf.getInt(MULTIPART_UPLOAD_PART_SIZE_KEY, 0));
     conf.setInt(Constants.UPLOAD_ACTIVE_BLOCKS_KEY, 20);
@@ -155,10 +164,8 @@ public void testMultiPartUploadConcurrent() throws IOException {
 
   @Test
   public void testHugeUpload() throws IOException {
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
-        MULTIPART_UPLOAD_PART_SIZE_DEFAULT - 1);
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
-        MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), PART_SIZE - 1);
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), PART_SIZE);
     ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
         MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
     bufferDirShouldEmpty();
@@ -202,4 +209,44 @@ private void bufferDirShouldEmpty() throws IOException {
     // Temporary file should be deleted
     assertEquals(0, files.length);
   }
+
+  @Test
+  public void testDirectoryAllocator() throws Throwable {
+    Configuration conf = fs.getConf();
+    File tmp = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf);
+    assertTrue("not found: " + tmp, tmp.exists());
+    tmp.delete();
+
+    // tmp should not in DeleteOnExitHook
+    try {
+      Class<?> c = Class.forName("java.io.DeleteOnExitHook");
+      Field field = c.getDeclaredField("files");
+      field.setAccessible(true);
+      String name = field.getName();
+      LinkedHashSet<String> files = (LinkedHashSet<String>)field.get(name);
+      assertTrue("in DeleteOnExitHook", files.isEmpty());
+      assertFalse("in DeleteOnExitHook",
+          (new ArrayList<>(files)).contains(tmp.getPath()));
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
+  @Test
+  public void testDirectoryAllocatorRR() throws Throwable {
+    File dir1 = GenericTestUtils.getRandomizedTestDir();
+    File dir2 = GenericTestUtils.getRandomizedTestDir();
+    dir1.mkdirs();
+    dir2.mkdirs();
+
+    Configuration conf = new Configuration();
+    conf.set(BUFFER_DIR_KEY, dir1 + ", " + dir2);
+    fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+    File tmp1 = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf);
+    tmp1.delete();
+    File tmp2 = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf);
+    tmp2.delete();
+    assertNotEquals("round robin not working",
+        tmp1.getParent(), tmp2.getParent());
+  }
 }
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
index e6467f49f49f8..4ae09623b20a3 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -47,6 +47,8 @@
 public class TestAliyunOSSFileSystemContract
     extends FileSystemContractBaseTest {
   public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
+  public static final String FS_OSS_IMPL_DISABLE_CACHE
+      = "fs.oss.impl.disable.cache";
   private static Path testRootPath =
       new Path(AliyunOSSTestUtils.generateUniqueTestPath());
 
@@ -413,7 +415,7 @@ private void testRenameDir(boolean changing, boolean result, boolean empty)
     Thread thread = new Thread(task);
     thread.start();
     while (!task.isRunning()) {
-      Thread.sleep(1000);
+      Thread.sleep(1);
     }
 
     if (changing) {
@@ -421,7 +423,11 @@ private void testRenameDir(boolean changing, boolean result, boolean empty)
     }
 
     thread.join();
-    assertEquals(result, task.isSucceed());
+    if (changing) {
+      assertTrue(task.isSucceed() || fs.exists(this.path("a")));
+    } else {
+      assertEquals(result, task.isSucceed());
+    }
   }
 
   class TestRenameTask implements Runnable {
@@ -451,6 +457,8 @@ public void run() {
         running = true;
         result = fs.rename(srcPath, dstPath);
       } catch (Exception e) {
+        e.printStackTrace();
+        this.result = false;
       }
     }
   }
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
index ebd555d643f4f..f85871dd86e3d 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.aliyun.oss;
 
+import com.aliyun.oss.model.ObjectMetadata;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.junit.After;
@@ -89,13 +90,17 @@ protected void writeRenameReadCompare(Path path, long len)
 
     assertTrue("Exists", fs.exists(path));
 
+    ObjectMetadata srcMeta = fs.getStore().getObjectMetadata(
+        path.toUri().getPath().substring(1));
+
     Path copyPath = path.suffix(".copy");
-    long start = System.currentTimeMillis();
     fs.rename(path, copyPath);
 
     assertTrue("Copy exists", fs.exists(copyPath));
-    // should less than 1 second
-    assertTrue(System.currentTimeMillis() - start < 1000);
+    // file type should not change
+    ObjectMetadata dstMeta = fs.getStore().getObjectMetadata(
+        copyPath.toUri().getPath().substring(1));
+    assertEquals(srcMeta.getObjectType(), dstMeta.getObjectType());
     // Download file from Aliyun OSS and compare the digest against the original
     MessageDigest digest2 = MessageDigest.getInstance("MD5");
     InputStream in = new BufferedInputStream(
@@ -120,6 +125,7 @@ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
   public void testLargeUpload()
       throws IOException, NoSuchAlgorithmException {
     // Multipart upload, shallow copy
-    writeRenameReadCompare(new Path("/test/xlarge"), 2147483648L); // 2GB
+    writeRenameReadCompare(new Path("/test/xlarge"),
+        Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
   }
 }
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java
new file mode 100644
index 0000000000000..2f1213c65ded2
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.apache.hadoop.fs.aliyun.oss.Constants;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test getFileStatus and related listing operations,
+ * using the v1 List Objects API.
+ */
+public class TestAliyunOSSContractGetFileStatusV1List
+    extends AbstractContractGetFileStatusTest {
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new AliyunOSSContract(conf);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    getLogger().info("FS details {}", getFileSystem());
+    super.teardown();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    AliyunOSSTestUtils.disableFilesystemCaching(conf);
+    conf.setInt(Constants.MAX_PAGING_KEYS_KEY, 2);
+    // Use v1 List Objects API
+    conf.setInt(Constants.LIST_VERSION, 1);
+    return conf;
+  }
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
index 9ec4be6c8afaa..ac7c38c48c23f 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
@@ -95,7 +95,7 @@
 
     <property>
         <name>fs.contract.rename-overwrites-dest</name>
-        <value>true</value>
+        <value>false</value>
     </property>
 
     <property>
diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml
index aa98c1bf3547e..f6154484ad270 100644
--- a/hadoop-tools/hadoop-archive-logs/pom.xml
+++ b/hadoop-tools/hadoop-archive-logs/pom.xml
@@ -59,6 +59,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -204,38 +210,6 @@
           <effort>Max</effort>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index 966fff7ae7c5b..b16b88d11dada 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -79,6 +79,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
index 861eb83584e5e..39a9e51ac8125 100644
--- a/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-aws/dev-support/findbugs-exclude.xml
@@ -28,19 +28,10 @@
     <Method name="s3Exists" />
     <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE" />
   </Match>
-
-  <!--
-    This extends the serializable S3Object, so findbug checks
-    serializability. It is never serialized however, so its
-    warnings are false positives.
-  -->
-  <Match>
-    <Class name="org.apache.hadoop.fs.s3a.InconsistentS3Object" />
-    <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" />
-  </Match>
+  <!-- we are using completable futures, so ignore the Future which submit() returns -->
   <Match>
-    <Class name="org.apache.hadoop.fs.s3a.InconsistentS3Object" />
-    <Bug pattern="SE_NO_SERIALVERSIONID" />
+    <Class name="org.apache.hadoop.fs.s3a.S3AFileSystem$InputStreamCallbacksImpl" />
+    <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
   </Match>
 
   <!--
@@ -68,6 +59,11 @@
     <Method name="openFileWithOptions"/>
     <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.fs.s3a.S3AFileSystem"/>
+    <Field name="futurePool"/>
+    <Bug pattern="IS2_INCONSISTENT_SYNC"/>
+  </Match>
   <Match>
     <Class name="org.apache.hadoop.fs.s3a.s3guard.S3GuardTool$BucketInfo"/>
     <Method name="run"/>
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index dddbe6fb34641..6ebf1c71f0d5b 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -45,10 +45,7 @@
     <fs.s3a.scale.test.huge.partitionsize>unset</fs.s3a.scale.test.huge.partitionsize>
     <!-- Timeout in seconds for scale tests.-->
     <fs.s3a.scale.test.timeout>3600</fs.s3a.scale.test.timeout>
-    <!-- are scale tests enabled ? -->
-    <fs.s3a.s3guard.test.enabled>false</fs.s3a.s3guard.test.enabled>
-    <fs.s3a.s3guard.test.authoritative>false</fs.s3a.s3guard.test.authoritative>
-    <fs.s3a.s3guard.test.implementation>local</fs.s3a.s3guard.test.implementation>
+
     <!-- Set a longer timeout for integration test (in milliseconds) -->
     <test.integration.timeout>200000</test.integration.timeout>
 
@@ -56,6 +53,9 @@
     <fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit>
     <!--    marker retention policy -->
     <fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention>
+
+    <!-- Is prefetch enabled? -->
+    <fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled>
   </properties>
 
   <profiles>
@@ -130,6 +130,8 @@
                 <!-- Markers-->
                 <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                 <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                <!-- Prefetch -->
+                <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
               </systemPropertyVariables>
             </configuration>
           </plugin>
@@ -166,13 +168,11 @@
                     <fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
                     <fs.s3a.scale.test.huge.huge.partitionsize>${fs.s3a.scale.test.huge.partitionsize}</fs.s3a.scale.test.huge.huge.partitionsize>
                     <fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
-                    <!-- S3Guard -->
-                    <fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
-                    <fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
-                    <fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
 
                     <test.default.timeout>${test.integration.timeout}</test.default.timeout>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <!-- Some tests cannot run in parallel.  Tests that cover -->
                   <!-- access to the root directory must run in isolation -->
@@ -193,14 +193,10 @@
                     <exclude>**/ITestS3AFileContextStatistics.java</exclude>
                     <exclude>**/ITestS3AEncryptionSSEC*.java</exclude>
                     <exclude>**/ITestS3AHuge*.java</exclude>
-                    <!-- this sets out to overlaod DynamoDB, so must be run standalone -->
-                    <exclude>**/ITestDynamoDBMetadataStoreScale.java</exclude>
                     <!-- Terasort MR jobs spawn enough processes that they use up all RAM -->
                     <exclude>**/ITestTerasort*.java</exclude>
                     <!-- Root marker tool tests -->
                     <exclude>**/ITestMarkerToolRootOperations.java</exclude>
-                    <!-- operations across the metastore -->
-                    <exclude>**/ITestS3GuardDDBRootOperations.java</exclude>
                     <!-- leave this until the end for better statistics -->
                     <exclude>**/ITestAggregateIOStatistics.java</exclude>
                   </excludes>
@@ -223,13 +219,11 @@
                     <fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
                     <fs.s3a.scale.test.huge.huge.partitionsize>${fs.s3a.scale.test.huge.partitionsize}</fs.s3a.scale.test.huge.huge.partitionsize>
                     <fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
-                    <!-- S3Guard -->
-                    <fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
-                    <fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
-                    <fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
                     <!-- Markers-->
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                     <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <!-- Do a sequential run for tests that cannot handle -->
                   <!-- parallel execution. -->
@@ -239,8 +233,6 @@
                     <include>**/ITestS3AHuge*.java</include>
                     <!-- SSE encrypted files confuse everything else -->
                     <include>**/ITestS3AEncryptionSSEC*.java</include>
-                    <!-- this sets out to overlaod DynamoDB, so must be run standalone -->
-                    <include>**/ITestDynamoDBMetadataStoreScale.java</include>
                     <!-- the terasort tests both work with a file in the same path in -->
                     <!--  the local FS. Running them sequentially guarantees isolation -->
                     <!-- and that they don't conflict with the other MR jobs for RAM -->
@@ -249,9 +241,8 @@
                     <!-- MUST be run before the other root ops so there's
                      more likelihood of files in the bucket -->
                     <include>**/ITestMarkerToolRootOperations.java</include>
-                    <!-- operations across the metastore -->
+                    <!-- operations on the root dir -->
                     <include>**/ITestS3AContractRootDir.java</include>
-                    <include>**/ITestS3GuardDDBRootOperations.java</include>
                     <!-- leave this until the end for better statistics -->
                     <include>**/ITestAggregateIOStatistics.java</include>
                   </includes>
@@ -286,13 +277,11 @@
                     <fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
                     <fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
                     <fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
-                    <!-- S3Guard -->
-                    <fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
-                    <fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
-                    <fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
                     <!-- Markers-->
                     <fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
                     <fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
+                    <!-- Prefetch -->
+                    <fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
                   </systemPropertyVariables>
                   <forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
                 </configuration>
@@ -316,46 +305,6 @@
       </properties>
     </profile>
 
-    <!-- Turn on S3Guard tests-->
-    <profile>
-      <id>s3guard</id>
-      <activation>
-        <property>
-          <name>s3guard</name>
-        </property>
-      </activation>
-      <properties >
-        <fs.s3a.s3guard.test.enabled>true</fs.s3a.s3guard.test.enabled>
-      </properties>
-    </profile>
-
-    <!-- Switch to DynamoDB for S3Guard. Has no effect unless S3Guard is enabled -->
-    <profile>
-      <id>dynamo</id>
-      <activation>
-        <property>
-          <name>dynamo</name>
-        </property>
-      </activation>
-      <properties >
-        <fs.s3a.s3guard.test.implementation>dynamo</fs.s3a.s3guard.test.implementation>
-      </properties>
-    </profile>
-
-    <!-- Switch S3Guard from Authoritative=false to true
-     Has no effect unless S3Guard is enabled -->
-    <profile>
-      <id>auth</id>
-      <activation>
-        <property>
-          <name>auth</name>
-        </property>
-      </activation>
-      <properties >
-        <fs.s3a.s3guard.test.authoritative>true</fs.s3a.s3guard.test.authoritative>
-      </properties>
-    </profile>
-
     <!-- Directory marker retention options, all from the -Dmarkers value-->
     <profile>
       <id>keep-markers</id>
@@ -394,6 +343,19 @@
       </properties>
     </profile>
 
+    <!-- Turn on prefetching-->
+    <profile>
+      <id>prefetch</id>
+      <activation>
+        <property>
+          <name>prefetch</name>
+        </property>
+      </activation>
+      <properties>
+        <fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled>
+      </properties>
+    </profile>
+
   </profiles>
 
   <build>
@@ -470,13 +432,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
         <executions>
           <execution>
             <id>banned-illegal-imports</id>
@@ -486,12 +441,20 @@
             </goals>
             <configuration>
               <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
+                <restrictImports>
+                  <includeTestCode>false</includeTestCode>
+                  <reason>Restrict mapreduce imports to committer code</reason>
+                  <exclusions>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter</exclusion>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory</exclusion>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory</exclusion>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.impl.*</exclusion>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.magic.*</exclusion>
+                    <exclusion>org.apache.hadoop.fs.s3a.commit.staging.*</exclusion>
+                  </exclusions>
                   <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
+                    <bannedImport>org.apache.hadoop.mapreduce.**</bannedImport>
+                    <bannedImport>org.apache.hadoop.mapred.**</bannedImport>
                   </bannedImports>
                 </restrictImports>
               </rules>
@@ -507,6 +470,16 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.enterprise</groupId>
+          <artifactId>cdi-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -566,6 +539,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
index f31f3c79b355e..d3c5f888c7370 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
@@ -20,7 +20,7 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.SdkBaseException;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
index 5f8b95dcb6402..f4d0a8d091249 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -32,7 +32,7 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,7 +61,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class AWSCredentialProviderList implements AWSCredentialsProvider,
+public final class AWSCredentialProviderList implements AWSCredentialsProvider,
     AutoCloseable {
 
   private static final Logger LOG = LoggerFactory.getLogger(
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
index 0cb9fd4e57118..564c03bf731d7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
@@ -34,9 +34,13 @@
  * Please note that users may reference this class name from configuration
  * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
  * would be a backward-incompatible change.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Stable
+@Deprecated
 public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
 
   public static final String NAME
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
index 7c866ac96765d..0294f7722905d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
@@ -21,12 +21,12 @@
 import javax.annotation.Nonnull;
 
 import com.amazonaws.arn.Arn;
-import com.amazonaws.regions.RegionUtils;
 
 /**
  * Represents an Arn Resource, this can be an accesspoint or bucket.
  */
 public final class ArnResource {
+  private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
 
   /**
    * Resource name.
@@ -106,8 +106,7 @@ public String getFullArn() {
    * @return resource endpoint.
    */
   public String getEndpoint() {
-    return RegionUtils.getRegion(accessPointRegionKey)
-        .getServiceEndpoint("s3");
+    return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 0eadc35c3c27f..e5b0a9b5aa163 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -20,16 +20,20 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
 
 import java.util.concurrent.TimeUnit;
 
 /**
- * All the constants used with the {@link S3AFileSystem}.
+ * Constants used with the {@link S3AFileSystem}.
  *
  * Some of the strings are marked as {@code Unstable}. This means
- * that they may be unsupported in future; at which point they will be marked
+ * that they may be Unsupported in future; at which point they will be marked
  * as deprecated and simply ignored.
+ *
+ * All S3Guard related constants are marked as Deprecated and either ignored (ddb config)
+ * or rejected (setting the metastore to anything other than the null store)
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
@@ -130,12 +134,13 @@ private Constants() {
   /**
    * JSON policy containing the policy to apply to the role: {@value}.
    * This is not used for delegation tokens, which generate the policy
-   * automatically, and restrict it to the S3, KMS and S3Guard services
+   * automatically, and restrict it to the S3 and KMS services
    * needed.
    */
   public static final String ASSUMED_ROLE_POLICY =
       "fs.s3a.assumed.role.policy";
 
+  @SuppressWarnings("deprecation")
   public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT =
       SimpleAWSCredentialsProvider.NAME;
 
@@ -154,6 +159,15 @@ private Constants() {
       "fs.s3a.connection.ssl.enabled";
   public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
 
+  /**
+   * Configuration option for S3 Requester Pays feature: {@value}.
+   */
+  public static final String ALLOW_REQUESTER_PAYS = "fs.s3a.requester.pays.enabled";
+  /**
+   * Default configuration for {@value ALLOW_REQUESTER_PAYS}: {@value}.
+   */
+  public static final boolean DEFAULT_ALLOW_REQUESTER_PAYS = false;
+
   // use OpenSSL or JSEE for secure connections
   public static final String SSL_CHANNEL_MODE =  "fs.s3a.ssl.channel.mode";
   public static final DelegatingSSLSocketFactory.SSLChannelMode
@@ -198,6 +212,8 @@ private Constants() {
   public static final String PROXY_PASSWORD = "fs.s3a.proxy.password";
   public static final String PROXY_DOMAIN = "fs.s3a.proxy.domain";
   public static final String PROXY_WORKSTATION = "fs.s3a.proxy.workstation";
+  /** Is the proxy secured(proxyProtocol = HTTPS)? */
+  public static final String PROXY_SECURED = "fs.s3a.proxy.ssl.enabled";
 
   /**
    * Number of times the AWS client library should retry errors before
@@ -387,7 +403,7 @@ private Constants() {
    * Value: {@value}.
    */
   public static final boolean DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT =
-      false;
+      true;
 
   /**
    * The capacity of executor queues for operations other than block
@@ -416,6 +432,57 @@ private Constants() {
    */
   public static final String CONTENT_ENCODING = "fs.s3a.object.content.encoding";
 
+  /**
+   * S3 storage class: standard, reduced_redundancy, intelligent_tiering etc.
+   * Value {@value }.
+   */
+  public static final String STORAGE_CLASS = "fs.s3a.create.storage.class";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_STANDARD = "standard";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_REDUCED_REDUNDANCY = "reduced_redundancy";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_GLACIER = "glacier";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_STANDARD_INFREQUENT_ACCESS = "standard_ia";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_ONEZONE_INFREQUENT_ACCESS = "onezone_ia";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_INTELLIGENT_TIERING = "intelligent_tiering";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_DEEP_ARCHIVE = "deep_archive";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_OUTPOSTS = "outposts";
+
+  /**
+   * S3 Storage option: {@value}.
+   */
+  public static final String STORAGE_CLASS_GLACIER_INSTANT_RETRIEVAL = "glacier_ir";
+
   // should we try to purge old multipart uploads when starting up
   public static final String PURGE_EXISTING_MULTIPART =
       "fs.s3a.multipart.purge";
@@ -494,20 +561,17 @@ private Constants() {
   public static final String CUSTOM_SIGNERS = "fs.s3a.custom.signers";
 
   /**
-   * There's 3 parameters that can be used to specify a non-default signing
+   * Multiple parameters can be used to specify a non-default signing
    * algorithm.<br>
    * fs.s3a.signing-algorithm - This property has existed for the longest time.
-   * If specified, without either of the other 2 properties being specified,
-   * this signing algorithm will be used for S3 and DDB (S3Guard). <br>
-   * The other 2 properties override this value for S3 or DDB. <br>
+   * If specified, without other properties being specified,
+   * this signing algorithm will be used for all services. <br>
+   * Another property overrides this value for S3. <br>
    * fs.s3a.s3.signing-algorithm - Allows overriding the S3 Signing algorithm.
-   * This does not affect DDB. Specifying this property without specifying
+   * Specifying this property without specifying
    * fs.s3a.signing-algorithm will only update the signing algorithm for S3
-   * requests, and the default will be used for DDB.<br>
-   * fs.s3a.ddb.signing-algorithm - Allows overriding the DDB Signing algorithm.
-   * This does not affect S3. Specifying this property without specifying
-   * fs.s3a.signing-algorithm will only update the signing algorithm for
-   * DDB requests, and the default will be used for S3.
+   * requests.
+   * {@code fs.s3a.sts.signing-algorithm}: algorithm to use for STS interaction.
    */
   public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm";
 
@@ -515,6 +579,7 @@ private Constants() {
       "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_S3.toLowerCase()
           + ".signing-algorithm";
 
+  @Deprecated
   public static final String SIGNING_ALGORITHM_DDB =
       "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_DDB.toLowerCase()
           + "signing-algorithm";
@@ -540,13 +605,23 @@ private Constants() {
 
   public static final String USER_AGENT_PREFIX = "fs.s3a.user.agent.prefix";
 
-  /** Whether or not to allow MetadataStore to be source of truth for a path prefix */
+  /**
+   * Paths considered "authoritative".
+   * When S3guard was supported, this skipped checks to s3 on directory listings.
+   * It is also use to optionally disable marker retentation purely on these
+   * paths -a feature which is still retained/available.
+   * */
   public static final String AUTHORITATIVE_PATH = "fs.s3a.authoritative.path";
   public static final String[] DEFAULT_AUTHORITATIVE_PATH = {};
 
-  /** Whether or not to allow MetadataStore to be source of truth. */
+  /**
+   * Whether or not to allow MetadataStore to be source of truth.
+   * @deprecated no longer supported
+   */
+  @Deprecated
   public static final String METADATASTORE_AUTHORITATIVE =
       "fs.s3a.metadatastore.authoritative";
+  @Deprecated
   public static final boolean DEFAULT_METADATASTORE_AUTHORITATIVE = false;
 
   /**
@@ -565,13 +640,16 @@ private Constants() {
 
   /**
    * How long a directory listing in the MS is considered as authoritative.
+   * @deprecated no longer supported
    */
+  @Deprecated
   public static final String METADATASTORE_METADATA_TTL =
       "fs.s3a.metadatastore.metadata.ttl";
 
   /**
    * Default TTL in milliseconds: 15 minutes.
    */
+  @Deprecated
   public static final long DEFAULT_METADATASTORE_METADATA_TTL =
       TimeUnit.MINUTES.toMillis(15);
 
@@ -579,37 +657,69 @@ private Constants() {
   public static final String READAHEAD_RANGE = "fs.s3a.readahead.range";
   public static final long DEFAULT_READAHEAD_RANGE = 64 * 1024;
 
+  /**
+   * The threshold at which drain operations switch
+   * to being asynchronous with the schedule/wait overhead
+   * compared to synchronous.
+   * Value: {@value}
+   */
+  public static final String ASYNC_DRAIN_THRESHOLD = "fs.s3a.input.async.drain.threshold";
+
+  /**
+   * This is a number based purely on experimentation in
+   * {@code ITestS3AInputStreamPerformance}.
+   * Value: {@value}
+   */
+  public static final int DEFAULT_ASYNC_DRAIN_THRESHOLD = 16_000;
+
   /**
    * Which input strategy to use for buffering, seeking and similar when
    * reading data.
    * Value: {@value}
    */
-  @InterfaceStability.Unstable
   public static final String INPUT_FADVISE =
       "fs.s3a.experimental.input.fadvise";
 
+  /**
+   * The default value for this FS.
+   * Which for S3A, is adaptive.
+   * Value: {@value}
+   * @deprecated use the {@link Options.OpenFileOptions} value
+   * in code which only needs to be compiled against newer hadoop
+   * releases.
+   */
+  public static final String INPUT_FADV_DEFAULT =
+      Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT;
+
   /**
    * General input. Some seeks, some reads.
+   * The policy name "default" is standard across different stores,
+   * and should be preferred.
    * Value: {@value}
    */
-  @InterfaceStability.Unstable
   public static final String INPUT_FADV_NORMAL = "normal";
 
   /**
    * Optimized for sequential access.
    * Value: {@value}
+   * @deprecated use the {@link Options.OpenFileOptions} value
+   * in code which only needs to be compiled against newer hadoop
+   * releases.
    */
-  @InterfaceStability.Unstable
-  public static final String INPUT_FADV_SEQUENTIAL = "sequential";
+  public static final String INPUT_FADV_SEQUENTIAL =
+      Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
 
   /**
    * Optimized purely for random seek+read/positionedRead operations;
    * The performance of sequential IO may be reduced in exchange for
    * more efficient {@code seek()} operations.
    * Value: {@value}
+   * @deprecated use the {@link Options.OpenFileOptions} value
+   * in code which only needs to be compiled against newer hadoop
+   * releases.
    */
-  @InterfaceStability.Unstable
-  public static final String INPUT_FADV_RANDOM = "random";
+  public static final String INPUT_FADV_RANDOM =
+      Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM;
 
   /**
    * Gauge name for the input policy : {@value}.
@@ -625,6 +735,7 @@ private Constants() {
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
+  @SuppressWarnings("deprecation")
   public static final Class<? extends S3ClientFactory>
       DEFAULT_S3_CLIENT_FACTORY_IMPL =
           DefaultS3ClientFactory.class;
@@ -635,202 +746,117 @@ private Constants() {
   @InterfaceAudience.Private
   public static final int MAX_MULTIPART_COUNT = 10000;
 
-  /* Constants. */
+  /*
+   * Obsolete S3Guard-related options, retained purely because this file
+   * is @Public/@Evolving.
+   */
+  @Deprecated
   public static final String S3_METADATA_STORE_IMPL =
       "fs.s3a.metadatastore.impl";
-
-  /**
-   * Whether to fail when there is an error writing to the metadata store.
-   */
+  @Deprecated
   public static final String FAIL_ON_METADATA_WRITE_ERROR =
       "fs.s3a.metadatastore.fail.on.write.error";
-
-  /**
-   * Default value ({@value}) for FAIL_ON_METADATA_WRITE_ERROR.
-   */
+  @Deprecated
   public static final boolean FAIL_ON_METADATA_WRITE_ERROR_DEFAULT = true;
-
-  /** Minimum period of time (in milliseconds) to keep metadata (may only be
-   * applied when a prune command is manually run).
-   */
   @InterfaceStability.Unstable
+  @Deprecated
   public static final String S3GUARD_CLI_PRUNE_AGE =
       "fs.s3a.s3guard.cli.prune.age";
-
-  /**
-   * The region of the DynamoDB service.
-   *
-   * This config has no default value. If the user does not set this, the
-   * S3Guard will operate table in the associated S3 bucket region.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_REGION_KEY =
       "fs.s3a.s3guard.ddb.region";
-
-  /**
-   * The DynamoDB table name to use.
-   *
-   * This config has no default value. If the user does not set this, the
-   * S3Guard implementation will use the respective S3 bucket name.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_NAME_KEY =
       "fs.s3a.s3guard.ddb.table";
-
-  /**
-   * A prefix for adding tags to the DDB Table upon creation.
-   *
-   * For example:
-   * fs.s3a.s3guard.ddb.table.tag.mytag
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_TAG =
       "fs.s3a.s3guard.ddb.table.tag.";
-
-  /**
-   * Whether to create the DynamoDB table if the table does not exist.
-   * Value: {@value}.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_CREATE_KEY =
       "fs.s3a.s3guard.ddb.table.create";
-
-  /**
-   * Read capacity when creating a table.
-   * When it and the write capacity are both "0", a per-request table is
-   * created.
-   * Value: {@value}.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY =
       "fs.s3a.s3guard.ddb.table.capacity.read";
-
-  /**
-   * Default read capacity when creating a table.
-   * Value: {@value}.
-   */
+  @Deprecated
   public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 0;
-
-  /**
-   * Write capacity when creating a table.
-   * When it and the read capacity are both "0", a per-request table is
-   * created.
-   * Value: {@value}.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY =
       "fs.s3a.s3guard.ddb.table.capacity.write";
-
-  /**
-   * Default write capacity when creating a table.
-   * Value: {@value}.
-   */
+  @Deprecated
   public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 0;
-
-  /**
-   * Whether server-side encryption (SSE) is enabled or disabled on the table.
-   * By default it's disabled, meaning SSE is set to AWS owned CMK.
-   * @see com.amazonaws.services.dynamodbv2.model.SSESpecification#setEnabled
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_SSE_ENABLED =
       "fs.s3a.s3guard.ddb.table.sse.enabled";
-
-  /**
-   * The KMS Master Key (CMK) used for the KMS encryption on the table.
-   *
-   * To specify a CMK, this config value can be its key ID, Amazon Resource
-   * Name (ARN), alias name, or alias ARN. Users only provide this config
-   * if the key is different from the default DynamoDB KMS Master Key, which is
-   * alias/aws/dynamodb.
-   */
+  @Deprecated
   public static final String S3GUARD_DDB_TABLE_SSE_CMK =
       "fs.s3a.s3guard.ddb.table.sse.cmk";
-
-  /**
-   * The maximum put or delete requests per BatchWriteItem request.
-   *
-   * Refer to Amazon API reference for this limit.
-   */
+  @Deprecated
   public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25;
-
+  @Deprecated
   public static final String S3GUARD_DDB_MAX_RETRIES =
       "fs.s3a.s3guard.ddb.max.retries";
-
-  /**
-   * Max retries on batched/throttled DynamoDB operations before giving up and
-   * throwing an IOException.  Default is {@value}. See core-default.xml for
-   * more detail.
-   */
+  @Deprecated
   public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT =
       DEFAULT_MAX_ERROR_RETRIES;
-
+  @Deprecated
   public static final String S3GUARD_DDB_THROTTLE_RETRY_INTERVAL =
       "fs.s3a.s3guard.ddb.throttle.retry.interval";
+  @Deprecated
   public static final String S3GUARD_DDB_THROTTLE_RETRY_INTERVAL_DEFAULT =
       "100ms";
-
-  /**
-   * Period of time (in milliseconds) to sleep between batches of writes.
-   * Currently only applies to prune operations, as they are naturally a
-   * lower priority than other operations.
-   */
+  @Deprecated
   @InterfaceStability.Unstable
   public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY =
       "fs.s3a.s3guard.ddb.background.sleep";
+  @Deprecated
   public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25;
 
   /**
    * The default "Null" metadata store: {@value}.
    */
+  @Deprecated
   public static final String S3GUARD_METASTORE_NULL
       = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
-
-  /**
-   * Use Local memory for the metadata: {@value}.
-   * This is not coherent across processes and must be used for testing only.
-   */
+  @Deprecated
   @InterfaceStability.Unstable
   public static final String S3GUARD_METASTORE_LOCAL
       = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore";
-
-  /**
-   * Maximum number of records in LocalMetadataStore.
-   */
   @InterfaceStability.Unstable
+  @Deprecated
   public static final String S3GUARD_METASTORE_LOCAL_MAX_RECORDS =
       "fs.s3a.s3guard.local.max_records";
+  @Deprecated
   public static final int DEFAULT_S3GUARD_METASTORE_LOCAL_MAX_RECORDS = 256;
-
-  /**
-   * Time to live in milliseconds in LocalMetadataStore.
-   * If zero, time-based expiration is disabled.
-   */
   @InterfaceStability.Unstable
+  @Deprecated
   public static final String S3GUARD_METASTORE_LOCAL_ENTRY_TTL =
       "fs.s3a.s3guard.local.ttl";
+  @Deprecated
   public static final int DEFAULT_S3GUARD_METASTORE_LOCAL_ENTRY_TTL
       = 60 * 1000;
-
-  /**
-   * Use DynamoDB for the metadata: {@value}.
-   */
+  @Deprecated
   public static final String S3GUARD_METASTORE_DYNAMO
       = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
-
-  /**
-   * The warn level if S3Guard is disabled.
-   */
+  @Deprecated
   public static final String S3GUARD_DISABLED_WARN_LEVEL
       = "fs.s3a.s3guard.disabled.warn.level";
+  @Deprecated
   public static final String DEFAULT_S3GUARD_DISABLED_WARN_LEVEL =
       "SILENT";
 
   /**
    * Inconsistency (visibility delay) injection settings.
+   * No longer used.
    */
-  @InterfaceStability.Unstable
+  @Deprecated
   public static final String FAIL_INJECT_INCONSISTENCY_KEY =
       "fs.s3a.failinject.inconsistency.key.substring";
 
-  @InterfaceStability.Unstable
+  @Deprecated
   public static final String FAIL_INJECT_INCONSISTENCY_MSEC =
       "fs.s3a.failinject.inconsistency.msec";
 
-  @InterfaceStability.Unstable
+  @Deprecated
   public static final String FAIL_INJECT_INCONSISTENCY_PROBABILITY =
       "fs.s3a.failinject.inconsistency.probability";
 
@@ -990,17 +1016,20 @@ private Constants() {
    * Number of times to retry any repeatable S3 client request on failure,
    * excluding throttling requests: {@value}.
    */
+  @Deprecated
   public static final String S3GUARD_CONSISTENCY_RETRY_LIMIT =
       "fs.s3a.s3guard.consistency.retry.limit";
 
   /**
    * Default retry limit: {@value}.
    */
+  @Deprecated
   public static final int S3GUARD_CONSISTENCY_RETRY_LIMIT_DEFAULT = 7;
 
   /**
    * Initial retry interval: {@value}.
    */
+  @Deprecated
   public static final String S3GUARD_CONSISTENCY_RETRY_INTERVAL =
       "fs.s3a.s3guard.consistency.retry.interval";
 
@@ -1010,10 +1039,12 @@ private Constants() {
    * each probe can cause the S3 load balancers to retain any 404 in
    * its cache for longer. See HADOOP-16490.
    */
+  @Deprecated
   public static final String S3GUARD_CONSISTENCY_RETRY_INTERVAL_DEFAULT =
       "2s";
 
   public static final String AWS_SERVICE_IDENTIFIER_S3 = "S3";
+  @Deprecated
   public static final String AWS_SERVICE_IDENTIFIER_DDB = "DDB";
   public static final String AWS_SERVICE_IDENTIFIER_STS = "STS";
 
@@ -1132,4 +1163,96 @@ private Constants() {
    * Require that all S3 access is made through Access Points.
    */
   public static final String AWS_S3_ACCESSPOINT_REQUIRED = "fs.s3a.accesspoint.required";
+
+  /**
+   * Flag for create performance.
+   * This is *not* a configuration option; it is for use in the
+   * {code createFile()} builder.
+   * Value {@value}.
+   */
+  public static final String FS_S3A_CREATE_PERFORMANCE = "fs.s3a.create.performance";
+
+  /**
+   * Prefix for adding a header to the object when created.
+   * The actual value must have a "." suffix and then the actual header.
+   * This is *not* a configuration option; it is only for use in the
+   * {code createFile()} builder.
+   * Value {@value}.
+   */
+  public static final String FS_S3A_CREATE_HEADER = "fs.s3a.create.header";
+
+  /**
+   * What is the smallest reasonable seek in bytes such
+   * that we group ranges together during vectored read operation.
+   * Value : {@value}.
+   */
+  public static final String AWS_S3_VECTOR_READS_MIN_SEEK_SIZE =
+          "fs.s3a.vectored.read.min.seek.size";
+
+  /**
+   * What is the largest merged read size in bytes such
+   * that we group ranges together during vectored read.
+   * Setting this value to 0 will disable merging of ranges.
+   * Value : {@value}.
+   */
+  public static final String AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE =
+          "fs.s3a.vectored.read.max.merged.size";
+
+  /**
+   * Default minimum seek in bytes during vectored reads : {@value}.
+   */
+  public static final int DEFAULT_AWS_S3_VECTOR_READS_MIN_SEEK_SIZE = 4896; // 4K
+
+  /**
+   * Default maximum read size in bytes during vectored reads : {@value}.
+   */
+  public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M
+
+  /**
+   * Maximum number of range reads a single input stream can have
+   * active (downloading, or queued) to the central FileSystem
+   * instance's pool of queued operations.
+   * This stops a single stream overloading the shared thread pool.
+   * {@value}
+   * <p>
+   * Default is {@link #DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS}
+   */
+  public static final String AWS_S3_VECTOR_ACTIVE_RANGE_READS =
+          "fs.s3a.vectored.active.ranged.reads";
+
+  /**
+   * Limit of queued range data download operations during vectored
+   * read. Value: {@value}
+   */
+  public static final int DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS = 4;
+
+  /**
+   * Prefix of auth classes in AWS SDK V1.
+   */
+  public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth";
+
+  /**
+   * Controls whether the prefetching input stream is enabled.
+   */
+  public static final String PREFETCH_ENABLED_KEY = "fs.s3a.prefetch.enabled";
+
+  /**
+   * Default option as to whether the prefetching input stream is enabled.
+   */
+  public static final boolean  PREFETCH_ENABLED_DEFAULT = false;
+
+  // If the default values are used, each file opened for reading will consume
+  // 64 MB of heap space (8 blocks x 8 MB each).
+
+  /**
+   * The size of a single prefetched block in number of bytes.
+   */
+  public static final String PREFETCH_BLOCK_SIZE_KEY = "fs.s3a.prefetch.block.size";
+  public static final int PREFETCH_BLOCK_DEFAULT_SIZE = 8 * 1024 * 1024;
+
+  /**
+   * Maximum number of blocks prefetched at any given time.
+   */
+  public static final String PREFETCH_BLOCK_COUNT_KEY = "fs.s3a.prefetch.block.count";
+  public static final int PREFETCH_BLOCK_DEFAULT_COUNT = 8;
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 764cb5ce97d73..f724f86e4afcd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -41,7 +41,7 @@
 import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
 import com.amazonaws.util.AwsHostNameUtils;
 import com.amazonaws.util.RuntimeHttpUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,6 +54,7 @@
 import org.apache.hadoop.fs.s3a.statistics.impl.AwsStatisticsCollector;
 import org.apache.hadoop.fs.store.LogExactlyOnce;
 
+import static com.amazonaws.services.s3.Headers.REQUESTER_PAYS_HEADER;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_CENTRAL_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING;
@@ -70,11 +71,14 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
+@SuppressWarnings("deprecation")
 public class DefaultS3ClientFactory extends Configured
     implements S3ClientFactory {
 
   private static final String S3_SERVICE_NAME = "s3";
 
+  private static final String REQUESTER_PAYS_HEADER_VALUE = "requester";
+
   /**
    * Subclasses refer to this.
    */
@@ -118,6 +122,11 @@ public AmazonS3 createS3Client(
     parameters.getHeaders().forEach((h, v) ->
         awsConf.addHeader(h, v));
 
+    if (parameters.isRequesterPays()) {
+      // All calls must acknowledge requester will pay via header.
+      awsConf.addHeader(REQUESTER_PAYS_HEADER, REQUESTER_PAYS_HEADER_VALUE);
+    }
+
     // When EXPERIMENTAL_AWS_INTERNAL_THROTTLING is false
     // throttling is explicitly disabled on the S3 client so that
     // all failures are collected in S3A instrumentation, and its
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
index f6bfc9aa92802..cfd7046e8abfa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/FailureInjectionPolicy.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,11 +28,6 @@
 
 /**
  * Simple object which stores current failure injection settings.
- * "Delaying a key" can mean:
- *    - Removing it from the S3 client's listings while delay is in effect.
- *    - Causing input stream reads to fail.
- *    - Causing the S3 side of getFileStatus(), i.e.
- *      AmazonS3#getObjectMetadata(), to throw FileNotFound.
  */
 public class FailureInjectionPolicy {
   /**
@@ -40,29 +35,9 @@ public class FailureInjectionPolicy {
    */
   public static final String DEFAULT_DELAY_KEY_SUBSTRING = "DELAY_LISTING_ME";
 
-  /**
-   * How many seconds affected keys will have delayed visibility.
-   * This should probably be a config value.
-   */
-  public static final long DEFAULT_DELAY_KEY_MSEC = 5 * 1000;
-
-  public static final float DEFAULT_DELAY_KEY_PROBABILITY = 1.0f;
-
-  /** Special config value since we can't store empty strings in XML. */
-  public static final String MATCH_ALL_KEYS = "*";
-
   private static final Logger LOG =
       LoggerFactory.getLogger(InconsistentAmazonS3Client.class);
 
-  /** Empty string matches all keys. */
-  private String delayKeySubstring;
-
-  /** Probability to delay visibility of a matching key. */
-  private float delayKeyProbability;
-
-  /** Time in milliseconds to delay visibility of newly modified object. */
-  private long delayKeyMsec;
-
   /**
    * Probability of throttling a request.
    */
@@ -75,33 +50,10 @@ public class FailureInjectionPolicy {
 
   public FailureInjectionPolicy(Configuration conf) {
 
-    this.delayKeySubstring = conf.get(FAIL_INJECT_INCONSISTENCY_KEY,
-        DEFAULT_DELAY_KEY_SUBSTRING);
-    // "" is a substring of all strings, use it to match all keys.
-    if (this.delayKeySubstring.equals(MATCH_ALL_KEYS)) {
-      this.delayKeySubstring = "";
-    }
-    this.delayKeyProbability = validProbability(
-        conf.getFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY,
-            DEFAULT_DELAY_KEY_PROBABILITY));
-    this.delayKeyMsec = conf.getLong(FAIL_INJECT_INCONSISTENCY_MSEC,
-        DEFAULT_DELAY_KEY_MSEC);
     this.setThrottleProbability(conf.getFloat(FAIL_INJECT_THROTTLE_PROBABILITY,
         0.0f));
   }
 
-  public String getDelayKeySubstring() {
-    return delayKeySubstring;
-  }
-
-  public float getDelayKeyProbability() {
-    return delayKeyProbability;
-  }
-
-  public long getDelayKeyMsec() {
-    return delayKeyMsec;
-  }
-
   public float getThrottleProbability() {
     return throttleProbability;
   }
@@ -126,25 +78,10 @@ public static boolean trueWithProbability(float p) {
     return Math.random() < p;
   }
 
-  /**
-   * Should we delay listing visibility for this key?
-   * @param key key which is being put
-   * @return true if we should delay
-   */
-  public boolean shouldDelay(String key) {
-    float p = getDelayKeyProbability();
-    boolean delay = key.contains(getDelayKeySubstring());
-    delay = delay && trueWithProbability(p);
-    LOG.debug("{}, p={} -> {}", key, p, delay);
-    return delay;
-  }
-
   @Override
   public String toString() {
     return String.format("FailureInjectionPolicy:" +
-            " %s msec delay, substring %s, delay probability %s;" +
             " throttle probability %s" + "; failure limit %d",
-        delayKeyMsec, delayKeySubstring, delayKeyProbability,
         throttleProbability, failureLimit);
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
index e54c21d853e6a..c6d17a32b64b1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
@@ -18,13 +18,8 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
@@ -53,19 +48,20 @@
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
 
 /**
  * A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects
- * inconsistency and/or errors.  Used for testing S3Guard.
- * Currently only delays listing visibility, not affecting GET.
+ * failures.
+ * It used to also inject inconsistency, but this was removed with S3Guard;
+ * what is retained is the ability to throttle AWS operations and for the
+ * input stream to be inconsistent.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -81,38 +77,6 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
    */
   private final AtomicLong failureCounter = new AtomicLong(0);
 
-  /**
-   * Composite of data we need to track about recently deleted objects:
-   * when it was deleted (same was with recently put objects) and the object
-   * summary (since we should keep returning it for sometime after its
-   * deletion).
-   */
-  private static class Delete {
-    private Long time;
-    private S3ObjectSummary summary;
-
-    Delete(Long time, S3ObjectSummary summary) {
-      this.time = time;
-      this.summary = summary;
-    }
-
-    public Long time() {
-      return time;
-    }
-
-    public S3ObjectSummary summary() {
-      return summary;
-    }
-  }
-
-  /**
-   * Map of key to delay -> time it was deleted + object summary (object summary
-   * is null for prefixes.
-   */
-  private Map<String, Delete> delayedDeletes = new HashMap<>();
-
-  /** Map of key to delay -> time it was created. */
-  private Map<String, Long> delayedPutKeys = new HashMap<>();
 
   /**
    * Instantiate.
@@ -130,19 +94,6 @@ public InconsistentAmazonS3Client(AWSCredentialsProvider credentials,
     policy = new FailureInjectionPolicy(conf);
   }
 
-
-  /**
-   * Clear any accumulated inconsistency state. Used by tests to make paths
-   * visible again.
-   * @param fs S3AFileSystem under test
-   * @throws Exception on failure
-   */
-  public static void clearInconsistency(S3AFileSystem fs) throws Exception {
-    AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard");
-    InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
-    ic.clearInconsistency();
-  }
-
   /**
    * A way for tests to patch in a different fault injection policy at runtime.
    * @param fs filesystem under test
@@ -166,17 +117,6 @@ public String toString() {
         policy, failureCounter.get());
   }
 
-  /**
-   * Clear all oustanding inconsistent keys.  After calling this function,
-   * listings should behave normally (no failure injection), until additional
-   * keys are matched for delay, e.g. via putObject(), deleteObject().
-   */
-  public void clearInconsistency() {
-    LOG.info("clearing all delayed puts / deletes");
-    delayedDeletes.clear();
-    delayedPutKeys.clear();
-  }
-
   /**
    * Convenience function for test code to cast from supertype.
    * @param c supertype to cast from
@@ -199,12 +139,6 @@ public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
       deleteObjectsRequest)
       throws AmazonClientException, AmazonServiceException {
     maybeFail();
-    LOG.info("registering bulk delete of objects");
-    for (DeleteObjectsRequest.KeyVersion keyVersion :
-        deleteObjectsRequest.getKeys()) {
-      registerDeleteObject(keyVersion.getKey(),
-          deleteObjectsRequest.getBucketName());
-    }
     return super.deleteObjects(deleteObjectsRequest);
   }
 
@@ -214,7 +148,6 @@ public void deleteObject(DeleteObjectRequest deleteObjectRequest)
     String key = deleteObjectRequest.getKey();
     LOG.debug("key {}", key);
     maybeFail();
-    registerDeleteObject(key, deleteObjectRequest.getBucketName());
     super.deleteObject(deleteObjectRequest);
   }
 
@@ -224,7 +157,6 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest)
       throws AmazonClientException, AmazonServiceException {
     LOG.debug("key {}", putObjectRequest.getKey());
     maybeFail();
-    registerPutObject(putObjectRequest);
     return super.putObject(putObjectRequest);
   }
 
@@ -233,283 +165,17 @@ public PutObjectResult putObject(PutObjectRequest putObjectRequest)
   public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
       throws AmazonClientException, AmazonServiceException {
     maybeFail();
-    return innerlistObjects(listObjectsRequest);
+    return super.listObjects(listObjectsRequest);
   }
 
-  /**
-   * Run the list object call without any failure probability.
-   * This stops a very aggressive failure rate from completely overloading
-   * the retry logic.
-   * @param listObjectsRequest request
-   * @return listing
-   * @throws AmazonClientException failure
-   */
-  private ObjectListing innerlistObjects(ListObjectsRequest listObjectsRequest)
-      throws AmazonClientException, AmazonServiceException {
-    LOG.debug("prefix {}", listObjectsRequest.getPrefix());
-    ObjectListing listing = super.listObjects(listObjectsRequest);
-    listing = filterListObjects(listing);
-    listing = restoreListObjects(listObjectsRequest, listing);
-    return listing;
-  }
-
-  /* We should only need to override these versions of listObjects() */
+  /* consistent listing with possibility of failing. */
   @Override
   public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request)
       throws AmazonClientException, AmazonServiceException {
     maybeFail();
-    return innerListObjectsV2(request);
-  }
-
-  /**
-   * Non failing V2 list object request.
-   * @param request request
-   * @return result.
-   */
-  private ListObjectsV2Result innerListObjectsV2(ListObjectsV2Request request) {
-    LOG.debug("prefix {}", request.getPrefix());
-    ListObjectsV2Result listing = super.listObjectsV2(request);
-    listing = filterListObjectsV2(listing);
-    listing = restoreListObjectsV2(request, listing);
-    return listing;
-  }
-
-  private void addSummaryIfNotPresent(List<S3ObjectSummary> list,
-      S3ObjectSummary item) {
-    // Behavior of S3ObjectSummary
-    String key = item.getKey();
-    if (list.stream().noneMatch((member) -> member.getKey().equals(key))) {
-      LOG.debug("Reinstate summary {}", key);
-      list.add(item);
-    }
-  }
-
-  /**
-   * Add prefix of child to given list.  The added prefix will be equal to
-   * ancestor plus one directory past ancestor.  e.g.:
-   * if ancestor is "/a/b/c" and child is "/a/b/c/d/e/file" then "a/b/c/d" is
-   * added to list.
-   * @param prefixes list to add to
-   * @param ancestor path we are listing in
-   * @param child full path to get prefix from
-   */
-  private void addPrefixIfNotPresent(List<String> prefixes, String ancestor,
-      String child) {
-    Path prefixCandidate = new Path(child).getParent();
-    Path ancestorPath = new Path(ancestor);
-    Preconditions.checkArgument(child.startsWith(ancestor), "%s does not " +
-        "start with %s", child, ancestor);
-    while (!prefixCandidate.isRoot()) {
-      Path nextParent = prefixCandidate.getParent();
-      if (nextParent.equals(ancestorPath)) {
-        String prefix = prefixCandidate.toString();
-        if (!prefixes.contains(prefix)) {
-          LOG.debug("Reinstate prefix {}", prefix);
-          prefixes.add(prefix);
-        }
-        return;
-      }
-      prefixCandidate = nextParent;
-    }
-  }
-
-  /**
-   * Checks that the parent key is an ancestor of the child key.
-   * @param parent key that may be the parent.
-   * @param child key that may be the child.
-   * @param recursive if false, only return true for direct children.  If
-   *                  true, any descendant will count.
-   * @return true if parent is an ancestor of child
-   */
-  private boolean isDescendant(String parent, String child, boolean recursive) {
-    if (recursive) {
-      if (!parent.endsWith("/")) {
-        parent = parent + "/";
-      }
-      return child.startsWith(parent);
-    } else {
-      Path actualParentPath = new Path(child).getParent();
-      Path expectedParentPath = new Path(parent);
-      // children which are directory markers are excluded here
-      return actualParentPath.equals(expectedParentPath)
-          && !child.endsWith("/");
-    }
-  }
-
-  /**
-   * Simulate eventual consistency of delete for this list operation:  Any
-   * recently-deleted keys will be added.
-   * @param request List request
-   * @param rawListing listing returned from underlying S3
-   * @return listing with recently-deleted items restored
-   */
-  private ObjectListing restoreListObjects(ListObjectsRequest request,
-      ObjectListing rawListing) {
-    List<S3ObjectSummary> outputList = rawListing.getObjectSummaries();
-    List<String> outputPrefixes = rawListing.getCommonPrefixes();
-    // recursive list has no delimiter, returns everything that matches a
-    // prefix.
-    boolean recursiveObjectList = !("/".equals(request.getDelimiter()));
-    String prefix = request.getPrefix();
-
-    restoreDeleted(outputList, outputPrefixes, recursiveObjectList, prefix);
-    return new CustomObjectListing(rawListing, outputList, outputPrefixes);
-  }
-
-  /**
-   * V2 list API variant of
-   * {@link #restoreListObjects(ListObjectsRequest, ObjectListing)}.
-   * @param request original v2 list request
-   * @param result raw s3 result
-   */
-  private ListObjectsV2Result restoreListObjectsV2(ListObjectsV2Request request,
-      ListObjectsV2Result result) {
-    List<S3ObjectSummary> outputList = result.getObjectSummaries();
-    List<String> outputPrefixes = result.getCommonPrefixes();
-    // recursive list has no delimiter, returns everything that matches a
-    // prefix.
-    boolean recursiveObjectList = !("/".equals(request.getDelimiter()));
-    String prefix = request.getPrefix();
-
-    restoreDeleted(outputList, outputPrefixes, recursiveObjectList, prefix);
-    return new CustomListObjectsV2Result(result, outputList, outputPrefixes);
-  }
-
-
-  /**
-   * Main logic for
-   * {@link #restoreListObjects(ListObjectsRequest, ObjectListing)} and
-   * the v2 variant above.
-   * @param summaries object summary list to modify.
-   * @param prefixes prefix list to modify
-   * @param recursive true if recursive list request
-   * @param prefix prefix for original list request
-   */
-  private void restoreDeleted(List<S3ObjectSummary> summaries,
-      List<String> prefixes, boolean recursive, String prefix) {
-
-    // Go through all deleted keys
-    for (String key : new HashSet<>(delayedDeletes.keySet())) {
-      Delete delete = delayedDeletes.get(key);
-      if (isKeyDelayed(delete.time(), key)) {
-        if (isDescendant(prefix, key, recursive)) {
-          if (delete.summary() != null) {
-            addSummaryIfNotPresent(summaries, delete.summary());
-          }
-        }
-        // Non-recursive list has delimiter: will return rolled-up prefixes for
-        // all keys that are not direct children
-        if (!recursive) {
-          if (isDescendant(prefix, key, true)) {
-            addPrefixIfNotPresent(prefixes, prefix, key);
-          }
-        }
-      } else {
-        // Clean up any expired entries
-        LOG.debug("Remove expired key {}", key);
-        delayedDeletes.remove(key);
-      }
-    }
-  }
-
-  private ObjectListing filterListObjects(ObjectListing rawListing) {
-
-    // Filter object listing
-    List<S3ObjectSummary> outputList = filterSummaries(
-        rawListing.getObjectSummaries());
-
-    // Filter prefixes (directories)
-    List<String> outputPrefixes = filterPrefixes(
-        rawListing.getCommonPrefixes());
-
-    return new CustomObjectListing(rawListing, outputList, outputPrefixes);
-  }
-
-  private ListObjectsV2Result filterListObjectsV2(ListObjectsV2Result raw) {
-    // Filter object listing
-    List<S3ObjectSummary> outputList = filterSummaries(
-        raw.getObjectSummaries());
-
-    // Filter prefixes (directories)
-    List<String> outputPrefixes = filterPrefixes(raw.getCommonPrefixes());
-
-    return new CustomListObjectsV2Result(raw, outputList, outputPrefixes);
+    return super.listObjectsV2(request);
   }
 
-  private List<S3ObjectSummary> filterSummaries(
-      List<S3ObjectSummary> summaries) {
-    List<S3ObjectSummary> outputList = new ArrayList<>();
-    for (S3ObjectSummary s : summaries) {
-      String key = s.getKey();
-      if (!isKeyDelayed(delayedPutKeys.get(key), key)) {
-        outputList.add(s);
-      }
-    }
-    return outputList;
-  }
-
-  private List<String> filterPrefixes(List<String> prefixes) {
-    return prefixes.stream()
-        .filter(key -> !isKeyDelayed(delayedPutKeys.get(key), key))
-        .collect(Collectors.toList());
-  }
-
-  private boolean isKeyDelayed(Long enqueueTime, String key) {
-    if (enqueueTime == null) {
-      LOG.debug("no delay for key {}", key);
-      return false;
-    }
-    long currentTime = System.currentTimeMillis();
-    long deadline = enqueueTime + policy.getDelayKeyMsec();
-    if (currentTime >= deadline) {
-      delayedDeletes.remove(key);
-      LOG.debug("no longer delaying {}", key);
-      return false;
-    } else {
-      LOG.info("delaying {}", key);
-      return true;
-    }
-  }
-
-  private void registerDeleteObject(String key, String bucket) {
-    if (policy.shouldDelay(key)) {
-      Delete delete = delayedDeletes.get(key);
-      if (delete != null && isKeyDelayed(delete.time(), key)) {
-        // there is already an entry in the delayed delete list,
-        // so ignore the operation
-        LOG.debug("Ignoring delete of already deleted object");
-      } else {
-        // Record summary so we can add it back for some time post-deletion
-        ListObjectsRequest request = new ListObjectsRequest()
-            .withBucketName(bucket)
-            .withPrefix(key);
-        S3ObjectSummary summary = innerlistObjects(request).getObjectSummaries()
-            .stream()
-            .filter(result -> result.getKey().equals(key))
-            .findFirst()
-            .orElse(null);
-        delayedDeletes.put(key, new Delete(System.currentTimeMillis(),
-            summary));
-      }
-    }
-  }
-
-  private void registerPutObject(PutObjectRequest req) {
-    String key = req.getKey();
-    if (policy.shouldDelay(key)) {
-      enqueueDelayedPut(key);
-    }
-  }
-
-  /**
-   * Record this key as something that should not become visible in
-   * listObject replies for a while, to simulate eventual list consistency.
-   * @param key key to delay visibility of
-   */
-  private void enqueueDelayedPut(String key) {
-    LOG.debug("delaying put of {}", key);
-    delayedPutKeys.put(key, System.currentTimeMillis());
-  }
 
   @Override
   public CompleteMultipartUploadResult completeMultipartUpload(
@@ -542,10 +208,6 @@ public MultipartUploadListing listMultipartUploads(
     return super.listMultipartUploads(listMultipartUploadsRequest);
   }
 
-  public long getDelayKeyMsec() {
-    return policy.getDelayKeyMsec();
-  }
-
   /**
    * Set the probability of throttling a request.
    * @param throttleProbability the probability of a request being throttled.
@@ -565,7 +227,7 @@ private void maybeFail(String errorMsg, int statusCode)
       throws AmazonClientException {
     // code structure here is to line up for more failures later
     AmazonServiceException ex = null;
-    if (policy.trueWithProbability(policy.getThrottleProbability())) {
+    if (FailureInjectionPolicy.trueWithProbability(policy.getThrottleProbability())) {
       // throttle the request
       ex = new AmazonServiceException(errorMsg
           + " count = " + (failureCounter.get() + 1), null);
@@ -599,18 +261,16 @@ public void setFailureLimit(int limit) {
   @Override
   public S3Object getObject(GetObjectRequest var1) throws SdkClientException,
       AmazonServiceException {
-    maybeFail("file not found", 404);
-    S3Object o = super.getObject(var1);
-    LOG.debug("Wrapping in InconsistentS3Object for key {}", var1.getKey());
-    return new InconsistentS3Object(o, policy);
+    maybeFail();
+    return super.getObject(var1);
   }
 
   @Override
   public S3Object getObject(String bucketName, String key)
       throws SdkClientException, AmazonServiceException {
-    S3Object o = super.getObject(bucketName, key);
-    LOG.debug("Wrapping in InconsistentS3Object for key {}", key);
-    return new InconsistentS3Object(o, policy);
+    maybeFail();
+    return super.getObject(bucketName, key);
+
   }
 
   /** Since ObjectListing is immutable, we just override it with wrapper. */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
index c11581f1d5d78..4bfcc8aba3af3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java
@@ -42,6 +42,7 @@ protected AmazonS3 buildAmazonS3Client(
       final ClientConfiguration awsConf,
       final S3ClientCreationParameters parameters) {
     LOG.warn("** FAILURE INJECTION ENABLED.  Do not run in production! **");
+    LOG.warn("List inconsistency is no longer emulated; only throttling and read errors");
     InconsistentAmazonS3Client s3
         = new InconsistentAmazonS3Client(
             parameters.getCredentialSet(), awsConf, getConf());
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3Object.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3Object.java
deleted file mode 100644
index 496ca1b9085c3..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3Object.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import com.amazonaws.services.s3.internal.AmazonS3ExceptionBuilder;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Wrapper around S3Object so we can do failure injection on
- * getObjectContent() and S3ObjectInputStream.
- * See also {@link InconsistentAmazonS3Client}.
- */
-@SuppressWarnings({"NonSerializableFieldInSerializableClass", "serial"})
-public class InconsistentS3Object extends S3Object {
-
-  // This should be configurable, probably.
-  public static final int MAX_READ_FAILURES = 100;
-
-  private static int readFailureCounter = 0;
-  private transient S3Object wrapped;
-  private transient FailureInjectionPolicy policy;
-  private final static transient Logger LOG = LoggerFactory.getLogger(
-      InconsistentS3Object.class);
-
-  public InconsistentS3Object(S3Object wrapped, FailureInjectionPolicy policy) {
-    this.wrapped = wrapped;
-    this.policy = policy;
-  }
-
-  @Override
-  public S3ObjectInputStream getObjectContent() {
-    return new InconsistentS3InputStream(wrapped.getObjectContent());
-  }
-
-  @Override
-  public String toString() {
-    return "InconsistentS3Object wrapping: " + wrapped.toString();
-  }
-
-  @Override
-  public ObjectMetadata getObjectMetadata() {
-    return wrapped.getObjectMetadata();
-  }
-
-  @Override
-  public void setObjectMetadata(ObjectMetadata metadata) {
-    wrapped.setObjectMetadata(metadata);
-  }
-
-  @Override
-  public void setObjectContent(S3ObjectInputStream objectContent) {
-    wrapped.setObjectContent(objectContent);
-  }
-
-  @Override
-  public void setObjectContent(InputStream objectContent) {
-    wrapped.setObjectContent(objectContent);
-  }
-
-  @Override
-  public String getBucketName() {
-    return wrapped.getBucketName();
-  }
-
-  @Override
-  public void setBucketName(String bucketName) {
-    wrapped.setBucketName(bucketName);
-  }
-
-  @Override
-  public String getKey() {
-    return wrapped.getKey();
-  }
-
-  @Override
-  public void setKey(String key) {
-    wrapped.setKey(key);
-  }
-
-  @Override
-  public String getRedirectLocation() {
-    return wrapped.getRedirectLocation();
-  }
-
-  @Override
-  public void setRedirectLocation(String redirectLocation) {
-    wrapped.setRedirectLocation(redirectLocation);
-  }
-
-  @Override
-  public Integer getTaggingCount() {
-    return wrapped.getTaggingCount();
-  }
-
-  @Override
-  public void setTaggingCount(Integer taggingCount) {
-    wrapped.setTaggingCount(taggingCount);
-  }
-
-  @Override
-  public void close() throws IOException {
-    wrapped.close();
-  }
-
-  @Override
-  public boolean isRequesterCharged() {
-    return wrapped.isRequesterCharged();
-  }
-
-  @Override
-  public void setRequesterCharged(boolean isRequesterCharged) {
-    wrapped.setRequesterCharged(isRequesterCharged);
-  }
-
-  private AmazonS3Exception mockException(String msg, int httpResponse) {
-    AmazonS3ExceptionBuilder builder = new AmazonS3ExceptionBuilder();
-    builder.setErrorMessage(msg);
-    builder.setStatusCode(httpResponse); // this is the important part
-    builder.setErrorCode(String.valueOf(httpResponse));
-    return builder.build();
-  }
-
-  /**
-   * Insert a failiure injection point for a read call.
-   * @throw IOException, as codepath is on InputStream, not other SDK call.
-   */
-  private void readFailpoint(int off, int len) throws IOException {
-    if (shouldInjectFailure(getKey())) {
-      String error = String.format(
-          "read(b, %d, %d) on key %s failed: injecting error %d/%d" +
-              " for test.", off, len, getKey(), readFailureCounter,
-          MAX_READ_FAILURES);
-      throw new FileNotFoundException(error);
-    }
-  }
-
-  /**
-   * Insert a failiure injection point for an InputStream skip() call.
-   * @throw IOException, as codepath is on InputStream, not other SDK call.
-   */
-  private void skipFailpoint(long len) throws IOException {
-    if (shouldInjectFailure(getKey())) {
-      String error = String.format(
-          "skip(%d) on key %s failed: injecting error %d/%d for test.",
-          len, getKey(), readFailureCounter, MAX_READ_FAILURES);
-      throw new FileNotFoundException(error);
-    }
-  }
-
-  private boolean shouldInjectFailure(String key) {
-    if (policy.shouldDelay(key) &&
-        readFailureCounter < MAX_READ_FAILURES) {
-      readFailureCounter++;
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * Wraps S3ObjectInputStream and implements failure injection.
-   */
-  protected class InconsistentS3InputStream extends S3ObjectInputStream {
-    private S3ObjectInputStream wrapped;
-
-    public InconsistentS3InputStream(S3ObjectInputStream wrapped) {
-      // seems awkward to have the stream wrap itself.
-      super(wrapped, wrapped.getHttpRequest());
-      this.wrapped = wrapped;
-    }
-
-    @Override
-    public void abort() {
-      wrapped.abort();
-    }
-
-    @Override
-    public int available() throws IOException {
-      return wrapped.available();
-    }
-
-    @Override
-    public void close() throws IOException {
-      wrapped.close();
-    }
-
-    @Override
-    public long skip(long n) throws IOException {
-      skipFailpoint(n);
-      return wrapped.skip(n);
-    }
-
-    @Override
-    public int read() throws IOException {
-      LOG.debug("read() for key {}", getKey());
-      readFailpoint(0, 1);
-      return wrapped.read();
-    }
-
-    @Override
-    public int read(byte[] b, int off, int len) throws IOException {
-      LOG.debug("read(b, {}, {}) for key {}", off, len, getKey());
-      readFailpoint(off, len);
-      return wrapped.read(b, off, len);
-    }
-
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
index a851f0fbac687..279bfeba98769 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
@@ -21,20 +21,25 @@
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.util.Optional;
+import java.util.concurrent.Future;
 import javax.annotation.Nullable;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.SdkBaseException;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.statistics.DurationTracker;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.functional.CallableRaisingIOE;
+import org.apache.hadoop.util.functional.FutureIO;
 import org.apache.hadoop.util.functional.InvocationRaisingIOE;
+import org.apache.hadoop.util.Preconditions;
+
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
 
 /**
  * Class to provide lambda expression invocation of AWS operations.
@@ -120,6 +125,31 @@ public static <T> T once(String action, String path,
     }
   }
 
+  /**
+   * Execute a function, translating any exception into an IOException.
+   * The supplied duration tracker instance is updated with success/failure.
+   * @param action action to execute (used in error messages)
+   * @param path path of work (used in error messages)
+   * @param tracker tracker to update
+   * @param operation operation to execute
+   * @param <T> type of return value
+   * @return the result of the function call
+   * @throws IOException any IOE raised, or translated exception
+   */
+  @Retries.OnceTranslated
+  public static <T> T onceTrackingDuration(
+      final String action,
+      final String path,
+      final DurationTracker tracker,
+      final CallableRaisingIOE<T> operation)
+      throws IOException {
+    try {
+      return invokeTrackingDuration(tracker, operation);
+    } catch (AmazonClientException e) {
+      throw S3AUtils.translateException(action, path, e);
+    }
+  }
+
   /**
    * Execute an operation with no result.
    * @param action action to execute (used in error messages)
@@ -137,6 +167,30 @@ public static void once(String action, String path,
         });
   }
 
+
+  /**
+   *
+   * Wait for a future, translating AmazonClientException into an IOException.
+   * @param action action to execute (used in error messages)
+   * @param path path of work (used in error messages)
+   * @param future future to await for
+   * @param <T> type of return value
+   * @return the result of the function call
+   * @throws IOException any IOE raised, or translated exception
+   * @throws RuntimeException any other runtime exception
+   */
+  @Retries.OnceTranslated
+  public static <T> T onceInTheFuture(String action,
+      String path,
+      final Future<T> future)
+      throws IOException {
+    try (DurationInfo ignored = new DurationInfo(LOG, false, "%s", action)) {
+      return FutureIO.awaitFuture(future);
+    } catch (AmazonClientException e) {
+      throw S3AUtils.translateException(action, path, e);
+    }
+  }
+
   /**
    * Execute an operation and ignore all raised IOExceptions; log at INFO;
    * full stack only at DEBUG.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 334fb354b8948..6c39cc4b64240 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -18,15 +18,10 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import javax.annotation.Nullable;
-
-import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 
-import org.apache.hadoop.classification.VisibleForTesting;
-
-import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
@@ -35,12 +30,10 @@
 import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation;
 import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreListFilesIterator;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
 import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.util.functional.RemoteIterators;
@@ -48,30 +41,21 @@
 import org.slf4j.Logger;
 
 import java.io.Closeable;
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.time.Instant;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
-import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.StringJoiner;
 
-import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture;
 import static org.apache.hadoop.fs.s3a.Constants.S3N_FOLDER_SUFFIX;
+import static org.apache.hadoop.fs.s3a.Invoker.onceInTheFuture;
 import static org.apache.hadoop.fs.s3a.S3AUtils.ACCEPT_ALL;
 import static org.apache.hadoop.fs.s3a.S3AUtils.createFileStatus;
 import static org.apache.hadoop.fs.s3a.S3AUtils.maybeAddTrailingSlash;
 import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory;
 import static org.apache.hadoop.fs.s3a.S3AUtils.stringify;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
 import static org.apache.hadoop.fs.s3a.auth.RoleModel.pathToKey;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
@@ -136,30 +120,6 @@ public static RemoteIterator<S3AFileStatus> toProvidedFileStatusIterator(
         Listing.ACCEPT_ALL_BUT_S3N::accept);
   }
 
-  /**
-   * Create a FileStatus iterator against a path, with a given list object
-   * request.
-   *
-   * @param listPath path of the listing
-   * @param request initial request to make
-   * @param filter the filter on which paths to accept
-   * @param acceptor the class/predicate to decide which entries to accept
-   * in the listing based on the full file status.
-   * @param span audit span for this iterator
-   * @return the iterator
-   * @throws IOException IO Problems
-   */
-  @Retries.RetryRaw
-  public FileStatusListingIterator createFileStatusListingIterator(
-      Path listPath,
-      S3ListRequest request,
-      PathFilter filter,
-      Listing.FileStatusAcceptor acceptor,
-      AuditSpan span) throws IOException {
-    return createFileStatusListingIterator(listPath, request, filter, acceptor,
-        null, span);
-  }
-
   /**
    * Create a FileStatus iterator against a path, with a given
    * list object request.
@@ -168,8 +128,6 @@ public FileStatusListingIterator createFileStatusListingIterator(
    * @param filter the filter on which paths to accept
    * @param acceptor the class/predicate to decide which entries to accept
    * in the listing based on the full file status.
-   * @param providedStatus the provided list of file status, which may contain
-   *                       items that are not listed from source.
    * @param span audit span for this iterator
    * @return the iterator
    * @throws IOException IO Problems
@@ -179,14 +137,12 @@ public FileStatusListingIterator createFileStatusListingIterator(
       Path listPath,
       S3ListRequest request,
       PathFilter filter,
-      Listing.FileStatusAcceptor acceptor,
-      RemoteIterator<S3AFileStatus> providedStatus,
+      FileStatusAcceptor acceptor,
       AuditSpan span) throws IOException {
     return new FileStatusListingIterator(
         createObjectListingIterator(listPath, request, span),
         filter,
-        acceptor,
-        providedStatus);
+        acceptor);
   }
 
   /**
@@ -219,28 +175,6 @@ public RemoteIterator<S3ALocatedFileStatus> createLocatedFileStatusIterator(
         listingOperationCallbacks::toLocatedFileStatus);
   }
 
-  /**
-   * Create an located status iterator that wraps another to filter out a set
-   * of recently deleted items.
-   * @param iterator an iterator over the remote located status entries.
-   * @param tombstones set of paths that are recently deleted and should be
-   *                   filtered.
-   * @return a new remote iterator.
-   */
-  @VisibleForTesting
-  RemoteIterator<S3ALocatedFileStatus> createTombstoneReconcilingIterator(
-      RemoteIterator<S3ALocatedFileStatus> iterator,
-      @Nullable Set<Path> tombstones) {
-    if (tombstones == null || tombstones.isEmpty()) {
-      // no need to filter.
-      return iterator;
-    } else {
-      return filteringRemoteIterator(
-          iterator,
-          candidate -> !tombstones.contains(candidate.getPath()));
-    }
-  }
-
   /**
    * Create a remote iterator from a single status entry.
    * @param status status
@@ -256,20 +190,14 @@ public RemoteIterator<S3ALocatedFileStatus> createSingleStatusIterator(
    * @param path input path.
    * @param recursive recursive listing?
    * @param acceptor file status filter
-   * @param collectTombstones should tombstones be collected from S3Guard?
-   * @param forceNonAuthoritativeMS forces metadata store to act like non
-   *                                authoritative. This is useful when
-   *                                listFiles output is used by import tool.
    * @param span audit span for this iterator
    * @return an iterator over listing.
    * @throws IOException any exception.
    */
   public RemoteIterator<S3ALocatedFileStatus> getListFilesAssumingDir(
-          Path path,
-          boolean recursive, Listing.FileStatusAcceptor acceptor,
-          boolean collectTombstones,
-          boolean forceNonAuthoritativeMS,
-          AuditSpan span) throws IOException {
+      Path path,
+      boolean recursive, FileStatusAcceptor acceptor,
+      AuditSpan span) throws IOException {
 
     String key = maybeAddTrailingSlash(pathToKey(path));
     String delimiter = recursive ? null : "/";
@@ -279,82 +207,19 @@ public RemoteIterator<S3ALocatedFileStatus> getListFilesAssumingDir(
       LOG.debug("Requesting all entries under {} with delimiter '{}'",
           key, delimiter);
     }
-    final RemoteIterator<S3AFileStatus> cachedFilesIterator;
-    final Set<Path> tombstones;
-    boolean allowAuthoritative = listingOperationCallbacks
-            .allowAuthoritative(path);
-    if (recursive) {
-      final PathMetadata pm = getStoreContext()
-              .getMetadataStore()
-              .get(path, true);
-      if (pm != null) {
-        if (pm.isDeleted()) {
-          OffsetDateTime deletedAt = OffsetDateTime
-                  .ofInstant(Instant.ofEpochMilli(
-                          pm.getFileStatus().getModificationTime()),
-                          ZoneOffset.UTC);
-          throw new FileNotFoundException("Path " + path + " is recorded as " +
-                  "deleted by S3Guard at " + deletedAt);
-        }
-      }
-      MetadataStoreListFilesIterator metadataStoreListFilesIterator =
-              new MetadataStoreListFilesIterator(
-                      getStoreContext().getMetadataStore(),
-                      pm,
-                      allowAuthoritative);
-      tombstones = metadataStoreListFilesIterator.listTombstones();
-      // if all of the below is true
-      //  - authoritative access is allowed for this metadatastore
-      //  for this directory,
-      //  - all the directory listings are authoritative on the client
-      //  - the caller does not force non-authoritative access
-      // return the listing without any further s3 access
-      if (!forceNonAuthoritativeMS &&
-              allowAuthoritative &&
-              metadataStoreListFilesIterator.isRecursivelyAuthoritative()) {
-        S3AFileStatus[] statuses = S3AUtils.iteratorToStatuses(
-                metadataStoreListFilesIterator, tombstones);
-        cachedFilesIterator = createProvidedFileStatusIterator(
-                statuses, ACCEPT_ALL, acceptor);
-        return createLocatedFileStatusIterator(cachedFilesIterator);
-      }
-      cachedFilesIterator = metadataStoreListFilesIterator;
-    } else {
-      DirListingMetadata meta =
-              S3Guard.listChildrenWithTtl(
-                      getStoreContext().getMetadataStore(),
-                      path,
-                      listingOperationCallbacks.getUpdatedTtlTimeProvider(),
-                      allowAuthoritative);
-      if (meta != null) {
-        tombstones = meta.listTombstones();
-      } else {
-        tombstones = null;
-      }
-      cachedFilesIterator = createProvidedFileStatusIterator(
-              S3Guard.dirMetaToStatuses(meta), ACCEPT_ALL, acceptor);
-      if (allowAuthoritative && meta != null && meta.isAuthoritative()) {
-        // metadata listing is authoritative, so return it directly
-        return createLocatedFileStatusIterator(cachedFilesIterator);
-      }
-    }
-    return createTombstoneReconcilingIterator(
-            createLocatedFileStatusIterator(
-                    createFileStatusListingIterator(path,
-                                    listingOperationCallbacks
-                                    .createListObjectsRequest(key,
-                                        delimiter,
-                                        span),
-                            ACCEPT_ALL,
-                            acceptor,
-                            cachedFilesIterator,
-                            span)),
-            collectTombstones ? tombstones : null);
+    return createLocatedFileStatusIterator(
+        createFileStatusListingIterator(path,
+            listingOperationCallbacks
+                .createListObjectsRequest(key,
+                    delimiter,
+                    span),
+            ACCEPT_ALL,
+            acceptor,
+            span));
   }
 
   /**
    * Generate list located status for a directory.
-   * Also performing tombstone reconciliation for guarded directories.
    * @param dir directory to check.
    * @param filter a path filter.
    * @param span audit span for this iterator
@@ -365,51 +230,14 @@ public RemoteIterator<S3ALocatedFileStatus> getLocatedFileStatusIteratorForDir(
           Path dir, PathFilter filter, AuditSpan span) throws IOException {
     span.activate();
     final String key = maybeAddTrailingSlash(pathToKey(dir));
-    final Listing.FileStatusAcceptor acceptor =
-            new Listing.AcceptAllButSelfAndS3nDirs(dir);
-    boolean allowAuthoritative = listingOperationCallbacks
-            .allowAuthoritative(dir);
-    DirListingMetadata meta =
-            S3Guard.listChildrenWithTtl(getStoreContext().getMetadataStore(),
-                    dir,
-                    listingOperationCallbacks
-                            .getUpdatedTtlTimeProvider(),
-                    allowAuthoritative);
-    if (meta != null) {
-      // there's metadata
-      // convert to an iterator
-      final RemoteIterator<S3AFileStatus> cachedFileStatusIterator =
-          createProvidedFileStatusIterator(
-              S3Guard.dirMetaToStatuses(meta), filter, acceptor);
-
-      // if the dir is authoritative and the data considers itself
-      // to be authorititative.
-      if (allowAuthoritative && meta.isAuthoritative()) {
-        // return the list
-        return createLocatedFileStatusIterator(cachedFileStatusIterator);
-      } else {
-        // merge the datasets
-        return createTombstoneReconcilingIterator(
-            createLocatedFileStatusIterator(
-                createFileStatusListingIterator(dir,
-                    listingOperationCallbacks
-                        .createListObjectsRequest(key, "/", span),
-                    filter,
-                    acceptor,
-                    cachedFileStatusIterator,
-                    span)),
-            meta.listTombstones());
-      }
-    } else {
-      // Unguarded
-      return createLocatedFileStatusIterator(
-          createFileStatusListingIterator(dir,
-              listingOperationCallbacks
-                  .createListObjectsRequest(key, "/", span),
-              filter,
-              acceptor,
-              span));
-    }
+
+    return createLocatedFileStatusIterator(
+        createFileStatusListingIterator(dir,
+            listingOperationCallbacks
+                .createListObjectsRequest(key, "/", span),
+            filter,
+            new AcceptAllButSelfAndS3nDirs(dir),
+            span));
   }
 
   /**
@@ -417,10 +245,11 @@ public RemoteIterator<S3ALocatedFileStatus> getLocatedFileStatusIteratorForDir(
    * to be a non-empty directory.
    * @param path input path.
    * @param span audit span for this iterator
-   * @return Triple of file statuses, metaData, auth flag.
+   * @return iterator of file statuses.
    * @throws IOException Any IO problems.
    */
-  public Triple<RemoteIterator<S3AFileStatus>, DirListingMetadata, Boolean>
+  @Retries.RetryRaw
+  public RemoteIterator<S3AFileStatus>
         getFileStatusesAssumingNonEmptyDir(Path path, final AuditSpan span)
           throws IOException {
     String key = pathToKey(path);
@@ -428,39 +257,16 @@ public RemoteIterator<S3ALocatedFileStatus> getLocatedFileStatusIteratorForDir(
       key = key + '/';
     }
 
-    boolean allowAuthoritative = listingOperationCallbacks
-            .allowAuthoritative(path);
-    DirListingMetadata dirMeta =
-            S3Guard.listChildrenWithTtl(
-                    getStoreContext().getMetadataStore(),
-                    path,
-                    listingOperationCallbacks.getUpdatedTtlTimeProvider(),
-                    allowAuthoritative);
-    // In auth mode return directly with auth flag.
-    if (allowAuthoritative && dirMeta != null && dirMeta.isAuthoritative()) {
-      RemoteIterator<S3AFileStatus> mfsItr = createProvidedFileStatusIterator(
-              S3Guard.dirMetaToStatuses(dirMeta),
-              ACCEPT_ALL,
-              Listing.ACCEPT_ALL_BUT_S3N);
-      return Triple.of(mfsItr,
-              dirMeta, Boolean.TRUE);
-    }
-
     S3ListRequest request = createListObjectsRequest(key, "/", span);
     LOG.debug("listStatus: doing listObjects for directory {}", key);
 
-    FileStatusListingIterator filesItr = createFileStatusListingIterator(
-            path,
-            request,
-            ACCEPT_ALL,
-            new Listing.AcceptAllButSelfAndS3nDirs(path),
-            span);
-
     // return the results obtained from s3.
-    return Triple.of(
-            filesItr,
-            dirMeta,
-            Boolean.FALSE);
+    return createFileStatusListingIterator(
+        path,
+        request,
+        ACCEPT_ALL,
+        new AcceptAllButSelfAndS3nDirs(path),
+        span);
   }
 
   public S3ListRequest createListObjectsRequest(String key,
@@ -529,7 +335,7 @@ interface FileStatusAcceptor {
    * Thread safety: None.
    */
   class FileStatusListingIterator
-      implements RemoteIterator<S3AFileStatus>, IOStatisticsSource {
+      implements RemoteIterator<S3AFileStatus>, IOStatisticsSource, Closeable {
 
     /** Source of objects. */
     private final ObjectListingIterator source;
@@ -542,8 +348,6 @@ class FileStatusListingIterator
     /** Iterator over the current set of results. */
     private ListIterator<S3AFileStatus> statusBatchIterator;
 
-    private final Map<Path, S3AFileStatus> providedStatus;
-    private Iterator<S3AFileStatus> providedStatusIterator;
 
     /**
      * Create an iterator over file status entries.
@@ -551,27 +355,17 @@ class FileStatusListingIterator
      * @param filter the filter on which paths to accept
      * @param acceptor the class/predicate to decide which entries to accept
      * in the listing based on the full file status.
-     * @param providedStatus the provided list of file status, which may contain
-     *                       items that are not listed from source.
      * @throws IOException IO Problems
      */
     @Retries.RetryTranslated
     FileStatusListingIterator(ObjectListingIterator source,
         PathFilter filter,
-        FileStatusAcceptor acceptor,
-        @Nullable RemoteIterator<S3AFileStatus> providedStatus)
+        FileStatusAcceptor acceptor)
         throws IOException {
       this.source = source;
       this.filter = filter;
       this.acceptor = acceptor;
-      this.providedStatus = new HashMap<>();
-      for (; providedStatus != null && providedStatus.hasNext();) {
-        final S3AFileStatus status = providedStatus.next();
-        Path path = status.getPath();
-        if (filter.accept(path) && acceptor.accept(status)) {
-          this.providedStatus.put(path, status);
-        }
-      }
+
       // build the first set of results. This will not trigger any
       // remote IO, assuming the source iterator is in its initial
       // iteration
@@ -586,26 +380,17 @@ class FileStatusListingIterator
      * Lastly, return true if the {@code providedStatusIterator}
      * has left items.
      * @return true if a call to {@link #next()} will succeed.
-     * @throws IOException
+     * @throws IOException IO Problems
      */
     @Override
     @Retries.RetryTranslated
     public boolean hasNext() throws IOException {
-      return sourceHasNext() || providedStatusIterator.hasNext();
+      return sourceHasNext();
     }
 
     @Retries.RetryTranslated
     private boolean sourceHasNext() throws IOException {
-      if (statusBatchIterator.hasNext() || requestNextBatch()) {
-        return true;
-      } else {
-        // turn to file status that are only in provided list
-        if (providedStatusIterator == null) {
-          LOG.debug("Start iterating the provided status.");
-          providedStatusIterator = providedStatus.values().iterator();
-        }
-        return false;
-      }
+      return statusBatchIterator.hasNext() || requestNextBatch();
     }
 
     @Override
@@ -614,29 +399,20 @@ public S3AFileStatus next() throws IOException {
       final S3AFileStatus status;
       if (sourceHasNext()) {
         status = statusBatchIterator.next();
-        // We remove from provided map the file status listed by S3 so that
-        // this does not return duplicate items.
-
-        // The provided status is returned as it is assumed to have the better
-        // metadata (i.e. the eTag and versionId from S3Guard)
-        S3AFileStatus provided = providedStatus.remove(status.getPath());
-        if (provided != null) {
-          LOG.debug(
-              "Removed and returned the status from provided file status {}",
-              status);
-          return provided;
-        }
       } else {
-        if (providedStatusIterator.hasNext()) {
-          status = providedStatusIterator.next();
-          LOG.debug("Returning provided file status {}", status);
-        } else {
-          throw new NoSuchElementException();
-        }
+        throw new NoSuchElementException();
       }
       return status;
     }
 
+    /**
+     * Close, if called, will update
+     * the thread statistics context with the value.
+     */
+    @Override
+    public void close() {
+      source.close();
+    }
     /**
      * Try to retrieve another batch.
      * Note that for the initial batch,
@@ -779,6 +555,11 @@ class ObjectListingIterator implements RemoteIterator<S3ListResult>,
 
     private final AuditSpan span;
 
+    /**
+     * Context statistics aggregator.
+     */
+    private final IOStatisticsAggregator aggregator;
+
     /** The most recent listing results. */
     private S3ListResult objects;
 
@@ -835,6 +616,8 @@ class ObjectListingIterator implements RemoteIterator<S3ListResult>,
       this.span = span;
       this.s3ListResultFuture = listingOperationCallbacks
           .listObjectsAsync(request, iostats, span);
+      this.aggregator = IOStatisticsContext.getCurrentIOStatisticsContext()
+          .getAggregator();
     }
 
     /**
@@ -865,24 +648,20 @@ public S3ListResult next() throws IOException {
         // clear the firstListing flag for future calls.
         firstListing = false;
         // Calculating the result of last async list call.
-        objects = awaitFuture(s3ListResultFuture);
+        objects = onceInTheFuture("listObjects()", listPath.toString(), s3ListResultFuture);
         fetchNextBatchAsyncIfPresent();
       } else {
-        try {
-          if (objectsPrev!= null && !objectsPrev.isTruncated()) {
-            // nothing more to request: fail.
-            throw new NoSuchElementException("No more results in listing of "
-                    + listPath);
-          }
-          // Calculating the result of last async list call.
-          objects = awaitFuture(s3ListResultFuture);
-          // Requesting next batch of results.
-          fetchNextBatchAsyncIfPresent();
-          listingCount++;
-          LOG.debug("New listing status: {}", this);
-        } catch (AmazonClientException e) {
-          throw translateException("listObjects()", listPath, e);
+        if (objectsPrev!= null && !objectsPrev.isTruncated()) {
+          // nothing more to request: fail.
+          throw new NoSuchElementException("No more results in listing of "
+              + listPath);
         }
+        // Calculating the result of last async list call.
+        objects = onceInTheFuture("listObjects()", listPath.toString(), s3ListResultFuture);
+        // Requesting next batch of results.
+        fetchNextBatchAsyncIfPresent();
+        listingCount++;
+        LOG.debug("New listing status: {}", this);
       }
       // Storing the current result to be used by hasNext() call.
       objectsPrev = objects;
@@ -891,9 +670,8 @@ public S3ListResult next() throws IOException {
 
     /**
      * If there are more listings present, call for next batch async.
-     * @throws IOException
      */
-    private void fetchNextBatchAsyncIfPresent() throws IOException {
+    private void fetchNextBatchAsyncIfPresent() {
       if (objects.isTruncated()) {
         LOG.debug("[{}], Requesting next {} objects under {}",
                 listingCount, maxKeys, listPath);
@@ -932,11 +710,12 @@ public int getListingCount() {
     }
 
     /**
-     * Close, if actually called, will close the span
-     * this listing was created with.
+     * Close, if called, will update
+     * the thread statistics context with the value.
      */
     @Override
     public void close() {
+      aggregator.aggregate(getIOStatistics());
     }
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataPersistenceException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataPersistenceException.java
deleted file mode 100644
index e55b7e8a5b188..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/MetadataPersistenceException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.PathIOException;
-
-/**
- * Indicates the metadata associated with the given Path could not be persisted
- * to the metadata store (e.g. S3Guard / DynamoDB).  When this occurs, the
- * file itself has been successfully written to S3, but the metadata may be out
- * of sync.  The metadata can be corrected with the "s3guard import" command
- * provided by {@link org.apache.hadoop.fs.s3a.s3guard.S3GuardTool}.
- */
-public class MetadataPersistenceException extends PathIOException {
-
-  /**
-   * Constructs a MetadataPersistenceException.
-   * @param path path of the affected file
-   * @param cause cause of the issue
-   */
-  public MetadataPersistenceException(String path, Throwable cause) {
-    super(path, cause);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RemoteFileChangedException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RemoteFileChangedException.java
index ce0b9a819b8fb..a3d07df8c4e05 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RemoteFileChangedException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RemoteFileChangedException.java
@@ -25,7 +25,7 @@
 /**
  * Indicates the S3 object is out of sync with the expected version.  Thrown in
  * cases such as when the object is updated while an {@link S3AInputStream} is
- * open, or when a file expected was never found.
+ * open, or when a file to be renamed disappeared during the operation.
  */
 @SuppressWarnings("serial")
 @InterfaceAudience.Public
@@ -36,18 +36,10 @@ public class RemoteFileChangedException extends PathIOException {
       "Constraints of request were unsatisfiable";
 
   /**
-   * While trying to get information on a file known to S3Guard, the
-   * file never became visible in S3.
-   */
-  public static final String FILE_NEVER_FOUND =
-      "File to rename not found on guarded S3 store after repeated attempts";
-
-  /**
-   * The file wasn't found in rename after a single attempt -the unguarded
-   * codepath.
+   * The file disappeaded during a rename between LIST and COPY.
    */
   public static final String FILE_NOT_FOUND_SINGLE_ATTEMPT =
-      "File to rename not found on unguarded S3 store";
+      "File to rename disappeared during the rename operation.";
 
   /**
    * Constructs a RemoteFileChangedException.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
index 78643cc5e0424..ec433fa95c207 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
@@ -33,10 +33,10 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public class S3A extends DelegateToFileSystem{
+public class S3A extends DelegateToFileSystem {
 
   public S3A(URI theUri, Configuration conf)
-          throws IOException, URISyntaxException {
+      throws IOException, URISyntaxException {
     super(theUri, new S3AFileSystem(), conf, "s3a", false);
   }
 
@@ -54,4 +54,13 @@ public String toString() {
     sb.append('}');
     return sb.toString();
   }
+
+  /**
+   * Close the file system; the FileContext API doesn't have an explicit close.
+   */
+  @Override
+  protected void finalize() throws Throwable {
+    fsImpl.close();
+    super.finalize();
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index ddb933d6eb9a7..19943ff2f70da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -40,7 +40,9 @@
 import com.amazonaws.services.s3.model.PutObjectResult;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
@@ -68,6 +70,7 @@
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
 
@@ -103,6 +106,11 @@ class S3ABlockOutputStream extends OutputStream implements
   /** IO Statistics. */
   private final IOStatistics iostatistics;
 
+  /**
+   * The options this instance was created with.
+   */
+  private final BlockOutputStreamBuilder builder;
+
   /** Total bytes for uploads submitted so far. */
   private long bytesSubmitted;
 
@@ -158,6 +166,9 @@ class S3ABlockOutputStream extends OutputStream implements
   /** is client side encryption enabled? */
   private final boolean isCSEEnabled;
 
+  /** Thread level IOStatistics Aggregator. */
+  private final IOStatisticsAggregator threadIOStatisticsAggregator;
+
   /**
    * An S3A output stream which uploads partitions in a separate pool of
    * threads; different {@link S3ADataBlocks.BlockFactory}
@@ -167,6 +178,7 @@ class S3ABlockOutputStream extends OutputStream implements
   S3ABlockOutputStream(BlockOutputStreamBuilder builder)
       throws IOException {
     builder.validate();
+    this.builder = builder;
     this.key = builder.key;
     this.blockFactory = builder.blockFactory;
     this.blockSize = (int) builder.blockSize;
@@ -193,6 +205,7 @@ class S3ABlockOutputStream extends OutputStream implements
       initMultipartUpload();
     }
     this.isCSEEnabled = builder.isCSEEnabled;
+    this.threadIOStatisticsAggregator = builder.ioStatisticsAggregator;
   }
 
   /**
@@ -332,6 +345,7 @@ public synchronized void write(byte[] source, int offset, int len)
    *                     initializing the upload, or if a previous operation
    *                     has failed.
    */
+  @Retries.RetryTranslated
   private synchronized void uploadCurrentBlock(boolean isLast)
       throws IOException {
     Preconditions.checkState(hasActiveBlock(), "No active block");
@@ -353,6 +367,7 @@ private synchronized void uploadCurrentBlock(boolean isLast)
    * can take time and potentially fail.
    * @throws IOException failure to initialize the upload
    */
+  @Retries.RetryTranslated
   private void initMultipartUpload() throws IOException {
     if (multiPartUpload == null) {
       LOG.debug("Initiating Multipart upload");
@@ -444,11 +459,23 @@ public void close() throws IOException {
    */
   private synchronized void cleanupOnClose() {
     cleanupWithLogger(LOG, getActiveBlock(), blockFactory);
+    mergeThreadIOStatistics(statistics.getIOStatistics());
     LOG.debug("Statistics: {}", statistics);
     cleanupWithLogger(LOG, statistics);
     clearActiveBlock();
   }
 
+  /**
+   * Merging the current thread's IOStatistics with the current IOStatistics
+   * context.
+   *
+   * @param streamStatistics Stream statistics to be merged into thread
+   *                         statistics aggregator.
+   */
+  private void mergeThreadIOStatistics(IOStatistics streamStatistics) {
+    getThreadIOStatistics().aggregate(streamStatistics);
+  }
+
   /**
    * Best effort abort of the multipart upload; sets
    * the field to null afterwards.
@@ -546,9 +573,15 @@ private int putObject() throws IOException {
     int size = block.dataSize();
     final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
     final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
-        writeOperationHelper.createPutObjectRequest(key, uploadData.getFile())
-        : writeOperationHelper.createPutObjectRequest(key,
-            uploadData.getUploadStream(), size, null);
+        writeOperationHelper.createPutObjectRequest(
+            key,
+            uploadData.getFile(),
+            builder.putOptions)
+        : writeOperationHelper.createPutObjectRequest(
+            key,
+            uploadData.getUploadStream(),
+            size,
+            builder.putOptions);
     BlockUploadProgress callback =
         new BlockUploadProgress(
             block, progressListener,  now());
@@ -559,7 +592,7 @@ private int putObject() throws IOException {
           try {
             // the putObject call automatically closes the input
             // stream afterwards.
-            return writeOperationHelper.putObject(putObjectRequest);
+            return writeOperationHelper.putObject(putObjectRequest, builder.putOptions);
           } finally {
             cleanupWithLogger(LOG, uploadData, block);
           }
@@ -646,6 +679,10 @@ public boolean hasCapability(String capability) {
     case StreamCapabilities.ABORTABLE_STREAM:
       return true;
 
+      // IOStatistics context support for thread-level IOStatistics.
+    case StreamCapabilities.IOSTATISTICS_CONTEXT:
+      return true;
+
     default:
       return false;
     }
@@ -674,7 +711,7 @@ private void handleSyncableInvocation() {
     }
     // downgrading.
     WARN_ON_SYNCABLE.warn("Application invoked the Syncable API against"
-        + " stream writing to {}. This is unsupported",
+        + " stream writing to {}. This is Unsupported",
         key);
     // and log at debug
     LOG.debug("Downgrading Syncable call", ex);
@@ -685,6 +722,14 @@ public IOStatistics getIOStatistics() {
     return iostatistics;
   }
 
+  /**
+   * Get the IOStatistics aggregator passed in the builder.
+   * @return an aggregator
+   */
+  protected IOStatisticsAggregator getThreadIOStatistics() {
+    return threadIOStatisticsAggregator;
+  }
+
   /**
    * Multiple partition upload.
    */
@@ -702,8 +747,21 @@ private class MultiPartUpload {
      */
     private IOException blockUploadFailure;
 
+    /**
+     * Constructor.
+     * Initiates the MPU request against S3.
+     * @param key upload destination
+     * @throws IOException failure
+     */
+
+    @Retries.RetryTranslated
     MultiPartUpload(String key) throws IOException {
-      this.uploadId = writeOperationHelper.initiateMultiPartUpload(key);
+      this.uploadId = trackDuration(statistics,
+          OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(),
+          () -> writeOperationHelper.initiateMultiPartUpload(
+              key,
+              builder.putOptions));
+
       this.partETagsFutures = new ArrayList<>(2);
       LOG.debug("Initiated multi-part upload for {} with " +
           "id '{}'", writeOperationHelper, uploadId);
@@ -887,7 +945,8 @@ private void complete(List<PartETag> partETags)
                   uploadId,
                   partETags,
                   bytesSubmitted,
-                  errorCount);
+                  errorCount,
+                  builder.putOptions);
             });
       } finally {
         statistics.exceptionInMultipartComplete(errorCount.get());
@@ -1057,6 +1116,16 @@ public static final class BlockOutputStreamBuilder {
     /** is Client side Encryption enabled? */
     private boolean isCSEEnabled;
 
+    /**
+     * Put object options.
+     */
+    private PutObjectOptions putOptions;
+
+    /**
+     * thread-level IOStatistics Aggregator.
+     */
+    private IOStatisticsAggregator ioStatisticsAggregator;
+
     private BlockOutputStreamBuilder() {
     }
 
@@ -1070,8 +1139,10 @@ public void validate() {
       requireNonNull(statistics, "null statistics");
       requireNonNull(writeOperations, "null writeOperationHelper");
       requireNonNull(putTracker, "null putTracker");
+      requireNonNull(putOptions, "null putOptions");
       Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE,
           "Block size is too small: %s", blockSize);
+      requireNonNull(ioStatisticsAggregator, "null ioStatisticsAggregator");
     }
 
     /**
@@ -1182,5 +1253,28 @@ public BlockOutputStreamBuilder withCSEEnabled(boolean value) {
       isCSEEnabled = value;
       return this;
     }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public BlockOutputStreamBuilder withPutOptions(
+        final PutObjectOptions value) {
+      putOptions = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     *
+     * @param value new value
+     * @return the builder
+     */
+    public BlockOutputStreamBuilder withIOStatisticsAggregator(
+        final IOStatisticsAggregator value) {
+      ioStatisticsAggregator = value;
+      return this;
+    }
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 250317706900c..03b5bd96162af 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -32,7 +32,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
index bc6df7aed8146..eda9177cf6bc9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.EtagSource;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 
@@ -30,7 +31,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class S3AFileStatus extends FileStatus {
+public class S3AFileStatus extends FileStatus implements EtagSource {
 
   private static final long serialVersionUID = -5955674081978903922L;
 
@@ -166,8 +167,16 @@ public void setIsEmptyDirectory(Tristate isEmptyDirectory) {
 
   /**
    * @return the S3 object eTag when available, else null.
+   * @deprecated use {@link EtagSource#getEtag()} for
+   * public access.
    */
+  @Deprecated
   public String getETag() {
+    return getEtag();
+  }
+
+  @Override
+  public String getEtag() {
     return eTag;
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b7a522d9c3dd5..3e6f2322d3b00 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -22,24 +22,24 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.file.AccessDeniedException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.time.Instant;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
-import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.Objects;
+import java.util.TreeSet;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -54,6 +54,8 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.Headers;
 import com.amazonaws.services.s3.model.CannedAccessControlList;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.DeleteObjectsResult;
@@ -70,6 +72,9 @@
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.PutObjectResult;
 import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.SelectObjectContentRequest;
+import com.amazonaws.services.s3.model.SelectObjectContentResult;
+import com.amazonaws.services.s3.model.StorageClass;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
 import com.amazonaws.services.s3.transfer.Copy;
@@ -80,35 +85,35 @@
 import com.amazonaws.services.s3.transfer.model.UploadResult;
 import com.amazonaws.event.ProgressListener;
 
-import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
-import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
-import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
-import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonPathCapabilities;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
 import org.apache.hadoop.fs.Globber;
+import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.impl.OpenFileParameters;
 import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
 import org.apache.hadoop.fs.s3a.auth.SignerManager;
 import org.apache.hadoop.fs.s3a.auth.delegation.DelegationOperations;
 import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
 import org.apache.hadoop.fs.s3a.impl.BulkDeleteRetryHandler;
 import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
 import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
+import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation;
 import org.apache.hadoop.fs.s3a.impl.CopyOutcome;
+import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder;
 import org.apache.hadoop.fs.s3a.impl.DeleteOperation;
 import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
 import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl;
@@ -117,16 +122,17 @@
 import org.apache.hadoop.fs.s3a.impl.InternalConstants;
 import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
 import org.apache.hadoop.fs.s3a.impl.MkdirOperation;
-import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport;
+import org.apache.hadoop.fs.s3a.impl.OpenFileSupport;
 import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.impl.RenameOperation;
 import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl;
 import org.apache.hadoop.fs.s3a.impl.S3AMultipartUploaderBuilder;
 import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
 import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.select.InternalSelectConstants;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
+import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
 import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations;
 import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl;
 import org.apache.hadoop.fs.statistics.DurationTracker;
@@ -134,16 +140,20 @@
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 import org.apache.hadoop.fs.store.audit.AuditEntryPoint;
+import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.DelegationTokenIssuer;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.LambdaUtils;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -173,13 +183,10 @@
 import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
 import org.apache.hadoop.fs.s3a.select.SelectBinding;
 import org.apache.hadoop.fs.s3a.select.SelectConstants;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
 import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
 import org.apache.hadoop.fs.s3a.statistics.impl.BondedS3AStatisticsContext;
 import org.apache.hadoop.fs.s3native.S3xLoginHelper;
@@ -198,7 +205,8 @@
 import static java.util.Objects.requireNonNull;
 import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL;
 import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT;
-import static org.apache.hadoop.fs.impl.AbstractFSBuilderImpl.rejectUnknownMandatoryKeys;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.Invoker.*;
@@ -213,15 +221,14 @@
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
-import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions;
+import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE;
+import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE;
 import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound;
 import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_S3GUARD_INCOMPATIBLE;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_S3GUARD_INCOMPATIBLE;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403;
@@ -229,7 +236,7 @@
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.UPLOAD_PART_COUNT_LIMIT;
 import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.fixBucketRegion;
 import static org.apache.hadoop.fs.s3a.impl.NetworkBinding.logDnsLookup;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.dirMetaToStatuses;
+import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.checkNoS3Guard;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
@@ -239,6 +246,7 @@
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation;
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 import static org.apache.hadoop.util.functional.RemoteIterators.typeCastingRemoteIterator;
 
 /**
@@ -274,10 +282,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   // APIs on an uninitialized filesystem.
   private Invoker invoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
       Invoker.LOG_EVENT);
-  // Only used for very specific code paths which behave differently for
-  // S3Guard. Retries FileNotFound, so be careful if you use this.
-  private Invoker s3guardInvoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
-      Invoker.LOG_EVENT);
+
   private final Retried onRetry = this::operationRetried;
 
   /**
@@ -293,6 +298,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   private TransferManager transfers;
   private ExecutorService boundedThreadPool;
   private ThreadPoolExecutor unboundedThreadPool;
+
+  // S3 reads are prefetched asynchronously using this future pool.
+  private ExecutorServiceFuturePool futurePool;
+
+  // If true, the prefetching input stream is used for reads.
+  private boolean prefetchEnabled;
+
+  // Size in bytes of a single prefetch block.
+  private int prefetchBlockSize;
+
+  // Size of prefetch queue (in number of blocks).
+  private int prefetchBlockCount;
+
   private int executorCapacity;
   private long multiPartThreshold;
   public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class);
@@ -300,7 +318,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
       LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress");
   private LocalDirAllocator directoryAllocator;
   private CannedAccessControlList cannedACL;
-  private boolean failOnMetadataWriteError;
 
   /**
    * This must never be null; until initialized it just declares that there
@@ -314,13 +331,24 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   /** Storage Statistics Bonded to the instrumentation. */
   private S3AStorageStatistics storageStatistics;
 
-  private long readAhead;
+  /**
+   * Default input policy; may be overridden in
+   * {@code openFile()}.
+   */
   private S3AInputPolicy inputPolicy;
+  /** Vectored IO context. */
+  private VectoredIOContext vectoredIOContext;
+
+  /**
+   * Maximum number of active range read operation a single
+   * input stream can have.
+   */
+  private int vectoredActiveRangeReads;
+
+  private long readAhead;
   private ChangeDetectionPolicy changeDetectionPolicy;
   private final AtomicBoolean closed = new AtomicBoolean(false);
   private volatile boolean isClosed = false;
-  private MetadataStore metadataStore;
-  private boolean allowAuthoritativeMetadataStore;
   private Collection<String> allowAuthoritativePaths;
 
   /** Delegation token integration; non-empty when DT support is enabled. */
@@ -332,15 +360,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
   private String blockOutputBuffer;
   private S3ADataBlocks.BlockFactory blockFactory;
   private int blockOutputActiveBlocks;
-  private WriteOperationHelper writeHelper;
   private boolean useListV1;
   private MagicCommitIntegration committerIntegration;
 
   private AWSCredentialProviderList credentials;
   private SignerManager signerManager;
 
-  private ITtlTimeProvider ttlTimeProvider;
-
   /**
    * Page size for deletions.
    */
@@ -348,6 +373,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
 
   private final ListingOperationCallbacks listingOperationCallbacks =
           new ListingOperationCallbacksImpl();
+
+  /**
+   * Helper for the openFile() method.
+   */
+  private OpenFileSupport openFileHelper;
+
   /**
    * Directory policy.
    */
@@ -383,6 +414,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
    */
   private ArnResource accessPoint;
 
+  /**
+   * A cache of files that should be deleted when the FileSystem is closed
+   * or the JVM is exited.
+   */
+  private final Set<Path> deleteOnExit = new TreeSet<>();
+
   /** Add any deprecated keys. */
   @SuppressWarnings("deprecation")
   private static void addDeprecatedKeys() {
@@ -476,12 +513,6 @@ public void initialize(URI name, Configuration originalConf)
       workingDir = new Path("/user", username)
           .makeQualified(this.uri, this.getWorkingDirectory());
 
-      s3guardInvoker = new Invoker(new S3GuardExistsRetryPolicy(getConf()),
-          onRetry);
-
-      failOnMetadataWriteError = conf.getBoolean(FAIL_ON_METADATA_WRITE_ERROR,
-          FAIL_ON_METADATA_WRITE_ERROR_DEFAULT);
-
       maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
       partSize = getMultipartSizeProperty(conf,
           MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
@@ -492,8 +523,16 @@ public void initialize(URI name, Configuration originalConf)
       longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1);
       enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
 
-      readAhead = longBytesOption(conf, READAHEAD_RANGE,
-          DEFAULT_READAHEAD_RANGE, 0);
+      this.prefetchEnabled = conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
+      long prefetchBlockSizeLong =
+          longBytesOption(conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE,
+              PREFETCH_BLOCK_DEFAULT_SIZE);
+      if (prefetchBlockSizeLong > (long) Integer.MAX_VALUE) {
+        throw new IOException("S3A prefatch block size exceeds int limit");
+      }
+      this.prefetchBlockSize = (int) prefetchBlockSizeLong;
+      this.prefetchBlockCount =
+          intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1);
 
       initThreadPools(conf);
 
@@ -519,11 +558,6 @@ public void initialize(URI name, Configuration originalConf)
       // requires the audit manager to be initialized.
       requestFactory = createRequestFactory();
 
-      // create the static write operation helper.
-      // this doesn't have a short-lived span; auditors which
-      // require one may reject usages.
-      writeHelper = createWriteOperationHelper(getActiveAuditSpan());
-
       // create an initial span for all other operations.
       span = createSpan(INITIALIZE_SPAN, bucket, null);
 
@@ -540,7 +574,9 @@ public void initialize(URI name, Configuration originalConf)
       doBucketProbing();
 
       inputPolicy = S3AInputPolicy.getPolicy(
-          conf.getTrimmed(INPUT_FADVISE, INPUT_FADV_NORMAL));
+          conf.getTrimmed(INPUT_FADVISE,
+              Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT),
+          S3AInputPolicy.Normal);
       LOG.debug("Input fadvise policy = {}", inputPolicy);
       changeDetectionPolicy = ChangeDetectionPolicy.getPolicy(conf);
       LOG.debug("Change detection policy = {}", changeDetectionPolicy);
@@ -570,32 +606,11 @@ public void initialize(URI name, Configuration originalConf)
       LOG.debug("Using S3ABlockOutputStream with buffer = {}; block={};" +
               " queue limit={}",
           blockOutputBuffer, partSize, blockOutputActiveBlocks);
-      long authDirTtl = conf.getTimeDuration(METADATASTORE_METADATA_TTL,
-          DEFAULT_METADATASTORE_METADATA_TTL, TimeUnit.MILLISECONDS);
-      ttlTimeProvider = new S3Guard.TtlTimeProvider(authDirTtl);
+      // verify there's no S3Guard in the store config.
+      checkNoS3Guard(this.getUri(), getConf());
 
-      setMetadataStore(S3Guard.getMetadataStore(this, ttlTimeProvider));
-      allowAuthoritativeMetadataStore = conf.getBoolean(METADATASTORE_AUTHORITATIVE,
-          DEFAULT_METADATASTORE_AUTHORITATIVE);
       allowAuthoritativePaths = S3Guard.getAuthoritativePaths(this);
 
-      if (hasMetadataStore()) {
-        LOG.debug("Using metadata store {}, authoritative store={}, authoritative path={}",
-            getMetadataStore(), allowAuthoritativeMetadataStore, allowAuthoritativePaths);
-        if (isCSEEnabled) {
-          throw new PathIOException(uri.toString(), CSE_S3GUARD_INCOMPATIBLE);
-        }
-        if (accessPoint != null) {
-          throw new PathIOException(uri.toString(), AP_S3GUARD_INCOMPATIBLE);
-        }
-      }
-
-      // LOG if S3Guard is disabled on the warn level set in config
-      if (!hasMetadataStore()) {
-        String warnLevel = conf.getTrimmed(S3GUARD_DISABLED_WARN_LEVEL,
-            DEFAULT_S3GUARD_DISABLED_WARN_LEVEL);
-        S3Guard.logS3GuardDisabled(LOG, warnLevel, bucket);
-      }
       // directory policy, which may look at authoritative paths
       directoryPolicy = DirectoryPolicyImpl.getDirectoryPolicy(conf,
           this::allowAuthoritative);
@@ -605,7 +620,23 @@ public void initialize(URI name, Configuration originalConf)
 
       pageSize = intOption(getConf(), BULK_DELETE_PAGE_SIZE,
           BULK_DELETE_PAGE_SIZE_DEFAULT, 0);
+      checkArgument(pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE,
+              "page size out of range: %s", pageSize);
       listing = new Listing(listingOperationCallbacks, createStoreContext());
+      // now the open file logic
+      openFileHelper = new OpenFileSupport(
+          changeDetectionPolicy,
+          longBytesOption(conf, READAHEAD_RANGE,
+              DEFAULT_READAHEAD_RANGE, 0),
+          username,
+          intOption(conf, IO_FILE_BUFFER_SIZE_KEY,
+              IO_FILE_BUFFER_SIZE_DEFAULT, 0),
+          longBytesOption(conf, ASYNC_DRAIN_THRESHOLD,
+                        DEFAULT_ASYNC_DRAIN_THRESHOLD, 0),
+          inputPolicy);
+      vectoredActiveRangeReads = intOption(conf,
+              AWS_S3_VECTOR_ACTIVE_RANGE_READS, DEFAULT_AWS_S3_VECTOR_ACTIVE_RANGE_READS, 1);
+      vectoredIOContext = populateVectoredIOContext(conf);
     } catch (AmazonClientException e) {
       // amazon client exception: stop all services then throw the translation
       cleanupWithLogger(LOG, span);
@@ -619,6 +650,23 @@ public void initialize(URI name, Configuration originalConf)
     }
   }
 
+  /**
+   * Populates the configurations related to vectored IO operation
+   * in the context which has to passed down to input streams.
+   * @param conf configuration object.
+   * @return VectoredIOContext.
+   */
+  private VectoredIOContext populateVectoredIOContext(Configuration conf) {
+    final int minSeekVectored = (int) longBytesOption(conf, AWS_S3_VECTOR_READS_MIN_SEEK_SIZE,
+            DEFAULT_AWS_S3_VECTOR_READS_MIN_SEEK_SIZE, 0);
+    final int maxReadSizeVectored = (int) longBytesOption(conf, AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE,
+            DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE, 0);
+    return new VectoredIOContext()
+            .setMinSeekForVectoredReads(minSeekVectored)
+            .setMaxReadSizeForVectoredReads(maxReadSizeVectored)
+            .build();
+  }
+
   /**
    * Set the client side encryption gauge to 0 or 1, indicating if CSE is
    * enabled through the gauge or not.
@@ -718,9 +766,13 @@ private void initThreadPools(Configuration conf) {
         MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS, 1);
     long keepAliveTime = longOption(conf, KEEPALIVE_TIME,
         DEFAULT_KEEPALIVE_TIME, 0);
+    int numPrefetchThreads = this.prefetchEnabled ? this.prefetchBlockCount : 0;
+
+    int activeTasksForBoundedThreadPool = maxThreads;
+    int waitingTasksForBoundedThreadPool = maxThreads + totalTasks + numPrefetchThreads;
     boundedThreadPool = BlockingThreadPoolExecutorService.newInstance(
-        maxThreads,
-        maxThreads + totalTasks,
+        activeTasksForBoundedThreadPool,
+        waitingTasksForBoundedThreadPool,
         keepAliveTime, TimeUnit.SECONDS,
         name + "-bounded");
     unboundedThreadPool = new ThreadPoolExecutor(
@@ -732,6 +784,16 @@ private void initThreadPools(Configuration conf) {
     unboundedThreadPool.allowCoreThreadTimeOut(true);
     executorCapacity = intOption(conf,
         EXECUTOR_CAPACITY, DEFAULT_EXECUTOR_CAPACITY, 1);
+    if (prefetchEnabled) {
+      final S3AInputStreamStatistics s3AInputStreamStatistics =
+          statisticsContext.newInputStreamStatistics();
+      futurePool = new ExecutorServiceFuturePool(
+          new SemaphoredDelegatingExecutor(
+              boundedThreadPool,
+              activeTasksForBoundedThreadPool + waitingTasksForBoundedThreadPool,
+              true,
+              s3AInputStreamStatistics));
+    }
   }
 
   /**
@@ -836,6 +898,7 @@ public Listing getListing() {
    * @param dtEnabled are delegation tokens enabled?
    * @throws IOException failure.
    */
+  @SuppressWarnings("deprecation")
   private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
     Configuration conf = getConf();
     credentials = null;
@@ -848,6 +911,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
       // with it if so.
 
       LOG.debug("Using delegation tokens");
+      V2Migration.v1DelegationTokenCredentialProvidersUsed();
       S3ADelegationTokens tokens = new S3ADelegationTokens();
       this.delegationTokens = Optional.of(tokens);
       tokens.bindToFileSystem(getCanonicalUri(),
@@ -888,10 +952,12 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
     S3ClientFactory.S3ClientCreationParameters parameters = null;
     parameters = new S3ClientFactory.S3ClientCreationParameters()
         .withCredentialSet(credentials)
+        .withPathUri(name)
         .withEndpoint(endpoint)
         .withMetrics(statisticsContext.newStatisticsFromAwsSdk())
         .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))
         .withUserAgentSuffix(uaSuffix)
+        .withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
         .withRequestHandlers(auditManager.createRequestHandlers());
 
     s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
@@ -991,6 +1057,22 @@ protected RequestFactory createRequestFactory() {
     // Any encoding type
     String contentEncoding = getConf().getTrimmed(CONTENT_ENCODING, null);
 
+    String storageClassConf = getConf()
+        .getTrimmed(STORAGE_CLASS, "")
+        .toUpperCase(Locale.US);
+    StorageClass storageClass = null;
+    if (!storageClassConf.isEmpty()) {
+      try {
+        storageClass = StorageClass.fromValue(storageClassConf);
+      } catch (IllegalArgumentException e) {
+        LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
+            STORAGE_CLASS, storageClassConf);
+      }
+    } else {
+      LOG.debug("Unset storage class property {}; falling back to default storage class",
+          STORAGE_CLASS);
+    }
+
     return RequestFactoryImpl.builder()
         .withBucket(requireNonNull(bucket))
         .withCannedACL(getCannedACL())
@@ -998,6 +1080,7 @@ protected RequestFactory createRequestFactory() {
         .withMultipartPartCountLimit(partCountLimit)
         .withRequestPreparer(getAuditManager()::requestCreated)
         .withContentEncoding(contentEncoding)
+        .withStorageClass(storageClass)
         .build();
   }
 
@@ -1156,7 +1239,7 @@ public int getDefaultPort() {
    * This is for internal use within the S3A code itself.
    * @return AmazonS3Client
    */
-  AmazonS3 getAmazonS3Client() {
+  private AmazonS3 getAmazonS3Client() {
     return s3;
   }
 
@@ -1170,6 +1253,7 @@ AmazonS3 getAmazonS3Client() {
   @VisibleForTesting
   public AmazonS3 getAmazonS3ClientForTesting(String reason) {
     LOG.warn("Access to S3A client requested, reason {}", reason);
+    V2Migration.v1S3ClientRequested();
     return s3;
   }
 
@@ -1228,15 +1312,6 @@ public String getBucketLocation(String bucketName) throws IOException {
     return fixBucketRegion(region);
   }
 
-  /**
-   * Returns the read ahead range value used by this filesystem.
-   * @return the readahead range
-   */
-  @VisibleForTesting
-  long getReadAheadRange() {
-    return readAhead;
-  }
-
   /**
    * Get the input policy for this FS instance.
    * @return the input policy
@@ -1318,13 +1393,16 @@ CannedAccessControlList getCannedACL() {
 
   /**
    * Change the input policy for this FS.
+   * This is now a no-op, retained in case some application
+   * or external test invokes it.
+   *
+   * @deprecated use openFile() options
    * @param inputPolicy new policy
    */
   @InterfaceStability.Unstable
+  @Deprecated
   public void setInputPolicy(S3AInputPolicy inputPolicy) {
-    Objects.requireNonNull(inputPolicy, "Null inputStrategy");
-    LOG.debug("Setting input strategy: {}", inputPolicy);
-    this.inputPolicy = inputPolicy;
+    LOG.warn("setInputPolicy is no longer supported");
   }
 
   /**
@@ -1442,64 +1520,61 @@ protected URI canonicalizeUri(URI rawUri) {
   @Retries.RetryTranslated
   public FSDataInputStream open(Path f, int bufferSize)
       throws IOException {
-    return open(f, Optional.empty(), Optional.empty());
+    return executeOpen(qualify(f),
+        openFileHelper.openSimpleFile(bufferSize));
   }
 
   /**
    * Opens an FSDataInputStream at the indicated Path.
-   * if status contains an S3AFileStatus reference, it is used
-   * and so a HEAD request to the store is avoided.
-   *
-   * @param file the file to open
-   * @param options configuration options if opened with the builder API.
-   * @param providedStatus optional file status.
+   * The {@code fileInformation} parameter controls how the file
+   * is opened, whether it is normal vs. an S3 select call,
+   * can a HEAD be skipped, etc.
+   * @param path the file to open
+   * @param fileInformation information about the file to open
    * @throws IOException IO failure.
    */
-  @Retries.RetryTranslated
   @AuditEntryPoint
-  private FSDataInputStream open(
-      final Path file,
-      final Optional<Configuration> options,
-      final Optional<S3AFileStatus> providedStatus)
+  @Retries.RetryTranslated
+  private FSDataInputStream executeOpen(
+      final Path path,
+      final OpenFileSupport.OpenFileInformation fileInformation)
       throws IOException {
-
-    final Path path = qualify(file);
+    // create the input stream statistics before opening
+    // the file so that the time to prepare to open the file is included.
+    S3AInputStreamStatistics inputStreamStats =
+        statisticsContext.newInputStreamStatistics();
     // this span is passed into the stream.
     final AuditSpan auditSpan = entryPoint(INVOCATION_OPEN, path);
-    S3AFileStatus fileStatus = extractOrFetchSimpleFileStatus(path,
-        providedStatus);
-
-    S3AReadOpContext readContext;
-    if (options.isPresent()) {
-      Configuration o = options.get();
-      // normal path. Open the file with the chosen seek policy, if different
-      // from the normal one.
-      // and readahead.
-      S3AInputPolicy policy = S3AInputPolicy.getPolicy(
-          o.get(INPUT_FADVISE, inputPolicy.toString()));
-      long readAheadRange2 = o.getLong(READAHEAD_RANGE, readAhead);
-      // TODO support change detection policy from options?
-      readContext = createReadContext(
-          fileStatus,
-          policy,
-          changeDetectionPolicy,
-          readAheadRange2,
-          auditSpan);
-    } else {
-      readContext = createReadContext(
-          fileStatus,
-          inputPolicy,
-          changeDetectionPolicy,
-          readAhead,
-          auditSpan);
-    }
+    final S3AFileStatus fileStatus =
+        trackDuration(inputStreamStats,
+            ACTION_FILE_OPENED.getSymbol(), () ->
+            extractOrFetchSimpleFileStatus(path, fileInformation));
+    S3AReadOpContext readContext = createReadContext(
+        fileStatus,
+        auditSpan);
+    fileInformation.applyOptions(readContext);
     LOG.debug("Opening '{}'", readContext);
 
-    return new FSDataInputStream(
-        new S3AInputStream(
-            readContext,
-            createObjectAttributes(fileStatus),
-            createInputStreamCallbacks(auditSpan)));
+    if (this.prefetchEnabled) {
+      return new FSDataInputStream(
+          new S3APrefetchingInputStream(
+              readContext.build(),
+              createObjectAttributes(path, fileStatus),
+              createInputStreamCallbacks(auditSpan),
+              inputStreamStats));
+    } else {
+      return new FSDataInputStream(
+          new S3AInputStream(
+              readContext.build(),
+              createObjectAttributes(path, fileStatus),
+              createInputStreamCallbacks(auditSpan),
+                  inputStreamStats,
+                  new SemaphoredDelegatingExecutor(
+                          boundedThreadPool,
+                          vectoredActiveRangeReads,
+                          true,
+                          inputStreamStats)));
+    }
   }
 
   /**
@@ -1553,36 +1628,67 @@ public S3Object getObject(GetObjectRequest request) {
         return s3.getObject(request);
       }
     }
+
+    @Override
+    public <T> CompletableFuture<T> submit(final CallableRaisingIOE<T> operation) {
+      CompletableFuture<T> result = new CompletableFuture<>();
+      unboundedThreadPool.submit(() ->
+          LambdaUtils.eval(result, () -> {
+            LOG.debug("Starting submitted operation in {}", auditSpan.getSpanId());
+            try (AuditSpan span = auditSpan.activate()) {
+              return operation.apply();
+            } finally {
+              LOG.debug("Completed submitted operation in {}", auditSpan.getSpanId());
+            }
+          }));
+      return result;
+    }
   }
 
+  /**
+   * Callbacks for WriteOperationHelper.
+   */
+  private final class WriteOperationHelperCallbacksImpl
+      implements WriteOperationHelper.WriteOperationHelperCallbacks {
+
+    @Override
+    public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
+      return s3.selectObjectContent(request);
+    }
+
+    @Override
+    public CompleteMultipartUploadResult completeMultipartUpload(
+        CompleteMultipartUploadRequest request) {
+      return s3.completeMultipartUpload(request);
+    }
+  }
+
+
   /**
    * Create the read context for reading from the referenced file,
    * using FS state as well as the status.
    * @param fileStatus file status.
-   * @param seekPolicy input policy for this operation
-   * @param changePolicy change policy for this operation.
-   * @param readAheadRange readahead value.
    * @param auditSpan audit span.
    * @return a context for read and select operations.
    */
   @VisibleForTesting
   protected S3AReadOpContext createReadContext(
       final FileStatus fileStatus,
-      final S3AInputPolicy seekPolicy,
-      final ChangeDetectionPolicy changePolicy,
-      final long readAheadRange,
       final AuditSpan auditSpan) {
-    return new S3AReadOpContext(fileStatus.getPath(),
-        hasMetadataStore(),
+    final S3AReadOpContext roc = new S3AReadOpContext(
+        fileStatus.getPath(),
         invoker,
-        s3guardInvoker,
         statistics,
         statisticsContext,
         fileStatus,
-        seekPolicy,
-        changePolicy,
-        readAheadRange,
-        auditSpan);
+        vectoredIOContext,
+        IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator(),
+        futurePool,
+        prefetchBlockSize,
+        prefetchBlockCount)
+        .withAuditSpan(auditSpan);
+    openFileHelper.applyDefaultOptions(roc);
+    return roc.build();
   }
 
   /**
@@ -1610,14 +1716,16 @@ private S3ObjectAttributes createObjectAttributes(
 
   /**
    * Create the attributes of an object for subsequent use.
+   * @param path path -this is used over the file status path.
    * @param fileStatus file status to build from.
    * @return attributes to use when building the query.
    */
   private S3ObjectAttributes createObjectAttributes(
+      final Path path,
       final S3AFileStatus fileStatus) {
     return createObjectAttributes(
-        fileStatus.getPath(),
-        fileStatus.getETag(),
+        path,
+        fileStatus.getEtag(),
         fileStatus.getVersionId(),
         fileStatus.getLen());
   }
@@ -1645,10 +1753,15 @@ public FSDataOutputStream create(Path f, FsPermission permission,
       boolean overwrite, int bufferSize, short replication, long blockSize,
       Progressable progress) throws IOException {
     final Path path = qualify(f);
+
     // the span will be picked up inside the output stream
     return trackDurationAndSpan(INVOCATION_CREATE, path, () ->
-        innerCreateFile(path, permission, overwrite, bufferSize, replication,
-            blockSize, progress));
+        innerCreateFile(path,
+            progress,
+            getActiveAuditSpan(),
+            overwrite
+                ? OPTIONS_CREATE_FILE_OVERWRITE
+                : OPTIONS_CREATE_FILE_NO_OVERWRITE));
   }
 
   /**
@@ -1656,58 +1769,68 @@ public FSDataOutputStream create(Path f, FsPermission permission,
    * reporting; in the active span.
    * Retry policy: retrying, translated on the getFileStatus() probe.
    * No data is uploaded to S3 in this call, so no retry issues related to that.
+   * The "performance" flag disables safety checks for the path being a file,
+   * parent directory existing, and doesn't attempt to delete
+   * dir markers, irrespective of FS settings.
+   * If true, this method call does no IO at all.
    * @param path the file name to open
-   * @param permission the permission to set.
-   * @param overwrite if a file with this name already exists, then if true,
-   *   the file will be overwritten, and if false an error will be thrown.
-   * @param bufferSize the size of the buffer to be used.
-   * @param replication required block replication for the file.
-   * @param blockSize the requested block size.
    * @param progress the progress reporter.
+   * @param auditSpan audit span
+   * @param options options for the file
    * @throws IOException in the event of IO related errors.
-   * @see #setPermission(Path, FsPermission)
    */
   @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
   @Retries.RetryTranslated
-  private FSDataOutputStream innerCreateFile(Path path,
-      FsPermission permission,
-      boolean overwrite,
-      int bufferSize,
-      short replication,
-      long blockSize,
-      Progressable progress) throws IOException {
+  private FSDataOutputStream innerCreateFile(
+      final Path path,
+      final Progressable progress,
+      final AuditSpan auditSpan,
+      final CreateFileBuilder.CreateFileOptions options) throws IOException {
+    auditSpan.activate();
     String key = pathToKey(path);
-    FileStatus status = null;
-    try {
-      // get the status or throw an FNFE.
-      // when overwriting, there is no need to look for any existing file,
-      // and attempting to do so can poison the load balancers with 404
-      // entries.
-      status = innerGetFileStatus(path, false,
-          overwrite
-              ? StatusProbeEnum.DIRECTORIES
-              : StatusProbeEnum.ALL);
-
-      // if the thread reaches here, there is something at the path
-      if (status.isDirectory()) {
-        // path references a directory: automatic error
-        throw new FileAlreadyExistsException(path + " is a directory");
-      }
-      if (!overwrite) {
-        // path references a file and overwrite is disabled
-        throw new FileAlreadyExistsException(path + " already exists");
+    EnumSet<CreateFlag> flags = options.getFlags();
+    boolean overwrite = flags.contains(CreateFlag.OVERWRITE);
+    boolean performance = options.isPerformance();
+    boolean skipProbes = performance || isUnderMagicCommitPath(path);
+    if (skipProbes) {
+      LOG.debug("Skipping existence/overwrite checks");
+    } else {
+      try {
+        // get the status or throw an FNFE.
+        // when overwriting, there is no need to look for any existing file,
+        // just a directory (for safety)
+        FileStatus status = innerGetFileStatus(path, false,
+            overwrite
+                ? StatusProbeEnum.DIRECTORIES
+                : StatusProbeEnum.ALL);
+
+        // if the thread reaches here, there is something at the path
+        if (status.isDirectory()) {
+          // path references a directory: automatic error
+          throw new FileAlreadyExistsException(path + " is a directory");
+        }
+        if (!overwrite) {
+          // path references a file and overwrite is disabled
+          throw new FileAlreadyExistsException(path + " already exists");
+        }
+        LOG.debug("Overwriting file {}", path);
+      } catch (FileNotFoundException e) {
+        // this means there is nothing at the path; all good.
       }
-      LOG.debug("Overwriting file {}", path);
-    } catch (FileNotFoundException e) {
-      // this means the file is not found
-
     }
     instrumentation.fileCreated();
-    PutTracker putTracker =
-        committerIntegration.createTracker(path, key);
-    String destKey = putTracker.getDestKey();
     final BlockOutputStreamStatistics outputStreamStatistics
         = statisticsContext.newOutputStreamStatistics();
+    PutTracker putTracker =
+        committerIntegration.createTracker(path, key, outputStreamStatistics);
+    String destKey = putTracker.getDestKey();
+
+    // put options are derived from the path and the
+    // option builder.
+    boolean keep = performance || keepDirectoryMarkers(path);
+    final PutObjectOptions putOptions =
+        new PutObjectOptions(keep, null, options.getHeaders());
+
     final S3ABlockOutputStream.BlockOutputStreamBuilder builder =
         S3ABlockOutputStream.builder()
         .withKey(destKey)
@@ -1717,7 +1840,7 @@ private FSDataOutputStream innerCreateFile(Path path,
         .withProgress(progress)
         .withPutTracker(putTracker)
         .withWriteOperations(
-            createWriteOperationHelper(getActiveAuditSpan()))
+            createWriteOperationHelper(auditSpan))
         .withExecutorService(
             new SemaphoredDelegatingExecutor(
                 boundedThreadPool,
@@ -1728,12 +1851,14 @@ private FSDataOutputStream innerCreateFile(Path path,
             getConf().getBoolean(
                 DOWNGRADE_SYNCABLE_EXCEPTIONS,
                 DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT))
-        .withCSEEnabled(isCSEEnabled);
+        .withCSEEnabled(isCSEEnabled)
+        .withPutOptions(putOptions)
+        .withIOStatisticsAggregator(
+            IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator());
     return new FSDataOutputStream(
         new S3ABlockOutputStream(builder),
         null);
   }
-
   /**
    * Create a Write Operation Helper with the current active span.
    * All operations made through this helper will activate the
@@ -1763,13 +1888,69 @@ public WriteOperationHelper createWriteOperationHelper(AuditSpan auditSpan) {
         getConf(),
         statisticsContext,
         getAuditSpanSource(),
-        auditSpan);
+        auditSpan,
+        new WriteOperationHelperCallbacksImpl());
+  }
+
+  /**
+   * Create instance of an FSDataOutputStreamBuilder for
+   * creating a file at the given path.
+   * @param path path to create
+   * @return a builder.
+   * @throws UncheckedIOException for problems creating the audit span
+   */
+  @Override
+  @AuditEntryPoint
+  public FSDataOutputStreamBuilder createFile(final Path path) {
+    try {
+      final Path qualified = qualify(path);
+      final AuditSpan span = entryPoint(INVOCATION_CREATE_FILE,
+          pathToKey(qualified),
+          null);
+      return new CreateFileBuilder(this,
+          qualified,
+          new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span))
+            .create()
+            .overwrite(true);
+    } catch (IOException e) {
+      // catch any IOEs raised in span creation and convert to
+      // an UncheckedIOException
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  /**
+   * Callback for create file operations.
+   */
+  private final class CreateFileBuilderCallbacksImpl implements
+      CreateFileBuilder.CreateFileBuilderCallbacks {
+
+    private final Statistic statistic;
+    /** span for operations. */
+    private final AuditSpan span;
+
+    private CreateFileBuilderCallbacksImpl(
+        final Statistic statistic,
+        final AuditSpan span) {
+      this.statistic = statistic;
+      this.span = span;
+    }
+
+    @Override
+    public FSDataOutputStream createFileFromBuilder(
+        final Path path,
+        final Progressable progress,
+        final CreateFileBuilder.CreateFileOptions options) throws IOException {
+      // the span will be picked up inside the output stream
+      return trackDuration(getDurationTrackerFactory(), statistic.getSymbol(), () ->
+          innerCreateFile(path, progress, span, options));
+    }
   }
 
   /**
    * {@inheritDoc}
-   * @throws FileNotFoundException if the parent directory is not present -or
-   * is not a directory.
+   * The S3A implementations downgrades to the recursive creation, to avoid
+   * any race conditions with parent entries "disappearing".
    */
   @Override
   @AuditEntryPoint
@@ -1781,30 +1962,23 @@ public FSDataOutputStream createNonRecursive(Path p,
       long blockSize,
       Progressable progress) throws IOException {
     final Path path = makeQualified(p);
-    // this span is passed into the stream.
-    try (AuditSpan span = entryPoint(INVOCATION_CREATE_NON_RECURSIVE, path)) {
-      Path parent = path.getParent();
-      // expect this to raise an exception if there is no parent dir
-      if (parent != null && !parent.isRoot()) {
-        S3AFileStatus status;
-        try {
-          // optimize for the directory existing: Call list first
-          status = innerGetFileStatus(parent, false,
-              StatusProbeEnum.DIRECTORIES);
-        } catch (FileNotFoundException e) {
-          // no dir, fall back to looking for a file
-          // (failure condition if true)
-          status = innerGetFileStatus(parent, false,
-              StatusProbeEnum.HEAD_ONLY);
-        }
-        if (!status.isDirectory()) {
-          throw new FileAlreadyExistsException("Not a directory: " + parent);
-        }
-      }
-      return innerCreateFile(path, permission,
-          flags.contains(CreateFlag.OVERWRITE), bufferSize,
-          replication, blockSize, progress);
+
+    // span is created and passed in to the callbacks.
+    final AuditSpan span = entryPoint(INVOCATION_CREATE_NON_RECURSIVE,
+        pathToKey(path),
+        null);
+    // uses the CreateFileBuilder, filling it in with the relevant arguments.
+    final CreateFileBuilder builder = new CreateFileBuilder(this,
+        path,
+        new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_NON_RECURSIVE, span))
+        .create()
+        .withFlags(flags)
+        .blockSize(blockSize)
+        .bufferSize(bufferSize);
+    if (progress != null) {
+      builder.progress(progress);
     }
+    return builder.build();
   }
 
   /**
@@ -1941,18 +2115,15 @@ private Pair<S3AFileStatus, S3AFileStatus> initiateRename(
           // threads.
           S3AFileStatus dstParentStatus = innerGetFileStatus(parent,
               false, StatusProbeEnum.FILE);
-          // if this doesn't raise an exception then it's one of
-          // raw S3: parent is a file: error
-          // guarded S3: parent is a file or a dir.
+          // if this doesn't raise an exception then
+          // the parent is a file or a dir.
           if (!dstParentStatus.isDirectory()) {
             throw new RenameFailedException(src, dst,
                 "destination parent is not a directory");
           }
         } catch (FileNotFoundException expected) {
           // nothing was found. Don't worry about it;
-          // expect rename to implicitly create the parent dir (raw S3)
-          // or the s3guard parents (guarded)
-
+          // expect rename to implicitly create the parent dir
         }
       }
     }
@@ -1994,7 +2165,6 @@ private long innerRename(Path source, Path dest)
 
     // Initiate the rename.
     // this will call back into this class via the rename callbacks
-    // and interact directly with any metastore.
     RenameOperation renameOperation = new RenameOperation(
         createStoreContext(),
         src, srcKey, p.getLeft(),
@@ -2037,14 +2207,14 @@ public S3ObjectAttributes createObjectAttributes(final Path path,
     @Override
     public S3ObjectAttributes createObjectAttributes(
         final S3AFileStatus fileStatus) {
-      return S3AFileSystem.this.createObjectAttributes(fileStatus);
+      return S3AFileSystem.this.createObjectAttributes(
+          fileStatus.getPath(),
+          fileStatus);
     }
 
     @Override
     public S3AReadOpContext createReadContext(final FileStatus fileStatus) {
       return S3AFileSystem.this.createReadContext(fileStatus,
-          inputPolicy,
-          changeDetectionPolicy, readAhead,
           auditSpan);
     }
 
@@ -2052,13 +2222,11 @@ public S3AReadOpContext createReadContext(final FileStatus fileStatus) {
     @Retries.RetryTranslated
     public void deleteObjectAtPath(final Path path,
         final String key,
-        final boolean isFile,
-        final BulkOperationState operationState)
+        final boolean isFile)
         throws IOException {
       auditSpan.activate();
       once("delete", path.toString(), () ->
-          S3AFileSystem.this.deleteObjectAtPath(path, key, isFile,
-              operationState));
+          S3AFileSystem.this.deleteObjectAtPath(path, key, isFile));
     }
 
     @Override
@@ -2066,7 +2234,6 @@ public void deleteObjectAtPath(final Path path,
     public RemoteIterator<S3ALocatedFileStatus> listFilesAndDirectoryMarkers(
         final Path path,
         final S3AFileStatus status,
-        final boolean collectTombstones,
         final boolean includeSelf) throws IOException {
       auditSpan.activate();
       return innerListFiles(
@@ -2075,9 +2242,8 @@ public RemoteIterator<S3ALocatedFileStatus> listFilesAndDirectoryMarkers(
           includeSelf
               ? Listing.ACCEPT_ALL_BUT_S3N
               : new Listing.AcceptAllButSelfAndS3nDirs(path),
-          status,
-          collectTombstones,
-          true);
+          status
+      );
     }
 
     @Override
@@ -2091,16 +2257,12 @@ public CopyResult copyFile(final String srcKey,
     }
 
     @Override
-    public DeleteObjectsResult removeKeys(
-        final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-        final boolean deleteFakeDir,
-        final List<Path> undeletedObjectsOnFailure,
-        final BulkOperationState operationState,
-        final boolean quiet)
+    public void removeKeys(
+            final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+            final boolean deleteFakeDir)
         throws MultiObjectDeleteException, AmazonClientException, IOException {
       auditSpan.activate();
-      return S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir,
-          undeletedObjectsOnFailure, operationState, quiet);
+      S3AFileSystem.this.removeKeys(keysToDelete, deleteFakeDir);
     }
 
     @Override
@@ -2111,17 +2273,12 @@ public void finishRename(final Path sourceRenamed, final Path destCreated)
       if (!sourceRenamed.getParent().equals(destParent)) {
         LOG.debug("source & dest parents are different; fix up dir markers");
         if (!keepDirectoryMarkers(destParent)) {
-          deleteUnnecessaryFakeDirectories(destParent, null);
+          deleteUnnecessaryFakeDirectories(destParent);
         }
         maybeCreateFakeParentDirectory(sourceRenamed);
       }
     }
 
-    @Override
-    public boolean allowAuthoritative(final Path p) {
-      return S3AFileSystem.this.allowAuthoritative(p);
-    }
-
     @Override
     @Retries.RetryTranslated
     public RemoteIterator<S3AFileStatus> listObjects(
@@ -2133,7 +2290,6 @@ public RemoteIterator<S3AFileStatus> listObjects(
               createListObjectsRequest(key, null),
               ACCEPT_ALL,
               Listing.ACCEPT_ALL_BUT_S3N,
-              null,
               auditSpan));
     }
   }
@@ -2148,12 +2304,10 @@ protected class ListingOperationCallbacksImpl implements
           ListingOperationCallbacks {
 
     @Override
-    @Retries.RetryRaw
     public CompletableFuture<S3ListResult> listObjectsAsync(
         S3ListRequest request,
         DurationTrackerFactory trackerFactory,
-        AuditSpan span)
-            throws IOException {
+        AuditSpan span) {
       return submit(unboundedThreadPool, span, () ->
           listObjects(request,
               pairedTrackerFactory(trackerFactory,
@@ -2166,8 +2320,7 @@ public CompletableFuture<S3ListResult> continueListObjectsAsync(
         S3ListRequest request,
         S3ListResult prevResult,
         DurationTrackerFactory trackerFactory,
-        AuditSpan span)
-            throws IOException {
+        AuditSpan span) {
       return submit(unboundedThreadPool, span,
           () -> continueListObjects(request, prevResult,
               pairedTrackerFactory(trackerFactory,
@@ -2200,15 +2353,6 @@ public int getMaxKeys() {
       return S3AFileSystem.this.getMaxKeys();
     }
 
-    @Override
-    public ITtlTimeProvider getUpdatedTtlTimeProvider() {
-      return S3AFileSystem.this.ttlTimeProvider;
-    }
-
-    @Override
-    public boolean allowAuthoritative(final Path p) {
-      return S3AFileSystem.this.allowAuthoritative(p);
-    }
   }
 
   /**
@@ -2226,6 +2370,7 @@ public boolean allowAuthoritative(final Path p) {
   @Retries.RetryTranslated
   @InterfaceStability.Evolving
   public ObjectMetadata getObjectMetadata(Path path) throws IOException {
+    V2Migration.v1GetObjectMetadataCalled();
     return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
         getObjectMetadata(makeQualified(path), null, invoker,
             "getObjectMetadata"));
@@ -2245,49 +2390,12 @@ private ObjectMetadata getObjectMetadata(Path path,
       ChangeTracker changeTracker, Invoker changeInvoker, String operation)
       throws IOException {
     String key = pathToKey(path);
-    return once(operation, path.toString(),
-        () ->
-            // this always does a full HEAD to the object
+    return once(operation, path.toString(), () ->
+            // HEAD against the object
             getObjectMetadata(
                 key, changeTracker, changeInvoker, operation));
   }
 
-  /**
-   * Does this Filesystem have a metadata store?
-   * @return true iff the FS has been instantiated with a metadata store
-   */
-  public boolean hasMetadataStore() {
-    return !S3Guard.isNullMetadataStore(metadataStore);
-  }
-
-  /**
-   * Does the filesystem have an authoritative metadata store?
-   * @return true if there is a metadata store and the authoritative flag
-   * is set for this filesystem.
-   */
-  @VisibleForTesting
-  public boolean hasAuthoritativeMetadataStore() {
-    return hasMetadataStore() && allowAuthoritativeMetadataStore;
-  }
-
-  /**
-   * Get the metadata store.
-   * This will always be non-null, but may be bound to the
-   * {@code NullMetadataStore}.
-   * @return the metadata store of this FS instance
-   */
-  @VisibleForTesting
-  public MetadataStore getMetadataStore() {
-    return metadataStore;
-  }
-
-  /** For testing only.  See ITestS3GuardEmptyDirs. */
-  @VisibleForTesting
-  void setMetadataStore(MetadataStore ms) {
-    Preconditions.checkNotNull(ms);
-    metadataStore = ms;
-  }
-
   /**
    * Entry point to an operation.
    * Increments the statistic; verifies the FS is active.
@@ -2408,7 +2516,9 @@ protected void incrementGauge(Statistic statistic, long count) {
    */
   public void operationRetried(Exception ex) {
     if (isThrottleException(ex)) {
-      operationThrottled(false);
+      LOG.debug("Request throttled");
+      incrementStatistic(STORE_IO_THROTTLED);
+      statisticsContext.addValueToQuantiles(STORE_IO_THROTTLE_RATE, 1);
     } else {
       incrementStatistic(STORE_IO_RETRY);
       incrementStatistic(IGNORED_ERRORS);
@@ -2430,46 +2540,6 @@ public void operationRetried(
     operationRetried(ex);
   }
 
-  /**
-   * Callback from {@link Invoker} when an operation against a metastore
-   * is retried.
-   * Always increments the {@link Statistic#S3GUARD_METADATASTORE_RETRY}
-   * statistic/counter;
-   * if it is a throttling exception will update the associated
-   * throttled metrics/statistics.
-   *
-   * @param ex exception
-   * @param retries number of retries
-   * @param idempotent is the method idempotent
-   */
-  public void metastoreOperationRetried(Exception ex,
-      int retries,
-      boolean idempotent) {
-    incrementStatistic(S3GUARD_METADATASTORE_RETRY);
-    if (isThrottleException(ex)) {
-      operationThrottled(true);
-    } else {
-      incrementStatistic(IGNORED_ERRORS);
-    }
-  }
-
-  /**
-   * Note that an operation was throttled -this will update
-   * specific counters/metrics.
-   * @param metastore was the throttling observed in the S3Guard metastore?
-   */
-  private void operationThrottled(boolean metastore) {
-    LOG.debug("Request throttled on {}", metastore ? "S3": "DynamoDB");
-    if (metastore) {
-      incrementStatistic(S3GUARD_METADATASTORE_THROTTLED);
-      statisticsContext.addValueToQuantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
-          1);
-    } else {
-      incrementStatistic(STORE_IO_THROTTLED);
-      statisticsContext.addValueToQuantiles(STORE_IO_THROTTLE_RATE, 1);
-    }
-  }
-
   /**
    * Get the storage statistics of this filesystem.
    * @return the storage statistics
@@ -2671,8 +2741,7 @@ public void incrementWriteOperations() {
   }
 
   /**
-   * Delete an object. This is the low-level internal call which
-   * <i>does not</i> update the metastore.
+   * Delete an object.
    * Increments the {@code OBJECT_DELETE_REQUESTS} and write
    * operation statistics.
    * This call does <i>not</i> create any mock parent entries.
@@ -2706,21 +2775,19 @@ protected void deleteObject(String key)
   }
 
   /**
-   * Delete an object, also updating the metastore.
+   * Delete an object.
    * This call does <i>not</i> create any mock parent entries.
    * Retry policy: retry untranslated; delete considered idempotent.
    * @param f path path to delete
    * @param key key of entry
    * @param isFile is the path a file (used for instrumentation only)
-   * @param operationState (nullable) operational state for a bulk update
    * @throws AmazonClientException problems working with S3
-   * @throws IOException IO failure in the metastore
+   * @throws IOException from invoker signature only -should not be raised.
    */
-  @Retries.RetryMixed
+  @Retries.RetryRaw
   void deleteObjectAtPath(Path f,
       String key,
-      boolean isFile,
-      @Nullable final BulkOperationState operationState)
+      boolean isFile)
       throws AmazonClientException, IOException {
     if (isFile) {
       instrumentation.fileDeleted(1);
@@ -2728,7 +2795,6 @@ void deleteObjectAtPath(Path f,
       instrumentation.directoryDeleted();
     }
     deleteObject(key);
-    metadataStore.delete(f, operationState);
   }
 
   /**
@@ -2745,8 +2811,7 @@ private void blockRootDelete(String key) throws InvalidRequestException {
   }
 
   /**
-   * Perform a bulk object delete operation against S3; leaves S3Guard
-   * alone.
+   * Perform a bulk object delete operation against S3.
    * Increments the {@code OBJECT_DELETE_REQUESTS} and write
    * operation statistics
    * <p></p>
@@ -2812,7 +2877,7 @@ private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest)
    */
   public PutObjectRequest newPutObjectRequest(String key,
       ObjectMetadata metadata, File srcfile) {
-    return requestFactory.newPutObjectRequest(key, metadata, srcfile);
+    return requestFactory.newPutObjectRequest(key, metadata, null, srcfile);
   }
 
   /**
@@ -2862,16 +2927,15 @@ public UploadInfo putObject(PutObjectRequest putObjectRequest) {
    * Auditing: must be inside an audit span.
    * <i>Important: this call will close any input stream in the request.</i>
    * @param putObjectRequest the request
+   * @param putOptions put object options
    * @return the upload initiated
    * @throws AmazonClientException on problems
-   * @throws MetadataPersistenceException if metadata about the write could
-   * not be saved to the metadata store and
-   * fs.s3a.metadatastore.fail.on.write.error=true
    */
   @VisibleForTesting
-  @Retries.OnceRaw("For PUT; post-PUT actions are RetryTranslated")
-  PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest)
-      throws AmazonClientException, MetadataPersistenceException {
+  @Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed")
+  PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest,
+      PutObjectOptions putOptions)
+      throws AmazonClientException {
     long len = getPutRequestLength(putObjectRequest);
     LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey());
     incrementPutStartStatistics(len);
@@ -2881,9 +2945,10 @@ PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest)
           OBJECT_PUT_REQUESTS.getSymbol(), () ->
               s3.putObject(putObjectRequest));
       incrementPutCompletedStatistics(true, len);
-      // update metadata
+      // apply any post-write actions.
       finishedWrite(putObjectRequest.getKey(), len,
-          result.getETag(), result.getVersionId(), null);
+          result.getETag(), result.getVersionId(),
+          putOptions);
       return result;
     } catch (SdkBaseException e) {
       incrementPutCompletedStatistics(false, len);
@@ -2982,15 +3047,10 @@ public void incrementPutProgressStatistics(String key, long bytes) {
 
   /**
    * Delete a list of keys on a s3-backend.
-   * This does <i>not</i> update the metastore.
    * Retry policy: retry untranslated; delete considered idempotent.
    * @param keysToDelete collection of keys to delete on the s3-backend.
    *        if empty, no request is made of the object store.
    * @param deleteFakeDir indicates whether this is for deleting fake dirs
-   * @param quiet should a bulk query be quiet, or should its result list
-   * all deleted keys?
-   * @return the deletion result if a multi object delete was invoked
-   * and it returned without a failure.
    * @throws InvalidRequestException if the request was rejected due to
    * a mistaken attempt to delete the root directory.
    * @throws MultiObjectDeleteException one or more of the keys could not
@@ -3000,10 +3060,9 @@ public void incrementPutProgressStatistics(String key, long bytes) {
    * @throws AmazonClientException other amazon-layer failure.
    */
   @Retries.RetryRaw
-  private DeleteObjectsResult removeKeysS3(
-      List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean deleteFakeDir,
-      boolean quiet)
+  private void removeKeysS3(
+          List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+          boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException,
       IOException {
     if (LOG.isDebugEnabled()) {
@@ -3014,18 +3073,30 @@ private DeleteObjectsResult removeKeysS3(
             key.getVersion() != null ? key.getVersion() : "");
       }
     }
-    DeleteObjectsResult result = null;
     if (keysToDelete.isEmpty()) {
       // exit fast if there are no keys to delete
-      return result;
+      return;
     }
     for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
       blockRootDelete(keyVersion.getKey());
     }
     try {
       if (enableMultiObjectsDelete) {
-        result = deleteObjects(
-            getRequestFactory().newBulkDeleteRequest(keysToDelete, quiet));
+        if (keysToDelete.size() <= pageSize) {
+          deleteObjects(getRequestFactory()
+                  .newBulkDeleteRequest(keysToDelete));
+        } else {
+          // Multi object deletion of more than 1000 keys is not supported
+          // by s3. So we are paging the keys by page size.
+          LOG.debug("Partitioning the keys to delete as it is more than " +
+                  "page size. Number of keys: {}, Page size: {}",
+                  keysToDelete.size(), pageSize);
+          for (List<DeleteObjectsRequest.KeyVersion> batchOfKeysToDelete :
+                  Lists.partition(keysToDelete, pageSize)) {
+            deleteObjects(getRequestFactory()
+                    .newBulkDeleteRequest(batchOfKeysToDelete));
+          }
+        }
       } else {
         for (DeleteObjectsRequest.KeyVersion keyVersion : keysToDelete) {
           deleteObject(keyVersion.getKey());
@@ -3041,7 +3112,6 @@ private DeleteObjectsResult removeKeysS3(
       throw ex;
     }
     noteDeleted(keysToDelete.size(), deleteFakeDir);
-    return result;
   }
 
   /**
@@ -3058,13 +3128,13 @@ private void noteDeleted(final int count, final boolean deleteFakeDir) {
   }
 
   /**
-   * Invoke {@link #removeKeysS3(List, boolean, boolean)} with handling of
-   * {@code MultiObjectDeleteException}.
+   * Invoke {@link #removeKeysS3(List, boolean)}.
+   * If a {@code MultiObjectDeleteException} is raised, the
+   * relevant statistics are updated.
    *
    * @param keysToDelete collection of keys to delete on the s3-backend.
    *        if empty, no request is made of the object store.
    * @param deleteFakeDir indicates whether this is for deleting fake dirs
-   * @param operationState (nullable) operational state for a bulk update
    * @throws InvalidRequestException if the request was rejected due to
    * a mistaken attempt to delete the root directory.
    * @throws MultiObjectDeleteException one or more of the keys could not
@@ -3073,78 +3143,15 @@ private void noteDeleted(final int count, final boolean deleteFakeDir) {
    * @throws IOException other IO Exception.
    */
   @VisibleForTesting
-  @Retries.RetryMixed
+  @Retries.RetryRaw
   public void removeKeys(
       final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      final boolean deleteFakeDir,
-      final BulkOperationState operationState)
+      final boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException,
       IOException {
-    removeKeys(keysToDelete, deleteFakeDir, new ArrayList<>(), operationState,
-        true);
-  }
-
-  /**
-   * Invoke {@link #removeKeysS3(List, boolean, boolean)} with handling of
-   * {@code MultiObjectDeleteException} before the exception is rethrown.
-   * Specifically:
-   * <ol>
-   *   <li>Failure and !deleteFakeDir: S3Guard is updated with all
-   *    deleted entries</li>
-   *   <li>Failure where deleteFakeDir == true: do nothing with S3Guard</li>
-   *   <li>Success: do nothing with S3Guard</li>
-   * </ol>
-   * @param keysToDelete collection of keys to delete on the s3-backend.
-   *        if empty, no request is made of the object store.
-   * @param deleteFakeDir indicates whether this is for deleting fake dirs.
-   * @param undeletedObjectsOnFailure List which will be built up of all
-   * files that were not deleted. This happens even as an exception
-   * is raised.
-   * @param operationState (nullable) operational state for a bulk update
-   * @param quiet should a bulk query be quiet, or should its result list
-   * all deleted keys
-   * @return the deletion result if a multi object delete was invoked
-   * and it returned without a failure, else null.
-   * @throws InvalidRequestException if the request was rejected due to
-   * a mistaken attempt to delete the root directory.
-   * @throws MultiObjectDeleteException one or more of the keys could not
-   * be deleted in a multiple object delete operation.
-   * @throws AmazonClientException amazon-layer failure.
-   * @throws IOException other IO Exception.
-   */
-  @Retries.RetryMixed
-  private DeleteObjectsResult removeKeys(
-      final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      final boolean deleteFakeDir,
-      final List<Path> undeletedObjectsOnFailure,
-      final BulkOperationState operationState,
-      final boolean quiet)
-      throws MultiObjectDeleteException, AmazonClientException, IOException {
-    undeletedObjectsOnFailure.clear();
     try (DurationInfo ignored = new DurationInfo(LOG, false,
-        "Deleting %d keys", keysToDelete.size())) {
-      return removeKeysS3(keysToDelete, deleteFakeDir, quiet);
-    } catch (MultiObjectDeleteException ex) {
-      LOG.debug("Partial delete failure");
-      // what to do if an IOE was raised? Given an exception was being
-      // raised anyway, and the failures are logged, do nothing.
-      if (!deleteFakeDir) {
-        // when deleting fake directories we don't want to delete metastore
-        // entries so we only process these failures on "real" deletes.
-        Triple<List<Path>, List<Path>, List<Pair<Path, IOException>>> results =
-            new MultiObjectDeleteSupport(createStoreContext(), operationState)
-                .processDeleteFailure(ex, keysToDelete, new ArrayList<Path>());
-        undeletedObjectsOnFailure.addAll(results.getLeft());
-      }
-      throw ex;
-    } catch (AmazonClientException | IOException ex) {
-      List<Path> paths = new MultiObjectDeleteSupport(
-          createStoreContext(),
-          operationState)
-          .processDeleteFailureGenericException(ex, keysToDelete);
-      // other failures. Assume nothing was deleted
-      undeletedObjectsOnFailure.addAll(paths);
-      throw ex;
+            "Deleting %d keys", keysToDelete.size())) {
+      removeKeysS3(keysToDelete, deleteFakeDir);
     }
   }
 
@@ -3166,6 +3173,24 @@ private DeleteObjectsResult removeKeys(
   @AuditEntryPoint
   public boolean delete(Path f, boolean recursive) throws IOException {
     checkNotClosed();
+    return deleteWithoutCloseCheck(f, recursive);
+  }
+
+  /**
+   * Same as delete(), except that it does not check if fs is closed.
+   *
+   * @param f the path to delete.
+   * @param recursive if path is a directory and set to
+   * true, the directory is deleted else throws an exception. In
+   * case of a file the recursive can be set to either true or false.
+   * @return true if the path existed and then was deleted; false if there
+   * was no path in the first place, or the corner cases of root path deletion
+   * have surfaced.
+   * @throws IOException due to inability to delete a directory or file.
+   */
+
+  @VisibleForTesting
+  protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOException {
     final Path path = qualify(f);
     // span covers delete, getFileStatus, fake directory operations.
     try (AuditSpan span = createSpan(INVOCATION_DELETE.getSymbol(),
@@ -3213,7 +3238,7 @@ private void createFakeDirectoryIfNecessary(Path f)
     // is mostly harmless to create a new one.
     if (!key.isEmpty() && !s3Exists(f, StatusProbeEnum.DIRECTORIES)) {
       LOG.debug("Creating new fake directory at {}", f);
-      createFakeDirectory(key);
+      createFakeDirectory(key, putOptionsForPath(f));
     }
   }
 
@@ -3228,7 +3253,7 @@ private void createFakeDirectoryIfNecessary(Path f)
   protected void maybeCreateFakeParentDirectory(Path path)
       throws IOException, AmazonClientException {
     Path parent = path.getParent();
-    if (parent != null && !parent.isRoot()) {
+    if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) {
       createFakeDirectoryIfNecessary(parent);
     }
   }
@@ -3266,8 +3291,7 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException,
     Path path = qualify(f);
     return trackDurationAndSpan(INVOCATION_LIST_STATUS, path, () ->
         once("listStatus", path.toString(),
-            () -> iteratorToStatuses(innerListStatus(path),
-                new HashSet<>())));
+            () -> iteratorToStatuses(innerListStatus(path))));
   }
 
   /**
@@ -3289,15 +3313,9 @@ private RemoteIterator<S3AFileStatus> innerListStatus(Path f)
     Path path = qualify(f);
     LOG.debug("List status for path: {}", path);
 
-    Triple<RemoteIterator<S3AFileStatus>, DirListingMetadata, Boolean>
-            statusesAssumingNonEmptyDir = listing
-            .getFileStatusesAssumingNonEmptyDir(path, getActiveAuditSpan());
-
-    if (!statusesAssumingNonEmptyDir.getLeft().hasNext() &&
-            statusesAssumingNonEmptyDir.getRight()) {
-      // We are sure that this is an empty directory in auth mode.
-      return statusesAssumingNonEmptyDir.getLeft();
-    } else if (!statusesAssumingNonEmptyDir.getLeft().hasNext()) {
+    final RemoteIterator<S3AFileStatus> statusIt = listing
+        .getFileStatusesAssumingNonEmptyDir(path, getActiveAuditSpan());
+    if (!statusIt.hasNext()) {
       // We may have an empty dir, or may have file or may have nothing.
       // So we call innerGetFileStatus to get the status, this may throw
       // FileNotFoundException if we have nothing.
@@ -3316,31 +3334,19 @@ private RemoteIterator<S3AFileStatus> innerListStatus(Path f)
       }
     }
     // Here we have a directory which may or may not be empty.
-    // So we update the metastore and return.
-    return S3Guard.dirListingUnion(
-            metadataStore,
-            path,
-            statusesAssumingNonEmptyDir.getLeft(),
-            statusesAssumingNonEmptyDir.getMiddle(),
-            allowAuthoritative(path),
-            ttlTimeProvider, p ->
-                    listing.createProvidedFileStatusIterator(
-                    dirMetaToStatuses(statusesAssumingNonEmptyDir.getMiddle()),
-                    ACCEPT_ALL,
-                    Listing.ACCEPT_ALL_BUT_S3N));
+    return statusIt;
   }
 
   /**
    * Is a path to be considered as authoritative?
-   * True iff there is an authoritative metastore or if there
-   * is a non-auth store with the supplied path under
+   * is a  store with the supplied path under
    * one of the paths declared as authoritative.
    * @param path path
    * @return true if the path is auth
    */
   public boolean allowAuthoritative(final Path path) {
     return S3Guard.allowAuthoritative(path, this,
-        allowAuthoritativeMetadataStore, allowAuthoritativePaths);
+        allowAuthoritativePaths);
   }
 
   /**
@@ -3418,6 +3424,11 @@ public UserGroupInformation getOwner() {
    * Make the given path and all non-existent parents into
    * directories. Has the semantics of Unix {@code 'mkdir -p'}.
    * Existence of the directory hierarchy is not an error.
+   * Parent elements are scanned to see if any are a file,
+   * <i>except under __magic</i> paths.
+   * There the FS assumes that the destination directory creation
+   * did that scan and that paths in job/task attempts are all
+   * "well formed"
    * @param p path to create
    * @param permission to apply to path
    * @return true if a directory was created or already existed
@@ -3435,7 +3446,8 @@ public boolean mkdirs(Path p, FsPermission permission) throws IOException,
         new MkdirOperation(
             createStoreContext(),
             path,
-            createMkdirOperationCallbacks()));
+            createMkdirOperationCallbacks(),
+            isMagicCommitPath(path)));
   }
 
   /**
@@ -3461,9 +3473,13 @@ public S3AFileStatus probePathStatus(final Path path,
     }
 
     @Override
-    public void createFakeDirectory(final String key)
+    public void createFakeDirectory(final Path dir, final boolean keepMarkers)
         throws IOException {
-      S3AFileSystem.this.createEmptyObject(key);
+      S3AFileSystem.this.createFakeDirectory(
+          pathToKey(dir),
+          keepMarkers
+              ? PutObjectOptions.keepingDirs()
+              : putOptionsForPath(dir));
     }
   }
 
@@ -3509,9 +3525,9 @@ public S3AFileStatus probePathStatus(final Path path,
     }
 
     @Override
-    public RemoteIterator<S3AFileStatus> listStatusIterator(final Path path)
-        throws IOException {
-      return S3AFileSystem.this.innerListStatus(path);
+    public RemoteIterator<S3ALocatedFileStatus> listFilesIterator(final Path path,
+        final boolean recursive) throws IOException {
+      return S3AFileSystem.this.innerListFiles(path, recursive, Listing.ACCEPT_ALL_BUT_S3N, null);
     }
   }
 
@@ -3562,13 +3578,7 @@ public FileStatus getFileStatus(final Path f) throws IOException {
   }
 
   /**
-   * Get the status of a file or directory, first through S3Guard and then
-   * through S3.
-   * The S3 probes can leave 404 responses in the S3 load balancers; if
-   * a check is only needed for a directory, declaring this saves time and
-   * avoids creating one for the object.
-   * When only probing for directories, if an entry for a file is found in
-   * S3Guard it is returned, but checks for updated values are skipped.
+   * Get the status of a file or directory.
    * Internal version of {@link #getFileStatus(Path)}.
    * @param f The path we want information from
    * @param needEmptyDirectoryFlag if true, implementation will calculate
@@ -3587,120 +3597,15 @@ S3AFileStatus innerGetFileStatus(final Path f,
     String key = pathToKey(path);
     LOG.debug("Getting path status for {}  ({}); needEmptyDirectory={}",
         path, key, needEmptyDirectoryFlag);
+    return s3GetFileStatus(path,
+        key,
+        probes,
+        needEmptyDirectoryFlag);
 
-    boolean allowAuthoritative = allowAuthoritative(path);
-    // Check MetadataStore, if any.
-    PathMetadata pm = null;
-    if (hasMetadataStore()) {
-      pm = S3Guard.getWithTtl(metadataStore, path, ttlTimeProvider,
-          needEmptyDirectoryFlag, allowAuthoritative);
-    }
-    Set<Path> tombstones = Collections.emptySet();
-    if (pm != null) {
-      S3AFileStatus msStatus = pm.getFileStatus();
-      if (pm.isDeleted()) {
-        OffsetDateTime deletedAt = OffsetDateTime.ofInstant(
-            Instant.ofEpochMilli(msStatus.getModificationTime()),
-            ZoneOffset.UTC);
-        throw new FileNotFoundException("Path " + path + " is recorded as " +
-            "deleted by S3Guard at " + deletedAt);
-      }
-
-      // if ms is not authoritative, check S3 if there's any recent
-      // modification - compare the modTime to check if metadata is up to date
-      // Skip going to s3 if the file checked is a directory. Because if the
-      // dest is also a directory, there's no difference.
-
-      if (!msStatus.isDirectory() &&
-          !allowAuthoritative &&
-          probes.contains(StatusProbeEnum.Head)) {
-        // a file has been found in a non-auth path and the caller has not said
-        // they only care about directories
-        LOG.debug("Metadata for {} found in the non-auth metastore.", path);
-        final long msModTime = pm.getFileStatus().getModificationTime();
-
-        S3AFileStatus s3AFileStatus;
-        try {
-          s3AFileStatus = s3GetFileStatus(path,
-              key,
-              probes,
-              tombstones,
-              needEmptyDirectoryFlag);
-        } catch (FileNotFoundException fne) {
-          LOG.trace("File Not Found from probes for {}", key, fne);
-          s3AFileStatus = null;
-        }
-        if (s3AFileStatus == null) {
-          LOG.warn("Failed to find file {}. Either it is not yet visible, or "
-              + "it has been deleted.", path);
-        } else {
-          final long s3ModTime = s3AFileStatus.getModificationTime();
-
-          if(s3ModTime > msModTime) {
-            LOG.debug("S3Guard metadata for {} is outdated;"
-                + " s3modtime={}; msModTime={} updating metastore",
-                path, s3ModTime, msModTime);
-            return S3Guard.putAndReturn(metadataStore, s3AFileStatus,
-                ttlTimeProvider);
-          }
-        }
-      }
-
-      if (needEmptyDirectoryFlag && msStatus.isDirectory()) {
-        // the caller needs to know if a directory is empty,
-        // and that this is a directory.
-        if (pm.isEmptyDirectory() != Tristate.UNKNOWN) {
-          // We have a definitive true / false from MetadataStore, we are done.
-          return msStatus;
-        } else {
-          // execute a S3Guard listChildren command to list tombstones under the
-          // path.
-          // This list will be used in the forthcoming s3GetFileStatus call.
-          DirListingMetadata children =
-              S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider,
-                  allowAuthoritative);
-          if (children != null) {
-            tombstones = children.listTombstones();
-          }
-          LOG.debug("MetadataStore doesn't know if {} is empty, using S3.",
-              path);
-        }
-      } else {
-        // Either this is not a directory, or we don't care if it is empty
-        return msStatus;
-      }
-
-      // now issue the S3 getFileStatus call.
-      try {
-        S3AFileStatus s3FileStatus = s3GetFileStatus(path,
-            key,
-            probes,
-            tombstones,
-            true);
-        // entry was found, so save in S3Guard and return the final value.
-        return S3Guard.putAndReturn(metadataStore, s3FileStatus,
-            ttlTimeProvider);
-      } catch (FileNotFoundException e) {
-        // If the metadata store has no children for it and it's not listed in
-        // S3 yet, we'll conclude that it is an empty directory
-        return S3AFileStatus.fromFileStatus(msStatus, Tristate.TRUE,
-            null, null);
-      }
-    } else {
-      // there was no entry in S3Guard
-      // retrieve the data and update the metadata store in the process.
-      return S3Guard.putAndReturn(metadataStore,
-          s3GetFileStatus(path,
-              key,
-              probes,
-              tombstones,
-              needEmptyDirectoryFlag),
-          ttlTimeProvider);
-    }
   }
 
   /**
-   * Raw {@code getFileStatus} that talks direct to S3.
+   * Probe store for file status with control of which probes are issued..
    * Used to implement {@link #innerGetFileStatus(Path, boolean, Set)},
    * and for direct management of empty directory blobs.
    *
@@ -3711,13 +3616,7 @@ S3AFileStatus innerGetFileStatus(final Path f,
    *     the key doesn't end in "/"
    *   </li>
    *   <li>
-   *     DirMarker: look for the directory marker -the key with a trailing /
-   *     if not passed in.
-   *     If an object was found with size 0 bytes, a directory status entry
-   *     is returned which declares that the directory is empty.
-   *   </li>
-   *    <li>
-   *     List: issue a LIST on the key (with / if needed), require one
+   *     DirMarker/List: issue a LIST on the key (with / if needed), require one
    *     entry to be found for the path to be considered a non-empty directory.
    *   </li>
    * </ol>
@@ -3725,12 +3624,8 @@ S3AFileStatus innerGetFileStatus(final Path f,
    * Notes:
    * <ul>
    *   <li>
-   *     Objects ending in / which are not 0-bytes long are not treated as
-   *     directory markers, but instead as files.
-   *   </li>
-   *   <li>
-   *     There's ongoing discussions about whether a dir marker
-   *     should be interpreted as an empty dir.
+   *     Objects ending in / are treated as directory markers,
+   *     irrespective of length.
    *   </li>
    *   <li>
    *     The HEAD requests require the permissions to read an object,
@@ -3739,8 +3634,7 @@ S3AFileStatus innerGetFileStatus(final Path f,
    *     the client for the HEAD to work.
    *   </li>
    *   <li>
-   *     The List probe needs list permission; it is also more prone to
-   *     inconsistency, even on newly created files.
+   *     The List probe needs list permission.
    *   </li>
    * </ul>
    *
@@ -3748,7 +3642,6 @@ S3AFileStatus innerGetFileStatus(final Path f,
    * @param path Qualified path
    * @param key  Key string for the path
    * @param probes probes to make
-   * @param tombstones tombstones to filter
    * @param needEmptyDirectoryFlag if true, implementation will calculate
    *        a TRUE or FALSE value for {@link S3AFileStatus#isEmptyDirectory()}
    * @return Status
@@ -3760,7 +3653,6 @@ S3AFileStatus innerGetFileStatus(final Path f,
   S3AFileStatus s3GetFileStatus(final Path path,
       final String key,
       final Set<StatusProbeEnum> probes,
-      @Nullable final Set<Path> tombstones,
       final boolean needEmptyDirectoryFlag) throws IOException {
     LOG.debug("S3GetFileStatus {}", path);
     // either you aren't looking for the directory flag, or you are,
@@ -3769,7 +3661,6 @@ S3AFileStatus s3GetFileStatus(final Path path,
         || probes.contains(StatusProbeEnum.List),
         "s3GetFileStatus(%s) wants to know if a directory is empty but"
             + " does not request a list probe", path);
-
     if (key.isEmpty() && !needEmptyDirectoryFlag) {
       return new S3AFileStatus(Tristate.UNKNOWN, path, username);
     }
@@ -3797,6 +3688,8 @@ S3AFileStatus s3GetFileStatus(final Path path,
       } catch (AmazonServiceException e) {
         // if the response is a 404 error, it just means that there is
         // no file at that path...the remaining checks will be needed.
+        // But: an empty bucket is also a 404, so check for that
+        // and fail.
         if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
           throw translateException("getFileStatus", path, e);
         }
@@ -3814,30 +3707,16 @@ S3AFileStatus s3GetFileStatus(final Path path,
         // children, so ask for two entries, so as to find
         // a child
         String dirKey = maybeAddTrailingSlash(key);
-        // list size is dir marker + at least one non-tombstone entry
-        // there's a corner case: more tombstones than you have in a
-        // single page list. We assume that if you have been deleting
-        // that many files, then the AWS listing will have purged some
-        // by the time of listing so that the response includes some
-        // which have not.
-
-        int listSize;
-        if (tombstones == null) {
-          // no tombstones so look for a marker and at least one child.
-          listSize = 2;
-        } else {
-          // build a listing > tombstones. If the caller has many thousands
-          // of tombstones this won't work properly, which is why pruning
-          // of expired tombstones matters.
-          listSize = Math.min(2 + tombstones.size(), Math.max(2, maxKeys));
-        }
+        // list size is dir marker + at least one entry
+
+        final int listSize = 2;
         S3ListRequest request = createListObjectsRequest(dirKey, "/",
             listSize);
         // execute the request
         S3ListResult listResult = listObjects(request,
             getDurationTrackerFactory());
 
-        if (listResult.hasPrefixesOrObjects(contextAccessors, tombstones)) {
+        if (listResult.hasPrefixesOrObjects()) {
           if (LOG.isDebugEnabled()) {
             LOG.debug("Found path as directory (with /)");
             listResult.logAtDebug(LOG);
@@ -3847,8 +3726,7 @@ S3AFileStatus s3GetFileStatus(final Path path,
           // children.
           // So the listing must contain the marker entry only.
           if (needEmptyDirectoryFlag
-              && listResult.representsEmptyDirectory(
-                  contextAccessors, dirKey, tombstones)) {
+              && listResult.representsEmptyDirectory(dirKey)) {
             return new S3AFileStatus(Tristate.TRUE, path, username);
           }
           // either an empty directory is not needed, or the
@@ -3872,8 +3750,7 @@ S3AFileStatus s3GetFileStatus(final Path path,
   }
 
   /**
-   * Raw version of {@link FileSystem#exists(Path)} which uses S3 only:
-   * S3Guard MetadataStore, if any, will be skipped.
+   * Probe S3 for a file or dir existing, with the given probe set.
    * Retry policy: retrying; translated.
    * @param path qualified path to look for
    * @param probes probes to make
@@ -3885,7 +3762,7 @@ private boolean s3Exists(final Path path, final Set<StatusProbeEnum> probes)
       throws IOException {
     String key = pathToKey(path);
     try {
-      s3GetFileStatus(path, key, probes, null, false);
+      s3GetFileStatus(path, key, probes, false);
       return true;
     } catch (FileNotFoundException e) {
       return false;
@@ -3926,12 +3803,12 @@ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
   }
 
   protected CopyFromLocalOperation.CopyFromLocalOperationCallbacks
-  createCopyFromLocalCallbacks() throws IOException {
+      createCopyFromLocalCallbacks() throws IOException {
     LocalFileSystem local = getLocal(getConf());
     return new CopyFromLocalCallbacksImpl(local);
   }
 
-  protected class CopyFromLocalCallbacksImpl implements
+  protected final class CopyFromLocalCallbacksImpl implements
       CopyFromLocalOperation.CopyFromLocalOperationCallbacks {
     private final LocalFileSystem local;
 
@@ -3968,7 +3845,7 @@ public void copyLocalFileFromTo(File file, Path from, Path to) throws IOExceptio
             S3AFileSystem.this.invoker.retry(
                 "putObject(" + "" + ")", to.toString(),
                 true,
-                () -> executePut(putObjectRequest, progress));
+                () -> executePut(putObjectRequest, progress, putOptionsForPath(to)));
 
             return null;
           });
@@ -3987,28 +3864,28 @@ public boolean createEmptyDir(Path path, StoreContext storeContext)
           new MkdirOperation(
               storeContext,
               path,
-              createMkdirOperationCallbacks()));
+              createMkdirOperationCallbacks(), false));
     }
   }
 
   /**
-   * Execute a PUT via the transfer manager, blocking for completion,
-   * updating the metastore afterwards.
+   * Execute a PUT via the transfer manager, blocking for completion.
    * If the waiting for completion is interrupted, the upload will be
    * aborted before an {@code InterruptedIOException} is thrown.
    * @param putObjectRequest request
    * @param progress optional progress callback
+   * @param putOptions put object options
    * @return the upload result
    * @throws InterruptedIOException if the blocking was interrupted.
-   * @throws MetadataPersistenceException if metadata about the write could
-   * not be saved to the metadata store and
-   * fs.s3a.metadatastore.fail.on.write.error=true
-   */
-  @Retries.OnceRaw("For PUT; post-PUT actions are RetryTranslated")
-  UploadResult executePut(PutObjectRequest putObjectRequest,
-      Progressable progress)
-      throws InterruptedIOException, MetadataPersistenceException {
+   */
+  @Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed")
+  UploadResult executePut(
+      final PutObjectRequest putObjectRequest,
+      final Progressable progress,
+      final PutObjectOptions putOptions)
+      throws InterruptedIOException {
     String key = putObjectRequest.getKey();
+    long len = getPutRequestLength(putObjectRequest);
     UploadInfo info = putObject(putObjectRequest);
     Upload upload = info.getUpload();
     ProgressableProgressListener listener = new ProgressableProgressListener(
@@ -4016,9 +3893,10 @@ UploadResult executePut(PutObjectRequest putObjectRequest,
     upload.addProgressListener(listener);
     UploadResult result = waitForUploadCompletion(key, info);
     listener.uploadCompleted();
+
     // post-write actions
-    finishedWrite(key, info.getLength(),
-        result.getETag(), result.getVersionId(), null);
+    finishedWrite(key, len,
+        result.getETag(), result.getVersionId(), putOptions);
     return result;
   }
 
@@ -4027,7 +3905,9 @@ UploadResult executePut(PutObjectRequest putObjectRequest,
    * If the waiting for completion is interrupted, the upload will be
    * aborted before an {@code InterruptedIOException} is thrown.
    * If the upload (or its result collection) failed, this is where
-   * the failure is raised as an AWS exception
+   * the failure is raised as an AWS exception.
+   * Calls {@link #incrementPutCompletedStatistics(boolean, long)}
+   * to update the statistics.
    * @param key destination key
    * @param uploadInfo upload to wait for
    * @return the upload result
@@ -4052,6 +3932,61 @@ UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
     }
   }
 
+  /**
+   * This override bypasses checking for existence.
+   *
+   * @param f the path to delete; this may be unqualified.
+   * @return true, always.   * @param f the path to delete.
+   * @return  true if deleteOnExit is successful, otherwise false.
+   * @throws IOException IO failure
+   */
+  @Override
+  public boolean deleteOnExit(Path f) throws IOException {
+    Path qualifedPath = makeQualified(f);
+    synchronized (deleteOnExit) {
+      deleteOnExit.add(qualifedPath);
+    }
+    return true;
+  }
+
+  /**
+   * Cancel the scheduled deletion of the path when the FileSystem is closed.
+   * @param f the path to cancel deletion
+   * @return true if the path was found in the delete-on-exit list.
+   */
+  @Override
+  public boolean cancelDeleteOnExit(Path f) {
+    Path qualifedPath = makeQualified(f);
+    synchronized (deleteOnExit) {
+      return deleteOnExit.remove(qualifedPath);
+    }
+  }
+
+  /**
+   * Delete all paths that were marked as delete-on-exit. This recursively
+   * deletes all files and directories in the specified paths. It does not
+   * check if file exists and filesystem is closed.
+   *
+   * The time to process this operation is {@code O(paths)}, with the actual
+   * time dependent on the time for existence and deletion operations to
+   * complete, successfully or not.
+   */
+  @Override
+  protected void processDeleteOnExit() {
+    synchronized (deleteOnExit) {
+      for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
+        Path path = iter.next();
+        try {
+          deleteWithoutCloseCheck(path, true);
+        } catch (IOException e) {
+          LOG.info("Ignoring failure to deleteOnExit for path {}", path);
+          LOG.debug("The exception for deleteOnExit is {}", e);
+        }
+        iter.remove();
+      }
+    }
+  }
+
   /**
    * Close the filesystem. This shuts down all transfers.
    * @throws IOException IO problem
@@ -4109,9 +4044,12 @@ protected synchronized void stopAllServices() {
     HadoopExecutors.shutdown(unboundedThreadPool, LOG,
         THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
     unboundedThreadPool = null;
+    if (futurePool != null) {
+      futurePool.shutdown(LOG, THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
+      futurePool = null;
+    }
     // other services are shutdown.
     cleanupWithLogger(LOG,
-        metadataStore,
         instrumentation,
         delegationTokens.orElse(null),
         signerManager,
@@ -4214,7 +4152,6 @@ public DelegationTokenIssuer[] getAdditionalTokenIssuers()
   /**
    * Build the AWS policy for restricted access to the resources needed
    * by this bucket.
-   * The policy generated includes S3 access, S3Guard access
    * if needed, and KMS operations.
    * @param access access level desired.
    * @return a policy for use in roles
@@ -4236,11 +4173,6 @@ public List<RoleModel.Statement> listAWSPolicyRules(
     // too much encryption access.
     statements.add(STATEMENT_ALLOW_SSE_KMS_RW);
 
-    // add any metastore policies
-    if (metadataStore instanceof AWSPolicyProvider) {
-      statements.addAll(
-          ((AWSPolicyProvider) metadataStore).listAWSPolicyRules(access));
-    }
     return statements;
   }
 
@@ -4292,21 +4224,14 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
     } catch (FileNotFoundException e) {
       // if rename fails at this point it means that the expected file was not
       // found.
-      // The cause is believed to always be one of
-      //  - File was deleted since LIST/S3Guard metastore.list.() knew of it.
-      //  - S3Guard is asking for a specific version and it's been removed by
-      //    lifecycle rules.
-      //  - there's a 404 cached in the S3 load balancers.
+      // This means the File was deleted since LIST enumerated it.
       LOG.debug("getObjectMetadata({}) failed to find an expected file",
           srcKey, e);
       // We create an exception, but the text depends on the S3Guard state
-      String message = hasMetadataStore()
-          ? RemoteFileChangedException.FILE_NEVER_FOUND
-          : RemoteFileChangedException.FILE_NOT_FOUND_SINGLE_ATTEMPT;
       throw new RemoteFileChangedException(
           keyToQualifiedPath(srcKey).toString(),
           action,
-          message,
+          RemoteFileChangedException.FILE_NOT_FOUND_SINGLE_ATTEMPT,
           e);
     }
 
@@ -4363,134 +4288,74 @@ InitiateMultipartUploadResult initiateMultipartUpload(
 
   /**
    * Perform post-write actions.
-   * <p></p>
+   * <p>
    * This operation MUST be called after any PUT/multipart PUT completes
    * successfully.
-   * <p></p>
-   * The actions include:
-   * <ol>
-   *   <li>
-   *     Calling
-   *     {@link #deleteUnnecessaryFakeDirectories(Path, BulkOperationState)}
-   *     if directory markers are not being retained.
-   *   </li>
-   *   <li>
-   *     Updating any metadata store with details on the newly created
-   *     object.
-   *     </li>
-   * </ol>
+   * <p>
+   * The actions include calling
+   * {@link #deleteUnnecessaryFakeDirectories(Path)}
+   * if directory markers are not being retained.
    * @param key key written to
    * @param length  total length of file written
    * @param eTag eTag of the written object
    * @param versionId S3 object versionId of the written object
-   * @param operationState state of any ongoing bulk operation.
-   * @throws MetadataPersistenceException if metadata about the write could
-   * not be saved to the metadata store and
-   * fs.s3a.metadatastore.fail.on.write.error=true
+   * @param putOptions put object options
    */
   @InterfaceAudience.Private
-  @Retries.RetryTranslated("Except if failOnMetadataWriteError=false, in which"
-      + " case RetryExceptionsSwallowed")
-  void finishedWrite(String key, long length, String eTag, String versionId,
-      @Nullable final BulkOperationState operationState)
-      throws MetadataPersistenceException {
+  @Retries.RetryExceptionsSwallowed
+  void finishedWrite(
+      String key,
+      long length,
+      String eTag,
+      String versionId,
+      PutObjectOptions putOptions) {
     LOG.debug("Finished write to {}, len {}. etag {}, version {}",
         key, length, eTag, versionId);
-    Path p = keyToQualifiedPath(key);
     Preconditions.checkArgument(length >= 0, "content length is negative");
-    final boolean isDir = objectRepresentsDirectory(key);
-    // kick off an async delete
-    CompletableFuture<?> deletion;
-    if (!keepDirectoryMarkers(p)) {
-      deletion = submit(
-          unboundedThreadPool, getActiveAuditSpan(),
-          () -> {
-            deleteUnnecessaryFakeDirectories(
-                p.getParent(),
-                operationState);
-            return null;
-          });
-    } else {
-      deletion = null;
-    }
-    // this is only set if there is a metastore to update and the
-    // operationState parameter passed in was null.
-    BulkOperationState stateToClose = null;
-
-    // See note about failure semantics in S3Guard documentation
-    try {
-      if (hasMetadataStore()) {
-        BulkOperationState activeState = operationState;
-        if (activeState == null) {
-          // create an operation state if there was none, so that the
-          // information gleaned from addAncestors is preserved into the
-          // subsequent put.
-          stateToClose = S3Guard.initiateBulkWrite(metadataStore,
-              isDir
-                  ? BulkOperationState.OperationType.Mkdir
-                  : BulkOperationState.OperationType.Put,
-              keyToPath(key));
-          activeState = stateToClose;
-        }
-        S3Guard.addAncestors(metadataStore, p, ttlTimeProvider, activeState);
-        S3AFileStatus status = createUploadFileStatus(p,
-            isDir, length,
-            getDefaultBlockSize(p), username, eTag, versionId);
-        boolean authoritative = false;
-        if (isDir) {
-          // this is a directory marker so put it as such.
-          status.setIsEmptyDirectory(Tristate.TRUE);
-          // and maybe mark as auth
-          authoritative = allowAuthoritative(p);
-        }
-        if (!authoritative) {
-          // for files and non-auth directories
-          S3Guard.putAndReturn(metadataStore, status,
-              ttlTimeProvider,
-              activeState);
-        } else {
-          // authoritative directory
-          S3Guard.putAuthDirectoryMarker(metadataStore, status,
-              ttlTimeProvider,
-              activeState);
-        }
-      }
-      // and catch up with any delete operation.
-      waitForCompletionIgnoringExceptions(deletion);
-    } catch (IOException e) {
-      if (failOnMetadataWriteError) {
-        throw new MetadataPersistenceException(p.toString(), e);
-      } else {
-        LOG.error("S3Guard: Error updating MetadataStore for write to {}",
-            p, e);
-      }
-      instrumentation.errorIgnored();
-    } finally {
-      // if a new operation state was created, close it.
-      IOUtils.cleanupWithLogger(LOG, stateToClose);
+    if (!putOptions.isKeepMarkers()) {
+      Path p = keyToQualifiedPath(key);
+      deleteUnnecessaryFakeDirectories(p.getParent());
     }
   }
 
   /**
    * Should we keep directory markers under the path being created
    * by mkdir/file creation/rename?
+   * This is done if marker retention is enabled for the path,
+   * or it is under a magic path where we are saving IOPs
+   * knowing that all committers are on the same code version and
+   * therefore marker aware.
    * @param path path to probe
    * @return true if the markers MAY be retained,
    * false if they MUST be deleted
    */
   private boolean keepDirectoryMarkers(Path path) {
-    return directoryPolicy.keepDirectoryMarkers(path);
+    return directoryPolicy.keepDirectoryMarkers(path)
+        || isUnderMagicCommitPath(path);
+  }
+
+  /**
+   * Should we keep directory markers under the path being created
+   * by mkdir/file creation/rename?
+   * See {@link #keepDirectoryMarkers(Path)} for the policy.
+   *
+   * @param path path to probe
+   * @return the options to use with the put request
+   */
+  private PutObjectOptions putOptionsForPath(Path path) {
+    return keepDirectoryMarkers(path)
+        ? PutObjectOptions.keepingDirs()
+        : PutObjectOptions.deletingDirs();
   }
 
   /**
    * Delete mock parent directories which are no longer needed.
    * Retry policy: retrying; exceptions swallowed.
    * @param path path
-   * @param operationState (nullable) operational state for a bulk update
+   *
    */
   @Retries.RetryExceptionsSwallowed
-  private void deleteUnnecessaryFakeDirectories(Path path,
-      final BulkOperationState operationState) {
+  private void deleteUnnecessaryFakeDirectories(Path path) {
     List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>();
     while (!path.isRoot()) {
       String key = pathToKey(path);
@@ -4500,7 +4365,7 @@ private void deleteUnnecessaryFakeDirectories(Path path,
       path = path.getParent();
     }
     try {
-      removeKeys(keysToRemove, true, operationState);
+      removeKeys(keysToRemove, true);
     } catch(AmazonClientException | IOException e) {
       instrumentation.errorIgnored();
       if (LOG.isDebugEnabled()) {
@@ -4517,27 +4382,32 @@ private void deleteUnnecessaryFakeDirectories(Path path,
    * Create a fake directory, always ending in "/".
    * Retry policy: retrying; translated.
    * @param objectName name of directory object.
+   * @param putOptions put object options
    * @throws IOException IO failure
    */
   @Retries.RetryTranslated
-  private void createFakeDirectory(final String objectName)
+  private void createFakeDirectory(final String objectName,
+      final PutObjectOptions putOptions)
       throws IOException {
-    createEmptyObject(objectName);
+    createEmptyObject(objectName, putOptions);
   }
 
   /**
    * Used to create an empty file that represents an empty directory.
+   * The policy for deleting parent dirs depends on the path, dir
+   * status and the putOptions value.
    * Retry policy: retrying; translated.
    * @param objectName object to create
+   * @param putOptions put object options
    * @throws IOException IO failure
    */
   @Retries.RetryTranslated
-  private void createEmptyObject(final String objectName)
+  private void createEmptyObject(final String objectName, PutObjectOptions putOptions)
       throws IOException {
     invoker.retry("PUT 0-byte object ", objectName,
          true, () ->
             putObjectDirect(getRequestFactory()
-                .newDirectoryMarkerRequest(objectName)));
+                .newDirectoryMarkerRequest(objectName), putOptions));
     incrementPutProgressStatistics(objectName, 0);
     instrumentation.directoryCreated();
   }
@@ -4545,9 +4415,7 @@ private void createEmptyObject(final String objectName)
   /**
    * Return the number of bytes that large input files should be optimally
    * be split into to minimize I/O time.
-   * @deprecated use {@link #getDefaultBlockSize(Path)} instead
    */
-  @Deprecated
   public long getDefaultBlockSize() {
     return getConf().getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE);
   }
@@ -4566,14 +4434,15 @@ public String toString() {
         "S3AFileSystem{");
     sb.append("uri=").append(uri);
     sb.append(", workingDir=").append(workingDir);
-    sb.append(", inputPolicy=").append(inputPolicy);
     sb.append(", partSize=").append(partSize);
     sb.append(", enableMultiObjectsDelete=").append(enableMultiObjectsDelete);
     sb.append(", maxKeys=").append(maxKeys);
     if (cannedACL != null) {
-      sb.append(", cannedACL=").append(cannedACL.toString());
+      sb.append(", cannedACL=").append(cannedACL);
+    }
+    if (openFileHelper != null) {
+      sb.append(", ").append(openFileHelper);
     }
-    sb.append(", readAhead=").append(readAhead);
     if (getConf() != null) {
       sb.append(", blockSize=").append(getDefaultBlockSize());
     }
@@ -4587,8 +4456,6 @@ public String toString() {
       sb.append(", blockFactory=").append(blockFactory);
     }
     sb.append(", auditManager=").append(auditManager);
-    sb.append(", metastore=").append(metadataStore);
-    sb.append(", authoritativeStore=").append(allowAuthoritativeMetadataStore);
     sb.append(", authoritativePath=").append(allowAuthoritativePaths);
     sb.append(", useListV1=").append(useListV1);
     if (committerIntegration != null) {
@@ -4649,14 +4516,26 @@ public boolean isMagicCommitEnabled() {
 
   /**
    * Predicate: is a path a magic commit path?
-   * True if magic commit is enabled and the path qualifies as special.
+   * True if magic commit is enabled and the path qualifies as special,
+   * and is not a a .pending or .pendingset file,
    * @param path path to examine
-   * @return true if the path is or is under a magic directory
+   * @return true if writing a file to the path triggers a "magic" write.
    */
   public boolean isMagicCommitPath(Path path) {
     return committerIntegration.isMagicCommitPath(path);
   }
 
+  /**
+   * Predicate: is a path under a magic commit path?
+   * True if magic commit is enabled and the path is under __magic,
+   * irrespective of file type.
+   * @param path path to examine
+   * @return true if the path is in a magic dir and the FS has magic writes enabled.
+   */
+  private boolean isUnderMagicCommitPath(Path path) {
+    return committerIntegration.isUnderMagicPath(path);
+  }
+
   /**
    * Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
    * Override superclass so as to disable symlink resolution as symlinks
@@ -4910,7 +4789,7 @@ public RemoteIterator<LocatedFileStatus> listFiles(Path f,
     return toLocatedFileStatusIterator(
         trackDurationAndSpan(INVOCATION_LIST_FILES, path, () ->
             innerListFiles(path, recursive,
-                new Listing.AcceptFilesOnly(path), null, true, false)));
+                new Listing.AcceptFilesOnly(path), null)));
   }
 
   /**
@@ -4929,64 +4808,22 @@ public RemoteIterator<S3ALocatedFileStatus> listFilesAndEmptyDirectories(
     return trackDurationAndSpan(INVOCATION_LIST_FILES, path, () ->
         innerListFiles(path, recursive,
             Listing.ACCEPT_ALL_BUT_S3N,
-            null, true, false));
-  }
-
-  /**
-   * Recursive List of files and empty directories, force metadatastore
-   * to act like it is non-authoritative.
-   * @param f path to list from
-   * @param recursive recursive listing?
-   * @return an iterator.
-   * @throws IOException failure
-   */
-  @InterfaceAudience.Private
-  @Retries.RetryTranslated
-  @AuditEntryPoint
-  public RemoteIterator<S3ALocatedFileStatus> listFilesAndEmptyDirectoriesForceNonAuth(
-      Path f, boolean recursive) throws IOException {
-    final Path path = qualify(f);
-    return trackDurationAndSpan(INVOCATION_LIST_FILES, path, () ->
-        innerListFiles(path, recursive,
-            Listing.ACCEPT_ALL_BUT_S3N,
-            null, true, true));
+            null));
   }
 
   /**
    * List files under the path.
    * <ol>
    *   <li>
-   *     If the path is authoritative on the client,
-   *     only S3Guard will be queried.
-   *   </li>
-   *   <li>
-   *     Otherwise, the S3Guard values are returned first, then the S3
-   *     entries will be retrieved and returned if not already listed.</li>
-   *   <li>
-   *     when collectTombstones} is true, S3Guard tombstones will
-   *     be used to filter out deleted files.
-   *     They MUST be used for normal listings; it is only for
-   *     deletion and low-level operations that they MAY be bypassed.
-   *   </li>
-   *   <li>
    *     The optional {@code status} parameter will be used to skip the
    *     initial getFileStatus call.
    *   </li>
    * </ol>
    *
-   * In case of recursive listing, if any of the directories reachable from
-   * the path are not authoritative on the client, this method will query S3
-   * for all the directories in the listing in addition to returning S3Guard
-   * entries.
-   *
    * @param f path
    * @param recursive recursive listing?
    * @param acceptor file status filter
    * @param status optional status of path to list.
-   * @param collectTombstones should tombstones be collected from S3Guard?
-   * @param forceNonAuthoritativeMS forces metadata store to act like non
-   *                                authoritative. This is useful when
-   *                                listFiles output is used by import tool.
    * @return an iterator over the listing.
    * @throws IOException failure
    */
@@ -4995,9 +4832,7 @@ private RemoteIterator<S3ALocatedFileStatus> innerListFiles(
       final Path f,
       final boolean recursive,
       final Listing.FileStatusAcceptor acceptor,
-      final S3AFileStatus status,
-      final boolean collectTombstones,
-      final boolean forceNonAuthoritativeMS) throws IOException {
+      final S3AFileStatus status) throws IOException {
     Path path = qualify(f);
     LOG.debug("listFiles({}, {})", path, recursive);
     try {
@@ -5014,9 +4849,7 @@ private RemoteIterator<S3ALocatedFileStatus> innerListFiles(
               listing.getListFilesAssumingDir(path,
                       recursive,
                       acceptor,
-                      collectTombstones,
-                      forceNonAuthoritativeMS,
-                      getActiveAuditSpan());
+                  getActiveAuditSpan());
       // If there are no list entries present, we
       // fallback to file existence check as the path
       // can be a file or empty directory.
@@ -5242,15 +5075,31 @@ public boolean hasPathCapability(final Path path, final String capability)
     case STORE_CAPABILITY_DIRECTORY_MARKER_AWARE:
       return true;
 
-    /*
+    // etags are avaialable in listings, but they
+    // are not consistent across renames.
+    // therefore, only availability is declared
+    case CommonPathCapabilities.ETAGS_AVAILABLE:
+      return true;
+
+      /*
      * Marker policy capabilities are handed off.
      */
     case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP:
     case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE:
     case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE:
+      return getDirectoryMarkerPolicy().hasPathCapability(path, cap);
+
+     // keep for a magic path or if the policy retains it
     case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP:
+      return keepDirectoryMarkers(path);
+    // delete is the opposite of keep
     case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE:
-      return getDirectoryMarkerPolicy().hasPathCapability(path, cap);
+      return !keepDirectoryMarkers(path);
+
+    // create file options
+    case FS_S3A_CREATE_PERFORMANCE:
+    case FS_S3A_CREATE_HEADER:
+      return true;
 
     default:
       return super.hasPathCapability(p, cap);
@@ -5289,47 +5138,38 @@ public AWSCredentialProviderList shareCredentials(final String purpose) {
     return credentials.share();
   }
 
-  @VisibleForTesting
-  public ITtlTimeProvider getTtlTimeProvider() {
-    return ttlTimeProvider;
-  }
-
-  @VisibleForTesting
-  protected void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider) {
-    this.ttlTimeProvider = ttlTimeProvider;
-    metadataStore.setTtlTimeProvider(ttlTimeProvider);
-  }
-
   /**
    * This is a proof of concept of a select API.
    * @param source path to source data
-   * @param expression select expression
    * @param options request configuration from the builder.
-   * @param providedStatus any passed in status
+   * @param fileInformation any passed in information.
    * @return the stream of the results
    * @throws IOException IO failure
    */
   @Retries.RetryTranslated
   @AuditEntryPoint
   private FSDataInputStream select(final Path source,
-      final String expression,
       final Configuration options,
-      final Optional<S3AFileStatus> providedStatus)
+      final OpenFileSupport.OpenFileInformation fileInformation)
       throws IOException {
-    final AuditSpan auditSpan = entryPoint(OBJECT_SELECT_REQUESTS, source);
     requireSelectSupport(source);
+    final AuditSpan auditSpan = entryPoint(OBJECT_SELECT_REQUESTS, source);
     final Path path = makeQualified(source);
+    String expression = fileInformation.getSql();
     final S3AFileStatus fileStatus = extractOrFetchSimpleFileStatus(path,
-        providedStatus);
+        fileInformation);
 
     // readahead range can be dynamically set
-    long ra = options.getLong(READAHEAD_RANGE, readAhead);
-    S3ObjectAttributes objectAttributes = createObjectAttributes(fileStatus);
-    S3AReadOpContext readContext = createReadContext(fileStatus, inputPolicy,
-        changeDetectionPolicy, ra, auditSpan);
+    S3ObjectAttributes objectAttributes = createObjectAttributes(
+        path, fileStatus);
+    ChangeDetectionPolicy changePolicy = fileInformation.getChangePolicy();
+    S3AReadOpContext readContext = createReadContext(
+        fileStatus,
+        auditSpan);
+    fileInformation.applyOptions(readContext);
 
-    if (changeDetectionPolicy.getSource() != ChangeDetectionPolicy.Source.None
-        && fileStatus.getETag() != null) {
+    if (changePolicy.getSource() != ChangeDetectionPolicy.Source.None
+        && fileStatus.getEtag() != null) {
       // if there is change detection, and the status includes at least an
       // etag,
       // check that the object metadata lines up with what is expected
@@ -5340,7 +5180,7 @@ private FSDataInputStream select(final Path source,
       // version in the final read; nor can we check the etag match)
       ChangeTracker changeTracker =
           new ChangeTracker(uri.toString(),
-              changeDetectionPolicy,
+              changePolicy,
               readContext.getS3AStatisticsContext()
                   .newInputStreamStatistics()
                   .getChangeTrackerStatistics(),
@@ -5378,38 +5218,36 @@ private void requireSelectSupport(final Path source) throws
   }
 
   /**
-   * Extract the status from the optional parameter, querying
-   * S3Guard/s3 if it is absent.
-   * @param path path of the status
-   * @param optStatus optional status
+   * Get the file status of the source file.
+   * If in the fileInformation parameter return that
+   * if not found, issue a HEAD request, looking for a
+   * file only.
+   * @param path path of the file to open
+   * @param fileInformation information on the file to open
    * @return a file status
-   * @throws FileNotFoundException if there is no normal file at that path
+   * @throws FileNotFoundException if a HEAD request found no file
    * @throws IOException IO failure
    */
   private S3AFileStatus extractOrFetchSimpleFileStatus(
-      final Path path, final Optional<S3AFileStatus> optStatus)
+      final Path path,
+      final OpenFileSupport.OpenFileInformation fileInformation)
       throws IOException {
-    S3AFileStatus fileStatus;
-    if (optStatus.isPresent()) {
-      fileStatus = optStatus.get();
-    } else {
-      // this looks at S3guard and gets any type of status back,
-      // if it falls back to S3 it does a HEAD only.
-      // therefore: if there is no S3Guard and there is a dir, this
-      // will raise a FileNotFoundException
+    S3AFileStatus fileStatus = fileInformation.getStatus();
+    if (fileStatus == null) {
+      // we check here for the passed in status
+      // being a directory
       fileStatus = innerGetFileStatus(path, false,
           StatusProbeEnum.HEAD_ONLY);
     }
-    // we check here for the passed in status or the S3Guard value
-    // for being a directory
     if (fileStatus.isDirectory()) {
       throw new FileNotFoundException(path.toString() + " is a directory");
     }
+
     return fileStatus;
   }
 
   /**
-   * Initiate the open or select operation.
+   * Initiate the open() or select() operation.
    * This is invoked from both the FileSystem and FileContext APIs.
    * It's declared as an audit entry point but the span creation is pushed
    * down into the open/select methods it ultimately calls.
@@ -5428,54 +5266,17 @@ public CompletableFuture<FSDataInputStream> openFileWithOptions(
       final Path rawPath,
       final OpenFileParameters parameters) throws IOException {
     final Path path = qualify(rawPath);
-    Configuration options = parameters.getOptions();
-    Set<String> mandatoryKeys = parameters.getMandatoryKeys();
-    String sql = options.get(SelectConstants.SELECT_SQL, null);
-    boolean isSelect = sql != null;
-    // choice of keys depends on open type
-    if (isSelect) {
-      rejectUnknownMandatoryKeys(
-          mandatoryKeys,
-          InternalSelectConstants.SELECT_OPTIONS,
-          "for " + path + " in S3 Select operation");
-    } else {
-      rejectUnknownMandatoryKeys(
-          mandatoryKeys,
-          InternalConstants.STANDARD_OPENFILE_KEYS,
-          "for " + path + " in non-select file I/O");
-    }
-    FileStatus providedStatus = parameters.getStatus();
-    S3AFileStatus fileStatus;
-    if (providedStatus != null) {
-      Preconditions.checkArgument(path.equals(providedStatus.getPath()),
-          "FileStatus parameter is not for the path %s: %s",
-          path, providedStatus);
-      if (providedStatus instanceof S3AFileStatus) {
-        // can use this status to skip our own probes,
-        // including etag and version.
-        LOG.debug("File was opened with a supplied S3AFileStatus;"
-            + " skipping getFileStatus call in open() operation: {}",
-            providedStatus);
-        fileStatus = (S3AFileStatus) providedStatus;
-      } else if (providedStatus instanceof S3ALocatedFileStatus) {
-        LOG.debug("File was opened with a supplied S3ALocatedFileStatus;"
-            + " skipping getFileStatus call in open() operation: {}",
-            providedStatus);
-        fileStatus = ((S3ALocatedFileStatus) providedStatus).toS3AFileStatus();
-      } else {
-        LOG.debug("Ignoring file status {}", providedStatus);
-        fileStatus = null;
-      }
-    } else {
-      fileStatus = null;
-    }
-    Optional<S3AFileStatus> ost = Optional.ofNullable(fileStatus);
+    OpenFileSupport.OpenFileInformation fileInformation =
+        openFileHelper.prepareToOpenFile(
+            path,
+            parameters,
+            getDefaultBlockSize());
     CompletableFuture<FSDataInputStream> result = new CompletableFuture<>();
-    if (!isSelect) {
+    if (!fileInformation.isS3Select()) {
       // normal path.
       unboundedThreadPool.submit(() ->
           LambdaUtils.eval(result,
-              () -> open(path, Optional.of(options), ost)));
+              () -> executeOpen(path, fileInformation)));
     } else {
       // it is a select statement.
       // fail fast if the operation is not available
@@ -5483,7 +5284,7 @@ public CompletableFuture<FSDataInputStream> openFileWithOptions(
       // submit the query
       unboundedThreadPool.submit(() ->
           LambdaUtils.eval(result,
-              () -> select(path, sql, options, ost)));
+              () -> select(path, parameters.getOptions(), fileInformation)));
     }
     return result;
   }
@@ -5530,10 +5331,8 @@ public StoreContext createStoreContext() {
         .setInputPolicy(getInputPolicy())
         .setChangeDetectionPolicy(changeDetectionPolicy)
         .setMultiObjectDeleteEnabled(enableMultiObjectsDelete)
-        .setMetadataStore(metadataStore)
         .setUseListV1(useListV1)
         .setContextAccessors(new ContextAccessorsImpl())
-        .setTimeProvider(getTtlTimeProvider())
         .setAuditor(getAuditor())
         .setEnableCSE(isCSEEnabled)
         .build();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputPolicy.java
index c018410e11767..b90d0f2a61605 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputPolicy.java
@@ -18,32 +18,46 @@
 
 package org.apache.hadoop.fs.s3a;
 
+import javax.annotation.Nullable;
+import java.util.Collection;
+import java.util.Locale;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Locale;
 
-import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_VECTOR;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
 
 /**
- * Filesystem input policy.
+ * Stream input policy.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public enum S3AInputPolicy {
 
-  Normal(INPUT_FADV_NORMAL),
-  Sequential(INPUT_FADV_SEQUENTIAL),
-  Random(INPUT_FADV_RANDOM);
+  Normal(FS_OPTION_OPENFILE_READ_POLICY_DEFAULT, false, true),
+  Random(FS_OPTION_OPENFILE_READ_POLICY_RANDOM, true, false),
+  Sequential(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL, false, false);
 
-  private static final Logger LOG =
-      LoggerFactory.getLogger(S3AInputPolicy.class);
+  /** Policy name. */
   private final String policy;
 
-  S3AInputPolicy(String policy) {
+  /** Is this random IO? */
+  private final boolean randomIO;
+
+  /** Is this an adaptive policy? */
+  private final boolean adaptive;
+
+  S3AInputPolicy(String policy,
+      boolean randomIO,
+      boolean adaptive) {
     this.policy = policy;
+    this.randomIO = randomIO;
+    this.adaptive = adaptive;
   }
 
   @Override
@@ -51,26 +65,63 @@ public String toString() {
     return policy;
   }
 
+  String getPolicy() {
+    return policy;
+  }
+
+  boolean isRandomIO() {
+    return randomIO;
+  }
+
+  boolean isAdaptive() {
+    return adaptive;
+  }
+
   /**
-   * Choose an FS access policy.
-   * Always returns something,
-   * primarily by downgrading to "normal" if there is no other match.
+   * Choose an access policy.
    * @param name strategy name from a configuration option, etc.
+   * @param defaultPolicy default policy to fall back to.
    * @return the chosen strategy
    */
-  public static S3AInputPolicy getPolicy(String name) {
+  public static S3AInputPolicy getPolicy(
+      String name,
+      @Nullable S3AInputPolicy defaultPolicy) {
     String trimmed = name.trim().toLowerCase(Locale.ENGLISH);
     switch (trimmed) {
-    case INPUT_FADV_NORMAL:
+    case FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE:
+    case FS_OPTION_OPENFILE_READ_POLICY_DEFAULT:
+    case Constants.INPUT_FADV_NORMAL:
       return Normal;
-    case INPUT_FADV_RANDOM:
+
+    // all these options currently map to random IO.
+    case FS_OPTION_OPENFILE_READ_POLICY_RANDOM:
+    case FS_OPTION_OPENFILE_READ_POLICY_VECTOR:
       return Random;
-    case INPUT_FADV_SEQUENTIAL:
+
+    case FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL:
+    case FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE:
       return Sequential;
     default:
-      LOG.warn("Unrecognized " + INPUT_FADVISE + " value: \"{}\"", trimmed);
-      return Normal;
+      return defaultPolicy;
+    }
+  }
+
+  /**
+   * Scan the list of input policies, returning the first one supported.
+   * @param policies list of policies.
+   * @param defaultPolicy fallback
+   * @return a policy or the defaultPolicy, which may be null
+   */
+  public static S3AInputPolicy getFirstSupportedPolicy(
+      Collection<String> policies,
+      @Nullable S3AInputPolicy defaultPolicy) {
+    for (String s : policies) {
+      S3AInputPolicy nextPolicy = S3AInputPolicy.getPolicy(s, null);
+      if (nextPolicy != null) {
+        return nextPolicy;
+      }
     }
+    return defaultPolicy;
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index fe62cf1a3f52c..4b50ab2c04bd9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -19,36 +19,56 @@
 package org.apache.hadoop.fs.s3a;
 
 import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.IntFunction;
 
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.CanSetReadahead;
 import org.apache.hadoop.fs.CanUnbuffer;
 import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
-import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
-import org.apache.hadoop.fs.statistics.IOStatistics;
-import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileRange;
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.StreamCapabilities;
-import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.impl.CombinedFileRange;
+import org.apache.hadoop.fs.VectoredReadUtils;
+import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
+import org.apache.hadoop.fs.s3a.impl.InternalConstants;
+import org.apache.hadoop.fs.s3a.impl.SDKStreamDrainer;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.EOFException;
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-
+import static java.util.Objects.requireNonNull;
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.VectoredReadUtils.isOrderedDisjoint;
+import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges;
+import static org.apache.hadoop.fs.VectoredReadUtils.validateNonOverlappingAndReturnSortedRanges;
+import static org.apache.hadoop.fs.s3a.Invoker.onceTrackingDuration;
 import static org.apache.hadoop.util.StringUtils.toLowerCase;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * The input stream for an S3A object.
@@ -78,6 +98,21 @@ public class S3AInputStream extends FSInputStream implements  CanSetReadahead,
   public static final String OPERATION_OPEN = "open";
   public static final String OPERATION_REOPEN = "re-open";
 
+  /**
+   * This is the maximum temporary buffer size we use while
+   * populating the data in direct byte buffers during a vectored IO
+   * operation. This is to ensure that when a big range of data is
+   * requested in direct byte buffer doesn't leads to OOM errors.
+   */
+  private static final int TMP_BUFFER_MAX_SIZE = 64 * 1024;
+
+  /**
+   * Atomic boolean variable to stop all ongoing vectored read operation
+   * for this input stream. This will be set to true when the stream is
+   * closed or unbuffer is called.
+   */
+  private final AtomicBoolean stopVectoredIOOperations = new AtomicBoolean(false);
+
   /**
    * This is the public position; the one set in {@link #seek(long)}
    * and returned in {@link #getPos()}.
@@ -100,6 +135,11 @@ public class S3AInputStream extends FSInputStream implements  CanSetReadahead,
   private S3ObjectInputStream wrappedStream;
   private final S3AReadOpContext context;
   private final InputStreamCallbacks client;
+
+  /**
+   * Thread pool used for vectored IO operation.
+   */
+  private final ExecutorService boundedThreadPool;
   private final String bucket;
   private final String key;
   private final String pathStr;
@@ -111,6 +151,9 @@ public class S3AInputStream extends FSInputStream implements  CanSetReadahead,
   private S3AInputPolicy inputPolicy;
   private long readahead = Constants.DEFAULT_READAHEAD_RANGE;
 
+  /** Vectored IO context. */
+  private final VectoredIOContext vectoredIOContext;
+
   /**
    * This is the actual position within the object, used by
    * lazy seek to decide whether to seek on the next read or not.
@@ -136,6 +179,15 @@ public class S3AInputStream extends FSInputStream implements  CanSetReadahead,
    */
   private final IOStatistics ioStatistics;
 
+  /**
+   * Threshold for stream reads to switch to
+   * asynchronous draining.
+   */
+  private long asyncDrainThreshold;
+
+  /** Aggregator used to aggregate per thread IOStatistics. */
+  private final IOStatisticsAggregator threadIOStatistics;
+
   /**
    * Create the stream.
    * This does not attempt to open it; that is only done on the first
@@ -143,10 +195,14 @@ public class S3AInputStream extends FSInputStream implements  CanSetReadahead,
    * @param ctx operation context
    * @param s3Attributes object attributes
    * @param client S3 client to use
+   * @param streamStatistics stream io stats.
+   * @param boundedThreadPool thread pool to use.
    */
   public S3AInputStream(S3AReadOpContext ctx,
-      S3ObjectAttributes s3Attributes,
-      InputStreamCallbacks client) {
+                        S3ObjectAttributes s3Attributes,
+                        InputStreamCallbacks client,
+                        S3AInputStreamStatistics streamStatistics,
+                        ExecutorService boundedThreadPool) {
     Preconditions.checkArgument(isNotEmpty(s3Attributes.getBucket()),
         "No Bucket");
     Preconditions.checkArgument(isNotEmpty(s3Attributes.getKey()), "No Key");
@@ -155,12 +211,11 @@ public S3AInputStream(S3AReadOpContext ctx,
     this.context = ctx;
     this.bucket = s3Attributes.getBucket();
     this.key = s3Attributes.getKey();
-    this.pathStr = ctx.dstFileStatus.getPath().toString();
+    this.pathStr = s3Attributes.getPath().toString();
     this.contentLength = l;
     this.client = client;
     this.uri = "s3a://" + this.bucket + "/" + this.key;
-    this.streamStatistics = ctx.getS3AStatisticsContext()
-        .newInputStreamStatistics();
+    this.streamStatistics = streamStatistics;
     this.ioStatistics = streamStatistics.getIOStatistics();
     this.changeTracker = new ChangeTracker(uri,
         ctx.getChangeDetectionPolicy(),
@@ -168,6 +223,10 @@ public S3AInputStream(S3AReadOpContext ctx,
         s3Attributes);
     setInputPolicy(ctx.getInputPolicy());
     setReadahead(ctx.getReadahead());
+    this.asyncDrainThreshold = ctx.getAsyncDrainThreshold();
+    this.boundedThreadPool = boundedThreadPool;
+    this.vectoredIOContext = context.getVectoredIOContext();
+    this.threadIOStatistics = requireNonNull(ctx.getIOStatisticsAggregator());
   }
 
   /**
@@ -180,6 +239,15 @@ private void setInputPolicy(S3AInputPolicy inputPolicy) {
     streamStatistics.inputPolicySet(inputPolicy.ordinal());
   }
 
+  /**
+   * Get the current input policy.
+   * @return input policy.
+   */
+  @VisibleForTesting
+  public S3AInputPolicy getInputPolicy() {
+    return inputPolicy;
+  }
+
   /**
    * Opens up the stream at specified target position and for given length.
    *
@@ -193,7 +261,7 @@ private synchronized void reopen(String reason, long targetPos, long length,
           boolean forceAbort) throws IOException {
 
     if (isObjectStreamOpen()) {
-      closeStream("reopen(" + reason + ")", contentRangeFinish, forceAbort);
+      closeStream("reopen(" + reason + ")", forceAbort, false);
     }
 
     contentRangeFinish = calculateRequestLimit(inputPolicy, targetPos,
@@ -211,21 +279,10 @@ private synchronized void reopen(String reason, long targetPos, long length,
         operation, uri, targetPos);
     changeTracker.maybeApplyConstraint(request);
 
-    DurationTracker tracker = streamStatistics.initiateGetRequest();
-    try {
-      object = Invoker.once(text, uri,
-          () -> client.getObject(request));
-    } catch(IOException e) {
-      // input function failed: note it
-      tracker.failed();
-      // and rethrow
-      throw e;
-    } finally {
-      // update the tracker.
-      // this is called after any catch() call will have
-      // set the failed flag.
-      tracker.close();
-    }
+    object = onceTrackingDuration(text, uri,
+        streamStatistics.initiateGetRequest(), () ->
+            client.getObject(request));
+
 
     changeTracker.processResponse(object, operation,
         targetPos);
@@ -333,7 +390,7 @@ private void seekInStream(long targetPos, long length) throws IOException {
       streamStatistics.seekBackwards(diff);
       // if the stream is in "Normal" mode, switch to random IO at this
       // point, as it is indicative of columnar format IO
-      if (inputPolicy.equals(S3AInputPolicy.Normal)) {
+      if (inputPolicy.isAdaptive()) {
         LOG.info("Switching to Random IO seek policy");
         setInputPolicy(S3AInputPolicy.Random);
       }
@@ -348,7 +405,7 @@ private void seekInStream(long targetPos, long length) throws IOException {
 
     // if the code reaches here, the stream needs to be reopened.
     // close the stream; if read the object will be opened at the new pos
-    closeStream("seekInStream()", this.contentRangeFinish, false);
+    closeStream("seekInStream()", false, false);
     pos = targetPos;
   }
 
@@ -366,10 +423,6 @@ public boolean seekToNewSource(long targetPos) throws IOException {
   @Retries.RetryTranslated
   private void lazySeek(long targetPos, long len) throws IOException {
 
-    // With S3Guard, the metadatastore gave us metadata for the file in
-    // open(), so we use a slightly different retry policy, but only on initial
-    // open.  After that, an exception generally means the file has changed
-    // and there is no point retrying anymore.
     Invoker invoker = context.getReadInvoker();
     invoker.maybeRetry(streamStatistics.getOpenOperations() == 0,
         "lazySeek", pathStr, true,
@@ -397,7 +450,7 @@ private void incrementBytesRead(long bytesRead) {
   }
 
   @Override
-  @Retries.RetryTranslated  // Some retries only happen w/ S3Guard, as intended.
+  @Retries.RetryTranslated
   public synchronized int read() throws IOException {
     checkNotClosed();
     if (this.contentLength == 0 || (nextReadPos >= contentLength)) {
@@ -410,10 +463,6 @@ public synchronized int read() throws IOException {
       return -1;
     }
 
-    // With S3Guard, the metadatastore gave us metadata for the file in
-    // open(), so we use a slightly different retry policy.
-    // read() may not be likely to fail, but reopen() does a GET which
-    // certainly could.
     Invoker invoker = context.getReadInvoker();
     int byteRead = invoker.retry("read", pathStr, true,
         () -> {
@@ -466,7 +515,7 @@ private void onReadFailure(IOException ioe, boolean forceAbort) {
           uri, client, object);
     }
     streamStatistics.readException();
-    closeStream("failure recovery", contentRangeFinish, forceAbort);
+    closeStream("failure recovery", forceAbort, false);
   }
 
   /**
@@ -478,7 +527,7 @@ private void onReadFailure(IOException ioe, boolean forceAbort) {
    * @throws IOException if there are other problems
    */
   @Override
-  @Retries.RetryTranslated  // Some retries only happen w/ S3Guard, as intended.
+  @Retries.RetryTranslated
   public synchronized int read(byte[] buf, int off, int len)
       throws IOException {
     checkNotClosed();
@@ -499,10 +548,6 @@ public synchronized int read(byte[] buf, int off, int len)
       return -1;
     }
 
-    // With S3Guard, the metadatastore gave us metadata for the file in
-    // open(), so we use a slightly different retry policy.
-    // read() may not be likely to fail, but reopen() does a GET which
-    // certainly could.
     Invoker invoker = context.getReadInvoker();
 
     streamStatistics.readOperationStarted(nextReadPos, len);
@@ -563,9 +608,9 @@ public synchronized void close() throws IOException {
     if (!closed) {
       closed = true;
       try {
-        // close or abort the stream
-        closeStream("close() operation", this.contentRangeFinish, false);
-        LOG.debug("Statistics of stream {}\n{}", key, streamStatistics);
+        stopVectoredIOOperations.set(true);
+        // close or abort the stream; blocking
+        closeStream("close() operation", false, true);
         // end the client+audit span.
         client.close();
         // this is actually a no-op
@@ -573,26 +618,49 @@ public synchronized void close() throws IOException {
       } finally {
         // merge the statistics back into the FS statistics.
         streamStatistics.close();
+        // Collect ThreadLevel IOStats
+        mergeThreadIOStatistics(streamStatistics.getIOStatistics());
       }
     }
   }
 
+  /**
+   * Merging the current thread's IOStatistics with the current IOStatistics
+   * context.
+   *
+   * @param streamIOStats Stream statistics to be merged into thread
+   *                      statistics aggregator.
+   */
+  private void mergeThreadIOStatistics(IOStatistics streamIOStats) {
+    threadIOStatistics.aggregate(streamIOStats);
+  }
+
   /**
    * Close a stream: decide whether to abort or close, based on
    * the length of the stream and the current position.
    * If a close() is attempted and fails, the operation escalates to
    * an abort.
    *
+   * The close is potentially; a future is returned.
+   * It's the draining of a stream which is time consuming so
+   * worth scheduling on a separate thread.
+   * In stream close, when an abort is issued or when there's no
+   * data to drain, block.
    * This does not set the {@link #closed} flag.
    * @param reason reason for stream being closed; used in messages
-   * @param length length of the stream.
    * @param forceAbort force an abort; used if explicitly requested.
+   * @param blocking should the call block for completion, or is async IO allowed
+   * @return a future for the async operation
    */
   @Retries.OnceRaw
-  private void closeStream(String reason, long length, boolean forceAbort) {
+  private CompletableFuture<Boolean> closeStream(
+      final String reason,
+      final boolean forceAbort,
+      final boolean blocking) {
+
     if (!isObjectStreamOpen()) {
       // steam is already closed
-      return;
+      return CompletableFuture.completedFuture(false);
     }
 
     // if the amount of data remaining in the current request is greater
@@ -601,56 +669,33 @@ private void closeStream(String reason, long length, boolean forceAbort) {
     LOG.debug("Closing stream {}: {}", reason,
         forceAbort ? "abort" : "soft");
     boolean shouldAbort = forceAbort || remaining > readahead;
+    CompletableFuture<Boolean> operation;
+    SDKStreamDrainer drainer = new SDKStreamDrainer(
+        uri,
+        object,
+        wrappedStream,
+        shouldAbort,
+        (int) remaining,
+        streamStatistics,
+        reason);
+
+    if (blocking || shouldAbort || remaining <= asyncDrainThreshold) {
+      // don't bother with async IO if the caller plans to wait for
+      // the result, there's an abort (which is fast), or
+      // there is not much data to read.
+      operation = CompletableFuture.completedFuture(drainer.apply());
 
-    try {
-      if (!shouldAbort) {
-        try {
-          // clean close. This will read to the end of the stream,
-          // so, while cleaner, can be pathological on a multi-GB object
-
-          // explicitly drain the stream
-          long drained = 0;
-          while (wrappedStream.read() >= 0) {
-            drained++;
-          }
-          LOG.debug("Drained stream of {} bytes", drained);
-
-          // now close it
-          wrappedStream.close();
-          // this MUST come after the close, so that if the IO operations fail
-          // and an abort is triggered, the initial attempt's statistics
-          // aren't collected.
-          streamStatistics.streamClose(false, drained);
-        } catch (Exception e) {
-          // exception escalates to an abort
-          LOG.debug("When closing {} stream for {}, will abort the stream",
-              uri, reason, e);
-          shouldAbort = true;
-        }
-      }
-      if (shouldAbort) {
-        // Abort, rather than just close, the underlying stream.  Otherwise, the
-        // remaining object payload is read from S3 while closing the stream.
-        LOG.debug("Aborting stream {}", uri);
-        try {
-          wrappedStream.abort();
-        } catch (Exception e) {
-          LOG.warn("When aborting {} stream after failing to close it for {}",
-              uri, reason, e);
-        }
-        streamStatistics.streamClose(true, remaining);
-      }
-      LOG.debug("Stream {} {}: {}; remaining={} streamPos={},"
-              + " nextReadPos={}," +
-              " request range {}-{} length={}",
-          uri, (shouldAbort ? "aborted" : "closed"), reason,
-          remaining, pos, nextReadPos,
-          contentRangeStart, contentRangeFinish,
-          length);
-    } finally {
-      wrappedStream = null;
-      object = null;
+    } else {
+      LOG.debug("initiating asynchronous drain of {} bytes", remaining);
+      // schedule an async drain/abort
+      operation = client.submit(drainer);
     }
+
+    // either the stream is closed in the blocking call or the async call is
+    // submitted with its own copy of the references
+    wrappedStream = null;
+    object = null;
+    return operation;
   }
 
   /**
@@ -660,17 +705,17 @@ private void closeStream(String reason, long length, boolean forceAbort) {
    *
    * This is potentially very inefficient, and should only be invoked
    * in extreme circumstances. It logs at info for this reason.
+   *
+   * Blocks until the abort is completed.
+   *
    * @return true if the connection was actually reset.
    * @throws IOException if invoked on a closed stream.
    */
   @InterfaceStability.Unstable
   public synchronized boolean resetConnection() throws IOException {
     checkNotClosed();
-    if (isObjectStreamOpen()) {
-      LOG.info("Forced reset of connection to {}", uri);
-      closeStream("reset()", contentRangeFinish, true);
-    }
-    return isObjectStreamOpen();
+    LOG.info("Forcing reset of connection to {}", uri);
+    return awaitFuture(closeStream("reset()", true, true));
   }
 
   @Override
@@ -747,6 +792,7 @@ public String toString() {
       sb.append(" remainingInCurrentRequest=")
           .append(remainingInCurrentRequest());
       sb.append(" ").append(changeTracker);
+      sb.append(" ").append(vectoredIOContext);
       sb.append('\n').append(s);
       sb.append('}');
       return sb.toString();
@@ -766,7 +812,7 @@ public String toString() {
    *
    */
   @Override
-  @Retries.RetryTranslated  // Some retries only happen w/ S3Guard, as intended.
+  @Retries.RetryTranslated
   public void readFully(long position, byte[] buffer, int offset, int length)
       throws IOException {
     checkNotClosed();
@@ -793,6 +839,324 @@ public void readFully(long position, byte[] buffer, int offset, int length)
     }
   }
 
+  /**
+   * {@inheritDoc}.
+   */
+  @Override
+  public int minSeekForVectorReads() {
+    return vectoredIOContext.getMinSeekForVectorReads();
+  }
+
+  /**
+   * {@inheritDoc}.
+   */
+  @Override
+  public int maxReadSizeForVectorReads() {
+    return vectoredIOContext.getMaxReadSizeForVectorReads();
+  }
+
+  /**
+   * {@inheritDoc}
+   * Vectored read implementation for S3AInputStream.
+   * @param ranges the byte ranges to read.
+   * @param allocate the function to allocate ByteBuffer.
+   * @throws IOException IOE if any.
+   */
+  @Override
+  public void readVectored(List<? extends FileRange> ranges,
+                           IntFunction<ByteBuffer> allocate) throws IOException {
+    LOG.debug("Starting vectored read on path {} for ranges {} ", pathStr, ranges);
+    checkNotClosed();
+    if (stopVectoredIOOperations.getAndSet(false)) {
+      LOG.debug("Reinstating vectored read operation for path {} ", pathStr);
+    }
+    List<? extends FileRange> sortedRanges = validateNonOverlappingAndReturnSortedRanges(ranges);
+    for (FileRange range : ranges) {
+      validateRangeRequest(range);
+      CompletableFuture<ByteBuffer> result = new CompletableFuture<>();
+      range.setData(result);
+    }
+
+    if (isOrderedDisjoint(sortedRanges, 1, minSeekForVectorReads())) {
+      LOG.debug("Not merging the ranges as they are disjoint");
+      streamStatistics.readVectoredOperationStarted(sortedRanges.size(), sortedRanges.size());
+      for (FileRange range: sortedRanges) {
+        ByteBuffer buffer = allocate.apply(range.getLength());
+        boundedThreadPool.submit(() -> readSingleRange(range, buffer));
+      }
+    } else {
+      LOG.debug("Trying to merge the ranges as they are not disjoint");
+      List<CombinedFileRange> combinedFileRanges = mergeSortedRanges(sortedRanges,
+              1, minSeekForVectorReads(),
+              maxReadSizeForVectorReads());
+      streamStatistics.readVectoredOperationStarted(sortedRanges.size(), combinedFileRanges.size());
+      LOG.debug("Number of original ranges size {} , Number of combined ranges {} ",
+              ranges.size(), combinedFileRanges.size());
+      for (CombinedFileRange combinedFileRange: combinedFileRanges) {
+        boundedThreadPool.submit(
+            () -> readCombinedRangeAndUpdateChildren(combinedFileRange, allocate));
+      }
+    }
+    LOG.debug("Finished submitting vectored read to threadpool" +
+            " on path {} for ranges {} ", pathStr, ranges);
+  }
+
+  /**
+   * Read the data from S3 for the bigger combined file range and update all the
+   * underlying ranges.
+   * @param combinedFileRange big combined file range.
+   * @param allocate method to create byte buffers to hold result data.
+   */
+  private void readCombinedRangeAndUpdateChildren(CombinedFileRange combinedFileRange,
+                                                  IntFunction<ByteBuffer> allocate) {
+    LOG.debug("Start reading combined range {} from path {} ", combinedFileRange, pathStr);
+    // This reference must be kept till all buffers are populated as this is a
+    // finalizable object which closes the internal stream when gc triggers.
+    S3Object objectRange = null;
+    S3ObjectInputStream objectContent = null;
+    try {
+      objectRange = getS3ObjectAndValidateNotNull("readCombinedFileRange",
+              combinedFileRange.getOffset(),
+              combinedFileRange.getLength());
+      objectContent = objectRange.getObjectContent();
+      populateChildBuffers(combinedFileRange, objectContent, allocate);
+    } catch (Exception ex) {
+      LOG.debug("Exception while reading a range {} from path {} ", combinedFileRange, pathStr, ex);
+      for(FileRange child : combinedFileRange.getUnderlying()) {
+        child.getData().completeExceptionally(ex);
+      }
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+    }
+    LOG.debug("Finished reading range {} from path {} ", combinedFileRange, pathStr);
+  }
+
+  /**
+   * Populate underlying buffers of the child ranges.
+   * @param combinedFileRange big combined file range.
+   * @param objectContent data from s3.
+   * @param allocate method to allocate child byte buffers.
+   * @throws IOException any IOE.
+   */
+  private void populateChildBuffers(CombinedFileRange combinedFileRange,
+                                    S3ObjectInputStream objectContent,
+                                    IntFunction<ByteBuffer> allocate) throws IOException {
+    // If the combined file range just contains a single child
+    // range, we only have to fill that one child buffer else
+    // we drain the intermediate data between consecutive ranges
+    // and fill the buffers one by one.
+    if (combinedFileRange.getUnderlying().size() == 1) {
+      FileRange child = combinedFileRange.getUnderlying().get(0);
+      ByteBuffer buffer = allocate.apply(child.getLength());
+      populateBuffer(child.getLength(), buffer, objectContent);
+      child.getData().complete(buffer);
+    } else {
+      FileRange prev = null;
+      for (FileRange child : combinedFileRange.getUnderlying()) {
+        if (prev != null && prev.getOffset() + prev.getLength() < child.getOffset()) {
+          long drainQuantity = child.getOffset() - prev.getOffset() - prev.getLength();
+          drainUnnecessaryData(objectContent, drainQuantity);
+        }
+        ByteBuffer buffer = allocate.apply(child.getLength());
+        populateBuffer(child.getLength(), buffer, objectContent);
+        child.getData().complete(buffer);
+        prev = child;
+      }
+    }
+  }
+
+  /**
+   * Drain unnecessary data in between ranges.
+   * @param objectContent s3 data stream.
+   * @param drainQuantity how many bytes to drain.
+   * @throws IOException any IOE.
+   */
+  private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQuantity)
+          throws IOException {
+    int drainBytes = 0;
+    int readCount;
+    while (drainBytes < drainQuantity) {
+      if (drainBytes + InternalConstants.DRAIN_BUFFER_SIZE <= drainQuantity) {
+        byte[] drainBuffer = new byte[InternalConstants.DRAIN_BUFFER_SIZE];
+        readCount = objectContent.read(drainBuffer);
+      } else {
+        byte[] drainBuffer = new byte[(int) (drainQuantity - drainBytes)];
+        readCount = objectContent.read(drainBuffer);
+      }
+      drainBytes += readCount;
+    }
+    streamStatistics.readVectoredBytesDiscarded(drainBytes);
+    LOG.debug("{} bytes drained from stream ", drainBytes);
+  }
+
+  /**
+   * Validates range parameters.
+   * In case of S3 we already have contentLength from the first GET request
+   * during an open file operation so failing fast here.
+   * @param range requested range.
+   * @throws EOFException end of file exception.
+   */
+  private void validateRangeRequest(FileRange range) throws EOFException {
+    VectoredReadUtils.validateRangeRequest(range);
+    if(range.getOffset() + range.getLength() > contentLength) {
+      final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s",
+              range.getOffset(), range.getLength(), pathStr);
+      LOG.warn(errMsg);
+      throw new EOFException(errMsg);
+    }
+  }
+
+  /**
+   * Read data from S3 for this range and populate the buffer.
+   * @param range range of data to read.
+   * @param buffer buffer to fill.
+   */
+  private void readSingleRange(FileRange range, ByteBuffer buffer) {
+    LOG.debug("Start reading range {} from path {} ", range, pathStr);
+    // This reference must be kept till all buffers are populated as this is a
+    // finalizable object which closes the internal stream when gc triggers.
+    S3Object objectRange = null;
+    S3ObjectInputStream objectContent = null;
+    try {
+      long position = range.getOffset();
+      int length = range.getLength();
+      objectRange = getS3ObjectAndValidateNotNull("readSingleRange", position, length);
+      objectContent = objectRange.getObjectContent();
+      populateBuffer(length, buffer, objectContent);
+      range.getData().complete(buffer);
+    } catch (Exception ex) {
+      LOG.warn("Exception while reading a range {} from path {} ", range, pathStr, ex);
+      range.getData().completeExceptionally(ex);
+    } finally {
+      IOUtils.cleanupWithLogger(LOG, objectRange, objectContent);
+    }
+    LOG.debug("Finished reading range {} from path {} ", range, pathStr);
+  }
+
+  /**
+   * Get the s3 object for S3 server for a specified range.
+   * Also checks if the vectored io operation has been stopped before and after
+   * the http get request such that we don't waste time populating the buffers.
+   * @param operationName name of the operation for which get object on S3 is called.
+   * @param position position of the object to be read from S3.
+   * @param length length from position of the object to be read from S3.
+   * @return result s3 object.
+   * @throws IOException exception if any.
+   */
+  private S3Object getS3ObjectAndValidateNotNull(final String operationName,
+                                                 final long position,
+                                                 final int length) throws IOException {
+    checkIfVectoredIOStopped();
+    S3Object objectRange = getS3Object(operationName, position, length);
+    if (objectRange.getObjectContent() == null) {
+      throw new PathIOException(uri,
+              "Null IO stream received during " + operationName);
+    }
+    checkIfVectoredIOStopped();
+    return objectRange;
+  }
+
+  /**
+   * Populates the buffer with data from objectContent
+   * till length. Handles both direct and heap byte buffers.
+   * @param length length of data to populate.
+   * @param buffer buffer to fill.
+   * @param objectContent result retrieved from S3 store.
+   * @throws IOException any IOE.
+   */
+  private void populateBuffer(int length,
+                              ByteBuffer buffer,
+                              S3ObjectInputStream objectContent) throws IOException {
+
+    if (buffer.isDirect()) {
+      VectoredReadUtils.readInDirectBuffer(length, buffer,
+          (position, tmp, offset, currentLength) -> {
+            readByteArray(objectContent, tmp, offset, currentLength);
+            return null;
+          });
+      buffer.flip();
+    } else {
+      readByteArray(objectContent, buffer.array(), 0, length);
+    }
+    // update io stats.
+    incrementBytesRead(length);
+  }
+
+
+  /**
+   * Read data into destination buffer from s3 object content.
+   * @param objectContent result from S3.
+   * @param dest destination buffer.
+   * @param offset start offset of dest buffer.
+   * @param length number of bytes to fill in dest.
+   * @throws IOException any IOE.
+   */
+  private void readByteArray(S3ObjectInputStream objectContent,
+                            byte[] dest,
+                            int offset,
+                            int length) throws IOException {
+    int readBytes = 0;
+    while (readBytes < length) {
+      int readBytesCurr = objectContent.read(dest,
+              offset + readBytes,
+              length - readBytes);
+      readBytes +=readBytesCurr;
+      if (readBytesCurr < 0) {
+        throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
+      }
+    }
+  }
+
+  /**
+   * Read data from S3 using a http request with retries.
+   * This also handles if file has been changed while the
+   * http call is getting executed. If the file has been
+   * changed RemoteFileChangedException is thrown.
+   * @param operationName name of the operation for which get object on S3 is called.
+   * @param position position of the object to be read from S3.
+   * @param length length from position of the object to be read from S3.
+   * @return S3Object result s3 object.
+   * @throws IOException exception if any.
+   */
+  private S3Object getS3Object(String operationName, long position,
+                               int length) throws IOException {
+    final GetObjectRequest request = client.newGetRequest(key)
+            .withRange(position, position + length - 1);
+    changeTracker.maybeApplyConstraint(request);
+    DurationTracker tracker = streamStatistics.initiateGetRequest();
+    S3Object objectRange;
+    Invoker invoker = context.getReadInvoker();
+    try {
+      objectRange = invoker.retry(operationName, pathStr, true,
+        () -> {
+          checkIfVectoredIOStopped();
+          return client.getObject(request);
+        });
+
+    } catch (IOException ex) {
+      tracker.failed();
+      throw ex;
+    } finally {
+      tracker.close();
+    }
+    changeTracker.processResponse(objectRange, operationName,
+            position);
+    return objectRange;
+  }
+
+  /**
+   * Check if vectored io operation has been stooped. This happens
+   * when the stream is closed or unbuffer is called.
+   * @throws InterruptedIOException throw InterruptedIOException such
+   *                                that all running vectored io is
+   *                                terminated thus releasing resources.
+   */
+  private void checkIfVectoredIOStopped() throws InterruptedIOException {
+    if (stopVectoredIOOperations.get()) {
+      throw new InterruptedIOException("Stream closed or unbuffer is called");
+    }
+  }
+
   /**
    * Access the input stream statistics.
    * This is for internal testing and may be removed without warning.
@@ -800,6 +1164,7 @@ public void readFully(long position, byte[] buffer, int offset, int length)
    */
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
+  @VisibleForTesting
   public S3AInputStreamStatistics getS3AStreamStatistics() {
     return streamStatistics;
   }
@@ -878,13 +1243,23 @@ public static long validateReadahead(@Nullable Long readahead) {
   /**
    * Closes the underlying S3 stream, and merges the {@link #streamStatistics}
    * instance associated with the stream.
+   * Also sets the {@code stopVectoredIOOperations} flag to true such that
+   * active vectored read operations are terminated. However termination of
+   * old vectored reads are not guaranteed if a new vectored read operation
+   * is initiated after unbuffer is called.
    */
   @Override
   public synchronized void unbuffer() {
     try {
-      closeStream("unbuffer()", contentRangeFinish, false);
+      stopVectoredIOOperations.set(true);
+      closeStream("unbuffer()", false, false);
     } finally {
       streamStatistics.unbuffered();
+      if (inputPolicy.isAdaptive()) {
+        S3AInputPolicy policy = S3AInputPolicy.Random;
+        LOG.debug("Switching to seek policy {} after unbuffer() invoked", policy);
+        setInputPolicy(policy);
+      }
     }
   }
 
@@ -892,8 +1267,10 @@ public synchronized void unbuffer() {
   public boolean hasCapability(String capability) {
     switch (toLowerCase(capability)) {
     case StreamCapabilities.IOSTATISTICS:
+    case StreamCapabilities.IOSTATISTICS_CONTEXT:
     case StreamCapabilities.READAHEAD:
     case StreamCapabilities.UNBUFFER:
+    case StreamCapabilities.VECTOREDIO:
       return true;
     default:
       return false;
@@ -930,6 +1307,14 @@ public interface InputStreamCallbacks extends Closeable {
     @Retries.OnceRaw
     S3Object getObject(GetObjectRequest request);
 
+    /**
+     * Submit some asynchronous work, for example, draining a stream.
+     * @param operation operation to invoke
+     * @param <T> return type
+     * @return a future.
+     */
+    <T> CompletableFuture<T> submit(CallableRaisingIOE<T> operation);
+
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 65a91fe365949..46568ec2a8d3d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.s3guard.MetastoreInstrumentation;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.ChangeTrackerStatistics;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
@@ -42,6 +41,7 @@
 import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
 import org.apache.hadoop.fs.statistics.StreamStatisticNames;
 import org.apache.hadoop.fs.statistics.DurationTracker;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding;
@@ -164,13 +164,7 @@ public class S3AInstrumentation implements Closeable, MetricsSource,
 
   private final MetricsRegistry registry =
       new MetricsRegistry("s3aFileSystem").setContext(CONTEXT);
-  private final MutableQuantiles putLatencyQuantile;
   private final MutableQuantiles throttleRateQuantile;
-  private final MutableQuantiles s3GuardThrottleRateQuantile;
-
-  /** Instantiate this without caring whether or not S3Guard is enabled. */
-  private final S3GuardInstrumentation s3GuardInstrumentation
-      = new S3GuardInstrumentation();
 
   /**
    * This is the IOStatistics store for the S3AFileSystem
@@ -224,10 +218,6 @@ public S3AInstrumentation(URI name) {
 
     //todo need a config for the quantiles interval?
     int interval = 1;
-    putLatencyQuantile = quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
-        "ops", "latency", interval);
-    s3GuardThrottleRateQuantile = quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
-        "events", "frequency (Hz)", interval);
     throttleRateQuantile = quantiles(STORE_IO_THROTTLE_RATE,
         "events", "frequency (Hz)", interval);
 
@@ -677,15 +667,6 @@ public S3AInputStreamStatistics newInputStreamStatistics(
     return new InputStreamStatistics(filesystemStatistics);
   }
 
-  /**
-   * Create a MetastoreInstrumentation instrumentation instance.
-   * There's likely to be at most one instance of this per FS instance.
-   * @return the S3Guard instrumentation point.
-   */
-  public MetastoreInstrumentation getS3GuardInstrumentation() {
-    return s3GuardInstrumentation;
-  }
-
   /**
    * Create a new instance of the committer statistics.
    * @return a new committer statistics instance
@@ -703,9 +684,7 @@ public void close() {
     synchronized (METRICS_SYSTEM_LOCK) {
       // it is critical to close each quantile, as they start a scheduled
       // task in a shared thread pool.
-      putLatencyQuantile.stop();
       throttleRateQuantile.stop();
-      s3GuardThrottleRateQuantile.stop();
       metricsSystem.unregisterSource(metricsSourceName);
       metricsSourceActiveCounter--;
       int activeSources = metricsSourceActiveCounter;
@@ -824,6 +803,10 @@ private final class InputStreamStatistics
     private final AtomicLong readOperations;
     private final AtomicLong readFullyOperations;
     private final AtomicLong seekOperations;
+    private final AtomicLong readVectoredOperations;
+    private final AtomicLong bytesDiscardedInVectoredIO;
+    private final AtomicLong readVectoredIncomingRanges;
+    private final AtomicLong readVectoredCombinedRanges;
 
     /** Bytes read by the application and any when draining streams . */
     private final AtomicLong totalBytesRead;
@@ -857,9 +840,24 @@ private InputStreamStatistics(
               StreamStatisticNames.STREAM_READ_SEEK_BYTES_SKIPPED,
               StreamStatisticNames.STREAM_READ_TOTAL_BYTES,
               StreamStatisticNames.STREAM_READ_UNBUFFERED,
+              StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES,
+              StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES,
+              StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+              StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED,
               StreamStatisticNames.STREAM_READ_VERSION_MISMATCHES)
-          .withGauges(STREAM_READ_GAUGE_INPUT_POLICY)
-          .withDurationTracking(ACTION_HTTP_GET_REQUEST)
+          .withGauges(STREAM_READ_GAUGE_INPUT_POLICY,
+              STREAM_READ_BLOCKS_IN_FILE_CACHE.getSymbol(),
+              STREAM_READ_ACTIVE_PREFETCH_OPERATIONS.getSymbol(),
+              STREAM_READ_ACTIVE_MEMORY_IN_USE.getSymbol()
+              )
+          .withDurationTracking(ACTION_HTTP_GET_REQUEST,
+              ACTION_EXECUTOR_ACQUIRED,
+              StoreStatisticNames.ACTION_FILE_OPENED,
+              StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED,
+              StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED,
+              StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS,
+              StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ,
+              StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ)
           .build();
       setIOStatistics(st);
       aborted = st.getCounterReference(
@@ -890,6 +888,14 @@ private InputStreamStatistics(
           StreamStatisticNames.STREAM_READ_OPERATIONS_INCOMPLETE);
       readOperations = st.getCounterReference(
           StreamStatisticNames.STREAM_READ_OPERATIONS);
+      readVectoredOperations = st.getCounterReference(
+          StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS);
+      bytesDiscardedInVectoredIO =  st.getCounterReference(
+              StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED);
+      readVectoredIncomingRanges = st.getCounterReference(
+              StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES);
+      readVectoredCombinedRanges = st.getCounterReference(
+              StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES);
       readFullyOperations = st.getCounterReference(
           StreamStatisticNames.STREAM_READ_FULLY_OPERATIONS);
       seekOperations = st.getCounterReference(
@@ -920,6 +926,18 @@ private long increment(String name, long value) {
       return incCounter(name, value);
     }
 
+    /**
+     * Increment the Statistic gauge and the local IOStatistics
+     * equivalent.
+     * @param statistic statistic
+     * @param v value.
+     * @return local IOStatistic value
+     */
+    private long incAllGauges(Statistic statistic, long v) {
+      incrementGauge(statistic, v);
+      return incGauge(statistic.getSymbol(), v);
+    }
+
     /**
      * {@inheritDoc}.
      * Increments the number of seek operations,
@@ -1035,6 +1053,25 @@ public void readOperationCompleted(int requested, int actual) {
       }
     }
 
+    @Override
+    public void readVectoredOperationStarted(int numIncomingRanges,
+                                             int numCombinedRanges) {
+      readVectoredIncomingRanges.addAndGet(numIncomingRanges);
+      readVectoredCombinedRanges.addAndGet(numCombinedRanges);
+      readVectoredOperations.incrementAndGet();
+    }
+
+    @Override
+    public void readVectoredBytesDiscarded(int discarded) {
+      bytesDiscardedInVectoredIO.addAndGet(discarded);
+    }
+
+    @Override
+    public void executorAcquired(Duration timeInQueue) {
+      // update the duration fields in the IOStatistics.
+      localIOStatistics().addTimedOperation(ACTION_EXECUTOR_ACQUIRED, timeInQueue);
+    }
+
     /**
      * {@code close()} merges the stream statistics into the filesystem's
      * instrumentation instance.
@@ -1293,6 +1330,43 @@ public DurationTracker initiateGetRequest() {
       return trackDuration(ACTION_HTTP_GET_REQUEST);
     }
 
+    @Override
+    public DurationTracker initiateInnerStreamClose(final boolean abort) {
+      return trackDuration(abort
+          ? StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED
+          : StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED);
+    }
+
+    @Override
+    public DurationTracker prefetchOperationStarted() {
+      incAllGauges(STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, 1);
+      return trackDuration(StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS);
+    }
+
+    @Override
+    public void blockAddedToFileCache() {
+      incAllGauges(STREAM_READ_BLOCKS_IN_FILE_CACHE, 1);
+    }
+
+    @Override
+    public void blockRemovedFromFileCache() {
+      incAllGauges(STREAM_READ_BLOCKS_IN_FILE_CACHE, -1);
+    }
+
+    @Override
+    public void prefetchOperationCompleted() {
+      incAllGauges(STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, -1);
+    }
+
+    @Override
+    public void memoryAllocated(int size) {
+      incAllGauges(STREAM_READ_ACTIVE_MEMORY_IN_USE, size);
+    }
+
+    @Override
+    public void memoryFreed(int size) {
+      incAllGauges(STREAM_READ_ACTIVE_MEMORY_IN_USE, -size);
+    }
   }
 
   /**
@@ -1320,8 +1394,18 @@ private void mergeOutputStreamStatistics(
     incrementCounter(STREAM_WRITE_EXCEPTIONS,
         source.lookupCounterValue(
             StreamStatisticNames.STREAM_WRITE_EXCEPTIONS));
+
     // merge in all the IOStatistics
-    this.getIOStatistics().aggregate(source.getIOStatistics());
+    final IOStatisticsStore sourceIOStatistics = source.getIOStatistics();
+    this.getIOStatistics().aggregate(sourceIOStatistics);
+
+    // propagate any extra values into the FS-level stats.
+    incrementMutableCounter(OBJECT_PUT_REQUESTS.getSymbol(),
+        sourceIOStatistics.counters().get(OBJECT_PUT_REQUESTS.getSymbol()));
+    incrementMutableCounter(
+        COMMITTER_MAGIC_MARKER_PUT.getSymbol(),
+        sourceIOStatistics.counters().get(COMMITTER_MAGIC_MARKER_PUT.getSymbol()));
+
   }
 
   /**
@@ -1378,9 +1462,12 @@ private OutputStreamStatistics(
               STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING.getSymbol())
           .withDurationTracking(
               ACTION_EXECUTOR_ACQUIRED,
+              COMMITTER_MAGIC_MARKER_PUT.getSymbol(),
               INVOCATION_ABORT.getSymbol(),
+              MULTIPART_UPLOAD_COMPLETED.getSymbol(),
               OBJECT_MULTIPART_UPLOAD_ABORTED.getSymbol(),
-              MULTIPART_UPLOAD_COMPLETED.getSymbol())
+              OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(),
+              OBJECT_PUT_REQUESTS.getSymbol())
           .build();
       setIOStatistics(st);
       // these are extracted to avoid lookups on heavily used counters.
@@ -1617,64 +1704,6 @@ public String toString() {
     }
   }
 
-  /**
-   * Instrumentation exported to S3Guard.
-   */
-  private final class S3GuardInstrumentation
-      implements MetastoreInstrumentation {
-
-    @Override
-    public void initialized() {
-      incrementCounter(S3GUARD_METADATASTORE_INITIALIZATION, 1);
-    }
-
-    @Override
-    public void storeClosed() {
-
-    }
-
-    @Override
-    public void throttled() {
-      // counters are incremented by owner.
-    }
-
-    @Override
-    public void retrying() {
-      // counters are incremented by owner.
-    }
-
-    @Override
-    public void recordsDeleted(int count) {
-      incrementCounter(S3GUARD_METADATASTORE_RECORD_DELETES, count);
-    }
-
-    @Override
-    public void recordsRead(int count) {
-      incrementCounter(S3GUARD_METADATASTORE_RECORD_READS, count);
-    }
-
-    @Override
-    public void recordsWritten(int count) {
-      incrementCounter(S3GUARD_METADATASTORE_RECORD_WRITES, count);
-    }
-
-    @Override
-    public void directoryMarkedAuthoritative() {
-      incrementCounter(
-          S3GUARD_METADATASTORE_AUTHORITATIVE_DIRECTORIES_UPDATED,
-          1);
-    }
-
-    @Override
-    public void entryAdded(final long durationNanos) {
-      addValueToQuantiles(
-          S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
-          durationNanos);
-      incrementCounter(S3GUARD_METADATASTORE_PUT_PATH_REQUEST, 1);
-    }
-
-  }
-
   /**
    * Instrumentation exported to S3A Committers.
    * The S3AInstrumentation metrics and
@@ -1700,6 +1729,7 @@ private CommitterStatisticsImpl() {
               COMMITTER_TASKS_SUCCEEDED.getSymbol())
           .withDurationTracking(
               COMMITTER_COMMIT_JOB.getSymbol(),
+              COMMITTER_LOAD_SINGLE_PENDING_FILE.getSymbol(),
               COMMITTER_MATERIALIZE_FILE.getSymbol(),
               COMMITTER_STAGE_FILE_UPLOAD.getSymbol())
           .build();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ALocatedFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ALocatedFileStatus.java
index 725a7fe0971b8..d29afb2e34fb3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ALocatedFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ALocatedFileStatus.java
@@ -19,14 +19,15 @@
 package org.apache.hadoop.fs.s3a;
 
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.EtagSource;
 import org.apache.hadoop.fs.LocatedFileStatus;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 /**
  * {@link LocatedFileStatus} extended to also carry ETag and object version ID.
  */
-public class S3ALocatedFileStatus extends LocatedFileStatus {
+public class S3ALocatedFileStatus extends LocatedFileStatus implements EtagSource {
 
   private static final long serialVersionUID = 3597192103662929338L;
 
@@ -37,12 +38,23 @@ public class S3ALocatedFileStatus extends LocatedFileStatus {
 
   public S3ALocatedFileStatus(S3AFileStatus status, BlockLocation[] locations) {
     super(checkNotNull(status), locations);
-    this.eTag = status.getETag();
+    this.eTag = status.getEtag();
     this.versionId = status.getVersionId();
     isEmptyDirectory = status.isEmptyDirectory();
   }
 
+  /**
+   * @return the S3 object eTag when available, else null.
+   * @deprecated use {@link EtagSource#getEtag()} for
+   * public access.
+   */
+  @Deprecated
   public String getETag() {
+    return getEtag();
+  }
+
+  @Override
+  public String getEtag() {
     return eTag;
   }
 
@@ -77,7 +89,7 @@ public S3AFileStatus toS3AFileStatus() {
         getModificationTime(),
         getBlockSize(),
         getOwner(),
-        getETag(),
+        getEtag(),
         getVersionId());
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOpContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOpContext.java
index aca1fa5e1408f..d37438a9197e1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOpContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AOpContext.java
@@ -20,7 +20,7 @@
 
 import javax.annotation.Nullable;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -38,66 +38,36 @@
 @SuppressWarnings("visibilitymodifier")
 public class S3AOpContext extends ActiveOperationContext {
 
-  final boolean isS3GuardEnabled;
   final Invoker invoker;
   @Nullable final FileSystem.Statistics stats;
-  @Nullable final Invoker s3guardInvoker;
 
   /** FileStatus for "destination" path being operated on. */
   protected final FileStatus dstFileStatus;
 
   /**
-   * Alternate constructor that allows passing in two invokers, the common
-   * one, and another with the S3Guard Retry Policy.
-   * @param isS3GuardEnabled true if s3Guard is active
+   * Constructor.
    * @param invoker invoker, which contains retry policy
-   * @param s3guardInvoker s3guard-specific retry policy invoker
    * @param stats optional stats object
    * @param instrumentation instrumentation to use
    * @param dstFileStatus file status from existence check
    */
-  public S3AOpContext(boolean isS3GuardEnabled, Invoker invoker,
-      @Nullable Invoker s3guardInvoker,
+  public S3AOpContext(Invoker invoker,
       @Nullable FileSystem.Statistics stats,
       S3AStatisticsContext instrumentation,
       FileStatus dstFileStatus) {
 
     super(newOperationId(),
-        instrumentation,
-        null);
+        instrumentation
+    );
     Preconditions.checkNotNull(invoker, "Null invoker arg");
     Preconditions.checkNotNull(instrumentation, "Null instrumentation arg");
     Preconditions.checkNotNull(dstFileStatus, "Null dstFileStatus arg");
-    this.isS3GuardEnabled = isS3GuardEnabled;
-    Preconditions.checkArgument(!isS3GuardEnabled || s3guardInvoker != null,
-        "S3Guard invoker required: S3Guard is enabled.");
+
     this.invoker = invoker;
-    this.s3guardInvoker = s3guardInvoker;
     this.stats = stats;
     this.dstFileStatus = dstFileStatus;
   }
 
-  /**
-   * Constructor using common invoker and retry policy.
-   * @param isS3GuardEnabled true if s3Guard is active
-   * @param invoker invoker, which contains retry policy
-   * @param stats optional stats object
-   * @param instrumentation instrumentation to use
-   * @param dstFileStatus file status from existence check
-   */
-  public S3AOpContext(boolean isS3GuardEnabled,
-      Invoker invoker,
-      @Nullable FileSystem.Statistics stats,
-      S3AStatisticsContext instrumentation,
-      FileStatus dstFileStatus) {
-    this(isS3GuardEnabled, invoker, null, stats, instrumentation,
-        dstFileStatus);
-  }
-
-  public boolean isS3GuardEnabled() {
-    return isS3GuardEnabled;
-  }
-
   public Invoker getInvoker() {
     return invoker;
   }
@@ -107,11 +77,6 @@ public FileSystem.Statistics getStats() {
     return stats;
   }
 
-  @Nullable
-  public Invoker getS3guardInvoker() {
-    return s3guardInvoker;
-  }
-
   public FileStatus getDstFileStatus() {
     return dstFileStatus;
   }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java
index 43398fc4a167b..b19a70dec915e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java
@@ -21,15 +21,17 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
 import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
 import javax.annotation.Nullable;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Read-specific operation context struct.
@@ -44,71 +46,106 @@ public class S3AReadOpContext extends S3AOpContext {
   /**
    * Initial input policy of the stream.
    */
-  private final S3AInputPolicy inputPolicy;
+  private S3AInputPolicy inputPolicy;
 
   /**
    * How to detect and deal with the object being updated during read.
    */
-  private final ChangeDetectionPolicy changeDetectionPolicy;
+  private ChangeDetectionPolicy changeDetectionPolicy;
 
   /**
    * Readahead for GET operations/skip, etc.
    */
-  private final long readahead;
+  private long readahead;
 
-  private final AuditSpan auditSpan;
+  private AuditSpan auditSpan;
+
+  /**
+   * Threshold for stream reads to switch to
+   * asynchronous draining.
+   */
+  private long asyncDrainThreshold;
+
+  /**
+   * Vectored IO context for vectored read api
+   * in {@code S3AInputStream#readVectored(List, IntFunction)}.
+   */
+  private final VectoredIOContext vectoredIOContext;
+
+  /** Thread-level IOStatistics aggregator. **/
+  private final IOStatisticsAggregator ioStatisticsAggregator;
+
+  // S3 reads are prefetched asynchronously using this future pool.
+  private ExecutorServiceFuturePool futurePool;
+
+  // Size in bytes of a single prefetch block.
+  private final int prefetchBlockSize;
+
+  // Size of prefetch queue (in number of blocks).
+  private final int prefetchBlockCount;
 
   /**
    * Instantiate.
    * @param path path of read
-   * @param isS3GuardEnabled true iff S3Guard is enabled.
    * @param invoker invoker for normal retries.
-   * @param s3guardInvoker S3Guard-specific retry invoker.
    * @param stats Fileystem statistics (may be null)
    * @param instrumentation statistics context
    * @param dstFileStatus target file status
-   * @param inputPolicy the input policy
-   * @param changeDetectionPolicy change detection policy.
-   * @param readahead readahead for GET operations/skip, etc.
-   * @param auditSpan active audit
+   * @param vectoredIOContext context for vectored read operation.
+   * @param ioStatisticsAggregator IOStatistics aggregator for each thread.
+   * @param futurePool the ExecutorServiceFuturePool instance used by async prefetches.
+   * @param prefetchBlockSize the size (in number of bytes) of each prefetched block.
+   * @param prefetchBlockCount maximum number of prefetched blocks.
    */
   public S3AReadOpContext(
       final Path path,
-      boolean isS3GuardEnabled,
       Invoker invoker,
-      @Nullable Invoker s3guardInvoker,
       @Nullable FileSystem.Statistics stats,
       S3AStatisticsContext instrumentation,
       FileStatus dstFileStatus,
-      S3AInputPolicy inputPolicy,
-      ChangeDetectionPolicy changeDetectionPolicy,
-      final long readahead,
-      final AuditSpan auditSpan) {
+      VectoredIOContext vectoredIOContext,
+      IOStatisticsAggregator ioStatisticsAggregator,
+      ExecutorServiceFuturePool futurePool,
+      int prefetchBlockSize,
+      int prefetchBlockCount) {
 
-    super(isS3GuardEnabled, invoker, s3guardInvoker, stats, instrumentation,
+    super(invoker, stats, instrumentation,
         dstFileStatus);
-    this.path = checkNotNull(path);
-    this.auditSpan = auditSpan;
+    this.path = requireNonNull(path);
+    this.vectoredIOContext = requireNonNull(vectoredIOContext, "vectoredIOContext");
+    this.ioStatisticsAggregator = ioStatisticsAggregator;
+    this.futurePool = futurePool;
+    Preconditions.checkArgument(
+        prefetchBlockSize > 0, "invalid prefetchBlockSize %d", prefetchBlockSize);
+    this.prefetchBlockSize = prefetchBlockSize;
+    Preconditions.checkArgument(
+        prefetchBlockCount > 0, "invalid prefetchBlockCount %d", prefetchBlockCount);
+    this.prefetchBlockCount = prefetchBlockCount;
+  }
+
+  /**
+   * validate the context.
+   * @return a read operation context ready for use.
+   */
+  public S3AReadOpContext build() {
+    requireNonNull(inputPolicy, "inputPolicy");
+    requireNonNull(changeDetectionPolicy, "changeDetectionPolicy");
+    requireNonNull(auditSpan, "auditSpan");
+    requireNonNull(inputPolicy, "inputPolicy");
     Preconditions.checkArgument(readahead >= 0,
         "invalid readahead %d", readahead);
-    this.inputPolicy = checkNotNull(inputPolicy);
-    this.changeDetectionPolicy = checkNotNull(changeDetectionPolicy);
-    this.readahead = readahead;
+    Preconditions.checkArgument(asyncDrainThreshold >= 0,
+        "invalid drainThreshold %d", asyncDrainThreshold);
+    requireNonNull(ioStatisticsAggregator, "ioStatisticsAggregator");
+    return this;
   }
 
   /**
    * Get invoker to use for read operations.
-   * When S3Guard is enabled we use the S3Guard invoker,
-   * which deals with things like FileNotFoundException
-   * differently.
    * @return invoker to use for read codepaths
    */
   public Invoker getReadInvoker() {
-    if (isS3GuardEnabled) {
-      return s3guardInvoker;
-    } else {
-      return invoker;
-    }
+    return invoker;
   }
 
   /**
@@ -147,6 +184,105 @@ public AuditSpan getAuditSpan() {
     return auditSpan;
   }
 
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public S3AReadOpContext withInputPolicy(final S3AInputPolicy value) {
+    inputPolicy = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public S3AReadOpContext withChangeDetectionPolicy(
+      final ChangeDetectionPolicy value) {
+    changeDetectionPolicy = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public S3AReadOpContext withReadahead(final long value) {
+    readahead = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public S3AReadOpContext withAuditSpan(final AuditSpan value) {
+    auditSpan = value;
+    return this;
+  }
+
+  /**
+   * Set builder value.
+   * @param value new value
+   * @return the builder
+   */
+  public S3AReadOpContext withAsyncDrainThreshold(final long value) {
+    asyncDrainThreshold = value;
+    return this;
+  }
+
+  public long getAsyncDrainThreshold() {
+    return asyncDrainThreshold;
+  }
+
+  /**
+   * Get Vectored IO context for this this read op.
+   * @return vectored IO context.
+   */
+  public VectoredIOContext getVectoredIOContext() {
+    return vectoredIOContext;
+  }
+
+  /**
+   * Return the IOStatistics aggregator.
+   *
+   * @return instance of IOStatisticsAggregator.
+   */
+  public IOStatisticsAggregator getIOStatisticsAggregator() {
+    return ioStatisticsAggregator;
+  }
+
+  /**
+   * Gets the {@code ExecutorServiceFuturePool} used for asynchronous prefetches.
+   *
+   * @return the {@code ExecutorServiceFuturePool} used for asynchronous prefetches.
+   */
+  public ExecutorServiceFuturePool getFuturePool() {
+    return this.futurePool;
+  }
+
+  /**
+   * Gets the size in bytes of a single prefetch block.
+   *
+   * @return the size in bytes of a single prefetch block.
+   */
+  public int getPrefetchBlockSize() {
+    return this.prefetchBlockSize;
+  }
+
+  /**
+   * Gets the size of prefetch queue (in number of blocks).
+   *
+   * @return the size of prefetch queue (in number of blocks).
+   */
+  public int getPrefetchBlockCount() {
+    return this.prefetchBlockCount;
+  }
+
   @Override
   public String toString() {
     final StringBuilder sb = new StringBuilder(
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
index 286dc213b2187..30427f7672a43 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -31,8 +31,7 @@
 import java.util.concurrent.TimeUnit;
 
 import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -77,7 +76,6 @@
  * untranslated exceptions, as well as the translated ones.
  * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html">S3 Error responses</a>
  * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html">Amazon S3 Error Best Practices</a>
- * @see <a href="http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/CommonErrors.html">Dynamo DB Commmon errors</a>
  */
 @SuppressWarnings("visibilitymodifier")  // I want a struct of finals, for real.
 public class S3ARetryPolicy implements RetryPolicy {
@@ -191,10 +189,6 @@ protected Map<Class<? extends Exception>, RetryPolicy> createExceptionMap() {
     policyMap.put(UnknownStoreException.class, fail);
     policyMap.put(InvalidRequestException.class, fail);
 
-    // metadata stores should do retries internally when it makes sense
-    // so there is no point doing another layer of retries after that
-    policyMap.put(MetadataPersistenceException.class, fail);
-
     // once the file has changed, trying again is not going to help
     policyMap.put(RemoteFileChangedException.class, fail);
 
@@ -234,11 +228,6 @@ protected Map<Class<? extends Exception>, RetryPolicy> createExceptionMap() {
     policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
     policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
 
-    // Dynamo DB exceptions
-    // asking for more than you should get. It's a retry but should be logged
-    // trigger sleep
-    policyMap.put(ProvisionedThroughputExceededException.class, throttlePolicy);
-
     return policyMap;
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 3a98aec1b6a8e..8a1947f3e42dc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -27,15 +27,11 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
 import com.amazonaws.retry.RetryUtils;
-import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
-import com.amazonaws.services.dynamodbv2.model.LimitExceededException;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -50,6 +46,7 @@
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
 import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
 import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
 import org.apache.hadoop.fs.s3native.S3xLoginHelper;
 import org.apache.hadoop.net.ConnectTimeoutException;
 import org.apache.hadoop.security.ProviderUtils;
@@ -96,6 +93,8 @@
 
 /**
  * Utility methods for S3A code.
+ * Some methods are marked LimitedPrivate since they are being used in an
+ * external project.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -163,7 +162,6 @@ private S3AUtils() {
    *
    * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html">S3 Error responses</a>
    * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html">Amazon S3 Error Best Practices</a>
-   * @see <a href="http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/CommonErrors.html">Dynamo DB Commmon errors</a>
    * @param operation operation
    * @param path path operated on (must not be null)
    * @param exception amazon exception raised
@@ -213,11 +211,6 @@ public static IOException translateException(@Nullable String operation,
       }
       return new AWSClientIOException(message, exception);
     } else {
-      if (exception instanceof AmazonDynamoDBException) {
-        // special handling for dynamo DB exceptions
-        return translateDynamoDBException(path, message,
-            (AmazonDynamoDBException)exception);
-      }
       IOException ioe;
       AmazonServiceException ase = (AmazonServiceException) exception;
       // this exception is non-null if the service exception is an s3 one
@@ -386,6 +379,7 @@ private static InterruptedIOException translateInterruptedException(
     } else {
       String name = innerCause.getClass().getName();
       if (name.endsWith(".ConnectTimeoutException")
+          || name.endsWith(".ConnectionPoolTimeoutException")
           || name.endsWith("$ConnectTimeoutException")) {
         // TCP connection http timeout from the shaded or unshaded filenames
         // com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException
@@ -401,8 +395,7 @@ private static InterruptedIOException translateInterruptedException(
 
   /**
    * Is the exception an instance of a throttling exception. That
-   * is an AmazonServiceException with a 503 response, any
-   * exception from DynamoDB for limits exceeded, an
+   * is an AmazonServiceException with a 503 response, an
    * {@link AWSServiceThrottledException},
    * or anything which the AWS SDK's RetryUtils considers to be
    * a throttling exception.
@@ -411,8 +404,6 @@ private static InterruptedIOException translateInterruptedException(
    */
   public static boolean isThrottleException(Exception ex) {
     return ex instanceof AWSServiceThrottledException
-        || ex instanceof ProvisionedThroughputExceededException
-        || ex instanceof LimitExceededException
         || (ex instanceof AmazonServiceException
             && 503  == ((AmazonServiceException)ex).getStatusCode())
         || (ex instanceof SdkBaseException
@@ -431,49 +422,6 @@ public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
             ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
   }
 
-  /**
-   * Translate a DynamoDB exception into an IOException.
-   *
-   * @param path path in the DDB
-   * @param message preformatted message for the exception
-   * @param ddbException exception
-   * @return an exception to throw.
-   */
-  public static IOException translateDynamoDBException(final String path,
-      final String message,
-      final AmazonDynamoDBException ddbException) {
-    if (isThrottleException(ddbException)) {
-      return new AWSServiceThrottledException(message, ddbException);
-    }
-    if (ddbException instanceof ResourceNotFoundException) {
-      return (FileNotFoundException) new FileNotFoundException(message)
-          .initCause(ddbException);
-    }
-    final int statusCode = ddbException.getStatusCode();
-    final String errorCode = ddbException.getErrorCode();
-    IOException result = null;
-    // 400 gets used a lot by DDB
-    if (statusCode == 400) {
-      switch (errorCode) {
-      case "AccessDeniedException":
-        result = (IOException) new AccessDeniedException(
-            path,
-            null,
-            ddbException.toString())
-            .initCause(ddbException);
-        break;
-
-      default:
-        result = new AWSBadRequestException(message, ddbException);
-      }
-
-    }
-    if (result ==  null) {
-      result = new AWSServiceIOException(message, ddbException);
-    }
-    return result;
-  }
-
   /**
    * Get low level details of an amazon exception for logging; multi-line.
    * @param e exception
@@ -605,6 +553,7 @@ public static long dateToLong(final Date date) {
   /**
    * The standard AWS provider list for AWS connections.
    */
+  @SuppressWarnings("deprecation")
   public static final List<Class<?>>
       STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
       Arrays.asList(
@@ -691,6 +640,13 @@ public static AWSCredentialProviderList buildAWSProviderList(
     AWSCredentialProviderList providers = new AWSCredentialProviderList();
     for (Class<?> aClass : awsClasses) {
 
+      // List of V1 credential providers that will be migrated with V2 upgrade
+      if (!Arrays.asList("EnvironmentVariableCredentialsProvider",
+              "EC2ContainerCredentialsProviderWrapper", "InstanceProfileCredentialsProvider")
+          .contains(aClass.getSimpleName()) && aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
+        V2Migration.v1ProviderReferenced(aClass.getName());
+      }
+
       if (forbidden.contains(aClass)) {
         throw new IOException(E_FORBIDDEN_AWS_PROVIDER
             + " in option " + key + ": " + aClass);
@@ -884,6 +840,8 @@ public static String lookupPassword(
   /**
    * Get a password from a configuration, including JCEKS files, handling both
    * the absolute key and bucket override.
+   * <br>
+   * <i>Note:</i> LimitedPrivate for ranger repository to get secrets.
    * @param bucket bucket or "" if none known
    * @param conf configuration
    * @param baseKey base key to look up, e.g "fs.s3a.secret.key"
@@ -894,6 +852,7 @@ public static String lookupPassword(
    * @throws IOException on any IO problem
    * @throws IllegalArgumentException bad arguments
    */
+  @InterfaceAudience.LimitedPrivate("Ranger")
   public static String lookupPassword(
       String bucket,
       Configuration conf,
@@ -1152,10 +1111,15 @@ private static Method getFactoryMethod(Class<?> cl, Class<?> returnType,
    * This method does not propagate security provider path information from
    * the S3A property into the Hadoop common provider: callers must call
    * {@link #patchSecurityCredentialProviders(Configuration)} explicitly.
+   *
+   * <br>
+   * <i>Note:</i> LimitedPrivate for ranger repository to set up
+   * per-bucket configurations.
    * @param source Source Configuration object.
    * @param bucket bucket name. Must not be empty.
    * @return a (potentially) patched clone of the original.
    */
+  @InterfaceAudience.LimitedPrivate("Ranger")
   public static Configuration propagateBucketOptions(Configuration source,
       String bucket) {
 
@@ -1248,7 +1212,7 @@ public static ClientConfiguration createAwsConf(Configuration conf,
    * @param conf The Hadoop configuration
    * @param bucket Optional bucket to use to look up per-bucket proxy secrets
    * @param awsServiceIdentifier a string representing the AWS service (S3,
-   * DDB, etc) for which the ClientConfiguration is being created.
+   * etc) for which the ClientConfiguration is being created.
    * @return new AWS client configuration
    * @throws IOException problem creating AWS client configuration
    */
@@ -1265,9 +1229,6 @@ public static ClientConfiguration createAwsConf(Configuration conf,
       case AWS_SERVICE_IDENTIFIER_S3:
         configKey = SIGNING_ALGORITHM_S3;
         break;
-      case AWS_SERVICE_IDENTIFIER_DDB:
-        configKey = SIGNING_ALGORITHM_DDB;
-        break;
       case AWS_SERVICE_IDENTIFIER_STS:
         configKey = SIGNING_ALGORITHM_STS;
         break;
@@ -1351,6 +1312,8 @@ private static void initProtocolSettings(Configuration conf,
   /**
    * Initializes AWS SDK proxy support in the AWS client configuration
    * if the S3A settings enable it.
+   * <br>
+   * <i>Note:</i> LimitedPrivate to provide proxy support in ranger repository.
    *
    * @param conf Hadoop configuration
    * @param bucket Optional bucket to use to look up per-bucket proxy secrets
@@ -1358,6 +1321,7 @@ private static void initProtocolSettings(Configuration conf,
    * @throws IllegalArgumentException if misconfigured
    * @throws IOException problem getting username/secret from password source.
    */
+  @InterfaceAudience.LimitedPrivate("Ranger")
   public static void initProxySupport(Configuration conf,
       String bucket,
       ClientConfiguration awsConf) throws IllegalArgumentException,
@@ -1387,13 +1351,17 @@ public static void initProxySupport(Configuration conf,
         LOG.error(msg);
         throw new IllegalArgumentException(msg);
       }
+      boolean isProxySecured = conf.getBoolean(PROXY_SECURED, false);
       awsConf.setProxyUsername(proxyUsername);
       awsConf.setProxyPassword(proxyPassword);
       awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
       awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+      awsConf.setProxyProtocol(isProxySecured ? Protocol.HTTPS : Protocol.HTTP);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
-                "domain {} as workstation {}", awsConf.getProxyHost(),
+        LOG.debug("Using proxy server {}://{}:{} as user {} with password {} "
+                + "on domain {} as workstation {}",
+            awsConf.getProxyProtocol(),
+            awsConf.getProxyHost(),
             awsConf.getProxyPort(),
             String.valueOf(awsConf.getProxyUsername()),
             awsConf.getProxyPassword(), awsConf.getProxyDomain(),
@@ -1430,21 +1398,16 @@ private static void initUserAgent(Configuration conf,
 
   /**
    * Convert the data of an iterator of {@link S3AFileStatus} to
-   * an array. Given tombstones are filtered out. If the iterator
-   * does return any item, an empty array is returned.
+   * an array.
    * @param iterator a non-null iterator
-   * @param tombstones possibly empty set of tombstones
    * @return a possibly-empty array of file status entries
    * @throws IOException failure
    */
   public static S3AFileStatus[] iteratorToStatuses(
-      RemoteIterator<S3AFileStatus> iterator, Set<Path> tombstones)
+      RemoteIterator<S3AFileStatus> iterator)
       throws IOException {
-    // this will close the span afterwards
-    RemoteIterator<S3AFileStatus> source = filteringRemoteIterator(iterator,
-        st -> !tombstones.contains(st.getPath()));
     S3AFileStatus[] statuses = RemoteIterators
-        .toArray(source, new S3AFileStatus[0]);
+        .toArray(iterator, new S3AFileStatus[0]);
     return statuses;
   }
 
@@ -1519,17 +1482,22 @@ public static <T> List<T> flatmapLocatedFiles(
   /**
    * List located files and filter them as a classic listFiles(path, filter)
    * would do.
+   * This will be incremental, fetching pages async.
+   * While it is rare for job to have many thousands of files, jobs
+   * against versioned buckets may return earlier if there are many
+   * non-visible objects.
    * @param fileSystem filesystem
    * @param path path to list
    * @param recursive recursive listing?
    * @param filter filter for the filename
-   * @return the filtered list of entries
+   * @return interator over the entries.
    * @throws IOException IO failure.
    */
-  public static List<LocatedFileStatus> listAndFilter(FileSystem fileSystem,
+  public static RemoteIterator<LocatedFileStatus> listAndFilter(FileSystem fileSystem,
       Path path, boolean recursive, PathFilter filter) throws IOException {
-    return flatmapLocatedFiles(fileSystem.listFiles(path, recursive),
-        status -> maybe(filter.accept(status.getPath()), status));
+    return filteringRemoteIterator(
+        fileSystem.listFiles(path, recursive),
+        status -> filter.accept(status.getPath()));
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index 5ef99ed6f5c3c..9010f34dc259c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -44,9 +44,12 @@
  * implementing only the deprecated method will work.
  * See https://github.com/apache/hbase-filesystem
  *
+ * @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of
+ * upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.LimitedPrivate("HBoss")
 @InterfaceStability.Evolving
+@Deprecated
 public interface S3ClientFactory {
 
   /**
@@ -101,7 +104,7 @@ final class S3ClientCreationParameters {
     private boolean pathStyleAccess;
 
     /**
-     * This is in the settings awaiting wiring up and testing.
+     * Permit requests to requester pays buckets.
      */
     private boolean requesterPays;
 
@@ -115,6 +118,12 @@ final class S3ClientCreationParameters {
      */
     private String userAgentSuffix = "";
 
+    /**
+     * S3A path.
+     * added in HADOOP-18330
+     */
+    private URI pathUri;
+
     /**
      * List of request handlers to include in the chain
      * of request execution in the SDK.
@@ -168,7 +177,7 @@ public S3ClientCreationParameters withMetrics(
     }
 
     /**
-     * Requester pays option. Not yet wired up.
+     * Set requester pays option.
      * @param value new value
      * @return the builder
      */
@@ -264,5 +273,26 @@ public S3ClientCreationParameters withHeader(
     public Map<String, String> getHeaders() {
       return headers;
     }
+
+    /**
+     * Get the full s3 path.
+     * added in HADOOP-18330
+     * @return path URI
+     */
+    public URI getPathUri() {
+      return pathUri;
+    }
+
+    /**
+     * Set full s3a path.
+     * added in HADOOP-18330
+     * @param value new value
+     * @return the builder
+     */
+    public S3ClientCreationParameters withPathUri(
+        final URI value) {
+      pathUri = value;
+      return this;
+    }
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3GuardExistsRetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3GuardExistsRetryPolicy.java
deleted file mode 100644
index 079c94e445ace..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3GuardExistsRetryPolicy.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.FileNotFoundException;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.retry.RetryPolicy;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_INTERVAL;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_INTERVAL_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_LIMIT;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_LIMIT_DEFAULT;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithProportionalSleep;
-
-/**
- * Slightly-modified retry policy for cases when the file is present in the
- * MetadataStore, but may be still throwing FileNotFoundException from S3.
- */
-public class S3GuardExistsRetryPolicy extends S3ARetryPolicy {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      S3GuardExistsRetryPolicy.class);
-
-  /**
-   * Instantiate.
-   * @param conf configuration to read.
-   */
-  public S3GuardExistsRetryPolicy(Configuration conf) {
-    super(conf);
-  }
-
-  @Override
-  protected Map<Class<? extends Exception>, RetryPolicy> createExceptionMap() {
-    Map<Class<? extends Exception>, RetryPolicy> b = super.createExceptionMap();
-    Configuration conf = getConfiguration();
-
-    // base policy from configuration
-    int limit = conf.getInt(S3GUARD_CONSISTENCY_RETRY_LIMIT,
-        S3GUARD_CONSISTENCY_RETRY_LIMIT_DEFAULT);
-    long interval = conf.getTimeDuration(S3GUARD_CONSISTENCY_RETRY_INTERVAL,
-        S3GUARD_CONSISTENCY_RETRY_INTERVAL_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    RetryPolicy retryPolicy = retryUpToMaximumCountWithProportionalSleep(
-        limit,
-        interval,
-        TimeUnit.MILLISECONDS);
-    LOG.debug("Retrying on recoverable S3Guard table/S3 inconsistencies {}"
-        + " times with an initial interval of {}ms", limit, interval);
-
-    b.put(FileNotFoundException.class, retryPolicy);
-    b.put(RemoteFileChangedException.class, retryPolicy);
-    return b;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
index 69794c04db53c..69c42bfe1471a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListResult.java
@@ -20,7 +20,6 @@
 
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
@@ -28,9 +27,6 @@
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.slf4j.Logger;
 
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
-
 /**
  * API version-independent container for S3 List responses.
  */
@@ -101,25 +97,6 @@ public List<String> getCommonPrefixes() {
     }
   }
 
-  /**
-   * Is the list of object summaries empty
-   * after accounting for tombstone markers (if provided)?
-   * @param accessors callback for key to path mapping.
-   * @param tombstones Set of tombstone markers, or null if not applicable.
-   * @return false if summaries contains objects not accounted for by
-   * tombstones.
-   */
-  public boolean isEmptyOfObjects(
-      final ContextAccessors accessors,
-      final Set<Path> tombstones) {
-    if (tombstones == null) {
-      return getObjectSummaries().isEmpty();
-    }
-    return isEmptyOfKeys(accessors,
-        objectSummaryKeys(),
-        tombstones);
-  }
-
   /**
    * Get the list of keys in the object summary.
    * @return a possibly empty list
@@ -131,55 +108,22 @@ private List<String> objectSummaryKeys() {
   }
 
   /**
-   * Does this listing have prefixes or objects after entries with
-   * tombstones have been stripped?
-   * @param accessors callback for key to path mapping.
-   * @param tombstones Set of tombstone markers, or null if not applicable.
-   * @return true if the reconciled list is non-empty
+   * Does this listing have prefixes or objects?
+   * @return true if the result is non-empty
    */
-  public boolean hasPrefixesOrObjects(
-      final ContextAccessors accessors,
-      final Set<Path> tombstones) {
+  public boolean hasPrefixesOrObjects() {
 
-    return !isEmptyOfKeys(accessors, getCommonPrefixes(), tombstones)
-        || !isEmptyOfObjects(accessors, tombstones);
-  }
-
-  /**
-   * Helper function to determine if a collection of keys is empty
-   * after accounting for tombstone markers (if provided).
-   * @param accessors callback for key to path mapping.
-   * @param keys Collection of path (prefixes / directories or keys).
-   * @param tombstones Set of tombstone markers, or null if not applicable.
-   * @return true if the list is considered empty.
-   */
-  public boolean isEmptyOfKeys(
-      final ContextAccessors accessors,
-      final Collection<String> keys,
-      final Set<Path> tombstones) {
-    if (tombstones == null) {
-      return keys.isEmpty();
-    }
-    for (String key : keys) {
-      Path qualified = accessors.keyToPath(key);
-      if (!tombstones.contains(qualified)) {
-        return false;
-      }
-    }
-    return true;
+    return !(getCommonPrefixes()).isEmpty()
+        || !getObjectSummaries().isEmpty();
   }
 
   /**
    * Does this listing represent an empty directory?
-   * @param contextAccessors callback for key to path mapping.
    * @param dirKey directory key
-   * @param tombstones Set of tombstone markers, or null if not applicable.
    * @return true if the list is considered empty.
    */
   public boolean representsEmptyDirectory(
-      final ContextAccessors contextAccessors,
-      final String dirKey,
-      final Set<Path> tombstones) {
+      final String dirKey) {
     // If looking for an empty directory, the marker must exist but
     // no children.
     // So the listing must contain the marker entry only as an object,
@@ -190,7 +134,7 @@ public boolean representsEmptyDirectory(
   }
 
   /**
-   * Dmp the result at debug level.
+   * Dump the result at debug level.
    * @param log log to use
    */
   public void logAtDebug(Logger log) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
index 5a8dfc7bc8c0e..0a0454854b2ac 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.fs.Path;
 
 /**
- * This class holds attributed of an object independent of the
+ * This class holds attributes of an object independent of the
  * file status type.
  * It is used in {@link S3AInputStream} and the select equivalent.
  * as a way to reduce parameters being passed
@@ -44,6 +44,17 @@ public class S3ObjectAttributes {
   private final String versionId;
   private final long len;
 
+  /**
+   * Constructor.
+   * @param bucket s3 bucket
+   * @param path path
+   * @param key object key
+   * @param serverSideEncryptionAlgorithm current encryption algorithm
+   * @param serverSideEncryptionKey any server side encryption key?
+   * @param len object length
+   * @param eTag optional etag
+   * @param versionId optional version id
+   */
   public S3ObjectAttributes(
       String bucket,
       Path path,
@@ -66,11 +77,11 @@ public S3ObjectAttributes(
   /**
    * Construct from the result of a copy and those parameters
    * which aren't included in an AWS SDK response.
-   * @param path
+   * @param path path
    * @param copyResult copy result.
    * @param serverSideEncryptionAlgorithm current encryption algorithm
    * @param serverSideEncryptionKey any server side encryption key?
-   * @param len
+   * @param len object length
    */
   public S3ObjectAttributes(
       final Path path,
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
index 5eba675cb82c2..6579a2bc3e7d2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
@@ -39,6 +39,6 @@
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
-public final class SharedInstanceCredentialProvider extends
-    IAMInstanceCredentialsProvider {
+@SuppressWarnings("deprecation")
+public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider {
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index a8e08e158cac4..50a2dd5fb3fc2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -41,9 +41,13 @@
  * Please note that users may reference this class name from configuration
  * property fs.s3a.aws.credentials.provider.  Therefore, changing the class name
  * would be a backward-incompatible change.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
+@Deprecated
 public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
 
   public static final String NAME
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index fb28a40c3b8a6..651769ff283bd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -55,6 +55,10 @@ public enum Statistic {
       StoreStatisticNames.ACTION_HTTP_HEAD_REQUEST,
       "HEAD request.",
       TYPE_DURATION),
+  ACTION_FILE_OPENED(
+      StoreStatisticNames.ACTION_FILE_OPENED,
+      "File opened.",
+      TYPE_DURATION),
   ACTION_HTTP_GET_REQUEST(
       StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
       "GET request.",
@@ -107,6 +111,10 @@ public enum Statistic {
       StoreStatisticNames.OP_CREATE,
       "Calls of create()",
       TYPE_DURATION),
+  INVOCATION_CREATE_FILE(
+      StoreStatisticNames.OP_CREATE_FILE,
+      "Calls of createFile()",
+      TYPE_DURATION),
   INVOCATION_CREATE_NON_RECURSIVE(
       StoreStatisticNames.OP_CREATE_NON_RECURSIVE,
       "Calls of createNonRecursive()",
@@ -175,6 +183,10 @@ public enum Statistic {
       StoreStatisticNames.OP_OPEN,
       "Calls of open()",
       TYPE_COUNTER),
+  INVOCATION_OPENFILE(
+      StoreStatisticNames.OP_OPENFILE,
+      "Calls of openFile()",
+      TYPE_COUNTER),
   INVOCATION_RENAME(
       StoreStatisticNames.OP_RENAME,
       "Calls of rename()",
@@ -296,6 +308,32 @@ public enum Statistic {
       StreamStatisticNames.STREAM_READ_OPERATIONS,
       "Count of read() operations in an input stream",
       TYPE_COUNTER),
+  STREAM_READ_VECTORED_OPERATIONS(
+          StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+          "Count of readVectored() operations in an input stream.",
+          TYPE_COUNTER),
+  STREAM_READ_VECTORED_READ_BYTES_DISCARDED(
+          StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED,
+          "Count of bytes discarded during readVectored() operation." +
+                  " in an input stream",
+          TYPE_COUNTER),
+  STREAM_READ_VECTORED_INCOMING_RANGES(
+          StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES,
+          "Count of incoming file ranges during readVectored() operation.",
+          TYPE_COUNTER),
+  STREAM_READ_VECTORED_COMBINED_RANGES(
+          StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES,
+          "Count of combined file ranges during readVectored() operation.",
+          TYPE_COUNTER),
+  STREAM_READ_REMOTE_STREAM_ABORTED(
+      StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED,
+      "Duration of aborting a remote stream during stream IO",
+      TYPE_DURATION),
+  STREAM_READ_REMOTE_STREAM_CLOSED(
+      StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED,
+      "Duration of closing a remote stream during stream IO",
+      TYPE_DURATION),
+
   STREAM_READ_OPERATIONS_INCOMPLETE(
       StreamStatisticNames.STREAM_READ_OPERATIONS_INCOMPLETE,
       "Count of incomplete read() operations in an input stream",
@@ -340,6 +378,22 @@ public enum Statistic {
       StreamStatisticNames.STREAM_READ_TOTAL_BYTES,
       "Total count of bytes read from an input stream",
       TYPE_COUNTER),
+  STREAM_READ_UNBUFFERED(
+      StreamStatisticNames.STREAM_READ_UNBUFFERED,
+      "Total count of input stream unbuffering operations",
+      TYPE_COUNTER),
+  STREAM_READ_BLOCKS_IN_FILE_CACHE(
+      StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE,
+      "Gauge of blocks in disk cache",
+      TYPE_GAUGE),
+  STREAM_READ_ACTIVE_PREFETCH_OPERATIONS(
+      StreamStatisticNames.STREAM_READ_ACTIVE_PREFETCH_OPERATIONS,
+      "Gauge of active prefetches",
+      TYPE_GAUGE),
+  STREAM_READ_ACTIVE_MEMORY_IN_USE(
+      StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE,
+      "Gauge of active memory in use",
+      TYPE_GAUGE),
 
   /* Stream Write statistics */
 
@@ -442,10 +496,19 @@ public enum Statistic {
       "committer_commits_reverted",
       "Count of commits reverted",
       TYPE_COUNTER),
+  COMMITTER_LOAD_SINGLE_PENDING_FILE(
+      "committer_load_single_pending_file",
+      "Duration to load a single pending file in task commit",
+      TYPE_DURATION),
   COMMITTER_MAGIC_FILES_CREATED(
       "committer_magic_files_created",
       "Count of files created under 'magic' paths",
       TYPE_COUNTER),
+
+  COMMITTER_MAGIC_MARKER_PUT(
+      "committer_magic_marker_put",
+      "Duration Tracking of marker files created under 'magic' paths",
+      TYPE_DURATION),
   COMMITTER_MATERIALIZE_FILE(
       "committer_materialize_file",
       "Duration Tracking of time to materialize a file in job commit",
@@ -455,47 +518,6 @@ public enum Statistic {
       "Duration Tracking of files uploaded from a local staging path",
       TYPE_DURATION),
 
-  /* S3guard stats */
-  S3GUARD_METADATASTORE_PUT_PATH_REQUEST(
-      "s3guard_metadatastore_put_path_request",
-      "S3Guard metadata store put one metadata path request",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_PUT_PATH_LATENCY(
-      "s3guard_metadatastore_put_path_latency",
-      "S3Guard metadata store put one metadata path latency",
-      TYPE_QUANTILE),
-  S3GUARD_METADATASTORE_INITIALIZATION(
-      "s3guard_metadatastore_initialization",
-      "S3Guard metadata store initialization times",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_RECORD_DELETES(
-      "s3guard_metadatastore_record_deletes",
-      "S3Guard metadata store records deleted",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_RECORD_READS(
-      "s3guard_metadatastore_record_reads",
-      "S3Guard metadata store records read",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_RECORD_WRITES(
-      "s3guard_metadatastore_record_writes",
-      "S3Guard metadata store records written",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_RETRY("s3guard_metadatastore_retry",
-      "S3Guard metadata store retry events",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_THROTTLED("s3guard_metadatastore_throttled",
-      "S3Guard metadata store throttled events",
-      TYPE_COUNTER),
-  S3GUARD_METADATASTORE_THROTTLE_RATE(
-      "s3guard_metadatastore_throttle_rate",
-      "S3Guard metadata store throttle rate",
-      TYPE_QUANTILE),
-  S3GUARD_METADATASTORE_AUTHORITATIVE_DIRECTORIES_UPDATED(
-      "s3guard_metadatastore_authoritative_directories_updated",
-      "S3Guard metadata store authoritative directories updated from S3",
-      TYPE_COUNTER),
-
-
   /* General Store operations */
   STORE_EXISTS_PROBE(StoreStatisticNames.STORE_EXISTS_PROBE,
       "Store Existence Probe",
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index 9c71ab458624e..db3d0bb13297c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -43,11 +43,14 @@
  *
  * This credential provider must not fail in creation because that will
  * break a chain of credential providers.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
-public class TemporaryAWSCredentialsProvider extends
-    AbstractSessionCredentialsProvider {
+@Deprecated
+public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider {
 
   public static final String NAME
       = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/VectoredIOContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/VectoredIOContext.java
new file mode 100644
index 0000000000000..31f0ae4cb5515
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/VectoredIOContext.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.util.List;
+import java.util.function.IntFunction;
+
+/**
+ * Context related to vectored IO operation.
+ * See {@link S3AInputStream#readVectored(List, IntFunction)}.
+ */
+public class VectoredIOContext {
+
+  /**
+   * What is the smallest reasonable seek that we should group
+   * ranges together during vectored read operation.
+   */
+  private int minSeekForVectorReads;
+
+  /**
+   * What is the largest size that we should group ranges
+   * together during vectored read operation.
+   * Setting this value 0 will disable merging of ranges.
+   */
+  private int maxReadSizeForVectorReads;
+
+  /**
+   * Default no arg constructor.
+   */
+  public VectoredIOContext() {
+  }
+
+  public VectoredIOContext setMinSeekForVectoredReads(int minSeek) {
+    this.minSeekForVectorReads = minSeek;
+    return this;
+  }
+
+  public VectoredIOContext setMaxReadSizeForVectoredReads(int maxSize) {
+    this.maxReadSizeForVectorReads = maxSize;
+    return this;
+  }
+
+  public VectoredIOContext build() {
+    return this;
+  }
+
+  public int getMinSeekForVectorReads() {
+    return minSeekForVectorReads;
+  }
+
+  public int getMaxReadSizeForVectorReads() {
+    return maxReadSizeForVectorReads;
+  }
+
+  @Override
+  public String toString() {
+    return "VectoredIOContext{" +
+            "minSeekForVectorReads=" + minSeekForVectorReads +
+            ", maxReadSizeForVectorReads=" + maxReadSizeForVectorReads +
+            '}';
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
index 2eb4841d9bd06..14ffeed4a55bb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -24,7 +24,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.amazonaws.services.s3.model.AmazonS3Exception;
@@ -40,8 +39,6 @@
 import com.amazonaws.services.s3.model.SelectObjectContentResult;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.model.UploadResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,17 +48,17 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.s3a.api.RequestFactory;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
 import org.apache.hadoop.fs.s3a.select.SelectBinding;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.functional.CallableRaisingIOE;
+import org.apache.hadoop.util.Preconditions;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.s3a.Invoker.*;
 import static org.apache.hadoop.fs.store.audit.AuditingFunctions.withinAuditSpan;
 
@@ -82,7 +79,7 @@
  *   upload process.</li>
  *   <li>Other low-level access to S3 functions, for private use.</li>
  *   <li>Failure handling, including converting exceptions to IOEs.</li>
- *   <li>Integration with instrumentation and S3Guard.</li>
+ *   <li>Integration with instrumentation.</li>
  *   <li>Evolution to add more low-level operations, such as S3 select.</li>
  * </ul>
  *
@@ -147,6 +144,11 @@ public class WriteOperationHelper implements WriteOperations {
    */
   private final RequestFactory requestFactory;
 
+  /**
+   * WriteOperationHelper callbacks.
+   */
+  private final WriteOperationHelperCallbacks writeOperationHelperCallbacks;
+
   /**
    * Constructor.
    * @param owner owner FS creating the helper
@@ -154,13 +156,14 @@ public class WriteOperationHelper implements WriteOperations {
    * @param statisticsContext statistics context
    * @param auditSpanSource source of spans
    * @param auditSpan span to activate
-   *
+   * @param writeOperationHelperCallbacks callbacks used by writeOperationHelper
    */
   protected WriteOperationHelper(S3AFileSystem owner,
       Configuration conf,
       S3AStatisticsContext statisticsContext,
       final AuditSpanSource auditSpanSource,
-      final AuditSpan auditSpan) {
+      final AuditSpan auditSpan,
+      final WriteOperationHelperCallbacks writeOperationHelperCallbacks) {
     this.owner = owner;
     this.invoker = new Invoker(new S3ARetryPolicy(conf),
         this::operationRetried);
@@ -171,6 +174,7 @@ protected WriteOperationHelper(S3AFileSystem owner,
     this.auditSpanSource = auditSpanSource;
     this.auditSpan = checkNotNull(auditSpan);
     this.requestFactory = owner.getRequestFactory();
+    this.writeOperationHelperCallbacks = writeOperationHelperCallbacks;
   }
 
   /**
@@ -236,22 +240,20 @@ private void deactivateAuditSpan() {
    * @param destKey destination key
    * @param inputStream source data.
    * @param length size, if known. Use -1 for not known
-   * @param headers optional map of custom headers.
+   * @param options options for the request
    * @return the request
    */
   @Retries.OnceRaw
   public PutObjectRequest createPutObjectRequest(String destKey,
       InputStream inputStream,
       long length,
-      final Map<String, String> headers) {
+      final PutObjectOptions options) {
     activateAuditSpan();
     ObjectMetadata objectMetadata = newObjectMetadata(length);
-    if (headers != null) {
-      objectMetadata.setUserMetadata(headers);
-    }
     return getRequestFactory().newPutObjectRequest(
         destKey,
         objectMetadata,
+        options,
         inputStream);
   }
 
@@ -259,18 +261,26 @@ public PutObjectRequest createPutObjectRequest(String destKey,
    * Create a {@link PutObjectRequest} request to upload a file.
    * @param dest key to PUT to.
    * @param sourceFile source file
+   * @param options options for the request
    * @return the request
    */
   @Retries.OnceRaw
-  public PutObjectRequest createPutObjectRequest(String dest,
-      File sourceFile) {
+  public PutObjectRequest createPutObjectRequest(
+      String dest,
+      File sourceFile,
+      final PutObjectOptions options) {
     Preconditions.checkState(sourceFile.length() < Integer.MAX_VALUE,
         "File length is too big for a single PUT upload");
     activateAuditSpan();
-    return getRequestFactory().
+    final ObjectMetadata objectMetadata =
+        newObjectMetadata((int) sourceFile.length());
+
+    PutObjectRequest putObjectRequest = getRequestFactory().
         newPutObjectRequest(dest,
-            newObjectMetadata((int) sourceFile.length()),
+            objectMetadata,
+            options,
             sourceFile);
+    return putObjectRequest;
   }
 
   /**
@@ -300,21 +310,20 @@ public ObjectMetadata newObjectMetadata(long length) {
   }
 
   /**
-   * Start the multipart upload process.
-   * Retry policy: retrying, translated.
-   * @param destKey destination of upload
-   * @return the upload result containing the ID
-   * @throws IOException IO problem
+   * {@inheritDoc}
    */
   @Retries.RetryTranslated
-  public String initiateMultiPartUpload(String destKey) throws IOException {
+  public String initiateMultiPartUpload(
+      final String destKey,
+      final PutObjectOptions options)
+      throws IOException {
     LOG.debug("Initiating Multipart upload to {}", destKey);
     try (AuditSpan span = activateAuditSpan()) {
       return retry("initiate MultiPartUpload", destKey, true,
           () -> {
             final InitiateMultipartUploadRequest initiateMPURequest =
                 getRequestFactory().newMultipartUploadRequest(
-                    destKey);
+                    destKey, options);
             return owner.initiateMultipartUpload(initiateMPURequest)
                 .getUploadId();
           });
@@ -324,15 +333,15 @@ public String initiateMultiPartUpload(String destKey) throws IOException {
   /**
    * Finalize a multipart PUT operation.
    * This completes the upload, and, if that works, calls
-   * {@link S3AFileSystem#finishedWrite(String, long, String, String, BulkOperationState)}
+   * {@link S3AFileSystem#finishedWrite(String, long, String, String, org.apache.hadoop.fs.s3a.impl.PutObjectOptions)}
    * to update the filesystem.
    * Retry policy: retrying, translated.
    * @param destKey destination of the commit
    * @param uploadId multipart operation Id
    * @param partETags list of partial uploads
    * @param length length of the upload
+   * @param putOptions put object options
    * @param retrying retrying callback
-   * @param operationState (nullable) operational state for a bulk update
    * @return the result of the operation.
    * @throws IOException on problems.
    */
@@ -342,8 +351,8 @@ private CompleteMultipartUploadResult finalizeMultipartUpload(
       String uploadId,
       List<PartETag> partETags,
       long length,
-      Retried retrying,
-      @Nullable BulkOperationState operationState) throws IOException {
+      PutObjectOptions putOptions,
+      Retried retrying) throws IOException {
     if (partETags.isEmpty()) {
       throw new PathIOException(destKey,
           "No upload parts in multipart upload");
@@ -357,11 +366,11 @@ private CompleteMultipartUploadResult finalizeMultipartUpload(
             final CompleteMultipartUploadRequest request =
                 getRequestFactory().newCompleteMultipartUploadRequest(
                     destKey, uploadId, partETags);
-            return owner.getAmazonS3Client().completeMultipartUpload(
-                  request);
+            return writeOperationHelperCallbacks.completeMultipartUpload(request);
           });
       owner.finishedWrite(destKey, length, uploadResult.getETag(),
-          uploadResult.getVersionId(), operationState);
+          uploadResult.getVersionId(),
+          putOptions);
       return uploadResult;
     }
   }
@@ -377,6 +386,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload(
    * @param length length of the upload
    * @param errorCount a counter incremented by 1 on every error; for
    * use in statistics
+   * @param putOptions put object options
    * @return the result of the operation.
    * @throws IOException if problems arose which could not be retried, or
    * the retry count was exceeded
@@ -387,7 +397,8 @@ public CompleteMultipartUploadResult completeMPUwithRetries(
       String uploadId,
       List<PartETag> partETags,
       long length,
-      AtomicInteger errorCount)
+      AtomicInteger errorCount,
+      PutObjectOptions putOptions)
       throws IOException {
     checkNotNull(uploadId);
     checkNotNull(partETags);
@@ -397,8 +408,8 @@ public CompleteMultipartUploadResult completeMPUwithRetries(
         uploadId,
         partETags,
         length,
-        (text, e, r, i) -> errorCount.incrementAndGet(),
-        null);
+        putOptions,
+        (text, e, r, i) -> errorCount.incrementAndGet());
   }
 
   /**
@@ -554,63 +565,67 @@ public String toString() {
    * Byte length is calculated from the file length, or, if there is no
    * file, from the content length of the header.
    * @param putObjectRequest the request
+   * @param putOptions put object options
    * @return the upload initiated
    * @throws IOException on problems
    */
   @Retries.RetryTranslated
-  public PutObjectResult putObject(PutObjectRequest putObjectRequest)
+  public PutObjectResult putObject(PutObjectRequest putObjectRequest,
+      PutObjectOptions putOptions)
       throws IOException {
     return retry("Writing Object",
         putObjectRequest.getKey(), true,
         withinAuditSpan(getAuditSpan(), () ->
-            owner.putObjectDirect(putObjectRequest)));
+            owner.putObjectDirect(putObjectRequest, putOptions)));
   }
 
   /**
-   * PUT an object via the transfer manager.
+   * PUT an object.
+   *
    * @param putObjectRequest the request
-   * @return the result of the operation
+   * @param putOptions put object options
+   *
    * @throws IOException on problems
    */
   @Retries.RetryTranslated
-  public UploadResult uploadObject(PutObjectRequest putObjectRequest)
+  public void uploadObject(PutObjectRequest putObjectRequest,
+      PutObjectOptions putOptions)
       throws IOException {
-    // no retry; rely on xfer manager logic
-    return retry("Writing Object",
+
+    retry("Writing Object",
         putObjectRequest.getKey(), true,
         withinAuditSpan(getAuditSpan(), () ->
-            owner.executePut(putObjectRequest, null)));
+            owner.putObjectDirect(putObjectRequest, putOptions)));
   }
 
   /**
    * Revert a commit by deleting the file.
-   * Relies on retry code in filesystem
+   * Relies on retry code in filesystem.
+   * Does not attempt to recreate the parent directory
    * @throws IOException on problems
    * @param destKey destination key
-   * @param operationState operational state for a bulk update
    */
   @Retries.OnceTranslated
-  public void revertCommit(String destKey,
-      @Nullable BulkOperationState operationState) throws IOException {
+  public void revertCommit(String destKey) throws IOException {
     once("revert commit", destKey,
         withinAuditSpan(getAuditSpan(), () -> {
           Path destPath = owner.keyToQualifiedPath(destKey);
           owner.deleteObjectAtPath(destPath,
-              destKey, true, operationState);
-          owner.maybeCreateFakeParentDirectory(destPath);
+              destKey, true);
         }));
   }
 
   /**
    * This completes a multipart upload to the destination key via
    * {@code finalizeMultipartUpload()}.
+   * Markers are never deleted on commit; this avoids having to
+   * issue many duplicate deletions.
    * Retry policy: retrying, translated.
    * Retries increment the {@code errorCount} counter.
    * @param destKey destination
    * @param uploadId multipart operation Id
    * @param partETags list of partial uploads
    * @param length length of the upload
-   * @param operationState operational state for a bulk update
    * @return the result of the operation.
    * @throws IOException if problems arose which could not be retried, or
    * the retry count was exceeded
@@ -620,8 +635,7 @@ public CompleteMultipartUploadResult commitUpload(
       String destKey,
       String uploadId,
       List<PartETag> partETags,
-      long length,
-      @Nullable BulkOperationState operationState)
+      long length)
       throws IOException {
     checkNotNull(uploadId);
     checkNotNull(partETags);
@@ -631,32 +645,8 @@ public CompleteMultipartUploadResult commitUpload(
         uploadId,
         partETags,
         length,
-        Invoker.NO_OP,
-        operationState);
-  }
-
-  /**
-   * Initiate a commit operation through any metastore.
-   * @param path path under which the writes will all take place.
-   * @return an possibly null operation state from the metastore.
-   * @throws IOException failure to instantiate.
-   */
-  public BulkOperationState initiateCommitOperation(
-      Path path) throws IOException {
-    return initiateOperation(path, BulkOperationState.OperationType.Commit);
-  }
-
-  /**
-   * Initiate a commit operation through any metastore.
-   * @param path path under which the writes will all take place.
-   * @param operationType operation to initiate
-   * @return an possibly null operation state from the metastore.
-   * @throws IOException failure to instantiate.
-   */
-  public BulkOperationState initiateOperation(final Path path,
-      final BulkOperationState.OperationType operationType) throws IOException {
-    return S3Guard.initiateBulkWrite(owner.getMetadataStore(),
-        operationType, path);
+        PutObjectOptions.keepingDirs(),
+        Invoker.NO_OP);
   }
 
   /**
@@ -732,7 +722,7 @@ public SelectObjectContentResult select(
           try (DurationInfo ignored =
                    new DurationInfo(LOG, "S3 Select operation")) {
             try {
-              return owner.getAmazonS3Client().selectObjectContent(request);
+              return writeOperationHelperCallbacks.selectObjectContent(request);
             } catch (AmazonS3Exception e) {
               LOG.error("Failure of S3 Select request against {}",
                   source);
@@ -774,4 +764,25 @@ public RequestFactory getRequestFactory() {
     return requestFactory;
   }
 
+  /***
+   * Callbacks for writeOperationHelper.
+   */
+  public interface WriteOperationHelperCallbacks {
+
+    /**
+     * Initiates a select request.
+     * @param request selectObjectContent request
+     * @return selectObjectContentResult
+     */
+    SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request);
+
+    /**
+     * Initiates a complete multi-part upload request.
+     * @param request Complete multi-part upload request
+     * @return completeMultipartUploadResult
+     */
+    CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request);
+
+  }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
index 9400ef2c490fe..321390446f705 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java
@@ -25,7 +25,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
@@ -38,12 +37,11 @@
 import com.amazonaws.services.s3.model.SelectObjectContentResult;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
-import com.amazonaws.services.s3.transfer.model.UploadResult;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
 import org.apache.hadoop.util.functional.CallableRaisingIOE;
 
@@ -80,22 +78,25 @@ <T> T retry(String action,
    * @param destKey destination key
    * @param inputStream source data.
    * @param length size, if known. Use -1 for not known
-   * @param headers optional map of custom headers.
+   * @param options options for the request
    * @return the request
    */
   PutObjectRequest createPutObjectRequest(String destKey,
       InputStream inputStream,
       long length,
-      @Nullable Map<String, String> headers);
+      @Nullable PutObjectOptions options);
 
   /**
    * Create a {@link PutObjectRequest} request to upload a file.
    * @param dest key to PUT to.
    * @param sourceFile source file
+   * @param options options for the request
    * @return the request
    */
-  PutObjectRequest createPutObjectRequest(String dest,
-      File sourceFile);
+  PutObjectRequest createPutObjectRequest(
+      String dest,
+      File sourceFile,
+      @Nullable PutObjectOptions options);
 
   /**
    * Callback on a successful write.
@@ -122,11 +123,12 @@ PutObjectRequest createPutObjectRequest(String dest,
    * Start the multipart upload process.
    * Retry policy: retrying, translated.
    * @param destKey destination of upload
+   * @param options options for the put request
    * @return the upload result containing the ID
    * @throws IOException IO problem
    */
   @Retries.RetryTranslated
-  String initiateMultiPartUpload(String destKey) throws IOException;
+  String initiateMultiPartUpload(String destKey, PutObjectOptions options) throws IOException;
 
   /**
    * This completes a multipart upload to the destination key via
@@ -139,6 +141,7 @@ PutObjectRequest createPutObjectRequest(String dest,
    * @param length length of the upload
    * @param errorCount a counter incremented by 1 on every error; for
    * use in statistics
+   * @param putOptions put object options
    * @return the result of the operation.
    * @throws IOException if problems arose which could not be retried, or
    * the retry count was exceeded
@@ -149,7 +152,8 @@ CompleteMultipartUploadResult completeMPUwithRetries(
       String uploadId,
       List<PartETag> partETags,
       long length,
-      AtomicInteger errorCount)
+      AtomicInteger errorCount,
+      PutObjectOptions putOptions)
       throws IOException;
 
   /**
@@ -239,33 +243,37 @@ UploadPartRequest newUploadPartRequest(
    * Byte length is calculated from the file length, or, if there is no
    * file, from the content length of the header.
    * @param putObjectRequest the request
+   * @param putOptions put object options
    * @return the upload initiated
    * @throws IOException on problems
    */
   @Retries.RetryTranslated
-  PutObjectResult putObject(PutObjectRequest putObjectRequest)
+  PutObjectResult putObject(PutObjectRequest putObjectRequest,
+      PutObjectOptions putOptions)
       throws IOException;
 
   /**
    * PUT an object via the transfer manager.
+   *
    * @param putObjectRequest the request
-   * @return the result of the operation
+   * @param putOptions put object options
+   *
    * @throws IOException on problems
    */
   @Retries.RetryTranslated
-  UploadResult uploadObject(PutObjectRequest putObjectRequest)
+  void uploadObject(PutObjectRequest putObjectRequest,
+      PutObjectOptions putOptions)
       throws IOException;
 
   /**
    * Revert a commit by deleting the file.
-   * Relies on retry code in filesystem
+   * No attempt is made to probe for/recreate a parent dir marker
+   * Relies on retry code in filesystem.
    * @throws IOException on problems
    * @param destKey destination key
-   * @param operationState operational state for a bulk update
    */
   @Retries.OnceTranslated
-  void revertCommit(String destKey,
-      @Nullable BulkOperationState operationState) throws IOException;
+  void revertCommit(String destKey) throws IOException;
 
   /**
    * This completes a multipart upload to the destination key via
@@ -276,7 +284,6 @@ void revertCommit(String destKey,
    * @param uploadId multipart operation Id
    * @param partETags list of partial uploads
    * @param length length of the upload
-   * @param operationState operational state for a bulk update
    * @return the result of the operation.
    * @throws IOException if problems arose which could not be retried, or
    * the retry count was exceeded
@@ -286,29 +293,9 @@ CompleteMultipartUploadResult commitUpload(
       String destKey,
       String uploadId,
       List<PartETag> partETags,
-      long length,
-      @Nullable BulkOperationState operationState)
+      long length)
       throws IOException;
 
-  /**
-   * Initiate a commit operation through any metastore.
-   * @param path path under which the writes will all take place.
-   * @return an possibly null operation state from the metastore.
-   * @throws IOException failure to instantiate.
-   */
-  BulkOperationState initiateCommitOperation(
-      Path path) throws IOException;
-
-  /**
-   * Initiate a commit operation through any metastore.
-   * @param path path under which the writes will all take place.
-   * @param operationType operation to initiate
-   * @return an possibly null operation state from the metastore.
-   * @throws IOException failure to instantiate.
-   */
-  BulkOperationState initiateOperation(Path path,
-      BulkOperationState.OperationType operationType) throws IOException;
-
   /**
    * Upload part of a multi-partition file.
    * @param request request
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
index ee5728688b74e..cae4d3ef034e8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
@@ -44,11 +44,13 @@
 import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
 import com.amazonaws.services.s3.model.SSECustomerKey;
 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
+import com.amazonaws.services.s3.model.StorageClass;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 
 /**
  * Factory for S3 objects.
@@ -106,6 +108,12 @@ public interface RequestFactory {
    */
   String getContentEncoding();
 
+  /**
+   * Get the object storage class, return null if none.
+   * @return storage class
+   */
+  StorageClass getStorageClass();
+
   /**
    * Create a new object metadata instance.
    * Any standard metadata headers are added here, for example:
@@ -134,11 +142,12 @@ CopyObjectRequest newCopyObjectRequest(String srcKey,
    * Adds the ACL and metadata
    * @param key key of object
    * @param metadata metadata header
+   * @param options options for the request
    * @param srcfile source file
    * @return the request
    */
   PutObjectRequest newPutObjectRequest(String key,
-      ObjectMetadata metadata, File srcfile);
+      ObjectMetadata metadata, PutObjectOptions options, File srcfile);
 
   /**
    * Create a {@link PutObjectRequest} request.
@@ -146,11 +155,13 @@ PutObjectRequest newPutObjectRequest(String key,
    * operation.
    * @param key key of object
    * @param metadata metadata header
+   * @param options options for the request
    * @param inputStream source data.
    * @return the request
    */
   PutObjectRequest newPutObjectRequest(String key,
       ObjectMetadata metadata,
+      PutObjectOptions options,
       InputStream inputStream);
 
   /**
@@ -183,10 +194,12 @@ AbortMultipartUploadRequest newAbortMultipartUploadRequest(
   /**
    * Start a multipart upload.
    * @param destKey destination object key
+   * @param options options for the request
    * @return the request.
    */
   InitiateMultipartUploadRequest newMultipartUploadRequest(
-      String destKey);
+      String destKey,
+      @Nullable PutObjectOptions options);
 
   /**
    * Complete a multipart upload.
@@ -291,12 +304,9 @@ ListObjectsV2Request newListObjectsV2Request(String key,
   /**
    * Bulk delete request.
    * @param keysToDelete list of keys to delete.
-   * @param quiet should a bulk query be quiet, or should its result list
-   * all deleted keys?
    * @return the request
    */
   DeleteObjectsRequest newBulkDeleteRequest(
-      List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean quiet);
+          List<DeleteObjectsRequest.KeyVersion> keysToDelete);
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
index a5a6dbc84797f..f2963d7319205 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
@@ -210,7 +210,7 @@ private RequestInfo writing(final String verb,
    * @param request request
    * @return true if the transfer manager creates them.
    */
-  public static final boolean
+  public static boolean
       isRequestNotAlwaysInSpan(final Object request) {
     return request instanceof CopyPartRequest
         || request instanceof CompleteMultipartUploadRequest
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
index 7b9e66901ec4c..bc964b430eb91 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
@@ -33,6 +33,8 @@
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 
 import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED_DEFAULT;
 import static org.apache.hadoop.fs.s3a.audit.impl.S3AInternalAuditConstants.AUDIT_SPAN_HANDLER_CONTEXT;
 
 /**
@@ -58,8 +60,14 @@ private AuditIntegration() {
   public static AuditManagerS3A createAndStartAuditManager(
       Configuration conf,
       IOStatisticsStore iostatistics) {
-    ActiveAuditManagerS3A auditManager = new ActiveAuditManagerS3A(
-        requireNonNull(iostatistics));
+    AuditManagerS3A auditManager;
+    if (conf.getBoolean(AUDIT_ENABLED, AUDIT_ENABLED_DEFAULT)) {
+      auditManager = new ActiveAuditManagerS3A(
+          requireNonNull(iostatistics));
+    } else {
+      LOG.debug("auditing is disabled");
+      auditManager = stubAuditManager();
+    }
     auditManager.init(conf);
     auditManager.start();
     LOG.debug("Started Audit Manager {}", auditManager);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java
index 672bcdf7f9c77..10c8f871cad32 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java
@@ -70,4 +70,15 @@ default boolean checkAccess(Path path, S3AFileStatus status, FsAction mode)
    * @return ID
    */
   String getAuditorId();
+
+  /**
+   * Span reference lost from GC operations.
+   * This is only called when an attempt is made to retrieve on
+   * the active thread or when a prune operation is cleaning up.
+   *
+   * @param threadId thread ID.
+   */
+  default void noteSpanReferenceLost(long threadId) {
+
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
index 2b08893df6e82..1d76833f8ceab 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
@@ -34,6 +34,19 @@ private S3AAuditConstants() {
    */
   public static final String UNAUDITED_OPERATION = "unaudited operation";
 
+  /**
+   * Is auditing enabled?
+   * Value: {@value}.
+   */
+  public static final String AUDIT_ENABLED = "fs.s3a.audit.enabled";
+
+  /**
+   * Default auditing flag.
+   * Value: {@value}.
+   */
+  public static final boolean AUDIT_ENABLED_DEFAULT = true;
+
+
   /**
    * Name of class used for audit logs: {@value}.
    */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
index e7e775dafc0d0..3d2102d305c7d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
@@ -23,6 +23,7 @@
 import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.amazonaws.AmazonWebServiceRequest;
 import com.amazonaws.HandlerContextAware;
@@ -33,16 +34,17 @@
 import com.amazonaws.handlers.HandlerBeforeAttemptContext;
 import com.amazonaws.handlers.RequestHandler2;
 import com.amazonaws.http.HttpResponse;
-import com.amazonaws.services.s3.transfer.Transfer;
 import com.amazonaws.services.s3.transfer.internal.TransferStateChangeListener;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.audit.AWSAuditEventCallbacks;
@@ -88,6 +90,11 @@
  * then the IOStatistics counter for {@code AUDIT_FAILURE}
  * is incremented.
  *
+ * Uses the WeakReferenceThreadMap to store spans for threads.
+ * Provided a calling class retains a reference to the span,
+ * the active span will be retained.
+ *
+ *
  */
 @InterfaceAudience.Private
 public final class ActiveAuditManagerS3A
@@ -111,6 +118,14 @@ public final class ActiveAuditManagerS3A
   public static final String NOT_A_WRAPPED_SPAN
       = "Span attached to request is not a wrapped span";
 
+  /**
+   * Arbitrary threshold for triggering pruning on deactivation.
+   * High enough it doesn't happen very often, low enough
+   * that it will happen regularly on a busy system.
+   * Value: {@value}.
+   */
+  static final int PRUNE_THRESHOLD = 10_000;
+
   /**
    * Audit service.
    */
@@ -127,12 +142,27 @@ public final class ActiveAuditManagerS3A
    */
   private WrappingAuditSpan unbondedSpan;
 
+  /**
+   * How many spans have to be deactivated before a prune is triggered?
+   * Fixed as a constant for now unless/until some pressing need
+   * for it to be made configurable ever surfaces.
+   */
+  private final int pruneThreshold = PRUNE_THRESHOLD;
+
+  /**
+   * Count down to next pruning.
+   */
+  private final AtomicInteger deactivationsBeforePrune = new AtomicInteger();
+
   /**
    * Thread local span. This defaults to being
    * the unbonded span.
    */
-  private final ThreadLocal<WrappingAuditSpan> activeSpan =
-      ThreadLocal.withInitial(() -> getUnbondedSpan());
+
+  private final WeakReferenceThreadMap<WrappingAuditSpan> activeSpanMap =
+      new WeakReferenceThreadMap<>(
+          (k) -> getUnbondedSpan(),
+          this::noteSpanReferenceLost);
 
   /**
    * Destination for recording statistics, especially duration/count of
@@ -147,6 +177,7 @@ public final class ActiveAuditManagerS3A
   public ActiveAuditManagerS3A(final IOStatisticsStore iostatistics) {
     super("ActiveAuditManagerS3A");
     this.ioStatisticsStore = iostatistics;
+    this.deactivationsBeforePrune.set(pruneThreshold);
   }
 
   @Override
@@ -178,6 +209,13 @@ protected void serviceStart() throws Exception {
     LOG.debug("Started audit service {}", auditor);
   }
 
+  @Override
+  protected void serviceStop() throws Exception {
+    // clear all references.
+    activeSpanMap.clear();
+    super.serviceStop();
+  }
+
   @Override
   public String toString() {
     final StringBuilder sb = new StringBuilder(super.toString());
@@ -225,7 +263,7 @@ public AuditSpanS3A getActiveAuditSpan() {
    * @return the active WrappingAuditSpan
    */
   private WrappingAuditSpan activeSpan() {
-    return activeSpan.get();
+    return activeSpanMap.getForCurrentThread();
   }
 
   /**
@@ -247,13 +285,66 @@ private AuditSpanS3A setActiveThreadSpan(AuditSpanS3A span) {
    */
   private WrappingAuditSpan switchToActiveSpan(WrappingAuditSpan span) {
     if (span != null && span.isValidSpan()) {
-      activeSpan.set(span);
+      activeSpanMap.setForCurrentThread(span);
     } else {
-      activeSpan.set(unbondedSpan);
+      activeSpanMap.removeForCurrentThread();
     }
     return activeSpan();
   }
 
+  /**
+   * Span reference lost from GC operations.
+   * This is only called when an attempt is made to retrieve on
+   * the active thread or when a prune operation is cleaning up.
+   *
+   * @param threadId thread ID.
+   */
+  private void noteSpanReferenceLost(long threadId) {
+    auditor.noteSpanReferenceLost(threadId);
+  }
+
+  /**
+   * Prune all null weak references, calling the referenceLost
+   * callback for each one.
+   *
+   * non-atomic and non-blocking.
+   * @return the number of entries pruned.
+   */
+  @VisibleForTesting
+  int prune() {
+    return activeSpanMap.prune();
+  }
+
+  /**
+   * remove the span from the reference map, shrinking the map in the process.
+   * if/when a new span is activated in the thread, a new entry will be created.
+   * and if queried for a span, the unbounded span will be automatically
+   * added to the map for this thread ID.
+   *
+   */
+  @VisibleForTesting
+  boolean removeActiveSpanFromMap() {
+    // remove from the map
+    activeSpanMap.removeForCurrentThread();
+    if (deactivationsBeforePrune.decrementAndGet() == 0) {
+      // trigger a prune
+      activeSpanMap.prune();
+      deactivationsBeforePrune.set(pruneThreshold);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Get the map of threads to active spans; allows
+   * for testing of weak reference resolution after GC.
+   * @return the span map
+   */
+  @VisibleForTesting
+  WeakReferenceThreadMap<WrappingAuditSpan> getActiveSpanMap() {
+    return activeSpanMap;
+  }
+
   /**
    * The Span ID in the audit manager is the ID of the auditor,
    * which can be used in the filesystem toString() method
@@ -331,13 +422,7 @@ public List<RequestHandler2> createRequestHandlers()
   @Override
   public TransferStateChangeListener createStateChangeListener() {
     final WrappingAuditSpan span = activeSpan();
-    return new TransferStateChangeListener() {
-      @Override
-      public void transferStateChanged(final Transfer transfer,
-          final Transfer.TransferState state) {
-        switchToActiveSpan(span);
-      }
-    };
+    return (transfer, state) -> switchToActiveSpan(span);
   }
 
   @Override
@@ -641,16 +726,21 @@ public AuditSpanS3A activate() {
      */
     @Override
     public void deactivate() {
-      // no-op for invalid spans,
+
+      // span is inactive; ignore
+      if (!isActive()) {
+        return;
+      }
+      // skipped for invalid spans,
       // so as to prevent the unbounded span from being closed
       // and everything getting very confused.
-      if (!isValid || !isActive()) {
-        return;
+      if (isValid) {
+        // deactivate the span
+        span.deactivate();
       }
-      // deactivate the span
-      span.deactivate();
-      // and go to the unbounded one.
-      switchToActiveSpan(getUnbondedSpan());
+      // remove the span from the reference map,
+      // sporadically triggering a prune operation.
+      removeActiveSpanFromMap();
     }
 
     /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
index 2e51699da6f44..d1ebd922e073d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
@@ -44,8 +44,6 @@
 /**
  * Simple No-op audit manager for use before a real
  * audit chain is set up, and for testing.
- * Audit spans always have a unique ID and the activation/deactivation
- * operations on them will update this audit manager's active span.
  * It does have the service lifecycle, so do
  * create a unique instance whenever used.
  */
@@ -59,14 +57,7 @@ public class NoopAuditManagerS3A extends CompositeService
   /**
    * The inner auditor.
    */
-  private NoopAuditor auditor = NOOP_AUDITOR;
-
-  /**
-   * Thread local span. This defaults to being
-   * the unbonded span.
-   */
-  private final ThreadLocal<AuditSpanS3A> activeSpan =
-      ThreadLocal.withInitial(this::getUnbondedSpan);
+  private final NoopAuditor auditor = NOOP_AUDITOR;
 
   /**
    * ID which is returned as a span ID in the audit event
@@ -160,7 +151,7 @@ public boolean checkAccess(final Path path,
 
   @Override
   public void activate(final AuditSpanS3A span) {
-    activeSpan.set(span);
+    /* no-op */
   }
 
   @Override
@@ -180,6 +171,6 @@ public static AuditSpanS3A createNewSpan(
       final String name,
       final String path1,
       final String path2) {
-    return NOOP_AUDITOR.createSpan(name, path1, path2);
+    return NoopSpan.INSTANCE;
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
index 1f714b0555285..1815285738b0e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java
@@ -28,7 +28,11 @@
 /**
  * Base class for AWS credential providers which
  * take a URI and config in their constructor.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
+@Deprecated
 public abstract class AbstractAWSCredentialProvider
     implements AWSCredentialsProvider {
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
index 30939767e4379..5b1829e096123 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java
@@ -35,13 +35,17 @@
 
 /**
  * Base class for session credential support.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Private
+@Deprecated
 public abstract class AbstractSessionCredentialsProvider
     extends AbstractAWSCredentialProvider {
 
   /** Credentials, created in {@link #init()}. */
-  private AWSCredentials awsCredentials;
+  private volatile AWSCredentials awsCredentials;
 
   /** Atomic flag for on-demand initialization. */
   private final AtomicBoolean initialized = new AtomicBoolean(false);
@@ -50,7 +54,7 @@ public abstract class AbstractSessionCredentialsProvider
    * The (possibly translated) initialization exception.
    * Used for testing.
    */
-  private IOException initializationException;
+  private volatile IOException initializationException;
 
   /**
    * Constructor.
@@ -69,9 +73,9 @@ public AbstractSessionCredentialsProvider(
    * @throws IOException on any failure.
    */
   @Retries.OnceTranslated
-  protected void init() throws IOException {
+  protected synchronized void init() throws IOException {
     // stop re-entrant attempts
-    if (initialized.getAndSet(true)) {
+    if (isInitialized()) {
       return;
     }
     try {
@@ -80,6 +84,8 @@ protected void init() throws IOException {
     } catch (IOException e) {
       initializationException = e;
       throw e;
+    } finally {
+      initialized.set(true);
     }
   }
 
@@ -128,13 +134,15 @@ public AWSCredentials getCredentials() throws SdkBaseException {
     }
     if (awsCredentials == null) {
       throw new CredentialInitializationException(
-          "Provider " + this + " has no credentials");
+          "Provider " + this + " has no credentials: " +
+             (initializationException != null ? initializationException.toString() : ""),
+          initializationException);
     }
     return awsCredentials;
   }
 
   public final boolean hasCredentials() {
-    return awsCredentials == null;
+    return awsCredentials != null;
   }
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
index 104e135ed79df..1e2ac16075aeb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
@@ -60,9 +60,13 @@
  * unless overridden, creating a session name from the current user.
  *
  * Classname is used in configuration files; do not move.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
+@Deprecated
 public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
     Closeable {
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
index 1bb30ed5c0dc9..ca9c518d30048 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
@@ -40,9 +40,13 @@
  * as a non-recoverable failure.
  * <p>
  * It is implicitly public; marked evolving as we can change its semantics.
+ *
+ * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
+ * as part of upgrading S3A to SDK V2. See HADOOP-18073.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
+@Deprecated
 public class IAMInstanceCredentialsProvider
     implements AWSCredentialsProvider, Closeable {
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
index f9e840e776a54..29e815560a8a9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java
@@ -207,9 +207,11 @@ public static MarshalledCredentials requestSessionCredentials(
               stsEndpoint.isEmpty() ? null : stsEndpoint,
               stsRegion)
               .build();
-      return fromSTSCredentials(
-          STSClientFactory.createClientConnection(tokenService, invoker)
-              .requestSessionCredentials(duration, TimeUnit.SECONDS));
+      try (STSClientFactory.STSClient stsClient = STSClientFactory.createClientConnection(
+          tokenService, invoker)) {
+        return fromSTSCredentials(stsClient.requestSessionCredentials(duration,
+            TimeUnit.SECONDS));
+      }
     } catch (SdkClientException e) {
       if (stsRegion.isEmpty()) {
         LOG.error("Region must be provided when requesting session credentials.",
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
index b0a1272256eaa..8bd04744cd8c0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java
@@ -28,7 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.CredentialInitializationException;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.toAWSCredentials;
 
 /**
@@ -40,6 +40,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
+@SuppressWarnings("deprecation")
 public class MarshalledCredentialProvider extends
     AbstractSessionCredentialsProvider {
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RoleModel.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RoleModel.java
index 6676b49b6a45a..cf5d0ac4a210e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RoleModel.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RoleModel.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.util.JsonSerialization;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.util.Preconditions.checkState;
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -45,7 +45,6 @@
  * don't expect to be able to parse everything.
  * It can generate simple models.
  * @see <a href="https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html">Example S3 Policies</a>
- * @see <a href="https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/api-permissions-reference.html">Dynamno DB Permissions</a>
  */
 @InterfaceAudience.LimitedPrivate("Tests")
 @InterfaceStability.Unstable
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
index d4fd5e1b72f72..940742c11e24b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java
@@ -32,7 +32,7 @@
 
 /**
  * Operations, statements and policies covering the operations
- * needed to work with S3 and S3Guard.
+ * needed to work with S3.
  */
 @InterfaceAudience.LimitedPrivate("Tests")
 @InterfaceStability.Unstable
@@ -200,7 +200,7 @@ private RolePolicies() {
 
   /**
    * Actions needed to read a file in S3 through S3A, excluding
-   * S3Guard and SSE-KMS.
+   * SSE-KMS.
    */
   private static final String[] S3_PATH_READ_OPERATIONS =
       new String[]{
@@ -213,7 +213,6 @@ private RolePolicies() {
    * <ol>
    *   <li>bucket-level operations</li>
    *   <li>SSE-KMS key operations</li>
-   *   <li>DynamoDB operations for S3Guard.</li>
    * </ol>
    * As this excludes the bucket list operations, it is not sufficient
    * to read from a bucket on its own.
@@ -230,7 +229,6 @@ private RolePolicies() {
    * Policies which can be applied to bucket resources for read operations.
    * <ol>
    *   <li>SSE-KMS key operations</li>
-   *   <li>DynamoDB operations for S3Guard.</li>
    * </ol>
    */
   public static final String[] S3_BUCKET_READ_OPERATIONS =
@@ -242,7 +240,7 @@ private RolePolicies() {
   /**
    * Actions needed to write data to an S3A Path.
    * This includes the appropriate read operations, but
-   * not SSE-KMS or S3Guard support.
+   * not SSE-KMS support.
    */
   public static final List<String> S3_PATH_RW_OPERATIONS =
       Collections.unmodifiableList(Arrays.asList(new String[]{
@@ -258,7 +256,7 @@ private RolePolicies() {
    * This is purely the extra operations needed for writing atop
    * of the read operation set.
    * Deny these and a path is still readable, but not writeable.
-   * Excludes: bucket-ARN, SSE-KMS and S3Guard permissions.
+   * Excludes: bucket-ARN and SSE-KMS permissions.
    */
   public static final List<String> S3_PATH_WRITE_OPERATIONS =
       Collections.unmodifiableList(Arrays.asList(new String[]{
@@ -270,7 +268,7 @@ private RolePolicies() {
 
   /**
    * Actions needed for R/W IO from the root of a bucket.
-   * Excludes: bucket-ARN, SSE-KMS and S3Guard permissions.
+   * Excludes: bucket-ARN and SSE-KMS permissions.
    */
   public static final List<String> S3_ROOT_RW_OPERATIONS =
       Collections.unmodifiableList(Arrays.asList(new String[]{
@@ -281,79 +279,9 @@ private RolePolicies() {
           S3_ABORT_MULTIPART_UPLOAD,
       }));
 
-  /**
-   * All DynamoDB operations: {@value}.
-   */
-  public static final String DDB_ALL_OPERATIONS = "dynamodb:*";
-
-  /**
-   * Operations needed for DDB/S3Guard Admin.
-   * For now: make this {@link #DDB_ALL_OPERATIONS}.
-   */
-  public static final String DDB_ADMIN = DDB_ALL_OPERATIONS;
-
-  /**
-   * Permission for DDB describeTable() operation: {@value}.
-   * This is used during initialization.
-   */
-  public static final String DDB_DESCRIBE_TABLE = "dynamodb:DescribeTable";
-
-  /**
-   * Permission to query the DDB table: {@value}.
-   */
-  public static final String DDB_QUERY = "dynamodb:Query";
-
-  /**
-   * Permission for DDB operation to get a record: {@value}.
-   */
-  public static final String DDB_GET_ITEM = "dynamodb:GetItem";
-
-  /**
-   * Permission for DDB write record operation: {@value}.
-   */
-  public static final String DDB_PUT_ITEM = "dynamodb:PutItem";
-
-  /**
-   * Permission for DDB update single item operation: {@value}.
-   */
-  public static final String DDB_UPDATE_ITEM = "dynamodb:UpdateItem";
-
-  /**
-   * Permission for DDB delete operation: {@value}.
-   */
-  public static final String DDB_DELETE_ITEM = "dynamodb:DeleteItem";
-
-  /**
-   * Permission for DDB operation: {@value}.
-   */
-  public static final String DDB_BATCH_GET_ITEM = "dynamodb:BatchGetItem";
-
-  /**
-   * Batch write permission for DDB: {@value}.
-   */
-  public static final String DDB_BATCH_WRITE_ITEM = "dynamodb:BatchWriteItem";
-
-  /**
-   * All DynamoDB tables: {@value}.
-   */
-  public static final String ALL_DDB_TABLES = "arn:aws:dynamodb:*";
-
-  /**
-   * Statement to allow all DDB access.
-   */
-  public static final Statement STATEMENT_ALL_DDB =
-      allowAllDynamoDBOperations(ALL_DDB_TABLES);
-
-  /**
-   * Statement to allow all client operations needed for S3Guard,
-   * but none of the admin operations.
-   */
-  public static final Statement STATEMENT_S3GUARD_CLIENT =
-      allowS3GuardClientOperations(ALL_DDB_TABLES);
-
   /**
    * Allow all S3 Operations.
-   * This does not cover DDB or S3-KMS
+   * This does not cover S3-KMS
    */
   public static final Statement STATEMENT_ALL_S3 = statement(true,
       S3_ALL_BUCKETS,
@@ -368,36 +296,6 @@ private RolePolicies() {
           S3_ALL_BUCKETS,
           S3_GET_BUCKET_LOCATION);
 
-  /**
-   * Policy for all S3 and S3Guard operations, and SSE-KMS.
-   */
-  public static final Policy ALLOW_S3_AND_SGUARD = policy(
-      STATEMENT_ALL_S3,
-      STATEMENT_ALL_DDB,
-      STATEMENT_ALLOW_SSE_KMS_RW,
-      STATEMENT_ALL_S3_GET_BUCKET_LOCATION
-  );
-
-  public static Statement allowS3GuardClientOperations(String tableArn) {
-    return statement(true,
-        tableArn,
-        DDB_BATCH_GET_ITEM,
-        DDB_BATCH_WRITE_ITEM,
-        DDB_DELETE_ITEM,
-        DDB_DESCRIBE_TABLE,
-        DDB_GET_ITEM,
-        DDB_PUT_ITEM,
-        DDB_QUERY,
-        DDB_UPDATE_ITEM
-    );
-  }
-
-  public static Statement allowAllDynamoDBOperations(String tableArn) {
-    return statement(true,
-        tableArn,
-        DDB_ALL_OPERATIONS);
-  }
-
   /**
    * From an S3 bucket name, build an ARN to refer to it.
    * @param bucket bucket name.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
index cb6848a2c026d..82d4fa588164d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java
@@ -30,7 +30,7 @@
 import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
 import com.amazonaws.services.securitytoken.model.Credentials;
 import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -149,12 +149,10 @@ public static AWSSecurityTokenServiceClientBuilder builder(
    * @param tokenService STS instance
    * @param invoker invoker to use
    * @return an STS client bonded to that interface.
-   * @throws IOException on any failure
    */
   public static STSClient createClientConnection(
       final AWSSecurityTokenService tokenService,
-      final Invoker invoker)
-      throws IOException {
+      final Invoker invoker) {
     return new STSClient(tokenService, invoker);
   }
 
@@ -175,12 +173,9 @@ private STSClient(final AWSSecurityTokenService tokenService,
 
     @Override
     public void close() throws IOException {
-      try {
-        tokenService.shutdown();
-      } catch (UnsupportedOperationException ignored) {
-        // ignore this, as it is what the STS client currently
-        // does.
-      }
+      // Since we are not using AbstractAWSSecurityTokenService, we
+      // don't need to worry about catching UnsupportedOperationException.
+      tokenService.shutdown();
     }
 
     /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
index cda769a789c30..e162428787cc4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java
@@ -29,6 +29,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
+import org.apache.hadoop.fs.s3a.impl.V2Migration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 
@@ -70,6 +71,8 @@ public void initCustomSigners() {
       return;
     }
 
+    V2Migration.v1CustomSignerUsed();
+
     for (String customSigner : customSigners) {
       String[] parts = customSigner.split(":");
       if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AWSPolicyProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AWSPolicyProvider.java
index aaca10f1aea5e..371a0dceaf7e9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AWSPolicyProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AWSPolicyProvider.java
@@ -31,8 +31,8 @@
  * The permissions requested are from the perspective of
  * S3A filesystem operations on the data, <i>not</i> the simpler
  * model of "permissions on the the remote service".
- * As an example, to use S3Guard effectively, the client needs full CRUD
- * access to the table, even for {@link AccessLevel#READ}.
+ * As an example, AWS-KMS encryption permissions must
+ * also be requested.
  */
 public interface AWSPolicyProvider {
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
index 41dac8ad683be..b40c8b2aabe62 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.net.URI;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
index fc7b095f0ddb4..6e31c006e26fc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
@@ -210,7 +210,7 @@ public abstract AWSCredentialProviderList deployUnbonded()
 
   /**
    * Bind to the token identifier, returning the credential providers to use
-   * for the owner to talk to S3, DDB and related AWS Services.
+   * for the owner to talk to S3 and related AWS Services.
    * @param retrievedIdentifier the unmarshalled data
    * @return non-empty list of AWS credential providers to use for
    * authenticating this client with AWS services.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
index 79553175e76f5..b39e81668a37a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
@@ -22,7 +22,7 @@
 import java.net.URI;
 import java.util.Optional;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
index c10561c5517b4..9b06031d5866a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
@@ -25,7 +25,7 @@
 
 import com.amazonaws.services.securitytoken.model.Credentials;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
index 839ab0e9d3263..bfb7e6966457b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
@@ -47,8 +47,8 @@
 
 import static org.apache.hadoop.fs.s3a.Statistic.DELEGATION_TOKENS_ISSUED;
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkState;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DEFAULT_DELEGATION_TOKEN_BINDING;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DURATION_LOG_AT_INFO;
@@ -128,8 +128,7 @@ public class S3ADelegationTokens extends AbstractDTService {
 
   /**
    * The access policies we want for operations.
-   * There's no attempt to ask for "admin" permissions here, e.g.
-   * those to manipulate S3Guard tables.
+   * There's no attempt to ask for "admin" permissions here.
    */
   protected static final EnumSet<AWSPolicyProvider.AccessLevel> ACCESS_POLICY
       = EnumSet.of(
@@ -420,8 +419,6 @@ public Token<AbstractS3ATokenIdentifier> createDelegationToken(
     requireServiceStarted();
     checkArgument(encryptionSecrets != null,
         "Null encryption secrets");
-    // this isn't done in in advance as it needs S3Guard initialized in the
-    // filesystem before it can generate complete policies.
     List<RoleModel.Statement> statements = getPolicyProvider()
         .listAWSPolicyRules(ACCESS_POLICY);
     Optional<RoleModel.Policy> rolePolicy =
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
index 391e3dcbd2fe5..d6044edde29dd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
@@ -26,32 +26,31 @@
 import java.util.Date;
 import java.util.List;
 import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 
 import com.amazonaws.services.s3.model.MultipartUpload;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.audit.AuditConstants;
-import org.apache.hadoop.fs.audit.CommonAuditContext;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
 import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+import org.apache.hadoop.fs.s3a.commit.impl.AuditContextUpdater;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
@@ -62,24 +61,33 @@
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.util.DurationInfo;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.util.functional.InvocationRaisingIOE;
+import org.apache.hadoop.util.functional.TaskPool;
 
-import static org.apache.hadoop.fs.s3a.Constants.THREAD_POOL_SHUTDOWN_DELAY_SECONDS;
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_TOTAL_TASKS;
+import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.MAX_TOTAL_TASKS;
 import static org.apache.hadoop.fs.s3a.Invoker.ignoreIOExceptions;
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_COMMIT_JOB;
 import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
 import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
-import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
 import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.E_NO_SPARK_UUID;
 import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID;
 import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID_SOURCE;
 import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.SPARK_WRITE_UUID;
+import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*;
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createJobSummaryFilename;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
 
 /**
  * Abstract base class for S3A committers; allows for any commonality
@@ -136,8 +144,6 @@ public abstract class AbstractS3ACommitter extends PathOutputCommitter
    */
   private final JobUUIDSource uuidSource;
 
-  private final CommonAuditContext commonAuditContext;
-
   /**
    * Has this instance been used for job setup?
    * If so then it is safe for a locally generated
@@ -145,11 +151,6 @@ public abstract class AbstractS3ACommitter extends PathOutputCommitter
    */
   private boolean jobSetup;
 
-  /**
-   * Thread pool for task execution.
-   */
-  private ExecutorService threadPool;
-
   /** Underlying commit operations. */
   private final CommitOperations commitOperations;
 
@@ -203,9 +204,8 @@ protected AbstractS3ACommitter(
       Path outputPath,
       TaskAttemptContext context) throws IOException {
     super(outputPath, context);
-    Preconditions.checkArgument(outputPath != null, "null output path");
-    Preconditions.checkArgument(context != null, "null job context");
-    this.jobContext = context;
+    setOutputPath(outputPath);
+    this.jobContext = requireNonNull(context, "null job context");
     this.role = "Task committer " + context.getTaskAttemptID();
     setConf(context.getConfiguration());
     Pair<String, JobUUIDSource> id = buildJobUUID(
@@ -219,17 +219,20 @@ protected AbstractS3ACommitter(
     S3AFileSystem fs = getDestS3AFS();
     // set this thread's context with the job ID.
     // audit spans created in this thread will pick
-    // up this value.
-    this.commonAuditContext = currentAuditContext();
-    updateCommonContext();
+    // up this value., including the commit operations instance
+    // soon to be created.
+    new AuditContextUpdater(jobContext)
+        .updateCurrentAuditContext();
+
     // the filesystem is the span source, always.
-    auditSpanSource = fs.getAuditSpanSource();
+    this.auditSpanSource = fs.getAuditSpanSource();
     this.createJobMarker = context.getConfiguration().getBoolean(
         CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
         DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER);
     // the statistics are shared between this committer and its operations.
     this.committerStatistics = fs.newCommitterStatistics();
-    this.commitOperations = new CommitOperations(fs, committerStatistics);
+    this.commitOperations = new CommitOperations(fs, committerStatistics,
+        outputPath.toString());
   }
 
   /**
@@ -267,8 +270,7 @@ public final Path getOutputPath() {
    * @param outputPath new value
    */
   protected final void setOutputPath(Path outputPath) {
-    Preconditions.checkNotNull(outputPath, "Null output path");
-    this.outputPath = outputPath;
+    this.outputPath = requireNonNull(outputPath, "Null output path");
   }
 
   /**
@@ -338,6 +340,12 @@ public Path getJobAttemptPath(JobContext context) {
     return getJobAttemptPath(getAppAttemptId(context));
   }
 
+  /**
+   * Compute the path under which all job attempts will be placed.
+   * @return the path to store job attempt data.
+   */
+  protected abstract Path getJobPath();
+
   /**
    * Compute the path where the output of a given job attempt will be placed.
    * @param appAttemptId the ID of the application attempt for this job.
@@ -440,8 +448,9 @@ protected FileSystem getDestinationFS(Path out, Configuration config)
   protected boolean requiresDelayedCommitOutputInFileSystem() {
     return false;
   }
+
   /**
-   * Task recovery considered unsupported: Warn and fail.
+   * Task recovery considered Unsupported: Warn and fail.
    * @param taskContext Context of the task whose output is being recovered
    * @throws IOException always.
    */
@@ -450,20 +459,24 @@ public void recoverTask(TaskAttemptContext taskContext) throws IOException {
     LOG.warn("Cannot recover task {}", taskContext.getTaskAttemptID());
     throw new PathCommitException(outputPath,
         String.format("Unable to recover task %s",
-        taskContext.getTaskAttemptID()));
+            taskContext.getTaskAttemptID()));
   }
 
   /**
    * if the job requires a success marker on a successful job,
    * create the file {@link CommitConstants#_SUCCESS}.
    *
-   * While the classic committers create a 0-byte file, the S3Guard committers
+   * While the classic committers create a 0-byte file, the S3A committers
    * PUT up a the contents of a {@link SuccessData} file.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending the pending commits
+   *
+   * @return the success data, even if the marker wasn't created
+   *
    * @throws IOException IO failure
    */
-  protected void maybeCreateSuccessMarkerFromCommits(JobContext context,
+  protected SuccessData maybeCreateSuccessMarkerFromCommits(
+      final CommitContext commitContext,
       ActiveCommit pending) throws IOException {
     List<String> filenames = new ArrayList<>(pending.size());
     // The list of committed objects in pending is size limited in
@@ -472,41 +485,92 @@ protected void maybeCreateSuccessMarkerFromCommits(JobContext context,
     // load in all the pending statistics
     IOStatisticsSnapshot snapshot = new IOStatisticsSnapshot(
         pending.getIOStatistics());
+    // and the current statistics
     snapshot.aggregate(getIOStatistics());
 
-    maybeCreateSuccessMarker(context, filenames, snapshot);
+    // and include the context statistics if enabled
+    if (commitContext.isCollectIOStatistics()) {
+      snapshot.aggregate(commitContext.getIOStatisticsContext()
+              .getIOStatistics());
+    }
+
+    return maybeCreateSuccessMarker(commitContext.getJobContext(), filenames, snapshot);
   }
 
   /**
    * if the job requires a success marker on a successful job,
-   * create the file {@link CommitConstants#_SUCCESS}.
+   * create the {@code _SUCCESS} file.
    *
-   * While the classic committers create a 0-byte file, the S3Guard committers
+   * While the classic committers create a 0-byte file, the S3A committers
    * PUT up a the contents of a {@link SuccessData} file.
+   * The file is returned, even if no marker is created.
+   * This is so it can be saved to a report directory.
    * @param context job context
    * @param filenames list of filenames.
    * @param ioStatistics any IO Statistics to include
    * @throws IOException IO failure
+   * @return the success data.
    */
-  protected void maybeCreateSuccessMarker(JobContext context,
-      List<String> filenames,
+  protected SuccessData maybeCreateSuccessMarker(
+      final JobContext context,
+      final List<String> filenames,
       final IOStatisticsSnapshot ioStatistics)
       throws IOException {
+
+    SuccessData successData =
+        createSuccessData(context, filenames, ioStatistics,
+            getDestFS().getConf());
     if (createJobMarker) {
-      // create a success data structure and then save it
-      SuccessData successData = new SuccessData();
-      successData.setCommitter(getName());
-      successData.setJobId(uuid);
-      successData.setJobIdSource(uuidSource.getText());
-      successData.setDescription(getRole());
-      successData.setHostname(NetUtils.getLocalHostname());
-      Date now = new Date();
-      successData.setTimestamp(now.getTime());
-      successData.setDate(now.toString());
-      successData.setFilenames(filenames);
-      successData.getIOStatistics().aggregate(ioStatistics);
+      // save it to the job dest dir
       commitOperations.createSuccessMarker(getOutputPath(), successData, true);
     }
+    return successData;
+  }
+
+  /**
+   * Create the success data structure from a job context.
+   * @param context job context.
+   * @param filenames short list of filenames; nullable
+   * @param ioStatistics IOStatistics snapshot
+   * @param destConf config of the dest fs, can be null
+   * @return the structure
+   *
+   */
+  private SuccessData createSuccessData(final JobContext context,
+      final List<String> filenames,
+      final IOStatisticsSnapshot ioStatistics,
+      final Configuration destConf) {
+    // create a success data structure
+    SuccessData successData = new SuccessData();
+    successData.setCommitter(getName());
+    successData.setJobId(uuid);
+    successData.setJobIdSource(uuidSource.getText());
+    successData.setDescription(getRole());
+    successData.setHostname(NetUtils.getLocalHostname());
+    Date now = new Date();
+    successData.setTimestamp(now.getTime());
+    successData.setDate(now.toString());
+    if (filenames != null) {
+      successData.setFilenames(filenames);
+    }
+    successData.getIOStatistics().aggregate(ioStatistics);
+    // attach some config options as diagnostics to assist
+    // in debugging performance issues.
+
+    // commit thread pool size
+    successData.addDiagnostic(FS_S3A_COMMITTER_THREADS,
+        Integer.toString(getJobCommitThreadCount(context)));
+
+    // and filesystem http connection and thread pool sizes
+    if (destConf != null) {
+      successData.addDiagnostic(MAXIMUM_CONNECTIONS,
+          destConf.get(MAXIMUM_CONNECTIONS,
+              Integer.toString(DEFAULT_MAXIMUM_CONNECTIONS)));
+      successData.addDiagnostic(MAX_TOTAL_TASKS,
+          destConf.get(MAX_TOTAL_TASKS,
+              Integer.toString(DEFAULT_MAX_TOTAL_TASKS)));
+    }
+    return successData;
   }
 
   /**
@@ -556,7 +620,11 @@ public void setupJob(JobContext context) throws IOException {
   @Override
   public void setupTask(TaskAttemptContext context) throws IOException {
     TaskAttemptID attemptID = context.getTaskAttemptID();
-    updateCommonContext();
+
+    // update the context so that task IO in the same thread has
+    // the relevant values.
+    new AuditContextUpdater(context)
+        .updateCurrentAuditContext();
 
     try (DurationInfo d = new DurationInfo(LOG, "Setup Task %s",
         attemptID)) {
@@ -572,6 +640,9 @@ && getUUIDSource() == JobUUIDSource.GeneratedLocally) {
       }
       Path taskAttemptPath = getTaskAttemptPath(context);
       FileSystem fs = taskAttemptPath.getFileSystem(getConf());
+      // delete that ta path if somehow it was there
+      fs.delete(taskAttemptPath, true);
+      // create an empty directory
       fs.mkdirs(taskAttemptPath);
     }
   }
@@ -596,25 +667,23 @@ protected FileSystem getTaskAttemptFilesystem(TaskAttemptContext context)
    * On a failure or abort of a single file's commit, all its uploads are
    * aborted.
    * The revert operation lists the files already committed and deletes them.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending  pending uploads
    * @throws IOException on any failure
    */
   protected void commitPendingUploads(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
     if (pending.isEmpty()) {
       LOG.warn("{}: No pending uploads to commit", getRole());
     }
     try (DurationInfo ignored = new DurationInfo(LOG,
-        "committing the output of %s task(s)", pending.size());
-        CommitOperations.CommitContext commitContext
-            = initiateCommitOperation()) {
+        "committing the output of %s task(s)", pending.size())) {
 
-      Tasks.foreach(pending.getSourceFiles())
+      TaskPool.foreach(pending.getSourceFiles())
           .stopOnFailure()
           .suppressExceptions(false)
-          .executeWith(buildSubmitter(context))
+          .executeWith(commitContext.getOuterSubmitter())
           .abortWith(status ->
               loadAndAbort(commitContext, pending, status, true, false))
           .revertWith(status ->
@@ -629,54 +698,61 @@ protected void commitPendingUploads(
    * This check avoids the situation where the inability to read
    * a file only surfaces partway through the job commit, so
    * results in the destination being tainted.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending the pending operations
    * @throws IOException any failure
    */
   protected void precommitCheckPendingFiles(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
 
     FileSystem sourceFS = pending.getSourceFS();
     try (DurationInfo ignored =
              new DurationInfo(LOG, "Preflight Load of pending files")) {
 
-      Tasks.foreach(pending.getSourceFiles())
+      TaskPool.foreach(pending.getSourceFiles())
           .stopOnFailure()
           .suppressExceptions(false)
-          .executeWith(buildSubmitter(context))
-          .run(status -> PendingSet.load(sourceFS, status));
+          .executeWith(commitContext.getOuterSubmitter())
+          .run(status -> PersistentCommitData.load(sourceFS, status,
+              commitContext.getPendingSetSerializer()));
     }
   }
 
   /**
    * Load a pendingset file and commit all of its contents.
+   * Invoked within a parallel run; the commitContext thread
+   * pool is already busy/possibly full, so do not
+   * execute work through the same submitter.
    * @param commitContext context to commit through
    * @param activeCommit commit state
    * @param status file to load
    * @throws IOException failure
    */
   private void loadAndCommit(
-      final CommitOperations.CommitContext commitContext,
+      final CommitContext commitContext,
       final ActiveCommit activeCommit,
       final FileStatus status) throws IOException {
 
     final Path path = status.getPath();
+    commitContext.switchToIOStatisticsContext();
     try (DurationInfo ignored =
              new DurationInfo(LOG,
                  "Loading and committing files in pendingset %s", path)) {
-      PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(),
-          status);
+      PendingSet pendingSet = PersistentCommitData.load(
+          activeCommit.getSourceFS(),
+          status,
+          commitContext.getPendingSetSerializer());
       String jobId = pendingSet.getJobId();
       if (!StringUtils.isEmpty(jobId) && !getUUID().equals(jobId)) {
         throw new PathCommitException(path,
             String.format("Mismatch in Job ID (%s) and commit job ID (%s)",
                 getUUID(), jobId));
       }
-      Tasks.foreach(pendingSet.getCommits())
+      TaskPool.foreach(pendingSet.getCommits())
           .stopOnFailure()
           .suppressExceptions(false)
-          .executeWith(singleThreadSubmitter())
+          .executeWith(commitContext.getInnerSubmitter())
           .onFailure((commit, exception) ->
               commitContext.abortSingleCommit(commit))
           .abortWith(commitContext::abortSingleCommit)
@@ -692,22 +768,28 @@ private void loadAndCommit(
 
   /**
    * Load a pendingset file and revert all of its contents.
+   * Invoked within a parallel run; the commitContext thread
+   * pool is already busy/possibly full, so do not
+   * execute work through the same submitter.
    * @param commitContext context to commit through
    * @param activeCommit commit state
    * @param status status of file to load
    * @throws IOException failure
    */
   private void loadAndRevert(
-      final CommitOperations.CommitContext commitContext,
+      final CommitContext commitContext,
       final ActiveCommit activeCommit,
       final FileStatus status) throws IOException {
 
     final Path path = status.getPath();
+    commitContext.switchToIOStatisticsContext();
     try (DurationInfo ignored =
              new DurationInfo(LOG, false, "Committing %s", path)) {
-      PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(),
-          status);
-      Tasks.foreach(pendingSet.getCommits())
+      PendingSet pendingSet = PersistentCommitData.load(
+          activeCommit.getSourceFS(),
+          status,
+          commitContext.getPendingSetSerializer());
+      TaskPool.foreach(pendingSet.getCommits())
           .suppressExceptions(true)
           .run(commitContext::revertCommit);
     }
@@ -715,6 +797,9 @@ private void loadAndRevert(
 
   /**
    * Load a pendingset file and abort all of its contents.
+   * Invoked within a parallel run; the commitContext thread
+   * pool is already busy/possibly full, so do not
+   * execute work through the same submitter.
    * @param commitContext context to commit through
    * @param activeCommit commit state
    * @param status status of file to load
@@ -722,20 +807,23 @@ private void loadAndRevert(
    * @throws IOException failure
    */
   private void loadAndAbort(
-      final CommitOperations.CommitContext commitContext,
+      final CommitContext commitContext,
       final ActiveCommit activeCommit,
       final FileStatus status,
       final boolean suppressExceptions,
       final boolean deleteRemoteFiles) throws IOException {
 
     final Path path = status.getPath();
+    commitContext.switchToIOStatisticsContext();
     try (DurationInfo ignored =
              new DurationInfo(LOG, false, "Aborting %s", path)) {
-      PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(),
-          status);
+      PendingSet pendingSet = PersistentCommitData.load(
+          activeCommit.getSourceFS(),
+          status,
+          commitContext.getPendingSetSerializer());
       FileSystem fs = getDestFS();
-      Tasks.foreach(pendingSet.getCommits())
-          .executeWith(singleThreadSubmitter())
+      TaskPool.foreach(pendingSet.getCommits())
+          .executeWith(commitContext.getInnerSubmitter())
           .suppressExceptions(suppressExceptions)
           .run(commit -> {
             try {
@@ -752,28 +840,64 @@ private void loadAndAbort(
   }
 
   /**
-   * Start the final commit/abort commit operations.
+   * Start the final job commit/abort commit operations.
+   * If configured to collect statistics,
+   * The IO StatisticsContext is reset.
+   * @param context job context
+   * @return a commit context through which the operations can be invoked.
+   * @throws IOException failure.
+   */
+  protected CommitContext initiateJobOperation(
+      final JobContext context)
+      throws IOException {
+
+    IOStatisticsContext ioStatisticsContext =
+        IOStatisticsContext.getCurrentIOStatisticsContext();
+    CommitContext commitContext = getCommitOperations().createCommitContext(
+        context,
+        getOutputPath(),
+        getJobCommitThreadCount(context),
+        ioStatisticsContext);
+    commitContext.maybeResetIOStatisticsContext();
+    return commitContext;
+  }
+
+  /**
+   * Start a ask commit/abort commit operations.
+   * This may have a different thread count.
+   * If configured to collect statistics,
+   * The IO StatisticsContext is reset.
+   * @param context job or task context
    * @return a commit context through which the operations can be invoked.
    * @throws IOException failure.
    */
-  protected CommitOperations.CommitContext initiateCommitOperation()
+  protected CommitContext initiateTaskOperation(
+      final JobContext context)
       throws IOException {
-    return getCommitOperations().initiateCommitOperation(getOutputPath());
+
+    CommitContext commitContext = getCommitOperations().createCommitContext(
+        context,
+        getOutputPath(),
+        getTaskCommitThreadCount(context),
+        IOStatisticsContext.getCurrentIOStatisticsContext());
+    commitContext.maybeResetIOStatisticsContext();
+    return commitContext;
   }
 
   /**
    * Internal Job commit operation: where the S3 requests are made
    * (potentially in parallel).
-   * @param context job context
+   * @param commitContext commit context
    * @param pending pending commits
    * @throws IOException any failure
    */
-  protected void commitJobInternal(JobContext context,
-      ActiveCommit pending)
+  protected void commitJobInternal(
+      final CommitContext commitContext,
+      final ActiveCommit pending)
       throws IOException {
     trackDurationOfInvocation(committerStatistics,
         COMMITTER_COMMIT_JOB.getSymbol(),
-        () -> commitPendingUploads(context, pending));
+        () -> commitPendingUploads(commitContext, pending));
   }
 
   @Override
@@ -782,7 +906,9 @@ public void abortJob(JobContext context, JobStatus.State state)
     LOG.info("{}: aborting job {} in state {}",
         getRole(), jobIdString(context), state);
     // final cleanup operations
-    abortJobInternal(context, false);
+    try (CommitContext commitContext = initiateJobOperation(context)){
+      abortJobInternal(commitContext, false);
+    }
   }
 
 
@@ -790,30 +916,33 @@ public void abortJob(JobContext context, JobStatus.State state)
    * The internal job abort operation; can be overridden in tests.
    * This must clean up operations; it is called when a commit fails, as
    * well as in an {@link #abortJob(JobContext, JobStatus.State)} call.
-   * The base implementation calls {@link #cleanup(JobContext, boolean)}
+   * The base implementation calls {@link #cleanup(CommitContext, boolean)}
    * so cleans up the filesystems and destroys the thread pool.
    * Subclasses must always invoke this superclass method after their
    * own operations.
-   * @param context job context
+   * Creates and closes its own commit context.
+   *
+   * @param commitContext commit context
    * @param suppressExceptions should exceptions be suppressed?
    * @throws IOException any IO problem raised when suppressExceptions is false.
    */
-  protected void abortJobInternal(JobContext context,
+  protected void abortJobInternal(CommitContext commitContext,
       boolean suppressExceptions)
       throws IOException {
-    cleanup(context, suppressExceptions);
+    cleanup(commitContext, suppressExceptions);
   }
 
   /**
    * Abort all pending uploads to the destination directory during
    * job cleanup operations.
    * Note: this instantiates the thread pool if required -so
-   * {@link #destroyThreadPool()} must be called after this.
    * @param suppressExceptions should exceptions be suppressed
+   * @param commitContext commit context
    * @throws IOException IO problem
    */
   protected void abortPendingUploadsInCleanup(
-      boolean suppressExceptions) throws IOException {
+      boolean suppressExceptions,
+      CommitContext commitContext) throws IOException {
     // return early if aborting is disabled.
     if (!shouldAbortUploadsInCleanup()) {
       LOG.debug("Not cleanup up pending uploads to {} as {} is false ",
@@ -824,9 +953,7 @@ protected void abortPendingUploadsInCleanup(
     Path dest = getOutputPath();
     try (DurationInfo ignored =
              new DurationInfo(LOG, "Aborting all pending commits under %s",
-                 dest);
-         CommitOperations.CommitContext commitContext
-             = initiateCommitOperation()) {
+                 dest)) {
       CommitOperations ops = getCommitOperations();
       List<MultipartUpload> pending;
       try {
@@ -840,8 +967,8 @@ protected void abortPendingUploadsInCleanup(
         LOG.warn("{} pending uploads were found -aborting", pending.size());
         LOG.warn("If other tasks/jobs are writing to {},"
             + "this action may cause them to fail", dest);
-        Tasks.foreach(pending)
-            .executeWith(buildSubmitter(getJobContext()))
+        TaskPool.foreach(pending)
+            .executeWith(commitContext.getOuterSubmitter())
             .suppressExceptions(suppressExceptions)
             .run(u -> commitContext.abortMultipartCommit(
                 u.getKey(), u.getUploadId()));
@@ -863,12 +990,12 @@ private boolean shouldAbortUploadsInCleanup() {
    * they can be loaded.
    * The Magic committer does not, because of the overhead of reading files
    * from S3 makes it too expensive.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending the pending operations
    * @throws IOException any failure
    */
   @VisibleForTesting
-  public void preCommitJob(JobContext context,
+  public void preCommitJob(CommitContext commitContext,
       ActiveCommit pending) throws IOException {
   }
 
@@ -890,21 +1017,55 @@ public void preCommitJob(JobContext context,
   @Override
   public void commitJob(JobContext context) throws IOException {
     String id = jobIdString(context);
+    // the commit context is created outside a try-with-resources block
+    // so it can be used in exception handling.
+    CommitContext commitContext = null;
+
+    SuccessData successData = null;
+    IOException failure = null;
+    String stage = "preparing";
     try (DurationInfo d = new DurationInfo(LOG,
         "%s: commitJob(%s)", getRole(), id)) {
+      commitContext = initiateJobOperation(context);
       ActiveCommit pending
-          = listPendingUploadsToCommit(context);
-      preCommitJob(context, pending);
-      commitJobInternal(context, pending);
+          = listPendingUploadsToCommit(commitContext);
+      stage = "precommit";
+      preCommitJob(commitContext, pending);
+      stage = "commit";
+      commitJobInternal(commitContext, pending);
+      stage = "completed";
       jobCompleted(true);
-      maybeCreateSuccessMarkerFromCommits(context, pending);
-      cleanup(context, false);
+      stage = "marker";
+      successData = maybeCreateSuccessMarkerFromCommits(commitContext, pending);
+      stage = "cleanup";
+      cleanup(commitContext, false);
     } catch (IOException e) {
+      // failure. record it for the summary
+      failure = e;
       LOG.warn("Commit failure for job {}", id, e);
       jobCompleted(false);
-      abortJobInternal(context, true);
+      abortJobInternal(commitContext, true);
       throw e;
+    } finally {
+      // save the report summary, even on failure
+      if (commitContext != null) {
+        if (successData == null) {
+          // if the commit did not get as far as creating success data, create one.
+          successData = createSuccessData(context, null, null,
+              getDestFS().getConf());
+        }
+        // save quietly, so no exceptions are raised
+        maybeSaveSummary(stage,
+            commitContext,
+            successData,
+            failure,
+            true,
+            true);
+        // and close that commit context
+        commitContext.close();
+      }
     }
+
   }
 
   /**
@@ -923,28 +1084,27 @@ protected void jobCompleted(boolean success) {
 
   /**
    * Get the list of pending uploads for this job attempt.
-   * @param context job context
+   * @param commitContext commit context
    * @return a list of pending uploads.
    * @throws IOException Any IO failure
    */
   protected abstract ActiveCommit listPendingUploadsToCommit(
-      JobContext context)
+      CommitContext commitContext)
       throws IOException;
 
   /**
    * Cleanup the job context, including aborting anything pending
    * and destroying the thread pool.
-   * @param context job context
+   * @param commitContext commit context
    * @param suppressExceptions should exceptions be suppressed?
    * @throws IOException any failure if exceptions were not suppressed.
    */
-  protected void cleanup(JobContext context,
+  protected void cleanup(CommitContext commitContext,
       boolean suppressExceptions) throws IOException {
     try (DurationInfo d = new DurationInfo(LOG,
-        "Cleanup job %s", jobIdString(context))) {
-      abortPendingUploadsInCleanup(suppressExceptions);
+        "Cleanup job %s", jobIdString(commitContext.getJobContext()))) {
+      abortPendingUploadsInCleanup(suppressExceptions, commitContext);
     } finally {
-      destroyThreadPool();
       cleanupStagingDirs();
     }
   }
@@ -955,8 +1115,9 @@ public void cleanupJob(JobContext context) throws IOException {
     String r = getRole();
     String id = jobIdString(context);
     LOG.warn("{}: using deprecated cleanupJob call for {}", r, id);
-    try (DurationInfo d = new DurationInfo(LOG, "%s: cleanup Job %s", r, id)) {
-      cleanup(context, true);
+    try (DurationInfo d = new DurationInfo(LOG, "%s: cleanup Job %s", r, id);
+         CommitContext commitContext = initiateJobOperation(context)) {
+      cleanup(commitContext, true);
     }
   }
 
@@ -1013,137 +1174,26 @@ protected String getRole() {
     return role;
   }
 
-  /**
-   * Returns an {@link Tasks.Submitter} for parallel tasks. The number of
-   * threads in the thread-pool is set by fs.s3a.committer.threads.
-   * If num-threads is 0, this will return null;
-   * this is used in Tasks as a cue
-   * to switch to single-threaded execution.
-   *
-   * @param context the JobContext for this commit
-   * @return a submitter or null
-   */
-  protected Tasks.Submitter buildSubmitter(
-      JobContext context) {
-    if (getThreadCount(context) > 0) {
-      return new PoolSubmitter(context);
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Returns an {@link ExecutorService} for parallel tasks. The number of
-   * threads in the thread-pool is set by fs.s3a.committer.threads.
-   * If num-threads is 0, this will raise an exception.
-   *
-   * @param context the JobContext for this commit
-   * @param numThreads threads
-   * @return an {@link ExecutorService} for the number of threads
-   */
-  private synchronized ExecutorService buildThreadPool(
-      JobContext context, int numThreads) {
-    Preconditions.checkArgument(numThreads > 0,
-        "Cannot create a thread pool with no threads");
-    if (threadPool == null) {
-      LOG.debug("{}: creating thread pool of size {}", getRole(), numThreads);
-      threadPool = HadoopExecutors.newFixedThreadPool(numThreads,
-          new ThreadFactoryBuilder()
-              .setDaemon(true)
-              .setNameFormat(THREAD_PREFIX + context.getJobID() + "-%d")
-              .build());
-    }
-    return threadPool;
-  }
-
   /**
    * Get the thread count for this job's commit operations.
    * @param context the JobContext for this commit
    * @return a possibly zero thread count.
    */
-  private int getThreadCount(final JobContext context) {
+  private int getJobCommitThreadCount(final JobContext context) {
     return context.getConfiguration().getInt(
         FS_S3A_COMMITTER_THREADS,
         DEFAULT_COMMITTER_THREADS);
   }
 
   /**
-   * Submit a runnable.
-   * This will demand-create the thread pool if needed.
-   * <p></p>
-   * This is synchronized to ensure the thread pool is always valid when
-   * work is synchronized. See HADOOP-16798.
+   * Get the thread count for this task's commit operations.
    * @param context the JobContext for this commit
-   * @param task task to execute
-   * @return the future of the submitted task.
-   */
-  private synchronized Future<?> submitRunnable(
-      final JobContext context,
-      final Runnable task) {
-    return buildThreadPool(context, getThreadCount(context)).submit(task);
-  }
-
-  /**
-   * The real task submitter, which hands off the work to
-   * the current thread pool.
-   */
-  private final class PoolSubmitter implements Tasks.Submitter {
-
-    private final JobContext context;
-
-    private final int numThreads;
-
-    private PoolSubmitter(final JobContext context) {
-      this.numThreads = getThreadCount(context);
-      Preconditions.checkArgument(numThreads > 0,
-          "Cannot create a thread pool with no threads");
-      this.context = context;
-    }
-
-    @Override
-    public Future<?> submit(final Runnable task) {
-      return submitRunnable(context, task);
-    }
-
-  }
-
-  /**
-   * Destroy any thread pools; wait for that to finish,
-   * but don't overreact if it doesn't finish in time.
-   */
-  protected void destroyThreadPool() {
-    ExecutorService pool;
-    // reset the thread pool in a sync block, then shut it down
-    // afterwards. This allows for other threads to create a
-    // new thread pool on demand.
-    synchronized(this) {
-      pool = this.threadPool;
-      threadPool = null;
-    }
-    if (pool != null) {
-      LOG.debug("Destroying thread pool");
-      HadoopExecutors.shutdown(pool, LOG,
-          THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
-    }
-  }
-
-  /**
-   * Get the thread pool for executing the single file commit/revert
-   * within the commit of all uploads of a single task.
-   * This is currently null; it is here to allow the Tasks class to
-   * provide the logic for execute/revert.
-   * @return null. always.
-   */
-  protected final synchronized Tasks.Submitter singleThreadSubmitter() {
-    return null;
-  }
-
-  /**
-   * Does this committer have a thread pool?
-   * @return true if a thread pool exists.
+   * @return a possibly zero thread count.
    */
-  public synchronized boolean hasThreadPool() {
-    return threadPool != null;
+  private int getTaskCommitThreadCount(final JobContext context) {
+    return context.getConfiguration().getInt(
+        FS_S3A_COMMITTER_THREADS,
+        DEFAULT_COMMITTER_THREADS);
   }
 
   /**
@@ -1161,24 +1211,23 @@ protected void deleteTaskAttemptPathQuietly(TaskAttemptContext context) {
    * Abort all pending uploads in the list.
    * This operation is used by the magic committer as part of its
    * rollback after a failure during task commit.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending pending uploads
    * @param suppressExceptions should exceptions be suppressed
    * @throws IOException any exception raised
    */
-  protected void abortPendingUploads(JobContext context,
-      List<SinglePendingCommit> pending,
-      boolean suppressExceptions)
+  protected void abortPendingUploads(
+      final CommitContext commitContext,
+      final List<SinglePendingCommit> pending,
+      final boolean suppressExceptions)
       throws IOException {
     if (pending == null || pending.isEmpty()) {
       LOG.info("{}: no pending commits to abort", getRole());
     } else {
       try (DurationInfo d = new DurationInfo(LOG,
-          "Aborting %s uploads", pending.size());
-           CommitOperations.CommitContext commitContext
-               = initiateCommitOperation()) {
-        Tasks.foreach(pending)
-            .executeWith(buildSubmitter(context))
+          "Aborting %s uploads", pending.size())) {
+        TaskPool.foreach(pending)
+            .executeWith(commitContext.getOuterSubmitter())
             .suppressExceptions(suppressExceptions)
             .run(commitContext::abortSingleCommit);
       }
@@ -1187,27 +1236,24 @@ protected void abortPendingUploads(JobContext context,
 
   /**
    * Abort all pending uploads in the list.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending pending uploads
    * @param suppressExceptions should exceptions be suppressed?
    * @param deleteRemoteFiles should remote files be deleted?
    * @throws IOException any exception raised
    */
   protected void abortPendingUploads(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending,
       final boolean suppressExceptions,
       final boolean deleteRemoteFiles) throws IOException {
-
     if (pending.isEmpty()) {
       LOG.info("{}: no pending commits to abort", getRole());
     } else {
       try (DurationInfo d = new DurationInfo(LOG,
-          "Aborting %s uploads", pending.size());
-           CommitOperations.CommitContext commitContext
-               = initiateCommitOperation()) {
-        Tasks.foreach(pending.getSourceFiles())
-            .executeWith(buildSubmitter(context))
+          "Aborting %s uploads", pending.size())) {
+        TaskPool.foreach(pending.getSourceFiles())
+            .executeWith(commitContext.getOuterSubmitter())
             .suppressExceptions(suppressExceptions)
             .run(path ->
                 loadAndAbort(commitContext,
@@ -1389,13 +1435,7 @@ public String toString() {
    */
   protected final void updateCommonContext() {
     currentAuditContext().put(AuditConstants.PARAM_JOB_ID, uuid);
-  }
 
-  /**
-   * Remove JobID from the current thread's context.
-   */
-  protected final void resetCommonContext() {
-    currentAuditContext().remove(AuditConstants.PARAM_JOB_ID);
   }
 
   protected AuditSpanSource getAuditSpanSource() {
@@ -1421,6 +1461,77 @@ protected AuditSpan startOperation(String name,
     return getAuditSpanSource().createSpan(name, path1, path2);
   }
 
+
+  /**
+   * Save a summary to the report dir if the config option
+   * is set.
+   * The report will be updated with the current active stage,
+   * and if {@code thrown} is non-null, it will be added to the
+   * diagnostics (and the job tagged as a failure).
+   * Static for testability.
+   * @param activeStage active stage
+   * @param context commit context.
+   * @param report summary file.
+   * @param thrown any exception indicting failure.
+   * @param quiet should exceptions be swallowed.
+   * @param overwrite should the existing file be overwritten
+   * @return the path of a file, if successfully saved
+   * @throws IOException if a failure occured and quiet==false
+   */
+  private static Path maybeSaveSummary(
+      String activeStage,
+      CommitContext context,
+      SuccessData report,
+      Throwable thrown,
+      boolean quiet,
+      boolean overwrite) throws IOException {
+    Configuration conf = context.getConf();
+    String reportDir = conf.getTrimmed(OPT_SUMMARY_REPORT_DIR, "");
+    if (reportDir.isEmpty()) {
+      LOG.debug("No summary directory set in " + OPT_SUMMARY_REPORT_DIR);
+      return null;
+    }
+    LOG.debug("Summary directory set to {}", reportDir);
+
+    Path reportDirPath = new Path(reportDir);
+    Path path = new Path(reportDirPath,
+        createJobSummaryFilename(context.getJobId()));
+
+    if (thrown != null) {
+      report.recordJobFailure(thrown);
+    }
+    report.putDiagnostic(STAGE, activeStage);
+    // the store operations here is explicitly created for the FS where
+    // the reports go, which may not be the target FS of the job.
+
+    final FileSystem fs = path.getFileSystem(conf);
+    try (ManifestStoreOperations operations = new ManifestStoreOperationsThroughFileSystem(
+        fs)) {
+      if (!overwrite) {
+        // check for file existence so there is no need to worry about
+        // precisely what exception is raised when overwrite=false and dest file
+        // exists
+        try {
+          FileStatus st = operations.getFileStatus(path);
+          // get here and the file exists
+          LOG.debug("Report already exists: {}", st);
+          return null;
+        } catch (FileNotFoundException ignored) {
+        }
+      }
+      report.save(fs, path, SuccessData.serializer());
+      LOG.info("Job summary saved to {}", path);
+      return path;
+    } catch (IOException e) {
+      LOG.debug("Failed to save summary to {}", path, e);
+      if (quiet) {
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   /**
    * State of the active commit operation.
    *
@@ -1442,7 +1553,7 @@ protected AuditSpan startOperation(String name,
    *   </li>
    * </ol>
    */
-  public static class ActiveCommit {
+  public static final class ActiveCommit {
 
     private static final AbstractS3ACommitter.ActiveCommit EMPTY
         = new ActiveCommit(null, new ArrayList<>());
@@ -1483,6 +1594,7 @@ public static class ActiveCommit {
      * @param sourceFS filesystem containing the list of pending files
      * @param sourceFiles .pendingset files to load and commit.
      */
+    @SuppressWarnings("unchecked")
     public ActiveCommit(
         final FileSystem sourceFS,
         final List<? extends FileStatus> sourceFiles) {
@@ -1493,13 +1605,14 @@ public ActiveCommit(
     /**
      * Create an active commit of the given pending files.
      * @param pendingFS source filesystem.
-     * @param statuses list of file status or subclass to use.
+     * @param statuses iterator of file status or subclass to use.
      * @return the commit
+     * @throws IOException if the iterator raises one.
      */
-    public static ActiveCommit fromStatusList(
+    public static ActiveCommit fromStatusIterator(
         final FileSystem pendingFS,
-        final List<? extends FileStatus> statuses) {
-      return new ActiveCommit(pendingFS, statuses);
+        final RemoteIterator<? extends FileStatus> statuses) throws IOException {
+      return new ActiveCommit(pendingFS, toList(statuses));
     }
 
     /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
index bbc59f168f60d..6e2a5d8c9fbce 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
@@ -22,7 +22,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
-import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.COMMITTER_FACTORY_SCHEME_PATTERN;
 
 /**
  * Constants for working with committers.
@@ -57,6 +56,12 @@ private CommitConstants() {
    */
   public static final String PENDINGSET_SUFFIX = ".pendingset";
 
+
+  /**
+   * Prefix to use for config options: {@value}.
+   */
+  public static final String OPT_PREFIX = "fs.s3a.committer.";
+
   /**
    * Flag to indicate whether support for the Magic committer is enabled
    * in the filesystem.
@@ -121,9 +126,8 @@ private CommitConstants() {
 
   /**
    * Temp data which is not auto-committed: {@value}.
-   * Uses a different name from normal just to make clear it is different.
    */
-  public static final String TEMP_DATA = "__temp-data";
+  public static final String TEMP_DATA = TEMPORARY;
 
 
   /**
@@ -144,7 +148,7 @@ private CommitConstants() {
    * Key to set for the S3A schema to use the specific committer.
    */
   public static final String S3A_COMMITTER_FACTORY_KEY = String.format(
-      COMMITTER_FACTORY_SCHEME_PATTERN, "s3a");
+      "mapreduce.outputcommitter.factory.scheme.s3a");
 
   /**
    * S3 Committer factory: {@value}.
@@ -222,13 +226,19 @@ private CommitConstants() {
   /**
    * Number of threads in committers for parallel operations on files
    * (upload, commit, abort, delete...): {@value}.
+   * Two thread pools this size are created, one for the outer
+   * task-level parallelism, and one for parallel execution
+   * within tasks (POSTs to commit individual uploads)
+   * If the value is negative, it is inverted and then multiplied
+   * by the number of cores in the CPU.
    */
   public static final String FS_S3A_COMMITTER_THREADS =
       "fs.s3a.committer.threads";
+
   /**
    * Default value for {@link #FS_S3A_COMMITTER_THREADS}: {@value}.
    */
-  public static final int DEFAULT_COMMITTER_THREADS = 8;
+  public static final int DEFAULT_COMMITTER_THREADS = 32;
 
   /**
    * Path  in the cluster filesystem for temporary data: {@value}.
@@ -330,4 +340,38 @@ private CommitConstants() {
   public static final String XA_MAGIC_MARKER = XA_HEADER_PREFIX
       + X_HEADER_MAGIC_MARKER;
 
+  /**
+   * Task Attempt ID query header: {@value}.
+   */
+  public static final String PARAM_TASK_ATTEMPT_ID = "ta";
+
+  /**
+   * Directory for saving job summary reports.
+   * These are the _SUCCESS files, but are saved even on
+   * job failures.
+   * Value: {@value}.
+   */
+  public static final String OPT_SUMMARY_REPORT_DIR =
+      OPT_PREFIX + "summary.report.directory";
+
+  /**
+   * Experimental feature to collect thread level IO statistics.
+   * When set the committers will reset the statistics in
+   * task setup and propagate to the job committer.
+   * The job comitter will include those and its own statistics.
+   * Do not use if the execution engine is collecting statistics,
+   * as the multiple reset() operations will result in incomplete
+   * statistics.
+   * Value: {@value}.
+   */
+  public static final String S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS =
+      OPT_PREFIX + "experimental.collect.iostatistics";
+
+  /**
+   * Default value for {@link #S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS}.
+   * Value: {@value}.
+   */
+  public static final boolean S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT =
+      false;
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java
new file mode 100644
index 0000000000000..20af292b3eb66
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
+
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_RENAME;
+
+/**
+ * Statistic names for committers.
+ * Please keep in sync with org.apache.hadoop.fs.s3a.Statistic
+ * so that S3A and manifest committers are in sync.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class CommitterStatisticNames {
+
+
+  /** Amount of data committed: {@value}. */
+  public static final String COMMITTER_BYTES_COMMITTED_COUNT =
+      "committer_bytes_committed";
+
+  /** Duration Tracking of time to commit an entire job: {@value}. */
+  public static final String COMMITTER_COMMIT_JOB =
+      "committer_commit_job";
+
+  /** Number of files committed: {@value}. */
+  public static final String COMMITTER_FILES_COMMITTED_COUNT =
+      "committer_files_committed";
+  /** "Count of successful tasks:: {@value}. */
+  public static final String COMMITTER_TASKS_COMPLETED_COUNT =
+      "committer_tasks_completed";
+
+  /** Count of failed tasks: {@value}. */
+  public static final String COMMITTER_TASKS_FAILED_COUNT =
+      "committer_tasks_failed";
+
+  /** Count of commits aborted: {@value}. */
+  public static final String COMMITTER_COMMITS_ABORTED_COUNT =
+      "committer_commits_aborted";
+
+  /** Count of commits reverted: {@value}. */
+  public static final String COMMITTER_COMMITS_REVERTED_COUNT =
+      "committer_commits_reverted";
+
+  /** Count of commits failed: {@value}. */
+  public static final String COMMITTER_COMMITS_FAILED =
+      "committer_commits" + StoreStatisticNames.SUFFIX_FAILURES;
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_FILE_COUNT_MEAN =
+      "committer_task_file_count";
+
+  /**
+   * File Size.
+   */
+  public static final String COMMITTER_FILE_SIZE_MEAN =
+      "committer_task_file_size";
+
+  /**
+   * What is a task attempt's directory count.
+   */
+  public static final String COMMITTER_TASK_DIRECTORY_COUNT_MEAN =
+      "committer_task_directory_count";
+
+  /**
+   * What is the depth of a task attempt's directory tree.
+   */
+  public static final String COMMITTER_TASK_DIRECTORY_DEPTH_MEAN =
+      "committer_task_directory_depth";
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_TASK_FILE_COUNT_MEAN =
+      "committer_task_file_count";
+
+  /**
+   * The number of files in a task. This will be a MeanStatistic.
+   */
+  public static final String COMMITTER_TASK_FILE_SIZE_MEAN =
+      "committer_task_file_size";
+
+  /** Directory creation {@value}. */
+  public static final String OP_CREATE_DIRECTORIES = "op_create_directories";
+
+  /** Creating a single directory {@value}. */
+  public static final String OP_CREATE_ONE_DIRECTORY =
+      "op_create_one_directory";
+
+  /** Directory scan {@value}. */
+  public static final String OP_DIRECTORY_SCAN = "op_directory_scan";
+
+  /**
+   * Overall job commit {@value}.
+   */
+  public static final String OP_STAGE_JOB_COMMIT = COMMITTER_COMMIT_JOB;
+
+  /** {@value}. */
+  public static final String OP_LOAD_ALL_MANIFESTS = "op_load_all_manifests";
+
+  /**
+   * Load a task manifest: {@value}.
+   */
+  public static final String OP_LOAD_MANIFEST = "op_load_manifest";
+
+  /** Rename a file: {@value}. */
+  public static final String OP_RENAME_FILE = OP_RENAME;
+
+  /**
+   * Save a task manifest: {@value}.
+   */
+  public static final String OP_SAVE_TASK_MANIFEST =
+      "op_task_stage_save_task_manifest";
+
+  /**
+   * Task abort: {@value}.
+   */
+  public static final String OP_STAGE_TASK_ABORT_TASK
+      = "op_task_stage_abort_task";
+
+  /**
+   * Job abort: {@value}.
+   */
+  public static final String OP_STAGE_JOB_ABORT = "op_job_stage_abort";
+
+  /**
+   * Job cleanup: {@value}.
+   */
+  public static final String OP_STAGE_JOB_CLEANUP = "op_job_stage_cleanup";
+
+  /**
+   * Prepare Directories Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_CREATE_TARGET_DIRS =
+      "op_job_stage_create_target_dirs";
+
+  /**
+   * Load Manifest Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_LOAD_MANIFESTS =
+      "op_job_stage_load_manifests";
+
+  /**
+   * Rename files stage duration: {@value}.
+   */
+  public static final String OP_STAGE_JOB_RENAME_FILES =
+      "op_job_stage_rename_files";
+
+
+  /**
+   * Job Setup Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_SETUP = "op_job_stage_setup";
+
+  /**
+   * Job saving _SUCCESS marker Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_SAVE_SUCCESS =
+      "op_job_stage_save_success_marker";
+
+  /**
+   * Output Validation (within job commit) Stage: {@value}.
+   */
+  public static final String OP_STAGE_JOB_VALIDATE_OUTPUT =
+
+      "op_job_stage_optional_validate_output";
+  /**
+   * Task saving manifest file Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_SAVE_MANIFEST =
+      "op_task_stage_save_manifest";
+
+  /**
+   * Task Setup Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_SETUP = "op_task_stage_setup";
+
+
+  /**
+   * Task Commit Stage: {@value}.
+   */
+  public static final String OP_STAGE_TASK_COMMIT = "op_stage_task_commit";
+
+  /** Task Scan directory Stage: {@value}. */
+  public static final String OP_STAGE_TASK_SCAN_DIRECTORY
+      = "op_stage_task_scan_directory";
+
+  private CommitterStatisticNames() {
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
index fcafdd1ed1280..b2d4bfaeeabab 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
@@ -35,6 +35,12 @@
 @InterfaceStability.Unstable
 public final class InternalCommitterConstants {
 
+  /**
+   * How long threads in the thread pool stay alive when
+   * idle. Value in seconds: {@value}.
+   */
+  public static final long THREAD_KEEP_ALIVE_TIME = 60L;
+
   private InternalCommitterConstants() {
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
index 41d36b2a8d7a0..e6524c91961dc 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker;
 import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation;
+import org.apache.hadoop.fs.s3a.statistics.PutTrackerStatistics;
 
 import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
 
@@ -38,8 +39,9 @@
  * in this case:
  * <ol>
  *   <li>{@link #isMagicCommitPath(Path)} will always return false.</li>
- *   <li>{@link #createTracker(Path, String)} will always return an instance
- *   of {@link PutTracker}.</li>
+ *   <li>{@link #isUnderMagicPath(Path)} will always return false.</li>
+ *   <li>{@link #createTracker(Path, String, PutTrackerStatistics)} will always
+ *   return an instance of {@link PutTracker}.</li>
  * </ol>
  *
  * <p>Important</p>: must not directly or indirectly import a class which
@@ -88,9 +90,11 @@ public String keyOfFinalDestination(List<String> elements, String key) {
    * for the commit tracker.
    * @param path path of nominal write
    * @param key key of path of nominal write
+   * @param trackerStatistics tracker statistics
    * @return the tracker for this operation.
    */
-  public PutTracker createTracker(Path path, String key) {
+  public PutTracker createTracker(Path path, String key,
+      PutTrackerStatistics trackerStatistics) {
     final List<String> elements = splitPathToElements(path);
     PutTracker tracker;
 
@@ -106,7 +110,8 @@ public PutTracker createTracker(Path path, String key) {
             key,
             destKey,
             pendingsetPath,
-            owner.getWriteOperationHelper());
+            owner.getWriteOperationHelper(),
+            trackerStatistics);
         LOG.debug("Created {}", tracker);
       } else {
         LOG.warn("File being created has a \"magic\" path, but the filesystem"
@@ -184,4 +189,13 @@ private boolean isCommitMetadataFile(List<String> elements) {
         || last.endsWith(CommitConstants.PENDINGSET_SUFFIX);
   }
 
+  /**
+   * Is this path in/under a magic path...regardless of file type.
+   * This is used to optimize create() operations.
+   * @param path path to check
+   * @return true if the path is one a magic file write expects.
+   */
+  public boolean isUnderMagicPath(Path path) {
+    return magicCommitEnabled && isMagicPath(splitPathToElements(path));
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
index 19713909f50c4..18bb90da8208b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
 import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.E_NO_MAGIC_PATH_ELEMENT;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java
deleted file mode 100644
index c318e86605e0c..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.commit;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utility class for parallel execution, takes closures for the various
- * actions.
- * There is no retry logic: it is expected to be handled by the closures.
- */
-public final class Tasks {
-  private static final Logger LOG = LoggerFactory.getLogger(Tasks.class);
-
-  private Tasks() {
-  }
-
-  /**
-   * Callback invoked to process an item.
-   * @param <I> item type being processed
-   * @param <E> exception class which may be raised
-   */
-  @FunctionalInterface
-  public interface Task<I, E extends Exception> {
-    void run(I item) throws E;
-  }
-
-  /**
-   * Callback invoked on a failure.
-   * @param <I> item type being processed
-   * @param <E> exception class which may be raised
-   */
-  @FunctionalInterface
-  public interface FailureTask<I, E extends Exception> {
-
-    /**
-     * process a failure.
-     * @param item item the task is processing
-     * @param exception the exception which was raised.
-     * @throws E Exception of type E
-     */
-    void run(I item, Exception exception) throws E;
-  }
-
-  /**
-   * Builder for task execution.
-   * @param <I> item type
-   */
-  public static class Builder<I> {
-    private final Iterable<I> items;
-    private Submitter service = null;
-    private FailureTask<I, ?> onFailure = null;
-    private boolean stopOnFailure = false;
-    private boolean suppressExceptions = false;
-    private Task<I, ?> revertTask = null;
-    private boolean stopRevertsOnFailure = false;
-    private Task<I, ?> abortTask = null;
-    private boolean stopAbortsOnFailure = false;
-
-    /**
-     * Create the builder.
-     * @param items items to process
-     */
-    Builder(Iterable<I> items) {
-      this.items = items;
-    }
-
-    /**
-     * Declare executor service: if null, the tasks are executed in a single
-     * thread.
-     * @param submitter service to schedule tasks with.
-     * @return this builder.
-     */
-    public Builder<I> executeWith(Submitter submitter) {
-      this.service = submitter;
-      return this;
-    }
-
-    public Builder<I> onFailure(FailureTask<I, ?> task) {
-      this.onFailure = task;
-      return this;
-    }
-
-    public Builder<I> stopOnFailure() {
-      this.stopOnFailure = true;
-      return this;
-    }
-
-    public Builder<I> suppressExceptions() {
-      return suppressExceptions(true);
-    }
-
-    public Builder<I> suppressExceptions(boolean suppress) {
-      this.suppressExceptions = suppress;
-      return this;
-    }
-
-    public Builder<I> revertWith(Task<I, ?> task) {
-      this.revertTask = task;
-      return this;
-    }
-
-    public Builder<I> stopRevertsOnFailure() {
-      this.stopRevertsOnFailure = true;
-      return this;
-    }
-
-    public Builder<I> abortWith(Task<I, ?> task) {
-      this.abortTask = task;
-      return this;
-    }
-
-    public Builder<I> stopAbortsOnFailure() {
-      this.stopAbortsOnFailure = true;
-      return this;
-    }
-
-    public <E extends Exception> boolean run(Task<I, E> task) throws E {
-      if (service != null) {
-        return runParallel(task);
-      } else {
-        return runSingleThreaded(task);
-      }
-    }
-
-    private <E extends Exception> boolean runSingleThreaded(Task<I, E> task)
-        throws E {
-      List<I> succeeded = new ArrayList<>();
-      List<Exception> exceptions = new ArrayList<>();
-
-      Iterator<I> iterator = items.iterator();
-      boolean threw = true;
-      try {
-        while (iterator.hasNext()) {
-          I item = iterator.next();
-          try {
-            task.run(item);
-            succeeded.add(item);
-
-          } catch (Exception e) {
-            exceptions.add(e);
-
-            if (onFailure != null) {
-              try {
-                onFailure.run(item, e);
-              } catch (Exception failException) {
-                LOG.error("Failed to clean up on failure", e);
-                // keep going
-              }
-            }
-
-            if (stopOnFailure) {
-              break;
-            }
-          }
-        }
-
-        threw = false;
-
-      } finally {
-        // threw handles exceptions that were *not* caught by the catch block,
-        // and exceptions that were caught and possibly handled by onFailure
-        // are kept in exceptions.
-        if (threw || !exceptions.isEmpty()) {
-          if (revertTask != null) {
-            boolean failed = false;
-            for (I item : succeeded) {
-              try {
-                revertTask.run(item);
-              } catch (Exception e) {
-                LOG.error("Failed to revert task", e);
-                failed = true;
-                // keep going
-              }
-              if (stopRevertsOnFailure && failed) {
-                break;
-              }
-            }
-          }
-
-          if (abortTask != null) {
-            boolean failed = false;
-            while (iterator.hasNext()) {
-              try {
-                abortTask.run(iterator.next());
-              } catch (Exception e) {
-                failed = true;
-                LOG.error("Failed to abort task", e);
-                // keep going
-              }
-              if (stopAbortsOnFailure && failed) {
-                break;
-              }
-            }
-          }
-        }
-      }
-
-      if (!suppressExceptions && !exceptions.isEmpty()) {
-        Tasks.<E>throwOne(exceptions);
-      }
-
-      return !threw && exceptions.isEmpty();
-    }
-
-    private <E extends Exception> boolean runParallel(final Task<I, E> task)
-        throws E {
-      final Queue<I> succeeded = new ConcurrentLinkedQueue<>();
-      final Queue<Exception> exceptions = new ConcurrentLinkedQueue<>();
-      final AtomicBoolean taskFailed = new AtomicBoolean(false);
-      final AtomicBoolean abortFailed = new AtomicBoolean(false);
-      final AtomicBoolean revertFailed = new AtomicBoolean(false);
-
-      List<Future<?>> futures = new ArrayList<>();
-
-      for (final I item : items) {
-        // submit a task for each item that will either run or abort the task
-        futures.add(service.submit(new Runnable() {
-          @Override
-          public void run() {
-            if (!(stopOnFailure && taskFailed.get())) {
-              // run the task
-              boolean threw = true;
-              try {
-                LOG.debug("Executing task");
-                task.run(item);
-                succeeded.add(item);
-                LOG.debug("Task succeeded");
-
-                threw = false;
-
-              } catch (Exception e) {
-                taskFailed.set(true);
-                exceptions.add(e);
-                LOG.info("Task failed", e);
-
-                if (onFailure != null) {
-                  try {
-                    onFailure.run(item, e);
-                  } catch (Exception failException) {
-                    LOG.error("Failed to clean up on failure", e);
-                    // swallow the exception
-                  }
-                }
-              } finally {
-                if (threw) {
-                  taskFailed.set(true);
-                }
-              }
-
-            } else if (abortTask != null) {
-              // abort the task instead of running it
-              if (stopAbortsOnFailure && abortFailed.get()) {
-                return;
-              }
-
-              boolean failed = true;
-              try {
-                LOG.info("Aborting task");
-                abortTask.run(item);
-                failed = false;
-              } catch (Exception e) {
-                LOG.error("Failed to abort task", e);
-                // swallow the exception
-              } finally {
-                if (failed) {
-                  abortFailed.set(true);
-                }
-              }
-            }
-          }
-        }));
-      }
-
-      // let the above tasks complete (or abort)
-      waitFor(futures);
-      int futureCount = futures.size();
-      futures.clear();
-
-      if (taskFailed.get() && revertTask != null) {
-        // at least one task failed, revert any that succeeded
-        LOG.info("Reverting all {} succeeded tasks from {} futures",
-            succeeded.size(), futureCount);
-        for (final I item : succeeded) {
-          futures.add(service.submit(() -> {
-            if (stopRevertsOnFailure && revertFailed.get()) {
-              return;
-            }
-
-            boolean failed = true;
-            try {
-              revertTask.run(item);
-              failed = false;
-            } catch (Exception e) {
-              LOG.error("Failed to revert task", e);
-              // swallow the exception
-            } finally {
-              if (failed) {
-                revertFailed.set(true);
-              }
-            }
-          }));
-        }
-
-        // let the revert tasks complete
-        waitFor(futures);
-      }
-
-      if (!suppressExceptions && !exceptions.isEmpty()) {
-        Tasks.<E>throwOne(exceptions);
-      }
-
-      return !taskFailed.get();
-    }
-  }
-
-  /**
-   * Wait for all the futures to complete; there's a small sleep between
-   * each iteration; enough to yield the CPU.
-   * @param futures futures.
-   */
-  private static void waitFor(Collection<Future<?>> futures) {
-    int size = futures.size();
-    LOG.debug("Waiting for {} tasks to complete", size);
-    int oldNumFinished = 0;
-    while (true) {
-      int numFinished = (int) futures.stream().filter(Future::isDone).count();
-
-      if (oldNumFinished != numFinished) {
-        LOG.debug("Finished count -> {}/{}", numFinished, size);
-        oldNumFinished = numFinished;
-      }
-
-      if (numFinished == size) {
-        // all of the futures are done, stop looping
-        break;
-      } else {
-        try {
-          Thread.sleep(10);
-        } catch (InterruptedException e) {
-          futures.forEach(future -> future.cancel(true));
-          Thread.currentThread().interrupt();
-          break;
-        }
-      }
-    }
-  }
-
-  public static <I> Builder<I> foreach(Iterable<I> items) {
-    return new Builder<>(items);
-  }
-
-  public static <I> Builder<I> foreach(I[] items) {
-    return new Builder<>(Arrays.asList(items));
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <E extends Exception> void throwOne(
-      Collection<Exception> exceptions)
-      throws E {
-    Iterator<Exception> iter = exceptions.iterator();
-    Exception e = iter.next();
-    Class<? extends Exception> exceptionClass = e.getClass();
-
-    while (iter.hasNext()) {
-      Exception other = iter.next();
-      if (!exceptionClass.isInstance(other)) {
-        e.addSuppressed(other);
-      }
-    }
-
-    Tasks.<E>castAndThrow(e);
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <E extends Exception> void castAndThrow(Exception e) throws E {
-    if (e instanceof RuntimeException) {
-      throw (RuntimeException) e;
-    }
-    throw (E) e;
-  }
-
-  /**
-   * Interface to whatever lets us submit tasks.
-   */
-  public interface Submitter {
-
-    /**
-     * Submit work.
-     * @param task task to execute
-     * @return the future of the submitted task.
-     */
-    Future<?> submit(Runnable task);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
index fd7341025667a..108e6d45bc621 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3a.commit.files;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.util.ArrayList;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.util.JsonSerialization;
@@ -62,8 +64,7 @@
 @SuppressWarnings("unused")
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public class PendingSet extends PersistentCommitData
-    implements IOStatisticsSource {
+public class PendingSet extends PersistentCommitData<PendingSet> {
   private static final Logger LOG = LoggerFactory.getLogger(PendingSet.class);
 
   /**
@@ -111,42 +112,32 @@ public PendingSet(int size) {
   }
 
   /**
-   * Get a JSON serializer for this class.
+   * Get a shared JSON serializer for this class.
    * @return a serializer.
    */
   public static JsonSerialization<PendingSet> serializer() {
-    return new JsonSerialization<>(PendingSet.class, false, true);
+    return new JsonSerialization<>(PendingSet.class, false, false);
   }
 
+
   /**
    * Load an instance from a file, then validate it.
    * @param fs filesystem
    * @param path path
+   * @param status status of file to load
    * @return the loaded instance
    * @throws IOException IO failure
    * @throws ValidationFailure if the data is invalid
    */
-  public static PendingSet load(FileSystem fs, Path path)
+  public static PendingSet load(FileSystem fs, Path path,
+      @Nullable FileStatus status)
       throws IOException {
     LOG.debug("Reading pending commits in file {}", path);
-    PendingSet instance = serializer().load(fs, path);
+    PendingSet instance = serializer().load(fs, path, status);
     instance.validate();
     return instance;
   }
 
-  /**
-   * Load an instance from a file, then validate it.
-   * @param fs filesystem
-   * @param status status of file to load
-   * @return the loaded instance
-   * @throws IOException IO failure
-   * @throws ValidationFailure if the data is invalid
-   */
-  public static PendingSet load(FileSystem fs, FileStatus status)
-      throws IOException {
-    return load(fs, status.getPath());
-  }
-
   /**
    * Add a commit.
    * @param commit the single commit
@@ -195,8 +186,8 @@ public void validate() throws ValidationFailure {
   }
 
   @Override
-  public byte[] toBytes() throws IOException {
-    return serializer().toBytes(this);
+  public byte[] toBytes(JsonSerialization<PendingSet> serializer) throws IOException {
+    return serializer.toBytes(this);
   }
 
   /**
@@ -208,9 +199,10 @@ public int size() {
   }
 
   @Override
-  public void save(FileSystem fs, Path path, boolean overwrite)
-      throws IOException {
-    serializer().save(fs, path, this, overwrite);
+  public IOStatistics save(final FileSystem fs,
+      final Path path,
+      final JsonSerialization<PendingSet> serializer) throws IOException {
+    return saveFile(fs, path, this, serializer, true);
   }
 
   /** @return the version marker. */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
index dba44b9a011d9..be2de6cf89cfd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
@@ -21,19 +21,33 @@
 import java.io.IOException;
 import java.io.Serializable;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
 
 /**
  * Class for single/multiple commit data structures.
+ * The mapreduce hierarchy {@code AbstractManifestData} is a fork
+ * of this; the Success data JSON format must stay compatible
  */
-@SuppressWarnings("serial")
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public abstract class PersistentCommitData implements Serializable {
+public abstract class PersistentCommitData<T extends PersistentCommitData>
+    implements Serializable, IOStatisticsSource {
+  private static final Logger LOG = LoggerFactory.getLogger(PersistentCommitData.class);
 
   /**
    * Supported version value: {@value}.
@@ -52,18 +66,109 @@ public abstract class PersistentCommitData implements Serializable {
    * Serialize to JSON and then to a byte array, after performing a
    * preflight validation of the data to be saved.
    * @return the data in a persistable form.
+   * @param serializer serializer to use
    * @throws IOException serialization problem or validation failure.
    */
-  public abstract byte[] toBytes() throws IOException;
+  public abstract byte[] toBytes(JsonSerialization<T> serializer) throws IOException;
 
   /**
    * Save to a hadoop filesystem.
+   * The destination file is overwritten, and on s3a stores the
+   * performance flag is set to turn off all existence checks and
+   * parent dir cleanup.
+   * The assumption here is: the job knows what it is doing.
+   *
    * @param fs filesystem
    * @param path path
-   * @param overwrite should any existing file be overwritten
+   * @param serializer serializer to use
+   * @return IOStats from the output stream.
+   *
    * @throws IOException IO exception
    */
-  public abstract void save(FileSystem fs, Path path, boolean overwrite)
+  public abstract IOStatistics save(FileSystem fs, Path path, JsonSerialization<T> serializer)
       throws IOException;
 
+  /**
+   * Load an instance from a status, then validate it.
+   * This uses the openFile() API, which S3A supports for
+   * faster load and declaring sequential access, always
+   * @param <T> type of persistent format
+   * @param fs filesystem
+   * @param status status of file to load
+   * @param serializer serializer to use
+   * @return the loaded instance
+   * @throws IOException IO failure
+   * @throws ValidationFailure if the data is invalid
+   */
+  public static <T extends PersistentCommitData> T load(FileSystem fs,
+      FileStatus status,
+      JsonSerialization<T> serializer)
+      throws IOException {
+    Path path = status.getPath();
+    LOG.debug("Reading commit data from file {}", path);
+    T result = serializer.load(fs, path, status);
+    result.validate();
+    return result;
+  }
+
+  /**
+   * Save to a file.
+   * This uses the createFile() API, which S3A supports for
+   * faster load and declaring sequential access, always
+   *
+   * @param <T> type of persistent format
+   * @param fs filesystem
+   * @param path path to save to
+   * @param instance data to save
+   * @param serializer serializer to use
+   * @param performance skip all safety check on the write
+   *
+   * @return any IOStatistics from the output stream, or null
+   *
+   * @throws IOException IO failure
+   */
+  public static <T extends PersistentCommitData> IOStatistics saveFile(
+      final FileSystem fs,
+      final Path path,
+      final T instance,
+      final JsonSerialization<T> serializer,
+      final boolean performance)
+      throws IOException {
+
+    FSDataOutputStreamBuilder builder = fs.createFile(path)
+        .create()
+        .recursive()
+        .overwrite(true);
+    // switch to performance mode
+    builder.opt(FS_S3A_CREATE_PERFORMANCE, performance);
+    return saveToStream(path, instance, builder, serializer);
+  }
+
+  /**
+   * Save to a file.
+   * This uses the createFile() API, which S3A supports for
+   * faster load and declaring sequential access, always
+   * @param <T> type of persistent format
+   * @param path path to save to (used for logging)
+   * @param instance data to save
+   * @param builder builder already prepared for the write
+   * @param serializer serializer to use
+   * @return any IOStatistics from the output stream, or null
+   * @throws IOException IO failure
+   */
+  public static <T extends PersistentCommitData> IOStatistics saveToStream(
+      final Path path,
+      final T instance,
+      final FSDataOutputStreamBuilder builder,
+      final JsonSerialization<T> serializer) throws IOException {
+    LOG.debug("saving commit data to file {}", path);
+    FSDataOutputStream dataOutputStream = builder.build();
+    try {
+      dataOutputStream.write(serializer.toBytes(instance));
+    } finally {
+      dataOutputStream.close();
+    }
+    return dataOutputStream.getIOStatistics();
+  }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
index 021cc7b34b1f5..77c3fed11fb24 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3a.commit.files;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.Serializable;
@@ -33,17 +34,18 @@
 import com.amazonaws.services.s3.model.PartETag;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.util.JsonSerialization;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.fs.s3a.commit.CommitUtils.validateCollectionClass;
 import static org.apache.hadoop.fs.s3a.commit.ValidationFailure.verify;
@@ -67,8 +69,8 @@
 @SuppressWarnings("unused")
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public class SinglePendingCommit extends PersistentCommitData
-    implements Iterable<String>, IOStatisticsSource {
+public class SinglePendingCommit extends PersistentCommitData<SinglePendingCommit>
+    implements Iterable<String> {
 
   /**
    * Serialization ID: {@value}.
@@ -139,20 +141,45 @@ public SinglePendingCommit() {
    * @return a serializer.
    */
   public static JsonSerialization<SinglePendingCommit> serializer() {
-    return new JsonSerialization<>(SinglePendingCommit.class, false, true);
+    return new JsonSerialization<>(SinglePendingCommit.class, false, false);
   }
 
   /**
    * Load an instance from a file, then validate it.
    * @param fs filesystem
    * @param path path
+   * @param status nullable status of file to load
+   * @param serDeser serializer; if null use the shared static one.
    * @return the loaded instance
    * @throws IOException IO failure
    * @throws ValidationFailure if the data is invalid
    */
-  public static SinglePendingCommit load(FileSystem fs, Path path)
+  public static SinglePendingCommit load(FileSystem fs,
+      Path path,
+      FileStatus status,
+      JsonSerialization<SinglePendingCommit> serDeser)
       throws IOException {
-    SinglePendingCommit instance = serializer().load(fs, path);
+    return load(fs, path, serDeser, null);
+  }
+
+  /**
+   * Load an instance from a file, then validate it.
+   * @param fs filesystem
+   * @param path path
+   * @param serDeser deserializer
+   * @param status status of file to load or null
+   * @return the loaded instance
+   * @throws IOException IO failure
+   * @throws ValidationFailure if the data is invalid
+   */
+  public static SinglePendingCommit load(FileSystem fs,
+      Path path,
+      JsonSerialization<SinglePendingCommit> serDeser,
+      @Nullable FileStatus status)
+      throws IOException {
+    JsonSerialization<SinglePendingCommit> jsonSerialization =
+        serDeser != null ? serDeser : serializer();
+    SinglePendingCommit instance = jsonSerialization.load(fs, path, status);
     instance.filename = path.toString();
     instance.validate();
     return instance;
@@ -246,15 +273,16 @@ public String toString() {
   }
 
   @Override
-  public byte[] toBytes() throws IOException {
+  public byte[] toBytes(JsonSerialization<SinglePendingCommit> serializer) throws IOException {
     validate();
-    return serializer().toBytes(this);
+    return serializer.toBytes(this);
   }
 
   @Override
-  public void save(FileSystem fs, Path path, boolean overwrite)
-      throws IOException {
-    serializer().save(fs, path, this, overwrite);
+  public IOStatistics save(final FileSystem fs,
+      final Path path,
+      final JsonSerialization<SinglePendingCommit> serializer) throws IOException {
+    return saveFile(fs, path, this, serializer, true);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
index 4500e1429e447..0e0a03f587081 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
@@ -30,13 +30,14 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
-import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.util.JsonSerialization;
 
 /**
@@ -47,14 +48,28 @@
  * <ol>
  *   <li>File length == 0: classic {@code FileOutputCommitter}.</li>
  *   <li>Loadable as {@link SuccessData}:
- *   A s3guard committer with name in in {@link #committer} field.</li>
+ *   An S3A committer with name in in {@link #committer} field.</li>
  *   <li>Not loadable? Something else.</li>
  * </ol>
  *
- * This is an unstable structure intended for diagnostics and testing.
- * Applications reading this data should use/check the {@link #name} field
- * to differentiate from any other JSON-based manifest and to identify
- * changes in the output format.
+ * This should be considered public, and MUST stay compatible
+ * at the JSON format level with that of
+ * {@code org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData}
+ * <p>
+ * The JSON format SHOULD be considered public and evolving
+ * with compatibility across versions.
+ * <p>
+ * All the Java serialization data is different and may change
+ * across versions with no stability guarantees other than
+ * "manifest summaries MAY be serialized between processes with
+ * the exact same version of this binary on their classpaths."
+ * That is sufficient for testing in Spark.
+ * <p>
+ * To aid with Java serialization, the maps and lists are
+ * exclusively those which serialize well.
+ * IOStatisticsSnapshot has a lot of complexity in marshalling
+ * there; this class doesn't worry about concurrent access
+ * so is simpler.
  *
  * Note: to deal with scale issues, the S3A committers do not include any
  * more than the number of objects listed in
@@ -65,8 +80,7 @@
 @SuppressWarnings("unused")
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-public class SuccessData extends PersistentCommitData
-    implements IOStatisticsSource {
+public class SuccessData extends PersistentCommitData<SuccessData> {
 
   private static final Logger LOG = LoggerFactory.getLogger(SuccessData.class);
 
@@ -80,7 +94,7 @@ public class SuccessData extends PersistentCommitData
   /**
    * Serialization ID: {@value}.
    */
-  private static final long serialVersionUID = 507133045258460083L + VERSION;
+  private static final long serialVersionUID = 507133045258460084L + VERSION;
 
   /**
    * Name to include in persisted data, so as to differentiate from
@@ -92,7 +106,14 @@ public class SuccessData extends PersistentCommitData
   /**
    * Name of file; includes version marker.
    */
-  private String name;
+  private String name = NAME;
+
+  /**
+   * Did this succeed?
+   * It is implicitly true in a _SUCCESS file, but if the file
+   * is also saved to a log dir, then it depends on the outcome
+   */
+  private boolean success = true;
 
   /** Timestamp of creation. */
   private long timestamp;
@@ -142,7 +163,17 @@ public class SuccessData extends PersistentCommitData
    * IOStatistics.
    */
   @JsonProperty("iostatistics")
-  private IOStatisticsSnapshot iostats = new IOStatisticsSnapshot();
+  private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot();
+
+  /**
+   * State (committed, aborted).
+   */
+  private String state;
+
+  /**
+   * Stage: last stage executed.
+   */
+  private String stage;
 
   @Override
   public void validate() throws ValidationFailure {
@@ -153,16 +184,17 @@ public void validate() throws ValidationFailure {
   }
 
   @Override
-  public byte[] toBytes() throws IOException {
-    return serializer().toBytes(this);
+  public byte[] toBytes(JsonSerialization<SuccessData> serializer) throws IOException {
+    return serializer.toBytes(this);
   }
 
   @Override
-  public void save(FileSystem fs, Path path, boolean overwrite)
-      throws IOException {
+  public IOStatistics save(final FileSystem fs,
+      final Path path,
+      final JsonSerialization<SuccessData> serializer) throws IOException {
     // always set the name field before being saved.
     name = NAME;
-    serializer().save(fs, path, this, overwrite);
+    return saveFile(fs, path, this, serializer, true);
   }
 
   @Override
@@ -250,8 +282,8 @@ public static SuccessData load(FileSystem fs, Path path)
    * Get a JSON serializer for this class.
    * @return a serializer.
    */
-  private static JsonSerialization<SuccessData> serializer() {
-    return new JsonSerialization<>(SuccessData.class, false, true);
+  public static JsonSerialization<SuccessData> serializer() {
+    return new JsonSerialization<>(SuccessData.class, false, false);
   }
 
   public String getName() {
@@ -371,10 +403,59 @@ public void setJobIdSource(final String jobIdSource) {
 
   @Override
   public IOStatisticsSnapshot getIOStatistics() {
-    return iostats;
+    return iostatistics;
   }
 
   public void setIOStatistics(final IOStatisticsSnapshot ioStatistics) {
-    this.iostats = ioStatistics;
+    this.iostatistics = ioStatistics;
+  }
+
+  /**
+   * Set the success flag.
+   * @param success did the job succeed?
+   */
+  public void setSuccess(boolean success) {
+    this.success = success;
+  }
+
+  /**
+   * Get the success flag.
+   * @return did the job succeed?
+   */
+  public boolean getSuccess() {
+    return success;
+  }
+
+  public String getState() {
+    return state;
+  }
+
+  public void setState(String state) {
+    this.state = state;
+  }
+
+  public String getStage() {
+    return stage;
+  }
+
+  /**
+   * Add a diagnostics entry.
+   * @param key name
+   * @param value value
+   */
+  public void putDiagnostic(String key, String value) {
+    diagnostics.put(key, value);
+  }
+
+  /**
+   * Note a failure by setting success flag to false,
+   * then add the exception to the diagnostics.
+   * @param thrown throwable
+   */
+  public void recordJobFailure(Throwable thrown) {
+    setSuccess(false);
+    String stacktrace = ExceptionUtils.getStackTrace(thrown);
+    diagnostics.put("exception", thrown.toString());
+    diagnostics.put("stacktrace", stacktrace);
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
index 0d574948e91a3..36eedba83b854 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
@@ -29,12 +29,11 @@
  *   <li>The summary information saved in the {@code _SUCCESS} file.</li>
  * </ol>
  *
- * There are no guarantees of stability between versions; these are internal
- * structures.
  *
  * The {@link org.apache.hadoop.fs.s3a.commit.files.SuccessData} file is
- * the one visible to callers after a job completes; it is an unstable
- * manifest intended for testing only.
+ * the one visible to callers after a job completes; it is compatible with
+ * the manifest committer format persisted in
+ * {@code org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData}
  *
  */
 @InterfaceAudience.Private
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java
new file mode 100644
index 0000000000000..17f63e6dff7f0
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.impl;
+
+import org.apache.hadoop.fs.audit.AuditConstants;
+import org.apache.hadoop.fs.audit.CommonAuditContext;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext;
+
+/**
+ * Class to track/update context information to set
+ * in threads.
+ */
+public final class AuditContextUpdater {
+
+  /**
+   * Job ID.
+   */
+  private final String jobId;
+
+  /**
+   * Task attempt ID for auditing.
+   */
+  private final String taskAttemptId;
+
+  /**
+   * Construct. Stores job information
+   * to attach to thread contexts.
+   * @param jobContext job/task context.
+   */
+  public AuditContextUpdater(final JobContext jobContext) {
+    JobID contextJobID = jobContext.getJobID();
+    this.jobId = contextJobID != null
+            ? contextJobID.toString()
+            : null;
+
+    if (jobContext instanceof TaskAttemptContext) {
+      // it's a task, extract info for auditing
+      final TaskAttemptID tid = ((TaskAttemptContext) jobContext).getTaskAttemptID();
+      this.taskAttemptId = tid != null
+          ? tid.toString()
+          : null;
+    } else {
+      this.taskAttemptId = null;
+    }
+  }
+
+  public AuditContextUpdater(String jobId) {
+    this.jobId = jobId;
+    this.taskAttemptId = null;
+  }
+
+  /**
+   * Add job/task info to current audit context.
+   */
+  public void updateCurrentAuditContext() {
+    final CommonAuditContext auditCtx = currentAuditContext();
+    if (jobId != null) {
+      auditCtx.put(AuditConstants.PARAM_JOB_ID, jobId);
+    } else {
+      currentAuditContext().remove(AuditConstants.PARAM_JOB_ID);
+    }
+    if (taskAttemptId != null) {
+      auditCtx.put(AuditConstants.PARAM_TASK_ATTEMPT_ID, taskAttemptId);
+    } else {
+      currentAuditContext().remove(CommitConstants.PARAM_TASK_ATTEMPT_ID);
+    }
+
+  }
+
+  /**
+   * Remove job/task info from the current audit context.
+   */
+  public void resetCurrentAuditContext() {
+    currentAuditContext().remove(AuditConstants.PARAM_JOB_ID);
+    currentAuditContext().remove(CommitConstants.PARAM_TASK_ATTEMPT_ID);
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java
new file mode 100644
index 0000000000000..c93d2d8f73913
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.impl;
+
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.JsonSerialization;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.fs.s3a.Constants.THREAD_POOL_SHUTDOWN_DELAY_SECONDS;
+import static org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter.THREAD_PREFIX;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.THREAD_KEEP_ALIVE_TIME;
+
+/**
+ * Commit context.
+ *
+ * It is used to manage the final commit sequence where files become
+ * visible.
+ *
+ * Once the commit operation has completed, it must be closed.
+ * It MUST NOT be reused.
+ *
+ * Audit integration: job and task attributes are added to the thread local context
+ * on create, removed on close().
+ *
+ * JSON Serializers are created on demand, on a per thread basis.
+ * A {@link WeakReferenceThreadMap} is used here; a GC may lose the
+ * references, but they will recreated as needed.
+ */
+public final class CommitContext implements Closeable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      CommitContext.class);
+
+  /**
+   * The actual commit operations.
+   */
+  private final CommitOperations commitOperations;
+
+  /**
+   * Job Context.
+   */
+  private final JobContext jobContext;
+
+  /**
+   * Serializer pool.
+   */
+
+  private final WeakReferenceThreadMap<JsonSerialization<PendingSet>>
+      pendingSetSerializer =
+        new WeakReferenceThreadMap<>((k) -> PendingSet.serializer(), null);
+
+  private final WeakReferenceThreadMap<JsonSerialization<SinglePendingCommit>>
+      singleCommitSerializer =
+        new WeakReferenceThreadMap<>((k) -> SinglePendingCommit.serializer(), null);
+
+  /**
+   * Submitter for per task operations, e.g loading manifests.
+   */
+  private PoolSubmitter outerSubmitter;
+
+  /**
+   * Submitter for operations within the tasks,
+   * such as POSTing the final commit operations.
+   */
+  private PoolSubmitter innerSubmitter;
+
+  /**
+   * Job Configuration.
+   */
+  private final Configuration conf;
+
+  /**
+   * Job ID.
+   */
+  private final String jobId;
+
+  /**
+   * Audit context; will be reset when this is closed.
+   */
+  private final AuditContextUpdater auditContextUpdater;
+
+  /**
+   * Number of committer threads.
+   */
+  private final int committerThreads;
+
+  /**
+   * Should IOStatistics be collected by the committer?
+   */
+  private final boolean collectIOStatistics;
+
+  /**
+   * IOStatisticsContext to switch to in all threads
+   * taking part in the commit operation.
+   * This ensures that the IOStatistics collected in the
+   * worker threads will be aggregated into the total statistics
+   * of the thread calling the committer commit/abort methods.
+   */
+  private final IOStatisticsContext ioStatisticsContext;
+
+  /**
+   * Create.
+   * @param commitOperations commit callbacks
+   * @param jobContext job context
+   * @param committerThreads number of commit threads
+   * @param ioStatisticsContext IOStatistics context of current thread
+   */
+  public CommitContext(
+      final CommitOperations commitOperations,
+      final JobContext jobContext,
+      final int committerThreads,
+      final IOStatisticsContext ioStatisticsContext) {
+    this.commitOperations = commitOperations;
+    this.jobContext = jobContext;
+    this.conf = jobContext.getConfiguration();
+    JobID contextJobID = jobContext.getJobID();
+    // either the job ID or make one up as it will be
+    // used for the filename of any reports.
+    this.jobId = contextJobID != null
+        ? contextJobID.toString()
+        : ("job-without-id-at-" + System.currentTimeMillis());
+    this.collectIOStatistics = conf.getBoolean(
+        S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS,
+        S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT);
+    this.ioStatisticsContext = Objects.requireNonNull(ioStatisticsContext);
+    this.auditContextUpdater = new AuditContextUpdater(jobContext);
+    this.auditContextUpdater.updateCurrentAuditContext();
+    this.committerThreads = committerThreads;
+    buildSubmitters();
+  }
+
+  /**
+   * Create for testing.
+   * This has no job context; instead the values
+   * are set explicitly.
+   * @param commitOperations commit callbacks
+   * @param conf job conf
+   * @param jobId ID
+   * @param committerThreads number of commit threads
+   * @param ioStatisticsContext IOStatistics context of current thread
+   */
+  public CommitContext(final CommitOperations commitOperations,
+      final Configuration conf,
+      final String jobId,
+      final int committerThreads,
+      final IOStatisticsContext ioStatisticsContext) {
+    this.commitOperations = commitOperations;
+    this.jobContext = null;
+    this.conf = conf;
+    this.jobId = jobId;
+    this.collectIOStatistics = false;
+    this.ioStatisticsContext = Objects.requireNonNull(ioStatisticsContext);
+    this.auditContextUpdater = new AuditContextUpdater(jobId);
+    this.auditContextUpdater.updateCurrentAuditContext();
+    this.committerThreads = committerThreads;
+    buildSubmitters();
+  }
+
+  /**
+   * Build the submitters and thread pools if the number of committerThreads
+   * is greater than zero.
+   * This should only be called in constructors; it is synchronized to keep
+   * SpotBugs happy.
+   */
+  private synchronized void buildSubmitters() {
+    if (committerThreads != 0) {
+      outerSubmitter = new PoolSubmitter(buildThreadPool(committerThreads));
+    }
+  }
+
+  /**
+   * Returns an {@link ExecutorService} for parallel tasks. The number of
+   * threads in the thread-pool is set by fs.s3a.committer.threads.
+   * If num-threads is 0, this will raise an exception.
+   * The threads have a lifespan set by
+   * {@link InternalCommitterConstants#THREAD_KEEP_ALIVE_TIME}.
+   * When the thread pool is full, the caller runs
+   * policy takes over.
+   * @param numThreads thread count, may be negative.
+   * @return an {@link ExecutorService} for the number of threads
+   */
+  private ExecutorService buildThreadPool(
+      int numThreads) {
+    if (numThreads < 0) {
+      // a negative number means "multiple of available processors"
+      numThreads = numThreads * -Runtime.getRuntime().availableProcessors();
+    }
+    Preconditions.checkArgument(numThreads > 0,
+        "Cannot create a thread pool with no threads");
+    LOG.debug("creating thread pool of size {}", numThreads);
+    final ThreadFactory factory = new ThreadFactoryBuilder()
+        .setDaemon(true)
+        .setNameFormat(THREAD_PREFIX + jobId + "-%d")
+        .build();
+    return new HadoopThreadPoolExecutor(0, numThreads,
+        THREAD_KEEP_ALIVE_TIME,
+        TimeUnit.SECONDS,
+        new LinkedBlockingQueue<>(),
+        factory,
+        new ThreadPoolExecutor.CallerRunsPolicy());
+  }
+
+  /**
+   * Commit the operation, throwing an exception on any failure.
+   * See {@code CommitOperations#commitOrFail(SinglePendingCommit)}.
+   * @param commit commit to execute
+   * @throws IOException on a failure
+   */
+  public void commitOrFail(SinglePendingCommit commit) throws IOException {
+    commitOperations.commitOrFail(commit);
+  }
+
+  /**
+   * Commit a single pending commit; exceptions are caught
+   * and converted to an outcome.
+   * See {@link CommitOperations#commit(SinglePendingCommit, String)}.
+   * @param commit entry to commit
+   * @param origin origin path/string for outcome text
+   * @return the outcome
+   */
+  public CommitOperations.MaybeIOE commit(SinglePendingCommit commit,
+      String origin) {
+    return commitOperations.commit(commit, origin);
+  }
+
+  /**
+   * See {@link CommitOperations#abortSingleCommit(SinglePendingCommit)}.
+   * @param commit pending commit to abort
+   * @throws FileNotFoundException if the abort ID is unknown
+   * @throws IOException on any failure
+   */
+  public void abortSingleCommit(final SinglePendingCommit commit)
+      throws IOException {
+    commitOperations.abortSingleCommit(commit);
+  }
+
+  /**
+   * See {@link CommitOperations#revertCommit(SinglePendingCommit)}.
+   * @param commit pending commit
+   * @throws IOException failure
+   */
+  public void revertCommit(final SinglePendingCommit commit)
+      throws IOException {
+    commitOperations.revertCommit(commit);
+  }
+
+  /**
+   * See {@link CommitOperations#abortMultipartCommit(String, String)}..
+   * @param destKey destination key
+   * @param uploadId upload to cancel
+   * @throws FileNotFoundException if the abort ID is unknown
+   * @throws IOException on any failure
+   */
+  public void abortMultipartCommit(
+      final String destKey,
+      final String uploadId)
+      throws IOException {
+    commitOperations.abortMultipartCommit(destKey, uploadId);
+  }
+
+  @Override
+  public synchronized void close() throws IOException {
+
+    destroyThreadPools();
+    auditContextUpdater.resetCurrentAuditContext();
+  }
+
+  @Override
+  public String toString() {
+    return "CommitContext{}";
+  }
+
+  /**
+   * Job Context.
+   * @return job context.
+   */
+  public JobContext getJobContext() {
+    return jobContext;
+  }
+
+  /**
+   * Return a submitter.
+   * If created with 0 threads, this returns null so
+   * TaskPool knows to run it in the current thread.
+   * @return a submitter or null
+   */
+  public synchronized TaskPool.Submitter getOuterSubmitter() {
+    return outerSubmitter;
+  }
+
+  /**
+   * Return a submitter. As this pool is used less often,
+   * create it on demand.
+   * If created with 0 threads, this returns null so
+   * TaskPool knows to run it in the current thread.
+   * @return a submitter or null
+   */
+  public synchronized TaskPool.Submitter getInnerSubmitter() {
+    if (innerSubmitter == null && committerThreads > 0) {
+      innerSubmitter = new PoolSubmitter(buildThreadPool(committerThreads));
+    }
+    return innerSubmitter;
+  }
+
+  /**
+   * Get a serializer for .pending files.
+   * @return a serializer.
+   */
+  public JsonSerialization<SinglePendingCommit> getSinglePendingFileSerializer() {
+    return singleCommitSerializer.getForCurrentThread();
+  }
+
+  /**
+   * Get a serializer for .pendingset files.
+   * @return a serializer.
+   */
+  public JsonSerialization<PendingSet> getPendingSetSerializer() {
+    return pendingSetSerializer.getForCurrentThread();
+  }
+
+  /**
+   * Destroy any thread pools; wait for that to finish,
+   * but don't overreact if it doesn't finish in time.
+   */
+  private synchronized void destroyThreadPools() {
+    try {
+      IOUtils.cleanupWithLogger(LOG, outerSubmitter, innerSubmitter);
+    } finally {
+      outerSubmitter = null;
+      innerSubmitter = null;
+    }
+  }
+
+  /**
+   * Job configuration.
+   * @return configuration (never null)
+   */
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Get the job ID.
+   * @return job ID.
+   */
+  public String getJobId() {
+    return jobId;
+  }
+
+  /**
+   * Collecting thread level IO statistics?
+   * @return true if thread level IO stats should be collected.
+   */
+  public boolean isCollectIOStatistics() {
+    return collectIOStatistics;
+  }
+
+  /**
+   * IOStatistics context of the created thread.
+   * @return the IOStatistics.
+   */
+  public IOStatisticsContext getIOStatisticsContext() {
+    return ioStatisticsContext;
+  }
+
+  /**
+   * Switch to the context IOStatistics context,
+   * if needed.
+   */
+  public void switchToIOStatisticsContext() {
+    IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext);
+  }
+
+  /**
+   * Reset the IOStatistics context if statistics are being
+   * collected.
+   * Logs at info.
+   */
+  public void maybeResetIOStatisticsContext() {
+    if (collectIOStatistics) {
+
+      LOG.info("Resetting IO statistics context {}",
+          ioStatisticsContext.getID());
+      ioStatisticsContext.reset();
+    }
+  }
+
+  /**
+   * Submitter for a given thread pool.
+   */
+  private final class PoolSubmitter implements TaskPool.Submitter, Closeable {
+
+    private ExecutorService executor;
+
+    private PoolSubmitter(ExecutorService executor) {
+      this.executor = executor;
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+      if (executor != null) {
+        HadoopExecutors.shutdown(executor, LOG,
+            THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS);
+      }
+      executor = null;
+    }
+
+    /**
+     * Forward to the submitter, wrapping in task
+     * context setting, so as to ensure that all operations
+     * have job/task attributes.
+     * @param task task to execute
+     * @return the future.
+     */
+    @Override
+    public Future<?> submit(Runnable task) {
+      return executor.submit(() -> {
+        auditContextUpdater.updateCurrentAuditContext();
+        try {
+          task.run();
+        } finally {
+          auditContextUpdater.resetCurrentAuditContext();
+        }
+      });
+    }
+
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
similarity index 75%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java
rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
index a4f13b78a3f0c..eb23f299a02e3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java
@@ -16,30 +16,30 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.commit;
+package org.apache.hadoop.fs.s3a.commit.impl;
 
-import javax.annotation.Nullable;
-import java.io.Closeable;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import javax.annotation.Nullable;
+
 import com.amazonaws.services.s3.model.MultipartUpload;
 import com.amazonaws.services.s3.model.PartETag;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
@@ -49,28 +49,34 @@
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3AUtils;
 import org.apache.hadoop.fs.s3a.WriteOperations;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.PathCommitException;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
 import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
 import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation;
 import org.apache.hadoop.fs.s3a.impl.HeaderProcessing;
 import org.apache.hadoop.fs.s3a.impl.InternalConstants;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
 import org.apache.hadoop.fs.statistics.DurationTracker;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
-import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.functional.TaskPool;
 
 import static java.util.Objects.requireNonNull;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.listAndFilter;
 import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_COMMIT_JOB;
 import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MATERIALIZE_FILE;
+import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_LOAD_SINGLE_PENDING_FILE;
 import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_STAGE_FILE_UPLOAD;
-import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.XA_MAGIC_MARKER;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants._SUCCESS;
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
 import static org.apache.hadoop.util.functional.RemoteIterators.cleanupRemoteIterator;
 
@@ -120,7 +126,7 @@ public class CommitOperations extends AbstractStoreOperation
    * @throws IOException failure to bind.
    */
   public CommitOperations(S3AFileSystem fs) throws IOException {
-    this(requireNonNull(fs), fs.newCommitterStatistics());
+    this(requireNonNull(fs), fs.newCommitterStatistics(), "/");
   }
 
   /**
@@ -128,10 +134,12 @@ public CommitOperations(S3AFileSystem fs) throws IOException {
    * the commit operations.
    * @param fs FS to bind to
    * @param committerStatistics committer statistics
+   * @param outputPath destination of work.
    * @throws IOException failure to bind.
    */
   public CommitOperations(S3AFileSystem fs,
-      CommitterStatistics committerStatistics) throws IOException {
+      CommitterStatistics committerStatistics,
+      String outputPath) throws IOException {
     super(requireNonNull(fs).createStoreContext());
     this.fs = fs;
     statistics = requireNonNull(committerStatistics);
@@ -139,7 +147,7 @@ public CommitOperations(S3AFileSystem fs,
     writeOperations = fs.createWriteOperationHelper(
         fs.getAuditSpanSource().createSpan(
             COMMITTER_COMMIT_JOB.getSymbol(),
-            "/", null));
+            outputPath, null));
   }
 
   /**
@@ -171,13 +179,11 @@ public IOStatistics getIOStatistics() {
   /**
    * Commit the operation, throwing an exception on any failure.
    * @param commit commit to execute
-   * @param operationState S3Guard state of ongoing operation.
    * @throws IOException on a failure
    */
-  private void commitOrFail(
-      final SinglePendingCommit commit,
-      final BulkOperationState operationState) throws IOException {
-    commit(commit, commit.getFilename(), operationState).maybeRethrow();
+  public void commitOrFail(
+      final SinglePendingCommit commit) throws IOException {
+    commit(commit, commit.getFilename()).maybeRethrow();
   }
 
   /**
@@ -185,13 +191,11 @@ private void commitOrFail(
    * and converted to an outcome.
    * @param commit entry to commit
    * @param origin origin path/string for outcome text
-   * @param operationState S3Guard state of ongoing operation.
    * @return the outcome
    */
-  private MaybeIOE commit(
+  public MaybeIOE commit(
       final SinglePendingCommit commit,
-      final String origin,
-      final BulkOperationState operationState) {
+      final String origin) {
     LOG.debug("Committing single commit {}", commit);
     MaybeIOE outcome;
     String destKey = "unknown destination";
@@ -203,7 +207,7 @@ private MaybeIOE commit(
       commit.validate();
       destKey = commit.getDestinationKey();
       long l = trackDuration(statistics, COMMITTER_MATERIALIZE_FILE.getSymbol(),
-          () -> innerCommit(commit, operationState));
+          () -> innerCommit(commit));
       LOG.debug("Successful commit of file length {}", l);
       outcome = MaybeIOE.NONE;
       statistics.commitCompleted(commit.getLength());
@@ -226,20 +230,17 @@ private MaybeIOE commit(
   /**
    * Inner commit operation.
    * @param commit entry to commit
-   * @param operationState S3Guard state of ongoing operation.
    * @return bytes committed.
    * @throws IOException failure
    */
   private long innerCommit(
-      final SinglePendingCommit commit,
-      final BulkOperationState operationState) throws IOException {
+      final SinglePendingCommit commit) throws IOException {
     // finalize the commit
     writeOperations.commitUpload(
         commit.getDestinationKey(),
-              commit.getUploadId(),
-              toPartEtags(commit.getEtags()),
-              commit.getLength(),
-              operationState);
+        commit.getUploadId(),
+        toPartEtags(commit.getEtags()),
+        commit.getLength());
     return commit.getLength();
   }
 
@@ -247,43 +248,69 @@ private long innerCommit(
    * Locate all files with the pending suffix under a directory.
    * @param pendingDir directory
    * @param recursive recursive listing?
-   * @return the list of all located entries
+   * @return iterator of all located entries
    * @throws IOException if there is a problem listing the path.
    */
-  public List<LocatedFileStatus> locateAllSinglePendingCommits(
+  public RemoteIterator<LocatedFileStatus> locateAllSinglePendingCommits(
       Path pendingDir,
       boolean recursive) throws IOException {
     return listAndFilter(fs, pendingDir, recursive, PENDING_FILTER);
   }
 
   /**
-   * Load all single pending commits in the directory.
+   * Load all single pending commits in the directory, using the
+   * outer submitter.
    * All load failures are logged and then added to list of files which would
    * not load.
+   *
    * @param pendingDir directory containing commits
    * @param recursive do a recursive scan?
+   * @param commitContext commit context
+   *
    * @return tuple of loaded entries and those pending files which would
    * not load/validate.
+   *
    * @throws IOException on a failure to list the files.
    */
   public Pair<PendingSet,
       List<Pair<LocatedFileStatus, IOException>>>
-      loadSinglePendingCommits(Path pendingDir, boolean recursive)
+      loadSinglePendingCommits(Path pendingDir,
+      boolean recursive,
+      CommitContext commitContext)
       throws IOException {
 
-    List<LocatedFileStatus> statusList = locateAllSinglePendingCommits(
-        pendingDir, recursive);
-    PendingSet commits = new PendingSet(
-        statusList.size());
-    List<Pair<LocatedFileStatus, IOException>> failures = new ArrayList<>(1);
-    for (LocatedFileStatus status : statusList) {
-      try {
-        commits.add(SinglePendingCommit.load(fs, status.getPath()));
-      } catch (IOException e) {
-        LOG.warn("Failed to load commit file {}", status.getPath(), e);
-        failures.add(Pair.of(status, e));
-      }
-    }
+    PendingSet commits = new PendingSet();
+    List<SinglePendingCommit> pendingFiles = Collections.synchronizedList(
+        new ArrayList<>(1));
+    List<Pair<LocatedFileStatus, IOException>> failures = Collections.synchronizedList(
+        new ArrayList<>(1));
+
+    TaskPool.foreach(locateAllSinglePendingCommits(pendingDir, recursive))
+        //. stopOnFailure()
+        .suppressExceptions(false)
+        .executeWith(commitContext.getOuterSubmitter())
+        .run(status -> {
+          Path path = status.getPath();
+          try {
+            // load the file
+            SinglePendingCommit singleCommit = trackDuration(statistics,
+                COMMITTER_LOAD_SINGLE_PENDING_FILE.getSymbol(), () ->
+                    SinglePendingCommit.load(fs,
+                        path,
+                        status,
+                        commitContext.getSinglePendingFileSerializer()));
+            // aggregate stats
+            commits.getIOStatistics()
+                .aggregate(singleCommit.getIOStatistics());
+            // then clear so they aren't marshalled again.
+            singleCommit.getIOStatistics().clear();
+            pendingFiles.add(singleCommit);
+          } catch (IOException e) {
+            LOG.warn("Failed to load commit file {}", path, e);
+            failures.add(Pair.of(status, e));
+          }
+        });
+    commits.setCommits(pendingFiles);
     return Pair.of(commits, failures);
   }
 
@@ -307,7 +334,7 @@ public IOException makeIOE(String key, Exception ex) {
    * @throws FileNotFoundException if the abort ID is unknown
    * @throws IOException on any failure
    */
-  private void abortSingleCommit(SinglePendingCommit commit)
+  public void abortSingleCommit(SinglePendingCommit commit)
       throws IOException {
     String destKey = commit.getDestinationKey();
     String origin = commit.getFilename() != null
@@ -326,7 +353,7 @@ private void abortSingleCommit(SinglePendingCommit commit)
    * @throws FileNotFoundException if the abort ID is unknown
    * @throws IOException on any failure
    */
-  private void abortMultipartCommit(String destKey, String uploadId)
+  public void abortMultipartCommit(String destKey, String uploadId)
       throws IOException {
     try (DurationInfo d = new DurationInfo(LOG,
         "Aborting commit ID %s to path %s", uploadId, destKey)) {
@@ -339,11 +366,13 @@ private void abortMultipartCommit(String destKey, String uploadId)
   /**
    * Enumerate all pending files in a dir/tree, abort.
    * @param pendingDir directory of pending operations
+   * @param commitContext commit context
    * @param recursive recurse?
    * @return the outcome of all the abort operations
    * @throws IOException if there is a problem listing the path.
    */
   public MaybeIOE abortAllSinglePendingCommits(Path pendingDir,
+      CommitContext commitContext,
       boolean recursive)
       throws IOException {
     Preconditions.checkArgument(pendingDir != null, "null pendingDir");
@@ -361,10 +390,14 @@ public MaybeIOE abortAllSinglePendingCommits(Path pendingDir,
       LOG.debug("No files to abort under {}", pendingDir);
     }
     while (pendingFiles.hasNext()) {
-      Path pendingFile = pendingFiles.next().getPath();
+      final LocatedFileStatus status = pendingFiles.next();
+      Path pendingFile = status.getPath();
       if (pendingFile.getName().endsWith(CommitConstants.PENDING_SUFFIX)) {
         try {
-          abortSingleCommit(SinglePendingCommit.load(fs, pendingFile));
+          abortSingleCommit(SinglePendingCommit.load(fs,
+              pendingFile,
+              status,
+              commitContext.getSinglePendingFileSerializer()));
         } catch (FileNotFoundException e) {
           LOG.debug("listed file already deleted: {}", pendingFile);
         } catch (IOException | IllegalArgumentException e) {
@@ -439,14 +472,6 @@ public void createSuccessMarker(Path outputPath,
     if (addMetrics) {
       addFileSystemStatistics(successData.getMetrics());
     }
-    // add any diagnostics
-    Configuration conf = fs.getConf();
-    successData.addDiagnostic(S3_METADATA_STORE_IMPL,
-        conf.getTrimmed(S3_METADATA_STORE_IMPL, ""));
-    successData.addDiagnostic(METADATASTORE_AUTHORITATIVE,
-        conf.getTrimmed(METADATASTORE_AUTHORITATIVE, "false"));
-    successData.addDiagnostic(AUTHORITATIVE_PATH,
-        conf.getTrimmed(AUTHORITATIVE_PATH, ""));
 
     // now write
     Path markerPath = new Path(outputPath, _SUCCESS);
@@ -454,21 +479,19 @@ public void createSuccessMarker(Path outputPath,
         successData);
     try (DurationInfo ignored = new DurationInfo(LOG,
         "Writing success file %s", markerPath)) {
-      successData.save(fs, markerPath, true);
+      successData.save(fs, markerPath, SuccessData.serializer());
     }
   }
 
   /**
    * Revert a pending commit by deleting the destination.
    * @param commit pending commit
-   * @param operationState nullable operational state for a bulk update
    * @throws IOException failure
    */
-  public void revertCommit(SinglePendingCommit commit,
-      BulkOperationState operationState) throws IOException {
+  public void revertCommit(SinglePendingCommit commit) throws IOException {
     LOG.info("Revert {}", commit);
     try {
-      writeOperations.revertCommit(commit.getDestinationKey(), operationState);
+      writeOperations.revertCommit(commit.getDestinationKey());
     } finally {
       statistics.commitReverted();
     }
@@ -485,7 +508,7 @@ public void revertCommit(SinglePendingCommit commit,
    * @return a pending upload entry
    * @throws IOException failure
    */
-  public SinglePendingCommit  uploadFileToPendingCommit(File localFile,
+  public SinglePendingCommit uploadFileToPendingCommit(File localFile,
       Path destPath,
       String partition,
       long uploadPartSize,
@@ -513,7 +536,8 @@ public SinglePendingCommit  uploadFileToPendingCommit(File localFile,
         destPath)) {
 
       statistics.commitCreated();
-      uploadId = writeOperations.initiateMultiPartUpload(destKey);
+      uploadId = writeOperations.initiateMultiPartUpload(destKey,
+          PutObjectOptions.keepingDirs());
       long length = localFile.length();
 
       SinglePendingCommit commitData = new SinglePendingCommit();
@@ -611,13 +635,45 @@ public void jobCompleted(boolean success) {
   }
 
   /**
-   * Begin the final commit.
+   * Create a commit context for a job or task.
+   *
+   * @param context job context
+   * @param path path for all work.
+   * @param committerThreads thread pool size
+   * @param ioStatisticsContext IOStatistics context of current thread
+   * @return the commit context to pass in.
+   * @throws IOException failure.
+   */
+  public CommitContext createCommitContext(
+      JobContext context,
+      Path path,
+      int committerThreads,
+      IOStatisticsContext ioStatisticsContext) throws IOException {
+    return new CommitContext(this, context,
+        committerThreads,
+        ioStatisticsContext);
+  }
+
+  /**
+   * Create a stub commit context for tests.
+   * There's no job context.
    * @param path path for all work.
+   * @param jobId job ID; if null a random UUID is generated.
+   * @param committerThreads number of committer threads.
    * @return the commit context to pass in.
    * @throws IOException failure.
    */
-  public CommitContext initiateCommitOperation(Path path) throws IOException {
-    return new CommitContext(writeOperations.initiateCommitOperation(path));
+  public CommitContext createCommitContextForTesting(
+      Path path, @Nullable String jobId, int committerThreads) throws IOException {
+    final String id = jobId != null
+        ? jobId
+        : UUID.randomUUID().toString();
+
+    return new CommitContext(this,
+        getStoreContext().getConfiguration(),
+        id,
+        committerThreads,
+        IOStatisticsContext.getCurrentIOStatisticsContext());
   }
 
   /**
@@ -643,110 +699,6 @@ public static Optional<Long> extractMagicFileLength(FileSystem fs, Path path)
     return HeaderProcessing.extractXAttrLongValue(bytes);
   }
 
-  /**
-   * Commit context.
-   *
-   * It is used to manage the final commit sequence where files become
-   * visible. It contains a {@link BulkOperationState} field, which, if
-   * there is a metastore, will be requested from the store so that it
-   * can track multiple creation operations within the same overall operation.
-   * This will be null if there is no metastore, or the store chooses not
-   * to provide one.
-   *
-   * This can only be created through {@link #initiateCommitOperation(Path)}.
-   *
-   * Once the commit operation has completed, it must be closed.
-   * It must not be reused.
-   */
-  public final class CommitContext implements Closeable {
-
-    /**
-     * State of any metastore.
-     */
-    private final BulkOperationState operationState;
-
-    /**
-     * Create.
-     * @param operationState any S3Guard bulk state.
-     */
-    private CommitContext(@Nullable final BulkOperationState operationState) {
-      this.operationState = operationState;
-    }
-
-    /**
-     * Commit the operation, throwing an exception on any failure.
-     * See {@link CommitOperations#commitOrFail(SinglePendingCommit, BulkOperationState)}.
-     * @param commit commit to execute
-     * @throws IOException on a failure
-     */
-    public void commitOrFail(SinglePendingCommit commit) throws IOException {
-      CommitOperations.this.commitOrFail(commit, operationState);
-    }
-
-    /**
-     * Commit a single pending commit; exceptions are caught
-     * and converted to an outcome.
-     * See {@link CommitOperations#commit(SinglePendingCommit, String, BulkOperationState)}.
-     * @param commit entry to commit
-     * @param origin origin path/string for outcome text
-     * @return the outcome
-     */
-    public MaybeIOE commit(SinglePendingCommit commit,
-        String origin) {
-      return CommitOperations.this.commit(commit, origin, operationState);
-    }
-
-    /**
-     * See {@link CommitOperations#abortSingleCommit(SinglePendingCommit)}.
-     * @param commit pending commit to abort
-     * @throws FileNotFoundException if the abort ID is unknown
-     * @throws IOException on any failure
-     */
-    public void abortSingleCommit(final SinglePendingCommit commit)
-        throws IOException {
-      CommitOperations.this.abortSingleCommit(commit);
-    }
-
-    /**
-     * See {@link CommitOperations#revertCommit(SinglePendingCommit, BulkOperationState)}.
-     * @param commit pending commit
-     * @throws IOException failure
-     */
-    public void revertCommit(final SinglePendingCommit commit)
-        throws IOException {
-      CommitOperations.this.revertCommit(commit, operationState);
-    }
-
-    /**
-     * See {@link CommitOperations#abortMultipartCommit(String, String)}..
-     * @param destKey destination key
-     * @param uploadId upload to cancel
-     * @throws FileNotFoundException if the abort ID is unknown
-     * @throws IOException on any failure
-     */
-    public void abortMultipartCommit(
-        final String destKey,
-        final String uploadId)
-        throws IOException {
-      CommitOperations.this.abortMultipartCommit(destKey, uploadId);
-    }
-
-    @Override
-    public void close() throws IOException {
-      IOUtils.cleanupWithLogger(LOG, operationState);
-    }
-
-    @Override
-    public String toString() {
-      final StringBuilder sb = new StringBuilder(
-          "CommitContext{");
-      sb.append("operationState=").append(operationState);
-      sb.append('}');
-      return sb.toString();
-    }
-
-  }
-
   /**
    * A holder for a possible IOException; the call {@link #maybeRethrow()}
    * will throw any exception passed into the constructor, and be a no-op
@@ -819,5 +771,4 @@ public static MaybeIOE of(IOException ex) {
     }
   }
 
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java
similarity index 74%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java
rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java
index 9e5ee860e85ff..c38ab2e9ba1af 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.commit;
+package org.apache.hadoop.fs.s3a.commit.impl;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -68,31 +68,67 @@ public static int getAppAttemptId(JobContext context) {
   /**
    * Compute the "magic" path for a job attempt.
    * @param jobUUID unique Job ID.
+   * @param appAttemptId the ID of the application attempt for this job.
    * @param dest the final output directory
    * @return the path to store job attempt data.
    */
-  public static Path getMagicJobAttemptPath(String jobUUID, Path dest) {
-    return new Path(getMagicJobAttemptsPath(dest),
-        formatAppAttemptDir(jobUUID));
+  public static Path getMagicJobAttemptPath(String jobUUID,
+      int appAttemptId,
+      Path dest) {
+    return new Path(
+        getMagicJobAttemptsPath(dest),
+        formatAppAttemptDir(jobUUID, appAttemptId));
+  }
+
+  /**
+   * Compute the "magic" path for a job.
+   * @param jobUUID unique Job ID.
+   * @param dest the final output directory
+   * @return the path to store job attempt data.
+   */
+  public static Path getMagicJobPath(String jobUUID,
+      Path dest) {
+    return new Path(
+        getMagicJobAttemptsPath(dest),
+        formatJobDir(jobUUID));
   }
 
   /**
-   * Format the application attempt directory.
+   * Build the name of the job directory, without
+   * app attempt.
+   * This is the path to use for cleanup.
    * @param jobUUID unique Job ID.
-   * @return the directory name for the application attempt
+   * @return the directory name for the job
    */
-  public static String formatAppAttemptDir(String jobUUID) {
+  public static String formatJobDir(
+      String jobUUID) {
     return String.format("job-%s", jobUUID);
   }
 
+  /**
+   * Build the name of the job attempt directory.
+   * @param jobUUID unique Job ID.
+   * @param appAttemptId the ID of the application attempt for this job.
+   * @return the directory tree for the application attempt
+   */
+  public static String formatAppAttemptDir(
+      String jobUUID,
+      int appAttemptId) {
+    return formatJobDir(jobUUID) + String.format("/%02d", appAttemptId);
+  }
+
   /**
    * Compute the path where the output of magic task attempts are stored.
    * @param jobUUID unique Job ID.
    * @param dest destination of work
+   * @param appAttemptId the ID of the application attempt for this job.
    * @return the path where the output of magic task attempts are stored.
    */
-  public static Path getMagicTaskAttemptsPath(String jobUUID, Path dest) {
-    return new Path(getMagicJobAttemptPath(jobUUID, dest), "tasks");
+  public static Path getMagicTaskAttemptsPath(
+      String jobUUID,
+      Path dest,
+      int appAttemptId) {
+    return new Path(getMagicJobAttemptPath(jobUUID, appAttemptId, dest), "tasks");
   }
 
   /**
@@ -115,6 +151,8 @@ public static Path getMagicTaskAttemptPath(TaskAttemptContext context,
   /**
    * Get the base Magic attempt path, without any annotations to mark relative
    * references.
+   * If there is an app attempt property in the context configuration, that
+   * is included.
    * @param context task context.
    * @param jobUUID unique Job ID.
    * @param dest The output path to commit work into
@@ -123,8 +161,9 @@ public static Path getMagicTaskAttemptPath(TaskAttemptContext context,
   public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context,
       String jobUUID,
       Path dest) {
-    return new Path(getMagicTaskAttemptsPath(jobUUID, dest),
-          String.valueOf(context.getTaskAttemptID()));
+    return new Path(
+        getMagicTaskAttemptsPath(jobUUID, dest, getAppAttemptId(context)),
+        String.valueOf(context.getTaskAttemptID()));
   }
 
   /**
@@ -132,12 +171,13 @@ public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context,
    * This data is <i>not magic</i>
    * @param jobUUID unique Job ID.
    * @param out output directory of job
+   * @param appAttemptId the ID of the application attempt for this job.
    * @return the path to store temporary job attempt data.
    */
   public static Path getTempJobAttemptPath(String jobUUID,
-      Path out) {
+      Path out, final int appAttemptId) {
     return new Path(new Path(out, TEMP_DATA),
-        formatAppAttemptDir(jobUUID));
+        formatAppAttemptDir(jobUUID, appAttemptId));
   }
 
   /**
@@ -150,7 +190,7 @@ public static Path getTempJobAttemptPath(String jobUUID,
   public static Path getTempTaskAttemptPath(TaskAttemptContext context,
       final String jobUUID, Path out) {
     return new Path(
-        getTempJobAttemptPath(jobUUID, out),
+        getTempJobAttemptPath(jobUUID, out, getAppAttemptId(context)),
         String.valueOf(context.getTaskAttemptID()));
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java
new file mode 100644
index 0000000000000..b4977f34d57f4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal classes which make use of mapreduce code.
+ * These MUST NOT be referred to in production code except
+ * in org.apache.hadoop.fs.s3a.commit classes which are only
+ * used within job/task committers.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.commit.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
index 71af2bef24bd8..c85571a1949a1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
@@ -26,20 +26,26 @@
 
 import com.amazonaws.services.s3.model.PartETag;
 import com.amazonaws.services.s3.model.PutObjectRequest;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Retries;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 import org.apache.hadoop.fs.s3a.commit.PutTracker;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
+import org.apache.hadoop.fs.s3a.statistics.PutTrackerStatistics;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.util.Preconditions;
 
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_MARKER_PUT;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.X_HEADER_MAGIC_MARKER;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
 
 /**
  * Put tracker for Magic commits.
@@ -57,6 +63,7 @@ public class MagicCommitTracker extends PutTracker {
   private final WriteOperationHelper writer;
   private final String bucket;
   private static final byte[] EMPTY = new byte[0];
+  private final PutTrackerStatistics trackerStatistics;
 
   /**
    * Magic commit tracker.
@@ -66,19 +73,22 @@ public class MagicCommitTracker extends PutTracker {
    * @param destKey key for the destination
    * @param pendingsetKey key of the pendingset file
    * @param writer writer instance to use for operations; includes audit span
+   * @param trackerStatistics tracker statistics
    */
   public MagicCommitTracker(Path path,
       String bucket,
       String originalDestKey,
       String destKey,
       String pendingsetKey,
-      WriteOperationHelper writer) {
+      WriteOperationHelper writer,
+      PutTrackerStatistics trackerStatistics) {
     super(destKey);
     this.bucket = bucket;
     this.path = path;
     this.originalDestKey = originalDestKey;
     this.pendingPartKey = pendingsetKey;
     this.writer = writer;
+    this.trackerStatistics = requireNonNull(trackerStatistics);
     LOG.info("File {} is written as magic file to path {}",
         path, destKey);
   }
@@ -126,6 +136,19 @@ public boolean aboutToComplete(String uploadId,
     Preconditions.checkArgument(!parts.isEmpty(),
         "No uploaded parts to save");
 
+    // put a 0-byte file with the name of the original under-magic path
+    // Add the final file length as a header
+    // this is done before the task commit, so its duration can be
+    // included in the statistics
+    Map<String, String> headers = new HashMap<>();
+    headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten));
+    PutObjectRequest originalDestPut = writer.createPutObjectRequest(
+        originalDestKey,
+        new ByteArrayInputStream(EMPTY),
+        0,
+        new PutObjectOptions(true, null, headers));
+    upload(originalDestPut);
+
     // build the commit summary
     SinglePendingCommit commitData = new SinglePendingCommit();
     commitData.touch(System.currentTimeMillis());
@@ -138,7 +161,8 @@ public boolean aboutToComplete(String uploadId,
     commitData.bindCommitData(parts);
     commitData.setIOStatistics(
         new IOStatisticsSnapshot(iostatistics));
-    byte[] bytes = commitData.toBytes();
+
+    byte[] bytes = commitData.toBytes(SinglePendingCommit.serializer());
     LOG.info("Uncommitted data pending to file {};"
             + " commit metadata for {} parts in {}. size: {} byte(s)",
         path.toUri(), parts.size(), pendingPartKey, bytesWritten);
@@ -148,19 +172,20 @@ public boolean aboutToComplete(String uploadId,
         pendingPartKey,
         new ByteArrayInputStream(bytes),
         bytes.length, null);
-    writer.uploadObject(put);
-
-    // Add the final file length as a header
-    Map<String, String> headers = new HashMap<>();
-    headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten));
-    // now put a 0-byte file with the name of the original under-magic path
-    PutObjectRequest originalDestPut = writer.createPutObjectRequest(
-        originalDestKey,
-        new ByteArrayInputStream(EMPTY),
-        0,
-        headers);
-    writer.uploadObject(originalDestPut);
+    upload(put);
     return false;
+
+  }
+  /**
+   * PUT an object.
+   * @param request the request
+   * @throws IOException on problems
+   */
+  @Retries.RetryTranslated
+  private void upload(PutObjectRequest request) throws IOException {
+    trackDurationOfInvocation(trackerStatistics,
+        COMMITTER_MAGIC_MARKER_PUT.getSymbol(), () ->
+            writer.uploadObject(request, PutObjectOptions.keepingDirs()));
   }
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
index 34bbfd4ed76e2..9ded64eedc056 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
@@ -32,11 +32,12 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.Invoker;
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR;
 import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -45,9 +46,10 @@
 
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TASK_ATTEMPT_ID;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TEMP_DATA;
 import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
 import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
-import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
+import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.demandStringifyIOStatistics;
 
 /**
@@ -100,25 +102,28 @@ public void setupJob(JobContext context) throws IOException {
     try (DurationInfo d = new DurationInfo(LOG,
         "Setup Job %s", jobIdString(context))) {
       super.setupJob(context);
-      Path jobAttemptPath = getJobAttemptPath(context);
-      getDestinationFS(jobAttemptPath,
-          context.getConfiguration()).mkdirs(jobAttemptPath);
+      Path jobPath = getJobPath();
+      final FileSystem destFS = getDestinationFS(jobPath,
+          context.getConfiguration());
+      destFS.delete(jobPath, true);
+      destFS.mkdirs(jobPath);
     }
   }
 
   /**
    * Get the list of pending uploads for this job attempt, by listing
    * all .pendingset files in the job attempt directory.
-   * @param context job context
+   * @param commitContext job context
    * @return a list of pending commits.
    * @throws IOException Any IO failure
    */
   protected ActiveCommit listPendingUploadsToCommit(
-      JobContext context)
+      CommitContext commitContext)
       throws IOException {
     FileSystem fs = getDestFS();
-    return ActiveCommit.fromStatusList(fs,
-        listAndFilter(fs, getJobAttemptPath(context), false,
+    return ActiveCommit.fromStatusIterator(fs,
+        listAndFilter(fs, getJobAttemptPath(commitContext.getJobContext()),
+            false,
             CommitOperations.PENDINGSET_FILTER));
   }
 
@@ -126,11 +131,16 @@ protected ActiveCommit listPendingUploadsToCommit(
    * Delete the magic directory.
    */
   public void cleanupStagingDirs() {
-    Path path = magicSubdir(getOutputPath());
+    final Path out = getOutputPath();
+    Path path = magicSubdir(out);
     try(DurationInfo ignored = new DurationInfo(LOG, true,
         "Deleting magic directory %s", path)) {
       Invoker.ignoreIOExceptions(LOG, "cleanup magic directory", path.toString(),
           () -> deleteWithWarning(getDestFS(), path, true));
+      // and the job temp directory with manifests
+      Invoker.ignoreIOExceptions(LOG, "cleanup job directory", path.toString(),
+          () -> deleteWithWarning(getDestFS(),
+              new Path(out, TEMP_DATA), true));
     }
   }
 
@@ -146,13 +156,8 @@ public void cleanupStagingDirs() {
   @Override
   public boolean needsTaskCommit(TaskAttemptContext context)
       throws IOException {
-    Path taskAttemptPath = getTaskAttemptPath(context);
-    try (DurationInfo d = new DurationInfo(LOG,
-        "needsTaskCommit task %s", context.getTaskAttemptID())) {
-      return taskAttemptPath.getFileSystem(
-          context.getConfiguration())
-          .exists(taskAttemptPath);
-    }
+    // return true as a dir was created here in setup;
+    return true;
   }
 
   @Override
@@ -167,8 +172,9 @@ public void commitTask(TaskAttemptContext context) throws IOException {
       throw e;
     } finally {
       // delete the task attempt so there's no possibility of a second attempt
+      // incurs a LIST, a bulk DELETE and maybe a parent dir creation, however
+      // as it happens during task commit, it should be off the critical path.
       deleteTaskAttemptPathQuietly(context);
-      destroyThreadPool();
     }
     getCommitOperations().taskCompleted(true);
     LOG.debug("aggregate statistics\n{}",
@@ -190,43 +196,55 @@ private PendingSet innerCommitTask(
     Path taskAttemptPath = getTaskAttemptPath(context);
     // load in all pending commits.
     CommitOperations actions = getCommitOperations();
-    Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
-        loaded = actions.loadSinglePendingCommits(
-            taskAttemptPath, true);
-    PendingSet pendingSet = loaded.getKey();
-    List<Pair<LocatedFileStatus, IOException>> failures = loaded.getValue();
-    if (!failures.isEmpty()) {
-      // At least one file failed to load
-      // revert all which did; report failure with first exception
-      LOG.error("At least one commit file could not be read: failing");
-      abortPendingUploads(context, pendingSet.getCommits(), true);
-      throw failures.get(0).getValue();
-    }
-    // patch in IDs
-    String jobId = getUUID();
-    String taskId = String.valueOf(context.getTaskAttemptID());
-    for (SinglePendingCommit commit : pendingSet.getCommits()) {
-      commit.setJobId(jobId);
-      commit.setTaskId(taskId);
-    }
-    pendingSet.putExtraData(TASK_ATTEMPT_ID, taskId);
-    pendingSet.setJobId(jobId);
-    Path jobAttemptPath = getJobAttemptPath(context);
-    TaskAttemptID taskAttemptID = context.getTaskAttemptID();
-    Path taskOutcomePath = new Path(jobAttemptPath,
-        taskAttemptID.getTaskID().toString() +
-        CommitConstants.PENDINGSET_SUFFIX);
-    LOG.info("Saving work of {} to {}", taskAttemptID, taskOutcomePath);
-    LOG.debug("task statistics\n{}",
-        IOStatisticsLogging.demandStringifyIOStatisticsSource(pendingSet));
-    try {
-      // We will overwrite if there exists a pendingSet file already
-      pendingSet.save(getDestFS(), taskOutcomePath, true);
-    } catch (IOException e) {
-      LOG.warn("Failed to save task commit data to {} ",
-          taskOutcomePath, e);
-      abortPendingUploads(context, pendingSet.getCommits(), true);
-      throw e;
+    PendingSet pendingSet;
+    try (CommitContext commitContext = initiateTaskOperation(context)) {
+      Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
+          loaded = actions.loadSinglePendingCommits(
+              taskAttemptPath, true, commitContext);
+      pendingSet = loaded.getKey();
+      List<Pair<LocatedFileStatus, IOException>> failures = loaded.getValue();
+      if (!failures.isEmpty()) {
+        // At least one file failed to load
+        // revert all which did; report failure with first exception
+        LOG.error("At least one commit file could not be read: failing");
+        abortPendingUploads(commitContext, pendingSet.getCommits(), true);
+        throw failures.get(0).getValue();
+      }
+      // patch in IDs
+      String jobId = getUUID();
+      String taskId = String.valueOf(context.getTaskAttemptID());
+      for (SinglePendingCommit commit : pendingSet.getCommits()) {
+        commit.setJobId(jobId);
+        commit.setTaskId(taskId);
+      }
+      pendingSet.putExtraData(TASK_ATTEMPT_ID, taskId);
+      pendingSet.setJobId(jobId);
+      // add in the IOStatistics of all the file loading
+      if (commitContext.isCollectIOStatistics()) {
+        pendingSet.getIOStatistics()
+            .aggregate(
+                commitContext.getIOStatisticsContext().getIOStatistics());
+      }
+
+      Path jobAttemptPath = getJobAttemptPath(context);
+      TaskAttemptID taskAttemptID = context.getTaskAttemptID();
+      Path taskOutcomePath = new Path(jobAttemptPath,
+          taskAttemptID.getTaskID().toString() +
+              CommitConstants.PENDINGSET_SUFFIX);
+      LOG.info("Saving work of {} to {}", taskAttemptID, taskOutcomePath);
+      LOG.debug("task statistics\n{}",
+          IOStatisticsLogging.demandStringifyIOStatisticsSource(pendingSet));
+      try {
+        // We will overwrite if there exists a pendingSet file already
+        pendingSet.save(getDestFS(),
+            taskOutcomePath,
+            commitContext.getPendingSetSerializer());
+      } catch (IOException e) {
+        LOG.warn("Failed to save task commit data to {} ",
+            taskOutcomePath, e);
+        abortPendingUploads(commitContext, pendingSet.getCommits(), true);
+        throw e;
+      }
     }
     return pendingSet;
   }
@@ -245,24 +263,35 @@ private PendingSet innerCommitTask(
   public void abortTask(TaskAttemptContext context) throws IOException {
     Path attemptPath = getTaskAttemptPath(context);
     try (DurationInfo d = new DurationInfo(LOG,
-        "Abort task %s", context.getTaskAttemptID())) {
-      getCommitOperations().abortAllSinglePendingCommits(attemptPath, true);
+        "Abort task %s", context.getTaskAttemptID());
+        CommitContext commitContext = initiateTaskOperation(context)) {
+      getCommitOperations().abortAllSinglePendingCommits(attemptPath,
+          commitContext,
+          true);
     } finally {
       deleteQuietly(
           attemptPath.getFileSystem(context.getConfiguration()),
           attemptPath, true);
-      destroyThreadPool();
     }
   }
 
+  /**
+   * Compute the path under which all job attempts will be placed.
+   * @return the path to store job attempt data.
+   */
+  @Override
+  protected Path getJobPath() {
+    return getMagicJobPath(getUUID(), getOutputPath());
+  }
+
   /**
    * Compute the path where the output of a given job attempt will be placed.
    * For the magic committer, the path includes the job UUID.
    * @param appAttemptId the ID of the application attempt for this job.
    * @return the path to store job attempt data.
    */
-  protected Path getJobAttemptPath(int appAttemptId) {
-    return getMagicJobAttemptPath(getUUID(), getOutputPath());
+  protected final Path getJobAttemptPath(int appAttemptId) {
+    return getMagicJobAttemptPath(getUUID(), appAttemptId, getOutputPath());
   }
 
   /**
@@ -272,12 +301,12 @@ protected Path getJobAttemptPath(int appAttemptId) {
    * @param context the context of the task attempt.
    * @return the path where a task attempt should be stored.
    */
-  public Path getTaskAttemptPath(TaskAttemptContext context) {
+  public final Path getTaskAttemptPath(TaskAttemptContext context) {
     return getMagicTaskAttemptPath(context, getUUID(), getOutputPath());
   }
 
   @Override
-  protected Path getBaseTaskAttemptPath(TaskAttemptContext context) {
+  protected final Path getBaseTaskAttemptPath(TaskAttemptContext context) {
     return getBaseMagicTaskAttemptPath(context, getUUID(), getOutputPath());
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
index 1a5a63c940f47..99683db984983 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathExistsException;
 import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
@@ -98,17 +99,18 @@ public void setupJob(JobContext context) throws IOException {
    * Pre-commit actions for a job.
    * Here: look at the conflict resolution mode and choose
    * an action based on the current policy.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending pending commits
    * @throws IOException any failure
    */
   @Override
   public void preCommitJob(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
 
+    final JobContext context = commitContext.getJobContext();
     // see if the files can be loaded.
-    super.preCommitJob(context, pending);
+    super.preCommitJob(commitContext, pending);
     Path outputPath = getOutputPath();
     FileSystem fs = getDestFS();
     Configuration fsConf = fs.getConf();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
index 214c7abdc732a..5d1a20e4240c2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
@@ -32,12 +32,13 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.commit.PathCommitException;
-import org.apache.hadoop.fs.s3a.commit.Tasks;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
-import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.functional.TaskPool;
 
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.COMMITTER_NAME_PARTITIONED;
 
@@ -89,7 +90,8 @@ public String toString() {
 
   @Override
   protected int commitTaskInternal(TaskAttemptContext context,
-      List<? extends FileStatus> taskOutput) throws IOException {
+      List<? extends FileStatus> taskOutput,
+      CommitContext commitContext) throws IOException {
     Path attemptPath = getTaskAttemptPath(context);
     Set<String> partitions = Paths.getPartitions(attemptPath, taskOutput);
 
@@ -109,7 +111,7 @@ protected int commitTaskInternal(TaskAttemptContext context,
         }
       }
     }
-    return super.commitTaskInternal(context, taskOutput);
+    return super.commitTaskInternal(context, taskOutput, commitContext);
   }
 
   /**
@@ -121,13 +123,13 @@ protected int commitTaskInternal(TaskAttemptContext context,
    *   <li>APPEND: allowed.; no need to check.</li>
    *   <li>REPLACE deletes all existing partitions.</li>
    * </ol>
-   * @param context job context
+   * @param commitContext commit context
    * @param pending the pending operations
    * @throws IOException any failure
    */
   @Override
   public void preCommitJob(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
 
     FileSystem fs = getDestFS();
@@ -135,7 +137,7 @@ public void preCommitJob(
     // enforce conflict resolution
     Configuration fsConf = fs.getConf();
     boolean shouldPrecheckPendingFiles = true;
-    switch (getConflictResolutionMode(context, fsConf)) {
+    switch (getConflictResolutionMode(commitContext.getJobContext(), fsConf)) {
     case FAIL:
       // FAIL checking is done on the task side, so this does nothing
       break;
@@ -144,17 +146,17 @@ public void preCommitJob(
       break;
     case REPLACE:
       // identify and replace the destination partitions
-      replacePartitions(context, pending);
+      replacePartitions(commitContext, pending);
       // and so there is no need to do another check.
       shouldPrecheckPendingFiles = false;
       break;
     default:
       throw new PathCommitException("",
           getRole() + ": unknown conflict resolution mode: "
-          + getConflictResolutionMode(context, fsConf));
+          + getConflictResolutionMode(commitContext.getJobContext(), fsConf));
     }
     if (shouldPrecheckPendingFiles) {
-      precommitCheckPendingFiles(context, pending);
+      precommitCheckPendingFiles(commitContext, pending);
     }
   }
 
@@ -176,17 +178,16 @@ public void preCommitJob(
    *   }
    * </pre>
    *
-   * @param context job context
+   * @param commitContext commit context
    * @param pending the pending operations
    * @throws IOException any failure
    */
   private void replacePartitions(
-      final JobContext context,
+      final CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
 
     Map<Path, String> partitions = new ConcurrentHashMap<>();
     FileSystem sourceFS = pending.getSourceFS();
-    Tasks.Submitter submitter = buildSubmitter(context);
     try (DurationInfo ignored =
              new DurationInfo(LOG, "Replacing partitions")) {
 
@@ -194,13 +195,15 @@ private void replacePartitions(
       // for a marginal optimisation, the previous parent is tracked, so
       // if a task writes many files to the same dir, the synchronized map
       // is updated only once.
-      Tasks.foreach(pending.getSourceFiles())
+      TaskPool.foreach(pending.getSourceFiles())
           .stopOnFailure()
           .suppressExceptions(false)
-          .executeWith(submitter)
+          .executeWith(commitContext.getOuterSubmitter())
           .run(status -> {
-            PendingSet pendingSet = PendingSet.load(sourceFS,
-                status);
+            PendingSet pendingSet = PersistentCommitData.load(
+                sourceFS,
+                status,
+                commitContext.getPendingSetSerializer());
             Path lastParent = null;
             for (SinglePendingCommit commit : pendingSet.getCommits()) {
               Path parent = commit.destinationPath().getParent();
@@ -213,10 +216,10 @@ private void replacePartitions(
     }
     // now do the deletes
     FileSystem fs = getDestFS();
-    Tasks.foreach(partitions.keySet())
+    TaskPool.foreach(partitions.keySet())
         .stopOnFailure()
         .suppressExceptions(false)
-        .executeWith(submitter)
+        .executeWith(commitContext.getOuterSubmitter())
         .run(partitionPath -> {
           LOG.debug("{}: removing partition path to be replaced: " +
               getRole(), partitionPath);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
index 2593cc18607b6..add916a2355d4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
@@ -25,7 +25,7 @@
 import java.util.concurrent.ExecutionException;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncheckedExecutionException;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
index ea49988c101e8..d764055c45bcd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
@@ -39,24 +39,25 @@
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
-import org.apache.hadoop.fs.s3a.commit.Tasks;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
 import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.functional.TaskPool;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static java.util.Objects.requireNonNull;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.Invoker.*;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
 import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
-import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
+import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*;
 import static org.apache.hadoop.util.functional.RemoteIterators.cleanupRemoteIterator;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
 
 /**
  * Committer based on the contributed work of the
@@ -111,7 +112,7 @@ public class StagingCommitter extends AbstractS3ACommitter {
   public StagingCommitter(Path outputPath,
       TaskAttemptContext context) throws IOException {
     super(outputPath, context);
-    this.constructorOutputPath = checkNotNull(getOutputPath(), "output path");
+    this.constructorOutputPath = requireNonNull(getOutputPath(), "output path");
     Configuration conf = getConf();
     this.uploadPartSize = conf.getLongBytes(
         MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
@@ -121,8 +122,8 @@ public StagingCommitter(Path outputPath,
     setWorkPath(buildWorkPath(context, getUUID()));
     this.wrappedCommitter = createWrappedCommitter(context, conf);
     setOutputPath(constructorOutputPath);
-    Path finalOutputPath = getOutputPath();
-    checkNotNull(finalOutputPath, "Output path cannot be null");
+    Path finalOutputPath = requireNonNull(getOutputPath(),
+        "Output path cannot be null");
     S3AFileSystem fs = getS3AFileSystem(finalOutputPath,
         context.getConfiguration(), false);
     s3KeyPrefix = fs.pathToKey(finalOutputPath);
@@ -243,10 +244,18 @@ private static Path getJobAttemptPath(int appAttemptId, Path out) {
 
   @Override
   protected Path getJobAttemptPath(int appAttemptId) {
-    return new Path(getPendingJobAttemptsPath(commitsDirectory),
+    return new Path(getJobPath(),
         String.valueOf(appAttemptId));
   }
 
+  /**
+   * Compute the path under which all job attempts will be placed.
+   * @return the path to store job attempt data.
+   */
+  protected Path getJobPath() {
+    return getPendingJobAttemptsPath(commitsDirectory);
+  }
+
   /**
    * Compute the path where the output of pending task attempts are stored.
    * @param context the context of the job with pending tasks.
@@ -275,7 +284,7 @@ public static Path getTaskAttemptPath(TaskAttemptContext context, Path out) {
    * @return the location of pending job attempts.
    */
   private static Path getPendingJobAttemptsPath(Path out) {
-    checkNotNull(out, "Null 'out' path");
+    requireNonNull(out, "Null 'out' path");
     return new Path(out, TEMPORARY);
   }
 
@@ -296,12 +305,12 @@ public Path getCommittedTaskPath(TaskAttemptContext context) {
    * @param context task context
    */
   private static void validateContext(TaskAttemptContext context) {
-    checkNotNull(context, "null context");
-    checkNotNull(context.getTaskAttemptID(),
+    requireNonNull(context, "null context");
+    requireNonNull(context.getTaskAttemptID(),
         "null task attempt ID");
-    checkNotNull(context.getTaskAttemptID().getTaskID(),
+    requireNonNull(context.getTaskAttemptID().getTaskID(),
         "null task ID");
-    checkNotNull(context.getTaskAttemptID().getJobID(),
+    requireNonNull(context.getTaskAttemptID().getJobID(),
         "null job ID");
   }
 
@@ -342,14 +351,13 @@ protected List<LocatedFileStatus> getTaskOutput(TaskAttemptContext context)
       throws IOException {
 
     // get files on the local FS in the attempt path
-    Path attemptPath = getTaskAttemptPath(context);
-    checkNotNull(attemptPath,
-        "No attemptPath path in {}", this);
+    Path attemptPath = requireNonNull(getTaskAttemptPath(context),
+        "No attemptPath path");
 
     LOG.debug("Scanning {} for files to commit", attemptPath);
 
-    return listAndFilter(getTaskAttemptFilesystem(context),
-        attemptPath, true, HIDDEN_FILE_FILTER);
+    return toList(listAndFilter(getTaskAttemptFilesystem(context),
+        attemptPath, true, HIDDEN_FILE_FILTER));
   }
 
   /**
@@ -425,46 +433,46 @@ public void setupJob(JobContext context) throws IOException {
 
   /**
    * Get the list of pending uploads for this job attempt.
-   * @param context job context
+   * @param commitContext job context
    * @return a list of pending uploads.
    * @throws IOException Any IO failure
    */
   @Override
   protected ActiveCommit listPendingUploadsToCommit(
-      JobContext context)
+      CommitContext commitContext)
       throws IOException {
-    return listPendingUploads(context, false);
+    return listPendingUploads(commitContext, false);
   }
 
   /**
    * Get the list of pending uploads for this job attempt, swallowing
    * exceptions.
-   * @param context job context
+   * @param commitContext commit context
    * @return a list of pending uploads. If an exception was swallowed,
    * then this may not match the actual set of pending operations
    * @throws IOException shouldn't be raised, but retained for the compiler
    */
   protected ActiveCommit listPendingUploadsToAbort(
-      JobContext context) throws IOException {
-    return listPendingUploads(context, true);
+      CommitContext commitContext) throws IOException {
+    return listPendingUploads(commitContext, true);
   }
 
   /**
    * Get the list of pending uploads for this job attempt.
-   * @param context job context
+   * @param commitContext commit context
    * @param suppressExceptions should exceptions be swallowed?
    * @return a list of pending uploads. If exceptions are being swallowed,
    * then this may not match the actual set of pending operations
    * @throws IOException Any IO failure which wasn't swallowed.
    */
   protected ActiveCommit listPendingUploads(
-      JobContext context, boolean suppressExceptions) throws IOException {
+      CommitContext commitContext, boolean suppressExceptions) throws IOException {
     try (DurationInfo ignored = new DurationInfo(LOG,
         "Listing pending uploads")) {
-      Path wrappedJobAttemptPath = getJobAttemptPath(context);
+      Path wrappedJobAttemptPath = getJobAttemptPath(commitContext.getJobContext());
       final FileSystem attemptFS = wrappedJobAttemptPath.getFileSystem(
-          context.getConfiguration());
-      return ActiveCommit.fromStatusList(attemptFS,
+          commitContext.getConf());
+      return ActiveCommit.fromStatusIterator(attemptFS,
           listAndFilter(attemptFS,
               wrappedJobAttemptPath, false,
               HIDDEN_FILE_FILTER));
@@ -491,27 +499,39 @@ public void cleanupStagingDirs() {
     }
   }
 
+  /**
+   * Staging committer cleanup includes calling wrapped committer's
+   * cleanup method, and removing all destination paths in the final
+   * filesystem.
+   * @param commitContext commit context
+   * @param suppressExceptions should exceptions be suppressed?
+   * @throws IOException IO failures if exceptions are not suppressed.
+   */
   @Override
   @SuppressWarnings("deprecation")
-  protected void cleanup(JobContext context,
+  protected void cleanup(CommitContext commitContext,
       boolean suppressExceptions)
       throws IOException {
     maybeIgnore(suppressExceptions, "Cleanup wrapped committer",
-        () -> wrappedCommitter.cleanupJob(context));
+        () -> wrappedCommitter.cleanupJob(
+            commitContext.getJobContext()));
     maybeIgnore(suppressExceptions, "Delete destination paths",
-        () -> deleteDestinationPaths(context));
-    super.cleanup(context, suppressExceptions);
+        () -> deleteDestinationPaths(
+            commitContext.getJobContext()));
+    super.cleanup(commitContext, suppressExceptions);
   }
 
   @Override
-  protected void abortJobInternal(JobContext context,
+  protected void abortJobInternal(CommitContext commitContext,
       boolean suppressExceptions) throws IOException {
     String r = getRole();
+    JobContext context = commitContext.getJobContext();
     boolean failed = false;
     try (DurationInfo d = new DurationInfo(LOG,
         "%s: aborting job in state %s ", r, jobIdString(context))) {
-      ActiveCommit pending = listPendingUploadsToAbort(context);
-      abortPendingUploads(context, pending, suppressExceptions, true);
+      ActiveCommit pending = listPendingUploadsToAbort(commitContext);
+      abortPendingUploads(commitContext,
+          pending, suppressExceptions, true);
     } catch (FileNotFoundException e) {
       // nothing to list
       LOG.debug("No job directory to read uploads from");
@@ -519,7 +539,7 @@ protected void abortJobInternal(JobContext context,
       failed = true;
       maybeIgnore(suppressExceptions, "aborting job", e);
     } finally {
-      super.abortJobInternal(context, failed || suppressExceptions);
+      super.abortJobInternal(commitContext, failed || suppressExceptions);
     }
   }
 
@@ -590,16 +610,16 @@ public boolean needsTaskCommit(TaskAttemptContext context)
   @Override
   public void commitTask(TaskAttemptContext context) throws IOException {
     try (DurationInfo d = new DurationInfo(LOG,
-        "%s: commit task %s", getRole(), context.getTaskAttemptID())) {
-      int count = commitTaskInternal(context, getTaskOutput(context));
+        "%s: commit task %s", getRole(), context.getTaskAttemptID());
+         CommitContext commitContext
+             = initiateTaskOperation(context)) {
+      int count = commitTaskInternal(context, getTaskOutput(context), commitContext);
       LOG.info("{}: upload file count: {}", getRole(), count);
     } catch (IOException e) {
       LOG.error("{}: commit of task {} failed",
           getRole(), context.getTaskAttemptID(), e);
       getCommitOperations().taskCompleted(false);
       throw e;
-    } finally {
-      destroyThreadPool();
     }
     getCommitOperations().taskCompleted(true);
   }
@@ -609,11 +629,13 @@ public void commitTask(TaskAttemptContext context) throws IOException {
    * writing a pending entry for them.
    * @param context task context
    * @param taskOutput list of files from the output
+   * @param commitContext commit context
    * @return number of uploads committed.
    * @throws IOException IO Failures.
    */
   protected int commitTaskInternal(final TaskAttemptContext context,
-      List<? extends FileStatus> taskOutput)
+      List<? extends FileStatus> taskOutput,
+      CommitContext commitContext)
       throws IOException {
     LOG.debug("{}: commitTaskInternal", getRole());
     Configuration conf = context.getConfiguration();
@@ -648,10 +670,10 @@ protected int commitTaskInternal(final TaskAttemptContext context,
       pendingCommits.putExtraData(TASK_ATTEMPT_ID,
           context.getTaskAttemptID().toString());
       try {
-        Tasks.foreach(taskOutput)
+        TaskPool.foreach(taskOutput)
             .stopOnFailure()
             .suppressExceptions(false)
-            .executeWith(buildSubmitter(context))
+            .executeWith(commitContext.getOuterSubmitter())
             .run(stat -> {
               Path path = stat.getPath();
               File localFile = new File(path.toUri().getPath());
@@ -674,14 +696,22 @@ protected int commitTaskInternal(final TaskAttemptContext context,
           pendingCommits.add(commit);
         }
 
+        // maybe add in the IOStatistics the thread
+        if (commitContext.isCollectIOStatistics()) {
+          pendingCommits.getIOStatistics().aggregate(
+              commitContext.getIOStatisticsContext()
+              .getIOStatistics());
+        }
+
         // save the data
-        // although overwrite=false, there's still a risk of > 1 entry being
-        // committed if the FS doesn't have create-no-overwrite consistency.
+        // overwrite any existing file, so whichever task attempt
+        // committed last wins.
 
         LOG.debug("Saving {} pending commit(s)) to file {}",
             pendingCommits.size(),
             commitsAttemptPath);
-        pendingCommits.save(commitsFS, commitsAttemptPath, false);
+        pendingCommits.save(commitsFS, commitsAttemptPath,
+            commitContext.getPendingSetSerializer());
         threw = false;
 
       } finally {
@@ -689,12 +719,11 @@ protected int commitTaskInternal(final TaskAttemptContext context,
           LOG.error(
               "{}: Exception during commit process, aborting {} commit(s)",
               getRole(), commits.size());
-          try(CommitOperations.CommitContext commitContext
-                  = initiateCommitOperation();
-              DurationInfo ignored = new DurationInfo(LOG,
+          try(DurationInfo ignored = new DurationInfo(LOG,
                   "Aborting %s uploads", commits.size())) {
-            Tasks.foreach(commits)
+            TaskPool.foreach(commits)
                 .suppressExceptions()
+                .executeWith(commitContext.getOuterSubmitter())
                 .run(commitContext::abortSingleCommit);
           }
           deleteTaskAttemptPathQuietly(context);
@@ -737,8 +766,6 @@ public void abortTask(TaskAttemptContext context) throws IOException {
       LOG.error("{}: exception when aborting task {}",
           getRole(), context.getTaskAttemptID(), e);
       throw e;
-    } finally {
-      destroyThreadPool();
     }
   }
 
@@ -857,16 +884,16 @@ public static String getConfictModeOption(JobContext context,
    * Pre-commit actions for a job.
    * Loads all the pending files to verify they can be loaded
    * and parsed.
-   * @param context job context
+   * @param commitContext commit context
    * @param pending pending commits
    * @throws IOException any failure
    */
   @Override
   public void preCommitJob(
-      final JobContext context,
+      CommitContext commitContext,
       final ActiveCommit pending) throws IOException {
 
     // see if the files can be loaded.
-    precommitCheckPendingFiles(context, pending);
+    precommitCheckPendingFiles(commitContext, pending);
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AbstractStoreOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AbstractStoreOperation.java
index cbca14660d35f..3537755e790c6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AbstractStoreOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AbstractStoreOperation.java
@@ -18,9 +18,9 @@
 
 package org.apache.hadoop.fs.s3a.impl;
 
-import org.apache.hadoop.fs.store.audit.AuditSpan;
+import javax.annotation.Nullable;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
 
 /**
  * Base class of operations in the store.
@@ -37,7 +37,7 @@ public abstract class AbstractStoreOperation {
   /**
    * Audit Span.
    */
-  private AuditSpan auditSpan;
+  private final AuditSpan auditSpan;
 
   /**
    * Constructor.
@@ -45,8 +45,11 @@ public abstract class AbstractStoreOperation {
    * stores it for later.
    * @param storeContext store context.
    */
-  protected AbstractStoreOperation(final StoreContext storeContext) {
-    this(storeContext, storeContext.getActiveAuditSpan());
+  protected AbstractStoreOperation(final @Nullable StoreContext storeContext) {
+    this(storeContext,
+        storeContext != null
+        ? storeContext.getActiveAuditSpan()
+        : null);
   }
 
   /**
@@ -54,10 +57,11 @@ protected AbstractStoreOperation(final StoreContext storeContext) {
    * @param storeContext store context.
    * @param auditSpan active span
    */
-  protected AbstractStoreOperation(final StoreContext storeContext,
+  protected AbstractStoreOperation(
+      final @Nullable StoreContext storeContext,
       final AuditSpan auditSpan) {
-    this.storeContext = checkNotNull(storeContext);
-    this.auditSpan = checkNotNull(auditSpan);
+    this.storeContext = storeContext;
+    this.auditSpan = auditSpan;
   }
 
   /**
@@ -70,7 +74,7 @@ public final StoreContext getStoreContext() {
 
   /**
    * Get the audit span this object was created with.
-   * @return the current span
+   * @return the current span or null
    */
   public AuditSpan getAuditSpan() {
     return auditSpan;
@@ -80,6 +84,8 @@ public AuditSpan getAuditSpan() {
    * Activate the audit span.
    */
   public void activateAuditSpan() {
-    auditSpan.activate();
+    if (auditSpan != null) {
+      auditSpan.activate();
+    }
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ActiveOperationContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ActiveOperationContext.java
index 3ad2bbff3ba92..9ad3c0a541003 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ActiveOperationContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ActiveOperationContext.java
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.fs.s3a.impl;
 
-import javax.annotation.Nullable;
-
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
 
 /**
@@ -41,19 +38,12 @@ public class ActiveOperationContext {
    */
   private final S3AStatisticsContext statisticsContext;
 
-  /**
-   * S3Guard bulk operation state, if (currently) set.
-   */
-  @Nullable private BulkOperationState bulkOperationState;
-
   public ActiveOperationContext(
       final long operationId,
-      final S3AStatisticsContext statisticsContext,
-      @Nullable final BulkOperationState bulkOperationState) {
+      final S3AStatisticsContext statisticsContext) {
     this.operationId = operationId;
     this.statisticsContext = Objects.requireNonNull(statisticsContext,
         "null statistics context");
-    this.bulkOperationState = bulkOperationState;
   }
 
   @Override
@@ -61,16 +51,10 @@ public String toString() {
     final StringBuilder sb = new StringBuilder(
         "ActiveOperation{");
     sb.append("operationId=").append(operationId);
-    sb.append(", bulkOperationState=").append(bulkOperationState);
     sb.append('}');
     return sb.toString();
   }
 
-  @Nullable
-  public BulkOperationState getBulkOperationState() {
-    return bulkOperationState;
-  }
-
   public long getOperationId() {
     return operationId;
   }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
index 259738f99892a..0156207419210 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CallableSupplier.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.util.DurationInfo;
 
-import static org.apache.hadoop.fs.impl.FutureIOSupport.raiseInnerCause;
+import static org.apache.hadoop.util.functional.FutureIO.raiseInnerCause;
 
 /**
  * A bridge from Callable to Supplier; catching exceptions
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
index 43a5805c22056..e7dd75c581131 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ChangeTracker.java
@@ -39,12 +39,12 @@
 import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
 import org.apache.hadoop.fs.s3a.statistics.ChangeTrackerStatistics;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 /**
  * Change tracking for input streams: the version ID or etag of the object is
  * tracked and compared on open/re-open.  An initial version ID or etag may or
- * may not be available, depending on usage (e.g. if S3Guard is utilized).
+ * may not be available.
  *
  * Self-contained for testing and use in different streams.
  */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java
new file mode 100644
index 0000000000000..0392afac59d91
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.Nonnull;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.util.Progressable;
+
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CREATE_FILE_KEYS;
+
+/**
+ * Builder used in create file; takes a callback to the operation
+ * to create the file.
+ * Is non-recursive unless explicitly changed.
+ */
+public class CreateFileBuilder extends
+    FSDataOutputStreamBuilder<FSDataOutputStream, CreateFileBuilder> {
+
+  /**
+   * Flag set to create with overwrite.
+   */
+  public static final EnumSet<CreateFlag> CREATE_OVERWRITE_FLAGS =
+      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
+
+  /**
+    * Flag set to create without overwrite.
+   */
+  public static final EnumSet<CreateFlag> CREATE_NO_OVERWRITE_FLAGS =
+      EnumSet.of(CreateFlag.CREATE);
+
+  /**
+   * Classic create file option set: overwriting.
+   */
+  public static final CreateFileOptions OPTIONS_CREATE_FILE_OVERWRITE =
+      new CreateFileOptions(CREATE_OVERWRITE_FLAGS, true, false, null);
+
+  /**
+   * Classic create file option set: no overwrite.
+   */
+  public static final CreateFileOptions OPTIONS_CREATE_FILE_NO_OVERWRITE =
+      new CreateFileOptions(CREATE_NO_OVERWRITE_FLAGS, true, false, null);
+
+  /**
+   * Callback interface.
+   */
+  private final CreateFileBuilderCallbacks callbacks;
+
+  /**
+   * Constructor.
+   * @param fileSystem fs; used by superclass.
+   * @param path qualified path to create
+   * @param callbacks callbacks.
+   */
+  public CreateFileBuilder(
+      @Nonnull final FileSystem fileSystem,
+      @Nonnull final Path path,
+      @Nonnull final CreateFileBuilderCallbacks callbacks) {
+
+    super(fileSystem, path);
+    this.callbacks = callbacks;
+  }
+
+  @Override
+  public CreateFileBuilder getThisBuilder() {
+    return this;
+  }
+
+  @Override
+  public FSDataOutputStream build() throws IOException {
+    Path path = getPath();
+
+    final Configuration options = getOptions();
+    final Map<String, String> headers = new HashMap<>();
+    final Set<String> mandatoryKeys = getMandatoryKeys();
+    final Set<String> keysToValidate = new HashSet<>();
+
+    // pick up all headers from the mandatory list and strip them before
+    // validating the keys
+    String headerPrefix = FS_S3A_CREATE_HEADER + ".";
+    final int prefixLen = headerPrefix.length();
+    mandatoryKeys.stream().forEach(key -> {
+      if (key.startsWith(headerPrefix) && key.length() > prefixLen) {
+        headers.put(key.substring(prefixLen), options.get(key));
+      } else {
+        keysToValidate.add(key);
+      }
+    });
+
+    rejectUnknownMandatoryKeys(keysToValidate, CREATE_FILE_KEYS, "for " + path);
+
+    // and add any optional headers
+    getOptionalKeys().stream()
+        .filter(key -> key.startsWith(headerPrefix) && key.length() > prefixLen)
+        .forEach(key -> headers.put(key.substring(prefixLen), options.get(key)));
+
+
+    EnumSet<CreateFlag> flags = getFlags();
+    if (flags.contains(CreateFlag.APPEND)) {
+      throw new UnsupportedOperationException("Append is not supported");
+    }
+    if (!flags.contains(CreateFlag.CREATE) &&
+        !flags.contains(CreateFlag.OVERWRITE)) {
+      throw new PathIOException(path.toString(),
+          "Must specify either create or overwrite");
+    }
+
+    final boolean performance =
+        options.getBoolean(Constants.FS_S3A_CREATE_PERFORMANCE, false);
+    return callbacks.createFileFromBuilder(
+        path,
+        getProgress(),
+        new CreateFileOptions(flags, isRecursive(), performance, headers));
+
+  }
+
+  /**
+   * Pass flags down.
+   * @param flags input flags.
+   * @return this builder.
+   */
+  public CreateFileBuilder withFlags(EnumSet<CreateFlag> flags) {
+    if (flags.contains(CreateFlag.CREATE)) {
+      create();
+    }
+    if (flags.contains(CreateFlag.APPEND)) {
+      append();
+    }
+    overwrite(flags.contains(CreateFlag.OVERWRITE));
+    return this;
+  }
+
+  /**
+   * make the flag getter public.
+   * @return creation flags.
+   */
+  public EnumSet<CreateFlag> getFlags() {
+    return super.getFlags();
+  }
+
+  /**
+   * Callbacks for creating the file.
+   */
+  public interface CreateFileBuilderCallbacks {
+
+    /**
+     * Create a file from the builder.
+     * @param path path to file
+     * @param progress progress callback
+     * @param options options for the file
+     * @return the stream
+     * @throws IOException any IO problem
+     */
+    FSDataOutputStream createFileFromBuilder(
+        Path path,
+        Progressable progress,
+        CreateFileOptions options) throws IOException;
+  }
+
+  /**
+   * Create file options as built from the builder set or the classic
+   * entry point.
+   */
+  public static final class CreateFileOptions {
+
+    /**
+     * creation flags.
+     * create parent dirs?
+     * progress callback.
+     * performance flag.
+     */
+    private final EnumSet<CreateFlag> flags;
+
+    /**
+     * create parent dirs?
+     */
+    private final boolean recursive;
+
+    /**
+     * performance flag.
+     */
+    private final boolean performance;
+
+    /**
+     * Headers; may be null.
+     */
+    private final Map<String, String> headers;
+
+    /**
+     * @param flags creation flags
+     * @param recursive create parent dirs?
+     * @param performance performance flag
+     * @param headers nullable header map.
+     */
+    public CreateFileOptions(
+        final EnumSet<CreateFlag> flags,
+        final boolean recursive,
+        final boolean performance,
+        final Map<String, String> headers) {
+      this.flags = flags;
+      this.recursive = recursive;
+      this.performance = performance;
+      this.headers = headers;
+    }
+
+    @Override
+    public String toString() {
+      return "CreateFileOptions{" +
+          "flags=" + flags +
+          ", recursive=" + recursive +
+          ", performance=" + performance +
+          ", headers=" + headers +
+          '}';
+    }
+
+    public EnumSet<CreateFlag> getFlags() {
+      return flags;
+    }
+
+    public boolean isRecursive() {
+      return recursive;
+    }
+
+    public boolean isPerformance() {
+      return performance;
+    }
+
+    public Map<String, String> getHeaders() {
+      return headers;
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
index 5db996763885b..a45bfe46f169f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.s3a.impl;
 
-import javax.annotation.Nullable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -26,9 +25,8 @@
 import java.util.stream.Collectors;
 
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,82 +38,31 @@
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
 import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.DurationInfo;
 
 import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.maybeAwaitCompletion;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
 
 /**
  * Implementation of the delete() operation.
- * <p>
- * How S3Guard/Store inconsistency is handled:
- * <ol>
- *   <li>
- *     The list operation does not ask for tombstone markers; objects
- *     under tombstones will be found and deleted.
- *     The {@code extraFilesDeleted} counter will be incremented here.
- *   </li>
- *   <li>
- *     That may result in recently deleted files being found and
- *     duplicate delete requests issued. This is mostly harmless.
- *   </li>
- *   <li>
- *     If a path is considered authoritative on the client, so only S3Guard
- *     is used for listings, we wrap up the delete with a scan of raw S3.
- *     This will find and eliminate OOB additions.
- *   </li>
- *   <li>
- *     Exception 1: simple directory markers of the form PATH + "/".
- *     These are treated as a signal that there are no children; no
- *     listing is made.
- *   </li>
- *   <li>
- *     Exception 2: delete(path, true) where path has a tombstone in S3Guard.
- *     Here the delete is downgraded to a no-op even before this operation
- *     is created. Thus: no listings of S3.
- *   </li>
- * </ol>
- * If this class is logged at debug, requests will be audited:
- * the response to a bulk delete call will be reviewed to see if there
- * were fewer files deleted than requested; that will be printed
- * at WARN level. This is independent of handling rejected delete
- * requests which raise exceptions -those are processed lower down.
- * <p>
- * Performance tuning:
- * <p>
- * The operation to POST a delete request (or issue many individual
- * DELETE calls) then update the S3Guard table is done in an async
- * operation so that it can overlap with the LIST calls for data.
- * However, only one single operation is queued at a time.
- * <p>
- * Executing more than one batch delete is possible, it just
- * adds complexity in terms of error handling as well as in
- * the datastructures used to track outstanding operations.
+ * This issues only one bulk delete at a time,
+ * intending to update S3Guard after every request succeeded.
+ * Now that S3Guard has been removed, it
+ * would be possible to issue multiple delete calls
+ * in parallel.
  * If this is done, then it may be good to experiment with different
  * page sizes. The default value is
  * {@link InternalConstants#MAX_ENTRIES_TO_DELETE}, the maximum a single
  * POST permits.
  * <p>
- * 1. Smaller pages executed in parallel may have different
- * performance characteristics when deleting very large directories,
- * because it will be the DynamoDB calls which will come to dominate.
+ * Smaller pages executed in parallel may have different
+ * performance characteristics when deleting very large directories.
  * Any exploration of options here MUST be done with performance
- * measurements taken from test runs in EC2 against local DDB and S3 stores,
+ * measurements taken from test runs in EC2 against local S3 stores,
  * so as to ensure network latencies do not skew the results.
- * <p>
- * 2. Note that as the DDB thread/connection pools will be shared across
- * all active delete operations, speedups will be minimal unless
- * those pools are large enough to cope the extra load.
- * <p>
- * There are also some opportunities to explore in
- * {@code DynamoDBMetadataStore} with batching delete requests
- * in the DDB APIs.
+
  */
 public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
 
@@ -142,11 +89,6 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
    */
   private final int pageSize;
 
-  /**
-   * Metastore -never null but may be the NullMetadataStore.
-   */
-  private final MetadataStore metadataStore;
-
   /**
    * Executor for async operations.
    */
@@ -157,35 +99,16 @@ public class DeleteOperation extends ExecutingStoreOperation<Boolean> {
    */
   private List<DeleteEntry> keys;
 
-  /**
-   * List of paths built up for incremental deletion on tree delete.
-   * At the end of the entire delete the full tree is scanned in S3Guard
-   * and tombstones added. For this reason this list of paths <i>must not</i>
-   * include directory markers, as that will break the scan.
-   */
-  private List<Path> paths;
-
   /**
    * The single async delete operation, or null.
    */
   private CompletableFuture<Void> deleteFuture;
 
-  /**
-   * Bulk Operation state if this is a bulk operation.
-   */
-  private BulkOperationState operationState;
-
   /**
    * Counter of deleted files.
    */
   private long filesDeleted;
 
-  /**
-   * Counter of files found in the S3 Store during a raw scan of the store
-   * after the previous listing was in auth-mode.
-   */
-  private long extraFilesDeleted;
-
   /**
    * Constructor.
    * @param context store context
@@ -208,7 +131,6 @@ public DeleteOperation(final StoreContext context,
             && pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE,
         "page size out of range: %s", pageSize);
     this.pageSize = pageSize;
-    metadataStore = context.getMetadataStore();
     executor = MoreExecutors.listeningDecorator(
         context.createThrottledExecutor(1));
   }
@@ -217,10 +139,6 @@ public long getFilesDeleted() {
     return filesDeleted;
   }
 
-  public long getExtraFilesDeleted() {
-    return extraFilesDeleted;
-  }
-
   /**
    * Delete a file or directory tree.
    * <p>
@@ -230,10 +148,14 @@ public long getExtraFilesDeleted() {
    * Only one delete at a time is submitted, however, to reduce the
    * complexity of recovering from failures.
    * <p>
-   * The DynamoDB store deletes paths in parallel itself, so that
-   * potentially slow part of the process is somewhat speeded up.
-   * The extra parallelization here is to list files from the store/DDB while
-   * that delete operation is in progress.
+   * With S3Guard removed, the problem of updating any
+   * DynamoDB store has gone away -delete calls could now be issued
+   * in parallel.
+   * However, rate limiting may be required to keep write load
+   * below the throttling point. Every entry in a single
+   * bulk delete call counts as a single write request -overloading
+   * an S3 partition with delete calls has been a problem in
+   * the past.
    *
    * @return true, except in the corner cases of root directory deletion
    * @throws PathIsNotEmptyDirectoryException if the path is a dir and this
@@ -294,9 +216,7 @@ public Boolean execute() throws IOException {
    * Delete a directory tree.
    * <p>
    * This is done by asking the filesystem for a list of all objects under
-   * the directory path, without using any S3Guard tombstone markers to hide
-   * objects which may be returned in S3 listings but which are considered
-   * deleted.
+   * the directory path.
    * <p>
    * Once the first {@link #pageSize} worth of objects has been listed, a batch
    * delete is queued for execution in a separate thread; subsequent batches
@@ -304,9 +224,6 @@ public Boolean execute() throws IOException {
    * being deleted in the separate thread.
    * <p>
    * After all listed objects are queued for deletion,
-   * if the path is considered authoritative in the client, a final scan
-   * of S3 <i>without S3Guard</i> is executed, so as to find and delete
-   * any out-of-band objects in the tree.
    * @param path directory path
    * @param dirKey directory key
    * @throws IOException failure
@@ -314,12 +231,7 @@ public Boolean execute() throws IOException {
   @Retries.RetryTranslated
   protected void deleteDirectoryTree(final Path path,
       final String dirKey) throws IOException {
-    // create an operation state so that the store can manage the bulk
-    // operation if it needs to
-    operationState = S3Guard.initiateBulkWrite(
-        metadataStore,
-        BulkOperationState.OperationType.Delete,
-        path);
+
     try (DurationInfo ignored =
              new DurationInfo(LOG, false, "deleting %s", dirKey)) {
 
@@ -327,11 +239,11 @@ protected void deleteDirectoryTree(final Path path,
       resetDeleteList();
       deleteFuture = null;
 
-      // list files including any under tombstones through S3Guard
+      // list files
       LOG.debug("Getting objects for directory prefix {} to delete", dirKey);
       final RemoteIterator<S3ALocatedFileStatus> locatedFiles =
           callbacks.listFilesAndDirectoryMarkers(path, status,
-              false, true);
+              true);
 
       // iterate through and delete. The next() call will block when a new S3
       // page is required; this any active delete submitted to the executor
@@ -345,50 +257,6 @@ protected void deleteDirectoryTree(final Path path,
       submitNextBatch();
       maybeAwaitCompletion(deleteFuture);
 
-      // if s3guard is authoritative we follow up with a bulk list and
-      // delete process on S3 this helps recover from any situation where S3
-      // and S3Guard have become inconsistent.
-      // This is only needed for auth paths; by performing the previous listing
-      // without tombstone filtering, any files returned by the non-auth
-      // S3 list which were hidden under tombstones will have been found
-      // and deleted.
-
-      if (callbacks.allowAuthoritative(path)) {
-        LOG.debug("Path is authoritatively guarded;"
-            + " listing files on S3 for completeness");
-        // let the ongoing delete finish to avoid duplicates
-        final RemoteIterator<S3AFileStatus> objects =
-            callbacks.listObjects(path, dirKey);
-
-        // iterate through and delete. The next() call will block when a new S3
-        // page is required; this any active delete submitted to the executor
-        // will run in parallel with this.
-        while (objects.hasNext()) {
-          // get the next entry in the listing.
-          extraFilesDeleted++;
-          S3AFileStatus next = objects.next();
-          LOG.debug("Found Unlisted entry {}", next);
-          queueForDeletion(deletionKey(next), null,
-              next.isDirectory());
-        }
-        if (extraFilesDeleted > 0) {
-          LOG.debug("Raw S3 Scan found {} extra file(s) to delete",
-              extraFilesDeleted);
-          // there is no more data:
-          // await any ongoing operation
-          submitNextBatch();
-          maybeAwaitCompletion(deleteFuture);
-        }
-      }
-
-      // final cleanup of the directory tree in the metastore, including the
-      // directory entry itself.
-      try (DurationInfo ignored2 =
-               new DurationInfo(LOG, false, "Delete metastore")) {
-        metadataStore.deleteSubtree(path, operationState);
-      }
-    } finally {
-      IOUtils.cleanupWithLogger(LOG, operationState);
     }
     LOG.debug("Delete \"{}\" completed; deleted {} objects", path,
         filesDeleted);
@@ -412,7 +280,7 @@ private String deletionKey(final S3AFileStatus stat) {
    */
   private void queueForDeletion(
       final S3AFileStatus stat) throws IOException {
-    queueForDeletion(deletionKey(stat), stat.getPath(), stat.isDirectory());
+    queueForDeletion(deletionKey(stat), stat.isDirectory());
   }
 
   /**
@@ -422,21 +290,13 @@ private void queueForDeletion(
    * complete.
    *
    * @param key key to delete
-   * @param deletePath nullable path of the key
    * @param isDirMarker is the entry a directory?
    * @throws IOException failure of the previous batch of deletions.
    */
   private void queueForDeletion(final String key,
-      @Nullable final Path deletePath,
       boolean isDirMarker) throws IOException {
     LOG.debug("Adding object to delete: \"{}\"", key);
     keys.add(new DeleteEntry(key, isDirMarker));
-    if (deletePath != null) {
-      if (!isDirMarker) {
-        paths.add(deletePath);
-      }
-    }
-
     if (keys.size() == pageSize) {
       submitNextBatch();
     }
@@ -455,7 +315,7 @@ private void submitNextBatch()
     maybeAwaitCompletion(deleteFuture);
 
     // delete the current page of keys and paths
-    deleteFuture = submitDelete(keys, paths);
+    deleteFuture = submitDelete(keys);
     // reset the references so a new list can be built up.
     resetDeleteList();
   }
@@ -466,7 +326,6 @@ private void submitNextBatch()
    */
   private void resetDeleteList() {
     keys = new ArrayList<>(pageSize);
-    paths = new ArrayList<>(pageSize);
   }
 
   /**
@@ -484,34 +343,28 @@ private void deleteObjectAtPath(
       throws IOException {
     LOG.debug("delete: {} {}", (isFile ? "file" : "dir marker"), key);
     filesDeleted++;
-    callbacks.deleteObjectAtPath(path, key, isFile, operationState);
+    callbacks.deleteObjectAtPath(path, key, isFile);
   }
 
   /**
-   * Delete a single page of keys and optionally the metadata.
-   * For a large page, it is the metadata size which dominates.
-   * Its possible to invoke this with empty lists of keys or paths.
-   * If both lists are empty no work is submitted and null is returned.
+   * Delete a single page of keys.
+   * If the list is empty no work is submitted and null is returned.
    *
    * @param keyList keys to delete.
-   * @param pathList paths to update the metastore with.
    * @return the submitted future or null
    */
   private CompletableFuture<Void> submitDelete(
-      final List<DeleteEntry> keyList,
-      final List<Path> pathList) {
+      final List<DeleteEntry> keyList) {
 
-    if (keyList.isEmpty() && pathList.isEmpty()) {
+    if (keyList.isEmpty()) {
       return null;
     }
     filesDeleted += keyList.size();
     return submit(executor,
         callableWithinAuditSpan(
             getAuditSpan(), () -> {
-              asyncDeleteAction(operationState,
-                  keyList,
-                  pathList,
-                  LOG.isDebugEnabled());
+              asyncDeleteAction(
+                  keyList);
               return null;
             }));
   }
@@ -520,26 +373,17 @@ private CompletableFuture<Void> submitDelete(
    * The action called in the asynchronous thread to delete
    * the keys from S3 and paths from S3Guard.
    *
-   * @param state ongoing operation state
    * @param keyList keys to delete.
-   * @param pathList paths to update the metastore with.
-   * @param auditDeletedKeys should the results be audited and undeleted
    * entries logged?
    * @throws IOException failure
    */
   @Retries.RetryTranslated
   private void asyncDeleteAction(
-      final BulkOperationState state,
-      final List<DeleteEntry> keyList,
-      final List<Path> pathList,
-      final boolean auditDeletedKeys)
+      final List<DeleteEntry> keyList)
       throws IOException {
-    List<DeleteObjectsResult.DeletedObject> deletedObjects = new ArrayList<>();
     try (DurationInfo ignored =
              new DurationInfo(LOG, false,
                  "Delete page of %d keys", keyList.size())) {
-      DeleteObjectsResult result = null;
-      List<Path> undeletedObjects = new ArrayList<>();
       if (!keyList.isEmpty()) {
         // first delete the files.
         List<DeleteObjectsRequest.KeyVersion> files = keyList.stream()
@@ -547,66 +391,35 @@ private void asyncDeleteAction(
             .map(e -> e.keyVersion)
             .collect(Collectors.toList());
         LOG.debug("Deleting of {} file objects", files.size());
-        result = Invoker.once("Remove S3 Files",
+        Invoker.once("Remove S3 Files",
             status.getPath().toString(),
             () -> callbacks.removeKeys(
                 files,
-                false,
-                undeletedObjects,
-                state,
-                !auditDeletedKeys));
-        if (result != null) {
-          deletedObjects.addAll(result.getDeletedObjects());
-        }
+                false
+            ));
         // now the dirs
         List<DeleteObjectsRequest.KeyVersion> dirs = keyList.stream()
             .filter(e -> e.isDirMarker)
             .map(e -> e.keyVersion)
             .collect(Collectors.toList());
         LOG.debug("Deleting of {} directory markers", dirs.size());
-        // This is invoked with deleteFakeDir = true, so
-        // S3Guard is not updated.
-        result = Invoker.once("Remove S3 Dir Markers",
+        // This is invoked with deleteFakeDir.
+        Invoker.once("Remove S3 Dir Markers",
             status.getPath().toString(),
             () -> callbacks.removeKeys(
                 dirs,
-                true,
-                undeletedObjects,
-                state,
-                !auditDeletedKeys));
-        if (result != null) {
-          deletedObjects.addAll(result.getDeletedObjects());
-        }
-      }
-      if (!pathList.isEmpty()) {
-        // delete file paths only. This stops tombstones
-        // being added until the final directory cleanup
-        // (HADOOP-17244)
-        metadataStore.deletePaths(pathList, state);
-      }
-      if (auditDeletedKeys) {
-        // audit the deleted keys
-        if (deletedObjects.size() != keyList.size()) {
-          // size mismatch
-          LOG.warn("Size mismatch in deletion operation. "
-                  + "Expected count of deleted files: {}; "
-                  + "actual: {}",
-              keyList.size(), deletedObjects.size());
-          // strip out the deleted keys
-          for (DeleteObjectsResult.DeletedObject del : deletedObjects) {
-            keyList.removeIf(kv -> kv.getKey().equals(del.getKey()));
-          }
-          for (DeleteEntry kv : keyList) {
-            LOG.debug("{}", kv.getKey());
-          }
-        }
+                true
+            ));
       }
     }
   }
 
   /**
-   * Deletion entry; dir marker state is tracked to control S3Guard
-   * update policy.
+   * Deletion entry; dir marker state is tracked to allow
+   * delete requests to be split into file
+   * and marker delete phases.
+   * Without S3Guard, the split is only used
+   * to choose which statistics to update.
    */
   private static final class DeleteEntry {
     private final DeleteObjectsRequest.KeyVersion keyVersion;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java
index ca04fed65a539..9390c699335b7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirMarkerTracker.java
@@ -34,20 +34,20 @@
  * Tracks directory markers which have been reported in object listings.
  * This is needed for auditing and cleanup, including during rename
  * operations.
- * <p></p>
+ * <p>
  * Designed to be used while scanning through the results of listObject
  * calls, where are we assume the results come in alphanumeric sort order
  * and parent entries before children.
- * <p></p>
+ * <p>
  * This lets as assume that we can identify all leaf markers as those
  * markers which were added to set of leaf markers and not subsequently
  * removed as a child entries were discovered.
- * <p></p>
+ * <p>
  * To avoid scanning datastructures excessively, the path of the parent
  * directory of the last file added is cached. This allows for a
  * quick bailout when many children of the same directory are
  * returned in a listing.
- * <p></p>
+ * <p>
  * Consult the directory_markers document for details on this feature,
  * including terminology.
  */
@@ -106,7 +106,7 @@ public class DirMarkerTracker {
 
   /**
    * Construct.
-   * <p></p>
+   * <p>
    * The base path is currently only used for information rather than
    * validating paths supplied in other methods.
    * @param basePath base path of track
@@ -128,7 +128,7 @@ public Path getBasePath() {
 
   /**
    * A marker has been found; this may or may not be a leaf.
-   * <p></p>
+   * <p>
    * Trigger a move of all markers above it into the surplus map.
    * @param path marker path
    * @param key object key
@@ -160,7 +160,7 @@ public List<Marker> fileFound(Path path,
 
   /**
    * A path has been found.
-   * <p></p>
+   * <p>
    * Declare all markers above it as surplus
    * @param path marker path
    * @param key object key
@@ -187,7 +187,7 @@ private List<Marker> pathFound(Path path,
   /**
    * Remove all markers from the path and its parents from the
    * {@link #leafMarkers} map.
-   * <p></p>
+   * <p>
    * if {@link #recordSurplusMarkers} is true, the marker is
    * moved to the surplus map. Not doing this is simply an
    * optimisation designed to reduce risk of excess memory consumption
@@ -223,7 +223,7 @@ public Map<Path, Marker> getLeafMarkers() {
 
   /**
    * Get the map of surplus markers.
-   * <p></p>
+   * <p>
    * Empty if they were not being recorded.
    * @return all surplus markers.
    */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java
index 36dd2e4fd2496..6ba74c7e971d2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java
@@ -69,21 +69,21 @@ enum MarkerPolicy {
 
     /**
      * Delete markers.
-     * <p></p>
+     * <p>
      * This is the classic S3A policy,
      */
     Delete(DIRECTORY_MARKER_POLICY_DELETE),
 
     /**
      * Keep markers.
-     * <p></p>
+     * <p>
      * This is <i>Not backwards compatible</i>.
      */
     Keep(DIRECTORY_MARKER_POLICY_KEEP),
 
     /**
      * Keep markers in authoritative paths only.
-     * <p></p>
+     * <p>
      * This is <i>Not backwards compatible</i> within the
      * auth paths, but is outside these.
      */
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ExecutingStoreOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ExecutingStoreOperation.java
index fdaeb8ebba457..4e2df30c5985d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ExecutingStoreOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ExecutingStoreOperation.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.util.functional.CallableRaisingIOE;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java
index 23631c67e30a1..257cef8192b2c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java
@@ -20,6 +20,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Set;
 
 import org.slf4j.Logger;
@@ -34,22 +35,15 @@
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
 
 import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
 
 /**
  * GetContentSummary operation.
- * This is based on {@code FileSystem.get#getContentSummary};
- * its still doing sequential treewalk with the efficiency
- * issues.
  *
- * Changes:
- * 1. On the recursive calls there
- * is no probe to see if the path is a file: we know the
- * recursion only happens with a dir.
- * 2. If a subdirectory is not found during the walk, that
- * does not trigger an error. The directory is clearly
- * not part of the content any more.
+ * It is optimized for s3 and performs a deep tree listing,
+ * inferring directory counts from the paths returned.
  *
  * The Operation serves up IOStatistics; this counts
  * the cost of all the list operations, but not the
@@ -122,9 +116,7 @@ public ContentSummary execute() throws IOException {
 
   /**
    * Return the {@link ContentSummary} of a given directory.
-   * This is a recursive operation (as the original is);
-   * it'd be more efficient of stack and heap if it managed its
-   * own stack.
+   *
    * @param dir dir to scan
    * @throws FileNotFoundException if the path does not resolve
    * @throws IOException IO failure
@@ -133,34 +125,65 @@ public ContentSummary execute() throws IOException {
    * @throws IOException failure
    */
   public ContentSummary getDirSummary(Path dir) throws IOException {
+
     long totalLength = 0;
     long fileCount = 0;
     long dirCount = 1;
-    final RemoteIterator<S3AFileStatus> it
-        = callbacks.listStatusIterator(dir);
+
+    RemoteIterator<S3ALocatedFileStatus> it = callbacks.listFilesIterator(dir, true);
+
+    Set<Path> dirSet = new HashSet<>();
+    Set<Path> pathsTraversed = new HashSet<>();
 
     while (it.hasNext()) {
-      final S3AFileStatus s = it.next();
-      if (s.isDirectory()) {
-        try {
-          ContentSummary c = getDirSummary(s.getPath());
-          totalLength += c.getLength();
-          fileCount += c.getFileCount();
-          dirCount += c.getDirectoryCount();
-        } catch (FileNotFoundException ignored) {
-          // path was deleted during the scan; exclude from
-          // summary.
-        }
-      } else {
-        totalLength += s.getLen();
+      S3ALocatedFileStatus fileStatus = it.next();
+      Path filePath = fileStatus.getPath();
+
+      if (fileStatus.isDirectory() && !filePath.equals(dir)) {
+        dirSet.add(filePath);
+        buildDirectorySet(dirSet, pathsTraversed, dir, filePath.getParent());
+      } else if (!fileStatus.isDirectory()) {
         fileCount += 1;
+        totalLength += fileStatus.getLen();
+        buildDirectorySet(dirSet, pathsTraversed, dir, filePath.getParent());
       }
+
     }
+
     // Add the list's IOStatistics
     iostatistics.aggregate(retrieveIOStatistics(it));
+
     return new ContentSummary.Builder().length(totalLength).
-        fileCount(fileCount).directoryCount(dirCount).
-        spaceConsumed(totalLength).build();
+            fileCount(fileCount).directoryCount(dirCount + dirSet.size()).
+            spaceConsumed(totalLength).build();
+  }
+
+  /***
+   * This method builds the set of all directories found under the base path. We need to do this
+   * because if the directory structure /a/b/c was created with a single mkdirs() call, it is
+   * stored as 1 object in S3 and the list files iterator will only return a single entry /a/b/c.
+   *
+   * We keep track of paths traversed so far to prevent duplication of work. For eg, if we had
+   * a/b/c/file-1.txt and /a/b/c/file-2.txt, we will only recurse over the complete path once
+   * and won't have to do anything for file-2.txt.
+   *
+   * @param dirSet Set of all directories found in the path
+   * @param pathsTraversed Set of all paths traversed so far
+   * @param basePath Path of directory to scan
+   * @param parentPath Parent path of the current file/directory in the iterator
+   */
+  private void buildDirectorySet(Set<Path> dirSet, Set<Path> pathsTraversed, Path basePath,
+      Path parentPath) {
+
+    if (parentPath == null || pathsTraversed.contains(parentPath) || parentPath.equals(basePath)) {
+      return;
+    }
+
+    dirSet.add(parentPath);
+
+    buildDirectorySet(dirSet, pathsTraversed, basePath, parentPath.getParent());
+
+    pathsTraversed.add(parentPath);
   }
 
   /**
@@ -186,23 +209,23 @@ public interface GetContentSummaryCallbacks {
 
     /**
      * Get the status of a path.
-     * @param path path to probe.
+     *
+     * @param path   path to probe.
      * @param probes probes to exec
      * @return the status
      * @throws IOException failure
      */
     @Retries.RetryTranslated
-    S3AFileStatus probePathStatus(Path path,
-        Set<StatusProbeEnum> probes) throws IOException;
+    S3AFileStatus probePathStatus(Path path, Set<StatusProbeEnum> probes) throws IOException;
 
-    /**
-     * Incremental list of all entries in a directory.
-     * @param path path of dir
-     * @return an iterator
+    /***
+     * List all entries under a path.
+     * @param path path.
+     * @param recursive if the subdirectories need to be traversed recursively
+     * @return an iterator over the listing.
      * @throws IOException failure
      */
-    RemoteIterator<S3AFileStatus> listStatusIterator(Path path)
+    RemoteIterator<S3ALocatedFileStatus> listFilesIterator(Path path, boolean recursive)
         throws IOException;
-
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
index e7b1ba35e5297..2c34e7b9b6ec2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java
@@ -22,12 +22,16 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.s3a.Constants;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
+
 /**
  * Internal constants private only to the S3A codebase.
  * Please don't refer to these outside of this module &amp; its tests.
@@ -47,6 +51,11 @@ public final class InternalConstants {
   */
   public static final boolean DELETE_CONSIDERED_IDEMPOTENT = true;
 
+  /**
+   * size of a buffer to create when draining the stream.
+   */
+  public static final int DRAIN_BUFFER_SIZE = 16384;
+
   private InternalConstants() {
   }
 
@@ -89,11 +98,17 @@ private InternalConstants() {
    * used becomes that of the select operation.
    */
   @InterfaceStability.Unstable
-  public static final Set<String> STANDARD_OPENFILE_KEYS =
-      Collections.unmodifiableSet(
-          new HashSet<>(
-              Arrays.asList(Constants.INPUT_FADVISE,
-                  Constants.READAHEAD_RANGE)));
+  public static final Set<String> S3A_OPENFILE_KEYS;
+
+  static {
+    Set<String> keys = Stream.of(
+        Constants.ASYNC_DRAIN_THRESHOLD,
+        Constants.INPUT_FADVISE,
+        Constants.READAHEAD_RANGE)
+        .collect(Collectors.toSet());
+    keys.addAll(FS_OPTION_OPENFILE_STANDARD_OPTIONS);
+    S3A_OPENFILE_KEYS = Collections.unmodifiableSet(keys);
+  }
 
   /** 403 error code. */
   public static final int SC_403 = 403;
@@ -105,6 +120,12 @@ private InternalConstants() {
   public static final String THROTTLE_LOG_NAME =
       "org.apache.hadoop.fs.s3a.throttled";
 
+  /**
+   * Name of the log for events related to the SDK V2 upgrade.
+   */
+  public static final String SDK_V2_UPGRADE_LOG_NAME =
+      "org.apache.hadoop.fs.s3a.SDKV2Upgrade";
+
   /** Directory marker attribute: see HADOOP-16613. Value: {@value}. */
   public static final String X_DIRECTORY =
       "application/x-directory";
@@ -137,17 +158,6 @@ private InternalConstants() {
    */
   public static final int CSE_PADDING_LENGTH = 16;
 
-  /**
-   * Error message to indicate S3-CSE is incompatible with S3Guard.
-   */
-  public static final String CSE_S3GUARD_INCOMPATIBLE = "S3-CSE cannot be "
-      + "used with S3Guard";
-
-  /**
-   * Error message to indicate Access Points are incompatible with S3Guard.
-   */
-  public static final String AP_S3GUARD_INCOMPATIBLE = "Access Points cannot be used with S3Guard";
-
   /**
    * Error message to indicate Access Points are required to be used for S3 access.
    */
@@ -164,4 +174,12 @@ private InternalConstants() {
    * will go through the AccessPoint.
    */
   public static final String ARN_BUCKET_OPTION = "fs.s3a.bucket.%s.accesspoint.arn";
+
+  /**
+   * The known keys used in a createFile call.
+   */
+  public static final Set<String> CREATE_FILE_KEYS =
+      Collections.unmodifiableSet(
+          new HashSet<>(Arrays.asList(Constants.FS_S3A_CREATE_PERFORMANCE)));
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java
index 99fb6809d93a3..f4d7a4349e484 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
 import org.apache.hadoop.fs.s3a.S3ListRequest;
 import org.apache.hadoop.fs.s3a.S3ListResult;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
 import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
@@ -44,37 +43,33 @@ public interface ListingOperationCallbacks {
    * Initiate a {@code listObjectsAsync} operation, incrementing metrics
    * in the process.
    *
-   * Retry policy: retry untranslated.
+   * Retry policy: failures will come from the future.
    * @param request request to initiate
    * @param trackerFactory tracker with statistics to update
    * @param span audit span for this operation
    * @return the results
-   * @throws IOException if the retry invocation raises one (it shouldn't).
    */
   @Retries.RetryRaw
   CompletableFuture<S3ListResult> listObjectsAsync(
       S3ListRequest request,
       DurationTrackerFactory trackerFactory,
-      AuditSpan span)
-          throws IOException;
+      AuditSpan span);
 
   /**
    * List the next set of objects.
-   * Retry policy: retry untranslated.
+   * Retry policy: failures will come from the future.
    * @param request last list objects request to continue
    * @param prevResult last paged result to continue from
    * @param trackerFactory tracker with statistics to update
    * @param span audit span for the IO
    * @return the next result object
-   * @throws IOException none, just there for retryUntranslated.
    */
   @Retries.RetryRaw
   CompletableFuture<S3ListResult> continueListObjectsAsync(
       S3ListRequest request,
       S3ListResult prevResult,
       DurationTrackerFactory trackerFactory,
-      AuditSpan span)
-          throws IOException;
+      AuditSpan span);
 
   /**
    * Build a {@link S3ALocatedFileStatus} from a {@link FileStatus} instance.
@@ -116,19 +111,4 @@ S3ListRequest createListObjectsRequest(
    */
   int getMaxKeys();
 
-  /**
-   * Get the updated time provider for the current fs instance.
-   * @return implementation of {@link ITtlTimeProvider}
-   */
-  ITtlTimeProvider getUpdatedTtlTimeProvider();
-
-  /**
-   * Is the path for this instance considered authoritative on the client,
-   * that is: will listing/status operations only be handled by the metastore,
-   * with no fallback to S3.
-   * @param p path
-   * @return true iff the path is authoritative on the client.
-   */
-  boolean allowAuthoritative(Path p);
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java
index 1d824201abeae..98a91b1881ba1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java
@@ -41,6 +41,18 @@
  * It performs the directory listing probe ahead of the simple object HEAD
  * call for this reason -the object is the failure mode which SHOULD NOT
  * be encountered on normal execution.
+ *
+ * Magic paths are handled specially
+ * <ul>
+ *   <li>The only path check is for a directory already existing there.</li>
+ *   <li>No ancestors are checked</li>
+ *   <li>Parent markers are never deleted, irrespective of FS settings</li>
+ * </ul>
+ * As a result, irrespective of depth, the operations performed are only
+ * <ol>
+ *   <li>One LIST</li>
+ *   <li>If needed, one PUT</li>
+ * </ol>
  */
 public class MkdirOperation extends ExecutingStoreOperation<Boolean> {
 
@@ -51,13 +63,21 @@ public class MkdirOperation extends ExecutingStoreOperation<Boolean> {
 
   private final MkdirCallbacks callbacks;
 
+  /**
+   * Should checks for ancestors existing be skipped?
+   * This flag is set when working with magic directories.
+   */
+  private final boolean isMagicPath;
+
   public MkdirOperation(
       final StoreContext storeContext,
       final Path dir,
-      final MkdirCallbacks callbacks) {
+      final MkdirCallbacks callbacks,
+      final boolean isMagicPath) {
     super(storeContext);
     this.dir = dir;
     this.callbacks = callbacks;
+    this.isMagicPath = isMagicPath;
   }
 
   /**
@@ -77,6 +97,14 @@ public Boolean execute() throws IOException {
       return true;
     }
 
+    // get the file status of the path.
+    // this is done even for a magic path, to avoid always  issuing PUT
+    // requests. Doing that without a check wouild seem to be an
+    // optimization, but it is not because
+    // 1. PUT is slower than HEAD
+    // 2. Write capacity is less than read capacity on a shard
+    // 3. It adds needless entries in versioned buckets, slowing
+    //    down subsequent operations.
     FileStatus fileStatus = getPathStatusExpectingDir(dir);
     if (fileStatus != null) {
       if (fileStatus.isDirectory()) {
@@ -85,7 +113,17 @@ public Boolean execute() throws IOException {
         throw new FileAlreadyExistsException("Path is a file: " + dir);
       }
     }
-    // dir, walk up tree
+    // file status was null
+
+    // is the path magic?
+    // If so, we declare success without looking any further
+    if (isMagicPath) {
+      // Create the marker file immediately,
+      // and don't delete markers
+      callbacks.createFakeDirectory(dir, true);
+      return true;
+    }
+
     // Walk path to root, ensuring closest ancestor is a directory, not file
     Path fPart = dir.getParent();
     try {
@@ -110,15 +148,15 @@ public Boolean execute() throws IOException {
       LOG.info("mkdirs({}}: Access denied when looking"
               + " for parent directory {}; skipping checks",
           dir, fPart);
-      LOG.debug("{}", e.toString(), e);
+      LOG.debug("{}", e, e);
     }
 
     // if we get here there is no directory at the destination.
     // so create one.
-    String key = getStoreContext().pathToKey(dir);
-    // this will create the marker file, delete the parent entries
-    // and update S3Guard
-    callbacks.createFakeDirectory(key);
+
+    // Create the marker file, delete the parent entries
+    // if the filesystem isn't configured to retain them
+    callbacks.createFakeDirectory(dir, false);
     return true;
   }
 
@@ -141,15 +179,21 @@ private S3AFileStatus probePathStatusOrNull(final Path path,
   /**
    * Get the status of a path -optimized for paths
    * where there is a directory marker or child entries.
+   *
+   * Under a magic path, there's no check for a file,
+   * just the listing.
+   *
    * @param path path to probe.
+   *
    * @return the status
+   *
    * @throws IOException failure
    */
   private S3AFileStatus getPathStatusExpectingDir(final Path path)
       throws IOException {
     S3AFileStatus status = probePathStatusOrNull(path,
         StatusProbeEnum.DIRECTORIES);
-    if (status == null) {
+    if (status == null && !isMagicPath) {
       status = probePathStatusOrNull(path,
           StatusProbeEnum.FILE);
     }
@@ -175,10 +219,15 @@ S3AFileStatus probePathStatus(Path path,
     /**
      * Create a fake directory, always ending in "/".
      * Retry policy: retrying; translated.
-     * @param key name of directory object.
+     * the keepMarkers flag controls whether or not markers
+     * are automatically kept (this is set when creating
+     * directories under a magic path, always)
+     * @param dir dir to create
+     * @param keepMarkers always keep markers
+     *
      * @throws IOException IO failure
      */
     @Retries.RetryTranslated
-    void createFakeDirectory(String key) throws IOException;
+    void createFakeDirectory(Path dir, boolean keepMarkers) throws IOException;
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java
index 69964ad256358..96e32f362dfd9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MultiObjectDeleteSupport.java
@@ -20,50 +20,27 @@
 
 import java.io.IOException;
 import java.nio.file.AccessDeniedException;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
-import java.util.Objects;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.AWSS3IOException;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Support for Multi Object Deletion.
+ * This is used to be a complex piece of code as it was required to
+ * update s3guard.
+ * Now all that is left is the exception extraction for better
+ * reporting,
  */
-public final class MultiObjectDeleteSupport extends AbstractStoreOperation {
+public final class MultiObjectDeleteSupport {
 
   private static final Logger LOG = LoggerFactory.getLogger(
       MultiObjectDeleteSupport.class);
 
-  private final BulkOperationState operationState;
-
-  /**
-   * Initiate with a store context.
-   * @param context store context.
-   * @param operationState any ongoing bulk operation.
-   */
-  public MultiObjectDeleteSupport(final StoreContext context,
-      final BulkOperationState operationState) {
-    super(context);
-    this.operationState = operationState;
+  private MultiObjectDeleteSupport() {
   }
 
   /**
@@ -89,7 +66,7 @@ public static IOException translateDeleteException(
       final MultiObjectDeleteException deleteException) {
     List<MultiObjectDeleteException.DeleteError> errors
         = deleteException.getErrors();
-    LOG.warn("Bulk delete operation failed to delete all objects;"
+    LOG.info("Bulk delete operation failed to delete all objects;"
             + " failure count = {}",
         errors.size());
     final StringBuilder result = new StringBuilder(
@@ -104,7 +81,7 @@ public static IOException translateDeleteException(
               ? (" (" + error.getVersionId() + ")")
               : ""),
           error.getMessage());
-      LOG.warn(item);
+      LOG.info(item);
       result.append(item);
       if (exitCode == null || exitCode.isEmpty() || ACCESS_DENIED.equals(code)) {
         exitCode = code;
@@ -117,301 +94,4 @@ public static IOException translateDeleteException(
       return new AWSS3IOException(result.toString(), deleteException);
     }
   }
-
-  /**
-   * Process a multi object delete exception by building two paths from
-   * the delete request: one of all deleted files, one of all undeleted values.
-   * The latter are those rejected in the delete call.
-   * @param deleteException the delete exception.
-   * @param keysToDelete the keys in the delete request
-   * @return tuple of (undeleted, deleted) paths.
-   */
-  public Pair<List<KeyPath>, List<KeyPath>> splitUndeletedKeys(
-      final MultiObjectDeleteException deleteException,
-      final Collection<DeleteObjectsRequest.KeyVersion> keysToDelete) {
-    LOG.debug("Processing delete failure; keys to delete count = {};"
-            + " errors in exception {}; successful deletions = {}",
-        keysToDelete.size(),
-        deleteException.getErrors().size(),
-        deleteException.getDeletedObjects().size());
-    // convert the collection of keys being deleted into paths
-    final List<KeyPath> pathsBeingDeleted = keysToKeyPaths(keysToDelete);
-    // Take this ist of paths
-    // extract all undeleted entries contained in the exception and
-    // then remove them from the original list.
-    List<KeyPath> undeleted = removeUndeletedPaths(deleteException,
-        pathsBeingDeleted,
-        getStoreContext()::keyToPath);
-    return Pair.of(undeleted, pathsBeingDeleted);
-  }
-
-  /**
-   * Given a list of delete requests, convert them all to paths.
-   * @param keysToDelete list of keys for the delete operation.
-   * @return the paths.
-   */
-  public List<Path> keysToPaths(
-      final Collection<DeleteObjectsRequest.KeyVersion> keysToDelete) {
-    return toPathList(keysToKeyPaths(keysToDelete));
-  }
-
-  /**
-   * Given a list of delete requests, convert them all to keypaths.
-   * @param keysToDelete list of keys for the delete operation.
-   * @return list of keypath entries
-   */
-  public List<KeyPath> keysToKeyPaths(
-      final Collection<DeleteObjectsRequest.KeyVersion> keysToDelete) {
-    return convertToKeyPaths(keysToDelete,
-        getStoreContext()::keyToPath);
-  }
-
-  /**
-   * Given a list of delete requests, convert them all to paths.
-   * @param keysToDelete list of keys for the delete operation.
-   * @param qualifier path qualifier
-   * @return the paths.
-   */
-  public static List<KeyPath> convertToKeyPaths(
-      final Collection<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      final Function<String, Path> qualifier) {
-    List<KeyPath> l = new ArrayList<>(keysToDelete.size());
-    for (DeleteObjectsRequest.KeyVersion kv : keysToDelete) {
-      String key = kv.getKey();
-      Path p = qualifier.apply(key);
-      boolean isDir = key.endsWith("/");
-      l.add(new KeyPath(key, p, isDir));
-    }
-    return l;
-  }
-
-  /**
-   * Process a delete failure by removing from the metastore all entries
-   * which where deleted, as inferred from the delete failures exception
-   * and the original list of files to delete declares to have been deleted.
-   * @param deleteException the delete exception.
-   * @param keysToDelete collection of keys which had been requested.
-   * @param retainedMarkers list built up of retained markers.
-   * @return a tuple of (undeleted, deleted, failures)
-   */
-  public Triple<List<Path>, List<Path>, List<Pair<Path, IOException>>>
-      processDeleteFailure(
-      final MultiObjectDeleteException deleteException,
-      final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      final List<Path> retainedMarkers) {
-    final MetadataStore metadataStore =
-        checkNotNull(getStoreContext().getMetadataStore(),
-            "context metadatastore");
-    final List<Pair<Path, IOException>> failures = new ArrayList<>();
-    final Pair<List<KeyPath>, List<KeyPath>> outcome =
-        splitUndeletedKeys(deleteException, keysToDelete);
-    List<KeyPath> deleted = outcome.getRight();
-    List<Path> deletedPaths = new ArrayList<>();
-    List<KeyPath> undeleted = outcome.getLeft();
-    retainedMarkers.clear();
-    List<Path> undeletedPaths = toPathList((List<KeyPath>) undeleted);
-    // sort shorter keys first,
-    // so that if the left key is longer than the first it is considered
-    // smaller, so appears in the list first.
-    // thus when we look for a dir being empty, we know it holds
-    deleted.sort((l, r) -> r.getKey().length() - l.getKey().length());
-
-    // now go through and delete from S3Guard all paths listed in
-    // the result which are either files or directories with
-    // no children.
-    deleted.forEach(kp -> {
-      Path path = kp.getPath();
-      try{
-        boolean toDelete = true;
-        if (kp.isDirectoryMarker()) {
-          // its a dir marker, which could be an empty dir
-          // (which is then tombstoned), or a non-empty dir, which
-          // is not tombstoned.
-          // for this to be handled, we have to have removed children
-          // from the store first, which relies on the sort
-          PathMetadata pmentry = metadataStore.get(path, true);
-          if (pmentry != null && !pmentry.isDeleted()) {
-            toDelete = pmentry.getFileStatus().isEmptyDirectory()
-                == Tristate.TRUE;
-          } else {
-            toDelete = false;
-          }
-        }
-        if (toDelete) {
-          LOG.debug("Removing deleted object from S3Guard Store {}", path);
-          metadataStore.delete(path, operationState);
-        } else {
-          LOG.debug("Retaining S3Guard directory entry {}", path);
-          retainedMarkers.add(path);
-        }
-      } catch (IOException e) {
-        // trouble: we failed to delete the far end entry
-        // try with the next one.
-        // if this is a big network failure, this is going to be noisy.
-        LOG.warn("Failed to update S3Guard store with deletion of {}", path);
-        failures.add(Pair.of(path, e));
-      }
-      // irrespective of the S3Guard outcome, it is declared as deleted, as
-      // it is no longer in the S3 store.
-      deletedPaths.add(path);
-    });
-    if (LOG.isDebugEnabled()) {
-      undeleted.forEach(p -> LOG.debug("Deleted {}", p));
-    }
-    return Triple.of(undeletedPaths, deletedPaths, failures);
-  }
-
-  /**
-   * Given a list of keypaths, convert to a list of paths.
-   * @param keyPaths source list
-   * @return a listg of paths
-   */
-  public static List<Path> toPathList(final List<KeyPath> keyPaths) {
-    return keyPaths.stream()
-        .map(KeyPath::getPath)
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Build a list of undeleted paths from a {@code MultiObjectDeleteException}.
-   * Outside of unit tests, the qualifier function should be
-   * {@link S3AFileSystem#keyToQualifiedPath(String)}.
-   * @param deleteException the delete exception.
-   * @param qualifierFn function to qualify paths
-   * @return the possibly empty list of paths.
-   */
-  @VisibleForTesting
-  public static List<Path> extractUndeletedPaths(
-      final MultiObjectDeleteException deleteException,
-      final Function<String, Path> qualifierFn) {
-    return toPathList(extractUndeletedKeyPaths(deleteException, qualifierFn));
-  }
-
-  /**
-   * Build a list of undeleted paths from a {@code MultiObjectDeleteException}.
-   * Outside of unit tests, the qualifier function should be
-   * {@link S3AFileSystem#keyToQualifiedPath(String)}.
-   * @param deleteException the delete exception.
-   * @param qualifierFn function to qualify paths
-   * @return the possibly empty list of paths.
-   */
-  @VisibleForTesting
-  public static List<KeyPath> extractUndeletedKeyPaths(
-      final MultiObjectDeleteException deleteException,
-      final Function<String, Path> qualifierFn) {
-
-    List<MultiObjectDeleteException.DeleteError> errors
-        = deleteException.getErrors();
-    return errors.stream()
-        .map((error) -> {
-          String key = error.getKey();
-          Path path = qualifierFn.apply(key);
-          boolean isDir = key.endsWith("/");
-          return new KeyPath(key, path, isDir);
-        })
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Process a {@code MultiObjectDeleteException} by
-   * removing all undeleted paths from the list of paths being deleted.
-   * The original list is updated, and so becomes the list of successfully
-   * deleted paths.
-   * @param deleteException the delete exception.
-   * @param pathsBeingDeleted list of paths which were being deleted.
-   * This has all undeleted paths removed, leaving only those deleted.
-   * @return the list of undeleted entries
-   */
-  @VisibleForTesting
-  static List<KeyPath> removeUndeletedPaths(
-      final MultiObjectDeleteException deleteException,
-      final Collection<KeyPath> pathsBeingDeleted,
-      final Function<String, Path> qualifier) {
-    // get the undeleted values
-    List<KeyPath> undeleted = extractUndeletedKeyPaths(deleteException,
-        qualifier);
-    // and remove them from the undeleted list, matching on key
-    for (KeyPath undel : undeleted) {
-      pathsBeingDeleted.removeIf(kp -> kp.getPath().equals(undel.getPath()));
-    }
-    return undeleted;
-  }
-
-  /**
-   * A delete operation failed.
-   * Currently just returns the list of all paths.
-   * @param ex exception.
-   * @param keysToDelete the keys which were being deleted.
-   * @return all paths which were not deleted.
-   */
-  public List<Path> processDeleteFailureGenericException(Exception ex,
-      final List<DeleteObjectsRequest.KeyVersion> keysToDelete) {
-    return keysToPaths(keysToDelete);
-  }
-
-  /**
-   * Representation of a (key, path) which couldn't be deleted;
-   * the dir marker flag is inferred from the key suffix.
-   * <p>
-   * Added because Pairs of Lists of Triples was just too complex
-   * for Java code.
-   * </p>
-   */
-  public static final class KeyPath {
-    /** Key in bucket. */
-    private final String key;
-    /** Full path. */
-    private final Path path;
-    /** Is this a directory marker? */
-    private final boolean directoryMarker;
-
-    public KeyPath(final String key,
-        final Path path,
-        final boolean directoryMarker) {
-      this.key = key;
-      this.path = path;
-      this.directoryMarker = directoryMarker;
-    }
-
-    public String getKey() {
-      return key;
-    }
-
-    public Path getPath() {
-      return path;
-    }
-
-    public boolean isDirectoryMarker() {
-      return directoryMarker;
-    }
-
-    @Override
-    public String toString() {
-      return "KeyPath{" +
-          "key='" + key + '\'' +
-          ", path=" + path +
-          ", directoryMarker=" + directoryMarker +
-          '}';
-    }
-
-    /**
-     * Equals test is on key alone.
-     */
-    @Override
-    public boolean equals(final Object o) {
-      if (this == o) {
-        return true;
-      }
-      if (o == null || getClass() != o.getClass()) {
-        return false;
-      }
-      KeyPath keyPath = (KeyPath) o;
-      return key.equals(keyPath.key);
-    }
-
-    @Override
-    public int hashCode() {
-      return Objects.hash(key);
-    }
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
index 409ac7bebc488..575a3d1b2de81 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
@@ -118,7 +118,7 @@ void configureSocketFactory(ClientConfiguration awsConf,
    * See also {@code com.amazonaws.services.s3.model.Region.fromValue()}
    * for its conversion logic.
    * @param region region from S3 call.
-   * @return the region to use in DDB etc.
+   * @return the region to use in AWS services.
    */
   public static String fixBucketRegion(final String region) {
     return region == null || region.equals("US")
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OpenFileSupport.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OpenFileSupport.java
new file mode 100644
index 0000000000000..70a99d5318c06
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OpenFileSupport.java
@@ -0,0 +1,600 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.select.InternalSelectConstants;
+import org.apache.hadoop.fs.s3a.select.SelectConstants;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_END;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_START;
+import static org.apache.hadoop.fs.impl.AbstractFSBuilderImpl.rejectUnknownMandatoryKeys;
+import static org.apache.hadoop.fs.s3a.Constants.ASYNC_DRAIN_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
+import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+
+/**
+ * Helper class for openFile() logic, especially processing file status
+ * args and length/etag/versionID.
+ * <p>
+ *  This got complex enough it merited removal from S3AFileSystem -which
+ *  also permits unit testing.
+ * </p>
+ * <p>
+ *   The default values are those from the FileSystem configuration.
+ *   in openFile(), they can all be changed by specific options;
+ *   in FileSystem.open(path, buffersize) only the buffer size is
+ *   set.
+ * </p>
+ */
+public class OpenFileSupport {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(OpenFileSupport.class);
+
+  /**
+   * For use when a value of an split/file length is unknown.
+   */
+  private static final int LENGTH_UNKNOWN = -1;
+
+  /**  Default change detection policy. */
+  private final ChangeDetectionPolicy changePolicy;
+
+  /** Default read ahead range. */
+  private final long defaultReadAhead;
+
+  /** Username. */
+  private final String username;
+
+  /** Default buffer size. */
+  private final int defaultBufferSize;
+
+  /**
+   * Threshold for stream reads to switch to
+   * asynchronous draining.
+   */
+  private final long defaultAsyncDrainThreshold;
+
+  /**
+   * Default input policy; may be overridden in
+   * {@code openFile()}.
+   */
+  private final S3AInputPolicy defaultInputPolicy;
+
+  /**
+   * Instantiate with the default options from the filesystem.
+   * @param changePolicy change detection policy
+   * @param defaultReadAhead read ahead range
+   * @param username username
+   * @param defaultBufferSize buffer size
+   * @param defaultAsyncDrainThreshold drain threshold
+   * @param defaultInputPolicy input policy
+   */
+  public OpenFileSupport(
+      final ChangeDetectionPolicy changePolicy,
+      final long defaultReadAhead,
+      final String username,
+      final int defaultBufferSize,
+      final long defaultAsyncDrainThreshold,
+      final S3AInputPolicy defaultInputPolicy) {
+    this.changePolicy = changePolicy;
+    this.defaultReadAhead = defaultReadAhead;
+    this.username = username;
+    this.defaultBufferSize = defaultBufferSize;
+    this.defaultAsyncDrainThreshold = defaultAsyncDrainThreshold;
+    this.defaultInputPolicy = defaultInputPolicy;
+  }
+
+  public ChangeDetectionPolicy getChangePolicy() {
+    return changePolicy;
+  }
+
+  public long getDefaultReadAhead() {
+    return defaultReadAhead;
+  }
+
+  public int getDefaultBufferSize() {
+    return defaultBufferSize;
+  }
+
+  public long getDefaultAsyncDrainThreshold() {
+    return defaultAsyncDrainThreshold;
+  }
+
+  /**
+   * Propagate the default options to the operation context
+   * being built up.
+   * @param roc context
+   * @return the context
+   */
+  public S3AReadOpContext applyDefaultOptions(S3AReadOpContext roc) {
+    return roc
+        .withInputPolicy(defaultInputPolicy)
+        .withChangeDetectionPolicy(changePolicy)
+        .withAsyncDrainThreshold(defaultAsyncDrainThreshold)
+        .withReadahead(defaultReadAhead);
+  }
+
+  /**
+   * Prepare to open a file from the openFile parameters.
+   * @param path path to the file
+   * @param parameters open file parameters from the builder.
+   * @param blockSize for fileStatus
+   * @return open file options
+   * @throws IOException failure to resolve the link.
+   * @throws IllegalArgumentException unknown mandatory key
+   */
+  @SuppressWarnings("ChainOfInstanceofChecks")
+  public OpenFileInformation prepareToOpenFile(
+      final Path path,
+      final OpenFileParameters parameters,
+      final long blockSize) throws IOException {
+    Configuration options = parameters.getOptions();
+    Set<String> mandatoryKeys = parameters.getMandatoryKeys();
+    String sql = options.get(SelectConstants.SELECT_SQL, null);
+    boolean isSelect = sql != null;
+    // choice of keys depends on open type
+    if (isSelect) {
+      // S3 Select call adds a large set of supported mandatory keys
+      rejectUnknownMandatoryKeys(
+          mandatoryKeys,
+          InternalSelectConstants.SELECT_OPTIONS,
+          "for " + path + " in S3 Select operation");
+    } else {
+      rejectUnknownMandatoryKeys(
+          mandatoryKeys,
+          InternalConstants.S3A_OPENFILE_KEYS,
+          "for " + path + " in non-select file I/O");
+    }
+
+    // where does a read end?
+    long fileLength = LENGTH_UNKNOWN;
+
+    // was a status passed in via a withStatus() invocation in
+    // the builder API?
+    FileStatus providedStatus = parameters.getStatus();
+    S3AFileStatus fileStatus = null;
+    if (providedStatus != null) {
+      // there's a file status
+
+      // make sure the file name matches -the rest of the path
+      // MUST NOT be checked.
+      Path providedStatusPath = providedStatus.getPath();
+      checkArgument(path.getName().equals(providedStatusPath.getName()),
+          "Filename mismatch between file being opened %s and"
+              + " supplied filestatus %s",
+          path, providedStatusPath);
+
+      // make sure the status references a file
+      if (providedStatus.isDirectory()) {
+        throw new FileNotFoundException(
+            "Supplied status references a directory " + providedStatus);
+      }
+      // build up the values
+      long len = providedStatus.getLen();
+      long modTime = providedStatus.getModificationTime();
+      String versionId;
+      String eTag;
+      // can use this status to skip our own probes,
+      LOG.debug("File was opened with a supplied FileStatus;"
+              + " skipping getFileStatus call in open() operation: {}",
+          providedStatus);
+
+      // what type is the status (and hence: what information does it contain?)
+      if (providedStatus instanceof S3AFileStatus) {
+        // is it an S3AFileSystem status?
+        S3AFileStatus st = (S3AFileStatus) providedStatus;
+        versionId = st.getVersionId();
+        eTag = st.getEtag();
+      } else if (providedStatus instanceof S3ALocatedFileStatus) {
+
+        //  S3ALocatedFileStatus instance may supply etag and version.
+        S3ALocatedFileStatus st = (S3ALocatedFileStatus) providedStatus;
+        versionId = st.getVersionId();
+        eTag = st.getEtag();
+      } else {
+        // it is another type.
+        // build a status struct without etag or version.
+        LOG.debug("Converting file status {}", providedStatus);
+        versionId = null;
+        eTag = null;
+      }
+      // Construct a new file status with the real path of the file.
+      fileStatus = new S3AFileStatus(
+          len,
+          modTime,
+          path,
+          blockSize,
+          username,
+          eTag,
+          versionId);
+      // set the end of the read to the file length
+      fileLength = fileStatus.getLen();
+    }
+    // determine start and end of file.
+    long splitStart = options.getLong(FS_OPTION_OPENFILE_SPLIT_START, 0);
+
+    // split end
+    long splitEnd = options.getLong(FS_OPTION_OPENFILE_SPLIT_END,
+        LENGTH_UNKNOWN);
+    if (splitStart > 0 && splitStart > splitEnd) {
+      LOG.warn("Split start {} is greater than split end {}, resetting",
+          splitStart, splitEnd);
+      splitStart = 0;
+    }
+
+    // read end is the open file value
+    fileLength = options.getLong(FS_OPTION_OPENFILE_LENGTH, fileLength);
+
+    // if the read end has come from options, use that
+    // in creating a file status
+    if (fileLength >= 0 && fileStatus == null) {
+      fileStatus = createStatus(path, fileLength, blockSize);
+    }
+
+    // Build up the input policy.
+    // seek policy from default, s3a opt or standard option
+    // read from the FS standard option.
+    Collection<String> policies =
+        options.getStringCollection(FS_OPTION_OPENFILE_READ_POLICY);
+    if (policies.isEmpty()) {
+      // fall back to looking at the S3A-specific option.
+      policies = options.getStringCollection(INPUT_FADVISE);
+    }
+
+    return new OpenFileInformation()
+        .withS3Select(isSelect)
+        .withSql(sql)
+        .withAsyncDrainThreshold(
+            options.getLong(ASYNC_DRAIN_THRESHOLD,
+                defaultReadAhead))
+        .withBufferSize(
+            options.getInt(FS_OPTION_OPENFILE_BUFFER_SIZE, defaultBufferSize))
+        .withChangePolicy(changePolicy)
+        .withFileLength(fileLength)
+        .withInputPolicy(
+            S3AInputPolicy.getFirstSupportedPolicy(policies, defaultInputPolicy))
+        .withReadAheadRange(
+            options.getLong(READAHEAD_RANGE, defaultReadAhead))
+        .withSplitStart(splitStart)
+        .withSplitEnd(splitEnd)
+        .withStatus(fileStatus)
+        .build();
+
+  }
+
+  /**
+   * Create a minimal file status.
+   * @param path path
+   * @param length file length/read end
+   * @param blockSize block size
+   * @return a new status
+   */
+  private S3AFileStatus createStatus(Path path, long length, long blockSize) {
+    return new S3AFileStatus(
+        length,
+        0,
+        path,
+        blockSize,
+        username,
+        null,
+        null);
+  }
+
+  /**
+   * Open a simple file, using all the default
+   * options.
+   * @return the parameters needed to open a file through
+   * {@code open(path, bufferSize)}.
+   * @param bufferSize  buffer size
+   */
+  public OpenFileInformation openSimpleFile(final int bufferSize) {
+    return new OpenFileInformation()
+        .withS3Select(false)
+        .withAsyncDrainThreshold(defaultAsyncDrainThreshold)
+        .withBufferSize(bufferSize)
+        .withChangePolicy(changePolicy)
+        .withFileLength(LENGTH_UNKNOWN)
+        .withInputPolicy(defaultInputPolicy)
+        .withReadAheadRange(defaultReadAhead)
+        .withSplitStart(0)
+        .withSplitEnd(LENGTH_UNKNOWN)
+        .build();
+  }
+
+  @Override
+  public String toString() {
+    return "OpenFileSupport{" +
+        "changePolicy=" + changePolicy +
+        ", defaultReadAhead=" + defaultReadAhead +
+        ", defaultBufferSize=" + defaultBufferSize +
+        ", defaultAsyncDrainThreshold=" + defaultAsyncDrainThreshold +
+        ", defaultInputPolicy=" + defaultInputPolicy +
+        '}';
+  }
+
+  /**
+   * The information on a file needed to open it.
+   */
+  public static final class OpenFileInformation {
+
+    /** Is this SQL? */
+    private boolean isS3Select;
+
+    /** File status; may be null. */
+    private S3AFileStatus status;
+
+    /** SQL string if this is a SQL select file. */
+    private String sql;
+
+    /** Active input policy. */
+    private S3AInputPolicy inputPolicy;
+
+    /** Change detection policy. */
+    private ChangeDetectionPolicy changePolicy;
+
+    /** Read ahead range. */
+    private long readAheadRange;
+
+    /** Buffer size. Currently ignored. */
+    private int bufferSize;
+
+    /**
+     * Where does the read start from. 0 unless known.
+     */
+    private long splitStart;
+
+    /**
+     * What is the split end?
+     * Negative if not known.
+     */
+    private long splitEnd = -1;
+
+    /**
+     * What is the file length?
+     * Negative if not known.
+     */
+    private long fileLength = -1;
+
+    /**
+     * Threshold for stream reads to switch to
+     * asynchronous draining.
+     */
+    private long asyncDrainThreshold;
+
+    /**
+     * Constructor.
+     */
+    public OpenFileInformation() {
+    }
+
+    /**
+     * Build.
+     * @return this object
+     */
+    public OpenFileInformation build() {
+      return this;
+    }
+
+    public boolean isS3Select() {
+      return isS3Select;
+    }
+
+    public S3AFileStatus getStatus() {
+      return status;
+    }
+
+    public String getSql() {
+      return sql;
+    }
+
+    public S3AInputPolicy getInputPolicy() {
+      return inputPolicy;
+    }
+
+    public ChangeDetectionPolicy getChangePolicy() {
+      return changePolicy;
+    }
+
+    public long getReadAheadRange() {
+      return readAheadRange;
+    }
+
+    public int getBufferSize() {
+      return bufferSize;
+    }
+
+    public long getSplitStart() {
+      return splitStart;
+    }
+
+    public long getSplitEnd() {
+      return splitEnd;
+    }
+
+    @Override
+    public String toString() {
+      return "OpenFileInformation{" +
+          "isSql=" + isS3Select +
+          ", status=" + status +
+          ", sql='" + sql + '\'' +
+          ", inputPolicy=" + inputPolicy +
+          ", changePolicy=" + changePolicy +
+          ", readAheadRange=" + readAheadRange +
+          ", splitStart=" + splitStart +
+          ", splitEnd=" + splitEnd +
+          ", bufferSize=" + bufferSize +
+          ", drainThreshold=" + asyncDrainThreshold +
+          '}';
+    }
+
+    /**
+     * Get the file length.
+     * @return the file length; -1 if not known.
+     */
+    public long getFileLength() {
+      return fileLength;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withS3Select(final boolean value) {
+      isS3Select = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withStatus(final S3AFileStatus value) {
+      status = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withSql(final String value) {
+      sql = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withInputPolicy(final S3AInputPolicy value) {
+      inputPolicy = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withChangePolicy(final ChangeDetectionPolicy value) {
+      changePolicy = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withReadAheadRange(final long value) {
+      readAheadRange = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withBufferSize(final int value) {
+      bufferSize = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withSplitStart(final long value) {
+      splitStart = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withSplitEnd(final long value) {
+      splitEnd = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withFileLength(final long value) {
+      fileLength = value;
+      return this;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public OpenFileInformation withAsyncDrainThreshold(final long value) {
+      asyncDrainThreshold = value;
+      return this;
+    }
+
+    /**
+     * Propagate the options to the operation context
+     * being built up.
+     * @param roc context
+     * @return the context
+     */
+    public S3AReadOpContext applyOptions(S3AReadOpContext roc) {
+      return roc
+          .withInputPolicy(inputPolicy)
+          .withChangeDetectionPolicy(changePolicy)
+          .withAsyncDrainThreshold(asyncDrainThreshold)
+          .withReadahead(readAheadRange);
+    }
+
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
index 3391097fc5730..5d17ae91b81e7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
@@ -24,7 +24,6 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import com.amazonaws.services.s3.transfer.model.CopyResult;
 
@@ -37,7 +36,6 @@
 import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
 import org.apache.hadoop.fs.s3a.S3AReadOpContext;
 import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 
 /**
  * These are all the callbacks which the {@link RenameOperation}
@@ -87,21 +85,18 @@ S3AReadOpContext createReadContext(
   void finishRename(Path sourceRenamed, Path destCreated) throws IOException;
 
   /**
-   * Delete an object, also updating the metastore.
+   * Delete an object.
    * This call does <i>not</i> create any mock parent entries.
    * Retry policy: retry untranslated; delete considered idempotent.
    * @param path path to delete
    * @param key key of entry
    * @param isFile is the path a file (used for instrumentation only)
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws AmazonClientException problems working with S3
-   * @throws IOException IO failure in the metastore
+   * @throws IOException from invoker signature only -should not be raised.
    */
   @Retries.RetryTranslated
   void deleteObjectAtPath(Path path,
       String key,
-      boolean isFile,
-      BulkOperationState operationState)
+      boolean isFile)
       throws IOException;
 
   /**
@@ -109,7 +104,6 @@ void deleteObjectAtPath(Path path,
    *
    * @param path path to list from
    * @param status optional status of path to list.
-   * @param collectTombstones should tombstones be collected from S3Guard?
    * @param includeSelf should the listing include this path if present?
    * @return an iterator.
    * @throws IOException failure
@@ -118,7 +112,6 @@ void deleteObjectAtPath(Path path,
   RemoteIterator<S3ALocatedFileStatus> listFilesAndDirectoryMarkers(
       Path path,
       S3AFileStatus status,
-      boolean collectTombstones,
       boolean includeSelf) throws IOException;
 
   /**
@@ -126,6 +119,7 @@ RemoteIterator<S3ALocatedFileStatus> listFilesAndDirectoryMarkers(
    * There's no update of metadata, directory markers, etc.
    * Callers must implement.
    * @param srcKey source object path
+   * @param destKey destination object path
    * @param srcAttributes S3 attributes of the source object
    * @param readContext the read context
    * @return the result of the copy
@@ -140,21 +134,10 @@ CopyResult copyFile(String srcKey,
       throws IOException;
 
   /**
-   * Remove keys from the store, updating the metastore on a
-   * partial delete represented as a MultiObjectDeleteException failure by
-   * deleting all those entries successfully deleted and then rethrowing
-   * the MultiObjectDeleteException.
+   * Remove keys from the store.
    * @param keysToDelete collection of keys to delete on the s3-backend.
    *        if empty, no request is made of the object store.
    * @param deleteFakeDir indicates whether this is for deleting fake dirs.
-   * @param undeletedObjectsOnFailure List which will be built up of all
-   * files that were not deleted. This happens even as an exception
-   * is raised.
-   * @param operationState bulk operation state
-   * @param quiet should a bulk query be quiet, or should its result list
-   * all deleted keys
-   * @return the deletion result if a multi object delete was invoked
-   * and it returned without a failure, else null.
    * @throws InvalidRequestException if the request was rejected due to
    * a mistaken attempt to delete the root directory.
    * @throws MultiObjectDeleteException one or more of the keys could not
@@ -162,28 +145,15 @@ CopyResult copyFile(String srcKey,
    * @throws AmazonClientException amazon-layer failure.
    * @throws IOException other IO Exception.
    */
-  @Retries.RetryMixed
-  DeleteObjectsResult removeKeys(
-      List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean deleteFakeDir,
-      List<Path> undeletedObjectsOnFailure,
-      BulkOperationState operationState,
-      boolean quiet)
+  @Retries.RetryRaw
+  void removeKeys(
+          List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+          boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException,
       IOException;
 
   /**
-   * Is the path for this instance considered authoritative on the client,
-   * that is: will listing/status operations only be handled by the metastore,
-   * with no fallback to S3.
-   * @param p path
-   * @return true iff the path is authoritative on the client.
-   */
-  boolean allowAuthoritative(Path p);
-
-  /**
-   * Create an iterator over objects in S3 only; S3Guard
-   * is not involved.
+   * Create an iterator over objects in S3.
    * The listing includes the key itself, if found.
    * @param path  path of the listing.
    * @param key object key
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java
new file mode 100644
index 0000000000000..e14285a1ca8b1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.util.Map;
+import javax.annotation.Nullable;
+
+/**
+ * Extensible structure for options when putting/writing objects.
+ */
+public final class PutObjectOptions {
+
+  /**
+   * Can the PUT operation skip marker deletion?
+   */
+  private final boolean keepMarkers;
+
+  /**
+   * Storage class, if not null.
+   */
+  private final String storageClass;
+
+  /**
+   * Headers; may be null.
+   */
+  private final Map<String, String> headers;
+
+  /**
+   * Constructor.
+   * @param keepMarkers Can the PUT operation skip marker deletion?
+   * @param storageClass Storage class, if not null.
+   * @param headers Headers; may be null.
+   */
+  public PutObjectOptions(
+      final boolean keepMarkers,
+      @Nullable final String storageClass,
+      @Nullable final Map<String, String> headers) {
+    this.keepMarkers = keepMarkers;
+    this.storageClass = storageClass;
+    this.headers = headers;
+  }
+
+  /**
+   * Get the marker retention flag.
+   * @return true if markers are to be retained.
+   */
+  public boolean isKeepMarkers() {
+    return keepMarkers;
+  }
+
+  /**
+   * Headers for the put/post request.
+   * @return headers or null.
+   */
+  public Map<String, String> getHeaders() {
+    return headers;
+  }
+
+  @Override
+  public String toString() {
+    return "PutObjectOptions{" +
+        "keepMarkers=" + keepMarkers +
+        ", storageClass='" + storageClass + '\'' +
+        '}';
+  }
+
+  private static final PutObjectOptions KEEP_DIRS = new PutObjectOptions(true,
+      null, null);
+  private static final PutObjectOptions DELETE_DIRS = new PutObjectOptions(false,
+      null, null);
+
+  /**
+   * Get the options to keep directories.
+   * @return an instance which keeps dirs
+   */
+  public static PutObjectOptions keepingDirs() {
+    return KEEP_DIRS;
+  }
+
+  /**
+   * Get the options to delete directory markers.
+   * @return an instance which deletes dirs
+   */
+  public static PutObjectOptions deletingDirs() {
+    return DELETE_DIRS;
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
index b7e0f353fabb0..ae4d2fe7a3431 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RenameOperation.java
@@ -26,14 +26,14 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.SdkBaseException;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.transfer.model.CopyResult;
-import org.apache.hadoop.util.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.Invoker;
 import org.apache.hadoop.fs.s3a.RenameFailedException;
 import org.apache.hadoop.fs.s3a.Retries;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
@@ -41,36 +41,28 @@
 import org.apache.hadoop.fs.s3a.S3AReadOpContext;
 import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
 import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.RenameTracker;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.OperationDuration;
 
+import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
 import static org.apache.hadoop.fs.store.audit.AuditingFunctions.callableWithinAuditSpan;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_BLOCK_SIZE;
-import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
-import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_BLOCKSIZE;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.RENAME_PARALLEL_LIMIT;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 
 /**
- * A parallelized rename operation which updates the metastore in the
- * process, through whichever {@link RenameTracker} the store provides.
- * <p></p>
+ * A parallelized rename operation.
+ * <p>
  * The parallel execution is in groups of size
  * {@link InternalConstants#RENAME_PARALLEL_LIMIT}; it is only
  * after one group completes that the next group is initiated.
- * <p></p>
+ * <p>
  * Once enough files have been copied that they meet the
  * {@link InternalConstants#MAX_ENTRIES_TO_DELETE} threshold, a delete
  * is initiated.
  * If it succeeds, the rename continues with the next group of files.
- * <p></p>
- * The RenameTracker has the task of keeping the metastore up to date
- * as the rename proceeds.
- * <p></p>
+ * <p>
  * Directory Markers which have child entries are never copied; only those
  * which represent empty directories are copied in the rename.
  * The {@link DirMarkerTracker} tracks which markers must be copied, and
@@ -79,10 +71,10 @@
  * the copied tree. This is to ensure that even if a directory tree
  * is copied from an authoritative path to a non-authoritative one
  * there is never any contamination of the non-auth path with markers.
- * <p></p>
+ * <p>
  * The rename operation implements the classic HDFS rename policy of
  * rename(file, dir) renames the file under the directory.
- * <p></p>
+ * <p>
  *
  * There is <i>no</i> validation of input and output paths.
  * Callers are required to themselves verify that destination is not under
@@ -121,11 +113,6 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
    */
   private final int pageSize;
 
-  /**
-   * Rename tracker.
-   */
-  private RenameTracker renameTracker;
-
   /**
    * List of active copies.
    */
@@ -138,14 +125,6 @@ public class RenameOperation extends ExecutingStoreOperation<Long> {
   private final List<DeleteObjectsRequest.KeyVersion> keysToDelete =
       new ArrayList<>();
 
-  /**
-   * List of paths to delete, which will be passed to the rename
-   * tracker after the deletion succeeds.
-   */
-  private final List<Path> pathsToDelete = new ArrayList<>();
-
-  private final long blocksize;
-
   /**
    * Initiate the rename.
    *
@@ -177,8 +156,9 @@ public RenameOperation(
     this.destKey = destKey;
     this.destStatus = destStatus;
     this.callbacks = callbacks;
-    blocksize = storeContext.getConfiguration()
-        .getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE);
+    checkArgument(pageSize > 0
+                    && pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE,
+            "page size out of range: %s", pageSize);
     this.pageSize = pageSize;
   }
 
@@ -198,7 +178,7 @@ private void completeActiveCopies(String reason) throws IOException {
 
   /**
    * Queue an object for deletion.
-   * <p></p>
+   * <p>
    * This object will be deleted when the next page of objects to delete
    * is posted to S3. Therefore, the COPY must have finished
    * before that deletion operation takes place.
@@ -212,13 +192,6 @@ private void completeActiveCopies(String reason) throws IOException {
    *     Only queuing objects here whose copy operation has
    *     been submitted and so is in that thread pool.
    *   </li>
-   *   <li>
-   *     If a path is supplied, then after the delete is executed
-   *     (and completes) the rename tracker from S3Guard will be
-   *     told of its deletion. Do not set this for directory
-   *     markers with children, as it may mistakenly add
-   *     tombstones into the table.
-   *   </li>
    * </ol>
    * This method must only be called from the primary thread.
    * @param path path to the object.
@@ -226,17 +199,14 @@ private void completeActiveCopies(String reason) throws IOException {
    */
   private void queueToDelete(Path path, String key) {
     LOG.debug("Queueing to delete {}", path);
-    if (path != null) {
-      pathsToDelete.add(path);
-    }
     keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
   }
 
   /**
    * Queue a list of markers for deletion.
-   * <p></p>
+   * <p>
    * no-op if the list is empty.
-   * <p></p>
+   * <p>
    * See {@link #queueToDelete(Path, String)} for
    * details on safe use of this method.
    *
@@ -251,7 +221,7 @@ private void queueToDelete(
 
   /**
    * Queue a single marker for deletion.
-   * <p></p>
+   * <p>
    * See {@link #queueToDelete(Path, String)} for
    * details on safe use of this method.
    *
@@ -272,28 +242,15 @@ private void completeActiveCopiesAndDeleteSources(String reason)
       throws IOException {
     completeActiveCopies(reason);
     removeSourceObjects(
-        keysToDelete,
-        pathsToDelete);
+        keysToDelete
+    );
     // now reset the lists.
     keysToDelete.clear();
-    pathsToDelete.clear();
   }
 
   @Retries.RetryMixed
   public Long execute() throws IOException {
     executeOnlyOnce();
-    final StoreContext storeContext = getStoreContext();
-    final MetadataStore metadataStore = checkNotNull(
-        storeContext.getMetadataStore(),
-        "No metadata store in context");
-
-    // Validation completed: time to begin the operation.
-    // The store-specific rename tracker is used to keep the store
-    // to date with the in-progress operation.
-    // for the null store, these are all no-ops.
-    renameTracker = metadataStore.initiateRenameOperation(
-        storeContext,
-        sourcePath, sourceStatus, destPath);
 
     // The path to whichever file or directory is created by the
     // rename. When deleting markers all parents of
@@ -317,21 +274,12 @@ public Long execute() throws IOException {
       try {
         completeActiveCopies("failure handling");
       } catch (IOException e) {
-        // a failure to update the metastore after a rename failure is what
-        // we'd see on a network problem, expired credentials and other
-        // unrecoverable errors.
         // Downgrading to warn because an exception is already
         // about to be thrown.
         LOG.warn("While completing all active copies", e);
       }
-      // notify the rename tracker of the failure
-      throw renameTracker.renameFailed(ex);
+      throw convertToIOException(ex);
     }
-
-    // At this point the rename has completed successfully in the S3 store.
-    // Tell the metastore this fact and let it complete its changes
-    renameTracker.completeRename();
-
     callbacks.finishRename(sourcePath, destCreated);
     return bytesCopied.get();
   }
@@ -362,19 +310,15 @@ protected Path renameFileToDest() throws IOException {
     // destination either does not exist or is a file to overwrite.
     LOG.debug("rename: renaming file {} to {}", sourcePath,
         copyDestinationPath);
-    copySourceAndUpdateTracker(
-        sourcePath,
+    copySource(
         sourceKey,
         sourceAttributes,
         readContext,
         copyDestinationPath,
-        copyDestinationKey,
-        false);
+        copyDestinationKey);
     bytesCopied.addAndGet(sourceStatus.getLen());
     // delete the source
-    callbacks.deleteObjectAtPath(sourcePath, sourceKey, true, null);
-    // and update the tracker
-    renameTracker.sourceObjectsDeleted(Lists.newArrayList(sourcePath));
+    callbacks.deleteObjectAtPath(sourcePath, sourceKey, true);
     return copyDestinationPath;
   }
 
@@ -402,15 +346,12 @@ protected void recursiveDirectoryRename() throws IOException {
     if (destStatus != null
         && destStatus.isEmptyDirectory() == Tristate.TRUE) {
       // delete unnecessary fake directory at the destination.
-      // this MUST be done before anything else so that
-      // rollback code doesn't get confused and insert a tombstone
-      // marker.
+
       LOG.debug("Deleting fake directory marker at destination {}",
           destStatus.getPath());
       // Although the dir marker policy doesn't always need to do this,
       // it's simplest just to be consistent here.
-      // note: updates the metastore as well a S3.
-      callbacks.deleteObjectAtPath(destStatus.getPath(), dstKey, false, null);
+      callbacks.deleteObjectAtPath(destStatus.getPath(), dstKey, false);
     }
 
     Path parentPath = storeContext.keyToPath(srcKey);
@@ -423,7 +364,6 @@ protected void recursiveDirectoryRename() throws IOException {
     final RemoteIterator<S3ALocatedFileStatus> iterator =
         callbacks.listFilesAndDirectoryMarkers(parentPath,
             sourceStatus,
-            true,
             true);
     while (iterator.hasNext()) {
       // get the next entry in the listing.
@@ -464,7 +404,7 @@ protected void recursiveDirectoryRename() throws IOException {
         queueToDelete(childSourcePath, key);
           // now begin the single copy
         CompletableFuture<Path> copy = initiateCopy(child, key,
-            childSourcePath, newDestKey, childDestPath);
+            newDestKey, childDestPath);
         activeCopies.add(copy);
         bytesCopied.addAndGet(sourceStatus.getLen());
       }
@@ -483,14 +423,11 @@ protected void recursiveDirectoryRename() throws IOException {
     // have been deleted.
     completeActiveCopiesAndDeleteSources("final copy and delete");
 
-    // We moved all the children, now move the top-level dir
-    // Empty directory should have been added as the object summary
-    renameTracker.moveSourceDirectory();
   }
 
   /**
    * Operations to perform at the end of every loop iteration.
-   * <p></p>
+   * <p>
    * This may block the thread waiting for copies to complete
    * and/or delete a page of data.
    */
@@ -511,12 +448,11 @@ private void endOfLoopActions() throws IOException {
   /**
    * Process all directory markers at the end of the rename.
    * All leaf markers are queued to be copied in the store;
-   * this updates the metastore tracker as it does so.
-   * <p></p>
+   * <p>
    * Why not simply create new markers? All the metadata
    * gets copied too, so if there was anything relevant then
    * it would be preserved.
-   * <p></p>
+   * <p>
    * At the same time: markers aren't valued much and may
    * be deleted without any safety checks -so if there was relevant
    * data it is at risk of destruction at any point.
@@ -525,7 +461,7 @@ private void endOfLoopActions() throws IOException {
    * Be advised though: the costs of the copy not withstanding,
    * it is a lot easier to have one single type of scheduled copy operation
    * than have copy and touch calls being scheduled.
-   * <p></p>
+   * <p>
    * The duration returned is the time to initiate all copy/delete operations,
    * including any blocking waits for active copies and paged deletes
    * to execute. There may still be outstanding operations
@@ -553,7 +489,6 @@ private OperationDuration copyEmptyDirectoryMarkers(
         "copying %d leaf markers with %d surplus not copied",
         leafMarkers.size(), surplus.size());
     for (DirMarkerTracker.Marker entry: leafMarkers.values()) {
-      Path source = entry.getPath();
       String key = entry.getKey();
       String newDestKey =
           dstKey + key.substring(srcKey.length());
@@ -564,7 +499,6 @@ private OperationDuration copyEmptyDirectoryMarkers(
           initiateCopy(
               entry.getStatus(),
               key,
-              source,
               newDestKey,
               childDestPath));
       queueToDelete(entry);
@@ -579,7 +513,6 @@ private OperationDuration copyEmptyDirectoryMarkers(
    * Initiate a copy operation in the executor.
    * @param source status of the source object.
    * @param key source key
-   * @param childSourcePath source as a path.
    * @param newDestKey destination key
    * @param childDestPath destination path.
    * @return the future.
@@ -587,125 +520,78 @@ private OperationDuration copyEmptyDirectoryMarkers(
   protected CompletableFuture<Path> initiateCopy(
       final S3ALocatedFileStatus source,
       final String key,
-      final Path childSourcePath,
       final String newDestKey,
       final Path childDestPath) {
     S3ObjectAttributes sourceAttributes =
         callbacks.createObjectAttributes(
             source.getPath(),
-            source.getETag(),
+            source.getEtag(),
             source.getVersionId(),
             source.getLen());
     // queue the copy operation for execution in the thread pool
     return submit(getStoreContext().getExecutor(),
         callableWithinAuditSpan(getAuditSpan(), () ->
-            copySourceAndUpdateTracker(
-                childSourcePath,
+            copySource(
                 key,
                 sourceAttributes,
                 callbacks.createReadContext(source),
                 childDestPath,
-                newDestKey,
-                true)));
+                newDestKey)));
   }
 
   /**
-   * This invoked to copy a file or directory marker then update the
-   * rename operation on success.
+   * This is invoked to copy a file or directory marker.
    * It may be called in its own thread.
-   * @param sourceFile source path of the copy; may have a trailing / on it.
    * @param srcKey source key
    * @param srcAttributes status of the source object
    * @param destination destination as a qualified path.
    * @param destinationKey destination key
-   * @param addAncestors should ancestors be added to the metastore?
    * @return the destination path.
    * @throws IOException failure
    */
   @Retries.RetryTranslated
-  private Path copySourceAndUpdateTracker(
-      final Path sourceFile,
+  private Path copySource(
       final String srcKey,
       final S3ObjectAttributes srcAttributes,
       final S3AReadOpContext readContext,
       final Path destination,
-      final String destinationKey,
-      final boolean addAncestors) throws IOException {
+      final String destinationKey) throws IOException {
     long len = srcAttributes.getLen();
-    CopyResult copyResult;
     try (DurationInfo ignored = new DurationInfo(LOG, false,
         "Copy file from %s to %s (length=%d)", srcKey, destinationKey, len)) {
-      copyResult = callbacks.copyFile(srcKey, destinationKey,
+      callbacks.copyFile(srcKey, destinationKey,
           srcAttributes, readContext);
     }
-    if (objectRepresentsDirectory(srcKey)) {
-      renameTracker.directoryMarkerCopied(
-          sourceFile,
-          destination,
-          addAncestors);
-    } else {
-      S3ObjectAttributes destAttributes = new S3ObjectAttributes(
-          destination,
-          copyResult,
-          srcAttributes.getServerSideEncryptionAlgorithm(),
-          srcAttributes.getServerSideEncryptionKey(),
-          len);
-      renameTracker.fileCopied(
-          sourceFile,
-          srcAttributes,
-          destAttributes,
-          destination,
-          blocksize,
-          addAncestors);
-    }
     return destination;
   }
 
   /**
-   * Remove source objects and update the metastore by way of
-   * the rename tracker.
+   * Remove source objects.
    * @param keys list of keys to delete
-   * @param paths list of paths matching the keys to delete 1:1.
    * @throws IOException failure
    */
   @Retries.RetryTranslated
   private void removeSourceObjects(
-      final List<DeleteObjectsRequest.KeyVersion> keys,
-      final List<Path> paths)
+      final List<DeleteObjectsRequest.KeyVersion> keys)
       throws IOException {
-    List<Path> undeletedObjects = new ArrayList<>();
-    try {
-      // remove the keys
-
-      // list what is being deleted for the interest of anyone
-      // who is trying to debug why objects are no longer there.
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Initiating delete operation for {} objects", keys.size());
-        for (DeleteObjectsRequest.KeyVersion key : keys) {
-          LOG.debug(" {} {}", key.getKey(),
-              key.getVersion() != null ? key.getVersion() : "");
-        }
+    // remove the keys
+
+    // list what is being deleted for the interest of anyone
+    // who is trying to debug why objects are no longer there.
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Initiating delete operation for {} objects", keys.size());
+      for (DeleteObjectsRequest.KeyVersion key : keys) {
+        LOG.debug(" {} {}", key.getKey(),
+            key.getVersion() != null ? key.getVersion() : "");
       }
-      // this will update the metastore on a failure, but on
-      // a successful operation leaves the store as is.
-      callbacks.removeKeys(
-          keys,
-          false,
-          undeletedObjects,
-          renameTracker.getOperationState(),
-          true);
-      // and clear the list.
-    } catch (AmazonClientException | IOException e) {
-      // Failed.
-      // Notify the rename tracker.
-      // removeKeys will have already purged the metastore of
-      // all keys it has known to delete; this is just a final
-      // bit of housekeeping and a chance to tune exception
-      // reporting.
-      // The returned IOE is rethrown.
-      throw renameTracker.deleteFailed(e, paths, undeletedObjects);
     }
-    renameTracker.sourceObjectsDeleted(paths);
+
+    Invoker.once("rename " + sourcePath + " to " + destPath,
+        sourcePath.toString(), () ->
+            callbacks.removeKeys(
+                keys,
+                false
+            ));
   }
 
   /**
@@ -724,4 +610,22 @@ private String maybeAddTrailingSlash(String key) {
     }
   }
 
+  /**
+   * Convert a passed in exception (expected to be an IOE or AWS exception)
+   * into an IOException.
+   * @param ex exception caught
+   * @return the exception to throw in the failure handler.
+   */
+  protected IOException convertToIOException(final Exception ex) {
+    if (ex instanceof IOException) {
+      return (IOException) ex;
+    } else if (ex instanceof SdkBaseException) {
+      return translateException("rename " + sourcePath + " to " + destPath,
+          sourcePath.toString(),
+          (SdkBaseException) ex);
+    } else {
+      // should never happen, but for completeness
+      return new IOException(ex);
+    }
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
index db4a6090e328b..ce11df0383929 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
@@ -23,7 +23,9 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import javax.annotation.Nullable;
 
 import com.amazonaws.AmazonWebServiceRequest;
 import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
@@ -46,8 +48,9 @@
 import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
 import com.amazonaws.services.s3.model.SSECustomerKey;
 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
+import com.amazonaws.services.s3.model.StorageClass;
 import com.amazonaws.services.s3.model.UploadPartRequest;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -60,8 +63,8 @@
 
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 /**
  * The standard implementation of the request factory.
@@ -106,13 +109,6 @@ public class RequestFactoryImpl implements RequestFactory {
    */
   private final long multipartPartCountLimit;
 
-  /**
-   * Requester Pays.
-   * This is to be wired up in a PR with its
-   * own tests and docs.
-   */
-  private final boolean requesterPays;
-
   /**
    * Callback to prepare requests.
    */
@@ -123,6 +119,11 @@ public class RequestFactoryImpl implements RequestFactory {
    */
   private final String contentEncoding;
 
+  /**
+   * Storage class.
+   */
+  private final StorageClass storageClass;
+
   /**
    * Constructor.
    * @param builder builder with all the configuration.
@@ -133,9 +134,9 @@ protected RequestFactoryImpl(
     this.cannedACL = builder.cannedACL;
     this.encryptionSecrets = builder.encryptionSecrets;
     this.multipartPartCountLimit = builder.multipartPartCountLimit;
-    this.requesterPays = builder.requesterPays;
     this.requestPreparer = builder.requestPreparer;
     this.contentEncoding = builder.contentEncoding;
+    this.storageClass = builder.storageClass;
   }
 
   /**
@@ -208,6 +209,15 @@ public String getContentEncoding() {
     return contentEncoding;
   }
 
+  /**
+   * Get the object storage class, return null if none.
+   * @return storage class
+   */
+  @Override
+  public StorageClass getStorageClass() {
+    return storageClass;
+  }
+
   /**
    * Sets server side encryption parameters to the part upload
    * request when encryption is enabled.
@@ -351,20 +361,27 @@ protected void copyEncryptionParameters(
   }
   /**
    * Create a putObject request.
-   * Adds the ACL and metadata
+   * Adds the ACL, storage class and metadata
    * @param key key of object
    * @param metadata metadata header
+   * @param options options for the request, including headers
    * @param srcfile source file
    * @return the request
    */
   @Override
   public PutObjectRequest newPutObjectRequest(String key,
-      ObjectMetadata metadata, File srcfile) {
+      ObjectMetadata metadata,
+      final PutObjectOptions options,
+      File srcfile) {
     Preconditions.checkNotNull(srcfile);
     PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key,
         srcfile);
+    maybeSetMetadata(options, metadata);
     setOptionalPutRequestParameters(putObjectRequest);
     putObjectRequest.setCannedAcl(cannedACL);
+    if (storageClass != null) {
+      putObjectRequest.setStorageClass(storageClass);
+    }
     putObjectRequest.setMetadata(metadata);
     return prepareRequest(putObjectRequest);
   }
@@ -375,19 +392,25 @@ public PutObjectRequest newPutObjectRequest(String key,
    * operation.
    * @param key key of object
    * @param metadata metadata header
+   * @param options options for the request
    * @param inputStream source data.
    * @return the request
    */
   @Override
   public PutObjectRequest newPutObjectRequest(String key,
       ObjectMetadata metadata,
+      @Nullable final PutObjectOptions options,
       InputStream inputStream) {
     Preconditions.checkNotNull(inputStream);
     Preconditions.checkArgument(isNotEmpty(key), "Null/empty key");
+    maybeSetMetadata(options, metadata);
     PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key,
         inputStream, metadata);
     setOptionalPutRequestParameters(putObjectRequest);
     putObjectRequest.setCannedAcl(cannedACL);
+    if (storageClass != null) {
+      putObjectRequest.setStorageClass(storageClass);
+    }
     return prepareRequest(putObjectRequest);
   }
 
@@ -396,19 +419,22 @@ public PutObjectRequest newDirectoryMarkerRequest(String directory) {
     String key = directory.endsWith("/")
         ? directory
         : (directory + "/");
-    // an input stream which is laways empty
-    final InputStream im = new InputStream() {
+    // an input stream which is always empty
+    final InputStream inputStream = new InputStream() {
       @Override
       public int read() throws IOException {
         return -1;
       }
     };
     // preparation happens in here
-    final ObjectMetadata md = createObjectMetadata(0L, true);
-    md.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY);
-    PutObjectRequest putObjectRequest =
-        newPutObjectRequest(key, md, im);
-    return putObjectRequest;
+    final ObjectMetadata metadata = createObjectMetadata(0L, true);
+    metadata.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY);
+
+    PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key,
+        inputStream, metadata);
+    setOptionalPutRequestParameters(putObjectRequest);
+    putObjectRequest.setCannedAcl(cannedACL);
+    return prepareRequest(putObjectRequest);
   }
 
   @Override
@@ -433,12 +459,18 @@ public AbortMultipartUploadRequest newAbortMultipartUploadRequest(
 
   @Override
   public InitiateMultipartUploadRequest newMultipartUploadRequest(
-      String destKey) {
+      final String destKey,
+      @Nullable final PutObjectOptions options) {
+    final ObjectMetadata objectMetadata = newObjectMetadata(-1);
+    maybeSetMetadata(options, objectMetadata);
     final InitiateMultipartUploadRequest initiateMPURequest =
         new InitiateMultipartUploadRequest(getBucket(),
             destKey,
-            newObjectMetadata(-1));
+            objectMetadata);
     initiateMPURequest.setCannedACL(getCannedACL());
+    if (getStorageClass() != null) {
+      initiateMPURequest.withStorageClass(getStorageClass());
+    }
     setOptionalMultipartUploadRequestParameters(initiateMPURequest);
     return prepareRequest(initiateMPURequest);
   }
@@ -575,12 +607,11 @@ public DeleteObjectRequest newDeleteObjectRequest(String key) {
 
   @Override
   public DeleteObjectsRequest newBulkDeleteRequest(
-      List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean quiet) {
+          List<DeleteObjectsRequest.KeyVersion> keysToDelete) {
     return prepareRequest(
         new DeleteObjectsRequest(bucket)
             .withKeys(keysToDelete)
-            .withQuiet(quiet));
+            .withQuiet(true));
   }
 
   @Override
@@ -588,6 +619,23 @@ public void setEncryptionSecrets(final EncryptionSecrets secrets) {
     encryptionSecrets = secrets;
   }
 
+  /**
+   * Set the metadata from the options if the options are not
+   * null and the metadata contains headers.
+   * @param options options for the request
+   * @param objectMetadata metadata to patch
+   */
+  private void maybeSetMetadata(
+      @Nullable PutObjectOptions options,
+      final ObjectMetadata objectMetadata) {
+    if (options != null) {
+      Map<String, String> headers = options.getHeaders();
+      if (headers != null) {
+        objectMetadata.setUserMetadata(headers);
+      }
+    }
+  }
+
   /**
    * Create a builder.
    * @return new builder.
@@ -616,12 +664,14 @@ public static final class RequestFactoryBuilder {
      */
     private CannedAccessControlList cannedACL = null;
 
-    /** Requester Pays flag. */
-    private boolean requesterPays = false;
-
     /** Content Encoding. */
     private String contentEncoding;
 
+    /**
+     * Storage class.
+     */
+    private StorageClass storageClass;
+
     /**
      * Multipart limit.
      */
@@ -653,6 +703,16 @@ public RequestFactoryBuilder withContentEncoding(final String value) {
       return this;
     }
 
+    /**
+     * Storage class.
+     * @param value new value
+     * @return the builder
+     */
+    public RequestFactoryBuilder withStorageClass(final StorageClass value) {
+      storageClass = value;
+      return this;
+    }
+
     /**
      * Target bucket.
      * @param value new value
@@ -685,17 +745,6 @@ public RequestFactoryBuilder withCannedACL(
       return this;
     }
 
-    /**
-     * Requester Pays flag.
-     * @param value new value
-     * @return the builder
-     */
-    public RequestFactoryBuilder withRequesterPays(
-        final boolean value) {
-      requesterPays = value;
-      return this;
-    }
-
     /**
      * Multipart limit.
      * @param value new value
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java
index 545cf0f9bb6e1..3a6a04e5e711c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java
@@ -38,13 +38,12 @@
 import com.amazonaws.services.s3.model.PartETag;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import com.amazonaws.services.s3.model.UploadPartResult;
-import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.BBPartHandle;
 import org.apache.hadoop.fs.BBUploadHandle;
 import org.apache.hadoop.fs.PartHandle;
@@ -55,8 +54,8 @@
 import org.apache.hadoop.fs.impl.AbstractMultipartUploader;
 import org.apache.hadoop.fs.s3a.WriteOperations;
 import org.apache.hadoop.fs.s3a.statistics.S3AMultipartUploaderStatistics;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToString;
 
@@ -80,16 +79,6 @@ class S3AMultipartUploader extends AbstractMultipartUploader {
 
   private final S3AMultipartUploaderStatistics statistics;
 
-  /**
-   * Bulk state; demand created and then retained.
-   */
-  private BulkOperationState operationState;
-
-  /**
-   * Was an operation state requested but not returned?
-   */
-  private boolean noOperationState;
-
   /**
    * Instatiate; this is called by the builder.
    * @param builder builder
@@ -109,14 +98,6 @@ class S3AMultipartUploader extends AbstractMultipartUploader {
     this.statistics = Objects.requireNonNull(statistics);
   }
 
-  @Override
-  public void close() throws IOException {
-    if (operationState != null) {
-      operationState.close();
-    }
-    super.close();
-  }
-
   @Override
   public IOStatistics getIOStatistics() {
     return statistics.getIOStatistics();
@@ -133,22 +114,6 @@ public String toString() {
     return sb.toString();
   }
 
-  /**
-   * Retrieve the operation state; create one on demand if needed
-   * <i>and there has been no unsuccessful attempt to create one.</i>
-   * @return an active operation state.
-   * @throws IOException failure
-   */
-  private synchronized BulkOperationState retrieveOperationState()
-      throws IOException {
-    if (operationState == null && !noOperationState) {
-      operationState = writeOperations.initiateOperation(getBasePath(),
-          BulkOperationState.OperationType.Upload);
-      noOperationState = operationState != null;
-    }
-    return operationState;
-  }
-
   @Override
   public CompletableFuture<UploadHandle> startUpload(
       final Path filePath)
@@ -158,7 +123,8 @@ public CompletableFuture<UploadHandle> startUpload(
     String key = context.pathToKey(dest);
     return context.submit(new CompletableFuture<>(),
         () -> {
-          String uploadId = writeOperations.initiateMultiPartUpload(key);
+          String uploadId = writeOperations.initiateMultiPartUpload(key,
+              PutObjectOptions.keepingDirs());
           statistics.uploadStarted();
           return BBUploadHandle.from(ByteBuffer.wrap(
               uploadId.getBytes(Charsets.UTF_8)));
@@ -238,7 +204,6 @@ public CompletableFuture<PathHandle> complete(
         "Duplicate PartHandles");
 
     // retrieve/create operation state for scalability of completion.
-    final BulkOperationState state = retrieveOperationState();
     long finalLen = totalLength;
     return context.submit(new CompletableFuture<>(),
         () -> {
@@ -247,8 +212,8 @@ public CompletableFuture<PathHandle> complete(
                   key,
                   uploadIdStr,
                   eTags,
-                  finalLen,
-                  state);
+                  finalLen
+              );
 
           byte[] eTag = result.getETag().getBytes(Charsets.UTF_8);
           statistics.uploadCompleted();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java
new file mode 100644
index 0000000000000..b566f9ad42765
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.Closeable;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.Nullable;
+
+import com.amazonaws.internal.SdkFilterInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+
+/**
+ * Drains/aborts s3 or other AWS SDK streams.
+ * It is callable so can be passed directly to a submitter
+ * for async invocation.
+ * A request object may be passed in; it will be implicitly
+ * cached until this object is GCd.
+ * This is because in some versions of the AWS SDK, the S3Object
+ * has a finalize() method which releases the http connection,
+ * even when the stream is still open.
+ * See HADOOP-17338 for details.
+ */
+public class SDKStreamDrainer implements CallableRaisingIOE<Boolean> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      SDKStreamDrainer.class);
+
+  /**
+   * URI for log messages.
+   */
+  private final String uri;
+
+  /**
+   * Request object; usually S3Object
+   * Never used, but needed to keep the http connection
+   * open long enough for draining to take place.
+   */
+  @Nullable
+  private final Closeable requestObject;
+
+  /**
+   * Stream from the {@link #requestObject} for draining and closing.
+   */
+  private final SdkFilterInputStream sdkStream;
+
+  /**
+   * Should the request be aborted?
+   */
+  private final boolean shouldAbort;
+
+  /**
+   * How many bytes remaining?
+   * This is decremented as the stream is
+   * drained;
+   * If the stream finished before the expected
+   * remaining value was read, this will show how many
+   * bytes were still expected.
+   */
+  private int remaining;
+
+  /**
+   * Statistics to update with the duration.
+   */
+  private final S3AInputStreamStatistics streamStatistics;
+
+  /**
+   * Reason? for log messages.
+   */
+  private final String reason;
+
+  /**
+   * Has the operation executed yet?
+   */
+  private final AtomicBoolean executed = new AtomicBoolean(false);
+
+  /**
+   * Any exception caught during execution.
+   */
+  private Exception thrown;
+
+  /**
+   * Was the stream aborted?
+   */
+  private boolean aborted;
+
+  /**
+   * how many bytes were drained?
+   */
+  private int drained = 0;
+
+  /**
+   * Prepare to drain the stream.
+   * @param uri URI for messages
+   * @param requestObject http request object; needed to avoid GC issues.
+   * @param sdkStream stream to close.
+   * @param shouldAbort force an abort; used if explicitly requested.
+   * @param streamStatistics stats to update
+   * @param reason reason for stream being closed; used in messages
+   * @param remaining remaining bytes
+   */
+  public SDKStreamDrainer(final String uri,
+      @Nullable final Closeable requestObject,
+      final SdkFilterInputStream sdkStream,
+      final boolean shouldAbort,
+      final int remaining,
+      final S3AInputStreamStatistics streamStatistics,
+      final String reason) {
+    this.uri = uri;
+    this.requestObject = requestObject;
+    this.sdkStream = requireNonNull(sdkStream);
+    this.shouldAbort = shouldAbort;
+    this.remaining = remaining;
+    this.streamStatistics = requireNonNull(streamStatistics);
+    this.reason = reason;
+  }
+
+  /**
+   * drain the stream. This method is intended to be
+   * used directly or asynchronously, and measures the
+   * duration of the operation in the stream statistics.
+   * @return was the stream aborted?
+   */
+  @Override
+  public Boolean apply() {
+    try {
+      Boolean outcome = invokeTrackingDuration(
+          streamStatistics.initiateInnerStreamClose(shouldAbort),
+          this::drainOrAbortHttpStream);
+      aborted = outcome;
+      return outcome;
+    } catch (Exception e) {
+      thrown = e;
+      return aborted;
+    }
+  }
+
+  /**
+   * Apply, raising any exception.
+   * For testing.
+   * @return the outcome.
+   * @throws Exception anything raised.
+   */
+  @VisibleForTesting
+  boolean applyRaisingException() throws Exception {
+    Boolean outcome = apply();
+    if (thrown != null) {
+      throw thrown;
+    }
+    return outcome;
+  }
+
+  /**
+   * Drain or abort the inner stream.
+   * Exceptions are saved then swallowed.
+   * If a close() is attempted and fails, the operation escalates to
+   * an abort.
+   * @return true if the stream was aborted.
+   */
+  private boolean drainOrAbortHttpStream() {
+    if (executed.getAndSet(true)) {
+      throw new IllegalStateException(
+          "duplicate invocation of drain operation");
+    }
+    boolean executeAbort = shouldAbort;
+    LOG.debug("drain or abort reason {} remaining={} abort={}",
+        reason, remaining, executeAbort);
+
+    if (!executeAbort) {
+      try {
+        // clean close. This will read to the end of the stream,
+        // so, while cleaner, can be pathological on a multi-GB object
+
+        if (remaining > 0) {
+          // explicitly drain the stream
+          LOG.debug("draining {} bytes", remaining);
+          drained = 0;
+          int size = Math.min(remaining, DRAIN_BUFFER_SIZE);
+          byte[] buffer = new byte[size];
+          // read the data; bail out early if
+          // the connection breaks.
+          // this may be a bit overaggressive on buffer underflow.
+          while (remaining > 0) {
+            final int count = sdkStream.read(buffer);
+            LOG.debug("read {} bytes", count);
+            if (count <= 0) {
+              // no more data is left
+              break;
+            }
+            drained += count;
+            remaining -= count;
+          }
+          LOG.debug("Drained stream of {} bytes", drained);
+        }
+
+        if (remaining != 0) {
+          // fewer bytes than expected came back; not treating as a
+          // reason to escalate to an abort().
+          // just log.
+          LOG.debug("drained fewer bytes than expected; {} remaining",
+              remaining);
+        }
+
+        // now close it.
+        // if there is still data in the stream, the SDK
+        // will warn and escalate to an abort itself.
+        LOG.debug("Closing stream");
+        sdkStream.close();
+
+        cleanupWithLogger(LOG, requestObject);
+        // this MUST come after the close, so that if the IO operations fail
+        // and an abort is triggered, the initial attempt's statistics
+        // aren't collected.
+        streamStatistics.streamClose(false, drained);
+        return false;
+      } catch (Exception e) {
+        // exception escalates to an abort
+        LOG.debug("When closing {} stream for {}, will abort the stream",
+            uri, reason, e);
+        thrown = e;
+      }
+    }
+    // Abort, rather than just close, the underlying stream. Otherwise, the
+    // remaining object payload is read from S3 while closing the stream.
+    LOG.debug("Aborting stream {}", uri);
+    try {
+      sdkStream.abort();
+    } catch (Exception e) {
+      LOG.warn("When aborting {} stream after failing to close it for {}",
+          uri, reason, e);
+      thrown = e;
+    } finally {
+      cleanupWithLogger(LOG, requestObject);
+    }
+
+    streamStatistics.streamClose(true, remaining);
+    LOG.debug("Stream {} aborted: {}; remaining={}",
+        uri, reason, remaining);
+    return true;
+  }
+
+  public String getUri() {
+    return uri;
+  }
+
+  public Object getRequestObject() {
+    return requestObject;
+  }
+
+  public SdkFilterInputStream getSdkStream() {
+    return sdkStream;
+  }
+
+  public boolean shouldAbort() {
+    return shouldAbort;
+  }
+
+  public int getRemaining() {
+    return remaining;
+  }
+
+  public S3AInputStreamStatistics getStreamStatistics() {
+    return streamStatistics;
+  }
+
+  public String getReason() {
+    return reason;
+  }
+
+  public boolean executed() {
+    return executed.get();
+  }
+
+  public Exception getThrown() {
+    return thrown;
+  }
+
+  public int getDrained() {
+    return drained;
+  }
+
+  public boolean aborted() {
+    return aborted;
+  }
+
+  @Override
+  public String toString() {
+    return "SDKStreamDrainer{" +
+        "uri='" + uri + '\'' +
+        ", reason='" + reason + '\'' +
+        ", shouldAbort=" + shouldAbort +
+        ", remaining=" + remaining +
+        ", executed=" + executed.get() +
+        ", aborted=" + aborted +
+        ", inner=" + sdkStream +
+        ", thrown=" + thrown +
+        '}';
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java
index d60e1a2cd289e..4b8a28f3e7bb0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java
@@ -40,8 +40,6 @@
 import org.apache.hadoop.fs.s3a.S3AStorageStatistics;
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
 import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
@@ -111,21 +109,8 @@ public class StoreContext implements ActiveThreadSpanSource<AuditSpan> {
   /** List algorithm. */
   private final boolean useListV1;
 
-  /**
-   * To allow this context to be passed down to the metastore, this field
-   * wll be null until initialized.
-   */
-  private final MetadataStore metadataStore;
-
   private final ContextAccessors contextAccessors;
 
-  /**
-   * Source of time.
-   */
-
-  /** Time source for S3Guard TTLs. */
-  private final ITtlTimeProvider timeProvider;
-
   /** Operation Auditor. */
   private final AuditSpanSource<AuditSpanS3A> auditor;
 
@@ -149,10 +134,8 @@ public class StoreContext implements ActiveThreadSpanSource<AuditSpan> {
       final S3AInputPolicy inputPolicy,
       final ChangeDetectionPolicy changeDetectionPolicy,
       final boolean multiObjectDeleteEnabled,
-      final MetadataStore metadataStore,
       final boolean useListV1,
       final ContextAccessors contextAccessors,
-      final ITtlTimeProvider timeProvider,
       final AuditSpanSource<AuditSpanS3A> auditor,
       final boolean isCSEEnabled) {
     this.fsURI = fsURI;
@@ -171,10 +154,8 @@ public class StoreContext implements ActiveThreadSpanSource<AuditSpan> {
     this.inputPolicy = inputPolicy;
     this.changeDetectionPolicy = changeDetectionPolicy;
     this.multiObjectDeleteEnabled = multiObjectDeleteEnabled;
-    this.metadataStore = metadataStore;
     this.useListV1 = useListV1;
     this.contextAccessors = contextAccessors;
-    this.timeProvider = timeProvider;
     this.auditor = auditor;
     this.isCSEEnabled = isCSEEnabled;
   }
@@ -224,10 +205,6 @@ public boolean isMultiObjectDeleteEnabled() {
     return multiObjectDeleteEnabled;
   }
 
-  public MetadataStore getMetadataStore() {
-    return metadataStore;
-  }
-
   public boolean isUseListV1() {
     return useListV1;
   }
@@ -368,14 +345,6 @@ public String getBucketLocation() throws IOException {
     return contextAccessors.getBucketLocation();
   }
 
-  /**
-   * Get the time provider.
-   * @return the time source.
-   */
-  public ITtlTimeProvider getTimeProvider() {
-    return timeProvider;
-  }
-
   /**
    * Build the full S3 key for a request from the status entry,
    * possibly adding a "/" if it represents directory and it does
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java
index d4021a731cca2..cff38b9fc4b7d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java
@@ -27,8 +27,6 @@
 import org.apache.hadoop.fs.s3a.S3AStorageStatistics;
 import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
 import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
 import org.apache.hadoop.security.UserGroupInformation;
 
@@ -63,14 +61,10 @@ public class StoreContextBuilder {
 
   private boolean multiObjectDeleteEnabled = true;
 
-  private MetadataStore metadataStore;
-
   private boolean useListV1 = false;
 
   private ContextAccessors contextAccessors;
 
-  private ITtlTimeProvider timeProvider;
-
   private AuditSpanSource<AuditSpanS3A> auditor;
 
   private boolean isCSEEnabled;
@@ -147,12 +141,6 @@ public StoreContextBuilder setMultiObjectDeleteEnabled(
     return this;
   }
 
-  public StoreContextBuilder setMetadataStore(
-      final MetadataStore store) {
-    this.metadataStore = store;
-    return this;
-  }
-
   public StoreContextBuilder setUseListV1(
       final boolean useV1) {
     this.useListV1 = useV1;
@@ -165,12 +153,6 @@ public StoreContextBuilder setContextAccessors(
     return this;
   }
 
-  public StoreContextBuilder setTimeProvider(
-      final ITtlTimeProvider provider) {
-    this.timeProvider = provider;
-    return this;
-  }
-
   /**
    * Set builder value.
    * @param value new value
@@ -193,7 +175,6 @@ public StoreContextBuilder setEnableCSE(
     return this;
   }
 
-  @SuppressWarnings("deprecation")
   public StoreContext build() {
     return new StoreContext(fsURI,
         bucket,
@@ -208,10 +189,8 @@ public StoreContext build() {
         inputPolicy,
         changeDetectionPolicy,
         multiObjectDeleteEnabled,
-        metadataStore,
         useListV1,
         contextAccessors,
-        timeProvider,
         auditor,
         isCSEEnabled);
   }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java
new file mode 100644
index 0000000000000..3aa8ad270eedd
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME;
+
+/**
+ * This class provides utility methods required for migrating S3A to AWS Java SDK V2.
+ * For more information on the upgrade, see HADOOP-18073.
+ */
+public final class V2Migration {
+
+  private V2Migration() { }
+
+  public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME);
+
+  private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
+  private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
+  private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
+  private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
+  private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA =
+      new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
+
+  /**
+   * Warns on an AWS V1 credential provider being referenced directly.
+   * @param name name of the credential provider
+   */
+  public static void v1ProviderReferenced(String name) {
+    WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn(
+        "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential "
+            + "providers will be removed once S3A is upgraded to SDK V2", name);
+  }
+
+  /**
+   * Warns on the v1 s3 client being requested.
+   */
+  public static void v1S3ClientRequested() {
+    WARN_ON_GET_S3_CLIENT.warn(
+        "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2");
+  }
+
+  /**
+   * Warns when v1 credential providers are used with delegation tokens.
+   */
+  public static void v1DelegationTokenCredentialProvidersUsed() {
+    WARN_ON_DELEGATION_TOKENS.warn(
+        "The credential provider interface has changed in AWS SDK V2, custom credential "
+            + "providers used in delegation tokens binding classes will need to be updated once "
+            + "S3A is upgraded to SDK V2");
+  }
+
+  /**
+   * Warns on use of custom signers.
+   */
+  public static void v1CustomSignerUsed() {
+    WARN_OF_CUSTOM_SIGNER.warn(
+        "The signer interface has changed in AWS SDK V2, custom signers will need to be updated "
+            + "once S3A is upgraded to SDK V2");
+  }
+
+  /**
+   * Warns on use of getObjectMetadata.
+   */
+  public static void v1GetObjectMetadataCalled() {
+    WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response "
+        + "will be changed as part of upgrading S3A to AWS SDK V2");
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java
new file mode 100644
index 0000000000000..adbb06bf14237
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.BlockManager;
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+
+/**
+ * Provides read access to S3 file one block at a time.
+ *
+ * A naive implementation of a {@code BlockManager} that provides no prefetching or caching.
+ * Useful baseline for comparing performance difference against {@code S3ACachingBlockManager}.
+ */
+public class S3ABlockManager extends BlockManager {
+
+  /**
+   * Reader that reads from S3 file.
+   */
+  private final S3ARemoteObjectReader reader;
+
+  /**
+   * Constructs an instance of {@code S3ABlockManager}.
+   *
+   * @param reader a reader that reads from S3 file.
+   * @param blockData information about each block of the S3 file.
+   *
+   * @throws IllegalArgumentException if reader is null.
+   * @throws IllegalArgumentException if blockData is null.
+   */
+  public S3ABlockManager(S3ARemoteObjectReader reader, BlockData blockData) {
+    super(blockData);
+
+    Validate.checkNotNull(reader, "reader");
+
+    this.reader = reader;
+  }
+
+  /**
+   * Reads into the given {@code buffer} {@code size} bytes from the underlying file
+   * starting at {@code startOffset}.
+   *
+   * @param buffer the buffer to read data in to.
+   * @param startOffset the offset at which reading starts.
+   * @param size the number bytes to read.
+   * @return number of bytes read.
+   */
+  @Override
+  public int read(ByteBuffer buffer, long startOffset, int size)
+      throws IOException {
+    return reader.read(buffer, startOffset, size);
+  }
+
+  @Override
+  public void close() {
+    reader.close();
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java
new file mode 100644
index 0000000000000..f82786659da05
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.CachingBlockManager;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+
+/**
+ * Provides access to S3 file one block at a time.
+ */
+public class S3ACachingBlockManager extends CachingBlockManager {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3ACachingBlockManager.class);
+
+  /**
+   * Reader that reads from S3 file.
+   */
+  private final S3ARemoteObjectReader reader;
+
+  /**
+   * Constructs an instance of a {@code S3ACachingBlockManager}.
+   *
+   * @param futurePool asynchronous tasks are performed in this pool.
+   * @param reader reader that reads from S3 file.
+   * @param blockData information about each block of the S3 file.
+   * @param bufferPoolSize size of the in-memory cache in terms of number of blocks.
+   * @param streamStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if reader is null.
+   */
+  public S3ACachingBlockManager(
+      ExecutorServiceFuturePool futurePool,
+      S3ARemoteObjectReader reader,
+      BlockData blockData,
+      int bufferPoolSize,
+      S3AInputStreamStatistics streamStatistics) {
+    super(futurePool, blockData, bufferPoolSize, streamStatistics);
+
+    Validate.checkNotNull(reader, "reader");
+
+    this.reader = reader;
+  }
+
+  protected S3ARemoteObjectReader getReader() {
+    return this.reader;
+  }
+
+  /**
+   * Reads into the given {@code buffer} {@code size} bytes from the underlying file
+   * starting at {@code startOffset}.
+   *
+   * @param buffer the buffer to read data in to.
+   * @param startOffset the offset at which reading starts.
+   * @param size the number bytes to read.
+   * @return number of bytes read.
+   */
+  @Override
+  public int read(ByteBuffer buffer, long startOffset, int size)
+      throws IOException {
+    return this.reader.read(buffer, startOffset, size);
+  }
+
+  @Override
+  public synchronized void close() {
+    this.reader.close();
+
+    super.close();
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java
new file mode 100644
index 0000000000000..f9ee4e412fc4f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.BlockManager;
+import org.apache.hadoop.fs.impl.prefetch.BufferData;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.impl.prefetch.FilePosition;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration;
+
+/**
+ * Provides an {@code InputStream} that allows reading from an S3 file.
+ * Prefetched blocks are cached to local disk if a seek away from the
+ * current block is issued.
+ */
+public class S3ACachingInputStream extends S3ARemoteInputStream {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3ACachingInputStream.class);
+
+  /**
+   * Number of blocks queued for prefching.
+   */
+  private final int numBlocksToPrefetch;
+
+  private final BlockManager blockManager;
+
+  /**
+   * Initializes a new instance of the {@code S3ACachingInputStream} class.
+   *
+   * @param context read-specific operation context.
+   * @param s3Attributes attributes of the S3 object being read.
+   * @param client callbacks used for interacting with the underlying S3 client.
+   * @param streamStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if context is null.
+   * @throws IllegalArgumentException if s3Attributes is null.
+   * @throws IllegalArgumentException if client is null.
+   */
+  public S3ACachingInputStream(
+      S3AReadOpContext context,
+      S3ObjectAttributes s3Attributes,
+      S3AInputStream.InputStreamCallbacks client,
+      S3AInputStreamStatistics streamStatistics) {
+    super(context, s3Attributes, client, streamStatistics);
+
+    this.numBlocksToPrefetch = this.getContext().getPrefetchBlockCount();
+    int bufferPoolSize = this.numBlocksToPrefetch + 1;
+    this.blockManager = this.createBlockManager(
+        this.getContext().getFuturePool(),
+        this.getReader(),
+        this.getBlockData(),
+        bufferPoolSize);
+    int fileSize = (int) s3Attributes.getLen();
+    LOG.debug("Created caching input stream for {} (size = {})", this.getName(),
+        fileSize);
+  }
+
+  @Override
+  public void close() throws IOException {
+    // Close the BlockManager first, cancelling active prefetches,
+    // deleting cached files and freeing memory used by buffer pool.
+    blockManager.close();
+    super.close();
+    LOG.info("closed: {}", getName());
+  }
+
+  @Override
+  protected boolean ensureCurrentBuffer() throws IOException {
+    if (isClosed()) {
+      return false;
+    }
+
+    long readPos = getNextReadPos();
+    if (!getBlockData().isValidOffset(readPos)) {
+      return false;
+    }
+
+    // Determine whether this is an out of order read.
+    FilePosition filePosition = getFilePosition();
+    boolean outOfOrderRead = !filePosition.setAbsolute(readPos);
+
+    if (!outOfOrderRead && filePosition.buffer().hasRemaining()) {
+      // Use the current buffer.
+      return true;
+    }
+
+    if (filePosition.isValid()) {
+      // We are jumping out of the current buffer. There are two cases to consider:
+      if (filePosition.bufferFullyRead()) {
+        // This buffer was fully read:
+        // it is very unlikely that this buffer will be needed again;
+        // therefore we release the buffer without caching.
+        blockManager.release(filePosition.data());
+      } else {
+        // We will likely need this buffer again (as observed empirically for Parquet)
+        // therefore we issue an async request to cache this buffer.
+        blockManager.requestCaching(filePosition.data());
+      }
+      filePosition.invalidate();
+    }
+
+    int prefetchCount;
+    if (outOfOrderRead) {
+      LOG.debug("lazy-seek({})", getOffsetStr(readPos));
+      blockManager.cancelPrefetches();
+
+      // We prefetch only 1 block immediately after a seek operation.
+      prefetchCount = 1;
+    } else {
+      // A sequential read results in a prefetch.
+      prefetchCount = numBlocksToPrefetch;
+    }
+
+    int toBlockNumber = getBlockData().getBlockNumber(readPos);
+    long startOffset = getBlockData().getStartOffset(toBlockNumber);
+
+    for (int i = 1; i <= prefetchCount; i++) {
+      int b = toBlockNumber + i;
+      if (b < getBlockData().getNumBlocks()) {
+        blockManager.requestPrefetch(b);
+      }
+    }
+
+    BufferData data = invokeTrackingDuration(
+        getS3AStreamStatistics()
+            .trackDuration(STREAM_READ_BLOCK_ACQUIRE_AND_READ),
+        () -> blockManager.get(toBlockNumber));
+
+    filePosition.setData(data, startOffset, readPos);
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    if (isClosed()) {
+      return "closed";
+    }
+
+    StringBuilder sb = new StringBuilder();
+    sb.append(String.format("%s%n", super.toString()));
+    sb.append(blockManager.toString());
+    return sb.toString();
+  }
+
+  protected BlockManager createBlockManager(
+      ExecutorServiceFuturePool futurePool,
+      S3ARemoteObjectReader reader,
+      BlockData blockData,
+      int bufferPoolSize) {
+    return new S3ACachingBlockManager(futurePool, reader, blockData,
+        bufferPoolSize,
+        getS3AStreamStatistics());
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java
new file mode 100644
index 0000000000000..e8bfe946f4abf
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.impl.prefetch.BufferData;
+import org.apache.hadoop.fs.impl.prefetch.FilePosition;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+
+/**
+ * Provides an {@code InputStream} that allows reading from an S3 file.
+ * The entire file is read into memory before reads can begin.
+ *
+ * Use of this class is recommended only for small files that can fit
+ * entirely in memory.
+ */
+public class S3AInMemoryInputStream extends S3ARemoteInputStream {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3AInMemoryInputStream.class);
+
+  private ByteBuffer buffer;
+
+  /**
+   * Initializes a new instance of the {@code S3AInMemoryInputStream} class.
+   *
+   * @param context read-specific operation context.
+   * @param s3Attributes attributes of the S3 object being read.
+   * @param client callbacks used for interacting with the underlying S3 client.
+   * @param streamStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if context is null.
+   * @throws IllegalArgumentException if s3Attributes is null.
+   * @throws IllegalArgumentException if client is null.
+   */
+  public S3AInMemoryInputStream(
+      S3AReadOpContext context,
+      S3ObjectAttributes s3Attributes,
+      S3AInputStream.InputStreamCallbacks client,
+      S3AInputStreamStatistics streamStatistics) {
+    super(context, s3Attributes, client, streamStatistics);
+    int fileSize = (int) s3Attributes.getLen();
+    this.buffer = ByteBuffer.allocate(fileSize);
+    LOG.debug("Created in-memory input stream for {} (size = {})",
+        getName(), fileSize);
+  }
+
+  /**
+   * Ensures that a non-empty valid buffer is available for immediate reading.
+   * It returns true when at least one such buffer is available for reading.
+   * It returns false on reaching the end of the stream.
+   *
+   * @return true if at least one such buffer is available for reading, false otherwise.
+   */
+  @Override
+  protected boolean ensureCurrentBuffer() throws IOException {
+    if (isClosed()) {
+      return false;
+    }
+
+    if (getBlockData().getFileSize() == 0) {
+      return false;
+    }
+
+    FilePosition filePosition = getFilePosition();
+    if (filePosition.isValid()) {
+      // Update current position (lazy seek).
+      filePosition.setAbsolute(getNextReadPos());
+    } else {
+      // Read entire file into buffer.
+      buffer.clear();
+      int numBytesRead =
+          getReader().read(buffer, 0, buffer.capacity());
+      if (numBytesRead <= 0) {
+        return false;
+      }
+      BufferData data = new BufferData(0, buffer);
+      filePosition.setData(data, 0, getNextReadPos());
+    }
+
+    return filePosition.buffer().hasRemaining();
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java
new file mode 100644
index 0000000000000..f778f40b74c0a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+
+/**
+ * Enhanced {@code InputStream} for reading from S3.
+ *
+ * This implementation provides improved read throughput by asynchronously prefetching
+ * blocks of configurable size from the underlying S3 file.
+ */
+public class S3APrefetchingInputStream
+    extends FSInputStream
+    implements CanSetReadahead, StreamCapabilities, IOStatisticsSource {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3APrefetchingInputStream.class);
+
+  /**
+   * Underlying input stream used for reading S3 file.
+   */
+  private S3ARemoteInputStream inputStream;
+
+  /**
+   * To be only used by synchronized getPos().
+   */
+  private long lastReadCurrentPos = 0;
+
+  /**
+   * To be only used by getIOStatistics().
+   */
+  private IOStatistics ioStatistics = null;
+
+  /**
+   * To be only used by getS3AStreamStatistics().
+   */
+  private S3AInputStreamStatistics inputStreamStatistics = null;
+
+  /**
+   * Initializes a new instance of the {@code S3APrefetchingInputStream} class.
+   *
+   * @param context read-specific operation context.
+   * @param s3Attributes attributes of the S3 object being read.
+   * @param client callbacks used for interacting with the underlying S3 client.
+   * @param streamStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if context is null.
+   * @throws IllegalArgumentException if s3Attributes is null.
+   * @throws IllegalArgumentException if client is null.
+   */
+  public S3APrefetchingInputStream(
+      S3AReadOpContext context,
+      S3ObjectAttributes s3Attributes,
+      S3AInputStream.InputStreamCallbacks client,
+      S3AInputStreamStatistics streamStatistics) {
+
+    Validate.checkNotNull(context, "context");
+    Validate.checkNotNull(s3Attributes, "s3Attributes");
+    Validate.checkNotNullAndNotEmpty(s3Attributes.getBucket(),
+        "s3Attributes.getBucket()");
+    Validate.checkNotNullAndNotEmpty(s3Attributes.getKey(),
+        "s3Attributes.getKey()");
+    Validate.checkNotNegative(s3Attributes.getLen(), "s3Attributes.getLen()");
+    Validate.checkNotNull(client, "client");
+    Validate.checkNotNull(streamStatistics, "streamStatistics");
+
+    long fileSize = s3Attributes.getLen();
+    if (fileSize <= context.getPrefetchBlockSize()) {
+      LOG.debug("Creating in memory input stream for {}", context.getPath());
+      this.inputStream = new S3AInMemoryInputStream(
+          context,
+          s3Attributes,
+          client,
+          streamStatistics);
+    } else {
+      LOG.debug("Creating in caching input stream for {}", context.getPath());
+      this.inputStream = new S3ACachingInputStream(
+          context,
+          s3Attributes,
+          client,
+          streamStatistics);
+    }
+  }
+
+  /**
+   * Returns the number of bytes available for reading without blocking.
+   *
+   * @return the number of bytes available for reading without blocking.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized int available() throws IOException {
+    throwIfClosed();
+    return inputStream.available();
+  }
+
+  /**
+   * Gets the current position. If the underlying S3 input stream is closed,
+   * it returns last read current position from the underlying steam. If the
+   * current position was never read and the underlying input stream is closed,
+   * this would return 0.
+   *
+   * @return the current position.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized long getPos() throws IOException {
+    if (!isClosed()) {
+      lastReadCurrentPos = inputStream.getPos();
+    }
+    return lastReadCurrentPos;
+  }
+
+  /**
+   * Reads and returns one byte from this stream.
+   *
+   * @return the next byte from this stream.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized int read() throws IOException {
+    throwIfClosed();
+    return inputStream.read();
+  }
+
+  /**
+   * Reads up to {@code len} bytes from this stream and copies them into
+   * the given {@code buffer} starting at the given {@code offset}.
+   * Returns the number of bytes actually copied in to the given buffer.
+   *
+   * @param buffer the buffer to copy data into.
+   * @param offset data is copied starting at this offset.
+   * @param len max number of bytes to copy.
+   * @return the number of bytes actually copied in to the given buffer.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized int read(byte[] buffer, int offset, int len)
+      throws IOException {
+    throwIfClosed();
+    return inputStream.read(buffer, offset, len);
+  }
+
+  /**
+   * Closes this stream and releases all acquired resources.
+   *
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized void close() throws IOException {
+    if (inputStream != null) {
+      inputStream.close();
+      inputStream = null;
+      super.close();
+    }
+  }
+
+  /**
+   * Updates internal data such that the next read will take place at the given {@code pos}.
+   *
+   * @param pos new read position.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public synchronized void seek(long pos) throws IOException {
+    throwIfClosed();
+    inputStream.seek(pos);
+  }
+
+  /**
+   * Sets the number of bytes to read ahead each time.
+   *
+   * @param readahead the number of bytes to read ahead each time..
+   */
+  @Override
+  public synchronized void setReadahead(Long readahead) {
+    if (!isClosed()) {
+      inputStream.setReadahead(readahead);
+    }
+  }
+
+  /**
+   * Indicates whether the given {@code capability} is supported by this stream.
+   *
+   * @param capability the capability to check.
+   * @return true if the given {@code capability} is supported by this stream, false otherwise.
+   */
+  @Override
+  public boolean hasCapability(String capability) {
+    if (!isClosed()) {
+      return inputStream.hasCapability(capability);
+    }
+
+    return false;
+  }
+
+  /**
+   * Access the input stream statistics.
+   * This is for internal testing and may be removed without warning.
+   * @return the statistics for this input stream
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  @VisibleForTesting
+  public S3AInputStreamStatistics getS3AStreamStatistics() {
+    if (!isClosed()) {
+      inputStreamStatistics = inputStream.getS3AStreamStatistics();
+    }
+    return inputStreamStatistics;
+  }
+
+  /**
+   * Gets the internal IO statistics.
+   *
+   * @return the internal IO statistics.
+   */
+  @Override
+  public IOStatistics getIOStatistics() {
+    if (!isClosed()) {
+      ioStatistics = inputStream.getIOStatistics();
+    }
+    return ioStatistics;
+  }
+
+  protected boolean isClosed() {
+    return inputStream == null;
+  }
+
+  protected void throwIfClosed() throws IOException {
+    if (isClosed()) {
+      throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+    }
+  }
+
+  // Unsupported functions.
+
+  @Override
+  public boolean seekToNewSource(long targetPos) throws IOException {
+    return false;
+  }
+
+  @Override
+  public boolean markSupported() {
+    return false;
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java
new file mode 100644
index 0000000000000..38d740bd74f94
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.FilePosition;
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Provides an {@link InputStream} that allows reading from an S3 file.
+ */
+public abstract class S3ARemoteInputStream
+    extends InputStream
+    implements CanSetReadahead, StreamCapabilities, IOStatisticsSource {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3ARemoteInputStream.class);
+
+  /**
+   * The S3 file read by this instance.
+   */
+  private S3ARemoteObject remoteObject;
+
+  /**
+   * Reading of S3 file takes place through this reader.
+   */
+  private S3ARemoteObjectReader reader;
+
+  /**
+   * Name of this stream. Used only for logging.
+   */
+  private final String name;
+
+  /**
+   * Indicates whether the stream has been closed.
+   */
+  private volatile boolean closed;
+
+  /**
+   * Internal position within the file. Updated lazily
+   * after a seek before a read.
+   */
+  private FilePosition fpos;
+
+  /**
+   * This is the actual position within the file, used by
+   * lazy seek to decide whether to seek on the next read or not.
+   */
+  private long nextReadPos;
+
+  /** Information about each block of the mapped S3 file. */
+  private BlockData blockData;
+
+  /** Read-specific operation context. */
+  private S3AReadOpContext context;
+
+  /** Attributes of the S3 object being read. */
+  private S3ObjectAttributes s3Attributes;
+
+  /** Callbacks used for interacting with the underlying S3 client. */
+  private S3AInputStream.InputStreamCallbacks client;
+
+  /** Used for reporting input stream access statistics. */
+  private final S3AInputStreamStatistics streamStatistics;
+
+  private S3AInputPolicy inputPolicy;
+
+  private final ChangeTracker changeTracker;
+
+  private final IOStatistics ioStatistics;
+
+  /**
+   * Initializes a new instance of the {@code S3ARemoteInputStream} class.
+   *
+   * @param context read-specific operation context.
+   * @param s3Attributes attributes of the S3 object being read.
+   * @param client callbacks used for interacting with the underlying S3 client.
+   * @param streamStatistics statistics for this stream.
+   *
+   * @throws IllegalArgumentException if context is null.
+   * @throws IllegalArgumentException if s3Attributes is null.
+   * @throws IllegalArgumentException if client is null.
+   */
+  public S3ARemoteInputStream(
+      S3AReadOpContext context,
+      S3ObjectAttributes s3Attributes,
+      S3AInputStream.InputStreamCallbacks client,
+      S3AInputStreamStatistics streamStatistics) {
+
+    this.context = requireNonNull(context);
+    this.s3Attributes = requireNonNull(s3Attributes);
+    this.client = requireNonNull(client);
+    this.streamStatistics = requireNonNull(streamStatistics);
+    this.ioStatistics = streamStatistics.getIOStatistics();
+    this.name = S3ARemoteObject.getPath(s3Attributes);
+    this.changeTracker = new ChangeTracker(
+        this.name,
+        context.getChangeDetectionPolicy(),
+        this.streamStatistics.getChangeTrackerStatistics(),
+        s3Attributes);
+
+    setInputPolicy(context.getInputPolicy());
+    setReadahead(context.getReadahead());
+
+    long fileSize = s3Attributes.getLen();
+    int bufferSize = context.getPrefetchBlockSize();
+
+    this.blockData = new BlockData(fileSize, bufferSize);
+    this.fpos = new FilePosition(fileSize, bufferSize);
+    this.remoteObject = getS3File();
+    this.reader = new S3ARemoteObjectReader(remoteObject);
+
+    this.nextReadPos = 0;
+  }
+
+  /**
+   * Gets the internal IO statistics.
+   *
+   * @return the internal IO statistics.
+   */
+  @Override
+  public IOStatistics getIOStatistics() {
+    return ioStatistics;
+  }
+
+  /**
+   * Access the input stream statistics.
+   * This is for internal testing and may be removed without warning.
+   * @return the statistics for this input stream
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  public S3AInputStreamStatistics getS3AStreamStatistics() {
+    return streamStatistics;
+  }
+
+  /**
+   * Sets the number of bytes to read ahead each time.
+   *
+   * @param readahead the number of bytes to read ahead each time..
+   */
+  @Override
+  public synchronized void setReadahead(Long readahead) {
+    // We support read head by prefetching therefore we ignore the supplied value.
+    if (readahead != null) {
+      Validate.checkNotNegative(readahead, "readahead");
+    }
+  }
+
+  /**
+   * Indicates whether the given {@code capability} is supported by this stream.
+   *
+   * @param capability the capability to check.
+   * @return true if the given {@code capability} is supported by this stream, false otherwise.
+   */
+  @Override
+  public boolean hasCapability(String capability) {
+    return capability.equalsIgnoreCase(StreamCapabilities.IOSTATISTICS)
+        || capability.equalsIgnoreCase(StreamCapabilities.READAHEAD);
+  }
+
+  /**
+   * Set/update the input policy of the stream.
+   * This updates the stream statistics.
+   * @param inputPolicy new input policy.
+   */
+  private void setInputPolicy(S3AInputPolicy inputPolicy) {
+    this.inputPolicy = inputPolicy;
+    streamStatistics.inputPolicySet(inputPolicy.ordinal());
+  }
+
+  /**
+   * Returns the number of bytes that can read from this stream without blocking.
+   */
+  @Override
+  public int available() throws IOException {
+    throwIfClosed();
+
+    // Update the current position in the current buffer, if possible.
+    if (!fpos.setAbsolute(nextReadPos)) {
+      return 0;
+    }
+
+    return fpos.buffer().remaining();
+  }
+
+  /**
+   * Gets the current position.
+   *
+   * @return the current position.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  public long getPos() throws IOException {
+    throwIfClosed();
+
+    return nextReadPos;
+  }
+
+  /**
+   * Moves the current read position so that the next read will occur at {@code pos}.
+   *
+   * @param pos the absolute position to seek to.
+   * @throws IOException if there an error during this operation.
+   *
+   * @throws IllegalArgumentException if pos is outside of the range [0, file size].
+   */
+  public void seek(long pos) throws IOException {
+    throwIfClosed();
+    throwIfInvalidSeek(pos);
+
+    nextReadPos = pos;
+  }
+
+  /**
+   * Ensures that a non-empty valid buffer is available for immediate reading.
+   * It returns true when at least one such buffer is available for reading.
+   * It returns false on reaching the end of the stream.
+   *
+   * @return true if at least one such buffer is available for reading, false otherwise.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  protected abstract boolean ensureCurrentBuffer() throws IOException;
+
+  @Override
+  public int read() throws IOException {
+    throwIfClosed();
+
+    if (remoteObject.size() == 0
+        || nextReadPos >= remoteObject.size()) {
+      return -1;
+    }
+
+    if (!ensureCurrentBuffer()) {
+      return -1;
+    }
+
+    nextReadPos++;
+    incrementBytesRead(1);
+
+    return fpos.buffer().get() & 0xff;
+  }
+
+  /**
+   * Reads bytes from this stream and copies them into
+   * the given {@code buffer} starting at the beginning (offset 0).
+   * Returns the number of bytes actually copied in to the given buffer.
+   *
+   * @param buffer the buffer to copy data into.
+   * @return the number of bytes actually copied in to the given buffer.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public int read(byte[] buffer) throws IOException {
+    return read(buffer, 0, buffer.length);
+  }
+
+  /**
+   * Reads up to {@code len} bytes from this stream and copies them into
+   * the given {@code buffer} starting at the given {@code offset}.
+   * Returns the number of bytes actually copied in to the given buffer.
+   *
+   * @param buffer the buffer to copy data into.
+   * @param offset data is copied starting at this offset.
+   * @param len max number of bytes to copy.
+   * @return the number of bytes actually copied in to the given buffer.
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public int read(byte[] buffer, int offset, int len) throws IOException {
+    throwIfClosed();
+
+    if (len == 0) {
+      return 0;
+    }
+
+    if (remoteObject.size() == 0
+        || nextReadPos >= remoteObject.size()) {
+      return -1;
+    }
+
+    if (!ensureCurrentBuffer()) {
+      return -1;
+    }
+
+    int numBytesRead = 0;
+    int numBytesRemaining = len;
+
+    while (numBytesRemaining > 0) {
+      if (!ensureCurrentBuffer()) {
+        break;
+      }
+
+      ByteBuffer buf = fpos.buffer();
+      int bytesToRead = Math.min(numBytesRemaining, buf.remaining());
+      buf.get(buffer, offset, bytesToRead);
+      nextReadPos += bytesToRead;
+      incrementBytesRead(bytesToRead);
+      offset += bytesToRead;
+      numBytesRemaining -= bytesToRead;
+      numBytesRead += bytesToRead;
+    }
+
+    return numBytesRead;
+  }
+
+  protected S3ARemoteObject getFile() {
+    return remoteObject;
+  }
+
+  protected S3ARemoteObjectReader getReader() {
+    return reader;
+  }
+
+  protected S3ObjectAttributes getS3ObjectAttributes() {
+    return s3Attributes;
+  }
+
+  protected FilePosition getFilePosition() {
+    return fpos;
+  }
+
+  protected String getName() {
+    return name;
+  }
+
+  protected boolean isClosed() {
+    return closed;
+  }
+
+  protected long getNextReadPos() {
+    return nextReadPos;
+  }
+
+  protected BlockData getBlockData() {
+    return blockData;
+  }
+
+  protected S3AReadOpContext getContext() {
+    return context;
+  }
+
+  private void incrementBytesRead(int bytesRead) {
+    if (bytesRead > 0) {
+      streamStatistics.bytesRead(bytesRead);
+      if (getContext().getStats() != null) {
+        getContext().getStats().incrementBytesRead(bytesRead);
+      }
+      fpos.incrementBytesRead(bytesRead);
+    }
+  }
+
+  protected S3ARemoteObject getS3File() {
+    return new S3ARemoteObject(
+        context,
+        s3Attributes,
+        client,
+        streamStatistics,
+        changeTracker
+    );
+  }
+
+  protected String getOffsetStr(long offset) {
+    int blockNumber = -1;
+
+    if (blockData.isValidOffset(offset)) {
+      blockNumber = blockData.getBlockNumber(offset);
+    }
+
+    return String.format("%d:%d", blockNumber, offset);
+  }
+
+  /**
+   * Closes this stream and releases all acquired resources.
+   *
+   * @throws IOException if there is an IO error during this operation.
+   */
+  @Override
+  public void close() throws IOException {
+    if (closed) {
+      return;
+    }
+    closed = true;
+
+    blockData = null;
+    reader.close();
+    reader = null;
+    remoteObject = null;
+    fpos.invalidate();
+    try {
+      client.close();
+    } finally {
+      streamStatistics.close();
+    }
+    client = null;
+  }
+
+  @Override
+  public boolean markSupported() {
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    if (isClosed()) {
+      return "closed";
+    }
+
+    StringBuilder sb = new StringBuilder();
+    sb.append(String.format("nextReadPos = (%d)%n", nextReadPos));
+    sb.append(String.format("fpos = (%s)", fpos));
+    return sb.toString();
+  }
+
+  protected void throwIfClosed() throws IOException {
+    if (closed) {
+      throw new IOException(
+          name + ": " + FSExceptionMessages.STREAM_IS_CLOSED);
+    }
+  }
+
+  protected void throwIfInvalidSeek(long pos) throws EOFException {
+    if (pos < 0) {
+      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK + " " + pos);
+    } else if (pos > this.getBlockData().getFileSize()) {
+      throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF + " " + pos);
+    }
+  }
+
+  // Unsupported functions.
+
+  @Override
+  public void mark(int readlimit) {
+    throw new UnsupportedOperationException("mark not supported");
+  }
+
+  @Override
+  public void reset() {
+    throw new UnsupportedOperationException("reset not supported");
+  }
+
+  @Override
+  public long skip(long n) {
+    throw new UnsupportedOperationException("skip not supported");
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java
new file mode 100644
index 0000000000000..3ab0022bb082e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
+import org.apache.hadoop.fs.s3a.impl.SDKStreamDrainer;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.statistics.DurationTracker;
+
+/**
+ * Encapsulates low level interactions with S3 object on AWS.
+ */
+public class S3ARemoteObject {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(S3ARemoteObject.class);
+
+  /**
+   * Read-specific operation context.
+   */
+  private final S3AReadOpContext context;
+
+  /**
+   * S3 object attributes.
+   */
+  private final S3ObjectAttributes s3Attributes;
+
+  /**
+   * Callbacks used for interacting with the underlying S3 client.
+   */
+  private final S3AInputStream.InputStreamCallbacks client;
+
+  /**
+   * Used for reporting input stream access statistics.
+   */
+  private final S3AInputStreamStatistics streamStatistics;
+
+  /**
+   * Enforces change tracking related policies.
+   */
+  private final ChangeTracker changeTracker;
+
+  /**
+   * Maps a stream returned by openForRead() to the associated S3 object.
+   * That allows us to close the object when closing the stream.
+   */
+  private final Map<InputStream, S3Object> s3Objects;
+
+  /**
+   * uri of the object being read.
+   */
+  private final String uri;
+
+  /**
+   * size of a buffer to create when draining the stream.
+   */
+  private static final int DRAIN_BUFFER_SIZE = 16384;
+
+  /**
+   * Initializes a new instance of the {@code S3ARemoteObject} class.
+   *
+   * @param context read-specific operation context.
+   * @param s3Attributes attributes of the S3 object being read.
+   * @param client callbacks used for interacting with the underlying S3 client.
+   * @param streamStatistics statistics about stream access.
+   * @param changeTracker helps enforce change tracking policy.
+   *
+   * @throws IllegalArgumentException if context is null.
+   * @throws IllegalArgumentException if s3Attributes is null.
+   * @throws IllegalArgumentException if client is null.
+   * @throws IllegalArgumentException if streamStatistics is null.
+   * @throws IllegalArgumentException if changeTracker is null.
+   */
+  public S3ARemoteObject(
+      S3AReadOpContext context,
+      S3ObjectAttributes s3Attributes,
+      S3AInputStream.InputStreamCallbacks client,
+      S3AInputStreamStatistics streamStatistics,
+      ChangeTracker changeTracker) {
+
+    Validate.checkNotNull(context, "context");
+    Validate.checkNotNull(s3Attributes, "s3Attributes");
+    Validate.checkNotNull(client, "client");
+    Validate.checkNotNull(streamStatistics, "streamStatistics");
+    Validate.checkNotNull(changeTracker, "changeTracker");
+
+    this.context = context;
+    this.s3Attributes = s3Attributes;
+    this.client = client;
+    this.streamStatistics = streamStatistics;
+    this.changeTracker = changeTracker;
+    this.s3Objects = new IdentityHashMap<>();
+    this.uri = this.getPath();
+  }
+
+  /**
+   * Gets an instance of {@code Invoker} for interacting with S3 API.
+   *
+   * @return an instance of {@code Invoker} for interacting with S3 API.
+   */
+  public Invoker getReadInvoker() {
+    return context.getReadInvoker();
+  }
+
+  /**
+   * Gets an instance of {@code S3AInputStreamStatistics} used for reporting access metrics.
+   *
+   * @return an instance of {@code S3AInputStreamStatistics} used for reporting access metrics.
+   */
+  public S3AInputStreamStatistics getStatistics() {
+    return streamStatistics;
+  }
+
+  /**
+   * Gets the path of this file.
+   *
+   * @return the path of this file.
+   */
+  public String getPath() {
+    return getPath(s3Attributes);
+  }
+
+  /**
+   * Gets the path corresponding to the given s3Attributes.
+   *
+   * @param s3Attributes attributes of an S3 object.
+   * @return the path corresponding to the given s3Attributes.
+   */
+  public static String getPath(S3ObjectAttributes s3Attributes) {
+    return String.format("s3a://%s/%s", s3Attributes.getBucket(),
+        s3Attributes.getKey());
+  }
+
+  /**
+   * Gets the size of this file.
+   * Its value is cached once obtained from AWS.
+   *
+   * @return the size of this file.
+   */
+  public long size() {
+    return s3Attributes.getLen();
+  }
+
+  /**
+   * Opens a section of the file for reading.
+   *
+   * @param offset Start offset (0 based) of the section to read.
+   * @param size Size of the section to read.
+   * @return an {@code InputStream} corresponding to the given section of this file.
+   *
+   * @throws IOException if there is an error opening this file section for reading.
+   * @throws IllegalArgumentException if offset is negative.
+   * @throws IllegalArgumentException if offset is greater than or equal to file size.
+   * @throws IllegalArgumentException if size is greater than the remaining bytes.
+   */
+  public InputStream openForRead(long offset, int size) throws IOException {
+    Validate.checkNotNegative(offset, "offset");
+    Validate.checkLessOrEqual(offset, "offset", size(), "size()");
+    Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset");
+
+    streamStatistics.streamOpened();
+    final GetObjectRequest request =
+        client.newGetRequest(s3Attributes.getKey())
+            .withRange(offset, offset + size - 1);
+    changeTracker.maybeApplyConstraint(request);
+
+    String operation = String.format(
+        "%s %s at %d", S3AInputStream.OPERATION_OPEN, uri, offset);
+    DurationTracker tracker = streamStatistics.initiateGetRequest();
+    S3Object object = null;
+
+    try {
+      object = Invoker.once(operation, uri, () -> client.getObject(request));
+    } catch (IOException e) {
+      tracker.failed();
+      throw e;
+    } finally {
+      tracker.close();
+    }
+
+    changeTracker.processResponse(object, operation, offset);
+    InputStream stream = object.getObjectContent();
+    synchronized (s3Objects) {
+      s3Objects.put(stream, object);
+    }
+
+    return stream;
+  }
+
+  void close(InputStream inputStream, int numRemainingBytes) {
+    S3Object obj;
+    synchronized (s3Objects) {
+      obj = s3Objects.remove(inputStream);
+      if (obj == null) {
+        throw new IllegalArgumentException("inputStream not found");
+      }
+    }
+    SDKStreamDrainer drainer = new SDKStreamDrainer(
+        uri,
+        obj,
+        (S3ObjectInputStream)inputStream,
+        false,
+        numRemainingBytes,
+        streamStatistics,
+        "close() operation");
+    if (numRemainingBytes <= context.getAsyncDrainThreshold()) {
+      // don't bother with async io.
+      drainer.apply();
+    } else {
+      LOG.debug("initiating asynchronous drain of {} bytes", numRemainingBytes);
+      // schedule an async drain/abort
+      client.submit(drainer);
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java
new file mode 100644
index 0000000000000..89ea77d6d0ebb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation;
+
+/**
+ * Provides functionality to read S3 file one block at a time.
+ */
+public class S3ARemoteObjectReader implements Closeable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      S3ARemoteObjectReader.class);
+
+  /** We read from the underlying input stream in blocks of this size. */
+  private static final int READ_BUFFER_SIZE = 64 * 1024;
+
+  /** The S3 file to read. */
+  private final S3ARemoteObject remoteObject;
+
+  /** Set to true by close(). */
+  private volatile boolean closed;
+
+  private final S3AInputStreamStatistics streamStatistics;
+
+  /**
+   * Constructs an instance of {@link S3ARemoteObjectReader}.
+   *
+   * @param remoteObject The S3 file to read.
+   *
+   * @throws IllegalArgumentException if remoteObject is null.
+   */
+  public S3ARemoteObjectReader(S3ARemoteObject remoteObject) {
+    Validate.checkNotNull(remoteObject, "remoteObject");
+
+    this.remoteObject = remoteObject;
+    this.streamStatistics = this.remoteObject.getStatistics();
+  }
+
+  /**
+   * Stars reading at {@code offset} and reads upto {@code size} bytes into {@code buffer}.
+   *
+   * @param buffer the buffer into which data is returned
+   * @param offset the absolute offset into the underlying file where reading starts.
+   * @param size the number of bytes to be read.
+   *
+   * @return number of bytes actually read.
+   * @throws IOException if there is an error reading from the file.
+   *
+   * @throws IllegalArgumentException if buffer is null.
+   * @throws IllegalArgumentException if offset is outside of the range [0, file size].
+   * @throws IllegalArgumentException if size is zero or negative.
+   */
+  public int read(ByteBuffer buffer, long offset, int size) throws IOException {
+    Validate.checkNotNull(buffer, "buffer");
+    Validate.checkWithinRange(offset, "offset", 0, this.remoteObject.size());
+    Validate.checkPositiveInteger(size, "size");
+
+    if (this.closed) {
+      return -1;
+    }
+
+    int reqSize = (int) Math.min(size, this.remoteObject.size() - offset);
+    return readOneBlockWithRetries(buffer, offset, reqSize);
+  }
+
+  @Override
+  public void close() {
+    this.closed = true;
+  }
+
+  private int readOneBlockWithRetries(ByteBuffer buffer, long offset, int size)
+      throws IOException {
+
+    this.streamStatistics.readOperationStarted(offset, size);
+    Invoker invoker = this.remoteObject.getReadInvoker();
+
+    int invokerResponse =
+        invoker.retry("read", this.remoteObject.getPath(), true,
+            trackDurationOfOperation(streamStatistics,
+                STREAM_READ_REMOTE_BLOCK_READ, () -> {
+                  try {
+                    this.readOneBlock(buffer, offset, size);
+                  } catch (EOFException e) {
+                    // the base implementation swallows EOFs.
+                    return -1;
+                  } catch (SocketTimeoutException e) {
+                    throw e;
+                  } catch (IOException e) {
+                    this.remoteObject.getStatistics().readException();
+                    throw e;
+                  }
+                  return 0;
+                }));
+
+    int numBytesRead = buffer.position();
+    buffer.limit(numBytesRead);
+    this.remoteObject.getStatistics()
+        .readOperationCompleted(size, numBytesRead);
+
+    if (invokerResponse < 0) {
+      return invokerResponse;
+    } else {
+      return numBytesRead;
+    }
+  }
+
+  private void readOneBlock(ByteBuffer buffer, long offset, int size)
+      throws IOException {
+    int readSize = Math.min(size, buffer.remaining());
+    if (readSize == 0) {
+      return;
+    }
+
+    InputStream inputStream = remoteObject.openForRead(offset, readSize);
+    int numRemainingBytes = readSize;
+    byte[] bytes = new byte[READ_BUFFER_SIZE];
+
+    int numBytesToRead;
+    int numBytes;
+
+    try {
+      do {
+        numBytesToRead = Math.min(READ_BUFFER_SIZE, numRemainingBytes);
+        numBytes = inputStream.read(bytes, 0, numBytesToRead);
+        if (numBytes < 0) {
+          String message = String.format(
+              "Unexpected end of stream: buffer[%d], readSize = %d, numRemainingBytes = %d",
+              buffer.capacity(), readSize, numRemainingBytes);
+          throw new EOFException(message);
+        }
+
+        if (numBytes > 0) {
+          buffer.put(bytes, 0, numBytes);
+          numRemainingBytes -= numBytes;
+        }
+      }
+      while (!this.closed && (numRemainingBytes > 0));
+    } finally {
+      remoteObject.close(inputStream, numRemainingBytes);
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java
new file mode 100644
index 0000000000000..f953d35dcca87
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * High performance s3 input stream which reads in
+ * blocks and can cache blocks in the local filesystem.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardDynamoDBDiagnostic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardDynamoDBDiagnostic.java
deleted file mode 100644
index 83495ca310623..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardDynamoDBDiagnostic.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.service.launcher.AbstractLaunchableService;
-import org.apache.hadoop.service.launcher.ServiceLaunchException;
-
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_FAIL;
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
-
-/**
- * Entry point for S3Guard diagnostics operations against DynamoDB tables.
- */
-public class AbstractS3GuardDynamoDBDiagnostic
-    extends AbstractLaunchableService {
-
-  private S3AFileSystem filesystem;
-
-  private DynamoDBMetadataStore store;
-
-  private URI uri;
-
-  private List<String> arguments;
-
-  /**
-   * Constructor.
-   * @param name entry point name.
-   */
-  public AbstractS3GuardDynamoDBDiagnostic(final String name) {
-    super(name);
-  }
-
-  /**
-   * Constructor. If the store is set then that is the store for the operation,
-   * otherwise the filesystem's binding is used instead.
-   * @param name entry point name.
-   * @param filesystem filesystem
-   * @param store optional metastore.
-   * @param uri URI. Must be set if filesystem == null.
-   */
-  public AbstractS3GuardDynamoDBDiagnostic(
-      final String name,
-      @Nullable final S3AFileSystem filesystem,
-      @Nullable final DynamoDBMetadataStore store,
-      @Nullable final URI uri) {
-    super(name);
-    this.store = store;
-    this.filesystem = filesystem;
-    if (store == null) {
-      require(filesystem != null, "No filesystem or URI");
-      bindStore(filesystem);
-    }
-    if (uri == null) {
-      require(filesystem != null, "No filesystem or URI");
-      setUri(filesystem.getUri());
-    } else {
-      setUri(uri);
-    }
-  }
-
-  /**
-   * Require a condition to hold, otherwise an exception is thrown.
-   * @param condition condition to be true
-   * @param error text on failure.
-   * @throws ServiceLaunchException if the condition is not met
-   */
-  protected static void require(boolean condition, String error) {
-    if (!condition) {
-      throw failure(error);
-    }
-  }
-
-  /**
-   * Generate a failure exception for throwing.
-   * @param message message
-   * @param ex optional nested exception.
-   * @return an exception to throw
-   */
-  protected static ServiceLaunchException failure(String message,
-      Throwable ex) {
-    return new ServiceLaunchException(EXIT_FAIL, message, ex);
-  }
-
-  /**
-   * Generate a failure exception for throwing.
-   * @param message message
-   * @return an exception to throw
-   */
-  protected static ServiceLaunchException failure(String message) {
-    return new ServiceLaunchException(EXIT_FAIL, message);
-  }
-
-  @Override
-  public Configuration bindArgs(final Configuration config,
-      final List<String> args)
-      throws Exception {
-    this.arguments = args;
-    return super.bindArgs(config, args);
-  }
-
-  /**
-   * Get the argument list.
-   * @return the argument list.
-   */
-  protected List<String> getArguments() {
-    return arguments;
-  }
-
-  /**
-   * Bind to the store from a CLI argument.
-   * @param fsURI filesystem URI
-   * @throws IOException failure
-   */
-  protected void bindFromCLI(String fsURI)
-      throws IOException {
-    Configuration conf = getConfig();
-    setUri(fsURI);
-    FileSystem fs = FileSystem.get(getUri(), conf);
-    require(fs instanceof S3AFileSystem,
-        "Not an S3A Filesystem:  " + fsURI);
-    filesystem = (S3AFileSystem) fs;
-    bindStore(filesystem);
-    setUri(fs.getUri());
-  }
-
-  /**
-   * Binds the {@link #store} field to the metastore of
-   * the filesystem -which must have a DDB metastore.
-   * @param fs filesystem to bind the store to.
-   */
-  private void bindStore(final S3AFileSystem fs) {
-    require(fs.hasMetadataStore(),
-        "Filesystem has no metadata store: " + fs.getUri());
-    MetadataStore ms = fs.getMetadataStore();
-    require(ms instanceof DynamoDBMetadataStore,
-        "Filesystem " + fs.getUri()
-            + " does not have a DynamoDB metadata store:  " + ms);
-    store = (DynamoDBMetadataStore) ms;
-  }
-
-  protected DynamoDBMetadataStore getStore() {
-    return store;
-  }
-
-  public S3AFileSystem getFilesystem() {
-    return filesystem;
-  }
-
-  public URI getUri() {
-    return uri;
-  }
-
-  public void setUri(final URI uri) {
-    String fsURI = uri.toString();
-    if (!fsURI.endsWith("/")) {
-      setUri(fsURI);
-    } else {
-      this.uri = uri;
-    }
-  }
-
-  /**
-   * Set the URI from a string; will add a "/" if needed.
-   * @param fsURI filesystem URI.
-   * @throws RuntimeException if the fsURI parameter is not a valid URI.
-   */
-  public void setUri(String fsURI) {
-    if (fsURI != null) {
-      if (!fsURI.endsWith("/")) {
-        fsURI += "/";
-      }
-      try {
-        setUri(new URI(fsURI));
-      } catch (URISyntaxException e) {
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-  /**
-   * Get the list of arguments, after validating the list size.
-   * @param argMin minimum number of entries.
-   * @param argMax maximum number of entries.
-   * @param usage Usage message.
-   * @return the argument list, which will be in the range.
-   * @throws ServiceLaunchException if the argument list is not valid.
-   */
-  protected List<String> getArgumentList(final int argMin,
-      final int argMax,
-      final String usage) {
-    List<String> arg = getArguments();
-    if (arg == null || arg.size() < argMin || arg.size() > argMax) {
-      // no arguments: usage message
-      throw new ServiceLaunchException(EXIT_USAGE, usage);
-    }
-    return arg;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AuthoritativeAuditOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AuthoritativeAuditOperation.java
deleted file mode 100644
index eac980c765bde..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AuthoritativeAuditOperation.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Collection;
-import java.util.Queue;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.service.launcher.LauncherExitCodes;
-import org.apache.hadoop.util.DurationInfo;
-import org.apache.hadoop.util.ExitCodeProvider;
-import org.apache.hadoop.util.ExitUtil;
-
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE;
-
-/**
- * Audit a directory tree for being authoritative.
- * One aspect of the audit to be aware of: the root directory is
- * always considered authoritative, even though, because there is no
- * matching entry in any of the stores, it is not strictly true.
- */
-public class AuthoritativeAuditOperation extends AbstractStoreOperation {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      AuthoritativeAuditOperation.class);
-
-  /**
-   * Exception error code when a path is non-auth in the DB}.
-   */
-  public static final int ERROR_ENTRY_NOT_AUTH_IN_DDB = EXIT_NOT_ACCEPTABLE;
-
-  /**
-   * Exception error code when a path is not configured to be
-   * auth in the S3A FS Config: {@value}.
-   */
-  public static final int ERROR_PATH_NOT_AUTH_IN_FS = 5;
-
-  /**
-   * Exception error string: {@value}.
-   */
-  public static final String E_NONAUTH
-      = "Directory is not marked as authoritative in the S3Guard store";
-
-  /** The metastore to audit. */
-  private final DynamoDBMetadataStore metastore;
-
-  /**  require all directories to be authoritative. */
-  private final boolean requireAuthoritative;
-
-  /**
-   * Verbose switch.
-   */
-  private final boolean verbose;
-
-  /**
-   * Constructor.
-   * @param storeContext store context.
-   * @param metastore metastore
-   * @param requireAuthoritative require all directories to be authoritative
-   * @param verbose verbose output
-   */
-  public AuthoritativeAuditOperation(
-      final StoreContext storeContext,
-      final DynamoDBMetadataStore metastore,
-      final boolean requireAuthoritative,
-      final boolean verbose) {
-    super(storeContext);
-    this.metastore = metastore;
-    this.requireAuthoritative = requireAuthoritative;
-    this.verbose = verbose;
-  }
-
-  /**
-   * Examine the path metadata and verify that the dir is authoritative.
-   * @param md metadata.
-   * @param requireAuth require all directories to be authoritative
-   * @throws NonAuthoritativeDirException if it is !auth and requireAuth=true.
-   */
-  private void verifyAuthDir(final DDBPathMetadata md,
-      final boolean requireAuth)
-      throws PathIOException {
-    final Path path = md.getFileStatus().getPath();
-    boolean isAuth = path.isRoot() || md.isAuthoritativeDir();
-    if (!isAuth && requireAuth) {
-      throw new NonAuthoritativeDirException(path);
-    }
-  }
-
-  /**
-   * Examine the path metadata, declare whether it should be queued for
-   * recursive scanning.
-   * @param md metadata.
-   * @return true if it is a dir to scan.
-   */
-  private boolean isDirectory(PathMetadata md) {
-    return !md.getFileStatus().isFile();
-  }
-
-  /**
-   * Audit the tree.
-   * @param path qualified path to scan
-   * @return tuple(dirs scanned, nonauth dirs found)
-   * @throws IOException IO failure
-   * @throws ExitUtil.ExitException if a non-auth dir was found.
-   */
-  public Pair<Integer, Integer> audit(Path path) throws IOException {
-    try (DurationInfo ignored =
-             new DurationInfo(LOG, "Audit %s", path)) {
-      return executeAudit(path, requireAuthoritative, true);
-    }
-  }
-
-  /**
-   * Audit the tree.
-   * This is the internal code which throws a NonAuthoritativePathException
-   * on failures; tests may use it.
-   * @param path path to scan
-   * @param requireAuth require all directories to be authoritative
-   * @param recursive recurse?
-   * @return tuple(dirs scanned, nonauth dirs found)
-   * @throws IOException IO failure
-   * @throws NonAuthoritativeDirException if a non-auth dir was found.
-   */
-  @VisibleForTesting
-  Pair<Integer, Integer> executeAudit(
-      final Path path,
-      final boolean requireAuth,
-      final boolean recursive) throws IOException {
-    int dirs = 0;
-    int nonauth = 0;
-    final Queue<DDBPathMetadata> queue = new ArrayDeque<>();
-    final boolean isRoot = path.isRoot();
-    final DDBPathMetadata baseData = metastore.get(path);
-    if (baseData == null) {
-      throw new ExitUtil.ExitException(LauncherExitCodes.EXIT_NOT_FOUND,
-          "No S3Guard entry for path " + path);
-    }
-
-    if (isRoot || isDirectory(baseData)) {
-      // we have the root entry or an authoritative a directory
-      queue.add(baseData);
-    } else {
-      LOG.info("Path represents file");
-      return Pair.of(0, 0);
-    }
-
-    while (!queue.isEmpty()) {
-      dirs++;
-      final DDBPathMetadata dir = queue.poll();
-      final Path p = dir.getFileStatus().getPath();
-      LOG.debug("Directory {}", dir.prettyPrint());
-      // log a message about the dir state, with root treated specially
-      if (!p.isRoot()) {
-        if (!dir.isAuthoritativeDir()) {
-          LOG.warn("Directory {} is not authoritative", p);
-          nonauth++;
-          verifyAuthDir(dir, requireAuth);
-        } else {
-          LOG.info("Directory {}", p);
-        }
-      } else {
-        // this is done to avoid the confusing message about root not being
-        // authoritative
-        LOG.info("Root directory {}", p);
-      }
-
-      // list its children
-      if (recursive) {
-        final DirListingMetadata entry = metastore.listChildren(p);
-
-        if (entry != null) {
-          final Collection<PathMetadata> listing = entry.getListing();
-          int files = 0, subdirs = 0;
-          for (PathMetadata e : listing) {
-            if (isDirectory(e)) {
-              // queue for auditing
-              queue.add((DDBPathMetadata) e);
-              subdirs++;
-            } else {
-              files++;
-            }
-          }
-          if (verbose && files > 0 || subdirs > 0) {
-            LOG.info("  files {}; directories {}", files, subdirs);
-          }
-        } else {
-          LOG.info("Directory {} has been deleted", dir);
-        }
-      }
-    }
-    // end of scan
-    if (dirs == 1 && isRoot) {
-      LOG.info("The store has no directories to scan");
-    } else {
-      LOG.info("Scanned {} directories - {} were not marked as authoritative",
-          dirs, nonauth);
-    }
-    return Pair.of(dirs, nonauth);
-  }
-
-  /**
-   * A directory was found which was non-authoritative.
-   * The exit code for this operation is
-   * {@link LauncherExitCodes#EXIT_NOT_ACCEPTABLE} -This is what the S3Guard
-   * will return.
-   */
-  public static final class NonAuthoritativeDirException
-      extends PathIOException implements ExitCodeProvider {
-
-    /**
-     * Instantiate.
-     * @param path the path which is non-authoritative.
-     */
-    private NonAuthoritativeDirException(final Path path) {
-      super(path.toString(), E_NONAUTH);
-    }
-
-    @Override
-    public int getExitCode() {
-      return ERROR_ENTRY_NOT_AUTH_IN_DDB;
-    }
-
-    @Override
-    public String toString() {
-      return getMessage();
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/BulkOperationState.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/BulkOperationState.java
deleted file mode 100644
index b4974b7356514..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/BulkOperationState.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * This represents state which may be passed to bulk IO operations
- * to enable them to store information about the state of the ongoing
- * operation across invocations.
- * <p>
- * A bulk operation state <i>MUST</i> only be be used for the single store
- * from which it was created, and <i>MUST</i>only for the duration of a single
- * bulk update operation.
- * <p>
- * Passing in the state is to allow the stores to maintain state about
- * updates they have already made to their store during this single operation:
- * a cache of what has happened. It is not a list of operations to be applied.
- * If a list of operations to perform is built up (e.g. during rename)
- * that is the duty of the caller, not this state.
- * <p>
- * After the operation has completed, it <i>MUST</i> be closed so
- * as to guarantee that all state is released.
- */
-public class BulkOperationState implements Closeable {
-
-  private final OperationType operation;
-
-  /**
-   * Constructor.
-   * @param operation the type of the operation.
-   */
-  public BulkOperationState(final OperationType operation) {
-    this.operation = operation;
-  }
-
-  /**
-   * Get the operation type.
-   * @return the operation type.
-   */
-  public OperationType getOperation() {
-    return operation;
-  }
-
-  @Override
-  public void close() throws IOException {
-
-  }
-
-  /**
-   * Enumeration of operations which can be performed in bulk.
-   * This can be used by the stores however they want.
-   * One special aspect: renames are to be done through a {@link RenameTracker}.
-   * Callers will be blocked from initiating a rename through
-   * {@code S3Guard#initiateBulkWrite()}
-   */
-  public enum OperationType {
-    /** Writing data. */
-    Put,
-    /**
-     * Rename: add and delete.
-     * After the rename, the tree under the destination path
-     * can be tagged as authoritative.
-     */
-    Rename,
-    /** Pruning: deleting entries and updating parents. */
-    Prune,
-    /** Commit operation. */
-    Commit,
-    /** Deletion operation. */
-    Delete,
-    /** FSCK operation. */
-    Fsck,
-    /**
-     * Bulk directory tree import.
-     * After an import, the entire tree under the path has been
-     * enumerated and should be tagged as authoritative.
-     */
-    Import,
-    /**
-     * Listing update.
-     */
-    Listing,
-    /**
-     * Mkdir operation.
-     */
-    Mkdir,
-    /**
-     * Multipart upload operation.
-     */
-    Upload
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java
deleted file mode 100644
index 292c0161348e9..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-/**
- * {@code DDBPathMetadata} wraps {@link PathMetadata} and adds the
- * isAuthoritativeDir flag to provide support for authoritative directory
- * listings in {@link DynamoDBMetadataStore}.
- */
-public class DDBPathMetadata extends PathMetadata {
-
-  private boolean isAuthoritativeDir;
-
-  public DDBPathMetadata(PathMetadata pmd) {
-    super(pmd.getFileStatus(), pmd.isEmptyDirectory(), pmd.isDeleted(),
-        pmd.getLastUpdated());
-    this.isAuthoritativeDir = false;
-    this.setLastUpdated(pmd.getLastUpdated());
-  }
-
-  public DDBPathMetadata(S3AFileStatus fileStatus) {
-    super(fileStatus);
-    this.isAuthoritativeDir = false;
-  }
-
-  public DDBPathMetadata(S3AFileStatus fileStatus, Tristate isEmptyDir,
-      boolean isDeleted, long lastUpdated) {
-    super(fileStatus, isEmptyDir, isDeleted, lastUpdated);
-    this.isAuthoritativeDir = false;
-  }
-
-  public DDBPathMetadata(S3AFileStatus fileStatus, Tristate isEmptyDir,
-      boolean isDeleted, boolean isAuthoritativeDir, long lastUpdated) {
-    super(fileStatus, isEmptyDir, isDeleted, lastUpdated);
-    this.isAuthoritativeDir = isAuthoritativeDir;
-  }
-
-  public boolean isAuthoritativeDir() {
-    return isAuthoritativeDir;
-  }
-
-  public void setAuthoritativeDir(boolean authoritativeDir) {
-    isAuthoritativeDir = authoritativeDir;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    return super.equals(o);
-  }
-
-  @Override public int hashCode() {
-    return super.hashCode();
-  }
-
-  @Override public String toString() {
-    return "DDBPathMetadata{" +
-        "isAuthoritativeDir=" + isAuthoritativeDir +
-        ", PathMetadata=" + super.toString() +
-        '}';
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DelayedUpdateRenameTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DelayedUpdateRenameTracker.java
deleted file mode 100644
index 51a9821ace106..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DelayedUpdateRenameTracker.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.amazonaws.SdkBaseException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.util.DurationInfo;
-
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.addMoveAncestors;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.addMoveDir;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.addMoveFile;
-
-/**
- * This is the rename updating strategy originally used:
- * a collection of source paths and a list of destinations are created,
- * then updated at the end (possibly slow).
- * <p>
- * It is not currently instantiated by any of the active trackers,
- * but is preserved to show that the original rename strategy
- * can be implemented via the tracker model.
- */
-public class DelayedUpdateRenameTracker extends RenameTracker {
-
-  private final MetadataStore metadataStore;
-
-  private final Collection<Path> sourcePaths = new HashSet<>();
-
-  private final List<PathMetadata> destMetas = new ArrayList<>();
-
-  private final List<Path> deletedPaths = new ArrayList<>();
-
-  public DelayedUpdateRenameTracker(
-      final StoreContext storeContext,
-      final MetadataStore metadataStore,
-      final Path sourceRoot,
-      final Path dest,
-      final BulkOperationState operationState) {
-    super("DelayedUpdateRenameTracker", storeContext, metadataStore,
-        sourceRoot, dest, operationState);
-    this.metadataStore = storeContext.getMetadataStore();
-  }
-
-  @Override
-  public synchronized void fileCopied(
-      final Path sourcePath,
-      final S3ObjectAttributes sourceAttributes,
-      final S3ObjectAttributes destAttributes,
-      final Path destPath,
-      final long blockSize,
-      final boolean addAncestors) throws IOException {
-
-    addMoveFile(metadataStore,
-        sourcePaths,
-        destMetas,
-        sourcePath,
-        destPath,
-        sourceAttributes.getLen(),
-        blockSize,
-        getOwner(),
-        destAttributes.getETag(),
-        destAttributes.getVersionId());
-    // Ancestor directories may not be listed, so we explicitly add them
-    if (addAncestors) {
-      addMoveAncestors(metadataStore,
-          sourcePaths,
-          destMetas,
-          getSourceRoot(),
-          sourcePath,
-          destPath,
-          getOwner());
-    }
-  }
-
-  @Override
-  public synchronized void directoryMarkerCopied(final Path sourcePath,
-      final Path destPath,
-      final boolean addAncestors) throws IOException {
-    addMoveDir(metadataStore, sourcePaths, destMetas,
-        sourcePath,
-        destPath, getOwner());
-    // Ancestor directories may not be listed, so we explicitly add them
-    if (addAncestors) {
-      addMoveAncestors(metadataStore,
-          sourcePaths,
-          destMetas,
-          getSourceRoot(),
-          sourcePath,
-          destPath,
-          getOwner());
-    }
-  }
-
-  @Override
-  public synchronized void moveSourceDirectory() throws IOException {
-    if (!sourcePaths.contains(getSourceRoot())) {
-      addMoveDir(metadataStore, sourcePaths, destMetas,
-          getSourceRoot(),
-          getDest(), getOwner());
-    }
-  }
-
-  @Override
-  public synchronized void sourceObjectsDeleted(
-      final Collection<Path> paths) throws IOException {
-    // add to the list of deleted paths.
-    deletedPaths.addAll(paths);
-  }
-
-  @Override
-  public void completeRename() throws IOException {
-    metadataStore.move(sourcePaths, destMetas, getOperationState());
-    super.completeRename();
-  }
-
-  @Override
-  public IOException renameFailed(final Exception ex) {
-    LOG.warn("Rename has failed; updating s3guard with destination state");
-    try (DurationInfo ignored = new DurationInfo(LOG,
-        "Cleaning up deleted paths")) {
-      // the destination paths are updated; the source is left alone.
-      metadataStore.move(new ArrayList<>(0), destMetas, getOperationState());
-      for (Path deletedPath : deletedPaths) {
-        // this is not ideal in that it may leave parent stuff around.
-        metadataStore.delete(deletedPath, getOperationState());
-      }
-      deleteParentPaths();
-    } catch (IOException | SdkBaseException e) {
-      LOG.warn("Ignoring error raised in AWS SDK ", e);
-    }
-
-    return super.renameFailed(ex);
-  }
-
-  /**
-   * Delete all the parent paths we know to be empty (by walking up the tree
-   * deleting as appropriate).
-   * @throws IOException failure
-   */
-  private void deleteParentPaths() throws IOException {
-    Set<Path> parentPaths = new HashSet<>();
-    for (Path deletedPath : deletedPaths) {
-      Path parent = deletedPath.getParent();
-      if (!parent.equals(getSourceRoot())) {
-        parentPaths.add(parent);
-      }
-    }
-    // now there's a set of parent paths. We now want to
-    // get them ordered by depth, so that deeper entries come first
-    // that way: when we check for a parent path existing we can
-    // see if it really is empty.
-    List<Path> parents = new ArrayList<>(parentPaths);
-    parents.sort(PathOrderComparators.TOPMOST_PATH_LAST);
-    for (Path parent : parents) {
-      PathMetadata md = metadataStore.get(parent, true);
-      if (md != null && md.isEmptyDirectory() == Tristate.TRUE) {
-        // if were confident that this is empty: delete it.
-        metadataStore.delete(parent, getOperationState());
-      }
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
deleted file mode 100644
index a69109c9c88bb..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.NoSuchElementException;
-import java.util.Queue;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-
-/**
- * {@code DescendantsIterator} is a {@link RemoteIterator} that implements
- * pre-ordering breadth-first traversal (BFS) of a path and all of its
- * descendants recursively.  After visiting each path, that path's direct
- * children are discovered by calling {@link MetadataStore#listChildren(Path)}.
- * Each iteration returns the next direct child, and if that child is a
- * directory, also pushes it onto a queue to discover its children later.
- *
- * For example, assume the consistent store contains metadata representing this
- * file system structure:
- *
- * <pre>
- * /dir1
- * |-- dir2
- * |   |-- file1
- * |   `-- file2
- * `-- dir3
- *     |-- dir4
- *     |   `-- file3
- *     |-- dir5
- *     |   `-- file4
- *     `-- dir6
- * </pre>
- *
- * Consider this code sample:
- * <pre>
- * final PathMetadata dir1 = get(new Path("/dir1"));
- * for (DescendantsIterator descendants = new DescendantsIterator(dir1);
- *     descendants.hasNext(); ) {
- *   final FileStatus status = descendants.next().getFileStatus();
- *   System.out.printf("%s %s%n", status.isDirectory() ? 'D' : 'F',
- *       status.getPath());
- * }
- * </pre>
- *
- * The output is:
- * <pre>
- * D /dir1
- * D /dir1/dir2
- * D /dir1/dir3
- * F /dir1/dir2/file1
- * F /dir1/dir2/file2
- * D /dir1/dir3/dir4
- * D /dir1/dir3/dir5
- * F /dir1/dir3/dir4/file3
- * F /dir1/dir3/dir5/file4
- * D /dir1/dir3/dir6
- * </pre>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class DescendantsIterator implements RemoteIterator<S3AFileStatus> {
-
-  private final MetadataStore metadataStore;
-  private final Queue<PathMetadata> queue = new LinkedList<>();
-
-  /**
-   * Creates a new {@code DescendantsIterator}.
-   *
-   * @param ms the associated {@link MetadataStore}
-   * @param meta base path for descendants iteration, which will be the first
-   *     returned during iteration (except root). Null makes empty iterator.
-   * @throws IOException if errors happen during metadata store listing
-   */
-  public DescendantsIterator(MetadataStore ms, PathMetadata meta)
-      throws IOException {
-    Preconditions.checkNotNull(ms);
-    this.metadataStore = ms;
-
-    if (meta != null) {
-      final Path path = meta.getFileStatus().getPath();
-      if (path.isRoot()) {
-        DirListingMetadata rootListing = ms.listChildren(path);
-        if (rootListing != null) {
-          rootListing = rootListing.withoutTombstones();
-          queue.addAll(rootListing.getListing());
-        }
-      } else {
-        queue.add(meta);
-      }
-    }
-  }
-
-  @Override
-  public boolean hasNext() throws IOException {
-    return !queue.isEmpty();
-  }
-
-  @Override
-  public S3AFileStatus next() throws IOException {
-    if (!hasNext()) {
-      throw new NoSuchElementException("No more descendants.");
-    }
-    PathMetadata next;
-    next = queue.poll();
-    if (next.getFileStatus().isDirectory()) {
-      final Path path = next.getFileStatus().getPath();
-      DirListingMetadata meta = metadataStore.listChildren(path);
-      if (meta != null) {
-        Collection<PathMetadata> more = meta.withoutTombstones().getListing();
-        if (!more.isEmpty()) {
-          queue.addAll(more);
-        }
-      }
-    }
-    return next.getFileStatus();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
deleted file mode 100644
index 63600b3a94202..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-/**
- * {@code DirListingMetadata} models a directory listing stored in a
- * {@link MetadataStore}.  Instances of this class are mutable and thread-safe.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class DirListingMetadata extends ExpirableMetadata {
-
-  /**
-   * Convenience parameter for passing into constructor.
-   */
-  public static final Collection<PathMetadata> EMPTY_DIR =
-      Collections.emptyList();
-
-  private final Path path;
-
-  /** Using a map for fast find / remove with large directories. */
-  private Map<Path, PathMetadata> listMap = new ConcurrentHashMap<>();
-
-  private boolean isAuthoritative;
-
-  /**
-   * Create a directory listing metadata container.
-   *
-   * @param path Path of the directory. If this path has a host component, then
-   *     all paths added later via {@link #put(PathMetadata)} must also have
-   *     the same host.
-   * @param listing Entries in the directory.
-   * @param isAuthoritative true iff listing is the full contents of the
-   *     directory, and the calling client reports that this may be cached as
-   *     the full and authoritative listing of all files in the directory.
-   * @param lastUpdated last updated time on which expiration is based.
-   */
-  public DirListingMetadata(Path path, Collection<PathMetadata> listing,
-      boolean isAuthoritative, long lastUpdated) {
-
-    checkPathAbsolute(path);
-    this.path = path;
-
-    if (listing != null) {
-      for (PathMetadata entry : listing) {
-        Path childPath = entry.getFileStatus().getPath();
-        checkChildPath(childPath);
-        listMap.put(childPath, entry);
-      }
-    }
-    this.isAuthoritative  = isAuthoritative;
-    this.setLastUpdated(lastUpdated);
-  }
-
-  public DirListingMetadata(Path path, Collection<PathMetadata> listing,
-      boolean isAuthoritative) {
-    this(path, listing, isAuthoritative, 0);
-  }
-
-  /**
-   * Copy constructor.
-   * @param d the existing {@link DirListingMetadata} object.
-   */
-  public DirListingMetadata(DirListingMetadata d) {
-    path = d.path;
-    isAuthoritative = d.isAuthoritative;
-    this.setLastUpdated(d.getLastUpdated());
-    listMap = new ConcurrentHashMap<>(d.listMap);
-  }
-
-  /**
-   * @return {@code Path} of the directory that contains this listing.
-   */
-  public Path getPath() {
-    return path;
-  }
-
-  /**
-   * @return entries in the directory
-   */
-  public Collection<PathMetadata> getListing() {
-    return Collections.unmodifiableCollection(listMap.values());
-  }
-
-  /**
-   * List all tombstones.
-   * @return all tombstones in the listing.
-   */
-  public Set<Path> listTombstones() {
-    Set<Path> tombstones = new HashSet<>();
-    for (PathMetadata meta : listMap.values()) {
-      if (meta.isDeleted()) {
-        tombstones.add(meta.getFileStatus().getPath());
-      }
-    }
-    return tombstones;
-  }
-
-  /**
-   * Get the directory listing excluding tombstones.
-   * Returns a new DirListingMetadata instances, without the tombstones -the
-   * lastUpdated field is copied from this instance.
-   * @return a new DirListingMetadata without the tombstones.
-   */
-  public DirListingMetadata withoutTombstones() {
-    Collection<PathMetadata> filteredList = new ArrayList<>();
-    for (PathMetadata meta : listMap.values()) {
-      if (!meta.isDeleted()) {
-        filteredList.add(meta);
-      }
-    }
-    return new DirListingMetadata(path, filteredList, isAuthoritative,
-        this.getLastUpdated());
-  }
-
-  /**
-   * @return number of entries tracked.  This is not the same as the number
-   * of entries in the actual directory unless {@link #isAuthoritative()} is
-   * true.
-   * It will also include any tombstones.
-   */
-  public int numEntries() {
-    return listMap.size();
-  }
-
-  /**
-   * @return true iff this directory listing is full and authoritative within
-   * the scope of the {@code MetadataStore} that returned it.
-   */
-  public boolean isAuthoritative() {
-    return isAuthoritative;
-  }
-
-
-  /**
-   * Is the underlying directory known to be empty?
-   * @return FALSE if directory is known to have a child entry, TRUE if
-   * directory is known to be empty, UNKNOWN otherwise.
-   */
-  public Tristate isEmpty() {
-    if (getListing().isEmpty()) {
-      if (isAuthoritative()) {
-        return Tristate.TRUE;
-      } else {
-        // This listing is empty, but may not be full list of underlying dir.
-        return Tristate.UNKNOWN;
-      }
-    } else { // not empty listing
-      // There exists at least one child, dir not empty.
-      return Tristate.FALSE;
-    }
-  }
-
-  /**
-   * Marks this directory listing as full and authoritative.
-   * @param authoritative see {@link #isAuthoritative()}.
-   */
-  public void setAuthoritative(boolean authoritative) {
-    this.isAuthoritative = authoritative;
-  }
-
-  /**
-   * Lookup entry within this directory listing.  This may return null if the
-   * {@code MetadataStore} only tracks a partial set of the directory entries.
-   * In the case where {@link #isAuthoritative()} is true, however, this
-   * function returns null iff the directory is known not to contain the listing
-   * at given path (within the scope of the {@code MetadataStore} that returned
-   * it).
-   *
-   * @param childPath path of entry to look for.
-   * @return entry, or null if it is not present or not being tracked.
-   */
-  public PathMetadata get(Path childPath) {
-    checkChildPath(childPath);
-    return listMap.get(childPath);
-  }
-
-  /**
-   * Replace an entry with a tombstone.
-   * @param childPath path of entry to replace.
-   */
-  public void markDeleted(Path childPath, long lastUpdated) {
-    checkChildPath(childPath);
-    listMap.put(childPath, PathMetadata.tombstone(childPath, lastUpdated));
-  }
-
-  /**
-   * Remove entry from this directory.
-   *
-   * @param childPath path of entry to remove.
-   */
-  public void remove(Path childPath) {
-    checkChildPath(childPath);
-    listMap.remove(childPath);
-  }
-
-  /**
-   * Add an entry to the directory listing.  If this listing already contains a
-   * {@code FileStatus} with the same path, it will be replaced.
-   *
-   * @param childPathMetadata entry to add to this directory listing.
-   * @return true if the status was added or replaced with a new value. False
-   * if the same FileStatus value was already present.
-   */
-  public boolean put(PathMetadata childPathMetadata) {
-    Preconditions.checkNotNull(childPathMetadata,
-        "childPathMetadata must be non-null");
-    final S3AFileStatus fileStatus = childPathMetadata.getFileStatus();
-    Path childPath = childStatusToPathKey(fileStatus);
-    PathMetadata newValue = childPathMetadata;
-    PathMetadata oldValue = listMap.put(childPath, childPathMetadata);
-    return oldValue == null || !oldValue.equals(newValue);
-  }
-
-  @Override
-  public String toString() {
-    return "DirListingMetadata{" +
-        "path=" + path +
-        ", listMap=" + listMap +
-        ", isAuthoritative=" + isAuthoritative +
-        ", lastUpdated=" + this.getLastUpdated() +
-        '}';
-  }
-
-  /**
-   * Remove expired entries from the listing based on TTL.
-   * @param ttl the ttl time
-   * @param now the current time
-   * @return the expired values.
-   */
-  public synchronized List<PathMetadata> removeExpiredEntriesFromListing(
-      long ttl, long now) {
-    List<PathMetadata> expired = new ArrayList<>();
-    final Iterator<Map.Entry<Path, PathMetadata>> iterator =
-        listMap.entrySet().iterator();
-    while (iterator.hasNext()) {
-      final Map.Entry<Path, PathMetadata> entry = iterator.next();
-      // we filter iff the lastupdated is not 0 and the entry is expired
-      PathMetadata metadata = entry.getValue();
-      if (metadata.getLastUpdated() != 0
-          && (metadata.getLastUpdated() + ttl) <= now) {
-        expired.add(metadata);
-        iterator.remove();
-      }
-    }
-    return expired;
-  }
-
-  /**
-   * Log contents to supplied StringBuilder in a pretty fashion.
-   * @param sb target StringBuilder
-   */
-  public void prettyPrint(StringBuilder sb) {
-    sb.append(String.format("DirMeta %-20s %-18s",
-        path.toString(),
-        isAuthoritative ? "Authoritative" : "Not Authoritative"));
-    for (Map.Entry<Path, PathMetadata> entry : listMap.entrySet()) {
-      sb.append("\n   key: ").append(entry.getKey()).append(": ");
-      entry.getValue().prettyPrint(sb);
-    }
-    sb.append("\n");
-  }
-
-  public String prettyPrint() {
-    StringBuilder sb = new StringBuilder();
-    prettyPrint(sb);
-    return sb.toString();
-  }
-
-  /**
-   * Checks that child path is valid.
-   * @param childPath path to check.
-   */
-  private void checkChildPath(Path childPath) {
-    checkPathAbsolute(childPath);
-
-    // If this dir's path has host (and thus scheme), so must its children
-    URI parentUri = path.toUri();
-    URI childUri = childPath.toUri();
-    if (parentUri.getHost() != null) {
-      Preconditions.checkNotNull(childUri.getHost(), "Expected non-null URI " +
-          "host: %s", childUri);
-      Preconditions.checkArgument(
-          childUri.getHost().equals(parentUri.getHost()),
-          "childUri %s and parentUri %s must have the same host",
-          childUri, parentUri);
-      Preconditions.checkNotNull(childUri.getScheme(), "No scheme in path %s",
-          childUri);
-    }
-    Preconditions.checkArgument(!childPath.isRoot(),
-        "childPath cannot be the root path: %s", childPath);
-    Preconditions.checkArgument(parentUri.getPath().equals(
-        childPath.getParent().toUri().getPath()),
-        "childPath %s must be a child of %s", childPath, path);
-  }
-
-  /**
-   * For Paths that are handed in directly, we assert they are in consistent
-   * format with checkPath().  For paths that are supplied embedded in
-   * FileStatus, we attempt to fill in missing scheme and host, when this
-   * DirListingMetadata is associated with one.
-   *
-   * @return Path suitable for consistent hashtable lookups
-   * @throws NullPointerException null status argument
-   * @throws IllegalArgumentException bad status values or failure to
-   *                                  create a URI.
-   */
-  private Path childStatusToPathKey(FileStatus status) {
-    Path p = status.getPath();
-    Preconditions.checkNotNull(p, "Child status' path cannot be null");
-    Preconditions.checkArgument(!p.isRoot(),
-        "childPath cannot be the root path: %s", p);
-    Preconditions.checkArgument(p.getParent().equals(path),
-        "childPath %s must be a child of %s", p, path);
-    URI uri = p.toUri();
-    URI parentUri = path.toUri();
-    // If FileStatus' path is missing host, but should have one, add it.
-    if (uri.getHost() == null && parentUri.getHost() != null) {
-      try {
-        return new Path(new URI(parentUri.getScheme(), parentUri.getHost(),
-            uri.getPath(), uri.getFragment()));
-      } catch (URISyntaxException e) {
-        throw new IllegalArgumentException("FileStatus path invalid with" +
-            " added " + parentUri.getScheme() + "://" + parentUri.getHost() +
-            " added", e);
-      }
-    }
-    return p;
-  }
-
-  private void checkPathAbsolute(Path p) {
-    Preconditions.checkNotNull(p, "path must be non-null");
-    Preconditions.checkArgument(p.isAbsolute(), "path must be absolute: %s", p);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java
deleted file mode 100644
index e4c9ef6de8ba3..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DumpS3GuardDynamoTable.java
+++ /dev/null
@@ -1,792 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.URI;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Deque;
-import java.util.List;
-
-import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.Listing;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
-import org.apache.hadoop.fs.s3a.S3ListRequest;
-import org.apache.hadoop.fs.store.audit.AuditSpan;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.launcher.LauncherExitCodes;
-import org.apache.hadoop.service.launcher.ServiceLaunchException;
-import org.apache.hadoop.service.launcher.ServiceLauncher;
-import org.apache.hadoop.util.DurationInfo;
-import org.apache.hadoop.util.ExitUtil;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.S3AUtils.ACCEPT_ALL;
-
-/**
- * This is a low-level diagnostics entry point which does a CVE/TSV dump of
- * the DDB state.
- * As it also lists the filesystem, it actually changes the state of the store
- * during the operation.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class DumpS3GuardDynamoTable extends AbstractS3GuardDynamoDBDiagnostic {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(DumpS3GuardDynamoTable.class);
-
-  /**
-   * Application name.
-   */
-  public static final String NAME = "DumpS3GuardDynamoTable";
-
-  /**
-   * Usage.
-   */
-  private static final String USAGE_MESSAGE = NAME
-      + " <filesystem> <dest-file>";
-
-  /**
-   * Suffix for the flat list: {@value}.
-   */
-  public static final String FLAT_CSV = "-flat.csv";
-
-  /**
-   * Suffix for the raw S3 dump: {@value}.
-   */
-  public static final String RAW_CSV = "-s3.csv";
-
-  /**
-   * Suffix for the DDB scan: {@value}.
-   */
-  public static final String SCAN_CSV = "-scan.csv";
-
-  /**
-   * Suffix for the second DDB scan: : {@value}.
-   */
-  public static final String SCAN2_CSV = "-scan-2.csv";
-
-  /**
-   * Suffix for the treewalk scan of the S3A Filesystem: {@value}.
-   */
-  public static final String TREE_CSV = "-tree.csv";
-
-  /**
-   * Suffix for a recursive treewalk through the metastore: {@value}.
-   */
-  public static final String STORE_CSV = "-store.csv";
-
-  /**
-   * Path in the local filesystem to save the data.
-   */
-  private String destPath;
-
-  private Pair<Long, Long> scanEntryResult;
-
-  private Pair<Long, Long> secondScanResult;
-
-  private long rawObjectStoreCount;
-
-  private long listStatusCount;
-
-  private long treewalkCount;
-
-  /**
-   * Instantiate.
-   * @param name application name.
-   */
-  public DumpS3GuardDynamoTable(final String name) {
-    super(name);
-  }
-
-  /**
-   * Instantiate with default name.
-   */
-  public DumpS3GuardDynamoTable() {
-    this(NAME);
-  }
-
-  /**
-   * Bind to a specific FS + store.
-   * @param fs filesystem
-   * @param store metastore to use
-   * @param destFile the base filename for output
-   * @param uri URI of store -only needed if FS is null.
-   */
-  public DumpS3GuardDynamoTable(
-      final S3AFileSystem fs,
-      final DynamoDBMetadataStore store,
-      final File destFile,
-      final URI uri) {
-    super(NAME, fs, store, uri);
-    this.destPath = destFile.getAbsolutePath();
-  }
-
-  /**
-   * Bind to the argument list, including validating the CLI.
-   * @throws Exception failure.
-   */
-  @Override
-  protected void serviceStart() throws Exception {
-    if (getStore() == null) {
-      List<String> arg = getArgumentList(2, 2, USAGE_MESSAGE);
-      bindFromCLI(arg.get(0));
-      destPath = arg.get(1);
-    }
-  }
-
-  /**
-   * Dump the filesystem and the metastore.
-   * @return the exit code.
-   * @throws ServiceLaunchException on failure.
-   * @throws IOException IO failure.
-   */
-  @Override
-  public int execute() throws ServiceLaunchException, IOException {
-
-    try {
-      final File scanFile = new File(
-          destPath + SCAN_CSV).getCanonicalFile();
-      File parentDir = scanFile.getParentFile();
-      if (!parentDir.mkdirs() && !parentDir.isDirectory()) {
-        throw new PathIOException(parentDir.toString(),
-            "Could not create destination directory");
-      }
-
-      try (CsvFile csv = new CsvFile(scanFile);
-           DurationInfo ignored = new DurationInfo(LOG,
-               "scanFile dump to %s", scanFile)) {
-        scanEntryResult = scanMetastore(csv);
-      }
-
-      if (getFilesystem() != null) {
-
-        Path basePath = getFilesystem().qualify(new Path(getUri()));
-
-        final File destFile = new File(destPath + STORE_CSV)
-            .getCanonicalFile();
-        LOG.info("Writing Store details to {}", destFile);
-        try (CsvFile csv = new CsvFile(destFile);
-             DurationInfo ignored = new DurationInfo(LOG, "List metastore")) {
-
-          LOG.info("Base path: {}", basePath);
-          dumpMetastore(csv, basePath);
-        }
-
-        // these operations all update the metastore as they list,
-        // that is: they are side-effecting.
-        final File treewalkFile = new File(destPath + TREE_CSV)
-            .getCanonicalFile();
-
-        try (CsvFile csv = new CsvFile(treewalkFile);
-             DurationInfo ignored = new DurationInfo(LOG,
-                 "Treewalk to %s", treewalkFile)) {
-          treewalkCount = treewalkFilesystem(csv, basePath);
-        }
-        final File flatlistFile = new File(
-            destPath + FLAT_CSV).getCanonicalFile();
-
-        try (CsvFile csv = new CsvFile(flatlistFile);
-             DurationInfo ignored = new DurationInfo(LOG,
-                 "Flat list to %s", flatlistFile)) {
-          listStatusCount = listStatusFilesystem(csv, basePath);
-        }
-        final File rawFile = new File(
-            destPath + RAW_CSV).getCanonicalFile();
-
-        try (CsvFile csv = new CsvFile(rawFile);
-             DurationInfo ignored = new DurationInfo(LOG,
-                 "Raw dump to %s", rawFile)) {
-          rawObjectStoreCount = dumpRawS3ObjectStore(csv);
-        }
-        final File scanFile2 = new File(
-            destPath + SCAN2_CSV).getCanonicalFile();
-
-        try (CsvFile csv = new CsvFile(scanFile);
-             DurationInfo ignored = new DurationInfo(LOG,
-                 "scanFile dump to %s", scanFile2)) {
-          secondScanResult = scanMetastore(csv);
-        }
-      }
-
-      return LauncherExitCodes.EXIT_SUCCESS;
-    } catch (IOException | RuntimeException e) {
-      LOG.error("failure", e);
-      throw e;
-    }
-  }
-
-  /**
-   * Push all elements of a list to a queue, such that the first entry
-   * on the list becomes the head of the queue.
-   * @param queue queue to update
-   * @param entries list of entries to add.
-   * @param <T> type of queue
-   */
-  private <T> void pushAll(Deque<T> queue, List<T> entries) {
-    Collections.reverse(entries);
-    for (T t : entries) {
-      queue.push(t);
-    }
-  }
-
-  /**
-   * Dump the filesystem via a treewalk.
-   * If metastore entries mark directories as deleted, this
-   * walk will not explore them.
-   * @param csv destination.
-   * @param base base path.
-   * @return number of entries found.
-   * @throws IOException IO failure.
-   */
-  protected long treewalkFilesystem(
-      final CsvFile csv,
-      final Path base) throws IOException {
-    ArrayDeque<Path> queue = new ArrayDeque<>();
-    queue.add(base);
-    long count = 0;
-    while (!queue.isEmpty()) {
-      Path path = queue.pop();
-      count++;
-      FileStatus[] fileStatuses;
-      try {
-        fileStatuses = getFilesystem().listStatus(path);
-      } catch (FileNotFoundException e) {
-        LOG.warn("File {} was not found", path);
-        continue;
-      }
-      // entries
-      for (FileStatus fileStatus : fileStatuses) {
-        csv.entry((S3AFileStatus) fileStatus);
-      }
-      // scan through the list, building up a reverse list of all directories
-      // found.
-      List<Path> dirs = new ArrayList<>(fileStatuses.length);
-      for (FileStatus fileStatus : fileStatuses) {
-        if (fileStatus.isDirectory()
-            && !(fileStatus.getPath().equals(path))) {
-          // directory: add to the end of the queue.
-          dirs.add(fileStatus.getPath());
-        } else {
-          // file: just increment the count
-          count++;
-        }
-        // now push the dirs list in reverse
-        // so that they have been added in the sort order as returned.
-        pushAll(queue, dirs);
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Dump the filesystem via a recursive listStatus call.
-   * @param csv destination.
-   * @return number of entries found.
-   * @throws IOException IO failure.
-   */
-  protected long listStatusFilesystem(
-      final CsvFile csv,
-      final Path path) throws IOException {
-    long count = 0;
-    RemoteIterator<S3ALocatedFileStatus> iterator = getFilesystem()
-        .listFilesAndEmptyDirectories(path, true);
-    while (iterator.hasNext()) {
-      S3ALocatedFileStatus status = iterator.next();
-      csv.entry(status.toS3AFileStatus());
-    }
-    return count;
-  }
-
-  /**
-   * Dump the raw S3 Object Store.
-   * @param csv destination.
-   * @return number of entries found.
-   * @throws IOException IO failure.
-   */
-  protected long dumpRawS3ObjectStore(
-      final CsvFile csv) throws IOException {
-    S3AFileSystem fs = getFilesystem();
-    long count = 0;
-    Path rootPath = fs.qualify(new Path("/"));
-    try (AuditSpan span = fs.createSpan("DumpS3GuardDynamoTable",
-        rootPath.toString(), null)) {
-      Listing listing = fs.getListing();
-      S3ListRequest request = listing.createListObjectsRequest("", null, span);
-      count = 0;
-      RemoteIterator<S3AFileStatus> st =
-          listing.createFileStatusListingIterator(rootPath, request,
-              ACCEPT_ALL,
-              new Listing.AcceptAllButSelfAndS3nDirs(rootPath),
-              span);
-      while (st.hasNext()) {
-        count++;
-        S3AFileStatus next = st.next();
-        LOG.debug("[{}] {}", count, next);
-        csv.entry(next);
-      }
-      LOG.info("entry count: {}", count);
-    }
-    return count;
-  }
-
-  /**
-   * list children under the metastore from a base path, through
-   * a recursive query + walk strategy.
-   * @param csv dest
-   * @param basePath base path
-   * @throws IOException failure.
-   */
-  protected void dumpMetastore(final CsvFile csv,
-      final Path basePath) throws IOException {
-    dumpStoreEntries(csv, getStore().listChildren(basePath));
-  }
-
-  /**
-   * Recursive Store Dump.
-   * @param csv open CSV file.
-   * @param dir directory listing
-   * @return (directories, files)
-   * @throws IOException failure
-   */
-  private Pair<Long, Long> dumpStoreEntries(
-      CsvFile csv,
-      DirListingMetadata dir) throws IOException {
-    ArrayDeque<DirListingMetadata> queue = new ArrayDeque<>();
-    queue.add(dir);
-    long files = 0, dirs = 1;
-    while (!queue.isEmpty()) {
-      DirListingMetadata next = queue.pop();
-      List<DDBPathMetadata> childDirs = new ArrayList<>();
-      Collection<PathMetadata> listing = next.getListing();
-      // sort by name
-      List<PathMetadata> sorted = new ArrayList<>(listing);
-      sorted.sort(new PathOrderComparators.PathMetadataComparator(
-          (l, r) -> l.compareTo(r)));
-
-      for (PathMetadata pmd : sorted) {
-        DDBPathMetadata ddbMd = (DDBPathMetadata) pmd;
-        dumpEntry(csv, ddbMd);
-        if (ddbMd.getFileStatus().isDirectory()) {
-          childDirs.add(ddbMd);
-        } else {
-          files++;
-        }
-      }
-      List<DirListingMetadata> childMD = new ArrayList<>(childDirs.size());
-      for (DDBPathMetadata childDir : childDirs) {
-        childMD.add(getStore().listChildren(
-            childDir.getFileStatus().getPath()));
-      }
-      pushAll(queue, childMD);
-    }
-
-    return Pair.of(dirs, files);
-  }
-
-
-  /**
-   * Dump a single entry, and log it.
-   * @param csv CSV output file.
-   * @param md metadata to log.
-   */
-  private void dumpEntry(CsvFile csv, DDBPathMetadata md) {
-    LOG.debug("{}", md.prettyPrint());
-    csv.entry(md);
-  }
-
-  /**
-   * Scan the metastore for all entries and dump them.
-   * There's no attempt to sort the output.
-   * @param csv file
-   * @return tuple of (live entries, tombstones).
-   */
-  private Pair<Long, Long> scanMetastore(CsvFile csv) {
-    S3GuardTableAccess tableAccess = new S3GuardTableAccess(getStore());
-    ExpressionSpecBuilder builder = new ExpressionSpecBuilder();
-    Iterable<DDBPathMetadata> results =
-        getStore().wrapWithRetries(tableAccess.scanMetadata(builder));
-    long live = 0;
-    long tombstone = 0;
-    for (DDBPathMetadata md : results) {
-      if (!(md instanceof S3GuardTableAccess.VersionMarker)) {
-        // print it
-        csv.entry(md);
-        if (md.isDeleted()) {
-          tombstone++;
-        } else {
-          live++;
-        }
-
-      }
-    }
-    return Pair.of(live, tombstone);
-  }
-
-  public Pair<Long, Long> getScanEntryResult() {
-    return scanEntryResult;
-  }
-
-  public Pair<Long, Long> getSecondScanResult() {
-    return secondScanResult;
-  }
-
-  public long getRawObjectStoreCount() {
-    return rawObjectStoreCount;
-  }
-
-  public long getListStatusCount() {
-    return listStatusCount;
-  }
-
-  public long getTreewalkCount() {
-    return treewalkCount;
-  }
-
-  /**
-   * Convert a timestamp in milliseconds to a human string.
-   * @param millis epoch time in millis
-   * @return a string for the CSV file.
-   */
-  private static String stringify(long millis) {
-    return new Date(millis).toString();
-  }
-
-  /**
-   * This is the JVM entry point for the service launcher.
-   *
-   * Converts the arguments to a list, then invokes
-   * {@link #serviceMain(List, AbstractS3GuardDynamoDBDiagnostic)}.
-   * @param args command line arguments.
-   */
-  public static void main(String[] args) {
-    try {
-      serviceMain(Arrays.asList(args), new DumpS3GuardDynamoTable());
-    } catch (ExitUtil.ExitException e) {
-      ExitUtil.terminate(e);
-    }
-  }
-
-  /**
-   * The real main function, which takes the arguments as a list.
-   * Argument 0 MUST be the service classname
-   * @param argsList the list of arguments
-   * @param service service to launch.
-   */
-  static void serviceMain(
-      final List<String> argsList,
-      final AbstractS3GuardDynamoDBDiagnostic service) {
-    ServiceLauncher<Service> serviceLauncher =
-        new ServiceLauncher<>(service.getName());
-
-    ExitUtil.ExitException ex = serviceLauncher.launchService(
-        new Configuration(),
-        service,
-        argsList,
-        false,
-        true);
-    if (ex != null) {
-      throw ex;
-    }
-  }
-
-  /**
-   * Entry point to dump the metastore and s3 store world views
-   * <p>
-   * Both the FS and the store will be dumped: the store is scanned
-   * before and after the sequence to show what changes were made to
-   * the store during the list operation.
-   * @param fs fs to dump. If null a store must be provided.
-   * @param store store to dump (fallback to FS)
-   * @param conf configuration to use (fallback to fs)
-   * @param destFile base name of the output files.
-   * @param uri URI of store -only needed if FS is null.
-   * @throws ExitUtil.ExitException failure.
-   * @return the store
-   */
-  public static DumpS3GuardDynamoTable dumpStore(
-      @Nullable final S3AFileSystem fs,
-      @Nullable DynamoDBMetadataStore store,
-      @Nullable Configuration conf,
-      final File destFile,
-      @Nullable URI uri) throws ExitUtil.ExitException {
-    ServiceLauncher<Service> serviceLauncher =
-        new ServiceLauncher<>(NAME);
-
-    if (conf == null) {
-      conf = checkNotNull(fs, "No filesystem").getConf();
-    }
-    if (store == null) {
-      store = (DynamoDBMetadataStore) checkNotNull(fs, "No filesystem")
-          .getMetadataStore();
-    }
-    DumpS3GuardDynamoTable dump = new DumpS3GuardDynamoTable(fs,
-        store,
-        destFile,
-        uri);
-    ExitUtil.ExitException ex = serviceLauncher.launchService(
-        conf,
-        dump,
-        Collections.emptyList(),
-        false,
-        true);
-    if (ex != null && ex.getExitCode() != 0) {
-      throw ex;
-    }
-    LOG.info("Results:");
-    Pair<Long, Long> r = dump.getScanEntryResult();
-    LOG.info("Metastore entries: {}", r);
-    LOG.info("Metastore scan total {}, entries {}, tombstones {}",
-        r.getLeft() + r.getRight(),
-        r.getLeft(),
-        r.getRight());
-    LOG.info("S3 count {}", dump.getRawObjectStoreCount());
-    LOG.info("Treewalk Count {}", dump.getTreewalkCount());
-    LOG.info("List Status Count {}", dump.getListStatusCount());
-    r = dump.getSecondScanResult();
-    if (r != null) {
-      LOG.info("Second metastore scan total {}, entries {}, tombstones {}",
-          r.getLeft() + r.getRight(),
-          r.getLeft(),
-          r.getRight());
-    }
-    return dump;
-  }
-
-  /**
-   * Writer for generating test CSV files.
-   *
-   * Quotes are manged by passing in a long whose specific bits control
-   * whether or not a row is quoted, bit 0 for column 0, etc.
-   *
-   * There is no escaping of values here.
-   */
-  private static final class CsvFile implements Closeable {
-
-
-    /** constant to quote all columns. */
-    public static final long ALL_QUOTES = 0x7fffffff;
-
-    /** least significant bit is used for first column; 1  mean 'quote'. */
-    public static final int ROW_QUOTE_MAP = 0b1110_1001_1111;
-
-    /** quote nothing: {@value}. */
-    public static final long NO_QUOTES = 0;
-
-    private final Path path;
-
-    private final PrintWriter out;
-
-    private final String separator;
-
-    private final String eol;
-
-    private final String quote;
-
-    /**
-     * Create.
-     * @param path filesystem path.
-     * @param out output write.
-     * @param separator separator of entries.
-     * @param eol EOL marker.
-     * @param quote quote marker.
-     * @throws IOException failure.
-     */
-    private CsvFile(
-        final Path path,
-        final PrintWriter out,
-        final String separator,
-        final String eol,
-        final String quote) throws IOException {
-      this.separator = checkNotNull(separator);
-      this.eol = checkNotNull(eol);
-      this.quote = checkNotNull(quote);
-      this.path = path;
-      this.out = checkNotNull(out);
-      header();
-    }
-
-    /**
-     * Create to a file, with UTF-8 output and the standard
-     * options of the TSV file.
-     * @param file destination file.
-     * @throws IOException failure.
-     */
-    private CsvFile(File file) throws IOException {
-      this(null,
-          new PrintWriter(file, "UTF-8"), "\t", "\n", "\"");
-    }
-
-    /**
-     * Close the file, if not already done.
-     * @throws IOException on a failure.
-     */
-    @Override
-    public synchronized void close() throws IOException {
-      if (out != null) {
-        out.close();
-      }
-    }
-
-    public Path getPath() {
-      return path;
-    }
-
-    public String getSeparator() {
-      return separator;
-    }
-
-    public String getEol() {
-      return eol;
-    }
-
-    /**
-     * Write a row.
-     * Entries are quoted if the bit for that column is true.
-     * @param quotes quote policy: every bit defines the rule for that element
-     * @param columns columns to write
-     * @return self for ease of chaining.
-     */
-    public CsvFile row(long quotes, Object... columns) {
-      checkNotNull(out);
-      for (int i = 0; i < columns.length; i++) {
-        if (i != 0) {
-          out.write(separator);
-        }
-        boolean toQuote = (quotes & 1) == 1;
-        // unsigned right shift to make next column flag @ position 0
-        quotes = quotes >>> 1;
-        if (toQuote) {
-          out.write(quote);
-        }
-        Object column = columns[i];
-        out.write(column != null ? column.toString() : "");
-        if (toQuote) {
-          out.write(quote);
-        }
-      }
-      out.write(eol);
-      return this;
-    }
-
-    /**
-     * Write a line.
-     * @param line line to print
-     * @return self for ease of chaining.
-     */
-    public CsvFile line(String line) {
-      out.write(line);
-      out.write(eol);
-      return this;
-    }
-
-    /**
-     * Get the output stream.
-     * @return the stream.
-     */
-    public PrintWriter getOut() {
-      return out;
-    }
-
-    /**
-     * Print the header.
-     */
-    void header() {
-      row(CsvFile.ALL_QUOTES,
-          "type",
-          "deleted",
-          "path",
-          "is_auth_dir",
-          "is_empty_dir",
-          "len",
-          "updated",
-          "updated_s",
-          "last_modified",
-          "last_modified_s",
-          "etag",
-          "version");
-    }
-
-    /**
-     * Add a metadata entry.
-     * @param md metadata.
-     */
-    void entry(DDBPathMetadata md) {
-      S3AFileStatus fileStatus = md.getFileStatus();
-      row(ROW_QUOTE_MAP,
-          fileStatus.isDirectory() ? "dir" : "file",
-          md.isDeleted(),
-          fileStatus.getPath().toString(),
-          md.isAuthoritativeDir(),
-          md.isEmptyDirectory().name(),
-          fileStatus.getLen(),
-          md.getLastUpdated(),
-          stringify(md.getLastUpdated()),
-          fileStatus.getModificationTime(),
-          stringify(fileStatus.getModificationTime()),
-          fileStatus.getETag(),
-          fileStatus.getVersionId());
-    }
-
-    /**
-     * filesystem entry: no metadata.
-     * @param fileStatus file status
-     */
-    void entry(S3AFileStatus fileStatus) {
-      row(ROW_QUOTE_MAP,
-          fileStatus.isDirectory() ? "dir" : "file",
-          "false",
-          fileStatus.getPath().toString(),
-          "",
-          fileStatus.isEmptyDirectory().name(),
-          fileStatus.getLen(),
-          "",
-          "",
-          fileStatus.getModificationTime(),
-          stringify(fileStatus.getModificationTime()),
-          fileStatus.getETag(),
-          fileStatus.getVersionId());
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
deleted file mode 100644
index e4275ccb51c4c..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
-
-/**
- * Interface to create a DynamoDB client.
- *
- * Implementation should be configured for setting and getting configuration.
- */
-@InterfaceAudience.Private
-public interface DynamoDBClientFactory extends Configurable {
-  Logger LOG = LoggerFactory.getLogger(DynamoDBClientFactory.class);
-
-  /**
-   * Create a DynamoDB client object from configuration.
-   *
-   * The DynamoDB client to create does not have to relate to any S3 buckets.
-   * All information needed to create a DynamoDB client is from the hadoop
-   * configuration. Specially, if the region is not configured, it will use the
-   * provided region parameter. If region is neither configured nor provided,
-   * it will indicate an error.
-   *
-   * @param defaultRegion the default region of the AmazonDynamoDB client
-   * @param bucket Optional bucket to use to look up per-bucket proxy secrets
-   * @param credentials credentials to use for authentication.
-   * @return a new DynamoDB client
-   * @throws IOException if any IO error happens
-   */
-  AmazonDynamoDB createDynamoDBClient(final String defaultRegion,
-      final String bucket,
-      final AWSCredentialsProvider credentials) throws IOException;
-
-  /**
-   * The default implementation for creating an AmazonDynamoDB.
-   */
-  class DefaultDynamoDBClientFactory extends Configured
-      implements DynamoDBClientFactory {
-    @Override
-    public AmazonDynamoDB createDynamoDBClient(String defaultRegion,
-        final String bucket,
-        final AWSCredentialsProvider credentials)
-        throws IOException {
-      Preconditions.checkNotNull(getConf(),
-          "Should have been configured before usage");
-
-      final Configuration conf = getConf();
-      final ClientConfiguration awsConf = S3AUtils
-          .createAwsConf(conf, bucket, Constants.AWS_SERVICE_IDENTIFIER_DDB);
-
-      final String region = getRegion(conf, defaultRegion);
-      LOG.debug("Creating DynamoDB client in region {}", region);
-
-      return AmazonDynamoDBClientBuilder.standard()
-          .withCredentials(credentials)
-          .withClientConfiguration(awsConf)
-          .withRegion(region)
-          .build();
-    }
-
-    /**
-     * Helper method to get and validate the AWS region for DynamoDBClient.
-     *
-     * @param conf configuration
-     * @param defaultRegion the default region
-     * @return configured region or else the provided default region
-     * @throws IOException if the region is not valid
-     */
-    static String getRegion(Configuration conf, String defaultRegion)
-        throws IOException {
-      String region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
-      if (StringUtils.isEmpty(region)) {
-        region = defaultRegion;
-      }
-      try {
-        Regions.fromName(region);
-      } catch (IllegalArgumentException | NullPointerException e) {
-        throw new IOException("Invalid region specified: " + region + "; " +
-            "Region can be configured with " + S3GUARD_DDB_REGION_KEY + ": " +
-            validRegionsString());
-      }
-      return region;
-    }
-
-    private static String validRegionsString() {
-      final String delimiter = ", ";
-      Regions[] regions = Regions.values();
-      StringBuilder sb = new StringBuilder();
-      for (int i = 0; i < regions.length; i++) {
-        if (i > 0) {
-          sb.append(delimiter);
-        }
-        sb.append(regions[i].getName());
-      }
-      return sb.toString();
-
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
deleted file mode 100644
index a031d33ba3d5c..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ /dev/null
@@ -1,2534 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.UncheckedIOException;
-import java.net.URI;
-import java.nio.file.AccessDeniedException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.ItemCollection;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.document.PutItemOutcome;
-import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
-import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.document.TableWriteItems;
-import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport;
-import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
-import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
-import com.amazonaws.services.dynamodbv2.document.utils.ValueMap;
-import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
-import com.amazonaws.services.dynamodbv2.model.TableDescription;
-import com.amazonaws.services.dynamodbv2.model.WriteRequest;
-
-import org.apache.hadoop.fs.s3a.impl.InternalConstants;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.util.functional.CallableRaisingIOE;
-import org.apache.hadoop.util.functional.RemoteIterators;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.AWSServiceThrottledException;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3a.auth.RolePolicies;
-import org.apache.hadoop.fs.s3a.auth.delegation.AWSPolicyProvider;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
-import org.apache.hadoop.util.DurationInfo;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowAllDynamoDBOperations;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3GuardClientOperations;
-import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
-import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
-import static org.apache.hadoop.fs.s3a.s3guard.PathOrderComparators.TOPMOST_PM_LAST;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.*;
-
-/**
- * DynamoDBMetadataStore is a {@link MetadataStore} that persists
- * file system metadata to DynamoDB.
- *
- * The current implementation uses a schema consisting of a single table.  The
- * name of the table can be configured by config key
- * {@link org.apache.hadoop.fs.s3a.Constants#S3GUARD_DDB_TABLE_NAME_KEY}.
- * By default, it matches the name of the S3 bucket.  Each item in the table
- * represents a single directory or file.  Its path is split into separate table
- * attributes:
- * <ul>
- * <li> parent (absolute path of the parent, with bucket name inserted as
- * first path component). </li>
- * <li> child (path of that specific child, relative to parent). </li>
- * <li> optional boolean attribute tracking whether the path is a directory.
- *      Absence or a false value indicates the path is a file. </li>
- * <li> optional long attribute revealing modification time of file.
- *      This attribute is meaningful only to file items.</li>
- * <li> optional long attribute revealing file length.
- *      This attribute is meaningful only to file items.</li>
- * <li> optional long attribute revealing block size of the file.
- *      This attribute is meaningful only to file items.</li>
- * <li> optional string attribute tracking the s3 eTag of the file.
- *      May be absent if the metadata was entered with a version of S3Guard
- *      before this was tracked.
- *      This attribute is meaningful only to file items.</li>
-  * <li> optional string attribute tracking the s3 versionId of the file.
- *      May be absent if the metadata was entered with a version of S3Guard
- *      before this was tracked.
- *      This attribute is meaningful only to file items.</li>
- * </ul>
- *
- * The DynamoDB partition key is the parent, and the range key is the child.
- *
- * To allow multiple buckets to share the same DynamoDB table, the bucket
- * name is treated as the root directory.
- *
- * For example, assume the consistent store contains metadata representing this
- * file system structure:
- *
- * <pre>
- * s3a://bucket/dir1
- * |-- dir2
- * |   |-- file1
- * |   `-- file2
- * `-- dir3
- *     |-- dir4
- *     |   `-- file3
- *     |-- dir5
- *     |   `-- file4
- *     `-- dir6
- * </pre>
- *
- * This is persisted to a single DynamoDB table as:
- *
- * <pre>
- * ====================================================================================
- * | parent                 | child | is_dir | mod_time | len | etag | ver_id |  ...  |
- * ====================================================================================
- * | /bucket                | dir1  | true   |          |     |      |        |       |
- * | /bucket/dir1           | dir2  | true   |          |     |      |        |       |
- * | /bucket/dir1           | dir3  | true   |          |     |      |        |       |
- * | /bucket/dir1/dir2      | file1 |        |   100    | 111 | abc  |  mno   |       |
- * | /bucket/dir1/dir2      | file2 |        |   200    | 222 | def  |  pqr   |       |
- * | /bucket/dir1/dir3      | dir4  | true   |          |     |      |        |       |
- * | /bucket/dir1/dir3      | dir5  | true   |          |     |      |        |       |
- * | /bucket/dir1/dir3/dir4 | file3 |        |   300    | 333 | ghi  |  stu   |       |
- * | /bucket/dir1/dir3/dir5 | file4 |        |   400    | 444 | jkl  |  vwx   |       |
- * | /bucket/dir1/dir3      | dir6  | true   |          |     |      |        |       |
- * ====================================================================================
- * </pre>
- *
- * This choice of schema is efficient for read access patterns.
- * {@link #get(Path)} can be served from a single item lookup.
- * {@link #listChildren(Path)} can be served from a query against all rows
- * matching the parent (the partition key) and the returned list is guaranteed
- * to be sorted by child (the range key).  Tracking whether or not a path is a
- * directory helps prevent unnecessary queries during traversal of an entire
- * sub-tree.
- *
- * Some mutating operations, notably
- * {@link MetadataStore#deleteSubtree(Path, BulkOperationState)} and
- * {@link MetadataStore#move(Collection, Collection, BulkOperationState)}
- * are less efficient with this schema.
- * They require mutating multiple items in the DynamoDB table.
- *
- * By default, DynamoDB access is performed within the same AWS region as
- * the S3 bucket that hosts the S3A instance.  During initialization, it checks
- * the location of the S3 bucket and creates a DynamoDB client connected to the
- * same region. The region may also be set explicitly by setting the config
- * parameter {@code fs.s3a.s3guard.ddb.region} to the corresponding region.
- */
-@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class DynamoDBMetadataStore implements MetadataStore,
-    AWSPolicyProvider {
-  public static final Logger LOG = LoggerFactory.getLogger(
-      DynamoDBMetadataStore.class);
-
-  /**
-   * Name of the operations log.
-   */
-  public static final String OPERATIONS_LOG_NAME =
-      "org.apache.hadoop.fs.s3a.s3guard.Operations";
-
-  /**
-   * A log of all state changing operations to the store;
-   * only updated at debug level.
-   */
-  public static final Logger OPERATIONS_LOG = LoggerFactory.getLogger(
-      OPERATIONS_LOG_NAME);
-
-  /** parent/child name to use in the version marker. */
-  public static final String VERSION_MARKER_ITEM_NAME = "../VERSION";
-
-  /** parent/child name to use in the version marker. */
-  public static final String VERSION_MARKER_TAG_NAME = "s3guard_version";
-
-  /** Current version number. */
-  public static final int VERSION = 100;
-
-  @VisibleForTesting
-  static final String BILLING_MODE
-      = "billing-mode";
-
-  @VisibleForTesting
-  static final String BILLING_MODE_PER_REQUEST
-      = "per-request";
-
-  @VisibleForTesting
-  static final String BILLING_MODE_PROVISIONED
-      = "provisioned";
-
-  @VisibleForTesting
-  static final String DESCRIPTION
-      = "S3Guard metadata store in DynamoDB";
-  @VisibleForTesting
-  static final String READ_CAPACITY = "read-capacity";
-  @VisibleForTesting
-  static final String WRITE_CAPACITY = "write-capacity";
-  @VisibleForTesting
-  static final String STATUS = "status";
-  @VisibleForTesting
-  static final String TABLE = "table";
-
-  @VisibleForTesting
-  static final String HINT_DDB_IOPS_TOO_LOW
-      = " This may be because the write threshold of DynamoDB is set too low.";
-
-  @VisibleForTesting
-  static final String THROTTLING = "Throttling";
-
-  public static final String E_ON_DEMAND_NO_SET_CAPACITY
-      = "Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when BillingMode is PAY_PER_REQUEST";
-
-  @VisibleForTesting
-  static final String E_INCONSISTENT_UPDATE
-      = "Duplicate and inconsistent entry in update operation";
-
-  private static final ValueMap DELETE_TRACKING_VALUE_MAP =
-      new ValueMap().withBoolean(":false", false);
-
-  /**
-   * The maximum number of outstanding operations to submit
-   * before blocking to await completion of all the executors.
-   * Paging work like this is less efficient, but it ensures that
-   * failure (auth, network, etc) are picked up before many more
-   * operations are submitted.
-   *
-   * Arbitrary Choice.
-   * Value: {@value}.
-   */
-  private static final int S3GUARD_DDB_SUBMITTED_TASK_LIMIT = 50;
-
-  private AmazonDynamoDB amazonDynamoDB;
-  private DynamoDB dynamoDB;
-  private AWSCredentialProviderList credentials;
-  private String region;
-  private Table table;
-  private String tableName;
-  private Configuration conf;
-  private String username;
-
-  /**
-   * This policy is mostly for batched writes, not for processing
-   * exceptions in invoke() calls.
-   * It also has a role purpose in
-   * {@link DynamoDBMetadataStoreTableManager#getVersionMarkerItem()};
-   * look at that method for the details.
-   */
-  private RetryPolicy batchWriteRetryPolicy;
-
-  /**
-   * The instrumentation is never null -if/when bound to an owner file system
-   * That filesystem statistics will be updated as appropriate.
-   */
-  private MetastoreInstrumentation instrumentation
-      = new MetastoreInstrumentationImpl();
-
-  /** Owner FS: only valid if configured with an owner FS. */
-  private S3AFileSystem owner;
-
-  /** Invoker for IO. Until configured properly, use try-once. */
-  private Invoker invoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
-      Invoker.NO_OP
-  );
-
-  /** Invoker for read operations. */
-  private Invoker readOp;
-
-  /** Invoker for write operations. */
-  private Invoker writeOp;
-
-  /** Invoker for scan operations. */
-  private Invoker scanOp;
-
-  private final AtomicLong readThrottleEvents = new AtomicLong(0);
-  private final AtomicLong writeThrottleEvents = new AtomicLong(0);
-  private final AtomicLong scanThrottleEvents = new AtomicLong(0);
-  private final AtomicLong batchWriteCapacityExceededEvents = new AtomicLong(0);
-
-  /**
-   * Total limit on the number of throttle events after which
-   * we stop warning in the log. Keeps the noise down.
-   */
-  private static final int THROTTLE_EVENT_LOG_LIMIT = 100;
-
-  /**
-   * Count of the total number of throttle events; used to crank back logging.
-   */
-  private AtomicInteger throttleEventCount = new AtomicInteger(0);
-
-  /**
-   * Executor for submitting operations.
-   */
-  private ListeningExecutorService executor;
-
-  /**
-   * Time source. This is used during writes when parent
-   * entries need to be created.
-   */
-  private ITtlTimeProvider ttlTimeProvider;
-
-  private DynamoDBMetadataStoreTableManager tableHandler;
-
-  /**
-   * A utility function to create DynamoDB instance.
-   * @param conf the file system configuration
-   * @param s3Region region of the associated S3 bucket (if any).
-   * @param bucket Optional bucket to use to look up per-bucket proxy secrets
-   * @param credentials credentials.
-   * @return DynamoDB instance.
-   * @throws IOException I/O error.
-   */
-  private DynamoDB createDynamoDB(
-      final Configuration conf,
-      final String s3Region,
-      final String bucket,
-      final AWSCredentialsProvider credentials)
-      throws IOException {
-    if (amazonDynamoDB == null) {
-      Preconditions.checkNotNull(conf);
-      final Class<? extends DynamoDBClientFactory> cls =
-          conf.getClass(S3GUARD_DDB_CLIENT_FACTORY_IMPL,
-          S3GUARD_DDB_CLIENT_FACTORY_IMPL_DEFAULT, DynamoDBClientFactory.class);
-      LOG.debug("Creating DynamoDB client {} with S3 region {}", cls, s3Region);
-      amazonDynamoDB = ReflectionUtils.newInstance(cls, conf)
-          .createDynamoDBClient(s3Region, bucket, credentials);
-    }
-    return new DynamoDB(amazonDynamoDB);
-  }
-
-  /**
-   * {@inheritDoc}.
-   * The credentials for authenticating with S3 are requested from the
-   * FS via {@link S3AFileSystem#shareCredentials(String)}; this will
-   * increment the reference counter of these credentials.
-   * @param fs {@code S3AFileSystem} associated with the MetadataStore
-   * @param ttlTp the time provider to use for metadata expiry
-   * @throws IOException on a failure
-   */
-  @Override
-  @Retries.OnceRaw
-  public void initialize(FileSystem fs, ITtlTimeProvider ttlTp)
-      throws IOException {
-    Preconditions.checkNotNull(fs, "Null filesystem");
-    Preconditions.checkArgument(fs instanceof S3AFileSystem,
-        "DynamoDBMetadataStore only supports S3A filesystem - not %s",
-        fs);
-    bindToOwnerFilesystem((S3AFileSystem) fs);
-    final String bucket = owner.getBucket();
-    String confRegion = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
-    if (!StringUtils.isEmpty(confRegion)) {
-      region = confRegion;
-      LOG.debug("Overriding S3 region with configured DynamoDB region: {}",
-          region);
-    } else {
-      try {
-        region = owner.getBucketLocation();
-      } catch (AccessDeniedException e) {
-        // access denied here == can't call getBucket. Report meaningfully
-        URI uri = owner.getUri();
-        String message =
-            "Failed to get bucket location as client lacks permission "
-                + RolePolicies.S3_GET_BUCKET_LOCATION + " for " + uri;
-        LOG.error(message);
-        throw (IOException)new AccessDeniedException(message).initCause(e);
-      }
-      LOG.debug("Inferring DynamoDB region from S3 bucket: {}", region);
-    }
-    credentials = owner.shareCredentials("s3guard");
-    dynamoDB = createDynamoDB(conf, region, bucket, credentials);
-
-    // use the bucket as the DynamoDB table name if not specified in config
-    tableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY, bucket);
-    initDataAccessRetries(conf);
-
-    this.ttlTimeProvider = ttlTp;
-
-    tableHandler = new DynamoDBMetadataStoreTableManager(
-        dynamoDB, tableName, region, amazonDynamoDB, conf, readOp,
-        batchWriteRetryPolicy);
-    this.table = tableHandler.initTable();
-
-    instrumentation.initialized();
-  }
-
-  /**
-   * Declare that this table is owned by the specific S3A FS instance.
-   * This will bind some fields to the values provided by the owner,
-   * including wiring up the instrumentation.
-   * @param fs owner filesystem
-   */
-  @VisibleForTesting
-  void bindToOwnerFilesystem(final S3AFileSystem fs) {
-    owner = fs;
-    conf = owner.getConf();
-    StoreContext context = owner.createStoreContext();
-    instrumentation = context.getInstrumentation()
-        .getS3GuardInstrumentation();
-    username = context.getUsername();
-    executor = MoreExecutors.listeningDecorator(
-        context.createThrottledExecutor());
-    ttlTimeProvider = Preconditions.checkNotNull(
-        context.getTimeProvider(),
-        "ttlTimeProvider must not be null");
-  }
-
-  /**
-   * Performs one-time initialization of the metadata store via configuration.
-   *
-   * This initialization depends on the configuration object to get AWS
-   * credentials, DynamoDBFactory implementation class, DynamoDB endpoints,
-   * DynamoDB table names etc. After initialization, this metadata store does
-   * not explicitly relate to any S3 bucket, which be nonexistent.
-   *
-   * This is used to operate the metadata store directly beyond the scope of the
-   * S3AFileSystem integration, e.g. command line tools.
-   * Generally, callers should use
-   * {@link MetadataStore#initialize(FileSystem, ITtlTimeProvider)}
-   * with an initialized {@code S3AFileSystem} instance.
-   *
-   * Without a filesystem to act as a reference point, the configuration itself
-   * must declare the table name and region in the
-   * {@link Constants#S3GUARD_DDB_TABLE_NAME_KEY} and
-   * {@link Constants#S3GUARD_DDB_REGION_KEY} respectively.
-   * It also creates a new credential provider list from the configuration,
-   * using the base fs.s3a.* options, as there is no bucket to infer per-bucket
-   * settings from.
-   *
-   * @see MetadataStore#initialize(FileSystem, ITtlTimeProvider)
-   * @throws IOException if there is an error
-   * @throws IllegalArgumentException if the configuration is incomplete
-   */
-  @Override
-  @Retries.OnceRaw
-  public void initialize(Configuration config,
-      ITtlTimeProvider ttlTp) throws IOException {
-    conf = config;
-    // use the bucket as the DynamoDB table name if not specified in config
-    tableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY);
-
-    Preconditions.checkArgument(!StringUtils.isEmpty(tableName),
-        "No DynamoDB table name configured");
-    region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
-    Preconditions.checkArgument(!StringUtils.isEmpty(region),
-        "No DynamoDB region configured");
-    // there's no URI here, which complicates life: you cannot
-    // create AWS providers here which require one.
-    credentials = createAWSCredentialProviderSet(null, conf);
-    dynamoDB = createDynamoDB(conf, region, null, credentials);
-
-    username = UserGroupInformation.getCurrentUser().getShortUserName();
-    // without an executor from the owner FS, create one using
-    // the executor capacity for work.
-    int executorCapacity = intOption(conf,
-        EXECUTOR_CAPACITY, DEFAULT_EXECUTOR_CAPACITY, 1);
-    executor = MoreExecutors.listeningDecorator(
-        BlockingThreadPoolExecutorService.newInstance(
-            executorCapacity,
-            executorCapacity * 2,
-              longOption(conf, KEEPALIVE_TIME,
-                  DEFAULT_KEEPALIVE_TIME, 0),
-                  TimeUnit.SECONDS,
-                  "s3a-ddb-" + tableName));
-    initDataAccessRetries(conf);
-    this.ttlTimeProvider = ttlTp;
-
-    tableHandler = new DynamoDBMetadataStoreTableManager(
-        dynamoDB, tableName, region, amazonDynamoDB, conf, readOp,
-        batchWriteRetryPolicy);
-    this.table = tableHandler.initTable();
-  }
-
-  /**
-   * Set retry policy. This is driven by the value of
-   * {@link Constants#S3GUARD_DDB_MAX_RETRIES} with an exponential backoff
-   * between each attempt of {@link Constants#S3GUARD_DDB_THROTTLE_RETRY_INTERVAL}
-   * milliseconds.
-   * @param config configuration for data access
-   */
-  private void initDataAccessRetries(Configuration config) {
-    batchWriteRetryPolicy = RetryPolicies
-        .exponentialBackoffRetry(
-            config.getInt(S3GUARD_DDB_MAX_RETRIES,
-                S3GUARD_DDB_MAX_RETRIES_DEFAULT),
-            conf.getTimeDuration(S3GUARD_DDB_THROTTLE_RETRY_INTERVAL,
-                S3GUARD_DDB_THROTTLE_RETRY_INTERVAL_DEFAULT,
-                TimeUnit.MILLISECONDS),
-            TimeUnit.MILLISECONDS);
-    final RetryPolicy throttledRetryRetryPolicy
-        = new S3GuardDataAccessRetryPolicy(config);
-    readOp = new Invoker(throttledRetryRetryPolicy, this::readRetryEvent);
-    writeOp = new Invoker(throttledRetryRetryPolicy, this::writeRetryEvent);
-    scanOp = new Invoker(throttledRetryRetryPolicy, this::scanRetryEvent);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void delete(Path path,
-      final BulkOperationState operationState)
-      throws IOException {
-    innerDelete(path, true,
-        extractOrCreate(operationState,
-            BulkOperationState.OperationType.Delete));
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void forgetMetadata(Path path) throws IOException {
-    LOG.debug("Forget metadata for {}", path);
-    innerDelete(path, false, null);
-  }
-
-  /**
-   * Inner delete option, action based on the {@code tombstone} flag.
-   * No tombstone: delete the entry. Tombstone: create a tombstone entry.
-   * There is no check as to whether the entry exists in the table first.
-   * @param path path to delete
-   * @param tombstone flag to create a tombstone marker
-   * @param ancestorState ancestor state for context.
-   * @throws IOException I/O error.
-   */
-  @Retries.RetryTranslated
-  private void innerDelete(final Path path,
-      final boolean tombstone,
-      final AncestorState ancestorState)
-      throws IOException {
-    checkPath(path);
-    LOG.debug("Deleting from table {} in region {}: {}",
-        tableName, region, path);
-
-    // deleting nonexistent item consumes 1 write capacity; skip it
-    if (path.isRoot()) {
-      LOG.debug("Skip deleting root directory as it does not exist in table");
-      return;
-    }
-    // the policy on whether repeating delete operations is based
-    // on that of S3A itself
-    boolean idempotent = InternalConstants.DELETE_CONSIDERED_IDEMPOTENT;
-    if (tombstone) {
-      Preconditions.checkArgument(ttlTimeProvider != null, "ttlTimeProvider "
-          + "must not be null");
-      final PathMetadata pmTombstone = PathMetadata.tombstone(path,
-          ttlTimeProvider.getNow());
-      Item item = PathMetadataDynamoDBTranslation.pathMetadataToItem(
-          new DDBPathMetadata(pmTombstone));
-      writeOp.retry(
-          "Put tombstone",
-          path.toString(),
-          idempotent,
-          () -> {
-            logPut(ancestorState, item);
-            recordsWritten(1);
-            table.putItem(item);
-          });
-    } else {
-      PrimaryKey key = pathToKey(path);
-      writeOp.retry(
-          "Delete key",
-          path.toString(),
-          idempotent,
-          () -> {
-            // record the attempt so even on retry the counter goes up.
-            logDelete(ancestorState, key);
-            recordsDeleted(1);
-            table.deleteItem(key);
-          });
-    }
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void deleteSubtree(Path path,
-      final BulkOperationState operationState)
-      throws IOException {
-    checkPath(path);
-    LOG.debug("Deleting subtree from table {} in region {}: {}",
-        tableName, region, path);
-
-    final PathMetadata meta = get(path);
-    if (meta == null) {
-      LOG.debug("Subtree path {} does not exist; this will be a no-op", path);
-      return;
-    }
-    if (meta.isDeleted()) {
-      LOG.debug("Subtree path {} is deleted; this will be a no-op", path);
-      return;
-    }
-    deleteEntries(RemoteIterators.mappingRemoteIterator(
-        new DescendantsIterator(this, meta),
-        FileStatus::getPath),
-        operationState);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void deletePaths(Collection<Path> paths,
-      final BulkOperationState operationState)
-      throws IOException {
-    deleteEntries(RemoteIterators.remoteIteratorFromIterable(paths),
-        operationState);
-  }
-
-  /**
-   * Delete the entries under an iterator.
-   * There's no attempt to order the paths: they are
-   * deleted in the order passed in.
-   * @param entries entries to delete.
-   * @param operationState Nullable operation state
-   * @throws IOException failure
-   */
-  @Retries.RetryTranslated
-  private void deleteEntries(RemoteIterator<Path> entries,
-      final BulkOperationState operationState)
-      throws IOException {
-    final List<CompletableFuture<Void>> futures = new ArrayList<>();
-    AncestorState state = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Delete);
-
-    while (entries.hasNext()) {
-      final Path pathToDelete = entries.next();
-      futures.add(submit(executor, () -> {
-        innerDelete(pathToDelete, true, state);
-        return null;
-      }));
-      if (futures.size() > S3GUARD_DDB_SUBMITTED_TASK_LIMIT) {
-        // first batch done; block for completion.
-        waitForCompletion(futures);
-        futures.clear();
-      }
-    }
-    // now wait for the final set.
-    waitForCompletion(futures);
-  }
-
-  /**
-   * Get a consistent view of an item.
-   * @param path path to look up in the database
-   * @return the result
-   * @throws IOException failure
-   */
-  @Retries.RetryTranslated
-  private Item getConsistentItem(final Path path) throws IOException {
-    PrimaryKey key = pathToKey(path);
-    final GetItemSpec spec = new GetItemSpec()
-        .withPrimaryKey(key)
-        .withConsistentRead(true); // strictly consistent read
-    return readOp.retry("get",
-        path.toString(),
-        true,
-        () -> {
-          recordsRead(1);
-          return table.getItem(spec);
-        });
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public DDBPathMetadata get(Path path) throws IOException {
-    return get(path, false);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public DDBPathMetadata get(Path path, boolean wantEmptyDirectoryFlag)
-      throws IOException {
-    checkPath(path);
-    LOG.debug("Get from table {} in region {}: {} ; wantEmptyDirectory={}",
-        tableName, region, path, wantEmptyDirectoryFlag);
-    DDBPathMetadata result = innerGet(path, wantEmptyDirectoryFlag);
-    LOG.debug("result of get {} is: {}", path, result);
-    return result;
-  }
-
-  /**
-   * Inner get operation, as invoked in the retry logic.
-   * @param path the path to get
-   * @param wantEmptyDirectoryFlag Set to true to give a hint to the
-   *   MetadataStore that it should try to compute the empty directory flag.
-   * @return metadata for {@code path}, {@code null} if not found
-   * @throws IOException IO problem
-   */
-  @Retries.RetryTranslated
-  private DDBPathMetadata innerGet(Path path, boolean wantEmptyDirectoryFlag)
-      throws IOException {
-    final DDBPathMetadata meta;
-    if (path.isRoot()) {
-      // Root does not persist in the table
-      meta =
-          new DDBPathMetadata(makeDirStatus(username, path));
-    } else {
-      final Item item = getConsistentItem(path);
-      meta = itemToPathMetadata(item, username);
-      LOG.debug("Get from table {} in region {} returning for {}: {}",
-          tableName, region, path, meta);
-    }
-
-    if (wantEmptyDirectoryFlag && meta != null && !meta.isDeleted()) {
-      final FileStatus status = meta.getFileStatus();
-      // for a non-deleted directory, we query its direct undeleted children
-      // to determine the isEmpty bit. There's no TTL checking going on here.
-      if (status.isDirectory()) {
-        final QuerySpec spec = new QuerySpec()
-            .withHashKey(pathToParentKeyAttribute(path))
-            .withConsistentRead(true)
-            .withFilterExpression(IS_DELETED + " = :false")
-            .withValueMap(DELETE_TRACKING_VALUE_MAP);
-        boolean hasChildren = readOp.retry("get/hasChildren",
-            path.toString(),
-            true,
-            () -> {
-              // issue the query
-              final IteratorSupport<Item, QueryOutcome> it = table.query(
-                  spec).iterator();
-              // if non empty, log the result to aid with some debugging
-              if (it.hasNext()) {
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug("Dir {} is non-empty", status.getPath());
-                  while(it.hasNext()) {
-                    LOG.debug("{}", itemToPathMetadata(it.next(), username));
-                  }
-                }
-                return true;
-              } else {
-                return false;
-              }
-          });
-
-        // If directory is authoritative, we can set the empty directory flag
-        // to TRUE or FALSE. Otherwise FALSE, or UNKNOWN.
-        if (meta.isAuthoritativeDir()) {
-          meta.setIsEmptyDirectory(
-              hasChildren ? Tristate.FALSE : Tristate.TRUE);
-        } else {
-          meta.setIsEmptyDirectory(
-              hasChildren ? Tristate.FALSE : Tristate.UNKNOWN);
-        }
-      }
-    }
-
-    return meta;
-  }
-
-  /**
-   * Make a S3AFileStatus object for a directory at given path.
-   * The FileStatus only contains what S3A needs, and omits mod time
-   * since S3A uses its own implementation which returns current system time.
-   * @param dirOwner  username of owner
-   * @param path   path to dir
-   * @return new S3AFileStatus
-   */
-  private S3AFileStatus makeDirStatus(String dirOwner, Path path) {
-    return new S3AFileStatus(Tristate.UNKNOWN, path, dirOwner);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public DirListingMetadata listChildren(final Path path) throws IOException {
-    checkPath(path);
-    LOG.debug("Listing table {} in region {}: {}", tableName, region, path);
-
-    final QuerySpec spec = new QuerySpec()
-        .withHashKey(pathToParentKeyAttribute(path))
-        .withConsistentRead(true); // strictly consistent read
-    final List<PathMetadata> metas = new ArrayList<>();
-    // find the children in the table
-    final ItemCollection<QueryOutcome> items = scanOp.retry(
-        "listChildren",
-        path.toString(),
-        true,
-        () -> table.query(spec));
-    // now wrap the result with retry logic
-    try {
-      for (Item item : wrapWithRetries(items)) {
-        metas.add(itemToPathMetadata(item, username));
-      }
-    } catch (UncheckedIOException e) {
-      // failure in the iterators; unwrap.
-      throw e.getCause();
-    }
-
-    // Minor race condition here - if the path is deleted between
-    // getting the list of items and the directory metadata we might
-    // get a null in DDBPathMetadata.
-    return getDirListingMetadataFromDirMetaAndList(path, metas,
-        get(path));
-  }
-
-  DirListingMetadata getDirListingMetadataFromDirMetaAndList(Path path,
-      List<PathMetadata> metas, DDBPathMetadata dirPathMeta) {
-    boolean isAuthoritative = false;
-    if (dirPathMeta != null) {
-      isAuthoritative = dirPathMeta.isAuthoritativeDir();
-    }
-
-    LOG.trace("Listing table {} in region {} for {} returning {}",
-        tableName, region, path, metas);
-
-    if (!metas.isEmpty() && dirPathMeta == null) {
-      // We handle this case as the directory is deleted.
-      LOG.warn("Directory marker is deleted, but the list of the directory "
-          + "elements is not empty: {}. This case is handled as if the "
-          + "directory was deleted.", metas);
-      return null;
-    }
-
-    if(metas.isEmpty() && dirPathMeta == null) {
-      return null;
-    }
-
-    return new DirListingMetadata(path, metas, isAuthoritative,
-        dirPathMeta.getLastUpdated());
-  }
-
-  /**
-   * Origin of entries in the ancestor map built up in
-   * {@link #completeAncestry(Collection, AncestorState)}.
-   * This is done to stop generated ancestor entries to overwriting those
-   * in the store, while allowing those requested in the API call to do this.
-   */
-  private enum EntryOrigin {
-    Requested,  // requested in method call
-    Retrieved,  // retrieved from DDB: do not resubmit
-    Generated   // generated ancestor.
-  }
-
-  /**
-   * Build the list of all parent entries.
-   * <p>
-   * <b>Thread safety:</b> none. Callers must synchronize access.
-   * <p>
-   * Callers are required to synchronize on ancestorState.
-   * @param pathsToCreate paths to create
-   * @param ancestorState ongoing ancestor state.
-   * @return the full ancestry paths
-   */
-  private Collection<DDBPathMetadata> completeAncestry(
-      final Collection<DDBPathMetadata> pathsToCreate,
-      final AncestorState ancestorState) throws IOException {
-    // Key on path to allow fast lookup
-    Map<Path, Pair<EntryOrigin, DDBPathMetadata>> ancestry = new HashMap<>();
-    LOG.debug("Completing ancestry for {} paths", pathsToCreate.size());
-    // we sort the inputs to guarantee that the topmost entries come first.
-    // that way if the put request contains both parents and children
-    // then the existing parents will not be re-created -they will just
-    // be added to the ancestor list first.
-    List<DDBPathMetadata> sortedPaths = new ArrayList<>(pathsToCreate);
-    sortedPaths.sort(PathOrderComparators.TOPMOST_PM_FIRST);
-    // iterate through the paths.
-    for (DDBPathMetadata entry : sortedPaths) {
-      Preconditions.checkArgument(entry != null);
-      Path path = entry.getFileStatus().getPath();
-      LOG.debug("Adding entry {}", path);
-      if (path.isRoot()) {
-        // this is a root entry: do not add it.
-        break;
-      }
-      // add it to the ancestor state, failing if it is already there and
-      // of a different type.
-      DDBPathMetadata oldEntry = ancestorState.put(path, entry);
-      boolean addAncestors = true;
-      if (oldEntry != null) {
-        // check for and warn if the existing bulk operation has an inconsistent
-        // entry.
-        // two directories or two files are both allowed.
-        // file-over-file can happen in multipart uploaders when the same
-        // uploader is overwriting file entries to the same destination as
-        // part of its bulk operation.
-        boolean oldWasDir = oldEntry.getFileStatus().isDirectory();
-        boolean newIsDir = entry.getFileStatus().isDirectory();
-        if ((oldWasDir && !newIsDir)
-            || (!oldWasDir && newIsDir)) {
-          LOG.warn("Overwriting a S3Guard file created in the operation: {}",
-              oldEntry);
-          LOG.warn("With new entry: {}", entry);
-          // restore the old state
-          ancestorState.put(path, oldEntry);
-          // then raise an exception
-          throw new PathIOException(path.toString(),
-              String.format("%s old %s new %s",
-                  E_INCONSISTENT_UPDATE,
-                  oldEntry,
-                  entry));
-        } else {
-          // a directory is already present. Log and continue.
-          LOG.debug("Directory at {} being updated with value {}",
-              path, entry);
-          // and we skip the the subsequent parent scan as we've already been
-          // here
-          addAncestors = false;
-        }
-      }
-      // add the entry to the ancestry map as an explicitly requested entry.
-      ancestry.put(path, Pair.of(EntryOrigin.Requested, entry));
-      // now scan up the ancestor tree to see if there are any
-      // immediately missing entries.
-      Path parent = path.getParent();
-      while (addAncestors
-          && !parent.isRoot() && !ancestry.containsKey(parent)) {
-        if (!ancestorState.findEntry(parent, true)) {
-          // there is no entry in the ancestor state.
-          // look in the store
-          DDBPathMetadata md;
-          Pair<EntryOrigin, DDBPathMetadata> newEntry;
-          final Item item = getConsistentItem(parent);
-          if (item != null && !itemToPathMetadata(item, username).isDeleted()) {
-            // This is an undeleted entry found in the database.
-            // register it in ancestor state and in the map of entries to create
-            // as a retrieved entry
-            md = itemToPathMetadata(item, username);
-            LOG.debug("Found existing entry for parent: {}", md);
-            newEntry = Pair.of(EntryOrigin.Retrieved, md);
-            // and we break, assuming that if there is an entry, its parents
-            // are valid too.
-            addAncestors = false;
-          } else {
-            // A directory entry was not found in the DB. Create one.
-            LOG.debug("auto-create ancestor path {} for child path {}",
-                parent, path);
-            final S3AFileStatus status = makeDirStatus(parent, username);
-            md = new DDBPathMetadata(status, Tristate.FALSE,
-                false, false, ttlTimeProvider.getNow());
-            // declare to be a generated entry
-            newEntry =  Pair.of(EntryOrigin.Generated, md);
-          }
-          // insert into the ancestor state to avoid further checks
-          ancestorState.put(parent, md);
-          ancestry.put(parent, newEntry);
-        }
-        parent = parent.getParent();
-      }
-    }
-    // we now have a list of entries which were not in the operation state.
-    // Filter out those which were retrieved, to produce a list of those
-    // which must be written to the database.
-    // TODO sort in reverse order of existence
-    return ancestry.values().stream()
-        .filter(p -> p.getLeft() != EntryOrigin.Retrieved)
-        .map(Pair::getRight)
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * {@inheritDoc}
-   * <p>
-   * The implementation scans all up the directory tree and does a get()
-   * for each entry; at each level one is found it is added to the ancestor
-   * state.
-   * <p>
-   * The original implementation would stop on finding the first non-empty
-   * parent. This (re) implementation issues a GET for every parent entry
-   * and so detects and recovers from a tombstone marker further up the tree
-   * (i.e. an inconsistent store is corrected for).
-   * <p>
-   * if {@code operationState} is not null, when this method returns the
-   * operation state will be updated with all new entries created.
-   * This ensures that subsequent operations with the same store will not
-   * trigger new updates.
-   * @param qualifiedPath path to update
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException on failure.
-   */
-  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
-  @Override
-  @Retries.RetryTranslated
-  public void addAncestors(final Path qualifiedPath,
-      @Nullable final BulkOperationState operationState) throws IOException {
-
-    Collection<DDBPathMetadata> newDirs = new ArrayList<>();
-    final AncestorState ancestorState = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Put);
-    Path parent = qualifiedPath.getParent();
-    boolean entryFound = false;
-
-    // Iterate up the parents.
-    // note that only ancestorState get/set operations are synchronized;
-    // the DDB read between them is not. As a result, more than one
-    // thread may probe the state, find the entry missing, do the database
-    // query and add the entry.
-    // This is done to avoid making the remote dynamo query part of the
-    // synchronized block.
-    // If a race does occur, the cost is simply one extra GET and potentially
-    // one extra PUT.
-    while (!parent.isRoot()) {
-      synchronized (ancestorState) {
-        if (ancestorState.contains(parent)) {
-          // the ancestry map contains the key, so no need to even look for it.
-          break;
-        }
-      }
-      // we don't worry about tombstone expiry here as expired or not,
-      // a directory entry will go in.
-      PathMetadata directory = get(parent);
-      if (directory == null || directory.isDeleted()) {
-        if (entryFound) {
-          LOG.warn("Inconsistent S3Guard table: adding directory {}", parent);
-        }
-        S3AFileStatus status = makeDirStatus(username, parent);
-        LOG.debug("Adding new ancestor entry {}", status);
-        DDBPathMetadata meta = new DDBPathMetadata(status, Tristate.FALSE,
-            false, ttlTimeProvider.getNow());
-        newDirs.add(meta);
-        // Do not update ancestor state here, as it
-        // will happen in the innerPut() call. Were we to add it
-        // here that put operation would actually (mistakenly) skip
-        // creating the entry.
-      } else {
-        // an entry was found. Check its type
-        entryFound = true;
-        if (directory.getFileStatus().isFile()) {
-          throw new PathIOException(parent.toString(),
-              "Cannot overwrite parent file: metastore is"
-                  + " in an inconsistent state");
-        }
-        // the directory exists. Add it to the ancestor state for next time.
-        synchronized (ancestorState) {
-          ancestorState.put(parent, new DDBPathMetadata(directory));
-        }
-      }
-      parent = parent.getParent();
-    }
-    // the listing of directories to put is all those parents which we know
-    // are not in the store or BulkOperationState.
-    if (!newDirs.isEmpty()) {
-      // patch up the time.
-      patchLastUpdated(newDirs, ttlTimeProvider);
-      innerPut(newDirs, operationState);
-    }
-  }
-
-  /**
-   * {@inheritDoc}.
-   *
-   * The DDB implementation sorts all the paths such that new items
-   * are ordered highest level entry first; deleted items are ordered
-   * lowest entry first.
-   *
-   * This is to ensure that if a client failed partway through the update,
-   * there will no entries in the table which lack parent entries.
-   * @param pathsToDelete Collection of all paths that were removed from the
-   *                      source directory tree of the move.
-   * @param pathsToCreate Collection of all PathMetadata for the new paths
-   *                      that were created at the destination of the rename
-   *                      ().
-   * @param operationState Any ongoing state supplied to the rename tracker
-   *                      which is to be passed in with each move operation.
-   * @throws IOException if there is an error
-   */
-  @Override
-  @Retries.RetryTranslated
-  public void move(@Nullable Collection<Path> pathsToDelete,
-      @Nullable Collection<PathMetadata> pathsToCreate,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    if (pathsToDelete == null && pathsToCreate == null) {
-      return;
-    }
-
-    LOG.debug("Moving paths of table {} in region {}: {} paths to delete and {}"
-        + " paths to create", tableName, region,
-        pathsToDelete == null ? 0 : pathsToDelete.size(),
-        pathsToCreate == null ? 0 : pathsToCreate.size());
-    LOG.trace("move: pathsToDelete = {}, pathsToCreate = {}", pathsToDelete,
-        pathsToCreate);
-
-    // In DynamoDBMetadataStore implementation, we assume that if a path
-    // exists, all its ancestors will also exist in the table.
-    // Following code is to maintain this invariant by putting all ancestor
-    // directories of the paths to create.
-    // ancestor paths that are not explicitly added to paths to create
-    AncestorState ancestorState = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Rename);
-    List<DDBPathMetadata> newItems = new ArrayList<>();
-    if (pathsToCreate != null) {
-      // create all parent entries.
-      // this is synchronized on the move state so that across both serialized
-      // and parallelized renames, duplicate ancestor entries are not created.
-      synchronized (ancestorState) {
-        newItems.addAll(
-            completeAncestry(
-                pathMetaToDDBPathMeta(pathsToCreate),
-                ancestorState));
-      }
-    }
-    // sort all the new items topmost first.
-    newItems.sort(PathOrderComparators.TOPMOST_PM_FIRST);
-
-    // now process the deletions.
-    if (pathsToDelete != null) {
-      List<DDBPathMetadata> tombstones = new ArrayList<>(pathsToDelete.size());
-      for (Path meta : pathsToDelete) {
-        Preconditions.checkArgument(ttlTimeProvider != null, "ttlTimeProvider"
-            + " must not be null");
-        final PathMetadata pmTombstone = PathMetadata.tombstone(meta,
-            ttlTimeProvider.getNow());
-        tombstones.add(new DDBPathMetadata(pmTombstone));
-      }
-      // sort all the tombstones lowest first.
-      tombstones.sort(TOPMOST_PM_LAST);
-      newItems.addAll(tombstones);
-    }
-
-    processBatchWriteRequest(ancestorState,
-        null, pathMetadataToItem(newItems));
-  }
-
-  /**
-   * Helper method to issue a batch write request to DynamoDB.
-   * <ol>
-   *   <li>Keys to delete are processed ahead of writing new items.</li>
-   *   <li>No attempt is made to sort the input: the caller must do that</li>
-   * </ol>
-   * As well as retrying on the operation invocation, incomplete
-   * batches are retried until all have been processed.
-   *
-   * @param ancestorState ancestor state for logging
-   * @param keysToDelete primary keys to be deleted; can be null
-   * @param itemsToPut new items to be put; can be null
-   * @return the number of iterations needed to complete the call.
-   */
-  @Retries.RetryTranslated("Outstanding batch items are updated with backoff")
-  private int processBatchWriteRequest(
-      @Nullable AncestorState ancestorState,
-      PrimaryKey[] keysToDelete,
-      Item[] itemsToPut) throws IOException {
-    final int totalToDelete = (keysToDelete == null ? 0 : keysToDelete.length);
-    final int totalToPut = (itemsToPut == null ? 0 : itemsToPut.length);
-    if (totalToPut == 0 && totalToDelete == 0) {
-      LOG.debug("Ignoring empty batch write request");
-      return 0;
-    }
-    int count = 0;
-    int batches = 0;
-    while (count < totalToDelete + totalToPut) {
-      final TableWriteItems writeItems = new TableWriteItems(tableName);
-      int numToDelete = 0;
-      if (keysToDelete != null
-          && count < totalToDelete) {
-        numToDelete = Math.min(S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT,
-            totalToDelete - count);
-        PrimaryKey[] toDelete = Arrays.copyOfRange(keysToDelete,
-            count, count + numToDelete);
-        LOG.debug("Deleting {} entries: {}", toDelete.length, toDelete);
-        writeItems.withPrimaryKeysToDelete(toDelete);
-        count += numToDelete;
-      }
-
-      if (numToDelete < S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT
-          && itemsToPut != null
-          && count < totalToDelete + totalToPut) {
-        final int numToPut = Math.min(
-            S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT - numToDelete,
-            totalToDelete + totalToPut - count);
-        final int index = count - totalToDelete;
-        writeItems.withItemsToPut(
-            Arrays.copyOfRange(itemsToPut, index, index + numToPut));
-        count += numToPut;
-      }
-
-      // if there's a retry and another process updates things then it's not
-      // quite idempotent, but this was the case anyway
-      batches++;
-      BatchWriteItemOutcome res = writeOp.retry(
-          "batch write",
-          "",
-          true,
-          () -> dynamoDB.batchWriteItem(writeItems));
-      // Check for unprocessed keys in case of exceeding provisioned throughput
-      Map<String, List<WriteRequest>> unprocessed = res.getUnprocessedItems();
-      int retryCount = 0;
-      while (!unprocessed.isEmpty()) {
-        batchWriteCapacityExceededEvents.incrementAndGet();
-        batches++;
-        retryBackoffOnBatchWrite(retryCount++);
-        // use a different reference to keep the compiler quiet
-        final Map<String, List<WriteRequest>> upx = unprocessed;
-        res = writeOp.retry(
-            "batch write",
-            "",
-            true,
-            () -> dynamoDB.batchWriteItemUnprocessed(upx));
-        unprocessed = res.getUnprocessedItems();
-      }
-    }
-    if (itemsToPut != null) {
-      recordsWritten(itemsToPut.length);
-      logPut(ancestorState, itemsToPut);
-    }
-    if (keysToDelete != null) {
-      recordsDeleted(keysToDelete.length);
-      logDelete(ancestorState, keysToDelete);
-
-    }
-    return batches;
-  }
-
-  /**
-   * Put the current thread to sleep to implement exponential backoff
-   * depending on retryCount.  If max retries are exceeded, throws an
-   * exception instead.
-   *
-   * @param retryCount number of retries so far
-   * @throws IOException when max retryCount is exceeded.
-   */
-  private void retryBackoffOnBatchWrite(int retryCount) throws IOException {
-    try {
-      // Our RetryPolicy ignores everything but retryCount here.
-      RetryPolicy.RetryAction action = batchWriteRetryPolicy.shouldRetry(
-          null,
-          retryCount, 0, true);
-      if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
-        // Create an AWSServiceThrottledException, with a fake inner cause
-        // which we fill in to look like a real exception so
-        // error messages look sensible
-        AmazonServiceException cause = new AmazonServiceException(
-            "Throttling");
-        cause.setServiceName("S3Guard");
-        cause.setStatusCode(AWSServiceThrottledException.STATUS_CODE);
-        cause.setErrorCode(THROTTLING);  // used in real AWS errors
-        cause.setErrorType(AmazonServiceException.ErrorType.Service);
-        cause.setErrorMessage(THROTTLING);
-        cause.setRequestId("n/a");
-        throw new AWSServiceThrottledException(
-            String.format("Max retries during batch write exceeded"
-                    + " (%d) for DynamoDB."
-                    + HINT_DDB_IOPS_TOO_LOW,
-                retryCount),
-            cause);
-      } else {
-        LOG.debug("Sleeping {} msec before next retry", action.delayMillis);
-        Thread.sleep(action.delayMillis);
-      }
-    } catch (InterruptedException e) {
-      throw (IOException)new InterruptedIOException(e.toString()).initCause(e);
-    } catch (IOException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new IOException("Unexpected exception " + e, e);
-    }
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void put(final PathMetadata meta) throws IOException {
-    put(meta, null);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void put(
-      final PathMetadata meta,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    // For a deeply nested path, this method will automatically create the full
-    // ancestry and save respective item in DynamoDB table.
-    // So after put operation, we maintain the invariant that if a path exists,
-    // all its ancestors will also exist in the table.
-    // For performance purpose, we generate the full paths to put and use batch
-    // write item request to save the items.
-    LOG.debug("Saving to table {} in region {}: {}", tableName, region, meta);
-
-    Collection<PathMetadata> wrapper = new ArrayList<>(1);
-    wrapper.add(meta);
-    put(wrapper, operationState);
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void put(
-      final Collection<? extends PathMetadata> metas,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    innerPut(pathMetaToDDBPathMeta(metas), operationState);
-  }
-
-  /**
-   * Internal put operation.
-   * <p>
-   * The ancestors to all entries are added to the set of entries to write,
-   * provided they are not already stored in any supplied operation state.
-   * Both the supplied metadata entries and ancestor entries are sorted
-   * so that the topmost entries are written first.
-   * This is to ensure that a failure partway through the operation will not
-   * create entries in the table without parents.
-   * @param metas metadata entries to write.
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException failure.
-   */
-  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
-  @Retries.RetryTranslated
-  private void innerPut(
-      final Collection<DDBPathMetadata> metas,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    if (metas.isEmpty()) {
-      // Happens when someone calls put() with an empty list.
-      LOG.debug("Ignoring empty list of entries to put");
-      return;
-    }
-    // always create or retrieve an ancestor state instance, so it can
-    // always be used for synchronization.
-    final AncestorState ancestorState = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Put);
-
-    Item[] items;
-    synchronized (ancestorState) {
-      items = pathMetadataToItem(
-          completeAncestry(metas, ancestorState));
-    }
-    LOG.debug("Saving batch of {} items to table {}, region {}", items.length,
-        tableName, region);
-    processBatchWriteRequest(ancestorState, null, items);
-  }
-
-  /**
-   * Get full path of ancestors that are nonexistent in table.
-   *
-   * This queries DDB when looking for parents which are not in
-   * any supplied ongoing operation state.
-   * Updates the operation state with found entries to reduce further checks.
-   *
-   * @param meta metadata to put
-   * @param operationState ongoing bulk state
-   * @return a possibly empty list of entries to put.
-   * @throws IOException failure
-   */
-  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
-  @VisibleForTesting
-  @Retries.RetryTranslated
-  List<DDBPathMetadata> fullPathsToPut(DDBPathMetadata meta,
-      @Nullable BulkOperationState operationState)
-      throws IOException {
-    checkPathMetadata(meta);
-    final List<DDBPathMetadata> metasToPut = new ArrayList<>();
-    // root path is not persisted
-    if (!meta.getFileStatus().getPath().isRoot()) {
-      metasToPut.add(meta);
-    }
-
-    // put all its ancestors if not present; as an optimization we return at its
-    // first existent ancestor
-    final AncestorState ancestorState = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Put);
-    Path path = meta.getFileStatus().getPath().getParent();
-    while (path != null && !path.isRoot()) {
-      synchronized (ancestorState) {
-        if (ancestorState.findEntry(path, true)) {
-          break;
-        }
-      }
-      final Item item = getConsistentItem(path);
-      if (!itemExists(item)) {
-        final S3AFileStatus status = makeDirStatus(path, username);
-        metasToPut.add(new DDBPathMetadata(status, Tristate.FALSE, false,
-            meta.isAuthoritativeDir(), meta.getLastUpdated()));
-        path = path.getParent();
-      } else {
-        // found the entry in the table, so add it to the ancestor state
-        synchronized (ancestorState) {
-          ancestorState.put(path, itemToPathMetadata(item, username));
-        }
-        // then break out of the loop.
-        break;
-      }
-    }
-    return metasToPut;
-  }
-
-  /**
-   * Does an item represent an object which exists?
-   * @param item item retrieved in a query.
-   * @return true iff the item isn't null and, if there is an is_deleted
-   * column, that its value is false.
-   */
-  private static boolean itemExists(Item item) {
-    if (item == null) {
-      return false;
-    }
-    if (item.hasAttribute(IS_DELETED) &&
-        item.getBoolean(IS_DELETED)) {
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Get the value of an optional boolean attribute, falling back to the
-   * default value if the attribute is absent.
-   * @param item Item
-   * @param attrName Attribute name
-   * @param defVal Default value
-   * @return The value or the default
-   */
-  private static boolean getBoolAttribute(Item item,
-      String attrName,
-      boolean defVal) {
-    return item.hasAttribute(attrName) ? item.getBoolean(attrName) : defVal;
-  }
-
-  /** Create a directory FileStatus using 0 for the lastUpdated time. */
-  static S3AFileStatus makeDirStatus(Path f, String owner) {
-    return new S3AFileStatus(Tristate.UNKNOWN, f, owner);
-  }
-
-  /**
-   * {@inheritDoc}.
-   * There is retry around building the list of paths to update, but
-   * the call to
-   * {@link #processBatchWriteRequest(DynamoDBMetadataStore.AncestorState, PrimaryKey[], Item[])}
-   * is only tried once.
-   * @param meta Directory listing metadata.
-   * @param unchangedEntries unchanged child entry paths
-   * @param operationState operational state for a bulk update
-   * @throws IOException IO problem
-   */
-  @Override
-  @Retries.RetryTranslated
-  public void put(
-      final DirListingMetadata meta,
-      final List<Path> unchangedEntries,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    LOG.debug("Saving {} dir meta for {} to table {} in region {}: {}",
-        meta.isAuthoritative() ? "auth" : "nonauth",
-        meta.getPath(),
-        tableName, region, meta);
-    // directory path
-    Path path = meta.getPath();
-    DDBPathMetadata ddbPathMeta =
-        new DDBPathMetadata(makeDirStatus(path, username), meta.isEmpty(),
-            false, meta.isAuthoritative(), meta.getLastUpdated());
-    // put all its ancestors if not present
-    final AncestorState ancestorState = extractOrCreate(operationState,
-        BulkOperationState.OperationType.Put);
-    // First add any missing ancestors...
-    final List<DDBPathMetadata> metasToPut = fullPathsToPut(ddbPathMeta,
-        ancestorState);
-
-    // next add all changed children of the directory
-    // ones that came from the previous listing are left as-is
-    final Collection<PathMetadata> children = meta.getListing()
-        .stream()
-        .filter(e -> !unchangedEntries.contains(e.getFileStatus().getPath()))
-        .collect(Collectors.toList());
-
-    metasToPut.addAll(pathMetaToDDBPathMeta(children));
-
-    // sort so highest-level entries are written to the store first.
-    // if a sequence fails, no orphan entries will have been written.
-    metasToPut.sort(PathOrderComparators.TOPMOST_PM_FIRST);
-    processBatchWriteRequest(ancestorState,
-        null,
-        pathMetadataToItem(metasToPut));
-    // and add the ancestors
-    synchronized (ancestorState) {
-      metasToPut.forEach(ancestorState::put);
-    }
-  }
-
-  @Override
-  public synchronized void close() {
-    instrumentation.storeClosed();
-    try {
-      if (dynamoDB != null) {
-        LOG.debug("Shutting down {}", this);
-        dynamoDB.shutdown();
-        dynamoDB = null;
-      }
-    } finally {
-      closeAutocloseables(LOG, credentials);
-      credentials = null;
-    }
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void destroy() throws IOException {
-    tableHandler.destroy();
-  }
-
-  @Retries.RetryTranslated
-  private ItemCollection<ScanOutcome> expiredFiles(PruneMode pruneMode,
-      long cutoff, String keyPrefix) throws IOException {
-
-    String filterExpression;
-    String projectionExpression;
-    ValueMap map;
-
-    switch (pruneMode) {
-    case ALL_BY_MODTIME:
-      // filter all files under the given parent older than the modtime.
-      // this implicitly skips directories, because they lack a modtime field.
-      // however we explicitly exclude directories to make clear that
-      // directories are to be excluded and avoid any confusion
-      // see: HADOOP-16725.
-      // note: files lack the is_dir field entirely, so we use a `not` to
-      // filter out the directories.
-      filterExpression =
-          "mod_time < :mod_time and begins_with(parent, :parent)"
-              + " and not is_dir = :is_dir";
-      projectionExpression = "parent,child";
-      map = new ValueMap()
-          .withLong(":mod_time", cutoff)
-          .withString(":parent", keyPrefix)
-          .withBoolean(":is_dir", true);
-      break;
-    case TOMBSTONES_BY_LASTUPDATED:
-      filterExpression =
-          "last_updated < :last_updated and begins_with(parent, :parent) "
-              + "and is_deleted = :is_deleted";
-      projectionExpression = "parent,child,is_deleted";
-      map = new ValueMap()
-          .withLong(":last_updated", cutoff)
-          .withString(":parent", keyPrefix)
-          .withBoolean(":is_deleted", true);
-      break;
-    default:
-      throw new UnsupportedOperationException("Unsupported prune mode: "
-          + pruneMode);
-    }
-
-    return readOp.retry(
-        "scan",
-        keyPrefix,
-        true,
-        () -> table.scan(filterExpression, projectionExpression, null, map));
-  }
-
-  @Override
-  @Retries.RetryTranslated
-  public void prune(PruneMode pruneMode, long cutoff) throws IOException {
-    prune(pruneMode, cutoff, "/");
-  }
-
-  /**
-   * Prune files, in batches. There's optionally a sleep between each batch.
-   *
-   * @param pruneMode The mode of operation for the prune For details see
-   *                  {@link MetadataStore#prune(PruneMode, long)}
-   * @param cutoff Oldest modification time to allow
-   * @param keyPrefix The prefix for the keys that should be removed
-   * @throws IOException Any IO/DDB failure.
-   * @throws InterruptedIOException if the prune was interrupted
-   * @return count of pruned items.
-   */
-  @Override
-  @Retries.RetryTranslated
-  public long prune(PruneMode pruneMode, long cutoff, String keyPrefix)
-      throws IOException {
-    LOG.debug("Prune {} under {} with age {}",
-        pruneMode == PruneMode.ALL_BY_MODTIME
-            ? "files and tombstones" : "tombstones",
-        keyPrefix, cutoff);
-    final ItemCollection<ScanOutcome> items =
-        expiredFiles(pruneMode, cutoff, keyPrefix);
-    return innerPrune(pruneMode, cutoff, keyPrefix, items);
-  }
-
-  /**
-   * Prune files, in batches. There's optionally a sleep between each batch.
-   *
-   * @param pruneMode The mode of operation for the prune For details see
-   *                  {@link MetadataStore#prune(PruneMode, long)}
-   * @param cutoff Oldest modification time to allow
-   * @param keyPrefix The prefix for the keys that should be removed
-   * @param items expired items
-   * @return count of pruned items.
-   * @throws IOException Any IO/DDB failure.
-   * @throws InterruptedIOException if the prune was interrupted
-   */
-  private int innerPrune(
-      final PruneMode pruneMode, final long cutoff, final String keyPrefix,
-      final ItemCollection<ScanOutcome> items)
-      throws IOException {
-    int itemCount = 0;
-    try (AncestorState state = initiateBulkWrite(
-        BulkOperationState.OperationType.Prune, null);
-         DurationInfo ignored =
-             new DurationInfo(LOG, "Pruning DynamoDB Store")) {
-      ArrayList<Path> deletionBatch =
-          new ArrayList<>(S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT);
-      long delay = conf.getTimeDuration(
-          S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
-          S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT,
-          TimeUnit.MILLISECONDS);
-      Set<Path> parentPathSet = new HashSet<>();
-      Set<Path> clearedParentPathSet = new HashSet<>();
-      // declare the operation to delete a batch as a function so
-      // as to keep the code consistent across multiple uses.
-      CallableRaisingIOE<Void> deleteBatchOperation =
-          () -> {
-            // lowest path entries get deleted first.
-            deletionBatch.sort(PathOrderComparators.TOPMOST_PATH_LAST);
-            processBatchWriteRequest(state, pathToKey(deletionBatch), null);
-
-            // set authoritative false for each pruned dir listing
-            // if at least one entry was not a tombstone
-            removeAuthoritativeDirFlag(parentPathSet, state);
-            // already cleared parent paths.
-            clearedParentPathSet.addAll(parentPathSet);
-            parentPathSet.clear();
-            return null;
-          };
-      for (Item item : items) {
-        DDBPathMetadata md = PathMetadataDynamoDBTranslation
-            .itemToPathMetadata(item, username);
-        Path path = md.getFileStatus().getPath();
-        boolean tombstone = md.isDeleted();
-        LOG.debug("Prune entry {}", path);
-        deletionBatch.add(path);
-
-        // add parent path of item so it can be marked as non-auth.
-        // this is only done if
-        // * it has not already been processed
-        // * the entry pruned is not a tombstone (no need to update)
-        // * the file is not in the root dir
-        Path parentPath = path.getParent();
-        if (!tombstone
-            && parentPath != null
-            && !parentPath.isRoot()
-            && !clearedParentPathSet.contains(parentPath)) {
-          parentPathSet.add(parentPath);
-        }
-
-        itemCount++;
-        if (deletionBatch.size() == S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT) {
-          deleteBatchOperation.apply();
-          deletionBatch.clear();
-          if (delay > 0) {
-            Thread.sleep(delay);
-          }
-        }
-      }
-      // final batch of deletes
-      if (!deletionBatch.isEmpty()) {
-        deleteBatchOperation.apply();
-      }
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      throw new InterruptedIOException("Pruning was interrupted");
-    } catch (AmazonDynamoDBException e) {
-      throw translateDynamoDBException(keyPrefix,
-          "Prune of " + keyPrefix + " failed", e);
-    }
-    LOG.info("Finished pruning {} items in batches of {}", itemCount,
-        S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT);
-    return itemCount;
-  }
-
-  /**
-   * Remove the Authoritative Directory Marker from a set of paths, if
-   * those paths are in the store.
-   * <p>
-   * This operation is <i>only</i>for pruning; it does not raise an error
-   * if, during the prune phase, the table appears inconsistent.
-   * This is not unusual as it can happen in a number of ways
-   * <ol>
-   *   <li>The state of the table changes during a slow prune operation which
-   *   deliberately inserts pauses to avoid overloading prepaid IO capacity.
-   *   </li>
-   *   <li>Tombstone markers have been left in the table after many other
-   *   operations have taken place, including deleting/replacing
-   *   parents.</li>
-   * </ol>
-   * <p>
-   *
-   * If an exception is raised in the get/update process, then the exception
-   * is caught and only rethrown after all the other paths are processed.
-   * This is to ensure a best-effort attempt to update the store.
-   * @param pathSet set of paths.
-   * @param state ongoing operation state.
-   * @throws IOException only after a best effort is made to update the store.
-   */
-  private void removeAuthoritativeDirFlag(
-      final Set<Path> pathSet,
-      final AncestorState state) throws IOException {
-
-    AtomicReference<IOException> rIOException = new AtomicReference<>();
-
-    Set<DDBPathMetadata> metas = pathSet.stream().map(path -> {
-      try {
-        if (path.isRoot()) {
-          LOG.debug("ignoring root path");
-          return null;
-        }
-        if (state != null && state.get(path) != null) {
-          // there's already an entry for this path
-          LOG.debug("Ignoring update of entry already in the state map");
-          return null;
-        }
-        DDBPathMetadata ddbPathMetadata = get(path);
-        if (ddbPathMetadata == null) {
-          // there is no entry.
-          LOG.debug("No parent {}; skipping", path);
-          return null;
-        }
-        if (ddbPathMetadata.isDeleted()) {
-          // the parent itself is deleted
-          LOG.debug("Parent has been deleted {}; skipping", path);
-          return null;
-        }
-        if (!ddbPathMetadata.getFileStatus().isDirectory()) {
-          // the parent itself is deleted
-          LOG.debug("Parent is not a directory {}; skipping", path);
-          return null;
-        }
-        LOG.debug("Setting isAuthoritativeDir==false on {}", ddbPathMetadata);
-        ddbPathMetadata.setAuthoritativeDir(false);
-        ddbPathMetadata.setLastUpdated(ttlTimeProvider.getNow());
-        return ddbPathMetadata;
-      } catch (IOException e) {
-        String msg = String.format("IOException while getting PathMetadata "
-            + "on path: %s.", path);
-        LOG.error(msg, e);
-        rIOException.set(e);
-        return null;
-      }
-    }).filter(Objects::nonNull).collect(Collectors.toSet());
-
-    try {
-      LOG.debug("innerPut on metas: {}", metas);
-      if (!metas.isEmpty()) {
-        innerPut(metas, state);
-      }
-    } catch (IOException e) {
-      String msg = String.format("IOException while setting false "
-          + "authoritative directory flag on: %s.", metas);
-      LOG.error(msg, e);
-      rIOException.set(e);
-    }
-
-    if (rIOException.get() != null) {
-      throw rIOException.get();
-    }
-  }
-
-  @VisibleForTesting
-  public AmazonDynamoDB getAmazonDynamoDB() {
-    return amazonDynamoDB;
-  }
-
-  @Override
-  public String toString() {
-    return getClass().getSimpleName() + '{'
-        + "region=" + region
-        + ", tableName=" + tableName
-        + ", tableArn=" + tableHandler.getTableArn()
-        + '}';
-  }
-
-  /**
-   * The administrative policy includes all DDB table operations;
-   * application access is restricted to those operations S3Guard operations
-   * require when working with data in a guarded bucket.
-   * @param access access level desired.
-   * @return a possibly empty list of statements.
-   */
-  @Override
-  public List<RoleModel.Statement> listAWSPolicyRules(
-      final Set<AccessLevel> access) {
-    Preconditions.checkState(tableHandler.getTableArn() != null,
-        "TableARN not known");
-    if (access.isEmpty()) {
-      return Collections.emptyList();
-    }
-    RoleModel.Statement stat;
-    if (access.contains(AccessLevel.ADMIN)) {
-      stat = allowAllDynamoDBOperations(tableHandler.getTableArn());
-    } else {
-      stat = allowS3GuardClientOperations(tableHandler.getTableArn());
-    }
-    return Lists.newArrayList(stat);
-  }
-
-  /**
-   * PUT a single item to the table.
-   * @param item item to put
-   * @return the outcome.
-   */
-  @Retries.OnceRaw
-  private PutItemOutcome putItem(Item item) {
-    LOG.debug("Putting item {}", item);
-    return table.putItem(item);
-  }
-
-  @VisibleForTesting
-  Table getTable() {
-    return table;
-  }
-
-  String getRegion() {
-    return region;
-  }
-
-  @VisibleForTesting
-  public String getTableName() {
-    return tableName;
-  }
-
-  @VisibleForTesting
-  DynamoDB getDynamoDB() {
-    return dynamoDB;
-  }
-
-  /**
-   * Validates a path object; it must be absolute, have an s3a:/// scheme
-   * and contain a host (bucket) component.
-   * @param path path to check
-   * @return the path passed in
-   */
-  private Path checkPath(Path path) {
-    Preconditions.checkNotNull(path);
-    Preconditions.checkArgument(path.isAbsolute(), "Path %s is not absolute",
-        path);
-    URI uri = path.toUri();
-    Preconditions.checkNotNull(uri.getScheme(), "Path %s missing scheme", path);
-    Preconditions.checkArgument(uri.getScheme().equals(Constants.FS_S3A),
-        "Path %s scheme must be %s", path, Constants.FS_S3A);
-    Preconditions.checkArgument(!StringUtils.isEmpty(uri.getHost()), "Path %s" +
-        " is missing bucket.", path);
-    return path;
-  }
-
-  /**
-   * Validates a path meta-data object.
-   */
-  private static void checkPathMetadata(PathMetadata meta) {
-    Preconditions.checkNotNull(meta);
-    Preconditions.checkNotNull(meta.getFileStatus());
-    Preconditions.checkNotNull(meta.getFileStatus().getPath());
-  }
-
-  @Override
-  @Retries.OnceRaw
-  public Map<String, String> getDiagnostics() throws IOException {
-    Map<String, String> map = new TreeMap<>();
-    if (table != null) {
-      TableDescription desc = getTableDescription(true);
-      map.put("name", desc.getTableName());
-      map.put(STATUS, desc.getTableStatus());
-      map.put("ARN", desc.getTableArn());
-      map.put("size", desc.getTableSizeBytes().toString());
-      map.put(TABLE, desc.toString());
-      ProvisionedThroughputDescription throughput
-          = desc.getProvisionedThroughput();
-      map.put(READ_CAPACITY, throughput.getReadCapacityUnits().toString());
-      map.put(WRITE_CAPACITY, throughput.getWriteCapacityUnits().toString());
-      map.put(BILLING_MODE,
-          throughput.getWriteCapacityUnits() == 0
-              ? BILLING_MODE_PER_REQUEST
-              : BILLING_MODE_PROVISIONED);
-      map.put("sse", desc.getSSEDescription() == null
-          ? "DISABLED"
-          : desc.getSSEDescription().toString());
-      map.put(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT,
-          Boolean.toString(true));
-    } else {
-      map.put("name", "DynamoDB Metadata Store");
-      map.put(TABLE, "none");
-      map.put(STATUS, "undefined");
-    }
-    map.put("description", DESCRIPTION);
-    map.put("region", region);
-    if (batchWriteRetryPolicy != null) {
-      map.put("retryPolicy", batchWriteRetryPolicy.toString());
-    }
-    return map;
-  }
-
-  @Retries.OnceRaw
-  private TableDescription getTableDescription(boolean forceUpdate) {
-    TableDescription desc = table.getDescription();
-    if (desc == null || forceUpdate) {
-      desc = table.describe();
-    }
-    return desc;
-  }
-
-  @Override
-  @Retries.OnceRaw
-  public void updateParameters(Map<String, String> parameters)
-      throws IOException {
-    Preconditions.checkNotNull(table, "Not initialized");
-    TableDescription desc = getTableDescription(true);
-    ProvisionedThroughputDescription current
-        = desc.getProvisionedThroughput();
-
-    long currentRead = current.getReadCapacityUnits();
-    long newRead = getLongParam(parameters,
-        S3GUARD_DDB_TABLE_CAPACITY_READ_KEY,
-        currentRead);
-    long currentWrite = current.getWriteCapacityUnits();
-    long newWrite = getLongParam(parameters,
-            S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY,
-            currentWrite);
-
-    if (currentRead == 0 || currentWrite == 0) {
-      // table is pay on demand
-      throw new IOException(E_ON_DEMAND_NO_SET_CAPACITY);
-    }
-
-    if (newRead != currentRead || newWrite != currentWrite) {
-      LOG.info("Current table capacity is read: {}, write: {}",
-          currentRead, currentWrite);
-      LOG.info("Changing capacity of table to read: {}, write: {}",
-          newRead, newWrite);
-      tableHandler.provisionTableBlocking(newRead, newWrite);
-    } else {
-      LOG.info("Table capacity unchanged at read: {}, write: {}",
-          newRead, newWrite);
-    }
-  }
-
-  private long getLongParam(Map<String, String> parameters,
-      String key,
-      long defVal) {
-    String k = parameters.get(key);
-    if (k != null) {
-      return Long.parseLong(k);
-    } else {
-      return defVal;
-    }
-  }
-
-  /**
-   * Callback on a read operation retried.
-   * @param text text of the operation
-   * @param ex exception
-   * @param attempts number of attempts
-   * @param idempotent is the method idempotent (this is assumed to be true)
-   */
-  void readRetryEvent(
-      String text,
-      IOException ex,
-      int attempts,
-      boolean idempotent) {
-    readThrottleEvents.incrementAndGet();
-    retryEvent(text, ex, attempts, true);
-  }
-
-  /**
-   * Callback  on a write operation retried.
-   * @param text text of the operation
-   * @param ex exception
-   * @param attempts number of attempts
-   * @param idempotent is the method idempotent (this is assumed to be true)
-   */
-  void writeRetryEvent(
-      String text,
-      IOException ex,
-      int attempts,
-      boolean idempotent) {
-    writeThrottleEvents.incrementAndGet();
-    retryEvent(text, ex, attempts, idempotent);
-  }
-
-  /**
-   * Callback on a scan operation retried.
-   * @param text text of the operation
-   * @param ex exception
-   * @param attempts number of attempts
-   * @param idempotent is the method idempotent (this is assumed to be true)
-   */
-  void scanRetryEvent(
-      String text,
-      IOException ex,
-      int attempts,
-      boolean idempotent) {
-    scanThrottleEvents.incrementAndGet();
-    retryEvent(text, ex, attempts, idempotent);
-  }
-
-  /**
-   * Callback from {@link Invoker} when an operation is retried.
-   * @param text text of the operation
-   * @param ex exception
-   * @param attempts number of attempts
-   * @param idempotent is the method idempotent
-   */
-  void retryEvent(
-      String text,
-      IOException ex,
-      int attempts,
-      boolean idempotent) {
-    if (S3AUtils.isThrottleException(ex)) {
-      // throttled
-      instrumentation.throttled();
-      int eventCount = throttleEventCount.addAndGet(1);
-      if (attempts == 1 && eventCount < THROTTLE_EVENT_LOG_LIMIT) {
-        LOG.warn("DynamoDB IO limits reached in {};"
-                + " consider increasing capacity: {}", text, ex.toString());
-        LOG.debug("Throttled", ex);
-      } else {
-        // user has been warned already, log at debug only.
-        LOG.debug("DynamoDB IO limits reached in {};"
-                + " consider increasing capacity: {}", text, ex.toString());
-      }
-    } else if (attempts == 1) {
-      // not throttled. Log on the first attempt only
-      LOG.info("Retrying {}: {}", text, ex.toString());
-      LOG.debug("Retrying {}", text, ex);
-    }
-
-    // note a retry
-    instrumentation.retrying();
-    if (owner != null) {
-      owner.metastoreOperationRetried(ex, attempts, idempotent);
-    }
-  }
-
-  /**
-   * Get the count of read throttle events.
-   * @return the current count of read throttle events.
-   */
-  @VisibleForTesting
-  public long getReadThrottleEventCount() {
-    return readThrottleEvents.get();
-  }
-
-  /**
-   * Get the count of write throttle events.
-   * @return the current count of write throttle events.
-   */
-  @VisibleForTesting
-  public long getWriteThrottleEventCount() {
-    return writeThrottleEvents.get();
-  }
-
-  /**
-   * Get the count of scan throttle events.
-   * @return the current count of scan throttle events.
-   */
-  @VisibleForTesting
-  public long getScanThrottleEventCount() {
-    return scanThrottleEvents.get();
-  }
-
-  @VisibleForTesting
-  public long getBatchWriteCapacityExceededCount() {
-    return batchWriteCapacityExceededEvents.get();
-  }
-
-  /**
-   * Get the operation invoker for write operations.
-   * @return an invoker for retrying mutating operations on a store.
-   */
-  public Invoker getInvoker() {
-    return writeOp;
-  }
-
-  /**
-   * Wrap an iterator returned from any scan with a retrying one.
-   * This includes throttle handling.
-   * Retries will update the relevant counters/metrics for scan operations.
-   * @param source source iterator
-   * @return a retrying iterator.
-   */
-  public <T> Iterable<T> wrapWithRetries(
-      final Iterable<T> source) {
-    return new RetryingCollection<>("scan dynamoDB table", scanOp, source);
-  }
-
-  /**
-   * Record the number of records written.
-   * @param count count of records.
-   */
-  private void recordsWritten(final int count) {
-    instrumentation.recordsWritten(count);
-  }
-
-  /**
-   * Record the number of records read.
-   * @param count count of records.
-   */
-  private void recordsRead(final int count) {
-    instrumentation.recordsRead(count);
-  }
-  /**
-   * Record the number of records deleted.
-   * @param count count of records.
-   */
-  private void recordsDeleted(final int count) {
-    instrumentation.recordsDeleted(count);
-  }
-
-  /**
-   * Initiate the rename operation by creating the tracker for the filesystem
-   * to keep up to date with state changes in the S3A bucket.
-   * @param storeContext store context.
-   * @param source source path
-   * @param sourceStatus status of the source file/dir
-   * @param dest destination path.
-   * @return the rename tracker
-   */
-  @Override
-  public RenameTracker initiateRenameOperation(
-      final StoreContext storeContext,
-      final Path source,
-      final S3AFileStatus sourceStatus,
-      final Path dest) {
-    return new ProgressiveRenameTracker(storeContext, this, source, dest,
-        new AncestorState(this, BulkOperationState.OperationType.Rename, dest));
-  }
-
-  /**
-   * Mark the directories instantiated under the destination path
-   * as authoritative. That is: all entries in the
-   * operationState (which must be an AncestorState instance),
-   * that are under the destination path.
-   *
-   * The database update synchronized on the operationState, so all other
-   * threads trying to update that state will be blocked until completion.
-   *
-   * This operation is only used in import and at the end of a rename,
-   * so this is not considered an issue.
-   * @param dest destination path.
-   * @param operationState active state.
-   * @throws IOException failure.
-   * @return the number of directories marked.
-   */
-  @Override
-  public int markAsAuthoritative(
-      final Path dest,
-      final BulkOperationState operationState) throws IOException {
-    if (operationState == null) {
-      return 0;
-    }
-    Preconditions.checkArgument(operationState instanceof AncestorState,
-        "Not an AncestorState %s", operationState);
-    final AncestorState state = (AncestorState)operationState;
-    // only mark paths under the dest as auth
-    final String simpleDestKey = pathToParentKey(dest);
-    final String destPathKey = simpleDestKey + "/";
-    final String opId = AncestorState.stateAsString(state);
-    LOG.debug("{}: marking directories under {} as authoritative",
-        opId, destPathKey);
-
-    // the list of dirs to build up.
-    final List<DDBPathMetadata> dirsToUpdate = new ArrayList<>();
-    synchronized (state) {
-      for (Map.Entry<Path, DDBPathMetadata> entry :
-          state.getAncestry().entrySet()) {
-        final Path path = entry.getKey();
-        final DDBPathMetadata md = entry.getValue();
-        final String key = pathToParentKey(path);
-        if (md.getFileStatus().isDirectory()
-            && (key.equals(simpleDestKey) || key.startsWith(destPathKey))) {
-          // the updated entry is under the destination.
-          md.setAuthoritativeDir(true);
-          md.setLastUpdated(ttlTimeProvider.getNow());
-          LOG.debug("{}: added {}", opId, key);
-          dirsToUpdate.add(md);
-        }
-      }
-      processBatchWriteRequest(state,
-          null, pathMetadataToItem(dirsToUpdate));
-    }
-    return dirsToUpdate.size();
-  }
-
-  @Override
-  public AncestorState initiateBulkWrite(
-      final BulkOperationState.OperationType operation,
-      final Path dest) {
-    return new AncestorState(this, operation, dest);
-  }
-
-  @Override
-  public void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider) {
-    this.ttlTimeProvider = ttlTimeProvider;
-  }
-
-  /**
-   * Username.
-   * @return the current username
-   */
-  String getUsername() {
-    return username;
-  }
-
-  /**
-   * Log a PUT into the operations log at debug level.
-   * @param state optional ancestor state.
-   * @param items items which have been PUT
-   */
-  private static void logPut(
-      @Nullable AncestorState state,
-      Item[] items) {
-    if (OPERATIONS_LOG.isDebugEnabled()) {
-      // log the operations
-      String stateStr = AncestorState.stateAsString(state);
-      for (Item item : items) {
-        boolean tombstone = !itemExists(item);
-        boolean isDir = getBoolAttribute(item, IS_DIR, false);
-        boolean auth = getBoolAttribute(item, IS_AUTHORITATIVE, false);
-        OPERATIONS_LOG.debug("{} {} {}{}{}",
-            stateStr,
-            tombstone ? "TOMBSTONE" : "PUT",
-            itemPrimaryKeyToString(item),
-            auth ? " [auth]" : "",
-            isDir ? " directory" : "");
-      }
-    }
-  }
-
-  /**
-   * Log a PUT into the operations log at debug level.
-   * @param state optional ancestor state.
-   * @param item item PUT.
-   */
-  private static void logPut(
-      @Nullable AncestorState state,
-      Item item) {
-    if (OPERATIONS_LOG.isDebugEnabled()) {
-      // log the operations
-      logPut(state, new Item[]{item});
-    }
-  }
-
-  /**
-   * Log a DELETE into the operations log at debug level.
-   * @param state optional ancestor state.
-   * @param keysDeleted keys which were deleted.
-   */
-  private static void logDelete(
-      @Nullable AncestorState state,
-      PrimaryKey[] keysDeleted) {
-    if (OPERATIONS_LOG.isDebugEnabled()) {
-      // log the operations
-      String stateStr = AncestorState.stateAsString(state);
-      for (PrimaryKey key : keysDeleted) {
-        OPERATIONS_LOG.debug("{} DELETE {}",
-            stateStr, primaryKeyToString(key));
-      }
-    }
-  }
-
-  /**
-   * Log a DELETE into the operations log at debug level.
-   * @param state optional ancestor state.
-   * @param key Deleted key
-   */
-  private static void logDelete(
-      @Nullable AncestorState state,
-      PrimaryKey key) {
-    if (OPERATIONS_LOG.isDebugEnabled()) {
-      logDelete(state, new PrimaryKey[]{key});
-    }
-  }
-
-  /**
-   * Get the move state passed in; create a new one if needed.
-   * @param state state.
-   * @param operation the type of the operation to use if the state is created.
-   * @return the cast or created state.
-   */
-  private AncestorState extractOrCreate(@Nullable BulkOperationState state,
-      BulkOperationState.OperationType operation) {
-    if (state != null) {
-      return (AncestorState) state;
-    } else {
-      return new AncestorState(this, operation, null);
-    }
-  }
-
-  @Override
-  public MetastoreInstrumentation getInstrumentation() {
-    return instrumentation;
-  }
-
-  /**
-   * This tracks all the ancestors created,
-   * across multiple move/write operations.
-   * This is to avoid duplicate creation of ancestors during bulk commits
-   * and rename operations managed by a rename tracker.
-   *
-   * There is no thread safety: callers must synchronize as appropriate.
-   */
-  @VisibleForTesting
-  static final class AncestorState extends BulkOperationState {
-
-    /**
-     * Counter of IDs issued.
-     */
-    private static final AtomicLong ID_COUNTER = new AtomicLong(0);
-
-    /** Owning store. */
-    private final DynamoDBMetadataStore store;
-
-    /** The ID of the state; for logging. */
-    private final long id;
-
-    /**
-     * Map of ancestors.
-     */
-    private final Map<Path, DDBPathMetadata> ancestry = new HashMap<>();
-
-    /**
-     * Destination path.
-     */
-    private final Path dest;
-
-    /**
-     * Create the state.
-     * @param store the store, for use in validation.
-     * If null: no validation (test only operation)
-     * @param operation the type of the operation.
-     * @param dest destination path.
-     */
-    AncestorState(
-        @Nullable final DynamoDBMetadataStore store,
-        final OperationType operation,
-        @Nullable final Path dest) {
-      super(operation);
-      this.store = store;
-      this.dest = dest;
-      this.id = ID_COUNTER.addAndGet(1);
-    }
-
-    int size() {
-      return ancestry.size();
-    }
-
-    /**
-     * Get the ancestry. Not thread safe.
-     * @return the map of ancestors.
-     */
-    Map<Path, DDBPathMetadata> getAncestry() {
-      return ancestry;
-    }
-
-    public Path getDest() {
-      return dest;
-    }
-
-    long getId() {
-      return id;
-    }
-
-    @Override
-    public String toString() {
-      final StringBuilder sb = new StringBuilder(
-          "AncestorState{");
-      sb.append("operation=").append(getOperation());
-      sb.append("id=").append(id);
-      sb.append("; dest=").append(dest);
-      sb.append("; size=").append(size());
-      sb.append("; paths={")
-          .append(StringUtils.join(ancestry.keySet(), " "))
-          .append('}');
-      sb.append('}');
-      return sb.toString();
-    }
-
-    /**
-     * Does the ancestor state contain a path?
-     * @param p path to check
-     * @return true if the state has an entry
-     */
-    boolean contains(Path p) {
-      return get(p) != null;
-    }
-
-    DDBPathMetadata put(Path p, DDBPathMetadata md) {
-      return ancestry.put(p, md);
-    }
-
-    DDBPathMetadata put(DDBPathMetadata md) {
-      return ancestry.put(md.getFileStatus().getPath(), md);
-    }
-
-    DDBPathMetadata get(Path p) {
-      return ancestry.get(p);
-    }
-
-    /**
-     * Find an entry in the ancestor state, warning and optionally
-     * raising an exception if there is a file at the path.
-     * @param path path to look up
-     * @param failOnFile fail if a file was found.
-     * @return true iff a directory was found in the ancestor state.
-     * @throws PathIOException if there was a file at the path.
-     */
-    boolean findEntry(
-        final Path path,
-        final boolean failOnFile) throws PathIOException {
-      final DDBPathMetadata ancestor = get(path);
-      if (ancestor != null) {
-        // there's an entry in the ancestor state
-        if (!ancestor.getFileStatus().isDirectory()) {
-          // but: its a file, which means this update is now inconsistent.
-          final String message = E_INCONSISTENT_UPDATE + " entry is " + ancestor
-              .getFileStatus();
-          LOG.error(message);
-          if (failOnFile) {
-            // errors trigger failure
-            throw new PathIOException(path.toString(), message);
-          }
-        }
-        return true;
-      } else {
-        return false;
-      }
-    }
-
-    /**
-     * If debug logging is enabled, this does an audit of the store state.
-     * it only logs this; the error messages are created so as they could
-     * be turned into exception messages.
-     * Audit failures aren't being turned into IOEs is that
-     * rename operations delete the source entry and that ends up in the
-     * ancestor state as present
-     * @throws IOException failure
-     */
-    @Override
-    public void close() throws IOException {
-      if (LOG.isDebugEnabled() && store != null) {
-        LOG.debug("Auditing {}", stateAsString(this));
-        for (Map.Entry<Path, DDBPathMetadata> entry : ancestry
-            .entrySet()) {
-          Path path = entry.getKey();
-          DDBPathMetadata expected = entry.getValue();
-          if (expected.isDeleted()) {
-            // file was deleted in bulk op; we don't care about it
-            // any more
-            continue;
-          }
-          DDBPathMetadata actual;
-          try {
-            actual = store.get(path);
-          } catch (IOException e) {
-            LOG.debug("Retrieving {}", path, e);
-            // this is for debug; don't be ambitious
-            return;
-          }
-          if (actual == null || actual.isDeleted()) {
-            String message = "Metastore entry for path "
-                + path + " deleted during bulk "
-                + getOperation() + " operation";
-            LOG.debug(message);
-          } else {
-            if (actual.getFileStatus().isDirectory() !=
-                expected.getFileStatus().isDirectory()) {
-              // the type of the entry has changed
-              String message = "Metastore entry for path "
-                  + path + " changed during bulk "
-                  + getOperation() + " operation"
-                  + " from " + expected
-                  + " to " + actual;
-              LOG.debug(message);
-            }
-          }
-
-        }
-      }
-    }
-
-    /**
-     * Create a string from the state including operation and ID.
-     * @param state state to use -may be null
-     * @return a string for logging.
-     */
-    private static String stateAsString(@Nullable AncestorState state) {
-      String stateStr;
-      if (state != null) {
-        stateStr = String.format("#(%s-%04d)",
-            state.getOperation(),
-            state.getId());
-      } else {
-        stateStr = "#()";
-      }
-      return stateStr;
-    }
-  }
-
-  protected DynamoDBMetadataStoreTableManager getTableHandler() {
-    Preconditions.checkNotNull(tableHandler, "Not initialized");
-    return tableHandler;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
deleted file mode 100644
index 9b9e62287ede4..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStoreTableManager.java
+++ /dev/null
@@ -1,756 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.nio.file.AccessDeniedException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SdkBaseException;
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.document.PutItemOutcome;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
-import com.amazonaws.services.dynamodbv2.model.BillingMode;
-import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
-import com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
-import com.amazonaws.services.dynamodbv2.model.ResourceInUseException;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import com.amazonaws.services.dynamodbv2.model.SSESpecification;
-import com.amazonaws.services.dynamodbv2.model.ScanRequest;
-import com.amazonaws.services.dynamodbv2.model.ScanResult;
-import com.amazonaws.services.dynamodbv2.model.TableDescription;
-import com.amazonaws.services.dynamodbv2.model.Tag;
-import com.amazonaws.services.dynamodbv2.model.TagResourceRequest;
-import com.amazonaws.waiters.WaiterTimedOutException;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSClientIOException;
-import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-
-import static java.lang.String.valueOf;
-
-import static org.apache.commons.lang3.StringUtils.isEmpty;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_READ_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_SSE_CMK;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_SSE_ENABLED;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
-import static org.apache.hadoop.fs.s3a.S3AUtils.lookupPassword;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateDynamoDBException;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.E_ON_DEMAND_NO_SET_CAPACITY;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION_MARKER_ITEM_NAME;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION_MARKER_TAG_NAME;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.attributeDefinitions;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.createVersionMarker;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.createVersionMarkerPrimaryKey;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.extractCreationTimeFromMarker;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.extractVersionFromMarker;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.keySchema;
-
-/**
- * Managing dynamo tables for S3Guard dynamodb based metadatastore.
- * Factored out from DynamoDBMetadataStore.
- */
-public class DynamoDBMetadataStoreTableManager {
-  public static final Logger LOG = LoggerFactory.getLogger(
-      DynamoDBMetadataStoreTableManager.class);
-
-  /** Error: version marker not found in table but the table is not empty. */
-  public static final String E_NO_VERSION_MARKER_AND_NOT_EMPTY
-      = "S3Guard table lacks version marker, and it is not empty.";
-
-  /** Error: version mismatch. */
-  public static final String E_INCOMPATIBLE_TAG_VERSION
-      = "Database table is from an incompatible S3Guard version based on table TAG.";
-
-  /** Error: version mismatch. */
-  public static final String E_INCOMPATIBLE_ITEM_VERSION
-      = "Database table is from an incompatible S3Guard version based on table ITEM.";
-
-  /** The AWS managed CMK for DynamoDB server side encryption. */
-  public static final String SSE_DEFAULT_MASTER_KEY = "alias/aws/dynamodb";
-
-  /** Invoker for IO. Until configured properly, use try-once. */
-  private Invoker invoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
-      Invoker.NO_OP
-  );
-
-  final private AmazonDynamoDB amazonDynamoDB;
-  final private DynamoDB dynamoDB;
-  final private String tableName;
-  final private String region;
-  final private Configuration conf;
-  final private Invoker readOp;
-  final private RetryPolicy batchWriteRetryPolicy;
-
-  private Table table;
-  private String tableArn;
-
-  public DynamoDBMetadataStoreTableManager(DynamoDB dynamoDB,
-      String tableName,
-      String region,
-      AmazonDynamoDB amazonDynamoDB,
-      Configuration conf,
-      Invoker readOp,
-      RetryPolicy batchWriteCapacityExceededEvents) {
-    this.dynamoDB = dynamoDB;
-    this.amazonDynamoDB = amazonDynamoDB;
-    this.tableName = tableName;
-    this.region = region;
-    this.conf = conf;
-    this.readOp = readOp;
-    this.batchWriteRetryPolicy = batchWriteCapacityExceededEvents;
-  }
-
-  /**
-   * Create a table if it does not exist and wait for it to become active.
-   *
-   * If a table with the intended name already exists, then it uses that table.
-   * Otherwise, it will automatically create the table if the config
-   * {@link org.apache.hadoop.fs.s3a.Constants#S3GUARD_DDB_TABLE_CREATE_KEY} is
-   * enabled. The DynamoDB table creation API is asynchronous.  This method wait
-   * for the table to become active after sending the creation request, so
-   * overall, this method is synchronous, and the table is guaranteed to exist
-   * after this method returns successfully.
-   *
-   * The wait for a table becoming active is Retry+Translated; it can fail
-   * while a table is not yet ready.
-   *
-   * @throws IOException if table does not exist and auto-creation is disabled;
-   * or table is being deleted, or any other I/O exception occurred.
-   */
-  @VisibleForTesting
-  @Retries.RetryTranslated
-  Table initTable() throws IOException {
-    table = dynamoDB.getTable(tableName);
-    try {
-      try {
-        LOG.debug("Binding to table {}", tableName);
-        TableDescription description = table.describe();
-        LOG.debug("Table state: {}", description);
-        tableArn = description.getTableArn();
-        final String status = description.getTableStatus();
-        switch (status) {
-        case "CREATING":
-          LOG.debug("Table {} in region {} is being created/updated. This may"
-                  + " indicate that the table is being operated by another "
-                  + "concurrent thread or process. Waiting for active...",
-              tableName, region);
-          waitForTableActive(table);
-          break;
-        case "DELETING":
-          throw new FileNotFoundException("DynamoDB table "
-              + "'" + tableName + "' is being "
-              + "deleted in region " + region);
-        case "UPDATING":
-          // table being updated; it can still be used.
-          LOG.debug("Table is being updated.");
-          break;
-        case "ACTIVE":
-          break;
-        default:
-          throw new IOException("Unknown DynamoDB table status " + status
-              + ": tableName='" + tableName + "', region=" + region);
-        }
-
-        verifyVersionCompatibility();
-        final Item versionMarker = getVersionMarkerItem();
-        Long created = extractCreationTimeFromMarker(versionMarker);
-        LOG.debug("Using existing DynamoDB table {} in region {} created {}",
-            tableName, region, (created != null) ? new Date(created) : null);
-      } catch (ResourceNotFoundException rnfe) {
-        if (conf.getBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false)) {
-          long readCapacity = conf.getLong(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY,
-              S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT);
-          long writeCapacity = conf.getLong(
-              S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY,
-              S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT);
-          ProvisionedThroughput capacity;
-          if (readCapacity > 0 && writeCapacity > 0) {
-            capacity = new ProvisionedThroughput(
-                readCapacity,
-                writeCapacity);
-          } else {
-            // at least one capacity value is <= 0
-            // verify they are both exactly zero
-            Preconditions.checkArgument(
-                readCapacity == 0 && writeCapacity == 0,
-                "S3Guard table read capacity %d and and write capacity %d"
-                    + " are inconsistent", readCapacity, writeCapacity);
-            // and set the capacity to null for per-request billing.
-            capacity = null;
-          }
-
-          createTable(capacity);
-        } else {
-          throw (FileNotFoundException) new FileNotFoundException(
-              "DynamoDB table '" + tableName + "' does not "
-                  + "exist in region " + region +
-                  "; auto-creation is turned off")
-              .initCause(rnfe);
-        }
-      }
-
-    } catch (AmazonClientException e) {
-      throw translateException("initTable", tableName, e);
-    }
-
-    return table;
-  }
-
-  protected void tagTableWithVersionMarker() throws AmazonDynamoDBException {
-    try {
-      TagResourceRequest tagResourceRequest = new TagResourceRequest()
-          .withResourceArn(table.getDescription().getTableArn())
-          .withTags(newVersionMarkerTag());
-      amazonDynamoDB.tagResource(tagResourceRequest);
-    } catch (AmazonDynamoDBException e) {
-      LOG.debug("Exception during tagging table: {}", e.getMessage(), e);
-    }
-  }
-
-  protected static Item getVersionMarkerFromTags(Table table,
-      AmazonDynamoDB addb) throws IOException {
-    List<Tag> tags = null;
-    try {
-      final TableDescription description = table.describe();
-      ListTagsOfResourceRequest listTagsOfResourceRequest =
-          new ListTagsOfResourceRequest()
-              .withResourceArn(description.getTableArn());
-      tags = addb.listTagsOfResource(listTagsOfResourceRequest).getTags();
-    } catch (ResourceNotFoundException e) {
-      LOG.error("Table: {} not found.", table.getTableName());
-      throw e;
-    } catch (AmazonDynamoDBException e) {
-      LOG.debug("Exception while getting tags from the dynamo table: {}",
-          e.getMessage(), e);
-      throw translateDynamoDBException(table.getTableName(),
-          "Retrieving tags.", e);
-    }
-
-    if (tags == null) {
-      return null;
-    }
-
-    final Optional<Tag> first = tags.stream()
-        .filter(tag -> tag.getKey().equals(VERSION_MARKER_TAG_NAME)).findFirst();
-    if (first.isPresent()) {
-      final Tag vmTag = first.get();
-      return createVersionMarker(
-          vmTag.getKey(), Integer.parseInt(vmTag.getValue()), 0
-      );
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Create a table, wait for it to become active, then add the version
-   * marker.
-   * Creating an setting up the table isn't wrapped by any retry operations;
-   * the wait for a table to become available is RetryTranslated.
-   * The tags are added to the table during creation, not after creation.
-   * We can assume that tagging and creating the table is a single atomic
-   * operation.
-   *
-   * @param capacity capacity to provision. If null: create a per-request
-   * table.
-   * @throws IOException on any failure.
-   * @throws InterruptedIOException if the wait was interrupted
-   */
-  @Retries.OnceMixed
-  private void createTable(ProvisionedThroughput capacity) throws IOException {
-    try {
-      String mode;
-      CreateTableRequest request = new CreateTableRequest()
-          .withTableName(tableName)
-          .withKeySchema(keySchema())
-          .withAttributeDefinitions(attributeDefinitions())
-          .withSSESpecification(getSseSpecFromConfig())
-          .withTags(getTableTagsFromConfig());
-      if (capacity != null) {
-        mode = String.format("with provisioned read capacity %d and"
-                + " write capacity %s",
-            capacity.getReadCapacityUnits(), capacity.getWriteCapacityUnits());
-        request.withProvisionedThroughput(capacity);
-      } else {
-        mode = "with pay-per-request billing";
-        request.withBillingMode(BillingMode.PAY_PER_REQUEST);
-      }
-      LOG.info("Creating non-existent DynamoDB table {} in region {} {}",
-          tableName, region, mode);
-      table = dynamoDB.createTable(request);
-      LOG.debug("Awaiting table becoming active");
-    } catch (ResourceInUseException e) {
-      LOG.warn("ResourceInUseException while creating DynamoDB table {} "
-              + "in region {}.  This may indicate that the table was "
-              + "created by another concurrent thread or process.",
-          tableName, region);
-    }
-    waitForTableActive(table);
-    putVersionMarkerItemToTable();
-  }
-
-  /**
-   * Get DynamoDB table server side encryption (SSE) settings from configuration.
-   */
-  private SSESpecification getSseSpecFromConfig() {
-    final SSESpecification sseSpecification = new SSESpecification();
-    boolean enabled = conf.getBoolean(S3GUARD_DDB_TABLE_SSE_ENABLED, false);
-    if (!enabled) {
-      // Do not set other options if SSE is disabled. Otherwise it will throw
-      // ValidationException.
-      return sseSpecification;
-    }
-    sseSpecification.setEnabled(Boolean.TRUE);
-    String cmk = null;
-    try {
-      // Get DynamoDB table SSE CMK from a configuration/credential provider.
-      cmk = lookupPassword("", conf, S3GUARD_DDB_TABLE_SSE_CMK);
-    } catch (IOException e) {
-      LOG.error("Cannot retrieve " + S3GUARD_DDB_TABLE_SSE_CMK, e);
-    }
-    if (isEmpty(cmk)) {
-      // Using Amazon managed default master key for DynamoDB table
-      return sseSpecification;
-    }
-    if (SSE_DEFAULT_MASTER_KEY.equals(cmk)) {
-      LOG.warn("Ignoring default DynamoDB table KMS Master Key {}",
-          SSE_DEFAULT_MASTER_KEY);
-    } else {
-      sseSpecification.setSSEType("KMS");
-      sseSpecification.setKMSMasterKeyId(cmk);
-    }
-    return sseSpecification;
-  }
-
-  /**
-   *  Return tags from configuration and the version marker for adding to
-   *  dynamo table during creation.
-   */
-  @Retries.OnceRaw
-  public List<Tag> getTableTagsFromConfig() {
-    List<Tag> tags = new ArrayList<>();
-
-    // from configuration
-    Map<String, String> tagProperties =
-        conf.getPropsWithPrefix(S3GUARD_DDB_TABLE_TAG);
-    for (Map.Entry<String, String> tagMapEntry : tagProperties.entrySet()) {
-      Tag tag = new Tag().withKey(tagMapEntry.getKey())
-          .withValue(tagMapEntry.getValue());
-      tags.add(tag);
-    }
-    // add the version marker
-    tags.add(newVersionMarkerTag());
-    return tags;
-  }
-
-  /**
-   * Create a new version marker tag.
-   * @return a new version marker tag
-   */
-  private static Tag newVersionMarkerTag() {
-    return new Tag().withKey(VERSION_MARKER_TAG_NAME).withValue(valueOf(VERSION));
-  }
-
-  /**
-   * Verify that a table version is compatible with this S3Guard client.
-   *
-   * Checks for consistency between the version marker as the item and tag.
-   *
-   * <pre>
-   *   1. If the table lacks both version markers AND it's empty,
-   *      both markers will be added.
-   *      If the table is not empty the check throws IOException
-   *   2. If there's no version marker ITEM, the compatibility with the TAG
-   *      will be checked, and the version marker ITEM will be added if the
-   *      TAG version is compatible.
-   *      If the TAG version is not compatible, the check throws OException
-   *   3. If there's no version marker TAG, the compatibility with the ITEM
-   *      version marker will be checked, and the version marker ITEM will be
-   *      added if the ITEM version is compatible.
-   *      If the ITEM version is not compatible, the check throws IOException
-   *   4. If the TAG and ITEM versions are both present then both will be checked
-   *      for compatibility. If the ITEM or TAG version marker is not compatible,
-   *      the check throws IOException
-   * </pre>
-   *
-   * @throws IOException on any incompatibility
-   */
-  @VisibleForTesting
-  protected void verifyVersionCompatibility() throws IOException {
-    final Item versionMarkerItem = getVersionMarkerItem();
-    Item versionMarkerFromTag = null;
-    boolean canReadDdbTags = true;
-
-    try {
-      versionMarkerFromTag = getVersionMarkerFromTags(table, amazonDynamoDB);
-    } catch (AccessDeniedException e) {
-      LOG.debug("Can not read tags of table.");
-      canReadDdbTags = false;
-    }
-
-    LOG.debug("versionMarkerItem: {};  versionMarkerFromTag: {}",
-        versionMarkerItem, versionMarkerFromTag);
-
-    if (versionMarkerItem == null && versionMarkerFromTag == null) {
-      if (!isEmptyTable(tableName, amazonDynamoDB)) {
-        LOG.error("Table is not empty but missing the version maker. Failing.");
-        throw new IOException(E_NO_VERSION_MARKER_AND_NOT_EMPTY
-            + " Table: " + tableName);
-      }
-
-      if (canReadDdbTags) {
-        LOG.info("Table {} contains no version marker item and tag. " +
-            "The table is empty, so the version marker will be added " +
-            "as TAG and ITEM.", tableName);
-        putVersionMarkerItemToTable();
-        tagTableWithVersionMarker();
-      }
-
-      if (!canReadDdbTags) {
-        LOG.info("Table {} contains no version marker item and the tags are not readable. " +
-            "The table is empty, so the ITEM version marker will be added .", tableName);
-        putVersionMarkerItemToTable();
-      }
-    }
-
-    if (versionMarkerItem == null && versionMarkerFromTag != null) {
-      final int tagVersionMarker =
-          extractVersionFromMarker(versionMarkerFromTag);
-      throwExceptionOnVersionMismatch(tagVersionMarker, tableName,
-          E_INCOMPATIBLE_TAG_VERSION);
-
-      LOG.info("Table {} contains no version marker ITEM but contains " +
-              "compatible version marker TAG. Restoring the version marker " +
-              "item from tag.", tableName);
-
-      putVersionMarkerItemToTable();
-    }
-
-    if (versionMarkerItem != null && versionMarkerFromTag == null
-        && canReadDdbTags) {
-      final int itemVersionMarker =
-          extractVersionFromMarker(versionMarkerItem);
-      throwExceptionOnVersionMismatch(itemVersionMarker, tableName,
-          E_INCOMPATIBLE_ITEM_VERSION);
-
-      LOG.info("Table {} contains no version marker TAG but contains " +
-          "compatible version marker ITEM. Restoring the version marker " +
-          "item from item.", tableName);
-
-      tagTableWithVersionMarker();
-    }
-
-    if (versionMarkerItem != null && versionMarkerFromTag != null) {
-      final int tagVersionMarker =
-          extractVersionFromMarker(versionMarkerFromTag);
-      final int itemVersionMarker =
-          extractVersionFromMarker(versionMarkerItem);
-
-      throwExceptionOnVersionMismatch(tagVersionMarker, tableName,
-          E_INCOMPATIBLE_TAG_VERSION);
-      throwExceptionOnVersionMismatch(itemVersionMarker, tableName,
-          E_INCOMPATIBLE_ITEM_VERSION);
-
-      LOG.debug("Table {} contains correct version marker TAG and ITEM.",
-          tableName);
-    }
-  }
-
-  private static boolean isEmptyTable(String tableName, AmazonDynamoDB aadb) {
-    final ScanRequest req = new ScanRequest().withTableName(
-        tableName).withLimit(1);
-    final ScanResult result = aadb.scan(req);
-    return result.getCount() == 0;
-  }
-
-  private static void throwExceptionOnVersionMismatch(int actual,
-      String tableName,
-      String exMsg) throws IOException {
-
-    if (VERSION != actual) {
-      throw new IOException(exMsg + " Table " + tableName
-          + " Expected version: " + VERSION + " actual tag version: " +
-          actual);
-    }
-  }
-
-  /**
-   * Add version marker to the dynamo table.
-   */
-  @Retries.OnceRaw
-  private void putVersionMarkerItemToTable() {
-    final Item marker = createVersionMarker(VERSION_MARKER_ITEM_NAME, VERSION,
-        System.currentTimeMillis());
-    putItem(marker);
-  }
-
-  /**
-   * Wait for table being active.
-   * @param t table to block on.
-   * @throws IOException IO problems
-   * @throws InterruptedIOException if the wait was interrupted
-   * @throws IllegalArgumentException if an exception was raised in the waiter
-   */
-  @Retries.RetryTranslated
-  private void waitForTableActive(Table t) throws IOException {
-    invoker.retry("Waiting for active state of table " + tableName,
-        null,
-        true,
-        () -> {
-          try {
-            t.waitForActive();
-          } catch (IllegalArgumentException ex) {
-            throw translateTableWaitFailure(tableName, ex);
-          } catch (InterruptedException e) {
-            LOG.warn("Interrupted while waiting for table {} in region {}"
-                    + " active",
-                tableName, region, e);
-            Thread.currentThread().interrupt();
-            throw (InterruptedIOException)
-                new InterruptedIOException("DynamoDB table '"
-                    + tableName + "' is not active yet in region " + region)
-                    .initCause(e);
-          }
-        });
-  }
-
-  /**
-   * Handle a table wait failure by extracting any inner cause and
-   * converting it, or, if unconvertable by wrapping
-   * the IllegalArgumentException in an IOE.
-   *
-   * @param name name of the table
-   * @param e exception
-   * @return an IOE to raise.
-   */
-  @VisibleForTesting
-  static IOException translateTableWaitFailure(
-      final String name, IllegalArgumentException e) {
-    final SdkBaseException ex = extractInnerException(e);
-    if (ex != null) {
-      if (ex instanceof WaiterTimedOutException) {
-        // a timeout waiting for state change: extract the
-        // message from the outer exception, but translate
-        // the inner one for the throttle policy.
-        return new AWSClientIOException(e.getMessage(), ex);
-      } else {
-        return translateException(e.getMessage(), name, ex);
-      }
-    } else {
-      return new IOException(e);
-    }
-  }
-
-  /**
-   * Take an {@code IllegalArgumentException} raised by a DDB operation
-   * and if it contains an inner SDK exception, unwrap it.
-   * @param ex exception.
-   * @return the inner AWS exception or null.
-   */
-  public static SdkBaseException extractInnerException(
-      IllegalArgumentException ex) {
-    if (ex.getCause() instanceof SdkBaseException) {
-      return (SdkBaseException) ex.getCause();
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Get the version mark item in the existing DynamoDB table.
-   *
-   * As the version marker item may be created by another concurrent thread or
-   * process, we sleep and retry a limited number times if the lookup returns
-   * with a null value.
-   * DDB throttling is always retried.
-   */
-  @VisibleForTesting
-  @Retries.RetryTranslated
-  protected Item getVersionMarkerItem() throws IOException {
-    final PrimaryKey versionMarkerKey =
-        createVersionMarkerPrimaryKey(VERSION_MARKER_ITEM_NAME);
-    int retryCount = 0;
-    // look for a version marker, with usual throttling/failure retries.
-    Item versionMarker = queryVersionMarker(versionMarkerKey);
-    while (versionMarker == null) {
-      // The marker was null.
-      // Two possibilities
-      // 1. This isn't a S3Guard table.
-      // 2. This is a S3Guard table in construction; another thread/process
-      //    is about to write/actively writing the version marker.
-      // So that state #2 is handled, batchWriteRetryPolicy is used to manage
-      // retries.
-      // This will mean that if the cause is actually #1, failure will not
-      // be immediate. As this will ultimately result in a failure to
-      // init S3Guard and the S3A FS, this isn't going to be a performance
-      // bottleneck -simply a slightly slower failure report than would otherwise
-      // be seen.
-      // "if your settings are broken, performance is not your main issue"
-      try {
-        RetryPolicy.RetryAction action = batchWriteRetryPolicy.shouldRetry(null,
-            retryCount, 0, true);
-        if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
-          break;
-        } else {
-          LOG.warn("No version marker found in the DynamoDB table: {}. " +
-              "Sleeping {} ms before next retry", tableName, action.delayMillis);
-          Thread.sleep(action.delayMillis);
-        }
-      } catch (Exception e) {
-        throw new IOException("initTable: Unexpected exception " + e, e);
-      }
-      retryCount++;
-      versionMarker = queryVersionMarker(versionMarkerKey);
-    }
-    return versionMarker;
-  }
-
-  /**
-   * Issue the query to get the version marker, with throttling for overloaded
-   * DDB tables.
-   * @param versionMarkerKey key to look up
-   * @return the marker
-   * @throws IOException failure
-   */
-  @Retries.RetryTranslated
-  private Item queryVersionMarker(final PrimaryKey versionMarkerKey)
-      throws IOException {
-    return readOp.retry("getVersionMarkerItem",
-        VERSION_MARKER_ITEM_NAME, true,
-        () -> table.getItem(versionMarkerKey));
-  }
-
-  /**
-   * PUT a single item to the table.
-   * @param item item to put
-   * @return the outcome.
-   */
-  @Retries.OnceRaw
-  private PutItemOutcome putItem(Item item) {
-    LOG.debug("Putting item {}", item);
-    return table.putItem(item);
-  }
-
-  /**
-   * Provision the table with given read and write capacity units.
-   * Call will fail if the table is busy, or the new values match the current
-   * ones.
-   * <p>
-   * Until the AWS SDK lets us switch a table to on-demand, an attempt to
-   * set the I/O capacity to zero will fail.
-   * @param readCapacity read units: must be greater than zero
-   * @param writeCapacity write units: must be greater than zero
-   * @throws IOException on a failure
-   */
-  @Retries.RetryTranslated
-  void provisionTable(Long readCapacity, Long writeCapacity)
-      throws IOException {
-
-    if (readCapacity == 0 || writeCapacity == 0) {
-      // table is pay on demand
-      throw new IOException(E_ON_DEMAND_NO_SET_CAPACITY);
-    }
-    final ProvisionedThroughput toProvision = new ProvisionedThroughput()
-        .withReadCapacityUnits(readCapacity)
-        .withWriteCapacityUnits(writeCapacity);
-    invoker.retry("ProvisionTable", tableName, true,
-        () -> {
-          final ProvisionedThroughputDescription p =
-              table.updateTable(toProvision).getProvisionedThroughput();
-          LOG.info("Provision table {} in region {}: readCapacityUnits={}, "
-                  + "writeCapacityUnits={}",
-              tableName, region, p.getReadCapacityUnits(),
-              p.getWriteCapacityUnits());
-        });
-  }
-
-  @Retries.RetryTranslated
-  public void destroy() throws IOException {
-    if (table == null) {
-      LOG.info("In destroy(): no table to delete");
-      return;
-    }
-    LOG.info("Deleting DynamoDB table {} in region {}", tableName, region);
-    Preconditions.checkNotNull(dynamoDB, "Not connected to DynamoDB");
-    try {
-      invoker.retry("delete", null, true,
-          () -> table.delete());
-      table.waitForDelete();
-    } catch (IllegalArgumentException ex) {
-      throw new TableDeleteTimeoutException(tableName,
-          "Timeout waiting for the table " + getTableArn()
-              + " to be deleted", ex);
-    } catch (FileNotFoundException rnfe) {
-      LOG.info("FileNotFoundException while deleting DynamoDB table {} in "
-              + "region {}.  This may indicate that the table does not exist, "
-              + "or has been deleted by another concurrent thread or process.",
-          tableName, region);
-    } catch (InterruptedException ie) {
-      Thread.currentThread().interrupt();
-      LOG.warn("Interrupted while waiting for DynamoDB table {} being deleted",
-          tableName, ie);
-      throw new InterruptedIOException("Table " + tableName
-          + " in region " + region + " has not been deleted");
-    }
-  }
-
-  @Retries.RetryTranslated
-  @VisibleForTesting
-  void provisionTableBlocking(Long readCapacity, Long writeCapacity)
-      throws IOException {
-    provisionTable(readCapacity, writeCapacity);
-    waitForTableActive(table);
-  }
-
-  public Table getTable() {
-    return table;
-  }
-
-  public String getTableArn() {
-    return tableArn;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java
deleted file mode 100644
index aa7fc4721b483..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ITtlTimeProvider.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-/**
- * This interface is defined for handling TTL expiry of metadata in S3Guard.
- *
- * TTL can be tested by implementing this interface and setting is as
- * {@code S3Guard.ttlTimeProvider}. By doing this, getNow() can return any
- * value preferred and flaky tests could be avoided. By default getNow()
- * returns the EPOCH in runtime.
- *
- * Time is measured in milliseconds,
- */
-public interface ITtlTimeProvider {
-
-  /**
-   * The current time in milliseconds.
-   * Assuming this calls System.currentTimeMillis(), this is a native iO call
-   * and so should be invoked sparingly (i.e. evaluate before any loop, rather
-   * than inside).
-   * @return the current time.
-   */
-  long getNow();
-
-  /**
-   * The TTL of the metadata.
-   * @return time in millis after which metadata is considered out of date.
-   */
-  long getMetadataTtl();
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ImportOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ImportOperation.java
deleted file mode 100644
index 9c8b702228608..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ImportOperation.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
-import org.apache.hadoop.fs.s3a.impl.ExecutingStoreOperation;
-import org.apache.hadoop.util.DurationInfo;
-
-/**
- * Import a directory tree into the metastore.
- * This code was moved from S3GuardTool and enhanced to mark
- * the destination tree as authoritative.
- */
-class ImportOperation extends ExecutingStoreOperation<Long> {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      ImportOperation.class);
-
-  /**
-   * Source file system: must not be guarded.
-   */
-  private final S3AFileSystem filesystem;
-
-  /**
-   * Destination metadata store.
-   */
-  private final MetadataStore store;
-
-  /**
-   * Source entry: File or directory.
-   */
-  private final S3AFileStatus status;
-
-  /**
-   * If importing the directory tree -should it be marked
-   * authoritative afterwards?
-   */
-  private final boolean authoritative;
-
-  private final boolean verbose;
-
-  /**
-   * For DDB the BulkOperation tracking eliminates the need for this cache,
-   * but it is retained here for the local store and to allow for
-   * ease of moving to operations which may update the store in parallel with
-   * writing.
-   */
-  private final Set<Path> dirCache = new HashSet<>();
-
-  /**
-   * Import.
-   * @param filesystem Unguarded FS to scan.
-   * @param store store to update
-   * @param status source status
-   * @param authoritative should the imported tree be marked as authoritative
-   * @param verbose Verbose output
-   */
-  ImportOperation(final S3AFileSystem filesystem,
-      final MetadataStore store,
-      final S3AFileStatus status,
-      final boolean authoritative,
-      final boolean verbose) {
-    super(filesystem.createStoreContext());
-    this.verbose = verbose;
-    Preconditions.checkState(!filesystem.hasMetadataStore(),
-        "Source filesystem for import has a metadata store");
-    this.filesystem = filesystem;
-    this.store = store;
-    this.status = status;
-    this.authoritative = authoritative;
-  }
-
-  private S3AFileSystem getFilesystem() {
-    return filesystem;
-  }
-
-  private MetadataStore getStore() {
-    return store;
-  }
-
-  private FileStatus getStatus() {
-    return status;
-  }
-
-  @Override
-  public Long execute() throws IOException {
-    final long items;
-    if (status.isFile()) {
-      PathMetadata meta = new PathMetadata(status);
-      getStore().put(meta, null);
-      items = 1;
-    } else {
-      try (DurationInfo ignored =
-               new DurationInfo(LOG, "Importing %s", getStatus().getPath())) {
-        items = importDir();
-      }
-    }
-    return items;
-  }
-
-  /**
-   * Recursively import every path under path.
-   * @return number of items inserted into MetadataStore
-   * @throws IOException on I/O errors.
-   */
-  private long importDir() throws IOException {
-    Preconditions.checkArgument(status.isDirectory());
-    long totalCountOfEntriesWritten = 0;
-    final Path basePath = status.getPath();
-    final MetadataStore ms = getStore();
-    LOG.info("Importing directory {}", basePath);
-    try (BulkOperationState operationState = ms
-        .initiateBulkWrite(
-            BulkOperationState.OperationType.Import,
-            basePath)) {
-      long countOfFilesWritten = 0;
-      long countOfDirsWritten = 0;
-      RemoteIterator<S3ALocatedFileStatus> it = getFilesystem()
-          .listFilesAndEmptyDirectoriesForceNonAuth(basePath, true);
-      while (it.hasNext()) {
-        S3ALocatedFileStatus located = it.next();
-        S3AFileStatus child;
-        final Path path = located.getPath();
-        final boolean isDirectory = located.isDirectory();
-        if (isDirectory) {
-          child = DynamoDBMetadataStore.makeDirStatus(path,
-              located.getOwner());
-          dirCache.add(path);
-          // and update the dir count
-          countOfDirsWritten++;
-        } else {
-          child = located.toS3AFileStatus();
-        }
-
-        int parentsWritten = putParentsIfNotPresent(child, operationState);
-        LOG.debug("Wrote {} parent entries", parentsWritten);
-
-        // We don't blindly overwrite any existing file entry in S3Guard with a
-        // new one, Because that may lose the version information.
-        // instead we merge them
-        if (!isDirectory) {
-          final PathMetadata existingEntry = S3Guard.getWithTtl(ms, path, null,
-              false, true);
-          if (existingEntry != null) {
-            final S3AFileStatus existingStatus = existingEntry.getFileStatus();
-            if (existingStatus.isFile()) {
-              // source is also a file.
-              // we only worry about an update if the timestamp is different,
-              final String existingEtag = existingStatus.getETag();
-              final String childEtag = child.getETag();
-              if (child.getModificationTime()
-                  != existingStatus.getModificationTime()
-                  || existingStatus.getLen() != child.getLen()
-                  || existingEtag == null
-                  || !existingEtag.equals(childEtag)) {
-                // files are potentially different, though a modtime change
-                // can just be a clock skew problem
-                // so if the etag is unchanged, we propagate any versionID
-                if (childEtag.equals(existingEtag)) {
-                  // copy over any version ID.
-                  child.setVersionId(existingStatus.getVersionId());
-                }
-              } else {
-                // the entry modtimes match
-                child = null;
-              }
-            }
-          }
-          if (child != null) {
-            countOfFilesWritten++;
-          }
-        }
-        if (child != null) {
-          // there's an entry to add.
-
-          // log entry spaced to same width
-          String t = isDirectory ? "Dir " : "File";
-          if (verbose) {
-            LOG.info("{} {}", t, path);
-          } else {
-            LOG.debug("{} {}", t, path);
-          }
-          S3Guard.putWithTtl(
-              ms,
-              new PathMetadata(child),
-              getFilesystem().getTtlTimeProvider(),
-              operationState);
-          totalCountOfEntriesWritten++;
-        }
-      }
-      LOG.info("Updated S3Guard with {} files and {} directory entries",
-          countOfFilesWritten, countOfDirsWritten);
-
-      // here all entries are imported.
-      // tell the store that everything should be marked as auth
-      if (authoritative) {
-        LOG.info("Marking directory tree {} as authoritative",
-            basePath);
-        ms.markAsAuthoritative(basePath, operationState);
-      }
-    }
-    return totalCountOfEntriesWritten;
-  }
-
-  /**
-   * Put parents into metastore and cache if the parents are not present.
-   *
-   * There's duplication here with S3Guard DDB ancestor state, but this
-   * is designed to work across implementations.
-   * @param fileStatus the file or an empty directory.
-   * @param operationState store's bulk update state.
-   * @return number of entries written.
-   * @throws IOException on I/O errors.
-   */
-  private int putParentsIfNotPresent(FileStatus fileStatus,
-      @Nullable BulkOperationState operationState) throws IOException {
-    Preconditions.checkNotNull(fileStatus);
-    Path parent = fileStatus.getPath().getParent();
-    int count = 0;
-    while (parent != null) {
-      if (dirCache.contains(parent)) {
-        return count;
-      }
-      final ITtlTimeProvider timeProvider
-          = getFilesystem().getTtlTimeProvider();
-      final PathMetadata pmd = S3Guard.getWithTtl(getStore(), parent,
-          timeProvider, false, true);
-      if (pmd == null || pmd.isDeleted()) {
-        S3AFileStatus dir = DynamoDBMetadataStore.makeDirStatus(parent,
-            fileStatus.getOwner());
-        S3Guard.putWithTtl(getStore(), new PathMetadata(dir),
-            timeProvider,
-            operationState);
-        count++;
-      }
-      dirCache.add(parent);
-      parent = parent.getParent();
-    }
-    return count;
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
deleted file mode 100644
index 540507497586e..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-
-/**
- * LocalMetadataEntry is used to store entries in the cache of
- * LocalMetadataStore. PathMetadata or dirListingMetadata can be null. The
- * entry is not immutable.
- */
-public final class LocalMetadataEntry {
-  @Nullable
-  private PathMetadata pathMetadata;
-  @Nullable
-  private DirListingMetadata dirListingMetadata;
-
-  LocalMetadataEntry() {
-  }
-
-  LocalMetadataEntry(PathMetadata pmd){
-    pathMetadata = pmd;
-    dirListingMetadata = null;
-  }
-
-  LocalMetadataEntry(DirListingMetadata dlm){
-    pathMetadata = null;
-    dirListingMetadata = dlm;
-  }
-
-  public PathMetadata getFileMeta() {
-    return pathMetadata;
-  }
-
-  public DirListingMetadata getDirListingMeta() {
-    return dirListingMetadata;
-  }
-
-
-  public boolean hasPathMeta() {
-    return this.pathMetadata != null;
-  }
-
-  public boolean hasDirMeta() {
-    return this.dirListingMetadata != null;
-  }
-
-  public void setPathMetadata(PathMetadata pathMetadata) {
-    this.pathMetadata = pathMetadata;
-  }
-
-  public void setDirListingMetadata(DirListingMetadata dirListingMetadata) {
-    this.dirListingMetadata = dirListingMetadata;
-  }
-
-  @Override public String toString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("LocalMetadataEntry{");
-    if(pathMetadata != null) {
-      sb.append("pathMetadata=" + pathMetadata.getFileStatus().getPath());
-    }
-    if(dirListingMetadata != null){
-      sb.append("; dirListingMetadata=" + dirListingMetadata.getPath());
-    }
-    sb.append("}");
-    return sb.toString();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
deleted file mode 100644
index 0b9ac46b409bc..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ /dev/null
@@ -1,651 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * This is a local, in-memory implementation of MetadataStore.
- * This is <i>not</i> a coherent cache across processes.  It is only
- * locally-coherent.
- *
- * The purpose of this is for unit and integration testing.
- * It could also be used to accelerate local-only operations where only one
- * process is operating on a given object store, or multiple processes are
- * accessing a read-only storage bucket.
- *
- * This MetadataStore does not enforce filesystem rules such as disallowing
- * non-recursive removal of non-empty directories.  It is assumed the caller
- * already has to perform these sorts of checks.
- *
- * Contains one cache internally with time based eviction.
- */
-public class LocalMetadataStore implements MetadataStore {
-
-  public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
-
-  /** Contains directory and file listings. */
-  private Cache<Path, LocalMetadataEntry> localCache;
-
-  private FileSystem fs;
-  /* Null iff this FS does not have an associated URI host. */
-  private String uriHost;
-
-  private String username;
-
-  private ITtlTimeProvider ttlTimeProvider;
-
-  @Override
-  public void initialize(FileSystem fileSystem,
-      ITtlTimeProvider ttlTp) throws IOException {
-    Preconditions.checkNotNull(fileSystem);
-    fs = fileSystem;
-    URI fsURI = fs.getUri();
-    uriHost = fsURI.getHost();
-    if (uriHost != null && uriHost.equals("")) {
-      uriHost = null;
-    }
-
-    initialize(fs.getConf(), ttlTp);
-  }
-
-  @Override
-  public void initialize(Configuration conf, ITtlTimeProvider ttlTp)
-      throws IOException {
-    Preconditions.checkNotNull(conf);
-    int maxRecords = conf.getInt(S3GUARD_METASTORE_LOCAL_MAX_RECORDS,
-        DEFAULT_S3GUARD_METASTORE_LOCAL_MAX_RECORDS);
-    if (maxRecords < 4) {
-      maxRecords = 4;
-    }
-    int ttl = conf.getInt(S3GUARD_METASTORE_LOCAL_ENTRY_TTL,
-        DEFAULT_S3GUARD_METASTORE_LOCAL_ENTRY_TTL);
-
-    CacheBuilder builder = CacheBuilder.newBuilder().maximumSize(maxRecords);
-    if (ttl >= 0) {
-      builder.expireAfterAccess(ttl, TimeUnit.MILLISECONDS);
-    }
-
-    localCache = builder.build();
-    username = UserGroupInformation.getCurrentUser().getShortUserName();
-    this.ttlTimeProvider = ttlTp;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb = new StringBuilder(
-        "LocalMetadataStore{");
-    sb.append("uriHost='").append(uriHost).append('\'');
-    sb.append('}');
-    return sb.toString();
-  }
-
-  @Override
-  public void delete(Path p,
-      final BulkOperationState operationState)
-      throws IOException {
-    doDelete(p, false, true);
-  }
-
-  @Override
-  public void forgetMetadata(Path p) throws IOException {
-    doDelete(p, false, false);
-  }
-
-  @Override
-  public void deleteSubtree(Path path,
-      final BulkOperationState operationState)
-      throws IOException {
-    doDelete(path, true, true);
-  }
-
-  private synchronized void doDelete(Path p, boolean recursive,
-      boolean tombstone) {
-
-    Path path = standardize(p);
-
-    // Delete entry from file cache, then from cached parent directory, if any
-    deleteCacheEntries(path, tombstone);
-
-    if (recursive) {
-      // Remove all entries that have this dir as path prefix.
-      deleteEntryByAncestor(path, localCache, tombstone, ttlTimeProvider);
-    }
-  }
-
-  @Override
-  public void deletePaths(final Collection<Path> paths,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    for (Path path : paths) {
-      doDelete(path, false, true);
-    }
-  }
-
-  @Override
-  public synchronized PathMetadata get(Path p) throws IOException {
-    return get(p, false);
-  }
-
-  @Override
-  public PathMetadata get(Path p, boolean wantEmptyDirectoryFlag)
-      throws IOException {
-    Path path = standardize(p);
-    synchronized (this) {
-      PathMetadata m = getFileMeta(path);
-
-      if (wantEmptyDirectoryFlag && m != null &&
-          m.getFileStatus().isDirectory()) {
-        m.setIsEmptyDirectory(isEmptyDirectory(p));
-      }
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("get({}) -> {}", path, m == null ? "null" : m.prettyPrint());
-      }
-      return m;
-    }
-  }
-
-  /**
-   * Determine if directory is empty.
-   * Call with lock held.
-   * @param p a Path, already filtered through standardize()
-   * @return TRUE / FALSE if known empty / not-empty, UNKNOWN otherwise.
-   */
-  private Tristate isEmptyDirectory(Path p) {
-    DirListingMetadata dlm = getDirListingMeta(p);
-    return dlm.withoutTombstones().isEmpty();
-  }
-
-  @Override
-  public synchronized DirListingMetadata listChildren(Path p) throws
-      IOException {
-    Path path = standardize(p);
-    DirListingMetadata listing = getDirListingMeta(path);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("listChildren({}) -> {}", path,
-          listing == null ? "null" : listing.prettyPrint());
-    }
-
-    if (listing != null) {
-      // Make a copy so callers can mutate without affecting our state
-      return new DirListingMetadata(listing);
-    }
-    return null;
-  }
-
-  @Override
-  public void move(@Nullable Collection<Path> pathsToDelete,
-      @Nullable Collection<PathMetadata> pathsToCreate,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    LOG.info("Move {} to {}", pathsToDelete, pathsToCreate);
-
-    if (pathsToCreate == null) {
-      pathsToCreate = Collections.emptyList();
-    }
-    if (pathsToDelete == null) {
-      pathsToDelete = Collections.emptyList();
-    }
-
-    // I feel dirty for using reentrant lock. :-|
-    synchronized (this) {
-
-      // 1. Delete pathsToDelete
-      for (Path meta : pathsToDelete) {
-        LOG.debug("move: deleting metadata {}", meta);
-        delete(meta, null);
-      }
-
-      // 2. Create new destination path metadata
-      for (PathMetadata meta : pathsToCreate) {
-        LOG.debug("move: adding metadata {}", meta);
-        put(meta, null);
-      }
-
-      // 3. We now know full contents of all dirs in destination subtree
-      for (PathMetadata meta : pathsToCreate) {
-        FileStatus status = meta.getFileStatus();
-        if (status == null || status.isDirectory()) {
-          continue;
-        }
-        DirListingMetadata dir = listChildren(status.getPath());
-        if (dir != null) {  // could be evicted already
-          dir.setAuthoritative(true);
-        }
-      }
-    }
-  }
-
-  @Override
-  public void put(final PathMetadata meta) throws IOException {
-    put(meta, null);
-  }
-
-  @Override
-  public void put(PathMetadata meta,
-      final BulkOperationState operationState) throws IOException {
-
-    Preconditions.checkNotNull(meta);
-    S3AFileStatus status = meta.getFileStatus();
-    Path path = standardize(status.getPath());
-    synchronized (this) {
-
-      /* Add entry for this file. */
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("put {} -> {}", path, meta.prettyPrint());
-      }
-      LocalMetadataEntry entry = localCache.getIfPresent(path);
-      if(entry == null){
-        entry = new LocalMetadataEntry(meta);
-      } else {
-        entry.setPathMetadata(meta);
-      }
-
-      /* Directory case:
-       * We also make sure we have an entry in the dirCache, so subsequent
-       * listStatus(path) at least see the directory.
-       *
-       * If we had a boolean flag argument "isNew", we would know whether this
-       * is an existing directory the client discovered via getFileStatus(),
-       * or if it is a newly-created directory.  In the latter case, we would
-       * be able to mark the directory as authoritative (fully-cached),
-       * saving round trips to underlying store for subsequent listStatus()
-       */
-
-      // only create DirListingMetadata if the entry does not have one
-      if (status.isDirectory() && !entry.hasDirMeta()) {
-        DirListingMetadata dlm =
-            new DirListingMetadata(path, DirListingMetadata.EMPTY_DIR, false);
-        entry.setDirListingMetadata(dlm);
-      }
-      localCache.put(path, entry);
-
-      /* Update cached parent dir. */
-      Path parentPath = path.getParent();
-      if (parentPath != null) {
-        LocalMetadataEntry parentMeta = localCache.getIfPresent(parentPath);
-
-        // Create empty parent LocalMetadataEntry if it doesn't exist
-        if (parentMeta == null){
-          parentMeta = new LocalMetadataEntry();
-          localCache.put(parentPath, parentMeta);
-        }
-
-        // If there is no directory metadata on the parent entry, create
-        // an empty one
-        if (!parentMeta.hasDirMeta()) {
-          DirListingMetadata parentDirMeta =
-              new DirListingMetadata(parentPath, DirListingMetadata.EMPTY_DIR,
-                  false);
-          parentDirMeta.setLastUpdated(meta.getLastUpdated());
-          parentMeta.setDirListingMetadata(parentDirMeta);
-        }
-
-        // Add the child pathMetadata to the listing
-        parentMeta.getDirListingMeta().put(meta);
-
-        // Mark the listing entry as deleted if the meta is set to deleted
-        if(meta.isDeleted()) {
-          parentMeta.getDirListingMeta().markDeleted(path,
-              ttlTimeProvider.getNow());
-        }
-      }
-    }
-  }
-
-  @Override
-  public synchronized void put(DirListingMetadata meta,
-      final List<Path> unchangedEntries,
-      final BulkOperationState operationState) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("put dirMeta {}", meta.prettyPrint());
-    }
-    LocalMetadataEntry entry =
-        localCache.getIfPresent(standardize(meta.getPath()));
-    if (entry == null) {
-      localCache.put(standardize(meta.getPath()), new LocalMetadataEntry(meta));
-    } else {
-      entry.setDirListingMetadata(meta);
-    }
-    put(meta.getListing(), null);
-  }
-
-  public synchronized void put(Collection<? extends PathMetadata> metas,
-      final BulkOperationState operationState) throws
-      IOException {
-    for (PathMetadata meta : metas) {
-      put(meta, operationState);
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-
-  }
-
-  @Override
-  public void destroy() throws IOException {
-    if (localCache != null) {
-      localCache.invalidateAll();
-    }
-  }
-
-  @Override
-  public void prune(PruneMode pruneMode, long cutoff) throws IOException{
-    prune(pruneMode, cutoff, "");
-  }
-
-  @Override
-  public synchronized long prune(PruneMode pruneMode, long cutoff,
-      String keyPrefix) {
-    // prune files
-    AtomicLong count = new AtomicLong();
-    // filter path_metadata (files), filter expired, remove expired
-    localCache.asMap().entrySet().stream()
-        .filter(entry -> entry.getValue().hasPathMeta())
-        .filter(entry -> expired(pruneMode,
-            entry.getValue().getFileMeta(), cutoff, keyPrefix))
-        .forEach(entry -> {
-          localCache.invalidate(entry.getKey());
-          count.incrementAndGet();
-        });
-
-
-    // prune dirs
-    // filter DIR_LISTING_METADATA, remove expired, remove authoritative bit
-    localCache.asMap().entrySet().stream()
-        .filter(entry -> entry.getValue().hasDirMeta())
-        .forEach(entry -> {
-          Path path = entry.getKey();
-          DirListingMetadata metadata = entry.getValue().getDirListingMeta();
-          Collection<PathMetadata> oldChildren = metadata.getListing();
-          Collection<PathMetadata> newChildren = new LinkedList<>();
-
-          for (PathMetadata child : oldChildren) {
-            if (!expired(pruneMode, child, cutoff, keyPrefix)) {
-              newChildren.add(child);
-            } else {
-              count.incrementAndGet();
-            }
-          }
-          removeAuthoritativeFromParent(path, oldChildren, newChildren);
-        });
-    return count.get();
-  }
-
-  private void removeAuthoritativeFromParent(Path path,
-      Collection<PathMetadata> oldChildren,
-      Collection<PathMetadata> newChildren) {
-    if (newChildren.size() != oldChildren.size()) {
-      DirListingMetadata dlm =
-          new DirListingMetadata(path, newChildren, false);
-      localCache.put(path, new LocalMetadataEntry(dlm));
-      if (!path.isRoot()) {
-        DirListingMetadata parent = getDirListingMeta(path.getParent());
-        if (parent != null) {
-          parent.setAuthoritative(false);
-        }
-      }
-    }
-  }
-
-  private boolean expired(PruneMode pruneMode, PathMetadata metadata,
-      long cutoff, String keyPrefix) {
-    final S3AFileStatus status = metadata.getFileStatus();
-    final URI statusUri = status.getPath().toUri();
-
-    // remove the protocol from path string to be able to compare
-    String bucket = statusUri.getHost();
-    String statusTranslatedPath = "";
-    if(bucket != null && !bucket.isEmpty()){
-      // if there's a bucket, (well defined host in Uri) the pathToParentKey
-      // can be used to get the path from the status
-      statusTranslatedPath =
-          PathMetadataDynamoDBTranslation.pathToParentKey(status.getPath());
-    } else {
-      // if there's no bucket in the path the pathToParentKey will fail, so
-      // this is the fallback to get the path from status
-      statusTranslatedPath = statusUri.getPath();
-    }
-
-    boolean expired;
-    switch (pruneMode) {
-    case ALL_BY_MODTIME:
-      // Note: S3 doesn't track modification time on directories, so for
-      // consistency with the DynamoDB implementation we ignore that here
-      expired = status.getModificationTime() < cutoff && !status.isDirectory()
-          && statusTranslatedPath.startsWith(keyPrefix);
-      break;
-    case TOMBSTONES_BY_LASTUPDATED:
-      expired = metadata.getLastUpdated() < cutoff && metadata.isDeleted()
-          && statusTranslatedPath.startsWith(keyPrefix);
-      break;
-    default:
-      throw new UnsupportedOperationException("Unsupported prune mode: "
-          + pruneMode);
-    }
-
-    return expired;
-  }
-
-  @VisibleForTesting
-  static void deleteEntryByAncestor(Path ancestor,
-      Cache<Path, LocalMetadataEntry> cache, boolean tombstone,
-      ITtlTimeProvider ttlTimeProvider) {
-
-    cache.asMap().entrySet().stream()
-        .filter(entry -> isAncestorOf(ancestor, entry.getKey()))
-        .forEach(entry -> {
-          LocalMetadataEntry meta = entry.getValue();
-          Path path = entry.getKey();
-          if(meta.hasDirMeta()){
-            cache.invalidate(path);
-          } else if(tombstone && meta.hasPathMeta()){
-            final PathMetadata pmTombstone = PathMetadata.tombstone(path,
-                ttlTimeProvider.getNow());
-            meta.setPathMetadata(pmTombstone);
-          } else {
-            cache.invalidate(path);
-          }
-        });
-  }
-
-  /**
-   * @return true if 'ancestor' is ancestor dir in path 'f'.
-   * All paths here are absolute.  Dir does not count as its own ancestor.
-   */
-  private static boolean isAncestorOf(Path ancestor, Path f) {
-    String aStr = ancestor.toString();
-    if (!ancestor.isRoot()) {
-      aStr += "/";
-    }
-    String fStr = f.toString();
-    return (fStr.startsWith(aStr));
-  }
-
-  /**
-   * Update fileCache and dirCache to reflect deletion of file 'f'.  Call with
-   * lock held.
-   */
-  private void deleteCacheEntries(Path path, boolean tombstone) {
-    LocalMetadataEntry entry = localCache.getIfPresent(path);
-    // If there's no entry, delete should silently succeed
-    // (based on MetadataStoreTestBase#testDeleteNonExisting)
-    if(entry == null){
-      LOG.warn("Delete: path {} is missing from cache.", path);
-      return;
-    }
-
-    // Remove target file entry
-    LOG.debug("delete file entry for {}", path);
-    if(entry.hasPathMeta()){
-      if (tombstone) {
-        PathMetadata pmd = PathMetadata.tombstone(path,
-            ttlTimeProvider.getNow());
-        entry.setPathMetadata(pmd);
-      } else {
-        entry.setPathMetadata(null);
-      }
-    }
-
-    // If this path is a dir, remove its listing
-    if(entry.hasDirMeta()) {
-      LOG.debug("removing listing of {}", path);
-      entry.setDirListingMetadata(null);
-    }
-
-    // If the entry is empty (contains no dirMeta or pathMeta) remove it from
-    // the cache.
-    if(!entry.hasDirMeta() && !entry.hasPathMeta()){
-      localCache.invalidate(entry);
-    }
-
-    /* Remove this path from parent's dir listing */
-    Path parent = path.getParent();
-    if (parent != null) {
-      DirListingMetadata dir = getDirListingMeta(parent);
-      if (dir != null) {
-        LOG.debug("removing parent's entry for {} ", path);
-        if (tombstone) {
-          dir.markDeleted(path, ttlTimeProvider.getNow());
-        } else {
-          dir.remove(path);
-        }
-      }
-    }
-  }
-
-  /**
-   * Return a "standardized" version of a path so we always have a consistent
-   * hash value.  Also asserts the path is absolute, and contains host
-   * component.
-   * @param p input Path
-   * @return standardized version of Path, suitable for hash key
-   */
-  private Path standardize(Path p) {
-    Preconditions.checkArgument(p.isAbsolute(), "Path must be absolute");
-    URI uri = p.toUri();
-    if (uriHost != null) {
-      Preconditions.checkArgument(StringUtils.isNotEmpty(uri.getHost()));
-    }
-    return p;
-  }
-
-  @Override
-  public Map<String, String> getDiagnostics() throws IOException {
-    Map<String, String> map = new HashMap<>();
-    map.put("name", "local://metadata");
-    map.put("uriHost", uriHost);
-    map.put("description", "Local in-VM metadata store for testing");
-    map.put(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT,
-        Boolean.toString(true));
-    return map;
-  }
-
-  @Override
-  public void updateParameters(Map<String, String> parameters)
-      throws IOException {
-  }
-
-  PathMetadata getFileMeta(Path p){
-    LocalMetadataEntry entry = localCache.getIfPresent(p);
-    if(entry != null && entry.hasPathMeta()){
-      return entry.getFileMeta();
-    } else {
-      return null;
-    }
-  }
-
-  DirListingMetadata getDirListingMeta(Path p){
-    LocalMetadataEntry entry = localCache.getIfPresent(p);
-    if(entry != null && entry.hasDirMeta()){
-      return entry.getDirListingMeta();
-    } else {
-      return null;
-    }
-  }
-
-  @Override
-  public RenameTracker initiateRenameOperation(final StoreContext storeContext,
-      final Path source,
-      final S3AFileStatus sourceStatus, final Path dest) throws IOException {
-    return new ProgressiveRenameTracker(storeContext, this, source, dest,
-        null);
-  }
-
-  @Override
-  public synchronized void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider) {
-    this.ttlTimeProvider = ttlTimeProvider;
-  }
-
-  @Override
-  public synchronized void addAncestors(final Path qualifiedPath,
-      @Nullable final BulkOperationState operationState) throws IOException {
-
-    Collection<PathMetadata> newDirs = new ArrayList<>();
-    Path parent = qualifiedPath.getParent();
-    while (!parent.isRoot()) {
-      PathMetadata directory = get(parent);
-      if (directory == null || directory.isDeleted()) {
-        S3AFileStatus status = new S3AFileStatus(Tristate.FALSE, parent,
-            username);
-        PathMetadata meta = new PathMetadata(status, Tristate.FALSE, false,
-            ttlTimeProvider.getNow());
-        newDirs.add(meta);
-      } else {
-        break;
-      }
-      parent = parent.getParent();
-    }
-    if (!newDirs.isEmpty()) {
-      put(newDirs, operationState);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
deleted file mode 100644
index 33d98e92148cc..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStore.java
+++ /dev/null
@@ -1,438 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.Retries.RetryTranslated;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-
-/**
- * {@code MetadataStore} defines the set of operations that any metadata store
- * implementation must provide.  Note that all {@link Path} objects provided
- * to methods must be absolute, not relative paths.
- * Implementations must implement any retries needed internally, such that
- * transient errors are generally recovered from without throwing exceptions
- * from this API.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface MetadataStore extends Closeable {
-
-  /**
-   * Performs one-time initialization of the metadata store.
-   *
-   * @param fs {@code FileSystem} associated with the MetadataStore
-   * @param ttlTimeProvider the time provider to use for metadata expiry
-   * @throws IOException if there is an error
-   */
-  void initialize(FileSystem fs, ITtlTimeProvider ttlTimeProvider)
-      throws IOException;
-
-  /**
-   * Performs one-time initialization of the metadata store via configuration.
-   * @see #initialize(FileSystem, ITtlTimeProvider)
-   * @param conf Configuration.
-   * @param ttlTimeProvider the time provider to use for metadata expiry
-   * @throws IOException if there is an error
-   */
-  void initialize(Configuration conf,
-      ITtlTimeProvider ttlTimeProvider) throws IOException;
-
-  /**
-   * Deletes exactly one path, leaving a tombstone to prevent lingering,
-   * inconsistent copies of it from being listed.
-   *
-   * Deleting an entry with a tombstone needs a
-   * {@link org.apache.hadoop.fs.s3a.s3guard.S3Guard.TtlTimeProvider} because
-   * the lastUpdated field of the record has to be updated to <pre>now</pre>.
-   *
-   * @param path the path to delete
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException if there is an error
-   */
-  void delete(Path path,
-      @Nullable BulkOperationState operationState)
-      throws IOException;
-
-  /**
-   * Removes the record of exactly one path.  Does not leave a tombstone (see
-   * {@link MetadataStore#delete(Path, BulkOperationState)}. It is currently
-   * intended for testing only, and a need to use it as part of normal
-   * FileSystem usage is not anticipated.
-   *
-   * @param path the path to delete
-   * @throws IOException if there is an error
-   */
-  @VisibleForTesting
-  void forgetMetadata(Path path) throws IOException;
-
-  /**
-   * Deletes the entire sub-tree rooted at the given path, leaving tombstones
-   * to prevent lingering, inconsistent copies of it from being listed.
-   *
-   * In addition to affecting future calls to {@link #get(Path)},
-   * implementations must also update any stored {@code DirListingMetadata}
-   * objects which track the parent of this file.
-   *
-   * Deleting a subtree with a tombstone needs a
-   * {@link org.apache.hadoop.fs.s3a.s3guard.S3Guard.TtlTimeProvider} because
-   * the lastUpdated field of all records have to be updated to <pre>now</pre>.
-   *
-   * @param path the root of the sub-tree to delete
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException if there is an error
-   */
-  @Retries.RetryTranslated
-  void deleteSubtree(Path path,
-      @Nullable BulkOperationState operationState)
-      throws IOException;
-
-  /**
-   * Delete the paths.
-   * There's no attempt to order the paths: they are
-   * deleted in the order passed in.
-   * @param paths paths to delete.
-   * @param operationState Nullable operation state
-   * @throws IOException failure
-   */
-
-  @RetryTranslated
-  void deletePaths(Collection<Path> paths,
-      @Nullable BulkOperationState operationState)
-      throws IOException;
-
-  /**
-   * Gets metadata for a path.
-   *
-   * @param path the path to get
-   * @return metadata for {@code path}, {@code null} if not found
-   * @throws IOException if there is an error
-   */
-  PathMetadata get(Path path) throws IOException;
-
-  /**
-   * Gets metadata for a path.  Alternate method that includes a hint
-   * whether or not the MetadataStore should do work to compute the value for
-   * {@link PathMetadata#isEmptyDirectory()}.  Since determining emptiness
-   * may be an expensive operation, this can save wasted work.
-   *
-   * @param path the path to get
-   * @param wantEmptyDirectoryFlag Set to true to give a hint to the
-   *   MetadataStore that it should try to compute the empty directory flag.
-   * @return metadata for {@code path}, {@code null} if not found
-   * @throws IOException if there is an error
-   */
-  PathMetadata get(Path path, boolean wantEmptyDirectoryFlag)
-      throws IOException;
-
-  /**
-   * Lists metadata for all direct children of a path.
-   *
-   * @param path the path to list
-   * @return metadata for all direct children of {@code path} which are being
-   *     tracked by the MetadataStore, or {@code null} if the path was not found
-   *     in the MetadataStore.
-   * @throws IOException if there is an error
-   */
-  @Retries.RetryTranslated
-  DirListingMetadata listChildren(Path path) throws IOException;
-
-  /**
-   * This adds all new ancestors of a path as directories.
-   * <p>
-   * Important: to propagate TTL information, any new ancestors added
-   * must have their last updated timestamps set through
-   * {@link S3Guard#patchLastUpdated(Collection, ITtlTimeProvider)}.
-   * @param qualifiedPath path to update
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException failure
-   */
-  @RetryTranslated
-  void addAncestors(Path qualifiedPath,
-      @Nullable BulkOperationState operationState) throws IOException;
-
-  /**
-   * Record the effects of a {@link FileSystem#rename(Path, Path)} in the
-   * MetadataStore.  Clients provide explicit enumeration of the affected
-   * paths (recursively), before and after the rename.
-   *
-   * This operation is not atomic, unless specific implementations claim
-   * otherwise.
-   *
-   * On the need to provide an enumeration of directory trees instead of just
-   * source and destination paths:
-   * Since a MetadataStore does not have to track all metadata for the
-   * underlying storage system, and a new MetadataStore may be created on an
-   * existing underlying filesystem, this move() may be the first time the
-   * MetadataStore sees the affected paths.  Therefore, simply providing src
-   * and destination paths may not be enough to record the deletions (under
-   * src path) and creations (at destination) that are happening during the
-   * rename().
-   *
-   * @param pathsToDelete Collection of all paths that were removed from the
-   *                      source directory tree of the move.
-   * @param pathsToCreate Collection of all PathMetadata for the new paths
-   *                      that were created at the destination of the rename().
-   * @param operationState     Any ongoing state supplied to the rename tracker
-   *                      which is to be passed in with each move operation.
-   * @throws IOException if there is an error
-   */
-  void move(@Nullable Collection<Path> pathsToDelete,
-      @Nullable Collection<PathMetadata> pathsToCreate,
-      @Nullable BulkOperationState operationState) throws IOException;
-
-  /**
-   * Saves metadata for exactly one path.
-   *
-   * Implementations may pre-create all the path's ancestors automatically.
-   * Implementations must update any {@code DirListingMetadata} objects which
-   * track the immediate parent of this file.
-   *
-   * @param meta the metadata to save
-   * @throws IOException if there is an error
-   */
-  @RetryTranslated
-  void put(PathMetadata meta) throws IOException;
-
-  /**
-   * Saves metadata for exactly one path, potentially
-   * using any bulk operation state to eliminate duplicate work.
-   *
-   * Implementations may pre-create all the path's ancestors automatically.
-   * Implementations must update any {@code DirListingMetadata} objects which
-   * track the immediate parent of this file.
-   *
-   * @param meta the metadata to save
-   * @param operationState operational state for a bulk update
-   * @throws IOException if there is an error
-   */
-  @RetryTranslated
-  void put(PathMetadata meta,
-      @Nullable BulkOperationState operationState) throws IOException;
-
-  /**
-   * Saves metadata for any number of paths.
-   *
-   * Semantics are otherwise the same as single-path puts.
-   *
-   * @param metas the metadata to save
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException if there is an error
-   */
-  void put(Collection<? extends PathMetadata> metas,
-      @Nullable BulkOperationState operationState) throws IOException;
-
-  /**
-   * Save directory listing metadata. Callers may save a partial directory
-   * listing for a given path, or may store a complete and authoritative copy
-   * of the directory listing.  {@code MetadataStore} implementations may
-   * subsequently keep track of all modifications to the directory contents at
-   * this path, and return authoritative results from subsequent calls to
-   * {@link #listChildren(Path)}. See {@link DirListingMetadata}.
-   *
-   * Any authoritative results returned are only authoritative for the scope
-   * of the {@code MetadataStore}:  A per-process {@code MetadataStore}, for
-   * example, would only show results visible to that process, potentially
-   * missing metadata updates (create, delete) made to the same path by
-   * another process.
-   *
-   * To optimize updates and avoid overwriting existing entries which
-   * may contain extra data, entries in the list of unchangedEntries may
-   * be excluded. That is: the listing metadata has the full list of
-   * what it believes are children, but implementations can opt to ignore
-   * some.
-   * @param meta Directory listing metadata.
-   * @param unchangedEntries list of entries in the dir listing which have
-   * not changed since the directory was list scanned on s3guard.
-   * @param operationState operational state for a bulk update
-   * @throws IOException if there is an error
-   */
-  void put(DirListingMetadata meta,
-      final List<Path> unchangedEntries,
-      @Nullable BulkOperationState operationState) throws IOException;
-
-  /**
-   * Destroy all resources associated with the metadata store.
-   *
-   * The destroyed resources can be DynamoDB tables, MySQL databases/tables, or
-   * HDFS directories. Any operations after calling this method may possibly
-   * fail.
-   *
-   * This operation is idempotent.
-   *
-   * @throws IOException if there is an error
-   */
-  void destroy() throws IOException;
-
-  /**
-   * Prune method with two modes of operation:
-   * <ul>
-   *   <li>
-   *    {@link PruneMode#ALL_BY_MODTIME}
-   *    Clear any metadata older than a specified mod_time from the store.
-   *    Note that this modification time is the S3 modification time from the
-   *    object's metadata - from the object store.
-   *    Implementations MUST clear file metadata, and MAY clear directory
-   *    metadata (s3a itself does not track modification time for directories).
-   *    Implementations may also choose to throw UnsupportedOperationException
-   *    instead. Note that modification times must be in UTC, as returned by
-   *    System.currentTimeMillis at the time of modification.
-   *   </li>
-   * </ul>
-   *
-   * <ul>
-   *   <li>
-   *    {@link PruneMode#TOMBSTONES_BY_LASTUPDATED}
-   *    Clear any tombstone updated earlier than a specified time from the
-   *    store. Note that this last_updated is the time when the metadata
-   *    entry was last updated and maintained by the metadata store.
-   *    Implementations MUST clear file metadata, and MAY clear directory
-   *    metadata (s3a itself does not track modification time for directories).
-   *    Implementations may also choose to throw UnsupportedOperationException
-   *    instead. Note that last_updated must be in UTC, as returned by
-   *    System.currentTimeMillis at the time of modification.
-   *   </li>
-   * </ul>
-   *
-   * @param pruneMode Prune Mode
-   * @param cutoff Oldest time to allow (UTC)
-   * @throws IOException if there is an error
-   * @throws UnsupportedOperationException if not implemented
-   */
-  void prune(PruneMode pruneMode, long cutoff) throws IOException,
-      UnsupportedOperationException;
-
-  /**
-   * Same as {@link MetadataStore#prune(PruneMode, long)}, but with an
-   * additional keyPrefix parameter to filter the pruned keys with a prefix.
-   *
-   * @param pruneMode Prune Mode
-   * @param cutoff Oldest time in milliseconds to allow (UTC)
-   * @param keyPrefix The prefix for the keys that should be removed
-   * @throws IOException if there is an error
-   * @throws UnsupportedOperationException if not implemented
-   * @return the number of pruned entries
-   */
-  long prune(PruneMode pruneMode, long cutoff, String keyPrefix)
-      throws IOException, UnsupportedOperationException;
-
-  /**
-   * Get any diagnostics information from a store, as a list of (key, value)
-   * tuples for display. Arbitrary values; no guarantee of stability.
-   * These are for debugging and testing only.
-   * @return a map of strings.
-   * @throws IOException if there is an error
-   */
-  Map<String, String> getDiagnostics() throws IOException;
-
-  /**
-   * Tune/update parameters for an existing table.
-   * @param parameters map of params to change.
-   * @throws IOException if there is an error
-   */
-  void updateParameters(Map<String, String> parameters) throws IOException;
-
-  /**
-   * Mark all directories created/touched in an operation as authoritative.
-   * The metastore can now update that path with any authoritative
-   * flags it chooses.
-   * The store may assume that therefore the operation state is complete.
-   * This holds for rename and needs to be documented for import.
-   * @param dest destination path.
-   * @param operationState active state.
-   * @throws IOException failure.
-   * @return the number of directories marked.
-   */
-  default int markAsAuthoritative(Path dest,
-      BulkOperationState operationState)
-      throws IOException {
-    return 0;
-  }
-
-  /**
-   * Modes of operation for prune.
-   * For details see {@link MetadataStore#prune(PruneMode, long)}
-   */
-  enum PruneMode {
-    ALL_BY_MODTIME,
-    TOMBSTONES_BY_LASTUPDATED
-  }
-
-  /**
-   * Start a rename operation.
-   *
-   * @param storeContext store context.
-   * @param source source path
-   * @param sourceStatus status of the source file/dir
-   * @param dest destination path.
-   * @return the rename tracker
-   * @throws IOException Failure.
-   */
-  RenameTracker initiateRenameOperation(
-      StoreContext storeContext,
-      Path source,
-      S3AFileStatus sourceStatus,
-      Path dest)
-      throws IOException;
-
-  /**
-   * Initiate a bulk update and create an operation state for it.
-   * This may then be passed into put operations.
-   * @param operation the type of the operation.
-   * @param dest path under which updates will be explicitly put.
-   * @return null or a store-specific state to pass into the put operations.
-   * @throws IOException failure
-   */
-  default BulkOperationState initiateBulkWrite(
-      BulkOperationState.OperationType operation,
-      Path dest) throws IOException {
-    return new BulkOperationState(operation);
-  }
-
-  /**
-   * The TtlTimeProvider has to be set during the initialization for the
-   * metadatastore, but this method can be used for testing, and change the
-   * instance during runtime.
-   *
-   * @param ttlTimeProvider
-   */
-  void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider);
-
-  /**
-   * Get any S3GuardInstrumentation for this store...must not be null.
-   * @return any store instrumentation.
-   */
-  default MetastoreInstrumentation getInstrumentation() {
-    return new MetastoreInstrumentationImpl();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
deleted file mode 100644
index a92aaeb05604a..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Set;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-
-/**
- * {@code MetadataStoreListFilesIterator} is a {@link RemoteIterator} that
- * is similar to {@code DescendantsIterator} but does not return directories
- * that have (or may have) children, and will also provide access to the set of
- * tombstones to allow recently deleted S3 objects to be filtered out from a
- * corresponding request.  In other words, it returns tombstones and the same
- * set of objects that should exist in S3: empty directories, and files, and not
- * other directories whose existence is inferred therefrom.
- *
- * For example, assume the consistent store contains metadata representing this
- * file system structure:
- *
- * <pre>
- * /dir1
- * |-- dir2
- * |   |-- file1
- * |   `-- file2
- * `-- dir3
- *     |-- dir4
- *     |   `-- file3
- *     |-- dir5
- *     |   `-- file4
- *     `-- dir6
- * </pre>
- *
- * Consider this code sample:
- * <pre>
- * final PathMetadata dir1 = get(new Path("/dir1"));
- * for (MetadataStoreListFilesIterator files =
- *     new MetadataStoreListFilesIterator(dir1); files.hasNext(); ) {
- *   final FileStatus status = files.next().getFileStatus();
- *   System.out.printf("%s %s%n", status.isDirectory() ? 'D' : 'F',
- *       status.getPath());
- * }
- * </pre>
- *
- * The output is:
- * <pre>
- * F /dir1/dir2/file1
- * F /dir1/dir2/file2
- * F /dir1/dir3/dir4/file3
- * F /dir1/dir3/dir5/file4
- * D /dir1/dir3/dir6
- * </pre>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class MetadataStoreListFilesIterator implements
-    RemoteIterator<S3AFileStatus> {
-  public static final Logger LOG = LoggerFactory.getLogger(
-      MetadataStoreListFilesIterator.class);
-
-  private final boolean allowAuthoritative;
-  private final MetadataStore metadataStore;
-  private final Set<Path> tombstones = new HashSet<>();
-  private final boolean recursivelyAuthoritative;
-  private Iterator<S3AFileStatus> leafNodesIterator = null;
-
-  public MetadataStoreListFilesIterator(MetadataStore ms, PathMetadata meta,
-      boolean allowAuthoritative) throws IOException {
-    Preconditions.checkNotNull(ms);
-    this.metadataStore = ms;
-    this.allowAuthoritative = allowAuthoritative;
-    this.recursivelyAuthoritative = prefetch(meta);
-  }
-
-  /**
-   * Walks the listing tree, starting from given metadata path. All
-   * encountered files and empty directories are added to
-   * {@link leafNodesIterator} unless a directory seems to be empty
-   * and at least one of the following conditions hold:
-   * <ul>
-   *   <li>
-   *     The directory listing is not marked authoritative
-   *   </li>
-   *   <li>
-   *     Authoritative mode is not allowed
-   *   </li>
-   * </ul>
-   * @param meta starting point for tree walk
-   * @return {@code true} if all encountered directory listings
-   *          are marked as authoritative
-   * @throws IOException
-   */
-  private boolean prefetch(PathMetadata meta) throws IOException {
-    final Queue<PathMetadata> queue = new LinkedList<>();
-    final Collection<S3AFileStatus> leafNodes = new ArrayList<>();
-
-    boolean allListingsAuthoritative = true;
-    if (meta != null) {
-      final Path path = meta.getFileStatus().getPath();
-      if (path.isRoot()) {
-        DirListingMetadata rootListing = metadataStore.listChildren(path);
-        if (rootListing != null) {
-          if (!rootListing.isAuthoritative()) {
-            allListingsAuthoritative = false;
-          }
-          tombstones.addAll(rootListing.listTombstones());
-          queue.addAll(rootListing.withoutTombstones().getListing());
-        }
-      } else {
-        queue.add(meta);
-      }
-    } else {
-      allListingsAuthoritative = false;
-    }
-
-    while(!queue.isEmpty()) {
-      PathMetadata nextMetadata = queue.poll();
-      S3AFileStatus nextStatus = nextMetadata.getFileStatus();
-      if (nextStatus.isFile()) {
-        // All files are leaf nodes by definition
-        leafNodes.add(nextStatus);
-        continue;
-      }
-      if (nextStatus.isDirectory()) {
-        final Path path = nextStatus.getPath();
-        DirListingMetadata children = metadataStore.listChildren(path);
-        if (children != null) {
-          if (!children.isAuthoritative()) {
-            allListingsAuthoritative = false;
-          }
-          tombstones.addAll(children.listTombstones());
-          Collection<PathMetadata> liveChildren =
-              children.withoutTombstones().getListing();
-          if (!liveChildren.isEmpty()) {
-            // If it's a directory, has children, not all deleted, then we
-            // add the children to the queue and move on to the next node
-            queue.addAll(liveChildren);
-            continue;
-          } else if (allowAuthoritative && children.isAuthoritative()) {
-            leafNodes.add(nextStatus);
-          }
-        } else {
-          // we do not have a listing, so directory definitely non-authoritative
-          allListingsAuthoritative = false;
-        }
-      }
-      // Directories that *might* be empty are ignored for now, since we
-      // cannot confirm that they are empty without incurring other costs.
-      // Users of this class can still discover empty directories via S3's
-      // fake directories, subject to the same consistency semantics as before.
-      // The only other possibility is a symlink, which is unsupported on S3A.
-    }
-    leafNodesIterator = leafNodes.iterator();
-    return allListingsAuthoritative;
-  }
-
-  @Override
-  public boolean hasNext() {
-    return leafNodesIterator.hasNext();
-  }
-
-  @Override
-  public S3AFileStatus next() {
-    return leafNodesIterator.next();
-  }
-
-  public boolean isRecursivelyAuthoritative() {
-    return recursivelyAuthoritative;
-  }
-
-  public Set<Path> listTombstones() {
-    return tombstones;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentation.java
deleted file mode 100644
index b5ac00eafe909..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentation.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-/**
- * Instrumentation exported to S3Guard.
- */
-public interface MetastoreInstrumentation {
-
-  /** Initialized event. */
-  void initialized();
-
-  /** Store has been closed. */
-  void storeClosed();
-
-  /**
-   * Throttled request.
-   */
-  void throttled();
-
-  /**
-   * S3Guard is retrying after a (retryable) failure.
-   */
-  void retrying();
-
-  /**
-   * Records have been deleted.
-   * @param count the number of records deleted.
-   */
-  void recordsDeleted(int count);
-
-  /**
-   * Records have been read.
-   * @param count the number of records read
-   */
-  void recordsRead(int count);
-
-  /**
-   * records have been written (including tombstones).
-   * @param count number of records written.
-   */
-  void recordsWritten(int count);
-
-  /**
-   * A directory has been tagged as authoritative.
-   */
-  void directoryMarkedAuthoritative();
-
-  /**
-   * An entry was added.
-   * @param durationNanos time to add
-   */
-  void entryAdded(long durationNanos);
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
deleted file mode 100644
index 722f42176ef2f..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A no-op implementation of MetadataStore.  Clients that use this
- * implementation should behave the same as they would without any
- * MetadataStore.
- */
-public class NullMetadataStore implements MetadataStore {
-
-  @Override
-  public void initialize(FileSystem fs, ITtlTimeProvider ttlTimeProvider)
-      throws IOException {
-  }
-
-  @Override
-  public void initialize(Configuration conf, ITtlTimeProvider ttlTimeProvider)
-      throws IOException {
-  }
-
-  @Override
-  public void close() throws IOException {
-  }
-
-  @Override
-  public void delete(Path path,
-      final BulkOperationState operationState)
-      throws IOException {
-  }
-
-  @Override
-  public void forgetMetadata(Path path) throws IOException {
-  }
-
-  @Override
-  public void deleteSubtree(Path path,
-      final BulkOperationState operationState)
-      throws IOException {
-  }
-
-  @Override
-  public void deletePaths(final Collection<Path> paths,
-      @Nullable final BulkOperationState operationState) throws IOException {
-
-  }
-
-  @Override
-  public PathMetadata get(Path path) throws IOException {
-    return null;
-  }
-
-  @Override
-  public PathMetadata get(Path path, boolean wantEmptyDirectoryFlag)
-      throws IOException {
-    return null;
-  }
-
-  @Override
-  public DirListingMetadata listChildren(Path path) throws IOException {
-    return null;
-  }
-
-  @Override
-  public void move(Collection<Path> pathsToDelete,
-      Collection<PathMetadata> pathsToCreate,
-      final BulkOperationState operationState) throws IOException {
-  }
-
-  @Override
-  public void put(final PathMetadata meta) throws IOException {
-  }
-
-  @Override
-  public void put(PathMetadata meta,
-      final BulkOperationState operationState) throws IOException {
-  }
-
-  @Override
-  public void put(Collection<? extends PathMetadata> meta,
-      final BulkOperationState operationState) throws IOException {
-  }
-
-  @Override
-  public void put(DirListingMetadata meta,
-      final List<Path> unchangedEntries,
-      final BulkOperationState operationState) throws IOException {
-  }
-
-  @Override
-  public void destroy() throws IOException {
-  }
-
-  @Override
-  public void prune(PruneMode pruneMode, long cutoff) {
-  }
-
-  @Override
-  public long prune(PruneMode pruneMode, long cutoff, String keyPrefix) {
-    return 0;
-  }
-
-  @Override
-  public String toString() {
-    return "NullMetadataStore";
-  }
-
-  @Override
-  public Map<String, String> getDiagnostics() throws IOException {
-    Map<String, String> map = new HashMap<>();
-    map.put("name", "Null Metadata Store");
-    map.put("description", "This is not a real metadata store");
-    return map;
-  }
-
-  @Override
-  public void updateParameters(Map<String, String> parameters)
-      throws IOException {
-  }
-
-  @Override
-  public RenameTracker initiateRenameOperation(final StoreContext storeContext,
-      final Path source,
-      final S3AFileStatus sourceStatus,
-      final Path dest)
-      throws IOException {
-    return new NullRenameTracker(storeContext, source, dest, this);
-  }
-
-  @Override
-  public void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider) {
-  }
-
-  @Override
-  public void addAncestors(final Path qualifiedPath,
-      @Nullable final BulkOperationState operationState) throws IOException {
-  }
-
-  private static final class NullRenameTracker extends RenameTracker {
-
-    private NullRenameTracker(
-        final StoreContext storeContext,
-        final Path source,
-        final Path dest,
-        MetadataStore metadataStore) {
-      super("NullRenameTracker", storeContext, metadataStore, source, dest,
-          null);
-    }
-
-    @Override
-    public void fileCopied(final Path childSource,
-        final S3ObjectAttributes sourceAttributes,
-        final S3ObjectAttributes destAttributes,
-        final Path destPath,
-        final long blockSize,
-        final boolean addAncestors) throws IOException {
-
-    }
-
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
deleted file mode 100644
index 09297056f6539..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-/**
- * {@code PathMetadata} models path metadata stored in the
- * {@link MetadataStore}. The lastUpdated field is implicitly set to 0 in the
- * constructors without that parameter to show that it will be initialized
- * with 0 if not set otherwise.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class PathMetadata extends ExpirableMetadata {
-
-  private S3AFileStatus fileStatus;
-  private Tristate isEmptyDirectory;
-  private boolean isDeleted;
-
-  /**
-   * Create a tombstone from the current time.
-   * It is mandatory to set the lastUpdated field to update when the
-   * tombstone state has changed to set when the entry got deleted.
-   *
-   * @param path path to tombstone
-   * @param lastUpdated last updated time on which expiration is based.
-   * @return the entry.
-   */
-  public static PathMetadata tombstone(Path path, long lastUpdated) {
-    S3AFileStatus s3aStatus = new S3AFileStatus(0,
-        System.currentTimeMillis(), path, 0, null,
-        null, null);
-    return new PathMetadata(s3aStatus, Tristate.UNKNOWN, true, lastUpdated);
-  }
-
-  /**
-   * Creates a new {@code PathMetadata} containing given {@code FileStatus}.
-   * lastUpdated field will be updated to 0 implicitly in this constructor.
-   *
-   * @param fileStatus file status containing an absolute path.
-   */
-  public PathMetadata(S3AFileStatus fileStatus) {
-    this(fileStatus, Tristate.UNKNOWN, false, 0);
-  }
-
-  /**
-   * Creates a new {@code PathMetadata} containing given {@code FileStatus}.
-   *
-   * @param fileStatus file status containing an absolute path.
-   * @param lastUpdated last updated time on which expiration is based.
-   */
-  public PathMetadata(S3AFileStatus fileStatus, long lastUpdated) {
-    this(fileStatus, Tristate.UNKNOWN, false, lastUpdated);
-  }
-
-  /**
-   * Creates a new {@code PathMetadata}.
-   * lastUpdated field will be updated to 0 implicitly in this constructor.
-   *
-   * @param fileStatus file status containing an absolute path.
-   * @param isEmptyDir empty directory {@link Tristate}
-   */
-  public PathMetadata(S3AFileStatus fileStatus, Tristate isEmptyDir) {
-    this(fileStatus, isEmptyDir, false, 0);
-  }
-
-  /**
-   * Creates a new {@code PathMetadata}.
-   * lastUpdated field will be updated to 0 implicitly in this constructor.
-   *
-   * @param fileStatus file status containing an absolute path.
-   * @param isEmptyDir empty directory {@link Tristate}
-   * @param isDeleted deleted / tombstoned flag
-   */
-  public PathMetadata(S3AFileStatus fileStatus, Tristate isEmptyDir,
-      boolean isDeleted) {
-    this(fileStatus, isEmptyDir, isDeleted, 0);
-  }
-
-  /**
-   * Creates a new {@code PathMetadata}.
-   *
-   * @param fileStatus file status containing an absolute path.
-   * @param isEmptyDir empty directory {@link Tristate}
-   * @param isDeleted deleted / tombstoned flag
-   * @param lastUpdated last updated time on which expiration is based.
-   */
-  public PathMetadata(S3AFileStatus fileStatus, Tristate isEmptyDir, boolean
-      isDeleted, long lastUpdated) {
-    Preconditions.checkNotNull(fileStatus, "fileStatus must be non-null");
-    Preconditions.checkNotNull(fileStatus.getPath(), "fileStatus path must be" +
-        " non-null");
-    Preconditions.checkArgument(fileStatus.getPath().isAbsolute(), "path must" +
-        " be absolute");
-    Preconditions.checkArgument(lastUpdated >=0, "lastUpdated parameter must "
-        + "be greater or equal to 0.");
-    this.fileStatus = fileStatus;
-    this.isEmptyDirectory = isEmptyDir;
-    this.isDeleted = isDeleted;
-    this.setLastUpdated(lastUpdated);
-  }
-
-  /**
-   * @return {@code FileStatus} contained in this {@code PathMetadata}.
-   */
-  public final S3AFileStatus getFileStatus() {
-    return fileStatus;
-  }
-
-  /**
-   * Query if a directory is empty.
-   * @return Tristate.TRUE if this is known to be an empty directory,
-   * Tristate.FALSE if known to not be empty, and Tristate.UNKNOWN if the
-   * MetadataStore does have enough information to determine either way.
-   */
-  public Tristate isEmptyDirectory() {
-    return isEmptyDirectory;
-  }
-
-  void setIsEmptyDirectory(Tristate isEmptyDirectory) {
-    this.isEmptyDirectory = isEmptyDirectory;
-    fileStatus.setIsEmptyDirectory(isEmptyDirectory);
-  }
-
-  public boolean isDeleted() {
-    return isDeleted;
-  }
-
-  void setIsDeleted(boolean isDeleted) {
-    this.isDeleted = isDeleted;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (!(o instanceof PathMetadata)) {
-      return false;
-    }
-    return this.fileStatus.equals(((PathMetadata)o).fileStatus);
-  }
-
-  @Override
-  public int hashCode() {
-    return fileStatus.hashCode();
-  }
-
-  @Override
-  public String toString() {
-    return "PathMetadata{" +
-        "fileStatus=" + fileStatus +
-        "; isEmptyDirectory=" + isEmptyDirectory +
-        "; isDeleted=" + isDeleted +
-        "; lastUpdated=" + super.getLastUpdated() +
-        '}';
-  }
-
-  /**
-   * Log contents to supplied StringBuilder in a pretty fashion.
-   * @param sb target StringBuilder
-   */
-  public void prettyPrint(StringBuilder sb) {
-    sb.append(String.format("%-5s %-20s %-7d %-8s %-6s %-20s %-20s",
-        fileStatus.isDirectory() ? "dir" : "file",
-        fileStatus.getPath().toString(), fileStatus.getLen(),
-        isEmptyDirectory.name(), isDeleted,
-        fileStatus.getETag(), fileStatus.getVersionId()));
-    sb.append(fileStatus);
-  }
-
-  public String prettyPrint() {
-    StringBuilder sb = new StringBuilder();
-    prettyPrint(sb);
-    return sb.toString();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java
deleted file mode 100644
index 2c82fb79b2c75..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.KeyAttribute;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
-import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
-import com.amazonaws.services.dynamodbv2.model.KeyType;
-import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-/**
- * Defines methods for translating between domain model objects and their
- * representations in the DynamoDB schema.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-@VisibleForTesting
-public final class PathMetadataDynamoDBTranslation {
-
-  /** The HASH key name of each item. */
-  @VisibleForTesting
-  static final String PARENT = "parent";
-  /** The RANGE key name of each item. */
-  @VisibleForTesting
-  static final String CHILD = "child";
-  @VisibleForTesting
-  static final String IS_DIR = "is_dir";
-  @VisibleForTesting
-  static final String MOD_TIME = "mod_time";
-  @VisibleForTesting
-  static final String FILE_LENGTH = "file_length";
-  @VisibleForTesting
-  static final String BLOCK_SIZE = "block_size";
-  static final String IS_DELETED = "is_deleted";
-  static final String IS_AUTHORITATIVE = "is_authoritative";
-  static final String LAST_UPDATED = "last_updated";
-  static final String ETAG = "etag";
-  static final String VERSION_ID = "version_id";
-
-  /** Used while testing backward compatibility. */
-  @VisibleForTesting
-  static final Set<String> IGNORED_FIELDS = new HashSet<>();
-
-  /** Table version field {@value} in version marker item. */
-  @VisibleForTesting
-  static final String TABLE_VERSION = "table_version";
-
-  /** Table creation timestampfield {@value} in version marker item. */
-  @VisibleForTesting
-  static final String TABLE_CREATED = "table_created";
-
-  /** The version marker field is invalid. */
-  static final String E_NOT_VERSION_MARKER = "Not a version marker: ";
-
-  /**
-   * Returns the key schema for the DynamoDB table.
-   *
-   * @return DynamoDB key schema
-   */
-  static Collection<KeySchemaElement> keySchema() {
-    return Arrays.asList(
-        new KeySchemaElement(PARENT, KeyType.HASH),
-        new KeySchemaElement(CHILD, KeyType.RANGE));
-  }
-
-  /**
-   * Returns the attribute definitions for the DynamoDB table.
-   *
-   * @return DynamoDB attribute definitions
-   */
-  static Collection<AttributeDefinition> attributeDefinitions() {
-    return Arrays.asList(
-        new AttributeDefinition(PARENT, ScalarAttributeType.S),
-        new AttributeDefinition(CHILD, ScalarAttributeType.S));
-  }
-
-  /**
-   * Converts a DynamoDB item to a {@link DDBPathMetadata}.
-   *
-   * @param item DynamoDB item to convert
-   * @return {@code item} converted to a {@link DDBPathMetadata}
-   */
-  static DDBPathMetadata itemToPathMetadata(Item item, String username) {
-    if (item == null) {
-      return null;
-    }
-
-    String parentStr = item.getString(PARENT);
-    Preconditions.checkNotNull(parentStr, "No parent entry in item %s", item);
-    String childStr = item.getString(CHILD);
-    Preconditions.checkNotNull(childStr, "No child entry in item %s", item);
-
-    // Skip table version markers, which are only non-absolute paths stored.
-    Path rawPath = new Path(parentStr, childStr);
-    if (!rawPath.isAbsoluteAndSchemeAuthorityNull()) {
-      return null;
-    }
-
-    Path parent = new Path(Constants.FS_S3A + ":/" + parentStr + "/");
-    Path path = new Path(parent, childStr);
-
-    boolean isDir = item.hasAttribute(IS_DIR) && item.getBoolean(IS_DIR);
-    boolean isAuthoritativeDir = false;
-    final S3AFileStatus fileStatus;
-    long lastUpdated = 0;
-    if (isDir) {
-      isAuthoritativeDir = !IGNORED_FIELDS.contains(IS_AUTHORITATIVE)
-          && item.hasAttribute(IS_AUTHORITATIVE)
-          && item.getBoolean(IS_AUTHORITATIVE);
-      fileStatus = DynamoDBMetadataStore.makeDirStatus(path, username);
-    } else {
-      long len = item.hasAttribute(FILE_LENGTH) ? item.getLong(FILE_LENGTH) : 0;
-      long modTime = item.hasAttribute(MOD_TIME) ? item.getLong(MOD_TIME) : 0;
-      long block = item.hasAttribute(BLOCK_SIZE) ? item.getLong(BLOCK_SIZE) : 0;
-      String eTag = item.getString(ETAG);
-      String versionId = item.getString(VERSION_ID);
-      fileStatus = new S3AFileStatus(
-          len, modTime, path, block, username, eTag, versionId);
-    }
-    lastUpdated =
-        !IGNORED_FIELDS.contains(LAST_UPDATED)
-            && item.hasAttribute(LAST_UPDATED)
-            ? item.getLong(LAST_UPDATED) : 0;
-
-    boolean isDeleted =
-        item.hasAttribute(IS_DELETED) && item.getBoolean(IS_DELETED);
-
-    return new DDBPathMetadata(fileStatus, Tristate.UNKNOWN, isDeleted,
-        isAuthoritativeDir, lastUpdated);
-  }
-
-  /**
-   * Converts a {@link DDBPathMetadata} to a DynamoDB item.
-   *
-   * Can ignore {@code IS_AUTHORITATIVE} flag if {@code ignoreIsAuthFlag} is
-   * true.
-   *
-   * @param meta {@link DDBPathMetadata} to convert
-   * @return {@code meta} converted to DynamoDB item
-   */
-  static Item pathMetadataToItem(DDBPathMetadata meta) {
-    Preconditions.checkNotNull(meta);
-    final S3AFileStatus status = meta.getFileStatus();
-    final Item item = new Item().withPrimaryKey(pathToKey(status.getPath()));
-    if (status.isDirectory()) {
-      item.withBoolean(IS_DIR, true);
-      if (!IGNORED_FIELDS.contains(IS_AUTHORITATIVE)) {
-        item.withBoolean(IS_AUTHORITATIVE, meta.isAuthoritativeDir());
-      }
-    } else {
-      item.withLong(FILE_LENGTH, status.getLen())
-          .withLong(MOD_TIME, status.getModificationTime())
-          .withLong(BLOCK_SIZE, status.getBlockSize());
-      if (status.getETag() != null) {
-        item.withString(ETAG, status.getETag());
-      }
-      if (status.getVersionId() != null) {
-        item.withString(VERSION_ID, status.getVersionId());
-      }
-    }
-    item.withBoolean(IS_DELETED, meta.isDeleted());
-
-    if(!IGNORED_FIELDS.contains(LAST_UPDATED)) {
-      item.withLong(LAST_UPDATED, meta.getLastUpdated());
-    }
-
-    return item;
-  }
-
-  /**
-   * The version marker has a primary key whose PARENT is {@code name};
-   * this MUST NOT be a value which represents an absolute path.
-   * @param name name of the version marker
-   * @param version version number
-   * @param timestamp creation timestamp
-   * @return an item representing a version marker.
-   */
-  static Item createVersionMarker(String name, int version, long timestamp) {
-    return new Item().withPrimaryKey(createVersionMarkerPrimaryKey(name))
-        .withInt(TABLE_VERSION, version)
-        .withLong(TABLE_CREATED, timestamp);
-  }
-
-  /**
-   * Create the primary key of the version marker.
-   * @param name key name
-   * @return the key to use when registering or resolving version markers
-   */
-  static PrimaryKey createVersionMarkerPrimaryKey(String name) {
-    return new PrimaryKey(PARENT, name, CHILD, name);
-  }
-
-  /**
-   * Extract the version from a version marker item.
-   * @param marker version marker item
-   * @return the extracted version field
-   * @throws IOException if the item is not a version marker
-   */
-  static int extractVersionFromMarker(Item marker) throws IOException {
-    if (marker.hasAttribute(TABLE_VERSION)) {
-      return marker.getInt(TABLE_VERSION);
-    } else {
-      throw new IOException(E_NOT_VERSION_MARKER + marker);
-    }
-  }
-
-  /**
-   * Extract the creation time, if present.
-   * @param marker version marker item
-   * @return the creation time, or null
-   * @throws IOException if the item is not a version marker
-   */
-  static Long extractCreationTimeFromMarker(Item marker) {
-    if (marker.hasAttribute(TABLE_CREATED)) {
-      return marker.getLong(TABLE_CREATED);
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Converts a collection {@link DDBPathMetadata} to a collection DynamoDB
-   * items.
-   *
-   * @see #pathMetadataToItem(DDBPathMetadata)
-   */
-  static Item[] pathMetadataToItem(Collection<DDBPathMetadata> metas) {
-    if (metas == null) {
-      return null;
-    }
-
-    final Item[] items = new Item[metas.size()];
-    int i = 0;
-    for (DDBPathMetadata meta : metas) {
-      items[i++] = pathMetadataToItem(meta);
-    }
-    return items;
-  }
-
-  /**
-   * Converts a {@link Path} to a DynamoDB equality condition on that path as
-   * parent, suitable for querying all direct children of the path.
-   *
-   * @param path the path; can not be null
-   * @return DynamoDB equality condition on {@code path} as parent
-   */
-  static KeyAttribute pathToParentKeyAttribute(Path path) {
-    return new KeyAttribute(PARENT, pathToParentKey(path));
-  }
-
-  /**
-   * e.g. {@code pathToParentKey(s3a://bucket/path/a) -> /bucket/path/a}
-   * @param path path to convert
-   * @return string for parent key
-   */
-  @VisibleForTesting
-  public static String pathToParentKey(Path path) {
-    Preconditions.checkNotNull(path);
-    Preconditions.checkArgument(path.isUriPathAbsolute(),
-        "Path not absolute: '%s'", path);
-    URI uri = path.toUri();
-    String bucket = uri.getHost();
-    Preconditions.checkArgument(!StringUtils.isEmpty(bucket),
-        "Path missing bucket %s", path);
-    String pKey = "/" + bucket + uri.getPath();
-
-    // Strip trailing slash
-    if (pKey.endsWith("/")) {
-      pKey = pKey.substring(0, pKey.length() - 1);
-    }
-    return pKey;
-  }
-
-  /**
-   * Converts a {@link Path} to a DynamoDB key, suitable for getting the item
-   * matching the path.
-   *
-   * @param path the path; can not be null
-   * @return DynamoDB key for item matching {@code path}
-   */
-  static PrimaryKey pathToKey(Path path) {
-    Preconditions.checkArgument(!path.isRoot(),
-        "Root path is not mapped to any PrimaryKey");
-    String childName = path.getName();
-    PrimaryKey key = new PrimaryKey(PARENT,
-        pathToParentKey(path.getParent()), CHILD,
-        childName);
-    for (KeyAttribute attr : key.getComponents()) {
-      String name = attr.getName();
-      Object v = attr.getValue();
-      Preconditions.checkNotNull(v,
-          "Null value for DynamoDB attribute \"%s\"", name);
-      Preconditions.checkState(!((String)v).isEmpty(),
-          "Empty string value for DynamoDB attribute \"%s\"", name);
-    }
-    return key;
-
-  }
-
-  /**
-   * Converts a collection of {@link Path} to a collection of DynamoDB keys.
-   *
-   * @see #pathToKey(Path)
-   */
-  static PrimaryKey[] pathToKey(Collection<Path> paths) {
-    if (paths == null) {
-      return null;
-    }
-
-    final PrimaryKey[] keys = new PrimaryKey[paths.size()];
-    int i = 0;
-    for (Path p : paths) {
-      keys[i++] = pathToKey(p);
-    }
-    return keys;
-  }
-
-  /**
-   * There is no need to instantiate this class.
-   */
-  private PathMetadataDynamoDBTranslation() {
-  }
-
-  /**
-   * Convert a collection of metadata entries to a list
-   * of DDBPathMetadata entries.
-   * If the sources are already DDBPathMetadata instances, they
-   * are copied directly into the new list, otherwise new
-   * instances are created.
-   * @param pathMetadatas source data
-   * @return the converted list.
-   */
-  static List<DDBPathMetadata> pathMetaToDDBPathMeta(
-      Collection<? extends PathMetadata> pathMetadatas) {
-    return pathMetadatas.stream().map(p ->
-        (p instanceof DDBPathMetadata)
-            ? (DDBPathMetadata) p
-            : new DDBPathMetadata(p))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Convert an item's (parent, child) key to a string value
-   * for logging. There is no validation of the item.
-   * @param item item.
-   * @return an s3a:// prefixed string.
-   */
-  static String itemPrimaryKeyToString(Item item) {
-    String parent = item.getString(PARENT);
-    String child = item.getString(CHILD);
-    return "s3a://" + parent + "/" + child;
-  }
-  /**
-   * Convert an item's (parent, child) key to a string value
-   * for logging. There is no validation of the item.
-   * @param item item.
-   * @return an s3a:// prefixed string.
-   */
-  static String primaryKeyToString(PrimaryKey item) {
-    Collection<KeyAttribute> c = item.getComponents();
-    String parent = "";
-    String child = "";
-    for (KeyAttribute attr : c) {
-      switch (attr.getName()) {
-      case PARENT:
-        parent = attr.getValue().toString();
-        break;
-      case CHILD:
-        child = attr.getValue().toString();
-        break;
-      default:
-      }
-    }
-    return "s3a://" + parent + "/" + child;
-  }
-
-  /**
-   * Create an empty dir marker which, when passed to the
-   * DDB metastore, is considered authoritative.
-   * @param status file status
-   * @return path metadata.
-   */
-  static PathMetadata authoritativeEmptyDirectoryMarker(
-      final S3AFileStatus status) {
-    return new DDBPathMetadata(status, Tristate.TRUE,
-        false, true, 0);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathOrderComparators.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathOrderComparators.java
deleted file mode 100644
index cbf41b4bab099..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathOrderComparators.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.Serializable;
-import java.util.Comparator;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Comparator of path ordering for sorting collections.
- *
- * The definition of "topmost" is:
- * <ol>
- *   <li>The depth of a path is the primary comparator.</li>
- *   <li>Root is topmost, "0"</li>
- *   <li>If two paths are of equal depth, {@link Path#compareTo(Path)}</li>
- *   is used. This delegates to URI compareTo.
- *   <li>repeated sorts do not change the order</li>
- * </ol>
- */
-final class PathOrderComparators {
-
-  private PathOrderComparators() {
-  }
-
-  /**
-   * The shallowest paths come first.
-   * This is to be used when adding entries.
-   */
-  static final Comparator<Path> TOPMOST_PATH_FIRST
-      = new TopmostFirst();
-
-  /**
-   * The leaves come first.
-   * This is to be used when deleting entries.
-   */
-  static final Comparator<Path> TOPMOST_PATH_LAST
-      = new TopmostLast();
-
-  /**
-   * The shallowest paths come first.
-   * This is to be used when adding entries.
-   */
-  static final Comparator<PathMetadata> TOPMOST_PM_FIRST
-      = new PathMetadataComparator(TOPMOST_PATH_FIRST);
-
-  /**
-   * The leaves come first.
-   * This is to be used when deleting entries.
-   */
-  static final Comparator<PathMetadata> TOPMOST_PM_LAST
-      = new PathMetadataComparator(TOPMOST_PATH_LAST);
-
-  private static class TopmostFirst implements Comparator<Path>, Serializable {
-
-    @Override
-    public int compare(Path pathL, Path pathR) {
-      // exit fast on equal values.
-      if (pathL.equals(pathR)) {
-        return 0;
-      }
-      int depthL = pathL.depth();
-      int depthR = pathR.depth();
-      if (depthL < depthR) {
-        // left is higher up than the right.
-        return -1;
-      }
-      if (depthR < depthL) {
-        // right is higher up than the left
-        return 1;
-      }
-      // and if they are of equal depth, use the "classic" comparator
-      // of paths.
-      return pathL.compareTo(pathR);
-    }
-  }
-
-  /**
-   * Compare the topmost last.
-   * For some reason the .reverse() option wasn't giving the
-   * correct outcome.
-   */
-  private static final class TopmostLast extends TopmostFirst {
-
-    @Override
-    public int compare(final Path pathL, final Path pathR) {
-      int compare = super.compare(pathL, pathR);
-      if (compare < 0) {
-        return 1;
-      }
-      if (compare > 0) {
-        return -1;
-      }
-      return 0;
-    }
-  }
-
-  /**
-   * Compare on path status.
-   */
-  static final class PathMetadataComparator implements
-      Comparator<PathMetadata>, Serializable {
-
-    private final Comparator<Path> inner;
-
-    PathMetadataComparator(final Comparator<Path> inner) {
-      this.inner = inner;
-    }
-
-    @Override
-    public int compare(final PathMetadata o1, final PathMetadata o2) {
-      return inner.compare(o1.getFileStatus().getPath(),
-          o2.getFileStatus().getPath());
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ProgressiveRenameTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ProgressiveRenameTracker.java
deleted file mode 100644
index 700b2ab71a6c8..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ProgressiveRenameTracker.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.util.DurationInfo;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.addMoveAncestors;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.addMoveDir;
-
-/**
- * This rename tracker progressively updates the metadata store
- * as it proceeds, during the parallelized copy operation.
- * <p>
- * Algorithm
- * <ol>
- *   <li>
- *     As {@code RenameTracker.fileCopied()} callbacks
- *     are raised, the metastore is updated with the new file entry.
- *   </li>
- *   <li>
- *     Including parent entries, as appropriate.
- *   </li>
- *   <li>
- *     All directories which have been created are tracked locally,
- *     to avoid needing to read the store; this is a thread-safe structure.
- *   </li>
- *   <li>
- *    The actual update is performed out of any synchronized block.
- *   </li>
- *   <li>
- *     When deletes are executed, the store is also updated.
- *   </li>
- *   <li>
- *     And at the completion of a successful rename, the source directory
- *     is also removed.
- *   </li>
- * </ol>
- * <pre>
- *
- * </pre>
- */
-public class ProgressiveRenameTracker extends RenameTracker {
-
-  /**
-   * The collection of paths to delete; this is added as individual files
-   * are renamed.
-   * <p>
-   * The metastore is only updated with these entries after the DELETE
-   * call containing these paths succeeds.
-   * <p>
-   * If the DELETE fails; the filesystem will use
-   * {@code MultiObjectDeleteSupport} to remove all successfully deleted
-   * entries from the metastore.
-   */
-  private final Collection<Path> pathsToDelete = new HashSet<>();
-
-  public ProgressiveRenameTracker(
-      final StoreContext storeContext,
-      final MetadataStore metadataStore,
-      final Path sourceRoot,
-      final Path dest,
-      final BulkOperationState operationState) {
-    super("ProgressiveRenameTracker",
-        storeContext, metadataStore, sourceRoot, dest, operationState);
-  }
-
-  /**
-   * When a file is copied, any ancestors
-   * are calculated and then the store is updated with
-   * the destination entries.
-   * <p>
-   * The source entries are added to the {@link #pathsToDelete} list.
-   * @param sourcePath path of source
-   * @param sourceAttributes status of source.
-   * @param destAttributes destination attributes
-   * @param destPath destination path.
-   * @param blockSize block size.
-   * @param addAncestors should ancestors be added?
-   * @throws IOException failure
-   */
-  @Override
-  public void fileCopied(
-      final Path sourcePath,
-      final S3ObjectAttributes sourceAttributes,
-      final S3ObjectAttributes destAttributes,
-      final Path destPath,
-      final long blockSize,
-      final boolean addAncestors) throws IOException {
-
-    // build the list of entries to add in a synchronized block.
-    final List<PathMetadata> entriesToAdd = new ArrayList<>(1);
-    LOG.debug("Updating store with copied file {}", sourcePath);
-    MetadataStore store = getMetadataStore();
-    synchronized (this) {
-      checkArgument(!pathsToDelete.contains(sourcePath),
-          "File being renamed is already processed %s", destPath);
-      // create the file metadata and update the lists
-      // the pathsToDelete field is incremented with the new source path,
-      // for deletion after the DELETE operation succeeds;
-      // the entriesToAdd variable is filled in with all entries
-      // to add within this method
-      S3Guard.addMoveFile(
-          store,
-          pathsToDelete,
-          entriesToAdd,
-          sourcePath,
-          destPath,
-          sourceAttributes.getLen(),
-          blockSize,
-          getOwner(),
-          destAttributes.getETag(),
-          destAttributes.getVersionId());
-      LOG.debug("New metastore entry : {}", entriesToAdd.get(0));
-      if (addAncestors) {
-        // add all new ancestors to the lists
-        addMoveAncestors(
-            store,
-            pathsToDelete,
-            entriesToAdd,
-            getSourceRoot(),
-            sourcePath,
-            destPath,
-            getOwner());
-      }
-    }
-
-    // outside the lock, the entriesToAdd variable has all the new entries to
-    // create. ...so update the store.
-    // no entries are deleted at this point.
-    try (DurationInfo ignored = new DurationInfo(LOG, false,
-        "Adding new metastore entries")) {
-      store.move(null, entriesToAdd, getOperationState());
-    }
-  }
-
-  /**
-   * A directory marker has been added.
-   * Add the new entry and record the source path as another entry to delete.
-   * @param sourcePath status of source.
-   * @param destPath destination path.
-   * @param addAncestors should ancestors be added?
-   * @throws IOException failure.
-   */
-  @Override
-  public void directoryMarkerCopied(
-      final Path sourcePath,
-      final Path destPath,
-      final boolean addAncestors) throws IOException {
-    // this list is created on demand.
-    final List<PathMetadata> entriesToAdd = new ArrayList<>(1);
-    MetadataStore store = getMetadataStore();
-    synchronized (this) {
-      addMoveDir(store,
-          pathsToDelete,
-          entriesToAdd,
-          sourcePath,
-          destPath,
-          getOwner());
-      // Ancestor directories may not be listed, so we explicitly add them
-      if (addAncestors) {
-        addMoveAncestors(store,
-            pathsToDelete,
-            entriesToAdd,
-            getSourceRoot(),
-            sourcePath,
-            destPath,
-            getOwner());
-      }
-    }
-    // outside the lock, the entriesToAdd list has all new files to create.
-    // ...so update the store.
-    try (DurationInfo ignored = new DurationInfo(LOG, false,
-        "adding %s metastore entries", entriesToAdd.size())) {
-      store.move(null, entriesToAdd, getOperationState());
-    }
-  }
-
-  @Override
-  public synchronized void moveSourceDirectory() throws IOException {
-    // this moves the source directory in the metastore if it has not
-    // already been processed.
-    if (!pathsToDelete.contains(getSourceRoot())) {
-      final List<Path> toDelete = new ArrayList<>(1);
-      final List<PathMetadata> toAdd = new ArrayList<>(1);
-
-      addMoveDir(getMetadataStore(), pathsToDelete, toAdd,
-          getSourceRoot(),
-          getDest(),
-          getOwner());
-      getMetadataStore().move(toDelete, toAdd, getOperationState());
-    }
-    getMetadataStore().markAsAuthoritative(
-        getDest(), getOperationState());
-  }
-
-  /**
-   * As source objects are deleted, so is the list of entries.
-   * @param paths path of objects deleted.
-   * @throws IOException failure.
-   */
-  @Override
-  public void sourceObjectsDeleted(
-      final Collection<Path> paths) throws IOException {
-
-    // delete the paths from the metastore
-    try (DurationInfo ignored = new DurationInfo(LOG, false,
-        "delete %s metastore entries", paths.size())) {
-      getMetadataStore().move(paths, null, getOperationState());
-      getMetadataStore().deletePaths(paths, getOperationState());
-    }
-  }
-
-  @Override
-  public synchronized void completeRename() throws IOException {
-    // mark dest tree as authoritative all the way down.
-    // finish off by deleting source directories.
-    sourceObjectsDeleted(pathsToDelete);
-    super.completeRename();
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PurgeS3GuardDynamoTable.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PurgeS3GuardDynamoTable.java
deleted file mode 100644
index 746d34fda5c12..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PurgeS3GuardDynamoTable.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.launcher.LauncherExitCodes;
-import org.apache.hadoop.service.launcher.ServiceLaunchException;
-import org.apache.hadoop.service.launcher.ServiceLauncher;
-import org.apache.hadoop.util.DurationInfo;
-import org.apache.hadoop.util.ExitUtil;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.s3guard.DumpS3GuardDynamoTable.serviceMain;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.PARENT;
-
-/**
- * Purge the S3Guard table of a FileSystem from all entries related to
- * that table.
- * Will fail if there is no table, or the store is in auth mode.
- * <pre>
- *   hadoop org.apache.hadoop.fs.s3a.s3guard.PurgeS3GuardDynamoTable \
- *   -force s3a://example-bucket/
- * </pre>
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class PurgeS3GuardDynamoTable
-    extends AbstractS3GuardDynamoDBDiagnostic {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PurgeS3GuardDynamoTable.class);
-
-  public static final String NAME = "PurgeS3GuardDynamoTable";
-
-  /**
-   * Name of the force option.
-   */
-  public static final String FORCE = "-force";
-
-  /**
-   * Usage message.
-   */
-  private static final String USAGE_MESSAGE = NAME
-      + " [-force] <filesystem>";
-
-  /**
-   * Flag which actually triggers the delete.
-   */
-  private boolean force;
-
-  private long filesFound;
-  private long filesDeleted;
-
-  public PurgeS3GuardDynamoTable(final String name) {
-    super(name);
-  }
-
-  public PurgeS3GuardDynamoTable() {
-    this(NAME);
-  }
-
-  public PurgeS3GuardDynamoTable(
-      final S3AFileSystem filesystem,
-      final DynamoDBMetadataStore store,
-      final URI uri,
-      final boolean force) {
-    super(NAME, filesystem, store, uri);
-    this.force = force;
-  }
-
-  /**
-   * Bind to the argument list, including validating the CLI.
-   * @throws Exception failure.
-   */
-  @Override
-  protected void serviceStart() throws Exception {
-    if (getStore() == null) {
-      List<String> arg = getArgumentList(1, 2, USAGE_MESSAGE);
-      String fsURI = arg.get(0);
-      if (arg.size() == 2) {
-        if (!arg.get(0).equals(FORCE)) {
-          throw new ServiceLaunchException(LauncherExitCodes.EXIT_USAGE,
-              USAGE_MESSAGE);
-        }
-        force = true;
-        fsURI = arg.get(1);
-      }
-      bindFromCLI(fsURI);
-    }
-  }
-
-  /**
-   * Extract the host from the FS URI, then scan and
-   * delete all entries from that bucket.
-   * @return the exit code.
-   * @throws ServiceLaunchException on failure.
-   * @throws IOException IO failure.
-   */
-  @Override
-  public int execute() throws ServiceLaunchException, IOException {
-
-    URI uri = getUri();
-    String host = uri.getHost();
-    String prefix = "/" + host + "/";
-    DynamoDBMetadataStore ddbms = getStore();
-    S3GuardTableAccess tableAccess = new S3GuardTableAccess(ddbms);
-    ExpressionSpecBuilder builder = new ExpressionSpecBuilder();
-    builder.withKeyCondition(
-        ExpressionSpecBuilder.S(PARENT).beginsWith(prefix));
-
-    LOG.info("Scanning for entries with prefix {} to delete from {}",
-        prefix, ddbms);
-
-    Iterable<DDBPathMetadata> entries =
-        ddbms.wrapWithRetries(tableAccess.scanMetadata(builder));
-    List<Path> list = new ArrayList<>();
-    entries.iterator().forEachRemaining(e -> {
-      if (!(e instanceof S3GuardTableAccess.VersionMarker)) {
-        Path p = e.getFileStatus().getPath();
-        String type = e.getFileStatus().isFile() ? "file" : "directory";
-        boolean tombstone = e.isDeleted();
-        if (tombstone) {
-          type = "tombstone " + type;
-        }
-        LOG.info("{} {}", type, p);
-        list.add(p);
-      }
-    });
-    int count = list.size();
-    filesFound = count;
-    LOG.info("Found {} entries{}",
-        count,
-        (count == 0 ? " -nothing to purge": ""));
-    if (count > 0) {
-      if (force) {
-        DurationInfo duration =
-            new DurationInfo(LOG,
-                "deleting %s entries from %s",
-                count, ddbms.toString());
-        // sending this in one by one for more efficient retries
-        for (Path path: list) {
-          ddbms.getInvoker()
-              .retry("delete",
-                  prefix,
-                  true,
-                  () -> tableAccess.delete(path));
-        }
-        duration.close();
-        long durationMillis = duration.value();
-        long timePerEntry = durationMillis / count;
-        LOG.info("Time per entry: {} ms", timePerEntry);
-        filesDeleted = count;
-      } else {
-        LOG.info("Delete process will only be executed when "
-            + FORCE + " is set");
-      }
-    }
-    return LauncherExitCodes.EXIT_SUCCESS;
-  }
-
-  /**
-   * This is the Main entry point for the service launcher.
-   *
-   * Converts the arguments to a list, instantiates a instance of the class
-   * then executes it.
-   * @param args command line arguments.
-   */
-  public static void main(String[] args) {
-    try {
-      serviceMain(Arrays.asList(args), new PurgeS3GuardDynamoTable());
-    } catch (ExitUtil.ExitException e) {
-      ExitUtil.terminate(e);
-    }
-  }
-
-  /**
-   * API Entry point to dump the metastore and S3 store world views
-   * <p>
-   * Both the FS and the store will be dumped: the store is scanned
-   * before and after the sequence to show what changes were made to
-   * the store during the list operation.
-   * @param fs fs to dump. If null a store must be provided.
-   * @param store store to dump (fallback to FS)
-   * @param conf configuration to use (fallback to fs)
-   * @param uri URI of store -only needed if FS is null.
-   * @param force force the actual delete
-   * @return (filesFound, filesDeleted)
-   * @throws ExitUtil.ExitException failure.
-   */
-  @InterfaceAudience.Private
-  @InterfaceStability.Unstable
-  public static Pair<Long, Long> purgeStore(
-      @Nullable final S3AFileSystem fs,
-      @Nullable DynamoDBMetadataStore store,
-      @Nullable Configuration conf,
-      @Nullable URI uri,
-      boolean force) throws ExitUtil.ExitException {
-    ServiceLauncher<Service> serviceLauncher =
-        new ServiceLauncher<>(NAME);
-
-    if (conf == null) {
-      conf = checkNotNull(fs, "No filesystem").getConf();
-    }
-    if (store == null) {
-      store = (DynamoDBMetadataStore) checkNotNull(fs, "No filesystem")
-          .getMetadataStore();
-    }
-    PurgeS3GuardDynamoTable purge = new PurgeS3GuardDynamoTable(fs,
-        store,
-        uri,
-        force);
-    ExitUtil.ExitException ex = serviceLauncher.launchService(
-        conf,
-        purge,
-        Collections.emptyList(),
-        false,
-        true);
-    if (ex != null && ex.getExitCode() != 0) {
-      throw ex;
-    }
-    return Pair.of(purge.filesFound, purge.filesDeleted);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RenameTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RenameTracker.java
deleted file mode 100644
index 5c112fc1047d5..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RenameTracker.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-
-import com.amazonaws.SdkBaseException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.DurationInfo;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
-
-/**
- * A class which manages updating the metastore with the rename process
- * as initiated in the S3AFilesystem rename.
- * <p>
- * Subclasses must provide an implementation and return it in
- * {@code MetadataStore.initiateRenameOperation()}.
- * <p>
- * The {@link #operationState} field/constructor argument is an opaque state to
- * be passed down to the metastore in its move operations; this allows the
- * stores to manage ongoing state -while still being able to share
- * rename tracker implementations.
- * <p>
- * This is to avoid performance problems wherein the progressive rename
- * tracker causes the store to repeatedly create and write duplicate
- * ancestor entries for every file added.
- */
-public abstract class RenameTracker extends AbstractStoreOperation {
-
-  public static final Logger LOG = LoggerFactory.getLogger(
-      RenameTracker.class);
-
-  /** source path. */
-  private final Path sourceRoot;
-
-  /** destination path. */
-  private final Path dest;
-
-  /**
-   * Track the duration of this operation.
-   */
-  private final DurationInfo durationInfo;
-
-  /**
-   * Generated name for strings.
-   */
-  private final String name;
-
-  /**
-   * Any ongoing state supplied to the rename tracker
-   * which is to be passed in with each move operation.
-   * This must be closed at the end of the tracker's life.
-   */
-  private final BulkOperationState operationState;
-
-  /**
-   * The metadata store for this tracker.
-   * Always non-null.
-   * <p>
-   * This is passed in separate from the store context to guarantee
-   * that whichever store creates a tracker is explicitly bound to that
-   * instance.
-   */
-  private final MetadataStore metadataStore;
-
-  /**
-   * Constructor.
-   * @param name tracker name for logs.
-   * @param storeContext store context.
-   * @param metadataStore the store
-   * @param sourceRoot source path.
-   * @param dest destination path.
-   * @param operationState ongoing move state.
-   */
-  protected RenameTracker(
-      final String name,
-      final StoreContext storeContext,
-      final MetadataStore metadataStore,
-      final Path sourceRoot,
-      final Path dest,
-      final BulkOperationState operationState) {
-    super(checkNotNull(storeContext));
-    checkNotNull(storeContext.getUsername(), "No username");
-    this.metadataStore = checkNotNull(metadataStore);
-    this.sourceRoot = checkNotNull(sourceRoot);
-    this.dest = checkNotNull(dest);
-    this.operationState = operationState;
-    this.name = String.format("%s (%s, %s)", name, sourceRoot, dest);
-    durationInfo = new DurationInfo(LOG, false,
-        name +" (%s, %s)", sourceRoot, dest);
-  }
-
-  @Override
-  public String toString() {
-    return name;
-  }
-
-  public Path getSourceRoot() {
-    return sourceRoot;
-  }
-
-  public Path getDest() {
-    return dest;
-  }
-
-  public String getOwner() {
-    return getStoreContext().getUsername();
-  }
-
-  public BulkOperationState getOperationState() {
-    return operationState;
-  }
-
-  /**
-   * Get the metadata store.
-   * @return a non-null store.
-   */
-  protected MetadataStore getMetadataStore() {
-    return metadataStore;
-  }
-
-  /**
-   * A file has been copied.
-   *
-   * @param childSource source of the file. This may actually be different
-   * from the path of the sourceAttributes. (HOW?)
-   * @param sourceAttributes status of source.
-   * @param destAttributes destination attributes
-   * @param destPath destination path.
-   * @param blockSize block size.
-   * @param addAncestors should ancestors be added?
-   * @throws IOException failure.
-   */
-  public abstract void fileCopied(
-      Path childSource,
-      S3ObjectAttributes sourceAttributes,
-      S3ObjectAttributes destAttributes,
-      Path destPath,
-      long blockSize,
-      boolean addAncestors) throws IOException;
-
-  /**
-   * A directory marker has been copied.
-   * @param sourcePath source path.
-   * @param destPath destination path.
-   * @param addAncestors should ancestors be added?
-   * @throws IOException failure.
-   */
-  public void directoryMarkerCopied(
-      Path sourcePath,
-      Path destPath,
-      boolean addAncestors) throws IOException {
-  }
-
-  /**
-   * The delete failed.
-   * <p>
-   * By the time this is called, the metastore will already have
-   * been updated with the results of any partial delete failure,
-   * such that all files known to have been deleted will have been
-   * removed.
-   * @param e exception
-   * @param pathsToDelete paths which were to be deleted.
-   * @param undeletedObjects list of objects which were not deleted.
-   */
-  public IOException deleteFailed(
-      final Exception e,
-      final List<Path> pathsToDelete,
-      final List<Path> undeletedObjects) {
-
-    return convertToIOException(e);
-  }
-
-  /**
-   * Top level directory move.
-   * This is invoked after all child entries have been copied
-   * @throws IOException on failure
-   */
-  public void moveSourceDirectory() throws IOException {
-  }
-
-  /**
-   * Note that source objects have been deleted.
-   * The metastore will already have been updated.
-   * @param paths path of objects deleted.
-   */
-  public void sourceObjectsDeleted(
-      final Collection<Path> paths) throws IOException {
-  }
-
-  /**
-   * Complete the operation.
-   * @throws IOException failure.
-   */
-  public void completeRename() throws IOException {
-    IOUtils.cleanupWithLogger(LOG, operationState);
-    noteRenameFinished();
-  }
-
-  /**
-   * Note that the rename has finished by closing the duration info;
-   * this will log the duration of the operation at debug.
-   */
-  protected void noteRenameFinished() {
-    durationInfo.close();
-  }
-
-  /**
-   * Rename has failed.
-   * <p>
-   * The metastore now needs to be updated with its current state
-   * even though the operation is incomplete.
-   * Implementations MUST NOT throw exceptions here, as this is going to
-   * be invoked in an exception handler.
-   * catch and log or catch and return/wrap.
-   * <p>
-   * The base implementation returns the IOE passed in and translates
-   * any AWS exception into an IOE.
-   * @param ex the exception which caused the failure.
-   * This is either an IOException or and AWS exception
-   * @return an IOException to throw in an exception.
-   */
-  public IOException renameFailed(Exception ex) {
-    LOG.debug("Rename has failed", ex);
-    IOUtils.cleanupWithLogger(LOG, operationState);
-    noteRenameFinished();
-    return convertToIOException(ex);
-  }
-
-  /**
-   * Convert a passed in exception (expected to be an IOE or AWS exception)
-   * into an IOException.
-   * @param ex exception caught
-   * @return the exception to throw in the failure handler.
-   */
-  protected IOException convertToIOException(final Exception ex) {
-    if (ex instanceof IOException) {
-      return (IOException) ex;
-    } else if (ex instanceof SdkBaseException) {
-      return translateException("rename " + sourceRoot + " to " + dest,
-          sourceRoot.toString(),
-          (SdkBaseException) ex);
-    } else {
-      // should never happen, but for completeness
-      return new IOException(ex);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RetryingCollection.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RetryingCollection.java
deleted file mode 100644
index 108d205f74b05..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/RetryingCollection.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.Retries;
-
-/**
- * A collection which wraps the result of a query or scan
- * with retries.
- * Important: iterate through this only once; the outcome
- * of repeating an iteration is "undefined"
- * @param <T> type of outcome.
- */
-class RetryingCollection<T> implements Iterable<T> {
-
-  /**
-   * Source iterable.
-   */
-  private final Iterable<T> source;
-
-  /**
-   * Invoker for retries.
-   */
-  private final Invoker invoker;
-
-  /**
-   * Operation name for invoker.retry messages.
-   */
-  private final String operation;
-
-  /**
-   * Constructor.
-   * @param operation Operation name for invoker.retry messages.
-   * @param invoker Invoker for retries.
-   * @param source Source iterable.
-   */
-  RetryingCollection(
-      final String operation,
-      final Invoker invoker,
-      final Iterable<T> source) {
-    this.operation = operation;
-    this.source = source;
-    this.invoker = invoker;
-  }
-
-  /**
-   * Demand creates a new iterator which will retry all hasNext/next
-   * operations through the invoker supplied in the constructor.
-   * @return a new iterator.
-   */
-  @Override
-  public Iterator<T> iterator() {
-    return new RetryingIterator(source.iterator());
-  }
-
-  /**
-   * An iterator which wraps a non-retrying iterator of scan results
-   * (i.e {@code S3GuardTableAccess.DDBPathMetadataIterator}.
-   */
-  private final class RetryingIterator implements Iterator<T> {
-
-    private final Iterator<T> iterator;
-
-    private RetryingIterator(final Iterator<T> iterator) {
-      this.iterator = iterator;
-    }
-
-    /**
-     * {@inheritDoc}.
-     * @throws UncheckedIOException for IO failure, including throttling.
-     */
-    @Override
-    @Retries.RetryTranslated
-    public boolean hasNext() {
-      try {
-        return invoker.retry(
-            operation,
-            null,
-            true,
-            iterator::hasNext);
-      } catch (IOException e) {
-        throw new UncheckedIOException(e);
-      }
-    }
-
-    /**
-     * {@inheritDoc}.
-     * @throws UncheckedIOException for IO failure, including throttling.
-     */
-    @Override
-    @Retries.RetryTranslated
-    public T next() {
-      try {
-        return invoker.retry(
-            "Scan Dynamo",
-            null,
-            true,
-            iterator::next);
-      } catch (IOException e) {
-        throw new UncheckedIOException(e);
-      }
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index 62d53d06f76a5..5e3e5dcab77d4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -18,1003 +18,107 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
-import javax.annotation.Nullable;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.Retries.RetryTranslated;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
-import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.S3AUtils.createUploadFileStatus;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.authoritativeEmptyDirectoryMarker;
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_BAD_CONFIGURATION;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_DYNAMO;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
+import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 
 /**
  * Logic for integrating MetadataStore with S3A.
+ * Most of the methods here were deleted when the S3Guard feature was removed.
  */
+@SuppressWarnings("deprecation")
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public final class S3Guard {
   private static final Logger LOG = LoggerFactory.getLogger(S3Guard.class);
 
-  @InterfaceAudience.Private
-  @InterfaceStability.Unstable
-  @VisibleForTesting
-  public static final String S3GUARD_DDB_CLIENT_FACTORY_IMPL =
-      "fs.s3a.s3guard.ddb.client.factory.impl";
-
-  static final Class<? extends DynamoDBClientFactory>
-      S3GUARD_DDB_CLIENT_FACTORY_IMPL_DEFAULT =
-      DynamoDBClientFactory.DefaultDynamoDBClientFactory.class;
-  private static final S3AFileStatus[] EMPTY_LISTING = new S3AFileStatus[0];
-
-  /**
-   * Hard-coded policy : {@value}.
-   * If true, when merging an S3 LIST with S3Guard in non-auth mode,
-   * only updated entries are added; new entries are left out.
-   * This policy choice reduces the amount of data stored in Dynamo,
-   * and hence the complexity of the merge in a non-auth listing.
-   */
-  @VisibleForTesting
-  public static final boolean DIR_MERGE_UPDATES_ALL_RECORDS_NONAUTH = false;
+  static final String NULL_METADATA_STORE
+      = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
 
   // Utility class.  All static functions.
-  private S3Guard() { }
-
-  /* Utility functions. */
-
-  /**
-   * Create a new instance of the configured MetadataStore.
-   * The returned MetadataStore will have been initialized via
-   * {@link MetadataStore#initialize(FileSystem, ITtlTimeProvider)}
-   * by this function before returning it.  Callers must clean up by calling
-   * {@link MetadataStore#close()} when done using the MetadataStore.
-   *
-   * @param fs  FileSystem whose Configuration specifies which
-   *            implementation to use.
-   * @param ttlTimeProvider
-   * @return Reference to new MetadataStore.
-   * @throws IOException if the metadata store cannot be instantiated
-   */
-  @Retries.OnceTranslated
-  public static MetadataStore getMetadataStore(FileSystem fs,
-      ITtlTimeProvider ttlTimeProvider)
-      throws IOException {
-    Preconditions.checkNotNull(fs);
-    Configuration conf = fs.getConf();
-    Preconditions.checkNotNull(conf);
-    MetadataStore msInstance;
-    try {
-      Class<? extends MetadataStore> msClass = getMetadataStoreClass(conf);
-      msInstance = ReflectionUtils.newInstance(msClass, conf);
-      LOG.debug("Using {} metadata store for {} filesystem",
-          msClass.getSimpleName(), fs.getScheme());
-      msInstance.initialize(fs, ttlTimeProvider);
-      return msInstance;
-    } catch (FileNotFoundException e) {
-      // Don't log this exception as it means the table doesn't exist yet;
-      // rely on callers to catch and treat specially
-      throw e;
-    } catch (RuntimeException | IOException e) {
-      String message = "Failed to instantiate metadata store " +
-          conf.get(S3_METADATA_STORE_IMPL)
-          + " defined in " + S3_METADATA_STORE_IMPL
-          + ": " + e;
-      LOG.error(message, e);
-      if (e instanceof IOException) {
-        throw e;
-      } else {
-        throw new IOException(message, e);
-      }
-    }
-  }
-
-  static Class<? extends MetadataStore> getMetadataStoreClass(
-      Configuration conf) {
-    if (conf == null) {
-      return NullMetadataStore.class;
-    }
-    if (conf.get(S3_METADATA_STORE_IMPL) != null && LOG.isDebugEnabled()) {
-      LOG.debug("Metastore option source {}",
-          (Object)conf.getPropertySources(S3_METADATA_STORE_IMPL));
-    }
-
-    Class<? extends MetadataStore> aClass = conf.getClass(
-        S3_METADATA_STORE_IMPL, NullMetadataStore.class,
-        MetadataStore.class);
-    return aClass;
-  }
-
-
-  /**
-   * We update the metastore for the specific case of S3 value == S3Guard value
-   * so as to place a more recent modtime in the store.
-   * because if not, we will continue to probe S3 whenever we look for this
-   * object, even we only do this if confident the S3 status is the same
-   * as the one in the store (i.e. it is not an older version)
-   * @param metadataStore MetadataStore to {@code put()} into.
-   * @param pm current data
-   * @param s3AFileStatus status to store
-   * @param timeProvider Time provider to use when writing entries
-   * @return true if the entry was updated.
-   * @throws IOException if metadata store update failed
-   */
-  @RetryTranslated
-  public static boolean refreshEntry(
-      MetadataStore metadataStore,
-      PathMetadata pm,
-      S3AFileStatus s3AFileStatus,
-      ITtlTimeProvider timeProvider) throws IOException {
-    // the modtime of the data is the same as/older than the s3guard value
-    // either an old object has been found, or the existing one was retrieved
-    // in both cases -return s3guard value
-    S3AFileStatus msStatus = pm.getFileStatus();
-
-    // first check: size
-    boolean sizeMatch = msStatus.getLen() == s3AFileStatus.getLen();
-
-    // etags are expected on all objects, but handle the situation
-    // that a third party store doesn't serve them.
-    String s3Etag = s3AFileStatus.getETag();
-    String pmEtag = msStatus.getETag();
-    boolean etagsMatch = s3Etag != null && s3Etag.equals(pmEtag);
-
-    // version ID: only in some stores, and will be missing in the metastore
-    // if the entry was created through a list operation.
-    String s3VersionId = s3AFileStatus.getVersionId();
-    String pmVersionId = msStatus.getVersionId();
-    boolean versionsMatchOrMissingInMetastore =
-        pmVersionId == null || pmVersionId.equals(s3VersionId);
-    if (sizeMatch && etagsMatch && versionsMatchOrMissingInMetastore) {
-      // update the store, return the new value
-      LOG.debug("Refreshing the metastore entry/timestamp");
-      putAndReturn(metadataStore, s3AFileStatus, timeProvider);
-      return true;
-    }
-    return false;
-  }
-
-  /**
-   * Helper function which puts a given S3AFileStatus into the MetadataStore and
-   * returns the same S3AFileStatus. Instrumentation monitors the put operation.
-   * @param ms MetadataStore to {@code put()} into.
-   * @param status status to store
-   * @param timeProvider Time provider to use when writing entries
-   * @return The same status as passed in
-   * @throws IOException if metadata store update failed
-   */
-  @RetryTranslated
-  public static S3AFileStatus putAndReturn(MetadataStore ms,
-      S3AFileStatus status,
-      ITtlTimeProvider timeProvider) throws IOException {
-    return putAndReturn(ms, status, timeProvider, null);
-  }
-
-  /**
-   * Helper function which puts a given S3AFileStatus into the MetadataStore and
-   * returns the same S3AFileStatus. Instrumentation monitors the put operation.
-   * @param ms MetadataStore to {@code put()} into.
-   * @param status status to store
-   * @param timeProvider Time provider to use when writing entries
-   * @param operationState possibly-null metastore state tracker.
-   * @return The same status as passed in
-   * @throws IOException if metadata store update failed
-   */
-  @RetryTranslated
-  public static S3AFileStatus putAndReturn(
-      final MetadataStore ms,
-      final S3AFileStatus status,
-      final ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    long startTimeNano = System.nanoTime();
-    try {
-      putWithTtl(ms, new PathMetadata(status), timeProvider, operationState);
-    } finally {
-      ms.getInstrumentation().entryAdded((System.nanoTime() - startTimeNano));
-    }
-    return status;
-  }
-
-  /**
-   * Creates an authoritative directory marker for the store.
-   * @param ms MetadataStore to {@code put()} into.
-   * @param status status to store
-   * @param timeProvider Time provider to use when writing entries
-   * @param operationState possibly-null metastore state tracker.
-   * @throws IOException if metadata store update failed
-   */
-  @RetryTranslated
-  public static void putAuthDirectoryMarker(
-      final MetadataStore ms,
-      final S3AFileStatus status,
-      final ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    long startTimeNano = System.nanoTime();
-    try {
-      final PathMetadata fileMeta = authoritativeEmptyDirectoryMarker(status);
-      putWithTtl(ms, fileMeta, timeProvider, operationState);
-    } finally {
-      ms.getInstrumentation().directoryMarkedAuthoritative();
-      ms.getInstrumentation().entryAdded((System.nanoTime() - startTimeNano));
-    }
-  }
-
-  /**
-   * Initiate a bulk write and create an operation state for it.
-   * This may then be passed into put operations.
-   * @param metastore store
-   * @param operation the type of the operation.
-   * @param path path under which updates will be explicitly put.
-   * @return a store-specific state to pass into the put operations, or null
-   * @throws IOException failure
-   */
-  public static BulkOperationState initiateBulkWrite(
-      @Nullable final MetadataStore metastore,
-      final BulkOperationState.OperationType operation,
-      final Path path) throws IOException {
-    Preconditions.checkArgument(
-        operation != BulkOperationState.OperationType.Rename,
-        "Rename operations cannot be started through initiateBulkWrite");
-    if (metastore == null || isNullMetadataStore(metastore)) {
-      return null;
-    } else {
-      return metastore.initiateBulkWrite(operation, path);
-    }
-  }
-
-  /**
-   * Convert the data of a directory listing to an array of {@link FileStatus}
-   * entries. Tombstones are filtered out at this point. If the listing is null
-   * an empty array is returned.
-   * @param dirMeta directory listing -may be null
-   * @return a possibly-empty array of file status entries
-   */
-  public static S3AFileStatus[] dirMetaToStatuses(DirListingMetadata dirMeta)  {
-    if (dirMeta == null) {
-      return EMPTY_LISTING;
-    }
-
-    Collection<PathMetadata> listing = dirMeta.getListing();
-    List<FileStatus> statuses = new ArrayList<>();
-
-    for (PathMetadata pm : listing) {
-      if (!pm.isDeleted()) {
-        statuses.add(pm.getFileStatus());
-      }
-    }
-
-    return statuses.toArray(new S3AFileStatus[0]);
+  private S3Guard() {
   }
 
   /**
-   * Given directory listing metadata from both the backing store and the
-   * MetadataStore, merge the two sources of truth to create a consistent
-   * view of the current directory contents, which can be returned to clients.
-   *
-   * Also update the MetadataStore to reflect the resulting directory listing.
-   *
-   * In not authoritative case: update file metadata if mod_time in listing
-   * of a file is greater then what is currently in the ms
-   *
-   * @param ms MetadataStore to use.
-   * @param path path to directory
-   * @param backingStatuses Directory listing from the backing store.
-   * @param dirMeta  Directory listing from MetadataStore.  May be null.
-   * @param isAuthoritative State of authoritative mode
-   * @param timeProvider Time provider to use when updating entries
-   * @param toStatusItr function to convert array of file status to
-   *                    RemoteIterator.
-   * @return Final result of directory listing.
-   * @throws IOException if metadata store update failed
-   */
-  public static RemoteIterator<S3AFileStatus> dirListingUnion(
-          MetadataStore ms, Path path,
-          RemoteIterator<S3AFileStatus> backingStatuses,
-          DirListingMetadata dirMeta, boolean isAuthoritative,
-          ITtlTimeProvider timeProvider,
-          Function<S3AFileStatus[], RemoteIterator<S3AFileStatus>> toStatusItr)
-          throws IOException {
-
-    // Fast-path for NullMetadataStore
-    if (isNullMetadataStore(ms)) {
-      return backingStatuses;
-    }
-
-    assertQualified(path);
-
-    if (dirMeta == null) {
-      // The metadataStore had zero state for this directory
-      dirMeta = new DirListingMetadata(path, DirListingMetadata.EMPTY_DIR,
-          false);
-    }
-
-    // Since we treat the MetadataStore as a "fresher" or "consistent" view
-    // of metadata, we always use its metadata first.
-
-    // Since the authoritative case is already handled outside this function,
-    // we will basically start with the set of directory entries in the
-    // DirListingMetadata, and add any that only exist in the backingStatuses.
-    //
-    // We try to avoid writing any more child entries than need be to :-
-    //  (a) save time and money.
-    //  (b) avoid overwriting the authoritative bit of children (HADOOP-16746).
-    // For auth mode updates, we supply the full listing and a list of which
-    // child entries have not been changed; the store gets to optimize its
-    // update however it chooses.
-    //
-    // for non-auth-mode S3Guard, we just build a list of entries to add and
-    // submit them in a batch; this is more efficient than trickling out the
-    // updates one-by-one.
-
-    BulkOperationState operationState = ms.initiateBulkWrite(
-        BulkOperationState.OperationType.Listing,
-        path);
-    if (isAuthoritative) {
-      authoritativeUnion(ms, path, backingStatuses, dirMeta,
-          timeProvider, operationState);
-    } else {
-      nonAuthoritativeUnion(ms, path, backingStatuses, dirMeta,
-          timeProvider, operationState);
-    }
-    IOUtils.cleanupWithLogger(LOG, operationState);
-
-    return toStatusItr.apply(dirMetaToStatuses(dirMeta));
-  }
-
-  /**
-   * Perform the authoritative union operation.
-   * Here all updated/missing entries are added back; we take care
-   * not to overwrite unchanged entries as that will lose their
-   * isAuthoritative bit (HADOOP-16746).
-   * @param ms MetadataStore to use.
-   * @param path path to directory
-   * @param backingStatuses Directory listing from the backing store.
-   * @param dirMeta  Directory listing from MetadataStore.  May be null.
-   * @param timeProvider Time provider to use when updating entries
-   * @param operationState ongoing operation
-   * @throws IOException if metadata store update failed
-   */
-  private static void authoritativeUnion(
-      final MetadataStore ms,
-      final Path path,
-      final RemoteIterator<S3AFileStatus> backingStatuses,
-      final DirListingMetadata dirMeta,
-      final ITtlTimeProvider timeProvider,
-      final BulkOperationState operationState) throws IOException {
-    // track all unchanged entries; used so the metastore can identify entries
-    // it doesn't need to update
-    List<Path> unchangedEntries = new ArrayList<>(dirMeta.getListing().size());
-    boolean changed = !dirMeta.isAuthoritative();
-    Set<Path> deleted = dirMeta.listTombstones();
-    final Map<Path, PathMetadata> dirMetaMap = dirMeta.getListing().stream()
-        .collect(Collectors.toMap(pm -> pm.getFileStatus().getPath(), pm -> pm));
-    while (backingStatuses.hasNext()) {
-      S3AFileStatus s = backingStatuses.next();
-      final Path statusPath = s.getPath();
-      if (deleted.contains(statusPath)) {
-        continue;
-      }
-
-      // this is built up to be whatever entry is to be added to the dirMeta
-      // collection
-      PathMetadata pathMetadata = dirMetaMap.get(statusPath);
-
-      if (pathMetadata == null) {
-        // there's no entry in the listing, so create one.
-        pathMetadata = new PathMetadata(s);
-      } else {
-        // no change -add the path to the list of unchangedEntries
-        unchangedEntries.add(statusPath);
-      }
-
-      // Minor race condition here.  Multiple threads could add to this
-      // mutable DirListingMetadata.  Since it is backed by a
-      // ConcurrentHashMap, the last put() wins.
-      // More concerning is two threads racing on listStatus() and delete().
-      // Any FileSystem has similar race conditions, but we could persist
-      // a stale entry longer.  We could expose an atomic
-      // DirListingMetadata#putIfNotPresent()
-      changed |= dirMeta.put(pathMetadata);
-    }
-
-    if (changed) {
-      // in an authoritative update, we pass in the full list of entries,
-      // but do declare which have not changed to avoid needless and potentially
-      // destructive overwrites.
-      LOG.debug("Marking the directory {} as authoritative", path);
-      ms.getInstrumentation().directoryMarkedAuthoritative();
-      dirMeta.setAuthoritative(true); // This is the full directory contents
-      // write the updated dir entry and any changed children.
-      S3Guard.putWithTtl(ms, dirMeta, unchangedEntries, timeProvider, operationState);
-    }
-  }
-
-  /**
-   * Perform the authoritative union operation.
-   * @param ms MetadataStore to use.
-   * @param path path to directory
-   * @param backingStatuses Directory listing from the backing store.
-   * @param dirMeta  Directory listing from MetadataStore.  May be null.
-   * @param timeProvider Time provider to use when updating entries
-   * @param operationState ongoing operation
-   * @throws IOException if metadata store update failed
-   */
-  private static void nonAuthoritativeUnion(
-      final MetadataStore ms,
-      final Path path,
-      final RemoteIterator<S3AFileStatus> backingStatuses,
-      final DirListingMetadata dirMeta,
-      final ITtlTimeProvider timeProvider,
-      final BulkOperationState operationState) throws IOException {
-    List<PathMetadata> entriesToAdd = new ArrayList<>();
-    Set<Path> deleted = dirMeta.listTombstones();
-
-    final Map<Path, PathMetadata> dirMetaMap = dirMeta.getListing().stream()
-        .collect(Collectors.toMap(pm -> pm.getFileStatus().getPath(), pm -> pm));
-    while (backingStatuses.hasNext()) {
-      S3AFileStatus s = backingStatuses.next();
-      final Path statusPath = s.getPath();
-      if (deleted.contains(statusPath)) {
-        continue;
-      }
-
-      // this is the record in dynamo
-      PathMetadata pathMetadata = dirMetaMap.get(statusPath);
-
-      // in non-auth listings, we compare the file status of the metastore
-      // list with those in the FS, and overwrite the MS entry if
-      // either of two conditions are met
-      // - there is no entry in the metastore and
-      //   DIR_MERGE_UPDATES_ALL_RECORDS_NONAUTH is compiled to true
-      // - there is an entry in the metastore the FS entry is newer.
-      boolean shouldUpdate;
-      if (pathMetadata != null) {
-        // entry is in DDB; check modification time
-        shouldUpdate = s.getModificationTime() > (pathMetadata.getFileStatus())
-            .getModificationTime();
-        // create an updated record.
-        pathMetadata = new PathMetadata(s);
-      } else {
-        // entry is not present. Create for insertion into dirMeta
-        pathMetadata = new PathMetadata(s);
-        // use hard-coded policy about updating
-        shouldUpdate = DIR_MERGE_UPDATES_ALL_RECORDS_NONAUTH;
-      }
-      if (shouldUpdate) {
-        // we do want to update DDB and the listing with a new entry.
-        LOG.debug("Update ms with newer metadata of: {}", s);
-        // ensure it gets into the dirListing
-        // add to the list of entries to add later,
-        entriesToAdd.add(pathMetadata);
-      }
-      // add the entry to the union; no-op if it was already there.
-      dirMeta.put(pathMetadata);
-    }
-
-    if (!entriesToAdd.isEmpty()) {
-        // non-auth, just push out the updated entry list
-      LOG.debug("Adding {} entries under directory {}", entriesToAdd.size(), path);
-      putWithTtl(ms, entriesToAdd, timeProvider, operationState);
-    }
-  }
-
-  /**
-   * Although NullMetadataStore does nothing, callers may wish to avoid work
-   * (fast path) when the NullMetadataStore is in use.
-   * @param ms The MetadataStore to test
-   * @return true iff the MetadataStore is the null, or no-op, implementation.
-   */
-  public static boolean isNullMetadataStore(MetadataStore ms) {
-    return (ms instanceof NullMetadataStore);
-  }
-
-  /**
-   * Update MetadataStore to reflect creation of the given  directories.
-   *
-   * If an IOException is raised while trying to update the entry, this
-   * operation catches the exception, swallows it and returns.
-   *
-   * @deprecated this is no longer called by {@code S3AFilesystem.innerMkDirs}.
-   * See: HADOOP-15079 (January 2018).
-   * It is currently retained because of its discussion in the method on
-   * atomicity and in case we need to reinstate it or adapt the current
-   * process of directory marker creation.
-   * But it is not being tested and so may age with time...consider
-   * deleting it in future if it's clear there's no need for it.
-   * @param ms    MetadataStore to update.
-   * @param dirs  null, or an ordered list of directories from leaf to root.
-   *              E.g. if /a/ exists, and  mkdirs(/a/b/c/d) is called, this
-   *              list will contain [/a/b/c/d, /a/b/c, /a/b].   /a/b/c/d is
-   *              an empty, dir, and the other dirs only contain their child
-   *              dir.
-   * @param owner Hadoop user name.
-   * @param authoritative Whether to mark new directories as authoritative.
-   * @param timeProvider Time provider.
-   */
-  @Deprecated
-  @Retries.OnceExceptionsSwallowed
-  public static void makeDirsOrdered(MetadataStore ms, List<Path> dirs,
-      String owner, boolean authoritative, ITtlTimeProvider timeProvider) {
-    if (dirs == null) {
-      return;
-    }
-
-    /* We discussed atomicity of this implementation.
-     * The concern is that multiple clients could race to write different
-     * cached directories to the MetadataStore.  Two solutions are proposed:
-     * 1. Move mkdirs() into MetadataStore interface and let implementations
-     *    ensure they are atomic.
-     * 2. Specify that the semantics of MetadataStore#putListStatus() is
-     *    always additive,  That is, if MetadataStore has listStatus() state
-     *    for /a/b that contains [/a/b/file0, /a/b/file1], and we then call
-     *    putListStatus(/a/b -> [/a/b/file2, /a/b/file3], isAuthoritative=true),
-     *    then we will end up with final state of
-     *    [/a/b/file0, /a/b/file1, /a/b/file2, /a/b/file3], isAuthoritative =
-     *    true
-     */
-    S3AFileStatus prevStatus = null;
-
-    // Use new batched put to reduce round trips.
-    List<PathMetadata> pathMetas = new ArrayList<>(dirs.size());
-
-    try {
-      // Iterate from leaf to root
-      for (int i = 0; i < dirs.size(); i++) {
-        boolean isLeaf = (prevStatus == null);
-        Path f = dirs.get(i);
-        assertQualified(f);
-        S3AFileStatus status =
-            createUploadFileStatus(f, true, 0, 0, owner, null, null);
-
-        // We only need to put a DirListingMetadata if we are setting
-        // authoritative bit
-        DirListingMetadata dirMeta = null;
-        if (authoritative) {
-          Collection<PathMetadata> children;
-          if (isLeaf) {
-            children = DirListingMetadata.EMPTY_DIR;
-          } else {
-            children = new ArrayList<>(1);
-            children.add(new PathMetadata(prevStatus));
-          }
-          dirMeta = new DirListingMetadata(f, children, authoritative);
-          S3Guard.putWithTtl(ms, dirMeta, Collections.emptyList(), timeProvider, null);
-        }
-
-        pathMetas.add(new PathMetadata(status));
-        prevStatus = status;
-      }
-
-      // Batched put
-      S3Guard.putWithTtl(ms, pathMetas, timeProvider, null);
-    } catch (IOException ioe) {
-      LOG.error("MetadataStore#put() failure:", ioe);
-    }
-  }
-
-  /**
-   * Helper function that records the move of directory paths, adding
-   * resulting metadata to the supplied lists.
-   * Does not store in MetadataStore.
-   * @param ms  MetadataStore, used to make this a no-op, when it is
-   *            NullMetadataStore.
-   * @param srcPaths stores the source path here
-   * @param dstMetas stores destination metadata here
-   * @param srcPath  source path to store
-   * @param dstPath  destination path to store
-   * @param owner file owner to use in created records
-   */
-  public static void addMoveDir(MetadataStore ms, Collection<Path> srcPaths,
-      Collection<PathMetadata> dstMetas, Path srcPath, Path dstPath,
-      String owner) {
-    if (isNullMetadataStore(ms)) {
-      return;
-    }
-    assertQualified(srcPath, dstPath);
-
-    S3AFileStatus dstStatus = createUploadFileStatus(dstPath, true, 0,
-        0, owner, null, null);
-    addMoveStatus(srcPaths, dstMetas, srcPath, dstStatus);
-  }
-
-  /**
-   * Like {@link #addMoveDir(MetadataStore, Collection, Collection, Path,
-   * Path, String)} (), but for files.
-   * @param ms  MetadataStore, used to make this a no-op, when it is
-   *            NullMetadataStore.
-   * @param srcPaths stores the source path here
-   * @param dstMetas stores destination metadata here
-   * @param srcPath  source path to store
-   * @param dstPath  destination path to store
-   * @param size length of file moved
-   * @param blockSize  blocksize to associate with destination file
-   * @param owner file owner to use in created records
-   * @param eTag the s3 object eTag of file moved
-   * @param versionId the s3 object versionId of file moved
-   */
-  public static void addMoveFile(MetadataStore ms, Collection<Path> srcPaths,
-      Collection<PathMetadata> dstMetas, Path srcPath, Path dstPath,
-      long size, long blockSize, String owner, String eTag, String versionId) {
-    if (isNullMetadataStore(ms)) {
-      return;
-    }
-    assertQualified(srcPath, dstPath);
-    S3AFileStatus dstStatus = createUploadFileStatus(dstPath, false,
-        size, blockSize, owner, eTag, versionId);
-    addMoveStatus(srcPaths, dstMetas, srcPath, dstStatus);
-  }
-
-  /**
-   * Helper method that records the move of all ancestors of a path.
-   *
-   * In S3A, an optimization is to delete unnecessary fake directory objects if
-   * the directory is non-empty. In that case, for a nested child to move, S3A
-   * is not listing and thus moving all its ancestors (up to source root). So we
-   * take care of those inferred directories of this path explicitly.
-   *
-   * As {@link #addMoveFile} and {@link #addMoveDir}, this method adds resulting
-   * metadata to the supplied lists. It does not update the MetadataStore.
-   *
-   * @param ms MetadataStore, no-op if it is NullMetadataStore
-   * @param srcPaths stores the source path here
-   * @param dstMetas stores destination metadata here
-   * @param srcRoot source root up to which (exclusive) should we add ancestors
-   * @param srcPath source path of the child to add ancestors
-   * @param dstPath destination path of the child to add ancestors
-   * @param owner Hadoop user name
-   */
-  public static void addMoveAncestors(MetadataStore ms,
-      Collection<Path> srcPaths, Collection<PathMetadata> dstMetas,
-      Path srcRoot, Path srcPath, Path dstPath, String owner) {
-    if (isNullMetadataStore(ms)) {
-      return;
-    }
-
-    assertQualified(srcRoot, srcPath, dstPath);
-
-    if (srcPath.equals(srcRoot)) {
-      LOG.debug("Skip moving ancestors of source root directory {}", srcRoot);
-      return;
-    }
-
-    Path parentSrc = srcPath.getParent();
-    Path parentDst = dstPath.getParent();
-    while (parentSrc != null
-        && !parentSrc.isRoot()
-        && !parentSrc.equals(srcRoot)
-        && !srcPaths.contains(parentSrc)) {
-      LOG.debug("Renaming non-listed parent {} to {}", parentSrc, parentDst);
-      S3Guard.addMoveDir(ms, srcPaths, dstMetas, parentSrc, parentDst, owner);
-      parentSrc = parentSrc.getParent();
-      parentDst = parentDst.getParent();
-    }
-  }
-
-  /**
-   * This adds all new ancestors of a path as directories.
-   * This forwards to
-   * {@link MetadataStore#addAncestors(Path, BulkOperationState)}.
-   * <p>
-   * Originally it implemented the logic to probe for an add ancestors,
-   * but with the addition of a store-specific bulk operation state
-   * it became unworkable.
-   *
-   * @param metadataStore store
-   * @param qualifiedPath path to update
-   * @param operationState (nullable) operational state for a bulk update
-   * @throws IOException failure
-   */
-  @Retries.RetryTranslated
-  public static void addAncestors(
-      final MetadataStore metadataStore,
-      final Path qualifiedPath,
-      final ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    metadataStore.addAncestors(qualifiedPath, operationState);
-  }
-
-  /**
-   * Add the fact that a file was moved from a source path to a destination.
-   * @param srcPaths collection of source paths to update
-   * @param dstMetas collection of destination meta data entries to update.
-   * @param srcPath path of the source file.
-   * @param dstStatus status of the source file after it was copied.
-   */
-  private static void addMoveStatus(Collection<Path> srcPaths,
-      Collection<PathMetadata> dstMetas,
-      Path srcPath,
-      S3AFileStatus dstStatus) {
-    srcPaths.add(srcPath);
-    dstMetas.add(new PathMetadata(dstStatus));
-  }
-
-  /**
-   * Assert that the path is qualified with a host and scheme.
-   * @param p path to check
-   * @throws NullPointerException if either argument does not hold
-   */
-  public static void assertQualified(Path p) {
-    URI uri = p.toUri();
-    // Paths must include bucket in case MetadataStore is shared between
-    // multiple S3AFileSystem instances
-    Preconditions.checkNotNull(uri.getHost(), "Null host in " + uri);
-
-    // This should never fail, but is retained for completeness.
-    Preconditions.checkNotNull(uri.getScheme(), "Null scheme in " + uri);
-  }
-
-  /**
-   * Assert that all paths are valid.
-   * @param paths path to check
-   * @throws NullPointerException if either argument does not hold
-   */
-  public static void assertQualified(Path...paths) {
-    for (Path path : paths) {
-      assertQualified(path);
-    }
-  }
-
-  /**
-   * Runtime implementation for TTL Time Provider interface.
-   */
-  public static class TtlTimeProvider implements ITtlTimeProvider {
-    private long authoritativeDirTtl;
-
-    public TtlTimeProvider(long authoritativeDirTtl) {
-      this.authoritativeDirTtl = authoritativeDirTtl;
-    }
-
-    public TtlTimeProvider(Configuration conf) {
-      this.authoritativeDirTtl =
-          conf.getTimeDuration(METADATASTORE_METADATA_TTL,
-              DEFAULT_METADATASTORE_METADATA_TTL, TimeUnit.MILLISECONDS);
-    }
-
-    @Override
-    public long getNow() {
-      return System.currentTimeMillis();
-    }
-
-    @Override public long getMetadataTtl() {
-      return authoritativeDirTtl;
-    }
-
-    @Override
-    public boolean equals(final Object o) {
-      if (this == o) { return true; }
-      if (o == null || getClass() != o.getClass()) { return false; }
-      final TtlTimeProvider that = (TtlTimeProvider) o;
-      return authoritativeDirTtl == that.authoritativeDirTtl;
-    }
-
-    @Override
-    public int hashCode() {
-      return Objects.hash(authoritativeDirTtl);
-    }
-
-    @Override
-    public String toString() {
-      final StringBuilder sb = new StringBuilder(
-          "TtlTimeProvider{");
-      sb.append("authoritativeDirTtl=").append(authoritativeDirTtl);
-      sb.append(" millis}");
-      return sb.toString();
-    }
-  }
-
-  /**
-   * Put a directory entry, setting the updated timestamp of the
-   * directory and its children.
-   * @param ms metastore
-   * @param dirMeta directory
-   * @param unchangedEntries list of unchanged entries from the listing
-   * @param timeProvider nullable time provider
-   * @throws IOException failure.
-   */
-  public static void putWithTtl(MetadataStore ms,
-      DirListingMetadata dirMeta,
-      final List<Path> unchangedEntries,
-      final ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState)
-      throws IOException {
-    long now = timeProvider.getNow();
-    dirMeta.setLastUpdated(now);
-    dirMeta.getListing()
-        .forEach(pm -> pm.setLastUpdated(now));
-    ms.put(dirMeta, unchangedEntries, operationState);
-  }
-
-  /**
-   * Put an entry, using the time provider to set its timestamp.
-   * @param ms metastore
-   * @param fileMeta entry to write
-   * @param timeProvider nullable time provider
-   * @param operationState nullable state for a bulk update
-   * @throws IOException failure.
-   */
-  public static void putWithTtl(MetadataStore ms, PathMetadata fileMeta,
-      @Nullable ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState) throws IOException {
-    if (timeProvider != null) {
-      fileMeta.setLastUpdated(timeProvider.getNow());
-    } else {
-      LOG.debug("timeProvider is null, put {} without setting last_updated",
-          fileMeta);
-    }
-    ms.put(fileMeta, operationState);
-  }
-
-  /**
-   * Put entries, using the time provider to set their timestamp.
-   * @param ms metastore
-   * @param fileMetas file metadata entries.
-   * @param timeProvider nullable time provider
-   * @param operationState nullable state for a bulk update
-   * @throws IOException failure.
-   */
-  public static void putWithTtl(MetadataStore ms,
-      Collection<? extends PathMetadata> fileMetas,
-      @Nullable ITtlTimeProvider timeProvider,
-      @Nullable final BulkOperationState operationState)
-      throws IOException {
-    patchLastUpdated(fileMetas, timeProvider);
-    ms.put(fileMetas, operationState);
-  }
-
-  /**
-   * Patch any collection of metadata entries with the timestamp
-   * of a time provider.
-   * This <i>MUST</i> be used when creating new entries for directories.
-   * @param fileMetas file metadata entries.
-   * @param timeProvider nullable time provider
-   */
-  static void patchLastUpdated(
-      final Collection<? extends PathMetadata> fileMetas,
-      @Nullable final ITtlTimeProvider timeProvider) {
-    if (timeProvider != null) {
-      final long now = timeProvider.getNow();
-      fileMetas.forEach(fileMeta -> fileMeta.setLastUpdated(now));
-    } else {
-      LOG.debug("timeProvider is null, put {} without setting last_updated",
-          fileMetas);
-    }
-  }
-
-  /**
-   * Get a path entry provided it is not considered expired.
-   * If the allowAuthoritative flag is true, return without
-   * checking for TTL expiry.
-   * @param ms metastore
-   * @param path path to look up.
-   * @param timeProvider nullable time provider
-   * @param needEmptyDirectoryFlag if true, implementation will
-   * return known state of directory emptiness.
-   * @param allowAuthoritative if this flag is true, the ttl won't apply to the
-   * metadata - so it will be returned regardless of it's expiry.
-   * @return the metadata or null if there as no entry.
-   * @throws IOException failure.
-   */
-  public static PathMetadata getWithTtl(MetadataStore ms, Path path,
-      @Nullable ITtlTimeProvider timeProvider,
-      final boolean needEmptyDirectoryFlag,
-      final boolean allowAuthoritative) throws IOException {
-    final PathMetadata pathMetadata = ms.get(path, needEmptyDirectoryFlag);
-    // if timeProvider is null let's return with what the ms has
-    if (timeProvider == null) {
-      LOG.debug("timeProvider is null, returning pathMetadata as is");
-      return pathMetadata;
-    }
-
-    // authoritative mode is enabled for this directory, return what the ms has
-    if (allowAuthoritative) {
-      LOG.debug("allowAuthoritative is true, returning pathMetadata as is");
-      return pathMetadata;
-    }
-
-    long ttl = timeProvider.getMetadataTtl();
-
-    if (pathMetadata != null) {
-      // Special case: the path metadata's last updated is 0. This can happen
-      // eg. with an old db using this implementation
-      if (pathMetadata.getLastUpdated() == 0) {
-        LOG.debug("PathMetadata TTL for {} is 0, so it will be returned as "
-            + "not expired.", path);
-        return pathMetadata;
-      }
-
-      if (!pathMetadata.isExpired(ttl, timeProvider.getNow())) {
-        return pathMetadata;
-      } else {
-        LOG.debug("PathMetadata TTl for {} is expired in metadata store"
-                + " -removing entry", path);
-        // delete the tombstone
-        ms.forgetMetadata(path);
-        return null;
-      }
-    }
-
-    return null;
-  }
-
-  /**
-   * List children; mark the result as non-auth if the TTL has expired.
-   * If the allowAuthoritative flag is true, return without filtering or
-   * checking for TTL expiry.
-   * If false: the expiry scan takes place and the
-   * TODO: should we always purge tombstones? Even in auth?
-   * @param ms metastore
-   * @param path path to look up.
-   * @param timeProvider nullable time provider
-   * @param allowAuthoritative if this flag is true, the ttl won't apply to the
-   * metadata - so it will be returned regardless of it's expiry.
-   * @return the listing of entries under a path, or null if there as no entry.
-   * @throws IOException failure.
-   */
-  @RetryTranslated
-  public static DirListingMetadata listChildrenWithTtl(MetadataStore ms,
-      Path path,
-      @Nullable ITtlTimeProvider timeProvider,
-      boolean allowAuthoritative)
-      throws IOException {
-    DirListingMetadata dlm = ms.listChildren(path);
-
-    if (timeProvider == null) {
-      LOG.debug("timeProvider is null, returning DirListingMetadata as is");
-      return dlm;
-    }
-
-    if (allowAuthoritative) {
-      LOG.debug("allowAuthoritative is true, returning pathMetadata as is");
-      return dlm;
-    }
+   * Assert that the FS is not configured to use an unsupported S3Guard
+   * option.
+   * the empty/null option is preferred.
+   * if the local store or null store is requested, the configuration is
+   * allowed.
+   * request for a DynamoDB or other store class will raise an exception.
+   * @param fsURI FileSystem URI
+   * @param conf configuration
+   * @return true if an option was set but ignored
+   * @throws PathIOException if an unsupported metastore was found.
+   */
+  public static boolean checkNoS3Guard(URI fsURI, Configuration conf) throws PathIOException {
+    final String classname = conf.getTrimmed(S3_METADATA_STORE_IMPL, "");
+
+    if (classname.isEmpty()) {
+      // all good. declare nothing was found.
+      return false;
+    }
+    // there is a s3guard configuration option
+    // ignore if harmless; reject if DDB or unknown
+    final String[] sources = conf.getPropertySources(S3_METADATA_STORE_IMPL);
+    final String origin = sources == null
+        ? "unknown"
+        : sources[0];
+    final String fsPath = fsURI.toString();
+    switch (classname) {
+    case NULL_METADATA_STORE:
+      // harmless
+      LOG.debug("Ignoring S3Guard store option of {} -no longer needed " +
+              "Origin {}",
+          NULL_METADATA_STORE, origin);
+      break;
+    case S3GUARD_METASTORE_LOCAL:
+      // used in some libraries (e.g. hboss) to force a consistent s3 in a test
+      // run.
+      // print a message and continue
+      LOG.warn("Ignoring S3Guard store option of {} -no longer needed or supported. "
+              + "Origin {}",
+          S3GUARD_METASTORE_LOCAL, origin);
+      break;
+    case S3GUARD_METASTORE_DYNAMO:
+      // this is the dangerous one, as it is a sign that a config is in use where
+      // older releases will use DDB for listing metadata, yet this
+      // client will not update it.
+      final String message = String.format("S3Guard is no longer needed/supported,"
+              + " yet %s is configured to use DynamoDB as the S3Guard metadata store."
+              + " This is no longer needed or supported. " +
+              "Origin of setting is %s",
+          fsPath, origin);
+      LOG.error(message);
+      throw new PathIOException(fsPath, message);
 
-    // filter expired entries
-    if (dlm != null) {
-      List<PathMetadata> expired = dlm.removeExpiredEntriesFromListing(
-          timeProvider.getMetadataTtl(),
-          timeProvider.getNow());
-      // now purge the tombstones
-      for (PathMetadata metadata : expired) {
-        if (metadata.isDeleted()) {
-          ms.forgetMetadata(metadata.getFileStatus().getPath());
-        }
-      }
+    default:
+      // an unknown store entirely.
+      throw new PathIOException(fsPath,
+          "Filesystem is configured to use unknown S3Guard store " + classname
+              + " origin " + origin);
     }
 
-    return dlm;
+    // an option was set, but it was harmless
+    return true;
   }
 
   public static Collection<String> getAuthoritativePaths(S3AFileSystem fs) {
@@ -1064,16 +168,12 @@ static Collection<String> getAuthoritativePaths(
    * Is the path for the given FS instance authoritative?
    * @param p path
    * @param fs filesystem
-   * @param authMetadataStore is the MS authoritative.
    * @param authPaths possibly empty list of authoritative paths
    * @return true iff the path is authoritative
    */
   public static boolean allowAuthoritative(Path p, S3AFileSystem fs,
-      boolean authMetadataStore, Collection<String> authPaths) {
+      Collection<String> authPaths) {
     String haystack = fs.maybeAddTrailingSlash(fs.qualify(p).toString());
-    if (authMetadataStore) {
-      return true;
-    }
     if (!authPaths.isEmpty()) {
       for (String needle : authPaths) {
         if (haystack.startsWith(needle)) {
@@ -1084,64 +184,4 @@ public static boolean allowAuthoritative(Path p, S3AFileSystem fs,
     return false;
   }
 
-  /**
-   * Format string to use when warning that S3Guard is disabled.
-   */
-  @VisibleForTesting
-  public static final String DISABLED_LOG_MSG =
-      "S3Guard is disabled on this bucket: %s";
-
-  /**
-   * Error string use in exception raised on an unknown log level.
-   */
-  public static final String UNKNOWN_WARN_LEVEL =
-      "Unknown " + S3GUARD_DISABLED_WARN_LEVEL + " value: ";
-
-  /**
-   * Warning levels to use when reporting S3Guard as disabled.
-   */
-  public enum DisabledWarnLevel {
-    SILENT,
-    INFORM,
-    WARN,
-    FAIL
-  }
-
-  /**
-   * Log that S3Guard is disabled -optionally raise an exception.
-   * @param logger Log to log to
-   * @param warnLevelStr string value of warn action.
-   * @param bucket bucket to use in log/error messages
-   * @throws ExitUtil.ExitException if s3guard was disabled
-   *                                and the log level is "fail"
-   * @throws IllegalArgumentException unknown warning level.
-   */
-  public static void logS3GuardDisabled(Logger logger, String warnLevelStr,
-      String bucket)
-      throws ExitUtil.ExitException, IllegalArgumentException {
-    final DisabledWarnLevel warnLevel;
-    try {
-      warnLevel = DisabledWarnLevel.valueOf(warnLevelStr.toUpperCase(Locale.US));
-    } catch (IllegalArgumentException e) {
-      throw new IllegalArgumentException(UNKNOWN_WARN_LEVEL + warnLevelStr, e);
-    }
-
-    String text = String.format(DISABLED_LOG_MSG, bucket);
-    switch (warnLevel) {
-    case SILENT:
-      logger.debug(text);
-      break;
-    case INFORM:
-      logger.info(text);
-      break;
-    case WARN:
-      logger.warn(text);
-      break;
-    case FAIL:
-      logger.error(text);
-      throw new ExitUtil.ExitException(EXIT_BAD_CONFIGURATION, text);
-    default:
-      throw new IllegalArgumentException(UNKNOWN_WARN_LEVEL + warnLevelStr);
-    }
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardDataAccessRetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardDataAccessRetryPolicy.java
deleted file mode 100644
index 915b94a0b7ba8..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardDataAccessRetryPolicy.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
-import org.apache.hadoop.io.retry.RetryPolicy;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry;
-
-/**
- * A Retry policy whose throttling comes from the S3Guard config options.
- */
-public class S3GuardDataAccessRetryPolicy extends S3ARetryPolicy {
-
-  public S3GuardDataAccessRetryPolicy(final Configuration conf) {
-    super(conf);
-  }
-
-  protected RetryPolicy createThrottleRetryPolicy(final Configuration conf) {
-    return exponentialBackoffRetry(
-        conf.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT),
-        conf.getTimeDuration(S3GUARD_DDB_THROTTLE_RETRY_INTERVAL,
-            S3GUARD_DDB_THROTTLE_RETRY_INTERVAL_DEFAULT,
-            TimeUnit.MILLISECONDS),
-        TimeUnit.MILLISECONDS);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsck.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsck.java
deleted file mode 100644
index 112deca54f11e..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsck.java
+++ /dev/null
@@ -1,764 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.security.InvalidParameterException;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport;
-import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
-import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.util.StopWatch;
-
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toSet;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.itemToPathMetadata;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.pathToKey;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.pathToParentKey;
-
-/**
- * Main class for the FSCK factored out from S3GuardTool
- * The implementation uses fixed DynamoDBMetadataStore as the backing store
- * for metadata.
- *
- * Functions:
- * <ul>
- *   <li>Checking metadata consistency between S3 and metadatastore</li>
- *   <li>Checking the internal metadata consistency</li>
- * </ul>
- */
-public class S3GuardFsck {
-  private static final Logger LOG = LoggerFactory.getLogger(S3GuardFsck.class);
-  public static final String ROOT_PATH_STRING = "/";
-
-  private final S3AFileSystem rawFS;
-  private final DynamoDBMetadataStore metadataStore;
-
-  private static final long MOD_TIME_RANGE = 2000L;
-
-  /**
-   * Creates an S3GuardFsck.
-   * @param fs the filesystem to compare to
-   * @param ms metadatastore the metadatastore to compare with (dynamo)
-   */
-  public S3GuardFsck(S3AFileSystem fs, MetadataStore ms)
-      throws InvalidParameterException {
-    this.rawFS = fs;
-
-    if (ms == null) {
-      throw new InvalidParameterException("S3A Bucket " + fs.getBucket()
-          + " should be guarded by a "
-          + DynamoDBMetadataStore.class.getCanonicalName());
-    }
-    this.metadataStore = (DynamoDBMetadataStore) ms;
-
-    Preconditions.checkArgument(!rawFS.hasMetadataStore(),
-        "Raw fs should not have a metadatastore.");
-  }
-
-  /**
-   * Compares S3 to MS.
-   * Iterative breadth first walk on the S3 structure from a given root.
-   * Creates a list of pairs (metadata in S3 and in the MetadataStore) where
-   * the consistency or any rule is violated.
-   * Uses {@link S3GuardFsckViolationHandler} to handle violations.
-   * The violations are listed in Enums: {@link Violation}
-   *
-   * @param p the root path to start the traversal
-   * @return a list of {@link ComparePair}
-   * @throws IOException
-   */
-  public List<ComparePair> compareS3ToMs(Path p) throws IOException {
-    StopWatch stopwatch = new StopWatch();
-    stopwatch.start();
-    int scannedItems = 0;
-
-    final Path rootPath = rawFS.qualify(p);
-    S3AFileStatus root = (S3AFileStatus) rawFS.getFileStatus(rootPath);
-    final List<ComparePair> comparePairs = new ArrayList<>();
-    final Queue<S3AFileStatus> queue = new ArrayDeque<>();
-    queue.add(root);
-
-    while (!queue.isEmpty()) {
-      final S3AFileStatus currentDir = queue.poll();
-
-
-      final Path currentDirPath = currentDir.getPath();
-      try {
-        List<FileStatus> s3DirListing = Arrays.asList(
-            rawFS.listStatus(currentDirPath));
-
-        // Check authoritative directory flag.
-        compareAuthoritativeDirectoryFlag(comparePairs, currentDirPath,
-            s3DirListing);
-        // Add all descendant directory to the queue
-        s3DirListing.stream().filter(pm -> pm.isDirectory())
-            .map(S3AFileStatus.class::cast)
-            .forEach(pm -> queue.add(pm));
-
-        // Check file and directory metadata for consistency.
-        final List<S3AFileStatus> children = s3DirListing.stream()
-            .filter(status -> !status.isDirectory())
-            .map(S3AFileStatus.class::cast).collect(toList());
-        final List<ComparePair> compareResult =
-            compareS3DirContentToMs(currentDir, children);
-        comparePairs.addAll(compareResult);
-
-        // Increase the scanned file size.
-        // One for the directory, one for the children.
-        scannedItems++;
-        scannedItems += children.size();
-      } catch (FileNotFoundException e) {
-        LOG.error("The path has been deleted since it was queued: "
-            + currentDirPath, e);
-      }
-
-    }
-    stopwatch.stop();
-
-    // Create a handler and handle each violated pairs
-    S3GuardFsckViolationHandler handler =
-        new S3GuardFsckViolationHandler(rawFS, metadataStore);
-    for (ComparePair comparePair : comparePairs) {
-      handler.logError(comparePair);
-    }
-
-    LOG.info("Total scan time: {}s", stopwatch.now(TimeUnit.SECONDS));
-    LOG.info("Scanned entries: {}", scannedItems);
-
-    return comparePairs;
-  }
-
-  /**
-   * Compare the directory contents if the listing is authoritative.
-   *
-   * @param comparePairs the list of compare pairs to add to
-   *                     if it contains a violation
-   * @param currentDirPath the current directory path
-   * @param s3DirListing the s3 directory listing to compare with
-   * @throws IOException
-   */
-  private void compareAuthoritativeDirectoryFlag(List<ComparePair> comparePairs,
-      Path currentDirPath, List<FileStatus> s3DirListing) throws IOException {
-    final DirListingMetadata msDirListing =
-        metadataStore.listChildren(currentDirPath);
-    if (msDirListing != null && msDirListing.isAuthoritative()) {
-      ComparePair cP = new ComparePair(s3DirListing, msDirListing);
-
-      if (s3DirListing.size() != msDirListing.numEntries()) {
-        cP.violations.add(Violation.AUTHORITATIVE_DIRECTORY_CONTENT_MISMATCH);
-      } else {
-        final Set<Path> msPaths = msDirListing.getListing().stream()
-                .map(pm -> pm.getFileStatus().getPath()).collect(toSet());
-        final Set<Path> s3Paths = s3DirListing.stream()
-                .map(pm -> pm.getPath()).collect(toSet());
-        if (!s3Paths.equals(msPaths)) {
-          cP.violations.add(Violation.AUTHORITATIVE_DIRECTORY_CONTENT_MISMATCH);
-        }
-      }
-
-      if (cP.containsViolation()) {
-        comparePairs.add(cP);
-      }
-    }
-  }
-
-  /**
-   * Compares S3 directory content to the metadata store.
-   *
-   * @param s3CurrentDir file status of the current directory
-   * @param children the contents of the directory
-   * @return the compare pairs with violations of consistency
-   * @throws IOException
-   */
-  protected List<ComparePair> compareS3DirContentToMs(
-      S3AFileStatus s3CurrentDir,
-      List<S3AFileStatus> children) throws IOException {
-    final Path path = s3CurrentDir.getPath();
-    final PathMetadata pathMetadata = metadataStore.get(path);
-    List<ComparePair> violationComparePairs = new ArrayList<>();
-
-    final ComparePair rootComparePair =
-        compareFileStatusToPathMetadata(s3CurrentDir, pathMetadata);
-    if (rootComparePair.containsViolation()) {
-      violationComparePairs.add(rootComparePair);
-    }
-
-    children.forEach(s3ChildMeta -> {
-      try {
-        final PathMetadata msChildMeta =
-            metadataStore.get(s3ChildMeta.getPath());
-        final ComparePair comparePair =
-            compareFileStatusToPathMetadata(s3ChildMeta, msChildMeta);
-        if (comparePair.containsViolation()) {
-          violationComparePairs.add(comparePair);
-        }
-      } catch (Exception e) {
-        LOG.error(e.getMessage(), e);
-      }
-    });
-
-    return violationComparePairs;
-  }
-
-  /**
-   * Compares a {@link S3AFileStatus} from S3 to a {@link PathMetadata}
-   * from the metadata store. Finds violated invariants and consistency
-   * issues.
-   *
-   * @param s3FileStatus the file status from S3
-   * @param msPathMetadata the path metadata from metadatastore
-   * @return {@link ComparePair} with the found issues
-   * @throws IOException
-   */
-  protected ComparePair compareFileStatusToPathMetadata(
-      S3AFileStatus s3FileStatus,
-      PathMetadata msPathMetadata) throws IOException {
-    final Path path = s3FileStatus.getPath();
-
-    if (msPathMetadata != null) {
-      LOG.info("Path: {} - Length S3: {}, MS: {} " +
-              "- Etag S3: {}, MS: {} ",
-          path,
-          s3FileStatus.getLen(), msPathMetadata.getFileStatus().getLen(),
-          s3FileStatus.getETag(), msPathMetadata.getFileStatus().getETag());
-    } else {
-      LOG.info("Path: {} - Length S3: {} - Etag S3: {}, no record in MS.",
-              path, s3FileStatus.getLen(), s3FileStatus.getETag());
-    }
-
-    ComparePair comparePair = new ComparePair(s3FileStatus, msPathMetadata);
-
-    if (!path.equals(path(ROOT_PATH_STRING))) {
-      final Path parentPath = path.getParent();
-      final PathMetadata parentPm = metadataStore.get(parentPath);
-
-      if (parentPm == null) {
-        comparePair.violations.add(Violation.NO_PARENT_ENTRY);
-      } else {
-        if (!parentPm.getFileStatus().isDirectory()) {
-          comparePair.violations.add(Violation.PARENT_IS_A_FILE);
-        }
-        if (parentPm.isDeleted()) {
-          comparePair.violations.add(Violation.PARENT_TOMBSTONED);
-        }
-      }
-    } else {
-      LOG.debug("Entry is in the root directory, so there's no parent");
-    }
-
-    // If the msPathMetadata is null, we RETURN because
-    // there is no metadata compare with
-    if (msPathMetadata == null) {
-      comparePair.violations.add(Violation.NO_METADATA_ENTRY);
-      return comparePair;
-    }
-
-    final S3AFileStatus msFileStatus = msPathMetadata.getFileStatus();
-    if (s3FileStatus.isDirectory() && !msFileStatus.isDirectory()) {
-      comparePair.violations.add(Violation.DIR_IN_S3_FILE_IN_MS);
-    }
-    if (!s3FileStatus.isDirectory() && msFileStatus.isDirectory()) {
-      comparePair.violations.add(Violation.FILE_IN_S3_DIR_IN_MS);
-    }
-
-    if(msPathMetadata.isDeleted()) {
-      comparePair.violations.add(Violation.TOMBSTONED_IN_MS_NOT_DELETED_IN_S3);
-    }
-
-    /**
-     * Attribute check
-     */
-    if (s3FileStatus.getLen() != msFileStatus.getLen()) {
-      comparePair.violations.add(Violation.LENGTH_MISMATCH);
-    }
-
-    // ModTime should be in the accuracy range defined.
-    long modTimeDiff = Math.abs(
-        s3FileStatus.getModificationTime() - msFileStatus.getModificationTime()
-    );
-    if (modTimeDiff > MOD_TIME_RANGE) {
-      comparePair.violations.add(Violation.MOD_TIME_MISMATCH);
-    }
-
-    if(msPathMetadata.getFileStatus().getVersionId() == null
-        || s3FileStatus.getVersionId() == null) {
-      LOG.debug("Missing versionIDs skipped. A HEAD request is "
-          + "required for each object to get the versionID.");
-    } else if(!s3FileStatus.getVersionId().equals(msFileStatus.getVersionId())) {
-      comparePair.violations.add(Violation.VERSIONID_MISMATCH);
-    }
-
-    // check etag only for files, and not directories
-    if (!s3FileStatus.isDirectory()) {
-      if (msPathMetadata.getFileStatus().getETag() == null) {
-        comparePair.violations.add(Violation.NO_ETAG);
-      } else if (s3FileStatus.getETag() != null &&
-          !s3FileStatus.getETag().equals(msFileStatus.getETag())) {
-        comparePair.violations.add(Violation.ETAG_MISMATCH);
-      }
-    }
-
-    return comparePair;
-  }
-
-  private Path path(String s) {
-    return rawFS.makeQualified(new Path(s));
-  }
-
-  /**
-   * Fix violations found during check.
-   *
-   * Currently only supports handling the following violation:
-   * - Violation.ORPHAN_DDB_ENTRY
-   *
-   * @param violations to be handled
-   * @throws IOException throws the error if there's any during handling
-   */
-  public void fixViolations(List<ComparePair> violations) throws IOException {
-    S3GuardFsckViolationHandler handler =
-        new S3GuardFsckViolationHandler(rawFS, metadataStore);
-
-    for (ComparePair v : violations) {
-      if (v.getViolations().contains(Violation.ORPHAN_DDB_ENTRY)) {
-        try {
-          handler.doFix(v);
-        } catch (IOException e) {
-          LOG.error("Error during handling the violation: ", e);
-          throw e;
-        }
-      }
-    }
-  }
-
-  /**
-   * A compare pair with the pair of metadata and the list of violations.
-   */
-  public static class ComparePair {
-    private final S3AFileStatus s3FileStatus;
-    private final PathMetadata msPathMetadata;
-
-    private final List<FileStatus> s3DirListing;
-    private final DirListingMetadata msDirListing;
-
-    private final Path path;
-
-    private final Set<Violation> violations = new HashSet<>();
-
-    ComparePair(S3AFileStatus status, PathMetadata pm) {
-      this.s3FileStatus = status;
-      this.msPathMetadata = pm;
-      this.s3DirListing = null;
-      this.msDirListing = null;
-      if (status != null) {
-        this.path = status.getPath();
-      } else {
-        this.path = pm.getFileStatus().getPath();
-      }
-    }
-
-    ComparePair(List<FileStatus> s3DirListing, DirListingMetadata msDirListing) {
-      this.s3DirListing = s3DirListing;
-      this.msDirListing = msDirListing;
-      this.s3FileStatus = null;
-      this.msPathMetadata = null;
-      this.path = msDirListing.getPath();
-    }
-
-    public S3AFileStatus getS3FileStatus() {
-      return s3FileStatus;
-    }
-
-    public PathMetadata getMsPathMetadata() {
-      return msPathMetadata;
-    }
-
-    public Set<Violation> getViolations() {
-      return violations;
-    }
-
-    public boolean containsViolation() {
-      return !violations.isEmpty();
-    }
-
-    public DirListingMetadata getMsDirListing() {
-      return msDirListing;
-    }
-
-    public List<FileStatus> getS3DirListing() {
-      return s3DirListing;
-    }
-
-    public Path getPath() {
-      return path;
-    }
-
-    @Override public String toString() {
-      return "ComparePair{" + "s3FileStatus=" + s3FileStatus
-          + ", msPathMetadata=" + msPathMetadata + ", s3DirListing=" +
-          s3DirListing + ", msDirListing=" + msDirListing + ", path="
-          + path + ", violations=" + violations + '}';
-    }
-  }
-
-  /**
-   * Check the DynamoDB metadatastore internally for consistency.
-   * <pre>
-   * Tasks to do here:
-   *  - find orphan entries (entries without a parent).
-   *  - find if a file's parent is not a directory (so the parent is a file).
-   *  - find entries where the parent is a tombstone.
-   *  - warn: no lastUpdated field.
-   * </pre>
-   */
-  public List<ComparePair> checkDdbInternalConsistency(Path basePath)
-      throws IOException {
-    Preconditions.checkArgument(basePath.isAbsolute(), "path must be absolute");
-
-    List<ComparePair> comparePairs = new ArrayList<>();
-    String rootStr = basePath.toString();
-    LOG.info("Root for internal consistency check: {}", rootStr);
-    StopWatch stopwatch = new StopWatch();
-    stopwatch.start();
-
-    final Table table = metadataStore.getTable();
-    final String username = metadataStore.getUsername();
-    DDBTree ddbTree = new DDBTree();
-
-    /*
-     * I. Root node construction
-     * - If the root node is the real bucket root, a node is constructed instead of
-     *   doing a query to the ddb because the bucket root is not stored.
-     * - If the root node is not a real bucket root then the entry is queried from
-     *   the ddb and constructed from the result.
-     */
-
-    DDBPathMetadata baseMeta;
-
-    if (!basePath.isRoot()) {
-      PrimaryKey rootKey = pathToKey(basePath);
-      final GetItemSpec spec = new GetItemSpec()
-          .withPrimaryKey(rootKey)
-          .withConsistentRead(true);
-      final Item baseItem = table.getItem(spec);
-      baseMeta = itemToPathMetadata(baseItem, username);
-
-      if (baseMeta == null) {
-        throw new FileNotFoundException(
-            "Base element metadata is null. " +
-                "This means the base path element is missing, or wrong path was " +
-                "passed as base path to the internal ddb consistency checker.");
-      }
-    } else {
-      baseMeta = new DDBPathMetadata(
-          new S3AFileStatus(Tristate.UNKNOWN, basePath, username)
-      );
-    }
-
-    DDBTreeNode root = new DDBTreeNode(baseMeta);
-    ddbTree.addNode(root);
-    ddbTree.setRoot(root);
-
-    /*
-     * II. Build and check the descendant tree:
-     * 1. query all nodes where the prefix is the given root, and put it in the tree
-     * 2. Check connectivity: check if each parent is in the hashmap
-     *    - This is done in O(n): we only need to find the parent based on the
-     *      path with a hashmap lookup.
-     *    - Do a test if the graph is connected - if the parent is not in the
-     *      hashmap, we found an orphan entry.
-     *
-     * 3. Do test the elements for errors:
-     *    - File is a parent of a file.
-     *    - Entries where the parent is tombstoned but the entries are not.
-     *    - Warn on no lastUpdated field.
-     *
-     */
-    ExpressionSpecBuilder builder = new ExpressionSpecBuilder();
-    builder.withCondition(
-        ExpressionSpecBuilder.S("parent")
-            .beginsWith(pathToParentKey(basePath))
-    );
-    final IteratorSupport<Item, ScanOutcome> resultIterator = table.scan(
-        builder.buildForScan()).iterator();
-    resultIterator.forEachRemaining(item -> {
-      final DDBPathMetadata pmd = itemToPathMetadata(item, username);
-      DDBTreeNode ddbTreeNode = new DDBTreeNode(pmd);
-      ddbTree.addNode(ddbTreeNode);
-    });
-
-    LOG.debug("Root: {}", ddbTree.getRoot());
-
-    for (Map.Entry<Path, DDBTreeNode> entry : ddbTree.getContentMap().entrySet()) {
-      final DDBTreeNode node = entry.getValue();
-      final ComparePair pair = new ComparePair(null, node.val);
-      // let's skip the root node when checking.
-      if (node.getVal().getFileStatus().getPath().isRoot()) {
-        continue;
-      }
-
-      if(node.getVal().getLastUpdated() == 0) {
-        pair.violations.add(Violation.NO_LASTUPDATED_FIELD);
-      }
-
-      // skip further checking the basenode which is not the actual bucket root.
-      if (node.equals(ddbTree.getRoot())) {
-        continue;
-      }
-
-      final Path parent = node.getFileStatus().getPath().getParent();
-      final DDBTreeNode parentNode = ddbTree.getContentMap().get(parent);
-      if (parentNode == null) {
-        pair.violations.add(Violation.ORPHAN_DDB_ENTRY);
-      } else {
-        if (!node.isTombstoned() && !parentNode.isDirectory()) {
-          pair.violations.add(Violation.PARENT_IS_A_FILE);
-        }
-        if(!node.isTombstoned() && parentNode.isTombstoned()) {
-          pair.violations.add(Violation.PARENT_TOMBSTONED);
-        }
-      }
-
-      if (!pair.violations.isEmpty()) {
-        comparePairs.add(pair);
-      }
-
-      node.setParent(parentNode);
-    }
-
-    // Create a handler and handle each violated pairs
-    S3GuardFsckViolationHandler handler =
-        new S3GuardFsckViolationHandler(rawFS, metadataStore);
-    for (ComparePair comparePair : comparePairs) {
-      handler.logError(comparePair);
-    }
-
-    stopwatch.stop();
-    LOG.info("Total scan time: {}s", stopwatch.now(TimeUnit.SECONDS));
-    LOG.info("Scanned entries: {}", ddbTree.contentMap.size());
-
-    return comparePairs;
-  }
-
-  /**
-   * DDBTree is the tree that represents the structure of items in the DynamoDB.
-   */
-  public static class DDBTree {
-    private final Map<Path, DDBTreeNode> contentMap = new HashMap<>();
-    private DDBTreeNode root;
-
-    public DDBTree() {
-    }
-
-    public Map<Path, DDBTreeNode> getContentMap() {
-      return contentMap;
-    }
-
-    public DDBTreeNode getRoot() {
-      return root;
-    }
-
-    public void setRoot(DDBTreeNode root) {
-      this.root = root;
-    }
-
-    public void addNode(DDBTreeNode pm) {
-      contentMap.put(pm.getVal().getFileStatus().getPath(), pm);
-    }
-
-    @Override
-    public String toString() {
-      return "DDBTree{" +
-          "contentMap=" + contentMap +
-          ", root=" + root +
-          '}';
-    }
-  }
-
-  /**
-   * Tree node for DDBTree.
-   */
-  private static final class DDBTreeNode {
-    private final DDBPathMetadata val;
-    private DDBTreeNode parent;
-    private final List<DDBPathMetadata> children;
-
-    private DDBTreeNode(DDBPathMetadata pm) {
-      this.val = pm;
-      this.parent = null;
-      this.children = new ArrayList<>();
-    }
-
-    public DDBPathMetadata getVal() {
-      return val;
-    }
-
-    public DDBTreeNode getParent() {
-      return parent;
-    }
-
-    public void setParent(DDBTreeNode parent) {
-      this.parent = parent;
-    }
-
-    public List<DDBPathMetadata> getChildren() {
-      return children;
-    }
-
-    public boolean isDirectory() {
-      return val.getFileStatus().isDirectory();
-    }
-
-    public S3AFileStatus getFileStatus() {
-      return val.getFileStatus();
-    }
-
-    public boolean isTombstoned() {
-      return val.isDeleted();
-    }
-
-    @Override
-    public String toString() {
-      return "DDBTreeNode{" +
-          "val=" + val +
-          ", parent=" + parent +
-          ", children=" + children +
-          '}';
-    }
-  }
-
-  /**
-   * Violation with severity and the handler.
-   * Defines the severity of the violation between 0-2
-   * where 0 is the most severe and 2 is the least severe.
-   */
-  public enum Violation {
-    /**
-     * No entry in metadatastore.
-     */
-    NO_METADATA_ENTRY(1,
-        S3GuardFsckViolationHandler.NoMetadataEntry.class),
-    /**
-     * A file or directory entry does not have a parent entry - excluding
-     * files and directories in the root.
-     */
-    NO_PARENT_ENTRY(0,
-        S3GuardFsckViolationHandler.NoParentEntry.class),
-    /**
-     * An entry’s parent is a file.
-     */
-    PARENT_IS_A_FILE(0,
-        S3GuardFsckViolationHandler.ParentIsAFile.class),
-    /**
-     * A file exists under a path for which there is a
-     * tombstone entry in the MS.
-     */
-    PARENT_TOMBSTONED(0,
-        S3GuardFsckViolationHandler.ParentTombstoned.class),
-    /**
-     * A directory in S3 is a file entry in the MS.
-     */
-    DIR_IN_S3_FILE_IN_MS(0,
-        S3GuardFsckViolationHandler.DirInS3FileInMs.class),
-    /**
-     * A file in S3 is a directory in the MS.
-     */
-    FILE_IN_S3_DIR_IN_MS(0,
-        S3GuardFsckViolationHandler.FileInS3DirInMs.class),
-    AUTHORITATIVE_DIRECTORY_CONTENT_MISMATCH(1,
-        S3GuardFsckViolationHandler.AuthDirContentMismatch.class),
-    /**
-     * An entry in the MS is tombstoned, but the object is not deleted on S3.
-     */
-    TOMBSTONED_IN_MS_NOT_DELETED_IN_S3(0,
-        S3GuardFsckViolationHandler.TombstonedInMsNotDeletedInS3.class),
-    /**
-     * Attribute mismatch.
-     */
-    LENGTH_MISMATCH(0,
-        S3GuardFsckViolationHandler.LengthMismatch.class),
-    MOD_TIME_MISMATCH(2,
-        S3GuardFsckViolationHandler.ModTimeMismatch.class),
-    /**
-     * If there's a versionID the mismatch is severe.
-     */
-    VERSIONID_MISMATCH(0,
-        S3GuardFsckViolationHandler.VersionIdMismatch.class),
-    /**
-     * If there's an etag the mismatch is severe.
-     */
-    ETAG_MISMATCH(0,
-        S3GuardFsckViolationHandler.EtagMismatch.class),
-    /**
-     * Don't worry too much if we don't have an etag.
-     */
-    NO_ETAG(2,
-        S3GuardFsckViolationHandler.NoEtag.class),
-    /**
-     * The entry does not have a parent in ddb.
-     */
-    ORPHAN_DDB_ENTRY(0, S3GuardFsckViolationHandler.OrphanDDBEntry.class),
-    /**
-     * The entry's lastUpdated field is empty.
-     */
-    NO_LASTUPDATED_FIELD(2,
-        S3GuardFsckViolationHandler.NoLastUpdatedField.class);
-
-    private final int severity;
-    private final Class<? extends S3GuardFsckViolationHandler.ViolationHandler> handler;
-
-    Violation(int s,
-        Class<? extends S3GuardFsckViolationHandler.ViolationHandler> h) {
-      this.severity = s;
-      this.handler = h;
-    }
-
-    public int getSeverity() {
-      return severity;
-    }
-
-    public Class<? extends S3GuardFsckViolationHandler.ViolationHandler> getHandler() {
-      return handler;
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsckViolationHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsckViolationHandler.java
deleted file mode 100644
index ec09b4af6b903..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsckViolationHandler.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Arrays;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-
-/**
- * Violation handler for the S3Guard's fsck.
- */
-public class S3GuardFsckViolationHandler {
-  private static final Logger LOG = LoggerFactory.getLogger(
-      S3GuardFsckViolationHandler.class);
-
-  // The rawFS and metadataStore are here to prepare when the ViolationHandlers
-  // will not just log, but fix the violations, so they will have access.
-  private final S3AFileSystem rawFs;
-  private final DynamoDBMetadataStore metadataStore;
-
-  private static String newLine = System.getProperty("line.separator");
-
-  public enum HandleMode {
-    FIX, LOG
-  }
-
-  public S3GuardFsckViolationHandler(S3AFileSystem fs,
-      DynamoDBMetadataStore ddbms) {
-
-    this.metadataStore = ddbms;
-    this.rawFs = fs;
-  }
-
-  public void logError(S3GuardFsck.ComparePair comparePair) throws IOException {
-    if (!comparePair.containsViolation()) {
-      LOG.debug("There is no violation in the compare pair: {}", comparePair);
-      return;
-    }
-
-    StringBuilder sB = new StringBuilder();
-    sB.append(newLine)
-        .append("On path: ").append(comparePair.getPath()).append(newLine);
-
-    handleComparePair(comparePair, sB, HandleMode.LOG);
-
-    LOG.error(sB.toString());
-  }
-
-  public void doFix(S3GuardFsck.ComparePair comparePair) throws IOException {
-    if (!comparePair.containsViolation()) {
-      LOG.debug("There is no violation in the compare pair: {}", comparePair);
-      return;
-    }
-
-    StringBuilder sB = new StringBuilder();
-    sB.append(newLine)
-        .append("On path: ").append(comparePair.getPath()).append(newLine);
-
-    handleComparePair(comparePair, sB, HandleMode.FIX);
-
-    LOG.info(sB.toString());
-  }
-
-  /**
-   * Create a new instance of the violation handler for all the violations
-   * found in the compare pair and use it.
-   *
-   * @param comparePair the compare pair with violations
-   * @param sB StringBuilder to append error strings from violations.
-   */
-  protected void handleComparePair(S3GuardFsck.ComparePair comparePair,
-      StringBuilder sB, HandleMode handleMode) throws IOException {
-
-    for (S3GuardFsck.Violation violation : comparePair.getViolations()) {
-      try {
-        ViolationHandler handler = violation.getHandler()
-            .getDeclaredConstructor(S3GuardFsck.ComparePair.class)
-            .newInstance(comparePair);
-
-        switch (handleMode) {
-          case FIX:
-            final String errorStr = handler.getError();
-            sB.append(errorStr);
-            break;
-          case LOG:
-            final String fixStr = handler.fixViolation(rawFs, metadataStore);
-            sB.append(fixStr);
-            break;
-          default:
-            throw new UnsupportedOperationException("Unknown handleMode: " + handleMode);
-        }
-
-      } catch (NoSuchMethodException e) {
-        LOG.error("Can not find declared constructor for handler: {}",
-            violation.getHandler());
-      } catch (IllegalAccessException | InstantiationException | InvocationTargetException e) {
-        LOG.error("Can not instantiate handler: {}",
-            violation.getHandler());
-      }
-      sB.append(newLine);
-    }
-  }
-
-  /**
-   * Violation handler abstract class.
-   * This class should be extended for violation handlers.
-   */
-  public static abstract class ViolationHandler {
-    private final PathMetadata pathMetadata;
-    private final S3AFileStatus s3FileStatus;
-    private final S3AFileStatus msFileStatus;
-    private final List<FileStatus> s3DirListing;
-    private final DirListingMetadata msDirListing;
-
-    public ViolationHandler(S3GuardFsck.ComparePair comparePair) {
-      pathMetadata = comparePair.getMsPathMetadata();
-      s3FileStatus = comparePair.getS3FileStatus();
-      if (pathMetadata != null) {
-        msFileStatus = pathMetadata.getFileStatus();
-      } else {
-        msFileStatus = null;
-      }
-      s3DirListing = comparePair.getS3DirListing();
-      msDirListing = comparePair.getMsDirListing();
-    }
-
-    public abstract String getError();
-
-    public PathMetadata getPathMetadata() {
-      return pathMetadata;
-    }
-
-    public S3AFileStatus getS3FileStatus() {
-      return s3FileStatus;
-    }
-
-    public S3AFileStatus getMsFileStatus() {
-      return msFileStatus;
-    }
-
-    public List<FileStatus> getS3DirListing() {
-      return s3DirListing;
-    }
-
-    public DirListingMetadata getMsDirListing() {
-      return msDirListing;
-    }
-
-    public String fixViolation(S3AFileSystem fs,
-        DynamoDBMetadataStore ddbms) throws IOException {
-      return String.format("Fixing of violation: %s is not supported yet.",
-          this.getClass().getSimpleName());
-    }
-  }
-
-  /**
-   * The violation handler when there's no matching metadata entry in the MS.
-   */
-  public static class NoMetadataEntry extends ViolationHandler {
-
-    public NoMetadataEntry(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "No PathMetadata for this path in the MS.";
-    }
-  }
-
-  /**
-   * The violation handler when there's no parent entry.
-   */
-  public static class NoParentEntry extends ViolationHandler {
-
-    public NoParentEntry(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "Entry does not have a parent entry (not root)";
-    }
-  }
-
-  /**
-   * The violation handler when the parent of an entry is a file.
-   */
-  public static class ParentIsAFile extends ViolationHandler {
-
-    public ParentIsAFile(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "The entry's parent in the metastore database is a file.";
-    }
-  }
-
-  /**
-   * The violation handler when the parent of an entry is tombstoned.
-   */
-  public static class ParentTombstoned extends ViolationHandler {
-
-    public ParentTombstoned(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "The entry in the metastore database has a parent entry " +
-          "which is a tombstone marker";
-    }
-  }
-
-  /**
-   * The violation handler when there's a directory is a file metadata in MS.
-   */
-  public static class DirInS3FileInMs extends ViolationHandler {
-
-    public DirInS3FileInMs(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "A directory in S3 is a file entry in the MS";
-    }
-  }
-
-  /**
-   * The violation handler when a file metadata is a directory in MS.
-   */
-  public static class FileInS3DirInMs extends ViolationHandler {
-
-    public FileInS3DirInMs(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "A file in S3 is a directory entry in the MS";
-    }
-  }
-
-  /**
-   * The violation handler when there's a directory listing content mismatch.
-   */
-  public static class AuthDirContentMismatch extends ViolationHandler {
-
-    public AuthDirContentMismatch(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      final String str = String.format(
-          "The content of an authoritative directory listing does "
-              + "not match the content of the S3 listing. S3: %s, MS: %s",
-          Arrays.asList(getS3DirListing()), getMsDirListing().getListing());
-      return str;
-    }
-  }
-
-  /**
-   * The violation handler when there's a length mismatch.
-   */
-  public static class LengthMismatch extends ViolationHandler {
-
-    public LengthMismatch(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override public String getError() {
-      return String.format("File length mismatch - S3: %s, MS: %s",
-          getS3FileStatus().getLen(), getMsFileStatus().getLen());
-    }
-  }
-
-  /**
-   * The violation handler when there's a modtime mismatch.
-   */
-  public static class ModTimeMismatch extends ViolationHandler {
-
-    public ModTimeMismatch(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return String.format("File timestamp mismatch - S3: %s, MS: %s",
-          getS3FileStatus().getModificationTime(),
-          getMsFileStatus().getModificationTime());
-    }
-  }
-
-  /**
-   * The violation handler when there's a version id mismatch.
-   */
-  public static class VersionIdMismatch extends ViolationHandler {
-
-    public VersionIdMismatch(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return String.format("getVersionId mismatch - S3: %s, MS: %s",
-          getS3FileStatus().getVersionId(), getMsFileStatus().getVersionId());
-    }
-  }
-
-  /**
-   * The violation handler when there's an etag mismatch.
-   */
-  public static class EtagMismatch extends ViolationHandler {
-
-    public EtagMismatch(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return String.format("Etag mismatch - S3: %s, MS: %s",
-        getS3FileStatus().getETag(), getMsFileStatus().getETag());
-    }
-  }
-
-  /**
-   * The violation handler when there's no etag.
-   */
-  public static class NoEtag extends ViolationHandler {
-
-    public NoEtag(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "No etag.";
-    }
-  }
-
-  /**
-   * The violation handler when there's a tombstoned entry in the ms is
-   * present, but the object is not deleted in S3.
-   */
-  public static class TombstonedInMsNotDeletedInS3 extends ViolationHandler {
-
-    public TombstonedInMsNotDeletedInS3(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "The entry for the path is tombstoned in the MS.";
-    }
-  }
-
-  /**
-   * The violation handler there's no parent in the MetadataStore.
-   */
-  public static class OrphanDDBEntry extends ViolationHandler {
-
-    public OrphanDDBEntry(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "The DDB entry is orphan - there is no parent in the MS.";
-    }
-
-    @Override
-    public String fixViolation(S3AFileSystem fs, DynamoDBMetadataStore ddbms)
-        throws IOException {
-      final Path path = getPathMetadata().getFileStatus().getPath();
-      ddbms.forgetMetadata(path);
-      return String.format(
-          "Fixing violation by removing metadata entry from the " +
-              "MS on path: %s", path);
-    }
-  }
-
-  /**
-   * The violation handler when there's no last updated field for the entry.
-   */
-  public static class NoLastUpdatedField extends ViolationHandler {
-
-    public NoLastUpdatedField(S3GuardFsck.ComparePair comparePair) {
-      super(comparePair);
-    }
-
-    @Override
-    public String getError() {
-      return "No lastUpdated field provided for the entry.";
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTableAccess.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTableAccess.java
deleted file mode 100644
index ca88dadeecf83..0000000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTableAccess.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.Collection;
-import java.util.Iterator;
-
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.ItemCollection;
-import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
-import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport;
-import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
-import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION_MARKER_ITEM_NAME;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.CHILD;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.PARENT;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.TABLE_VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.itemToPathMetadata;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.pathToKey;
-
-/**
- * Package-scoped accessor to table state in S3Guard.
- * This is for maintenance, diagnostics and testing: it is <i>not</i> to
- * be used otherwise.
- * <ol>
- *   <li>
- *     Some of the operations here may dramatically alter the state of
- *     a table, so use carefully.
- *   </li>
- *   <li>
- *     Operations to assess consistency of a store are best executed
- *     against a table which is otherwise inactive.
- *   </li>
- *   <li>
- *     No retry/throttling or AWS to IOE logic here.
- *   </li>
- *   <li>
- *     If a scan or query includes the version marker in the result, it
- *     is converted to a {@link VersionMarker} instance.
- *   </li>
- * </ol>
- *
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-@Retries.OnceRaw
-class S3GuardTableAccess {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(S3GuardTableAccess.class);
-
-  /**
-   * Store instance to work with.
-   */
-  private final DynamoDBMetadataStore store;
-
-  /**
-   * Table; retrieved from the store.
-   */
-  private final Table table;
-
-  /**
-   * Construct.
-   * @param store store to work with.
-   */
-  S3GuardTableAccess(final DynamoDBMetadataStore store) {
-    this.store = checkNotNull(store);
-    this.table = checkNotNull(store.getTable());
-  }
-
-  /**
-   * Username of user in store.
-   * @return a string.
-   */
-  private String getUsername() {
-    return store.getUsername();
-  }
-
-  /**
-   * Execute a query.
-   * @param spec query spec.
-   * @return the outcome.
-   */
-  @Retries.OnceRaw
-  ItemCollection<QueryOutcome> query(QuerySpec spec) {
-    return table.query(spec);
-  }
-
-  /**
-   * Issue a query where the result is to be an iterator over
-   * the entries
-   * of DDBPathMetadata instances.
-   * @param spec query spec.
-   * @return an iterator over path entries.
-   */
-  @Retries.OnceRaw
-  Iterable<DDBPathMetadata> queryMetadata(QuerySpec spec) {
-    return new DDBPathMetadataCollection<>(query(spec));
-  }
-
-  @Retries.OnceRaw
-  ItemCollection<ScanOutcome> scan(ExpressionSpecBuilder spec) {
-    return table.scan(spec.buildForScan());
-  }
-
-  @Retries.OnceRaw
-  Iterable<DDBPathMetadata> scanMetadata(ExpressionSpecBuilder spec) {
-    return new DDBPathMetadataCollection<>(scan(spec));
-  }
-
-  @Retries.OnceRaw
-  void delete(Collection<Path> paths) {
-    paths.stream()
-        .map(PathMetadataDynamoDBTranslation::pathToKey)
-        .forEach(table::deleteItem);
-  }
-
-  @Retries.OnceRaw
-  void delete(Path path) {
-    table.deleteItem(pathToKey(path));
-  }
-
-  /**
-   * A collection which wraps the result of a query or scan.
-   * Important: iterate through this only once; the outcome
-   * of repeating an iteration is "undefined"
-   * @param <T> type of outcome.
-   */
-  private final class DDBPathMetadataCollection<T>
-      implements Iterable<DDBPathMetadata> {
-
-    /**
-     * Query/scan result.
-     */
-    private final ItemCollection<T> outcome;
-
-    /**
-     * Instantiate.
-     * @param outcome query/scan outcome.
-     */
-    private DDBPathMetadataCollection(final ItemCollection<T> outcome) {
-      this.outcome = outcome;
-    }
-
-    /**
-     * Get the iterator.
-     * @return the iterator.
-     */
-    @Override
-    public Iterator<DDBPathMetadata> iterator() {
-      return new DDBPathMetadataIterator<>(outcome.iterator());
-    }
-
-  }
-
-  /**
-   * An iterator which converts the iterated-over result of
-   * a query or scan into a {@code DDBPathMetadataIterator} entry.
-   * @param <T> type of source.
-   */
-  private final class DDBPathMetadataIterator<T> implements
-      Iterator<DDBPathMetadata> {
-
-    /**
-     * Iterator to invoke.
-     */
-    private final IteratorSupport<Item, T> it;
-
-    /**
-     * Instantiate.
-     * @param it Iterator to invoke.
-     */
-    private DDBPathMetadataIterator(final IteratorSupport<Item, T> it) {
-      this.it = it;
-    }
-
-    @Override
-    @Retries.OnceRaw
-    public boolean hasNext() {
-      return it.hasNext();
-    }
-
-    @Override
-    @Retries.OnceRaw
-    public DDBPathMetadata next() {
-      Item item = it.next();
-      Pair<String, String> key = primaryKey(item);
-      if (VERSION_MARKER_ITEM_NAME.equals(key.getLeft()) &&
-          VERSION_MARKER_ITEM_NAME.equals(key.getRight())) {
-        // a version marker is found, return the special type
-        return new VersionMarker(item);
-      } else {
-        return itemToPathMetadata(item, getUsername());
-      }
-    }
-
-  }
-
-  /**
-   * DDBPathMetadata subclass returned when a query returns
-   * the version marker.
-   * There is a FileStatus returned where the owner field contains
-   * the table version; the path is always the unqualified path "/VERSION".
-   * Because it is unqualified, operations which treat this as a normal
-   * DDB metadata entry usually fail.
-   */
-  static final class VersionMarker extends DDBPathMetadata {
-
-    /**
-     * Instantiate.
-     * @param versionMarker the version marker.
-     */
-    VersionMarker(Item versionMarker) {
-      super(new S3AFileStatus(true, new Path("/VERSION"),
-          "" + versionMarker.getString(TABLE_VERSION)));
-    }
-  }
-
-  /**
-   * Given an item, split it to the parent and child fields.
-   * @param item item to split.
-   * @return (parent, child).
-   */
-  private static Pair<String, String> primaryKey(Item item) {
-    return Pair.of(item.getString(PARENT), item.getString(CHILD));
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 71e10048b38f4..3c16d87fe1376 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -28,38 +28,29 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
 import java.util.Scanner;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import com.amazonaws.services.s3.model.MultipartUpload;
 
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.DurationFormatUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.MultipartUtils;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AUtils;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 import org.apache.hadoop.fs.s3a.auth.RolePolicies;
 import org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens;
@@ -82,11 +73,8 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.Invoker.LOG_EVENT;
-import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
-import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
 import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.FILESYSTEM_TEMP_PATH;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.SSE_DEFAULT_MASTER_KEY;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
 import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.MULTIPART_UPLOAD_ABORTED;
@@ -94,7 +82,7 @@
 
 /**
  * CLI to manage S3Guard Metadata Store.
- * <p></p>
+ * <p>
  * Some management tools invoke this class directly.
  */
 @InterfaceAudience.LimitedPrivate("management tools")
@@ -102,34 +90,39 @@
 public abstract class S3GuardTool extends Configured implements Tool,
     Closeable {
   private static final Logger LOG = LoggerFactory.getLogger(S3GuardTool.class);
+  private static final String ENTRY_POINT = "s3guard";
 
-  private static final String NAME = "s3guard";
+  private static final String NAME = ENTRY_POINT;
   private static final String COMMON_USAGE =
       "When possible and not overridden by more specific options, metadata\n" +
-      "repository information will be inferred from the S3A URL (if provided)" +
-      "\n\n" +
-      "Generic options supported are:\n" +
-      "  -conf <config file> - specify an application configuration file\n" +
-      "  -D <property=value> - define a value for a given property\n";
-
-  private static final String USAGE = NAME +
+          "repository information will be inferred from the S3A URL (if provided)" +
+          "\n\n" +
+          "Generic options supported are:\n" +
+          "  -conf <config file> - specify an application configuration file\n" +
+          "  -D <property=value> - define a value for a given property\n";
+
+  static final List<String> UNSUPPORTED_COMMANDS = Arrays.asList(
+      "init",
+      "destroy",
+      "authoritative",
+      "diff",
+      "fsck",
+      "import",
+      "prune",
+      "set-capacity");
+
+  /**
+   * Usage includes supported commands, but not the excluded ones.
+   */
+  private static final String USAGE = ENTRY_POINT +
       " [command] [OPTIONS] [s3a://BUCKET]\n\n" +
       "Commands: \n" +
-      "\t" + Init.NAME + " - " + Init.PURPOSE + "\n" +
-      "\t" + Destroy.NAME + " - " + Destroy.PURPOSE + "\n" +
-      "\t" + Authoritative.NAME + " - " + Authoritative.PURPOSE + "\n" +
       "\t" + BucketInfo.NAME + " - " + BucketInfo.PURPOSE + "\n" +
-      "\t" + Diff.NAME + " - " + Diff.PURPOSE + "\n" +
-      "\t" + Fsck.NAME + " - " + Fsck.PURPOSE + "\n" +
-      "\t" + Import.NAME + " - " + Import.PURPOSE + "\n" +
       "\t" + MarkerTool.MARKERS + " - " + MarkerTool.PURPOSE + "\n" +
-      "\t" + Prune.NAME + " - " + Prune.PURPOSE + "\n" +
-      "\t" + SetCapacity.NAME + " - " + SetCapacity.PURPOSE + "\n" +
       "\t" + SelectTool.NAME + " - " + SelectTool.PURPOSE + "\n" +
       "\t" + Uploads.NAME + " - " + Uploads.PURPOSE + "\n";
 
-  private static final String DATA_IN_S3_IS_PRESERVED
-      = "(all data in S3 is preserved)";
+  private static final String E_UNSUPPORTED = "This command is no longer supported";
 
   public abstract String getUsage();
 
@@ -141,6 +134,7 @@ public abstract class S3GuardTool extends Configured implements Tool,
   static final int ERROR = EXIT_FAIL;
   static final int E_BAD_STATE = EXIT_NOT_ACCEPTABLE;
   static final int E_NOT_FOUND = EXIT_NOT_FOUND;
+  static final int E_S3GUARD_UNSUPPORTED = ERROR;
 
   /** Error String when the wrong FS is used for binding: {@value}. **/
   @VisibleForTesting
@@ -151,12 +145,11 @@ public abstract class S3GuardTool extends Configured implements Tool,
    */
   private FileSystem baseFS;
   private S3AFileSystem filesystem;
-  private MetadataStore store;
   private final CommandFormat commandFormat;
 
   public static final String META_FLAG = "meta";
 
-  // These are common to prune, upload subcommands.
+  // These are common options
   public static final String DAYS_FLAG = "days";
   public static final String HOURS_FLAG = "hours";
   public static final String MINUTES_FLAG = "minutes";
@@ -164,13 +157,6 @@ public abstract class S3GuardTool extends Configured implements Tool,
   public static final String AGE_OPTIONS_USAGE = "[-days <days>] "
       + "[-hours <hours>] [-minutes <minutes>] [-seconds <seconds>]";
 
-  public static final String REGION_FLAG = "region";
-  public static final String READ_FLAG = "read";
-  public static final String WRITE_FLAG = "write";
-  public static final String SSE_FLAG = "sse";
-  public static final String CMK_FLAG = "cmk";
-  public static final String TAG_FLAG = "tag";
-
   public static final String VERBOSE = "verbose";
 
   /**
@@ -178,19 +164,9 @@ public abstract class S3GuardTool extends Configured implements Tool,
    * @param conf Configuration.
    * @param opts any boolean options to support
    */
-  protected S3GuardTool(Configuration conf, String...opts) {
+  protected S3GuardTool(Configuration conf, String... opts) {
     super(conf);
-
-    // Set s3guard is off warn level to silent, as the fs is often instantiated
-    // without s3guard on purpose.
-    conf.set(S3GUARD_DISABLED_WARN_LEVEL,
-        S3Guard.DisabledWarnLevel.SILENT.toString());
-
     commandFormat = new CommandFormat(0, Integer.MAX_VALUE, opts);
-    // For metadata store URI
-    commandFormat.addOptionWithValue(META_FLAG);
-    // DDB region.
-    commandFormat.addOptionWithValue(REGION_FLAG);
   }
 
   /**
@@ -200,66 +176,15 @@ protected S3GuardTool(Configuration conf, String...opts) {
   public abstract String getName();
 
   /**
-   * Close the FS and metastore.
+   * Close the FS.
    * @throws IOException on failure.
    */
   @Override
   public void close() throws IOException {
     IOUtils.cleanupWithLogger(LOG,
-        baseFS, store);
+        baseFS);
     baseFS = null;
     filesystem = null;
-    store = null;
-  }
-
-  /**
-   * Parse DynamoDB region from either -m option or a S3 path.
-   *
-   * Note that as a side effect, if the paths included an S3 path,
-   * and there is no region set on the CLI, then the S3A FS is
-   * initialized, after which {@link #filesystem} will no longer
-   * be null.
-   *
-   * @param paths remaining parameters from CLI.
-   * @throws IOException on I/O errors.
-   * @throws ExitUtil.ExitException on validation errors
-   */
-  protected void parseDynamoDBRegion(List<String> paths) throws IOException {
-    Configuration conf = getConf();
-    String fromCli = getCommandFormat().getOptValue(REGION_FLAG);
-    String fromConf = conf.get(S3GUARD_DDB_REGION_KEY);
-    boolean hasS3Path = !paths.isEmpty();
-
-    if (fromCli != null) {
-      if (fromCli.isEmpty()) {
-        throw invalidArgs("No region provided with -" + REGION_FLAG + " flag");
-      }
-      if (hasS3Path) {
-        throw invalidArgs("Providing both an S3 path and the"
-            + " -" + REGION_FLAG
-            + " flag is not supported. If you need to specify a different "
-            + "region than the S3 bucket, configure " + S3GUARD_DDB_REGION_KEY);
-      }
-      conf.set(S3GUARD_DDB_REGION_KEY, fromCli);
-      return;
-    }
-
-    if (fromConf != null) {
-      if (fromConf.isEmpty()) {
-        throw invalidArgs("No region provided with config %s",
-            S3GUARD_DDB_REGION_KEY);
-      }
-      return;
-    }
-
-    if (hasS3Path) {
-      String s3Path = paths.get(0);
-      initS3AFileSystem(s3Path);
-      return;
-    }
-
-    throw invalidArgs("No region found from -" + REGION_FLAG + " flag, " +
-        "config, or S3 bucket");
   }
 
   private long getDeltaComponent(TimeUnit unit, String arg) {
@@ -292,111 +217,8 @@ protected void addAgeOptions() {
     format.addOptionWithValue(SECONDS_FLAG);
   }
 
-  protected void checkIfS3BucketIsGuarded(List<String> paths)
-      throws IOException {
-    // be sure that path is provided in params, so there's no IOoBE
-    String s3Path = "";
-    if(!paths.isEmpty()) {
-      s3Path = paths.get(0);
-    }
-
-    // Check if DynamoDB url is set from arguments.
-    String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
-    if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
-      // If not set, check if filesystem is guarded by creating an
-      // S3AFileSystem and check if hasMetadataStore is true
-      try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
-          S3AFileSystem.newInstance(toUri(s3Path), getConf())){
-        Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
-            "The S3 bucket is unguarded. " + getName()
-                + " can not be used on an unguarded bucket.");
-      }
-    }
-  }
-
-  /**
-   * Check if bucket or DDB table name is set.
-   * @param paths position arguments in which S3 path is provided.
-   */
-  protected void checkBucketNameOrDDBTableNameProvided(List<String> paths) {
-    String s3Path = null;
-    if(!paths.isEmpty()) {
-      s3Path = paths.get(0);
-    }
-
-    String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
-
-    if(metadataStoreUri == null && s3Path == null) {
-      throw invalidArgs("S3 bucket url or DDB table name have to be provided "
-          + "explicitly to use " + getName() + " command.");
-    }
-  }
-
-  /**
-   * Parse metadata store from command line option or HDFS configuration.
-   *
-   * @param forceCreate override the auto-creation setting to true.
-   * @return a initialized metadata store.
-   * @throws IOException on unsupported metadata store.
-   */
-  protected MetadataStore initMetadataStore(boolean forceCreate)
-      throws IOException {
-    if (getStore() != null) {
-      return getStore();
-    }
-    Configuration conf;
-    if (filesystem == null) {
-      conf = getConf();
-    } else {
-      conf = filesystem.getConf();
-    }
-    String metaURI = getCommandFormat().getOptValue(META_FLAG);
-    if (metaURI != null && !metaURI.isEmpty()) {
-      URI uri = URI.create(metaURI);
-      LOG.info("Create metadata store: {}", uri + " scheme: "
-          + uri.getScheme());
-      switch (uri.getScheme().toLowerCase(Locale.ENGLISH)) {
-      case "local":
-        setStore(new LocalMetadataStore());
-        break;
-      case "dynamodb":
-        setStore(new DynamoDBMetadataStore());
-        conf.set(S3GUARD_DDB_TABLE_NAME_KEY, uri.getAuthority());
-        if (forceCreate) {
-          conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
-        }
-        break;
-      default:
-        throw new IOException(
-            String.format("Metadata store %s is not supported", uri));
-      }
-    } else {
-      // CLI does not specify metadata store URI, it uses default metadata store
-      // DynamoDB instead.
-      setStore(new DynamoDBMetadataStore());
-      if (forceCreate) {
-        conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
-      }
-    }
-
-    if (filesystem == null) {
-      getStore().initialize(conf, new S3Guard.TtlTimeProvider(conf));
-    } else {
-      getStore().initialize(filesystem, new S3Guard.TtlTimeProvider(conf));
-    }
-    LOG.info("Metadata store {} is initialized.", getStore());
-    return getStore();
-  }
-
   /**
    * Create and initialize a new S3A FileSystem instance.
-   * This instance is always created without S3Guard, so allowing
-   * a previously created metastore to be patched in.
-   *
-   * Note: this is a bit convoluted as it needs to also handle the situation
-   * of a per-bucket option in core-site.xml, which isn't easily overridden.
-   * The new config and the setting of the values before any
-   * {@code Configuration.get()} calls are critical.
    *
    * @param path s3a URI
    * @throws IOException failure to init filesystem
@@ -405,45 +227,7 @@ protected MetadataStore initMetadataStore(boolean forceCreate)
   protected void initS3AFileSystem(String path) throws IOException {
     LOG.debug("Initializing S3A FS to {}", path);
     URI uri = toUri(path);
-    // Make sure that S3AFileSystem does not hold an actual MetadataStore
-    // implementation.
-    Configuration conf = new Configuration(getConf());
-    String nullStore = NullMetadataStore.class.getName();
-    conf.set(S3_METADATA_STORE_IMPL, nullStore);
-    String bucket = uri.getHost();
-    S3AUtils.setBucketOption(conf,
-        bucket,
-        S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-    String updatedBucketOption = S3AUtils.getBucketOption(conf, bucket,
-        S3_METADATA_STORE_IMPL);
-    LOG.debug("updated bucket store option {}", updatedBucketOption);
-    Preconditions.checkState(S3GUARD_METASTORE_NULL.equals(updatedBucketOption),
-        "Expected bucket option to be %s but was %s",
-        S3GUARD_METASTORE_NULL, updatedBucketOption);
-
-    bindFilesystem(FileSystem.newInstance(uri, conf));
-  }
-
-  /**
-   * Initialize the filesystem if there is none bonded to already and
-   * the command line path list is not empty.
-   * @param paths path list.
-   * @return true if at the end of the call, getFilesystem() is not null
-   * @throws IOException failure to instantiate.
-   */
-  @VisibleForTesting
-  boolean maybeInitFilesystem(final List<String> paths)
-      throws IOException {
-    // is there an S3 FS to create?
-    if (getFilesystem() == null) {
-      // none yet -create one
-      if (!paths.isEmpty()) {
-        initS3AFileSystem(paths.get(0));
-      } else {
-        LOG.debug("No path on command line, so not instantiating FS");
-      }
-    }
-    return getFilesystem() != null;
+    bindFilesystem(FileSystem.newInstance(uri, getConf()));
   }
 
   /**
@@ -477,28 +261,16 @@ protected S3AFileSystem bindFilesystem(FileSystem bindingFS) {
     if (!(fs instanceof S3AFileSystem)) {
       throw new ExitUtil.ExitException(EXIT_SERVICE_UNAVAILABLE,
           WRONG_FILESYSTEM + "URI " + fs.getUri() + " : "
-          + fs.getClass().getName());
+              + fs.getClass().getName());
     }
     filesystem = (S3AFileSystem) fs;
     return filesystem;
   }
 
-  @VisibleForTesting
-  public MetadataStore getStore() {
-    return store;
-  }
-
-  @VisibleForTesting
-  protected void setStore(MetadataStore store) {
-    Preconditions.checkNotNull(store);
-    this.store = store;
-  }
-
   /**
    * Reset the store and filesystem bindings.
    */
   protected void resetBindings() {
-    store = null;
     filesystem = null;
   }
 
@@ -548,675 +320,6 @@ protected void dumpFileSystemStatistics(PrintStream stream) {
     println(stream, "");
   }
 
-  /**
-   * Create the metadata store.
-   */
-  static class Init extends S3GuardTool {
-    public static final String NAME = "init";
-    public static final String PURPOSE = "initialize metadata repository";
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-          "(implementation-specific)\n" +
-        "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + REGION_FLAG + " REGION - Service region for connections\n" +
-        "  -" + READ_FLAG + " UNIT - Provisioned read throughput units\n" +
-        "  -" + WRITE_FLAG + " UNIT - Provisioned write through put units\n" +
-        "  -" + SSE_FLAG + " - Enable server side encryption\n" +
-        "  -" + CMK_FLAG + " KEY - Customer managed CMK\n" +
-        "  -" + TAG_FLAG + " key=value; list of tags to tag dynamo table\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.\n"
-        + "To create a table with per-request billing, set the read and write\n"
-        + "capacities to 0";
-
-    Init(Configuration conf) {
-      super(conf, SSE_FLAG);
-      // read capacity.
-      getCommandFormat().addOptionWithValue(READ_FLAG);
-      // write capacity.
-      getCommandFormat().addOptionWithValue(WRITE_FLAG);
-      // customer managed customer master key (CMK) for server side encryption
-      getCommandFormat().addOptionWithValue(CMK_FLAG);
-      // tag
-      getCommandFormat().addOptionWithValue(TAG_FLAG);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    @Override
-    public int run(String[] args, PrintStream out) throws Exception {
-      List<String> paths = parseArgs(args);
-      try {
-        checkBucketNameOrDDBTableNameProvided(paths);
-      } catch (ExitUtil.ExitException e) {
-        errorln(USAGE);
-        throw e;
-      }
-      CommandFormat commands = getCommandFormat();
-      String readCap = commands.getOptValue(READ_FLAG);
-      if (readCap != null && !readCap.isEmpty()) {
-        int readCapacity = Integer.parseInt(readCap);
-        getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, readCapacity);
-      }
-      String writeCap = commands.getOptValue(WRITE_FLAG);
-      if (writeCap != null && !writeCap.isEmpty()) {
-        int writeCapacity = Integer.parseInt(writeCap);
-        getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCapacity);
-      }
-      if (!paths.isEmpty()) {
-        String s3path = paths.get(0);
-        URI fsURI = new URI(s3path);
-        Configuration bucketConf = propagateBucketOptions(getConf(),
-            fsURI.getHost());
-        setConf(bucketConf);
-      }
-
-      String cmk = commands.getOptValue(CMK_FLAG);
-      if (commands.getOpt(SSE_FLAG)) {
-        getConf().setBoolean(S3GUARD_DDB_TABLE_SSE_ENABLED, true);
-        LOG.debug("SSE flag is passed to command {}", this.getName());
-        if (!StringUtils.isEmpty(cmk)) {
-          if (SSE_DEFAULT_MASTER_KEY.equals(cmk)) {
-            LOG.warn("Ignoring default DynamoDB table KMS Master Key " +
-                "alias/aws/dynamodb in configuration");
-          } else {
-            LOG.debug("Setting customer managed CMK {}", cmk);
-            getConf().set(S3GUARD_DDB_TABLE_SSE_CMK, cmk);
-          }
-        }
-      } else if (!StringUtils.isEmpty(cmk)) {
-        throw invalidArgs("Option %s can only be used with option %s",
-            CMK_FLAG, SSE_FLAG);
-      }
-
-      String tags = commands.getOptValue(TAG_FLAG);
-      if (tags != null && !tags.isEmpty()) {
-        String[] stringList = tags.split(";");
-        Map<String, String> tagsKV = new HashMap<>();
-        for(String kv : stringList) {
-          if(kv.isEmpty() || !kv.contains("=")){
-            continue;
-          }
-          String[] kvSplit = kv.split("=");
-          tagsKV.put(kvSplit[0], kvSplit[1]);
-        }
-
-        for (Map.Entry<String, String> kv : tagsKV.entrySet()) {
-          getConf().set(S3GUARD_DDB_TABLE_TAG + kv.getKey(), kv.getValue());
-        }
-      }
-
-      // Validate parameters.
-      try {
-        parseDynamoDBRegion(paths);
-      } catch (ExitUtil.ExitException e) {
-        errorln(USAGE);
-        throw e;
-      }
-      MetadataStore store = initMetadataStore(true);
-      printStoreDiagnostics(out, store);
-      return SUCCESS;
-    }
-  }
-
-  /**
-   * Change the capacity of the metadata store.
-   */
-  static class SetCapacity extends S3GuardTool {
-    public static final String NAME = "set-capacity";
-    public static final String PURPOSE = "Alter metadata store IO capacity";
-    public static final String READ_CAP_INVALID = "Read capacity must have "
-        + "value greater than or equal to 1.";
-    public static final String WRITE_CAP_INVALID = "Write capacity must have "
-        + "value greater than or equal to 1.";
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-          "(implementation-specific)\n" +
-        "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + READ_FLAG + " UNIT - Provisioned read throughput units\n" +
-        "  -" + WRITE_FLAG + " UNIT - Provisioned write through put units\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.";
-
-    SetCapacity(Configuration conf) {
-      super(conf);
-      // read capacity.
-      getCommandFormat().addOptionWithValue(READ_FLAG);
-      // write capacity.
-      getCommandFormat().addOptionWithValue(WRITE_FLAG);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    @Override
-    public int run(String[] args, PrintStream out) throws Exception {
-      List<String> paths = parseArgs(args);
-      if (paths.isEmpty()) {
-        errorln(getUsage());
-        throw invalidArgs("no arguments");
-      }
-      Map<String, String> options = new HashMap<>();
-      checkIfS3BucketIsGuarded(paths);
-
-      String readCap = getCommandFormat().getOptValue(READ_FLAG);
-      if (StringUtils.isNotEmpty(readCap)) {
-        Preconditions.checkArgument(Integer.parseInt(readCap) > 0,
-            READ_CAP_INVALID);
-
-        S3GuardTool.println(out, "Read capacity set to %s", readCap);
-        options.put(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, readCap);
-      }
-      String writeCap = getCommandFormat().getOptValue(WRITE_FLAG);
-      if (StringUtils.isNotEmpty(writeCap)) {
-        Preconditions.checkArgument(Integer.parseInt(writeCap) > 0,
-            WRITE_CAP_INVALID);
-
-        S3GuardTool.println(out, "Write capacity set to %s", writeCap);
-        options.put(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCap);
-      }
-
-      // Validate parameters.
-      try {
-        parseDynamoDBRegion(paths);
-        maybeInitFilesystem(paths);
-      } catch (ExitUtil.ExitException e) {
-        errorln(USAGE);
-        throw e;
-      }
-      MetadataStore store = initMetadataStore(false);
-      store.updateParameters(options);
-      printStoreDiagnostics(out, store);
-      return SUCCESS;
-    }
-  }
-
-
-  /**
-   * Destroy a metadata store.
-   */
-  static class Destroy extends S3GuardTool {
-    public static final String NAME = "destroy";
-    public static final String PURPOSE = "destroy the Metadata Store including its"
-        + " contents" + DATA_IN_S3_IS_PRESERVED;
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-          "(implementation-specific)\n" +
-        "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + REGION_FLAG + " REGION - Service region for connections\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.";
-
-    Destroy(Configuration conf) {
-      super(conf);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    public int run(String[] args, PrintStream out) throws Exception {
-      List<String> paths = parseArgs(args);
-      try {
-        checkBucketNameOrDDBTableNameProvided(paths);
-        checkIfS3BucketIsGuarded(paths);
-        parseDynamoDBRegion(paths);
-        maybeInitFilesystem(paths);
-      } catch (ExitUtil.ExitException e) {
-        errorln(USAGE);
-        throw e;
-      }
-
-      try {
-        initMetadataStore(false);
-      } catch (FileNotFoundException e) {
-        // indication that the table was not found
-        println(out, "Metadata Store does not exist.");
-        LOG.debug("Failed to bind to store to be destroyed", e);
-        return SUCCESS;
-      }
-
-      Preconditions.checkState(getStore() != null,
-          "Metadata Store is not initialized");
-
-      try {
-        getStore().destroy();
-      } catch (TableDeleteTimeoutException e) {
-        LOG.warn("The table is been deleted but it is still (briefly)"
-            + " listed as present in AWS");
-        LOG.debug("Timeout waiting for table disappearing", e);
-      }
-      println(out, "Metadata store is deleted.");
-      return SUCCESS;
-    }
-  }
-
-  /**
-   * Import s3 metadata to the metadata store.
-   */
-  static class Import extends S3GuardTool {
-    public static final String NAME = "import";
-    public static final String PURPOSE = "import metadata from existing S3 " +
-        "data";
-    public static final String AUTH_FLAG = "authoritative";
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://PATH]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + AUTH_FLAG + " - Mark imported directory data as authoritative.\n" +
-        "  -" + VERBOSE + " - Verbose Output.\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-        "(implementation-specific)\n" +
-        "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + REGION_FLAG + " REGION - Service region for connections\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.";
-
-    Import(Configuration conf) {
-      super(conf, AUTH_FLAG, VERBOSE);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    @Override
-    public int run(String[] args, PrintStream out) throws Exception {
-      List<String> paths = parseArgs(args);
-      if (paths.isEmpty()) {
-        errorln(getUsage());
-        throw invalidArgs("no arguments");
-      }
-      String s3Path = paths.get(0);
-      initS3AFileSystem(s3Path);
-
-      URI uri = toUri(s3Path);
-      String filePath = uri.getPath();
-      if (filePath.isEmpty()) {
-        // If they specify a naked S3 URI (e.g. s3a://bucket), we'll consider
-        // root to be the path
-        filePath = "/";
-      }
-      Path path = new Path(filePath);
-      S3AFileStatus status = (S3AFileStatus) getFilesystem()
-          .getFileStatus(path);
-
-      try {
-        initMetadataStore(false);
-      } catch (FileNotFoundException e) {
-        throw storeNotFound(e);
-      }
-
-      final CommandFormat commandFormat = getCommandFormat();
-
-      final boolean verbose = commandFormat.getOpt(VERBOSE);
-      final ImportOperation importer = new ImportOperation(
-          getFilesystem(),
-          getStore(),
-          status,
-          commandFormat.getOpt(AUTH_FLAG),
-          verbose);
-      long items = importer.execute();
-      println(out, "Inserted %d items into Metadata Store", items);
-      if (verbose) {
-        dumpFileSystemStatistics(out);
-      }
-      return SUCCESS;
-    }
-
-  }
-
-  /**
-   * Show diffs between the s3 and metadata store.
-   */
-  static class Diff extends S3GuardTool {
-    public static final String NAME = "diff";
-    public static final String PURPOSE = "report on delta between S3 and " +
-        "repository";
-    private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-        "(implementation-specific)\n" +
-        "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + REGION_FLAG + " REGION - Service region for connections\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.";
-
-    private static final String SEP = "\t";
-    static final String S3_PREFIX = "S3";
-    static final String MS_PREFIX = "MS";
-
-    Diff(Configuration conf) {
-      super(conf);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    /**
-     * Formats the output of printing a FileStatus in S3guard diff tool.
-     * @param status the status to print.
-     * @return the string of output.
-     */
-    private static String formatFileStatus(FileStatus status) {
-      return String.format("%s%s%d%s%s",
-          status.isDirectory() ? "D" : "F",
-          SEP,
-          status.getLen(),
-          SEP,
-          status.getPath().toString());
-    }
-
-    /**
-     * Compares metadata from 2 S3 FileStatus's to see if they differ.
-     * @param thisOne
-     * @param thatOne
-     * @return true if the metadata is not identical
-     */
-    private static boolean differ(FileStatus thisOne, FileStatus thatOne) {
-      Preconditions.checkArgument(!(thisOne == null && thatOne == null));
-      return (thisOne == null || thatOne == null) ||
-          (thisOne.getLen() != thatOne.getLen()) ||
-          (thisOne.isDirectory() != thatOne.isDirectory()) ||
-          (!thisOne.isDirectory() &&
-              thisOne.getModificationTime() != thatOne.getModificationTime());
-    }
-
-    /**
-     * Print difference, if any, between two file statuses to the output stream.
-     *
-     * @param msStatus file status from metadata store.
-     * @param s3Status file status from S3.
-     * @param out output stream.
-     */
-    private static void printDiff(FileStatus msStatus,
-                                  FileStatus s3Status,
-                                  PrintStream out) {
-      Preconditions.checkArgument(!(msStatus == null && s3Status == null));
-      if (msStatus != null && s3Status != null) {
-        Preconditions.checkArgument(
-            msStatus.getPath().equals(s3Status.getPath()),
-            String.format("The path from metadata store and s3 are different:" +
-            " ms=%s s3=%s", msStatus.getPath(), s3Status.getPath()));
-      }
-
-      if (differ(msStatus, s3Status)) {
-        if (s3Status != null) {
-          println(out, "%s%s%s", S3_PREFIX, SEP, formatFileStatus(s3Status));
-        }
-        if (msStatus != null) {
-          println(out, "%s%s%s", MS_PREFIX, SEP, formatFileStatus(msStatus));
-        }
-      }
-    }
-
-    /**
-     * Compare the metadata of the directory with the same path, on S3 and
-     * the metadata store, respectively. If one of them is null, consider the
-     * metadata of the directory and all its subdirectories are missing from
-     * the source.
-     *
-     * Pass the FileStatus obtained from s3 and metadata store to avoid one
-     * round trip to fetch the same metadata twice, because the FileStatus
-     * hve already been obtained from listStatus() / listChildren operations.
-     *
-     * @param msDir the directory FileStatus obtained from the metadata store.
-     * @param s3Dir the directory FileStatus obtained from S3.
-     * @param out the output stream to generate diff results.
-     * @throws IOException on I/O errors.
-     */
-    private void compareDir(FileStatus msDir, FileStatus s3Dir,
-                            PrintStream out) throws IOException {
-      Preconditions.checkArgument(!(msDir == null && s3Dir == null),
-          "The path does not exist in metadata store and on s3.");
-
-      if (msDir != null && s3Dir != null) {
-        Preconditions.checkArgument(msDir.getPath().equals(s3Dir.getPath()),
-            String.format("The path from metadata store and s3 are different:" +
-             " ms=%s s3=%s", msDir.getPath(), s3Dir.getPath()));
-      }
-
-      Map<Path, FileStatus> s3Children = new HashMap<>();
-      if (s3Dir != null && s3Dir.isDirectory()) {
-        for (FileStatus status : getFilesystem().listStatus(s3Dir.getPath())) {
-          s3Children.put(status.getPath(), status);
-        }
-      }
-
-      Map<Path, FileStatus> msChildren = new HashMap<>();
-      if (msDir != null && msDir.isDirectory()) {
-        DirListingMetadata dirMeta =
-            getStore().listChildren(msDir.getPath());
-
-        if (dirMeta != null) {
-          for (PathMetadata meta : dirMeta.getListing()) {
-            FileStatus status = meta.getFileStatus();
-            msChildren.put(status.getPath(), status);
-          }
-        }
-      }
-
-      Set<Path> allPaths = new HashSet<>(s3Children.keySet());
-      allPaths.addAll(msChildren.keySet());
-
-      for (Path path : allPaths) {
-        FileStatus s3Status = s3Children.get(path);
-        FileStatus msStatus = msChildren.get(path);
-        printDiff(msStatus, s3Status, out);
-        if ((s3Status != null && s3Status.isDirectory()) ||
-            (msStatus != null && msStatus.isDirectory())) {
-          compareDir(msStatus, s3Status, out);
-        }
-      }
-      out.flush();
-    }
-
-    /**
-     * Compare both metadata store and S3 on the same path.
-     *
-     * @param path the path to be compared.
-     * @param out  the output stream to display results.
-     * @throws IOException on I/O errors.
-     */
-    private void compareRoot(Path path, PrintStream out) throws IOException {
-      Path qualified = getFilesystem().makeQualified(path);
-      FileStatus s3Status = null;
-      try {
-        s3Status = getFilesystem().getFileStatus(qualified);
-      } catch (FileNotFoundException e) {
-        /* ignored */
-      }
-      PathMetadata meta = getStore().get(qualified);
-      FileStatus msStatus = (meta != null && !meta.isDeleted()) ?
-          meta.getFileStatus() : null;
-      compareDir(msStatus, s3Status, out);
-    }
-
-    @VisibleForTesting
-    public int run(String[] args, PrintStream out) throws IOException {
-      List<String> paths = parseArgs(args);
-      if (paths.isEmpty()) {
-        out.println(USAGE);
-        throw invalidArgs("no arguments");
-      }
-      String s3Path = paths.get(0);
-      initS3AFileSystem(s3Path);
-      initMetadataStore(false);
-
-      URI uri = toUri(s3Path);
-      Path root;
-      if (uri.getPath().isEmpty()) {
-        root = new Path("/");
-      } else {
-        root = new Path(uri.getPath());
-      }
-      root = getFilesystem().makeQualified(root);
-      compareRoot(root, out);
-      out.flush();
-      return SUCCESS;
-    }
-
-  }
-
-  /**
-   * Prune metadata that has not been modified recently.
-   */
-  static class Prune extends S3GuardTool {
-    public static final String NAME = "prune";
-    public static final String PURPOSE = "truncate older metadata from " +
-        "repository "
-        + DATA_IN_S3_IS_PRESERVED;;
-
-    public static final String TOMBSTONE = "tombstone";
-
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + META_FLAG + " URL - Metadata repository details " +
-        "(implementation-specific)\n" +
-        "[-" + TOMBSTONE + "]\n" +
-        "Age options. Any combination of these integer-valued options:\n" +
-        AGE_OPTIONS_USAGE + "\n" +
-        "Amazon DynamoDB-specific options:\n" +
-        "  -" + REGION_FLAG + " REGION - Service region for connections\n" +
-        "\n" +
-        "  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.\n" +
-        "  Specifying both the -" + REGION_FLAG + " option and an S3A path\n" +
-        "  is not supported.";
-
-    Prune(Configuration conf) {
-      super(conf, TOMBSTONE);
-      addAgeOptions();
-    }
-
-    @VisibleForTesting
-    void setMetadataStore(MetadataStore ms) {
-      Preconditions.checkNotNull(ms);
-      this.setStore(ms);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    public int run(String[] args, PrintStream out) throws
-        InterruptedException, IOException {
-      List<String> paths = parseArgs(args);
-      try {
-        checkBucketNameOrDDBTableNameProvided(paths);
-        parseDynamoDBRegion(paths);
-      } catch (ExitUtil.ExitException e) {
-        errorln(USAGE);
-        throw e;
-      }
-      maybeInitFilesystem(paths);
-      initMetadataStore(false);
-
-      Configuration conf = getConf();
-      long confDelta = conf.getLong(S3GUARD_CLI_PRUNE_AGE, 0);
-
-      long cliDelta = ageOptionsToMsec();
-
-      if (confDelta <= 0 && cliDelta <= 0) {
-        errorln("You must specify a positive age for metadata to prune.");
-      }
-
-      // A delta provided on the CLI overrides if one is configured
-      long delta = confDelta;
-      if (cliDelta > 0) {
-        delta = cliDelta;
-      }
-
-      long now = System.currentTimeMillis();
-      long divide = now - delta;
-
-      // remove the protocol from path string to get keyPrefix
-      // by default the keyPrefix is "/" - unless the s3 URL is provided
-      String keyPrefix = "/";
-      if(paths.size() > 0) {
-        Path path = new Path(paths.get(0));
-        keyPrefix = PathMetadataDynamoDBTranslation.pathToParentKey(path);
-      }
-
-      MetadataStore.PruneMode mode
-          = MetadataStore.PruneMode.ALL_BY_MODTIME;
-      if (getCommandFormat().getOpt(TOMBSTONE)) {
-        mode = MetadataStore.PruneMode.TOMBSTONES_BY_LASTUPDATED;
-      }
-      try {
-        getStore().prune(mode, divide,
-            keyPrefix);
-      } catch (UnsupportedOperationException e){
-        errorln("Prune operation not supported in metadata store.");
-      }
-
-      out.flush();
-      return SUCCESS;
-    }
-
-  }
-
   /**
    * Get info about a bucket and its S3Guard integration status.
    */
@@ -1232,21 +335,22 @@ public static class BucketInfo extends S3GuardTool {
     public static final String MARKERS_FLAG = "markers";
     public static final String MARKERS_AWARE = "aware";
 
-    public static final String PURPOSE = "provide/check S3Guard information"
+    public static final String PURPOSE = "provide/check information"
         + " about a specific bucket";
 
     private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n"
         + "\t" + PURPOSE + "\n\n"
         + "Common options:\n"
-        + "  -" + GUARDED_FLAG + " - Require S3Guard\n"
-        + "  -" + UNGUARDED_FLAG + " - Force S3Guard to be disabled\n"
         + "  -" + AUTH_FLAG + " - Require the S3Guard mode to be \"authoritative\"\n"
         + "  -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n"
-        + "  -" + MAGIC_FLAG + " - Require the S3 filesystem to be support the \"magic\" committer\n"
+        + "  -" + MAGIC_FLAG +
+        " - Require the S3 filesystem to be support the \"magic\" committer\n"
         + "  -" + ENCRYPTION_FLAG
         + " (none, sse-s3, sse-kms) - Require encryption policy\n"
         + "  -" + MARKERS_FLAG
-        + " (aware, keep, delete, authoritative) - directory markers policy\n";
+        + " (aware, keep, delete, authoritative) - directory markers policy\n"
+        + "  -" + GUARDED_FLAG + " - Require S3Guard. Will always fail.\n"
+        + "  -" + UNGUARDED_FLAG + " - Force S3Guard to be disabled (always true)\n";
 
     /**
      * Output when the client cannot get the location of a bucket.
@@ -1256,7 +360,6 @@ public static class BucketInfo extends S3GuardTool {
         "Location unknown -caller lacks "
             + RolePolicies.S3_GET_BUCKET_LOCATION + " permission";
 
-
     @VisibleForTesting
     public static final String IS_MARKER_AWARE =
         "\tThe S3A connector is compatible with buckets where"
@@ -1290,20 +393,10 @@ public int run(String[] args, PrintStream out)
       CommandFormat commands = getCommandFormat();
       URI fsURI = toUri(s3Path);
 
-      // check if UNGUARDED_FLAG is passed and use NullMetadataStore in
-      // config to avoid side effects like creating the table if not exists
-      Configuration unguardedConf = getConf();
-      if (commands.getOpt(UNGUARDED_FLAG)) {
-        LOG.debug("Unguarded flag is passed to command :" + this.getName());
-        clearBucketOption(unguardedConf, fsURI.getHost(), S3_METADATA_STORE_IMPL);
-        unguardedConf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-      }
-
       S3AFileSystem fs = bindFilesystem(
-          FileSystem.newInstance(fsURI, unguardedConf));
+          FileSystem.newInstance(fsURI, getConf()));
       Configuration conf = fs.getConf();
       URI fsUri = fs.getUri();
-      MetadataStore store = fs.getMetadataStore();
       println(out, "Filesystem %s", fsUri);
       try {
         println(out, "Location: %s", fs.getBucketLocation());
@@ -1314,35 +407,17 @@ public int run(String[] args, PrintStream out)
         LOG.debug("failed to get bucket location", e);
         println(out, LOCATION_UNKNOWN);
       }
-      boolean usingS3Guard = !(store instanceof NullMetadataStore);
-      boolean authMode = false;
-      if (usingS3Guard) {
-        out.printf("Filesystem %s is using S3Guard with store %s%n",
-            fsUri, store.toString());
-        printOption(out, "Authoritative Metadata Store",
-            METADATASTORE_AUTHORITATIVE, "false");
-        printOption(out, "Authoritative Path",
-              AUTHORITATIVE_PATH, "");
-        final Collection<String> authoritativePaths
-            = S3Guard.getAuthoritativePaths(fs);
-        if (!authoritativePaths.isEmpty()) {
-          println(out, "Qualified Authoritative Paths:");
-          for (String path : authoritativePaths) {
-            println(out, "\t%s", path);
-          }
-          println(out, "");
+
+      // print any auth paths for directory marker info
+      final Collection<String> authoritativePaths
+          = S3Guard.getAuthoritativePaths(fs);
+      if (!authoritativePaths.isEmpty()) {
+        println(out, "Qualified Authoritative Paths:");
+        for (String path : authoritativePaths) {
+          println(out, "\t%s", path);
         }
-        authMode = conf.getBoolean(METADATASTORE_AUTHORITATIVE, false);
-        final long ttl = conf.getTimeDuration(METADATASTORE_METADATA_TTL,
-            DEFAULT_METADATASTORE_METADATA_TTL, TimeUnit.MILLISECONDS);
-        println(out, "\tMetadata time to live: (set in %s) = %s",
-            METADATASTORE_METADATA_TTL,
-            DurationFormatUtils.formatDurationHMS(ttl));
-        printStoreDiagnostics(out, store);
-      } else {
-        println(out, "Filesystem %s is not using S3Guard", fsUri);
+        println(out, "");
       }
-
       println(out, "%nS3A Client");
       printOption(out, "\tSigning Algorithm", SIGNING_ALGORITHM, "(unset)");
       String endpoint = conf.getTrimmed(ENDPOINT, "");
@@ -1352,7 +427,8 @@ public int run(String[] args, PrintStream out)
       String encryption =
           printOption(out, "\tEncryption", Constants.S3_ENCRYPTION_ALGORITHM,
               "none");
-      printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL);
+      printOption(out, "\tInput seek policy", INPUT_FADVISE,
+          Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT);
       printOption(out, "\tChange Detection Source", CHANGE_DETECT_SOURCE,
           CHANGE_DETECT_SOURCE_DEFAULT);
       printOption(out, "\tChange Detection Mode", CHANGE_DETECT_MODE,
@@ -1421,23 +497,8 @@ public int run(String[] args, PrintStream out)
       } else {
         println(out, "\tDelegation token support is disabled");
       }
-
-      if (usingS3Guard) {
-        if (commands.getOpt(UNGUARDED_FLAG)) {
-          throw badState("S3Guard is enabled for %s", fsUri);
-        }
-        if (commands.getOpt(AUTH_FLAG) && !authMode) {
-          throw badState("S3Guard is enabled for %s,"
-              + " but not in authoritative mode", fsUri);
-        }
-        if (commands.getOpt(NONAUTH_FLAG) && authMode) {
-          throw badState("S3Guard is enabled in authoritative mode for %s",
-              fsUri);
-        }
-      } else {
-        if (commands.getOpt(GUARDED_FLAG)) {
-          throw badState("S3Guard is not enabled for %s", fsUri);
-        }
+      if (commands.getOpt(GUARDED_FLAG)) {
+        throw badState("S3Guard is not supported");
       }
       if (commands.getOpt(MAGIC_FLAG) && !magic) {
         throw badState("The magic committer is not enabled for %s", fsUri);
@@ -1448,8 +509,8 @@ public int run(String[] args, PrintStream out)
       if (StringUtils.isNotEmpty(desiredEncryption)
           && !desiredEncryption.equalsIgnoreCase(encryption)) {
         throw badState("Bucket %s: required encryption is %s"
-                    + " but actual encryption is %s",
-                fsUri, desiredEncryption, encryption);
+                + " but actual encryption is %s",
+            fsUri, desiredEncryption, encryption);
       }
 
       // directory markers
@@ -1465,13 +526,12 @@ public int run(String[] args, PrintStream out)
      * Validate the marker options.
      * @param out output stream
      * @param fs filesystem
-     * @param path test path
      * @param marker desired marker option -may be null.
      */
     private void processMarkerOption(final PrintStream out,
         final S3AFileSystem fs,
         final String marker) {
-      println(out, "%nSecurity");
+      println(out, "%nDirectory Markers");
       DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy();
       String desc = markerPolicy.describe();
       println(out, "\tThe directory marker policy is \"%s\"", desc);
@@ -1530,8 +590,8 @@ static class Uploads extends S3GuardTool {
         "s3a://BUCKET[/path]\n"
         + "\t" + PURPOSE + "\n\n"
         + "Common options:\n"
-        + " (-" + LIST + " | -" + EXPECT +" <num-uploads> | -" + ABORT
-        + ") [-" + VERBOSE +"] "
+        + " (-" + LIST + " | -" + EXPECT + " <num-uploads> | -" + ABORT
+        + ") [-" + VERBOSE + "] "
         + "[<age-options>] [-force]\n"
         + "\t - Under given path, list or delete all uploads," +
         " or only those \n"
@@ -1548,7 +608,8 @@ static class Uploads extends S3GuardTool {
     public static final String TOTAL = "Total";
 
     /** Runs in one of three modes. */
-    private enum Mode { LIST, EXPECT, ABORT };
+    private enum Mode {LIST, EXPECT, ABORT}
+
     private Mode mode = null;
 
     /** For Mode == EXPECT, expected listing size. */
@@ -1736,228 +797,6 @@ private void vprintln(PrintStream out, String format, Object...
     }
   }
 
-  /**
-   * Fsck - check for consistency between S3 and the metadatastore.
-   */
-  static class Fsck extends S3GuardTool {
-    public static final String CHECK_FLAG = "check";
-    public static final String DDB_MS_CONSISTENCY_FLAG = "internal";
-    public static final String FIX_FLAG = "fix";
-
-    public static final String NAME = "fsck";
-    public static final String PURPOSE = "Compares S3 with MetadataStore, and "
-        + "returns a failure status if any rules or invariants are violated. "
-        + "Only works with DynamoDB metadata stores.";
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://BUCKET]\n" +
-        "\t" + PURPOSE + "\n\n" +
-        "Common options:\n" +
-        "  -" + CHECK_FLAG + " Check the metadata store for errors, but do "
-        + "not fix any issues.\n" +
-        "  -" + DDB_MS_CONSISTENCY_FLAG + " Check the dynamodb metadata store "
-        + "for internal consistency.\n" +
-        "  -" + FIX_FLAG + " Fix the errors found in the metadata store. Can " +
-        "be used with " + CHECK_FLAG + " or " + DDB_MS_CONSISTENCY_FLAG + " flags. "
-        + "\n\t\tFixes: \n" +
-        "\t\t\t- Remove orphan entries from DDB." +
-        "\n";
-
-    Fsck(Configuration conf) {
-      super(conf, CHECK_FLAG, DDB_MS_CONSISTENCY_FLAG, FIX_FLAG);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    public int run(String[] args, PrintStream out) throws
-        InterruptedException, IOException {
-      List<String> paths = parseArgs(args);
-      if (paths.isEmpty()) {
-        out.println(USAGE);
-        throw invalidArgs("no arguments");
-      }
-      int exitValue = EXIT_SUCCESS;
-
-      final CommandFormat commandFormat = getCommandFormat();
-
-      // check if there's more than one arguments
-      // from CHECK and INTERNAL CONSISTENCY
-      int flags = countTrue(commandFormat.getOpt(CHECK_FLAG),
-          commandFormat.getOpt(DDB_MS_CONSISTENCY_FLAG));
-      if (flags > 1) {
-        out.println(USAGE);
-        throw invalidArgs("There should be only one parameter used for checking.");
-      }
-      if (flags == 0 && commandFormat.getOpt(FIX_FLAG)) {
-        errorln(FIX_FLAG + " flag can be used with either " + CHECK_FLAG + " or " +
-            DDB_MS_CONSISTENCY_FLAG + " flag, but not alone.");
-        errorln(USAGE);
-        return ERROR;
-      }
-
-      String s3Path = paths.get(0);
-      try {
-        initS3AFileSystem(s3Path);
-      } catch (Exception e) {
-        errorln("Failed to initialize S3AFileSystem from path: " + s3Path);
-        throw e;
-      }
-
-      URI uri = toUri(s3Path);
-      Path root;
-      if (uri.getPath().isEmpty()) {
-        root = new Path("/");
-      } else {
-        root = new Path(uri.getPath());
-      }
-
-      final S3AFileSystem fs = getFilesystem();
-      initMetadataStore(false);
-      final MetadataStore ms = getStore();
-
-      if (ms == null ||
-          !(ms instanceof DynamoDBMetadataStore)) {
-        errorln(s3Path + " path uses metadata store: " + ms);
-        errorln(NAME + " can be only used with a DynamoDB backed s3a bucket.");
-        errorln(USAGE);
-        return ERROR;
-      }
-
-      List<S3GuardFsck.ComparePair> violations;
-
-      if (commandFormat.getOpt(CHECK_FLAG)) {
-        // do the check
-        S3GuardFsck s3GuardFsck = new S3GuardFsck(fs, ms);
-        try {
-          violations = s3GuardFsck.compareS3ToMs(fs.qualify(root));
-        } catch (IOException e) {
-          throw e;
-        }
-      } else if (commandFormat.getOpt(DDB_MS_CONSISTENCY_FLAG)) {
-        S3GuardFsck s3GuardFsck = new S3GuardFsck(fs, ms);
-        violations = s3GuardFsck.checkDdbInternalConsistency(fs.qualify(root));
-      } else {
-        errorln("No supported operation is selected.");
-        errorln(USAGE);
-        return ERROR;
-      }
-
-      if (commandFormat.getOpt(FIX_FLAG)) {
-        S3GuardFsck s3GuardFsck = new S3GuardFsck(fs, ms);
-        s3GuardFsck.fixViolations(violations);
-      }
-
-      out.flush();
-
-      // We fail if there were compare pairs, as the returned compare pairs
-      // contain issues.
-      if (violations == null || violations.size() > 0) {
-        exitValue = EXIT_FAIL;
-      }
-      return exitValue;
-    }
-
-    int countTrue(Boolean... bools) {
-      return (int) Arrays.stream(bools).filter(p -> p).count();
-    }
-  }
-  /**
-   * Audits a DynamoDB S3Guard repository for all the entries being
-   * 'authoritative'.
-   * Checks bucket settings if {@link #CHECK_FLAG} is set, then
-   * treewalk.
-   */
-  static class Authoritative extends S3GuardTool {
-
-    public static final String NAME = "authoritative";
-
-    public static final String CHECK_FLAG = "check-config";
-    public static final String REQUIRE_AUTH = "required";
-
-    public static final String PURPOSE = "Audits a DynamoDB S3Guard "
-        + "repository for all the entries being 'authoritative'";
-
-    private static final String USAGE = NAME + " [OPTIONS] [s3a://PATH]\n"
-        + "\t" + PURPOSE + "\n\n"
-        + "Options:\n"
-        + "  -" + REQUIRE_AUTH + " - Require directories under the path to"
-        + " be authoritative.\n"
-        + "  -" + CHECK_FLAG + " - Check the configuration for the path to"
-        + " be authoritative\n"
-        + "  -" + VERBOSE + " - Verbose Output.\n";
-
-    Authoritative(Configuration conf) {
-      super(conf, CHECK_FLAG, REQUIRE_AUTH, VERBOSE);
-    }
-
-    @Override
-    public String getName() {
-      return NAME;
-    }
-
-    @Override
-    public String getUsage() {
-      return USAGE;
-    }
-
-    public int run(String[] args, PrintStream out) throws
-        InterruptedException, IOException {
-      List<String> paths = parseArgs(args);
-      if (paths.isEmpty()) {
-        out.println(USAGE);
-        throw invalidArgs("no arguments");
-      }
-      maybeInitFilesystem(paths);
-      initMetadataStore(false);
-      String s3Path = paths.get(0);
-
-      URI uri = toUri(s3Path);
-      Path auditPath;
-      if (uri.getPath().isEmpty()) {
-        auditPath = new Path("/");
-      } else {
-        auditPath = new Path(uri.getPath());
-      }
-
-      final S3AFileSystem fs = getFilesystem();
-      final MetadataStore ms = getStore();
-
-      if (!(ms instanceof DynamoDBMetadataStore)) {
-        errorln(s3Path + " path uses MS: " + ms);
-        errorln(NAME + " can be only used with a DynamoDB-backed S3Guard table.");
-        errorln(USAGE);
-        return ERROR;
-      }
-
-      final CommandFormat commandFormat = getCommandFormat();
-      if (commandFormat.getOpt(CHECK_FLAG)) {
-        // check that the path is auth
-        if (!fs.allowAuthoritative(auditPath)) {
-          // path isn't considered auth in the S3A bucket info
-          errorln("Path " + auditPath
-              + " is not configured to be authoritative");
-          return AuthoritativeAuditOperation.ERROR_PATH_NOT_AUTH_IN_FS;
-        }
-      }
-
-      final AuthoritativeAuditOperation audit = new AuthoritativeAuditOperation(
-          fs.createStoreContext(),
-          (DynamoDBMetadataStore) ms,
-          commandFormat.getOpt(REQUIRE_AUTH),
-          commandFormat.getOpt(VERBOSE));
-      audit.audit(fs.qualify(auditPath));
-
-      out.flush();
-      return EXIT_SUCCESS;
-    }
-  }
-
   private static S3GuardTool command;
 
   /**
@@ -1980,10 +819,9 @@ protected static URI toUri(String s3Path) {
   private static void printHelp() {
     if (command == null) {
       errorln("Usage: hadoop " + USAGE);
-      errorln("\tperform S3Guard metadata store " +
-          "administrative commands.");
+      errorln("\tperform S3A connector administrative commands.");
     } else {
-      errorln("Usage: hadoop " + command.getUsage());
+      errorln("Usage: hadoop " + ENTRY_POINT + command.getUsage());
     }
     errorln();
     errorln(COMMON_USAGE);
@@ -2010,29 +848,12 @@ protected static void println(PrintStream out,
   }
 
   /**
-   * Retrieve and Print store diagnostics.
-   * @param out output stream
-   * @param store store
-   * @throws IOException Failure to retrieve the data.
-   */
-  protected static void printStoreDiagnostics(PrintStream out,
-      MetadataStore store)
-      throws IOException {
-    Map<String, String> diagnostics = store.getDiagnostics();
-    out.println("Metadata Store Diagnostics:");
-    for (Map.Entry<String, String> entry : diagnostics.entrySet()) {
-      println(out, "\t%s=%s", entry.getKey(), entry.getValue());
-    }
-  }
-
-
-  /**
-   * Handle store not found by converting to an exit exception
+   * Handle FileNotFoundException by converting to an exit exception
    * with specific error code.
    * @param e exception
    * @return a new exception to throw
    */
-  protected static ExitUtil.ExitException storeNotFound(
+  protected static ExitUtil.ExitException notFound(
       FileNotFoundException e) {
     return new ExitUtil.ExitException(
         E_NOT_FOUND, e.toString(), e);
@@ -2045,7 +866,7 @@ protected static ExitUtil.ExitException storeNotFound(
    * @return a new exception to throw
    */
   protected static ExitUtil.ExitException invalidArgs(
-      String format, Object...args) {
+      String format, Object... args) {
     return exitException(INVALID_ARGUMENT, format, args);
   }
 
@@ -2056,9 +877,16 @@ protected static ExitUtil.ExitException invalidArgs(
    * @return a new exception to throw
    */
   protected static ExitUtil.ExitException badState(
-      String format, Object...args) {
-    int exitCode = E_BAD_STATE;
-    return exitException(exitCode, format, args);
+      String format, Object... args) {
+    return exitException(E_BAD_STATE, format, args);
+  }
+
+  /**
+   * Crate an exception declaring S3Guard is unsupported.
+   * @return an exception raise.
+   */
+  protected static ExitUtil.ExitException s3guardUnsupported() {
+    throw exitException(E_S3GUARD_UNSUPPORTED, E_UNSUPPORTED);
   }
 
   /**
@@ -2068,7 +896,7 @@ protected static ExitUtil.ExitException badState(
    * @return a new exception to throw
    */
   protected static ExitUtil.ExitException userAborted(
-      String format, Object...args) {
+      String format, Object... args) {
     return exitException(ERROR, format, args);
   }
 
@@ -2095,7 +923,7 @@ protected static ExitUtil.ExitException exitException(
    * @return exit code.
    * @throws Exception on I/O errors.
    */
-  public static int run(Configuration conf, String...args) throws
+  public static int run(Configuration conf, String... args) throws
       Exception {
     /* ToolRunner.run does this too, but we must do it before looking at
     subCommand or instantiating the cmd object below */
@@ -2107,31 +935,17 @@ public static int run(Configuration conf, String...args) throws
     }
     final String subCommand = otherArgs[0];
     LOG.debug("Executing command {}", subCommand);
+    // if it is no longer supported: raise an exception
+    if (UNSUPPORTED_COMMANDS.contains(subCommand)) {
+      throw s3guardUnsupported();
+    }
     switch (subCommand) {
-    case Init.NAME:
-      command = new Init(conf);
-      break;
-    case Destroy.NAME:
-      command = new Destroy(conf);
-      break;
-    case Import.NAME:
-      command = new Import(conf);
-      break;
     case BucketInfo.NAME:
       command = new BucketInfo(conf);
       break;
-    case Diff.NAME:
-      command = new Diff(conf);
-      break;
     case MarkerTool.MARKERS:
       command = new MarkerTool(conf);
       break;
-    case Prune.NAME:
-      command = new Prune(conf);
-      break;
-    case SetCapacity.NAME:
-      command = new SetCapacity(conf);
-      break;
     case Uploads.NAME:
       command = new Uploads(conf);
       break;
@@ -2140,12 +954,6 @@ public static int run(Configuration conf, String...args) throws
       // because this is the defacto S3 CLI.
       command = new SelectTool(conf);
       break;
-    case Fsck.NAME:
-      command = new Fsck(conf);
-      break;
-    case Authoritative.NAME:
-      command = new Authoritative(conf);
-      break;
     default:
       printHelp();
       throw new ExitUtil.ExitException(E_USAGE,
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
index d4303150d1d35..89c0300cd91f2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/package-info.java
@@ -17,10 +17,9 @@
  */
 
 /**
- * This package contains classes related to S3Guard: a feature of S3A to mask
- * the eventual consistency behavior of S3 and optimize access patterns by
- * coordinating with a strongly consistent external store for file system
- * metadata.
+ * This package contained S3Guard support; now the feature has been removed,
+ * its contents are limited to the public command line and some
+ * methods still used by directory marker code.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/InternalSelectConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/InternalSelectConstants.java
index 4f387d8ccfa75..fbf5226afb82f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/InternalSelectConstants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/InternalSelectConstants.java
@@ -71,7 +71,7 @@ private InternalSelectConstants() {
         CSV_OUTPUT_QUOTE_FIELDS,
         CSV_OUTPUT_RECORD_DELIMITER
     ));
-    options.addAll(InternalConstants.STANDARD_OPENFILE_KEYS);
+    options.addAll(InternalConstants.S3A_OPENFILE_KEYS);
     SELECT_OPTIONS = Collections.unmodifiableSet(options);
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java
index 150043aea9f1a..9c79cc1004ce1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectBinding.java
@@ -28,7 +28,7 @@
 import com.amazonaws.services.s3.model.OutputSerialization;
 import com.amazonaws.services.s3.model.QuoteFields;
 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,7 +42,7 @@
 import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.hadoop.fs.s3a.select.SelectConstants.*;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java
index 02d1e53c7ba81..f6ae52eba5346 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectInputStream.java
@@ -28,7 +28,7 @@
 import com.amazonaws.services.s3.model.SelectObjectContentEventVisitor;
 import com.amazonaws.services.s3.model.SelectObjectContentResult;
 import com.amazonaws.services.s3.model.SelectRecordsInputStream;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,7 +44,7 @@
 import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.io.IOUtils;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.hadoop.fs.s3a.Invoker.once;
 import static org.apache.hadoop.fs.s3a.S3AInputStream.validateReadahead;
@@ -337,7 +337,7 @@ public synchronized void seek(long newPos) throws IOException {
 
   /**
    * Build an exception to raise when an operation is not supported here.
-   * @param action action which is unsupported.
+   * @param action action which is Unsupported.
    * @return an exception to throw.
    */
   protected PathIOException unsupported(final String action) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java
index 21c98fd6f8da5..7a6c1afdc1fc3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/select/SelectTool.java
@@ -39,12 +39,12 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool;
 import org.apache.hadoop.fs.shell.CommandFormat;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.OperationDuration;
+import org.apache.hadoop.util.functional.FutureIO;
 
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
@@ -261,10 +261,10 @@ public int run(String[] args, PrintStream out)
     FSDataInputStream stream;
     try(DurationInfo ignored =
             new DurationInfo(LOG, "Selecting stream")) {
-      stream = FutureIOSupport.awaitFuture(builder.build());
+      stream = FutureIO.awaitFuture(builder.build());
     } catch (FileNotFoundException e) {
       // the source file is missing.
-      throw storeNotFound(e);
+      throw notFound(e);
     }
     try {
       if (toConsole) {
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java
index 772b965d4f4a3..bd1466b2a432f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java
@@ -25,7 +25,8 @@
  * Block output stream statistics.
  */
 public interface BlockOutputStreamStatistics extends Closeable,
-    S3AStatisticInterface {
+    S3AStatisticInterface,
+    PutTrackerStatistics {
 
   /**
    * Block is queued for upload.
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java
index fd232a058d0b8..53c25b0c65aea 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.s3a.statistics;
 
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+
 /**
  * Statistics for S3A committers.
  */
@@ -63,4 +65,10 @@ public interface CommitterStatistics
    * @param success success flag
    */
   void jobCompleted(boolean success);
+
+  /**
+   * Return the writeable IOStatisticsStore.
+   * @return the statistics
+   */
+  IOStatisticsStore getIOStatistics();
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java
new file mode 100644
index 0000000000000..b422e0c9d53aa
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.statistics;
+
+/**
+ * Interface for PUT tracking.
+ * It is subclassed by {@link BlockOutputStreamStatistics},
+ * so that operations performed by the PutTracker update
+ * the stream statistics.
+ * Having a separate interface helps isolate operations.
+ */
+public interface PutTrackerStatistics extends S3AStatisticInterface {
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java
index 328d9f7c4ce4a..69f2c279ddaa8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.s3a.statistics;
 
+import org.apache.hadoop.fs.impl.prefetch.PrefetchingStatistics;
 import org.apache.hadoop.fs.statistics.DurationTracker;
 
 /**
@@ -26,7 +27,7 @@
  * It also contains getters for tests.
  */
 public interface S3AInputStreamStatistics extends AutoCloseable,
-    S3AStatisticInterface {
+    S3AStatisticInterface, PrefetchingStatistics {
 
   /**
    * Seek backwards, incrementing the seek and backward seek counters.
@@ -96,6 +97,20 @@ void streamClose(boolean abortedConnection,
    */
   void readOperationCompleted(int requested, int actual);
 
+  /**
+   * A vectored read operation has started..
+   * @param numIncomingRanges number of input ranges.
+   * @param numCombinedRanges number of combined ranges.
+   */
+  void readVectoredOperationStarted(int numIncomingRanges,
+                                    int numCombinedRanges);
+
+  /**
+   * Number of bytes discarded during vectored read.
+   * @param discarded discarded bytes during vectored read.
+   */
+  void readVectoredBytesDiscarded(int discarded);
+
   @Override
   void close();
 
@@ -188,4 +203,11 @@ void streamClose(boolean abortedConnection,
    */
   DurationTracker initiateGetRequest();
 
+  /**
+   * Initiate a stream close/abort.
+   * @param abort was the stream aborted?
+   * @return duration tracker;
+   */
+  DurationTracker initiateInnerStreamClose(boolean abort);
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AStatisticsContext.java
index 27f1398d4ea51..5378c41edf70d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AStatisticsContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AStatisticsContext.java
@@ -18,19 +18,11 @@
 
 package org.apache.hadoop.fs.s3a.statistics;
 
-import org.apache.hadoop.fs.s3a.s3guard.MetastoreInstrumentation;
-
 /**
  * This is the statistics context for ongoing operations in S3A.
  */
 public interface S3AStatisticsContext extends CountersAndGauges {
 
-  /**
-   * Get the metastore instrumentation.
-   * @return an instance of the metastore statistics tracking.
-   */
-  MetastoreInstrumentation getS3GuardInstrumentation();
-
   /**
    * Create a stream input statistics instance.
    * @return the new instance
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/BondedS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/BondedS3AStatisticsContext.java
index 51bb4afebc4ff..1ad4f3ff68baa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/BondedS3AStatisticsContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/BondedS3AStatisticsContext.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.s3a.S3AInstrumentation;
 import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.s3guard.MetastoreInstrumentation;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
 import org.apache.hadoop.fs.s3a.statistics.DelegationTokenStatistics;
@@ -94,16 +93,6 @@ private FileSystem.Statistics getInstanceStatistics() {
     return statisticsSource.getInstanceStatistics();
   }
 
-  /**
-   * Get a MetastoreInstrumentation getInstrumentation() instance for this
-   * context.
-   * @return the S3Guard getInstrumentation() point.
-   */
-  @Override
-  public MetastoreInstrumentation getS3GuardInstrumentation() {
-    return getInstrumentation().getS3GuardInstrumentation();
-  }
-
   /**
    * Create a stream input statistics instance.
    * The FileSystem.Statistics instance of the {@link #statisticsSource}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java
index 3a651026a0ba6..d10b6484175b1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java
@@ -22,8 +22,6 @@
 import java.time.Duration;
 
 import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.s3guard.MetastoreInstrumentation;
-import org.apache.hadoop.fs.s3a.s3guard.MetastoreInstrumentationImpl;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.ChangeTrackerStatistics;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
@@ -35,6 +33,7 @@
 import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 
 import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatistics;
 import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTracker;
@@ -49,9 +48,6 @@
  */
 public final class EmptyS3AStatisticsContext implements S3AStatisticsContext {
 
-  public static final MetastoreInstrumentation
-      METASTORE_INSTRUMENTATION = new MetastoreInstrumentationImpl();
-
   public static final S3AInputStreamStatistics
       EMPTY_INPUT_STREAM_STATISTICS = new EmptyInputStreamStatistics();
 
@@ -69,11 +65,6 @@ public final class EmptyS3AStatisticsContext implements S3AStatisticsContext {
   public static final StatisticsFromAwsSdk
       EMPTY_STATISTICS_FROM_AWS_SDK = new EmptyStatisticsFromAwsSdk();
 
-  @Override
-  public MetastoreInstrumentation getS3GuardInstrumentation() {
-    return METASTORE_INSTRUMENTATION;
-  }
-
   @Override
   public S3AInputStreamStatistics newInputStreamStatistics() {
     return EMPTY_INPUT_STREAM_STATISTICS;
@@ -147,6 +138,7 @@ private static class EmptyS3AStatisticImpl implements
     public DurationTracker trackDuration(String key, long count) {
       return stubDurationTracker();
     }
+
   }
 
   /**
@@ -203,6 +195,17 @@ public void readOperationCompleted(final int requested, final int actual) {
 
     }
 
+    @Override
+    public void readVectoredOperationStarted(int numIncomingRanges,
+                                             int numCombinedRanges) {
+
+    }
+
+    @Override
+    public void readVectoredBytesDiscarded(int discarded) {
+
+    }
+
     @Override
     public void close() {
 
@@ -218,6 +221,41 @@ public void unbuffered() {
 
     }
 
+    @Override
+    public DurationTracker prefetchOperationStarted() {
+      return stubDurationTracker();
+    }
+
+    @Override
+    public void prefetchOperationCompleted() {
+
+    }
+
+    @Override
+    public void blockAddedToFileCache() {
+
+    }
+
+    @Override
+    public void blockRemovedFromFileCache() {
+
+    }
+
+    @Override
+    public void executorAcquired(Duration timeInQueue) {
+
+    }
+
+    @Override
+    public void memoryAllocated(int size) {
+
+    }
+
+    @Override
+    public void memoryFreed(int size) {
+
+    }
+
     /**
      * Return an IO statistics instance.
      * @return an empty IO statistics instance.
@@ -347,6 +385,11 @@ public DurationTracker initiateGetRequest() {
       return stubDurationTracker();
     }
 
+    @Override
+    public DurationTracker initiateInnerStreamClose(final boolean abort) {
+      return stubDurationTracker();
+    }
+
   }
 
   /**
@@ -387,6 +430,11 @@ public void taskCompleted(final boolean success) {
     @Override
     public void jobCompleted(final boolean success) {
     }
+
+    @Override
+    public IOStatisticsStore getIOStatistics() {
+      return null;
+    }
   }
 
   private static final class EmptyBlockOutputStreamStatistics
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
index dd760cfd97ab0..4ddc5f9478bb1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
@@ -36,7 +36,7 @@
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,10 +73,7 @@
 import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
 
 /**
- * Audit and S3 bucket for directory markers.
- * <p></p>
- * This tool does not go anywhere near S3Guard; its scan bypasses any
- * metastore as we are explicitly looking for marker objects.
+ * Audit an S3 bucket for directory markers.
  */
 @InterfaceAudience.LimitedPrivate("management tools")
 @InterfaceStability.Unstable
@@ -150,7 +147,7 @@ public final class MarkerTool extends S3GuardTool {
   /**
    * Constant to use when there is no limit on the number of
    * objects listed: {@value}.
-   * <p></p>
+   * <p>
    * The value is 0 and not -1 because it allows for the limit to be
    * set on the command line {@code -limit 0}.
    * The command line parser rejects {@code -limit -1} as the -1
@@ -478,17 +475,23 @@ public String toString() {
           '}';
     }
 
-    /** Exit code to report. */
+    /**
+     * @return Exit code to report.
+     */
     public int getExitCode() {
       return exitCode;
     }
 
-    /** Tracker which did the scan. */
+    /**
+     * @return Tracker which did the scan.
+     */
     public DirMarkerTracker getTracker() {
       return tracker;
     }
 
-    /** Summary of purge. Null if none took place. */
+    /**
+     * @return Summary of purge. Null if none took place.
+     */
     public MarkerPurgeSummary getPurgeSummary() {
       return purgeSummary;
     }
@@ -664,7 +667,7 @@ private String suffix(final int size) {
    * @param path path to scan
    * @param tracker tracker to update
    * @param limit limit of files to scan; -1 for 'unlimited'
-   * @return true if the scan completedly scanned the entire tree
+   * @return true if the scan completely scanned the entire tree
    * @throws IOException IO failure
    */
   @Retries.RetryTranslated
@@ -818,10 +821,9 @@ pages, suffix(pages),
       int end = Math.min(start + deletePageSize, size);
       List<DeleteObjectsRequest.KeyVersion> page = markerKeys.subList(start,
           end);
-      List<Path> undeleted = new ArrayList<>();
       once("Remove S3 Keys",
           tracker.getBasePath().toString(), () ->
-              operations.removeKeys(page, true, undeleted, null, false));
+              operations.removeKeys(page, true));
       summary.deleteRequests++;
       // and move to the start of the next page
       start = end;
@@ -844,6 +846,7 @@ public void setVerbose(final boolean verbose) {
    * Execute the marker tool, with no checks on return codes.
    *
    * @param scanArgs set of args for the scanner.
+   * @throws IOException IO failure
    * @return the result
    */
   @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
@@ -857,9 +860,9 @@ public static MarkerTool.ScanResult execMarkerTool(
 
   /**
    * Arguments for the scan.
-   * <p></p>
+   * <p>
    * Uses a builder/argument object because too many arguments were
-   * being created and it was making maintenance harder.
+   * being created, and it was making maintenance harder.
    */
   public static final class ScanArgs {
 
@@ -964,43 +967,71 @@ public static final class ScanArgsBuilder {
     /** Consider only markers in nonauth paths as errors. */
     private boolean nonAuth = false;
 
-    /** Source FS; must be or wrap an S3A FS. */
+    /**
+     * Source FS; must be or wrap an S3A FS.
+     * @param source Source FileSystem
+     * @return the builder class after scanning source FS
+     */
     public ScanArgsBuilder withSourceFS(final FileSystem source) {
       this.sourceFS = source;
       return this;
     }
 
-    /** Path to scan. */
+    /**
+     * Path to scan.
+     * @param p path to scan
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withPath(final Path p) {
       this.path = p;
       return this;
     }
 
-    /** Purge? */
+    /**
+     * Should the markers be purged? This is also enabled when using the clean flag on the CLI.
+     * @param d set to purge if true
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withDoPurge(final boolean d) {
       this.doPurge = d;
       return this;
     }
 
-    /** Min marker count (ignored on purge). */
+    /**
+     * Min marker count an audit must find (ignored on purge).
+     * @param min Minimum Marker Count (default 0)
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withMinMarkerCount(final int min) {
       this.minMarkerCount = min;
       return this;
     }
 
-    /** Max marker count (ignored on purge). */
+    /**
+     * Max marker count an audit must find (ignored on purge).
+     * @param max Maximum Marker Count (default 0)
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withMaxMarkerCount(final int max) {
       this.maxMarkerCount = max;
       return this;
     }
 
-    /** Limit of files to scan; 0 for 'unlimited'. */
+    /**
+     * Limit of files to scan; 0 for 'unlimited'.
+     * @param l Limit of files to scan
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withLimit(final int l) {
       this.limit = l;
       return this;
     }
 
-    /** Consider only markers in nonauth paths as errors. */
+    /**
+     * Consider only markers in non-authoritative paths as errors.
+     * @param b True if tool should only consider markers in non-authoritative paths
+     * @return builder class for method chaining
+     */
     public ScanArgsBuilder withNonAuth(final boolean b) {
       this.nonAuth = b;
       return this;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java
index 9ab7636d6c99f..a701f86f7b0c3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperations.java
@@ -23,7 +23,6 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 
 import org.apache.hadoop.fs.InvalidRequestException;
@@ -31,7 +30,6 @@
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.s3a.Retries;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 
 /**
  * Operations which must be offered by the store for {@link MarkerTool}.
@@ -41,8 +39,7 @@
 public interface MarkerToolOperations {
 
   /**
-   * Create an iterator over objects in S3 only; S3Guard
-   * is not involved.
+   * Create an iterator over objects in S3.
    * The listing includes the key itself, if found.
    * @param path  path of the listing.
    * @param key object key
@@ -56,21 +53,11 @@ RemoteIterator<S3AFileStatus> listObjects(
       throws IOException;
 
   /**
-   * Remove keys from the store, updating the metastore on a
-   * partial delete represented as a MultiObjectDeleteException failure by
-   * deleting all those entries successfully deleted and then rethrowing
-   * the MultiObjectDeleteException.
+   * Remove keys from the store.
    * @param keysToDelete collection of keys to delete on the s3-backend.
    *        if empty, no request is made of the object store.
    * @param deleteFakeDir indicates whether this is for deleting fake dirs.
-   * @param undeletedObjectsOnFailure List which will be built up of all
-   * files that were not deleted. This happens even as an exception
-   * is raised.
-   * @param operationState bulk operation state
-   * @param quiet should a bulk query be quiet, or should its result list
    * all deleted keys
-   * @return the deletion result if a multi object delete was invoked
-   * and it returned without a failure, else null.
    * @throws InvalidRequestException if the request was rejected due to
    * a mistaken attempt to delete the root directory.
    * @throws MultiObjectDeleteException one or more of the keys could not
@@ -79,12 +66,9 @@ RemoteIterator<S3AFileStatus> listObjects(
    * @throws IOException other IO Exception.
    */
   @Retries.RetryMixed
-  DeleteObjectsResult removeKeys(
+  void removeKeys(
       List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean deleteFakeDir,
-      List<Path> undeletedObjectsOnFailure,
-      BulkOperationState operationState,
-      boolean quiet)
+      boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException,
              IOException;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java
index d14bb6b1d8ebb..ccf80e1dde00e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerToolOperationsImpl.java
@@ -23,14 +23,12 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 
 /**
  * Implement the marker tool operations by forwarding to the
@@ -56,15 +54,12 @@ public RemoteIterator<S3AFileStatus> listObjects(final Path path,
   }
 
   @Override
-  public DeleteObjectsResult removeKeys(
+  public void removeKeys(
       final List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      final boolean deleteFakeDir,
-      final List<Path> undeletedObjectsOnFailure,
-      final BulkOperationState operationState,
-      final boolean quiet)
+      final boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException, IOException {
-    return operationCallbacks.removeKeys(keysToDelete, deleteFakeDir,
-        undeletedObjectsOnFailure, operationState, quiet);
+    operationCallbacks.removeKeys(keysToDelete, deleteFakeDir
+    );
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index 89ae3ea1b17c6..b1a8c96880237 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -23,7 +23,7 @@
 import java.util.Objects;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-aws/src/main/shellprofile.d/hadoop-s3guard.sh b/hadoop-tools/hadoop-aws/src/main/shellprofile.d/hadoop-s3guard.sh
index 039b0772e76a1..4a4dc734e7850 100644
--- a/hadoop-tools/hadoop-aws/src/main/shellprofile.d/hadoop-s3guard.sh
+++ b/hadoop-tools/hadoop-aws/src/main/shellprofile.d/hadoop-s3guard.sh
@@ -18,7 +18,7 @@
 if ! declare -f hadoop_subcommand_s3guard >/dev/null 2>/dev/null; then
 
   if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
-    hadoop_add_subcommand "s3guard" client "manage metadata on S3"
+    hadoop_add_subcommand "s3guard" client "S3 Commands"
   fi
 
   # this can't be indented otherwise shelldocs won't get it
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
index 96bd64811aac4..094ea5668c05e 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
@@ -39,7 +39,7 @@ are, how to configure their policies, etc.
 * You need a role to assume, and know its "ARN".
 * You need a pair of long-lived IAM User credentials, not the root account set.
 * Have the AWS CLI installed, and test that it works there.
-* Give the role access to S3, and, if using S3Guard, to DynamoDB.
+* Give the role access to S3.
 * For working with data encrypted with SSE-KMS, the role must
 have access to the appropriate KMS keys.
 
@@ -80,7 +80,7 @@ and for background refreshes, a different credential provider must be
 created, one which uses long-lived credentials (secret keys, environment variables).
 Short lived credentials (e.g other session tokens, EC2 instance credentials) cannot be used.
 
-A list of providers can be set in `s.s3a.assumed.role.credentials.provider`;
+A list of providers can be set in `fs.s3a.assumed.role.credentials.provider`;
 if unset the standard `BasicAWSCredentialsProvider` credential provider is used,
 which uses `fs.s3a.access.key` and `fs.s3a.secret.key`.
 
@@ -234,9 +234,6 @@ s3:Get*
 s3:ListBucket
 ```
 
-When using S3Guard, the client needs the appropriate
-<a href="s3guard-permissions">DynamoDB access permissions</a>
-
 To use SSE-KMS encryption, the client needs the
 <a href="sse-kms-permissions">SSE-KMS Permissions</a> to access the
 KMS key(s).
@@ -277,47 +274,6 @@ If the caller doesn't have these permissions, the operation will fail with an
 `AccessDeniedException`: the S3 Store does not provide the specifics of
 the cause of the failure.
 
-### <a name="s3guard-permissions"></a> S3Guard Permissions
-
-To use S3Guard, all clients must have a subset of the
-[AWS DynamoDB Permissions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/api-permissions-reference.html).
-
-To work with buckets protected with S3Guard, the client must have
-all the following rights on the DynamoDB Table used to protect that bucket.
-
-```
-dynamodb:BatchGetItem
-dynamodb:BatchWriteItem
-dynamodb:DeleteItem
-dynamodb:DescribeTable
-dynamodb:GetItem
-dynamodb:PutItem
-dynamodb:Query
-dynamodb:UpdateItem
-```
-
-This is true, *even if the client only has read access to the data*.
-
-For the `hadoop s3guard` table management commands, _extra_ permissions are required:
-
-```
-dynamodb:CreateTable
-dynamodb:DescribeLimits
-dynamodb:DeleteTable
-dynamodb:Scan
-dynamodb:TagResource
-dynamodb:UntagResource
-dynamodb:UpdateTable
-```
-
-Without these permissions, tables cannot be created, destroyed or have their IO capacity
-changed through the `s3guard set-capacity` call.
-The `dynamodb:Scan` permission is needed for `s3guard prune`
-
-The `dynamodb:CreateTable` permission is needed by a client when it tries to
-create the DynamoDB table on startup, that is
-`fs.s3a.s3guard.ddb.table.create` is `true` and the table does not already exist.
-
 ### <a name="mixed-permissions"></a> Mixed Permissions in a single S3 Bucket
 
 Mixing permissions down the "directory tree" is limited
@@ -348,10 +304,6 @@ file will exist.
 For a directory copy, only a partial copy of the source data may take place
 before the permission failure is raised.
 
-
-*S3Guard*: if [S3Guard](s3guard.html) is used to manage the directory listings,
-then after partial failures of rename/copy the DynamoDB tables can get out of sync.
-
 ### Example: Read access to the base, R/W to the path underneath
 
 This example has the base bucket read only, and a directory underneath,
@@ -818,29 +770,6 @@ Caused by:  com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Se
 Note: the ability to read encrypted data in the store does not guarantee that the caller can encrypt new data.
 It is a separate permission.
 
-
-### <a name="dynamodb_exception"></a> `AccessDeniedException` + `AmazonDynamoDBException`
-
-```
-java.nio.file.AccessDeniedException: bucket1:
-  com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException:
-  User: arn:aws:sts::980678866538:assumed-role/s3guard-test-role/test is not authorized to perform:
-  dynamodb:DescribeTable on resource: arn:aws:dynamodb:us-west-1:980678866538:table/bucket1
-   (Service: AmazonDynamoDBv2; Status Code: 400;
-```
-
-The caller is trying to access an S3 bucket which uses S3Guard, but the caller
-lacks the relevant DynamoDB access permissions.
-
-The `dynamodb:DescribeTable` operation is the first one used in S3Guard to access,
-the DynamoDB table, so it is often the first to fail. It can be a sign
-that the role has no permissions at all to access the table named in the exception,
-or just that this specific permission has been omitted.
-
-If the role policy requested for the assumed role didn't ask for any DynamoDB
-permissions, this is where all attempts to work with a S3Guarded bucket will
-fail. Check the value of `fs.s3a.assumed.role.policy`
-
 ### Error `Unable to execute HTTP request`
 
 This is a low-level networking error. Possible causes include:
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
index 15a6470c5504c..8ccc36cf83bb1 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
@@ -22,6 +22,20 @@ and inside the AWS S3 SDK, immediately before the request is executed.
 The full architecture is covered in [Auditing Architecture](auditing_architecture.html);
 this document covers its use.
 
+## Important: Auditing is disabled by default
+
+Due to a memory leak from the use of `ThreadLocal` fields, this auditing feature
+leaked memory as S3A filesystem instances were created and deleted.
+This caused problems in long-lived processes which either do not re-use filesystem
+instances, or attempt to delete all instances belonging to specific users.
+See [HADOOP-18091](https://issues.apache.org/jira/browse/HADOOP-18091) _S3A auditing leaks memory through ThreadLocal references_.
+
+To avoid these memory leaks, auditing was disabled by default in the hadoop 3.3.2 release.
+
+As these memory leaks have now been fixed, auditing has been re-enabled.
+
+To disable it, set `fs.s3a.audit.enabled` to `false`.
+
 ## Auditing workflow
 
 1. An _Auditor Service_ can be instantiated for each S3A FileSystem instance,
@@ -63,12 +77,16 @@ ideally even identifying the process/job generating load.
 
 ## Using Auditing
 
-The Logging Auditor is enabled by default; it annotates the S3 logs.
+Auditing is disabled by default.
+When auditing enabled, a Logging Auditor will annotate the S3 logs through a custom
+HTTP Referrer header in requests made to S3.
+Other auditor classes may be used instead.
 
 ### Auditor Options
 
 | Option | Meaning | Default Value |
 |--------|---------|---------------|
+| `fs.s3a.audit.enabled` | Is auditing enabled? | `true` |
 | `fs.s3a.audit.service.classname` | Auditor classname | `org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor` |
 | `fs.s3a.audit.request.handlers` | List of extra subclasses of AWS SDK RequestHandler2 to include in handler chain | `""` |
 | `fs.s3a.audit.referrer.enabled` | Logging auditor to publish the audit information in the HTTP Referrer header | `true` |
@@ -76,14 +94,26 @@ The Logging Auditor is enabled by default; it annotates the S3 logs.
 | `fs.s3a.audit.reject.out.of.span.operations` | Auditor to reject operations "outside of a span" | `false` |
 
 
-### Disabling Auditing with the No-op Auditor
+### Disabling Auditing.
 
-The No-op auditor does not perform any logging of audit events.
+In this release of Hadoop, auditing is disabled.
+
+This can be explicitly set globally or for specific buckets
 
 ```xml
 <property>
-  <name>fs.s3a.audit.service.classname</name>
-  <value>org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor</value>
+  <name>fs.s3a.audit.enabled</name>
+  <value>false</value>
+</property>
+```
+
+Specific buckets can have auditing disabled, even when it is enabled globally.
+
+```xml
+<property>
+  <name>fs.s3a.bucket.landsat-pds.audit.enabled</name>
+  <value>false</value>
+  <description>Do not audit landsat bucket operations</description>
 </property>
 ```
 
@@ -92,20 +122,24 @@ The No-op auditor does not perform any logging of audit events.
 The "Logging Auditor" is the default auditor.
 It provides two forms of logging
 
-1. Logging of operations in the client via Log4J.
+1. Logging of operations in the client via the active SLF4J imolementation.
 1. Dynamic generation of the HTTP Referrer header for S3 requests.
 
 The Logging Auditor is enabled by providing its classname in the option
 `fs.s3a.audit.service.classname`.
 
 ```xml
+<property>
+  <name>fs.s3a.audit.enabled</name>
+  <value>true</value>
+</property>
+
 <property>
   <name>fs.s3a.audit.service.classname</name>
   <value>org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor</value>
 </property>
 ```
 
-
 To print auditing events in the local client logs, set the associated Log4J log
 to log at debug:
 
@@ -192,18 +226,25 @@ If any of the field values were `null`, the field is omitted.
 | `cm` | Command | `S3GuardTool$BucketInfo` |
 | `fs` | FileSystem ID | `af5943a9-b6f6-4eec-9c58-008982fc492a` |
 | `id` | Span ID | `3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3` |
-| `ji` | Job ID | `(Generated by query engine)` |
+| `ji` | Job ID  (S3A committer)| `(Generated by query engine)` |
 | `op` | Filesystem API call | `op_rename` |
 | `p1` | Path 1 of operation | `s3a://alice-london/path1` |
 | `p2` | Path 2 of operation | `s3a://alice-london/path2` |
 | `pr` | Principal | `alice` |
 | `ps` | Unique process UUID | `235865a0-d399-4696-9978-64568db1b51c` |
+| `ta` | Task Attempt ID (S3A committer) | |
 | `t0` | Thread 0: thread span was created in | `100` |
 | `t1` | Thread 1: thread this operation was executed in | `200` |
 | `ts` | Timestamp (UTC epoch millis) | `1617116985923` |
 
+_Notes_
 
-Thread IDs are from the current thread in the JVM.
+* Thread IDs are from the current thread in the JVM, so can be compared to those in`````````
+  Log4J logs. They are never unique.
+* Task Attempt/Job IDs are only ever set during operations involving the S3A committers, specifically
+  all operations excecuted by the committer.
+  Operations executed in the same thread as the committer's instantiation _may_ also report the
+  IDs, even if they are unrelated to the actual task. Consider them "best effort".
 
 ```java
 Long.toString(Thread.currentThread().getId())
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md
index 50e8fe79fc669..aa35fae92cebf 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md
@@ -117,7 +117,80 @@ the auditor is bound to.
 
 The auditor then creates and returns a span for the specific operation.
 The AuditManagerS3A will automatically activate the span returned by the auditor
-(i.e. assign it the thread local variable tracking the active span in each thread)
+(i.e. assign it the thread local variable tracking the active span in each thread).
+
+### Memory Leakage through `ThreadLocal` misuse
+
+The original implementation of the integration with the S3AFileSystem class
+contained a critical defect,
+[HADOOP-18091](https://issues.apache.org/jira/browse/HADOOP-18091) _S3A auditing leaks memory through ThreadLocal references_.
+
+The original code was written with the assumption that when the `ActiveAuditManagerS3A` service was
+garbage collected, references in its `ThreadLocal` field would be freed.
+In fact, they are retained until all threads with references are terminated.
+If any long-lived thread had performed an s3 operation which created a span,
+a reference back to the audit manager instance was created
+*whose lifetime was that of the thread*
+
+In short-lived processes, and long-lived processes where a limited set of
+`S3AFileSystem` instances were reused, this had no adverse effect.
+Indeed, if the filesystem instances were retained in the cache until
+the process was shut down, there would be strong references to the owning
+`S3AFileSystem` instance anyway.
+
+Where it did have problems was when the following conditions were met
+1. Process was long-lived.
+2. Long-lived threads in the process invoked filesystem operations on `s3a://` URLs.
+3. Either `S3AFileSystem` instances were created repeatedly, rather than retrieved
+   from the cache of active instances.
+4. Or, after a query for a specific user was completed,
+   `FileSystem.closeAllForUGI(UserGroupInformation)` was invoked to remove all
+   cached FS instances of that user.
+
+Conditions 1, 2 and 4 are exactly those which long-lived Hive services can
+encounter.
+
+_Auditing was disabled by default until a fix was implemented._
+
+The memory leak has been fixed using a new class `org.apache.hadoop.util.WeakReferenceMap`
+to store a map of thread IDs to active spans. When the S3A filesystem is closed,
+its audit manager service is stopped and all references to spans removed from the
+map of thread ID to span.
+
+Weak References are used for the map so that span references do not consume memory even if
+threads are terminated without resetting the span reference of that thread.
+There is therefore a theoretical risk that if a garbage collection takes place during
+execution of a spanned operation, the reference will be lost.
+
+This is not considered an issue as all bounded entry points into the S3A filesystem
+retain a strong reference to their audit span.
+
+All entry points which return an object which can invoke s3 operations (input and output
+streams, list iterators, etc.) also retain a strong reference to their span, a reference
+they activate before performing S3 operations.
+A factory method is provided to demand-generate a new span if, somehow, these conditions
+are not met. The "unbounded span" is used here.
+Except in deployments where `fs.s3a.audit.reject.out.of.span.operations` is true,
+invoking S3 operations within the unbounded span are permitted.
+That option is set to `true` within S3A test suites.
+Therefore it is unlikely that any operations are invoked in unbounded spans except
+for the special case of copy operations invoked by the transfer manager threads.
+Those are already ignored in the logging auditor, whose unbounded span ignores
+requests which `AWSRequestAnalyzer.isRequestNotAlwaysInSpan()` indicates
+may happen outside of a span.
+This is restricted to bucket location probes possibly performed by the SDK
+on instantiation, and copy part/complete calls.
+
+
+```java
+  public static boolean
+      isRequestNotAlwaysInSpan(final Object request) {
+    return request instanceof CopyPartRequest
+        || request instanceof CompleteMultipartUploadRequest
+        || request instanceof GetBucketLocationRequest;
+  }
+```
+
 
 ### Class `org.apache.hadoop.fs.audit.CommonAuditContext`
 
@@ -138,11 +211,55 @@ Spans may be used on different thread from that which they were created.
 Spans MUST always use the values from the `currentAuditContext()` in the creation
 thread.
 
+#### Memory Usage of `CommonAuditContext`
+
+The `CommonAuditContext` map has a `ThreadLocal` field to store global
+information which is intended to span multiple operations across multiple
+filesystems, for example the MapReduce or Spark job ID, which is set
+in the S3A committers.
+
+Applications and Hadoop code MUST NOT attach context entries
+which directly or indirectly consumes lots of memory, as the life
+of that memory use will become that of the thread.
+
+Applications and Hadoop code SHOULD remove context entries when
+no-longer needed.
+
+If memory leakage is suspected here, set the log
+`org.apache.hadoop.fs.audit.CommonAuditContext` to `TRACE`
+to log the origin of operations which add log entries.
+
+This will produce a log entry whose stack trace will show the caller chain.
+```
+2022-01-26 16:10:28,384 TRACE audit.CommonAuditContext (CommonAuditContext.java:put(149)) - Adding context entry t1
+java.lang.Exception: t1
+    at org.apache.hadoop.fs.audit.CommonAuditContext.put(CommonAuditContext.java:149)
+    at org.apache.hadoop.fs.audit.CommonAuditContext.init(CommonAuditContext.java:192)
+    at org.apache.hadoop.fs.audit.CommonAuditContext.createInstance(CommonAuditContext.java:210)
+    at org.apache.hadoop.fs.audit.CommonAuditContext.lambda$static$0(CommonAuditContext.java:119)
+    at java.lang.ThreadLocal$SuppliedThreadLocal.initialValue(ThreadLocal.java:284)
+    at java.lang.ThreadLocal.setInitialValue(ThreadLocal.java:180)
+    at java.lang.ThreadLocal.get(ThreadLocal.java:170)
+    at org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext(CommonAuditContext.java:219)
+    at org.apache.hadoop.fs.audit.TestCommonAuditContext.<init>(TestCommonAuditContext.java:54)
+```
+
 
 ### class `NoopAuditor`
 
-This auditor creates spans which perform no auditing.
-It is very efficient and reliable.
+This auditor creates spans which doesn't do anything with the events.
+
+```xml
+<property>
+  <name>fs.s3a.audit.service.classname</name>
+  <value>org.apache.hadoop.fs.s3a.audit.impl.NoopAuditor</value>
+</property>
+```
+
+This is *not* the same as disabling auditing, as it still uses the `ActiveAuditManagerS3A` class
+which is the source of memory leaks.
+
+Avoid using it except in tests as there is no benefit -simply significant cost.
 
 ### class `LoggingAuditor`
 
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
new file mode 100644
index 0000000000000..e649a8d76d539
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md
@@ -0,0 +1,76 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Upgrading S3A to AWS SDK V2
+
+This document explains the upcoming work for upgrading S3A to AWS SDK V2.
+This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073).
+
+## Why the upgrade?
+
+- Moving to SDK V2 will provide performance benefits.
+For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/)
+is built using java bindings of the AWS Common Runtime S3
+client (https://github.com/awslabs/aws-crt-java) (CRT).
+CRT is a set of packages written in C, designed for maximising performance when interacting with AWS
+services such as S3.
+- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/)
+which S3A will benefit from are not available in SDKV1.
+
+## What's changing?
+
+The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from
+[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A.
+A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes).
+
+The major changes and how this affects S3A are listed below.
+
+### Package Change
+
+Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes
+were under `com.amazonaws`.
+
+### Credential Providers
+
+- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java)
+has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java).
+- Credential provider class changes: the package and class names of credential providers have
+changed.
+
+The change in interface will mean that custom credential providers will need to be updated to now
+implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`.
+
+Due to change in class names, references to SDK V1 credential providers
+in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers.
+
+### Delegation Tokens
+
+Custom credential providers used in delegation token binding classes will also need to be updated.
+
+### AmazonS3 replaced by S3Client
+
+The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`.
+
+For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2
+`S3Client`.
+
+The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`.
+
+### Signers
+
+Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java)
+has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java).
+
+The change in signers will mean the custom signers will need to be updated to implement the new
+interface.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
index 048f08cf7c0f0..b69be8ae336d3 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
@@ -1818,7 +1818,7 @@ directory on the job commit, so is *very* expensive, and not something which
 we recommend when working with S3.
 
 
-To use a S3Guard committer, it must also be identified as the Parquet committer.
+To use an S3A committer, it must also be identified as the Parquet committer.
 The fact that instances are dynamically instantiated somewhat complicates the process.
 
 In early tests; we can switch committers for ORC output without making any changes
@@ -1928,12 +1928,6 @@ files.
 ### Security Risks of all committers
 
 
-#### Visibility
-
-[Obsolete] If S3Guard is used for storing metadata, then the metadata is visible to
-all users with read access. A malicious user with write access could delete
-entries of newly generated files, so they would not be visible.
-
 
 #### Malicious Serialized Data
 
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
index 6cd847b4abb67..38aea18cad1fd 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
@@ -51,7 +51,7 @@ obsolete.
 ## Introduction: The Commit Problem
 
 Apache Hadoop MapReduce (and behind the scenes, Apache Spark) often write
-the output of their work to filesystems
+the output of their work to filesystems.
 
 Normally, Hadoop uses the `FileOutputFormatCommitter` to manage the
 promotion of files created in a single task attempt to the final output of
@@ -68,37 +68,37 @@ process across the cluster may rename a file or directory to the same path.
 If the rename fails for any reason, either the data is at the original location,
 or it is at the destination, -in which case the rename actually succeeded.
 
-**The S3 object store and the `s3a://` filesystem client cannot meet these requirements.*
+_The S3 object store and the `s3a://` filesystem client cannot meet these requirements._
 
-Although S3A is (now) consistent, the S3A client still mimics `rename()`
+Although S3 is (now) consistent, the S3A client still mimics `rename()`
 by copying files and then deleting the originals.
 This can fail partway through, and there is nothing to prevent any other process
 in the cluster attempting a rename at the same time.
 
 As a result,
 
-* If a rename fails, the data is left in an unknown state.
+* If a 'rename' fails, the data is left in an unknown state.
 * If more than one process attempts to commit work simultaneously, the output
 directory may contain the results of both processes: it is no longer an exclusive
 operation.
-*. Commit time is still
-proportional to the amount of data created. It still can't handle task failure.
+* Commit time is still proportional to the amount of data created.
+It still can't handle task failure.
 
 **Using the "classic" `FileOutputCommmitter` to commit work to Amazon S3 risks
-loss or corruption of generated data**
+loss or corruption of generated data**.
 
 
-To address these problems there is now explicit support in the `hadop-aws`
-module for committing work to Amazon S3 via the S3A filesystem client,
-*the S3A Committers*
+To address these problems there is now explicit support in the `hadoop-aws`
+module for committing work to Amazon S3 via the S3A filesystem client:
+*the S3A Committers*.
 
 
 For safe, as well as high-performance output of work to S3,
-we need use "a committer" explicitly written to work with S3, treating it as
-an object store with special features.
+we need to use "a committer" explicitly written to work with S3,
+treating it as an object store with special features.
 
 
-### Background : Hadoop's "Commit Protocol"
+### Background: Hadoop's "Commit Protocol"
 
 How exactly is work written to its final destination? That is accomplished by
 a "commit protocol" between the workers and the job manager.
@@ -106,10 +106,10 @@ a "commit protocol" between the workers and the job manager.
 This protocol is implemented in Hadoop MapReduce, with a similar but extended
 version in Apache Spark:
 
-1. A "Job" is the entire query, with inputs to outputs
+1. The "Job" is the entire query. It takes a given set of input and produces some output.
 1. The "Job Manager" is the process in charge of choreographing the execution
 of the job. It may perform some of the actual computation too.
-1. The job has "workers", which are processes which work the actual data
+1. The job has "workers", which are processes which work with the actual data
 and write the results.
 1. Workers execute "Tasks", which are fractions of the job, a job whose
 input has been *partitioned* into units of work which can be executed independently.
@@ -126,7 +126,7 @@ this "speculation" delivers speedup as it can address the "straggler problem".
 When multiple workers are working on the same data, only one worker is allowed
 to write the final output.
 1. The entire job may fail (often from the failure of the Job Manager (MR Master, Spark Driver, ...)).
-1, The network may partition, with workers isolated from each other or
+1. The network may partition, with workers isolated from each other or
 the process managing the entire commit.
 1. Restarted jobs may recover from a failure by reusing the output of all
 completed tasks (MapReduce with the "v1" algorithm), or just by rerunning everything
@@ -137,34 +137,34 @@ What is "the commit protocol" then? It is the requirements on workers as to
 when their data is made visible, where, for a filesystem, "visible" means "can
 be seen in the destination directory of the query."
 
-* There is a destination directory of work, "the output directory."
-* The final output of tasks must be in this directory *or paths underneath it*.
+* There is a destination directory of work: "the output directory".
+The final output of tasks must be in this directory *or paths underneath it*.
 * The intermediate output of a task must not be visible in the destination directory.
 That is: they must not write directly to the destination.
 * The final output of a task *may* be visible under the destination.
-* The Job Manager makes the decision as to whether a task's data is to be "committed",
-be it directly to the final directory or to some intermediate store..
-* Individual workers communicate with the Job manager to manage the commit process:
-whether the output is to be *committed* or *aborted*
+* Individual workers communicate with the Job manager to manage the commit process.
+* The Job Manager makes the decision on if a task's output data is to be "committed",
+be it directly to the final directory or to some intermediate store.
 * When a worker commits the output of a task, it somehow promotes its intermediate work to becoming
 final.
 * When a worker aborts a task's output, that output must not become visible
 (i.e. it is not committed).
 * Jobs themselves may be committed/aborted (the nature of "when" is not covered here).
 * After a Job is committed, all its work must be visible.
-* And a file `_SUCCESS` may be written to the output directory.
+A file named `_SUCCESS` may be written to the output directory.
 * After a Job is aborted, all its intermediate data is lost.
 * Jobs may also fail. When restarted, the successor job must be able to clean up
 all the intermediate and committed work of its predecessor(s).
 * Task and Job processes measure the intervals between communications with their
 Application Master and YARN respectively.
-When the interval has grown too large they must conclude
+When the interval has grown too large, they must conclude
 that the network has partitioned and that they must abort their work.
 
 
 That's "essentially" it. When working with HDFS and similar filesystems,
 directory `rename()` is the mechanism used to commit the work of tasks and
 jobs.
+
 * Tasks write data to task attempt directories under the directory `_temporary`
 underneath the final destination directory.
 * When a task is committed, these files are renamed to the destination directory
@@ -180,20 +180,19 @@ and restarting the job.
 whose output is in the job attempt directory, *and only rerunning all uncommitted tasks*.
 
 
-This algorithm does not works safely or swiftly with AWS S3 storage because 
-tenames go from being fast, atomic operations to slow operations which can fail partway through.
+This algorithm does not work safely or swiftly with AWS S3 storage because
+renames go from being fast, atomic operations to slow operations which can fail partway through.
 
 This then is the problem which the S3A committers address:
-
-*How to safely and reliably commit work to Amazon S3 or compatible object store*
+*How to safely and reliably commit work to Amazon S3 or compatible object store.*
 
 
 ## Meet the S3A Committers
 
 Since Hadoop 3.1, the S3A FileSystem has been accompanied by classes
-designed to integrate with the Hadoop and Spark job commit protocols, classes
-which interact with the S3A filesystem to reliably commit work work to S3:
-*The S3A Committers*
+designed to integrate with the Hadoop and Spark job commit protocols,
+classes which interact with the S3A filesystem to reliably commit work to S3:
+*The S3A Committers*.
 
 The underlying architecture of this process is very complex, and
 covered in [the committer architecture documentation](./committer_architecture.html).
@@ -219,8 +218,8 @@ conflict with existing files is resolved.
 
 | feature | staging | magic |
 |--------|---------|---|
-| task output destination | local disk | S3A *without completing the write* |
-| task commit process | upload data from disk to S3 | list all pending uploads on s3 and write details to job attempt directory |
+| task output destination | write to local disk | upload to S3 *without completing the write* |
+| task commit process | upload data from disk to S3 *without completing the write* | list all pending uploads on S3 and write details to job attempt directory |
 | task abort process | delete local disk data | list all pending uploads and abort them |
 | job commit | list & complete pending uploads | list & complete pending uploads |
 
@@ -228,33 +227,30 @@ The other metric is "maturity". There, the fact that the Staging committers
 are based on Netflix's production code counts in its favor.
 
 
-### The Staging Committer
+### The Staging Committers
 
-This is based on work from Netflix. It "stages" data into the local filesystem.
-It also requires the cluster to have HDFS, so that
+This is based on work from Netflix.
+It "stages" data into the local filesystem, using URLs with `file://` schemas.
 
-Tasks write to URLs with `file://` schemas. When a task is committed,
-its files are listed, uploaded to S3 as incompleted Multipart Uploads.
+When a task is committed, its files are listed and uploaded to S3 as incomplete Multipart Uploads.
 The information needed to complete the uploads is saved to HDFS where
 it is committed through the standard "v1" commit algorithm.
 
 When the Job is committed, the Job Manager reads the lists of pending writes from its
 HDFS Job destination directory and completes those uploads.
 
-Canceling a task is straightforward: the local directory is deleted with
-its staged data. Canceling a job is achieved by reading in the lists of
+Canceling a _task_ is straightforward: the local directory is deleted with its staged data.
+Canceling a _job_ is achieved by reading in the lists of
 pending writes from the HDFS job attempt directory, and aborting those
 uploads. For extra safety, all outstanding multipart writes to the destination directory
 are aborted.
 
-The staging committer comes in two slightly different forms, with slightly
-different conflict resolution policies:
-
+There are two staging committers with slightly different conflict resolution behaviors:
 
-* **Directory**: the entire directory tree of data is written or overwritten,
+* **Directory Committer**: the entire directory tree of data is written or overwritten,
 as normal.
 
-* **Partitioned**: special handling of partitioned directory trees of the form
+* **Partitioned Committer**: special handling of partitioned directory trees of the form
 `YEAR=2017/MONTH=09/DAY=19`: conflict resolution is limited to the partitions
 being updated.
 
@@ -265,13 +261,16 @@ directories containing new data. It is intended for use with Apache Spark
 only.
 
 
-## Conflict Resolution in the Staging Committers
+#### Conflict Resolution in the Staging Committers
 
 The Staging committers offer the ability to replace the conflict policy
 of the execution engine with policy designed to work with the tree of data.
 This is based on the experience and needs of Netflix, where efficiently adding
 new data to an existing partitioned directory tree is a common operation.
 
+An XML configuration is shown below.
+The default conflict mode if unset would be `append`.
+
 ```xml
 <property>
   <name>fs.s3a.committer.staging.conflict-mode</name>
@@ -283,40 +282,37 @@ new data to an existing partitioned directory tree is a common operation.
 </property>
 ```
 
-**replace** : when the job is committed (and not before), delete files in
+The _Directory Committer_ uses the entire directory tree for conflict resolution.
+For this committer, the behavior of each conflict mode is shown below:
+
+- **replace**: When the job is committed (and not before), delete files in
 directories into which new data will be written.
 
-**fail**: when there are existing files in the destination, fail the job.
+- **fail**: When there are existing files in the destination, fail the job.
 
-**append**: Add new data to the directories at the destination; overwriting
+- **append**: Add new data to the directories at the destination; overwriting
 any with the same name. Reliable use requires unique names for generated files,
 which the committers generate
 by default.
 
-The difference between the two staging committers are as follows:
+The _Partitioned Committer_ calculates the partitions into which files are added,
+the final directories in the tree, and uses that in its conflict resolution process.
+For the _Partitioned Committer_, the behavior of each mode is as follows:
 
-The Directory Committer uses the entire directory tree for conflict resolution.
-If any file exists at the destination it will fail in job setup; if the resolution
-mechanism is "replace" then all existing files will be deleted.
-
-The partitioned committer calculates the partitions into which files are added,
-the final directories in the tree, and uses that in its conflict resolution
-process:
-
-
-**replace** : delete all data in the destination partition before committing
+- **replace**: Delete all data in the destination _partition_ before committing
 the new files.
 
-**fail**: fail if there is data in the destination partition, ignoring the state
+- **fail**: Fail if there is data in the destination _partition_, ignoring the state
 of any parallel partitions.
 
-**append**: add the new data.
+- **append**: Add the new data to the destination _partition_,
+  overwriting any files with the same name.
 
-It's intended for use in Apache Spark Dataset operations, rather
+The _Partitioned Committer_ is intended for use in Apache Spark Dataset operations, rather
 than Hadoop's original MapReduce engine, and only in jobs
 where adding new data to an existing dataset is the desired goal.
 
-Prerequisites for successful work
+Prerequisites for success with the _Partitioned Committer_:
 
 1. The output is written into partitions via `PARTITIONED BY` or `partitionedBy()`
 instructions.
@@ -356,19 +352,20 @@ task commit.
 
 However, it has extra requirements of the filesystem
 
-1. [Obsolete] It requires a consistent object store.
+1. The object store must be consistent.
 1. The S3A client must be configured to recognize interactions
-with the magic directories and treat them specially.
+with the magic directories and treat them as a special case.
 
-Now that Amazon S3 is consistent, the magic committer is enabled by default.
+Now that [Amazon S3 is consistent](https://aws.amazon.com/s3/consistency/),
+the magic directory path rewriting is enabled by default.
 
-It's also not been field tested to the extent of Netflix's committer; consider
-it the least mature of the committers.
+The Magic Committer has not been field tested to the extent of Netflix's committer;
+consider it the least mature of the committers.
 
 
-#### Which Committer to Use?
+### Which Committer to Use?
 
-1. If you want to create or update existing partitioned data trees in Spark, use thee
+1. If you want to create or update existing partitioned data trees in Spark, use the
 Partitioned Committer. Make sure you have enough hard disk capacity for all staged data.
 Do not use it in other situations.
 
@@ -398,8 +395,8 @@ This is done in `mapred-default.xml`
 </property>
 ```
 
-What is missing is an explicit choice of committer to use in the property
-`fs.s3a.committer.name`; so the classic (and unsafe) file committer is used.
+You must also choose which of the S3A committers to use with the `fs.s3a.committer.name` property.
+Otherwise, the classic (and unsafe) file committer is used.
 
 | `fs.s3a.committer.name` |  Committer |
 |--------|---------|
@@ -408,9 +405,7 @@ What is missing is an explicit choice of committer to use in the property
 | `magic` | the "magic" committer |
 | `file` | the original and unsafe File committer; (default) |
 
-
-
-## Using the Directory and Partitioned Staging Committers
+## Using the Staging Committers
 
 Generated files are initially written to a local directory underneath one of the temporary
 directories listed in `fs.s3a.buffer.dir`.
@@ -422,16 +417,14 @@ The staging committer needs a path in the cluster filesystem
 Temporary files are saved in HDFS (or other cluster filesystem) under the path
 `${fs.s3a.committer.staging.tmp.path}/${user}` where `user` is the name of the user running the job.
 The default value of `fs.s3a.committer.staging.tmp.path` is `tmp/staging`,
-Which will be converted at run time to a path under the current user's home directory,
-essentially `~/tmp/staging`
- so the temporary directory
+resulting in the HDFS directory `~/tmp/staging/${user}`.
 
 The application attempt ID is used to create a unique path under this directory,
 resulting in a path `~/tmp/staging/${user}/${application-attempt-id}/` under which
 summary data of each task's pending commits are managed using the standard
 `FileOutputFormat` committer.
 
-When a task is committed the data is uploaded under the destination directory.
+When a task is committed, the data is uploaded under the destination directory.
 The policy of how to react if the destination exists is defined by
 the `fs.s3a.committer.staging.conflict-mode` setting.
 
@@ -442,9 +435,9 @@ the `fs.s3a.committer.staging.conflict-mode` setting.
 | `append` | Add the new files to the existing directory tree |
 
 
-## The "Partitioned" Staging Committer
+### The "Partitioned" Staging Committer
 
-This committer an extension of the "Directory" committer which has a special conflict resolution
+This committer is an extension of the "Directory" committer which has a special conflict resolution
 policy designed to support operations which insert new data into a directory tree structured
 using Hive's partitioning strategy: different levels of the tree represent different columns.
 
@@ -471,10 +464,10 @@ logs/YEAR=2017/MONTH=04/
 A partitioned structure like this allows for queries using Hive or Spark to filter out
 files which do not contain relevant data.
 
-What the partitioned committer does is, where the tooling permits, allows callers
-to add data to an existing partitioned layout*.
+The partitioned committer allows callers to add new data to an existing partitioned layout,
+where the application supports it.
 
-More specifically, it does this by having a conflict resolution options which
+More specifically, it does this by reducing the scope of conflict resolution to
 only act on individual partitions, rather than across the entire output tree.
 
 | `fs.s3a.committer.staging.conflict-mode` | Meaning |
@@ -492,18 +485,18 @@ was written. With the policy of `append`, the new file would be added to
 the existing set of files.
 
 
-### Notes
+### Notes on using Staging Committers
 
 1. A deep partition tree can itself be a performance problem in S3 and the s3a client,
-or, more specifically. a problem with applications which use recursive directory tree
+or more specifically a problem with applications which use recursive directory tree
 walks to work with data.
 
 1. The outcome if you have more than one job trying simultaneously to write data
 to the same destination with any policy other than "append" is undefined.
 
 1. In the `append` operation, there is no check for conflict with file names.
-If, in the example above, the file `log-20170228.avro` already existed,
-it would be overridden. Set `fs.s3a.committer.staging.unique-filenames` to `true`
+If the file `log-20170228.avro` in the example above already existed, it would be overwritten.
+Set `fs.s3a.committer.staging.unique-filenames` to `true`
 to ensure that a UUID is included in every filename to avoid this.
 
 
@@ -514,7 +507,11 @@ performance.
 
 ### FileSystem client setup
 
-1. Turn the magic on by `fs.s3a.committer.magic.enabled"`
+The S3A connector can recognize files created under paths with `__magic/` as a parent directory.
+This allows it to handle those files in a special way, such as uploading to a different location
+and storing the information needed to complete pending multipart uploads.
+
+Turn the magic on by setting `fs.s3a.committer.magic.enabled` to `true`:
 
 ```xml
 <property>
@@ -526,46 +523,36 @@ performance.
 </property>
 ```
 
-
-
 ### Enabling the committer
 
+Set the committer used by S3A's committer factory to `magic`:
+
 ```xml
 <property>
   <name>fs.s3a.committer.name</name>
   <value>magic</value>
 </property>
-
 ```
 
 Conflict management is left to the execution engine itself.
 
-## Common Committer Options
+## Committer Options Reference
 
+### Common S3A Committer Options
+
+The table below provides a summary of each option.
 
 | Option | Meaning | Default |
 |--------|---------|---------|
 | `mapreduce.fileoutputcommitter.marksuccessfuljobs` | Write a `_SUCCESS` file on the successful completion of the job. | `true` |
-| `fs.s3a.buffer.dir` | Local filesystem directory for data being written and/or staged. | `${hadoop.tmp.dir}/s3a` |
-| `fs.s3a.committer.magic.enabled` | Enable "magic committer" support in the filesystem. | `false` |
+| `fs.s3a.buffer.dir` | Local filesystem directory for data being written and/or staged. | `${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a` |
+| `fs.s3a.committer.magic.enabled` | Enable "magic committer" support in the filesystem. | `true` |
 | `fs.s3a.committer.abort.pending.uploads` | list and abort all pending uploads under the destination path when the job is committed or aborted. | `true` |
-| `fs.s3a.committer.threads` | Number of threads in committers for parallel operations on files. | 8 |
+| `fs.s3a.committer.threads` | Number of threads in committers for parallel operations on files.| -4 |
 | `fs.s3a.committer.generate.uuid` | Generate a Job UUID if none is passed down from Spark | `false` |
 | `fs.s3a.committer.require.uuid` |Require the Job UUID to be passed down from Spark | `false` |
 
-
-## Staging committer (Directory and Partitioned) options
-
-
-| Option | Meaning | Default |
-|--------|---------|---------|
-| `fs.s3a.committer.staging.conflict-mode` | Conflict resolution: `fail`, `append` or `replace`| `append` |
-| `fs.s3a.committer.staging.tmp.path` | Path in the cluster filesystem for temporary data. | `tmp/staging` |
-| `fs.s3a.committer.staging.unique-filenames` | Generate unique filenames. | `true` |
-| `fs.s3a.committer.staging.abort.pending.uploads` | Deprecated; replaced by `fs.s3a.committer.abort.pending.uploads`. |  `(false)` |
-
-
-### Common Committer Options
+The examples below shows how these options can be configured in XML.
 
 ```xml
 <property>
@@ -587,10 +574,15 @@ Conflict management is left to the execution engine itself.
 
 <property>
   <name>fs.s3a.committer.threads</name>
-  <value>8</value>
+  <value>-4</value>
   <description>
     Number of threads in committers for parallel operations on files
-    (upload, commit, abort, delete...)
+    (upload, commit, abort, delete...).
+    Two thread pools this size are created, one for the outer
+    task-level parallelism, and one for parallel execution
+    within tasks (POSTs to commit individual uploads)
+    If the value is negative, it is inverted and then multiplied
+    by the number of cores in the CPU.
   </description>
 </property>
 
@@ -623,8 +615,8 @@ Conflict management is left to the execution engine itself.
   <name>fs.s3a.committer.require.uuid</name>
   <value>false</value>
   <description>
-    Should the committer fail to initialize if a unique ID isn't set in
-    "spark.sql.sources.writeJobUUID" or fs.s3a.committer.staging.uuid
+    Require the committer fail to initialize if a unique ID is not set in
+    "spark.sql.sources.writeJobUUID" or "fs.s3a.committer.uuid".
     This helps guarantee that unique IDs for jobs are being
     passed down in spark applications.
   
@@ -645,7 +637,14 @@ Conflict management is left to the execution engine itself.
 </property>
 ```
 
-### Staging Committer Options
+### Staging committer (Directory and Partitioned) options
+
+| Option | Meaning | Default |
+|--------|---------|---------|
+| `fs.s3a.committer.staging.conflict-mode` | Conflict resolution: `fail`, `append`, or `replace`.| `append` |
+| `fs.s3a.committer.staging.tmp.path` | Path in the cluster filesystem for temporary data. | `tmp/staging` |
+| `fs.s3a.committer.staging.unique-filenames` | Generate unique filenames. | `true` |
+| `fs.s3a.committer.staging.abort.pending.uploads` | Deprecated; replaced by `fs.s3a.committer.abort.pending.uploads`. |  `(false)` |
 
 ```xml
 <property>
@@ -667,7 +666,7 @@ Conflict management is left to the execution engine itself.
   <value>true</value>
   <description>
     Option for final files to have a unique name through job attempt info,
-    or the value of fs.s3a.committer.staging.uuid
+    or the value of fs.s3a.committer.uuid.
     When writing data with the "append" conflict option, this guarantees
     that new data will not overwrite any existing data.
   </description>
@@ -691,10 +690,9 @@ The magic committer recognizes when files are created under paths with `__magic/
 and redirects the upload to a different location, adding the information needed to complete the upload
 in the job commit operation.
 
-If, for some reason, you *do not* want these paths to be redirected and not manifest until later,
+If, for some reason, you *do not* want these paths to be redirected and completed later,
 the feature can be disabled by setting `fs.s3a.committer.magic.enabled` to false.
-
-By default it is true.
+By default, it is enabled.
 
 ```xml
 <property>
@@ -706,6 +704,8 @@ By default it is true.
 </property>
 ```
 
+You will not be able to use the Magic Committer if this option is disabled.
+
 ## <a name="concurrent-jobs"></a> Concurrent Jobs writing to the same destination
 
 It is sometimes possible for multiple jobs to simultaneously write to the same destination path.
@@ -725,7 +725,7 @@ be creating files with paths/filenames unique to the specific job.
 It is not enough for them to be unique by task `part-00000.snappy.parquet`,
 because each job will have tasks with the same name, so generate files with conflicting operations.
 
-For the staging committers, setting `fs.s3a.committer.staging.unique-filenames` to ensure unique names are
+For the staging committers, enable `fs.s3a.committer.staging.unique-filenames` to ensure unique names are
 generated during the upload. Otherwise, use what configuration options are available in the specific `FileOutputFormat`.
 
 Note: by default, the option `mapreduce.output.basename` sets the base name for files;
@@ -752,13 +752,12 @@ org.apache.hadoop.fs.s3a.commit.PathCommitException: `s3a://landsat-pds': Filesy
 in configuration option fs.s3a.committer.magic.enabled
 ```
 
-The Job is configured to use the magic committer, but the S3A bucket has not been explicitly
-declared as supporting it.
+The Job is configured to use the magic committer,
+but the S3A bucket has not been explicitly declared as supporting it.
 
-
-This can be done for those buckets which are known to be consistent, either
-because [S3Guard](s3guard.html) is used to provide consistency,
-or because the S3-compatible filesystem is known to be strongly consistent.
+Magic Committer support within the S3A filesystem has been enabled by default since Hadoop 3.3.1.
+This error will only surface with a configuration which has explicitly disabled it.
+Remove all global/per-bucket declarations of `fs.s3a.bucket.magic.enabled` or set them to `true`.
 
 ```xml
 <property>
@@ -767,29 +766,35 @@ or because the S3-compatible filesystem is known to be strongly consistent.
 </property>
 ```
 
-
 Tip: you can verify that a bucket supports the magic committer through the
 `hadoop s3guard bucket-info` command:
 
 
 ```
 > hadoop s3guard bucket-info -magic s3a://landsat-pds/
-
-Filesystem s3a://landsat-pds
 Location: us-west-2
-Filesystem s3a://landsat-pds is not using S3Guard
-The "magic" committer is not supported
 
 S3A Client
-  Signing Algorithm: fs.s3a.signing-algorithm=(unset)
-  Endpoint: fs.s3a.endpoint=s3.amazonaws.com
-  Encryption: fs.s3a.server-side-encryption-algorithm=none
-  Input seek policy: fs.s3a.experimental.input.fadvise=normal
-  Change Detection Source: fs.s3a.change.detection.source=etag
-  Change Detection Mode: fs.s3a.change.detection.mode=server
-Delegation token support is disabled
-2019-05-17 13:53:38,245 [main] INFO  util.ExitUtil (ExitUtil.java:terminate(210)) -
- Exiting with status 46: 46: The magic committer is not enabled for s3a://landsat-pds
+        Signing Algorithm: fs.s3a.signing-algorithm=(unset)
+        Endpoint: fs.s3a.endpoint=s3.amazonaws.com
+        Encryption: fs.s3a.encryption.algorithm=none
+        Input seek policy: fs.s3a.experimental.input.fadvise=normal
+        Change Detection Source: fs.s3a.change.detection.source=etag
+        Change Detection Mode: fs.s3a.change.detection.mode=server
+
+S3A Committers
+        The "magic" committer is supported in the filesystem
+        S3A Committer factory class: mapreduce.outputcommitter.factory.scheme.s3a=org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
+        S3A Committer name: fs.s3a.committer.name=magic
+        Store magic committer integration: fs.s3a.committer.magic.enabled=true
+
+Security
+        Delegation token support is disabled
+
+Directory Markers
+        The directory marker policy is "delete"
+        Available Policies: delete, keep, authoritative
+        Authoritative paths: fs.s3a.authoritative.path=```
 ```
 
 ### Error message: "File being created has a magic path, but the filesystem has magic file support disabled"
@@ -802,11 +807,6 @@ This message should not appear through the committer itself &mdash;it will
 fail with the error message in the previous section, but may arise
 if other applications are attempting to create files under the path `/__magic/`.
 
-Make sure the filesystem meets the requirements of the magic committer
-(a consistent S3A filesystem through S3Guard or the S3 service itself),
-and set the `fs.s3a.committer.magic.enabled` flag to indicate that magic file
-writes are supported.
-
 
 ### `FileOutputCommitter` appears to be still used (from logs or delays in commits)
 
@@ -840,7 +840,7 @@ the failure happen at the start of a job.
 (Setting this option will not interfere with the Staging Committers' use of HDFS,
 as it explicitly sets the algorithm to "2" for that part of its work).
 
-The other way to check which committer to use is to examine the `_SUCCESS` file.
+The other way to check which committer was used is to examine the `_SUCCESS` file.
 If it is 0-bytes long, the classic `FileOutputCommitter` committed the job.
 The S3A committers all write a non-empty JSON file; the `committer` field lists
 the committer used.
@@ -856,7 +856,7 @@ all committers registered for the s3a:// schema.
 1. The output format has overridden `FileOutputFormat.getOutputCommitter()`
 and is returning its own committer -one which is a subclass of `FileOutputCommitter`.
 
-That final cause. *the output format is returning its own committer*, is not
+The final cause "the output format is returning its own committer" is not
 easily fixed; it may be that the custom committer performs critical work
 during its lifecycle, and contains assumptions about the state of the written
 data during task and job commit (i.e. it is in the destination filesystem).
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md
index 90e4e5587d027..0ba516313f42d 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md
@@ -91,7 +91,7 @@ of:
 
 These credentials are obtained from the AWS Secure Token Service (STS) when the the token is issued.
 * A set of AWS session credentials binding the user to a specific AWS IAM Role,
-further restricted to only access the S3 bucket and matching S3Guard DynamoDB table.
+further restricted to only access the S3 bucket.
 Again, these credentials are requested when the token is issued.
 
 
@@ -404,7 +404,6 @@ Else: as with session delegation tokens, an STS client is created. This time
 set to restrict access to:
 
 * CRUD access the specific bucket a token is being requested for
-* CRUD access to the contents of any S3Guard DDB used (not admin rights though).
 * access to all KMS keys (assumption: AWS KMS is where restrictions are set up)
 
 *Example Generated Role Policy*
@@ -428,12 +427,7 @@ set to restrict access to:
     "Effect" : "Allow",
     "Action" : [ "kms:Decrypt", "kms:GenerateDataKey" ],
     "Resource" : "arn:aws:kms:*"
-  }, {
-    "Sid" : "9",
-    "Effect" : "Allow",
-    "Action" : [ "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:DescribeTable", "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem" ],
-    "Resource" : "arn:aws:dynamodb:eu-west-1:980678866fff:table/example-bucket"
-  } ]
+  }]
 }
 ```
 
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
index aad3f355b2de6..f8f9d88d1e7cf 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
@@ -39,8 +39,7 @@ Clients with this token have the full permissions of the user.
 
 *Role Delegation Tokens:* These contain an "STS Session Token" requested by by the
 STS "Assume Role" API, so grant the caller to interact with S3 as specific AWS
-role, *with permissions restricted to purely accessing the S3 bucket
-and associated S3Guard data*.
+role, *with permissions restricted to purely accessing that specific S3 bucket*.
 
 Role Delegation Tokens are the most powerful. By restricting the access rights
 of the granted STS token, no process receiving the token may perform
@@ -121,6 +120,8 @@ Because applications only collect Delegation Tokens in secure clusters,
 It does mean that to be able to submit delegation tokens in transient
 cloud-hosted Hadoop clusters, _these clusters must also have Kerberos enabled_.
 
+*Tip*: you should only be deploying Hadoop in public clouds with Kerberos enabled.
+
 
 ### <a name="session-tokens"></a> S3A Session Delegation Tokens
 
@@ -143,8 +144,7 @@ for specifics details on the (current) token lifespan.
 A Role Delegation Tokens is created by asking the AWS
 [Security Token Service](http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html)
 for set of "Assumed Role" credentials, with a AWS account specific role for a limited duration..
-This role is restricted to only grant access the S3 bucket, the S3Guard table
-and all KMS keys,
+This role is restricted to only grant access the S3 bucket and all KMS keys,
 They are marshalled into the S3A Delegation Token.
 
 Other S3A connectors can extract these credentials and use them to
@@ -389,10 +389,10 @@ There are some further configuration options:
 | `fs.s3a.assumed.role.sts.endpoint.region` | region for issued tokens |  (undefined) |
 
 The option `fs.s3a.assumed.role.arn` must be set to a role which the
-user can assume. It must have permissions to access the bucket, any
-associated S3Guard table and any KMS encryption keys. The actual
+user can assume. It must have permissions to access the bucket and
+any KMS encryption keys. The actual
 requested role will be this role, explicitly restricted to the specific
-bucket and S3Guard table.
+bucket.
 
 The XML settings needed to enable session tokens are:
 
@@ -416,14 +416,10 @@ A JSON role policy for the role/session will automatically be generated which wi
 consist of
 1. Full access to the S3 bucket for all operations used by the S3A client
 (read, write, list, multipart operations, get bucket location, etc).
-1. Full user access to any S3Guard DynamoDB table used by the bucket.
 1. Full user access to KMS keys. This is to be able to decrypt any data
 in the bucket encrypted with SSE-KMS, as well as encrypt new data if that
 is the encryption policy.
 
-If the client doesn't have S3Guard enabled, but the remote application does,
-the issued role tokens will not have permission to access the S3Guard table.
-
 ### <a name="enabling-full-tokens"></a> Enabling Full Delegation Tokens
 
 This passes the full credentials in, falling back to any session credentials
@@ -663,7 +659,7 @@ for unmarshalling.
 
 This identifier must contain all information needed at the far end to
 authenticate the caller with AWS services used by the S3A client: AWS S3 and
-potentially AWS KMS (for SSE-KMS) and AWS DynamoDB (for S3Guard).
+potentially AWS KMS (for SSE-KMS).
 
 It must have its own unique *Token Kind*, to ensure that it can be distinguished
 from the other token identifiers when tokens are being unmarshalled.
@@ -693,8 +689,7 @@ is needed to keep them out of logs.
 * Secrets MUST NOT be logged, even at debug level.
 * Prefer short-lived session secrets over long-term secrets.
 * Try to restrict the permissions to what a client with the delegated token
-  may perform to those needed to access data in the S3 bucket. This potentially
-  includes a DynamoDB table, KMS access, etc.
+  may perform to those needed to access data in the S3 bucket.
 * Implementations need to be resistant to attacks which pass in invalid data as
 their token identifier: validate the types of the unmarshalled data; set limits
 on the size of all strings and other arrays to read in, etc.
@@ -798,7 +793,6 @@ Create the delegation token.
 If non-empty, the `policy` argument contains an AWS policy model to grant access to:
 
 * The target S3 bucket.
-* Any S3Guard DDB table it is bonded to.
 * KMS key `"kms:GenerateDataKey` and `kms:Decrypt`permissions for all KMS keys.
 
 This can be converted to a string and passed to the AWS `assumeRole` operation.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md
index 65fcb6502fb94..27c337354c4d6 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/directory_markers.md
@@ -51,16 +51,6 @@ These branches have read-only compatibility.
   such directories have child entries.
 * They will open files under directories with such markers.
 
-However, they have limitations when writing/deleting directories.
-
-Specifically: S3Guard tables may not be correctly updated in
-all conditions, especially on the partial failure of delete
-operations. Specifically: they may mistakenly add a tombstone in
-the dynamoDB table and so future directory/directory tree listings
-will consider the directory to be nonexistent.
-
-_It is not safe for Hadoop releases before Hadoop 3.3.1 to write
-to S3 buckets which have directory markers when S3Guard is enabled_
 
 ## Verifying read compatibility.
 
@@ -261,7 +251,7 @@ This is backwards compatible _outside authoritative directories_.
 
 Until now, the notion of an "authoritative"
 directory has only been used as a performance optimization for deployments
-where it is known that all Applications are using the same S3Guard metastore
+where it is known that all applications were using the same S3Guard metastore
 when writing and reading data.
 In such a deployment, if it is also known that all applications are using a
 compatible version of the s3a connector, then they
@@ -272,41 +262,31 @@ every shipping Hadoop releases.
 
 ##  <a name="s3guard"></a> Directory Markers and S3Guard
 
-Applications which interact with S3A in S3A clients with S3Guard enabled still
-create and delete markers. There's no attempt to skip operations, such as by having
-`mkdirs() `create entries in the DynamoDB table but not the store.
-Having the client always update S3 ensures that other applications and clients
-do (eventually) see the changes made by the "guarded" application.
 
+The now-deleted S3Guard feature included the concept of "authoritative paths";
+paths where all clients were required to be using S3Guard and sharing the
+same metadata store.
+In such a setup, listing authoritative paths would skip all queries of the S3
+store -potentially being much faster.
+
+In production, authoritative paths were usually only ever for Hive managed
+tables, where access was strictly restricted to the Hive services.
+
+Likewise, the directory marker retention can enabled purely for authoritative
+paths.
 When S3Guard is configured to treat some directories as  [Authoritative](s3guard.html#authoritative)
 then an S3A connector with a retention policy of `fs.s3a.directory.marker.retention` of
 `authoritative` will omit deleting markers in authoritative directories.
 
-*Note* there may be further changes in directory semantics in "authoritative mode";
-only use in managed applications where all clients are using the same version of
-hadoop, and configured consistently.
-
-After the directory marker feature [HADOOP-13230](https://issues.apache.org/jira/browse/HADOOP-13230)
-was added, issues related to S3Guard integration surfaced:
-
-1. The incremental update of the S3Guard table was inserting tombstones
-   over directories as the markers were deleted, hiding files underneath.
-   This happened during directory `rename()` and `delete()`.
-1. The update of the S3Guard table after a partial failure of a bulk delete
-   operation would insert tombstones in S3Guard records of successfully
-   deleted markers, irrespective of the directory status.
-
-Issue #1 is unique to Hadoop branch 3.3; however issue #2 is s critical
-part of the S3Guard consistency handling.
-
-Both issues have been fixed in Hadoop 3.3.x,
-in [HADOOP-17244](https://issues.apache.org/jira/browse/HADOOP-17244)
-
-Issue #2, delete failure handling, is not easily backported and is
-not likely to be backported.
-
-Accordingly: Hadoop releases with read-only compatibility must not be used
-to rename or delete directories where markers are retained *when S3Guard is enabled.*
+```xml
+<property>
+  <name>fs.s3a.bucket.hive.authoritative.path</name>
+  <value>/tables</value>
+</property>
+```
+This an option to consider if not 100% confident that all
+applications interacting with a store are using an S3A client
+which is marker aware.
 
 ## <a name="bucket-info"></a> Verifying marker policy with `s3guard bucket-info`
 
@@ -334,7 +314,6 @@ Example: `s3guard bucket-info -markers aware` on a compatible release.
 > hadoop s3guard bucket-info -markers aware s3a://landsat-pds/
  Filesystem s3a://landsat-pds
  Location: us-west-2
- Filesystem s3a://landsat-pds is not using S3Guard
 
 ...
 
@@ -354,13 +333,9 @@ is unknown
 > hadoop s3guard bucket-info -markers aware s3a://landsat-pds/
 Illegal option -markers
 Usage: hadoop bucket-info [OPTIONS] s3a://BUCKET
-    provide/check S3Guard information about a specific bucket
+    provide/check information about a specific bucket
 
 Common options:
-  -guarded - Require S3Guard
-  -unguarded - Force S3Guard to be disabled
-  -auth - Require the S3Guard mode to be "authoritative"
-  -nonauth - Require the S3Guard mode to be "non-authoritative"
   -magic - Require the S3 filesystem to be support the "magic" committer
   -encryption -require {none, sse-s3, sse-kms} - Require encryption policy
 
@@ -380,7 +355,6 @@ A specific policy check verifies that the connector is configured as desired
 > hadoop s3guard bucket-info -markers delete s3a://landsat-pds/
 Filesystem s3a://landsat-pds
 Location: us-west-2
-Filesystem s3a://landsat-pds is not using S3Guard
 
 ...
 
@@ -394,16 +368,33 @@ does not match that requested:
 > hadoop s3guard bucket-info -markers keep s3a://landsat-pds/
 Filesystem s3a://landsat-pds
 Location: us-west-2
-Filesystem s3a://landsat-pds is not using S3Guard
 
-...
+S3A Client
+        Signing Algorithm: fs.s3a.signing-algorithm=(unset)
+        Endpoint: fs.s3a.endpoint=s3.amazonaws.com
+        Encryption: fs.s3a.encryption.algorithm=none
+        Input seek policy: fs.s3a.experimental.input.fadvise=normal
+        Change Detection Source: fs.s3a.change.detection.source=etag
+        Change Detection Mode: fs.s3a.change.detection.mode=server
+
+S3A Committers
+        The "magic" committer is supported in the filesystem
+        S3A Committer factory class: mapreduce.outputcommitter.factory.scheme.s3a=org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
+        S3A Committer name: fs.s3a.committer.name=magic
+        Store magic committer integration: fs.s3a.committer.magic.enabled=true
 
 Security
-    Delegation token support is disabled
+        Delegation token support is disabled
 
-The directory marker policy is "delete"
+Directory Markers
+        The directory marker policy is "delete"
+        Available Policies: delete, keep, authoritative
+        Authoritative paths: fs.s3a.authoritative.path=
+
+2021-11-22 16:03:59,175 [main] INFO  util.ExitUtil (ExitUtil.java:terminate(210))
+ -Exiting with status 46: 46: Bucket s3a://landsat-pds: required marker polic is
+  "keep" but actual policy is "delete"
 
-2020-08-12 17:14:30,563 [main] INFO  util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 46: 46: Bucket s3a://landsat-pds: required marker policy is "keep" but actual policy is "delete"
 ```
 
 
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
index ccdfeeda7e901..ce1286c414a60 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md
@@ -601,7 +601,6 @@ clients where S3-CSE has not been enabled.
 
 ### Limitations
 
-- S3Guard is not supported with S3-CSE.
 - Performance will be reduced. All encrypt/decrypt is now being done on the
  client.
 - Writing files may be slower, as only a single block can be encrypted and
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 9ed2419cd5cd5..cd7793bfa92d3 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -47,6 +47,8 @@ full details.
 * [Auditing](./auditing.html).
 * [Auditing Architecture](./auditing_architecture.html).
 * [Testing](./testing.html)
+* [Prefetching](./prefetching.html)
+* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html)
 
 ## <a name="overview"></a> Overview
 
@@ -84,11 +86,10 @@ schemes.
 properties, the Hadoop key management store and IAM roles.
 * Supports per-bucket configuration.
 * Supports S3 "Server Side Encryption" for both reading and writing:
- SSE-S3, SSE-KMS and SSE-C
+ SSE-S3, SSE-KMS and SSE-C.
 * Instrumented with Hadoop metrics.
 * Before S3 was consistent, provided a consistent view of inconsistent storage
   through [S3Guard](./s3guard.html).
-
 * Actively maintained by the open source community.
 
 
@@ -148,32 +149,11 @@ Amazon S3 is an example of "an object store". In order to achieve scalability
 and especially high availability, S3 has —as many other cloud object stores have
 done— relaxed some of the constraints which classic "POSIX" filesystems promise.
 
-The [S3Guard](./s3guard.html) feature attempts to address some of these, but
-it cannot do so completely. Do read these warnings and consider how
-they apply.
-
 For further discussion on these topics, please consult
 [The Hadoop FileSystem API Definition](../../../hadoop-project-dist/hadoop-common/filesystem/index.html).
 
-### Warning #1: S3 Consistency model
-
-Amazon S3 is an example of "an object store". In order to achieve scalability
-and especially high availability, S3 has —as many other cloud object stores have
-done— relaxed some of the constraints which classic "POSIX" filesystems promise.
-
-Specifically
-
-1. Files that are newly created from the Hadoop Filesystem APIs may not be
-immediately visible.
-2. File delete and update operations may not immediately propagate. Old
-copies of the file may exist for an indeterminate time period.
-3. Directory operations: `delete()` and `rename()` are implemented by
-recursive file-by-file operations. They take time at least proportional to
-the number of files, during which time partial updates may be visible. If
-the operations are interrupted, the filesystem is left in an intermediate state.
-
 
-### Warning #2: Directories are mimicked
+### Warning #1: Directories are mimicked
 
 The S3A clients mimics directories by:
 
@@ -213,7 +193,7 @@ to safely save the output of queries directly into S3 object stores
 through the S3A filesystem.
 
 
-### Warning #3: Object stores have different authorization models
+### Warning #2: Object stores have different authorization models
 
 The object authorization model of S3 is much different from the file
 authorization model of HDFS and traditional file systems.
@@ -256,6 +236,9 @@ needs the credentials needed to interact with buckets.
 The client supports multiple authentication mechanisms and can be configured as to
 which mechanisms to use, and their order of use. Custom implementations
 of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
+However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be
+updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
+For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
 
 *Important*: The S3A connector no longer supports username and secrets
 in URLs of the form `s3a://key:secret@bucket/`.
@@ -551,7 +534,7 @@ This means that the default S3A authentication chain can be defined as
     com.amazonaws.auth.AWSCredentialsProvider.
 
     When S3A delegation tokens are not enabled, this list will be used
-    to directly authenticate with S3 and DynamoDB services.
+    to directly authenticate with S3 and other AWS services.
     When S3A Delegation tokens are enabled, depending upon the delegation
     token binding it may be used
     to communicate with the STS endpoint to request session/role
@@ -778,10 +761,9 @@ All S3A client options are configured with options with the prefix `fs.s3a.`.
 The client supports <a href="per_bucket_configuration">Per-bucket configuration</a>
 to allow different buckets to override the shared settings. This is commonly
 used to change the endpoint, encryption and authentication mechanisms of buckets.
-S3Guard options, various minor options.
+and various minor options.
 
-Here are the S3A properties for use in production. The S3Guard options are
-documented in the [S3Guard documents](./s3guard.html); some testing-related
+Here are the S3A properties for use in production; some testing-related
 options are covered in [Testing](./testing.md).
 
 ```xml
@@ -990,9 +972,12 @@ options are covered in [Testing](./testing.md).
 
 <property>
   <name>fs.s3a.buffer.dir</name>
-  <value>${hadoop.tmp.dir}/s3a</value>
+  <value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value>
   <description>Comma separated list of directories that will be used to buffer file
-    uploads to.</description>
+    uploads to.
+    Yarn container path will be used as default value on yarn applications,
+    otherwise fall back to hadoop.tmp.dir
+  </description>
 </property>
 
 <property>
@@ -1037,6 +1022,14 @@ options are covered in [Testing](./testing.md).
   any call to setReadahead() is made to an open stream.</description>
 </property>
 
+<property>
+  <name>fs.s3a.input.async.drain.threshold</name>
+  <value>64K</value>
+  <description>Bytes to read ahead during a seek() before closing and
+  re-opening the S3 HTTP connection. This option will be overridden if
+  any call to setReadahead() is made to an open stream.</description>
+</property>
+
 <property>
   <name>fs.s3a.list.version</name>
   <value>2</value>
@@ -1091,6 +1084,40 @@ options are covered in [Testing](./testing.md).
   </description>
 </property>
 
+<property>
+  <name>fs.s3a.create.storage.class</name>
+  <value></value>
+  <description>
+      Storage class: standard, reduced_redundancy, intelligent_tiering, etc.
+      Specify the storage class for S3A PUT object requests.
+      If not set the storage class will be null
+      and mapped to default standard class on S3.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.prefetch.enabled</name>
+  <value>false</value>
+  <description>
+    Enables prefetching and caching when reading from input stream.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.prefetch.block.size</name>
+  <value>8MB</value>
+  <description>
+      The size of a single prefetched block of data.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.prefetch.block.count</name>
+  <value>8</value>
+  <description>
+      Maximum number of blocks prefetched concurrently at any given time.
+  </description>
+</property>
 ```
 
 ## <a name="retry_and_recovery"></a>Retry and Recovery
@@ -1176,10 +1203,10 @@ be the wrong decision: rebuild the `hadoop-aws` module with the constant
 
 
 
-### Throttled requests from S3 and Dynamo DB
+### Throttled requests from S3
 
 
-When S3A or Dynamo DB returns a response indicating that requests
+When AWS S3 returns a response indicating that requests
 from the caller are being throttled, an exponential back-off with
 an initial interval and a maximum number of requests.
 
@@ -1246,8 +1273,6 @@ performed by clients.
 !. Maybe: increase `fs.s3a.readahead.range` to increase the minimum amount
 of data asked for in every GET request, as well as how much data is
 skipped in the existing stream before aborting it and creating a new stream.
-1. If the DynamoDB tables used by S3Guard are being throttled, increase
-the capacity through `hadoop s3guard set-capacity` (and pay more, obviously).
 1. KMS: "consult AWS about increasing your capacity".
 
 
@@ -1323,13 +1348,6 @@ that had already read the first byte.  Seeks backward on the other hand can
 result in new 'Get Object' requests that can trigger the
 `RemoteFileChangedException`.
 
-Additionally, due to the eventual consistency of S3 in a read-after-overwrite
-scenario, visibility of a new write may be delayed, avoiding the
-`RemoteFileChangedException` for some readers.  That said, if a reader does not
-see `RemoteFileChangedException`, they will have at least read a consistent view
-of a single version of the file (the version available when they started
-reading).
-
 ### Change detection with S3 Versions.
 
 It is possible to switch to using the
@@ -1616,7 +1634,7 @@ Accessing data through an access point, is done by using its ARN, as opposed to
 You can set the Access Point ARN property using the following per bucket configuration property:
 ```xml
 <property>
-    <name>fs.s3a.sample-bucket.accesspoint.arn</name>
+    <name>fs.s3a.bucket.sample-bucket.accesspoint.arn</name>
     <value> {ACCESSPOINT_ARN_HERE} </value>
     <description>Configure S3a traffic to use this AccessPoint</description>
 </property>
@@ -1626,21 +1644,11 @@ This configures access to the `sample-bucket` bucket for S3A, to go through the
 new Access Point ARN. So, for example `s3a://sample-bucket/key` will now use your
 configured ARN when getting data from S3 instead of your bucket.
 
-You can also use an Access Point name as a path URI such as `s3a://finance-team-access/key`, by
-configuring the `.accesspoint.arn` property as a per-bucket override:
-```xml
-<property>
-    <name>fs.s3a.finance-team-access.accesspoint.arn</name>
-    <value> {ACCESSPOINT_ARN_HERE} </value>
-    <description>Configure S3a traffic to use this AccessPoint</description>
-</property>
-```
-
 The `fs.s3a.accesspoint.required` property can also require all access to S3 to go through Access
 Points. This has the advantage of increasing security inside a VPN / VPC as you only allow access
 to known sources of data defined through Access Points. In case there is a need to access a bucket
 directly (without Access Points) then you can use per bucket overrides to disable this setting on a
-bucket by bucket basis i.e. `fs.s3a.{YOUR-BUCKET}.accesspoint.required`.
+bucket by bucket basis i.e. `fs.s3a.bucket.{YOUR-BUCKET}.accesspoint.required`.
 
 ```xml
 <!-- Require access point only access -->
@@ -1650,7 +1658,7 @@ bucket by bucket basis i.e. `fs.s3a.{YOUR-BUCKET}.accesspoint.required`.
 </property>
 <!-- Disable it on a per-bucket basis if needed -->
 <property>
-    <name>fs.s3a.example-bucket.accesspoint.required</name>
+    <name>fs.s3a.bucket.example-bucket.accesspoint.required</name>
     <value>false</value>
 </property>
 ```
@@ -1662,6 +1670,45 @@ Before using Access Points make sure you're not impacted by the following:
 considering endpoints, if you have any custom signers that use the host endpoint property make
 sure to update them if needed;
 
+## <a name="requester_pays"></a>Requester Pays buckets
+
+S3A supports buckets with
+[Requester Pays](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html)
+enabled. When a bucket is configured with requester pays, the requester must cover
+the per-request cost.
+
+For requests to be successful, the S3 client must acknowledge that they will pay
+for these requests by setting a request flag, usually a header, on each request.
+
+To enable this feature within S3A, configure the `fs.s3a.requester.pays.enabled` property.
+
+```xml
+<property>
+    <name>fs.s3a.requester.pays.enabled</name>
+    <value>true</value>
+</property>
+```
+
+## <a name="storage_classes"></a>Storage Classes
+
+Amazon S3 offers a range of [Storage Classes](https://aws.amazon.com/s3/storage-classes/)
+that you can choose from based on behavior of your applications. By using the right
+storage class, you can reduce the cost of your bucket.
+
+S3A uses Standard storage class for PUT object requests by default, which is suitable for
+general use cases. To use a specific storage class, set the value in `fs.s3a.create.storage.class` property to
+the storage class you want.
+
+```xml
+<property>
+    <name>fs.s3a.create.storage.class</name>
+    <value>intelligent_tiering</value>
+</property>
+```
+
+Please note that S3A does not support reading from archive storage classes at the moment.
+`AccessDeniedException` with InvalidObjectState will be thrown if you're trying to do so.
+
 ## <a name="upload"></a>How S3A writes data to S3
 
 The original S3A client implemented file writes by
@@ -1778,9 +1825,12 @@ consumed, and so eliminates heap size as the limiting factor in queued uploads
 
 <property>
   <name>fs.s3a.buffer.dir</name>
-  <value>${hadoop.tmp.dir}/s3a</value>
+  <value>${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a</value>
   <description>Comma separated list of directories that will be used to buffer file
-    uploads to.</description>
+    uploads to.
+    Yarn container path will be used as default value on yarn applications,
+    otherwise fall back to hadoop.tmp.dir
+  </description>
 </property>
 ```
 
@@ -2092,7 +2142,7 @@ except also allows for a custom SignerInitializer
 (`org.apache.hadoop.fs.s3a.AwsSignerInitializer`) class to be specified.
 
 #### Usage of the Signers
-Signers can be set at a per service level(S3, dynamodb, etc) or a common
+Signers can be set at a per-service level (S3, etc) or a common
 signer for all services.
 
 ```xml
@@ -2102,12 +2152,6 @@ signer for all services.
   <description>Specify the signer for S3</description>
 </property>
 
-<property>
-  <name>fs.s3a.ddb.signing-algorithm</name>
-  <value>${DdbSignerName}</value>
-  <description>Specify the signer for DDB</description>
-</property>
-
 <property>
   <name>fs.s3a.signing-algorithm</name>
   <value>${SignerName}</value>
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
index ab8b922df2a70..88e6e8a0b2115 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md
@@ -55,15 +55,44 @@ it isn't, and some attempts to preserve the metaphor are "aggressively suboptima
 
 To make most efficient use of S3, care is needed.
 
-## <a name="s3guard"></a> Speeding up directory listing operations through S3Guard
+## <a name="vectoredIO"></a> Improving read performance using Vectored IO
+The S3A FileSystem supports implementation of vectored read api using which
+a client can provide a list of file ranges to read returning a future read
+object associated with each range. For full api specification please see
+[FSDataInputStream](../../hadoop-common-project/hadoop-common/filesystem/fsdatainputstream.html).
 
-[S3Guard](s3guard.html) provides significant speedups for operations which
-list files a lot. This includes the setup of all queries against data:
-MapReduce, Hive and Spark, as well as DistCP.
-
-
-Experiment with using it to see what speedup it delivers.
+The following properties can be configured to optimise vectored reads based
+on the client requirements.
 
+```xml
+<property>
+  <name>fs.s3a.vectored.read.min.seek.size</name>
+  <value>4K</value>
+  <description>
+     What is the smallest reasonable seek in bytes such
+     that we group ranges together during vectored
+     read operation.
+   </description>
+</property>
+<property>
+   <name>fs.s3a.vectored.read.max.merged.size</name>
+   <value>1M</value>
+   <description>
+      What is the largest merged read size in bytes such
+      that we group ranges together during vectored read.
+      Setting this value to 0 will disable merging of ranges.
+   </description>
+<property>
+   <name>fs.s3a.vectored.active.ranged.reads</name>
+   <value>4</value>
+   <description>
+      Maximum number of range reads a single input stream can have
+      active (downloading, or queued) to the central FileSystem
+      instance's pool of queued operations.
+      This stops a single stream overloading the shared thread pool.
+   </description>
+</property>
+```
 
 ## <a name="fadvise"></a> Improving data input performance through fadvise
 
@@ -157,9 +186,7 @@ When using S3 as a destination, this is slow because of the way `rename()`
 is mimicked with copy and delete.
 
 If committing output takes a long time, it is because you are using the standard
-`FileOutputCommitter`. If you are doing this on any S3 endpoint which lacks
-list consistency (Amazon S3 without [S3Guard](s3guard.html)), this committer
-is at risk of losing data!
+`FileOutputCommitter`.
 
 *Your problem may appear to be performance, but that is a symptom
 of the underlying problem: the way S3A fakes rename operations means that
@@ -448,27 +475,6 @@ If you believe that you are reaching these limits, you may be able to
 get them increased.
 Consult [the KMS Rate Limit documentation](http://docs.aws.amazon.com/kms/latest/developerguide/limits.html).
 
-### <a name="s3guard_throttling"></a> S3Guard and Throttling
-
-
-S3Guard uses DynamoDB for directory and file lookups;
-it is rate limited to the amount of (guaranteed) IO purchased for a
-table.
-
-To see the allocated capacity of a bucket, the `hadoop s3guard bucket-info s3a://bucket`
-command will print out the allocated capacity.
-
-
-If significant throttling events/rate is observed here, the pre-allocated
-IOPs can be increased with the `hadoop s3guard set-capacity` command, or
-through the AWS Console. Throttling events in S3Guard are noted in logs, and
-also in the S3A metrics `s3guard_metadatastore_throttle_rate` and
-`s3guard_metadatastore_throttled`.
-
-If you are using DistCP for a large backup to/from a S3Guarded bucket, it is
-actually possible to increase the capacity for the duration of the operation.
-
-
 ## <a name="coding"></a> Best Practises for Code
 
 Here are some best practises if you are writing applications to work with
@@ -484,10 +490,6 @@ Cache the outcome of `getFileStats()`, rather than repeatedly ask for it.
 That includes using `isFile()`, `isDirectory()`, which are simply wrappers
 around `getFileStatus()`.
 
-Don't immediately look for a file with a `getFileStatus()` or listing call
-after creating it, or try to read it immediately.
-This is where eventual consistency problems surface: the data may not yet be visible.
-
 Rely on `FileNotFoundException` being raised if the source of an operation is
 missing, rather than implementing your own probe for the file before
 conditionally calling the operation.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md
new file mode 100644
index 0000000000000..e966c2dce4cb3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md
@@ -0,0 +1,196 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# S3A Prefetching
+
+This document explains the `S3PrefetchingInputStream` and the various components it uses.
+
+This input stream implements prefetching and caching to improve read performance of the input
+stream.
+A high level overview of this feature was published in
+[Pinterest Engineering's blog post titled "Improving efficiency and reducing runtime using S3 read optimization"](https://medium.com/pinterest-engineering/improving-efficiency-and-reducing-runtime-using-s3-read-optimization-b31da4b60fa0).
+
+With prefetching, the input stream divides the remote file into blocks of a fixed size, associates
+buffers to these blocks and then reads data into these buffers asynchronously.
+It also potentially caches these blocks.
+
+### Basic Concepts
+
+* **Remote File**: A binary blob of data stored on some storage device.
+* **Block File**: Local file containing a block of the remote file.
+* **Block**: A file is divided into a number of blocks.
+The size of the first n-1 blocks is same, and the size of the last block may be same or smaller.
+* **Block based reading**: The granularity of read is one block.
+That is, either an entire block is read and returned or none at all.
+Multiple blocks may be read in parallel.
+
+### Configuring the stream
+
+|Property    |Meaning    |Default    |
+|---|---|---|
+|`fs.s3a.prefetch.enabled`    |Enable the prefetch input stream    |`false` |
+|`fs.s3a.prefetch.block.size`    |Size of a block    |`8M`    |
+|`fs.s3a.prefetch.block.count`    |Number of blocks to prefetch    |`8`    |
+
+### Key Components
+
+`S3PrefetchingInputStream` - When prefetching is enabled, S3AFileSystem will return an instance of
+this class as the input stream.
+Depending on the remote file size, it will either use
+the `S3InMemoryInputStream` or the `S3CachingInputStream` as the underlying input stream.
+
+`S3InMemoryInputStream` - Underlying input stream used when the remote file size < configured block
+size.
+Will read the entire remote file into memory.
+
+`S3CachingInputStream` - Underlying input stream used when remote file size > configured block size.
+Uses asynchronous prefetching of blocks and caching to improve performance.
+
+`BlockData` - Holds information about the blocks in a remote file, such as:
+
+* Number of blocks in the remote file
+* Block size
+* State of each block (initially all blocks have state *NOT_READY*).
+Other states are: Queued, Ready, Cached.
+
+`BufferData` - Holds the buffer and additional information about it such as:
+
+* The block number this buffer is for
+* State of the buffer (Unknown, Blank, Prefetching, Caching, Ready, Done).
+Initial state of a buffer is blank.
+
+`CachingBlockManager` - Implements reading data into the buffer, prefetching and caching.
+
+`BufferPool` - Manages a fixed sized pool of buffers.
+It's used by `CachingBlockManager` to acquire buffers.
+
+`S3File` - Implements operations to interact with S3 such as opening and closing the input stream to
+the remote file in S3.
+
+`S3Reader` - Implements reading from the stream opened by `S3File`.
+Reads from this input stream in blocks of 64KB.
+
+`FilePosition` - Provides functionality related to tracking the position in the file.
+Also gives access to the current buffer in use.
+
+`SingleFilePerBlockCache` - Responsible for caching blocks to the local file system.
+Each cache block is stored on the local disk as a separate block file.
+
+### Operation
+
+#### S3InMemoryInputStream
+
+For a remote file with size 5MB, and block size = 8MB, since file size is less than the block size,
+the `S3InMemoryInputStream` will be used.
+
+If the caller makes the following read calls:
+
+```
+in.read(buffer, 0, 3MB);
+in.read(buffer, 0, 2MB);
+```
+
+When the first read is issued, there is no buffer in use yet.
+The `S3InMemoryInputStream` gets the data in this remote file by calling the `ensureCurrentBuffer()`
+method, which ensures that a buffer with data is available to be read from.
+
+The `ensureCurrentBuffer()` then:
+
+* Reads data into a buffer by calling `S3Reader.read(ByteBuffer buffer, long offset, int size)`.
+* `S3Reader` uses `S3File` to open an input stream to the remote file in S3 by making
+  a `getObject()` request with range as `(0, filesize)`.
+* The `S3Reader` reads the entire remote file into the provided buffer, and once reading is complete
+  closes the S3 stream and frees all underlying resources.
+* Now the entire remote file is in a buffer, set this data in `FilePosition` so it can be accessed
+  by the input stream.
+
+The read operation now just gets the required bytes from the buffer in `FilePosition`.
+
+When the second read is issued, there is already a valid buffer which can be used.
+Don't do anything else, just read the required bytes from this buffer.
+
+#### S3CachingInputStream
+
+If there is a remote file with size 40MB and block size = 8MB, the `S3CachingInputStream` will be
+used.
+
+##### Sequential Reads
+
+If the caller makes the following calls:
+
+```
+in.read(buffer, 0, 5MB)
+in.read(buffer, 0, 8MB)
+```
+
+For the first read call, there is no valid buffer yet.
+`ensureCurrentBuffer()` is called, and for the first `read()`, prefetch count is set as 1.
+
+The current block (block 0) is read synchronously, while the blocks to be prefetched (block 1) is
+read asynchronously.
+
+The `CachingBlockManager` is responsible for getting buffers from the buffer pool and reading data
+into them. This process of acquiring the buffer pool works as follows:
+
+* The buffer pool keeps a map of allocated buffers and a pool of available buffers.
+The size of this pool is = prefetch block count + 1.
+If the prefetch block count is 8, the buffer pool has a size of 9.
+* If the pool is not yet at capacity, create a new buffer and add it to the pool.
+* If it's at capacity, check if any buffers with state = done can be released.
+Releasing a buffer means removing it from allocated and returning it back to the pool of available
+buffers.
+* If there are no buffers with state = done currently then nothing will be released, so retry the
+  above step at a fixed interval a few times till a buffer becomes available.
+* If after multiple retries there are still no available buffers, release a buffer in the ready state.
+The buffer for the block furthest from the current block is released.
+
+Once a buffer has been acquired by `CachingBlockManager`, if the buffer is in a *READY* state, it is
+returned.
+This means that data was already read into this buffer asynchronously by a prefetch.
+If its state is *BLANK* then data is read into it using
+`S3Reader.read(ByteBuffer buffer, long offset, int size).`
+
+For the second read call, `in.read(buffer, 0, 8MB)`, since the block sizes are of 8MB and only 5MB
+of block 0 has been read so far, 3MB of the required data will be read from the current block 0.
+Once all data has been read from this block, `S3CachingInputStream` requests the next block (
+block 1), which will already have been prefetched and so it can just start reading from it.
+Also, while reading from block 1 it will also issue prefetch requests for the next blocks.
+The number of blocks to be prefetched is determined by `fs.s3a.prefetch.block.count`.
+
+##### Random Reads
+
+The `CachingInputStream` also caches prefetched blocks. This happens when `read()` is issued
+after a `seek()` outside the current block, but the current block still has not been fully read.
+
+For example, consider the following calls:
+
+```
+in.read(buffer, 0, 5MB)
+in.seek(10MB)
+in.read(buffer, 0, 4MB)
+in.seek(2MB)
+in.read(buffer, 0, 4MB)
+```
+
+For the above read sequence, after the `seek(10MB)` call is issued, block 0 has not been read
+completely so the subsequent call to `read()` will cache it, on the assumption that the caller
+will probably want to read from it again.
+
+After `seek(2MB)` is called, the position is back inside block 0. The next read can now be
+satisfied from the locally cached block file, which is typically orders of magnitude faster
+than a network based read.
+
+NB: `seek()` is implemented lazily, so it only keeps track of the new position but does not
+otherwise affect the internal state of the stream. Only when a `read()` is issued, it will call
+the `ensureCurrentBuffer()` method and fetch a new block if required.
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
index a5aaae91454be..649c80a22d4f7 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3_select.md
@@ -934,6 +934,21 @@ Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: GZIP is not applic
   ...
 ```
 
+
+### AWSBadRequestException  `UnsupportedStorageClass`
+
+S3 Select doesn't work with some storage classes like Glacier or Reduced Redundancy.
+Make sure you've set `fs.s3a.create.storage.class` to a supported storage class for S3 Select.
+
+```
+org.apache.hadoop.fs.s3a.AWSBadRequestException:
+    Select on s3a://example/dataset.csv.gz:
+    com.amazonaws.services.s3.model.AmazonS3Exception:
+     We do not support REDUCED_REDUNDANCY storage class.
+     Please check the service documentation and try again.
+     (Service: Amazon S3; Status Code: 400; Error Code: UnsupportedStorageClass
+```
+
 ### `PathIOException`: "seek() not supported"
 
 The input stream returned by the select call does not support seeking
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index 27ac10c825fb5..046717bd5f4f6 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -12,14 +12,14 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# S3Guard: Consistency and Metadata Caching for S3A
+# S3Guard: Consistency and Metadata Caching for S3A (retired)
 
 <!-- MACRO{toc|fromDepth=0|toDepth=5} -->
 
 ## Overview
 
-*S3Guard* is a feature for the S3A client of the S3 object store,
-which can use a (consistent) database as the store of metadata about objects
+*S3Guard* was a feature for the S3A client of the S3 object store,
+which used a consistent database as the store of metadata about objects
 in an S3 bucket.
 
 It was written been 2016 and 2020, *when Amazon S3 was eventually consistent.*
@@ -32,7 +32,14 @@ It compensated for the following S3 inconsistencies:
 It did not compensate for update inconsistency, though by storing the etag
 values of objects in the database, it could detect and report problems.
 
-Now that S3 is consistent, there is no need for S3Guard at all.
+Now that S3 is consistent, _there is no need for S3Guard at all._
+Accordingly, it was removed from the source in 2022 in [HADOOP-17409](https://issues.apache.org/jira/browse/HADOOP-17409), _Remove S3Guard_.
+
+Attempting to create an S3A connector instance with S3Guard set to anything but the
+null or local metastores will now fail.
+
+
+### S3Guard History
 
 S3Guard
 
@@ -43,8 +50,6 @@ including those which take place during the partitioning period of query
 execution, the process where files are listed and the work divided up amongst
 processes.
 
-
-
 The basic idea was that, for each operation in the Hadoop S3 client (s3a) that
 reads or modifies metadata, a shadow copy of that metadata is stored in a
 separate MetadataStore implementation. The store was 
@@ -59,74 +64,39 @@ separate MetadataStore implementation. The store was
 For links to early design documents and related patches, see
 [HADOOP-13345](https://issues.apache.org/jira/browse/HADOOP-13345).
 
-*Important*
-
-* While all underlying data is persisted in S3, if, for some reason,
-the S3Guard-cached metadata becomes inconsistent with that in S3,
-queries on the data may become incorrect.
-For example, new datasets may be omitted, objects may be overwritten,
-or clients may not be aware that some data has been deleted.
-It is essential for all clients writing to an S3Guard-enabled
-S3 Repository to use the feature. Clients reading the data may work directly
-with the S3A data, in which case the normal S3 consistency guarantees apply.
 
 ## Moving off S3Guard
 
-How to move off S3Guard, given it is no longer needed.
+How to move off S3Guard
 
 1. Unset the option `fs.s3a.metadatastore.impl` globally/for all buckets for which it
    was selected.
-1. If the option `org.apache.hadoop.fs.s3a.s3guard.disabled.warn.level` has been changed from
-the default (`SILENT`), change it back. You no longer need to be warned that S3Guard is disabled.
 1. Restart all applications.
 
 Once you are confident that all applications have been restarted, _Delete the DynamoDB table_.
 This is to avoid paying for a database you no longer need.
-This is best done from the AWS GUI.
-
-## Setting up S3Guard
-
-### S3A to warn or fail if S3Guard is disabled
-A seemingly recurrent problem with S3Guard is that people think S3Guard is
-turned on but it isn't.
-You can set `org.apache.hadoop.fs.s3a.s3guard.disabled.warn.level`
-to avoid this. The property sets what to do when an S3A FS is instantiated
-without S3Guard. The following values are available:
+This can be done from the AWS GUI.
 
-* `SILENT`: Do nothing.
-* `INFORM`: Log at info level that FS is instantiated without S3Guard.
-* `WARN`: Warn that data may be at risk in workflows.
-* `FAIL`: S3AFileSystem instantiation will fail.
+## Removing S3Guard Configurations
 
-The default setting is `SILENT`. The setting is case insensitive.
-The required level can be set in the `core-site.xml`.
+The `fs.s3a.metadatastore.impl` option must be one of
+* unset
+* set to the empty string ""
+* set to the "Null" Metadata store `org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore`.
 
----
-The latest configuration parameters are defined in `core-default.xml`.  You
-should consult that file for full information, but a summary is provided here.
+To aid the migration of external components which used the Local store for a consistent
+view within the test process, the Local Metadata store option is also recognized:
+`org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore`.
+When this option is used the S3A connector will warn and continue.
 
 
-### 1. Choose the Database
-
-A core concept of S3Guard is that the directory listing data of the object
-store, *the metadata* is replicated in a higher-performance, consistent,
-database. In S3Guard, this database is called *The Metadata Store*
-
-By default, S3Guard is not enabled.
-
-The Metadata Store to use in production is bonded to Amazon's DynamoDB
-database service.  The following setting will enable this Metadata Store:
-
 ```xml
 <property>
     <name>fs.s3a.metadatastore.impl</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
+    <value></value>
 </property>
 ```
 
-Note that the `NullMetadataStore` store can be explicitly requested if desired.
-This offers no metadata storage, and effectively disables S3Guard.
-
 ```xml
 <property>
     <name>fs.s3a.metadatastore.impl</name>
@@ -134,675 +104,55 @@ This offers no metadata storage, and effectively disables S3Guard.
 </property>
 ```
 
-### 2. Configure S3Guard Settings
-
-More settings will may be added in the future.
-Currently the only Metadata Store-independent setting, besides the
-implementation class above, are the *allow authoritative* and *fail-on-error*
-flags.
-
-#### <a name="authoritative"></a>  Authoritative S3Guard
-
-Authoritative S3Guard is a complicated configuration which delivers performance
-at the expense of being unsafe for other applications to use the same directory
-tree/bucket unless configured consistently.
-
-It can also be used to support [directory marker retention](directory_markers.html)
-in higher-performance but non-backwards-compatible modes.
-
-Most deployments do not use this setting -it is ony used in deployments where
-specific parts of a bucket (e.g. Apache Hive managed tables) are known to
-have exclusive access by a single application (Hive) and other tools/applications
-from exactly the same Hadoop release.
-
-The _authoritative_ expression in S3Guard is present in two different layers, for
-two different reasons:
-
-* Authoritative S3Guard
-    * S3Guard can be set as authoritative, which means that an S3A client will
-    avoid round-trips to S3 when **getting file metadata**, and **getting
-    directory listings** if there is a fully cached version of the directory
-    stored in metadata store.
-    * This mode can be set as a configuration property
-    `fs.s3a.metadatastore.authoritative`
-    * It can also be set only on specific directories by setting
-    `fs.s3a.authoritative.path` to one or more prefixes, for example
-    `s3a://bucket/path` or "/auth1,/auth2".
-    * All interactions with the S3 bucket(s) must be through S3A clients sharing
-    the same metadata store.
-    * This is independent from which metadata store implementation is used.
-    * In authoritative mode the metadata TTL metadata expiry is not effective.
-    This means that the metadata entries won't expire on authoritative paths.
-
-* Authoritative directory listings (isAuthoritative bit)
-    * Tells if the stored directory listing metadata is complete.
-    * This is set by the FileSystem client (e.g. s3a) via the `DirListingMetadata`
-    class (`org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata`).
-    (The MetadataStore only knows what the FS client tells it.)
-    * If set to `TRUE`, we know that the directory listing
-    (`DirListingMetadata`) is full, and complete.
-    * If set to `FALSE` the listing may not be complete.
-    * Metadata store may persist the isAuthoritative bit on the metadata store.
-    * Currently `org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore` and
-    `org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore` implementation
-    supports authoritative bit.
-
-More on Authoritative S3Guard:
-
-* This setting is about treating the MetadataStore (e.g. dynamodb) as the source
- of truth in general, and also to short-circuit S3 list objects and serve
- listings from the MetadataStore in some circumstances.
-* For S3A to skip S3's get object metadata, and serve it directly from the
-MetadataStore, the following things must all be true:
-    1. The S3A client is configured to allow MetadataStore to be authoritative
-    source of a file metadata (`fs.s3a.metadatastore.authoritative=true`).
-    1. The MetadataStore has the file metadata for the path stored in it.
-* For S3A to skip S3's list objects on some path, and serve it directly from
-the MetadataStore, the following things must all be true:
-    1. The MetadataStore implementation persists the bit
-    `DirListingMetadata.isAuthorititative` set when calling
-    `MetadataStore#put` (`DirListingMetadata`)
-    1. The S3A client is configured to allow MetadataStore to be authoritative
-    source of a directory listing (`fs.s3a.metadatastore.authoritative=true`).
-    1. The MetadataStore has a **full listing for path** stored in it.  This only
-    happens if the FS client (s3a) explicitly has stored a full directory
-    listing with `DirListingMetadata.isAuthorititative=true` before the said
-    listing request happens.
-
-This configuration only enables authoritative mode in the client layer. It is
-recommended that you leave the default setting here:
-
-```xml
-<property>
-    <name>fs.s3a.metadatastore.authoritative</name>
-    <value>false</value>
-</property>
-```
-
-Note that a MetadataStore MAY persist this bit in the directory listings. (Not
-MUST).
-
-Note that if this is set to true, it may exacerbate or persist existing race
-conditions around multiple concurrent modifications and listings of a given
-directory tree.
-
-In particular: **If the Metadata Store is declared as authoritative,
-all interactions with the S3 bucket(s) must be through S3A clients sharing
-the same Metadata Store**
-
-#### TTL metadata expiry
-
-It can be configured how long an entry is valid in the MetadataStore
-**if the authoritative mode is turned off**, or the path is not
-configured to be authoritative.
-If `((lastUpdated + ttl) <= now)` is false for an entry, the entry will
-be expired, so the S3 bucket will be queried for fresh metadata.
-The time for expiry of metadata can be set as the following:
-
-```xml
-<property>
-    <name>fs.s3a.metadatastore.metadata.ttl</name>
-    <value>15m</value>
-</property>
-```
-
-#### Fail on Error
-
-By default, S3AFileSystem write operations will fail when updates to
-S3Guard metadata fail. S3AFileSystem first writes the file to S3 and then
-updates the metadata in S3Guard. If the metadata write fails,
-`MetadataPersistenceException` is thrown.  The file in S3 **is not** rolled
-back.
-
-If the write operation cannot be programmatically retried, the S3Guard metadata
-for the given file can be corrected with a command like the following:
-
-```bash
-hadoop s3guard import [-meta URI] s3a://my-bucket/file-with-bad-metadata
-```
-
-Programmatic retries of the original operation would require overwrite=true.
-Suppose the original operation was `FileSystem.create(myFile, overwrite=false)`.
-If this operation failed with `MetadataPersistenceException` a repeat of the
-same operation would result in `FileAlreadyExistsException` since the original
-operation successfully created the file in S3 and only failed in writing the
-metadata to S3Guard.
-
-Metadata update failures can be downgraded to ERROR logging instead of exception
-by setting the following configuration:
-
-```xml
-<property>
-    <name>fs.s3a.metadatastore.fail.on.write.error</name>
-    <value>false</value>
-</property>
-```
-
-Setting this false is dangerous as it could result in the type of issue S3Guard
-is designed to avoid. For example, a reader may see an inconsistent listing
-after a recent write since S3Guard may not contain metadata about the recently
-written file due to a metadata write error.
-
-As with the default setting, the new/updated file is still in S3 and **is not**
-rolled back. The S3Guard metadata is likely to be out of sync.
-
-### 3. Configure the Metadata Store.
-
-Here are the `DynamoDBMetadataStore` settings.  Other Metadata Store
-implementations will have their own configuration parameters.
-
-
-### 4. Name Your Table
-
-First, choose the name of the table you wish to use for the S3Guard metadata
-storage in your DynamoDB instance.  If you leave it unset/empty, a
-separate table will be created for each S3 bucket you access, and that
-bucket's name will be used for the name of the DynamoDB table.  For example,
-this sets the table name to `my-ddb-table-name`
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.ddb.table</name>
-  <value>my-ddb-table-name</value>
-  <description>
-    The DynamoDB table name to operate. Without this property, the respective
-    S3 bucket names will be used.
-  </description>
-</property>
-```
-
-It is good to share a table across multiple buckets for multiple reasons,
-especially if you are *not* using on-demand DynamoDB tables, and instead
-prepaying for provisioned I/O capacity.
-
-1. You are billed for the provisioned I/O capacity allocated to the table,
-*even when the table is not used*. Sharing capacity can reduce costs.
-
-1. You can share the "provision burden" across the buckets. That is, rather
-than allocating for the peak load on a single bucket, you can allocate for
-the peak load *across all the buckets*, which is likely to be significantly
-lower.
-
-1. It's easier to measure and tune the load requirements and cost of
-S3Guard, because there is only one table to review and configure in the
-AWS management console.
-
-1. When you don't grant the permission to create DynamoDB tables to users.
-A single pre-created table for all buckets avoids the needs for an administrator
-to create one for every bucket.
-
-When wouldn't you want to share a table?
-
-1. When you are using on-demand DynamoDB and want to keep each table isolated.
-1. When you do explicitly want to provision I/O capacity to a specific bucket
-and table, isolated from others.
-
-1. When you are using separate billing for specific buckets allocated
-to specific projects.
-
-1. When different users/roles have different access rights to different buckets.
-As S3Guard requires all users to have R/W access to the table, all users will
-be able to list the metadata in all buckets, even those to which they lack
-read access.
-
-### 5. Locate your Table
-
-You may also wish to specify the region to use for DynamoDB.  If a region
-is not configured, S3A will assume that it is in the same region as the S3
-bucket. A list of regions for the DynamoDB service can be found in
-[Amazon's documentation](http://docs.aws.amazon.com/general/latest/gr/rande.html#ddb_region).
-In this example, to use the US West 2 region:
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.ddb.region</name>
-  <value>us-west-2</value>
-</property>
-```
-
-When working with S3Guard-managed buckets from EC2 VMs running in AWS
-infrastructure, using a local DynamoDB region ensures the lowest latency
-and highest reliability, as well as avoiding all long-haul network charges.
-The S3Guard tables, and indeed, the S3 buckets, should all be in the same
-region as the VMs.
-
-### 6. Optional: Create your Table
-
-Next, you can choose whether or not the table will be automatically created
-(if it doesn't already exist).  If you want this feature, set the
-`fs.s3a.s3guard.ddb.table.create` option to `true`.
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.ddb.table.create</name>
-  <value>true</value>
-  <description>
-    If true, the S3A client will create the table if it does not already exist.
-  </description>
-</property>
-```
-
-### 7. If creating a table: Choose your billing mode (and perhaps I/O Capacity)
-
-Next, you need to decide whether to use On-Demand DynamoDB and its
-pay-per-request billing (recommended), or to explicitly request a
-provisioned IO capacity.
-
-Before AWS offered pay-per-request billing, the sole billing mechanism,
-was "provisioned capacity". This mechanism requires you to choose 
-the DynamoDB read and write throughput requirements you
-expect to need for your expected uses of the S3Guard table.
-Setting higher values cost you more money -*even when the table was idle*
-  *Note* that these settings only affect table creation when
-`fs.s3a.s3guard.ddb.table.create` is enabled.  To change the throughput for
-an existing table, use the AWS console or CLI tool.
-
-For more details on DynamoDB capacity units, see the AWS page on [Capacity
-Unit Calculations](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/WorkingWithTables.html#CapacityUnitCalculations).
-
-Provisioned IO capacity is billed per hour for the life of the table, *even when the
-table and the underlying S3 buckets are not being used*.
-
-There are also charges incurred for data storage and for data I/O outside of the
-region of the DynamoDB instance. S3Guard only stores metadata in DynamoDB: path names
-and summary details of objects —the actual data is stored in S3, so billed at S3
-rates.
-
-With provisioned I/O capacity, attempting to perform more I/O than the capacity
-requested throttles the operation and may result in operations failing.
-Larger I/O capacities cost more.
-
-With the introduction of On-Demand DynamoDB, you can now avoid paying for
-provisioned capacity by creating an on-demand table.
-With an on-demand table you are not throttled if your DynamoDB requests exceed
-any pre-provisioned limit, nor do you pay per hour even when a table is idle.
-
-You do, however, pay more per DynamoDB operation.
-Even so, the ability to cope with sudden bursts of read or write requests, combined
-with the elimination of charges for idle tables, suit the use patterns made of 
-S3Guard tables by applications interacting with S3. That is: periods when the table
-is rarely used, with intermittent high-load operations when directory trees
-are scanned (query planning and similar), or updated (rename and delete operations).
-
-
-We recommending using On-Demand DynamoDB for maximum performance in operations
-such as query planning, and lowest cost when S3 buckets are not being accessed.
-
-This is the default, as configured in the default configuration options.
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for read operations in terms of capacity
-    units for the DynamoDB table. This config value will only be used when
-    creating a new DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    If a positive integer is provided for this and the write capacity, then
-    a table with "provisioned capacity" will be created.
-    You can change the capacity of an existing provisioned-capacity table
-    through the "s3guard set-capacity" command.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for write operations in terms of
-    capacity units for the DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    Refer to related configuration option fs.s3a.s3guard.ddb.table.capacity.read
-  </description>
-</property>
-```
-
-### 8.  If creating a table: Enable server side encryption (SSE)
-
-Encryption at rest can help you protect sensitive data in your DynamoDB table.
-When creating a new table, you can set server side encryption on the table
-using the default AWS owned customer master key (CMK), AWS managed CMK, or
-customer managed CMK. S3Guard code accessing the table is all the same whether
-SSE is enabled or not. For more details on DynamoDB table server side
-encryption, see the AWS page on [Encryption at Rest: How It Works](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/encryption.howitworks.html).
-
-These are the default configuration options, as configured in `core-default.xml`.
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.enabled</name>
-  <value>false</value>
-  <description>
-    Whether server-side encryption (SSE) is enabled or disabled on the table.
-    By default it's disabled, meaning SSE is set to AWS owned CMK.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.cmk</name>
-  <value/>
-  <description>
-    The KMS Customer Master Key (CMK) used for the KMS encryption on the table.
-    To specify a CMK, this config value can be its key ID, Amazon Resource Name
-    (ARN), alias name, or alias ARN. Users only need to provide this config if
-    the key is different from the default DynamoDB KMS Master Key, which is
-    alias/aws/dynamodb.
-  </description>
-</property>
-```
-
-## Authenticating with S3Guard
-
-The DynamoDB metadata store takes advantage of the fact that the DynamoDB
-service uses the same authentication mechanisms as S3. S3Guard
-gets all its credentials from the S3A client that is using it.
-
-All existing S3 authentication mechanisms can be used.
-
-## Per-bucket S3Guard configuration
-
-In production, it is likely only some buckets will have S3Guard enabled;
-those which are read-only may have disabled, for example. Equally importantly,
-buckets in different regions should have different tables, each
-in the relevant region.
-
-These options can be managed through S3A's [per-bucket configuration
-mechanism](./index.html#Configuring_different_S3_buckets).
-All options with the under `fs.s3a.bucket.BUCKETNAME.KEY` are propagated
-to the options `fs.s3a.KEY` *for that bucket only*.
-
-As an example, here is a configuration to use different metadata stores
-and tables for different buckets
-
-First, we define shortcuts for the metadata store classnames:
-
-
-```xml
-<property>
-  <name>s3guard.null</name>
-  <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
-</property>
-
-<property>
-  <name>s3guard.dynamo</name>
-  <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
-</property>
-```
-
-Next, Amazon's public landsat database is configured with no
-metadata store:
-
-```xml
-<property>
-  <name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name>
-  <value>${s3guard.null}</value>
-  <description>The read-only landsat-pds repository isn't
-  managed by S3Guard</description>
-</property>
-```
-
-Next the `ireland-2` and `ireland-offline` buckets are configured with
-DynamoDB as the store, and a shared table `production-table`:
-
-
-```xml
-<property>
-  <name>fs.s3a.bucket.ireland-2.metadatastore.impl</name>
-  <value>${s3guard.dynamo}</value>
-</property>
-
-<property>
-  <name>fs.s3a.bucket.ireland-offline.metadatastore.impl</name>
-  <value>${s3guard.dynamo}</value>
-</property>
-
-<property>
-  <name>fs.s3a.bucket.ireland-2.s3guard.ddb.table</name>
-  <value>production-table</value>
-</property>
-```
-
-The region of this table is automatically set to be that of the buckets,
-here `eu-west-1`; the same table name may actually be used in different
-regions.
-
-Together then, this configuration enables the DynamoDB Metadata Store
-for two buckets with a shared table, while disabling it for the public
-bucket.
-
-
-### Out-of-band operations with S3Guard
-
-We call an operation out-of-band (OOB) when a bucket is used by a client with
- S3Guard, and another client runs a write (e.g delete, move, rename,
- overwrite) operation on an object in the same bucket without S3Guard.
+## Issue: Increased number/cost of S3 IO calls.
 
-The definition of behaviour in S3AFileSystem/MetadataStore in case of OOBs:
-* A client with S3Guard
-* B client without S3Guard (Directly to S3)
+More AWS S3 calls may be made once S3Guard is disabled, both for LIST and HEAD operations.
 
+While this may seem to increase cost, as the DDB table is no longer needed, users will
+save on DDB table storage and use costs.
 
-* OOB OVERWRITE, authoritative mode:
-  * A client creates F1 file
-  * B client overwrites F1 file with F2 (Same, or different file size)
-  * A client's getFileStatus returns F1 metadata
+Some deployments of Apache Hive declared their managed tables to be "authoritative".
+The S3 store was no longer checked when listing directories or for updates to
+entries. The S3Guard table in DynamoDB was used exclusively.
 
-* OOB OVERWRITE, NOT authoritative mode:
-  * A client creates F1 file
-  * B client overwrites F1 file with F2 (Same, or different file size)
-  * A client's getFileStatus returns F2 metadata. In not authoritative mode we
- check S3 for the file. If the modification time of the file in S3 is greater
- than in S3Guard, we can safely return the S3 file metadata and update the
- cache.
+Without S3Guard, listing performance may be slower. However, Hadoop 3.3.0+ has significantly
+improved listing performance ([HADOOP-17400](https://issues.apache.org/jira/browse/HADOOP-17400)
+_Optimize S3A for maximum performance in directory listings_) so this should not be apparent.
 
-* OOB DELETE, authoritative mode:
-  * A client creates F file
-  * B client deletes F file
-  * A client's getFileStatus returns that the file is still there
+We recommend disabling [directory marker deletion](directory_markers.html) to reduce
+the number of DELETE operations made when writing files.
+this reduces the load on the S3 partition and so the risk of throttling, which can
+impact performance.
+This is very important when working with versioned S3 buckets, as the tombstone markers
+created will slow down subsequent listing operations.
 
-* OOB DELETE, NOT authoritative mode:
-  * A client creates F file
-  * B client deletes F file
-  * A client's getFileStatus returns that the file is still there
-
-Note: authoritative and NOT authoritative mode behaves the same at
-OOB DELETE case.
-
-The behaviour in case of getting directory listings:
-* File status in metadata store gets updated during the listing the same way
-as in getFileStatus.
+Finally, the S3A [auditing](auditing.html) feature adds information to the S3 server logs
+about which jobs, users and filesystem operations have been making S3 requests.
+This auditing information can be used to identify opportunities to reduce load.
 
 
 ## S3Guard Command Line Interface (CLI)
 
-Note that in some cases an AWS region or `s3a://` URI can be provided.
-
-Metadata store URIs include a scheme that designates the backing store. For
-example (e.g. `dynamodb://table_name`;). As documented above, the
-AWS region can be inferred if the URI to an existing bucket is provided.
-
-
-The S3A URI must also be provided for per-bucket configuration options
-to be picked up. That is: when an s3a URL is provided on the command line,
-all its "resolved" per-bucket settings are used to connect to, authenticate
-with and configure the S3Guard table. If no such URL is provided, then
-the base settings are picked up.
-
-
-### Create a table: `s3guard init`
-
-```bash
-hadoop s3guard init -meta URI ( -region REGION | s3a://BUCKET )
-```
-
-Creates and initializes an empty metadata store.
-
-A DynamoDB metadata store can be initialized with additional parameters
-pertaining to capacity. 
-
-If these values are both zero, then an on-demand DynamoDB table is created;
-if positive values then they set the
-[Provisioned Throughput](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ProvisionedThroughput.html)
-of the table.
-
-
-```bash
-[-write PROVISIONED_WRITES] [-read PROVISIONED_READS]
-```
-
-Server side encryption (SSE) can be enabled with AWS managed customer master key
-(CMK), or customer managed CMK. By default the DynamoDB table will be encrypted
-with AWS owned CMK. To use a customer managed CMK, you can specify its KMS key
-ID, ARN, alias name, or alias ARN. If not specified, the default AWS managed CMK
-for DynamoDB "alias/aws/dynamodb" will be used.
-
-```bash
-[-sse [-cmk KMS_CMK_ID]]
-```
-
-Tag argument can be added with a key=value list of tags. The table for the
-metadata store will be created with these tags in DynamoDB.
-
-```bash
-[-tag key=value;]
-```
-
-
-Example 1
-
-```bash
-hadoop s3guard init -meta dynamodb://ireland-team -write 0 -read 0 s3a://ireland-1
-```
-
-Creates an on-demand table "ireland-team",
-in the same location as the S3 bucket "ireland-1".
-
-
-Example 2
-
-```bash
-hadoop s3guard init -meta dynamodb://ireland-team -region eu-west-1 --read 0 --write 0
-```
-
-Creates a table "ireland-team" in the region "eu-west-1.amazonaws.com"
-
-
-Example 3
-
-```bash
-hadoop s3guard init -meta dynamodb://ireland-team -tag tag1=first;tag2=second;
-```
-
-Creates a table "ireland-team" with tags "first" and "second". The read and
-write capacity will be those of the site configuration's values of
-`fs.s3a.s3guard.ddb.table.capacity.read` and `fs.s3a.s3guard.ddb.table.capacity.write`;
-if these are both zero then it will be an on-demand table.
-
-
-Example 4
-
-```bash
-hadoop s3guard init -meta dynamodb://ireland-team -sse
-```
-
-Creates a table "ireland-team" with server side encryption enabled. The CMK will
-be using the default AWS managed "alias/aws/dynamodb".
-
-
-### Import a bucket: `s3guard import`
-
-```bash
-hadoop s3guard import [-meta URI] [-authoritative] [-verbose] s3a://PATH
-```
-
-Pre-populates a metadata store according to the current contents of an S3
-bucket/path. If the `-meta` option is omitted, the binding information is taken
-from the `core-site.xml` configuration.
-
-Usage
-
-```
-hadoop s3guard import
-
-import [OPTIONS] [s3a://PATH]
-    import metadata from existing S3 data
-
-Common options:
-  -authoritative - Mark imported directory data as authoritative.
-  -verbose - Verbose Output.
-  -meta URL - Metadata repository details (implementation-specific)
-
-Amazon DynamoDB-specific options:
-  -region REGION - Service region for connections
-
-  URLs for Amazon DynamoDB are of the form dynamodb://TABLE_NAME.
-  Specifying both the -region option and an S3A path
-  is not supported.
-```
-
-Example
-
-Import all files and directories in a bucket into the S3Guard table.
-
-```bash
-hadoop s3guard import s3a://ireland-1
-```
-
-Import a directory tree, marking directories as authoritative.
-
-```bash
-hadoop s3guard import -authoritative -verbose s3a://ireland-1/fork-0008
-
-2020-01-03 12:05:18,321 [main] INFO - Metadata store DynamoDBMetadataStore{region=eu-west-1,
- tableName=s3guard-metadata, tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/s3guard-metadata} is initialized.
-2020-01-03 12:05:18,324 [main] INFO - Starting: Importing s3a://ireland-1/fork-0008
-2020-01-03 12:05:18,324 [main] INFO - Importing directory s3a://ireland-1/fork-0008
-2020-01-03 12:05:18,537 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-0-false
-2020-01-03 12:05:18,630 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-0-true
-2020-01-03 12:05:19,142 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-false/dir-0
-2020-01-03 12:05:19,191 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-false/dir-1
-2020-01-03 12:05:19,240 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-true/dir-0
-2020-01-03 12:05:19,289 [main] INFO - Dir  s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-true/dir-1
-2020-01-03 12:05:19,314 [main] INFO - Updated S3Guard with 0 files and 6 directory entries
-2020-01-03 12:05:19,315 [main] INFO - Marking directory tree s3a://ireland-1/fork-0008 as authoritative
-2020-01-03 12:05:19,342 [main] INFO - Importing s3a://ireland-1/fork-0008: duration 0:01.018s
-Inserted 6 items into Metadata Store
-```
-
-### Compare a S3Guard table and the S3 Store: `s3guard diff`
-
-```bash
-hadoop s3guard diff [-meta URI] s3a://BUCKET
-```
-
-Lists discrepancies between a metadata store and bucket. Note that depending on
-how S3Guard is used, certain discrepancies are to be expected.
-
-Example
-
-```bash
-hadoop s3guard diff s3a://ireland-1
-```
 
 ### Display information about a bucket, `s3guard bucket-info`
 
-Prints and optionally checks the s3guard and encryption status of a bucket.
+Prints and optionally checks the status of a bucket.
 
 ```bash
-hadoop s3guard bucket-info [-guarded] [-unguarded] [-auth] [-nonauth] [-magic] [-encryption ENCRYPTION] s3a://BUCKET
+hadoop s3guard bucket-info [-guarded] [-unguarded] [-auth] [-nonauth] [-magic] [-encryption ENCRYPTION] [-markers MARKER] s3a://BUCKET
 ```
 
 Options
 
 | argument | meaning |
 |-----------|-------------|
-| `-guarded` | Require S3Guard to be enabled |
-| `-unguarded` | Force S3Guard to be disabled |
-| `-auth` | Require the S3Guard mode to be "authoritative" |
-| `-nonauth` | Require the S3Guard mode to be "non-authoritative" |
+| `-guarded` | Require S3Guard to be enabled. This will now always fail |
+| `-unguarded` | Require S3Guard to be disabled. This will now always succeed |
+| `-auth` | Require the S3Guard mode to be "authoritative". This will now always fail |
+| `-nonauth` | Require the S3Guard mode to be "non-authoritative". This will now always fail |
 | `-magic` | Require the S3 filesystem to be support the "magic" committer |
-| `-encryption <type>` | Require a specific server-side encryption algorithm  |
+| `-markers` | Directory marker status: `aware`, `keep`, `delete`, `authoritative` |
+| `-encryption <type>` | Require a specific encryption algorithm  |
 
 The server side encryption options are not directly related to S3Guard, but
 it is often convenient to check them at the same time.
@@ -810,79 +160,36 @@ it is often convenient to check them at the same time.
 Example
 
 ```bash
-hadoop s3guard bucket-info -guarded -magic s3a://ireland-1
-```
-
-List the details of bucket `s3a://ireland-1`, mandating that it must have S3Guard enabled
-("-guarded") and that support for the magic S3A committer is enabled ("-magic")
+> hadoop s3guard bucket-info -magic -markers keep s3a://test-london/
 
-```
-Filesystem s3a://ireland-1
-Location: eu-west-1
-Filesystem s3a://ireland-1 is using S3Guard with store DynamoDBMetadataStore{region=eu-west-1, tableName=ireland-1}
-Authoritative S3Guard: fs.s3a.metadatastore.authoritative=false
-Metadata Store Diagnostics:
-  ARN=arn:aws:dynamodb:eu-west-1:00000000:table/ireland-1
-  billing-mode=provisioned
-  description=S3Guard metadata store in DynamoDB
-  name=ireland-1
-  read-capacity=20
-  region=eu-west-1
-  retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
-  size=12812
-  status=ACTIVE
-  table={AttributeDefinitions: [{AttributeName: child,AttributeType: S},
-    {AttributeName: parent,AttributeType: S}],TableName: ireland-1,
-    KeySchema: [{AttributeName: parent,KeyType: HASH}, {AttributeName: child,KeyType: RANGE}],
-    TableStatus: ACTIVE,
-    CreationDateTime: Fri Aug 25 19:07:25 BST 2017,
-    ProvisionedThroughput: {LastIncreaseDateTime: Tue Aug 29 11:45:18 BST 2017,
-    LastDecreaseDateTime: Wed Aug 30 15:37:51 BST 2017,
-    NumberOfDecreasesToday: 1,
-    ReadCapacityUnits: 20,WriteCapacityUnits: 20},
-    TableSizeBytes: 12812,ItemCount: 91,
-    TableArn: arn:aws:dynamodb:eu-west-1:00000000:table/ireland-1,}
-  write-capacity=20
-The "magic" committer is supported
+2021-11-22 15:21:00,289 [main] INFO  impl.DirectoryPolicyImpl (DirectoryPolicyImpl.java:getDirectoryPolicy(189)) - Directory markers will be kept
+Filesystem s3a://test-london
+Location: eu-west-2
 
 S3A Client
-  Signing Algorithm: fs.s3a.signing-algorithm=(unset)
-  Endpoint: fs.s3a.endpoint=s3-eu-west-1.amazonaws.com
-  Encryption: fs.s3a.server-side-encryption-algorithm=none
-  Input seek policy: fs.s3a.experimental.input.fadvise=normal
-  Change Detection Source: fs.s3a.change.detection.source=etag
-  Change Detection Mode: fs.s3a.change.detection.mode=server
-Delegation token support is disabled
-```
+        Signing Algorithm: fs.s3a.signing-algorithm=(unset)
+        Endpoint: fs.s3a.endpoint=(unset)
+        Encryption: fs.s3a.encryption.algorithm=none
+        Input seek policy: fs.s3a.experimental.input.fadvise=normal
+        Change Detection Source: fs.s3a.change.detection.source=etag
+        Change Detection Mode: fs.s3a.change.detection.mode=server
 
-This listing includes all the information about the table supplied from
-
-```bash
-hadoop s3guard bucket-info -unguarded -encryption none s3a://landsat-pds
-```
-
-List the S3Guard status of clients of the public `landsat-pds` bucket,
-and verifies that the data is neither tracked with S3Guard nor encrypted.
+S3A Committers
+        The "magic" committer is supported in the filesystem
+        S3A Committer factory class: mapreduce.outputcommitter.factory.scheme.s3a=org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
+        S3A Committer name: fs.s3a.committer.name=magic
+        Store magic committer integration: fs.s3a.committer.magic.enabled=true
 
+Security
+        Delegation token support is disabled
 
-```
-Filesystem s3a://landsat-pdsLocation: us-west-2
-Filesystem s3a://landsat-pds is not using S3Guard
-Endpoint: fs.s3a.endpoints3.amazonaws.com
-Encryption: fs.s3a.server-side-encryption-algorithm=none
-Input seek policy: fs.s3a.experimental.input.fadvise=normal
-```
+Directory Markers
+        The directory marker policy is "keep"
+        Available Policies: delete, keep, authoritative
+        Authoritative paths: fs.s3a.authoritative.path=
 
-Note that other clients may have a S3Guard table set up to store metadata
-on this bucket; the checks are all done from the perspective of the configuration
-settings of the current client.
-
-```bash
-hadoop s3guard bucket-info -guarded -auth s3a://landsat-pds
 ```
 
-Require the bucket to be using S3Guard in authoritative mode. This will normally
-fail against this specific bucket.
 
 ### List or Delete Leftover Multipart Uploads: `s3guard uploads`
 
@@ -918,975 +225,3 @@ default, and terminates with a success or failure exit code depending
 on whether or not the supplied number matches the number of uploads
 found that match the given options (path, age).
 
-
-### Delete a table: `s3guard destroy`
-
-Deletes a metadata store. With DynamoDB as the store, this means
-the specific DynamoDB table use to store the metadata.
-
-```bash
-hadoop s3guard destroy [-meta URI] ( -region REGION | s3a://BUCKET )
-```
-
-This *does not* delete the bucket, only the S3Guard table which it is bound
-to.
-
-
-Examples
-
-```bash
-hadoop s3guard destroy s3a://ireland-1
-```
-
-Deletes the table which the bucket ireland-1 is configured to use
-as its MetadataStore.
-
-```bash
-hadoop s3guard destroy -meta dynamodb://ireland-team -region eu-west-1
-```
-
-
-### Clean up a table, `s3guard prune`
-
-Delete all file entries in the MetadataStore table whose object "modification
-time" is older than the specified age.
-
-```bash
-hadoop s3guard prune [-days DAYS] [-hours HOURS] [-minutes MINUTES]
-    [-seconds SECONDS] [-tombstone] [-meta URI] ( -region REGION | s3a://BUCKET )
-```
-
-A time value of hours, minutes and/or seconds must be supplied.
-
-1. This does not delete the entries in the bucket itself.
-1. The modification time is effectively the creation time of the objects
-in the S3 Bucket.
-1. If an S3A URI is supplied, only the entries in the table specified by the
-URI and older than a specific age are deleted.
-
-
-The `-tombstone` option instructs the operation to only purge "tombstones",
-markers of deleted files. These tombstone markers are only used briefly,
-to indicate that a recently deleted file should not be found in listings.
-As a result, there is no adverse consequences in regularly pruning old
-tombstones.
-
-Example
-
-```bash
-hadoop s3guard prune -days 7 s3a://ireland-1
-```
-
-Deletes all entries in the S3Guard table for files older than seven days from
-the table associated with `s3a://ireland-1`.
-
-```bash
-hadoop s3guard prune -tombstone -days 7 s3a://ireland-1/path_prefix/
-```
-
-Deletes all entries in the S3Guard table for tombstones older than seven days from
-the table associated with `s3a://ireland-1` and with the prefix `path_prefix`
-
-```bash
-hadoop s3guard prune -hours 1 -minutes 30 -meta dynamodb://ireland-team -region eu-west-1
-```
-
-Delete all file entries more than 90 minutes old from the table "`ireland-team"` in
-the region `eu-west-1`.
-
-
-### Audit the "authoritative state of a DynamoDB Table, `s3guard authoritative`
-
-This recursively checks a S3Guard table to verify that all directories
-underneath are marked as "authoritative", and/or that the configuration
-is set for the S3A client to treat files and directories urnder the path
-as authoritative.
-
-```
-hadoop s3guard authoritative
-
-authoritative [OPTIONS] [s3a://PATH]
-    Audits a DynamoDB S3Guard repository for all the entries being 'authoritative'
-
-Options:
-  -required Require directories under the path to be authoritative.
-  -check-config Check the configuration for the path to be authoritative
-  -verbose Verbose Output.
-```
-
-Verify that a path under an object store is declared to be authoritative
-in the cluster configuration -and therefore that file entries will not be
-validated against S3, and that directories marked as "authoritative" in the
-S3Guard table will be treated as complete.
-
-```bash
-hadoop s3guard authoritative -check-config s3a:///ireland-1/fork-0003/test/
-
-2020-01-03 11:42:29,147 [main] INFO  Metadata store DynamoDBMetadataStore{
-  region=eu-west-1, tableName=s3guard-metadata, tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/s3guard-metadata} is initialized.
-Path /fork-0003/test is not configured to be authoritative
-```
-
-Scan a store and report which directories are not marked as authoritative.
-
-```bash
-hadoop s3guard authoritative s3a://ireland-1/
-
-2020-01-03 11:51:58,416 [main] INFO  - Metadata store DynamoDBMetadataStore{region=eu-west-1, tableName=s3guard-metadata, tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/s3guard-metadata} is initialized.
-2020-01-03 11:51:58,419 [main] INFO  - Starting: audit s3a://ireland-1/
-2020-01-03 11:51:58,422 [main] INFO  - Root directory s3a://ireland-1/
-2020-01-03 11:51:58,469 [main] INFO  -   files 4; directories 12
-2020-01-03 11:51:58,469 [main] INFO  - Directory s3a://ireland-1/Users
-2020-01-03 11:51:58,521 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,522 [main] INFO  - Directory s3a://ireland-1/fork-0007
-2020-01-03 11:51:58,573 [main] INFO  - Directory s3a://ireland-1/fork-0001
-2020-01-03 11:51:58,626 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,626 [main] INFO  - Directory s3a://ireland-1/fork-0006
-2020-01-03 11:51:58,676 [main] INFO  - Directory s3a://ireland-1/path
-2020-01-03 11:51:58,734 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,735 [main] INFO  - Directory s3a://ireland-1/fork-0008
-2020-01-03 11:51:58,802 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,802 [main] INFO  - Directory s3a://ireland-1/fork-0004
-2020-01-03 11:51:58,854 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,855 [main] WARN   - Directory s3a://ireland-1/fork-0003 is not authoritative
-2020-01-03 11:51:58,905 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:58,906 [main] INFO  - Directory s3a://ireland-1/fork-0005
-2020-01-03 11:51:58,955 [main] INFO  - Directory s3a://ireland-1/customsignerpath2
-2020-01-03 11:51:59,006 [main] INFO  - Directory s3a://ireland-1/fork-0002
-2020-01-03 11:51:59,063 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:59,064 [main] INFO  - Directory s3a://ireland-1/customsignerpath1
-2020-01-03 11:51:59,121 [main] INFO  - Directory s3a://ireland-1/Users/stevel
-2020-01-03 11:51:59,170 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:59,171 [main] INFO  - Directory s3a://ireland-1/fork-0001/test
-2020-01-03 11:51:59,233 [main] INFO  - Directory s3a://ireland-1/path/style
-2020-01-03 11:51:59,282 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:59,282 [main] INFO  - Directory s3a://ireland-1/fork-0008/test
-2020-01-03 11:51:59,338 [main] INFO  -   files 15; directories 10
-2020-01-03 11:51:59,339 [main] INFO  - Directory s3a://ireland-1/fork-0004/test
-2020-01-03 11:51:59,394 [main] WARN   - Directory s3a://ireland-1/fork-0003/test is not authoritative
-2020-01-03 11:51:59,451 [main] INFO  -   files 35; directories 1
-2020-01-03 11:51:59,451 [main] INFO  - Directory s3a://ireland-1/fork-0002/test
-2020-01-03 11:51:59,508 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects
-2020-01-03 11:51:59,558 [main] INFO  -   files 0; directories 1
-2020-01-03 11:51:59,559 [main] INFO  - Directory s3a://ireland-1/path/style/access
-2020-01-03 11:51:59,610 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-2-0-false
-2020-01-03 11:51:59,660 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-2-1-false
-2020-01-03 11:51:59,719 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-0-true
-2020-01-03 11:51:59,773 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-true
-2020-01-03 11:51:59,824 [main] INFO  -   files 0; directories 2
-2020-01-03 11:51:59,824 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-2-1-true
-2020-01-03 11:51:59,879 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-1-false
-2020-01-03 11:51:59,939 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-0-false
-2020-01-03 11:51:59,990 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-2-0-true
-2020-01-03 11:52:00,042 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-false
-2020-01-03 11:52:00,094 [main] INFO  -   files 0; directories 2
-2020-01-03 11:52:00,094 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-0-0-1-true
-2020-01-03 11:52:00,144 [main] WARN   - Directory s3a://ireland-1/fork-0003/test/ancestor is not authoritative
-2020-01-03 11:52:00,197 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk
-2020-01-03 11:52:00,245 [main] INFO  -   files 0; directories 1
-2020-01-03 11:52:00,245 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-true/dir-0
-2020-01-03 11:52:00,296 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-true/dir-1
-2020-01-03 11:52:00,346 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-false/dir-0
-2020-01-03 11:52:00,397 [main] INFO  - Directory s3a://ireland-1/fork-0008/test/doTestListFiles-2-0-0-false/dir-1
-2020-01-03 11:52:00,479 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools
-2020-01-03 11:52:00,530 [main] INFO  -   files 0; directories 1
-2020-01-03 11:52:00,530 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws
-2020-01-03 11:52:00,582 [main] INFO  -   files 0; directories 1
-2020-01-03 11:52:00,582 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target
-2020-01-03 11:52:00,636 [main] INFO  -   files 0; directories 1
-2020-01-03 11:52:00,637 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir
-2020-01-03 11:52:00,691 [main] INFO  -   files 0; directories 3
-2020-01-03 11:52:00,691 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir/2
-2020-01-03 11:52:00,752 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir/5
-2020-01-03 11:52:00,807 [main] INFO  - Directory s3a://ireland-1/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir/8
-2020-01-03 11:52:00,862 [main] INFO   - Scanned 45 directories - 3 were not marked as authoritative
-2020-01-03 11:52:00,863 [main] INFO  - audit s3a://ireland-1/: duration 0:02.444s
-```
-
-Scan the path/bucket and fail if any entry is non-authoritative.
-
-```bash
-hadoop s3guard authoritative -verbose -required s3a://ireland-1/
-
-2020-01-03 11:47:40,288 [main] INFO  - Metadata store DynamoDBMetadataStore{region=eu-west-1, tableName=s3guard-metadata, tableArn=arn:aws:dynamodb:eu-west-1:980678866538:table/s3guard-metadata} is initialized.
-2020-01-03 11:47:40,291 [main] INFO  - Starting: audit s3a://ireland-1/
-2020-01-03 11:47:40,295 [main] INFO  - Root directory s3a://ireland-1/
-2020-01-03 11:47:40,336 [main] INFO  -  files 4; directories 12
-2020-01-03 11:47:40,336 [main] INFO  - Directory s3a://ireland-1/Users
-2020-01-03 11:47:40,386 [main] INFO  -  files 0; directories 1
-2020-01-03 11:47:40,386 [main] INFO  - Directory s3a://ireland-1/fork-0007
-2020-01-03 11:47:40,435 [main] INFO  -  files 1; directories 0
-2020-01-03 11:47:40,435 [main] INFO  - Directory s3a://ireland-1/fork-0001
-2020-01-03 11:47:40,486 [main] INFO  -  files 0; directories 1
-2020-01-03 11:47:40,486 [main] INFO  - Directory s3a://ireland-1/fork-0006
-2020-01-03 11:47:40,534 [main] INFO  -  files 1; directories 0
-2020-01-03 11:47:40,535 [main] INFO  - Directory s3a://ireland-1/path
-2020-01-03 11:47:40,587 [main] INFO  -  files 0; directories 1
-2020-01-03 11:47:40,588 [main] INFO  - Directory s3a://ireland-1/fork-0008
-2020-01-03 11:47:40,641 [main] INFO  -  files 0; directories 1
-2020-01-03 11:47:40,642 [main] INFO  - Directory s3a://ireland-1/fork-0004
-2020-01-03 11:47:40,692 [main] INFO  -  files 0; directories 1
-2020-01-03 11:47:40,693 [main] WARN  - Directory s3a://ireland-1/fork-0003 is not authoritative
-2020-01-03 11:47:40,693 [main] INFO  - audit s3a://ireland-1/: duration 0:00.402s
-2020-01-03 11:47:40,698 [main] INFO  - Exiting with status 46: `s3a://ireland-1/fork-0003': Directory is not marked as authoritative in the S3Guard store
-```
-
-This command is primarily for testing.
-
-### Tune the I/O capacity of the DynamoDB Table, `s3guard set-capacity`
-
-Alter the read and/or write capacity of a s3guard table created with provisioned
-I/O capacity.
-
-```bash
-hadoop s3guard set-capacity [--read UNIT] [--write UNIT] ( -region REGION | s3a://BUCKET )
-```
-
-The `--read` and `--write` units are those of `s3guard init`.
-
-It cannot be used to change the I/O capacity of an on demand table (there is
-no need), and nor can it be used to convert an existing table to being
-on-demand. For that the AWS console must be used.
-
-Example
-
-```
-hadoop s3guard set-capacity  -read 20 -write 20 s3a://ireland-1
-```
-
-Set the capacity of the table used by bucket `s3a://ireland-1` to 20 read
-and 20 write. (This is a low number, incidentally)
-
-```
-2017-08-30 16:21:26,343 [main] INFO  s3guard.S3GuardTool (S3GuardTool.java:initMetadataStore(229)) - Metadata store DynamoDBMetadataStore{region=eu-west-1, tableName=ireland-1} is initialized.
-2017-08-30 16:21:26,344 [main] INFO  s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1084)) - Current table capacity is read: 25, write: 25
-2017-08-30 16:21:26,344 [main] INFO  s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1086)) - Changing capacity of table to read: 20, write: 20
-Metadata Store Diagnostics:
-  ARN=arn:aws:dynamodb:eu-west-1:00000000000:table/ireland-1
-  billing-mode=provisioned
-  description=S3Guard metadata store in DynamoDB
-  name=ireland-1
-  read-capacity=25
-  region=eu-west-1
-  retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
-  size=12812
-  status=UPDATING
-  table={ ... }
-  write-capacity=25
-```
-
-After the update, the table status changes to `UPDATING`; this is a sign that
-the capacity has been changed.
-
-Repeating the same command will not change the capacity, as both read and
-write values match that already in use.
-
-```
-2017-08-30 16:24:35,337 [main] INFO  s3guard.DynamoDBMetadataStore (DynamoDBMetadataStore.java:updateParameters(1090)) - Table capacity unchanged at read: 20, write: 20
-Metadata Store Diagnostics:
-  ARN=arn:aws:dynamodb:eu-west-1:00000000000:table/ireland-1
-  billing-mode=provisioned
-  description=S3Guard metadata store in DynamoDB
-  name=ireland-1
-  read-capacity=20
-  region=eu-west-1
-  retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=100 MILLISECONDS)
-  size=12812
-  status=ACTIVE
-  table={ ... }
-  write-capacity=20
-```
-*Note*: There is a limit to how many times in a 24 hour period the capacity
-of a bucket can be changed, either through this command or the AWS console.
-
-### Check the consistency of the metadata store, `s3guard fsck`
-
-Compares S3 with MetadataStore, and returns a failure status if any
-rules or invariants are violated. Only works with DynamoDB metadata stores.
-
-```bash
-hadoop s3guard fsck [-check | -internal] [-fix] (s3a://BUCKET | s3a://PATH_PREFIX)
-```
-
-`-check` operation checks the metadata store from the S3 perspective, but
-does not fix any issues.
-The consistency issues will be logged in ERROR loglevel.
-
-`-internal` operation checks the internal consistency of the metadata store,
-but does not fix any issues.
-
-`-fix` operation fixes consistency issues between the metadatastore and the S3
-bucket. This parameter is optional, and can be used together with check or
-internal parameters, but not alone.
-The following fix is implemented:
-- Remove orphan entries from DDB
-
-The errors found will be logged at the ERROR log level.
-
-*Note*: `-check` and `-internal` operations can be used only as separate
-commands. Running `fsck` with both will result in an error.
-
-Example
-
-```bash
-hadoop s3guard fsck -check s3a://ireland-1/path_prefix/
-```
-
-Checks the metadata store while iterating through the S3 bucket.
-The path_prefix will be used as the root element of the check.
-
-```bash
-hadoop s3guard fsck -internal s3a://ireland-1/path_prefix/
-```
-
-Checks the metadata store internal consistency.
-The path_prefix will be used as the root element of the check.
-
-
-## Debugging and Error Handling
-
-If you run into network connectivity issues, or have a machine failure in the
-middle of an operation, you may end up with your metadata store having state
-that differs from S3.  The S3Guard CLI commands, covered in the CLI section
-above, can be used to diagnose and repair these issues.
-
-There are some logs whose log level can be increased to provide more
-information.
-
-```properties
-# Log S3Guard classes
-log4j.logger.org.apache.hadoop.fs.s3a.s3guard=DEBUG
-
-# Log all S3A classes
-log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
-
-# Enable debug logging of AWS DynamoDB client
-log4j.logger.com.amazonaws.services.dynamodbv2.AmazonDynamoDB
-
-# Log all HTTP requests made; includes S3 interaction. This may
-# include sensitive information such as account IDs in HTTP headers.
-log4j.logger.com.amazonaws.request=DEBUG
-```
-
-If all else fails, S3Guard is designed to allow for easy recovery by deleting
-the metadata store data. In DynamoDB, this can be accomplished by simply
-deleting the table, and allowing S3Guard to recreate it from scratch.  Note
-that S3Guard tracks recent changes to file metadata to implement consistency.
-Deleting the metadata store table will simply result in a period of eventual
-consistency for any file modifications that were made right before the table
-was deleted.
-
-### Enabling a log message whenever S3Guard is *disabled*
-
-When dealing with support calls related to the S3A connector, "is S3Guard on?"
-is the usual opening question. This can be determined by looking at the application logs for
-messages about S3Guard starting -the absence of S3Guard can only be inferred by the absence
-of such messages.
-
-There is a another strategy: have the S3A Connector log whenever *S3Guard is not enabled*
-
-This can be done in the configuration option `fs.s3a.s3guard.disabled.warn.level`
-
-```xml
-<property>
- <name>fs.s3a.s3guard.disabled.warn.level</name>
- <value>silent</value>
- <description>
-   Level to print a message when S3Guard is disabled.
-   Values: 
-   "warn": log at WARN level
-   "inform": log at INFO level
-   "silent": log at DEBUG level
-   "fail": raise an exception
- </description>
-</property>
-```
-
-The `fail` option is clearly more than logging; it exists as an extreme debugging
-tool. Use with care.
-
-### Failure Semantics
-
-Operations which modify metadata will make changes to S3 first. If, and only
-if, those operations succeed, the equivalent changes will be made to the
-Metadata Store.
-
-These changes to S3 and Metadata Store are not fully-transactional:  If the S3
-operations succeed, and the subsequent Metadata Store updates fail, the S3
-changes will *not* be rolled back.  In this case, an error message will be
-logged.
-
-### Versioning
-
-S3Guard tables are created with a version marker entry and table tag.
-The entry is created with the primary key and child entry of `../VERSION`; 
-the use of a relative path guarantees that it will not be resolved.
-Table tag key is named `s3guard_version`.
-
-When the table is initialized by S3Guard, the table will be tagged during the 
-creating and the version marker entry will be created in the table.
-If the table lacks the version marker entry or tag, S3Guard will try to create
-it according to the following rules:
-
-1. If the table lacks both version markers AND it's empty, both markers will be added. 
-If the table is not empty the check throws IOException
-1. If there's no version marker ITEM, the compatibility with the TAG
-will be checked, and the version marker ITEM will be added if the
-TAG version is compatible.
-If the TAG version is not compatible, the check throws OException
-1. If there's no version marker TAG, the compatibility with the ITEM
-version marker will be checked, and the version marker ITEM will be
-added if the ITEM version is compatible.
-If the ITEM version is not compatible, the check throws IOException
-1. If the TAG and ITEM versions are both present then both will be checked
-for compatibility. If the ITEM or TAG version marker is not compatible,
-the check throws IOException
-
-*Note*: If the user does not have sufficient rights to tag the table the 
-initialization of S3Guard will not fail, but there will be no version marker tag
-on the dynamo table.
-
-*Versioning policy*
-
-1. The version number of an S3Guard table will only be incremented when
-an incompatible change is made to the table structure —that is, the structure
-has changed so that it is no longer readable by older versions, or because
-it has added new mandatory fields which older versions do not create.
-1. The version number of S3Guard tables will only be changed by incrementing
-the value.
-1. Updated versions of S3Guard MAY continue to support older version tables.
-1. If an incompatible change is made such that existing tables are not compatible,
-then a means shall be provided to update existing tables. For example:
-an option in the Command Line Interface, or an option to upgrade tables
-during S3Guard initialization.
-
-*Note*: this policy does not indicate any intent to upgrade table structures
-in an incompatible manner. The version marker in tables exists to support
-such an option if it ever becomes necessary, by ensuring that all S3Guard
-client can recognise any version mismatch.
-
-## Security
-
-All users of the DynamoDB table must have write access to it. This
-effectively means they must have write access to the entire object store.
-
-There's not been much testing of using a S3Guard Metadata Store
-with a read-only S3 Bucket. It *should* work, provided all users
-have write access to the DynamoDB table. And, as updates to the Metadata Store
-are only made after successful file creation, deletion and rename, the
-store is *unlikely* to get out of sync, it is still something which
-merits more testing before it could be considered reliable.
-
-## Managing DynamoDB I/O Capacity
-
-Historically, DynamoDB has been not only billed on use (data and I/O requests)
--but on provisioned I/O Capacity.
-
-With Provisioned IO, when an application makes more requests than
-the allocated capacity permits, the request is rejected; it is up to
-the calling application to detect when it is being so throttled and
-react. S3Guard does this, but as a result: when the client is being
-throttled, operations are slower. This capacity throttling is averaged
-over a few minutes: a briefly overloaded table will not be throttled,
-but the rate cannot be sustained.
-
-The load on a table is visible in the AWS console: go to the
-DynamoDB page for the table and select the "metrics" tab.
-If the graphs of throttled read or write
-requests show that a lot of throttling has taken place, then there is not
-enough allocated capacity for the applications making use of the table.
-
-Similarly, if the capacity graphs show that the read or write loads are
-low compared to the allocated capacities, then the table *may* be overprovisioned
-for the current workload.
-
-The S3Guard connector to DynamoDB can be configured to make
-multiple attempts to repeat a throttled request, with an exponential
-backoff between them.
-
-The relevant settings for managing retries in the connector are:
-
-```xml
-
-<property>
-  <name>fs.s3a.s3guard.ddb.max.retries</name>
-  <value>9</value>
-    <description>
-      Max retries on throttled/incompleted DynamoDB operations
-      before giving up and throwing an IOException.
-      Each retry is delayed with an exponential
-      backoff timer which starts at 100 milliseconds and approximately
-      doubles each time.  The minimum wait before throwing an exception is
-      sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.throttle.retry.interval</name>
-  <value>100ms</value>
-    <description>
-      Initial interval to retry after a request is throttled events;
-      the back-off policy is exponential until the number of retries of
-      fs.s3a.s3guard.ddb.max.retries is reached.
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.background.sleep</name>
-  <value>25ms</value>
-  <description>
-    Length (in milliseconds) of pause between each batch of deletes when
-    pruning metadata.  Prevents prune operations (which can typically be low
-    priority background operations) from overly interfering with other I/O
-    operations.
-  </description>
-</property>
-```
-
-Having a large value for `fs.s3a.s3guard.ddb.max.retries` will ensure
-that clients of an overloaded table will not fail immediately. However
-queries may be unexpectedly slow.
-
-If operations, especially directory operations, are slow, check the AWS
-console. It is also possible to set up AWS alerts for capacity limits
-being exceeded.
-
-### <a name="on-demand"></a> On-Demand Dynamo Capacity
-
-[Amazon DynamoDB On-Demand](https://aws.amazon.com/blogs/aws/amazon-dynamodb-on-demand-no-capacity-planning-and-pay-per-request-pricing/)
-removes the need to pre-allocate I/O capacity for S3Guard tables.
-Instead the caller is _only_ charged per I/O Operation.
-
-* There are no SLA capacity guarantees. This is generally not an issue
-for S3Guard applications.
-* There's no explicit limit on I/O capacity, so operations which make
-heavy use of S3Guard tables (for example: SQL query planning) do not
-get throttled.
-* You are charged more per DynamoDB API call, in exchange for paying nothing
-when you are not interacting with DynamoDB.
-* There's no way put a limit on the I/O; you may unintentionally run up
-large bills through sustained heavy load.
-* The `s3guard set-capacity` command fails: it does not make sense any more.
-
-When idle, S3Guard tables are only billed for the data stored, not for
-any unused capacity. For this reason, there is no performance benefit
-from sharing a single S3Guard table across multiple buckets.
-
-*Creating a S3Guard Table with On-Demand Tables*
-
-The default settings for S3Guard are to create on-demand tables; this
-can also be done explicitly in the `s3guard init` command by setting the
-read and write capacities to zero.
-
-
-```bash
-hadoop s3guard init -meta dynamodb://ireland-team -write 0 -read 0 s3a://ireland-1
-```
-
-*Enabling DynamoDB On-Demand for an existing S3Guard table*
-
-You cannot currently convert an existing S3Guard table to being an on-demand
-table through the `s3guard` command.
-
-It can be done through the AWS console or [the CLI](https://docs.aws.amazon.com/cli/latest/reference/dynamodb/update-table.html).
-From the Web console or the command line, switch the billing to pay-per-request.
-
-Once enabled, the read and write capacities of the table listed in the
-`hadoop s3guard bucket-info` command become "0", and the "billing-mode"
-attribute changes to "per-request":
-
-```
-> hadoop s3guard bucket-info s3a://example-bucket/
-
-Filesystem s3a://example-bucket
-Location: eu-west-1
-Filesystem s3a://example-bucket is using S3Guard with store
-  DynamoDBMetadataStore{region=eu-west-1, tableName=example-bucket,
-  tableArn=arn:aws:dynamodb:eu-west-1:11111122223333:table/example-bucket}
-Authoritative S3Guard: fs.s3a.metadatastore.authoritative=false
-Metadata Store Diagnostics:
-  ARN=arn:aws:dynamodb:eu-west-1:11111122223333:table/example-bucket
-  billing-mode=per-request
-  description=S3Guard metadata store in DynamoDB
-  name=example-bucket
-  persist.authoritative.bit=true
-  read-capacity=0
-  region=eu-west-1
-  retryPolicy=ExponentialBackoffRetry(maxRetries=9, sleepTime=250 MILLISECONDS)
-  size=66797
-  status=ACTIVE
-  table={AttributeDefinitions:
-    [{AttributeName: child,AttributeType: S},
-     {AttributeName: parent,AttributeType: S}],
-     TableName: example-bucket,
-     KeySchema: [{
-       AttributeName: parent,KeyType: HASH},
-       {AttributeName: child,KeyType: RANGE}],
-     TableStatus: ACTIVE,
-     CreationDateTime: Thu Oct 11 18:51:14 BST 2018,
-     ProvisionedThroughput: {
-       LastIncreaseDateTime: Tue Oct 30 16:48:45 GMT 2018,
-       LastDecreaseDateTime: Tue Oct 30 18:00:03 GMT 2018,
-       NumberOfDecreasesToday: 0,
-       ReadCapacityUnits: 0,
-       WriteCapacityUnits: 0},
-     TableSizeBytes: 66797,
-     ItemCount: 415,
-     TableArn: arn:aws:dynamodb:eu-west-1:11111122223333:table/example-bucket,
-     TableId: a7b0728a-f008-4260-b2a0-aaaaabbbbb,}
-  write-capacity=0
-The "magic" committer is supported
-```
-
-### <a name="autoscaling"></a> Autoscaling (Provisioned Capacity) S3Guard tables.
-
-[DynamoDB Auto Scaling](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AutoScaling.html)
-can automatically increase and decrease the allocated capacity.
-
-Before DynamoDB On-Demand was introduced, autoscaling was the sole form
-of dynamic scaling. 
-
-Experiments with S3Guard and DynamoDB Auto Scaling have shown that any Auto Scaling
-operation will only take place after callers have been throttled for a period of
-time. The clients will still need to be configured to retry when overloaded
-until any extra capacity is allocated. Furthermore, as this retrying will
-block the threads from performing other operations -including more I/O, the
-the autoscale may not scale fast enough.
-
-This is why the DynamoDB On-Demand appears is a better option for
-workloads with Hadoop, Spark, Hive and other applications.
-
-If autoscaling is to be used, we recommend experimenting with the option,
-based on usage information collected from previous days, and choosing a
-combination of retry counts and an interval which allow for the clients to cope with
-some throttling, but not to time-out other applications.
-
-## Read-After-Overwrite Consistency
-
-S3Guard provides read-after-overwrite consistency through ETags (default) or
-object versioning checked either on the server (default) or client. This works
-such that a reader reading a file after an overwrite either sees the new version
-of the file or an error. Without S3Guard, new readers may see the original
-version. Once S3 reaches eventual consistency, new readers will see the new
-version.
-
-Readers using S3Guard will usually see the new file version, but may
-in rare cases see `RemoteFileChangedException` instead. This would occur if
-an S3 object read cannot provide the version tracked in S3Guard metadata.
-
-S3Guard achieves this behavior by storing ETags and object version IDs in the
-S3Guard metadata store (e.g. DynamoDB). On opening a file, S3AFileSystem
-will look in S3 for the version of the file indicated by the ETag or object
-version ID stored in the metadata store. If that version is unavailable,
-`RemoteFileChangedException` is thrown. Whether ETag or version ID and
-server or client mode is used is determed by the
-[fs.s3a.change.detection configuration options](./index.html#Handling_Read-During-Overwrite).
-
-### No Versioning Metadata Available
-
-When the first S3AFileSystem clients are upgraded to a version of
-`S3AFileSystem` that contains these change tracking features, any existing
-S3Guard metadata will not contain ETags or object version IDs. Reads of files
-tracked in such S3Guard metadata will access whatever version of the file is
-available in S3 at the time of read. Only if the file is subsequently updated
-will S3Guard start tracking ETag and object version ID and as such generating
-`RemoteFileChangedException` if an inconsistency is detected.
-
-Similarly, when S3Guard metadata is pruned, S3Guard will no longer be able to
-detect an inconsistent read. S3Guard metadata should be retained for at least
-as long as the perceived possible read-after-overwrite temporary inconsistency
-window. That window is expected to be short, but there are no guarantees so it
-is at the administrator's discretion to weigh the risk.
-
-### Known Limitations
-
-#### S3 Select
-
-S3 Select does not provide a capability for server-side ETag or object
-version ID qualification. Whether `fs.s3a.change.detection.mode` is "client" or
-"server", S3Guard will cause a client-side check of the file version before
-opening the file with S3 Select. If the current version does not match the
-version tracked in S3Guard, `RemoteFileChangedException` is thrown.
-
-It is still possible that the S3 Select read will access a different version of
-the file, if the visible file version changes between the version check and
-the opening of the file. This can happen due to eventual consistency or
-an overwrite of the file between the version check and the open of the file.
-
-#### Rename
-
-Rename is implemented via copy in S3. With `fs.s3a.change.detection.mode` set
-to "client", a fully reliable mechansim for ensuring the copied content is the expected
-content is not possible. This is the case since there isn't necessarily a way
-to know the expected ETag or version ID to appear on the object resulting from
-the copy.
-
-Furthermore, if `fs.s3a.change.detection.mode` is "server" and a third-party S3
-implementation is used that doesn't honor the provided ETag or version ID,
-S3AFileSystem and S3Guard cannot detect it.
-
-When `fs.s3.change.detection.mode` is "client", a client-side check
-will be performed before the copy to ensure the current version of the file
-matches S3Guard metadata. If not, `RemoteFileChangedException` is thrown.
-Similar to as discussed with regard to S3 Select, this is not sufficient to
-guarantee that same version is the version copied.
-
-When `fs.s3.change.detection.mode` server, the expected version is also specified
-in the underlying S3 `CopyObjectRequest`. As long as the server honors it, the
-copied object will be correct.
-
-All this said, with the defaults of `fs.s3.change.detection.mode` of "server" and
-`fs.s3.change.detection.source` of "etag", when working with Amazon's S3, copy should in fact
-either copy the expected file version or, in the case of an eventual consistency
-anomaly, generate `RemoteFileChangedException`. The same should be true when
-`fs.s3.change.detection.source` = "versionid".
-
-#### Out of Sync Metadata
-
-The S3Guard version tracking metadata (ETag or object version ID) could become
-out of sync with the true current object metadata in S3.  For example, S3Guard
-is still tracking v1 of some file after v2 has been written.  This could occur
-for reasons such as a writer writing without utilizing S3Guard and/or
-S3AFileSystem or simply due to a write with S3AFileSystem and S3Guard that wrote
-successfully to S3, but failed in communication with S3Guard's metadata store
-(e.g. DynamoDB).
-
-If this happens, reads of the affected file(s) will result in
-`RemoteFileChangedException` until one of:
-
-* the S3Guard metadata is corrected out-of-band
-* the file is overwritten (causing an S3Guard metadata update)
-* the S3Guard metadata is pruned
-
-The S3Guard metadata for a file can be corrected with the `s3guard import`
-command as discussed above. The command can take a file URI instead of a
-bucket URI to correct the metadata for a single file. For example:
-
-```bash
-hadoop s3guard import [-meta URI] s3a://my-bucket/file-with-bad-metadata
-```
-
-## Troubleshooting
-
-### Error: `S3Guard table lacks version marker.`
-
-The table which was intended to be used as a S3guard metadata store
-does not have any version marker indicating that it is a S3Guard table.
-
-It may be that this is not a S3Guard table.
-
-* Make sure that this is the correct table name.
-* Delete the table, so it can be rebuilt.
-
-### Error: `Database table is from an incompatible S3Guard version`
-
-This indicates that the version of S3Guard which created (or possibly updated)
-the database table is from a different version that that expected by the S3A
-client.
-
-This error will also include the expected and actual version numbers.
-
-If the expected version is lower than the actual version, then the version
-of the S3A client library is too old to interact with this S3Guard-managed
-bucket. Upgrade the application/library.
-
-If the expected version is higher than the actual version, then the table
-itself will need upgrading.
-
-### Error `"DynamoDB table TABLE does not exist in region REGION; auto-creation is turned off"`
-
-S3Guard could not find the DynamoDB table for the Metadata Store,
-and it was not configured to create it. Either the table was missing,
-or the configuration is preventing S3Guard from finding the table.
-
-1. Verify that the value of `fs.s3a.s3guard.ddb.table` is correct.
-1. If the region for an existing table has been set in
-`fs.s3a.s3guard.ddb.region`, verify that the value is correct.
-1. If the region is not set, verify that the table exists in the same
-region as the bucket being used.
-1. Create the table if necessary.
-
-
-### Error `"The level of configured provisioned throughput for the table was exceeded"`
-
-```
-org.apache.hadoop.fs.s3a.AWSServiceThrottledException: listFiles on s3a://bucket/10/d1/d2/d3:
-com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException:
-The level of configured provisioned throughput for the table was exceeded.
-Consider increasing your provisioning level with the UpdateTable API.
-(Service: AmazonDynamoDBv2; Status Code: 400;
-Error Code: ProvisionedThroughputExceededException;
-```
-The I/O load of clients of the (shared) DynamoDB table was exceeded.
-
-1. Switch to On-Demand Dynamo DB tables (AWS console)
-1. Increase the capacity of the DynamoDB table (AWS console or `s3guard set-capacity`)/
-1. Increase the retry count and/or sleep time of S3Guard on throttle events (Hadoop configuration).
-
-### Error `Max retries exceeded`
-
-The I/O load of clients of the (shared) DynamoDB table was exceeded, and
-the number of attempts to retry the operation exceeded the configured amount.
-
-1. Switch to On-Demand Dynamo DB tables (AWS console).
-1. Increase the capacity of the DynamoDB table.
-1. Increase the retry count and/or sleep time of S3Guard on throttle events.
-
-
-### Error when running `set-capacity`: `org.apache.hadoop.fs.s3a.AWSServiceThrottledException: ProvisionTable`
-
-```
-org.apache.hadoop.fs.s3a.AWSServiceThrottledException: ProvisionTable on s3guard-example:
-com.amazonaws.services.dynamodbv2.model.LimitExceededException:
-Subscriber limit exceeded: Provisioned throughput decreases are limited within a given UTC day.
-After the first 4 decreases, each subsequent decrease in the same UTC day can be performed at most once every 3600 seconds.
-Number of decreases today: 6.
-Last decrease at Wednesday, July 25, 2018 8:48:14 PM UTC.
-Next decrease can be made at Wednesday, July 25, 2018 9:48:14 PM UTC
-```
-
-There's are limit on how often you can change the capacity of an DynamoDB table;
-if you call `set-capacity` too often, it fails. Wait until the after the time indicated
-and try again.
-
-### Error `Invalid region specified`
-
-```
-java.io.IOException: Invalid region specified "iceland-2":
-  Region can be configured with fs.s3a.s3guard.ddb.region:
-  us-gov-west-1, us-east-1, us-east-2, us-west-1, us-west-2,
-  eu-west-1, eu-west-2, eu-west-3, eu-central-1, ap-south-1,
-  ap-southeast-1, ap-southeast-2, ap-northeast-1, ap-northeast-2,
-  sa-east-1, cn-north-1, cn-northwest-1, ca-central-1
-  at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.getRegion
-  at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.createDynamoDBClient
-```
-
-The region specified in `fs.s3a.s3guard.ddb.region` is invalid.
-
-### "Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when BillingMode is PAY_PER_REQUEST"
-
-```
-ValidationException; One or more parameter values were invalid:
-  Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when
-  BillingMode is PAY_PER_REQUEST
-  (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ValidationException)
-```
-
-On-Demand DynamoDB tables do not have any fixed capacity -it is an error
-to try to change it with the `set-capacity` command.
-
-### `MetadataPersistenceException`
-
-A filesystem write operation failed to persist metadata to S3Guard. The file was
-successfully written to S3 and now the S3Guard metadata is likely to be out of
-sync.
-
-See [Fail on Error](#fail-on-error) for more detail.
-
-### Error `RemoteFileChangedException`
-
-An exception like the following could occur for a couple of reasons:
-
-* the S3Guard metadata is out of sync with the true S3 metadata.  For
-example, the S3Guard DynamoDB table is tracking a different ETag than the ETag
-shown in the exception.  This may suggest the object was updated in S3 without
-involvement from S3Guard or there was a transient failure when S3Guard tried to
-write to DynamoDB.
-
-* S3 is exhibiting read-after-overwrite temporary inconsistency.  The S3Guard
-metadata was updated with a new ETag during a recent write, but the current read
-is not seeing that ETag due to S3 eventual consistency.  This exception prevents
-the reader from an inconsistent read where the reader sees an older version of
-the file.
-
-```
-org.apache.hadoop.fs.s3a.RemoteFileChangedException: open 's3a://my-bucket/test/file.txt':
-  Change reported by S3 while reading at position 0.
-  ETag 4e886e26c072fef250cfaf8037675405 was unavailable
-  at org.apache.hadoop.fs.s3a.impl.ChangeTracker.processResponse(ChangeTracker.java:167)
-  at org.apache.hadoop.fs.s3a.S3AInputStream.reopen(S3AInputStream.java:207)
-  at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$lazySeek$1(S3AInputStream.java:355)
-  at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$2(Invoker.java:195)
-  at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
-  at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:265)
-  at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:322)
-  at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:261)
-  at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:193)
-  at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:215)
-  at org.apache.hadoop.fs.s3a.S3AInputStream.lazySeek(S3AInputStream.java:348)
-  at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:381)
-  at java.io.FilterInputStream.read(FilterInputStream.java:83)
-```
-
-### Error `AWSClientIOException: copyFile` caused by `NullPointerException`
-
-The AWS SDK has an [issue](https://github.com/aws/aws-sdk-java/issues/1644)
-where it will throw a relatively generic `AmazonClientException` caused by
-`NullPointerException` when copying a file and specifying a precondition
-that cannot be met.  This can bubble up from `S3AFileSystem.rename()`. It
-suggests that the file in S3 is inconsistent with the metadata in S3Guard.
-
-```
-org.apache.hadoop.fs.s3a.AWSClientIOException: copyFile(test/rename-eventually2.dat, test/dest2.dat) on test/rename-eventually2.dat: com.amazonaws.AmazonClientException: Unable to complete transfer: null: Unable to complete transfer: null
-  at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:201)
-  at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111)
-  at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:314)
-  at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:406)
-  at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:310)
-  at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:285)
-  at org.apache.hadoop.fs.s3a.S3AFileSystem.copyFile(S3AFileSystem.java:3034)
-  at org.apache.hadoop.fs.s3a.S3AFileSystem.innerRename(S3AFileSystem.java:1258)
-  at org.apache.hadoop.fs.s3a.S3AFileSystem.rename(S3AFileSystem.java:1119)
-  at org.apache.hadoop.fs.s3a.ITestS3ARemoteFileChanged.lambda$testRenameEventuallyConsistentFile2$6(ITestS3ARemoteFileChanged.java:556)
-  at org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:498)
-  at org.apache.hadoop.fs.s3a.ITestS3ARemoteFileChanged.testRenameEventuallyConsistentFile2(ITestS3ARemoteFileChanged.java:554)
-  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
-  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
-  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
-  at java.lang.reflect.Method.invoke(Method.java:498)
-  at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
-  at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
-  at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
-  at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
-  at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
-  at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
-  at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
-  at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)
-  at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292)
-  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
-  at java.lang.Thread.run(Thread.java:748)
-Caused by: com.amazonaws.AmazonClientException: Unable to complete transfer: null
-  at com.amazonaws.services.s3.transfer.internal.AbstractTransfer.unwrapExecutionException(AbstractTransfer.java:286)
-  at com.amazonaws.services.s3.transfer.internal.AbstractTransfer.rethrowExecutionException(AbstractTransfer.java:265)
-  at com.amazonaws.services.s3.transfer.internal.CopyImpl.waitForCopyResult(CopyImpl.java:67)
-  at org.apache.hadoop.fs.s3a.impl.CopyOutcome.waitForCopy(CopyOutcome.java:72)
-  at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$copyFile$14(S3AFileSystem.java:3047)
-  at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
-  ... 25 more
-Caused by: java.lang.NullPointerException
-  at com.amazonaws.services.s3.transfer.internal.CopyCallable.copyInOneChunk(CopyCallable.java:154)
-  at com.amazonaws.services.s3.transfer.internal.CopyCallable.call(CopyCallable.java:134)
-  at com.amazonaws.services.s3.transfer.internal.CopyMonitor.call(CopyMonitor.java:132)
-  at com.amazonaws.services.s3.transfer.internal.CopyMonitor.call(CopyMonitor.java:43)
-  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
-  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
-  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
-  ... 1 more
-```
-
-### Error `Attempt to change a resource which is still in use: Table is being deleted`
-
-```
-com.amazonaws.services.dynamodbv2.model.ResourceInUseException:
-  Attempt to change a resource which is still in use: Table is being deleted:
-   s3guard.test.testDynamoDBInitDestroy351245027
-    (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceInUseException;)
-```
-
-You have attempted to call `hadoop s3guard destroy` on a table which is already
-being destroyed.
-
-## Other Topics
-
-For details on how to test S3Guard, see [Testing S3Guard](./testing.html#s3guard)
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index f5da283db3689..d2ed9ede0171c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -12,7 +12,7 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 
-# Testing the S3A filesystem client and its features, including S3Guard
+# Testing the S3A filesystem client and its features
 
 <!-- MACRO{toc|fromDepth=0|toDepth=3} -->
 
@@ -318,7 +318,7 @@ mvn surefire-report:failsafe-report-only
 ## <a name="versioning"></a> Testing Versioned Stores
 
 Some tests (specifically some in `ITestS3ARemoteFileChanged`) require
-a versioned bucket for full test coverage as well as S3Guard being enabled.
+a versioned bucket for full test coverage.
 
 To enable versioning in a bucket.
 
@@ -388,11 +388,6 @@ By their very nature they are slow. And, as their execution time is often
 limited by bandwidth between the computer running the tests and the S3 endpoint,
 parallel execution does not speed these tests up.
 
-***Note: Running scale tests with `-Ds3guard` and `-Ddynamo` requires that
-you use a private, testing-only DynamoDB table.*** The tests do disruptive
-things such as deleting metadata and setting the provisioned throughput
-to very low values.
-
 ### <a name="enabling-scale"></a> Enabling the Scale Tests
 
 The tests are enabled if the `scale` property is set in the maven build
@@ -549,6 +544,18 @@ which address issues. In particular, we encourage testing of Hadoop release
 candidates, as these third-party endpoints get even less testing than the
 S3 endpoint itself.
 
+### Public datasets used in tests
+
+Some tests rely on the presence of existing public datasets available on Amazon S3.
+You may find a number of these in `org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils`.
+
+When testing against an endpoint which is not part of Amazon S3's standard commercial partition
+(`aws`) such as third-party implementations or AWS's China regions, you should replace these
+configurations with an empty space (` `) to disable the tests or an existing path in your object
+store that supports these tests.
+
+An example of this might be the MarkerTools tests which require a bucket with a large number of
+objects or the requester pays tests that require requester pays to be enabled for the bucket.
 
 ### Disabling the encryption tests
 
@@ -565,6 +572,18 @@ can be turned off.
 Encryption is only used for those specific test suites with `Encryption` in
 their classname.
 
+### Disabling the storage class tests
+
+When running storage class tests against third party object store that doesn't support
+S3 storage class, these tests might fail. They can be disabled.
+
+```xml
+<property>
+  <name>test.fs.s3a.create.storage.class.enabled</name>
+  <value>false</value>
+</property>
+```
+
 ### Configuring the CSV file read tests**
 
 To test on alternate infrastructures supporting
@@ -598,6 +617,45 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with
 a meaningful error ("S3 Select not supported") rather than a generic Bad Request
 exception.
 
+### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
+
+The tests are run with prefetch if the `prefetch` property is set in the
+maven build. This can be combined with the scale tests as well.
+
+```bash
+mvn verify -Dprefetch
+
+mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
+
+mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
+```
+
+
+### Testing Requester Pays
+
+By default, the requester pays tests will look for a bucket that exists on Amazon S3
+in us-east-1.
+
+If the endpoint does support requester pays, you can specify an alternative object.
+The test only requires an object of at least a few bytes in order
+to check that lists and basic reads work.
+
+```xml
+<property>
+  <name>test.fs.s3a.requester.pays.file</name>
+  <value>s3a://my-req-pays-enabled-bucket/on-another-endpoint.json</value>
+</property>
+```
+
+If the endpoint does not support requester pays, you can also disable the tests by configuring
+the test URI as a single space.
+
+```xml
+<property>
+  <name>test.fs.s3a.requester.pays.file</name>
+  <value> </value>
+</property>
+```
 
 ### Testing Session Credentials
 
@@ -891,28 +949,6 @@ This is invaluable for debugging test failures.
 How to set test options in your hadoop configuration rather
 than on the maven command line:
 
-As an example let's assume you want to run S3Guard integration tests using IDE.
-Please add the following properties in
-`hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml` file.
- Local configuration is stored in auth-keys.xml. The changes to this file won't be committed,
- so it's safe to store local config here.
-```xml
-<property>
-  <name>fs.s3a.s3guard.test.enabled</name>
-  <value>true</value>
-</property>
-```
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.test.implementation</name>
-  <value>dynamo</value>
-</property>
-```
-
-Warning : Although this is easier for IDE debugging setups, once you do this,
-you cannot change configurations on the mvn command line, such as testing without s3guard.
-
 ### Keeping AWS Costs down
 
 Most of the base S3 tests are designed to use public AWS data
@@ -923,7 +959,6 @@ so will cost more as well as generally take more time to execute.
 You are however billed for
 
 1. Data left in S3 after test runs.
-2. DynamoDB capacity reserved by S3Guard tables.
 3. HTTP operations on files (HEAD, LIST, GET).
 4. In-progress multipart uploads from bulk IO or S3A committer tests.
 5. Encryption/decryption using AWS KMS keys.
@@ -932,8 +967,6 @@ The GET/decrypt costs are incurred on each partial read of a file,
 so random IO can cost more than sequential IO; the speedup of queries with
 columnar data usually justifies this.
 
-The DynamoDB costs come from the number of entries stores and the allocated capacity.
-
 How to keep costs down
 
 * Don't run the scale tests with large datasets; keep `fs.s3a.scale.test.huge.filesize` unset, or a few MB (minimum: 5).
@@ -944,19 +977,7 @@ it can be manually done:
 * Abort all outstanding uploads:
 
       hadoop s3guard uploads -abort -force s3a://test-bucket/
-* If you don't need it, destroy the S3Guard DDB table.
-
-      hadoop s3guard destroy s3a://test-bucket/
 
-The S3Guard tests will automatically create the Dynamo DB table in runs with
-`-Ds3guard -Ddynamo` set; default capacity of these buckets
-tests is very small; it keeps costs down at the expense of IO performance
-and, for test runs in or near the S3/DDB stores, throttling events.
-
-If you want to manage capacity, use `s3guard set-capacity` to increase it
-(performance) or decrease it (costs).
-For remote `hadoop-aws` test runs, the read/write capacities of "0" each should suffice;
-increase it if parallel test run logs warn of throttling.
 
 ## <a name="tips"></a> Tips
 
@@ -983,25 +1004,21 @@ using an absolute XInclude reference to it.
 **Warning do not enable any type of failure injection in production.  The
 following settings are for testing only.**
 
-One of the challenges with S3A integration tests is the fact that S3 was an
+One of the challenges with S3A integration tests was the fact that S3 was an
 eventually-consistent storage system. To simulate inconsistencies more
 frequently than they would normally surface, S3A supports a shim layer on top of the `AmazonS3Client`
 class which artificially delays certain paths from appearing in listings.
 This is implemented in the class `InconsistentAmazonS3Client`.
 
-Now that S3 is consistent, injecting failures during integration and
-functional testing is less important.
-There's no need to enable it to verify that S3Guard can recover
-from consistencies, given that in production such consistencies
-will never surface.
+Now that S3 is consistent, injecting inconsistency is no longer needed
+during testing.
+However, it is stil useful to use the other feature of the client:
+throttling simulation.
 
 ## Simulating List Inconsistencies
 
 ### Enabling the InconsistentAmazonS3CClient
 
-There are two ways of enabling the `InconsistentAmazonS3Client`: at
-config-time, or programmatically. For an example of programmatic test usage,
-see `ITestS3GuardListConsistency`.
 
 To enable the fault-injecting client via configuration, switch the
 S3A client to use the "Inconsistent S3 Client Factory" when connecting to
@@ -1014,387 +1031,39 @@ S3:
 </property>
 ```
 
-The inconsistent client works by:
-
-1. Choosing which objects will be "inconsistent" at the time the object is
-created or deleted.
-2. When `listObjects()` is called, any keys that we have marked as
-inconsistent above will not be returned in the results (until the
-configured delay has elapsed). Similarly, deleted items may be *added* to
-missing results to delay the visibility of the delete.
-
-There are two ways of choosing which keys (filenames) will be affected: By
-substring, and by random probability.
+The inconsistent client will, on every AWS SDK request,
+generate a random number, and if less than the probability,
+raise a 503 exception.
 
 ```xml
-<property>
-  <name>fs.s3a.failinject.inconsistency.key.substring</name>
-  <value>DELAY_LISTING_ME</value>
-</property>
-
-<property>
-  <name>fs.s3a.failinject.inconsistency.probability</name>
-  <value>1.0</value>
-</property>
-```
-
-By default, any object which has the substring "DELAY_LISTING_ME" in its key
-will subject to delayed visibility. For example, the path
-`s3a://my-bucket/test/DELAY_LISTING_ME/file.txt` would match this condition.
-To match all keys use the value "\*" (a single asterisk). This is a special
-value: *We don't support arbitrary wildcards.*
-
-The default probability of delaying an object is 1.0. This means that *all*
-keys that match the substring will get delayed visibility. Note that we take
-the logical *and* of the two conditions (substring matches *and* probability
-random chance occurs). Here are some example configurations:
-
-```
-| substring | probability |  behavior                                  |
-|-----------|-------------|--------------------------------------------|
-|           | 0.001       | An empty <value> tag in .xml config will   |
-|           |             | be interpreted as unset and revert to the  |
-|           |             | default value, "DELAY_LISTING_ME"          |
-|           |             |                                            |
-| *         | 0.001       | 1/1000 chance of *any* key being delayed.  |
-|           |             |                                            |
-| delay     | 0.01        | 1/100 chance of any key containing "delay" |
-|           |             |                                            |
-| delay     | 1.0         | All keys containing substring "delay" ..   |
-```
-
-You can also configure how long you want the delay in visibility to last.
-The default is 5000 milliseconds (five seconds).
 
-```xml
 <property>
-  <name>fs.s3a.failinject.inconsistency.msec</name>
-  <value>5000</value>
+  <name>fs.s3a.failinject.throttle.probability</name>
+  <value>0.05</value>
 </property>
 ```
 
+These exceptions are returned to S3; they do not test the
+AWS SDK retry logic.
 
-### Limitations of Inconsistency Injection
-
-Although `InconsistentAmazonS3Client` can delay the visibility of an object
-or parent directory, it does not prevent the key of that object from
-appearing in all prefix searches. For example, if we create the following
-object with the default configuration above, in an otherwise empty bucket:
-
-```
-s3a://bucket/a/b/c/DELAY_LISTING_ME
-```
-
-Then the following paths will still be visible as directories (ignoring
-possible real-world inconsistencies):
-
-```
-s3a://bucket/a
-s3a://bucket/a/b
-```
-
-Whereas `getFileStatus()` on the following *will* be subject to delayed
-visibility (`FileNotFoundException` until delay has elapsed):
-
-```
-s3a://bucket/a/b/c
-s3a://bucket/a/b/c/DELAY_LISTING_ME
-```
-
-In real-life S3 inconsistency, however, we expect that all the above paths
-(including `a` and `b`) will be subject to delayed visibility.
 
 ### Using the `InconsistentAmazonS3CClient` in downstream integration tests
 
 The inconsistent client is shipped in the `hadoop-aws` JAR, so it can
-be used in applications which work with S3 to see how they handle
-inconsistent directory listings.
+be used in integration tests.
 
 ## <a name="s3guard"></a> Testing S3Guard
 
-[S3Guard](./s3guard.html) is an extension to S3A which added consistent metadata
-listings to the S3A client. 
-
-It has not been needed for applications to work safely with AWS S3 since November
-2020. However, it is currently still part of the codebase, and so something which
-needs to be tested.
-
-The basic strategy for testing S3Guard correctness consists of:
-
-1. MetadataStore Contract tests.
-
-    The MetadataStore contract tests are inspired by the Hadoop FileSystem and
-    `FileContext` contract tests.  Each implementation of the `MetadataStore` interface
-    subclasses the `MetadataStoreTestBase` class and customizes it to initialize
-    their MetadataStore.  This test ensures that the different implementations
-    all satisfy the semantics of the MetadataStore API.
-
-2. Running existing S3A unit and integration tests with S3Guard enabled.
-
-    You can run the S3A integration tests on top of S3Guard by configuring your
-    `MetadataStore` in your
-    `hadoop-tools/hadoop-aws/src/test/resources/core-site.xml` or
-    `hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml` files.
-    Next run the S3A integration tests as outlined in the *Running the Tests* section
-    of the [S3A documentation](./index.html)
-
-3. Running fault-injection tests that test S3Guard's consistency features.
-
-    The `ITestS3GuardListConsistency` uses failure injection to ensure
-    that list consistency logic is correct even when the underlying storage is
-    eventually consistent.
-
-    The integration test adds a shim above the Amazon S3 Client layer that injects
-    delays in object visibility.
-
-    All of these tests will be run if you follow the steps listed in step 2 above.
-
-    No charges are incurred for using this store, and its consistency
-    guarantees are that of the underlying object store instance. <!-- :) -->
-
-### Testing S3A with S3Guard Enabled
-
-All the S3A tests which work with a private repository can be configured to
-run with S3Guard by using the `s3guard` profile. When set, this will run
-all the tests with local memory for the metadata set to "non-authoritative" mode.
-
-```bash
-mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard
-```
-
-When the `s3guard` profile is enabled, following profiles can be specified:
-
-* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
-  not exist. You will have to pay the bills for DynamoDB web service.
-* `auth`: treat the S3Guard metadata as authoritative.
+As part of the removal of S3Guard from the production code, the tests have been updated
+so that
 
-```bash
-mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
-```
-
-When experimenting with options, it is usually best to run a single test suite
-at a time until the operations appear to be working.
-
-```bash
-mvn -T 1C verify -Dtest=skip -Dit.test=ITestS3AMiscOperations -Ds3guard -Ddynamo
-```
-
-### Notes
-
-1. If the `s3guard` profile is not set, then the S3Guard properties are those
-of the test configuration set in s3a contract xml file or `auth-keys.xml`
-
-If the `s3guard` profile *is* set:
-1. The S3Guard options from maven (the dynamo and authoritative flags)
-  overwrite any previously set in the configuration files.
-1. DynamoDB will be configured to create any missing tables.
-1. When using DynamoDB and running `ITestDynamoDBMetadataStore`,
-  the `fs.s3a.s3guard.ddb.test.table`
-property MUST be configured, and the name of that table MUST be different
- than what is used for `fs.s3a.s3guard.ddb.table`. The test table is destroyed
- and modified multiple times during the test.
- 1. Several of the tests create and destroy DynamoDB tables. The table names
- are prefixed with the value defined by
- `fs.s3a.s3guard.test.dynamo.table.prefix` (default="s3guard.test."). The user
- executing the tests will need sufficient privilege to create and destroy such
- tables. If the tests abort uncleanly, these tables may be left behind,
- incurring AWS charges.
-
-
-### How to Dump the Table and Metastore State
-
-There's an unstable entry point to list the contents of a table
-and S3 filesystem ot a set of Tab Separated Value files:
-
-```
-hadoop org.apache.hadoop.fs.s3a.s3guard.DumpS3GuardDynamoTable s3a://bucket/ dir/out
-```
-
-This generates a set of files prefixed `dir/out-` with different views of the
-world which can then be viewed on the command line or editor:
-
-```
-"type" "deleted" "path" "is_auth_dir" "is_empty_dir" "len" "updated" "updated_s" "last_modified" "last_modified_s" "etag" "version"
-"file" "true" "s3a://bucket/fork-0001/test/ITestS3AContractDistCp/testDirectWrite/remote" "false" "UNKNOWN" 0 1562171244451 "Wed Jul 03 17:27:24 BST 2019" 1562171244451 "Wed Jul 03 17:27:24 BST 2019" "" ""
-"file" "true" "s3a://bucket/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir/1/5xlPpalRwv/test/new/newdir/file1" "false" "UNKNOWN" 0 1562171518435 "Wed Jul 03 17:31:58 BST 2019" 1562171518435 "Wed Jul 03 17:31:58 BST 2019" "" ""
-"file" "true" "s3a://bucket/Users/stevel/Projects/hadoop-trunk/hadoop-tools/hadoop-aws/target/test-dir/1/5xlPpalRwv/test/new/newdir/subdir" "false" "UNKNOWN" 0 1562171518535 "Wed Jul 03 17:31:58 BST 2019" 1562171518535 "Wed Jul 03 17:31:58 BST 2019" "" ""
-"file" "true" "s3a://bucket/test/DELAY_LISTING_ME/testMRJob" "false" "UNKNOWN" 0 1562172036299 "Wed Jul 03 17:40:36 BST 2019" 1562172036299 "Wed Jul 03 17:40:36 BST 2019" "" ""
-```
-
-This is unstable: the output format may change without warning.
-To understand the meaning of the fields, consult the documentation.
-They are, currently:
-
-| field | meaning | source |
-|-------|---------| -------|
-| `type` | type | filestatus |
-| `deleted` | tombstone marker | metadata |
-| `path` | path of an entry | filestatus |
-| `is_auth_dir` | directory entry authoritative status | metadata |
-| `is_empty_dir` | does the entry represent an empty directory | metadata |
-| `len` | file length | filestatus |
-| `last_modified` | file status last modified | filestatus |
-| `last_modified_s` | file status last modified as string | filestatus |
-| `updated` | time (millis) metadata was updated | metadata |
-| `updated_s` | updated time as a string | metadata |
-| `etag` | any etag | filestatus |
-| `version` |  any version| filestatus |
-
-Files generated
-
-| suffix        | content |
-|---------------|---------|
-| `-scan.csv`   | Full scan/dump of the metastore |
-| `-store.csv`  | Recursive walk through the metastore |
-| `-tree.csv`   | Treewalk through filesystem `listStatus("/")` calls |
-| `-flat.csv`   | Flat listing through filesystem `listFiles("/", recursive)` |
-| `-s3.csv`     | Dump of the S3 Store *only* |
-| `-scan-2.csv` | Scan of the store after the previous operations |
-
-Why the two scan entries? The S3A listing and treewalk operations
-may add new entries to the metastore/DynamoDB table.
-
-Note 1: this is unstable; entry list and meaning may change, sorting of output,
-the listing algorithm, representation of types, etc. It's expected
-uses are: diagnostics, support calls and helping us developers
-work out what we've just broken.
-
-Note 2: This *is* safe to use against an active store; the tables may appear
-to be inconsistent due to changes taking place during the dump sequence.
-
-### Resetting the Metastore: `PurgeS3GuardDynamoTable`
-
-The `PurgeS3GuardDynamoTable` entry point
-`org.apache.hadoop.fs.s3a.s3guard.PurgeS3GuardDynamoTable` can
-list all entries in a store for a specific filesystem, and delete them.
-It *only* deletes those entries in the store for that specific filesystem,
-even if the store is shared.
-
-```bash
-hadoop org.apache.hadoop.fs.s3a.s3guard.PurgeS3GuardDynamoTable \
-  -force s3a://bucket/
-```
-
-Without the `-force` option the table is scanned, but no entries deleted;
-with it then all entries for that filesystem are deleted.
-No attempt is made to order the deletion; while the operation is under way
-the store is not fully connected (i.e. there may be entries whose parent has
-already been deleted).
-
-Needless to say: *it is not safe to use this against a table in active use.*
-
-### Scale Testing MetadataStore Directly
-
-There are some scale tests that exercise Metadata Store implementations
-directly. These ensure that S3Guard is are robust to things like DynamoDB
-throttling, and compare performance for different implementations. These
-are included in the scale tests executed when `-Dscale` is passed to
-the maven command line.
-
-The two S3Guard scale tests are `ITestDynamoDBMetadataStoreScale` and
-`ITestLocalMetadataStoreScale`.
-
-To run these tests, your DynamoDB table needs to be of limited capacity;
-the values in `ITestDynamoDBMetadataStoreScale` currently require a read capacity
-of 10 or less. a write capacity of 15 or more.
-
-The following settings allow us to run `ITestDynamoDBMetadataStoreScale` with
-artificially low read and write capacity provisioned, so we can judge the
-effects of being throttled by the DynamoDB service:
-
-```xml
-<property>
-  <name>scale.test.operation.count</name>
-  <value>10</value>
-</property>
-<property>
-  <name>scale.test.directory.count</name>
-  <value>3</value>
-</property>
-<property>
-  <name>fs.s3a.scale.test.enabled</name>
-  <value>true</value>
-</property>
-<property>
-  <name>fs.s3a.s3guard.ddb.table</name>
-  <value>my-scale-test</value>
-</property>
-<property>
-  <name>fs.s3a.s3guard.ddb.table.create</name>
-  <value>true</value>
-</property>
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-  <value>5</value>
-</property>
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-  <value>5</value>
-</property>
-```
-
-These tests verify that the invoked operations can trigger retries in the
-S3Guard code, rather than just in the AWS SDK level, so showing that if
-SDK operations fail, they get retried. They also verify that the filesystem
-statistics are updated to record that throttling took place.
-
-*Do not panic if these tests fail to detect throttling!*
-
-These tests are unreliable as they need certain conditions to be met
-to repeatedly fail:
-
-1. You must have a low-enough latency connection to the DynamoDB store that,
-for the capacity allocated, you can overload it.
-1. The AWS Console can give you a view of what is happening here.
-1. Running a single test on its own is less likely to trigger an overload
-than trying to run the whole test suite.
-1. And running the test suite more than once, back-to-back, can also help
-overload the cluster.
-1. Stepping through with a debugger will reduce load, so may not trigger
-failures.
-
-If the tests fail, it *probably* just means you aren't putting enough load
-on the table.
-
-These tests do not verify that the entire set of DynamoDB calls made
-during the use of a S3Guarded S3A filesystem are wrapped by retry logic.
-
-*The best way to verify resilience is to run the entire `hadoop-aws` test suite,
-or even a real application, with throttling enabled.
-
-### Testing encrypted DynamoDB tables
-
-By default, a DynamoDB table is encrypted using AWS owned customer master key
-(CMK). You can enable server side encryption (SSE) using AWS managed CMK or
-customer managed CMK in KMS before running the S3Guard tests.
-1. To enable AWS managed CMK, set the config
-`fs.s3a.s3guard.ddb.table.sse.enabled` to true in `auth-keys.xml`.
-1. To enable customer managed CMK, you need to create a KMS key and set the
-config in `auth-keys.xml`. The value can be the key ARN or alias. Example:
-```
-  <property>
-    <name>fs.s3a.s3guard.ddb.table.sse.enabled</name>
-    <value>true</value>
-  </property>
-  <property>
-    <name>fs.s3a.s3guard.ddb.table.sse.cmk</name>
-    <value>arn:aws:kms:us-west-2:360379543683:key/071a86ff-8881-4ba0-9230-95af6d01ca01</value>
-  </property>
-```
-For more details about SSE on DynamoDB table, please see [S3Guard doc](./s3guard.html).
+* All S3Guard-specific tests have been deleted.
+* All tests parameterized on S3Guard settings have had those test configurations removed.
+* The maven profiles option to run tests with S3Guard have been removed.
 
-### Testing only: Local Metadata Store
-
-There is an in-memory Metadata Store for testing.
-
-```xml
-<property>
-  <name>fs.s3a.metadatastore.impl</name>
-  <value>org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore</value>
-</property>
-```
+There is no need to test S3Guard -and so tests are lot faster.
+(We developers are all happy)
 
-This is not for use in production.
 
 ##<a name="assumed_roles"></a> Testing Assumed Roles
 
@@ -1413,11 +1082,8 @@ The specific tests an Assumed Role ARN is required for are
 To run these tests you need:
 
 1. A role in your AWS account will full read and write access rights to
-the S3 bucket used in the tests, DynamoDB, for S3Guard, and KMS for any
-SSE-KMS tests.
+the S3 bucket used in the tests, and KMS for any SSE-KMS tests.
 
-If your bucket is set up by default to use S3Guard, the role must have access
-to that service.
 
 1. Your IAM User to have the permissions to "assume" that role.
 
@@ -1445,7 +1111,7 @@ thorough test, by switching to the credentials provider.
 ```
 
 The usual credentials needed to log in to the bucket will be used, but now
-the credentials used to interact with S3 and DynamoDB will be temporary
+the credentials used to interact with S3 will be temporary
 role credentials, rather than the full credentials.
 
 ## <a name="qualifiying_sdk_updates"></a> Qualifying an AWS SDK Update
@@ -1464,7 +1130,7 @@ picked and reverted easily.
 1. Do not mix in an SDK update with any other piece of work, for the same reason.
 1. Plan for an afternoon's work, including before/after testing, log analysis
 and any manual tests.
-1. Make sure all the integration tests are running (including s3guard, ARN, encryption, scale)
+1. Make sure all the integration tests are running (including ARN, encryption, scale)
   *before you start the upgrade*.
 1. Create a JIRA for updating the SDK. Don't include the version (yet),
 as it may take a couple of SDK updates before it is ready.
@@ -1472,7 +1138,7 @@ as it may take a couple of SDK updates before it is ready.
 1. Create a private git branch of trunk for JIRA, and in
   `hadoop-project/pom.xml` update the `aws-java-sdk.version` to the new SDK version.
 1. Update AWS SDK versions in NOTICE.txt.
-1. Do a clean build and rerun all the `hadoop-aws` tests, with and without the `-Ds3guard -Ddynamo` options.
+1. Do a clean build and rerun all the `hadoop-aws` tests.
   This includes the `-Pscale` set, with a role defined for the assumed role tests.
   in `fs.s3a.assumed.role.arn` for testing assumed roles,
   and `fs.s3a.encryption.key` for encryption, for full coverage.
@@ -1502,7 +1168,15 @@ your IDE or via maven.
 
 We need a run through of the CLI to see if there have been changes there
 which cause problems, especially whether new log messages have surfaced,
-or whether some packaging change breaks that CLI
+or whether some packaging change breaks that CLI.
+
+It is always interesting when doing this to enable IOStatistics reporting
+```xml
+<property>
+  <name>fs.iostatistics.logging.level</name>
+  <value>info</value>
+</property>
+```
 
 From the root of the project, create a command line release `mvn package -Pdist -DskipTests -Dmaven.javadoc.skip=true  -DskipShade`;
 
@@ -1514,7 +1188,7 @@ From the root of the project, create a command line release `mvn package -Pdist
 export HADOOP_OPTIONAL_TOOLS="hadoop-aws"
 ```
 
-Run some basic s3guard commands as well as file operations.
+Run some basic s3guard CLI as well as file operations.
 
 ```bash
 export BUCKETNAME=example-bucket-name
@@ -1526,21 +1200,6 @@ bin/hadoop s3guard uploads $BUCKET
 # repeat twice, once with "no" and once with "yes" as responses
 bin/hadoop s3guard uploads -abort $BUCKET
 
-# ---------------------------------------------------
-# assuming s3guard is enabled
-# if on pay-by-request, expect an error message and exit code of -1
-bin/hadoop s3guard set-capacity $BUCKET
-
-# skip for PAY_PER_REQUEST
-bin/hadoop s3guard set-capacity -read 15 -write 15 $BUCKET
-
-bin/hadoop s3guard bucket-info -guarded $BUCKET 
-
-bin/hadoop s3guard diff $BUCKET/
-bin/hadoop s3guard prune -minutes 10 $BUCKET/
-bin/hadoop s3guard import -verbose $BUCKET/
-bin/hadoop s3guard authoritative -verbose $BUCKET
-
 # ---------------------------------------------------
 # root filesystem operatios
 # ---------------------------------------------------
@@ -1568,7 +1227,6 @@ bin/hadoop fs -rm -r $BUCKET/\*
 # ---------------------------------------------------
 
 bin/hadoop fs -mkdir $BUCKET/dir-no-trailing
-# used to fail with S3Guard
 bin/hadoop fs -mkdir $BUCKET/dir-trailing/
 bin/hadoop fs -touchz $BUCKET/file
 bin/hadoop fs -ls $BUCKET/
@@ -1638,6 +1296,21 @@ bin/hadoop s3guard markers -clean -verbose $BUCKET
 # expect success and exit code of 0
 bin/hadoop s3guard markers -audit -verbose $BUCKET
 
+# ---------------------------------------------------
+# Copy to from local
+# ---------------------------------------------------
+
+time bin/hadoop fs -copyFromLocal -t 10  share/hadoop/tools/lib/*aws*jar $BUCKET/
+
+# expect the iostatistics object_list_request value to be O(directories)
+bin/hadoop fs -ls -R $BUCKET/
+
+# expect the iostatistics object_list_request and op_get_content_summary values to be 1
+bin/hadoop fs -du -h -s $BUCKET/
+
+mkdir tmp
+time bin/hadoop fs -copyToLocal -t 10  $BUCKET/\*aws\* tmp
+
 # ---------------------------------------------------
 # S3 Select on Landsat
 # ---------------------------------------------------
@@ -1647,6 +1320,17 @@ export LANDSATGZ=s3a://landsat-pds/scene_list.gz
 bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \
  "SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100"
 
+
+# ---------------------------------------------------
+# Cloudstore
+# check out and build https://github.com/steveloughran/cloudstore
+# then for these tests, set CLOUDSTORE env var to point to the JAR
+# ---------------------------------------------------
+
+bin/hadoop jar $CLOUDSTORE storediag $BUCKET
+
+time bin/hadoop jar $CLOUDSTORE bandwidth 64M $BUCKET/testfile
+
 ```
 
 ### Other tests
@@ -1662,6 +1346,7 @@ to AWS services.
 * Try and get other people, especially anyone with their own endpoints,
   apps or different deployment environments, to run their own tests.
 * Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`.
+* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc)
 
 ### Dealing with Deprecated APIs and New Features
 
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 6f55d9effedd4..ef9f7f123e55c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -21,12 +21,13 @@
 Common problems working with S3 are:
 
 1. [Classpath setup](#classpath)
-1. [Authentication](#authentication)
-1. [Access Denial](#access_denied)
-1. [Connectivity Problems](#connectivity)
-1. [File System Semantics](#semantics)
-1. [Encryption](#encryption)
-1. [Other Errors](#other)
+2. [Authentication](#authentication)
+3. [Access Denial](#access_denied)
+4. [Connectivity Problems](#connectivity)
+5. [File System Semantics](#semantics)
+6. [Encryption](#encryption)
+7. [Other Errors](#other)
+8. [SDK Upgrade Warnings](#upgrade_warnings)
 
 This document also includes some [best pactises](#best) to aid troubleshooting.
 
@@ -336,13 +337,6 @@ spark.sql.hive.metastore.sharedPrefixes com.amazonaws.
 You are trying to use session/temporary credentials and the session token
 supplied is considered invalid.
 
-```
-org.apache.hadoop.fs.s3a.AWSBadRequestException: initTable on bucket:
-  com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException:
-  The security token included in the request is invalid
-  (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: UnrecognizedClientException)
-```
-
 This can surface if your configuration is setting the `fs.s3a.secret.key`,
 `fs.s3a.access.key` and `fs.s3a.session.key` correctly, but the
 AWS credential provider list set in `AWS_CREDENTIALS_PROVIDER` does not include
@@ -554,24 +548,55 @@ When trying to write or read SEE-KMS-encrypted data, the client gets a
 The caller does not have the permissions to access
 the key with which the data was encrypted.
 
+### <a name="access_denied_requester_pays"></a>`AccessDeniedException` when using a "Requester Pays" enabled bucket
+
+When making cross-account requests to a requester pays enabled bucket, all calls must acknowledge via a header that the requester will be billed.
+
+If you don't enable this acknowledgement within S3A, then you will see a message similar to this:
+
+```
+java.nio.file.AccessDeniedException: s3a://my-bucket/my-object: getFileStatus on s3a://my-bucket/my-object:
+com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403;
+Error Code: 403 Forbidden; Request ID: myshortreqid; S3 Extended Request ID: mylongreqid):403 Forbidden
+```
+
+To enable requester pays, set `fs.s3a.requester.pays.enabled` property to `true`.
+
+### <a name="access_denied_archive_storage_class"></a>`AccessDeniedException` "InvalidObjectState" when trying to read files
+
+```
+java.nio.file.AccessDeniedException: file1: copyFile(file1, file2) on file1: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=:InvalidObjectState
+
+Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Operation is not valid for the source object's storage class (Service: Amazon S3; Status Code: 403; Error Code: InvalidObjectState; Request ID: SK9EMPC1YRX75VZR; S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=; Proxy: null), S3 Extended Request ID: /nhUfdwJ+y5DLz6B4YR2FdA0FnQWwhDAkSCakn42zs2JssK3qWTrfwdNDiy6bOyXHOvJY0VAlHw=
+```
+
+This happens when you're trying to read or copy files that have archive storage class such as
+Glacier.
+
+If you want to access the file with S3A after writes, do not set `fs.s3a.create.storage.class` to `glacier` or `deep_archive`.
+
 ### <a name="no_region_session_credentials"></a> "Unable to find a region via the region provider chain." when using session credentials.
 
-Region must be provided when requesting session credentials, or an exception will be thrown with the message:
+Region must be provided when requesting session credentials, or an exception will be thrown with the
+message:
+
 ```
 com.amazonaws.SdkClientException: Unable to find a region via the region provider
 chain. Must provide an explicit region in the builder or setup environment to supply a region.
 ```
-In this case you have to set the `fs.s3a.assumed.role.sts.endpoint` property to a valid
-S3 sts endpoint and region like the following:
+
+In this case you have to set the `fs.s3a.assumed.role.sts.endpoint` property to a valid S3 sts
+endpoint and region like the following:
 
 ```xml
+
 <property>
     <name>fs.s3a.assumed.role.sts.endpoint</name>
     <value>${sts.endpoint}</value>
 </property>
 <property>
-    <name>fs.s3a.assumed.role.sts.endpoint.region</name>
-    <value>${sts.region}</value>
+<name>fs.s3a.assumed.role.sts.endpoint.region</name>
+<value>${sts.region}</value>
 </property>
 ```
 
@@ -924,30 +949,48 @@ connector isn't saving any data at all. The `Syncable` API, especially the
 `hsync()` call, are critical for applications such as HBase to safely
 persist data.
 
-The S3A connector throws an `UnsupportedOperationException` when these API calls
-are made, because the guarantees absolutely cannot be met: nothing is being flushed
-or saved.
+When configured to do so, the S3A connector throws an `UnsupportedOperationException`
+when these API calls are made, because the API guarantees absolutely cannot be met:
+_nothing is being flushed or saved_.
 
-* Applications which intend to invoke the Syncable APIs call `hasCapability("hsync")` on
+* Applications which intend to invoke the Syncable APIs should call `hasCapability("hsync")` on
   the stream to see if they are supported.
 * Or catch and downgrade `UnsupportedOperationException`.
 
-These recommendations _apply to all filesystems_. 
+These recommendations _apply to all filesystems_.
+
+For consistency with other filesystems, S3A output streams
+do not by default reject the `Syncable` calls -instead
+they print a warning of its use.
+
+
+The count of invocations of the two APIs are collected in the S3A filesystem
+Statistics/IOStatistics and so their use can be monitored.
 
-To downgrade the S3A connector to simply warning of the use of
+To switch the S3A connector to rejecting all use of
 `hsync()` or `hflush()` calls, set the option
-`fs.s3a.downgrade.syncable.exceptions` to true.
+`fs.s3a.downgrade.syncable.exceptions` to `false`.
 
 ```xml
 <property>
   <name>fs.s3a.downgrade.syncable.exceptions</name>
-  <value>true</value>
+  <value>false</value>
 </property>
 ```
 
-The count of invocations of the two APIs are collected
-in the S3A filesystem Statistics/IOStatistics and so
-their use can be monitored.
+Regardless of the setting, the `Syncable` API calls do not work.
+Telling the store to *not* downgrade the calls is a way to
+1. Prevent applications which require Syncable to work from being deployed
+  against S3.
+2. Identify applications which are making the calls even though they don't
+  need to. These applications can then be fixed -something which may take
+  time.
+
+Put differently: it is safest to disable downgrading syncable exceptions.
+However, enabling the downgrade stops applications unintentionally using the API
+from breaking.
+
+*Tip*: try turning it on in staging environments to see what breaks.
 
 ### `RemoteFileChangedException` and read-during-overwrite
 
@@ -1012,32 +1055,6 @@ before versioning was enabled.
 See [Handling Read-During-Overwrite](./index.html#handling_read-during-overwrite)
 for more information.
 
-### `RemoteFileChangedException`: "File to rename not found on guarded S3 store after repeated attempts"
-
-A file being renamed and listed in the S3Guard table could not be found
-in the S3 bucket even after multiple attempts.
-
-Now that S3 is consistent, this is sign that the S3Guard table is out of sync with
-the S3 Data. 
-
-Fix: disable S3Guard: it is no longer needed.
-
-```
-org.apache.hadoop.fs.s3a.RemoteFileChangedException: copyFile(/sourcedir/missing, /destdir/)
- `s3a://example/sourcedir/missing': File not found on S3 after repeated attempts: `s3a://example/sourcedir/missing'
-at org.apache.hadoop.fs.s3a.S3AFileSystem.copyFile(S3AFileSystem.java:3231)
-at org.apache.hadoop.fs.s3a.S3AFileSystem.access$700(S3AFileSystem.java:177)
-at org.apache.hadoop.fs.s3a.S3AFileSystem$RenameOperationCallbacksImpl.copyFile(S3AFileSystem.java:1368)
-at org.apache.hadoop.fs.s3a.impl.RenameOperation.copySourceAndUpdateTracker(RenameOperation.java:448)
-at org.apache.hadoop.fs.s3a.impl.RenameOperation.lambda$initiateCopy$0(RenameOperation.java:412)
-```
-
-If error occurs and the file is on S3, consider increasing the value of
-`fs.s3a.s3guard.consistency.retry.limit`.
-
-We also recommend using applications/application
-options which do  not rename files when committing work or when copying data
-to S3, but instead write directly to the final destination.
 
 ### Rename not behaving as "expected"
 
@@ -1435,30 +1452,6 @@ The user trying to use the KMS Key ID should have the right permissions to acces
 If not, then add permission(or IAM role) in "Key users" section by selecting the
 AWS-KMS CMK Key on AWS console.
 
-### S3-CSE cannot be used with S3Guard
-
-S3-CSE not supported for S3Guard enabled buckets.
-```
-org.apache.hadoop.fs.PathIOException: `s3a://test-bucket': S3-CSE cannot be used with S3Guard
-    at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:543)
-    at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3460)
-    at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:172)
-    at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3565)
-    at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3512)
-    at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:539)
-    at org.apache.hadoop.fs.Path.getFileSystem(Path.java:366)
-    at org.apache.hadoop.fs.shell.PathData.expandAsGlob(PathData.java:342)
-    at org.apache.hadoop.fs.shell.Command.expandArgument(Command.java:252)
-    at org.apache.hadoop.fs.shell.Command.expandArguments(Command.java:235)
-    at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:105)
-    at org.apache.hadoop.fs.shell.Command.run(Command.java:179)
-    at org.apache.hadoop.fs.FsShell.run(FsShell.java:327)
-    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81)
-    at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95)
-    at org.apache.hadoop.fs.FsShell.main(FsShell.java:390)
-```
-If you want to use S3Guard then disable S3-CSE or disable S3Guard if you want
-to use S3-CSE.
 
 ### <a name="not_all_bytes_were_read"></a> Message appears in logs "Not all bytes were read from the S3ObjectInputStream"
 
@@ -1535,6 +1528,56 @@ The specified bucket does not exist
     at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
 ```
 
+
+## <a name="s3guard"></a> S3Guard Errors
+
+S3Guard has been completely cut from the s3a connector
+[HADOOP-17409 Remove S3Guard - no longer needed](HADOOP-17409 Remove S3Guard - no longer needed).
+
+To avoid consistency problems with older releases, if an S3A filesystem is configured to use DynamoDB the filesystem
+will fail to initialize.
+
+### <a name="s3guard unsupported"></a> S3Guard is no longer needed/supported
+
+The option `fs.s3a.metadatastore.impl` or the per-bucket version has a value of
+`org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore`
+
+```
+org.apache.hadoop.fs.PathIOException: `s3a://production-london': S3Guard is no longer needed/supported,
+ yet s3a://production-london is configured to use DynamoDB as the S3Guard metadata store.
+ This is no longer needed or supported.
+ Origin of setting is fs.s3a.bucket.production-london.metadatastore.impl via [core-site.xml]
+        at org.apache.hadoop.fs.s3a.s3guard.S3Guard.checkNoS3Guard(S3Guard.java:111)
+        at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:540)
+        at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3459)
+        at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:171)
+        at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3564)
+        at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3511)
+        at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:538)
+        at org.apache.hadoop.fs.Path.getFileSystem(Path.java:366)
+        at org.apache.hadoop.fs.shell.PathData.expandAsGlob(PathData.java:342)
+        at org.apache.hadoop.fs.shell.Command.expandArgument(Command.java:252)
+        at org.apache.hadoop.fs.shell.Command.expandArguments(Command.java:235)
+        at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:105)
+        at org.apache.hadoop.fs.shell.Command.run(Command.java:179)
+        at org.apache.hadoop.fs.FsShell.run(FsShell.java:327)
+        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:81)
+        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:95)
+        at org.apache.hadoop.fs.FsShell.main(FsShell.java:390)
+
+ls: `s3a://production-london': S3Guard is no longer needed/supported,
+    yet s3a://production-london is configured to use DynamoDB as the S3Guard metadata store.
+    This is no longer needed or supported.
+    Origin of setting is fs.s3a.bucket.production-london.metadatastore.impl via [core-site.xml]
+```
+
+The error message will state the property from where it came, here `fs.s3a.bucket.production-london.metadatastore.impl` and which
+file the option was set if known, here `core-site.xml`.
+
+Fix: remove the configuration options enabling S3Guard.
+
+Consult the [S3Guard documentation](s3guard.html) for more details.
+
 ## <a name="other"></a> Other Errors
 
 ### <a name="integrity"></a> `SdkClientException` Unable to verify integrity of data upload
@@ -1908,44 +1951,6 @@ Please don't do that. Given that the emulated directory rename and delete operat
 are not atomic, even without retries, multiple S3 clients working with the same
 paths can interfere with each other
 
-### <a name="retries"></a> Tuning S3Guard open/rename Retry Policies
-
-When the S3A connector attempts to open a file for which it has an entry in
-its database, it will retry if the desired file is not found. This is
-done if:
-
-* No file is found in S3.
-* There is a file but its version or etag is not consistent with S3Guard table.
-
-These can be symptoms of S3's eventual consistency, hence the retries.
-They can also be caused by changes having been made to the S3 Store without
-SGuard being kept up to date.
-
-For this reason, the number of retry events are limited.
-
-```xml
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.limit</name>
-  <value>7</value>
-  <description>
-    Number of times to retry attempts to read/open/copy files when
-    S3Guard believes a specific version of the file to be available,
-    but the S3 request does not find any version of a file, or a different
-    version.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.interval</name>
-  <value>2s</value>
-  <description>
-    Initial interval between attempts to retry operations while waiting for S3
-    to become consistent with the S3Guard data.
-    An exponential back-off is used here: every failure doubles the delay.
-  </description>
-</property>
-```
-
 ### <a name="aws-timeouts"></a> Tuning AWS request timeouts
 
 It is possible to configure a global timeout for AWS service calls using following property:
@@ -1985,3 +1990,51 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request:
 
 When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it
 completely by setting it to `0`.
+
+## <a name="upgrade_warnings"></a> SDK Upgrade Warnings
+
+S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2).
+For more information on the upgrade and what's changing, see
+[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
+
+S3A logs the following warnings for things that will be changing in the upgrade. To disable these
+logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties.
+
+### <a name="ProviderReferenced"></a>  `Directly referencing AWS SDK V1 credential provider`
+
+This will be logged when an AWS credential provider is referenced directly in
+`fs.s3a.aws.credentials.provider`.
+For example, `com.amazonaws.auth.AWSSessionCredentialsProvider`
+
+To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`.
+Instead, use S3A's credential providers.
+
+### <a name="ClientRequested"></a>  `getAmazonS3ClientForTesting() will be removed`
+
+This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2,
+the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to
+`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed.
+
+### <a name="DelegationTokenProvider"></a>
+### `Custom credential providers used in delegation tokens binding classes will need to be updated`
+
+This will be logged when delegation tokens are used.
+Delegation tokens allow the use of custom binding classes which can implement custom credential
+providers.
+These credential providers will currently be implementing
+`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement
+`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
+
+### <a name="CustomSignerUsed"></a>
+### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated`
+
+This will be logged when a custom signer is used.
+Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be
+updated to implement `software.amazon.awssdk.core.signer.Signer`.
+
+### <a name="GetObjectMetadataCalled"></a>
+### `getObjectMetadata() called. This operation and it's response will be changed`
+
+This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to
+`headObject()` and will return a response of the type `HeadObjectResponse`.
+
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractContentSummary.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractContentSummary.java
new file mode 100644
index 0000000000000..ad83cfe52dadc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractContentSummary.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractContentSummaryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+
+public class ITestS3AContractContentSummary extends AbstractContractContentSummaryTest {
+
+  @Test
+  public void testGetContentSummaryDir() throws Throwable {
+    describe("getContentSummary on test dir with children");
+    S3AFileSystem fs = getFileSystem();
+    Path baseDir = methodPath();
+
+    // Nested folders created separately will return as separate objects in listFiles()
+    fs.mkdirs(new Path(baseDir, "a"));
+    fs.mkdirs(new Path(baseDir, "a/b"));
+    fs.mkdirs(new Path(baseDir, "a/b/a"));
+
+    // Will return as one object
+    fs.mkdirs(new Path(baseDir, "d/e/f"));
+
+    Path filePath = new Path(baseDir, "a/b/file");
+    touch(fs, filePath);
+
+    // look at path to see if it is a file
+    // it is not: so LIST
+    final ContentSummary summary = fs.getContentSummary(baseDir);
+
+    Assertions.assertThat(summary.getDirectoryCount()).as("Summary " + summary).isEqualTo(7);
+    Assertions.assertThat(summary.getFileCount()).as("Summary " + summary).isEqualTo(1);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3AContract(conf);
+  }
+
+  @Override
+  public S3AFileSystem getFileSystem() {
+    return (S3AFileSystem) super.getFileSystem();
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
index fd9497ba3ff54..d2a858f615ef6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
@@ -22,25 +22,11 @@
 import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 /**
  * S3A contract tests creating files.
  */
 public class ITestS3AContractCreate extends AbstractContractCreateTest {
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
index 95ea410fa66ae..a47dcaef61ee1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
@@ -22,25 +22,11 @@
 import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 /**
  * S3A contract tests covering deletes.
  */
 public class ITestS3AContractDelete extends AbstractContractDeleteTest {
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
index f4f5c176b37f8..e761e0d14bf83 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
@@ -20,7 +20,6 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.StorageStatistics;
@@ -41,7 +40,7 @@ protected int getTestTimeoutMillis() {
   }
 
   /**
-   * Create a configuration, possibly patching in S3Guard options.
+   * Create a configuration.
    * @return a configuration
    */
   @Override
@@ -49,8 +48,6 @@ protected Configuration createConfiguration() {
     Configuration newConf = super.createConfiguration();
     newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING);
     newConf.set(FAST_UPLOAD_BUFFER, FAST_UPLOAD_BUFFER_DISK);
-    // patch in S3Guard options
-    maybeEnableS3Guard(newConf);
     return newConf;
   }
 
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractEtag.java
similarity index 80%
rename from hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java
rename to hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractEtag.java
index 8f1edb9b6a3bd..824b2dab1f8df 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractEtag.java
@@ -16,17 +16,19 @@
  *  limitations under the License.
  */
 
-package org.apache.hadoop.fs.swift.contract;
+package org.apache.hadoop.fs.contract.s3a;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractContractEtagTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-public class TestSwiftContractRename extends AbstractContractRenameTest {
+/**
+ * Test S3A etag support.
+ */
+public class ITestS3AContractEtag extends AbstractContractEtagTest {
 
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
+    return new S3AContract(conf);
   }
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
index 638a22786b8cb..a68b0ea12af57 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
@@ -24,8 +24,6 @@
 import org.apache.hadoop.fs.s3a.S3ATestConstants;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 /**
  * S3A contract tests covering getFileStatus.
  * Some of the tests can take too long when the fault injection rate is high,
@@ -51,8 +49,6 @@ protected Configuration createConfiguration() {
     S3ATestUtils.disableFilesystemCaching(conf);
     // aggressively low page size forces tests to go multipage
     conf.setInt(Constants.MAX_PAGING_KEYS, 2);
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
     return conf;
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
index dba52e128d4fa..d953e7eb6aea9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
@@ -22,25 +22,11 @@
 import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 /**
  * Test dir operations on S3A.
  */
 public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
index 8222fff614598..48f5982bed656 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
@@ -31,7 +31,6 @@
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBytes;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 import static org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles.DEFAULT_HUGE_PARTITION_SIZE;
 
 /**
@@ -65,17 +64,6 @@ public S3AFileSystem getFileSystem() {
     return (S3AFileSystem) super.getFileSystem();
   }
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
@@ -96,15 +84,6 @@ protected boolean supportsConcurrentUploadsToSamePath() {
     return true;
   }
 
-  /**
-   * Provide a pessimistic time to become consistent.
-   * @return a time in milliseconds
-   */
-  @Override
-  protected int timeToBecomeConsistentMillis() {
-    return 30 * 1000;
-  }
-
   @Override
   protected boolean finalizeConsumesUploadIdImmediately() {
     return false;
@@ -130,7 +109,7 @@ public void setup() throws Exception {
    * S3 has no concept of directories, so this test does not apply.
    */
   public void testDirectoryInTheWay() throws Exception {
-    skip("unsupported");
+    skip("Unsupported");
   }
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
index d78273b14710a..82a7a3c63b37f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
@@ -18,29 +18,27 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
+import java.io.FileNotFoundException;
+
+import org.junit.Test;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * S3A contract tests opening files.
  */
 public class ITestS3AContractOpen extends AbstractContractOpenTest {
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
@@ -54,4 +52,59 @@ protected AbstractFSContract createContract(Configuration conf) {
   protected boolean areZeroByteFilesEncrypted() {
     return true;
   }
+
+  @Test
+  public void testOpenFileApplyReadBadName() throws Throwable {
+    describe("use the apply sequence to read a whole file");
+    Path path = methodPath();
+    FileSystem fs = getFileSystem();
+    touch(fs, path);
+    FileStatus st = fs.getFileStatus(path);
+    // The final element of the path is different, so
+    // openFile must fail
+    FileStatus st2 = new FileStatus(
+        0, false,
+        st.getReplication(),
+        st.getBlockSize(),
+        st.getModificationTime(),
+        st.getAccessTime(),
+        st.getPermission(),
+        st.getOwner(),
+        st.getGroup(),
+        new Path("gopher:///localhost/something.txt"));
+    intercept(IllegalArgumentException.class, () ->
+        fs.openFile(path)
+            .withFileStatus(st2)
+            .build());
+  }
+
+  /**
+   * Pass in a directory reference and expect the openFile call
+   * to fail.
+   */
+  @Test
+  public void testOpenFileDirectory() throws Throwable {
+    describe("Change the status to a directory");
+    Path path = methodPath();
+    FileSystem fs = getFileSystem();
+    int len = 4096;
+    createFile(fs, path, true,
+        dataset(len, 0x40, 0x80));
+    FileStatus st = fs.getFileStatus(path);
+    FileStatus st2 = new FileStatus(
+        len, true,
+        st.getReplication(),
+        st.getBlockSize(),
+        st.getModificationTime(),
+        st.getAccessTime(),
+        st.getPermission(),
+        st.getOwner(),
+        st.getGroup(),
+        path);
+    intercept(FileNotFoundException.class, () ->
+        fs.openFile(path)
+            .withFileStatus(st2)
+            .build());
+  }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
index e44df5facd2c3..d3ba7373cc944 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
@@ -18,21 +18,15 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.junit.Assume;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.Statistic;
@@ -41,70 +35,26 @@
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyFileContents;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.S3A_TEST_TIMEOUT;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 
 /**
  * S3A contract tests covering rename.
- * Parameterized for auth mode as testRenameWithNonEmptySubDir was failing
- * during HADOOP-16697 development; this lets us ensure that when S3Guard
- * is enabled, both auth and nonauth paths work
  */
-@RunWith(Parameterized.class)
 public class ITestS3AContractRename extends AbstractContractRenameTest {
 
   public static final Logger LOG = LoggerFactory.getLogger(
       ITestS3AContractRename.class);
 
-  private final boolean authoritative;
-
-  /**
-   * Parameterization.
-   */
-  @Parameterized.Parameters(name = "auth={0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {false},
-        {true}
-    });
-  }
-
-  public ITestS3AContractRename(boolean authoritative) {
-    this.authoritative = authoritative;
-  }
-
   @Override
   protected int getTestTimeoutMillis() {
     return S3A_TEST_TIMEOUT;
   }
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritative);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
   }
 
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    Assume.assumeTrue(
-        "Skipping auth mode tests when the FS doesn't have a metastore",
-        !authoritative || ((S3AFileSystem) getFileSystem()).hasMetadataStore());
-  }
-
   @Override
   public void testRenameDirIntoExistingDir() throws Throwable {
     describe("S3A rename into an existing directory returns false");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
index 6377b65f71394..5335de1b324d0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
@@ -18,20 +18,15 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
+import org.junit.Ignore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 /**
  * root dir operations against an S3 bucket.
  */
@@ -41,18 +36,6 @@ public class ITestS3AContractRootDir extends
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestS3AContractRootDir.class);
 
-  /**
-   * Create a configuration, possibly patching in S3Guard options.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
@@ -67,40 +50,4 @@ public S3AFileSystem getFileSystem() {
   @Ignore("S3 always return false when non-recursively remove root dir")
   public void testRmNonEmptyRootDirNonRecursive() throws Throwable {
   }
-
-  /**
-   * This is overridden to allow for eventual consistency on listings,
-   * but only if the store does not have S3Guard protecting it.
-   */
-  @Override
-  public void testListEmptyRootDirectory() throws IOException {
-    int maxAttempts = 10;
-    describe("Listing root directory; for consistency allowing "
-        + maxAttempts + " attempts");
-    for (int attempt = 1; attempt <= maxAttempts; ++attempt) {
-      try {
-        super.testListEmptyRootDirectory();
-        break;
-      } catch (AssertionError | FileNotFoundException e) {
-        if (attempt < maxAttempts) {
-          LOG.info("Attempt {} of {} for empty root directory test failed.  "
-              + "This is likely caused by eventual consistency of S3 "
-              + "listings.  Attempting retry.", attempt, maxAttempts,
-              e);
-          try {
-            Thread.sleep(1000);
-          } catch (InterruptedException e2) {
-            Thread.currentThread().interrupt();
-            fail("Test interrupted.");
-            break;
-          }
-        } else {
-          LOG.error(
-              "Empty root directory test failed {} attempts.  Failing test.",
-              maxAttempts);
-          throw e;
-        }
-      }
-    }
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
index 329b940a66a5d..dd41583de3fe4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -44,15 +44,14 @@
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
 import org.apache.hadoop.util.NativeCodeLoader;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
-import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_NORMAL;
-import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_RANDOM;
-import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_SEQUENTIAL;
 import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
 import static org.apache.hadoop.fs.s3a.Constants.SSL_CHANNEL_MODE;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.FS_S3A_IMPL_DISABLE_CACHE;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
         SSLChannelMode.Default_JSSE;
 import static org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory.
@@ -88,9 +87,9 @@ public class ITestS3AContractSeek extends AbstractContractSeekTest {
   @Parameterized.Parameters
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {INPUT_FADV_SEQUENTIAL, Default_JSSE},
-        {INPUT_FADV_RANDOM, OpenSSL},
-        {INPUT_FADV_NORMAL, Default_JSSE_with_GCM},
+        {FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL, Default_JSSE},
+        {FS_OPTION_OPENFILE_READ_POLICY_RANDOM, OpenSSL},
+        {FS_OPTION_OPENFILE_READ_POLICY_DEFAULT, Default_JSSE_with_GCM},
     });
   }
 
@@ -105,7 +104,7 @@ public ITestS3AContractSeek(final String seekPolicy,
   }
 
   /**
-   * Create a configuration, possibly patching in S3Guard options.
+   * Create a configuration.
    * The FS is set to be uncached and the readahead and seek policies
    * of the bucket itself are removed, so as to guarantee that the
    * parameterized and test settings are
@@ -114,8 +113,6 @@ public ITestS3AContractSeek(final String seekPolicy,
   @Override
   protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
     // purge any per-bucket overrides.
     try {
       URI bucketURI = new URI(checkNotNull(conf.get("fs.contract.test.fs.s3a")));
@@ -218,7 +215,8 @@ public void validateSSLChannelMode() {
   public void testReadPolicyInFS() throws Throwable {
     describe("Verify the read policy is being consistently set");
     S3AFileSystem fs = getFileSystem();
-    assertEquals(S3AInputPolicy.getPolicy(seekPolicy), fs.getInputPolicy());
+    assertEquals(S3AInputPolicy.getPolicy(seekPolicy, S3AInputPolicy.Normal),
+        fs.getInputPolicy());
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractUnbuffer.java
index d6dbce9233b1c..2c7149ff5cb15 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractUnbuffer.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractUnbuffer.java
@@ -22,18 +22,8 @@
 import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
-
 public class ITestS3AContractUnbuffer extends AbstractContractUnbufferTest {
 
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
new file mode 100644
index 0000000000000..4c357e288c84f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileRange;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.contract.AbstractContractVectoredReadTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
+import org.apache.hadoop.fs.statistics.StreamStatisticNames;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.returnBuffersToPoolPostRead;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.test.MoreAsserts.assertEqual;
+
+public class ITestS3AContractVectoredRead extends AbstractContractVectoredReadTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ITestS3AContractVectoredRead.class);
+
+  public ITestS3AContractVectoredRead(String bufferType) {
+    super(bufferType);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new S3AContract(conf);
+  }
+
+  /**
+   * Overriding in S3 vectored read api fails fast in case of EOF
+   * requested range.
+   */
+  @Override
+  public void testEOFRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = new ArrayList<>();
+    fileRanges.add(FileRange.createFileRange(DATASET_LEN, 100));
+    verifyExceptionalVectoredRead(fs, fileRanges, EOFException.class);
+  }
+
+  @Test
+  public void testMinSeekAndMaxSizeConfigsPropagation() throws Exception {
+    Configuration conf = getFileSystem().getConf();
+    S3ATestUtils.removeBaseAndBucketOverrides(conf,
+            Constants.AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE,
+            Constants.AWS_S3_VECTOR_READS_MIN_SEEK_SIZE);
+    S3ATestUtils.disableFilesystemCaching(conf);
+    final int configuredMinSeek = 2 * 1024;
+    final int configuredMaxSize = 10 * 1024 * 1024;
+    conf.set(Constants.AWS_S3_VECTOR_READS_MIN_SEEK_SIZE, "2K");
+    conf.set(Constants.AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE, "10M");
+    try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+      try (FSDataInputStream fis = fs.open(path(VECTORED_READ_FILE_NAME))) {
+        int newMinSeek = fis.minSeekForVectorReads();
+        int newMaxSize = fis.maxReadSizeForVectorReads();
+        assertEqual(newMinSeek, configuredMinSeek,
+                "configured s3a min seek for vectored reads");
+        assertEqual(newMaxSize, configuredMaxSize,
+                "configured s3a max size for vectored reads");
+      }
+    }
+  }
+
+  @Test
+  public void testMinSeekAndMaxSizeDefaultValues() throws Exception {
+    Configuration conf = getFileSystem().getConf();
+    S3ATestUtils.removeBaseAndBucketOverrides(conf,
+            Constants.AWS_S3_VECTOR_READS_MIN_SEEK_SIZE,
+            Constants.AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE);
+    try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+      try (FSDataInputStream fis = fs.open(path(VECTORED_READ_FILE_NAME))) {
+        int minSeek = fis.minSeekForVectorReads();
+        int maxSize = fis.maxReadSizeForVectorReads();
+        assertEqual(minSeek, Constants.DEFAULT_AWS_S3_VECTOR_READS_MIN_SEEK_SIZE,
+                "default s3a min seek for vectored reads");
+        assertEqual(maxSize, Constants.DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE,
+                "default s3a max read size for vectored reads");
+      }
+    }
+  }
+
+  @Test
+  public void testStopVectoredIoOperationsCloseStream() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = createSampleNonOverlappingRanges();
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))){
+      in.readVectored(fileRanges, getAllocate());
+      in.close();
+      LambdaTestUtils.intercept(InterruptedIOException.class,
+          () -> validateVectoredReadResult(fileRanges, DATASET));
+    }
+    // reopening the stream should succeed.
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))){
+      in.readVectored(fileRanges, getAllocate());
+      validateVectoredReadResult(fileRanges, DATASET);
+    }
+  }
+
+  @Test
+  public void testStopVectoredIoOperationsUnbuffer() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = createSampleNonOverlappingRanges();
+    try (FSDataInputStream in = fs.open(path(VECTORED_READ_FILE_NAME))){
+      in.readVectored(fileRanges, getAllocate());
+      in.unbuffer();
+      LambdaTestUtils.intercept(InterruptedIOException.class,
+          () -> validateVectoredReadResult(fileRanges, DATASET));
+      // re-initiating the vectored reads after unbuffer should succeed.
+      in.readVectored(fileRanges, getAllocate());
+      validateVectoredReadResult(fileRanges, DATASET);
+    }
+
+  }
+
+  /**
+   * S3 vectored IO doesn't support overlapping ranges.
+   */
+  @Override
+  public void testOverlappingRanges() throws Exception {
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = getSampleOverlappingRanges();
+    verifyExceptionalVectoredRead(fs, fileRanges, UnsupportedOperationException.class);
+  }
+
+  /**
+   * S3 vectored IO doesn't support overlapping ranges.
+   */
+  @Override
+  public void testSameRanges() throws Exception {
+    // Same ranges are special case of overlapping only.
+    FileSystem fs = getFileSystem();
+    List<FileRange> fileRanges = getSampleSameRanges();
+    verifyExceptionalVectoredRead(fs, fileRanges, UnsupportedOperationException.class);
+  }
+
+  /**
+   * As the minimum seek value is 4*1024, the first three ranges will be
+   * merged into and other two will remain as it is.
+   * */
+  @Test
+  public void testNormalReadVsVectoredReadStatsCollection() throws Exception {
+
+    try (S3AFileSystem fs = getTestFileSystemWithReadAheadDisabled()) {
+      List<FileRange> fileRanges = new ArrayList<>();
+      fileRanges.add(FileRange.createFileRange(10 * 1024, 100));
+      fileRanges.add(FileRange.createFileRange(8 * 1024, 100));
+      fileRanges.add(FileRange.createFileRange(14 * 1024, 100));
+      fileRanges.add(FileRange.createFileRange(2 * 1024 - 101, 100));
+      fileRanges.add(FileRange.createFileRange(40 * 1024, 1024));
+
+      FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME));
+      CompletableFuture<FSDataInputStream> builder =
+              fs.openFile(path(VECTORED_READ_FILE_NAME))
+                      .withFileStatus(fileStatus)
+                      .build();
+      try (FSDataInputStream in = builder.get()) {
+        in.readVectored(fileRanges, getAllocate());
+        validateVectoredReadResult(fileRanges, DATASET);
+        returnBuffersToPoolPostRead(fileRanges, getPool());
+
+        // audit the io statistics for this stream
+        IOStatistics st = in.getIOStatistics();
+        LOG.info("IOStats after readVectored operation {}", ioStatisticsToPrettyString(st));
+
+        // the vectored io operation must be tracked
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+                1);
+
+        // the vectored io operation is being called with 5 input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES,
+                5);
+
+        // 5 input ranges got combined in 3 as some of them are close.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES,
+                3);
+
+        // number of bytes discarded will be based on the above input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED,
+                5944);
+
+        verifyStatisticCounterValue(st,
+                StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+                3);
+
+        // read bytes should match the sum of requested length for each input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_BYTES,
+                1424);
+
+      }
+
+      CompletableFuture<FSDataInputStream> builder1 =
+              fs.openFile(path(VECTORED_READ_FILE_NAME))
+                      .withFileStatus(fileStatus)
+                      .build();
+
+      try (FSDataInputStream in = builder1.get()) {
+        for (FileRange range : fileRanges) {
+          byte[] temp = new byte[range.getLength()];
+          in.readFully((int) range.getOffset(), temp, 0, range.getLength());
+        }
+
+        // audit the statistics for this stream
+        IOStatistics st = in.getIOStatistics();
+        LOG.info("IOStats after read fully operation {}", ioStatisticsToPrettyString(st));
+
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+                0);
+
+        // all other counter values consistent.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED,
+                0);
+        verifyStatisticCounterValue(st,
+                StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+                5);
+
+        // read bytes should match the sum of requested length for each input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_BYTES,
+                1424);
+      }
+      // validate stats are getting merged at fs instance level.
+      IOStatistics fsStats = fs.getIOStatistics();
+      // only 1 vectored io call is made in this fs instance.
+      verifyStatisticCounterValue(fsStats,
+              StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+              1);
+      // 8 get requests were made in this fs instance.
+      verifyStatisticCounterValue(fsStats,
+              StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+              8);
+
+      verifyStatisticCounterValue(fsStats,
+              StreamStatisticNames.STREAM_READ_BYTES,
+              2848);
+    }
+  }
+
+  @Test
+  public void testMultiVectoredReadStatsCollection() throws Exception {
+    try (S3AFileSystem fs = getTestFileSystemWithReadAheadDisabled()) {
+      List<FileRange> ranges1 = getConsecutiveRanges();
+      List<FileRange> ranges2 = getConsecutiveRanges();
+      FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME));
+      CompletableFuture<FSDataInputStream> builder =
+              fs.openFile(path(VECTORED_READ_FILE_NAME))
+                      .withFileStatus(fileStatus)
+                      .build();
+      try (FSDataInputStream in = builder.get()) {
+        in.readVectored(ranges1, getAllocate());
+        in.readVectored(ranges2, getAllocate());
+        validateVectoredReadResult(ranges1, DATASET);
+        validateVectoredReadResult(ranges2, DATASET);
+        returnBuffersToPoolPostRead(ranges1, getPool());
+        returnBuffersToPoolPostRead(ranges2, getPool());
+
+        // audit the io statistics for this stream
+        IOStatistics st = in.getIOStatistics();
+
+        // 2 vectored io calls are made above.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+                2);
+
+        // 2 vectored io operation is being called with 2 input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES,
+                4);
+
+        // 2 ranges are getting merged in 1 during both vectored io operation.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES,
+                2);
+
+        // number of bytes discarded will be 0 as the ranges are consecutive.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED,
+                0);
+        // only 2 http get request will be made because ranges in both range list will be merged
+        // to 1 because they are consecutive.
+        verifyStatisticCounterValue(st,
+                StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+                2);
+        // read bytes should match the sum of requested length for each input ranges.
+        verifyStatisticCounterValue(st,
+                StreamStatisticNames.STREAM_READ_BYTES,
+                2000);
+      }
+      IOStatistics fsStats = fs.getIOStatistics();
+      // 2 vectored io calls are made in this fs instance.
+      verifyStatisticCounterValue(fsStats,
+              StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS,
+              2);
+      // 2 get requests were made in this fs instance.
+      verifyStatisticCounterValue(fsStats,
+              StoreStatisticNames.ACTION_HTTP_GET_REQUEST,
+              2);
+    }
+  }
+
+  private S3AFileSystem getTestFileSystemWithReadAheadDisabled() throws IOException {
+    Configuration conf = getFileSystem().getConf();
+    // also resetting the min seek and max size values is important
+    // as this same test suite has test which overrides these params.
+    S3ATestUtils.removeBaseAndBucketOverrides(conf,
+            Constants.READAHEAD_RANGE,
+            Constants.AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE,
+            Constants.AWS_S3_VECTOR_READS_MIN_SEEK_SIZE);
+    S3ATestUtils.disableFilesystemCaching(conf);
+    conf.setInt(Constants.READAHEAD_RANGE, 0);
+    return S3ATestUtils.createTestFileSystem(conf);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
index fb98657227a04..0d3dd4c2f6606 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
@@ -18,17 +18,11 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.impl.InternalConstants;
-
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfIOEContainsMessage;
 
 /**
  * The contract of S3A: only enabled if the test bucket is provided.
@@ -69,22 +63,6 @@ public S3AContract(Configuration conf, boolean addContractResource) {
     }
   }
 
-  /**
-   * Skip S3AFS initialization if S3-CSE and S3Guard are enabled.
-   *
-   */
-  @Override
-  public void init() throws IOException {
-    try {
-      super.init();
-    } catch (PathIOException ioe) {
-      // Skip the tests if (S3-CSE or Access Points) and S3-Guard are enabled.
-      skipIfIOEContainsMessage(ioe,
-          InternalConstants.CSE_S3GUARD_INCOMPATIBLE,
-          InternalConstants.AP_S3GUARD_INCOMPATIBLE);
-    }
-  }
-
   @Override
   public String getScheme() {
     return "s3a";
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
index 5765fe471c2bc..a46303f339678 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -26,8 +26,6 @@
 import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
 
 import org.junit.After;
 import org.junit.Before;
@@ -64,17 +62,19 @@ public void setup() throws Exception {
     s3 = fs.getAmazonS3ClientForTesting("mocking");
   }
 
+  @SuppressWarnings("deprecation")
   public Configuration createConfiguration() {
     Configuration conf = new Configuration();
     conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
         S3ClientFactory.class);
-    // We explicitly disable MetadataStore. For unit
-    // test we don't issue request to AWS DynamoDB service.
-    conf.setClass(S3_METADATA_STORE_IMPL, NullMetadataStore.class,
-        MetadataStore.class);
+
     // use minimum multipart size for faster triggering
     conf.setLong(Constants.MULTIPART_SIZE, MULTIPART_MIN_SIZE);
     conf.setInt(Constants.S3A_BUCKET_PROBE, 1);
+    // this is so stream draining is always blocking, allowing
+    // assertions to be safely made without worrying
+    // about any race conditions
+    conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE);
     return conf;
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index 242919192f19f..e90ad8b73efae 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.fs.contract.s3a.S3AContract;
 import org.apache.hadoop.fs.s3a.tools.MarkerTool;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.audit.AuditSpanSource;
 import org.apache.hadoop.io.IOUtils;
@@ -38,10 +39,10 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool;
 import static org.apache.hadoop.fs.s3a.S3AUtils.E_FS_CLOSED;
 import static org.apache.hadoop.fs.s3a.tools.MarkerTool.UNLIMITED_LISTING;
@@ -67,6 +68,15 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
    */
   private AuditSpanSource spanSource;
 
+  /**
+   * Atomic references to be used to re-throw an Exception or an ASE
+   * caught inside a lambda function.
+   */
+  private static final AtomicReference<Exception> FUTURE_EXCEPTION =
+      new AtomicReference<>();
+  private static final AtomicReference<AssertionError> FUTURE_ASE =
+      new AtomicReference<>();
+
   /**
    * Get the source.
    * @return span source
@@ -100,6 +110,9 @@ public void setup() throws Exception {
     S3AFileSystem.initializeClass();
     super.setup();
     setSpanSource(getFileSystem());
+    // Reset the current context's thread IOStatistics.`
+    // this ensures that the context stats will always be from the test case
+    IOStatisticsContext.getCurrentIOStatisticsContext().reset();
   }
 
   @Override
@@ -175,7 +188,7 @@ protected int getTestTimeoutMillis() {
   }
 
   /**
-   * Create a configuration, possibly patching in S3Guard options.
+   * Create a configuration.
    * @return a configuration
    */
   @Override
@@ -235,10 +248,6 @@ protected void writeThenReadFile(Path path, int len) throws IOException {
     ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
   }
 
-  protected String getTestTableName(String suffix) {
-    return getTestDynamoTablePrefix(getConfiguration()) + suffix;
-  }
-
   /**
    * Create a span from the source; returns a no-op if
    * creation fails or the source is null.
@@ -268,4 +277,53 @@ public void skipIfClientSideEncryption() {
     Assume.assumeTrue("Skipping test if CSE is enabled",
         !getFileSystem().isCSEEnabled());
   }
+
+  /**
+   * If an exception is caught while doing some work in a Lambda function,
+   * store it in an atomic reference to be thrown later on.
+   * @param exception Exception caught.
+   */
+  public static void setFutureException(Exception exception) {
+    FUTURE_EXCEPTION.set(exception);
+  }
+
+  /**
+   * If an Assertion is caught while doing some work in a Lambda function,
+   * store it in an atomic reference to be thrown later on.
+   *
+   * @param ase Assertion Error caught.
+   */
+  public static void setFutureAse(AssertionError ase) {
+    FUTURE_ASE.set(ase);
+  }
+
+  /**
+   * throw the caught exception from the atomic reference and also clear the
+   * atomic reference so that we don't rethrow in another test.
+   *
+   * @throws Exception the exception caught.
+   */
+  public static void maybeReThrowFutureException() throws Exception {
+    if (FUTURE_EXCEPTION.get() != null) {
+      Exception exceptionToThrow = FUTURE_EXCEPTION.get();
+      // reset the atomic ref before throwing.
+      setFutureAse(null);
+      throw exceptionToThrow;
+    }
+  }
+
+  /**
+   * throw the Assertion error from the atomic reference and also clear the
+   * atomic reference so that we don't rethrow in another test.
+   *
+   * @throws Exception Assertion error caught.
+   */
+  public static void maybeReThrowFutureASE() throws Exception {
+    if (FUTURE_ASE.get() != null) {
+      AssertionError aseToThrow = FUTURE_ASE.get();
+      // reset the atomic ref before throwing.
+      setFutureAse(null);
+      throw aseToThrow;
+    }
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
index 7945c820d060a..6df4f7593c9b8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a;
 
 import java.io.IOException;
+import java.nio.file.AccessDeniedException;
 
 import org.junit.Test;
 
@@ -77,14 +78,32 @@ protected void patchConfigurationEncryptionSettings(
       0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 12 - 1
   };
 
+  /**
+   * Skips the tests if encryption is not enabled in configuration.
+   *
+   * @implNote We can use {@link #createConfiguration()} here since
+   * it does not depend on any per-bucket based configuration.
+   * Otherwise, we would need to grab the configuration from an
+   * instance of {@link S3AFileSystem}.
+   */
   protected void requireEncryptedFileSystem() {
-    skipIfEncryptionTestsDisabled(getFileSystem().getConf());
+    skipIfEncryptionTestsDisabled(createConfiguration());
   }
 
+  /**
+   * Skipping tests when running against mandatory encryption bucket
+   * which allows only certain encryption method.
+   * S3 throw AmazonS3Exception with status 403 AccessDenied
+   * then it is translated into AccessDeniedException by S3AUtils.translateException(...)
+   */
   @Override
   public void setup() throws Exception {
-    super.setup();
-    requireEncryptedFileSystem();
+    try {
+      requireEncryptedFileSystem();
+      super.setup();
+    } catch (AccessDeniedException e) {
+      skip("Bucket does not allow " + getSSEAlgorithm() + " encryption method");
+    }
   }
 
   /**
@@ -104,7 +123,7 @@ public void testEncryptionSettingPropagation() throws Throwable {
   @Test
   public void testEncryption() throws Throwable {
     requireEncryptedFileSystem();
-    validateEncrytionSecrets(getFileSystem().getEncryptionSecrets());
+    validateEncryptionSecrets(getFileSystem().getEncryptionSecrets());
     for (int size: SIZES) {
       validateEncryptionForFilesize(size);
     }
@@ -116,7 +135,7 @@ public void testEncryptionOverRename() throws Throwable {
     byte[] data = dataset(1024, 'a', 'z');
     S3AFileSystem fs = getFileSystem();
     EncryptionSecrets secrets = fs.getEncryptionSecrets();
-    validateEncrytionSecrets(secrets);
+    validateEncryptionSecrets(secrets);
     writeDataset(fs, src, data, data.length, 1024 * 1024, true);
     ContractTestUtils.verifyFileContents(fs, src, data);
     // this file will be encrypted
@@ -135,7 +154,7 @@ public void testEncryptionOverRename() throws Throwable {
    * This makes sure that the settings have propagated properly.
    * @param secrets encryption secrets of the filesystem.
    */
-  protected void validateEncrytionSecrets(final EncryptionSecrets secrets) {
+  protected void validateEncryptionSecrets(final EncryptionSecrets secrets) {
     assertNotNull("No encryption secrets for filesystem", secrets);
     S3AEncryptionMethods sseAlgorithm = getSSEAlgorithm();
     assertEquals("Filesystem has wrong encryption algorithm",
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
deleted file mode 100644
index b1d742a400505..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestAuthoritativePath.java
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Collection;
-
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.checkListingContainsPath;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.checkListingDoesNotContainPath;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.junit.Assume.assumeTrue;
-
-public class ITestAuthoritativePath extends AbstractS3ATestBase {
-
-  public Path testRoot;
-
-  private S3AFileSystem fullyAuthFS;
-  private S3AFileSystem rawFS;
-
-  private MetadataStore ms;
-
-  @Before
-  public void setup() throws Exception {
-    super.setup();
-
-    long timestamp = System.currentTimeMillis();
-    testRoot = path("" + timestamp);
-
-    S3AFileSystem fs = getFileSystem();
-    // These test will fail if no ms
-    assumeTrue("FS needs to have a metadatastore.",
-        fs.hasMetadataStore());
-    assumeTrue("Metadatastore should persist authoritative bit",
-        metadataStorePersistsAuthoritativeBit(fs.getMetadataStore()));
-
-    // This test setup shares a single metadata store across instances,
-    // so that test runs with a local FS work.
-    // but this needs to be addressed in teardown, where the Auth fs
-    // needs to be detached from the metadata store before it is closed,
-    ms = fs.getMetadataStore();
-
-    fullyAuthFS = createFullyAuthFS();
-    assertTrue("No S3Guard store for fullyAuthFS",
-        fullyAuthFS.hasMetadataStore());
-    assertTrue("Authoritative mode off in fullyAuthFS",
-        fullyAuthFS.hasAuthoritativeMetadataStore());
-
-    rawFS = createRawFS();
-    assertFalse("UnguardedFS still has S3Guard",
-        rawFS.hasMetadataStore());
-  }
-
-  private void cleanUpFS(S3AFileSystem fs) {
-    // detach from the (shared) metadata store.
-    if (fs != null) {
-      fs.setMetadataStore(new NullMetadataStore());
-    }
-
-    IOUtils.cleanupWithLogger(LOG, fs);
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    if (fullyAuthFS != null) {
-      fullyAuthFS.delete(testRoot, true);
-    }
-
-    cleanUpFS(fullyAuthFS);
-    cleanUpFS(rawFS);
-    super.teardown();
-  }
-
-  private S3AFileSystem createFullyAuthFS()
-      throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    config.setBoolean(METADATASTORE_AUTHORITATIVE, true);
-    final S3AFileSystem newFS = createFS(uri, config);
-    // set back the same metadata store instance
-    newFS.setMetadataStore(ms);
-    return newFS;
-  }
-
-  private S3AFileSystem createSinglePathAuthFS(String authPath)
-      throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    config.set(AUTHORITATIVE_PATH, authPath.toString());
-    final S3AFileSystem newFS = createFS(uri, config);
-    // set back the same metadata store instance
-    newFS.setMetadataStore(ms);
-    return newFS;
-  }
-
-  private S3AFileSystem createMultiPathAuthFS(String first, String middle, String last)
-        throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    config.set(AUTHORITATIVE_PATH, first + "," + middle + "," + last);
-    final S3AFileSystem newFS = createFS(uri, config);
-    // set back the same metadata store instance
-    newFS.setMetadataStore(ms);
-    return newFS;
-  }
-
-  private S3AFileSystem createRawFS() throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        S3_METADATA_STORE_IMPL);
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    return createFS(uri, config);
-  }
-
-  /**
-   * Create and initialize a new filesystem.
-   * This filesystem MUST be closed in test teardown.
-   * @param uri FS URI
-   * @param config config.
-   * @return new instance
-   * @throws IOException failure
-   */
-  private S3AFileSystem createFS(final URI uri, final Configuration config)
-      throws IOException {
-    S3AFileSystem fs2 = new S3AFileSystem();
-    fs2.initialize(uri, config);
-    return fs2;
-  }
-
-  private void runTestOutsidePath(S3AFileSystem partiallyAuthFS, Path nonAuthPath) throws Exception {
-    Path inBandPath = new Path(nonAuthPath, "out-of-path-in-band");
-    Path outOfBandPath = new Path(nonAuthPath, "out-of-path-out-of-band");
-
-    touch(fullyAuthFS, inBandPath);
-
-    // trigger an authoritative write-back
-    fullyAuthFS.listStatus(inBandPath.getParent());
-
-    touch(rawFS, outOfBandPath);
-
-    // listing lacks outOfBandPath => short-circuited by auth mode
-    checkListingDoesNotContainPath(fullyAuthFS, outOfBandPath);
-
-    // partiallyAuthFS differs from fullyAuthFS because we're outside the path
-    checkListingContainsPath(partiallyAuthFS, outOfBandPath);
-
-    // sanity check that in-band operations are always visible
-    checkListingContainsPath(fullyAuthFS, inBandPath);
-    checkListingContainsPath(partiallyAuthFS, inBandPath);
-
-  }
-
-  private void runTestInsidePath(S3AFileSystem partiallyAuthFS, Path authPath) throws Exception {
-    Path inBandPath = new Path(authPath, "in-path-in-band");
-    Path outOfBandPath = new Path(authPath, "in-path-out-of-band");
-
-    touch(fullyAuthFS, inBandPath);
-
-    // trigger an authoritative write-back
-    fullyAuthFS.listStatus(inBandPath.getParent());
-
-    touch(rawFS, outOfBandPath);
-
-    // listing lacks outOfBandPath => short-circuited by auth mode
-    checkListingDoesNotContainPath(fullyAuthFS, outOfBandPath);
-    checkListingDoesNotContainPath(partiallyAuthFS, outOfBandPath);
-
-    // sanity check that in-band operations are always successful
-    checkListingContainsPath(fullyAuthFS, inBandPath);
-    checkListingContainsPath(partiallyAuthFS, inBandPath);
-  }
-
-  @Test
-  public void testSingleAuthPath() throws Exception {
-    Path authPath = new Path(testRoot, "testSingleAuthPath-auth");
-    Path nonAuthPath = new Path(testRoot, "testSingleAuthPath");
-    S3AFileSystem fs = createSinglePathAuthFS(authPath.toString());
-    try {
-      assertTrue("No S3Guard store for partially authoritative FS",
-            fs.hasMetadataStore());
-
-      runTestInsidePath(fs, authPath);
-      runTestOutsidePath(fs, nonAuthPath);
-    } finally {
-      cleanUpFS(fs);
-    }
-  }
-
-  @Test
-  public void testAuthPathWithOtherBucket() throws Exception {
-    Path authPath;
-    Path nonAuthPath;
-    S3AFileSystem fs = null;
-    String landsat = "s3a://landsat-pds/data";
-    String decoy2 = "/decoy2";
-
-    try {
-      authPath = new Path(testRoot, "testMultiAuthPath-first");
-      nonAuthPath = new Path(testRoot, "nonAuth-1");
-      fs = createMultiPathAuthFS(authPath.toString(), landsat, decoy2);
-      assertTrue("No S3Guard store for partially authoritative FS",
-          fs.hasMetadataStore());
-
-      runTestInsidePath(fs, authPath);
-      runTestOutsidePath(fs, nonAuthPath);
-    } finally {
-      cleanUpFS(fs);
-    }
-  }
-
-  @Test
-  public void testMultiAuthPath() throws Exception {
-    Path authPath;
-    Path nonAuthPath;
-    S3AFileSystem fs = null;
-    String decoy1 = "/decoy1";
-    String decoy2 = "/decoy2";
-
-    try {
-      authPath = new Path(testRoot, "testMultiAuthPath-first");
-      nonAuthPath = new Path(testRoot, "nonAuth-1");
-      fs = createMultiPathAuthFS(authPath.toString(), decoy1, decoy2);
-      assertTrue("No S3Guard store for partially authoritative FS",
-            fs.hasMetadataStore());
-
-      runTestInsidePath(fs, authPath);
-      runTestOutsidePath(fs, nonAuthPath);
-    } finally {
-      cleanUpFS(fs);
-    }
-
-    try {
-      authPath = new Path(testRoot, "testMultiAuthPath-middle");
-      nonAuthPath = new Path(testRoot, "nonAuth-2");
-      fs = createMultiPathAuthFS(decoy1, authPath.toString(), decoy2);
-      assertTrue("No S3Guard store for partially authoritative FS",
-            fs.hasMetadataStore());
-
-      runTestInsidePath(fs, authPath);
-      runTestOutsidePath(fs, nonAuthPath);
-    } finally {
-      cleanUpFS(fs);
-    }
-
-    try {
-      authPath = new Path(testRoot, "testMultiAuthPath-last");
-      nonAuthPath = new Path(testRoot, "nonAuth-3");
-      fs = createMultiPathAuthFS(decoy1, decoy2, authPath.toString());
-      assertTrue("No S3Guard store for partially authoritative FS",
-            fs.hasMetadataStore());
-
-      runTestInsidePath(fs, authPath);
-      runTestOutsidePath(fs, nonAuthPath);
-    } finally {
-      cleanUpFS(fs);
-    }
-  }
-
-  @Test
-  public void testPrefixVsDirectory() throws Exception {
-    S3AFileSystem fs = createSinglePathAuthFS("/auth");
-    Collection<String> authPaths = S3Guard.getAuthoritativePaths(fs);
-
-    try{
-      Path totalMismatch = new Path(testRoot, "/non-auth");
-      assertFalse(S3Guard.allowAuthoritative(totalMismatch, fs,
-          false, authPaths));
-
-      Path prefixMatch = new Path(testRoot, "/authoritative");
-      assertFalse(S3Guard.allowAuthoritative(prefixMatch, fs,
-          false, authPaths));
-
-      Path directoryMatch = new Path(testRoot, "/auth/oritative");
-      assertTrue(S3Guard.allowAuthoritative(directoryMatch, fs,
-          false, authPaths));
-
-      Path unqualifiedMatch = new Path(testRoot.toUri().getPath(), "/auth/oritative");
-      assertTrue(S3Guard.allowAuthoritative(unqualifiedMatch, fs,
-          false, authPaths));
-    } finally {
-      cleanUpFS(fs);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
index 55423273b9579..cf9ad877add89 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
@@ -43,7 +43,7 @@
 public class ITestBlockingThreadPoolExecutorService {
 
   private static final Logger LOG = LoggerFactory.getLogger(
-      BlockingThreadPoolExecutorService.class);
+      ITestBlockingThreadPoolExecutorService.class);
 
   private static final int NUM_ACTIVE_TASKS = 4;
   private static final int NUM_WAITING_TASKS = 2;
@@ -57,7 +57,7 @@ public class ITestBlockingThreadPoolExecutorService {
   private static BlockingThreadPoolExecutorService tpe;
 
   @Rule
-  public Timeout testTimeout = new Timeout(60 * 1000);
+  public Timeout testTimeout = new Timeout(60, TimeUnit.SECONDS);
 
   @AfterClass
   public static void afterClass() throws Exception {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java
index 0bcb11a823d3d..3ad679e6728d3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestDowngradeSyncable.java
@@ -44,7 +44,7 @@ public class ITestDowngradeSyncable extends AbstractS3ACostTest {
 
 
   public ITestDowngradeSyncable() {
-    super(false, true, false);
+    super(true);
   }
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestLocatedFileStatusFetcher.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestLocatedFileStatusFetcher.java
index 5b6e634a63168..fcf412dac8a79 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestLocatedFileStatusFetcher.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestLocatedFileStatusFetcher.java
@@ -19,13 +19,9 @@
 package org.apache.hadoop.fs.s3a;
 
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collection;
 
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,13 +35,6 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.extractStatistics;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
@@ -61,24 +50,11 @@
  * but whereas that tests failure paths, this looks at the performance
  * of successful invocations.
  */
-@RunWith(Parameterized.class)
 public class ITestLocatedFileStatusFetcher extends AbstractS3ATestBase {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestLocatedFileStatusFetcher.class);
 
-
-  /**
-   * Parameterization.
-   */
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {"raw", false},
-        {"nonauth", true}
-    });
-  }
-
   /** Filter to select everything. */
   private static final PathFilter EVERYTHING = t -> true;
 
@@ -115,10 +91,6 @@ public static Collection<Object[]> params() {
    */
   private static final int EXPECTED_LIST_COUNT = 4;
 
-  private final String name;
-
-  private final boolean s3guard;
-
   private Path basePath;
 
   private Path emptyDir;
@@ -137,40 +109,13 @@ public static Collection<Object[]> params() {
 
   private Configuration listConfig;
 
-  public ITestLocatedFileStatusFetcher(final String name,
-      final boolean s3guard) {
-    this.name = name;
-    this.s3guard = s3guard;
-  }
-
-  @Override
-  public Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    String bucketName = getTestBucketName(conf);
-
-    removeBaseAndBucketOverrides(bucketName, conf,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    removeBucketOverrides(bucketName, conf,
-        S3_METADATA_STORE_IMPL);
-    if (!s3guard) {
-      removeBaseAndBucketOverrides(bucketName, conf,
-          S3_METADATA_STORE_IMPL);
-    }
-    conf.setBoolean(METADATASTORE_AUTHORITATIVE, false);
-    disableFilesystemCaching(conf);
-    return conf;
-  }
   @Override
   public void setup() throws Exception {
     super.setup();
     S3AFileSystem fs
         = getFileSystem();
-    // avoiding the parameterization to steer clear of accidentally creating
-    // patterns; a timestamp is used to ensure tombstones from previous runs
-    // do not interfere
-    basePath = path("ITestLocatedFileStatusFetcher-" + name
-        + "-" + System.currentTimeMillis() / 1000);
+
+    basePath = methodPath();
 
     // define the paths and create them.
     describe("Creating test directories and files");
@@ -197,7 +142,6 @@ public void setup() throws Exception {
     listConfig = new Configuration(getConfiguration());
   }
 
-
   /**
    * Assert that the fetcher stats logs the expected number of calls.
    * @param fetcher fetcher
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
index 30f1f71eeea88..c13c3f48b8466 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -157,17 +157,21 @@ public void testBadCredentials() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAnonymousProvider() throws Exception {
     Configuration conf = new Configuration();
     conf.set(AWS_CREDENTIALS_PROVIDER,
         AnonymousAWSCredentialsProvider.class.getName());
     Path testFile = getCSVTestPath(conf);
-    FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
-    assertNotNull(fs);
-    assertTrue(fs instanceof S3AFileSystem);
-    FileStatus stat = fs.getFileStatus(testFile);
-    assertNotNull(stat);
-    assertEquals(testFile, stat.getPath());
+    try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
+      assertNotNull("S3AFileSystem instance must not be null", fs);
+      assertTrue("FileSystem must be the instance of S3AFileSystem", fs instanceof S3AFileSystem);
+      FileStatus stat = fs.getFileStatus(testFile);
+      assertNotNull("FileStatus with qualified path must not be null", stat);
+      assertEquals(
+          "The qualified path returned by getFileStatus should be same as the original file",
+          testFile, stat.getPath());
+    }
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
index a9c5e6dd43eb4..fb295f3f09ff4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
@@ -37,8 +37,6 @@
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
 import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
@@ -58,8 +56,6 @@ public class ITestS3ABucketExistence extends AbstractS3ATestBase {
   public void testNoBucketProbing() throws Exception {
     describe("Disable init-time probes and expect FS operations to fail");
     Configuration conf = createConfigurationWithProbe(0);
-    // metastores can bypass S3 checks, so disable S3Guard, always
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
 
     fs = FileSystem.get(uri, conf);
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
index 7424b6f90b3c3..1071582cc67d2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
 
 import static org.apache.hadoop.fs.s3a.Constants.CANNED_ACL;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
@@ -64,15 +65,17 @@ protected Configuration createConfiguration() {
   @Test
   public void testCreatedObjectsHaveACLs() throws Throwable {
     S3AFileSystem fs = getFileSystem();
-    Path dir = methodPath();
-    fs.mkdirs(dir);
-    assertObjectHasLoggingGrant(dir, false);
-    Path path = new Path(dir, "1");
-    ContractTestUtils.touch(fs, path);
-    assertObjectHasLoggingGrant(path, true);
-    Path path2 = new Path(dir, "2");
-    fs.rename(path, path2);
-    assertObjectHasLoggingGrant(path2, true);
+    try (AuditSpan span = span()) {
+      Path dir = methodPath();
+      fs.mkdirs(dir);
+      assertObjectHasLoggingGrant(dir, false);
+      Path path = new Path(dir, "1");
+      ContractTestUtils.touch(fs, path);
+      assertObjectHasLoggingGrant(path, true);
+      Path path2 = new Path(dir, "2");
+      fs.rename(path, path2);
+      assertObjectHasLoggingGrant(path2, true);
+    }
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 906cadd502411..26d00bc7d359a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -414,6 +414,7 @@ public void testRequestTimeout() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testCloseIdempotent() throws Throwable {
     conf = new Configuration();
     fs = S3ATestUtils.createTestFileSystem(conf);
@@ -456,17 +457,6 @@ public void testDirectoryAllocatorRR() throws Throwable {
         tmp1.getParent(), tmp2.getParent());
   }
 
-  @Test
-  public void testReadAheadRange() throws Exception {
-    conf = new Configuration();
-    conf.set(Constants.READAHEAD_RANGE, "300K");
-    fs = S3ATestUtils.createTestFileSystem(conf);
-    assertNotNull(fs);
-    long readAheadRange = fs.getReadAheadRange();
-    assertNotNull(readAheadRange);
-    assertEquals("Read Ahead Range Incorrect.", 300 * 1024, readAheadRange);
-  }
-
   @Test
   public void testUsernameFromUGI() throws Throwable {
     final String alice = "alice";
@@ -537,7 +527,7 @@ public void testS3SpecificSignerOverride() throws IOException {
     Assert.assertEquals(s3SignerOverride,
         clientConfiguration.getSignerOverride());
     clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB);
+        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_STS);
     Assert.assertNull(clientConfiguration.getSignerOverride());
 
     // Configured base SIGNING_ALGORITHM, overridden for S3 only
@@ -549,41 +539,9 @@ public void testS3SpecificSignerOverride() throws IOException {
     Assert.assertEquals(s3SignerOverride,
         clientConfiguration.getSignerOverride());
     clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB);
+        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_STS);
     Assert
         .assertEquals(signerOverride, clientConfiguration.getSignerOverride());
   }
 
-  @Test(timeout = 10_000L)
-  public void testDdbSpecificSignerOverride() throws IOException {
-    ClientConfiguration clientConfiguration = null;
-    Configuration config;
-
-    String signerOverride = "testSigner";
-    String ddbSignerOverride = "testDdbSigner";
-
-    // Default SIGNING_ALGORITHM, overridden for S3
-    config = new Configuration();
-    config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride);
-    clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB);
-    Assert.assertEquals(ddbSignerOverride,
-        clientConfiguration.getSignerOverride());
-    clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3);
-    Assert.assertNull(clientConfiguration.getSignerOverride());
-
-    // Configured base SIGNING_ALGORITHM, overridden for S3
-    config = new Configuration();
-    config.set(SIGNING_ALGORITHM, signerOverride);
-    config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride);
-    clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB);
-    Assert.assertEquals(ddbSignerOverride,
-        clientConfiguration.getSignerOverride());
-    clientConfiguration = S3AUtils
-        .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3);
-    Assert
-        .assertEquals(signerOverride, clientConfiguration.getSignerOverride());
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
index 42e905e416d16..6d950d9bfaaa4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AContractGetFileStatusV1List.java
@@ -25,7 +25,6 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.LIST_VERSION;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 
 /**
  * S3A contract tests for getFileStatus, using the v1 List Objects API.
@@ -50,7 +49,6 @@ protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     disableFilesystemCaching(conf);
     conf.setInt(Constants.MAX_PAGING_KEYS, 2);
-    maybeEnableS3Guard(conf);
 
     // Use v1 List Objects API
     conf.setInt(LIST_VERSION, 1);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
index 7b9e79e475386..ca9d185c3e9e1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
@@ -33,7 +33,6 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE;
 import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
 import static org.apache.hadoop.fs.s3a.Constants.RETRY_INTERVAL;
 import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
@@ -53,8 +52,7 @@ protected Configuration createConfiguration() {
         CHANGE_DETECT_SOURCE,
         CHANGE_DETECT_MODE,
         RETRY_LIMIT,
-        RETRY_INTERVAL,
-        METADATASTORE_AUTHORITATIVE);
+        RETRY_INTERVAL);
     conf.setInt(RETRY_LIMIT, 2);
     conf.set(RETRY_INTERVAL, "1ms");
     return conf;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java
new file mode 100644
index 0000000000000..31c58de629b5f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+
+/**
+ * Test deleteOnExit for S3A.
+ * The following cases for deleteOnExit are tested:
+ *  1. A nonexist file, which is added to deleteOnExit set.
+ *  2. An existing file
+ *  3. A file is added to deleteOnExist set first, then created.
+ *  4. A directory with some files under it.
+ */
+public class ITestS3ADeleteOnExit extends AbstractS3ATestBase {
+
+  private static final String PARENT_DIR_PATH_STR = "testDeleteOnExitDir";
+  private static final String NON_EXIST_FILE_PATH_STR =
+          PARENT_DIR_PATH_STR + "/nonExistFile";
+  private static final String INORDER_FILE_PATH_STR =
+          PARENT_DIR_PATH_STR + "/inOrderFile";
+  private static final String OUT_OF_ORDER_FILE_PATH_STR =
+          PARENT_DIR_PATH_STR + "/outOfOrderFile";
+  private static final String SUBDIR_PATH_STR =
+          PARENT_DIR_PATH_STR + "/subDir";
+  private static final String FILE_UNDER_SUBDIR_PATH_STR =
+          SUBDIR_PATH_STR + "/subDirFile";
+
+  @Test
+  public void testDeleteOnExit() throws Exception {
+    FileSystem fs = getFileSystem();
+
+    // Get a new filesystem object which is same as fs.
+    FileSystem s3aFs = new S3AFileSystem();
+    s3aFs.initialize(fs.getUri(), fs.getConf());
+    Path nonExistFilePath = path(NON_EXIST_FILE_PATH_STR);
+    Path inOrderFilePath = path(INORDER_FILE_PATH_STR);
+    Path outOfOrderFilePath = path(OUT_OF_ORDER_FILE_PATH_STR);
+    Path subDirPath = path(SUBDIR_PATH_STR);
+    Path fileUnderSubDirPath = path(FILE_UNDER_SUBDIR_PATH_STR);
+
+    // 1. set up the test directory.
+    Path dir = path("testDeleteOnExitDir");
+    s3aFs.mkdirs(dir);
+
+    // 2. Add a nonexisting file to DeleteOnExit set.
+    s3aFs.deleteOnExit(nonExistFilePath);
+    assertPathDoesNotExist("File " + NON_EXIST_FILE_PATH_STR + " should not exist",
+            nonExistFilePath);
+
+    // 3. create a file and then add it to DeleteOnExit set.
+    byte[] data = dataset(16, 'a', 26);
+    createFile(s3aFs, inOrderFilePath, true, data);
+    assertPathExists("File " + INORDER_FILE_PATH_STR + " should exist", inOrderFilePath);
+    s3aFs.deleteOnExit(inOrderFilePath);
+
+    // 4. add a path to DeleteOnExit set first, then create it.
+    s3aFs.deleteOnExit(outOfOrderFilePath);
+    createFile(s3aFs, outOfOrderFilePath, true, data);
+    assertPathExists("File " + OUT_OF_ORDER_FILE_PATH_STR + " should exist",
+            outOfOrderFilePath);
+
+    // 5. create a subdirectory, a file under it,  and add subdirectory DeleteOnExit set.
+    s3aFs.mkdirs(subDirPath);
+    s3aFs.deleteOnExit(subDirPath);
+    createFile(s3aFs, fileUnderSubDirPath, true, data);
+    assertPathExists("Directory " + SUBDIR_PATH_STR + " should exist", subDirPath);
+    assertPathExists("File " + FILE_UNDER_SUBDIR_PATH_STR + " should exist",
+            fileUnderSubDirPath);
+
+    s3aFs.close();
+
+    // After s3aFs is closed, make sure that all files/directories in deleteOnExit
+    // set are deleted.
+    assertPathDoesNotExist("File " + NON_EXIST_FILE_PATH_STR + " should not exist",
+            nonExistFilePath);
+    assertPathDoesNotExist("File " + INORDER_FILE_PATH_STR + " should not exist",
+            inOrderFilePath);
+    assertPathDoesNotExist("File " + OUT_OF_ORDER_FILE_PATH_STR + " should not exist",
+            outOfOrderFilePath);
+    assertPathDoesNotExist("Directory " + SUBDIR_PATH_STR + " should not exist",
+            subDirPath);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
index ff46e981eabaa..64e37bf832b87 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -41,19 +41,21 @@
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP;
 import static org.apache.hadoop.fs.s3a.Constants.ETAG_CHECKSUM_ENABLED;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Concrete class that extends {@link AbstractTestS3AEncryption}
  * and tests SSE-C encryption.
  * HEAD requests against SSE-C-encrypted data will fail if the wrong key
- * is presented, so the tests are very brittle to S3Guard being on vs. off.
+ * is presented -this used to cause problems with S3Guard.
  * Equally "vexing" has been the optimizations of getFileStatus(), wherein
  * LIST comes before HEAD path + /
  */
@@ -78,18 +80,11 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {"raw-keep-markers", false, true},
-        {"raw-delete-markers", false, false},
-        {"guarded-keep-markers", true, true},
-        {"guarded-delete-markers", true, false}
+        {"keep-markers", true},
+        {"delete-markers", false}
     });
   }
 
-  /**
-   * Parameter: should the stores be guarded?
-   */
-  private final boolean s3guard;
-
   /**
    * Parameter: should directory markers be retained?
    */
@@ -101,9 +96,7 @@ public static Collection<Object[]> params() {
   private S3AFileSystem fsKeyB;
 
   public ITestS3AEncryptionSSEC(final String name,
-      final boolean s3guard,
       final boolean keepMarkers) {
-    this.s3guard = s3guard;
     this.keepMarkers = keepMarkers;
   }
 
@@ -113,13 +106,6 @@ protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     disableFilesystemCaching(conf);
     String bucketName = getTestBucketName(conf);
-    removeBucketOverrides(bucketName, conf,
-        S3_METADATA_STORE_IMPL);
-    if (!s3guard) {
-      // in a raw run remove all s3guard settings
-      removeBaseAndBucketOverrides(bucketName, conf,
-          S3_METADATA_STORE_IMPL);
-    }
     // directory marker options
     removeBaseAndBucketOverrides(bucketName, conf,
         DIRECTORY_MARKER_POLICY,
@@ -335,7 +321,7 @@ public void testListStatusEncryptedFile() throws Exception {
    * @return true if check for a path being a file will issue a HEAD request.
    */
   private boolean statusProbesCheckS3(S3AFileSystem fs, Path path) {
-    return !fs.hasMetadataStore() || !fs.allowAuthoritative(path);
+    return true;
   }
 
   /**
@@ -370,9 +356,7 @@ public void testDeleteEncryptedObjectWithDifferentKey() throws Exception {
   }
 
   /**
-   * getFileChecksum always goes to S3, so when
-   * the caller lacks permissions, it fails irrespective
-   * of guard.
+   * getFileChecksum always goes to S3.
    */
   @Test
   public void testChecksumRequiresReadAccess() throws Throwable {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
index a0fb762d89914..56ce9300dc4e7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionWithDefaultS3Settings.java
@@ -135,7 +135,7 @@ public void testEncryptionOverRename2() throws Throwable {
     Path src = path(createFilename(1024));
     byte[] data = dataset(1024, 'a', 'z');
     EncryptionSecrets secrets = fs.getEncryptionSecrets();
-    validateEncrytionSecrets(secrets);
+    validateEncryptionSecrets(secrets);
     writeDataset(fs, src, data, data.length, 1024 * 1024, true);
     ContractTestUtils.verifyFileContents(fs, src, data);
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
index 761dd558063ba..add6502d7da71 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
@@ -165,6 +165,7 @@ private void createMarsNorth2Client(Configuration conf) throws Exception {
    * @throws URISyntaxException parse problems.
    * @throws IOException IO problems
    */
+  @SuppressWarnings("deprecation")
   private AmazonS3 createS3Client(Configuration conf,
       String endpoint,
       String expectedRegion)
@@ -176,6 +177,7 @@ private AmazonS3 createS3Client(Configuration conf,
     S3ClientFactory.S3ClientCreationParameters parameters
         = new S3ClientFactory.S3ClientCreationParameters()
         .withCredentialSet(new AnonymousAWSCredentialsProvider())
+        .withPathUri(new URI("s3a://localhost/"))
         .withEndpoint(endpoint)
         .withMetrics(new EmptyS3AStatisticsContext()
             .newStatisticsFromAwsSdk());
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
index e395207589812..c0f6a4b23226b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -19,16 +19,14 @@
 package org.apache.hadoop.fs.s3a;
 
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import org.apache.hadoop.util.Lists;
+
 import org.assertj.core.api.Assertions;
 import org.junit.Assume;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.statistics.StoreStatisticNames;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
@@ -42,10 +40,11 @@
 import java.nio.file.AccessDeniedException;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
 import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.*;
-import static org.apache.hadoop.fs.s3a.impl.TestPartialDeleteFailures.keysToDelete;
 import static org.apache.hadoop.test.LambdaTestUtils.*;
+import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
 
 /**
  * ITest for failure handling, primarily multipart deletion.
@@ -78,10 +77,41 @@ public void testMultiObjectDeleteNoFile() throws Throwable {
     removeKeys(getFileSystem(), "ITestS3AFailureHandling/missingFile");
   }
 
+  /**
+   * See HADOOP-18112.
+   */
+  @Test
+  public void testMultiObjectDeleteLargeNumKeys() throws Exception {
+    S3AFileSystem fs =  getFileSystem();
+    Path path = path("largeDir");
+    mkdirs(path);
+    createFiles(fs, path, 1, 1005, 0);
+    RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator =
+            fs.listFiles(path, false);
+    List<String> keys  = toList(mappingRemoteIterator(locatedFileStatusRemoteIterator,
+        locatedFileStatus -> fs.pathToKey(locatedFileStatus.getPath())));
+    // After implementation of paging during multi object deletion,
+    // no exception is encountered.
+    Long bulkDeleteReqBefore = getNumberOfBulkDeleteRequestsMadeTillNow(fs);
+    try (AuditSpan span = span()) {
+      fs.removeKeys(buildDeleteRequest(keys.toArray(new String[0])), false);
+    }
+    Long bulkDeleteReqAfter = getNumberOfBulkDeleteRequestsMadeTillNow(fs);
+    // number of delete requests is 5 as we have default page size of 250.
+    Assertions.assertThat(bulkDeleteReqAfter - bulkDeleteReqBefore)
+            .describedAs("Number of batched bulk delete requests")
+            .isEqualTo(5);
+  }
+
+  private Long getNumberOfBulkDeleteRequestsMadeTillNow(S3AFileSystem fs) {
+    return fs.getIOStatistics().counters()
+            .get(StoreStatisticNames.OBJECT_BULK_DELETE_REQUEST);
+  }
+
   private void removeKeys(S3AFileSystem fileSystem, String... keys)
       throws IOException {
     try (AuditSpan span = span()) {
-      fileSystem.removeKeys(buildDeleteRequest(keys), false, null);
+      fileSystem.removeKeys(buildDeleteRequest(keys), false);
     }
   }
 
@@ -132,51 +162,6 @@ public void testMultiObjectDeleteNoPermissions() throws Throwable {
                 fs.pathToKey(csvPath),
                 "missing-key.csv"
             });
-    MultiObjectDeleteException ex = intercept(
-        MultiObjectDeleteException.class,
-        () -> fs.removeKeys(keys, false, null));
-
-    final List<Path> undeleted
-        = extractUndeletedPaths(ex, fs::keyToQualifiedPath);
-    String undeletedFiles = join(undeleted);
-    failIf(undeleted.size() != 2,
-        "undeleted list size wrong: " + undeletedFiles,
-        ex);
-    assertTrue("no CSV in " +undeletedFiles, undeleted.contains(csvPath));
-
-    // and a full split, after adding a new key
-    String marker = "/marker";
-    Path markerPath = fs.keyToQualifiedPath(marker);
-    keys.add(new DeleteObjectsRequest.KeyVersion(marker));
-
-    Pair<List<KeyPath>, List<KeyPath>> pair =
-        new MultiObjectDeleteSupport(fs.createStoreContext(), null)
-        .splitUndeletedKeys(ex, keys);
-    assertEquals(undeleted, toPathList(pair.getLeft()));
-    List<KeyPath> right = pair.getRight();
-    Assertions.assertThat(right)
-        .hasSize(1);
-    assertEquals(markerPath, right.get(0).getPath());
-  }
-
-  /**
-   * See what happens when you delete two entries which do not exist.
-   * It must not raise an exception.
-   */
-  @Test
-  public void testMultiObjectDeleteMissingEntriesSucceeds() throws Throwable {
-    describe("Delete keys which don't exist");
-    Path base = path("missing");
-    S3AFileSystem fs = getFileSystem();
-    List<DeleteObjectsRequest.KeyVersion> keys = keysToDelete(
-        Lists.newArrayList(new Path(base, "1"), new Path(base, "2")));
-    try (AuditSpan span = span()) {
-      fs.removeKeys(keys, false, null);
-    }
-  }
-
-  private String join(final Iterable iterable) {
-    return "[" + StringUtils.join(iterable, ",") + "]";
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
index ca8e49cc33be2..dae6312d48098 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -18,14 +18,11 @@
 
 package org.apache.hadoop.fs.s3a;
 
-
-import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
 import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
 
-
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -43,14 +40,13 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
 import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Use metrics to assert about the cost of file API calls.
- * Parameterized on guarded vs raw. and directory marker keep vs delete
+ * Parameterized on directory marker keep vs delete.
  */
 @RunWith(Parameterized.class)
 public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
@@ -64,25 +60,19 @@ public class ITestS3AFileOperationCost extends AbstractS3ACostTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {"raw-keep-markers", false, true, false},
-        {"raw-delete-markers", false, false, false},
-        {"nonauth-keep-markers", true, true, false},
-        {"auth-delete-markers", true, false, true}
+        {"keep-markers", true},
+        {"delete-markers", false},
     });
   }
 
-  public ITestS3AFileOperationCost(final String name,
-      final boolean s3guard,
-      final boolean keepMarkers,
-      final boolean authoritative) {
-    super(s3guard, keepMarkers, authoritative);
+  public ITestS3AFileOperationCost(
+      final String name,
+      final boolean keepMarkers) {
+    super(keepMarkers);
   }
 
   /**
    * Test the cost of {@code listLocatedStatus(file)}.
-   * There's a minor inefficiency in that calling this on
-   * a file in S3Guard still executes a LIST call, even
-   * though the file record is in the store.
    */
   @Test
   public void testCostOfLocatedFileStatusOnFile() throws Throwable {
@@ -90,11 +80,8 @@ public void testCostOfLocatedFileStatusOnFile() throws Throwable {
     Path file = file(methodPath());
     S3AFileSystem fs = getFileSystem();
     verifyMetrics(() -> fs.listLocatedStatus(file),
-        whenRaw(FILE_STATUS_FILE_PROBE
-            .plus(LIST_LOCATED_STATUS_LIST_OP)),
-        whenAuthoritative(LIST_LOCATED_STATUS_LIST_OP),
-        whenNonauth(LIST_LOCATED_STATUS_LIST_OP
-            .plus(S3GUARD_NONAUTH_FILE_STATUS_PROBE)));
+        always(FILE_STATUS_FILE_PROBE
+            .plus(LIST_LOCATED_STATUS_LIST_OP)));
   }
 
   @Test
@@ -104,10 +91,8 @@ public void testCostOfListLocatedStatusOnEmptyDir() throws Throwable {
     S3AFileSystem fs = getFileSystem();
     verifyMetrics(() ->
             fs.listLocatedStatus(dir),
-        whenRaw(LIST_LOCATED_STATUS_LIST_OP
-            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)),
-        whenAuthoritative(NO_IO),
-        whenNonauth(LIST_LOCATED_STATUS_LIST_OP));
+        always(LIST_LOCATED_STATUS_LIST_OP
+            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)));
   }
 
   @Test
@@ -118,9 +103,7 @@ public void testCostOfListLocatedStatusOnNonEmptyDir() throws Throwable {
     Path file = file(new Path(dir, "file.txt"));
     verifyMetrics(() ->
           fs.listLocatedStatus(dir),
-        whenRaw(LIST_LOCATED_STATUS_LIST_OP),
-        whenAuthoritative(NO_IO),
-        whenNonauth(LIST_LOCATED_STATUS_LIST_OP));
+        always(LIST_LOCATED_STATUS_LIST_OP));
   }
 
   @Test
@@ -131,10 +114,8 @@ public void testCostOfListFilesOnFile() throws Throwable {
     touch(fs, file);
     verifyMetrics(() ->
             fs.listFiles(file, true),
-        whenRaw(LIST_LOCATED_STATUS_LIST_OP
-            .plus(GET_FILE_STATUS_ON_FILE)),
-        whenAuthoritative(NO_IO),
-        whenNonauth(LIST_LOCATED_STATUS_LIST_OP));
+        always(LIST_LOCATED_STATUS_LIST_OP
+            .plus(GET_FILE_STATUS_ON_FILE)));
   }
 
   @Test
@@ -146,10 +127,8 @@ public void testCostOfListFilesOnEmptyDir() throws Throwable {
     fs.mkdirs(dir);
     verifyMetrics(() ->
             fs.listFiles(dir, true),
-        whenRaw(LIST_FILES_LIST_OP
-            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)),
-        whenAuthoritative(NO_IO),
-        whenNonauth(LIST_FILES_LIST_OP));
+        always(LIST_FILES_LIST_OP
+            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)));
   }
 
   @Test
@@ -162,9 +141,7 @@ public void testCostOfListFilesOnNonEmptyDir() throws Throwable {
     touch(fs, file);
     verifyMetrics(() ->
             fs.listFiles(dir, true),
-        whenRaw(LIST_FILES_LIST_OP),
-        whenAuthoritative(NO_IO),
-        whenNonauth(LIST_FILES_LIST_OP));
+        always(LIST_FILES_LIST_OP));
   }
 
   @Test
@@ -174,7 +151,7 @@ public void testCostOfListFilesOnNonExistingDir() throws Throwable {
     S3AFileSystem fs = getFileSystem();
     verifyMetricsIntercepting(FileNotFoundException.class, "",
         () -> fs.listFiles(dir, true),
-        whenRaw(LIST_FILES_LIST_OP
+        always(LIST_FILES_LIST_OP
             .plus(GET_FILE_STATUS_FNFE)));
   }
 
@@ -186,11 +163,8 @@ public void testCostOfListStatusOnFile() throws Throwable {
     touch(fs, file);
     verifyMetrics(() ->
             fs.listStatus(file),
-            whenRaw(LIST_STATUS_LIST_OP
-                    .plus(GET_FILE_STATUS_ON_FILE)),
-            whenAuthoritative(LIST_STATUS_LIST_OP),
-            whenNonauth(LIST_STATUS_LIST_OP
-                .plus(S3GUARD_NONAUTH_FILE_STATUS_PROBE)));
+        always(LIST_STATUS_LIST_OP
+            .plus(GET_FILE_STATUS_ON_FILE)));
   }
 
   @Test
@@ -201,10 +175,8 @@ public void testCostOfListStatusOnEmptyDir() throws Throwable {
     fs.mkdirs(dir);
     verifyMetrics(() ->
             fs.listStatus(dir),
-            whenRaw(LIST_STATUS_LIST_OP
-            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)),
-            whenAuthoritative(NO_IO),
-            whenNonauth(LIST_STATUS_LIST_OP));
+        always(LIST_STATUS_LIST_OP
+            .plus(GET_FILE_STATUS_ON_EMPTY_DIR)));
   }
 
   @Test
@@ -217,16 +189,14 @@ public void testCostOfListStatusOnNonEmptyDir() throws Throwable {
     touch(fs, file);
     verifyMetrics(() ->
             fs.listStatus(dir),
-            whenRaw(LIST_STATUS_LIST_OP),
-            whenAuthoritative(NO_IO),
-            whenNonauth(LIST_STATUS_LIST_OP));
+        always(LIST_STATUS_LIST_OP));
   }
 
   @Test
   public void testCostOfGetFileStatusOnFile() throws Throwable {
     describe("performing getFileStatus on a file");
     Path simpleFile = file(methodPath());
-    S3AFileStatus status = verifyRawInnerGetFileStatus(simpleFile, true,
+    S3AFileStatus status = verifyInnerGetFileStatus(simpleFile, true,
         StatusProbeEnum.ALL,
         GET_FILE_STATUS_ON_FILE);
     assertTrue("not a file: " + status, status.isFile());
@@ -236,13 +206,13 @@ public void testCostOfGetFileStatusOnFile() throws Throwable {
   public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
     describe("performing getFileStatus on an empty directory");
     Path dir = dir(methodPath());
-    S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true,
+    S3AFileStatus status = verifyInnerGetFileStatus(dir, true,
         StatusProbeEnum.ALL,
         GET_FILE_STATUS_ON_DIR_MARKER);
     assertSame("not empty: " + status, Tristate.TRUE,
         status.isEmptyDirectory());
     // but now only ask for the directories and the file check is skipped.
-    verifyRawInnerGetFileStatus(dir, false,
+    verifyInnerGetFileStatus(dir, false,
         StatusProbeEnum.DIRECTORIES,
         FILE_STATUS_DIR_PROBE);
 
@@ -254,7 +224,7 @@ public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
   @Test
   public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
     describe("performing getFileStatus on a missing file");
-    interceptRawGetFileStatusFNFE(methodPath(), false,
+    interceptGetFileStatusFNFE(methodPath(), false,
         StatusProbeEnum.ALL,
         GET_FILE_STATUS_FNFE);
   }
@@ -262,7 +232,7 @@ public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
   @Test
   public void testCostOfRootFileStatus() throws Throwable {
     Path root = path("/");
-    S3AFileStatus rootStatus = verifyRawInnerGetFileStatus(
+    S3AFileStatus rootStatus = verifyInnerGetFileStatus(
             root,
             false,
             StatusProbeEnum.ALL,
@@ -275,7 +245,7 @@ public void testCostOfRootFileStatus() throws Throwable {
     Assertions.assertThat(rootStatus.isEmptyDirectory())
             .isEqualTo(Tristate.UNKNOWN);
 
-    rootStatus = verifyRawInnerGetFileStatus(
+    rootStatus = verifyInnerGetFileStatus(
             root,
             true,
             StatusProbeEnum.ALL,
@@ -305,7 +275,7 @@ public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
     describe("performing getFileStatus on a non-empty directory");
     Path dir = dir(methodPath());
     file(new Path(dir, "simple.txt"));
-    S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true,
+    S3AFileStatus status = verifyInnerGetFileStatus(dir, true,
         StatusProbeEnum.ALL,
         GET_FILE_STATUS_ON_DIR);
     assertEmptyDirStatus(status, Tristate.FALSE);
@@ -349,24 +319,23 @@ public void testCostOfCopyFromLocalFile() throws Throwable {
   @Test
   public void testDirProbes() throws Throwable {
     describe("Test directory probe cost");
-    assumeUnguarded();
     S3AFileSystem fs = getFileSystem();
     // Create the empty directory.
     Path emptydir = dir(methodPath());
 
     // head probe fails
-    interceptRawGetFileStatusFNFE(emptydir, false,
+    interceptGetFileStatusFNFE(emptydir, false,
         StatusProbeEnum.HEAD_ONLY,
         FILE_STATUS_FILE_PROBE);
 
     // a LIST will find it and declare as empty
-    S3AFileStatus status = verifyRawInnerGetFileStatus(emptydir, true,
+    S3AFileStatus status = verifyInnerGetFileStatus(emptydir, true,
         StatusProbeEnum.LIST_ONLY,
         FILE_STATUS_DIR_PROBE);
     assertEmptyDirStatus(status, Tristate.TRUE);
 
     // skip all probes and expect no operations to take place
-    interceptRawGetFileStatusFNFE(emptydir, false,
+    interceptGetFileStatusFNFE(emptydir, false,
         EnumSet.noneOf(StatusProbeEnum.class),
         NO_IO);
 
@@ -375,17 +344,16 @@ public void testDirProbes() throws Throwable {
     String emptyDirTrailingSlash = fs.pathToKey(emptydir.getParent())
         + "/" + emptydir.getName() +  "/";
     // A HEAD request does not probe for keys with a trailing /
-    interceptRaw(FileNotFoundException.class, "",
+    interceptOperation(FileNotFoundException.class, "",
         NO_IO, () ->
         fs.s3GetFileStatus(emptydir, emptyDirTrailingSlash,
-            StatusProbeEnum.HEAD_ONLY, null, false));
+            StatusProbeEnum.HEAD_ONLY, false));
 
     // but ask for a directory marker and you get the entry
-    status = verifyRaw(FILE_STATUS_DIR_PROBE, () ->
+    status = verify(FILE_STATUS_DIR_PROBE, () ->
         fs.s3GetFileStatus(emptydir,
             emptyDirTrailingSlash,
             StatusProbeEnum.LIST_ONLY,
-            null,
             true));
     assertEquals(emptydir, status.getPath());
     assertEmptyDirStatus(status, Tristate.TRUE);
@@ -397,80 +365,13 @@ public void testNeedEmptyDirectoryProbeRequiresList() throws Throwable {
 
     intercept(IllegalArgumentException.class, "", () ->
             fs.s3GetFileStatus(new Path("/something"), "/something",
-                StatusProbeEnum.HEAD_ONLY, null, true));
-  }
-  @Test
-  public void testCreateCost() throws Throwable {
-    describe("Test file creation cost -raw only");
-    assumeUnguarded();
-    Path testFile = methodPath();
-    // when overwrite is false, the path is checked for existence.
-    create(testFile, false,
-        CREATE_FILE_NO_OVERWRITE);
-    // but when true: only the directory checks take place.
-    create(testFile, true, CREATE_FILE_OVERWRITE);
-  }
-
-  @Test
-  public void testCreateCostFileExists() throws Throwable {
-    describe("Test cost of create file failing with existing file");
-    assumeUnguarded();
-    Path testFile = file(methodPath());
-
-    // now there is a file there, an attempt with overwrite == false will
-    // fail on the first HEAD.
-    interceptRaw(FileAlreadyExistsException.class, "",
-        FILE_STATUS_FILE_PROBE,
-        () -> file(testFile, false));
-  }
-
-  @Test
-  public void testCreateCostDirExists() throws Throwable {
-    describe("Test cost of create file failing with existing dir");
-    assumeUnguarded();
-    Path testFile = dir(methodPath());
-
-    // now there is a file there, an attempt with overwrite == false will
-    // fail on the first HEAD.
-    interceptRaw(FileAlreadyExistsException.class, "",
-        GET_FILE_STATUS_ON_DIR_MARKER,
-        () -> file(testFile, false));
-  }
-
-  /**
-   * Use the builder API.
-   * This always looks for a parent unless the caller says otherwise.
-   */
-  @Test
-  public void testCreateBuilder() throws Throwable {
-    describe("Test builder file creation cost -raw only");
-    assumeUnguarded();
-    Path testFile = methodPath();
-    dir(testFile.getParent());
-
-    // builder defaults to looking for parent existence (non-recursive)
-    buildFile(testFile, false,  false,
-        GET_FILE_STATUS_FNFE                // destination file
-            .plus(FILE_STATUS_DIR_PROBE));  // parent dir
-    // recursive = false and overwrite=true:
-    // only make sure the dest path isn't a directory.
-    buildFile(testFile, true, true,
-        FILE_STATUS_DIR_PROBE);
-
-    // now there is a file there, an attempt with overwrite == false will
-    // fail on the first HEAD.
-    interceptRaw(FileAlreadyExistsException.class, "",
-        GET_FILE_STATUS_ON_FILE,
-        () -> buildFile(testFile, false, true,
-            GET_FILE_STATUS_ON_FILE));
+                StatusProbeEnum.HEAD_ONLY, true));
   }
 
   @Test
   public void testCostOfGlobStatus() throws Throwable {
     describe("Test globStatus has expected cost");
     S3AFileSystem fs = getFileSystem();
-    assume("Unguarded FS only", !fs.hasMetadataStore());
-
     Path basePath = path("testCostOfGlobStatus/nextFolder/");
 
     // create a bunch of files
@@ -482,7 +383,7 @@ public void testCostOfGlobStatus() throws Throwable {
     fs.globStatus(basePath.suffix("/*"));
     // 2 head + 1 list from getFileStatus on path,
     // plus 1 list to match the glob pattern
-    verifyRaw(LIST_STATUS_LIST_OP,
+    verify(LIST_STATUS_LIST_OP,
         () -> fs.globStatus(basePath.suffix("/*")));
   }
 
@@ -490,8 +391,6 @@ public void testCostOfGlobStatus() throws Throwable {
   public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable {
     describe("Test globStatus does not attempt to resolve symlinks");
     S3AFileSystem fs = getFileSystem();
-    assume("Unguarded FS only", !fs.hasMetadataStore());
-
     Path basePath = path("testCostOfGlobStatusNoSymlinkResolution/f/");
 
     // create a single file, globStatus returning a single file on a pattern
@@ -501,7 +400,7 @@ public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable {
     // unguarded: 2 head + 1 list from getFileStatus on path,
     // plus 1 list to match the glob pattern
     // no additional operations from symlink resolution
-    verifyRaw(LIST_STATUS_LIST_OP,
+    verify(LIST_STATUS_LIST_OP,
         () -> fs.globStatus(basePath.suffix("/*")));
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
new file mode 100644
index 0000000000000..70dc5ee476c47
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsContextImpl;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertCapabilities;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disablePrefetching;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BYTES;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_WRITE_BYTES;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.enableIOStatisticsContext;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.getCurrentIOStatisticsContext;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.getThreadSpecificIOStatisticsContext;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.setThreadIOStatisticsContext;
+import static org.apache.hadoop.util.functional.RemoteIterators.foreach;
+
+/**
+ * Tests to verify the Thread-level IOStatistics.
+ */
+public class ITestS3AIOStatisticsContext extends AbstractS3ATestBase {
+
+  private static final int SMALL_THREADS = 2;
+  private static final int BYTES_BIG = 100;
+  private static final int BYTES_SMALL = 50;
+  private static final String[] IOSTATISTICS_CONTEXT_CAPABILITY =
+      new String[] {StreamCapabilities.IOSTATISTICS_CONTEXT};
+  private ExecutorService executor;
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration configuration = super.createConfiguration();
+    disablePrefetching(configuration);
+    enableIOStatisticsContext();
+    return configuration;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    executor = HadoopExecutors.newFixedThreadPool(SMALL_THREADS);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    if (executor != null) {
+      executor.shutdown();
+    }
+    super.teardown();
+  }
+
+  /**
+   * Verify that S3AInputStream aggregates per thread IOStats collection
+   * correctly.
+   */
+  @Test
+  public void testS3AInputStreamIOStatisticsContext()
+      throws Exception {
+    S3AFileSystem fs = getFileSystem();
+    Path path = path(getMethodName());
+    byte[] data = dataset(256, 'a', 'z');
+    byte[] readDataFirst = new byte[BYTES_BIG];
+    byte[] readDataSecond = new byte[BYTES_SMALL];
+    writeDataset(fs, path, data, data.length, 1024, true);
+
+    CountDownLatch latch = new CountDownLatch(SMALL_THREADS);
+
+    try {
+
+      for (int i = 0; i < SMALL_THREADS; i++) {
+        executor.submit(() -> {
+          try {
+            // get the thread context and reset
+            IOStatisticsContext context =
+                getAndResetThreadStatisticsContext();
+            try (FSDataInputStream in = fs.open(path)) {
+              // Assert the InputStream's stream capability to support
+              // IOStatisticsContext.
+              assertCapabilities(in, IOSTATISTICS_CONTEXT_CAPABILITY, null);
+              in.seek(50);
+              in.read(readDataFirst);
+            }
+            assertContextBytesRead(context, BYTES_BIG);
+            // Stream is closed for a thread. Re-open and do more operations.
+            try (FSDataInputStream in = fs.open(path)) {
+              in.seek(100);
+              in.read(readDataSecond);
+            }
+            assertContextBytesRead(context, BYTES_BIG + BYTES_SMALL);
+
+            latch.countDown();
+          } catch (Exception e) {
+            latch.countDown();
+            setFutureException(e);
+            LOG.error("An error occurred while doing a task in the thread", e);
+          } catch (AssertionError ase) {
+            latch.countDown();
+            setFutureAse(ase);
+            throw ase;
+          }
+        });
+      }
+      // wait for tasks to finish.
+      latch.await();
+    } finally {
+      executor.shutdown();
+    }
+
+    // Check if an Exception or ASE was caught while the test threads were running.
+    maybeReThrowFutureException();
+    maybeReThrowFutureASE();
+
+  }
+
+  /**
+   * get the thread context and reset.
+   * @return thread context
+   */
+  private static IOStatisticsContext getAndResetThreadStatisticsContext() {
+    assertTrue("thread-level IOStatistics should be enabled by default",
+        IOStatisticsContext.enabled());
+    IOStatisticsContext context =
+        IOStatisticsContext.getCurrentIOStatisticsContext();
+    context.reset();
+    return context;
+  }
+
+  /**
+   * Verify that S3ABlockOutputStream aggregates per thread IOStats collection
+   * correctly.
+   */
+  @Test
+  public void testS3ABlockOutputStreamIOStatisticsContext()
+      throws Exception {
+    S3AFileSystem fs = getFileSystem();
+    Path path = path(getMethodName());
+    byte[] writeDataFirst = new byte[BYTES_BIG];
+    byte[] writeDataSecond = new byte[BYTES_SMALL];
+
+    final ExecutorService executorService =
+        HadoopExecutors.newFixedThreadPool(SMALL_THREADS);
+    CountDownLatch latch = new CountDownLatch(SMALL_THREADS);
+
+    try {
+      for (int i = 0; i < SMALL_THREADS; i++) {
+        executorService.submit(() -> {
+          try {
+            // get the thread context and reset
+            IOStatisticsContext context =
+                getAndResetThreadStatisticsContext();
+            try (FSDataOutputStream out = fs.create(path)) {
+              // Assert the OutputStream's stream capability to support
+              // IOStatisticsContext.
+              assertCapabilities(out, IOSTATISTICS_CONTEXT_CAPABILITY, null);
+              out.write(writeDataFirst);
+            }
+            assertContextBytesWrite(context, BYTES_BIG);
+
+            // Stream is closed for a thread. Re-open and do more operations.
+            try (FSDataOutputStream out = fs.create(path)) {
+              out.write(writeDataSecond);
+            }
+            assertContextBytesWrite(context, BYTES_BIG + BYTES_SMALL);
+            latch.countDown();
+          } catch (Exception e) {
+            latch.countDown();
+            setFutureException(e);
+            LOG.error("An error occurred while doing a task in the thread", e);
+          } catch (AssertionError ase) {
+            latch.countDown();
+            setFutureAse(ase);
+            throw ase;
+          }
+        });
+      }
+      // wait for tasks to finish.
+      latch.await();
+    } finally {
+      executorService.shutdown();
+    }
+
+    // Check if an Excp or ASE was caught while the test threads were running.
+    maybeReThrowFutureException();
+    maybeReThrowFutureASE();
+  }
+
+  /**
+   * Verify stats collection and aggregation for constructor thread, Junit
+   * thread and a worker thread.
+   */
+  @Test
+  public void testThreadIOStatisticsForDifferentThreads()
+      throws IOException, InterruptedException {
+    S3AFileSystem fs = getFileSystem();
+    Path path = path(getMethodName());
+    byte[] data = new byte[BYTES_BIG];
+    long threadIdForTest = Thread.currentThread().getId();
+    IOStatisticsContext context =
+        getAndResetThreadStatisticsContext();
+    Assertions.assertThat(((IOStatisticsContextImpl)context).getThreadID())
+        .describedAs("Thread ID of %s", context)
+        .isEqualTo(threadIdForTest);
+    Assertions.assertThat(((IOStatisticsContextImpl)context).getID())
+        .describedAs("ID of %s", context)
+        .isGreaterThan(0);
+
+    // Write in the Junit thread.
+    try (FSDataOutputStream out = fs.create(path)) {
+      out.write(data);
+    }
+
+    // Read in the Junit thread.
+    try (FSDataInputStream in = fs.open(path)) {
+      in.read(data);
+    }
+
+    // Worker thread work and wait for it to finish.
+    TestWorkerThread workerThread = new TestWorkerThread(path, null);
+    long workerThreadID = workerThread.getId();
+    LOG.info("Worker thread ID: {} ", workerThreadID);
+    workerThread.start();
+    workerThread.join();
+
+    assertThreadStatisticsForThread(threadIdForTest, BYTES_BIG);
+    assertThreadStatisticsForThread(workerThreadID, BYTES_SMALL);
+  }
+
+  /**
+   * Verify stats collection and aggregation for constructor thread, Junit
+   * thread and a worker thread.
+   */
+  @Test
+  public void testThreadSharingIOStatistics()
+      throws IOException, InterruptedException {
+    S3AFileSystem fs = getFileSystem();
+    Path path = path(getMethodName());
+    byte[] data = new byte[BYTES_BIG];
+    long threadIdForTest = Thread.currentThread().getId();
+    IOStatisticsContext context =
+        getAndResetThreadStatisticsContext();
+
+
+    // Write in the Junit thread.
+    try (FSDataOutputStream out = fs.create(path)) {
+      out.write(data);
+    }
+
+    // Read in the Junit thread.
+    try (FSDataInputStream in = fs.open(path)) {
+      in.read(data);
+    }
+
+    // Worker thread will share the same context.
+    TestWorkerThread workerThread = new TestWorkerThread(path, context);
+    long workerThreadID = workerThread.getId();
+    workerThread.start();
+    workerThread.join();
+
+    assertThreadStatisticsForThread(threadIdForTest, BYTES_BIG + BYTES_SMALL);
+
+  }
+
+  /**
+   * Test to verify if setting the current IOStatisticsContext removes the
+   * current context and creates a new instance of it.
+   */
+  @Test
+  public void testSettingNullIOStatisticsContext() {
+    IOStatisticsContext ioStatisticsContextBefore =
+        getCurrentIOStatisticsContext();
+    // Set the current IOStatisticsContext to null, which should remove the
+    // context and set a new one.
+    setThreadIOStatisticsContext(null);
+    // Get the context again after setting.
+    IOStatisticsContext ioStatisticsContextAfter =
+        getCurrentIOStatisticsContext();
+    //Verify the context ID after setting to null is different than the previous
+    // one.
+    Assertions.assertThat(ioStatisticsContextBefore.getID())
+        .describedAs("A new IOStaticsContext should be set after setting the "
+            + "current to null")
+        .isNotEqualTo(ioStatisticsContextAfter.getID());
+  }
+
+  /**
+   * Assert bytes written by the statistics context.
+   *
+   * @param context statistics context.
+   * @param bytes expected bytes.
+   */
+  private void assertContextBytesWrite(IOStatisticsContext context,
+      int bytes) {
+    verifyStatisticCounterValue(
+        context.getIOStatistics(),
+        STREAM_WRITE_BYTES,
+        bytes);
+  }
+
+  /**
+   * Assert bytes read by the statistics context.
+   *
+   * @param context statistics context.
+   * @param readBytes expected bytes.
+   */
+  private void assertContextBytesRead(IOStatisticsContext context,
+      int readBytes) {
+    verifyStatisticCounterValue(
+        context.getIOStatistics(),
+        STREAM_READ_BYTES,
+        readBytes);
+  }
+
+  /**
+   * Assert fixed bytes wrote and read for a particular thread ID.
+   *
+   * @param testThreadId                thread ID.
+   * @param expectedBytesWrittenAndRead expected bytes.
+   */
+  private void assertThreadStatisticsForThread(long testThreadId,
+      int expectedBytesWrittenAndRead) {
+    LOG.info("Thread ID to be asserted: {}", testThreadId);
+    IOStatisticsContext ioStatisticsContext =
+        getThreadSpecificIOStatisticsContext(testThreadId);
+    Assertions.assertThat(ioStatisticsContext)
+        .describedAs("IOStatisticsContext for %d", testThreadId)
+        .isNotNull();
+
+
+    IOStatistics ioStatistics = ioStatisticsContext.snapshot();
+
+
+    assertThatStatisticCounter(ioStatistics,
+            STREAM_WRITE_BYTES)
+        .describedAs("Bytes written are not as expected for thread : %s",
+                testThreadId)
+        .isEqualTo(expectedBytesWrittenAndRead);
+
+    assertThatStatisticCounter(ioStatistics,
+        STREAM_READ_BYTES)
+        .describedAs("Bytes read are not as expected for thread : %s",
+                testThreadId)
+        .isEqualTo(expectedBytesWrittenAndRead);
+  }
+
+  @Test
+  public void testListingStatisticsContext() throws Throwable {
+    describe("verify the list operations update on close()");
+
+    S3AFileSystem fs = getFileSystem();
+    Path path = methodPath();
+    fs.mkdirs(methodPath());
+
+    // after all setup, get the reset context
+    IOStatisticsContext context =
+        getAndResetThreadStatisticsContext();
+    IOStatistics ioStatistics = context.getIOStatistics();
+
+    fs.listStatus(path);
+    verifyStatisticCounterValue(ioStatistics,
+        StoreStatisticNames.OBJECT_LIST_REQUEST,
+        1);
+
+    context.reset();
+    foreach(fs.listStatusIterator(path), i -> {});
+    verifyStatisticCounterValue(ioStatistics,
+        StoreStatisticNames.OBJECT_LIST_REQUEST,
+        1);
+
+    context.reset();
+    foreach(fs.listLocatedStatus(path), i -> {});
+    verifyStatisticCounterValue(ioStatistics,
+        StoreStatisticNames.OBJECT_LIST_REQUEST,
+        1);
+
+    context.reset();
+    foreach(fs.listFiles(path, true), i -> {});
+    verifyStatisticCounterValue(ioStatistics,
+        StoreStatisticNames.OBJECT_LIST_REQUEST,
+        1);
+  }
+
+  @Test
+  public void testListingThroughTaskPool() throws Throwable {
+    describe("verify the list operations are updated through taskpool");
+
+    S3AFileSystem fs = getFileSystem();
+    Path path = methodPath();
+    fs.mkdirs(methodPath());
+
+    // after all setup, get the reset context
+    IOStatisticsContext context =
+        getAndResetThreadStatisticsContext();
+    IOStatistics ioStatistics = context.getIOStatistics();
+
+    CloseableTaskPoolSubmitter submitter =
+        new CloseableTaskPoolSubmitter(executor);
+    TaskPool.foreach(fs.listStatusIterator(path))
+        .executeWith(submitter)
+        .run(i -> {});
+
+    verifyStatisticCounterValue(ioStatistics,
+        StoreStatisticNames.OBJECT_LIST_REQUEST,
+        1);
+
+  }
+
+  /**
+   * Simulating doing some work in a separate thread.
+   * If constructed with an IOStatisticsContext then
+   * that context is switched to before performing the IO.
+   */
+  private class TestWorkerThread extends Thread implements Runnable {
+    private final Path workerThreadPath;
+
+    private final IOStatisticsContext ioStatisticsContext;
+
+    /**
+     * create.
+     * @param workerThreadPath thread path.
+     * @param ioStatisticsContext optional statistics context     *
+     */
+    TestWorkerThread(
+        final Path workerThreadPath,
+        final IOStatisticsContext ioStatisticsContext) {
+      this.workerThreadPath = workerThreadPath;
+      this.ioStatisticsContext = ioStatisticsContext;
+    }
+
+    @Override
+    public void run() {
+      // Setting the worker thread's name.
+      Thread.currentThread().setName("worker thread");
+      S3AFileSystem fs = getFileSystem();
+      byte[] data = new byte[BYTES_SMALL];
+
+      // maybe switch context
+      if (ioStatisticsContext != null) {
+        IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext);
+      }
+      // Storing context in a field to not lose the reference in a GC.
+      IOStatisticsContext ioStatisticsContextWorkerThread =
+          getCurrentIOStatisticsContext();
+
+      // Write in the worker thread.
+      try (FSDataOutputStream out = fs.create(workerThreadPath)) {
+        out.write(data);
+      } catch (IOException e) {
+        throw new UncheckedIOException("Failure while writing", e);
+      }
+
+      //Read in the worker thread.
+      try (FSDataInputStream in = fs.open(workerThreadPath)) {
+        in.read(data);
+      } catch (IOException e) {
+        throw new UncheckedIOException("Failure while reading", e);
+      }
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInconsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInconsistency.java
deleted file mode 100644
index a49998d50d5a6..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInconsistency.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Source;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.test.LambdaTestUtils;
-
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.FailureInjectionPolicy.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.test.LambdaTestUtils.eventually;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Tests S3A behavior under forced inconsistency via {@link
- * InconsistentAmazonS3Client}.
- *
- * These tests are for validating expected behavior *without* S3Guard, but
- * may also run with S3Guard enabled.  For tests that validate S3Guard's
- * consistency features, see {@link ITestS3GuardListConsistency}.
- */
-public class ITestS3AInconsistency extends AbstractS3ATestBase {
-
-  private static final int OPEN_READ_ITERATIONS = 10;
-
-  public static final int INCONSISTENCY_MSEC = 800;
-
-  private static final int INITIAL_RETRY = 128;
-
-  private static final int RETRIES = 4;
-
-  /** By using a power of 2 for the initial time, the total is a shift left. */
-  private static final int TOTAL_RETRY_DELAY = INITIAL_RETRY << RETRIES;
-
-  /**
-   * S3 Client side encryption when enabled should skip this test.
-   */
-  @Before
-  public void setUp() {
-    skipIfClientSideEncryption();
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // reduce retry limit so FileNotFoundException cases timeout faster,
-    // speeding up the tests
-    removeBaseAndBucketOverrides(conf,
-        CHANGE_DETECT_SOURCE,
-        CHANGE_DETECT_MODE,
-        RETRY_LIMIT,
-        RETRY_INTERVAL,
-        METADATASTORE_AUTHORITATIVE,
-        S3_CLIENT_FACTORY_IMPL);
-    conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
-        S3ClientFactory.class);
-    conf.set(FAIL_INJECT_INCONSISTENCY_KEY, DEFAULT_DELAY_KEY_SUBSTRING);
-    // the reads are always inconsistent
-    conf.setFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY, 1.0f);
-    // but the inconsistent time is less than exponential growth of the
-    // retry interval (128 -> 256 -> 512 -> 1024
-    conf.setLong(FAIL_INJECT_INCONSISTENCY_MSEC, INCONSISTENCY_MSEC);
-    conf.setInt(RETRY_LIMIT, RETRIES);
-    conf.set(RETRY_INTERVAL, String.format("%dms", INITIAL_RETRY));
-    return conf;
-  }
-
-  @Test
-  public void testGetFileStatus() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-
-    // 1. Make sure no ancestor dirs exist
-    Path dir = path("ancestor");
-    fs.delete(dir, true);
-    waitUntilDeleted(dir);
-
-    // 2. Create a descendant file, which implicitly creates ancestors
-    // This file has delayed visibility.
-    touch(getFileSystem(),
-        path("ancestor/file-" + DEFAULT_DELAY_KEY_SUBSTRING));
-
-    // 3. Assert expected behavior.  If S3Guard is enabled, we should be able
-    // to get status for ancestor.  If S3Guard is *not* enabled, S3A will
-    // fail to infer the existence of the ancestor since visibility of the
-    // child file is delayed, and its key prefix search will return nothing.
-    try {
-      FileStatus status = fs.getFileStatus(dir);
-      if (fs.hasMetadataStore()) {
-        assertTrue("Ancestor is dir", status.isDirectory());
-      } else {
-        fail("getFileStatus should fail due to delayed visibility.");
-      }
-    } catch (FileNotFoundException e) {
-      if (fs.hasMetadataStore()) {
-        fail("S3Guard failed to list parent of inconsistent child.");
-      }
-      LOG.info("File not found, as expected.");
-    }
-  }
-
-
-  /**
-   * Ensure that deleting a file with an open read stream does eventually cause
-   * readers to get a FNFE, even with S3Guard and its retries enabled.
-   * In real usage, S3Guard should be enabled for all clients that modify the
-   * file, so the delete would be immediately recorded in the MetadataStore.
-   * Here, however, we test deletion from under S3Guard to make sure it still
-   * eventually propagates the FNFE after any retry policies are exhausted.
-   */
-  @Test
-  public void testOpenDeleteRead() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    ChangeDetectionPolicy changeDetectionPolicy =
-        fs.getChangeDetectionPolicy();
-    Assume.assumeFalse("FNF not expected when using a bucket with"
-            + " object versioning",
-        changeDetectionPolicy.getSource() == Source.VersionId);
-
-    Path p = path("testOpenDeleteRead.txt");
-    writeTextFile(fs, p, "1337c0d3z", true);
-    try (InputStream s = fs.open(p)) {
-      // Disable s3guard, delete file underneath it, re-enable s3guard
-      MetadataStore metadataStore = fs.getMetadataStore();
-      fs.setMetadataStore(new NullMetadataStore());
-      fs.delete(p, false);
-      fs.setMetadataStore(metadataStore);
-      eventually(TOTAL_RETRY_DELAY * 2, INITIAL_RETRY * 2, () -> {
-        intercept(FileNotFoundException.class, () -> s.read());
-      });
-    }
-  }
-
-  /**
-   * Test read() path behavior when getFileStatus() succeeds but subsequent
-   * read() on the input stream fails due to eventual consistency.
-   * There are many points in the InputStream codepaths that can fail. We set
-   * a probability of failure and repeat the test multiple times to achieve
-   * decent coverage.
-   */
-  @Test
-  public void testOpenFailOnRead() throws Exception {
-
-    S3AFileSystem fs = getFileSystem();
-
-    // 1. Patch in a different failure injection policy with <1.0 probability
-    Configuration conf = fs.getConf();
-    conf.setFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY, 0.5f);
-    InconsistentAmazonS3Client.setFailureInjectionPolicy(fs,
-        new FailureInjectionPolicy(conf));
-
-    // 2. Make sure no ancestor dirs exist
-    Path dir = path("ancestor");
-    fs.delete(dir, true);
-    waitUntilDeleted(dir);
-
-    // 3. Create a descendant file, which implicitly creates ancestors
-    // This file has delayed visibility.
-    describe("creating test file");
-    Path path = path("ancestor/file-to-read-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    writeTextFile(getFileSystem(), path, "Reading is fun", false);
-
-    // 4. Clear inconsistency so the first getFileStatus() can succeed, if we
-    // are not using S3Guard. If we are using S3Guard, it should tolerate the
-    // delayed visibility.
-    if (!fs.hasMetadataStore()) {
-      InconsistentAmazonS3Client.clearInconsistency(fs);
-    }
-
-    // ? Do we need multiple iterations when S3Guard is disabled?  For now,
-    // leaving it in
-    for (int i = 0; i < OPEN_READ_ITERATIONS; i++) {
-      doOpenFailOnReadTest(fs, path, i);
-    }
-  }
-
-  private void doOpenFailOnReadTest(S3AFileSystem fs, Path path, int iteration)
-      throws Exception {
-
-    // 4. Open the file
-    describe(String.format("i=%d: opening test file", iteration));
-    try(InputStream in = fs.open(path)) {
-      // 5. Assert expected behavior on read() failure.
-      int l = 4;
-      byte[] buf = new byte[l];
-      describe("reading test file");
-      // Use both read() variants
-      if ((iteration % 2) == 0) {
-        assertEquals(l, in.read(buf, 0, l));
-      } else {
-        in.read();
-      }
-    } catch (FileNotFoundException e) {
-      if (fs.hasMetadataStore()) {
-        LOG.error("Error:", e);
-        ContractTestUtils.fail("S3Guard failed to handle fail-on-read", e);
-      } else {
-        LOG.info("File not found on read(), as expected.");
-      }
-    }
-  }
-
-  private void waitUntilDeleted(final Path p) throws Exception {
-    LambdaTestUtils.eventually(30 * 1000, 1000,
-        () -> assertPathDoesNotExist("Dir should be deleted", p));
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMetadataPersistenceException.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMetadataPersistenceException.java
deleted file mode 100644
index 7a980a39effa6..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMetadataPersistenceException.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-import org.apache.hadoop.io.IOUtils;
-
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Tests failed writes to metadata store generate the expected
- * MetadataPersistenceException.
- */
-@RunWith(Parameterized.class)
-public class ITestS3AMetadataPersistenceException extends AbstractS3ATestBase {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ITestS3AMetadataPersistenceException.class);
-
-  private S3AFileSystem fs;
-  private IOException ioException;
-  private final boolean failOnError;
-
-  public ITestS3AMetadataPersistenceException(boolean failOnError) {
-    this.failOnError = failOnError;
-  }
-
-  @Parameterized.Parameters
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {true},
-        {false}
-    });
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    conf.set(Constants.FAIL_ON_METADATA_WRITE_ERROR,
-        Boolean.toString(failOnError));
-    // replaced in setup() by IOExceptionMetadataStore
-    conf.setClass(Constants.S3_METADATA_STORE_IMPL,
-        NullMetadataStore.class,
-        MetadataStore.class);
-    return conf;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    S3AFileSystem contractFs = getFileSystem();
-    fs = (S3AFileSystem) FileSystem.newInstance(
-        contractFs.getUri(), contractFs.getConf());
-    ioException = new IOException();
-    IOExceptionMetadataStore metadataStore =
-        new IOExceptionMetadataStore(ioException);
-    metadataStore.initialize(getConfiguration(),
-        new S3Guard.TtlTimeProvider(getConfiguration()));
-    fs.setMetadataStore(metadataStore);
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    IOUtils.cleanupWithLogger(LOG, fs);
-    super.teardown();
-  }
-
-  @Test
-  public void testFailedMetadataUpdate() throws Throwable {
-    // write a trivial file
-    Path testFile = path("testFailedMetadataUpdate");
-    try {
-      FSDataOutputStream outputStream = fs.create(testFile);
-      outputStream.write(1);
-
-      if (failOnError) {
-        // close should throw the expected exception
-        MetadataPersistenceException thrown =
-            intercept(
-                MetadataPersistenceException.class,
-                outputStream::close);
-        assertEquals("cause didn't match original exception",
-            ioException, thrown.getCause());
-      } else {
-        MetricDiff ignoredCount = new MetricDiff(fs, Statistic.IGNORED_ERRORS);
-
-        // close should merely log and increment the statistic
-        outputStream.close();
-        ignoredCount.assertDiffEquals("ignored errors", 1);
-      }
-    } finally {
-      // turn off the store and forcibly delete from the raw bucket.
-      fs.setMetadataStore(new NullMetadataStore());
-      fs.delete(testFile, false);
-    }
-  }
-
-  private static class IOExceptionMetadataStore extends LocalMetadataStore {
-    private final IOException ioException;
-
-    private IOExceptionMetadataStore(IOException ioException) {
-      this.ioException = ioException;
-    }
-
-    @Override
-    public void put(PathMetadata meta,
-        final BulkOperationState operationState) throws IOException {
-      throw ioException;
-    }
-
-    @Override
-    public void put(final PathMetadata meta) throws IOException {
-      put(meta, null);
-    }
-
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
index b4d2527a46af8..2d29282ad0195 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
@@ -36,8 +36,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonPathCapabilities;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.store.EtagChecksum;
 import org.apache.hadoop.test.LambdaTestUtils;
@@ -102,18 +102,6 @@ public void testCreateNonRecursiveSuccess() throws IOException {
     assertIsFile(shouldWork);
   }
 
-  @Test(expected = FileNotFoundException.class)
-  public void testCreateNonRecursiveNoParent() throws IOException {
-    createNonRecursive(path("/recursive/node"));
-  }
-
-  @Test(expected = FileAlreadyExistsException.class)
-  public void testCreateNonRecursiveParentIsFile() throws IOException {
-    Path parent = path("/file.txt");
-    touch(getFileSystem(), parent);
-    createNonRecursive(new Path(parent, "fail"));
-  }
-
   @Test
   public void testPutObjectDirect() throws Throwable {
     final S3AFileSystem fs = getFileSystem();
@@ -126,7 +114,7 @@ public void testPutObjectDirect() throws Throwable {
           new ByteArrayInputStream("PUT".getBytes()),
           metadata);
       LambdaTestUtils.intercept(IllegalStateException.class,
-          () -> fs.putObjectDirect(put));
+          () -> fs.putObjectDirect(put, PutObjectOptions.keepingDirs()));
       assertPathDoesNotExist("put object was created", path);
     }
   }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
new file mode 100644
index 0000000000000..b1ab4afce4760
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.net.URI;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_DEFAULT_SIZE;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMaximum;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticGaugeValue;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_EXECUTOR_ACQUIRED;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MAX;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_OPENED;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS;
+import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
+
+/**
+ * Test the prefetching input stream, validates that the underlying S3ACachingInputStream and
+ * S3AInMemoryInputStream are working as expected.
+ */
+public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest {
+
+  public ITestS3APrefetchingInputStream() {
+    super(true);
+  }
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestS3APrefetchingInputStream.class);
+
+  private static final int S_1K = 1024;
+  private static final int S_1M = S_1K * S_1K;
+  // Path for file which should have length > block size so S3ACachingInputStream is used
+  private Path largeFile;
+  private FileSystem largeFileFS;
+  private int numBlocks;
+  private int blockSize;
+  private long largeFileSize;
+  // Size should be < block size so S3AInMemoryInputStream is used
+  private static final int SMALL_FILE_SIZE = S_1K * 16;
+
+
+  @Override
+  public Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+    conf.setBoolean(PREFETCH_ENABLED_KEY, true);
+    return conf;
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    super.teardown();
+    cleanupWithLogger(LOG, largeFileFS);
+    largeFileFS = null;
+  }
+
+  private void openFS() throws Exception {
+    Configuration conf = getConfiguration();
+    String largeFileUri = S3ATestUtils.getCSVTestFile(conf);
+
+    largeFile = new Path(largeFileUri);
+    blockSize = conf.getInt(PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE);
+    largeFileFS = new S3AFileSystem();
+    largeFileFS.initialize(new URI(largeFileUri), getConfiguration());
+    FileStatus fileStatus = largeFileFS.getFileStatus(largeFile);
+    largeFileSize = fileStatus.getLen();
+    numBlocks = calculateNumBlocks(largeFileSize, blockSize);
+  }
+
+  private static int calculateNumBlocks(long largeFileSize, int blockSize) {
+    if (largeFileSize == 0) {
+      return 0;
+    } else {
+      return ((int) (largeFileSize / blockSize)) + (largeFileSize % blockSize > 0 ? 1 : 0);
+    }
+  }
+
+  @Test
+  public void testReadLargeFileFully() throws Throwable {
+    describe("read a large file fully, uses S3ACachingInputStream");
+    IOStatistics ioStats;
+    openFS();
+
+    try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+      ioStats = in.getIOStatistics();
+
+      byte[] buffer = new byte[S_1M * 10];
+      long bytesRead = 0;
+
+      while (bytesRead < largeFileSize) {
+        in.readFully(buffer, 0, (int) Math.min(buffer.length, largeFileSize - bytesRead));
+        bytesRead += buffer.length;
+        // Blocks are fully read, no blocks should be cached
+        verifyStatisticGaugeValue(ioStats, STREAM_READ_BLOCKS_IN_FILE_CACHE,
+            0);
+      }
+
+      // Assert that first block is read synchronously, following blocks are prefetched
+      verifyStatisticCounterValue(ioStats, STREAM_READ_PREFETCH_OPERATIONS,
+          numBlocks - 1);
+      verifyStatisticCounterValue(ioStats, ACTION_HTTP_GET_REQUEST, numBlocks);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_OPENED, numBlocks);
+    }
+    // Verify that once stream is closed, all memory is freed
+    verifyStatisticGaugeValue(ioStats, STREAM_READ_ACTIVE_MEMORY_IN_USE, 0);
+    assertThatStatisticMaximum(ioStats,
+        ACTION_EXECUTOR_ACQUIRED + SUFFIX_MAX).isGreaterThan(0);
+  }
+
+  @Test
+  public void testReadLargeFileFullyLazySeek() throws Throwable {
+    describe("read a large file using readFully(position,buffer,offset,length),"
+        + " uses S3ACachingInputStream");
+    IOStatistics ioStats;
+    openFS();
+
+    try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+      ioStats = in.getIOStatistics();
+
+      byte[] buffer = new byte[S_1M * 10];
+      long bytesRead = 0;
+
+      while (bytesRead < largeFileSize) {
+        in.readFully(bytesRead, buffer, 0, (int) Math.min(buffer.length,
+            largeFileSize - bytesRead));
+        bytesRead += buffer.length;
+        // Blocks are fully read, no blocks should be cached
+        verifyStatisticGaugeValue(ioStats, STREAM_READ_BLOCKS_IN_FILE_CACHE,
+                0);
+      }
+
+      // Assert that first block is read synchronously, following blocks are prefetched
+      verifyStatisticCounterValue(ioStats, STREAM_READ_PREFETCH_OPERATIONS,
+              numBlocks - 1);
+      verifyStatisticCounterValue(ioStats, ACTION_HTTP_GET_REQUEST, numBlocks);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_OPENED, numBlocks);
+    }
+    // Verify that once stream is closed, all memory is freed
+    verifyStatisticGaugeValue(ioStats, STREAM_READ_ACTIVE_MEMORY_IN_USE, 0);
+  }
+
+  @Test
+  public void testRandomReadLargeFile() throws Throwable {
+    describe("random read on a large file, uses S3ACachingInputStream");
+    IOStatistics ioStats;
+    openFS();
+
+    try (FSDataInputStream in = largeFileFS.open(largeFile)) {
+      ioStats = in.getIOStatistics();
+
+      byte[] buffer = new byte[blockSize];
+
+      // Don't read block 0 completely so it gets cached on read after seek
+      in.read(buffer, 0, blockSize - S_1K * 10);
+
+      // Seek to block 2 and read all of it
+      in.seek(blockSize * 2);
+      in.read(buffer, 0, blockSize);
+
+      // Seek to block 4 but don't read: noop.
+      in.seek(blockSize * 4);
+
+      // Backwards seek, will use cached block 0
+      in.seek(S_1K * 5);
+      in.read();
+
+      // Expected to get block 0 (partially read), 1 (prefetch), 2 (fully read), 3 (prefetch)
+      // Blocks 0, 1, 3 were not fully read, so remain in the file cache
+      verifyStatisticCounterValue(ioStats, ACTION_HTTP_GET_REQUEST, 4);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_OPENED, 4);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_PREFETCH_OPERATIONS, 2);
+      verifyStatisticGaugeValue(ioStats, STREAM_READ_BLOCKS_IN_FILE_CACHE, 3);
+    }
+    verifyStatisticGaugeValue(ioStats, STREAM_READ_BLOCKS_IN_FILE_CACHE, 0);
+    verifyStatisticGaugeValue(ioStats, STREAM_READ_ACTIVE_MEMORY_IN_USE, 0);
+  }
+
+  @Test
+  public void testRandomReadSmallFile() throws Throwable {
+    describe("random read on a small file, uses S3AInMemoryInputStream");
+
+    byte[] data = ContractTestUtils.dataset(SMALL_FILE_SIZE, 'a', 26);
+    Path smallFile = path("randomReadSmallFile");
+    ContractTestUtils.writeDataset(getFileSystem(), smallFile, data, data.length, 16, true);
+
+    try (FSDataInputStream in = getFileSystem().open(smallFile)) {
+      IOStatistics ioStats = in.getIOStatistics();
+
+      byte[] buffer = new byte[SMALL_FILE_SIZE];
+
+      in.read(buffer, 0, S_1K * 4);
+      in.seek(S_1K * 12);
+      in.read(buffer, 0, S_1K * 4);
+
+      verifyStatisticCounterValue(ioStats, ACTION_HTTP_GET_REQUEST, 1);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_OPENED, 1);
+      verifyStatisticCounterValue(ioStats, STREAM_READ_PREFETCH_OPERATIONS, 0);
+      // The buffer pool is not used
+      verifyStatisticGaugeValue(ioStats, STREAM_READ_ACTIVE_MEMORY_IN_USE, 0);
+      // no prefetch ops, so no action_executor_acquired
+      assertThatStatisticMaximum(ioStats,
+          ACTION_EXECUTOR_ACQUIRED + SUFFIX_MAX).isEqualTo(-1);
+    }
+  }
+
+  @Test
+  public void testStatusProbesAfterClosingStream() throws Throwable {
+    describe("When the underlying input stream is closed, the prefetch input stream"
+        + " should still support some status probes");
+
+    byte[] data = ContractTestUtils.dataset(SMALL_FILE_SIZE, 'a', 26);
+    Path smallFile = methodPath();
+    ContractTestUtils.writeDataset(getFileSystem(), smallFile, data, data.length, 16, true);
+
+    FSDataInputStream in = getFileSystem().open(smallFile);
+
+    byte[] buffer = new byte[SMALL_FILE_SIZE];
+    in.read(buffer, 0, S_1K * 4);
+    in.seek(S_1K * 12);
+    in.read(buffer, 0, S_1K * 4);
+
+    long pos = in.getPos();
+    IOStatistics ioStats = in.getIOStatistics();
+    S3AInputStreamStatistics inputStreamStatistics =
+        ((S3APrefetchingInputStream) (in.getWrappedStream())).getS3AStreamStatistics();
+
+    assertNotNull("Prefetching input IO stats should not be null", ioStats);
+    assertNotNull("Prefetching input stream stats should not be null", inputStreamStatistics);
+    assertNotEquals("Position retrieved from prefetching input stream should be greater than 0", 0,
+        pos);
+
+    in.close();
+
+    // status probes after closing the input stream
+    long newPos = in.getPos();
+    IOStatistics newIoStats = in.getIOStatistics();
+    S3AInputStreamStatistics newInputStreamStatistics =
+        ((S3APrefetchingInputStream) (in.getWrappedStream())).getS3AStreamStatistics();
+
+    assertNotNull("Prefetching input IO stats should not be null", newIoStats);
+    assertNotNull("Prefetching input stream stats should not be null", newInputStreamStatistics);
+    assertNotEquals("Position retrieved from prefetching input stream should be greater than 0", 0,
+        newPos);
+
+    // compare status probes after closing of the stream with status probes done before
+    // closing the stream
+    assertEquals("Position retrieved through stream before and after closing should match", pos,
+        newPos);
+    assertEquals("IO stats retrieved through stream before and after closing should match", ioStats,
+        newIoStats);
+    assertEquals("Stream stats retrieved through stream before and after closing should match",
+        inputStreamStatistics, newInputStreamStatistics);
+
+    assertFalse("seekToNewSource() not supported with prefetch", in.seekToNewSource(10));
+
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java
deleted file mode 100644
index 770a99599f508..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARemoteFileChanged.java
+++ /dev/null
@@ -1,1549 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Optional;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.CopyObjectResult;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3Object;
-import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.junit.Assume;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Mode;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Source;
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.readUTF8;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.CHANGE_DETECTED;
-import static org.apache.hadoop.fs.s3a.select.SelectConstants.S3_SELECT_CAPABILITY;
-import static org.apache.hadoop.fs.s3a.select.SelectConstants.SELECT_SQL;
-import static org.apache.hadoop.test.LambdaTestUtils.eventually;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.when;
-
-/**
- * Test S3A remote file change detection.
- * This is a very parameterized test; the first three parameters
- * define configuration options for the tests, while the final one
- * declares the expected outcomes given those options.
- *
- * This test uses mocking to insert transient failures into the S3 client,
- * underneath the S3A Filesystem instance.
- *
- * This is used to simulate eventual consistency, so force the change policy
- * failure modes to be encountered.
- *
- * If changes are made to the filesystem such that the number of calls to
- * operations such as {@link S3AFileSystem#getObjectMetadata(Path)} are
- * changed, the number of failures which the mock layer must generate may
- * change.
- *
- * As the S3Guard auth mode flag does control whether or not a HEAD is issued
- * in a call to {@code getFileStatus()}; the test parameter {@link #authMode}
- * is used to help predict this count.
- *
- * <i>Important:</i> if you are seeing failures in this test after changing
- * one of the rename/copy/open operations, it may be that an increase,
- * decrease or change in the number of low-level S3 HEAD/GET operations is
- * triggering the failures.
- * Please review the changes to see that you haven't unintentionally done this.
- * If it is intentional, please update the parameters here.
- *
- * If you are seeing failures without such a change, and nobody else is,
- * it is likely that you have a different bucket configuration option which
- * is somehow triggering a regression. If you can work out which option
- * this is, then extend {@link #createConfiguration()} to reset that parameter
- * too.
- *
- * Note: to help debug these issues, set the log for this to DEBUG:
- * <pre>
- *   log4j.logger.org.apache.hadoop.fs.s3a.ITestS3ARemoteFileChanged=DEBUG
- * </pre>
- * The debug information printed will include a trace of where operations
- * are being called from, to help understand why the test is failing.
- */
-@RunWith(Parameterized.class)
-public class ITestS3ARemoteFileChanged extends AbstractS3ATestBase {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ITestS3ARemoteFileChanged.class);
-
-  private static final String TEST_DATA = "Some test data";
-
-  private static final byte[] TEST_DATA_BYTES = TEST_DATA.getBytes(
-      Charsets.UTF_8);
-  private static final int TEST_MAX_RETRIES = 4;
-  private static final String TEST_RETRY_INTERVAL = "1ms";
-  private static final String QUOTED_TEST_DATA =
-      "\"" + TEST_DATA + "\"";
-
-  private Optional<AmazonS3> originalS3Client = Optional.empty();
-
-  private static final String INCONSISTENT = "inconsistent";
-
-  private static final String CONSISTENT = "consistent";
-
-  private enum InteractionType {
-    READ,
-    READ_AFTER_DELETE,
-    EVENTUALLY_CONSISTENT_READ,
-    COPY,
-    EVENTUALLY_CONSISTENT_COPY,
-    EVENTUALLY_CONSISTENT_METADATA,
-    SELECT,
-    EVENTUALLY_CONSISTENT_SELECT
-  }
-
-  private final String changeDetectionSource;
-  private final String changeDetectionMode;
-  private final boolean authMode;
-  private final Collection<InteractionType> expectedExceptionInteractions;
-  private S3AFileSystem fs;
-
-  /**
-   * Test parameters.
-   * <ol>
-   *   <li>Change detection source: etag or version.</li>
-   *   <li>Change detection policy: server, client, client+warn, none</li>
-   *   <li>Whether to enable auth mode on the filesystem.</li>
-   *   <li>Expected outcomes.</li>
-   * </ol>
-   * @return the test configuration.
-   */
-  @Parameterized.Parameters(name = "{0}-{1}-auth-{2}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        // make sure it works with invalid config
-        {"bogus", "bogus",
-            true,
-            Arrays.asList(
-                InteractionType.READ,
-                InteractionType.READ_AFTER_DELETE,
-                InteractionType.EVENTUALLY_CONSISTENT_READ,
-                InteractionType.COPY,
-                InteractionType.EVENTUALLY_CONSISTENT_COPY,
-                InteractionType.EVENTUALLY_CONSISTENT_METADATA,
-                InteractionType.SELECT,
-                InteractionType.EVENTUALLY_CONSISTENT_SELECT)},
-
-        // test with etag
-        {CHANGE_DETECT_SOURCE_ETAG, CHANGE_DETECT_MODE_SERVER,
-            true,
-            Arrays.asList(
-                InteractionType.READ,
-                InteractionType.READ_AFTER_DELETE,
-                InteractionType.EVENTUALLY_CONSISTENT_READ,
-                InteractionType.COPY,
-                InteractionType.EVENTUALLY_CONSISTENT_COPY,
-                InteractionType.EVENTUALLY_CONSISTENT_METADATA,
-                InteractionType.SELECT,
-                InteractionType.EVENTUALLY_CONSISTENT_SELECT)},
-        {CHANGE_DETECT_SOURCE_ETAG, CHANGE_DETECT_MODE_CLIENT,
-            false,
-            Arrays.asList(
-                InteractionType.READ,
-                InteractionType.EVENTUALLY_CONSISTENT_READ,
-                InteractionType.READ_AFTER_DELETE,
-                InteractionType.COPY,
-                // not InteractionType.EVENTUALLY_CONSISTENT_COPY as copy change
-                // detection can't really occur client-side.  The eTag of
-                // the new object can't be expected to match.
-                InteractionType.EVENTUALLY_CONSISTENT_METADATA,
-                InteractionType.SELECT,
-                InteractionType.EVENTUALLY_CONSISTENT_SELECT)},
-        {CHANGE_DETECT_SOURCE_ETAG, CHANGE_DETECT_MODE_WARN,
-            false,
-            Arrays.asList(
-                InteractionType.READ_AFTER_DELETE)},
-        {CHANGE_DETECT_SOURCE_ETAG, CHANGE_DETECT_MODE_NONE,
-            false,
-            Arrays.asList(
-                InteractionType.READ_AFTER_DELETE)},
-
-        // test with versionId
-        // when using server-side versionId, the exceptions
-        // shouldn't happen since the previous version will still be available
-        {CHANGE_DETECT_SOURCE_VERSION_ID, CHANGE_DETECT_MODE_SERVER,
-            true,
-            Arrays.asList(
-                InteractionType.EVENTUALLY_CONSISTENT_READ,
-                InteractionType.EVENTUALLY_CONSISTENT_COPY,
-                InteractionType.EVENTUALLY_CONSISTENT_METADATA,
-                InteractionType.EVENTUALLY_CONSISTENT_SELECT)},
-
-        // with client-side versionId it will behave similar to client-side eTag
-        {CHANGE_DETECT_SOURCE_VERSION_ID, CHANGE_DETECT_MODE_CLIENT,
-            false,
-            Arrays.asList(
-                InteractionType.READ,
-                InteractionType.READ_AFTER_DELETE,
-                InteractionType.EVENTUALLY_CONSISTENT_READ,
-                InteractionType.COPY,
-                // not InteractionType.EVENTUALLY_CONSISTENT_COPY as copy change
-                // detection can't really occur client-side.  The versionId of
-                // the new object can't be expected to match.
-                InteractionType.EVENTUALLY_CONSISTENT_METADATA,
-                InteractionType.SELECT,
-                InteractionType.EVENTUALLY_CONSISTENT_SELECT)},
-
-        {CHANGE_DETECT_SOURCE_VERSION_ID, CHANGE_DETECT_MODE_WARN,
-            true,
-            Arrays.asList(
-                InteractionType.READ_AFTER_DELETE)},
-        {CHANGE_DETECT_SOURCE_VERSION_ID, CHANGE_DETECT_MODE_NONE,
-            false,
-            Arrays.asList(
-                InteractionType.READ_AFTER_DELETE)}
-    });
-  }
-
-  public ITestS3ARemoteFileChanged(String changeDetectionSource,
-      String changeDetectionMode,
-      boolean authMode,
-      Collection<InteractionType> expectedExceptionInteractions) {
-    this.changeDetectionSource = changeDetectionSource;
-    this.changeDetectionMode = changeDetectionMode;
-    this.authMode = authMode;
-    this.expectedExceptionInteractions = expectedExceptionInteractions;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    // skip all versioned checks if the remote FS doesn't do
-    // versions.
-    fs = getFileSystem();
-    skipIfVersionPolicyAndNoVersionId();
-    // cache the original S3 client for teardown.
-    originalS3Client = Optional.of(
-        fs.getAmazonS3ClientForTesting("caching"));
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    // restore the s3 client so there's no mocking interfering with the teardown
-    if (fs != null) {
-      originalS3Client.ifPresent(fs::setAmazonS3Client);
-    }
-    super.teardown();
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        CHANGE_DETECT_SOURCE,
-        CHANGE_DETECT_MODE,
-        RETRY_LIMIT,
-        RETRY_INTERVAL,
-        S3GUARD_CONSISTENCY_RETRY_LIMIT,
-        S3GUARD_CONSISTENCY_RETRY_INTERVAL,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    conf.set(CHANGE_DETECT_SOURCE, changeDetectionSource);
-    conf.set(CHANGE_DETECT_MODE, changeDetectionMode);
-    conf.setBoolean(METADATASTORE_AUTHORITATIVE, authMode);
-    conf.set(AUTHORITATIVE_PATH, "");
-
-    // reduce retry limit so FileNotFoundException cases timeout faster,
-    // speeding up the tests
-    conf.setInt(RETRY_LIMIT, TEST_MAX_RETRIES);
-    conf.set(RETRY_INTERVAL, TEST_RETRY_INTERVAL);
-    conf.setInt(S3GUARD_CONSISTENCY_RETRY_LIMIT, TEST_MAX_RETRIES);
-    conf.set(S3GUARD_CONSISTENCY_RETRY_INTERVAL, TEST_RETRY_INTERVAL);
-
-    if (conf.getClass(S3_METADATA_STORE_IMPL, MetadataStore.class) ==
-        NullMetadataStore.class) {
-      LOG.debug("Enabling local S3Guard metadata store");
-      // favor LocalMetadataStore over NullMetadataStore
-      conf.setClass(S3_METADATA_STORE_IMPL,
-          LocalMetadataStore.class, MetadataStore.class);
-    }
-    S3ATestUtils.disableFilesystemCaching(conf);
-    return conf;
-  }
-
-  /**
-   * Get the path of this method, including parameterized values.
-   * @return a path unique to this method and parameters
-   * @throws IOException failure.
-   */
-  protected Path path() throws IOException {
-    return super.path(getMethodName());
-  }
-
-  /**
-   * How many HEAD requests are made in a call to
-   * {@link S3AFileSystem#getFileStatus(Path)}?
-   * @return a number >= 0.
-   */
-  private int getFileStatusHeadCount() {
-    return authMode ? 0 : 1;
-  }
-
-  /**
-   * Tests reading a file that is changed while the reader's InputStream is
-   * open.
-   */
-  @Test
-  public void testReadFileChangedStreamOpen() throws Throwable {
-    describe("Tests reading a file that is changed while the reader's "
-        + "InputStream is open.");
-    final int originalLength = 8192;
-    final byte[] originalDataset = dataset(originalLength, 'a', 32);
-    final int newLength = originalLength + 1;
-    final byte[] newDataset = dataset(newLength, 'A', 32);
-    final Path testpath = path("readFileToChange.txt");
-    // initial write
-    writeDataset(fs, testpath, originalDataset, originalDataset.length,
-        1024, false);
-
-    try(FSDataInputStream instream = fs.open(testpath)) {
-      // seek forward and read successfully
-      instream.seek(1024);
-      assertTrue("no data to read", instream.read() >= 0);
-
-      // overwrite
-      writeDataset(fs, testpath, newDataset, newDataset.length, 1024, true);
-      // here the new file length is larger. Probe the file to see if this is
-      // true, with a spin and wait
-      eventually(30 * 1000, 1000,
-          () -> {
-            assertEquals(newLength, fs.getFileStatus(testpath).getLen());
-          });
-
-      // With the new file version in place, any subsequent S3 read by
-      // eTag/versionId will fail.  A new read by eTag/versionId will occur in
-      // reopen() on read after a seek() backwards.  We verify seek backwards
-      // results in the expected exception and seek() forward works without
-      // issue.
-
-      // first check seek forward
-      instream.seek(2048);
-      assertTrue("no data to read", instream.read() >= 0);
-
-      // now check seek backward
-      instream.seek(instream.getPos() - 100);
-
-      if (expectedExceptionInteractions.contains(InteractionType.READ)) {
-        expectReadFailure(instream);
-      } else {
-        instream.read();
-      }
-
-      byte[] buf = new byte[256];
-
-      // seek backward
-      instream.seek(0);
-
-      if (expectedExceptionInteractions.contains(InteractionType.READ)) {
-        expectReadFailure(instream);
-        intercept(RemoteFileChangedException.class, "", "read",
-            () -> instream.read(0, buf, 0, buf.length));
-        intercept(RemoteFileChangedException.class,  "", "readfully",
-            () -> instream.readFully(0, buf));
-      } else {
-        instream.read(buf);
-        instream.read(0, buf, 0, buf.length);
-        instream.readFully(0, buf);
-      }
-
-      // delete the file. Reads must fail
-      fs.delete(testpath, false);
-
-      // seek backward
-      instream.seek(0);
-
-      if (expectedExceptionInteractions.contains(
-          InteractionType.READ_AFTER_DELETE)) {
-        intercept(FileNotFoundException.class, "", "read()",
-            () -> instream.read());
-        intercept(FileNotFoundException.class, "", "readfully",
-            () -> instream.readFully(2048, buf));
-      } else {
-        instream.read();
-        instream.readFully(2048, buf);
-      }
-    }
-  }
-
-  /**
-   * Tests reading a file where the version visible in S3 does not match the
-   * version tracked in the metadata store.
-   */
-  @Test
-  public void testReadFileChangedOutOfSyncMetadata() throws Throwable {
-    final Path testpath = writeOutOfSyncFileVersion("fileChangedOutOfSync.dat");
-
-    try (FSDataInputStream instream = fs.open(testpath)) {
-      if (expectedExceptionInteractions.contains(InteractionType.READ)) {
-        expectReadFailure(instream);
-      } else {
-        instream.read();
-      }
-    }
-  }
-
-  /**
-   * Verifies that when the openFile builder is passed in a status,
-   * then that is used to eliminate the getFileStatus call in open();
-   * thus the version and etag passed down are still used.
-   */
-  @Test
-  public void testOpenFileWithStatus() throws Throwable {
-    final Path testpath = path("testOpenFileWithStatus.dat");
-    final byte[] dataset = TEST_DATA_BYTES;
-    S3AFileStatus originalStatus =
-        writeFile(testpath, dataset, dataset.length, true);
-
-    // forge a file status with a different etag
-    // no attempt is made to change the versionID as it will
-    // get rejected by S3 as an invalid version
-    S3AFileStatus forgedStatus =
-        S3AFileStatus.fromFileStatus(originalStatus, Tristate.FALSE,
-            originalStatus.getETag() + "-fake",
-            originalStatus.getVersionId() + "");
-    fs.getMetadataStore().put(
-        new PathMetadata(forgedStatus, Tristate.FALSE, false));
-
-    // verify the bad etag gets picked up.
-    LOG.info("Opening stream with s3guard's (invalid) status.");
-    try (FSDataInputStream instream = fs.openFile(testpath)
-        .build()
-        .get()) {
-      try {
-        instream.read();
-        // No exception only if we don't enforce change detection as exception
-        assertTrue(
-            "Read did not raise an exception even though the change detection "
-                + "mode was " + changeDetectionMode
-                + " and the inserted file status was invalid",
-            changeDetectionMode.equals(CHANGE_DETECT_MODE_NONE)
-                || changeDetectionMode.equals(CHANGE_DETECT_MODE_WARN)
-                || changeDetectionSource.equals(CHANGE_DETECT_SOURCE_VERSION_ID));
-      } catch (RemoteFileChangedException ignored) {
-        // Ignored.
-      }
-    }
-
-    // By passing in the status open() doesn't need to check s3guard
-    // And hence the existing file is opened
-    LOG.info("Opening stream with the original status.");
-    try (FSDataInputStream instream = fs.openFile(testpath)
-        .withFileStatus(originalStatus)
-        .build()
-        .get()) {
-      instream.read();
-    }
-
-    // and this holds for S3A Located Status
-    LOG.info("Opening stream with S3ALocatedFileStatus.");
-    try (FSDataInputStream instream = fs.openFile(testpath)
-        .withFileStatus(new S3ALocatedFileStatus(originalStatus, null))
-        .build()
-        .get()) {
-      instream.read();
-    }
-
-    // if you pass in a status of a dir, it will be rejected
-    S3AFileStatus s2 = new S3AFileStatus(true, testpath, "alice");
-    assertTrue("not a directory " + s2, s2.isDirectory());
-    LOG.info("Open with directory status");
-    interceptFuture(FileNotFoundException.class, "",
-        fs.openFile(testpath)
-            .withFileStatus(s2)
-            .build());
-
-    // now, we delete the file from the store and s3guard
-    // when we pass in the status, there's no HEAD request, so it's only
-    // in the read call where the 404 surfaces.
-    // and there, when versionID is passed to the GET, the data is returned
-    LOG.info("Testing opening a deleted file");
-    fs.delete(testpath, false);
-    try (FSDataInputStream instream = fs.openFile(testpath)
-        .withFileStatus(originalStatus)
-        .build()
-        .get()) {
-      if (changeDetectionSource.equals(CHANGE_DETECT_SOURCE_VERSION_ID)
-          && changeDetectionMode.equals(CHANGE_DETECT_MODE_SERVER)) {
-          // the deleted file is still there if you know the version ID
-          // and the check is server-side
-          instream.read();
-      } else {
-        // all other cases, the read will return 404.
-        intercept(FileNotFoundException.class,
-            () -> instream.read());
-      }
-
-    }
-
-    // whereas without that status, you fail in the get() when a HEAD is
-    // issued
-    interceptFuture(FileNotFoundException.class, "",
-        fs.openFile(testpath).build());
-
-  }
-
-  /**
-   * Ensures a file can be read when there is no version metadata
-   * (ETag, versionId).
-   */
-  @Test
-  public void testReadWithNoVersionMetadata() throws Throwable {
-    final Path testpath = writeFileWithNoVersionMetadata("readnoversion.dat");
-
-    assertEquals("Contents of " + testpath,
-        TEST_DATA,
-        readUTF8(fs, testpath, -1));
-  }
-
-  /**
-   * Tests using S3 Select on a file where the version visible in S3 does not
-   * match the version tracked in the metadata store.
-   */
-  @Test
-  public void testSelectChangedFile() throws Throwable {
-    requireS3Select();
-    final Path testpath = writeOutOfSyncFileVersion("select.dat");
-
-    if (expectedExceptionInteractions.contains(InteractionType.SELECT)) {
-      interceptFuture(RemoteFileChangedException.class, "select",
-          fs.openFile(testpath)
-              .must(SELECT_SQL, "SELECT * FROM S3OBJECT").build());
-    } else {
-      fs.openFile(testpath)
-          .must(SELECT_SQL, "SELECT * FROM S3OBJECT")
-          .build()
-          .get()
-          .close();
-    }
-  }
-
-  /**
-   * Tests using S3 Select on a file where the version visible in S3 does not
-   * initially match the version tracked in the metadata store, but eventually
-   * (after retries) does.
-   */
-  @Test
-  public void testSelectEventuallyConsistentFile() throws Throwable {
-    describe("Eventually Consistent S3 Select");
-    requireS3Guard();
-    requireS3Select();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-
-    final Path testpath1 = writeEventuallyConsistentFileVersion(
-        "select1.dat", s3ClientSpy, 0, TEST_MAX_RETRIES, 0);
-
-    // should succeed since the inconsistency doesn't last longer than the
-    // configured retry limit
-    fs.openFile(testpath1)
-        .must(SELECT_SQL, "SELECT * FROM S3OBJECT")
-        .build()
-        .get()
-        .close();
-
-    // select() makes a getFileStatus() call before the consistency checking
-    // that will match the stub. As such, we need an extra inconsistency here
-    // to cross the threshold
-    int getMetadataInconsistencyCount = TEST_MAX_RETRIES + 2;
-    final Path testpath2 = writeEventuallyConsistentFileVersion(
-        "select2.dat", s3ClientSpy, 0, getMetadataInconsistencyCount, 0);
-
-    if (expectedExceptionInteractions.contains(
-        InteractionType.EVENTUALLY_CONSISTENT_SELECT)) {
-      // should fail since the inconsistency lasts longer than the configured
-      // retry limit
-      interceptFuture(RemoteFileChangedException.class, "select",
-          fs.openFile(testpath2)
-              .must(SELECT_SQL, "SELECT * FROM S3OBJECT").build());
-    } else {
-      fs.openFile(testpath2)
-          .must(SELECT_SQL, "SELECT * FROM S3OBJECT")
-          .build()
-          .get()
-          .close();
-    }
-  }
-
-  /**
-   * Ensures a file can be read via S3 Select when there is no version metadata
-   * (ETag, versionId).
-   */
-  @Test
-  public void testSelectWithNoVersionMetadata() throws Throwable {
-    requireS3Select();
-    final Path testpath =
-        writeFileWithNoVersionMetadata("selectnoversion.dat");
-
-    try (FSDataInputStream instream = fs.openFile(testpath)
-        .must(SELECT_SQL, "SELECT * FROM S3OBJECT")
-        .build()
-        .get()) {
-      assertEquals(QUOTED_TEST_DATA,
-          IOUtils.toString(instream, StandardCharsets.UTF_8).trim());
-    }
-  }
-
-  /**
-   * Tests doing a rename() on a file where the version visible in S3 does not
-   * match the version tracked in the metadata store.
-   * @throws Throwable failure
-   */
-  @Test
-  public void testRenameChangedFile() throws Throwable {
-    final Path testpath = writeOutOfSyncFileVersion("rename.dat");
-
-    final Path dest = path("dest.dat");
-    if (expectedExceptionInteractions.contains(InteractionType.COPY)) {
-      intercept(RemoteFileChangedException.class, "",
-          "expected copy() failure",
-          () -> fs.rename(testpath, dest));
-    } else {
-      fs.rename(testpath, dest);
-    }
-  }
-
-  /**
-   * Inconsistent response counts for getObjectMetadata() and
-   * copyObject() for a rename.
-   * @param metadataCallsExpectedBeforeRetryLoop number of getObjectMetadata
-   * calls expected before the consistency checking retry loop
-   * @return the inconsistencies for (metadata, copy)
-   */
-  private Pair<Integer, Integer> renameInconsistencyCounts(
-      int metadataCallsExpectedBeforeRetryLoop) {
-    int metadataInconsistencyCount = TEST_MAX_RETRIES
-        + metadataCallsExpectedBeforeRetryLoop;
-    int copyInconsistencyCount =
-        versionCheckingIsOnServer() ? TEST_MAX_RETRIES : 0;
-
-    return Pair.of(metadataInconsistencyCount, copyInconsistencyCount);
-  }
-
-  /**
-   * Tests doing a rename() on a file where the version visible in S3 does not
-   * match the version in the metadata store until a certain number of retries
-   * has been met.
-   */
-  @Test
-  public void testRenameEventuallyConsistentFile() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-
-    // Total inconsistent response count across getObjectMetadata() and
-    // copyObject().
-    // The split of inconsistent responses between getObjectMetadata() and
-    // copyObject() is arbitrary.
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(
-        getFileStatusHeadCount());
-    int metadataInconsistencyCount = counts.getLeft();
-    int copyInconsistencyCount = counts.getRight();
-    final Path testpath1 =
-        writeEventuallyConsistentFileVersion("rename-eventually1.dat",
-            s3ClientSpy,
-            0,
-            metadataInconsistencyCount,
-            copyInconsistencyCount);
-
-    final Path dest1 = path("dest1.dat");
-    // shouldn't fail since the inconsistency doesn't last through the
-    // configured retry limit
-    fs.rename(testpath1, dest1);
-  }
-
-  /**
-   * Tests doing a rename() on a file where the version visible in S3 does not
-   * match the version in the metadata store until a certain number of retries
-   * has been met.
-   * The test expects failure by AWSClientIOException caused by NPE due to
-   * https://github.com/aws/aws-sdk-java/issues/1644
-   */
-  @Test
-  public void testRenameEventuallyConsistentFileNPE() throws Throwable {
-    requireS3Guard();
-    skipIfVersionPolicyAndNoVersionId();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(
-        getFileStatusHeadCount());
-    int metadataInconsistencyCount = counts.getLeft();
-    int copyInconsistencyCount = counts.getRight();
-    // giving copyInconsistencyCount + 1 here should trigger the failure,
-    // exceeding the retry limit
-    final Path testpath2 =
-        writeEventuallyConsistentFileVersion("rename-eventuallyNPE.dat",
-            s3ClientSpy,
-            0,
-            metadataInconsistencyCount,
-            copyInconsistencyCount + 1);
-    final Path dest2 = path("destNPE.dat");
-    if (expectedExceptionInteractions.contains(
-        InteractionType.EVENTUALLY_CONSISTENT_COPY)) {
-      // should fail since the inconsistency is set up to persist longer than
-      // the configured retry limit
-      // the expected exception is not RemoteFileChangedException due to
-      // https://github.com/aws/aws-sdk-java/issues/1644
-      // If this test is failing after an AWS SDK update,
-      // then it means the SDK bug is fixed.
-      // Please update this test to match the new behavior.
-      AWSClientIOException exception =
-          intercept(AWSClientIOException.class,
-              "Unable to complete transfer: null",
-              "expected copy() failure",
-              () -> fs.rename(testpath2, dest2));
-      AmazonClientException cause = exception.getCause();
-      if (cause == null) {
-        // no cause; something else went wrong: throw.
-        throw new AssertionError("No inner cause",
-            exception);
-      }
-      Throwable causeCause = cause.getCause();
-      if (!(causeCause instanceof NullPointerException)) {
-        // null causeCause or it is the wrong type: throw
-        throw new AssertionError("Innermost cause is not NPE",
-            exception);
-      }
-    } else {
-      fs.rename(testpath2, dest2);
-    }
-  }
-
-  /**
-   * Tests doing a rename() on a file where the version visible in S3 does not
-   * match the version in the metadata store until a certain number of retries
-   * has been met.
-   * The test expects failure by RemoteFileChangedException.
-   */
-  @Test
-  public void testRenameEventuallyConsistentFileRFCE() throws Throwable {
-    requireS3Guard();
-    skipIfVersionPolicyAndNoVersionId();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(
-        getFileStatusHeadCount());
-    int metadataInconsistencyCount = counts.getLeft();
-    int copyInconsistencyCount = counts.getRight();
-    // giving metadataInconsistencyCount + 1 here should trigger the failure,
-    // exceeding the retry limit
-    final Path testpath2 =
-        writeEventuallyConsistentFileVersion("rename-eventuallyRFCE.dat",
-            s3ClientSpy,
-            0,
-            metadataInconsistencyCount + 1,
-            copyInconsistencyCount);
-    final Path dest2 = path("destRFCE.dat");
-    if (expectedExceptionInteractions.contains(
-        InteractionType.EVENTUALLY_CONSISTENT_METADATA)) {
-      // should fail since the inconsistency is set up to persist longer than
-      // the configured retry limit
-      intercept(RemoteFileChangedException.class,
-          CHANGE_DETECTED,
-          "expected copy() failure",
-          () -> fs.rename(testpath2, dest2));
-    } else {
-      fs.rename(testpath2, dest2);
-    }
-  }
-
-  /**
-   * Tests doing a rename() on a directory containing
-   * an file which is eventually consistent.
-   * There is no call to getFileStatus on the source file whose
-   * inconsistency is simulated; the state of S3Guard auth mode is not
-   * relevant.
-   */
-  @Test
-  public void testRenameEventuallyConsistentDirectory() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    Path basedir = path();
-    Path sourcedir = new Path(basedir, "sourcedir");
-    fs.mkdirs(sourcedir);
-    Path destdir = new Path(basedir, "destdir");
-    Path inconsistentFile = new Path(sourcedir, INCONSISTENT);
-    Path consistentFile = new Path(sourcedir, CONSISTENT);
-
-    // write the consistent data
-    writeDataset(fs, consistentFile, TEST_DATA_BYTES, TEST_DATA_BYTES.length,
-        1024, true, true);
-
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(0);
-    int metadataInconsistencyCount = counts.getLeft();
-    int copyInconsistencyCount = counts.getRight();
-
-    writeEventuallyConsistentData(
-        s3ClientSpy,
-        inconsistentFile,
-        TEST_DATA_BYTES,
-        0,
-        metadataInconsistencyCount,
-        copyInconsistencyCount);
-
-    // must not fail since the inconsistency doesn't last through the
-    // configured retry limit
-    fs.rename(sourcedir, destdir);
-  }
-
-  /**
-   * Tests doing a rename() on a file which is eventually visible.
-   */
-  @Test
-  public void testRenameEventuallyVisibleFile() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    Path basedir = path();
-    Path sourcedir = new Path(basedir, "sourcedir");
-    fs.mkdirs(sourcedir);
-    Path destdir = new Path(basedir, "destdir");
-    Path inconsistentFile = new Path(sourcedir, INCONSISTENT);
-    Path consistentFile = new Path(sourcedir, CONSISTENT);
-
-    // write the consistent data
-    writeDataset(fs, consistentFile, TEST_DATA_BYTES, TEST_DATA_BYTES.length,
-        1024, true, true);
-
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(0);
-    int metadataInconsistencyCount = counts.getLeft();
-
-    writeDataset(fs, inconsistentFile, TEST_DATA_BYTES, TEST_DATA_BYTES.length,
-        1024, true, true);
-
-    stubTemporaryNotFound(s3ClientSpy, metadataInconsistencyCount,
-        inconsistentFile);
-
-    // must not fail since the inconsistency doesn't last through the
-    // configured retry limit
-    fs.rename(sourcedir, destdir);
-  }
-
-  /**
-   * Tests doing a rename() on a file which never quite appears will
-   * fail with a RemoteFileChangedException rather than have the exception
-   * downgraded to a failure.
-   */
-  @Test
-  public void testRenameMissingFile()
-      throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    Path basedir = path();
-    Path sourcedir = new Path(basedir, "sourcedir");
-    fs.mkdirs(sourcedir);
-    Path destdir = new Path(basedir, "destdir");
-    Path inconsistentFile = new Path(sourcedir, INCONSISTENT);
-    Path consistentFile = new Path(sourcedir, CONSISTENT);
-
-    // write the consistent data
-    writeDataset(fs, consistentFile, TEST_DATA_BYTES, TEST_DATA_BYTES.length,
-        1024, true, true);
-
-    Pair<Integer, Integer> counts = renameInconsistencyCounts(0);
-    int metadataInconsistencyCount = counts.getLeft();
-
-    writeDataset(fs, inconsistentFile, TEST_DATA_BYTES, TEST_DATA_BYTES.length,
-        1024, true, true);
-
-    stubTemporaryNotFound(s3ClientSpy, metadataInconsistencyCount + 1,
-        inconsistentFile);
-
-    String expected = fs.hasMetadataStore()
-        ? RemoteFileChangedException.FILE_NEVER_FOUND
-        : RemoteFileChangedException.FILE_NOT_FOUND_SINGLE_ATTEMPT;
-    RemoteFileChangedException ex = intercept(
-        RemoteFileChangedException.class,
-        expected,
-        () -> fs.rename(sourcedir, destdir));
-    assertEquals("Path in " + ex,
-        inconsistentFile, ex.getPath());
-    if (!(ex.getCause() instanceof FileNotFoundException)) {
-      throw ex;
-    }
-  }
-
-  /**
-   * Ensures a file can be renamed when there is no version metadata
-   * (ETag, versionId).
-   */
-  @Test
-  public void testRenameWithNoVersionMetadata() throws Throwable {
-    final Path testpath =
-        writeFileWithNoVersionMetadata("renamenoversion.dat");
-
-    final Path dest = path("noversiondest.dat");
-    fs.rename(testpath, dest);
-    assertEquals("Contents of " + dest,
-        TEST_DATA,
-        readUTF8(fs, dest, -1));
-  }
-
-  /**
-   * Ensures S3Guard and retries allow an eventually consistent read.
-   */
-  @Test
-  public void testReadAfterEventuallyConsistentWrite() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    final Path testpath1 =
-        writeEventuallyConsistentFileVersion("eventually1.dat",
-            s3ClientSpy, TEST_MAX_RETRIES, 0 , 0);
-
-    try (FSDataInputStream instream1 = fs.open(testpath1)) {
-      // succeeds on the last retry
-      instream1.read();
-    }
-  }
-
-  /**
-   * Ensures S3Guard and retries allow an eventually consistent read.
-   */
-  @Test
-  public void testReadAfterEventuallyConsistentWrite2() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    final Path testpath2 =
-        writeEventuallyConsistentFileVersion("eventually2.dat",
-            s3ClientSpy, TEST_MAX_RETRIES + 1, 0, 0);
-
-    try (FSDataInputStream instream2 = fs.open(testpath2)) {
-      if (expectedExceptionInteractions.contains(
-          InteractionType.EVENTUALLY_CONSISTENT_READ)) {
-        // keeps retrying and eventually gives up with RemoteFileChangedException
-        expectReadFailure(instream2);
-      } else {
-        instream2.read();
-      }
-    }
-  }
-
-  /**
-   * Ensures read on re-open (after seek backwards) when S3 does not return the
-   * version of the file tracked in the metadata store fails immediately.  No
-   * retries should happen since a retry is not expected to recover.
-   */
-  @Test
-  public void testEventuallyConsistentReadOnReopen() throws Throwable {
-    requireS3Guard();
-    AmazonS3 s3ClientSpy = spyOnFilesystem();
-    String filename = "eventually-reopen.dat";
-    final Path testpath =
-        writeEventuallyConsistentFileVersion(filename,
-            s3ClientSpy, 0, 0, 0);
-
-    try (FSDataInputStream instream = fs.open(testpath)) {
-      instream.read();
-      // overwrite the file, returning inconsistent version for
-      // (effectively) infinite retries
-      writeEventuallyConsistentFileVersion(filename, s3ClientSpy,
-          Integer.MAX_VALUE, 0, 0);
-      instream.seek(0);
-      if (expectedExceptionInteractions.contains(InteractionType.READ)) {
-        // if it retries at all, it will retry forever, which should fail
-        // the test.  The expected behavior is immediate
-        // RemoteFileChangedException.
-        expectReadFailure(instream);
-      } else {
-        instream.read();
-      }
-    }
-  }
-
-  /**
-   * Writes a file with old ETag and versionId in the metadata store such
-   * that the metadata is out of sync with S3.  Attempts to read such a file
-   * should result in {@link RemoteFileChangedException}.
-   */
-  private Path writeOutOfSyncFileVersion(String filename) throws IOException {
-    final Path testpath = path(filename);
-    final byte[] dataset = TEST_DATA_BYTES;
-    S3AFileStatus originalStatus =
-        writeFile(testpath, dataset, dataset.length, false);
-
-    // overwrite with half the content
-    S3AFileStatus newStatus = writeFile(testpath, dataset, dataset.length / 2,
-        true);
-
-    // put back the original etag, versionId
-    S3AFileStatus forgedStatus =
-        S3AFileStatus.fromFileStatus(newStatus, Tristate.FALSE,
-            originalStatus.getETag(), originalStatus.getVersionId());
-    fs.getMetadataStore().put(
-        new PathMetadata(forgedStatus, Tristate.FALSE, false));
-
-    return testpath;
-  }
-
-  /**
-   * Write data to a file; return the status from the filesystem.
-   * @param path file path
-   * @param dataset dataset to write from
-   * @param length number of bytes from the dataset to write.
-   * @param overwrite overwrite flag
-   * @return the retrieved file status.
-   */
-  private S3AFileStatus writeFile(final Path path,
-      final byte[] dataset,
-      final int length,
-      final boolean overwrite) throws IOException {
-    writeDataset(fs, path, dataset, length,
-        1024, overwrite);
-    return (S3AFileStatus) fs.getFileStatus(path);
-  }
-
-  /**
-   * Writes {@link #TEST_DATA} to a file where the file will be inconsistent
-   * in S3 for a set of operations.
-   * The duration of the inconsistency is controlled by the
-   * getObjectInconsistencyCount, getMetadataInconsistencyCount, and
-   * copyInconsistentCallCount parameters.
-   * The inconsistency manifests in AmazonS3#getObject,
-   * AmazonS3#getObjectMetadata, and AmazonS3#copyObject.
-   * This method sets up the provided s3ClientSpy to return a response to each
-   * of these methods indicating an inconsistency where the requested object
-   * version (eTag or versionId) is not available until a certain retry
-   * threshold is met.
-   * Providing inconsistent call count values above or
-   * below the overall retry limit allows a test to simulate a condition that
-   * either should or should not result in an overall failure from retry
-   * exhaustion.
-   * @param filename name of file (will be under test path)
-   * @param s3ClientSpy s3 client to patch
-   * @param getObjectInconsistencyCount number of GET inconsistencies
-   * @param getMetadataInconsistencyCount number of HEAD inconsistencies
-   * @param copyInconsistencyCount number of COPY inconsistencies.
-   * @return the path written
-   * @throws IOException failure to write the test data.
-   */
-  private Path writeEventuallyConsistentFileVersion(String filename,
-      AmazonS3 s3ClientSpy,
-      int getObjectInconsistencyCount,
-      int getMetadataInconsistencyCount,
-      int copyInconsistencyCount)
-      throws IOException {
-    return writeEventuallyConsistentData(s3ClientSpy,
-        path(filename),
-        TEST_DATA_BYTES,
-        getObjectInconsistencyCount,
-        getMetadataInconsistencyCount,
-        copyInconsistencyCount);
-  }
-
-  /**
-   * Writes data to a path and configures the S3 client for inconsistent
-   * HEAD, GET or COPY operations.
-   * @param testpath absolute path of file
-   * @param s3ClientSpy s3 client to patch
-   * @param dataset bytes to write.
-   * @param getObjectInconsistencyCount number of GET inconsistencies
-   * @param getMetadataInconsistencyCount number of HEAD inconsistencies
-   * @param copyInconsistencyCount number of COPY inconsistencies.
-   * @return the path written
-   * @throws IOException failure to write the test data.
-   */
-  private Path writeEventuallyConsistentData(final AmazonS3 s3ClientSpy,
-      final Path testpath,
-      final byte[] dataset,
-      final int getObjectInconsistencyCount,
-      final int getMetadataInconsistencyCount,
-      final int copyInconsistencyCount)
-      throws IOException {
-    writeDataset(fs, testpath, dataset, dataset.length,
-        1024, true);
-    S3AFileStatus originalStatus = (S3AFileStatus) fs.getFileStatus(testpath);
-
-    // overwrite with half the content
-    writeDataset(fs, testpath, dataset, dataset.length / 2,
-        1024, true);
-
-    LOG.debug("Original file info: {}: version={}, etag={}", testpath,
-        originalStatus.getVersionId(), originalStatus.getETag());
-
-    S3AFileStatus newStatus = (S3AFileStatus) fs.getFileStatus(testpath);
-    LOG.debug("Updated file info: {}: version={}, etag={}", testpath,
-        newStatus.getVersionId(), newStatus.getETag());
-
-    LOG.debug("File {} will be inconsistent for {} HEAD and {} GET requests",
-        testpath, getMetadataInconsistencyCount, getObjectInconsistencyCount);
-
-    stubTemporaryUnavailable(s3ClientSpy, getObjectInconsistencyCount,
-        testpath, newStatus);
-
-    stubTemporaryWrongVersion(s3ClientSpy, getObjectInconsistencyCount,
-        testpath, originalStatus);
-
-    if (versionCheckingIsOnServer()) {
-      // only stub inconsistency when mode is server since no constraints that
-      // should trigger inconsistency are passed in any other mode
-      LOG.debug("File {} will be inconsistent for {} COPY operations",
-          testpath, copyInconsistencyCount);
-      stubTemporaryCopyInconsistency(s3ClientSpy, testpath, newStatus,
-          copyInconsistencyCount);
-    }
-
-    stubTemporaryMetadataInconsistency(s3ClientSpy, testpath, originalStatus,
-        newStatus, getMetadataInconsistencyCount);
-
-    return testpath;
-  }
-
-  /**
-   * Log the call hierarchy at debug level, helps track down
-   * where calls to operations are coming from.
-   */
-  private void logLocationAtDebug() {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Call hierarchy", new Exception("here"));
-    }
-  }
-
-  /**
-   * Stubs {@link AmazonS3#getObject(GetObjectRequest)}
-   * within s3ClientSpy to return null until inconsistentCallCount calls have
-   * been made.  The null response simulates what occurs when an object
-   * matching the specified ETag or versionId is not available.
-   * @param s3ClientSpy the spy to stub
-   * @param inconsistentCallCount the number of calls that should return the
-   * null response
-   * @param testpath the path of the object the stub should apply to
-   */
-  private void stubTemporaryUnavailable(AmazonS3 s3ClientSpy,
-      int inconsistentCallCount, Path testpath,
-      S3AFileStatus newStatus) {
-    Answer<S3Object> temporarilyUnavailableAnswer = new Answer<S3Object>() {
-      private int callCount = 0;
-
-      @Override
-      public S3Object answer(InvocationOnMock invocation) throws Throwable {
-        // simulates ETag or versionId constraint not met until
-        // inconsistentCallCount surpassed
-        callCount++;
-        if (callCount <= inconsistentCallCount) {
-          LOG.info("Temporarily unavailable {} count {} of {}",
-              testpath, callCount, inconsistentCallCount);
-          logLocationAtDebug();
-          return null;
-        }
-        return (S3Object) invocation.callRealMethod();
-      }
-    };
-
-    // match the requests that would be made in either server-side change
-    // detection mode
-    doAnswer(temporarilyUnavailableAnswer).when(s3ClientSpy)
-        .getObject(
-            matchingGetObjectRequest(
-                testpath, newStatus.getETag(), null));
-    doAnswer(temporarilyUnavailableAnswer).when(s3ClientSpy)
-        .getObject(
-            matchingGetObjectRequest(
-                testpath, null, newStatus.getVersionId()));
-  }
-
-  /**
-   * Stubs {@link AmazonS3#getObject(GetObjectRequest)}
-   * within s3ClientSpy to return an object modified to contain metadata
-   * from originalStatus until inconsistentCallCount calls have been made.
-   * @param s3ClientSpy the spy to stub
-   * @param testpath the path of the object the stub should apply to
-   * @param inconsistentCallCount the number of calls that should return the
-   * null response
-   * @param originalStatus the status metadata to inject into the
-   * inconsistentCallCount responses
-   */
-  private void stubTemporaryWrongVersion(AmazonS3 s3ClientSpy,
-      int inconsistentCallCount, Path testpath,
-      S3AFileStatus originalStatus) {
-    Answer<S3Object> temporarilyWrongVersionAnswer = new Answer<S3Object>() {
-      private int callCount = 0;
-
-      @Override
-      public S3Object answer(InvocationOnMock invocation) throws Throwable {
-        // simulates old ETag or versionId until inconsistentCallCount surpassed
-        callCount++;
-        S3Object s3Object = (S3Object) invocation.callRealMethod();
-        if (callCount <= inconsistentCallCount) {
-          LOG.info("Temporary Wrong Version {} count {} of {}",
-              testpath, callCount, inconsistentCallCount);
-          logLocationAtDebug();
-          S3Object objectSpy = Mockito.spy(s3Object);
-          ObjectMetadata metadataSpy =
-              Mockito.spy(s3Object.getObjectMetadata());
-          when(objectSpy.getObjectMetadata()).thenReturn(metadataSpy);
-          when(metadataSpy.getETag()).thenReturn(originalStatus.getETag());
-          when(metadataSpy.getVersionId())
-              .thenReturn(originalStatus.getVersionId());
-          return objectSpy;
-        }
-        return s3Object;
-      }
-    };
-
-    // match requests that would be made in client-side change detection
-    doAnswer(temporarilyWrongVersionAnswer).when(s3ClientSpy).getObject(
-        matchingGetObjectRequest(testpath, null, null));
-  }
-
-  /**
-   * Stubs {@link AmazonS3#copyObject(CopyObjectRequest)}
-   * within s3ClientSpy to return null (indicating preconditions not met) until
-   * copyInconsistentCallCount calls have been made.
-   * @param s3ClientSpy the spy to stub
-   * @param testpath the path of the object the stub should apply to
-   * @param newStatus the status metadata containing the ETag and versionId
-   * that should be matched in order for the stub to apply
-   * @param copyInconsistentCallCount how many times to return the
-   * precondition failed error
-   */
-  private void stubTemporaryCopyInconsistency(AmazonS3 s3ClientSpy,
-      Path testpath, S3AFileStatus newStatus,
-      int copyInconsistentCallCount) {
-    Answer<CopyObjectResult> temporarilyPreconditionsNotMetAnswer =
-        new Answer<CopyObjectResult>() {
-      private int callCount = 0;
-
-      @Override
-      public CopyObjectResult answer(InvocationOnMock invocation)
-          throws Throwable {
-        callCount++;
-        if (callCount <= copyInconsistentCallCount) {
-          String message = "preconditions not met on call " + callCount
-              + " of " + copyInconsistentCallCount;
-          LOG.info("Copying {}: {}", testpath, message);
-          logLocationAtDebug();
-          return null;
-        }
-        return (CopyObjectResult) invocation.callRealMethod();
-      }
-    };
-
-    // match requests made during copy
-    doAnswer(temporarilyPreconditionsNotMetAnswer).when(s3ClientSpy).copyObject(
-        matchingCopyObjectRequest(testpath, newStatus.getETag(), null));
-    doAnswer(temporarilyPreconditionsNotMetAnswer).when(s3ClientSpy).copyObject(
-        matchingCopyObjectRequest(testpath, null, newStatus.getVersionId()));
-  }
-
-  /**
-   * Stubs {@link AmazonS3#getObjectMetadata(GetObjectMetadataRequest)}
-   * within s3ClientSpy to return metadata from originalStatus until
-   * metadataInconsistentCallCount calls have been made.
-   * @param s3ClientSpy the spy to stub
-   * @param testpath the path of the object the stub should apply to
-   * @param originalStatus the inconsistent status metadata to return
-   * @param newStatus the status metadata to return after
-   * metadataInconsistentCallCount is met
-   * @param metadataInconsistentCallCount how many times to return the
-   * inconsistent metadata
-   */
-  private void stubTemporaryMetadataInconsistency(AmazonS3 s3ClientSpy,
-      Path testpath, S3AFileStatus originalStatus,
-      S3AFileStatus newStatus, int metadataInconsistentCallCount) {
-    Answer<ObjectMetadata> temporarilyOldMetadataAnswer =
-        new Answer<ObjectMetadata>() {
-      private int callCount = 0;
-
-      @Override
-      public ObjectMetadata answer(InvocationOnMock invocation)
-          throws Throwable {
-        ObjectMetadata objectMetadata =
-            (ObjectMetadata) invocation.callRealMethod();
-        callCount++;
-        if (callCount <= metadataInconsistentCallCount) {
-          LOG.info("Inconsistent metadata {} count {} of {}",
-              testpath, callCount, metadataInconsistentCallCount);
-          logLocationAtDebug();
-          ObjectMetadata metadataSpy =
-              Mockito.spy(objectMetadata);
-          when(metadataSpy.getETag()).thenReturn(originalStatus.getETag());
-          when(metadataSpy.getVersionId())
-              .thenReturn(originalStatus.getVersionId());
-          return metadataSpy;
-        }
-        return objectMetadata;
-      }
-    };
-
-    // match requests made during select
-    doAnswer(temporarilyOldMetadataAnswer).when(s3ClientSpy).getObjectMetadata(
-        matchingMetadataRequest(testpath, null));
-    doAnswer(temporarilyOldMetadataAnswer).when(s3ClientSpy).getObjectMetadata(
-        matchingMetadataRequest(testpath, newStatus.getVersionId()));
-  }
-
-  /**
-   * Writes a file with null ETag and versionId in the metadata store.
-   */
-  private Path writeFileWithNoVersionMetadata(String filename)
-      throws IOException {
-    final Path testpath = path(filename);
-    S3AFileStatus originalStatus = writeFile(testpath, TEST_DATA_BYTES,
-        TEST_DATA_BYTES.length, false);
-
-    // remove ETag and versionId
-    S3AFileStatus newStatus = S3AFileStatus.fromFileStatus(originalStatus,
-        Tristate.FALSE, null, null);
-    fs.getMetadataStore().put(new PathMetadata(newStatus, Tristate.FALSE,
-        false));
-
-    return testpath;
-  }
-
-  /**
-   * The test is invalid if the policy uses versionId but the bucket doesn't
-   * have versioning enabled.
-   *
-   * Tests the given file for a versionId to detect whether bucket versioning
-   * is enabled.
-   */
-  private void skipIfVersionPolicyAndNoVersionId(Path testpath)
-      throws IOException {
-    if (fs.getChangeDetectionPolicy().getSource() == Source.VersionId) {
-      // skip versionId tests if the bucket doesn't have object versioning
-      // enabled
-      Assume.assumeTrue(
-          "Target filesystem does not support versioning",
-          fs.getObjectMetadata(testpath).getVersionId() != null);
-    }
-  }
-
-  /**
-   * Like {@link #skipIfVersionPolicyAndNoVersionId(Path)} but generates a new
-   * file to test versionId against.
-   */
-  private void skipIfVersionPolicyAndNoVersionId() throws IOException {
-    if (fs.getChangeDetectionPolicy().getSource() == Source.VersionId) {
-      Path versionIdFeatureTestFile = path("versionIdTest");
-      writeDataset(fs, versionIdFeatureTestFile, TEST_DATA_BYTES,
-          TEST_DATA_BYTES.length, 1024, true, true);
-      skipIfVersionPolicyAndNoVersionId(versionIdFeatureTestFile);
-    }
-  }
-
-  private GetObjectRequest matchingGetObjectRequest(Path path, String eTag,
-      String versionId) {
-    return ArgumentMatchers.argThat(request -> {
-      if (request.getBucketName().equals(fs.getBucket())
-          && request.getKey().equals(fs.pathToKey(path))) {
-        if (eTag == null && !request.getMatchingETagConstraints().isEmpty()) {
-          return false;
-        }
-        if (eTag != null &&
-            !request.getMatchingETagConstraints().contains(eTag)) {
-          return false;
-        }
-        if (versionId == null && request.getVersionId() != null) {
-          return false;
-        }
-        if (versionId != null && !versionId.equals(request.getVersionId())) {
-          return false;
-        }
-        return true;
-      }
-      return false;
-    });
-  }
-
-  private CopyObjectRequest matchingCopyObjectRequest(Path path, String eTag,
-      String versionId) {
-    return ArgumentMatchers.argThat(request -> {
-      if (request.getSourceBucketName().equals(fs.getBucket())
-          && request.getSourceKey().equals(fs.pathToKey(path))) {
-        if (eTag == null && !request.getMatchingETagConstraints().isEmpty()) {
-          return false;
-        }
-        if (eTag != null &&
-            !request.getMatchingETagConstraints().contains(eTag)) {
-          return false;
-        }
-        if (versionId == null && request.getSourceVersionId() != null) {
-          return false;
-        }
-        if (versionId != null &&
-            !versionId.equals(request.getSourceVersionId())) {
-          return false;
-        }
-        return true;
-      }
-      return false;
-    });
-  }
-
-  private GetObjectMetadataRequest matchingMetadataRequest(Path path,
-      String versionId) {
-    return ArgumentMatchers.argThat(request -> {
-      if (request.getBucketName().equals(fs.getBucket())
-          && request.getKey().equals(fs.pathToKey(path))) {
-        if (versionId == null && request.getVersionId() != null) {
-          return false;
-        }
-        if (versionId != null &&
-            !versionId.equals(request.getVersionId())) {
-          return false;
-        }
-        return true;
-      }
-      return false;
-    });
-  }
-
-  /**
-   * Match any getObjectMetadata request against a given path.
-   * @param path path to to match.
-   * @return the matching request.
-   */
-  private GetObjectMetadataRequest matchingMetadataRequest(Path path) {
-    return ArgumentMatchers.argThat(request -> {
-      return request.getBucketName().equals(fs.getBucket())
-          && request.getKey().equals(fs.pathToKey(path));
-    });
-  }
-
-  /**
-   * Skip a test case if it needs S3Guard and the filesystem does
-   * not have it.
-   */
-  private void requireS3Guard() {
-    Assume.assumeTrue("S3Guard must be enabled", fs.hasMetadataStore());
-  }
-
-  /**
-   * Skip a test case if S3 Select is not supported on this store.
-   */
-  private void requireS3Select() {
-    Assume.assumeTrue("S3 Select is not enabled",
-        getFileSystem().hasCapability(S3_SELECT_CAPABILITY));
-  }
-
-  /**
-   * Spy on the filesystem at the S3 client level.
-   * @return a mocked S3 client to which the test FS is bonded.
-   */
-  private AmazonS3 spyOnFilesystem() {
-    AmazonS3 s3ClientSpy = Mockito.spy(
-        fs.getAmazonS3ClientForTesting("mocking"));
-    fs.setAmazonS3Client(s3ClientSpy);
-    return s3ClientSpy;
-  }
-
-  /**
-   * Expect reading this stream to fail.
-   * @param instream input stream.
-   * @return the caught exception.
-   * @throws Exception an other exception
-   */
-
-  private RemoteFileChangedException expectReadFailure(
-      final FSDataInputStream instream)
-      throws Exception {
-    return intercept(RemoteFileChangedException.class, "",
-        "read() returned",
-        () -> readToText(instream.read()));
-  }
-
-  /**
-   * Convert the result of a read to a text string for errors.
-   * @param r result of the read() call.
-   * @return a string for exception text.
-   */
-  private String readToText(int r) {
-    return r < 32
-        ? (String.format("%02d", r))
-        : (String.format("%c", (char) r));
-  }
-
-  /**
-   * Is the version checking on the server?
-   * @return true if the server returns 412 errors.
-   */
-  private boolean versionCheckingIsOnServer() {
-    return fs.getChangeDetectionPolicy().getMode() == Mode.Server;
-  }
-
-  /**
-   * Stubs {@link AmazonS3#getObject(GetObjectRequest)}
-   * within s3ClientSpy to return throw a FileNotFoundException
-   * until inconsistentCallCount calls have been made.
-   * This simulates the condition where the S3 endpoint is caching
-   * a 404 request, or there is a tombstone in the way which has yet
-   * to clear.
-   * @param s3ClientSpy the spy to stub
-   * @param inconsistentCallCount the number of calls that should return the
-   * null response
-   * @param testpath the path of the object the stub should apply to
-   */
-  private void stubTemporaryNotFound(AmazonS3 s3ClientSpy,
-      int inconsistentCallCount, Path testpath) {
-    Answer<ObjectMetadata> notFound = new Answer<ObjectMetadata>() {
-      private int callCount = 0;
-
-      @Override
-      public ObjectMetadata answer(InvocationOnMock invocation
-      ) throws Throwable {
-        // simulates delayed visibility.
-        callCount++;
-        if (callCount <= inconsistentCallCount) {
-          LOG.info("Temporarily unavailable {} count {} of {}",
-              testpath, callCount, inconsistentCallCount);
-          logLocationAtDebug();
-          throw new FileNotFoundException(testpath.toString());
-        }
-        return (ObjectMetadata) invocation.callRealMethod();
-      }
-    };
-
-    // HEAD requests will fail
-    doAnswer(notFound).when(s3ClientSpy).getObjectMetadata(
-        matchingMetadataRequest(testpath));
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
new file mode 100644
index 0000000000000..d3925d35a99d3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.nio.file.AccessDeniedException;
+
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils;
+import org.apache.hadoop.fs.statistics.IOStatisticAssertions;
+import org.apache.hadoop.fs.statistics.StreamStatisticNames;
+
+import static org.apache.hadoop.fs.s3a.Constants.ALLOW_REQUESTER_PAYS;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_DEFAULT;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Tests for Requester Pays feature.
+ */
+public class ITestS3ARequesterPays extends AbstractS3ATestBase {
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+
+    Path requesterPaysPath = getRequesterPaysPath(conf);
+    String requesterPaysBucketName = requesterPaysPath.toUri().getHost();
+    S3ATestUtils.removeBaseAndBucketOverrides(
+        requesterPaysBucketName,
+        conf,
+        ALLOW_REQUESTER_PAYS,
+        S3A_BUCKET_PROBE);
+
+    return conf;
+  }
+
+  @Test
+  public void testRequesterPaysOptionSuccess() throws Throwable {
+    describe("Test requester pays enabled case by reading last then first byte");
+
+    Configuration conf = this.createConfiguration();
+    conf.setBoolean(ALLOW_REQUESTER_PAYS, true);
+    // Enable bucket exists check, the first failure point people may encounter
+    conf.setInt(S3A_BUCKET_PROBE, 2);
+
+    Path requesterPaysPath = getRequesterPaysPath(conf);
+
+    try (
+        FileSystem fs = requesterPaysPath.getFileSystem(conf);
+        FSDataInputStream inputStream = fs.open(requesterPaysPath);
+    ) {
+      long fileLength = fs.getFileStatus(requesterPaysPath).getLen();
+
+      inputStream.seek(fileLength - 1);
+      inputStream.readByte();
+
+      // Jump back to the start, triggering a new GetObject request.
+      inputStream.seek(0);
+      inputStream.readByte();
+
+      if (conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT)) {
+        // For S3APrefetchingInputStream, verify a call was made
+        IOStatisticAssertions.assertThatStatisticCounter(inputStream.getIOStatistics(),
+            StreamStatisticNames.STREAM_READ_OPENED).isEqualTo(1);
+      } else {
+        // For S3AInputStream, verify > 1 call was made,
+        // so we're sure it is correctly configured for each request
+        IOStatisticAssertions.assertThatStatisticCounter(inputStream.getIOStatistics(),
+            StreamStatisticNames.STREAM_READ_OPENED).isGreaterThan(1);
+      }
+
+      // Check list calls work without error
+      fs.listFiles(requesterPaysPath.getParent(), false);
+    }
+  }
+
+  @Test
+  public void testRequesterPaysDisabledFails() throws Throwable {
+    describe("Verify expected failure for requester pays buckets when client has it disabled");
+
+    Configuration conf = this.createConfiguration();
+    conf.setBoolean(ALLOW_REQUESTER_PAYS, false);
+    Path requesterPaysPath = getRequesterPaysPath(conf);
+
+    try (FileSystem fs = requesterPaysPath.getFileSystem(conf)) {
+      intercept(
+          AccessDeniedException.class,
+          "403 Forbidden",
+          "Expected requester pays bucket to fail without header set",
+          () -> fs.open(requesterPaysPath).close()
+      );
+    }
+  }
+
+  private static Path getRequesterPaysPath(Configuration conf) {
+    return new Path(PublicDatasetTestUtils.getRequesterPaysObject(conf));
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java
new file mode 100644
index 0000000000000..7c56f8d2ea050
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.nio.file.AccessDeniedException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+
+import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER;
+import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_ARRAY;
+import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_DISK;
+import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BYTEBUFFER;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_REDUCED_REDUNDANCY;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.XA_STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.decodeBytes;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Tests of storage class.
+ */
+@RunWith(Parameterized.class)
+public class ITestS3AStorageClass extends AbstractS3ATestBase {
+
+  /**
+   * HADOOP-18339. Parameterized the test for different fast upload buffer types
+   * to ensure the storage class configuration works with all of them.
+   */
+  @Parameterized.Parameters(name = "fast-upload-buffer-{0}")
+  public static Collection<Object[]> params() {
+    return Arrays.asList(new Object[][]{
+        {FAST_UPLOAD_BUFFER_DISK},
+        {FAST_UPLOAD_BUFFER_ARRAY}
+    });
+  }
+
+  private final String fastUploadBufferType;
+
+  public ITestS3AStorageClass(String fastUploadBufferType) {
+    this.fastUploadBufferType = fastUploadBufferType;
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    disableFilesystemCaching(conf);
+    removeBaseAndBucketOverrides(conf, STORAGE_CLASS, FAST_UPLOAD_BUFFER);
+    conf.set(FAST_UPLOAD_BUFFER, fastUploadBufferType);
+
+    return conf;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    skipIfStorageClassTestsDisabled(getConfiguration());
+  }
+
+  /*
+   * This test ensures the default storage class configuration (no config or null)
+   * works well with create and copy operations
+   */
+  @Test
+  public void testCreateAndCopyObjectWithStorageClassDefault() throws Throwable {
+    Configuration conf = this.createConfiguration();
+    S3AContract contract = (S3AContract) createContract(conf);
+    contract.init();
+
+    FileSystem fs = contract.getTestFileSystem();
+    Path dir = methodPath();
+    fs.mkdirs(dir);
+    assertObjectHasNoStorageClass(dir);
+    Path path = new Path(dir, "file1");
+    ContractTestUtils.touch(fs, path);
+    assertObjectHasNoStorageClass(path);
+    Path path2 = new Path(dir, "file1");
+    fs.rename(path, path2);
+    assertObjectHasNoStorageClass(path2);
+  }
+
+  /*
+   * Verify object can be created and copied correctly
+   * with specified storage class
+   */
+  @Test
+  public void testCreateAndCopyObjectWithStorageClassReducedRedundancy() throws Throwable {
+    Configuration conf = this.createConfiguration();
+    conf.set(STORAGE_CLASS, STORAGE_CLASS_REDUCED_REDUNDANCY);
+    S3AContract contract = (S3AContract) createContract(conf);
+    contract.init();
+
+    FileSystem fs = contract.getTestFileSystem();
+    Path dir = methodPath();
+    fs.mkdirs(dir);
+    // even with storage class specified
+    // directories do not have storage class
+    assertObjectHasNoStorageClass(dir);
+    Path path = new Path(dir, "file1");
+    ContractTestUtils.touch(fs, path);
+    assertObjectHasStorageClass(path, STORAGE_CLASS_REDUCED_REDUNDANCY);
+    Path path2 = new Path(dir, "file1");
+    fs.rename(path, path2);
+    assertObjectHasStorageClass(path2, STORAGE_CLASS_REDUCED_REDUNDANCY);
+  }
+
+  /*
+   * Archive storage classes have different behavior
+   * from general storage classes
+   */
+  @Test
+  public void testCreateAndCopyObjectWithStorageClassGlacier() throws Throwable {
+    Configuration conf = this.createConfiguration();
+    conf.set(STORAGE_CLASS, STORAGE_CLASS_GLACIER);
+    S3AContract contract = (S3AContract) createContract(conf);
+    contract.init();
+
+    FileSystem fs = contract.getTestFileSystem();
+    Path dir = methodPath();
+    fs.mkdirs(dir);
+    // even with storage class specified
+    // directories do not have storage class
+    assertObjectHasNoStorageClass(dir);
+    Path path = new Path(dir, "file1");
+    ContractTestUtils.touch(fs, path);
+    assertObjectHasStorageClass(path, STORAGE_CLASS_GLACIER);
+    Path path2 = new Path(dir, "file2");
+
+    // this is the current behavior
+    // object with archive storage class can't be read directly
+    // when trying to read it, AccessDeniedException will be thrown
+    // with message InvalidObjectState
+    intercept(AccessDeniedException.class, "InvalidObjectState", () -> fs.rename(path, path2));
+  }
+
+  /*
+   * Verify object can be created and copied correctly
+   * with completely invalid storage class
+   */
+  @Test
+  public void testCreateAndCopyObjectWithStorageClassInvalid() throws Throwable {
+    Configuration conf = this.createConfiguration();
+    conf.set(STORAGE_CLASS, "testing");
+    S3AContract contract = (S3AContract) createContract(conf);
+    contract.init();
+
+    FileSystem fs = contract.getTestFileSystem();
+    Path dir = methodPath();
+    fs.mkdirs(dir);
+    // even with storage class specified
+    // directories do not have storage class
+    assertObjectHasNoStorageClass(dir);
+    Path path = new Path(dir, "file1");
+    ContractTestUtils.touch(fs, path);
+    assertObjectHasNoStorageClass(path);
+    Path path2 = new Path(dir, "file1");
+    fs.rename(path, path2);
+    assertObjectHasNoStorageClass(path2);
+  }
+
+  /*
+   * Verify object can be created and copied correctly
+   * with empty string configuration
+   */
+  @Test
+  public void testCreateAndCopyObjectWithStorageClassEmpty() throws Throwable {
+    Configuration conf = this.createConfiguration();
+    conf.set(STORAGE_CLASS, "");
+    S3AContract contract = (S3AContract) createContract(conf);
+    contract.init();
+
+    FileSystem fs = contract.getTestFileSystem();
+    Path dir = methodPath();
+    fs.mkdirs(dir);
+    // even with storage class specified
+    // directories do not have storage class
+    assertObjectHasNoStorageClass(dir);
+    Path path = new Path(dir, "file1");
+    ContractTestUtils.touch(fs, path);
+    assertObjectHasNoStorageClass(path);
+    Path path2 = new Path(dir, "file1");
+    fs.rename(path, path2);
+    assertObjectHasNoStorageClass(path2);
+  }
+
+  /**
+   * Assert that a given object has no storage class specified.
+   *
+   * @param path path
+   */
+  protected void assertObjectHasNoStorageClass(Path path) throws Throwable {
+    S3AFileSystem fs = getFileSystem();
+    Map<String, byte[]> xAttrs = fs.getXAttrs(path);
+    String storageClass = decodeBytes(xAttrs.get(XA_STORAGE_CLASS));
+
+    Assertions.assertThat(storageClass).describedAs("Storage class of object %s", path).isNull();
+  }
+
+  /**
+   * Assert that a given object has the given storage class specified.
+   *
+   * @param path                 path
+   * @param expectedStorageClass expected storage class for the object
+   */
+  protected void assertObjectHasStorageClass(Path path, String expectedStorageClass)
+      throws Throwable {
+    S3AFileSystem fs = getFileSystem();
+    Map<String, byte[]> xAttrs = fs.getXAttrs(path);
+    String actualStorageClass = decodeBytes(xAttrs.get(XA_STORAGE_CLASS));
+
+    Assertions.assertThat(actualStorageClass).describedAs("Storage class of object %s", path)
+        .isEqualToIgnoringCase(expectedStorageClass);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
index 041b6f4c1798e..0778662542d88 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
@@ -71,6 +71,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class);
 
+  @SuppressWarnings("deprecation")
   private static final String TEMPORARY_AWS_CREDENTIALS
       = TemporaryAWSCredentialsProvider.NAME;
 
@@ -125,13 +126,14 @@ public void testSTS() throws IOException {
         credentials,
         getStsEndpoint(conf),
         getStsRegion(conf));
-    STSClientFactory.STSClient clientConnection =
-        STSClientFactory.createClientConnection(
-            builder.build(),
-            new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT));
-    Credentials sessionCreds = clientConnection
-        .requestSessionCredentials(TEST_SESSION_TOKEN_DURATION_SECONDS,
-            TimeUnit.SECONDS);
+    Credentials sessionCreds;
+    try (STSClientFactory.STSClient clientConnection =
+        STSClientFactory.createClientConnection(builder.build(),
+            new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT))) {
+      sessionCreds = clientConnection
+          .requestSessionCredentials(
+              TEST_SESSION_TOKEN_DURATION_SECONDS, TimeUnit.SECONDS);
+    }
 
     // clone configuration so changes here do not affect the base FS.
     Configuration conf2 = new Configuration(conf);
@@ -174,6 +176,7 @@ protected String getStsRegion(final Configuration conf) {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testTemporaryCredentialValidation() throws Throwable {
     Configuration conf = new Configuration();
     conf.set(ACCESS_KEY, "accesskey");
@@ -275,8 +278,7 @@ public void testInvalidSTSBinding() throws Exception {
       // this is a failure path, so fail with a meaningful error
       fail("request to create a file should have failed");
     } catch (AWSBadRequestException expected){
-      // likely at two points in the operation, depending on
-      // S3Guard state
+      // could fail in fs creation or file IO
     } finally {
       IOUtils.closeStream(fs);
     }
@@ -358,6 +360,7 @@ public void testSessionCredentialsEndpointNoRegion() throws Throwable {
    * @return the caught exception.
    * @throws Exception any unexpected exception.
    */
+  @SuppressWarnings("deprecation")
   public <E extends Exception> E expectedSessionRequestFailure(
       final Class<E> clazz,
       final String endpoint,
@@ -380,11 +383,12 @@ public <E extends Exception> E expectedSessionRequestFailure(
             Invoker invoker = new Invoker(new S3ARetryPolicy(conf),
                 LOG_AT_ERROR);
 
-            STSClientFactory.STSClient stsClient
-                = STSClientFactory.createClientConnection(tokenService,
-                invoker);
-
-            return stsClient.requestSessionCredentials(30, TimeUnit.MINUTES);
+            try (STSClientFactory.STSClient stsClient =
+                STSClientFactory.createClientConnection(
+                    tokenService, invoker)) {
+              return stsClient.requestSessionCredentials(
+                  30, TimeUnit.MINUTES);
+            }
           });
     }
   }
@@ -414,6 +418,7 @@ public void testTemporaryCredentialValidationOnLoad() throws Throwable {
           return sc.toString();
         });
   }
+
   @Test
   public void testEmptyTemporaryCredentialValidation() throws Throwable {
     Configuration conf = new Configuration();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java
index 3d7ee0882efa4..3a2d1b1b09a49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java
@@ -20,6 +20,7 @@
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.fs.statistics.IOStatistics;
@@ -33,6 +34,7 @@
 
 import java.io.IOException;
 
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_BYTES;
 import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_BYTES_READ_CLOSE;
 import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_TOTAL_BYTES;
@@ -72,6 +74,7 @@ public void testUnbuffer() throws IOException {
     IOStatisticsSnapshot iostats = new IOStatisticsSnapshot();
     // Open file, read half the data, and then call unbuffer
     try (FSDataInputStream inputStream = getFileSystem().open(dest)) {
+      skipIfCannotUnbuffer(inputStream);
       assertTrue(inputStream.getWrappedStream() instanceof S3AInputStream);
       int bytesToRead = 8;
       readAndAssertBytesRead(inputStream, bytesToRead);
@@ -138,6 +141,7 @@ public void testUnbufferStreamStatistics() throws IOException {
     Object streamStatsStr;
     try {
       inputStream = fs.open(dest);
+      skipIfCannotUnbuffer(inputStream);
       streamStatsStr = demandStringifyIOStatisticsSource(inputStream);
 
       LOG.info("initial stream statistics {}", streamStatsStr);
@@ -192,6 +196,12 @@ private boolean isObjectStreamOpen(FSDataInputStream inputStream) {
     return ((S3AInputStream) inputStream.getWrappedStream()).isObjectStreamOpen();
   }
 
+  private void skipIfCannotUnbuffer(FSDataInputStream inputStream) {
+    if (!inputStream.hasCapability(StreamCapabilities.UNBUFFER)) {
+      skip("input stream does not support unbuffer");
+    }
+  }
+
   /**
    * Read the specified number of bytes from the given
    * {@link FSDataInputStream} and assert that
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardCreate.java
deleted file mode 100644
index dcc2538ec61d9..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardCreate.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.junit.Assume;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-
-/**
- * Home for testing the creation of new files and directories with S3Guard
- * enabled.
- */
-public class ITestS3GuardCreate extends AbstractS3ATestBase {
-
-  /**
-   * Test that ancestor creation during S3AFileSystem#create() is properly
-   * accounted for in the MetadataStore.  This should be handled by the
-   * FileSystem, and be a FS contract test, but S3A does not handle ancestors on
-   * create(), so we need to take care in the S3Guard code to do the right
-   * thing.  This may change: See HADOOP-13221 for more detail.
-   */
-  @Test
-  public void testCreatePopulatesFileAncestors() throws Exception {
-    final S3AFileSystem fs = getFileSystem();
-    Assume.assumeTrue(fs.hasMetadataStore());
-    final MetadataStore ms = fs.getMetadataStore();
-    final Path parent = path("testCreatePopulatesFileAncestors");
-
-    try {
-      fs.mkdirs(parent);
-      final Path nestedFile = new Path(parent, "dir1/dir2/file4");
-      touch(fs, nestedFile);
-
-      DirListingMetadata list = ms.listChildren(parent);
-      assertFalse("MetadataStore falsely reports authoritative empty list",
-          list.isEmpty() == Tristate.TRUE);
-    } finally {
-      fs.delete(parent, true);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java
deleted file mode 100644
index d603d38432285..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardEmptyDirs.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.stream.Stream;
-
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.assertj.core.api.Assertions;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.fs.s3a.s3guard.DDBPathMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.fs.store.audit.AuditSpan;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_MKDIRS;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeFilesystemHasMetadatastore;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getStatusWithEmptyDirFlag;
-
-/**
- * Test logic around whether or not a directory is empty, with S3Guard enabled.
- * The fact that S3AFileStatus has an isEmptyDirectory flag in it makes caching
- * S3AFileStatus's really tricky, as the flag can change as a side effect of
- * changes to other paths.
- * After S3Guard is merged to trunk, we should try to remove the
- * isEmptyDirectory flag from S3AFileStatus, or maintain it outside
- * of the MetadataStore.
- */
-public class ITestS3GuardEmptyDirs extends AbstractS3ATestBase {
-
-  /**
-   * Rename an empty directory, verify that the empty dir
-   * marker moves in both S3Guard and in the S3A FS.
-   */
-  @Test
-  public void testRenameEmptyDir() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
-    Path basePath = path(getMethodName());
-    Path sourceDir = new Path(basePath, "AAA-source");
-    String sourceDirMarker = fs.pathToKey(sourceDir) + "/";
-    Path destDir = new Path(basePath, "BBB-dest");
-    String destDirMarker = fs.pathToKey(destDir) + "/";
-    // set things up.
-    mkdirs(sourceDir);
-    // create a span for all the low level operations
-    span();
-    // there's source directory marker+
-    fs.getObjectMetadata(sourceDirMarker);
-    S3AFileStatus srcStatus = getEmptyDirStatus(sourceDir);
-    assertEquals("Must be an empty dir: " + srcStatus, Tristate.TRUE,
-        srcStatus.isEmptyDirectory());
-    // do the rename
-    assertRenameOutcome(fs, sourceDir, destDir, true);
-    S3AFileStatus destStatus = getEmptyDirStatus(destDir);
-    assertEquals("Must be an empty dir: " + destStatus, Tristate.TRUE,
-        destStatus.isEmptyDirectory());
-    // source does not exist.
-    intercept(FileNotFoundException.class,
-        () -> getEmptyDirStatus(sourceDir));
-    // and verify that there's no dir marker hidden under a tombstone
-    intercept(FileNotFoundException.class,
-        () -> Invoker.once("HEAD", sourceDirMarker, () -> {
-          span();
-          ObjectMetadata md = fs.getObjectMetadata(sourceDirMarker);
-          return String.format("Object %s of length %d",
-              sourceDirMarker, md.getInstanceLength());
-        }));
-
-    // the parent dir mustn't be confused
-    S3AFileStatus baseStatus = getEmptyDirStatus(basePath);
-    assertEquals("Must not be an empty dir: " + baseStatus, Tristate.FALSE,
-        baseStatus.isEmptyDirectory());
-    // and verify the dest dir has a marker
-    span();
-    fs.getObjectMetadata(destDirMarker);
-  }
-
-  private S3AFileStatus getEmptyDirStatus(Path dir) throws IOException {
-    try (AuditSpan span = span()) {
-      return getFileSystem().innerGetFileStatus(dir, true, StatusProbeEnum.ALL);
-    }
-  }
-
-  @Test
-  public void testEmptyDirs() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    assumeFilesystemHasMetadatastore(getFileSystem());
-    MetadataStore configuredMs = fs.getMetadataStore();
-    Path existingDir = path("existing-dir");
-    Path existingFile = path("existing-dir/existing-file");
-    try {
-      // 1. Simulate files already existing in the bucket before we started our
-      // cluster.  Temporarily disable the MetadataStore so it doesn't witness
-      // us creating these files.
-
-      fs.setMetadataStore(new NullMetadataStore());
-      assertTrue(fs.mkdirs(existingDir));
-      touch(fs, existingFile);
-
-
-      // 2. Simulate (from MetadataStore's perspective) starting our cluster and
-      // creating a file in an existing directory.
-      fs.setMetadataStore(configuredMs);  // "start cluster"
-      Path newFile = path("existing-dir/new-file");
-      touch(fs, newFile);
-      span();
-      S3AFileStatus status = fs.innerGetFileStatus(existingDir, true,
-          StatusProbeEnum.ALL);
-      assertEquals("Should not be empty dir", Tristate.FALSE,
-          status.isEmptyDirectory());
-
-      // 3. Assert that removing the only file the MetadataStore witnessed
-      // being created doesn't cause it to think the directory is now empty.
-      fs.delete(newFile, false);
-      span();
-      status = fs.innerGetFileStatus(existingDir, true, StatusProbeEnum.ALL);
-      assertEquals("Should not be empty dir", Tristate.FALSE,
-          status.isEmptyDirectory());
-
-      // 4. Assert that removing the final file, that existed "before"
-      // MetadataStore started, *does* cause the directory to be marked empty.
-      fs.delete(existingFile, false);
-      span();
-      status = fs.innerGetFileStatus(existingDir, true, StatusProbeEnum.ALL);
-      assertEquals("Should be empty dir now", Tristate.TRUE,
-          status.isEmptyDirectory());
-    } finally {
-      configuredMs.forgetMetadata(existingFile);
-      configuredMs.forgetMetadata(existingDir);
-    }
-  }
-
-  /**
-   * Test tombstones don't get in the way of a listing of the
-   * root dir.
-   * This test needs to create a path which appears first in the listing,
-   * and an entry which can come later. To allow the test to proceed
-   * while other tests are running, the filename "0000" is used for that
-   * deleted entry.
-   */
-  @Test
-  public void testTombstonesAndEmptyDirectories() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
-    assumeFilesystemHasMetadatastore(getFileSystem());
-
-    // Create the first and last files.
-    Path base = path(getMethodName());
-    // use something ahead of all the ASCII alphabet characters so
-    // even during parallel test runs, this test is expected to work.
-    String first = "0000";
-    Path firstPath = new Path(base, first);
-
-    // this path is near the bottom of the ASCII string space.
-    // This isn't so critical.
-    String last = "zzzz";
-    Path lastPath = new Path(base, last);
-    touch(fs, firstPath);
-    touch(fs, lastPath);
-    // Delete first entry (+assert tombstone)
-    assertDeleted(firstPath, false);
-    DynamoDBMetadataStore ddbMs = getRequiredDDBMetastore(fs);
-    DDBPathMetadata firstMD = ddbMs.get(firstPath);
-    assertNotNull("No MD for " + firstPath, firstMD);
-    assertTrue("Not a tombstone " + firstMD,
-        firstMD.isDeleted());
-    // PUT child to store going past the FS entirely.
-    // This is not going to show up on S3Guard.
-    Path child = new Path(firstPath, "child");
-    StoreContext ctx = fs.createStoreContext();
-    String childKey = ctx.pathToKey(child);
-    String baseKey = ctx.pathToKey(base) + "/";
-    AmazonS3 s3 = fs.getAmazonS3ClientForTesting("LIST");
-    String bucket = ctx.getBucket();
-    try {
-      createEmptyObject(fs, childKey);
-
-      // Do a list
-      span();
-      ListObjectsV2Request listReq = ctx.getRequestFactory()
-          .newListObjectsV2Request(baseKey, "/", 10);
-      ListObjectsV2Result listing = s3.listObjectsV2(listReq);
-
-      // the listing has the first path as a prefix, because of the child
-      Assertions.assertThat(listing.getCommonPrefixes())
-          .describedAs("The prefixes of a LIST of %s", base)
-          .contains(baseKey + first + "/");
-
-      // and the last file is one of the files
-      Stream<String> files = listing.getObjectSummaries()
-          .stream()
-          .map(S3ObjectSummary::getKey);
-      Assertions.assertThat(files)
-          .describedAs("The files of a LIST of %s", base)
-          .contains(baseKey + last);
-
-      // verify absolutely that the last file exists
-      assertPathExists("last file", lastPath);
-
-      boolean isDDB = fs.getMetadataStore() instanceof DynamoDBMetadataStore;
-      // if DDB is the metastore, then we expect no FS requests to be made
-      // at all.
-      S3ATestUtils.MetricDiff listMetric = new S3ATestUtils.MetricDiff(fs,
-          Statistic.OBJECT_LIST_REQUEST);
-      S3ATestUtils.MetricDiff getMetric = new S3ATestUtils.MetricDiff(fs,
-          Statistic.OBJECT_METADATA_REQUESTS);
-      // do a getFile status with empty dir flag
-      S3AFileStatus status = getStatusWithEmptyDirFlag(fs, base);
-      assertNonEmptyDir(status);
-      if (isDDB) {
-        listMetric.assertDiffEquals(
-            "FileSystem called S3 LIST rather than use DynamoDB",
-            0);
-        getMetric.assertDiffEquals(
-            "FileSystem called S3 GET rather than use DynamoDB",
-            0);
-        LOG.info("Verified that DDB directory status was accepted");
-      }
-
-    } finally {
-      // try to recover from the defective state.
-      span();
-      s3.deleteObject(bucket, childKey);
-      fs.delete(lastPath, true);
-      ddbMs.forgetMetadata(firstPath);
-    }
-  }
-
-  protected void assertNonEmptyDir(final S3AFileStatus status) {
-    assertEquals("Should not be empty dir: " + status, Tristate.FALSE,
-        status.isEmptyDirectory());
-  }
-
-  /**
-   * Get the DynamoDB metastore; assume false if it is of a different
-   * type.
-   * @return extracted and cast metadata store.
-   */
-  @SuppressWarnings("ConstantConditions")
-  private DynamoDBMetadataStore getRequiredDDBMetastore(S3AFileSystem fs) {
-    MetadataStore ms = fs.getMetadataStore();
-    assume("Not a DynamoDBMetadataStore: " + ms,
-        ms instanceof DynamoDBMetadataStore);
-    return (DynamoDBMetadataStore) ms;
-  }
-
-  /**
-   * From {@code S3AFileSystem.createEmptyObject()}.
-   * @param fs filesystem
-   * @param key key
-   */
-  private void createEmptyObject(S3AFileSystem fs, String key)
-      throws IOException {
-
-    try (AuditSpan span = fs.getAuditSpanSource()
-        .createSpan(INVOCATION_MKDIRS.getSymbol(), key, null)) {
-      fs.createMkdirOperationCallbacks().createFakeDirectory(key);
-    }
-  }
-
-  @Test
-  public void testDirMarkerDelete() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
-    assumeFilesystemHasMetadatastore(getFileSystem());
-    Path baseDir = methodPath();
-    Path subFile = new Path(baseDir, "subdir/file.txt");
-    // adds the s3guard entry
-    fs.mkdirs(baseDir);
-    touch(fs, subFile);
-    // PUT a marker
-    createEmptyObject(fs, fs.pathToKey(baseDir) + "/");
-    fs.delete(baseDir, true);
-    assertPathDoesNotExist("Should have been deleted", baseDir);
-
-    // now create the dir again
-    fs.mkdirs(baseDir);
-    FileStatus fileStatus = fs.getFileStatus(baseDir);
-    Assertions.assertThat(fileStatus)
-        .matches(FileStatus::isDirectory, "Not a directory");
-    Assertions.assertThat(fs.listStatus(baseDir))
-        .describedAs("listing of %s", baseDir)
-        .isEmpty();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
deleted file mode 100644
index a5bc420704d96..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
+++ /dev/null
@@ -1,769 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.hadoop.util.Lists;
-import org.assertj.core.api.Assertions;
-import org.junit.Assume;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.FailureInjectionPolicy.*;
-import static org.apache.hadoop.fs.s3a.InconsistentAmazonS3Client.*;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Test S3Guard list consistency feature by injecting delayed listObjects()
- * visibility via {@link InconsistentAmazonS3Client}.
- *
- * Tests here generally:
- * 1. Use the inconsistency injection mentioned above.
- * 2. Only run when S3Guard is enabled.
- */
-public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
-
-  private Invoker invoker;
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    invoker = new Invoker(new S3ARetryPolicy(getConfiguration()),
-        Invoker.NO_OP
-    );
-    skipIfClientSideEncryption();
-    Assume.assumeTrue("No metadata store in test filesystem",
-        getFileSystem().hasMetadataStore());
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    if (getFileSystem() != null && getFileSystem()
-        .getAmazonS3Client() instanceof InconsistentAmazonS3Client) {
-      clearInconsistency(getFileSystem());
-    }
-    super.teardown();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
-        S3ClientFactory.class);
-    // Other configs would break test assumptions
-    conf.set(FAIL_INJECT_INCONSISTENCY_KEY, DEFAULT_DELAY_KEY_SUBSTRING);
-    conf.setFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY, 1.0f);
-    // this is a long value to guarantee that the inconsistency holds
-    // even over long-haul connections, and in the debugger too/
-    conf.setLong(FAIL_INJECT_INCONSISTENCY_MSEC, 600_1000L);
-    return new S3AContract(conf);
-  }
-
-  /**
-   * Helper function for other test cases: does a single rename operation and
-   * validates the aftermath.
-   * @param mkdirs Directories to create
-   * @param srcdirs Source paths for rename operation
-   * @param dstdirs Destination paths for rename operation
-   * @param yesdirs Files that must exist post-rename (e.g. srcdirs children)
-   * @param nodirs Files that must not exist post-rename (e.g. dstdirs children)
-   * @throws Exception
-   */
-  private void doTestRenameSequence(Path[] mkdirs, Path[] srcdirs,
-      Path[] dstdirs, Path[] yesdirs, Path[] nodirs) throws Exception {
-    S3AFileSystem fs = getFileSystem();
-
-
-    if (mkdirs != null) {
-      for (Path mkdir : mkdirs) {
-        assertTrue(fs.mkdirs(mkdir));
-      }
-      clearInconsistency(fs);
-    }
-
-    assertEquals("srcdirs and dstdirs must have equal length",
-        srcdirs.length, dstdirs.length);
-    for (int i = 0; i < srcdirs.length; i++) {
-      assertTrue("Rename returned false: " + srcdirs[i] + " -> " + dstdirs[i],
-          fs.rename(srcdirs[i], dstdirs[i]));
-    }
-
-    for (Path yesdir : yesdirs) {
-      assertTrue("Path was supposed to exist: " + yesdir, fs.exists(yesdir));
-    }
-    for (Path nodir : nodirs) {
-      assertFalse("Path is not supposed to exist: " + nodir, fs.exists(nodir));
-    }
-  }
-
-  /**
-   * Delete an array of paths; log exceptions.
-   * @param paths paths to delete
-   */
-  private void deletePathsQuietly(Path...paths) {
-    for (Path dir : paths) {
-      try {
-        getFileSystem().delete(dir, true);
-      } catch (IOException e) {
-        LOG.info("Failed to delete {}: {}", dir, e.toString());
-        LOG.debug("Delete failure:, e");
-      }
-    }
-  }
-
-  /**
-   * Tests that after renaming a directory, the original directory and its
-   * contents are indeed missing and the corresponding new paths are visible.
-   * @throws Exception
-   */
-  @Test
-  public void testConsistentListAfterRename() throws Exception {
-    Path d1f = path("d1/f");
-    Path d1f2 = path("d1/f-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    Path[] mkdirs = {d1f, d1f2};
-    Path d1 = path("d1");
-    Path[] srcdirs = {d1};
-    Path d2 = path("d2");
-    Path[] dstdirs = {d2};
-    Path d2f2 = path("d2/f-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    Path[] yesdirs = {d2, path("d2/f"), d2f2};
-    Path[] nodirs = {
-        d1, d1f, d1f2};
-    try {
-      doTestRenameSequence(mkdirs, srcdirs, dstdirs, yesdirs, nodirs);
-    } finally {
-      clearInconsistency(getFileSystem());
-      deletePathsQuietly(d1, d2, d1f, d1f2, d2f2);
-    }
-  }
-
-  /**
-   * Tests a circular sequence of renames to verify that overwriting recently
-   * deleted files and reading recently created files from rename operations
-   * works as expected.
-   * @throws Exception
-   */
-  @Test
-  public void testRollingRenames() throws Exception {
-    Path[] dir0 = {path("rolling/1")};
-    Path[] dir1 = {path("rolling/2")};
-    Path[] dir2 = {path("rolling/3")};
-    // These sets have to be in reverse order compared to the movement
-    Path[] setA = {dir1[0], dir0[0]};
-    Path[] setB = {dir2[0], dir1[0]};
-    Path[] setC = {dir0[0], dir2[0]};
-
-    try {
-      for(int i = 0; i < 2; i++) {
-        Path[] firstSet = i == 0 ? setA : null;
-        doTestRenameSequence(firstSet, setA, setB, setB, dir0);
-        doTestRenameSequence(null, setB, setC, setC, dir1);
-        doTestRenameSequence(null, setC, setA, setA, dir2);
-      }
-
-      S3AFileSystem fs = getFileSystem();
-      intercept(FileNotFoundException.class, () ->
-          fs.rename(dir2[0], dir1[0]));
-      assertTrue("Renaming over existing file should have succeeded",
-          fs.rename(dir1[0], dir0[0]));
-    } finally {
-      clearInconsistency(getFileSystem());
-      deletePathsQuietly(dir0[0], dir1[0], dir2[0]);
-    }
-  }
-
-  /**
-   * Tests that deleted files immediately stop manifesting in list operations
-   * even when the effect in S3 is delayed.
-   * @throws Exception
-   */
-  @Test
-  public void testConsistentListAfterDelete() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-
-    // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
-    // in listObjects() results via InconsistentS3Client
-    Path inconsistentPath =
-        path("a/b/dir3-" + DEFAULT_DELAY_KEY_SUBSTRING);
-
-    Path dir1 = path("a/b/dir1");
-    Path dir2 = path("a/b/dir2");
-    Path[] testDirs = {
-        dir1,
-        dir2,
-        inconsistentPath};
-
-    for (Path path : testDirs) {
-      assertTrue("Can't create directory: " + path, fs.mkdirs(path));
-    }
-    clearInconsistency(fs);
-    for (Path path : testDirs) {
-      assertTrue("Can't delete path: " + path, fs.delete(path, false));
-    }
-
-    FileStatus[] paths = fs.listStatus(path("a/b/"));
-    List<Path> list = new ArrayList<>();
-    for (FileStatus fileState : paths) {
-      list.add(fileState.getPath());
-    }
-    Assertions.assertThat(list)
-        .describedAs("Expected deleted files to be excluded")
-        .doesNotContain(dir1)
-        .doesNotContain(dir2)
-        .doesNotContain(inconsistentPath);
-  }
-
-  /**
-   * Tests that rename immediately after files in the source directory are
-   * deleted results in exactly the correct set of destination files and none
-   * of the source files.
-   * @throws Exception
-   */
-  @Test
-  public void testConsistentRenameAfterDelete() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-
-    // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
-    // in listObjects() results via InconsistentS3Client
-    Path inconsistentPath =
-        path("a/b/dir3-" + DEFAULT_DELAY_KEY_SUBSTRING);
-
-    Path[] testDirs = {path("a/b/dir1"),
-        path("a/b/dir2"),
-        inconsistentPath};
-
-    for (Path path : testDirs) {
-      assertTrue(fs.mkdirs(path));
-    }
-    clearInconsistency(fs);
-    assertTrue(fs.delete(testDirs[1], false));
-    assertTrue(fs.delete(testDirs[2], false));
-
-    ContractTestUtils.rename(fs, path("a"), path("a3"));
-    ContractTestUtils.assertPathsDoNotExist(fs,
-            "Source paths shouldn't exist post rename operation",
-            testDirs[0], testDirs[1], testDirs[2]);
-    FileStatus[] paths = fs.listStatus(path("a3/b"));
-    List<Path> list = new ArrayList<>();
-    for (FileStatus fileState : paths) {
-      list.add(fileState.getPath());
-    }
-    Assertions.assertThat(list)
-        .contains(path("a3/b/dir1"))
-        .doesNotContain(path("a3/b/dir2"))
-        .doesNotContain(path("a3/b/dir3-" +
-            DEFAULT_DELAY_KEY_SUBSTRING));
-
-    intercept(FileNotFoundException.class, "",
-        "Recently renamed dir should not be visible",
-        () -> S3AUtils.mapLocatedFiles(
-            fs.listFilesAndEmptyDirectories(path("a"), true),
-            FileStatus::getPath));
-  }
-
-  @Test
-  public void testConsistentListStatusAfterPut() throws Exception {
-
-    S3AFileSystem fs = getFileSystem();
-
-    // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
-    // in listObjects() results via InconsistentS3Client
-    Path inconsistentPath =
-        path("a/b/dir3-" + DEFAULT_DELAY_KEY_SUBSTRING);
-
-    Path[] testDirs = {path("a/b/dir1"),
-        path("a/b/dir2"),
-        inconsistentPath};
-
-    for (Path path : testDirs) {
-      assertTrue(fs.mkdirs(path));
-    }
-
-    FileStatus[] paths = fs.listStatus(path("a/b/"));
-    List<Path> list = new ArrayList<>();
-    for (FileStatus fileState : paths) {
-      list.add(fileState.getPath());
-    }
-    Assertions.assertThat(list)
-        .contains(path("a/b/dir1"))
-        .contains(path("a/b/dir2"))
-        .contains(inconsistentPath);
-  }
-
-  /**
-   * Similar to {@link #testConsistentListStatusAfterPut()}, this tests that the
-   * FS listLocatedStatus() call will return consistent list.
-   */
-  @Test
-  public void testConsistentListLocatedStatusAfterPut() throws Exception {
-    final S3AFileSystem fs = getFileSystem();
-    String rootDir = "doTestConsistentListLocatedStatusAfterPut";
-    fs.mkdirs(path(rootDir));
-
-    final int[] numOfPaths = {0, 1, 5};
-    for (int normalPathNum : numOfPaths) {
-      for (int delayedPathNum : new int[] {0, 2}) {
-        LOG.info("Testing with normalPathNum={}, delayedPathNum={}",
-            normalPathNum, delayedPathNum);
-        doTestConsistentListLocatedStatusAfterPut(fs, rootDir, normalPathNum,
-            delayedPathNum);
-      }
-    }
-  }
-
-  /**
-   * Helper method to implement the tests of consistent listLocatedStatus().
-   * @param fs The S3 file system from contract
-   * @param normalPathNum number paths listed directly from S3 without delaying
-   * @param delayedPathNum number paths listed with delaying
-   * @throws Exception
-   */
-  private void doTestConsistentListLocatedStatusAfterPut(S3AFileSystem fs,
-      String rootDir, int normalPathNum, int delayedPathNum) throws Exception {
-    final List<Path> testDirs = new ArrayList<>(normalPathNum + delayedPathNum);
-    int index = 0;
-    for (; index < normalPathNum; index++) {
-      testDirs.add(path(rootDir + "/dir-" +
-          index));
-    }
-    for (; index < normalPathNum + delayedPathNum; index++) {
-      // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
-      // in listObjects() results via InconsistentS3Client
-      testDirs.add(path(rootDir + "/dir-" + index +
-          DEFAULT_DELAY_KEY_SUBSTRING));
-    }
-
-    for (Path path : testDirs) {
-      // delete the old test path (if any) so that when we call mkdirs() later,
-      // the to delay directories will be tracked via putObject() request.
-      fs.delete(path, true);
-      assertTrue(fs.mkdirs(path));
-    }
-
-    // this should return the union data from S3 and MetadataStore
-    final RemoteIterator<LocatedFileStatus> statusIterator =
-        fs.listLocatedStatus(path(rootDir + "/"));
-    List<Path> list = new ArrayList<>();
-    for (; statusIterator.hasNext();) {
-      list.add(statusIterator.next().getPath());
-    }
-
-    // This should fail without S3Guard, and succeed with it because part of the
-    // children under test path are delaying visibility
-    for (Path path : testDirs) {
-      assertTrue("listLocatedStatus should list " + path, list.contains(path));
-    }
-  }
-
-  /**
-   * Tests that the S3AFS listFiles() call will return consistent file list.
-   */
-  @Test
-  public void testConsistentListFiles() throws Exception {
-    final S3AFileSystem fs = getFileSystem();
-
-    final int[] numOfPaths = {0, 2};
-    for (int dirNum : numOfPaths) {
-      for (int normalFile : numOfPaths) {
-        for (int delayedFile : new int[] {0, 1}) {
-          for (boolean recursive : new boolean[] {true, false}) {
-            doTestListFiles(fs, dirNum, normalFile, delayedFile, recursive);
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Helper method to implement the tests of consistent listFiles().
-   *
-   * The file structure has dirNum subdirectories, and each directory (including
-   * the test base directory itself) has normalFileNum normal files and
-   * delayedFileNum delayed files.
-   *
-   * @param fs The S3 file system from contract
-   * @param dirNum number of subdirectories
-   * @param normalFileNum number files in each directory without delay to list
-   * @param delayedFileNum number files in each directory with delay to list
-   * @param recursive listFiles recursively if true
-   * @throws Exception if any unexpected error
-   */
-  private void doTestListFiles(S3AFileSystem fs, int dirNum, int normalFileNum,
-      int delayedFileNum, boolean recursive) throws Exception {
-    describe("Testing dirNum=%d, normalFile=%d, delayedFile=%d, "
-        + "recursive=%s", dirNum, normalFileNum, delayedFileNum, recursive);
-    final Path baseTestDir = path("doTestListFiles-" + dirNum + "-"
-        + normalFileNum + "-" + delayedFileNum + "-" + recursive);
-    // delete the old test path (if any) so that when we call mkdirs() later,
-    // the to delay sub directories will be tracked via putObject() request.
-    fs.delete(baseTestDir, true);
-
-    // make subdirectories (if any)
-    final List<Path> testDirs = new ArrayList<>(dirNum + 1);
-    assertTrue(fs.mkdirs(baseTestDir));
-    testDirs.add(baseTestDir);
-    for (int i = 0; i < dirNum; i++) {
-      final Path subdir = path(baseTestDir + "/dir-" + i);
-      assertTrue(fs.mkdirs(subdir));
-      testDirs.add(subdir);
-    }
-
-    final Collection<String> fileNames
-        = new ArrayList<>(normalFileNum + delayedFileNum);
-    int index = 0;
-    for (; index < normalFileNum; index++) {
-      fileNames.add("file-" + index);
-    }
-    for (; index < normalFileNum + delayedFileNum; index++) {
-      // Any S3 keys that contain DELAY_KEY_SUBSTRING will be delayed
-      // in listObjects() results via InconsistentS3Client
-      fileNames.add("file-" + index + "-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    }
-
-    int filesAndEmptyDirectories = 0;
-
-    // create files under each test directory
-    for (Path dir : testDirs) {
-      for (String fileName : fileNames) {
-        writeTextFile(fs, new Path(dir, fileName), "I, " + fileName, false);
-        filesAndEmptyDirectories++;
-      }
-    }
-
-    // this should return the union data from S3 and MetadataStore
-    final RemoteIterator<LocatedFileStatus> statusIterator
-        = fs.listFiles(baseTestDir, recursive);
-    final Collection<Path> listedFiles = new HashSet<>();
-    for (; statusIterator.hasNext();) {
-      final FileStatus status = statusIterator.next();
-      assertTrue("FileStatus " + status + " is not a file!", status.isFile());
-      listedFiles.add(status.getPath());
-    }
-    LOG.info("S3AFileSystem::listFiles('{}', {}) -> {}",
-        baseTestDir, recursive, listedFiles);
-
-    // This should fail without S3Guard, and succeed with it because part of the
-    // files to list are delaying visibility
-    if (!recursive) {
-      // in this case only the top level files are listed
-      verifyFileIsListed(listedFiles, baseTestDir, fileNames);
-      assertEquals("Unexpected number of files returned by listFiles() call",
-          normalFileNum + delayedFileNum, listedFiles.size());
-    } else {
-      for (Path dir : testDirs) {
-        verifyFileIsListed(listedFiles, dir, fileNames);
-      }
-      assertEquals("Unexpected number of files returned by listFiles() call",
-          filesAndEmptyDirectories,
-          listedFiles.size());
-    }
-  }
-
-  private static void verifyFileIsListed(Collection<Path> listedFiles,
-      Path currentDir, Collection<String> fileNames) {
-    for (String fileName : fileNames) {
-      final Path file = new Path(currentDir, fileName);
-      assertTrue(file + " should have been listed", listedFiles.contains(file));
-    }
-  }
-
-  @Test
-  public void testCommitByRenameOperations() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
-    Path work = path("test-commit-by-rename-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    Path task00 = new Path(work, "task00");
-    fs.mkdirs(task00);
-    String name = "part-00";
-    try (FSDataOutputStream out =
-             fs.create(new Path(task00, name), false)) {
-      out.writeChars("hello");
-    }
-    for (FileStatus stat : fs.listStatus(task00)) {
-      fs.rename(stat.getPath(), work);
-    }
-    List<FileStatus> files = new ArrayList<>(2);
-    for (FileStatus stat : fs.listStatus(work)) {
-      if (stat.isFile()) {
-        files.add(stat);
-      }
-    }
-    assertFalse("renamed file " + name + " not found in " + work,
-        files.isEmpty());
-    assertEquals("more files found than expected in " + work
-        + " " + ls(work), 1, files.size());
-    FileStatus status = files.get(0);
-    assertEquals("Wrong filename in " + status,
-        name, status.getPath().getName());
-  }
-
-  @Test
-  public void testInconsistentS3ClientDeletes() throws Throwable {
-    describe("Verify that delete adds tombstones which block entries"
-        + " returned in (inconsistent) listings");
-    // Test only implemented for v2 S3 list API
-    assumeV2ListAPI();
-
-    S3AFileSystem fs = getFileSystem();
-    Path root = path("testInconsistentS3ClientDeletes-"
-        + DEFAULT_DELAY_KEY_SUBSTRING);
-    for (int i = 0; i < 3; i++) {
-      fs.mkdirs(new Path(root, "dir-" + i));
-      touch(fs, new Path(root, "file-" + i));
-      for (int j = 0; j < 3; j++) {
-        touch(fs, new Path(new Path(root, "dir-" + i), "file-" + i + "-" + j));
-      }
-    }
-    clearInconsistency(fs);
-
-    String key = fs.pathToKey(root) + "/";
-
-    LOG.info("Listing objects before executing delete()");
-    ListObjectsV2Result preDeleteDelimited = listObjectsV2(fs, key, "/");
-    ListObjectsV2Result preDeleteUndelimited = listObjectsV2(fs, key, null);
-
-    LOG.info("Deleting the directory {}", root);
-    fs.delete(root, true);
-    LOG.info("Delete completed; listing results which must exclude deleted"
-        + " paths");
-
-    ListObjectsV2Result postDeleteDelimited = listObjectsV2(fs, key, "/");
-    boolean stripTombstones = false;
-    assertObjectSummariesEqual(
-        "InconsistentAmazonS3Client added back objects incorrectly " +
-            "in a non-recursive listing",
-        preDeleteDelimited, postDeleteDelimited,
-        stripTombstones);
-
-    assertListSizeEqual("InconsistentAmazonS3Client added back prefixes incorrectly " +
-            "in a non-recursive listing",
-        preDeleteDelimited.getCommonPrefixes(),
-        postDeleteDelimited.getCommonPrefixes());
-    LOG.info("Executing Deep listing");
-    ListObjectsV2Result postDeleteUndelimited = listObjectsV2(fs, key, null);
-    assertObjectSummariesEqual("InconsistentAmazonS3Client added back objects"
-            + " incorrectly in a recursive listing",
-        preDeleteUndelimited, postDeleteUndelimited,
-        stripTombstones);
-
-    assertListSizeEqual("InconsistentAmazonS3Client added back prefixes incorrectly " +
-            "in a recursive listing",
-        preDeleteUndelimited.getCommonPrefixes(),
-        postDeleteUndelimited.getCommonPrefixes()
-    );
-  }
-
-  private void assertObjectSummariesEqual(final String message,
-      final ListObjectsV2Result expected,
-      final ListObjectsV2Result actual,
-      final boolean stripTombstones) {
-    assertCollectionsEqual(
-        message,
-        stringify(expected.getObjectSummaries(), stripTombstones),
-        stringify(actual.getObjectSummaries(), stripTombstones));
-  }
-
-  List<String> stringify(List<S3ObjectSummary> objects,
-      boolean stripTombstones) {
-    return objects.stream()
-        .filter(s -> !stripTombstones || !(s.getKey().endsWith("/")))
-        .map(s -> s.getKey())
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Require the v2 S3 list API.
-   */
-  protected void assumeV2ListAPI() {
-    Assume.assumeTrue(getConfiguration()
-        .getInt(LIST_VERSION, DEFAULT_LIST_VERSION) == 2);
-  }
-
-  /**
-   * Verify that delayed S3 listings doesn't stop the FS from deleting
-   * a directory tree. This has not always been the case; this test
-   * verifies the fix and prevents regression.
-   */
-  @Test
-  public void testDeleteUsesS3Guard() throws Throwable {
-    describe("Verify that delete() uses S3Guard to get a consistent"
-        + " listing of its directory structure");
-    assumeV2ListAPI();
-    S3AFileSystem fs = getFileSystem();
-    Path root = path(
-        "testDeleteUsesS3Guard-" + DEFAULT_DELAY_KEY_SUBSTRING);
-    for (int i = 0; i < 3; i++) {
-      Path path = new Path(root, "file-" + i);
-      touch(fs, path);
-    }
-    // we now expect the listing to miss these
-    String key = fs.pathToKey(root) + "/";
-
-    // verify that the inconsistent listing does not show these
-    LOG.info("Listing objects before executing delete()");
-    List<Path> preDeletePaths = objectsToPaths(listObjectsV2(fs, key, null));
-    Assertions.assertThat(preDeletePaths)
-        .isEmpty();
-    // do the delete
-    fs.delete(root, true);
-
-    // now go through every file and verify that it is not there.
-    // if you comment out the delete above and run this test case,
-    // the assertion will fail; this is how the validity of the assertions
-    // were verified.
-    clearInconsistency(fs);
-    List<Path> postDeletePaths =
-        objectsToPaths(listObjectsV2(fs, key, null));
-    Assertions.assertThat(postDeletePaths)
-        .isEmpty();
-  }
-
-  private List<Path> objectsToPaths(ListObjectsV2Result r) {
-    S3AFileSystem fs = getFileSystem();
-    return r.getObjectSummaries().stream()
-        .map(s -> fs.keyToQualifiedPath(s.getKey()))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Tests that the file's eTag and versionId are preserved in recursive
-   * listings.
-   */
-  @Test
-  public void testListingReturnsVersionMetadata() throws Throwable {
-    S3AFileSystem fs = getFileSystem();
-
-    // write simple file
-    Path parent = path(getMethodName());
-    Path file = new Path(parent, "file1");
-    try (FSDataOutputStream outputStream = fs.create(file)) {
-      outputStream.writeChars("hello");
-    }
-
-    // get individual file status
-    FileStatus[] fileStatuses = fs.listStatus(file);
-    assertEquals(1, fileStatuses.length);
-    S3AFileStatus status = (S3AFileStatus) fileStatuses[0];
-    String eTag = status.getETag();
-    assertNotNull("Etag in " + eTag, eTag);
-    String versionId = status.getVersionId();
-
-    // get status through recursive directory listing
-    RemoteIterator<LocatedFileStatus> filesIterator = fs.listFiles(
-        parent, true);
-    List<LocatedFileStatus> files = Lists.newArrayList();
-    while (filesIterator.hasNext()) {
-      files.add(filesIterator.next());
-    }
-    Assertions.assertThat(files)
-        .hasSize(1);
-
-    // ensure eTag and versionId are preserved in directory listing
-    S3ALocatedFileStatus locatedFileStatus =
-        (S3ALocatedFileStatus) files.get(0);
-    assertEquals("etag of " + locatedFileStatus,
-        eTag, locatedFileStatus.getETag());
-    assertEquals("versionID of " + locatedFileStatus,
-        versionId, locatedFileStatus.getVersionId());
-  }
-
-  /**
-   * Assert that the two collections match using
-   * object equality of the elements within.
-   * @param message text for the assertion
-   * @param expected expected list
-   * @param actual actual list
-   * @param <T> type of list
-   */
-  private <T> void assertCollectionsEqual(String message,
-      Collection<T> expected,
-      Collection<T> actual) {
-    Assertions.assertThat(actual)
-        .describedAs(message)
-        .containsExactlyInAnyOrderElementsOf(expected);
-  }
-
-  /**
-   * Assert that the two list sizes match; failure message includes the lists.
-   * @param message text for the assertion
-   * @param expected expected list
-   * @param actual actual list
-   * @param <T> type of list
-   */
-  private <T> void assertListSizeEqual(String message,
-      List<T> expected,
-      List<T> actual) {
-    String leftContents = expected.stream()
-        .map(n -> n.toString())
-        .collect(Collectors.joining("\n"));
-    String rightContents = actual.stream()
-        .map(n -> n.toString())
-        .collect(Collectors.joining("\n"));
-    String summary = "\nExpected:" + leftContents
-        + "\n-----------\n"
-        + "Actual:" + rightContents
-        + "\n-----------\n";
-
-    if (expected.size() != actual.size()) {
-      LOG.error(message + summary);
-    }
-    assertEquals(message + summary, expected.size(), actual.size());
-  }
-
-  /**
-   * Retrying v2 list directly through the s3 client.
-   * @param fs filesystem
-   * @param key key to list under
-   * @param delimiter any delimiter
-   * @return the listing
-   * @throws IOException on error
-   */
-  @Retries.RetryRaw
-  private ListObjectsV2Result listObjectsV2(S3AFileSystem fs,
-      String key, String delimiter) throws IOException {
-    ListObjectsV2Request k = fs.createListObjectsRequest(key, delimiter)
-        .getV2();
-    return invoker.retryUntranslated("list", true,
-        () -> {
-          return fs.getAmazonS3ClientForTesting("list").listObjectsV2(k);
-        });
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java
deleted file mode 100644
index 2d4173d1c2ad4..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardOutOfBandOperations.java
+++ /dev/null
@@ -1,1191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.stream.Collectors;
-
-import org.assertj.core.api.Assertions;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
-import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Source;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.readBytesToString;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.readDataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile;
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE_NONE;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_METADATA_TTL;
-import static org.apache.hadoop.fs.s3a.Constants.RETRY_INTERVAL;
-import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_INTERVAL;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_CONSISTENCY_RETRY_LIMIT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.PROBE_INTERVAL_MILLIS;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.STABILIZATION_TIME;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.TIMESTAMP_SLEEP;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.checkListingContainsPath;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.checkListingDoesNotContainPath;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.read;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.readWithStatus;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.test.LambdaTestUtils.eventually;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
-import static org.junit.Assume.assumeTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- *
- * This integration test is for documenting and defining how S3Guard should
- * behave in case of out-of-band (OOB) operations.
- * <pre>
- * The behavior is the following in case of S3AFileSystem.getFileStatus:
- * A client with S3Guard
- * B client without S3Guard (Directly to S3)
- *
- * * OOB OVERWRITE, authoritative mode:
- * ** A client creates F1 file
- * ** B client overwrites F1 file with F2 (Same, or different file size)
- * ** A client's getFileStatus returns F1 metadata
- *
- * * OOB OVERWRITE, NOT authoritative mode:
- * ** A client creates F1 file
- * ** B client overwrites F1 file with F2 (Same, or different file size)
- * ** A client's getFileStatus returns F2 metadata. In not authoritative
- * mode we check S3 for the file. If the modification time of the file in S3
- * is greater than in S3Guard, we can safely return the S3 file metadata and
- * update the cache.
- *
- * * OOB DELETE, authoritative mode:
- * ** A client creates F file
- * ** B client deletes F file
- * ** A client's getFileStatus returns that the file is still there
- *
- * * OOB DELETE, NOT authoritative mode:
- * ** A client creates F file
- * ** B client deletes F file
- * ** A client's getFileStatus returns that the file is still there
- *
- * As you can see, authoritative and NOT authoritative mode behaves the same
- * at OOB DELETE case.
- *
- * The behavior is the following in case of S3AFileSystem.listStatus:
- * * File status in metadata store gets updated during the listing (in
- * S3Guard.dirListingUnion) the same way as in getFileStatus.
- * </pre>
- */
-@RunWith(Parameterized.class)
-public class ITestS3GuardOutOfBandOperations extends AbstractS3ATestBase {
-
-  private S3AFileSystem guardedFs;
-  private S3AFileSystem rawFS;
-
-  private MetadataStore realMs;
-
-  /**
-   * Is the "real" FS Authoritative.
-   */
-  private final boolean authoritative;
-
-  /**
-   * Test array for parameterized test runs.
-   * @return a list of parameter tuples.
-   */
-  @Parameterized.Parameters(name="auth={0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {true}, {false}
-    });
-  }
-
-  public ITestS3GuardOutOfBandOperations(final boolean authoritative) {
-    this.authoritative = authoritative;
-  }
-
-  /**
-   * By changing the method name, the thread name is changed and
-   * so you can see in the logs which mode is being tested.
-   * @return a string to use for the thread namer.
-   */
-  @Override
-  protected String getMethodName() {
-    return super.getMethodName() +
-        (authoritative ? "-auth" : "-nonauth");
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // reduce retry limit so FileNotFoundException cases timeout faster,
-    // speeding up the tests
-    removeBaseAndBucketOverrides(conf,
-        RETRY_LIMIT,
-        RETRY_INTERVAL,
-        S3GUARD_CONSISTENCY_RETRY_INTERVAL,
-        S3GUARD_CONSISTENCY_RETRY_LIMIT,
-        CHANGE_DETECT_MODE,
-        METADATASTORE_METADATA_TTL);
-    conf.setInt(RETRY_LIMIT, 3);
-    conf.setInt(S3GUARD_CONSISTENCY_RETRY_LIMIT, 3);
-    conf.set(CHANGE_DETECT_MODE, CHANGE_DETECT_MODE_NONE);
-    final String delay = "10ms";
-    conf.set(RETRY_INTERVAL, delay);
-    conf.set(S3GUARD_CONSISTENCY_RETRY_INTERVAL, delay);
-    conf.set(METADATASTORE_METADATA_TTL, delay);
-    return conf;
-  }
-
-  @Before
-  public void setup() throws Exception {
-    super.setup();
-    S3AFileSystem fs = getFileSystem();
-    // These test will fail if no ms
-    assumeTrue("FS needs to have a metadatastore.",
-        fs.hasMetadataStore());
-    assumeTrue("Metadatastore should persist authoritative bit",
-        metadataStorePersistsAuthoritativeBit(fs.getMetadataStore()));
-
-    // This test setup shares a single metadata store across instances,
-    // so that test runs with a local FS work.
-    // but this needs to be addressed in teardown, where the guarded fs
-    // needs to be detached from the metadata store before it is closed,
-    realMs = fs.getMetadataStore();
-    // now we create a new FS with the auth parameter
-    guardedFs = createGuardedFS(authoritative);
-    assertTrue("No S3Guard store for " + guardedFs,
-        guardedFs.hasMetadataStore());
-    assertEquals("Authoritative status in " + guardedFs,
-        authoritative, guardedFs.hasAuthoritativeMetadataStore());
-
-    // create raw fs without s3guard
-    rawFS = createUnguardedFS();
-    assertFalse("Raw FS still has S3Guard " + rawFS,
-        rawFS.hasMetadataStore());
-    nameThread();
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    if (guardedFs != null) {
-      // detach from the (shared) metadata store.
-      guardedFs.setMetadataStore(new NullMetadataStore());
-      // and only then close it.
-      IOUtils.cleanupWithLogger(LOG, guardedFs);
-    }
-    IOUtils.cleanupWithLogger(LOG, rawFS);
-    super.teardown();
-  }
-
-  /**
-   * Create a new FS which is the same config as the test FS, except
-   * that it is guarded with the specific authoritative mode.
-   * @param authoritativeMode mode of the new FS's metastore
-   * @return the new FS
-   */
-  private S3AFileSystem createGuardedFS(boolean authoritativeMode)
-      throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        CHANGE_DETECT_MODE,
-        METADATASTORE_AUTHORITATIVE,
-        METADATASTORE_METADATA_TTL,
-        AUTHORITATIVE_PATH);
-    config.setBoolean(METADATASTORE_AUTHORITATIVE, authoritativeMode);
-    config.setLong(METADATASTORE_METADATA_TTL,
-        5_000);
-    final S3AFileSystem gFs = createFS(uri, config);
-    // set back the same metadata store instance
-    gFs.setMetadataStore(realMs);
-    return gFs;
-  }
-
-  /**
-   * Create a test filesystem which is always unguarded.
-   * This filesystem MUST be closed in test teardown.
-   * @return the new FS
-   */
-  private S3AFileSystem createUnguardedFS() throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        S3_METADATA_STORE_IMPL);
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    return createFS(uri, config);
-  }
-
-  /**
-   * Create and initialize a new filesystem.
-   * This filesystem MUST be closed in test teardown.
-   * @param uri FS URI
-   * @param config config.
-   * @return new instance
-   * @throws IOException failure
-   */
-  private S3AFileSystem createFS(final URI uri, final Configuration config)
-      throws IOException {
-    S3AFileSystem fs2 = new S3AFileSystem();
-    fs2.initialize(uri, config);
-    return fs2;
-  }
-
-  @Test
-  public void testSameLengthOverwrite() throws Exception {
-    String firstText = "hello, world!";
-    String secondText = "HELLO, WORLD!";
-    overwriteFile(firstText, secondText);
-  }
-
-  @Test
-  public void testLongerLengthOverwrite() throws Exception {
-    String firstText = "Hello, World!";
-    String secondText = firstText + " " + firstText;
-    overwriteFile(firstText, secondText);
-  }
-
-  @Test
-  public void testOutOfBandDeletes() throws Exception {
-
-    Path testFileName = path("OutOfBandDelete-" + UUID.randomUUID());
-    outOfBandDeletes(testFileName, authoritative);
-  }
-
-  @Test
-  public void testListingSameLengthOverwrite() throws Exception {
-    overwriteFileInListing("THE TEXT", "the text");
-  }
-
-  @Test
-  public void testListingLongerLengthOverwrite() throws Exception {
-    overwriteFileInListing("THE TEXT", "THE LONGER TEXT");
-  }
-
-  /**
-   * Tests that tombstone expiry is implemented. If a file is created raw
-   * while the tombstone exist in ms for with the same name then S3Guard will
-   * check S3 for the file.
-   *
-   * Seq: create guarded; delete guarded; create raw (same path); read guarded;
-   * This will fail if no tombstone expiry is set
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testTombstoneExpiryGuardedDeleteRawCreate() throws Exception {
-    boolean allowAuthoritative = authoritative;
-    Path testFilePath = path("TEGDRC-" + UUID.randomUUID() + "/file");
-    LOG.info("Allow authoritative param: {}",  allowAuthoritative);
-    String originalText = "some test";
-    String newText = "the new originalText for test";
-
-    final ITtlTimeProvider originalTimeProvider =
-        guardedFs.getTtlTimeProvider();
-    try {
-      final AtomicLong now = new AtomicLong(1);
-      final AtomicLong metadataTtl = new AtomicLong(1);
-
-      // SET TTL TIME PROVIDER FOR TESTING
-      ITtlTimeProvider testTimeProvider =
-          new ITtlTimeProvider() {
-            @Override public long getNow() {
-              return now.get();
-            }
-
-            @Override public long getMetadataTtl() {
-              return metadataTtl.get();
-            }
-          };
-      guardedFs.setTtlTimeProvider(testTimeProvider);
-
-      // CREATE GUARDED
-      createAndAwaitFs(guardedFs, testFilePath, originalText);
-
-      // DELETE GUARDED
-      deleteGuardedTombstoned(guardedFs, testFilePath, now);
-
-      // CREATE RAW
-      createAndAwaitFs(rawFS, testFilePath, newText);
-
-      // CHECK LISTING - THE FILE SHOULD NOT BE THERE, EVEN IF IT'S CREATED RAW
-      checkListingDoesNotContainPath(guardedFs, testFilePath);
-
-      // CHANGE TTL SO ENTRY (& TOMBSTONE METADATA) WILL EXPIRE
-      long willExpire = now.get() + metadataTtl.get() + 1L;
-      now.set(willExpire);
-      LOG.info("willExpire: {}, ttlNow: {}; ttlTTL: {}", willExpire,
-          testTimeProvider.getNow(), testTimeProvider.getMetadataTtl());
-
-      // READ GUARDED
-      // This should fail in authoritative mode since we trust the metadatastore
-      // despite of the expiry. The metadata will not expire.
-      if (authoritative) {
-        intercept(FileNotFoundException.class, testFilePath.toString(),
-            "File should not be present in the metedatastore in authoritative mode.",
-            () -> readBytesToString(guardedFs, testFilePath, newText.length()));
-      } else {
-        String newRead = readBytesToString(guardedFs, testFilePath,
-            newText.length());
-
-        // CHECK LISTING - THE FILE SHOULD BE THERE, TOMBSTONE EXPIRED
-        checkListingContainsPath(guardedFs, testFilePath);
-
-        // we can assert that the originalText is the new one, which created raw
-        LOG.info("Old: {}, New: {}, Read: {}", originalText, newText, newRead);
-        assertEquals("The text should be modified with a new.", newText,
-            newRead);
-      }
-
-    } finally {
-      guardedFs.delete(testFilePath, true);
-      guardedFs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  private void createAndAwaitFs(S3AFileSystem fs, Path testFilePath,
-      String text) throws Exception {
-    writeTextFile(fs, testFilePath, text, true);
-    final FileStatus newStatus = awaitFileStatus(fs, testFilePath);
-    assertNotNull("Newly created file status should not be null.", newStatus);
-  }
-
-  private void deleteGuardedTombstoned(S3AFileSystem guarded,
-      Path testFilePath, AtomicLong now) throws Exception {
-    guarded.delete(testFilePath, true);
-
-    final PathMetadata metadata =
-        guarded.getMetadataStore().get(testFilePath);
-    assertNotNull("Created file metadata should not be null in ms",
-        metadata);
-    assertEquals("Created file metadata last_updated should equal with "
-            + "mocked now", now.get(), metadata.getLastUpdated());
-
-    intercept(FileNotFoundException.class, testFilePath.toString(),
-        "This file should throw FNFE when reading through "
-            + "the guarded fs, and the metadatastore tombstoned the file.",
-        () -> guarded.getFileStatus(testFilePath));
-  }
-
-  /**
-   * createNonRecursive must fail if the parent directory has been deleted,
-   * and succeed if the tombstone has expired and the directory has been
-   * created out of band.
-   */
-  @Test
-  public void testCreateNonRecursiveFailsIfParentDeleted() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-
-    String dirToDelete = methodName + UUID.randomUUID().toString();
-    String fileToTry = dirToDelete + "/theFileToTry";
-
-    final Path dirPath = path(dirToDelete);
-    final Path filePath = path(fileToTry);
-
-    // Create a directory with
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = guardedFs.getTtlTimeProvider();
-
-    try {
-      guardedFs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getNow()).thenReturn(100L);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(5L);
-
-      // CREATE DIRECTORY
-      guardedFs.mkdirs(dirPath);
-
-      // DELETE DIRECTORY
-      guardedFs.delete(dirPath, true);
-
-      // WRITE TO DELETED DIRECTORY - FAIL
-      intercept(FileNotFoundException.class,
-          dirToDelete,
-          "createNonRecursive must fail if the parent directory has been deleted.",
-          () -> createNonRecursive(guardedFs, filePath));
-
-      // CREATE THE DIRECTORY RAW
-      rawFS.mkdirs(dirPath);
-      awaitFileStatus(rawFS, dirPath);
-
-      // SET TIME SO METADATA EXPIRES
-      when(mockTimeProvider.getNow()).thenReturn(110L);
-
-      // WRITE TO DELETED DIRECTORY
-      // - FAIL ON AUTH = TRUE
-      // - SUCCESS ON AUTH = FALSE
-      if (authoritative) {
-        intercept(FileNotFoundException.class, filePath.getParent().toString(),
-            "Parent does not exist, so in authoritative mode this should fail.",
-            () -> createNonRecursive(guardedFs, filePath));
-      } else {
-        createNonRecursive(guardedFs, filePath);
-      }
-
-    } finally {
-      guardedFs.delete(filePath, true);
-      guardedFs.delete(dirPath, true);
-      guardedFs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * When lastUpdated = 0 the entry should not expire. This is a special case
-   * eg. for old metadata entries
-   */
-  @Test
-  public void testLastUpdatedZeroWontExpire() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-
-    String testFile = methodName + UUID.randomUUID().toString() +
-        "/theFileToTry";
-
-    long ttl = 10L;
-    final Path filePath = path(testFile);
-
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = guardedFs.getTtlTimeProvider();
-
-    try {
-      guardedFs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(ttl);
-
-      // create a file while the NOW is 0, so it will set 0 as the last_updated
-      when(mockTimeProvider.getNow()).thenReturn(0L);
-      touch(guardedFs, filePath);
-      deleteFile(guardedFs, filePath);
-
-      final PathMetadata pathMetadata =
-          guardedFs.getMetadataStore().get(filePath);
-      assertNotNull("pathMetadata should not be null after deleting with "
-          + "tombstones", pathMetadata);
-      assertEquals("pathMetadata lastUpdated field should be 0", 0,
-          pathMetadata.getLastUpdated());
-
-      // set the time, so the metadata would expire
-      when(mockTimeProvider.getNow()).thenReturn(2*ttl);
-      intercept(FileNotFoundException.class, filePath.toString(),
-          "This file should throw FNFE when reading through "
-              + "the guarded fs, and the metadatastore tombstoned the file. "
-              + "The tombstone won't expire if lastUpdated is set to 0.",
-          () -> guardedFs.getFileStatus(filePath));
-
-    } finally {
-      guardedFs.delete(filePath, true);
-      guardedFs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * 1. File is deleted in the guarded fs.
-   * 2. File is replaced in the raw fs.
-   * 3. File is deleted in the guarded FS after the expiry time.
-   * 4. File MUST NOT exist in raw FS.
-   */
-  @Test
-  public void deleteAfterTombstoneExpiryOobCreate() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-
-    String testFile = methodName + UUID.randomUUID().toString() +
-        "/theFileToTry";
-
-    long ttl = 10L;
-    final Path filePath = path(testFile);
-
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = guardedFs.getTtlTimeProvider();
-
-    try {
-      guardedFs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(ttl);
-
-      // CREATE AND DELETE WITH GUARDED FS
-      when(mockTimeProvider.getNow()).thenReturn(100L);
-      touch(guardedFs, filePath);
-      deleteFile(guardedFs, filePath);
-
-      final PathMetadata pathMetadata =
-          guardedFs.getMetadataStore().get(filePath);
-      assertNotNull("pathMetadata should not be null after deleting with "
-          + "tombstones", pathMetadata);
-
-      // REPLACE WITH RAW FS
-      touch(rawFS, filePath);
-      awaitFileStatus(rawFS, filePath);
-
-      // SET EXPIRY TIME, SO THE TOMBSTONE IS EXPIRED
-      when(mockTimeProvider.getNow()).thenReturn(100L + 2 * ttl);
-
-      // DELETE IN GUARDED FS
-      // NOTE: in auth this will be ineffective:
-      //  we already have the tombstone marker on the item, it won't expire,
-      //  so we don't delete the raw S3 file.
-      guardedFs.delete(filePath, true);
-
-      // FILE MUST NOT EXIST IN RAW
-      // If authoritative, the file status can be retrieved raw:
-      //    deleting with guarded FS won't do anything because the tombstone
-      //    marker won't expire in auth mode.
-      // If not authoritative, we go to the S3 bucket and get an FNFE
-      if (authoritative) {
-        rawFS.getFileStatus(filePath);
-      } else {
-        intercept(FileNotFoundException.class, filePath.toString(),
-            "This file should throw FNFE when reading through "
-                + "the raw fs, and the guarded fs deleted the file.",
-            () -> rawFS.getFileStatus(filePath));
-      }
-
-    } finally {
-      guardedFs.delete(filePath, true);
-      guardedFs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * Test that a tombstone won't hide an entry after it's expired in the
-   * listing.
-   */
-  @Test
-  public void testRootTombstones() throws Exception {
-    long ttl = 10L;
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    when(mockTimeProvider.getMetadataTtl()).thenReturn(ttl);
-    when(mockTimeProvider.getNow()).thenReturn(100L);
-    ITtlTimeProvider originalTimeProvider = guardedFs.getTtlTimeProvider();
-    guardedFs.setTtlTimeProvider(mockTimeProvider);
-
-    Path base = path(getMethodName() + UUID.randomUUID());
-    Path testFile = new Path(base, "test.file");
-
-    try {
-      touch(guardedFs, testFile);
-      ContractTestUtils.assertDeleted(guardedFs, testFile, false);
-
-      touch(rawFS, testFile);
-      awaitFileStatus(rawFS, testFile);
-
-      // the rawFS will include the file=
-      LambdaTestUtils.eventually(5000, 1000, () -> {
-        checkListingContainsPath(rawFS, testFile);
-      });
-
-      // it will be hidden because of the tombstone
-      checkListingDoesNotContainPath(guardedFs, testFile);
-
-      // the tombstone is expired, so we should detect the file
-      // in non-authoritative mode
-      when(mockTimeProvider.getNow()).thenReturn(100 + ttl);
-      if (authoritative) {
-        checkListingDoesNotContainPath(guardedFs, testFile);
-      } else {
-        checkListingContainsPath(guardedFs, testFile);
-      }
-    } finally {
-      // cleanup
-      guardedFs.delete(base, true);
-      guardedFs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * Perform an out-of-band delete.
-   * @param testFilePath filename
-   * @param allowAuthoritative  is the store authoritative
-   * @throws Exception failure
-   */
-  private void outOfBandDeletes(
-      final Path testFilePath,
-      final boolean allowAuthoritative)
-      throws Exception {
-    try {
-      // Create initial file
-      String text = "Hello, World!";
-      writeTextFile(guardedFs, testFilePath, text, true);
-      awaitFileStatus(rawFS, testFilePath);
-
-      // Delete the file without S3Guard (raw)
-      deleteFile(rawFS, testFilePath);
-
-      // The check is the same if s3guard is authoritative and if it's not
-      // it should be in the ms
-      FileStatus status = guardedFs.getFileStatus(testFilePath);
-      LOG.info("Authoritative: {} status path: {}",
-          allowAuthoritative, status.getPath());
-      final boolean versionedChangeDetection =
-          isVersionedChangeDetection();
-      if (!versionedChangeDetection) {
-        expectExceptionWhenReading(testFilePath, text);
-        expectExceptionWhenReadingOpenFileAPI(testFilePath, text, null);
-        expectExceptionWhenReadingOpenFileAPI(testFilePath, text, status);
-      } else {
-        // FNFE not expected when using a bucket with object versioning
-        final String read1 = read(guardedFs, testFilePath);
-        assertEquals("File read from the auth FS", text, read1);
-        // and when the status is passed in, even the raw FS will ask for it
-        // via the versionId in the status
-        final String read2 = readWithStatus(rawFS, status);
-        assertEquals("File read from the raw FS", text, read2);
-      }
-    } finally {
-      guardedFs.delete(testFilePath, true);
-    }
-  }
-
-  /**
-   * Overwrite a file out of band.
-   * @param firstText first text
-   * @param secondText second text
-   * @throws Exception failure
-   */
-  private void overwriteFile(String firstText, String secondText)
-      throws Exception {
-    boolean allowAuthoritative = authoritative;
-    Path testFilePath = path("OverwriteFileTest-" + UUID.randomUUID());
-    LOG.info("Allow authoritative param: {}",  allowAuthoritative);
-    try {
-      // Create initial file
-      writeTextFile(
-          guardedFs, testFilePath, firstText, true);
-      // and cache the value for later
-      final FileStatus origStatus = awaitFileStatus(rawFS, testFilePath);
-      waitForDifferentTimestamps();
-      // Overwrite the file without S3Guard
-      writeTextFile(
-          rawFS, testFilePath, secondText, true);
-
-      // Read the file and verify the data
-      eventually(STABILIZATION_TIME, PROBE_INTERVAL_MILLIS,
-          () -> {
-            FileStatus rawFileStatus = rawFS.getFileStatus(testFilePath);
-            final FileStatus guardedFileStatus =
-                guardedFs.getFileStatus(testFilePath);
-            verifyFileStatusAsExpected(firstText, secondText,
-                allowAuthoritative,
-                origStatus,
-                rawFileStatus,
-                guardedFileStatus);
-          });
-    } finally {
-      guardedFs.delete(testFilePath, true);
-    }
-  }
-
-  /**
-   * Assert that an array has a given size; in failure the full string values
-   * of the array will be included, one per line.
-   * @param message message for errors.
-   * @param expected expected length.
-   * @param array the array to probe
-   */
-  private <T> void assertArraySize(
-      final String message,
-      final int expected,
-      final T[] array) {
-    if (expected != array.length) {
-      // condition is not met, build an error which includes all the entries
-      String listing = Arrays.stream(array)
-          .map(Object::toString)
-          .collect(Collectors.joining("\n"));
-      fail(message + ": expected " + expected + " elements but found "
-          + array.length
-          + "\n" + listing);
-    }
-  }
-
-  /**
-   * Overwrite a file, verify that the text is different as is the timestamp.
-   * There are some pauses in the test to ensure that timestamps are different.
-   * @param firstText first text to write
-   * @param secondText second text to write
-   */
-  private void overwriteFileInListing(String firstText, String secondText)
-      throws Exception {
-    boolean allowAuthoritative = authoritative;
-
-    LOG.info("Authoritative mode enabled: {}", allowAuthoritative);
-    String rUUID = UUID.randomUUID().toString();
-    String testDir = "dir-" + rUUID + "/";
-    String testFile = testDir + "file-1-" + rUUID;
-    Path testDirPath = path(testDir);
-    Path testFilePath = guardedFs.qualify(path(testFile));
-
-    try {
-      // Create initial statusIterator with guarded ms
-      writeTextFile(guardedFs, testFilePath, firstText, true);
-      // and cache the value for later
-      final S3AFileStatus origStatus = awaitFileStatus(rawFS, testFilePath);
-      assertNotNull("No etag in raw status " + origStatus,
-          origStatus.getETag());
-
-      // Do a listing to cache the lists. Should be authoritative if it's set.
-      final S3AFileStatus[] origList = (S3AFileStatus[]) guardedFs.listStatus(
-          testDirPath);
-      assertArraySize("Added one file to the new dir, so the number of "
-              + "files in the dir should be one.", 1, origList);
-      S3AFileStatus origGuardedFileStatus = origList[0];
-      assertNotNull("No etag in origGuardedFileStatus " + origGuardedFileStatus,
-          origGuardedFileStatus.getETag());
-      final DirListingMetadata dirListingMetadata =
-          realMs.listChildren(guardedFs.qualify(testDirPath));
-      assertListingAuthority(allowAuthoritative, dirListingMetadata);
-
-      // a brief pause to guarantee timestamps are different.
-      waitForDifferentTimestamps();
-
-      // Update file with second text without S3Guard (raw)
-      deleteFile(rawFS, testFilePath);
-
-      // write to the test path with the second text
-      writeTextFile(rawFS, testFilePath, secondText, true);
-      // and await it becoming visible again.
-      final FileStatus rawFileStatus = awaitFileStatus(rawFS, testFilePath);
-
-      // check listing in guarded store.
-      final S3AFileStatus[] modList = (S3AFileStatus[]) guardedFs.listStatus(
-          testDirPath);
-      assertArraySize("Added one file to the new dir then modified it, "
-          + "so the number of files in the dir should be one.", 1,
-          modList);
-      assertEquals("The only file path in the directory listing should be "
-              + "equal to the testFilePath.", testFilePath,
-          modList[0].getPath());
-
-      // Read the file and verify the data
-      eventually(STABILIZATION_TIME, PROBE_INTERVAL_MILLIS,
-          () -> {
-            final FileStatus guardedFileStatus =
-                guardedFs.getFileStatus(testFilePath);
-            verifyFileStatusAsExpected(firstText, secondText,
-                allowAuthoritative,
-                origStatus,
-                rawFileStatus,
-                guardedFileStatus);
-          });
-    } finally {
-      guardedFs.delete(testDirPath, true);
-    }
-  }
-
-  private void deleteFile(final S3AFileSystem fs, final Path testFilePath)
-      throws Exception {
-    fs.delete(testFilePath, true);
-    awaitDeletedFileDisappearance(fs, testFilePath);
-  }
-
-
-  /**
-   * Verify that the file status of a file which has been overwritten
-   * is as expected, throwing informative exceptions if not.
-   * @param firstText text of the first write
-   * @param secondText text of the second
-   * @param allowAuthoritative is S3Guard being authoritative
-   * @param origStatus filestatus of the first written file
-   * @param rawFileStatus status of the updated file from the raw FS
-   * @param guardedFileStatus status of the updated file from the guarded FS
-   */
-  private void verifyFileStatusAsExpected(final String firstText,
-      final String secondText,
-      final boolean allowAuthoritative,
-      final FileStatus origStatus,
-      final FileStatus rawFileStatus,
-      final FileStatus guardedFileStatus) {
-    String stats = "\nRaw: " + rawFileStatus.toString() +
-        "\nGuarded: " + guardedFileStatus.toString();
-    if (firstText.length() != secondText.length()) {
-      // the file lengths are different, so compare that first.
-      // it's not going to be brittle to timestamps, and easy to understand
-      // when there is an error.
-
-      // check the file length in the raw FS To verify that status is actually
-      // stabilized w.r.t the last write.
-      long expectedLength = secondText.length();
-      assertEquals("Length of raw file status did not match the updated text "
-              + rawFileStatus,
-          expectedLength, rawFileStatus.getLen());
-      // now compare the lengths of the the raw and guarded files
-      long guardedLength = guardedFileStatus.getLen();
-      if (allowAuthoritative) {
-        // expect the length to be out of sync
-        assertNotEquals(
-            "File length in authoritative table with " + stats,
-            expectedLength, guardedLength);
-      } else {
-        assertEquals(
-            "File length in non-authoritative table with " + stats,
-            expectedLength, guardedLength);
-      }
-    }
-    // check etag. This relies on first and second text being different.
-    final S3AFileStatus rawS3AFileStatus = (S3AFileStatus) rawFileStatus;
-    final S3AFileStatus guardedS3AFileStatus = (S3AFileStatus)
-        guardedFileStatus;
-    final S3AFileStatus origS3AFileStatus = (S3AFileStatus) origStatus;
-    assertNotEquals(
-        "raw status still no to date with changes" + stats,
-        origS3AFileStatus.getETag(), rawS3AFileStatus.getETag());
-    if (allowAuthoritative) {
-      // expect the etag to be out of sync
-      assertNotEquals(
-          "etag in authoritative table with " + stats,
-          rawS3AFileStatus.getETag(), guardedS3AFileStatus.getETag());
-    } else {
-      assertEquals(
-          "etag in non-authoritative table with " + stats,
-          rawS3AFileStatus.getETag(), guardedS3AFileStatus.getETag());
-    }
-    // Next: modification time.
-    long rawModTime = rawFileStatus.getModificationTime();
-    long guardedModTime = guardedFileStatus.getModificationTime();
-    assertNotEquals(
-        "Updated file still has original timestamp\n"
-            + " original " + origStatus + stats,
-        origStatus.getModificationTime(), rawModTime);
-    if (allowAuthoritative) {
-      // If authoritative is allowed metadata is not updated, so mod_time
-      // won't match
-      assertNotEquals("Authoritative is enabled, so metadata is not "
-              + "updated in ms, so mod_time won't match. Expecting "
-              + "different values for raw and guarded filestatus."
-              + stats,
-          rawModTime,
-          guardedModTime);
-    } else {
-      // If authoritative is not enabled metadata is updated, mod_time
-      // will match
-      assertEquals("Authoritative is disabled, so metadata is"
-              + " updated in ms, so mod_time must match. Expecting "
-              + " same values for raw and guarded filestatus."
-              + stats,
-          rawModTime,
-          guardedModTime);
-    }
-  }
-
-  /**
-   * A brief pause to guarantee timestamps are different.
-   * This doesn't have to be as long as a stabilization delay.
-   */
-  private void waitForDifferentTimestamps() throws InterruptedException {
-    Thread.sleep(TIMESTAMP_SLEEP);
-  }
-
-  /**
-   * Assert that a listing has the specific authority.
-   * @param expectAuthoritative expect authority bit of listing
-   * @param dirListingMetadata listing to check
-   */
-  private void assertListingAuthority(final boolean expectAuthoritative,
-      final DirListingMetadata dirListingMetadata) {
-    if (expectAuthoritative) {
-      assertTrue("DirListingMeta should be authoritative if authoritative "
-              + "mode is enabled.",
-          dirListingMetadata.isAuthoritative());
-    } else {
-      assertFalse("DirListingMeta should not be authoritative if "
-              + "authoritative mode is disabled.",
-          dirListingMetadata.isAuthoritative());
-    }
-  }
-
-  /**
-   * Delete a file and use listStatus to build up the S3Guard cache.
-   */
-  @Test
-  public void testListingDelete() throws Exception {
-
-    boolean allowAuthoritative = authoritative;
-    LOG.info("Authoritative mode enabled: {}", allowAuthoritative);
-    String rUUID = UUID.randomUUID().toString();
-    String testDir = "dir-" + rUUID + "/";
-    String testFile = testDir + "file-1-" + rUUID;
-    Path testDirPath = path(testDir);
-    Path testFilePath = guardedFs.qualify(path(testFile));
-    String text = "Some random text";
-
-    try {
-      // Create initial statusIterator with real ms
-      writeTextFile(
-          guardedFs, testFilePath, text, true);
-      awaitFileStatus(rawFS, testFilePath);
-
-      // Do a listing to cache the lists. Should be authoritative if it's set.
-      final FileStatus[] origList = guardedFs.listStatus(testDirPath);
-      assertEquals("Added one file to the new dir, so the number of "
-          + "files in the dir should be one.", 1, origList.length);
-      final DirListingMetadata dirListingMetadata =
-          realMs.listChildren(guardedFs.qualify(testDirPath));
-      assertListingAuthority(allowAuthoritative, dirListingMetadata);
-
-      // Delete the file without S3Guard (raw)
-      deleteFile(rawFS, testFilePath);
-
-      // now, versioned FS or not, it will not be readable from the
-      // raw FS, and this will fail in both open APIs during the open
-      // phase, rather than when a read is attempted.
-      interceptFuture(FileNotFoundException.class, "",
-          rawFS.openFile(testFilePath).build());
-      intercept(FileNotFoundException.class, () ->
-          rawFS.open(testFilePath).close());
-
-      // File status will be still readable from s3guard
-      S3AFileStatus status = (S3AFileStatus)
-          guardedFs.getFileStatus(testFilePath);
-      LOG.info("authoritative: {} status: {}", allowAuthoritative, status);
-      if (isVersionedChangeDetection() && status.getVersionId() != null) {
-        // when the status entry has a version ID, then that may be used
-        // when opening the file on what is clearly a versioned store.
-        int length = text.length();
-        byte[] bytes = readOpenFileAPI(guardedFs, testFilePath, length, null);
-        Assertions.assertThat(toChar(bytes))
-            .describedAs("openFile(%s)", testFilePath)
-            .isEqualTo(text);
-        // reading the rawFS with status will also work.
-        bytes = readOpenFileAPI(rawFS, testFilePath, length, status);
-        Assertions.assertThat(toChar(bytes))
-            .describedAs("openFile(%s)", testFilePath)
-            .isEqualTo(text);
-        bytes = readDataset(guardedFs, testFilePath, length);
-        Assertions.assertThat(toChar(bytes))
-            .describedAs("open(%s)", testFilePath)
-            .isEqualTo(text);
-      } else {
-        // unversioned sequence
-        expectExceptionWhenReading(testFilePath, text);
-        expectExceptionWhenReadingOpenFileAPI(testFilePath, text, null);
-        expectExceptionWhenReadingOpenFileAPI(testFilePath, text, status);
-      }
-    } finally {
-      guardedFs.delete(testDirPath, true);
-    }
-  }
-
-  private boolean isVersionedChangeDetection() {
-    return getFileSystem().getChangeDetectionPolicy().getSource()
-        == Source.VersionId;
-  }
-
-  /**
-   * We expect the read to fail with an FNFE: open will be happy.
-   * @param testFilePath path of the test file
-   * @param text the context in the file.
-   * @throws Exception failure other than the FNFE
-   */
-  private void expectExceptionWhenReading(Path testFilePath, String text)
-      throws Exception {
-    try (FSDataInputStream in = guardedFs.open(testFilePath)) {
-      intercept(FileNotFoundException.class, () -> {
-        byte[] bytes = new byte[text.length()];
-        return in.read(bytes, 0, bytes.length);
-      });
-    }
-  }
-
-  /**
-   * We expect the read to fail with an FNFE: open will be happy.
-   * @param testFilePath path of the test file
-   * @param text the context in the file.
-   * @param status optional status for the withFileStatus operation.
-   * @throws Exception failure other than the FNFE
-   */
-  private void expectExceptionWhenReadingOpenFileAPI(
-      Path testFilePath, String text, FileStatus status)
-      throws Exception {
-    expectExceptionWhenReadingOpenFileAPI(guardedFs,
-        testFilePath, text, status);
-  }
-
-  /**
-   * We expect the read to fail with an FNFE: open will be happy.
-   * @param fs filesystem
-   * @param testFilePath path of the test file
-   * @param text the context in the file.
-   * @param status optional status for the withFileStatus operation.
-   * @throws Exception failure other than the FNFE
-   */
-  private void expectExceptionWhenReadingOpenFileAPI(
-      final S3AFileSystem fs,
-      final Path testFilePath
-      , final String text,
-      final FileStatus status)
-      throws Exception {
-    final FutureDataInputStreamBuilder builder
-        = fs.openFile(testFilePath);
-    if (status != null) {
-      builder.withFileStatus(status);
-    }
-    try (FSDataInputStream in = builder.build().get()) {
-      intercept(FileNotFoundException.class, () -> {
-        byte[] bytes = new byte[text.length()];
-        return in.read(bytes, 0, bytes.length);
-      });
-    }
-  }
-
-  /**
-   * Open and read a file with the openFile API.
-   * @param fs FS to read from
-   * @param testFilePath path of the test file
-   * @param len data length to read
-   * @param status optional status for the withFileStatus operation.
-   * @throws Exception failure
-   * @return the data
-   */
-  private byte[] readOpenFileAPI(
-      S3AFileSystem fs,
-      Path testFilePath,
-      int len,
-      FileStatus status) throws Exception {
-    FutureDataInputStreamBuilder builder = fs.openFile(testFilePath);
-    if (status != null) {
-      builder.withFileStatus(status);
-    }
-    try (FSDataInputStream in = builder.build().get()) {
-      byte[] bytes = new byte[len];
-      in.readFully(0, bytes);
-      return bytes;
-    }
-  }
-
-  /**
-   * Wait for a deleted file to no longer be visible.
-   * @param fs filesystem
-   * @param testFilePath path to query
-   * @throws Exception failure
-   */
-  private void awaitDeletedFileDisappearance(final S3AFileSystem fs,
-      final Path testFilePath) throws Exception {
-    eventually(
-        STABILIZATION_TIME, PROBE_INTERVAL_MILLIS,
-        () -> intercept(FileNotFoundException.class,
-            () -> fs.getFileStatus(testFilePath)));
-  }
-
-  /**
-   * Wait for a file to be visible.
-   * @param fs filesystem
-   * @param testFilePath path to query
-   * @return the file status.
-   * @throws Exception failure
-   */
-  private S3AFileStatus awaitFileStatus(S3AFileSystem fs,
-      final Path testFilePath)
-      throws Exception {
-    return (S3AFileStatus) eventually(
-        STABILIZATION_TIME, PROBE_INTERVAL_MILLIS,
-        () -> fs.getFileStatus(testFilePath));
-  }
-
-  @Test
-  public void testDeleteIgnoresTombstones() throws Throwable {
-    describe("Verify that directory delete goes behind tombstones");
-    Path dir = path("oobdir");
-    Path testFilePath = new Path(dir, "file");
-    // create a file under the store
-    createAndAwaitFs(guardedFs, testFilePath, "Original File is long");
-    // Delete the file leaving a tombstone in the metastore
-    LOG.info("Initial delete of guarded FS dir {}", dir);
-    guardedFs.delete(dir, true);
-//    deleteFile(guardedFs, testFilePath);
-    awaitDeletedFileDisappearance(guardedFs, testFilePath);
-    // add a new file in raw
-    createAndAwaitFs(rawFS, testFilePath, "hi!");
-    // now we need to be sure that the file appears in a listing
-    awaitListingContainsChild(rawFS, dir, testFilePath);
-
-    // now a hack to remove the empty dir up the tree
-    Path sibling = new Path(dir, "sibling");
-    guardedFs.mkdirs(sibling);
-    // now do a delete of the parent dir. This is required to also
-    // check the underlying fs.
-    LOG.info("Deleting guarded FS dir {} with OOB child", dir);
-    guardedFs.delete(dir, true);
-    LOG.info("Now waiting for child to be deleted in raw fs: {}", testFilePath);
-
-    // so eventually the file will go away.
-    // this is required to be true in auth as well as non-auth.
-
-    awaitDeletedFileDisappearance(rawFS, testFilePath);
-  }
-
-  /**
-   * Wait for a file to be visible.
-   * @param fs filesystem
-   * @param testFilePath path to query
-   * @throws Exception failure
-   */
-  private void awaitListingContainsChild(S3AFileSystem fs,
-      final Path dir,
-      final Path testFilePath)
-      throws Exception {
-    LOG.info("Awaiting list of {} to include {}", dir, testFilePath);
-    eventually(
-        STABILIZATION_TIME, PROBE_INTERVAL_MILLIS,
-        () -> {
-          FileStatus[] stats = fs.listStatus(dir);
-          Assertions.assertThat(stats)
-              .describedAs("listing of %s", dir)
-              .filteredOn(s -> s.getPath().equals(testFilePath))
-              .isNotEmpty();
-          return null;
-        });
-  }
-
-  private FSDataOutputStream createNonRecursive(FileSystem fs, Path path)
-      throws Exception {
-    return fs
-        .createNonRecursive(path, false, 4096, (short) 3, (short) 4096, null);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java
deleted file mode 100644
index 4143283d71fb8..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-
-import org.assertj.core.api.Assertions;
-import org.junit.Assume;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.isMetadataStoreAuthoritative;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * These tests are testing the S3Guard TTL (time to live) features.
- */
-@RunWith(Parameterized.class)
-public class ITestS3GuardTtl extends AbstractS3ATestBase {
-
-  private final boolean authoritative;
-
-  /**
-   * Test array for parameterized test runs.
-   * @return a list of parameter tuples.
-   */
-  @Parameterized.Parameters(name = "auth={0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {true}, {false}
-    });
-  }
-
-  /**
-   * By changing the method name, the thread name is changed and
-   * so you can see in the logs which mode is being tested.
-   * @return a string to use for the thread namer.
-   */
-  @Override
-  protected String getMethodName() {
-    return super.getMethodName() +
-        (authoritative ? "-auth" : "-nonauth");
-  }
-
-  public ITestS3GuardTtl(boolean authoritative) {
-    this.authoritative = authoritative;
-  }
-
-  /**
-   * Patch the configuration - this test needs disabled filesystem caching.
-   * These tests modify the fs instance that would cause flaky tests.
-   * @return a configuration
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration configuration = super.createConfiguration();
-    S3ATestUtils.disableFilesystemCaching(configuration);
-    configuration =
-        S3ATestUtils.prepareTestConfiguration(configuration);
-    configuration.setBoolean(METADATASTORE_AUTHORITATIVE, authoritative);
-    return configuration;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    Assume.assumeTrue(getFileSystem().hasMetadataStore());
-  }
-
-  @Test
-  public void testDirectoryListingAuthoritativeTtl() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-
-    final S3AFileSystem fs = getFileSystem();
-    Assume.assumeTrue(fs.hasMetadataStore());
-    final MetadataStore ms = fs.getMetadataStore();
-
-    Assume.assumeTrue("MetadataStore should be capable for authoritative "
-            + "storage of directories to run this test.",
-        metadataStorePersistsAuthoritativeBit(ms));
-
-    Assume.assumeTrue("MetadataStore should be authoritative for this test",
-        isMetadataStoreAuthoritative(getFileSystem().getConf()));
-
-    ITtlTimeProvider mockTimeProvider =
-        mock(ITtlTimeProvider.class);
-    ITtlTimeProvider restoreTimeProvider = fs.getTtlTimeProvider();
-    fs.setTtlTimeProvider(mockTimeProvider);
-    when(mockTimeProvider.getNow()).thenReturn(100L);
-    when(mockTimeProvider.getMetadataTtl()).thenReturn(1L);
-
-    Path dir = path("ttl/");
-    Path file = path("ttl/afile");
-
-    try {
-      fs.mkdirs(dir);
-      touch(fs, file);
-
-      // get an authoritative listing in ms
-      fs.listStatus(dir);
-
-      // check if authoritative
-      DirListingMetadata dirListing =
-          S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider, authoritative);
-      assertTrue("Listing should be authoritative.",
-          dirListing.isAuthoritative());
-      // change the time, and assume it's not authoritative anymore
-      // if the metadatastore is not authoritative.
-      when(mockTimeProvider.getNow()).thenReturn(102L);
-      dirListing = S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider,
-          authoritative);
-      if (authoritative) {
-        assertTrue("Listing should be authoritative.",
-            dirListing.isAuthoritative());
-      } else {
-        assertFalse("Listing should not be authoritative.",
-            dirListing.isAuthoritative());
-      }
-
-      // get an authoritative listing in ms again - retain test
-      fs.listStatus(dir);
-      // check if authoritative
-      dirListing = S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider,
-          authoritative);
-      assertTrue("Listing shoud be authoritative after listStatus.",
-          dirListing.isAuthoritative());
-    } finally {
-      fs.delete(dir, true);
-      fs.setTtlTimeProvider(restoreTimeProvider);
-    }
-  }
-
-  @Test
-  public void testFileMetadataExpiresTtl() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-
-    Path fileExpire1 = path("expirettl-" + UUID.randomUUID());
-    Path fileExpire2 = path("expirettl-" + UUID.randomUUID());
-    Path fileRetain = path("expirettl-" + UUID.randomUUID());
-
-    final S3AFileSystem fs = getFileSystem();
-    Assume.assumeTrue(fs.hasMetadataStore());
-    final MetadataStore ms = fs.getMetadataStore();
-
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = fs.getTtlTimeProvider();
-
-    try {
-      fs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(5L);
-
-      // set the time, so the fileExpire1 will expire
-      when(mockTimeProvider.getNow()).thenReturn(100L);
-      touch(fs, fileExpire1);
-      // set the time, so fileExpire2 will expire
-      when(mockTimeProvider.getNow()).thenReturn(101L);
-      touch(fs, fileExpire2);
-      // set the time, so fileRetain won't expire
-      when(mockTimeProvider.getNow()).thenReturn(109L);
-      touch(fs, fileRetain);
-      final FileStatus origFileRetainStatus = fs.getFileStatus(fileRetain);
-      // change time, so the first two file metadata is expired
-      when(mockTimeProvider.getNow()).thenReturn(110L);
-
-      // metadata is expired so this should refresh the metadata with
-      // last_updated to the getNow() if the store is not authoritative
-      final FileStatus fileExpire1Status = fs.getFileStatus(fileExpire1);
-      assertNotNull(fileExpire1Status);
-      if (authoritative) {
-        assertEquals(100L, ms.get(fileExpire1).getLastUpdated());
-      } else {
-        assertEquals(110L, ms.get(fileExpire1).getLastUpdated());
-      }
-
-      // metadata is expired so this should refresh the metadata with
-      // last_updated to the getNow() if the store is not authoritative
-      final FileStatus fileExpire2Status = fs.getFileStatus(fileExpire2);
-      assertNotNull(fileExpire2Status);
-      if (authoritative) {
-        assertEquals(101L, ms.get(fileExpire2).getLastUpdated());
-      } else {
-        assertEquals(110L, ms.get(fileExpire2).getLastUpdated());
-      }
-
-      final FileStatus fileRetainStatus = fs.getFileStatus(fileRetain);
-      assertEquals("Modification time of these files should be equal.",
-          origFileRetainStatus.getModificationTime(),
-          fileRetainStatus.getModificationTime());
-      assertNotNull(fileRetainStatus);
-      assertEquals(109L, ms.get(fileRetain).getLastUpdated());
-    } finally {
-      fs.delete(fileExpire1, true);
-      fs.delete(fileExpire2, true);
-      fs.delete(fileRetain, true);
-      fs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * create(tombstone file) must succeed irrespective of overwrite flag.
-   */
-  @Test
-  public void testCreateOnTombstonedFileSucceeds() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-    final S3AFileSystem fs = getFileSystem();
-
-    String fileToTry = methodName + UUID.randomUUID().toString();
-
-    final Path filePath = path(fileToTry);
-
-    // Create a directory with
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = fs.getTtlTimeProvider();
-
-    try {
-      fs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getNow()).thenReturn(100L);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(5L);
-
-      // CREATE A FILE
-      touch(fs, filePath);
-
-      // DELETE THE FILE - TOMBSTONE
-      fs.delete(filePath, true);
-
-      // CREATE THE SAME FILE WITHOUT ERROR DESPITE THE TOMBSTONE
-      touch(fs, filePath);
-
-    } finally {
-      fs.delete(filePath, true);
-      fs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * create("parent has tombstone") must always succeed (We dont check the
-   * parent), but after the file has been written, all entries up the tree
-   * must be valid. That is: the putAncestor code will correct everything
-   */
-  @Test
-  public void testCreateParentHasTombstone() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-    final S3AFileSystem fs = getFileSystem();
-
-    String dirToDelete = methodName + UUID.randomUUID().toString();
-    String fileToTry = dirToDelete + "/theFileToTry";
-
-    final Path dirPath = path(dirToDelete);
-    final Path filePath = path(fileToTry);
-
-    // Create a directory with
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = fs.getTtlTimeProvider();
-
-    try {
-      fs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getNow()).thenReturn(100L);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(5L);
-
-      // CREATE DIRECTORY
-      fs.mkdirs(dirPath);
-
-      // DELETE DIRECTORY
-      fs.delete(dirPath, true);
-
-      // WRITE TO DELETED DIRECTORY - SUCCESS
-      touch(fs, filePath);
-
-      // SET TIME SO METADATA EXPIRES
-      when(mockTimeProvider.getNow()).thenReturn(110L);
-
-      // WRITE TO DELETED DIRECTORY - SUCCESS
-      touch(fs, filePath);
-
-    } finally {
-      fs.delete(filePath, true);
-      fs.delete(dirPath, true);
-      fs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  /**
-   * Test that listing of metadatas is filtered from expired items.
-   */
-  @Test
-  public void testListingFilteredExpiredItems() throws Exception {
-    LOG.info("Authoritative mode: {}", authoritative);
-    final S3AFileSystem fs = getFileSystem();
-
-    long oldTime = 100L;
-    long newTime = 110L;
-    long ttl = 9L;
-    final String basedir = "testListingFilteredExpiredItems";
-    final Path tombstonedPath = path(basedir + "/tombstonedPath");
-    final Path baseDirPath = path(basedir);
-    final List<Path> filesToCreate = new ArrayList<>();
-    final MetadataStore ms = fs.getMetadataStore();
-
-    for (int i = 0; i < 10; i++) {
-      filesToCreate.add(path(basedir + "/file" + i));
-    }
-
-    ITtlTimeProvider mockTimeProvider = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = fs.getTtlTimeProvider();
-
-    try {
-      fs.setTtlTimeProvider(mockTimeProvider);
-      when(mockTimeProvider.getMetadataTtl()).thenReturn(ttl);
-
-      // add and delete entry with the oldtime
-      when(mockTimeProvider.getNow()).thenReturn(oldTime);
-      touch(fs, tombstonedPath);
-      fs.delete(tombstonedPath, false);
-
-      // create items with newTime
-      when(mockTimeProvider.getNow()).thenReturn(newTime);
-      for (Path path : filesToCreate) {
-        touch(fs, path);
-      }
-
-      // listing will contain the tombstone with oldtime
-      when(mockTimeProvider.getNow()).thenReturn(oldTime);
-      final DirListingMetadata fullDLM = getDirListingMetadata(ms, baseDirPath);
-      List<Path> containedPaths = fullDLM.getListing().stream()
-          .map(pm -> pm.getFileStatus().getPath())
-          .collect(Collectors.toList());
-      Assertions.assertThat(containedPaths)
-          .describedAs("Full listing of path %s", baseDirPath)
-          .hasSize(11)
-          .contains(tombstonedPath);
-
-      // listing will be filtered if the store is not authoritative,
-      // and won't contain the tombstone with oldtime
-      when(mockTimeProvider.getNow()).thenReturn(newTime);
-      final DirListingMetadata filteredDLM = S3Guard.listChildrenWithTtl(
-          ms, baseDirPath, mockTimeProvider, authoritative);
-      containedPaths = filteredDLM.getListing().stream()
-          .map(pm -> pm.getFileStatus().getPath())
-          .collect(Collectors.toList());
-      if (authoritative) {
-        Assertions.assertThat(containedPaths)
-            .describedAs("Full listing of path %s", baseDirPath)
-            .hasSize(11)
-            .contains(tombstonedPath);
-      } else {
-        Assertions.assertThat(containedPaths)
-            .describedAs("Full listing of path %s", baseDirPath)
-            .hasSize(10)
-            .doesNotContain(tombstonedPath);
-      }
-    } finally {
-      fs.delete(baseDirPath, true);
-      fs.setTtlTimeProvider(originalTimeProvider);
-    }
-  }
-
-  protected DirListingMetadata getDirListingMetadata(final MetadataStore ms,
-      final Path baseDirPath) throws IOException {
-    final DirListingMetadata fullDLM = ms.listChildren(baseDirPath);
-    Assertions.assertThat(fullDLM)
-        .describedAs("Metastrore directory listing of %s",
-            baseDirPath)
-        .isNotNull();
-    return fullDLM;
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardWriteBack.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardWriteBack.java
deleted file mode 100644
index c5e4cbcd45929..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardWriteBack.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.junit.Assume.assumeTrue;
-
-/**
- * Test cases that validate S3Guard's behavior for writing things like
- * directory listings back to the MetadataStore.
- */
-public class ITestS3GuardWriteBack extends AbstractS3ATestBase {
-
-  @Override
-  public void setup() throws Exception {
-    assumeTrue("dirListingUnion always writes back records",
-        !S3Guard.DIR_MERGE_UPDATES_ALL_RECORDS_NONAUTH);
-    super.setup();
-  }
-
-  /**
-   * In listStatus(), when S3Guard is enabled, the full listing for a
-   * directory is "written back" to the MetadataStore before the listing is
-   * returned.  Currently this "write back" behavior occurs when
-   * fs.s3a.metadatastore.authoritative is true.  This test validates this
-   * behavior.
-   * @throws Exception on failure
-   */
-  @Test
-  public void testListStatusWriteBack() throws Exception {
-    assumeTrue(getFileSystem().hasMetadataStore());
-
-    Path directory = path("ListStatusWriteBack");
-
-    // "raw" S3AFileSystem without S3Guard
-    S3AFileSystem noS3Guard = createTestFS(directory.toUri(), true, false);
-
-    // Another with S3Guard and write-back disabled
-    S3AFileSystem noWriteBack = createTestFS(directory.toUri(), false, false);
-
-    // Another S3Guard and write-back enabled
-    S3AFileSystem yesWriteBack = createTestFS(directory.toUri(), false, true);
-
-    // delete the existing directory (in case of last test failure)
-    noS3Guard.delete(directory, true);
-    // Create a directory on S3 only
-    Path onS3 = new Path(directory, "OnS3");
-    noS3Guard.mkdirs(onS3);
-    // Create a directory on both S3 and metadata store
-    Path onS3AndMS = new Path(directory, "OnS3AndMS");
-    ContractTestUtils.assertPathDoesNotExist(noWriteBack, "path", onS3AndMS);
-    noWriteBack.mkdirs(onS3AndMS);
-
-    FileStatus[] fsResults;
-    DirListingMetadata mdResults;
-
-    // FS should return both even though S3Guard is not writing back to MS
-    fsResults = noWriteBack.listStatus(directory);
-    assertEquals("Filesystem enabled S3Guard without write back should have "
-            + "both /OnS3 and /OnS3AndMS: " + Arrays.toString(fsResults),
-        2, fsResults.length);
-
-    // Metadata store without write-back should still only contain /OnS3AndMS,
-    // because newly discovered /OnS3 is not written back to metadata store
-    mdResults = noWriteBack.getMetadataStore().listChildren(directory);
-    assertNotNull("No results from noWriteBack listChildren " + directory,
-        mdResults);
-    assertEquals("Metadata store without write back should still only know "
-            + "about /OnS3AndMS, but it has: " + mdResults,
-        1, mdResults.numEntries());
-
-    // FS should return both (and will write it back)
-    fsResults = yesWriteBack.listStatus(directory);
-    assertEquals("Filesystem enabled S3Guard with write back should have"
-            + " both /OnS3 and /OnS3AndMS: " + Arrays.toString(fsResults),
-        2, fsResults.length);
-
-    // Metadata store with write-back should contain both because the newly
-    // discovered /OnS3 should have been written back to metadata store
-    mdResults = yesWriteBack.getMetadataStore().listChildren(directory);
-    assertEquals("Unexpected number of results from metadata store. "
-            + "Should have /OnS3 and /OnS3AndMS: " + mdResults,
-        2, mdResults.numEntries());
-
-    // If we don't clean this up, the next test run will fail because it will
-    // have recorded /OnS3 being deleted even after it's written to noS3Guard.
-    getFileSystem().getMetadataStore().forgetMetadata(onS3);
-  }
-
-  /**
-   * Create a separate S3AFileSystem instance for testing.
-   * There's a bit of complexity as it forces pushes up s3guard options from
-   * the base values to the per-bucket options. This stops explicit bucket
-   * settings in test XML configs from unintentionally breaking tests.
-   */
-  private S3AFileSystem createTestFS(URI fsURI, boolean disableS3Guard,
-      boolean authoritativeMeta) throws IOException {
-    Configuration conf;
-
-    // Create a FileSystem that is S3-backed only
-    conf = createConfiguration();
-    String host = fsURI.getHost();
-    String metastore;
-
-    metastore = S3GUARD_METASTORE_NULL;
-    if (!disableS3Guard) {
-      // pick up the metadata store used by the main test
-      metastore = getFileSystem().getConf().get(S3_METADATA_STORE_IMPL);
-      assertNotEquals(S3GUARD_METASTORE_NULL, metastore);
-    }
-
-    conf.set(Constants.S3_METADATA_STORE_IMPL, metastore);
-    conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritativeMeta);
-    conf.unset(AUTHORITATIVE_PATH);
-    S3AUtils.setBucketOption(conf, host,
-        METADATASTORE_AUTHORITATIVE,
-        Boolean.toString(authoritativeMeta));
-    S3AUtils.setBucketOption(conf, host,
-        S3_METADATA_STORE_IMPL, metastore);
-
-    S3AFileSystem fs = asS3AFS(FileSystem.newInstance(fsURI, conf));
-    // do a check to verify that everything got through
-    assertEquals("Metadata store should have been disabled: " + fs,
-        disableS3Guard, !fs.hasMetadataStore());
-    assertEquals("metastore option did not propagate",
-        metastore, fs.getConf().get(S3_METADATA_STORE_IMPL));
-
-    return fs;
-
-  }
-
-  private static S3AFileSystem asS3AFS(FileSystem fs) {
-    assertTrue("Not a S3AFileSystem: " + fs, fs instanceof S3AFileSystem);
-    return (S3AFileSystem)fs;
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
index 624bfd6c94501..a859cd534bb02 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
@@ -40,16 +40,20 @@
 import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
 import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl;
+import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
+import org.apache.hadoop.fs.s3a.impl.StubContextAccessor;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
 import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
+import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks;
 import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
 import org.apache.hadoop.util.Progressable;
 
 import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.noopAuditor;
 import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTrackerFactory;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 /**
  * Relays FS calls to the mocked FS, allows for some extra logging with
@@ -173,7 +177,8 @@ public void initialize(URI name, Configuration originalConf)
         conf,
         new EmptyS3AStatisticsContext(),
         noopAuditor(conf),
-        AuditTestSupport.NOOP_SPAN);
+        AuditTestSupport.NOOP_SPAN,
+        new MinimalWriteOperationHelperCallbacks());
   }
 
   @Override
@@ -212,8 +217,11 @@ public boolean exists(Path f) throws IOException {
   }
 
   @Override
-  void finishedWrite(String key, long length, String eTag, String versionId,
-          BulkOperationState operationState) {
+  void finishedWrite(String key,
+      long length,
+      String eTag,
+      String versionId,
+      final PutObjectOptions putOptions) {
 
   }
 
@@ -339,8 +347,7 @@ public long getDefaultBlockSize() {
   @Override
   void deleteObjectAtPath(Path f,
       String key,
-      boolean isFile,
-      final BulkOperationState operationState)
+      boolean isFile)
       throws AmazonClientException, IOException {
     deleteObject(key);
   }
@@ -380,11 +387,29 @@ public CommitterStatistics newCommitterStatistics() {
 
   @Override
   public void operationRetried(Exception ex) {
-    /** no-op */
+    /* no-op */
   }
 
   @Override
   protected DurationTrackerFactory getDurationTrackerFactory() {
     return stubDurationTrackerFactory();
   }
+
+  /**
+   * Build an immutable store context.
+   * If called while the FS is being initialized,
+   * some of the context will be incomplete.
+   * new store context instances should be created as appropriate.
+   * @return the store context of this FS.
+   */
+  public StoreContext createStoreContext() {
+    return new StoreContextBuilder().setFsURI(getUri())
+        .setBucket(getBucket())
+        .setConfiguration(getConf())
+        .setUsername(getUsername())
+        .setAuditor(getAuditor())
+        .setContextAccessors(new StubContextAccessor(getBucket()))
+        .build();
+  }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
index bd121ba2728eb..3240309aef971 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
@@ -31,6 +31,7 @@
  * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3}
  * interface suitable for unit testing.
  */
+@SuppressWarnings("deprecation")
 public class MockS3ClientFactory implements S3ClientFactory {
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
index 04c2b2a09bda2..9ea33cf69c92b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java
@@ -22,6 +22,7 @@
 import com.amazonaws.services.s3.model.PartETag;
 import com.amazonaws.services.s3.model.UploadPartRequest;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.io.IOUtils;
 
@@ -78,7 +79,7 @@ public static IdKey createPartUpload(S3AFileSystem fs, String key, int len,
       WriteOperationHelper writeHelper = fs.getWriteOperationHelper();
       byte[] data = dataset(len, 'a', 'z');
       InputStream in = new ByteArrayInputStream(data);
-      String uploadId = writeHelper.initiateMultiPartUpload(key);
+      String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.keepingDirs());
       UploadPartRequest req = writeHelper.newUploadPartRequest(key, uploadId,
           partNo, len, in, null, 0L);
       PartETag partEtag = writeHelper.uploadPart(req).getPartETag();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 945d3f01c0569..a6269c437665a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -20,6 +20,9 @@
 
 import java.time.Duration;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils;
+
 /**
  * Constants for S3A Testing.
  */
@@ -50,6 +53,11 @@ public interface S3ATestConstants {
    */
   String KEY_ENCRYPTION_TESTS = TEST_FS_S3A + "encryption.enabled";
 
+  /**
+   * A property set to true if storage class tests are enabled: {@value }.
+   */
+  String KEY_STORAGE_CLASS_TESTS_ENABLED = TEST_FS_S3A + "create.storage.class.enabled";
+
   /**
    * Tell tests that they are being executed in parallel: {@value}.
    */
@@ -97,6 +105,22 @@ public interface S3ATestConstants {
    */
   String DEFAULT_CSVTEST_FILE = LANDSAT_BUCKET + "scene_list.gz";
 
+  /**
+   * Configuration key for an existing object in a requester pays bucket: {@value}.
+   *
+   * Accessible via
+   * {@link PublicDatasetTestUtils#getRequesterPaysObject(Configuration)}.
+   */
+  String KEY_REQUESTER_PAYS_FILE = TEST_FS_S3A + "requester.pays.file";
+
+  /**
+   * Configuration key for an existing bucket with many objects: {@value}.
+   *
+   * This is used for tests depending on buckets with a large number of keys.
+   */
+  String KEY_BUCKET_WITH_MANY_OBJECTS
+      = TEST_FS_S3A + "bucket-with-many-objects";
+
   /**
    * Name of the property to define the timeout for scale tests: {@value}.
    * Measured in seconds.
@@ -163,21 +187,6 @@ public interface S3ATestConstants {
   @Deprecated
   String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
 
-  /**
-   * Various S3Guard tests.
-   */
-  String TEST_S3GUARD_PREFIX = "fs.s3a.s3guard.test";
-  String TEST_S3GUARD_ENABLED = TEST_S3GUARD_PREFIX + ".enabled";
-  String TEST_S3GUARD_AUTHORITATIVE = TEST_S3GUARD_PREFIX + ".authoritative";
-  String TEST_S3GUARD_IMPLEMENTATION = TEST_S3GUARD_PREFIX + ".implementation";
-  String TEST_S3GUARD_IMPLEMENTATION_LOCAL = "local";
-  String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo";
-  String TEST_S3GUARD_IMPLEMENTATION_NONE = "none";
-
-  String TEST_S3GUARD_DYNAMO_TABLE_PREFIX =
-      "fs.s3a.s3guard.test.dynamo.table.prefix";
-  String TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT = "s3guard.test.";
-
   /**
    * ACL for S3 Logging; used in some tests: {@value}.
    */
@@ -218,17 +227,6 @@ public interface S3ATestConstants {
   Duration TEST_SESSION_TOKEN_DURATION = Duration.ofSeconds(
       TEST_SESSION_TOKEN_DURATION_SECONDS);
 
-  /**
-   * Test table name to use during DynamoDB integration tests in
-   * {@code ITestDynamoDBMetadataStore}.
-   *
-   * The table will be modified, and deleted in the end of the tests.
-   * If this value is not set, the integration tests that would be destructive
-   * won't run.
-   */
-  String S3GUARD_DDB_TEST_TABLE_NAME_KEY =
-      "fs.s3a.s3guard.ddb.test.table";
-
   /**
    * Test option to enable audits of the method path after
    * every test case.
@@ -247,4 +245,10 @@ public interface S3ATestConstants {
    * used.
    */
   int KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN = 94;
+
+  /**
+   * Build directory property.
+   * Value: {@value}.
+   */
+  String PROJECT_BUILD_DIRECTORY_PROPERTY = "project.build.directory";
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 5897f3aa44b24..469562f9b33b9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -35,19 +35,15 @@
 import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding;
 import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-
 import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
 import org.apache.hadoop.fs.s3a.impl.ContextAccessors;
 import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
 import org.apache.hadoop.fs.s3a.impl.StoreContext;
 import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
+import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
 import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreCapabilities;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-import org.apache.hadoop.fs.s3a.test.OperationTrackingStore;
 import org.apache.hadoop.fs.s3native.S3xLoginHelper;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -56,13 +52,17 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
 import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
+import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.functional.CallableRaisingIOE;
+import org.apache.hadoop.util.functional.FutureIO;
 
 import com.amazonaws.auth.AWSCredentialsProvider;
 import org.assertj.core.api.Assertions;
-import org.hamcrest.core.Is;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.slf4j.Logger;
@@ -71,12 +71,13 @@
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.Callable;
@@ -84,17 +85,17 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
+import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
+import static org.apache.hadoop.test.GenericTestUtils.buildPaths;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH;
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
-import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture;
-import static org.apache.hadoop.fs.s3a.FailureInjectionPolicy.*;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3AUtils.buildEncryptionSecrets;
-import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
-import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assert.*;
 
@@ -104,8 +105,23 @@
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public final class S3ATestUtils {
+
   private static final Logger LOG = LoggerFactory.getLogger(
-      S3ATestUtils.class);
+          S3ATestUtils.class);
+
+  /** Many threads for scale performance: {@value}. */
+  public static final int EXECUTOR_THREAD_COUNT = 64;
+  /**
+   * For submitting work.
+   */
+  private static final ListeningExecutorService EXECUTOR =
+      MoreExecutors.listeningDecorator(
+          BlockingThreadPoolExecutorService.newInstance(
+              EXECUTOR_THREAD_COUNT,
+              EXECUTOR_THREAD_COUNT * 2,
+              30, TimeUnit.SECONDS,
+              "test-operations"));
+
 
   /**
    * Value to set a system property to (in maven) to declare that
@@ -114,10 +130,6 @@ public final class S3ATestUtils {
   public static final String UNSET_PROPERTY = "unset";
   public static final int PURGE_DELAY_SECONDS = 60 * 60;
 
-  public static final int TIMESTAMP_SLEEP = 2000;
-  public static final int STABILIZATION_TIME = 20_000;
-  public static final int PROBE_INTERVAL_MILLIS = 500;
-
   /** Add any deprecated keys. */
   @SuppressWarnings("deprecation")
   private static void addDeprecatedKeys() {
@@ -191,10 +203,7 @@ public static S3AFileSystem createTestFileSystem(Configuration conf,
     // make this whole class not run by default
     Assume.assumeTrue("No test filesystem in " + TEST_FS_S3A_NAME,
         liveTest);
-    // Skip if S3Guard and S3-CSE are enabled.
-    skipIfS3GuardAndS3CSEEnabled(conf);
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
+
     S3AFileSystem fs1 = new S3AFileSystem();
     //enable purging in tests
     if (purge) {
@@ -236,30 +245,10 @@ public static FileContext createTestFileContext(Configuration conf)
     // make this whole class not run by default
     Assume.assumeTrue("No test filesystem in " + TEST_FS_S3A_NAME,
         liveTest);
-    // Skip if S3Guard and S3-CSE are enabled.
-    skipIfS3GuardAndS3CSEEnabled(conf);
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
     FileContext fc = FileContext.getFileContext(testURI, conf);
     return fc;
   }
 
-  /**
-   * Skip if S3Guard and S3CSE are enabled together.
-   *
-   * @param conf Test Configuration.
-   */
-  private static void skipIfS3GuardAndS3CSEEnabled(Configuration conf)
-      throws IOException {
-    String encryptionMethod = getEncryptionAlgorithm(getTestBucketName(conf),
-        conf).getMethod();
-    String metaStore = conf.getTrimmed(S3_METADATA_STORE_IMPL, "");
-    if (encryptionMethod.equals(S3AEncryptionMethods.CSE_KMS.getMethod()) &&
-        !metaStore.equals(S3GUARD_METASTORE_NULL)) {
-      skip("Skipped if CSE is enabled with S3Guard.");
-    }
-  }
-
   /**
    * Skip if PathIOE occurred due to exception which contains a message which signals
    * an incompatibility or throw the PathIOE.
@@ -477,6 +466,17 @@ public static void skipIfEncryptionTestsDisabled(
     }
   }
 
+  /**
+   * Skip a test if storage class tests are disabled.
+   * @param configuration configuration to probe
+   */
+  public static void skipIfStorageClassTestsDisabled(
+      Configuration configuration) {
+    if (!configuration.getBoolean(KEY_STORAGE_CLASS_TESTS_ENABLED, true)) {
+      skip("Skipping storage class tests");
+    }
+  }
+
   /**
    * Create a test path, using the value of
    * {@link S3ATestConstants#TEST_UNIQUE_FORK_ID} if it is set.
@@ -490,105 +490,6 @@ public static Path createTestPath(Path defVal) {
         new Path("/" + testUniqueForkId, "test");
   }
 
-  /**
-   * Test assumption that S3Guard is/is not enabled.
-   * @param shouldBeEnabled should S3Guard be enabled?
-   * @param originalConf configuration to check
-   * @throws URISyntaxException
-   */
-  public static void assumeS3GuardState(boolean shouldBeEnabled,
-      Configuration originalConf) throws URISyntaxException {
-    boolean isEnabled = isS3GuardTestPropertySet(originalConf);
-    Assume.assumeThat("Unexpected S3Guard test state:"
-            + " shouldBeEnabled=" + shouldBeEnabled
-            + " and isEnabled=" + isEnabled,
-        shouldBeEnabled, Is.is(isEnabled));
-
-    final String fsname = originalConf.getTrimmed(TEST_FS_S3A_NAME);
-    Assume.assumeNotNull(fsname);
-    final String bucket = new URI(fsname).getHost();
-    final Configuration conf = propagateBucketOptions(originalConf, bucket);
-    boolean usingNullImpl = S3GUARD_METASTORE_NULL.equals(
-        conf.getTrimmed(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL));
-    Assume.assumeThat("Unexpected S3Guard test state:"
-            + " shouldBeEnabled=" + shouldBeEnabled
-            + " but usingNullImpl=" + usingNullImpl,
-        shouldBeEnabled, Is.is(!usingNullImpl));
-  }
-
-  /**
-   * Is the test option for S3Guard set?
-   * @param conf configuration to examine.
-   * @return true if the config or system property turns s3guard tests on
-   */
-  public static boolean isS3GuardTestPropertySet(final Configuration conf) {
-    return getTestPropertyBool(conf, TEST_S3GUARD_ENABLED,
-        conf.getBoolean(TEST_S3GUARD_ENABLED, false));
-  }
-
-  /**
-   * Conditionally set the S3Guard options from test properties.
-   * @param conf configuration
-   */
-  public static void maybeEnableS3Guard(Configuration conf) {
-    if (isS3GuardTestPropertySet(conf)) {
-      // S3Guard is enabled.
-      boolean authoritative = getTestPropertyBool(conf,
-          TEST_S3GUARD_AUTHORITATIVE,
-          conf.getBoolean(TEST_S3GUARD_AUTHORITATIVE, false));
-      String impl = getTestProperty(conf, TEST_S3GUARD_IMPLEMENTATION,
-          conf.get(TEST_S3GUARD_IMPLEMENTATION,
-              TEST_S3GUARD_IMPLEMENTATION_LOCAL));
-      String implClass = "";
-      switch (impl) {
-      case TEST_S3GUARD_IMPLEMENTATION_LOCAL:
-        implClass = S3GUARD_METASTORE_LOCAL;
-        break;
-      case TEST_S3GUARD_IMPLEMENTATION_DYNAMO:
-        implClass = S3GUARD_METASTORE_DYNAMO;
-        break;
-      case TEST_S3GUARD_IMPLEMENTATION_NONE:
-        implClass = S3GUARD_METASTORE_NULL;
-        break;
-      default:
-        fail("Unknown s3guard back end: \"" + impl + "\"");
-      }
-      LOG.debug("Enabling S3Guard, authoritative={}, implementation={}",
-          authoritative, implClass);
-      conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritative);
-      conf.set(AUTHORITATIVE_PATH, "");
-      conf.set(S3_METADATA_STORE_IMPL, implClass);
-      conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
-    }
-  }
-
-  /**
-   * Is there a MetadataStore configured for s3a with authoritative enabled?
-   * @param conf Configuration to test.
-   * @return true iff there is a MetadataStore configured, and it is
-   * configured allow authoritative results.  This can result in reducing
-   * round trips to S3 service for cached results, which may affect FS/FC
-   * statistics.
-   */
-  public static boolean isMetadataStoreAuthoritative(Configuration conf) {
-    if (conf == null) {
-      return Constants.DEFAULT_METADATASTORE_AUTHORITATIVE;
-    }
-    return conf.getBoolean(
-        Constants.METADATASTORE_AUTHORITATIVE,
-        Constants.DEFAULT_METADATASTORE_AUTHORITATIVE);
-  }
-
-  /**
-   * Require a filesystem to have a metadata store; skip test
-   * if not.
-   * @param fs filesystem to check
-   */
-  public static void assumeFilesystemHasMetadatastore(S3AFileSystem fs) {
-    assume("Filesystem does not have a metastore",
-        fs.hasMetadataStore());
-  }
-
   /**
    * Reset all metrics in a list.
    * @param metrics metrics to reset
@@ -650,14 +551,12 @@ public static <E extends Throwable, T extends Closeable> E interceptClosing(
 
   /**
    * Patch a configuration for testing.
-   * This includes possibly enabling s3guard, setting up the local
+   * This includes setting up the local
    * FS temp dir and anything else needed for test runs.
    * @param conf configuration to patch
    * @return the now-patched configuration
    */
   public static Configuration prepareTestConfiguration(final Configuration conf) {
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
     // set hadoop temp dir to a default value
     String testUniqueForkId =
         System.getProperty(TEST_UNIQUE_FORK_ID);
@@ -676,9 +575,26 @@ public static Configuration prepareTestConfiguration(final Configuration conf) {
         DEFAULT_DIRECTORY_MARKER_POLICY);
     conf.set(DIRECTORY_MARKER_POLICY, directoryRetention);
 
+    boolean prefetchEnabled =
+        getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
+    conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled);
+
     return conf;
   }
 
+  /**
+   * build dir.
+   * @return the directory for the project's build, as set by maven,
+   * falling back to pwd + "target" if running from an IDE;
+   */
+  public static File getProjectBuildDir() {
+    String propval = System.getProperty(PROJECT_BUILD_DIRECTORY_PROPERTY);
+    if (StringUtils.isEmpty(propval)) {
+      propval = "target";
+    }
+    return new File(propval).getAbsoluteFile();
+  }
+
   /**
    * Clear any Hadoop credential provider path.
    * This is needed if people's test setups switch to credential providers,
@@ -696,6 +612,7 @@ public static void unsetHadoopCredentialProviders(final Configuration conf) {
    * @return a set of credentials
    * @throws IOException on a failure
    */
+  @SuppressWarnings("deprecation")
   public static AWSCredentialsProvider buildAwsCredentialsProvider(
       final Configuration conf)
       throws IOException {
@@ -799,16 +716,6 @@ public static String getTestBucketName(final Configuration conf) {
     return URI.create(bucket).getHost();
   }
 
-  /**
-   * Get the prefix for DynamoDB table names used in tests.
-   * @param conf configuration to scan.
-   * @return the table name prefix
-   */
-  public static String getTestDynamoTablePrefix(final Configuration conf) {
-    return getTestProperty(conf, TEST_S3GUARD_DYNAMO_TABLE_PREFIX,
-        TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT);
-  }
-
   /**
    * Remove any values from a bucket.
    * @param bucket bucket whose overrides are to be removed. Can be null/empty
@@ -868,16 +775,6 @@ public static void removeBaseAndBucketOverrides(
     removeBaseAndBucketOverrides(getTestBucketName(conf), conf, options);
   }
 
-  /**
-   * Disable S3Guard from the test bucket in a configuration.
-   * @param conf configuration.
-   */
-  public static void disableS3GuardInTestBucket(Configuration conf) {
-    removeBaseAndBucketOverrides(getTestBucketName(conf), conf,
-        S3_METADATA_STORE_IMPL,
-        DIRECTORY_MARKER_POLICY);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-  }
   /**
    * Call a function; any exception raised is logged at info.
    * This is for test teardowns.
@@ -942,16 +839,14 @@ public static S3AFileStatus getStatusWithEmptyDirFlag(
   /**
    * Create mock implementation of store context.
    * @param multiDelete
-   * @param store
    * @param accessors
    * @return
    * @throws URISyntaxException
    * @throws IOException
    */
   public static StoreContext createMockStoreContext(
-          boolean multiDelete,
-          OperationTrackingStore store,
-          ContextAccessors accessors)
+      boolean multiDelete,
+      ContextAccessors accessors)
           throws URISyntaxException, IOException {
     URI name = new URI("s3a://bucket");
     Configuration conf = new Configuration();
@@ -975,13 +870,92 @@ public static StoreContext createMockStoreContext(
             ChangeDetectionPolicy.createPolicy(ChangeDetectionPolicy.Mode.None,
                 ChangeDetectionPolicy.Source.ETag, false))
         .setMultiObjectDeleteEnabled(multiDelete)
-        .setMetadataStore(store)
         .setUseListV1(false)
         .setContextAccessors(accessors)
-        .setTimeProvider(new S3Guard.TtlTimeProvider(conf))
         .build();
   }
 
+  /**
+   * Write the text to a file asynchronously. Logs the operation duration.
+   * @param fs filesystem
+   * @param path path
+   * @return future to the patch created.
+   */
+  private static CompletableFuture<Path> put(FileSystem fs,
+      Path path, String text) {
+    return submit(EXECUTOR, () -> {
+      try (DurationInfo ignore =
+               new DurationInfo(LOG, false, "Creating %s", path)) {
+        createFile(fs, path, true, text.getBytes(Charsets.UTF_8));
+        return path;
+      }
+    });
+  }
+
+  /**
+   * Build a set of files in a directory tree.
+   * @param fs filesystem
+   * @param destDir destination
+   * @param depth file depth
+   * @param fileCount number of files to create.
+   * @param dirCount number of dirs to create at each level
+   * @return the list of files created.
+   */
+  public static List<Path> createFiles(final FileSystem fs,
+      final Path destDir,
+      final int depth,
+      final int fileCount,
+      final int dirCount) throws IOException {
+    return createDirsAndFiles(fs, destDir, depth, fileCount, dirCount,
+        new ArrayList<>(fileCount),
+        new ArrayList<>(dirCount));
+  }
+
+  /**
+   * Build a set of files in a directory tree.
+   * @param fs filesystem
+   * @param destDir destination
+   * @param depth file depth
+   * @param fileCount number of files to create.
+   * @param dirCount number of dirs to create at each level
+   * @param paths [out] list of file paths created
+   * @param dirs [out] list of directory paths created.
+   * @return the list of files created.
+   */
+  public static List<Path> createDirsAndFiles(final FileSystem fs,
+      final Path destDir,
+      final int depth,
+      final int fileCount,
+      final int dirCount,
+      final List<Path> paths,
+      final List<Path> dirs) throws IOException {
+    buildPaths(paths, dirs, destDir, depth, fileCount, dirCount);
+    List<CompletableFuture<Path>> futures = new ArrayList<>(paths.size()
+        + dirs.size());
+
+    // create directories. With dir marker retention, that adds more entries
+    // to cause deletion issues
+    try (DurationInfo ignore =
+             new DurationInfo(LOG, "Creating %d directories", dirs.size())) {
+      for (Path path : dirs) {
+        futures.add(submit(EXECUTOR, () ->{
+          fs.mkdirs(path);
+          return path;
+        }));
+      }
+      waitForCompletion(futures);
+    }
+
+    try (DurationInfo ignore =
+            new DurationInfo(LOG, "Creating %d files", paths.size())) {
+      for (Path path : paths) {
+        futures.add(put(fs, path, path.getName()));
+      }
+      waitForCompletion(futures);
+      return paths;
+    }
+  }
+
   /**
    * Helper class to do diffs of metrics.
    */
@@ -1313,7 +1287,7 @@ public static String readWithStatus(
             .withFileStatus(status)
             .build();
 
-    try (FSDataInputStream in = awaitFuture(future)) {
+    try (FSDataInputStream in = FutureIO.awaitFuture(future)) {
       byte[] buf = new byte[(int) status.getLen()];
       in.readFully(0, buf);
       return new String(buf);
@@ -1340,63 +1314,12 @@ public static long lsR(FileSystem fileSystem, Path path, boolean recursive)
         (status) -> LOG.info("{}", status));
   }
 
-  /**
-   * Turn on the inconsistent S3A FS client in a configuration,
-   * with 100% probability of inconsistency, default delays.
-   * For this to go live, the paths must include the element
-   * {@link FailureInjectionPolicy#DEFAULT_DELAY_KEY_SUBSTRING}.
-   * @param conf configuration to patch
-   * @param delay delay in millis
-   */
-  public static void enableInconsistentS3Client(Configuration conf,
-      long delay) {
-    LOG.info("Enabling inconsistent S3 client");
-    conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
-        S3ClientFactory.class);
-    conf.set(FAIL_INJECT_INCONSISTENCY_KEY, DEFAULT_DELAY_KEY_SUBSTRING);
-    conf.setLong(FAIL_INJECT_INCONSISTENCY_MSEC, delay);
-    conf.setFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY, 0.0f);
-    conf.setFloat(FAIL_INJECT_THROTTLE_PROBABILITY, 0.0f);
-  }
-
-  /**
-   * Is the filesystem using the inconsistent/throttling/unreliable client?
-   * @param fs filesystem
-   * @return true if the filesystem's client is the inconsistent one.
-   */
-  public static boolean isFaultInjecting(S3AFileSystem fs) {
-    return fs.getAmazonS3Client() instanceof InconsistentAmazonS3Client;
-  }
-
-  /**
-   * Skip a test because the client is using fault injection.
-   * This should only be done for those tests which are measuring the cost
-   * of operations or otherwise cannot handle retries.
-   * @param fs filesystem to check
-   */
-  public static void skipDuringFaultInjection(S3AFileSystem fs) {
-    Assume.assumeFalse("Skipping as filesystem has fault injection",
-        isFaultInjecting(fs));
-  }
-
   /**
    * Date format used for mapping upload initiation time to human string.
    */
   public static final DateFormat LISTING_FORMAT = new SimpleDateFormat(
       "yyyy-MM-dd HH:mm:ss");
 
-  /**
-   * Skip a test if the FS isn't marked as supporting magic commits.
-   * @param fs filesystem
-   */
-  public static void assumeMagicCommitEnabled(S3AFileSystem fs)
-      throws IOException {
-    assume("Magic commit option disabled on " + fs,
-        fs.hasPathCapability(
-            fs.getWorkingDirectory(),
-            CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER));
-  }
-
   /**
    * Probe for the configuration containing a specific credential provider.
    * If the list is empty, there will be no match, even if the named provider
@@ -1412,27 +1335,6 @@ public static boolean authenticationContains(Configuration conf,
         .contains(providerClassname);
   }
 
-  public static boolean metadataStorePersistsAuthoritativeBit(MetadataStore ms)
-      throws IOException {
-    Map<String, String> diags = ms.getDiagnostics();
-    String persists =
-        diags.get(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT);
-    if(persists == null){
-      return false;
-    }
-    return Boolean.valueOf(persists);
-  }
-
-  /**
-   * Set the metadata store of a filesystem instance to the given
-   * store, via a package-private setter method.
-   * @param fs filesystem.
-   * @param ms metastore
-   */
-  public static void setMetadataStore(S3AFileSystem fs, MetadataStore ms) {
-    fs.setMetadataStore(ms);
-}
-
   public static void checkListingDoesNotContainPath(S3AFileSystem fs, Path filePath)
       throws IOException {
     final RemoteIterator<LocatedFileStatus> listIter =
@@ -1563,4 +1465,49 @@ public static void skipIfEncryptionNotSet(Configuration configuration,
           + " in " + secrets);
     }
   }
+
+  /**
+   * Get the input stream statistics of an input stream.
+   * Raises an exception if the inner stream is not an S3A input stream
+   * or prefetching input stream
+   * @param in wrapper
+   * @return the statistics for the inner stream
+   */
+  public static S3AInputStreamStatistics getInputStreamStatistics(
+      FSDataInputStream in) {
+
+    InputStream inner = in.getWrappedStream();
+    if (inner instanceof S3AInputStream) {
+      return ((S3AInputStream) inner).getS3AStreamStatistics();
+    } else if (inner instanceof S3APrefetchingInputStream) {
+      return ((S3APrefetchingInputStream) inner).getS3AStreamStatistics();
+    } else {
+      throw new AssertionError("Not an S3AInputStream or S3APrefetchingInputStream: " + inner);
+    }
+  }
+
+  /**
+   * Get the inner stream of an input stream.
+   * Raises an exception if the inner stream is not an S3A input stream
+   * @param in wrapper
+   * @return the inner stream
+   * @throws AssertionError if the inner stream is of the wrong type
+   */
+  public static S3AInputStream getS3AInputStream(
+          FSDataInputStream in) {
+    InputStream inner = in.getWrappedStream();
+    if (inner instanceof S3AInputStream) {
+      return (S3AInputStream) inner;
+    } else {
+      throw new AssertionError("Not an S3AInputStream: " + inner);
+    }
+  }
+
+  /**
+   * Disable Prefetching streams from S3AFileSystem in tests.
+   * @param conf Configuration to remove the prefetch property from.
+   */
+  public static void disablePrefetching(Configuration conf) {
+    removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
index 97069c7c8a100..36381bf14b169 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a;
 
 import com.amazonaws.regions.Regions;
+import org.assertj.core.api.Assertions;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,40 +28,54 @@
 
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
+/**
+ * Verifies the mapping of ARN declaration of resource to the associated
+ * access point.
+ * The region mapping assertions have been brittle to changes across AWS SDK
+ * versions to only verify partial matches, rather than the FQDN of the
+ * endpoints.
+ *
+ */
 public class TestArnResource extends HadoopTestBase {
   private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
 
+  private final static String MOCK_ACCOUNT = "123456789101";
+
   @Test
   public void parseAccessPointFromArn() throws IllegalArgumentException {
     describe("Parse AccessPoint ArnResource from arn string");
 
     String accessPoint = "testAp";
-    String accountId = "123456789101";
     String[][] regionPartitionEndpoints = new String[][] {
-        {Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"},
-        {Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
-            "s3-accesspoint.us-gov-east-1.amazonaws.com"},
-        {Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"},
+        {Regions.EU_WEST_1.getName(), "aws"},
+        {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"},
+        {Regions.CN_NORTH_1.getName(), "aws-cn"},
     };
 
     for (String[] testPair : regionPartitionEndpoints) {
       String region = testPair[0];
       String partition = testPair[1];
-      String endpoint = testPair[2];
 
-      // arn:partition:service:region:account-id:resource-type/resource-id
-      String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
-          accessPoint);
-
-      ArnResource resource = ArnResource.accessPointFromArn(arn);
-      assertEquals("Arn does not match", arn, resource.getFullArn());
+      ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
       assertEquals("Access Point name does not match", accessPoint, resource.getName());
-      assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
+      assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
       assertEquals("Region does not match", region, resource.getRegion());
-      assertEquals("Endpoint does not match", endpoint, resource.getEndpoint());
     }
   }
 
+  @Test
+  public void makeSureEndpointHasTheCorrectFormat() {
+    // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
+    // endpoints for the client are modified. This test makes sure endpoint is set up correctly.
+    ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
+        "test");
+    String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
+
+    Assertions.assertThat(accessPoint.getEndpoint())
+        .describedAs("Endpoint has invalid format. Access Point requests will not work")
+        .isEqualTo(expected);
+  }
+
   @Test
   public void invalidARNsMustThrow() throws Exception {
     describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@@ -69,6 +84,23 @@ public void invalidARNsMustThrow() throws Exception {
         ArnResource.accessPointFromArn("invalid:arn:resource"));
   }
 
+  /**
+   * Create an {@link ArnResource} from string components
+   * @param partition - partition for ARN
+   * @param region - region for ARN
+   * @param accountId - accountId for ARN
+   * @param resourceName - ARN resource name
+   * @return ArnResource described by its properties
+   */
+  private ArnResource getArnResourceFrom(String partition, String region, String accountId,
+      String resourceName) {
+    // arn:partition:service:region:account-id:resource-type/resource-id
+    String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
+        resourceName);
+
+    return ArnResource.accessPointFromArn(arn);
+  }
+
   private void describe(String message) {
     LOG.info(message);
   }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBucketConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBucketConfiguration.java
index 0ac49a39d38f1..07e07ba16115f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBucketConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBucketConfiguration.java
@@ -35,6 +35,8 @@
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.test.AbstractHadoopTestBase;
 
+import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE;
+import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE_CLIENT;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_BUCKET_PREFIX;
 import static org.apache.hadoop.fs.s3a.Constants.S3A_SECURITY_CREDENTIAL_PROVIDER_PATH;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
@@ -138,9 +140,9 @@ public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
     String impl = "fs.s3a.impl";
     config.set(impl, "orig");
     setBucketOption(config, "b", impl, "b");
-    String metastoreImpl = "fs.s3a.metadatastore.impl";
-    String ddb = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
-    setBucketOption(config, "b", metastoreImpl, ddb);
+    String changeDetectionMode = CHANGE_DETECT_MODE;
+    String client = CHANGE_DETECT_MODE_CLIENT;
+    setBucketOption(config, "b", changeDetectionMode, client);
     setBucketOption(config, "b", "impl2", "b2");
     setBucketOption(config, "b", "bucket.b.loop", "b3");
     assertOptionEquals(config, "fs.s3a.bucket.b.impl", "b");
@@ -148,7 +150,7 @@ public void testBucketConfigurationSkipsUnmodifiable() throws Throwable {
     Configuration updated = propagateBucketOptions(config, "b");
     assertOptionEquals(updated, impl, "orig");
     assertOptionEquals(updated, "fs.s3a.impl2", "b2");
-    assertOptionEquals(updated, metastoreImpl, ddb);
+    assertOptionEquals(updated, changeDetectionMode, client);
     assertOptionEquals(updated, "fs.s3a.bucket.b.loop", null);
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
index 4f063904127c3..35199f4092790 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
@@ -23,14 +23,12 @@
 import java.io.InterruptedIOException;
 import java.net.SocketTimeoutException;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.SdkBaseException;
 import com.amazonaws.SdkClientException;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import org.junit.Assert;
 import org.junit.Before;
@@ -261,14 +259,6 @@ public void testRetryThrottled() throws Throwable {
         ex, retries, false);
   }
 
-  @Test
-  public void testRetryThrottledDDB() throws Throwable {
-    assertRetryAction("Expected retry on connection timeout",
-        RETRY_POLICY, RetryPolicy.RetryAction.RETRY,
-        new ProvisionedThroughputExceededException("IOPs"), 1, false);
-  }
-
-
   protected AmazonServiceException newThrottledException() {
     return serviceException(
         AWSServiceThrottledException.STATUS_CODE, "throttled");
@@ -446,33 +436,6 @@ public void testQuietlyEvalReturnValueFail() {
         quietlyEval("", "", () -> 3 / d));
   }
 
-  @Test
-  public void testDynamoDBThrottleConversion() throws Throwable {
-    ProvisionedThroughputExceededException exceededException =
-        new ProvisionedThroughputExceededException("iops");
-    AWSServiceThrottledException ddb = verifyTranslated(
-        AWSServiceThrottledException.class, exceededException);
-    assertTrue(isThrottleException(exceededException));
-    assertTrue(isThrottleException(ddb));
-    assertRetryAction("Expected throttling retry",
-        RETRY_POLICY,
-        RetryPolicy.RetryAction.RETRY,
-        ddb, SAFE_RETRY_COUNT, false);
-    // and briefly attempt an operation
-    CatchCallback catcher = new CatchCallback();
-    AtomicBoolean invoked = new AtomicBoolean(false);
-    invoker.retry("test", null, false, catcher,
-        () -> {
-          if (!invoked.getAndSet(true)) {
-            throw exceededException;
-          }
-        });
-    // to verify that the ex was translated by the time it
-    // got to the callback
-    verifyExceptionClass(AWSServiceThrottledException.class,
-        catcher.lastException);
-  }
-
   /**
    * Catch the exception and preserve it for later queries.
    */
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
index 77ba31c3ce8ea..38993b43ebf45 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
@@ -18,27 +18,14 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.statistics.IOStatistics;
-import org.apache.hadoop.fs.statistics.IOStatisticsSource;
-import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import java.util.NoSuchElementException;
 
-import org.assertj.core.api.Assertions;
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.Set;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 
-import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.extractStatistics;
-import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
-import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
-import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
@@ -46,78 +33,6 @@
  */
 public class TestListing extends AbstractS3AMockTest {
 
-  private static class MockRemoteIterator<S3AFileStatus> implements
-      RemoteIterator<S3AFileStatus>, IOStatisticsSource {
-
-    private final IOStatisticsStore ioStatistics;
-
-    private Iterator<S3AFileStatus> iterator;
-
-    MockRemoteIterator(Collection<S3AFileStatus> source) {
-      iterator = source.iterator();
-      this.ioStatistics = iostatisticsStore()
-          .withDurationTracking(OBJECT_LIST_REQUEST)
-          .build();
-      ioStatistics.incrementCounter(OBJECT_LIST_REQUEST);
-    }
-
-    public boolean hasNext() {
-      return iterator.hasNext();
-    }
-
-    public S3AFileStatus next() {
-      return iterator.next();
-    }
-
-    @Override
-    public IOStatistics getIOStatistics() {
-      return ioStatistics;
-    }
-  }
-
-  private S3AFileStatus blankFileStatus(Path path) {
-    return new S3AFileStatus(Tristate.UNKNOWN, path, null);
-  }
-
-  @Test
-  public void testTombstoneReconcilingIterator() throws Exception {
-    Path parent = new Path("/parent");
-    Path liveChild = new Path(parent, "/liveChild");
-    Path deletedChild = new Path(parent, "/deletedChild");
-
-    Listing listing = fs.getListing();
-    Collection<S3AFileStatus> statuses = new ArrayList<>();
-    statuses.add(blankFileStatus(parent));
-    statuses.add(blankFileStatus(liveChild));
-    statuses.add(blankFileStatus(deletedChild));
-
-    Set<Path> tombstones = new HashSet<>();
-    tombstones.add(deletedChild);
-
-    RemoteIterator<S3AFileStatus> sourceIterator = new MockRemoteIterator(
-        statuses);
-    RemoteIterator<S3ALocatedFileStatus> locatedIterator =
-        listing.createLocatedFileStatusIterator(sourceIterator);
-    RemoteIterator<S3ALocatedFileStatus> reconcilingIterator =
-        listing.createTombstoneReconcilingIterator(locatedIterator, tombstones);
-
-    Set<Path> expectedPaths = new HashSet<>();
-    expectedPaths.add(parent);
-    expectedPaths.add(liveChild);
-
-    Set<Path> actualPaths = new HashSet<>();
-    while (reconcilingIterator.hasNext()) {
-      actualPaths.add(reconcilingIterator.next().getPath());
-    }
-    Assertions.assertThat(actualPaths)
-        .describedAs("paths from iterator")
-        .isEqualTo(expectedPaths);
-
-    // now verify the stats went all the way through.
-    IOStatistics iostats = extractStatistics(reconcilingIterator);
-    verifyStatisticCounterValue(iostats, OBJECT_LIST_REQUEST, 1);
-  }
-
   @Test
   public void testProvidedFileStatusIteratorEnd() throws Exception {
     S3AFileStatus s3aStatus = new S3AFileStatus(
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index f273e68371e58..6456cb5e12a7f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -22,9 +22,15 @@
 import java.io.InterruptedIOException;
 import java.net.URI;
 import java.nio.file.AccessDeniedException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
@@ -37,6 +43,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider;
 import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider;
 import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException;
 import org.apache.hadoop.io.retry.RetryPolicy;
@@ -46,6 +53,7 @@
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture;
 import static org.junit.Assert.*;
 
 /**
@@ -95,6 +103,7 @@ public void testProviderFailureError() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testInstantiationChain() throws Throwable {
     Configuration conf = new Configuration(false);
     conf.set(AWS_CREDENTIALS_PROVIDER,
@@ -114,6 +123,7 @@ public void testInstantiationChain() throws Throwable {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testDefaultChain() throws Exception {
     URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
     Configuration conf = new Configuration(false);
@@ -138,6 +148,7 @@ public void testDefaultChainNoURI() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testConfiguredChain() throws Exception {
     URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
     List<Class<?>> expectedClasses =
@@ -156,6 +167,7 @@ public void testConfiguredChain() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testConfiguredChainUsesSharedInstanceProfile() throws Exception {
     URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
     Configuration conf = new Configuration(false);
@@ -194,7 +206,7 @@ static abstract class AbstractProvider implements AWSCredentialsProvider {
   /**
    * A credential provider whose constructor signature doesn't match.
    */
-  static class ConstructorSignatureErrorProvider
+  protected static class ConstructorSignatureErrorProvider
       implements AWSCredentialsProvider {
 
     @SuppressWarnings("unused")
@@ -214,7 +226,7 @@ public void refresh() {
   /**
    * A credential provider whose constructor raises an NPE.
    */
-  static class ConstructorFailureProvider
+  protected static class ConstructorFailureProvider
       implements AWSCredentialsProvider {
 
     @SuppressWarnings("unused")
@@ -242,7 +254,7 @@ public void testAWSExceptionTranslation() throws Throwable {
     }
   }
 
-  static class AWSExceptionRaisingFactory implements AWSCredentialsProvider {
+  protected static class AWSExceptionRaisingFactory implements AWSCredentialsProvider {
 
     public static final String NO_AUTH = "No auth";
 
@@ -368,6 +380,7 @@ private static void assertCredentialProviders(
    * @see S3ATestUtils#authenticationContains(Configuration, String).
    */
   @Test
+  @SuppressWarnings("deprecation")
   public void testAuthenticationContainsProbes() {
     Configuration conf = new Configuration(false);
     assertFalse("found AssumedRoleCredentialProvider",
@@ -457,7 +470,7 @@ public void testIOEInConstructorPropagation() throws Throwable {
   /**
    * Credential provider which raises an IOE when constructed.
    */
-  private static class IOERaisingProvider implements AWSCredentialsProvider {
+  protected static class IOERaisingProvider implements AWSCredentialsProvider {
 
     public IOERaisingProvider(URI uri, Configuration conf)
         throws IOException {
@@ -475,4 +488,153 @@ public void refresh() {
     }
   }
 
+  private static final AWSCredentials EXPECTED_CREDENTIALS = new AWSCredentials() {
+    @Override
+    public String getAWSAccessKeyId() {
+      return "expectedAccessKey";
+    }
+
+    @Override
+    public String getAWSSecretKey() {
+      return "expectedSecret";
+    }
+  };
+
+  /**
+   * Credential provider that takes a long time.
+   */
+  protected static class SlowProvider extends AbstractSessionCredentialsProvider {
+
+    public SlowProvider(@Nullable URI uri, Configuration conf) {
+      super(uri, conf);
+    }
+
+    @Override
+    protected AWSCredentials createCredentials(Configuration config) throws IOException {
+      // yield to other callers to induce race condition
+      Thread.yield();
+      return EXPECTED_CREDENTIALS;
+    }
+  }
+
+  private static final int CONCURRENT_THREADS = 10;
+
+  @Test
+  public void testConcurrentAuthentication() throws Throwable {
+    Configuration conf = createProviderConfiguration(SlowProvider.class.getName());
+    Path testFile = getCSVTestPath(conf);
+
+    AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf);
+
+    SlowProvider provider = (SlowProvider) list.getProviders().get(0);
+
+    ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS);
+
+    List<Future<AWSCredentials>> results = new ArrayList<>();
+
+    try {
+      assertFalse(
+          "Provider not initialized. isInitialized should be false",
+          provider.isInitialized());
+      assertFalse(
+          "Provider not initialized. hasCredentials should be false",
+          provider.hasCredentials());
+      if (provider.getInitializationException() != null) {
+        throw new AssertionError(
+            "Provider not initialized. getInitializationException should return null",
+            provider.getInitializationException());
+      }
+
+      for (int i = 0; i < CONCURRENT_THREADS; i++) {
+        results.add(pool.submit(() -> list.getCredentials()));
+      }
+
+      for (Future<AWSCredentials> result : results) {
+        AWSCredentials credentials = result.get();
+        assertEquals("Access key from credential provider",
+                "expectedAccessKey", credentials.getAWSAccessKeyId());
+        assertEquals("Secret key from credential provider",
+                "expectedSecret", credentials.getAWSSecretKey());
+      }
+    } finally {
+      pool.awaitTermination(10, TimeUnit.SECONDS);
+      pool.shutdown();
+    }
+
+    assertTrue(
+        "Provider initialized without errors. isInitialized should be true",
+         provider.isInitialized());
+    assertTrue(
+        "Provider initialized without errors. hasCredentials should be true",
+        provider.hasCredentials());
+    if (provider.getInitializationException() != null) {
+      throw new AssertionError(
+          "Provider initialized without errors. getInitializationException should return null",
+          provider.getInitializationException());
+    }
+  }
+
+  /**
+   * Credential provider with error.
+   */
+  protected static class ErrorProvider extends AbstractSessionCredentialsProvider {
+
+    public ErrorProvider(@Nullable URI uri, Configuration conf) {
+      super(uri, conf);
+    }
+
+    @Override
+    protected AWSCredentials createCredentials(Configuration config) throws IOException {
+      throw new IOException("expected error");
+    }
+  }
+
+  @Test
+  public void testConcurrentAuthenticationError() throws Throwable {
+    Configuration conf = createProviderConfiguration(ErrorProvider.class.getName());
+    Path testFile = getCSVTestPath(conf);
+
+    AWSCredentialProviderList list = createAWSCredentialProviderSet(testFile.toUri(), conf);
+    ErrorProvider provider = (ErrorProvider) list.getProviders().get(0);
+
+    ExecutorService pool = Executors.newFixedThreadPool(CONCURRENT_THREADS);
+
+    List<Future<AWSCredentials>> results = new ArrayList<>();
+
+    try {
+      assertFalse("Provider not initialized. isInitialized should be false",
+          provider.isInitialized());
+      assertFalse("Provider not initialized. hasCredentials should be false",
+          provider.hasCredentials());
+      if (provider.getInitializationException() != null) {
+        throw new AssertionError(
+            "Provider not initialized. getInitializationException should return null",
+            provider.getInitializationException());
+      }
+
+      for (int i = 0; i < CONCURRENT_THREADS; i++) {
+        results.add(pool.submit(() -> list.getCredentials()));
+      }
+
+      for (Future<AWSCredentials> result : results) {
+        interceptFuture(CredentialInitializationException.class,
+            "expected error",
+            result
+        );
+      }
+    } finally {
+      pool.awaitTermination(10, TimeUnit.SECONDS);
+      pool.shutdown();
+    }
+
+    assertTrue(
+        "Provider initialization failed. isInitialized should be true",
+        provider.isInitialized());
+    assertFalse(
+        "Provider initialization failed. hasCredentials should be false",
+        provider.hasCredentials());
+    assertTrue(
+        "Provider initialization failed. getInitializationException should contain the error",
+        provider.getInitializationException().getMessage().contains("expected error"));
+  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java
index 9ae24c19f3da5..ffa2c81e58adf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java
@@ -22,7 +22,10 @@
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
 import org.apache.hadoop.fs.s3a.commit.PutTracker;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
+import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 import org.apache.hadoop.util.Progressable;
 import org.junit.Before;
 import org.junit.Test;
@@ -65,7 +68,12 @@ private S3ABlockOutputStream.BlockOutputStreamBuilder mockS3ABuilder() {
             .withKey("")
             .withProgress(progressable)
             .withPutTracker(putTracker)
-            .withWriteOperations(oHelper);
+            .withWriteOperations(oHelper)
+            .withPutOptions(PutObjectOptions.keepingDirs())
+            .withIOStatisticsAggregator(
+                IOStatisticsContext.getCurrentIOStatisticsContext()
+                    .getAggregator());
+
     return builder;
   }
 
@@ -95,7 +103,8 @@ public void testWriteOperationHelperPartLimits() throws Throwable {
         conf,
         new EmptyS3AStatisticsContext(),
         noopAuditor(conf),
-        AuditTestSupport.NOOP_SPAN);
+        AuditTestSupport.NOOP_SPAN,
+        new MinimalWriteOperationHelperCallbacks());
     ByteArrayInputStream inputStream = new ByteArrayInputStream(
         "a".getBytes());
     // first one works
@@ -141,6 +150,10 @@ public void testCallingCloseAfterCallingAbort() throws Exception {
    */
   @Test
   public void testSyncableUnsupported() throws Exception {
+    final S3ABlockOutputStream.BlockOutputStreamBuilder
+        builder = mockS3ABuilder();
+    builder.withDowngradeSyncableExceptions(false);
+    stream = spy(new S3ABlockOutputStream(builder));
     intercept(UnsupportedOperationException.class, () -> stream.hflush());
     intercept(UnsupportedOperationException.class, () -> stream.hsync());
   }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java
new file mode 100644
index 0000000000000..62a99d7209263
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Date;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * deleteOnExit test for S3A.
+ */
+public class TestS3ADeleteOnExit extends AbstractS3AMockTest {
+
+  static class TestS3AFileSystem extends S3AFileSystem {
+    private int deleteOnDnExitCount;
+
+    public int getDeleteOnDnExitCount() {
+      return deleteOnDnExitCount;
+    }
+
+    @Override
+    public boolean deleteOnExit(Path f) throws IOException {
+      deleteOnDnExitCount++;
+      return super.deleteOnExit(f);
+    }
+
+    // This is specifically designed for deleteOnExit processing.
+    // In this specific case, deleteWithoutCloseCheck() will only be called in the path of
+    // processDeleteOnExit.
+    @Override
+    protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOException {
+      boolean result = super.deleteWithoutCloseCheck(f, recursive);
+      deleteOnDnExitCount--;
+      return result;
+    }
+  }
+
+  @Test
+  public void testDeleteOnExit() throws Exception {
+    Configuration conf = createConfiguration();
+    TestS3AFileSystem testFs  = new TestS3AFileSystem();
+    URI uri = URI.create(FS_S3A + "://" + BUCKET);
+    // unset S3CSE property from config to avoid pathIOE.
+    conf.unset(Constants.S3_ENCRYPTION_ALGORITHM);
+    testFs.initialize(uri, conf);
+    AmazonS3 testS3 = testFs.getAmazonS3ClientForTesting("mocking");
+
+    Path path = new Path("/file");
+    String key = path.toUri().getPath().substring(1);
+    ObjectMetadata meta = new ObjectMetadata();
+    meta.setContentLength(1L);
+    meta.setLastModified(new Date(2L));
+    when(testS3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key))))
+            .thenReturn(meta);
+
+    testFs.deleteOnExit(path);
+    testFs.close();
+    assertEquals(0, testFs.getDeleteOnDnExitCount());
+  }
+
+  private ArgumentMatcher<GetObjectMetadataRequest> correctGetMetadataRequest(
+          String bucket, String key) {
+    return request -> request != null
+            && request.getBucketName().equals(bucket)
+            && request.getKey().equals(key);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java
index db5b5b56851ea..c62bf5daca3a4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.net.SocketException;
 import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CompletableFuture;
 
 import com.amazonaws.SdkClientException;
 import com.amazonaws.services.s3.model.GetObjectRequest;
@@ -34,9 +35,10 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.audit.impl.NoopSpan;
 import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
-import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
 
 import static java.lang.Math.min;
+import static org.apache.hadoop.util.functional.FutureIO.eval;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
@@ -54,7 +56,6 @@ public class TestS3AInputStreamRetry extends AbstractS3AMockTest {
   @Test
   public void testInputStreamReadRetryForException() throws IOException {
     S3AInputStream s3AInputStream = getMockedS3AInputStream();
-
     assertEquals("'a' from the test input stream 'ab' should be the first " +
         "character being read", INPUT.charAt(0), s3AInputStream.read());
     assertEquals("'b' from the test input stream 'ab' should be the second " +
@@ -103,13 +104,15 @@ private S3AInputStream getMockedS3AInputStream() {
         INPUT.length());
 
     S3AReadOpContext s3AReadOpContext = fs.createReadContext(
-        s3AFileStatus, S3AInputPolicy.Normal,
-        ChangeDetectionPolicy.getPolicy(fs.getConf()), 100, NoopSpan.INSTANCE);
+        s3AFileStatus,
+        NoopSpan.INSTANCE);
 
     return new S3AInputStream(
         s3AReadOpContext,
         s3ObjectAttributes,
-        getMockedInputStreamCallback());
+        getMockedInputStreamCallback(),
+        s3AReadOpContext.getS3AStatisticsContext().newInputStreamStatistics(),
+            null);
   }
 
   /**
@@ -151,6 +154,11 @@ public GetObjectRequest newGetRequest(String key) {
         return new GetObjectRequest(fs.getBucket(), key);
       }
 
+      @Override
+      public <T> CompletableFuture<T> submit(final CallableRaisingIOE<T> operation) {
+        return eval(operation);
+      }
+
       @Override
       public void close() {
       }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
new file mode 100644
index 0000000000000..e05ee25adfa74
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AProxy.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_HOST;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_PORT;
+import static org.apache.hadoop.fs.s3a.Constants.PROXY_SECURED;
+import static org.apache.hadoop.fs.s3a.S3AUtils.initProxySupport;
+
+/**
+ * Tests to verify {@link S3AUtils} translates the proxy configurations
+ * are set correctly to Client configurations which are later used to construct
+ * the proxy in AWS SDK.
+ */
+public class TestS3AProxy extends AbstractHadoopTestBase {
+
+  /**
+   * Verify Http proxy protocol.
+   */
+  @Test
+  public void testProxyHttp() throws IOException {
+    Configuration proxyConfigForHttp = createProxyConfig(false);
+    verifyProxy(proxyConfigForHttp, false);
+  }
+
+  /**
+   * Verify Https proxy protocol.
+   */
+  @Test
+  public void testProxyHttps() throws IOException {
+    Configuration proxyConfigForHttps = createProxyConfig(true);
+    verifyProxy(proxyConfigForHttps, true);
+  }
+
+  /**
+   * Verify default proxy protocol.
+   */
+  @Test
+  public void testProxyDefault() throws IOException {
+    Configuration proxyConfigDefault = new Configuration();
+    proxyConfigDefault.set(PROXY_HOST, "testProxyDefault");
+    verifyProxy(proxyConfigDefault, false);
+  }
+
+  /**
+   * Assert that the configuration set for a proxy gets translated to Client
+   * configuration with the correct protocol to be used by AWS SDK.
+   * @param proxyConfig Configuration used to set the proxy configs.
+   * @param isExpectedSecured What is the expected protocol for the proxy to
+   *                          be? true for https, and false for http.
+   * @throws IOException
+   */
+  private void verifyProxy(Configuration proxyConfig,
+      boolean isExpectedSecured)
+      throws IOException {
+    ClientConfiguration awsConf = new ClientConfiguration();
+    initProxySupport(proxyConfig, "test-bucket", awsConf);
+    Assertions.assertThat(awsConf.getProxyProtocol())
+        .describedAs("Proxy protocol not as expected")
+        .isEqualTo(isExpectedSecured ? Protocol.HTTPS : Protocol.HTTP);
+  }
+
+  /**
+   * Create a configuration file with proxy configs.
+   * @param isSecured Should the configured proxy be secured or not?
+   * @return configuration.
+   */
+  private Configuration createProxyConfig(boolean isSecured) {
+    Configuration conf = new Configuration();
+    conf.set(PROXY_HOST, "testProxy");
+    conf.set(PROXY_PORT, "1234");
+    conf.setBoolean(PROXY_SECURED, isSecured);
+    return conf;
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java
index c858c9933fc7e..0e105c25c3a45 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java
@@ -33,8 +33,9 @@
 import static org.junit.Assert.assertEquals;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.atLeast;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -59,6 +60,8 @@ public void testUnbuffer() throws IOException {
     // Create mock S3ObjectInputStream and S3Object for open()
     S3ObjectInputStream objectStream = mock(S3ObjectInputStream.class);
     when(objectStream.read()).thenReturn(-1);
+    when(objectStream.read(any(byte[].class))).thenReturn(-1);
+    when(objectStream.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
 
     S3Object s3Object = mock(S3Object.class);
     when(s3Object.getObjectContent()).thenReturn(objectStream);
@@ -67,10 +70,10 @@ public void testUnbuffer() throws IOException {
 
     // Call read and then unbuffer
     FSDataInputStream stream = fs.open(path);
-    assertEquals(0, stream.read(new byte[8])); // mocks read 0 bytes
+    assertEquals(-1, stream.read(new byte[8])); // mocks read 0 bytes
     stream.unbuffer();
 
     // Verify that unbuffer closed the object stream
-    verify(objectStream, times(1)).close();
+    verify(objectStream, atLeast(1)).close();
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java
index 4d3930fbc3c1d..42de7cdffc80e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestStreamChangeTracker.java
@@ -79,7 +79,7 @@ public void testVersionCheckingHandlingNoVersions() throws Throwable {
   public void testVersionCheckingHandlingNoVersionsVersionRequired()
       throws Throwable {
     LOG.info("If an endpoint doesn't return versions but we are configured to"
-        + "require them");
+        + " require them");
     ChangeTracker tracker = newTracker(
         ChangeDetectionPolicy.Mode.Client,
         ChangeDetectionPolicy.Source.VersionId,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java
index 9519c394ce23d..ad72d75081b27 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java
@@ -29,6 +29,7 @@
 import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_FAILURE;
 import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION;
 import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_SPAN_CREATION;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
 import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_REQUEST_HANDLERS;
 import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME;
 import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.LOGGING_AUDIT_SERVICE;
@@ -68,6 +69,7 @@ public static Configuration noopAuditConfig() {
     final Configuration conf = new Configuration(false);
     conf.set(
         AUDIT_SERVICE_CLASSNAME, NOOP_AUDIT_SERVICE);
+    conf.setBoolean(AUDIT_ENABLED, true);
     return conf;
   }
 
@@ -88,6 +90,7 @@ public static Configuration loggingAuditConfig() {
    */
   public static Configuration enableLoggingAuditor(final Configuration conf) {
     conf.set(AUDIT_SERVICE_CLASSNAME, LOGGING_AUDIT_SERVICE);
+    conf.setBoolean(AUDIT_ENABLED, true);
     conf.setBoolean(REJECT_OUT_OF_SPAN_OPERATIONS,  true);
     return conf;
   }
@@ -117,7 +120,8 @@ public static Configuration resetAuditOptions(Configuration conf) {
         REFERRER_HEADER_ENABLED,
         REJECT_OUT_OF_SPAN_OPERATIONS,
         AUDIT_REQUEST_HANDLERS,
-        AUDIT_SERVICE_CLASSNAME);
+        AUDIT_SERVICE_CLASSNAME,
+        AUDIT_ENABLED);
     return conf;
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java
index bd145fa9c7403..4b0b65fd783ae 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java
@@ -37,8 +37,9 @@
 import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_REQUEST_EXECUTION;
 import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_ACCESS;
 import static org.apache.hadoop.fs.s3a.Statistic.STORE_IO_REQUEST;
-import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME;
 import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.resetAuditOptions;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_ALL_PROBES;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_FILE_PROBE;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.ROOT_FILE_STATUS_PROBE;
@@ -67,13 +68,16 @@ public Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     resetAuditOptions(conf);
     conf.set(AUDIT_SERVICE_CLASSNAME, AccessCheckingAuditor.CLASS);
+    conf.setBoolean(AUDIT_ENABLED, true);
     return conf;
   }
 
   @Override
   public void setup() throws Exception {
     super.setup();
-    auditor = (AccessCheckingAuditor) getFileSystem().getAuditor();
+    final S3AFileSystem fs = getFileSystem();
+    auditor = (AccessCheckingAuditor) fs.getAuditor();
+    setSpanSource(fs);
   }
 
   @Test
@@ -87,7 +91,7 @@ public void testFileAccessAllowed() throws Throwable {
     verifyMetrics(
         () -> access(fs, path),
         with(INVOCATION_ACCESS, 1),
-        whenRaw(FILE_STATUS_FILE_PROBE));
+        always(FILE_STATUS_FILE_PROBE));
   }
 
   @Test
@@ -100,7 +104,7 @@ public void testDirAccessAllowed() throws Throwable {
     verifyMetrics(
         () -> access(fs, path),
         with(INVOCATION_ACCESS, 1),
-        whenRaw(FILE_STATUS_ALL_PROBES));
+        always(FILE_STATUS_ALL_PROBES));
   }
 
   @Test
@@ -113,7 +117,7 @@ public void testRootDirAccessAllowed() throws Throwable {
     verifyMetrics(
         () -> access(fs, path),
         with(INVOCATION_ACCESS, 1),
-        whenRaw(ROOT_FILE_STATUS_PROBE));
+        always(ROOT_FILE_STATUS_PROBE));
   }
 
   /**
@@ -136,7 +140,7 @@ public void testFileAccessDenied() throws Throwable {
         with(AUDIT_ACCESS_CHECK_FAILURE, 1),
         // one S3 request: a HEAD.
         with(AUDIT_REQUEST_EXECUTION, 1),
-        whenRaw(FILE_STATUS_FILE_PROBE));
+        always(FILE_STATUS_FILE_PROBE));
   }
 
   /**
@@ -160,7 +164,7 @@ public void testDirAccessDenied() throws Throwable {
         with(AUDIT_REQUEST_EXECUTION, 2),
         with(STORE_IO_REQUEST, 2),
         with(AUDIT_ACCESS_CHECK_FAILURE, 1),
-        whenRaw(FILE_STATUS_ALL_PROBES));
+        always(FILE_STATUS_ALL_PROBES));
   }
 
   /**
@@ -181,7 +185,7 @@ public void testMissingPathAccessFNFE() throws Throwable {
         // two S3 requests: a HEAD and a LIST.
         with(AUDIT_REQUEST_EXECUTION, 2),
         with(AUDIT_ACCESS_CHECK_FAILURE, 0),
-        whenRaw(FILE_STATUS_ALL_PROBES));
+        always(FILE_STATUS_ALL_PROBES));
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManagerDisabled.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManagerDisabled.java
new file mode 100644
index 0000000000000..68aaef51d60f6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManagerDisabled.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.audit;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.audit.impl.NoopAuditManagerS3A;
+import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+
+import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.NOOP_SPAN;
+import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.resetAuditOptions;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
+
+/**
+ * Verify that audit managers are disabled if set to false.
+ */
+public class ITestAuditManagerDisabled extends AbstractS3ACostTest {
+
+  public ITestAuditManagerDisabled() {
+    super(true);
+  }
+
+  @Override
+  public Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    resetAuditOptions(conf);
+    conf.setBoolean(AUDIT_ENABLED, false);
+    return conf;
+  }
+
+  /**
+   * Verify that the auditor is the no-op auditor if auditing is disabled.
+   */
+  @Test
+  public void testAuditorDisabled() {
+
+    final S3AFileSystem fs = getFileSystem();
+    final AuditManagerS3A auditManager = fs.getAuditManager();
+
+    Assertions.assertThat(auditManager)
+        .isInstanceOf(NoopAuditManagerS3A.class);
+  }
+
+  /**
+   * All the audit spans are the no-op span.
+   */
+  @Test
+  public void testAuditSpansAreAllTheSame() throws Throwable {
+
+    final S3AFileSystem fs = getFileSystem();
+    final AuditSpanS3A span1 = fs.createSpan("span1", null, null);
+    final AuditSpanS3A span2 = fs.createSpan("span2", null, null);
+    Assertions.assertThat(span1)
+        .describedAs("audit span 1")
+        .isSameAs(NOOP_SPAN);
+    Assertions.assertThat(span2)
+        .describedAs("audit span 2")
+        .isSameAs(span1);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/MemoryHungryAuditor.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/MemoryHungryAuditor.java
new file mode 100644
index 0000000000000..63e7922001e8b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/MemoryHungryAuditor.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.audit;
+
+import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.s3a.audit.impl.AbstractAuditSpanImpl;
+import org.apache.hadoop.fs.s3a.audit.impl.AbstractOperationAuditor;
+
+
+/**
+ * An audit service which consumes lots of memory.
+ */
+public class MemoryHungryAuditor extends AbstractOperationAuditor {
+
+  public static final String NAME = "org.apache.hadoop.fs.s3a.audit.MemoryHungryAuditor";
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MemoryHungryAuditor.class);
+  /**
+   * How big is each manager?
+   */
+  public static final int MANAGER_SIZE = 10 * 1024 * 1024;
+
+  /**
+   * How big is each span?
+   */
+  public static final int SPAN_SIZE = 512 * 1024;
+
+  private static final AtomicLong INSTANCE_COUNT = new AtomicLong();
+
+  private final AtomicLong spanCount = new AtomicLong();
+
+  private final byte[] data = new byte[MANAGER_SIZE];
+
+  /**
+   * unbonded span created on demand.
+   */
+  private AuditSpanS3A unbondedSpan;
+
+
+  /**
+   * Constructor.
+   */
+  public MemoryHungryAuditor() {
+    super("MemoryHungryAuditor");
+    INSTANCE_COUNT.incrementAndGet();
+  }
+
+  public long getSpanCount() {
+    return spanCount.get();
+  }
+
+  @Override
+  public AuditSpanS3A createSpan(
+      final String operation,
+      @Nullable final String path1,
+      @Nullable final String path2) {
+    spanCount.incrementAndGet();
+    return new MemorySpan(createSpanID(), operation);
+  }
+
+  @Override
+  public AuditSpanS3A getUnbondedSpan() {
+    if (unbondedSpan == null) {
+      unbondedSpan = new MemorySpan(createSpanID(), "unbonded");
+    }
+    return unbondedSpan;
+  }
+
+  @Override
+  public String toString() {
+    return String.format("%s instance %d span count %d",
+        super.toString(),
+        getInstanceCount(),
+        getSpanCount());
+  }
+
+  @Override
+  public void noteSpanReferenceLost(final long threadId) {
+    LOG.info("Span lost for thread {}", threadId);
+  }
+
+  public static long getInstanceCount() {
+    return INSTANCE_COUNT.get();
+  }
+
+  /**
+   * A span which consumes a lot of memory.
+   */
+  private static final class MemorySpan extends AbstractAuditSpanImpl {
+
+    private final byte[] data = new byte[SPAN_SIZE];
+
+    private MemorySpan(final String spanId, final String operationName) {
+      super(spanId, operationName);
+    }
+
+    @Override
+    public AuditSpanS3A activate() {
+      return this;
+    }
+
+    @Override
+    public void deactivate() {
+    }
+
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/impl/TestActiveAuditManagerThreadLeakage.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/impl/TestActiveAuditManagerThreadLeakage.java
new file mode 100644
index 0000000000000..901347d29d87b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/impl/TestActiveAuditManagerThreadLeakage.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.audit.impl;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import org.assertj.core.api.Assertions;
+import org.junit.After;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
+import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A;
+import org.apache.hadoop.fs.s3a.audit.MemoryHungryAuditor;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_SERVICE_CLASSNAME;
+import static org.apache.hadoop.fs.s3a.audit.impl.ActiveAuditManagerS3A.PRUNE_THRESHOLD;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatisticsStore;
+
+/**
+ * This test attempts to recreate the OOM problems of
+ * HADOOP-18091. S3A auditing leaks memory through ThreadLocal references
+ * it does this by creating a thread pool, then
+ * creates and destroys FS instances, with threads in
+ * the pool (but not the main JUnit test thread) creating
+ * audit spans.
+ *
+ * With a custom audit span with a large memory footprint,
+ * memory demands will be high, and if the closed instances
+ * don't get cleaned up, memory runs out.
+ * GCs are forced.
+ * It is critical no spans are created in the junit thread because they will
+ * last for the duration of the test JVM.
+ */
+@SuppressWarnings("ResultOfMethodCallIgnored")
+public class TestActiveAuditManagerThreadLeakage extends AbstractHadoopTestBase {
+  /**
+   * Logging.
+   */
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestActiveAuditManagerThreadLeakage.class);
+
+  /** how many managers to sequentially create. */
+  private static final int MANAGER_COUNT = 500;
+
+  /** size of long lived thread pool. */
+  private static final int THREAD_COUNT = 20;
+  private ExecutorService workers;
+
+  /**
+   * count of prunings which have taken place in the manager lifecycle
+   * operations.
+   */
+  private int pruneCount;
+
+  /**
+   * As audit managers are created they are added to the list,
+   * so we can verify they get GC'd.
+   */
+  private final List<WeakReference<ActiveAuditManagerS3A>> auditManagers =
+      new ArrayList<>();
+
+  @After
+  public void teardown() {
+    if (workers != null) {
+      workers.shutdown();
+    }
+  }
+
+
+  /**
+   * When the service is stopped, the span map is
+   * cleared immediately.
+   */
+  @Test
+  public void testSpanMapClearedInServiceStop() throws IOException {
+    try (ActiveAuditManagerS3A auditManager =
+             new ActiveAuditManagerS3A(emptyStatisticsStore())) {
+      auditManager.init(createMemoryHungryConfiguration());
+      auditManager.start();
+      auditManager.getActiveAuditSpan();
+      // get the span map
+      final WeakReferenceThreadMap<?> spanMap
+          = auditManager.getActiveSpanMap();
+      Assertions.assertThat(spanMap.size())
+          .describedAs("map size")
+          .isEqualTo(1);
+      auditManager.stop();
+      Assertions.assertThat(spanMap.size())
+          .describedAs("map size")
+          .isEqualTo(0);
+    }
+  }
+
+  @Test
+  public void testMemoryLeak() throws Throwable {
+    workers = Executors.newFixedThreadPool(THREAD_COUNT);
+    for (int i = 0; i < MANAGER_COUNT; i++) {
+      final long oneAuditConsumption = createAndTestOneAuditor();
+      LOG.info("manager {} memory retained {}", i, oneAuditConsumption);
+    }
+
+    // pruning must have taken place.
+    // that's somewhat implicit in the test not going OOM.
+    // but if memory allocation in test runs increase, it
+    // may cease to hold. in which case: create more
+    // audit managers
+    LOG.info("Total prune count {}", pruneCount);
+
+    Assertions.assertThat(pruneCount)
+        .describedAs("Total prune count")
+        .isNotZero();
+
+    // now count number of audit managers GC'd
+    // some must have been GC'd, showing that no other
+    // references are being retained internally.
+    Assertions.assertThat(auditManagers.stream()
+            .filter((r) -> r.get() == null)
+            .count())
+        .describedAs("number of audit managers garbage collected")
+        .isNotZero();
+  }
+
+  /**
+   * Create, use and then shutdown one auditor in a unique thread.
+   * @return memory consumed/released
+   */
+  private long createAndTestOneAuditor() throws Exception {
+    long original = Runtime.getRuntime().freeMemory();
+    ExecutorService factory = Executors.newSingleThreadExecutor();
+
+    try {
+      pruneCount += factory.submit(this::createAuditorAndWorkers).get();
+    } finally {
+      factory.shutdown();
+      factory.awaitTermination(60, TimeUnit.SECONDS);
+    }
+
+
+    final long current = Runtime.getRuntime().freeMemory();
+    return current - original;
+
+  }
+
+  /**
+   * This is the core of the leakage test.
+   * Create an audit manager and spans across multiple threads.
+   * The spans are created in the long-lived pool, so if there is
+   * any bonding of the life of managers/spans to that of threads,
+   * it will surface as OOM events.
+   * @return count of weak references whose reference values were
+   * nullified.
+   */
+  private int createAuditorAndWorkers()
+      throws IOException, InterruptedException, ExecutionException {
+    try (ActiveAuditManagerS3A auditManager =
+             new ActiveAuditManagerS3A(emptyStatisticsStore())) {
+      auditManager.init(createMemoryHungryConfiguration());
+      auditManager.start();
+      LOG.info("Using {}", auditManager);
+      auditManagers.add(new WeakReference<>(auditManager));
+
+      // no guarantee every thread gets used, so track
+      // in a set. This will give us the thread ID of every
+      // entry in the map.
+
+      Set<Long> threadIds = new HashSet<>();
+
+      List<Future<Result>> futures = new ArrayList<>(THREAD_COUNT);
+
+      // perform the spanning operation in a long lived thread.
+      for (int i = 0; i < THREAD_COUNT; i++) {
+        futures.add(workers.submit(() -> spanningOperation(auditManager)));
+      }
+
+      // get the results and so determine the thread IDs
+      for (Future<Result> future : futures) {
+        final Result r = future.get();
+        threadIds.add(r.getThreadId());
+      }
+
+      final int threadsUsed = threadIds.size();
+      final Long[] threadIdArray = threadIds.toArray(new Long[0]);
+
+      // gc
+      System.gc();
+      // get the span map
+      final WeakReferenceThreadMap<?> spanMap
+          = auditManager.getActiveSpanMap();
+
+      // count number of spans removed
+      final long derefenced = threadIds.stream()
+          .filter((id) -> !spanMap.containsKey(id))
+          .count();
+      if (derefenced > 0) {
+        LOG.info("{} executed across {} threads and dereferenced {} entries",
+            auditManager, threadsUsed, derefenced);
+      }
+
+      // resolve not quite all of the threads.
+      // why not all? leaves at least one for pruning
+      // but it does complicate some of the assertions...
+      int spansRecreated = 0;
+      int subset = threadIdArray.length - 1;
+      LOG.info("Resolving {} thread references", subset);
+      for (int i = 0; i < subset; i++) {
+        final long id = threadIdArray[i];
+
+        // note whether or not the span is present
+        final boolean present = spanMap.containsKey(id);
+
+        // get the the span for that ID. which must never be
+        // null
+        Assertions.assertThat(spanMap.get(id))
+            .describedAs("Span map entry for thread %d", id)
+            .isNotNull();
+
+        // if it wasn't present, the unbounded span must therefore have been
+        // bounded to this map entry.
+        if (!present) {
+          spansRecreated++;
+        }
+      }
+      LOG.info("Recreated {} spans", subset);
+
+      // if the number of spans lost is more than the number
+      // of entries not probed, then at least one span was
+      // recreated
+      if (derefenced > threadIdArray.length - subset) {
+        Assertions.assertThat(spansRecreated)
+            .describedAs("number of recreated spans")
+            .isGreaterThan(0);
+      }
+
+      // now prune.
+      int pruned = auditManager.prune();
+      if (pruned > 0) {
+        LOG.info("{} executed across {} threads and pruned {} entries",
+            auditManager, threadsUsed, pruned);
+      }
+      Assertions.assertThat(pruned)
+          .describedAs("Count of references pruned")
+          .isEqualTo(derefenced - spansRecreated);
+      return pruned + (int) derefenced;
+    }
+
+  }
+
+  private Configuration createMemoryHungryConfiguration() {
+    final Configuration conf = new Configuration(false);
+    conf.set(AUDIT_SERVICE_CLASSNAME, MemoryHungryAuditor.NAME);
+    return conf;
+  }
+
+  /**
+   * The operation in each worker thread.
+   * @param auditManager audit manager
+   * @return result of the call
+   * @throws IOException troluble
+   */
+  private Result spanningOperation(final ActiveAuditManagerS3A auditManager)
+      throws IOException {
+    auditManager.getActiveAuditSpan();
+    final AuditSpanS3A auditSpan =
+        auditManager.createSpan("span", null, null);
+    Assertions.assertThat(auditSpan)
+        .describedAs("audit span for current thread")
+        .isNotNull();
+    // this is needed to ensure that more of the thread pool is used up
+    Thread.yield();
+    return new Result(Thread.currentThread().getId());
+  }
+
+  /**
+   * Result of the spanning operation.
+   */
+  private static final class Result {
+    /** thread operation took place in. */
+    private final long threadId;
+
+
+    private Result(final long threadId) {
+      this.threadId = threadId;
+    }
+
+    private long getThreadId() {
+      return threadId;
+    }
+
+
+  }
+
+  /**
+   * Verify that pruning takes place intermittently.
+   */
+  @Test
+  public void testRegularPruning() throws Throwable {
+    try (ActiveAuditManagerS3A auditManager =
+             new ActiveAuditManagerS3A(emptyStatisticsStore())) {
+      auditManager.init(createMemoryHungryConfiguration());
+      auditManager.start();
+      // get the span map
+      final WeakReferenceThreadMap<?> spanMap
+          = auditManager.getActiveSpanMap();
+      // add a null entry at a thread ID other than this one
+      spanMap.put(Thread.currentThread().getId() + 1, null);
+
+      // remove this span enough times that pruning shall take
+      // place twice
+      // this verifies that pruning takes place and that the
+      // counter is reset
+      int pruningCount = 0;
+      for (int i = 0; i < PRUNE_THRESHOLD * 2 + 1; i++) {
+        boolean pruned = auditManager.removeActiveSpanFromMap();
+        if (pruned) {
+          pruningCount++;
+        }
+      }
+      // pruning must have taken place
+      Assertions.assertThat(pruningCount)
+          .describedAs("Intermittent pruning count")
+          .isEqualTo(2);
+    }
+  }
+
+  /**
+   * Verify span deactivation removes the entry from the map.
+   */
+  @Test
+  public void testSpanDeactivationRemovesEntryFromMap() throws Throwable {
+    try (ActiveAuditManagerS3A auditManager =
+             new ActiveAuditManagerS3A(emptyStatisticsStore())) {
+      auditManager.init(createMemoryHungryConfiguration());
+      auditManager.start();
+      // get the span map
+      final WeakReferenceThreadMap<?> spanMap
+          = auditManager.getActiveSpanMap();
+      final AuditSpanS3A auditSpan =
+          auditManager.createSpan("span", null, null);
+      Assertions.assertThat(auditManager.getActiveAuditSpan())
+          .describedAs("active span")
+          .isSameAs(auditSpan);
+      // this assert gets used repeatedly, so define a lambda-exp
+      // which can be envoked with different arguments
+      Consumer<Boolean> assertMapHasKey = expected ->
+          Assertions.assertThat(spanMap.containsKey(spanMap.currentThreadId()))
+              .describedAs("map entry for current thread")
+              .isEqualTo(expected);
+
+      // sets the span to null
+      auditSpan.deactivate();
+
+      // there's no entry
+      assertMapHasKey.accept(false);
+
+      // asking for the current span will return the unbonded one
+      final AuditSpanS3A newSpan = auditManager.getActiveAuditSpan();
+      Assertions.assertThat(newSpan)
+          .describedAs("active span")
+          .isNotNull()
+          .matches(s -> !s.isValidSpan());
+      // which is in the map
+      // there's an entry
+      assertMapHasKey.accept(true);
+
+      // deactivating the old span does nothing
+      auditSpan.deactivate();
+      assertMapHasKey.accept(true);
+
+      // deactivating the current unbounded span does
+      // remove the entry
+      newSpan.deactivate();
+      assertMapHasKey.accept(false);
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
index 814292c45d83b..9fb09b4cede52 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
@@ -47,9 +47,10 @@
 import org.apache.hadoop.fs.s3a.S3ATestConstants;
 import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool;
 import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
 
@@ -72,7 +73,7 @@
  * Tests use of assumed roles.
  * Only run if an assumed role is provided.
  */
-@SuppressWarnings({"IOResourceOpenedButNotSafelyClosed", "ThrowableNotThrown"})
+@SuppressWarnings("ThrowableNotThrown")
 public class ITestAssumeRole extends AbstractS3ATestBase {
 
   private static final Logger LOG =
@@ -139,6 +140,7 @@ private <E extends Throwable> E expectFileSystemCreateFailure(
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testCreateCredentialProvider() throws IOException {
     describe("Create the credential provider");
 
@@ -152,6 +154,7 @@ public void testCreateCredentialProvider() throws IOException {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testCreateCredentialProviderNoURI() throws IOException {
     describe("Create the credential provider");
 
@@ -169,6 +172,7 @@ public void testCreateCredentialProviderNoURI() throws IOException {
    * @return a configuration set to use to the role ARN.
    * @throws JsonProcessingException problems working with JSON policies.
    */
+  @SuppressWarnings("deprecation")
   protected Configuration createValidRoleConf() throws JsonProcessingException {
     String roleARN = getAssumedRoleARN();
 
@@ -182,6 +186,7 @@ protected Configuration createValidRoleConf() throws JsonProcessingException {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumedInvalidRole() throws Throwable {
     Configuration conf = new Configuration();
     conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE);
@@ -199,6 +204,7 @@ public void testAssumeRoleFSBadARN() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumeRoleNoARN() throws Exception {
     describe("Attemnpt to create the FS with no ARN");
     Configuration conf = createAssumedRoleConfig();
@@ -231,6 +237,7 @@ public void testAssumeRoleFSBadPolicy2() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumeRoleCannotAuthAssumedRole() throws Exception {
     describe("Assert that you can't use assumed roles to auth assumed roles");
 
@@ -244,13 +251,13 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumeRoleBadInnerAuth() throws Exception {
     describe("Try to authenticate with a keypair with spaces");
 
     Configuration conf = createAssumedRoleConfig();
     unsetHadoopCredentialProviders(conf);
-    conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
-        SimpleAWSCredentialsProvider.NAME);
+    conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME);
     conf.set(ACCESS_KEY, "not valid");
     conf.set(SECRET_KEY, "not secret");
     expectFileSystemCreateFailure(conf,
@@ -260,13 +267,13 @@ public void testAssumeRoleBadInnerAuth() throws Exception {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumeRoleBadInnerAuth2() throws Exception {
     describe("Try to authenticate with an invalid keypair");
 
     Configuration conf = createAssumedRoleConfig();
     unsetHadoopCredentialProviders(conf);
-    conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
-        SimpleAWSCredentialsProvider.NAME);
+    conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME);
     conf.set(ACCESS_KEY, "notvalid");
     conf.set(SECRET_KEY, "notsecret");
     expectFileSystemCreateFailure(conf,
@@ -344,6 +351,7 @@ private Configuration createAssumedRoleConfig(String roleARN) {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumeRoleUndefined() throws Throwable {
     describe("Verify that you cannot instantiate the"
         + " AssumedRoleCredentialProvider without a role ARN");
@@ -355,6 +363,7 @@ public void testAssumeRoleUndefined() throws Throwable {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumedIllegalDuration() throws Throwable {
     describe("Expect the constructor to fail if the session is to short");
     Configuration conf = new Configuration();
@@ -383,22 +392,12 @@ public void testAssumeRoleCreateFS() throws IOException {
   public void testAssumeRoleRestrictedPolicyFS() throws Exception {
     describe("Restrict the policy for this session; verify that reads fail.");
 
-    // there's some special handling of S3Guard here as operations
-    // which only go to DDB don't fail the way S3 would reject them.
     Configuration conf = createAssumedRoleConfig();
     bindRolePolicy(conf, RESTRICTED_POLICY);
     Path path = new Path(getFileSystem().getUri());
-    boolean guarded = getFileSystem().hasMetadataStore();
     try (FileSystem fs = path.getFileSystem(conf)) {
-      if (!guarded) {
-        // when S3Guard is enabled, the restricted policy still
-        // permits S3Guard record lookup, so getFileStatus calls
-        // will work iff the record is in the database.
-        // probe the store using a path other than /, so a HEAD
-        // request is issued.
-        forbidden("getFileStatus",
-            () -> fs.getFileStatus(methodPath()));
-      }
+      forbidden("getFileStatus",
+          () -> fs.getFileStatus(methodPath()));
       forbidden("",
           () -> fs.listStatus(ROOT));
       forbidden("",
@@ -428,7 +427,6 @@ public void testAssumeRolePoliciesOverrideRolePerms() throws Throwable {
         policy(
             statement(false, S3_ALL_BUCKETS, S3_GET_OBJECT_TORRENT),
             ALLOW_S3_GET_BUCKET_LOCATION,
-            STATEMENT_S3GUARD_CLIENT,
             STATEMENT_ALLOW_SSE_KMS_RW));
     Path path = path("testAssumeRoleStillIncludesRolePerms");
     roleFS = (S3AFileSystem) path.getFileSystem(conf);
@@ -438,7 +436,6 @@ public void testAssumeRolePoliciesOverrideRolePerms() throws Throwable {
   /**
    * After blocking all write verbs used by S3A, try to write data (fail)
    * and read data (succeed).
-   * For S3Guard: full DDB RW access is retained.
    * SSE-KMS key access is set to decrypt only.
    */
   @Test
@@ -451,7 +448,6 @@ public void testReadOnlyOperations() throws Throwable {
         policy(
             statement(false, S3_ALL_BUCKETS, S3_PATH_WRITE_OPERATIONS),
             STATEMENT_ALL_S3,
-            STATEMENT_S3GUARD_CLIENT,
             STATEMENT_ALLOW_SSE_KMS_READ));
     Path path = methodPath();
     roleFS = (S3AFileSystem) path.getFileSystem(conf);
@@ -499,7 +495,6 @@ public void testRestrictedWriteSubdir() throws Throwable {
     Configuration conf = createAssumedRoleConfig();
 
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALL_BUCKET_READ_ACCESS,
         STATEMENT_ALLOW_SSE_KMS_RW,
         new Statement(Effects.Allow)
@@ -542,6 +537,7 @@ public Path methodPath() throws IOException {
    * don't break.
    */
   @Test
+  @SuppressWarnings("deprecation")
   public void testAssumedRoleRetryHandler() throws Throwable {
     try(AssumedRoleCredentialProvider provider
             = new AssumedRoleCredentialProvider(getFileSystem().getUri(),
@@ -568,7 +564,6 @@ public void testRestrictedCommitActions() throws Throwable {
     fs.mkdirs(readOnlyDir);
 
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         STATEMENT_ALL_BUCKET_READ_ACCESS,
         new Statement(Effects.Allow)
@@ -578,9 +573,9 @@ public void testRestrictedCommitActions() throws Throwable {
     roleFS = (S3AFileSystem) writeableDir.getFileSystem(conf);
     CommitterStatistics committerStatistics = fs.newCommitterStatistics();
     CommitOperations fullOperations = new CommitOperations(fs,
-        committerStatistics);
+        committerStatistics, "/");
     CommitOperations operations = new CommitOperations(roleFS,
-        committerStatistics);
+        committerStatistics, "/");
 
     File localSrc = File.createTempFile("source", "");
     writeCSVData(localSrc);
@@ -610,37 +605,37 @@ public void testRestrictedCommitActions() throws Throwable {
           SinglePendingCommit pending =
               fullOperations.uploadFileToPendingCommit(src, dest, "",
                   uploadPartSize, progress);
-          pending.save(fs, new Path(readOnlyDir,
-              name + CommitConstants.PENDING_SUFFIX), true);
+          pending.save(fs,
+              new Path(readOnlyDir, name + CommitConstants.PENDING_SUFFIX),
+              SinglePendingCommit.serializer());
           assertTrue(src.delete());
         }));
     progress.assertCount("progress counter is not expected",
         range);
 
-    try {
+    try(CommitContext commitContext =
+            operations.createCommitContextForTesting(uploadDest,
+                null, 0)) {
       // we expect to be able to list all the files here
       Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
           pendingCommits = operations.loadSinglePendingCommits(readOnlyDir,
-          true);
+          true, commitContext);
 
       // all those commits must fail
       List<SinglePendingCommit> commits = pendingCommits.getLeft().getCommits();
       assertEquals(range, commits.size());
-      try(CommitOperations.CommitContext commitContext
-              = operations.initiateCommitOperation(uploadDest)) {
-        commits.parallelStream().forEach(
-            (c) -> {
-              CommitOperations.MaybeIOE maybeIOE =
-                  commitContext.commit(c, "origin");
-              Path path = c.destinationPath();
-              assertCommitAccessDenied(path, maybeIOE);
-            });
-      }
+      commits.parallelStream().forEach(
+          (c) -> {
+            CommitOperations.MaybeIOE maybeIOE =
+                commitContext.commit(c, "origin");
+            Path path = c.destinationPath();
+            assertCommitAccessDenied(path, maybeIOE);
+          });
 
       // fail of all list and abort of .pending files.
       LOG.info("abortAllSinglePendingCommits({})", readOnlyDir);
       assertCommitAccessDenied(readOnlyDir,
-          operations.abortAllSinglePendingCommits(readOnlyDir, true));
+          operations.abortAllSinglePendingCommits(readOnlyDir, commitContext, true));
 
       // try writing a magic file
       Path magicDestPath = new Path(readOnlyDir,
@@ -720,7 +715,6 @@ public void executePartialDelete(final Configuration conf,
     fs.delete(destDir, true);
 
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
         new Statement(Effects.Deny)
@@ -752,13 +746,7 @@ public void testBucketLocationForbidden() throws Throwable {
     describe("Restrict role to read only");
     Configuration conf = createAssumedRoleConfig();
 
-    // S3Guard is turned off so that it isn't trying to work out
-    // where any table is.
-    removeBaseAndBucketOverrides(getTestBucketName(conf), conf,
-        S3_METADATA_STORE_IMPL);
-
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
         statement(false, S3_ALL_BUCKETS, S3_GET_BUCKET_LOCATION));
@@ -773,28 +761,4 @@ public void testBucketLocationForbidden() throws Throwable {
     Assertions.assertThat(info)
         .contains(S3GuardTool.BucketInfo.LOCATION_UNKNOWN);
   }
-  /**
-   * Turn off access to dynamo DB Tags and see how DDB table init copes.
-   * There's no testing of the codepath other than checking the logs
-   * - this test does make sure that no regression stops the tag permission
-   * failures from halting the client
-   */
-  @Test
-  public void testRestrictDDBTagAccess() throws Throwable {
-
-    describe("extra policies in assumed roles need;"
-        + " all required policies stated");
-    Configuration conf = createAssumedRoleConfig();
-
-    bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
-        STATEMENT_ALLOW_SSE_KMS_RW,
-        STATEMENT_ALL_S3,
-        new Statement(Effects.Deny)
-            .addActions(S3_PATH_RW_OPERATIONS)
-            .addResources(ALL_DDB_TABLES));
-    Path path = path("testRestrictDDBTagAccess");
-
-    roleFS = (S3AFileSystem) path.getFileSystem(conf);
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
index 853810602baec..dabc0abc2af21 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumedRoleCommitOperations.java
@@ -37,11 +37,6 @@
 
 /**
  * Verify that the commit operations work with a restricted set of operations.
- * The superclass, {@link ITestCommitOperations} turns on an inconsistent client
- * to see how things work in the presence of inconsistency.
- * These tests disable it, to remove that as a factor in these tests, which are
- * verifying that the policy settings to enabled MPU list/commit/abort are all
- * enabled properly.
  */
 public class ITestAssumedRoleCommitOperations extends ITestCommitOperations {
 
@@ -58,11 +53,6 @@ public class ITestAssumedRoleCommitOperations extends ITestCommitOperations {
    */
   private S3AFileSystem roleFS;
 
-  @Override
-  public boolean useInconsistentClient() {
-    return false;
-  }
-
   @Override
   public void setup() throws Exception {
     super.setup();
@@ -72,7 +62,6 @@ public void setup() throws Exception {
     Configuration conf = newAssumedRoleConfig(getConfiguration(),
         getAssumedRoleARN());
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         statement(true, S3_ALL_BUCKETS, S3_BUCKET_READ_OPERATIONS),
         new RoleModel.Statement(RoleModel.Effects.Allow)
@@ -114,7 +103,7 @@ public S3AFileSystem getFullFileSystem() {
   }
 
   /**
-   * switch to an inconsistent path if in inconsistent mode.
+   * switch to an restricted path.
    * {@inheritDoc}
    */
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
index cbba326d5ecae..a829d470e7a66 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java
@@ -20,7 +20,6 @@
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -228,10 +227,13 @@ private String parseBucketFromHost(String host) {
       if (service.contains("s3-accesspoint") || service.contains("s3-outposts")
           || service.contains("s3-object-lambda")) {
         // If AccessPoint then bucketName is of format `accessPoint-accountId`;
-        String[] accessPointBits = hostBits[0].split("-");
-        int lastElem = accessPointBits.length - 1;
-        String accountId = accessPointBits[lastElem];
-        String accessPointName = String.join("", Arrays.copyOf(accessPointBits, lastElem));
+        String[] accessPointBits = bucketName.split("-");
+        String accountId = accessPointBits[accessPointBits.length - 1];
+        // Extract the access point name from bucket name. eg: if bucket name is
+        // test-custom-signer-<accountId>, get the access point name test-custom-signer by removing
+        // -<accountId> from the bucket name.
+        String accessPointName =
+            bucketName.substring(0, bucketName.length() - (accountId.length() + 1));
         Arn arn = Arn.builder()
             .withAccountId(accountId)
             .withPartition("aws")
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
index 402469eb3b736..a16e1b5e492fb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestRestrictedReadAccess.java
@@ -22,22 +22,15 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Optional;
 import java.util.concurrent.Callable;
 
 import org.assertj.core.api.Assertions;
-import org.assertj.core.api.Assumptions;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
@@ -46,23 +39,14 @@
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore;
 import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
 import org.apache.hadoop.mapreduce.lib.input.InvalidInputException;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
 import static org.apache.hadoop.fs.s3a.Constants.ASSUMED_ROLE_ARN;
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeS3GuardState;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.lsR;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
 import static org.apache.hadoop.fs.s3a.auth.RoleModel.Effects;
 import static org.apache.hadoop.fs.s3a.auth.RoleModel.Statement;
 import static org.apache.hadoop.fs.s3a.auth.RoleModel.directory;
@@ -79,21 +63,16 @@
 /**
  * This test creates a client with no read access to the underlying
  * filesystem and then tries to perform various read operations on it.
- * S3Guard in non-auth mode always goes to the FS, so we parameterize the
- * test for S3Guard + Auth to see how failures move around.
- * <ol>
- *   <li>Tests only run if an assumed role is provided.</li>
- *   <li>And the S3Guard tests require DynamoDB.</li>
- * </ol>
+ *
  * The tests are all bundled into one big test case.
  * From a purist unit test perspective, this is utterly wrong as it goes
  * against the
  * <i>"Each test case tests exactly one thing"</i>
  * philosophy of JUnit.
  * <p>
- * However is significantly reduces setup costs on the parameterized test runs,
+ * However is significantly reduces setup costs .
  * as it means that the filesystems and directories only need to be
- * created and destroyed once per parameterized suite, rather than
+ * created and destroyed once per suite, rather than
  * once per individual test.
  * <p>
  * All the test probes have informative messages so when a test failure
@@ -109,8 +88,6 @@
  * created paths.
  *
  */
-@SuppressWarnings("ThrowableNotThrown")
-@RunWith(Parameterized.class)
 public class ITestRestrictedReadAccess extends AbstractS3ATestBase {
 
   private static final Logger LOG =
@@ -147,34 +124,11 @@ public class ITestRestrictedReadAccess extends AbstractS3ATestBase {
    */
   public static final byte[] HELLO = "hello".getBytes(StandardCharsets.UTF_8);
 
-  private final boolean guardedInAuthMode;
-
   /**
    * Wildcard scan to find *.txt in the no-read directory.
-   * When a scan/glob is done with S3Guard in auth mode, the scan will
-   * succeed but the file open will fail for any non-empty file.
-   * In non-auth mode, the read restrictions will fail the actual scan.
    */
   private Path noReadWildcard;
 
-  /**
-   * Parameterization.
-   */
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {"raw", false, false},
-        {"nonauth", true, false},
-        {"auth", true, true}
-    });
-  }
-
-  private final String name;
-
-  private final boolean s3guard;
-
-  private final boolean authMode;
-
   private Path basePath;
 
   private Path noReadDir;
@@ -200,50 +154,9 @@ public static Collection<Object[]> params() {
    */
   private S3AFileSystem readonlyFS;
 
-  /**
-   * Test suite setup.
-   * @param name name for logs/paths.
-   * @param s3guard is S3Guard enabled?
-   * @param authMode is S3Guard in auth mode?
-   */
-  public ITestRestrictedReadAccess(
-      final String name,
-      final boolean s3guard,
-      final boolean authMode) {
-    this.name = name;
-    this.s3guard = s3guard;
-    this.authMode = authMode;
-    this.guardedInAuthMode = s3guard && authMode;
-
-  }
-
-  @Override
-  public Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    String bucketName = getTestBucketName(conf);
-
-    removeBaseAndBucketOverrides(bucketName, conf,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    removeBucketOverrides(bucketName, conf,
-        S3_METADATA_STORE_IMPL);
-    if (!s3guard) {
-      removeBaseAndBucketOverrides(bucketName, conf,
-          S3_METADATA_STORE_IMPL);
-    }
-    conf.setBoolean(METADATASTORE_AUTHORITATIVE, authMode);
-    disableFilesystemCaching(conf);
-    return conf;
-  }
-
   @Override
   public void setup() throws Exception {
     super.setup();
-    if (s3guard) {
-      // s3guard is required for those test runs where any of the
-      // guard options are set
-      assumeS3GuardState(true, getConfiguration());
-    }
     assumeRoleTests();
   }
 
@@ -311,13 +224,8 @@ public void initNoReadAccess() throws Throwable {
     describe("Setting up filesystem");
 
     S3AFileSystem realFS = getFileSystem();
-    verifyS3GuardSettings(realFS, "real filesystem");
 
-    // avoiding the parameterization to steer clear of accidentally creating
-    // patterns; a timestamp is used to ensure tombstones from previous runs
-    // do not interfere
-    basePath = path("testNoReadAccess-" + name
-        + "-" + System.currentTimeMillis() / 1000);
+    basePath = methodPath();
 
     // define the paths and create them.
     describe("Creating test directories and files");
@@ -347,51 +255,19 @@ public void initNoReadAccess() throws Throwable {
     subdir2File2 = new Path(subDir2, "subdir2File2.docx");
     createFile(realFS, subdir2File1, true, HELLO);
     createFile(realFS, subdir2File2, true, HELLO);
-    // execute a recursive list to make sure that S3Guard tables are always
-    // up to date
-    lsR(realFS, noReadDir, true);
 
     // create a role filesystem which does not have read access under a path
     // it still has write access, which can be explored in the final
     // step to delete files and directories.
     roleConfig = createAssumedRoleConfig();
     bindRolePolicyStatements(roleConfig,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         statement(true, S3_ALL_BUCKETS, S3_ALL_OPERATIONS),
         new Statement(Effects.Deny)
             .addActions(S3_ALL_GET)
             .addResources(directory(noReadDir)));
     readonlyFS = (S3AFileSystem) basePath.getFileSystem(roleConfig);
-    verifyS3GuardSettings(readonlyFS, "readonly");
-  }
 
-  /**
-   * Verify that the FS (real or restricted) meets the
-   * requirement of the test.
-   * S3Guard tests are skipped if the (default) store is not
-   * a DDB store consistent across all FS instances.
-   * The raw tests fail if somehow the FS does still have a S3Guard metastore.
-   * @param fs filesystem
-   * @param storeType store role for error messages.
-   */
-  protected void verifyS3GuardSettings(final S3AFileSystem fs,
-      final String storeType) {
-    if (s3guard) {
-      Assumptions.assumeThat(fs.getMetadataStore())
-          .describedAs("Metadata store in "
-                  + storeType
-                  + " fs: %s",
-              fs.getMetadataStore())
-          .isInstanceOf(DynamoDBMetadataStore.class);
-    } else {
-      Assertions.assertThat(fs.hasMetadataStore())
-          .describedAs("Metadata store in "
-                  + storeType
-                  + " fs: %s",
-              fs.getMetadataStore())
-          .isFalse();
-    }
   }
 
   /**
@@ -409,7 +285,7 @@ public void checkBasicFileOperations() throws Throwable {
     lsR(readonlyFS, basePath, true);
 
 
-    // this is HEAD + "/" on S3; get on S3Guard auth when the path exists,
+    // this is HEAD + "/" on S3
     readonlyFS.listStatus(emptyDir);
 
     // a recursive list of the no-read-directory works because
@@ -421,30 +297,16 @@ public void checkBasicFileOperations() throws Throwable {
     readonlyFS.getFileStatus(noReadDir);
 
     readonlyFS.getFileStatus(emptyDir);
-    // now look at a file; the outcome depends on the mode.
-    accessDeniedIf(!guardedInAuthMode, () ->
+    // now look at a file
+    accessDenied(() ->
         readonlyFS.getFileStatus(subdirFile));
 
-    // irrespective of mode, the attempt to read the data will fail.
-    // the only variable is where the failure occurs
+    // the attempt to read the data will also fail.
     accessDenied(() ->
         ContractTestUtils.readUTF8(readonlyFS, subdirFile, HELLO.length));
 
-    // the empty file is interesting.
-    // auth mode doesn't check the store.
-    // Furthermore, because it knows the file length is zero,
-    // it returns -1 without even opening the file.
-    // This means that permissions on the file do not get checked.
-    // See: HADOOP-16464.
-    Optional<FSDataInputStream> optIn = accessDeniedIf(
-        !guardedInAuthMode, () -> readonlyFS.open(emptyFile));
-    if (optIn.isPresent()) {
-      try (FSDataInputStream is = optIn.get()) {
-        Assertions.assertThat(is.read())
-            .describedAs("read of empty file")
-            .isEqualTo(-1);
-      }
-    }
+    accessDenied(() -> readonlyFS.open(emptyFile));
+
   }
 
   /**
@@ -455,22 +317,14 @@ public void checkGlobOperations() throws Throwable {
     describe("Glob Status operations");
     // baseline: the real filesystem on a subdir
     globFS(getFileSystem(), subdirFile, null, false, 1);
-    // a file fails if not in auth mode
-    globFS(readonlyFS, subdirFile, null, !guardedInAuthMode, 1);
+    // a file fails
+    globFS(readonlyFS, subdirFile, null, true, 1);
     // empty directories don't fail.
     FileStatus[] st = globFS(readonlyFS, emptyDir, null, false, 1);
-    if (s3guard) {
-      assertStatusPathEquals(emptyDir, st);
-    }
 
     st = globFS(readonlyFS,
         noReadWildcard,
         null, false, 2);
-    if (s3guard) {
-      Assertions.assertThat(st)
-          .extracting(FileStatus::getPath)
-          .containsExactlyInAnyOrder(subdirFile, subdir2File1);
-    }
 
     // there is precisely one .docx file (subdir2File2.docx)
     globFS(readonlyFS,
@@ -554,15 +408,8 @@ public void checkLocatedFileStatusScanFile() throws Throwable {
         true,
         TEXT_FILE,
         true);
-    accessDeniedIf(!guardedInAuthMode,
-        () -> fetcher.getFileStatuses())
-        .ifPresent(stats -> {
-          Assertions.assertThat(stats)
-              .describedAs("result of located scan")
-              .isNotNull()
-              .flatExtracting(FileStatus::getPath)
-              .containsExactly(subdirFile);
-        });
+    accessDenied(() -> fetcher.getFileStatuses());
+
   }
 
   /**
@@ -614,18 +461,13 @@ public void checkLocatedFileStatusNonexistentPath() throws Throwable {
    * Do some cleanup to see what happens with delete calls.
    * Cleanup happens in test teardown anyway; doing it here
    * just makes use of the delete calls to see how delete failures
-   * change with permissions and S3Guard settings.
+   * change with permissions.
    */
   public void checkDeleteOperations() throws Throwable {
     describe("Testing delete operations");
     readonlyFS.delete(emptyDir, true);
-    if (!authMode) {
-      // to fail on HEAD
-      accessDenied(() -> readonlyFS.delete(emptyFile, true));
-    } else {
-      // checks DDB for status and then issues the DELETE
-      readonlyFS.delete(emptyFile, true);
-    }
+    // to fail on HEAD
+    accessDenied(() -> readonlyFS.delete(emptyFile, true));
 
     // this will succeed for both
     readonlyFS.delete(subDir, true);
@@ -664,27 +506,6 @@ protected <T> AccessDeniedException accessDenied(final Callable<T> eval)
     return intercept(AccessDeniedException.class, eval);
   }
 
-  /**
-   * Conditionally expect an operation to fail with an AccessDeniedException.
-   * @param condition the condition which must be true for access to be denied
-   * @param eval closure to evaluate.
-   * @param <T> type of callable
-   * @return the return value if the call succeeded
-   * and did not return null.
-   * @throws Exception any unexpected exception
-   */
-  protected <T> Optional<T> accessDeniedIf(
-      final boolean condition,
-      final Callable<T> eval)
-      throws Exception {
-    if (condition) {
-      intercept(AccessDeniedException.class, eval);
-      return Optional.empty();
-    } else {
-      return Optional.ofNullable(eval.call());
-    }
-  }
-
   /**
    * Assert that a status array has exactly one element and its
    * value is as expected.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
index 7163c5f4f5353..37c2dce4e1d72 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java
@@ -73,16 +73,12 @@ public final class RoleTestUtils {
       =  statement(true, S3_ALL_BUCKETS, S3_GET_BUCKET_LOCATION);
 
   /**
-   * This is AWS policy removes read access from S3, leaves S3Guard access up.
-   * This will allow clients to use S3Guard list/HEAD operations, even
-   * the ability to write records, but not actually access the underlying
-   * data.
+   * This is AWS policy removes read access from S3.
    * The client does need {@link RolePolicies#S3_GET_BUCKET_LOCATION} to
    * get the bucket location.
    */
   public static final Policy RESTRICTED_POLICY = policy(
-      DENY_S3_GET_OBJECT, STATEMENT_ALL_DDB, ALLOW_S3_GET_BUCKET_LOCATION
-      );
+      DENY_S3_GET_OBJECT, ALLOW_S3_GET_BUCKET_LOCATION);
 
   private RoleTestUtils() {
   }
@@ -150,6 +146,7 @@ public static void assertTouchForbidden(final FileSystem fs, final Path path)
    * @param roleARN ARN of role
    * @return the new configuration
    */
+  @SuppressWarnings("deprecation")
   public static Configuration newAssumedRoleConfig(
       final Configuration srcConf,
       final String roleARN) {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
index 75ea01c0724c6..d5d62f2cae92c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
@@ -49,7 +49,6 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 import static java.util.Objects.requireNonNull;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assumeSessionTestsEnabled;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
@@ -162,11 +161,7 @@ protected YarnConfiguration createConfiguration() {
     conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
         10_000);
 
-    // turn off DDB for the job resource bucket
     String host = jobResource.getHost();
-    conf.set(
-        String.format("fs.s3a.bucket.%s.metadatastore.impl", host),
-        S3GUARD_METASTORE_NULL);
     // and fix to the main endpoint if the caller has moved
     conf.set(
         String.format("fs.s3a.bucket.%s.endpoint", host), "");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
index 6aed9e7e314e9..9598ef084fa49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.file.AccessDeniedException;
 
@@ -41,7 +42,6 @@
 import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
 import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.S3ClientFactory;
@@ -69,6 +69,7 @@
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
 import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH;
@@ -145,9 +146,14 @@ protected Configuration createConfiguration() {
     // disable if assume role opts are off
     assumeSessionTestsEnabled(conf);
     disableFilesystemCaching(conf);
-    String s3EncryptionMethod =
-        conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM,
-            S3AEncryptionMethods.SSE_KMS.getMethod());
+    String s3EncryptionMethod;
+    try {
+      s3EncryptionMethod =
+          getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod();
+    } catch (IOException e) {
+      throw new UncheckedIOException("Failed to lookup encryption algorithm.",
+          e);
+    }
     String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf);
     removeBaseAndBucketOverrides(conf,
         DELEGATION_TOKEN_BINDING,
@@ -248,6 +254,7 @@ public void testGetDTfromFileSystem() throws Throwable {
   }
 
   @Test
+  @SuppressWarnings("deprecation")
   public void testAddTokensFromFileSystem() throws Throwable {
     describe("verify FileSystem.addDelegationTokens() collects tokens");
     S3AFileSystem fs = getFileSystem();
@@ -323,7 +330,6 @@ public void testDelegatedFileSystem() throws Throwable {
         + " if role restricted, permissions are tightened.");
     S3AFileSystem fs = getFileSystem();
     // force a probe of the remote FS to make sure its endpoint is valid
-    // (this always hits S3, even when S3Guard is enabled)
     fs.getObjectMetadata(new Path("/"));
     readLandsatMetadata(fs);
 
@@ -571,6 +577,7 @@ public void testDelegationBindingMismatch2() throws Throwable {
    * @return result of the HEAD
    * @throws Exception failure
    */
+  @SuppressWarnings("deprecation")
   protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS)
       throws Exception {
     AWSCredentialProviderList testingCreds
@@ -584,6 +591,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS)
     S3ClientFactory.S3ClientCreationParameters parameters = null;
     parameters = new S3ClientFactory.S3ClientCreationParameters()
         .withCredentialSet(testingCreds)
+        .withPathUri(new URI("s3a://localhost/"))
         .withEndpoint(DEFAULT_ENDPOINT)
         .withMetrics(new EmptyS3AStatisticsContext()
             .newStatisticsFromAwsSdk())
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
index 7a70ac95fc4d9..fab7ffdbb76f8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
@@ -249,7 +249,7 @@ public void testCreateWithRenewer() throws Throwable {
    * @return the retrieved DT. This is only for error reporting.
    * @throws IOException failure.
    */
-  @SuppressWarnings("OptionalGetWithoutIsPresent")
+  @SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"})
   protected AbstractS3ATokenIdentifier verifyCredentialPropagation(
       final S3AFileSystem fs,
       final MarshalledCredentials session,
@@ -259,8 +259,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation(
     // clear any credential paths to ensure they don't get picked up and used
     // for authentication.
     unsetHadoopCredentialProviders(conf);
-    conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER,
-        TemporaryAWSCredentialsProvider.NAME);
+    conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.NAME);
     session.setSecretsInConfiguration(conf);
     try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) {
       delegationTokens2.bindToFileSystem(
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/MiniKerberizedHadoopCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/MiniKerberizedHadoopCluster.java
index 0a266dc3eaa6d..280c39b910d03 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/MiniKerberizedHadoopCluster.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/MiniKerberizedHadoopCluster.java
@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.util.Properties;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
index 012b6b9662cdb..9987257527176 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
@@ -18,13 +18,15 @@
 
 package org.apache.hadoop.fs.s3a.commit;
 
+import java.io.File;
 import java.io.IOException;
-import java.io.InterruptedIOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
 import java.util.List;
-import java.util.stream.Collectors;
 
-import com.amazonaws.services.s3.AmazonS3;
 import org.assertj.core.api.Assertions;
+import org.junit.AfterClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,20 +37,24 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.FailureInjectionPolicy;
-import org.apache.hadoop.fs.s3a.InconsistentAmazonS3Client;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.WriteOperationHelper;
-import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSupport;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
 
+import static java.time.temporal.ChronoField.DAY_OF_MONTH;
+import static java.time.temporal.ChronoField.MONTH_OF_YEAR;
+import static java.time.temporal.ChronoField.YEAR;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
@@ -58,47 +64,28 @@
 /**
  * Base test suite for committer operations.
  *
- * By default, these tests enable the inconsistent committer, with
- * a delay of {@link #CONSISTENCY_DELAY}; they may also have throttling
- * enabled/disabled.
- *
- * <b>Important:</b> all filesystem probes will have to wait for
- * the FS inconsistency delays and handle things like throttle exceptions,
- * or disable throttling and fault injection before the probe.
- *
  */
 public abstract class AbstractCommitITest extends AbstractS3ATestBase {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractCommitITest.class);
 
-  protected static final int CONSISTENCY_DELAY = 500;
-  protected static final int CONSISTENCY_PROBE_INTERVAL = 500;
-  protected static final int CONSISTENCY_WAIT = CONSISTENCY_DELAY * 2;
-
-  private InconsistentAmazonS3Client inconsistentClient;
-
+  /**
+   * Job statistics accrued across all test cases.
+   */
+  private static final IOStatisticsSnapshot JOB_STATISTICS =
+      IOStatisticsSupport.snapshotIOStatistics();
 
   /**
-   * Should the inconsistent S3A client be used?
-   * Default value: true.
-   * @return true for inconsistent listing
+   * Helper class for commit operations and assertions.
    */
-  public boolean useInconsistentClient() {
-    return true;
-  }
+  private CommitterTestHelper testHelper;
 
   /**
-   * switch to an inconsistent path if in inconsistent mode.
-   * {@inheritDoc}
+   * Directory for job summary reports.
+   * This should be set up in test suites testing against real object stores.
    */
-  @Override
-  protected Path path(String filepath) throws IOException {
-    return useInconsistentClient() ?
-           super.path(FailureInjectionPolicy.DEFAULT_DELAY_KEY_SUBSTRING
-               + "/" + filepath)
-           : super.path(filepath);
-  }
+  private File reportDir;
 
   /**
    * Creates a configuration for commit operations: commit is enabled in the FS
@@ -115,18 +102,24 @@ protected Configuration createConfiguration() {
         FS_S3A_COMMITTER_NAME,
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE,
         FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES,
-        FAST_UPLOAD_BUFFER);
+        FAST_UPLOAD_BUFFER,
+        S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS);
 
     conf.setBoolean(MAGIC_COMMITTER_ENABLED, DEFAULT_MAGIC_COMMITTER_ENABLED);
     conf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE);
     conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE);
     conf.set(FAST_UPLOAD_BUFFER, FAST_UPLOAD_BUFFER_ARRAY);
-    if (useInconsistentClient()) {
-      enableInconsistentS3Client(conf, CONSISTENCY_DELAY);
-    }
+    // and bind the report dir
+    conf.set(OPT_SUMMARY_REPORT_DIR, reportDir.toURI().toString());
+    conf.setBoolean(S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS, true);
     return conf;
   }
 
+  @AfterClass
+  public static void printStatistics() {
+    LOG.info("Aggregate job statistics {}\n",
+        IOStatisticsLogging.ioStatisticsToPrettyString(JOB_STATISTICS));
+  }
   /**
    * Get the log; can be overridden for test case log.
    * @return a log.
@@ -135,6 +128,36 @@ public Logger log() {
     return LOG;
   }
 
+  /**
+   * Get directory for reports; valid after
+   * setup.
+   * @return where success/failure reports go.
+   */
+  protected File getReportDir() {
+    return reportDir;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    // set the manifest committer to a localfs path for reports across
+    // all threads.
+    // do this before superclass setup so reportDir is non-null there
+    // and can be used in creating the configuration.
+    reportDir = new File(getProjectBuildDir(), "reports");
+    reportDir.mkdirs();
+
+    super.setup();
+    testHelper = new CommitterTestHelper(getFileSystem());
+  }
+
+  /**
+   * Get helper class.
+   * @return helper; only valid after setup.
+   */
+  public CommitterTestHelper getTestHelper() {
+    return testHelper;
+  }
+
   /***
    * Bind to the named committer.
    *
@@ -152,7 +175,6 @@ protected void bindCommitter(Configuration conf, String factory,
 
   /**
    * Clean up a directory.
-   * Waits for consistency if needed
    * @param dir directory
    * @param conf configuration
    * @throws IOException failure
@@ -161,31 +183,14 @@ public void rmdir(Path dir, Configuration conf) throws IOException {
     if (dir != null) {
       describe("deleting %s", dir);
       FileSystem fs = dir.getFileSystem(conf);
-      waitForConsistency();
+
       fs.delete(dir, true);
-      waitForConsistency();
-    }
-  }
 
-  /**
-   * Setup will use inconsistent client if {@link #useInconsistentClient()}
-   * is true.
-   * @throws Exception failure.
-   */
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    if (useInconsistentClient()) {
-      AmazonS3 client = getFileSystem()
-          .getAmazonS3ClientForTesting("fault injection");
-      if (client instanceof InconsistentAmazonS3Client) {
-        inconsistentClient = (InconsistentAmazonS3Client) client;
-      }
     }
   }
 
   /**
-   * Create a random Job ID using the fork ID as part of the number.
+   * Create a random Job ID using the fork ID and the current time.
    * @return fork ID string in a format parseable by Jobs
    * @throws Exception failure
    */
@@ -195,7 +200,14 @@ public static String randomJobId() throws Exception {
     String trailingDigits = testUniqueForkId.substring(l - 4, l);
     try {
       int digitValue = Integer.valueOf(trailingDigits);
-      return String.format("20070712%04d_%04d",
+      DateTimeFormatter formatter = new DateTimeFormatterBuilder()
+          .parseCaseInsensitive()
+          .appendValue(YEAR, 4)
+          .appendValue(MONTH_OF_YEAR, 2)
+          .appendValue(DAY_OF_MONTH, 2)
+          .toFormatter();
+      return String.format("%s%04d_%04d",
+          LocalDateTime.now().format(formatter),
           (long)(Math.random() * 1000),
           digitValue);
     } catch (NumberFormatException e) {
@@ -203,102 +215,15 @@ public static String randomJobId() throws Exception {
     }
   }
 
-  /**
-   * Teardown waits for the consistency delay and resets failure count, so
-   * FS is stable, before the superclass teardown is called. This
-   * should clean things up better.
-   * @throws Exception failure.
-   */
-  @Override
-  public void teardown() throws Exception {
-    Thread.currentThread().setName("teardown");
-    LOG.info("AbstractCommitITest::teardown");
-    waitForConsistency();
-    // make sure there are no failures any more
-    resetFailures();
-    super.teardown();
-  }
-
-  /**
-   * Wait a multiple of the inconsistency delay for things to stabilize;
-   * no-op if the consistent client is used.
-   * @throws InterruptedIOException if the sleep is interrupted
-   */
-  protected void waitForConsistency() throws InterruptedIOException {
-    if (useInconsistentClient() && inconsistentClient != null) {
-      try {
-        Thread.sleep(2* inconsistentClient.getDelayKeyMsec());
-      } catch (InterruptedException e) {
-        throw (InterruptedIOException)
-            (new InterruptedIOException("while waiting for consistency: " + e)
-                .initCause(e));
-      }
-    }
-  }
-
-  /**
-   * Set the throttling factor on requests.
-   * @param p probability of a throttling occurring: 0-1.0
-   */
-  protected void setThrottling(float p) {
-    if (inconsistentClient != null) {
-      inconsistentClient.setThrottleProbability(p);
-    }
-  }
-
-  /**
-   * Set the throttling factor on requests and number of calls to throttle.
-   * @param p probability of a throttling occurring: 0-1.0
-   * @param limit limit to number of calls which fail
-   */
-  protected void setThrottling(float p, int limit) {
-    if (inconsistentClient != null) {
-      inconsistentClient.setThrottleProbability(p);
-    }
-    setFailureLimit(limit);
-  }
-
-  /**
-   * Turn off throttling.
-   */
-  protected void resetFailures() {
-    if (inconsistentClient != null) {
-      setThrottling(0, 0);
-    }
-  }
-
-  /**
-   * Set failure limit.
-   * @param limit limit to number of calls which fail
-   */
-  private void setFailureLimit(int limit) {
-    if (inconsistentClient != null) {
-      inconsistentClient.setFailureLimit(limit);
-    }
-  }
-
   /**
    * Abort all multipart uploads under a path.
    * @param path path for uploads to abort; may be null
    * @return a count of aborts
    * @throws IOException trouble.
    */
-  protected int abortMultipartUploadsUnderPath(Path path) throws IOException {
-    S3AFileSystem fs = getFileSystem();
-    if (fs != null && path != null) {
-      String key = fs.pathToKey(path);
-      int count = 0;
-      try (AuditSpan span = span()) {
-        WriteOperationHelper writeOps = fs.getWriteOperationHelper();
-        count = writeOps.abortMultipartUploadsUnderPath(key);
-        if (count > 0) {
-          log().info("Multipart uploads deleted: {}", count);
-        }
-      }
-      return count;
-    } else {
-      return 0;
-    }
+  protected void abortMultipartUploadsUnderPath(Path path) throws IOException {
+    getTestHelper()
+        .abortMultipartUploadsUnderPath(path);
   }
 
   /**
@@ -320,10 +245,9 @@ protected void assertMultipartUploadsPending(Path path) throws IOException {
   protected void assertNoMultipartUploadsPending(Path path) throws IOException {
     List<String> uploads = listMultipartUploads(getFileSystem(),
         pathToPrefix(path));
-    if (!uploads.isEmpty()) {
-      String result = uploads.stream().collect(Collectors.joining("\n"));
-      fail("Multipart uploads in progress under " + path + " \n" + result);
-    }
+    Assertions.assertThat(uploads)
+        .describedAs("Multipart uploads in progress under " + path)
+        .isEmpty();
   }
 
   /**
@@ -478,11 +402,20 @@ public static SuccessData validateSuccessFile(final Path outputPath,
           .describedAs("JobID in " + commitDetails)
           .isEqualTo(jobId);
     }
+    // also load as a manifest success data file
+    // to verify consistency and that the CLI tool works.
+    Path success = new Path(outputPath, _SUCCESS);
+    final ManifestPrinter showManifest = new ManifestPrinter();
+    ManifestSuccessData manifestSuccessData =
+        showManifest.loadAndPrintManifest(fs, success);
+
     return successData;
   }
 
   /**
    * Load a success file; fail if the file is empty/nonexistent.
+   * The statistics in {@link #JOB_STATISTICS} are updated with
+   * the statistics from the success file
    * @param fs filesystem
    * @param outputPath directory containing the success file.
    * @param origin origin of the file
@@ -512,6 +445,8 @@ public static SuccessData loadSuccessFile(final FileSystem fs,
     String body = ContractTestUtils.readUTF8(fs, success, -1);
     LOG.info("Loading committer success file {}. Actual contents=\n{}", success,
         body);
-    return SuccessData.load(fs, success);
+    SuccessData successData = SuccessData.load(fs, success);
+    JOB_STATISTICS.aggregate(successData.getIOStatistics());
+    return successData;
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
index 1e5a9582b2c2d..b193cca03db00 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
@@ -71,6 +71,8 @@
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_INTELLIGENT_TIERING;
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter.E_SELF_GENERATED_JOB_UUID;
@@ -82,7 +84,7 @@
 import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_TASKS_SUCCEEDED;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
-import static org.apache.hadoop.test.LambdaTestUtils.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Test the job/task commit actions of an S3A Committer, including trying to
@@ -93,7 +95,7 @@
  * This is a complex test suite as it tries to explore the full lifecycle
  * of committers, and is designed for subclassing.
  */
-@SuppressWarnings({"unchecked", "ThrowableNotThrown", "unused"})
+@SuppressWarnings({"unchecked", "unused"})
 public abstract class AbstractITCommitProtocol extends AbstractCommitITest {
   private Path outDir;
 
@@ -124,7 +126,7 @@ public abstract class AbstractITCommitProtocol extends AbstractCommitITest {
   private static final Text VAL_2 = new Text("val2");
 
   /** A job to abort in test case teardown. */
-  private List<JobData> abortInTeardown = new ArrayList<>(1);
+  private final List<JobData> abortInTeardown = new ArrayList<>(1);
 
   private final StandardCommitterFactory
       standardCommitterFactory = new StandardCommitterFactory();
@@ -560,7 +562,7 @@ protected void commit(AbstractS3ACommitter committer,
       describe("\ncommitting job");
       committer.commitJob(jContext);
       describe("commit complete\n");
-      verifyCommitterHasNoThreads(committer);
+
     }
   }
 
@@ -605,7 +607,7 @@ public void executeWork(String name,
   @Test
   @SuppressWarnings("deprecation")
   public void testRecoveryAndCleanup() throws Exception {
-    describe("Test (unsupported) task recovery.");
+    describe("Test (Unsupported) task recovery.");
     JobData jobData = startJob(true);
     TaskAttemptContext tContext = jobData.tContext;
     AbstractS3ACommitter committer = jobData.committer;
@@ -615,6 +617,9 @@ public void testRecoveryAndCleanup() throws Exception {
     assertNotNull("null outputPath in committer " + committer,
         committer.getOutputPath());
 
+    // note the task attempt path.
+    Path job1TaskAttempt0Path = committer.getTaskAttemptPath(tContext);
+
     // Commit the task. This will promote data and metadata to where
     // job commits will pick it up on commit or abort.
     commitTask(committer, tContext);
@@ -634,6 +639,15 @@ public void testRecoveryAndCleanup() throws Exception {
     intercept(PathCommitException.class, "recover",
         () -> committer2.recoverTask(tContext2));
 
+    // the new task attempt path is different from the first, because the
+    // job attempt counter is used in the path
+    final Path job2TaskAttempt0Path = committer2.getTaskAttemptPath(tContext2);
+    LOG.info("Job attempt 1 task attempt path {}; attempt 2 path {}",
+        job1TaskAttempt0Path, job2TaskAttempt0Path);
+    assertNotEquals("Task attempt paths must differ",
+        job1TaskAttempt0Path,
+        job2TaskAttempt0Path);
+
     // at this point, task attempt 0 has failed to recover
     // it should be abortable though. This will be a no-op as it already
     // committed
@@ -643,7 +657,7 @@ public void testRecoveryAndCleanup() throws Exception {
     committer2.abortJob(jContext2, JobStatus.State.KILLED);
     // now, state of system may still have pending data
     assertNoMultipartUploadsPending(outDir);
-    verifyCommitterHasNoThreads(committer2);
+
   }
 
   protected void assertTaskAttemptPathDoesNotExist(
@@ -696,32 +710,31 @@ private void validateContent(Path dir,
   }
 
   /**
-   * Identify any path under the directory which begins with the
-   * {@code "part-m-00000"} sequence. There's some compensation for
-   * eventual consistency here.
-   * @param dir directory to scan
-   * @return the full path
-   * @throws FileNotFoundException the path is missing.
+   * Verify storage class of output file matches the expected storage class.
+   * @param dir output directory.
+   * @param expectedStorageClass expected storage class value.
    * @throws Exception failure.
    */
-  protected Path getPart0000(final Path dir) throws Exception {
-    final FileSystem fs = dir.getFileSystem(getConfiguration());
-    return eventually(CONSISTENCY_WAIT, CONSISTENCY_PROBE_INTERVAL,
-        () -> getPart0000Immediately(fs, dir));
+  private void validateStorageClass(Path dir, String expectedStorageClass) throws Exception {
+    Path expectedFile = getPart0000(dir);
+    S3AFileSystem fs = getFileSystem();
+    String actualStorageClass = fs.getObjectMetadata(expectedFile).getStorageClass();
+
+    Assertions.assertThat(actualStorageClass)
+        .describedAs("Storage class of object %s", expectedFile)
+        .isEqualToIgnoringCase(expectedStorageClass);
   }
 
   /**
    * Identify any path under the directory which begins with the
-   * {@code "part-m-00000"} sequence. There's some compensation for
-   * eventual consistency here.
-   * @param fs FS to probe
+   * {@code "part-m-00000"} sequence.
    * @param dir directory to scan
    * @return the full path
    * @throws FileNotFoundException the path is missing.
-   * @throws IOException failure.
+   * @throws Exception failure.
    */
-  private Path getPart0000Immediately(FileSystem fs, Path dir)
-      throws IOException {
+  protected Path getPart0000(final Path dir) throws Exception {
+    final FileSystem fs = dir.getFileSystem(getConfiguration());
     FileStatus[] statuses = fs.listStatus(dir,
         path -> path.getName().startsWith(PART_00000));
     if (statuses.length != 1) {
@@ -746,7 +759,9 @@ private void validateMapFileOutputContent(
     assertPathExists("Map output", expectedMapDir);
     assertIsDirectory(expectedMapDir);
     FileStatus[] files = fs.listStatus(expectedMapDir);
-    assertTrue("No files found in " + expectedMapDir, files.length > 0);
+    Assertions.assertThat(files)
+        .describedAs("Files found in " + expectedMapDir)
+        .hasSizeGreaterThan(0);
     assertPathExists("index file in " + expectedMapDir,
         new Path(expectedMapDir, MapFile.INDEX_FILE_NAME));
     assertPathExists("data file in " + expectedMapDir,
@@ -791,12 +806,12 @@ public void testCommitLifecycle() throws Exception {
     commitTask(committer, tContext);
 
     // this is only task commit; there MUST be no part- files in the dest dir
-    waitForConsistency();
+
     try {
       applyLocatedFiles(getFileSystem().listFiles(outDir, false),
-          (status) ->
-              assertFalse("task committed file to dest :" + status,
-                  status.getPath().toString().contains("part")));
+          (status) -> Assertions.assertThat(status.getPath().toString())
+              .describedAs("task committed file to dest :" + status)
+              .doesNotContain("part"));
     } catch (FileNotFoundException ignored) {
       log().info("Outdir {} is not created by task commit phase ",
           outDir);
@@ -813,6 +828,41 @@ public void testCommitLifecycle() throws Exception {
     assertNoMultipartUploadsPending(outDir);
   }
 
+  @Test
+  public void testCommitWithStorageClassConfig() throws Exception {
+    describe("Commit with specific storage class configuration;" +
+        " expect the final file has correct storage class.");
+
+    Configuration conf = getConfiguration();
+    skipIfStorageClassTestsDisabled(conf);
+    conf.set(STORAGE_CLASS, STORAGE_CLASS_INTELLIGENT_TIERING);
+
+    JobData jobData = startJob(false);
+    JobContext jContext = jobData.jContext;
+    TaskAttemptContext tContext = jobData.tContext;
+    AbstractS3ACommitter committer = jobData.committer;
+    validateTaskAttemptWorkingDirectory(committer, tContext);
+
+    // write output
+    writeTextOutput(tContext);
+
+    // commit task
+    dumpMultipartUploads();
+    commitTask(committer, tContext);
+
+    // commit job
+    assertMultipartUploadsPending(outDir);
+    commitJob(committer, jContext);
+
+    // validate output
+    validateContent(outDir, shouldExpectSuccessMarker(),
+        committer.getUUID());
+    assertNoMultipartUploadsPending(outDir);
+
+    // validate storage class
+    validateStorageClass(outDir, STORAGE_CLASS_INTELLIGENT_TIERING);
+  }
+
   @Test
   public void testCommitterWithDuplicatedCommit() throws Exception {
     describe("Call a task then job commit twice;" +
@@ -1020,7 +1070,7 @@ public void testMapFileOutputCommitter() throws Exception {
     // do commit
     commit(committer, jContext, tContext);
     S3AFileSystem fs = getFileSystem();
-    waitForConsistency();
+
     lsR(fs, outDir, true);
     String ls = ls(outDir);
     describe("\nvalidating");
@@ -1035,27 +1085,34 @@ public void testMapFileOutputCommitter() throws Exception {
     // hidden filenames (_ or . prefixes)
     describe("listing");
     FileStatus[] filtered = fs.listStatus(outDir, HIDDEN_FILE_FILTER);
-    assertEquals("listed children under " + ls,
-        1, filtered.length);
+    Assertions.assertThat(filtered)
+        .describedAs("listed children under " + ls)
+        .hasSize(1);
     FileStatus fileStatus = filtered[0];
-    assertTrue("Not the part file: " + fileStatus,
-        fileStatus.getPath().getName().startsWith(PART_00000));
+    Assertions.assertThat(fileStatus.getPath().getName())
+        .describedAs("Not a part file: " + fileStatus)
+        .startsWith(PART_00000);
 
     describe("getReaders()");
-    assertEquals("Number of MapFile.Reader entries with shared FS "
-            + outDir + " : " + ls,
-        1, getReaders(fs, outDir, conf).length);
+    Assertions.assertThat(getReaders(fs, outDir, conf))
+        .describedAs("Number of MapFile.Reader entries with shared FS %s: %s",
+            outDir, ls)
+        .hasSize(1);
 
     describe("getReaders(new FS)");
     FileSystem fs2 = FileSystem.get(outDir.toUri(), conf);
-    assertEquals("Number of MapFile.Reader entries with shared FS2 "
-            + outDir + " : " + ls,
-        1, getReaders(fs2, outDir, conf).length);
+    Assertions.assertThat(getReaders(fs2, outDir, conf))
+        .describedAs("Number of MapFile.Reader entries with shared FS2 %s: %s",
+            outDir, ls)
+        .hasSize(1);
 
     describe("MapFileOutputFormat.getReaders");
-    assertEquals("Number of MapFile.Reader entries with new FS in "
-            + outDir + " : " + ls,
-        1, MapFileOutputFormat.getReaders(outDir, conf).length);
+
+    Assertions.assertThat(MapFileOutputFormat.getReaders(outDir, conf))
+        .describedAs("Number of MapFile.Reader entries with new FS in %s: %s",
+            outDir, ls)
+        .hasSize(1);
+
   }
 
   /** Open the output generated by this format. */
@@ -1129,7 +1186,7 @@ public void testAbortTaskThenJob() throws Exception {
 
     committer.abortJob(jobData.jContext, JobStatus.State.FAILED);
     assertJobAbortCleanedUp(jobData);
-    verifyCommitterHasNoThreads(committer);
+
   }
 
   /**
@@ -1183,7 +1240,7 @@ public void testFailAbort() throws Exception {
     // try again; expect abort to be idempotent.
     committer.abortJob(jContext, JobStatus.State.FAILED);
     assertNoMultipartUploadsPending(outDir);
-    verifyCommitterHasNoThreads(committer);
+
   }
 
   public void assertPart0000DoesNotExist(Path dir) throws Exception {
@@ -1397,7 +1454,7 @@ public void testAMWorkflow() throws Throwable {
     AbstractS3ACommitter committer2 = (AbstractS3ACommitter)
         outputFormat.getOutputCommitter(newAttempt);
     committer2.abortTask(tContext);
-    verifyCommitterHasNoThreads(committer2);
+
     assertNoMultipartUploadsPending(getOutDir());
   }
 
@@ -1741,11 +1798,10 @@ public void testS3ACommitterFactoryBinding() throws Throwable {
     conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
     TaskAttemptContext tContext = new TaskAttemptContextImpl(conf,
         taskAttempt0);
-    String name = getCommitterName();
     S3ACommitterFactory factory = new S3ACommitterFactory();
-    assertEquals("Wrong committer from factory",
-        createCommitter(outDir, tContext).getClass(),
-        factory.createOutputCommitter(outDir, tContext).getClass());
+    Assertions.assertThat(factory.createOutputCommitter(outDir, tContext).getClass())
+        .describedAs("Committer from factory with name %s", getCommitterName())
+        .isEqualTo(createCommitter(outDir, tContext).getClass());
   }
 
   /**
@@ -1794,7 +1850,7 @@ protected void validateTaskAttemptWorkingDirectory(
   protected void commitTask(final AbstractS3ACommitter committer,
       final TaskAttemptContext tContext) throws IOException {
     committer.commitTask(tContext);
-    verifyCommitterHasNoThreads(committer);
+
   }
 
   /**
@@ -1806,15 +1862,7 @@ protected void commitTask(final AbstractS3ACommitter committer,
   protected void commitJob(final AbstractS3ACommitter committer,
       final JobContext jContext) throws IOException {
     committer.commitJob(jContext);
-    verifyCommitterHasNoThreads(committer);
-  }
 
-  /**
-   * Verify that the committer does not have a thread pool.
-   * @param committer committer to validate.
-   */
-  protected void verifyCommitterHasNoThreads(AbstractS3ACommitter committer) {
-    assertFalse("Committer has an active thread pool",
-        committer.hasThreadPool());
   }
+
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java
index 1b752a8db97f1..aa44c171d7715 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractYarnClusterITest.java
@@ -39,7 +39,7 @@
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool;
@@ -156,7 +156,7 @@ public void terminate() {
   /**
    * Create the cluster binding.
    * The configuration will be patched by propagating down options
-   * from the maven build (S3Guard binding etc) and turning off unwanted
+   * from the maven build and turning off unwanted
    * YARN features.
    *
    * If an HDFS cluster is requested,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java
new file mode 100644
index 0000000000000..cd703f96da8dd
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.MultipartTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.XA_MAGIC_MARKER;
+import static org.apache.hadoop.fs.s3a.commit.impl.CommitOperations.extractMagicFileLength;
+
+/**
+ * Helper for committer tests: extra assertions and the like.
+ */
+public class CommitterTestHelper {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CommitterTestHelper.class);
+
+  /**
+   * Filesystem under test.
+   */
+  private final S3AFileSystem fileSystem;
+
+  /**
+   * Constructor.
+   * @param fileSystem filesystem to work with.
+   */
+  public CommitterTestHelper(S3AFileSystem fileSystem) {
+    this.fileSystem = requireNonNull(fileSystem);
+  }
+
+  /**
+   * Get the filesystem.
+   * @return the filesystem.
+   */
+  public S3AFileSystem getFileSystem() {
+    return fileSystem;
+  }
+
+  /**
+   * Assert a path refers to a marker file of an expected length;
+   * the length is extracted from the custom header.
+   * @param path magic file.
+   * @param dataSize expected data size
+   * @throws IOException IO failure
+   */
+  public void assertIsMarkerFile(Path path, long dataSize) throws IOException {
+    final S3AFileSystem fs = getFileSystem();
+    FileStatus status = verifyPathExists(fs,
+        "uploaded file commit", path);
+    Assertions.assertThat(status.getLen())
+        .describedAs("Marker File file %s: %s", path, status)
+        .isEqualTo(0);
+    Assertions.assertThat(extractMagicFileLength(fs, path))
+        .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + path)
+        .isNotEmpty()
+        .hasValue(dataSize);
+  }
+
+  /**
+   * Assert a file does not have the magic marker header.
+   * @param path magic file.
+   * @throws IOException IO failure
+   */
+  public void assertFileLacksMarkerHeader(Path path) throws IOException {
+    Assertions.assertThat(extractMagicFileLength(getFileSystem(),
+            path))
+        .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + path)
+        .isEmpty();
+  }
+
+  /**
+   * Create a new path which has the same filename as the dest file, but
+   * is in a magic directory under the destination dir.
+   * @param destFile final destination file
+   * @return magic path
+   */
+  public static Path makeMagic(Path destFile) {
+    return new Path(destFile.getParent(),
+        MAGIC + '/' + BASE + "/" + destFile.getName());
+  }
+
+  /**
+   * Assert that an output stream is magic.
+   * @param stream stream to probe.
+   */
+  public static void assertIsMagicStream(final FSDataOutputStream stream) {
+    Assertions.assertThat(stream.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT))
+        .describedAs("Stream capability %s in stream %s",
+            STREAM_CAPABILITY_MAGIC_OUTPUT, stream)
+        .isTrue();
+  }
+
+  /**
+   * Abort all multipart uploads under a path.
+   * @param path path for uploads to abort; may be null
+   * @return a count of aborts
+   * @throws IOException trouble.
+   */
+  public void abortMultipartUploadsUnderPath(Path path) {
+
+    MultipartTestUtils.clearAnyUploads(getFileSystem(), path);
+  }
+
+  /**
+   * Get a list of all pending uploads under a prefix, one which can be printed.
+   * @param prefix prefix to look under
+   * @return possibly empty list
+   * @throws IOException IO failure.
+   */
+  public List<String> listMultipartUploads(
+      String prefix) throws IOException {
+
+    return MultipartTestUtils.listMultipartUploads(getFileSystem(), prefix);
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java
new file mode 100644
index 0000000000000..d0c86b738ca10
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
+import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
+import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
+
+import static org.apache.hadoop.fs.s3a.Statistic.ACTION_HTTP_GET_REQUEST;
+import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_FILES_CREATED;
+import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_MARKER_PUT;
+import static org.apache.hadoop.fs.s3a.Statistic.DIRECTORIES_CREATED;
+import static org.apache.hadoop.fs.s3a.Statistic.FAKE_DIRECTORIES_DELETED;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_LIST_REQUEST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_INITIATED;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_REQUESTS;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
+import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.assertIsMagicStream;
+import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.makeMagic;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_FILES_LIST_OP;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_OPERATION;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
+
+/**
+ * Assert cost of commit operations;
+ * <ol>
+ *   <li>Even on marker deleting filesystems,
+ *       operations under magic dirs do not trigger marker deletion.</li>
+ *   <li>Loading pending files from FileStatus entries skips HEAD checks.</li>
+ *   <li>Mkdir under magic dirs doesn't check ancestor or dest type</li>
+ * </ol>
+ */
+public class ITestCommitOperationCost extends AbstractS3ACostTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestCommitOperationCost.class);
+
+  /**
+   * Helper for the tests.
+   */
+  private CommitterTestHelper testHelper;
+
+  /**
+   * Create with markers kept, always.
+   */
+  public ITestCommitOperationCost() {
+    super(false);
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    testHelper = new CommitterTestHelper(getFileSystem());
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    try {
+      if (testHelper != null) {
+        testHelper.abortMultipartUploadsUnderPath(methodPath());
+      }
+    } finally {
+      super.teardown();
+    }
+  }
+
+  /**
+   * Get a method-relative path.
+   * @param filename filename
+   * @return new path
+   * @throws IOException failure to create/parse the path.
+   */
+  private Path methodSubPath(String filename) throws IOException {
+    return new Path(methodPath(), filename);
+  }
+
+  /**
+   * Return the FS IOStats, prettified.
+   * @return string for assertions.
+   */
+  protected String fileSystemIOStats() {
+    return ioStatisticsToPrettyString(getFileSystem().getIOStatistics());
+  }
+
+  @Test
+  public void testMagicMkdir() throws Throwable {
+    describe("Mkdirs __magic always skips dir marker deletion");
+    S3AFileSystem fs = getFileSystem();
+    Path baseDir = methodPath();
+    // create dest dir marker, always
+    fs.mkdirs(baseDir);
+    Path magicDir = new Path(baseDir, MAGIC);
+    verifyMetrics(() -> {
+      fs.mkdirs(magicDir);
+      return fileSystemIOStats();
+    },
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 0),
+        with(DIRECTORIES_CREATED, 1));
+    verifyMetrics(() -> {
+      fs.delete(magicDir, true);
+      return fileSystemIOStats();
+    },
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 1),
+        with(DIRECTORIES_CREATED, 0));
+    assertPathExists("parent", baseDir);
+  }
+
+  /**
+   * When a magic subdir is deleted, parent dirs are not recreated.
+   */
+  @Test
+  public void testMagicMkdirs() throws Throwable {
+    describe("Mkdirs __magic/subdir always skips dir marker deletion");
+    S3AFileSystem fs = getFileSystem();
+    Path baseDir = methodPath();
+    Path magicDir = new Path(baseDir, MAGIC);
+    fs.delete(baseDir, true);
+
+    Path magicSubdir = new Path(magicDir, "subdir");
+    verifyMetrics(() -> {
+      fs.mkdirs(magicSubdir, FsPermission.getDirDefault());
+      return "after mkdirs " + fileSystemIOStats();
+    },
+        always(LIST_OPERATION),
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 0),
+        with(DIRECTORIES_CREATED, 1));
+    assertPathExists("magicSubdir", magicSubdir);
+
+    verifyMetrics(() -> {
+      fs.delete(magicSubdir, true);
+      return "after delete " + fileSystemIOStats();
+    },
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 1),
+        with(OBJECT_LIST_REQUEST, 1),
+        with(OBJECT_METADATA_REQUESTS, 1),
+        with(DIRECTORIES_CREATED, 0));
+    // no marker dir creation
+    assertPathDoesNotExist("magicDir", magicDir);
+    assertPathDoesNotExist("baseDir", baseDir);
+  }
+
+  /**
+   * Active stream; a field is used so closures can write to
+   * it.
+   */
+  private FSDataOutputStream stream;
+
+  /**
+   * Abort any active stream.
+   * @throws IOException failure
+   */
+  private void abortActiveStream() throws IOException {
+    if (stream != null) {
+      stream.abort();
+      stream.close();
+    }
+  }
+
+  @Test
+  public void testCostOfCreatingMagicFile() throws Throwable {
+    describe("Files created under magic paths skip existence checks and marker deletes");
+    S3AFileSystem fs = getFileSystem();
+    Path destFile = methodSubPath("file.txt");
+    fs.delete(destFile.getParent(), true);
+    Path magicDest = makeMagic(destFile);
+
+    // when the file is created, there is no check for overwrites
+    // or the dest being a directory, even if overwrite=false
+    try {
+      verifyMetrics(() -> {
+        stream = fs.create(magicDest, false);
+        return stream.toString();
+      },
+          always(NO_HEAD_OR_LIST),
+          with(COMMITTER_MAGIC_FILES_CREATED, 1),
+          with(COMMITTER_MAGIC_MARKER_PUT, 0),
+          with(OBJECT_MULTIPART_UPLOAD_INITIATED, 1));
+      assertIsMagicStream(stream);
+
+      stream.write("hello".getBytes(StandardCharsets.UTF_8));
+
+      // when closing, there will be no directories deleted
+      // we do expect two PUT requests, because the marker and manifests
+      // are both written
+      LOG.info("closing magic stream to {}", magicDest);
+      verifyMetrics(() -> {
+        stream.close();
+        return stream.toString();
+      },
+          always(NO_HEAD_OR_LIST),
+          with(OBJECT_PUT_REQUESTS, 2),
+          with(COMMITTER_MAGIC_MARKER_PUT, 2),
+          with(OBJECT_BULK_DELETE_REQUEST, 0),
+          with(OBJECT_DELETE_REQUEST, 0));
+
+    } catch (Exception e) {
+      abortActiveStream();
+      throw e;
+    }
+    // list the manifests
+    final CommitOperations commitOperations = new CommitOperations(fs);
+    List<LocatedFileStatus> pending = verifyMetrics(() ->
+            toList(commitOperations.
+                locateAllSinglePendingCommits(magicDest.getParent(), false)),
+        always(LIST_FILES_LIST_OP));
+    Assertions.assertThat(pending)
+        .describedAs("pending commits")
+        .hasSize(1);
+
+    // load the only pending commit
+    SinglePendingCommit singleCommit = verifyMetrics(() ->
+            PersistentCommitData.load(fs,
+                pending.get(0),
+                SinglePendingCommit.serializer()),
+        always(NO_HEAD_OR_LIST),
+        with(ACTION_HTTP_GET_REQUEST, 1));
+
+    // commit it through the commit operations.
+    verifyMetrics(() -> {
+      commitOperations.commitOrFail(singleCommit);
+      return ioStatisticsToPrettyString(
+          commitOperations.getIOStatistics());
+    },
+        always(NO_HEAD_OR_LIST),  // no probes for the dest path
+        with(FAKE_DIRECTORIES_DELETED, 0),  // no fake dirs
+        with(OBJECT_DELETE_REQUEST, 0)); // no deletes
+
+    LOG.info("Final Statistics {}",
+        IOStatisticsLogging.ioStatisticsToPrettyString(stream.getIOStatistics()));
+  }
+
+  /**
+   * saving pending files MUST NOT trigger HEAD/LIST calls
+   * when created under a magic path; when opening
+   * with an S3AFileStatus the HEAD will be skipped too.
+   */
+  @Test
+  public void testCostOfSavingLoadingPendingFile() throws Throwable {
+    describe("Verify costs of saving .pending file under a magic path");
+
+    S3AFileSystem fs = getFileSystem();
+    Path partDir = methodSubPath("file.pending");
+    Path destFile = new Path(partDir, "file.pending");
+    Path magicDest = makeMagic(destFile);
+    // create a pending file with minimal values needed
+    // for validation to work
+    final SinglePendingCommit commit = new SinglePendingCommit();
+    commit.touch(System.currentTimeMillis());
+    commit.setUri(destFile.toUri().toString());
+    commit.setBucket(fs.getBucket());
+    commit.setLength(0);
+    commit.setDestinationKey(fs.pathToKey(destFile));
+    commit.setUploadId("uploadId");
+    commit.setEtags(new ArrayList<>());
+    // fail fast if the commit data is incomplete
+    commit.validate();
+
+    // save the file: no checks will be made
+    verifyMetrics(() -> {
+          commit.save(fs, magicDest,
+              SinglePendingCommit.serializer());
+          return commit.toString();
+        },
+        with(COMMITTER_MAGIC_FILES_CREATED, 0),
+        always(NO_HEAD_OR_LIST),
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 0)
+    );
+
+    LOG.info("File written; Validating");
+    testHelper.assertFileLacksMarkerHeader(magicDest);
+    FileStatus status = fs.getFileStatus(magicDest);
+
+    LOG.info("Reading file {}", status);
+    // opening a file with a status passed in will skip the HEAD
+    verifyMetrics(() ->
+            PersistentCommitData.load(fs, status, SinglePendingCommit.serializer()),
+        always(NO_HEAD_OR_LIST),
+        with(ACTION_HTTP_GET_REQUEST, 1));
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
index 2bc6434ccd494..a76a65be8bb3e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
@@ -23,9 +23,8 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
 
-import com.amazonaws.services.s3.model.PartETag;
-import org.apache.hadoop.util.Lists;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -34,13 +33,15 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.auth.ProgressCounter;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker;
 import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitter;
 import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory;
@@ -49,20 +50,22 @@
 import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
 import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.Lists;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
-import static org.apache.hadoop.fs.s3a.commit.CommitOperations.extractMagicFileLength;
 import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.assertIsMagicStream;
 import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
 import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS;
 import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Test the low-level binding of the S3A FS to the magic commit mechanism,
  * and handling of the commit operations.
- * This is done with an inconsistent client.
  */
 public class ITestCommitOperations extends AbstractCommitITest {
 
@@ -71,22 +74,9 @@ public class ITestCommitOperations extends AbstractCommitITest {
   private static final byte[] DATASET = dataset(1000, 'a', 32);
   private static final String S3A_FACTORY_KEY = String.format(
       COMMITTER_FACTORY_SCHEME_PATTERN, "s3a");
-  private ProgressCounter progress;
+  private static final String JOB_ID = UUID.randomUUID().toString();
 
-  /**
-   * A compile time flag which allows you to disable failure reset before
-   * assertions and teardown.
-   * As S3A is now required to be resilient to failure on all FS operations,
-   * setting it to false ensures that even the assertions are checking
-   * the resilience codepaths.
-   */
-  private static final boolean RESET_FAILURES_ENABLED = false;
-
-  private static final float HIGH_THROTTLE = 0.25f;
-
-  private static final float FULL_THROTTLE = 1.0f;
-
-  private static final int STANDARD_FAILURE_LIMIT = 2;
+  private ProgressCounter progress;
 
   @Override
   protected Configuration createConfiguration() {
@@ -96,18 +86,11 @@ protected Configuration createConfiguration() {
     return conf;
   }
 
-  @Override
-  public boolean useInconsistentClient() {
-    return true;
-  }
-
   @Override
   public void setup() throws Exception {
     FileSystem.closeAll();
     super.setup();
     verifyIsMagicCommitFS(getFileSystem());
-    // abort,; rethrow on failure
-    setThrottling(HIGH_THROTTLE, STANDARD_FAILURE_LIMIT);
     progress = new ProgressCounter();
     progress.assertCount("progress", 0);
   }
@@ -118,11 +101,13 @@ public void testCreateTrackerNormalPath() throws Throwable {
     MagicCommitIntegration integration
         = new MagicCommitIntegration(fs, true);
     String filename = "notdelayed.txt";
-    Path destFile = methodPath(filename);
+    Path destFile = methodSubPath(filename);
     String origKey = fs.pathToKey(destFile);
-    PutTracker tracker = integration.createTracker(destFile, origKey);
-    assertFalse("wrong type: " + tracker + " for " + destFile,
-        tracker instanceof MagicCommitTracker);
+    PutTracker tracker = integration.createTracker(destFile, origKey,
+        EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS);
+    Assertions.assertThat(tracker)
+        .describedAs("Tracker for %s", destFile)
+        .isNotInstanceOf(MagicCommitTracker.class);
   }
 
   /**
@@ -135,36 +120,45 @@ public void testCreateTrackerMagicPath() throws Throwable {
     MagicCommitIntegration integration
         = new MagicCommitIntegration(fs, true);
     String filename = "delayed.txt";
-    Path destFile = methodPath(filename);
+    Path destFile = methodSubPath(filename);
     String origKey = fs.pathToKey(destFile);
     Path pendingPath = makeMagic(destFile);
     verifyIsMagicCommitPath(fs, pendingPath);
     String pendingPathKey = fs.pathToKey(pendingPath);
-    assertTrue("wrong path of " + pendingPathKey,
-        pendingPathKey.endsWith(filename));
+    Assertions.assertThat(pendingPathKey)
+        .describedAs("pending path")
+        .endsWith(filename);
     final List<String> elements = splitPathToElements(pendingPath);
-    assertEquals("splitPathToElements()", filename, lastElement(elements));
+    Assertions.assertThat(lastElement(elements))
+        .describedAs("splitPathToElements(%s)", pendingPath)
+        .isEqualTo(filename);
     List<String> finalDestination = finalDestination(elements);
-    assertEquals("finalDestination()",
-        filename,
-        lastElement(finalDestination));
-    final String destKey = elementsToKey(finalDestination);
-    assertEquals("destination key", origKey, destKey);
+    Assertions.assertThat(lastElement(finalDestination))
+        .describedAs("finalDestination(%s)", pendingPath)
+        .isEqualTo(filename);
+    Assertions.assertThat(elementsToKey(finalDestination))
+        .describedAs("destination key")
+        .isEqualTo(origKey);
 
     PutTracker tracker = integration.createTracker(pendingPath,
-        pendingPathKey);
-    assertTrue("wrong type: " + tracker + " for " + pendingPathKey,
-        tracker instanceof MagicCommitTracker);
-    assertEquals("tracker destination key", origKey, tracker.getDestKey());
-
-    Path pendingSuffixedPath = new Path(pendingPath,
-        "part-0000" + PENDING_SUFFIX);
-    assertFalse("still a delayed complete path " + pendingSuffixedPath,
-        fs.isMagicCommitPath(pendingSuffixedPath));
-    Path pendingSet = new Path(pendingPath,
-        "part-0000" + PENDINGSET_SUFFIX);
-    assertFalse("still a delayed complete path " + pendingSet,
-        fs.isMagicCommitPath(pendingSet));
+        pendingPathKey, EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS);
+    Assertions.assertThat(tracker)
+        .describedAs("Tracker for %s", pendingPathKey)
+        .isInstanceOf(MagicCommitTracker.class);
+    Assertions.assertThat(tracker.getDestKey())
+        .describedAs("tracker destination key")
+        .isEqualTo(origKey);
+
+    assertNotDelayedWrite(new Path(pendingPath,
+        "part-0000" + PENDING_SUFFIX));
+    assertNotDelayedWrite(new Path(pendingPath,
+        "part-0000" + PENDINGSET_SUFFIX));
+  }
+
+  private void assertNotDelayedWrite(Path pendingSuffixedPath) {
+    Assertions.assertThat(getFileSystem().isMagicCommitPath(pendingSuffixedPath))
+        .describedAs("Expected %s to not be magic/delayed write", pendingSuffixedPath)
+        .isFalse();
   }
 
   @Test
@@ -172,41 +166,39 @@ public void testCreateAbortEmptyFile() throws Throwable {
     describe("create then abort an empty file; throttled");
     S3AFileSystem fs = getFileSystem();
     String filename = "empty-abort.txt";
-    Path destFile = methodPath(filename);
+    Path destFile = methodSubPath(filename);
     Path pendingFilePath = makeMagic(destFile);
     touch(fs, pendingFilePath);
-    waitForConsistency();
+
     validateIntermediateAndFinalPaths(pendingFilePath, destFile);
     Path pendingDataPath = validatePendingCommitData(filename,
         pendingFilePath);
 
     CommitOperations actions = newCommitOperations();
     // abort,; rethrow on failure
-    fullThrottle();
+
     LOG.info("Abort call");
-    actions.abortAllSinglePendingCommits(pendingDataPath.getParent(), true)
-        .maybeRethrow();
-    resetFailures();
+    Path parent = pendingDataPath.getParent();
+    try (CommitContext commitContext =
+             actions.createCommitContextForTesting(parent, JOB_ID, 0)) {
+      actions.abortAllSinglePendingCommits(parent, commitContext, true)
+          .maybeRethrow();
+    }
+
     assertPathDoesNotExist("pending file not deleted", pendingDataPath);
     assertPathDoesNotExist("dest file was created", destFile);
   }
 
-  private void fullThrottle() {
-    setThrottling(FULL_THROTTLE, STANDARD_FAILURE_LIMIT);
-  }
-
+  /**
+   * Create a new commit operations instance for the test FS.
+   * @return commit operations.
+   * @throws IOException IO failure.
+   */
   private CommitOperations newCommitOperations()
       throws IOException {
     return new CommitOperations(getFileSystem());
   }
 
-  @Override
-  protected void resetFailures() {
-    if (!RESET_FAILURES_ENABLED) {
-      super.resetFailures();
-    }
-  }
-
   /**
    * Create a new path which has the same filename as the dest file, but
    * is in a magic directory under the destination dir.
@@ -233,10 +225,16 @@ public void testCommitSmallFile() throws Throwable {
   @Test
   public void testAbortNonexistentDir() throws Throwable {
     describe("Attempt to abort a directory that does not exist");
-    Path destFile = methodPath("testAbortNonexistentPath");
-    newCommitOperations()
-        .abortAllSinglePendingCommits(destFile, true)
-        .maybeRethrow();
+    Path destFile = methodSubPath("testAbortNonexistentPath");
+    final CommitOperations operations = newCommitOperations();
+    try (CommitContext commitContext
+             = operations.createCommitContextForTesting(destFile, JOB_ID, 0)) {
+      final CommitOperations.MaybeIOE outcome = operations
+          .abortAllSinglePendingCommits(destFile, commitContext, true);
+      outcome.maybeRethrow();
+      Assertions.assertThat(outcome)
+          .isEqualTo(CommitOperations.MaybeIOE.NONE);
+    }
   }
 
   @Test
@@ -279,7 +277,7 @@ public void testBaseRelativePath() throws Throwable {
     describe("Test creating file with a __base marker and verify that it ends" +
         " up in where expected");
     S3AFileSystem fs = getFileSystem();
-    Path destDir = methodPath("testBaseRelativePath");
+    Path destDir = methodSubPath("testBaseRelativePath");
     fs.delete(destDir, true);
     Path pendingBaseDir = new Path(destDir, MAGIC + "/child/" + BASE);
     String child = "subdir/child.txt";
@@ -289,7 +287,7 @@ public void testBaseRelativePath() throws Throwable {
         expectedDestPath);
 
     createFile(fs, pendingChildPath, true, DATASET);
-    commit("child.txt", pendingChildPath, expectedDestPath, 0, 0);
+    commit("child.txt", pendingChildPath, expectedDestPath);
   }
 
   /**
@@ -305,14 +303,17 @@ public void testMarkerFileRename()
     fs.delete(destDir, true);
     Path magicDest = makeMagic(destFile);
     Path magicDir = magicDest.getParent();
-    fs.mkdirs(magicDir);
+    fs.mkdirs(magicDest);
 
     // use the builder API to verify it works exactly the
     // same.
-    try (FSDataOutputStream stream = fs.createFile(magicDest)
-        .overwrite(true)
-        .recursive()
-        .build()) {
+    FSDataOutputStreamBuilder builder = fs.createFile(magicDest)
+        .overwrite(true);
+    builder.recursive();
+    // this has a broken return type; not sure why
+    builder.must(FS_S3A_CREATE_PERFORMANCE, true);
+
+    try (FSDataOutputStream stream = builder.build()) {
       assertIsMagicStream(stream);
       stream.write(DATASET);
     }
@@ -321,9 +322,7 @@ public void testMarkerFileRename()
     fs.rename(magicDest, magic2);
 
     // the renamed file has no header
-    Assertions.assertThat(extractMagicFileLength(fs, magic2))
-        .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + magic2)
-        .isEmpty();
+    getTestHelper().assertFileLacksMarkerHeader(magic2);
     // abort the upload, which is driven by the .pending files
     // there must be 1 deleted file; during test debugging with aborted
     // runs there may be more.
@@ -333,17 +332,6 @@ public void testMarkerFileRename()
         .isGreaterThanOrEqualTo(1);
   }
 
-  /**
-   * Assert that an output stream is magic.
-   * @param stream stream to probe.
-   */
-  protected void assertIsMagicStream(final FSDataOutputStream stream) {
-    Assertions.assertThat(stream.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT))
-        .describedAs("Stream capability %s in stream %s",
-            STREAM_CAPABILITY_MAGIC_OUTPUT, stream)
-        .isTrue();
-  }
-
   /**
    * Create a file through the magic commit mechanism.
    * @param filename file to create (with __magic path.)
@@ -353,82 +341,62 @@ protected void assertIsMagicStream(final FSDataOutputStream stream) {
   private void createCommitAndVerify(String filename, byte[] data)
       throws Exception {
     S3AFileSystem fs = getFileSystem();
-    Path destFile = methodPath(filename);
+    Path destFile = methodSubPath(filename);
     fs.delete(destFile.getParent(), true);
     Path magicDest = makeMagic(destFile);
     assertPathDoesNotExist("Magic file should not exist", magicDest);
     long dataSize = data != null ? data.length : 0;
-    try(FSDataOutputStream stream = fs.create(magicDest, true)) {
+    try (FSDataOutputStream stream = fs.create(magicDest, true)) {
       assertIsMagicStream(stream);
       if (dataSize > 0) {
         stream.write(data);
       }
-      stream.close();
     }
-    FileStatus status = getFileStatusEventually(fs, magicDest,
-        CONSISTENCY_WAIT);
-    assertEquals("Magic marker file is not zero bytes: " + status,
-        0, 0);
-    Assertions.assertThat(extractMagicFileLength(fs,
-        magicDest))
-        .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + magicDest)
-        .isNotEmpty()
-        .hasValue(dataSize);
-    commit(filename, destFile, HIGH_THROTTLE, 0);
+    getTestHelper().assertIsMarkerFile(magicDest, dataSize);
+    commit(filename, destFile);
     verifyFileContents(fs, destFile, data);
     // the destination file doesn't have the attribute
-    Assertions.assertThat(extractMagicFileLength(fs,
-        destFile))
-        .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + destFile)
-        .isEmpty();
+    getTestHelper().assertFileLacksMarkerHeader(destFile);
   }
 
   /**
    * Commit the file, with before and after checks on the dest and magic
    * values.
-   * Failures can be set; they'll be reset after the commit.
    * @param filename filename of file
    * @param destFile destination path of file
-   * @param throttle probability of commit throttling
-   * @param failures failure limit
    * @throws Exception any failure of the operation
    */
   private void commit(String filename,
-      Path destFile,
-      float throttle,
-      int failures) throws Exception {
-    commit(filename, makeMagic(destFile), destFile, throttle, failures);
+      Path destFile) throws Exception {
+    commit(filename, makeMagic(destFile), destFile);
   }
 
   /**
    * Commit to a write to {@code magicFile} which is expected to
    * be saved to {@code destFile}.
-   * Failures can be set; they'll be reset after the commit.
    * @param magicFile path to write to
    * @param destFile destination to verify
-   * @param throttle probability of commit throttling
-   * @param failures failure limit
    */
   private void commit(String filename,
       Path magicFile,
-      Path destFile,
-      float throttle, int failures)
+      Path destFile)
       throws IOException {
-    resetFailures();
+
+    final CommitOperations actions = newCommitOperations();
     validateIntermediateAndFinalPaths(magicFile, destFile);
     SinglePendingCommit commit = SinglePendingCommit.load(getFileSystem(),
-        validatePendingCommitData(filename, magicFile));
-    setThrottling(throttle, failures);
-    commitOrFail(destFile, commit, newCommitOperations());
-    resetFailures();
-    verifyCommitExists(commit);
+        validatePendingCommitData(filename, magicFile),
+        null,
+        SinglePendingCommit.serializer());
+
+    commitOrFail(destFile, commit, actions);
   }
 
   private void commitOrFail(final Path destFile,
       final SinglePendingCommit commit, final CommitOperations actions)
       throws IOException {
-    try (CommitOperations.CommitContext commitContext
-             = actions.initiateCommitOperation(destFile)) {
+    try (CommitContext commitContext
+             = actions.createCommitContextForTesting(destFile, JOB_ID, 0)) {
       commitContext.commitOrFail(commit);
     }
   }
@@ -445,26 +413,6 @@ private void validateIntermediateAndFinalPaths(Path magicFilePath,
     assertPathDoesNotExist("dest file was created", destFile);
   }
 
-  /**
-   * Verify that the path at the end of a commit exists.
-   * This does not validate the size.
-   * @param commit commit to verify
-   * @throws FileNotFoundException dest doesn't exist
-   * @throws ValidationFailure commit arg is invalid
-   * @throws IOException invalid commit, IO failure
-   */
-  private void verifyCommitExists(SinglePendingCommit commit)
-      throws FileNotFoundException, ValidationFailure, IOException {
-    commit.validate();
-    // this will force an existence check
-    Path path = getFileSystem().keyToQualifiedPath(commit.getDestinationKey());
-    FileStatus status = getFileSystem().getFileStatus(path);
-    LOG.debug("Destination entry: {}", status);
-    if (!status.isFile()) {
-      throw new PathCommitException(path, "Not a file: " + status);
-    }
-  }
-
   /**
    * Validate that a pending commit data file exists, load it and validate
    * its contents.
@@ -487,14 +435,19 @@ private Path validatePendingCommitData(String filename,
     SinglePendingCommit persisted = SinglePendingCommit.serializer()
         .load(fs, pendingDataPath);
     persisted.validate();
-    assertTrue("created timestamp wrong in " + persisted,
-        persisted.getCreated() > 0);
-    assertTrue("saved timestamp wrong in " + persisted,
-        persisted.getSaved() > 0);
+    Assertions.assertThat(persisted.getCreated())
+        .describedAs("Created timestamp in %s", persisted)
+        .isGreaterThan(0);
+    Assertions.assertThat(persisted.getSaved())
+        .describedAs("saved timestamp in %s", persisted)
+        .isGreaterThan(0);
     List<String> etags = persisted.getEtags();
-    assertEquals("etag list " + persisted, 1, etags.size());
-    List<PartETag> partList = CommitOperations.toPartEtags(etags);
-    assertEquals("part list " + persisted, 1, partList.size());
+    Assertions.assertThat(etags)
+        .describedAs("Etag list")
+        .hasSize(1);
+    Assertions.assertThat(CommitOperations.toPartEtags(etags))
+        .describedAs("Etags to parts")
+        .hasSize(1);
     return pendingDataPath;
   }
 
@@ -504,27 +457,18 @@ private Path validatePendingCommitData(String filename,
    * @return new path
    * @throws IOException failure to create/parse the path.
    */
-  private Path methodPath(String filename) throws IOException {
+  private Path methodSubPath(String filename) throws IOException {
     return new Path(methodPath(), filename);
   }
 
-  /**
-   * Get a unique path for a method.
-   * @return a path
-   * @throws IOException
-   */
-  protected Path methodPath() throws IOException {
-    return path(getMethodName());
-  }
-
   @Test
   public void testUploadEmptyFile() throws Throwable {
+    describe("Upload a zero byte file to a magic path");
     File tempFile = File.createTempFile("commit", ".txt");
     CommitOperations actions = newCommitOperations();
-    Path dest = methodPath("testUploadEmptyFile");
+    Path dest = methodSubPath("testUploadEmptyFile");
     S3AFileSystem fs = getFileSystem();
     fs.delete(dest, false);
-    fullThrottle();
 
     SinglePendingCommit pendingCommit =
         actions.uploadFileToPendingCommit(tempFile,
@@ -532,16 +476,19 @@ public void testUploadEmptyFile() throws Throwable {
             null,
             DEFAULT_MULTIPART_SIZE,
             progress);
-    resetFailures();
+
     assertPathDoesNotExist("pending commit", dest);
-    fullThrottle();
+
     commitOrFail(dest, pendingCommit, actions);
-    resetFailures();
-    FileStatus status = verifyPathExists(fs,
-        "uploaded file commit", dest);
+
     progress.assertCount("Progress counter should be 1.",
         1);
-    assertEquals("File length in " + status, 0, status.getLen());
+    FileStatus status = verifyPathExists(fs,
+        "uploaded file commit", dest);
+    Assertions.assertThat(status.getLen())
+        .describedAs("Committed File file %s: %s", dest, status)
+        .isEqualTo(0);
+    getTestHelper().assertFileLacksMarkerHeader(dest);
   }
 
   @Test
@@ -550,10 +497,10 @@ public void testUploadSmallFile() throws Throwable {
     String text = "hello, world";
     FileUtils.write(tempFile, text, "UTF-8");
     CommitOperations actions = newCommitOperations();
-    Path dest = methodPath("testUploadSmallFile");
+    Path dest = methodSubPath("testUploadSmallFile");
     S3AFileSystem fs = getFileSystem();
     fs.delete(dest, true);
-    fullThrottle();
+
     assertPathDoesNotExist("test setup", dest);
     SinglePendingCommit pendingCommit =
         actions.uploadFileToPendingCommit(tempFile,
@@ -561,73 +508,66 @@ public void testUploadSmallFile() throws Throwable {
             null,
             DEFAULT_MULTIPART_SIZE,
             progress);
-    resetFailures();
+
     assertPathDoesNotExist("pending commit", dest);
-    fullThrottle();
+
     LOG.debug("Postcommit validation");
     commitOrFail(dest, pendingCommit, actions);
-    resetFailures();
+
     String s = readUTF8(fs, dest, -1);
-    assertEquals(text, s);
+    Assertions.assertThat(s)
+        .describedAs("contents of committed file %s", dest)
+        .isEqualTo(text);
     progress.assertCount("Progress counter should be 1.",
         1);
   }
 
-  @Test(expected = FileNotFoundException.class)
+  @Test
   public void testUploadMissingFile() throws Throwable {
     File tempFile = File.createTempFile("commit", ".txt");
     tempFile.delete();
     CommitOperations actions = newCommitOperations();
-    Path dest = methodPath("testUploadMissingile");
-    fullThrottle();
-    actions.uploadFileToPendingCommit(tempFile, dest, null,
-        DEFAULT_MULTIPART_SIZE, progress);
-    progress.assertCount("Progress counter should be 1.",
-        1);
+    Path dest = methodSubPath("testUploadMissingFile");
+    intercept(FileNotFoundException.class, () ->
+        actions.uploadFileToPendingCommit(tempFile, dest, null,
+            DEFAULT_MULTIPART_SIZE, progress));
+    progress.assertCount("Progress counter should be 0.",
+        0);
   }
 
   @Test
   public void testRevertCommit() throws Throwable {
-    Path destFile = methodPath("part-0000");
+    describe("Revert a commit; the destination file will be deleted");
+    Path destFile = methodSubPath("part-0000");
     S3AFileSystem fs = getFileSystem();
     touch(fs, destFile);
-    CommitOperations actions = newCommitOperations();
     SinglePendingCommit commit = new SinglePendingCommit();
+    CommitOperations actions = newCommitOperations();
     commit.setDestinationKey(fs.pathToKey(destFile));
-    fullThrottle();
-    actions.revertCommit(commit, null);
-    resetFailures();
-    assertPathExists("parent of reverted commit", destFile.getParent());
+    newCommitOperations().revertCommit(commit);
+    assertPathDoesNotExist("should have been reverted", destFile);
   }
 
   @Test
   public void testRevertMissingCommit() throws Throwable {
-    Path destFile = methodPath("part-0000");
+    Path destFile = methodSubPath("part-0000");
     S3AFileSystem fs = getFileSystem();
     fs.delete(destFile, false);
-    CommitOperations actions = newCommitOperations();
     SinglePendingCommit commit = new SinglePendingCommit();
     commit.setDestinationKey(fs.pathToKey(destFile));
-    fullThrottle();
-    actions.revertCommit(commit, null);
-    resetFailures();
-    assertPathExists("parent of reverted (nonexistent) commit",
-        destFile.getParent());
+    newCommitOperations().revertCommit(commit);
+    assertPathDoesNotExist("should have been reverted", destFile);
   }
 
   @Test
   public void testFailuresInAbortListing() throws Throwable {
-    CommitOperations actions = newCommitOperations();
     Path path = path("testFailuresInAbort");
     getFileSystem().mkdirs(path);
-    setThrottling(HIGH_THROTTLE);
     LOG.info("Aborting");
-    actions.abortPendingUploadsUnderPath(path);
+    newCommitOperations().abortPendingUploadsUnderPath(path);
     LOG.info("Abort completed");
-    resetFailures();
   }
 
-
   /**
    * Test a normal stream still works as expected in a magic filesystem,
    * with a call of {@code hasCapability()} to check that it is normal.
@@ -636,38 +576,29 @@ public void testFailuresInAbortListing() throws Throwable {
   @Test
   public void testWriteNormalStream() throws Throwable {
     S3AFileSystem fs = getFileSystem();
-    assumeMagicCommitEnabled(fs);
     Path destFile = path("normal");
     try (FSDataOutputStream out = fs.create(destFile, true)) {
       out.writeChars("data");
       assertFalse("stream has magic output: " + out,
           out.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT));
-      out.close();
     }
-    FileStatus status = getFileStatusEventually(fs, destFile,
-        CONSISTENCY_WAIT);
-    assertTrue("Empty marker file: " + status, status.getLen() > 0);
+    FileStatus status = fs.getFileStatus(destFile);
+    Assertions.assertThat(status.getLen())
+        .describedAs("Normal file %s: %s", destFile, status)
+        .isGreaterThan(0);
   }
 
   /**
    * Creates a bulk commit and commits multiple files.
-   * If the DDB metastore is in use, use the instrumentation to
-   * verify that the write count is as expected.
-   * This is done without actually looking into the store -just monitoring
-   * changes in the filesystem's instrumentation counters.
-   * As changes to the store may be made during get/list calls,
-   * when the counters must be reset before each commit, this must be
-   * *after* all probes for the outcome of the previous operation.
    */
   @Test
   public void testBulkCommitFiles() throws Throwable {
-    describe("verify bulk commit including metastore update count");
+    describe("verify bulk commit");
     File localFile = File.createTempFile("commit", ".txt");
     CommitOperations actions = newCommitOperations();
-    Path destDir = methodPath("out");
+    Path destDir = methodSubPath("out");
     S3AFileSystem fs = getFileSystem();
     fs.delete(destDir, false);
-    fullThrottle();
 
     Path destFile1 = new Path(destDir, "file1");
     // this subdir will only be created in the commit of file 2
@@ -679,7 +610,7 @@ public void testBulkCommitFiles() throws Throwable {
         destFile3);
     List<SinglePendingCommit> commits = new ArrayList<>(3);
 
-    for (Path destination : destinations) {
+    for (Path destination: destinations) {
       SinglePendingCommit commit1 =
           actions.uploadFileToPendingCommit(localFile,
               destination, null,
@@ -687,60 +618,31 @@ public void testBulkCommitFiles() throws Throwable {
               progress);
       commits.add(commit1);
     }
-    resetFailures();
+
     assertPathDoesNotExist("destination dir", destDir);
     assertPathDoesNotExist("subdirectory", subdir);
     LOG.info("Initiating commit operations");
-    try (CommitOperations.CommitContext commitContext
-             = actions.initiateCommitOperation(destDir)) {
-      // how many records have been written
-      MetricDiff writes = new MetricDiff(fs,
-          Statistic.S3GUARD_METADATASTORE_RECORD_WRITES);
+    try (CommitContext commitContext
+             = actions.createCommitContextForTesting(destDir, JOB_ID, 0)) {
       LOG.info("Commit #1");
       commitContext.commitOrFail(commits.get(0));
       final String firstCommitContextString = commitContext.toString();
       LOG.info("First Commit state {}", firstCommitContextString);
-      long writesOnFirstCommit = writes.diff();
       assertPathExists("destFile1", destFile1);
       assertPathExists("destination dir", destDir);
 
       LOG.info("Commit #2");
-      writes.reset();
       commitContext.commitOrFail(commits.get(1));
       assertPathExists("subdirectory", subdir);
       assertPathExists("destFile2", destFile2);
       final String secondCommitContextString = commitContext.toString();
       LOG.info("Second Commit state {}", secondCommitContextString);
 
-      if (writesOnFirstCommit != 0) {
-        LOG.info("DynamoDB Metastore is in use: checking write count");
-        // S3Guard is in use against DDB, so the metrics can be checked
-        // to see how many records were updated.
-        // there should only be two new entries: one for the file and
-        // one for the parent.
-        // we include the string values of the contexts because that includes
-        // the internals of the bulk operation state.
-        writes.assertDiffEquals("Number of records written after commit #2"
-                + "; first commit had " + writesOnFirstCommit
-                + "; first commit ancestors " + firstCommitContextString
-                + "; second commit ancestors: " + secondCommitContextString,
-            2);
-      }
-
       LOG.info("Commit #3");
-      writes.reset();
       commitContext.commitOrFail(commits.get(2));
       assertPathExists("destFile3", destFile3);
-      if (writesOnFirstCommit != 0) {
-        // this file is in the same dir as destFile2, so only its entry
-        // is added
-        writes.assertDiffEquals(
-            "Number of records written after third commit; "
-                + "first commit had " + writesOnFirstCommit,
-            1);
-      }
     }
-    resetFailures();
+
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java
index 0d17016927d02..2ad2568d5cc20 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java
@@ -107,11 +107,6 @@ public void setup() throws Exception {
     taskConfRef = tContext.getConfiguration();
   }
 
-  @Override
-  public boolean useInconsistentClient() {
-    return false;
-  }
-
   @Test
   public void testEverything() throws Throwable {
     testImplicitFileBinding();
@@ -162,7 +157,7 @@ public void testBindingsInFSConfig() throws Throwable {
   }
 
   /**
-   * Create an invalid committer via the FS binding,
+   * Create an invalid committer via the FS binding.
    */
   public void testInvalidFileBinding() throws Throwable {
     taskConfRef.unset(FS_S3A_COMMITTER_NAME);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java
index aa068c08d667c..622ead2617fd6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java
@@ -190,11 +190,6 @@ protected String committerName() {
     return committerTestBinding.getCommitterName();
   }
 
-  @Override
-  public boolean useInconsistentClient() {
-    return committerTestBinding.useInconsistentClient();
-  }
-
   /**
    * Verify that the committer binding is happy.
    */
@@ -218,9 +213,6 @@ public void test_200_execute() throws Exception {
     // that and URI creation fails.
 
     Path outputPath = path("ITestS3ACommitterMRJob-execute-"+ committerName());
-    // create and delete to force in a tombstone marker -see HADOOP-16207
-    fs.mkdirs(outputPath);
-    fs.delete(outputPath, true);
 
     String commitUUID = UUID.randomUUID().toString();
     String suffix = isUniqueFilenames() ? ("-" + commitUUID) : "";
@@ -303,7 +295,6 @@ public void test_200_execute() throws Exception {
       fail(message);
     }
 
-    waitForConsistency();
     Path successPath = new Path(outputPath, _SUCCESS);
     SuccessData successData = validateSuccessFile(outputPath,
         committerName(),
@@ -483,12 +474,6 @@ protected void applyCustomConfigOptions(JobConf jobConf)
         throws IOException {
     }
 
-    /**
-     * Should the inconsistent S3A client be used?
-     * @return true for inconsistent listing
-     */
-    public abstract boolean useInconsistentClient();
-
     /**
      * Override point for any committer specific validation operations;
      * called after the base assertions have all passed.
@@ -543,13 +528,6 @@ private DirectoryCommitterTestBinding() {
       super(DirectoryStagingCommitter.NAME);
     }
 
-    /**
-     * @return true for inconsistent listing
-     */
-    public boolean useInconsistentClient() {
-      return true;
-    }
-
     /**
      * Verify that staging commit dirs are made absolute under the user's
      * home directory, so, in a secure cluster, private.
@@ -584,12 +562,6 @@ private PartitionCommitterTestBinding() {
       super(PartitionedStagingCommitter.NAME);
     }
 
-    /**
-     * @return true for inconsistent listing
-     */
-    public boolean useInconsistentClient() {
-      return true;
-    }
   }
 
   /**
@@ -603,13 +575,6 @@ private MagicCommitterTestBinding() {
       super(MagicS3GuardCommitter.NAME);
     }
 
-    /**
-     * @return we need a consistent store.
-     */
-    public boolean useInconsistentClient() {
-      return false;
-    }
-
     /**
      * The result validation here is that there isn't a __magic directory
      * any more.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
index 2b2fc2bb446d7..32cab03770c2d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
@@ -25,8 +25,6 @@
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
@@ -34,7 +32,6 @@
 import org.apache.hadoop.fs.s3a.commit.AbstractITCommitProtocol;
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.CommitUtils;
 import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
 import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl;
@@ -45,7 +42,7 @@
 
 import static org.apache.hadoop.fs.s3a.S3AUtils.listAndFilter;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
-import static org.hamcrest.CoreMatchers.containsString;
+import static org.apache.hadoop.util.functional.RemoteIterators.toList;
 
 /**
  * Test the magic committer's commit protocol.
@@ -57,15 +54,6 @@ protected String suitename() {
     return "ITestMagicCommitProtocol";
   }
 
-  /**
-   * Need consistency here.
-   * @return false
-   */
-  @Override
-  public boolean useInconsistentClient() {
-    return false;
-  }
-
   @Override
   protected String getCommitterFactoryName() {
     return CommitConstants.S3A_COMMITTER_FACTORY;
@@ -108,8 +96,9 @@ public MagicS3GuardCommitter createFailingCommitter(
   protected void validateTaskAttemptPathDuringWrite(Path p,
       final long expectedLength) throws IOException {
     String pathStr = p.toString();
-    assertTrue("not magic " + pathStr,
-        pathStr.contains(MAGIC));
+    Assertions.assertThat(pathStr)
+        .describedAs("Magic path")
+        .contains(MAGIC);
     assertPathDoesNotExist("task attempt visible", p);
   }
 
@@ -125,9 +114,9 @@ protected void validateTaskAttemptPathAfterWrite(Path marker,
     // if you list the parent dir and find the marker, it
     // is really 0 bytes long
     String name = marker.getName();
-    List<LocatedFileStatus> filtered = listAndFilter(fs,
+    List<LocatedFileStatus> filtered = toList(listAndFilter(fs,
         marker.getParent(), false,
-        (path) -> path.getName().equals(name));
+        (path) -> path.getName().equals(name)));
     Assertions.assertThat(filtered)
         .hasSize(1);
     Assertions.assertThat(filtered.get(0))
@@ -135,14 +124,7 @@ protected void validateTaskAttemptPathAfterWrite(Path marker,
             "Listing should return 0 byte length");
 
     // marker file is empty
-    FileStatus st = fs.getFileStatus(marker);
-    assertEquals("file length in " + st, 0, st.getLen());
-    // xattr header
-    Assertions.assertThat(CommitOperations.extractMagicFileLength(fs,
-        marker))
-        .describedAs("XAttribute " + XA_MAGIC_MARKER)
-        .isNotEmpty()
-        .hasValue(expectedLength);
+    getTestHelper().assertIsMarkerFile(marker, expectedLength);
   }
 
   /**
@@ -160,8 +142,8 @@ protected void validateTaskAttemptWorkingDirectory(
     assertEquals("Wrong schema for working dir " + wd
         + " with committer " + committer,
         "s3a", wd.getScheme());
-    assertThat(wd.getPath(),
-        containsString('/' + CommitConstants.MAGIC + '/'));
+    Assertions.assertThat(wd.getPath())
+        .contains('/' + CommitConstants.MAGIC + '/');
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
index 3c15454e7edfb..aeea195aacf89 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
@@ -20,8 +20,8 @@
 
 import java.io.IOException;
 import java.util.List;
-import java.util.stream.Collectors;
 
+import org.assertj.core.api.Assertions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,10 +33,11 @@
 import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
 import org.apache.hadoop.fs.s3a.commit.CommitUtils;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles;
 
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads;
@@ -54,6 +55,7 @@
 public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles {
   private static final Logger LOG = LoggerFactory.getLogger(
       ITestS3AHugeMagicCommits.class);
+  private static final int COMMITTER_THREADS = 64;
 
   private Path magicDir;
   private Path jobDir;
@@ -123,16 +125,16 @@ public void test_030_postCreationAssertions() throws Throwable {
     Path destDir = getHugefile().getParent();
     assertPathExists("Magic dir", new Path(destDir, CommitConstants.MAGIC));
     String destDirKey = fs.pathToKey(destDir);
-    List<String> uploads = listMultipartUploads(fs, destDirKey);
 
-    assertEquals("Pending uploads: "
-        + uploads.stream()
-        .collect(Collectors.joining("\n")), 1, uploads.size());
+    Assertions.assertThat(listMultipartUploads(fs, destDirKey))
+        .describedAs("Pending uploads")
+        .hasSize(1);
     assertNotNull("jobDir", jobDir);
-    Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
-        results = operations.loadSinglePendingCommits(jobDir, false);
-    try(CommitOperations.CommitContext commitContext
-            = operations.initiateCommitOperation(jobDir)) {
+    try(CommitContext commitContext
+            = operations.createCommitContextForTesting(jobDir, null, COMMITTER_THREADS)) {
+      Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
+          results = operations.loadSinglePendingCommits(jobDir, false, commitContext
+      );
       for (SinglePendingCommit singlePendingCommit :
           results.getKey().getCommits()) {
         commitContext.commitOrFail(singlePendingCommit);
@@ -140,10 +142,9 @@ public void test_030_postCreationAssertions() throws Throwable {
     }
     timer.end("time to commit %s", pendingDataFile);
     // upload is no longer pending
-    uploads = listMultipartUploads(fs, destDirKey);
-    assertEquals("Pending uploads"
-            + uploads.stream().collect(Collectors.joining("\n")),
-        0, operations.listPendingUploadsUnderPath(destDir).size());
+    Assertions.assertThat(operations.listPendingUploadsUnderPath(destDir))
+        .describedAs("Pending uploads undedr path")
+        .isEmpty();
     // at this point, the huge file exists, so the normal assertions
     // on that file must be valid. Verify.
     super.test_030_postCreationAssertions();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
index d3da8185c8d65..beccf4f328c05 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
 import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientErrors;
 import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientResults;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
@@ -76,11 +77,12 @@ public void commitJob(JobContext context) throws IOException {
   }
 
   @Override
-  protected void maybeCreateSuccessMarker(JobContext context,
+  protected SuccessData maybeCreateSuccessMarker(JobContext context,
       List<String> filenames,
       final IOStatisticsSnapshot ioStatistics)
       throws IOException {
      //skipped
+    return null;
   }
 
   public ClientResults getResults() throws IOException {
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
index 6e13fd0227a3b..6f2953762439a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
@@ -105,10 +105,15 @@ public class StagingTestBase {
   /** The raw bucket URI Path before any canonicalization. */
   public static final URI RAW_BUCKET_URI =
       RAW_BUCKET_PATH.toUri();
-  public static Path outputPath =
+
+  @SuppressWarnings("StaticNonFinalField")
+  private static Path outputPath =
       new Path("s3a://" + BUCKET + "/" + OUTPUT_PREFIX);
-  public static URI outputPathUri = outputPath.toUri();
-  public static Path root;
+
+  @SuppressWarnings("StaticNonFinalField")
+  private static URI outputPathUri = getOutputPath().toUri();
+  @SuppressWarnings("StaticNonFinalField")
+  private static Path root;
 
   protected StagingTestBase() {
   }
@@ -131,8 +136,8 @@ protected static S3AFileSystem createAndBindMockFSInstance(Configuration conf,
     URI uri = RAW_BUCKET_URI;
     wrapperFS.initialize(uri, conf);
     root = wrapperFS.makeQualified(new Path("/"));
-    outputPath = new Path(root, OUTPUT_PREFIX);
-    outputPathUri = outputPath.toUri();
+    outputPath = new Path(getRoot(), OUTPUT_PREFIX);
+    outputPathUri = getOutputPath().toUri();
     FileSystemTestHelper.addFileSystemForTesting(uri, conf, wrapperFS);
     return mockFs;
   }
@@ -154,7 +159,7 @@ private static S3AFileSystem mockS3AFileSystemRobustly() {
    */
   public static MockS3AFileSystem lookupWrapperFS(Configuration conf)
       throws IOException {
-    return (MockS3AFileSystem) FileSystem.get(outputPathUri, conf);
+    return (MockS3AFileSystem) FileSystem.get(getOutputPathUri(), conf);
   }
 
   public static void verifyCompletion(FileSystem mockS3) throws IOException {
@@ -169,13 +174,13 @@ public static void verifyDeleted(FileSystem mockS3, Path path)
 
   public static void verifyDeleted(FileSystem mockS3, String child)
       throws IOException {
-    verifyDeleted(mockS3, new Path(outputPath, child));
+    verifyDeleted(mockS3, new Path(getOutputPath(), child));
   }
 
   public static void verifyCleanupTempFiles(FileSystem mockS3)
       throws IOException {
     verifyDeleted(mockS3,
-        new Path(outputPath, CommitConstants.TEMPORARY));
+        new Path(getOutputPath(), CommitConstants.TEMPORARY));
   }
 
   protected static void assertConflictResolution(
@@ -189,7 +194,7 @@ protected static void assertConflictResolution(
   public static void pathsExist(FileSystem mockS3, String... children)
       throws IOException {
     for (String child : children) {
-      pathExists(mockS3, new Path(outputPath, child));
+      pathExists(mockS3, new Path(getOutputPath(), child));
     }
   }
 
@@ -231,7 +236,7 @@ public static void mkdirsHasOutcome(FileSystem mockS3,
   public static void canDelete(FileSystem mockS3, String... children)
       throws IOException {
     for (String child : children) {
-      canDelete(mockS3, new Path(outputPath, child));
+      canDelete(mockS3, new Path(getOutputPath(), child));
     }
   }
 
@@ -243,7 +248,7 @@ public static void canDelete(FileSystem mockS3, Path f) throws IOException {
 
   public static void verifyExistenceChecked(FileSystem mockS3, String child)
       throws IOException {
-    verifyExistenceChecked(mockS3, new Path(outputPath, child));
+    verifyExistenceChecked(mockS3, new Path(getOutputPath(), child));
   }
 
   public static void verifyExistenceChecked(FileSystem mockS3, Path path)
@@ -262,6 +267,18 @@ public static void verifyMkdirsInvoked(FileSystem mockS3, Path path)
     verify(mockS3).mkdirs(path);
   }
 
+  protected static URI getOutputPathUri() {
+    return outputPathUri;
+  }
+
+  static Path getRoot() {
+    return root;
+  }
+
+  static Path getOutputPath() {
+    return outputPath;
+  }
+
   /**
    * Provides setup/teardown of a MiniDFSCluster for tests that need one.
    */
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java
index cb7202b10d133..4d24c07dacfe2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java
@@ -28,6 +28,7 @@
 import java.util.stream.IntStream;
 
 import com.amazonaws.services.s3.model.PartETag;
+
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.assertj.core.api.Assertions;
 import org.junit.AfterClass;
@@ -48,18 +49,21 @@
 import org.apache.hadoop.fs.s3a.commit.CommitConstants;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.JsonSerialization;
 
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.CONFLICT_MODE_APPEND;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_CONFLICT_MODE;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.PENDINGSET_SUFFIX;
 import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.BUCKET;
-import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.outputPath;
-import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.outputPathUri;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.getOutputPath;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.getOutputPathUri;
 import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.pathIsDirectory;
 
 /**
@@ -83,6 +87,7 @@ public class TestDirectoryCommitterScale
   public static final int TOTAL_COMMIT_COUNT = FILES_PER_TASK * TASKS;
 
   public static final int BLOCKS_PER_TASK = 1000;
+  private static final int COMMITTER_THREAD_COUNT = 100;
 
   private static File stagingDir;
 
@@ -95,13 +100,13 @@ public class TestDirectoryCommitterScale
 
   @Override
   DirectoryCommitterForTesting newJobCommitter() throws Exception {
-    return new DirectoryCommitterForTesting(outputPath,
+    return new DirectoryCommitterForTesting(getOutputPath(),
         createTaskAttemptForJob());
   }
 
   @BeforeClass
   public static void setupStaging() throws Exception {
-    stagingDir = File.createTempFile("staging", "");
+    stagingDir = File.createTempFile("staging", null);
     stagingDir.delete();
     stagingDir.mkdir();
     stagingPath = new Path(stagingDir.toURI());
@@ -125,7 +130,7 @@ protected JobConf createJobConf() {
     JobConf conf = super.createJobConf();
     conf.setInt(
         CommitConstants.FS_S3A_COMMITTER_THREADS,
-        100);
+        COMMITTER_THREAD_COUNT);
     return conf;
   }
 
@@ -149,7 +154,8 @@ public void test_010_createTaskFiles() throws Exception {
    */
   private void createTasks() throws IOException {
     // create a stub multipart commit containing multiple files.
-
+    JsonSerialization<SinglePendingCommit> serializer =
+        SinglePendingCommit.serializer();
     // step1: a list of tags.
     // this is the md5sum of hadoop 3.2.1.tar
     String tag = "9062dcf18ffaee254821303bbd11c72b";
@@ -164,12 +170,14 @@ private void createTasks() throws IOException {
     // these get overwritten
     base.setDestinationKey("/base");
     base.setUploadId("uploadId");
-    base.setUri(outputPathUri.toString());
+    base.setUri(getOutputPathUri().toString());
+    byte[] bytes = base.toBytes(serializer);
 
     SinglePendingCommit[] singles = new SinglePendingCommit[FILES_PER_TASK];
-    byte[] bytes = base.toBytes();
+
     for (int i = 0; i < FILES_PER_TASK; i++) {
-      singles[i] = SinglePendingCommit.serializer().fromBytes(bytes);
+
+      singles[i] = serializer.fromBytes(bytes);
     }
     // now create the files, using this as the template
 
@@ -182,7 +190,7 @@ private void createTasks() throws IOException {
         String uploadId = String.format("%05d-task-%04d-file-%02d",
             uploadCount, task, i);
         // longer paths to take up more space.
-        Path p = new Path(outputPath,
+        Path p = new Path(getOutputPath(),
             "datasets/examples/testdirectoryscale/"
                 + "year=2019/month=09/day=26/hour=20/second=53"
                 + uploadId);
@@ -199,7 +207,7 @@ private void createTasks() throws IOException {
       }
       Path path = new Path(stagingPath,
           String.format("task-%04d." + PENDINGSET_SUFFIX, task));
-      pending.save(localFS, path, true);
+      pending.save(localFS, path, PendingSet.serializer());
     }
   }
 
@@ -211,12 +219,14 @@ public void test_020_loadFilesToAttempt() throws Exception {
     Configuration jobConf = getJobConf();
     jobConf.set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
-    FileSystem mockS3 = getMockS3A();
-    pathIsDirectory(mockS3, outputPath);
-    try (DurationInfo ignored =
-             new DurationInfo(LOG, "listing pending uploads")) {
+    S3AFileSystem mockS3 = getMockS3A();
+    pathIsDirectory(mockS3, getOutputPath());
+    final CommitOperations operations = new CommitOperations(getWrapperFS());
+    try (CommitContext commitContext
+             = operations.createCommitContextForTesting(getOutputPath(),
+        null, COMMITTER_THREAD_COUNT)) {
       AbstractS3ACommitter.ActiveCommit activeCommit
-          = committer.listPendingUploadsToCommit(getJob());
+          = committer.listPendingUploadsToCommit(commitContext);
       Assertions.assertThat(activeCommit.getSourceFiles())
           .describedAs("Source files of %s", activeCommit)
           .hasSize(TASKS);
@@ -232,7 +242,7 @@ public void test_030_commitFiles() throws Exception {
     jobConf.set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
     S3AFileSystem mockS3 = getMockS3A();
-    pathIsDirectory(mockS3, outputPath);
+    pathIsDirectory(mockS3, getOutputPath());
 
     try (DurationInfo ignored =
              new DurationInfo(LOG, "Committing Job")) {
@@ -261,7 +271,7 @@ public void test_040_abortFiles() throws Exception {
     jobConf.set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
     FileSystem mockS3 = getMockS3A();
-    pathIsDirectory(mockS3, outputPath);
+    pathIsDirectory(mockS3, getOutputPath());
 
     committer.abortJob(getJob(), JobStatus.State.FAILED);
   }
@@ -304,11 +314,11 @@ public Path getJobAttemptPath(JobContext context) {
     }
 
     @Override
-    protected void commitJobInternal(final JobContext context,
+    protected void commitJobInternal(final CommitContext commitContext,
         final ActiveCommit pending)
         throws IOException {
       activeCommit = pending;
-      super.commitJobInternal(context, pending);
+      super.commitJobInternal(commitContext, pending);
     }
   }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
index 94c0b29a3cdcc..11edf0d216376 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
@@ -37,7 +37,6 @@
 
 import org.apache.hadoop.util.Sets;
 import org.assertj.core.api.Assertions;
-import org.hamcrest.core.StringStartsWith;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -58,6 +57,7 @@
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
 import org.apache.hadoop.fs.s3a.commit.PathCommitException;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.JobContext;
@@ -168,7 +168,7 @@ public void setupCommitter() throws Exception {
     this.tac = new TaskAttemptContextImpl(
         new Configuration(job.getConfiguration()), AID);
 
-    this.jobCommitter = new MockedStagingCommitter(outputPath, tac);
+    this.jobCommitter = new MockedStagingCommitter(getOutputPath(), tac);
     jobCommitter.setupJob(job);
 
     // get the task's configuration copy so modifications take effect
@@ -183,7 +183,7 @@ public void setupCommitter() throws Exception {
     this.conf.set(BUFFER_DIR,
         String.format("%s/local-0/, %s/local-1 ", tmp, tmp));
 
-    this.committer = new MockedStagingCommitter(outputPath, tac);
+    this.committer = new MockedStagingCommitter(getOutputPath(), tac);
     Paths.resetTempFolderCache();
   }
 
@@ -335,10 +335,11 @@ public void testAttemptPathConstructionWithSchema() throws Exception {
     config.set(BUFFER_DIR,
         "file:/tmp/mr-local-0,file:/tmp/mr-local-1");
 
-    assertThat("Path should be the same with file scheme",
+    Assertions.assertThat(
         getLocalTaskAttemptTempDir(config,
-            jobUUID, tac.getTaskAttemptID()).toString(),
-        StringStartsWith.startsWith(commonPath));
+            jobUUID, tac.getTaskAttemptID()).toString())
+        .describedAs("Path should be the same with file scheme")
+        .startsWith(commonPath);
   }
 
   @Test
@@ -379,7 +380,7 @@ public void testSingleTaskCommit() throws Exception {
     assertEquals("Should name the commits file with the task ID: " + results,
         "task_job_0001_r_000002", stats[0].getPath().getName());
 
-    PendingSet pending = PendingSet.load(dfs, stats[0]);
+    PendingSet pending = PersistentCommitData.load(dfs, stats[0], PendingSet.serializer());
     assertEquals("Should have one pending commit", 1, pending.size());
     SinglePendingCommit commit = pending.getCommits().get(0);
     assertEquals("Should write to the correct bucket:" + results,
@@ -419,7 +420,7 @@ public void testSingleTaskEmptyFileCommit() throws Exception {
     assertEquals("Should name the commits file with the task ID",
         "task_job_0001_r_000002", stats[0].getPath().getName());
 
-    PendingSet pending = PendingSet.load(dfs, stats[0]);
+    PendingSet pending = PersistentCommitData.load(dfs, stats[0], PendingSet.serializer());
     assertEquals("Should have one pending commit", 1, pending.size());
   }
 
@@ -442,7 +443,7 @@ public void testSingleTaskMultiFileCommit() throws Exception {
         "task_job_0001_r_000002", stats[0].getPath().getName());
 
     List<SinglePendingCommit> pending =
-        PendingSet.load(dfs, stats[0]).getCommits();
+        PersistentCommitData.load(dfs, stats[0], PendingSet.serializer()).getCommits();
     assertEquals("Should have correct number of pending commits",
         files.size(), pending.size());
 
@@ -717,7 +718,7 @@ private Set<String> runTasks(JobContext jobContext,
       TaskAttemptContext attempt = new TaskAttemptContextImpl(
           new Configuration(jobContext.getConfiguration()), attemptID);
       MockedStagingCommitter taskCommitter = new MockedStagingCommitter(
-          outputPath, attempt);
+          getOutputPath(), attempt);
       commitTask(taskCommitter, attempt, numFiles);
     }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
index 98075b827a7c2..439ef9aa44fcb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
@@ -29,8 +29,12 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
 import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
+import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
 import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
@@ -46,7 +50,7 @@ public class TestStagingDirectoryOutputCommitter
 
   @Override
   DirectoryStagingCommitter newJobCommitter() throws Exception {
-    return new DirectoryStagingCommitter(outputPath,
+    return new DirectoryStagingCommitter(getOutputPath(),
         createTaskAttemptForJob());
   }
 
@@ -63,7 +67,7 @@ public void testBadConflictMode() throws Throwable {
   public void testDefaultConflictResolution() throws Exception {
     getJob().getConfiguration().unset(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE);
-    pathIsDirectory(getMockS3A(), outputPath);
+    pathIsDirectory(getMockS3A(), getOutputPath());
     verifyJobSetupAndCommit();
   }
 
@@ -75,7 +79,8 @@ public void testFailConflictResolution() throws Exception {
   }
 
   protected void verifyFailureConflictOutcome() throws Exception {
-    pathIsDirectory(getMockS3A(), outputPath);
+    final S3AFileSystem mockFS = getMockS3A();
+    pathIsDirectory(mockFS, getOutputPath());
     final DirectoryStagingCommitter committer = newJobCommitter();
 
     // this should fail
@@ -86,20 +91,24 @@ protected void verifyFailureConflictOutcome() throws Exception {
 
     // but there are no checks in job commit (HADOOP-15469)
     // this is done by calling the preCommit method directly,
-    committer.preCommitJob(getJob(), AbstractS3ACommitter.ActiveCommit.empty());
 
-    reset(getMockS3A());
-    pathDoesNotExist(getMockS3A(), outputPath);
+    final CommitContext commitContext = new CommitOperations(getWrapperFS()).
+        createCommitContext(getJob(), getOutputPath(), 0,
+            IOStatisticsContext.getCurrentIOStatisticsContext());
+    committer.preCommitJob(commitContext, AbstractS3ACommitter.ActiveCommit.empty());
+
+    reset(mockFS);
+    pathDoesNotExist(mockFS, getOutputPath());
 
     committer.setupJob(getJob());
-    verifyExistenceChecked(getMockS3A(), outputPath);
-    verifyMkdirsInvoked(getMockS3A(), outputPath);
-    verifyNoMoreInteractions(getMockS3A());
+    verifyExistenceChecked(mockFS, getOutputPath());
+    verifyMkdirsInvoked(mockFS, getOutputPath());
+    verifyNoMoreInteractions(mockFS);
 
-    reset(getMockS3A());
-    pathDoesNotExist(getMockS3A(), outputPath);
+    reset(mockFS);
+    pathDoesNotExist(mockFS, getOutputPath());
     committer.commitJob(getJob());
-    verifyCompletion(getMockS3A());
+    verifyCompletion(mockFS);
   }
 
   @Test
@@ -108,7 +117,7 @@ public void testAppendConflictResolution() throws Exception {
     getJob().getConfiguration().set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
     FileSystem mockS3 = getMockS3A();
-    pathIsDirectory(mockS3, outputPath);
+    pathIsDirectory(mockS3, getOutputPath());
     verifyJobSetupAndCommit();
   }
 
@@ -120,7 +129,7 @@ protected void verifyJobSetupAndCommit()
     FileSystem mockS3 = getMockS3A();
 
     Mockito.reset(mockS3);
-    pathExists(mockS3, outputPath);
+    pathExists(mockS3, getOutputPath());
 
     committer.commitJob(getJob());
     verifyCompletion(mockS3);
@@ -130,7 +139,7 @@ protected void verifyJobSetupAndCommit()
   public void testReplaceConflictResolution() throws Exception {
     FileSystem mockS3 = getMockS3A();
 
-    pathIsDirectory(mockS3, outputPath);
+    pathIsDirectory(mockS3, getOutputPath());
 
     getJob().getConfiguration().set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
@@ -140,17 +149,17 @@ public void testReplaceConflictResolution() throws Exception {
     committer.setupJob(getJob());
 
     Mockito.reset(mockS3);
-    pathExists(mockS3, outputPath);
-    canDelete(mockS3, outputPath);
+    pathExists(mockS3, getOutputPath());
+    canDelete(mockS3, getOutputPath());
 
     committer.commitJob(getJob());
-    verifyDeleted(mockS3, outputPath);
+    verifyDeleted(mockS3, getOutputPath());
     verifyCompletion(mockS3);
   }
 
   @Test
   public void testReplaceConflictFailsIfDestIsFile() throws Exception {
-    pathIsFile(getMockS3A(), outputPath);
+    pathIsFile(getMockS3A(), getOutputPath());
 
     getJob().getConfiguration().set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
@@ -166,7 +175,7 @@ public void testReplaceConflictFailsIfDestIsFile() throws Exception {
 
   @Test
   public void testAppendConflictFailsIfDestIsFile() throws Exception {
-    pathIsFile(getMockS3A(), outputPath);
+    pathIsFile(getMockS3A(), getOutputPath());
 
     getJob().getConfiguration().set(
         FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
index 76a0de225371e..64a9be0888ffa 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
@@ -51,13 +51,13 @@ public class TestStagingPartitionedFileListing
 
   @Override
   PartitionedStagingCommitter newJobCommitter() throws IOException {
-    return new PartitionedStagingCommitter(outputPath,
+    return new PartitionedStagingCommitter(getOutputPath(),
         createTaskAttemptForJob());
   }
 
   @Override
   PartitionedStagingCommitter newTaskCommitter() throws IOException {
-    return new PartitionedStagingCommitter(outputPath, getTAC());
+    return new PartitionedStagingCommitter(getOutputPath(), getTAC());
   }
 
   private FileSystem attemptFS;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
index 86b677c70a305..28161979f0b79 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
@@ -34,7 +34,7 @@
 import org.apache.hadoop.fs.s3a.commit.PathCommitException;
 import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
 import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
-import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitContext;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -67,7 +67,7 @@ private final class PartitionedStagingCommitterForTesting
 
     private PartitionedStagingCommitterForTesting(TaskAttemptContext context)
         throws IOException {
-      super(StagingTestBase.outputPath, context);
+      super(StagingTestBase.getOutputPath(), context);
     }
 
     /**
@@ -75,14 +75,15 @@ private PartitionedStagingCommitterForTesting(TaskAttemptContext context)
      * This is quite complex as the mock pending uploads need to be saved
      * to a filesystem for the next stage of the commit process.
      * To simulate multiple commit, more than one .pendingset file is created,
-     * @param context job context
+     * @param commitContext job context
      * @return an active commit containing a list of paths to valid pending set
      * file.
      * @throws IOException IO failure
      */
+    @SuppressWarnings("CollectionDeclaredAsConcreteClass")
     @Override
     protected ActiveCommit listPendingUploadsToCommit(
-        JobContext context) throws IOException {
+        CommitContext commitContext) throws IOException {
 
       LocalFileSystem localFS = FileSystem.getLocal(getConf());
       ActiveCommit activeCommit = new ActiveCommit(localFS,
@@ -109,17 +110,17 @@ protected ActiveCommit listPendingUploadsToCommit(
         File file = File.createTempFile("staging", ".pendingset");
         file.deleteOnExit();
         Path path = new Path(file.toURI());
-        pendingSet.save(localFS, path, true);
+        pendingSet.save(localFS, path, PendingSet.serializer());
         activeCommit.add(localFS.getFileStatus(path));
       }
       return activeCommit;
     }
 
     @Override
-    protected void abortJobInternal(JobContext context,
+    protected void abortJobInternal(CommitContext commitContext,
         boolean suppressExceptions) throws IOException {
       this.aborted = true;
-      super.abortJobInternal(context, suppressExceptions);
+      super.abortJobInternal(commitContext, suppressExceptions);
     }
   }
 
@@ -242,7 +243,7 @@ public void testReplaceWithDeleteFailure() throws Exception {
     pathsExist(mockS3, "dateint=20161116/hour=14");
     when(mockS3
         .delete(
-            new Path(outputPath, "dateint=20161116/hour=14"),
+            new Path(getOutputPath(), "dateint=20161116/hour=14"),
             true))
         .thenThrow(new PathCommitException("fake",
             "Fake IOException for delete"));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
index fb252102491d6..4e82b94314d34 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
@@ -47,13 +47,13 @@ public class TestStagingPartitionedTaskCommit
 
   @Override
   PartitionedStagingCommitter newJobCommitter() throws IOException {
-    return new PartitionedStagingCommitter(outputPath,
+    return new PartitionedStagingCommitter(getOutputPath(),
         createTaskAttemptForJob());
   }
 
   @Override
   PartitionedStagingCommitter newTaskCommitter() throws Exception {
-    return new PartitionedStagingCommitter(outputPath, getTAC());
+    return new PartitionedStagingCommitter(getOutputPath(), getTAC());
   }
 
   // The set of files used by this test
@@ -104,7 +104,7 @@ public void testFail() throws Exception {
 
     // test failure when one partition already exists
     reset(mockS3);
-    Path existsPath = new Path(outputPath, relativeFiles.get(1)).getParent();
+    Path existsPath = new Path(getOutputPath(), relativeFiles.get(1)).getParent();
     pathExists(mockS3, existsPath);
 
     intercept(PathExistsException.class, "",
@@ -133,7 +133,7 @@ public void testAppend() throws Exception {
 
     // test success when one partition already exists
     reset(mockS3);
-    pathExists(mockS3, new Path(outputPath, relativeFiles.get(2)).getParent());
+    pathExists(mockS3, new Path(getOutputPath(), relativeFiles.get(2)).getParent());
 
     committer.commitTask(getTAC());
     verifyFilesCreated(committer);
@@ -178,7 +178,7 @@ public void testReplace() throws Exception {
 
     // test success when one partition already exists
     reset(mockS3);
-    pathExists(mockS3, new Path(outputPath, relativeFiles.get(3)).getParent());
+    pathExists(mockS3, new Path(getOutputPath(), relativeFiles.get(3)).getParent());
 
     committer.commitTask(getTAC());
     verifyFilesCreated(committer);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
index 36857669553f6..b19662c0117fd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
@@ -37,6 +37,7 @@
 
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.CONFLICT_MODE_APPEND;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_CONFLICT_MODE;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS;
 
 /** ITest of the low level protocol methods. */
 public class ITestDirectoryCommitProtocol extends ITestStagingCommitProtocol {
@@ -51,6 +52,14 @@ protected String getCommitterName() {
     return CommitConstants.COMMITTER_NAME_DIRECTORY;
   }
 
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    // turn off stats collection to verify that it works
+    conf.setBoolean(S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS, false);
+    return  conf;
+  }
+
   @Override
   protected AbstractS3ACommitter createCommitter(
       Path outputPath,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
index 826c3cd2743a7..bb3031b32c1a8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
@@ -27,8 +27,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.InconsistentS3ClientFactory;
-import org.apache.hadoop.fs.s3a.S3ClientFactory;
 import org.apache.hadoop.fs.s3a.commit.AbstractITCommitProtocol;
 import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
 import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
@@ -41,7 +39,6 @@
 import org.apache.hadoop.mapreduce.JobStatus;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
-import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL;
 import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
 
 /** Test the staging committer's handling of the base protocol operations. */
@@ -55,10 +52,7 @@ protected String suitename() {
   @Override
   protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
-    conf.setInt(FS_S3A_COMMITTER_THREADS, 1);
-    // switch to the inconsistent filesystem
-    conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
-        S3ClientFactory.class);
+    conf.setInt(FS_S3A_COMMITTER_THREADS, 4);
 
     // disable unique filenames so that the protocol tests of FileOutputFormat
     // and this test generate consistent names.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java
index 32f909231c217..d28ee5172b632 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java
@@ -125,15 +125,6 @@ public ITestTerasortOnS3A(final String committerName) {
     this.committerName = committerName;
   }
 
-  /**
-   * Not using special paths here.
-   * @return false
-   */
-  @Override
-  public boolean useInconsistentClient() {
-    return false;
-  }
-
   @Override
   protected String committerName() {
     return committerName;
@@ -339,7 +330,7 @@ public void test_140_teracomplete() throws Throwable {
     stage.accept("teravalidate");
     stage.accept("overall");
     String text = results.toString();
-    File resultsFile = File.createTempFile("results", ".csv");
+    File resultsFile = new File(getReportDir(), committerName + ".csv");
     FileUtils.write(resultsFile, text, StandardCharsets.UTF_8);
     LOG.info("Results are in {}\n{}", resultsFile, text);
   }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
index ca8ce15996388..d28f4279f1ba0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
@@ -13,6 +13,7 @@
  */
 package org.apache.hadoop.fs.s3a.fileContext;
 
+import java.io.IOException;
 import java.net.URI;
 
 import com.amazonaws.services.s3.model.CryptoStorageMode;
@@ -32,9 +33,10 @@
 import org.junit.Assert;
 import org.junit.Before;
 
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
-import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm;
+import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH;
 
 /**
@@ -83,12 +85,14 @@ protected void verifyReadBytes(FileSystem.Statistics stats) {
    * @param stats Filesystem statistics.
    */
   @Override
-  protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+  protected void verifyWrittenBytes(FileSystem.Statistics stats)
+      throws IOException {
     //No extra bytes are written
     long expectedBlockSize = blockSize;
-    if (conf.get(S3_ENCRYPTION_ALGORITHM, "")
-        .equals(S3AEncryptionMethods.CSE_KMS.getMethod())) {
-      String keyId = conf.get(S3_ENCRYPTION_KEY, "");
+    if (S3AEncryptionMethods.CSE_KMS.getMethod()
+        .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf)
+            .getMethod())) {
+      String keyId = getS3EncryptionKey(getTestBucketName(conf), conf);
       // Adding padding length and KMS key generation bytes written.
       expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length +
           KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
index 725646ce1bb25..bef359cca73c0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
@@ -16,29 +16,22 @@
 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContextURIBase;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestFileSystem;
-
 /**
  * S3a implementation of FileContextURIBase.
  */
 public class ITestS3AFileContextURI extends FileContextURIBase {
 
   private Configuration conf;
-  private boolean hasMetadataStore;
 
   @Before
   public void setUp() throws IOException, Exception {
     conf = new Configuration();
-    try(S3AFileSystem s3aFS = createTestFileSystem(conf)) {
-      hasMetadataStore = s3aFS.hasMetadataStore();
-    }
+
     fc1 = S3ATestUtils.createTestFileContext(conf);
     fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
     super.setUp();
@@ -51,11 +44,4 @@ public void testFileStatus() throws IOException {
     // (the statistics tested with this method are not relevant for an S3FS)
   }
 
-  @Test
-  @Override
-  public void testModificationTime() throws IOException {
-    // skip modtime tests as there may be some inconsistency during creation
-    assume("modification time tests are skipped", !hasMetadataStore);
-    super.testModificationTime();
-  }
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
index df45d0def5a4f..378f4a70433d7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java
@@ -26,15 +26,9 @@
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;;
 import org.assertj.core.api.Assertions;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -43,13 +37,11 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
 import org.apache.hadoop.util.DurationInfo;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
@@ -70,22 +62,14 @@
 import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.forbidden;
 import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.newAssumedRoleConfig;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
-import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
-import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.extractUndeletedPaths;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.removeUndeletedPaths;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.toPathList;
 import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.assertFileCount;
 import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.extractCause;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
-import static org.apache.hadoop.test.GenericTestUtils.buildPaths;
 import static org.apache.hadoop.test.LambdaTestUtils.eval;
 
 /**
- * Test partial failures of delete and rename operations, especially
- * that the S3Guard tables are consistent with the state of
- * the filesystem.
+ * Test partial failures of delete and rename operations,.
  *
  * All these test have a unique path for each run, with a roleFS having
  * full RW access to part of it, and R/O access to a restricted subdirectory
@@ -105,11 +89,6 @@
  *   </li>
  * </ol>
  *
- * This test manages to create lots of load on the s3guard prune command
- * when that is tested in a separate test suite;
- * too many tombstone files for the test to complete.
- * An attempt is made in {@link #deleteTestDirInTeardown()} to prune these test
- * files.
  */
 @SuppressWarnings("ThrowableNotThrown")
 @RunWith(Parameterized.class)
@@ -123,20 +102,6 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase {
   private static final Statement STATEMENT_ALL_BUCKET_READ_ACCESS
       = statement(true, S3_ALL_BUCKETS, S3_BUCKET_READ_OPERATIONS);
 
-  /** Many threads for scale performance: {@value}. */
-  public static final int EXECUTOR_THREAD_COUNT = 64;
-
-  /**
-   * For submitting work.
-   */
-  private static final ListeningExecutorService EXECUTOR =
-      MoreExecutors.listeningDecorator(
-          BlockingThreadPoolExecutorService.newInstance(
-              EXECUTOR_THREAD_COUNT,
-              EXECUTOR_THREAD_COUNT * 2,
-              30, TimeUnit.SECONDS,
-              "test-operations"));
-
 
   /**
    * The number of files in a non-scaled test.
@@ -261,7 +226,6 @@ public void setup() throws Exception {
     // create the baseline assumed role
     assumedRoleConfig = createAssumedRoleConfig();
     bindRolePolicyStatements(assumedRoleConfig,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         STATEMENT_ALL_BUCKET_READ_ACCESS,  // root:     r-x
         new Statement(Effects.Allow)       // dest:     rwx
@@ -335,14 +299,10 @@ protected Configuration createConfiguration() {
     removeBucketOverrides(bucketName, conf,
         MAX_THREADS,
         MAXIMUM_CONNECTIONS,
-        S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
         DIRECTORY_MARKER_POLICY,
         BULK_DELETE_PAGE_SIZE);
     conf.setInt(MAX_THREADS, EXECUTOR_THREAD_COUNT);
     conf.setInt(MAXIMUM_CONNECTIONS, EXECUTOR_THREAD_COUNT * 2);
-    // turn off prune delays, so as to stop scale tests creating
-    // so much cruft that future CLI prune commands take forever
-    conf.setInt(S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY, 0);
     // use the keep policy to ensure that surplus markers exist
     // to complicate failures
     conf.set(DIRECTORY_MARKER_POLICY, DIRECTORY_MARKER_POLICY_KEEP);
@@ -406,7 +366,6 @@ public void testRenameParentPathNotWriteable() throws Throwable {
     describe("rename with parent paths not writeable; multi=%s", multiDelete);
     final Configuration conf = createAssumedRoleConfig();
     bindRolePolicyStatements(conf,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         STATEMENT_ALL_BUCKET_READ_ACCESS,
         new Statement(Effects.Allow)
@@ -510,34 +469,19 @@ public void testRenameDirFailsInDelete() throws Throwable {
       // look in that exception for a multidelete
       MultiObjectDeleteException mde = extractCause(
           MultiObjectDeleteException.class, deniedException);
-      final List<Path> undeleted
-          = extractUndeletedPaths(mde, fs::keyToQualifiedPath);
-
-      List<Path> expectedUndeletedFiles = new ArrayList<>(createdFiles);
-      if (getFileSystem().getDirectoryMarkerPolicy()
-          .keepDirectoryMarkers(readOnlyDir)) {
-        // directory markers are being retained,
-        // so will also be in the list of undeleted files
-        expectedUndeletedFiles.addAll(dirs);
-      }
-      Assertions.assertThat(undeleted)
-          .as("files which could not be deleted")
-          .containsExactlyInAnyOrderElementsOf(expectedUndeletedFiles);
     }
     LOG.info("Result of renaming read-only files is as expected",
         deniedException);
     assertFileCount("files in the source directory", roleFS,
         readOnlyDir, expectedFileCount);
     // now lets look at the destination.
-    // even with S3Guard on, we expect the destination to match that of
+    // we expect the destination to match that of
     // the remote state.
-    // the test will exist
     describe("Verify destination directory exists");
     assertIsDirectory(writableDir);
     assertFileCount("files in the dest directory", roleFS,
         writableDir, expectedFileCount);
     // all directories in the source tree must still exist,
-    // which for S3Guard means no tombstone markers were added
     LOG.info("Verifying all directories still exist");
     for (Path dir : dirs) {
       assertIsDirectory(dir);
@@ -653,17 +597,6 @@ public void testPartialDirDelete() throws Throwable {
     Path head = deletableFiles.remove(0);
     assertTrue("delete " + head + " failed",
         roleFS.delete(head, false));
-    List<Path> allFiles = Stream.concat(
-        readOnlyFiles.stream(),
-        deletableFiles.stream())
-        .collect(Collectors.toList());
-    List<MultiObjectDeleteSupport.KeyPath> keyPaths = allFiles.stream()
-        .map(path ->
-            new MultiObjectDeleteSupport.KeyPath(
-                storeContext.pathToKey(path),
-                path,
-                false))
-        .collect(Collectors.toList());
 
     // this set can be deleted by the role FS
     MetricDiff rejectionCount = new MetricDiff(roleFS, FILES_DELETE_REJECTED);
@@ -697,34 +630,10 @@ public void testPartialDirDelete() throws Throwable {
     ex = expectDeleteForbidden(basePath);
     String iostats = ioStatisticsSourceToString(roleFS);
 
-    if (multiDelete) {
-      // multi-delete status checks
-      deleteVerbCount.assertDiffEquals("Wrong delete request count", 0);
-      bulkDeleteVerbCount.assertDiffEquals(
-          "Wrong count of delete operations in " + iostats, 1);
-      MultiObjectDeleteException mde = extractCause(
-          MultiObjectDeleteException.class, ex);
-      List<MultiObjectDeleteSupport.KeyPath> undeletedKeyPaths =
-          removeUndeletedPaths(mde, keyPaths, storeContext::keyToPath);
-      final List<Path> undeleted = toPathList(
-          undeletedKeyPaths);
-      deleteObjectCount.assertDiffEquals(
-          "Wrong count of objects in delete request",
-          allFiles.size());
-      Assertions.assertThat(undeleted)
-          .as("files which could not be deleted")
-          .containsExactlyInAnyOrderElementsOf(readOnlyFiles);
-      Assertions.assertThat(toPathList(keyPaths))
-          .as("files which were deleted")
-          .containsExactlyInAnyOrderElementsOf(deletableFiles);
-      rejectionCount.assertDiffEquals("Wrong rejection count",
-          readOnlyFiles.size());
-    }
     reset(rejectionCount, deleteVerbCount);
 
     // build the set of all paths under the directory tree through
     // a directory listing (i.e. not getFileStatus()).
-    // small risk of observed inconsistency here on unguarded stores.
     final Set<Path> readOnlyListing = listFilesUnderPath(readOnlyDir, true);
 
     String directoryList = readOnlyListing.stream()
@@ -736,25 +645,6 @@ public void testPartialDirDelete() throws Throwable {
         .containsExactlyInAnyOrderElementsOf(readOnlyFiles);
   }
 
-  /**
-   * Verifies the logic of handling directory markers in
-   * delete operations, specifically:
-   * <ol>
-   *   <li>all markers above empty directories MUST be deleted</li>
-   *   <li>all markers above non-empty directories MUST NOT be deleted</li>
-   * </ol>
-   * As the delete list may include subdirectories, we need to work up from
-   * the bottom of the list of deleted files before probing the parents,
-   * that being done by a s3guard get(path, need-empty-directory) call.
-   * <p></p>
-   * This is pretty sensitive code.
-   */
-  @Test
-  public void testSubdirDeleteFailures() throws Throwable {
-    describe("Multiobject delete handling of directorYesFory markers");
-    assume("Multiobject delete only", multiDelete);
-  }
-
   /**
    * Expect the delete() call to fail.
    * @param path path to delete.
@@ -828,94 +718,10 @@ private Set<Path> listFilesUnderPath(Path path, boolean recursive)
     return files;
   }
 
-  /**
-   * Write the text to a file asynchronously. Logs the operation duration.
-   * @param fs filesystem
-   * @param path path
-   * @return future to the patch created.
-   */
-  private static CompletableFuture<Path> put(FileSystem fs,
-      Path path, String text) {
-    return submit(EXECUTOR, () -> {
-      try (DurationInfo ignore =
-               new DurationInfo(LOG, false, "Creating %s", path)) {
-        createFile(fs, path, true, text.getBytes(Charsets.UTF_8));
-        return path;
-      }
-    });
-  }
-
-  /**
-   * Build a set of files in a directory tree.
-   * @param fs filesystem
-   * @param destDir destination
-   * @param depth file depth
-   * @param fileCount number of files to create.
-   * @param dirCount number of dirs to create at each level
-   * @return the list of files created.
-   */
-  public static List<Path> createFiles(final FileSystem fs,
-      final Path destDir,
-      final int depth,
-      final int fileCount,
-      final int dirCount) throws IOException {
-    return createDirsAndFiles(fs, destDir, depth, fileCount, dirCount,
-        new ArrayList<Path>(fileCount),
-        new ArrayList<Path>(dirCount));
-  }
-
-  /**
-   * Build a set of files in a directory tree.
-   * @param fs filesystem
-   * @param destDir destination
-   * @param depth file depth
-   * @param fileCount number of files to create.
-   * @param dirCount number of dirs to create at each level
-   * @param paths [out] list of file paths created
-   * @param dirs [out] list of directory paths created.
-   * @return the list of files created.
-   */
-  public static List<Path> createDirsAndFiles(final FileSystem fs,
-      final Path destDir,
-      final int depth,
-      final int fileCount,
-      final int dirCount,
-      final List<Path> paths,
-      final List<Path> dirs) throws IOException {
-    buildPaths(paths, dirs, destDir, depth, fileCount, dirCount);
-    List<CompletableFuture<Path>> futures = new ArrayList<>(paths.size()
-        + dirs.size());
-
-    // create directories. With dir marker retention, that adds more entries
-    // to cause deletion issues
-    try (DurationInfo ignore =
-             new DurationInfo(LOG, "Creating %d directories", dirs.size())) {
-      for (Path path : dirs) {
-        futures.add(submit(EXECUTOR, () ->{
-          fs.mkdirs(path);
-          return path;
-        }));
-      }
-      waitForCompletion(futures);
-    }
-
-    try (DurationInfo ignore =
-            new DurationInfo(LOG, "Creating %d files", paths.size())) {
-      for (Path path : paths) {
-        futures.add(put(fs, path, path.getName()));
-      }
-      waitForCompletion(futures);
-      return paths;
-    }
-  }
-
   /**
    * Verifies that s3:DeleteObjectVersion is not required for rename.
    * <p></p>
    * See HADOOP-17621.
-   * <p></p>
-   * This test will only show a regression if the bucket has versioning
-   * enabled *and* S3Guard is enabled.
    */
   @Test
   public void testRenamePermissionRequirements() throws Throwable {
@@ -928,7 +734,6 @@ public void testRenamePermissionRequirements() throws Throwable {
     // and then delete.
     Configuration roleConfig = createAssumedRoleConfig();
     bindRolePolicyStatements(roleConfig,
-        STATEMENT_S3GUARD_CLIENT,
         STATEMENT_ALLOW_SSE_KMS_RW,
         STATEMENT_ALL_BUCKET_READ_ACCESS,  // root:     r-x
         new Statement(Effects.Allow)       // dest:     rwx
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java
index 9885eb5698477..2610f54b44e9e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestRenameDeleteRace.java
@@ -39,7 +39,6 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableS3GuardInTestBucket;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit;
@@ -82,7 +81,7 @@ protected Configuration createConfiguration() {
     removeBaseAndBucketOverrides(getTestBucketName(conf),
         conf,
         DIRECTORY_MARKER_POLICY);
-    disableS3GuardInTestBucket(conf);
+
     return conf;
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
index 415dcba0f575f..3a390e34ecad2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestXAttrCost.java
@@ -58,8 +58,7 @@ public class ITestXAttrCost extends AbstractS3ACostTest {
   private static final int GET_METADATA_ON_DIR = GET_METADATA_ON_OBJECT * 2;
 
   public ITestXAttrCost() {
-    // no parameterization here
-    super(false, true, false);
+    super(true);
   }
 
   @Test
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java
new file mode 100644
index 0000000000000..41180667e1000
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.api.RequestFactory;
+import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
+
+/**
+ * A Stub context acccessor for test.
+ */
+public final class StubContextAccessor
+    implements ContextAccessors {
+
+  private final String bucket;
+
+  /**
+   * Construct.
+   * @param bucket bucket to use when qualifying keys.]=
+   */
+  public StubContextAccessor(String bucket) {
+    this.bucket = bucket;
+  }
+
+  @Override
+  public Path keyToPath(final String key) {
+    return new Path("s3a://" + bucket + "/" + key);
+  }
+
+  @Override
+  public String pathToKey(final Path path) {
+    return null;
+  }
+
+  @Override
+  public File createTempFile(final String prefix, final long size)
+      throws IOException {
+    throw new UnsupportedOperationException("unsppported");
+  }
+
+  @Override
+  public String getBucketLocation() throws IOException {
+    return null;
+  }
+
+  @Override
+  public Path makeQualified(final Path path) {
+    return path;
+  }
+
+  @Override
+  public AuditSpan getActiveAuditSpan() {
+    return AuditTestSupport.NOOP_SPAN;
+  }
+
+  @Override
+  public RequestFactory getRequestFactory() {
+    return MockS3AFileSystem.REQUEST_FACTORY;
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java
new file mode 100644
index 0000000000000..65d7aa6192dd8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.HadoopTestBase;
+import org.apache.hadoop.util.Progressable;
+
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit test of {@link CreateFileBuilder}.
+ */
+public class TestCreateFileBuilder extends HadoopTestBase {
+
+  private static final BuilderCallbacks CALLBACKS = new BuilderCallbacks();
+
+  private CreateFileBuilder mkBuilder() throws IOException {
+    return new CreateFileBuilder(
+        FileSystem.getLocal(new Configuration()),
+        new Path("/"),
+        CALLBACKS);
+  }
+
+  private BuilderOutputStream unwrap(FSDataOutputStream out) {
+    OutputStream s = out.getWrappedStream();
+    Assertions.assertThat(s)
+        .isInstanceOf(BuilderOutputStream.class);
+    return (BuilderOutputStream) s;
+  }
+
+  private BuilderOutputStream build(FSDataOutputStreamBuilder builder)
+      throws IOException {
+    return unwrap(builder.build());
+  }
+
+  @Test
+  public void testSimpleBuild() throws Throwable {
+    Assertions.assertThat(build(mkBuilder().create()))
+        .matches(p -> !p.isOverwrite())
+        .matches(p -> !p.isPerformance());
+  }
+
+  @Test
+  public void testAppendForbidden() throws Throwable {
+    intercept(UnsupportedOperationException.class, () ->
+        build(mkBuilder().append()));
+  }
+
+  @Test
+  public void testPerformanceSupport() throws Throwable {
+    CreateFileBuilder builder = mkBuilder().create();
+    builder.must(FS_S3A_CREATE_PERFORMANCE, true);
+    Assertions.assertThat(build(builder))
+        .matches(p -> p.isPerformance());
+  }
+
+  @Test
+  public void testHeaderOptions() throws Throwable {
+    final CreateFileBuilder builder = mkBuilder().create()
+        .must(FS_S3A_CREATE_HEADER + ".retention", "permanent")
+        .opt(FS_S3A_CREATE_HEADER + ".owner", "engineering");
+    final Map<String, String> headers = build(builder).getHeaders();
+    Assertions.assertThat(headers)
+        .containsEntry("retention", "permanent")
+        .containsEntry("owner", "engineering");
+  }
+
+  @Test
+  public void testIncompleteHeader() throws Throwable {
+    final CreateFileBuilder builder = mkBuilder().create()
+        .must(FS_S3A_CREATE_HEADER, "permanent");
+    intercept(IllegalArgumentException.class, () ->
+        build(builder));
+  }
+
+  private static final class BuilderCallbacks implements
+      CreateFileBuilder.CreateFileBuilderCallbacks {
+
+    @Override
+    public FSDataOutputStream createFileFromBuilder(final Path path,
+        final Progressable progress,
+        final CreateFileBuilder.CreateFileOptions options) throws IOException {
+      return new FSDataOutputStream(
+          new BuilderOutputStream(
+              progress,
+              options),
+          null);
+    }
+  }
+
+  /**
+   * Stream which will be wrapped and which returns the flags used
+   * creating the object.
+   */
+  private static final class BuilderOutputStream extends OutputStream {
+
+    private final Progressable progress;
+
+
+    private final CreateFileBuilder.CreateFileOptions options;
+
+    private BuilderOutputStream(final Progressable progress,
+        final CreateFileBuilder.CreateFileOptions options) {
+      this.progress = progress;
+      this.options = options;
+    }
+
+    private boolean isOverwrite() {
+      return options.getFlags().contains(CreateFlag.OVERWRITE);
+    }
+
+    private Progressable getProgress() {
+      return progress;
+    }
+
+    private boolean isPerformance() {
+      return options.isPerformance();
+    }
+
+    private CreateFileBuilder.CreateFileOptions getOptions() {
+      return options;
+    }
+
+    private Map<String, String> getHeaders() {
+      return options.getHeaders();
+    }
+
+    @Override
+    public void write(final int b) throws IOException {
+
+    }
+
+    @Override
+    public String toString() {
+      return "BuilderOutputStream{" +
+          "progress=" + progress +
+          ", options=" + options +
+          "} " + super.toString();
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java
index 15c7ae917dfc5..82592b1d01950 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestHeaderProcessing.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.api.RequestFactory;
 import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
-import org.apache.hadoop.fs.s3a.test.OperationTrackingStore;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.test.HadoopTestBase;
 
@@ -98,7 +97,7 @@ public void setup() throws Exception {
         X_HEADER_MAGIC_MARKER,
         Long.toString(MAGIC_LEN));
     context = S3ATestUtils.createMockStoreContext(true,
-        new OperationTrackingStore(), CONTEXT_ACCESSORS);
+        CONTEXT_ACCESSORS);
     headerProcessing = new HeaderProcessing(context, CONTEXT_ACCESSORS);
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestOpenFileSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestOpenFileSupport.java
new file mode 100644
index 0000000000000..17f210dd586e8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestOpenFileSupport.java
@@ -0,0 +1,429 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.ObjectAssert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static java.util.Collections.singleton;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_RANDOM;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_END;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_SPLIT_START;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ASYNC_DRAIN_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
+import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit tests for {@link OpenFileSupport} and the associated
+ * seek policy lookup in {@link S3AInputPolicy}.
+ */
+public class TestOpenFileSupport extends HadoopTestBase {
+
+  private static final ChangeDetectionPolicy CHANGE_POLICY =
+      ChangeDetectionPolicy.createPolicy(
+          ChangeDetectionPolicy.Mode.Server,
+          ChangeDetectionPolicy.Source.None,
+          false);
+
+  private static final long READ_AHEAD_RANGE = 16;
+
+  private static final String USERNAME = "hadoop";
+
+  public static final S3AInputPolicy INPUT_POLICY = S3AInputPolicy.Sequential;
+
+  public static final String TESTFILE = "s3a://bucket/name";
+
+  private static final Path TESTPATH = new Path(TESTFILE);
+
+  /**
+   * Create a OpenFileSupport instance.
+   */
+  private static final OpenFileSupport PREPARE =
+      new OpenFileSupport(
+          CHANGE_POLICY,
+          READ_AHEAD_RANGE,
+          USERNAME,
+          IO_FILE_BUFFER_SIZE_DEFAULT,
+          DEFAULT_ASYNC_DRAIN_THRESHOLD,
+          INPUT_POLICY);
+
+  @Test
+  public void testSimpleFile() throws Throwable {
+    ObjectAssert<OpenFileSupport.OpenFileInformation>
+        asst = assertFileInfo(
+            PREPARE.openSimpleFile(1024));
+
+    asst.extracting(f -> f.getChangePolicy())
+        .isEqualTo(CHANGE_POLICY);
+    asst.extracting(f -> f.getInputPolicy())
+        .isEqualTo(INPUT_POLICY);
+    asst.extracting(f -> f.getReadAheadRange())
+        .isEqualTo(READ_AHEAD_RANGE);
+  }
+
+  /**
+   * Initiate an assert from an open file information instance.
+   * @param fi file info
+   * @return an assert stream.
+   */
+  private ObjectAssert<OpenFileSupport.OpenFileInformation> assertFileInfo(
+      final OpenFileSupport.OpenFileInformation fi) {
+    return Assertions.assertThat(fi)
+        .describedAs("File Information %s", fi);
+  }
+
+  /**
+   * Create an assertion about the openFile information from a configuration
+   * with the given key/value option.
+   * @param key key to set.
+   * @param option option value.
+   * @return the constructed OpenFileInformation.
+   */
+  public ObjectAssert<OpenFileSupport.OpenFileInformation> assertOpenFile(
+      final String key,
+      final String option) throws IOException {
+    return assertFileInfo(prepareToOpenFile(params(key, option)));
+  }
+
+  @Test
+  public void testUnknownMandatoryOption() throws Throwable {
+
+    String key = "unknown";
+    intercept(IllegalArgumentException.class, key, () ->
+        prepareToOpenFile(params(key, "undefined")));
+  }
+
+  @Test
+  public void testSeekRandomIOPolicy() throws Throwable {
+
+    // ask for random IO
+    String option = FS_OPTION_OPENFILE_READ_POLICY_RANDOM;
+
+    // is picked up
+    assertOpenFile(INPUT_FADVISE, option)
+        .extracting(f -> f.getInputPolicy())
+        .isEqualTo(S3AInputPolicy.Random);
+    // and as neither status nor length was set: no file status
+    assertOpenFile(INPUT_FADVISE, option)
+        .extracting(f -> f.getStatus())
+        .isNull();
+  }
+
+  /**
+   * There's a standard policy name. 'adaptive',
+   * meaning 'whatever this stream does to adapt to the client's use'.
+   * On the S3A connector that is mapped to {@link S3AInputPolicy#Normal}.
+   */
+  @Test
+  public void testSeekPolicyAdaptive() throws Throwable {
+
+    // when caller asks for adaptive, they get "normal"
+    assertOpenFile(FS_OPTION_OPENFILE_READ_POLICY,
+        FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE)
+        .extracting(f -> f.getInputPolicy())
+        .isEqualTo(S3AInputPolicy.Normal);
+  }
+
+  /**
+   * Verify that an unknown seek policy falls back to
+   * {@link S3AInputPolicy#Normal}.
+   */
+  @Test
+  public void testUnknownSeekPolicyS3AOption() throws Throwable {
+    // fall back to the normal seek policy.
+    assertOpenFile(INPUT_FADVISE, "undefined")
+        .extracting(f -> f.getInputPolicy())
+        .isEqualTo(INPUT_POLICY);
+  }
+
+  /**
+   * The S3A option also supports a list of values.
+   */
+  @Test
+  public void testSeekPolicyListS3AOption() throws Throwable {
+    // fall back to the second seek policy if the first is unknown
+    assertOpenFile(INPUT_FADVISE, "hbase, random")
+        .extracting(f -> f.getInputPolicy())
+        .isEqualTo(S3AInputPolicy.Random);
+  }
+
+  /**
+   * Verify that if a list of policies is supplied in a configuration,
+   * the first recognized policy will be adopted.
+   */
+  @Test
+  public void testSeekPolicyExtractionFromList() throws Throwable {
+    String plist = "a, b, RandOm, other ";
+    Configuration conf = conf(FS_OPTION_OPENFILE_READ_POLICY, plist);
+    Collection<String> options = conf.getTrimmedStringCollection(
+        FS_OPTION_OPENFILE_READ_POLICY);
+    Assertions.assertThat(S3AInputPolicy.getFirstSupportedPolicy(options, null))
+        .describedAs("Policy from " + plist)
+        .isEqualTo(S3AInputPolicy.Random);
+  }
+
+  @Test
+  public void testAdaptiveSeekPolicyRecognized() throws Throwable {
+    Assertions.assertThat(S3AInputPolicy.getPolicy("adaptive", null))
+        .describedAs("adaptive")
+        .isEqualTo(S3AInputPolicy.Normal);
+  }
+
+  @Test
+  public void testUnknownSeekPolicyFallback() throws Throwable {
+    Assertions.assertThat(S3AInputPolicy.getPolicy("unknown", null))
+        .describedAs("unknown policy")
+        .isNull();
+  }
+
+  /**
+   * Test the mapping of the standard option names.
+   */
+  @Test
+  public void testInputPolicyMapping() throws Throwable {
+    Object[][] policyMapping = {
+        {"normal", S3AInputPolicy.Normal},
+        {FS_OPTION_OPENFILE_READ_POLICY_ADAPTIVE, S3AInputPolicy.Normal},
+        {FS_OPTION_OPENFILE_READ_POLICY_DEFAULT, S3AInputPolicy.Normal},
+        {FS_OPTION_OPENFILE_READ_POLICY_RANDOM, S3AInputPolicy.Random},
+        {FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL, S3AInputPolicy.Sequential},
+    };
+    for (Object[] mapping : policyMapping) {
+      String name = (String) mapping[0];
+      Assertions.assertThat(S3AInputPolicy.getPolicy(name, null))
+          .describedAs("Policy %s", name)
+          .isEqualTo(mapping[1]);
+    }
+  }
+
+  /**
+   * Verify readahead range is picked up.
+   */
+  @Test
+  public void testReadahead() throws Throwable {
+    // readahead range option
+    assertOpenFile(READAHEAD_RANGE, "4096")
+        .extracting(f -> f.getReadAheadRange())
+        .isEqualTo(4096L);
+  }
+
+  /**
+   * Verify buffer size  is picked up.
+   */
+  @Test
+  public void testBufferSize() throws Throwable {
+    // readahead range option
+    assertOpenFile(FS_OPTION_OPENFILE_BUFFER_SIZE, "4096")
+        .extracting(f -> f.getBufferSize())
+        .isEqualTo(4096);
+  }
+
+  @Test
+  public void testStatusWithValidFilename() throws Throwable {
+    Path p = new Path("file:///tmp/" + TESTPATH.getName());
+    ObjectAssert<OpenFileSupport.OpenFileInformation> asst =
+        assertFileInfo(prepareToOpenFile(
+            params(FS_OPTION_OPENFILE_LENGTH, "32")
+                .withStatus(status(p, 4096))));
+    asst.extracting(f -> f.getStatus().getVersionId())
+        .isEqualTo("version");
+    asst.extracting(f -> f.getStatus().getEtag())
+        .isEqualTo("etag");
+    asst.extracting(f -> f.getStatus().getLen())
+        .isEqualTo(4096L);
+  }
+
+  /**
+   * Verify S3ALocatedFileStatus is handled.
+   */
+  @Test
+  public void testLocatedStatus() throws Throwable {
+    Path p = new Path("file:///tmp/" + TESTPATH.getName());
+    ObjectAssert<OpenFileSupport.OpenFileInformation> asst =
+        assertFileInfo(
+            prepareToOpenFile(
+                params(FS_OPTION_OPENFILE_LENGTH, "32")
+                    .withStatus(
+                        new S3ALocatedFileStatus(
+                            status(p, 4096), null))));
+    asst.extracting(f -> f.getStatus().getVersionId())
+        .isEqualTo("version");
+    asst.extracting(f -> f.getStatus().getEtag())
+        .isEqualTo("etag");
+    asst.extracting(f -> f.getStatus().getLen())
+        .isEqualTo(4096L);
+  }
+
+  /**
+   * Callers cannot supply a directory status when opening a file.
+   */
+  @Test
+  public void testDirectoryStatus() throws Throwable {
+    intercept(FileNotFoundException.class, TESTFILE, () ->
+        prepareToOpenFile(
+            params(INPUT_FADVISE, "normal")
+                .withStatus(new S3AFileStatus(true, TESTPATH, USERNAME))));
+  }
+
+  /**
+   * File name must match the path argument to openFile().
+   */
+  @Test
+  public void testStatusWithInconsistentFilename() throws Throwable {
+    intercept(IllegalArgumentException.class, TESTFILE, () ->
+        prepareToOpenFile(params(INPUT_FADVISE, "normal")
+            .withStatus(new S3AFileStatus(true,
+                new Path(TESTFILE + "-"), USERNAME))));
+  }
+
+  /**
+   * Prepare to open a file with the set of parameters.
+   * @param parameters open a file
+   * @return
+   * @throws IOException
+   */
+  public OpenFileSupport.OpenFileInformation prepareToOpenFile(
+      final OpenFileParameters parameters)
+      throws IOException {
+    return PREPARE.prepareToOpenFile(TESTPATH,
+        parameters,
+        IO_FILE_BUFFER_SIZE_DEFAULT
+    );
+  }
+
+  /**
+   * If a file length option is set, a file status
+   * is created.
+   */
+  @Test
+  public void testFileLength() throws Throwable {
+    ObjectAssert<OpenFileSupport.OpenFileInformation> asst =
+        assertFileInfo(prepareToOpenFile(
+            params(FS_OPTION_OPENFILE_LENGTH, "8192")
+                .withStatus(null)));
+    asst.extracting(f -> f.getStatus())
+        .isNotNull();
+    asst.extracting(f -> f.getStatus().getPath())
+        .isEqualTo(TESTPATH);
+    asst.extracting(f -> f.getStatus().getLen())
+        .isEqualTo(8192L);
+  }
+
+  /**
+   * Verify that setting the split end sets the length.
+   * By passing in a value greater than the size of an int,
+   * the test verifies that the long is passed everywhere.
+   */
+  @Test
+  public void testSplitEndSetsLength() throws Throwable {
+    long bigFile = 2L ^ 34;
+    assertOpenFile(FS_OPTION_OPENFILE_SPLIT_END, Long.toString(bigFile))
+        .matches(p -> p.getSplitEnd() == bigFile, "split end")
+        .matches(p -> p.getFileLength() == -1, "file length")
+        .matches(p -> p.getStatus() == null, "status");
+  }
+
+  /**
+   * Semantics of split and length. Split end can only be safely treated
+   * as a hint unless the codec is known (how?) that it will never
+   * read past it.
+   */
+  @Test
+  public void testSplitEndAndLength() throws Throwable {
+    long splitEnd = 256;
+    long len = 8192;
+    Configuration conf = conf(FS_OPTION_OPENFILE_LENGTH,
+        Long.toString(len));
+    conf.setLong(FS_OPTION_OPENFILE_SPLIT_END, splitEnd);
+    conf.setLong(FS_OPTION_OPENFILE_SPLIT_START, 1024);
+    Set<String> s = new HashSet<>();
+    Collections.addAll(s,
+        FS_OPTION_OPENFILE_SPLIT_START,
+        FS_OPTION_OPENFILE_SPLIT_END,
+        FS_OPTION_OPENFILE_LENGTH);
+    assertFileInfo(prepareToOpenFile(
+        new OpenFileParameters()
+            .withMandatoryKeys(s)
+            .withOptions(conf)))
+        .matches(p -> p.getSplitStart() == 0, "split start")
+        .matches(p -> p.getSplitEnd() == splitEnd, "split end")
+        .matches(p -> p.getStatus().getLen() == len, "file length");
+  }
+
+  /**
+   * Create an S3A status entry with stub etag and versions, timestamp of 0.
+   * @param path status path
+   * @param length file length
+   * @return a status instance.
+   */
+  private S3AFileStatus status(final Path path, final int length) {
+    return new S3AFileStatus(length, 0,
+        path, 0, "", "etag", "version");
+  }
+
+  /**
+   * Create an instance of {@link OpenFileParameters} with
+   * the key as a mandatory parameter.
+   * @param key mandatory key
+   * @param val value
+   * @return the instance.
+   */
+  private OpenFileParameters params(final String key, final String val) {
+    return new OpenFileParameters()
+        .withMandatoryKeys(singleton(key))
+        .withOptions(conf(key, val));
+  }
+
+  /**
+   * Create a configuration with a single entry.
+   * @param key entry key
+   * @param val entry value
+   * @return a configuration
+   */
+  private Configuration conf(String key, Object val) {
+    Configuration c = new Configuration(false);
+    c.set(key, val.toString());
+    return c;
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java
deleted file mode 100644
index f43860e1e8b36..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestPartialDeleteFailures.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.MultiObjectDeleteException;
-import org.apache.hadoop.util.Lists;
-import org.assertj.core.api.Assertions;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.commons.lang3.tuple.Triple;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.api.RequestFactory;
-import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
-import org.apache.hadoop.fs.s3a.test.OperationTrackingStore;
-import org.apache.hadoop.fs.store.audit.AuditSpan;
-
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.ACCESS_DENIED;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.removeUndeletedPaths;
-import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.toPathList;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Unit test suite covering translation of AWS SDK exceptions to S3A exceptions,
- * and retry/recovery policies.
- */
-public class TestPartialDeleteFailures {
-
-  private static final ContextAccessors CONTEXT_ACCESSORS
-      = new MinimalContextAccessor();
-
-  private StoreContext context;
-
-  private static Path qualifyKey(String k) {
-    return new Path("s3a://bucket/" + k);
-  }
-
-  private static String toKey(Path path) {
-    return path.toUri().getPath();
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    context = S3ATestUtils.createMockStoreContext(true,
-        new OperationTrackingStore(), CONTEXT_ACCESSORS);
-  }
-
-  @Test
-  public void testDeleteExtraction() {
-    List<MultiObjectDeleteSupport.KeyPath> src = pathList("a", "a/b", "a/c");
-    List<MultiObjectDeleteSupport.KeyPath> rejected = pathList("a/b");
-    MultiObjectDeleteException ex = createDeleteException(ACCESS_DENIED,
-        rejected);
-    List<MultiObjectDeleteSupport.KeyPath> undeleted =
-        removeUndeletedPaths(ex, src,
-            TestPartialDeleteFailures::qualifyKey);
-    assertEquals("mismatch of rejected and undeleted entries",
-        rejected, undeleted);
-  }
-
-  @Test
-  public void testSplitKeysFromResults() throws Throwable {
-    List<MultiObjectDeleteSupport.KeyPath> src = pathList("a", "a/b", "a/c");
-    List<MultiObjectDeleteSupport.KeyPath> rejected = pathList("a/b");
-    List<DeleteObjectsRequest.KeyVersion> keys = keysToDelete(toPathList(src));
-    MultiObjectDeleteException ex = createDeleteException(ACCESS_DENIED,
-        rejected);
-    Pair<List<MultiObjectDeleteSupport.KeyPath>,
-        List<MultiObjectDeleteSupport.KeyPath>> pair =
-        new MultiObjectDeleteSupport(context, null)
-          .splitUndeletedKeys(ex, keys);
-    List<MultiObjectDeleteSupport.KeyPath> undeleted = pair.getLeft();
-    List<MultiObjectDeleteSupport.KeyPath> deleted = pair.getRight();
-    assertEquals(rejected, undeleted);
-    // now check the deleted list to verify that it is valid
-    src.remove(rejected.get(0));
-    assertEquals(src, deleted);
-  }
-
-  /**
-   * Build a list of qualified paths from vararg parameters.
-   * @param paths paths to qualify and then convert to a lst.
-   * @return same paths as a list.
-   */
-  private List<MultiObjectDeleteSupport.KeyPath> pathList(String... paths) {
-    return Arrays.stream(paths)
-        .map(k->
-            new MultiObjectDeleteSupport.KeyPath(k,
-                qualifyKey(k),
-                k.endsWith("/")))
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Build a delete exception containing all the rejected paths.
-   * The list of successful entries is empty.
-   * @param rejected the rejected paths.
-   * @return a new exception
-   */
-  private MultiObjectDeleteException createDeleteException(
-      final String code,
-      final List<MultiObjectDeleteSupport.KeyPath> rejected) {
-    List<MultiObjectDeleteException.DeleteError> errors = rejected.stream()
-        .map((kp) -> {
-          Path p = kp.getPath();
-          MultiObjectDeleteException.DeleteError e
-              = new MultiObjectDeleteException.DeleteError();
-          e.setKey(kp.getKey());
-          e.setCode(code);
-          e.setMessage("forbidden");
-          return e;
-        }).collect(Collectors.toList());
-    return new MultiObjectDeleteException(errors, Collections.emptyList());
-  }
-
-  /**
-   * From a list of paths, build up the list of KeyVersion records
-   * for a delete request.
-   * All the entries will be files (i.e. no trailing /)
-   * @param paths path list
-   * @return a key list suitable for a delete request.
-   */
-  public static List<DeleteObjectsRequest.KeyVersion> keysToDelete(
-      List<Path> paths) {
-    return paths.stream()
-        .map(p -> {
-          String uripath = p.toUri().getPath();
-          return uripath.substring(1);
-        })
-        .map(DeleteObjectsRequest.KeyVersion::new)
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * From a list of keys, build up the list of keys for a delete request.
-   * If a key has a trailing /, that will be retained, so it will be
-   * considered a directory during multi-object delete failure handling
-   * @param keys key list
-   * @return a key list suitable for a delete request.
-   */
-  public static List<DeleteObjectsRequest.KeyVersion> toDeleteRequests(
-      List<String> keys) {
-    return keys.stream()
-        .map(DeleteObjectsRequest.KeyVersion::new)
-        .collect(Collectors.toList());
-  }
-
-  /**
-   * Verify that on a partial delete, the S3Guard tables are updated
-   * with deleted items. And only them.
-   */
-  @Test
-  public void testProcessDeleteFailure() throws Throwable {
-    String keyA = "/a/";
-    String keyAB = "/a/b";
-    String keyAC = "/a/c";
-    Path pathA = qualifyKey(keyA);
-    Path pathAB = qualifyKey(keyAB);
-    Path pathAC = qualifyKey(keyAC);
-    List<String> srcKeys = Lists.newArrayList(keyA, keyAB, keyAC);
-    List<Path> src = Lists.newArrayList(pathA, pathAB, pathAC);
-    List<DeleteObjectsRequest.KeyVersion> keyList = toDeleteRequests(srcKeys);
-    List<Path> deleteForbidden = Lists.newArrayList(pathAB);
-    final List<Path> deleteAllowed = Lists.newArrayList(pathA, pathAC);
-    List<MultiObjectDeleteSupport.KeyPath> forbiddenKP =
-        Lists.newArrayList(
-            new MultiObjectDeleteSupport.KeyPath(keyAB, pathAB, true));
-    MultiObjectDeleteException ex = createDeleteException(ACCESS_DENIED,
-        forbiddenKP);
-    OperationTrackingStore store
-        = new OperationTrackingStore();
-    StoreContext storeContext = S3ATestUtils
-            .createMockStoreContext(true, store, CONTEXT_ACCESSORS);
-    MultiObjectDeleteSupport deleteSupport
-        = new MultiObjectDeleteSupport(storeContext, null);
-    List<Path> retainedMarkers = new ArrayList<>();
-    Triple<List<Path>, List<Path>, List<Pair<Path, IOException>>>
-        triple = deleteSupport.processDeleteFailure(ex,
-        keyList,
-        retainedMarkers);
-    Assertions.assertThat(triple.getRight())
-        .as("failure list")
-        .isEmpty();
-    List<Path> undeleted = triple.getLeft();
-    List<Path> deleted = triple.getMiddle();
-    Assertions.assertThat(deleted).
-        as("deleted files")
-        .containsAll(deleteAllowed)
-        .doesNotContainAnyElementsOf(deleteForbidden);
-    Assertions.assertThat(undeleted).
-        as("undeleted store entries")
-        .containsAll(deleteForbidden)
-        .doesNotContainAnyElementsOf(deleteAllowed);
-    // because dir marker retention is on, we expect at least one retained
-    // marker
-    Assertions.assertThat(retainedMarkers).
-        as("Retained Markers")
-        .containsExactly(pathA);
-    Assertions.assertThat(store.getDeleted()).
-        as("List of tombstoned records")
-        .doesNotContain(pathA);
-  }
-
-
-  private static final class MinimalContextAccessor
-      implements ContextAccessors {
-
-    @Override
-    public Path keyToPath(final String key) {
-      return qualifyKey(key);
-    }
-
-    @Override
-    public String pathToKey(final Path path) {
-      return null;
-    }
-
-    @Override
-    public File createTempFile(final String prefix, final long size)
-        throws IOException {
-      throw new UnsupportedOperationException("unsppported");
-    }
-
-    @Override
-    public String getBucketLocation() throws IOException {
-      return null;
-    }
-
-    @Override
-    public Path makeQualified(final Path path) {
-      return path;
-    }
-
-    @Override
-    public AuditSpan getActiveAuditSpan() {
-      return AuditTestSupport.NOOP_SPAN;
-    }
-
-    @Override
-    public RequestFactory getRequestFactory() {
-      return MockS3AFileSystem.REQUEST_FACTORY;
-    }
-
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
index dbd89b960f96d..5c243bb820f02 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
@@ -90,7 +90,7 @@ public void testRequestFactoryWithCannedACL() throws Throwable {
     ObjectMetadata md = factory.newObjectMetadata(128);
     Assertions.assertThat(
             factory.newPutObjectRequest(path, md,
-                    new ByteArrayInputStream(new byte[0]))
+                    null, new ByteArrayInputStream(new byte[0]))
                 .getCannedAcl())
         .describedAs("ACL of PUT")
         .isEqualTo(acl);
@@ -98,7 +98,8 @@ public void testRequestFactoryWithCannedACL() throws Throwable {
             .getCannedAccessControlList())
         .describedAs("ACL of COPY")
         .isEqualTo(acl);
-    Assertions.assertThat(factory.newMultipartUploadRequest(path)
+    Assertions.assertThat(factory.newMultipartUploadRequest(path,
+                null)
             .getCannedACL())
         .describedAs("ACL of MPU")
         .isEqualTo(acl);
@@ -164,7 +165,7 @@ private void createFactoryObjects(RequestFactory factory) {
         new ArrayList<>()));
     a(factory.newCopyObjectRequest(path, path2, md));
     a(factory.newDeleteObjectRequest(path));
-    a(factory.newBulkDeleteRequest(new ArrayList<>(), true));
+    a(factory.newBulkDeleteRequest(new ArrayList<>()));
     a(factory.newDirectoryMarkerRequest(path));
     a(factory.newGetObjectRequest(path));
     a(factory.newGetObjectMetadataRequest(path));
@@ -172,12 +173,12 @@ private void createFactoryObjects(RequestFactory factory) {
     a(factory.newListObjectsV1Request(path, "/", 1));
     a(factory.newListNextBatchOfObjectsRequest(new ObjectListing()));
     a(factory.newListObjectsV2Request(path, "/", 1));
-    a(factory.newMultipartUploadRequest(path));
+    a(factory.newMultipartUploadRequest(path, null));
     File srcfile = new File("/tmp/a");
     a(factory.newPutObjectRequest(path,
-        factory.newObjectMetadata(-1), srcfile));
+        factory.newObjectMetadata(-1), null, srcfile));
     ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]);
-    a(factory.newPutObjectRequest(path, md, stream));
+    a(factory.newPutObjectRequest(path, md, null, stream));
     a(factory.newSelectRequest(path));
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java
new file mode 100644
index 0000000000000..33a44a9ad78f7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.impl;
+
+import java.io.IOException;
+
+import com.amazonaws.internal.SdkFilterInputStream;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit tests for stream draining.
+ */
+public class TestSDKStreamDrainer extends HadoopTestBase {
+
+  public static final int BYTES = 100;
+
+  /**
+   * Aborting does as asked.
+   */
+  @Test
+  public void testDrainerAborted() throws Throwable {
+    assertAborted(drainer(BYTES, true, stream()));
+  }
+
+  /**
+   * Create a stream of the default length.
+   * @return a stream.
+   */
+  private static FakeSDKInputStream stream() {
+    return new FakeSDKInputStream(BYTES);
+  }
+
+  /**
+   * a normal drain; all bytes are read. No abort.
+   */
+  @Test
+  public void testDrainerDrained() throws Throwable {
+    assertBytesReadNotAborted(
+        drainer(BYTES, false, stream()),
+        BYTES);
+  }
+
+  /**
+   * Empty streams are fine.
+   */
+  @Test
+  public void testEmptyStream() throws Throwable {
+    int size = 0;
+    assertBytesReadNotAborted(
+        drainer(size, false, new FakeSDKInputStream(size)),
+        size);
+  }
+
+  /**
+   * Single char read; just a safety check on the test stream more than
+   * the production code.
+   */
+  @Test
+  public void testSingleChar() throws Throwable {
+    int size = 1;
+    assertBytesReadNotAborted(
+        drainer(size, false, new FakeSDKInputStream(size)),
+        size);
+  }
+
+  /**
+   * a read spanning multiple buffers.
+   */
+  @Test
+  public void testMultipleBuffers() throws Throwable {
+    int size = DRAIN_BUFFER_SIZE + 1;
+    assertBytesReadNotAborted(
+        drainer(size, false, new FakeSDKInputStream(size)),
+        size);
+  }
+
+  /**
+   * Read of exactly one buffer.
+   */
+  @Test
+  public void testExactlyOneBuffer() throws Throwable {
+    int size = DRAIN_BUFFER_SIZE;
+    assertBytesReadNotAborted(
+        drainer(size, false, new FakeSDKInputStream(size)),
+        size);
+  }
+
+  /**
+   * Less data than expected came back. not escalated.
+   */
+  @Test
+  public void testStreamUnderflow() throws Throwable {
+    int size = 50;
+    assertBytesReadNotAborted(
+        drainer(BYTES, false, new FakeSDKInputStream(size)),
+        size);
+  }
+
+  /**
+   * Test a drain where a read triggers an IOE; this must escalate
+   * to an abort.
+   */
+  @Test
+  public void testReadFailure() throws Throwable {
+    int threshold = 50;
+    SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/",
+        null,
+        new FakeSDKInputStream(BYTES, threshold),
+        false,
+        BYTES,
+        EMPTY_INPUT_STREAM_STATISTICS, "test");
+    intercept(IOException.class, "", () ->
+        drainer.applyRaisingException());
+
+    assertAborted(drainer);
+  }
+
+  /**
+   * abort does not read(), so the exception will not surface.
+   */
+  @Test
+  public void testReadFailureDoesNotSurfaceInAbort() throws Throwable {
+    int threshold = 50;
+    SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/",
+        null,
+        new FakeSDKInputStream(BYTES, threshold),
+        true,
+        BYTES,
+        EMPTY_INPUT_STREAM_STATISTICS, "test");
+    drainer.applyRaisingException();
+
+    assertAborted(drainer);
+  }
+
+  /**
+   * make sure the underlying stream read code works.
+   */
+  @Test
+  public void testFakeStreamRead() throws Throwable {
+    FakeSDKInputStream stream = stream();
+    int count = 0;
+    while (stream.read() > 0) {
+      count++;
+    }
+    Assertions.assertThat(count)
+        .describedAs("bytes read from %s", stream)
+        .isEqualTo(BYTES);
+  }
+
+  /**
+   * Create a drainer and invoke it, rethrowing any exception
+   * which occurred during the draining.
+   * @param remaining bytes remaining in the stream
+   * @param shouldAbort should we abort?
+   * @param in input stream.
+   * @return the drainer
+   * @throws Throwable something went wrong
+   */
+  private SDKStreamDrainer drainer(int remaining,
+      boolean shouldAbort,
+      FakeSDKInputStream in) throws Throwable {
+    SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/",
+        null,
+        in,
+        shouldAbort,
+        remaining,
+        EMPTY_INPUT_STREAM_STATISTICS, "test");
+    drainer.applyRaisingException();
+    return drainer;
+  }
+
+
+  /**
+   * The draining aborted.
+   * @param drainer drainer to assert on.
+   * @return the drainer.
+   */
+  private SDKStreamDrainer assertAborted(SDKStreamDrainer drainer) {
+    Assertions.assertThat(drainer)
+        .matches(SDKStreamDrainer::aborted, "aborted");
+    return drainer;
+  }
+
+  /**
+   * The draining was not aborted.
+   * @param drainer drainer to assert on.
+   * @return the drainer.
+   */
+  private SDKStreamDrainer assertNotAborted(SDKStreamDrainer drainer) {
+    Assertions.assertThat(drainer)
+        .matches(d -> !d.aborted(), "is not aborted");
+    return drainer;
+  }
+
+  /**
+   * The draining was not aborted and {@code bytes} were read.
+   * @param drainer drainer to assert on.
+   * @param bytes expected byte count
+   * @return the drainer.
+   */
+  private SDKStreamDrainer assertBytesReadNotAborted(SDKStreamDrainer drainer,
+      int bytes) {
+    return assertBytesRead(assertNotAborted(drainer), bytes);
+  }
+
+  /**
+   * Assert {@code bytes} were read.
+   * @param drainer drainer to assert on.
+   * @param bytes expected byte count
+   * @return the drainer.
+   */
+  private static SDKStreamDrainer assertBytesRead(final SDKStreamDrainer drainer,
+      final int bytes) {
+    Assertions.assertThat(drainer)
+        .describedAs("bytes read by %s", drainer)
+        .extracting(SDKStreamDrainer::getDrained)
+        .isEqualTo(bytes);
+    return drainer;
+  }
+
+
+  /**
+   * Fake stream; generates data dynamically.
+   * Only overrides the methods used in stream draining.
+   */
+  private static final class FakeSDKInputStream extends SdkFilterInputStream {
+
+    private final int capacity;
+
+    private final int readToRaiseIOE;
+
+    private int bytesRead;
+
+    private boolean closed;
+
+    private boolean aborted;
+
+    /**
+     * read up to the capacity; optionally trigger an IOE.
+     * @param capacity total capacity.
+     * @param readToRaiseIOE position to raise an IOE, or -1
+     */
+    private FakeSDKInputStream(final int capacity, final int readToRaiseIOE) {
+      super(null);
+      this.capacity = capacity;
+      this.readToRaiseIOE = readToRaiseIOE;
+    }
+
+    /**
+     * read up to the capacity.
+     * @param capacity total capacity.
+     */
+    private FakeSDKInputStream(final int capacity) {
+      this(capacity, -1);
+    }
+
+    @Override
+    public void abort() {
+      aborted = true;
+    }
+
+    @Override
+    protected boolean isAborted() {
+      return aborted;
+    }
+
+    @Override
+    public int read() throws IOException {
+      if (bytesRead >= capacity) {
+        // EOF
+        return -1;
+      }
+      bytesRead++;
+      if (readToRaiseIOE > 0 && bytesRead >= readToRaiseIOE) {
+        throw new IOException("IOE triggered on reading byte " + bytesRead);
+      }
+      return (int) '0' + (bytesRead % 10);
+    }
+
+    @Override
+    public int read(final byte[] bytes, final int off, final int len)
+        throws IOException {
+      int count = 0;
+
+      try {
+        while (count < len) {
+          int r = read();
+          if (r < 0) {
+            break;
+          }
+          bytes[off + count] = (byte) r;
+          count++;
+        }
+      } catch (IOException e) {
+        if (count == 0) {
+          // first byte
+          throw e;
+        }
+        // otherwise break loop
+      }
+      return count;
+    }
+
+    @Override
+    public void close() throws IOException {
+      closed = true;
+    }
+
+    @Override
+    public String toString() {
+      return "FakeSDKInputStream{" +
+          "capacity=" + capacity +
+          ", readToRaiseIOE=" + readToRaiseIOE +
+          ", bytesRead=" + bytesRead +
+          ", closed=" + closed +
+          ", aborted=" + aborted +
+          "} " + super.toString();
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
index 45639dae7b7b3..8bbf52b578e1a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
@@ -59,30 +59,11 @@
  */
 public class AbstractS3ACostTest extends AbstractS3ATestBase {
 
-  /**
-   * Parameter: should the stores be guarded?
-   */
-  private final boolean s3guard;
-
   /**
    * Parameter: should directory markers be retained?
    */
   private final boolean keepMarkers;
 
-  /**
-   * Is this an auth mode test run?
-   */
-  private final boolean authoritative;
-
-  /** probe states calculated from the configuration options. */
-  private boolean isGuarded;
-
-  private boolean isRaw;
-
-  private boolean isAuthoritative;
-
-  private boolean isNonAuth;
-
   private boolean isKeeping;
 
   private boolean isDeleting;
@@ -101,59 +82,31 @@ public class AbstractS3ACostTest extends AbstractS3ATestBase {
    */
   private Statistic deleteMarkerStatistic;
 
-  public AbstractS3ACostTest(
-      final boolean s3guard,
-      final boolean keepMarkers,
-      final boolean authoritative) {
-    this.s3guard = s3guard;
-    this.keepMarkers = keepMarkers;
-    this.authoritative = authoritative;
-  }
 
   /**
-   * Constructor for tests which don't include
-   * any for S3Guard.
+   * Constructor for parameterized tests.
    * @param keepMarkers should markers be tested.
    */
-  public AbstractS3ACostTest(
+  protected AbstractS3ACostTest(
       final boolean keepMarkers) {
-    this.s3guard = false;
     this.keepMarkers = keepMarkers;
-    this.authoritative = false;
   }
 
   @Override
   public Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     String bucketName = getTestBucketName(conf);
-    // If AccessPoint ARN is set guarded tests are skipped
     String arnKey = String.format(InternalConstants.ARN_BUCKET_OPTION, bucketName);
     String arn = conf.getTrimmed(arnKey, "");
-    if (isGuarded() && !arn.isEmpty()) {
-      ContractTestUtils.skip(
-          "Skipping test since AccessPoint ARN is set and is incompatible with S3Guard.");
-    }
 
-    removeBucketOverrides(bucketName, conf,
-        S3_METADATA_STORE_IMPL);
-    if (!isGuarded()) {
-      // in a raw run remove all s3guard settings
-      removeBaseAndBucketOverrides(bucketName, conf,
-          S3_METADATA_STORE_IMPL);
-    }
     removeBaseAndBucketOverrides(bucketName, conf,
         DIRECTORY_MARKER_POLICY,
-        METADATASTORE_AUTHORITATIVE,
         AUTHORITATIVE_PATH);
     // directory marker options
     conf.set(DIRECTORY_MARKER_POLICY,
         keepMarkers
             ? DIRECTORY_MARKER_POLICY_KEEP
             : DIRECTORY_MARKER_POLICY_DELETE);
-    if (isGuarded()) {
-      conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_DYNAMO);
-      conf.setBoolean(METADATASTORE_AUTHORITATIVE, authoritative);
-    }
     disableFilesystemCaching(conf);
 
     // AccessPoint ARN is the only per bucket configuration that must be kept.
@@ -167,21 +120,7 @@ public Configuration createConfiguration() {
   @Override
   public void setup() throws Exception {
     super.setup();
-    if (isGuarded()) {
-      // s3guard is required for those test runs where any of the
-      // guard options are set
-      assumeS3GuardState(true, getConfiguration());
-    }
     S3AFileSystem fs = getFileSystem();
-    skipDuringFaultInjection(fs);
-
-    // build up the states
-    isGuarded = isGuarded();
-
-    isRaw = !isGuarded;
-    isAuthoritative = isGuarded && authoritative;
-    isNonAuth = isGuarded && !authoritative;
-
     isKeeping = isKeepingMarkers();
 
     isDeleting = !isKeeping;
@@ -193,16 +132,7 @@ public void setup() throws Exception {
         .isEqualTo(isKeepingMarkers()
             ? DirectoryPolicy.MarkerPolicy.Keep
             : DirectoryPolicy.MarkerPolicy.Delete);
-    // All counter statistics of the filesystem are added as metrics.
-    // Durations too, as they have counters of success and failure.
-    OperationCostValidator.Builder builder = OperationCostValidator.builder(
-        getFileSystem());
-    EnumSet.allOf(Statistic.class).stream()
-        .filter(s ->
-            s.getType() == StatisticTypeEnum.TYPE_COUNTER
-                || s.getType() == StatisticTypeEnum.TYPE_DURATION)
-        .forEach(s -> builder.withMetric(s));
-    costValidator = builder.build();
+    setupCostValidator();
 
     // determine bulk delete settings
     final Configuration fsConf = getFileSystem().getConf();
@@ -215,40 +145,17 @@ public void setup() throws Exception {
     setSpanSource(fs);
   }
 
-  public void assumeUnguarded() {
-    assume("Unguarded FS only", !isGuarded());
-  }
-
-  /**
-   * Is the store guarded authoritatively on the test path?
-   * @return true if the condition is met on this test run.
-   */
-  public boolean isAuthoritative() {
-    return authoritative;
-  }
-
-  /**
-   * Is the store guarded?
-   * @return true if the condition is met on this test run.
-   */
-  public boolean isGuarded() {
-    return s3guard;
-  }
-
-  /**
-   * Is the store raw?
-   * @return true if the condition is met on this test run.
-   */
-  public boolean isRaw() {
-    return isRaw;
-  }
-
-  /**
-   * Is the store guarded non-authoritatively on the test path?
-   * @return true if the condition is met on this test run.
-   */
-  public boolean isNonAuth() {
-    return isNonAuth;
+  protected void setupCostValidator() {
+    // All counter statistics of the filesystem are added as metrics.
+    // Durations too, as they have counters of success and failure.
+    OperationCostValidator.Builder builder = OperationCostValidator.builder(
+        getFileSystem());
+    EnumSet.allOf(Statistic.class).stream()
+        .filter(s ->
+            s.getType() == StatisticTypeEnum.TYPE_COUNTER
+                || s.getType() == StatisticTypeEnum.TYPE_DURATION)
+        .forEach(s -> builder.withMetric(s));
+    costValidator = builder.build();
   }
 
   public boolean isDeleting() {
@@ -280,7 +187,7 @@ protected Path buildFile(Path path,
       boolean recursive,
       OperationCost cost) throws Exception {
     resetStatistics();
-    verifyRaw(cost, () -> {
+    verify(cost, () -> {
       FSDataOutputStreamBuilder builder = getFileSystem().createFile(path)
           .overwrite(overwrite);
       if (recursive) {
@@ -343,7 +250,7 @@ protected Path create(Path path) throws Exception {
    */
   protected Path create(Path path, boolean overwrite,
       OperationCost cost) throws Exception {
-    return verifyRaw(cost, () ->
+    return verify(cost, () ->
         file(path, overwrite));
   }
 
@@ -425,12 +332,12 @@ protected <T, E extends Throwable> E verifyMetricsIntercepting(
    * @return the exception caught.
    * @throws Exception any other exception
    */
-  protected <T, E extends Throwable> E interceptRaw(
+  protected <T, E extends Throwable> E interceptOperation(
       Class<E> clazz,
       String text,
       OperationCost cost,
       Callable<T> eval) throws Exception {
-    return verifyMetricsIntercepting(clazz, text, eval, whenRaw(cost));
+    return verifyMetricsIntercepting(clazz, text, eval, always(cost));
   }
 
   /**
@@ -443,48 +350,6 @@ protected OperationCostValidator.ExpectedProbe always(
     return expect(true, cost);
   }
 
-  /**
-   * Declare the expected cost on a raw FS.
-   * @param cost costs to expect
-   * @return a probe.
-   */
-  protected OperationCostValidator.ExpectedProbe whenRaw(
-      OperationCost cost) {
-    return expect(isRaw(), cost);
-  }
-
-  /**
-   * Declare the expected cost on a guarded FS.
-   * @param cost costs to expect
-   * @return a probe.
-   */
-  protected OperationCostValidator.ExpectedProbe whenGuarded(
-      OperationCost cost) {
-    return expect(isGuarded(), cost);
-  }
-
-  /**
-   * Declare the expected cost on a guarded auth FS.
-   * @param cost costs to expect
-   * @return a probe.
-   */
-  protected OperationCostValidator.ExpectedProbe whenAuthoritative(
-      OperationCost cost) {
-    return expect(isAuthoritative(), cost);
-  }
-
-
-  /**
-   * Declare the expected cost on a guarded nonauth FS.
-   * @param cost costs to expect
-   * @return a probe.
-   */
-  protected OperationCostValidator.ExpectedProbe whenNonauth(
-      OperationCost cost) {
-    return expect(isNonAuth(), cost);
-  }
-
-
   /**
    * A metric diff which must hold when the fs is keeping markers.
    * @param cost expected cost
@@ -506,8 +371,8 @@ protected OperationCostValidator.ExpectedProbe whenDeleting(
   }
 
   /**
-   * Execute a closure expecting a specific number of HEAD/LIST calls
-   * on <i>raw</i> S3 stores only. The operation is always evaluated.
+   * Execute a closure expecting a specific number of HEAD/LIST calls.
+   * The operation is always evaluated.
    * A span is always created prior to the invocation; saves trouble
    * in tests that way.
    * @param cost expected cost
@@ -515,29 +380,29 @@ protected OperationCostValidator.ExpectedProbe whenDeleting(
    * @param <T> return type of closure
    * @return the result of the evaluation
    */
-  protected <T> T verifyRaw(
+  protected <T> T verify(
       OperationCost cost,
       Callable<T> eval) throws Exception {
     return verifyMetrics(eval,
-        whenRaw(cost), OperationCostValidator.always());
+        always(cost), OperationCostValidator.always());
   }
 
   /**
    * Execute {@code S3AFileSystem#innerGetFileStatus(Path, boolean, Set)}
    * for the given probes.
-   * expect the specific HEAD/LIST count with a raw FS.
+   * expect the specific HEAD/LIST count.
    * @param path path
    * @param needEmptyDirectoryFlag look for empty directory
    * @param probes file status probes to perform
    * @param cost expected cost
    * @return the status
    */
-  public S3AFileStatus verifyRawInnerGetFileStatus(
+  public S3AFileStatus verifyInnerGetFileStatus(
       Path path,
       boolean needEmptyDirectoryFlag,
       Set<StatusProbeEnum> probes,
       OperationCost cost) throws Exception {
-    return verifyRaw(cost, () ->
+    return verify(cost, () ->
         innerGetFileStatus(getFileSystem(),
             path,
             needEmptyDirectoryFlag,
@@ -547,20 +412,20 @@ public S3AFileStatus verifyRawInnerGetFileStatus(
   /**
    * Execute {@code S3AFileSystem#innerGetFileStatus(Path, boolean, Set)}
    * for the given probes -expect a FileNotFoundException,
-   * and the specific HEAD/LIST count with a raw FS.
+   * and the specific HEAD/LIST count.
    * @param path path
    * @param needEmptyDirectoryFlag look for empty directory
    * @param probes file status probes to perform
    * @param cost expected cost
    */
 
-  public void interceptRawGetFileStatusFNFE(
+  public void interceptGetFileStatusFNFE(
       Path path,
       boolean needEmptyDirectoryFlag,
       Set<StatusProbeEnum> probes,
       OperationCost cost) throws Exception {
     try (AuditSpan span = span()) {
-      interceptRaw(FileNotFoundException.class, "",
+      interceptOperation(FileNotFoundException.class, "",
           cost, () ->
               innerGetFileStatus(getFileSystem(),
                   path,
@@ -571,15 +436,14 @@ public void interceptRawGetFileStatusFNFE(
 
   /**
    * Probe for a path being a directory.
-   * Metrics are only checked on unguarded stores.
    * @param path path
    * @param expected expected outcome
-   * @param cost expected cost on a Raw FS.
+   * @param cost expected cost
    */
   protected void isDir(Path path,
       boolean expected,
       OperationCost cost) throws Exception {
-    boolean b = verifyRaw(cost, () ->
+    boolean b = verify(cost, () ->
         getFileSystem().isDirectory(path));
     Assertions.assertThat(b)
         .describedAs("isDirectory(%s)", path)
@@ -588,15 +452,14 @@ protected void isDir(Path path,
 
   /**
    * Probe for a path being a file.
-   * Metrics are only checked on unguarded stores.
    * @param path path
    * @param expected expected outcome
-   * @param cost expected cost on a Raw FS.
+   * @param cost expected cost
    */
   protected void isFile(Path path,
       boolean expected,
       OperationCost cost) throws Exception {
-    boolean b = verifyRaw(cost, () ->
+    boolean b = verify(cost, () ->
         getFileSystem().isFile(path));
     Assertions.assertThat(b)
         .describedAs("isFile(%s)", path)
@@ -614,53 +477,6 @@ protected OperationCostValidator.ExpectedProbe with(
     return probe(stat, expected);
   }
 
-  /**
-   * A metric diff which must hold when the fs is unguarded.
-   * @param stat metric source
-   * @param expected expected value.
-   * @return the diff.
-   */
-  protected OperationCostValidator.ExpectedProbe withWhenRaw(
-      final Statistic stat, final int expected) {
-    return probe(isRaw(), stat, expected);
-  }
-
-  /**
-   * A metric diff which must hold when the fs is guarded.
-   * @param stat metric source
-   * @param expected expected value.
-   * @return the diff.
-   */
-  protected OperationCostValidator.ExpectedProbe withWhenGuarded(
-      final Statistic stat,
-      final int expected) {
-    return probe(isGuarded(), stat, expected);
-  }
-
-  /**
-   * A metric diff which must hold when the fs is guarded + authoritative.
-   * @param stat metric source
-   * @param expected expected value.
-   * @return the diff.
-   */
-  protected OperationCostValidator.ExpectedProbe withWhenAuthoritative(
-      final Statistic stat,
-      final int expected) {
-    return probe(isAuthoritative(), stat, expected);
-  }
-
-  /**
-   * A metric diff which must hold when the fs is guarded + authoritative.
-   * @param stat metric source
-   * @param expected expected value.
-   * @return the diff.
-   */
-  protected OperationCostValidator.ExpectedProbe withWhenNonauth(
-      final Statistic stat,
-      final int expected) {
-    return probe(isNonAuth(), stat, expected);
-  }
-
   /**
    * A metric diff which must hold when the fs is keeping markers.
    * @param stat metric source
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
new file mode 100644
index 0000000000000..39530d97bf794
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.performance;
+
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
+import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
+import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_OVERWRITE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_DIR_PROBE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_FILE_PROBE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_DIR_MARKER;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_FILE;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.HEAD_OPERATION;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST;
+
+/**
+ * Assert cost of createFile operations, especially
+ * with the FS_S3A_CREATE_PERFORMANCE option.
+ */
+@SuppressWarnings("resource")
+public class ITestCreateFileCost extends AbstractS3ACostTest {
+
+  /**
+   * Create with markers kept, always.
+   */
+  public ITestCreateFileCost() {
+    super(false);
+  }
+
+  @Test
+  public void testCreateNoOverwrite() throws Throwable {
+    describe("Test file creation without overwrite");
+    Path testFile = methodPath();
+    // when overwrite is false, the path is checked for existence.
+    create(testFile, false,
+        CREATE_FILE_NO_OVERWRITE);
+  }
+
+  @Test
+  public void testCreateOverwrite() throws Throwable {
+    describe("Test file creation with overwrite");
+    Path testFile = methodPath();
+    // when overwrite is true: only the directory checks take place.
+    create(testFile, true, CREATE_FILE_OVERWRITE);
+  }
+
+  @Test
+  public void testCreateNoOverwriteFileExists() throws Throwable {
+    describe("Test cost of create file failing with existing file");
+    Path testFile = file(methodPath());
+
+    // now there is a file there, an attempt with overwrite == false will
+    // fail on the first HEAD.
+    interceptOperation(FileAlreadyExistsException.class, "",
+        FILE_STATUS_FILE_PROBE,
+        () -> file(testFile, false));
+  }
+
+  @Test
+  public void testCreateFileOverDir() throws Throwable {
+    describe("Test cost of create file failing with existing dir");
+    Path testFile = dir(methodPath());
+
+    // now there is a file there, an attempt with overwrite == false will
+    // fail on the first HEAD.
+    interceptOperation(FileAlreadyExistsException.class, "",
+        GET_FILE_STATUS_ON_DIR_MARKER,
+        () -> file(testFile, false));
+  }
+
+  /**
+   * Use the builder API.
+   * on s3a this skips parent checks, always.
+   */
+  @Test
+  public void testCreateBuilderSequence() throws Throwable {
+    describe("Test builder file creation cost");
+    Path testFile = methodPath();
+    dir(testFile.getParent());
+
+    // s3a fs skips the recursive checks to avoid race
+    // conditions with other processes/threads deleting
+    // files and so briefly the path not being present
+    // only make sure the dest path isn't a directory.
+    buildFile(testFile, true, false,
+        FILE_STATUS_DIR_PROBE);
+
+    // now there is a file there, an attempt with overwrite == false will
+    // fail on the first HEAD.
+    interceptOperation(FileAlreadyExistsException.class, "",
+        GET_FILE_STATUS_ON_FILE,
+        () -> buildFile(testFile, false, true,
+            GET_FILE_STATUS_ON_FILE));
+  }
+
+  @Test
+  public void testCreateFilePerformanceFlag() throws Throwable {
+    describe("createFile with performance flag skips safety checks");
+    S3AFileSystem fs = getFileSystem();
+
+    Path path = methodPath();
+    FSDataOutputStreamBuilder builder = fs.createFile(path)
+        .overwrite(false)
+        .recursive();
+
+    // this has a broken return type; something to do with the return value of
+    // the createFile() call. only fixable via risky changes to the FileSystem class
+    builder.must(FS_S3A_CREATE_PERFORMANCE, true);
+
+    verifyMetrics(() -> build(builder),
+        always(NO_HEAD_OR_LIST),
+        with(OBJECT_BULK_DELETE_REQUEST, 0),
+        with(OBJECT_DELETE_REQUEST, 0));
+  }
+
+  @Test
+  public void testCreateFileRecursive() throws Throwable {
+    describe("createFile without performance flag performs overwrite safety checks");
+    S3AFileSystem fs = getFileSystem();
+
+    final Path path = methodPath();
+    FSDataOutputStreamBuilder builder = fs.createFile(path)
+        .recursive()
+        .overwrite(false);
+
+    // include a custom header to probe for after
+    final String custom = "custom";
+    builder.must(FS_S3A_CREATE_HEADER + ".h1", custom);
+
+    verifyMetrics(() -> build(builder),
+        always(CREATE_FILE_NO_OVERWRITE));
+
+    // the header is there and the probe should be a single HEAD call.
+    String header = verifyMetrics(() ->
+            toChar(requireNonNull(
+              fs.getXAttr(path, XA_HEADER_PREFIX + "h1"),
+              "no header")),
+        always(HEAD_OPERATION));
+    Assertions.assertThat(header)
+        .isEqualTo(custom);
+  }
+
+  @Test
+  public void testCreateFileNonRecursive() throws Throwable {
+    describe("nonrecursive createFile does not check parents");
+    S3AFileSystem fs = getFileSystem();
+
+    verifyMetrics(() ->
+            build(fs.createFile(methodPath()).overwrite(true)),
+        always(CREATE_FILE_OVERWRITE));
+  }
+
+
+  @Test
+  public void testCreateNonRecursive() throws Throwable {
+    describe("nonrecursive createFile does not check parents");
+    S3AFileSystem fs = getFileSystem();
+
+    verifyMetrics(() -> {
+      fs.createNonRecursive(methodPath(),
+              true, 1000, (short)1, 0L, null)
+          .close();
+      return "";
+    },
+        always(CREATE_FILE_OVERWRITE));
+  }
+
+  private FSDataOutputStream build(final FSDataOutputStreamBuilder builder)
+      throws IOException {
+    FSDataOutputStream out = builder.build();
+    out.close();
+    return out;
+  }
+
+  /**
+   * Shows how the performance option allows the FS to become ill-formed.
+   */
+  @Test
+  public void testPerformanceFlagPermitsInvalidStores() throws Throwable {
+    describe("createFile with performance flag over a directory");
+    S3AFileSystem fs = getFileSystem();
+
+    Path path = methodPath();
+    Path child = new Path(path, "child");
+    ContractTestUtils.touch(fs, child);
+    try {
+      FSDataOutputStreamBuilder builder = fs.createFile(path)
+          .overwrite(false);
+      // this has a broken return type; a java typesystem quirk.
+      builder.must(FS_S3A_CREATE_PERFORMANCE, true);
+
+      verifyMetrics(() -> build(builder),
+          always(NO_HEAD_OR_LIST),
+          with(OBJECT_BULK_DELETE_REQUEST, 0),
+          with(OBJECT_DELETE_REQUEST, 0));
+      // the file is there
+      assertIsFile(path);
+      // the child is there
+      assertIsFile(child);
+
+      // delete the path
+      fs.delete(path, true);
+      // the child is still there
+      assertIsFile(child);
+      // and the directory exists again
+      assertIsDirectory(path);
+    } finally {
+      // always delete the child, so if the test suite fails, the
+      // store is at least well-formed.
+      fs.delete(child, true);
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
index a08d77367b18f..ccb0c0e79ec7e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestDirectoryMarkerListing.java
@@ -49,13 +49,9 @@
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -199,12 +195,6 @@ protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     String bucketName = getTestBucketName(conf);
 
-    // Turn off S3Guard
-    removeBaseAndBucketOverrides(bucketName, conf,
-        S3_METADATA_STORE_IMPL,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-
     // directory marker options
     removeBaseAndBucketOverrides(bucketName, conf,
         DIRECTORY_MARKER_POLICY);
@@ -222,8 +212,6 @@ protected Configuration createConfiguration() {
   public void setup() throws Exception {
     super.setup();
     S3AFileSystem fs = getFileSystem();
-    assume("unguarded FS only",
-        !fs.hasMetadataStore());
     s3client = fs.getAmazonS3ClientForTesting("markers");
     bucket = fs.getBucket();
     Path base = new Path(methodPath(), "base");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
index 2901767128fce..be4de7942f7b9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
@@ -46,7 +46,7 @@
 /**
  * Use metrics to assert about the cost of file API calls.
  * <p></p>
- * Parameterized on guarded vs raw. and directory marker keep vs delete.
+ * Parameterized on directory marker keep vs delete.
  * There's extra complexity related to bulk/non-bulk delete calls.
  * If bulk deletes are disabled, many more requests are made to delete
  * parent directories. The counters of objects deleted are constant
@@ -64,20 +64,14 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {"raw-keep-markers", false, true, false},
-        {"raw-delete-markers", false, false, false},
-        {"nonauth-keep-markers", true, true, false},
-        {"nonauth-delete-markers", true, false, false},
-        {"auth-delete-markers", true, false, true},
-        {"auth-keep-markers", true, true, true}
+        {"keep-markers", true},
+        {"delete-markers", false},
     });
   }
 
   public ITestS3ADeleteCost(final String name,
-      final boolean s3guard,
-      final boolean keepMarkers,
-      final boolean authoritative) {
-    super(s3guard, keepMarkers, authoritative);
+      final boolean keepMarkers) {
+    super(keepMarkers);
   }
 
   @Override
@@ -105,8 +99,8 @@ public void testDeleteSingleFileInDir() throws Throwable {
     // still be there
     Path simpleFile = file(new Path(dir, "simple.txt"));
 
-    boolean rawAndKeeping = isRaw() && isDeleting();
-    boolean rawAndDeleting = isRaw() && isDeleting();
+    boolean rawAndKeeping = !isDeleting();
+    boolean rawAndDeleting = isDeleting();
     verifyMetrics(() -> {
           fs.delete(simpleFile, false);
           return "after fs.delete(simpleFile) " + getMetricSummary();
@@ -116,7 +110,7 @@ public void testDeleteSingleFileInDir() throws Throwable {
         // if deleting markers, look for the parent too
         probe(rawAndDeleting, OBJECT_METADATA_REQUESTS,
             FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H),
-        withWhenRaw(OBJECT_LIST_REQUEST,
+        with(OBJECT_LIST_REQUEST,
             FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L),
         with(DIRECTORIES_DELETED, 0),
         with(FILES_DELETED, 1),
@@ -137,7 +131,7 @@ public void testDeleteSingleFileInDir() throws Throwable {
     );
 
     // there is an empty dir for a parent
-    S3AFileStatus status = verifyRawInnerGetFileStatus(dir, true,
+    S3AFileStatus status = verifyInnerGetFileStatus(dir, true,
         StatusProbeEnum.ALL, GET_FILE_STATUS_ON_DIR);
     assertEmptyDirStatus(status, Tristate.TRUE);
   }
@@ -157,8 +151,8 @@ public void testDeleteFileInDir() throws Throwable {
     Path file1 = file(new Path(dir, "file1.txt"));
     Path file2 = file(new Path(dir, "file2.txt"));
 
-    boolean rawAndKeeping = isRaw() && isDeleting();
-    boolean rawAndDeleting = isRaw() && isDeleting();
+    boolean rawAndKeeping = !isDeleting();
+    boolean rawAndDeleting = isDeleting();
     verifyMetrics(() -> {
       fs.delete(file1, false);
       return "after fs.delete(file1) " + getMetricSummary();
@@ -169,7 +163,7 @@ public void testDeleteFileInDir() throws Throwable {
         // if deleting markers, look for the parent too
         probe(rawAndDeleting, OBJECT_METADATA_REQUESTS,
             FILESTATUS_FILE_PROBE_H + FILESTATUS_DIR_PROBE_H),
-        withWhenRaw(OBJECT_LIST_REQUEST,
+        with(OBJECT_LIST_REQUEST,
             FILESTATUS_FILE_PROBE_L + FILESTATUS_DIR_PROBE_L),
         with(DIRECTORIES_DELETED, 0),
         with(FILES_DELETED, 1),
@@ -272,9 +266,10 @@ public void testDirMarkersFileCreation() throws Throwable {
 
     final int directories = directoriesInPath(srcDir);
     verifyMetrics(() -> {
-      file(new Path(srcDir, "source.txt"));
+      final Path srcPath = new Path(srcDir, "source.txt");
+      file(srcPath);
       LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem());
-      return "after touch(fs, srcFilePath) " + getMetricSummary();
+      return "after touch(fs, " + srcPath + ")" + getMetricSummary();
     },
         with(DIRECTORIES_CREATED, 0),
         with(DIRECTORIES_DELETED, 0),
@@ -282,10 +277,10 @@ public void testDirMarkersFileCreation() throws Throwable {
         withWhenKeeping(getDeleteMarkerStatistic(), 0),
         withWhenKeeping(FAKE_DIRECTORIES_DELETED, 0),
         // delete all possible fake dirs above the file
-        withWhenDeleting(getDeleteMarkerStatistic(),
-            isBulkDelete() ? 1: directories),
         withWhenDeleting(FAKE_DIRECTORIES_DELETED,
-            directories));
+            directories),
+        withWhenDeleting(getDeleteMarkerStatistic(),
+            isBulkDelete() ? 1: directories));
   }
 
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java
index 6449d2a5e44c9..6a6baa03ead41 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java
@@ -30,18 +30,22 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.Statistic.AUDIT_SPAN_CREATION;
 import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_GET_CONTENT_SUMMARY;
 import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_LIST_REQUEST;
 import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS;
+import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.AUDIT_ENABLED;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILESTATUS_DIR_PROBE_L;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_FILE_PROBE;
 import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_OPERATION;
+import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
 
 /**
  * Use metrics to assert about the cost of misc operations.
@@ -53,20 +57,48 @@ public class ITestS3AMiscOperationCost extends AbstractS3ACostTest {
   private static final Logger LOG =
       LoggerFactory.getLogger(ITestS3AMiscOperationCost.class);
 
+  /**
+   * Parameter: should auditing be enabled?
+   */
+  private final boolean auditing;
+
   /**
    * Parameterization.
    */
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {"keep-markers", true},
-        {"delete-markers", false}
+        {"keep-markers-auditing", true, true},
+        {"delete-markers-unaudited", false, false}
     });
   }
 
   public ITestS3AMiscOperationCost(final String name,
-      final boolean keepMarkers) {
-    super(false, keepMarkers, false);
+      final boolean keepMarkers,
+      final boolean auditing) {
+    super(keepMarkers);
+    this.auditing = auditing;
+  }
+
+  @Override
+  public Configuration createConfiguration() {
+    final Configuration conf = super.createConfiguration();
+    removeBaseAndBucketOverrides(conf, AUDIT_ENABLED);
+    conf.setBoolean(AUDIT_ENABLED, auditing);
+    return conf;
+  }
+
+  /**
+   * Expected audit count when auditing is enabled; expect 0
+   * when disabled.
+   * @param expected expected value.
+   * @return the probe.
+   */
+  protected OperationCostValidator.ExpectedProbe withAuditCount(
+      final int expected) {
+    return probe(AUDIT_SPAN_CREATION,
+        auditing ? expected : 0);
+
   }
 
   /**
@@ -81,7 +113,7 @@ public void testMkdirOverDir() throws Throwable {
 
     // create the child; only assert on HEAD/GET IO
     verifyMetrics(() -> fs.mkdirs(baseDir),
-        with(AUDIT_SPAN_CREATION, 1),
+        withAuditCount(1),
         // full probe on dest plus list only on parent.
         with(OBJECT_METADATA_REQUESTS, 0),
         with(OBJECT_LIST_REQUEST, FILESTATUS_DIR_PROBE_L));
@@ -105,13 +137,15 @@ public void testGetContentSummaryDir() throws Throwable {
     Path childDir = new Path(baseDir, "subdir/child");
     touch(fs, childDir);
 
+    // look at path to see if it is a file
+    // it is not: so LIST
     final ContentSummary summary = verifyMetrics(
         () -> getContentSummary(baseDir),
         with(INVOCATION_GET_CONTENT_SUMMARY, 1),
-        with(AUDIT_SPAN_CREATION, 1),
-        whenRaw(FILE_STATUS_FILE_PROBE    // look at path to see if it is a file
-            .plus(LIST_OPERATION)         // it is not: so LIST
-            .plus(LIST_OPERATION)));       // and a LIST on the child dir
+        withAuditCount(1),
+        always(FILE_STATUS_FILE_PROBE    // look at path to see if it is a file
+            .plus(LIST_OPERATION)));         // it is not: so LIST
+
     Assertions.assertThat(summary.getDirectoryCount())
         .as("Summary " + summary)
         .isEqualTo(2);
@@ -127,8 +161,8 @@ public void testGetContentMissingPath() throws Throwable {
     verifyMetricsIntercepting(FileNotFoundException.class,
         "", () -> getContentSummary(baseDir),
         with(INVOCATION_GET_CONTENT_SUMMARY, 1),
-        with(AUDIT_SPAN_CREATION, 1),
-        whenRaw(FILE_STATUS_FILE_PROBE
+        withAuditCount(1),
+        always(FILE_STATUS_FILE_PROBE
             .plus(FILE_STATUS_FILE_PROBE)
             .plus(LIST_OPERATION)
             .plus(LIST_OPERATION)));
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java
index 639e1dddbae74..395eac2987b3c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java
@@ -61,7 +61,7 @@ public static Collection<Object[]> params() {
 
   public ITestS3AMkdirCost(final String name,
       final boolean keepMarkers) {
-    super(false, true, false);
+    super(keepMarkers);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AOpenCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AOpenCost.java
new file mode 100644
index 0000000000000..b0ee531112b50
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AOpenCost.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.performance;
+
+
+import java.io.EOFException;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.readStream;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile;
+import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_BYTES_READ_CLOSE;
+import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_OPENED;
+import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_SEEK_BYTES_SKIPPED;
+import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertDurationRange;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.extractStatistics;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.demandStringifyIOStatistics;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_FILE_OPENED;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Cost of openFile().
+ */
+public class ITestS3AOpenCost extends AbstractS3ACostTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestS3AOpenCost.class);
+
+  private Path testFile;
+
+  private FileStatus testFileStatus;
+
+  private long fileLength;
+
+  public ITestS3AOpenCost() {
+    super(true);
+  }
+
+  /**
+   * Setup creates a test file, saves is status and length
+   * to fields.
+   */
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    S3AFileSystem fs = getFileSystem();
+    testFile = methodPath();
+
+    writeTextFile(fs, testFile, "openfile", true);
+    testFileStatus = fs.getFileStatus(testFile);
+    fileLength = testFileStatus.getLen();
+  }
+
+  /**
+   * Test when openFile() performs GET requests when file status
+   * and length options are passed down.
+   * Note that the input streams only update the FS statistics
+   * in close(), so metrics cannot be verified until all operations
+   * on a stream are complete.
+   * This is slightly less than ideal.
+   */
+  @Test
+  public void testOpenFileWithStatusOfOtherFS() throws Throwable {
+    describe("Test cost of openFile with/without status; raw only");
+    S3AFileSystem fs = getFileSystem();
+
+    // now read that file back in using the openFile call.
+    // with a new FileStatus and a different path.
+    // this verifies that any FileStatus class/subclass is used
+    // as a source of the file length.
+    FileStatus st2 = new FileStatus(
+        fileLength, false,
+        testFileStatus.getReplication(),
+        testFileStatus.getBlockSize(),
+        testFileStatus.getModificationTime(),
+        testFileStatus.getAccessTime(),
+        testFileStatus.getPermission(),
+        testFileStatus.getOwner(),
+        testFileStatus.getGroup(),
+        new Path("gopher:///localhost/" + testFile.getName()));
+
+    // no IO in open
+    FSDataInputStream in = verifyMetrics(() ->
+            fs.openFile(testFile)
+                .withFileStatus(st2)
+                .build()
+                .get(),
+        always(NO_HEAD_OR_LIST),
+        with(STREAM_READ_OPENED, 0));
+
+    // the stream gets opened during read
+    long readLen = verifyMetrics(() ->
+            readStream(in),
+        always(NO_HEAD_OR_LIST),
+        with(STREAM_READ_OPENED, 1));
+    assertEquals("bytes read from file", fileLength, readLen);
+  }
+
+  @Test
+  public void testOpenFileShorterLength() throws Throwable {
+    // do a second read with the length declared as short.
+    // we now expect the bytes read to be shorter.
+    S3AFileSystem fs = getFileSystem();
+
+    S3ATestUtils.MetricDiff bytesDiscarded =
+        new S3ATestUtils.MetricDiff(fs, STREAM_READ_BYTES_READ_CLOSE);
+    int offset = 2;
+    long shortLen = fileLength - offset;
+    // open the file
+    FSDataInputStream in2 = verifyMetrics(() ->
+            fs.openFile(testFile)
+                .must(FS_OPTION_OPENFILE_READ_POLICY,
+                    FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+                .opt(FS_OPTION_OPENFILE_LENGTH, shortLen)
+                .build()
+                .get(),
+        always(NO_HEAD_OR_LIST),
+        with(STREAM_READ_OPENED, 0));
+
+    // verify that the statistics are in range
+    IOStatistics ioStatistics = extractStatistics(in2);
+    Object statsString = demandStringifyIOStatistics(ioStatistics);
+    LOG.info("Statistics of open stream {}", statsString);
+    verifyStatisticCounterValue(ioStatistics, ACTION_FILE_OPENED, 1);
+    // no network IO happened, duration is 0. There's a very small
+    // risk of some delay making it positive just from scheduling delays
+    assertDurationRange(ioStatistics, ACTION_FILE_OPENED, 0, 0);
+    // now read it
+    long r2 = verifyMetrics(() ->
+            readStream(in2),
+        always(NO_HEAD_OR_LIST),
+        with(STREAM_READ_OPENED, 1),
+        with(STREAM_READ_BYTES_READ_CLOSE, 0),
+        with(STREAM_READ_SEEK_BYTES_SKIPPED, 0));
+
+    LOG.info("Statistics of read stream {}", statsString);
+
+    assertEquals("bytes read from file", shortLen, r2);
+    // no bytes were discarded.
+    bytesDiscarded.assertDiffEquals(0);
+  }
+
+  @Test
+  public void testOpenFileLongerLength() throws Throwable {
+    // do a second read with the length declared as longer
+    // than it is.
+    // An EOF will be read on readFully(), -1 on a read()
+
+    S3AFileSystem fs = getFileSystem();
+    // set a length past the actual file length
+    long longLen = fileLength + 10;
+    FSDataInputStream in3 = verifyMetrics(() ->
+            fs.openFile(testFile)
+                .must(FS_OPTION_OPENFILE_READ_POLICY,
+                    FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+                .must(FS_OPTION_OPENFILE_LENGTH, longLen)
+                .build()
+                .get(),
+        always(NO_HEAD_OR_LIST));
+
+    // assert behaviors of seeking/reading past the file length.
+    // there is no attempt at recovery.
+    verifyMetrics(() -> {
+      byte[] out = new byte[(int) longLen];
+      intercept(EOFException.class,
+          () -> in3.readFully(0, out));
+      in3.seek(longLen - 1);
+      assertEquals("read past real EOF on " + in3,
+          -1, in3.read());
+      in3.close();
+      return in3.toString();
+    },
+        // two GET calls were made, one for readFully,
+        // the second on the read() past the EOF
+        // the operation has got as far as S3
+        with(STREAM_READ_OPENED, 2));
+
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java
index 0077503e87c1a..ff2b3c1161f2e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ARenameCost.java
@@ -40,7 +40,7 @@
 /**
  * Use metrics to assert about the cost of file API calls.
  * <p></p>
- * Parameterized on guarded vs raw. and directory marker keep vs delete
+ * Parameterized on directory marker keep vs delete
  */
 @RunWith(Parameterized.class)
 public class ITestS3ARenameCost extends AbstractS3ACostTest {
@@ -54,18 +54,15 @@ public class ITestS3ARenameCost extends AbstractS3ACostTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> params() {
     return Arrays.asList(new Object[][]{
-        {"raw-keep-markers", false, true, false},
-        {"raw-delete-markers", false, false, false},
-        {"nonauth-keep-markers", true, true, false},
-        {"auth-delete-markers", true, false, true}
+        {"keep-markers", true},
+        {"delete-markers", false},
     });
   }
 
   public ITestS3ARenameCost(final String name,
-      final boolean s3guard,
-      final boolean keepMarkers,
-      final boolean authoritative) {
-    super(s3guard, keepMarkers, authoritative);
+      final boolean keepMarkers) {
+    super(keepMarkers);
+
   }
 
   @Test
@@ -101,7 +98,7 @@ public void testRenameFileToDifferentDirectory() throws Throwable {
     final int directoriesInPath = directoriesInPath(destDir);
     verifyMetrics(() ->
             execRename(srcFilePath, destFilePath),
-        whenRaw(RENAME_SINGLE_FILE_DIFFERENT_DIR),
+        always(RENAME_SINGLE_FILE_DIFFERENT_DIR),
         with(DIRECTORIES_CREATED, 0),
         with(DIRECTORIES_DELETED, 0),
         // keeping: only the core delete operation is issued.
@@ -152,7 +149,7 @@ public void testRenameSameDirectory() throws Throwable {
     Path destFile = new Path(parent2, "dest");
     verifyMetrics(() ->
             execRename(sourceFile, destFile),
-        whenRaw(RENAME_SINGLE_FILE_SAME_DIR),
+        always(RENAME_SINGLE_FILE_SAME_DIR),
         with(OBJECT_COPY_REQUESTS, 1),
         with(DIRECTORIES_CREATED, 0),
         with(OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST),
@@ -174,7 +171,7 @@ public void testCostOfRootFileRename() throws Throwable {
         fs.rename(src, dest);
         return "after fs.rename(/src,/dest) " + getMetricSummary();
       },
-          whenRaw(FILE_STATUS_FILE_PROBE
+          always(FILE_STATUS_FILE_PROBE
               .plus(GET_FILE_STATUS_FNFE)
               .plus(COPY_OP)),
           // here we expect there to be no fake directories
@@ -213,7 +210,7 @@ public void testCostOfRootFileDelete() throws Throwable {
           with(FAKE_DIRECTORIES_DELETED, 0),
           with(FILES_DELETED, 1),
           with(OBJECT_DELETE_REQUEST, DELETE_OBJECT_REQUEST),
-          whenRaw(FILE_STATUS_FILE_PROBE)); /* no need to look at parent. */
+          always(FILE_STATUS_FILE_PROBE)); /* no need to look at parent. */
 
     } finally {
       fs.delete(src, false);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java
new file mode 100644
index 0000000000000..a03f181cb3893
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.performance;
+
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.Constants.ASYNC_DRAIN_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
+import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
+import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES;
+import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
+import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT;
+import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_ABORTED;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SEEK_POLICY_CHANGED;
+import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_UNBUFFERED;
+
+/**
+ * Test stream unbuffer performance/behavior with stream draining
+ * and aborting.
+ */
+public class ITestUnbufferDraining extends AbstractS3ACostTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestUnbufferDraining.class);
+
+  /**
+   * Readahead range to use, sets drain threshold too.
+   */
+  public static final int READAHEAD = 1000;
+
+  /**
+   * How big a file to create?
+   */
+  public static final int FILE_SIZE = 50_000;
+
+  /**
+   * Number of attempts to unbuffer on each stream.
+   */
+  public static final int ATTEMPTS = 10;
+
+  /**
+   * Test FS with a tiny connection pool and
+   * no recovery.
+   */
+  private FileSystem brittleFS;
+
+  /**
+   * Create with markers kept, always.
+   */
+  public ITestUnbufferDraining() {
+    super(false);
+  }
+
+  @Override
+  public Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    removeBaseAndBucketOverrides(conf,
+        ASYNC_DRAIN_THRESHOLD,
+        ESTABLISH_TIMEOUT,
+        INPUT_FADVISE,
+        MAX_ERROR_RETRIES,
+        MAXIMUM_CONNECTIONS,
+        PREFETCH_ENABLED_KEY,
+        READAHEAD_RANGE,
+        REQUEST_TIMEOUT,
+        RETRY_LIMIT,
+        SOCKET_TIMEOUT);
+
+    return conf;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    // now create a new FS with minimal http capacity and recovery
+    // a separate one is used to avoid test teardown suffering
+    // from the lack of http connections and short timeouts.
+    Configuration conf = getConfiguration();
+    // kick off async drain for any data
+    conf.setInt(ASYNC_DRAIN_THRESHOLD, 1);
+    conf.setInt(MAXIMUM_CONNECTIONS, 2);
+    conf.setInt(MAX_ERROR_RETRIES, 1);
+    conf.setInt(ESTABLISH_TIMEOUT, 1000);
+    conf.setInt(READAHEAD_RANGE, READAHEAD);
+    conf.setInt(RETRY_LIMIT, 1);
+
+    brittleFS = FileSystem.newInstance(getFileSystem().getUri(), conf);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    super.teardown();
+    FileSystem bfs = getBrittleFS();
+    FILESYSTEM_IOSTATS.aggregate(retrieveIOStatistics(bfs));
+    IOUtils.cleanupWithLogger(LOG, bfs);
+  }
+
+  public FileSystem getBrittleFS() {
+    return brittleFS;
+  }
+
+  /**
+   * Test stream close performance/behavior with stream draining
+   * and unbuffer.
+   */
+  @Test
+  public void testUnbufferDraining() throws Throwable {
+
+    describe("unbuffer draining");
+    FileStatus st = createTestFile();
+
+    IOStatistics brittleStats = retrieveIOStatistics(getBrittleFS());
+    long originalUnbuffered = lookupCounter(brittleStats,
+        STREAM_READ_UNBUFFERED);
+
+    int offset = FILE_SIZE - READAHEAD + 1;
+    try (FSDataInputStream in = getBrittleFS().openFile(st.getPath())
+        .withFileStatus(st)
+        .must(ASYNC_DRAIN_THRESHOLD, 1)
+        .build().get()) {
+      describe("Initiating unbuffer with async drain\n");
+      for (int i = 0; i < ATTEMPTS; i++) {
+        describe("Starting read/unbuffer #%d", i);
+        in.seek(offset);
+        in.read();
+        in.unbuffer();
+      }
+      // verify the policy switched.
+      assertReadPolicy(in, S3AInputPolicy.Random);
+      // assert that the statistics are as expected
+      IOStatistics stats = in.getIOStatistics();
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_UNBUFFERED,
+          ATTEMPTS);
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_ABORTED,
+          0);
+      // there's always a policy of 1, so
+      // this value must be 1 + 1
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_SEEK_POLICY_CHANGED,
+          2);
+    }
+    // filesystem statistic propagation
+    verifyStatisticCounterValue(brittleStats,
+        STREAM_READ_UNBUFFERED,
+        ATTEMPTS + originalUnbuffered);
+  }
+
+  /**
+   * Lookup a counter, returning 0 if it is not defined.
+   * @param statistics stats to probe
+   * @param key counter key
+   * @return the value or 0
+   */
+  private static long lookupCounter(
+      final IOStatistics statistics,
+      final String key) {
+    Long counter = statistics.counters().get(key);
+    return counter == null ? 0 : counter;
+  }
+
+  /**
+   * Assert that the read policy is as expected.
+   * @param in input stream
+   * @param policy read policy.
+   */
+  private static void assertReadPolicy(final FSDataInputStream in,
+      final S3AInputPolicy policy) {
+    S3AInputStream inner = (S3AInputStream) in.getWrappedStream();
+    Assertions.assertThat(inner.getInputPolicy())
+        .describedAs("input policy of %s", inner)
+        .isEqualTo(policy);
+  }
+
+  /**
+   * Test stream close performance/behavior with unbuffer
+   * aborting rather than draining.
+   */
+  @Test
+  public void testUnbufferAborting() throws Throwable {
+
+    describe("unbuffer aborting");
+    FileStatus st = createTestFile();
+    IOStatistics brittleStats = retrieveIOStatistics(getBrittleFS());
+    long originalUnbuffered =
+        lookupCounter(brittleStats, STREAM_READ_UNBUFFERED);
+    long originalAborted =
+        lookupCounter(brittleStats, STREAM_READ_ABORTED);
+
+    // open the file at the beginning with a whole file read policy,
+    // so even with s3a switching to random on unbuffer,
+    // this always does a full GET
+    try (FSDataInputStream in = getBrittleFS().openFile(st.getPath())
+        .withFileStatus(st)
+        .must(ASYNC_DRAIN_THRESHOLD, 1)
+        .must(FS_OPTION_OPENFILE_READ_POLICY,
+            FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+        .build().get()) {
+      assertReadPolicy(in, S3AInputPolicy.Sequential);
+
+      describe("Initiating unbuffer with async drain\n");
+      for (int i = 0; i < ATTEMPTS; i++) {
+        describe("Starting read/unbuffer #%d", i);
+        in.read();
+        in.unbuffer();
+        // because the read policy is sequential, it doesn't change
+        assertReadPolicy(in, S3AInputPolicy.Sequential);
+      }
+
+      // assert that the statistics are as expected
+      IOStatistics stats = in.getIOStatistics();
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_UNBUFFERED,
+          ATTEMPTS);
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_ABORTED,
+          ATTEMPTS);
+      // there's always a policy of 1.
+      verifyStatisticCounterValue(stats,
+          STREAM_READ_SEEK_POLICY_CHANGED,
+          1);
+    }
+    // look at FS statistics
+    verifyStatisticCounterValue(brittleStats,
+        STREAM_READ_UNBUFFERED,
+        ATTEMPTS + originalUnbuffered);
+    verifyStatisticCounterValue(brittleStats,
+        STREAM_READ_ABORTED,
+        ATTEMPTS + originalAborted);
+  }
+
+  private FileStatus createTestFile() throws IOException {
+    byte[] data = dataset(FILE_SIZE, '0', 10);
+    S3AFileSystem fs = getFileSystem();
+
+    Path path = methodPath();
+    ContractTestUtils.createFile(fs, path, true, data);
+    return fs.getFileStatus(path);
+  }
+
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
index af4cfba0aa078..08a19600d5830 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java
@@ -57,11 +57,18 @@ public final class OperationCost {
   public static final int DELETE_MARKER_REQUEST = DELETE_OBJECT_REQUEST;
 
   /**
-   * No IO takes place.
+   * No Head or List IO takes place; other operations
+   * may still take place.
    */
   public static final OperationCost NO_IO =
       new OperationCost(0, 0);
 
+  /**
+   * More detailed description of the NO_IO cost.
+   */
+  public static final OperationCost NO_HEAD_OR_LIST =
+      NO_IO;
+
   /** A HEAD operation. */
   public static final OperationCost HEAD_OPERATION = new OperationCost(1, 0);
 
@@ -81,7 +88,7 @@ public final class OperationCost {
   /**
    * Cost of getFileStatus on root directory.
    */
-  public static final OperationCost ROOT_FILE_STATUS_PROBE = NO_IO;
+  public static final OperationCost ROOT_FILE_STATUS_PROBE = NO_HEAD_OR_LIST;
 
   /**
    * Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#ALL}.
@@ -120,7 +127,6 @@ public final class OperationCost {
   public static final OperationCost LIST_STATUS_LIST_OP = LIST_OPERATION;
   /**
    * Metadata cost of a copy operation, as used during rename.
-   * This happens even if the store is guarded.
    */
   public static final OperationCost COPY_OP =
       new OperationCost(1, 0);
@@ -161,13 +167,6 @@ public final class OperationCost {
   public static final OperationCost CREATE_FILE_NO_OVERWRITE =
       FILE_STATUS_ALL_PROBES;
 
-  /**
-   * S3Guard in non-auth mode always attempts a single file
-   * status call.
-   */
-  public static final OperationCost S3GUARD_NONAUTH_FILE_STATUS_PROBE =
-      FILE_STATUS_FILE_PROBE;
-
   /** Expected HEAD count. */
   private final int head;
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java
index 72e51ee3b9958..3a8a1f6ad7b6f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java
@@ -20,6 +20,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -34,6 +35,7 @@
 import org.apache.hadoop.fs.s3a.S3AInstrumentation;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.Statistic;
+import org.apache.hadoop.fs.s3a.statistics.StatisticTypeEnum;
 import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
 import org.apache.hadoop.metrics2.lib.MutableCounter;
 import org.apache.hadoop.metrics2.lib.MutableMetric;
@@ -274,7 +276,7 @@ public Builder withMetric(Statistic statistic) {
 
     /**
      * Add a varargs list of metrics.
-     * @param stat statistics to monitor.
+     * @param stats statistics to monitor.
      * @return this.
      */
     public Builder withMetrics(Statistic...stats) {
@@ -282,6 +284,20 @@ public Builder withMetrics(Statistic...stats) {
       return this;
     }
 
+    /**
+     * Add all counters and duration types to the
+     * metrics which can be asserted over.
+     * @return this.
+     */
+    public Builder withAllCounters() {
+      EnumSet.allOf(Statistic.class).stream()
+          .filter(s ->
+              s.getType() == StatisticTypeEnum.TYPE_COUNTER
+                  || s.getType() == StatisticTypeEnum.TYPE_DURATION)
+          .forEach(metrics::add);
+      return this;
+    }
+
     /**
      * Instantiate.
      * @return the validator.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java
new file mode 100644
index 0000000000000..6e2f547a22ec1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.CompletableFuture;
+
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
+/**
+ * A mock s3 file with some fault injection.
+ */
+class MockS3ARemoteObject extends S3ARemoteObject {
+
+  private byte[] contents;
+
+  // If true, throws IOException on open request just once.
+  // That allows test code to validate behavior related to retries.
+  private boolean throwExceptionOnOpen;
+
+  private static final String BUCKET = "bucket";
+
+  private static final String KEY = "key";
+
+  MockS3ARemoteObject(int size) {
+    this(size, false);
+  }
+
+  MockS3ARemoteObject(int size, boolean throwExceptionOnOpen) {
+    super(
+        S3APrefetchFakes.createReadContext(null, KEY, size, 1, 1),
+        S3APrefetchFakes.createObjectAttributes(BUCKET, KEY, size),
+        S3APrefetchFakes.createInputStreamCallbacks(BUCKET, KEY),
+        EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS,
+        S3APrefetchFakes.createChangeTracker(BUCKET, KEY, size)
+    );
+
+    this.throwExceptionOnOpen = throwExceptionOnOpen;
+    this.contents = new byte[size];
+    for (int b = 0; b < size; b++) {
+      this.contents[b] = byteAtOffset(b);
+    }
+  }
+
+  @Override
+  public InputStream openForRead(long offset, int size) throws IOException {
+    Validate.checkLessOrEqual(offset, "offset", size(), "size()");
+    Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset");
+
+    if (throwExceptionOnOpen) {
+      throwExceptionOnOpen = false;
+      throw new IOException("Throwing because throwExceptionOnOpen is true ");
+    }
+    int bufSize = (int) Math.min(size, size() - offset);
+    return new ByteArrayInputStream(contents, (int) offset, bufSize);
+  }
+
+  @Override
+  public void close(InputStream inputStream, int numRemainingBytes) {
+    // do nothing since we do not use a real S3 stream.
+  }
+
+  public static byte byteAtOffset(int offset) {
+    return (byte) (offset % 128);
+  }
+
+  public static S3AInputStream.InputStreamCallbacks createClient(String bucketName) {
+    return new S3AInputStream.InputStreamCallbacks() {
+      @Override
+      public S3Object getObject(GetObjectRequest request) {
+        return null;
+      }
+
+      @Override
+      public <T> CompletableFuture<T> submit(CallableRaisingIOE<T> operation) {
+        return null;
+      }
+
+      @Override
+      public GetObjectRequest newGetRequest(String key) {
+        return new GetObjectRequest(bucketName, key);
+      }
+
+      @Override
+      public void close() {
+      }
+    };
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java
new file mode 100644
index 0000000000000..bab07f4f9ec83
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java
@@ -0,0 +1,413 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.impl.prefetch.BlockCache;
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.impl.prefetch.SingleFilePerBlockCache;
+import org.apache.hadoop.fs.impl.prefetch.Validate;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.VectoredIOContext;
+import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
+import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext;
+import org.apache.hadoop.fs.s3a.statistics.impl.CountingChangeTracker;
+import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
+
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatisticsStore;
+
+/**
+ * Provides 'fake' implementations of S3ARemoteInputStream variants.
+ *
+ * These implementations avoid accessing the following real resources:
+ * -- S3 store
+ * -- local filesystem
+ *
+ * This arrangement allows thorough multi-threaded testing of those
+ * implementations without accessing external resources. It also helps
+ * avoid test flakiness introduced by external factors.
+ */
+public final class S3APrefetchFakes {
+
+  private S3APrefetchFakes() {
+  }
+
+  public static final String E_TAG = "eTag";
+
+  public static final String OWNER = "owner";
+
+  public static final String VERSION_ID = "v1";
+
+  public static final long MODIFICATION_TIME = 0L;
+
+  public static final ChangeDetectionPolicy CHANGE_POLICY =
+      ChangeDetectionPolicy.createPolicy(
+          ChangeDetectionPolicy.Mode.None,
+          ChangeDetectionPolicy.Source.None,
+          false);
+
+  public static S3AFileStatus createFileStatus(String key, long fileSize) {
+    org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key);
+    long blockSize = fileSize;
+    return new S3AFileStatus(
+        fileSize, MODIFICATION_TIME, path, blockSize, OWNER, E_TAG, VERSION_ID);
+  }
+
+  public static S3ObjectAttributes createObjectAttributes(
+      String bucket,
+      String key,
+      long fileSize) {
+
+    org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key);
+    String encryptionKey = "";
+
+    return new S3ObjectAttributes(
+        bucket,
+        path,
+        key,
+        S3AEncryptionMethods.NONE,
+        encryptionKey,
+        E_TAG,
+        VERSION_ID,
+        fileSize);
+  }
+
+  public static S3AReadOpContext createReadContext(
+      ExecutorServiceFuturePool futurePool,
+      String key,
+      int fileSize,
+      int prefetchBlockSize,
+      int prefetchBlockCount) {
+
+    S3AFileStatus fileStatus = createFileStatus(key, fileSize);
+    org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key);
+    FileSystem.Statistics statistics = new FileSystem.Statistics("s3a");
+    S3AStatisticsContext statisticsContext = new EmptyS3AStatisticsContext();
+    RetryPolicy retryPolicy =
+        RetryPolicies.retryUpToMaximumCountWithFixedSleep(3, 10,
+            TimeUnit.MILLISECONDS);
+
+    return new S3AReadOpContext(
+        path,
+        new Invoker(retryPolicy, Invoker.LOG_EVENT),
+        statistics,
+        statisticsContext,
+        fileStatus,
+        new VectoredIOContext()
+            .setMinSeekForVectoredReads(1)
+            .setMaxReadSizeForVectoredReads(1)
+            .build(),
+        emptyStatisticsStore(),
+        futurePool,
+        prefetchBlockSize,
+        prefetchBlockCount)
+        .withChangeDetectionPolicy(
+            ChangeDetectionPolicy.createPolicy(ChangeDetectionPolicy.Mode.None,
+                ChangeDetectionPolicy.Source.ETag, false))
+        .withInputPolicy(S3AInputPolicy.Normal);
+  }
+
+  public static URI createUri(String bucket, String key) {
+    return URI.create(String.format("s3a://%s/%s", bucket, key));
+  }
+
+  public static ChangeTracker createChangeTracker(
+      String bucket,
+      String key,
+      long fileSize) {
+
+    return new ChangeTracker(
+        createUri(bucket, key).toString(),
+        CHANGE_POLICY,
+        new CountingChangeTracker(),
+        createObjectAttributes(bucket, key, fileSize));
+  }
+
+  public static S3ObjectInputStream createS3ObjectInputStream(byte[] buffer) {
+    return new S3ObjectInputStream(new ByteArrayInputStream(buffer), null);
+  }
+
+  public static S3AInputStream.InputStreamCallbacks createInputStreamCallbacks(
+      String bucket,
+      String key) {
+
+    S3Object object = new S3Object() {
+      @Override
+      public S3ObjectInputStream getObjectContent() {
+        return createS3ObjectInputStream(new byte[8]);
+      }
+
+      @Override
+      public ObjectMetadata getObjectMetadata() {
+        ObjectMetadata metadata = new ObjectMetadata();
+        metadata.setHeader("ETag", E_TAG);
+        return metadata;
+      }
+    };
+
+    return new S3AInputStream.InputStreamCallbacks() {
+      @Override
+      public S3Object getObject(GetObjectRequest request) {
+        return object;
+      }
+
+      @Override
+      public <T> CompletableFuture<T> submit(CallableRaisingIOE<T> operation) {
+        return null;
+      }
+
+      @Override
+      public GetObjectRequest newGetRequest(String key) {
+        return new GetObjectRequest(bucket, key);
+      }
+
+      @Override
+      public void close() {
+      }
+    };
+  }
+
+
+  public static S3ARemoteInputStream createInputStream(
+      Class<? extends S3ARemoteInputStream> clazz,
+      ExecutorServiceFuturePool futurePool,
+      String bucket,
+      String key,
+      int fileSize,
+      int prefetchBlockSize,
+      int prefetchBlockCount) {
+
+    org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key);
+
+    S3AFileStatus fileStatus = createFileStatus(key, fileSize);
+    S3ObjectAttributes s3ObjectAttributes =
+        createObjectAttributes(bucket, key, fileSize);
+    S3AReadOpContext s3AReadOpContext = createReadContext(
+        futurePool,
+        key,
+        fileSize,
+        prefetchBlockSize,
+        prefetchBlockCount);
+
+    S3AInputStream.InputStreamCallbacks callbacks =
+        createInputStreamCallbacks(bucket, key);
+    S3AInputStreamStatistics stats =
+        s3AReadOpContext.getS3AStatisticsContext().newInputStreamStatistics();
+
+    if (clazz == FakeS3AInMemoryInputStream.class) {
+      return new FakeS3AInMemoryInputStream(s3AReadOpContext,
+          s3ObjectAttributes, callbacks, stats);
+    } else if (clazz == FakeS3ACachingInputStream.class) {
+      return new FakeS3ACachingInputStream(s3AReadOpContext, s3ObjectAttributes,
+          callbacks, stats);
+    }
+
+    throw new RuntimeException("Unsupported class: " + clazz);
+  }
+
+  public static FakeS3AInMemoryInputStream createS3InMemoryInputStream(
+      ExecutorServiceFuturePool futurePool,
+      String bucket,
+      String key,
+      int fileSize) {
+
+    return (FakeS3AInMemoryInputStream) createInputStream(
+        FakeS3AInMemoryInputStream.class, futurePool, bucket, key, fileSize, 1,
+        1);
+  }
+
+  public static FakeS3ACachingInputStream createS3CachingInputStream(
+      ExecutorServiceFuturePool futurePool,
+      String bucket,
+      String key,
+      int fileSize,
+      int prefetchBlockSize,
+      int prefetchBlockCount) {
+
+    return (FakeS3ACachingInputStream) createInputStream(
+        FakeS3ACachingInputStream.class,
+        futurePool,
+        bucket,
+        key,
+        fileSize,
+        prefetchBlockSize,
+        prefetchBlockCount);
+  }
+
+  public static class FakeS3AInMemoryInputStream
+      extends S3AInMemoryInputStream {
+
+    public FakeS3AInMemoryInputStream(
+        S3AReadOpContext context,
+        S3ObjectAttributes s3Attributes,
+        S3AInputStream.InputStreamCallbacks client,
+        S3AInputStreamStatistics streamStatistics) {
+      super(context, s3Attributes, client, streamStatistics);
+    }
+
+    @Override
+    protected S3ARemoteObject getS3File() {
+      randomDelay(200);
+      return new MockS3ARemoteObject(
+          (int) this.getS3ObjectAttributes().getLen(), false);
+    }
+  }
+
+  public static class FakeS3FilePerBlockCache extends SingleFilePerBlockCache {
+
+    private final Map<Path, byte[]> files;
+
+    private final int readDelay;
+
+    private final int writeDelay;
+
+    public FakeS3FilePerBlockCache(int readDelay, int writeDelay) {
+      super(new EmptyS3AStatisticsContext().newInputStreamStatistics());
+      this.files = new ConcurrentHashMap<>();
+      this.readDelay = readDelay;
+      this.writeDelay = writeDelay;
+    }
+
+    @Override
+    protected int readFile(Path path, ByteBuffer buffer) {
+      byte[] source = this.files.get(path);
+      randomDelay(this.readDelay);
+      buffer.put(source);
+      return source.length;
+    }
+
+    @Override
+    protected void writeFile(Path path, ByteBuffer buffer) throws IOException {
+      Validate.checkPositiveInteger(buffer.limit(), "buffer.limit()");
+      byte[] dest = new byte[buffer.limit()];
+      randomDelay(this.writeDelay);
+      buffer.rewind();
+      buffer.get(dest);
+      this.files.put(path, dest);
+    }
+
+    private long fileCount = 0;
+
+    @Override
+    protected Path getCacheFilePath() throws IOException {
+      fileCount++;
+      return Paths.get(Long.toString(fileCount));
+    }
+
+    @Override
+    public void close() throws IOException {
+      this.files.clear();
+    }
+  }
+
+  private static final Random RANDOM = new Random();
+
+  private static void randomDelay(int delay) {
+    try {
+      Thread.sleep(RANDOM.nextInt(delay));
+    } catch (InterruptedException e) {
+
+    }
+  }
+
+  public static class FakeS3ACachingBlockManager
+      extends S3ACachingBlockManager {
+
+    public FakeS3ACachingBlockManager(
+        ExecutorServiceFuturePool futurePool,
+        S3ARemoteObjectReader reader,
+        BlockData blockData,
+        int bufferPoolSize) {
+      super(futurePool, reader, blockData, bufferPoolSize,
+          new EmptyS3AStatisticsContext().newInputStreamStatistics());
+    }
+
+    @Override
+    public int read(ByteBuffer buffer, long offset, int size)
+        throws IOException {
+      randomDelay(100);
+      return this.getReader().read(buffer, offset, size);
+    }
+
+    @Override
+    protected BlockCache createCache() {
+      final int readDelayMs = 50;
+      final int writeDelayMs = 200;
+      return new FakeS3FilePerBlockCache(readDelayMs, writeDelayMs);
+    }
+  }
+
+  public static class FakeS3ACachingInputStream extends S3ACachingInputStream {
+
+    public FakeS3ACachingInputStream(
+        S3AReadOpContext context,
+        S3ObjectAttributes s3Attributes,
+        S3AInputStream.InputStreamCallbacks client,
+        S3AInputStreamStatistics streamStatistics) {
+      super(context, s3Attributes, client, streamStatistics);
+    }
+
+    @Override
+    protected S3ARemoteObject getS3File() {
+      randomDelay(200);
+      return new MockS3ARemoteObject(
+          (int) this.getS3ObjectAttributes().getLen(), false);
+    }
+
+    @Override
+    protected S3ACachingBlockManager createBlockManager(
+        ExecutorServiceFuturePool futurePool,
+        S3ARemoteObjectReader reader,
+        BlockData blockData,
+        int bufferPoolSize) {
+      return new FakeS3ACachingBlockManager(futurePool, reader, blockData,
+          bufferPoolSize);
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java
new file mode 100644
index 0000000000000..c1b59d6f2e130
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.BufferData;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+
+public class TestS3ABlockManager extends AbstractHadoopTestBase {
+
+  static final int FILE_SIZE = 12;
+
+  static final int BLOCK_SIZE = 3;
+
+  @Test
+  public void testArgChecks() throws Exception {
+    BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE);
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+
+    // Should not throw.
+    new S3ABlockManager(reader, blockData);
+
+    // Verify it throws correctly.
+    intercept(
+        IllegalArgumentException.class,
+        "'reader' must not be null",
+        () -> new S3ABlockManager(null, blockData));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'blockData' must not be null",
+        () -> new S3ABlockManager(reader, null));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> new S3ABlockManager(reader, blockData).get(-1));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'data' must not be null",
+        () -> new S3ABlockManager(reader, blockData).release(null));
+  }
+
+  @Test
+  public void testGet() throws IOException {
+    BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE);
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+    S3ABlockManager blockManager = new S3ABlockManager(reader, blockData);
+
+    for (int b = 0; b < blockData.getNumBlocks(); b++) {
+      BufferData data = blockManager.get(b);
+      ByteBuffer buffer = data.getBuffer();
+      long startOffset = blockData.getStartOffset(b);
+      for (int i = 0; i < BLOCK_SIZE; i++) {
+        assertEquals(startOffset + i, buffer.get());
+      }
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java
new file mode 100644
index 0000000000000..aecf8802beb85
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.impl.prefetch.BlockData;
+import org.apache.hadoop.fs.impl.prefetch.BufferData;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+
+public class TestS3ACachingBlockManager extends AbstractHadoopTestBase {
+
+  static final int FILE_SIZE = 15;
+
+  static final int BLOCK_SIZE = 2;
+
+  static final int POOL_SIZE = 3;
+
+  private final ExecutorService threadPool = Executors.newFixedThreadPool(4);
+
+  private final ExecutorServiceFuturePool futurePool =
+      new ExecutorServiceFuturePool(threadPool);
+
+  private final S3AInputStreamStatistics streamStatistics =
+      new EmptyS3AStatisticsContext().newInputStreamStatistics();
+
+  private final BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE);
+
+  @Test
+  public void testArgChecks() throws Exception {
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+
+    // Should not throw.
+    S3ACachingBlockManager blockManager =
+        new S3ACachingBlockManager(futurePool, reader, blockData, POOL_SIZE,
+            streamStatistics);
+
+    // Verify it throws correctly.
+    intercept(
+        NullPointerException.class,
+        () -> new S3ACachingBlockManager(null, reader, blockData, POOL_SIZE,
+            streamStatistics));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'reader' must not be null",
+        () -> new S3ACachingBlockManager(futurePool, null, blockData, POOL_SIZE,
+            streamStatistics));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'blockData' must not be null",
+        () -> new S3ACachingBlockManager(futurePool, reader, null, POOL_SIZE,
+            streamStatistics));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'bufferPoolSize' must be a positive integer",
+        () -> new S3ACachingBlockManager(futurePool, reader, blockData, 0,
+            streamStatistics));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'bufferPoolSize' must be a positive integer",
+        () -> new S3ACachingBlockManager(futurePool, reader, blockData, -1,
+            streamStatistics));
+
+    intercept(NullPointerException.class,
+        () -> new S3ACachingBlockManager(futurePool, reader, blockData,
+            POOL_SIZE, null));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> blockManager.get(-1));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'data' must not be null",
+        () -> blockManager.release(null));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'blockNumber' must not be negative",
+        () -> blockManager.requestPrefetch(-1));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'data' must not be null",
+        () -> blockManager.requestCaching(null));
+  }
+
+  /**
+   * Extends S3ACachingBlockManager so that we can inject asynchronous failures.
+   */
+  private static final class BlockManagerForTesting
+      extends S3ACachingBlockManager {
+
+    BlockManagerForTesting(
+        ExecutorServiceFuturePool futurePool,
+        S3ARemoteObjectReader reader,
+        BlockData blockData,
+        int bufferPoolSize,
+        S3AInputStreamStatistics streamStatistics) {
+      super(futurePool, reader, blockData, bufferPoolSize, streamStatistics);
+    }
+
+    // If true, forces the next read operation to fail.
+    // Resets itself to false after one failure.
+    private boolean forceNextReadToFail;
+
+    @Override
+    public int read(ByteBuffer buffer, long offset, int size)
+        throws IOException {
+      if (forceNextReadToFail) {
+        forceNextReadToFail = false;
+        throw new RuntimeException("foo");
+      } else {
+        return super.read(buffer, offset, size);
+      }
+    }
+
+    // If true, forces the next cache-put operation to fail.
+    // Resets itself to false after one failure.
+    private boolean forceNextCachePutToFail;
+
+    @Override
+    protected void cachePut(int blockNumber, ByteBuffer buffer)
+        throws IOException {
+      if (forceNextCachePutToFail) {
+        forceNextCachePutToFail = false;
+        throw new RuntimeException("bar");
+      } else {
+        super.cachePut(blockNumber, buffer);
+      }
+    }
+  }
+
+  // @Ignore
+  @Test
+  public void testGet() throws Exception {
+    testGetHelper(false);
+  }
+
+  // @Ignore
+  @Test
+  public void testGetFailure() throws Exception {
+    testGetHelper(true);
+  }
+
+  private void testGetHelper(boolean forceReadFailure) throws Exception {
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, true);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+    BlockManagerForTesting blockManager =
+        new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE,
+            streamStatistics);
+
+    for (int b = 0; b < blockData.getNumBlocks(); b++) {
+      // We simulate caching failure for all even numbered blocks.
+      boolean forceFailure = forceReadFailure && (b % 2 == 0);
+
+      BufferData data = null;
+
+      if (forceFailure) {
+        blockManager.forceNextReadToFail = true;
+
+        intercept(
+            RuntimeException.class,
+            "foo",
+            () -> blockManager.get(3));
+      } else {
+        data = blockManager.get(b);
+
+        long startOffset = blockData.getStartOffset(b);
+        for (int i = 0; i < blockData.getSize(b); i++) {
+          assertEquals(startOffset + i, data.getBuffer().get());
+        }
+
+        blockManager.release(data);
+      }
+
+      assertEquals(POOL_SIZE, blockManager.numAvailable());
+    }
+  }
+
+  // @Ignore
+  @Test
+  public void testPrefetch() throws IOException, InterruptedException {
+    testPrefetchHelper(false);
+  }
+
+  // @Ignore
+  @Test
+  public void testPrefetchFailure() throws IOException, InterruptedException {
+    testPrefetchHelper(true);
+  }
+
+  private void testPrefetchHelper(boolean forcePrefetchFailure)
+      throws IOException, InterruptedException {
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+    BlockManagerForTesting blockManager =
+        new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE,
+            streamStatistics);
+    assertInitialState(blockManager);
+
+    int expectedNumErrors = 0;
+    int expectedNumSuccesses = 0;
+
+    for (int b = 0; b < POOL_SIZE; b++) {
+      // We simulate caching failure for all odd numbered blocks.
+      boolean forceFailure = forcePrefetchFailure && (b % 2 == 1);
+      if (forceFailure) {
+        expectedNumErrors++;
+        blockManager.forceNextReadToFail = true;
+      } else {
+        expectedNumSuccesses++;
+      }
+      blockManager.requestPrefetch(b);
+    }
+
+    assertEquals(0, blockManager.numCached());
+
+    blockManager.cancelPrefetches();
+    waitForCaching(blockManager, expectedNumSuccesses);
+    assertEquals(expectedNumErrors, this.totalErrors(blockManager));
+    assertEquals(expectedNumSuccesses, blockManager.numCached());
+  }
+
+  // @Ignore
+  @Test
+  public void testCachingOfPrefetched()
+      throws IOException, InterruptedException {
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+    S3ACachingBlockManager blockManager =
+        new S3ACachingBlockManager(futurePool, reader, blockData, POOL_SIZE,
+            streamStatistics);
+    assertInitialState(blockManager);
+
+    for (int b = 0; b < blockData.getNumBlocks(); b++) {
+      blockManager.requestPrefetch(b);
+      BufferData data = blockManager.get(b);
+      blockManager.requestCaching(data);
+    }
+
+    waitForCaching(blockManager, blockData.getNumBlocks());
+    assertEquals(blockData.getNumBlocks(), blockManager.numCached());
+    assertEquals(0, this.totalErrors(blockManager));
+  }
+
+  // @Ignore
+  @Test
+  public void testCachingOfGet() throws IOException, InterruptedException {
+    testCachingOfGetHelper(false);
+  }
+
+  // @Ignore
+  @Test
+  public void testCachingFailureOfGet()
+      throws IOException, InterruptedException {
+    testCachingOfGetHelper(true);
+  }
+
+  public void testCachingOfGetHelper(boolean forceCachingFailure)
+      throws IOException, InterruptedException {
+    MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+    BlockManagerForTesting blockManager =
+        new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE,
+            streamStatistics);
+    assertInitialState(blockManager);
+
+    int expectedNumErrors = 0;
+    int expectedNumSuccesses = 0;
+
+    for (int b = 0; b < blockData.getNumBlocks(); b++) {
+      // We simulate caching failure for all odd numbered blocks.
+      boolean forceFailure = forceCachingFailure && (b % 2 == 1);
+      if (forceFailure) {
+        expectedNumErrors++;
+      } else {
+        expectedNumSuccesses++;
+      }
+
+      BufferData data = blockManager.get(b);
+      if (forceFailure) {
+        blockManager.forceNextCachePutToFail = true;
+      }
+
+      blockManager.requestCaching(data);
+      waitForCaching(blockManager, expectedNumSuccesses);
+      assertEquals(expectedNumSuccesses, blockManager.numCached());
+
+      if (forceCachingFailure) {
+        assertEquals(expectedNumErrors, this.totalErrors(blockManager));
+      } else {
+        assertEquals(0, this.totalErrors(blockManager));
+      }
+    }
+  }
+
+  private void waitForCaching(
+      S3ACachingBlockManager blockManager,
+      int expectedCount)
+      throws InterruptedException {
+    // Wait for async cache operation to be over.
+    int numTrys = 0;
+    int count;
+    do {
+      Thread.sleep(100);
+      count = blockManager.numCached();
+      numTrys++;
+      if (numTrys > 600) {
+        String message = String.format(
+            "waitForCaching: expected: %d, actual: %d, read errors: %d, caching errors: %d",
+            expectedCount, count, blockManager.numReadErrors(),
+            blockManager.numCachingErrors());
+        throw new IllegalStateException(message);
+      }
+    }
+    while (count < expectedCount);
+  }
+
+  private int totalErrors(S3ACachingBlockManager blockManager) {
+    return blockManager.numCachingErrors() + blockManager.numReadErrors();
+  }
+
+  private void assertInitialState(S3ACachingBlockManager blockManager) {
+    assertEquals(0, blockManager.numCached());
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java
new file mode 100644
index 0000000000000..d449a79a5a851
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.impl.prefetch.ExceptionAsserts;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Applies the same set of tests to both S3ACachingInputStream and S3AInMemoryInputStream.
+ */
+public class TestS3ARemoteInputStream extends AbstractHadoopTestBase {
+
+  private static final int FILE_SIZE = 10;
+
+  private final ExecutorService threadPool = Executors.newFixedThreadPool(4);
+
+  private final ExecutorServiceFuturePool futurePool =
+      new ExecutorServiceFuturePool(threadPool);
+
+  private final S3AInputStream.InputStreamCallbacks client =
+      MockS3ARemoteObject.createClient("bucket");
+
+  @Test
+  public void testArgChecks() throws Exception {
+    S3AReadOpContext readContext =
+        S3APrefetchFakes.createReadContext(futurePool, "key", 10, 10, 1);
+    S3ObjectAttributes attrs =
+        S3APrefetchFakes.createObjectAttributes("bucket", "key", 10);
+    S3AInputStreamStatistics stats =
+        readContext.getS3AStatisticsContext().newInputStreamStatistics();
+
+    // Should not throw.
+    new S3ACachingInputStream(readContext, attrs, client, stats);
+
+    ExceptionAsserts.assertThrows(
+        NullPointerException.class,
+        () -> new S3ACachingInputStream(null, attrs, client, stats));
+
+    ExceptionAsserts.assertThrows(
+        NullPointerException.class,
+        () -> new S3ACachingInputStream(readContext, null, client, stats));
+
+    ExceptionAsserts.assertThrows(
+        NullPointerException.class,
+        () -> new S3ACachingInputStream(readContext, attrs, null, stats));
+
+    ExceptionAsserts.assertThrows(
+        NullPointerException.class,
+        () -> new S3ACachingInputStream(readContext, attrs, client, null));
+  }
+
+  @Test
+  public void testRead0SizedFile() throws Exception {
+    S3ARemoteInputStream inputStream =
+        S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket",
+            "key", 0);
+    testRead0SizedFileHelper(inputStream, 9);
+
+    inputStream =
+        S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key",
+            0, 5, 2);
+    testRead0SizedFileHelper(inputStream, 5);
+  }
+
+  private void testRead0SizedFileHelper(S3ARemoteInputStream inputStream,
+      int bufferSize)
+      throws Exception {
+    assertAvailable(0, inputStream);
+    assertEquals(-1, inputStream.read());
+    assertEquals(-1, inputStream.read());
+
+    byte[] buffer = new byte[2];
+    assertEquals(-1, inputStream.read(buffer));
+    assertEquals(-1, inputStream.read());
+  }
+
+  @Test
+  public void testRead() throws Exception {
+    S3ARemoteInputStream inputStream =
+        S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket",
+            "key", FILE_SIZE);
+    testReadHelper(inputStream, FILE_SIZE);
+
+    inputStream =
+        S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key",
+            FILE_SIZE, 5, 2);
+    testReadHelper(inputStream, 5);
+  }
+
+  private void testReadHelper(S3ARemoteInputStream inputStream, int bufferSize)
+      throws Exception {
+    assertEquals(0, inputStream.read());
+    assertAvailable(bufferSize - 1, inputStream);
+    assertEquals(1, inputStream.read());
+
+    byte[] buffer = new byte[2];
+    assertEquals(2, inputStream.read(buffer));
+    assertEquals(2, buffer[0]);
+    assertEquals(3, buffer[1]);
+
+    assertEquals(4, inputStream.read());
+
+    buffer = new byte[10];
+    int curPos = (int) inputStream.getPos();
+    int expectedRemainingBytes = (int) (FILE_SIZE - curPos);
+    int readStartOffset = 2;
+    assertEquals(
+        expectedRemainingBytes,
+        inputStream.read(buffer, readStartOffset, expectedRemainingBytes));
+
+    for (int i = 0; i < expectedRemainingBytes; i++) {
+      assertEquals(curPos + i, buffer[readStartOffset + i]);
+    }
+
+    assertEquals(-1, inputStream.read());
+    Thread.sleep(100);
+    assertEquals(-1, inputStream.read());
+    assertEquals(-1, inputStream.read());
+    assertEquals(-1, inputStream.read(buffer));
+    assertEquals(-1, inputStream.read(buffer, 1, 3));
+  }
+
+  @Test
+  public void testSeek() throws Exception {
+    S3ARemoteInputStream inputStream;
+    inputStream =
+        S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket",
+            "key", 9);
+    testSeekHelper(inputStream, 9, 9);
+
+    inputStream =
+        S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key",
+            9, 5, 1);
+    testSeekHelper(inputStream, 5, 9);
+  }
+
+  private void testSeekHelper(S3ARemoteInputStream inputStream,
+      int bufferSize,
+      int fileSize)
+      throws Exception {
+    assertAvailable(0, inputStream);
+    assertEquals(0, inputStream.getPos());
+    inputStream.seek(bufferSize);
+    assertAvailable(0, inputStream);
+    assertEquals(bufferSize, inputStream.getPos());
+    inputStream.seek(0);
+    assertAvailable(0, inputStream);
+
+    for (int i = 0; i < fileSize; i++) {
+      assertEquals(i, inputStream.read());
+    }
+
+    for (int i = 0; i < fileSize; i++) {
+      inputStream.seek(i);
+      for (int j = i; j < fileSize; j++) {
+        assertEquals(j, inputStream.read());
+      }
+    }
+
+    // Can seek to the EOF: read() will then return -1.
+    inputStream.seek(fileSize);
+    assertEquals(-1, inputStream.read());
+
+    // Test invalid seeks.
+    ExceptionAsserts.assertThrows(
+        EOFException.class,
+        FSExceptionMessages.NEGATIVE_SEEK,
+        () -> inputStream.seek(-1));
+
+    ExceptionAsserts.assertThrows(
+        EOFException.class,
+        FSExceptionMessages.CANNOT_SEEK_PAST_EOF,
+        () -> inputStream.seek(fileSize + 1));
+  }
+
+  @Test
+  public void testRandomSeek() throws Exception {
+    S3ARemoteInputStream inputStream;
+    inputStream =
+        S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket",
+            "key", 9);
+    testRandomSeekHelper(inputStream, 9, 9);
+
+    inputStream =
+        S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key",
+            9, 5, 1);
+    testRandomSeekHelper(inputStream, 5, 9);
+  }
+
+  private void testRandomSeekHelper(S3ARemoteInputStream inputStream,
+      int bufferSize,
+      int fileSize)
+      throws Exception {
+    assertEquals(0, inputStream.getPos());
+    inputStream.seek(7);
+    assertEquals(7, inputStream.getPos());
+    inputStream.seek(0);
+
+    assertAvailable(0, inputStream);
+    for (int i = 0; i < fileSize; i++) {
+      assertEquals(i, inputStream.read());
+    }
+
+    for (int i = 0; i < fileSize; i++) {
+      inputStream.seek(i);
+      for (int j = i; j < fileSize; j++) {
+        assertEquals(j, inputStream.read());
+      }
+
+      int seekFromEndPos = fileSize - i - 1;
+      inputStream.seek(seekFromEndPos);
+      for (int j = seekFromEndPos; j < fileSize; j++) {
+        assertEquals(j, inputStream.read());
+      }
+    }
+  }
+
+  @Test
+  public void testClose() throws Exception {
+    S3ARemoteInputStream inputStream =
+        S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket",
+            "key", 9);
+    testCloseHelper(inputStream, 9);
+
+    inputStream =
+        S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key",
+            9, 5, 3);
+    testCloseHelper(inputStream, 5);
+  }
+
+  private void testCloseHelper(S3ARemoteInputStream inputStream, int bufferSize)
+      throws Exception {
+    assertAvailable(0, inputStream);
+    assertEquals(0, inputStream.read());
+    assertEquals(1, inputStream.read());
+    assertAvailable(bufferSize - 2, inputStream);
+
+    inputStream.close();
+
+    ExceptionAsserts.assertThrows(
+        IOException.class,
+        FSExceptionMessages.STREAM_IS_CLOSED,
+        () -> inputStream.available());
+
+    ExceptionAsserts.assertThrows(
+        IOException.class,
+        FSExceptionMessages.STREAM_IS_CLOSED,
+        () -> inputStream.read());
+
+    byte[] buffer = new byte[10];
+    ExceptionAsserts.assertThrows(
+        IOException.class,
+        FSExceptionMessages.STREAM_IS_CLOSED,
+        () -> inputStream.read(buffer));
+
+    // Verify a second close() does not throw.
+    inputStream.close();
+  }
+
+  private static void assertAvailable(int expected, InputStream inputStream)
+      throws IOException {
+    assertThat(inputStream.available())
+        .describedAs("Check available bytes on stream %s", inputStream)
+        .isEqualTo(expected);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java
new file mode 100644
index 0000000000000..b3788aac80834
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.junit.Test;
+
+import org.apache.hadoop.fs.impl.prefetch.ExceptionAsserts;
+import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AReadOpContext;
+import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
+import org.apache.hadoop.fs.s3a.impl.ChangeTracker;
+import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+public class TestS3ARemoteObject extends AbstractHadoopTestBase {
+
+  private final ExecutorService threadPool = Executors.newFixedThreadPool(1);
+
+  private final ExecutorServiceFuturePool futurePool =
+      new ExecutorServiceFuturePool(threadPool);
+
+  private final S3AInputStream.InputStreamCallbacks client =
+      MockS3ARemoteObject.createClient("bucket");
+
+  @Test
+  public void testArgChecks() throws Exception {
+    S3AReadOpContext readContext =
+        S3APrefetchFakes.createReadContext(futurePool, "key", 10, 10, 1);
+    S3ObjectAttributes attrs =
+        S3APrefetchFakes.createObjectAttributes("bucket", "key", 10);
+    S3AInputStreamStatistics stats =
+        readContext.getS3AStatisticsContext().newInputStreamStatistics();
+    ChangeTracker changeTracker =
+        S3APrefetchFakes.createChangeTracker("bucket", "key", 10);
+
+    // Should not throw.
+    new S3ARemoteObject(readContext, attrs, client, stats, changeTracker);
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'context' must not be null",
+        () -> new S3ARemoteObject(null, attrs, client, stats, changeTracker));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'s3Attributes' must not be null",
+        () -> new S3ARemoteObject(readContext, null, client, stats,
+            changeTracker));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'client' must not be null",
+        () -> new S3ARemoteObject(readContext, attrs, null, stats,
+            changeTracker));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'streamStatistics' must not be null",
+        () -> new S3ARemoteObject(readContext, attrs, client, null,
+            changeTracker));
+
+    ExceptionAsserts.assertThrows(
+        IllegalArgumentException.class,
+        "'changeTracker' must not be null",
+        () -> new S3ARemoteObject(readContext, attrs, client, stats, null));
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java
new file mode 100644
index 0000000000000..db70c4f22bce9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.prefetch;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.junit.Assert.assertEquals;
+
+public class TestS3ARemoteObjectReader extends AbstractHadoopTestBase {
+
+  private static final int FILE_SIZE = 9;
+
+  private static final int BUFFER_SIZE = 2;
+
+  private final S3ARemoteObject remoteObject =
+      new MockS3ARemoteObject(FILE_SIZE, false);
+
+  @Test
+  public void testArgChecks() throws Exception {
+    // Should not throw.
+    S3ARemoteObjectReader reader = new S3ARemoteObjectReader(remoteObject);
+
+    // Verify it throws correctly.
+
+    intercept(
+        IllegalArgumentException.class,
+        "'remoteObject' must not be null",
+        () -> new S3ARemoteObjectReader(null));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'buffer' must not be null",
+        () -> reader.read(null, 10, 2));
+
+    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
+
+    intercept(
+        IllegalArgumentException.class,
+        "'offset' (-1) must be within the range [0, 9]",
+        () -> reader.read(buffer, -1, 2));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'offset' (11) must be within the range [0, 9]",
+        () -> reader.read(buffer, 11, 2));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> reader.read(buffer, 1, 0));
+
+    intercept(
+        IllegalArgumentException.class,
+        "'size' must be a positive integer",
+        () -> reader.read(buffer, 1, -1));
+  }
+
+  @Test
+  public void testGetWithOffset() throws Exception {
+    for (int i = 0; i < FILE_SIZE; i++) {
+      testGetHelper(false, i);  // no retry
+      testGetHelper(true, i);   // with retry
+    }
+  }
+
+  private void testGetHelper(boolean testWithRetry, long startOffset)
+      throws Exception {
+    int numBlocks = 0;
+    ByteBuffer buffer;
+    S3ARemoteObjectReader reader =
+        new S3ARemoteObjectReader(
+            new MockS3ARemoteObject(FILE_SIZE, testWithRetry));
+    int remainingSize = FILE_SIZE - (int) startOffset;
+    for (int bufferSize = 0; bufferSize <= FILE_SIZE + 1; bufferSize++) {
+      buffer = ByteBuffer.allocate(bufferSize);
+      for (int readSize = 1; readSize <= FILE_SIZE; readSize++) {
+        buffer.clear();
+        int numBytesRead = reader.read(buffer, startOffset, readSize);
+        int expectedNumBytesRead = Math.min(readSize, remainingSize);
+        expectedNumBytesRead = Math.min(bufferSize, expectedNumBytesRead);
+        assertEquals(expectedNumBytesRead, numBytesRead);
+
+        byte[] bytes = buffer.array();
+        for (int i = 0; i < expectedNumBytesRead; i++) {
+          assertEquals(startOffset + i, bytes[i]);
+        }
+      }
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index dddd84b9103be..95f22644c91ab 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -18,50 +18,26 @@
 
 package org.apache.hadoop.fs.s3a.s3guard;
 
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
 
-import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.UnknownStoreException;
-import org.apache.hadoop.util.StopWatch;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileSystem;
 import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.fs.s3a.UnknownStoreException;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
+
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.BucketInfo.IS_MARKER_AWARE;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_S3GUARD_UNSUPPORTED;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.INVALID_ARGUMENT;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.UNSUPPORTED_COMMANDS;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.runS3GuardCommand;
 import static org.apache.hadoop.fs.s3a.tools.MarkerTool.MARKERS;
@@ -73,31 +49,14 @@
  */
 public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
 
-  protected static final String OWNER = "hdfs";
-  protected static final String DYNAMODB_TABLE = "ireland-team";
   protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST
       = "s3a://this-bucket-does-not-exist-00000000000";
 
-  private static final int PRUNE_MAX_AGE_SECS = 2;
-
-  private MetadataStore ms;
-  private S3AFileSystem rawFs;
-
   /**
    * List of tools to close in test teardown.
    */
   private final List<S3GuardTool> toolsToClose = new ArrayList<>();
 
-  /**
-   * The test timeout is increased in case previous tests have created
-   * many tombstone markers which now need to be purged.
-   * @return the test timeout.
-   */
-  @Override
-  protected int getTestTimeoutMillis() {
-    return SCALE_TEST_TIMEOUT_SECONDS * 1000;
-  }
-
   /**
    * Declare that the tool is to be closed in teardown.
    * @param tool tool to close
@@ -173,272 +132,23 @@ protected void runToFailure(int status, Object... args)
     }
   }
 
-  protected MetadataStore getMetadataStore() {
-    return ms;
-  }
-
   @Override
   public void setup() throws Exception {
     super.setup();
-    S3ATestUtils.assumeS3GuardState(true, getConfiguration());
-    S3AFileSystem fs = getFileSystem();
-    ms = fs.getMetadataStore();
-
-    // Also create a "raw" fs without any MetadataStore configured
-    Configuration conf = new Configuration(getConfiguration());
-    clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-    URI fsUri = fs.getUri();
-    S3AUtils.setBucketOption(conf,fsUri.getHost(),
-        S3_METADATA_STORE_IMPL,
-        S3GUARD_METASTORE_NULL);
-    rawFs = (S3AFileSystem) FileSystem.newInstance(fsUri, conf);
   }
 
   @Override
   public void teardown() throws Exception {
     super.teardown();
     toolsToClose.forEach(t -> IOUtils.cleanupWithLogger(LOG, t));
-    IOUtils.cleanupWithLogger(LOG, ms);
-    IOUtils.closeStream(rawFs);
-  }
-
-  protected void mkdirs(Path path, boolean onS3, boolean onMetadataStore)
-      throws IOException {
-    Preconditions.checkArgument(onS3 || onMetadataStore);
-    // getFileSystem() returns an fs with MetadataStore configured
-    S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
-    if (onS3) {
-      fs.mkdirs(path);
-    } else if (onMetadataStore) {
-      S3AFileStatus status = new S3AFileStatus(true, path, OWNER);
-      ms.put(new PathMetadata(status), null);
-    }
-  }
-
-  protected static void putFile(MetadataStore ms, S3AFileStatus f)
-      throws IOException {
-    assertNotNull(f);
-    try (BulkOperationState bulkWrite =
-             ms.initiateBulkWrite(
-                 BulkOperationState.OperationType.Put,
-                 f.getPath())) {
-      ms.put(new PathMetadata(f), bulkWrite);
-      Path parent = f.getPath().getParent();
-      while (parent != null) {
-        S3AFileStatus dir = new S3AFileStatus(false, parent, f.getOwner());
-        ms.put(new PathMetadata(dir), bulkWrite);
-        parent = parent.getParent();
-      }
-    }
-  }
-
-  /**
-   * Create file either on S3 or in metadata store.
-   * @param path the file path.
-   * @param onS3 set to true to create the file on S3.
-   * @param onMetadataStore set to true to create the file on the
-   *                        metadata store.
-   * @throws IOException IO problem
-   */
-  protected void createFile(Path path, boolean onS3, boolean onMetadataStore)
-      throws IOException {
-    Preconditions.checkArgument(onS3 || onMetadataStore);
-    // getFileSystem() returns an fs with MetadataStore configured
-    S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
-    if (onS3) {
-      ContractTestUtils.touch(fs, path);
-    } else if (onMetadataStore) {
-      S3AFileStatus status = new S3AFileStatus(100L, System.currentTimeMillis(),
-          fs.makeQualified(path), 512L, "hdfs", null, null);
-      putFile(ms, status);
-    }
-  }
-
-  /**
-   * Attempt to test prune() with sleep() without having flaky tests
-   * when things run slowly. Test is basically:
-   * 1. Set max path age to X seconds
-   * 2. Create some files (which writes entries to MetadataStore)
-   * 3. Sleep X+2 seconds (all files from above are now "stale")
-   * 4. Create some other files (these are "fresh").
-   * 5. Run prune on MetadataStore.
-   * 6. Assert that only files that were created before the sleep() were pruned.
-   *
-   * Problem is: #6 can fail if X seconds elapse between steps 4 and 5, since
-   * the newer files also become stale and get pruned.  This is easy to
-   * reproduce by running all integration tests in parallel with a ton of
-   * threads, or anything else that slows down execution a lot.
-   *
-   * Solution: Keep track of time elapsed between #4 and #5, and if it
-   * exceeds X, just print a warn() message instead of failing.
-   *
-   * @param cmdConf configuration for command
-   * @param parent path
-   * @param args command args
-   * @throws Exception
-   */
-  private void testPruneCommand(Configuration cmdConf, Path parent,
-      String...args) throws Exception {
-    Path keepParent = path("prune-cli-keep");
-    StopWatch timer = new StopWatch();
-    final S3AFileSystem fs = getFileSystem();
-    S3GuardTool.Prune cmd = toClose(new S3GuardTool.Prune(cmdConf));
-    cmd.setMetadataStore(ms);
-    try {
-
-      fs.mkdirs(parent);
-      fs.mkdirs(keepParent);
-      createFile(new Path(parent, "stale"), true, true);
-      createFile(new Path(keepParent, "stale-to-keep"), true, true);
-
-      Thread.sleep(TimeUnit.SECONDS.toMillis(PRUNE_MAX_AGE_SECS + 2));
-
-      timer.start();
-      createFile(new Path(parent, "fresh"), true, true);
-
-      assertMetastoreListingCount(parent, "Children count before pruning", 2);
-      exec(cmd, args);
-      long msecElapsed = timer.now(TimeUnit.MILLISECONDS);
-      if (msecElapsed >= PRUNE_MAX_AGE_SECS * 1000) {
-        LOG.warn("Skipping an assertion: Test running too slowly ({} msec)",
-            msecElapsed);
-      } else {
-        assertMetastoreListingCount(parent, "Pruned children count remaining",
-            1);
-      }
-      assertMetastoreListingCount(keepParent,
-          "This child should have been kept (prefix restriction).", 1);
-    } finally {
-      fs.delete(parent, true);
-      fs.delete(keepParent, true);
-      ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME,
-          Long.MAX_VALUE,
-          fs.pathToKey(parent));
-      ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME,
-          Long.MAX_VALUE,
-          fs.pathToKey(keepParent));
-      // reset the store before we close the tool.
-      cmd.setMetadataStore(new NullMetadataStore());
-    }
-  }
-
-  private void assertMetastoreListingCount(Path parent,
-      String message,
-      int expected) throws IOException {
-    Collection<PathMetadata> listing = ms.listChildren(parent).getListing();
-    assertEquals(message +" [" + StringUtils.join(", ", listing) + "]",
-        expected, listing.size());
-  }
-
-  @Test
-  public void testPruneCommandCLI() throws Exception {
-    Path testPath = path("testPruneCommandCLI");
-    testPruneCommand(getFileSystem().getConf(), testPath,
-        "prune", "-seconds", String.valueOf(PRUNE_MAX_AGE_SECS),
-        testPath.toString());
-  }
-
-  @Test
-  public void testPruneCommandTombstones() throws Exception {
-    Path testPath = path("testPruneCommandTombstones");
-    getFileSystem().mkdirs(testPath);
-    getFileSystem().delete(testPath, true);
-    S3GuardTool.Prune cmd = toClose(
-        new S3GuardTool.Prune(getFileSystem().getConf()));
-    cmd.setMetadataStore(ms);
-    try {
-      exec(cmd,
-          "prune", "-" + S3GuardTool.Prune.TOMBSTONE,
-          "-seconds", "0",
-          testPath.toString());
-      assertNotNull("Command did not create a filesystem",
-          cmd.getFilesystem());
-    } finally {
-      // reset the store before we close the tool.
-      cmd.setMetadataStore(new NullMetadataStore());
-    }
-  }
-
-  /**
-   * HADOOP-16457. In certain cases prune doesn't create an FS.
-   */
-  @Test
-  public void testMaybeInitFilesystem() throws Exception {
-    Path testPath = path("maybeInitFilesystem");
-    try (S3GuardTool.Prune cmd =
-             new S3GuardTool.Prune(getFileSystem().getConf())) {
-      cmd.maybeInitFilesystem(Collections.singletonList(testPath.toString()));
-      assertNotNull("Command did not create a filesystem",
-          cmd.getFilesystem());
-    }
-  }
-
-  /**
-   * HADOOP-16457. In certain cases prune doesn't create an FS.
-   */
-  @Test
-  public void testMaybeInitFilesystemNoPath() throws Exception {
-    try (S3GuardTool.Prune cmd = new S3GuardTool.Prune(
-        getFileSystem().getConf())) {
-      cmd.maybeInitFilesystem(Collections.emptyList());
-      assertNull("Command should not have created a filesystem",
-          cmd.getFilesystem());
-    }
-  }
-
-  @Test
-  public void testPruneCommandNoPath() throws Exception {
-    runToFailure(INVALID_ARGUMENT,
-        S3GuardTool.Prune.NAME,
-        "-" + S3GuardTool.Prune.TOMBSTONE,
-        "-seconds", "0");
-  }
-
-  @Test
-  public void testPruneCommandConf() throws Exception {
-    getConfiguration().setLong(Constants.S3GUARD_CLI_PRUNE_AGE,
-        TimeUnit.SECONDS.toMillis(PRUNE_MAX_AGE_SECS));
-    Path testPath = path("testPruneCommandConf");
-    testPruneCommand(getConfiguration(), testPath,
-        "prune", testPath.toString());
-  }
-
-  @Test
-  public void testSetCapacityFailFastOnReadWriteOfZero() throws Exception{
-    Configuration conf = getConfiguration();
-    String bucket = getFileSystem().getBucket();
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, getFileSystem().getBucket());
-
-    S3GuardTool.SetCapacity cmdR = toClose(new S3GuardTool.SetCapacity(conf));
-    String[] argsR =
-        new String[]{cmdR.getName(), "-read", "0", "s3a://" + bucket};
-    intercept(IllegalArgumentException.class,
-        S3GuardTool.SetCapacity.READ_CAP_INVALID, () -> cmdR.run(argsR));
-
-    S3GuardTool.SetCapacity cmdW = toClose(new S3GuardTool.SetCapacity(conf));
-    String[] argsW =
-        new String[]{cmdW.getName(), "-write", "0", "s3a://" + bucket};
-    intercept(IllegalArgumentException.class,
-        S3GuardTool.SetCapacity.WRITE_CAP_INVALID, () -> cmdW.run(argsW));
   }
 
   @Test
   public void testBucketInfoUnguarded() throws Exception {
     final Configuration conf = getConfiguration();
     URI fsUri = getFileSystem().getUri();
-    conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
-    String bucket = fsUri.getHost();
-    clearBucketOption(conf, bucket,
-        S3GUARD_DDB_TABLE_CREATE_KEY);
-    clearBucketOption(conf, bucket, S3_METADATA_STORE_IMPL);
-    clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_NAME_KEY);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
-        "testBucketInfoUnguarded-" + UUID.randomUUID());
-
-    // run a bucket info command and look for
-    // confirmation that it got the output from DDB diags
+
+    // run a bucket info command
     S3GuardTool.BucketInfo infocmd = toClose(new S3GuardTool.BucketInfo(conf));
     String info = exec(infocmd, S3GuardTool.BucketInfo.NAME,
         "-" + S3GuardTool.BucketInfo.UNGUARDED_FLAG,
@@ -457,8 +167,7 @@ public void testBucketInfoMarkerAware() throws Throwable {
     final Configuration conf = getConfiguration();
     URI fsUri = getFileSystem().getUri();
 
-    // run a bucket info command and look for
-    // confirmation that it got the output from DDB diags
+    // run a bucket info command
     S3GuardTool.BucketInfo infocmd = toClose(new S3GuardTool.BucketInfo(conf));
     String info = exec(infocmd, S3GuardTool.BucketInfo.NAME,
         "-" + MARKERS, S3GuardTool.BucketInfo.MARKERS_AWARE,
@@ -477,45 +186,14 @@ public void testBucketInfoMarkerPolicyUnknown() throws Throwable {
     final Configuration conf = getConfiguration();
     URI fsUri = getFileSystem().getUri();
 
-    // run a bucket info command and look for
-    // confirmation that it got the output from DDB diags
+    // run a bucket info command and expect failure
     S3GuardTool.BucketInfo infocmd = toClose(new S3GuardTool.BucketInfo(conf));
-    intercept(ExitUtil.ExitException.class, ""+ EXIT_NOT_ACCEPTABLE, () ->
+    intercept(ExitUtil.ExitException.class, "" + EXIT_NOT_ACCEPTABLE, () ->
         exec(infocmd, S3GuardTool.BucketInfo.NAME,
             "-" + MARKERS, "unknown",
             fsUri.toString()));
   }
 
-  @Test
-  public void testSetCapacityFailFastIfNotGuarded() throws Exception{
-    Configuration conf = getConfiguration();
-    bindToNonexistentTable(conf);
-    String bucket = rawFs.getBucket();
-    clearBucketOption(conf, bucket, S3_METADATA_STORE_IMPL);
-    clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_NAME_KEY);
-    clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_CREATE_KEY);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
-
-    S3GuardTool.SetCapacity cmdR = toClose(new S3GuardTool.SetCapacity(conf));
-    String[] argsR = new String[]{
-        cmdR.getName(),
-        "s3a://" + getFileSystem().getBucket()
-    };
-
-    intercept(IllegalStateException.class, "unguarded",
-        () -> cmdR.run(argsR));
-  }
-
-  /**
-   * Binds the configuration to a nonexistent table.
-   * @param conf
-   */
-  private void bindToNonexistentTable(final Configuration conf) {
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
-    conf.unset(S3GUARD_DDB_REGION_KEY);
-    conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false);
-  }
-
   /**
    * Make an S3GuardTool of the specific subtype with binded configuration
    * to a nonexistent table.
@@ -524,20 +202,15 @@ private void bindToNonexistentTable(final Configuration conf) {
   private S3GuardTool makeBindedTool(Class<? extends S3GuardTool> tool)
       throws Exception {
     Configuration conf = getConfiguration();
-    // set a table as a safety check in case the test goes wrong
-    // and deletes it.
-    bindToNonexistentTable(conf);
     return tool.getDeclaredConstructor(Configuration.class).newInstance(conf);
   }
 
   @Test
   public void testToolsNoBucket() throws Throwable {
     List<Class<? extends S3GuardTool>> tools =
-        Arrays.asList(S3GuardTool.Destroy.class, S3GuardTool.BucketInfo.class,
-            S3GuardTool.Diff.class, S3GuardTool.Import.class,
-            S3GuardTool.Prune.class, S3GuardTool.SetCapacity.class,
-            S3GuardTool.Uploads.class,
-            S3GuardTool.Authoritative.class);
+        Arrays.asList(
+            S3GuardTool.BucketInfo.class,
+            S3GuardTool.Uploads.class);
 
     for (Class<? extends S3GuardTool> tool : tools) {
       S3GuardTool cmdR = makeBindedTool(tool);
@@ -551,27 +224,12 @@ public void testToolsNoBucket() throws Throwable {
     }
   }
 
-  @Test
-  public void testToolsNoArgsForBucketAndDDBTable() throws Throwable {
-    List<Class<? extends S3GuardTool>> tools =
-        Arrays.asList(S3GuardTool.Destroy.class, S3GuardTool.Init.class);
-
-    for (Class<? extends S3GuardTool> tool : tools) {
-      S3GuardTool cmdR = makeBindedTool(tool);
-      describe("Calling " + cmdR.getName() + " without any arguments.");
-      intercept(ExitUtil.ExitException.class,
-          "S3 bucket url or DDB table name have to be provided explicitly",
-          () -> cmdR.run(new String[]{tool.getName()}));
-    }
-  }
-
   @Test
   public void testToolsNoArgsForBucket() throws Throwable {
     List<Class<? extends S3GuardTool>> tools =
-        Arrays.asList(S3GuardTool.BucketInfo.class, S3GuardTool.Diff.class,
-            S3GuardTool.Import.class, S3GuardTool.Prune.class,
-            S3GuardTool.SetCapacity.class, S3GuardTool.Uploads.class,
-            S3GuardTool.Authoritative.class);
+        Arrays.asList(
+            S3GuardTool.BucketInfo.class,
+            S3GuardTool.Uploads.class);
 
     for (Class<? extends S3GuardTool> tool : tools) {
       S3GuardTool cmdR = makeBindedTool(tool);
@@ -582,6 +240,15 @@ public void testToolsNoArgsForBucket() throws Throwable {
     }
   }
 
+  @Test
+  public void testUnsupported() throws Throwable {
+    describe("Verify the unsupported tools are rejected");
+    for (String tool : UNSUPPORTED_COMMANDS) {
+      describe("Probing %s", tool);
+      runToFailure(E_S3GUARD_UNSUPPORTED, tool);
+    }
+  }
+
   @Test
   public void testProbeForMagic() throws Throwable {
     S3AFileSystem fs = getFileSystem();
@@ -589,7 +256,7 @@ public void testProbeForMagic() throws Throwable {
     S3GuardTool.BucketInfo cmd = new S3GuardTool.BucketInfo(
         getConfiguration());
     // this must always work
-      exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name);
+    exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name);
   }
 
   /**
@@ -606,86 +273,4 @@ protected void assertExitCode(final int expectedErrorCode,
     }
   }
 
-  @Test
-  public void testDestroyFailsIfNoBucketNameOrDDBTableSet()
-      throws Exception {
-    intercept(ExitUtil.ExitException.class,
-        () -> run(S3GuardTool.Destroy.NAME));
-  }
-
-  @Test
-  public void testInitFailsIfNoBucketNameOrDDBTableSet() throws Exception {
-    intercept(ExitUtil.ExitException.class,
-        () -> run(S3GuardTool.Init.NAME));
-  }
-
-  @Test
-  public void
-  testDiffCommand() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    ms = getMetadataStore();
-    Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
-    Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
-
-    Path testPath = path("test-diff");
-    // clean up through the store and behind it.
-    fs.delete(testPath, true);
-    rawFs.delete(testPath, true);
-    mkdirs(testPath, true, true);
-
-    Path msOnlyPath = new Path(testPath, "ms_only");
-    mkdirs(msOnlyPath, false, true);
-    filesOnMS.add(msOnlyPath);
-    for (int i = 0; i < 5; i++) {
-      Path file = new Path(msOnlyPath, String.format("file-%d", i));
-      createFile(file, false, true);
-      filesOnMS.add(file);
-    }
-
-    Path s3OnlyPath = new Path(testPath, "s3_only");
-    mkdirs(s3OnlyPath, true, false);
-    filesOnS3.add(s3OnlyPath);
-    for (int i = 0; i < 5; i++) {
-      Path file = new Path(s3OnlyPath, String.format("file-%d", i));
-      createFile(file, true, false);
-      filesOnS3.add(file);
-    }
-
-    ByteArrayOutputStream buf = new ByteArrayOutputStream();
-    S3GuardTool.Diff cmd = toClose(new S3GuardTool.Diff(fs.getConf()));
-    cmd.setStore(ms);
-    String table = "dynamo://" + getTestTableName(DYNAMODB_TABLE);
-    exec(0, "", cmd, buf, "diff", "-meta", table, testPath.toString());
-
-    Set<Path> actualOnS3 = new HashSet<>();
-    Set<Path> actualOnMS = new HashSet<>();
-    boolean duplicates = false;
-    try (BufferedReader reader =
-        new BufferedReader(new InputStreamReader(
-            new ByteArrayInputStream(buf.toByteArray())))) {
-      String line;
-      while ((line = reader.readLine()) != null) {
-        String[] fields = line.split("\\s");
-        assertEquals("[" + line + "] does not have enough fields",
-            4, fields.length);
-        String where = fields[0];
-        Path path = new Path(fields[3]);
-        if (S3GuardTool.Diff.S3_PREFIX.equals(where)) {
-          duplicates = duplicates || actualOnS3.contains(path);
-          actualOnS3.add(path);
-        } else if (S3GuardTool.Diff.MS_PREFIX.equals(where)) {
-          duplicates = duplicates || actualOnMS.contains(path);
-          actualOnMS.add(path);
-        } else {
-          fail("Unknown prefix: " + where);
-        }
-      }
-    }
-    String actualOut = buf.toString();
-    assertEquals("Mismatched metadata store outputs: " + actualOut,
-        filesOnMS, actualOnMS);
-    assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
-    assertFalse("Diff contained duplicates", duplicates);
-  }
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DDBCapacities.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DDBCapacities.java
deleted file mode 100644
index 3f1e99061b84e..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DDBCapacities.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.Map;
-import java.util.Objects;
-
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
-import org.junit.Assert;
-
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.READ_CAPACITY;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.WRITE_CAPACITY;
-
-/**
- * Tuple of read and write capacity of a DDB table.
- */
-class DDBCapacities {
-  private final long read, write;
-
-  DDBCapacities(long read, long write) {
-    this.read = read;
-    this.write = write;
-  }
-
-  public long getRead() {
-    return read;
-  }
-
-  public long getWrite() {
-    return write;
-  }
-
-  String getReadStr() {
-    return Long.toString(read);
-  }
-
-  String getWriteStr() {
-    return Long.toString(write);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    DDBCapacities that = (DDBCapacities) o;
-    return read == that.read && write == that.write;
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(read, write);
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder sb = new StringBuilder("Capacities{");
-    sb.append("read=").append(read);
-    sb.append(", write=").append(write);
-    sb.append('}');
-    return sb.toString();
-  }
-
-  /**
-   * Is the the capacity that of an On-Demand table?
-   * @return true if the capacities are both 0.
-   */
-  public boolean isOnDemandTable() {
-    return read == 0 && write == 0;
-  }
-
-  /**
-   * Given a diagnostics map from a DDB store, extract the capacities.
-   * @param diagnostics diagnostics map to examine.
-   * @return the capacities
-   * @throws AssertionError if the fields are missing.
-   */
-  public static DDBCapacities extractCapacities(
-      final Map<String, String> diagnostics) {
-    String read = diagnostics.get(READ_CAPACITY);
-    Assert.assertNotNull("No " + READ_CAPACITY + " attribute in diagnostics",
-        read);
-    return new DDBCapacities(
-        Long.parseLong(read),
-        Long.parseLong(diagnostics.get(WRITE_CAPACITY)));
-  }
-
-  /**
-   * Given a throughput information from table.describe(), build
-   * a DDBCapacities object.
-   * @param throughput throughput description.
-   * @return the capacities
-   */
-  public static DDBCapacities extractCapacities(
-      ProvisionedThroughputDescription throughput) {
-    return new DDBCapacities(throughput.getReadCapacityUnits(),
-        throughput.getWriteCapacityUnits());
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
deleted file mode 100644
index 93a9e33aaaf48..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
+++ /dev/null
@@ -1,1543 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import com.amazonaws.services.dynamodbv2.model.SSEDescription;
-import com.amazonaws.services.dynamodbv2.model.TableDescription;
-import com.amazonaws.services.dynamodbv2.model.Tag;
-import com.amazonaws.services.dynamodbv2.model.TagResourceRequest;
-import com.amazonaws.services.dynamodbv2.model.UntagResourceRequest;
-import org.apache.hadoop.util.Lists;
-import org.assertj.core.api.Assertions;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3ATestConstants;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.DurationInfo;
-
-import org.junit.AfterClass;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import static java.lang.String.valueOf;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_INCOMPATIBLE_ITEM_VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_INCOMPATIBLE_TAG_VERSION;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.E_NO_VERSION_MARKER_AND_NOT_EMPTY;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.getVersionMarkerFromTags;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
-import static org.apache.hadoop.test.LambdaTestUtils.*;
-
-/**
- * Test that {@link DynamoDBMetadataStore} implements {@link MetadataStore}.
- *
- * In this integration test, we use a real AWS DynamoDB. A
- * {@link DynamoDBMetadataStore} object is created in the @BeforeClass method,
- * and shared for all test in the @BeforeClass method. You will be charged
- * bills for AWS S3 or DynamoDB when you run these tests.
- *
- * According to the base class, every test case will have independent contract
- * to create a new {@link S3AFileSystem} instance and initializes it.
- * A table will be created and shared between the tests; some tests also
- * create their own.
- *
- * Important: Any new test which creates a table must do the following
- * <ol>
- *   <li>Enable on-demand pricing.</li>
- *   <li>Always destroy the table, even if an assertion fails.</li>
- * </ol>
- * This is needed to avoid "leaking" DDB tables and running up bills.
- */
-public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
-
-  public static final int MINUTE = 60_000;
-
-  public ITestDynamoDBMetadataStore() {
-    super();
-  }
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ITestDynamoDBMetadataStore.class);
-  public static final PrimaryKey
-      VERSION_MARKER_PRIMARY_KEY = createVersionMarkerPrimaryKey(
-      DynamoDBMetadataStore.VERSION_MARKER_ITEM_NAME);
-
-  private S3AFileSystem fileSystem;
-  private S3AContract s3AContract;
-  private DynamoDBMetadataStoreTableManager tableHandler;
-
-  private URI fsUri;
-
-  private String bucket;
-
-  @SuppressWarnings("StaticNonFinalField")
-  private static DynamoDBMetadataStore ddbmsStatic;
-
-  @SuppressWarnings("StaticNonFinalField")
-  private static String testDynamoDBTableName;
-
-  private static final List<Path> UNCHANGED_ENTRIES = Collections.emptyList();
-
-  /**
-   * Create a path under the test path provided by
-   * the FS contract.
-   * @param filepath path string in
-   * @return a path qualified by the test filesystem
-   */
-  protected Path path(String filepath) {
-    return getFileSystem().makeQualified(
-        new Path(s3AContract.getTestPath(), filepath));
-  }
-
-  @Override
-  public void setUp() throws Exception {
-    Configuration conf = prepareTestConfiguration(new Configuration());
-    assumeThatDynamoMetadataStoreImpl(conf);
-    Assume.assumeTrue("Test DynamoDB table name should be set to run "
-            + "integration tests.", testDynamoDBTableName != null);
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
-    enableOnDemand(conf);
-    s3AContract = new S3AContract(conf);
-    s3AContract.init();
-
-    fileSystem = (S3AFileSystem) s3AContract.getTestFileSystem();
-    assume("No test filesystem", s3AContract.isEnabled());
-    assertNotNull("No test filesystem", fileSystem);
-    fsUri = fileSystem.getUri();
-    bucket = fileSystem.getBucket();
-
-    try{
-      super.setUp();
-    } catch (FileNotFoundException e){
-      LOG.warn("MetadataStoreTestBase setup failed. Waiting for table to be "
-          + "deleted before trying again.", e);
-      try {
-        ddbmsStatic.getTable().waitForDelete();
-      } catch (IllegalArgumentException | InterruptedException ex) {
-        LOG.warn("When awaiting a table to be cleaned up", e);
-      }
-      super.setUp();
-    }
-    tableHandler = getDynamoMetadataStore().getTableHandler();
-  }
-
-  @BeforeClass
-  public static void beforeClassSetup() throws IOException {
-    Configuration conf = prepareTestConfiguration(new Configuration());
-    assumeThatDynamoMetadataStoreImpl(conf);
-    // S3GUARD_DDB_TEST_TABLE_NAME_KEY and S3GUARD_DDB_TABLE_NAME_KEY should
-    // be configured to use this test.
-    testDynamoDBTableName = conf.get(
-        S3ATestConstants.S3GUARD_DDB_TEST_TABLE_NAME_KEY);
-    String dynamoDbTableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY);
-    Assume.assumeTrue("No DynamoDB table name configured in "
-            + S3GUARD_DDB_TABLE_NAME_KEY,
-        !StringUtils.isEmpty(dynamoDbTableName));
-
-    // We should assert that the table name is configured, so the test should
-    // fail if it's not configured.
-    assertNotNull("Test DynamoDB table name '"
-        + S3ATestConstants.S3GUARD_DDB_TEST_TABLE_NAME_KEY + "'"
-        + " should be set to run integration tests.",
-        testDynamoDBTableName);
-
-    // We should assert that the test table is not the same as the production
-    // table, as the test table could be modified and destroyed multiple
-    // times during the test.
-    assertNotEquals("Test DynamoDB table name: "
-            + "'" + S3ATestConstants.S3GUARD_DDB_TEST_TABLE_NAME_KEY + "'"
-            + " and production table name: "
-            + "'" + S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
-        testDynamoDBTableName, conf.get(S3GUARD_DDB_TABLE_NAME_KEY));
-
-    // We can use that table in the test if these assertions are valid
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
-
-    // remove some prune delays
-    conf.setInt(S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY, 0);
-
-    // clear all table tagging config before this test
-    conf.getPropsWithPrefix(S3GUARD_DDB_TABLE_TAG).keySet().forEach(
-        propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey)
-    );
-
-    // set the tags on the table so that it can be tested later.
-    Map<String, String> tagMap = createTagMap();
-    for (Map.Entry<String, String> tagEntry : tagMap.entrySet()) {
-      conf.set(S3GUARD_DDB_TABLE_TAG + tagEntry.getKey(), tagEntry.getValue());
-    }
-    LOG.debug("Creating static ddbms which will be shared between tests.");
-    enableOnDemand(conf);
-
-    ddbmsStatic = new DynamoDBMetadataStore();
-    ddbmsStatic.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-  }
-
-  @AfterClass
-  public static void afterClassTeardown() {
-    LOG.debug("Destroying static DynamoDBMetadataStore.");
-    destroy(ddbmsStatic);
-    ddbmsStatic = null;
-  }
-
-  /**
-   * Destroy and then close() a metastore instance.
-   * Exceptions are caught and logged at debug.
-   * @param ddbms store -may be null.
-   */
-  private static void destroy(final DynamoDBMetadataStore ddbms) {
-    if (ddbms != null) {
-      try {
-        ddbms.destroy();
-        IOUtils.closeStream(ddbms);
-      } catch (IOException e) {
-        LOG.debug("On ddbms shutdown", e);
-      }
-    }
-  }
-
-  private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
-    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
-        conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
-            Constants.S3GUARD_METASTORE_DYNAMO));
-  }
-
-  /**
-   * This teardown does not call super.teardown() so as to avoid the DDMBS table
-   * from being destroyed.
-   * <p>
-   * This is potentially quite slow, depending on DDB IO Capacity and number
-   * of entries to forget.
-   */
-  @Override
-  public void tearDown() throws Exception {
-    LOG.info("Removing data from ddbms table in teardown.");
-    Thread.currentThread().setName("Teardown");
-    // The following is a way to be sure the table will be cleared and there
-    // will be no leftovers after the test.
-    try {
-      deleteAllMetadata();
-    } finally {
-      IOUtils.cleanupWithLogger(LOG, fileSystem);
-    }
-  }
-
-  /**
-   * Forget all metadata in the store.
-   * This originally did an iterate and forget, but using prune() hands off the
-   * bulk IO into the metastore itself; the forgetting is used
-   * to purge anything which wasn't pruned.
-   */
-  private void deleteAllMetadata() throws IOException {
-    // The following is a way to be sure the table will be cleared and there
-    // will be no leftovers after the test.
-    // only executed if there is a filesystem, as failure during test setup
-    // means that strToPath will NPE.
-    if (getContract() != null && getContract().getFileSystem() != null) {
-      deleteMetadataUnderPath(ddbmsStatic, strToPath("/"), true);
-    }
-  }
-
-  /**
-   * Delete all metadata under a path.
-   * Attempt to use prune first as it scales slightly better.
-   * @param ms store
-   * @param path path to prune under
-   * @param suppressErrors should errors be suppressed?
-   * @throws IOException if there is a failure and suppressErrors == false
-   */
-  public static void deleteMetadataUnderPath(final DynamoDBMetadataStore ms,
-      final Path path, final boolean suppressErrors) throws IOException {
-    ThrottleTracker throttleTracker = new ThrottleTracker(ms);
-    int forgotten = 0;
-    try (DurationInfo ignored = new DurationInfo(LOG, true, "forget")) {
-      PathMetadata meta = ms.get(path);
-      if (meta != null) {
-        for (DescendantsIterator desc = new DescendantsIterator(ms,
-            meta);
-            desc.hasNext();) {
-          forgotten++;
-          ms.forgetMetadata(desc.next().getPath());
-        }
-        LOG.info("Forgot {} entries", forgotten);
-      }
-    } catch (FileNotFoundException fnfe) {
-      // there is no table.
-      return;
-    } catch (IOException ioe) {
-      LOG.warn("Failed to forget entries under {}", path, ioe);
-      if (!suppressErrors) {
-        throw ioe;
-      }
-    }
-    LOG.info("Throttle statistics: {}", throttleTracker);
-  }
-
-  @Override protected String getPathStringForPrune(String path)
-      throws Exception {
-    String b = getTestBucketName(getContract().getFileSystem().getConf());
-    return "/" + b + "/dir2";
-  }
-
-  /**
-   * Each contract has its own S3AFileSystem and DynamoDBMetadataStore objects.
-   */
-  private class DynamoDBMSContract extends AbstractMSContract {
-
-    DynamoDBMSContract(Configuration conf) {
-    }
-
-    DynamoDBMSContract() {
-      this(new Configuration());
-    }
-
-    @Override
-    public S3AFileSystem getFileSystem() {
-      return ITestDynamoDBMetadataStore.this.fileSystem;
-    }
-
-    @Override
-    public DynamoDBMetadataStore getMetadataStore() {
-      return ITestDynamoDBMetadataStore.ddbmsStatic;
-    }
-  }
-
-  @Override
-  public DynamoDBMSContract createContract() {
-    return new DynamoDBMSContract();
-  }
-
-  @Override
-  public DynamoDBMSContract createContract(Configuration conf) {
-    return new DynamoDBMSContract(conf);
-  }
-
-  @Override
-  protected S3AFileStatus basicFileStatus(Path path, int size, boolean isDir)
-      throws IOException {
-    String owner = UserGroupInformation.getCurrentUser().getShortUserName();
-    return isDir
-        ? new S3AFileStatus(true, path, owner)
-        : new S3AFileStatus(size, getModTime(), path, BLOCK_SIZE, owner,
-            null, null);
-  }
-
-  /**
-   * Create a directory status entry.
-   * @param dir directory.
-   * @return the status
-   */
-  private S3AFileStatus dirStatus(Path dir) throws IOException {
-    return basicFileStatus(dir, 0, true);
-  }
-
-  private DynamoDBMetadataStore getDynamoMetadataStore() throws IOException {
-    return (DynamoDBMetadataStore) getContract().getMetadataStore();
-  }
-
-  private S3AFileSystem getFileSystem() {
-    return this.fileSystem;
-  }
-
-  /**
-   * Force the configuration into DDB on demand, so that
-   * even if a test bucket isn't cleaned up, the cost is $0.
-   * @param conf configuration to patch.
-   */
-  public static void enableOnDemand(Configuration conf) {
-    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, 0);
-    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, 0);
-  }
-
-  /**
-   * Get the configuration needed to create a table; extracts
-   * it from the filesystem then always patches it to be on demand.
-   * Why the patch? It means even if a cached FS has brought in
-   * some provisioned values, they get reset.
-   * @return a new configuration
-   */
-  private Configuration getTableCreationConfig() {
-    Configuration conf = new Configuration(getFileSystem().getConf());
-    enableOnDemand(conf);
-    return conf;
-  }
-
-  /**
-   * This tests that after initialize() using an S3AFileSystem object, the
-   * instance should have been initialized successfully, and tables are ACTIVE.
-   */
-  @Test
-  public void testInitialize() throws IOException {
-    final S3AFileSystem s3afs = this.fileSystem;
-    final String tableName =
-        getTestTableName("testInitialize");
-    Configuration conf = getFileSystem().getConf();
-    enableOnDemand(conf);
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
-    try {
-      ddbms.initialize(s3afs, new S3Guard.TtlTimeProvider(conf));
-      Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
-      verifyTableSse(conf, table.getDescription());
-      assertNotNull(ddbms.getTable());
-      assertEquals(tableName, ddbms.getTable().getTableName());
-
-      String expectedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
-          s3afs.getBucketLocation(bucket));
-      assertEquals("DynamoDB table should be in configured region or the same" +
-              " region as S3 bucket",
-          expectedRegion,
-          ddbms.getRegion());
-    } finally {
-      destroy(ddbms);
-    }
-  }
-
-  /**
-   * This tests that after initialize() using a Configuration object, the
-   * instance should have been initialized successfully, and tables are ACTIVE.
-   */
-  @Test
-  public void testInitializeWithConfiguration() throws IOException {
-    final String tableName =
-        getTestTableName("testInitializeWithConfiguration");
-    final Configuration conf = getTableCreationConfig();
-    conf.unset(S3GUARD_DDB_TABLE_NAME_KEY);
-    String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
-        getFileSystem().getBucketLocation());
-    conf.unset(S3GUARD_DDB_REGION_KEY);
-    try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
-      ddbms.initialize(conf,  new S3Guard.TtlTimeProvider(conf));
-      fail("Should have failed because the table name is not set!");
-    } catch (IllegalArgumentException ignored) {
-    }
-
-    // config table name
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
-      ddbms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-      fail("Should have failed because as the region is not set!");
-    } catch (IllegalArgumentException ignored) {
-    }
-
-    // config region
-    conf.set(S3GUARD_DDB_REGION_KEY, savedRegion);
-    doTestInitializeWithConfiguration(conf, tableName);
-
-    // config table server side encryption (SSE)
-    conf.setBoolean(S3GUARD_DDB_TABLE_SSE_ENABLED, true);
-    doTestInitializeWithConfiguration(conf, tableName);
-  }
-
-  /**
-   * Test initialize() using a Configuration object successfully.
-   */
-  private void doTestInitializeWithConfiguration(Configuration conf,
-      String tableName) throws IOException {
-    DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
-    try {
-      ddbms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-      Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
-      verifyTableSse(conf, table.getDescription());
-      assertNotNull(ddbms.getTable());
-      assertEquals(tableName, ddbms.getTable().getTableName());
-      assertEquals("Unexpected key schema found!",
-          keySchema(),
-          ddbms.getTable().describe().getKeySchema());
-    } finally {
-      destroy(ddbms);
-    }
-  }
-
-  /**
-   * This should really drive a parameterized test run of 5^2 entries, but it
-   * would require a major refactoring to set things up.
-   * For now, each source test has its own entry, with the destination written
-   * to.
-   * This seems to be enough to stop DDB throttling from triggering test
-   * timeouts.
-   */
-  private static final int[] NUM_METAS_TO_DELETE_OR_PUT = {
-      -1, // null
-      0, // empty collection
-      1, // one path
-      S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT, // exact limit of a batch request
-      S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT + 1 // limit + 1
-  };
-
-  @Test
-  public void testBatchWrite00() throws IOException {
-    doBatchWriteForOneSet(0);
-  }
-
-  @Test
-  public void testBatchWrite01() throws IOException {
-    doBatchWriteForOneSet(1);
-  }
-
-  @Test
-  public void testBatchWrite02() throws IOException {
-    doBatchWriteForOneSet(2);
-  }
-
-  @Test
-  public void testBatchWrite03() throws IOException {
-    doBatchWriteForOneSet(3);
-  }
-
-  @Test
-  public void testBatchWrite04() throws IOException {
-    doBatchWriteForOneSet(4);
-  }
-
-  /**
-   * Test that for a large batch write request, the limit is handled correctly.
-   * With cleanup afterwards.
-   */
-  private void doBatchWriteForOneSet(int index) throws IOException {
-    for (int numNewMetas : NUM_METAS_TO_DELETE_OR_PUT) {
-      doTestBatchWrite(NUM_METAS_TO_DELETE_OR_PUT[index],
-          numNewMetas,
-          getDynamoMetadataStore());
-    }
-    // The following is a way to be sure the table will be cleared and there
-    // will be no leftovers after the test.
-    deleteMetadataUnderPath(ddbmsStatic, strToPath("/"), false);
-  }
-
-  /**
-   * Test that for a large batch write request, the limit is handled correctly.
-   */
-  private void doTestBatchWrite(int numDelete, int numPut,
-      DynamoDBMetadataStore ms) throws IOException {
-    Path path = new Path(
-        "/ITestDynamoDBMetadataStore_testBatchWrite_" + numDelete + '_'
-            + numPut);
-    final Path root = fileSystem.makeQualified(path);
-    final Path oldDir = new Path(root, "oldDir");
-    final Path newDir = new Path(root, "newDir");
-    LOG.info("doTestBatchWrite: oldDir={}, newDir={}", oldDir, newDir);
-    Thread.currentThread()
-        .setName(String.format("Bulk put=%d; delete=%d", numPut, numDelete));
-
-    AncestorState putState = checkNotNull(ms.initiateBulkWrite(
-        BulkOperationState.OperationType.Put, newDir),
-        "No state from initiateBulkWrite()");
-    ms.put(new PathMetadata(dirStatus(oldDir)), putState);
-    ms.put(new PathMetadata(dirStatus(newDir)), putState);
-
-    final List<PathMetadata> oldMetas = numDelete < 0 ? null :
-        new ArrayList<>(numDelete);
-    for (int i = 0; i < numDelete; i++) {
-      oldMetas.add(new PathMetadata(
-          basicFileStatus(new Path(oldDir, "child" + i), i, false)));
-    }
-    final List<PathMetadata> newMetas = numPut < 0 ? null :
-        new ArrayList<>(numPut);
-    for (int i = 0; i < numPut; i++) {
-      newMetas.add(new PathMetadata(
-          basicFileStatus(new Path(newDir, "child" + i), i, false)));
-    }
-
-    Collection<Path> pathsToDelete = null;
-    if (oldMetas != null) {
-      // put all metadata of old paths and verify
-      ms.put(new DirListingMetadata(oldDir, oldMetas, false), UNCHANGED_ENTRIES,
-          putState);
-      assertEquals("Child count",
-          0, ms.listChildren(newDir).withoutTombstones().numEntries());
-      Assertions.assertThat(ms.listChildren(oldDir).getListing())
-          .describedAs("Old Directory listing")
-          .containsExactlyInAnyOrderElementsOf(oldMetas);
-
-      assertTrue(CollectionUtils
-          .isEqualCollection(oldMetas, ms.listChildren(oldDir).getListing()));
-
-      pathsToDelete = new ArrayList<>(oldMetas.size());
-      for (PathMetadata meta : oldMetas) {
-        pathsToDelete.add(meta.getFileStatus().getPath());
-      }
-    }
-
-    // move the old paths to new paths and verify
-    AncestorState state = checkNotNull(ms.initiateBulkWrite(
-        BulkOperationState.OperationType.Put, newDir),
-        "No state from initiateBulkWrite()");
-    assertEquals("bulk write destination", newDir, state.getDest());
-
-    ThrottleTracker throttleTracker = new ThrottleTracker(ms);
-    try(DurationInfo ignored = new DurationInfo(LOG, true,
-        "Move")) {
-      ms.move(pathsToDelete, newMetas, state);
-    }
-    LOG.info("Throttle status {}", throttleTracker);
-    assertEquals("Number of children in source directory",
-        0, ms.listChildren(oldDir).withoutTombstones().numEntries());
-    if (newMetas != null) {
-      Assertions.assertThat(ms.listChildren(newDir).getListing())
-          .describedAs("Directory listing")
-          .containsAll(newMetas);
-      if (!newMetas.isEmpty()) {
-        Assertions.assertThat(state.size())
-            .describedAs("Size of ancestor state")
-            .isGreaterThan(newMetas.size());
-      }
-    }
-  }
-
-  @Test
-  public void testInitExistingTable() throws IOException {
-    final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
-    final String tableName = ddbms.getTable().getTableName();
-    verifyTableInitialized(tableName, ddbms.getDynamoDB());
-    // create existing table
-    tableHandler.initTable();
-    verifyTableInitialized(tableName, ddbms.getDynamoDB());
-  }
-
-  /**
-   * Test versioning handling.
-   * <ol>
-   *   <li>Create the table.</li>
-   *   <li>Verify tag propagation.</li>
-   *   <li>Delete the version marker -verify failure.</li>
-   *   <li>Reinstate a different version marker -verify failure</li>
-   * </ol>
-   * Delete the version marker and verify that table init fails.
-   * This also includes the checks for tagging, which goes against all
-   * principles of unit tests.
-   * However, merging the routines saves
-   */
-  @Test
-  public void testTableVersioning() throws Exception {
-    String tableName = getTestTableName("testTableVersionRequired");
-    Configuration conf = getTableCreationConfig();
-    int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
-        S3GUARD_DDB_MAX_RETRIES_DEFAULT);
-    conf.setInt(S3GUARD_DDB_MAX_RETRIES, 3);
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    tagConfiguration(conf);
-    DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
-    try {
-      ddbms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-      DynamoDBMetadataStoreTableManager localTableHandler =
-          ddbms.getTableHandler();
-
-      Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
-      // check the tagging
-      verifyStoreTags(createTagMap(), ddbms);
-      // check version compatibility
-      checkVerifyVersionMarkerCompatibility(localTableHandler, table);
-
-      conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
-    } finally {
-      destroy(ddbms);
-    }
-  }
-
-  private void checkVerifyVersionMarkerCompatibility(
-      DynamoDBMetadataStoreTableManager localTableHandler, Table table)
-      throws Exception {
-    final AmazonDynamoDB addb
-        = getDynamoMetadataStore().getAmazonDynamoDB();
-    Item originalVersionMarker = table.getItem(VERSION_MARKER_PRIMARY_KEY);
-
-    LOG.info("1/6: remove version marker and tags from table " +
-        "the table is empty, so it should be initialized after the call");
-    deleteVersionMarkerItem(table);
-    removeVersionMarkerTag(table, addb);
-    localTableHandler.initTable();
-
-    final int versionFromItem = extractVersionFromMarker(
-        localTableHandler.getVersionMarkerItem());
-    final int versionFromTag = extractVersionFromMarker(
-        getVersionMarkerFromTags(table, addb));
-    assertEquals("Table should be tagged with the right version.",
-        VERSION, versionFromTag);
-    assertEquals("Table should have the right version marker.",
-        VERSION, versionFromItem);
-
-    LOG.info("2/6: if the table is not empty and there's no version marker " +
-        "it should fail");
-    deleteVersionMarkerItem(table);
-    removeVersionMarkerTag(table, addb);
-    String testKey = "coffee";
-    Item wrongItem =
-        createVersionMarker(testKey, VERSION * 2, 0);
-    table.putItem(wrongItem);
-    intercept(IOException.class, E_NO_VERSION_MARKER_AND_NOT_EMPTY,
-        () -> localTableHandler.initTable());
-
-    LOG.info("3/6: table has only version marker item then it will be tagged");
-    table.putItem(originalVersionMarker);
-    localTableHandler.initTable();
-    final int versionFromTag2 = extractVersionFromMarker(
-        getVersionMarkerFromTags(table, addb));
-    assertEquals("Table should have the right version marker tag " +
-        "if there was a version item.", VERSION, versionFromTag2);
-
-    LOG.info("4/6: table has only version marker tag then the version marker " +
-        "item will be created.");
-    deleteVersionMarkerItem(table);
-    removeVersionMarkerTag(table, addb);
-    localTableHandler.tagTableWithVersionMarker();
-    localTableHandler.initTable();
-    final int versionFromItem2 = extractVersionFromMarker(
-        localTableHandler.getVersionMarkerItem());
-    assertEquals("Table should have the right version marker item " +
-        "if there was a version tag.", VERSION, versionFromItem2);
-
-    LOG.info("5/6: add a different marker tag to the table: init should fail");
-    deleteVersionMarkerItem(table);
-    removeVersionMarkerTag(table, addb);
-    Item v200 = createVersionMarker(VERSION_MARKER_ITEM_NAME, VERSION * 2, 0);
-    table.putItem(v200);
-    intercept(IOException.class, E_INCOMPATIBLE_ITEM_VERSION,
-        () -> localTableHandler.initTable());
-
-    LOG.info("6/6: add a different marker item to the table: init should fail");
-    deleteVersionMarkerItem(table);
-    removeVersionMarkerTag(table, addb);
-    int wrongVersion = VERSION + 3;
-    tagTableWithCustomVersion(table, addb, wrongVersion);
-    intercept(IOException.class, E_INCOMPATIBLE_TAG_VERSION,
-        () -> localTableHandler.initTable());
-
-    // CLEANUP
-    table.putItem(originalVersionMarker);
-    localTableHandler.tagTableWithVersionMarker();
-    localTableHandler.initTable();
-  }
-
-  private void tagTableWithCustomVersion(Table table,
-      AmazonDynamoDB addb,
-      int wrongVersion) {
-    final Tag vmTag = new Tag().withKey(VERSION_MARKER_TAG_NAME)
-        .withValue(valueOf(wrongVersion));
-    TagResourceRequest tagResourceRequest = new TagResourceRequest()
-        .withResourceArn(table.getDescription().getTableArn())
-        .withTags(vmTag);
-    addb.tagResource(tagResourceRequest);
-  }
-
-  private void removeVersionMarkerTag(Table table, AmazonDynamoDB addb) {
-    addb.untagResource(new UntagResourceRequest()
-        .withResourceArn(table.describe().getTableArn())
-        .withTagKeys(VERSION_MARKER_TAG_NAME));
-  }
-
-  /**
-   * Deletes a version marker; spins briefly to await it disappearing.
-   * @param table table to delete the key
-   * @throws Exception failure
-   */
-  private void deleteVersionMarkerItem(Table table) throws Exception {
-    table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
-    eventually(30_000, 1_0, () ->
-        assertNull("Version marker should be null after deleting it " +
-            "from the table.", table.getItem(VERSION_MARKER_PRIMARY_KEY)));
-  }
-
-  /**
-   * Test that initTable fails with IOException when table does not exist and
-   * table auto-creation is disabled.
-   */
-  @Test
-  public void testFailNonexistentTable() throws IOException {
-    final String tableName =
-        getTestTableName("testFailNonexistentTable");
-    final S3AFileSystem s3afs = getFileSystem();
-    final Configuration conf = s3afs.getConf();
-    enableOnDemand(conf);
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    String b = fsUri.getHost();
-    clearBucketOption(conf, b, S3GUARD_DDB_TABLE_CREATE_KEY);
-    clearBucketOption(conf, b, S3_METADATA_STORE_IMPL);
-    clearBucketOption(conf, b, S3GUARD_DDB_TABLE_NAME_KEY);
-    conf.unset(S3GUARD_DDB_TABLE_CREATE_KEY);
-    try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
-      ddbms.initialize(s3afs, new S3Guard.TtlTimeProvider(conf));
-      // if an exception was not raised, a table was created.
-      // So destroy it before failing.
-      ddbms.destroy();
-      fail("Should have failed as table does not exist and table auto-creation"
-          + " is disabled");
-    } catch (IOException ignored) {
-    }
-  }
-
-  /**
-   * Test cases about root directory as it is not in the DynamoDB table.
-   */
-  @Test
-  public void testRootDirectory() throws IOException {
-    final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
-    Path rootPath = new Path(new Path(fsUri), "/");
-    verifyRootDirectory(ddbms.get(rootPath), true);
-
-    ddbms.put(new PathMetadata(new S3AFileStatus(true,
-        new Path(rootPath, "foo"),
-        UserGroupInformation.getCurrentUser().getShortUserName())),
-        null);
-    verifyRootDirectory(ddbms.get(rootPath), false);
-  }
-
-  private void verifyRootDirectory(PathMetadata rootMeta, boolean isEmpty) {
-    assertNotNull(rootMeta);
-    final FileStatus status = rootMeta.getFileStatus();
-    assertNotNull(status);
-    assertTrue(status.isDirectory());
-    // UNKNOWN is always a valid option, but true / false should not contradict
-    if (isEmpty) {
-      assertNotSame("Should not be marked non-empty",
-          Tristate.FALSE,
-          rootMeta.isEmptyDirectory());
-    } else {
-      assertNotSame("Should not be marked empty",
-          Tristate.TRUE,
-          rootMeta.isEmptyDirectory());
-    }
-  }
-
-  /**
-   * Test that when moving nested paths, all its ancestors up to destination
-   * root will also be created.
-   * Here is the directory tree before move:
-   * <pre>
-   * testMovePopulateAncestors
-   * ├── a
-   * │   └── b
-   * │       └── src
-   * │           ├── dir1
-   * │           │   └── dir2
-   * │           └── file1.txt
-   * └── c
-   *     └── d
-   *         └── dest
-   *</pre>
-   * As part of rename(a/b/src, d/c/dest), S3A will enumerate the subtree at
-   * a/b/src.  This test verifies that after the move, the new subtree at
-   * 'dest' is reachable from the root (i.e. c/ and c/d exist in the table.
-   * DynamoDBMetadataStore depends on this property to do recursive delete
-   * without a full table scan.
-   */
-  @Test
-  public void testMovePopulatesAncestors() throws IOException {
-    final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
-    final String testRoot = "/testMovePopulatesAncestors";
-    final String srcRoot = testRoot + "/a/b/src";
-    final String destRoot = testRoot + "/c/d/e/dest";
-
-    final Path nestedPath1 = strToPath(srcRoot + "/file1.txt");
-    AncestorState bulkWrite = ddbms.initiateBulkWrite(
-        BulkOperationState.OperationType.Put, nestedPath1);
-    ddbms.put(new PathMetadata(basicFileStatus(nestedPath1, 1024, false)),
-        bulkWrite);
-    final Path nestedPath2 = strToPath(srcRoot + "/dir1/dir2");
-    ddbms.put(new PathMetadata(basicFileStatus(nestedPath2, 0, true)),
-        bulkWrite);
-
-    // We don't put the destRoot path here, since put() would create ancestor
-    // entries, and we want to ensure that move() does it, instead.
-
-    // Build enumeration of src / dest paths and do the move()
-    final Collection<Path> fullSourcePaths = Lists.newArrayList(
-        strToPath(srcRoot),
-        strToPath(srcRoot + "/dir1"),
-        strToPath(srcRoot + "/dir1/dir2"),
-        strToPath(srcRoot + "/file1.txt"));
-    final String finalFile = destRoot + "/file1.txt";
-    final Collection<PathMetadata> pathsToCreate = Lists.newArrayList(
-        new PathMetadata(basicFileStatus(strToPath(destRoot),
-            0, true)),
-        new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1"),
-            0, true)),
-        new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1/dir2"),
-            0, true)),
-        new PathMetadata(basicFileStatus(strToPath(finalFile),
-            1024, false))
-    );
-
-    ddbms.move(fullSourcePaths, pathsToCreate, bulkWrite);
-    bulkWrite.close();
-    // assert that all the ancestors should have been populated automatically
-    List<String> paths = Lists.newArrayList(
-        testRoot + "/c", testRoot + "/c/d", testRoot + "/c/d/e", destRoot,
-        destRoot + "/dir1", destRoot + "/dir1/dir2");
-    for (String p : paths) {
-      assertCached(p);
-      verifyInAncestor(bulkWrite, p, true);
-    }
-    // Also check moved files while we're at it
-    assertCached(finalFile);
-    verifyInAncestor(bulkWrite, finalFile, false);
-  }
-
-  @Test
-  public void testAncestorOverwriteConflict() throws Throwable {
-    final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
-    String testRoot = "/" + getMethodName();
-    String parent = testRoot + "/parent";
-    Path parentPath = strToPath(parent);
-    String child = parent + "/child";
-    Path childPath = strToPath(child);
-    String grandchild = child + "/grandchild";
-    Path grandchildPath = strToPath(grandchild);
-    String child2 = parent + "/child2";
-    String grandchild2 = child2 + "/grandchild2";
-    Path grandchild2Path = strToPath(grandchild2);
-    AncestorState bulkWrite = ddbms.initiateBulkWrite(
-        BulkOperationState.OperationType.Put, parentPath);
-
-    // writing a child creates ancestors
-    ddbms.put(
-        new PathMetadata(basicFileStatus(childPath, 1024, false)),
-        bulkWrite);
-    verifyInAncestor(bulkWrite, child, false);
-    verifyInAncestor(bulkWrite, parent, true);
-
-    // overwrite an ancestor with a file entry in the same operation
-    // is an error.
-    intercept(PathIOException.class, E_INCONSISTENT_UPDATE,
-        () -> ddbms.put(
-            new PathMetadata(basicFileStatus(parentPath, 1024, false)),
-            bulkWrite));
-
-    // now put a file under the child and expect the put operation
-    // to fail fast, because the ancestor state includes a file at a parent.
-
-    intercept(PathIOException.class, E_INCONSISTENT_UPDATE,
-        () -> ddbms.put(
-            new PathMetadata(basicFileStatus(grandchildPath, 1024, false)),
-            bulkWrite));
-
-    // and expect a failure for directory update under the child
-    DirListingMetadata grandchildListing = new DirListingMetadata(
-        grandchildPath,
-        new ArrayList<>(), false);
-    intercept(PathIOException.class, E_INCONSISTENT_UPDATE,
-        () -> ddbms.put(grandchildListing, UNCHANGED_ENTRIES, bulkWrite));
-
-    // but a directory update under another path is fine
-    DirListingMetadata grandchild2Listing = new DirListingMetadata(
-        grandchild2Path,
-        new ArrayList<>(), false);
-    ddbms.put(grandchild2Listing, UNCHANGED_ENTRIES, bulkWrite);
-    // and it creates a new entry for its parent
-    verifyInAncestor(bulkWrite, child2, true);
-  }
-
-  /**
-   * Assert that a path has an entry in the ancestor state.
-   * @param state ancestor state
-   * @param path path to look for
-   * @param isDirectory is it a directory
-   * @return the value
-   * @throws IOException IO failure
-   * @throws AssertionError assertion failure.
-   */
-  private DDBPathMetadata verifyInAncestor(AncestorState state,
-      String path,
-      final boolean isDirectory)
-      throws IOException {
-    final Path p = strToPath(path);
-    assertTrue("Path " + p + " not found in ancestor state", state.contains(p));
-    final DDBPathMetadata md = state.get(p);
-    assertTrue("Ancestor value for "+  path,
-        isDirectory
-            ? md.getFileStatus().isDirectory()
-            : md.getFileStatus().isFile());
-    return md;
-  }
-
-  @Test
-  public void testDeleteTable() throws Exception {
-    final String tableName = getTestTableName("testDeleteTable");
-    Path testPath = new Path(new Path(fsUri), "/" + tableName);
-    final S3AFileSystem s3afs = getFileSystem();
-    // patch the filesystem config as this is one read in initialize()
-    final Configuration conf =  s3afs.getConf();
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    enableOnDemand(conf);
-    DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
-    try {
-      ddbms.initialize(s3afs, new S3Guard.TtlTimeProvider(conf));
-      // we can list the empty table
-      ddbms.listChildren(testPath);
-      DynamoDB dynamoDB = ddbms.getDynamoDB();
-      ddbms.destroy();
-      verifyTableNotExist(tableName, dynamoDB);
-
-      // delete table once more; the ResourceNotFoundException swallowed
-      // silently
-      ddbms.destroy();
-      verifyTableNotExist(tableName, dynamoDB);
-      intercept(IOException.class, "",
-          "Should have failed after the table is destroyed!",
-          () -> ddbms.listChildren(testPath));
-      ddbms.destroy();
-      intercept(FileNotFoundException.class, "",
-          "Destroyed table should raise FileNotFoundException when pruned",
-          () -> ddbms.prune(PruneMode.ALL_BY_MODTIME, 0));
-    } finally {
-      destroy(ddbms);
-    }
-  }
-
-  protected void verifyStoreTags(final Map<String, String> tagMap,
-      final DynamoDBMetadataStore store) {
-    List<Tag> tags = listTagsOfStore(store);
-    Map<String, String> actual = new HashMap<>();
-    tags.forEach(t -> actual.put(t.getKey(), t.getValue()));
-    Assertions.assertThat(actual)
-        .describedAs("Tags from DDB table")
-        .containsAllEntriesOf(tagMap);
-
-    // The version marker is always there in the tags.
-    // We have a plus one in tags we expect.
-    assertEquals(tagMap.size() + 1, tags.size());
-  }
-
-  protected List<Tag> listTagsOfStore(final DynamoDBMetadataStore store) {
-    ListTagsOfResourceRequest listTagsOfResourceRequest =
-        new ListTagsOfResourceRequest()
-            .withResourceArn(store.getTable().getDescription()
-                .getTableArn());
-    return store.getAmazonDynamoDB()
-        .listTagsOfResource(listTagsOfResourceRequest).getTags();
-  }
-
-  private static Map<String, String> createTagMap() {
-    Map<String, String> tagMap = new HashMap<>();
-    tagMap.put("hello", "dynamo");
-    tagMap.put("tag", "youre it");
-    return tagMap;
-  }
-
-  private static void tagConfiguration(Configuration conf) {
-    // set the tags on the table so that it can be tested later.
-    Map<String, String> tagMap = createTagMap();
-    for (Map.Entry<String, String> tagEntry : tagMap.entrySet()) {
-      conf.set(S3GUARD_DDB_TABLE_TAG + tagEntry.getKey(), tagEntry.getValue());
-    }
-  }
-
-  @Test
-  public void testGetEmptyDirFlagCanSetTrue() throws IOException {
-    boolean authoritativeDirectoryListing = true;
-    testGetEmptyDirFlagCanSetTrueOrUnknown(authoritativeDirectoryListing);
-  }
-
-  @Test
-  public void testGetEmptyDirFlagCanSetUnknown() throws IOException {
-    boolean authoritativeDirectoryListing = false;
-    testGetEmptyDirFlagCanSetTrueOrUnknown(authoritativeDirectoryListing);
-  }
-
-  private void testGetEmptyDirFlagCanSetTrueOrUnknown(boolean auth)
-      throws IOException {
-    // setup
-    final DynamoDBMetadataStore ms = getDynamoMetadataStore();
-    String rootPath = "/testAuthoritativeEmptyDirFlag-" + UUID.randomUUID();
-    String filePath = rootPath + "/file1";
-    final Path dirToPut = fileSystem.makeQualified(new Path(rootPath));
-    final Path fileToPut = fileSystem.makeQualified(new Path(filePath));
-
-    // Create non-auth DirListingMetadata
-    DirListingMetadata dlm =
-        new DirListingMetadata(dirToPut, new ArrayList<>(), auth);
-    if(auth){
-      assertEquals(Tristate.TRUE, dlm.isEmpty());
-    } else {
-      assertEquals(Tristate.UNKNOWN, dlm.isEmpty());
-    }
-    assertEquals(auth, dlm.isAuthoritative());
-
-    // Test with non-authoritative listing, empty dir
-    ms.put(dlm, UNCHANGED_ENTRIES, null);
-    final PathMetadata pmdResultEmpty = ms.get(dirToPut, true);
-    if(auth){
-      assertEquals(Tristate.TRUE, pmdResultEmpty.isEmptyDirectory());
-    } else {
-      assertEquals(Tristate.UNKNOWN, pmdResultEmpty.isEmptyDirectory());
-    }
-
-    // Test with non-authoritative listing, non-empty dir
-    dlm.put(new PathMetadata(basicFileStatus(fileToPut, 1, false)));
-    ms.put(dlm, UNCHANGED_ENTRIES, null);
-    final PathMetadata pmdResultNotEmpty = ms.get(dirToPut, true);
-    assertEquals(Tristate.FALSE, pmdResultNotEmpty.isEmptyDirectory());
-  }
-
-  /**
-   * This validates the table is created and ACTIVE in DynamoDB.
-   *
-   * This should not rely on the {@link DynamoDBMetadataStore} implementation.
-   * Return the table
-   */
-  private Table verifyTableInitialized(String tableName, DynamoDB dynamoDB) {
-    final Table table = dynamoDB.getTable(tableName);
-    final TableDescription td = table.describe();
-    assertEquals(tableName, td.getTableName());
-    assertEquals("ACTIVE", td.getTableStatus());
-    return table;
-  }
-
-  /**
-   * Verify the table is created with correct server side encryption (SSE).
-   */
-  private void verifyTableSse(Configuration conf, TableDescription td) {
-    SSEDescription sseDescription = td.getSSEDescription();
-    if (conf.getBoolean(S3GUARD_DDB_TABLE_SSE_ENABLED, false)) {
-      assertNotNull(sseDescription);
-      assertEquals("ENABLED", sseDescription.getStatus());
-      assertEquals("KMS", sseDescription.getSSEType());
-      // We do not test key ARN is the same as configured value,
-      // because in configuration, the ARN can be specified by alias.
-      assertNotNull(sseDescription.getKMSMasterKeyArn());
-    } else {
-      if (sseDescription != null) {
-        assertEquals("DISABLED", sseDescription.getStatus());
-      }
-    }
-  }
-
-  /**
-   * This validates the table is not found in DynamoDB.
-   *
-   * This should not rely on the {@link DynamoDBMetadataStore} implementation.
-   */
-  private void verifyTableNotExist(String tableName, DynamoDB dynamoDB) throws
-      Exception{
-    intercept(ResourceNotFoundException.class,
-        () -> dynamoDB.getTable(tableName).describe());
-  }
-
-  private String getTestTableName(String suffix) {
-    return getTestDynamoTablePrefix(s3AContract.getConf()) + suffix;
-  }
-
-  @Test
-  public void testPruneAgainstInvalidTable() throws Throwable {
-    describe("Create an Invalid listing and prune it");
-    DynamoDBMetadataStore ms
-        = ITestDynamoDBMetadataStore.ddbmsStatic;
-    String base = "/" + getMethodName();
-    String subdir = base + "/subdir";
-    Path subDirPath = strToPath(subdir);
-    createNewDirs(base, subdir);
-
-    String subFile = subdir + "/file1";
-    Path subFilePath = strToPath(subFile);
-    putListStatusFiles(subdir, true,
-        subFile);
-    final DDBPathMetadata subDirMetadataOrig = ms.get(subDirPath);
-    Assertions.assertThat(subDirMetadataOrig.isAuthoritativeDir())
-        .describedAs("Subdirectory %s", subDirMetadataOrig)
-        .isTrue();
-
-    // now let's corrupt the graph by putting a file
-    // over the subdirectory
-
-    long now = getTime();
-    long oldTime = now - MINUTE;
-    putFile(subdir, oldTime, null);
-    getFile(subdir);
-
-    Path basePath = strToPath(base);
-    DirListingMetadata listing = ms.listChildren(basePath);
-    String childText = listing.prettyPrint();
-    LOG.info("Listing {}", childText);
-    Collection<PathMetadata> childList = listing.getListing();
-    Assertions.assertThat(childList)
-        .as("listing of %s with %s", basePath, childText)
-        .hasSize(1);
-    PathMetadata[] pm = new PathMetadata[0];
-    S3AFileStatus status = childList.toArray(pm)[0]
-        .getFileStatus();
-    Assertions.assertThat(status.isFile())
-        .as("Entry %s", (Object)pm)
-        .isTrue();
-    getNonNull(subFile);
-
-    LOG.info("Pruning");
-    // now prune
-    ms.prune(PruneMode.ALL_BY_MODTIME,
-        now + MINUTE, subdir);
-    ms.get(subFilePath);
-
-    final PathMetadata subDirMetadataFinal = getNonNull(subdir);
-
-    Assertions.assertThat(subDirMetadataFinal.getFileStatus().isFile())
-        .describedAs("Subdirectory entry %s is still a file",
-            subDirMetadataFinal)
-        .isTrue();
-  }
-
-  @Test
-  public void testPutFileDirectlyUnderTombstone() throws Throwable {
-    describe("Put a file under a tombstone; verify the tombstone");
-    String base = "/" + getMethodName();
-    long now = getTime();
-    putTombstone(base, now, null);
-    PathMetadata baseMeta1 = get(base);
-    Assertions.assertThat(baseMeta1.isDeleted())
-        .as("Metadata %s", baseMeta1)
-        .isTrue();
-    String child = base + "/file";
-    putFile(child, now, null);
-    getDirectory(base);
-  }
-
-  @Test
-  public void testPruneTombstoneUnderTombstone() throws Throwable {
-    describe("Put a tombsteone under a tombstone, prune the pair");
-    String base = "/" + getMethodName();
-    long now = getTime();
-    String dir = base + "/dir";
-    putTombstone(dir, now, null);
-    assertIsTombstone(dir);
-    // parent dir is created
-    assertCached(base);
-    String child = dir + "/file";
-    String child2 = dir + "/file2";
-
-    // this will actually mark the parent as a dir,
-    // so that lists of that dir will pick up the tombstone
-    putTombstone(child, now, null);
-    getDirectory(dir);
-    // tombstone the dir
-    putTombstone(dir, now, null);
-    // add another child entry; this will update the dir entry from being
-    // tombstone to dir
-    putFile(child2, now, null);
-    getDirectory(dir);
-
-    // put a tombstone over the directory again
-    putTombstone(dir, now, null);
-    // verify
-    assertIsTombstone(dir);
-
-    //prune all tombstones
-    getDynamoMetadataStore().prune(PruneMode.TOMBSTONES_BY_LASTUPDATED,
-        now + MINUTE);
-
-    // the child is gone
-    assertNotFound(child);
-
-    // *AND* the parent dir has not been created
-    assertNotFound(dir);
-
-    // the child2 entry is still there, though it's now orphan (the store isn't
-    // meeting the rule "all entries must have a parent which exists"
-    getFile(child2);
-    // a full prune will still find and delete it, as this
-    // doesn't walk the tree
-    getDynamoMetadataStore().prune(PruneMode.ALL_BY_MODTIME,
-        now + MINUTE);
-    assertNotFound(child2);
-    assertNotFound(dir);
-  }
-
-  @Test
-  public void testPruneFileUnderTombstone() throws Throwable {
-    describe("Put a file under a tombstone, prune the pair");
-    String base = "/" + getMethodName();
-    long now = getTime();
-    String dir = base + "/dir";
-    putTombstone(dir, now, null);
-    assertIsTombstone(dir);
-    // parent dir is created
-    assertCached(base);
-    String child = dir + "/file";
-
-    // this will actually mark the parent as a dir,
-    // so that lists of that dir will pick up the tombstone
-    putFile(child, now, null);
-    // dir is reinstated
-    getDirectory(dir);
-
-    // put a tombstone
-    putTombstone(dir, now, null);
-    // prune all entries
-    getDynamoMetadataStore().prune(PruneMode.ALL_BY_MODTIME,
-        now + MINUTE);
-    // the child is gone
-    assertNotFound(child);
-
-    // *AND* the parent dir has not been created
-    assertNotFound(dir);
-  }
-
-
-  @Test
-  public void testPruneFilesNotDirs() throws Throwable {
-    describe("HADOOP-16725: directories cannot be pruned");
-    String base = "/" + getMethodName();
-    final long now = getTime();
-    // round it off for ease of interpreting results
-    final long t0 = now - (now % 100_000);
-    long interval = 1_000;
-    long t1 = t0 + interval;
-    long t2 = t1 + interval;
-    String dir = base + "/dir";
-    String dir2 = base + "/dir2";
-    String child1 = dir + "/file1";
-    String child2 = dir + "/file2";
-    final Path basePath = strToPath(base);
-    // put the dir at age t0
-    final DynamoDBMetadataStore ms = getDynamoMetadataStore();
-    final AncestorState ancestorState
-        = ms.initiateBulkWrite(
-            BulkOperationState.OperationType.Put,
-            basePath);
-    putDir(base, t0, ancestorState);
-    assertLastUpdated(base, t0);
-
-    putDir(dir, t0, ancestorState);
-    assertLastUpdated(dir, t0);
-    // base dir is unchanged
-    assertLastUpdated(base, t0);
-
-    // this directory will not have any children, so
-    // will be excluded from any ancestor re-creation
-    putDir(dir2, t0, ancestorState);
-
-    // child1 has age t0 and so will be pruned
-    putFile(child1, t0, ancestorState);
-
-    // child2 has age t2
-    putFile(child2, t2, ancestorState);
-
-    // close the ancestor state
-    ancestorState.close();
-
-    // make some assertions about state before the prune
-    assertLastUpdated(base, t0);
-    assertLastUpdated(dir, t0);
-    assertLastUpdated(dir2, t0);
-    assertLastUpdated(child1, t0);
-    assertLastUpdated(child2, t2);
-
-    // prune all entries older than t1 must delete child1 but
-    // not the directory, even though it is of the same age
-    LOG.info("Starting prune of all entries older than {}", t1);
-    ms.prune(PruneMode.ALL_BY_MODTIME, t1);
-    // child1 is gone
-    assertNotFound(child1);
-
-    // *AND* the parent dir has not been created
-    assertCached(dir);
-    assertCached(child2);
-    assertCached(dir2);
-
-  }
-
-  /**
-   * A cert that there is an entry for the given key and that its
-   * last updated timestamp matches that passed in.
-   * @param key Key to look up.
-   * @param lastUpdated Timestamp to expect.
-   * @throws IOException I/O failure.
-   */
-  protected void assertLastUpdated(final String key, final long lastUpdated)
-      throws IOException {
-    PathMetadata dirMD = verifyCached(key);
-    assertEquals("Last updated timestamp in MD " + dirMD,
-        lastUpdated, dirMD.getLastUpdated());
-  }
-
-  /**
-   * Keep in sync with code changes in S3AFileSystem.finishedWrite() so that
-   * the production code can be tested here.
-   */
-  @Test
-  public void testPutFileDeepUnderTombstone() throws Throwable {
-    describe("Put a file two levels under a tombstone");
-    String base = "/" + getMethodName();
-    String dir = base + "/dir";
-    long now = getTime();
-    // creating a file MUST create its parents
-    String child = dir + "/file";
-    Path childPath = strToPath(child);
-    putFile(child, now, null);
-    getFile(child);
-    getDirectory(dir);
-    getDirectory(base);
-
-    // now put the tombstone
-    putTombstone(base, now, null);
-    assertIsTombstone(base);
-
-    /*- --------------------------------------------*/
-    /* Begin S3FileSystem.finishedWrite() sequence. */
-    /* ---------------------------------------------*/
-    AncestorState ancestorState = getDynamoMetadataStore()
-        .initiateBulkWrite(BulkOperationState.OperationType.Put,
-            childPath);
-    S3Guard.addAncestors(getDynamoMetadataStore(),
-        childPath,
-        getTtlTimeProvider(),
-        ancestorState);
-    // now write the file again.
-    putFile(child, now, ancestorState);
-    /* -------------------------------------------*/
-    /* End S3FileSystem.finishedWrite() sequence. */
-    /* -------------------------------------------*/
-
-    getFile(child);
-    // the ancestor will now exist.
-    getDirectory(dir);
-    getDirectory(base);
-  }
-
-  @Test
-  public void testDumpTable() throws Throwable {
-    describe("Dump the table contents, but not the S3 Store");
-    String target = System.getProperty("test.build.dir", "target");
-    File buildDir = new File(target).getAbsoluteFile();
-    String name = "ITestDynamoDBMetadataStore";
-    File destFile = new File(buildDir, name);
-    DumpS3GuardDynamoTable.dumpStore(
-        null,
-        ddbmsStatic,
-        getFileSystem().getConf(),
-        destFile,
-        fsUri);
-    File storeFile = new File(buildDir, name + DumpS3GuardDynamoTable.SCAN_CSV);
-    try (BufferedReader in = new BufferedReader(new InputStreamReader(
-        new FileInputStream(storeFile), Charset.forName("UTF-8")))) {
-      for (String line : org.apache.commons.io.IOUtils.readLines(in)) {
-        LOG.info(line);
-      }
-    }
-  }
-
-  @Test
-  public void testPurgeTableNoForce() throws Throwable {
-    describe("Purge the table");
-
-    putTombstone("/" + getMethodName(), getTime(), null);
-    Pair<Long, Long> r = PurgeS3GuardDynamoTable.purgeStore(
-        null,
-        ddbmsStatic,
-        getFileSystem().getConf(),
-        fsUri,
-        false);
-
-    Assertions.assertThat(r.getLeft()).
-        describedAs("entries found in %s", r)
-        .isGreaterThanOrEqualTo(1);
-    Assertions.assertThat(r.getRight()).
-        describedAs("entries deleted in %s", r)
-        .isZero();
-  }
-
-  @Test
-  public void testPurgeTableForce() throws Throwable {
-    describe("Purge the table -force");
-    putTombstone("/" + getMethodName(), getTime(), null);
-    Pair<Long, Long> r = PurgeS3GuardDynamoTable.purgeStore(
-        null,
-        ddbmsStatic,
-        getFileSystem().getConf(),
-        fsUri,
-        true);
-    Assertions.assertThat(r.getLeft()).
-        describedAs("entries found in %s", r)
-        .isGreaterThanOrEqualTo(1);
-    Assertions.assertThat(r.getRight()).
-        describedAs("entries deleted in %s", r)
-        .isEqualTo(r.getLeft());
-
-    // second iteration will have zero entries
-
-    r = PurgeS3GuardDynamoTable.purgeStore(
-        null,
-        ddbmsStatic,
-        getFileSystem().getConf(),
-        fsUri,
-        true);
-    Assertions.assertThat(r.getLeft()).
-        describedAs("entries found in %s", r)
-        .isZero();
-    Assertions.assertThat(r.getRight()).
-        describedAs("entries deleted in %s", r)
-        .isZero();
-  }
-
-  /**
-   * Assert that an entry exists and is a directory.
-   * @param pathStr path
-   * @throws IOException IO failure.
-   */
-  protected DDBPathMetadata verifyAuthDirStatus(String pathStr,
-      boolean authDirFlag)
-      throws IOException {
-    DDBPathMetadata md = (DDBPathMetadata) getDirectory(pathStr);
-    assertEquals("isAuthoritativeDir() mismatch in " + md,
-        authDirFlag,
-        md.isAuthoritativeDir());
-    return md;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreAuthoritativeMode.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreAuthoritativeMode.java
deleted file mode 100644
index 3d9715ceb3ebb..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreAuthoritativeMode.java
+++ /dev/null
@@ -1,917 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-import org.assertj.core.api.Assertions;
-import org.junit.AfterClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.io.IOUtils;
-
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3AUtils.applyLocatedFiles;
-import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_LIST_REQUEST;
-import static org.apache.hadoop.fs.s3a.Statistic.S3GUARD_METADATASTORE_AUTHORITATIVE_DIRECTORIES_UPDATED;
-import static org.apache.hadoop.fs.s3a.Statistic.S3GUARD_METADATASTORE_RECORD_WRITES;
-import static org.apache.hadoop.fs.s3a.s3guard.AuthoritativeAuditOperation.ERROR_PATH_NOT_AUTH_IN_FS;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.authoritativeEmptyDirectoryMarker;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Authoritative.CHECK_FLAG;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Authoritative.REQUIRE_AUTH;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Import.AUTH_FLAG;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.VERBOSE;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.expectExecResult;
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_ACCEPTABLE;
-import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_NOT_FOUND;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Test to verify the expected behaviour of DynamoDB and authoritative mode.
- * The main testFS is non-auth; we also create a test FS which runs in auth
- * mode.
- * Making the default FS non-auth means that test path cleanup in the
- * superclass isn't going to get mislead by anything authoritative.
- *
- * For performance boosting we demand create the auth FS and its test
- * paths on the first test setup().
- * This also fixes the auth/nonauth paths so that a specific
- * bit of the FS is expected to be auth in the FS.
- *
- * This test is designed to run in parallel mode with other tests which
- * may or may not be auth mode.
- *
- * It shouldn't make any difference -tests here simply must not make
- * any assumptions about the state of any path outside the test tree.
- */
-@SuppressWarnings("StaticNonFinalField")
-public class ITestDynamoDBMetadataStoreAuthoritativeMode
-    extends AbstractS3ATestBase {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      ITestDynamoDBMetadataStoreAuthoritativeMode.class);
-
-  public static final String AUDIT = S3GuardTool.Authoritative.NAME;
-
-  public static final String IMPORT = S3GuardTool.Import.NAME;
-
-  private String fsUriStr;
-
-  /**
-   * Authoritative FS.
-   */
-  private static S3AFileSystem authFS;
-
-  /**
-   * The unguarded file system.
-   */
-  private static S3AFileSystem unguardedFS;
-
-  /**
-   * Base path in the store for auth and nonauth paths.
-   */
-  private static Path basePath;
-
-  /**
-   * Path under basePath which will be declared as authoritative.
-   */
-  private static Path authPath;
-
-  /**
-   * Path under basePath which will be declared as non-authoritative.
-   */
-  private static Path nonauthPath;
-
-  /**
-   * test method specific auth path.
-   */
-  private Path methodAuthPath;
-
-  /**
-   * test method specific non-auth path.
-   */
-  private Path methodNonauthPath;
-
-  /**
-   * Auditor of store state.
-   */
-  private AuthoritativeAuditOperation auditor;
-
-  /**
-   * Path {@code $methodAuthPath/dir}.
-   */
-  private Path dir;
-
-  /**
-   * Path {@code $methodAuthPath/dir/file}.
-   */
-  private Path dirFile;
-
-  /**
-   * List of tools to close in test teardown.
-   */
-  private final List<S3GuardTool> toolsToClose = new ArrayList<>();
-
-  /**
-   * The metastore of the auth filesystem.
-   */
-  private DynamoDBMetadataStore metastore;
-
-  /**
-   * After all tests have run, close the filesystems.
-   */
-  @AfterClass
-  public static void closeFileSystems() {
-    IOUtils.cleanupWithLogger(LOG, authFS, unguardedFS);
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
-        METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    conf.setTimeDuration(
-        S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
-        0,
-        TimeUnit.MILLISECONDS);
-    return conf;
-  }
-
-  /**
-   * Test case setup will on-demand create the class-level fields
-   * of the authFS and the auth/non-auth paths.
-   */
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    S3AFileSystem fs = getFileSystem();
-    Configuration conf = fs.getConf();
-    S3ATestUtils.assumeS3GuardState(true, conf);
-    assume("Filesystem isn't running DDB",
-        fs.getMetadataStore() instanceof DynamoDBMetadataStore);
-    URI fsURI = fs.getUri();
-    fsUriStr = fsURI.toString();
-    if (!fsUriStr.endsWith("/")) {
-      fsUriStr = fsUriStr + "/";
-    }
-
-
-    if (authFS == null) {
-      // creating the test stores
-      basePath = path("base");
-      authPath = new Path(basePath, "auth");
-      nonauthPath = new Path(basePath, "nonauth");
-      final Configuration authconf = new Configuration(conf);
-      final URI uri = authPath.toUri();
-      authconf.set(AUTHORITATIVE_PATH, uri.toString());
-      authFS = (S3AFileSystem) FileSystem.newInstance(uri, authconf);
-
-      // and create the unguarded at the same time
-      final Configuration unguardedConf = new Configuration(conf);
-      removeBaseAndBucketOverrides(unguardedConf,
-          S3_METADATA_STORE_IMPL);
-      unguardedFS = (S3AFileSystem) FileSystem.newInstance(uri, unguardedConf);
-    }
-    metastore = (DynamoDBMetadataStore) authFS.getMetadataStore();
-    auditor = new AuthoritativeAuditOperation(
-        authFS.createStoreContext(),
-        metastore,
-        true,
-        true);
-
-    cleanupMethodPaths();
-    dir = new Path(methodAuthPath, "dir");
-    dirFile = new Path(dir, "file");
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    toolsToClose.forEach(t -> IOUtils.cleanupWithLogger(LOG, t));
-    try {
-      cleanupMethodPaths();
-    } catch (IOException ignored) {
-    }
-    super.teardown();
-  }
-
-  /**
-   * Clean up from other test runs which halted.
-   * Uses the authfs; no-op if null.
-   * @throws IOException Failure
-   */
-  private void cleanupMethodPaths() throws IOException {
-    S3AFileSystem fs = authFS;
-    if (fs != null) {
-      methodAuthPath = new Path(authPath, getMethodName());
-      fs.delete(methodAuthPath, true);
-      methodNonauthPath = new Path(nonauthPath, getMethodName());
-      fs.delete(methodNonauthPath, true);
-    }
-  }
-
-  /**
-   * Declare that the tool is to be closed in teardown.
-   * @param tool tool to close
-   * @return the tool.
-   */
-  protected <T extends S3GuardTool> T toClose(T tool) {
-    toolsToClose.add(tool);
-    return tool;
-  }
-
-  /**
-   * Get the conf of the auth FS.
-   * @return the auth FS config.
-   */
-  private Configuration getAuthConf() {
-    return authFS.getConf();
-  }
-
-  @Test
-  public void testEmptyDirMarkerIsAuth() {
-    final S3AFileStatus st = new S3AFileStatus(true, dir, "root");
-    final DDBPathMetadata md = (DDBPathMetadata)
-        authoritativeEmptyDirectoryMarker(st);
-    Assertions.assertThat(md)
-        .describedAs("Metadata %s", md)
-        .matches(DDBPathMetadata::isAuthoritativeDir, "is auth dir")
-        .matches(d -> d.isEmptyDirectory() == Tristate.TRUE,
-            "isEmptyDirectory");
-  }
-
-  @Test
-  public void testMkDirAuth() throws Throwable {
-    describe("create an empty dir and assert it is tagged as authoritative");
-    authFS.mkdirs(dir);
-    expectAuthRecursive(dir);
-  }
-
-  @Test
-  public void testListStatusMakesEmptyDirAuth() throws Throwable {
-    describe("Verify listStatus marks an Empty dir as auth");
-    mkNonauthDir(dir);
-    // initial dir is non-auth
-    expectNonauthNonRecursive(dir);
-    authFS.listStatus(dir);
-    // dir is auth;
-    expectAuthRecursive(dir);
-    // Next list will not go to s3
-    assertListDoesNotUpdateAuth(dir);
-  }
-
-  @Test
-  public void testListFilesRecursiveWhenAllListingsAreAuthoritative()
-      throws Exception {
-    describe("listFiles does not make further calls to the fs when"
-        + "all nested directory listings are authoritative");
-    Set<Path> originalFiles = new HashSet<>();
-
-    Path parentDir = dir;
-    Path parentFile = dirFile;
-    Path nestedDir1 = new Path(dir, "nested1");
-    Path nestedFile1 = new Path(nestedDir1, "nestedFile1");
-    Path nestedDir2 = new Path(nestedDir1, "nested2/");
-    Path nestedFile2 = new Path(nestedDir2, "nestedFile2");
-
-    originalFiles.add(parentFile);
-    originalFiles.add(nestedFile1);
-    originalFiles.add(nestedFile2);
-
-    mkAuthDir(parentDir);
-    mkAuthDir(nestedDir1);
-    mkAuthDir(nestedDir2);
-    touchFile(parentFile);
-    touchFile(nestedFile1);
-    touchFile(nestedFile2);
-
-    S3ATestUtils.MetricDiff objListRequests =
-        new S3ATestUtils.MetricDiff(authFS, OBJECT_LIST_REQUEST);
-
-    RemoteIterator<LocatedFileStatus> statusIterator =
-        authFS.listFiles(dir, true);
-
-    List<Path> pathsFromStatusIterator = toPaths(statusIterator);
-
-    Assertions.assertThat(pathsFromStatusIterator)
-        .as("listFiles should return all the items in actual"
-            + "S3 directory and nothing more")
-        .hasSameElementsAs(originalFiles)
-        .hasSameSizeAs(originalFiles);
-
-    objListRequests.assertDiffEquals("There must not be any OBJECT_LIST "
-        + "requests as all directory listings are authoritative", 0);
-  }
-
-  @Test
-  public void testListFilesRecursiveWhenSomePathsAreNotAuthoritative()
-      throws Exception {
-    describe("listFiles correctly constructs recursive listing"
-        + "when authoritative and non-authoritative paths are mixed");
-    List<Path> originalFiles = new ArrayList<>();
-    Path parentDir = dir;
-    Path parentFile = dirFile;
-    Path nestedDir1 = new Path(dir, "nested1");
-    Path nestedFile1 = new Path(nestedDir1, "nestedFile1");
-    Path nestedDir2 = new Path(nestedDir1, "nested2/");
-    Path nestedFile2 = new Path(nestedDir2, "nestedFile2");
-
-    originalFiles.add(parentFile);
-    originalFiles.add(nestedFile1);
-    originalFiles.add(nestedFile2);
-
-    mkAuthDir(parentDir);
-    mkNonauthDir(nestedDir1);
-    mkAuthDir(nestedDir2);
-    touchFile(parentFile);
-    touchFile(nestedFile1);
-    touchFile(nestedFile2);
-
-    S3ATestUtils.MetricDiff objListRequests =
-        new S3ATestUtils.MetricDiff(authFS, OBJECT_LIST_REQUEST);
-
-    RemoteIterator<LocatedFileStatus> statusIterator =
-        authFS.listFiles(dir, true);
-
-    List<Path> pathsFromStatusIterator = toPaths(statusIterator);
-
-    Assertions.assertThat(pathsFromStatusIterator)
-        .as("listFiles should return all the items in actual"
-            + "S3 directory and nothing more")
-        .hasSameElementsAs(originalFiles)
-        .hasSameSizeAs(originalFiles);
-    objListRequests.assertDiffEquals("Only 1 OBJECT_LIST call is expected"
-        + "as a nested directory listing is not authoritative", 1);
-  }
-
-  @Test
-  public void testListStatusMakesDirAuth() throws Throwable {
-    describe("Verify listStatus marks a dir as auth");
-    final Path subdir = new Path(dir, "subdir");
-
-    mkAuthDir(dir);
-    expectAuthRecursive(dir);
-    authFS.mkdirs(subdir);
-    // dir and subdirs are auth
-    expectAuthRecursive(dir);
-    expectAuthRecursive(subdir);
-    // now mark the dir as nonauth
-    markDirNonauth(dir);
-    expectNonauthNonRecursive(dir);
-    expectAuthRecursive(subdir);
-
-    // look at the MD & make sure that the dir and subdir are auth
-    final DirListingMetadata listing = metastore.listChildren(dir);
-    Assertions.assertThat(listing)
-        .describedAs("metadata of %s", dir)
-        .matches(d -> !d.isAuthoritative(), "is not auth");
-    Assertions.assertThat(listing.getListing())
-        .describedAs("listing of %s", dir)
-        .hasSize(1)
-        .allMatch(md -> ((DDBPathMetadata) md).isAuthoritativeDir(),
-            "is auth");
-
-    // directory list makes the dir auth and leaves the child auth
-    assertListUpdatesAuth(dir);
-
-    // and afterwards, a followup list does not write anything to DDB
-    // (as the dir is auth, its not going to go near the FS to update...)
-    expectOperationUpdatesDDB(0, () -> authFS.listStatus(dir));
-    // mark the dir nonauth again
-    markDirNonauth(dir);
-    // and only one record is written to DDB, the dir marker as auth
-    // the subdir is not overwritten
-    expectOperationUpdatesDDB(1, () -> authFS.listStatus(dir));
-  }
-
-  @Test
-  public void testAddFileMarksNonAuth() throws Throwable {
-    describe("Adding a file marks dir as nonauth but leaves ancestors alone");
-    mkAuthDir(methodAuthPath);
-    touchFile(dirFile);
-    expectNonauthRecursive(dir);
-    assertListUpdatesAuth(dir);
-    expectAuthRecursive(methodAuthPath);
-  }
-
-  /**
-   * When you delete the single file in a directory then a fake directory
-   * marker is added. This must be auth.
-   */
-  @Test
-  public void testDeleteSingleFileLeavesMarkersAlone() throws Throwable {
-    describe("Deleting a file with no peers makes no changes to ancestors");
-    mkAuthDir(methodAuthPath);
-    touchFile(dirFile);
-    assertListUpdatesAuth(dir);
-    authFS.delete(dirFile, false);
-    expectAuthRecursive(methodAuthPath);
-  }
-
-  @Test
-  public void testDeleteMultipleFileLeavesMarkersAlone() throws Throwable {
-    describe("Deleting a file from a dir with >1 file makes no changes"
-        + " to ancestors");
-    mkAuthDir(methodAuthPath);
-    touchFile(dirFile);
-    Path file2 = new Path(dir, "file2");
-    touchFile(file2);
-    assertListUpdatesAuth(dir);
-    authFS.delete(dirFile, false);
-    expectAuthRecursive(methodAuthPath);
-  }
-
-  @Test
-  public void testDeleteEmptyDirLeavesParentAuth() throws Throwable {
-    describe("Deleting a directory retains the auth status "
-        + "of the parent directory");
-    mkAuthDir(dir);
-    mkAuthDir(dirFile);
-    expectAuthRecursive(dir);
-    authFS.delete(dirFile, false);
-    expectAuthRecursive(dir);
-  }
-
-  /**
-   * Assert the number of pruned files matches expectations.
-   * @param path path to prune
-   * @param mode prune mode
-   * @param limit timestamp before which files are deleted
-   * @param expected number of entries to be pruned
-   */
-  protected void assertPruned(final Path path,
-      final MetadataStore.PruneMode mode,
-      final long limit,
-      final int expected)
-      throws IOException {
-    String keyPrefix
-        = PathMetadataDynamoDBTranslation.pathToParentKey(path);
-    Assertions.assertThat(
-        authFS.getMetadataStore().prune(
-            mode,
-            limit,
-            keyPrefix))
-        .describedAs("Number of files pruned under %s", keyPrefix)
-        .isEqualTo(expected);
-  }
-
-  @Test
-  public void testPruneFilesMarksNonAuth() throws Throwable {
-    describe("Pruning a file marks dir as nonauth");
-    mkAuthDir(methodAuthPath);
-
-    touchFile(dirFile);
-    assertListUpdatesAuth(dir);
-
-    assertPruned(dir,
-        MetadataStore.PruneMode.ALL_BY_MODTIME,
-        Long.MAX_VALUE,
-        1);
-    expectNonauthRecursive(dir);
-  }
-
-  @Test
-  public void testPruneTombstoneRetainsAuth() throws Throwable {
-    describe("Verify that deleting and then pruning a file does not change"
-        + " the state of the parent.");
-    mkAuthDir(methodAuthPath);
-
-    touchFile(dirFile);
-    assertListUpdatesAuth(dir);
-    // add a second file to avoid hitting the mkdir-is-nonauth issue that causes
-    // testDeleteSingleFileLeavesMarkersAlone() to fail
-    Path file2 = new Path(dir, "file2");
-    touchFile(file2);
-    authFS.delete(dirFile, false);
-    expectAuthRecursive(dir);
-    assertPruned(dir, MetadataStore.PruneMode.TOMBSTONES_BY_LASTUPDATED,
-        Long.MAX_VALUE, 1);
-    expectAuthRecursive(dir);
-  }
-
-  @Test
-  public void testRenameFile() throws Throwable {
-    describe("renaming a file");
-    final Path source = new Path(dir, "source");
-    final Path dest = new Path(dir, "dest");
-    touchFile(source);
-    assertListUpdatesAuth(dir);
-    authFS.rename(source, dest);
-    expectAuthRecursive(dir);
-  }
-
-  @Test
-  public void testRenameDirMarksDestAsAuth() throws Throwable {
-    describe("renaming a dir must mark dest tree as auth");
-    final Path base = methodAuthPath;
-    mkAuthDir(base);
-    final Path source = new Path(base, "source");
-    final Path dest = new Path(base, "dest");
-    mkAuthDir(source);
-    expectAuthRecursive(base);
-    Path subdir = new Path(source, "subdir");
-    Path f = new Path(subdir, "file");
-    touchFile(f);
-    expectNonauthRecursive(base);
-    // list the source directories so everything is
-    // marked as auth
-    authFS.listStatus(source);
-    authFS.listStatus(subdir);
-    expectAuthRecursive(base);
-    authFS.rename(source, dest);
-    expectAuthRecursive(base);
-  }
-
-  @Test
-  @Ignore("TODO: HADOOP-16465")
-  public void testListLocatedStatusMarksDirAsAuth() throws Throwable {
-    describe("validate listLocatedStatus()");
-    final Path subdir = new Path(dir, "subdir");
-    final Path subdirfile = new Path(subdir, "file");
-    touchFile(subdirfile);
-    // Subdir list makes it auth
-    expectAuthoritativeUpdate(1, 1, () -> {
-      final RemoteIterator<LocatedFileStatus> st
-          = authFS.listLocatedStatus(subdir);
-      applyLocatedFiles(st,
-          f -> LOG.info("{}", f));
-      return null;
-    });
-    expectAuthNonRecursive(subdir);
-  }
-
-  @Test
-  public void testS3GuardImportMarksDirAsAuth() throws Throwable {
-    describe("import with authoritive=true marks directories");
-    // the base dir is auth
-    mkAuthDir(methodAuthPath);
-    int expected = 0;
-    final Path subdir = new Path(dir, "subdir");
-    final Path subdirfile = new Path(subdir, "file");
-    ContractTestUtils.touch(authFS, subdirfile);
-    expected++;
-    for (int i = 0; i < 5; i++) {
-      ContractTestUtils.touch(authFS, new Path(subdir, "file-" + i));
-      expected++;
-    }
-    final Path emptydir = new Path(dir, "emptydir");
-    unguardedFS.mkdirs(emptydir);
-    expected++;
-
-    S3AFileStatus status1 = (S3AFileStatus) authFS.getFileStatus(subdirfile);
-    final MetadataStore authMS = authFS.getMetadataStore();
-    final ImportOperation importer = new ImportOperation(unguardedFS,
-        authMS,
-        (S3AFileStatus) unguardedFS.getFileStatus(dir),
-        true, true);
-    final Long count = importer.execute();
-    expectAuthRecursive(dir);
-    // the parent dir shouldn't have changed
-    expectAuthRecursive(methodAuthPath);
-
-    // file entry
-    S3AFileStatus status2 = (S3AFileStatus) authFS.getFileStatus(subdirfile);
-    Assertions.assertThat(status2.getETag())
-        .describedAs("Etag of %s", status2)
-        .isEqualTo(status1.getETag());
-    // only picked up on versioned stores.
-    Assertions.assertThat(status2.getVersionId())
-        .describedAs("version ID of %s", status2)
-        .isEqualTo(status1.getVersionId());
-
-    // the import finds files and empty dirs
-    Assertions.assertThat(count)
-        .describedAs("Count of imports under %s", dir)
-        .isEqualTo(expected);
-  }
-
-  /**
-   * Given a flag, add a - prefix.
-   * @param flag flag to wrap
-   * @return a flag for use in the CLI
-   */
-  private String f(String flag) {
-    return "-" + flag;
-  }
-
-  @Test
-  public void testAuditS3GuardTool() throws Throwable {
-    describe("Test the s3guard audit CLI");
-    mkNonauthDir(methodAuthPath);
-    final String path = methodAuthPath.toString();
-    // this is non-auth, so the scan is rejected
-    expectExecResult(EXIT_NOT_ACCEPTABLE,
-        authTool(),
-        AUDIT,
-        f(CHECK_FLAG),
-        f(REQUIRE_AUTH),
-        f(VERBOSE),
-        path);
-    // a non-auth audit is fine
-    exec(authTool(),
-        AUDIT,
-        f(VERBOSE),
-        path);
-    // non-auth import
-    exec(importTool(),
-        IMPORT,
-        f(VERBOSE),
-        path);
-    // which will leave the result unchanged
-    expectExecResult(EXIT_NOT_ACCEPTABLE,
-        authTool(),
-        AUDIT,
-        f(CHECK_FLAG),
-        f(REQUIRE_AUTH),
-        f(VERBOSE),
-        path);
-    // auth import
-    exec(importTool(),
-        IMPORT,
-        f(AUTH_FLAG),
-        f(VERBOSE),
-        path);
-    // so now the audit succeeds
-    exec(authTool(),
-        AUDIT,
-        f(REQUIRE_AUTH),
-        path);
-  }
-
-  /**
-   * Create an import tool instance with the auth FS Config.
-   * It will be closed in teardown.
-   * @return a new instance.
-   */
-  protected S3GuardTool.Import importTool() {
-    return toClose(new S3GuardTool.Import(getAuthConf()));
-  }
-
-  /**
-   * Create an auth tool instance with the auth FS Config.
-   * It will be closed in teardown.
-   * @return a new instance.
-   */
-  protected S3GuardTool.Authoritative authTool() {
-    return toClose(new S3GuardTool.Authoritative(getAuthConf()));
-  }
-
-  @Test
-  public void testAuditS3GuardToolNonauthDir() throws Throwable {
-    describe("Test the s3guard audit -check-conf against a nonauth path");
-    mkdirs(methodNonauthPath);
-    expectExecResult(ERROR_PATH_NOT_AUTH_IN_FS,
-        authTool(),
-        AUDIT,
-        f(CHECK_FLAG),
-        methodNonauthPath.toString());
-  }
-
-  @Test
-  public void testImportNonauthDir() throws Throwable {
-    describe("s3guard import against a nonauth path marks the dirs as auth");
-    final String path = methodNonauthPath.toString();
-    mkdirs(methodNonauthPath);
-    // auth import
-    exec(importTool(),
-        IMPORT,
-        f(AUTH_FLAG),
-        f(VERBOSE),
-        path);
-    exec(authTool(),
-        AUDIT,
-        f(REQUIRE_AUTH),
-        f(VERBOSE),
-        path);
-  }
-
-  @Test
-  public void testAuditS3GuardTooMissingDir() throws Throwable {
-    describe("Test the s3guard audit against a missing path");
-    expectExecResult(EXIT_NOT_FOUND,
-        authTool(),
-        AUDIT,
-        methodAuthPath.toString());
-  }
-
-  /**
-   * Touch a file in the authoritative fs.
-   * @param file path of file
-   * @throws IOException Failure
-   */
-  protected void touchFile(final Path file) throws IOException {
-    ContractTestUtils.touch(authFS, file);
-  }
-
-  /**
-   * Invoke an operation expecting the meta store to have its
-   * directoryMarkedAuthoritative count to be be updated {@code updates}
-   * times and S3 LIST requests made {@code lists} times.
-   * @param <T> Return type
-   * @param updates Expected count
-   * @param lists Expected lists
-   * @param fn Function to invoke
-   * @return Result of the function call
-   * @throws Exception Failure
-   */
-  private <T> T expectAuthoritativeUpdate(
-      int updates,
-      int lists,
-      Callable<T> fn)
-      throws Exception {
-    S3ATestUtils.MetricDiff authDirsMarked = new S3ATestUtils.MetricDiff(authFS,
-        S3GUARD_METADATASTORE_AUTHORITATIVE_DIRECTORIES_UPDATED);
-    S3ATestUtils.MetricDiff listRequests = new S3ATestUtils.MetricDiff(authFS,
-        OBJECT_LIST_REQUEST);
-    final T call = fn.call();
-    authDirsMarked.assertDiffEquals(updates);
-    listRequests.assertDiffEquals(lists);
-    return call;
-  }
-
-  /**
-   * Invoke an operation expecting {@code writes} records written to DDB.
-   * @param <T> Return type
-   * @param writes Expected count
-   * @param fn Function to invoke
-   * @return Result of the function call
-   * @throws Exception Failure
-   */
-  private <T> T expectOperationUpdatesDDB(
-      int writes,
-      Callable<T> fn)
-      throws Exception {
-    S3ATestUtils.MetricDiff writeDiff = new S3ATestUtils.MetricDiff(authFS,
-        S3GUARD_METADATASTORE_RECORD_WRITES);
-    final T call = fn.call();
-    writeDiff.assertDiffEquals(writes);
-    return call;
-  }
-
-  /**
-   * Creates a Path array from all items retrieved from
-   * {@link RemoteIterator<LocatedFileStatus>}.
-   *
-   * @param remoteIterator iterator
-   * @return a list of Paths
-   * @throws IOException
-   */
-  private List<Path> toPaths(RemoteIterator<LocatedFileStatus> remoteIterator)
-      throws IOException {
-    List<Path> list = new ArrayList<>();
-    while (remoteIterator.hasNext()) {
-      LocatedFileStatus fileStatus = remoteIterator.next();
-      list.add(fileStatus.getPath());
-    }
-    return list;
-  }
-
-  /**
-   * Assert that a listStatus call increments the
-   * "s3guard_metadatastore_authoritative_directories_updated" counter.
-   * Then checks that the directory is recursively authoritative.
-   * @param path path to scan
-   */
-  private void assertListUpdatesAuth(Path path) throws Exception {
-    expectAuthoritativeUpdate(1, 1, () -> authFS.listStatus(path));
-    expectAuthRecursive(path);
-  }
-
-  /**
-   * Assert that a listStatus call does not increment the
-   * "s3guard_metadatastore_authoritative_directories_updated" counter.
-   * @param path path to scan
-   */
-  private void assertListDoesNotUpdateAuth(Path path) throws Exception {
-    expectAuthoritativeUpdate(0, 0, () -> authFS.listStatus(path));
-  }
-
-  /**
-   * Create a directory if needed, force it to be authoritatively listed.
-   * @param path dir
-   */
-  private void mkAuthDir(Path path) throws IOException {
-    authFS.mkdirs(path);
-  }
-
-  /**
-   * Create a non-auth directory.
-   * @param path dir
-   */
-  private void mkNonauthDir(Path path) throws IOException {
-    authFS.mkdirs(path);
-    // overwrite entry with a nonauth one
-    markDirNonauth(path);
-  }
-
-  /**
-   * Mark a directory as nonauth.
-   * @param path path to the directory
-   * @throws IOException failure
-   */
-  private void markDirNonauth(final Path path) throws IOException {
-    S3Guard.putWithTtl(metastore,
-        nonAuthEmptyDirectoryMarker((S3AFileStatus) authFS.getFileStatus(path)),
-        null, null);
-  }
-
-  /**
-   * Create an empty dir marker which, when passed to the
-   * DDB metastore, is considered authoritative.
-   * @param status file status
-   * @return path metadata.
-   */
-  private PathMetadata nonAuthEmptyDirectoryMarker(
-      final S3AFileStatus status) {
-    return new DDBPathMetadata(status, Tristate.TRUE,
-        false, false, 0);
-  }
-  /**
-   * Performed a recursive audit of the directory
-   * -require everything to be authoritative.
-   * @param path directory
-   */
-  private void expectAuthRecursive(Path path) throws Exception {
-    auditor.executeAudit(path, true, true);
-  }
-
-  /**
-   * Performed a non-recursive audit of the directory
-   * -require the directory to be authoritative.
-   * @param path directory
-   */
-  private void expectAuthNonRecursive(Path path) throws Exception {
-    auditor.executeAudit(path, true, false);
-  }
-
-  /**
-   * Performed a recursive audit of the directory
-   * -expect a failure.
-   * @param path directory
-   * @return the path returned by the exception
-   */
-  private Path expectNonauthRecursive(Path path) throws Exception {
-    return intercept(
-        AuthoritativeAuditOperation.NonAuthoritativeDirException.class,
-        () -> auditor.executeAudit(path, true, true))
-        .getPath();
-  }
-
-  /**
-   * Performed a recursive audit of the directory
-   * -expect a failure.
-   * @param path directory
-   * @return the path returned by the exception
-   */
-  private Path expectNonauthNonRecursive(Path path) throws Exception {
-    return intercept(
-        AuthoritativeAuditOperation.NonAuthoritativeDirException.class,
-        () -> auditor.executeAudit(path, true, true))
-        .getPath();
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreScale.java
deleted file mode 100644
index 0f6800f473517..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStoreScale.java
+++ /dev/null
@@ -1,691 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import javax.annotation.Nullable;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
-import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
-import org.assertj.core.api.Assertions;
-import org.junit.Assume;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.internal.AssumptionViolatedException;
-import org.junit.runners.MethodSorters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.StorageStatistics;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.AWSServiceThrottledException;
-import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AStorageStatistics;
-import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.scale.AbstractITestS3AMetadataStoreScale;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.DurationInfo;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.s3guard.MetadataStoreTestBase.basicFileStatus;
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.PARENT;
-import static org.junit.Assume.assumeTrue;
-
-/**
- * Scale test for DynamoDBMetadataStore.
- *
- * The throttle tests aren't quite trying to verify that throttling can
- * be recovered from, because that makes for very slow tests: you have
- * to overload the system and them have them back of until they finally complete.
- * <p>
- * With DDB on demand, throttling is very unlikely.
- * Here the tests simply run to completion, so act as regression tests of
- * parallel invocations on the metastore APIs
- */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class ITestDynamoDBMetadataStoreScale
-    extends AbstractITestS3AMetadataStoreScale {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      ITestDynamoDBMetadataStoreScale.class);
-
-  private static final long BATCH_SIZE = 25;
-
-  /**
-   * IO Units for batch size; this sets the size to use for IO capacity.
-   * Value: {@value}.
-   */
-  private static final long MAXIMUM_READ_CAPACITY = 10;
-  private static final long MAXIMUM_WRITE_CAPACITY = 15;
-
-  /**
-   * Time in milliseconds to sleep after a test throttled:
-   * {@value}.
-   * This is to help isolate throttling to the test which failed,
-   * rather than have it surface in a followup test.
-   * Also the test reports will record durations more accurately,
-   * as JUnit doesn't include setup/teardown times in its reports.
-   * There's a cost: single test runs will sleep, and the last test
-   * run may throttle when it doesn't need to.
-   * The last test {}@link {@link #test_999_delete_all_entries()}
-   * doesn't do the sleep so a full batch run should not suffer here.
-   */
-  public static final int THROTTLE_RECOVER_TIME_MILLIS = 5_000;
-
-  private DynamoDBMetadataStore ddbms;
-  private DynamoDBMetadataStoreTableManager tableHandler;
-
-  private DynamoDB ddb;
-
-  private Table table;
-
-  private String tableName;
-
-  /** was the provisioning changed in test_001_limitCapacity()? */
-  private boolean isOverProvisionedForTest;
-
-  private ProvisionedThroughputDescription originalCapacity;
-
-  private static final int THREADS = 40;
-
-  private static final int OPERATIONS_PER_THREAD = 50;
-
-  private boolean isOnDemandTable;
-
-  /**
-   * Create the metadata store. The table and region are determined from
-   * the attributes of the FS used in the tests.
-   * @return a new metadata store instance
-   * @throws IOException failure to instantiate
-   * @throws AssumptionViolatedException if the FS isn't running S3Guard + DDB/
-   */
-  @Override
-  public DynamoDBMetadataStore createMetadataStore() throws IOException {
-    S3AFileSystem fs = getFileSystem();
-    assumeTrue("S3Guard is disabled for " + fs.getUri(),
-        fs.hasMetadataStore());
-    MetadataStore store = fs.getMetadataStore();
-    assumeTrue("Metadata store for " + fs.getUri() + " is " + store
-            + " -not DynamoDBMetadataStore",
-        store instanceof DynamoDBMetadataStore);
-    DDBCapacities capacities = DDBCapacities.extractCapacities(
-        store.getDiagnostics());
-    isOnDemandTable = capacities.isOnDemandTable();
-
-    DynamoDBMetadataStore fsStore = (DynamoDBMetadataStore) store;
-    Configuration conf = new Configuration(fs.getConf());
-
-    tableName = fsStore.getTableName();
-    assertTrue("Null/Empty tablename in " + fsStore,
-        StringUtils.isNotEmpty(tableName));
-    String region = fsStore.getRegion();
-    assertTrue("Null/Empty region in " + fsStore,
-        StringUtils.isNotEmpty(region));
-    // create a new metastore configured to fail fast if throttling
-    // happens.
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-    conf.set(S3GUARD_DDB_REGION_KEY, region);
-    conf.set(S3GUARD_DDB_THROTTLE_RETRY_INTERVAL, "50ms");
-    conf.set(S3GUARD_DDB_MAX_RETRIES, "1");
-    conf.set(MAX_ERROR_RETRIES, "1");
-    conf.set(S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY, "5ms");
-
-    DynamoDBMetadataStore ms = new DynamoDBMetadataStore();
-    // init the metastore in a bigger retry loop than the test setup
-    // in case the previous test case overloaded things
-    final Invoker fsInvoker = fs.createStoreContext().getInvoker();
-    fsInvoker.retry("init metastore", null, true,
-        () -> ms.initialize(conf, new S3Guard.TtlTimeProvider(conf)));
-    // wire up the owner FS so that we can make assertions about throttle
-    // events
-    ms.bindToOwnerFilesystem(fs);
-    return ms;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    ddbms = (DynamoDBMetadataStore) createMetadataStore();
-    tableName = ddbms.getTableName();
-    tableHandler = ddbms.getTableHandler();
-    assertNotNull("table has no name", tableName);
-    ddb = ddbms.getDynamoDB();
-    table = ddb.getTable(tableName);
-    originalCapacity = table.describe().getProvisionedThroughput();
-
-    // is this table too big for throttling to surface?
-    isOverProvisionedForTest = (
-        originalCapacity.getReadCapacityUnits() > MAXIMUM_READ_CAPACITY
-            || originalCapacity.getWriteCapacityUnits() > MAXIMUM_WRITE_CAPACITY);
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    IOUtils.cleanupWithLogger(LOG, ddbms);
-    super.teardown();
-  }
-
-  /**
-   * Is throttling likely?
-   * @return true if the DDB table has prepaid IO and is small enough
-   * to throttle.
-   */
-  private boolean expectThrottling() {
-    return !isOverProvisionedForTest && !isOnDemandTable;
-  }
-
-  /**
-   * Recover from throttling by sleeping briefly.
-   */
-  private void recoverFromThrottling() throws InterruptedException {
-    LOG.info("Sleeping to recover from throttling for {} ms",
-        THROTTLE_RECOVER_TIME_MILLIS);
-    Thread.sleep(THROTTLE_RECOVER_TIME_MILLIS);
-  }
-
-  /**
-   * The subclass expects the superclass to be throttled; sometimes it is.
-   */
-  @Test
-  @Override
-  public void test_010_Put() throws Throwable {
-    ThrottleTracker tracker = new ThrottleTracker(ddbms);
-    try {
-      // if this doesn't throttle, all is well.
-      super.test_010_Put();
-    } catch (AWSServiceThrottledException ex) {
-      // if the service was throttled, all is good.
-      // log and continue
-      LOG.warn("DDB connection was throttled", ex);
-    } finally {
-      LOG.info("Statistics {}", tracker);
-    }
-  }
-
-  /**
-   * The subclass expects the superclass to be throttled; sometimes it is.
-   */
-  @Test
-  @Override
-  public void test_020_Moves() throws Throwable {
-    ThrottleTracker tracker = new ThrottleTracker(ddbms);
-    try {
-      // if this doesn't throttle, all is well.
-      super.test_020_Moves();
-    } catch (AWSServiceThrottledException ex) {
-      // if the service was throttled, all is good.
-      // log and continue
-      LOG.warn("DDB connection was throttled", ex);
-    } finally {
-      LOG.info("Statistics {}", tracker);
-    }
-  }
-
-  /**
-   * Though the AWS SDK claims in documentation to handle retries and
-   * exponential backoff, we have witnessed
-   * com.amazonaws...dynamodbv2.model.ProvisionedThroughputExceededException
-   * (Status Code: 400; Error Code: ProvisionedThroughputExceededException)
-   * Hypothesis:
-   * Happens when the size of a batched write is bigger than the number of
-   * provisioned write units.  This test ensures we handle the case
-   * correctly, retrying w/ smaller batch instead of surfacing exceptions.
-   */
-  @Test
-  public void test_030_BatchedWrite() throws Exception {
-
-    final int iterations = 15;
-    final ArrayList<PathMetadata> toCleanup = new ArrayList<>();
-    toCleanup.ensureCapacity(THREADS * iterations);
-
-    // Fail if someone changes a constant we depend on
-    assertTrue("Maximum batch size must big enough to run this test",
-        S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT >= BATCH_SIZE);
-
-
-    // We know the dynamodb metadata store will expand a put of a path
-    // of depth N into a batch of N writes (all ancestors are written
-    // separately up to the root).  (Ab)use this for an easy way to write
-    // a batch of stuff that is bigger than the provisioned write units
-    try {
-      describe("Running %d iterations of batched put, size %d", iterations,
-          BATCH_SIZE);
-      Path base = path(getMethodName());
-      final String pathKey = base.toUri().getPath();
-
-      ThrottleTracker result = execute("prune",
-          1,
-          expectThrottling(),
-          () -> {
-            ThrottleTracker tracker = new ThrottleTracker(ddbms);
-            long pruneItems = 0;
-            for (long i = 0; i < iterations; i++) {
-              Path longPath = pathOfDepth(BATCH_SIZE,
-                  pathKey, String.valueOf(i));
-              S3AFileStatus status = basicFileStatus(longPath, 0, false,
-                  12345);
-              PathMetadata pm = new PathMetadata(status);
-              synchronized (toCleanup) {
-                toCleanup.add(pm);
-              }
-
-              ddbms.put(pm, null);
-
-              pruneItems++;
-
-              if (pruneItems == BATCH_SIZE) {
-                describe("pruning files");
-                ddbms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME,
-                    Long.MAX_VALUE, pathKey);
-                pruneItems = 0;
-              }
-              if (tracker.probe()) {
-                // fail fast
-                break;
-              }
-            }
-          });
-      if (expectThrottling() && result.probeThrottlingDetected()) {
-        recoverFromThrottling();
-      }
-    } finally {
-      describe("Cleaning up table %s", tableName);
-      for (PathMetadata pm : toCleanup) {
-        cleanupMetadata(ddbms, pm);
-      }
-    }
-  }
-
-  /**
-   * Test Get throttling including using
-   * {@link MetadataStore#get(Path, boolean)},
-   * as that stresses more of the code.
-   */
-  @Test
-  public void test_040_get() throws Throwable {
-    // attempt to create many many get requests in parallel.
-    Path path = new Path("s3a://example.org/get");
-    S3AFileStatus status = new S3AFileStatus(true, path, "alice");
-    PathMetadata metadata = new PathMetadata(status);
-    ddbms.put(metadata, null);
-    try {
-      execute("get",
-          OPERATIONS_PER_THREAD,
-          expectThrottling(),
-          () -> ddbms.get(path, true)
-      );
-    } finally {
-      retryingDelete(path);
-    }
-  }
-
-  /**
-   * Ask for the version marker, which is where table init can be overloaded.
-   */
-  @Test
-  public void test_050_getVersionMarkerItem() throws Throwable {
-    execute("get",
-        OPERATIONS_PER_THREAD * 2,
-        expectThrottling(),
-        () -> {
-          try {
-            tableHandler.getVersionMarkerItem();
-          } catch (FileNotFoundException ignored) {
-          }
-        }
-    );
-  }
-
-  /**
-   * Cleanup with an extra bit of retry logic around it, in case things
-   * are still over the limit.
-   * @param path path
-   */
-  private void retryingDelete(final Path path) {
-    try {
-      ddbms.getInvoker().retry("Delete ", path.toString(), true,
-          () -> ddbms.delete(path, null));
-    } catch (IOException e) {
-      LOG.warn("Failed to delete {}: ", path, e);
-    }
-  }
-
-  @Test
-  public void test_060_list() throws Throwable {
-    // attempt to create many many get requests in parallel.
-    Path path = new Path("s3a://example.org/list");
-    S3AFileStatus status = new S3AFileStatus(true, path, "alice");
-    PathMetadata metadata = new PathMetadata(status);
-    ddbms.put(metadata, null);
-    try {
-      Path parent = path.getParent();
-      execute("list",
-          OPERATIONS_PER_THREAD,
-          expectThrottling(),
-          () -> ddbms.listChildren(parent)
-      );
-    } finally {
-      retryingDelete(path);
-    }
-  }
-
-  @Test
-  public void test_070_putDirMarker() throws Throwable {
-    // attempt to create many many get requests in parallel.
-    Path path = new Path("s3a://example.org/putDirMarker");
-    S3AFileStatus status = new S3AFileStatus(true, path, "alice");
-    PathMetadata metadata = new PathMetadata(status);
-    ddbms.put(metadata, null);
-    DirListingMetadata children = ddbms.listChildren(path.getParent());
-    try (DynamoDBMetadataStore.AncestorState state =
-             ddbms.initiateBulkWrite(
-                 BulkOperationState.OperationType.Put,
-                 path)) {
-      execute("list",
-          OPERATIONS_PER_THREAD,
-          expectThrottling(),
-          () -> ddbms.put(children, Collections.emptyList(), state));
-    } finally {
-      retryingDelete(path);
-    }
-  }
-
-  @Test
-  public void test_080_fullPathsToPut() throws Throwable {
-    // attempt to create many many get requests in parallel.
-    Path base = new Path("s3a://example.org/test_080_fullPathsToPut");
-    Path child = new Path(base, "child");
-    List<PathMetadata> pms = new ArrayList<>();
-    try {
-      try (BulkOperationState bulkUpdate
-              = ddbms.initiateBulkWrite(
-                  BulkOperationState.OperationType.Put, child)) {
-        ddbms.put(new PathMetadata(makeDirStatus(base)), bulkUpdate);
-        ddbms.put(new PathMetadata(makeDirStatus(child)), bulkUpdate);
-        ddbms.getInvoker().retry("set up directory tree",
-            base.toString(),
-            true,
-            () -> ddbms.put(pms, bulkUpdate));
-      }
-      try (BulkOperationState bulkUpdate
-              = ddbms.initiateBulkWrite(
-                  BulkOperationState.OperationType.Put, child)) {
-        DDBPathMetadata dirData = ddbms.get(child, true);
-        execute("put",
-            OPERATIONS_PER_THREAD,
-            expectThrottling(),
-            () -> ddbms.fullPathsToPut(dirData, bulkUpdate)
-        );
-      }
-    } finally {
-      ddbms.forgetMetadata(child);
-      ddbms.forgetMetadata(base);
-    }
-  }
-
-  /**
-   * Try many deletes in parallel; this will create tombstones.
-   */
-  @Test
-  public void test_090_delete() throws Throwable {
-    Path path = new Path("s3a://example.org/delete");
-    S3AFileStatus status = new S3AFileStatus(true, path, "alice");
-    PathMetadata metadata = new PathMetadata(status);
-    ddbms.put(metadata, null);
-    ITtlTimeProvider time = checkNotNull(getTtlTimeProvider(), "time provider");
-
-    try (DurationInfo ignored = new DurationInfo(LOG, true, "delete")) {
-      execute("delete",
-          OPERATIONS_PER_THREAD,
-          expectThrottling(),
-          () -> {
-            ddbms.delete(path, null);
-          });
-    }
-  }
-
-  /**
-   * Forget Metadata: delete entries without tombstones.
-   */
-  @Test
-  public void test_100_forgetMetadata() throws Throwable {
-    Path path = new Path("s3a://example.org/delete");
-    try (DurationInfo ignored = new DurationInfo(LOG, true, "delete")) {
-      execute("delete",
-          OPERATIONS_PER_THREAD,
-          expectThrottling(),
-          () -> ddbms.forgetMetadata(path)
-      );
-    }
-  }
-
-  @Test
-  public void test_900_instrumentation() throws Throwable {
-    describe("verify the owner FS gets updated after throttling events");
-    Assume.assumeTrue("No throttling expected", expectThrottling());
-    // we rely on the FS being shared
-    S3AFileSystem fs = getFileSystem();
-    String fsSummary = fs.toString();
-
-    S3AStorageStatistics statistics = fs.getStorageStatistics();
-    for (StorageStatistics.LongStatistic statistic : statistics) {
-      LOG.info("{}", statistic.toString());
-    }
-    String retryKey = Statistic.S3GUARD_METADATASTORE_RETRY.getSymbol();
-    assertTrue("No increment of " + retryKey + " in " + fsSummary,
-        statistics.getLong(retryKey) > 0);
-    String throttledKey = Statistic.S3GUARD_METADATASTORE_THROTTLED.getSymbol();
-    assertTrue("No increment of " + throttledKey + " in " + fsSummary,
-        statistics.getLong(throttledKey) > 0);
-  }
-
-  @Test
-  public void test_999_delete_all_entries() throws Throwable {
-    describe("Delete all entries from the table");
-    S3GuardTableAccess tableAccess = new S3GuardTableAccess(ddbms);
-    ExpressionSpecBuilder builder = new ExpressionSpecBuilder();
-    final String path = "/test/";
-    builder.withCondition(
-        ExpressionSpecBuilder.S(PARENT).beginsWith(path));
-    Iterable<DDBPathMetadata> entries =
-        ddbms.wrapWithRetries(tableAccess.scanMetadata(builder));
-    List<Path> list = new ArrayList<>();
-    try {
-      entries.iterator().forEachRemaining(e -> {
-        Path p = e.getFileStatus().getPath();
-        LOG.info("Deleting {}", p);
-        list.add(p);
-      });
-    } catch (UncheckedIOException e) {
-      // the iterator may have overloaded; swallow if so.
-      if (!(e.getCause() instanceof AWSServiceThrottledException)) {
-        throw e;
-      }
-    }
-    // sending this in one by one for more efficient retries
-    for (Path p : list) {
-      ddbms.getInvoker()
-          .retry("delete",
-              path,
-              true,
-              () -> tableAccess.delete(p));
-    }
-  }
-
-  /**
-   * Execute a set of operations in parallel, collect throttling statistics
-   * and return them.
-   * This execution will complete as soon as throttling is detected.
-   * This ensures that the tests do not run for longer than they should.
-   * @param operation string for messages.
-   * @param operationsPerThread number of times per thread to invoke the action.
-   * @param expectThrottling is throttling expected (and to be asserted on?)
-   * @param action action to invoke.
-   * @return the throttle statistics
-   */
-  public ThrottleTracker execute(String operation,
-      int operationsPerThread,
-      final boolean expectThrottling,
-      LambdaTestUtils.VoidCallable action)
-      throws Exception {
-
-    final ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
-    final ThrottleTracker tracker = new ThrottleTracker(ddbms);
-    final ExecutorService executorService = Executors.newFixedThreadPool(
-        THREADS);
-    final List<Callable<ExecutionOutcome>> tasks = new ArrayList<>(THREADS);
-
-    final AtomicInteger throttleExceptions = new AtomicInteger(0);
-    for (int i = 0; i < THREADS; i++) {
-      tasks.add(
-          () -> {
-            final ExecutionOutcome outcome = new ExecutionOutcome();
-            final ContractTestUtils.NanoTimer t
-                = new ContractTestUtils.NanoTimer();
-            for (int j = 0; j < operationsPerThread; j++) {
-              if (tracker.isThrottlingDetected()
-                  || throttleExceptions.get() > 0) {
-                outcome.skipped = true;
-                return outcome;
-              }
-              try {
-                action.call();
-                outcome.completed++;
-              } catch (AWSServiceThrottledException e) {
-                // this is possibly OK
-                LOG.info("Operation [{}] raised a throttled exception " + e, j, e);
-                LOG.debug(e.toString(), e);
-                throttleExceptions.incrementAndGet();
-                // consider it completed
-                outcome.throttleExceptions.add(e);
-                outcome.throttled++;
-              } catch (Exception e) {
-                LOG.error("Failed to execute {}", operation, e);
-                outcome.exceptions.add(e);
-                break;
-              }
-              tracker.probe();
-            }
-            LOG.info("Thread completed {} with in {} ms with outcome {}: {}",
-                operation, t.elapsedTimeMs(), outcome, tracker);
-            return outcome;
-          }
-      );
-    }
-    final List<Future<ExecutionOutcome>> futures =
-        executorService.invokeAll(tasks,
-        getTestTimeoutMillis(), TimeUnit.MILLISECONDS);
-    long elapsedMs = timer.elapsedTimeMs();
-    LOG.info("Completed {} with {}", operation, tracker);
-    LOG.info("time to execute: {} millis", elapsedMs);
-
-    Assertions.assertThat(futures)
-        .describedAs("Futures of all tasks")
-        .allMatch(Future::isDone);
-    tracker.probe();
-    if (expectThrottling() && tracker.probeThrottlingDetected()) {
-      recoverFromThrottling();
-    }
-    for (Future<ExecutionOutcome> future : futures) {
-
-      ExecutionOutcome outcome = future.get();
-      if (!outcome.exceptions.isEmpty()) {
-        throw outcome.exceptions.get(0);
-      }
-      if (!outcome.skipped) {
-        assertEquals("Future did not complete all operations",
-            operationsPerThread, outcome.completed + outcome.throttled);
-      }
-    }
-
-    return tracker;
-  }
-
-  /**
-   * Attempt to delete metadata, suppressing any errors, and retrying on
-   * throttle events just in case some are still surfacing.
-   * @param ms store
-   * @param pm path to clean up
-   */
-  private void cleanupMetadata(DynamoDBMetadataStore ms, PathMetadata pm) {
-    Path path = pm.getFileStatus().getPath();
-    try {
-      ITestDynamoDBMetadataStore.deleteMetadataUnderPath(ms, path, true);
-    } catch (IOException ioe) {
-      // Ignore.
-      LOG.info("Ignoring error while cleaning up {} in database", path, ioe);
-    }
-  }
-
-  private Path pathOfDepth(long n,
-      String name, @Nullable String fileSuffix) {
-    StringBuilder sb = new StringBuilder();
-    for (long i = 0; i < n; i++) {
-      sb.append(i == 0 ? "/" + name : String.format("level-%03d", i));
-      if (i == n - 1 && fileSuffix != null) {
-        sb.append(fileSuffix);
-      }
-      sb.append("/");
-    }
-    return new Path(getFileSystem().getUri().toString(), sb.toString());
-  }
-
-  /**
-   * Outcome of a thread's execution operation.
-   */
-  private static class ExecutionOutcome {
-    private int completed;
-    private int throttled;
-    private boolean skipped;
-    private final List<Exception> exceptions = new ArrayList<>(1);
-    private final List<Exception> throttleExceptions = new ArrayList<>(1);
-
-    @Override
-    public String toString() {
-      final StringBuilder sb = new StringBuilder(
-          "ExecutionOutcome{");
-      sb.append("completed=").append(completed);
-      sb.append(", skipped=").append(skipped);
-      sb.append(", throttled=").append(throttled);
-      sb.append(", exception count=").append(exceptions.size());
-      sb.append('}');
-      return sb.toString();
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
deleted file mode 100644
index 0b55e24e5a955..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import org.junit.Assume;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_READ_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY;
-
-/**
- * Tests concurrent operations on S3Guard.
- */
-public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
-
-  @Rule
-  public final Timeout timeout = new Timeout(5 * 60 * 1000);
-
-  protected Configuration createConfiguration() {
-    Configuration conf =  super.createConfiguration();
-    //patch the read/write capacity
-    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, 0);
-    conf.setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, 0);
-    return conf;
-  }
-
-  private void failIfTableExists(DynamoDB db, String tableName) {
-    boolean tableExists = true;
-    try {
-      Table table = db.getTable(tableName);
-      table.describe();
-    } catch (ResourceNotFoundException e) {
-      tableExists = false;
-    }
-    if (tableExists) {
-      fail("Table already exists: " + tableName);
-    }
-  }
-
-  private void deleteTable(DynamoDB db, String tableName) throws
-      InterruptedException {
-    try {
-      Table table = db.getTable(tableName);
-      table.waitForActive();
-      table.delete();
-      table.waitForDelete();
-    } catch (ResourceNotFoundException e) {
-      LOG.warn("Failed to delete {}, as it was not found", tableName, e);
-    }
-  }
-
-  @Test
-  public void testConcurrentTableCreations() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    final Configuration conf = fs.getConf();
-    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
-        conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
-            Constants.S3GUARD_METASTORE_DYNAMO));
-
-    AWSCredentialProviderList sharedCreds =
-        fs.shareCredentials("testConcurrentTableCreations");
-    // close that shared copy.
-    sharedCreds.close();
-    // this is the original reference count.
-    int originalRefCount = sharedCreds.getRefCount();
-
-    //now init the store; this should increment the ref count.
-    DynamoDBMetadataStore ms = new DynamoDBMetadataStore();
-    ms.initialize(fs, new S3Guard.TtlTimeProvider(conf));
-
-    // the ref count should have gone up
-    assertEquals("Credential Ref count unchanged after initializing metastore "
-        + sharedCreds,
-        originalRefCount + 1, sharedCreds.getRefCount());
-    try {
-      DynamoDB db = ms.getDynamoDB();
-
-      String tableName =
-          getTestTableName("testConcurrentTableCreations" +
-              new Random().nextInt());
-      conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
-      conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
-
-      String region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
-      if (StringUtils.isEmpty(region)) {
-        // no region set, so pick it up from the test bucket
-        conf.set(S3GUARD_DDB_REGION_KEY, fs.getBucketLocation());
-      }
-      int concurrentOps = 16;
-      int iterations = 4;
-
-      failIfTableExists(db, tableName);
-
-      for (int i = 0; i < iterations; i++) {
-        ExecutorService executor = Executors.newFixedThreadPool(
-            concurrentOps, new ThreadFactory() {
-              private AtomicInteger count = new AtomicInteger(0);
-
-              public Thread newThread(Runnable r) {
-                return new Thread(r,
-                    "testConcurrentTableCreations" + count.getAndIncrement());
-              }
-            });
-        ((ThreadPoolExecutor) executor).prestartAllCoreThreads();
-        Future<Exception>[] futures = new Future[concurrentOps];
-        for (int f = 0; f < concurrentOps; f++) {
-          final int index = f;
-          futures[f] = executor.submit(new Callable<Exception>() {
-            @Override
-            public Exception call() throws Exception {
-
-              ContractTestUtils.NanoTimer timer =
-                  new ContractTestUtils.NanoTimer();
-
-              Exception result = null;
-              try (DynamoDBMetadataStore store = new DynamoDBMetadataStore()) {
-                store.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-              } catch (Exception e) {
-                LOG.error(e.getClass() + ": " + e.getMessage());
-                result = e;
-              }
-
-              timer.end("Parallel DynamoDB client creation %d", index);
-              LOG.info("Parallel DynamoDB client creation {} ran from {} to {}",
-                  index, timer.getStartTime(), timer.getEndTime());
-              return result;
-            }
-          });
-        }
-        List<Exception> exceptions = new ArrayList<>(concurrentOps);
-        for (int f = 0; f < concurrentOps; f++) {
-          Exception outcome = futures[f].get();
-          if (outcome != null) {
-            exceptions.add(outcome);
-          }
-        }
-        deleteTable(db, tableName);
-        int exceptionsThrown = exceptions.size();
-        if (exceptionsThrown > 0) {
-          // at least one exception was thrown. Fail the test & nest the first
-          // exception caught
-          throw new AssertionError(exceptionsThrown + "/" + concurrentOps +
-              " threads threw exceptions while initializing on iteration " + i,
-              exceptions.get(0));
-        }
-      }
-    } finally {
-      ms.close();
-    }
-    assertEquals("Credential Ref count unchanged after closing metastore: "
-            + sharedCreds,
-        originalRefCount, sharedCreds.getRefCount());
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java
deleted file mode 100644
index f03d4e0924775..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardDDBRootOperations.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.File;
-import java.net.URI;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.assertj.core.api.Assertions;
-import org.junit.FixMethodOrder;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
-import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_DELETE;
-import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3AUtils.applyLocatedFiles;
-
-/**
- * This test run against the root of the FS, and operations which span the DDB
- * table and the filesystem.
- * For this reason, these tests are executed in the sequential phase of the
- * integration tests.
- * <p>
- * The tests only run if DynamoDB is the metastore.
- * <p></p>
- * The marker policy is fixed to "delete"
- */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public class ITestS3GuardDDBRootOperations extends AbstractS3ATestBase {
-
-  private StoreContext storeContext;
-
-  private String fsUriStr;
-
-  private DynamoDBMetadataStore metastore;
-
-  private String metastoreUriStr;
-
-  // this is a switch you can change in your IDE to enable
-  // or disable those tests which clean up the metastore.
-  private final boolean cleaning = true;
-
-  /**
-   * The test timeout is increased in case previous tests have created
-   * many tombstone markers which now need to be purged.
-   * @return the test timeout.
-   */
-  @Override
-  protected int getTestTimeoutMillis() {
-    return SCALE_TEST_TIMEOUT_SECONDS * 1000;
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    String bucketName = getTestBucketName(conf);
-    disableFilesystemCaching(conf);
-
-    removeBucketOverrides(bucketName, conf,
-        S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
-        ENABLE_MULTI_DELETE,
-        DIRECTORY_MARKER_POLICY);
-    conf.set(DIRECTORY_MARKER_POLICY,
-        DIRECTORY_MARKER_POLICY_DELETE);
-    // set a sleep time of 0 on pruning, for speedier test runs.
-    conf.setTimeDuration(
-        S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY,
-        0,
-        TimeUnit.MILLISECONDS);
-    return conf;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    S3AFileSystem fs = getFileSystem();
-    Configuration conf = fs.getConf();
-    S3ATestUtils.assumeS3GuardState(true, conf);
-    storeContext = fs.createStoreContext();
-    assume("Filesystem isn't running DDB",
-        storeContext.getMetadataStore() instanceof DynamoDBMetadataStore);
-    metastore = (DynamoDBMetadataStore) storeContext.getMetadataStore();
-    URI fsURI = storeContext.getFsURI();
-    fsUriStr = fsURI.toString();
-    if (!fsUriStr.endsWith("/")) {
-      fsUriStr = fsUriStr + "/";
-    }
-    metastoreUriStr = "dynamodb://" + metastore.getTableName() + "/";
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    Thread.currentThread().setName("teardown");
-    super.teardown();
-  }
-
-  private void assumeCleaningOperation() {
-    assume("Cleaning operation skipped", cleaning);
-  }
-
-  @Test
-  @Ignore
-  public void test_050_dump_metastore() throws Throwable {
-    File destFile = calculateDumpFileBase();
-    describe("Dumping S3Guard store under %s", destFile);
-    DumpS3GuardDynamoTable.dumpStore(
-        null,
-        metastore,
-        getConfiguration(),
-        destFile,
-        getFileSystem().getUri());
-  }
-
-  @Test
-  public void test_060_dump_metastore_and_s3() throws Throwable {
-    File destFile = calculateDumpFileBase();
-    describe("Dumping S3Guard store under %s", destFile);
-    DumpS3GuardDynamoTable.dumpStore(
-        getFileSystem(),
-        metastore,
-        getConfiguration(),
-        destFile,
-        getFileSystem().getUri());
-  }
-
-  @Test
-  public void test_100_FilesystemPrune() throws Throwable {
-    describe("Execute prune against a filesystem URI");
-    assumeCleaningOperation();
-    S3AFileSystem fs = getFileSystem();
-    Configuration conf = fs.getConf();
-    int result = S3GuardTool.run(conf,
-        S3GuardTool.Prune.NAME,
-        "-seconds", "1",
-        fsUriStr);
-    Assertions.assertThat(result)
-        .describedAs("Result of prune %s", fsUriStr)
-        .isEqualTo(0);
-  }
-
-
-  @Test
-  public void test_200_MetastorePruneTombstones() throws Throwable {
-    describe("Execute prune against a dynamo URL");
-    assumeCleaningOperation();
-    S3AFileSystem fs = getFileSystem();
-    Configuration conf = fs.getConf();
-    int result = S3GuardTool.run(conf,
-        S3GuardTool.Prune.NAME,
-        "-tombstone",
-        "-meta", checkNotNull(metastoreUriStr),
-        "-seconds", "1",
-        fs.qualify(new Path("/")).toString());
-    Assertions.assertThat(result)
-        .describedAs("Result of prune %s", fsUriStr)
-        .isEqualTo(0);
-  }
-
-  @Test
-  public void test_300_MetastorePrune() throws Throwable {
-    describe("Execute prune against a dynamo URL");
-    assumeCleaningOperation();
-    S3AFileSystem fs = getFileSystem();
-    Configuration conf = fs.getConf();
-    int result = S3GuardTool.run(conf,
-        S3GuardTool.Prune.NAME,
-        "-meta", checkNotNull(metastoreUriStr),
-        "-region", fs.getBucketLocation(),
-        "-seconds", "1");
-    Assertions.assertThat(result)
-        .describedAs("Result of prune %s", fsUriStr)
-        .isEqualTo(0);
-  }
-
-  @Test
-  public void test_400_rm_root_recursive() throws Throwable {
-    describe("Remove the root directory");
-    assumeCleaningOperation();
-    S3AFileSystem fs = getFileSystem();
-    Path root = new Path("/");
-    Path file = new Path("/test_400_rm_root_recursive-01");
-    Path file2 = new Path("/test_400_rm_root_recursive-02");
-    // recursive treewalk to delete all files
-    // does not delete directories.
-    applyLocatedFiles(fs.listFilesAndEmptyDirectories(root, true),
-        f -> {
-          Path p = f.getPath();
-          fs.delete(p, true);
-          assertPathDoesNotExist("expected file to be deleted", p);
-        });
-    ContractTestUtils.deleteChildren(fs, root, true);
-    // everything must be done by now
-    StringBuffer sb = new StringBuffer();
-    AtomicInteger foundFile = new AtomicInteger(0);
-    applyLocatedFiles(fs.listFilesAndEmptyDirectories(root, true),
-        f -> {
-          foundFile.addAndGet(1);
-          Path p = f.getPath();
-          sb.append(f.isDirectory()
-              ? "Dir  "
-              : "File ")
-            .append(p);
-          if (!f.isDirectory()) {
-            sb.append("[").append(f.getLen()).append("]");
-          }
-
-          fs.delete(p, true);
-        });
-
-    assertEquals("Remaining files " + sb,
-        0, foundFile.get());
-    try {
-      ContractTestUtils.touch(fs, file);
-      assertDeleted(file, false);
-
-
-      assertFalse("Root directory delete failed",
-          fs.delete(root, true));
-
-      ContractTestUtils.touch(fs, file2);
-      assertFalse("Root directory delete should have failed",
-          fs.delete(root, true));
-    } finally {
-      fs.delete(file, false);
-      fs.delete(file2, false);
-    }
-  }
-
-  @Test
-  @Ignore
-  public void test_600_dump_metastore() throws Throwable {
-    File destFile = calculateDumpFileBase();
-    describe("Dumping S3Guard store under %s", destFile);
-    DumpS3GuardDynamoTable.dumpStore(
-        getFileSystem(),
-        metastore,
-        getConfiguration(),
-        destFile,
-        getFileSystem().getUri());
-  }
-
-  protected File calculateDumpFileBase() {
-    String target = System.getProperty("test.build.dir", "target");
-    File buildDir = new File(target,
-        this.getClass().getSimpleName()).getAbsoluteFile();
-    buildDir.mkdirs();
-    return new File(buildDir, getMethodName());
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsShell.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsShell.java
deleted file mode 100644
index 51b23af4dc72d..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsShell.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.Arrays;
-import java.util.stream.Collectors;
-
-import org.junit.Test;
-
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.util.ToolRunner;
-
-/**
- * Test FS shell and S3Guard (and of course, the rest of S3).
- */
-public class ITestS3GuardFsShell extends AbstractS3ATestBase {
-
-  /**
-   * Run a shell command.
-   * @param args array of arguments.
-   * @return the exit code.
-   * @throws Exception any exception raised.
-   */
-  private int fsShell(String[] args) throws Exception {
-    FsShell shell = new FsShell(getConfiguration());
-    try {
-      return ToolRunner.run(shell, args);
-    } finally {
-      shell.close();
-    }
-  }
-
-  /**
-   * Execute a command and verify that it returned the specific exit code.
-   * @param expected expected value
-   * @param args array of arguments.
-   * @throws Exception any exception raised.
-   */
-  private void exec(int expected, String[] args) throws Exception {
-    int result = fsShell(args);
-    String argslist = Arrays.stream(args).collect(Collectors.joining(" "));
-    assertEquals("hadoop fs " + argslist, expected, result);
-  }
-
-  /**
-   * Execute a shell command expecting a result of 0.
-   * @param args array of arguments.
-   * @throws Exception any exception raised.
-   */
-  private void exec(String[] args) throws Exception {
-    exec(0, args);
-  }
-
-  /**
-   * Issue a mkdir without a trailing /.
-   */
-  @Test
-  public void testMkdirNoTrailing() throws Throwable {
-    Path dest = path("normal");
-    try {
-      String destStr = dest.toString();
-      mkdirs(destStr);
-      isDir(destStr);
-      rmdir(destStr);
-      isNotFound(destStr);
-    } finally {
-      getFileSystem().delete(dest, true);
-    }
-  }
-
-  /**
-   * Issue a mkdir with a trailing /.
-   */
-  @Test
-  public void testMkdirTrailing() throws Throwable {
-    Path base = path("trailing");
-    getFileSystem().delete(base, true);
-    try {
-      String destStr = base.toString() + "/";
-      mkdirs(destStr);
-      isDir(destStr);
-      isDir(base.toString());
-      rmdir(destStr);
-      isNotFound(destStr);
-    } finally {
-      getFileSystem().delete(base, true);
-    }
-  }
-
-  /**
-   * Create the destination path and then call mkdir, expect it to still work.
-   */
-  @Test
-  public void testMkdirTrailingExists() throws Throwable {
-    Path base = path("trailingexists");
-    getFileSystem().mkdirs(base);
-    try {
-      String destStr = base.toString() + "/";
-      // the path already exists
-      isDir(destStr);
-      mkdirs(destStr);
-      isDir(destStr);
-      rmdir(base.toString());
-      isNotFound(destStr);
-    } finally {
-      getFileSystem().delete(base, true);
-    }
-  }
-
-  private void isNotFound(final String destStr) throws Exception {
-    exec(1, new String[]{"-test", "-d", destStr});
-  }
-
-  private void mkdirs(final String destStr) throws Exception {
-    exec(new String[]{"-mkdir", "-p", destStr});
-  }
-
-  private void rmdir(final String destStr) throws Exception {
-    exec(new String[]{"-rmdir", destStr});
-  }
-
-  private void isDir(final String destStr) throws Exception {
-    exec(new String[]{"-test", "-d", destStr});
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java
deleted file mode 100644
index 28eb52ff41ada..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java
+++ /dev/null
@@ -1,646 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-import org.assertj.core.api.Assertions;
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-
-import static org.apache.hadoop.fs.s3a.Constants.AUTHORITATIVE_PATH;
-import static org.junit.Assume.assumeTrue;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-
-/**
- * Integration tests for the S3Guard Fsck against a dyamodb backed metadata
- * store.
- */
-public class ITestS3GuardFsck extends AbstractS3ATestBase {
-
-  private S3AFileSystem guardedFs;
-  private S3AFileSystem rawFs;
-
-  private MetadataStore metadataStore;
-
-  @Before
-  public void setup() throws Exception {
-    super.setup();
-    S3AFileSystem fs = getFileSystem();
-    // These test will fail if no ms
-    assumeTrue("FS needs to have a DynamoDB metadatastore.",
-        fs.getMetadataStore() instanceof DynamoDBMetadataStore);
-    assumeTrue("Metadatastore should persist authoritative bit",
-        metadataStorePersistsAuthoritativeBit(fs.getMetadataStore()));
-
-    guardedFs = fs;
-    metadataStore = fs.getMetadataStore();
-
-    // create raw fs without s3guard
-    rawFs = createUnguardedFS();
-    assertFalse("Raw FS still has S3Guard " + rawFs,
-        rawFs.hasMetadataStore());
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    if (guardedFs != null) {
-      IOUtils.cleanupWithLogger(LOG, guardedFs);
-    }
-    IOUtils.cleanupWithLogger(LOG, rawFs);
-    super.teardown();
-  }
-
-  /**
-   * Create a test filesystem which is always unguarded.
-   * This filesystem MUST be closed in test teardown.
-   * @return the new FS
-   */
-  private S3AFileSystem createUnguardedFS() throws Exception {
-    S3AFileSystem testFS = getFileSystem();
-    Configuration config = new Configuration(testFS.getConf());
-    URI uri = testFS.getUri();
-
-    removeBaseAndBucketOverrides(uri.getHost(), config,
-        S3_METADATA_STORE_IMPL, METADATASTORE_AUTHORITATIVE,
-        AUTHORITATIVE_PATH);
-    S3AFileSystem fs2 = new S3AFileSystem();
-    fs2.initialize(uri, config);
-    return fs2;
-  }
-
-  @Test
-  public void testIDetectNoMetadataEntry() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchRawAndWaitRaw(file);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 2);
-      final S3GuardFsck.ComparePair pair = comparePairs.get(0);
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.NO_METADATA_ENTRY);
-    } finally {
-      // delete the working directory with all of its contents
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectNoParentEntry() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchGuardedAndWaitRaw(file);
-      // delete the parent from the MS
-      metadataStore.forgetMetadata(cwd);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 2);
-      // check the parent that it does not exist
-      checkForViolationInPairs(cwd, comparePairs,
-          S3GuardFsck.Violation.NO_METADATA_ENTRY);
-      // check the child that there's no parent entry.
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.NO_PARENT_ENTRY);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectParentIsAFile() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchGuardedAndWaitRaw(file);
-      // modify the cwd metadata and set that it's not a directory
-      final S3AFileStatus newParentFile = MetadataStoreTestBase
-          .basicFileStatus(cwd, 1, false, 1);
-      metadataStore.put(new PathMetadata(newParentFile));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 2);
-      // check the parent that it does not exist
-      checkForViolationInPairs(cwd, comparePairs,
-          S3GuardFsck.Violation.DIR_IN_S3_FILE_IN_MS);
-      // check the child that the parent is a file.
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.PARENT_IS_A_FILE);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectParentTombstoned() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchGuardedAndWaitRaw(file);
-      // modify the parent metadata and set that it's not a directory
-      final PathMetadata cwdPmd = metadataStore.get(cwd);
-      cwdPmd.setIsDeleted(true);
-      metadataStore.put(cwdPmd);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      // check the child that the parent is tombstoned
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.PARENT_TOMBSTONED);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectDirInS3FileInMs() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    try {
-      // create a file with guarded fs
-      mkdirs(cwd);
-      // modify the cwd metadata and set that it's not a directory
-      final S3AFileStatus newParentFile = MetadataStoreTestBase
-          .basicFileStatus(cwd, 1, false, 1);
-      metadataStore.put(new PathMetadata(newParentFile));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-      assertComparePairsSize(comparePairs, 1);
-
-      // check the child that the dir in s3 is a file in the ms
-      checkForViolationInPairs(cwd, comparePairs,
-          S3GuardFsck.Violation.DIR_IN_S3_FILE_IN_MS);
-    } finally {
-      cleanup(cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectFileInS3DirInMs() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchGuardedAndWaitRaw(file);
-      // modify the cwd metadata and set that it's not a directory
-      final S3AFileStatus newFile = MetadataStoreTestBase
-          .basicFileStatus(file, 1, true, 1);
-      metadataStore.put(new PathMetadata(newFile));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-      // check the child that the dir in s3 is a file in the ms
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.FILE_IN_S3_DIR_IN_MS);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIAuthoritativeDirectoryContentMismatch() throws Exception {
-    assumeTrue("Authoritative directory listings should be enabled for this "
-            + "test", guardedFs.hasAuthoritativeMetadataStore());
-    // first dir listing will be correct
-    final Path cwdCorrect = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path fileC1 = new Path(cwdCorrect, "fileC1");
-    final Path fileC2 = new Path(cwdCorrect, "fileC2");
-
-    // second dir listing will be incorrect: missing entry from Dynamo
-    final Path cwdIncorrect = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path fileIc1 = new Path(cwdIncorrect, "fileC1");
-    final Path fileIc2 = new Path(cwdIncorrect, "fileC2");
-    try {
-      touchGuardedAndWaitRaw(fileC1);
-      touchGuardedAndWaitRaw(fileC2);
-      touchGuardedAndWaitRaw(fileIc1);
-
-      // get listing from ms and set it authoritative
-      final DirListingMetadata dlmC = metadataStore.listChildren(cwdCorrect);
-      final DirListingMetadata dlmIc = metadataStore.listChildren(cwdIncorrect);
-      dlmC.setAuthoritative(true);
-      dlmIc.setAuthoritative(true);
-      metadataStore.put(dlmC, Collections.emptyList(), null);
-      metadataStore.put(dlmIc, Collections.emptyList(), null);
-
-      // add a file raw so the listing will be different.
-      touchRawAndWaitRaw(fileIc2);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> pairsCorrect =
-          s3GuardFsck.compareS3ToMs(cwdCorrect);
-      final List<S3GuardFsck.ComparePair> pairsIncorrect =
-          s3GuardFsck.compareS3ToMs(cwdIncorrect);
-
-      // Assert that the correct dir does not contain the violation.
-      assertTrue(pairsCorrect.stream()
-          .noneMatch(p -> p.getPath().equals(cwdCorrect)));
-
-      // Assert that the incorrect listing contains the violation.
-      checkForViolationInPairs(cwdIncorrect, pairsIncorrect,
-          S3GuardFsck.Violation.AUTHORITATIVE_DIRECTORY_CONTENT_MISMATCH);
-    } finally {
-      cleanup(fileC1, fileC2, fileIc1, fileIc2, cwdCorrect, cwdIncorrect);
-    }
-  }
-
-  @Test
-  public void testIDetectLengthMismatch() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      // create a file with guarded fs
-      touchGuardedAndWaitRaw(file);
-
-      // modify the file metadata so the length will not match
-      final S3AFileStatus newFile = MetadataStoreTestBase
-          .basicFileStatus(file, 9999, false, 1);
-      metadataStore.put(new PathMetadata(newFile));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-      // Assert that the correct dir does not contain the violation.
-      assertTrue(comparePairs.stream()
-          .noneMatch(p -> p.getPath().equals(cwd)));
-      // Assert that the incorrect file meta contains the violation.
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.LENGTH_MISMATCH);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIDetectModTimeMismatch() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      // create a file with guarded fs
-      touchGuardedAndWaitRaw(file);
-      // modify the parent meta entry so the MOD_TIME will surely be up to date
-      final FileStatus oldCwdFileStatus = rawFs.getFileStatus(cwd);
-      final S3AFileStatus newCwdFileStatus = MetadataStoreTestBase
-          .basicFileStatus(cwd, 0, true,
-              oldCwdFileStatus.getModificationTime());
-      metadataStore.put(new PathMetadata(newCwdFileStatus));
-
-      // modify the file metadata so the length will not match
-      final S3AFileStatus newFileStatus = MetadataStoreTestBase
-          .basicFileStatus(file, 0, false, 1);
-      metadataStore.put(new PathMetadata(newFileStatus));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-      // Assert that the correct dir does not contain the violation.
-      assertTrue(comparePairs.stream()
-          .noneMatch(p -> p.getPath().equals(cwd)));
-      // check the file meta that there's a violation.
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.MOD_TIME_MISMATCH);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testIEtagMismatch() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      touchGuardedAndWaitRaw(file);
-      // modify the file metadata so the etag will not match
-      final S3AFileStatus newFileStatus = new S3AFileStatus(1, 1, file, 1, "",
-          "etag", "versionId");
-      metadataStore.put(new PathMetadata(newFileStatus));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-      // check the child that there's a BLOCKSIZE_MISMATCH
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.ETAG_MISMATCH);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testINoEtag() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file1 = new Path(cwd, "file1");
-    final Path file2 = new Path(cwd, "file2");
-    try {
-      // create a file1 with guarded fs
-      touchGuardedAndWaitRaw(file1);
-      touchGuardedAndWaitRaw(file2);
-      // modify the file1 metadata so there's no etag
-      final S3AFileStatus newFile1Status =
-          new S3AFileStatus(1, 1, file1, 1, "", null, "versionId");
-      final S3AFileStatus newFile2Status =
-          new S3AFileStatus(1, 1, file2, 1, "", "etag", "versionId");
-      metadataStore.put(new PathMetadata(newFile1Status));
-      metadataStore.put(new PathMetadata(newFile2Status));
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 2);
-
-      // check file 1 that there's NO_ETAG
-      checkForViolationInPairs(file1, comparePairs,
-          S3GuardFsck.Violation.NO_ETAG);
-      // check the child that there's no NO_ETAG violation
-      checkNoViolationInPairs(file2, comparePairs,
-          S3GuardFsck.Violation.NO_ETAG);
-    } finally {
-      cleanup(file1, file2, cwd);
-    }
-  }
-
-  @Test
-  public void testTombstonedInMsNotDeletedInS3() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      // create a file with guarded fs
-      touchGuardedAndWaitRaw(file);
-      // set isDeleted flag in ms to true (tombstone item)
-      final PathMetadata fileMeta = metadataStore.get(file);
-      fileMeta.setIsDeleted(true);
-      metadataStore.put(fileMeta);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.compareS3ToMs(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-
-      // check if the violation is there
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.TOMBSTONED_IN_MS_NOT_DELETED_IN_S3);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void checkDdbInternalConsistency() throws Exception {
-    final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-    final DynamoDBMetadataStore ms =
-        (DynamoDBMetadataStore) guardedFs.getMetadataStore();
-    s3GuardFsck.checkDdbInternalConsistency(
-        new Path("s3a://" + guardedFs.getBucket() + "/"));
-  }
-
-  @Test
-  public void testDdbInternalNoLastUpdatedField() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path file = new Path(cwd, "file");
-    try {
-      final S3AFileStatus s3AFileStatus = new S3AFileStatus(100, 100, file, 100,
-          "test", "etag", "version");
-      final PathMetadata pathMetadata = new PathMetadata(s3AFileStatus);
-      pathMetadata.setLastUpdated(0);
-      metadataStore.put(pathMetadata);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.checkDdbInternalConsistency(cwd);
-
-      assertComparePairsSize(comparePairs, 1);
-
-      // check if the violation is there
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.NO_LASTUPDATED_FIELD);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testDdbInternalOrphanEntry() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path parentDir = new Path(cwd, "directory");
-    final Path file = new Path(parentDir, "file");
-    try {
-      final S3AFileStatus s3AFileStatus = new S3AFileStatus(100, 100, file, 100,
-          "test", "etag", "version");
-      final PathMetadata pathMetadata = new PathMetadata(s3AFileStatus);
-      pathMetadata.setLastUpdated(1000);
-      metadataStore.put(pathMetadata);
-      metadataStore.forgetMetadata(parentDir);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.checkDdbInternalConsistency(cwd);
-
-      // check if the violation is there
-      assertComparePairsSize(comparePairs, 1);
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.ORPHAN_DDB_ENTRY);
-
-      // fix the violation
-      s3GuardFsck.fixViolations(
-          comparePairs.stream().filter(cP -> cP.getViolations()
-              .contains(S3GuardFsck.Violation.ORPHAN_DDB_ENTRY))
-              .collect(Collectors.toList())
-      );
-
-      // assert that the violation is fixed
-      final List<S3GuardFsck.ComparePair> fixedComparePairs =
-          s3GuardFsck.checkDdbInternalConsistency(cwd);
-      checkNoViolationInPairs(file, fixedComparePairs,
-          S3GuardFsck.Violation.ORPHAN_DDB_ENTRY);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testDdbInternalParentIsAFile() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path parentDir = new Path(cwd, "directory");
-    final Path file = new Path(parentDir, "file");
-    try {
-      final S3AFileStatus s3AFileStatus = new S3AFileStatus(100, 100, file, 100,
-          "test", "etag", "version");
-      final PathMetadata pathMetadata = new PathMetadata(s3AFileStatus);
-      pathMetadata.setLastUpdated(1000);
-      metadataStore.put(pathMetadata);
-
-      final S3AFileStatus dirAsFile = MetadataStoreTestBase
-          .basicFileStatus(parentDir, 1, false, 1);
-      final PathMetadata dirAsFilePm = new PathMetadata(dirAsFile);
-      dirAsFilePm.setLastUpdated(100);
-      metadataStore.put(dirAsFilePm);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.checkDdbInternalConsistency(cwd);
-
-      // check if the violation is there
-      assertComparePairsSize(comparePairs, 1);
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.PARENT_IS_A_FILE);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  @Test
-  public void testDdbInternalParentTombstoned() throws Exception {
-    final Path cwd = path("/" + getMethodName() + "-" + UUID.randomUUID());
-    final Path parentDir = new Path(cwd, "directory");
-    final Path file = new Path(parentDir, "file");
-    try {
-      final S3AFileStatus s3AFileStatus = new S3AFileStatus(100, 100, file, 100,
-          "test", "etag", "version");
-      final PathMetadata pathMetadata = new PathMetadata(s3AFileStatus);
-      pathMetadata.setLastUpdated(1000);
-      metadataStore.put(pathMetadata);
-      metadataStore.delete(parentDir, null);
-
-      final S3GuardFsck s3GuardFsck = new S3GuardFsck(rawFs, metadataStore);
-      final List<S3GuardFsck.ComparePair> comparePairs =
-          s3GuardFsck.checkDdbInternalConsistency(cwd);
-
-      // check if the violation is there
-      assertComparePairsSize(comparePairs, 1);
-      checkForViolationInPairs(file, comparePairs,
-          S3GuardFsck.Violation.PARENT_TOMBSTONED);
-    } finally {
-      cleanup(file, cwd);
-    }
-  }
-
-  protected void assertComparePairsSize(
-      List<S3GuardFsck.ComparePair> comparePairs, int num) {
-    Assertions.assertThat(comparePairs)
-        .describedAs("Number of compare pairs")
-        .hasSize(num);
-  }
-
-  private void touchGuardedAndWaitRaw(Path file) throws Exception {
-    touchAndWait(guardedFs, rawFs, file);
-  }
-
-  private void touchRawAndWaitRaw(Path file) throws Exception {
-    touchAndWait(rawFs, rawFs, file);
-  }
-
-  private void touchAndWait(FileSystem forTouch, FileSystem forWait, Path file)
-      throws IOException {
-    touch(forTouch, file);
-    touch(forWait, file);
-  }
-
-  private void checkForViolationInPairs(Path file,
-      List<S3GuardFsck.ComparePair> comparePairs,
-      S3GuardFsck.Violation violation) {
-    final S3GuardFsck.ComparePair childPair = comparePairs.stream()
-        .filter(p -> p.getPath().equals(file))
-        .findFirst().get();
-    assertNotNull("The pair should not be null.", childPair);
-    assertTrue("The pair must contain a violation.",
-        childPair.containsViolation());
-    Assertions.assertThat(childPair.getViolations())
-        .describedAs("Violations in the pair")
-        .contains(violation);
-  }
-
-  /**
-   * Check that there is no violation in the pair provided.
-   *
-   * @param file the path to filter to in the comparePairs list.
-   * @param comparePairs the list to validate.
-   * @param violation the violation that should not be in the list.
-   */
-  private void checkNoViolationInPairs(Path file,
-      List<S3GuardFsck.ComparePair> comparePairs,
-      S3GuardFsck.Violation violation) {
-
-    if (comparePairs.size() == 0) {
-      LOG.info("Compare pairs is empty, so there's no violation. (As expected.)");
-      return;
-    }
-
-    final S3GuardFsck.ComparePair comparePair = comparePairs.stream()
-        .filter(p -> p.getPath().equals(file))
-        .findFirst().get();
-    assertNotNull("The pair should not be null.", comparePair);
-    Assertions.assertThat(comparePair.getViolations())
-        .describedAs("Violations in the pair")
-        .doesNotContain(violation);
-  }
-
-  private void cleanup(Path... paths) {
-    for (Path path : paths) {
-      try {
-        metadataStore.forgetMetadata(path);
-        rawFs.delete(path, true);
-      } catch (IOException e) {
-        LOG.error("Error during cleanup.", e);
-      }
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
similarity index 57%
rename from hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
rename to hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
index ecddbbd820872..23b14fd3792ca 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
@@ -21,185 +21,37 @@
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.StringUtils;
-
 import org.junit.Test;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.UnknownStoreException;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
-import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE_DEFAULT;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
-import static org.apache.hadoop.fs.s3a.MultipartTestUtils.*;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.StringUtils;
+
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.countUploadsAt;
+import static org.apache.hadoop.fs.s3a.MultipartTestUtils.createPartUpload;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVFile;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.BucketInfo;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
- * Test S3Guard related CLI commands against a LocalMetadataStore.
- * Also responsible for testing the non s3guard-specific commands that, for
- * now, live under the s3guard CLI command.
+ * Test S3Guard Tool CLI commands.
  */
-public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
+public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
 
-  private static final String LOCAL_METADATA = "local://metadata";
-  private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
+  private static final String[] ABORT_FORCE_OPTIONS = new String[]{
+      "-abort",
       "-force", "-verbose"};
 
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    removeBaseAndBucketOverrides(conf,
-        S3_METADATA_STORE_IMPL, S3A_BUCKET_PROBE);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_LOCAL);
-    conf.setInt(S3A_BUCKET_PROBE, S3A_BUCKET_PROBE_DEFAULT);
-    return conf;
-  }
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    assertTrue("metadata store impl should be LocalMetadataStore.",
-        getMetadataStore() instanceof LocalMetadataStore);
-  }
-
-  @Test
-  public void testImportCommand() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    MetadataStore ms = getMetadataStore();
-    Path parent = path("test-import");
-    fs.mkdirs(parent);
-    Path dir = new Path(parent, "a");
-    fs.mkdirs(dir);
-    Path emptyDir = new Path(parent, "emptyDir");
-    fs.mkdirs(emptyDir);
-    for (int i = 0; i < 10; i++) {
-      String child = String.format("file-%d", i);
-      try (FSDataOutputStream out = fs.create(new Path(dir, child))) {
-        out.write(1);
-      }
-    }
-
-    S3GuardTool.Import cmd = toClose(new S3GuardTool.Import(fs.getConf()));
-    try {
-      cmd.setStore(ms);
-      exec(cmd, "import", parent.toString());
-    } finally {
-      cmd.setStore(new NullMetadataStore());
-    }
-
-    DirListingMetadata children =
-        ms.listChildren(dir);
-    assertEquals("Unexpected number of paths imported", 10, children
-        .getListing().size());
-    assertEquals("Expected 2 items: empty directory and a parent directory", 2,
-        ms.listChildren(parent).getListing().size());
-  }
-
-  @Test
-  public void testImportCommandRepairsETagAndVersionId() throws Exception {
-    S3AFileSystem fs = getFileSystem();
-    MetadataStore ms = getMetadataStore();
-    Path path = path("test-version-metadata");
-    try (FSDataOutputStream out = fs.create(path)) {
-      out.write(1);
-    }
-    S3AFileStatus originalStatus = (S3AFileStatus) fs.getFileStatus(path);
-
-    // put in bogus ETag and versionId
-    S3AFileStatus bogusStatus = S3AFileStatus.fromFileStatus(originalStatus,
-        Tristate.FALSE, "bogusETag", "bogusVersionId");
-    ms.put(new PathMetadata(bogusStatus));
-
-    // sanity check that bogus status is actually persisted
-    S3AFileStatus retrievedBogusStatus = (S3AFileStatus) fs.getFileStatus(path);
-    assertEquals("bogus ETag was not persisted",
-        "bogusETag", retrievedBogusStatus.getETag());
-    assertEquals("bogus versionId was not persisted",
-        "bogusVersionId", retrievedBogusStatus.getVersionId());
-
-    // execute the import
-    S3GuardTool.Import cmd = toClose(new S3GuardTool.Import(fs.getConf()));
-    cmd.setStore(ms);
-    try {
-      exec(cmd, "import", path.toString());
-    } finally {
-      cmd.setStore(new NullMetadataStore());
-    }
-
-    // make sure ETag and versionId were corrected
-    S3AFileStatus updatedStatus = (S3AFileStatus) fs.getFileStatus(path);
-    assertEquals("ETag was not corrected",
-        originalStatus.getETag(), updatedStatus.getETag());
-    assertEquals("VersionId was not corrected",
-        originalStatus.getVersionId(), updatedStatus.getVersionId());
-  }
-
-  @Test
-  public void testDestroyBucketExistsButNoTable() throws Throwable {
-    run(Destroy.NAME,
-        "-meta", LOCAL_METADATA,
-        getLandsatCSVFile(getConfiguration()));
-  }
-
-  @Test
-  public void testImportNoFilesystem() throws Throwable {
-    final Import importer = toClose(new S3GuardTool.Import(getConfiguration()));
-    importer.setStore(getMetadataStore());
-    try {
-      intercept(IOException.class,
-          () -> importer.run(
-              new String[]{
-                  "import",
-                  "-meta", LOCAL_METADATA,
-                  S3A_THIS_BUCKET_DOES_NOT_EXIST
-              }));
-    } finally {
-      importer.setStore(new NullMetadataStore());
-    }
-  }
-
-  @Test
-  public void testInfoBucketAndRegionNoFS() throws Throwable {
-    intercept(UnknownStoreException.class,
-        () -> run(BucketInfo.NAME, "-meta",
-            LOCAL_METADATA, "-region",
-            "any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST));
-  }
-
-  @Test
-  public void testInit() throws Throwable {
-    run(Init.NAME,
-        "-meta", LOCAL_METADATA,
-        "-region", "us-west-1");
-  }
-
-  @Test
-  public void testInitTwice() throws Throwable {
-    run(Init.NAME,
-        "-meta", LOCAL_METADATA,
-        "-region", "us-west-1");
-    run(Init.NAME,
-        "-meta", LOCAL_METADATA,
-        "-region", "us-west-1");
-  }
-
   @Test
   public void testLandsatBucketUnguarded() throws Throwable {
     run(BucketInfo.NAME,
@@ -213,7 +65,7 @@ public void testLandsatBucketRequireGuarded() throws Throwable {
         BucketInfo.NAME,
         "-" + BucketInfo.GUARDED_FLAG,
         getLandsatCSVFile(
-            ITestS3GuardToolLocal.this.getConfiguration()));
+            ITestS3GuardTool.this.getConfiguration()));
   }
 
   @Test
@@ -229,38 +81,17 @@ public void testLandsatBucketRequireEncrypted() throws Throwable {
         BucketInfo.NAME,
         "-" + BucketInfo.ENCRYPTION_FLAG,
         "AES256", getLandsatCSVFile(
-            ITestS3GuardToolLocal.this.getConfiguration()));
+            ITestS3GuardTool.this.getConfiguration()));
   }
 
   @Test
   public void testStoreInfo() throws Throwable {
     S3GuardTool.BucketInfo cmd =
         toClose(new S3GuardTool.BucketInfo(getFileSystem().getConf()));
-    cmd.setStore(getMetadataStore());
-    try {
-      String output = exec(cmd, cmd.getName(),
-          "-" + BucketInfo.GUARDED_FLAG,
-          getFileSystem().getUri().toString());
-      LOG.info("Exec output=\n{}", output);
-    } finally {
-      cmd.setStore(new NullMetadataStore());
-    }
-  }
-
-  @Test
-  public void testSetCapacity() throws Throwable {
-    S3GuardTool cmd = toClose(
-        new S3GuardTool.SetCapacity(getFileSystem().getConf()));
-    cmd.setStore(getMetadataStore());
-    try {
-      String output = exec(cmd, cmd.getName(),
-          "-" + READ_FLAG, "100",
-          "-" + WRITE_FLAG, "100",
-          getFileSystem().getUri().toString());
-      LOG.info("Exec output=\n{}", output);
-    } finally {
-      cmd.setStore(new NullMetadataStore());
-    }
+    String output = exec(cmd, cmd.getName(),
+        "-" + BucketInfo.UNGUARDED_FLAG,
+        getFileSystem().getUri().toString());
+    LOG.info("Exec output=\n{}", output);
   }
 
   private final static String UPLOAD_PREFIX = "test-upload-prefix";
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
deleted file mode 100644
index 19784ede5ea4d..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-
-import com.amazonaws.services.dynamodbv2.document.DynamoDB;
-import com.amazonaws.services.dynamodbv2.document.Table;
-import com.amazonaws.services.dynamodbv2.model.ListTagsOfResourceRequest;
-import com.amazonaws.services.dynamodbv2.model.ResourceInUseException;
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import com.amazonaws.services.dynamodbv2.model.Tag;
-
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.AssumptionViolatedException;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.UnknownStoreException;
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Destroy;
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Init;
-import org.apache.hadoop.util.ExitUtil;
-
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
-import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBucketOverrides;
-import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Test S3Guard related CLI commands against DynamoDB.
- */
-public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
-
-  @Override
-  public void setup() throws Exception {
-    super.setup();
-    try {
-      getMetadataStore();
-    } catch (ClassCastException e) {
-      throw new AssumptionViolatedException(
-          "Test only applies when DynamoDB is used for S3Guard Store",
-          e);
-    }
-  }
-
-  @Override
-  protected DynamoDBMetadataStore getMetadataStore() {
-    return (DynamoDBMetadataStore) super.getMetadataStore();
-  }
-
-  // Check the existence of a given DynamoDB table.
-  private static boolean exist(DynamoDB dynamoDB, String tableName) {
-    assertNotNull(dynamoDB);
-    assertNotNull(tableName);
-    assertFalse("empty table name", tableName.isEmpty());
-    try {
-      Table table = dynamoDB.getTable(tableName);
-      table.describe();
-    } catch (ResourceNotFoundException e) {
-      return false;
-    }
-    return true;
-  }
-
-  @Test
-  public void testInvalidRegion() throws Exception {
-    final String testTableName =
-        getTestTableName("testInvalidRegion" + new Random().nextInt());
-    final String testRegion = "invalidRegion";
-    // Initialize MetadataStore
-    final Init initCmd = toClose(new Init(getFileSystem().getConf()));
-    intercept(IOException.class,
-        () -> {
-          int res = initCmd.run(new String[]{
-              "init",
-              "-region", testRegion,
-              "-meta", "dynamodb://" + testTableName
-          });
-          return "Use of invalid region did not fail, returning " + res
-              + "- table may have been " +
-              "created and not cleaned up: " + testTableName;
-        });
-  }
-
-  @Test
-  public void testDynamoTableTagging() throws Exception {
-    Configuration conf = getConfiguration();
-    // If the region is not set in conf, skip the test.
-    String ddbRegion = conf.get(S3GUARD_DDB_REGION_KEY);
-    Assume.assumeTrue(
-        S3GUARD_DDB_REGION_KEY + " should be set to run this test",
-        ddbRegion != null && !ddbRegion.isEmpty()
-    );
-
-    // setup
-    // clear all table tagging config before this test
-    conf.getPropsWithPrefix(S3GUARD_DDB_TABLE_TAG).keySet().forEach(
-        propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey)
-    );
-
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY,
-        getTestTableName("testDynamoTableTagging-" + UUID.randomUUID()));
-    String bucket = getFileSystem().getBucket();
-    removeBucketOverrides(bucket, conf,
-        S3GUARD_DDB_TABLE_NAME_KEY,
-        S3GUARD_DDB_REGION_KEY);
-
-    S3GuardTool.Init cmdR = new S3GuardTool.Init(conf);
-    Map<String, String> tagMap = new HashMap<>();
-    tagMap.put("hello", "dynamo");
-    tagMap.put("tag", "youre it");
-
-    String[] argsR = new String[]{
-        cmdR.getName(),
-        "-tag", tagMapToStringParams(tagMap),
-        "s3a://" + bucket + "/"
-    };
-
-    // run
-    cmdR.run(argsR);
-
-    // Check. Should create new metadatastore with the table name set.
-    try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
-      ddbms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-      ListTagsOfResourceRequest listTagsOfResourceRequest = new ListTagsOfResourceRequest()
-          .withResourceArn(ddbms.getTable().getDescription().getTableArn());
-      List<Tag> tags = ddbms.getAmazonDynamoDB().listTagsOfResource(listTagsOfResourceRequest).getTags();
-
-      // assert
-      // table version is always there as a plus one tag.
-      assertEquals(tagMap.size() + 1, tags.size());
-      for (Tag tag : tags) {
-        // skip the version marker tag
-        if (tag.getKey().equals(VERSION_MARKER_TAG_NAME)) {
-          continue;
-        }
-        Assert.assertEquals(tagMap.get(tag.getKey()), tag.getValue());
-      }
-      // be sure to clean up - delete table
-      ddbms.destroy();
-    }
-  }
-
-  private String tagMapToStringParams(Map<String, String> tagMap) {
-    StringBuilder stringBuilder = new StringBuilder();
-
-    for (Map.Entry<String, String> kv : tagMap.entrySet()) {
-      stringBuilder.append(kv.getKey() + "=" + kv.getValue() + ";");
-    }
-
-    return stringBuilder.toString();
-  }
-
-  private DDBCapacities getCapacities() throws IOException {
-    return DDBCapacities.extractCapacities(getMetadataStore().getDiagnostics());
-  }
-
-  @Test
-  public void testDynamoDBInitDestroyCycle() throws Throwable {
-    String testTableName =
-        getTestTableName("testDynamoDBInitDestroy" + new Random().nextInt());
-    String testS3Url = path(testTableName).toString();
-    S3AFileSystem fs = getFileSystem();
-    DynamoDB db = null;
-    try {
-      try (Init initCmd = new Init(fs.getConf())) {
-      // Initialize MetadataStore
-        expectSuccess("Init command did not exit successfully - see output",
-            initCmd,
-            Init.NAME,
-            "-" + READ_FLAG, "0",
-            "-" + WRITE_FLAG, "0",
-            "-" + Init.SSE_FLAG,
-            "-" + META_FLAG, "dynamodb://" + testTableName,
-            testS3Url);
-      }
-      // Verify it exists
-      MetadataStore ms = getMetadataStore();
-      assertTrue("metadata store should be DynamoDBMetadataStore",
-          ms instanceof DynamoDBMetadataStore);
-      DynamoDBMetadataStore dynamoMs = (DynamoDBMetadataStore) ms;
-      db = dynamoMs.getDynamoDB();
-      assertTrue(String.format("%s does not exist", testTableName),
-          exist(db, testTableName));
-
-      Configuration conf = fs.getConf();
-      String bucket = fs.getBucket();
-      // force in a new bucket
-      setBucketOption(conf, bucket, Constants.S3_METADATA_STORE_IMPL,
-          Constants.S3GUARD_METASTORE_DYNAMO);
-      try (Init initCmd = new Init(conf)) {
-        String initOutput = exec(initCmd,
-            "init", "-meta", "dynamodb://" + testTableName, testS3Url);
-        assertTrue("No Dynamo diagnostics in output " + initOutput,
-            initOutput.contains(DESCRIPTION));
-      }
-
-      // run a bucket info command and look for
-      // confirmation that it got the output from DDB diags
-      String info;
-      try (S3GuardTool.BucketInfo infocmd = new S3GuardTool.BucketInfo(conf)) {
-        info = exec(infocmd, BucketInfo.NAME,
-            "-" + BucketInfo.GUARDED_FLAG,
-            testS3Url);
-        assertTrue("No Dynamo diagnostics in output " + info,
-            info.contains(DESCRIPTION));
-      }
-
-    // get the current values to set again
-
-      // play with the set-capacity option
-      String fsURI = getFileSystem().getUri().toString();
-      DDBCapacities original = getCapacities();
-      assertTrue("Wrong billing mode in " + info,
-          info.contains(BILLING_MODE_PER_REQUEST));
-      // per-request tables fail here, so expect that
-      intercept(IOException.class, E_ON_DEMAND_NO_SET_CAPACITY,
-          () -> exec(toClose(newSetCapacity()),
-              SetCapacity.NAME,
-              fsURI));
-
-         // Destroy MetadataStore
-      try (Destroy destroyCmd = new Destroy(fs.getConf())){
-        String destroyed = exec(destroyCmd,
-            "destroy", "-meta", "dynamodb://" + testTableName, testS3Url);
-        // Verify it does not exist
-        assertFalse(String.format("%s still exists", testTableName),
-            exist(db, testTableName));
-
-        // delete again and expect success again
-        expectSuccess("Destroy command did not exit successfully - see output",
-            destroyCmd,
-            "destroy", "-meta", "dynamodb://" + testTableName, testS3Url);
-      }
-    } catch (ResourceNotFoundException e) {
-      throw new AssertionError(
-          String.format("DynamoDB table %s does not exist", testTableName),
-          e);
-    } finally {
-      LOG.warn("Table may have not been cleaned up: " +
-          testTableName);
-      if (db != null) {
-        Table table = db.getTable(testTableName);
-        if (table != null) {
-          try {
-            table.delete();
-            table.waitForDelete();
-          } catch (ResourceNotFoundException | ResourceInUseException e) {
-            /* Ignore */
-          }
-        }
-      }
-    }
-  }
-
-  private S3GuardTool newSetCapacity() {
-    S3GuardTool setCapacity = new S3GuardTool.SetCapacity(
-        getFileSystem().getConf());
-    setCapacity.setStore(getMetadataStore());
-    return setCapacity;
-  }
-
-  @Test
-  public void testDestroyUnknownTable() throws Throwable {
-    run(S3GuardTool.Destroy.NAME,
-        "-region", "us-west-2",
-        "-meta", "dynamodb://" + getTestTableName(DYNAMODB_TABLE));
-  }
-
-  @Test
-  public void testCLIFsckWithoutParam() throws Exception {
-    intercept(ExitUtil.ExitException.class, () -> run(Fsck.NAME));
-  }
-
-  @Test
-  public void testCLIFsckWithParam() throws Exception {
-    LOG.info("This test serves the purpose to run fsck with the correct " +
-        "parameters, so there will be no exception thrown.");
-    final int result = run(S3GuardTool.Fsck.NAME, "-check",
-        "s3a://" + getFileSystem().getBucket());
-    LOG.info("The return value of the run: {}", result);
-  }
-
-  @Test
-  public void testCLIFsckWithParamParentOfRoot() throws Exception {
-    intercept(IOException.class, "Invalid URI",
-        () -> run(S3GuardTool.Fsck.NAME, "-check",
-            "s3a://" + getFileSystem().getBucket() + "/.."));
-  }
-
-  @Test
-  public void testCLIFsckFailInitializeFs() throws Exception {
-    intercept(UnknownStoreException.class,
-        () -> run(S3GuardTool.Fsck.NAME, "-check",
-            "s3a://this-bucket-does-not-exist-" + UUID.randomUUID()));
-  }
-
-  @Test
-  public void testCLIFsckDDbInternalWrongS3APath() throws Exception {
-    intercept(FileNotFoundException.class, "wrong path",
-        () -> run(S3GuardTool.Fsck.NAME, "-"+Fsck.DDB_MS_CONSISTENCY_FLAG,
-            "s3a://" + getFileSystem().getBucket() + "/" + UUID.randomUUID()));
-  }
-
-  @Test
-  public void testCLIFsckDDbInternalParam() throws Exception {
-    describe("This test serves the purpose to run fsck with the correct " +
-        "parameters, so there will be no exception thrown.");
-    final int result = run(S3GuardTool.Fsck.NAME,
-        "-" + Fsck.DDB_MS_CONSISTENCY_FLAG,
-        "s3a://" + getFileSystem().getBucket());
-    LOG.info("The return value of the run: {}", result);
-  }
-
-  @Test
-  public void testCLIFsckCheckExclusive() throws Exception {
-    describe("There should be only one check param when running fsck." +
-        "If more then one param is passed, the command should fail." +
-        "This provide exclusive run for checks so the user is able to define " +
-        "the order of checking.");
-    intercept(ExitUtil.ExitException.class, "only one parameter",
-        () -> run(S3GuardTool.Fsck.NAME,
-        "-" + Fsck.DDB_MS_CONSISTENCY_FLAG, "-" + Fsck.CHECK_FLAG,
-        "s3a://" + getFileSystem().getBucket()));
-  }
-
-  @Test
-  public void testCLIFsckDDbFixOnlyFails() throws Exception {
-    describe("This test serves the purpose to run fsck with the correct " +
-        "parameters, so there will be no exception thrown.");
-    final int result = run(S3GuardTool.Fsck.NAME,
-        "-" + Fsck.FIX_FLAG,
-        "s3a://" + getFileSystem().getBucket());
-    LOG.info("The return value of the run: {}", result);
-    assertEquals(ERROR, result);
-  }
-
-  /**
-   * Test that the fix flag is accepted by the fsck.
-   *
-   * Note that we don't have an assert at the end of this test because
-   * there maybe some errors found during the check and the returned value
-   * will be ERROR and not SUCCESS. So if we assert on SUCCESS, then the test
-   * could (likely) to be flaky.
-   * If the FIX_FLAG parameter is not accepted here an exception will be thrown
-   * so the test will break.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testCLIFsckDDbFixAndInternalSucceed() throws Exception {
-    describe("This test serves the purpose to run fsck with the correct " +
-        "parameters, so there will be no exception thrown.");
-    final int result = run(S3GuardTool.Fsck.NAME,
-        "-" + Fsck.FIX_FLAG,
-        "-" + Fsck.DDB_MS_CONSISTENCY_FLAG,
-        "s3a://" + getFileSystem().getBucket());
-    LOG.info("The return value of the run: {}", result);
-  }
-
-  /**
-   * Test that when init, the CMK option can not live without SSE enabled.
-   */
-  @Test
-  public void testCLIInitParamCmkWithoutSse() throws Exception {
-    intercept(ExitUtil.ExitException.class, "can only be used with",
-        () -> run(S3GuardTool.Init.NAME,
-            "-" + S3GuardTool.CMK_FLAG,
-            "alias/" + UUID.randomUUID(),
-            "s3a://" + getFileSystem().getBucket() + "/" + UUID.randomUUID()));
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
deleted file mode 100644
index 06148857ea7d1..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ /dev/null
@@ -1,1342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3ATestConstants;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.test.HadoopTestBase;
-
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
-
-/**
- * Main test class for MetadataStore implementations.
- * Implementations should each create a test by subclassing this and
- * overriding {@link #createContract()}.
- * If your implementation may return missing results for recently set paths,
- * override {@link MetadataStoreTestBase#allowMissing()}.
- */
-public abstract class MetadataStoreTestBase extends HadoopTestBase {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(MetadataStoreTestBase.class);
-
-  /** Some dummy values for sanity-checking FileStatus contents. */
-  static final long BLOCK_SIZE = 32 * 1024 * 1024;
-  static final int REPLICATION = 1;
-  static final String OWNER = "bob";
-  private final long modTime = System.currentTimeMillis() - 5000;
-
-  // attributes not supported by S3AFileStatus
-  static final FsPermission PERMISSION = null;
-  static final String GROUP = null;
-  private final long accessTime = 0;
-  private static ITtlTimeProvider ttlTimeProvider;
-
-  private static final List<Path> EMPTY_LIST = Collections.emptyList();
-
-  /**
-   * Each test should override this.  Will use a new Configuration instance.
-   * @return Contract which specifies the MetadataStore under test plus config.
-   */
-  public abstract AbstractMSContract createContract() throws IOException;
-
-  /**
-   * Each test should override this.
-   * @param conf Base configuration instance to use.
-   * @return Contract which specifies the MetadataStore under test plus config.
-   */
-  public abstract AbstractMSContract createContract(Configuration conf)
-      throws IOException;
-
-  /**
-   * Tests assume that implementations will return recently set results.  If
-   * your implementation does not always hold onto metadata (e.g. LRU or
-   * time-based expiry) you can override this to return false.
-   * @return true if the test should succeed when null results are returned
-   *  from the MetadataStore under test.
-   */
-  public boolean allowMissing() {
-    return false;
-  }
-
-  /**
-   * Pruning is an optional feature for metadata store implementations.
-   * Tests will only check that functionality if it is expected to work.
-   * @return true if the test should expect pruning to work.
-   */
-  public boolean supportsPruning() {
-    return true;
-  }
-
-  /** The MetadataStore contract used to test against. */
-  private AbstractMSContract contract;
-
-  protected MetadataStore ms;
-
-  /**
-   * @return reference to the test contract.
-   */
-  protected AbstractMSContract getContract() {
-    return contract;
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    Thread.currentThread().setName("setup");
-    LOG.debug("== Setup. ==");
-    contract = createContract();
-    ms = contract.getMetadataStore();
-    assertNotNull("null MetadataStore", ms);
-    assertNotNull("null FileSystem", contract.getFileSystem());
-    ms.initialize(contract.getFileSystem(),
-        new S3Guard.TtlTimeProvider(contract.getFileSystem().getConf()));
-    ttlTimeProvider =
-        new S3Guard.TtlTimeProvider(contract.getFileSystem().getConf());
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    Thread.currentThread().setName("teardown");
-    LOG.debug("== Tear down. ==");
-    if (ms != null) {
-      try {
-        ms.destroy();
-      } catch (Exception e) {
-        LOG.warn("Failed to destroy tables in teardown", e);
-      }
-      IOUtils.closeStream(ms);
-      ms = null;
-    }
-  }
-
-  /**
-   * Describe a test in the logs.
-   * @param text text to print
-   * @param args arguments to format in the printing
-   */
-  protected void describe(String text, Object... args) {
-    LOG.info("\n\n{}: {}\n",
-        getMethodName(),
-        String.format(text, args));
-  }
-
-  /**
-   * Helper function for verifying DescendantsIterator and
-   * MetadataStoreListFilesIterator behavior.
-   * @param createNodes List of paths to create
-   * @param checkNodes List of paths that the iterator should return
-   */
-  private void doTestDescendantsIterator(
-      Class implementation, String[] createNodes,
-      String[] checkNodes) throws Exception {
-    // we set up the example file system tree in metadata store
-    for (String pathStr : createNodes) {
-      final S3AFileStatus status = pathStr.contains("file")
-          ? basicFileStatus(strToPath(pathStr), 100, false)
-          : basicFileStatus(strToPath(pathStr), 0, true);
-      ms.put(new PathMetadata(status), null);
-    }
-
-    final PathMetadata rootMeta = new PathMetadata(makeDirStatus("/"));
-    RemoteIterator<S3AFileStatus> iterator;
-    if (implementation == DescendantsIterator.class) {
-      iterator = new DescendantsIterator(ms, rootMeta);
-    } else if (implementation == MetadataStoreListFilesIterator.class) {
-      iterator = new MetadataStoreListFilesIterator(ms, rootMeta, false);
-    } else {
-      throw new UnsupportedOperationException("Unrecognized class");
-    }
-
-    final Set<String> actual = new HashSet<>();
-    while (iterator.hasNext()) {
-      final Path p = iterator.next().getPath();
-      actual.add(Path.getPathWithoutSchemeAndAuthority(p).toString());
-    }
-    LOG.info("We got {} by iterating DescendantsIterator", actual);
-
-    if (!allowMissing()) {
-      Assertions.assertThat(actual)
-          .as("files listed through DescendantsIterator")
-          .containsExactlyInAnyOrder(checkNodes);
-    }
-  }
-
-  /**
-   * Test that we can get the whole sub-tree by iterating DescendantsIterator.
-   *
-   * The tree is similar to or same as the example in code comment.
-   */
-  @Test
-  public void testDescendantsIterator() throws Exception {
-    final String[] tree = new String[] {
-        "/dir1",
-        "/dir1/dir2",
-        "/dir1/dir3",
-        "/dir1/dir2/file1",
-        "/dir1/dir2/file2",
-        "/dir1/dir3/dir4",
-        "/dir1/dir3/dir5",
-        "/dir1/dir3/dir4/file3",
-        "/dir1/dir3/dir5/file4",
-        "/dir1/dir3/dir6"
-    };
-    doTestDescendantsIterator(DescendantsIterator.class,
-        tree, tree);
-  }
-
-  /**
-   * Test that we can get the correct subset of the tree with
-   * MetadataStoreListFilesIterator.
-   *
-   * The tree is similar to or same as the example in code comment.
-   */
-  @Test
-  public void testMetadataStoreListFilesIterator() throws Exception {
-    final String[] wholeTree = new String[] {
-        "/dir1",
-        "/dir1/dir2",
-        "/dir1/dir3",
-        "/dir1/dir2/file1",
-        "/dir1/dir2/file2",
-        "/dir1/dir3/dir4",
-        "/dir1/dir3/dir5",
-        "/dir1/dir3/dir4/file3",
-        "/dir1/dir3/dir5/file4",
-        "/dir1/dir3/dir6"
-    };
-    final String[] leafNodes = new String[] {
-        "/dir1/dir2/file1",
-        "/dir1/dir2/file2",
-        "/dir1/dir3/dir4/file3",
-        "/dir1/dir3/dir5/file4"
-    };
-    doTestDescendantsIterator(MetadataStoreListFilesIterator.class, wholeTree,
-        leafNodes);
-  }
-
-  @Test
-  public void testPutNew() throws Exception {
-    /* create three dirs /da1, /da2, /da3 */
-    createNewDirs("/da1", "/da2", "/da3");
-
-    /* It is caller's responsibility to set up ancestor entries beyond the
-     * containing directory.  We only track direct children of the directory.
-     * Thus this will not affect entry for /da1.
-     */
-    ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc1", 100)), null);
-
-    assertEmptyDirs("/da2", "/da3");
-    assertDirectorySize("/da1/db1", 1);
-
-    /* Check contents of dir status. */
-    PathMetadata dirMeta = ms.get(strToPath("/da1"));
-    if (!allowMissing() || dirMeta != null) {
-      verifyDirStatus(dirMeta.getFileStatus());
-    }
-
-    /* This already exists, and should silently replace it. */
-    ms.put(new PathMetadata(makeDirStatus("/da1/db1")), null);
-
-    /* If we had putNew(), and used it above, this would be empty again. */
-    assertDirectorySize("/da1", 1);
-
-    assertEmptyDirs("/da2", "/da3");
-
-    /* Ensure new files update correct parent dirs. */
-    ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc1", 100)), null);
-    ms.put(new PathMetadata(makeFileStatus("/da1/db1/fc2", 200)), null);
-    assertDirectorySize("/da1", 1);
-    assertDirectorySize("/da1/db1", 2);
-    assertEmptyDirs("/da2", "/da3");
-    PathMetadata meta = ms.get(strToPath("/da1/db1/fc2"));
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get file after put new.", meta);
-      verifyFileStatus(meta.getFileStatus(), 200);
-    }
-  }
-
-  @Test
-  public void testPutOverwrite() throws Exception {
-    final String filePath = "/a1/b1/c1/some_file";
-    final String dirPath = "/a1/b1/c1/d1";
-    ms.put(new PathMetadata(makeFileStatus(filePath, 100)), null);
-    ms.put(new PathMetadata(makeDirStatus(dirPath)), null);
-    PathMetadata meta = ms.get(strToPath(filePath));
-    if (!allowMissing() || meta != null) {
-      verifyFileStatus(meta.getFileStatus(), 100);
-    }
-
-    ms.put(new PathMetadata(basicFileStatus(strToPath(filePath), 9999, false)),
-        null);
-    meta = ms.get(strToPath(filePath));
-    if (!allowMissing() || meta != null) {
-      verifyFileStatus(meta.getFileStatus(), 9999);
-    }
-  }
-
-  @Test
-  public void testRootDirPutNew() throws Exception {
-    Path rootPath = strToPath("/");
-
-    ms.put(new PathMetadata(makeFileStatus("/file1", 100)), null);
-    DirListingMetadata dir = ms.listChildren(rootPath);
-    if (!allowMissing() || dir != null) {
-      assertNotNull("Root dir cached", dir);
-      assertFalse("Root not fully cached", dir.isAuthoritative());
-      final Collection<PathMetadata> listing = dir.getListing();
-      Assertions.assertThat(listing)
-          .describedAs("Root dir listing")
-          .isNotNull()
-          .extracting(p -> p.getFileStatus().getPath())
-          .containsExactly(strToPath("/file1"));
-    }
-  }
-
-  @Test
-  public void testDelete() throws Exception {
-    setUpDeleteTest();
-
-    ms.delete(strToPath("/ADirectory1/db1/file2"), null);
-
-    /* Ensure delete happened. */
-    assertDirectorySize("/ADirectory1/db1", 1);
-    PathMetadata meta = ms.get(strToPath("/ADirectory1/db1/file2"));
-    assertTrue("File deleted", meta == null || meta.isDeleted());
-  }
-
-  @Test
-  public void testDeleteSubtree() throws Exception {
-    deleteSubtreeHelper("");
-  }
-
-  @Test
-  public void testDeleteSubtreeHostPath() throws Exception {
-    deleteSubtreeHelper(contract.getFileSystem().getUri().toString());
-  }
-
-  private void deleteSubtreeHelper(String pathPrefix) throws Exception {
-
-    String p = pathPrefix;
-    setUpDeleteTest(p);
-    createNewDirs(p + "/ADirectory1/db1/dc1", p + "/ADirectory1/db1/dc1/dd1");
-    ms.put(new PathMetadata(
-        makeFileStatus(p + "/ADirectory1/db1/dc1/dd1/deepFile", 100)),
-        null);
-    if (!allowMissing()) {
-      assertCached(p + "/ADirectory1/db1");
-    }
-    ms.deleteSubtree(strToPath(p + "/ADirectory1/db1/"), null);
-
-    assertEmptyDirectory(p + "/ADirectory1");
-    assertDeleted(p + "/ADirectory1/db1");
-    assertDeleted(p + "/ADirectory1/file1");
-    assertDeleted(p + "/ADirectory1/file2");
-    assertDeleted(p + "/ADirectory1/db1/dc1/dd1/deepFile");
-    assertEmptyDirectory(p + "/ADirectory2");
-  }
-
-
-  /*
-   * Some implementations might not support this.  It was useful to test
-   * correctness of the LocalMetadataStore implementation, but feel free to
-   * override this to be a no-op.
-   */
-  @Test
-  public void testDeleteRecursiveRoot() throws Exception {
-    setUpDeleteTest();
-
-    ms.deleteSubtree(strToPath("/"), null);
-    assertDeleted("/ADirectory1");
-    assertDeleted("/ADirectory2");
-    assertDeleted("/ADirectory2/db1");
-    assertDeleted("/ADirectory2/db1/file1");
-    assertDeleted("/ADirectory2/db1/file2");
-  }
-
-  @Test
-  public void testDeleteNonExisting() throws Exception {
-    // Path doesn't exist, but should silently succeed
-    ms.delete(strToPath("/bobs/your/uncle"), null);
-
-    // Ditto.
-    ms.deleteSubtree(strToPath("/internets"), null);
-  }
-
-
-  private void setUpDeleteTest() throws IOException {
-    setUpDeleteTest("");
-  }
-
-  private void setUpDeleteTest(String prefix) throws IOException {
-    createNewDirs(prefix + "/ADirectory1", prefix + "/ADirectory2",
-        prefix + "/ADirectory1/db1");
-    ms.put(new PathMetadata(makeFileStatus(prefix + "/ADirectory1/db1/file1",
-        100)),
-        null);
-    ms.put(new PathMetadata(makeFileStatus(prefix + "/ADirectory1/db1/file2",
-        100)),
-        null);
-
-    PathMetadata meta = ms.get(strToPath(prefix + "/ADirectory1/db1/file2"));
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Found test file", meta);
-      assertDirectorySize(prefix + "/ADirectory1/db1", 2);
-    }
-  }
-
-  @Test
-  public void testGet() throws Exception {
-    final String filePath = "/a1/b1/c1/some_file";
-    final String dirPath = "/a1/b1/c1/d1";
-    ms.put(new PathMetadata(makeFileStatus(filePath, 100)), null);
-    ms.put(new PathMetadata(makeDirStatus(dirPath)), null);
-    PathMetadata meta = ms.get(strToPath(filePath));
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get found file", meta);
-      verifyFileStatus(meta.getFileStatus(), 100);
-    }
-
-    if (!(ms instanceof NullMetadataStore)) {
-      ms.delete(strToPath(filePath), null);
-      meta = ms.get(strToPath(filePath));
-      assertTrue("Tombstone not left for deleted file", meta.isDeleted());
-    }
-
-    meta = ms.get(strToPath(dirPath));
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get found file (dir)", meta);
-      assertTrue("Found dir", meta.getFileStatus().isDirectory());
-    }
-
-    meta = ms.get(strToPath("/bollocks"));
-    assertNull("Don't get non-existent file", meta);
-  }
-
-  @Test
-  public void testGetEmptyDir() throws Exception {
-    final String dirPath = "/a1/b1/c1/d1";
-    // Creates /a1/b1/c1/d1 as an empty dir
-    setupListStatus();
-
-    // 1. Tell MetadataStore (MS) that there are zero children
-    putListStatusFiles(dirPath, true /* authoritative */
-        /* zero children */);
-
-    // 2. Request a file status for dir, including whether or not the dir
-    // is empty.
-    PathMetadata meta = ms.get(strToPath(dirPath), true);
-
-    // 3. Check that either (a) the MS doesn't track whether or not it is
-    // empty (which is allowed), or (b) the MS knows the dir is empty.
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get should find meta for dir", meta);
-      assertNotEquals("Dir is empty or unknown", Tristate.FALSE,
-          meta.isEmptyDirectory());
-    }
-  }
-
-  @Test
-  public void testGetNonEmptyDir() throws Exception {
-    final String dirPath = "/a1/b1/c1";
-    // Creates /a1/b1/c1 as an non-empty dir
-    setupListStatus();
-
-    // Request a file status for dir, including whether or not the dir
-    // is empty.
-    PathMetadata meta = ms.get(strToPath(dirPath), true);
-
-    // MetadataStore knows /a1/b1/c1 has at least one child.  It is valid
-    // for it to answer either (a) UNKNOWN: the MS doesn't track whether
-    // or not the dir is empty, or (b) the MS knows the dir is non-empty.
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get should find meta for dir", meta);
-      assertNotEquals("Dir is non-empty or unknown", Tristate.TRUE,
-          meta.isEmptyDirectory());
-    }
-  }
-
-  @Test
-  public void testGetDirUnknownIfEmpty() throws Exception {
-    final String dirPath = "/a1/b1/c1/d1";
-    // 1. Create /a1/b1/c1/d1 as an empty dir, but do not tell MetadataStore
-    // (MS) whether or not it has any children.
-    setupListStatus();
-
-    // 2. Request a file status for dir, including whether or not the dir
-    // is empty.
-    PathMetadata meta = ms.get(strToPath(dirPath), true);
-
-    // 3. Assert MS reports isEmptyDir as UNKONWN: We haven't told MS
-    // whether or not the directory has any children.
-    if (!allowMissing() || meta != null) {
-      assertNotNull("Get should find meta for dir", meta);
-      assertEquals("Dir empty is unknown", Tristate.UNKNOWN,
-          meta.isEmptyDirectory());
-    }
-  }
-
-  @Test
-  public void testListChildren() throws Exception {
-    setupListStatus();
-
-    DirListingMetadata dirMeta;
-    dirMeta = ms.listChildren(strToPath("/"));
-    if (!allowMissing()) {
-      assertNotNull(dirMeta);
-        /* Cache has no way of knowing it has all entries for root unless we
-         * specifically tell it via put() with
-         * DirListingMetadata.isAuthoritative = true */
-      assertFalse("Root dir is not cached, or partially cached",
-          dirMeta.isAuthoritative());
-      assertListingsEqual(dirMeta.getListing(), "/a1", "/a2");
-    }
-
-    dirMeta = ms.listChildren(strToPath("/a1"));
-    if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
-      assertListingsEqual(dirMeta.getListing(), "/a1/b1", "/a1/b2");
-    }
-
-    dirMeta = ms.listChildren(strToPath("/a1/b1"));
-    if (!allowMissing() || dirMeta != null) {
-      assertListingsEqual(dirMeta.getListing(), "/a1/b1/file1", "/a1/b1/file2",
-          "/a1/b1/c1");
-    }
-  }
-
-
-
-  @Test
-  public void testListChildrenAuthoritative() throws IOException {
-    Assume.assumeTrue("MetadataStore should be capable for authoritative "
-        + "storage of directories to run this test.",
-        metadataStorePersistsAuthoritativeBit(ms));
-
-    setupListStatus();
-
-    DirListingMetadata dirMeta = ms.listChildren(strToPath("/a1/b1"));
-    dirMeta.setAuthoritative(true);
-    dirMeta.put(new PathMetadata(
-        makeFileStatus("/a1/b1/file_new", 100)));
-    ms.put(dirMeta, EMPTY_LIST, null);
-
-    dirMeta = ms.listChildren(strToPath("/a1/b1"));
-    assertListingsEqual(dirMeta.getListing(), "/a1/b1/file1", "/a1/b1/file2",
-        "/a1/b1/c1", "/a1/b1/file_new");
-    assertTrue(dirMeta.isAuthoritative());
-  }
-
-  @Test
-  public void testDirListingRoot() throws Exception {
-    commonTestPutListStatus("/");
-  }
-
-  @Test
-  public void testPutDirListing() throws Exception {
-    commonTestPutListStatus("/a");
-  }
-
-  @Test
-  public void testInvalidListChildren() throws Exception {
-    setupListStatus();
-    assertNull("missing path returns null",
-        ms.listChildren(strToPath("/a1/b1x")));
-  }
-
-  @Test
-  public void testMove() throws Exception {
-    // Create test dir structure
-    createNewDirs("/a1", "/a2", "/a3");
-    createNewDirs("/a1/b1", "/a1/b2");
-    putListStatusFiles("/a1/b1", false, "/a1/b1/file1", "/a1/b1/file2");
-
-    // Assert root listing as expected
-    Collection<PathMetadata> entries;
-    DirListingMetadata dirMeta = ms.listChildren(strToPath("/"));
-    if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
-      assertNotNull("Listing root", dirMeta);
-      entries = dirMeta.getListing();
-      assertListingsEqual(entries, "/a1", "/a2", "/a3");
-    }
-
-    // Assert src listing as expected
-    dirMeta = ms.listChildren(strToPath("/a1/b1"));
-    if (!allowMissing() || dirMeta != null) {
-      assertNotNull("Listing /a1/b1", dirMeta);
-      entries = dirMeta.getListing();
-      assertListingsEqual(entries, "/a1/b1/file1", "/a1/b1/file2");
-    }
-
-    // Do the move(): rename(/a1/b1, /b1)
-    Collection<Path> srcPaths = Arrays.asList(strToPath("/a1/b1"),
-        strToPath("/a1/b1/file1"), strToPath("/a1/b1/file2"));
-
-    ArrayList<PathMetadata> destMetas = new ArrayList<>();
-    destMetas.add(new PathMetadata(makeDirStatus("/b1")));
-    destMetas.add(new PathMetadata(makeFileStatus("/b1/file1", 100)));
-    destMetas.add(new PathMetadata(makeFileStatus("/b1/file2", 100)));
-    ms.move(srcPaths, destMetas, null);
-
-    // Assert src is no longer there
-    dirMeta = ms.listChildren(strToPath("/a1"));
-    if (!allowMissing() || dirMeta != null) {
-      assertNotNull("Listing /a1", dirMeta);
-      entries = dirMeta.withoutTombstones().getListing();
-      assertListingsEqual(entries, "/a1/b2");
-    }
-
-    PathMetadata meta = ms.get(strToPath("/a1/b1/file1"));
-    assertTrue("Src path deleted", meta == null || meta.isDeleted());
-
-    // Assert dest looks right
-    meta = ms.get(strToPath("/b1/file1"));
-    if (!allowMissing() || meta != null) {
-      assertNotNull("dest file not null", meta);
-      verifyFileStatus(meta.getFileStatus(), 100);
-    }
-
-    dirMeta = ms.listChildren(strToPath("/b1"));
-    if (!allowMissing() || dirMeta != null) {
-      assertNotNull("dest listing not null", dirMeta);
-      entries = dirMeta.getListing();
-      assertListingsEqual(entries, "/b1/file1", "/b1/file2");
-    }
-  }
-
-  /**
-   * Test that the MetadataStore differentiates between the same path in two
-   * different buckets.
-   */
-  @Test
-  public void testMultiBucketPaths() throws Exception {
-    String p1 = "s3a://bucket-a/path1";
-    String p2 = "s3a://bucket-b/path2";
-
-    // Make sure we start out empty
-    PathMetadata meta = ms.get(new Path(p1));
-    assertNull("Path should not be present yet.", meta);
-    meta = ms.get(new Path(p2));
-    assertNull("Path2 should not be present yet.", meta);
-
-    // Put p1, assert p2 doesn't match
-    ms.put(new PathMetadata(makeFileStatus(p1, 100)), null);
-    meta = ms.get(new Path(p2));
-    assertNull("Path 2 should not match path 1.", meta);
-
-    // Make sure delete is correct as well
-    if (!allowMissing()) {
-      ms.delete(new Path(p2), null);
-      meta = ms.get(new Path(p1));
-      assertNotNull("Path should not have been deleted", meta);
-    }
-    ms.delete(new Path(p1), null);
-  }
-
-  @Test
-  public void testPruneFiles() throws Exception {
-    Assume.assumeTrue(supportsPruning());
-    createNewDirs("/pruneFiles");
-
-    long oldTime = getTime();
-    ms.put(new PathMetadata(makeFileStatus("/pruneFiles/old", 1, oldTime)),
-        null);
-    DirListingMetadata ls2 = ms.listChildren(strToPath("/pruneFiles"));
-    if (!allowMissing()) {
-      assertListingsEqual(ls2.getListing(), "/pruneFiles/old");
-    }
-
-    // It's possible for the Local implementation to get from /pruneFiles/old's
-    // modification time to here in under 1ms, causing it to not get pruned
-    Thread.sleep(1);
-    long cutoff = System.currentTimeMillis();
-    long newTime = getTime();
-    ms.put(new PathMetadata(makeFileStatus("/pruneFiles/new", 1, newTime)),
-        null);
-
-    DirListingMetadata ls;
-    ls = ms.listChildren(strToPath("/pruneFiles"));
-    if (!allowMissing()) {
-      assertListingsEqual(ls.getListing(), "/pruneFiles/new",
-          "/pruneFiles/old");
-    }
-    ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME, cutoff);
-    ls = ms.listChildren(strToPath("/pruneFiles"));
-    if (allowMissing()) {
-      assertDeleted("/pruneFiles/old");
-    } else {
-      assertListingsEqual(ls.getListing(), "/pruneFiles/new");
-    }
-  }
-
-  @Test
-  public void testPruneDirs() throws Exception {
-    Assume.assumeTrue(supportsPruning());
-
-    // We only test that files, not dirs, are removed during prune.
-    // We specifically allow directories to remain, as it is more robust
-    // for DynamoDBMetadataStore's prune() implementation: If a
-    // file was created in a directory while it was being pruned, it would
-    // violate the invariant that all ancestors of a file exist in the table.
-
-    createNewDirs("/pruneDirs/dir");
-
-    long oldTime = getTime();
-    ms.put(new PathMetadata(makeFileStatus("/pruneDirs/dir/file",
-        1, oldTime)), null);
-
-    // It's possible for the Local implementation to get from the old
-    // modification time to here in under 1ms, causing it to not get pruned
-    Thread.sleep(1);
-    long cutoff = getTime();
-
-    ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME, cutoff);
-
-    assertDeleted("/pruneDirs/dir/file");
-  }
-
-  @Test
-  public void testPruneUnsetsAuthoritative() throws Exception {
-    String rootDir = "/unpruned-root-dir";
-    String grandparentDir = rootDir + "/pruned-grandparent-dir";
-    String parentDir = grandparentDir + "/pruned-parent-dir";
-    String staleFile = parentDir + "/stale-file";
-    String freshFile = rootDir + "/fresh-file";
-    String[] directories = {rootDir, grandparentDir, parentDir};
-
-    createNewDirs(rootDir, grandparentDir, parentDir);
-    long time = System.currentTimeMillis();
-    ms.put(new PathMetadata(
-        basicFileStatus(0, false, 0, time - 1, strToPath(staleFile)),
-        Tristate.FALSE, false),
-        null);
-    ms.put(new PathMetadata(
-        basicFileStatus(0, false, 0, time + 1, strToPath(freshFile)),
-        Tristate.FALSE, false),
-        null);
-
-    // set parent dir as authoritative
-    if (!allowMissing()) {
-      DirListingMetadata parentDirMd = ms.listChildren(strToPath(parentDir));
-      parentDirMd.setAuthoritative(true);
-      ms.put(parentDirMd, EMPTY_LIST, null);
-    }
-
-    ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME, time);
-    DirListingMetadata listing;
-    for (String directory : directories) {
-      Path path = strToPath(directory);
-      if (ms.get(path) != null) {
-        listing = ms.listChildren(path);
-        assertFalse(listing.isAuthoritative());
-      }
-    }
-  }
-
-  @Test
-  public void testPrunePreservesAuthoritative() throws Exception {
-    String rootDir = "/unpruned-root-dir";
-    String grandparentDir = rootDir + "/pruned-grandparent-dir";
-    String parentDir = grandparentDir + "/pruned-parent-dir";
-    String staleFile = parentDir + "/stale-file";
-    String freshFile = rootDir + "/fresh-file";
-    String[] directories = {rootDir, grandparentDir, parentDir};
-
-    // create dirs
-    createNewDirs(rootDir, grandparentDir, parentDir);
-    long time = System.currentTimeMillis();
-    ms.put(new PathMetadata(
-        basicFileStatus(0, false, 0, time + 1, strToPath(staleFile)),
-        Tristate.FALSE, false),
-        null);
-    ms.put(new PathMetadata(
-        basicFileStatus(0, false, 0, time + 1, strToPath(freshFile)),
-        Tristate.FALSE, false),
-        null);
-
-    if (!allowMissing()) {
-      // set parent dir as authoritative
-      DirListingMetadata parentDirMd = ms.listChildren(strToPath(parentDir));
-      parentDirMd.setAuthoritative(true);
-      ms.put(parentDirMd, EMPTY_LIST, null);
-
-      // prune the ms
-      ms.prune(MetadataStore.PruneMode.ALL_BY_MODTIME, time);
-
-      // get the directory listings
-      DirListingMetadata rootDirMd = ms.listChildren(strToPath(rootDir));
-      DirListingMetadata grandParentDirMd =
-          ms.listChildren(strToPath(grandparentDir));
-      parentDirMd = ms.listChildren(strToPath(parentDir));
-
-      // assert that parent dir is still authoritative (no removed elements
-      // during prune)
-      assertFalse(rootDirMd.isAuthoritative());
-      assertFalse(grandParentDirMd.isAuthoritative());
-      assertTrue(parentDirMd.isAuthoritative());
-    }
-  }
-
-  @Test
-  public void testPutDirListingMetadataPutsFileMetadata()
-      throws IOException {
-    boolean authoritative = true;
-    String[] filenames = {"/dir1/file1", "/dir1/file2", "/dir1/file3"};
-    String dirPath = "/dir1";
-
-    ArrayList<PathMetadata> metas = new ArrayList<>(filenames.length);
-    for (String filename : filenames) {
-      metas.add(new PathMetadata(makeFileStatus(filename, 100)));
-    }
-    DirListingMetadata dirMeta =
-        new DirListingMetadata(strToPath(dirPath), metas, authoritative);
-    ms.put(dirMeta, EMPTY_LIST, null);
-
-    if (!allowMissing()) {
-      assertDirectorySize(dirPath, filenames.length);
-      PathMetadata metadata;
-      for(String fileName : filenames){
-        metadata = ms.get(strToPath(fileName));
-        assertNotNull(String.format(
-            "PathMetadata for file %s should not be null.", fileName),
-            metadata);
-      }
-    }
-  }
-
-  @Test
-  public void testPutRetainsIsDeletedInParentListing() throws Exception {
-    final Path path = strToPath("/a/b");
-    final S3AFileStatus fileStatus = basicFileStatus(path, 0, false);
-    PathMetadata pm = new PathMetadata(fileStatus);
-    pm.setIsDeleted(true);
-    ms.put(pm, null);
-    if(!allowMissing()) {
-      final PathMetadata pathMetadata =
-          ms.listChildren(path.getParent()).get(path);
-      assertTrue("isDeleted should be true on the parent listing",
-          pathMetadata.isDeleted());
-    }
-  }
-
-  @Test
-  public void testPruneExpiredTombstones() throws Exception {
-    List<String> keepFilenames = new ArrayList<>(
-        Arrays.asList("/dir1/fileK1", "/dir1/fileK2", "/dir1/fileK3"));
-    List<String> removeFilenames = new ArrayList<>(
-        Arrays.asList("/dir1/fileR1", "/dir1/fileR2", "/dir1/fileR3"));
-
-    long cutoff = 9001;
-
-    for(String fN : keepFilenames) {
-      final PathMetadata pathMetadata = new PathMetadata(makeFileStatus(fN, 1));
-      pathMetadata.setLastUpdated(9002L);
-      ms.put(pathMetadata);
-    }
-
-    for(String fN : removeFilenames) {
-      final PathMetadata pathMetadata = new PathMetadata(makeFileStatus(fN, 1));
-      pathMetadata.setLastUpdated(9000L);
-      // tombstones are the deleted files!
-      pathMetadata.setIsDeleted(true);
-      ms.put(pathMetadata);
-    }
-
-    ms.prune(MetadataStore.PruneMode.TOMBSTONES_BY_LASTUPDATED, cutoff);
-
-    if (!allowMissing()) {
-      for (String fN : keepFilenames) {
-        final PathMetadata pathMetadata = ms.get(strToPath(fN));
-        assertNotNull("Kept files should be in the metastore after prune",
-            pathMetadata);
-      }
-    }
-
-    for(String fN : removeFilenames) {
-      final PathMetadata pathMetadata = ms.get(strToPath(fN));
-      assertNull("Expired tombstones should be removed from metastore after "
-          + "the prune.", pathMetadata);
-    }
-  }
-
-  @Test
-  public void testPruneExpiredTombstonesSpecifiedPath() throws Exception {
-    List<String> keepFilenames = new ArrayList<>(
-        Arrays.asList("/dir1/fileK1", "/dir1/fileK2", "/dir1/fileK3"));
-    List<String> removeFilenames = new ArrayList<>(
-        Arrays.asList("/dir2/fileR1", "/dir2/fileR2", "/dir2/fileR3"));
-
-    long cutoff = 9001;
-
-    // Both are expired. Difference is it will only delete the specified one.
-    for (String fN : keepFilenames) {
-      final PathMetadata pathMetadata = new PathMetadata(makeFileStatus(fN, 1));
-      pathMetadata.setLastUpdated(9002L);
-      ms.put(pathMetadata);
-    }
-
-    for (String fN : removeFilenames) {
-      final PathMetadata pathMetadata = new PathMetadata(makeFileStatus(fN, 1));
-      pathMetadata.setLastUpdated(9000L);
-      // tombstones are the deleted files!
-      pathMetadata.setIsDeleted(true);
-      ms.put(pathMetadata);
-    }
-
-    final String prunePath = getPathStringForPrune("/dir2");
-    ms.prune(MetadataStore.PruneMode.TOMBSTONES_BY_LASTUPDATED, cutoff,
-        prunePath);
-
-    if (!allowMissing()) {
-      for (String fN : keepFilenames) {
-        final PathMetadata pathMetadata = ms.get(strToPath(fN));
-        assertNotNull("Kept files should be in the metastore after prune",
-            pathMetadata);
-      }
-    }
-
-    for (String fN : removeFilenames) {
-      final PathMetadata pathMetadata = ms.get(strToPath(fN));
-      assertNull("Expired tombstones should be removed from metastore after "
-          + "the prune.", pathMetadata);
-    }
-  }
-
-  /*
-   * Helper functions.
-   */
-
-  /** Modifies paths input array and returns it. */
-  private String[] buildPathStrings(String parent, String... paths)
-      throws IOException {
-    for (int i = 0; i < paths.length; i++) {
-      Path p = new Path(strToPath(parent), paths[i]);
-      paths[i] = p.toString();
-    }
-    return paths;
-  }
-
-
-  /**
-   * The prune operation needs the path with the bucket name as a string in
-   * {@link DynamoDBMetadataStore}, but not for {@link LocalMetadataStore}.
-   * This is an implementation detail of the ms, so this should be
-   * implemented in the subclasses.
-   */
-  protected abstract String getPathStringForPrune(String path)
-      throws Exception;
-
-  private void commonTestPutListStatus(final String parent) throws IOException {
-    putListStatusFiles(parent, true, buildPathStrings(parent, "file1", "file2",
-        "file3"));
-    DirListingMetadata dirMeta = ms.listChildren(strToPath(parent));
-    if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
-      assertNotNull("list after putListStatus", dirMeta);
-      Collection<PathMetadata> entries = dirMeta.getListing();
-      assertNotNull("listStatus has entries", entries);
-      assertListingsEqual(entries,
-          buildPathStrings(parent, "file1", "file2", "file3"));
-    }
-  }
-
-  private void setupListStatus() throws IOException {
-    createNewDirs("/a1", "/a2", "/a1/b1", "/a1/b2", "/a1/b1/c1",
-        "/a1/b1/c1/d1");
-    ms.put(new PathMetadata(makeFileStatus("/a1/b1/file1", 100)), null);
-    ms.put(new PathMetadata(makeFileStatus("/a1/b1/file2", 100)), null);
-  }
-
-  private void assertListingsEqual(Collection<PathMetadata> listing,
-      String ...pathStrs) throws IOException {
-    Set<Path> a = new HashSet<>();
-    for (PathMetadata meta : listing) {
-      a.add(meta.getFileStatus().getPath());
-    }
-
-    Set<Path> b = new HashSet<>();
-    for (String ps : pathStrs) {
-      b.add(strToPath(ps));
-    }
-    Assertions.assertThat(a)
-        .as("Directory Listing")
-        .containsExactlyInAnyOrderElementsOf(b);
-  }
-
-  protected void putListStatusFiles(String dirPath, boolean authoritative,
-      String... filenames) throws IOException {
-    ArrayList<PathMetadata> metas = new ArrayList<>(filenames .length);
-    for (String filename : filenames) {
-      metas.add(new PathMetadata(makeFileStatus(filename, 100)));
-    }
-    DirListingMetadata dirMeta =
-        new DirListingMetadata(strToPath(dirPath), metas, authoritative);
-    ms.put(dirMeta, EMPTY_LIST, null);
-  }
-
-  protected void createNewDirs(String... dirs)
-      throws IOException {
-    for (String pathStr : dirs) {
-      ms.put(new PathMetadata(makeDirStatus(pathStr)), null);
-    }
-  }
-
-  private void assertDirectorySize(String pathStr, int size)
-      throws IOException {
-    DirListingMetadata dirMeta = ms.listChildren(strToPath(pathStr));
-    if (!allowMissing()) {
-      assertNotNull("Directory " + pathStr + " is null in cache", dirMeta);
-    }
-    if (!allowMissing() || dirMeta != null) {
-      dirMeta = dirMeta.withoutTombstones();
-      Assertions.assertThat(nonDeleted(dirMeta.getListing()))
-          .as("files in directory %s", pathStr)
-          .hasSize(size);
-    }
-  }
-
-  /** @return only file statuses which are *not* marked deleted. */
-  private Collection<PathMetadata> nonDeleted(
-      Collection<PathMetadata> statuses) {
-    Collection<PathMetadata> currentStatuses = new ArrayList<>();
-    for (PathMetadata status : statuses) {
-      if (!status.isDeleted()) {
-        currentStatuses.add(status);
-      }
-    }
-    return currentStatuses;
-  }
-
-  protected PathMetadata get(final String pathStr) throws IOException {
-    Path path = strToPath(pathStr);
-    return ms.get(path);
-  }
-
-  protected PathMetadata getNonNull(final String pathStr) throws IOException {
-    return checkNotNull(get(pathStr), "No metastore entry for %s", pathStr);
-  }
-
-  /**
-   * Assert that either a path has no entry or that it is marked as deleted.
-   * @param pathStr path
-   * @throws IOException IO failure.
-   */
-  protected void assertDeleted(String pathStr) throws IOException {
-    PathMetadata meta = get(pathStr);
-    boolean cached = meta != null && !meta.isDeleted();
-    assertFalse(pathStr + " should not be cached: " + meta, cached);
-  }
-
-  protected void assertCached(String pathStr) throws IOException {
-    verifyCached(pathStr);
-  }
-
-  /**
-   * Get an entry which must exist and not be deleted.
-   * @param pathStr path
-   * @return the entry
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata verifyCached(final String pathStr) throws IOException {
-    PathMetadata meta = getNonNull(pathStr);
-    assertFalse(pathStr + " was found but marked deleted: "+ meta,
-        meta.isDeleted());
-    return meta;
-  }
-
-  /**
-   * Assert that an entry exists and is a file.
-   * @param pathStr path
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata verifyIsFile(String pathStr) throws IOException {
-    PathMetadata md = verifyCached(pathStr);
-    assertTrue("Not a file: " + md,
-        md.getFileStatus().isFile());
-    return md;
-  }
-
-  /**
-   * Assert that an entry exists and is a tombstone.
-   * @param pathStr path
-   * @throws IOException IO failure.
-   */
-  protected void assertIsTombstone(String pathStr) throws IOException {
-    PathMetadata meta = getNonNull(pathStr);
-    assertTrue(pathStr + " must be a tombstone: " + meta, meta.isDeleted());
-  }
-
-  /**
-   * Assert that an entry does not exist.
-   * @param pathStr path
-   * @throws IOException IO failure.
-   */
-  protected void assertNotFound(String pathStr) throws IOException {
-    PathMetadata meta = get(pathStr);
-    assertNull("Unexpectedly found entry at path " + pathStr,
-        meta);
-  }
-
-  /**
-   * Get an entry which must be a file.
-   * @param pathStr path
-   * @return the entry
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata getFile(final String pathStr) throws IOException {
-    PathMetadata meta = verifyCached(pathStr);
-    assertFalse(pathStr + " is not a file: " + meta,
-        meta.getFileStatus().isDirectory());
-    return meta;
-  }
-
-  /**
-   * Get an entry which must be a directory.
-   * @param pathStr path
-   * @return the entry
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata getDirectory(final String pathStr) throws IOException {
-    PathMetadata meta = verifyCached(pathStr);
-    assertTrue(pathStr + " is not a directory: " + meta,
-        meta.getFileStatus().isDirectory());
-    return meta;
-  }
-
-  /**
-   * Get an entry which must not be marked as an empty directory:
-   * its empty directory field must be FALSE or UNKNOWN.
-   * @param pathStr path
-   * @return the entry
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata getNonEmptyDirectory(final String pathStr)
-      throws IOException {
-    PathMetadata meta = getDirectory(pathStr);
-    assertNotEquals("Path " + pathStr
-            + " is considered an empty dir " + meta,
-        Tristate.TRUE,
-        meta.isEmptyDirectory());
-    return meta;
-  }
-
-  /**
-   * Get an entry which must be an empty directory.
-   * its empty directory field must be TRUE.
-   * @param pathStr path
-   * @return the entry
-   * @throws IOException IO failure.
-   */
-  protected PathMetadata getEmptyDirectory(final String pathStr)
-      throws IOException {
-    PathMetadata meta = getDirectory(pathStr);
-    assertEquals("Path " + pathStr
-            + " is not considered an empty dir " + meta,
-        Tristate.TRUE,
-        meta.isEmptyDirectory());
-    return meta;
-  }
-
-  /**
-   * Convenience to create a fully qualified Path from string.
-   */
-  protected Path strToPath(String p) throws IOException {
-    final Path path = new Path(p);
-    assertTrue("Non-absolute path: " + path,  path.isAbsolute());
-    return path.makeQualified(contract.getFileSystem().getUri(), null);
-  }
-
-  protected void assertEmptyDirectory(String pathStr) throws IOException {
-    assertDirectorySize(pathStr, 0);
-  }
-
-  protected void assertEmptyDirs(String...dirs) throws IOException {
-    for (String pathStr : dirs) {
-      assertEmptyDirectory(pathStr);
-    }
-  }
-
-  protected S3AFileStatus basicFileStatus(Path path, int size, boolean isDir)
-      throws IOException {
-    return basicFileStatus(path, size, isDir, modTime);
-  }
-
-  public static S3AFileStatus basicFileStatus(int size, boolean isDir,
-      long blockSize, long modificationTime, Path path) {
-    if (isDir) {
-      return new S3AFileStatus(Tristate.UNKNOWN, path, null);
-    } else {
-      return new S3AFileStatus(size, modificationTime, path, blockSize, null,
-          null, null);
-    }
-  }
-
-  public static S3AFileStatus basicFileStatus(Path path, int size,
-      boolean isDir, long newModTime) throws IOException {
-    if (isDir) {
-      return new S3AFileStatus(Tristate.UNKNOWN, path, OWNER);
-    } else {
-      return new S3AFileStatus(size, newModTime, path, BLOCK_SIZE, OWNER,
-          "etag", "version");
-    }
-  }
-
-  protected S3AFileStatus makeFileStatus(String pathStr, int size) throws
-      IOException {
-    return makeFileStatus(pathStr, size, modTime);
-  }
-
-  protected S3AFileStatus makeFileStatus(String pathStr, int size,
-      long newModTime) throws IOException {
-    return basicFileStatus(strToPath(pathStr), size, false,
-        newModTime);
-  }
-
-  protected void verifyFileStatus(FileStatus status, long size) {
-    S3ATestUtils.verifyFileStatus(status, size, BLOCK_SIZE, modTime);
-  }
-
-  protected S3AFileStatus makeDirStatus(String pathStr) throws IOException {
-    return basicFileStatus(strToPath(pathStr), 0, true, modTime);
-  }
-
-  /**
-   * Verify the directory file status. Subclass may verify additional fields.
-   */
-  protected void verifyDirStatus(S3AFileStatus status) {
-    assertTrue("Is a dir", status.isDirectory());
-    assertEquals("zero length", 0, status.getLen());
-  }
-
-  long getModTime() {
-    return modTime;
-  }
-
-  long getAccessTime() {
-    return accessTime;
-  }
-
-  protected static long getTime() {
-    return System.currentTimeMillis();
-  }
-
-  protected static ITtlTimeProvider getTtlTimeProvider() {
-    return ttlTimeProvider;
-  }
-
-  /**
-   * Put a file to the shared DDB table.
-   * @param key key
-   * @param time timestamp.
-   * @param operationState ongoing state
-   * @return the entry
-   * @throws IOException IO failure
-   */
-  protected PathMetadata putFile(
-      final String key,
-      final long time,
-      BulkOperationState operationState) throws IOException {
-    PathMetadata meta = new PathMetadata(makeFileStatus(key, 1, time));
-    meta.setLastUpdated(time);
-    ms.put(meta, operationState);
-    return meta;
-  }
-
-  /**
-   * Put a dir to the shared DDB table.
-   * @param key key
-   * @param time timestamp.
-   * @param operationState ongoing state
-   * @return the entry
-   * @throws IOException IO failure
-   */
-  protected PathMetadata putDir(
-      final String key,
-      final long time,
-      BulkOperationState operationState) throws IOException {
-    PathMetadata meta = new PathMetadata(
-        basicFileStatus(strToPath(key), 0, true, time));
-    meta.setLastUpdated(time);
-    ms.put(meta, operationState);
-    return meta;
-  }
-
-  /**
-   * Put a tombstone to the shared DDB table.
-   * @param key key
-   * @param time timestamp.
-   * @param operationState ongoing state
-   * @return the entry
-   * @throws IOException IO failure
-   */
-  protected PathMetadata putTombstone(
-      final String key,
-      final long time,
-      BulkOperationState operationState) throws IOException {
-    PathMetadata meta = tombstone(strToPath(key), time);
-    ms.put(meta, operationState);
-    return meta;
-  }
-
-  /**
-   * Create a tombstone from the timestamp.
-   * @param path path to tombstone
-   * @param time timestamp.
-   * @return the entry.
-   */
-  public static PathMetadata tombstone(Path path, long time) {
-    S3AFileStatus s3aStatus = new S3AFileStatus(0,
-        time, path, 0, null,
-        null, null);
-    return new PathMetadata(s3aStatus, Tristate.UNKNOWN, true);
-  }
-
-  @Override
-  protected Timeout retrieveTestTimeout() {
-    return Timeout.millis(S3ATestConstants.S3A_TEST_TIMEOUT);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
deleted file mode 100644
index 5f7a6fbd072df..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.assertj.core.api.Assertions;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.junit.Assert.*;
-
-/**
- * Unit tests of {@link DirListingMetadata}.
- */
-public class TestDirListingMetadata {
-
-  private static final String TEST_OWNER = "hadoop";
-  public static final String TEST_ETAG = "abc";
-  public static final String TEST_VERSION_ID = "def";
-
-  @Rule
-  public ExpectedException exception = ExpectedException.none();
-
-  @Test
-  public void testNullPath() {
-    exception.expect(NullPointerException.class);
-    exception.expectMessage(notNullValue(String.class));
-    new DirListingMetadata(null, null, false);
-  }
-
-  @Test
-  public void testNullListing() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertTrue(meta.getListing().isEmpty());
-    assertFalse(meta.isAuthoritative());
-  }
-
-  @Test
-  public void testEmptyListing() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path,
-        new ArrayList<PathMetadata>(0),
-        false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertTrue(meta.getListing().isEmpty());
-    assertFalse(meta.isAuthoritative());
-  }
-
-  @Test
-  public void testListing() {
-    Path path = new Path("/path");
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER,
-            TEST_ETAG, TEST_VERSION_ID));
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    DirListingMetadata meta = new DirListingMetadata(path, listing, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertFalse(meta.getListing().isEmpty());
-    assertTrue(meta.getListing().contains(pathMeta1));
-    assertTrue(meta.getListing().contains(pathMeta2));
-    assertTrue(meta.getListing().contains(pathMeta3));
-    assertFalse(meta.isAuthoritative());
-  }
-
-  @Test
-  public void testListingUnmodifiable() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = makeTwoDirsOneFile(path);
-    assertNotNull(meta.getListing());
-    exception.expect(UnsupportedOperationException.class);
-    meta.getListing().clear();
-  }
-
-  @Test
-  public void testAuthoritative() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, true);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertTrue(meta.getListing().isEmpty());
-    assertTrue(meta.isAuthoritative());
-  }
-
-  @Test
-  public void testSetAuthoritative() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertTrue(meta.getListing().isEmpty());
-    assertFalse(meta.isAuthoritative());
-    meta.setAuthoritative(true);
-    assertTrue(meta.isAuthoritative());
-  }
-
-  @Test
-  public void testGet() {
-    Path path = new Path("/path");
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER,
-            TEST_ETAG, TEST_VERSION_ID));
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    DirListingMetadata meta = new DirListingMetadata(path, listing, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertFalse(meta.getListing().isEmpty());
-    assertTrue(meta.getListing().contains(pathMeta1));
-    assertTrue(meta.getListing().contains(pathMeta2));
-    assertTrue(meta.getListing().contains(pathMeta3));
-    assertFalse(meta.isAuthoritative());
-    assertEquals(pathMeta1, meta.get(pathMeta1.getFileStatus().getPath()));
-    assertEquals(pathMeta2, meta.get(pathMeta2.getFileStatus().getPath()));
-    assertEquals(pathMeta3, meta.get(pathMeta3.getFileStatus().getPath()));
-    assertNull(meta.get(new Path(path, "notfound")));
-  }
-
-  @Test
-  public void testGetNull() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(NullPointerException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.get(null);
-  }
-
-  @Test
-  public void testGetRoot() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.get(new Path("/"));
-  }
-
-  @Test
-  public void testGetNotChild() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.get(new Path("/different/ancestor"));
-  }
-
-  @Test
-  public void testPut() {
-    Path path = new Path("/path");
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER,
-            TEST_ETAG, TEST_VERSION_ID));
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    DirListingMetadata meta = new DirListingMetadata(path, listing, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertFalse(meta.getListing().isEmpty());
-    assertTrue(meta.getListing().contains(pathMeta1));
-    assertTrue(meta.getListing().contains(pathMeta2));
-    assertTrue(meta.getListing().contains(pathMeta3));
-    assertFalse(meta.isAuthoritative());
-    PathMetadata pathMeta4 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir3"), TEST_OWNER));
-    meta.put(pathMeta4);
-    assertTrue(meta.getListing().contains(pathMeta4));
-    assertEquals(pathMeta4, meta.get(pathMeta4.getFileStatus().getPath()));
-  }
-
-  @Test
-  public void testPutNull() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(NullPointerException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.put(null);
-  }
-
-  @Test
-  public void testPutNullPath() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(NullPointerException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.put(new PathMetadata(new S3AFileStatus(true, null, TEST_OWNER)));
-  }
-
-  @Test
-  public void testPutRoot() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.put(new PathMetadata(new S3AFileStatus(true, new Path("/"),
-        TEST_OWNER)));
-  }
-
-  @Test
-  public void testPutNotChild() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.put(new PathMetadata(
-        new S3AFileStatus(true, new Path("/different/ancestor"), TEST_OWNER)));
-  }
-
-  @Test
-  public void testRemove() {
-    Path path = new Path("/path");
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER,
-            TEST_ETAG, TEST_VERSION_ID));
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    DirListingMetadata meta = new DirListingMetadata(path, listing, false);
-    assertEquals(path, meta.getPath());
-    assertNotNull(meta.getListing());
-    assertFalse(meta.getListing().isEmpty());
-    assertTrue(meta.getListing().contains(pathMeta1));
-    assertTrue(meta.getListing().contains(pathMeta2));
-    assertTrue(meta.getListing().contains(pathMeta3));
-    assertFalse(meta.isAuthoritative());
-    meta.remove(pathMeta1.getFileStatus().getPath());
-    assertFalse(meta.getListing().contains(pathMeta1));
-    assertNull(meta.get(pathMeta1.getFileStatus().getPath()));
-  }
-
-  @Test
-  public void testRemoveNull() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(NullPointerException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.remove(null);
-  }
-
-  @Test
-  public void testRemoveRoot() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.remove(new Path("/"));
-  }
-
-  @Test
-  public void testRemoveNotChild() {
-    Path path = new Path("/path");
-    DirListingMetadata meta = new DirListingMetadata(path, null, false);
-    exception.expect(IllegalArgumentException.class);
-    exception.expectMessage(notNullValue(String.class));
-    meta.remove(new Path("/different/ancestor"));
-  }
-
-
-  @Test
-  public void testRemoveExpiredEntriesFromListing() {
-    long ttl = 9;
-    long oldTime = 100;
-    long newTime = 110;
-    long now = 110;
-
-    Path path = new Path("/path");
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(path, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(path, "file1"), 8192, TEST_OWNER,
-            TEST_ETAG, TEST_VERSION_ID));
-    pathMeta1.setLastUpdated(oldTime);
-    pathMeta2.setLastUpdated(0);
-    pathMeta3.setLastUpdated(newTime);
-
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    DirListingMetadata meta = new DirListingMetadata(path, listing, false);
-
-    List<PathMetadata> expired = meta.removeExpiredEntriesFromListing(ttl,
-        now);
-
-    Assertions.assertThat(meta.getListing())
-        .describedAs("Metadata listing for %s", path)
-        .doesNotContain(pathMeta1)
-        .contains(pathMeta2)
-        .contains(pathMeta3);
-    Assertions.assertThat(expired)
-        .describedAs("Expire entries underr %s", path)
-        .doesNotContain(pathMeta2)
-        .contains(pathMeta1);
-  }
-
-  /**
-   * Create DirListingMetadata with two dirs and one file living in directory
-   * 'parent'.
-   */
-  private static DirListingMetadata makeTwoDirsOneFile(Path parent) {
-    PathMetadata pathMeta1 = new PathMetadata(
-        new S3AFileStatus(true, new Path(parent, "dir1"), TEST_OWNER));
-    PathMetadata pathMeta2 = new PathMetadata(
-        new S3AFileStatus(true, new Path(parent, "dir2"), TEST_OWNER));
-    PathMetadata pathMeta3 = new PathMetadata(
-        new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192,
-            TEST_OWNER, TEST_ETAG, TEST_VERSION_ID));
-    List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
-    return new DirListingMetadata(parent, listing, false);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
deleted file mode 100644
index 602a072aac2f6..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.List;
-
-import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
-import com.amazonaws.waiters.WaiterTimedOutException;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.s3a.AWSClientIOException;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.test.HadoopTestBase;
-import org.apache.hadoop.fs.Path;
-
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStoreTableManager.translateTableWaitFailure;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-
-/**
- * Unit test suite for misc dynamoDB metastore operations.
- */
-public class TestDynamoDBMiscOperations extends HadoopTestBase {
-
-  private static final String TIMEOUT_ERROR_MESSAGE
-      = "Table table-name did not transition into ACTIVE state.";
-
-  @Test
-  public void testUnwrapTableWaitTimeout() throws Throwable {
-    final Exception waiterTimedOut =
-        new WaiterTimedOutException("waiter timed out");
-    final AWSClientIOException ex = intercept(AWSClientIOException.class,
-        TIMEOUT_ERROR_MESSAGE,
-        () -> {
-          throw translateTableWaitFailure("example",
-              new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE,
-                  waiterTimedOut));
-        });
-    assertEquals(waiterTimedOut, ex.getCause());
-  }
-
-  @Test
-  public void testTranslateIllegalArgumentException() throws Throwable {
-    final IllegalArgumentException e =
-        new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE);
-    final IOException ex = intercept(IOException.class,
-        TIMEOUT_ERROR_MESSAGE,
-        () -> {
-          throw translateTableWaitFailure("example", e);
-        });
-    assertEquals(e, ex.getCause());
-  }
-
-  @Test
-  public void testTranslateWrappedDDBException() throws Throwable {
-    final Exception inner = new ResourceNotFoundException("ddb");
-    final IllegalArgumentException e =
-        new IllegalArgumentException("outer", inner);
-    final FileNotFoundException ex = intercept(FileNotFoundException.class,
-        "outer",
-        () -> {
-          throw translateTableWaitFailure("example", e);
-        });
-    assertEquals(inner, ex.getCause());
-  }
-
-  @Test
-  public void testTranslateWrappedOtherException() throws Throwable {
-    final Exception inner = new NullPointerException("npe");
-    final IllegalArgumentException e =
-        new IllegalArgumentException("outer", inner);
-    final IOException ex = intercept(IOException.class,
-        "outer",
-        () -> {
-          throw translateTableWaitFailure("example", e);
-        });
-    assertEquals(e, ex.getCause());
-  }
-
-  @Test
-  public void testInnerListChildrenDirectoryNpe() throws Exception {
-    DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
-    Path p = mock(Path.class);
-    List<PathMetadata> metas = mock(List.class);
-
-    when(metas.isEmpty()).thenReturn(false);
-    DDBPathMetadata dirPathMeta = null;
-
-    assertNull("The return value should be null.",
-        ddbms.getDirListingMetadataFromDirMetaAndList(p, metas, dirPathMeta));
-  }
-
-  @Test
-  public void testAncestorStateForDir() throws Throwable {
-    final DynamoDBMetadataStore.AncestorState ancestorState
-        = new DynamoDBMetadataStore.AncestorState(
-            null, BulkOperationState.OperationType.Rename, null);
-
-    // path 1 is a directory
-    final Path path1 = new Path("s3a://bucket/1");
-    final S3AFileStatus status1 = new S3AFileStatus(true,
-        path1, "hadoop");
-    final DDBPathMetadata md1 = new DDBPathMetadata(
-        status1);
-    ancestorState.put(md1);
-    assertTrue("Status not found in ancestors",
-        ancestorState.contains(path1));
-    final DDBPathMetadata result = ancestorState.get(path1);
-    assertEquals(status1, result.getFileStatus());
-    assertTrue("Lookup failed",
-        ancestorState.findEntry(path1, true));
-    final Path path2 = new Path("s3a://bucket/2");
-    assertFalse("Lookup didn't fail",
-        ancestorState.findEntry(path2, true));
-    assertFalse("Lookup didn't fail",
-        ancestorState.contains(path2));
-    assertNull("Lookup didn't fail",
-        ancestorState.get(path2));
-  }
-
-  @Test
-  public void testAncestorStateForFile() throws Throwable {
-    final DynamoDBMetadataStore.AncestorState ancestorState
-        = new DynamoDBMetadataStore.AncestorState(
-            null, BulkOperationState.OperationType.Rename, null);
-
-    // path 1 is a file
-    final Path path1 = new Path("s3a://bucket/1");
-    final S3AFileStatus status1 = new S3AFileStatus(
-        1024_1024_1024L,
-        0,
-        path1,
-        32_000_000,
-        "hadoop",
-        "e4",
-        "f5");
-    final DDBPathMetadata md1 = new DDBPathMetadata(
-        status1);
-    ancestorState.put(md1);
-    assertTrue("Lookup failed",
-        ancestorState.findEntry(path1, false));
-    intercept(PathIOException.class,
-        DynamoDBMetadataStore.E_INCONSISTENT_UPDATE,
-        () -> ancestorState.findEntry(path1, true));
-  }
-
-  @Test
-  public void testNoBulkRenameThroughInitiateBulkWrite() throws Throwable {
-    intercept(IllegalArgumentException.class,
-        () -> S3Guard.initiateBulkWrite(null,
-            BulkOperationState.OperationType.Rename, null));
-  }
-  @Test
-  public void testInitiateBulkWrite() throws Throwable {
-    assertNull(
-        S3Guard.initiateBulkWrite(null,
-            BulkOperationState.OperationType.Put, null));
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
deleted file mode 100644
index 8df61dbe1a836..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Ticker;
-import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-import org.junit.Assume;
-import org.junit.Test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * MetadataStore unit test for {@link LocalMetadataStore}.
- */
-public class TestLocalMetadataStore extends MetadataStoreTestBase {
-
-
-  private final static class LocalMSContract extends AbstractMSContract {
-
-    private FileSystem fs;
-
-    private LocalMSContract() throws IOException {
-      this(new Configuration());
-    }
-
-    private LocalMSContract(Configuration config) throws IOException {
-      fs = FileSystem.getLocal(config);
-    }
-
-    @Override
-    public FileSystem getFileSystem() {
-      return fs;
-    }
-
-    @Override
-    public MetadataStore getMetadataStore() throws IOException {
-      LocalMetadataStore lms = new LocalMetadataStore();
-      return lms;
-    }
-  }
-
-  @Override
-  public AbstractMSContract createContract() throws IOException {
-    return new LocalMSContract();
-  }
-
-  @Override
-  public AbstractMSContract createContract(Configuration conf) throws
-      IOException {
-    return new LocalMSContract(conf);
-  }
-
-  @Override protected String getPathStringForPrune(String path)
-      throws Exception{
-    return path;
-  }
-
-  @Test
-  public void testClearByAncestor() throws Exception {
-    Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder().build();
-
-    // 1. Test paths without scheme/host
-    assertClearResult(cache, "", "/", 0);
-    assertClearResult(cache, "", "/dirA/dirB", 2);
-    assertClearResult(cache, "", "/invalid", 5);
-
-
-    // 2. Test paths w/ scheme/host
-    String p = "s3a://fake-bucket-name";
-    assertClearResult(cache, p, "/", 0);
-    assertClearResult(cache, p, "/dirA/dirB", 2);
-    assertClearResult(cache, p, "/invalid", 5);
-  }
-
-  static class TestTicker extends Ticker {
-    private long myTicker = 0;
-    @Override public long read() {
-      return myTicker;
-    }
-    public void set(long val) {
-      this.myTicker = val;
-    }
-
-  }
-
-  /**
-   * Test that time eviction in cache used in {@link LocalMetadataStore}
-   * implementation working properly.
-   *
-   * The test creates a Ticker instance, which will be used to control the
-   * internal clock of the cache to achieve eviction without having to wait
-   * for the system clock.
-   * The test creates 3 entry: 2nd and 3rd entry will survive the eviction,
-   * because it will be created later than the 1st - using the ticker.
-   */
-  @Test
-  public void testCacheTimedEvictionAfterWrite() {
-    TestTicker testTicker = new TestTicker();
-    final long t0 = testTicker.read();
-    final long t1 = t0 + 100;
-    final long t2 = t1 + 100;
-
-    final long ttl = t1 + 50; // between t1 and t2
-
-    Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder()
-        .expireAfterWrite(ttl,
-            TimeUnit.NANOSECONDS /* nanos to avoid conversions */)
-        .ticker(testTicker)
-        .build();
-
-    String p = "s3a://fake-bucket-name";
-    Path path1 = new Path(p + "/dirA/dirB/file1");
-    Path path2 = new Path(p + "/dirA/dirB/file2");
-    Path path3 = new Path(p + "/dirA/dirB/file3");
-
-    // Test time is t0
-    populateEntry(cache, path1);
-
-    // set new value on the ticker, so the next two entries will be added later
-    testTicker.set(t1);  // Test time is now t1
-    populateEntry(cache, path2);
-    populateEntry(cache, path3);
-
-    assertEquals("Cache should contain 3 records before eviction",
-        3, cache.size());
-    LocalMetadataEntry pm1 = cache.getIfPresent(path1);
-    assertNotNull("PathMetadata should not be null before eviction", pm1);
-
-    // set the ticker to a time when timed eviction should occur
-    // for the first entry
-    testTicker.set(t2);
-
-    // call cleanup explicitly, as timed expiration is performed with
-    // periodic maintenance during writes and occasionally during reads only
-    cache.cleanUp();
-
-    assertEquals("Cache size should be 2 after eviction", 2, cache.size());
-    pm1 = cache.getIfPresent(path1);
-    assertNull("PathMetadata should be null after eviction", pm1);
-  }
-
-
-  @Test
-  public void testUpdateParentLastUpdatedOnPutNewParent() throws Exception {
-    Assume.assumeTrue("This test only applies if metadatastore does not allow"
-        + " missing values (skip for NullMS).", !allowMissing());
-
-    ITtlTimeProvider tp = mock(ITtlTimeProvider.class);
-    ITtlTimeProvider originalTimeProvider = getTtlTimeProvider();
-
-    long now = 100L;
-
-    final String parent = "/parentUpdated-" + UUID.randomUUID();
-    final String child = parent + "/file1";
-
-    try {
-      when(tp.getNow()).thenReturn(now);
-
-      // create a file
-      ms.put(new PathMetadata(makeFileStatus(child, 100), tp.getNow()),
-          null);
-      final PathMetadata fileMeta = ms.get(strToPath(child));
-      assertEquals("lastUpdated field of first file should be equal to the "
-          + "mocked value", now, fileMeta.getLastUpdated());
-
-      final DirListingMetadata listing = ms.listChildren(strToPath(parent));
-      assertEquals("Listing lastUpdated field should be equal to the mocked "
-          + "time value.", now, listing.getLastUpdated());
-
-    } finally {
-      ms.setTtlTimeProvider(originalTimeProvider);
-    }
-
-  }
-
-
-  private static void populateMap(Cache<Path, LocalMetadataEntry> cache,
-      String prefix) {
-    populateEntry(cache, new Path(prefix + "/dirA/dirB/"));
-    populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC"));
-    populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC/file1"));
-    populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC/file2"));
-    populateEntry(cache, new Path(prefix + "/dirA/file1"));
-  }
-
-  private static void populateEntry(Cache<Path, LocalMetadataEntry> cache,
-      Path path) {
-    S3AFileStatus s3aStatus = new S3AFileStatus(Tristate.UNKNOWN, path, null);
-    cache.put(path, new LocalMetadataEntry(new PathMetadata(s3aStatus)));
-  }
-
-  private static long sizeOfMap(Cache<Path, LocalMetadataEntry> cache) {
-    return cache.asMap().values().stream()
-        .filter(entry -> !entry.getFileMeta().isDeleted())
-        .count();
-  }
-
-  private static void assertClearResult(Cache<Path, LocalMetadataEntry> cache,
-      String prefixStr, String pathStr, int leftoverSize) throws IOException {
-    populateMap(cache, prefixStr);
-    LocalMetadataStore.deleteEntryByAncestor(new Path(prefixStr + pathStr),
-        cache, true, getTtlTimeProvider());
-    assertEquals(String.format("Cache should have %d entries", leftoverSize),
-        leftoverSize, sizeOfMap(cache));
-    cache.invalidateAll();
-  }
-
-  @Override
-  protected void verifyFileStatus(FileStatus status, long size) {
-    S3ATestUtils.verifyFileStatus(status, size, REPLICATION, getModTime(),
-        getAccessTime(),
-        BLOCK_SIZE, OWNER, GROUP, PERMISSION);
-  }
-
-  @Override
-  protected void verifyDirStatus(S3AFileStatus status) {
-    S3ATestUtils.verifyDirStatus(status, REPLICATION, OWNER);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestMetastoreChecking.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestMetastoreChecking.java
new file mode 100644
index 0000000000000..185ed024e4689
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestMetastoreChecking.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.net.URI;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_DYNAMO;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_LOCAL;
+import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
+import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.NULL_METADATA_STORE;
+import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.checkNoS3Guard;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Verify thqt the metastore checking
+ * is forgiving for local/null stores, and unforgiving for
+ * DDB or other bindings.
+ */
+@SuppressWarnings("deprecation")
+public class TestMetastoreChecking extends AbstractHadoopTestBase {
+
+  private final Path root = new Path("/");
+
+  private URI fsUri;
+
+  private static final String BASE = "s3a://bucket";
+
+  @Before
+  public void setup() throws Exception {
+    fsUri = new URI(BASE +"/");
+  }
+
+  private Configuration chooseStore(String classname) {
+    Configuration conf = new Configuration(false);
+    if (classname != null) {
+      conf.set(S3_METADATA_STORE_IMPL, classname, "code");
+    }
+    return conf;
+  }
+
+  @Test
+  public void testNoClass() throws Throwable {
+    checkOutcome(null, false);
+  }
+
+  @Test
+  public void testNullClass() throws Throwable {
+    checkOutcome(NULL_METADATA_STORE, true);
+  }
+
+  @Test
+  public void testLocalStore() throws Throwable {
+    checkOutcome(S3GUARD_METASTORE_LOCAL, true);
+  }
+
+  @Test
+  public void testDDBStore() throws Throwable {
+    intercept(PathIOException.class, () ->
+        checkOutcome(S3GUARD_METASTORE_DYNAMO, false));
+  }
+
+  @Test
+  public void testUnknownStore() throws Throwable {
+    intercept(PathIOException.class, "unknownStore", () ->
+        checkOutcome("unknownStore", false));
+  }
+
+  private void checkOutcome(final String classname, final boolean outcome) throws PathIOException {
+    Configuration conf = chooseStore(classname);
+
+    Assertions.assertThat(checkNoS3Guard(fsUri, conf))
+        .describedAs("check with classname %s", classname)
+        .isEqualTo(outcome);
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
deleted file mode 100644
index 2e0bc4b7e4f0e..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-
-import java.io.IOException;
-
-/**
- * Run MetadataStore unit tests on the NullMetadataStore implementation.
- */
-public class TestNullMetadataStore extends MetadataStoreTestBase {
-  private static class NullMSContract extends AbstractMSContract {
-    @Override
-    public FileSystem getFileSystem() throws IOException {
-      Configuration config = new Configuration();
-      return FileSystem.getLocal(config);
-    }
-
-    @Override
-    public MetadataStore getMetadataStore() throws IOException {
-      return new NullMetadataStore();
-    }
-  }
-
-  /** This MetadataStore always says "I don't know, ask the backing store". */
-  @Override
-  public boolean allowMissing() {
-    return true;
-  }
-
-  @Override protected String getPathStringForPrune(String path)
-      throws Exception {
-    return path;
-  }
-
-  @Override
-  public AbstractMSContract createContract() {
-    return new NullMSContract();
-  }
-
-  @Override
-  public AbstractMSContract createContract(Configuration conf) {
-    return createContract();
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestObjectChangeDetectionAttributes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestObjectChangeDetectionAttributes.java
deleted file mode 100644
index f001262b36a04..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestObjectChangeDetectionAttributes.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-
-import com.amazonaws.services.s3.Headers;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.amazonaws.services.s3.model.UploadPartResult;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.AbstractS3AMockTest;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_MODE_SERVER;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE_ETAG;
-import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE_VERSION_ID;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.when;
-import static org.mockito.hamcrest.MockitoHamcrest.argThat;
-
-/**
- * Unit tests to ensure object eTag and versionId are captured on S3 PUT and
- * used on GET.
- * Further (integration) testing is performed in
- * {@link org.apache.hadoop.fs.s3a.ITestS3ARemoteFileChanged}.
- */
-@RunWith(Parameterized.class)
-public class TestObjectChangeDetectionAttributes extends AbstractS3AMockTest {
-  private final String changeDetectionSource;
-
-  public TestObjectChangeDetectionAttributes(String changeDetectionSource) {
-    this.changeDetectionSource = changeDetectionSource;
-  }
-
-  @Parameterized.Parameters(name = "change={0}")
-  public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][]{
-        {CHANGE_DETECT_SOURCE_ETAG},
-        {CHANGE_DETECT_SOURCE_VERSION_ID}
-    });
-  }
-
-  @Override
-  public Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    conf.setClass(Constants.S3_METADATA_STORE_IMPL,
-        LocalMetadataStore.class, MetadataStore.class);
-    conf.set(CHANGE_DETECT_SOURCE, changeDetectionSource);
-    conf.set(CHANGE_DETECT_MODE, CHANGE_DETECT_MODE_SERVER);
-    return conf;
-  }
-
-  /**
-   * Tests a file uploaded with a single PUT to ensure eTag is captured and used
-   * on file read.
-   */
-  @Test
-  public void testCreateAndReadFileSinglePart() throws Exception {
-    String bucket = "s3a://mock-bucket/";
-    String file = "single-part-file";
-    Path path = new Path(bucket, file);
-    byte[] content = "content".getBytes();
-    String eTag = "abc";
-    String versionId = "def";
-
-    putObject(file, path, content, eTag, versionId);
-
-    // make sure the eTag and versionId were put into the metadataStore
-    assertVersionAttributes(path, eTag, versionId);
-
-    // Ensure underlying S3 getObject call uses the stored eTag or versionId
-    // when reading data back.  If it doesn't, the read won't work and the
-    // assert will fail.
-    assertContent(file, path, content, eTag, versionId);
-
-    // test overwrite
-    byte[] newConent = "newcontent".getBytes();
-    String newETag = "newETag";
-    String newVersionId = "newVersionId";
-
-    putObject(file, path, newConent, newETag, newVersionId);
-    assertVersionAttributes(path, newETag, newVersionId);
-    assertContent(file, path, newConent, newETag, newVersionId);
-  }
-
-  /**
-   * Tests a file uploaded with multi-part upload to ensure eTag is captured
-   * and used on file read.
-   */
-  @Test
-  public void testCreateAndReadFileMultiPart() throws Exception {
-    String bucket = "s3a://mock-bucket/";
-    String file = "multi-part-file";
-    Path path = new Path(bucket, file);
-    byte[] content = new byte[Constants.MULTIPART_MIN_SIZE + 1];
-    String eTag = "abc";
-    String versionId = "def";
-
-    multipartUpload(file, path, content, eTag, versionId);
-
-    // make sure the eTag and versionId were put into the metadataStore
-    assertVersionAttributes(path, eTag, versionId);
-
-    // Ensure underlying S3 getObject call uses the stored eTag or versionId
-    // when reading data back.  If it doesn't, the read won't work and the
-    // assert will fail.
-    assertContent(file, path, content, eTag, versionId);
-
-    // test overwrite
-    byte[] newContent = new byte[Constants.MULTIPART_MIN_SIZE + 1];
-    Arrays.fill(newContent, (byte) 1);
-    String newETag = "newETag";
-    String newVersionId = "newVersionId";
-
-    multipartUpload(file, path, newContent, newETag, newVersionId);
-    assertVersionAttributes(path, newETag, newVersionId);
-    assertContent(file, path, newContent, newETag, newVersionId);
-  }
-
-  private void putObject(String file, Path path, byte[] content,
-      String eTag, String versionId) throws IOException {
-    PutObjectResult putObjectResult = new PutObjectResult();
-    ObjectMetadata objectMetadata = new ObjectMetadata();
-    objectMetadata.setContentLength(content.length);
-    putObjectResult.setMetadata(objectMetadata);
-    putObjectResult.setETag(eTag);
-    putObjectResult.setVersionId(versionId);
-
-    when(s3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
-        .thenThrow(NOT_FOUND);
-    when(s3.putObject(argThat(correctPutObjectRequest(file))))
-        .thenReturn(putObjectResult);
-    ListObjectsV2Result emptyListing = new ListObjectsV2Result();
-    when(s3.listObjectsV2(argThat(correctListObjectsRequest(file + "/"))))
-        .thenReturn(emptyListing);
-
-    FSDataOutputStream outputStream = fs.create(path);
-    outputStream.write(content);
-    outputStream.close();
-  }
-
-  private void multipartUpload(String file, Path path, byte[] content,
-      String eTag, String versionId) throws IOException {
-    CompleteMultipartUploadResult uploadResult =
-        new CompleteMultipartUploadResult();
-    uploadResult.setVersionId(versionId);
-
-    when(s3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
-        .thenThrow(NOT_FOUND);
-
-    InitiateMultipartUploadResult initiateMultipartUploadResult =
-        new InitiateMultipartUploadResult();
-    initiateMultipartUploadResult.setUploadId("uploadId");
-    when(s3.initiateMultipartUpload(
-        argThat(correctInitiateMultipartUploadRequest(file))))
-        .thenReturn(initiateMultipartUploadResult);
-
-    UploadPartResult uploadPartResult = new UploadPartResult();
-    uploadPartResult.setETag("partETag");
-    when(s3.uploadPart(argThat(correctUploadPartRequest(file))))
-        .thenReturn(uploadPartResult);
-
-    CompleteMultipartUploadResult multipartUploadResult =
-        new CompleteMultipartUploadResult();
-    multipartUploadResult.setETag(eTag);
-    multipartUploadResult.setVersionId(versionId);
-    when(s3.completeMultipartUpload(
-        argThat(correctMultipartUploadRequest(file))))
-        .thenReturn(multipartUploadResult);
-
-    ListObjectsV2Result emptyListing = new ListObjectsV2Result();
-    when(s3.listObjectsV2(argThat(correctListObjectsRequest(file + "/"))))
-        .thenReturn(emptyListing);
-
-    FSDataOutputStream outputStream = fs.create(path);
-    outputStream.write(content);
-    outputStream.close();
-  }
-
-  private void assertContent(String file, Path path, byte[] content,
-      String eTag, String versionId) throws IOException {
-    S3Object s3Object = new S3Object();
-    ObjectMetadata metadata = new ObjectMetadata();
-    metadata.setHeader(Headers.S3_VERSION_ID, versionId);
-    metadata.setHeader(Headers.ETAG, eTag);
-    s3Object.setObjectMetadata(metadata);
-    s3Object.setObjectContent(new ByteArrayInputStream(content));
-    when(s3.getObject(argThat(correctGetObjectRequest(file, eTag, versionId))))
-        .thenReturn(s3Object);
-    FSDataInputStream inputStream = fs.open(path);
-    byte[] readContent = IOUtils.toByteArray(inputStream);
-    assertArrayEquals(content, readContent);
-  }
-
-  private void assertVersionAttributes(Path path, String eTag, String versionId)
-      throws IOException {
-    MetadataStore metadataStore = fs.getMetadataStore();
-    PathMetadata pathMetadata = metadataStore.get(path);
-    assertNotNull(pathMetadata);
-    S3AFileStatus fileStatus = pathMetadata.getFileStatus();
-    assertEquals(eTag, fileStatus.getETag());
-    assertEquals(versionId, fileStatus.getVersionId());
-  }
-
-  private Matcher<GetObjectRequest> correctGetObjectRequest(final String key,
-      final String eTag, final String versionId) {
-    return new BaseMatcher<GetObjectRequest>() {
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof GetObjectRequest) {
-          GetObjectRequest getObjectRequest = (GetObjectRequest) item;
-          if (getObjectRequest.getKey().equals(key)) {
-            if (changeDetectionSource.equals(
-                CHANGE_DETECT_SOURCE_ETAG)) {
-              return getObjectRequest.getMatchingETagConstraints()
-                  .contains(eTag);
-            } else if (changeDetectionSource.equals(
-                CHANGE_DETECT_SOURCE_VERSION_ID)) {
-              return getObjectRequest.getVersionId().equals(versionId);
-            }
-          }
-        }
-        return false;
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key and "
-            + changeDetectionSource
-            + " matches");
-      }
-    };
-  }
-
-  private Matcher<UploadPartRequest> correctUploadPartRequest(
-      final String key) {
-    return new BaseMatcher<UploadPartRequest>() {
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof UploadPartRequest) {
-          UploadPartRequest request = (UploadPartRequest) item;
-          return request.getKey().equals(key);
-        }
-        return false;
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key matches");
-      }
-    };
-  }
-
-  private Matcher<InitiateMultipartUploadRequest>
-      correctInitiateMultipartUploadRequest(final String key) {
-    return new BaseMatcher<InitiateMultipartUploadRequest>() {
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key matches");
-      }
-
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof InitiateMultipartUploadRequest) {
-          InitiateMultipartUploadRequest request =
-              (InitiateMultipartUploadRequest) item;
-          return request.getKey().equals(key);
-        }
-        return false;
-      }
-    };
-  }
-
-  private Matcher<CompleteMultipartUploadRequest>
-      correctMultipartUploadRequest(final String key) {
-    return new BaseMatcher<CompleteMultipartUploadRequest>() {
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof CompleteMultipartUploadRequest) {
-          CompleteMultipartUploadRequest request =
-              (CompleteMultipartUploadRequest) item;
-          return request.getKey().equals(key);
-        }
-        return false;
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key matches");
-      }
-    };
-  }
-
-  private Matcher<ListObjectsV2Request> correctListObjectsRequest(
-      final String key) {
-    return new BaseMatcher<ListObjectsV2Request>() {
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof ListObjectsV2Request) {
-          ListObjectsV2Request listObjectsRequest =
-              (ListObjectsV2Request) item;
-          return listObjectsRequest.getPrefix().equals(key);
-        }
-        return false;
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key matches");
-      }
-    };
-  }
-
-  private Matcher<PutObjectRequest> correctPutObjectRequest(
-      final String key) {
-    return new BaseMatcher<PutObjectRequest>() {
-      @Override
-      public boolean matches(Object item) {
-        if (item instanceof PutObjectRequest) {
-          PutObjectRequest putObjectRequest = (PutObjectRequest) item;
-          return putObjectRequest.getKey().equals(key);
-        }
-        return false;
-      }
-
-      @Override
-      public void describeTo(Description description) {
-        description.appendText("key matches");
-      }
-    };
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
deleted file mode 100644
index f2d27f333b03e..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.Callable;
-
-import com.amazonaws.services.dynamodbv2.document.Item;
-import com.amazonaws.services.dynamodbv2.document.KeyAttribute;
-import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
-import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
-import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
-
-import static com.amazonaws.services.dynamodbv2.model.KeyType.HASH;
-import static com.amazonaws.services.dynamodbv2.model.KeyType.RANGE;
-import static com.amazonaws.services.dynamodbv2.model.ScalarAttributeType.S;
-import static org.hamcrest.CoreMatchers.anyOf;
-import static org.hamcrest.CoreMatchers.is;
-
-import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION_MARKER_ITEM_NAME;
-import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.VERSION;
-import static org.mockito.Mockito.never;
-
-/**
- * Test the PathMetadataDynamoDBTranslation is able to translate between domain
- * model objects and DynamoDB items.
- */
-@RunWith(Parameterized.class)
-public class TestPathMetadataDynamoDBTranslation extends Assert {
-
-  private static final Path TEST_DIR_PATH = new Path("s3a://test-bucket/myDir");
-  private static final long TEST_FILE_LENGTH = 100;
-  private static final long TEST_MOD_TIME = 9999;
-  private static final long TEST_BLOCK_SIZE = 128;
-  private static final String TEST_ETAG = "abc";
-  private static final String TEST_VERSION_ID = "def";
-  private static final Path TEST_FILE_PATH = new Path(TEST_DIR_PATH, "myFile");
-
-  private final Item testFileItem;
-  private final DDBPathMetadata testFilePathMetadata;
-  private final Item testDirItem;
-  private final DDBPathMetadata testDirPathMetadata;
-
-  @Parameterized.Parameters
-  public static Collection<Object[]> params() throws IOException {
-    String username = UserGroupInformation.getCurrentUser().getShortUserName();
-
-    return Arrays.asList(new Object[][]{
-        // with etag and versionId
-        {
-            new Item()
-                .withPrimaryKey(PARENT,
-                    pathToParentKey(TEST_FILE_PATH.getParent()),
-                    CHILD, TEST_FILE_PATH.getName())
-                .withBoolean(IS_DIR, false)
-                .withLong(FILE_LENGTH, TEST_FILE_LENGTH)
-                .withLong(MOD_TIME, TEST_MOD_TIME)
-                .withLong(BLOCK_SIZE, TEST_BLOCK_SIZE)
-                .withString(ETAG, TEST_ETAG)
-                .withString(VERSION_ID, TEST_VERSION_ID),
-            new DDBPathMetadata(
-                new S3AFileStatus(TEST_FILE_LENGTH, TEST_MOD_TIME,
-                    TEST_FILE_PATH, TEST_BLOCK_SIZE, username, TEST_ETAG,
-                    TEST_VERSION_ID))
-        },
-        // without etag or versionId
-        {
-            new Item()
-                .withPrimaryKey(PARENT,
-                    pathToParentKey(TEST_FILE_PATH.getParent()),
-                    CHILD, TEST_FILE_PATH.getName())
-                .withBoolean(IS_DIR, false)
-                .withLong(FILE_LENGTH, TEST_FILE_LENGTH)
-                .withLong(MOD_TIME, TEST_MOD_TIME)
-                .withLong(BLOCK_SIZE, TEST_BLOCK_SIZE),
-            new DDBPathMetadata(
-                new S3AFileStatus(TEST_FILE_LENGTH, TEST_MOD_TIME,
-                    TEST_FILE_PATH, TEST_BLOCK_SIZE, username, null, null))
-        }
-    });
-  }
-
-  public TestPathMetadataDynamoDBTranslation(Item item,
-      DDBPathMetadata metadata) throws IOException {
-    testFileItem = item;
-    testFilePathMetadata = metadata;
-
-    String username = UserGroupInformation.getCurrentUser().getShortUserName();
-
-    testDirPathMetadata = new DDBPathMetadata(new S3AFileStatus(false,
-        TEST_DIR_PATH, username));
-
-    testDirItem = new Item();
-    testDirItem
-        .withPrimaryKey(PARENT, "/test-bucket", CHILD, TEST_DIR_PATH.getName())
-        .withBoolean(IS_DIR, true);
-  }
-
-  /**
-   * It should not take long time as it doesn't involve remote server operation.
-   */
-  @Rule
-  public final Timeout timeout = new Timeout(30 * 1000);
-
-  @Test
-  public void testKeySchema() {
-    final Collection<KeySchemaElement> keySchema =
-        PathMetadataDynamoDBTranslation.keySchema();
-    assertNotNull(keySchema);
-    assertEquals("There should be HASH and RANGE key in key schema",
-        2, keySchema.size());
-    for (KeySchemaElement element : keySchema) {
-      assertThat(element.getAttributeName(), anyOf(is(PARENT), is(CHILD)));
-      assertThat(element.getKeyType(),
-          anyOf(is(HASH.toString()), is(RANGE.toString())));
-    }
-  }
-
-  @After
-  public void tearDown() {
-    PathMetadataDynamoDBTranslation.IGNORED_FIELDS.clear();
-  }
-
-  @Test
-  public void testAttributeDefinitions() {
-    final Collection<AttributeDefinition> attrs =
-        PathMetadataDynamoDBTranslation.attributeDefinitions();
-    assertNotNull(attrs);
-    assertEquals("There should be HASH and RANGE attributes", 2, attrs.size());
-    for (AttributeDefinition definition : attrs) {
-      assertThat(definition.getAttributeName(), anyOf(is(PARENT), is(CHILD)));
-      assertEquals(S.toString(), definition.getAttributeType());
-    }
-  }
-
-  @Test
-  public void testItemToPathMetadata() throws IOException {
-    final String user =
-        UserGroupInformation.getCurrentUser().getShortUserName();
-    assertNull(itemToPathMetadata(null, user));
-
-    verify(testDirItem, itemToPathMetadata(testDirItem, user));
-    verify(testFileItem, itemToPathMetadata(testFileItem, user));
-  }
-
-  /**
-   * Verify that the Item and PathMetadata objects hold the same information.
-   */
-  private static void verify(Item item, PathMetadata meta) {
-    assertNotNull(meta);
-    final S3AFileStatus status = meta.getFileStatus();
-    final Path path = status.getPath();
-    assertEquals(item.get(PARENT), pathToParentKey(path.getParent()));
-    assertEquals(item.get(CHILD), path.getName());
-    boolean isDir = item.hasAttribute(IS_DIR) && item.getBoolean(IS_DIR);
-    assertEquals(isDir, status.isDirectory());
-    long len = item.hasAttribute(FILE_LENGTH) ? item.getLong(FILE_LENGTH) : 0;
-    assertEquals(len, status.getLen());
-    long bSize = item.hasAttribute(BLOCK_SIZE) ? item.getLong(BLOCK_SIZE) : 0;
-    assertEquals(bSize, status.getBlockSize());
-    String eTag = item.getString(ETAG);
-    assertEquals(eTag, status.getETag());
-    String versionId = item.getString(VERSION_ID);
-    assertEquals(versionId, status.getVersionId());
-
-    /*
-     * S3AFileStatue#getModificationTime() reports the current time, so the
-     * following assertion is failing.
-     *
-     * long modTime = item.hasAttribute(MOD_TIME) ? item.getLong(MOD_TIME) : 0;
-     * assertEquals(modTime, status.getModificationTime());
-     */
-  }
-
-  @Test
-  public void testPathMetadataToItem() {
-    verify(pathMetadataToItem(testDirPathMetadata), testDirPathMetadata);
-    verify(pathMetadataToItem(testFilePathMetadata),
-        testFilePathMetadata);
-  }
-
-  @Test
-  public void testPathToParentKeyAttribute() {
-    doTestPathToParentKeyAttribute(TEST_DIR_PATH);
-    doTestPathToParentKeyAttribute(TEST_FILE_PATH);
-  }
-
-  private static void doTestPathToParentKeyAttribute(Path path) {
-    final KeyAttribute attr = pathToParentKeyAttribute(path);
-    assertNotNull(attr);
-    assertEquals(PARENT, attr.getName());
-    // this path is expected as parent filed
-    assertEquals(pathToParentKey(path), attr.getValue());
-  }
-
-  private static String pathToParentKey(Path p) {
-    Preconditions.checkArgument(p.isUriPathAbsolute());
-    URI parentUri = p.toUri();
-    String bucket = parentUri.getHost();
-    Preconditions.checkNotNull(bucket);
-    String s =  "/" + bucket + parentUri.getPath();
-    // strip trailing slash
-    if (s.endsWith("/")) {
-      s = s.substring(0, s.length()-1);
-    }
-    return s;
-  }
-
-  @Test
-  public void testPathToKey() throws Exception {
-    LambdaTestUtils.intercept(IllegalArgumentException.class,
-        () -> pathToKey(new Path("/")));
-    doTestPathToKey(TEST_DIR_PATH);
-    doTestPathToKey(TEST_FILE_PATH);
-  }
-
-  @Test
-  public void testPathToKeyTrailing() throws Exception {
-    doTestPathToKey(new Path("s3a://test-bucket/myDir/trailing//"));
-    doTestPathToKey(new Path("s3a://test-bucket/myDir/trailing/"));
-    LambdaTestUtils.intercept(IllegalArgumentException.class,
-        () -> pathToKey(new Path("s3a://test-bucket//")));
-  }
-
-  private static void doTestPathToKey(Path path) {
-    final PrimaryKey key = pathToKey(path);
-    assertNotNull(key);
-    assertEquals("There should be both HASH and RANGE keys",
-        2, key.getComponents().size());
-
-    for (KeyAttribute keyAttribute : key.getComponents()) {
-      assertThat(keyAttribute.getName(), anyOf(is(PARENT), is(CHILD)));
-      if (PARENT.equals(keyAttribute.getName())) {
-        assertEquals(pathToParentKey(path.getParent()),
-            keyAttribute.getValue());
-      } else {
-        assertEquals(path.getName(), keyAttribute.getValue());
-      }
-    }
-  }
-
-  @Test
-  public void testVersionRoundTrip() throws Throwable {
-    final Item marker = createVersionMarker(VERSION_MARKER_ITEM_NAME, VERSION, 0);
-    assertEquals("Extracted version from " + marker,
-        VERSION, extractVersionFromMarker(marker));
-  }
-
-  @Test
-  public void testVersionMarkerNotStatusIllegalPath() throws Throwable {
-    final Item marker = createVersionMarker(VERSION_MARKER_ITEM_NAME, VERSION, 0);
-    assertNull("Path metadata fromfrom " + marker,
-        itemToPathMetadata(marker, "alice"));
-  }
-
-  /**
-   * Test when translating an {@link Item} to {@link DDBPathMetadata} works
-   * if {@code IS_AUTHORITATIVE} flag is ignored.
-   */
-  @Test
-  public void testIsAuthoritativeCompatibilityItemToPathMetadata()
-      throws Exception {
-    Item item = Mockito.spy(testDirItem);
-    item.withBoolean(IS_AUTHORITATIVE, true);
-    PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(IS_AUTHORITATIVE);
-
-    final String user =
-        UserGroupInformation.getCurrentUser().getShortUserName();
-    DDBPathMetadata meta = itemToPathMetadata(item, user);
-
-    Mockito.verify(item, Mockito.never()).getBoolean(IS_AUTHORITATIVE);
-    assertFalse(meta.isAuthoritativeDir());
-  }
-
-  /**
-   * Test when translating an {@link DDBPathMetadata} to {@link Item} works
-   * if {@code IS_AUTHORITATIVE} flag is ignored.
-   */
-  @Test
-  public void testIsAuthoritativeCompatibilityPathMetadataToItem() {
-    DDBPathMetadata meta = Mockito.spy(testFilePathMetadata);
-    meta.setAuthoritativeDir(true);
-    PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(IS_AUTHORITATIVE);
-
-    Item item = pathMetadataToItem(meta);
-
-    Mockito.verify(meta, never()).isAuthoritativeDir();
-    assertFalse(item.hasAttribute(IS_AUTHORITATIVE));
-  }
-
-
-  /**
-   * Test when translating an {@link Item} to {@link DDBPathMetadata} works
-   * if {@code LAST_UPDATED} flag is ignored.
-   */
-  @Test
-  public void testIsLastUpdatedCompatibilityItemToPathMetadata()
-      throws Exception {
-    Item item = Mockito.spy(testDirItem);
-    item.withLong(LAST_UPDATED, 100);
-    PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(LAST_UPDATED);
-
-    final String user =
-        UserGroupInformation.getCurrentUser().getShortUserName();
-    DDBPathMetadata meta = itemToPathMetadata(item, user);
-
-    Mockito.verify(item, Mockito.never()).getLong(LAST_UPDATED);
-    assertFalse(meta.isAuthoritativeDir());
-  }
-
-  /**
-   * Test when translating an {@link DDBPathMetadata} to {@link Item} works
-   * if {@code LAST_UPDATED} flag is ignored.
-   */
-  @Test
-  public void testIsLastUpdatedCompatibilityPathMetadataToItem() {
-    DDBPathMetadata meta = Mockito.spy(testFilePathMetadata);
-    meta.setLastUpdated(100);
-    PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(LAST_UPDATED);
-
-    Item item = pathMetadataToItem(meta);
-
-    Mockito.verify(meta, never()).getLastUpdated();
-    assertFalse(item.hasAttribute(LAST_UPDATED));
-  }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathOrderComparators.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathOrderComparators.java
deleted file mode 100644
index 03233df69e198..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathOrderComparators.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.hadoop.util.Lists;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.Path;
-
-import static org.apache.hadoop.fs.s3a.s3guard.PathOrderComparators.TOPMOST_PATH_FIRST;
-import static org.apache.hadoop.fs.s3a.s3guard.PathOrderComparators.TOPMOST_PATH_LAST;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test ordering of paths with the comparator matches requirements.
- */
-public class TestPathOrderComparators {
-
-  private static final Path ROOT = new Path("s3a://bucket/");
-
-  public static final Path DIR_A = new Path(ROOT, "dirA");
-
-  public static final Path DIR_B = new Path(ROOT, "dirB");
-
-  public static final Path DIR_A_FILE_1 = new Path(DIR_A, "file1");
-
-  public static final Path DIR_A_FILE_2 = new Path(DIR_A, "file2");
-
-  public static final Path DIR_B_FILE_3 = new Path(DIR_B, "file3");
-
-  public static final Path DIR_B_FILE_4 = new Path(DIR_B, "file4");
-
-  @Test
-  public void testRootEqual() throws Throwable {
-    assertComparesEqual(ROOT, ROOT);
-  }
-
-  @Test
-  public void testRootFirst() throws Throwable {
-    assertComparesTopmost(ROOT, DIR_A_FILE_1);
-  }
-
-  @Test
-  public void testDirOrdering() throws Throwable {
-    assertComparesTopmost(DIR_A, DIR_B);
-  }
-
-  @Test
-  public void testFilesEqual() throws Throwable {
-    assertComparesEqual(DIR_A_FILE_1, DIR_A_FILE_1);
-  }
-
-  @Test
-  public void testFilesInSameDir() throws Throwable {
-    assertComparesTopmost(ROOT, DIR_A_FILE_1);
-    assertComparesTopmost(DIR_A, DIR_A_FILE_1);
-    assertComparesTopmost(DIR_A, DIR_A_FILE_2);
-    assertComparesTopmost(DIR_A_FILE_1, DIR_A_FILE_2);
-  }
-
-  @Test
-  public void testReversedFiles() throws Throwable {
-    assertReverseOrder(DIR_A_FILE_1, ROOT);
-    assertReverseOrder(DIR_A_FILE_1, DIR_A);
-    assertReverseOrder(DIR_A_FILE_2, DIR_A);
-    assertReverseOrder(DIR_A_FILE_2, DIR_A_FILE_1);
-  }
-
-  @Test
-  public void testFilesAndDifferentShallowDir() throws Throwable {
-    assertComparesTopmost(DIR_B, DIR_A_FILE_1);
-    assertComparesTopmost(DIR_A, DIR_B_FILE_3);
-  }
-
-  @Test
-  public void testOrderRoot() throws Throwable {
-    verifySorted(ROOT);
-  }
-
-  @Test
-  public void testOrderRootDirs() throws Throwable {
-    verifySorted(ROOT, DIR_A, DIR_B);
-  }
-
-  @Test
-  public void testOrderRootDirsAndFiles() throws Throwable {
-    verifySorted(ROOT, DIR_A, DIR_B, DIR_A_FILE_1, DIR_A_FILE_2);
-  }
-
-  @Test
-  public void testOrderRootDirsAndAllFiles() throws Throwable {
-    verifySorted(ROOT, DIR_A, DIR_B,
-        DIR_A_FILE_1, DIR_A_FILE_2,
-        DIR_B_FILE_3, DIR_B_FILE_4);
-  }
-
-  @Test
-  public void testSortOrderConstant() throws Throwable {
-    List<Path> sort1 = verifySorted(ROOT, DIR_A, DIR_B,
-        DIR_A_FILE_1, DIR_A_FILE_2,
-        DIR_B_FILE_3, DIR_B_FILE_4);
-    List<Path> sort2 = Lists.newArrayList(sort1);
-    assertSortsTo(sort2, sort1, true);
-  }
-
-  @Test
-  public void testSortReverse() throws Throwable {
-    List<Path> sort1 = Lists.newArrayList(
-        ROOT,
-        DIR_A,
-        DIR_B,
-        DIR_A_FILE_1,
-        DIR_A_FILE_2,
-        DIR_B_FILE_3,
-        DIR_B_FILE_4);
-    List<Path> expected = Lists.newArrayList(
-        DIR_B_FILE_4,
-        DIR_B_FILE_3,
-        DIR_A_FILE_2,
-        DIR_A_FILE_1,
-        DIR_B,
-        DIR_A,
-        ROOT);
-    assertSortsTo(expected, sort1, false);
-  }
-
-
-  private List<Path> verifySorted(Path... paths) {
-    List<Path> original = Lists.newArrayList(paths);
-    List<Path> sorted = Lists.newArrayList(paths);
-    assertSortsTo(original, sorted, true);
-    return sorted;
-  }
-
-  private void assertSortsTo(
-      final List<Path> original,
-      final List<Path> sorted,
-      boolean topmost) {
-    sorted.sort(topmost ? TOPMOST_PATH_FIRST : TOPMOST_PATH_LAST);
-    assertThat(sorted)
-        .as("Sorted paths")
-        .containsExactlyElementsOf(original);
-  }
-
-  private void assertComparesEqual(Path l, Path r) {
-    assertOrder(0, l, r);
-  }
-
-  private void assertComparesTopmost(Path l, Path r) {
-    assertOrder(-1, l, r);
-    assertOrder(1, r, l);
-  }
-
-  private void assertReverseOrder(Path l, Path r) {
-    assertComparesTo(-1, TOPMOST_PATH_LAST, l, r);
-    assertComparesTo(1, TOPMOST_PATH_LAST, r, l);
-  }
-
-  private void assertOrder(int res,
-      Path l, Path r) {
-    assertComparesTo(res, TOPMOST_PATH_FIRST, l, r);
-  }
-
-  private void assertComparesTo(final int expected,
-      final Comparator<Path> comparator,
-      final Path l, final Path r) {
-    int actual = comparator.compare(l, r);
-    if (actual < -1) {
-      actual = -1;
-    }
-    if (actual > 1) {
-      actual = 1;
-    }
-    assertEquals("Comparing " + l + " to " + r,
-        expected, actual);
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java
deleted file mode 100644
index eaa363bbf19b9..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java
+++ /dev/null
@@ -1,526 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.service.launcher.LauncherExitCodes;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.apache.hadoop.util.ExitUtil;
-
-import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_METADATASTORE_METADATA_TTL;
-import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_METADATA_TTL;
-import static org.apache.hadoop.fs.s3a.Listing.toProvidedFileStatusIterator;
-import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.dirMetaToStatuses;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * Tests for the {@link S3Guard} utility class.
- */
-public class TestS3Guard extends Assert {
-
-  public static final String MS_FILE_1 = "s3a://bucket/dir/ms-file1";
-
-  public static final String MS_FILE_2 = "s3a://bucket/dir/ms-file2";
-
-  public static final String S3_FILE_3 = "s3a://bucket/dir/s3-file3";
-
-  public static final String S3_DIR_4 = "s3a://bucket/dir/s3-dir4";
-
-  public static final Path DIR_PATH = new Path("s3a://bucket/dir");
-
-  private MetadataStore ms;
-
-  private ITtlTimeProvider timeProvider;
-
-  @Before
-  public void setUp() throws Exception {
-    final Configuration conf = new Configuration(false);
-    ms = new LocalMetadataStore();
-    ms.initialize(conf, new S3Guard.TtlTimeProvider(conf));
-    timeProvider = new S3Guard.TtlTimeProvider(
-        DEFAULT_METADATASTORE_METADATA_TTL);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    if (ms != null) {
-       ms.destroy();
-    }
-  }
-
-  /**
-   * Basic test to ensure results from S3 and MetadataStore are merged
-   * correctly.
-   */
-  @Test
-  public void testDirListingUnionNonauth() throws Exception {
-
-    // Two files in metadata store listing
-    PathMetadata m1 = makePathMeta(MS_FILE_1, false);
-    PathMetadata m2 = makePathMeta(MS_FILE_2, false);
-    DirListingMetadata dirMeta = new DirListingMetadata(DIR_PATH,
-        Arrays.asList(m1, m2), false);
-
-    // Two other entries in s3
-    final S3AFileStatus s1Status = makeFileStatus(S3_FILE_3, false);
-    final S3AFileStatus s2Status = makeFileStatus(S3_DIR_4, true);
-    List<S3AFileStatus> s3Listing = Arrays.asList(
-        s1Status,
-        s2Status);
-    RemoteIterator<S3AFileStatus> storeItr = toProvidedFileStatusIterator(
-            s3Listing.toArray(new S3AFileStatus[0]));
-    RemoteIterator<S3AFileStatus> resultItr = S3Guard.dirListingUnion(
-            ms, DIR_PATH, storeItr, dirMeta, false,
-            timeProvider, s3AFileStatuses ->
-                    toProvidedFileStatusIterator(dirMetaToStatuses(dirMeta)));
-    S3AFileStatus[] result = S3AUtils.iteratorToStatuses(
-            resultItr, new HashSet<>());
-
-    assertEquals("listing length", 4, result.length);
-    assertContainsPaths(result, MS_FILE_1, MS_FILE_2, S3_FILE_3, S3_DIR_4);
-
-    // check the MS doesn't contain the s3 entries as nonauth
-    // unions should block them
-    assertNoRecord(ms, S3_FILE_3);
-    assertNoRecord(ms, S3_DIR_4);
-
-    // for entries which do exist, when updated in S3, the metastore is updated
-    S3AFileStatus f1Status2 = new S3AFileStatus(
-        200, System.currentTimeMillis(), new Path(MS_FILE_1),
-        1, null, "tag2", "ver2");
-    S3AFileStatus[] f1Statuses = new S3AFileStatus[1];
-    f1Statuses[0] = f1Status2;
-    RemoteIterator<S3AFileStatus> itr = toProvidedFileStatusIterator(
-            f1Statuses);
-    FileStatus[] result2 = S3AUtils.iteratorToStatuses(
-            S3Guard.dirListingUnion(
-                    ms, DIR_PATH, itr, dirMeta,
-                    false, timeProvider,
-              s3AFileStatuses ->
-                      toProvidedFileStatusIterator(
-                              dirMetaToStatuses(dirMeta))),
-            new HashSet<>());
-    // the listing returns the new status
-    Assertions.assertThat(find(result2, MS_FILE_1))
-        .describedAs("Entry in listing results for %s", MS_FILE_1)
-        .isSameAs(f1Status2);
-    // as does a query of the MS
-    final PathMetadata updatedMD = verifyRecord(ms, MS_FILE_1);
-    Assertions.assertThat(updatedMD.getFileStatus())
-        .describedAs("Entry in metastore for %s: %s", MS_FILE_1, updatedMD)
-        .isEqualTo(f1Status2);
-  }
-
-  /**
-   * Auth mode unions are different.
-   */
-  @Test
-  public void testDirListingUnionAuth() throws Exception {
-
-    // Two files in metadata store listing
-    PathMetadata m1 = makePathMeta(MS_FILE_1, false);
-    PathMetadata m2 = makePathMeta(MS_FILE_2, false);
-    DirListingMetadata dirMeta = new DirListingMetadata(DIR_PATH,
-        Arrays.asList(m1, m2), true);
-
-    // Two other entries in s3
-    S3AFileStatus s1Status = makeFileStatus(S3_FILE_3, false);
-    S3AFileStatus s2Status = makeFileStatus(S3_DIR_4, true);
-    List<S3AFileStatus> s3Listing = Arrays.asList(
-        s1Status,
-        s2Status);
-
-    ITtlTimeProvider timeProvider = new S3Guard.TtlTimeProvider(
-        DEFAULT_METADATASTORE_METADATA_TTL);
-
-    RemoteIterator<S3AFileStatus> storeItr = toProvidedFileStatusIterator(
-            s3Listing.toArray(new S3AFileStatus[0]));
-    RemoteIterator<S3AFileStatus> resultItr = S3Guard
-            .dirListingUnion(ms, DIR_PATH, storeItr, dirMeta,
-                    true, timeProvider,
-              s3AFileStatuses ->
-                      toProvidedFileStatusIterator(
-                              dirMetaToStatuses(dirMeta)));
-
-    S3AFileStatus[] result = S3AUtils.iteratorToStatuses(
-            resultItr, new HashSet<>());
-    assertEquals("listing length", 4, result.length);
-    assertContainsPaths(result, MS_FILE_1, MS_FILE_2, S3_FILE_3, S3_DIR_4);
-
-    // now verify an auth scan added the records
-    PathMetadata file3Meta = verifyRecord(ms, S3_FILE_3);
-    PathMetadata dir4Meta = verifyRecord(ms, S3_DIR_4);
-
-    // we can't check auth flag handling because local FS doesn't have one
-    // so do just check the dir status still all good.
-    Assertions.assertThat(dir4Meta)
-        .describedAs("Metastore entry for dir %s", dir4Meta)
-        .matches(m -> m.getFileStatus().isDirectory());
-
-    DirListingMetadata dirMeta2 = new DirListingMetadata(DIR_PATH,
-        Arrays.asList(m1, m2, file3Meta, dir4Meta), true);
-    // now s1 status is updated on S3
-    S3AFileStatus s1Status2 = new S3AFileStatus(
-        200, System.currentTimeMillis(), new Path(S3_FILE_3),
-        1, null, "tag2", "ver2");
-    S3AFileStatus[] f1Statuses = new S3AFileStatus[1];
-    f1Statuses[0] = s1Status2;
-    RemoteIterator<S3AFileStatus> itr =
-            toProvidedFileStatusIterator(f1Statuses);
-    FileStatus[] result2 = S3AUtils.iteratorToStatuses(
-            S3Guard.dirListingUnion(ms, DIR_PATH, itr, dirMeta,
-                    true, timeProvider,
-              s3AFileStatuses ->
-                      toProvidedFileStatusIterator(
-                              dirMetaToStatuses(dirMeta))),
-            new HashSet<>());
-
-    // but the result of the listing contains the old entry
-    // because auth mode doesn't pick up changes in S3 which
-    // didn't go through s3guard
-    Assertions.assertThat(find(result2, S3_FILE_3))
-        .describedAs("Entry in listing results for %s", S3_FILE_3)
-        .isSameAs(file3Meta.getFileStatus());
-  }
-
-  /**
-   * Assert there is no record in the store.
-   * @param ms metastore
-   * @param path path
-   * @throws IOException IOError
-   */
-  private void assertNoRecord(MetadataStore ms, String path)
-      throws IOException {
-    Assertions.assertThat(lookup(ms, path))
-        .describedAs("Metastore entry for %s", path)
-        .isNull();
-  }
-
-  /**
-   * Assert there is arecord in the store, then return it.
-   * @param ms metastore
-   * @param path path
-   * @return the record.
-   * @throws IOException IO Error
-   */
-  private PathMetadata verifyRecord(MetadataStore ms, String path)
-     throws IOException {
-   final PathMetadata md = lookup(ms, path);
-   Assertions.assertThat(md)
-       .describedAs("Metastore entry for %s", path)
-       .isNotNull();
-   return md;
-  }
-
-  /**
-   * Look up a record.
-   * @param ms store
-   * @param path path
-   * @return the record or null
-   * @throws IOException IO Error
-   */
-  private PathMetadata lookup(final MetadataStore ms, final String path)
-      throws IOException {
-    return ms.get(new Path(path));
-  }
-
-  @Test
-  public void testPutWithTtlDirListingMeta() throws Exception {
-    // arrange
-    DirListingMetadata dlm = new DirListingMetadata(new Path("/"), null,
-        false);
-    MetadataStore ms = spy(MetadataStore.class);
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(100L);
-
-    // act
-    S3Guard.putWithTtl(ms, dlm, Collections.emptyList(), timeProvider, null);
-
-    // assert
-    assertEquals("last update in " + dlm, 100L, dlm.getLastUpdated());
-    verify(timeProvider, times(1)).getNow();
-    verify(ms, times(1)).put(dlm, Collections.emptyList(), null);
-  }
-
-  @Test
-  public void testPutWithTtlFileMeta() throws Exception {
-    // arrange
-    S3AFileStatus fileStatus = mock(S3AFileStatus.class);
-    when(fileStatus.getPath()).thenReturn(new Path("/"));
-    PathMetadata pm = new PathMetadata(fileStatus);
-    MetadataStore ms = spy(MetadataStore.class);
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(100L);
-
-    // act
-    S3Guard.putWithTtl(ms, pm, timeProvider, null);
-
-    // assert
-    assertEquals("last update in " + pm, 100L, pm.getLastUpdated());
-    verify(timeProvider, times(1)).getNow();
-    verify(ms, times(1)).put(pm, null);
-  }
-
-  @Test
-  public void testPutWithTtlCollection() throws Exception {
-    // arrange
-    S3AFileStatus fileStatus = mock(S3AFileStatus.class);
-    when(fileStatus.getPath()).thenReturn(new Path("/"));
-    Collection<PathMetadata> pmCollection = new ArrayList<>();
-    for (int i = 0; i < 10; i++) {
-      pmCollection.add(new PathMetadata(fileStatus));
-    }
-    MetadataStore ms = spy(MetadataStore.class);
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(100L);
-
-    // act
-    S3Guard.putWithTtl(ms, pmCollection, timeProvider, null);
-
-    // assert
-    pmCollection.forEach(
-        pm -> assertEquals(100L, pm.getLastUpdated())
-    );
-    verify(timeProvider, times(1)).getNow();
-    verify(ms, times(1)).put(pmCollection, null);
-  }
-
-  @Test
-  public void testGetWithTtlExpired() throws Exception {
-    // arrange
-    S3AFileStatus fileStatus = mock(S3AFileStatus.class);
-    Path path = new Path("/file");
-    when(fileStatus.getPath()).thenReturn(path);
-    PathMetadata pm = new PathMetadata(fileStatus);
-    pm.setLastUpdated(100L);
-
-    MetadataStore ms = mock(MetadataStore.class);
-    when(ms.get(path, false)).thenReturn(pm);
-
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(101L);
-    when(timeProvider.getMetadataTtl()).thenReturn(1L);
-
-    // act
-    final PathMetadata pmExpired = S3Guard.getWithTtl(ms, path, timeProvider,
-        false, false);
-
-    // assert
-    assertNull(pmExpired);
-  }
-
-  @Test
-  public void testGetWithTtlNotExpired() throws Exception {
-    // arrange
-    S3AFileStatus fileStatus = mock(S3AFileStatus.class);
-    Path path = new Path("/file");
-    when(fileStatus.getPath()).thenReturn(path);
-    PathMetadata pm = new PathMetadata(fileStatus);
-    pm.setLastUpdated(100L);
-
-    MetadataStore ms = mock(MetadataStore.class);
-    when(ms.get(path, false)).thenReturn(pm);
-
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(101L);
-    when(timeProvider.getMetadataTtl()).thenReturn(2L);
-
-    // act
-    final PathMetadata pmNotExpired =
-        S3Guard.getWithTtl(ms, path, timeProvider, false, false);
-
-    // assert
-    assertNotNull(pmNotExpired);
-  }
-
-  @Test
-  public void testGetWithZeroLastUpdatedNotExpired() throws Exception {
-    // arrange
-    S3AFileStatus fileStatus = mock(S3AFileStatus.class);
-    Path path = new Path("/file");
-    when(fileStatus.getPath()).thenReturn(path);
-    PathMetadata pm = new PathMetadata(fileStatus);
-    // we set 0 this time as the last updated: can happen eg. when we use an
-    // old dynamo table
-    pm.setLastUpdated(0L);
-
-    MetadataStore ms = mock(MetadataStore.class);
-    when(ms.get(path, false)).thenReturn(pm);
-
-    ITtlTimeProvider timeProvider =
-        mock(ITtlTimeProvider.class);
-    when(timeProvider.getNow()).thenReturn(101L);
-    when(timeProvider.getMetadataTtl()).thenReturn(2L);
-
-    // act
-    final PathMetadata pmExpired = S3Guard.getWithTtl(ms, path, timeProvider,
-        false, false);
-
-    // assert
-    assertNotNull(pmExpired);
-  }
-
-
-  /**
-   * Makes sure that all uses of TTL timeouts use a consistent time unit.
-   * @throws Throwable failure
-   */
-  @Test
-  public void testTTLConstruction() throws Throwable {
-    // first one
-    ITtlTimeProvider timeProviderExplicit = new S3Guard.TtlTimeProvider(
-        DEFAULT_METADATASTORE_METADATA_TTL);
-
-    // mirror the FS construction,
-    // from a config guaranteed to be empty (i.e. the code defval)
-    Configuration conf = new Configuration(false);
-    long millitime = conf.getTimeDuration(METADATASTORE_METADATA_TTL,
-        DEFAULT_METADATASTORE_METADATA_TTL, TimeUnit.MILLISECONDS);
-    assertEquals(15 * 60_000, millitime);
-    S3Guard.TtlTimeProvider fsConstruction = new S3Guard.TtlTimeProvider(
-        millitime);
-    assertEquals("explicit vs fs construction", timeProviderExplicit,
-        fsConstruction);
-    assertEquals("first and second constructor", timeProviderExplicit,
-        new S3Guard.TtlTimeProvider(conf));
-    // set the conf to a time without unit
-    conf.setLong(METADATASTORE_METADATA_TTL,
-        DEFAULT_METADATASTORE_METADATA_TTL);
-    assertEquals("first and second time set through long", timeProviderExplicit,
-        new S3Guard.TtlTimeProvider(conf));
-    double timeInSeconds = DEFAULT_METADATASTORE_METADATA_TTL / 1000;
-    double timeInMinutes = timeInSeconds / 60;
-    String timeStr = String.format("%dm", (int) timeInMinutes);
-    assertEquals(":wrong time in minutes from " + timeInMinutes,
-        "15m", timeStr);
-    conf.set(METADATASTORE_METADATA_TTL, timeStr);
-    assertEquals("Time in millis as string from "
-            + conf.get(METADATASTORE_METADATA_TTL),
-        timeProviderExplicit,
-        new S3Guard.TtlTimeProvider(conf));
-  }
-
-  @Test
-  public void testLogS3GuardDisabled() throws Exception {
-    final Logger localLogger = LoggerFactory.getLogger(
-        TestS3Guard.class.toString() + UUID.randomUUID());
-    S3Guard.logS3GuardDisabled(localLogger,
-        S3Guard.DisabledWarnLevel.SILENT.toString(), "bucket");
-    S3Guard.logS3GuardDisabled(localLogger,
-        S3Guard.DisabledWarnLevel.INFORM.toString(), "bucket");
-    S3Guard.logS3GuardDisabled(localLogger,
-        S3Guard.DisabledWarnLevel.WARN.toString(), "bucket");
-
-    // Test that lowercase setting is accepted
-    S3Guard.logS3GuardDisabled(localLogger,
-        S3Guard.DisabledWarnLevel.WARN.toString()
-            .toLowerCase(Locale.US), "bucket");
-
-    ExitUtil.ExitException ex = LambdaTestUtils.intercept(
-        ExitUtil.ExitException.class,
-        String.format(S3Guard.DISABLED_LOG_MSG, "bucket"),
-        () -> S3Guard.logS3GuardDisabled(
-            localLogger, S3Guard.DisabledWarnLevel.FAIL.toString(), "bucket"));
-    if (ex.getExitCode() != LauncherExitCodes.EXIT_BAD_CONFIGURATION) {
-      throw ex;
-    }
-    LambdaTestUtils.intercept(IllegalArgumentException.class,
-        S3Guard.UNKNOWN_WARN_LEVEL,
-        () -> S3Guard.logS3GuardDisabled(
-            localLogger, "FOO_BAR_LEVEL", "bucket"));
-  }
-
-  void assertContainsPaths(FileStatus[] statuses, String...pathStr) {
-    for (String s :pathStr) {
-      assertContainsPath(statuses, s);
-    }
-  }
-
-  void assertContainsPath(FileStatus[] statuses, String pathStr) {
-    find(statuses, pathStr);
-  }
-
-  /**
-   * Look up an entry or raise an assertion
-   * @param statuses list of statuses
-   * @param pathStr path to search
-   * @return the entry if found
-   */
-  private FileStatus find(FileStatus[] statuses, String pathStr) {
-    for (FileStatus s : statuses) {
-      if (s.getPath().toString().equals(pathStr)) {
-        return s;
-      }
-    }
-    // no match, fail meaningfully
-    Assertions.assertThat(statuses)
-        .anyMatch(s -> s.getPath().toString().equals(pathStr));
-    return null;
-  }
-
-  private PathMetadata makePathMeta(String pathStr, boolean isDir) {
-    return new PathMetadata(makeFileStatus(pathStr, isDir));
-  }
-
-  private S3AFileStatus makeFileStatus(String pathStr, boolean isDir) {
-    Path p = new Path(pathStr);
-    S3AFileStatus fileStatus;
-    if (isDir) {
-      fileStatus = new S3AFileStatus(Tristate.UNKNOWN, p, null);
-    } else {
-      fileStatus = new S3AFileStatus(
-          100, System.currentTimeMillis(), p, 1, null, null, null);
-    }
-    return fileStatus;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardCLI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardCLI.java
index 43256b9f5fb1b..26ccf1284efbf 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardCLI.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardCLI.java
@@ -22,7 +22,6 @@
 import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3ATestConstants;
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.ExitUtil;
 
@@ -83,39 +82,4 @@ public void testUnknownCommand() throws Throwable {
     runToFailure(E_USAGE, "unknown");
   }
 
-  @Test
-  public void testPruneNoArgs() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Prune.NAME);
-  }
-
-  @Test
-  public void testDiffNoArgs() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Diff.NAME);
-  }
-
-  @Test
-  public void testImportNoArgs() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Import.NAME);
-  }
-
-  @Test
-  public void testDestroyNoArgs() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Destroy.NAME);
-  }
-
-  @Test
-  public void testDestroyUnknownTableNoRegion() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Destroy.NAME,
-        "-meta", "dynamodb://ireland-team");
-  }
-
-  @Test
-  public void testInitBucketAndRegion() throws Throwable {
-    runToFailure(INVALID_ARGUMENT, Init.NAME,
-        "-meta", "dynamodb://ireland-team",
-        "-region", "eu-west-1",
-        S3ATestConstants.DEFAULT_CSVTEST_FILE
-    );
-  }
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ThrottleTracker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ThrottleTracker.java
deleted file mode 100644
index 0dad1bf03d7bd..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ThrottleTracker.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Something to track throttles in DynamoDB metastores.
- * The constructor sets the counters to the current count in the
- * DDB table; a call to {@link #reset()} will set it to the latest values.
- * The {@link #probe()} will pick up the latest values to compare them with
- * the original counts.
- * <p>
- * The toString value logs the state.
- * <p>
- * This class was originally part of ITestDynamoDBMetadataStoreScale;
- * it was converted to a toplevel class for broader use.
- */
-class ThrottleTracker {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      ThrottleTracker.class);
-  private final DynamoDBMetadataStore ddbms;
-
-  private long writeThrottleEventOrig;
-
-  private long readThrottleEventOrig;
-
-  private long batchWriteThrottleCountOrig;
-
-  private long scanThrottleCountOrig;
-
-  private long readThrottles;
-
-  private long writeThrottles;
-
-  private long batchThrottles;
-
-  private long scanThrottles;
-
-  ThrottleTracker(final DynamoDBMetadataStore ddbms) {
-    this.ddbms = ddbms;
-    reset();
-  }
-
-  /**
-   * Reset the counters.
-   */
-  public synchronized void reset() {
-    writeThrottleEventOrig
-        = ddbms.getWriteThrottleEventCount();
-
-    readThrottleEventOrig
-        = ddbms.getReadThrottleEventCount();
-
-    batchWriteThrottleCountOrig
-        = ddbms.getBatchWriteCapacityExceededCount();
-
-    scanThrottleCountOrig
-        = ddbms.getScanThrottleEventCount();
-  }
-
-  /**
-   * Update the latest throttle count; synchronized.
-   * @return true if throttling has been detected.
-   */
-  public synchronized boolean probe() {
-    setReadThrottles(
-        ddbms.getReadThrottleEventCount() - readThrottleEventOrig);
-    setWriteThrottles(ddbms.getWriteThrottleEventCount()
-        - writeThrottleEventOrig);
-    setBatchThrottles(ddbms.getBatchWriteCapacityExceededCount()
-        - batchWriteThrottleCountOrig);
-    setScanThrottles(ddbms.getScanThrottleEventCount()
-        - scanThrottleCountOrig);
-    return isThrottlingDetected();
-  }
-
-  @Override
-  public String toString() {
-    return String.format(
-        "Tracker with read throttle events = %d;"
-            + " write throttles = %d;"
-            + " batch throttles = %d;"
-            + " scan throttles = %d",
-        getReadThrottles(), getWriteThrottles(), getBatchThrottles(),
-        getScanThrottles());
-  }
-
-  /**
-   * Check that throttling was detected; Warn if not.
-   * @return true if throttling took place.
-   */
-  public boolean probeThrottlingDetected() {
-    if (!isThrottlingDetected()) {
-      LOG.warn("No throttling detected in {} against {}",
-          this, ddbms);
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Has there been any throttling on an operation?
-   * @return true if any operations were throttled.
-   */
-  public boolean isThrottlingDetected() {
-    return getReadThrottles() > 0
-        || getWriteThrottles() > 0
-        || getBatchThrottles() > 0
-        || getScanThrottles() > 0;
-  }
-
-  public long getReadThrottles() {
-    return readThrottles;
-  }
-
-  public void setReadThrottles(long readThrottles) {
-    this.readThrottles = readThrottles;
-  }
-
-  public long getWriteThrottles() {
-    return writeThrottles;
-  }
-
-  public void setWriteThrottles(long writeThrottles) {
-    this.writeThrottles = writeThrottles;
-  }
-
-  public long getBatchThrottles() {
-    return batchThrottles;
-  }
-
-  public void setBatchThrottles(long batchThrottles) {
-    this.batchThrottles = batchThrottles;
-  }
-
-  public long getScanThrottles() {
-    return scanThrottles;
-  }
-
-  public void setScanThrottles(final long scanThrottles) {
-    this.scanThrottles = scanThrottles;
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
deleted file mode 100644
index 8bbec1fdec628..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
-
-import org.junit.Before;
-import org.junit.FixMethodOrder;
-import org.junit.Test;
-import org.junit.runners.MethodSorters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.NanoTimer;
-
-/**
- * Test the performance of a MetadataStore.  Useful for load testing.
- * Could be separated from S3A code, but we're using the S3A scale test
- * framework for convenience.
- */
-@FixMethodOrder(MethodSorters.NAME_ASCENDING)
-public abstract class AbstractITestS3AMetadataStoreScale extends
-    S3AScaleTestBase {
-  private static final Logger LOG = LoggerFactory.getLogger(
-      AbstractITestS3AMetadataStoreScale.class);
-
-  /** Some dummy values for FileStatus contents. */
-  static final long BLOCK_SIZE = 32 * 1024 * 1024;
-  static final long SIZE = BLOCK_SIZE * 2;
-  static final String OWNER = "bob";
-  static final long ACCESS_TIME = System.currentTimeMillis();
-
-  static final Path BUCKET_ROOT = new Path("s3a://fake-bucket/");
-  private ITtlTimeProvider ttlTimeProvider;
-
-  @Before
-  public void initialize() {
-    ttlTimeProvider = new S3Guard.TtlTimeProvider(new Configuration());
-  }
-
-  /**
-   * Subclasses should override this to provide the MetadataStore they which
-   * to test.
-   * @return MetadataStore to test against
-   * @throws IOException
-   */
-  public abstract MetadataStore createMetadataStore() throws IOException;
-
-  protected ITtlTimeProvider getTtlTimeProvider() {
-    return ttlTimeProvider;
-  }
-
-  @Test
-  public void test_010_Put() throws Throwable {
-    describe("Test workload of put() operations");
-
-    // As described in hadoop-aws site docs, count parameter is used for
-    // width and depth of directory tree
-    int width = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
-    int depth = width;
-
-    List<PathMetadata> paths = new ArrayList<>();
-    createDirTree(BUCKET_ROOT, depth, width, paths);
-
-    long count = 1; // Some value in case we throw an exception below
-    try (MetadataStore ms = createMetadataStore()) {
-
-      try {
-        count = populateMetadataStore(paths, ms);
-      } finally {
-        clearMetadataStore(ms, count);
-      }
-    }
-  }
-
-  @Test
-  public void test_020_Moves() throws Throwable {
-    describe("Test workload of batched move() operations");
-
-    // As described in hadoop-aws site docs, count parameter is used for
-    // width and depth of directory tree
-    int width = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
-    int depth = width;
-
-    long operations = getConf().getLong(KEY_OPERATION_COUNT,
-        DEFAULT_OPERATION_COUNT);
-
-    List<PathMetadata> origMetas = new ArrayList<>();
-    createDirTree(BUCKET_ROOT, depth, width, origMetas);
-
-    // Pre-compute source and destination paths for move() loop below
-    List<Path> origPaths = metasToPaths(origMetas);
-    List<PathMetadata> movedMetas = moveMetas(origMetas, BUCKET_ROOT,
-        new Path(BUCKET_ROOT, "moved-here"));
-    List<Path> movedPaths = metasToPaths(movedMetas);
-
-    long count = 1; // Some value in case we throw an exception below
-    try (MetadataStore ms = createMetadataStore()) {
-
-      try {
-        // Setup
-        count = populateMetadataStore(origMetas, ms);
-
-        // Main loop: move things back and forth
-        describe("Running move workload");
-        NanoTimer moveTimer = new NanoTimer();
-        LOG.info("Running {} moves of {} paths each", operations,
-            origMetas.size());
-        for (int i = 0; i < operations; i++) {
-          Collection<Path> toDelete;
-          Collection<PathMetadata> toCreate;
-          if (i % 2 == 0) {
-            toDelete = origPaths;
-            toCreate = movedMetas;
-          } else {
-            toDelete = movedPaths;
-            toCreate = origMetas;
-          }
-          ms.move(toDelete, toCreate, null);
-        }
-        moveTimer.end();
-        printTiming(LOG, "move", moveTimer, operations);
-      } finally {
-        // Cleanup
-        clearMetadataStore(ms, count);
-        ms.move(origPaths, null, null);
-        ms.move(movedPaths, null, null);
-      }
-    }
-  }
-
-  /**
-   * Create a copy of given list of PathMetadatas with the paths moved from
-   * src to dest.
-   */
-  protected List<PathMetadata> moveMetas(List<PathMetadata> metas, Path src,
-      Path dest) throws IOException {
-    List<PathMetadata> moved = new ArrayList<>(metas.size());
-    for (PathMetadata srcMeta : metas) {
-      S3AFileStatus status = copyStatus((S3AFileStatus)srcMeta.getFileStatus());
-      status.setPath(movePath(status.getPath(), src, dest));
-      moved.add(new PathMetadata(status));
-    }
-    return moved;
-  }
-
-  protected Path movePath(Path p, Path src, Path dest) {
-    String srcStr = src.toUri().getPath();
-    String pathStr = p.toUri().getPath();
-    // Strip off src dir
-    pathStr = pathStr.substring(srcStr.length());
-    // Prepend new dest
-    return new Path(dest, pathStr);
-  }
-
-  protected S3AFileStatus copyStatus(S3AFileStatus status) {
-    if (status.isDirectory()) {
-      return new S3AFileStatus(status.isEmptyDirectory(), status.getPath(),
-          status.getOwner());
-    } else {
-      return new S3AFileStatus(status.getLen(), status.getModificationTime(),
-          status.getPath(), status.getBlockSize(), status.getOwner(),
-          status.getETag(), status.getVersionId());
-    }
-  }
-
-  /** @return number of PathMetadatas put() into MetadataStore */
-  private long populateMetadataStore(Collection<PathMetadata> paths,
-      MetadataStore ms) throws IOException {
-    long count = 0;
-    NanoTimer putTimer = new NanoTimer();
-    describe("Inserting into MetadataStore");
-    try (BulkOperationState operationState =
-            ms.initiateBulkWrite(BulkOperationState.OperationType.Put,
-                BUCKET_ROOT)) {
-      for (PathMetadata p : paths) {
-        ms.put(p, operationState);
-        count++;
-      }
-    }
-    putTimer.end();
-    printTiming(LOG, "put", putTimer, count);
-    return count;
-  }
-
-  protected void clearMetadataStore(MetadataStore ms, long count)
-      throws IOException {
-    describe("Recursive deletion");
-    NanoTimer deleteTimer = new NanoTimer();
-    ms.deleteSubtree(BUCKET_ROOT, null);
-    deleteTimer.end();
-    printTiming(LOG, "delete", deleteTimer, count);
-  }
-
-  private static void printTiming(Logger log, String op, NanoTimer timer,
-      long count) {
-    double msec = (double)timer.duration() / 1000;
-    double msecPerOp = msec / count;
-    log.info(String.format("Elapsed %.2f msec. %.3f msec / %s (%d ops)", msec,
-        msecPerOp, op, count));
-  }
-
-  protected static S3AFileStatus makeFileStatus(Path path) throws IOException {
-    return new S3AFileStatus(SIZE, ACCESS_TIME, path, BLOCK_SIZE, OWNER,
-        null, null);
-  }
-
-  protected static S3AFileStatus makeDirStatus(Path p) throws IOException {
-    return new S3AFileStatus(false, p, OWNER);
-  }
-
-  protected List<Path> metasToPaths(List<PathMetadata> metas) {
-    List<Path> paths = new ArrayList<>(metas.size());
-    for (PathMetadata meta : metas) {
-      paths.add(meta.getFileStatus().getPath());
-    }
-    return paths;
-  }
-
-  /**
-   * Recursively create a directory tree.
-   * @param parent Parent dir of the paths to create.
-   * @param depth How many more levels deep past parent to create.
-   * @param width Number of files (and directories, if depth > 0) per directory.
-   * @param paths List to add generated paths to.
-   */
-  protected static void createDirTree(Path parent, int depth, int width,
-      Collection<PathMetadata> paths) throws IOException {
-
-    // Create files
-    for (int i = 0; i < width; i++) {
-      Path p = new Path(parent, String.format("file-%d", i));
-      PathMetadata meta = new PathMetadata(makeFileStatus(p));
-      paths.add(meta);
-    }
-
-    if (depth == 0) {
-      return;
-    }
-
-    // Create directories if there is depth remaining
-    for (int i = 0; i < width; i++) {
-      Path dir = new Path(parent, String.format("dir-%d", i));
-      PathMetadata meta = new PathMetadata(makeDirStatus(dir));
-      paths.add(meta);
-      createDirTree(dir, depth-1, width, paths);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index 2dd92bbfc8c9d..f8d47011de3f0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -19,8 +19,13 @@
 package org.apache.hadoop.fs.s3a.scale;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.IntFunction;
 
 import com.amazonaws.event.ProgressEvent;
 import com.amazonaws.event.ProgressEventType;
@@ -35,7 +40,9 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileRange;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.Constants;
@@ -47,6 +54,7 @@
 import org.apache.hadoop.util.Progressable;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING;
@@ -383,7 +391,7 @@ private void logFSState() {
 
   /**
    * This is the set of actions to perform when verifying the file actually
-   * was created. With the s3guard committer, the file doesn't come into
+   * was created. With the S3A committer, the file doesn't come into
    * existence; a different set of assertions must be checked.
    */
   @Test
@@ -446,6 +454,30 @@ public void test_040_PositionedReadHugeFile() throws Throwable {
         toHuman(timer.nanosPerOperation(ops)));
   }
 
+  @Test
+  public void test_045_vectoredIOHugeFile() throws Throwable {
+    assumeHugeFileExists();
+    List<FileRange> rangeList = new ArrayList<>();
+    rangeList.add(FileRange.createFileRange(5856368, 116770));
+    rangeList.add(FileRange.createFileRange(3520861, 116770));
+    rangeList.add(FileRange.createFileRange(8191913, 116770));
+    rangeList.add(FileRange.createFileRange(1520861, 116770));
+    rangeList.add(FileRange.createFileRange(2520861, 116770));
+    rangeList.add(FileRange.createFileRange(9191913, 116770));
+    rangeList.add(FileRange.createFileRange(2820861, 156770));
+    IntFunction<ByteBuffer> allocate = ByteBuffer::allocate;
+    FileSystem fs = getFileSystem();
+    CompletableFuture<FSDataInputStream> builder =
+            fs.openFile(hugefile).build();
+    try (FSDataInputStream in = builder.get()) {
+      in.readVectored(rangeList, allocate);
+      byte[] readFullRes = new byte[(int)filesize];
+      in.readFully(0, readFullRes);
+      // Comparing vectored read results with read fully.
+      validateVectoredReadResult(rangeList, readFullRes);
+    }
+  }
+
   /**
    * Read in the entire file using read() calls.
    * @throws Throwable failure
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java
index 3fa288417c4d3..813eea8389f64 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java
@@ -30,7 +30,7 @@
 import java.util.concurrent.ExecutorService;
 
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.assertj.core.api.Assertions;
 import org.junit.Assume;
@@ -45,6 +45,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.auth.delegation.Csvout;
@@ -144,26 +145,25 @@ public ILoadTestS3ABulkDeleteThrottling(
   @Override
   protected Configuration createScaleConfiguration() {
     Configuration conf = super.createScaleConfiguration();
-    S3ATestUtils.disableFilesystemCaching(conf);
-    return conf;
-  }
 
-  @Override
-  public void setup() throws Exception {
-    final Configuration conf = getConf();
     S3ATestUtils.removeBaseAndBucketOverrides(conf,
         EXPERIMENTAL_AWS_INTERNAL_THROTTLING,
         BULK_DELETE_PAGE_SIZE,
-        USER_AGENT_PREFIX);
+        USER_AGENT_PREFIX,
+        ENABLE_MULTI_DELETE);
     conf.setBoolean(EXPERIMENTAL_AWS_INTERNAL_THROTTLING, throttle);
-    Assertions.assertThat(pageSize)
-        .describedAs("page size")
-        .isGreaterThan(0);
     conf.setInt(BULK_DELETE_PAGE_SIZE, pageSize);
     conf.set(USER_AGENT_PREFIX,
         String.format("ILoadTestS3ABulkDeleteThrottling-%s-%04d",
             throttle, pageSize));
 
+    S3ATestUtils.disableFilesystemCaching(conf);
+    return conf;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    final Configuration conf = getConf();
     super.setup();
     Assume.assumeTrue("multipart delete disabled",
         conf.getBoolean(ENABLE_MULTI_DELETE, true));
@@ -246,8 +246,8 @@ private File deleteFiles(final int requestCount,
         final ContractTestUtils.NanoTimer timer =
             new ContractTestUtils.NanoTimer();
         Exception ex = null;
-        try {
-          fs.removeKeys(fileList, false, null);
+        try (AuditSpan span = span())  {
+          fs.removeKeys(fileList, false);
         } catch (IOException e) {
           ex = e;
         }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
index a724b9737c948..dbdd8b5da6a3c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
@@ -38,7 +38,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.EXPERIMENTAL_AWS_INTERNAL_THROTTLING;
 import static org.apache.hadoop.fs.s3a.Constants.USER_AGENT_PREFIX;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.lsR;
-import static org.apache.hadoop.fs.s3a.impl.ITestPartialRenamesDeletes.createFiles;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
 import static org.apache.hadoop.test.GenericTestUtils.filenameOfIndex;
 
 /**
@@ -77,7 +77,6 @@ protected Configuration createScaleConfiguration() {
    * set too low. Alternatively, consider reducing the
    * <code>scale.test.operation.count</code> parameter in
    * <code>getOperationCount()</code>.
-   * If it is slow: look at the size of any S3Guard Table used.
    * @see #getOperationCount()
    */
   @Test
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
index d5e4788036ebe..91ea0c8e62fb3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a.scale;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -30,6 +31,7 @@
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 import org.apache.hadoop.fs.s3a.api.RequestFactory;
+import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 import org.apache.hadoop.util.functional.RemoteIterators;
@@ -53,7 +55,6 @@
 
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_KEEP;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
@@ -148,15 +149,45 @@ public void testListOperations() throws Throwable {
           listContinueRequests,
           listStatusCalls,
           getFileStatusCalls);
-      if (!fs.hasMetadataStore()) {
-        assertEquals(listRequests.toString(), 1, listRequests.diff());
-      }
+      assertEquals(listRequests.toString(), 1, listRequests.diff());
       reset(metadataRequests,
           listRequests,
           listContinueRequests,
           listStatusCalls,
           getFileStatusCalls);
 
+      describe("Get content summary for directory");
+
+      NanoTimer getContentSummaryTimer = new NanoTimer();
+
+      ContentSummary rootPathSummary = fs.getContentSummary(scaleTestDir);
+      ContentSummary testPathSummary = fs.getContentSummary(listDir);
+
+      getContentSummaryTimer.end("getContentSummary of %s", created);
+
+      // only two list operations should have taken place
+      print(LOG,
+          metadataRequests,
+          listRequests,
+          listContinueRequests,
+          listStatusCalls,
+          getFileStatusCalls);
+      assertEquals(listRequests.toString(), 2, listRequests.diff());
+      reset(metadataRequests,
+          listRequests,
+          listContinueRequests,
+          listStatusCalls,
+          getFileStatusCalls);
+
+      assertTrue("Root directory count should be > test path",
+          rootPathSummary.getDirectoryCount() > testPathSummary.getDirectoryCount());
+      assertTrue("Root file count should be >= to test path",
+          rootPathSummary.getFileCount() >= testPathSummary.getFileCount());
+      assertEquals("Incorrect directory count", created.getDirCount() + 1,
+          testPathSummary.getDirectoryCount());
+      assertEquals("Incorrect file count", created.getFileCount(),
+          testPathSummary.getFileCount());
+
     } finally {
       describe("deletion");
       // deletion at the end of the run
@@ -197,7 +228,6 @@ public void testMultiPagesListingPerformanceAndCorrectness()
             getConfigurationWithConfiguredBatchSize(batchSize);
 
     removeBaseAndBucketOverrides(conf,
-        S3_METADATA_STORE_IMPL,
         DIRECTORY_MARKER_POLICY);
     // force directory markers = keep to save delete requests on every
     // file created.
@@ -210,7 +240,6 @@ public void testMultiPagesListingPerformanceAndCorrectness()
 
     NanoTimer uploadTimer = new NanoTimer();
     try {
-      assume("Test is only for raw fs", !fs.hasMetadataStore());
       fs.create(dir);
 
       // create a span for the write operations
@@ -229,9 +258,9 @@ public void testMultiPagesListingPerformanceAndCorrectness()
         ObjectMetadata om = fs.newObjectMetadata(0L);
         PutObjectRequest put = requestFactory
             .newPutObjectRequest(fs.pathToKey(file), om,
-                new FailingInputStream());
+                null, new FailingInputStream());
         futures.add(submit(executorService, () ->
-            writeOperationHelper.putObject(put)));
+            writeOperationHelper.putObject(put, PutObjectOptions.keepingDirs())));
       }
       LOG.info("Waiting for PUTs to complete");
       waitForCompletion(futures);
@@ -321,9 +350,6 @@ public void testMultiPagesListingPerformanceAndCorrectness()
           .describedAs("Listing results using listLocatedStatus() must" +
               "match with original list of files")
           .hasSameElementsAs(originalListOfFiles);
-      // delete in this FS so S3Guard is left out of it.
-      // and so that the incremental listing is tested through
-      // the delete operation.
       fs.delete(dir, true);
     } finally {
       executorService.shutdown();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
index d2e04d928df4b..3dc64ea3b45fd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java
@@ -23,6 +23,9 @@
 import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
+import java.nio.file.AccessDeniedException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
@@ -41,10 +44,20 @@ public class ITestS3AHugeFilesSSECDiskBlocks
   private static final String KEY_1
       = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs=";
 
+  /**
+   * Skipping tests when running against mandatory encryption bucket
+   * which allows only certain encryption method.
+   * S3 throw AmazonS3Exception with status 403 AccessDenied
+   * then it is translated into AccessDeniedException by S3AUtils.translateException(...)
+   */
   @Override
   public void setup() throws Exception {
-    super.setup();
-    skipIfEncryptionTestsDisabled(getConfiguration());
+    try {
+      super.setup();
+      skipIfEncryptionTestsDisabled(getConfiguration());
+    } catch (AccessDeniedException e) {
+      skip("Bucket does not allow " + S3AEncryptionMethods.SSE_C + " encryption method");
+    }
   }
 
   @SuppressWarnings("deprecation")
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
new file mode 100644
index 0000000000000..99407467df56d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesStorageClass.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.bandwidth;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.toHuman;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_REDUCED_REDUNDANCY;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled;
+
+/**
+ * Class to verify that {@link Constants#STORAGE_CLASS} is set correctly
+ * for creating and renaming huge files with multipart upload requests.
+ */
+public class ITestS3AHugeFilesStorageClass extends AbstractSTestS3AHugeFiles {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ITestS3AHugeFilesStorageClass.class);
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    skipIfStorageClassTestsDisabled(getConfiguration());
+  }
+
+  @Override
+  protected Configuration createScaleConfiguration() {
+    Configuration conf = super.createScaleConfiguration();
+    disableFilesystemCaching(conf);
+    removeBaseAndBucketOverrides(conf, STORAGE_CLASS);
+
+    conf.set(STORAGE_CLASS, STORAGE_CLASS_REDUCED_REDUNDANCY);
+    return conf;
+  }
+
+  @Override
+  protected String getBlockOutputBufferName() {
+    return Constants.FAST_UPLOAD_BUFFER_ARRAY;
+  }
+
+  @Override
+  public void test_010_CreateHugeFile() throws IOException {
+    super.test_010_CreateHugeFile();
+    assertStorageClass(getPathOfFileToCreate());
+  }
+
+  @Override
+  public void test_030_postCreationAssertions() throws Throwable {
+    super.test_030_postCreationAssertions();
+    assertStorageClass(getPathOfFileToCreate());
+  }
+
+  @Override
+  public void test_040_PositionedReadHugeFile() throws Throwable {
+    skipQuietly("PositionedReadHugeFile");
+  }
+
+  @Override
+  public void test_050_readHugeFile() throws Throwable {
+    skipQuietly("readHugeFile");
+  }
+
+  @Override
+  public void test_090_verifyRenameSourceEncryption() throws IOException {
+    skipQuietly("verifyRenameSourceEncryption");
+  }
+
+  @Override
+  public void test_100_renameHugeFile() throws Throwable {
+    Path hugefile = getHugefile();
+    Path hugefileRenamed = getHugefileRenamed();
+    assumeHugeFileExists();
+    describe("renaming %s to %s", hugefile, hugefileRenamed);
+    S3AFileSystem fs = getFileSystem();
+    FileStatus status = fs.getFileStatus(hugefile);
+    long size = status.getLen();
+    fs.delete(hugefileRenamed, false);
+    ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+    fs.rename(hugefile, hugefileRenamed);
+    long mb = Math.max(size / _1MB, 1);
+    timer.end("time to rename file of %d MB", mb);
+    LOG.info("Time per MB to rename = {} nS", toHuman(timer.nanosPerOperation(mb)));
+    bandwidth(timer, size);
+    FileStatus destFileStatus = fs.getFileStatus(hugefileRenamed);
+    assertEquals(size, destFileStatus.getLen());
+    assertStorageClass(hugefileRenamed);
+  }
+
+  @Override
+  public void test_110_verifyRenameDestEncryption() throws IOException {
+    skipQuietly("verifyRenameDestEncryption");
+  }
+
+  private void skipQuietly(String text) {
+    describe("Skipping: %s", text);
+  }
+
+  protected void assertStorageClass(Path hugeFile) throws IOException {
+    S3AFileSystem fs = getFileSystem();
+    String actual = fs.getObjectMetadata(hugeFile).getStorageClass();
+
+    assertTrue(
+        "Storage class of object is " + actual + ", expected " + STORAGE_CLASS_REDUCED_REDUNDANCY,
+        STORAGE_CLASS_REDUCED_REDUNDANCY.equalsIgnoreCase(actual));
+  }
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
index 27798e21ed246..eea70ced13c92 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -22,11 +22,13 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3AInputPolicy;
 import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
@@ -41,7 +43,6 @@
 import org.assertj.core.api.Assertions;
 import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -50,27 +51,34 @@
 import java.io.EOFException;
 import java.io.IOException;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_BUFFER_SIZE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue;
-import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsSourceToString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToString;
 import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MAX;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MEAN;
 import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MIN;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
- * Look at the performance of S3a operations.
+ * Look at the performance of S3a Input Stream Reads.
  */
 public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
   private static final Logger LOG = LoggerFactory.getLogger(
       ITestS3AInputStreamPerformance.class);
+  private static final int READAHEAD_128K = 128 * _1KB;
 
   private S3AFileSystem s3aFS;
   private Path testData;
@@ -87,6 +95,14 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
   private boolean testDataAvailable = true;
   private String assumptionMessage = "test file";
 
+  @Override
+  protected Configuration createScaleConfiguration() {
+    Configuration conf = super.createScaleConfiguration();
+    S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+    conf.setBoolean(PREFETCH_ENABLED_KEY, false);
+    return conf;
+  }
+
   /**
    * Open the FS and the test data. The input stream is always set up here.
    * @throws IOException IO Problems.
@@ -128,14 +144,16 @@ public void cleanup() {
     describe("cleanup");
     IOUtils.closeStream(in);
     if (in != null) {
+      final IOStatistics stats = in.getIOStatistics();
       LOG.info("Stream statistics {}",
-          ioStatisticsSourceToString(in));
-      IOSTATS.aggregate(in.getIOStatistics());
+          ioStatisticsToPrettyString(stats));
+      IOSTATS.aggregate(stats);
     }
     if (s3aFS != null) {
+      final IOStatistics stats = s3aFS.getIOStatistics();
       LOG.info("FileSystem statistics {}",
-          ioStatisticsSourceToString(s3aFS));
-      FILESYSTEM_IOSTATS.aggregate(s3aFS.getIOStatistics());
+          ioStatisticsToPrettyString(stats));
+      FILESYSTEM_IOSTATS.aggregate(stats);
       IOUtils.closeStream(s3aFS);
     }
   }
@@ -177,7 +195,7 @@ FSDataInputStream openTestFile() throws IOException {
   FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
       throws IOException {
     requireCSVTestData();
-    return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
+    return openDataFile(s3aFS, testData, inputPolicy, readahead, testDataStatus.getLen());
   }
 
   /**
@@ -187,27 +205,28 @@ FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
    * @param path path to open
    * @param inputPolicy input policy to use
    * @param readahead readahead/buffer size
+   * @param length
    * @return the stream, wrapping an S3a one
    * @throws IOException IO problems
    */
   private FSDataInputStream openDataFile(S3AFileSystem fs,
       Path path,
       S3AInputPolicy inputPolicy,
-      long readahead) throws IOException {
+      long readahead,
+      final long length) throws IOException {
     int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
         DEFAULT_READ_BUFFER_SIZE);
-    S3AInputPolicy policy = fs.getInputPolicy();
-    fs.setInputPolicy(inputPolicy);
-    try {
-      FSDataInputStream stream = fs.open(path, bufferSize);
-      if (readahead >= 0) {
-        stream.setReadahead(readahead);
-      }
-      streamStatistics = getInputStreamStatistics(stream);
-      return stream;
-    } finally {
-      fs.setInputPolicy(policy);
+    final FutureDataInputStreamBuilder builder = fs.openFile(path)
+        .opt(FS_OPTION_OPENFILE_READ_POLICY,
+            inputPolicy.toString())
+        .opt(FS_OPTION_OPENFILE_LENGTH, length)
+        .opt(FS_OPTION_OPENFILE_BUFFER_SIZE, bufferSize);
+    if (readahead > 0) {
+      builder.opt(READAHEAD_RANGE, readahead);
     }
+    FSDataInputStream stream = awaitFuture(builder.build());
+    streamStatistics = getInputStreamStatistics(stream);
+    return stream;
   }
 
   /**
@@ -293,8 +312,10 @@ public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
       if (bandwidth(blockTimer, blockSize) < minimumBandwidth) {
         LOG.warn("Bandwidth {} too low on block {}: resetting connection",
             bw, blockId);
-        Assert.assertTrue("Bandwidth of " + bw +" too low after  "
-            + resetCount + " attempts", resetCount <= maxResetCount);
+        Assertions.assertThat(resetCount)
+            .describedAs("Bandwidth of %s too low after  %s attempts",
+                bw, resetCount)
+            .isLessThanOrEqualTo(maxResetCount);
         resetCount++;
         // reset the connection
         getS3AInputStream(in).resetConnection();
@@ -359,7 +380,7 @@ public void testReadWithNormalPolicy() throws Throwable {
   public void testDecompressionSequential128K() throws Throwable {
     describe("Decompress with a 128K readahead");
     skipIfClientSideEncryption();
-    executeDecompression(128 * _1KB, S3AInputPolicy.Sequential);
+    executeDecompression(READAHEAD_128K, S3AInputPolicy.Sequential);
     assertStreamOpenedExactlyOnce();
   }
 
@@ -558,7 +579,7 @@ public void testRandomReadOverBuffer() throws Throwable {
     byte[] buffer = new byte[datasetLen];
     int readahead = _8K;
     int halfReadahead = _4K;
-    in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
+    in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead, datasetLen);
 
     LOG.info("Starting initial reads");
     S3AInputStream s3aStream = getS3aStream();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java
index 231cfd884e0c8..b83d12b4c1a66 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java
@@ -35,7 +35,7 @@
 import org.apache.hadoop.fs.s3a.S3AInstrumentation;
 import org.apache.hadoop.fs.s3a.Statistic;
 import org.apache.hadoop.fs.s3a.auth.ProgressCounter;
-import org.apache.hadoop.fs.s3a.commit.CommitOperations;
+import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations;
 
 import static org.apache.hadoop.fs.StreamCapabilities.ABORTABLE_STREAM;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index d95b46b10d786..514c6cf886918 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -19,19 +19,14 @@
 package org.apache.hadoop.fs.s3a.scale;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
-import org.apache.hadoop.fs.s3a.S3AInputStream;
 import org.apache.hadoop.fs.s3a.S3ATestConstants;
 import org.apache.hadoop.fs.s3a.Statistic;
-import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.InputStream;
-
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupGaugeStatistic;
 
@@ -154,34 +149,6 @@ protected int getTestTimeoutMillis() {
     return getTestTimeoutSeconds() * 1000;
   }
 
-  /**
-   * Get the input stream statistics of an input stream.
-   * Raises an exception if the inner stream is not an S3A input stream
-   * @param in wrapper
-   * @return the statistics for the inner stream
-   */
-  protected S3AInputStreamStatistics getInputStreamStatistics(
-      FSDataInputStream in) {
-    return getS3AInputStream(in).getS3AStreamStatistics();
-  }
-
-  /**
-   * Get the inner stream of an input stream.
-   * Raises an exception if the inner stream is not an S3A input stream
-   * @param in wrapper
-   * @return the inner stream
-   * @throws AssertionError if the inner stream is of the wrong type
-   */
-  protected S3AInputStream getS3AInputStream(
-      FSDataInputStream in) {
-    InputStream inner = in.getWrappedStream();
-    if (inner instanceof S3AInputStream) {
-      return (S3AInputStream) inner;
-    } else {
-      throw new AssertionError("Not an S3AInputStream: " + inner);
-    }
-  }
-
   /**
    * Get the gauge value of a statistic from the
    * IOStatistics of the filesystem. Raises an assertion if
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java
index 56d99d1abe391..2c1a10a21d0ab 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/AbstractS3SelectTest.java
@@ -60,11 +60,13 @@
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.DurationInfo;
 
-import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture;
+import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVPath;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.select.CsvFile.ALL_QUOTES;
 import static org.apache.hadoop.fs.s3a.select.SelectConstants.*;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * Superclass for S3 Select tests.
@@ -280,6 +282,14 @@ boolean isSelectAvailable(final FileSystem filesystem) {
         .hasCapability(S3_SELECT_CAPABILITY);
   }
 
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    removeBaseAndBucketOverrides(conf, STORAGE_CLASS);
+
+    return conf;
+  }
+
   /**
    * Setup: requires select to be available.
    */
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/CsvFile.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/CsvFile.java
index 8e3da890d7e69..17574372de429 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/CsvFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/CsvFile.java
@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java
index 6918941295699..97fbaae0ae4d3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3Select.java
@@ -63,8 +63,8 @@
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.util.DurationInfo;
 
-import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE;
-import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADV_NORMAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_DEFAULT;
 import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE;
 import static org.apache.hadoop.fs.s3a.select.CsvFile.ALL_QUOTES;
 import static org.apache.hadoop.fs.s3a.select.SelectBinding.expandBackslashChars;
@@ -496,7 +496,7 @@ public void testSelectFileExample() throws Throwable {
    */
   @Test
   public void testSelectUnsupportedInputFormat() throws Throwable {
-    describe("Request an unsupported input format");
+    describe("Request an Unsupported input format");
     FutureDataInputStreamBuilder builder = getFileSystem().openFile(csvPath)
         .must(SELECT_SQL, SELECT_ODD_ENTRIES)
         .must(SELECT_INPUT_FORMAT, "pptx");
@@ -510,7 +510,7 @@ public void testSelectUnsupportedInputFormat() throws Throwable {
    */
   @Test
   public void testSelectUnsupportedOutputFormat() throws Throwable {
-    describe("Request a (currently) unsupported output format");
+    describe("Request a (currently) Unsupported output format");
     FutureDataInputStreamBuilder builder = getFileSystem().openFile(csvPath)
         .must(SELECT_SQL, SELECT_ODD_ENTRIES)
         .must(SELECT_INPUT_FORMAT, "csv")
@@ -567,7 +567,7 @@ public void testSelectRootFails() throws Throwable {
     FutureDataInputStreamBuilder builder =
         getFileSystem().openFile(path("/"))
             .must(SELECT_SQL, SELECT_ODD_ENTRIES);
-    interceptFuture(FileNotFoundException.class,
+    interceptFuture(IOException.class,
         "", builder.build());
   }
 
@@ -767,7 +767,8 @@ public void testSelectOddRecordsIgnoreHeaderV1()
     JobConf conf = createJobConf();
     inputOpt(conf, CSV_INPUT_HEADER, CSV_HEADER_OPT_NONE);
     inputMust(conf, CSV_INPUT_HEADER, CSV_HEADER_OPT_IGNORE);
-    inputMust(conf, INPUT_FADVISE, INPUT_FADV_NORMAL);
+    inputMust(conf, FS_OPTION_OPENFILE_READ_POLICY,
+        FS_OPTION_OPENFILE_READ_POLICY_DEFAULT);
     inputMust(conf, SELECT_ERRORS_INCLUDE_SQL, "true");
     verifySelectionCount(EVEN_ROWS_COUNT,
         SELECT_EVEN_ROWS_NO_HEADER,
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectCLI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectCLI.java
index b22d2f53b4bde..f5c630aa5bdba 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectCLI.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectCLI.java
@@ -42,7 +42,7 @@
 import org.apache.hadoop.util.OperationDuration;
 import org.apache.hadoop.util.ToolRunner;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
 import static org.apache.hadoop.fs.s3a.select.ITestS3SelectLandsat.SELECT_NOTHING;
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java
index 8b0578df11c01..51da971fb7063 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java
@@ -415,7 +415,7 @@ public void testSelectSeekFullLandsat() throws Throwable {
       long increment = 64 * _1KB;
 
       // seek forward, comparing bytes
-      for(offset = 32 * _1KB; offset < actualLen; offset += increment) {
+      for(offset = 32 * _1KB; offset < _1MB; offset += increment) {
         seek(seekStream, offset);
         assertEquals("Seek position in " + seekStream,
             offset, seekStream.getPos());
@@ -423,11 +423,6 @@ public void testSelectSeekFullLandsat() throws Throwable {
         assertEquals("byte at seek position",
             dataset[(int) seekStream.getPos()], seekStream.read());
       }
-      for(; offset < len; offset += _1MB) {
-        seek(seekStream, offset);
-        assertEquals("Seek position in " + seekStream,
-            offset, seekStream.getPos());
-      }
       // there's no knowledge of how much data is left, but with Gzip
       // involved there can be a lot. To keep the test duration down,
       // this test, unlike the simpler one, doesn't try to read past the
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectMRJob.java
index 52a591384772a..c1c7b89dce83d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectMRJob.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectMRJob.java
@@ -30,7 +30,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.examples.WordCount;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 import org.apache.hadoop.fs.s3a.S3AUtils;
@@ -44,6 +43,7 @@
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.functional.FutureIO;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 
@@ -203,7 +203,7 @@ public void testLandsatSelect() throws Exception {
   private String readStringFromFile(Path path) throws IOException {
     int bytesLen = (int)fs.getFileStatus(path).getLen();
     byte[] buffer = new byte[bytesLen];
-    return FutureIOSupport.awaitFuture(
+    return FutureIO.awaitFuture(
         fs.openFile(path).build().thenApply(in -> {
           try {
             IOUtils.readFully(in, buffer, 0, bytesLen);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestS3AContractStreamIOStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestS3AContractStreamIOStatistics.java
index 8bed174fd31e1..0f6b69cd54d89 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestS3AContractStreamIOStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestS3AContractStreamIOStatistics.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.fs.contract.s3a.S3AContract;
 import org.apache.hadoop.fs.statistics.StreamStatisticNames;
 
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
 import static org.apache.hadoop.fs.statistics.StreamStatisticNames.*;
 
 /**
@@ -36,14 +35,6 @@
 public class ITestS3AContractStreamIOStatistics extends
     AbstractContractStreamIOStatisticsTest {
 
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    // patch in S3Guard options
-    maybeEnableS3Guard(conf);
-    return conf;
-  }
-
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new S3AContract(conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java
index 678e5d482d1a5..2a8cfc663a53a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.fs.s3a.S3ListRequest;
 import org.apache.hadoop.fs.s3a.S3ListResult;
 import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
 import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
@@ -40,8 +39,7 @@ public class MinimalListingOperationCallbacks
   @Override
   public CompletableFuture<S3ListResult> listObjectsAsync(
       final S3ListRequest request,
-      final DurationTrackerFactory trackerFactory, AuditSpan span)
-      throws IOException {
+      final DurationTrackerFactory trackerFactory, AuditSpan span) {
     return null;
   }
 
@@ -49,8 +47,7 @@ public CompletableFuture<S3ListResult> listObjectsAsync(
   public CompletableFuture<S3ListResult> continueListObjectsAsync(
       final S3ListRequest request,
       final S3ListResult prevResult,
-      final DurationTrackerFactory trackerFactory, AuditSpan span)
-      throws IOException {
+      final DurationTrackerFactory trackerFactory, AuditSpan span) {
     return null;
   }
 
@@ -78,14 +75,4 @@ public int getMaxKeys() {
     return 0;
   }
 
-  @Override
-  public ITtlTimeProvider getUpdatedTtlTimeProvider() {
-    return null;
-  }
-
-  @Override
-  public boolean allowAuthoritative(Path p) {
-    return false;
-  }
-
 }
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java
index a50b944c798ca..fa1ad2db62af7 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalOperationCallbacks.java
@@ -23,7 +23,6 @@
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.DeleteObjectsResult;
 import com.amazonaws.services.s3.model.MultiObjectDeleteException;
 import com.amazonaws.services.s3.transfer.model.CopyResult;
 
@@ -35,7 +34,6 @@
 import org.apache.hadoop.fs.s3a.S3AReadOpContext;
 import org.apache.hadoop.fs.s3a.S3ObjectAttributes;
 import org.apache.hadoop.fs.s3a.impl.OperationCallbacks;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
 
 /**
  * Stub implementation of {@link OperationCallbacks}.
@@ -76,8 +74,7 @@ public void finishRename(
   public void deleteObjectAtPath(
       Path path,
       String key,
-      boolean isFile,
-      BulkOperationState operationState)
+      boolean isFile)
       throws IOException {
 
   }
@@ -86,7 +83,6 @@ public void deleteObjectAtPath(
   public RemoteIterator<S3ALocatedFileStatus> listFilesAndDirectoryMarkers(
       final Path path,
       final S3AFileStatus status,
-      final boolean collectTombstones,
       final boolean includeSelf) throws IOException {
     return null;
   }
@@ -102,20 +98,11 @@ public CopyResult copyFile(
   }
 
   @Override
-  public DeleteObjectsResult removeKeys(
-      List<DeleteObjectsRequest.KeyVersion> keysToDelete,
-      boolean deleteFakeDir,
-      List<Path> undeletedObjectsOnFailure,
-      BulkOperationState operationState,
-      boolean quiet)
+  public void removeKeys(
+          List<DeleteObjectsRequest.KeyVersion> keysToDelete,
+          boolean deleteFakeDir)
       throws MultiObjectDeleteException, AmazonClientException,
              IOException {
-    return null;
-  }
-
-  @Override
-  public boolean allowAuthoritative(Path p) {
-    return false;
   }
 
   @Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java
similarity index 52%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java
rename to hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java
index 7884d8e830f59..ffba558d11fd0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetastoreInstrumentationImpl.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java
@@ -16,57 +16,31 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.s3guard;
+package org.apache.hadoop.fs.s3a.test;
 
-/**
- * A no-op implementation of {@link MetastoreInstrumentation}
- * which allows metastores to always return an instance
- * when requested.
- */
-public class MetastoreInstrumentationImpl implements MetastoreInstrumentation {
-
-  @Override
-  public void initialized() {
-
-  }
-
-  @Override
-  public void storeClosed() {
-
-  }
-
-  @Override
-  public void throttled() {
-
-  }
-
-  @Override
-  public void retrying() {
-
-  }
-
-  @Override
-  public void recordsDeleted(final int count) {
-
-  }
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.SelectObjectContentRequest;
+import com.amazonaws.services.s3.model.SelectObjectContentResult;
 
-  @Override
-  public void recordsRead(final int count) {
+import org.apache.hadoop.fs.s3a.WriteOperationHelper;
 
-  }
+/**
+ * Stub implementation of writeOperationHelper callbacks.
+ */
+public class MinimalWriteOperationHelperCallbacks
+    implements WriteOperationHelper.WriteOperationHelperCallbacks {
 
   @Override
-  public void recordsWritten(final int count) {
-
+  public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
+    return null;
   }
 
   @Override
-  public void directoryMarkedAuthoritative() {
-
+  public CompleteMultipartUploadResult completeMultipartUpload(
+      CompleteMultipartUploadRequest request) {
+    return null;
   }
 
-  @Override
-  public void entryAdded(final long durationNanos) {
+};
 
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/OperationTrackingStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/OperationTrackingStore.java
deleted file mode 100644
index 1bf0c3e6fc1e9..0000000000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/OperationTrackingStore.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.test;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.impl.StoreContext;
-import org.apache.hadoop.fs.s3a.s3guard.BulkOperationState;
-import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.ITtlTimeProvider;
-import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
-import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
-import org.apache.hadoop.fs.s3a.s3guard.RenameTracker;
-
-/**
- * MetadataStore which tracks what is deleted and added.
- */
-public class OperationTrackingStore implements MetadataStore {
-
-  private final List<Path> deleted = new ArrayList<>();
-
-  private final List<Path> created = new ArrayList<>();
-
-  @Override
-  public void initialize(final FileSystem fs,
-      ITtlTimeProvider ttlTimeProvider) {
-  }
-
-  @Override
-  public void initialize(final Configuration conf,
-      ITtlTimeProvider ttlTimeProvider) {
-  }
-
-  @Override
-  public void forgetMetadata(final Path path) {
-  }
-
-  @Override
-  public PathMetadata get(final Path path) {
-    return null;
-  }
-
-  @Override
-  public PathMetadata get(final Path path,
-      final boolean wantEmptyDirectoryFlag) {
-    return null;
-  }
-
-  @Override
-  public DirListingMetadata listChildren(final Path path) {
-    return null;
-  }
-
-  @Override
-  public void put(final PathMetadata meta) {
-    put(meta, null);
-  }
-
-  @Override
-  public void put(final PathMetadata meta,
-      final BulkOperationState operationState) {
-    created.add(meta.getFileStatus().getPath());
-  }
-
-  @Override
-  public void put(final Collection<? extends PathMetadata> metas,
-      final BulkOperationState operationState) {
-    metas.stream().forEach(meta -> put(meta, null));
-  }
-
-  @Override
-  public void put(final DirListingMetadata meta,
-      final List<Path> unchangedEntries,
-      final BulkOperationState operationState) {
-    created.add(meta.getPath());
-  }
-
-  @Override
-  public void destroy() {
-  }
-
-  @Override
-  public void delete(final Path path,
-      final BulkOperationState operationState) {
-    deleted.add(path);
-  }
-
-  @Override
-  public void deletePaths(final Collection<Path> paths,
-      @Nullable final BulkOperationState operationState)
-      throws IOException {
-    deleted.addAll(paths);
-  }
-
-  @Override
-  public void deleteSubtree(final Path path,
-      final BulkOperationState operationState) {
-
-  }
-
-  @Override
-  public void move(@Nullable final Collection<Path> pathsToDelete,
-      @Nullable final Collection<PathMetadata> pathsToCreate,
-      @Nullable final BulkOperationState operationState) {
-  }
-
-  @Override
-  public void prune(final PruneMode pruneMode, final long cutoff) {
-  }
-
-  @Override
-  public long prune(final PruneMode pruneMode,
-      final long cutoff,
-      final String keyPrefix) {
-    return 0;
-  }
-
-  @Override
-  public BulkOperationState initiateBulkWrite(
-      final BulkOperationState.OperationType operation,
-      final Path dest) {
-    return null;
-  }
-
-  @Override
-  public void setTtlTimeProvider(ITtlTimeProvider ttlTimeProvider) {
-  }
-
-  @Override
-  public Map<String, String> getDiagnostics() {
-    return null;
-  }
-
-  @Override
-  public void updateParameters(final Map<String, String> parameters) {
-  }
-
-  @Override
-  public void close() {
-  }
-
-  public List<Path> getDeleted() {
-    return deleted;
-  }
-
-  public List<Path> getCreated() {
-    return created;
-  }
-
-  @Override
-  public RenameTracker initiateRenameOperation(
-      final StoreContext storeContext,
-      final Path source,
-      final S3AFileStatus sourceStatus,
-      final Path dest) {
-    throw new UnsupportedOperationException("unsupported");
-  }
-
-  @Override
-  public void addAncestors(final Path qualifiedPath,
-      @Nullable final BulkOperationState operationState) {
-
-  }
-}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
new file mode 100644
index 0000000000000..669acd8b8bd56
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.test;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_BUCKET_WITH_MANY_OBJECTS;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_REQUESTER_PAYS_FILE;
+
+/**
+ * Provides S3A filesystem URIs for public data sets for specific use cases.
+ *
+ * This allows for the contract between S3A tests and the existence of data sets
+ * to be explicit and also standardizes access and configuration of
+ * replacements.
+ *
+ * Bucket specific configuration such as endpoint or requester pays should be
+ * configured within "hadoop-tools/hadoop-aws/src/test/resources/core-site.xml".
+ *
+ * Warning: methods may mutate the configuration instance passed in.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class PublicDatasetTestUtils {
+
+  /**
+   * Private constructor for utility class.
+   */
+  private PublicDatasetTestUtils() {}
+
+  /**
+   * Default path for an object inside a requester pays bucket: {@value}.
+   */
+  private static final String DEFAULT_REQUESTER_PAYS_FILE
+      = "s3a://usgs-landsat/collection02/catalog.json";
+
+  /**
+   * Default bucket for an S3A file system with many objects: {@value}.
+   *
+   * We use a subdirectory to ensure we have permissions on all objects
+   * contained within as well as permission to inspect the directory itself.
+   */
+  private static final String DEFAULT_BUCKET_WITH_MANY_OBJECTS
+      = "s3a://usgs-landsat/collection02/level-1/";
+
+  /**
+   * Provide a URI for a directory containing many objects.
+   *
+   * Unless otherwise configured,
+   * this will be {@value DEFAULT_BUCKET_WITH_MANY_OBJECTS}.
+   *
+   * @param conf Hadoop configuration
+   * @return S3A FS URI
+   */
+  public static String getBucketPrefixWithManyObjects(Configuration conf) {
+    return fetchFromConfig(conf,
+        KEY_BUCKET_WITH_MANY_OBJECTS, DEFAULT_BUCKET_WITH_MANY_OBJECTS);
+  }
+
+  /**
+   * Provide a URI to an object within a requester pays enabled bucket.
+   *
+   * Unless otherwise configured,
+   * this will be {@value DEFAULT_REQUESTER_PAYS_FILE}.
+   *
+   * @param conf Hadoop configuration
+   * @return S3A FS URI
+   */
+  public static String getRequesterPaysObject(Configuration conf) {
+    return fetchFromConfig(conf,
+        KEY_REQUESTER_PAYS_FILE, DEFAULT_REQUESTER_PAYS_FILE);
+  }
+
+  private static String fetchFromConfig(Configuration conf, String key, String defaultValue) {
+    String value = conf.getTrimmed(key, defaultValue);
+
+    S3ATestUtils.assume("Empty test property: " + key, !value.isEmpty());
+
+    return value;
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
index 797d33c8d90d7..0c2473a5f61da 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/AbstractMarkerToolTest.java
@@ -73,12 +73,9 @@ protected Configuration createConfiguration() {
     removeBaseAndBucketOverrides(bucketName, conf,
         S3A_BUCKET_PROBE,
         DIRECTORY_MARKER_POLICY,
-        S3_METADATA_STORE_IMPL,
-        METADATASTORE_AUTHORITATIVE,
         AUTHORITATIVE_PATH);
     // base FS is legacy
     conf.set(DIRECTORY_MARKER_POLICY, DIRECTORY_MARKER_POLICY_DELETE);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
 
     // turn off bucket probes for a bit of speedup in the connectors we create.
     conf.setInt(S3A_BUCKET_PROBE, 0);
@@ -192,7 +189,6 @@ protected S3AFileSystem createFS(String markerPolicy,
     String bucketName = getTestBucketName(conf);
     removeBucketOverrides(bucketName, conf,
         DIRECTORY_MARKER_POLICY,
-        S3_METADATA_STORE_IMPL,
         BULK_DELETE_PAGE_SIZE,
         AUTHORITATIVE_PATH);
     if (authPath != null) {
@@ -201,7 +197,6 @@ protected S3AFileSystem createFS(String markerPolicy,
     // Use a very small page size to force the paging
     // code to be tested.
     conf.setInt(BULK_DELETE_PAGE_SIZE, 2);
-    conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
     conf.set(DIRECTORY_MARKER_POLICY, markerPolicy);
     S3AFileSystem fs2 = new S3AFileSystem();
     fs2.initialize(testFSUri, conf);
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java
index fc1abc19dd8d8..127fcbab75023 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java
@@ -28,9 +28,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_MARKER_POLICY_AUTHORITATIVE;
@@ -307,22 +309,25 @@ public void testRunLimitedAudit() throws Throwable {
   }
 
   /**
-   * Run an audit against the landsat bucket.
+   * Run an audit against a bucket with a large number of objects.
    * <p></p>
    * This tests paging/scale against a larger bucket without
    * worrying about setup costs.
    */
   @Test
-  public void testRunLimitedLandsatAudit() throws Throwable {
-    describe("Audit a few thousand landsat objects");
+  public void testRunAuditManyObjectsInBucket() throws Throwable {
+    describe("Audit a few thousand objects");
     final File audit = tempAuditFile();
 
+    Configuration conf = super.createConfiguration();
+    String bucketUri = PublicDatasetTestUtils.getBucketPrefixWithManyObjects(conf);
+
     runToFailure(EXIT_INTERRUPTED,
         MARKERS,
         AUDIT,
         m(OPT_LIMIT), 3000,
         m(OPT_OUT), audit,
-        LANDSAT_BUCKET);
+        bucketUri);
     readOutput(audit);
   }
 
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
index 87160937a1825..87b37b7f8ffbc 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
@@ -119,7 +119,7 @@ public void testWithMiniCluster() throws Exception {
 
     Path success = new Path(output, _SUCCESS);
     FileStatus status = fs.getFileStatus(success);
-    assertTrue("0 byte success file - not a s3guard committer " + success,
+    assertTrue("0 byte success file - not an S3A committer " + success,
         status.getLen() > 0);
     SuccessData successData = SuccessData.load(fs, success);
     String commitDetails = successData.toString();
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 21e30b6d66cb1..1533477229642 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -30,6 +30,8 @@
     <final>false</final>
   </property>
 
+  <!-- Per-bucket configurations: landsat-pds -->
+
   <property>
     <name>fs.s3a.bucket.landsat-pds.endpoint</name>
     <value>${central.endpoint}</value>
@@ -43,14 +45,6 @@
     it will only create log noise.</description>
   </property>
 
-  <!-- Make sure S3Guard is disabled for read-only bucket tests. -->
-  <property>
-    <name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name>
-    <value>${s3guard.null}</value>
-    <description>The read-only landsat-pds repository isn't
-        managed by s3guard</description>
-  </property>
-
   <property>
     <name>fs.s3a.bucket.landsat-pds.probe</name>
     <value>0</value>
@@ -63,17 +57,32 @@
     <description>Do not add the referrer header to landsat operations</description>
   </property>
 
-  <!-- Convenience definitions. -->
+  <!-- Per-bucket configurations: usgs-landsat -->
+
   <property>
-    <name>s3guard.null</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
+    <name>fs.s3a.bucket.usgs-landsat.endpoint</name>
+    <value>${central.endpoint}</value>
   </property>
 
   <property>
-    <name>s3guard.dynamo</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
+    <name>fs.s3a.bucket.usgs-landsat.requester.pays.enabled</name>
+    <value>true</value>
+    <description>usgs-landsat requires requester pays enabled</description>
   </property>
 
+  <property>
+    <name>fs.s3a.bucket.usgs-landsat.multipart.purge</name>
+    <value>false</value>
+    <description>Don't try to purge uploads in the read-only bucket, as
+      it will only create log noise.</description>
+  </property>
+
+  <property>
+    <name>fs.s3a.bucket.usgs-landsat.audit.add.referrer.header</name>
+    <value>false</value>
+  </property>
+
+
   <!--
    This is the default endpoint, which can be used to interact
    with any v2 region.
@@ -162,29 +171,29 @@
     <value>simple</value>
   </property>
 
-  <!-- Reduce DDB capacity on auto-created tables, to keep bills down. -->
+  <!-- auditing rejects out of span operations; this ensures that
+        no regressions take place. -->
   <property>
-    <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-    <value>0</value>
+    <name>fs.s3a.audit.reject.out.of.span.operations</name>
+    <value>true</value>
   </property>
+
+  <!-- Enable IOStatisticsContext support for Thread level. -->
   <property>
-    <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-    <value>0</value>
+    <name>fs.iostatistics.thread.level.enabled</name>
+    <value>true</value>
   </property>
 
-  <!-- auditing rejects out of span operations; this ensures that
-        no regressions take place. -->
   <property>
-    <name>fs.s3a.audit.reject.out.of.span.operations</name>
-    <value>true</value>
+    <name>fs.s3a.connection.request.timeout</name>
+    <value>10s</value>
   </property>
 
   <!--
   To run these tests.
 
   # Create a file auth-keys.xml  - DO NOT ADD TO REVISION CONTROL
-  # add the property test.fs.s3n.name to point to an S3 filesystem URL
-  # Add the credentials for the service you are testing against
+  # see testing.md for details on what to set.
   -->
   <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
     <fallback/>
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index 653c8a75e69ff..0ec8d52042807 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -52,20 +52,14 @@ log4j.logger.org.apache.hadoop.ipc.Server=WARN
 
 # for debugging low level S3a operations, uncomment these lines
 # Log all S3A classes
-#log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
+log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
 #log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO
 #log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO
-# Log S3Guard classes
-#log4j.logger.org.apache.hadoop.fs.s3a.s3guard=DEBUG
-# if set to debug, this will log the PUT/DELETE operations on a store
-log4j.logger.org.apache.hadoop.fs.s3a.s3guard.Operations=DEBUG
+log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN
 
 # Log Committer classes
 #log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG
 
-# Enable debug logging of AWS Dynamo client
-#log4j.logger.com.amazonaws.services.dynamodbv2.AmazonDynamoDB=DEBUG
-
 # Log all HTTP requests made; includes S3 interaction. This may
 # include sensitive information such as account IDs in HTTP headers.
 #log4j.logger.com.amazonaws.request=DEBUG
@@ -89,3 +83,6 @@ log4j.logger.org.apache.hadoop.fs.s3a.S3AStorageStatistics=INFO
 #log4j.logger.org.apache.hadoop.fs.s3a.audit=DEBUG
 # log request creation, span lifecycle and other low-level details
 #log4j.logger.org.apache.hadoop.fs.s3a.audit=TRACE
+
+# uncomment this to trace where context entries are set
+# log4j.logger.org.apache.hadoop.fs.audit.CommonAuditContext=TRACE
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 6d73cfc09a719..cec050d2c1be7 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -29,7 +29,6 @@
   </description>
   <packaging>jar</packaging>
   <properties>
-    <okHttpVersion>${okhttp.version}</okHttpVersion>
     <minimalJsonVersion>0.9.1</minimalJsonVersion>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
@@ -65,38 +64,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
 
 
@@ -150,12 +117,6 @@
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>com.squareup.okhttp</groupId>
-      <artifactId>okhttp</artifactId>
-      <version>${okhttp.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -173,12 +134,6 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-      <groupId>com.squareup.okhttp</groupId>
-      <artifactId>mockwebserver</artifactId>
-      <version>${okhttp.version}</version>
-      <scope>test</scope>
-    </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 06b82f8a835c7..8d9677dc0ea07 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -27,7 +27,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.microsoft.azure.datalake.store.ADLStoreClient;
 import com.microsoft.azure.datalake.store.ADLStoreOptions;
 import com.microsoft.azure.datalake.store.DirectoryEntry;
diff --git a/hadoop-tools/hadoop-azure/.gitignore b/hadoop-tools/hadoop-azure/.gitignore
index 0e17efaa1eb24..522210137ec16 100644
--- a/hadoop-tools/hadoop-azure/.gitignore
+++ b/hadoop-tools/hadoop-azure/.gitignore
@@ -1,5 +1,6 @@
 .checkstyle
 bin/
-src/test/resources/combinationConfigFiles
 src/test/resources/abfs-combination-test-configs.xml
 dev-support/testlogs
+src/test/resources/accountSettings/*
+!src/test/resources/accountSettings/accountName_settings.xml.template
diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
index 25d9593d573eb..400d0a23834a5 100755
--- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
@@ -2,7 +2,7 @@
 
 # shellcheck disable=SC2034
 # unused variables are global in nature and used in testsupport.sh
-
+test
 set -eo pipefail
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,36 +22,154 @@ set -eo pipefail
 
 # shellcheck disable=SC1091
 . dev-support/testrun-scripts/testsupport.sh
+init
+
+resourceDir=src/test/resources/
+logdir=dev-support/testlogs/
+azureTestXml=azure-auth-keys.xml
+azureTestXmlPath=$resourceDir$azureTestXml
+processCount=8
 
-begin
+## SECTION: TEST COMBINATION METHODS
 
-### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
-### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
+runHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
+runHNSSharedKeyTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("SharedKey")
+  triggerRun "HNS-SharedKey" "$accountName"  "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth")
-generateconfigs
+runNonHNSSharedKeyTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("SharedKey")
+  triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=AppendBlob-HNS-OAuth
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
-"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
-values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
-generateconfigs
+runAppendBlobHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+  VALUES=("OAuth" "true")
+  triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
-generateconfigs
+runTest=false
+cleanUpTestContainers=false
+echo 'Ensure below are complete before running script:'
+echo '1. Account specific settings file is present.'
+echo '   Copy accountName_settings.xml.template to accountName_settings.xml'
+echo '   where accountName in copied file name should be the test account name without domain'
+echo '   (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+echo '   folder. New account settings file to be added to same folder.)'
+echo '   Follow instructions in the template to populate settings correctly for the account'
+echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+echo '   where accountNames should be the test account names without domain'
+echo ' '
+echo ' '
+echo 'Choose action:'
+echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+do
+  case $scriptMode in
+  SET_ACTIVE_TEST_CONFIG)
+    runTest=false
+    break
+    ;;
+  RUN_TEST)
+    runTest=true
+    read -r -p "Enter parallel test run process count [default - 8]: " processCount
+    processCount=${processCount:-8}
+    break
+    ;;
+  CLEAN_UP_OLD_TEST_CONTAINERS)
+    runTest=false
+    cleanUpTestContainers=true
+    break
+    ;;
+  SET_OR_CHANGE_TEST_ACCOUNT)
+    runTest=false
+    cleanUpTestContainers=false
+    accountSettingsFile="src/test/resources/azure-auth-keys.xml"
+    if [[ ! -f "$accountSettingsFile" ]];
+    then
+      logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
+      cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
+    fi
 
-combination=NonHNS-SharedKey
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
-values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
-generateconfigs
+    vi $accountSettingsFile
+    exit 0
+    break
+    ;;
+  PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
+    runTest=false
+    cleanUpTestContainers=false
+    logFilePaths=/tmp/logPaths
+    find target/ -name "*output.txt" > $logFilePaths
+    logOutput "$(cat $logFilePaths)"
+    rm $logFilePaths
+    exit 0
+    break
+    ;;
+  *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
 
+## SECTION: COMBINATION DEFINITIONS AND TRIGGER
 
-### DO NOT EDIT THE LINES BELOW.
+echo ' '
+echo 'Set the active test combination to run the action:'
+select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
+do
+   case $combo in
+      HNS-OAuth)
+         runHNSOAuthTest
+         break
+         ;;
+      HNS-SharedKey)
+         runHNSSharedKeyTest
+         break
+         ;;
+      nonHNS-SharedKey)
+         runNonHNSSharedKeyTest
+         break
+         ;;
+       AppendBlob-HNS-OAuth)
+         runAppendBlobHNSOAuthTest
+         break
+         ;;
+      AllCombinationsTestRun)
+        if [ $runTest == false ]
+        then
+          logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
+          break
+        fi
+         runHNSOAuthTest
+         runHNSSharedKeyTest
+         runNonHNSSharedKeyTest
+         runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
+         break
+         ;;
+      Quit)
+         exit 0
+         ;;
+      *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
 
-runtests "$@"
+if [ $runTest == true ]
+then
+  printAggregate
+fi
diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
index 1b118ae1e8297..28f96edd27354 100644
--- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
+++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
@@ -15,117 +15,88 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-testresourcesdir=src/test/resources
-combconfsdir=$testresourcesdir/combinationConfigFiles
-combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
+resourceDir=src/test/resources/
+accountSettingsFolderName=accountSettings
+combtestfile=$resourceDir
+combtestfile+=abfs-combination-test-configs.xml
+logdir=dev-support/testlogs/
 
-logdir=dev-support/testlogs
 testresultsregex="Results:(\n|.)*?Tests run:"
-testresultsfilename=
-starttime=
-threadcount=
-defaultthreadcount=8
-
-properties=
-values=
-
-validate() {
-  if [ -z "$threadcount" ] ; then
-    threadcount=$defaultthreadcount
-  fi
-  numberegex='^[0-9]+$'
-  if ! [[ $threadcount =~ $numberegex ]] ; then
-    echo "Exiting. The script param (threadcount) should be a number"
-    exit -1
-  fi
-  if [ -z "$combination" ]; then
-   echo "Exiting. combination cannot be empty"
-   exit -1
-  fi
-  propertiessize=${#properties[@]}
-  valuessize=${#values[@]}
-  if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
-    echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
-    exit -1
-  fi
-
-  for filename in "${combinations[@]}"; do
-    if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
-      echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
-      exit -1
-    fi
-  done
-}
-
-checkdependencies() {
-  if ! [ "$(command -v pcregrep)" ]; then
-    echo "Exiting. pcregrep is required to run the script."
-    exit -1
-  fi
-  if ! [ "$(command -v xmlstarlet)" ]; then
-    echo "Exiting. xmlstarlet is required to run the script."
-    exit -1
+accountConfigFileSuffix="_settings.xml"
+testOutputLogFolder=$logdir
+testlogfilename=combinationTestLogFile
+
+fullRunStartTime=$(date +%s)
+STARTTIME=$(date +%s)
+ENDTIME=$(date +%s)
+
+outputFormatOn="\033[0;95m"
+outputFormatOff="\033[0m"
+
+triggerRun()
+{
+  echo ' '
+  combination=$1
+  accountName=$2
+  runTest=$3
+  processcount=$4
+  cleanUpTestContainers=$5
+
+  if [ -z "$accountName" ]; then
+    logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
+    exit 1;
   fi
-}
-
-cleancombinationconfigs() {
-  rm -rf $combconfsdir
-  mkdir -p $combconfsdir
-}
-
-generateconfigs() {
-  combconffile="$combconfsdir/$combination.xml"
-  rm -rf "$combconffile"
-  cat > "$combconffile" << ENDOFFILE
+  accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
+  rm -rf $combtestfile
+  cat > $combtestfile << ENDOFFILE
 <configuration>
 
 </configuration>
 ENDOFFILE
-
-  propertiessize=${#properties[@]}
-  valuessize=${#values[@]}
+  propertiessize=${#PROPERTIES[@]}
+  valuessize=${#VALUES[@]}
   if [ "$propertiessize" -ne "$valuessize" ]; then
-    echo "Exiting. Number of properties and values differ for $combination"
-    exit -1
+    logOutput "Exiting. Number of properties and values differ for $combination"
+    exit 1
   fi
   for ((i = 0; i < propertiessize; i++)); do
-    key=${properties[$i]}
-    val=${values[$i]}
+    key=${PROPERTIES[$i]}
+    val=${VALUES[$i]}
+    echo "Combination specific property setting: [ key=$key , value=$val ]"
     changeconf "$key" "$val"
   done
-  formatxml "$combconffile"
-}
-
-formatxml() {
-  xmlstarlet fo -s 2 "$1" > "$1.tmp"
-  mv "$1.tmp" "$1"
-}
-
-setactiveconf() {
-  if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
-    echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
-    exit -1
-  fi
-  rm -rf $combtestfile
-  cat > $combtestfile << ENDOFFILE
-<configuration>
-
-</configuration>
-ENDOFFILE
+  formatxml "$combtestfile"
   xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
-  xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
+  xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
   xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
   formatxml $combtestfile
-}
+  echo ' '
+  echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
+  testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
+  touch "$testlogfilename"
 
-changeconf() {
-  xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
-  xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
-  if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
+  if [ "$runTest" == true ]
   then
-    echo "Exiting. Changing config property failed."
-    exit -1
+    STARTTIME=$(date +%s)
+    echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
+    logOutput "Test run report can be seen in $testlogfilename"
+    mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
+    ENDTIME=$(date +%s)
+    summary
   fi
+
+  if [ "$cleanUpTestContainers" == true ]
+  then
+    mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
+    if grep -q "There are test failures" "$testlogfilename";
+    then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
+    pcregrep -M "$testresultsregex" "$testlogfilename"
+    exit 0
+    fi
+
+    logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
+  fi
+
 }
 
 summary() {
@@ -134,108 +105,68 @@ summary() {
     echo "$combination"
     echo "========================"
     pcregrep -M "$testresultsregex" "$testlogfilename"
-  } >> "$testresultsfilename"
+  } >> "$aggregatedTestResult"
   printf "\n----- Test results -----\n"
   pcregrep -M "$testresultsregex" "$testlogfilename"
-
   secondstaken=$((ENDTIME - STARTTIME))
   mins=$((secondstaken / 60))
   secs=$((secondstaken % 60))
   printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
-  echo "Find test logs for the combination ($combination) in: $testlogfilename"
-  echo "Find consolidated test results in: $testresultsfilename"
-  echo "----------"
+  echo "Find test result for the combination ($combination) in: $testlogfilename"
+  logOutput "Consolidated test result is saved in: $aggregatedTestResult"
+  echo "------------------------"
 }
 
-init() {
-  checkdependencies
-  if ! mvn clean install -DskipTests
-  then
-    echo ""
-    echo "Exiting. Build failed."
-    exit -1
+checkdependencies() {
+  if ! [ "$(command -v pcregrep)" ]; then
+    logOutput "Exiting. pcregrep is required to run the script."
+    exit 1
   fi
-  starttime=$(date +"%Y-%m-%d_%H-%M-%S")
-  mkdir -p "$logdir"
-  testresultsfilename="$logdir/$starttime/Test-Results.txt"
-  if [[ -z "$combinations" ]]; then
-    combinations=( $( ls $combconfsdir/*.xml ))
+  if ! [ "$(command -v xmlstarlet)" ]; then
+    logOutput "Exiting. xmlstarlet is required to run the script."
+    exit 1
   fi
 }
 
-runtests() {
-  parseoptions "$@"
-  validate
-  if [ -z "$starttime" ]; then
-    init
-  fi
-  shopt -s nullglob
-  for combconffile in "${combinations[@]}"; do
-    STARTTIME=$(date +%s)
-    combination=$(basename "$combconffile" .xml)
-    mkdir -p "$logdir/$starttime"
-    testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
-    printf "\nRunning the combination: %s..." "$combination"
-    setactiveconf
-    mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
-    ENDTIME=$(date +%s)
-    summary
-  done
+formatxml() {
+  xmlstarlet fo -s 2 "$1" > "$1.tmp"
+  mv "$1.tmp" "$1"
 }
 
-begin() {
-  cleancombinationconfigs
+changeconf() {
+  xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
+  xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
+  if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
+  then
+    logOutput "Exiting. Changing config property failed."
+    exit 1
+  fi
 }
 
-parseoptions() {
-runactivate=0
-runtests=0
-  while getopts ":c:a:t:" option; do
-    case "${option}" in
-      a)
-        if [[ "$runactivate" -eq "1" ]]; then
-          echo "-a Option is not multivalued"
-          exit 1
-        fi
-        runactivate=1
-        combination=$(basename "$OPTARG" .xml)
-        ;;
-      c)
-        runtests=1
-        combination=$(basename "$OPTARG" .xml)
-        combinations+=("$combination")
-        ;;
-      t)
-        threadcount=$OPTARG
-        ;;
-      *|?|h)
-        if [[ -z "$combinations" ]]; then
-          combinations=( $( ls $combconfsdir/*.xml ))
-        fi
-      combstr=""
-        for combconffile in "${combinations[@]}"; do
-          combname=$(basename "$combconffile" .xml)
-          combstr="${combname}, ${combstr}"
-        done
-        combstr=${combstr:0:-2}
-
-        echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
-        echo ""
-        echo "Where:"
-        echo "  -a COMBINATION_NAME   Specify the combination name which needs to be activated."
-        echo "                        Configured combinations: ${combstr}"
-        echo "  -c COMBINATION_NAME   Specify the combination name for test runs"
-        echo "  -t THREAD_COUNT       Specify the thread count"
-        exit 1
-        ;;
-    esac
-  done
-  if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
-    echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
+init() {
+  checkdependencies
+  if ! mvn clean install -DskipTests
+  then
+    echo ""
+    echo "Exiting. Build failed."
     exit 1
   fi
-  if [[ "$runactivate" -eq "1" ]]; then
-        setactiveconf
-        exit 0
-  fi
+  starttime=$(date +"%Y-%m-%d_%H-%M-%S")
+  testOutputLogFolder+=$starttime
+  mkdir -p "$testOutputLogFolder"
+  aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
+ }
+
+ printAggregate() {
+   echo  :::: AGGREGATED TEST RESULT ::::
+   cat "$aggregatedTestResult"
+  fullRunEndTime=$(date +%s)
+  fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
+  mins=$((fullRunTimeInSecs / 60))
+  secs=$((fullRunTimeInSecs % 60))
+  printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
+ }
+
+logOutput() {
+  echo -e "$outputFormatOn" "$1" "$outputFormatOff"
 }
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index df852d5b426ae..c313fa28a3ab4 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -100,38 +100,6 @@
           </execution>
         </executions>
        </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
 
     </plugins>
 
@@ -210,18 +178,6 @@
       <scope>compile</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
-      <scope>compile</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-core-asl</artifactId>
-      <scope>compile</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.wildfly.openssl</groupId>
       <artifactId>wildfly-openssl</artifactId>
@@ -263,6 +219,67 @@
       <scope>test</scope>
     </dependency>
 
+    <!--
+    the mapreduce-client-core module is compiled against for the
+    manifest committer support. It is not needed on the classpath
+    except when using this committer, so it only tagged as
+    "provided".
+    It is not exported as a transitive dependency of the JAR.
+    Applications which wish to to use the manifest committer
+    will need to explicity add the mapreduce JAR to their classpath.
+    This is already done by MapReduce and Spark.
+    -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <!--
+    These are only added to the classpath for tests,
+    and are not exported by the test JAR as transitive
+    dependencies.
+    -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-tests</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-hs</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-examples</artifactId>
+      <scope>test</scope>
+      <type>jar</type>
+    </dependency>
+    <!-- artifacts needed to bring up a Mini MR Yarn cluster-->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-app</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-distcp</artifactId>
@@ -363,7 +380,7 @@
                     <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                     <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                     <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                     <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -394,7 +411,7 @@
                     <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                     <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                     <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                     <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -436,7 +453,7 @@
                     <!-- surefire.forkNumber won't do the parameter -->
                     <!-- substitution.  Putting a prefix in front of it like -->
                     <!-- "fork-" makes it work. -->
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <!-- Propagate scale parameters -->
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
@@ -526,7 +543,7 @@
                     <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                     <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                     <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                     <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -570,7 +587,7 @@
                     <!-- surefire.forkNumber won't do the parameter -->
                     <!-- substitution.  Putting a prefix in front of it like -->
                     <!-- "fork-" makes it work. -->
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <!-- Propagate scale parameters -->
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
@@ -587,7 +604,9 @@
                     <exclude>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</exclude>
                     <exclude>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</exclude>
                     <exclude>**/azurebfs/ITestSmallWriteOptimization.java</exclude>
+                    <exclude>**/azurebfs/ITestAbfsStreamStatistics*.java</exclude>
                     <exclude>**/azurebfs/services/ITestReadBufferManager.java</exclude>
+                    <exclude>**/azurebfs/commit/*.java</exclude>
                   </excludes>
 
                 </configuration>
@@ -616,7 +635,7 @@
                     <!-- surefire.forkNumber won't do the parameter -->
                     <!-- substitution.  Putting a prefix in front of it like -->
                     <!-- "fork-" makes it work. -->
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <!-- Propagate scale parameters -->
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
@@ -629,6 +648,8 @@
                     <include>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</include>
                     <include>**/azurebfs/ITestSmallWriteOptimization.java</include>
                     <include>**/azurebfs/services/ITestReadBufferManager.java</include>
+                    <include>**/azurebfs/ITestAbfsStreamStatistics*.java</include>
+                    <include>**/azurebfs/commit/*.java</include>
                   </includes>
                 </configuration>
               </execution>
@@ -678,7 +699,7 @@
                     <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                     <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                     <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                     <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -708,7 +729,7 @@
                     <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                     <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                     <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
                     <fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -750,7 +771,7 @@
                     <!-- surefire.forkNumber won't do the parameter -->
                     <!-- substitution.  Putting a prefix in front of it like -->
                     <!-- "fork-" makes it work. -->
-                    <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+                    <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
                     <!-- Propagate scale parameters -->
                     <fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
                     <fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
index 070c8c1fe827a..fd2a7c210e706 100644
--- a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
+++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
@@ -48,4 +48,7 @@
               files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]utils[\\/]Base64.java"/>
     <suppress checks="ParameterNumber|VisibilityModifier"
               files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]ITestSmallWriteOptimization.java"/>
+    <!-- allow tests to use _ for ordering. -->
+    <suppress checks="MethodName"
+              files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]commit[\\/]ITestAbfsTerasort.java"/>
 </suppressions>
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 3fc1a9a9fc27c..6e50cfc627e05 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -282,9 +282,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
   // Retry parameter defaults.
   //
 
-  private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 1s
+  private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s
   private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s
-  private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 1s
+  private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s
   private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30;
 
   private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3  * 1000;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index 84f5e3e71acc7..4c8d5fb6a5f71 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -43,7 +43,7 @@
 
 import org.apache.hadoop.fs.impl.StoreImplementationUtils;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.fs.FSExceptionMessages;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
index 923155f4b119f..3c4803cd3df7a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.azure;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 1c2c7aaa27c15..babbf791e9626 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -89,7 +89,7 @@
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Time;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java
index c918518bff24a..75bc99b2cf8d8 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java
@@ -23,7 +23,7 @@
 import java.net.HttpURLConnection;
 import java.util.Map;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
index 32df94223862c..e595b2f4efac0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfs.java
@@ -43,4 +43,13 @@ public class Abfs extends DelegateToFileSystem {
   public int getUriDefaultPort() {
     return -1;
   }
+
+  /**
+   * Close the file system; the FileContext API doesn't have an explicit close.
+   */
+  @Override
+  protected void finalize() throws Throwable {
+    fsImpl.close();
+    super.finalize();
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 5c806e10e73d8..80f803d80dab0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -22,7 +22,7 @@
 import java.lang.reflect.Field;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -117,6 +117,10 @@ public class AbfsConfiguration{
       DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ)
   private boolean optimizeFooterRead;
 
+  @BooleanConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED,
+      DefaultValue = DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED)
+  private boolean accountThrottlingEnabled;
+
   @IntegerConfigurationValidatorAnnotation(ConfigurationKey = AZURE_READ_BUFFER_SIZE,
       MinValue = MIN_BUFFER_SIZE,
       MaxValue = MAX_BUFFER_SIZE,
@@ -260,6 +264,19 @@ public class AbfsConfiguration{
       DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING)
   private boolean enableAutoThrottling;
 
+  @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT,
+      DefaultValue = DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS)
+  private int accountOperationIdleTimeout;
+
+  @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ANALYSIS_PERIOD,
+          DefaultValue = DEFAULT_ANALYSIS_PERIOD_MS)
+  private int analysisPeriod;
+
+  @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT,
+      MinValue = 0,
+      DefaultValue = RATE_LIMIT_DEFAULT)
+  private int rateLimit;
+
   @StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY,
       DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX)
   private String userAgentId;
@@ -297,6 +314,11 @@ public class AbfsConfiguration{
           DefaultValue = DEFAULT_ABFS_LATENCY_TRACK)
   private boolean trackLatency;
 
+  @BooleanConfigurationValidatorAnnotation(
+      ConfigurationKey = FS_AZURE_ENABLE_READAHEAD,
+      DefaultValue = DEFAULT_ENABLE_READAHEAD)
+  private boolean enabledReadAhead;
+
   @LongConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS,
       MinValue = 0,
       DefaultValue = DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS)
@@ -684,6 +706,10 @@ public String getAppendBlobDirs() {
     return this.azureAppendBlobDirs;
   }
 
+  public boolean accountThrottlingEnabled() {
+    return accountThrottlingEnabled;
+  }
+
   public String getAzureInfiniteLeaseDirs() {
     return this.azureInfiniteLeaseDirs;
   }
@@ -726,6 +752,18 @@ public boolean isAutoThrottlingEnabled() {
     return this.enableAutoThrottling;
   }
 
+  public int getAccountOperationIdleTimeout() {
+    return accountOperationIdleTimeout;
+  }
+
+  public int getAnalysisPeriod() {
+    return analysisPeriod;
+  }
+
+  public int getRateLimit() {
+    return rateLimit;
+  }
+
   public String getCustomUserAgentPrefix() {
     return this.userAgentId;
   }
@@ -906,6 +944,15 @@ public SASTokenProvider getSASTokenProvider() throws AzureBlobFileSystemExceptio
     }
   }
 
+  public boolean isReadAheadEnabled() {
+    return this.enabledReadAhead;
+  }
+
+  @VisibleForTesting
+  void setReadAheadEnabled(final boolean enabledReadAhead) {
+    this.enabledReadAhead = enabledReadAhead;
+  }
+
   public int getReadAheadRange() {
     return this.readAheadRange;
   }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java
index c478256e706bc..67ee8e90efb3d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java
@@ -87,7 +87,11 @@ public class AbfsCountersImpl implements AbfsCounters {
       BYTES_RECEIVED,
       READ_THROTTLES,
       WRITE_THROTTLES,
-      SERVER_UNAVAILABLE
+      SERVER_UNAVAILABLE,
+      RENAME_RECOVERY,
+      METADATA_INCOMPLETE_RENAME_FAILURES,
+      RENAME_PATH_ATTEMPTS
+
   };
 
   private static final AbfsStatistic[] DURATION_TRACKER_LIST = {
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java
index bb65b0c902120..3a77e82ffb4fb 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java
@@ -100,7 +100,16 @@ public enum AbfsStatistic {
       AbfsHttpConstants.HTTP_METHOD_PATCH),
   HTTP_POST_REQUEST(StoreStatisticNames.ACTION_HTTP_POST_REQUEST,
       "Time taken to complete a POST request",
-      AbfsHttpConstants.HTTP_METHOD_POST);
+      AbfsHttpConstants.HTTP_METHOD_POST),
+
+  // Rename recovery
+  RENAME_RECOVERY("rename_recovery",
+      "Number of times Rename recoveries happened"),
+  METADATA_INCOMPLETE_RENAME_FAILURES("metadata_incomplete_rename_failures",
+      "Number of times rename operation failed due to metadata being "
+          + "incomplete"),
+  RENAME_PATH_ATTEMPTS("rename_path_attempts",
+      "Number of times we attempt to rename a path internally");
 
   private String statName;
   private String statDescription;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
index c33265ce32406..ba20bbb5d7668 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/Abfss.java
@@ -43,4 +43,13 @@ public class Abfss extends DelegateToFileSystem {
   public int getUriDefaultPort() {
     return -1;
   }
+
+  /**
+   * Close the file system; the FileContext API doesn't have an explicit close.
+   */
+  @Override
+  protected void finalize() throws Throwable {
+    fsImpl.close();
+    super.finalize();
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index d30129b7a7d68..21501d28f4238 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -27,6 +27,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.AccessDeniedException;
+import java.time.Duration;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.ArrayList;
@@ -42,15 +43,18 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.apache.hadoop.io.IOUtils;
+import javax.annotation.Nullable;
+
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.azurebfs.commit.ResilientCommitByRename;
 import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
-import org.apache.hadoop.fs.azurebfs.services.AbfsClientThrottlingIntercept;
 import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -63,8 +67,10 @@
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
 import org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations;
@@ -79,6 +85,7 @@
 import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
 import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
 import org.apache.hadoop.fs.azurebfs.services.AbfsCounters;
+import org.apache.hadoop.fs.azurebfs.services.AbfsLocatedFileStatus;
 import org.apache.hadoop.fs.azurebfs.utils.Listener;
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
 import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat;
@@ -91,9 +98,12 @@
 import org.apache.hadoop.fs.statistics.IOStatistics;
 import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 import org.apache.hadoop.fs.store.DataBlocks;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.RateLimiting;
+import org.apache.hadoop.util.RateLimitingFactory;
 import org.apache.hadoop.util.functional.RemoteIterators;
 import org.apache.hadoop.util.DurationInfo;
 import org.apache.hadoop.util.LambdaUtils;
@@ -109,6 +119,8 @@
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT;
 import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
 import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
+import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator;
+import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
 
 /**
  * A {@link org.apache.hadoop.fs.FileSystem} for reading and writing files stored on <a
@@ -138,6 +150,9 @@ public class AzureBlobFileSystem extends FileSystem
   /** Maximum Active blocks per OutputStream. */
   private int blockOutputActiveBlocks;
 
+  /** Rate limiting for operations which use it to throttle their IO. */
+  private RateLimiting rateLimiting;
+
   @Override
   public void initialize(URI uri, Configuration configuration)
       throws IOException {
@@ -209,8 +224,7 @@ public void initialize(URI uri, Configuration configuration)
       }
     }
 
-    AbfsClientThrottlingIntercept.initializeSingleton(abfsConfiguration.isAutoThrottlingEnabled());
-
+    rateLimiting = RateLimitingFactory.create(abfsConfiguration.getRateLimit());
     LOG.debug("Initializing AzureBlobFileSystem for {} complete", uri);
   }
 
@@ -256,7 +270,7 @@ private FSDataInputStream open(final Path path,
       InputStream inputStream = abfsStore
           .openFileForRead(qualifiedPath, parameters, statistics, tracingContext);
       return new FSDataInputStream(inputStream);
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       checkException(path, ex);
       return null;
     }
@@ -285,8 +299,13 @@ protected CompletableFuture<FSDataInputStream> openFileWithOptions(
   }
 
   @Override
-  public FSDataOutputStream create(final Path f, final FsPermission permission, final boolean overwrite, final int bufferSize,
-      final short replication, final long blockSize, final Progressable progress) throws IOException {
+  public FSDataOutputStream create(final Path f,
+      final FsPermission permission,
+      final boolean overwrite,
+      final int bufferSize,
+      final short replication,
+      final long blockSize,
+      final Progressable progress) throws IOException {
     LOG.debug("AzureBlobFileSystem.create path: {} permission: {} overwrite: {} bufferSize: {}",
         f,
         permission,
@@ -306,7 +325,7 @@ public FSDataOutputStream create(final Path f, final FsPermission permission, fi
           FsPermission.getUMask(getConf()), tracingContext);
       statIncrement(FILES_CREATED);
       return new FSDataOutputStream(outputStream, statistics);
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       checkException(f, ex);
       return null;
     }
@@ -335,8 +354,12 @@ public FSDataOutputStream createNonRecursive(final Path f, final FsPermission pe
 
   @Override
   @SuppressWarnings("deprecation")
-  public FSDataOutputStream createNonRecursive(final Path f, final FsPermission permission,
-      final EnumSet<CreateFlag> flags, final int bufferSize, final short replication, final long blockSize,
+  public FSDataOutputStream createNonRecursive(final Path f,
+      final FsPermission permission,
+      final EnumSet<CreateFlag> flags,
+      final int bufferSize,
+      final short replication,
+      final long blockSize,
       final Progressable progress) throws IOException {
 
     // Check if file should be appended or overwritten. Assume that the file
@@ -360,7 +383,8 @@ public FSDataOutputStream createNonRecursive(final Path f,
   }
 
   @Override
-  public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress) throws IOException {
+  public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress)
+      throws IOException {
     LOG.debug(
         "AzureBlobFileSystem.append path: {} bufferSize: {}",
         f.toString(),
@@ -375,7 +399,7 @@ public FSDataOutputStream append(final Path f, final int bufferSize, final Progr
       OutputStream outputStream = abfsStore
           .openFileForWrite(qualifiedPath, statistics, false, tracingContext);
       return new FSDataOutputStream(outputStream, statistics);
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       checkException(f, ex);
       return null;
     }
@@ -398,7 +422,7 @@ public boolean rename(final Path src, final Path dst) throws IOException {
         fileSystemId, FSOperationType.RENAME, true, tracingHeaderFormat,
         listener);
     // rename under same folder;
-    if(makeQualified(parentFolder).equals(qualifiedDstPath)) {
+    if (makeQualified(parentFolder).equals(qualifiedDstPath)) {
       return tryGetFileStatus(qualifiedSrcPath, tracingContext) != null;
     }
 
@@ -433,24 +457,99 @@ public boolean rename(final Path src, final Path dst) throws IOException {
 
       qualifiedDstPath = makeQualified(adjustedDst);
 
-      abfsStore.rename(qualifiedSrcPath, qualifiedDstPath, tracingContext);
+      abfsStore.rename(qualifiedSrcPath, qualifiedDstPath, tracingContext, null);
       return true;
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       LOG.debug("Rename operation failed. ", ex);
       checkException(
-              src,
-              ex,
-              AzureServiceErrorCode.PATH_ALREADY_EXISTS,
-              AzureServiceErrorCode.INVALID_RENAME_SOURCE_PATH,
-              AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND,
-              AzureServiceErrorCode.INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE,
-              AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND,
-              AzureServiceErrorCode.INTERNAL_OPERATION_ABORT);
+          src,
+          ex,
+          AzureServiceErrorCode.PATH_ALREADY_EXISTS,
+          AzureServiceErrorCode.INVALID_RENAME_SOURCE_PATH,
+          AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND,
+          AzureServiceErrorCode.INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE,
+          AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND,
+          AzureServiceErrorCode.INTERNAL_OPERATION_ABORT);
       return false;
     }
 
   }
 
+  /**
+   * Private method to create resilient commit support.
+   * @return a new instance
+   * @param path destination path
+   * @throws IOException problem probing store capabilities
+   * @throws UnsupportedOperationException if the store lacks this support
+   */
+  @InterfaceAudience.Private
+  public ResilientCommitByRename createResilientCommitSupport(final Path path)
+      throws IOException {
+
+    if (!hasPathCapability(path,
+        CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME)) {
+      throw new UnsupportedOperationException(
+          "Resilient commit support not available for " + path);
+    }
+    return new ResilientCommitByRenameImpl();
+  }
+
+  /**
+   * Resilient commit support.
+   * Provided as a nested class to avoid contaminating the
+   * FS instance with too many private methods which end up
+   * being used widely (as has happened to the S3A FS)
+   */
+  public class ResilientCommitByRenameImpl implements ResilientCommitByRename {
+
+    /**
+     * Perform the rename.
+     * This will be rate limited, as well as able to recover
+     * from rename errors if the etag was passed in.
+     * @param source path to source file
+     * @param dest destination of rename.
+     * @param sourceEtag etag of source file. may be null or empty
+     * @return the outcome of the operation
+     * @throws IOException any rename failure which was not recovered from.
+     */
+    public Pair<Boolean, Duration> commitSingleFileByRename(
+        final Path source,
+        final Path dest,
+        @Nullable final String sourceEtag) throws IOException {
+
+      LOG.debug("renameFileWithEtag source: {} dest: {} etag {}", source, dest, sourceEtag);
+      statIncrement(CALL_RENAME);
+
+      trailingPeriodCheck(dest);
+      Path qualifiedSrcPath = makeQualified(source);
+      Path qualifiedDstPath = makeQualified(dest);
+
+      TracingContext tracingContext = new TracingContext(clientCorrelationId,
+          fileSystemId, FSOperationType.RENAME, true, tracingHeaderFormat,
+          listener);
+
+      if (qualifiedSrcPath.equals(qualifiedDstPath)) {
+        // rename to itself is forbidden
+        throw new PathIOException(qualifiedSrcPath.toString(), "cannot rename object onto self");
+      }
+
+      // acquire one IO permit
+      final Duration waitTime = rateLimiting.acquire(1);
+
+      try {
+        final boolean recovered = abfsStore.rename(qualifiedSrcPath,
+            qualifiedDstPath, tracingContext, sourceEtag);
+        return Pair.of(recovered, waitTime);
+      } catch (AzureBlobFileSystemException ex) {
+        LOG.debug("Rename operation failed. ", ex);
+        checkException(source, ex);
+        // never reached
+        return null;
+      }
+
+    }
+  }
+
   @Override
   public boolean delete(final Path f, final boolean recursive) throws IOException {
     LOG.debug(
@@ -528,7 +627,7 @@ private void incrementStatistic(AbfsStatistic statistic) {
    * @throws IllegalArgumentException if the path has a trailing period (.)
    */
   private void trailingPeriodCheck(Path path) throws IllegalArgumentException {
-    while (!path.isRoot()){
+    while (!path.isRoot()) {
       String pathToString = path.toString();
       if (pathToString.length() != 0) {
         if (pathToString.charAt(pathToString.length() - 1) == '.') {
@@ -536,8 +635,7 @@ private void trailingPeriodCheck(Path path) throws IllegalArgumentException {
               "ABFS does not allow files or directories to end with a dot.");
         }
         path = path.getParent();
-      }
-      else {
+      } else {
         break;
       }
     }
@@ -596,10 +694,10 @@ public synchronized void close() throws IOException {
 
   @Override
   public FileStatus getFileStatus(final Path f) throws IOException {
-      TracingContext tracingContext = new TracingContext(clientCorrelationId,
-          fileSystemId, FSOperationType.GET_FILESTATUS, tracingHeaderFormat,
-          listener);
-      return getFileStatus(f, tracingContext);
+    TracingContext tracingContext = new TracingContext(clientCorrelationId,
+        fileSystemId, FSOperationType.GET_FILESTATUS, tracingHeaderFormat,
+        listener);
+    return getFileStatus(f, tracingContext);
   }
 
   private FileStatus getFileStatus(final Path path,
@@ -610,7 +708,7 @@ private FileStatus getFileStatus(final Path path,
 
     try {
       return abfsStore.getFileStatus(qualifiedPath, tracingContext);
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       checkException(path, ex);
       return null;
     }
@@ -634,7 +732,7 @@ public void breakLease(final Path f) throws IOException {
           fileSystemId, FSOperationType.BREAK_LEASE, tracingHeaderFormat,
           listener);
       abfsStore.breakLease(qualifiedPath, tracingContext);
-    } catch(AzureBlobFileSystemException ex) {
+    } catch (AzureBlobFileSystemException ex) {
       checkException(f, ex);
     }
   }
@@ -661,7 +759,6 @@ public Path makeQualified(Path path) {
     return super.makeQualified(path);
   }
 
-
   @Override
   public Path getWorkingDirectory() {
     return this.workingDir;
@@ -684,8 +781,8 @@ public String getScheme() {
   @Override
   public Path getHomeDirectory() {
     return makeQualified(new Path(
-            FileSystemConfigurations.USER_HOME_DIRECTORY_PREFIX
-                + "/" + abfsStore.getUser()));
+        FileSystemConfigurations.USER_HOME_DIRECTORY_PREFIX
+            + "/" + abfsStore.getUser()));
   }
 
   /**
@@ -709,8 +806,8 @@ public BlockLocation[] getFileBlockLocations(FileStatus file,
     }
     final String blobLocationHost = abfsStore.getAbfsConfiguration().getAzureBlockLocationHost();
 
-    final String[] name = { blobLocationHost };
-    final String[] host = { blobLocationHost };
+    final String[] name = {blobLocationHost};
+    final String[] host = {blobLocationHost};
     long blockSize = file.getBlockSize();
     if (blockSize <= 0) {
       throw new IllegalArgumentException(
@@ -785,15 +882,14 @@ public Void call() throws Exception {
           }
         });
       }
-    }
-    finally {
+    } finally {
       executorService.shutdownNow();
     }
 
     return true;
   }
 
-   /**
+  /**
    * Set owner of a path (i.e. a file or a directory).
    * The parameters owner and group cannot both be null.
    *
@@ -823,9 +919,9 @@ public void setOwner(final Path path, final String owner, final String group)
 
     try {
       abfsStore.setOwner(qualifiedPath,
-              owner,
-              group,
-              tracingContext);
+          owner,
+          group,
+          tracingContext);
     } catch (AzureBlobFileSystemException ex) {
       checkException(path, ex);
     }
@@ -842,7 +938,10 @@ public void setOwner(final Path path, final String owner, final String group)
    * @throws IllegalArgumentException If name is null or empty or if value is null
    */
   @Override
-  public void setXAttr(final Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag)
+  public void setXAttr(final Path path,
+      final String name,
+      final byte[] value,
+      final EnumSet<XAttrSetFlag> flag)
       throws IOException {
     LOG.debug("AzureBlobFileSystem.setXAttr path: {}", path);
 
@@ -966,7 +1065,7 @@ public void modifyAclEntries(final Path path, final List<AclEntry> aclSpec)
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "modifyAclEntries is only supported by storage accounts with the "
-          + "hierarchical namespace enabled.");
+              + "hierarchical namespace enabled.");
     }
 
     if (aclSpec == null || aclSpec.isEmpty()) {
@@ -1001,7 +1100,7 @@ public void removeAclEntries(final Path path, final List<AclEntry> aclSpec)
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "removeAclEntries is only supported by storage accounts with the "
-          + "hierarchical namespace enabled.");
+              + "hierarchical namespace enabled.");
     }
 
     if (aclSpec == null || aclSpec.isEmpty()) {
@@ -1033,7 +1132,7 @@ public void removeDefaultAcl(final Path path) throws IOException {
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "removeDefaultAcl is only supported by storage accounts with the "
-          + "hierarchical namespace enabled.");
+              + "hierarchical namespace enabled.");
     }
 
     Path qualifiedPath = makeQualified(path);
@@ -1063,7 +1162,7 @@ public void removeAcl(final Path path) throws IOException {
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "removeAcl is only supported by storage accounts with the "
-          + "hierarchical namespace enabled.");
+              + "hierarchical namespace enabled.");
     }
 
     Path qualifiedPath = makeQualified(path);
@@ -1096,7 +1195,7 @@ public void setAcl(final Path path, final List<AclEntry> aclSpec)
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "setAcl is only supported by storage accounts with the hierarchical "
-          + "namespace enabled.");
+              + "namespace enabled.");
     }
 
     if (aclSpec == null || aclSpec.size() == 0) {
@@ -1128,7 +1227,7 @@ public AclStatus getAclStatus(final Path path) throws IOException {
     if (!getIsNamespaceEnabled(tracingContext)) {
       throw new UnsupportedOperationException(
           "getAclStatus is only supported by storage account with the "
-          + "hierarchical namespace enabled.");
+              + "hierarchical namespace enabled.");
     }
 
     Path qualifiedPath = makeQualified(path);
@@ -1188,7 +1287,7 @@ public RemoteIterator<FileStatus> listStatusIterator(Path path)
       TracingContext tracingContext = new TracingContext(clientCorrelationId,
           fileSystemId, FSOperationType.LISTSTATUS, true, tracingHeaderFormat, listener);
       AbfsListStatusRemoteIterator abfsLsItr =
-          new AbfsListStatusRemoteIterator(getFileStatus(path, tracingContext), abfsStore,
+          new AbfsListStatusRemoteIterator(path, abfsStore,
               tracingContext);
       return RemoteIterators.typeCastingRemoteIterator(abfsLsItr);
     } else {
@@ -1196,6 +1295,36 @@ public RemoteIterator<FileStatus> listStatusIterator(Path path)
     }
   }
 
+  /**
+   * Incremental listing of located status entries,
+   * preserving etags.
+   * @param path path to list
+   * @param filter a path filter
+   * @return iterator of results.
+   * @throws FileNotFoundException source path not found.
+   * @throws IOException other values.
+   */
+  @Override
+  protected RemoteIterator<LocatedFileStatus> listLocatedStatus(
+      final Path path,
+      final PathFilter filter)
+      throws FileNotFoundException, IOException {
+
+    LOG.debug("AzureBlobFileSystem.listStatusIterator path : {}", path);
+    // get a paged iterator over the source data, filtering out non-matching
+    // entries.
+    final RemoteIterator<FileStatus> sourceEntries = filteringRemoteIterator(
+        listStatusIterator(path),
+        (st) -> filter.accept(st.getPath()));
+    // and then map that to a remote iterator of located file status
+    // entries, propagating any etags.
+    return mappingRemoteIterator(sourceEntries,
+        st -> new AbfsLocatedFileStatus(st,
+            st.isFile()
+                ? getFileBlockLocations(st, 0, st.getLen())
+                : null));
+  }
+
   private FileStatus tryGetFileStatus(final Path f, TracingContext tracingContext) {
     try {
       return getFileStatus(f, tracingContext);
@@ -1208,7 +1337,7 @@ private FileStatus tryGetFileStatus(final Path f, TracingContext tracingContext)
 
   private boolean fileSystemExists() throws IOException {
     LOG.debug(
-            "AzureBlobFileSystem.fileSystemExists uri: {}", uri);
+        "AzureBlobFileSystem.fileSystemExists uri: {}", uri);
     try {
       TracingContext tracingContext = new TracingContext(clientCorrelationId,
           fileSystemId, FSOperationType.TEST_OP, tracingHeaderFormat, listener);
@@ -1333,9 +1462,9 @@ private void checkCheckAccessException(final Path path,
    * @throws IOException if the exception error code is not on the allowed list.
    */
   @VisibleForTesting
-  static void checkException(final Path path,
-                              final AzureBlobFileSystemException exception,
-                              final AzureServiceErrorCode... allowedErrorCodesList) throws IOException {
+  public static void checkException(final Path path,
+      final AzureBlobFileSystemException exception,
+      final AzureServiceErrorCode... allowedErrorCodesList) throws IOException {
     if (exception instanceof AbfsRestOperationException) {
       AbfsRestOperationException ere = (AbfsRestOperationException) exception;
 
@@ -1445,7 +1574,7 @@ public boolean failed() {
   }
 
   @VisibleForTesting
-  AzureBlobFileSystemStore getAbfsStore() {
+  public AzureBlobFileSystemStore getAbfsStore() {
     return abfsStore;
   }
 
@@ -1498,7 +1627,10 @@ public boolean hasPathCapability(final Path path, final String capability)
     switch (validatePathCapabilityArgs(p, capability)) {
     case CommonPathCapabilities.FS_PERMISSIONS:
     case CommonPathCapabilities.FS_APPEND:
+    case CommonPathCapabilities.ETAGS_AVAILABLE:
       return true;
+
+    case CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME:
     case CommonPathCapabilities.FS_ACLS:
       return getIsNamespaceEnabled(
           new TracingContext(clientCorrelationId, fileSystemId,
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 4fd194b1e01cb..e5e7056126564 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -55,7 +55,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
@@ -65,6 +65,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.EtagSource;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -95,6 +96,7 @@
 import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
 import org.apache.hadoop.fs.azurebfs.services.AbfsClientContext;
 import org.apache.hadoop.fs.azurebfs.services.AbfsClientContextBuilder;
+import org.apache.hadoop.fs.azurebfs.services.AbfsClientRenameResult;
 import org.apache.hadoop.fs.azurebfs.services.AbfsCounters;
 import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
 import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
@@ -130,6 +132,8 @@
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.http.client.utils.URIBuilder;
 
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.METADATA_INCOMPLETE_RENAME_FAILURES;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_RECOVERY;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_EQUALS;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_FORWARD_SLASH;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_HYPHEN;
@@ -764,7 +768,7 @@ public AbfsInputStream openFileForRead(Path path,
         eTag = ((VersionedFileStatus) fileStatus).getVersion();
       } else {
         if (fileStatus != null) {
-          LOG.warn(
+          LOG.debug(
               "Fallback to getPathStatus REST call as provided filestatus "
                   + "is not of type VersionedFileStatus");
         }
@@ -804,6 +808,7 @@ private AbfsInputStreamContext populateAbfsInputStreamContext(
             .withReadBufferSize(abfsConfiguration.getReadBufferSize())
             .withReadAheadQueueDepth(abfsConfiguration.getReadAheadQueueDepth())
             .withTolerateOobAppends(abfsConfiguration.getTolerateOobAppends())
+            .isReadAheadEnabled(abfsConfiguration.isReadAheadEnabled())
             .withReadSmallFilesCompletely(abfsConfiguration.readSmallFilesCompletely())
             .withOptimizeFooterRead(abfsConfiguration.optimizeFooterRead())
             .withReadAheadRange(abfsConfiguration.getReadAheadRange())
@@ -877,7 +882,22 @@ public void breakLease(final Path path, final TracingContext tracingContext) thr
     client.breakLease(getRelativePath(path), tracingContext);
   }
 
-  public void rename(final Path source, final Path destination, TracingContext tracingContext) throws
+  /**
+   * Rename a file or directory.
+   * If a source etag is passed in, the operation will attempt to recover
+   * from a missing source file by probing the destination for
+   * existence and comparing etags.
+   * @param source path to source file
+   * @param destination destination of rename.
+   * @param tracingContext trace context
+   * @param sourceEtag etag of source file. may be null or empty
+   * @throws AzureBlobFileSystemException failure, excluding any recovery from overload failures.
+   * @return true if recovery was needed and succeeded.
+   */
+  public boolean rename(final Path source,
+      final Path destination,
+      final TracingContext tracingContext,
+      final String sourceEtag) throws
           AzureBlobFileSystemException {
     final Instant startAggregate = abfsPerfTracker.getLatencyInstant();
     long countAggregate = 0;
@@ -897,23 +917,30 @@ public void rename(final Path source, final Path destination, TracingContext tra
 
     String sourceRelativePath = getRelativePath(source);
     String destinationRelativePath = getRelativePath(destination);
+    // was any operation recovered from?
+    boolean recovered = false;
 
     do {
       try (AbfsPerfInfo perfInfo = startTracking("rename", "renamePath")) {
-        AbfsRestOperation op = client
-            .renamePath(sourceRelativePath, destinationRelativePath,
-                continuation, tracingContext);
+        final AbfsClientRenameResult abfsClientRenameResult =
+            client.renamePath(sourceRelativePath, destinationRelativePath,
+                continuation, tracingContext, sourceEtag, false);
+
+        AbfsRestOperation op = abfsClientRenameResult.getOp();
         perfInfo.registerResult(op.getResult());
         continuation = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION);
         perfInfo.registerSuccess(true);
         countAggregate++;
         shouldContinue = continuation != null && !continuation.isEmpty();
-
+        // update the recovery flag.
+        recovered |= abfsClientRenameResult.isRenameRecovered();
+        populateRenameRecoveryStatistics(abfsClientRenameResult);
         if (!shouldContinue) {
           perfInfo.registerAggregates(startAggregate, countAggregate);
         }
       }
     } while (shouldContinue);
+    return recovered;
   }
 
   public void delete(final Path path, final boolean recursive,
@@ -975,7 +1002,7 @@ public FileStatus getFileStatus(final Path path,
       final long blockSize = abfsConfiguration.getAzureBlockSize();
       final AbfsHttpOperation result = op.getResult();
 
-      final String eTag = result.getResponseHeader(HttpHeaderConfigurations.ETAG);
+      String eTag = extractEtagHeader(result);
       final String lastModified = result.getResponseHeader(HttpHeaderConfigurations.LAST_MODIFIED);
       final String permissions = result.getResponseHeader((HttpHeaderConfigurations.X_MS_PERMISSIONS));
       final boolean hasAcl = AbfsPermission.isExtendedAcl(permissions);
@@ -1733,10 +1760,27 @@ private AbfsPerfInfo startTracking(String callerName, String calleeName) {
     return new AbfsPerfInfo(abfsPerfTracker, callerName, calleeName);
   }
 
-  private static class VersionedFileStatus extends FileStatus {
-    private final String version;
+  /**
+   * A File status with version info extracted from the etag value returned
+   * in a LIST or HEAD request.
+   * The etag is included in the java serialization.
+   */
+  private static final class VersionedFileStatus extends FileStatus
+      implements EtagSource {
 
-    VersionedFileStatus(
+    /**
+     * The superclass is declared serializable; this subclass can also
+     * be serialized.
+     */
+    private static final long serialVersionUID = -2009013240419749458L;
+
+    /**
+     * The etag of an object.
+     * Not-final so that serialization via reflection will preserve the value.
+     */
+    private String version;
+
+    private VersionedFileStatus(
             final String owner, final String group, final FsPermission fsPermission, final boolean hasAcl,
             final long length, final boolean isdir, final int blockReplication,
             final long blocksize, final long modificationTime, final Path path,
@@ -1797,6 +1841,11 @@ public String getVersion() {
       return this.version;
     }
 
+    @Override
+    public String getEtag() {
+      return getVersion();
+    }
+
     @Override
     public String toString() {
       final StringBuilder sb = new StringBuilder(
@@ -1860,7 +1909,7 @@ public AzureBlobFileSystemStoreBuilder build() {
   }
 
   @VisibleForTesting
-  AbfsClient getClient() {
+  public AbfsClient getClient() {
     return this.client;
   }
 
@@ -1902,4 +1951,45 @@ boolean areLeasesFreed() {
     }
     return true;
   }
+
+  /**
+   * Get the etag header from a response, stripping any quotations.
+   * see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
+   * @param result response to process.
+   * @return the quote-unwrapped etag.
+   */
+  public static String extractEtagHeader(AbfsHttpOperation result) {
+    String etag = result.getResponseHeader(HttpHeaderConfigurations.ETAG);
+    if (etag != null) {
+      // strip out any wrapper "" quotes which come back, for consistency with
+      // list calls
+      if (etag.startsWith("W/\"")) {
+        // Weak etag
+        etag = etag.substring(3);
+      } else if (etag.startsWith("\"")) {
+        // strong etag
+        etag = etag.substring(1);
+      }
+      if (etag.endsWith("\"")) {
+        // trailing quote
+        etag = etag.substring(0, etag.length() - 1);
+      }
+    }
+    return etag;
+  }
+
+  /**
+   * Increment rename recovery based counters in IOStatistics.
+   *
+   * @param abfsClientRenameResult Result of an ABFS rename operation.
+   */
+  private void populateRenameRecoveryStatistics(
+      AbfsClientRenameResult abfsClientRenameResult) {
+    if (abfsClientRenameResult.isRenameRecovered()) {
+      abfsCounters.incrementCounter(RENAME_RECOVERY, 1);
+    }
+    if (abfsClientRenameResult.isIncompleteMetadataState()) {
+      abfsCounters.incrementCounter(METADATA_INCOMPLETE_RENAME_FAILURES, 1);
+    }
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java
new file mode 100644
index 0000000000000..efba9244af700
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.time.Duration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+
+/**
+ * Extension of StoreOperationsThroughFileSystem with ABFS awareness.
+ * Purely for use by jobs committing work through the manifest committer.
+ * The {@link AzureManifestCommitterFactory} will configure
+ * this as the binding to the FS.
+ *
+ * ADLS Gen2 stores support etag-recovery on renames, but not WASB
+ * stores.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce")
+@InterfaceStability.Unstable
+public class AbfsManifestStoreOperations extends
+    ManifestStoreOperationsThroughFileSystem {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbfsManifestStoreOperations.class);
+
+  /**
+   * Classname, which can be declared in jpb configurations.
+   */
+  public static final String NAME = AbfsManifestStoreOperations.class.getName();
+
+  /**
+   * Resilient rename calls; only available on an ADLS Gen2 store.
+   * Will be null after binding if the FS isn't compatible.
+   */
+  private ResilientCommitByRename resilientCommitByRename;
+
+  @Override
+  public AzureBlobFileSystem getFileSystem() {
+    return (AzureBlobFileSystem) super.getFileSystem();
+  }
+
+  /**
+   * Bind to the store.
+   *
+   * @param filesystem FS.
+   * @param path path to work under
+   * @throws IOException binding problems.
+   */
+  @Override
+  public void bindToFileSystem(FileSystem filesystem, Path path) throws IOException {
+    if (!(filesystem instanceof AzureBlobFileSystem)) {
+      throw new PathIOException(path.toString(),
+          "Not an abfs filesystem: " + filesystem.getClass());
+    }
+    super.bindToFileSystem(filesystem, path);
+    try {
+      resilientCommitByRename = getFileSystem().createResilientCommitSupport(path);
+      LOG.debug("Bonded to filesystem with resilient commits under path {}", path);
+    } catch (UnsupportedOperationException e) {
+      LOG.debug("No resilient commit support under path {}", path);
+    }
+  }
+
+  @Override
+  public boolean storePreservesEtagsThroughRenames(final Path path) {
+    return true;
+  }
+
+  /**
+   * Resilient commits available on hierarchical stores.
+   * @return true if the FS can use etags on renames.
+   */
+  @Override
+  public boolean storeSupportsResilientCommit() {
+    return resilientCommitByRename != null;
+  }
+
+  /**
+   * Commit a file through an internal ABFS operation.
+   * If resilient commit is unavailable, invokes the superclass, which
+   * will raise an UnsupportedOperationException
+   * @param entry entry to commit
+   * @return the outcome
+   * @throws IOException any failure in resilient commit.
+   * @throws UnsupportedOperationException if not available.
+   */
+  @Override
+  public CommitFileResult commitFile(final FileEntry entry) throws IOException {
+
+    if (resilientCommitByRename != null) {
+      final Pair<Boolean, Duration> result =
+          resilientCommitByRename.commitSingleFileByRename(
+              entry.getSourcePath(),
+              entry.getDestPath(),
+              entry.getEtag());
+      return CommitFileResult.fromResilientCommit(result.getLeft(),
+          result.getRight());
+    } else {
+      return super.commitFile(entry);
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java
new file mode 100644
index 0000000000000..b760fa7a4ac53
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+
+/**
+ * A Committer for the manifest committer which performs all bindings needed
+ * to work best with abfs.
+ * This includes, at a minimum, switching to the abfs-specific manifest store operations.
+ *
+ * This classname is referenced in configurations, so MUST NOT change.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class AzureManifestCommitterFactory extends ManifestCommitterFactory {
+
+  /**
+   * Classname, which can be declared in job configurations.
+   */
+  public static final String NAME = ManifestCommitterFactory.class.getName();
+
+  @Override
+  public ManifestCommitter createOutputCommitter(final Path outputPath,
+      final TaskAttemptContext context) throws IOException {
+    final Configuration conf = context.getConfiguration();
+    // use ABFS Store operations
+    conf.set(OPT_STORE_OPERATIONS_CLASS,
+        AbfsManifestStoreOperations.NAME);
+    return super.createOutputCommitter(outputPath, context);
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java
new file mode 100644
index 0000000000000..2e91392a661b1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+
+/**
+ * API exclusively for committing files.
+ *
+ * This is only for use by (@link {@link AbfsManifestStoreOperations},
+ * and is intended to be implemented by ABFS.
+ * To ensure that there is no need to add mapreduce JARs to the
+ * classpath just to work with ABFS, this interface
+ * MUST NOT refer to anything in the
+ * {@code org.apache.hadoop.mapreduce} package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface ResilientCommitByRename extends IOStatisticsSource {
+
+  /**
+   * Rename source file to dest path *Exactly*; no subdirectory games here.
+   * if the method does not raise an exception,then
+   * the data at dest is the data which was at source.
+   *
+   * Requirements
+   *
+   * <pre>
+   *   exists(FS, source) else raise FileNotFoundException
+   *   source != dest else raise PathIOException
+   *   not exists(FS, dest)
+   *   isDir(FS, dest.getParent)
+   * </pre>
+   * <ol>
+   *   <li>source != dest else raise PathIOException</li>
+   *   <li>source must exist else raise FileNotFoundException</li>
+   *   <li>source must exist and be a file</li>
+   *   <li>dest must not exist; </li>
+   *   <li>dest.getParent() must be a dir</li>
+   *   <li>if sourceEtag is non-empty, it MAY be used to qualify/validate the rename.</li>
+   * </ol>
+   *
+   * The outcome of the operation is undefined if source is not a file, dest exists,
+   * dest.getParent() doesn't exist/is a file.
+   * That is: implementations SHOULD assume that the code calling this method has
+   * set up the destination directory tree and is only invoking this call on a file.
+   * Accordingly: <i>implementations MAY skip validation checks</i>
+   *
+   * Post Conditions on a successful operation:
+   * <pre>
+   * FS' where:
+   *     not exists(FS', source)
+   *     and exists(FS', dest)
+   *     and data(FS', dest) == data (FS, source)
+   * </pre>
+   * This is exactly the same outcome as `FileSystem.rename()` when the same preconditions
+   * are met. This API call simply restricts the operation to file rename with strict
+   * conditions, (no need to be 'clever' about dest path calculation) and the ability
+   * to pass in etags, modtimes and file status values.
+   *
+   * @param source path to source file
+   * @param dest destination of rename.
+   * @param sourceEtag etag of source file. may be null or empty
+   * @return true if recovery was needed.
+   * @throws FileNotFoundException source file not found
+   * @throws PathIOException failure, including source and dest being the same path
+   * @throws IOException any other exception
+   */
+  Pair<Boolean, Duration> commitSingleFileByRename(
+      Path source,
+      Path dest,
+      @Nullable String sourceEtag) throws IOException;
+
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
new file mode 100644
index 0000000000000..3567377350d6b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Support for manifest committer.
+ * Unless otherwise stated: classes are private.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 12beb5a9bbabe..a59f76b6d0fe0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -38,6 +38,7 @@ public final class ConfigurationKeys {
   public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key";
   public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)";
   public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode";
+  public static final String FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = "fs.azure.account.throttling.enabled";
 
   // Retry strategy defined by the user
   public static final String AZURE_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval";
@@ -116,6 +117,8 @@ public final class ConfigurationKeys {
   public static final String AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION = "fs.azure.createRemoteFileSystemDuringInitialization";
   public static final String AZURE_SKIP_USER_GROUP_METADATA_DURING_INITIALIZATION = "fs.azure.skipUserGroupMetadataDuringInitialization";
   public static final String FS_AZURE_ENABLE_AUTOTHROTTLING = "fs.azure.enable.autothrottling";
+  public static final String FS_AZURE_ACCOUNT_OPERATION_IDLE_TIMEOUT = "fs.azure.account.operation.idle.timeout";
+  public static final String FS_AZURE_ANALYSIS_PERIOD = "fs.azure.analysis.period";
   public static final String FS_AZURE_ALWAYS_USE_HTTPS = "fs.azure.always.use.https";
   public static final String FS_AZURE_ATOMIC_RENAME_KEY = "fs.azure.atomic.rename.key";
   /** This config ensures that during create overwrite an existing file will be
@@ -186,6 +189,13 @@ public final class ConfigurationKeys {
   public static final String FS_AZURE_SKIP_SUPER_USER_REPLACEMENT = "fs.azure.identity.transformer.skip.superuser.replacement";
   public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER = "fs.azure.account.keyprovider";
   public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
+
+  /**
+   * Enable or disable readahead buffer in AbfsInputStream.
+   * Value: {@value}.
+   */
+  public static final String FS_AZURE_ENABLE_READAHEAD = "fs.azure.enable.readahead";
+
   /** Setting this true will make the driver use it's own RemoteIterator implementation */
   public static final String FS_AZURE_ENABLE_ABFS_LIST_ITERATOR = "fs.azure.enable.abfslistiterator";
   /** Server side encryption key */
@@ -220,6 +230,9 @@ public final class ConfigurationKeys {
   /** Key for enabling the tracking of ABFS API latency and sending the latency numbers to the ABFS API service */
   public static final String FS_AZURE_ABFS_LATENCY_TRACK = "fs.azure.abfs.latency.track";
 
+  /** Key for rate limit capacity, as used by IO operations which try to throttle themselves. */
+  public static final String FS_AZURE_ABFS_IO_RATE_LIMIT = "fs.azure.io.rate.limit";
+
   public static String accountProperty(String property, String account) {
     return property + "." + account;
   }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index f58c61e8908a6..0ea2c929800a3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -94,6 +94,9 @@ public final class FileSystemConfigurations {
   public static final boolean DEFAULT_ENABLE_FLUSH = true;
   public static final boolean DEFAULT_DISABLE_OUTPUTSTREAM_FLUSH = true;
   public static final boolean DEFAULT_ENABLE_AUTOTHROTTLING = true;
+  public static final boolean DEFAULT_FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED = true;
+  public static final int DEFAULT_ACCOUNT_OPERATION_IDLE_TIMEOUT_MS = 60_000;
+  public static final int DEFAULT_ANALYSIS_PERIOD_MS = 10_000;
 
   public static final DelegatingSSLSocketFactory.SSLChannelMode DEFAULT_FS_AZURE_SSL_CHANNEL_MODE
       = DelegatingSSLSocketFactory.SSLChannelMode.Default;
@@ -106,6 +109,7 @@ public final class FileSystemConfigurations {
   public static final boolean DEFAULT_ABFS_LATENCY_TRACK = false;
   public static final long DEFAULT_SAS_TOKEN_RENEW_PERIOD_FOR_STREAMS_IN_SECONDS = 120;
 
+  public static final boolean DEFAULT_ENABLE_READAHEAD = false;
   public static final String DEFAULT_FS_AZURE_USER_AGENT_PREFIX = EMPTY_STRING;
   public static final String DEFAULT_VALUE_UNKNOWN = "UNKNOWN";
 
@@ -133,5 +137,10 @@ public final class FileSystemConfigurations {
   public static final String DATA_BLOCKS_BUFFER_DEFAULT =
       DATA_BLOCKS_BUFFER_DISK;
 
+  /**
+   * IO rate limit. Value: {@value}
+   */
+  public static final int RATE_LIMIT_DEFAULT = 10_000;
+
   private FileSystemConfigurations() {}
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java
index cdf3decdc98bc..a9883dd2ce5fc 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultEntrySchema.java
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.fs.azurebfs.contracts.services;
 
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java
index e3519fb429bff..dc7da04b5bd4f 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/ListResultSchema.java
@@ -20,8 +20,8 @@
 
 import java.util.List;
 
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
 
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
index 49f90feb22e95..dd4ec7c2009c2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java
@@ -29,10 +29,11 @@
 import java.util.Hashtable;
 import java.util.Map;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.JsonToken;
+import org.apache.hadoop.util.Preconditions;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -451,7 +452,7 @@ private static AzureADToken parseTokenFromStream(
       long expiresOnInSecs = -1;
 
       JsonFactory jf = new JsonFactory();
-      JsonParser jp = jf.createJsonParser(httpResponseStream);
+      JsonParser jp = jf.createParser(httpResponseStream);
       String fieldName, fieldValue;
       jp.nextToken();
       while (jp.hasCurrentToken()) {
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/ClientCredsTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/ClientCredsTokenProvider.java
index d432e3b2ce0c5..1b8a74543712c 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/ClientCredsTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/ClientCredsTokenProvider.java
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
index 14914101e5cca..889976041d46e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
@@ -22,7 +22,8 @@
 import java.io.IOException;
 import java.net.URI;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -138,4 +139,9 @@ public String getUserAgentSuffix() {
     String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, "");
     return suffix != null ? suffix : "";
   }
+
+  @VisibleForTesting
+  protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() {
+    return adaptee;
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java
index dea2f96b60f0a..f9aa1a0d75b88 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/IdentityTransformer.java
@@ -21,7 +21,7 @@
 import java.util.List;
 import java.util.Locale;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/RefreshTokenBasedTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/RefreshTokenBasedTokenProvider.java
index 4f35d1522d99f..a24a4447d3098 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/RefreshTokenBasedTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/RefreshTokenBasedTokenProvider.java
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
index 3d9d8b865a059..e0e483f75b704 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/UserPasswordTokenProvider.java
@@ -19,7 +19,7 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java
index 4108daaa2171b..85d0434a687d0 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/AbfsDelegationTokenManager.java
@@ -24,7 +24,7 @@
 import java.net.URI;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 17d5c03a819ff..25562660ae231 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -28,7 +28,6 @@
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
@@ -39,7 +38,8 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.store.LogExactlyOnce;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
@@ -63,18 +63,21 @@
 import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
 import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
-import org.apache.hadoop.fs.azurebfs.utils.DateTimeUtils;
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS;
+import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME;
 import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.*;
 import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.*;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND;
 
 /**
  * AbfsClient.
@@ -98,9 +101,14 @@ public class AbfsClient implements Closeable {
   private AccessTokenProvider tokenProvider;
   private SASTokenProvider sasTokenProvider;
   private final AbfsCounters abfsCounters;
+  private final AbfsThrottlingIntercept intercept;
 
   private final ListeningScheduledExecutorService executorService;
 
+  /** logging the rename failure if metadata is in an incomplete state. */
+  private static final LogExactlyOnce ABFS_METADATA_INCOMPLETE_RENAME_FAILURE =
+      new LogExactlyOnce(LOG);
+
   private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCredentials,
                     final AbfsConfiguration abfsConfiguration,
                     final AbfsClientContext abfsClientContext)
@@ -113,6 +121,7 @@ private AbfsClient(final URL baseUrl, final SharedKeyCredentials sharedKeyCreden
     this.retryPolicy = abfsClientContext.getExponentialRetryPolicy();
     this.accountName = abfsConfiguration.getAccountName().substring(0, abfsConfiguration.getAccountName().indexOf(AbfsHttpConstants.DOT));
     this.authType = abfsConfiguration.getAuthType(accountName);
+    this.intercept = AbfsThrottlingInterceptFactory.getInstance(accountName, abfsConfiguration);
 
     String encryptionKey = this.abfsConfiguration
         .getClientProvidedEncryptionKey();
@@ -209,6 +218,10 @@ SharedKeyCredentials getSharedKeyCredentials() {
     return sharedKeyCredentials;
   }
 
+  AbfsThrottlingIntercept getIntercept() {
+    return intercept;
+  }
+
   List<AbfsHttpHeader> createDefaultHeaders() {
     final List<AbfsHttpHeader> requestHeaders = new ArrayList<AbfsHttpHeader>();
     requestHeaders.add(new AbfsHttpHeader(X_MS_VERSION, xMsVersion));
@@ -480,8 +493,34 @@ public AbfsRestOperation breakLease(final String path,
     return op;
   }
 
-  public AbfsRestOperation renamePath(String source, final String destination,
-      final String continuation, TracingContext tracingContext)
+
+  /**
+   * Rename a file or directory.
+   * If a source etag is passed in, the operation will attempt to recover
+   * from a missing source file by probing the destination for
+   * existence and comparing etags.
+   * The second value in the result will be true to indicate that this
+   * took place.
+   * As rename recovery is only attempted if the source etag is non-empty,
+   * in normal rename operations rename recovery will never happen.
+   * @param source path to source file
+   * @param destination destination of rename.
+   * @param continuation continuation.
+   * @param tracingContext trace context
+   * @param sourceEtag etag of source file. may be null or empty
+   * @param isMetadataIncompleteState was there a rename failure due to
+   *                                  incomplete metadata state?
+   * @return AbfsClientRenameResult result of rename operation indicating the
+   * AbfsRest operation, rename recovery and incomplete metadata state failure.
+   * @throws AzureBlobFileSystemException failure, excluding any recovery from overload failures.
+   */
+  public AbfsClientRenameResult renamePath(
+      final String source,
+      final String destination,
+      final String continuation,
+      final TracingContext tracingContext,
+      final String sourceEtag,
+      boolean isMetadataIncompleteState)
       throws AzureBlobFileSystemException {
     final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
 
@@ -507,76 +546,109 @@ public AbfsRestOperation renamePath(String source, final String destination,
             HTTP_METHOD_PUT,
             url,
             requestHeaders);
-    Instant renameRequestStartTime = Instant.now();
     try {
+      incrementAbfsRenamePath();
       op.execute(tracingContext);
+      // AbfsClientResult contains the AbfsOperation, If recovery happened or
+      // not, and the incompleteMetaDataState is true or false.
+      // If we successfully rename a path and isMetadataIncompleteState was
+      // true, then rename was recovered, else it didn't, this is why
+      // isMetadataIncompleteState is used for renameRecovery(as the 2nd param).
+      return new AbfsClientRenameResult(op, isMetadataIncompleteState, isMetadataIncompleteState);
     } catch (AzureBlobFileSystemException e) {
         // If we have no HTTP response, throw the original exception.
         if (!op.hasResult()) {
           throw e;
         }
-        final AbfsRestOperation idempotencyOp = renameIdempotencyCheckOp(
-            renameRequestStartTime, op, destination, tracingContext);
-        if (idempotencyOp.getResult().getStatusCode()
-            == op.getResult().getStatusCode()) {
+
+        // ref: HADOOP-18242. Rename failure occurring due to a rare case of
+        // tracking metadata being in incomplete state.
+        if (op.getResult().getStorageErrorCode()
+            .equals(RENAME_DESTINATION_PARENT_PATH_NOT_FOUND.getErrorCode())
+            && !isMetadataIncompleteState) {
+          //Logging
+          ABFS_METADATA_INCOMPLETE_RENAME_FAILURE
+              .info("Rename Failure attempting to resolve tracking metadata state and retrying.");
+
+          // Doing a HEAD call resolves the incomplete metadata state and
+          // then we can retry the rename operation.
+          AbfsRestOperation sourceStatusOp = getPathStatus(source, false,
+              tracingContext);
+          isMetadataIncompleteState = true;
+          // Extract the sourceEtag, using the status Op, and set it
+          // for future rename recovery.
+          AbfsHttpOperation sourceStatusResult = sourceStatusOp.getResult();
+          String sourceEtagAfterFailure = extractEtagHeader(sourceStatusResult);
+          renamePath(source, destination, continuation, tracingContext,
+              sourceEtagAfterFailure, isMetadataIncompleteState);
+        }
+        // if we get out of the condition without a successful rename, then
+        // it isn't metadata incomplete state issue.
+        isMetadataIncompleteState = false;
+
+        boolean etagCheckSucceeded = renameIdempotencyCheckOp(
+            source,
+            sourceEtag, op, destination, tracingContext);
+        if (!etagCheckSucceeded) {
           // idempotency did not return different result
           // throw back the exception
           throw e;
-        } else {
-          return idempotencyOp;
         }
+      return new AbfsClientRenameResult(op, true, isMetadataIncompleteState);
     }
+  }
 
-    return op;
+  private void incrementAbfsRenamePath() {
+    abfsCounters.incrementCounter(RENAME_PATH_ATTEMPTS, 1);
   }
 
   /**
    * Check if the rename request failure is post a retry and if earlier rename
    * request might have succeeded at back-end.
    *
-   * If there is a parallel rename activity happening from any other store
-   * interface, the logic here will detect the rename to have happened due to
-   * the one initiated from this ABFS filesytem instance as it was retried. This
-   * should be a corner case hence going ahead with LMT check.
-   * @param renameRequestStartTime startTime for the rename request
+   * If a source etag was passed in, and the error was 404, get the
+   * etag of any file at the destination.
+   * If it matches the source etag, then the rename is considered
+   * a success.
+   * Exceptions raised in the probe of the destination are swallowed,
+   * so that they do not interfere with the original rename failures.
+   * @param source source path
    * @param op Rename request REST operation response with non-null HTTP response
    * @param destination rename destination path
+   * @param sourceEtag etag of source file. may be null or empty
    * @param tracingContext Tracks identifiers for request header
-   * @return REST operation response post idempotency check
-   * @throws AzureBlobFileSystemException if GetFileStatus hits any exception
+   * @return true if the file was successfully copied
    */
-  public AbfsRestOperation renameIdempotencyCheckOp(
-      final Instant renameRequestStartTime,
+  public boolean renameIdempotencyCheckOp(
+      final String source,
+      final String sourceEtag,
       final AbfsRestOperation op,
       final String destination,
-      TracingContext tracingContext) throws AzureBlobFileSystemException {
+      TracingContext tracingContext) {
     Preconditions.checkArgument(op.hasResult(), "Operations has null HTTP response");
+
     if ((op.isARetriedRequest())
-        && (op.getResult().getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND)) {
+        && (op.getResult().getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND)
+        && isNotEmpty(sourceEtag)) {
+
       // Server has returned HTTP 404, which means rename source no longer
-      // exists. Check on destination status and if it has a recent LMT timestamp.
-      // If yes, return success, else fall back to original rename request failure response.
+      // exists. Check on destination status and if its etag matches
+      // that of the source, consider it to be a success.
+      LOG.debug("rename {} to {} failed, checking etag of destination",
+          source, destination);
 
       try {
         final AbfsRestOperation destStatusOp = getPathStatus(destination,
             false, tracingContext);
-        if (destStatusOp.getResult().getStatusCode()
-            == HttpURLConnection.HTTP_OK) {
-          String lmt = destStatusOp.getResult().getResponseHeader(
-              HttpHeaderConfigurations.LAST_MODIFIED);
-
-          if (DateTimeUtils.isRecentlyModified(lmt, renameRequestStartTime)) {
-            LOG.debug("Returning success response from rename idempotency logic");
-            return destStatusOp;
-          }
-        }
-      } catch (AzureBlobFileSystemException e) {
-        // GetFileStatus on the destination failed, return original op
-        return op;
+        final AbfsHttpOperation result = destStatusOp.getResult();
+
+        return result.getStatusCode() == HttpURLConnection.HTTP_OK
+            && sourceEtag.equals(extractEtagHeader(result));
+      } catch (AzureBlobFileSystemException ignored) {
+        // GetFileStatus on the destination failed, the rename did not take place
       }
     }
-
-    return op;
+    return false;
   }
 
   public AbfsRestOperation append(final String path, final byte[] buffer,
@@ -1064,6 +1136,10 @@ private String appendSASTokenToQuery(String path,
           sasToken = cachedSasToken;
           LOG.trace("Using cached SAS token.");
         }
+        // if SAS Token contains a prefix of ?, it should be removed
+        if (sasToken.charAt(0) == '?') {
+          sasToken = sasToken.substring(1);
+        }
         queryBuilder.setSASToken(sasToken);
         LOG.trace("SAS token fetch complete for {} on {}", operation, path);
       } catch (Exception ex) {
@@ -1207,6 +1283,14 @@ protected AbfsCounters getAbfsCounters() {
     return abfsCounters;
   }
 
+  /**
+   * Getter for abfsConfiguration from AbfsClient.
+   * @return AbfsConfiguration instance
+   */
+  protected AbfsConfiguration getAbfsConfiguration() {
+    return abfsConfiguration;
+  }
+
   public int getNumLeaseThreads() {
     return abfsConfiguration.getNumLeaseThreads();
   }
@@ -1223,4 +1307,9 @@ public ListenableFuture<?> submit(Runnable runnable) {
   public <V> void addCallback(ListenableFuture<V> future, FutureCallback<V> callback) {
     Futures.addCallback(future, callback, executorService);
   }
+
+  @VisibleForTesting
+  protected AccessTokenProvider getTokenProvider() {
+    return tokenProvider;
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientRenameResult.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientRenameResult.java
new file mode 100644
index 0000000000000..86e3473a9fe5d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientRenameResult.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+/**
+ * A class to store the Result of an AbfsClient rename operation, signifying the
+ * AbfsRestOperation result and the rename recovery.
+ */
+public class AbfsClientRenameResult {
+
+  /** Abfs Rest Operation. */
+  private final AbfsRestOperation op;
+  /** Flag indicating recovery took place. */
+  private final boolean renameRecovered;
+  /** Abfs storage tracking metadata is in an incomplete state. */
+  private final boolean isIncompleteMetadataState;
+
+  /**
+   * Constructing an ABFS rename operation result.
+   * @param op The AbfsRestOperation.
+   * @param renameRecovered Did rename recovery took place?
+   * @param isIncompleteMetadataState Did the rename failed due to incomplete
+   *                                 metadata state and had to be retried?
+   */
+  public AbfsClientRenameResult(
+      AbfsRestOperation op,
+      boolean renameRecovered,
+      boolean isIncompleteMetadataState) {
+    this.op = op;
+    this.renameRecovered = renameRecovered;
+    this.isIncompleteMetadataState = isIncompleteMetadataState;
+  }
+
+  public AbfsRestOperation getOp() {
+    return op;
+  }
+
+  public boolean isRenameRecovered() {
+    return renameRecovered;
+  }
+
+  public boolean isIncompleteMetadataState() {
+    return isIncompleteMetadataState;
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingAnalyzer.java
index 4780ce2be753b..f1eb3a2a77476 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingAnalyzer.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingAnalyzer.java
@@ -20,20 +20,23 @@
 
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.util.Time.now;
+
 class AbfsClientThrottlingAnalyzer {
   private static final Logger LOG = LoggerFactory.getLogger(
       AbfsClientThrottlingAnalyzer.class);
-  private static final int DEFAULT_ANALYSIS_PERIOD_MS = 10 * 1000;
   private static final int MIN_ANALYSIS_PERIOD_MS = 1000;
   private static final int MAX_ANALYSIS_PERIOD_MS = 30000;
   private static final double MIN_ACCEPTABLE_ERROR_PERCENTAGE = .1;
@@ -50,42 +53,38 @@ class AbfsClientThrottlingAnalyzer {
   private String name = null;
   private Timer timer = null;
   private AtomicReference<AbfsOperationMetrics> blobMetrics = null;
+  private AtomicLong lastExecutionTime = null;
+  private final AtomicBoolean isOperationOnAccountIdle = new AtomicBoolean(false);
+  private AbfsConfiguration abfsConfiguration = null;
+  private boolean accountLevelThrottlingEnabled = true;
 
   private AbfsClientThrottlingAnalyzer() {
     // hide default constructor
   }
 
-  /**
-   * Creates an instance of the <code>AbfsClientThrottlingAnalyzer</code> class with
-   * the specified name.
-   *
-   * @param name a name used to identify this instance.
-   * @throws IllegalArgumentException if name is null or empty.
-   */
-  AbfsClientThrottlingAnalyzer(String name) throws IllegalArgumentException {
-    this(name, DEFAULT_ANALYSIS_PERIOD_MS);
-  }
-
   /**
    * Creates an instance of the <code>AbfsClientThrottlingAnalyzer</code> class with
    * the specified name and period.
    *
    * @param name   A name used to identify this instance.
-   * @param period The frequency, in milliseconds, at which metrics are
-   *               analyzed.
+   * @param abfsConfiguration The configuration set.
    * @throws IllegalArgumentException If name is null or empty.
    *                                  If period is less than 1000 or greater than 30000 milliseconds.
    */
-  AbfsClientThrottlingAnalyzer(String name, int period)
+  AbfsClientThrottlingAnalyzer(String name, AbfsConfiguration abfsConfiguration)
       throws IllegalArgumentException {
     Preconditions.checkArgument(
         StringUtils.isNotEmpty(name),
         "The argument 'name' cannot be null or empty.");
+    int period = abfsConfiguration.getAnalysisPeriod();
     Preconditions.checkArgument(
         period >= MIN_ANALYSIS_PERIOD_MS && period <= MAX_ANALYSIS_PERIOD_MS,
         "The argument 'period' must be between 1000 and 30000.");
     this.name = name;
-    this.analysisPeriodMs = period;
+    this.abfsConfiguration = abfsConfiguration;
+    this.accountLevelThrottlingEnabled = abfsConfiguration.accountThrottlingEnabled();
+    this.analysisPeriodMs = abfsConfiguration.getAnalysisPeriod();
+    this.lastExecutionTime = new AtomicLong(now());
     this.blobMetrics = new AtomicReference<AbfsOperationMetrics>(
         new AbfsOperationMetrics(System.currentTimeMillis()));
     this.timer = new Timer(
@@ -95,6 +94,47 @@ private AbfsClientThrottlingAnalyzer() {
         analysisPeriodMs);
   }
 
+  /**
+   * Resumes the timer if it was stopped.
+   */
+  private void resumeTimer() {
+    blobMetrics = new AtomicReference<AbfsOperationMetrics>(
+            new AbfsOperationMetrics(System.currentTimeMillis()));
+    timer.schedule(new TimerTaskImpl(),
+            analysisPeriodMs,
+            analysisPeriodMs);
+    isOperationOnAccountIdle.set(false);
+  }
+
+  /**
+   * Synchronized method to suspend or resume timer.
+   * @param timerFunctionality resume or suspend.
+   * @param timerTask The timertask object.
+   * @return true or false.
+   */
+  private synchronized boolean timerOrchestrator(TimerFunctionality timerFunctionality,
+      TimerTask timerTask) {
+    switch (timerFunctionality) {
+    case RESUME:
+      if (isOperationOnAccountIdle.get()) {
+        resumeTimer();
+      }
+      break;
+    case SUSPEND:
+      if (accountLevelThrottlingEnabled && (System.currentTimeMillis()
+          - lastExecutionTime.get() >= getOperationIdleTimeout())) {
+        isOperationOnAccountIdle.set(true);
+        timerTask.cancel();
+        timer.purge();
+        return true;
+      }
+      break;
+    default:
+      break;
+    }
+    return false;
+  }
+
   /**
    * Updates metrics with results from the current storage operation.
    *
@@ -104,12 +144,13 @@ private AbfsClientThrottlingAnalyzer() {
   public void addBytesTransferred(long count, boolean isFailedOperation) {
     AbfsOperationMetrics metrics = blobMetrics.get();
     if (isFailedOperation) {
-      metrics.bytesFailed.addAndGet(count);
-      metrics.operationsFailed.incrementAndGet();
+      metrics.addBytesFailed(count);
+      metrics.incrementOperationsFailed();
     } else {
-      metrics.bytesSuccessful.addAndGet(count);
-      metrics.operationsSuccessful.incrementAndGet();
+      metrics.addBytesSuccessful(count);
+      metrics.incrementOperationsSuccessful();
     }
+    blobMetrics.set(metrics);
   }
 
   /**
@@ -117,6 +158,8 @@ public void addBytesTransferred(long count, boolean isFailedOperation) {
    * @return true if Thread sleeps(Throttling occurs) else false.
    */
   public boolean suspendIfNecessary() {
+    lastExecutionTime.set(now());
+    timerOrchestrator(TimerFunctionality.RESUME, null);
     int duration = sleepDuration;
     if (duration > 0) {
       try {
@@ -134,19 +177,27 @@ int getSleepDuration() {
     return sleepDuration;
   }
 
+  int getOperationIdleTimeout() {
+    return abfsConfiguration.getAccountOperationIdleTimeout();
+  }
+
+  AtomicBoolean getIsOperationOnAccountIdle() {
+    return isOperationOnAccountIdle;
+  }
+
   private int analyzeMetricsAndUpdateSleepDuration(AbfsOperationMetrics metrics,
                                                    int sleepDuration) {
     final double percentageConversionFactor = 100;
-    double bytesFailed = metrics.bytesFailed.get();
-    double bytesSuccessful = metrics.bytesSuccessful.get();
-    double operationsFailed = metrics.operationsFailed.get();
-    double operationsSuccessful = metrics.operationsSuccessful.get();
+    double bytesFailed = metrics.getBytesFailed().get();
+    double bytesSuccessful = metrics.getBytesSuccessful().get();
+    double operationsFailed = metrics.getOperationsFailed().get();
+    double operationsSuccessful = metrics.getOperationsSuccessful().get();
     double errorPercentage = (bytesFailed <= 0)
         ? 0
         : (percentageConversionFactor
         * bytesFailed
         / (bytesFailed + bytesSuccessful));
-    long periodMs = metrics.endTime - metrics.startTime;
+    long periodMs = metrics.getEndTime() - metrics.getStartTime();
 
     double newSleepDuration;
 
@@ -238,10 +289,13 @@ public void run() {
         }
 
         long now = System.currentTimeMillis();
-        if (now - blobMetrics.get().startTime >= analysisPeriodMs) {
+        if (timerOrchestrator(TimerFunctionality.SUSPEND, this)) {
+          return;
+        }
+        if (now - blobMetrics.get().getStartTime() >= analysisPeriodMs) {
           AbfsOperationMetrics oldMetrics = blobMetrics.getAndSet(
               new AbfsOperationMetrics(now));
-          oldMetrics.endTime = now;
+          oldMetrics.setEndTime(now);
           sleepDuration = analyzeMetricsAndUpdateSleepDuration(oldMetrics,
               sleepDuration);
         }
@@ -252,24 +306,4 @@ public void run() {
       }
     }
   }
-
-  /**
-   * Stores Abfs operation metrics during each analysis period.
-   */
-  static class AbfsOperationMetrics {
-    private AtomicLong bytesFailed;
-    private AtomicLong bytesSuccessful;
-    private AtomicLong operationsFailed;
-    private AtomicLong operationsSuccessful;
-    private long endTime;
-    private long startTime;
-
-    AbfsOperationMetrics(long startTime) {
-      this.startTime = startTime;
-      this.bytesFailed = new AtomicLong();
-      this.bytesSuccessful = new AtomicLong();
-      this.operationsFailed = new AtomicLong();
-      this.operationsSuccessful = new AtomicLong();
-    }
-  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
index 7303e833418db..52a46bc746934 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientThrottlingIntercept.java
@@ -19,10 +19,12 @@
 package org.apache.hadoop.fs.azurebfs.services;
 
 import java.net.HttpURLConnection;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.azurebfs.AbfsStatistic;
 import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
 
@@ -38,35 +40,89 @@
  * and sleeps just enough to minimize errors, allowing optimal ingress and/or
  * egress throughput.
  */
-public final class AbfsClientThrottlingIntercept {
+public final class AbfsClientThrottlingIntercept implements AbfsThrottlingIntercept {
   private static final Logger LOG = LoggerFactory.getLogger(
       AbfsClientThrottlingIntercept.class);
   private static final String RANGE_PREFIX = "bytes=";
-  private static AbfsClientThrottlingIntercept singleton = null;
-  private AbfsClientThrottlingAnalyzer readThrottler = null;
-  private AbfsClientThrottlingAnalyzer writeThrottler = null;
-  private static boolean isAutoThrottlingEnabled = false;
+  private static AbfsClientThrottlingIntercept singleton; // singleton, initialized in static initialization block
+  private static final ReentrantLock LOCK = new ReentrantLock();
+  private final AbfsClientThrottlingAnalyzer readThrottler;
+  private final AbfsClientThrottlingAnalyzer writeThrottler;
+  private final String accountName;
 
   // Hide default constructor
-  private AbfsClientThrottlingIntercept() {
-    readThrottler = new AbfsClientThrottlingAnalyzer("read");
-    writeThrottler = new AbfsClientThrottlingAnalyzer("write");
+  public AbfsClientThrottlingIntercept(String accountName, AbfsConfiguration abfsConfiguration) {
+    this.accountName = accountName;
+    this.readThrottler = setAnalyzer("read " + accountName, abfsConfiguration);
+    this.writeThrottler = setAnalyzer("write " + accountName, abfsConfiguration);
+    LOG.debug("Client-side throttling is enabled for the ABFS file system for the account : {}", accountName);
   }
 
-  public static synchronized void initializeSingleton(boolean enableAutoThrottling) {
-    if (!enableAutoThrottling) {
-      return;
-    }
+  // Hide default constructor
+  private AbfsClientThrottlingIntercept(AbfsConfiguration abfsConfiguration) {
+    //Account name is kept as empty as same instance is shared across all accounts
+    this.accountName = "";
+    this.readThrottler = setAnalyzer("read", abfsConfiguration);
+    this.writeThrottler = setAnalyzer("write", abfsConfiguration);
+    LOG.debug("Client-side throttling is enabled for the ABFS file system using singleton intercept");
+  }
+
+  /**
+   * Sets the analyzer for the intercept.
+   * @param name Name of the analyzer.
+   * @param abfsConfiguration The configuration.
+   * @return AbfsClientThrottlingAnalyzer instance.
+   */
+  private AbfsClientThrottlingAnalyzer setAnalyzer(String name, AbfsConfiguration abfsConfiguration) {
+    return new AbfsClientThrottlingAnalyzer(name, abfsConfiguration);
+  }
+
+  /**
+   * Returns the analyzer for read operations.
+   * @return AbfsClientThrottlingAnalyzer for read.
+   */
+  AbfsClientThrottlingAnalyzer getReadThrottler() {
+    return readThrottler;
+  }
+
+  /**
+   * Returns the analyzer for write operations.
+   * @return AbfsClientThrottlingAnalyzer for write.
+   */
+  AbfsClientThrottlingAnalyzer getWriteThrottler() {
+    return writeThrottler;
+  }
+
+  /**
+   * Creates a singleton object of the AbfsClientThrottlingIntercept.
+   * which is shared across all filesystem instances.
+   * @param abfsConfiguration configuration set.
+   * @return singleton object of intercept.
+   */
+  static AbfsClientThrottlingIntercept initializeSingleton(AbfsConfiguration abfsConfiguration) {
     if (singleton == null) {
-      singleton = new AbfsClientThrottlingIntercept();
-      isAutoThrottlingEnabled = true;
-      LOG.debug("Client-side throttling is enabled for the ABFS file system.");
+      LOCK.lock();
+      try {
+        if (singleton == null) {
+          singleton = new AbfsClientThrottlingIntercept(abfsConfiguration);
+          LOG.debug("Client-side throttling is enabled for the ABFS file system.");
+        }
+      } finally {
+        LOCK.unlock();
+      }
     }
+    return singleton;
   }
 
-  static void updateMetrics(AbfsRestOperationType operationType,
-                            AbfsHttpOperation abfsHttpOperation) {
-    if (!isAutoThrottlingEnabled || abfsHttpOperation == null) {
+  /**
+   * Updates the metrics for successful and failed read and write operations.
+   * @param operationType Only applicable for read and write operations.
+   * @param abfsHttpOperation Used for status code and data transferred.
+   */
+  @Override
+  public void updateMetrics(AbfsRestOperationType operationType,
+      AbfsHttpOperation abfsHttpOperation) {
+    if (abfsHttpOperation == null) {
       return;
     }
 
@@ -82,7 +138,7 @@ static void updateMetrics(AbfsRestOperationType operationType,
       case Append:
         contentLength = abfsHttpOperation.getBytesSent();
         if (contentLength > 0) {
-          singleton.writeThrottler.addBytesTransferred(contentLength,
+          writeThrottler.addBytesTransferred(contentLength,
               isFailedOperation);
         }
         break;
@@ -90,7 +146,7 @@ static void updateMetrics(AbfsRestOperationType operationType,
         String range = abfsHttpOperation.getConnection().getRequestProperty(HttpHeaderConfigurations.RANGE);
         contentLength = getContentLengthIfKnown(range);
         if (contentLength > 0) {
-          singleton.readThrottler.addBytesTransferred(contentLength,
+          readThrottler.addBytesTransferred(contentLength,
               isFailedOperation);
         }
         break;
@@ -104,21 +160,18 @@ static void updateMetrics(AbfsRestOperationType operationType,
    * uses this to suspend the request, if necessary, to minimize errors and
    * maximize throughput.
    */
-  static void sendingRequest(AbfsRestOperationType operationType,
+  @Override
+  public void sendingRequest(AbfsRestOperationType operationType,
       AbfsCounters abfsCounters) {
-    if (!isAutoThrottlingEnabled) {
-      return;
-    }
-
     switch (operationType) {
       case ReadFile:
-        if (singleton.readThrottler.suspendIfNecessary()
+        if (readThrottler.suspendIfNecessary()
             && abfsCounters != null) {
           abfsCounters.incrementCounter(AbfsStatistic.READ_THROTTLES, 1);
         }
         break;
       case Append:
-        if (singleton.writeThrottler.suspendIfNecessary()
+        if (writeThrottler.suspendIfNecessary()
             && abfsCounters != null) {
           abfsCounters.incrementCounter(AbfsStatistic.WRITE_THROTTLES, 1);
         }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
index 6e13d4cec6737..413bf3686898b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java
@@ -30,11 +30,11 @@
 
 import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
 import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.JsonToken;
-import org.codehaus.jackson.map.ObjectMapper;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -467,7 +467,7 @@ private void processStorageErrorResponse() {
         return;
       }
       JsonFactory jf = new JsonFactory();
-      try (JsonParser jp = jf.createJsonParser(stream)) {
+      try (JsonParser jp = jf.createParser(stream)) {
         String fieldName, fieldValue;
         jp.nextToken();  // START_OBJECT - {
         jp.nextToken();  // FIELD_NAME - "error":
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index 42c1f4d4fc0f6..8f12484a55c9d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -26,7 +26,7 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -137,7 +137,7 @@ public AbfsInputStream(
     this.tolerateOobAppends = abfsInputStreamContext.isTolerateOobAppends();
     this.eTag = eTag;
     this.readAheadRange = abfsInputStreamContext.getReadAheadRange();
-    this.readAheadEnabled = true;
+    this.readAheadEnabled = abfsInputStreamContext.isReadAheadEnabled();
     this.alwaysReadBufferSize
         = abfsInputStreamContext.shouldReadBufferSizeAlways();
     this.bufferedPreadDisabled = abfsInputStreamContext
@@ -745,6 +745,11 @@ byte[] getBuffer() {
     return buffer;
   }
 
+  @VisibleForTesting
+  public boolean isReadAheadEnabled() {
+    return readAheadEnabled;
+  }
+
   @VisibleForTesting
   public int getReadAheadRange() {
     return readAheadRange;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java
index 55f01bf15bcf7..f6b330934cf5e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java
@@ -20,7 +20,7 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Class to hold extra input stream configs.
@@ -35,6 +35,8 @@ public class AbfsInputStreamContext extends AbfsStreamContext {
 
   private boolean tolerateOobAppends;
 
+  private boolean isReadAheadEnabled = false;
+
   private boolean alwaysReadBufferSize;
 
   private int readAheadBlockSize;
@@ -72,6 +74,12 @@ public AbfsInputStreamContext withTolerateOobAppends(
     return this;
   }
 
+  public AbfsInputStreamContext isReadAheadEnabled(
+          final boolean isReadAheadEnabled) {
+    this.isReadAheadEnabled = isReadAheadEnabled;
+    return this;
+  }
+
   public AbfsInputStreamContext withReadAheadRange(
           final int readAheadRange) {
     this.readAheadRange = readAheadRange;
@@ -141,6 +149,10 @@ public boolean isTolerateOobAppends() {
     return tolerateOobAppends;
   }
 
+  public boolean isReadAheadEnabled() {
+    return isReadAheadEnabled;
+  }
+
   public int getReadAheadRange() {
     return readAheadRange;
   }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java
new file mode 100644
index 0000000000000..0b63a342936a8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * Class to store listStatus results for AbfsListStatusRemoteIterator. The
+ * results can either be of type Iterator or an exception thrown during the
+ * operation
+ */
+public class AbfsListResult {
+  private IOException listException = null;
+
+  private Iterator<FileStatus> fileStatusIterator
+      = Collections.emptyIterator();
+
+  AbfsListResult(IOException ex) {
+    this.listException = ex;
+  }
+
+  AbfsListResult(Iterator<FileStatus> fileStatusIterator) {
+    this.fileStatusIterator = fileStatusIterator;
+  }
+
+  IOException getListingException() {
+    return listException;
+  }
+
+  Iterator<FileStatus> getFileStatusIterator() {
+    return fileStatusIterator;
+  }
+
+  boolean isFailedListing() {
+    return (listException != null);
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
index 835217f945af5..3fecb1f059115 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java
@@ -27,13 +27,15 @@
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
-import javax.activation.UnsupportedDataTypeException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
 
 public class AbfsListStatusRemoteIterator
@@ -46,9 +48,9 @@ public class AbfsListStatusRemoteIterator
   private static final int MAX_QUEUE_SIZE = 10;
   private static final long POLL_WAIT_TIME_IN_MS = 250;
 
-  private final FileStatus fileStatus;
+  private final Path path;
   private final ListingSupport listingSupport;
-  private final ArrayBlockingQueue<Object> iteratorsQueue;
+  private final ArrayBlockingQueue<AbfsListResult> listResultQueue;
   private final TracingContext tracingContext;
 
   private volatile boolean isAsyncInProgress = false;
@@ -56,13 +58,15 @@ public class AbfsListStatusRemoteIterator
   private String continuation;
   private Iterator<FileStatus> currIterator;
 
-  public AbfsListStatusRemoteIterator(final FileStatus fileStatus,
-      final ListingSupport listingSupport, TracingContext tracingContext) {
-    this.fileStatus = fileStatus;
+  public AbfsListStatusRemoteIterator(final Path path,
+      final ListingSupport listingSupport, TracingContext tracingContext)
+      throws IOException {
+    this.path = path;
     this.listingSupport = listingSupport;
     this.tracingContext = tracingContext;
-    iteratorsQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
+    listResultQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE);
     currIterator = Collections.emptyIterator();
+    addNextBatchIteratorToQueue();
     fetchBatchesAsync();
   }
 
@@ -86,19 +90,17 @@ public FileStatus next() throws IOException {
   private Iterator<FileStatus> getNextIterator() throws IOException {
     fetchBatchesAsync();
     try {
-      Object obj = null;
-      while (obj == null
-          && (!isIterationComplete || !iteratorsQueue.isEmpty())) {
-        obj = iteratorsQueue.poll(POLL_WAIT_TIME_IN_MS, TimeUnit.MILLISECONDS);
+      AbfsListResult listResult = null;
+      while (listResult == null
+          && (!isIterationComplete || !listResultQueue.isEmpty())) {
+        listResult = listResultQueue.poll(POLL_WAIT_TIME_IN_MS, TimeUnit.MILLISECONDS);
       }
-      if (obj == null) {
+      if (listResult == null) {
         return Collections.emptyIterator();
-      } else if (obj instanceof Iterator) {
-        return (Iterator<FileStatus>) obj;
-      } else if (obj instanceof IOException) {
-        throw (IOException) obj;
+      } else if (listResult.isFailedListing()) {
+        throw listResult.getListingException();
       } else {
-        throw new UnsupportedDataTypeException();
+        return listResult.getFileStatusIterator();
       }
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
@@ -122,20 +124,17 @@ private void fetchBatchesAsync() {
 
   private void asyncOp() {
     try {
-      while (!isIterationComplete && iteratorsQueue.size() <= MAX_QUEUE_SIZE) {
+      while (!isIterationComplete && listResultQueue.size() <= MAX_QUEUE_SIZE) {
         addNextBatchIteratorToQueue();
       }
     } catch (IOException ioe) {
       LOG.error("Fetching filestatuses failed", ioe);
       try {
-        iteratorsQueue.put(ioe);
+        listResultQueue.put(new AbfsListResult(ioe));
       } catch (InterruptedException interruptedException) {
         Thread.currentThread().interrupt();
         LOG.error("Thread got interrupted: {}", interruptedException);
       }
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      LOG.error("Thread got interrupted: {}", e);
     } finally {
       synchronized (this) {
         isAsyncInProgress = false;
@@ -143,19 +142,25 @@ private void asyncOp() {
     }
   }
 
-  private void addNextBatchIteratorToQueue()
-      throws IOException, InterruptedException {
+  private synchronized void addNextBatchIteratorToQueue()
+      throws IOException {
     List<FileStatus> fileStatuses = new ArrayList<>();
-    continuation = listingSupport
-        .listStatus(fileStatus.getPath(), null, fileStatuses, FETCH_ALL_FALSE,
-            continuation, tracingContext);
-    if (!fileStatuses.isEmpty()) {
-      iteratorsQueue.put(fileStatuses.iterator());
-    }
-    synchronized (this) {
-      if (continuation == null || continuation.isEmpty()) {
-        isIterationComplete = true;
+    try {
+      try {
+        continuation = listingSupport.listStatus(path, null, fileStatuses,
+            FETCH_ALL_FALSE, continuation, tracingContext);
+      } catch (AbfsRestOperationException ex) {
+        AzureBlobFileSystem.checkException(path, ex);
+      }
+      if (!fileStatuses.isEmpty()) {
+        listResultQueue.put(new AbfsListResult(fileStatuses.iterator()));
       }
+    } catch (InterruptedException ie) {
+      Thread.currentThread().interrupt();
+      LOG.error("Thread interrupted", ie);
+    }
+    if (continuation == null || continuation.isEmpty()) {
+      isIterationComplete = true;
     }
   }
 
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java
new file mode 100644
index 0000000000000..325c929c16a7a
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsLocatedFileStatus.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.util.Preconditions;
+
+/**
+ * {@link LocatedFileStatus} extended to also carry an ETag.
+ */
+public class AbfsLocatedFileStatus extends LocatedFileStatus implements EtagSource {
+
+  private static final long serialVersionUID = -8185960773314341594L;
+
+  /**
+   * etag; may be null.
+   */
+  private final String etag;
+
+  public AbfsLocatedFileStatus(FileStatus status, BlockLocation[] locations) {
+    super(Preconditions.checkNotNull(status), locations);
+    if (status instanceof EtagSource) {
+      this.etag = ((EtagSource) status).getEtag();
+    } else {
+      this.etag = null;
+    }
+  }
+
+  @Override
+  public String getEtag() {
+    return etag;
+  }
+
+  @Override
+  public String toString() {
+    return "AbfsLocatedFileStatus{"
+        + "etag='" + etag + '\'' + "} "
+        + super.toString();
+  }
+  // equals() and hashCode() overridden to avoid FindBugs warning.
+  // Base implementation is equality on Path only, which is still appropriate.
+
+  @Override
+  public boolean equals(Object o) {
+    return super.equals(o);
+  }
+
+  @Override
+  public int hashCode() {
+    return super.hashCode();
+  }
+
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsNoOpThrottlingIntercept.java
similarity index 61%
rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java
rename to hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsNoOpThrottlingIntercept.java
index 5776bd4d34501..6b84e583c337a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsNoOpThrottlingIntercept.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -16,24 +16,22 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.fs.s3a.s3guard;
+package org.apache.hadoop.fs.azurebfs.services;
 
-/**
- * Expirable Metadata abstract class is for storing the field needed for
- * metadata classes in S3Guard that could be expired with TTL.
- */
-public abstract class ExpirableMetadata {
-  private long lastUpdated = 0;
+final class AbfsNoOpThrottlingIntercept implements AbfsThrottlingIntercept {
+
+  public static final AbfsNoOpThrottlingIntercept INSTANCE = new AbfsNoOpThrottlingIntercept();
 
-  public long getLastUpdated() {
-    return lastUpdated;
+  private AbfsNoOpThrottlingIntercept() {
   }
 
-  public void setLastUpdated(long lastUpdated) {
-    this.lastUpdated = lastUpdated;
+  @Override
+  public void updateMetrics(final AbfsRestOperationType operationType,
+      final AbfsHttpOperation abfsHttpOperation) {
   }
 
-  public boolean isExpired(long ttl, long now) {
-    return (lastUpdated + ttl) <= now;
+  @Override
+  public void sendingRequest(final AbfsRestOperationType operationType,
+      final AbfsCounters abfsCounters) {
   }
 }
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOperationMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOperationMetrics.java
new file mode 100644
index 0000000000000..2e53367d39fd2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOperationMetrics.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Stores Abfs operation metrics during each analysis period.
+ */
+class AbfsOperationMetrics {
+
+  /**
+   * No of bytes which could not be transferred due to a failed operation.
+  */
+  private final AtomicLong bytesFailed;
+
+  /**
+   * No of bytes successfully transferred during a successful operation.
+   */
+  private final AtomicLong bytesSuccessful;
+
+  /**
+   * Total no of failed operations.
+   */
+  private final AtomicLong operationsFailed;
+
+  /**
+   * Total no of successful operations.
+   */
+  private final AtomicLong operationsSuccessful;
+
+  /**
+   * Time when collection of metrics ended.
+   */
+  private long endTime;
+
+  /**
+   * Time when the collection of metrics started.
+   */
+  private final long startTime;
+
+  AbfsOperationMetrics(long startTime) {
+    this.startTime = startTime;
+    this.bytesFailed = new AtomicLong();
+    this.bytesSuccessful = new AtomicLong();
+    this.operationsFailed = new AtomicLong();
+    this.operationsSuccessful = new AtomicLong();
+  }
+
+  /**
+   *
+   * @return bytes failed to transfer.
+   */
+  AtomicLong getBytesFailed() {
+    return bytesFailed;
+  }
+
+  /**
+   *
+   * @return bytes successfully transferred.
+   */
+  AtomicLong getBytesSuccessful() {
+    return bytesSuccessful;
+  }
+
+  /**
+   *
+   * @return no of operations failed.
+   */
+  AtomicLong getOperationsFailed() {
+    return operationsFailed;
+  }
+
+  /**
+   *
+   * @return no of successful operations.
+   */
+  AtomicLong getOperationsSuccessful() {
+    return operationsSuccessful;
+  }
+
+  /**
+   *
+   * @return end time of metric collection.
+   */
+  long getEndTime() {
+    return endTime;
+  }
+
+  /**
+   *
+   * @param endTime sets the end time.
+   */
+  void setEndTime(final long endTime) {
+    this.endTime = endTime;
+  }
+
+  /**
+   *
+   * @return start time of metric collection.
+   */
+  long getStartTime() {
+    return startTime;
+  }
+
+  void addBytesFailed(long bytes) {
+    this.getBytesFailed().addAndGet(bytes);
+  }
+
+  void addBytesSuccessful(long bytes) {
+    this.getBytesSuccessful().addAndGet(bytes);
+  }
+
+  void incrementOperationsFailed() {
+    this.getOperationsFailed().incrementAndGet();
+  }
+
+  void incrementOperationsSuccessful() {
+    this.getOperationsSuccessful().incrementAndGet();
+  }
+
+}
+
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index c181417b5c4a8..620616b993fe2 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -27,7 +27,7 @@
 import java.util.UUID;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
 import org.slf4j.Logger;
@@ -58,7 +58,7 @@
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.APPEND_MODE;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.FLUSH_CLOSE_MODE;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters.Mode.FLUSH_MODE;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.util.Preconditions.checkState;
 
 /**
  * The BlobFsOutputStream for Rest AbfsClient.
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
index 74b267d563eb2..00da9b6601338 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java
@@ -45,6 +45,8 @@ public class AbfsRestOperation {
   private final AbfsRestOperationType operationType;
   // Blob FS client, which has the credentials, retry policy, and logs.
   private final AbfsClient client;
+  // Return intercept instance
+  private final AbfsThrottlingIntercept intercept;
   // the HTTP method (PUT, PATCH, POST, GET, HEAD, or DELETE)
   private final String method;
   // full URL including query parameters
@@ -145,6 +147,7 @@ String getSasToken() {
             || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method));
     this.sasToken = sasToken;
     this.abfsCounters = client.getAbfsCounters();
+    this.intercept = client.getIntercept();
   }
 
   /**
@@ -241,7 +244,8 @@ private void completeExecute(TracingContext tracingContext)
    */
   private boolean executeHttpOperation(final int retryCount,
     TracingContext tracingContext) throws AzureBlobFileSystemException {
-    AbfsHttpOperation httpOperation = null;
+    AbfsHttpOperation httpOperation;
+
     try {
       // initialize the HTTP request and open the connection
       httpOperation = new AbfsHttpOperation(url, method, requestHeaders);
@@ -278,8 +282,7 @@ private boolean executeHttpOperation(final int retryCount,
       // dump the headers
       AbfsIoUtils.dumpHeadersToDebugLog("Request Headers",
           httpOperation.getConnection().getRequestProperties());
-      AbfsClientThrottlingIntercept.sendingRequest(operationType, abfsCounters);
-
+      intercept.sendingRequest(operationType, abfsCounters);
       if (hasRequestBody) {
         // HttpUrlConnection requires
         httpOperation.sendRequest(buffer, bufferOffset, bufferLength);
@@ -317,7 +320,7 @@ private boolean executeHttpOperation(final int retryCount,
 
       return false;
     } finally {
-      AbfsClientThrottlingIntercept.updateMetrics(operationType, httpOperation);
+      intercept.updateMetrics(operationType, httpOperation);
     }
 
     LOG.debug("HttpRequest: {}: {}", operationType, httpOperation);
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingIntercept.java
new file mode 100644
index 0000000000000..57b5095bb3219
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingIntercept.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An interface for Abfs Throttling Interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface AbfsThrottlingIntercept {
+
+  /**
+   * Updates the metrics for successful and failed read and write operations.
+   * @param operationType Only applicable for read and write operations.
+   * @param abfsHttpOperation Used for status code and data transferred.
+   */
+  void updateMetrics(AbfsRestOperationType operationType,
+      AbfsHttpOperation abfsHttpOperation);
+
+  /**
+   * Called before the request is sent.  Client-side throttling
+   * uses this to suspend the request, if necessary, to minimize errors and
+   * maximize throughput.
+   * @param operationType Only applicable for read and write operations.
+   * @param abfsCounters Used for counters.
+   */
+  void sendingRequest(AbfsRestOperationType operationType,
+      AbfsCounters abfsCounters);
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingInterceptFactory.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingInterceptFactory.java
new file mode 100644
index 0000000000000..279b7a318caf0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsThrottlingInterceptFactory.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+import org.apache.hadoop.util.WeakReferenceMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class to get an instance of throttling intercept class per account.
+ */
+final class AbfsThrottlingInterceptFactory {
+
+  private AbfsThrottlingInterceptFactory() {
+  }
+
+  private static AbfsConfiguration abfsConfig;
+
+  /**
+   * List of references notified of loss.
+   */
+  private static List<String> lostReferences = new ArrayList<>();
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbfsThrottlingInterceptFactory.class);
+
+  /**
+   * Map which stores instance of ThrottlingIntercept class per account.
+   */
+  private static WeakReferenceMap<String, AbfsThrottlingIntercept>
+      interceptMap = new WeakReferenceMap<>(
+      AbfsThrottlingInterceptFactory::factory,
+      AbfsThrottlingInterceptFactory::referenceLost);
+
+  /**
+   * Returns instance of throttling intercept.
+   * @param accountName Account name.
+   * @return instance of throttling intercept.
+   */
+  private static AbfsClientThrottlingIntercept factory(final String accountName) {
+    return new AbfsClientThrottlingIntercept(accountName, abfsConfig);
+  }
+
+  /**
+   * Reference lost callback.
+   * @param accountName key lost.
+   */
+  private static void referenceLost(String accountName) {
+    lostReferences.add(accountName);
+  }
+
+  /**
+   * Returns an instance of AbfsThrottlingIntercept.
+   *
+   * @param accountName The account for which we need instance of throttling intercept.
+     @param abfsConfiguration The object of abfsconfiguration class.
+    * @return Instance of AbfsThrottlingIntercept.
+   */
+  static synchronized AbfsThrottlingIntercept getInstance(String accountName,
+      AbfsConfiguration abfsConfiguration) {
+    abfsConfig =  abfsConfiguration;
+    AbfsThrottlingIntercept intercept;
+    if (!abfsConfiguration.isAutoThrottlingEnabled()) {
+      return AbfsNoOpThrottlingIntercept.INSTANCE;
+    }
+    // If singleton is enabled use a static instance of the intercept class for all accounts
+    if (!abfsConfiguration.accountThrottlingEnabled()) {
+      intercept = AbfsClientThrottlingIntercept.initializeSingleton(
+          abfsConfiguration);
+    } else {
+      // Return the instance from the map
+      intercept = interceptMap.get(accountName);
+      if (intercept == null) {
+        intercept = new AbfsClientThrottlingIntercept(accountName,
+            abfsConfiguration);
+        interceptMap.put(accountName, intercept);
+      }
+    }
+    return intercept;
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
index bb1ec9e4a3fb5..e3adc59afac5e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
@@ -48,9 +48,15 @@ public String getStorageAccountKey(String accountName, Configuration rawConfig)
       // Validating the key.
       validateStorageAccountKey(key);
     } catch (IllegalAccessException | InvalidConfigurationValueException e) {
-      throw new KeyProviderException("Failure to initialize configuration", e);
+      LOG.debug("Failure to retrieve storage account key for {}", accountName,
+          e);
+      throw new KeyProviderException("Failure to initialize configuration for "
+          + accountName
+          + " key =\"" + key + "\""
+          + ": " + e, e);
     } catch(IOException ioe) {
-      LOG.warn("Unable to get key from credential providers. {}", ioe);
+      LOG.warn("Unable to get key for {} from credential providers. {}",
+          accountName, ioe, ioe);
     }
 
     return key;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
new file mode 100644
index 0000000000000..bf7da69ec4982
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public enum TimerFunctionality {
+  RESUME,
+
+  SUSPEND
+}
+
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java
index 523fd3179405a..c5057417a3d75 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.fs.azurebfs.utils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 
 import java.io.File;
@@ -29,6 +29,7 @@
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.LineIterator;
+import org.apache.hadoop.io.IOUtils;
 
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING;
@@ -189,7 +190,7 @@ private static void loadMap(HashMap<String, String> cache, String fileLocation,
     } catch (ArrayIndexOutOfBoundsException e) {
       LOG.error("Error while parsing mapping file", e);
     } finally {
-      LineIterator.closeQuietly(it);
+      IOUtils.cleanupWithLogger(LOG, it);
     }
   }
 }
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index dfb7f3f42a5cf..31498df17904a 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -676,8 +676,6 @@ cause problems.
 As with all Azure storage services, the Azure Datalake Gen 2 store offers
 a fully consistent view of the store, with complete
 Create, Read, Update, and Delete consistency for data and metadata.
-(Compare and contrast with S3 which only offers Create consistency;
-S3Guard adds CRUD to metadata, but not the underlying data).
 
 ### <a name="performance"></a> Performance and Scalability
 
@@ -769,6 +767,12 @@ Hflush() being the only documented API that can provide persistent data
 transfer, Flush() also attempting to persist buffered data will lead to
 performance issues.
 
+
+### <a name="accountlevelthrottlingoptions"></a> Account level throttling Options
+
+`fs.azure.account.operation.idle.timeout`: This value specifies the time after which the timer for the analyzer (read or
+write) should be paused until no new request is made again. The default value for the same is 60 seconds.
+
 ### <a name="hnscheckconfigoptions"></a> HNS Check Options
 Config `fs.azure.account.hns.enabled` provides an option to specify whether
  the storage account is HNS enabled or not. In case the config is not provided,
@@ -879,6 +883,9 @@ when there are too many writes from the same process.
  tuned with this config considering each queued request holds a buffer. Set
  the value 3 or 4 times the value set for s.azure.write.max.concurrent.requests.
 
+`fs.azure.analysis.period`: The time after which sleep duration is recomputed after analyzing metrics. The default value
+for the same is 10 seconds.
+
 ### <a name="securityconfigoptions"></a> Security Options
 `fs.azure.always.use.https`: Enforces to use HTTPS instead of HTTP when the flag
 is made true. Irrespective of the flag, `AbfsClient` will use HTTPS if the secure
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
index 933f86be3e896..e256fbef546cb 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
@@ -602,26 +602,76 @@ various test combinations, it will:
 2. Run tests for all combinations
 3. Summarize results across all the test combination runs.
 
-As a pre-requisite step, fill config values for test accounts and credentials
-needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
-and rename as `src/test/resources/azure-auth-keys.xml`.
+Below are the pre-requiste steps to follow:
+1. Copy
 
-**To add a new test combination:** Templates for mandatory test combinations
-for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
-If a new one needs to be added, add a combination set within
-`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
-and
-1. Provide a new combination name
-2. Update properties and values array which need to be effective for the test
-combination
-3. Call generateconfigs
+        ./src/test/resources/azure-auth-keys.xml.template
+        TO
+        ./src/test/resources/azure-auth-keys.xml
+  Update account names that should be used in the test run for HNS and non-HNS
+  combinations in the 2 properties present in the xml (account name should be
+  without domain part), namely
+
+    fs.azure.hnsTestAccountName
+    fs.azure.nonHnsTestAccountName
+  azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
+
+```
+XInclude is supported, so for extra security secrets may be
+kept out of the source tree then referenced through an an XInclude element:
+
+      <include xmlns="http://www.w3.org/2001/XInclude"
+        href="/users/self/.secrets/auth-keys.xml" />
+```
+
+2. Create account config files (one config file per account) in folder:
+
+        ./src/test/resources/accountSettings/
+   Follow the instruction in the start of the template file
+
+        accountName_settings.xml.template
+   within accountSettings folder while creating account config file.
+   New files created in folder accountSettings is listed in .gitignore to
+   prevent accidental cred leaks.
 
 **To run PR validation:** Running command
-* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
-each of the combinations defined and run tests for all the combinations.
-* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
-combinations can be provided with -c option. If combinations are provided
-with -c option, tests for only those combinations will be run.
+* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
+```bash
+Choose action:
+[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
+1) SET_ACTIVE_TEST_CONFIG               4) SET_OR_CHANGE_TEST_ACCOUNT
+2) RUN_TEST                             5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+3) CLEAN_UP_OLD_TEST_CONTAINERS
+#? 2
+```
+Enter 1: for setting active combination for IDE test run/single mvn test class runs.
+
+Enter 2: for choosing the combination to choose for mvn full test suite.
+
+Enter 3: For clean-up of any abruptly ending test leaving auto generated test
+container on the account.
+
+Enter 4: To create/modify the config file that decides the account to use for specific test combination.
+
+Enter 5: To print the log4j paths the last test runs.
+
+On next prompt, current list of combinations to choose are provided.
+Sample for Run_TEST action:
+```bash
+Enter parallel test run process count [default - 8]: 4
+Set the active test combination to run the action:
+1) HNS-OAuth               3) nonHNS-SharedKey        5) AllCombinationsTestRun
+2) HNS-SharedKey           4) AppendBlob-HNS-OAuth    6) Quit
+#? 1
+
+Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
+
+Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
+Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
+Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
+````
+
+Provide the option for the action chosen first.
 
 **Test logs:** Test runs will create a folder within dev-support/testlogs to
 save the test logs. Folder name will be the test start timestamp. The mvn verify
@@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
 Test-Results.log in the same folder. When run for PR validation, the
 consolidated test results needs to be pasted into the PR comment section.
 
-**To generate config for use in IDE:** Running command with -a (activate) option
-`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
-the effective config relevant for the specific test combination. Hence the same
-config files used by the mvn test runs can be used for IDE without any manual
-updates needed within config file.
-
-**Other command line options:**
-* -a <COMBINATION_NAME> Specify the combination name which needs to be
-activated. This is to be used to generate config for use in IDE.
-* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
-config is specified, tests for only the specified combinations will run. All
-combinations of tests will be running if this config is not specified.
-* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
-are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
-
-In order to test ABFS, please add the following configuration to your
-`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
-compatibility tests which require WASB credentials, in addition to the ABFS
-credentials.
+**To add a new test combination:** Templates for mandatory test combinations
+for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
+If a new one needs to be added, add a combination to
+`dev-support/testrun-scripts/runtests.sh`.
+(Refer to current active combinations within
+`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
+`SECTION: TEST COMBINATION METHODS` in the script).
+
+**Test Configuration Details:**
+
+ Note that the ABFS tests include compatibility tests which require WASB
+ credentials, in addition to the ABFS credentials.
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index 56d553819feae..c5bf85a4f8190 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -37,10 +37,12 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
 import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
 import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
 import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
 import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
 import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -149,6 +151,17 @@ protected boolean getIsNamespaceEnabled(AzureBlobFileSystem fs)
     return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false));
   }
 
+  public static TracingContext getSampleTracingContext(AzureBlobFileSystem fs,
+      boolean needsPrimaryReqId) {
+    String correlationId, fsId;
+    TracingHeaderFormat format;
+    correlationId = "test-corr-id";
+    fsId = "test-filesystem-id";
+    format = TracingHeaderFormat.ALL_ID_FORMAT;
+    return new TracingContext(correlationId, fsId,
+        FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
+  }
+
   public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
       boolean needsPrimaryReqId) {
     String correlationId, fsId;
@@ -167,7 +180,6 @@ public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
         FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
   }
 
-
   @Before
   public void setup() throws Exception {
     //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
@@ -241,6 +253,9 @@ public Hashtable<String, String> call() throws Exception {
     }
   }
 
+  public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
+    return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
+  }
 
   public void loadConfiguredFileSystem() throws Exception {
       // disable auto-creation of filesystem
@@ -271,11 +286,12 @@ protected void createFilesystemForSASTests() throws Exception {
     // The SAS tests do not have permission to create a filesystem
     // so first create temporary instance of the filesystem using SharedKey
     // then re-use the filesystem it creates with SAS auth instead of SharedKey.
-    AzureBlobFileSystem tempFs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig);
-    ContractTestUtils.assertPathExists(tempFs, "This path should exist",
-        new Path("/"));
-    abfsConfig.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.name());
-    usingFilesystemForSASTests = true;
+    try (AzureBlobFileSystem tempFs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig)){
+      ContractTestUtils.assertPathExists(tempFs, "This path should exist",
+          new Path("/"));
+      abfsConfig.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.name());
+      usingFilesystemForSASTests = true;
+    }
   }
 
   public AzureBlobFileSystem getFileSystem() throws IOException {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
index 9e81a0127b6cb..ea1d0e26facec 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java
@@ -21,17 +21,20 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.assertj.core.api.Assertions;
+import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
@@ -39,6 +42,7 @@
 import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator;
 import org.apache.hadoop.fs.azurebfs.services.ListingSupport;
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.apache.hadoop.test.LambdaTestUtils;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
@@ -52,6 +56,8 @@
 public class ITestAbfsListStatusRemoteIterator extends AbstractAbfsIntegrationTest {
 
   private static final int TEST_FILES_NUMBER = 1000;
+  private static final Logger LOG = LoggerFactory.getLogger(
+      ITestAbfsListStatusRemoteIterator.class);
 
   public ITestAbfsListStatusRemoteIterator() throws Exception {
   }
@@ -60,13 +66,11 @@ public ITestAbfsListStatusRemoteIterator() throws Exception {
   public void testAbfsIteratorWithHasNext() throws Exception {
     Path testDir = createTestDirectory();
     setPageSize(10);
-    final List<String> fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER,
-        testDir, "testListPath");
+    final List<String> fileNames = createFilesUnderDirectory(testDir);
 
-    ListingSupport listngSupport = Mockito.spy(getFileSystem().getAbfsStore());
-    RemoteIterator<FileStatus> fsItr = new AbfsListStatusRemoteIterator(
-        getFileSystem().getFileStatus(testDir), listngSupport,
-        getTestTracingContext(getFileSystem(), true));
+    ListingSupport listingSupport = Mockito.spy(getFileSystem().getAbfsStore());
+    RemoteIterator<FileStatus> fsItr = new AbfsListStatusRemoteIterator(testDir,
+        listingSupport, getTestTracingContext(getFileSystem(), true));
     Assertions.assertThat(fsItr)
         .describedAs("RemoteIterator should be instance of "
             + "AbfsListStatusRemoteIterator by default")
@@ -74,20 +78,12 @@ public void testAbfsIteratorWithHasNext() throws Exception {
     int itrCount = 0;
     while (fsItr.hasNext()) {
       FileStatus fileStatus = fsItr.next();
-      String pathStr = fileStatus.getPath().toString();
-      fileNames.remove(pathStr);
+      verifyIteratorResultContent(fileStatus, fileNames);
       itrCount++;
     }
-    Assertions.assertThat(itrCount)
-        .describedAs("Number of iterations should be equal to the files "
-            + "created")
-        .isEqualTo(TEST_FILES_NUMBER);
-    Assertions.assertThat(fileNames.size())
-        .describedAs("After removing every iterm found from the iterator, "
-            + "there should be no more elements in the fileNames")
-        .isEqualTo(0);
-    int minNumberOfInvokations = TEST_FILES_NUMBER / 10;
-    verify(listngSupport, Mockito.atLeast(minNumberOfInvokations))
+    verifyIteratorResultCount(itrCount, fileNames);
+    int minNumberOfInvocations = TEST_FILES_NUMBER / 10;
+    verify(listingSupport, Mockito.atLeast(minNumberOfInvocations))
         .listStatus(any(Path.class), nullable(String.class),
             anyList(), anyBoolean(),
             nullable(String.class),
@@ -98,13 +94,11 @@ public void testAbfsIteratorWithHasNext() throws Exception {
   public void testAbfsIteratorWithoutHasNext() throws Exception {
     Path testDir = createTestDirectory();
     setPageSize(10);
-    final List<String> fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER,
-        testDir, "testListPath");
+    final List<String> fileNames = createFilesUnderDirectory(testDir);
 
-    ListingSupport listngSupport = Mockito.spy(getFileSystem().getAbfsStore());
-    RemoteIterator<FileStatus> fsItr = new AbfsListStatusRemoteIterator(
-        getFileSystem().getFileStatus(testDir), listngSupport,
-        getTestTracingContext(getFileSystem(), true));
+    ListingSupport listingSupport = Mockito.spy(getFileSystem().getAbfsStore());
+    RemoteIterator<FileStatus> fsItr = new AbfsListStatusRemoteIterator(testDir,
+        listingSupport, getTestTracingContext(getFileSystem(), true));
     Assertions.assertThat(fsItr)
         .describedAs("RemoteIterator should be instance of "
             + "AbfsListStatusRemoteIterator by default")
@@ -112,25 +106,13 @@ public void testAbfsIteratorWithoutHasNext() throws Exception {
     int itrCount = 0;
     for (int i = 0; i < TEST_FILES_NUMBER; i++) {
       FileStatus fileStatus = fsItr.next();
-      String pathStr = fileStatus.getPath().toString();
-      fileNames.remove(pathStr);
+      verifyIteratorResultContent(fileStatus, fileNames);
       itrCount++;
     }
-    Assertions.assertThatThrownBy(() -> fsItr.next())
-        .describedAs(
-            "next() should throw NoSuchElementException since next has been "
-                + "called " + TEST_FILES_NUMBER + " times")
-        .isInstanceOf(NoSuchElementException.class);
-    Assertions.assertThat(itrCount)
-        .describedAs("Number of iterations should be equal to the files "
-            + "created")
-        .isEqualTo(TEST_FILES_NUMBER);
-    Assertions.assertThat(fileNames.size())
-        .describedAs("After removing every iterm found from the iterator, "
-            + "there should be no more elements in the fileNames")
-        .isEqualTo(0);
-    int minNumberOfInvokations = TEST_FILES_NUMBER / 10;
-    verify(listngSupport, Mockito.atLeast(minNumberOfInvokations))
+    LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next);
+    verifyIteratorResultCount(itrCount, fileNames);
+    int minNumberOfInvocations = TEST_FILES_NUMBER / 10;
+    verify(listingSupport, Mockito.atLeast(minNumberOfInvocations))
         .listStatus(any(Path.class), nullable(String.class),
             anyList(), anyBoolean(),
             nullable(String.class),
@@ -141,9 +123,8 @@ public void testAbfsIteratorWithoutHasNext() throws Exception {
   public void testWithAbfsIteratorDisabled() throws Exception {
     Path testDir = createTestDirectory();
     setPageSize(10);
-    setEnableAbfsIterator(false);
-    final List<String> fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER,
-        testDir, "testListPath");
+    disableAbfsIterator();
+    final List<String> fileNames = createFilesUnderDirectory(testDir);
 
     RemoteIterator<FileStatus> fsItr =
         getFileSystem().listStatusIterator(testDir);
@@ -154,73 +135,45 @@ public void testWithAbfsIteratorDisabled() throws Exception {
     int itrCount = 0;
     while (fsItr.hasNext()) {
       FileStatus fileStatus = fsItr.next();
-      String pathStr = fileStatus.getPath().toString();
-      fileNames.remove(pathStr);
+      verifyIteratorResultContent(fileStatus, fileNames);
       itrCount++;
     }
-    Assertions.assertThat(itrCount)
-        .describedAs("Number of iterations should be equal to the files "
-            + "created")
-        .isEqualTo(TEST_FILES_NUMBER);
-    Assertions.assertThat(fileNames.size())
-        .describedAs("After removing every iterm found from the iterator, "
-            + "there should be no more elements in the fileNames")
-        .isEqualTo(0);
+    verifyIteratorResultCount(itrCount, fileNames);
   }
 
   @Test
   public void testWithAbfsIteratorDisabledWithoutHasNext() throws Exception {
     Path testDir = createTestDirectory();
     setPageSize(10);
-    setEnableAbfsIterator(false);
-    final List<String> fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER,
-        testDir, "testListPath");
-
-    RemoteIterator<FileStatus> fsItr =
-        getFileSystem().listStatusIterator(testDir);
-    Assertions.assertThat(fsItr)
-        .describedAs("RemoteIterator should not be instance of "
-            + "AbfsListStatusRemoteIterator when it is disabled")
+    disableAbfsIterator();
+    final List<String> fileNames = createFilesUnderDirectory(testDir);
+
+    RemoteIterator<FileStatus> fsItr = getFileSystem().listStatusIterator(
+        testDir);
+    Assertions.assertThat(fsItr).describedAs(
+            "RemoteIterator should not be instance of "
+                + "AbfsListStatusRemoteIterator when it is disabled")
         .isNotInstanceOf(AbfsListStatusRemoteIterator.class);
-    int itrCount = 0;
-    for (int i = 0; i < TEST_FILES_NUMBER; i++) {
+    int itrCount;
+    for (itrCount = 0; itrCount < TEST_FILES_NUMBER; itrCount++) {
       FileStatus fileStatus = fsItr.next();
-      String pathStr = fileStatus.getPath().toString();
-      fileNames.remove(pathStr);
-      itrCount++;
+      verifyIteratorResultContent(fileStatus, fileNames);
     }
-    Assertions.assertThatThrownBy(() -> fsItr.next())
-        .describedAs(
-            "next() should throw NoSuchElementException since next has been "
-                + "called " + TEST_FILES_NUMBER + " times")
-        .isInstanceOf(NoSuchElementException.class);
-    Assertions.assertThat(itrCount)
-        .describedAs("Number of iterations should be equal to the files "
-            + "created")
-        .isEqualTo(TEST_FILES_NUMBER);
-    Assertions.assertThat(fileNames.size())
-        .describedAs("After removing every iterm found from the iterator, "
-            + "there should be no more elements in the fileNames")
-        .isEqualTo(0);
+    LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next);
+    verifyIteratorResultCount(itrCount, fileNames);
   }
 
   @Test
   public void testNextWhenNoMoreElementsPresent() throws Exception {
     Path testDir = createTestDirectory();
     setPageSize(10);
-    RemoteIterator fsItr =
-        new AbfsListStatusRemoteIterator(getFileSystem().getFileStatus(testDir),
-            getFileSystem().getAbfsStore(),
-            getTestTracingContext(getFileSystem(), true));
+    RemoteIterator<FileStatus> fsItr = new AbfsListStatusRemoteIterator(testDir,
+        getFileSystem().getAbfsStore(),
+        getTestTracingContext(getFileSystem(), true));
     fsItr = Mockito.spy(fsItr);
     Mockito.doReturn(false).when(fsItr).hasNext();
 
-    RemoteIterator<FileStatus> finalFsItr = fsItr;
-    Assertions.assertThatThrownBy(() -> finalFsItr.next())
-        .describedAs(
-        "next() should throw NoSuchElementException if hasNext() return "
-            + "false")
-        .isInstanceOf(NoSuchElementException.class);
+    LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next);
   }
 
   @Test
@@ -256,39 +209,47 @@ public void testIOException() throws Exception {
     getFileSystem().mkdirs(testDir);
 
     String exceptionMessage = "test exception";
-    ListingSupport lsSupport =getMockListingSupport(exceptionMessage);
-    RemoteIterator fsItr =
-        new AbfsListStatusRemoteIterator(getFileSystem().getFileStatus(testDir),
-        lsSupport, getTestTracingContext(getFileSystem(), true));
-
-    Assertions.assertThatThrownBy(() -> fsItr.next())
-        .describedAs(
-        "When ioException is not null and queue is empty exception should be "
-            + "thrown")
-        .isInstanceOf(IOException.class)
-        .hasMessage(exceptionMessage);
+    ListingSupport lsSupport = getMockListingSupport(exceptionMessage);
+
+    LambdaTestUtils.intercept(IOException.class,
+        () -> new AbfsListStatusRemoteIterator(testDir, lsSupport,
+            getTestTracingContext(getFileSystem(), true)));
   }
 
   @Test
-  public void testNonExistingPath() throws Throwable {
+  public void testNonExistingPath() throws Exception {
     Path nonExistingDir = new Path("nonExistingPath");
-    Assertions.assertThatThrownBy(
-        () -> getFileSystem().listStatusIterator(nonExistingDir)).describedAs(
-        "test the listStatusIterator call on a path which is not "
-            + "present should result in FileNotFoundException")
-        .isInstanceOf(FileNotFoundException.class);
+    LambdaTestUtils.intercept(FileNotFoundException.class,
+        () -> getFileSystem().listStatusIterator(nonExistingDir));
+  }
+
+  private void verifyIteratorResultContent(FileStatus fileStatus,
+      List<String> fileNames) {
+    String pathStr = fileStatus.getPath().toString();
+    Assert.assertTrue(
+        String.format("Could not remove path %s from filenames %s", pathStr,
+            fileNames), fileNames.remove(pathStr));
+  }
+
+  private void verifyIteratorResultCount(int itrCount, List<String> fileNames) {
+    Assertions.assertThat(itrCount).describedAs(
+            "Number of iterations should be equal to the files created")
+        .isEqualTo(TEST_FILES_NUMBER);
+    Assertions.assertThat(fileNames)
+        .describedAs("After removing every item found from the iterator, "
+            + "there should be no more elements in the fileNames")
+        .hasSize(0);
   }
 
   private ListingSupport getMockListingSupport(String exceptionMessage) {
     return new ListingSupport() {
       @Override
-      public FileStatus[] listStatus(Path path, TracingContext tracingContext) throws IOException {
+      public FileStatus[] listStatus(Path path, TracingContext tracingContext) {
         return null;
       }
 
       @Override
-      public FileStatus[] listStatus(Path path, String startFrom, TracingContext tracingContext)
-          throws IOException {
+      public FileStatus[] listStatus(Path path, String startFrom, TracingContext tracingContext) {
         return null;
       }
 
@@ -303,15 +264,14 @@ public String listStatus(Path path, String startFrom,
   }
 
   private Path createTestDirectory() throws IOException {
-    String testDirectoryName = "testDirectory" + System.currentTimeMillis();
-    Path testDirectory = path(testDirectoryName);
+    Path testDirectory = path("testDirectory");
     getFileSystem().mkdirs(testDirectory);
     return testDirectory;
   }
 
-  private void setEnableAbfsIterator(boolean shouldEnable) throws IOException {
+  private void disableAbfsIterator() throws IOException {
     AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem());
-    abfsStore.getAbfsConfiguration().setEnableAbfsListIterator(shouldEnable);
+    abfsStore.getAbfsConfiguration().setEnableAbfsListIterator(false);
   }
 
   private void setPageSize(int pageSize) throws IOException {
@@ -319,21 +279,21 @@ private void setPageSize(int pageSize) throws IOException {
     abfsStore.getAbfsConfiguration().setListMaxResults(pageSize);
   }
 
-  private List<String> createFilesUnderDirectory(int numFiles, Path rootPath,
-      String filenamePrefix)
+  private List<String> createFilesUnderDirectory(Path rootPath)
       throws ExecutionException, InterruptedException, IOException {
     final List<Future<Void>> tasks = new ArrayList<>();
-    final List<String> fileNames = new ArrayList<>();
+    final List<String> fileNames = Collections.synchronizedList(new ArrayList<>());
     ExecutorService es = Executors.newFixedThreadPool(10);
     try {
-      for (int i = 0; i < numFiles; i++) {
-        final Path filePath = new Path(rootPath, filenamePrefix + i);
-        Callable<Void> callable = () -> {
-          getFileSystem().create(filePath);
-          fileNames.add(makeQualified(filePath).toString());
+      for (int i = 0; i < ITestAbfsListStatusRemoteIterator.TEST_FILES_NUMBER; i++) {
+        Path filePath = makeQualified(new Path(rootPath, "testListPath" + i));
+        tasks.add(es.submit(() -> {
+          touch(filePath);
+          synchronized (fileNames) {
+            Assert.assertTrue(fileNames.add(filePath.toString()));
+          }
           return null;
-        };
-        tasks.add(es.submit(callable));
+        }));
       }
       for (Future<Void> task : tasks) {
         task.get();
@@ -341,6 +301,10 @@ private List<String> createFilesUnderDirectory(int numFiles, Path rootPath,
     } finally {
       es.shutdownNow();
     }
+    LOG.debug(fileNames.toString());
+    Assertions.assertThat(fileNames)
+        .describedAs("File creation incorrect or fileNames not added to list")
+        .hasSize(ITestAbfsListStatusRemoteIterator.TEST_FILES_NUMBER);
     return fileNames;
   }
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
index b0e82444afb34..beada775ae87b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
@@ -32,11 +32,14 @@
 import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
 import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
 import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.APPENDBLOB_MAX_WRITE_BUFFER_SIZE;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
 
 /**
  * Test read, write and seek.
@@ -47,18 +50,27 @@
 public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
   private static final String TEST_PATH = "/testfile";
 
-  @Parameterized.Parameters(name = "Size={0}")
+  /**
+   * Parameterize on read buffer size and readahead.
+   * For test performance, a full x*y test matrix is not used.
+   * @return the test parameters
+   */
+  @Parameterized.Parameters(name = "Size={0}-readahead={1}")
   public static Iterable<Object[]> sizes() {
-    return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE},
-        {DEFAULT_READ_BUFFER_SIZE},
-        {APPENDBLOB_MAX_WRITE_BUFFER_SIZE},
-        {MAX_BUFFER_SIZE}});
+    return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE, true},
+        {DEFAULT_READ_BUFFER_SIZE, false},
+        {DEFAULT_READ_BUFFER_SIZE, true},
+        {APPENDBLOB_MAX_WRITE_BUFFER_SIZE, false},
+        {MAX_BUFFER_SIZE, true}});
   }
 
   private final int size;
+  private final boolean readaheadEnabled;
 
-  public ITestAbfsReadWriteAndSeek(final int size) throws Exception {
+  public ITestAbfsReadWriteAndSeek(final int size,
+      final boolean readaheadEnabled) throws Exception {
     this.size = size;
+    this.readaheadEnabled = readaheadEnabled;
   }
 
   @Test
@@ -71,18 +83,25 @@ private void testReadWriteAndSeek(int bufferSize) throws Exception {
     final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
     abfsConfiguration.setWriteBufferSize(bufferSize);
     abfsConfiguration.setReadBufferSize(bufferSize);
+    abfsConfiguration.setReadAheadEnabled(readaheadEnabled);
 
     final byte[] b = new byte[2 * bufferSize];
     new Random().nextBytes(b);
 
     Path testPath = path(TEST_PATH);
-    try (FSDataOutputStream stream = fs.create(testPath)) {
+    FSDataOutputStream stream = fs.create(testPath);
+    try {
       stream.write(b);
+    } finally{
+    stream.close();
     }
+    logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream);
 
     final byte[] readBuffer = new byte[2 * bufferSize];
     int result;
+    IOStatisticsSource statisticsSource = null;
     try (FSDataInputStream inputStream = fs.open(testPath)) {
+      statisticsSource = inputStream;
       ((AbfsInputStream) inputStream.getWrappedStream()).registerListener(
           new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
               fs.getFileSystemId(), FSOperationType.READ, true, 0,
@@ -100,6 +119,8 @@ private void testReadWriteAndSeek(int bufferSize) throws Exception {
       inputStream.seek(0);
       result = inputStream.read(readBuffer, 0, bufferSize);
     }
+    logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, statisticsSource);
+
     assertNotEquals("data read in final read()", -1, result);
     assertArrayEquals(readBuffer, b);
   }
@@ -110,6 +131,7 @@ public void testReadAheadRequestID() throws java.io.IOException {
     final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration();
     int bufferSize = MIN_BUFFER_SIZE;
     abfsConfiguration.setReadBufferSize(bufferSize);
+    abfsConfiguration.setReadAheadEnabled(readaheadEnabled);
 
     final byte[] b = new byte[bufferSize * 10];
     new Random().nextBytes(b);
@@ -121,8 +143,10 @@ public void testReadAheadRequestID() throws java.io.IOException {
               ((AbfsOutputStream) stream.getWrappedStream())
                   .getStreamID()));
       stream.write(b);
+      logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream);
     }
 
+
     final byte[] readBuffer = new byte[4 * bufferSize];
     int result;
     fs.registerListener(
@@ -135,6 +159,7 @@ public void testReadAheadRequestID() throws java.io.IOException {
               ((AbfsInputStream) inputStream.getWrappedStream())
                   .getStreamID()));
       result = inputStream.read(readBuffer, 0, bufferSize*4);
+      logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, inputStream);
     }
     fs.registerListener(null);
   }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
index a71e7bc815f75..3fe3557d501dc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
@@ -74,8 +74,9 @@ public void testAbfsRestOperationExceptionFormat() throws IOException {
       // verify its format
       String errorMessage = ex.getLocalizedMessage();
       String[] errorFields = errorMessage.split(",");
-
-      Assert.assertEquals(6, errorFields.length);
+      Assertions.assertThat(errorFields)
+          .describedAs("fields in exception of %s", ex)
+          .hasSize(6);
       // Check status message, status code, HTTP Request Type and URL.
       Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim());
       Assert.assertEquals("404", errorFields[1].trim());
@@ -111,7 +112,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
     final AzureBlobFileSystem fs1 =
         (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
         config);
-    RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
+    RetryTestTokenProvider retryTestTokenProvider
+        = RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
+        getAccessTokenProvider(fs1));
+    retryTestTokenProvider.resetStatusToFirstTokenFetch();
 
     intercept(Exception.class,
         ()-> {
@@ -119,10 +123,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
         });
 
     // Number of retries done should be as configured
-    Assert.assertTrue(
-        "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
-            + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
-            + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
+    Assert.assertEquals(
+        "Number of token fetch retries done does not match with fs.azure"
+            + ".custom.token.fetch.retry.count configured", numOfRetries,
+        retryTestTokenProvider.getRetryCount());
   }
 
   @Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
index ea9fba62579f6..b164689ef807b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
@@ -401,7 +401,8 @@ public void testSignatureMask() throws Exception {
     fs.create(new Path(src)).close();
     AbfsRestOperation abfsHttpRestOperation = fs.getAbfsClient()
         .renamePath(src, "/testABC" + "/abc.txt", null,
-            getTestTracingContext(fs, false));
+            getTestTracingContext(fs, false), null, false)
+        .getOp();
     AbfsHttpOperation result = abfsHttpRestOperation.getResult();
     String url = result.getMaskedUrl();
     String encodedUrl = result.getMaskedEncodedUrl();
@@ -418,7 +419,7 @@ public void testSignatureMaskOnExceptionMessage() throws Exception {
     intercept(IOException.class, "sig=XXXX",
         () -> getFileSystem().getAbfsClient()
             .renamePath("testABC/test.xt", "testABC/abc.txt", null,
-                getTestTracingContext(getFileSystem(), false)));
+                getTestTracingContext(getFileSystem(), false), null, false));
   }
 
   @Test
@@ -478,4 +479,17 @@ public void testSetPermissionWithoutAgentForNonOwner() throws Exception {
         "r--r-----",
         fileStatus.getPermission().toString());
   }
+
+  @Test
+  public void testSASQuesMarkPrefix() throws Exception {
+    AbfsConfiguration testConfig = this.getConfiguration();
+    // the SAS Token Provider is changed
+    testConfig.set(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, "org.apache.hadoop.fs.azurebfs.extensions.MockWithPrefixSASTokenProvider");
+
+    AzureBlobFileSystem testFs = (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration());
+    Path testFile = new Path("/testSASPrefixQuesMark");
+
+    // the creation of this filesystem should work correctly even when a SAS Token is generated with a ? prefix
+    testFs.create(testFile).close();
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
index b9b846f52d2dd..db181fb5dd660 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
@@ -192,6 +192,7 @@ public void testDeleteIdempotency() throws Exception {
 
     // Mock delete response to 404
     when(op.getResult()).thenReturn(http404Op);
+    when(op.hasResult()).thenReturn(true);
 
     Assertions.assertThat(testClient.deleteIdempotencyCheckOp(op)
         .getResult()
@@ -207,6 +208,7 @@ public void testDeleteIdempotency() throws Exception {
 
     // Mock delete response to 400
     when(op.getResult()).thenReturn(http400Op);
+    when(op.hasResult()).thenReturn(true);
 
     Assertions.assertThat(testClient.deleteIdempotencyCheckOp(op)
         .getResult()
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
index b12af5b0826ab..ea07650e90110 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.azurebfs;
 
-import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
@@ -26,36 +25,22 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.assertj.core.api.Assertions;
-import org.junit.Test;
 import org.junit.Assert;
+import org.junit.Test;
 
-import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
-import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
-import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
-import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticAssertions;
+import org.apache.hadoop.fs.statistics.IOStatistics;
 
-import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static java.util.UUID.randomUUID;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
-
-import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
 
 /**
  * Test rename operation.
@@ -63,9 +48,6 @@
 public class ITestAzureBlobFileSystemRename extends
     AbstractAbfsIntegrationTest {
 
-  private static final int REDUCED_RETRY_COUNT = 1;
-  private static final int REDUCED_MAX_BACKOFF_INTERVALS_MS = 5000;
-
   public ITestAzureBlobFileSystemRename() throws Exception {
     super();
   }
@@ -191,151 +173,29 @@ public void testPosixRenameDirectory() throws Exception {
   }
 
   @Test
-  public void testRenameRetryFailureAsHTTP400() throws Exception {
-    // Rename failed as Bad Request
-    // RenameIdempotencyCheck should throw back the rename failure Op
-    testRenameTimeout(HTTP_BAD_REQUEST, HTTP_BAD_REQUEST, false,
-        "renameIdempotencyCheckOp should return rename BadRequest "
-            + "response itself.");
-  }
-
-  @Test
-  public void testRenameRetryFailureAsHTTP404() throws Exception {
-    // Rename failed as FileNotFound and the destination LMT is
-    // within TimespanForIdentifyingRecentOperationThroughLMT
-    testRenameTimeout(HTTP_NOT_FOUND, HTTP_OK, false,
-        "Rename should return success response because the destination "
-            + "path is present and its LMT is within "
-            + "TimespanForIdentifyingRecentOperationThroughLMT.");
-  }
-
-  @Test
-  public void testRenameRetryFailureWithDestOldLMT() throws Exception {
-    // Rename failed as FileNotFound and the destination LMT is
-    // older than TimespanForIdentifyingRecentOperationThroughLMT
-    testRenameTimeout(HTTP_NOT_FOUND, HTTP_NOT_FOUND, true,
-        "Rename should return original rename failure response "
-            + "because the destination path LMT is older than "
-            + "TimespanForIdentifyingRecentOperationThroughLMT.");
-  }
-
-  @Test
-  public void testRenameIdempotencyTriggerHttpNotFound() throws Exception {
-    AbfsHttpOperation http404Op = mock(AbfsHttpOperation.class);
-    when(http404Op.getStatusCode()).thenReturn(HTTP_NOT_FOUND);
-
-    AbfsHttpOperation http200Op = mock(AbfsHttpOperation.class);
-    when(http200Op.getStatusCode()).thenReturn(HTTP_OK);
-
-    // Check 1 where idempotency check fails to find dest path
-    // Rename should throw exception
-    testRenameIdempotencyTriggerChecks(http404Op);
-
-    // Check 2 where idempotency check finds the dest path
-    // Renam will be successful
-    testRenameIdempotencyTriggerChecks(http200Op);
-  }
-
-  private void testRenameIdempotencyTriggerChecks(
-      AbfsHttpOperation idempotencyRetHttpOp) throws Exception {
-
-    final AzureBlobFileSystem fs = getFileSystem();
-    AbfsClient client = TestAbfsClient.getMockAbfsClient(
-        fs.getAbfsStore().getClient(),
-        this.getConfiguration());
-
-    AbfsRestOperation idempotencyRetOp = mock(AbfsRestOperation.class);
-    when(idempotencyRetOp.getResult()).thenReturn(idempotencyRetHttpOp);
-    doReturn(idempotencyRetOp).when(client).renameIdempotencyCheckOp(any(),
-        any(), any(), any());
-    when(client.renamePath(any(), any(), any(), any())).thenCallRealMethod();
-
-    // rename on non-existing source file will trigger idempotency check
-    if (idempotencyRetHttpOp.getStatusCode() == HTTP_OK) {
-      // idempotency check found that destination exists and is recently created
-      Assertions.assertThat(client.renamePath(
-          "/NonExistingsourcepath",
-          "/destpath",
-          null,
-          getTestTracingContext(fs, true))
-          .getResult()
-          .getStatusCode())
-          .describedAs("Idempotency check reports recent successful "
-              + "rename. 200OK should be returned")
-          .isEqualTo(idempotencyRetOp.getResult().getStatusCode());
-    } else {
-      // rename dest not found. Original exception should be returned.
-      intercept(AbfsRestOperationException.class,
-          () -> client.renamePath(
-              "/NonExistingsourcepath",
-              "/destpath",
-              "",
-              getTestTracingContext(fs, true)));
-    }
-  }
-
-  private void testRenameTimeout(
-      int renameRequestStatus,
-      int renameIdempotencyCheckStatus,
-      boolean isOldOp,
-      String assertMessage) throws Exception {
-    // Config to reduce the retry and maxBackoff time for test run
-    AbfsConfiguration abfsConfig
-        = TestAbfsConfigurationFieldsValidation.updateRetryConfigs(
-        getConfiguration(),
-        REDUCED_RETRY_COUNT, REDUCED_MAX_BACKOFF_INTERVALS_MS);
+  public void testRenameWithNoDestinationParentDir() throws Exception {
+    describe("Verifying the expected behaviour of ABFS rename when "
+        + "destination parent Dir doesn't exist.");
 
     final AzureBlobFileSystem fs = getFileSystem();
-    AbfsClient abfsClient = fs.getAbfsStore().getClient();
-    AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext(
-        abfsClient,
-        abfsConfig);
-
-    // Mock instance of AbfsRestOperation
-    AbfsRestOperation op = mock(AbfsRestOperation.class);
-    // Set retryCount to non-zero
-    when(op.isARetriedRequest()).thenReturn(true);
-
-    // Mock instance of Http Operation response. This will return HTTP:Bad Request
-    AbfsHttpOperation http400Op = mock(AbfsHttpOperation.class);
-    when(http400Op.getStatusCode()).thenReturn(HTTP_BAD_REQUEST);
-
-    // Mock instance of Http Operation response. This will return HTTP:Not Found
-    AbfsHttpOperation http404Op = mock(AbfsHttpOperation.class);
-    when(http404Op.getStatusCode()).thenReturn(HTTP_NOT_FOUND);
-
-    Path destinationPath = fs.makeQualified(
-        new Path("destination" + randomUUID().toString()));
-
-    Instant renameRequestStartTime = Instant.now();
-
-    if (renameRequestStatus == HTTP_BAD_REQUEST) {
-      when(op.getResult()).thenReturn(http400Op);
-    } else if (renameRequestStatus == HTTP_NOT_FOUND) {
-      // Create the file new.
-      fs.create(destinationPath).close();
-      when(op.getResult()).thenReturn(http404Op);
-
-      if (isOldOp) {
-        // instead of sleeping for DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS
-        // which will affect test run time
-        // will modify renameRequestStartTime to a future time so that
-        // lmt will qualify for old op
-        renameRequestStartTime = renameRequestStartTime.plusSeconds(
-            DEFAULT_CLOCK_SKEW_WITH_SERVER_IN_MS);
-      }
-
-    }
-
-    Assertions.assertThat(testClient.renameIdempotencyCheckOp(
-        renameRequestStartTime,
-        op,
-        destinationPath.toUri().getPath(),
-        getTestTracingContext(fs, true))
-        .getResult()
-        .getStatusCode())
-        .describedAs(assertMessage)
-        .isEqualTo(renameIdempotencyCheckStatus);
+    Path sourcePath = path(getMethodName());
+    Path destPath = new Path("falseParent", "someChildFile");
+
+    byte[] data = dataset(1024, 'a', 'z');
+    writeDataset(fs, sourcePath, data, data.length, 1024, true);
+
+    // Verify that renaming on a destination with no parent dir wasn't
+    // successful.
+    assertFalse("Rename result expected to be false with no Parent dir",
+        fs.rename(sourcePath, destPath));
+
+    // Verify that metadata was in an incomplete state after the rename
+    // failure, and we retired the rename once more.
+    IOStatistics ioStatistics = fs.getIOStatistics();
+    IOStatisticAssertions.assertThatStatisticCounter(ioStatistics,
+        RENAME_PATH_ATTEMPTS.getStatName())
+        .describedAs("There should be 2 rename attempts if metadata "
+            + "incomplete state failure is hit")
+        .isEqualTo(2);
   }
-
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
index b7b1a3bba2db6..58c3be6688330 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
@@ -37,7 +37,7 @@
 
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Lists;
 import org.assertj.core.api.Assertions;
 import org.junit.Assume;
@@ -526,7 +526,8 @@ private void testRenamePath(final boolean isWithCPK) throws Exception {
     AbfsClient abfsClient = fs.getAbfsClient();
     AbfsRestOperation abfsRestOperation = abfsClient
         .renamePath(testFileName, newName, null,
-            getTestTracingContext(fs, false));
+            getTestTracingContext(fs, false), null, false)
+        .getOp();
     assertCPKHeaders(abfsRestOperation, false);
     assertNoCPKResponseHeadersPresent(abfsRestOperation);
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
index 006004850d0df..b91a3e2208b57 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
@@ -44,7 +44,7 @@
 import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_CLIENT_CORRELATIONID;
@@ -130,10 +130,10 @@ public void runCorrelationTestForAllMethods() throws Exception {
 
     testClasses.put(new ITestAzureBlobFileSystemListStatus(), //liststatus
         ITestAzureBlobFileSystemListStatus.class.getMethod("testListPath"));
-    testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE), //open,
+    testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE, true), //open,
         // read, write
         ITestAbfsReadWriteAndSeek.class.getMethod("testReadAheadRequestID"));
-    testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE), //read (bypassreadahead)
+    testClasses.put(new ITestAbfsReadWriteAndSeek(MIN_BUFFER_SIZE, false), //read (bypassreadahead)
         ITestAbfsReadWriteAndSeek.class
             .getMethod("testReadAndWriteWithDifferentBufferSizesAndSeek"));
     testClasses.put(new ITestAzureBlobFileSystemAppend(), //append
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java
new file mode 100644
index 0000000000000..8160cdc64c546
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+
+/**
+ * Helper methods for committer tests on ABFS.
+ */
+final class AbfsCommitTestHelper {
+  private AbfsCommitTestHelper() {
+  }
+
+  /**
+   * Prepare the test configuration.
+   * @param contractTestBinding test binding
+   * @return an extracted and patched configuration.
+   */
+  static Configuration prepareTestConfiguration(
+      ABFSContractTestBinding contractTestBinding) {
+    final Configuration conf =
+        contractTestBinding.getRawConfiguration();
+
+    // use ABFS Store operations
+    conf.set(OPT_STORE_OPERATIONS_CLASS,
+        AbfsManifestStoreOperations.NAME);
+
+    return conf;
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java
new file mode 100644
index 0000000000000..55752055f0c31
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.AbstractManifestCommitterTest;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.util.DurationInfo;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled;
+import static org.apache.hadoop.io.IOUtils.closeStream;
+
+/**
+ * Tests which create a yarn minicluster.
+ * These are all considered scale tests; the probe for
+ * scale tests being enabled is executed before the cluster
+ * is set up to avoid wasting time on non-scale runs.
+ */
+public abstract class AbstractAbfsClusterITest extends
+    AbstractManifestCommitterTest {
+
+  public static final int NO_OF_NODEMANAGERS = 2;
+
+  private final ABFSContractTestBinding binding;
+
+
+  /**
+   * The static cluster binding with the lifecycle of this test; served
+   * through instance-level methods for sharing across methods in the
+   * suite.
+   */
+  @SuppressWarnings("StaticNonFinalField")
+  private static ClusterBinding clusterBinding;
+
+  protected AbstractAbfsClusterITest() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  protected int getTestTimeoutMillis() {
+    return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS;
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+    requireScaleTestsEnabled();
+    if (getClusterBinding() == null) {
+      clusterBinding = demandCreateClusterBinding();
+    }
+    assertNotNull("cluster is not bound", getClusterBinding());
+  }
+
+  @AfterClass
+  public static void teardownClusters() throws IOException {
+    terminateCluster(clusterBinding);
+    clusterBinding = null;
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  /**
+   * This is the cluster binding which every subclass must create.
+   */
+  protected static final class ClusterBinding {
+
+    private String clusterName;
+
+    private final MiniMRYarnCluster yarn;
+
+    public ClusterBinding(
+        final String clusterName,
+        final MiniMRYarnCluster yarn) {
+      this.clusterName = clusterName;
+      this.yarn = requireNonNull(yarn);
+    }
+
+
+    /**
+     * Get the cluster FS, which will either be HDFS or the local FS.
+     * @return a filesystem.
+     * @throws IOException failure
+     */
+    public FileSystem getClusterFS() throws IOException {
+      return FileSystem.getLocal(yarn.getConfig());
+    }
+
+    public MiniMRYarnCluster getYarn() {
+      return yarn;
+    }
+
+    public Configuration getConf() {
+      return getYarn().getConfig();
+    }
+
+    public String getClusterName() {
+      return clusterName;
+    }
+
+    public void terminate() {
+      closeStream(getYarn());
+    }
+  }
+
+  /**
+   * Create the cluster binding.
+   * The configuration will be patched by propagating down options
+   * from the maven build (S3Guard binding etc) and turning off unwanted
+   * YARN features.
+   *
+   * If an HDFS cluster is requested,
+   * the HDFS and YARN clusters will share the same configuration, so
+   * the HDFS cluster binding is implicitly propagated to YARN.
+   * If one is not requested, the local filesystem is used as the cluster FS.
+   * @param conf configuration to start with.
+   * @return the cluster binding.
+   * @throws IOException failure.
+   */
+  protected static ClusterBinding createCluster(
+      final JobConf conf) throws IOException {
+    try (DurationInfo d = new DurationInfo(LOG, "Creating YARN MiniCluster")) {
+      conf.setBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, false);
+      // create a unique cluster name based on the current time in millis.
+      String timestamp = LocalDateTime.now().format(
+          DateTimeFormatter.ofPattern("yyyy-MM-dd-HH.mm.ss.SS"));
+      String clusterName = "yarn-" + timestamp;
+      MiniMRYarnCluster yarnCluster =
+          new MiniMRYarnCluster(clusterName, NO_OF_NODEMANAGERS);
+      yarnCluster.init(conf);
+      yarnCluster.start();
+      return new ClusterBinding(clusterName, yarnCluster);
+    }
+  }
+
+  /**
+   * Terminate the cluster if it is not null.
+   * @param cluster the cluster
+   */
+  protected static void terminateCluster(ClusterBinding cluster) {
+    if (cluster != null) {
+      cluster.terminate();
+    }
+  }
+
+  /**
+   * Get the cluster binding for this subclass.
+   * @return the cluster binding
+   */
+  protected ClusterBinding getClusterBinding() {
+    return clusterBinding;
+  }
+
+  protected MiniMRYarnCluster getYarn() {
+    return getClusterBinding().getYarn();
+  }
+
+
+  /**
+   * We stage work into a temporary directory rather than directly under
+   * the user's home directory, as that is often rejected by CI test
+   * runners.
+   */
+  @Rule
+  public final TemporaryFolder stagingFilesDir = new TemporaryFolder();
+
+
+  /**
+   * binding on demand rather than in a BeforeClass static method.
+   * Subclasses can override this to change the binding options.
+   * @return the cluster binding
+   */
+  protected ClusterBinding demandCreateClusterBinding() throws Exception {
+    return createCluster(new JobConf());
+  }
+
+  /**
+   * Create a job configuration.
+   * This creates a new job conf from the yarn
+   * cluster configuration then calls
+   * {@link #applyCustomConfigOptions(JobConf)} to allow it to be customized.
+   * @return the new job configuration.
+   * @throws IOException failure
+   */
+  protected JobConf newJobConf() throws IOException {
+    JobConf jobConf = new JobConf(getYarn().getConfig());
+    jobConf.addResource(getConfiguration());
+    applyCustomConfigOptions(jobConf);
+    return jobConf;
+  }
+
+  /**
+   * Patch the (job) configuration for this committer.
+   * @param jobConf configuration to patch
+   * @return a configuration which will run this configuration.
+   */
+  protected Configuration patchConfigurationForCommitter(
+      final Configuration jobConf) {
+    enableManifestCommitter(jobConf);
+    return jobConf;
+  }
+
+  /**
+   * Override point to let implementations tune the MR Job conf.
+   * @param jobConf configuration
+   */
+  protected void applyCustomConfigOptions(JobConf jobConf) throws IOException {
+
+  }
+
+
+  /**
+   * Assume that scale tests are enabled.
+   */
+  protected void requireScaleTestsEnabled() {
+    assumeScaleTestsEnabled(getConfiguration());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java
new file mode 100644
index 0000000000000..a597c35376a71
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCleanupStage;
+
+/**
+ * Cleanup logic on ABFS.
+ */
+public class ITestAbfsCleanupStage extends TestCleanupStage {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsCleanupStage() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java
new file mode 100644
index 0000000000000..a0aaec8532850
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCommitTaskStage;
+
+/**
+ * ABFS storage test of task committer.
+ */
+public class ITestAbfsCommitTaskStage extends TestCommitTaskStage {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsCommitTaskStage() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java
new file mode 100644
index 0000000000000..6621b80da00c1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCreateOutputDirectoriesStage;
+
+/**
+ * ABFS storage test of directory creation.
+ */
+public class ITestAbfsCreateOutputDirectoriesStage extends TestCreateOutputDirectoriesStage {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsCreateOutputDirectoriesStage() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java
new file mode 100644
index 0000000000000..4e4c4f5996bc2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.FixMethodOrder;
+import org.junit.runners.MethodSorters;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestJobThroughManifestCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+
+/**
+ * Test the Manifest committer stages against ABFS.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class ITestAbfsJobThroughManifestCommitter
+    extends TestJobThroughManifestCommitter {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsJobThroughManifestCommitter() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return enableManifestCommitter(prepareTestConfiguration(binding));
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+  @Override
+  protected boolean shouldDeleteTestRootAtEndOfTestRun() {
+    return true;
+  }
+
+  /**
+   * Add read of manifest and validate of output's etags.
+   * @param attemptId attempt ID
+   * @param files files which were created.
+   * @param manifest manifest
+   * @throws IOException failure
+   */
+  @Override
+  protected void validateTaskAttemptManifest(String attemptId,
+      List<Path> files,
+      TaskManifest manifest) throws IOException {
+    super.validateTaskAttemptManifest(attemptId, files, manifest);
+    final List<FileEntry> commit = manifest.getFilesToCommit();
+    final ManifestStoreOperations operations = getStoreOperations();
+    for (FileEntry entry : commit) {
+      Assertions.assertThat(entry.getEtag())
+          .describedAs("Etag of %s", entry)
+          .isNotEmpty();
+      final FileStatus sourceStatus = operations.getFileStatus(entry.getSourcePath());
+      final String etag = ManifestCommitterSupport.getEtag(sourceStatus);
+      Assertions.assertThat(etag)
+          .describedAs("Etag of %s", sourceStatus)
+          .isEqualTo(entry.getEtag());
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java
new file mode 100644
index 0000000000000..acd693e39a0f0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestLoadManifestsStage;
+
+/**
+ * ABFS storage test of saving and loading a large number
+ * of manifests.
+ */
+public class ITestAbfsLoadManifestsStage extends TestLoadManifestsStage {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsLoadManifestsStage() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java
new file mode 100644
index 0000000000000..aac06f952dab2
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestManifestCommitProtocol;
+
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+
+/**
+ * Test the Manifest protocol against ABFS.
+ */
+public class ITestAbfsManifestCommitProtocol extends
+    TestManifestCommitProtocol {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsManifestCommitProtocol() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return enableManifestCommitter(prepareTestConfiguration(binding));
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+
+  @Override
+  protected String suitename() {
+    return "ITestAbfsManifestCommitProtocol";
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java
new file mode 100644
index 0000000000000..922782da29c5f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.nio.charset.StandardCharsets;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.AbstractManifestCommitterTest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME;
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Test {@link AbfsManifestStoreOperations}.
+ * As this looks at etag handling through FS operations, it's actually testing how etags work
+ * in ABFS (preservation across renames) and in the client (are they consistent
+ * in LIST and HEAD calls).
+ *
+ * Skipped when tested against wasb-compatible stores.
+ */
+public class ITestAbfsManifestStoreOperations extends AbstractManifestCommitterTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAbfsManifestStoreOperations.class);
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsManifestStoreOperations() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+
+    // skip tests on non-HNS stores
+    assumeTrue("Resilient rename not available",
+        getFileSystem().hasPathCapability(getContract().getTestPath(),
+            ETAGS_PRESERVED_IN_RENAME));
+
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return enableManifestCommitter(prepareTestConfiguration(binding));
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+  /**
+   * basic consistency across operations, as well as being non-empty.
+   */
+  @Test
+  public void testEtagConsistencyAcrossListAndHead() throws Throwable {
+    describe("Etag values must be non-empty and consistent across LIST and HEAD Calls.");
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    ContractTestUtils.touch(fs, path);
+    final ManifestStoreOperations operations = createManifestStoreOperations();
+    Assertions.assertThat(operations)
+        .describedAs("Store operations class loaded via Configuration")
+        .isInstanceOf(AbfsManifestStoreOperations.class);
+
+    final FileStatus st = operations.getFileStatus(path);
+    final String etag = operations.getEtag(st);
+    Assertions.assertThat(etag)
+        .describedAs("Etag of %s", st)
+        .isNotBlank();
+    LOG.info("etag of empty file is \"{}\"", etag);
+
+    final FileStatus[] statuses = fs.listStatus(path);
+    Assertions.assertThat(statuses)
+        .describedAs("List(%s)", path)
+        .hasSize(1);
+    final FileStatus lsStatus = statuses[0];
+    Assertions.assertThat(operations.getEtag(lsStatus))
+        .describedAs("etag of list status (%s) compared to HEAD value of %s", lsStatus, st)
+        .isEqualTo(etag);
+  }
+
+  @Test
+  public void testEtagsOfDifferentDataDifferent() throws Throwable {
+    describe("Verify that two different blocks of data written have different tags");
+
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    Path src = new Path(path, "src");
+
+    ContractTestUtils.createFile(fs, src, true,
+        "data1234".getBytes(StandardCharsets.UTF_8));
+    final ManifestStoreOperations operations = createManifestStoreOperations();
+    final FileStatus srcStatus = operations.getFileStatus(src);
+    final String srcTag = operations.getEtag(srcStatus);
+    LOG.info("etag of file 1 is \"{}\"", srcTag);
+
+    // now overwrite with data of same length
+    // (ensure that path or length aren't used exclusively as tag)
+    ContractTestUtils.createFile(fs, src, true,
+        "1234data".getBytes(StandardCharsets.UTF_8));
+
+    // validate
+    final String tag2 = operations.getEtag(operations.getFileStatus(src));
+    LOG.info("etag of file 2 is \"{}\"", tag2);
+
+    Assertions.assertThat(tag2)
+        .describedAs("etag of updated file")
+        .isNotEqualTo(srcTag);
+  }
+
+  @Test
+  public void testEtagConsistencyAcrossRename() throws Throwable {
+    describe("Verify that when a file is renamed, the etag remains unchanged");
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    Path src = new Path(path, "src");
+    Path dest = new Path(path, "dest");
+
+    ContractTestUtils.createFile(fs, src, true,
+        "sample data".getBytes(StandardCharsets.UTF_8));
+    final ManifestStoreOperations operations = createManifestStoreOperations();
+    final FileStatus srcStatus = operations.getFileStatus(src);
+    final String srcTag = operations.getEtag(srcStatus);
+    LOG.info("etag of short file is \"{}\"", srcTag);
+
+    Assertions.assertThat(srcTag)
+        .describedAs("Etag of %s", srcStatus)
+        .isNotBlank();
+
+    // rename
+    operations.commitFile(new FileEntry(src, dest, 0, srcTag));
+
+    // validate
+    FileStatus destStatus = operations.getFileStatus(dest);
+    final String destTag = operations.getEtag(destStatus);
+    Assertions.assertThat(destTag)
+        .describedAs("etag of list status (%s) compared to HEAD value of %s", destStatus, srcStatus)
+        .isEqualTo(srcTag);
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
new file mode 100644
index 0000000000000..6b9830e8f33fc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure;
+/**
+ * Rename failure logic on ABFS.
+ * This will go through the resilient rename operation.
+ */
+public class ITestAbfsRenameStageFailure extends TestRenameStageFailure {
+
+  /**
+   * How many files to create.
+   */
+  private static final int FILES_TO_CREATE = 20;
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsRenameStageFailure() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  protected boolean isNamespaceEnabled() throws AzureBlobFileSystemException {
+    AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
+    return fs.getAbfsStore().getIsNamespaceEnabled(AbstractAbfsIntegrationTest.getSampleTracingContext(fs, false));
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+  @Override
+  protected boolean requireRenameResilience() throws AzureBlobFileSystemException {
+    return isNamespaceEnabled();
+  }
+
+  @Override
+  protected int filesToCreate() {
+    return FILES_TO_CREATE;
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java
new file mode 100644
index 0000000000000..d2fe9de115c3b
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestTaskManifestFileIO;
+
+/**
+ * Test Reading/writing manifest file through ABFS.
+ */
+public class ITestAbfsTaskManifestFileIO extends TestTaskManifestFileIO {
+
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsTaskManifestFileIO() throws Exception {
+    binding = new ABFSContractTestBinding();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, binding.isSecureMode());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java
new file mode 100644
index 0000000000000..4b21b838decc5
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Consumer;
+
+import org.junit.Assume;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.examples.terasort.TeraGen;
+import org.apache.hadoop.examples.terasort.TeraSort;
+import org.apache.hadoop.examples.terasort.TeraSortConfigKeys;
+import org.apache.hadoop.examples.terasort.TeraValidate;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import static java.util.Optional.empty;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.loadSuccessFile;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.validateSuccessFile;
+
+/**
+ * Runs Terasort against ABFS using the manifest committer.
+ * The tests run in sequence, so each operation is isolated.
+ * Scale test only (it is big and slow)
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+@SuppressWarnings({"StaticNonFinalField", "OptionalUsedAsFieldOrParameterType"})
+public class ITestAbfsTerasort extends AbstractAbfsClusterITest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAbfsTerasort.class);
+
+  public static final int EXPECTED_PARTITION_COUNT = 10;
+
+  public static final int PARTITION_SAMPLE_SIZE = 1000;
+
+  public static final int ROW_COUNT = 1000;
+
+  /**
+   * This has to be common across all test methods.
+   */
+  private static final Path TERASORT_PATH = new Path("/ITestAbfsTerasort");
+
+  /**
+   * Duration tracker created in the first of the test cases and closed
+   * in {@link #test_140_teracomplete()}.
+   */
+  private static Optional<DurationInfo> terasortDuration = empty();
+
+  /**
+   * Tracker of which stages are completed and how long they took.
+   */
+  private static final Map<String, DurationInfo> COMPLETED_STAGES = new HashMap<>();
+
+  /**
+   * FileSystem statistics are collected from the _SUCCESS markers.
+   */
+  protected static final IOStatisticsSnapshot JOB_IOSTATS =
+      snapshotIOStatistics();
+
+  /** Base path for all the terasort input and output paths. */
+  private Path terasortPath;
+
+  /** Input (teragen) path. */
+  private Path sortInput;
+
+  /** Path where sorted data goes. */
+  private Path sortOutput;
+
+  /** Path for validated job's output. */
+  private Path sortValidate;
+
+  public ITestAbfsTerasort() throws Exception {
+  }
+
+
+  @Override
+  public void setup() throws Exception {
+    // superclass calls requireScaleTestsEnabled();
+    super.setup();
+    prepareToTerasort();
+  }
+
+  /**
+   * Set up the job conf with the options for terasort chosen by the scale
+   * options.
+   * @param conf configuration
+   */
+  @Override
+  protected void applyCustomConfigOptions(JobConf conf) {
+    // small sample size for faster runs
+    conf.setInt(TeraSortConfigKeys.SAMPLE_SIZE.key(),
+        getSampleSizeForEachPartition());
+    conf.setInt(TeraSortConfigKeys.NUM_PARTITIONS.key(),
+        getExpectedPartitionCount());
+    conf.setBoolean(
+        TeraSortConfigKeys.USE_SIMPLE_PARTITIONER.key(),
+        false);
+  }
+
+  private int getExpectedPartitionCount() {
+    return EXPECTED_PARTITION_COUNT;
+  }
+
+  private int getSampleSizeForEachPartition() {
+    return PARTITION_SAMPLE_SIZE;
+  }
+
+  protected int getRowCount() {
+    return ROW_COUNT;
+  }
+
+  /**
+   * Set up the terasort by initializing paths variables
+   * The paths used must be unique across parameterized runs but
+   * common across all test cases in a single parameterized run.
+   */
+  private void prepareToTerasort() {
+    terasortPath = getFileSystem().makeQualified(TERASORT_PATH);
+    sortInput = new Path(terasortPath, "sortin");
+    sortOutput = new Path(terasortPath, "sortout");
+    sortValidate = new Path(terasortPath, "validate");
+  }
+
+  /**
+   * Declare that a stage has completed.
+   * @param stage stage name/key in the map
+   * @param d duration.
+   */
+  private static void completedStage(final String stage,
+      final DurationInfo d) {
+    COMPLETED_STAGES.put(stage, d);
+  }
+
+  /**
+   * Declare a stage which is required for this test case.
+   * @param stage stage name
+   */
+  private static void requireStage(final String stage) {
+    Assume.assumeTrue(
+        "Required stage was not completed: " + stage,
+        COMPLETED_STAGES.get(stage) != null);
+  }
+
+  /**
+   * Execute a single stage in the terasort.
+   * Updates the completed stages map with the stage duration -if successful.
+   * @param stage Stage name for the stages map.
+   * @param jobConf job conf
+   * @param dest destination directory -the _SUCCESS file will be expected here.
+   * @param tool tool to run.
+   * @param args args for the tool.
+   * @param minimumFileCount minimum number of files to have been created
+   * @throws Exception any failure
+   */
+  private void executeStage(
+      final String stage,
+      final JobConf jobConf,
+      final Path dest,
+      final Tool tool,
+      final String[] args,
+      final int minimumFileCount) throws Exception {
+    int result;
+
+    // the duration info is created outside a try-with-resources
+    // clause as it is used later.
+    DurationInfo d = new DurationInfo(LOG, stage);
+    try {
+      result = ToolRunner.run(jobConf, tool, args);
+    } finally {
+      d.close();
+    }
+    dumpOutputTree(dest);
+    assertEquals(stage
+        + "(" + StringUtils.join(", ", args) + ")"
+        + " failed", 0, result);
+    final ManifestSuccessData successFile = validateSuccessFile(getFileSystem(), dest,
+        minimumFileCount, "");
+    JOB_IOSTATS.aggregate(successFile.getIOStatistics());
+
+    completedStage(stage, d);
+  }
+
+  /**
+   * Set up terasort by cleaning out the destination, and note the initial
+   * time before any of the jobs are executed.
+   *
+   * This is executed first <i>for each parameterized run</i>.
+   * It is where all variables which need to be reset for each run need
+   * to be reset.
+   */
+  @Test
+  public void test_100_terasort_setup() throws Throwable {
+    describe("Setting up for a terasort");
+
+    getFileSystem().delete(terasortPath, true);
+    terasortDuration = Optional.of(new DurationInfo(LOG, false, "Terasort"));
+  }
+
+  @Test
+  public void test_110_teragen() throws Throwable {
+    describe("Teragen to %s", sortInput);
+    getFileSystem().delete(sortInput, true);
+
+    JobConf jobConf = newJobConf();
+    patchConfigurationForCommitter(jobConf);
+    executeStage("teragen",
+        jobConf,
+        sortInput,
+        new TeraGen(),
+        new String[]{Integer.toString(getRowCount()), sortInput.toString()},
+        1);
+  }
+
+
+  @Test
+  public void test_120_terasort() throws Throwable {
+    describe("Terasort from %s to %s", sortInput, sortOutput);
+    requireStage("teragen");
+    getFileSystem().delete(sortOutput, true);
+
+    loadSuccessFile(getFileSystem(), sortInput);
+    JobConf jobConf = newJobConf();
+    patchConfigurationForCommitter(jobConf);
+    executeStage("terasort",
+        jobConf,
+        sortOutput,
+        new TeraSort(),
+        new String[]{sortInput.toString(), sortOutput.toString()},
+        1);
+  }
+
+  @Test
+  public void test_130_teravalidate() throws Throwable {
+    describe("TeraValidate from %s to %s", sortOutput, sortValidate);
+    requireStage("terasort");
+    getFileSystem().delete(sortValidate, true);
+    loadSuccessFile(getFileSystem(), sortOutput);
+    JobConf jobConf = newJobConf();
+    patchConfigurationForCommitter(jobConf);
+    executeStage("teravalidate",
+        jobConf,
+        sortValidate,
+        new TeraValidate(),
+        new String[]{sortOutput.toString(), sortValidate.toString()},
+        1);
+  }
+
+  /**
+   * Print the results, and save to the base dir as a CSV file.
+   * Why there? Makes it easy to list and compare.
+   */
+  @Test
+  public void test_140_teracomplete() throws Throwable {
+    terasortDuration.ifPresent(d -> {
+      d.close();
+      completedStage("overall", d);
+    });
+
+    // IO Statistics
+    IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, JOB_IOSTATS);
+
+    // and the summary
+    final StringBuilder results = new StringBuilder();
+    results.append("\"Operation\"\t\"Duration\"\n");
+
+    // this is how you dynamically create a function in a method
+    // for use afterwards.
+    // Works because there's no IOEs being raised in this sequence.
+    Consumer<String> stage = (s) -> {
+      DurationInfo duration = COMPLETED_STAGES.get(s);
+      results.append(String.format("\"%s\"\t\"%s\"\n",
+          s,
+          duration == null ? "" : duration));
+    };
+
+    stage.accept("teragen");
+    stage.accept("terasort");
+    stage.accept("teravalidate");
+    stage.accept("overall");
+    String text = results.toString();
+    File resultsFile = File.createTempFile("results", ".csv");
+    FileUtils.write(resultsFile, text, StandardCharsets.UTF_8);
+    LOG.info("Results are in {}\n{}", resultsFile, text);
+  }
+
+  /**
+   * Reset the duration so if two committer tests are run sequentially.
+   * Without this the total execution time is reported as from the start of
+   * the first test suite to the end of the second.
+   */
+  @Test
+  public void test_150_teracleanup() throws Throwable {
+    terasortDuration = Optional.empty();
+  }
+
+  @Test
+  public void test_200_directory_deletion() throws Throwable {
+    getFileSystem().delete(terasortPath, true);
+  }
+
+  /**
+   * Dump the files under a path -but not fail if the path is not present.,
+   * @param path path to dump
+   * @throws Exception any failure.
+   */
+  protected void dumpOutputTree(Path path) throws Exception {
+    LOG.info("Files under output directory {}", path);
+    try {
+      RemoteIterators.foreach(getFileSystem().listFiles(path, true),
+          (status) -> LOG.info("{}", status));
+    } catch (FileNotFoundException e) {
+      LOG.info("Output directory {} not found", path);
+    }
+  }
+}
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
similarity index 80%
rename from hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
index 571491c03540f..3d49d62eaa865 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
@@ -17,7 +17,8 @@
  */
 
 /**
- * Resource handling plugins used internal to the Hadoop build.
- * IA.Private (build structure encourages not using the actual annotations)
+ * Unit and integration tests for the manifest committer.
+ * JSON job reports will be saved to
+ * {@code target/reports}
  */
-package org.apache.hadoop.maven.plugin.shade.resource;
+package org.apache.hadoop.fs.azurebfs.commit;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
index 565eb38c4f70a..9e40f22d231b0 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java
@@ -24,6 +24,9 @@
 public final class TestConfigurationKeys {
   public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name";
   public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
+  public static final String FS_AZURE_ABFS_ACCOUNT1_NAME = "fs.azure.abfs.account1.name";
+  public static final String FS_AZURE_ENABLE_AUTOTHROTTLING = "fs.azure.enable.autothrottling";
+  public static final String FS_AZURE_ANALYSIS_PERIOD = "fs.azure.analysis.period";
   public static final String FS_AZURE_ACCOUNT_KEY = "fs.azure.account.key";
   public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
   public static final String FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT = "fs.azure.test.namespace.enabled";
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
index 62bcca174ef8d..1319ea44c7c07 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
@@ -34,7 +34,7 @@ public class AbfsFileSystemContract extends AbstractBondedFSContract {
   public static final String CONTRACT_XML = "abfs.xml";
   private final boolean isSecure;
 
-  protected AbfsFileSystemContract(final Configuration conf, boolean secure) {
+  public AbfsFileSystemContract(final Configuration conf, boolean secure) {
     super(conf);
     //insert the base features
     addConfResource(CONTRACT_XML);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
new file mode 100644
index 0000000000000..d498ae71a4b6f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractEtag.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractEtagTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Contract test for etag support.
+ */
+public class ITestAbfsFileSystemContractEtag extends AbstractContractEtagTest {
+  private final boolean isSecure;
+  private final ABFSContractTestBinding binding;
+
+  public ITestAbfsFileSystemContractEtag() throws Exception {
+    binding = new ABFSContractTestBinding();
+    this.isSecure = binding.isSecureMode();
+  }
+
+  @Override
+  public void setup() throws Exception {
+    binding.setup();
+    super.setup();
+    // Base rename contract test class re-uses the test folder
+    // This leads to failures when the test is re-run as same ABFS test
+    // containers are re-used for test run and creation of source and
+    // destination test paths fail, as they are already present.
+    binding.getFileSystem().delete(binding.getTestPath(), true);
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return binding.getRawConfiguration();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(final Configuration conf) {
+    return new AbfsFileSystemContract(conf, isSecure);
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java
index 3222a15cd527d..aaf47f7a9c871 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java
@@ -34,10 +34,11 @@
 
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_AHEAD_RANGE;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_READ_BUFFER_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_READAHEAD;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.impl.FutureIOSupport.awaitFuture;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * Contract test for seek operation.
@@ -68,6 +69,7 @@ protected Configuration createConfiguration() {
   protected AbstractFSContract createContract(final Configuration conf) {
     conf.setInt(AZURE_READ_AHEAD_RANGE, MIN_BUFFER_SIZE);
     conf.setInt(AZURE_READ_BUFFER_SIZE, MIN_BUFFER_SIZE);
+    conf.setBoolean(FS_AZURE_ENABLE_READAHEAD, true);
     return new AbfsFileSystemContract(conf, isSecure);
   }
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java
index a9fa2d77194aa..e99d0895d11ee 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java
@@ -27,7 +27,9 @@
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -41,6 +43,7 @@ public ITestAzureBlobFileSystemBasics() throws Exception {
     // If all contract tests are running in parallel, some root level tests in FileSystemContractBaseTest will fail
     // due to the race condition. Hence for this contract test it should be tested in different container
     binding = new ABFSContractTestBinding(false);
+    globalTimeout = Timeout.millis(TEST_TIMEOUT);
   }
 
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java
index 8a33ea5de0641..3f6a4872c5d17 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java
@@ -20,7 +20,7 @@
 
 import java.io.IOException;
 
-import org.codehaus.jackson.map.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultEntrySchema;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/ClassicDelegationTokenManager.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/ClassicDelegationTokenManager.java
index f74e3e3b3e9a7..1f0cbc0a1672b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/ClassicDelegationTokenManager.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/ClassicDelegationTokenManager.java
@@ -22,7 +22,7 @@
 import java.net.URI;
 import java.nio.charset.Charset;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockWithPrefixSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockWithPrefixSASTokenProvider.java
new file mode 100644
index 0000000000000..ed701c4669c02
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockWithPrefixSASTokenProvider.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.extensions;
+
+import java.io.IOException;
+
+public class MockWithPrefixSASTokenProvider extends MockSASTokenProvider {
+
+    /**
+     * Function to return an already generated SAS Token with a '?' prefix
+     * @param accountName the name of the storage account.
+     * @param fileSystem the name of the fileSystem.
+     * @param path the file or directory path.
+     * @param operation the operation to be performed on the path.
+     * @return
+     * @throws IOException
+     */
+    @Override
+    public String getSASToken(String accountName, String fileSystem, String path,
+                              String operation) throws IOException {
+        String token = super.getSASToken(accountName, fileSystem, path, operation);
+        return "?" + token;
+    }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/StubAbfsTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/StubAbfsTokenIdentifier.java
index b04517c8ac4df..426c2ba3c09eb 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/StubAbfsTokenIdentifier.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/StubAbfsTokenIdentifier.java
@@ -29,7 +29,7 @@
 import java.util.Objects;
 import java.util.UUID;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
index 3566ebbaaaa2a..7427add29086c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
@@ -30,12 +30,12 @@
  */
 public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
 
-  // Need to track first token fetch otherwise will get counted as a retry too.
-  private static boolean isThisFirstTokenFetch = true;
-  public static int reTryCount = 0;
+  private static final Logger LOG = LoggerFactory.getLogger(
+      RetryTestTokenProvider.class);
 
-  private static final Logger LOG = LoggerFactory
-      .getLogger(RetryTestTokenProvider.class);
+  // Need to track first token fetch otherwise will get counted as a retry too.
+  private boolean isThisFirstTokenFetch = true;
+  private int retryCount = 0;
 
   @Override
   public void initialize(Configuration configuration, String accountName)
@@ -43,9 +43,13 @@ public void initialize(Configuration configuration, String accountName)
 
   }
 
-  public static void ResetStatusToFirstTokenFetch() {
+  /**
+   * Clear earlier retry details and reset RetryTestTokenProvider instance to
+   * state of first access token fetch call.
+   */
+  public void resetStatusToFirstTokenFetch() {
     isThisFirstTokenFetch = true;
-    reTryCount = 0;
+    retryCount = 0;
   }
 
   @Override
@@ -53,7 +57,7 @@ public String getAccessToken() throws IOException {
     if (isThisFirstTokenFetch) {
       isThisFirstTokenFetch = false;
     } else {
-      reTryCount++;
+      retryCount++;
     }
 
     LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens");
@@ -64,4 +68,13 @@ public String getAccessToken() throws IOException {
   public Date getExpiryTime() {
     return new Date();
   }
+
+  public static RetryTestTokenProvider getCurrentRetryTestProviderInstance(
+      AccessTokenProvider customTokenProvider) {
+    return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee();
+  }
+
+  public int getRetryCount() {
+    return retryCount;
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java
index a5d60c4b41a92..66f072501dc4d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java
@@ -81,7 +81,7 @@ protected void testWithNoOptimization(final FileSystem fs,
         }
         assertEquals(length, bytesRead);
         assertContentReadCorrectly(fileContent,
-            (int) (seekPos + totalBytesRead - length), length, buffer);
+            (int) (seekPos + totalBytesRead - length), length, buffer, testFilePath);
 
         assertTrue(abfsInputStream.getFCursor() >= seekPos + totalBytesRead);
         assertTrue(abfsInputStream.getFCursorAfterLastRead() >= seekPos + totalBytesRead);
@@ -133,7 +133,7 @@ private void testExceptionInOptimization(final FileSystem fs,
         actualLength = length - delta;
       }
       assertEquals(bytesRead, actualLength);
-      assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer);
+      assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
       assertEquals(fileContent.length, abfsInputStream.getFCursor());
       assertEquals(fileContent.length, abfsInputStream.getFCursorAfterLastRead());
       assertEquals(actualLength, abfsInputStream.getBCursor());
@@ -200,24 +200,24 @@ protected Map<String, Long> getInstrumentationMap(FileSystem fs)
   }
 
   protected void assertContentReadCorrectly(byte[] actualFileContent, int from,
-      int len, byte[] contentRead) {
+      int len, byte[] contentRead, Path testFilePath) {
     for (int i = 0; i < len; i++) {
-      assertEquals(contentRead[i], actualFileContent[i + from]);
+      assertEquals("The test file path is " + testFilePath, contentRead[i], actualFileContent[i + from]);
     }
   }
 
   protected void assertBuffersAreNotEqual(byte[] actualContent,
-      byte[] contentRead, AbfsConfiguration conf) {
-    assertBufferEquality(actualContent, contentRead, conf, false);
+      byte[] contentRead, AbfsConfiguration conf, Path testFilePath) {
+    assertBufferEquality(actualContent, contentRead, conf, false, testFilePath);
   }
 
   protected void assertBuffersAreEqual(byte[] actualContent, byte[] contentRead,
-      AbfsConfiguration conf) {
-    assertBufferEquality(actualContent, contentRead, conf, true);
+      AbfsConfiguration conf, Path testFilePath) {
+    assertBufferEquality(actualContent, contentRead, conf, true, testFilePath);
   }
 
   private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
-      AbfsConfiguration conf, boolean assertEqual) {
+      AbfsConfiguration conf, boolean assertEqual, Path testFilePath) {
     int bufferSize = conf.getReadBufferSize();
     int actualContentSize = actualContent.length;
     int n = (actualContentSize < bufferSize) ? actualContentSize : bufferSize;
@@ -228,9 +228,9 @@ private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
       }
     }
     if (assertEqual) {
-      assertEquals(n, matches);
+      assertEquals("The test file path is " + testFilePath, n, matches);
     } else {
-      assertNotEquals(n, matches);
+      assertNotEquals("The test file path is " + testFilePath, n, matches);
     }
   }
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
index 1cb8ed3ea8cee..cb3eaffe029c9 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
@@ -235,14 +235,14 @@ private void seekReadAndTest(final FileSystem fs, final Path testFilePath,
       assertEquals(expectedBCurson, abfsInputStream.getBCursor());
       assertEquals(actualLength, bytesRead);
       //  Verify user-content read
-      assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer);
+      assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
       //  Verify data read to AbfsInputStream buffer
       int from = seekPos;
       if (optimizationOn) {
         from = (int) max(0, actualContentLength - bufferSize);
       }
       assertContentReadCorrectly(fileContent, from, (int) abfsInputStream.getLimit(),
-          abfsInputStream.getBuffer());
+          abfsInputStream.getBuffer(), testFilePath);
     }
   }
 
@@ -280,7 +280,7 @@ private void testPartialReadWithNoData(final FileSystem fs,
       byte[] buffer = new byte[length];
       int bytesRead = iStream.read(buffer, 0, length);
       assertEquals(length, bytesRead);
-      assertContentReadCorrectly(fileContent, seekPos, length, buffer);
+      assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
       assertEquals(fileContent.length, abfsInputStream.getFCursor());
       assertEquals(length, abfsInputStream.getBCursor());
       assertTrue(abfsInputStream.getLimit() >= length);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java
index d85da5aaeabd7..baa664d6e7fee 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java
@@ -189,7 +189,7 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
       byte[] buffer = new byte[length];
       int bytesRead = iStream.read(buffer, 0, length);
       assertEquals(bytesRead, length);
-      assertContentReadCorrectly(fileContent, seekPos, length, buffer);
+      assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
       AbfsInputStream abfsInputStream = (AbfsInputStream) iStream
           .getWrappedStream();
 
@@ -199,16 +199,16 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
       int expectedLimit, expectedFCursor;
       int expectedBCursor;
       if (conf.readSmallFilesCompletely() && smallFile) {
-        assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf);
+        assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
         expectedFCursor = fileContentLength;
         expectedLimit = fileContentLength;
         expectedBCursor = seekPos + length;
       } else {
         if ((seekPos == 0)) {
-          assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf);
+          assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
         } else {
           assertBuffersAreNotEqual(fileContent, abfsInputStream.getBuffer(),
-              conf);
+              conf, testFilePath);
         }
         expectedBCursor = length;
         expectedFCursor = (fileContentLength < (seekPos + readBufferSize))
@@ -260,7 +260,7 @@ private void partialReadWithNoData(final FileSystem fs,
       byte[] buffer = new byte[length];
       int bytesRead = iStream.read(buffer, 0, length);
       assertEquals(bytesRead, length);
-      assertContentReadCorrectly(fileContent, seekPos, length, buffer);
+      assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
       assertEquals(fileContent.length, abfsInputStream.getFCursor());
       assertEquals(fileContent.length,
           abfsInputStream.getFCursorAfterLastRead());
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
index a725bf3175a5c..08eb3adc92697 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
@@ -306,6 +306,11 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance,
     when(client.getAccessToken()).thenCallRealMethod();
     when(client.getSharedKeyCredentials()).thenCallRealMethod();
     when(client.createDefaultHeaders()).thenCallRealMethod();
+    when(client.getAbfsConfiguration()).thenReturn(abfsConfig);
+    when(client.getIntercept()).thenReturn(
+        AbfsThrottlingInterceptFactory.getInstance(
+            abfsConfig.getAccountName().substring(0,
+                abfsConfig.getAccountName().indexOf(DOT)), abfsConfig));
 
     // override baseurl
     client = TestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
@@ -395,4 +400,8 @@ public static AbfsRestOperation getRestOp(AbfsRestOperationType type,
         url,
         requestHeaders);
   }
+
+  public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
+    return client.getTokenProvider();
+  }
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClientThrottlingAnalyzer.java
index 3f680e499300d..22649cd190d83 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClientThrottlingAnalyzer.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClientThrottlingAnalyzer.java
@@ -18,9 +18,15 @@
 
 package org.apache.hadoop.fs.azurebfs.services;
 
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.junit.Test;
 
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ANALYSIS_PERIOD;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -33,6 +39,15 @@ public class TestAbfsClientThrottlingAnalyzer {
       + ANALYSIS_PERIOD / 10;
   private static final long MEGABYTE = 1024 * 1024;
   private static final int MAX_ACCEPTABLE_PERCENT_DIFFERENCE = 20;
+  private AbfsConfiguration abfsConfiguration;
+
+  public TestAbfsClientThrottlingAnalyzer() throws IOException, IllegalAccessException {
+    final Configuration configuration = new Configuration();
+    configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+    configuration.setInt(FS_AZURE_ANALYSIS_PERIOD, 1000);
+    this.abfsConfiguration = new AbfsConfiguration(configuration,
+            "dummy");
+  }
 
   private void sleep(long milliseconds) {
     try {
@@ -82,8 +97,7 @@ private void validateLessThanOrEqual(long maxExpected, long actual) {
   @Test
   public void testNoMetricUpdatesThenNoWaiting() {
     AbfsClientThrottlingAnalyzer analyzer = new AbfsClientThrottlingAnalyzer(
-        "test",
-        ANALYSIS_PERIOD);
+        "test", abfsConfiguration);
     validate(0, analyzer.getSleepDuration());
     sleep(ANALYSIS_PERIOD_PLUS_10_PERCENT);
     validate(0, analyzer.getSleepDuration());
@@ -96,8 +110,7 @@ public void testNoMetricUpdatesThenNoWaiting() {
   @Test
   public void testOnlySuccessThenNoWaiting() {
     AbfsClientThrottlingAnalyzer analyzer = new AbfsClientThrottlingAnalyzer(
-        "test",
-        ANALYSIS_PERIOD);
+        "test", abfsConfiguration);
     analyzer.addBytesTransferred(8 * MEGABYTE, false);
     validate(0, analyzer.getSleepDuration());
     sleep(ANALYSIS_PERIOD_PLUS_10_PERCENT);
@@ -112,8 +125,7 @@ public void testOnlySuccessThenNoWaiting() {
   @Test
   public void testOnlyErrorsAndWaiting() {
     AbfsClientThrottlingAnalyzer analyzer = new AbfsClientThrottlingAnalyzer(
-        "test",
-        ANALYSIS_PERIOD);
+        "test", abfsConfiguration);
     validate(0, analyzer.getSleepDuration());
     analyzer.addBytesTransferred(4 * MEGABYTE, true);
     sleep(ANALYSIS_PERIOD_PLUS_10_PERCENT);
@@ -132,8 +144,7 @@ public void testOnlyErrorsAndWaiting() {
   @Test
   public void testSuccessAndErrorsAndWaiting() {
     AbfsClientThrottlingAnalyzer analyzer = new AbfsClientThrottlingAnalyzer(
-        "test",
-        ANALYSIS_PERIOD);
+        "test", abfsConfiguration);
     validate(0, analyzer.getSleepDuration());
     analyzer.addBytesTransferred(8 * MEGABYTE, false);
     analyzer.addBytesTransferred(2 * MEGABYTE, true);
@@ -157,8 +168,7 @@ public void testSuccessAndErrorsAndWaiting() {
   @Test
   public void testManySuccessAndErrorsAndWaiting() {
     AbfsClientThrottlingAnalyzer analyzer = new AbfsClientThrottlingAnalyzer(
-        "test",
-        ANALYSIS_PERIOD);
+        "test", abfsConfiguration);
     validate(0, analyzer.getSleepDuration());
     final int numberOfRequests = 20;
     for (int i = 0; i < numberOfRequests; i++) {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
index b5ae9b737842d..69795ee5bd857 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java
@@ -106,6 +106,7 @@ private AbfsClient getMockAbfsClient() {
   private AbfsInputStream getAbfsInputStream(AbfsClient mockAbfsClient,
       String fileName) throws IOException {
     AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1);
+    inputStreamContext.isReadAheadEnabled(true);
     // Create AbfsInputStream with the client instance
     AbfsInputStream inputStream = new AbfsInputStream(
         mockAbfsClient,
@@ -131,6 +132,7 @@ public AbfsInputStream getAbfsInputStream(AbfsClient abfsClient,
       boolean alwaysReadBufferSize,
       int readAheadBlockSize) throws IOException {
     AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1);
+    inputStreamContext.isReadAheadEnabled(true);
     // Create AbfsInputStream with the client instance
     AbfsInputStream inputStream = new AbfsInputStream(
         abfsClient,
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java
new file mode 100644
index 0000000000000..65ea79b36bd0e
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
+import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PUT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Testing Abfs Rename recovery using Mockito.
+ */
+public class TestAbfsRenameRetryRecovery extends AbstractAbfsIntegrationTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestAbfsRenameRetryRecovery.class);
+
+  public TestAbfsRenameRetryRecovery() throws Exception {
+  }
+
+  /**
+   * Mock the AbfsClient to run a metadata incomplete scenario with recovery
+   * rename.
+   */
+  @Test
+  public void testRenameFailuresDueToIncompleteMetadata() throws Exception {
+    String sourcePath = getMethodName() + "Source";
+    String destNoParentPath = "/NoParent/Dest";
+    AzureBlobFileSystem fs = getFileSystem();
+
+    AbfsClient mockClient = TestAbfsClient.getMockAbfsClient(
+        fs.getAbfsStore().getClient(),
+        fs.getAbfsStore().getAbfsConfiguration());
+
+    AbfsCounters abfsCounters = mock(AbfsCounters.class);
+    when(mockClient.getAbfsCounters()).thenReturn(abfsCounters);
+    // SuccessFul Result.
+    AbfsRestOperation successOp =
+        new AbfsRestOperation(AbfsRestOperationType.RenamePath, mockClient,
+            HTTP_METHOD_PUT, null, null);
+    AbfsClientRenameResult successResult = mock(AbfsClientRenameResult.class);
+    doReturn(successOp).when(successResult).getOp();
+    when(successResult.isIncompleteMetadataState()).thenReturn(false);
+
+    // Failed Result.
+    AbfsRestOperation failedOp = new AbfsRestOperation(AbfsRestOperationType.RenamePath, mockClient,
+        HTTP_METHOD_PUT, null, null);
+    AbfsClientRenameResult recoveredMetaDataIncompleteResult =
+        mock(AbfsClientRenameResult.class);
+
+    doReturn(failedOp).when(recoveredMetaDataIncompleteResult).getOp();
+    when(recoveredMetaDataIncompleteResult.isIncompleteMetadataState()).thenReturn(true);
+
+    // No destination Parent dir exception.
+    AzureBlobFileSystemException destParentNotFound
+        = getMockAbfsRestOperationException(
+        RENAME_DESTINATION_PARENT_PATH_NOT_FOUND.getStatusCode(),
+        RENAME_DESTINATION_PARENT_PATH_NOT_FOUND.getErrorCode());
+
+    // We need to throw an exception once a rename is triggered with
+    // destination having no parent, but after a retry it needs to succeed.
+    when(mockClient.renamePath(sourcePath, destNoParentPath, null, null,
+        null, false))
+        .thenThrow(destParentNotFound)
+        .thenReturn(recoveredMetaDataIncompleteResult);
+
+    // Dest parent not found exc. to be raised.
+    intercept(AzureBlobFileSystemException.class,
+        () -> mockClient.renamePath(sourcePath,
+        destNoParentPath, null, null,
+        null, false));
+
+    AbfsClientRenameResult resultOfSecondRenameCall =
+        mockClient.renamePath(sourcePath,
+        destNoParentPath, null, null,
+        null, false);
+
+    // the second rename call should be the recoveredResult due to
+    // metaDataIncomplete
+    Assertions.assertThat(resultOfSecondRenameCall)
+        .describedAs("This result should be recovered result due to MetaData "
+            + "being in incomplete state")
+        .isSameAs(recoveredMetaDataIncompleteResult);
+    // Verify Incomplete metadata state happened for our second rename call.
+    assertTrue("Metadata incomplete state should be true if a rename is "
+            + "retried after no Parent directory is found",
+        resultOfSecondRenameCall.isIncompleteMetadataState());
+
+
+    // Verify renamePath occurred two times implying a retry was attempted.
+    verify(mockClient, times(2))
+        .renamePath(sourcePath, destNoParentPath, null, null, null, false);
+
+  }
+
+  /**
+   * Method to create an AbfsRestOperationException.
+   * @param statusCode status code to be used.
+   * @param errorCode error code to be used.
+   * @return the exception.
+   */
+  private AbfsRestOperationException getMockAbfsRestOperationException(
+      int statusCode, String errorCode) {
+    return new AbfsRestOperationException(statusCode, errorCode,
+        "No Parent found for the Destination file",
+        new Exception());
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
index 0f8dc55aa14a4..a1fc4e138d6cd 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
@@ -18,13 +18,35 @@
 
 package org.apache.hadoop.fs.azurebfs.services;
 
+import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
+
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_BACKOFF_INTERVAL;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_BACKOFF_INTERVAL;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MIN_BACKOFF_INTERVAL;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENABLE_AUTOTHROTTLING;
+import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT1_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME;
+
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.fs.FSDataInputStream;
 
+import org.assertj.core.api.Assertions;
+import org.junit.Assume;
+import org.mockito.Mockito;
+
+import java.net.URI;
 import java.util.Random;
 
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -41,6 +63,9 @@ public class TestExponentialRetryPolicy extends AbstractAbfsIntegrationTest {
   private final int noRetryCount = 0;
   private final int retryCount = new Random().nextInt(maxRetryCount);
   private final int retryCountBeyondMax = maxRetryCount + 1;
+  private static final String TEST_PATH = "/testfile";
+  private static final double MULTIPLYING_FACTOR = 1.5;
+  private static final int ANALYSIS_PERIOD = 10000;
 
 
   public TestExponentialRetryPolicy() throws Exception {
@@ -67,6 +92,173 @@ public void testDefaultMaxIORetryCount() throws Exception {
     testMaxIOConfig(abfsConfig);
   }
 
+  @Test
+  public void testThrottlingIntercept() throws Exception {
+    AzureBlobFileSystem fs = getFileSystem();
+    final Configuration configuration = new Configuration();
+    configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+    configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, false);
+
+    // On disabling throttling AbfsNoOpThrottlingIntercept object is returned
+    AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+        "dummy.dfs.core.windows.net");
+    AbfsThrottlingIntercept intercept;
+    AbfsClient abfsClient = TestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration);
+    intercept = abfsClient.getIntercept();
+    Assertions.assertThat(intercept)
+        .describedAs("AbfsNoOpThrottlingIntercept instance expected")
+        .isInstanceOf(AbfsNoOpThrottlingIntercept.class);
+
+    configuration.setBoolean(FS_AZURE_ENABLE_AUTOTHROTTLING, true);
+    configuration.setBoolean(FS_AZURE_ACCOUNT_LEVEL_THROTTLING_ENABLED, true);
+    // On disabling throttling AbfsClientThrottlingIntercept object is returned
+    AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration,
+        "dummy1.dfs.core.windows.net");
+    AbfsClient abfsClient1 = TestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1);
+    intercept = abfsClient1.getIntercept();
+    Assertions.assertThat(intercept)
+        .describedAs("AbfsClientThrottlingIntercept instance expected")
+        .isInstanceOf(AbfsClientThrottlingIntercept.class);
+  }
+
+  @Test
+  public void testCreateMultipleAccountThrottling() throws Exception {
+    Configuration config = new Configuration(getRawConfiguration());
+    String accountName = config.get(FS_AZURE_ACCOUNT_NAME);
+    if (accountName == null) {
+      // check if accountName is set using different config key
+      accountName = config.get(FS_AZURE_ABFS_ACCOUNT1_NAME);
+    }
+    assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT1_NAME,
+        accountName != null && !accountName.isEmpty());
+
+    Configuration rawConfig1 = new Configuration();
+    rawConfig1.addResource(TEST_CONFIGURATION_FILE_NAME);
+
+    AbfsRestOperation successOp = mock(AbfsRestOperation.class);
+    AbfsHttpOperation http500Op = mock(AbfsHttpOperation.class);
+    when(http500Op.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR);
+    when(successOp.getResult()).thenReturn(http500Op);
+
+    AbfsConfiguration configuration = Mockito.mock(AbfsConfiguration.class);
+    when(configuration.getAnalysisPeriod()).thenReturn(ANALYSIS_PERIOD);
+    when(configuration.isAutoThrottlingEnabled()).thenReturn(true);
+    when(configuration.accountThrottlingEnabled()).thenReturn(false);
+
+    AbfsThrottlingIntercept instance1 = AbfsThrottlingInterceptFactory.getInstance(accountName, configuration);
+    String accountName1 = config.get(FS_AZURE_ABFS_ACCOUNT1_NAME);
+
+    assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT1_NAME,
+        accountName1 != null && !accountName1.isEmpty());
+
+    AbfsThrottlingIntercept instance2 = AbfsThrottlingInterceptFactory.getInstance(accountName1, configuration);
+    //if singleton is enabled, for different accounts both the instances should return same value
+    Assertions.assertThat(instance1)
+        .describedAs(
+            "if singleton is enabled, for different accounts both the instances should return same value")
+        .isEqualTo(instance2);
+
+    when(configuration.accountThrottlingEnabled()).thenReturn(true);
+    AbfsThrottlingIntercept instance3 = AbfsThrottlingInterceptFactory.getInstance(accountName, configuration);
+    AbfsThrottlingIntercept instance4 = AbfsThrottlingInterceptFactory.getInstance(accountName1, configuration);
+    AbfsThrottlingIntercept instance5 = AbfsThrottlingInterceptFactory.getInstance(accountName, configuration);
+    //if singleton is not enabled, for different accounts instances should return different value
+    Assertions.assertThat(instance3)
+        .describedAs(
+            "iff singleton is not enabled, for different accounts instances should return different value")
+        .isNotEqualTo(instance4);
+
+    //if singleton is not enabled, for same accounts instances should return same value
+    Assertions.assertThat(instance3)
+        .describedAs(
+            "if singleton is not enabled, for same accounts instances should return same value")
+        .isEqualTo(instance5);
+  }
+
+  @Test
+  public void testOperationOnAccountIdle() throws Exception {
+    //Get the filesystem.
+    AzureBlobFileSystem fs = getFileSystem();
+    AbfsClient client = fs.getAbfsStore().getClient();
+    AbfsConfiguration configuration1 = client.getAbfsConfiguration();
+    Assume.assumeTrue(configuration1.isAutoThrottlingEnabled());
+    Assume.assumeTrue(configuration1.accountThrottlingEnabled());
+
+    AbfsClientThrottlingIntercept accountIntercept
+        = (AbfsClientThrottlingIntercept) client.getIntercept();
+    final byte[] b = new byte[2 * MIN_BUFFER_SIZE];
+    new Random().nextBytes(b);
+
+    Path testPath = path(TEST_PATH);
+
+    //Do an operation on the filesystem.
+    try (FSDataOutputStream stream = fs.create(testPath)) {
+      stream.write(b);
+    }
+
+    //Don't perform any operation on the account.
+    int sleepTime = (int) ((getAbfsConfig().getAccountOperationIdleTimeout()) * MULTIPLYING_FACTOR);
+    Thread.sleep(sleepTime);
+
+    try (FSDataInputStream streamRead = fs.open(testPath)) {
+      streamRead.read(b);
+    }
+
+    //Perform operations on another account.
+    AzureBlobFileSystem fs1 = new AzureBlobFileSystem();
+    Configuration config = new Configuration(getRawConfiguration());
+    String accountName1 = config.get(FS_AZURE_ABFS_ACCOUNT1_NAME);
+    assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT1_NAME,
+        accountName1 != null && !accountName1.isEmpty());
+    final String abfsUrl1 = this.getFileSystemName() + "12" + "@" + accountName1;
+    URI defaultUri1 = null;
+    defaultUri1 = new URI("abfss", abfsUrl1, null, null, null);
+    fs1.initialize(defaultUri1, getRawConfiguration());
+    AbfsClient client1 = fs1.getAbfsStore().getClient();
+    AbfsClientThrottlingIntercept accountIntercept1
+        = (AbfsClientThrottlingIntercept) client1.getIntercept();
+    try (FSDataOutputStream stream1 = fs1.create(testPath)) {
+      stream1.write(b);
+    }
+
+    //Verify the write analyzer for first account is idle but the read analyzer is not idle.
+    Assertions.assertThat(accountIntercept.getWriteThrottler()
+            .getIsOperationOnAccountIdle()
+            .get())
+        .describedAs("Write analyzer for first account should be idle the first time")
+        .isTrue();
+
+    Assertions.assertThat(
+            accountIntercept.getReadThrottler()
+                .getIsOperationOnAccountIdle()
+                .get())
+        .describedAs("Read analyzer for first account should not be idle")
+        .isFalse();
+
+    //Verify the write analyzer for second account is not idle.
+    Assertions.assertThat(
+            accountIntercept1.getWriteThrottler()
+                .getIsOperationOnAccountIdle()
+                .get())
+        .describedAs("Write analyzer for second account should not be idle")
+        .isFalse();
+
+    //Again perform an operation on the first account.
+    try (FSDataOutputStream stream2 = fs.create(testPath)) {
+      stream2.write(b);
+    }
+
+    //Verify the write analyzer on first account is not idle.
+    Assertions.assertThat(
+
+            accountIntercept.getWriteThrottler()
+                .getIsOperationOnAccountIdle()
+                .get())
+        .describedAs(
+            "Write analyzer for first account should not be idle second time")
+        .isFalse();
+  }
+
   @Test
   public void testAbfsConfigConstructor() throws Exception {
     // Ensure we choose expected values that are not defaults
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java
new file mode 100644
index 0000000000000..b8272319ab851
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
+
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+
+/**
+ * This looks like a test, but it is really a command to invoke to
+ * clean up containers created in other test runs.
+ *
+ */
+public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
+  private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
+  private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
+
+  public CleanupTestContainers() throws Exception {
+  }
+
+  @org.junit.Test
+  public void testDeleteContainers() throws Throwable {
+    int count = 0;
+    AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
+    String accountName = abfsConfig.getAccountName().split("\\.")[0];
+    LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
+
+    String accountKey = abfsConfig.getStorageAccountKey();
+    if ((accountKey == null) || (accountKey.isEmpty())) {
+      LOG.debug("Clean up not possible. Account ket not present in config");
+    }
+    final StorageCredentials credentials;
+    credentials = new StorageCredentialsAccountAndKey(
+        accountName, accountKey);
+    CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
+    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+    Iterable<CloudBlobContainer> containers
+        = blobClient.listContainers(CONTAINER_PREFIX);
+    for (CloudBlobContainer container : containers) {
+      LOG.info("Container {} URI {}",
+          container.getName(),
+          container.getUri());
+      if (container.deleteIfExists()) {
+        count++;
+        LOG.info("Current deleted test containers count - #{}", count);
+      }
+    }
+    LOG.info("Summary: Deleted {} test containers", count);
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template
new file mode 100644
index 0000000000000..062b2f4bf3ad1
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template
@@ -0,0 +1,185 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  NOTE:
+  Each config file will hold configs for one storage account.
+  ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
+  Below template collects information needed for both Auth settings on the account.
+
+  OAuth test mode works with token provider type - ClientCredsTokenProvider
+  For specific details on test config or pre-requisite, refer to
+  https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+
+  Pre-requisite before configuring:
+  1. An ABFS account
+  2. A container within account
+  3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
+  4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
+  5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
+  6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
+
+  Replace below strings with actual value:
+  ## ACCOUNT SETTINGS ##
+    1. ACCOUNTNAME -> to account name without domain
+    2. IS_NAMESPACE_ENABLED -> to true or false
+    3. CONTAINER_NAME -> name of an existing container
+    4. ACCOUNT_KEY -> Account access key
+  ## IDENTITY SETTINGS ##
+    5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
+    6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
+    7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
+
+    8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
+    9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
+    10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
+
+    11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
+    12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
+
+    13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
+    14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
+-->
+
+<configuration>
+
+  <!-- ACCOUNT SETTINGS -->
+  <property>
+    <name>fs.azure.abfs.account.name</name>
+    <value>ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.abfs</name>
+    <value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.abfss</name>
+    <value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.wasb.account.name</name>
+    <value>ACCOUNTNAME.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.scale.test.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.test.namespace.enabled</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.hns.enabled</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+
+  <!-- SUPERUSER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.endpoint</name>
+    <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth.provider.type</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>SUPERUSER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.id</name>
+    <value>SUPERUSER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>SUPERUSER_CLIENT_SECRET</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.secret</name>
+    <value>SUPERUSER_CLIENT_SECRET</value>
+  </property>
+
+  <!-- NO RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.enable.check.access</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>fs.azure.account.test.oauth2.client.id</name>
+    <value>NO_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.test.oauth2.client.secret</name>
+    <value>NO_RBAC_USER_CLIENT_SECRET</value>
+  </property>
+  <property>
+    <name>fs.azure.check.access.testuser.guid</name>
+    <value>NO_RBAC_USER_OID</value>
+  </property>
+
+  <!-- CONTRIBUTOR RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.id</name>
+    <value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.secret</name>
+    <value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
+  </property>
+
+  <!-- READER RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.id</name>
+    <value>READER_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.secret</name>
+    <value>READER_RBAC_USER_CLIENT_ID</value>
+  </property>
+</configuration>
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
index 12dbbfab47970..636816551ddf7 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
@@ -14,162 +14,16 @@
 -->
 
 <configuration>
+  <!-- Account names without domain -->
 
-  <!--====================== IMPORTANT!! ========================-->
-  <!--
-      1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
-      2. UPDATE runtests.sh
-         A.  "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
-             ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
-             SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
-         B.  PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
-             DIFFERENT VARIANTS OF CONFIGS.
-      3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
-      4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
-  -->
-
-  <!--=============== Auth type ===============-->
-  <property>
-    <name>fs.azure.account.auth.type</name>
-    <value>SharedKey</value>
-  </property>
-
-  <!--=============== Auth related accounts ===============-->
-  <!-- This set of configs needs to be provided for all the accounts with which
-   the tests needs to be ran. -->
-  <property>
-    <name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
-    <value>{ACCOUNT_ACCESS_KEY}</value>
-    <description>Account access key</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
-    <description>OAuth token provider implementation class</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>Token end point, this can be found through Azure portal
-    </description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>{client id}</value>
-    <description>AAD client id.</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>{client secret}</value>
-    <description>AAD client secret</description>
-  </property>
-
-  <!--=============== Configs for hadoop contract tests for ABFS ===============-->
-  <property>
-    <name>fs.contract.test.fs.abfs</name>
-    <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.contract.test.fs.abfss</name>
-    <value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
-  </property>
-
-  <!--=============== WASB Configs ===============-->
-  <property>
-    <name>fs.azure.wasb.account.name</name>
-    <value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
-    <value>WASB account key</value>
-  </property>
-  <property>
-    <name>fs.contract.test.fs.wasb</name>
-    <value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
-    </value>
-  </property>
-
-  <!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.id</name>
-    <value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.secret</name>
-    <value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.id</name>
-    <value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.secret</name>
-    <value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
-  </property>
-
-<!--===========================   FOR CheckAccess =========================-->
-<!-- To run ABFS CheckAccess tests, you must register an app, with no role
- assignments, and set the configuration discussed below:
-
-    1) Register a new app with no RBAC
-    2) As part of the test configs you need to provide the guid for the above
-created app. Please follow the below steps to fetch the guid.
-      a) Get an access token with the above created app. Please refer the
- following documentation for the same. https://docs.microsoft
-.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
-      b) Decode the token fetched with the above step. You may use https
-://jwt.ms/ to decode the token
-      d) The oid field in the decoded string is the guid.
-    3) Set the following configurations:
--->
-
-  <property>
-    <name>fs.azure.account.test.oauth2.client.id</name>
-    <value>{client id}</value>
-    <description>The client id(app id) for the app created on step 1
-    </description>
-  </property>
-  <property>
-    <name>fs.azure.account.test.oauth2.client.secret</name>
-    <value>{client secret}</value>
-    <description>
-The client secret(application's secret) for the app created on step 1
-    </description>
-  </property>
-  <property>
-    <name>fs.azure.check.access.testuser.guid</name>
-    <value>{guid}</value>
-    <description>The guid fetched on step 2</description>
-  </property>
   <property>
-    <name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
-.windows.net</name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>
-Token end point. This can be found through Azure portal. As part of CheckAccess
-test cases. The access will be tested for an FS instance created with the
-above mentioned client credentials. So this configuration is necessary to
-create the test FS instance.
-    </description>
+    <name>fs.azure.hnsTestAccountName</name>
+    <value></value>
   </property>
 
-  <!--========== Append blob related configs ===========-->
   <property>
-    <name>fs.azure.test.appendblob.enabled</name>
-    <value>false</value>
-    <description>If made true, tests will be running under the assumption that
-      append blob is enabled and the root directory and contract test root
-      directory will be part of the append blob directories. Should be false for
-      non-HNS accounts.
-    </description>
+    <name>fs.azure.nonHnsTestAccountName</name>
+    <value></value>
   </property>
 
 </configuration>
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
index 24ffeb5d107a0..6730021974e5f 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
+++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
@@ -28,11 +28,8 @@
     <value>false</value>
   </property>
 
-  <property>
-    <name>fs.azure.test.namespace.enabled</name>
-    <value>true</value>
-  </property>
-
+  <!--====================  ABFS CONFIGURATION ====================-->
+  <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
   <property>
     <name>fs.azure.abfs.latency.track</name>
     <value>false</value>
@@ -43,9 +40,6 @@
     <value>true</value>
   </property>
 
-  <!--====================  ABFS CONFIGURATION ====================-->
-  <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
-
   <!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
   <property>
     <name>fs.azure.user.agent.prefix</name>
@@ -59,7 +53,40 @@
       STORE THE CONFIGURATION PROPERTIES WITHIN IT.
       TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
       THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
-      IT IN PATCHES OR COMMITS. -->
+      IT IN PATCHES OR COMMITS.
+
+      TEST SCRIPT RUNS:
+      ================
+      FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
+      TRIGGERED OVER SCRIPT:
+      ./dev-support/testrun-scripts/runtests.sh
+      (FROM hadoop-azure ROOT PROJECT PATH)
+
+      TO USE THE TEST SCRIPT,
+      1. COPY
+         ./src/test/resources/azure-auth-keys.xml.template
+         TO
+         ./src/test/resources/azure-auth-keys.xml
+         UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
+         FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
+         PRESENT IN THE XML, NAMELY
+         fs.azure.hnsTestAccountName and
+         fs.azure.nonHnsTestAccountName
+         (ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
+
+         azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
+         ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
+
+      2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
+         PER ACCOUNT) IN FOLDER:
+         ./src/test/resources/accountSettings/
+
+         FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
+         accountName_settings.xml.template
+         WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
+
+         NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
+         TO PREVENT ACCIDENTAL CRED LEAKS. -->
   <!--=============================================================-->
 
   <include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml b/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml
new file mode 100644
index 0000000000000..7d2d11c04eff3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<configuration>
+  <include xmlns="http://www.w3.org/2001/XInclude" href="azure-test.xml">
+    <fallback/>
+  </include>
+</configuration>
diff --git a/hadoop-tools/hadoop-benchmark/pom.xml b/hadoop-tools/hadoop-benchmark/pom.xml
new file mode 100644
index 0000000000000..3d742fab5c877
--- /dev/null
+++ b/hadoop-tools/hadoop-benchmark/pom.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.4.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project/pom.xml</relativePath>
+  </parent>
+  <artifactId>hadoop-benchmark</artifactId>
+  <version>3.4.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <name>Apache Hadoop Common Benchmark</name>
+  <description>Apache Hadoop Common Benchmark</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <mainClass>org.apache.hadoop.benchmark.VectoredReadBenchmark</mainClass>
+            </manifest>
+          </archive>
+          <descriptors>
+            <descriptor>src/main/assembly/uber.xml</descriptor>
+          </descriptors>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id> <!-- this is used for inheritance merges -->
+            <phase>package</phase> <!-- bind to the packaging phase -->
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>findbugs-maven-plugin</artifactId>
+          <configuration>
+            <excludeFilterFile>${basedir}/src/main/findbugs/exclude.xml</excludeFilterFile>
+          </configuration>
+      </plugin>
+      <plugin>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
+        <configuration>
+          <excludeFilterFile>${basedir}/src/main/findbugs/exclude.xml</excludeFilterFile>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/hadoop-tools/hadoop-benchmark/src/main/assembly/uber.xml b/hadoop-tools/hadoop-benchmark/src/main/assembly/uber.xml
new file mode 100644
index 0000000000000..014eab951b3cf
--- /dev/null
+++ b/hadoop-tools/hadoop-benchmark/src/main/assembly/uber.xml
@@ -0,0 +1,33 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<assembly>
+  <id>uber</id>
+  <formats>
+    <format>jar</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>/</outputDirectory>
+      <useProjectArtifact>true</useProjectArtifact>
+      <unpack>true</unpack>
+      <scope>runtime</scope>
+    </dependencySet>
+  </dependencySets>
+  <containerDescriptorHandlers>
+    <containerDescriptorHandler>
+      <handlerName>metaInf-services</handlerName>
+    </containerDescriptorHandler>
+  </containerDescriptorHandlers>
+</assembly>
diff --git a/hadoop-tools/hadoop-benchmark/src/main/findbugs/exclude.xml b/hadoop-tools/hadoop-benchmark/src/main/findbugs/exclude.xml
new file mode 100644
index 0000000000000..05f2a067cf01e
--- /dev/null
+++ b/hadoop-tools/hadoop-benchmark/src/main/findbugs/exclude.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<FindBugsFilter>
+  <Match>
+    <Class name="~org\.apache\.hadoop\.benchmark\.generated.*"/>
+  </Match>
+  <Match>
+    <Class name="~org\.openjdk\.jmh\.infra\.generated.*"/>
+  </Match>
+</FindBugsFilter>
diff --git a/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/VectoredReadBenchmark.java b/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/VectoredReadBenchmark.java
new file mode 100644
index 0000000000000..5df46c36786df
--- /dev/null
+++ b/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/VectoredReadBenchmark.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.benchmark;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.CompletionHandler;
+import java.nio.file.FileSystems;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.IntFunction;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileRange;
+import org.apache.hadoop.fs.impl.FileRangeImpl;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+public class VectoredReadBenchmark {
+
+  static final Path DATA_PATH = getTestDataPath();
+  static final String DATA_PATH_PROPERTY = "bench.data";
+  static final int READ_SIZE = 64 * 1024;
+  static final long SEEK_SIZE = 1024L * 1024;
+
+
+  static Path getTestDataPath() {
+    String value = System.getProperty(DATA_PATH_PROPERTY);
+    return new Path(value == null ? "/tmp/taxi.orc" : value);
+  }
+
+  @State(Scope.Thread)
+  public static class FileSystemChoice {
+
+    @Param({"local", "raw"})
+    private String fileSystemKind;
+
+    private Configuration conf;
+    private FileSystem fs;
+
+    @Setup(Level.Trial)
+    public void setup() {
+      conf = new Configuration();
+      try {
+        LocalFileSystem local = FileSystem.getLocal(conf);
+        fs = "raw".equals(fileSystemKind) ? local.getRaw() : local;
+      } catch (IOException e) {
+        throw new IllegalArgumentException("Can't get filesystem", e);
+      }
+    }
+  }
+
+  @State(Scope.Thread)
+  public static class BufferChoice {
+    @Param({"direct", "array"})
+    private String bufferKind;
+
+    private IntFunction<ByteBuffer> allocate;
+    @Setup(Level.Trial)
+    public void setup() {
+      allocate = "array".equals(bufferKind)
+                     ? ByteBuffer::allocate : ByteBuffer::allocateDirect;
+    }
+  }
+
+  @Benchmark
+  public void asyncRead(FileSystemChoice fsChoice,
+                        BufferChoice bufferChoice,
+                        Blackhole blackhole) throws Exception {
+    FSDataInputStream stream = fsChoice.fs.open(DATA_PATH);
+    List<FileRange> ranges = new ArrayList<>();
+    for(int m=0; m < 100; ++m) {
+      FileRange range = FileRange.createFileRange(m * SEEK_SIZE, READ_SIZE);
+      ranges.add(range);
+    }
+    stream.readVectored(ranges, bufferChoice.allocate);
+    for(FileRange range: ranges) {
+      blackhole.consume(range.getData().get());
+    }
+    stream.close();
+  }
+
+  static class Joiner implements CompletionHandler<ByteBuffer, FileRange> {
+    private int remaining;
+    private final ByteBuffer[] result;
+    private Throwable exception = null;
+
+    Joiner(int total) {
+      remaining = total;
+      result = new ByteBuffer[total];
+    }
+
+    synchronized void finish() {
+      remaining -= 1;
+      if (remaining == 0) {
+        notify();
+      }
+    }
+
+    synchronized ByteBuffer[] join() throws InterruptedException, IOException {
+      while (remaining > 0 && exception == null) {
+        wait();
+      }
+      if (exception != null) {
+        throw new IOException("problem reading", exception);
+      }
+      return result;
+    }
+
+
+    @Override
+    public synchronized void completed(ByteBuffer buffer, FileRange attachment) {
+      result[--remaining] = buffer;
+      if (remaining == 0) {
+        notify();
+      }
+    }
+
+    @Override
+    public synchronized void failed(Throwable exc, FileRange attachment) {
+      this.exception = exc;
+      notify();
+    }
+  }
+
+  static class FileRangeCallback extends FileRangeImpl implements
+          CompletionHandler<Integer, FileRangeCallback> {
+    private final AsynchronousFileChannel channel;
+    private final ByteBuffer buffer;
+    private int completed = 0;
+    private final Joiner joiner;
+
+    FileRangeCallback(AsynchronousFileChannel channel, long offset,
+                      int length, Joiner joiner, ByteBuffer buffer) {
+      super(offset, length, null);
+      this.channel = channel;
+      this.joiner = joiner;
+      this.buffer = buffer;
+    }
+
+    @Override
+    public void completed(Integer result, FileRangeCallback attachment) {
+      final int bytes = result;
+      if (bytes == -1) {
+        failed(new EOFException("Read past end of file"), this);
+      }
+      completed += bytes;
+      if (completed < this.getLength()) {
+        channel.read(buffer, this.getOffset() + completed, this, this);
+      } else {
+        buffer.flip();
+        joiner.finish();
+      }
+    }
+
+    @Override
+    public void failed(Throwable exc, FileRangeCallback attachment) {
+      joiner.failed(exc, this);
+    }
+  }
+
+  @Benchmark
+  public void asyncFileChanArray(BufferChoice bufferChoice,
+                                 Blackhole blackhole) throws Exception {
+    java.nio.file.Path path = FileSystems.getDefault().getPath(DATA_PATH.toString());
+    AsynchronousFileChannel channel = AsynchronousFileChannel.open(path, StandardOpenOption.READ);
+    List<FileRangeImpl> ranges = new ArrayList<>();
+    Joiner joiner = new Joiner(100);
+    for(int m=0; m < 100; ++m) {
+      ByteBuffer buffer = bufferChoice.allocate.apply(READ_SIZE);
+      FileRangeCallback range = new FileRangeCallback(channel, m * SEEK_SIZE,
+          READ_SIZE, joiner, buffer);
+      ranges.add(range);
+      channel.read(buffer, range.getOffset(), range, range);
+    }
+    joiner.join();
+    channel.close();
+    blackhole.consume(ranges);
+  }
+
+  @Benchmark
+  public void syncRead(FileSystemChoice fsChoice,
+                       Blackhole blackhole) throws Exception {
+    FSDataInputStream stream = fsChoice.fs.open(DATA_PATH);
+    List<byte[]> result = new ArrayList<>();
+    for(int m=0; m < 100; ++m) {
+      byte[] buffer = new byte[READ_SIZE];
+      stream.readFully(m * SEEK_SIZE, buffer);
+      result.add(buffer);
+    }
+    blackhole.consume(result);
+    stream.close();
+  }
+
+  /**
+   * Run the benchmarks.
+   * @param args the pathname of a 100MB data file
+   * @throws Exception any ex.
+   */
+  public static void main(String[] args) throws Exception {
+    OptionsBuilder opts = new OptionsBuilder();
+    opts.include("VectoredReadBenchmark");
+    opts.jvmArgs("-server", "-Xms256m", "-Xmx2g",
+        "-D" + DATA_PATH_PROPERTY + "=" + args[0]);
+    opts.forks(1);
+    new Runner(opts.build()).run();
+  }
+}
diff --git a/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/package-info.java b/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/package-info.java
new file mode 100644
index 0000000000000..95d6977e3aba7
--- /dev/null
+++ b/hadoop-tools/hadoop-benchmark/src/main/java/org/apache/hadoop/benchmark/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Benchmark for Vectored Read IO operations.
+ */
+package org.apache.hadoop.benchmark;
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index ec77281b66350..890e60a318513 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -90,6 +90,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 20109b0cf61f7..5194e51d81fbf 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -85,6 +85,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -128,7 +134,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <forkCount>1</forkCount>
           <reuseForks>false</reuseForks>
           <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
@@ -223,38 +228,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 93b4ecaa0cfe9..141f45d61f372 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.util.Random;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -473,13 +473,18 @@ private static Configuration getDefaultConf() {
     return config;
   }
 
-  private synchronized void cleanup() {
+  /**
+   * Clean the staging folder created by distcp.
+   */
+  protected synchronized void cleanup() {
     try {
       if (metaFolder != null) {
-        if (jobFS != null) {
-          jobFS.delete(metaFolder, true);
+        synchronized (this) {
+          if (jobFS != null) {
+            jobFS.delete(metaFolder, true);
+          }
+          metaFolder = null;
         }
-        metaFolder = null;
       }
     } catch (IOException e) {
       LOG.error("Unable to cleanup meta folder: " + metaFolder, e);
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index c75c0e85dd791..289d552b86219 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -86,6 +86,8 @@ private DistCpConstants() {
   public static final String CONF_LABEL_SPLIT_RATIO =
       "distcp.dynamic.split.ratio";
   public static final String CONF_LABEL_DIRECT_WRITE = "distcp.direct.write";
+  public static final String CONF_LABEL_UPDATE_ROOT =
+          "distcp.update.root.attributes";
 
   /* Total bytes to be copied. Updated by copylisting. Unfiltered count */
   public static final String CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED = "mapred.total.bytes.expected";
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
index 8e9d64a376e9f..8c443f66a0529 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java
@@ -175,6 +175,10 @@ public boolean shouldUseIterator() {
     return options.shouldUseIterator();
   }
 
+  public boolean shouldUpdateRoot() {
+    return options.shouldUpdateRoot();
+  }
+
   public final boolean splitLargeFile() {
     return options.getBlocksPerChunk() > 0;
   }
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index 4163f8274d967..becd537f68b41 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -244,8 +244,12 @@ public enum DistCpOptionSwitch {
   USE_ITERATOR(DistCpConstants.CONF_LABEL_USE_ITERATOR,
       new Option("useiterator", false,
           "Use single threaded list status iterator to build "
-              + "the listing to save the memory utilisation at the client"));
+              + "the listing to save the memory utilisation at the client")),
 
+  UPDATE_ROOT(DistCpConstants.CONF_LABEL_UPDATE_ROOT,
+      new Option("updateRoot", false,
+      "Update root directory attributes "
+          + "(eg permissions, ownership ...)"));
 
   public static final String PRESERVE_STATUS_DEFAULT = "-prbugpct";
   private final String confLabel;
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index a219586dc8314..2745d828c361b 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.tools;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -30,8 +30,10 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.tools.util.DistCpUtils;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Set;
@@ -160,6 +162,8 @@ public final class DistCpOptions {
 
   private final boolean useIterator;
 
+  private final boolean updateRoot;
+
   /**
    * File attributes for preserve.
    *
@@ -226,6 +230,8 @@ private DistCpOptions(Builder builder) {
     this.directWrite = builder.directWrite;
 
     this.useIterator = builder.useIterator;
+
+    this.updateRoot = builder.updateRoot;
   }
 
   public Path getSourceFileListing() {
@@ -234,7 +240,18 @@ public Path getSourceFileListing() {
 
   public List<Path> getSourcePaths() {
     return sourcePaths == null ?
-        null : Collections.unmodifiableList(sourcePaths);
+        null :
+        Collections.unmodifiableList(getUniquePaths(sourcePaths));
+  }
+
+  private List<Path> getUniquePaths(List<Path> srcPaths) {
+    Set<Path> uniquePaths = new LinkedHashSet<>();
+    for (Path path : srcPaths) {
+      if (!uniquePaths.add(path)) {
+        LOG.info("Path: {} added multiple times, ignoring the redundant entry.", path);
+      }
+    }
+    return new ArrayList<>(uniquePaths);
   }
 
   public Path getTargetPath() {
@@ -361,6 +378,10 @@ public boolean shouldUseIterator() {
     return useIterator;
   }
 
+  public boolean shouldUpdateRoot() {
+    return updateRoot;
+  }
+
   /**
    * Add options to configuration. These will be used in the Mapper/committer
    *
@@ -414,6 +435,9 @@ public void appendToConf(Configuration conf) {
 
     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.USE_ITERATOR,
         String.valueOf(useIterator));
+
+    DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.UPDATE_ROOT,
+        String.valueOf(updateRoot));
   }
 
   /**
@@ -452,6 +476,7 @@ public String toString() {
         ", verboseLog=" + verboseLog +
         ", directWrite=" + directWrite +
         ", useiterator=" + useIterator +
+        ", updateRoot=" + updateRoot +
         '}';
   }
 
@@ -505,6 +530,8 @@ public static class Builder {
 
     private boolean useIterator = false;
 
+    private boolean updateRoot = false;
+
     public Builder(List<Path> sourcePaths, Path targetPath) {
       Preconditions.checkArgument(sourcePaths != null && !sourcePaths.isEmpty(),
           "Source paths should not be null or empty!");
@@ -657,11 +684,28 @@ public Builder withAppend(boolean newAppend) {
       return this;
     }
 
+    /**
+     * whether builder with crc.
+     * @param newSkipCRC whether to skip crc check
+     * @return  Builder object whether to skip crc check
+     * @deprecated Use {@link #withSkipCRC(boolean)} instead.
+     */
+    @Deprecated
     public Builder withCRC(boolean newSkipCRC) {
       this.skipCRC = newSkipCRC;
       return this;
     }
 
+    /**
+     * whether builder with crc.
+     * @param newSkipCRC whether to skip crc check
+     * @return  Builder object whether to skip crc check
+     */
+    public Builder withSkipCRC(boolean newSkipCRC) {
+      this.skipCRC = newSkipCRC;
+      return this;
+    }
+
     public Builder withBlocking(boolean newBlocking) {
       this.blocking = newBlocking;
       return this;
@@ -767,6 +811,11 @@ public Builder withUseIterator(boolean useItr) {
       this.useIterator = useItr;
       return this;
     }
+
+    public Builder withUpdateRoot(boolean updateRootAttrs) {
+      this.updateRoot = updateRootAttrs;
+      return this;
+    }
   }
 
 }
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
index e52c1687553d4..1cf2d97ec1f38 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
@@ -260,7 +260,7 @@ private boolean getAllDiffs() throws IOException {
               list.add(new DiffInfo(source, target, dt));
             } else {
               list = diffMap.get(SnapshotDiffReport.DiffType.DELETE);
-              DiffInfo info = new DiffInfo(source, target,
+              DiffInfo info = new DiffInfo(source, null,
                   SnapshotDiffReport.DiffType.DELETE);
               list.add(info);
               if (deletedByExclusionDiffs == null) {
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index a4c3b0f851f7b..49ee09f400400 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -33,7 +33,7 @@
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.fs.Path;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * The OptionsParser parses out the command-line options passed to DistCp,
@@ -108,7 +108,7 @@ public static DistCpOptions parse(String[] args)
             command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch()))
         .withAppend(
             command.hasOption(DistCpOptionSwitch.APPEND.getSwitch()))
-        .withCRC(
+        .withSkipCRC(
             command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch()))
         .withBlocking(
             !command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch()))
@@ -117,7 +117,9 @@ public static DistCpOptions parse(String[] args)
         .withDirectWrite(
             command.hasOption(DistCpOptionSwitch.DIRECT_WRITE.getSwitch()))
         .withUseIterator(
-            command.hasOption(DistCpOptionSwitch.USE_ITERATOR.getSwitch()));
+            command.hasOption(DistCpOptionSwitch.USE_ITERATOR.getSwitch()))
+        .withUpdateRoot(
+            command.hasOption(DistCpOptionSwitch.UPDATE_ROOT.getSwitch()));
 
     if (command.hasOption(DistCpOptionSwitch.DIFF.getSwitch())) {
       String[] snapshots = getVals(command,
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index e9eb4c0923657..c3f097da4ba96 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -616,10 +616,12 @@ private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
       DistCpContext context) throws IOException {
     boolean syncOrOverwrite = context.shouldSyncFolder() ||
         context.shouldOverwrite();
+    boolean skipRootPath = syncOrOverwrite && !context.shouldUpdateRoot();
     for (CopyListingFileStatus fs : fileStatus) {
       if (fs.getPath().equals(sourcePathRoot) &&
-          fs.isDirectory() && syncOrOverwrite) {
-        // Skip the root-paths when syncOrOverwrite
+          fs.isDirectory() && skipRootPath) {
+        // Skip the root-paths when skipRootPath (syncOrOverwrite and
+        // update root directory is not a must).
         LOG.debug("Skip {}", fs.getPath());
         return;
       }
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 2272781f72476..8c2bc82d3fc7f 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -75,6 +75,7 @@ public class CopyCommitter extends FileOutputCommitter {
   private boolean ignoreFailures = false;
   private boolean skipCrc = false;
   private int blocksPerChunk = 0;
+  private boolean updateRoot = false;
 
   /**
    * Create a output committer
@@ -100,6 +101,8 @@ public void commitJob(JobContext jobContext) throws IOException {
     Configuration conf = jobContext.getConfiguration();
     syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
     overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
+    updateRoot =
+        conf.getBoolean(CONF_LABEL_UPDATE_ROOT, false);
     targetPathExists = conf.getBoolean(
         DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
     ignoreFailures = conf.getBoolean(
@@ -336,9 +339,12 @@ private void preserveFileAttributesForDirectories(Configuration conf)
 
         Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
         //
-        // Skip the root folder when syncOrOverwrite is true.
+        // Skip the root folder when skipRoot is true.
         //
-        if (targetRoot.equals(targetFile) && syncOrOverwrite) continue;
+        boolean skipRoot = syncOrOverwrite && !updateRoot;
+        if (targetRoot.equals(targetFile) && skipRoot) {
+          continue;
+        }
 
         FileSystem targetFS = targetFile.getFileSystem(conf);
         DistCpUtils.preserve(targetFS, targetFile, srcFileStatus, attributes,
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/DeletedDirTracker.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/DeletedDirTracker.java
index 7e11add9b7a79..7d3a952eab592 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/DeletedDirTracker.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/DeletedDirTracker.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.tools.mapred;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
index e836ccce41223..ae6b734f48603 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
@@ -52,7 +52,10 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
 import static org.apache.hadoop.tools.mapred.CopyMapper.getFileAttributeSettings;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
 
 /**
  * This class extends RetriableCommand to implement the copy of files,
@@ -154,7 +157,7 @@ private long doCopy(CopyListingFileStatus source, Path target,
       if (!source.isSplit()) {
         DistCpUtils.compareFileLengthsAndChecksums(source.getLen(), sourceFS,
                 sourcePath, sourceChecksum, targetFS,
-                targetPath, skipCrc, source.getLen());
+                targetPath, skipCrc, offset + bytesRead);
       }
       // it's not append or direct write (preferred for s3a) case, thus we first
       // write to a temporary file, then rename it to the target path.
@@ -362,7 +365,11 @@ private static ThrottledInputStream getInputStream(Path path,
       FileSystem fs = path.getFileSystem(conf);
       float bandwidthMB = conf.getFloat(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
               DistCpConstants.DEFAULT_BANDWIDTH_MB);
-      FSDataInputStream in = fs.open(path);
+      // open with sequential read, but not whole-file
+      FSDataInputStream in = awaitFuture(fs.openFile(path)
+          .opt(FS_OPTION_OPENFILE_READ_POLICY,
+              FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+          .build());
       return new ThrottledInputStream(in, bandwidthMB * 1024 * 1024);
     }
     catch (IOException e) {
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index 136b6c8ca1339..a86e41c6668fd 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -49,7 +49,7 @@ Overview
 
   [The erstwhile implementation of DistCp]
   (http://hadoop.apache.org/docs/r1.2.1/distcp.html) has its share of quirks
-  and drawbacks, both in its usage, as well as its extensibility and
+  and drawbacks, both in its usage and its extensibility and
   performance. The purpose of the DistCp refactor was to fix these
   shortcomings, enabling it to be used and extended programmatically. New
   paradigms have been introduced to improve runtime and setup performance,
@@ -179,7 +179,7 @@ $H3 Update and Overwrite
     hdfs://nn2:8020/target/10 32
     hdfs://nn2:8020/target/20 64
 
-  Will effect:
+  The result will be:
 
     hdfs://nn2:8020/target/1 32
     hdfs://nn2:8020/target/2 32
@@ -190,7 +190,7 @@ $H3 Update and Overwrite
   because it doesn't exist at the target. `10` and `20` are overwritten since
   the contents don't match the source.
 
-  If `-update` is used, `1` is skipped because the file-length and contents match. `2` is copied because it doesn’t exist at the target. `10` and `20` are overwritten since the contents don’t match the source. However, if `-append` is additionally used, then only `10` is overwritten (source length less than destination) and `20` is appended with the change in file (if the files match up to the destination's original length).
+  If `-update` is used, `1` is skipped because the file-length and contents match. `2` is copied because it doesn't exist at the target. `10` and `20` are overwritten since the contents don’t match the source. However, if `-append` is additionally used, then only `10` is overwritten (source length less than destination) and `20` is appended with the change in file (if the files match up to the destination's original length).
 
   If `-overwrite` is used, `1` is overwritten as well.
 
@@ -269,7 +269,7 @@ $H4 Experiment 1: Syncing diff of two adjacent snapshots
 
 $H4 Experiment 2: syncing diff of two non-adjacent snapshots
 
-  First do a clean up from Experiment 1.
+  First do a cleanup from Experiment 1.
 
     hdfs dfs -rm -skipTrash /dst/1.txt
 
@@ -363,6 +363,7 @@ Command Line Options
 | `-xtrack <path>` | Save information about missing source files to the specified path. | This option is only valid with `-update` option. This is an experimental property and it cannot be used with `-atomic` option. |
 | `-direct` | Write directly to destination paths | Useful for avoiding potentially very expensive temporary file rename operations when the destination is an object store |
 | `-useiterator` | Uses single threaded listStatusIterator to build listing | Useful for saving memory at the client side. Using this option will ignore the numListstatusThreads option |
+| `-updateRoot` | Update root directory attributes (eg permissions, ownership ...) | Useful if you need to enforce root directory attributes update when using distcp |
 
 Architecture of DistCp
 ----------------------
@@ -513,7 +514,7 @@ $H3 InputFormats and MapReduce Components
      * A file with the same name exists at target, but `-overwrite` is
        specified.
      * A file with the same name exists at target, but differs in block-size
-       (and block-size needs to be preserved.
+       and block-size needs to be preserved.
 
   * **CopyCommitter:** This class is responsible for the commit-phase of the
     DistCp job, including:
@@ -575,12 +576,12 @@ $H3 MapReduce and other side-effects
     map on a re-execution will be marked as "skipped".
   * If a map fails `mapreduce.map.maxattempts` times, the remaining map tasks
     will be killed (unless `-i` is set).
-  * If `mapreduce.map.speculative` is set set final and true, the result of the
+  * If `mapreduce.map.speculative` is set to be true, the result of the
     copy is undefined.
 
 $H3 DistCp and Object Stores
 
-DistCp works with Object Stores such as Amazon S3, Azure WASB and OpenStack Swift.
+DistCp works with Object Stores such as Amazon S3, Azure ABFS and Google GCS.
 
 Prequisites
 
@@ -623,7 +624,7 @@ And to use `-update` to only copy changed files.
 
 ```bash
 hadoop distcp -update -numListstatusThreads 20  \
-  swift://history.cluster1/2016 \
+  s3a://history/2016 \
   hdfs://nn1:8020/history/2016
 ```
 
@@ -690,7 +691,7 @@ Frequently Asked Questions
      directory is copied over, rather than the source-directory itself. This
      behaviour is consistent with the legacy DistCp implementation as well.
 
-  2. **How does the new DistCp differ in semantics from the Legacy DistCp?**
+  2. **How does the new DistCp differs in semantics from the Legacy DistCp?**
 
      * Files that are skipped during copy used to also have their
        file-attributes (permissions, owner/group info, etc.) unchanged, when
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
index ce7d00d2bd732..69e1421f08447 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
@@ -167,6 +167,29 @@ public void testDuplicates() {
     }
   }
 
+  @Test
+  public void testDuplicateSourcePaths() throws Exception {
+    FileSystem fs = FileSystem.get(getConf());
+    List<Path> srcPaths = new ArrayList<Path>();
+    try {
+      srcPaths.add(new Path("/tmp/in"));
+      srcPaths.add(new Path("/tmp/in"));
+      TestDistCpUtils.createFile(fs, "/tmp/in/src1/1.txt");
+      TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt");
+      Path target = new Path("/tmp/out");
+      Path listingFile = new Path("/tmp/list");
+      final DistCpOptions options =
+          new DistCpOptions.Builder(srcPaths, target).build();
+      final DistCpContext context = new DistCpContext(options);
+      CopyListing listing =
+          CopyListing.getCopyListing(getConf(), CREDENTIALS, context);
+      listing.buildListing(listingFile, context);
+      Assert.assertTrue(fs.exists(listingFile));
+    } finally {
+      TestDistCpUtils.delete(fs, "/tmp");
+    }
+  }
+
   @Test(timeout=10000)
   public void testBuildListing() {
     FileSystem fs = null;
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
index 13497029a0767..d126bfdc4f975 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
@@ -128,7 +128,7 @@ public void testSetSkipCRC() {
         new Path("hdfs://localhost:8020/target/"));
     Assert.assertFalse(builder.build().shouldSkipCRC());
 
-    final DistCpOptions options = builder.withSyncFolder(true).withCRC(true)
+    final DistCpOptions options = builder.withSyncFolder(true).withSkipCRC(true)
         .build();
     Assert.assertTrue(options.shouldSyncFolder());
     Assert.assertTrue(options.shouldSkipCRC());
@@ -289,7 +289,7 @@ public void testToString() {
         "atomicWorkPath=null, logPath=null, sourceFileListing=abc, " +
         "sourcePaths=null, targetPath=xyz, filtersFile='null', " +
         "blocksPerChunk=0, copyBufferSize=8192, verboseLog=false, " +
-        "directWrite=false, useiterator=false}";
+        "directWrite=false, useiterator=false, updateRoot=false}";
     String optionString = option.toString();
     Assert.assertEquals(val, optionString);
     Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
@@ -391,7 +391,7 @@ public void testAppendOption() {
           new Path("hdfs://localhost:8020/target/"))
           .withSyncFolder(true)
           .withAppend(true)
-          .withCRC(true)
+          .withSkipCRC(true)
           .build();
       fail("Append should fail if skipCrc option is specified");
     } catch (IllegalArgumentException e) {
@@ -563,4 +563,15 @@ public void testAppendToConf() {
             "otherwise it may not be fetched properly",
             expectedValForEmptyConfigKey, config.get(""));
   }
+
+  @Test
+  public void testUpdateRoot() {
+    final DistCpOptions options = new DistCpOptions.Builder(
+        Collections.singletonList(
+            new Path("hdfs://localhost:8020/source")),
+        new Path("hdfs://localhost:8020/target/"))
+        .withUpdateRoot(true)
+        .build();
+    Assert.assertTrue(options.shouldUpdateRoot());
+  }
 }
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
index 61c72746add7e..93796e752ebc9 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -1154,6 +1155,84 @@ public void testSyncSnapshotDiffWithWebHdfs3() throws Exception {
     snapshotDiffWithPaths(webHdfsSource, webHdfsTarget);
   }
 
+  @Test
+  public void testRenameWithFilter() throws Exception {
+    java.nio.file.Path filterFile = null;
+    try {
+      Path sourcePath = new Path(dfs.getWorkingDirectory(), "source");
+
+      // Create some dir inside source
+      dfs.mkdirs(new Path(sourcePath, "dir1"));
+      dfs.mkdirs(new Path(sourcePath, "dir2"));
+
+      // Allow & Create snapshot at source.
+      dfs.allowSnapshot(sourcePath);
+      dfs.createSnapshot(sourcePath, "s1");
+
+      filterFile = Files.createTempFile("filters", "txt");
+      String str = ".*filterDir1.*";
+      try (BufferedWriter writer = new BufferedWriter(
+          new FileWriter(filterFile.toString()))) {
+        writer.write(str);
+      }
+      final DistCpOptions.Builder builder =
+          new DistCpOptions.Builder(new ArrayList<>(Arrays.asList(sourcePath)),
+              target).withFiltersFile(filterFile.toString())
+              .withSyncFolder(true);
+      new DistCp(conf, builder.build()).execute();
+
+      // Check the two directories get copied.
+      ContractTestUtils
+          .assertPathExists(dfs, "dir1 should get copied to target",
+              new Path(target, "dir1"));
+      ContractTestUtils
+          .assertPathExists(dfs, "dir2 should get copied to target",
+              new Path(target, "dir2"));
+
+      // Allow & create initial snapshots on target.
+      dfs.allowSnapshot(target);
+      dfs.createSnapshot(target, "s1");
+
+      // Now do a rename to a filtered name on source.
+      dfs.rename(new Path(sourcePath, "dir1"),
+          new Path(sourcePath, "filterDir1"));
+
+      ContractTestUtils
+          .assertPathExists(dfs, "'filterDir1' should be there on source",
+              new Path(sourcePath, "filterDir1"));
+
+      // Create the incremental snapshot.
+      dfs.createSnapshot(sourcePath, "s2");
+
+      final DistCpOptions.Builder diffBuilder =
+          new DistCpOptions.Builder(new ArrayList<>(Arrays.asList(sourcePath)),
+              target).withUseDiff("s1", "s2")
+              .withFiltersFile(filterFile.toString()).withSyncFolder(true);
+      new DistCp(conf, diffBuilder.build()).execute();
+
+      // Check the only qualified directory dir2 is there in target
+      ContractTestUtils.assertPathExists(dfs, "dir2 should be there on target",
+          new Path(target, "dir2"));
+
+      // Check the filtered directory is not there.
+      ContractTestUtils.assertPathDoesNotExist(dfs,
+          "Filtered directory 'filterDir1' shouldn't get copied",
+          new Path(target, "filterDir1"));
+
+      // Check the renamed directory gets deleted.
+      ContractTestUtils.assertPathDoesNotExist(dfs,
+          "Renamed directory 'dir1' should get deleted",
+          new Path(target, "dir1"));
+
+      // Check the filtered directory isn't there in the home directory.
+      ContractTestUtils.assertPathDoesNotExist(dfs,
+          "Filtered directory 'filterDir1' shouldn't get copied to home directory",
+          new Path("filterDir1"));
+    } finally {
+      deleteFilterFile(filterFile);
+    }
+  }
+
   private void snapshotDiffWithPaths(Path sourceFSPath,
       Path targetFSPath) throws Exception {
 
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
index 47b850f4ba3e2..64c6800f9446a 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
@@ -21,6 +21,7 @@
 import static org.apache.hadoop.test.GenericTestUtils.getMethodName;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
@@ -44,6 +45,8 @@
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.tools.util.DistCpTestUtils;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -551,4 +554,44 @@ public void testSourceRoot() throws Exception {
     String[] args2 = new String[]{rootStr, tgtStr2};
     Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0));
   }
+
+  @Test
+  public void testUpdateRoot() throws Exception {
+    FileSystem fs = cluster.getFileSystem();
+
+    Path source = new Path("/src");
+    Path dest1 = new Path("/dest1");
+    Path dest2 = new Path("/dest2");
+
+    fs.delete(source, true);
+    fs.delete(dest1, true);
+    fs.delete(dest2, true);
+
+    // Create a source dir
+    fs.mkdirs(source);
+    fs.setOwner(source, "userA", "groupA");
+    fs.setTimes(source, new Random().nextLong(), new Random().nextLong());
+
+    GenericTestUtils.createFiles(fs, source, 3, 5, 5);
+
+    // should not preserve attrs
+    DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, source.toString(),
+        dest1.toString(), "-p -update", conf);
+
+    FileStatus srcStatus = fs.getFileStatus(source);
+    FileStatus destStatus1 = fs.getFileStatus(dest1);
+    assertNotEquals(srcStatus.getOwner(), destStatus1.getOwner());
+    assertNotEquals(srcStatus.getModificationTime(),
+        destStatus1.getModificationTime());
+
+    // should preserve attrs
+    DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, source.toString(),
+        dest2.toString(), "-p -update -updateRoot",
+        conf);
+
+    FileStatus destStatus2 = fs.getFileStatus(dest2);
+    assertEquals(srcStatus.getOwner(), destStatus2.getOwner());
+    assertEquals(srcStatus.getModificationTime(),
+        destStatus2.getModificationTime());
+  }
 }
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index 85b312a94b52f..1ffdd89073dec 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -804,4 +804,18 @@ public void testExclusionsOption() {
         "hdfs://localhost:8020/target/"});
     assertThat(options.getFiltersFile()).isEqualTo("/tmp/filters.txt");
   }
+
+  @Test
+  public void testParseUpdateRoot() {
+    DistCpOptions options = OptionsParser.parse(new String[] {
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    Assert.assertFalse(options.shouldUpdateRoot());
+
+    options = OptionsParser.parse(new String[] {
+        "-updateRoot",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    Assert.assertTrue(options.shouldUpdateRoot());
+  }
 }
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
index 04aeea665c244..8545df30bac8e 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
@@ -349,7 +349,7 @@ private Job distCpUpdate(final Path srcDir, final Path destDir)
             Collections.singletonList(srcDir), destDir)
             .withDeleteMissing(true)
             .withSyncFolder(true)
-            .withCRC(true)
+            .withSkipCRC(true)
             .withDirectWrite(shouldUseDirectWrite())
             .withOverwrite(false)));
   }
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 62940f64b3488..599f3ec2db61e 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -536,7 +536,7 @@ private void testCommitWithChecksumMismatch(boolean skipCrc)
           Collections.singletonList(new Path(sourceBase)),
           new Path("/out"))
           .withBlocksPerChunk(blocksPerChunk)
-          .withCRC(skipCrc)
+          .withSkipCRC(skipCrc)
           .build();
       options.appendToConf(conf);
       conf.setBoolean(
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
index 3a1c3e9506b33..b31f26163265f 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml
@@ -60,6 +60,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <type>test-jar</type>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -137,38 +143,6 @@
           </environmentVariables>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AMOptions.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AMOptions.java
index 77f8c2ce57224..b7726e4d7337e 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AMOptions.java
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/AMOptions.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.tools.dynamometer;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java
index 6c09e27352f0c..0c57542747e41 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java
@@ -19,7 +19,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.base.Splitter;
 import org.apache.hadoop.util.Lists;
 import java.util.Optional;
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java
index ee0810d6439d7..f6c8a6ac4d58b 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/DynoInfraUtils.java
@@ -54,9 +54,10 @@
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
 import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.JsonToken;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
 import org.slf4j.Logger;
 
 
@@ -484,7 +485,7 @@ static Set<String> parseStaleDataNodeList(String liveNodeJsonString,
     final Set<String> dataNodesToReport = new HashSet<>();
 
     JsonFactory fac = new JsonFactory();
-    JsonParser parser = fac.createJsonParser(IOUtils
+    JsonParser parser = fac.createParser(IOUtils
         .toInputStream(liveNodeJsonString, StandardCharsets.UTF_8.name()));
 
     int objectDepth = 0;
@@ -554,7 +555,7 @@ static String fetchNameNodeJMXValue(Properties nameNodeProperties,
     }
     InputStream in = conn.getInputStream();
     JsonFactory fac = new JsonFactory();
-    JsonParser parser = fac.createJsonParser(in);
+    JsonParser parser = fac.createParser(in);
     if (parser.nextToken() != JsonToken.START_OBJECT
         || parser.nextToken() != JsonToken.FIELD_NAME
         || !parser.getCurrentName().equals("beans")
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/SimulatedDataNodes.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/SimulatedDataNodes.java
index 520077e0823cb..0189d1fd1a1f1 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/SimulatedDataNodes.java
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/SimulatedDataNodes.java
@@ -131,7 +131,7 @@ public int run(String[] args) throws Exception {
           + " block listing files; launching DataNodes accordingly.");
       mc.startDataNodes(getConf(), blockListFiles.size(), null, false,
           StartupOption.REGULAR, null, null, null, null, false, true, true,
-          null);
+          null, null, null);
       long startTime = Time.monotonicNow();
       System.out.println("Waiting for DataNodes to connect to NameNode and "
           + "init storage directories.");
diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
index 316a2bd276c61..8818b1ef24461 100644
--- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
+++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java
@@ -76,6 +76,7 @@
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -111,6 +112,7 @@
  * property to point directly to a Hadoop tarball which is present locally and
  * no download will occur.
  */
+@Ignore
 public class TestDynamometerInfra {
 
   private static final Logger LOG =
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index 160f8b4023efb..d6e0ba07dc4f8 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -74,6 +74,16 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.enterprise</groupId>
+          <artifactId>cdi-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -90,6 +100,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-federation-balance/pom.xml b/hadoop-tools/hadoop-federation-balance/pom.xml
index f048e2a11f639..63269b3878cbd 100644
--- a/hadoop-tools/hadoop-federation-balance/pom.xml
+++ b/hadoop-tools/hadoop-federation-balance/pom.xml
@@ -85,6 +85,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -138,7 +144,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
-          <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
           <forkCount>1</forkCount>
           <reuseForks>false</reuseForks>
           <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
@@ -233,38 +238,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/hadoop-tools/hadoop-federation-balance/src/site/markdown/HDFSFederationBalance.md b/hadoop-tools/hadoop-federation-balance/src/site/markdown/HDFSFederationBalance.md
index c9d643bf542cd..e665528f7b9a2 100644
--- a/hadoop-tools/hadoop-federation-balance/src/site/markdown/HDFSFederationBalance.md
+++ b/hadoop-tools/hadoop-federation-balance/src/site/markdown/HDFSFederationBalance.md
@@ -90,7 +90,7 @@ Usage
 
 ### Command Options
 
-Command `submit` has 4 options:
+Command `submit` has the following options:
 
 | Option key                     | Description                          | Default |
 | ------------------------------ | ------------------------------------ | ------- |
diff --git a/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestDistCpProcedure.java b/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestDistCpProcedure.java
index 0bed359903694..39986d865aea4 100644
--- a/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestDistCpProcedure.java
+++ b/hadoop-tools/hadoop-federation-balance/src/test/java/org/apache/hadoop/tools/fedbalance/TestDistCpProcedure.java
@@ -33,7 +33,9 @@
 import org.apache.hadoop.tools.fedbalance.procedure.BalanceProcedureScheduler;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -44,6 +46,7 @@
 import java.io.ByteArrayInputStream;
 import java.net.URI;
 import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 import static junit.framework.TestCase.assertTrue;
 import static org.apache.hadoop.tools.fedbalance.FedBalanceConfigs.SCHEDULER_JOURNAL_URI;
@@ -74,6 +77,11 @@ public class TestDistCpProcedure {
           new FileEntry(SRCDAT + "/b/c", false)};
   private static String nnUri;
 
+  @Rule
+  // There are multiple unit tests with different timeouts that fail multiple times because of
+  // DataStreamer#waitAndQueuePacket, so we set a larger global timeout.
+  public Timeout globalTimeout = new Timeout(180000, TimeUnit.MILLISECONDS);
+
   @BeforeClass
   public static void beforeClass() throws IOException {
     DistCpProcedure.enableForTest();
@@ -98,7 +106,7 @@ public static void afterClass() {
     }
   }
 
-  @Test(timeout = 90000)
+  @Test
   public void testSuccessfulDistCpProcedure() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -140,7 +148,7 @@ public void testSuccessfulDistCpProcedure() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testInitDistCp() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -198,7 +206,7 @@ public void testDiffThreshold() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testDiffDistCp() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -235,7 +243,7 @@ public void testDiffDistCp() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testStageFinalDistCp() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -260,7 +268,7 @@ public void testStageFinalDistCp() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testStageFinish() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -293,7 +301,7 @@ public void testStageFinish() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testRecoveryByStage() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -334,7 +342,7 @@ public void testRecoveryByStage() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testShutdown() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
@@ -359,7 +367,7 @@ public void testShutdown() throws Exception {
     cleanup(fs, new Path(testRoot));
   }
 
-  @Test(timeout = 30000)
+  @Test
   public void testDisableWrite() throws Exception {
     String testRoot = nnUri + "/user/foo/testdir." + getMethodName();
     DistributedFileSystem fs =
diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml
index 66f971c9e1a74..8c5ed0224bf14 100644
--- a/hadoop-tools/hadoop-fs2img/pom.xml
+++ b/hadoop-tools/hadoop-fs2img/pom.xml
@@ -41,6 +41,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -107,38 +113,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index b903bb6f097f5..21d786c342d6c 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -90,6 +90,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -188,38 +194,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/hadoop-tools/hadoop-openstack/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-openstack/dev-support/findbugs-exclude.xml
deleted file mode 100644
index cfb75c73081b5..0000000000000
--- a/hadoop-tools/hadoop-openstack/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~   you may not use this file except in compliance with the License.
-  ~   You may obtain a copy of the License at
-  ~   
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~   
-  ~   Unless required by applicable law or agreed to in writing, software
-  ~   distributed under the License is distributed on an "AS IS" BASIS,
-  ~   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~   See the License for the specific language governing permissions and
-  ~   limitations under the License. See accompanying LICENSE file.
-  -->
-<FindBugsFilter>
-
-  <!--
-  OpenStack Swift FS module -closes streams in a different method
-  from where they are opened.
-  -->
-  <Match>
-    <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
-    <Method name="uploadFileAttempt"/>
-    <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
-    <Bug code="OBL"/>
-  </Match>
-  <Match>
-    <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
-    <Method name="uploadFilePartAttempt"/>
-    <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
-    <Bug code="OBL"/>
-  </Match>
-
-</FindBugsFilter>
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index e1065883feb9e..a3f0e748454a1 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -26,9 +26,10 @@
   <version>3.4.0-SNAPSHOT</version>
   <name>Apache Hadoop OpenStack support</name>
   <description>
-    This module contains code to support integration with OpenStack.
-    Currently this consists of a filesystem client to read data from
-    and write data to an OpenStack Swift object store.
+    This module used to contain code to support integration with OpenStack.
+    It has been deleted as unsupported; the JAR is still published so as to
+    not break applications which declare an explicit maven/ivy/SBT dependency
+    on the module.
   </description>
   <packaging>jar</packaging>
 
@@ -37,32 +38,6 @@
     <downloadSources>true</downloadSources>
   </properties>
 
-  <profiles>
-    <profile>
-      <id>tests-off</id>
-      <activation>
-        <file>
-          <missing>src/test/resources/auth-keys.xml</missing>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>true</maven.test.skip>
-      </properties>
-    </profile>
-    <profile>
-      <id>tests-on</id>
-      <activation>
-        <file>
-          <exists>src/test/resources/auth-keys.xml</exists>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>false</maven.test.skip>
-      </properties>
-    </profile>
-
-  </profiles>
-
   <build>
     <plugins>
       <plugin>
@@ -70,71 +45,11 @@
         <artifactId>spotbugs-maven-plugin</artifactId>
         <configuration>
           <xmlOutput>true</xmlOutput>
-          <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
-          </excludeFilterFile>
           <effort>Max</effort>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>deplist</id>
-            <phase>compile</phase>
-            <goals>
-              <goal>list</goal>
-            </goals>
-            <configuration>
-              <!-- build a shellprofile -->
-              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-annotations</artifactId>
-      <scope>compile</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpcore</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>commons-logging</groupId>
-      <artifactId>commons-logging</artifactId>
-      <scope>compile</scope>
-    </dependency>
 
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-    </dependency>
-  </dependencies>
 </project>
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java
deleted file mode 100644
index e25d17d2fb898..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Class that represents authentication request to Openstack Keystone.
- * Contains basic authentication information.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS
- */
-public class ApiKeyAuthenticationRequest extends AuthenticationRequest {
-  /**
-   * Credentials for login
-   */
-  private ApiKeyCredentials apiKeyCredentials;
-
-  /**
-   * API key auth
-   * @param tenantName tenant
-   * @param apiKeyCredentials credentials
-   */
-  public ApiKeyAuthenticationRequest(String tenantName, ApiKeyCredentials apiKeyCredentials) {
-    this.tenantName = tenantName;
-    this.apiKeyCredentials = apiKeyCredentials;
-  }
-
-  /**
-   * @return credentials for login into Keystone
-   */
-  @JsonProperty("RAX-KSKEY:apiKeyCredentials")
-  public ApiKeyCredentials getApiKeyCredentials() {
-    return apiKeyCredentials;
-  }
-
-  /**
-   * @param apiKeyCredentials credentials for login into Keystone
-   */
-  public void setApiKeyCredentials(ApiKeyCredentials apiKeyCredentials) {
-    this.apiKeyCredentials = apiKeyCredentials;
-  }
-
-  @Override
-  public String toString() {
-    return "Auth as " +
-           "tenant '" + tenantName + "' "
-           + apiKeyCredentials;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyCredentials.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyCredentials.java
deleted file mode 100644
index 412ce81daa300..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyCredentials.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-
-/**
- * Describes credentials to log in Swift using Keystone authentication.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class ApiKeyCredentials {
-  /**
-   * user login
-   */
-  private String username;
-
-  /**
-   * user password
-   */
-  private String apikey;
-
-  /**
-   * default constructor
-   */
-  public ApiKeyCredentials() {
-  }
-
-  /**
-   * @param username user login
-   * @param apikey user api key
-   */
-  public ApiKeyCredentials(String username, String apikey) {
-    this.username = username;
-    this.apikey = apikey;
-  }
-
-  /**
-   * @return user api key
-   */
-  public String getApiKey() {
-    return apikey;
-  }
-
-  /**
-   * @param apikey user api key
-   */
-  public void setApiKey(String apikey) {
-    this.apikey = apikey;
-  }
-
-  /**
-   * @return login
-   */
-  public String getUsername() {
-    return username;
-  }
-
-  /**
-   * @param username login
-   */
-  public void setUsername(String username) {
-    this.username = username;
-  }
-
-  @Override
-  public String toString() {
-    return "user " +
-           "'" + username + '\'' +
-           " with key of length " + ((apikey == null) ? 0 : apikey.length());
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequest.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequest.java
deleted file mode 100644
index a2a3b55e76f2e..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * Class that represents authentication request to Openstack Keystone.
- * Contains basic authentication information.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class AuthenticationRequest {
-
-  /**
-   * tenant name
-   */
-  protected String tenantName;
-
-  public AuthenticationRequest() {
-  }
-
-  /**
-   * @return tenant name for Keystone authorization
-   */
-  public String getTenantName() {
-    return tenantName;
-  }
-
-  /**
-   * @param tenantName tenant name for authorization
-   */
-  public void setTenantName(String tenantName) {
-    this.tenantName = tenantName;
-  }
-
-  @Override
-  public String toString() {
-    return "AuthenticationRequest{" +
-           "tenantName='" + tenantName + '\'' +
-           '}';
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequestWrapper.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequestWrapper.java
deleted file mode 100644
index f30e90dad384d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequestWrapper.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * This class is used for correct hierarchy mapping of
- * Keystone authentication model and java code.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class AuthenticationRequestWrapper {
-  /**
-   * authentication request
-   */
-  private AuthenticationRequest auth;
-
-  /**
-   * default constructor used for json parsing
-   */
-  public AuthenticationRequestWrapper() {
-  }
-
-  /**
-   * @param auth authentication requests
-   */
-  public AuthenticationRequestWrapper(AuthenticationRequest auth) {
-    this.auth = auth;
-  }
-
-  /**
-   * @return authentication request
-   */
-  public AuthenticationRequest getAuth() {
-    return auth;
-  }
-
-  /**
-   * @param auth authentication request
-   */
-  public void setAuth(AuthenticationRequest auth) {
-    this.auth = auth;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationResponse.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationResponse.java
deleted file mode 100644
index f09ec0c5fb99e..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationResponse.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
-import org.apache.hadoop.fs.swift.auth.entities.Catalog;
-import org.apache.hadoop.fs.swift.auth.entities.User;
-
-import java.util.List;
-
-/**
- * Response from KeyStone deserialized into AuthenticationResponse class.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class AuthenticationResponse {
-  private Object metadata;
-  private List<Catalog> serviceCatalog;
-  private User user;
-  private AccessToken token;
-
-  public Object getMetadata() {
-    return metadata;
-  }
-
-  public void setMetadata(Object metadata) {
-    this.metadata = metadata;
-  }
-
-  public List<Catalog> getServiceCatalog() {
-    return serviceCatalog;
-  }
-
-  public void setServiceCatalog(List<Catalog> serviceCatalog) {
-    this.serviceCatalog = serviceCatalog;
-  }
-
-  public User getUser() {
-    return user;
-  }
-
-  public void setUser(User user) {
-    this.user = user;
-  }
-
-  public AccessToken getToken() {
-    return token;
-  }
-
-  public void setToken(AccessToken token) {
-    this.token = token;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeyStoneAuthRequest.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeyStoneAuthRequest.java
deleted file mode 100644
index c3abbac88f452..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeyStoneAuthRequest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * Class that represents authentication to OpenStack Keystone.
- * Contains basic authentication information.
- * Used when {@link ApiKeyAuthenticationRequest} is not applicable.
- * (problem with different Keystone installations/versions/modifications)
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class KeyStoneAuthRequest extends AuthenticationRequest {
-
-  /**
-   * Credentials for Keystone authentication
-   */
-  private KeystoneApiKeyCredentials apiAccessKeyCredentials;
-
-  /**
-   * @param tenant                  Keystone tenant name for authentication
-   * @param apiAccessKeyCredentials Credentials for authentication
-   */
-  public KeyStoneAuthRequest(String tenant, KeystoneApiKeyCredentials apiAccessKeyCredentials) {
-    this.apiAccessKeyCredentials = apiAccessKeyCredentials;
-    this.tenantName = tenant;
-  }
-
-  public KeystoneApiKeyCredentials getApiAccessKeyCredentials() {
-    return apiAccessKeyCredentials;
-  }
-
-  public void setApiAccessKeyCredentials(KeystoneApiKeyCredentials apiAccessKeyCredentials) {
-    this.apiAccessKeyCredentials = apiAccessKeyCredentials;
-  }
-
-  @Override
-  public String toString() {
-    return "KeyStoneAuthRequest as " +
-            "tenant '" + tenantName + "' "
-            + apiAccessKeyCredentials;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeystoneApiKeyCredentials.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeystoneApiKeyCredentials.java
deleted file mode 100644
index 75202b3a6d23a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeystoneApiKeyCredentials.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * Class for Keystone authentication.
- * Used when {@link ApiKeyCredentials} is not applicable
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class KeystoneApiKeyCredentials {
-
-  /**
-   * User access key
-   */
-  private String accessKey;
-
-  /**
-   * User access secret
-   */
-  private String secretKey;
-
-  public KeystoneApiKeyCredentials(String accessKey, String secretKey) {
-    this.accessKey = accessKey;
-    this.secretKey = secretKey;
-  }
-
-  public String getAccessKey() {
-    return accessKey;
-  }
-
-  public void setAccessKey(String accessKey) {
-    this.accessKey = accessKey;
-  }
-
-  public String getSecretKey() {
-    return secretKey;
-  }
-
-  public void setSecretKey(String secretKey) {
-    this.secretKey = secretKey;
-  }
-
-  @Override
-  public String toString() {
-    return "user " +
-            "'" + accessKey + '\'' +
-            " with key of length " + ((secretKey == null) ? 0 : secretKey.length());
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordAuthenticationRequest.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordAuthenticationRequest.java
deleted file mode 100644
index ee519f3f8daa1..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordAuthenticationRequest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * Class that represents authentication request to Openstack Keystone.
- * Contains basic authentication information.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class PasswordAuthenticationRequest extends AuthenticationRequest {
-  /**
-   * Credentials for login
-   */
-  private PasswordCredentials passwordCredentials;
-
-  /**
-   * @param tenantName tenant
-   * @param passwordCredentials password credentials
-   */
-  public PasswordAuthenticationRequest(String tenantName, PasswordCredentials passwordCredentials) {
-    this.tenantName = tenantName;
-    this.passwordCredentials = passwordCredentials;
-  }
-
-  /**
-   * @return credentials for login into Keystone
-   */
-  public PasswordCredentials getPasswordCredentials() {
-    return passwordCredentials;
-  }
-
-  /**
-   * @param passwordCredentials credentials for login into Keystone
-   */
-  public void setPasswordCredentials(PasswordCredentials passwordCredentials) {
-    this.passwordCredentials = passwordCredentials;
-  }
-
-  @Override
-  public String toString() {
-    return "Authenticate as " +
-           "tenant '" + tenantName + "' "
-           + passwordCredentials;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordCredentials.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordCredentials.java
deleted file mode 100644
index 40d8c77feb49d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordCredentials.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-
-/**
- * Describes credentials to log in Swift using Keystone authentication.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class PasswordCredentials {
-  /**
-   * user login
-   */
-  private String username;
-
-  /**
-   * user password
-   */
-  private String password;
-
-  /**
-   * default constructor
-   */
-  public PasswordCredentials() {
-  }
-
-  /**
-   * @param username user login
-   * @param password user password
-   */
-  public PasswordCredentials(String username, String password) {
-    this.username = username;
-    this.password = password;
-  }
-
-  /**
-   * @return user password
-   */
-  public String getPassword() {
-    return password;
-  }
-
-  /**
-   * @param password user password
-   */
-  public void setPassword(String password) {
-    this.password = password;
-  }
-
-  /**
-   * @return login
-   */
-  public String getUsername() {
-    return username;
-  }
-
-  /**
-   * @param username login
-   */
-  public void setUsername(String username) {
-    this.username = username;
-  }
-
-  @Override
-  public String toString() {
-    return "PasswordCredentials{username='" + username + "'}";
-  }
-}
-
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/Roles.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/Roles.java
deleted file mode 100644
index 57f2fa6d45108..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/Roles.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth;
-
-/**
- * Describes user roles in Openstack system.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-public class Roles {
-  /**
-   * role name
-   */
-  private String name;
-
-  /**
-   * This field user in RackSpace auth model
-   */
-  private String id;
-
-  /**
-   * This field user in RackSpace auth model
-   */
-  private String description;
-
-  /**
-   * Service id used in HP public Cloud
-   */
-  private String serviceId;
-
-  /**
-   * Service id used in HP public Cloud
-   */
-  private String tenantId;
-
-  /**
-   * @return role name
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * @param name role name
-   */
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getId() {
-    return id;
-  }
-
-  public void setId(String id) {
-    this.id = id;
-  }
-
-  public String getDescription() {
-    return description;
-  }
-
-  public void setDescription(String description) {
-    this.description = description;
-  }
-
-  public String getServiceId() {
-    return serviceId;
-  }
-
-  public void setServiceId(String serviceId) {
-    this.serviceId = serviceId;
-  }
-
-  public String getTenantId() {
-    return tenantId;
-  }
-
-  public void setTenantId(String tenantId) {
-    this.tenantId = tenantId;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java
deleted file mode 100644
index b38d4660e5ae6..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth.entities;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/**
- * Access token representation of Openstack Keystone authentication.
- * Class holds token id, tenant and expiration time.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- *
- * Example:
- * <pre>
- * "token" : {
- *   "RAX-AUTH:authenticatedBy" : [ "APIKEY" ],
- *   "expires" : "2013-07-12T05:19:24.685-05:00",
- *   "id" : "8bbea4215113abdab9d4c8fb0d37",
- *   "tenant" : { "id" : "01011970",
- *   "name" : "77777"
- *   }
- *  }
- * </pre>
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-
-public class AccessToken {
-  /**
-   * token expiration time
-   */
-  private String expires;
-  /**
-   * token id
-   */
-  private String id;
-  /**
-   * tenant name for whom id is attached
-   */
-  private Tenant tenant;
-
-  /**
-   * @return token expiration time
-   */
-  public String getExpires() {
-    return expires;
-  }
-
-  /**
-   * @param expires the token expiration time
-   */
-  public void setExpires(String expires) {
-    this.expires = expires;
-  }
-
-  /**
-   * @return token value
-   */
-  public String getId() {
-    return id;
-  }
-
-  /**
-   * @param id token value
-   */
-  public void setId(String id) {
-    this.id = id;
-  }
-
-  /**
-   * @return tenant authenticated in Openstack Keystone
-   */
-  public Tenant getTenant() {
-    return tenant;
-  }
-
-  /**
-   * @param tenant tenant authenticated in Openstack Keystone
-   */
-  public void setTenant(Tenant tenant) {
-    this.tenant = tenant;
-  }
-
-  @Override
-  public String toString() {
-    return "AccessToken{" +
-            "id='" + id + '\'' +
-            ", tenant=" + tenant +
-            ", expires='" + expires + '\'' +
-            '}';
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java
deleted file mode 100644
index 76e161b064277..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth.entities;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-import java.util.List;
-
-/**
- * Describes Openstack Swift REST endpoints.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-
-public class Catalog {
-  /**
-   * List of valid swift endpoints
-   */
-  private List<Endpoint> endpoints;
-  /**
-   * endpoint links are additional information description
-   * which aren't used in Hadoop and Swift integration scope
-   */
-  private List<Object> endpoints_links;
-  /**
-   * Openstack REST service name. In our case name = "keystone"
-   */
-  private String name;
-
-  /**
-   * Type of REST service. In our case type = "identity"
-   */
-  private String type;
-
-  /**
-   * @return List of endpoints
-   */
-  public List<Endpoint> getEndpoints() {
-    return endpoints;
-  }
-
-  /**
-   * @param endpoints list of endpoints
-   */
-  public void setEndpoints(List<Endpoint> endpoints) {
-    this.endpoints = endpoints;
-  }
-
-  /**
-   * @return list of endpoint links
-   */
-  public List<Object> getEndpoints_links() {
-    return endpoints_links;
-  }
-
-  /**
-   * @param endpoints_links list of endpoint links
-   */
-  public void setEndpoints_links(List<Object> endpoints_links) {
-    this.endpoints_links = endpoints_links;
-  }
-
-  /**
-   * @return name of Openstack REST service
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * @param name of Openstack REST service
-   */
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  /**
-   * @return type of Openstack REST service
-   */
-  public String getType() {
-    return type;
-  }
-
-  /**
-   * @param type of REST service
-   */
-  public void setType(String type) {
-    this.type = type;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java
deleted file mode 100644
index b1cbf2acc7b4a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth.entities;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-import java.net.URI;
-
-/**
- * Openstack Swift endpoint description.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-
-public class Endpoint {
-
-  /**
-   * endpoint id
-   */
-  private String id;
-
-  /**
-   * Keystone admin URL
-   */
-  private URI adminURL;
-
-  /**
-   * Keystone internal URL
-   */
-  private URI internalURL;
-
-  /**
-   * public accessible URL
-   */
-  private URI publicURL;
-
-  /**
-   * public accessible URL#2
-   */
-  private URI publicURL2;
-
-  /**
-   * Openstack region name
-   */
-  private String region;
-
-  /**
-   * This field is used in RackSpace authentication model
-   */
-  private String tenantId;
-
-  /**
-   * This field user in RackSpace auth model
-   */
-  private String versionId;
-
-  /**
-   * This field user in RackSpace auth model
-   */
-  private String versionInfo;
-
-  /**
-   * This field user in RackSpace auth model
-   */
-  private String versionList;
-
-
-  /**
-   * @return endpoint id
-   */
-  public String getId() {
-    return id;
-  }
-
-  /**
-   * @param id endpoint id
-   */
-  public void setId(String id) {
-    this.id = id;
-  }
-
-  /**
-   * @return Keystone admin URL
-   */
-  public URI getAdminURL() {
-    return adminURL;
-  }
-
-  /**
-   * @param adminURL Keystone admin URL
-   */
-  public void setAdminURL(URI adminURL) {
-    this.adminURL = adminURL;
-  }
-
-  /**
-   * @return internal Keystone
-   */
-  public URI getInternalURL() {
-    return internalURL;
-  }
-
-  /**
-   * @param internalURL Keystone internal URL
-   */
-  public void setInternalURL(URI internalURL) {
-    this.internalURL = internalURL;
-  }
-
-  /**
-   * @return public accessible URL
-   */
-  public URI getPublicURL() {
-    return publicURL;
-  }
-
-  /**
-   * @param publicURL public URL
-   */
-  public void setPublicURL(URI publicURL) {
-    this.publicURL = publicURL;
-  }
-
-  public URI getPublicURL2() {
-    return publicURL2;
-  }
-
-  public void setPublicURL2(URI publicURL2) {
-    this.publicURL2 = publicURL2;
-  }
-
-  /**
-   * @return Openstack region name
-   */
-  public String getRegion() {
-    return region;
-  }
-
-  /**
-   * @param region Openstack region name
-   */
-  public void setRegion(String region) {
-    this.region = region;
-  }
-
-  public String getTenantId() {
-    return tenantId;
-  }
-
-  public void setTenantId(String tenantId) {
-    this.tenantId = tenantId;
-  }
-
-  public String getVersionId() {
-    return versionId;
-  }
-
-  public void setVersionId(String versionId) {
-    this.versionId = versionId;
-  }
-
-  public String getVersionInfo() {
-    return versionInfo;
-  }
-
-  public void setVersionInfo(String versionInfo) {
-    this.versionInfo = versionInfo;
-  }
-
-  public String getVersionList() {
-    return versionList;
-  }
-
-  public void setVersionList(String versionList) {
-    this.versionList = versionList;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java
deleted file mode 100644
index 405d2c853682c..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth.entities;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/**
- * Tenant is abstraction in Openstack which describes all account
- * information and user privileges in system.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class Tenant {
-
-  /**
-   * tenant id
-   */
-  private String id;
-
-  /**
-   * tenant short description which Keystone returns
-   */
-  private String description;
-
-  /**
-   * boolean enabled user account or no
-   */
-  private boolean enabled;
-
-  /**
-   * tenant human readable name
-   */
-  private String name;
-
-  /**
-   * @return tenant name
-   */
-  public String getName() {
-    return name;
-  }
-
-  /**
-   * @param name tenant name
-   */
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  /**
-   * @return true if account enabled and false otherwise
-   */
-  public boolean isEnabled() {
-    return enabled;
-  }
-
-  /**
-   * @param enabled enable or disable
-   */
-  public void setEnabled(boolean enabled) {
-    this.enabled = enabled;
-  }
-
-  /**
-   * @return account short description
-   */
-  public String getDescription() {
-    return description;
-  }
-
-  /**
-   * @param description set account description
-   */
-  public void setDescription(String description) {
-    this.description = description;
-  }
-
-  /**
-   * @return set tenant id
-   */
-  public String getId() {
-    return id;
-  }
-
-  /**
-   * @param id tenant id
-   */
-  public void setId(String id) {
-    this.id = id;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java
deleted file mode 100644
index da3bac20f2b42..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.auth.entities;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import org.apache.hadoop.fs.swift.auth.Roles;
-
-import java.util.List;
-
-/**
- * Describes user entity in Keystone
- * In different Swift installations User is represented differently.
- * To avoid any JSON deserialization failures this entity is ignored.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class User {
-
-  /**
-   * user id in Keystone
-   */
-  private String id;
-
-  /**
-   * user human readable name
-   */
-  private String name;
-
-  /**
-   * user roles in Keystone
-   */
-  private List<Roles> roles;
-
-  /**
-   * links to user roles
-   */
-  private List<Object> roles_links;
-
-  /**
-   * human readable username in Keystone
-   */
-  private String username;
-
-  /**
-   * @return user id
-   */
-  public String getId() {
-    return id;
-  }
-
-  /**
-   * @param id user id
-   */
-  public void setId(String id) {
-    this.id = id;
-  }
-
-
-  /**
-   * @return user name
-   */
-  public String getName() {
-    return name;
-  }
-
-
-  /**
-   * @param name user name
-   */
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  /**
-   * @return user roles
-   */
-  public List<Roles> getRoles() {
-    return roles;
-  }
-
-  /**
-   * @param roles sets user roles
-   */
-  public void setRoles(List<Roles> roles) {
-    this.roles = roles;
-  }
-
-  /**
-   * @return user roles links
-   */
-  public List<Object> getRoles_links() {
-    return roles_links;
-  }
-
-  /**
-   * @param roles_links user roles links
-   */
-  public void setRoles_links(List<Object> roles_links) {
-    this.roles_links = roles_links;
-  }
-
-  /**
-   * @return username
-   */
-  public String getUsername() {
-    return username;
-  }
-
-  /**
-   * @param username human readable user name
-   */
-  public void setUsername(String username) {
-    this.username = username;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java
deleted file mode 100644
index fdb9a3973ad08..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-import org.apache.http.HttpResponse;
-
-import java.net.URI;
-
-/**
- * An exception raised when an authentication request was rejected
- */
-public class SwiftAuthenticationFailedException extends SwiftInvalidResponseException {
-
-  public SwiftAuthenticationFailedException(String message,
-                                            int statusCode,
-                                            String operation,
-                                            URI uri) {
-    super(message, statusCode, operation, uri);
-  }
-
-  public SwiftAuthenticationFailedException(String message,
-                                            String operation,
-                                            URI uri,
-                                            HttpResponse resp) {
-    super(message, operation, uri, resp);
-  }
-
-  @Override
-  public String exceptionTitle() {
-    return "Authentication Failure";
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java
deleted file mode 100644
index f5b2abde0a9a8..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-import org.apache.http.HttpResponse;
-
-import java.net.URI;
-
-/**
- * Thrown to indicate that data locality can't be calculated or requested path is incorrect.
- * Data locality can't be calculated if Openstack Swift version is old.
- */
-public class SwiftBadRequestException extends SwiftInvalidResponseException {
-
-  public SwiftBadRequestException(String message,
-                                  String operation,
-                                  URI uri,
-                                  HttpResponse resp) {
-    super(message, operation, uri, resp);
-  }
-
-  public SwiftBadRequestException(String message,
-                                  int statusCode,
-                                  String operation,
-                                  URI uri) {
-    super(message, statusCode, operation, uri);
-  }
-
-  @Override
-  public String exceptionTitle() {
-    return "BadRequest";
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConfigurationException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConfigurationException.java
deleted file mode 100644
index 3651f2e050557..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConfigurationException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-/**
- * Exception raised to indicate there is some problem with how the Swift FS
- * is configured
- */
-public class SwiftConfigurationException extends SwiftException {
-  public SwiftConfigurationException(String message) {
-    super(message);
-  }
-
-  public SwiftConfigurationException(String message, Throwable cause) {
-    super(message, cause);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInternalStateException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInternalStateException.java
deleted file mode 100644
index 0f3e5d988497f..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInternalStateException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-/**
- * The internal state of the Swift client is wrong -presumably a sign
- * of some bug
- */
-public class SwiftInternalStateException extends SwiftException {
-
-  public SwiftInternalStateException(String message) {
-    super(message);
-  }
-
-  public SwiftInternalStateException(String message, Throwable cause) {
-    super(message, cause);
-  }
-
-  public SwiftInternalStateException(Throwable cause) {
-    super(cause);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java
deleted file mode 100644
index e90e57519b939..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-import org.apache.hadoop.fs.swift.util.HttpResponseUtils;
-import org.apache.http.HttpResponse;
-
-import java.io.IOException;
-import java.net.URI;
-
-/**
- * Exception raised when the HTTP code is invalid. The status code,
- * method name and operation URI are all in the response.
- */
-public class SwiftInvalidResponseException extends SwiftConnectionException {
-
-  public final int statusCode;
-  public final String operation;
-  public final URI uri;
-  public final String body;
-
-  public SwiftInvalidResponseException(String message,
-                                       int statusCode,
-                                       String operation,
-                                       URI uri) {
-    super(message);
-    this.statusCode = statusCode;
-    this.operation = operation;
-    this.uri = uri;
-    this.body = "";
-  }
-
-  public SwiftInvalidResponseException(String message,
-                                       String operation,
-                                       URI uri,
-                                       HttpResponse resp) {
-    super(message);
-    this.statusCode = resp.getStatusLine().getStatusCode();
-    this.operation = operation;
-    this.uri = uri;
-    String bodyAsString;
-    try {
-      bodyAsString = HttpResponseUtils.getResponseBodyAsString(resp);
-      if (bodyAsString == null) {
-        bodyAsString = "";
-      }
-    } catch (IOException e) {
-      bodyAsString = "";
-    }
-    this.body = bodyAsString;
-  }
-
-  public int getStatusCode() {
-    return statusCode;
-  }
-
-  public String getOperation() {
-    return operation;
-  }
-
-  public URI getUri() {
-    return uri;
-  }
-
-  public String getBody() {
-    return body;
-  }
-
-  /**
-   * Override point: title of an exception -this is used in the
-   * toString() method.
-   * @return the new exception title
-   */
-  public String exceptionTitle() {
-    return "Invalid Response";
-  }
-
-  /**
-   * Build a description that includes the exception title, the URI,
-   * the message, the status code -and any body of the response
-   * @return the string value for display
-   */
-  @Override
-  public String toString() {
-    StringBuilder msg = new StringBuilder();
-    msg.append(exceptionTitle());
-    msg.append(": ");
-    msg.append(getMessage());
-    msg.append("  ");
-    msg.append(operation);
-    msg.append(" ");
-    msg.append(uri);
-    msg.append(" => ");
-    msg.append(statusCode);
-    if (body != null && !body.isEmpty()) {
-      msg.append(" : ");
-      msg.append(body);
-    }
-
-    return msg.toString();
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java
deleted file mode 100644
index 0b078d7f43337..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-/**
- * Exception raised when the J/O mapping fails.
- */
-public class SwiftJsonMarshallingException extends SwiftException {
-
-  public SwiftJsonMarshallingException(String message) {
-    super(message);
-  }
-
-  public SwiftJsonMarshallingException(String message, Throwable cause) {
-    super(message, cause);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java
deleted file mode 100644
index 8f78f70f44b5d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-/**
- * Used to relay exceptions upstream from the inner implementation
- * to the public API, where it is downgraded to a log+failure.
- * Making it visible internally aids testing
- */
-public class SwiftOperationFailedException extends SwiftException {
-
-  public SwiftOperationFailedException(String message) {
-    super(message);
-  }
-
-  public SwiftOperationFailedException(String message, Throwable cause) {
-    super(message, cause);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java
deleted file mode 100644
index 1e7ca67d1b00f..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-import org.apache.http.HttpResponse;
-
-import java.net.URI;
-
-/**
- * Exception raised if a Swift endpoint returned a HTTP response indicating
- * the caller is being throttled.
- */
-public class SwiftThrottledRequestException extends
-                                            SwiftInvalidResponseException {
-  public SwiftThrottledRequestException(String message,
-                                        String operation,
-                                        URI uri,
-                                        HttpResponse resp) {
-    super(message, operation, uri, resp);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java
deleted file mode 100644
index b7e011c59ab7c..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.exceptions;
-
-/**
- * Exception raised on an unsupported feature in the FS API -such as
- * <code>append()</code>
- */
-public class SwiftUnsupportedFeatureException extends SwiftException {
-
-  public SwiftUnsupportedFeatureException(String message) {
-    super(message);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java
deleted file mode 100644
index d159caa66909a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.net.ConnectException;
-import java.net.NoRouteToHostException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-
-/**
- * Variant of Hadoop NetUtils exception wrapping with URI awareness and
- * available in branch-1 too.
- */
-public class ExceptionDiags {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ExceptionDiags.class);
-
-  /** text to point users elsewhere: {@value} */
-  private static final String FOR_MORE_DETAILS_SEE
-    = " For more details see:  ";
-  /** text included in wrapped exceptions if the host is null: {@value} */
-  public static final String UNKNOWN_HOST = "(unknown)";
-  /** Base URL of the Hadoop Wiki: {@value} */
-  public static final String HADOOP_WIKI = "http://wiki.apache.org/hadoop/";
-
-  /**
-   * Take an IOException and a URI, wrap it where possible with
-   * something that includes the URI
-   *
-   * @param dest target URI
-   * @param operation operation
-   * @param exception the caught exception.
-   * @return an exception to throw
-   */
-  public static IOException wrapException(final String dest,
-                                          final String operation,
-                                          final IOException exception) {
-    String action = operation + " " + dest;
-    String xref = null;
-
-    if (exception instanceof ConnectException) {
-      xref = "ConnectionRefused";
-    } else if (exception instanceof UnknownHostException) {
-      xref = "UnknownHost";
-    } else if (exception instanceof SocketTimeoutException) {
-      xref = "SocketTimeout";
-    } else if (exception instanceof NoRouteToHostException) {
-      xref = "NoRouteToHost";
-    }
-    String msg = action
-                 + " failed on exception: "
-                 + exception;
-    if (xref != null) {
-       msg = msg + ";" + see(xref);
-    }
-    return wrapWithMessage(exception, msg);
-  }
-
-  private static String see(final String entry) {
-    return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry;
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <T extends IOException> T wrapWithMessage(
-    T exception, String msg) {
-    Class<? extends Throwable> clazz = exception.getClass();
-    try {
-      Constructor<? extends Throwable> ctor =
-        clazz.getConstructor(String.class);
-      Throwable t = ctor.newInstance(msg);
-      return (T) (t.initCause(exception));
-    } catch (Throwable e) {
-      return exception;
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java
deleted file mode 100644
index bd025aca1b8d7..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-
-/**
- * This replaces the input stream release class from JetS3t and AWS;
- * # Failures in the constructor are relayed up instead of simply logged.
- * # it is set up to be more robust at teardown
- * # release logic is thread safe
- * Note that the thread safety of the inner stream contains no thread
- * safety guarantees -this stream is not to be read across streams.
- * The thread safety logic here is to ensure that even if somebody ignores
- * that rule, the release code does not get entered twice -and that
- * any release in one thread is picked up by read operations in all others.
- */
-public class HttpInputStreamWithRelease extends InputStream {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(HttpInputStreamWithRelease.class);
-  private final URI uri;
-  private HttpRequestBase req;
-  private HttpResponse resp;
-  //flag to say the stream is released -volatile so that read operations
-  //pick it up even while unsynchronized.
-  private volatile boolean released;
-  //volatile flag to verify that data is consumed.
-  private volatile boolean dataConsumed;
-  private InputStream inStream;
-  /**
-   * In debug builds, this is filled in with the construction-time
-   * stack, which is then included in logs from the finalize(), method.
-   */
-  private final Exception constructionStack;
-
-  /**
-   * Why the stream is closed
-   */
-  private String reasonClosed = "unopened";
-
-  public HttpInputStreamWithRelease(URI uri, HttpRequestBase req,
-      HttpResponse resp) throws IOException {
-    this.uri = uri;
-    this.req = req;
-    this.resp = resp;
-    constructionStack = LOG.isDebugEnabled() ? new Exception("stack") : null;
-    if (req == null) {
-      throw new IllegalArgumentException("Null 'request' parameter ");
-    }
-    try {
-      inStream = resp.getEntity().getContent();
-    } catch (IOException e) {
-      inStream = new ByteArrayInputStream(new byte[]{});
-      throw releaseAndRethrow("getResponseBodyAsStream() in constructor -" + e, e);
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    release("close()", null);
-  }
-
-  /**
-   * Release logic
-   * @param reason reason for release (used in debug messages)
-   * @param ex exception that is a cause -null for non-exceptional releases
-   * @return true if the release took place here
-   * @throws IOException if the abort or close operations failed.
-   */
-  private synchronized boolean release(String reason, Exception ex) throws
-                                                                   IOException {
-    if (!released) {
-      reasonClosed = reason;
-      try {
-        LOG.debug("Releasing connection to {}:  {}", uri, reason, ex);
-        if (req != null) {
-          if (!dataConsumed) {
-            req.abort();
-          }
-          req.releaseConnection();
-        }
-        if (inStream != null) {
-          //this guard may seem un-needed, but a stack trace seen
-          //on the JetS3t predecessor implied that it
-          //is useful
-          inStream.close();
-        }
-        return true;
-      } finally {
-        //if something went wrong here, we do not want the release() operation
-        //to try and do anything in advance.
-        released = true;
-        dataConsumed = true;
-      }
-    } else {
-      return false;
-    }
-  }
-
-  /**
-   * Release the method, using the exception as a cause
-   * @param operation operation that failed
-   * @param ex the exception which triggered it.
-   * @return the exception to throw
-   */
-  private IOException releaseAndRethrow(String operation, IOException ex) {
-    try {
-      release(operation, ex);
-    } catch (IOException ioe) {
-      LOG.debug("Exception during release: {}", operation, ioe);
-      //make this the exception if there was none before
-      if (ex == null) {
-        ex = ioe;
-      }
-    }
-    return ex;
-  }
-
-  /**
-   * Assume that the connection is not released: throws an exception if it is
-   * @throws SwiftConnectionClosedException
-   */
-  private synchronized void assumeNotReleased() throws SwiftConnectionClosedException {
-    if (released || inStream == null) {
-      throw new SwiftConnectionClosedException(reasonClosed);
-    }
-  }
-
-  @Override
-  public int available() throws IOException {
-    assumeNotReleased();
-    try {
-      return inStream.available();
-    } catch (IOException e) {
-      throw releaseAndRethrow("available() failed -" + e, e);
-    }
-  }
-
-  @Override
-  public int read() throws IOException {
-    assumeNotReleased();
-    int read = 0;
-    try {
-      read = inStream.read();
-    } catch (EOFException e) {
-      LOG.debug("EOF exception", e);
-      read = -1;
-    } catch (IOException e) {
-      throw releaseAndRethrow("read()", e);
-    }
-    if (read < 0) {
-      dataConsumed = true;
-      release("read() -all data consumed", null);
-    }
-    return read;
-  }
-
-  @Override
-  public int read(byte[] b, int off, int len) throws IOException {
-    SwiftUtils.validateReadArgs(b, off, len);
-    if (len == 0) {
-      return 0;
-    }
-    //if the stream is already closed, then report an exception.
-    assumeNotReleased();
-    //now read in a buffer, reacting differently to different operations
-    int read;
-    try {
-      read = inStream.read(b, off, len);
-    } catch (EOFException e) {
-      LOG.debug("EOF exception", e);
-      read = -1;
-    } catch (IOException e) {
-      throw releaseAndRethrow("read(b, off, " + len + ")", e);
-    }
-    if (read < 0) {
-      dataConsumed = true;
-      release("read() -all data consumed", null);
-    }
-    return read;
-  }
-
-  /**
-   * Finalizer does release the stream, but also logs at WARN level
-   * including the URI at fault
-   */
-  @Override
-  protected void finalize() {
-    try {
-      if (release("finalize()", constructionStack)) {
-        LOG.warn("input stream of {}" +
-                 " not closed properly -cleaned up in finalize()", uri);
-      }
-    } catch (Exception e) {
-      //swallow anything that failed here
-      LOG.warn("Exception while releasing {} in finalizer", uri, e);
-    }
-  }
-
-  @Override
-  public String toString() {
-    return "HttpInputStreamWithRelease working with " + uri
-      +" released=" + released
-      +" dataConsumed=" + dataConsumed;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java
deleted file mode 100644
index f6917d3ffaea6..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-
-import java.net.URI;
-import java.util.Properties;
-
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;
-
-/**
- * This class implements the binding logic between Hadoop configurations
- * and the swift rest client.
- * <p>
- * The swift rest client takes a Properties instance containing
- * the string values it uses to bind to a swift endpoint.
- * <p>
- * This class extracts the values for a specific filesystem endpoint
- * and then builds an appropriate Properties file.
- */
-public final class RestClientBindings {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RestClientBindings.class);
-
-  public static final String E_INVALID_NAME = "Invalid swift hostname '%s':" +
-          " hostname must in form container.service";
-
-  /**
-   * Public for testing : build the full prefix for use in resolving
-   * configuration items
-   *
-   * @param service service to use
-   * @return the prefix string <i>without any trailing "."</i>
-   */
-  public static String buildSwiftInstancePrefix(String service) {
-    return SWIFT_SERVICE_PREFIX + service;
-  }
-
-  /**
-   * Raise an exception for an invalid service name
-   *
-   * @param hostname hostname that was being parsed
-   * @return an exception to throw
-   */
-  private static SwiftConfigurationException invalidName(String hostname) {
-    return new SwiftConfigurationException(
-            String.format(E_INVALID_NAME, hostname));
-  }
-
-  /**
-   * Get the container name from the hostname -the single element before the
-   * first "." in the hostname
-   *
-   * @param hostname hostname to split
-   * @return the container
-   * @throws SwiftConfigurationException
-   */
-  public static String extractContainerName(String hostname) throws
-          SwiftConfigurationException {
-    int i = hostname.indexOf(".");
-    if (i <= 0) {
-      throw invalidName(hostname);
-    }
-    return hostname.substring(0, i);
-  }
-
-  public static String extractContainerName(URI uri) throws
-          SwiftConfigurationException {
-    return extractContainerName(uri.getHost());
-  }
-
-  /**
-   * Get the service name from a longer hostname string
-   *
-   * @param hostname hostname
-   * @return the separated out service name
-   * @throws SwiftConfigurationException if the hostname was invalid
-   */
-  public static String extractServiceName(String hostname) throws
-          SwiftConfigurationException {
-    int i = hostname.indexOf(".");
-    if (i <= 0) {
-      throw invalidName(hostname);
-    }
-    String service = hostname.substring(i + 1);
-    if (service.isEmpty() || service.contains(".")) {
-      //empty service contains dots in -not currently supported
-      throw invalidName(hostname);
-    }
-    return service;
-  }
-
-  public static String extractServiceName(URI uri) throws
-          SwiftConfigurationException {
-    return extractServiceName(uri.getHost());
-  }
-
-  /**
-   * Build a properties instance bound to the configuration file -using
-   * the filesystem URI as the source of the information.
-   *
-   * @param fsURI filesystem URI
-   * @param conf  configuration
-   * @return a properties file with the instance-specific properties extracted
-   *         and bound to the swift client properties.
-   * @throws SwiftConfigurationException if the configuration is invalid
-   */
-  public static Properties bind(URI fsURI, Configuration conf) throws
-          SwiftConfigurationException {
-    String host = fsURI.getHost();
-    if (host == null || host.isEmpty()) {
-      //expect shortnames -> conf names
-      throw invalidName(host);
-    }
-
-    String container = extractContainerName(host);
-    String service = extractServiceName(host);
-
-    //build filename schema
-    String prefix = buildSwiftInstancePrefix(service);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Filesystem " + fsURI
-              + " is using configuration keys " + prefix);
-    }
-    Properties props = new Properties();
-    props.setProperty(SWIFT_SERVICE_PROPERTY, service);
-    props.setProperty(SWIFT_CONTAINER_PROPERTY, container);
-    copy(conf, prefix + DOT_AUTH_URL, props, SWIFT_AUTH_PROPERTY, true);
-    copy(conf, prefix + DOT_USERNAME, props, SWIFT_USERNAME_PROPERTY, true);
-    copy(conf, prefix + DOT_APIKEY, props, SWIFT_APIKEY_PROPERTY, false);
-    copy(conf, prefix + DOT_PASSWORD, props, SWIFT_PASSWORD_PROPERTY,
-            props.contains(SWIFT_APIKEY_PROPERTY) ? true : false);
-    copy(conf, prefix + DOT_TENANT, props, SWIFT_TENANT_PROPERTY, false);
-    copy(conf, prefix + DOT_REGION, props, SWIFT_REGION_PROPERTY, false);
-    copy(conf, prefix + DOT_HTTP_PORT, props, SWIFT_HTTP_PORT_PROPERTY, false);
-    copy(conf, prefix +
-            DOT_HTTPS_PORT, props, SWIFT_HTTPS_PORT_PROPERTY, false);
-
-    copyBool(conf, prefix + DOT_PUBLIC, props, SWIFT_PUBLIC_PROPERTY, false);
-    copyBool(conf, prefix + DOT_LOCATION_AWARE, props,
-             SWIFT_LOCATION_AWARE_PROPERTY, false);
-
-    return props;
-  }
-
-  /**
-   * Extract a boolean value from the configuration and copy it to the
-   * properties instance.
-   * @param conf     source configuration
-   * @param confKey  key in the configuration file
-   * @param props    destination property set
-   * @param propsKey key in the property set
-   * @param defVal default value
-   */
-  private static void copyBool(Configuration conf,
-                               String confKey,
-                               Properties props,
-                               String propsKey,
-                               boolean defVal) {
-    boolean b = conf.getBoolean(confKey, defVal);
-    props.setProperty(propsKey, Boolean.toString(b));
-  }
-
-  private static void set(Properties props, String key, String optVal) {
-    if (optVal != null) {
-      props.setProperty(key, optVal);
-    }
-  }
-
-  /**
-   * Copy a (trimmed) property from the configuration file to the properties file.
-   * <p>
-   * If marked as required and not found in the configuration, an
-   * exception is raised.
-   * If not required -and missing- then the property will not be set.
-   * In this case, if the property is already in the Properties instance,
-   * it will remain untouched.
-   *
-   * @param conf     source configuration
-   * @param confKey  key in the configuration file
-   * @param props    destination property set
-   * @param propsKey key in the property set
-   * @param required is the property required
-   * @throws SwiftConfigurationException if the property is required but was
-   *                                     not found in the configuration instance.
-   */
-  public static void copy(Configuration conf, String confKey, Properties props,
-                          String propsKey,
-                          boolean required) throws SwiftConfigurationException {
-    //TODO: replace. version compatibility issue conf.getTrimmed fails with NoSuchMethodError
-    String val = conf.get(confKey);
-    if (val != null) {
-      val = val.trim();
-    }
-    if (required && val == null) {
-      throw new SwiftConfigurationException(
-              "Missing mandatory configuration option: "
-                      +
-                      confKey);
-    }
-    set(props, propsKey, val);
-  }
-
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java
deleted file mode 100644
index a01f32c18b2b9..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.apache.hadoop.util.VersionInfo;
-
-/**
- * Constants used in the Swift REST protocol,
- * and in the properties used to configure the {@link SwiftRestClient}.
- */
-public class SwiftProtocolConstants {
-  /**
-   * Swift-specific header for authentication: {@value}
-   */
-  public static final String HEADER_AUTH_KEY = "X-Auth-Token";
-
-  /**
-   * Default port used by Swift for HTTP
-   */
-  public static final int SWIFT_HTTP_PORT = 8080;
-
-  /**
-   * Default port used by Swift Auth for HTTPS
-   */
-  public static final int SWIFT_HTTPS_PORT = 443;
-
-  /** HTTP standard {@value} header */
-  public static final String HEADER_RANGE = "Range";
-
-  /** HTTP standard {@value} header */
-  public static final String HEADER_DESTINATION = "Destination";
-
-  /** HTTP standard {@value} header */
-  public static final String HEADER_LAST_MODIFIED = "Last-Modified";
-
-  /** HTTP standard {@value} header */
-  public static final String HEADER_CONTENT_LENGTH = "Content-Length";
-
-  /** HTTP standard {@value} header */
-  public static final String HEADER_CONTENT_RANGE = "Content-Range";
-
-  /**
-   * Patten for range headers
-   */
-  public static final String SWIFT_RANGE_HEADER_FORMAT_PATTERN = "bytes=%d-%d";
-
-  /**
-   *  section in the JSON catalog provided after auth listing the swift FS:
-   *  {@value}
-   */
-  public static final String SERVICE_CATALOG_SWIFT = "swift";
-  /**
-   *  section in the JSON catalog provided after auth listing the cloud files;
-   * this is an alternate catalog entry name
-   *  {@value}
-   */
-  public static final String SERVICE_CATALOG_CLOUD_FILES = "cloudFiles";
-  /**
-   *  section in the JSON catalog provided after auth listing the object store;
-   * this is an alternate catalog entry name
-   *  {@value}
-   */
-  public static final String SERVICE_CATALOG_OBJECT_STORE = "object-store";
-
-  /**
-   * entry in the swift catalog defining the prefix used to talk to objects
-   *  {@value}
-   */
-  public static final String SWIFT_OBJECT_AUTH_ENDPOINT =
-          "/object_endpoint/";
-  /**
-   * Swift-specific header: object manifest used in the final upload
-   * of a multipart operation: {@value}
-   */
-  public static final String X_OBJECT_MANIFEST = "X-Object-Manifest";
-  /**
-   * Swift-specific header -#of objects in a container: {@value}
-   */
-  public static final String X_CONTAINER_OBJECT_COUNT =
-          "X-Container-Object-Count";
-  /**
-   * Swift-specific header: no. of bytes used in a container {@value}
-   */
-  public static final String X_CONTAINER_BYTES_USED = "X-Container-Bytes-Used";
-
-  /**
-   * Header to set when requesting the latest version of a file: : {@value}
-   */
-  public static final String X_NEWEST = "X-Newest";
-
-  /**
-   * throttled response sent by some endpoints.
-   */
-  public static final int SC_THROTTLED_498 = 498;
-  /**
-   * W3C recommended status code for throttled operations
-   */
-  public static final int SC_TOO_MANY_REQUESTS_429 = 429;
-
-  public static final String FS_SWIFT = "fs.swift";
-
-  /**
-   * Prefix for all instance-specific values in the configuration: {@value}
-   */
-  public static final String SWIFT_SERVICE_PREFIX = FS_SWIFT + ".service.";
-
-  /**
-   * timeout for all connections: {@value}
-   */
-  public static final String SWIFT_CONNECTION_TIMEOUT =
-          FS_SWIFT + ".connect.timeout";
-
-  /**
-   * timeout for all connections: {@value}
-   */
-  public static final String SWIFT_SOCKET_TIMEOUT =
-          FS_SWIFT + ".socket.timeout";
-
-  /**
-   * the default socket timeout in millis {@value}.
-   * This controls how long the connection waits for responses from
-   * servers.
-   */
-  public static final int DEFAULT_SOCKET_TIMEOUT = 60000;
-
-  /**
-   * connection retry count for all connections: {@value}
-   */
-  public static final String SWIFT_RETRY_COUNT =
-          FS_SWIFT + ".connect.retry.count";
-
-  /**
-   * delay in millis between bulk (delete, rename, copy operations: {@value}
-   */
-  public static final String SWIFT_THROTTLE_DELAY =
-          FS_SWIFT + ".connect.throttle.delay";
-
-  /**
-   * the default throttle delay in millis {@value}
-   */
-  public static final int DEFAULT_THROTTLE_DELAY = 0;
-
-  /**
-   * blocksize for all filesystems: {@value}
-   */
-  public static final String SWIFT_BLOCKSIZE =
-          FS_SWIFT + ".blocksize";
-
-  /**
-   * the default blocksize for filesystems in KB: {@value}
-   */
-  public static final int DEFAULT_SWIFT_BLOCKSIZE = 32 * 1024;
-
-  /**
-   * partition size for all filesystems in KB: {@value}
-   */
-  public static final String SWIFT_PARTITION_SIZE =
-    FS_SWIFT + ".partsize";
-
-  /**
-   * The default partition size for uploads: {@value}
-   */
-  public static final int DEFAULT_SWIFT_PARTITION_SIZE = 4608*1024;
-
-  /**
-   * request size for reads in KB: {@value}
-   */
-  public static final String SWIFT_REQUEST_SIZE =
-    FS_SWIFT + ".requestsize";
-
-  /**
-   * The default request size for reads: {@value}
-   */
-  public static final int DEFAULT_SWIFT_REQUEST_SIZE = 64;
-
-
-  public static final String HEADER_USER_AGENT="User-Agent";
-
-  /**
-   * The user agent sent in requests.
-   */
-  public static final String SWIFT_USER_AGENT= "Apache Hadoop Swift Client "
-                                               + VersionInfo.getBuildVersion();
-
-  /**
-   * Key for passing the service name as a property -not read from the
-   * configuration : {@value}
-   */
-  public static final String DOT_SERVICE = ".SERVICE-NAME";
-
-  /**
-   * Key for passing the container name as a property -not read from the
-   * configuration : {@value}
-   */
-  public static final String DOT_CONTAINER = ".CONTAINER-NAME";
-
-  public static final String DOT_AUTH_URL = ".auth.url";
-  public static final String DOT_TENANT = ".tenant";
-  public static final String DOT_USERNAME = ".username";
-  public static final String DOT_PASSWORD = ".password";
-  public static final String DOT_HTTP_PORT = ".http.port";
-  public static final String DOT_HTTPS_PORT = ".https.port";
-  public static final String DOT_REGION = ".region";
-  public static final String DOT_PROXY_HOST = ".proxy.host";
-  public static final String DOT_PROXY_PORT = ".proxy.port";
-  public static final String DOT_LOCATION_AWARE = ".location-aware";
-  public static final String DOT_APIKEY = ".apikey";
-  public static final String DOT_USE_APIKEY = ".useApikey";
-
-  /**
-   * flag to say use public URL
-   */
-  public static final String DOT_PUBLIC = ".public";
-
-  public static final String SWIFT_SERVICE_PROPERTY = FS_SWIFT + DOT_SERVICE;
-  public static final String SWIFT_CONTAINER_PROPERTY = FS_SWIFT + DOT_CONTAINER;
-
-  public static final String SWIFT_AUTH_PROPERTY = FS_SWIFT + DOT_AUTH_URL;
-  public static final String SWIFT_TENANT_PROPERTY = FS_SWIFT + DOT_TENANT;
-  public static final String SWIFT_USERNAME_PROPERTY = FS_SWIFT + DOT_USERNAME;
-  public static final String SWIFT_PASSWORD_PROPERTY = FS_SWIFT + DOT_PASSWORD;
-  public static final String SWIFT_APIKEY_PROPERTY = FS_SWIFT + DOT_APIKEY;
-  public static final String SWIFT_HTTP_PORT_PROPERTY = FS_SWIFT + DOT_HTTP_PORT;
-  public static final String SWIFT_HTTPS_PORT_PROPERTY = FS_SWIFT
-          + DOT_HTTPS_PORT;
-  public static final String SWIFT_REGION_PROPERTY = FS_SWIFT + DOT_REGION;
-  public static final String SWIFT_PUBLIC_PROPERTY = FS_SWIFT + DOT_PUBLIC;
-
-  public static final String SWIFT_USE_API_KEY_PROPERTY = FS_SWIFT + DOT_USE_APIKEY;
-
-  public static final String SWIFT_LOCATION_AWARE_PROPERTY = FS_SWIFT +
-            DOT_LOCATION_AWARE;
-
-  public static final String SWIFT_PROXY_HOST_PROPERTY = FS_SWIFT + DOT_PROXY_HOST;
-  public static final String SWIFT_PROXY_PORT_PROPERTY = FS_SWIFT + DOT_PROXY_PORT;
-  public static final String HTTP_ROUTE_DEFAULT_PROXY =
-    "http.route.default-proxy";
-  /**
-   * Topology to return when a block location is requested
-   */
-  public static final String TOPOLOGY_PATH = "/swift/unknown";
-  /**
-   * Block location to return when a block location is requested
-   */
-  public static final String BLOCK_LOCATION = "/default-rack/swift";
-  /**
-   * Default number of attempts to retry a connect request: {@value}
-   */
-  static final int DEFAULT_RETRY_COUNT = 3;
-  /**
-   * Default timeout in milliseconds for connection requests: {@value}
-   */
-  static final int DEFAULT_CONNECT_TIMEOUT = 15000;
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java
deleted file mode 100644
index cf6bf9b972a10..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java
+++ /dev/null
@@ -1,1879 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.apache.hadoop.fs.swift.util.HttpResponseUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.config.SocketConfig;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.swift.auth.ApiKeyAuthenticationRequest;
-import org.apache.hadoop.fs.swift.auth.ApiKeyCredentials;
-import org.apache.hadoop.fs.swift.auth.AuthenticationRequest;
-import org.apache.hadoop.fs.swift.auth.AuthenticationRequestWrapper;
-import org.apache.hadoop.fs.swift.auth.AuthenticationResponse;
-import org.apache.hadoop.fs.swift.auth.AuthenticationWrapper;
-import org.apache.hadoop.fs.swift.auth.KeyStoneAuthRequest;
-import org.apache.hadoop.fs.swift.auth.KeystoneApiKeyCredentials;
-import org.apache.hadoop.fs.swift.auth.PasswordAuthenticationRequest;
-import org.apache.hadoop.fs.swift.auth.PasswordCredentials;
-import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
-import org.apache.hadoop.fs.swift.auth.entities.Catalog;
-import org.apache.hadoop.fs.swift.auth.entities.Endpoint;
-import org.apache.hadoop.fs.swift.exceptions.SwiftAuthenticationFailedException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftThrottledRequestException;
-import org.apache.hadoop.fs.swift.util.Duration;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.DurationStatsTable;
-import org.apache.hadoop.fs.swift.util.JSONUtil;
-import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.util.List;
-import java.util.Properties;
-
-import static org.apache.http.HttpStatus.*;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;
-
-/**
- * This implements the client-side of the Swift REST API.
- *
- * The core actions put, get and query data in the Swift object store,
- * after authenticating the client.
- *
- * <b>Logging:</b>
- *
- * Logging at DEBUG level displays detail about the actions of this
- * client, including HTTP requests and responses -excluding authentication
- * details.
- */
-public final class SwiftRestClient {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftRestClient.class);
-
-  /**
-   * Header that says "use newest version" -ensures that
-   * the query doesn't pick up older versions served by
-   * an eventually consistent filesystem (except in the special case
-   * of a network partition, at which point no guarantees about
-   * consistency can be made.
-   */
-  public static final Header NEWEST =
-           new BasicHeader(SwiftProtocolConstants.X_NEWEST, "true");
-
-  /**
-   * the authentication endpoint as supplied in the configuration.
-   */
-  private final URI authUri;
-
-  /**
-   * Swift region. Some OpenStack installations has more than one region.
-   * In this case user can specify the region with which Hadoop will be working
-   */
-  private final String region;
-
-  /**
-   * tenant name.
-   */
-  private final String tenant;
-
-  /**
-   * username name.
-   */
-  private final String username;
-
-  /**
-   * user password.
-   */
-  private final String password;
-
-  /**
-   * user api key.
-   */
-  private final String apiKey;
-
-  /**
-   * The authentication request used to authenticate with Swift.
-   */
-  private final AuthenticationRequest authRequest;
-
-  /**
-   * This auth request is similar to @see authRequest,
-   * with one difference: it has another json representation when
-   * authRequest one is not applicable.
-   */
-  private AuthenticationRequest keystoneAuthRequest;
-
-  private boolean useKeystoneAuthentication = false;
-
-  /**
-   * The container this client is working with.
-   */
-  private final String container;
-  private final String serviceDescription;
-
-  /**
-   * Access token (Secret).
-   */
-  private AccessToken token;
-
-  /**
-   * Endpoint for swift operations, obtained after authentication.
-   */
-  private URI endpointURI;
-
-  /**
-   * URI under which objects can be found.
-   * This is set when the user is authenticated -the URI
-   * is returned in the body of the success response.
-   */
-  private URI objectLocationURI;
-
-  /**
-   * The name of the service provider.
-   */
-  private final String serviceProvider;
-
-  /**
-   * Should the public swift endpoint be used, rather than the in-cluster one?
-   */
-  private final boolean usePublicURL;
-
-  /**
-   * Number of times to retry a connection.
-   */
-  private final int retryCount;
-
-  /**
-   * How long (in milliseconds) should a connection be attempted.
-   */
-  private final int connectTimeout;
-
-  /**
-   * How long (in milliseconds) should a connection be attempted.
-   */
-  private final int socketTimeout;
-
-  /**
-   * How long (in milliseconds) between bulk operations.
-   */
-  private final int throttleDelay;
-
-  /**
-  * the name of a proxy host (can be null, in which case there is no proxy).
-   */
-  private String proxyHost;
-
-  /**
-   * The port of a proxy. This is ignored if {@link #proxyHost} is null.
-   */
-  private int proxyPort;
-
-  /**
-   * Flag to indicate whether or not the client should
-   * query for file location data.
-   */
-  private final boolean locationAware;
-
-  private final int partSizeKB;
-  /**
-   * The blocksize of this FS
-   */
-  private final int blocksizeKB;
-  private final int bufferSizeKB;
-
-  private final DurationStatsTable durationStats = new DurationStatsTable();
-  /**
-   * objects query endpoint. This is synchronized
-   * to handle a simultaneous update of all auth data in one
-   * go.
-   */
-  private synchronized URI getEndpointURI() {
-    return endpointURI;
-  }
-
-  /**
-   * token for Swift communication.
-   */
-  private synchronized AccessToken getToken() {
-    return token;
-  }
-
-  /**
-   * Setter of authentication and endpoint details.
-   * Being synchronized guarantees that all three fields are set up together.
-   * It is up to the reader to read all three fields in their own
-   * synchronized block to be sure that they are all consistent.
-   *
-   * @param endpoint endpoint URI
-   * @param objectLocation object location URI
-   * @param authToken auth token
-   */
-  private void setAuthDetails(URI endpoint,
-                              URI objectLocation,
-                              AccessToken authToken) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(String.format("setAuth: endpoint=%s; objectURI=%s; token=%s",
-              endpoint, objectLocation, authToken));
-    }
-    synchronized (this) {
-      endpointURI = endpoint;
-      objectLocationURI = objectLocation;
-      token = authToken;
-    }
-  }
-
-
-  /**
-   * Base class for all Swift REST operations.
-   *
-   * @param <M> request
-   * @param <R> result
-   */
-  private static abstract class HttpRequestProcessor
-      <M extends HttpUriRequest, R> {
-    public final M createRequest(String uri) throws IOException {
-      final M req = doCreateRequest(uri);
-      setup(req);
-      return req;
-    }
-
-    /**
-     * Override it to return some result after request is executed.
-     */
-    public abstract R extractResult(M req, HttpResponse resp)
-        throws IOException;
-
-    /**
-     * Factory method to create a REST method against the given URI.
-     *
-     * @param uri target
-     * @return method to invoke
-     */
-    protected abstract M doCreateRequest(String uri) throws IOException;
-
-    /**
-     * Override port to set up the request before it is executed.
-     */
-    protected void setup(M req) throws IOException {
-    }
-
-    /**
-     * Override point: what are the status codes that this operation supports?
-     *
-     * @return an array with the permitted status code(s)
-     */
-    protected int[] getAllowedStatusCodes() {
-      return new int[]{
-              SC_OK,
-              SC_CREATED,
-              SC_ACCEPTED,
-              SC_NO_CONTENT,
-              SC_PARTIAL_CONTENT,
-      };
-    }
-  }
-
-  private static abstract class GetRequestProcessor<R>
-      extends HttpRequestProcessor<HttpGet, R> {
-    @Override
-    protected final HttpGet doCreateRequest(String uri) {
-      return new HttpGet(uri);
-    }
-  }
-
-  private static abstract class PostRequestProcessor<R>
-      extends HttpRequestProcessor<HttpPost, R> {
-    @Override
-    protected final HttpPost doCreateRequest(String uri) {
-      return new HttpPost(uri);
-    }
-  }
-
-  /**
-   * There's a special type for auth messages, so that low-level
-   * message handlers can react to auth failures differently from everything
-   * else.
-   */
-  private static final class AuthPostRequest extends HttpPost {
-    private AuthPostRequest(String uri) {
-      super(uri);
-    }
-  }
-
-  /**
-   * Generate an auth message.
-   * @param <R> response
-   */
-  private static abstract class AuthRequestProcessor<R>
-      extends HttpRequestProcessor<AuthPostRequest, R> {
-    @Override
-    protected final AuthPostRequest doCreateRequest(String uri) {
-      return new AuthPostRequest(uri);
-    }
-  }
-
-  private static abstract class PutRequestProcessor<R>
-      extends HttpRequestProcessor<HttpPut, R> {
-    @Override
-    protected final HttpPut doCreateRequest(String uri) {
-      return new HttpPut(uri);
-    }
-
-    /**
-     * Override point: what are the status codes that this operation supports?
-     *
-     * @return the list of status codes to accept
-     */
-    @Override
-    protected int[] getAllowedStatusCodes() {
-      return new int[]{
-              SC_OK,
-              SC_CREATED,
-              SC_NO_CONTENT,
-              SC_ACCEPTED,
-      };
-    }
-  }
-
-  /**
-   * Create operation.
-   *
-   * @param <R> result type
-   */
-  private static abstract class CopyRequestProcessor<R>
-      extends HttpRequestProcessor<CopyRequest, R> {
-    @Override
-    protected final CopyRequest doCreateRequest(String uri)
-        throws SwiftException {
-      CopyRequest copy = new CopyRequest();
-      try {
-        copy.setURI(new URI(uri));
-      } catch (URISyntaxException e) {
-        throw new SwiftException("Failed to create URI from: " + uri);
-      }
-      return copy;
-    }
-
-    /**
-     * The only allowed status code is 201:created.
-     * @return an array with the permitted status code(s)
-     */
-    @Override
-    protected int[] getAllowedStatusCodes() {
-      return new int[]{
-              SC_CREATED
-      };
-    }
-  }
-
-  /**
-   * Delete operation.
-   *
-   * @param <R>
-   */
-  private static abstract class DeleteRequestProcessor<R>
-      extends HttpRequestProcessor<HttpDelete, R> {
-    @Override
-    protected final HttpDelete doCreateRequest(String uri) {
-      return new HttpDelete(uri);
-    }
-
-    @Override
-    protected int[] getAllowedStatusCodes() {
-      return new int[]{
-              SC_OK,
-              SC_ACCEPTED,
-              SC_NO_CONTENT,
-              SC_NOT_FOUND
-      };
-    }
-  }
-
-  private static abstract class HeadRequestProcessor<R>
-      extends HttpRequestProcessor<HttpHead, R> {
-    @Override
-    protected final HttpHead doCreateRequest(String uri) {
-      return new HttpHead(uri);
-    }
-  }
-
-
-  /**
-   * Create a Swift Rest Client instance.
-   *
-   * @param filesystemURI filesystem URI
-   * @param conf The configuration to use to extract the binding
-   * @throws SwiftConfigurationException the configuration is not valid for
-   * defining a rest client against the service
-   */
-  private SwiftRestClient(URI filesystemURI,
-                          Configuration conf)
-      throws SwiftConfigurationException {
-    Properties props = RestClientBindings.bind(filesystemURI, conf);
-    String stringAuthUri = getOption(props, SWIFT_AUTH_PROPERTY);
-    username = getOption(props, SWIFT_USERNAME_PROPERTY);
-    password = props.getProperty(SWIFT_PASSWORD_PROPERTY);
-    apiKey = props.getProperty(SWIFT_APIKEY_PROPERTY);
-    //optional
-    region = props.getProperty(SWIFT_REGION_PROPERTY);
-    //tenant is optional
-    tenant = props.getProperty(SWIFT_TENANT_PROPERTY);
-    //service is used for diagnostics
-    serviceProvider = props.getProperty(SWIFT_SERVICE_PROPERTY);
-    container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
-    String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
-    usePublicURL = "true".equals(isPubProp);
-
-        if (apiKey == null && password == null) {
-            throw new SwiftConfigurationException(
-                    "Configuration for " + filesystemURI +" must contain either "
-                            + SWIFT_PASSWORD_PROPERTY + " or "
-                            + SWIFT_APIKEY_PROPERTY);
-        }
-        //create the (reusable) authentication request
-        if (password != null) {
-            authRequest = new PasswordAuthenticationRequest(tenant,
-                    new PasswordCredentials(
-                            username,
-                            password));
-        } else {
-            authRequest = new ApiKeyAuthenticationRequest(tenant,
-                    new ApiKeyCredentials(
-                            username, apiKey));
-            keystoneAuthRequest = new KeyStoneAuthRequest(tenant,
-                    new KeystoneApiKeyCredentials(username, apiKey));
-    }
-    locationAware = "true".equals(
-      props.getProperty(SWIFT_LOCATION_AWARE_PROPERTY, "false"));
-
-    //now read in properties that are shared across all connections
-
-    //connection and retries
-    try {
-      retryCount = conf.getInt(SWIFT_RETRY_COUNT, DEFAULT_RETRY_COUNT);
-      connectTimeout = conf.getInt(SWIFT_CONNECTION_TIMEOUT,
-                                   DEFAULT_CONNECT_TIMEOUT);
-      socketTimeout = conf.getInt(SWIFT_SOCKET_TIMEOUT,
-                                   DEFAULT_SOCKET_TIMEOUT);
-
-      throttleDelay = conf.getInt(SWIFT_THROTTLE_DELAY,
-                                  DEFAULT_THROTTLE_DELAY);
-
-      //proxy options
-      proxyHost = conf.get(SWIFT_PROXY_HOST_PROPERTY);
-      proxyPort = conf.getInt(SWIFT_PROXY_PORT_PROPERTY, 8080);
-
-      blocksizeKB = conf.getInt(SWIFT_BLOCKSIZE,
-                                DEFAULT_SWIFT_BLOCKSIZE);
-      if (blocksizeKB <= 0) {
-        throw new SwiftConfigurationException("Invalid blocksize set in "
-                          + SWIFT_BLOCKSIZE
-                          + ": " + blocksizeKB);
-      }
-      partSizeKB = conf.getInt(SWIFT_PARTITION_SIZE,
-                               DEFAULT_SWIFT_PARTITION_SIZE);
-      if (partSizeKB <=0) {
-        throw new SwiftConfigurationException("Invalid partition size set in "
-                                              + SWIFT_PARTITION_SIZE
-                                              + ": " + partSizeKB);
-      }
-
-      bufferSizeKB = conf.getInt(SWIFT_REQUEST_SIZE,
-                                 DEFAULT_SWIFT_REQUEST_SIZE);
-      if (bufferSizeKB <=0) {
-        throw new SwiftConfigurationException("Invalid buffer size set in "
-                          + SWIFT_REQUEST_SIZE
-                          + ": " + bufferSizeKB);
-      }
-    } catch (NumberFormatException e) {
-      //convert exceptions raised parsing ints and longs into
-      // SwiftConfigurationException instances
-      throw new SwiftConfigurationException(e.toString(), e);
-    }
-    //everything you need for diagnostics. The password is omitted.
-    serviceDescription = String.format(
-      "Service={%s} container={%s} uri={%s}"
-      + " tenant={%s} user={%s} region={%s}"
-      + " publicURL={%b}"
-      + " location aware={%b}"
-      + " partition size={%d KB}, buffer size={%d KB}"
-      + " block size={%d KB}"
-      + " connect timeout={%d}, retry count={%d}"
-      + " socket timeout={%d}"
-      + " throttle delay={%d}"
-      ,
-      serviceProvider,
-      container,
-      stringAuthUri,
-      tenant,
-      username,
-      region != null ? region : "(none)",
-      usePublicURL,
-      locationAware,
-      partSizeKB,
-      bufferSizeKB,
-      blocksizeKB,
-      connectTimeout,
-      retryCount,
-      socketTimeout,
-      throttleDelay
-      );
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(serviceDescription);
-    }
-    try {
-      this.authUri = new URI(stringAuthUri);
-    } catch (URISyntaxException e) {
-      throw new SwiftConfigurationException("The " + SWIFT_AUTH_PROPERTY
-              + " property was incorrect: "
-              + stringAuthUri, e);
-    }
-  }
-
-  /**
-   * Get a mandatory configuration option.
-   *
-   * @param props property set
-   * @param key   key
-   * @return value of the configuration
-   * @throws SwiftConfigurationException if there was no match for the key
-   */
-  private static String getOption(Properties props, String key) throws
-          SwiftConfigurationException {
-    String val = props.getProperty(key);
-    if (val == null) {
-      throw new SwiftConfigurationException("Undefined property: " + key);
-    }
-    return val;
-  }
-
-  /**
-   * Make an HTTP GET request to Swift to get a range of data in the object.
-   *
-   * @param path   path to object
-   * @param offset offset from file beginning
-   * @param length file length
-   * @return The input stream -which must be closed afterwards.
-   * @throws IOException Problems
-   * @throws SwiftException swift specific error
-   * @throws FileNotFoundException path is not there
-   */
-  public HttpBodyContent getData(SwiftObjectPath path,
-                                 long offset,
-                                 long length) throws IOException {
-    if (offset < 0) {
-      throw new SwiftException("Invalid offset: " + offset
-                            + " in getDataAsInputStream( path=" + path
-                            + ", offset=" + offset
-                            + ", length =" + length + ")");
-    }
-    if (length <= 0) {
-      throw new SwiftException("Invalid length: " + length
-                + " in getDataAsInputStream( path="+ path
-                            + ", offset=" + offset
-                            + ", length ="+ length + ")");
-    }
-
-    final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
-            offset,
-            offset + length - 1);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("getData:" + range);
-    }
-
-    return getData(path,
-                   new BasicHeader(HEADER_RANGE, range),
-                   SwiftRestClient.NEWEST);
-  }
-
-  /**
-   * Returns object length.
-   *
-   * @param uri file URI
-   * @return object length
-   * @throws SwiftException on swift-related issues
-   * @throws IOException on network/IO problems
-   */
-  public long getContentLength(URI uri) throws IOException {
-    preRemoteCommand("getContentLength");
-    return perform("getContentLength", uri, new HeadRequestProcessor<Long>() {
-      @Override
-      public Long extractResult(HttpHead req, HttpResponse resp)
-          throws IOException {
-        return HttpResponseUtils.getContentLength(resp);
-      }
-
-      @Override
-      protected void setup(HttpHead req) throws IOException {
-        super.setup(req);
-        req.addHeader(NEWEST);
-      }
-    });
-  }
-
-  /**
-   * Get the length of the remote object.
-   * @param path object to probe
-   * @return the content length
-   * @throws IOException on any failure
-   */
-  public long getContentLength(SwiftObjectPath path) throws IOException {
-    return getContentLength(pathToURI(path));
-  }
-
-  /**
-   * Get the path contents as an input stream.
-   * <b>Warning:</b> this input stream must be closed to avoid
-   * keeping Http connections open.
-   *
-   * @param path path to file
-   * @param requestHeaders http headers
-   * @return byte[] file data or null if the object was not found
-   * @throws IOException on IO Faults
-   * @throws FileNotFoundException if there is nothing at the path
-   */
-  public HttpBodyContent getData(SwiftObjectPath path,
-                                 final Header... requestHeaders)
-          throws IOException {
-    preRemoteCommand("getData");
-    return doGet(pathToURI(path),
-            requestHeaders);
-  }
-
-  /**
-   * Returns object location as byte[].
-   *
-   * @param path path to file
-   * @param requestHeaders http headers
-   * @return byte[] file data or null if the object was not found
-   * @throws IOException on IO Faults
-   */
-  public byte[] getObjectLocation(SwiftObjectPath path,
-                                  final Header... requestHeaders) throws IOException {
-    if (!isLocationAware()) {
-      //if the filesystem is not location aware, do not ask for this information
-      return null;
-    }
-    preRemoteCommand("getObjectLocation");
-    try {
-      return perform("getObjectLocation", pathToObjectLocation(path),
-          new GetRequestProcessor<byte[]>() {
-            @Override
-            protected int[] getAllowedStatusCodes() {
-              return new int[]{
-                  SC_OK,
-                  SC_FORBIDDEN,
-                  SC_NO_CONTENT
-              };
-            }
-
-            @Override
-            public byte[] extractResult(HttpGet req, HttpResponse resp) throws
-                IOException {
-
-              //TODO: remove SC_NO_CONTENT if it depends on Swift versions
-              int statusCode = resp.getStatusLine().getStatusCode();
-              if (statusCode == SC_NOT_FOUND
-                  || statusCode == SC_FORBIDDEN
-                  || statusCode == SC_NO_CONTENT
-                  || resp.getEntity().getContent() == null) {
-                return null;
-              }
-              final InputStream responseBodyAsStream =
-                  resp.getEntity().getContent();
-              final byte[] locationData = new byte[1024];
-
-              return responseBodyAsStream.read(locationData) > 0 ?
-                  locationData : null;
-            }
-
-            @Override
-            protected void setup(HttpGet req)
-                throws SwiftInternalStateException {
-              setHeaders(req, requestHeaders);
-            }
-          });
-    } catch (IOException e) {
-      LOG.warn("Failed to get the location of " + path + ": " + e, e);
-      return null;
-    }
-  }
-
-  /**
-   * Create the URI needed to query the location of an object.
-   * @param path object path to retrieve information about
-   * @return the URI for the location operation
-   * @throws SwiftException if the URI could not be constructed
-   */
-  private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException {
-    URI uri;
-    String dataLocationURI = objectLocationURI.toString();
-    try {
-      if (path.toString().startsWith("/")) {
-        dataLocationURI = dataLocationURI.concat(path.toUriPath());
-      } else {
-        dataLocationURI = dataLocationURI.concat("/").concat(path.toUriPath());
-      }
-
-      uri = new URI(dataLocationURI);
-    } catch (URISyntaxException e) {
-      throw new SwiftException(e);
-    }
-    return uri;
-  }
-
-  /**
-   * Find objects under a prefix.
-   *
-   * @param path path prefix
-   * @param requestHeaders optional request headers
-   * @return byte[] file data or null if the object was not found
-   * @throws IOException on IO Faults
-   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
-   * the directory is empty
-   */
-  public byte[] findObjectsByPrefix(SwiftObjectPath path,
-                          final Header... requestHeaders) throws IOException {
-    preRemoteCommand("findObjectsByPrefix");
-    URI uri;
-    String dataLocationURI = getEndpointURI().toString();
-    try {
-      String object = path.getObject();
-      if (object.startsWith("/")) {
-        object = object.substring(1);
-      }
-      object = encodeUrl(object);
-      dataLocationURI = dataLocationURI.concat("/")
-              .concat(path.getContainer())
-              .concat("/?prefix=")
-              .concat(object)
-      ;
-      uri = new URI(dataLocationURI);
-    } catch (URISyntaxException e) {
-      throw new SwiftException("Bad URI: " + dataLocationURI, e);
-    }
-
-    return perform("findObjectsByPrefix", uri,
-        new GetRequestProcessor<byte[]>() {
-          @Override
-          public byte[] extractResult(HttpGet req, HttpResponse resp)
-              throws IOException {
-            if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) {
-              //no result
-              throw new FileNotFoundException("Not found " + req.getURI());
-            }
-            return HttpResponseUtils.getResponseBody(resp);
-          }
-
-          @Override
-          protected int[] getAllowedStatusCodes() {
-            return new int[]{
-                SC_OK,
-                SC_NOT_FOUND
-            };
-          }
-
-          @Override
-          protected void setup(HttpGet req) throws SwiftInternalStateException {
-            setHeaders(req, requestHeaders);
-          }
-        });
-  }
-
-  /**
-   * Find objects in a directory.
-   *
-   * @param path path prefix
-   * @param requestHeaders optional request headers
-   * @return byte[] file data or null if the object was not found
-   * @throws IOException on IO Faults
-   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
-   * the directory is empty
-   */
-  public byte[] listDeepObjectsInDirectory(SwiftObjectPath path,
-                                           boolean listDeep,
-                                       final Header... requestHeaders)
-          throws IOException {
-    preRemoteCommand("listDeepObjectsInDirectory");
-
-    String endpoint = getEndpointURI().toString();
-    StringBuilder dataLocationURI = new StringBuilder();
-    dataLocationURI.append(endpoint);
-    String object = path.getObject();
-    if (object.startsWith("/")) {
-      object = object.substring(1);
-    }
-    if (!object.endsWith("/")) {
-      object = object.concat("/");
-    }
-
-    if (object.equals("/")) {
-      object = "";
-    }
-
-    dataLocationURI = dataLocationURI.append("/")
-            .append(path.getContainer())
-            .append("/?prefix=")
-            .append(object)
-            .append("&format=json");
-
-    //in listing deep set param to false
-    if (listDeep == false) {
-        dataLocationURI.append("&delimiter=/");
-    }
-
-    return findObjects(dataLocationURI.toString(), requestHeaders);
-  }
-
-  /**
-   * Find objects in a location.
-   * @param location URI
-   * @param requestHeaders optional request headers
-   * @return the body of te response
-   * @throws IOException IO problems
-   */
-  private byte[] findObjects(String location, final Header[] requestHeaders)
-      throws IOException {
-    URI uri;
-    preRemoteCommand("findObjects");
-    try {
-      uri = new URI(location);
-    } catch (URISyntaxException e) {
-      throw new SwiftException("Bad URI: " + location, e);
-    }
-
-    return perform("findObjects", uri,
-        new GetRequestProcessor<byte[]>() {
-          @Override
-          public byte[] extractResult(HttpGet req, HttpResponse resp)
-              throws IOException {
-            if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) {
-              //no result
-              throw new FileNotFoundException("Not found " + req.getURI());
-            }
-            return HttpResponseUtils.getResponseBody(resp);
-          }
-
-          @Override
-          protected int[] getAllowedStatusCodes() {
-            return new int[]{
-                SC_OK,
-                SC_NOT_FOUND
-            };
-          }
-
-          @Override
-          protected void setup(HttpGet req) throws SwiftInternalStateException {
-            setHeaders(req, requestHeaders);
-          }
-        });
-  }
-
-  /**
-   * Copy an object. This is done by sending a COPY method to the filesystem
-   * which is required to handle this WebDAV-level extension to the
-   * base HTTP operations.
-   *
-   * @param src source path
-   * @param dst destination path
-   * @param headers any headers
-   * @return true if the status code was considered successful
-   * @throws IOException on IO Faults
-   */
-  public boolean copyObject(SwiftObjectPath src, final SwiftObjectPath dst,
-                            final Header... headers) throws IOException {
-
-    preRemoteCommand("copyObject");
-
-    return perform("copy", pathToURI(src),
-        new CopyRequestProcessor<Boolean>() {
-          @Override
-          public Boolean extractResult(CopyRequest req, HttpResponse resp)
-              throws IOException {
-            return resp.getStatusLine().getStatusCode() != SC_NOT_FOUND;
-          }
-
-          @Override
-          protected void setup(CopyRequest req) throws
-              SwiftInternalStateException {
-            setHeaders(req, headers);
-            req.addHeader(HEADER_DESTINATION, dst.toUriPath());
-          }
-        });
-  }
-
-  /**
-   * Uploads file as Input Stream to Swift.
-   * The data stream will be closed after the request.
-   *
-   * @param path path to Swift
-   * @param data object data
-   * @param length length of data
-   * @param requestHeaders http headers
-   * @throws IOException on IO Faults
-   */
-  public void upload(SwiftObjectPath path,
-                     final InputStream data,
-                     final long length,
-                     final Header... requestHeaders)
-          throws IOException {
-    preRemoteCommand("upload");
-
-    try {
-      perform("upload", pathToURI(path), new PutRequestProcessor<byte[]>() {
-        @Override
-        public byte[] extractResult(HttpPut req, HttpResponse resp)
-            throws IOException {
-          return HttpResponseUtils.getResponseBody(resp);
-        }
-
-        @Override
-        protected void setup(HttpPut req) throws
-                        SwiftInternalStateException {
-          req.setEntity(new InputStreamEntity(data, length));
-          setHeaders(req, requestHeaders);
-        }
-      });
-    } finally {
-      data.close();
-    }
-
-  }
-
-
-  /**
-   * Deletes object from swift.
-   * The result is true if this operation did the deletion.
-   *
-   * @param path           path to file
-   * @param requestHeaders http headers
-   * @throws IOException on IO Faults
-   */
-  public boolean delete(SwiftObjectPath path, final Header... requestHeaders) throws IOException {
-    preRemoteCommand("delete");
-
-    return perform("", pathToURI(path), new DeleteRequestProcessor<Boolean>() {
-      @Override
-      public Boolean extractResult(HttpDelete req, HttpResponse resp)
-          throws IOException {
-        return resp.getStatusLine().getStatusCode() == SC_NO_CONTENT;
-      }
-
-      @Override
-      protected void setup(HttpDelete req) throws
-                    SwiftInternalStateException {
-        setHeaders(req, requestHeaders);
-      }
-    });
-  }
-
-  /**
-   * Issue a head request.
-   * @param reason reason -used in logs
-   * @param path path to query
-   * @param requestHeaders request header
-   * @return the response headers. This may be an empty list
-   * @throws IOException IO problems
-   * @throws FileNotFoundException if there is nothing at the end
-   */
-  public Header[] headRequest(String reason,
-                              SwiftObjectPath path,
-                              final Header... requestHeaders)
-          throws IOException {
-
-    preRemoteCommand("headRequest: "+ reason);
-    return perform(reason, pathToURI(path),
-        new HeadRequestProcessor<Header[]>() {
-          @Override
-          public Header[] extractResult(HttpHead req, HttpResponse resp)
-              throws IOException {
-            if (resp.getStatusLine().getStatusCode() == SC_NOT_FOUND) {
-              throw new FileNotFoundException("Not Found " + req.getURI());
-            }
-            return resp.getAllHeaders();
-          }
-
-          @Override
-          protected void setup(HttpHead req) throws
-              SwiftInternalStateException {
-            setHeaders(req, requestHeaders);
-          }
-        });
-  }
-
-  /**
-   * Issue a put request.
-   * @param path path
-   * @param requestHeaders optional headers
-   * @return the HTTP response
-   * @throws IOException any problem
-   */
-  public int putRequest(SwiftObjectPath path, final Header... requestHeaders)
-          throws IOException {
-
-    preRemoteCommand("putRequest");
-    return perform(pathToURI(path), new PutRequestProcessor<Integer>() {
-
-      @Override
-      public Integer extractResult(HttpPut req, HttpResponse resp)
-          throws IOException {
-        return resp.getStatusLine().getStatusCode();
-      }
-
-      @Override
-      protected void setup(HttpPut req) throws
-                    SwiftInternalStateException {
-        setHeaders(req, requestHeaders);
-      }
-    });
-  }
-
-  /**
-   * Authenticate to Openstack Keystone.
-   * As well as returning the access token, the member fields {@link #token},
-   * {@link #endpointURI} and {@link #objectLocationURI} are set up for re-use.
-   * <p>
-   * This method is re-entrant -if more than one thread attempts to authenticate
-   * neither will block -but the field values with have those of the last caller.
-   *
-   * @return authenticated access token
-   */
-  public AccessToken authenticate() throws IOException {
-    final AuthenticationRequest authenticationRequest;
-    if (useKeystoneAuthentication) {
-      authenticationRequest = keystoneAuthRequest;
-    } else {
-      authenticationRequest = authRequest;
-    }
-
-    LOG.debug("started authentication");
-    return perform("authentication",
-                   authUri,
-                   new AuthenticationPost(authenticationRequest));
-  }
-
-  private final class AuthenticationPost extends
-      AuthRequestProcessor<AccessToken> {
-    final AuthenticationRequest authenticationRequest;
-
-    private AuthenticationPost(AuthenticationRequest authenticationRequest) {
-      this.authenticationRequest = authenticationRequest;
-    }
-
-    @Override
-    protected void setup(AuthPostRequest req) throws IOException {
-      req.setEntity(getAuthenticationRequst(authenticationRequest));
-    }
-
-    /**
-     * specification says any of the 2xxs are OK, so list all
-     * the standard ones
-     * @return a set of 2XX status codes.
-     */
-    @Override
-    protected int[] getAllowedStatusCodes() {
-      return new int[]{
-        SC_OK,
-        SC_BAD_REQUEST,
-        SC_CREATED,
-        SC_ACCEPTED,
-        SC_NON_AUTHORITATIVE_INFORMATION,
-        SC_NO_CONTENT,
-        SC_RESET_CONTENT,
-        SC_PARTIAL_CONTENT,
-        SC_MULTI_STATUS,
-        SC_UNAUTHORIZED //if request unauthorized, try another method
-      };
-    }
-
-    @Override
-    public AccessToken extractResult(AuthPostRequest req, HttpResponse resp)
-        throws IOException {
-      //initial check for failure codes leading to authentication failures
-      if (resp.getStatusLine().getStatusCode() == SC_BAD_REQUEST) {
-        throw new SwiftAuthenticationFailedException(
-       authenticationRequest.toString(), "POST", authUri, resp);
-      }
-
-      final AuthenticationResponse access =
-          JSONUtil.toObject(HttpResponseUtils.getResponseBodyAsString(resp),
-                            AuthenticationWrapper.class).getAccess();
-      final List<Catalog> serviceCatalog = access.getServiceCatalog();
-      //locate the specific service catalog that defines Swift; variations
-      //in the name of this add complexity to the search
-      StringBuilder catList = new StringBuilder();
-      StringBuilder regionList = new StringBuilder();
-
-      //these fields are all set together at the end of the operation
-      URI endpointURI = null;
-      URI objectLocation;
-      Endpoint swiftEndpoint = null;
-      AccessToken accessToken;
-
-      for (Catalog catalog : serviceCatalog) {
-        String name = catalog.getName();
-        String type = catalog.getType();
-        String descr = String.format("[%s: %s]; ", name, type);
-        catList.append(descr);
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Catalog entry " + descr);
-        }
-        if (name.equals(SERVICE_CATALOG_SWIFT)
-            || name.equals(SERVICE_CATALOG_CLOUD_FILES)
-            || type.equals(SERVICE_CATALOG_OBJECT_STORE)) {
-          //swift is found
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Found swift catalog as " + name + " => " + type);
-          }
-          //now go through the endpoints
-          for (Endpoint endpoint : catalog.getEndpoints()) {
-            String endpointRegion = endpoint.getRegion();
-            URI publicURL = endpoint.getPublicURL();
-            URI internalURL = endpoint.getInternalURL();
-            descr = String.format("[%s => %s / %s]; ",
-                                  endpointRegion,
-                                  publicURL,
-                                  internalURL);
-            regionList.append(descr);
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Endpoint " + descr);
-            }
-            if (region == null || endpointRegion.equals(region)) {
-              endpointURI = usePublicURL ? publicURL : internalURL;
-              swiftEndpoint = endpoint;
-              break;
-            }
-          }
-        }
-      }
-      if (endpointURI == null) {
-        String message = "Could not find swift service from auth URL "
-                         + authUri
-                         + " and region '" + region + "'. "
-                         + "Categories: " + catList
-                         + ((regionList.length() > 0) ?
-                            ("regions: " + regionList)
-                                                      : "No regions");
-        throw new SwiftInvalidResponseException(message,
-                                                SC_OK,
-                                                "authenticating",
-                                                authUri);
-
-      }
-
-
-      accessToken = access.getToken();
-      String path = SWIFT_OBJECT_AUTH_ENDPOINT
-                    + swiftEndpoint.getTenantId();
-      String host = endpointURI.getHost();
-      try {
-        objectLocation = new URI(endpointURI.getScheme(),
-                                 null,
-                                 host,
-                                 endpointURI.getPort(),
-                                 path,
-                                 null,
-                                 null);
-      } catch (URISyntaxException e) {
-        throw new SwiftException("object endpoint URI is incorrect: "
-                                 + endpointURI
-                                 + " + " + path,
-                                 e);
-      }
-      setAuthDetails(endpointURI, objectLocation, accessToken);
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("authenticated against " + endpointURI);
-      }
-      createDefaultContainer();
-      return accessToken;
-    }
-  }
-
-  private StringEntity getAuthenticationRequst(
-      AuthenticationRequest authenticationRequest) throws IOException {
-    final String data = JSONUtil.toJSON(new AuthenticationRequestWrapper(
-            authenticationRequest));
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Authenticating with " + authenticationRequest);
-    }
-    return new StringEntity(data, ContentType.create("application/json",
-        "UTF-8"));
-  }
-
-  /**
-   * create default container if it doesn't exist for Hadoop Swift integration.
-   * non-reentrant, as this should only be needed once.
-   *
-   * @throws IOException IO problems.
-   */
-  private synchronized void createDefaultContainer() throws IOException {
-    createContainer(container);
-  }
-
-  /**
-   * Create a container -if it already exists, do nothing.
-   *
-   * @param containerName the container name
-   * @throws IOException IO problems
-   * @throws SwiftBadRequestException invalid container name
-   * @throws SwiftInvalidResponseException error from the server
-   */
-  public void createContainer(String containerName) throws IOException {
-    SwiftObjectPath objectPath = new SwiftObjectPath(containerName, "");
-    try {
-      //see if the data is there
-      headRequest("createContainer", objectPath, NEWEST);
-    } catch (FileNotFoundException ex) {
-      int status = 0;
-      try {
-        status = putRequest(objectPath);
-      } catch (FileNotFoundException e) {
-        //triggered by a very bad container name.
-        //re-insert the 404 result into the status
-        status = SC_NOT_FOUND;
-      }
-      if (status == SC_BAD_REQUEST) {
-        throw new SwiftBadRequestException(
-          "Bad request -authentication failure or bad container name?",
-          status,
-          "PUT",
-          null);
-      }
-      if (!isStatusCodeExpected(status,
-              SC_OK,
-              SC_CREATED,
-              SC_ACCEPTED,
-              SC_NO_CONTENT)) {
-        throw new SwiftInvalidResponseException("Couldn't create container "
-                + containerName +
-                " for storing data in Swift." +
-                " Try to create container " +
-                containerName + " manually ",
-                status,
-                "PUT",
-                null);
-      } else {
-        throw ex;
-      }
-    }
-  }
-
-  /**
-   * Trigger an initial auth operation if some of the needed
-   * fields are missing.
-   *
-   * @throws IOException on problems
-   */
-  private void authIfNeeded() throws IOException {
-    if (getEndpointURI() == null) {
-      authenticate();
-    }
-  }
-
-  /**
-   * Pre-execution actions to be performed by methods. Currently this
-   * <ul>
-   *   <li>Logs the operation at TRACE</li>
-   *   <li>Authenticates the client -if needed</li>
-   * </ul>
-   * @throws IOException
-   */
-  private void preRemoteCommand(String operation) throws IOException {
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("Executing " + operation);
-    }
-    authIfNeeded();
-  }
-
-
-  /**
-   * Performs the HTTP request, validates the response code and returns
-   * the received data. HTTP Status codes are converted into exceptions.
-   *
-   * @param uri URI to source
-   * @param processor HttpMethodProcessor
-   * @param <M> method
-   * @param <R> result type
-   * @return result of HTTP request
-   * @throws IOException IO problems
-   * @throws SwiftBadRequestException the status code indicated "Bad request"
-   * @throws SwiftInvalidResponseException the status code is out of range
-   * for the action (excluding 404 responses)
-   * @throws SwiftInternalStateException the internal state of this client
-   * is invalid
-   * @throws FileNotFoundException a 404 response was returned
-   */
-  private <M extends HttpRequestBase, R> R perform(URI uri,
-                      HttpRequestProcessor<M, R> processor)
-    throws IOException,
-           SwiftBadRequestException,
-           SwiftInternalStateException,
-           SwiftInvalidResponseException,
-           FileNotFoundException {
-    return perform("",uri, processor);
-  }
-
-  /**
-   * Performs the HTTP request, validates the response code and returns
-   * the received data. HTTP Status codes are converted into exceptions.
-   * @param reason why is this operation taking place. Used for statistics
-   * @param uri URI to source
-   * @param processor HttpMethodProcessor
-   * @param <M> method
-   * @param <R> result type
-   * @return result of HTTP request
-   * @throws IOException IO problems
-   * @throws SwiftBadRequestException the status code indicated "Bad request"
-   * @throws SwiftInvalidResponseException the status code is out of range
-   * for the action (excluding 404 responses)
-   * @throws SwiftInternalStateException the internal state of this client
-   * is invalid
-   * @throws FileNotFoundException a 404 response was returned
-   */
-  private <M extends HttpRequestBase, R> R perform(String reason, URI uri,
-      HttpRequestProcessor<M, R> processor)
-      throws IOException, SwiftBadRequestException, SwiftInternalStateException,
-            SwiftInvalidResponseException, FileNotFoundException {
-    checkNotNull(uri);
-    checkNotNull(processor);
-
-    final M req = processor.createRequest(uri.toString());
-    req.addHeader(HEADER_USER_AGENT, SWIFT_USER_AGENT);
-    //retry policy
-    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
-    clientBuilder.setRetryHandler(
-        new DefaultHttpRequestRetryHandler(retryCount, false));
-    RequestConfig.Builder requestConfigBuilder =
-        RequestConfig.custom().setConnectTimeout(connectTimeout);
-    if (proxyHost != null) {
-      requestConfigBuilder.setProxy(new HttpHost(proxyHost, proxyPort));
-    }
-    clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
-    clientBuilder.setDefaultSocketConfig(
-        SocketConfig.custom().setSoTimeout(socketTimeout).build());
-    Duration duration = new Duration();
-    boolean success = false;
-    HttpResponse resp;
-    try {
-      // client should not be closed in this method because
-      // the connection can be used later
-      CloseableHttpClient client = clientBuilder.build();
-      int statusCode = 0;
-      try {
-        resp = exec(client, req);
-        statusCode = checkNotNull(resp.getStatusLine().getStatusCode());
-      } catch (IOException e) {
-        //rethrow with extra diagnostics and wiki links
-        throw ExceptionDiags.wrapException(uri.toString(), req.getMethod(), e);
-      }
-
-      //look at the response and see if it was valid or not.
-      //Valid is more than a simple 200; even 404 "not found" is considered
-      //valid -which it is for many methods.
-
-      //validate the allowed status code for this operation
-      int[] allowedStatusCodes = processor.getAllowedStatusCodes();
-      boolean validResponse = isStatusCodeExpected(statusCode,
-          allowedStatusCodes);
-
-      if (!validResponse) {
-        IOException ioe = buildException(uri, req, resp, statusCode);
-        throw ioe;
-      }
-
-      R r = processor.extractResult(req, resp);
-      success = true;
-      return r;
-    } catch (IOException e) {
-      //release the connection -always
-      req.releaseConnection();
-      throw e;
-    } finally {
-      duration.finished();
-      durationStats.add(req.getMethod() + " " + reason, duration, success);
-    }
-  }
-
-  /**
-   * Build an exception from a failed operation. This can include generating
-   * specific exceptions (e.g. FileNotFound), as well as the default
-   * {@link SwiftInvalidResponseException}.
-   *
-   * @param uri URI for operation
-   * @param resp operation that failed
-   * @param statusCode status code
-   * @param <M> method type
-   * @return an exception to throw
-   */
-  private <M extends HttpUriRequest> IOException buildException(
-      URI uri, M req, HttpResponse resp, int statusCode) {
-    IOException fault;
-
-    //log the failure @debug level
-    String errorMessage = String.format("Method %s on %s failed, status code: %d," +
-            " status line: %s",
-            req.getMethod(),
-            uri,
-            statusCode,
-            resp.getStatusLine()
-    );
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(errorMessage);
-    }
-    //send the command
-    switch (statusCode) {
-    case SC_NOT_FOUND:
-      fault = new FileNotFoundException("Operation " + req.getMethod()
-          + " on " + uri);
-      break;
-
-    case SC_BAD_REQUEST:
-      //bad HTTP request
-      fault =  new SwiftBadRequestException("Bad request against " + uri,
-          req.getMethod(), uri, resp);
-      break;
-
-    case SC_REQUESTED_RANGE_NOT_SATISFIABLE:
-      //out of range
-      StringBuilder errorText = new StringBuilder(
-          resp.getStatusLine().getReasonPhrase());
-      //get the requested length
-      Header requestContentLen = req.getFirstHeader(HEADER_CONTENT_LENGTH);
-      if (requestContentLen != null) {
-        errorText.append(" requested ").append(requestContentLen.getValue());
-      }
-      //and the result
-      Header availableContentRange = resp.getFirstHeader(HEADER_CONTENT_RANGE);
-
-      if (availableContentRange != null) {
-        errorText.append(" available ")
-            .append(availableContentRange.getValue());
-      }
-      fault = new EOFException(errorText.toString());
-      break;
-
-    case SC_UNAUTHORIZED:
-      //auth failure; should only happen on the second attempt
-      fault  = new SwiftAuthenticationFailedException(
-          "Operation not authorized- current access token =" + getToken(),
-          req.getMethod(),
-          uri,
-          resp);
-      break;
-
-    case SwiftProtocolConstants.SC_TOO_MANY_REQUESTS_429:
-    case SwiftProtocolConstants.SC_THROTTLED_498:
-      //response code that may mean the client is being throttled
-      fault  = new SwiftThrottledRequestException(
-          "Client is being throttled: too many requests",
-          req.getMethod(),
-          uri,
-          resp);
-      break;
-
-    default:
-      //return a generic invalid HTTP response
-      fault = new SwiftInvalidResponseException(
-          errorMessage,
-          req.getMethod(),
-          uri,
-          resp);
-    }
-
-    return fault;
-  }
-
-  /**
-   * Exec a GET request and return the input stream of the response.
-   *
-   * @param uri URI to GET
-   * @param requestHeaders request headers
-   * @return the input stream. This must be closed to avoid log errors
-   * @throws IOException
-   */
-  private HttpBodyContent doGet(final URI uri, final Header... requestHeaders) throws IOException {
-    return perform("", uri, new GetRequestProcessor<HttpBodyContent>() {
-      @Override
-      public HttpBodyContent extractResult(HttpGet req, HttpResponse resp)
-          throws IOException {
-        return new HttpBodyContent(
-            new HttpInputStreamWithRelease(uri, req, resp),
-            HttpResponseUtils.getContentLength(resp));
-      }
-
-      @Override
-      protected void setup(HttpGet req) throws
-                    SwiftInternalStateException {
-        setHeaders(req, requestHeaders);
-      }
-    });
-  }
-
-  /**
-   * Create an instance against a specific FS URI.
-   *
-   * @param filesystemURI filesystem to bond to
-   * @param config source of configuration data
-   * @return REST client instance
-   * @throws IOException on instantiation problems
-   */
-  public static SwiftRestClient getInstance(URI filesystemURI,
-                                            Configuration config) throws IOException {
-    return new SwiftRestClient(filesystemURI, config);
-  }
-
-
-  /**
-   * Converts Swift path to URI to make request.
-   * This is public for unit testing
-   *
-   * @param path path to object
-   * @param endpointURI domain url e.g. http://domain.com
-   * @return valid URI for object
-   * @throws SwiftException
-   */
-  public static URI pathToURI(SwiftObjectPath path,
-                              URI endpointURI) throws SwiftException {
-    checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");
-
-    String dataLocationURI = endpointURI.toString();
-    try {
-
-      dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
-      return new URI(dataLocationURI);
-    } catch (URISyntaxException e) {
-      throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
-    }
-  }
-
-  /**
-   * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
-   * with a replacement of + with %20.
-   * @param url URL string
-   * @return an encoded string
-   * @throws SwiftException if the URL cannot be encoded
-   */
-  private static String encodeUrl(String url) throws SwiftException {
-    if (url.matches(".*\\s+.*")) {
-      try {
-        url = URLEncoder.encode(url, "UTF-8");
-        url = url.replace("+", "%20");
-      } catch (UnsupportedEncodingException e) {
-        throw new SwiftException("failed to encode URI", e);
-      }
-    }
-
-    return url;
-  }
-
-  /**
-   * Convert a swift path to a URI relative to the current endpoint.
-   *
-   * @param path path
-   * @return an path off the current endpoint URI.
-   * @throws SwiftException
-   */
-  private URI pathToURI(SwiftObjectPath path) throws SwiftException {
-    return pathToURI(path, getEndpointURI());
-  }
-
-  /**
-   * Add the headers to the method, and the auth token (which must be set).
-   * @param method method to update
-   * @param requestHeaders the list of headers
-   * @throws SwiftInternalStateException not yet authenticated
-   */
-  private void setHeaders(HttpUriRequest method, Header[] requestHeaders)
-      throws SwiftInternalStateException {
-    for (Header header : requestHeaders) {
-      method.addHeader(header);
-    }
-    setAuthToken(method, getToken());
-  }
-
-
-  /**
-   * Set the auth key header of the method to the token ID supplied.
-   *
-   * @param method method
-   * @param accessToken access token
-   * @throws SwiftInternalStateException if the client is not yet authenticated
-   */
-  private void setAuthToken(HttpUriRequest method, AccessToken accessToken)
-      throws SwiftInternalStateException {
-    checkNotNull(accessToken,"Not authenticated");
-    method.addHeader(HEADER_AUTH_KEY, accessToken.getId());
-  }
-
-  /**
-   * Execute a method in a new HttpClient instance. If the auth failed,
-   * authenticate then retry the method.
-   *
-   * @param req request to exec
-   * @param client client to use
-   * @param <M> Request type
-   * @return the status code
-   * @throws IOException on any failure
-   */
-  private <M extends HttpUriRequest> HttpResponse exec(HttpClient client, M req)
-      throws IOException {
-    HttpResponse resp = execWithDebugOutput(req, client);
-    int statusCode = resp.getStatusLine().getStatusCode();
-    if ((statusCode == HttpStatus.SC_UNAUTHORIZED
-            || statusCode == HttpStatus.SC_BAD_REQUEST)
-        && req instanceof AuthPostRequest
-            && !useKeystoneAuthentication) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Operation failed with status " + statusCode
-            + " attempting keystone auth");
-      }
-      //if rackspace key authentication failed - try custom Keystone authentication
-      useKeystoneAuthentication = true;
-      final AuthPostRequest authentication = (AuthPostRequest) req;
-      //replace rackspace auth with keystone one
-      authentication.setEntity(getAuthenticationRequst(keystoneAuthRequest));
-      resp = execWithDebugOutput(req, client);
-    }
-
-    if (statusCode == HttpStatus.SC_UNAUTHORIZED ) {
-      //unauthed -or the auth uri rejected it.
-
-      if (req instanceof AuthPostRequest) {
-          //unauth response from the AUTH URI itself.
-          throw new SwiftAuthenticationFailedException(authRequest.toString(),
-                                                       "auth",
-                                                       authUri,
-                                                       resp);
-      }
-      //any other URL: try again
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Reauthenticating");
-      }
-      //re-auth, this may recurse into the same dir
-      authenticate();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Retrying original request");
-      }
-      resp = execWithDebugOutput(req, client);
-    }
-    return resp;
-  }
-
-  /**
-   * Execute the request with the request and response logged at debug level.
-   * @param req request to execute
-   * @param client client to use
-   * @param <M> method type
-   * @return the status code
-   * @throws IOException any failure reported by the HTTP client.
-   */
-  private <M extends HttpUriRequest> HttpResponse execWithDebugOutput(M req,
-      HttpClient client) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      StringBuilder builder = new StringBuilder(
-              req.getMethod() + " " + req.getURI() + "\n");
-      for (Header header : req.getAllHeaders()) {
-        builder.append(header.toString());
-      }
-      LOG.debug(builder.toString());
-    }
-    HttpResponse resp = client.execute(req);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Status code = " + resp.getStatusLine().getStatusCode());
-    }
-    return resp;
-  }
-
-  /**
-   * Ensures that an object reference passed as a parameter to the calling
-   * method is not null.
-   *
-   * @param reference an object reference
-   * @return the non-null reference that was validated
-   * @throws NullPointerException if {@code reference} is null
-   */
-  private static <T> T checkNotNull(T reference) throws
-            SwiftInternalStateException {
-    return checkNotNull(reference, "Null Reference");
-  }
-
-  private static <T> T checkNotNull(T reference, String message) throws
-            SwiftInternalStateException {
-    if (reference == null) {
-      throw new SwiftInternalStateException(message);
-    }
-    return reference;
-  }
-
-  /**
-   * Check for a status code being expected -takes a list of expected values
-   *
-   * @param status received status
-   * @param expected expected value
-   * @return true if status is an element of [expected]
-   */
-  private boolean isStatusCodeExpected(int status, int... expected) {
-    for (int code : expected) {
-      if (status == code) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-
-  @Override
-  public String toString() {
-    return "Swift client: " + serviceDescription;
-  }
-
-  /**
-   * Get the region which this client is bound to
-   * @return the region
-   */
-  public String getRegion() {
-    return region;
-  }
-
-  /**
-   * Get the tenant to which this client is bound
-   * @return the tenant
-   */
-  public String getTenant() {
-    return tenant;
-  }
-
-  /**
-   * Get the username this client identifies itself as
-   * @return the username
-   */
-  public String getUsername() {
-    return username;
-  }
-
-  /**
-   * Get the container to which this client is bound
-   * @return the container
-   */
-  public String getContainer() {
-    return container;
-  }
-
-  /**
-   * Is this client bound to a location aware Swift blobstore
-   * -that is, can you query for the location of partitions?
-   * @return true iff the location of multipart file uploads
-   * can be determined.
-   */
-  public boolean isLocationAware() {
-    return locationAware;
-  }
-
-  /**
-   * Get the blocksize of this filesystem
-   * @return a blocksize &gt; 0
-   */
-  public long getBlocksizeKB() {
-    return blocksizeKB;
-  }
-
-  /**
-   * Get the partition size in KB.
-   * @return the partition size
-   */
-  public int getPartSizeKB() {
-    return partSizeKB;
-  }
-
-  /**
-   * Get the buffer size in KB.
-   * @return the buffer size wanted for reads
-   */
-  public int getBufferSizeKB() {
-    return bufferSizeKB;
-  }
-
-  public int getProxyPort() {
-    return proxyPort;
-  }
-
-  public String getProxyHost() {
-    return proxyHost;
-  }
-
-  public int getRetryCount() {
-    return retryCount;
-  }
-
-  public int getConnectTimeout() {
-    return connectTimeout;
-  }
-
-  public boolean isUsePublicURL() {
-    return usePublicURL;
-  }
-
-  public int getThrottleDelay() {
-    return throttleDelay;
-  }
-
-  /**
-   * Get the current operation statistics.
-   * @return a snapshot of the statistics
-   */
-
-  public List<DurationStats> getOperationStatistics() {
-    return durationStats.getDurationStatistics();
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html
deleted file mode 100644
index ad900f90d06f0..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html
+++ /dev/null
@@ -1,81 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
-        "http://www.w3.org/TR/html4/loose.dtd">
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~  or more contributor license agreements.  See the NOTICE file
-  ~  distributed with this work for additional information
-  ~  regarding copyright ownership.  The ASF licenses this file
-  ~  to you under the Apache License, Version 2.0 (the
-  ~  "License"); you may not use this file except in compliance
-  ~  with the License.  You may obtain a copy of the License at
-  ~
-  ~       http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~  Unless required by applicable law or agreed to in writing, software
-  ~  distributed under the License is distributed on an "AS IS" BASIS,
-  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~  See the License for the specific language governing permissions and
-  ~  limitations under the License.
-  -->
-
-<html>
-<head>
-    <title>Swift Filesystem Client for Apache Hadoop</title>
-        <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
-<body>
-
-<h1>
-    Swift Filesystem Client for Apache Hadoop
-</h1>
-
-<h2>Introduction</h2>
-
-<div>This package provides support in Apache Hadoop for the OpenStack Swift
-    Key-Value store, allowing client applications -including MR Jobs- to
-    read and write data in Swift.
-</div>
-
-<div>Design Goals</div>
-<ol>
-    <li>Give clients access to SwiftFS files, similar to S3n:</li>
-    <li>maybe: support a Swift Block store -- at least until Swift's
-        support for &gt;5GB files has stabilized.
-    </li>
-    <li>Support for data-locality if the Swift FS provides file location information</li>
-    <li>Support access to multiple Swift filesystems in the same client/task.</li>
-    <li>Authenticate using the Keystone APIs.</li>
-    <li>Avoid dependency on unmaintained libraries.</li>
-</ol>
-
-
-<h2>Supporting multiple Swift Filesystems</h2>
-
-The goal of supporting multiple swift filesystems simultaneously changes how
-clusters are named and authenticated. In Hadoop's S3 and S3N filesystems, the "bucket" into
-which objects are stored is directly named in the URL, such as
-<code>s3n://bucket/object1</code>. The Hadoop configuration contains a
-single set of login credentials for S3 (username and key), which are used to
-authenticate the HTTP operations.
-
-For swift, we need to know not only which "container" name, but which credentials
-to use to authenticate with it -and which URL to use for authentication.
-
-This has led to a different design pattern from S3, as instead of simple bucket names,
-the hostname of an S3 container is two-level, the name of the service provider
-being the second path: <code>swift://bucket.service/</code>
-
-The <code>service</code> portion of this domain name is used as a reference into
-the client settings -and so identify the service provider of that container.
-
-
-<h2>Testing</h2>
-
-<div>
-    The client code can be tested against public or private Swift instances; the
-    public services are (at the time of writing -January 2013-), Rackspace and
-    HP Cloud. Testing against both instances is how interoperability
-    can be verified.
-</div>
-
-</body>
-</html>
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java
deleted file mode 100644
index 794219f31a4a6..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * Add stricter compliance with the evolving FS specifications
- */
-public class StrictBufferedFSInputStream extends BufferedFSInputStream {
-
-  public StrictBufferedFSInputStream(FSInputStream in,
-                                     int size) {
-    super(in, size);
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    if (pos < 0) {
-      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
-    }
-    if (in == null) {
-      throw new SwiftConnectionClosedException(FSExceptionMessages.STREAM_IS_CLOSED);
-    }
-    super.seek(pos);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
deleted file mode 100644
index 725cae1e3b8e3..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-
-/**
- * A subclass of {@link FileStatus} that contains the
- * Swift-specific rules of when a file is considered to be a directory.
- */
-public class SwiftFileStatus extends FileStatus {
-
-  public SwiftFileStatus() {
-  }
-
-  public SwiftFileStatus(long length,
-                         boolean isdir,
-                         int block_replication,
-                         long blocksize, long modification_time, Path path) {
-    super(length, isdir, block_replication, blocksize, modification_time, path);
-  }
-
-  public SwiftFileStatus(long length,
-                         boolean isdir,
-                         int block_replication,
-                         long blocksize,
-                         long modification_time,
-                         long access_time,
-                         FsPermission permission,
-                         String owner, String group, Path path) {
-    super(length, isdir, block_replication, blocksize, modification_time,
-            access_time, permission, owner, group, path);
-  }
-
-  //HDFS2+ only
-
-  public SwiftFileStatus(long length,
-                         boolean isdir,
-                         int block_replication,
-                         long blocksize,
-                         long modification_time,
-                         long access_time,
-                         FsPermission permission,
-                         String owner, String group, Path symlink, Path path) {
-    super(length, isdir, block_replication, blocksize, modification_time,
-          access_time, permission, owner, group, symlink, path);
-  }
-
-  /**
-   * Declare that the path represents a directory, which in the
-   * SwiftNativeFileSystem means "is a directory or a 0 byte file"
-   *
-   * @return true if the status is considered to be a file
-   */
-  @Override
-  public boolean isDirectory() {
-    return super.isDirectory() || getLen() == 0;
-  }
-
-  /**
-   * A entry is a file if it is not a directory.
-   * By implementing it <i>and not marking as an override</i> this
-   * subclass builds and runs in both Hadoop versions.
-   * @return the opposite value to {@link #isDirectory()}
-   */
-  @Override
-  public boolean isFile() {
-    return !this.isDirectory();
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append(getClass().getSimpleName());
-    sb.append("{ ");
-    sb.append("path=").append(getPath());
-    sb.append("; isDirectory=").append(isDirectory());
-    sb.append("; length=").append(getLen());
-    sb.append("; blocksize=").append(getBlockSize());
-    sb.append("; modification_time=").append(getModificationTime());
-    sb.append("}");
-    return sb.toString();
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
deleted file mode 100644
index 560eadd930917..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java
+++ /dev/null
@@ -1,761 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftUnsupportedFeatureException;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-import org.apache.hadoop.util.Progressable;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * Swift file system implementation. Extends Hadoop FileSystem
- */
-public class SwiftNativeFileSystem extends FileSystem {
-
-  /** filesystem prefix: {@value} */
-  public static final String SWIFT = "swift";
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftNativeFileSystem.class);
-
-  /**
-   * path to user work directory for storing temporary files
-   */
-  private Path workingDir;
-
-  /**
-   * Swift URI
-   */
-  private URI uri;
-
-  /**
-   * reference to swiftFileSystemStore
-   */
-  private SwiftNativeFileSystemStore store;
-
-  /**
-   * Default constructor for Hadoop
-   */
-  public SwiftNativeFileSystem() {
-    // set client in initialize()
-  }
-
-  /**
-   * This constructor used for testing purposes
-   */
-  public SwiftNativeFileSystem(SwiftNativeFileSystemStore store) {
-    this.store = store;
-  }
-
-  /**
-   * This is for testing
-   * @return the inner store class
-   */
-  public SwiftNativeFileSystemStore getStore() {
-    return store;
-  }
-
-  @Override
-  public String getScheme() {
-    return SWIFT;
-  }
-
-  /**
-   * default class initialization.
-   *
-   * @param fsuri path to Swift
-   * @param conf  Hadoop configuration
-   * @throws IOException
-   */
-  @Override
-  public void initialize(URI fsuri, Configuration conf) throws IOException {
-    super.initialize(fsuri, conf);
-
-    setConf(conf);
-    if (store == null) {
-      store = new SwiftNativeFileSystemStore();
-    }
-    this.uri = fsuri;
-    String username;
-    try {
-      username = UserGroupInformation.getCurrentUser().getShortUserName();
-    } catch (IOException ex) {
-      LOG.warn("Unable to get user name. Fall back to system property " +
-          "user.name", ex);
-      username = System.getProperty("user.name");
-    }
-    this.workingDir = new Path("/user", username)
-      .makeQualified(uri, new Path(username));
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Initializing SwiftNativeFileSystem against URI " + uri
-              + " and working dir " + workingDir);
-    }
-    store.initialize(uri, conf);
-    LOG.debug("SwiftFileSystem initialized");
-  }
-
-  /**
-   * @return path to Swift
-   */
-  @Override
-  public URI getUri() {
-
-    return uri;
-  }
-
-  @Override
-  public String toString() {
-    return "Swift FileSystem " + store;
-  }
-
-  /**
-   * Path to user working directory
-   *
-   * @return Hadoop path
-   */
-  @Override
-  public Path getWorkingDirectory() {
-    return workingDir;
-  }
-
-  /**
-   * @param dir user working directory
-   */
-  @Override
-  public void setWorkingDirectory(Path dir) {
-    workingDir = makeAbsolute(dir);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("SwiftFileSystem.setWorkingDirectory to " + dir);
-    }
-  }
-
-  /**
-   * Return a file status object that represents the path.
-   *
-   * @param path The path we want information from
-   * @return a FileStatus object
-   */
-  @Override
-  public FileStatus getFileStatus(Path path) throws IOException {
-    Path absolutePath = makeAbsolute(path);
-    return store.getObjectMetadata(absolutePath);
-  }
-
-  /**
-   * The blocksize of this filesystem is set by the property
-   * SwiftProtocolConstants.SWIFT_BLOCKSIZE;the default is the value of
-   * SwiftProtocolConstants.DEFAULT_SWIFT_BLOCKSIZE;
-   * @return the blocksize for this FS.
-   */
-  @Override
-  public long getDefaultBlockSize() {
-    return store.getBlocksize();
-  }
-
-  /**
-   * The blocksize for this filesystem.
-   * @see #getDefaultBlockSize()
-   * @param f path of file
-   * @return the blocksize for the path
-   */
-  @Override
-  public long getDefaultBlockSize(Path f) {
-    return store.getBlocksize();
-  }
-
-  @Override
-  public long getBlockSize(Path path) throws IOException {
-    return store.getBlocksize();
-  }
-
-  @Override
-  @SuppressWarnings("deprecation")
-  public boolean isFile(Path f) throws IOException {
-    try {
-      FileStatus fileStatus = getFileStatus(f);
-      return !SwiftUtils.isDirectory(fileStatus);
-    } catch (FileNotFoundException e) {
-      return false;               // f does not exist
-    }
-  }
-
-  @SuppressWarnings("deprecation")
-  @Override
-  public boolean isDirectory(Path f) throws IOException {
-
-    try {
-      FileStatus fileStatus = getFileStatus(f);
-      return SwiftUtils.isDirectory(fileStatus);
-    } catch (FileNotFoundException e) {
-      return false;               // f does not exist
-    }
-  }
-
-  /**
-   * Override getCononicalServiceName because we don't support token in Swift
-   */
-  @Override
-  public String getCanonicalServiceName() {
-    // Does not support Token
-    return null;
-  }
-
-  /**
-   * Return an array containing hostnames, offset and size of
-   * portions of the given file.  For a nonexistent
-   * file or regions, null will be returned.
-   * <p>
-   * This call is most helpful with DFS, where it returns
-   * hostnames of machines that contain the given file.
-   * <p>
-   * The FileSystem will simply return an elt containing 'localhost'.
-   */
-  @Override
-  public BlockLocation[] getFileBlockLocations(FileStatus file,
-                                               long start,
-                                               long len) throws IOException {
-    //argument checks
-    if (file == null) {
-      return null;
-    }
-
-    if (start < 0 || len < 0) {
-      throw new IllegalArgumentException("Negative start or len parameter" +
-                                         " to getFileBlockLocations");
-    }
-    if (file.getLen() <= start) {
-      return new BlockLocation[0];
-    }
-
-    // Check if requested file in Swift is more than 5Gb. In this case
-    // each block has its own location -which may be determinable
-    // from the Swift client API, depending on the remote server
-    final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(),
-                                                             false,
-                                                             true);
-    List<URI> locations = new ArrayList<URI>();
-    if (listOfFileBlocks.length > 1) {
-      for (FileStatus fileStatus : listOfFileBlocks) {
-        if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
-                .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
-          continue;
-        }
-        locations.addAll(store.getObjectLocation(fileStatus.getPath()));
-      }
-    } else {
-      locations = store.getObjectLocation(file.getPath());
-    }
-
-    if (locations.isEmpty()) {
-      LOG.debug("No locations returned for " + file.getPath());
-      //no locations were returned for the object
-      //fall back to the superclass
-
-      String[] name = {SwiftProtocolConstants.BLOCK_LOCATION};
-      String[] host = { "localhost" };
-      String[] topology={SwiftProtocolConstants.TOPOLOGY_PATH};
-      return new BlockLocation[] {
-        new BlockLocation(name, host, topology,0, file.getLen())
-      };
-    }
-
-    final String[] names = new String[locations.size()];
-    final String[] hosts = new String[locations.size()];
-    int i = 0;
-    for (URI location : locations) {
-      hosts[i] = location.getHost();
-      names[i] = location.getAuthority();
-      i++;
-    }
-    return new BlockLocation[]{
-            new BlockLocation(names, hosts, 0, file.getLen())
-    };
-  }
-
-  /**
-   * Create the parent directories.
-   * As an optimization, the entire hierarchy of parent
-   * directories is <i>Not</i> polled. Instead
-   * the tree is walked up from the last to the first,
-   * creating directories until one that exists is found.
-   *
-   * This strategy means if a file is created in an existing directory,
-   * one quick poll suffices.
-   *
-   * There is a big assumption here: that all parent directories of an existing
-   * directory also exists.
-   * @param path path to create.
-   * @param permission to apply to files
-   * @return true if the operation was successful
-   * @throws IOException on a problem
-   */
-  @Override
-  public boolean mkdirs(Path path, FsPermission permission) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("SwiftFileSystem.mkdirs: " + path);
-    }
-    Path directory = makeAbsolute(path);
-
-    //build a list of paths to create
-    List<Path> paths = new ArrayList<Path>();
-    while (shouldCreate(directory)) {
-      //this directory needs creation, add to the list
-      paths.add(0, directory);
-      //now see if the parent needs to be created
-      directory = directory.getParent();
-    }
-
-    //go through the list of directories to create
-    for (Path p : paths) {
-      if (isNotRoot(p)) {
-        //perform a mkdir operation without any polling of
-        //the far end first
-        forceMkdir(p);
-      }
-    }
-
-    //if an exception was not thrown, this operation is considered
-    //a success
-    return true;
-  }
-
-  private boolean isNotRoot(Path absolutePath) {
-    return !isRoot(absolutePath);
-  }
-
-  private boolean isRoot(Path absolutePath) {
-    return absolutePath.getParent() == null;
-  }
-
-  /**
-   * internal implementation of directory creation.
-   *
-   * @param path path to file
-   * @return boolean file is created; false: no need to create
-   * @throws IOException if specified path is file instead of directory
-   */
-  private boolean mkdir(Path path) throws IOException {
-    Path directory = makeAbsolute(path);
-    boolean shouldCreate = shouldCreate(directory);
-    if (shouldCreate) {
-      forceMkdir(directory);
-    }
-    return shouldCreate;
-  }
-
-  /**
-   * Should mkdir create this directory?
-   * If the directory is root : false
-   * If the entry exists and is a directory: false
-   * If the entry exists and is a file: exception
-   * else: true
-   * @param directory path to query
-   * @return true iff the directory should be created
-   * @throws IOException IO problems
-   * @throws ParentNotDirectoryException if the path references a file
-   */
-  private boolean shouldCreate(Path directory) throws IOException {
-    FileStatus fileStatus;
-    boolean shouldCreate;
-    if (isRoot(directory)) {
-      //its the base dir, bail out immediately
-      return false;
-    }
-    try {
-      //find out about the path
-      fileStatus = getFileStatus(directory);
-
-      if (!SwiftUtils.isDirectory(fileStatus)) {
-        //if it's a file, raise an error
-        throw new ParentNotDirectoryException(
-                String.format("%s: can't mkdir since it exists and is not a directory: %s",
-                    directory, fileStatus));
-      } else {
-        //path exists, and it is a directory
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("skipping mkdir(" + directory + ") as it exists already");
-        }
-        shouldCreate = false;
-      }
-    } catch (FileNotFoundException e) {
-      shouldCreate = true;
-    }
-    return shouldCreate;
-  }
-
-  /**
-   * mkdir of a directory -irrespective of what was there underneath.
-   * There are no checks for the directory existing, there not
-   * being a path there, etc. etc. Those are assumed to have
-   * taken place already
-   * @param absolutePath path to create
-   * @throws IOException IO problems
-   */
-  private void forceMkdir(Path absolutePath) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Making dir '" + absolutePath + "' in Swift");
-    }
-    //file is not found: it must be created
-    store.createDirectory(absolutePath);
-  }
-
-  /**
-   * List the statuses of the files/directories in the given path if the path is
-   * a directory.
-   *
-   * @param path given path
-   * @return the statuses of the files/directories in the given path
-   * @throws IOException
-   */
-  @Override
-  public FileStatus[] listStatus(Path path) throws IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("SwiftFileSystem.listStatus for: " + path);
-    }
-    return store.listSubPaths(makeAbsolute(path), false, true);
-  }
-
-  /**
-   * This optional operation is not supported
-   */
-  @Override
-  public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
-      throws IOException {
-    LOG.debug("SwiftFileSystem.append");
-    throw new SwiftUnsupportedFeatureException("Not supported: append()");
-  }
-
-  /**
-   * @param permission Currently ignored.
-   */
-  @Override
-  public FSDataOutputStream create(Path file, FsPermission permission,
-                                   boolean overwrite, int bufferSize,
-                                   short replication, long blockSize,
-                                   Progressable progress)
-          throws IOException {
-    LOG.debug("SwiftFileSystem.create");
-
-    FileStatus fileStatus = null;
-    Path absolutePath = makeAbsolute(file);
-    try {
-      fileStatus = getFileStatus(absolutePath);
-    } catch (FileNotFoundException e) {
-      //the file isn't there.
-    }
-
-    if (fileStatus != null) {
-      //the path exists -action depends on whether or not it is a directory,
-      //and what the overwrite policy is.
-
-      //What is clear at this point is that if the entry exists, there's
-      //no need to bother creating any parent entries
-      if (fileStatus.isDirectory()) {
-        //here someone is trying to create a file over a directory
-
-/*    we can't throw an exception here as there is no easy way to distinguish
-     a file from the dir
-
-        throw new SwiftPathExistsException("Cannot create a file over a directory:"
-                                           + file);
- */
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Overwriting either an empty file or a directory");
-        }
-      }
-      if (overwrite) {
-        //overwrite set -> delete the object.
-        store.delete(absolutePath, true);
-      } else {
-        throw new FileAlreadyExistsException("Path exists: " + file);
-      }
-    } else {
-      // destination does not exist -trigger creation of the parent
-      Path parent = file.getParent();
-      if (parent != null) {
-        if (!mkdirs(parent)) {
-          throw new SwiftOperationFailedException(
-            "Mkdirs failed to create " + parent);
-        }
-      }
-    }
-
-    SwiftNativeOutputStream out = createSwiftOutputStream(file);
-    return new FSDataOutputStream(out, statistics);
-  }
-
-  /**
-   * Create the swift output stream
-   * @param path path to write to
-   * @return the new file
-   * @throws IOException
-   */
-  protected SwiftNativeOutputStream createSwiftOutputStream(Path path) throws
-                                                                       IOException {
-    long partSizeKB = getStore().getPartsizeKB();
-    return new SwiftNativeOutputStream(getConf(),
-            getStore(),
-            path.toUri().toString(),
-            partSizeKB);
-  }
-
-  /**
-   * Opens an FSDataInputStream at the indicated Path.
-   *
-   * @param path       the file name to open
-   * @param bufferSize the size of the buffer to be used.
-   * @return the input stream
-   * @throws FileNotFoundException if the file is not found
-   * @throws IOException any IO problem
-   */
-  @Override
-  public FSDataInputStream open(Path path, int bufferSize) throws IOException {
-    int bufferSizeKB = getStore().getBufferSizeKB();
-    long readBlockSize = bufferSizeKB * 1024L;
-    return open(path, bufferSize, readBlockSize);
-  }
-
-  /**
-   * Low-level operation to also set the block size for this operation
-   * @param path       the file name to open
-   * @param bufferSize the size of the buffer to be used.
-   * @param readBlockSize how big should the read block/buffer size be?
-   * @return the input stream
-   * @throws FileNotFoundException if the file is not found
-   * @throws IOException any IO problem
-   */
-  public FSDataInputStream open(Path path,
-                                int bufferSize,
-                                long readBlockSize) throws IOException {
-    if (readBlockSize <= 0) {
-      throw new SwiftConfigurationException("Bad remote buffer size");
-    }
-    Path absolutePath = makeAbsolute(path);
-    return new FSDataInputStream(
-            new StrictBufferedFSInputStream(
-                    new SwiftNativeInputStream(store,
-                                       statistics,
-                                       absolutePath,
-                                       readBlockSize),
-                    bufferSize));
-  }
-
-  /**
-   * Renames Path src to Path dst. On swift this uses copy-and-delete
-   * and <i>is not atomic</i>.
-   *
-   * @param src path
-   * @param dst path
-   * @return true if directory renamed, false otherwise
-   * @throws IOException on problems
-   */
-  @Override
-  public boolean rename(Path src, Path dst) throws IOException {
-
-    try {
-      store.rename(makeAbsolute(src), makeAbsolute(dst));
-      //success
-      return true;
-    } catch (SwiftOperationFailedException
-        | FileAlreadyExistsException
-        | FileNotFoundException
-        | ParentNotDirectoryException e) {
-      //downgrade to a failure
-      LOG.debug("rename({}, {}) failed",src, dst, e);
-      return false;
-    }
-  }
-
-
-  /**
-   * Delete a file or directory
-   *
-   * @param path      the path to delete.
-   * @param recursive if path is a directory and set to
-   *                  true, the directory is deleted else throws an exception if the
-   *                  directory is not empty
-   *                  case of a file the recursive can be set to either true or false.
-   * @return true if the object was deleted
-   * @throws IOException IO problems
-   */
-  @Override
-  public boolean delete(Path path, boolean recursive) throws IOException {
-    try {
-      return store.delete(path, recursive);
-    } catch (FileNotFoundException e) {
-      //base path was not found.
-      return false;
-    }
-  }
-
-  /**
-   * Delete a file.
-   * This method is abstract in Hadoop 1.x; in 2.x+ it is non-abstract
-   * and deprecated
-   */
-  @Override
-  public boolean delete(Path f) throws IOException {
-    return delete(f, true);
-  }
-
-  /**
-   * Makes path absolute
-   *
-   * @param path path to file
-   * @return absolute path
-   */
-  protected Path makeAbsolute(Path path) {
-    if (path.isAbsolute()) {
-      return path;
-    }
-    return new Path(workingDir, path);
-  }
-
-  /**
-   * Get the current operation statistics
-   * @return a snapshot of the statistics
-   */
-  public List<DurationStats> getOperationStatistics() {
-    return store.getOperationStatistics();
-  }
-
-  /**
-   * Low level method to do a deep listing of all entries, not stopping
-   * at the next directory entry. This is to let tests be confident that
-   * recursive deletes really are working.
-   * @param path path to recurse down
-   * @param newest ask for the newest data, potentially slower than not.
-   * @return a potentially empty array of file status
-   * @throws IOException any problem
-   */
-  @InterfaceAudience.Private
-  public FileStatus[] listRawFileStatus(Path path, boolean newest) throws IOException {
-    return store.listSubPaths(makeAbsolute(path), true, newest);
-  }
-
-  /**
-   * Get the number of partitions written by an output stream
-   * This is for testing
-   * @param outputStream output stream
-   * @return the #of partitions written by that stream
-   */
-  @InterfaceAudience.Private
-  public static int getPartitionsWritten(FSDataOutputStream outputStream) {
-    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
-    return snos.getPartitionsWritten();
-  }
-
-  private static SwiftNativeOutputStream getSwiftNativeOutputStream(
-    FSDataOutputStream outputStream) {
-    OutputStream wrappedStream = outputStream.getWrappedStream();
-    return (SwiftNativeOutputStream) wrappedStream;
-  }
-
-  /**
-   * Get the size of partitions written by an output stream
-   * This is for testing
-   *
-   * @param outputStream output stream
-   * @return partition size in bytes
-   */
-  @InterfaceAudience.Private
-  public static long getPartitionSize(FSDataOutputStream outputStream) {
-    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
-    return snos.getFilePartSize();
-  }
-
-  /**
-   * Get the the number of bytes written to an output stream
-   * This is for testing
-   *
-   * @param outputStream output stream
-   * @return partition size in bytes
-   */
-  @InterfaceAudience.Private
-  public static long getBytesWritten(FSDataOutputStream outputStream) {
-    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
-    return snos.getBytesWritten();
-  }
-
-  /**
-   * Get the the number of bytes uploaded by an output stream
-   * to the swift cluster.
-   * This is for testing
-   *
-   * @param outputStream output stream
-   * @return partition size in bytes
-   */
-  @InterfaceAudience.Private
-  public static long getBytesUploaded(FSDataOutputStream outputStream) {
-    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
-    return snos.getBytesUploaded();
-  }
-
-  /**
-   * {@inheritDoc}
-   * @throws FileNotFoundException if the parent directory is not present -or
-   * is not a directory.
-   */
-  @Override
-  public FSDataOutputStream createNonRecursive(Path path,
-      FsPermission permission,
-      EnumSet<CreateFlag> flags,
-      int bufferSize,
-      short replication,
-      long blockSize,
-      Progressable progress) throws IOException {
-    Path parent = path.getParent();
-    if (parent != null) {
-      // expect this to raise an exception if there is no parent
-      if (!getFileStatus(parent).isDirectory()) {
-        throw new FileAlreadyExistsException("Not a directory: " + parent);
-      }
-    }
-    return create(path, permission,
-        flags.contains(CreateFlag.OVERWRITE), bufferSize,
-        replication, blockSize, progress);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
deleted file mode 100644
index 5e4800900920a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java
+++ /dev/null
@@ -1,986 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.swift.snative;
-
-import com.fasterxml.jackson.databind.type.CollectionType;
-
-import org.apache.http.Header;
-import org.apache.http.HttpStatus;
-import org.apache.http.message.BasicHeader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
-import org.apache.hadoop.fs.swift.http.HttpBodyContent;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.http.SwiftRestClient;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.JSONUtil;
-import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InterruptedIOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.Charset;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * File system store implementation.
- * Makes REST requests, parses data from responses
- */
-public class SwiftNativeFileSystemStore {
-  private static final Pattern URI_PATTERN = Pattern.compile("\"\\S+?\"");
-  private static final String PATTERN = "EEE, d MMM yyyy hh:mm:ss zzz";
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftNativeFileSystemStore.class);
-  private URI uri;
-  private SwiftRestClient swiftRestClient;
-
-  /**
-   * Initalize the filesystem store -this creates the REST client binding.
-   *
-   * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
-   *                      options in the configuration file
-   * @param configuration configuration
-   * @throws IOException on any failure.
-   */
-  public void initialize(URI fsURI, Configuration configuration) throws IOException {
-    this.uri = fsURI;
-    this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
-  }
-
-  @Override
-  public String toString() {
-    return "SwiftNativeFileSystemStore with "
-            + swiftRestClient;
-  }
-
-  /**
-   * Get the default blocksize of this (bound) filesystem
-   * @return the blocksize returned for all FileStatus queries,
-   * which is used by the MapReduce splitter.
-   */
-  public long getBlocksize() {
-    return 1024L * swiftRestClient.getBlocksizeKB();
-  }
-
-  public long getPartsizeKB() {
-    return swiftRestClient.getPartSizeKB();
-  }
-
-  public int getBufferSizeKB() {
-    return swiftRestClient.getBufferSizeKB();
-  }
-
-  public int getThrottleDelay() {
-    return swiftRestClient.getThrottleDelay();
-  }
-  /**
-   * Upload a file/input stream of a specific length.
-   *
-   * @param path        destination path in the swift filesystem
-   * @param inputStream input data. This is closed afterwards, always
-   * @param length      length of the data
-   * @throws IOException on a problem
-   */
-  public void uploadFile(Path path, InputStream inputStream, long length)
-          throws IOException {
-      swiftRestClient.upload(toObjectPath(path), inputStream, length);
-  }
-
-  /**
-   * Upload part of a larger file.
-   *
-   * @param path        destination path
-   * @param partNumber  item number in the path
-   * @param inputStream input data
-   * @param length      length of the data
-   * @throws IOException on a problem
-   */
-  public void uploadFilePart(Path path, int partNumber,
-                             InputStream inputStream, long length)
-          throws IOException {
-
-    String stringPath = path.toUri().toString();
-    String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
-      partNumber);
-    if (stringPath.endsWith("/")) {
-      stringPath = stringPath.concat(partitionFilename);
-    } else {
-      stringPath = stringPath.concat("/").concat(partitionFilename);
-    }
-
-    swiftRestClient.upload(
-      new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
-            inputStream,
-            length);
-  }
-
-  /**
-   * Tell the Swift server to expect a multi-part upload by submitting
-   * a 0-byte file with the X-Object-Manifest header
-   *
-   * @param path path of final final
-   * @throws IOException
-   */
-  public void createManifestForPartUpload(Path path) throws IOException {
-    String pathString = toObjectPath(path).toString();
-    if (!pathString.endsWith("/")) {
-      pathString = pathString.concat("/");
-    }
-    if (pathString.startsWith("/")) {
-      pathString = pathString.substring(1);
-    }
-
-    swiftRestClient.upload(toObjectPath(path),
-        new ByteArrayInputStream(new byte[0]),
-        0,
-        new BasicHeader(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString));
-  }
-
-  /**
-   * Get the metadata of an object
-   *
-   * @param path path
-   * @return file metadata. -or null if no headers were received back from the server.
-   * @throws IOException           on a problem
-   * @throws FileNotFoundException if there is nothing at the end
-   */
-  public SwiftFileStatus getObjectMetadata(Path path) throws IOException {
-    return getObjectMetadata(path, true);
-  }
-
-  /**
-   * Get the HTTP headers, in case you really need the low-level
-   * metadata
-   * @param path path to probe
-   * @param newest newest or oldest?
-   * @return the header list
-   * @throws IOException IO problem
-   * @throws FileNotFoundException if there is nothing at the end
-   */
-  public Header[] getObjectHeaders(Path path, boolean newest)
-    throws IOException, FileNotFoundException {
-    SwiftObjectPath objectPath = toObjectPath(path);
-    return stat(objectPath, newest);
-  }
-
-  /**
-   * Get the metadata of an object
-   *
-   * @param path path
-   * @param newest flag to say "set the newest header", otherwise take any entry
-   * @return file metadata. -or null if no headers were received back from the server.
-   * @throws IOException           on a problem
-   * @throws FileNotFoundException if there is nothing at the end
-   */
-  public SwiftFileStatus getObjectMetadata(Path path, boolean newest)
-    throws IOException, FileNotFoundException {
-
-    SwiftObjectPath objectPath = toObjectPath(path);
-    final Header[] headers = stat(objectPath, newest);
-    //no headers is treated as a missing file
-    if (headers.length == 0) {
-      throw new FileNotFoundException("Not Found " + path.toUri());
-    }
-
-    boolean isDir = false;
-    long length = 0;
-    long lastModified = 0 ;
-    for (Header header : headers) {
-      String headerName = header.getName();
-      if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
-              headerName.equals(SwiftProtocolConstants.X_CONTAINER_BYTES_USED)) {
-        length = 0;
-        isDir = true;
-      }
-      if (SwiftProtocolConstants.HEADER_CONTENT_LENGTH.equals(headerName)) {
-        length = Long.parseLong(header.getValue());
-      }
-      if (SwiftProtocolConstants.HEADER_LAST_MODIFIED.equals(headerName)) {
-        final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(PATTERN);
-        try {
-          lastModified = simpleDateFormat.parse(header.getValue()).getTime();
-        } catch (ParseException e) {
-          throw new SwiftException("Failed to parse " + header.toString(), e);
-        }
-      }
-    }
-    if (lastModified == 0) {
-      lastModified = System.currentTimeMillis();
-    }
-
-    Path correctSwiftPath = getCorrectSwiftPath(path);
-    return new SwiftFileStatus(length,
-                               isDir,
-                               1,
-                               getBlocksize(),
-                               lastModified,
-                               correctSwiftPath);
-  }
-
-  private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
-                                                                    IOException {
-    Header[] headers;
-    if (newest) {
-      headers = swiftRestClient.headRequest("getObjectMetadata-newest",
-                                            objectPath, SwiftRestClient.NEWEST);
-    } else {
-      headers = swiftRestClient.headRequest("getObjectMetadata",
-                                            objectPath);
-    }
-    return headers;
-  }
-
-  /**
-   * Get the object as an input stream
-   *
-   * @param path object path
-   * @return the input stream -this must be closed to terminate the connection
-   * @throws IOException           IO problems
-   * @throws FileNotFoundException path doesn't resolve to an object
-   */
-  public HttpBodyContent getObject(Path path) throws IOException {
-    return swiftRestClient.getData(toObjectPath(path),
-                                   SwiftRestClient.NEWEST);
-  }
-
-  /**
-   * Get the input stream starting from a specific point.
-   *
-   * @param path           path to object
-   * @param byteRangeStart starting point
-   * @param length         no. of bytes
-   * @return an input stream that must be closed
-   * @throws IOException IO problems
-   */
-  public HttpBodyContent getObject(Path path, long byteRangeStart, long length)
-          throws IOException {
-    return swiftRestClient.getData(
-      toObjectPath(path), byteRangeStart, length);
-  }
-
-  /**
-   * List a directory.
-   * This is O(n) for the number of objects in this path.
-   *
-   *
-   *
-   * @param path working path
-   * @param listDeep ask for all the data
-   * @param newest ask for the newest data
-   * @return Collection of file statuses
-   * @throws IOException IO problems
-   * @throws FileNotFoundException if the path does not exist
-   */
-  private List<FileStatus> listDirectory(SwiftObjectPath path,
-                                         boolean listDeep,
-                                         boolean newest) throws IOException {
-    final byte[] bytes;
-    final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
-    final Path correctSwiftPath = getCorrectSwiftPath(path);
-    try {
-      bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep);
-    } catch (FileNotFoundException e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("" +
-                "File/Directory not found " + path);
-      }
-      if (SwiftUtils.isRootDir(path)) {
-        return Collections.emptyList();
-      } else {
-        throw e;
-      }
-    } catch (SwiftInvalidResponseException e) {
-      //bad HTTP error code
-      if (e.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
-        //this can come back on a root list if the container is empty
-        if (SwiftUtils.isRootDir(path)) {
-          return Collections.emptyList();
-        } else {
-          //NO_CONTENT returned on something other than the root directory;
-          //see if it is there, and convert to empty list or not found
-          //depending on whether the entry exists.
-          FileStatus stat = getObjectMetadata(correctSwiftPath, newest);
-
-          if (stat.isDirectory()) {
-            //it's an empty directory. state that
-            return Collections.emptyList();
-          } else {
-            //it's a file -return that as the status
-            files.add(stat);
-            return files;
-          }
-        }
-      } else {
-        //a different status code: rethrow immediately
-        throw e;
-      }
-    }
-
-    final CollectionType collectionType = JSONUtil.getJsonMapper().getTypeFactory().
-            constructCollectionType(List.class, SwiftObjectFileStatus.class);
-
-    final List<SwiftObjectFileStatus> fileStatusList = JSONUtil.toObject(
-        new String(bytes, Charset.forName("UTF-8")), collectionType);
-
-    //this can happen if user lists file /data/files/file
-    //in this case swift will return empty array
-    if (fileStatusList.isEmpty()) {
-      SwiftFileStatus objectMetadata = getObjectMetadata(correctSwiftPath,
-                                                         newest);
-      if (objectMetadata.isFile()) {
-        files.add(objectMetadata);
-      }
-
-      return files;
-    }
-
-    for (SwiftObjectFileStatus status : fileStatusList) {
-      if (status.getName() != null) {
-          files.add(new SwiftFileStatus(status.getBytes(),
-                  status.getBytes() == 0,
-                  1,
-                  getBlocksize(),
-                  status.getLast_modified().getTime(),
-                  getCorrectSwiftPath(new Path(status.getName()))));
-      }
-    }
-
-    return files;
-  }
-
-  /**
-   * List all elements in this directory
-   *
-   *
-   *
-   * @param path     path to work with
-   * @param recursive do a recursive get
-   * @param newest ask for the newest, or can some out of date data work?
-   * @return the file statuses, or an empty array if there are no children
-   * @throws IOException           on IO problems
-   * @throws FileNotFoundException if the path is nonexistent
-   */
-  public FileStatus[] listSubPaths(Path path,
-                                   boolean recursive,
-                                   boolean newest) throws IOException {
-    final Collection<FileStatus> fileStatuses;
-    fileStatuses = listDirectory(toDirPath(path), recursive, newest);
-    return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
-  }
-
-  /**
-   * Create a directory
-   *
-   * @param path path
-   * @throws IOException
-   */
-  public void createDirectory(Path path) throws IOException {
-    innerCreateDirectory(toDirPath(path));
-  }
-
-  /**
-   * The inner directory creation option. This only creates
-   * the dir at the given path, not any parent dirs.
-   * @param swiftObjectPath swift object path at which a 0-byte blob should be
-   * put
-   * @throws IOException IO problems
-   */
-  private void innerCreateDirectory(SwiftObjectPath swiftObjectPath)
-          throws IOException {
-
-    swiftRestClient.putRequest(swiftObjectPath);
-  }
-
-  private SwiftObjectPath toDirPath(Path path) throws
-          SwiftConfigurationException {
-    return SwiftObjectPath.fromPath(uri, path, false);
-  }
-
-  private SwiftObjectPath toObjectPath(Path path) throws
-          SwiftConfigurationException {
-    return SwiftObjectPath.fromPath(uri, path);
-  }
-
-  /**
-   * Try to find the specific server(s) on which the data lives
-   * @param path path to probe
-   * @return a possibly empty list of locations
-   * @throws IOException on problems determining the locations
-   */
-  public List<URI> getObjectLocation(Path path) throws IOException {
-    final byte[] objectLocation;
-    objectLocation = swiftRestClient.getObjectLocation(toObjectPath(path));
-    if (objectLocation == null || objectLocation.length == 0) {
-      //no object location, return an empty list
-      return new LinkedList<URI>();
-    }
-    return extractUris(new String(objectLocation, Charset.forName("UTF-8")), path);
-  }
-
-  /**
-   * deletes object from Swift
-   *
-   * @param path path to delete
-   * @return true if the path was deleted by this specific operation.
-   * @throws IOException on a failure
-   */
-  public boolean deleteObject(Path path) throws IOException {
-    SwiftObjectPath swiftObjectPath = toObjectPath(path);
-    if (!SwiftUtils.isRootDir(swiftObjectPath)) {
-      return swiftRestClient.delete(swiftObjectPath);
-    } else {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Not deleting root directory entry");
-      }
-      return true;
-    }
-  }
-
-  /**
-   * deletes a directory from Swift. This is not recursive
-   *
-   * @param path path to delete
-   * @return true if the path was deleted by this specific operation -or
-   *         the path was root and not acted on.
-   * @throws IOException on a failure
-   */
-  public boolean rmdir(Path path) throws IOException {
-    return deleteObject(path);
-  }
-
-  /**
-   * Does the object exist
-   *
-   * @param path object path
-   * @return true if the metadata of an object could be retrieved
-   * @throws IOException IO problems other than FileNotFound, which
-   *                     is downgraded to an object does not exist return code
-   */
-  public boolean objectExists(Path path) throws IOException {
-    return objectExists(toObjectPath(path));
-  }
-
-  /**
-   * Does the object exist
-   *
-   * @param path swift object path
-   * @return true if the metadata of an object could be retrieved
-   * @throws IOException IO problems other than FileNotFound, which
-   *                     is downgraded to an object does not exist return code
-   */
-  public boolean objectExists(SwiftObjectPath path) throws IOException {
-    try {
-      Header[] headers = swiftRestClient.headRequest("objectExists",
-                                                     path,
-                                                     SwiftRestClient.NEWEST);
-      //no headers is treated as a missing file
-      return headers.length != 0;
-    } catch (FileNotFoundException e) {
-      return false;
-    }
-  }
-
-  /**
-   * Rename through copy-and-delete. this is a consequence of the
-   * Swift filesystem using the path as the hash
-   * into the Distributed Hash Table, "the ring" of filenames.
-   * <p>
-   * Because of the nature of the operation, it is not atomic.
-   *
-   * @param src source file/dir
-   * @param dst destination
-   * @throws IOException                   IO failure
-   * @throws SwiftOperationFailedException if the rename failed
-   * @throws FileNotFoundException         if the source directory is missing, or
-   *                                       the parent directory of the destination
-   */
-  public void rename(Path src, Path dst)
-    throws FileNotFoundException, SwiftOperationFailedException, IOException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("mv " + src + " " + dst);
-    }
-    boolean renamingOnToSelf = src.equals(dst);
-
-    SwiftObjectPath srcObject = toObjectPath(src);
-    SwiftObjectPath destObject = toObjectPath(dst);
-
-    if (SwiftUtils.isRootDir(srcObject)) {
-      throw new SwiftOperationFailedException("cannot rename root dir");
-    }
-
-    final SwiftFileStatus srcMetadata;
-    srcMetadata = getObjectMetadata(src);
-    SwiftFileStatus dstMetadata;
-    try {
-      dstMetadata = getObjectMetadata(dst);
-    } catch (FileNotFoundException e) {
-      //destination does not exist.
-      LOG.debug("Destination does not exist");
-      dstMetadata = null;
-    }
-
-    //check to see if the destination parent directory exists
-    Path srcParent = src.getParent();
-    Path dstParent = dst.getParent();
-    //skip the overhead of a HEAD call if the src and dest share the same
-    //parent dir (in which case the dest dir exists), or the destination
-    //directory is root, in which case it must also exist
-    if (dstParent != null && !dstParent.equals(srcParent)) {
-      SwiftFileStatus fileStatus;
-      try {
-        fileStatus = getObjectMetadata(dstParent);
-      } catch (FileNotFoundException e) {
-        //destination parent doesn't exist; bail out
-        LOG.debug("destination parent directory " + dstParent + " doesn't exist");
-        throw e;
-      }
-      if (!fileStatus.isDir()) {
-        throw new ParentNotDirectoryException(dstParent.toString());
-      }
-    }
-
-    boolean destExists = dstMetadata != null;
-    boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
-    //calculate the destination
-    SwiftObjectPath destPath;
-
-    //enum the child entries and everything underneath
-    List<FileStatus> childStats = listDirectory(srcObject, true, true);
-    boolean srcIsFile = !srcMetadata.isDirectory();
-    if (srcIsFile) {
-
-      //source is a simple file OR a partitioned file
-      // outcomes:
-      // #1 dest exists and is file: fail
-      // #2 dest exists and is dir: destination path becomes under dest dir
-      // #3 dest does not exist: use dest as name
-      if (destExists) {
-
-        if (destIsDir) {
-          //outcome #2 -move to subdir of dest
-          destPath = toObjectPath(new Path(dst, src.getName()));
-        } else {
-          //outcome #1 dest it's a file: fail if different
-          if (!renamingOnToSelf) {
-            throw new FileAlreadyExistsException(
-                    "cannot rename a file over one that already exists");
-          } else {
-            //is mv self self where self is a file. this becomes a no-op
-            LOG.debug("Renaming file onto self: no-op => success");
-            return;
-          }
-        }
-      } else {
-        //outcome #3 -new entry
-        destPath = toObjectPath(dst);
-      }
-      int childCount = childStats.size();
-      //here there is one of:
-      // - a single object ==> standard file
-      // ->
-      if (childCount == 0) {
-        copyThenDeleteObject(srcObject, destPath);
-      } else {
-        //do the copy
-        SwiftUtils.debug(LOG, "Source file appears to be partitioned." +
-                              " copying file and deleting children");
-
-        copyObject(srcObject, destPath);
-        for (FileStatus stat : childStats) {
-          SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
-          deleteObject(stat.getPath());
-        }
-
-        swiftRestClient.delete(srcObject);
-      }
-    } else {
-
-      //here the source exists and is a directory
-      // outcomes (given we know the parent dir exists if we get this far)
-      // #1 destination is a file: fail
-      // #2 destination is a directory: create a new dir under that one
-      // #3 destination doesn't exist: create a new dir with that name
-      // #3 and #4 are only allowed if the dest path is not == or under src
-
-
-      if (destExists && !destIsDir) {
-        // #1 destination is a file: fail
-        throw new FileAlreadyExistsException(
-                "the source is a directory, but not the destination");
-      }
-      Path targetPath;
-      if (destExists) {
-        // #2 destination is a directory: create a new dir under that one
-        targetPath = new Path(dst, src.getName());
-      } else {
-        // #3 destination doesn't exist: create a new dir with that name
-        targetPath = dst;
-      }
-      SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
-      //final check for any recursive operations
-      if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
-        //you can't rename a directory onto itself
-        throw new SwiftOperationFailedException(
-          "cannot move a directory under itself");
-      }
-
-
-      LOG.info("mv  " + srcObject + " " + targetPath);
-
-      logDirectory("Directory to copy ", srcObject, childStats);
-
-      // iterative copy of everything under the directory.
-      // by listing all children this can be done iteratively
-      // rather than recursively -everything in this list is either a file
-      // or a 0-byte-len file pretending to be a directory.
-      String srcURI = src.toUri().toString();
-      int prefixStripCount = srcURI.length() + 1;
-      for (FileStatus fileStatus : childStats) {
-        Path copySourcePath = fileStatus.getPath();
-        String copySourceURI = copySourcePath.toUri().toString();
-
-        String copyDestSubPath = copySourceURI.substring(prefixStripCount);
-
-        Path copyDestPath = new Path(targetPath, copyDestSubPath);
-        if (LOG.isTraceEnabled()) {
-          //trace to debug some low-level rename path problems; retained
-          //in case they ever come back.
-          LOG.trace("srcURI=" + srcURI
-                  + "; copySourceURI=" + copySourceURI
-                  + "; copyDestSubPath=" + copyDestSubPath
-                  + "; copyDestPath=" + copyDestPath);
-        }
-        SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
-
-        try {
-          copyThenDeleteObject(toObjectPath(copySourcePath),
-                  copyDestination);
-        } catch (FileNotFoundException e) {
-          LOG.info("Skipping rename of " + copySourcePath);
-        }
-        //add a throttle delay
-        throttle();
-      }
-      //now rename self. If missing, create the dest directory and warn
-      if (!SwiftUtils.isRootDir(srcObject)) {
-        try {
-          copyThenDeleteObject(srcObject,
-                  targetObjectPath);
-        } catch (FileNotFoundException e) {
-          //create the destination directory
-          LOG.warn("Source directory deleted during rename", e);
-          innerCreateDirectory(destObject);
-        }
-      }
-    }
-  }
-
-  /**
-   * Debug action to dump directory statuses to the debug log
-   *
-   * @param message    explanation
-   * @param objectPath object path (can be null)
-   * @param statuses   listing output
-   */
-  private void logDirectory(String message, SwiftObjectPath objectPath,
-                            Iterable<FileStatus> statuses) {
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(message + ": listing of " + objectPath);
-      for (FileStatus fileStatus : statuses) {
-        LOG.debug(fileStatus.getPath().toString());
-      }
-    }
-  }
-
-  public void copy(Path srcKey, Path dstKey) throws IOException {
-    SwiftObjectPath srcObject = toObjectPath(srcKey);
-    SwiftObjectPath destObject = toObjectPath(dstKey);
-    swiftRestClient.copyObject(srcObject, destObject);
-  }
-
-
-  /**
-   * Copy an object then, if the copy worked, delete it.
-   * If the copy failed, the source object is not deleted.
-   *
-   * @param srcObject  source object path
-   * @param destObject destination object path
-   * @throws IOException IO problems
-
-   */
-  private void copyThenDeleteObject(SwiftObjectPath srcObject,
-                                    SwiftObjectPath destObject) throws
-          IOException {
-
-
-    //do the copy
-    copyObject(srcObject, destObject);
-    //getting here means the copy worked
-    swiftRestClient.delete(srcObject);
-  }
-  /**
-   * Copy an object
-   * @param srcObject  source object path
-   * @param destObject destination object path
-   * @throws IOException IO problems
-   */
-  private void copyObject(SwiftObjectPath srcObject,
-                                    SwiftObjectPath destObject) throws
-          IOException {
-    if (srcObject.isEqualToOrParentOf(destObject)) {
-      throw new SwiftException(
-        "Can't copy " + srcObject + " onto " + destObject);
-    }
-    //do the copy
-    boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
-    if (!copySucceeded) {
-      throw new SwiftException("Copy of " + srcObject + " to "
-              + destObject + "failed");
-    }
-  }
-
-  /**
-   * Take a Hadoop path and return one which uses the URI prefix and authority
-   * of this FS. It doesn't make a relative path absolute
-   * @param path path in
-   * @return path with a URI bound to this FS
-   * @throws SwiftException URI cannot be created.
-   */
-  public Path getCorrectSwiftPath(Path path) throws
-          SwiftException {
-    try {
-      final URI fullUri = new URI(uri.getScheme(),
-              uri.getAuthority(),
-              path.toUri().getPath(),
-              null,
-              null);
-
-      return new Path(fullUri);
-    } catch (URISyntaxException e) {
-      throw new SwiftException("Specified path " + path + " is incorrect", e);
-    }
-  }
-
-  /**
-   * Builds a hadoop-Path from a swift path, inserting the URI authority
-   * of this FS instance
-   * @param path swift object path
-   * @return Hadoop path
-   * @throws SwiftException if the URI couldn't be created.
-   */
-  private Path getCorrectSwiftPath(SwiftObjectPath path) throws
-          SwiftException {
-    try {
-      final URI fullUri = new URI(uri.getScheme(),
-              uri.getAuthority(),
-              path.getObject(),
-              null,
-              null);
-
-      return new Path(fullUri);
-    } catch (URISyntaxException e) {
-      throw new SwiftException("Specified path " + path + " is incorrect", e);
-    }
-  }
-
-
-  /**
-   * extracts URIs from json
-   * @param json json to parse
-   * @param path path (used in exceptions)
-   * @return URIs
-   * @throws SwiftOperationFailedException on any problem parsing the JSON
-   */
-  public static List<URI> extractUris(String json, Path path) throws
-                                                   SwiftOperationFailedException {
-    final Matcher matcher = URI_PATTERN.matcher(json);
-    final List<URI> result = new ArrayList<URI>();
-    while (matcher.find()) {
-      final String s = matcher.group();
-      final String uri = s.substring(1, s.length() - 1);
-      try {
-        URI createdUri = URI.create(uri);
-        result.add(createdUri);
-      } catch (IllegalArgumentException e) {
-        //failure to create the URI, which means this is bad JSON. Convert
-        //to an exception with useful text
-        throw new SwiftOperationFailedException(
-          String.format(
-            "could not convert \"%s\" into a URI." +
-            " source: %s " +
-            " first JSON: %s",
-            uri, path, json.substring(0, 256)));
-      }
-    }
-    return result;
-  }
-
-  /**
-   * Insert a throttled wait if the throttle delay &gt; 0
-   * @throws InterruptedIOException if interrupted during sleep
-   */
-  public void throttle() throws InterruptedIOException {
-    int throttleDelay = getThrottleDelay();
-    if (throttleDelay > 0) {
-      try {
-        Thread.sleep(throttleDelay);
-      } catch (InterruptedException e) {
-        //convert to an IOE
-        throw (InterruptedIOException) new InterruptedIOException(e.toString())
-          .initCause(e);
-      }
-    }
-  }
-
-  /**
-   * Get the current operation statistics
-   * @return a snapshot of the statistics
-   */
-  public List<DurationStats> getOperationStatistics() {
-    return swiftRestClient.getOperationStatistics();
-  }
-
-
-  /**
-   * Delete the entire tree. This is an internal one with slightly different
-   * behavior: if an entry is missing, a {@link FileNotFoundException} is
-   * raised. This lets the caller distinguish a file not found with
-   * other reasons for failure, so handles race conditions in recursive
-   * directory deletes better.
-   * <p>
-   * The problem being addressed is: caller A requests a recursive directory
-   * of directory /dir ; caller B requests a delete of a file /dir/file,
-   * between caller A enumerating the files contents, and requesting a delete
-   * of /dir/file. We want to recognise the special case
-   * "directed file is no longer there" and not convert that into a failure
-   *
-   * @param absolutePath  the path to delete.
-   * @param recursive if path is a directory and set to
-   *                  true, the directory is deleted else throws an exception if the
-   *                  directory is not empty
-   *                  case of a file the recursive can be set to either true or false.
-   * @return true if the object was deleted
-   * @throws IOException           IO problems
-   * @throws FileNotFoundException if a file/dir being deleted is not there -
-   *                               this includes entries below the specified path, (if the path is a dir
-   *                               and recursive is true)
-   */
-  public boolean delete(Path absolutePath, boolean recursive) throws IOException {
-    Path swiftPath = getCorrectSwiftPath(absolutePath);
-    SwiftUtils.debug(LOG, "Deleting path '%s' recursive=%b",
-                     absolutePath,
-                     recursive);
-    boolean askForNewest = true;
-    SwiftFileStatus fileStatus = getObjectMetadata(swiftPath, askForNewest);
-
-    //ask for the file/dir status, but don't demand the newest, as we
-    //don't mind if the directory has changed
-    //list all entries under this directory.
-    //this will throw FileNotFoundException if the file isn't there
-    FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest);
-    if (statuses == null) {
-      //the directory went away during the non-atomic stages of the operation.
-      // Return false as it was not this thread doing the deletion.
-      SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'",
-                       absolutePath,
-                       recursive);
-      return false;
-    }
-    int filecount = statuses.length;
-    SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
-                     absolutePath,
-                     filecount);
-
-    if (filecount == 0) {
-      //it's an empty directory or a path
-      rmdir(absolutePath);
-      return true;
-    }
-
-    if (LOG.isDebugEnabled()) {
-      SwiftUtils.debug(LOG, "%s", SwiftUtils.fileStatsToString(statuses, "\n"));
-    }
-
-    if (filecount == 1 && swiftPath.equals(statuses[0].getPath())) {
-      // 1 entry => simple file and it is the target
-      //simple file: delete it
-      SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
-      deleteObject(absolutePath);
-      return true;
-    }
-
-    //>1 entry implies directory with children. Run through them,
-    // but first check for the recursive flag and reject it *unless it looks
-    // like a partitioned file (len > 0 && has children)
-    if (!fileStatus.isDirectory()) {
-      LOG.debug("Multiple child entries but entry has data: assume partitioned");
-    } else if (!recursive) {
-      //if there are children, unless this is a recursive operation, fail immediately
-      throw new SwiftOperationFailedException("Directory " + fileStatus
-                                              + " is not empty: "
-                                              + SwiftUtils.fileStatsToString(
-                                                        statuses, "; "));
-    }
-
-    //delete the entries. including ourselves.
-    for (FileStatus entryStatus : statuses) {
-      Path entryPath = entryStatus.getPath();
-      try {
-        boolean deleted = deleteObject(entryPath);
-        if (!deleted) {
-          SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
-                           entryPath);
-        }
-      } catch (FileNotFoundException e) {
-        //the path went away -race conditions.
-        //do not fail, as the outcome is still OK.
-        SwiftUtils.debug(LOG, "Path '%s' is no longer present; continuing",
-                         entryPath);
-      }
-      throttle();
-    }
-    //now delete self
-    SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
-    deleteObject(absolutePath);
-
-    return true;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java
deleted file mode 100644
index bce7325c980e7..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftException;
-import org.apache.hadoop.fs.swift.http.HttpBodyContent;
-import org.apache.hadoop.fs.swift.http.HttpInputStreamWithRelease;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * The input stream from remote Swift blobs.
- * The class attempts to be buffer aware, and react to a forward seek operation
- * by trying to scan ahead through the current block of data to find it.
- * This accelerates some operations that do a lot of seek()/read() actions,
- * including work (such as in the MR engine) that do a seek() immediately after
- * an open().
- */
-class SwiftNativeInputStream extends FSInputStream {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftNativeInputStream.class);
-
-  /**
-   *  range requested off the server: {@value}
-   */
-  private final long bufferSize;
-
-  /**
-   * File nativeStore instance
-   */
-  private final SwiftNativeFileSystemStore nativeStore;
-
-  /**
-   * Hadoop statistics. Used to get info about number of reads, writes, etc.
-   */
-  private final FileSystem.Statistics statistics;
-
-  /**
-   * Data input stream
-   */
-  private HttpInputStreamWithRelease httpStream;
-
-  /**
-   * File path
-   */
-  private final Path path;
-
-  /**
-   * Current position
-   */
-  private long pos = 0;
-
-  /**
-   * Length of the file picked up at start time
-   */
-  private long contentLength = -1;
-
-  /**
-   * Why the stream is closed
-   */
-  private String reasonClosed = "unopened";
-
-  /**
-   * Offset in the range requested last
-   */
-  private long rangeOffset = 0;
-
-  public SwiftNativeInputStream(SwiftNativeFileSystemStore storeNative,
-      FileSystem.Statistics statistics, Path path, long bufferSize)
-          throws IOException {
-    this.nativeStore = storeNative;
-    this.statistics = statistics;
-    this.path = path;
-    if (bufferSize <= 0) {
-      throw new IllegalArgumentException("Invalid buffer size");
-    }
-    this.bufferSize = bufferSize;
-    //initial buffer fill
-    this.httpStream = storeNative.getObject(path).getInputStream();
-    //fillBuffer(0);
-  }
-
-  /**
-   * Move to a new position within the file relative to where the pointer is now.
-   * Always call from a synchronized clause
-   * @param offset offset
-   */
-  private synchronized void incPos(int offset) {
-    pos += offset;
-    rangeOffset += offset;
-    SwiftUtils.trace(LOG, "Inc: pos=%d bufferOffset=%d", pos, rangeOffset);
-  }
-
-  /**
-   * Update the start of the buffer; always call from a sync'd clause
-   * @param seekPos position sought.
-   * @param contentLength content length provided by response (may be -1)
-   */
-  private synchronized void updateStartOfBufferPosition(long seekPos,
-                                                        long contentLength) {
-    //reset the seek pointer
-    pos = seekPos;
-    //and put the buffer offset to 0
-    rangeOffset = 0;
-    this.contentLength = contentLength;
-    SwiftUtils.trace(LOG, "Move: pos=%d; bufferOffset=%d; contentLength=%d",
-                     pos,
-                     rangeOffset,
-                     contentLength);
-  }
-
-  @Override
-  public synchronized int read() throws IOException {
-    verifyOpen();
-    int result = -1;
-    try {
-      result = httpStream.read();
-    } catch (IOException e) {
-      String msg = "IOException while reading " + path
-                   + ": " +e + ", attempting to reopen.";
-      LOG.debug(msg, e);
-      if (reopenBuffer()) {
-        result = httpStream.read();
-      }
-    }
-    if (result != -1) {
-      incPos(1);
-    }
-    if (statistics != null && result != -1) {
-      statistics.incrementBytesRead(1);
-    }
-    return result;
-  }
-
-  @Override
-  public synchronized int read(byte[] b, int off, int len) throws IOException {
-    SwiftUtils.debug(LOG, "read(buffer, %d, %d)", off, len);
-    SwiftUtils.validateReadArgs(b, off, len);
-    if (len == 0) {
-      return 0;
-    }
-    int result = -1;
-    try {
-      verifyOpen();
-      result = httpStream.read(b, off, len);
-    } catch (IOException e) {
-      //other IO problems are viewed as transient and re-attempted
-      LOG.info("Received IOException while reading '" + path +
-               "', attempting to reopen: " + e);
-      LOG.debug("IOE on read()" + e, e);
-      if (reopenBuffer()) {
-        result = httpStream.read(b, off, len);
-      }
-    }
-    if (result > 0) {
-      incPos(result);
-      if (statistics != null) {
-        statistics.incrementBytesRead(result);
-      }
-    }
-
-    return result;
-  }
-
-  /**
-   * Re-open the buffer
-   * @return true iff more data could be added to the buffer
-   * @throws IOException if not
-   */
-  private boolean reopenBuffer() throws IOException {
-    innerClose("reopening buffer to trigger refresh");
-    boolean success = false;
-    try {
-      fillBuffer(pos);
-      success =  true;
-    } catch (EOFException eof) {
-      //the EOF has been reached
-      this.reasonClosed = "End of file";
-    }
-    return success;
-  }
-
-  /**
-   * close the stream. After this the stream is not usable -unless and until
-   * it is re-opened (which can happen on some of the buffer ops)
-   * This method is thread-safe and idempotent.
-   *
-   * @throws IOException on IO problems.
-   */
-  @Override
-  public synchronized void close() throws IOException {
-    innerClose("closed");
-  }
-
-  private void innerClose(String reason) throws IOException {
-    try {
-      if (httpStream != null) {
-        reasonClosed = reason;
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Closing HTTP input stream : " + reason);
-        }
-        httpStream.close();
-      }
-    } finally {
-      httpStream = null;
-    }
-  }
-
-  /**
-   * Assume that the connection is not closed: throws an exception if it is
-   * @throws SwiftConnectionClosedException
-   */
-  private void verifyOpen() throws SwiftConnectionClosedException {
-    if (httpStream == null) {
-      throw new SwiftConnectionClosedException(reasonClosed);
-    }
-  }
-
-  @Override
-  public synchronized String toString() {
-    return "SwiftNativeInputStream" +
-           " position=" + pos
-           + " buffer size = " + bufferSize
-           + " "
-           + (httpStream != null ? httpStream.toString()
-                                 : (" no input stream: " + reasonClosed));
-  }
-
-  /**
-   * Treats any finalize() call without the input stream being closed
-   * as a serious problem, logging at error level
-   * @throws Throwable n/a
-   */
-  @Override
-  protected void finalize() throws Throwable {
-    if (httpStream != null) {
-      LOG.error(
-        "Input stream is leaking handles by not being closed() properly: "
-        + httpStream.toString());
-    }
-  }
-
-  /**
-   * Read through the specified number of bytes.
-   * The implementation iterates a byte a time, which may seem inefficient
-   * compared to the read(bytes[]) method offered by input streams.
-   * However, if you look at the code that implements that method, it comes
-   * down to read() one char at a time -only here the return value is discarded.
-   *
-   *<p/>
-   * This is a no-op if the stream is closed
-   * @param bytes number of bytes to read.
-   * @throws IOException IO problems
-   * @throws SwiftException if a read returned -1.
-   */
-  private int chompBytes(long bytes) throws IOException {
-    int count = 0;
-    if (httpStream != null) {
-      int result;
-      for (long i = 0; i < bytes; i++) {
-        result = httpStream.read();
-        if (result < 0) {
-          throw new SwiftException("Received error code while chomping input");
-        }
-        count ++;
-        incPos(1);
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Seek to an offset. If the data is already in the buffer, move to it
-   * @param targetPos target position
-   * @throws IOException on any problem
-   */
-  @Override
-  public synchronized void seek(long targetPos) throws IOException {
-    if (targetPos < 0) {
-      throw new EOFException(
-          FSExceptionMessages.NEGATIVE_SEEK);
-    }
-    //there's some special handling of near-local data
-    //as the seek can be omitted if it is in/adjacent
-    long offset = targetPos - pos;
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Seek to " + targetPos + "; current pos =" + pos
-                + "; offset="+offset);
-    }
-    if (offset == 0) {
-      LOG.debug("seek is no-op");
-      return;
-    }
-
-    if (offset < 0) {
-      LOG.debug("seek is backwards");
-    } else if ((rangeOffset + offset < bufferSize)) {
-      //if the seek is in  range of that requested, scan forwards
-      //instead of closing and re-opening a new HTTP connection
-      SwiftUtils.debug(LOG,
-                       "seek is within current stream"
-                       + "; pos= %d ; targetPos=%d; "
-                       + "offset= %d ; bufferOffset=%d",
-                       pos, targetPos, offset, rangeOffset);
-      try {
-        LOG.debug("chomping ");
-        chompBytes(offset);
-      } catch (IOException e) {
-        //this is assumed to be recoverable with a seek -or more likely to fail
-        LOG.debug("while chomping ",e);
-      }
-      if (targetPos - pos == 0) {
-        LOG.trace("chomping successful");
-        return;
-      }
-      LOG.trace("chomping failed");
-    } else {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Seek is beyond buffer size of " + bufferSize);
-      }
-    }
-
-    innerClose("seeking to " + targetPos);
-    fillBuffer(targetPos);
-  }
-
-  /**
-   * Fill the buffer from the target position
-   * If the target position == current position, the
-   * read still goes ahead; this is a way of handling partial read failures
-   * @param targetPos target position
-   * @throws IOException IO problems on the read
-   */
-  private void fillBuffer(long targetPos) throws IOException {
-    long length = targetPos + bufferSize;
-    SwiftUtils.debug(LOG, "Fetching %d bytes starting at %d", length, targetPos);
-    HttpBodyContent blob = nativeStore.getObject(path, targetPos, length);
-    httpStream = blob.getInputStream();
-    updateStartOfBufferPosition(targetPos, blob.getContentLength());
-  }
-
-  @Override
-  public synchronized long getPos() throws IOException {
-    return pos;
-  }
-
-  /**
-   * This FS doesn't explicitly support multiple data sources, so
-   * return false here.
-   * @param targetPos the desired target position
-   * @return true if a new source of the data has been set up
-   * as the source of future reads
-   * @throws IOException IO problems
-   */
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java
deleted file mode 100644
index ac49a8a649535..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftException;
-import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Output stream, buffers data on local disk.
- * Writes to Swift on the close() method, unless the
- * file is significantly large that it is being written as partitions.
- * In this case, the first partition is written on the first write that puts
- * data over the partition, as may later writes. The close() then causes
- * the final partition to be written, along with a partition manifest.
- */
-class SwiftNativeOutputStream extends OutputStream {
-  public static final int ATTEMPT_LIMIT = 3;
-  private long filePartSize;
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftNativeOutputStream.class);
-  private Configuration conf;
-  private String key;
-  private File backupFile;
-  private OutputStream backupStream;
-  private SwiftNativeFileSystemStore nativeStore;
-  private boolean closed;
-  private int partNumber;
-  private long blockOffset;
-  private long bytesWritten;
-  private long bytesUploaded;
-  private boolean partUpload = false;
-  final byte[] oneByte = new byte[1];
-
-  /**
-   * Create an output stream
-   * @param conf configuration to use
-   * @param nativeStore native store to write through
-   * @param key the key to write
-   * @param partSizeKB the partition size
-   * @throws IOException
-   */
-  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
-  public SwiftNativeOutputStream(Configuration conf,
-                                 SwiftNativeFileSystemStore nativeStore,
-                                 String key,
-                                 long partSizeKB) throws IOException {
-    this.conf = conf;
-    this.key = key;
-    this.backupFile = newBackupFile();
-    this.nativeStore = nativeStore;
-    this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile));
-    this.partNumber = 1;
-    this.blockOffset = 0;
-    this.filePartSize = 1024L * partSizeKB;
-  }
-
-  private File newBackupFile() throws IOException {
-    File dir = new File(conf.get("hadoop.tmp.dir"));
-    if (!dir.mkdirs() && !dir.exists()) {
-      throw new SwiftException("Cannot create Swift buffer directory: " + dir);
-    }
-    File result = File.createTempFile("output-", ".tmp", dir);
-    result.deleteOnExit();
-    return result;
-  }
-
-  /**
-   * Flush the local backing stream.
-   * This does not trigger a flush of data to the remote blobstore.
-   * @throws IOException
-   */
-  @Override
-  public void flush() throws IOException {
-    backupStream.flush();
-  }
-
-  /**
-   * check that the output stream is open
-   *
-   * @throws SwiftException if it is not
-   */
-  private synchronized void verifyOpen() throws SwiftException {
-    if (closed) {
-      throw new SwiftConnectionClosedException();
-    }
-  }
-
-  /**
-   * Close the stream. This will trigger the upload of all locally cached
-   * data to the remote blobstore.
-   * @throws IOException IO problems uploading the data.
-   */
-  @Override
-  public synchronized void close() throws IOException {
-    if (closed) {
-      return;
-    }
-
-    try {
-      closed = true;
-      //formally declare as closed.
-      backupStream.close();
-      backupStream = null;
-      Path keypath = new Path(key);
-      if (partUpload) {
-        partUpload(true);
-        nativeStore.createManifestForPartUpload(keypath);
-      } else {
-        uploadOnClose(keypath);
-      }
-    } finally {
-      delete(backupFile);
-      backupFile = null;
-    }
-    assert backupStream == null: "backup stream has been reopened";
-  }
-
-  /**
-   * Upload a file when closed, either in one go, or, if the file is
-   * already partitioned, by uploading the remaining partition and a manifest.
-   * @param keypath key as a path
-   * @throws IOException IO Problems
-   */
-  private void uploadOnClose(Path keypath) throws IOException {
-    boolean uploadSuccess = false;
-    int attempt = 0;
-    while (!uploadSuccess) {
-      try {
-        ++attempt;
-        bytesUploaded += uploadFileAttempt(keypath, attempt);
-        uploadSuccess = true;
-      } catch (IOException e) {
-        LOG.info("Upload failed " + e, e);
-        if (attempt > ATTEMPT_LIMIT) {
-          throw e;
-        }
-      }
-    }
-}
-
-  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
-  private long uploadFileAttempt(Path keypath, int attempt) throws IOException {
-    long uploadLen = backupFile.length();
-    SwiftUtils.debug(LOG, "Closing write of file %s;" +
-                          " localfile=%s of length %d - attempt %d",
-                     key,
-                     backupFile,
-                     uploadLen,
-                     attempt);
-
-    nativeStore.uploadFile(keypath,
-                           new FileInputStream(backupFile),
-                           uploadLen);
-    return uploadLen;
-  }
-
-  @Override
-  protected void finalize() throws Throwable {
-    if(!closed) {
-      LOG.warn("stream not closed");
-    }
-    if (backupFile != null) {
-      LOG.warn("Leaking backing file " + backupFile);
-    }
-  }
-
-  private void delete(File file) {
-    if (file != null) {
-      SwiftUtils.debug(LOG, "deleting %s", file);
-      if (!file.delete()) {
-        LOG.warn("Could not delete " + file);
-      }
-    }
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    //insert to a one byte array
-    oneByte[0] = (byte) b;
-    //then delegate to the array writing routine
-    write(oneByte, 0, 1);
-  }
-
-  @Override
-  public synchronized void write(byte[] buffer, int offset, int len) throws
-                                                                     IOException {
-    //validate args
-    if (offset < 0 || len < 0 || (offset + len) > buffer.length) {
-      throw new IndexOutOfBoundsException("Invalid offset/length for write");
-    }
-    //validate the output stream
-    verifyOpen();
-    SwiftUtils.debug(LOG, " write(offset=%d, len=%d)", offset, len);
-
-    // if the size of file is greater than the partition limit
-    while (blockOffset + len >= filePartSize) {
-      // - then partition the blob and upload as many partitions
-      // are needed.
-      //how many bytes to write for this partition.
-      int subWriteLen = (int) (filePartSize - blockOffset);
-      if (subWriteLen < 0 || subWriteLen > len) {
-        throw new SwiftInternalStateException("Invalid subwrite len: "
-                                              + subWriteLen
-                                              + " -buffer len: " + len);
-      }
-      writeToBackupStream(buffer, offset, subWriteLen);
-      //move the offset along and length down
-      offset += subWriteLen;
-      len -= subWriteLen;
-      //now upload the partition that has just been filled up
-      // (this also sets blockOffset=0)
-      partUpload(false);
-    }
-    //any remaining data is now written
-    writeToBackupStream(buffer, offset, len);
-  }
-
-  /**
-   * Write to the backup stream.
-   * Guarantees:
-   * <ol>
-   *   <li>backupStream is open</li>
-   *   <li>blockOffset + len &lt; filePartSize</li>
-   * </ol>
-   * @param buffer buffer to write
-   * @param offset offset in buffer
-   * @param len length of write.
-   * @throws IOException backup stream write failing
-   */
-  private void writeToBackupStream(byte[] buffer, int offset, int len) throws
-                                                                       IOException {
-    assert len >= 0  : "remainder to write is negative";
-    SwiftUtils.debug(LOG," writeToBackupStream(offset=%d, len=%d)", offset, len);
-    if (len == 0) {
-      //no remainder -downgrade to no-op
-      return;
-    }
-
-    //write the new data out to the backup stream
-    backupStream.write(buffer, offset, len);
-    //increment the counters
-    blockOffset += len;
-    bytesWritten += len;
-  }
-
-  /**
-   * Upload a single partition. This deletes the local backing-file,
-   * and re-opens it to create a new one.
-   * @param closingUpload is this the final upload of an upload
-   * @throws IOException on IO problems
-   */
-  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
-  private void partUpload(boolean closingUpload) throws IOException {
-    if (backupStream != null) {
-      backupStream.close();
-    }
-
-    if (closingUpload && partUpload && backupFile.length() == 0) {
-      //skipping the upload if
-      // - it is close time
-      // - the final partition is 0 bytes long
-      // - one part has already been written
-      SwiftUtils.debug(LOG, "skipping upload of 0 byte final partition");
-      delete(backupFile);
-    } else {
-      partUpload = true;
-      boolean uploadSuccess = false;
-      int attempt = 0;
-      while(!uploadSuccess) {
-        try {
-          ++attempt;
-          bytesUploaded += uploadFilePartAttempt(attempt);
-          uploadSuccess = true;
-        } catch (IOException e) {
-          LOG.info("Upload failed " + e, e);
-          if (attempt > ATTEMPT_LIMIT) {
-            throw e;
-          }
-        }
-      }
-      delete(backupFile);
-      partNumber++;
-      blockOffset = 0;
-      if (!closingUpload) {
-        //if not the final upload, create a new output stream
-        backupFile = newBackupFile();
-        backupStream =
-          new BufferedOutputStream(new FileOutputStream(backupFile));
-      }
-    }
-  }
-
-  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
-  private long uploadFilePartAttempt(int attempt) throws IOException {
-    long uploadLen = backupFile.length();
-    SwiftUtils.debug(LOG, "Uploading part %d of file %s;" +
-                          " localfile=%s of length %d  - attempt %d",
-                     partNumber,
-                     key,
-                     backupFile,
-                     uploadLen,
-                     attempt);
-    nativeStore.uploadFilePart(new Path(key),
-                               partNumber,
-                               new FileInputStream(backupFile),
-                               uploadLen);
-    return uploadLen;
-  }
-
-  /**
-   * Get the file partition size
-   * @return the partition size
-   */
-  long getFilePartSize() {
-    return filePartSize;
-  }
-
-  /**
-   * Query the number of partitions written
-   * This is intended for testing
-   * @return the of partitions already written to the remote FS
-   */
-  synchronized int getPartitionsWritten() {
-    return partNumber - 1;
-  }
-
-  /**
-   * Get the number of bytes written to the output stream.
-   * This should always be less than or equal to bytesUploaded.
-   * @return the number of bytes written to this stream
-   */
-  long getBytesWritten() {
-    return bytesWritten;
-  }
-
-  /**
-   * Get the number of bytes uploaded to remote Swift cluster.
-   * bytesUploaded -bytesWritten = the number of bytes left to upload
-   * @return the number of bytes written to the remote endpoint
-   */
-  long getBytesUploaded() {
-    return bytesUploaded;
-  }
-
-  @Override
-  public String toString() {
-    return "SwiftNativeOutputStream{" +
-           ", key='" + key + '\'' +
-           ", backupFile=" + backupFile +
-           ", closed=" + closed +
-           ", filePartSize=" + filePartSize +
-           ", partNumber=" + partNumber +
-           ", blockOffset=" + blockOffset +
-           ", partUpload=" + partUpload +
-           ", nativeStore=" + nativeStore +
-           ", bytesWritten=" + bytesWritten +
-           ", bytesUploaded=" + bytesUploaded +
-           '}';
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
deleted file mode 100644
index ca8adc6244c73..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.snative;
-
-import java.util.Date;
-
-/**
- * Java mapping of Swift JSON file status.
- * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
- * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
- */
-
-class SwiftObjectFileStatus {
-  private long bytes;
-  private String content_type;
-  private String hash;
-  private Date last_modified;
-  private String name;
-  private String subdir;
-
-  SwiftObjectFileStatus() {
-  }
-
-  SwiftObjectFileStatus(long bytes, String content_type, String hash,
-                        Date last_modified, String name) {
-    this.bytes = bytes;
-    this.content_type = content_type;
-    this.hash = hash;
-    this.last_modified = last_modified;
-    this.name = name;
-  }
-
-  public long getBytes() {
-    return bytes;
-  }
-
-  public void setBytes(long bytes) {
-    this.bytes = bytes;
-  }
-
-  public String getContent_type() {
-    return content_type;
-  }
-
-  public void setContent_type(String content_type) {
-    this.content_type = content_type;
-  }
-
-  public String getHash() {
-    return hash;
-  }
-
-  public void setHash(String hash) {
-    this.hash = hash;
-  }
-
-  public Date getLast_modified() {
-    return last_modified;
-  }
-
-  public void setLast_modified(Date last_modified) {
-    this.last_modified = last_modified;
-  }
-
-  public String getName() {
-    return pathToRootPath(name);
-  }
-
-  public void setName(String name) {
-    this.name = name;
-  }
-
-  public String getSubdir() {
-    return pathToRootPath(subdir);
-  }
-
-  public void setSubdir(String subdir) {
-    this.subdir = subdir;
-  }
-
-  /**
-   * If path doesn't starts with '/'
-   * method will concat '/'
-   *
-   * @param path specified path
-   * @return root path string
-   */
-  private String pathToRootPath(String path) {
-    if (path == null) {
-      return null;
-    }
-
-    if (path.startsWith("/")) {
-      return path;
-    }
-
-    return "/".concat(path);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java
deleted file mode 100644
index 3071f946824c6..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-public class Duration {
-
-  private final long started;
-  private long finished;
-
-  public Duration() {
-    started = time();
-    finished = started;
-  }
-
-  private long time() {
-    return System.currentTimeMillis();
-  }
-
-  public void finished() {
-    finished = time();
-  }
-
-  public String getDurationString() {
-    return humanTime(value());
-  }
-
-  public static String humanTime(long time) {
-    long seconds = (time / 1000);
-    long minutes = (seconds / 60);
-    return String.format("%d:%02d:%03d", minutes, seconds % 60, time % 1000);
-  }
-
-  @Override
-  public String toString() {
-    return getDurationString();
-  }
-
-  public long value() {
-    return finished -started;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java
deleted file mode 100644
index 734cf8b6dc100..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-/**
- * Build ongoing statistics from duration data
- */
-public class DurationStats {
-
-  final String operation;
-  int n;
-  long sum;
-  long min;
-  long max;
-  double mean, m2;
-
-  /**
-   * Construct statistics for a given operation.
-   * @param operation operation
-   */
-  public DurationStats(String operation) {
-    this.operation = operation;
-    reset();
-  }
-
-  /**
-   * construct from another stats entry;
-   * all value are copied.
-   * @param that the source statistics
-   */
-  public DurationStats(DurationStats that) {
-    operation = that.operation;
-    n = that.n;
-    sum = that.sum;
-    min = that.min;
-    max = that.max;
-    mean = that.mean;
-    m2 = that.m2;
-  }
-
-  /**
-   * Add a duration
-   * @param duration the new duration
-   */
-  public void add(Duration duration) {
-    add(duration.value());
-  }
-
-  /**
-   * Add a number
-   * @param x the number
-   */
-  public void add(long x) {
-    n++;
-    sum += x;
-    double delta = x - mean;
-    mean += delta / n;
-    m2 += delta * (x - mean);
-    if (x < min) {
-      min = x;
-    }
-    if (x > max) {
-      max = x;
-    }
-  }
-
-  /**
-   * Reset the data
-   */
-  public void reset() {
-    n = 0;
-    sum = 0;
-    sum = 0;
-    min = 10000000;
-    max = 0;
-    mean = 0;
-    m2 = 0;
-  }
-
-  /**
-   * Get the number of entries sampled
-   * @return the number of durations added
-   */
-  public int getCount() {
-    return n;
-  }
-
-  /**
-   * Get the sum of all durations
-   * @return all the durations
-   */
-  public long getSum() {
-    return sum;
-  }
-
-  /**
-   * Get the arithmetic mean of the aggregate statistics
-   * @return the arithmetic mean
-   */
-  public double getArithmeticMean() {
-    return mean;
-  }
-
-  /**
-   * Variance, sigma^2
-   * @return variance, or, if no samples are there, 0.
-   */
-  public double getVariance() {
-    return n > 0 ? (m2 / (n - 1)) : 0;
-  }
-
-  /**
-   * Get the std deviation, sigma
-   * @return the stddev, 0 may mean there are no samples.
-   */
-  public double getDeviation() {
-    double variance = getVariance();
-    return (variance > 0) ? Math.sqrt(variance) : 0;
-  }
-
-  /**
-   * Covert to a useful string
-   * @return a human readable summary
-   */
-  @Override
-  public String toString() {
-    return String.format(
-      "%s count=%d total=%.3fs mean=%.3fs stddev=%.3fs min=%.3fs max=%.3fs",
-      operation,
-      n,
-      sum / 1000.0,
-      mean / 1000.0,
-      getDeviation() / 1000000.0,
-      min / 1000.0,
-      max / 1000.0);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java
deleted file mode 100644
index 58f8f0b641de5..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.swift.util;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Build a duration stats table to which you can add statistics.
- * Designed to be multithreaded
- */
-public class DurationStatsTable {
-
-  private Map<String,DurationStats> statsTable
-    = new HashMap<String, DurationStats>(6);
-
-  /**
-   * Add an operation
-   * @param operation operation name
-   * @param duration duration
-   */
-  public void add(String operation, Duration duration, boolean success) {
-    DurationStats durationStats;
-    String key = operation;
-    if (!success) {
-      key += "-FAIL";
-    }
-    synchronized (this) {
-      durationStats = statsTable.get(key);
-      if (durationStats == null) {
-        durationStats = new DurationStats(key);
-        statsTable.put(key, durationStats);
-      }
-    }
-    synchronized (durationStats) {
-      durationStats.add(duration);
-    }
-  }
-
-  /**
-   * Get the current duration statistics
-   * @return a snapshot of the statistics
-   */
-   public synchronized List<DurationStats> getDurationStatistics() {
-     List<DurationStats> results = new ArrayList<DurationStats>(statsTable.size());
-     for (DurationStats stat: statsTable.values()) {
-       results.add(new DurationStats(stat));
-     }
-     return results;
-   }
-
-  /**
-   * reset the values of the statistics. This doesn't delete them, merely zeroes them.
-   */
-  public synchronized void reset() {
-    for (DurationStats stat : statsTable.values()) {
-      stat.reset();
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java
deleted file mode 100644
index 1cc340d83d9b4..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/HttpResponseUtils.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.util.EncodingUtils;
-
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.HEADER_CONTENT_LENGTH;
-
-/**
- * Utility class for parsing HttpResponse. This class is implemented like
- * {@code org.apache.commons.httpclient.HttpMethodBase.java} in httpclient 3.x.
- */
-public abstract class HttpResponseUtils {
-
-  /**
-   * Returns the response body of the HTTPResponse, if any, as an array of bytes.
-   * If response body is not available or cannot be read, returns <tt>null</tt>
-   *
-   * Note: This will cause the entire response body to be buffered in memory. A
-   * malicious server may easily exhaust all the VM memory. It is strongly
-   * recommended, to use getResponseAsStream if the content length of the
-   * response is unknown or reasonably large.
-   *
-   * @param resp HttpResponse
-   * @return The response body
-   * @throws IOException If an I/O (transport) problem occurs while obtaining
-   * the response body.
-   */
-  public static byte[] getResponseBody(HttpResponse resp) throws IOException {
-    try(InputStream instream = resp.getEntity().getContent()) {
-      if (instream != null) {
-        long contentLength = resp.getEntity().getContentLength();
-        if (contentLength > Integer.MAX_VALUE) {
-          //guard integer cast from overflow
-          throw new IOException("Content too large to be buffered: "
-              + contentLength +" bytes");
-        }
-        ByteArrayOutputStream outstream = new ByteArrayOutputStream(
-            contentLength > 0 ? (int) contentLength : 4*1024);
-        byte[] buffer = new byte[4096];
-        int len;
-        while ((len = instream.read(buffer)) > 0) {
-          outstream.write(buffer, 0, len);
-        }
-        outstream.close();
-        return outstream.toByteArray();
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Returns the response body of the HTTPResponse, if any, as a {@link String}.
-   * If response body is not available or cannot be read, returns <tt>null</tt>
-   * The string conversion on the data is done using UTF-8.
-   *
-   * Note: This will cause the entire response body to be buffered in memory. A
-   * malicious server may easily exhaust all the VM memory. It is strongly
-   * recommended, to use getResponseAsStream if the content length of the
-   * response is unknown or reasonably large.
-   *
-   * @param resp HttpResponse
-   * @return The response body.
-   * @throws IOException If an I/O (transport) problem occurs while obtaining
-   * the response body.
-   */
-  public static String getResponseBodyAsString(HttpResponse resp)
-      throws IOException {
-    byte[] rawdata = getResponseBody(resp);
-    if (rawdata != null) {
-      return EncodingUtils.getString(rawdata, "UTF-8");
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Return the length (in bytes) of the response body, as specified in a
-   * <tt>Content-Length</tt> header.
-   *
-   * <p>
-   * Return <tt>-1</tt> when the content-length is unknown.
-   * </p>
-   *
-   * @param resp HttpResponse
-   * @return content length, if <tt>Content-Length</tt> header is available.
-   *          <tt>0</tt> indicates that the request has no body.
-   *          If <tt>Content-Length</tt> header is not present, the method
-   *          returns <tt>-1</tt>.
-   */
-  public static long getContentLength(HttpResponse resp) {
-    Header header = resp.getFirstHeader(HEADER_CONTENT_LENGTH);
-    if (header == null) {
-      return -1;
-    } else {
-      return Long.parseLong(header.getValue());
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java
deleted file mode 100644
index fee7e7f5697e3..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.type.CollectionType;
-import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.io.Writer;
-
-
-public class JSONUtil {
-  private static ObjectMapper jsonMapper = new ObjectMapper();
-
-  /**
-   * Private constructor.
-   */
-  private JSONUtil() {
-  }
-
-  /**
-   * Converting object to JSON string. If errors appears throw
-   * MeshinException runtime exception.
-   *
-   * @param object The object to convert.
-   * @return The JSON string representation.
-   * @throws IOException IO issues
-   * @throws SwiftJsonMarshallingException failure to generate JSON
-   */
-  public static String toJSON(Object object) throws
-                                             IOException {
-    Writer json = new StringWriter();
-    try {
-      jsonMapper.writeValue(json, object);
-      return json.toString();
-    } catch (JsonGenerationException | JsonMappingException e) {
-      throw new SwiftJsonMarshallingException(e.toString(), e);
-    }
-  }
-
-  /**
-   * Convert string representation to object. If errors appears throw
-   * Exception runtime exception.
-   *
-   * @param value The JSON string.
-   * @param klazz The class to convert.
-   * @return The Object of the given class.
-   */
-  public static <T> T toObject(String value, Class<T> klazz) throws
-                                                             IOException {
-    try {
-      return jsonMapper.readValue(value, klazz);
-    } catch (JsonGenerationException e) {
-      throw new SwiftJsonMarshallingException(e.toString()
-                                              + " source: " + value,
-                                              e);
-    } catch (JsonMappingException e) {
-      throw new SwiftJsonMarshallingException(e.toString()
-                                              + " source: " + value,
-                                              e);
-    }
-  }
-
-  /**
-   * @param value         json string
-   * @param typeReference class type reference
-   * @param <T>           type
-   * @return deserialized  T object
-   */
-  @SuppressWarnings("unchecked")
-  public static <T> T toObject(String value,
-                               final TypeReference<T> typeReference)
-            throws IOException {
-    try {
-      return (T)jsonMapper.readValue(value, typeReference);
-    } catch (JsonGenerationException | JsonMappingException e) {
-      throw new SwiftJsonMarshallingException("Error generating response", e);
-    }
-  }
-
-  /**
-   * @param value          json string
-   * @param collectionType class describing how to deserialize collection of objects
-   * @param <T>            type
-   * @return deserialized  T object
-   */
-  @SuppressWarnings("unchecked")
-  public static <T> T toObject(String value,
-                               final CollectionType collectionType)
-              throws IOException {
-    try {
-      return (T)jsonMapper.readValue(value, collectionType);
-    } catch (JsonGenerationException | JsonMappingException e) {
-      throw new SwiftJsonMarshallingException(e.toString()
-                                              + " source: " + value,
-                                              e);
-    }
-  }
-
-  public static ObjectMapper getJsonMapper() {
-    return jsonMapper;
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java
deleted file mode 100644
index 791509a9e0372..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.apache.hadoop.fs.swift.http.RestClientBindings;
-
-import java.net.URI;
-import java.util.regex.Pattern;
-
-/**
- * Swift hierarchy mapping of (container, path)
- */
-public final class SwiftObjectPath {
-  private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/");
-
-  /**
-   * Swift container
-   */
-  private final String container;
-
-  /**
-   * swift object
-   */
-  private final String object;
-
-  private final String uriPath;
-
-  /**
-   * Build an instance from a (host, object) pair
-   *
-   * @param container container name
-   * @param object    object ref underneath the container
-   */
-  public SwiftObjectPath(String container, String object) {
-
-    if (object == null) {
-      throw new IllegalArgumentException("object name can't be null");
-    }
-
-    this.container = container;
-    this.object = URI.create(object).getPath();
-    uriPath = buildUriPath();
-  }
-
-  public String getContainer() {
-    return container;
-  }
-
-  public String getObject() {
-    return object;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (!(o instanceof SwiftObjectPath)) return false;
-    final SwiftObjectPath that = (SwiftObjectPath) o;
-    return this.toUriPath().equals(that.toUriPath());
-  }
-
-  @Override
-  public int hashCode() {
-    int result = container.hashCode();
-    result = 31 * result + object.hashCode();
-    return result;
-  }
-
-  private String buildUriPath() {
-    return SwiftUtils.joinPaths(container, object);
-  }
-
-  public String toUriPath() {
-    return uriPath;
-  }
-
-  @Override
-  public String toString() {
-    return toUriPath();
-  }
-
-  /**
-   * Test for the object matching a path, ignoring the container
-   * value.
-   *
-   * @param path path string
-   * @return true iff the object's name matches the path
-   */
-  public boolean objectMatches(String path) {
-    return object.equals(path);
-  }
-
-
-  /**
-   * Query to see if the possibleChild object is a child path of this.
-   * object.
-   *
-   * The test is done by probing for the path of the this object being
-   * at the start of the second -with a trailing slash, and both
-   * containers being equal
-   *
-   * @param possibleChild possible child dir
-   * @return true iff the possibleChild is under this object
-   */
-  public boolean isEqualToOrParentOf(SwiftObjectPath possibleChild) {
-    String origPath = toUriPath();
-    String path = origPath;
-    if (!path.endsWith("/")) {
-      path = path + "/";
-    }
-    String childPath = possibleChild.toUriPath();
-    return childPath.equals(origPath) || childPath.startsWith(path);
-  }
-
-  /**
-   * Create a path tuple of (container, path), where the container is
-   * chosen from the host of the URI.
-   *
-   * @param uri  uri to start from
-   * @param path path underneath
-   * @return a new instance.
-   * @throws SwiftConfigurationException if the URI host doesn't parse into
-   *                                     container.service
-   */
-  public static SwiftObjectPath fromPath(URI uri,
-                                         Path path)
-          throws SwiftConfigurationException {
-    return fromPath(uri, path, false);
-  }
-
-  /**
-   * Create a path tuple of (container, path), where the container is
-   * chosen from the host of the URI.
-   * A trailing slash can be added to the path. This is the point where
-   * these /-es need to be appended, because when you construct a {@link Path}
-   * instance, {@link Path#normalizePath(String, String)} is called
-   * -which strips off any trailing slash.
-   *
-   * @param uri              uri to start from
-   * @param path             path underneath
-   * @param addTrailingSlash should a trailing slash be added if there isn't one.
-   * @return a new instance.
-   * @throws SwiftConfigurationException if the URI host doesn't parse into
-   *                                     container.service
-   */
-  public static SwiftObjectPath fromPath(URI uri,
-                                         Path path,
-                                         boolean addTrailingSlash)
-          throws SwiftConfigurationException {
-
-    String url =
-            path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), "");
-    //add a trailing slash if needed
-    if (addTrailingSlash && !url.endsWith("/")) {
-      url += "/";
-    }
-
-    String container = uri.getHost();
-    if (container == null) {
-      //no container, not good: replace with ""
-      container = "";
-    } else if (container.contains(".")) {
-      //its a container.service URI. Strip the container
-      container = RestClientBindings.extractContainerName(container);
-    }
-    return new SwiftObjectPath(container, url);
-  }
-
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
deleted file mode 100644
index 2e3abce251a2d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
+++ /dev/null
@@ -1,547 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.junit.internal.AssumptionViolatedException;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Properties;
-
-/**
- * Utilities used across test cases
- */
-public class SwiftTestUtils extends org.junit.Assert {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SwiftTestUtils.class);
-
-  public static final String TEST_FS_SWIFT = "test.fs.swift.name";
-  public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
-
-  /**
-   * Get the test URI
-   * @param conf configuration
-   * @throws SwiftConfigurationException missing parameter or bad URI
-   */
-  public static URI getServiceURI(Configuration conf) throws
-                                                      SwiftConfigurationException {
-    String instance = conf.get(TEST_FS_SWIFT);
-    if (instance == null) {
-      throw new SwiftConfigurationException(
-        "Missing configuration entry " + TEST_FS_SWIFT);
-    }
-    try {
-      return new URI(instance);
-    } catch (URISyntaxException e) {
-      throw new SwiftConfigurationException("Bad URI: " + instance);
-    }
-  }
-
-  public static boolean hasServiceURI(Configuration conf) {
-    String instance = conf.get(TEST_FS_SWIFT);
-    return instance != null;
-  }
-
-  /**
-   * Assert that a property in the property set matches the expected value
-   * @param props property set
-   * @param key property name
-   * @param expected expected value. If null, the property must not be in the set
-   */
-  public static void assertPropertyEquals(Properties props,
-                                          String key,
-                                          String expected) {
-    String val = props.getProperty(key);
-    if (expected == null) {
-      assertNull("Non null property " + key + " = " + val, val);
-    } else {
-      assertEquals("property " + key + " = " + val,
-                          expected,
-                          val);
-    }
-  }
-
-  /**
-   *
-   * Write a file and read it in, validating the result. Optional flags control
-   * whether file overwrite operations should be enabled, and whether the
-   * file should be deleted afterwards.
-   *
-   * If there is a mismatch between what was written and what was expected,
-   * a small range of bytes either side of the first error are logged to aid
-   * diagnosing what problem occurred -whether it was a previous file
-   * or a corrupting of the current file. This assumes that two
-   * sequential runs to the same path use datasets with different character
-   * moduli.
-   *
-   * @param fs filesystem
-   * @param path path to write to
-   * @param len length of data
-   * @param overwrite should the create option allow overwrites?
-   * @param delete should the file be deleted afterwards? -with a verification
-   * that it worked. Deletion is not attempted if an assertion has failed
-   * earlier -it is not in a <code>finally{}</code> block.
-   * @throws IOException IO problems
-   */
-  public static void writeAndRead(FileSystem fs,
-                                  Path path,
-                                  byte[] src,
-                                  int len,
-                                  int blocksize,
-                                  boolean overwrite,
-                                  boolean delete) throws IOException {
-    fs.mkdirs(path.getParent());
-
-    writeDataset(fs, path, src, len, blocksize, overwrite);
-
-    byte[] dest = readDataset(fs, path, len);
-
-    compareByteArrays(src, dest, len);
-
-    if (delete) {
-      boolean deleted = fs.delete(path, false);
-      assertTrue("Deleted", deleted);
-      assertPathDoesNotExist(fs, "Cleanup failed", path);
-    }
-  }
-
-  /**
-   * Write a file.
-   * Optional flags control
-   * whether file overwrite operations should be enabled
-   * @param fs filesystem
-   * @param path path to write to
-   * @param len length of data
-   * @param overwrite should the create option allow overwrites?
-   * @throws IOException IO problems
-   */
-  public static void writeDataset(FileSystem fs,
-                                   Path path,
-                                   byte[] src,
-                                   int len,
-                                   int blocksize,
-                                   boolean overwrite) throws IOException {
-    assertTrue(
-      "Not enough data in source array to write " + len + " bytes",
-      src.length >= len);
-    FSDataOutputStream out = fs.create(path,
-                                       overwrite,
-                                       fs.getConf()
-                                         .getInt(IO_FILE_BUFFER_SIZE,
-                                                 4096),
-                                       (short) 1,
-                                       blocksize);
-    out.write(src, 0, len);
-    out.close();
-    assertFileHasLength(fs, path, len);
-  }
-
-  /**
-   * Read the file and convert to a byte dataset
-   * @param fs filesystem
-   * @param path path to read from
-   * @param len length of data to read
-   * @return the bytes
-   * @throws IOException IO problems
-   */
-  public static byte[] readDataset(FileSystem fs, Path path, int len)
-      throws IOException {
-    FSDataInputStream in = fs.open(path);
-    byte[] dest = new byte[len];
-    try {
-      in.readFully(0, dest);
-    } finally {
-      in.close();
-    }
-    return dest;
-  }
-
-  /**
-   * Assert that the array src[0..len] and dest[] are equal
-   * @param src source data
-   * @param dest actual
-   * @param len length of bytes to compare
-   */
-  public static void compareByteArrays(byte[] src,
-                                       byte[] dest,
-                                       int len) {
-    assertEquals("Number of bytes read != number written",
-                        len, dest.length);
-    int errors = 0;
-    int first_error_byte = -1;
-    for (int i = 0; i < len; i++) {
-      if (src[i] != dest[i]) {
-        if (errors == 0) {
-          first_error_byte = i;
-        }
-        errors++;
-      }
-    }
-
-    if (errors > 0) {
-      String message = String.format(" %d errors in file of length %d",
-                                     errors, len);
-      LOG.warn(message);
-      // the range either side of the first error to print
-      // this is a purely arbitrary number, to aid user debugging
-      final int overlap = 10;
-      for (int i = Math.max(0, first_error_byte - overlap);
-           i < Math.min(first_error_byte + overlap, len);
-           i++) {
-        byte actual = dest[i];
-        byte expected = src[i];
-        String letter = toChar(actual);
-        String line = String.format("[%04d] %2x %s%n", i, actual, letter);
-        if (expected != actual) {
-          line = String.format("[%04d] %2x %s -expected %2x %s%n",
-                               i,
-                               actual,
-                               letter,
-                               expected,
-                               toChar(expected));
-        }
-        LOG.warn(line);
-      }
-      fail(message);
-    }
-  }
-
-  /**
-   * Convert a byte to a character for printing. If the
-   * byte value is &lt; 32 -and hence unprintable- the byte is
-   * returned as a two digit hex value
-   * @param b byte
-   * @return the printable character string
-   */
-  public static String toChar(byte b) {
-    if (b >= 0x20) {
-      return Character.toString((char) b);
-    } else {
-      return String.format("%02x", b);
-    }
-  }
-
-  public static String toChar(byte[] buffer) {
-    StringBuilder builder = new StringBuilder(buffer.length);
-    for (byte b : buffer) {
-      builder.append(toChar(b));
-    }
-    return builder.toString();
-  }
-
-  public static byte[] toAsciiByteArray(String s) {
-    char[] chars = s.toCharArray();
-    int len = chars.length;
-    byte[] buffer = new byte[len];
-    for (int i = 0; i < len; i++) {
-      buffer[i] = (byte) (chars[i] & 0xff);
-    }
-    return buffer;
-  }
-
-  public static void cleanupInTeardown(FileSystem fileSystem,
-                                       String cleanupPath) {
-    cleanup("TEARDOWN", fileSystem, cleanupPath);
-  }
-
-  public static void cleanup(String action,
-                             FileSystem fileSystem,
-                             String cleanupPath) {
-    noteAction(action);
-    try {
-      if (fileSystem != null) {
-        fileSystem.delete(fileSystem.makeQualified(new Path(cleanupPath)),
-                          true);
-      }
-    } catch (Exception e) {
-      LOG.error("Error deleting in "+ action + " - "  + cleanupPath + ": " + e, e);
-    }
-  }
-
-  public static void noteAction(String action) {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("==============  "+ action +" =============");
-    }
-  }
-
-  /**
-   * downgrade a failure to a message and a warning, then an
-   * exception for the Junit test runner to mark as failed
-   * @param message text message
-   * @param failure what failed
-   * @throws AssumptionViolatedException always
-   */
-  public static void downgrade(String message, Throwable failure) {
-    LOG.warn("Downgrading test " + message, failure);
-    AssumptionViolatedException ave =
-      new AssumptionViolatedException(failure, null);
-    throw ave;
-  }
-
-  /**
-   * report an overridden test as unsupported
-   * @param message message to use in the text
-   * @throws AssumptionViolatedException always
-   */
-  public static void unsupported(String message) {
-    throw new AssumptionViolatedException(message);
-  }
-
-  /**
-   * report a test has been skipped for some reason
-   * @param message message to use in the text
-   * @throws AssumptionViolatedException always
-   */
-  public static void skip(String message) {
-    throw new AssumptionViolatedException(message);
-  }
-
-
-  /**
-   * Make an assertion about the length of a file
-   * @param fs filesystem
-   * @param path path of the file
-   * @param expected expected length
-   * @throws IOException on File IO problems
-   */
-  public static void assertFileHasLength(FileSystem fs, Path path,
-                                         int expected) throws IOException {
-    FileStatus status = fs.getFileStatus(path);
-    assertEquals(
-      "Wrong file length of file " + path + " status: " + status,
-      expected,
-      status.getLen());
-  }
-
-  /**
-   * Assert that a path refers to a directory
-   * @param fs filesystem
-   * @param path path of the directory
-   * @throws IOException on File IO problems
-   */
-  public static void assertIsDirectory(FileSystem fs,
-                                       Path path) throws IOException {
-    FileStatus fileStatus = fs.getFileStatus(path);
-    assertIsDirectory(fileStatus);
-  }
-
-  /**
-   * Assert that a path refers to a directory
-   * @param fileStatus stats to check
-   */
-  public static void assertIsDirectory(FileStatus fileStatus) {
-    assertTrue("Should be a dir -but isn't: " + fileStatus,
-                      fileStatus.isDirectory());
-  }
-
-  /**
-   * Write the text to a file, returning the converted byte array
-   * for use in validating the round trip
-   * @param fs filesystem
-   * @param path path of file
-   * @param text text to write
-   * @param overwrite should the operation overwrite any existing file?
-   * @return the read bytes
-   * @throws IOException on IO problems
-   */
-  public static byte[] writeTextFile(FileSystem fs,
-                                   Path path,
-                                   String text,
-                                   boolean overwrite) throws IOException {
-    FSDataOutputStream stream = fs.create(path, overwrite);
-    byte[] bytes = new byte[0];
-    if (text != null) {
-      bytes = toAsciiByteArray(text);
-      stream.write(bytes);
-    }
-    stream.close();
-    return bytes;
-  }
-
-  /**
-   * Touch a file: fails if it is already there
-   * @param fs filesystem
-   * @param path path
-   * @throws IOException IO problems
-   */
-  public static void touch(FileSystem fs,
-                           Path path) throws IOException {
-    fs.delete(path, true);
-    writeTextFile(fs, path, null, false);
-  }
-
-  public static void assertDeleted(FileSystem fs,
-                                   Path file,
-                                   boolean recursive) throws IOException {
-    assertPathExists(fs, "about to be deleted file", file);
-    boolean deleted = fs.delete(file, recursive);
-    String dir = ls(fs, file.getParent());
-    assertTrue("Delete failed on " + file + ": " + dir, deleted);
-    assertPathDoesNotExist(fs, "Deleted file", file);
-  }
-
-  /**
-   * Read in "length" bytes, convert to an ascii string
-   * @param fs filesystem
-   * @param path path to read
-   * @param length #of bytes to read.
-   * @return the bytes read and converted to a string
-   * @throws IOException
-   */
-  public static String readBytesToString(FileSystem fs,
-                                  Path path,
-                                  int length) throws IOException {
-    FSDataInputStream in = fs.open(path);
-    try {
-      byte[] buf = new byte[length];
-      in.readFully(0, buf);
-      return toChar(buf);
-    } finally {
-      in.close();
-    }
-  }
-
-  public static String getDefaultWorkingDirectory() {
-    return "/user/" + System.getProperty("user.name");
-  }
-
-  public static String ls(FileSystem fileSystem, Path path) throws IOException {
-    return SwiftUtils.ls(fileSystem, path);
-  }
-
-  public static String dumpStats(String pathname, FileStatus[] stats) {
-    return pathname + SwiftUtils.fileStatsToString(stats,"\n");
-  }
-
-  /**
-   /**
-   * Assert that a file exists and whose {@link FileStatus} entry
-   * declares that this is a file and not a symlink or directory.
-   * @param fileSystem filesystem to resolve path against
-   * @param filename name of the file
-   * @throws IOException IO problems during file operations
-   */
-  public static void assertIsFile(FileSystem fileSystem, Path filename) throws
-                                                                 IOException {
-    assertPathExists(fileSystem, "Expected file", filename);
-    FileStatus status = fileSystem.getFileStatus(filename);
-    String fileInfo = filename + "  " + status;
-    assertFalse("File claims to be a directory " + fileInfo,
-                status.isDirectory());
-/* disabled for Hadoop v1 compatibility
-    assertFalse("File claims to be a symlink " + fileInfo,
-                       status.isSymlink());
-*/
-  }
-
-  /**
-   * Create a dataset for use in the tests; all data is in the range
-   * base to (base+modulo-1) inclusive
-   * @param len length of data
-   * @param base base of the data
-   * @param modulo the modulo
-   * @return the newly generated dataset
-   */
-  public static byte[] dataset(int len, int base, int modulo) {
-    byte[] dataset = new byte[len];
-    for (int i = 0; i < len; i++) {
-      dataset[i] = (byte) (base + (i % modulo));
-    }
-    return dataset;
-  }
-
-  /**
-   * Assert that a path exists -but make no assertions as to the
-   * type of that entry
-   *
-   * @param fileSystem filesystem to examine
-   * @param message message to include in the assertion failure message
-   * @param path path in the filesystem
-   * @throws IOException IO problems
-   */
-  public static void assertPathExists(FileSystem fileSystem, String message,
-                               Path path) throws IOException {
-    try {
-      fileSystem.getFileStatus(path);
-    } catch (FileNotFoundException e) {
-      //failure, report it
-      throw (IOException)new FileNotFoundException(message + ": not found "
-          + path + " in " + path.getParent() + ": " + e + " -- "
-           + ls(fileSystem, path.getParent())).initCause(e);
-    }
-  }
-
-  /**
-   * Assert that a path does not exist
-   *
-   * @param fileSystem filesystem to examine
-   * @param message message to include in the assertion failure message
-   * @param path path in the filesystem
-   * @throws IOException IO problems
-   */
-  public static void assertPathDoesNotExist(FileSystem fileSystem,
-                                            String message,
-                                            Path path) throws IOException {
-    try {
-      FileStatus status = fileSystem.getFileStatus(path);
-      fail(message + ": unexpectedly found " + path + " as  " + status);
-    } catch (FileNotFoundException expected) {
-      //this is expected
-
-    }
-  }
-
-
-  /**
-   * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
-   * @param fs filesystem
-   * @param dir directory to scan
-   * @param subdir full path to look for
-   * @throws IOException IO problems
-   */
-  public static void assertListStatusFinds(FileSystem fs,
-                                           Path dir,
-                                           Path subdir) throws IOException {
-    FileStatus[] stats = fs.listStatus(dir);
-    boolean found = false;
-    StringBuilder builder = new StringBuilder();
-    for (FileStatus stat : stats) {
-      builder.append(stat.toString()).append('\n');
-      if (stat.getPath().equals(subdir)) {
-        found = true;
-      }
-    }
-    assertTrue("Path " + subdir
-                      + " not found in directory " + dir + ":" + builder,
-                      found);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
deleted file mode 100644
index f218a80595a8a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.util;
-
-import org.slf4j.Logger;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-/**
- * Various utility classes for SwiftFS support
- */
-public final class SwiftUtils {
-
-  public static final String READ = "read(buffer, offset, length)";
-
-  /**
-   * Join two (non null) paths, inserting a forward slash between them
-   * if needed
-   *
-   * @param path1 first path
-   * @param path2 second path
-   * @return the combined path
-   */
-  public static String joinPaths(String path1, String path2) {
-    StringBuilder result =
-            new StringBuilder(path1.length() + path2.length() + 1);
-    result.append(path1);
-    boolean insertSlash = true;
-    if (path1.endsWith("/")) {
-      insertSlash = false;
-    } else if (path2.startsWith("/")) {
-      insertSlash = false;
-    }
-    if (insertSlash) {
-      result.append("/");
-    }
-    result.append(path2);
-    return result.toString();
-  }
-
-  /**
-   * This test contains the is-directory logic for Swift, so if
-   * changed there is only one place for it.
-   *
-   * @param fileStatus status to examine
-   * @return true if we consider this status to be representative of a
-   *         directory.
-   */
-  public static boolean isDirectory(FileStatus fileStatus) {
-    return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
-  }
-
-  /**
-   * Test for the entry being a file that is treated as if it is a
-   * directory
-   *
-   * @param fileStatus status
-   * @return true if it meets the rules for being a directory
-   */
-  public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
-    return fileStatus.getLen() == 0;
-  }
-
-  /**
-   * Predicate: Is a swift object referring to the root directory?
-   * @param swiftObject object to probe
-   * @return true iff the object refers to the root
-   */
-  public static boolean isRootDir(SwiftObjectPath swiftObject) {
-    return swiftObject.objectMatches("") || swiftObject.objectMatches("/");
-  }
-
-  /**
-   * Sprintf() to the log iff the log is at debug level. If the log
-   * is not at debug level, the printf operation is skipped, so
-   * no time is spent generating the string.
-   * @param log log to use
-   * @param text text message
-   * @param args args arguments to the print statement
-   */
-  public static void debug(Logger log, String text, Object... args) {
-    if (log.isDebugEnabled()) {
-      log.debug(String.format(text, args));
-    }
-  }
-
-  /**
-   * Log an exception (in text and trace) iff the log is at debug
-   * @param log Log to use
-   * @param text text message
-   * @param ex exception
-   */
-  public static void debugEx(Logger log, String text, Exception ex) {
-    if (log.isDebugEnabled()) {
-      log.debug(text + ex, ex);
-    }
-  }
-
-  /**
-   * Sprintf() to the log iff the log is at trace level. If the log
-   * is not at trace level, the printf operation is skipped, so
-   * no time is spent generating the string.
-   * @param log log to use
-   * @param text text message
-   * @param args args arguments to the print statement
-   */
-  public static void trace(Logger log, String text, Object... args) {
-    if (log.isTraceEnabled()) {
-      log.trace(String.format(text, args));
-    }
-  }
-
-  /**
-   * Given a partition number, calculate the partition value.
-   * This is used in the SwiftNativeOutputStream, and is placed
-   * here for tests to be able to calculate the filename of
-   * a partition.
-   * @param partNumber part number
-   * @return a string to use as the filename
-   */
-  public static String partitionFilenameFromNumber(int partNumber) {
-    return String.format("%06d", partNumber);
-  }
-
-  /**
-   * List a a path to string
-   * @param fileSystem filesystem
-   * @param path directory
-   * @return a listing of the filestatuses of elements in the directory, one
-   * to a line, preceded by the full path of the directory
-   * @throws IOException connectivity problems
-   */
-  public static String ls(FileSystem fileSystem, Path path) throws
-                                                            IOException {
-    if (path == null) {
-      //surfaces when someone calls getParent() on something at the top of the path
-      return "/";
-    }
-    FileStatus[] stats;
-    String pathtext = "ls " + path;
-    try {
-      stats = fileSystem.listStatus(path);
-    } catch (FileNotFoundException e) {
-      return pathtext + " -file not found";
-    } catch (IOException e) {
-      return pathtext + " -failed: " + e;
-    }
-    return pathtext + fileStatsToString(stats, "\n");
-  }
-
-  /**
-   * Take an array of filestatus and convert to a string (prefixed w/ a [01] counter
-   * @param stats array of stats
-   * @param separator separator after every entry
-   * @return a stringified set
-   */
-  public static String fileStatsToString(FileStatus[] stats, String separator) {
-    StringBuilder buf = new StringBuilder(stats.length * 128);
-    for (int i = 0; i < stats.length; i++) {
-      buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
-    }
-    return buf.toString();
-  }
-
-  /**
-   * Verify that the basic args to a read operation are valid;
-   * throws an exception if not -with meaningful text including
-   * @param buffer destination buffer
-   * @param off offset
-   * @param len number of bytes to read
-   * @throws NullPointerException null buffer
-   * @throws IndexOutOfBoundsException on any invalid range.
-   */
-  public static void validateReadArgs(byte[] buffer, int off, int len) {
-    if (buffer == null) {
-      throw new NullPointerException("Null byte array in"+ READ);
-    }
-    if (off < 0 ) {
-      throw new IndexOutOfBoundsException("Negative buffer offset "
-                                          + off
-                                          + " in " + READ);
-    }
-    if (len < 0 ) {
-      throw new IndexOutOfBoundsException("Negative read length "
-                                          + len
-                                          + " in " + READ);
-    }
-    if (off > buffer.length) {
-      throw new IndexOutOfBoundsException("Buffer offset of "
-                                          + off
-                                          + "beyond buffer size of "
-                                          + buffer.length
-                                          + " in " + READ);
-    }
-  } 
-}
diff --git a/hadoop-tools/hadoop-openstack/src/site/markdown/index.md b/hadoop-tools/hadoop-openstack/src/site/markdown/index.md
deleted file mode 100644
index 1815f60c613a9..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/site/markdown/index.md
+++ /dev/null
@@ -1,549 +0,0 @@
-<!---
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-* [Hadoop OpenStack Support: Swift Object Store](#Hadoop_OpenStack_Support:_Swift_Object_Store)
-    * [Introduction](#Introduction)
-    * [Features](#Features)
-    * [Using the Hadoop Swift Filesystem Client](#Using_the_Hadoop_Swift_Filesystem_Client)
-        * [Concepts: services and containers](#Concepts:_services_and_containers)
-        * [Containers and Objects](#Containers_and_Objects)
-        * [Eventual Consistency](#Eventual_Consistency)
-        * [Non-atomic "directory" operations.](#Non-atomic_directory_operations.)
-    * [Working with Swift Object Stores in Hadoop](#Working_with_Swift_Object_Stores_in_Hadoop)
-        * [Swift Filesystem URIs](#Swift_Filesystem_URIs)
-        * [Installing](#Installing)
-        * [Configuring](#Configuring)
-            * [Example: Rackspace US, in-cluster access using API key](#Example:_Rackspace_US_in-cluster_access_using_API_key)
-            * [Example: Rackspace UK: remote access with password authentication](#Example:_Rackspace_UK:_remote_access_with_password_authentication)
-            * [Example: HP cloud service definition](#Example:_HP_cloud_service_definition)
-        * [General Swift Filesystem configuration options](#General_Swift_Filesystem_configuration_options)
-            * [Blocksize fs.swift.blocksize](#Blocksize_fs.swift.blocksize)
-            * [Partition size fs.swift.partsize](#Partition_size_fs.swift.partsize)
-            * [Request size fs.swift.requestsize](#Request_size_fs.swift.requestsize)
-            * [Connection timeout fs.swift.connect.timeout](#Connection_timeout_fs.swift.connect.timeout)
-            * [Connection timeout fs.swift.socket.timeout](#Connection_timeout_fs.swift.socket.timeout)
-            * [Connection Retry Count fs.swift.connect.retry.count](#Connection_Retry_Count_fs.swift.connect.retry.count)
-            * [Connection Throttle Delay fs.swift.connect.throttle.delay](#Connection_Throttle_Delay_fs.swift.connect.throttle.delay)
-            * [HTTP Proxy](#HTTP_Proxy)
-        * [Troubleshooting](#Troubleshooting)
-            * [ClassNotFoundException](#ClassNotFoundException)
-            * [Failure to Authenticate](#Failure_to_Authenticate)
-            * [Timeout connecting to the Swift Service](#Timeout_connecting_to_the_Swift_Service)
-        * [Warnings](#Warnings)
-        * [Limits](#Limits)
-        * [Testing the hadoop-openstack module](#Testing_the_hadoop-openstack_module)
-
-Hadoop OpenStack Support: Swift Object Store
-============================================
-
-Introduction
-------------
-
-[OpenStack](http://www.openstack.org/) is an open source cloud infrastructure which can be accessed from multiple public IaaS providers, and deployed privately. It offers infrastructure services such as VM hosting (Nova), authentication (Keystone) and storage of binary objects (Swift).
-
-This module enables Apache Hadoop applications -including MapReduce jobs, read and write data to and from instances of the [OpenStack Swift object store](http://www.openstack.org/software/openstack-storage/).
-
-To make it part of Apache Hadoop's default classpath, simply make sure that
-HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-openstack' in the list.
-
-Features
---------
-
-* Read and write of data stored in a Swift object store
-
-* Support of a pseudo-hierachical file system (directories, subdirectories and
-  files)
-
-* Standard filesystem operations: `create`, `delete`, `mkdir`,
-  `ls`, `mv`, `stat`.
-
-* Can act as a source of data in a MapReduce job, or a sink.
-
-* Support for multiple OpenStack services, and multiple containers from a
-  single service.
-
-* Supports in-cluster and remote access to Swift data.
-
-* Supports OpenStack Keystone authentication with password or token.
-
-* Released under the Apache Software License
-
-* Tested against the Hadoop 3.x and 1.x branches, against multiple public
-  OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
-
-* Tested against private OpenStack clusters, including scalability tests of
-  large file uploads.
-
-Using the Hadoop Swift Filesystem Client
-----------------------------------------
-
-### Concepts: services and containers
-
-OpenStack swift is an *Object Store*; also known as a *blobstore*. It stores arbitrary binary objects by name in a *container*.
-
-The Hadoop Swift filesystem library adds another concept, the *service*, which defines which Swift blobstore hosts a container -and how to connect to it.
-
-### Containers and Objects
-
-*   Containers are created by users with accounts on the Swift filestore, and hold
-    *objects*.
-
-*   Objects can be zero bytes long, or they can contain data.
-
-*   Objects in the container can be up to 5GB; there is a special support for
-    larger files than this, which merges multiple objects in to one.
-
-*   Each object is referenced by it's *name*; there is no notion of directories.
-
-*   You can use any characters in an object name that can be 'URL-encoded'; the
-    maximum length of a name is 1034 characters -after URL encoding.
-
-*   Names can have `/` characters in them, which are used to create the illusion of
-    a directory structure. For example `dir/dir2/name`. Even though this looks
-    like a directory, *it is still just a name*. There is no requirement to have
-    any entries in the container called `dir` or `dir/dir2`
-
-*   That said. if the container has zero-byte objects that look like directory
-    names above other objects, they can pretend to be directories. Continuing the
-    example, a 0-byte object called `dir` would tell clients that it is a
-    directory while `dir/dir2` or `dir/dir2/name` were present. This creates an
-    illusion of containers holding a filesystem.
-
-Client applications talk to Swift over HTTP or HTTPS, reading, writing and deleting objects using standard HTTP operations (GET, PUT and DELETE, respectively). There is also a COPY operation, that creates a new object in the container, with a new name, containing the old data. There is no rename operation itself, objects need to be copied -then the original entry deleted.
-
-### Eventual Consistency
-
-The Swift Filesystem is \*eventually consistent\*: an operation on an object may not be immediately visible to that client, or other clients. This is a consequence of the goal of the filesystem: to span a set of machines, across multiple datacenters, in such a way that the data can still be available when many of them fail. (In contrast, the Hadoop HDFS filesystem is \*immediately consistent\*, but it does not span datacenters.)
-
-Eventual consistency can cause surprises for client applications that expect immediate consistency: after an object is deleted or overwritten, the object may still be visible -or the old data still retrievable. The Swift Filesystem client for Apache Hadoop attempts to handle this, in conjunction with the MapReduce engine, but there may be still be occasions when eventual consistency causes surprises.
-
-### Non-atomic "directory" operations.
-
-Hadoop expects some operations to be atomic, especially `rename()`, which is something the MapReduce layer relies on to commit the output of a job, renaming data from a temp directory to the final path. Because a rename is implemented as a copy of every blob under the directory's path, followed by a delete of the originals, the intermediate state of the operation will be visible to other clients. If two Reducer tasks to rename their temp directory to the final path, both operations may succeed, with the result that output directory contains mixed data. This can happen if MapReduce jobs are being run with *speculation* enabled and Swift used as the direct output of the MR job (it can also happen against Amazon S3).
-
-Other consequences of the non-atomic operations are:
-
-1.  If a program is looking for the presence of the directory before acting
-    on the data -it may start prematurely. This can be avoided by using
-    other mechanisms to co-ordinate the programs, such as the presence of a file
-    that is written *after* any bulk directory operations.
-
-2.  A `rename()` or `delete()` operation may include files added under
-    the source directory tree during the operation, may unintentionally delete
-    it, or delete the 0-byte swift entries that mimic directories and act
-    as parents for the files. Try to avoid doing this.
-
-The best ways to avoid all these problems is not using Swift as the filesystem between MapReduce jobs or other Hadoop workflows. It can act as a source of data, and a final destination, but it doesn't meet all of Hadoop's expectations of what a filesystem is -it's a *blobstore*.
-
-Working with Swift Object Stores in Hadoop
-------------------------------------------
-
-Once installed, the Swift FileSystem client can be used by any Hadoop application to read from or write to data stored in a Swift container.
-
-Data stored in Swift can be used as the direct input to a MapReduce job -simply use the `swift:` URL (see below) to declare the source of the data.
-
-This Swift Filesystem client is designed to work with multiple Swift object stores, both public and private. This allows the client to work with different clusters, reading and writing data to and from either of them.
-
-It can also work with the same object stores using multiple login details.
-
-These features are achieved by one basic concept: using a service name in the URI referring to a swift filesystem, and looking up all the connection and login details for that specific service. Different service names can be defined in the Hadoop XML configuration file, so defining different clusters, or providing different login details for the same object store(s).
-
-### Swift Filesystem URIs
-
-Hadoop uses URIs to refer to files within a filesystem. Some common examples are:
-
-        local://etc/hosts
-        hdfs://cluster1/users/example/data/set1
-        hdfs://cluster2.example.org:8020/users/example/data/set1
-
-The Swift Filesystem Client adds a new URL type `swift`. In a Swift Filesystem URL, the hostname part of a URL identifies the container and the service to work with; the path the name of the object. Here are some examples
-
-        swift://container.rackspace/my-object.csv
-        swift://data.hpcloud/data/set1
-        swift://dmitry.privatecloud/out/results
-
-In the last two examples, the paths look like directories: it is not, they are simply the objects named `data/set1` and `out/results` respectively.
-
-### Installing
-
-The `hadoop-openstack` JAR must be on the classpath of the Hadoop program trying to talk to the Swift service. If installed in the classpath of the Hadoop MapReduce service, then all programs started by the MR engine will pick up the JAR automatically. This is the easiest way to give all Hadoop jobs access to Swift.
-
-Alternatively, the JAR can be included as one of the JAR files that an application uses. This lets the Hadoop jobs work with a Swift object store even if the Hadoop cluster is not pre-configured for this.
-
-The library also depends upon the Apache HttpComponents library, which must also be on the classpath.
-
-### Configuring
-
-To talk to a swift service, the user must must provide:
-
-1.  The URL defining the container and the service.
-
-2.  In the cluster/job configuration, the login details of that service.
-
-Multiple service definitions can co-exist in the same configuration file: just use different names for them.
-
-#### Example: Rackspace US, in-cluster access using API key
-
-This service definition is for use in a Hadoop cluster deployed within Rackspace's US infrastructure.
-
-        <property>
-          <name>fs.swift.service.rackspace.auth.url</name>
-          <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
-          <description>Rackspace US (multiregion)</description>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspace.username</name>
-          <value>user4</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspace.region</name>
-          <value>DFW</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspace.apikey</name>
-          <value>fe806aa86dfffe2f6ed8</value>
-        </property>
-
-Here the API key visible in the account settings API keys page is used to log in. No property for public/private access -the default is to use the private endpoint for Swift operations.
-
-This configuration also selects one of the regions, DFW, for its data.
-
-A reference to this service would use the `rackspace` service name:
-
-        swift://hadoop-container.rackspace/
-
-#### Example: Rackspace UK: remote access with password authentication
-
-This connects to Rackspace's UK ("LON") datacenter.
-
-        <property>
-          <name>fs.swift.service.rackspaceuk.auth.url</name>
-          <value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
-          <description>Rackspace UK</description>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspaceuk.username</name>
-          <value>user4</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspaceuk.password</name>
-          <value>insert-password-here/value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.rackspace.public</name>
-          <value>true</value>
-        </property>
-
-This is a public access point connection, using a password over an API key.
-
-A reference to this service would use the `rackspaceuk` service name:
-
-        swift://hadoop-container.rackspaceuk/
-
-Because the public endpoint is used, if this service definition is used within the London datacenter, all accesses will be billed at the public upload/download rates, *irrespective of where the Hadoop cluster is*.
-
-#### Example: HP cloud service definition
-
-Here is an example that connects to the HP Cloud object store.
-
-        <property>
-          <name>fs.swift.service.hpcloud.auth.url</name>
-          <value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
-          </value>
-          <description>HP Cloud</description>
-        </property>
-
-        <property>
-          <name>fs.swift.service.hpcloud.tenant</name>
-          <value>FE806AA86</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.hpcloud.username</name>
-          <value>FE806AA86DFFFE2F6ED8</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.hpcloud.password</name>
-          <value>secret-password-goes-here</value>
-        </property>
-
-        <property>
-          <name>fs.swift.service.hpcloud.public</name>
-          <value>true</value>
-        </property>
-
-A reference to this service would use the `hpcloud` service name:
-
-        swift://hadoop-container.hpcloud/
-
-### General Swift Filesystem configuration options
-
-Some configuration options apply to the Swift client, independent of the specific Swift filesystem chosen.
-
-#### Blocksize fs.swift.blocksize
-
-Swift does not break up files into blocks, except in the special case of files over 5GB in length. Accordingly, there isn't a notion of a "block size" to define where the data is kept.
-
-Hadoop's MapReduce layer depends on files declaring their block size, so that it knows how to partition work. Too small a blocksize means that many mappers work on small pieces of data; too large a block size means that only a few mappers get started.
-
-The block size value reported by Swift, therefore, controls the basic workload partioning of the MapReduce engine -and can be an important parameter to tune for performance of the cluster.
-
-The property has a unit of kilobytes; the default value is `32*1024`: 32 MB
-
-        <property>
-          <name>fs.swift.blocksize</name>
-          <value>32768</value>
-        </property>
-
-This blocksize has no influence on how files are stored in Swift; it only controls what the reported size of blocks are - a value used in Hadoop MapReduce to divide work.
-
-Note that the MapReduce engine's split logic can be tuned independently by setting the `mapred.min.split.size` and `mapred.max.split.size` properties, which can be done in specific job configurations.
-
-        <property>
-          <name>mapred.min.split.size</name>
-          <value>524288</value>
-        </property>
-
-        <property>
-          <name>mapred.max.split.size</name>
-          <value>1048576</value>
-        </property>
-
-In an Apache Pig script, these properties would be set as:
-
-        mapred.min.split.size 524288
-        mapred.max.split.size 1048576
-
-#### Partition size fs.swift.partsize
-
-The Swift filesystem client breaks very large files into partitioned files, uploading each as it progresses, and writing any remaning data and an XML manifest when a partitioned file is closed.
-
-The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that Swift can support.
-
-It is possible to set a smaller partition size, in the `fs.swift.partsize` option. This takes a value in KB.
-
-        <property>
-          <name>fs.swift.partsize</name>
-          <value>1024</value>
-          <description>upload every MB</description>
-        </property>
-
-When should this value be changed from its default?
-
-While there is no need to ever change it for basic operation of the Swift filesystem client, it can be tuned
-
-*   If a Swift filesystem is location aware, then breaking a file up into
-    smaller partitions scatters the data round the cluster. For best performance,
-    the property `fs.swift.blocksize` should be set to a smaller value than the
-    partition size of files.
-
-*   When writing to an unpartitioned file, the entire write is done in the
-    `close()` operation. When a file is partitioned, the outstanding data to
-    be written whenever the outstanding amount of data is greater than the
-    partition size. This means that data will be written more incrementally
-
-#### Request size fs.swift.requestsize
-
-The Swift filesystem client reads files in HTTP GET operations, asking for a block of data at a time.
-
-The default value is 64KB. A larger value may be more efficient over faster networks, as it reduces the overhead of setting up the HTTP operation.
-
-However, if the file is read with many random accesses, requests for data will be made from different parts of the file -discarding some of the previously requested data. The benefits of larger request sizes may be wasted.
-
-The property `fs.swift.requestsize` sets the request size in KB.
-
-        <property>
-          <name>fs.swift.requestsize</name>
-          <value>128</value>
-        </property>
-
-#### Connection timeout fs.swift.connect.timeout
-
-This sets the timeout in milliseconds to connect to a Swift service.
-
-        <property>
-          <name>fs.swift.connect.timeout</name>
-          <value>15000</value>
-        </property>
-
-A shorter timeout means that connection failures are raised faster -but may trigger more false alarms. A longer timeout is more resilient to network problems -and may be needed when talking to remote filesystems.
-
-#### Connection timeout fs.swift.socket.timeout
-
-This sets the timeout in milliseconds to wait for data from a connected socket.
-
-        <property>
-          <name>fs.swift.socket.timeout</name>
-          <value>60000</value>
-        </property>
-
-A shorter timeout means that connection failures are raised faster -but may trigger more false alarms. A longer timeout is more resilient to network problems -and may be needed when talking to remote filesystems.
-
-#### Connection Retry Count fs.swift.connect.retry.count
-
-This sets the number of times to try to connect to a service whenever an HTTP request is made.
-
-        <property>
-          <name>fs.swift.connect.retry.count</name>
-          <value>3</value>
-        </property>
-
-The more retries, the more resilient it is to transient outages -and the less rapid it is at detecting and reporting server connectivity problems.
-
-#### Connection Throttle Delay fs.swift.connect.throttle.delay
-
-This property adds a delay between bulk file copy and delete operations, to prevent requests being throttled or blocked by the remote service
-
-        <property>
-          <name>fs.swift.connect.throttle.delay</name>
-          <value>0</value>
-        </property>
-
-It is measured in milliseconds; "0" means do not add any delay.
-
-Throttling is enabled on the public endpoints of some Swift services. If `rename()` or `delete()` operations fail with `SwiftThrottledRequestException` exceptions, try setting this property.
-
-#### HTTP Proxy
-
-If the client can only access the Swift filesystem via a web proxy server, the client configuration must specify the proxy via the `fs.swift.connect.proxy.host` and `fs.swift.connect.proxy.port` properties.
-
-        <property>
-          <name>fs.swift.proxy.host</name>
-          <value>web-proxy</value>
-        </property>
-
-        <property>
-          <name>fs.swift.proxy.port</name>
-          <value>8088</value>
-        </property>
-
-If the host is declared, the proxy port must be set to a valid integer value.
-
-### Troubleshooting
-
-#### ClassNotFoundException
-
-The `hadoop-openstack` JAR -or any dependencies- may not be on your classpath.
-
-Make sure that the:
-* JAR is installed on the servers in the cluster.
-* 'hadoop-openstack' is on the HADOOP_OPTIONAL_TOOLS entry in hadoop-env.sh or that the job submission process uploads the JAR file to the distributed cache.
-
-#### Failure to Authenticate
-
-A `SwiftAuthenticationFailedException` is thrown when the client cannot authenticate with the OpenStack keystone server. This could be because the URL in the service definition is wrong, or because the supplied credentials are invalid.
-
-1.  Check the authentication URL through `curl` or your browser
-
-2.  Use a Swift client such as CyberDuck to validate your credentials
-
-3.  If you have included a tenant ID, try leaving it out. Similarly,
-    try adding it if you had not included it.
-
-4.  Try switching from API key authentication to password-based authentication,
-    by setting the password.
-
-5.  Change your credentials. As with Amazon AWS clients, some credentials
-    don't seem to like going over the network.
-
-#### Timeout connecting to the Swift Service
-
-This happens if the client application is running outside an OpenStack cluster, where it does not have access to the private hostname/IP address for filesystem operations. Set the `public` flag to true -but remember to set it to false for use in-cluster.
-
-### Warnings
-
-1.  Do not share your login details with anyone, which means do not log the
-    details, or check the XML configuration files into any revision control system
-    to which you do not have exclusive access.
-
-2.  Similarly, do not use your real account details in any
-    documentation \*or any bug reports submitted online\*
-
-3.  Prefer the apikey authentication over passwords as it is easier
-    to revoke a key -and some service providers allow you to set
-    an automatic expiry date on a key when issued.
-
-4.  Do not use the public service endpoint from within a public OpenStack
-    cluster, as it will run up large bills.
-
-5.  Remember: it's not a real filesystem or hierarchical directory structure.
-    Some operations (directory rename and delete) take time and are not atomic or
-    isolated from other operations taking place.
-
-6.  Append is not supported.
-
-7.  Unix-style permissions are not supported. All accounts with write access to
-    a repository have unlimited access; the same goes for those with read access.
-
-8.  In the public clouds, do not make the containers public unless you are happy
-    with anyone reading your data, and are prepared to pay the costs of their
-    downloads.
-
-### Limits
-
-*   Maximum length of an object path: 1024 characters
-
-*   Maximum size of a binary object: no absolute limit. Files \> 5GB are
-    partitioned into separate files in the native filesystem, and merged during
-    retrieval. *Warning:* the partitioned/large file support is the
-    most complex part of the Hadoop/Swift FS integration, and, along with
-    authentication, the most troublesome to support.
-
-### Testing the hadoop-openstack module
-
-The `hadoop-openstack` can be remotely tested against any public or private cloud infrastructure which supports the OpenStack Keystone authentication mechanism. It can also be tested against private OpenStack clusters. OpenStack Development teams are strongly encouraged to test the Hadoop swift filesystem client against any version of Swift that they are developing or deploying, to stress their cluster and to identify bugs early.
-
-The module comes with a large suite of JUnit tests -tests that are only executed if the source tree includes credentials to test against a specific cluster.
-
-After checking out the Hadoop source tree, create the file:
-
-      hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
-
-Into this file, insert the credentials needed to bond to the test filesystem, as decribed above.
-
-Next set the property `test.fs.swift.name` to the URL of a swift container to test against. The tests expect exclusive access to this container -do not keep any other data on it, or expect it to be preserved.
-
-        <property>
-          <name>test.fs.swift.name</name>
-          <value>swift://test.myswift/</value>
-        </property>
-
-In the base hadoop directory, run:
-
-       mvn clean install -DskipTests
-
-This builds a set of Hadoop JARs consistent with the `hadoop-openstack` module that is about to be tested.
-
-In the `hadoop-tools/hadoop-openstack` directory run
-
-       mvn test -Dtest=TestSwiftRestClient
-
-This runs some simple tests which include authenticating against the remote swift service. If these tests fail, so will all the rest. If it does fail: check your authentication.
-
-Once this test succeeds, you can run the full test suite
-
-        mvn test
-
-Be advised that these tests can take an hour or more, especially against a remote Swift service -or one that throttles bulk operations.
-
-Once the `auth-keys.xml` file is in place, the `mvn test` runs from the Hadoop source base directory will automatically run these OpenStack tests While this ensures that no regressions have occurred, it can also add significant time to test runs, and may run up bills, depending on who is providingthe Swift storage service. We recommend having a separate source tree set up purely for the Swift tests, and running it manually or by the CI tooling at a lower frequency than normal test runs.
-
-Finally: Apache Hadoop is an open source project. Contributions of code -including more tests- are very welcome.
diff --git a/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css b/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css
deleted file mode 100644
index f830baafa8cc8..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-#banner {
-  height: 93px;
-  background: none;
-}
-
-#bannerLeft img {
-  margin-left: 30px;
-  margin-top: 10px;
-}
-
-#bannerRight img {
-  margin: 17px;
-}
-
diff --git a/hadoop-tools/hadoop-openstack/src/site/site.xml b/hadoop-tools/hadoop-openstack/src/site/site.xml
deleted file mode 100644
index e2941380e2acf..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/site/site.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~  or more contributor license agreements.  See the NOTICE file
-  ~  distributed with this work for additional information
-  ~  regarding copyright ownership.  The ASF licenses this file
-  ~  to you under the Apache License, Version 2.0 (the
-  ~  "License"); you may not use this file except in compliance
-  ~  with the License.  You may obtain a copy of the License at
-  ~
-  ~       http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~  Unless required by applicable law or agreed to in writing, software
-  ~  distributed under the License is distributed on an "AS IS" BASIS,
-  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~  See the License for the specific language governing permissions and
-  ~  limitations under the License.
-  -->
-
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project name="Apache Hadoop ${project.version}">
-
-  <skin>
-    <groupId>org.apache.maven.skins</groupId>
-    <artifactId>maven-stylus-skin</artifactId>
-    <version>${maven-stylus-skin.version}</version>
-  </skin>
-
-  <body>
-    <links>
-      <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
-    </links>
-  </body>
-
-</project>
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
deleted file mode 100644
index 4361a06a9498f..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;
-
-/**
- * This is the base class for most of the Swift tests
- */
-public class SwiftFileSystemBaseTest extends Assert implements
-                                                    SwiftTestConstants {
-
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(SwiftFileSystemBaseTest.class);
-  protected SwiftNativeFileSystem fs;
-  protected static SwiftNativeFileSystem lastFs;
-  protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
-  private Configuration conf;
-
-  @Before
-  public void setUp() throws Exception {
-    noteAction("setup");
-    final URI uri = getFilesystemURI();
-    conf = createConfiguration();
-
-    fs = createSwiftFS();
-    try {
-      fs.initialize(uri, conf);
-    } catch (IOException e) {
-      //FS init failed, set it to null so that teardown doesn't
-      //attempt to use it
-      fs = null;
-      throw e;
-    }
-    //remember the last FS
-    lastFs = fs;
-    noteAction("setup complete");
-  }
-
-  /**
-   * Configuration generator. May be overridden to inject
-   * some custom options
-   * @return a configuration with which to create FS instances
-   */
-  protected Configuration createConfiguration() {
-    return new Configuration();
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    cleanupInTeardown(fs, "/test");
-  }
-
-  @AfterClass
-  public static void classTearDown() throws Exception {
-    if (lastFs  != null) {
-      List<DurationStats> statistics = lastFs.getOperationStatistics();
-      for (DurationStats stat : statistics) {
-        LOG.info(stat.toString());
-      }
-    }
-  }
-
-  /**
-   * Get the configuration used to set up the FS
-   * @return the configuration
-   */
-  public Configuration getConf() {
-    return conf;
-  }
-
-  /**
-   * Describe the test, combining some logging with details
-   * for people reading the code
-   *
-   * @param description test description
-   */
-  protected void describe(String description) {
-    noteAction(description);
-  }
-
-  protected URI getFilesystemURI() throws URISyntaxException, IOException {
-    return getServiceURI(createConfiguration());
-  }
-
-  protected SwiftNativeFileSystem createSwiftFS() throws IOException {
-    SwiftNativeFileSystem swiftNativeFileSystem =
-      new SwiftNativeFileSystem();
-    return swiftNativeFileSystem;
-  }
-
-  protected int getBlockSize() {
-    return 1024;
-  }
-
-  /**
-   * Is rename supported?
-   * @return true
-   */
-  protected boolean renameSupported() {
-    return true;
-  }
-
-  /**
-   * assume in a test that rename is supported;
-   * skip it if not
-   */
-  protected void assumeRenameSupported() {
-    Assume.assumeTrue(renameSupported());
-  }
-
-  /**
-   * Take an unqualified path, and qualify it w.r.t the
-   * current filesystem
-   * @param pathString source path
-   * @return a qualified path instance
-   */
-  protected Path path(String pathString) {
-    return fs.makeQualified(new Path(pathString));
-  }
-
-  /**
-   * Get the filesystem
-   * @return the current FS
-   */
-  public SwiftNativeFileSystem getFs() {
-    return fs;
-  }
-
-  /**
-   * Create a file using the standard {@link #data} bytes.
-   *
-   * @param path path to write
-   * @throws IOException on any problem
-   */
-  protected void createFile(Path path) throws IOException {
-    createFile(path, data);
-  }
-
-  /**
-   * Create a file with the given data.
-   *
-   * @param path       path to write
-   * @param sourceData source dataset
-   * @throws IOException on any problem
-   */
-  protected void createFile(Path path, byte[] sourceData) throws IOException {
-    FSDataOutputStream out = fs.create(path);
-    out.write(sourceData, 0, sourceData.length);
-    out.close();
-  }
-
-  /**
-   * Create and then close a file
-   * @param path path to create
-   * @throws IOException on a failure
-   */
-  protected void createEmptyFile(Path path) throws IOException {
-    FSDataOutputStream out = fs.create(path);
-    out.close();
-  }
-
-  /**
-   * Get the inner store -useful for lower level operations
-   *
-   * @return the store
-   */
-  protected SwiftNativeFileSystemStore getStore() {
-    return fs.getStore();
-  }
-
-  /**
-   * Rename a path
-   * @param src source
-   * @param dst dest
-   * @param renameMustSucceed flag to say "this rename must exist"
-   * @param srcExists add assert that the source exists afterwards
-   * @param dstExists add assert the dest exists afterwards
-   * @throws IOException IO trouble
-   */
-  protected void rename(Path src, Path dst, boolean renameMustSucceed,
-                        boolean srcExists, boolean dstExists) throws IOException {
-    if (renameMustSucceed) {
-      renameToSuccess(src, dst, srcExists, dstExists);
-    } else {
-      renameToFailure(src, dst);
-    }
-  }
-
-  /**
-   * Get a string describing the outcome of a rename, by listing the dest
-   * path and its parent along with some covering text
-   * @param src source path
-   * @param dst dest path
-   * @return a string for logs and exceptions
-   * @throws IOException IO problems
-   */
-  private String getRenameOutcome(Path src, Path dst) throws IOException {
-    String lsDst = ls(dst);
-    Path parent = dst.getParent();
-    String lsParent = parent != null ? ls(parent) : "";
-    return "  result of " + src + " => " + dst
-            + " - " + lsDst
-            + " \n" + lsParent;
-  }
-
-  /**
-   * Rename, expecting an exception to be thrown
-   *
-   * @param src source
-   * @param dst dest
-   * @throws IOException a failure other than an
-   *                     expected SwiftRenameException or FileNotFoundException
-   */
-  protected void renameToFailure(Path src, Path dst) throws IOException {
-    try {
-      getStore().rename(src, dst);
-      fail("Expected failure renaming " + src + " to " + dst
-              + "- but got success");
-    } catch (SwiftOperationFailedException e) {
-      LOG.debug("Rename failed (expected):" + e);
-    } catch (FileNotFoundException e) {
-      LOG.debug("Rename failed (expected):" + e);
-    }
-  }
-
-  /**
-   * Rename to success
-   *
-   * @param src source
-   * @param dst dest
-   * @param srcExists add assert that the source exists afterwards
-   * @param dstExists add assert the dest exists afterwards
-   * @throws SwiftOperationFailedException operation failure
-   * @throws IOException IO problems
-   */
-  protected void renameToSuccess(Path src, Path dst,
-                                 boolean srcExists, boolean dstExists)
-      throws SwiftOperationFailedException, IOException {
-    getStore().rename(src, dst);
-    String outcome = getRenameOutcome(src, dst);
-    assertEquals("Source " + src + "exists: " + outcome,
-            srcExists, fs.exists(src));
-    assertEquals("Destination " + dstExists + " exists" + outcome,
-            dstExists, fs.exists(dst));
-  }
-
-  /**
-   * List a path in the test FS
-   * @param path path to list
-   * @return the contents of the path/dir
-   * @throws IOException IO problems
-   */
-  protected String ls(Path path) throws IOException {
-    return SwiftTestUtils.ls(fs, path);
-  }
-
-  /**
-   * assert that a path exists
-   * @param message message to use in an assertion
-   * @param path path to probe
-   * @throws IOException IO problems
-   */
-  public void assertExists(String message, Path path) throws IOException {
-    assertPathExists(fs, message, path);
-  }
-
-  /**
-   * assert that a path does not
-   * @param message message to use in an assertion
-   * @param path path to probe
-   * @throws IOException IO problems
-   */
-  public void assertPathDoesNotExist(String message, Path path) throws
-          IOException {
-    SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
-  }
-
-  /**
-   * Assert that a file exists and whose {@link FileStatus} entry
-   * declares that this is a file and not a symlink or directory.
-   *
-   * @param filename name of the file
-   * @throws IOException IO problems during file operations
-   */
-  protected void assertIsFile(Path filename) throws IOException {
-    SwiftTestUtils.assertIsFile(fs, filename);
-  }
-
-  /**
-   * Assert that a file exists and whose {@link FileStatus} entry
-   * declares that this is a file and not a symlink or directory.
-   *
-   * @throws IOException IO problems during file operations
-   */
-  protected void mkdirs(Path path) throws IOException {
-    assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
-  }
-
-  /**
-   * Assert that a delete succeeded
-   * @param path path to delete
-   * @param recursive recursive flag
-   * @throws IOException IO problems
-   */
-  protected void assertDeleted(Path path, boolean recursive) throws IOException {
-    SwiftTestUtils.assertDeleted(fs, path, recursive);
-  }
-
-  /**
-   * Assert that a value is not equal to the expected value
-   * @param message message if the two values are equal
-   * @param expected expected value
-   * @param actual actual value
-   */
-  protected void assertNotEqual(String message, int expected, int actual) {
-    assertTrue(message,
-               actual != expected);
-  }
-
-  /**
-   * Get the number of partitions written from the Swift Native FS APIs
-   * @param out output stream
-   * @return the number of partitioned files written by the stream
-   */
-  protected int getPartitionsWritten(FSDataOutputStream out) {
-    return SwiftNativeFileSystem.getPartitionsWritten(out);
-  }
-
-  /**
-   * Assert that the no. of partitions written matches expectations
-   * @param action operation (for use in the assertions)
-   * @param out output stream
-   * @param expected expected no. of partitions
-   */
-  protected void assertPartitionsWritten(String action, FSDataOutputStream out,
-                                         long expected) {
-    OutputStream nativeStream = out.getWrappedStream();
-    int written = getPartitionsWritten(out);
-    if(written !=expected) {
-    Assert.fail(action + ": " +
-                TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT 
-                + " + expected: " + expected + " actual: " + written
-                + " -- " + nativeStream);
-    }
-  }
-
-  /**
-   * Assert that the result value == -1; which implies
-   * that a read was successful
-   * @param text text to include in a message (usually the operation)
-   * @param result read result to validate
-   */
-  protected void assertMinusOne(String text, int result) {
-    assertEquals(text + " wrong read result " + result, -1, result);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java
deleted file mode 100644
index 6948cf92fa6c0..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-/**
- * Hard coded constants for the test timeouts
- */
-public interface SwiftTestConstants {
-  /**
-   * Timeout for swift tests: {@value}
-   */
-  int SWIFT_TEST_TIMEOUT = 5 * 60 * 1000;
-
-  /**
-   * Timeout for tests performing bulk operations: {@value}
-   */
-  int SWIFT_BULK_IO_TEST_TIMEOUT = 12 * 60 * 1000;
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java
deleted file mode 100644
index b595f1c2d14ab..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.fs.swift;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSMainOperationsBaseTest;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import static org.apache.hadoop.fs.swift.SwiftTestConstants.SWIFT_TEST_TIMEOUT;
-import java.io.IOException;
-import java.net.URI;
-
-public class TestFSMainOperationsSwift extends FSMainOperationsBaseTest {
-
-  @Override
-  @Before
-  public void setUp() throws Exception {
-    Configuration conf = new Configuration();
-    //small blocksize for faster remote tests
-    conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, 2);
-    URI serviceURI = SwiftTestUtils.getServiceURI(conf);
-    fSys = FileSystem.get(serviceURI, conf);
-    super.setUp();
-  }
-
-  private Path wd = null;
-
-  @Override
-  protected FileSystem createFileSystem() throws Exception {
-    return fSys;
-  }
-
-  @Override
-  protected Path getDefaultWorkingDirectory() throws IOException {
-    if (wd == null) {
-      wd = fSys.getWorkingDirectory();
-    }
-    return wd;
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWDAbsolute() throws IOException {
-    Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
-    fSys.mkdirs(absoluteDir);
-    fSys.setWorkingDirectory(absoluteDir);
-    Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testListStatusThrowsExceptionForUnreadableDir() {
-    SwiftTestUtils.skip("unsupported");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusThrowsExceptionForUnreadableDir() {
-    SwiftTestUtils.skip("unsupported");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testFsStatus() throws Exception {
-    super.testFsStatus();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWorkingDirectory() throws Exception {
-    super.testWorkingDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testMkdirs() throws Exception {
-    super.testMkdirs();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
-    super.testMkdirsFailsForSubdirectoryOfExistingFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGetFileStatusThrowsExceptionForNonExistentFile() throws
-                                                                   Exception {
-    super.testGetFileStatusThrowsExceptionForNonExistentFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testListStatusThrowsExceptionForNonExistentFile() throws
-                                                                Exception {
-    super.testListStatusThrowsExceptionForNonExistentFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testListStatus() throws Exception {
-    super.testListStatus();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testListStatusFilterWithNoMatches() throws Exception {
-    super.testListStatusFilterWithNoMatches();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testListStatusFilterWithSomeMatches() throws Exception {
-    super.testListStatusFilterWithSomeMatches();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusNonExistentFile() throws Exception {
-    super.testGlobStatusNonExistentFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusWithNoMatchesInPath() throws Exception {
-    super.testGlobStatusWithNoMatchesInPath();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusSomeMatchesInDirectories() throws Exception {
-    super.testGlobStatusSomeMatchesInDirectories();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusWithMultipleWildCardMatches() throws Exception {
-    super.testGlobStatusWithMultipleWildCardMatches();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception {
-    super.testGlobStatusWithMultipleMatchesOfSingleChar();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithEmptyPathResults() throws Exception {
-    super.testGlobStatusFilterWithEmptyPathResults();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithSomePathMatchesAndTrivialFilter() throws
-                                                                        Exception {
-    super.testGlobStatusFilterWithSomePathMatchesAndTrivialFilter();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter() throws
-                                                                                Exception {
-    super.testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter() throws
-                                                                               Exception {
-    super.testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter() throws
-                                                                           Exception {
-    super.testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter() throws
-                                                                                 Exception {
-    super.testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteReadAndDeleteEmptyFile() throws Exception {
-    super.testWriteReadAndDeleteEmptyFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteReadAndDeleteHalfABlock() throws Exception {
-    super.testWriteReadAndDeleteHalfABlock();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteReadAndDeleteOneBlock() throws Exception {
-    super.testWriteReadAndDeleteOneBlock();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
-    super.testWriteReadAndDeleteOneAndAHalfBlocks();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteReadAndDeleteTwoBlocks() throws Exception {
-    super.testWriteReadAndDeleteTwoBlocks();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testOverwrite() throws IOException {
-    super.testOverwrite();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testWriteInNonExistentDirectory() throws IOException {
-    super.testWriteInNonExistentDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testDeleteNonExistentFile() throws IOException {
-    super.testDeleteNonExistentFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testDeleteRecursively() throws IOException {
-    super.testDeleteRecursively();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testDeleteEmptyDirectory() throws IOException {
-    super.testDeleteEmptyDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameNonExistentPath() throws Exception {
-    super.testRenameNonExistentPath();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileToNonExistentDirectory() throws Exception {
-    super.testRenameFileToNonExistentDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileToDestinationWithParentFile() throws Exception {
-    super.testRenameFileToDestinationWithParentFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileToExistingParent() throws Exception {
-    super.testRenameFileToExistingParent();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileToItself() throws Exception {
-    super.testRenameFileToItself();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileAsExistingFile() throws Exception {
-    super.testRenameFileAsExistingFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameFileAsExistingDirectory() throws Exception {
-    super.testRenameFileAsExistingDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryToItself() throws Exception {
-    super.testRenameDirectoryToItself();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryToNonExistentParent() throws Exception {
-    super.testRenameDirectoryToNonExistentParent();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryAsNonExistentDirectory() throws Exception {
-    super.testRenameDirectoryAsNonExistentDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryAsEmptyDirectory() throws Exception {
-    super.testRenameDirectoryAsEmptyDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryAsNonEmptyDirectory() throws Exception {
-    super.testRenameDirectoryAsNonEmptyDirectory();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testRenameDirectoryAsFile() throws Exception {
-    super.testRenameDirectoryAsFile();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testInputStreamClosedTwice() throws IOException {
-    super.testInputStreamClosedTwice();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testOutputStreamClosedTwice() throws IOException {
-    super.testOutputStreamClosedTwice();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testGetWrappedInputStream() throws IOException {
-    super.testGetWrappedInputStream();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  @Override
-  public void testCopyToLocalWithUseRawLocalFileSystemOption() throws
-                                                               Exception {
-    super.testCopyToLocalWithUseRawLocalFileSystemOption();
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java
deleted file mode 100644
index 99c6962cb48e5..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.junit.Test;
-
-import java.net.URL;
-
-/**
- * This test just debugs which log resources are being picked up
- */
-public class TestLogResources implements SwiftTestConstants {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestLogResources.class);
-
-  private void printf(String format, Object... args) {
-    String msg = String.format(format, args);
-    System.out.printf(msg + "\n");
-    LOG.info(msg);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testWhichLog4JPropsFile() throws Throwable {
-    locateResource("log4j.properties");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testWhichLog4JXMLFile() throws Throwable {
-    locateResource("log4j.XML");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testCommonsLoggingProps() throws Throwable {
-    locateResource("commons-logging.properties");
-  }
-
-  private void locateResource(String resource) {
-    URL url = this.getClass().getClassLoader().getResource(resource);
-    if (url != null) {
-      printf("resource %s is at %s", resource, url);
-    } else {
-      printf("resource %s is not on the classpath", resource);
-    }
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java
deleted file mode 100644
index c195bffc5137f..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.After;
-import org.junit.Test;
-
-/**
- * Seek tests verify that
- * <ol>
- *   <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
- *   <li>When you seek past the end of a file, it's an error that should
- *   raise -what- EOFException?</li>
- *   <li>when you seek forwards, you get new data</li>
- *   <li>when you seek backwards, you get the previous data</li>
- *   <li>That this works for big multi-MB files as well as small ones.</li>
- * </ol>
- * These may seem "obvious", but the more the input streams try to be clever
- * about offsets and buffering, the more likely it is that seek() will start
- * to get confused.
- */
-public class TestReadPastBuffer extends SwiftFileSystemBaseTest {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestReadPastBuffer.class);
-  public static final int SWIFT_READ_BLOCKSIZE = 4096;
-  public static final int SEEK_FILE_LEN = SWIFT_READ_BLOCKSIZE * 2;
-
-  private Path testPath;
-  private Path readFile;
-  private Path zeroByteFile;
-  private FSDataInputStream instream;
-
-
-  /**
-   * Get a configuration which a small blocksize reported to callers
-   * @return a configuration for this test
-   */
-  @Override
-  public Configuration getConf() {
-    Configuration conf = super.getConf();
-    /*
-     * set to 4KB
-     */
-    conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, SWIFT_READ_BLOCKSIZE);
-    return conf;
-  }
-
-  /**
-   * Setup creates dirs under test/hadoop
-   *
-   * @throws Exception
-   */
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-    byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);
-
-    //delete the test directory
-    testPath = path("/test");
-    readFile = new Path(testPath, "TestReadPastBuffer.txt");
-    createFile(readFile, block);
-  }
-
-  @After
-  public void cleanFile() {
-    IOUtils.closeStream(instream);
-    instream = null;
-  }
-
-  /**
-   * Create a config with a 1KB request size
-   * @return a config
-   */
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
-    return conf;
-  }
-
-  /**
-   * Seek past the buffer then read
-   * @throws Throwable problems
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekAndReadPastEndOfFile() throws Throwable {
-    instream = fs.open(readFile);
-    assertEquals(0, instream.getPos());
-    //expect that seek to 0 works
-    //go just before the end
-    instream.seek(SEEK_FILE_LEN - 2);
-    assertTrue("Premature EOF", instream.read() != -1);
-    assertTrue("Premature EOF", instream.read() != -1);
-    assertMinusOne("read past end of file", instream.read());
-  }
-
-  /**
-   * Seek past the buffer and attempt a read(buffer)
-   * @throws Throwable failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekBulkReadPastEndOfFile() throws Throwable {
-    instream = fs.open(readFile);
-    assertEquals(0, instream.getPos());
-    //go just before the end
-    instream.seek(SEEK_FILE_LEN - 1);
-    byte[] buffer = new byte[1];
-    int result = instream.read(buffer, 0, 1);
-    //next byte is expected to fail
-    result = instream.read(buffer, 0, 1);
-    assertMinusOne("read past end of file", result);
-    //and this one
-    result = instream.read(buffer, 0, 1);
-    assertMinusOne("read past end of file", result);
-
-    //now do an 0-byte read and expect it to
-    //to be checked first
-    result = instream.read(buffer, 0, 0);
-    assertEquals("EOF checks coming before read range check", 0, result);
-
-  }
-
-
-
-  /**
-   * Read past the buffer size byte by byte and verify that it refreshed
-   * @throws Throwable
-   */
-  @Test
-  public void testReadPastBufferSize() throws Throwable {
-    instream = fs.open(readFile);
-
-    while (instream.read() != -1);
-    //here we have gone past the end of a file and its buffer. Now try again
-    assertMinusOne("reading after the (large) file was read: "+ instream,
-                   instream.read());
-  }
-}
-
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java
deleted file mode 100644
index 51fa92a2eb3a1..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.After;
-import org.junit.Test;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * Seek tests verify that
- * <ol>
- *   <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
- *   <li>When you seek past the end of a file, it's an error that should
- *   raise -what- EOFException?</li>
- *   <li>when you seek forwards, you get new data</li>
- *   <li>when you seek backwards, you get the previous data</li>
- *   <li>That this works for big multi-MB files as well as small ones.</li>
- * </ol>
- * These may seem "obvious", but the more the input streams try to be clever
- * about offsets and buffering, the more likely it is that seek() will start
- * to get confused.
- */
-public class TestSeek extends SwiftFileSystemBaseTest {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestSeek.class);
-  public static final int SMALL_SEEK_FILE_LEN = 256;
-
-  private Path testPath;
-  private Path smallSeekFile;
-  private Path zeroByteFile;
-  private FSDataInputStream instream;
-
-  /**
-   * Setup creates dirs under test/hadoop
-   *
-   * @throws Exception
-   */
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-    //delete the test directory
-    testPath = path("/test");
-    smallSeekFile = new Path(testPath, "seekfile.txt");
-    zeroByteFile = new Path(testPath, "zero.txt");
-    byte[] block = SwiftTestUtils.dataset(SMALL_SEEK_FILE_LEN, 0, 255);
-    //this file now has a simple rule: offset => value
-    createFile(smallSeekFile, block);
-    createEmptyFile(zeroByteFile);
-  }
-
-  @After
-  public void cleanFile() {
-    IOUtils.closeStream(instream);
-    instream = null;
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekZeroByteFile() throws Throwable {
-    instream = fs.open(zeroByteFile);
-    assertEquals(0, instream.getPos());
-    //expect initial read to fai;
-    int result = instream.read();
-    assertMinusOne("initial byte read", result);
-    byte[] buffer = new byte[1];
-    //expect that seek to 0 works
-    instream.seek(0);
-    //reread, expect same exception
-    result = instream.read();
-    assertMinusOne("post-seek byte read", result);
-    result = instream.read(buffer, 0, 1);
-    assertMinusOne("post-seek buffer read", result);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testBlockReadZeroByteFile() throws Throwable {
-    instream = fs.open(zeroByteFile);
-    assertEquals(0, instream.getPos());
-    //expect that seek to 0 works
-    byte[] buffer = new byte[1];
-    int result = instream.read(buffer, 0, 1);
-    assertMinusOne("block read zero byte file", result);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekReadClosedFile() throws Throwable {
-    instream = fs.open(smallSeekFile);
-    instream.close();
-    try {
-      instream.seek(0);
-    } catch (SwiftConnectionClosedException e) {
-      //expected a closed file
-    }
-    try {
-      instream.read();
-    } catch (IOException e) {
-      //expected a closed file
-    }
-    try {
-      byte[] buffer = new byte[1];
-      int result = instream.read(buffer, 0, 1);
-    } catch (IOException e) {
-      //expected a closed file
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testNegativeSeek() throws Throwable {
-    instream = fs.open(smallSeekFile);
-    assertEquals(0, instream.getPos());
-    try {
-      instream.seek(-1);
-      long p = instream.getPos();
-      LOG.warn("Seek to -1 returned a position of " + p);
-      int result = instream.read();
-      fail(
-        "expected an exception, got data " + result + " at a position of " + p);
-    } catch (IOException e) {
-      //bad seek -expected
-    }
-    assertEquals(0, instream.getPos());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekFile() throws Throwable {
-    instream = fs.open(smallSeekFile);
-    assertEquals(0, instream.getPos());
-    //expect that seek to 0 works
-    instream.seek(0);
-    int result = instream.read();
-    assertEquals(0, result);
-    assertEquals(1, instream.read());
-    assertEquals(2, instream.getPos());
-    assertEquals(2, instream.read());
-    assertEquals(3, instream.getPos());
-    instream.seek(128);
-    assertEquals(128, instream.getPos());
-    assertEquals(128, instream.read());
-    instream.seek(63);
-    assertEquals(63, instream.read());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekAndReadPastEndOfFile() throws Throwable {
-    instream = fs.open(smallSeekFile);
-    assertEquals(0, instream.getPos());
-    //expect that seek to 0 works
-    //go just before the end
-    instream.seek(SMALL_SEEK_FILE_LEN - 2);
-    assertTrue("Premature EOF", instream.read() != -1);
-    assertTrue("Premature EOF", instream.read() != -1);
-    assertMinusOne("read past end of file", instream.read());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekAndPastEndOfFileThenReseekAndRead() throws Throwable {
-    instream = fs.open(smallSeekFile);
-    //go just before the end. This may or may not fail; it may be delayed until the
-    //read
-    try {
-      instream.seek(SMALL_SEEK_FILE_LEN);
-      //if this doesn't trigger, then read() is expected to fail
-      assertMinusOne("read after seeking past EOF", instream.read());
-    } catch (EOFException expected) {
-      //here an exception was raised in seek
-    }
-    instream.seek(1);
-    assertTrue("Premature EOF", instream.read() != -1);
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
-    return conf;
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSeekBigFile() throws Throwable {
-    Path testSeekFile = new Path(testPath, "bigseekfile.txt");
-    byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
-    createFile(testSeekFile, block);
-    instream = fs.open(testSeekFile);
-    assertEquals(0, instream.getPos());
-    //expect that seek to 0 works
-    instream.seek(0);
-    int result = instream.read();
-    assertEquals(0, result);
-    assertEquals(1, instream.read());
-    assertEquals(2, instream.read());
-
-    //do seek 32KB ahead
-    instream.seek(32768);
-    assertEquals("@32768", block[32768], (byte) instream.read());
-    instream.seek(40000);
-    assertEquals("@40000", block[40000], (byte) instream.read());
-    instream.seek(8191);
-    assertEquals("@8191", block[8191], (byte) instream.read());
-    instream.seek(0);
-    assertEquals("@0", 0, (byte) instream.read());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
-    Path testSeekFile = new Path(testPath, "bigseekfile.txt");
-    byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
-    createFile(testSeekFile, block);
-    instream = fs.open(testSeekFile);
-    instream.seek(39999);
-    assertTrue(-1 != instream.read());
-    assertEquals (40000, instream.getPos());
-
-    byte[] readBuffer = new byte[256];
-    instream.read(128, readBuffer, 0, readBuffer.length);
-    //have gone back
-    assertEquals(40000, instream.getPos());
-    //content is the same too
-    assertEquals("@40000", block[40000], (byte) instream.read());
-    //now verify the picked up data
-    for (int i = 0; i < 256; i++) {
-      assertEquals("@" + i, block[i + 128], readBuffer[i]);
-    }
-  }
-
-  /**
-   * work out the expected byte from a specific offset
-   * @param offset offset in the file
-   * @return the value
-   */
-  int expectedByte(int offset) {
-    return offset & 0xff;
-  }
-}
-
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java
deleted file mode 100644
index 0212b4d9c65dc..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.swift.http.SwiftRestClient;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_LOCATION_AWARE;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_TENANT;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_BLOCKSIZE;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONNECTION_TIMEOUT;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PARTITION_SIZE;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_HOST_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_PORT_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_RETRY_COUNT;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PREFIX;
-
-/**
- * Test the swift service-specific configuration binding features
- */
-public class TestSwiftConfig extends Assert {
-
-
-  public static final String SERVICE = "openstack";
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testEmptyUrl() throws Exception {
-    final Configuration configuration = new Configuration();
-
-    set(configuration, DOT_TENANT, "tenant");
-    set(configuration, DOT_USERNAME, "username");
-    set(configuration, DOT_PASSWORD, "password");
-    mkInstance(configuration);
-  }
-
-@Test
-  public void testEmptyTenant() throws Exception {
-    final Configuration configuration = new Configuration();
-    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
-    set(configuration, DOT_USERNAME, "username");
-    set(configuration, DOT_PASSWORD, "password");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testEmptyUsername() throws Exception {
-    final Configuration configuration = new Configuration();
-    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
-    set(configuration, DOT_TENANT, "tenant");
-    set(configuration, DOT_PASSWORD, "password");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testEmptyPassword() throws Exception {
-    final Configuration configuration = new Configuration();
-    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
-    set(configuration, DOT_TENANT, "tenant");
-    set(configuration, DOT_USERNAME, "username");
-    mkInstance(configuration);
-  }
-
-  @Test
-  public void testGoodRetryCount() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_RETRY_COUNT, "3");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testBadRetryCount() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_RETRY_COUNT, "three");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testBadConnectTimeout() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_CONNECTION_TIMEOUT, "three");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testZeroBlocksize() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_BLOCKSIZE, "0");
-    mkInstance(configuration);
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testNegativeBlocksize() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_BLOCKSIZE, "-1");
-    mkInstance(configuration);
-  }
-
-  @Test
-  public void testPositiveBlocksize() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    int size = 127;
-    configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
-    SwiftRestClient restClient = mkInstance(configuration);
-    assertEquals(size, restClient.getBlocksizeKB());
-  }
-
-  @Test
-  public void testLocationAwareTruePropagates() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    set(configuration, DOT_LOCATION_AWARE, "true");
-    SwiftRestClient restClient = mkInstance(configuration);
-    assertTrue(restClient.isLocationAware());
-  }
-
-  @Test
-  public void testLocationAwareFalsePropagates() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    set(configuration, DOT_LOCATION_AWARE, "false");
-    SwiftRestClient restClient = mkInstance(configuration);
-    assertFalse(restClient.isLocationAware());
-  }
-
-  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
-  public void testNegativePartsize() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    configuration.set(SWIFT_PARTITION_SIZE, "-1");
-    SwiftRestClient restClient = mkInstance(configuration);
-  }
-
-  @Test
-  public void testPositivePartsize() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    int size = 127;
-    configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
-    SwiftRestClient restClient = mkInstance(configuration);
-    assertEquals(size, restClient.getPartSizeKB());
-  }
-
-  @Test
-  public void testProxyData() throws Exception {
-    final Configuration configuration = createCoreConfig();
-    String proxy="web-proxy";
-    int port = 8088;
-    configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
-    configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
-    SwiftRestClient restClient = mkInstance(configuration);
-    assertEquals(proxy, restClient.getProxyHost());
-    assertEquals(port, restClient.getProxyPort());
-  }
-
-  private Configuration createCoreConfig() {
-    final Configuration configuration = new Configuration();
-    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
-    set(configuration, DOT_TENANT, "tenant");
-    set(configuration, DOT_USERNAME, "username");
-    set(configuration, DOT_PASSWORD, "password");
-    return configuration;
-  }
-
-  private void set(Configuration configuration, String field, String value) {
-    configuration.set(SWIFT_SERVICE_PREFIX + SERVICE + field, value);
-  }
-
-  private SwiftRestClient mkInstance(Configuration configuration) throws
-          IOException,
-          URISyntaxException {
-    URI uri = new URI("swift://container.openstack/");
-    return SwiftRestClient.getInstance(uri, configuration);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
deleted file mode 100644
index 516dc99fab02d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertFileHasLength;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertIsDirectory;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
-
-
-/**
- * Test basic filesystem operations.
- * Many of these are similar to those in {@link TestSwiftFileSystemContract}
- * -this is a JUnit4 test suite used to initially test the Swift
- * component. Once written, there's no reason not to retain these tests.
- */
-public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSwiftFileSystemBasicOps.class);
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLsRoot() throws Throwable {
-    Path path = new Path("/");
-    FileStatus[] statuses = fs.listStatus(path);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testMkDir() throws Throwable {
-    Path path = new Path("/test/MkDir");
-    fs.mkdirs(path);
-    //success then -so try a recursive operation
-    fs.delete(path, true);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteNonexistentFile() throws Throwable {
-    Path path = new Path("/test/DeleteNonexistentFile");
-    assertFalse("delete returned true", fs.delete(path, false));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testPutFile() throws Throwable {
-    Path path = new Path("/test/PutFile");
-    Exception caught = null;
-    writeTextFile(fs, path, "Testing a put to a file", false);
-    assertDeleted(path, false);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testPutGetFile() throws Throwable {
-    Path path = new Path("/test/PutGetFile");
-    try {
-      String text = "Testing a put and get to a file "
-              + System.currentTimeMillis();
-      writeTextFile(fs, path, text, false);
-
-      String result = readBytesToString(fs, path, text.length());
-      assertEquals(text, result);
-    } finally {
-      delete(fs, path);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testPutDeleteFileInSubdir() throws Throwable {
-    Path path =
-            new Path("/test/PutDeleteFileInSubdir/testPutDeleteFileInSubdir");
-    String text = "Testing a put and get to a file in a subdir "
-            + System.currentTimeMillis();
-    writeTextFile(fs, path, text, false);
-    assertDeleted(path, false);
-    //now delete the parent that should have no children
-    assertDeleted(new Path("/test/PutDeleteFileInSubdir"), false);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRecursiveDelete() throws Throwable {
-    Path childpath =
-            new Path("/test/testRecursiveDelete");
-    String text = "Testing a put and get to a file in a subdir "
-            + System.currentTimeMillis();
-    writeTextFile(fs, childpath, text, false);
-    //now delete the parent that should have no children
-    assertDeleted(new Path("/test"), true);
-    assertFalse("child entry still present " + childpath, fs.exists(childpath));
-  }
-
-  private void delete(SwiftNativeFileSystem fs, Path path) {
-    try {
-      if (!fs.delete(path, false)) {
-        LOG.warn("Failed to delete " + path);
-      }
-    } catch (IOException e) {
-      LOG.warn("deleting " + path, e);
-    }
-  }
-
-  private void deleteR(SwiftNativeFileSystem fs, Path path) {
-    try {
-      if (!fs.delete(path, true)) {
-        LOG.warn("Failed to delete " + path);
-      }
-    } catch (IOException e) {
-      LOG.warn("deleting " + path, e);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testOverwrite() throws Throwable {
-    Path path = new Path("/test/Overwrite");
-    try {
-      String text = "Testing a put to a file "
-              + System.currentTimeMillis();
-      writeTextFile(fs, path, text, false);
-      assertFileHasLength(fs, path, text.length());
-      String text2 = "Overwriting a file "
-              + System.currentTimeMillis();
-      writeTextFile(fs, path, text2, true);
-      assertFileHasLength(fs, path, text2.length());
-      String result = readBytesToString(fs, path, text2.length());
-      assertEquals(text2, result);
-    } finally {
-      delete(fs, path);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testOverwriteDirectory() throws Throwable {
-    Path path = new Path("/test/testOverwriteDirectory");
-    try {
-      fs.mkdirs(path.getParent());
-      String text = "Testing a put to a file "
-              + System.currentTimeMillis();
-      writeTextFile(fs, path, text, false);
-      assertFileHasLength(fs, path, text.length());
-    } finally {
-      delete(fs, path);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testFileStatus() throws Throwable {
-    Path path = new Path("/test/FileStatus");
-    try {
-      String text = "Testing File Status "
-              + System.currentTimeMillis();
-      writeTextFile(fs, path, text, false);
-      SwiftTestUtils.assertIsFile(fs, path);
-    } finally {
-      delete(fs, path);
-    }
-  }
-
-  /**
-   * Assert that a newly created directory is a directory
-   *
-   * @throws Throwable if not, or if something else failed
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDirStatus() throws Throwable {
-    Path path = new Path("/test/DirStatus");
-    try {
-      fs.mkdirs(path);
-      assertIsDirectory(fs, path);
-    } finally {
-      delete(fs, path);
-    }
-  }
-
-  /**
-   * Assert that if a directory that has children is deleted, it is still
-   * a directory
-   *
-   * @throws Throwable if not, or if something else failed
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDirStaysADir() throws Throwable {
-    Path path = new Path("/test/dirStaysADir");
-    Path child = new Path(path, "child");
-    try {
-      //create the dir
-      fs.mkdirs(path);
-      //assert the parent has the directory nature
-      assertIsDirectory(fs, path);
-      //create the child dir
-      writeTextFile(fs, child, "child file", true);
-      //assert the parent has the directory nature
-      assertIsDirectory(fs, path);
-      //now rm the child
-      delete(fs, child);
-    } finally {
-      deleteR(fs, path);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testCreateMultilevelDir() throws Throwable {
-    Path base = new Path("/test/CreateMultilevelDir");
-    Path path = new Path(base, "1/2/3");
-    fs.mkdirs(path);
-    assertExists("deep multilevel dir not created", path);
-    fs.delete(base, true);
-    assertPathDoesNotExist("Multilevel delete failed", path);
-    assertPathDoesNotExist("Multilevel delete failed", base);
-
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testCreateDirWithFileParent() throws Throwable {
-    Path path = new Path("/test/CreateDirWithFileParent");
-    Path child = new Path(path, "subdir/child");
-    fs.mkdirs(path.getParent());
-    try {
-      //create the child dir
-      writeTextFile(fs, path, "parent", true);
-      try {
-        fs.mkdirs(child);
-      } catch (ParentNotDirectoryException expected) {
-        LOG.debug("Expected Exception", expected);
-      }
-    } finally {
-      fs.delete(path, true);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLongObjectNamesForbidden() throws Throwable {
-    StringBuilder buffer = new StringBuilder(1200);
-    buffer.append("/");
-    for (int i = 0; i < (1200 / 4); i++) {
-      buffer.append(String.format("%04x", i));
-    }
-    String pathString = buffer.toString();
-    Path path = new Path(pathString);
-    try {
-      writeTextFile(fs, path, pathString, true);
-      //if we get here, problems.
-      fs.delete(path, false);
-      fail("Managed to create an object with a name of length "
-              + pathString.length());
-    } catch (SwiftBadRequestException e) {
-      //expected
-      //LOG.debug("Caught exception " + e, e);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLsNonExistentFile() throws Exception {
-    try {
-      Path path = new Path("/test/hadoop/file");
-      FileStatus[] statuses = fs.listStatus(path);
-      fail("Should throw FileNotFoundException on " + path
-              + " but got list of length " + statuses.length);
-    } catch (FileNotFoundException fnfe) {
-      // expected
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testGetCanonicalServiceName() {
-    Assert.assertNull(fs.getCanonicalServiceName());
-  }
-
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java
deleted file mode 100644
index 1ad28a6cc4521..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-/**
- * Test block location logic.
- * The endpoint may or may not be location-aware
- */
-public class TestSwiftFileSystemBlockLocation extends SwiftFileSystemBaseTest {
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateSingleFileBlocks() throws Throwable {
-    describe("verify that a file returns 1+ blocks");
-    FileStatus fileStatus = createFileAndGetStatus();
-    BlockLocation[] locations =
-      getFs().getFileBlockLocations(fileStatus, 0, 1);
-    assertNotEqual("No block locations supplied for " + fileStatus, 0,
-                   locations.length);
-    for (BlockLocation location : locations) {
-      assertLocationValid(location);
-    }
-  }
-
-  private void assertLocationValid(BlockLocation location) throws
-                                                           IOException {
-    LOG.info("{}", location);
-    String[] hosts = location.getHosts();
-    String[] names = location.getNames();
-    assertNotEqual("No hosts supplied for " + location, 0, hosts.length);
-    //for every host, there's a name.
-    assertEquals("Unequal names and hosts in " + location,
-                 hosts.length, names.length);
-    assertEquals(SwiftProtocolConstants.BLOCK_LOCATION,
-                 location.getNames()[0]);
-    assertEquals(SwiftProtocolConstants.TOPOLOGY_PATH,
-                 location.getTopologyPaths()[0]);
-  }
-
-  private FileStatus createFileAndGetStatus() throws IOException {
-    Path path = path("/test/locatedFile");
-    createFile(path);
-    return fs.getFileStatus(path);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateNullStatus() throws Throwable {
-    describe("verify that a null filestatus maps to a null location array");
-    BlockLocation[] locations =
-      getFs().getFileBlockLocations((FileStatus) null, 0, 1);
-    assertNull(locations);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateNegativeSeek() throws Throwable {
-    describe("verify that a negative offset is illegal");
-    try {
-      BlockLocation[] locations =
-        getFs().getFileBlockLocations(createFileAndGetStatus(),
-                                      -1,
-                                      1);
-      fail("Expected an exception, got " + locations.length + " locations");
-    } catch (IllegalArgumentException e) {
-      //expected
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateNegativeLen() throws Throwable {
-    describe("verify that a negative length is illegal");
-    try {
-      BlockLocation[] locations =
-        getFs().getFileBlockLocations(createFileAndGetStatus(),
-                                      0,
-                                      -1);
-      fail("Expected an exception, got " + locations.length + " locations");
-    } catch (IllegalArgumentException e) {
-      //expected
-    }
-  }
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateOutOfRangeLen() throws Throwable {
-    describe("overshooting the length is legal, as long as the" +
-             " origin location is valid");
-
-    BlockLocation[] locations =
-      getFs().getFileBlockLocations(createFileAndGetStatus(),
-                                    0,
-                                    data.length + 100);
-    assertNotNull(locations);
-    assertTrue(locations.length > 0);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateOutOfRangeSrc() throws Throwable {
-    describe("Seeking out of the file length returns an empty array");
-
-    BlockLocation[] locations =
-      getFs().getFileBlockLocations(createFileAndGetStatus(),
-                                    data.length + 100,
-                                    1);
-    assertEmptyBlockLocations(locations);
-  }
-
-  private void assertEmptyBlockLocations(BlockLocation[] locations) {
-    assertNotNull(locations);
-    if (locations.length!=0) {
-      fail("non empty locations[] with first entry of " + locations[0]);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateDirectory() throws Throwable {
-    describe("verify that locating a directory is an error");
-    createFile(path("/test/filename"));
-    FileStatus status = fs.getFileStatus(path("/test"));
-    LOG.info("Filesystem is " + fs + "; target is " + status);
-    SwiftTestUtils.assertIsDirectory(status);
-    BlockLocation[] locations;
-    locations = getFs().getFileBlockLocations(status,
-                                              0,
-                                              1);
-    assertEmptyBlockLocations(locations);
-  }
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLocateRootDirectory() throws Throwable {
-    describe("verify that locating the root directory is an error");
-    FileStatus status = fs.getFileStatus(path("/"));
-    SwiftTestUtils.assertIsDirectory(status);
-    BlockLocation[] locations;
-    locations = getFs().getFileBlockLocations(status,
-                                              0,
-                                              1);
-    assertEmptyBlockLocations(locations);
-  }
-
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java
deleted file mode 100644
index 0211163248681..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-/**
- * Tests that blocksize is never zero for a file, either in the FS default
- * or the FileStatus value of a queried file 
- */
-public class TestSwiftFileSystemBlocksize extends SwiftFileSystemBaseTest {
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDefaultBlocksizeNonZero() throws Throwable {
-    assertTrue("Zero default blocksize", 0L != getFs().getDefaultBlockSize());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDefaultBlocksizeRootPathNonZero() throws Throwable {
-    assertTrue("Zero default blocksize",
-               0L != getFs().getDefaultBlockSize(new Path("/")));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDefaultBlocksizeOtherPathNonZero() throws Throwable {
-    assertTrue("Zero default blocksize",
-               0L != getFs().getDefaultBlockSize(new Path("/test")));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testBlocksizeNonZeroForFile() throws Throwable {
-    Path smallfile = new Path("/test/smallfile");
-    SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
-    createFile(smallfile);
-    FileStatus status = getFs().getFileStatus(smallfile);
-    assertTrue("Zero blocksize in " + status,
-               status.getBlockSize() != 0L);
-    assertTrue("Zero replication in " + status,
-               status.getReplication() != 0L);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java
deleted file mode 100644
index c447919efa461..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Test Swift FS concurrency logic. This isn't a very accurate test,
- * because it is hard to consistently generate race conditions.
- * Consider it "best effort"
- */
-public class TestSwiftFileSystemConcurrency extends SwiftFileSystemBaseTest {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestSwiftFileSystemConcurrency.class);
-  private Exception thread1Ex, thread2Ex;
-  public static final String TEST_RACE_CONDITION_ON_DELETE_DIR =
-    "/test/testraceconditionondirdeletetest";
-
-  /**
-   * test on concurrent file system changes
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRaceConditionOnDirDeleteTest() throws Exception {
-    SwiftTestUtils.skip("Skipping unreliable test");
-
-    final String message = "message";
-    final Path fileToRead = new Path(
-      TEST_RACE_CONDITION_ON_DELETE_DIR +"/files/many-files/file");
-    final ExecutorService executorService = Executors.newFixedThreadPool(2);
-    fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR +"/file/test/file1"));
-    fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR + "/documents/doc1"));
-    fs.create(new Path(
-      TEST_RACE_CONDITION_ON_DELETE_DIR + "/pictures/picture"));
-
-
-    executorService.execute(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          assertDeleted(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR), true);
-        } catch (IOException e) {
-          LOG.warn("deletion thread:" + e, e);
-          thread1Ex = e;
-          throw new RuntimeException(e);
-        }
-      }
-    });
-    executorService.execute(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          final FSDataOutputStream outputStream = fs.create(fileToRead);
-          outputStream.write(message.getBytes());
-          outputStream.close();
-        } catch (IOException e) {
-          LOG.warn("writer thread:" + e, e);
-          thread2Ex = e;
-          throw new RuntimeException(e);
-        }
-      }
-    });
-
-    executorService.awaitTermination(1, TimeUnit.MINUTES);
-    if (thread1Ex != null) {
-      throw thread1Ex;
-    }
-    if (thread2Ex != null) {
-      throw thread2Ex;
-    }
-    try {
-      fs.open(fileToRead);
-      LOG.info("concurrency test failed to trigger a failure");
-    } catch (FileNotFoundException expected) {
-
-    }
-
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
deleted file mode 100644
index 1655b95231c1d..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * This is the full filesystem contract test -which requires the
- * Default config set up to point to a filesystem.
- *
- * Some of the tests override the base class tests -these
- * are where SwiftFS does not implement those features, or
- * when the behavior of SwiftFS does not match the normal
- * contract -which normally means that directories and equal files
- * are being treated as equal.
- */
-public class TestSwiftFileSystemContract
-        extends FileSystemContractBaseTest {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSwiftFileSystemContract.class);
-
-  /**
-   * Override this if the filesystem is not case sensitive
-   * @return true if the case detection/preservation tests should run
-   */
-  protected boolean filesystemIsCaseSensitive() {
-    return false;
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    final URI uri = getFilesystemURI();
-    final Configuration conf = new Configuration();
-    fs = createSwiftFS();
-    try {
-      fs.initialize(uri, conf);
-    } catch (IOException e) {
-      //FS init failed, set it to null so that teardown doesn't
-      //attempt to use it
-      fs = null;
-      throw e;
-    }
-  }
-
-  protected URI getFilesystemURI() throws URISyntaxException, IOException {
-    return SwiftTestUtils.getServiceURI(new Configuration());
-  }
-
-  protected SwiftNativeFileSystem createSwiftFS() throws IOException {
-    SwiftNativeFileSystem swiftNativeFileSystem =
-            new SwiftNativeFileSystem();
-    return swiftNativeFileSystem;
-  }
-
-  @Test
-  public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
-    Path testDir = path("/test/hadoop");
-    assertFalse(fs.exists(testDir));
-    assertTrue(fs.mkdirs(testDir));
-    assertTrue(fs.exists(testDir));
-
-    Path filepath = path("/test/hadoop/file");
-    SwiftTestUtils.writeTextFile(fs, filepath, "hello, world", false);
-
-    Path testSubDir = new Path(filepath, "subdir");
-    SwiftTestUtils.assertPathDoesNotExist(fs, "subdir before mkdir", testSubDir);
-
-    try {
-      fs.mkdirs(testSubDir);
-      fail("Should throw IOException.");
-    } catch (ParentNotDirectoryException e) {
-      // expected
-    }
-    //now verify that the subdir path does not exist
-    SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir);
-
-    Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir");
-    try {
-      fs.mkdirs(testDeepSubDir);
-      fail("Should throw IOException.");
-    } catch (ParentNotDirectoryException e) {
-      // expected
-    }
-    SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir  after mkdir",
-                                          testDeepSubDir);
-
-  }
-
-  @Test
-  public void testWriteReadAndDeleteEmptyFile() throws Exception {
-    try {
-      super.testWriteReadAndDeleteEmptyFile();
-    } catch (AssertionError e) {
-      SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
-    }
-  }
-
-  @Test
-  public void testMkdirsWithUmask() throws Exception {
-    //unsupported
-  }
-
-  @Test
-  public void testZeroByteFilesAreFiles() throws Exception {
-//    SwiftTestUtils.unsupported("testZeroByteFilesAreFiles");
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java
deleted file mode 100644
index 81af49c2a34e3..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-/**
- * Test deletion operations
- */
-public class TestSwiftFileSystemDelete extends SwiftFileSystemBaseTest {
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteEmptyFile() throws IOException {
-    final Path file = new Path("/test/testDeleteEmptyFile");
-    createEmptyFile(file);
-    SwiftTestUtils.noteAction("about to delete");
-    assertDeleted(file, true);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteEmptyFileTwice() throws IOException {
-    final Path file = new Path("/test/testDeleteEmptyFileTwice");
-    createEmptyFile(file);
-    assertDeleted(file, true);
-    SwiftTestUtils.noteAction("multiple creates, and deletes");
-    assertFalse("Delete returned true", fs.delete(file, false));
-    createEmptyFile(file);
-    assertDeleted(file, true);
-    assertFalse("Delete returned true", fs.delete(file, false));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteNonEmptyFile() throws IOException {
-    final Path file = new Path("/test/testDeleteNonEmptyFile");
-    createFile(file);
-    assertDeleted(file, true);
-  }
-  
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteNonEmptyFileTwice() throws IOException {
-    final Path file = new Path("/test/testDeleteNonEmptyFileTwice");
-    createFile(file);
-    assertDeleted(file, true);
-    assertFalse("Delete returned true", fs.delete(file, false));
-    createFile(file);
-    assertDeleted(file, true);
-    assertFalse("Delete returned true", fs.delete(file, false));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDeleteTestDir() throws IOException {
-    final Path file = new Path("/test/");
-    fs.delete(file, true);
-    assertPathDoesNotExist("Test dir found", file);
-  }
-
-  /**
-   * Test recursive root directory deletion fails if there is an entry underneath
-   * @throws Throwable
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRmRootDirRecursiveIsForbidden() throws Throwable {
-    Path root = path("/");
-    Path testFile = path("/test");
-    createFile(testFile);
-    assertTrue("rm(/) returned false", fs.delete(root, true));
-    assertExists("Root dir is missing", root);
-    assertPathDoesNotExist("test file not deleted", testFile);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
deleted file mode 100644
index 9b4ba5e8c6f90..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-
-/**
- * Test swift-specific directory logic.
- * This class is HDFS-1 compatible; its designed to be subclasses by something
- * with HDFS2 extensions
- */
-public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
-
-  /**
-   * Asserts that a zero byte file has a status of file and not
-   * directory or symlink
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testZeroByteFilesAreDirectories() throws Exception {
-    Path src = path("/test/testZeroByteFilesAreFiles");
-    //create a zero byte file
-    SwiftTestUtils.touch(fs, src);
-    SwiftTestUtils.assertIsDirectory(fs, src);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testNoStatusForMissingDirectories() throws Throwable {
-    Path missing = path("/test/testNoStatusForMissingDirectories");
-    assertPathDoesNotExist("leftover?", missing);
-    try {
-      FileStatus[] statuses = fs.listStatus(missing);
-      //not expected
-      fail("Expected a FileNotFoundException, got the status " + statuses);
-    } catch (FileNotFoundException expected) {
-      //expected
-    }
-  }
-
-  /**
-   * test that a dir off root has a listStatus() call that
-   * works as expected. and that when a child is added. it changes
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
-    Path test = path("/test");
-    fs.delete(test, true);
-    mkdirs(test);
-    assertExists("created test directory", test);
-    FileStatus[] statuses = fs.listStatus(test);
-    String statusString = statusToString(test.toString(), statuses);
-    assertEquals("Wrong number of elements in file status " + statusString, 0,
-                 statuses.length);
-
-    Path src = path("/test/file");
-
-    //create a zero byte file
-    SwiftTestUtils.touch(fs, src);
-    //stat it
-    statuses = fs.listStatus(test);
-    statusString = statusToString(test.toString(), statuses);
-    assertEquals("Wrong number of elements in file status " + statusString, 1,
-                 statuses.length);
-    SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
-    assertTrue("isDir(): Not a directory: " + stat, stat.isDirectory());
-    extraStatusAssertions(stat);
-  }
-
-  /**
-   * test that a dir two levels down has a listStatus() call that
-   * works as expected.
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
-    Path test = path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
-    fs.delete(test, true);
-    mkdirs(test);
-    assertExists("created test sub directory", test);
-    FileStatus[] statuses = fs.listStatus(test);
-    String statusString = statusToString(test.toString(), statuses);
-    assertEquals("Wrong number of elements in file status " + statusString,0,
-                 statuses.length);
-  }
-
-  private String statusToString(String pathname,
-                                FileStatus[] statuses) {
-    assertNotNull(statuses);
-    return SwiftTestUtils.dumpStats(pathname,statuses);
-  }
-
-  /**
-   * method for subclasses to add extra assertions
-   * @param stat status to look at
-   */
-  protected void extraStatusAssertions(SwiftFileStatus stat) {
-
-  }
-
-  /**
-   * Asserts that a zero byte file has a status of file and not
-   * directory or symlink
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testMultiByteFilesAreFiles() throws Exception {
-    Path src = path("/test/testMultiByteFilesAreFiles");
-    SwiftTestUtils.writeTextFile(fs, src, "testMultiByteFilesAreFiles", false);
-    assertIsFile(src);
-    FileStatus status = fs.getFileStatus(src);
-    assertFalse(status.isDirectory());
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java
deleted file mode 100644
index 844463db6d979..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.RestClientBindings;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.StringUtils;
-import org.junit.Test;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-
-public class TestSwiftFileSystemExtendedContract extends SwiftFileSystemBaseTest {
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testOpenNonExistingFile() throws IOException {
-    final Path p = new Path("/test/testOpenNonExistingFile");
-    //open it as a file, should get FileNotFoundException
-    try {
-      final FSDataInputStream in = fs.open(p);
-      in.close();
-      fail("didn't expect to get here");
-    } catch (FileNotFoundException fnfe) {
-      LOG.debug("Expected: " + fnfe, fnfe);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testFilesystemHasURI() throws Throwable {
-    assertNotNull(fs.getUri());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testCreateFile() throws Exception {
-    final Path f = new Path("/test/testCreateFile");
-    final FSDataOutputStream fsDataOutputStream = fs.create(f);
-    fsDataOutputStream.close();
-    assertExists("created file", f);
-  }
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testWriteReadFile() throws Exception {
-    final Path f = new Path("/test/test");
-    final FSDataOutputStream fsDataOutputStream = fs.create(f);
-    final String message = "Test string";
-    fsDataOutputStream.write(message.getBytes());
-    fsDataOutputStream.close();
-    assertExists("created file", f);
-    FSDataInputStream open = null;
-    try {
-      open = fs.open(f);
-      final byte[] bytes = new byte[512];
-      final int read = open.read(bytes);
-      final byte[] buffer = new byte[read];
-      System.arraycopy(bytes, 0, buffer, 0, read);
-      assertEquals(message, new String(buffer));
-    } finally {
-      fs.delete(f, false);
-      IOUtils.closeStream(open);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testConfDefinesFilesystem() throws Throwable {
-    Configuration conf = new Configuration();
-    SwiftTestUtils.getServiceURI(conf);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testConfIsValid() throws Throwable {
-    Configuration conf = new Configuration();
-    URI fsURI = SwiftTestUtils.getServiceURI(conf);
-    RestClientBindings.bind(fsURI, conf);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testGetSchemeImplemented() throws Throwable {
-    String scheme = fs.getScheme();
-    assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
-  }
-
-  /**
-   * Assert that a filesystem is case sensitive.
-   * This is done by creating a mixed-case filename and asserting that
-   * its lower case version is not there.
-   *
-   * @throws Exception failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testFilesystemIsCaseSensitive() throws Exception {
-    String mixedCaseFilename = "/test/UPPER.TXT";
-    Path upper = path(mixedCaseFilename);
-    Path lower = path(StringUtils.toLowerCase(mixedCaseFilename));
-    assertFalse("File exists" + upper, fs.exists(upper));
-    assertFalse("File exists" + lower, fs.exists(lower));
-    FSDataOutputStream out = fs.create(upper);
-    out.writeUTF("UPPER");
-    out.close();
-    FileStatus upperStatus = fs.getFileStatus(upper);
-    assertExists("Original upper case file" + upper, upper);
-    //verify the lower-case version of the filename doesn't exist
-    assertPathDoesNotExist("lower case file", lower);
-    //now overwrite the lower case version of the filename with a
-    //new version.
-    out = fs.create(lower);
-    out.writeUTF("l");
-    out.close();
-    assertExists("lower case file", lower);
-    //verify the length of the upper file hasn't changed
-    assertExists("Original upper case file " + upper, upper);
-    FileStatus newStatus = fs.getFileStatus(upper);
-    assertEquals("Expected status:" + upperStatus
-            + " actual status " + newStatus,
-            upperStatus.getLen(),
-            newStatus.getLen());
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java
deleted file mode 100644
index 5e2b1b7231910..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertListStatusFinds;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dumpStats;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.touch;
-
-/**
- * Test the FileSystem#listStatus() operations
- */
-public class TestSwiftFileSystemLsOperations extends SwiftFileSystemBaseTest {
-
-  private Path[] testDirs;
-
-    /**
-   * Setup creates dirs under test/hadoop
-   *
-   * @throws Exception
-   */
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-    //delete the test directory
-    Path test = path("/test");
-    fs.delete(test, true);
-    mkdirs(test);
-  }
-
-  /**
-   * Create subdirectories and files under test/ for those tests
-   * that want them. Doing so adds overhead to setup and teardown,
-   * so should only be done for those tests that need them.
-   * @throws IOException on an IO problem
-   */
-  private void createTestSubdirs() throws IOException {
-    testDirs = new Path[]{
-              path("/test/hadoop/a"),
-              path("/test/hadoop/b"),
-              path("/test/hadoop/c/1"),
-      };
-
-    assertPathDoesNotExist("test directory setup", testDirs[0]);
-    for (Path path : testDirs) {
-      mkdirs(path);
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListLevelTest() throws Exception {
-    createTestSubdirs();
-    FileStatus[] paths = fs.listStatus(path("/test"));
-    assertEquals(dumpStats("/test", paths), 1, paths.length);
-    assertEquals(path("/test/hadoop"), paths[0].getPath());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListLevelTestHadoop() throws Exception {
-    createTestSubdirs();
-    FileStatus[] paths;
-    paths = fs.listStatus(path("/test/hadoop"));
-    String stats = dumpStats("/test/hadoop", paths);
-    assertEquals("Paths.length wrong in " + stats, 3, paths.length);
-    assertEquals("Path element[0] wrong: " + stats, path("/test/hadoop/a"),
-                 paths[0].getPath());
-    assertEquals("Path element[1] wrong: " + stats, path("/test/hadoop/b"),
-                 paths[1].getPath());
-    assertEquals("Path element[2] wrong: " + stats, path("/test/hadoop/c"),
-                 paths[2].getPath());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListStatusEmptyDirectory() throws Exception {
-    createTestSubdirs();
-    FileStatus[] paths;
-    paths = fs.listStatus(path("/test/hadoop/a"));
-    assertEquals(dumpStats("/test/hadoop/a", paths), 0,
-                 paths.length);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListStatusFile() throws Exception {
-    describe("Create a single file under /test;" +
-             " assert that listStatus(/test) finds it");
-    Path file = path("/test/filename");
-    createFile(file);
-    FileStatus[] pathStats = fs.listStatus(file);
-    assertEquals(dumpStats("/test/", pathStats),
-                 1,
-                 pathStats.length);
-    //and assert that the len of that ls'd path is the same as the original
-    FileStatus lsStat = pathStats[0];
-    assertEquals("Wrong file len in listing of " + lsStat,
-      data.length, lsStat.getLen());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListEmptyRoot() throws Throwable {
-    describe("Empty the root dir and verify that an LS / returns {}");
-    cleanup("testListEmptyRoot", fs, "/test");
-    cleanup("testListEmptyRoot", fs, "/user");
-    FileStatus[] fileStatuses = fs.listStatus(path("/"));
-    assertEquals("Non-empty root" + dumpStats("/", fileStatuses),
-                 0,
-                 fileStatuses.length);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListNonEmptyRoot() throws Throwable {
-    Path test = path("/test");
-    touch(fs, test);
-    FileStatus[] fileStatuses = fs.listStatus(path("/"));
-    String stats = dumpStats("/", fileStatuses);
-    assertEquals("Wrong #of root children" + stats, 1, fileStatuses.length);
-    FileStatus status = fileStatuses[0];
-    assertEquals("Wrong path value" + stats,test, status.getPath());
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListStatusRootDir() throws Throwable {
-    Path dir = path("/");
-    Path child = path("/test");
-    touch(fs, child);
-    assertListStatusFinds(fs, dir, child);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListStatusFiltered() throws Throwable {
-    Path dir = path("/");
-    Path child = path("/test");
-    touch(fs, child);
-    FileStatus[] stats = fs.listStatus(dir, new AcceptAllFilter());
-    boolean found = false;
-    StringBuilder builder = new StringBuilder();
-    for (FileStatus stat : stats) {
-      builder.append(stat.toString()).append('\n');
-      if (stat.getPath().equals(child)) {
-        found = true;
-      }
-    }
-    assertTrue("Path " + child
-                      + " not found in directory " + dir + ":" + builder,
-                      found);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
deleted file mode 100644
index 419d0303a0402..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
+++ /dev/null
@@ -1,442 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.http.Header;
-import org.junit.Test;
-import org.junit.internal.AssumptionViolatedException;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
-
-/**
- * Test partitioned uploads.
- * This is done by forcing a very small partition size and verifying that it
- * is picked up.
- */
-public class TestSwiftFileSystemPartitionedUploads extends
-                                                   SwiftFileSystemBaseTest {
-
-  public static final String WRONG_PARTITION_COUNT =
-    "wrong number of partitions written into ";
-  public static final int PART_SIZE = 1;
-  public static final int PART_SIZE_BYTES = PART_SIZE * 1024;
-  public static final int BLOCK_SIZE = 1024;
-  private URI uri;
-
-  @Override
-  protected Configuration createConfiguration() {
-    Configuration conf = super.createConfiguration();
-    //set the partition size to 1 KB
-    conf.setInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE, PART_SIZE);
-    return conf;
-  }
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testPartitionPropertyPropagatesToConf() throws Throwable {
-    assertEquals(1,
-                 getConf().getInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE,
-                                  0));
-  }
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testPartionPropertyPropagatesToStore() throws Throwable {
-    assertEquals(1, fs.getStore().getPartsizeKB());
-  }
-
-  /**
-   * tests functionality for big files ( > 5Gb) upload
-   */
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testFilePartUpload() throws Throwable {
-
-    final Path path = new Path("/test/testFilePartUpload");
-
-    int len = 8192;
-    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
-    FSDataOutputStream out = fs.create(path,
-                                       false,
-                                       getBufferSize(),
-                                       (short) 1,
-                                       BLOCK_SIZE);
-
-    try {
-      int totalPartitionsToWrite = len / PART_SIZE_BYTES;
-      assertPartitionsWritten("Startup", out, 0);
-      //write 2048
-      int firstWriteLen = 2048;
-      out.write(src, 0, firstWriteLen);
-      //assert
-      long expected = getExpectedPartitionsWritten(firstWriteLen,
-                                                   PART_SIZE_BYTES,
-                                                   false);
-      SwiftUtils.debug(LOG, "First write: predict %d partitions written",
-                       expected);
-      assertPartitionsWritten("First write completed", out, expected);
-      //write the rest
-      int remainder = len - firstWriteLen;
-      SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
-
-      out.write(src, firstWriteLen, remainder);
-      expected =
-        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
-      assertPartitionsWritten("Remaining data", out, expected);
-      out.close();
-      expected =
-        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
-      assertPartitionsWritten("Stream closed", out, expected);
-
-      Header[] headers = fs.getStore().getObjectHeaders(path, true);
-      for (Header header : headers) {
-        LOG.info(header.toString());
-      }
-
-      byte[] dest = readDataset(fs, path, len);
-      LOG.info("Read dataset from " + path + ": data length =" + len);
-      //compare data
-      SwiftTestUtils.compareByteArrays(src, dest, len);
-      FileStatus status;
-
-      final Path qualifiedPath = fs.makeQualified(path);
-      status = fs.getFileStatus(qualifiedPath);
-      //now see what block location info comes back.
-      //This will vary depending on the Swift version, so the results
-      //aren't checked -merely that the test actually worked
-      BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
-      assertNotNull("Null getFileBlockLocations()", locations);
-      assertTrue("empty array returned for getFileBlockLocations()",
-                 locations.length > 0);
-
-      //last bit of test -which seems to play up on partitions, which we download
-      //to a skip
-      try {
-        validatePathLen(path, len);
-      } catch (AssertionError e) {
-        //downgrade to a skip
-        throw new AssumptionViolatedException(e, null);
-      }
-
-    } finally {
-      IOUtils.closeStream(out);
-    }
-  }
-  /**
-   * tests functionality for big files ( > 5Gb) upload
-   */
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
-
-    final Path path = new Path("/test/testFilePartUploadLengthCheck");
-
-    int len = 8192;
-    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
-    FSDataOutputStream out = fs.create(path,
-                                       false,
-                                       getBufferSize(),
-                                       (short) 1,
-                                       BLOCK_SIZE);
-
-    try {
-      int totalPartitionsToWrite = len / PART_SIZE_BYTES;
-      assertPartitionsWritten("Startup", out, 0);
-      //write 2048
-      int firstWriteLen = 2048;
-      out.write(src, 0, firstWriteLen);
-      //assert
-      long expected = getExpectedPartitionsWritten(firstWriteLen,
-                                                   PART_SIZE_BYTES,
-                                                   false);
-      SwiftUtils.debug(LOG, "First write: predict %d partitions written",
-                       expected);
-      assertPartitionsWritten("First write completed", out, expected);
-      //write the rest
-      int remainder = len - firstWriteLen;
-      SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
-
-      out.write(src, firstWriteLen, remainder);
-      expected =
-        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
-      assertPartitionsWritten("Remaining data", out, expected);
-      out.close();
-      expected =
-        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
-      assertPartitionsWritten("Stream closed", out, expected);
-
-      Header[] headers = fs.getStore().getObjectHeaders(path, true);
-      for (Header header : headers) {
-        LOG.info(header.toString());
-      }
-
-      byte[] dest = readDataset(fs, path, len);
-      LOG.info("Read dataset from " + path + ": data length =" + len);
-      //compare data
-      SwiftTestUtils.compareByteArrays(src, dest, len);
-      FileStatus status = fs.getFileStatus(path);
-
-      //now see what block location info comes back.
-      //This will vary depending on the Swift version, so the results
-      //aren't checked -merely that the test actually worked
-      BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
-      assertNotNull("Null getFileBlockLocations()", locations);
-      assertTrue("empty array returned for getFileBlockLocations()",
-                 locations.length > 0);
-    } finally {
-      IOUtils.closeStream(out);
-    }
-  }
-
-  private FileStatus validatePathLen(Path path, int len) throws IOException {
-    //verify that the length is what was written in a direct status check
-    final Path qualifiedPath = fs.makeQualified(path);
-    FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
-    StringBuilder listing = lsToString(parentDirListing);
-    String parentDirLS = listing.toString();
-    FileStatus status = fs.getFileStatus(qualifiedPath);
-    assertEquals("Length of written file " + qualifiedPath
-                 + " from status check " + status
-                 + " in dir " + listing,
-                 len,
-                 status.getLen());
-    String fileInfo = qualifiedPath + "  " + status;
-    assertFalse("File claims to be a directory " + fileInfo,
-                status.isDirectory());
-
-    FileStatus listedFileStat = resolveChild(parentDirListing, qualifiedPath);
-    assertNotNull("Did not find " + path + " in " + parentDirLS,
-                  listedFileStat);
-    //file is in the parent dir. Now validate it's stats
-    assertEquals("Wrong len for " + path + " in listing " + parentDirLS,
-                 len,
-                 listedFileStat.getLen());
-    listedFileStat.toString();
-    return status;
-  }
-
-  private FileStatus resolveChild(FileStatus[] parentDirListing,
-                                  Path childPath) {
-    FileStatus listedFileStat = null;
-    for (FileStatus stat : parentDirListing) {
-      if (stat.getPath().equals(childPath)) {
-        listedFileStat = stat;
-      }
-    }
-    return listedFileStat;
-  }
-
-  private StringBuilder lsToString(FileStatus[] parentDirListing) {
-    StringBuilder listing = new StringBuilder();
-    for (FileStatus stat : parentDirListing) {
-      listing.append(stat).append("\n");
-    }
-    return listing;
-  }
-
-  /**
-   * Calculate the #of partitions expected from the upload
-   * @param uploaded number of bytes uploaded
-   * @param partSizeBytes the partition size
-   * @param closed whether or not the stream has closed
-   * @return the expected number of partitions, for use in assertions.
-   */
-  private int getExpectedPartitionsWritten(long uploaded,
-                                           int partSizeBytes,
-                                           boolean closed) {
-    //#of partitions in total
-    int partitions = (int) (uploaded / partSizeBytes);
-    //#of bytes past the last partition
-    int remainder = (int) (uploaded % partSizeBytes);
-    if (closed) {
-      //all data is written, so if there was any remainder, it went up
-      //too
-      return partitions + ((remainder > 0) ? 1 : 0);
-    } else {
-      //not closed. All the remainder is buffered,
-      return partitions;
-    }
-  }
-
-  private int getBufferSize() {
-    return fs.getConf().getInt("io.file.buffer.size", 4096);
-  }
-
-  /**
-   * Test sticks up a very large partitioned file and verifies that
-   * it comes back unchanged.
-   * @throws Throwable
-   */
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testManyPartitionedFile() throws Throwable {
-    final Path path = new Path("/test/testManyPartitionedFile");
-
-    int len = PART_SIZE_BYTES * 15;
-    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
-    FSDataOutputStream out = fs.create(path,
-                                       false,
-                                       getBufferSize(),
-                                       (short) 1,
-                                       BLOCK_SIZE);
-
-    out.write(src, 0, src.length);
-    int expected =
-      getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
-    out.close();
-    assertPartitionsWritten("write completed", out, expected);
-    assertEquals("too few bytes written", len,
-                 SwiftNativeFileSystem.getBytesWritten(out));
-    assertEquals("too few bytes uploaded", len,
-                 SwiftNativeFileSystem.getBytesUploaded(out));
-    //now we verify that the data comes back. If it
-    //doesn't, it means that the ordering of the partitions
-    //isn't right
-    byte[] dest = readDataset(fs, path, len);
-    //compare data
-    SwiftTestUtils.compareByteArrays(src, dest, len);
-    //finally, check the data
-    FileStatus[] stats = fs.listStatus(path);
-    assertEquals("wrong entry count in "
-                 + SwiftTestUtils.dumpStats(path.toString(), stats),
-                 expected, stats.length);
-  }
-
-  /**
-   * Test that when a partitioned file is overwritten by a smaller one,
-   * all the old partitioned files go away
-   * @throws Throwable
-   */
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testOverwritePartitionedFile() throws Throwable {
-    final Path path = new Path("/test/testOverwritePartitionedFile");
-
-    final int len1 = 8192;
-    final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
-    FSDataOutputStream out = fs.create(path,
-                                       false,
-                                       getBufferSize(),
-                                       (short) 1,
-                                       1024);
-    out.write(src1, 0, len1);
-    out.close();
-    long expected = getExpectedPartitionsWritten(len1,
-                                                 PART_SIZE_BYTES,
-                                                 false);
-    assertPartitionsWritten("initial upload", out, expected);
-    assertExists("Exists", path);
-    FileStatus status = fs.getFileStatus(path);
-    assertEquals("Length", len1, status.getLen());
-    //now write a shorter file with a different dataset
-    final int len2 = 4095;
-    final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
-    out = fs.create(path,
-                    true,
-                    getBufferSize(),
-                    (short) 1,
-                    1024);
-    out.write(src2, 0, len2);
-    out.close();
-    status = fs.getFileStatus(path);
-    assertEquals("Length", len2, status.getLen());
-    byte[] dest = readDataset(fs, path, len2);
-    //compare data
-    SwiftTestUtils.compareByteArrays(src2, dest, len2);
-  }
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testDeleteSmallPartitionedFile() throws Throwable {
-    final Path path = new Path("/test/testDeleteSmallPartitionedFile");
-
-    final int len1 = 1024;
-    final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
-    SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
-    assertExists("Exists", path);
-
-    Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
-    Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
-    String ls = SwiftTestUtils.ls(fs, path);
-    assertExists("Partition 0001 Exists in " + ls, part_0001);
-    assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
-    assertExists("Partition 0002 Exists in " + ls, part_0001);
-    fs.delete(path, false);
-    assertPathDoesNotExist("deleted file still there", path);
-    ls = SwiftTestUtils.ls(fs, path);
-    assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
-  }
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testDeletePartitionedFile() throws Throwable {
-    final Path path = new Path("/test/testDeletePartitionedFile");
-
-    SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
-    assertExists("Exists", path);
-
-    Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
-    Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
-    String ls = SwiftTestUtils.ls(fs, path);
-    assertExists("Partition 0001 Exists in " + ls, part_0001);
-    assertExists("Partition 0002 Exists in " + ls, part_0001);
-    fs.delete(path, false);
-    assertPathDoesNotExist("deleted file still there", path);
-    ls = SwiftTestUtils.ls(fs, path);
-    assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
-    assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
-  }
-
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testRenamePartitionedFile() throws Throwable {
-    Path src = new Path("/test/testRenamePartitionedFileSrc");
-
-    int len = data.length;
-    SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
-    assertExists("Exists", src);
-
-    String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
-    Path srcPart = new Path(src, partOneName);
-    Path dest = new Path("/test/testRenamePartitionedFileDest");
-    Path destPart = new Path(src, partOneName);
-    assertExists("Partition Exists", srcPart);
-    fs.rename(src, dest);
-    assertPathExists(fs, "dest file missing", dest);
-    FileStatus status = fs.getFileStatus(dest);
-    assertEquals("Length of renamed file is wrong", len, status.getLen());
-    byte[] destData = readDataset(fs, dest, len);
-    //compare data
-    SwiftTestUtils.compareByteArrays(data, destData, len);
-    String srcLs = SwiftTestUtils.ls(fs, src);
-    String destLs = SwiftTestUtils.ls(fs, dest);
-
-    assertPathDoesNotExist("deleted file still found in " + srcLs, src);
-
-    assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
-  }
-
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java
deleted file mode 100644
index 84794cb725074..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
-
-/**
- * Test filesystem read operations
- */
-public class TestSwiftFileSystemRead extends SwiftFileSystemBaseTest {
-
-
-  /**
-   * Read past the end of a file: expect the operation to fail
-   * @throws IOException
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testOverRead() throws IOException {
-    final String message = "message";
-    final Path filePath = new Path("/test/file.txt");
-
-    writeTextFile(fs, filePath, message, false);
-
-    try {
-      readBytesToString(fs, filePath, 20);
-      fail("expected an exception");
-    } catch (EOFException e) {
-      //expected
-    }
-  }
-
-  /**
-   * Read and write some JSON
-   * @throws IOException
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRWJson() throws IOException {
-    final String message = "{" +
-                           " 'json': { 'i':43, 'b':true}," +
-                           " 's':'string'" +
-                           "}";
-    final Path filePath = new Path("/test/file.json");
-
-    writeTextFile(fs, filePath, message, false);
-    String readJson = readBytesToString(fs, filePath, message.length());
-    assertEquals(message,readJson);
-    //now find out where it is
-    FileStatus status = fs.getFileStatus(filePath);
-    BlockLocation[] locations = fs.getFileBlockLocations(status, 0, 10);
-  }
-
-  /**
-   * Read and write some XML
-   * @throws IOException
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRWXML() throws IOException {
-    final String message = "<x>" +
-                           " <json i='43' 'b'=true/>" +
-                           " string" +
-                           "</x>";
-    final Path filePath = new Path("/test/file.xml");
-
-    writeTextFile(fs, filePath, message, false);
-    String read = readBytesToString(fs, filePath, message.length());
-    assertEquals(message,read);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
deleted file mode 100644
index f5ad155ffe3d7..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.compareByteArrays;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dataset;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeDataset;
-
-public class TestSwiftFileSystemRename extends SwiftFileSystemBaseTest {
-
-  /**
-   * Rename a file into a directory
-   *
-   * @throws Exception
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameFileIntoExistingDirectory() throws Exception {
-    assumeRenameSupported();
-
-    Path src = path("/test/olddir/file");
-    createFile(src);
-    Path dst = path("/test/new/newdir");
-    fs.mkdirs(dst);
-    rename(src, dst, true, false, true);
-    Path newFile = path("/test/new/newdir/file");
-    if (!fs.exists(newFile)) {
-      String ls = ls(dst);
-      LOG.info(ls(path("/test/new")));
-      LOG.info(ls(path("/test/hadoop")));
-      fail("did not find " + newFile + " - directory: " + ls);
-    }
-    assertTrue("Destination changed",
-            fs.exists(path("/test/new/newdir/file")));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameFile() throws Exception {
-    assumeRenameSupported();
-
-    final Path old = new Path("/test/alice/file");
-    final Path newPath = new Path("/test/bob/file");
-    fs.mkdirs(newPath.getParent());
-    final FSDataOutputStream fsDataOutputStream = fs.create(old);
-    final byte[] message = "Some data".getBytes();
-    fsDataOutputStream.write(message);
-    fsDataOutputStream.close();
-
-    assertTrue(fs.exists(old));
-    rename(old, newPath, true, false, true);
-
-    final FSDataInputStream bobStream = fs.open(newPath);
-    final byte[] bytes = new byte[512];
-    final int read = bobStream.read(bytes);
-    bobStream.close();
-    final byte[] buffer = new byte[read];
-    System.arraycopy(bytes, 0, buffer, 0, read);
-    assertEquals(new String(message), new String(buffer));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameDirectory() throws Exception {
-    assumeRenameSupported();
-
-    final Path old = new Path("/test/data/logs");
-    final Path newPath = new Path("/test/var/logs");
-    fs.mkdirs(old);
-    fs.mkdirs(newPath.getParent());
-    assertTrue(fs.exists(old));
-    rename(old, newPath, true, false, true);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameTheSameDirectory() throws Exception {
-    assumeRenameSupported();
-
-    final Path old = new Path("/test/usr/data");
-    fs.mkdirs(old);
-    rename(old, old, false, true, true);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameDirectoryIntoExistingDirectory() throws Exception {
-    assumeRenameSupported();
-
-    Path src = path("/test/olddir/dir");
-    fs.mkdirs(src);
-    createFile(path("/test/olddir/dir/file1"));
-    createFile(path("/test/olddir/dir/subdir/file2"));
-
-    Path dst = path("/test/new/newdir");
-    fs.mkdirs(dst);
-    //this renames into a child
-    rename(src, dst, true, false, true);
-    assertExists("new dir", path("/test/new/newdir/dir"));
-    assertExists("Renamed nested file1", path("/test/new/newdir/dir/file1"));
-    assertPathDoesNotExist("Nested file1 should have been deleted",
-            path("/test/olddir/dir/file1"));
-    assertExists("Renamed nested subdir",
-            path("/test/new/newdir/dir/subdir/"));
-    assertExists("file under subdir",
-            path("/test/new/newdir/dir/subdir/file2"));
-
-    assertPathDoesNotExist("Nested /test/hadoop/dir/subdir/file2 still exists",
-            path("/test/olddir/dir/subdir/file2"));
-  }
-
-  /**
-   * trying to rename a directory onto itself should fail,
-   * preserving everything underneath.
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameDirToSelf() throws Throwable {
-    assumeRenameSupported();
-    Path parentdir = path("/test/parentdir");
-    fs.mkdirs(parentdir);
-    Path child = new Path(parentdir, "child");
-    createFile(child);
-
-    rename(parentdir, parentdir, false, true, true);
-    //verify the child is still there
-    assertIsFile(child);
-  }
-
-  /**
-   * Assert that root directory renames are not allowed
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameRootDirForbidden() throws Exception {
-    assumeRenameSupported();
-    rename(path("/"),
-            path("/test/newRootDir"),
-            false, true, false);
-  }
-
-  /**
-   * Assert that renaming a parent directory to be a child
-   * of itself is forbidden
-   *
-   * @throws Exception on failures
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameChildDirForbidden() throws Exception {
-    assumeRenameSupported();
-
-    Path parentdir = path("/test/parentdir");
-    fs.mkdirs(parentdir);
-    Path childFile = new Path(parentdir, "childfile");
-    createFile(childFile);
-    //verify one level down
-    Path childdir = new Path(parentdir, "childdir");
-    rename(parentdir, childdir, false, true, false);
-    //now another level
-    fs.mkdirs(childdir);
-    Path childchilddir = new Path(childdir, "childdir");
-    rename(parentdir, childchilddir, false, true, false);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameFileAndVerifyContents() throws IOException {
-    assumeRenameSupported();
-
-    final Path filePath = new Path("/test/home/user/documents/file.txt");
-    final Path newFilePath = new Path("/test/home/user/files/file.txt");
-    mkdirs(newFilePath.getParent());
-    int len = 1024;
-    byte[] dataset = dataset(len, 'A', 26);
-    writeDataset(fs, filePath, dataset, len, len, false);
-    rename(filePath, newFilePath, true, false, true);
-    byte[] dest = readDataset(fs, newFilePath, len);
-    compareByteArrays(dataset, dest, len);
-    String reread = readBytesToString(fs, newFilePath, 20);
-  }
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testMoveFileUnderParent() throws Throwable {
-    if (!renameSupported()) return;
-    Path filepath = path("test/file");
-    createFile(filepath);
-    //HDFS expects rename src, src -> true
-    rename(filepath, filepath, true, true, true);
-    //verify the file is still there
-    assertIsFile(filepath);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testMoveDirUnderParent() throws Throwable {
-    if (!renameSupported()) {
-      return;
-    }
-    Path testdir = path("test/dir");
-    fs.mkdirs(testdir);
-    Path parent = testdir.getParent();
-    //the outcome here is ambiguous, so is not checked
-    try {
-      fs.rename(testdir, parent);
-    } catch (SwiftOperationFailedException e) {
-      // allowed
-    }
-    assertExists("Source directory has been deleted ", testdir);
-  }
-
-  /**
-   * trying to rename a file onto itself should succeed (it's a no-op)
-   */
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameFileToSelf() throws Throwable {
-    if (!renameSupported()) return;
-    Path filepath = path("test/file");
-    createFile(filepath);
-    //HDFS expects rename src, src -> true
-    rename(filepath, filepath, true, true, true);
-    //verify the file is still there
-    assertIsFile(filepath);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenamedConsistence() throws IOException {
-    assumeRenameSupported();
-    describe("verify that overwriting a file with new data doesn't impact" +
-            " the existing content");
-
-    final Path filePath = new Path("/test/home/user/documents/file.txt");
-    final Path newFilePath = new Path("/test/home/user/files/file.txt");
-    mkdirs(newFilePath.getParent());
-    int len = 1024;
-    byte[] dataset = dataset(len, 'A', 26);
-    byte[] dataset2 = dataset(len, 'a', 26);
-    writeDataset(fs, filePath, dataset, len, len, false);
-    rename(filePath, newFilePath, true, false, true);
-    SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
-    byte[] dest = readDataset(fs, newFilePath, len);
-    compareByteArrays(dataset, dest, len);
-    String reread = readBytesToString(fs, newFilePath, 20);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRenameMissingFile() throws Throwable {
-    assumeRenameSupported();
-    Path path = path("/test/RenameMissingFile");
-    Path path2 = path("/test/RenameMissingFileDest");
-    mkdirs(path("test"));
-    rename(path, path2, false, false, false);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java
deleted file mode 100644
index 5692b48f116be..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.swift;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.http.RestClientBindings;
-import org.apache.hadoop.fs.swift.http.SwiftRestClient;
-import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
-import org.apache.hadoop.fs.swift.util.SwiftUtils;
-import org.junit.Test;
-
-import java.net.URI;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Unit tests for SwiftObjectPath class.
- */
-public class TestSwiftObjectPath implements SwiftTestConstants {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSwiftObjectPath.class);
-
-  /**
-   * What an endpoint looks like. This is derived from a (valid)
-   * rackspace endpoint address
-   */
-  private static final String ENDPOINT =
-          "https://storage101.region1.example.org/v1/MossoCloudFS_9fb40cc0-1234-5678-9abc-def000c9a66";
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testParsePath() throws Exception {
-    final String pathString = "/home/user/files/file1";
-    final Path path = new Path(pathString);
-    final URI uri = new URI("http://container.localhost");
-    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
-    final SwiftObjectPath actual = new SwiftObjectPath(
-            RestClientBindings.extractContainerName(uri),
-            pathString);
-
-    assertEquals(expected, actual);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testParseUrlPath() throws Exception {
-    final String pathString = "swift://container.service1/home/user/files/file1";
-    final URI uri = new URI(pathString);
-    final Path path = new Path(pathString);
-    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
-    final SwiftObjectPath actual = new SwiftObjectPath(
-            RestClientBindings.extractContainerName(uri),
-            "/home/user/files/file1");
-
-    assertEquals(expected, actual);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testHandleUrlAsPath() throws Exception {
-    final String hostPart = "swift://container.service1";
-    final String pathPart = "/home/user/files/file1";
-    final String uriString = hostPart + pathPart;
-
-    final SwiftObjectPath expected = new SwiftObjectPath(uriString, pathPart);
-    final SwiftObjectPath actual = new SwiftObjectPath(uriString, uriString);
-
-    assertEquals(expected, actual);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testParseAuthenticatedUrl() throws Exception {
-    final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
-    final URI uri = new URI(pathString);
-    final Path path = new Path(pathString);
-    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
-    final SwiftObjectPath actual = new SwiftObjectPath(
-            RestClientBindings.extractContainerName(uri),
-            "/home/tom/documents/finance.docx");
-
-    assertEquals(expected, actual);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testConvertToPath() throws Throwable {
-    String initialpath = "/dir/file1";
-    Path ipath = new Path(initialpath);
-    SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
-            ipath);
-    URI endpoint = new URI(ENDPOINT);
-    URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
-    LOG.info("Inital Hadoop Path =" + initialpath);
-    LOG.info("Merged URI=" + uri);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRootDirProbeEmptyPath() throws Throwable {
-    SwiftObjectPath object=new SwiftObjectPath("container","");
-    assertTrue(SwiftUtils.isRootDir(object));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testRootDirProbeRootPath() throws Throwable {
-    SwiftObjectPath object=new SwiftObjectPath("container","/");
-    assertTrue(SwiftUtils.isRootDir(object));
-  }
-
-  private void assertParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
-    assertTrue(p1.toString() + " is not a parent of " + p2 ,p1.isEqualToOrParentOf(
-      p2));
-  }
-
-  private void assertNotParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
-    assertFalse(p1.toString() + " is a parent of " + p2, p1.isEqualToOrParentOf(
-      p2));
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testChildOfProbe() throws Throwable {
-    SwiftObjectPath parent = new SwiftObjectPath("container",
-                                                 "/parent");
-    SwiftObjectPath parent2 = new SwiftObjectPath("container",
-                                                 "/parent2");
-    SwiftObjectPath child = new SwiftObjectPath("container",
-                                                 "/parent/child");
-    SwiftObjectPath sibling = new SwiftObjectPath("container",
-                                                 "/parent/sibling");
-    SwiftObjectPath grandchild = new SwiftObjectPath("container",
-                                                     "/parent/child/grandchild");
-    assertParentOf(parent, child);
-    assertParentOf(parent, grandchild);
-    assertParentOf(child, grandchild);
-    assertParentOf(parent, parent);
-    assertNotParentOf(child, parent);
-    assertParentOf(child, child);
-    assertNotParentOf(parent, parent2);
-    assertNotParentOf(grandchild, parent);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testChildOfRoot() throws Throwable {
-    SwiftObjectPath root = new SwiftObjectPath("container", "/");
-    SwiftObjectPath child = new SwiftObjectPath("container", "child");
-    SwiftObjectPath grandchild = new SwiftObjectPath("container",
-                                                     "/child/grandchild");
-    assertParentOf(root, child);
-    assertParentOf(root, grandchild);
-    assertParentOf(child, grandchild);
-    assertParentOf(root, root);
-    assertNotParentOf(child, root);
-    assertParentOf(child, child);
-    assertNotParentOf(grandchild, root);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java
deleted file mode 100644
index 99f72b7be981a..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
-import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
-
-/**
- * The contract of OpenStack Swift: only enabled if the test binding data is provided
- */
-public class SwiftContract extends AbstractBondedFSContract {
-
-  public static final String CONTRACT_XML = "contract/swift.xml";
-
-  public SwiftContract(Configuration conf) {
-    super(conf);
-    //insert the base features
-    addConfResource(CONTRACT_XML);
-  }
-
-
-  @Override
-  public String getScheme() {
-    return SwiftNativeFileSystem.SWIFT;
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java
deleted file mode 100644
index df15a0a84c31e..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestSwiftContractCreate extends AbstractContractCreateTest {
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
-  }
-
-  @Override
-  public void testOverwriteEmptyDirectory() throws Throwable {
-    ContractTestUtils.skip("blobstores can't distinguish empty directories from files");
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java
deleted file mode 100644
index b82ba776386e9..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestSwiftContractMkdir extends AbstractContractMkdirTest {
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java
deleted file mode 100644
index 0f91b6f823e13..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestSwiftContractOpen extends AbstractContractOpenTest {
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
-  }
-
-  @Override
-  public void testOpenReadDir() throws Throwable {
-    ContractTestUtils.skip("Skipping object-store quirk");
-  }
-
-  @Override
-  public void testOpenReadDirWithChild() throws Throwable {
-    ContractTestUtils.skip("Skipping object-store quirk");
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java
deleted file mode 100644
index c7b766edd49ff..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * root dir operations against an S3 bucket
- */
-public class TestSwiftContractRootDir extends
-    AbstractContractRootDirectoryTest {
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java
deleted file mode 100644
index d045980e698cc..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestSwiftContractSeek extends AbstractContractSeekTest {
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new SwiftContract(conf);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java
deleted file mode 100644
index cb64bef6c5cd7..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.hdfs2;
-
-import org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories;
-import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
-
-/**
- * Add some HDFS-2 only assertions to {@link TestSwiftFileSystemDirectories}
- */
-public class TestSwiftFileSystemDirectoriesHdfs2 extends
-                                                 TestSwiftFileSystemDirectories {
-
-
-  /**
-   * make assertions about fields that only appear in
-   * FileStatus in HDFS2
-   * @param stat status to look at
-   */
-  protected void extraStatusAssertions(SwiftFileStatus stat) {
-    //HDFS2
-    assertTrue("isDirectory(): Not a directory: " + stat, stat.isDirectory());
-    assertFalse("isFile(): declares itself a file: " + stat, stat.isFile());
-    assertFalse("isFile(): declares itself a file: " + stat, stat.isSymlink());
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
deleted file mode 100644
index 833b91d57f2c5..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.hdfs2;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class TestV2LsOperations extends SwiftFileSystemBaseTest {
-
-  private Path[] testDirs;
-
-  /**
-   * Setup creates dirs under test/hadoop
-   * @throws Exception
-   */
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-    //delete the test directory
-    Path test = path("/test");
-    fs.delete(test, true);
-    mkdirs(test);
-  }
-
-  /**
-   * Create subdirectories and files under test/ for those tests
-   * that want them. Doing so adds overhead to setup and teardown,
-   * so should only be done for those tests that need them.
-   * @throws IOException on an IO problem
-   */
-  private void createTestSubdirs() throws IOException {
-    testDirs = new Path[]{
-      path("/test/hadoop/a"),
-      path("/test/hadoop/b"),
-      path("/test/hadoop/c/1"),
-    };
-    assertPathDoesNotExist("test directory setup", testDirs[0]);
-    for (Path path : testDirs) {
-      mkdirs(path);
-    }
-  }
-
-  /**
-   * To get this project to compile under Hadoop 1, this code needs to be
-   * commented out
-   *
-   *
-   * @param fs filesystem
-   * @param dir dir
-   * @param subdir subdir
-   * @param recursive recurse?
-   * @throws IOException IO problems
-   */
-  public static void assertListFilesFinds(FileSystem fs,
-                                          Path dir,
-                                          Path subdir,
-                                          boolean recursive) throws IOException {
-    RemoteIterator<LocatedFileStatus> iterator =
-      fs.listFiles(dir, recursive);
-    boolean found = false;
-    int entries = 0;
-    StringBuilder builder = new StringBuilder();
-    while (iterator.hasNext()) {
-      LocatedFileStatus next = iterator.next();
-      entries++;
-      builder.append(next.toString()).append('\n');
-      if (next.getPath().equals(subdir)) {
-        found = true;
-      }
-    }
-    assertTrue("Path " + subdir
-               + " not found in directory " + dir + " : "
-               + " entries=" + entries
-               + " content"
-               + builder.toString(),
-               found);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListFilesRootDir() throws Throwable {
-    Path dir = path("/");
-    Path child = new Path(dir, "test");
-    fs.delete(child, true);
-    SwiftTestUtils.writeTextFile(fs, child, "text", false);
-    assertListFilesFinds(fs, dir, child, false);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListFilesSubDir() throws Throwable {
-    createTestSubdirs();
-    Path dir = path("/test/subdir");
-    Path child = new Path(dir, "text.txt");
-    SwiftTestUtils.writeTextFile(fs, child, "text", false);
-    assertListFilesFinds(fs, dir, child, false);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testListFilesRecursive() throws Throwable {
-    createTestSubdirs();
-    Path dir = path("/test/recursive");
-    Path child = new Path(dir, "hadoop/a/a.txt");
-    SwiftTestUtils.writeTextFile(fs, child, "text", false);
-    assertListFilesFinds(fs, dir, child, true);
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java
deleted file mode 100644
index 8075e08404ae8..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.swift.SwiftTestConstants;
-import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Properties;
-
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_AUTH_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONTAINER_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTPS_PORT_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTP_PORT_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PASSWORD_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_REGION_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_TENANT_PROPERTY;
-import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_USERNAME_PROPERTY;
-import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPropertyEquals;
-
-public class TestRestClientBindings extends Assert
-  implements SwiftTestConstants {
-
-  private static final String SERVICE = "sname";
-  private static final String CONTAINER = "cname";
-  private static final String FS_URI = "swift://"
-          + CONTAINER + "." + SERVICE + "/";
-  private static final String AUTH_URL = "http://localhost:8080/auth";
-  private static final String USER = "user";
-  private static final String PASS = "pass";
-  private static final String TENANT = "tenant";
-  private URI filesysURI;
-  private Configuration conf;
-
-  @Before
-  public void setup() throws URISyntaxException {
-    filesysURI = new URI(FS_URI);
-    conf = new Configuration(true);
-    setInstanceVal(conf, SERVICE, DOT_AUTH_URL, AUTH_URL);
-    setInstanceVal(conf, SERVICE, DOT_USERNAME, USER);
-    setInstanceVal(conf, SERVICE, DOT_PASSWORD, PASS);
-  }
-
-  private void setInstanceVal(Configuration conf,
-                              String host,
-                              String key,
-                              String val) {
-    String instance = RestClientBindings.buildSwiftInstancePrefix(host);
-    String confkey = instance
-            + key;
-    conf.set(confkey, val);
-  }
-
-  public void testPrefixBuilder() throws Throwable {
-    String built = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
-    assertEquals("fs.swift.service." + SERVICE, built);
-  }
-
-  public void testBindAgainstConf() throws Exception {
-    Properties props = RestClientBindings.bind(filesysURI, conf);
-    assertPropertyEquals(props, SWIFT_CONTAINER_PROPERTY, CONTAINER);
-    assertPropertyEquals(props, SWIFT_SERVICE_PROPERTY, SERVICE);
-    assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
-    assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
-    assertPropertyEquals(props, SWIFT_USERNAME_PROPERTY, USER);
-    assertPropertyEquals(props, SWIFT_PASSWORD_PROPERTY, PASS);
-
-    assertPropertyEquals(props, SWIFT_TENANT_PROPERTY, null);
-    assertPropertyEquals(props, SWIFT_REGION_PROPERTY, null);
-    assertPropertyEquals(props, SWIFT_HTTP_PORT_PROPERTY, null);
-    assertPropertyEquals(props, SWIFT_HTTPS_PORT_PROPERTY, null);
-  }
-
-  public void expectBindingFailure(URI fsURI, Configuration config) {
-    try {
-      Properties binding = RestClientBindings.bind(fsURI, config);
-      //if we get here, binding didn't fail- there is something else.
-      //list the properties but not the values.
-      StringBuilder details = new StringBuilder() ;
-      for (Object key: binding.keySet()) {
-        details.append(key.toString()).append(" ");
-      }
-      fail("Expected a failure, got the binding [ "+ details+"]");
-    } catch (SwiftConfigurationException expected) {
-
-    }
-  }
-
-  public void testBindAgainstConfMissingInstance() throws Exception {
-    Configuration badConf = new Configuration();
-    expectBindingFailure(filesysURI, badConf);
-  }
-
-
-/* Hadoop 2.x+ only, as conf.unset() isn't a v1 feature
-  public void testBindAgainstConfIncompleteInstance() throws Exception {
-    String instance = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
-    conf.unset(instance + DOT_PASSWORD);
-    expectBindingFailure(filesysURI, conf);
-  }
-*/
-
-  @Test(expected = SwiftConfigurationException.class)
-  public void testDottedServiceURL() throws Exception {
-    RestClientBindings.bind(new URI("swift://hadoop.apache.org/"), conf);
-  }
-
-  @Test(expected = SwiftConfigurationException.class)
-  public void testMissingServiceURL() throws Exception {
-    RestClientBindings.bind(new URI("swift:///"), conf);
-  }
-
-  /**
-   * inner test method that expects container extraction to fail
-   * -if not prints a meaningful error message.
-   *
-   * @param hostname hostname to parse
-   */
-  private static void expectExtractContainerFail(String hostname) {
-    try {
-      String container = RestClientBindings.extractContainerName(hostname);
-      fail("Expected an error -got a container of '" + container
-              + "' from " + hostname);
-    } catch (SwiftConfigurationException expected) {
-      //expected
-    }
-  }
-
-  /**
-   * inner test method that expects service extraction to fail
-   * -if not prints a meaningful error message.
-   *
-   * @param hostname hostname to parse
-   */
-  public static void expectExtractServiceFail(String hostname) {
-    try {
-      String service = RestClientBindings.extractServiceName(hostname);
-      fail("Expected an error -got a service of '" + service
-              + "' from " + hostname);
-    } catch (SwiftConfigurationException expected) {
-      //expected
-    }
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testEmptyHostname() throws Throwable {
-    expectExtractContainerFail("");
-    expectExtractServiceFail("");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testDot() throws Throwable {
-    expectExtractContainerFail(".");
-    expectExtractServiceFail(".");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testSimple() throws Throwable {
-    expectExtractContainerFail("simple");
-    expectExtractServiceFail("simple");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testTrailingDot() throws Throwable {
-    expectExtractServiceFail("simple.");
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testLeadingDot() throws Throwable {
-    expectExtractServiceFail(".leading");
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java
deleted file mode 100644
index 7568c11c562c3..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.http;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.SwiftTestConstants;
-import org.apache.hadoop.fs.swift.util.Duration;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.apache.http.Header;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-
-public class TestSwiftRestClient implements SwiftTestConstants {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSwiftRestClient.class);
-
-  private Configuration conf;
-  private boolean runTests;
-  private URI serviceURI;
-
-  @Before
-  public void setup() throws IOException {
-    conf = new Configuration();
-    runTests = SwiftTestUtils.hasServiceURI(conf);
-    if (runTests) {
-      serviceURI = SwiftTestUtils.getServiceURI(conf);
-    }
-  }
-
-  protected void assumeEnabled() {
-    Assume.assumeTrue(runTests);
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testCreate() throws Throwable {
-    assumeEnabled();
-    SwiftRestClient client = createClient();
-  }
-
-  private SwiftRestClient createClient() throws IOException {
-    return SwiftRestClient.getInstance(serviceURI, conf);
-  }
-
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testAuthenticate() throws Throwable {
-    assumeEnabled();
-    SwiftRestClient client = createClient();
-    client.authenticate();
-  }
-
-  @Test(timeout = SWIFT_TEST_TIMEOUT)
-  public void testPutAndDelete() throws Throwable {
-    assumeEnabled();
-    SwiftRestClient client = createClient();
-    client.authenticate();
-    Path path = new Path("restTestPutAndDelete");
-    SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path);
-    byte[] stuff = new byte[1];
-    stuff[0] = 'a';
-    client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length);
-    //check file exists
-    Duration head = new Duration();
-    Header[] responseHeaders = client.headRequest("expect success",
-                                                  sobject,
-                                                  SwiftRestClient.NEWEST);
-    head.finished();
-    LOG.info("head request duration " + head);
-    for (Header header: responseHeaders) {
-      LOG.info(header.toString());
-    }
-    //delete the file
-    client.delete(sobject);
-    //check file is gone
-    try {
-      Header[] headers = client.headRequest("expect fail",
-                                            sobject,
-                                            SwiftRestClient.NEWEST);
-      Assert.fail("Expected deleted file, but object is still present: "
-                  + sobject);
-    } catch (FileNotFoundException e) {
-      //expected
-    }
-    for (DurationStats stats: client.getOperationStatistics()) {
-      LOG.info(stats.toString());
-    }
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java
deleted file mode 100644
index 314e7a1dfb88e..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.scale;
-
-import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
-
-/**
- * Base class for scale tests; here is where the common scale configuration
- * keys are defined
- */
-
-public class SwiftScaleTestBase extends SwiftFileSystemBaseTest {
-
-  public static final String SCALE_TEST = "scale.test.";
-  public static final String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
-  public static final long DEFAULT_OPERATION_COUNT = 10;
-
-  protected long getOperationCount() {
-    return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT);
-  }
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java
deleted file mode 100644
index 1d6cfa2e8668e..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.swift.scale;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.swift.util.Duration;
-import org.apache.hadoop.fs.swift.util.DurationStats;
-import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
-import org.junit.Test;
-
-public class TestWriteManySmallFiles extends SwiftScaleTestBase {
-
-  public static final Logger LOG =
-      LoggerFactory.getLogger(TestWriteManySmallFiles.class);
-
-  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
-  public void testScaledWriteThenRead() throws Throwable {
-    Path dir = new Path("/test/manysmallfiles");
-    Duration rm1 = new Duration();
-    fs.delete(dir, true);
-    rm1.finished();
-    fs.mkdirs(dir);
-    Duration ls1 = new Duration();
-    fs.listStatus(dir);
-    ls1.finished();
-    long count = getOperationCount();
-    SwiftTestUtils.noteAction("Beginning Write of "+ count + " files ");
-    DurationStats writeStats = new DurationStats("write");
-    DurationStats readStats = new DurationStats("read");
-    String format = "%08d";
-    for (long l = 0; l < count; l++) {
-      String name = String.format(format, l);
-      Path p = new Path(dir, "part-" + name);
-      Duration d = new Duration();
-      SwiftTestUtils.writeTextFile(fs, p, name, false);
-      d.finished();
-      writeStats.add(d);
-      Thread.sleep(1000);
-    }
-    //at this point, the directory is full.
-    SwiftTestUtils.noteAction("Beginning ls");
-
-    Duration ls2 = new Duration();
-    FileStatus[] status2 = (FileStatus[]) fs.listStatus(dir);
-    ls2.finished();
-    assertEquals("Not enough entries in the directory", count, status2.length);
-
-    SwiftTestUtils.noteAction("Beginning read");
-
-    for (long l = 0; l < count; l++) {
-      String name = String.format(format, l);
-      Path p = new Path(dir, "part-" + name);
-      Duration d = new Duration();
-      String result = SwiftTestUtils.readBytesToString(fs, p, name.length());
-      assertEquals(name, result);
-      d.finished();
-      readStats.add(d);
-    }
-    //do a recursive delete
-    SwiftTestUtils.noteAction("Beginning delete");
-    Duration rm2 = new Duration();
-    fs.delete(dir, true);
-    rm2.finished();
-    //print the stats
-    LOG.info(String.format("'filesystem','%s'",fs.getUri()));
-    LOG.info(writeStats.toString());
-    LOG.info(readStats.toString());
-    LOG.info(String.format(
-      "'rm1',%d,'ls1',%d",
-      rm1.value(),
-      ls1.value()));
-    LOG.info(String.format(
-      "'rm2',%d,'ls2',%d",
-      rm2.value(),
-      ls2.value()));
-  }
-
-}
diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties b/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties
deleted file mode 100644
index a3bb8204f0df8..0000000000000
--- a/hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.target=System.out
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n"
-#log4j.logger.org.apache.hadoop.fs.swift=DEBUG
diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml
index 961aa2933e3b0..89e248f9a7efe 100644
--- a/hadoop-tools/hadoop-resourceestimator/pom.xml
+++ b/hadoop-tools/hadoop-resourceestimator/pom.xml
@@ -79,8 +79,22 @@
             <artifactId>jersey-server</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.sun.jersey</groupId>
+            <groupId>com.github.pjfanning</groupId>
             <artifactId>jersey-json</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.jaxrs</groupId>
+                    <artifactId>jackson-jaxrs-json-provider</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index fd3b5c9a1b8fa..9a966d55c0ed1 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -86,6 +86,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfigurationParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
index 7e79179721f8a..9cd2f4778fc7f 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
@@ -25,6 +25,8 @@
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
+import org.apache.hadoop.util.XMLUtils;
+
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
@@ -55,7 +57,7 @@ static Properties parse(InputStream input) throws IOException {
     Properties result = new Properties();
 
     try {
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
 
       DocumentBuilder db = dbf.newDocumentBuilder();
 
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
index 1d85872c08d7b..a6c8bdad87d04 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
@@ -17,28 +17,27 @@
  */
 package org.apache.hadoop.tools.rumen;
 
+import java.io.IOException;
+import java.io.StringReader;
 import java.util.Properties;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
-import java.io.InputStream;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-
-import java.nio.charset.Charset;
-
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.util.XMLUtils;
+
 import org.w3c.dom.Document;
 import org.w3c.dom.NodeList;
 import org.w3c.dom.Node;
 import org.w3c.dom.Element;
 import org.w3c.dom.Text;
 
+import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 
 class ParsedConfigFile {
@@ -46,7 +45,6 @@ class ParsedConfigFile {
       Pattern.compile("_(job_[0-9]+_[0-9]+)_");
   private static final Pattern heapPattern =
       Pattern.compile("-Xmx([0-9]+)([mMgG])");
-  private static final Charset UTF_8 = Charset.forName("UTF-8");
 
   final int heapMegabytes;
 
@@ -103,13 +101,11 @@ private int maybeGetIntValue(String propName, String attr, String value,
     }
 
     try {
-      InputStream is = new ByteArrayInputStream(xmlString.getBytes(UTF_8));
-
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
 
       DocumentBuilder db = dbf.newDocumentBuilder();
 
-      Document doc = db.parse(is);
+      Document doc = db.parse(new InputSource(new StringReader(xmlString)));
 
       Element root = doc.getDocumentElement();
 
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index 329cc790fb9e5..208cbdf6c142a 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -147,38 +147,6 @@
           </excludes>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinition.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinition.java
new file mode 100644
index 0000000000000..1f9e351c4295f
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinition.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
+
+import java.util.List;
+
+public abstract class AMDefinition {
+  protected int jobCount;
+  protected String amType;
+  protected String user;
+  protected String queue;
+  protected long jobStartTime;
+  protected long jobFinishTime;
+  protected List<ContainerSimulator> taskContainers;
+  protected Resource amResource;
+  protected String labelExpression;
+  protected String oldAppId;
+
+  public AMDefinition(AmDefinitionBuilder builder) {
+    this.jobStartTime = builder.jobStartTime;
+    this.jobFinishTime = builder.jobFinishTime;
+    this.amType = builder.amType;
+    this.taskContainers = builder.taskContainers;
+    this.labelExpression = builder.labelExpression;
+    this.user = builder.user;
+    this.amResource = builder.amResource;
+    this.queue = builder.queue;
+    this.jobCount = builder.jobCount;
+    this.oldAppId = builder.jobId;
+  }
+
+  public String getAmType() {
+    return amType;
+  }
+
+  public String getUser() {
+    return user;
+  }
+
+  public String getOldAppId() {
+    return oldAppId;
+  }
+
+  public long getJobStartTime() {
+    return jobStartTime;
+  }
+
+  public long getJobFinishTime() {
+    return jobFinishTime;
+  }
+
+  public List<ContainerSimulator> getTaskContainers() {
+    return taskContainers;
+  }
+
+  public Resource getAmResource() {
+    return amResource;
+  }
+
+  public String getLabelExpression() {
+    return labelExpression;
+  }
+
+  public String getQueue() {
+    return queue;
+  }
+
+  public int getJobCount() {
+    return jobCount;
+  }
+
+
+  public abstract static class AmDefinitionBuilder {
+    private static final String DEFAULT_USER = "default";
+
+    protected int jobCount = 1;
+    protected String amType = AMDefinitionFactory.DEFAULT_JOB_TYPE;
+    protected String user = DEFAULT_USER;
+    protected String queue;
+    protected String jobId;
+    protected long jobStartTime;
+    protected long jobFinishTime;
+    protected List<ContainerSimulator> taskContainers;
+    protected Resource amResource;
+    protected String labelExpression = null;
+
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionFactory.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionFactory.java
new file mode 100644
index 0000000000000..61975f04a375b
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionFactory.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.tools.rumen.LoggedJob;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public final class AMDefinitionFactory {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AMDefinitionFactory.class);
+  public final static String DEFAULT_JOB_TYPE = "mapreduce";
+
+  private AMDefinitionFactory() {}
+
+  public static AMDefinitionSLS createFromSlsTrace(Map<String, String> jsonJob,
+      SLSRunner slsRunner) throws YarnException {
+    AMDefinitionSLS amDefinition = AMDefinitionSLS.Builder.create(jsonJob)
+        .withAmType(SLSConfiguration.AM_TYPE)
+        .withAmResource(getAMContainerResourceSLS(jsonJob, slsRunner))
+        .withTaskContainers(
+            AMDefinitionSLS.getTaskContainers(jsonJob, slsRunner))
+        .withJobStartTime(SLSConfiguration.JOB_START_MS)
+        .withJobFinishTime(SLSConfiguration.JOB_END_MS)
+        .withLabelExpression(SLSConfiguration.JOB_LABEL_EXPR)
+        .withUser(SLSConfiguration.JOB_USER)
+        .withQueue(SLSConfiguration.JOB_QUEUE_NAME)
+        .withJobId(SLSConfiguration.JOB_ID)
+        .withJobCount(SLSConfiguration.JOB_COUNT)
+        .build();
+    slsRunner.increaseQueueAppNum(amDefinition.getQueue());
+    return amDefinition;
+  }
+
+  public static AMDefinitionRumen createFromRumenTrace(LoggedJob job,
+      long baselineTimeMs, SLSRunner slsRunner) throws YarnException {
+    AMDefinitionRumen amDefinition = AMDefinitionRumen.Builder.create()
+        .withAmType(DEFAULT_JOB_TYPE)
+        .withAmResource(getAMContainerResourceSynthAndRumen(slsRunner))
+        .withTaskContainers(
+            AMDefinitionRumen.getTaskContainers(job, slsRunner))
+        .withJobStartTime(job.getSubmitTime())
+        .withJobFinishTime(job.getFinishTime())
+        .withBaseLineTimeMs(baselineTimeMs)
+        .withUser(job.getUser())
+        .withQueue(job.getQueue().getValue())
+        .withJobId(job.getJobID().toString())
+        .build();
+    slsRunner.increaseQueueAppNum(amDefinition.getQueue());
+    return amDefinition;
+  }
+
+  public static AMDefinitionSynth createFromSynth(SynthJob job,
+      SLSRunner slsRunner) throws YarnException {
+    AMDefinitionSynth amDefinition =
+        AMDefinitionSynth.Builder.create()
+            .withAmType(job.getType())
+            .withAmResource(getAMContainerResourceSynthAndRumen(slsRunner))
+            .withTaskContainers(
+                AMDefinitionSynth.getTaskContainers(job, slsRunner))
+            .withUser(job.getUser())
+            .withQueue(job.getQueueName())
+            .withJobId(job.getJobID().toString())
+            .withJobStartTime(job.getSubmissionTime())
+            .withJobFinishTime(job.getSubmissionTime() + job.getDuration())
+            .withBaseLineTimeMs(0)
+            .build();
+
+    slsRunner.increaseQueueAppNum(amDefinition.getQueue());
+    return amDefinition;
+  }
+
+  private static Resource getAMContainerResourceSLS(Map<String, String> jsonJob,
+      Configured configured) {
+    Resource amContainerResource =
+        SLSConfiguration.getAMContainerResource(configured.getConf());
+    if (jsonJob == null) {
+      return amContainerResource;
+    }
+
+    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+    for (ResourceInformation info : infors) {
+      String key = SLSConfiguration.JOB_AM_PREFIX + info.getName();
+      if (jsonJob.containsKey(key)) {
+        long value = Long.parseLong(jsonJob.get(key));
+        amContainerResource.setResourceValue(info.getName(), value);
+      }
+    }
+
+    return amContainerResource;
+  }
+
+  private static Resource getAMContainerResourceSynthAndRumen(
+      Configured configured) {
+    return SLSConfiguration.getAMContainerResource(configured.getConf());
+  }
+
+  static void adjustTimeValuesToBaselineTime(AMDefinition amDef,
+      AMDefinition.AmDefinitionBuilder builder, long baselineTimeMs) {
+    builder.jobStartTime -= baselineTimeMs;
+    builder.jobFinishTime -= baselineTimeMs;
+    if (builder.jobStartTime < 0) {
+      LOG.warn("Warning: reset job {} start time to 0.", amDef.getOldAppId());
+      builder.jobFinishTime = builder.jobFinishTime - builder.jobStartTime;
+      builder.jobStartTime = 0;
+    }
+    amDef.jobStartTime = builder.jobStartTime;
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionRumen.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionRumen.java
new file mode 100644
index 0000000000000..cc97a90ec00b7
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionRumen.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.tools.rumen.LoggedJob;
+import org.apache.hadoop.tools.rumen.LoggedTask;
+import org.apache.hadoop.tools.rumen.LoggedTaskAttempt;
+import org.apache.hadoop.tools.rumen.datatypes.UserName;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.sls.AMDefinitionFactory.adjustTimeValuesToBaselineTime;
+
+public class AMDefinitionRumen extends AMDefinition {
+  public final static int DEFAULT_MAPPER_PRIORITY = 20;
+  private final static int DEFAULT_REDUCER_PRIORITY = 10;
+
+  public AMDefinitionRumen(AmDefinitionBuilder builder) {
+    super(builder);
+  }
+
+  public static List<ContainerSimulator> getTaskContainers(LoggedJob job,
+      SLSRunner slsRunner) throws YarnException {
+    List<ContainerSimulator> containerList = new ArrayList<>();
+
+    TaskContainerDefinition.Builder builder =
+        TaskContainerDefinition.Builder.create()
+            .withCount(1)
+            .withResource(slsRunner.getDefaultContainerResource())
+            .withExecutionType(ExecutionType.GUARANTEED)
+            .withAllocationId(-1)
+            .withRequestDelay(0);
+
+    // mapper
+    for (LoggedTask mapTask : job.getMapTasks()) {
+      if (mapTask.getAttempts().size() == 0) {
+        throw new YarnException("Invalid map task, no attempt for a mapper!");
+      }
+      LoggedTaskAttempt taskAttempt =
+          mapTask.getAttempts().get(mapTask.getAttempts().size() - 1);
+      TaskContainerDefinition containerDef = builder
+          .withHostname(taskAttempt.getHostName().getValue())
+          .withDuration(taskAttempt.getFinishTime() -
+              taskAttempt.getStartTime())
+          .withPriority(DEFAULT_MAPPER_PRIORITY)
+          .withType("map")
+          .build();
+      containerList.add(
+          ContainerSimulator.createFromTaskContainerDefinition(containerDef));
+    }
+
+    // reducer
+    for (LoggedTask reduceTask : job.getReduceTasks()) {
+      if (reduceTask.getAttempts().size() == 0) {
+        throw new YarnException(
+            "Invalid reduce task, no attempt for a reducer!");
+      }
+      LoggedTaskAttempt taskAttempt =
+          reduceTask.getAttempts().get(reduceTask.getAttempts().size() - 1);
+      TaskContainerDefinition containerDef = builder
+          .withHostname(taskAttempt.getHostName().getValue())
+          .withDuration(taskAttempt.getFinishTime() -
+              taskAttempt.getStartTime())
+          .withPriority(DEFAULT_REDUCER_PRIORITY)
+          .withType("reduce")
+          .build();
+      containerList.add(
+          ContainerSimulator.createFromTaskContainerDefinition(containerDef));
+    }
+
+    return containerList;
+  }
+
+
+  public static final class Builder extends AmDefinitionBuilder {
+    private long baselineTimeMs;
+
+    private Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    public Builder withAmType(String amType) {
+      this.amType = amType;
+      return this;
+    }
+
+    public Builder withUser(UserName user) {
+      if (user != null) {
+        this.user = user.getValue();
+      }
+      return this;
+    }
+
+    public Builder withQueue(String queue) {
+      this.queue = queue;
+      return this;
+    }
+
+    public Builder withJobId(String oldJobId) {
+      this.jobId = oldJobId;
+      return this;
+    }
+
+    public Builder withJobStartTime(long time) {
+      this.jobStartTime = time;
+      return this;
+    }
+
+    public Builder withJobFinishTime(long time) {
+      this.jobFinishTime = time;
+      return this;
+    }
+
+    public Builder withBaseLineTimeMs(long baselineTimeMs) {
+      this.baselineTimeMs = baselineTimeMs;
+      return this;
+    }
+
+    public Builder withLabelExpression(String expr) {
+      this.labelExpression = expr;
+      return this;
+    }
+
+    public AMDefinitionRumen.Builder withTaskContainers(
+        List<ContainerSimulator> taskContainers) {
+      this.taskContainers = taskContainers;
+      return this;
+    }
+
+    public AMDefinitionRumen.Builder withAmResource(Resource amResource) {
+      this.amResource = amResource;
+      return this;
+    }
+
+    public AMDefinitionRumen build() {
+      AMDefinitionRumen amDef = new AMDefinitionRumen(this);
+
+      if (baselineTimeMs == 0) {
+        baselineTimeMs = jobStartTime;
+      }
+      adjustTimeValuesToBaselineTime(amDef, this, baselineTimeMs);
+      return amDef;
+    }
+
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSLS.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSLS.java
new file mode 100644
index 0000000000000..a84c9240cb6a3
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSLS.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AMDefinitionSLS extends AMDefinition {
+  public AMDefinitionSLS(AmDefinitionBuilder builder) {
+    super(builder);
+  }
+
+  public String getQueue() {
+    return queue;
+  }
+
+  public static List<ContainerSimulator> getTaskContainers(Map<String, ?> jsonJob,
+      SLSRunner slsRunner) throws YarnException {
+    List<Map<String, String>> tasks = (List) jsonJob.get(SLSConfiguration.JOB_TASKS);
+    if (tasks == null || tasks.size() == 0) {
+      throw new YarnException("No task for the job!");
+    }
+
+    List<ContainerSimulator> containers = new ArrayList<>();
+    for (Map<String, String> jsonTask : tasks) {
+      TaskContainerDefinition containerDef =
+          TaskContainerDefinition.Builder.create()
+              .withCount(jsonTask, SLSConfiguration.COUNT)
+              .withHostname(jsonTask.get(SLSConfiguration.TASK_HOST))
+              .withDuration(jsonTask, SLSConfiguration.TASK_DURATION_MS)
+              .withDurationLegacy(jsonTask, SLSConfiguration.DURATION_MS)
+              .withTaskStart(jsonTask, SLSConfiguration.TASK_START_MS)
+              .withTaskFinish(jsonTask, SLSConfiguration.TASK_END_MS)
+              .withResource(getResourceForContainer(jsonTask, slsRunner))
+              .withPriority(jsonTask, SLSConfiguration.TASK_PRIORITY)
+              .withType(jsonTask, SLSConfiguration.TASK_TYPE)
+              .withExecutionType(jsonTask, SLSConfiguration.TASK_EXECUTION_TYPE)
+              .withAllocationId(jsonTask, SLSConfiguration.TASK_ALLOCATION_ID)
+              .withRequestDelay(jsonTask, SLSConfiguration.TASK_REQUEST_DELAY)
+              .build();
+
+      for (int i = 0; i < containerDef.getCount(); i++) {
+        containers.add(ContainerSimulator.
+            createFromTaskContainerDefinition(containerDef));
+      }
+    }
+    return containers;
+  }
+
+  private static Resource getResourceForContainer(Map<String, String> jsonTask,
+      SLSRunner slsRunner) {
+    Resource res = slsRunner.getDefaultContainerResource();
+    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+    for (ResourceInformation info : infors) {
+      if (jsonTask.containsKey(SLSConfiguration.TASK_PREFIX + info.getName())) {
+        long value = Long.parseLong(
+            jsonTask.get(SLSConfiguration.TASK_PREFIX + info.getName()));
+        res.setResourceValue(info.getName(), value);
+      }
+    }
+    return res;
+  }
+
+  public static final class Builder extends AmDefinitionBuilder {
+    private final Map<String, String> jsonJob;
+
+    private Builder(Map<String, String> jsonJob) {
+      this.jsonJob = jsonJob;
+    }
+
+    public static Builder create(Map<String, String> jsonJob) {
+      return new Builder(jsonJob);
+    }
+
+    public Builder withAmType(String key) {
+      if (jsonJob.containsKey(key)) {
+        String amType = jsonJob.get(key);
+        if (amType != null) {
+          this.amType = amType;
+        }
+      }
+      return this;
+    }
+
+    public Builder withUser(String key) {
+      if (jsonJob.containsKey(key)) {
+        String user = jsonJob.get(key);
+        if (user != null) {
+          this.user = user;
+        }
+      }
+      return this;
+    }
+
+    public Builder withQueue(String key) {
+      if (jsonJob.containsKey(key)) {
+        this.queue = jsonJob.get(key);
+      }
+      return this;
+    }
+
+    public Builder withJobId(String key) {
+      if (jsonJob.containsKey(key)) {
+        this.jobId = jsonJob.get(key);
+      }
+      return this;
+    }
+
+    public Builder withJobCount(String key) {
+      if (jsonJob.containsKey(key)) {
+        jobCount = Integer.parseInt(jsonJob.get(key));
+        jobCount = Math.max(jobCount, 1);
+      }
+      return this;
+    }
+
+    public Builder withJobStartTime(String key) {
+      if (jsonJob.containsKey(key)) {
+        this.jobStartTime = Long.parseLong(jsonJob.get(key));
+      }
+      return this;
+    }
+
+    public Builder withJobFinishTime(String key) {
+      if (jsonJob.containsKey(key)) {
+        this.jobFinishTime = Long.parseLong(jsonJob.get(key));
+      }
+      return this;
+    }
+
+    public Builder withLabelExpression(String key) {
+      if (jsonJob.containsKey(key)) {
+        this.labelExpression = jsonJob.get(key);
+      }
+      return this;
+    }
+
+    public AMDefinitionSLS.Builder withTaskContainers(
+        List<ContainerSimulator> taskContainers) {
+      this.taskContainers = taskContainers;
+      return this;
+    }
+
+    public AMDefinitionSLS.Builder withAmResource(Resource amResource) {
+      this.amResource = amResource;
+      return this;
+    }
+
+    public AMDefinitionSLS build() {
+      AMDefinitionSLS amDef = new AMDefinitionSLS(this);
+      // Job id is generated automatically if this job configuration allows
+      // multiple job instances
+      if (jobCount > 1) {
+        amDef.oldAppId = null;
+      } else {
+        amDef.oldAppId = jobId;
+      }
+      amDef.jobCount = jobCount;
+      return amDef;
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSynth.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSynth.java
new file mode 100644
index 0000000000000..db736f06f7212
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMDefinitionSynth.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import static org.apache.hadoop.yarn.sls.AMDefinitionFactory.adjustTimeValuesToBaselineTime;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
+import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
+
+public class AMDefinitionSynth extends AMDefinition {
+  public AMDefinitionSynth(AmDefinitionBuilder builder) {
+    super(builder);
+  }
+
+  public static List<ContainerSimulator> getTaskContainers(
+      SynthJob job, SLSRunner slsRunner) throws YarnException {
+    List<ContainerSimulator> containerList = new ArrayList<>();
+    ArrayList<NodeId> keyAsArray = new ArrayList<>(
+        slsRunner.getNmMap().keySet());
+    Random rand = new Random(slsRunner.getStjp().getSeed());
+
+    for (SynthJob.SynthTask task : job.getTasks()) {
+      RMNode node = getRandomNode(slsRunner, keyAsArray, rand);
+      TaskContainerDefinition containerDef =
+          TaskContainerDefinition.Builder.create()
+              .withCount(1)
+              .withHostname("/" + node.getRackName() + "/" + node.getHostName())
+              .withDuration(task.getTime())
+              .withResource(Resource
+                  .newInstance((int) task.getMemory(), (int) task.getVcores()))
+              .withPriority(task.getPriority())
+              .withType(task.getType())
+              .withExecutionType(task.getExecutionType())
+              .withAllocationId(-1)
+              .withRequestDelay(0)
+              .build();
+      containerList.add(
+          ContainerSimulator.createFromTaskContainerDefinition(containerDef));
+    }
+
+    return containerList;
+  }
+
+  private static RMNode getRandomNode(SLSRunner slsRunner,
+      ArrayList<NodeId> keyAsArray, Random rand) {
+    int randomIndex = rand.nextInt(keyAsArray.size());
+    return slsRunner.getNmMap().get(keyAsArray.get(randomIndex)).getNode();
+  }
+
+  public static final class Builder extends AmDefinitionBuilder {
+    private long baselineTimeMs;
+
+    private Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    public Builder withAmType(String amType) {
+      this.amType = amType;
+      return this;
+    }
+
+    public Builder withUser(String user) {
+      if (user != null) {
+        this.user = user;
+      }
+      return this;
+    }
+
+    public Builder withQueue(String queue) {
+      this.queue = queue;
+      return this;
+    }
+
+    public Builder withJobId(String oldJobId) {
+      this.jobId = oldJobId;
+      return this;
+    }
+
+    public Builder withJobStartTime(long time) {
+      this.jobStartTime = time;
+      return this;
+    }
+
+    public Builder withJobFinishTime(long time) {
+      this.jobFinishTime = time;
+      return this;
+    }
+
+    public Builder withBaseLineTimeMs(long baselineTimeMs) {
+      this.baselineTimeMs = baselineTimeMs;
+      return this;
+    }
+
+    public AMDefinitionSynth.Builder withLabelExpression(String expr) {
+      this.labelExpression = expr;
+      return this;
+    }
+
+    public AMDefinitionSynth.Builder withTaskContainers(
+        List<ContainerSimulator> taskContainers) {
+      this.taskContainers = taskContainers;
+      return this;
+    }
+
+    public AMDefinitionSynth.Builder withAmResource(Resource amResource) {
+      this.amResource = amResource;
+      return this;
+    }
+
+    public AMDefinitionSynth build() {
+      AMDefinitionSynth amDef = new AMDefinitionSynth(this);
+
+      if (baselineTimeMs == 0) {
+        baselineTimeMs = jobStartTime;
+      }
+      adjustTimeValuesToBaselineTime(amDef, this, baselineTimeMs);
+      return amDef;
+    }
+  }
+
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMRunner.java
new file mode 100644
index 0000000000000..2dc09de665368
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/AMRunner.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.tools.rumen.JobTraceReader;
+import org.apache.hadoop.tools.rumen.LoggedJob;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.SLSRunner.TraceType;
+import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
+import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
+import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+
+public class AMRunner {
+  private static final Logger LOG = LoggerFactory.getLogger(AMRunner.class);
+  int remainingApps = 0;
+
+  private final Configuration conf;
+  private int AM_ID;
+  private Map<String, AMSimulator> amMap;
+  private Map<ApplicationId, AMSimulator> appIdAMSim;
+  private Set<String> trackedApps;
+  private Map<String, Class> amClassMap;
+  private TraceType inputType;
+  private String[] inputTraces;
+  private final TaskRunner runner;
+  private final SLSRunner slsRunner;
+  private int numAMs, numTasks;
+  private long maxRuntime;
+  private ResourceManager rm;
+
+  public AMRunner(TaskRunner runner, SLSRunner slsRunner) {
+    this.runner = runner;
+    this.slsRunner = slsRunner;
+    this.conf = slsRunner.getConf();
+  }
+
+
+  public void init(Configuration conf) throws ClassNotFoundException {
+    amMap = new ConcurrentHashMap<>();
+    amClassMap = new HashMap<>();
+    appIdAMSim = new ConcurrentHashMap<>();
+    // <AMType, Class> map
+    for (Map.Entry<String, String> e : conf) {
+      String key = e.getKey();
+      if (key.startsWith(SLSConfiguration.AM_TYPE_PREFIX)) {
+        String amType = key.substring(SLSConfiguration.AM_TYPE_PREFIX.length());
+        amClassMap.put(amType, Class.forName(conf.get(key)));
+      }
+    }
+  }
+
+  public void startAM() throws YarnException, IOException {
+    switch (inputType) {
+      case SLS:
+        for (String inputTrace : inputTraces) {
+          startAMFromSLSTrace(inputTrace);
+        }
+        break;
+      case RUMEN:
+        long baselineTimeMS = 0;
+        for (String inputTrace : inputTraces) {
+          startAMFromRumenTrace(inputTrace, baselineTimeMS);
+        }
+        break;
+      case SYNTH:
+        startAMFromSynthGenerator();
+        break;
+      default:
+        throw new YarnException("Input configuration not recognized, "
+            + "trace type should be SLS, RUMEN, or SYNTH");
+    }
+
+    numAMs = amMap.size();
+    remainingApps = numAMs;
+  }
+
+  /**
+   * Parse workload from a SLS trace file.
+   */
+  private void startAMFromSLSTrace(String inputTrace) throws IOException {
+    JsonFactory jsonF = new JsonFactory();
+    ObjectMapper mapper = new ObjectMapper();
+
+    try (Reader input = new InputStreamReader(
+        new FileInputStream(inputTrace), StandardCharsets.UTF_8)) {
+      JavaType type = mapper.getTypeFactory().
+          constructMapType(Map.class, String.class, String.class);
+      Iterator<Map<String, String>> jobIter = mapper.readValues(
+          jsonF.createParser(input), type);
+
+      while (jobIter.hasNext()) {
+        try {
+          Map<String, String> jsonJob = jobIter.next();
+          AMDefinitionSLS amDef = AMDefinitionFactory.createFromSlsTrace(jsonJob, slsRunner);
+          startAMs(amDef);
+        } catch (Exception e) {
+          LOG.error("Failed to create an AM: {}", e.getMessage());
+        }
+      }
+    }
+  }
+
+  /**
+   * parse workload information from synth-generator trace files.
+   */
+  private void startAMFromSynthGenerator() throws YarnException, IOException {
+    Configuration localConf = new Configuration();
+    localConf.set("fs.defaultFS", "file:///");
+    //if we use the nodeFile this could have been not initialized yet.
+    if (slsRunner.getStjp() == null) {
+      slsRunner.setStjp(new SynthTraceJobProducer(conf, new Path(inputTraces[0])));
+    }
+
+    SynthJob job;
+    // we use stjp, a reference to the job producer instantiated during node
+    // creation
+    while ((job = (SynthJob) slsRunner.getStjp().getNextJob()) != null) {
+      ReservationId reservationId = null;
+      if (job.hasDeadline()) {
+        reservationId = ReservationId
+            .newInstance(rm.getStartTime(), AM_ID);
+      }
+      AMDefinitionSynth amDef = AMDefinitionFactory.createFromSynth(job, slsRunner);
+      startAMs(amDef, reservationId, job.getParams(), job.getDeadline());
+    }
+  }
+
+  /**
+   * Parse workload from a rumen trace file.
+   */
+  private void startAMFromRumenTrace(String inputTrace, long baselineTimeMS)
+      throws IOException {
+    Configuration conf = new Configuration();
+    conf.set("fs.defaultFS", "file:///");
+    File fin = new File(inputTrace);
+
+    try (JobTraceReader reader = new JobTraceReader(
+        new Path(fin.getAbsolutePath()), conf)) {
+      LoggedJob job = reader.getNext();
+
+      while (job != null) {
+        try {
+          AMDefinitionRumen amDef =
+              AMDefinitionFactory.createFromRumenTrace(job, baselineTimeMS,
+                  slsRunner);
+          startAMs(amDef);
+        } catch (Exception e) {
+          LOG.error("Failed to create an AM", e);
+        }
+        job = reader.getNext();
+      }
+    }
+  }
+
+  private void startAMs(AMDefinition amDef) {
+    for (int i = 0; i < amDef.getJobCount(); i++) {
+      JobDefinition jobDef = JobDefinition.Builder.create()
+          .withAmDefinition(amDef)
+          .withDeadline(-1)
+          .withReservationId(null)
+          .withParams(null)
+          .build();
+      runNewAM(jobDef);
+    }
+  }
+
+  private void startAMs(AMDefinition amDef,
+      ReservationId reservationId,
+      Map<String, String> params, long deadline) {
+    for (int i = 0; i < amDef.getJobCount(); i++) {
+      JobDefinition jobDef = JobDefinition.Builder.create()
+          .withAmDefinition(amDef)
+          .withReservationId(reservationId)
+          .withParams(params)
+          .withDeadline(deadline)
+          .build();
+      runNewAM(jobDef);
+    }
+  }
+
+  private void runNewAM(JobDefinition jobDef) {
+    AMDefinition amDef = jobDef.getAmDefinition();
+    String oldJobId = amDef.getOldAppId();
+    AMSimulator amSim =
+        createAmSimulator(amDef.getAmType());
+
+    if (amSim != null) {
+      int heartbeatInterval = conf.getInt(
+          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS,
+          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS_DEFAULT);
+      boolean isTracked = trackedApps.contains(oldJobId);
+
+      if (oldJobId == null) {
+        oldJobId = Integer.toString(AM_ID);
+      }
+      AM_ID++;
+      amSim.init(amDef, rm, slsRunner, isTracked, runner.getStartTimeMS(), heartbeatInterval, appIdAMSim);
+      if (jobDef.getReservationId() != null) {
+        // if we have a ReservationId, delegate reservation creation to
+        // AMSim (reservation shape is impl specific)
+        UTCClock clock = new UTCClock();
+        amSim.initReservation(jobDef.getReservationId(), jobDef.getDeadline(), clock.getTime());
+      }
+      runner.schedule(amSim);
+      maxRuntime = Math.max(maxRuntime, amDef.getJobFinishTime());
+      numTasks += amDef.getTaskContainers().size();
+      amMap.put(oldJobId, amSim);
+    }
+  }
+
+  private AMSimulator createAmSimulator(String jobType) {
+    return (AMSimulator) ReflectionUtils.newInstance(
+        amClassMap.get(jobType), new Configuration());
+  }
+
+  public AMSimulator getAMSimulator(ApplicationId appId) {
+    return appIdAMSim.get(appId);
+  }
+
+  public void setInputType(TraceType inputType) {
+    this.inputType = inputType;
+  }
+
+  public void setInputTraces(String[] inputTraces) {
+    this.inputTraces = inputTraces.clone();
+  }
+
+  public void setResourceManager(ResourceManager rm) {
+    this.rm = rm;
+  }
+
+  public Set<String> getTrackedApps() {
+    return trackedApps;
+  }
+
+  public void setTrackedApps(Set<String> trackApps) {
+    this.trackedApps = trackApps;
+  }
+
+  public int getNumAMs() {
+    return numAMs;
+  }
+
+  public int getNumTasks() {
+    return numTasks;
+  }
+
+  public long getMaxRuntime() {
+    return maxRuntime;
+  }
+
+  public Map<String, AMSimulator> getAmMap() {
+    return amMap;
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/JobDefinition.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/JobDefinition.java
new file mode 100644
index 0000000000000..4a39d3710c92a
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/JobDefinition.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import java.util.Map;
+
+public class JobDefinition {
+  private AMDefinition amDefinition;
+  private ReservationId reservationId;
+  private long deadline;
+  private Map<String, String> params;
+
+  public AMDefinition getAmDefinition() {
+    return amDefinition;
+  }
+
+  public ReservationId getReservationId() {
+    return reservationId;
+  }
+
+  public long getDeadline() {
+    return deadline;
+  }
+
+  //Currently unused
+  public Map<String, String> getParams() {
+    return params;
+  }
+
+  public static final class Builder {
+    private AMDefinition amDefinition;
+    private ReservationId reservationId;
+    private long deadline;
+    private Map<String, String> params;
+
+    private Builder() {
+    }
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    public Builder withAmDefinition(AMDefinition amDefinition) {
+      this.amDefinition = amDefinition;
+      return this;
+    }
+
+    public Builder withReservationId(ReservationId reservationId) {
+      this.reservationId = reservationId;
+      return this;
+    }
+
+    public Builder withDeadline(long deadline) {
+      this.deadline = deadline;
+      return this;
+    }
+
+    public Builder withParams(Map<String, String> params) {
+      this.params = params;
+      return this;
+    }
+
+    public JobDefinition build() {
+      JobDefinition jobDef = new JobDefinition();
+      jobDef.params = this.params;
+      jobDef.amDefinition = this.amDefinition;
+      jobDef.reservationId = this.reservationId;
+      jobDef.deadline = this.deadline;
+      return jobDef;
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/NMRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/NMRunner.java
new file mode 100644
index 0000000000000..14524a4e04ac5
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/NMRunner.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.sls.SLSRunner.TraceType;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
+import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
+import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
+import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class NMRunner {
+  private static final Logger LOG = LoggerFactory.getLogger(NMRunner.class);
+  
+  // other simulation information
+  private int numNMs, numRacks;
+
+  // NM simulator
+  private Map<NodeId, NMSimulator> nmMap;
+  private Resource nodeManagerResource;
+  private String nodeFile;
+  private TaskRunner taskRunner;
+  private Configuration conf;
+  private ResourceManager rm;
+  private String tableMapping;
+  private int threadPoolSize;
+  private TraceType inputType;
+  private String[] inputTraces;
+  private SynthTraceJobProducer stjp;
+
+  public NMRunner(TaskRunner taskRunner, Configuration conf, ResourceManager rm, String tableMapping, int threadPoolSize) {
+    this.taskRunner = taskRunner;
+    this.conf = conf;
+    this.rm = rm;
+    this.tableMapping = tableMapping;
+    this.threadPoolSize = threadPoolSize;
+    this.nmMap = new ConcurrentHashMap<>();
+    this.nodeManagerResource = getNodeManagerResourceFromConf();
+  }
+
+  public void startNM() throws YarnException, IOException,
+      InterruptedException {
+    // nm configuration
+    int heartbeatInterval = conf.getInt(
+        SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
+        SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
+    float resourceUtilizationRatio = conf.getFloat(
+        SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO,
+        SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO_DEFAULT);
+    // nm information (fetch from topology file, or from sls/rumen json file)
+    Set<SLSRunner.NodeDetails> nodeSet = null;
+    if (nodeFile.isEmpty()) {
+      for (String inputTrace : inputTraces) {
+        switch (inputType) {
+          case SLS:
+            nodeSet = SLSUtils.parseNodesFromSLSTrace(inputTrace);
+            break;
+          case RUMEN:
+            nodeSet = SLSUtils.parseNodesFromRumenTrace(inputTrace);
+            break;
+          case SYNTH:
+            stjp = new SynthTraceJobProducer(conf, new Path(inputTraces[0]));
+            nodeSet = SLSUtils.generateNodes(stjp.getNumNodes(),
+                stjp.getNumNodes()/stjp.getNodesPerRack());
+            break;
+          default:
+            throw new YarnException("Input configuration not recognized, "
+                + "trace type should be SLS, RUMEN, or SYNTH");
+        }
+      }
+    } else {
+      nodeSet = SLSUtils.parseNodesFromNodeFile(nodeFile,
+          nodeManagerResource);
+    }
+
+    if (nodeSet == null || nodeSet.isEmpty()) {
+      throw new YarnException("No node! Please configure nodes.");
+    }
+
+    SLSUtils.generateNodeTableMapping(nodeSet, tableMapping);
+
+    // create NM simulators
+    Random random = new Random();
+    Set<String> rackSet = ConcurrentHashMap.newKeySet();
+    int threadPoolSize = Math.max(this.threadPoolSize,
+        SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT);
+    ExecutorService executorService = Executors.
+        newFixedThreadPool(threadPoolSize);
+    for (SLSRunner.NodeDetails nodeDetails : nodeSet) {
+      executorService.submit(new Runnable() {
+        @Override public void run() {
+          try {
+            // we randomize the heartbeat start time from zero to 1 interval
+            NMSimulator nm = new NMSimulator();
+            Resource nmResource = nodeManagerResource;
+            String hostName = nodeDetails.getHostname();
+            if (nodeDetails.getNodeResource() != null) {
+              nmResource = nodeDetails.getNodeResource();
+            }
+            Set<NodeLabel> nodeLabels = nodeDetails.getLabels();
+            nm.init(hostName, nmResource,
+                random.nextInt(heartbeatInterval),
+                heartbeatInterval, rm, resourceUtilizationRatio, nodeLabels);
+            nmMap.put(nm.getNode().getNodeID(), nm);
+            taskRunner.schedule(nm);
+            rackSet.add(nm.getNode().getRackName());
+          } catch (IOException | YarnException e) {
+            LOG.error("Got an error while adding node", e);
+          }
+        }
+      });
+    }
+    executorService.shutdown();
+    executorService.awaitTermination(10, TimeUnit.MINUTES);
+    numRacks = rackSet.size();
+    numNMs = nmMap.size();
+  }
+
+  void waitForNodesRunning() throws InterruptedException {
+    long startTimeMS = System.currentTimeMillis();
+    while (true) {
+      int numRunningNodes = 0;
+      for (RMNode node : rm.getRMContext().getRMNodes().values()) {
+        if (node.getState() == NodeState.RUNNING) {
+          numRunningNodes++;
+        }
+      }
+      if (numRunningNodes == numNMs) {
+        break;
+      }
+      LOG.info("SLSRunner is waiting for all nodes RUNNING."
+          + " {} of {} NMs initialized.", numRunningNodes, numNMs);
+      Thread.sleep(1000);
+    }
+    LOG.info("SLSRunner takes {} ms to launch all nodes.",
+        System.currentTimeMillis() - startTimeMS);
+  }
+
+  private Resource getNodeManagerResourceFromConf() {
+    Resource resource = Resources.createResource(0);
+    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+    for (ResourceInformation info : infors) {
+      long value;
+      if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+        value = conf.getInt(SLSConfiguration.NM_MEMORY_MB,
+            SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+      } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+        value = conf.getInt(SLSConfiguration.NM_VCORES,
+            SLSConfiguration.NM_VCORES_DEFAULT);
+      } else {
+        value = conf.getLong(SLSConfiguration.NM_PREFIX +
+            info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+      }
+
+      resource.setResourceValue(info.getName(), value);
+    }
+
+    return resource;
+  }
+
+  public void setNodeFile(String nodeFile) {
+    this.nodeFile = nodeFile;
+  }
+
+
+  public void setInputType(TraceType inputType) {
+    this.inputType = inputType;
+  }
+
+  public void setInputTraces(String[] inputTraces) {
+    this.inputTraces = inputTraces.clone();
+  }
+
+  public int getNumNMs() {
+    return numNMs;
+  }
+
+  public int getNumRacks() {
+    return numRacks;
+  }
+
+  public Resource getNodeManagerResource() {
+    return nodeManagerResource;
+  }
+
+  public Map<NodeId, NMSimulator> getNmMap() {
+    return nmMap;
+  }
+
+  public SynthTraceJobProducer getStjp() {
+    return stjp;
+  }
+
+  public void setTableMapping(String tableMapping) {
+    this.tableMapping = tableMapping;
+  }
+
+  public void setRm(ResourceManager rm) {
+    this.rm = rm;
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RMRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RMRunner.java
new file mode 100644
index 0000000000000..dbded4b306e19
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RMRunner.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.net.DNSToSwitchMapping;
+import org.apache.hadoop.net.TableMapping;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher;
+import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
+import org.apache.hadoop.yarn.sls.scheduler.SLSFairScheduler;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
+import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RMRunner {
+  private ResourceManager rm;
+  private String metricsOutputDir;
+  private Configuration conf;
+  private SLSRunner slsRunner;
+  private String tableMapping;
+  private Map<String, Integer> queueAppNumMap;
+
+  public RMRunner(Configuration conf, SLSRunner slsRunner) {
+    this.conf = conf;
+    this.slsRunner = slsRunner;
+    this.queueAppNumMap = new HashMap<>();
+  }
+
+  public void startRM() throws ClassNotFoundException, YarnException {
+    Configuration rmConf = new YarnConfiguration(conf);
+    String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
+
+    if (Class.forName(schedulerClass) == CapacityScheduler.class) {
+      rmConf.set(YarnConfiguration.RM_SCHEDULER,
+          SLSCapacityScheduler.class.getName());
+      rmConf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+      rmConf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+          ProportionalCapacityPreemptionPolicy.class.getName());
+    } else if (Class.forName(schedulerClass) == FairScheduler.class) {
+      rmConf.set(YarnConfiguration.RM_SCHEDULER,
+          SLSFairScheduler.class.getName());
+    } else if (Class.forName(schedulerClass) == FifoScheduler.class) {
+      // TODO add support for FifoScheduler
+      throw new YarnException("Fifo Scheduler is not supported yet.");
+    }
+    rmConf.setClass(
+        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+        TableMapping.class, DNSToSwitchMapping.class);
+    rmConf.set(
+        CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY,
+        tableMapping);
+    rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
+
+    rm = new ResourceManager() {
+      @Override
+      protected ApplicationMasterLauncher createAMLauncher() {
+        return new MockAMLauncher(slsRunner, this.rmContext);
+      }
+    };
+
+    // Across runs of parametrized tests, the JvmMetrics objects is retained,
+    // but is not registered correctly
+    JvmMetrics jvmMetrics = JvmMetrics.initSingleton("ResourceManager", null);
+    jvmMetrics.registerIfNeeded();
+
+    // Init and start the actual ResourceManager
+    rm.init(rmConf);
+    rm.start();
+  }
+
+  public void increaseQueueAppNum(String queue) throws YarnException {
+    SchedulerWrapper wrapper = (SchedulerWrapper)rm.getResourceScheduler();
+    String queueName = wrapper.getRealQueueName(queue);
+    Integer appNum = queueAppNumMap.get(queueName);
+    if (appNum == null) {
+      appNum = 1;
+    } else {
+      appNum = appNum + 1;
+    }
+
+    queueAppNumMap.put(queueName, appNum);
+    SchedulerMetrics metrics = wrapper.getSchedulerMetrics();
+    if (metrics != null) {
+      metrics.trackQueue(queueName);
+    }
+  }
+
+  public void setMetricsOutputDir(String metricsOutputDir) {
+    this.metricsOutputDir = metricsOutputDir;
+  }
+
+  public String getTableMapping() {
+    return tableMapping;
+  }
+
+  public void setTableMapping(String tableMapping) {
+    this.tableMapping = tableMapping;
+  }
+
+  public void stop() {
+    rm.stop();
+  }
+
+  public ResourceManager getRm() {
+    return rm;
+  }
+
+  public Map<String, Integer> getQueueAppNumMap() {
+    return queueAppNumMap;
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
index 7c10a57b1df32..2d04bb240c1a6 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/ReservationClientUtil.java
@@ -70,9 +70,7 @@ public static ReservationSubmissionRequest createMRReservation(
         deadline, reservationRequests, name);
 
     // outermost request
-    ReservationSubmissionRequest request = ReservationSubmissionRequest
+    return ReservationSubmissionRequest
         .newInstance(resDef, queueName, reservationId);
-
-    return request;
   }
 }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
index 76bcb157188e5..2cdfe236c410d 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
@@ -25,6 +25,7 @@
 import java.io.OutputStreamWriter;
 import java.io.Reader;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
@@ -54,7 +55,7 @@ public class RumenToSLSConverter {
   private static Map<String, Set<String>> rackNodeMap =
           new TreeMap<String, Set<String>>();
 
-  public static void main(String args[]) throws Exception {
+  public static void main(String[] args) throws Exception {
     Options options = new Options();
     options.addOption("input", true, "input rumen json file");
     options.addOption("outputJobs", true, "output jobs file");
@@ -121,9 +122,10 @@ public static void main(String args[]) throws Exception {
   private static void generateSLSLoadFile(String inputFile, String outputFile)
           throws IOException {
     try (Reader input =
-        new InputStreamReader(new FileInputStream(inputFile), "UTF-8")) {
+        new InputStreamReader(new FileInputStream(inputFile),
+            StandardCharsets.UTF_8)) {
       try (Writer output =
-          new OutputStreamWriter(new FileOutputStream(outputFile), "UTF-8")) {
+          new OutputStreamWriter(new FileOutputStream(outputFile), StandardCharsets.UTF_8)) {
         ObjectMapper mapper = new ObjectMapper();
         ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
         Iterator<Map> i = mapper.readValues(
@@ -140,7 +142,7 @@ private static void generateSLSLoadFile(String inputFile, String outputFile)
   private static void generateSLSNodeFile(String outputFile)
           throws IOException {
     try (Writer output =
-        new OutputStreamWriter(new FileOutputStream(outputFile), "UTF-8")) {
+        new OutputStreamWriter(new FileOutputStream(outputFile), StandardCharsets.UTF_8)) {
       ObjectMapper mapper = new ObjectMapper();
       ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
       for (Map.Entry<String, Set<String>> entry : rackNodeMap.entrySet()) {
@@ -218,7 +220,7 @@ private static List createSLSTasks(String taskType,
         task.put("container.priority", priority);
         task.put("container.type", taskType);
         array.add(task);
-        String rackHost[] = SLSUtils.getRackHostName(hostname);
+        String[] rackHost = SLSUtils.getRackHostName(hostname);
         if (rackNodeMap.containsKey(rackHost[0])) {
           rackNodeMap.get(rackHost[0]).add(rackHost[1]);
         } else {
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index a8d2aa6584bcb..c14d5685b7db0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -18,18 +18,13 @@
 package org.apache.hadoop.yarn.sls;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
 import java.security.Security;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
 import java.util.Collections;
@@ -38,9 +33,6 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.GnuParser;
@@ -52,50 +44,22 @@
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.TableMapping;
-import org.apache.hadoop.tools.rumen.JobTraceReader;
-import org.apache.hadoop.tools.rumen.LoggedJob;
-import org.apache.hadoop.tools.rumen.LoggedTask;
-import org.apache.hadoop.tools.rumen.LoggedTaskAttempt;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
-import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
 import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
 import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
-import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher;
-import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler;
-import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
-import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
-import org.apache.hadoop.yarn.sls.scheduler.SLSFairScheduler;
-import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
 import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
-import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
+import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
+import org.apache.hadoop.yarn.sls.scheduler.Tracker;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
-import org.apache.hadoop.yarn.sls.utils.SLSUtils;
-import org.apache.hadoop.yarn.util.UTCClock;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -103,47 +67,25 @@
 @Private
 @Unstable
 public class SLSRunner extends Configured implements Tool {
-  // RM, Runner
-  private ResourceManager rm;
-  private static TaskRunner runner = new TaskRunner();
+  private static final TaskRunner runner = new TaskRunner();
   private String[] inputTraces;
-  private Map<String, Integer> queueAppNumMap;
-  private int poolSize;
-
-  // NM simulator
-  private Map<NodeId, NMSimulator> nmMap;
-  private Resource nodeManagerResource;
-  private String nodeFile;
-
-  // AM simulator
-  private int AM_ID;
-  private Map<String, AMSimulator> amMap;
-  private Map<ApplicationId, AMSimulator> appIdAMSim;
-  private Set<String> trackedApps;
-  private Map<String, Class> amClassMap;
-  private static int remainingApps = 0;
 
   // metrics
-  private String metricsOutputDir;
   private boolean printSimulation;
 
-  // other simulation information
-  private int numNMs, numRacks, numAMs, numTasks;
-  private long maxRuntime;
-  private String tableMapping;
-
   private final static Map<String, Object> simulateInfoMap =
-          new HashMap<String, Object>();
+      new HashMap<>();
 
   // logger
   public final static Logger LOG = LoggerFactory.getLogger(SLSRunner.class);
 
-  private final static int DEFAULT_MAPPER_PRIORITY = 20;
-  private final static int DEFAULT_REDUCER_PRIORITY = 10;
-
   private static boolean exitAtTheFinish = false;
+  private AMRunner amRunner;
+  private RMRunner rmRunner;
+  private NMRunner nmRunner;
 
-  private static final String DEFAULT_USER = "default";
+  private TraceType inputType;
+  private SynthTraceJobProducer stjp;
 
   /**
    * The type of trace in input.
@@ -156,19 +98,16 @@ public enum TraceType {
   public static final String NETWORK_NEGATIVE_CACHE_TTL =
       "networkaddress.cache.negative.ttl";
 
-  private TraceType inputType;
-  private SynthTraceJobProducer stjp;
-
-  public static int getRemainingApps() {
-    return remainingApps;
+  public int getRemainingApps() {
+    return amRunner.remainingApps;
   }
 
-  public SLSRunner() throws ClassNotFoundException {
+  public SLSRunner() throws ClassNotFoundException, YarnException {
     Configuration tempConf = new Configuration(false);
     init(tempConf);
   }
 
-  public SLSRunner(Configuration tempConf) throws ClassNotFoundException {
+  public SLSRunner(Configuration tempConf) throws ClassNotFoundException, YarnException {
     init(tempConf);
   }
 
@@ -182,51 +121,31 @@ public void setConf(Configuration conf) {
     super.setConf(conf);
   }
 
-  private void init(Configuration tempConf) throws ClassNotFoundException {
-    nmMap = new ConcurrentHashMap<>();
-    queueAppNumMap = new HashMap<>();
-    amMap = new ConcurrentHashMap<>();
-    amClassMap = new HashMap<>();
-    appIdAMSim = new ConcurrentHashMap<>();
+  private void init(Configuration tempConf) throws ClassNotFoundException, YarnException {
     // runner configuration
     setConf(tempConf);
-
-    // runner
-    poolSize = tempConf.getInt(SLSConfiguration.RUNNER_POOL_SIZE,
+    
+    int poolSize = tempConf.getInt(SLSConfiguration.RUNNER_POOL_SIZE,
         SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT);
     SLSRunner.runner.setQueueSize(poolSize);
-    // <AMType, Class> map
-    for (Map.Entry e : tempConf) {
-      String key = e.getKey().toString();
-      if (key.startsWith(SLSConfiguration.AM_TYPE_PREFIX)) {
-        String amType = key.substring(SLSConfiguration.AM_TYPE_PREFIX.length());
-        amClassMap.put(amType, Class.forName(tempConf.get(key)));
-      }
-    }
 
-    nodeManagerResource = getNodeManagerResource();
+    rmRunner = new RMRunner(getConf(), this);
+    nmRunner = new NMRunner(runner, getConf(), rmRunner.getRm(), rmRunner.getTableMapping(), poolSize);
+    amRunner = new AMRunner(runner, this);
+    amRunner.init(tempConf);
   }
 
-  private Resource getNodeManagerResource() {
-    Resource resource = Resources.createResource(0);
-    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
-    for (ResourceInformation info : infors) {
-      long value;
-      if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
-        value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
-            SLSConfiguration.NM_MEMORY_MB_DEFAULT);
-      } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
-        value = getConf().getInt(SLSConfiguration.NM_VCORES,
-            SLSConfiguration.NM_VCORES_DEFAULT);
-      } else {
-        value = getConf().getLong(SLSConfiguration.NM_PREFIX +
-            info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+  private SynthTraceJobProducer getSynthJobTraceProducer() throws YarnException {
+    // if we use the nodeFile this could have been not initialized yet.
+    if (nmRunner.getStjp() != null) {
+      return nmRunner.getStjp();
+    } else {
+      try {
+        return new SynthTraceJobProducer(getConf(), new Path(inputTraces[0]));
+      } catch (IOException e) {
+        throw new YarnException("Failed to initialize SynthTraceJobProducer", e);
       }
-
-      resource.setResourceValue(info.getName(), value);
     }
-
-    return resource;
   }
 
   /**
@@ -236,39 +155,63 @@ public static Map<String, Object> getSimulateInfoMap() {
     return Collections.unmodifiableMap(simulateInfoMap);
   }
 
-  public void setSimulationParams(TraceType inType, String[] inTraces,
-      String nodes, String outDir, Set<String> trackApps,
-      boolean printsimulation) throws IOException, ClassNotFoundException {
-
-    this.inputType = inType;
+  /**
+   * This is invoked before start.
+   * @param inputType The trace type
+   * @param inTraces Input traces
+   * @param nodes The node file
+   * @param metricsOutputDir Output dir for metrics
+   * @param trackApps Track these applications
+   * @param printSimulation Whether to print the simulation
+   */
+  public void setSimulationParams(TraceType inputType, String[] inTraces,
+      String nodes, String metricsOutputDir, Set<String> trackApps,
+      boolean printSimulation) throws YarnException {
+    this.inputType = inputType;
     this.inputTraces = inTraces.clone();
-    this.nodeFile = nodes;
-    this.trackedApps = trackApps;
-    this.printSimulation = printsimulation;
-    metricsOutputDir = outDir;
-    tableMapping = outDir + "/tableMapping.csv";
+    this.amRunner.setInputType(inputType);
+    this.amRunner.setInputTraces(this.inputTraces);
+    this.amRunner.setTrackedApps(trackApps);
+    this.nmRunner.setNodeFile(nodes);
+    this.nmRunner.setInputType(inputType);
+    this.nmRunner.setInputTraces(this.inputTraces);
+    this.printSimulation = printSimulation;
+    this.rmRunner.setMetricsOutputDir(metricsOutputDir);
+    String tableMapping = metricsOutputDir + "/tableMapping.csv";
+    this.rmRunner.setTableMapping(tableMapping);
+    this.nmRunner.setTableMapping(tableMapping);
+    
+    //We need this.inputTraces to set before creating SynthTraceJobProducer
+    if (inputType == TraceType.SYNTH) {
+      this.stjp = getSynthJobTraceProducer();
+    }
   }
 
   public void start() throws IOException, ClassNotFoundException, YarnException,
       InterruptedException {
-
     enableDNSCaching(getConf());
 
     // start resource manager
-    startRM();
+    rmRunner.startRM();
+    nmRunner.setRm(rmRunner.getRm());
+    amRunner.setResourceManager(rmRunner.getRm());
+    
     // start node managers
-    startNM();
+    nmRunner.startNM();
     // start application masters
-    startAM();
+    amRunner.startAM();
+
     // set queue & tracked apps information
-    ((SchedulerWrapper) rm.getResourceScheduler()).getTracker()
-        .setQueueSet(this.queueAppNumMap.keySet());
-    ((SchedulerWrapper) rm.getResourceScheduler()).getTracker()
-        .setTrackedAppSet(this.trackedApps);
+    SchedulerWrapper resourceScheduler =
+        (SchedulerWrapper) rmRunner.getRm().getResourceScheduler();
+    resourceScheduler.setSLSRunner(this);
+    Tracker tracker = resourceScheduler.getTracker();
+    tracker.setQueueSet(rmRunner.getQueueAppNumMap().keySet());
+    tracker.setTrackedAppSet(amRunner.getTrackedApps());
     // print out simulation info
     printSimulationInfo();
     // blocked until all nodes RUNNING
-    waitForNodesRunning();
+    nmRunner.waitForNodesRunning();
     // starting the runner once everything is ready to go,
     runner.start();
   }
@@ -290,428 +233,7 @@ static void enableDNSCaching(Configuration conf) {
     }
   }
 
-  private void startRM() throws ClassNotFoundException, YarnException {
-    Configuration rmConf = new YarnConfiguration(getConf());
-    String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
-
-    if (Class.forName(schedulerClass) == CapacityScheduler.class) {
-      rmConf.set(YarnConfiguration.RM_SCHEDULER,
-          SLSCapacityScheduler.class.getName());
-      rmConf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
-      rmConf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
-          ProportionalCapacityPreemptionPolicy.class.getName());
-    } else if (Class.forName(schedulerClass) == FairScheduler.class) {
-      rmConf.set(YarnConfiguration.RM_SCHEDULER,
-          SLSFairScheduler.class.getName());
-    } else if (Class.forName(schedulerClass) == FifoScheduler.class) {
-      // TODO add support for FifoScheduler
-      throw new YarnException("Fifo Scheduler is not supported yet.");
-    }
-    rmConf.setClass(
-        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
-        TableMapping.class, DNSToSwitchMapping.class);
-    rmConf.set(
-        CommonConfigurationKeysPublic.NET_TOPOLOGY_TABLE_MAPPING_FILE_KEY,
-        tableMapping);
-    rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
-
-    final SLSRunner se = this;
-    rm = new ResourceManager() {
-      @Override
-      protected ApplicationMasterLauncher createAMLauncher() {
-        return new MockAMLauncher(se, this.rmContext, appIdAMSim);
-      }
-    };
-
-    // Across runs of parametrized tests, the JvmMetrics objects is retained,
-    // but is not registered correctly
-    JvmMetrics jvmMetrics = JvmMetrics.initSingleton("ResourceManager", null);
-    jvmMetrics.registerIfNeeded();
-
-    // Init and start the actual ResourceManager
-    rm.init(rmConf);
-    rm.start();
-  }
-
-  private void startNM() throws YarnException, IOException,
-      InterruptedException {
-    // nm configuration
-    int heartbeatInterval = getConf().getInt(
-        SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
-        SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
-    float resourceUtilizationRatio = getConf().getFloat(
-        SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO,
-        SLSConfiguration.NM_RESOURCE_UTILIZATION_RATIO_DEFAULT);
-    // nm information (fetch from topology file, or from sls/rumen json file)
-    Set<NodeDetails> nodeSet = null;
-    if (nodeFile.isEmpty()) {
-      for (String inputTrace : inputTraces) {
-        switch (inputType) {
-        case SLS:
-          nodeSet = SLSUtils.parseNodesFromSLSTrace(inputTrace);
-          break;
-        case RUMEN:
-          nodeSet = SLSUtils.parseNodesFromRumenTrace(inputTrace);
-          break;
-        case SYNTH:
-          stjp = new SynthTraceJobProducer(getConf(), new Path(inputTraces[0]));
-          nodeSet = SLSUtils.generateNodes(stjp.getNumNodes(),
-              stjp.getNumNodes()/stjp.getNodesPerRack());
-          break;
-        default:
-          throw new YarnException("Input configuration not recognized, "
-              + "trace type should be SLS, RUMEN, or SYNTH");
-        }
-      }
-    } else {
-      nodeSet = SLSUtils.parseNodesFromNodeFile(nodeFile,
-          nodeManagerResource);
-    }
-
-    if (nodeSet == null || nodeSet.isEmpty()) {
-      throw new YarnException("No node! Please configure nodes.");
-    }
-
-    SLSUtils.generateNodeTableMapping(nodeSet, tableMapping);
-
-    // create NM simulators
-    Random random = new Random();
-    Set<String> rackSet = ConcurrentHashMap.newKeySet();
-    int threadPoolSize = Math.max(poolSize,
-        SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT);
-    ExecutorService executorService = Executors.
-        newFixedThreadPool(threadPoolSize);
-    for (NodeDetails nodeDetails : nodeSet) {
-      executorService.submit(new Runnable() {
-        @Override public void run() {
-          try {
-            // we randomize the heartbeat start time from zero to 1 interval
-            NMSimulator nm = new NMSimulator();
-            Resource nmResource = nodeManagerResource;
-            String hostName = nodeDetails.getHostname();
-            if (nodeDetails.getNodeResource() != null) {
-              nmResource = nodeDetails.getNodeResource();
-            }
-            Set<NodeLabel> nodeLabels = nodeDetails.getLabels();
-            nm.init(hostName, nmResource,
-                random.nextInt(heartbeatInterval),
-                heartbeatInterval, rm, resourceUtilizationRatio, nodeLabels);
-            nmMap.put(nm.getNode().getNodeID(), nm);
-            runner.schedule(nm);
-            rackSet.add(nm.getNode().getRackName());
-          } catch (IOException | YarnException e) {
-            LOG.error("Got an error while adding node", e);
-          }
-        }
-      });
-    }
-    executorService.shutdown();
-    executorService.awaitTermination(10, TimeUnit.MINUTES);
-    numRacks = rackSet.size();
-    numNMs = nmMap.size();
-  }
-
-  private void waitForNodesRunning() throws InterruptedException {
-    long startTimeMS = System.currentTimeMillis();
-    while (true) {
-      int numRunningNodes = 0;
-      for (RMNode node : rm.getRMContext().getRMNodes().values()) {
-        if (node.getState() == NodeState.RUNNING) {
-          numRunningNodes++;
-        }
-      }
-      if (numRunningNodes == numNMs) {
-        break;
-      }
-      LOG.info("SLSRunner is waiting for all nodes RUNNING."
-          + " {} of {} NMs initialized.", numRunningNodes, numNMs);
-      Thread.sleep(1000);
-    }
-    LOG.info("SLSRunner takes {} ms to launch all nodes.",
-        System.currentTimeMillis() - startTimeMS);
-  }
-
-  @SuppressWarnings("unchecked")
-  private void startAM() throws YarnException, IOException {
-    switch (inputType) {
-    case SLS:
-      for (String inputTrace : inputTraces) {
-        startAMFromSLSTrace(inputTrace);
-      }
-      break;
-    case RUMEN:
-      long baselineTimeMS = 0;
-      for (String inputTrace : inputTraces) {
-        startAMFromRumenTrace(inputTrace, baselineTimeMS);
-      }
-      break;
-    case SYNTH:
-      startAMFromSynthGenerator();
-      break;
-    default:
-      throw new YarnException("Input configuration not recognized, "
-          + "trace type should be SLS, RUMEN, or SYNTH");
-    }
-
-    numAMs = amMap.size();
-    remainingApps = numAMs;
-  }
-
-  /**
-   * Parse workload from a SLS trace file.
-   */
-  @SuppressWarnings("unchecked")
-  private void startAMFromSLSTrace(String inputTrace) throws IOException {
-    JsonFactory jsonF = new JsonFactory();
-    ObjectMapper mapper = new ObjectMapper();
-
-    try (Reader input = new InputStreamReader(
-        new FileInputStream(inputTrace), "UTF-8")) {
-      Iterator<Map> jobIter = mapper.readValues(
-          jsonF.createParser(input), Map.class);
-
-      while (jobIter.hasNext()) {
-        try {
-          createAMForJob(jobIter.next());
-        } catch (Exception e) {
-          LOG.error("Failed to create an AM: {}", e.getMessage());
-        }
-      }
-    }
-  }
-
-  private void createAMForJob(Map jsonJob) throws YarnException {
-    long jobStartTime = Long.parseLong(
-        jsonJob.get(SLSConfiguration.JOB_START_MS).toString());
-
-    long jobFinishTime = 0;
-    if (jsonJob.containsKey(SLSConfiguration.JOB_END_MS)) {
-      jobFinishTime = Long.parseLong(
-          jsonJob.get(SLSConfiguration.JOB_END_MS).toString());
-    }
-
-    String jobLabelExpr = null;
-    if (jsonJob.containsKey(SLSConfiguration.JOB_LABEL_EXPR)) {
-      jobLabelExpr = jsonJob.get(SLSConfiguration.JOB_LABEL_EXPR).toString();
-    }
-
-    String user = (String) jsonJob.get(SLSConfiguration.JOB_USER);
-    if (user == null) {
-      user = "default";
-    }
-
-    String queue = jsonJob.get(SLSConfiguration.JOB_QUEUE_NAME).toString();
-    increaseQueueAppNum(queue);
-
-    String amType = (String)jsonJob.get(SLSConfiguration.AM_TYPE);
-    if (amType == null) {
-      amType = SLSUtils.DEFAULT_JOB_TYPE;
-    }
-
-    int jobCount = 1;
-    if (jsonJob.containsKey(SLSConfiguration.JOB_COUNT)) {
-      jobCount = Integer.parseInt(
-          jsonJob.get(SLSConfiguration.JOB_COUNT).toString());
-    }
-    jobCount = Math.max(jobCount, 1);
-
-    String oldAppId = (String)jsonJob.get(SLSConfiguration.JOB_ID);
-    // Job id is generated automatically if this job configuration allows
-    // multiple job instances
-    if(jobCount > 1) {
-      oldAppId = null;
-    }
-
-    for (int i = 0; i < jobCount; i++) {
-      runNewAM(amType, user, queue, oldAppId, jobStartTime, jobFinishTime,
-          getTaskContainers(jsonJob), getAMContainerResource(jsonJob),
-          jobLabelExpr);
-    }
-  }
-
-  private List<ContainerSimulator> getTaskContainers(Map jsonJob)
-      throws YarnException {
-    List<ContainerSimulator> containers = new ArrayList<>();
-    List tasks = (List) jsonJob.get(SLSConfiguration.JOB_TASKS);
-    if (tasks == null || tasks.size() == 0) {
-      throw new YarnException("No task for the job!");
-    }
-
-    for (Object o : tasks) {
-      Map jsonTask = (Map) o;
-
-      String hostname = (String) jsonTask.get(SLSConfiguration.TASK_HOST);
-
-      long duration = 0;
-      if (jsonTask.containsKey(SLSConfiguration.TASK_DURATION_MS)) {
-        duration = Integer.parseInt(
-            jsonTask.get(SLSConfiguration.TASK_DURATION_MS).toString());
-      } else if (jsonTask.containsKey(SLSConfiguration.DURATION_MS)) {
-        // Also support "duration.ms" for backward compatibility
-        duration = Integer.parseInt(
-            jsonTask.get(SLSConfiguration.DURATION_MS).toString());
-      } else if (jsonTask.containsKey(SLSConfiguration.TASK_START_MS) &&
-          jsonTask.containsKey(SLSConfiguration.TASK_END_MS)) {
-        long taskStart = Long.parseLong(
-            jsonTask.get(SLSConfiguration.TASK_START_MS).toString());
-        long taskFinish = Long.parseLong(
-            jsonTask.get(SLSConfiguration.TASK_END_MS).toString());
-        duration = taskFinish - taskStart;
-      }
-      if (duration <= 0) {
-        throw new YarnException("Duration of a task shouldn't be less or equal"
-            + " to 0!");
-      }
-
-      Resource res = getResourceForContainer(jsonTask);
-
-      int priority = DEFAULT_MAPPER_PRIORITY;
-      if (jsonTask.containsKey(SLSConfiguration.TASK_PRIORITY)) {
-        priority = Integer.parseInt(
-            jsonTask.get(SLSConfiguration.TASK_PRIORITY).toString());
-      }
-
-      String type = "map";
-      if (jsonTask.containsKey(SLSConfiguration.TASK_TYPE)) {
-        type = jsonTask.get(SLSConfiguration.TASK_TYPE).toString();
-      }
-
-      int count = 1;
-      if (jsonTask.containsKey(SLSConfiguration.COUNT)) {
-        count = Integer.parseInt(
-            jsonTask.get(SLSConfiguration.COUNT).toString());
-      }
-      count = Math.max(count, 1);
-
-      ExecutionType executionType = ExecutionType.GUARANTEED;
-      if (jsonTask.containsKey(SLSConfiguration.TASK_EXECUTION_TYPE)) {
-        executionType = ExecutionType.valueOf(
-            jsonTask.get(SLSConfiguration.TASK_EXECUTION_TYPE).toString());
-      }
-      long allocationId = -1;
-      if (jsonTask.containsKey(SLSConfiguration.TASK_ALLOCATION_ID)) {
-        allocationId = Long.parseLong(
-            jsonTask.get(SLSConfiguration.TASK_ALLOCATION_ID).toString());
-      }
-
-      long requestDelay = 0;
-      if (jsonTask.containsKey(SLSConfiguration.TASK_REQUEST_DELAY)) {
-        requestDelay = Long.parseLong(
-            jsonTask.get(SLSConfiguration.TASK_REQUEST_DELAY).toString());
-      }
-      requestDelay = Math.max(requestDelay, 0);
-
-      for (int i = 0; i < count; i++) {
-        containers.add(
-            new ContainerSimulator(res, duration, hostname, priority, type,
-                executionType, allocationId, requestDelay));
-      }
-    }
-
-    return containers;
-  }
-
-  private Resource getResourceForContainer(Map jsonTask) {
-    Resource res = getDefaultContainerResource();
-    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
-    for (ResourceInformation info : infors) {
-      if (jsonTask.containsKey(SLSConfiguration.TASK_PREFIX + info.getName())) {
-        long value = Long.parseLong(
-            jsonTask.get(SLSConfiguration.TASK_PREFIX + info.getName())
-                .toString());
-        res.setResourceValue(info.getName(), value);
-      }
-    }
-
-    return res;
-  }
-
-  /**
-   * Parse workload from a rumen trace file.
-   */
-  @SuppressWarnings("unchecked")
-  private void startAMFromRumenTrace(String inputTrace, long baselineTimeMS)
-      throws IOException {
-    Configuration conf = new Configuration();
-    conf.set("fs.defaultFS", "file:///");
-    File fin = new File(inputTrace);
-
-    try (JobTraceReader reader = new JobTraceReader(
-        new Path(fin.getAbsolutePath()), conf)) {
-      LoggedJob job = reader.getNext();
-
-      while (job != null) {
-        try {
-          createAMForJob(job, baselineTimeMS);
-        } catch (Exception e) {
-          LOG.error("Failed to create an AM", e);
-        }
-
-        job = reader.getNext();
-      }
-    }
-  }
-
-  private void createAMForJob(LoggedJob job, long baselineTimeMs)
-      throws YarnException {
-    String user = job.getUser() == null ? "default" :
-        job.getUser().getValue();
-    String jobQueue = job.getQueue().getValue();
-    String oldJobId = job.getJobID().toString();
-    long jobStartTimeMS = job.getSubmitTime();
-    long jobFinishTimeMS = job.getFinishTime();
-    if (baselineTimeMs == 0) {
-      baselineTimeMs = job.getSubmitTime();
-    }
-    jobStartTimeMS -= baselineTimeMs;
-    jobFinishTimeMS -= baselineTimeMs;
-    if (jobStartTimeMS < 0) {
-      LOG.warn("Warning: reset job {} start time to 0.", oldJobId);
-      jobFinishTimeMS = jobFinishTimeMS - jobStartTimeMS;
-      jobStartTimeMS = 0;
-    }
-
-    increaseQueueAppNum(jobQueue);
-
-    List<ContainerSimulator> containerList = new ArrayList<>();
-    // mapper
-    for (LoggedTask mapTask : job.getMapTasks()) {
-      if (mapTask.getAttempts().size() == 0) {
-        throw new YarnException("Invalid map task, no attempt for a mapper!");
-      }
-      LoggedTaskAttempt taskAttempt =
-          mapTask.getAttempts().get(mapTask.getAttempts().size() - 1);
-      String hostname = taskAttempt.getHostName().getValue();
-      long containerLifeTime = taskAttempt.getFinishTime() -
-          taskAttempt.getStartTime();
-      containerList.add(
-          new ContainerSimulator(getDefaultContainerResource(),
-              containerLifeTime, hostname, DEFAULT_MAPPER_PRIORITY, "map"));
-    }
-
-    // reducer
-    for (LoggedTask reduceTask : job.getReduceTasks()) {
-      if (reduceTask.getAttempts().size() == 0) {
-        throw new YarnException(
-            "Invalid reduce task, no attempt for a reducer!");
-      }
-      LoggedTaskAttempt taskAttempt =
-          reduceTask.getAttempts().get(reduceTask.getAttempts().size() - 1);
-      String hostname = taskAttempt.getHostName().getValue();
-      long containerLifeTime = taskAttempt.getFinishTime() -
-          taskAttempt.getStartTime();
-      containerList.add(
-          new ContainerSimulator(getDefaultContainerResource(),
-              containerLifeTime, hostname, DEFAULT_REDUCER_PRIORITY, "reduce"));
-    }
-
-    // Only supports the default job type currently
-    runNewAM(SLSUtils.DEFAULT_JOB_TYPE, user, jobQueue, oldJobId,
-        jobStartTimeMS, jobFinishTimeMS, containerList,
-        getAMContainerResource(null));
-  }
-
-  private Resource getDefaultContainerResource() {
+  Resource getDefaultContainerResource() {
     int containerMemory = getConf().getInt(SLSConfiguration.CONTAINER_MEMORY_MB,
         SLSConfiguration.CONTAINER_MEMORY_MB_DEFAULT);
     int containerVCores = getConf().getInt(SLSConfiguration.CONTAINER_VCORES,
@@ -719,178 +241,23 @@ private Resource getDefaultContainerResource() {
     return Resources.createResource(containerMemory, containerVCores);
   }
 
-  /**
-   * parse workload information from synth-generator trace files.
-   */
-  @SuppressWarnings("unchecked")
-  private void startAMFromSynthGenerator() throws YarnException, IOException {
-    Configuration localConf = new Configuration();
-    localConf.set("fs.defaultFS", "file:///");
-    long baselineTimeMS = 0;
-
-    // if we use the nodeFile this could have been not initialized yet.
-    if (stjp == null) {
-      stjp = new SynthTraceJobProducer(getConf(), new Path(inputTraces[0]));
-    }
-
-    SynthJob job = null;
-    // we use stjp, a reference to the job producer instantiated during node
-    // creation
-    while ((job = (SynthJob) stjp.getNextJob()) != null) {
-      // only support MapReduce currently
-      String user = job.getUser() == null ? DEFAULT_USER :
-              job.getUser();
-      String jobQueue = job.getQueueName();
-      String oldJobId = job.getJobID().toString();
-      long jobStartTimeMS = job.getSubmissionTime();
-
-      // CARLO: Finish time is only used for logging, omit for now
-      long jobFinishTimeMS = jobStartTimeMS + job.getDuration();
-
-      if (baselineTimeMS == 0) {
-        baselineTimeMS = jobStartTimeMS;
-      }
-      jobStartTimeMS -= baselineTimeMS;
-      jobFinishTimeMS -= baselineTimeMS;
-      if (jobStartTimeMS < 0) {
-        LOG.warn("Warning: reset job {} start time to 0.", oldJobId);
-        jobFinishTimeMS = jobFinishTimeMS - jobStartTimeMS;
-        jobStartTimeMS = 0;
-      }
-
-      increaseQueueAppNum(jobQueue);
-
-      List<ContainerSimulator> containerList =
-          new ArrayList<ContainerSimulator>();
-      ArrayList<NodeId> keyAsArray = new ArrayList<NodeId>(nmMap.keySet());
-      Random rand = new Random(stjp.getSeed());
-
-      for (SynthJob.SynthTask task : job.getTasks()) {
-        RMNode node = nmMap.get(keyAsArray.get(rand.nextInt(keyAsArray.size())))
-            .getNode();
-        String hostname = "/" + node.getRackName() + "/" + node.getHostName();
-        long containerLifeTime = task.getTime();
-        Resource containerResource = Resource
-            .newInstance((int) task.getMemory(), (int) task.getVcores());
-        containerList.add(
-            new ContainerSimulator(containerResource, containerLifeTime,
-                hostname, task.getPriority(), task.getType(),
-                task.getExecutionType()));
-      }
-
-
-      ReservationId reservationId = null;
-
-      if(job.hasDeadline()){
-        reservationId = ReservationId
-            .newInstance(this.rm.getStartTime(), AM_ID);
-      }
-
-      runNewAM(job.getType(), user, jobQueue, oldJobId,
-          jobStartTimeMS, jobFinishTimeMS, containerList, reservationId,
-          job.getDeadline(), getAMContainerResource(null), null,
-          job.getParams());
-    }
-  }
-
-  private Resource getAMContainerResource(Map jsonJob) {
-    Resource amContainerResource =
-        SLSConfiguration.getAMContainerResource(getConf());
-
-    if (jsonJob == null) {
-      return amContainerResource;
-    }
-
-    ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
-    for (ResourceInformation info : infors) {
-      String key = SLSConfiguration.JOB_AM_PREFIX + info.getName();
-      if (jsonJob.containsKey(key)) {
-        long value = Long.parseLong(jsonJob.get(key).toString());
-        amContainerResource.setResourceValue(info.getName(), value);
-      }
-    }
-
-    return amContainerResource;
-  }
-
-  private void increaseQueueAppNum(String queue) throws YarnException {
-    SchedulerWrapper wrapper = (SchedulerWrapper)rm.getResourceScheduler();
-    String queueName = wrapper.getRealQueueName(queue);
-    Integer appNum = queueAppNumMap.get(queueName);
-    if (appNum == null) {
-      appNum = 1;
-    } else {
-      appNum = appNum + 1;
-    }
-
-    queueAppNumMap.put(queueName, appNum);
-    SchedulerMetrics metrics = wrapper.getSchedulerMetrics();
-    if (metrics != null) {
-      metrics.trackQueue(queueName);
-    }
-  }
-
-  private void runNewAM(String jobType, String user,
-      String jobQueue, String oldJobId, long jobStartTimeMS,
-      long jobFinishTimeMS, List<ContainerSimulator> containerList,
-      Resource amContainerResource) {
-    runNewAM(jobType, user, jobQueue, oldJobId, jobStartTimeMS,
-        jobFinishTimeMS, containerList, null,  -1,
-        amContainerResource, null, null);
-  }
-
-  private void runNewAM(String jobType, String user,
-      String jobQueue, String oldJobId, long jobStartTimeMS,
-      long jobFinishTimeMS, List<ContainerSimulator> containerList,
-      Resource amContainerResource, String labelExpr) {
-    runNewAM(jobType, user, jobQueue, oldJobId, jobStartTimeMS,
-        jobFinishTimeMS, containerList, null,  -1,
-        amContainerResource, labelExpr, null);
-  }
-
-  @SuppressWarnings("checkstyle:parameternumber")
-  private void runNewAM(String jobType, String user,
-      String jobQueue, String oldJobId, long jobStartTimeMS,
-      long jobFinishTimeMS, List<ContainerSimulator> containerList,
-      ReservationId reservationId, long deadline, Resource amContainerResource,
-      String labelExpr, Map<String, String> params) {
-    AMSimulator amSim = (AMSimulator) ReflectionUtils.newInstance(
-        amClassMap.get(jobType), new Configuration());
-
-    if (amSim != null) {
-      int heartbeatInterval = getConf().getInt(
-          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS,
-          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS_DEFAULT);
-      boolean isTracked = trackedApps.contains(oldJobId);
-
-      if (oldJobId == null) {
-        oldJobId = Integer.toString(AM_ID);
-      }
-      AM_ID++;
-      amSim.init(heartbeatInterval, containerList, rm, this, jobStartTimeMS,
-          jobFinishTimeMS, user, jobQueue, isTracked, oldJobId,
-          runner.getStartTimeMS(), amContainerResource, labelExpr, params,
-          appIdAMSim);
-      if(reservationId != null) {
-        // if we have a ReservationId, delegate reservation creation to
-        // AMSim (reservation shape is impl specific)
-        UTCClock clock = new UTCClock();
-        amSim.initReservation(reservationId, deadline, clock.getTime());
-      }
-      runner.schedule(amSim);
-      maxRuntime = Math.max(maxRuntime, jobFinishTimeMS);
-      numTasks += containerList.size();
-      amMap.put(oldJobId, amSim);
-    }
+  public void increaseQueueAppNum(String queue) throws YarnException {
+    rmRunner.increaseQueueAppNum(queue);
   }
 
   private void printSimulationInfo() {
+    final int numAMs = amRunner.getNumAMs();
+    final int numTasks = amRunner.getNumTasks();
+    final long maxRuntime = amRunner.getMaxRuntime();
+    Map<String, AMSimulator> amMap = amRunner.getAmMap();
+    Map<String, Integer> queueAppNumMap = rmRunner.getQueueAppNumMap();
+
     if (printSimulation) {
       // node
       LOG.info("------------------------------------");
       LOG.info("# nodes = {}, # racks = {}, capacity " +
               "of each node {}.",
-              numNMs, numRacks, nodeManagerResource);
+              nmRunner.getNumNMs(), nmRunner.getNumRacks(), nmRunner.getNodeManagerResource());
       LOG.info("------------------------------------");
       // job
       LOG.info("# applications = {}, # total " +
@@ -914,12 +281,12 @@ private void printSimulationInfo() {
       LOG.info("------------------------------------");
     }
     // package these information in the simulateInfoMap used by other places
-    simulateInfoMap.put("Number of racks", numRacks);
-    simulateInfoMap.put("Number of nodes", numNMs);
+    simulateInfoMap.put("Number of racks", nmRunner.getNumRacks());
+    simulateInfoMap.put("Number of nodes", nmRunner.getNumNMs());
     simulateInfoMap.put("Node memory (MB)",
-        nodeManagerResource.getResourceValue(ResourceInformation.MEMORY_URI));
+        nmRunner.getNodeManagerResource().getResourceValue(ResourceInformation.MEMORY_URI));
     simulateInfoMap.put("Node VCores",
-        nodeManagerResource.getResourceValue(ResourceInformation.VCORES_URI));
+        nmRunner.getNodeManagerResource().getResourceValue(ResourceInformation.VCORES_URI));
     simulateInfoMap.put("Number of applications", numAMs);
     simulateInfoMap.put("Number of tasks", numTasks);
     simulateInfoMap.put("Average tasks per applicaion",
@@ -932,11 +299,14 @@ private void printSimulationInfo() {
   }
 
   public Map<NodeId, NMSimulator> getNmMap() {
-    return nmMap;
+    return nmRunner.getNmMap();
   }
 
-  public static void decreaseRemainingApps() {
-    remainingApps--;
+  public void decreaseRemainingApps() {
+    amRunner.remainingApps--;
+    if (amRunner.remainingApps == 0) {
+      exitSLSRunner();
+    }
   }
 
   public static void exitSLSRunner() {
@@ -947,13 +317,12 @@ public static void exitSLSRunner() {
   }
 
   public void stop() throws InterruptedException {
-    rm.stop();
+    rmRunner.stop();
     runner.stop();
   }
 
   public int run(final String[] argv) throws IOException, InterruptedException,
       ParseException, ClassNotFoundException, YarnException {
-
     Options options = new Options();
 
     // Left for compatibility
@@ -974,24 +343,15 @@ public int run(final String[] argv) throws IOException, InterruptedException,
     CommandLineParser parser = new GnuParser();
     CommandLine cmd = parser.parse(options, argv);
 
-    String traceType = null;
-    String traceLocation = null;
-
     // compatibility with old commandline
-    if (cmd.hasOption("inputrumen")) {
-      traceType = "RUMEN";
-      traceLocation = cmd.getOptionValue("inputrumen");
-    }
-    if (cmd.hasOption("inputsls")) {
-      traceType = "SLS";
-      traceLocation = cmd.getOptionValue("inputsls");
-    }
-
-    if (cmd.hasOption("tracetype")) {
-      traceType = cmd.getOptionValue("tracetype");
-      traceLocation = cmd.getOptionValue("tracelocation");
-    }
-
+    boolean hasInputRumenOption = cmd.hasOption("inputrumen");
+    boolean hasInputSlsOption = cmd.hasOption("inputsls");
+    boolean hasTraceTypeOption = cmd.hasOption("tracetype");
+    TraceType traceType = determineTraceType(cmd, hasInputRumenOption,
+        hasInputSlsOption, hasTraceTypeOption);
+    String traceLocation = determineTraceLocation(cmd, hasInputRumenOption,
+        hasInputSlsOption, hasTraceTypeOption);
+    
     String output = cmd.getOptionValue("output");
 
     File outputFile = new File(output);
@@ -1001,41 +361,66 @@ public int run(final String[] argv) throws IOException, InterruptedException,
       throw new YarnException("Cannot create output directory");
     }
 
-    Set<String> trackedJobSet = new HashSet<String>();
+    Set<String> trackedJobSet = new HashSet<>();
     if (cmd.hasOption("trackjobs")) {
       String trackjobs = cmd.getOptionValue("trackjobs");
-      String jobIds[] = trackjobs.split(",");
+      String[] jobIds = trackjobs.split(",");
       trackedJobSet.addAll(Arrays.asList(jobIds));
     }
 
     String tempNodeFile =
         cmd.hasOption("nodes") ? cmd.getOptionValue("nodes") : "";
 
-    TraceType tempTraceType = TraceType.SLS;
+    String[] inputFiles = traceLocation.split(",");
+
+    setSimulationParams(traceType, inputFiles, tempNodeFile, output,
+        trackedJobSet, cmd.hasOption("printsimulation"));
+    
+    start();
+
+    return 0;
+  }
+
+  private TraceType determineTraceType(CommandLine cmd, boolean hasInputRumenOption,
+      boolean hasInputSlsOption, boolean hasTraceTypeOption) throws YarnException {
+    String traceType = null;
+    if (hasInputRumenOption) {
+      traceType = "RUMEN";
+    }
+    if (hasInputSlsOption) {
+      traceType = "SLS";
+    }
+    if (hasTraceTypeOption) {
+      traceType = cmd.getOptionValue("tracetype");
+    }
+    if (traceType == null) {
+      throw new YarnException("Misconfigured input");
+    }
     switch (traceType) {
     case "SLS":
-      tempTraceType = TraceType.SLS;
-      break;
+      return TraceType.SLS;
     case "RUMEN":
-      tempTraceType = TraceType.RUMEN;
-      break;
-
+      return TraceType.RUMEN;
     case "SYNTH":
-      tempTraceType = TraceType.SYNTH;
-      break;
+      return TraceType.SYNTH;
     default:
       printUsage();
       throw new YarnException("Misconfigured input");
     }
+  }
 
-    String[] inputFiles = traceLocation.split(",");
-
-    setSimulationParams(tempTraceType, inputFiles, tempNodeFile, output,
-        trackedJobSet, cmd.hasOption("printsimulation"));
-
-    start();
-
-    return 0;
+  private String determineTraceLocation(CommandLine cmd, boolean hasInputRumenOption,
+      boolean hasInputSlsOption, boolean hasTraceTypeOption) throws YarnException {
+    if (hasInputRumenOption) {
+      return cmd.getOptionValue("inputrumen");
+    }
+    if (hasInputSlsOption) {
+      return cmd.getOptionValue("inputsls");
+    }
+    if (hasTraceTypeOption) {
+      return cmd.getOptionValue("tracelocation");
+    }
+    throw new YarnException("Misconfigured input! ");
   }
 
   public static void main(String[] argv) throws Exception {
@@ -1106,9 +491,7 @@ public boolean equals(Object o) {
       NodeDetails that = (NodeDetails) o;
 
       return StringUtils.equals(hostname, that.hostname) && (
-          nodeResource == null ?
-              that.nodeResource == null :
-              nodeResource.equals(that.nodeResource)) && SetUtils
+          Objects.equals(nodeResource, that.nodeResource)) && SetUtils
           .isEqualSet(labels, that.labels);
     }
 
@@ -1121,4 +504,16 @@ public int hashCode() {
       return result;
     }
   }
+
+  public SynthTraceJobProducer getStjp() {
+    return stjp;
+  }
+
+  public void setStjp(SynthTraceJobProducer stjp) {
+    this.stjp = stjp;
+  }
+
+  public AMSimulator getAMSimulatorByAppId(ApplicationId appId) {
+    return amRunner.getAMSimulator(appId);
+  }
 }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/TaskContainerDefinition.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/TaskContainerDefinition.java
new file mode 100644
index 0000000000000..04bea3ab2cd83
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/TaskContainerDefinition.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.sls.AMDefinitionRumen.DEFAULT_MAPPER_PRIORITY;
+
+public class TaskContainerDefinition {
+  private long duration;
+  private Resource resource;
+  private int priority;
+  private String type;
+  private int count;
+  private ExecutionType executionType;
+  private long allocationId = -1;
+  private long requestDelay = 0;
+  private String hostname;
+
+  public long getDuration() {
+    return duration;
+  }
+
+  public Resource getResource() {
+    return resource;
+  }
+
+  public int getPriority() {
+    return priority;
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public int getCount() {
+    return count;
+  }
+
+  public ExecutionType getExecutionType() {
+    return executionType;
+  }
+
+  public long getAllocationId() {
+    return allocationId;
+  }
+
+  public long getRequestDelay() {
+    return requestDelay;
+  }
+
+  public String getHostname() {
+    return hostname;
+  }
+
+  public static final class Builder {
+    private long duration = -1;
+    private long durationLegacy = -1;
+    private long taskStart = -1;
+    private long taskFinish = -1;
+    private Resource resource;
+    private int priority = DEFAULT_MAPPER_PRIORITY;
+    private String type = "map";
+    private int count = 1;
+    private ExecutionType executionType = ExecutionType.GUARANTEED;
+    private long allocationId = -1;
+    private long requestDelay = 0;
+    private String hostname;
+
+    public static Builder create() {
+      return new Builder();
+    }
+
+    public Builder withDuration(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.duration = Integer.parseInt(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withDuration(long duration) {
+      this.duration = duration;
+      return this;
+    }
+
+    /**
+     * Also support "duration.ms" for backward compatibility.
+     * @param jsonTask the json representation of the task.
+     * @param key The json key.
+     * @return the builder
+     */
+    public Builder withDurationLegacy(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.durationLegacy = Integer.parseInt(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withTaskStart(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.taskStart = Long.parseLong(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withTaskFinish(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.taskFinish = Long.parseLong(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withResource(Resource resource) {
+      this.resource = resource;
+      return this;
+    }
+
+    public Builder withPriority(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.priority = Integer.parseInt(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withPriority(int priority) {
+      this.priority = priority;
+      return this;
+    }
+
+    public Builder withType(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.type = jsonTask.get(key);
+      }
+      return this;
+    }
+
+    public Builder withType(String type) {
+      this.type = type;
+      return this;
+    }
+
+    public Builder withCount(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        count = Integer.parseInt(jsonTask.get(key));
+        count = Math.max(count, 1);
+      }
+      return this;
+    }
+
+    public Builder withCount(int count) {
+      this.count = count;
+      return this;
+    }
+
+    public Builder withExecutionType(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.executionType = ExecutionType.valueOf(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withExecutionType(ExecutionType executionType) {
+      this.executionType = executionType;
+      return this;
+    }
+
+    public Builder withAllocationId(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        this.allocationId = Long.parseLong(jsonTask.get(key));
+      }
+      return this;
+    }
+
+    public Builder withAllocationId(long allocationId) {
+      this.allocationId = allocationId;
+      return this;
+    }
+
+    public Builder withRequestDelay(Map<String, String> jsonTask, String key) {
+      if (jsonTask.containsKey(key)) {
+        requestDelay = Long.parseLong(jsonTask.get(key));
+        requestDelay = Math.max(requestDelay, 0);
+      }
+      return this;
+    }
+
+    public Builder withRequestDelay(long requestDelay) {
+      this.requestDelay = requestDelay;
+      return this;
+    }
+
+    public Builder withHostname(String hostname) {
+      this.hostname = hostname;
+      return this;
+    }
+
+    public TaskContainerDefinition build() throws YarnException {
+      TaskContainerDefinition taskContainerDef =
+          new TaskContainerDefinition();
+      taskContainerDef.duration = validateAndGetDuration(this);
+      taskContainerDef.resource = this.resource;
+      taskContainerDef.type = this.type;
+      taskContainerDef.requestDelay = this.requestDelay;
+      taskContainerDef.priority = this.priority;
+      taskContainerDef.count = this.count;
+      taskContainerDef.allocationId = this.allocationId;
+      taskContainerDef.executionType = this.executionType;
+      taskContainerDef.hostname = this.hostname;
+      return taskContainerDef;
+    }
+
+    private long validateAndGetDuration(Builder builder) throws YarnException {
+      long duration = 0;
+
+      if (builder.duration != -1) {
+        duration = builder.duration;
+      } else if (builder.durationLegacy != -1) {
+        duration = builder.durationLegacy;
+      } else if (builder.taskStart != -1 && builder.taskFinish != -1) {
+        duration = builder.taskFinish - builder.taskStart;
+      }
+
+      if (duration <= 0) {
+        throw new YarnException("Duration of a task shouldn't be less or equal"
+            + " to 0!");
+      }
+      return duration;
+    }
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index 5315eaa1a3118..0a87a6c207058 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -62,6 +62,7 @@
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.AMDefinition;
 import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
@@ -75,6 +76,7 @@
 @Private
 @Unstable
 public abstract class AMSimulator extends TaskRunner.Task {
+  private static final long FINISH_TIME_NOT_INITIALIZED = Long.MIN_VALUE;
   // resource manager
   protected ResourceManager rm;
   // main
@@ -102,7 +104,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
   protected long traceStartTimeMS;
   protected long traceFinishTimeMS;
   protected long simulateStartTimeMS;
-  protected long simulateFinishTimeMS;
+  protected long simulateFinishTimeMS = FINISH_TIME_NOT_INITIALIZED;
   // whether tracked in Metrics
   protected boolean isTracked;
   // progress
@@ -127,27 +129,25 @@ public AMSimulator() {
     this.responseQueue = new LinkedBlockingQueue<>();
   }
 
-  @SuppressWarnings("checkstyle:parameternumber")
-  public void init(int heartbeatInterval,
-      List<ContainerSimulator> containerList, ResourceManager resourceManager,
-      SLSRunner slsRunnner, long startTime, long finishTime, String simUser,
-      String simQueue, boolean tracked, String oldApp, long baseTimeMS,
-      Resource amResource, String nodeLabelExpr, Map<String, String> params,
-      Map<ApplicationId, AMSimulator> appIdAMSim) {
-    super.init(startTime, startTime + 1000000L * heartbeatInterval,
-        heartbeatInterval);
-    this.user = simUser;
-    this.rm = resourceManager;
-    this.se = slsRunnner;
-    this.queue = simQueue;
-    this.oldAppId = oldApp;
+  public void init(AMDefinition amDef, ResourceManager rm, SLSRunner slsRunner,
+      boolean tracked, long baselineTimeMS, long heartbeatInterval,
+      Map<ApplicationId, AMSimulator> appIdToAMSim) {
+    long startTime = amDef.getJobStartTime();
+    long endTime = startTime + 1000000L * heartbeatInterval;
+    super.init(startTime, endTime, heartbeatInterval);
+
+    this.user = amDef.getUser();
+    this.queue = amDef.getQueue();
+    this.oldAppId = amDef.getOldAppId();
+    this.amContainerResource = amDef.getAmResource();
+    this.nodeLabelExpression = amDef.getLabelExpression();
+    this.traceStartTimeMS = amDef.getJobStartTime();
+    this.traceFinishTimeMS = amDef.getJobFinishTime();
+    this.rm = rm;
+    this.se = slsRunner;
     this.isTracked = tracked;
-    this.baselineTimeMS = baseTimeMS;
-    this.traceStartTimeMS = startTime;
-    this.traceFinishTimeMS = finishTime;
-    this.amContainerResource = amResource;
-    this.nodeLabelExpression = nodeLabelExpr;
-    this.appIdToAMSim = appIdAMSim;
+    this.baselineTimeMS = baselineTimeMS;
+    this.appIdToAMSim = appIdToAMSim;
   }
 
   /**
@@ -226,6 +226,16 @@ public void middleStep() throws Exception {
 
   @Override
   public void lastStep() throws Exception {
+    if (simulateFinishTimeMS != FINISH_TIME_NOT_INITIALIZED) {
+      // The finish time is already recorded.
+      // Different value from zero means lastStep was called before.
+      // We want to prevent lastStep to be called more than once.
+      // See YARN-10427 for more details.
+      LOG.warn("Method AMSimulator#lastStep was already called. " +
+          "Skipping execution of method for application: {}", appId);
+      return;
+    }
+
     LOG.info("Application {} is shutting down.", appId);
     // unregister tracking
     if (isTracked) {
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/DAGAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/DAGAMSimulator.java
index 83467e0e5cf46..418408db9fbdc 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/DAGAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/DAGAMSimulator.java
@@ -32,10 +32,10 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.AMDefinition;
 import org.apache.hadoop.yarn.sls.SLSRunner;
 import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
 import org.slf4j.Logger;
@@ -93,19 +93,15 @@ public class DAGAMSimulator extends AMSimulator {
       LoggerFactory.getLogger(DAGAMSimulator.class);
 
   @SuppressWarnings("checkstyle:parameternumber")
-  public void init(int heartbeatInterval,
-      List<ContainerSimulator> containerList, ResourceManager resourceManager,
-      SLSRunner slsRunnner, long startTime, long finishTime, String simUser,
-      String simQueue, boolean tracked, String oldApp, long baseTimeMS,
-      Resource amResource, String nodeLabelExpr, Map<String, String> params,
-      Map<ApplicationId, AMSimulator> appIdAMSim) {
-    super.init(heartbeatInterval, containerList, resourceManager, slsRunnner,
-        startTime, finishTime, simUser, simQueue, tracked, oldApp, baseTimeMS,
-        amResource, nodeLabelExpr, params, appIdAMSim);
+  public void init(AMDefinition amDef, ResourceManager rm, SLSRunner slsRunner,
+      boolean tracked, long baselineTimeMS, long heartbeatInterval,
+      Map<ApplicationId, AMSimulator> appIdToAMSim) {
+    super.init(amDef, rm, slsRunner, tracked, baselineTimeMS, heartbeatInterval,
+        appIdToAMSim);
     super.amtype = "dag";
 
-    allContainers.addAll(containerList);
-    pendingContainers.addAll(containerList);
+    allContainers.addAll(amDef.getTaskContainers());
+    pendingContainers.addAll(amDef.getTaskContainers());
     totalContainers = allContainers.size();
 
     LOG.info("Added new job with {} containers", allContainers.size());
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
index 184fdca2e575c..976c0229b86f3 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
@@ -45,6 +45,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.AMDefinition;
 import org.apache.hadoop.yarn.sls.ReservationClientUtil;
 import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
 import org.apache.hadoop.yarn.sls.SLSRunner;
@@ -123,19 +124,15 @@ scheduled when all maps have finished (not support slow-start currently).
       LoggerFactory.getLogger(MRAMSimulator.class);
 
   @SuppressWarnings("checkstyle:parameternumber")
-  public void init(int heartbeatInterval,
-      List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
-      long traceStartTime, long traceFinishTime, String user, String queue,
-      boolean isTracked, String oldAppId, long baselineStartTimeMS,
-      Resource amContainerResource, String nodeLabelExpr,
-      Map<String, String> params, Map<ApplicationId, AMSimulator> appIdAMSim) {
-    super.init(heartbeatInterval, containerList, rm, se, traceStartTime,
-        traceFinishTime, user, queue, isTracked, oldAppId, baselineStartTimeMS,
-        amContainerResource, nodeLabelExpr, params, appIdAMSim);
+  public void init(AMDefinition amDef, ResourceManager rm, SLSRunner slsRunner,
+      boolean tracked, long baselineTimeMS, long heartbeatInterval,
+      Map<ApplicationId, AMSimulator> appIdToAMSim) {
+    super.init(amDef, rm, slsRunner, tracked, baselineTimeMS,
+        heartbeatInterval, appIdToAMSim);
     amtype = "mapreduce";
 
     // get map/reduce tasks
-    for (ContainerSimulator cs : containerList) {
+    for (ContainerSimulator cs : amDef.getTaskContainers()) {
       if (cs.getType().equals("map")) {
         cs.setPriority(PRIORITY_MAP);
         allMaps.add(cs);
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/StreamAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/StreamAMSimulator.java
index 7e3545191f20e..09297afd4d0e8 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/StreamAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/StreamAMSimulator.java
@@ -30,11 +30,11 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.sls.AMDefinition;
 import org.apache.hadoop.yarn.sls.SLSRunner;
 import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
 import org.slf4j.Logger;
@@ -93,21 +93,14 @@ public class StreamAMSimulator extends AMSimulator {
       LoggerFactory.getLogger(StreamAMSimulator.class);
 
   @SuppressWarnings("checkstyle:parameternumber")
-  public void init(int heartbeatInterval,
-      List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
-      long traceStartTime, long traceFinishTime, String user, String queue,
-      boolean isTracked, String oldAppId, long baselineStartTimeMS,
-      Resource amContainerResource, String nodeLabelExpr,
-      Map<String, String> params, Map<ApplicationId, AMSimulator> appIdAMSim) {
-    super.init(heartbeatInterval, containerList, rm, se, traceStartTime,
-        traceFinishTime, user, queue, isTracked, oldAppId, baselineStartTimeMS,
-        amContainerResource, nodeLabelExpr, params, appIdAMSim);
+  public void init(AMDefinition amDef, ResourceManager rm, SLSRunner slsRunner,
+      boolean tracked, long baselineTimeMS, long heartbeatInterval,
+      Map<ApplicationId, AMSimulator> appIdToAMSim) {
+    super.init(amDef, rm, slsRunner, tracked, baselineTimeMS,
+        heartbeatInterval, appIdToAMSim);
     amtype = "stream";
-
-    allStreams.addAll(containerList);
-
-    duration = traceFinishTime - traceStartTime;
-
+    allStreams.addAll(amDef.getTaskContainers());
+    duration = amDef.getJobFinishTime() - amDef.getJobStartTime();
     LOG.info("Added new job with {} streams, running for {}",
         allStreams.size(), duration);
   }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
index 0d304419351c2..4898ea736f9c7 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
@@ -85,7 +85,7 @@ public void init(String nodeIdStr, Resource nodeResource, int dispatchTime,
     super.init(dispatchTime, dispatchTime + 1000000L * heartBeatInterval,
         heartBeatInterval);
     // create resource
-    String rackHostName[] = SLSUtils.getRackHostName(nodeIdStr);
+    String[] rackHostName = SLSUtils.getRackHostName(nodeIdStr);
     this.node = NodeInfo.newNodeInfo(rackHostName[0], rackHostName[1],
         Resources.clone(nodeResource));
     this.rm = pRm;
@@ -128,7 +128,7 @@ public void firstStep() {
   @Override
   public void middleStep() throws Exception {
     // we check the lifetime for each running containers
-    ContainerSimulator cs = null;
+    ContainerSimulator cs;
     synchronized(completedContainerList) {
       while ((cs = containerQueue.poll()) != null) {
         runningContainers.remove(cs.getId());
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 32567db666ef3..5d1069a5ca298 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -186,7 +186,6 @@ public Set<String> getNodeLabels() {
 
     @Override
     public List<Container> pullNewlyIncreasedContainers() {
-      // TODO Auto-generated method stub
       return null;
     }
 
@@ -259,9 +258,8 @@ public static RMNode newNodeInfo(String rackName, String hostName,
                               final Resource resource, int port) {
     final NodeId nodeId = newNodeID(hostName, port);
     final String nodeAddr = hostName + ":" + port;
-    final String httpAddress = hostName;
-    
-    return new FakeRMNodeImpl(nodeId, nodeAddr, httpAddress,
+
+    return new FakeRMNodeImpl(nodeId, nodeAddr, hostName,
         resource, rackName, "Me good",
         port, hostName, null);
   }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
index d28407669cbe3..aec9f0f105b46 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/resourcemanager/MockAMLauncher.java
@@ -37,22 +37,17 @@
 import org.apache.hadoop.yarn.sls.SLSRunner;
 import org.apache.hadoop.yarn.sls.appmaster.AMSimulator;
 
-import java.util.Map;
 
 public class MockAMLauncher extends ApplicationMasterLauncher
     implements EventHandler<AMLauncherEvent> {
   private static final Logger LOG = LoggerFactory.getLogger(
       MockAMLauncher.class);
 
-  private Map<ApplicationId, AMSimulator> appIdAMSim;
+  private SLSRunner slsRunner;
 
-  SLSRunner se;
-
-  public MockAMLauncher(SLSRunner se, RMContext rmContext,
-      Map<ApplicationId, AMSimulator> appIdAMSim) {
+  public MockAMLauncher(SLSRunner slsRunner, RMContext rmContext) {
     super(rmContext);
-    this.appIdAMSim = appIdAMSim;
-    this.se = se;
+    this.slsRunner = slsRunner;
   }
 
   @Override
@@ -79,12 +74,11 @@ private void setupAMRMToken(RMAppAttempt appAttempt) {
   }
 
   @Override
-  @SuppressWarnings("unchecked")
   public void handle(AMLauncherEvent event) {
     ApplicationId appId =
         event.getAppAttempt().getAppAttemptId().getApplicationId();
     // find AMSimulator
-    AMSimulator ams = appIdAMSim.get(appId);
+    AMSimulator ams = slsRunner.getAMSimulatorByAppId(appId);
     if (ams == null) {
       throw new YarnRuntimeException(
           "Didn't find any AMSimulator for applicationId=" + appId);
@@ -103,7 +97,7 @@ public void handle(AMLauncherEvent event) {
             event.getAppAttempt().getMasterContainer());
         LOG.info("Notify AM launcher launched:" + amContainer.getId());
 
-        se.getNmMap().get(amContainer.getNodeId())
+        slsRunner.getNmMap().get(amContainer.getNodeId())
             .addNewContainer(amContainer, -1, appId);
         ams.getRanNodes().add(amContainer.getNodeId());
         return;
@@ -111,7 +105,7 @@ public void handle(AMLauncherEvent event) {
         throw new YarnRuntimeException(e);
       }
     case CLEANUP:
-      se.getNmMap().get(amContainer.getNodeId())
+      slsRunner.getNmMap().get(amContainer.getNodeId())
           .cleanupContainer(amContainer.getId());
       break;
     default:
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
index e83ee91d8e13e..8f119943570e2 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java
@@ -26,54 +26,41 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.sls.TaskContainerDefinition;
 
 @Private
 @Unstable
 public class ContainerSimulator implements Delayed {
-  // id
   private ContainerId id;
-  // resource allocated
   private Resource resource;
-  // end time
   private long endTime;
   // life time (ms)
   private long lifeTime;
   // time(ms) after which container would be requested by AM
   private long requestDelay;
-  // host name
   private String hostname;
-  // priority
   private int priority;
-  // type 
   private String type;
-  // execution type
   private ExecutionType executionType = ExecutionType.GUARANTEED;
-  // allocation id
   private long allocationId;
 
   /**
-   * invoked when AM schedules containers to allocate.
+   * Invoked when AM schedules containers to allocate.
+   * @param def The task's definition object.
+   * @return ContainerSimulator object
    */
-  public ContainerSimulator(Resource resource, long lifeTime,
-      String hostname, int priority, String type) {
-    this(resource, lifeTime, hostname, priority, type,
-        ExecutionType.GUARANTEED);
+  public static ContainerSimulator createFromTaskContainerDefinition(
+      TaskContainerDefinition def) {
+    return new ContainerSimulator(def.getResource(), def.getDuration(),
+        def.getHostname(), def.getPriority(), def.getType(),
+        def.getExecutionType(), def.getAllocationId(), def.getRequestDelay());
   }
 
   /**
-   * invoked when AM schedules containers to allocate.
-   */
-  public ContainerSimulator(Resource resource, long lifeTime,
-      String hostname, int priority, String type, ExecutionType executionType) {
-    this(resource, lifeTime, hostname, priority, type,
-        executionType, -1, 0);
-  }
-
-  /**
-   * invoked when AM schedules containers to allocate.
+   * Invoked when AM schedules containers to allocate.
    */
   @SuppressWarnings("checkstyle:parameternumber")
-  public ContainerSimulator(Resource resource, long lifeTime,
+  private ContainerSimulator(Resource resource, long lifeTime,
       String hostname, int priority, String type, ExecutionType executionType,
       long allocationId, long requestDelay) {
     this.resource = resource;
@@ -87,7 +74,7 @@ public ContainerSimulator(Resource resource, long lifeTime,
   }
 
   /**
-   * invoke when NM schedules containers to run.
+   * Invoked when NM schedules containers to run.
    */
   public ContainerSimulator(ContainerId id, Resource resource, long endTime,
       long lifeTime, long allocationId) {
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index b5ae4f5b3c0ae..dbbc88fb52d00 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -99,6 +99,11 @@ public Resource getTotalCapability() {
     return node.getTotalCapability();
   }
 
+  @Override
+  public Resource getAllocatedContainerResource() {
+    return node.getAllocatedContainerResource();
+  }
+
   @Override
   public String getRackName() {
     return node.getRackName();
@@ -171,7 +176,6 @@ public Set<String> getNodeLabels() {
 
   @Override
   public List<Container> pullNewlyIncreasedContainers() {
-    // TODO Auto-generated method stub
     return Collections.emptyList();
   }
 
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index e0cb15138f451..6cc4cf7597e32 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -17,12 +17,7 @@
  */
 package org.apache.hadoop.yarn.sls.scheduler;
 
-import java.io.IOException;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -30,119 +25,65 @@
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.sls.SLSRunner;
-import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
-import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.codahale.metrics.Timer;
-
 @Private
 @Unstable
 public class SLSCapacityScheduler extends CapacityScheduler implements
-        SchedulerWrapper,Configurable {
-  private Configuration conf;
- 
-  private Map<ApplicationAttemptId, String> appQueueMap =
-          new ConcurrentHashMap<ApplicationAttemptId, String>();
-
-  private Map<ContainerId, Resource> preemptionContainerMap =
-          new ConcurrentHashMap<ContainerId, Resource>();
+        SchedulerWrapper, Configurable {
 
-  // metrics
-  private SchedulerMetrics schedulerMetrics;
-  private boolean metricsON;
-  private Tracker tracker;
-
-  // logger
+  private final SLSSchedulerCommons schedulerCommons;
+  private Configuration conf;
+  private SLSRunner runner;
   private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
 
-  public Tracker getTracker() {
-    return tracker;
-  }
-
   public SLSCapacityScheduler() {
-    tracker = new Tracker();
+    schedulerCommons = new SLSSchedulerCommons(this);
   }
 
   @Override
   public void setConf(Configuration conf) {
     this.conf = conf;
     super.setConf(conf);
-    metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
-    if (metricsON) {
-      try {
-        schedulerMetrics = SchedulerMetrics.getInstance(conf,
-            CapacityScheduler.class);
-        schedulerMetrics.init(this, conf);
-      } catch (Exception e) {
-        LOG.error("Caught exception while initializing schedulerMetrics", e);
-      }
-    }
+    schedulerCommons.initMetrics(CapacityScheduler.class, conf);
   }
 
   @Override
   public Allocation allocate(ApplicationAttemptId attemptId,
       List<ResourceRequest> resourceRequests,
       List<SchedulingRequest> schedulingRequests, List<ContainerId> containerIds,
-      List<String> strings, List<String> strings2, ContainerUpdates updateRequests) {
-    if (metricsON) {
-      final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer()
-          .time();
-      Allocation allocation = null;
-      try {
-        allocation = super
-            .allocate(attemptId, resourceRequests, schedulingRequests,
-                containerIds, strings,
-                strings2, updateRequests);
-        return allocation;
-      } catch (Exception e) {
-        LOG.error("Caught exception from allocate", e);
-        throw e;
-      } finally {
-        context.stop();
-        schedulerMetrics.increaseSchedulerAllocationCounter();
-        try {
-          updateQueueWithAllocateRequest(allocation, attemptId,
-                  resourceRequests, containerIds);
-        } catch (IOException e) {
-          LOG.error("Caught exception while executing finally block", e);
-        }
-      }
-    } else {
-      return super.allocate(attemptId, resourceRequests, schedulingRequests,
-          containerIds, strings,
-          strings2, updateRequests);
-    }
+      List<String> blacklistAdditions, List<String> blacklistRemovals,
+      ContainerUpdates updateRequests) {
+    return schedulerCommons.allocate(attemptId, resourceRequests, schedulingRequests,
+        containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
   }
 
+  @Override
+  public Allocation allocatePropagated(ApplicationAttemptId attemptId,
+      List<ResourceRequest> resourceRequests,
+      List<SchedulingRequest> schedulingRequests,
+      List<ContainerId> containerIds, List<String> blacklistAdditions,
+      List<String> blacklistRemovals, ContainerUpdates updateRequests) {
+    return super.allocate(attemptId, resourceRequests, schedulingRequests,
+        containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
+  }
 
   @Override
   public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
       boolean updatePending) {
-    if (metricsON) {
+    if (schedulerCommons.isMetricsON()) {
       boolean isSuccess = false;
       long startTimeNs = System.nanoTime();
       try {
@@ -151,13 +92,13 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
       } finally {
         long elapsedNs = System.nanoTime() - startTimeNs;
         if (isSuccess) {
-          schedulerMetrics.getSchedulerCommitSuccessTimer()
+          getSchedulerMetrics().getSchedulerCommitSuccessTimer()
               .update(elapsedNs, TimeUnit.NANOSECONDS);
-          schedulerMetrics.increaseSchedulerCommitSuccessCounter();
+          getSchedulerMetrics().increaseSchedulerCommitSuccessCounter();
         } else {
-          schedulerMetrics.getSchedulerCommitFailureTimer()
+          getSchedulerMetrics().getSchedulerCommitFailureTimer()
               .update(elapsedNs, TimeUnit.NANOSECONDS);
-          schedulerMetrics.increaseSchedulerCommitFailureCounter();
+          getSchedulerMetrics().increaseSchedulerCommitFailureCounter();
         }
       }
     } else {
@@ -167,222 +108,36 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
 
   @Override
   public void handle(SchedulerEvent schedulerEvent) {
-    if (!metricsON) {
-      super.handle(schedulerEvent);
-      return;
-    }
-
-    if (!schedulerMetrics.isRunning()) {
-      schedulerMetrics.setRunning(true);
-    }
-
-    Timer.Context handlerTimer = null;
-    Timer.Context operationTimer = null;
-
-    NodeUpdateSchedulerEventWrapper eventWrapper;
     try {
-      if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
-          && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
-        eventWrapper = new NodeUpdateSchedulerEventWrapper(
-            (NodeUpdateSchedulerEvent)schedulerEvent);
-        schedulerEvent = eventWrapper;
-        updateQueueWithNodeUpdate(eventWrapper);
-      } else if (schedulerEvent.getType() ==
-          SchedulerEventType.APP_ATTEMPT_REMOVED
-          && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
-        // check if having AM Container, update resource usage information
-        AppAttemptRemovedSchedulerEvent appRemoveEvent =
-            (AppAttemptRemovedSchedulerEvent) schedulerEvent;
-        ApplicationAttemptId appAttemptId =
-            appRemoveEvent.getApplicationAttemptID();
-        String queue = appQueueMap.get(appAttemptId);
-        SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
-        if (!app.getLiveContainers().isEmpty()) {  // have 0 or 1
-          // should have one container which is AM container
-          RMContainer rmc = app.getLiveContainers().iterator().next();
-          schedulerMetrics.updateQueueMetricsByRelease(
-              rmc.getContainer().getResource(), queue);
-        }
-      }
-
-      handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time();
-      operationTimer = schedulerMetrics.getSchedulerHandleTimer(
-          schedulerEvent.getType()).time();
-
-      super.handle(schedulerEvent);
-    } finally {
-      if (handlerTimer != null) {
-        handlerTimer.stop();
-      }
-      if (operationTimer != null) {
-        operationTimer.stop();
-      }
-      schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType());
-
-      if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
-          && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
-        SLSRunner.decreaseRemainingApps();
-        AppAttemptRemovedSchedulerEvent appRemoveEvent =
-            (AppAttemptRemovedSchedulerEvent) schedulerEvent;
-        appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
-        if (SLSRunner.getRemainingApps() == 0) {
-          try {
-            getSchedulerMetrics().tearDown();
-            SLSRunner.exitSLSRunner();
-          } catch (Exception e) {
-            LOG.error("Scheduler Metrics failed to tear down.", e);
-          }
-        }
-      } else if (schedulerEvent.getType() ==
-          SchedulerEventType.APP_ATTEMPT_ADDED
-          && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
-        AppAttemptAddedSchedulerEvent appAddEvent =
-            (AppAttemptAddedSchedulerEvent) schedulerEvent;
-        SchedulerApplication app =
-            applications.get(appAddEvent.getApplicationAttemptId()
-                .getApplicationId());
-        appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue()
-            .getQueueName());
-      }
-    }
-  }
-
-  private void updateQueueWithNodeUpdate(
-          NodeUpdateSchedulerEventWrapper eventWrapper) {
-    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
-    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
-    for (UpdatedContainerInfo info : containerList) {
-      for (ContainerStatus status : info.getCompletedContainers()) {
-        ContainerId containerId = status.getContainerId();
-        SchedulerAppReport app = super.getSchedulerAppInfo(
-                containerId.getApplicationAttemptId());
-
-        if (app == null) {
-          // this happens for the AM container
-          // The app have already removed when the NM sends the release
-          // information.
-          continue;
-        }
-
-        String queue = appQueueMap.get(containerId.getApplicationAttemptId());
-        int releasedMemory = 0, releasedVCores = 0;
-        if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
-          for (RMContainer rmc : app.getLiveContainers()) {
-            if (rmc.getContainerId() == containerId) {
-              releasedMemory += rmc.getContainer().getResource().getMemorySize();
-              releasedVCores += rmc.getContainer()
-                      .getResource().getVirtualCores();
-              break;
-            }
-          }
-        } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
-          if (preemptionContainerMap.containsKey(containerId)) {
-            Resource preResource = preemptionContainerMap.get(containerId);
-            releasedMemory += preResource.getMemorySize();
-            releasedVCores += preResource.getVirtualCores();
-            preemptionContainerMap.remove(containerId);
-          }
-        }
-        // update queue counters
-        schedulerMetrics.updateQueueMetricsByRelease(
-            Resource.newInstance(releasedMemory, releasedVCores), queue);
-      }
+      schedulerCommons.handle(schedulerEvent);
+    } catch(Exception e) {
+      LOG.error("Caught exception while handling scheduler event", e);
+      throw e;
     }
   }
 
-  private void updateQueueWithAllocateRequest(Allocation allocation,
-                        ApplicationAttemptId attemptId,
-                        List<ResourceRequest> resourceRequests,
-                        List<ContainerId> containerIds) throws IOException {
-    // update queue information
-    Resource pendingResource = Resources.createResource(0, 0);
-    Resource allocatedResource = Resources.createResource(0, 0);
-    String queueName = appQueueMap.get(attemptId);
-    // container requested
-    for (ResourceRequest request : resourceRequests) {
-      if (request.getResourceName().equals(ResourceRequest.ANY)) {
-        Resources.addTo(pendingResource,
-                Resources.multiply(request.getCapability(),
-                        request.getNumContainers()));
-      }
-    }
-    // container allocated
-    for (Container container : allocation.getContainers()) {
-      Resources.addTo(allocatedResource, container.getResource());
-      Resources.subtractFrom(pendingResource, container.getResource());
-    }
-    // container released from AM
-    SchedulerAppReport report = super.getSchedulerAppInfo(attemptId);
-    for (ContainerId containerId : containerIds) {
-      Container container = null;
-      for (RMContainer c : report.getLiveContainers()) {
-        if (c.getContainerId().equals(containerId)) {
-          container = c.getContainer();
-          break;
-        }
-      }
-      if (container != null) {
-        // released allocated containers
-        Resources.subtractFrom(allocatedResource, container.getResource());
-      } else {
-        for (RMContainer c : report.getReservedContainers()) {
-          if (c.getContainerId().equals(containerId)) {
-            container = c.getContainer();
-            break;
-          }
-        }
-        if (container != null) {
-          // released reserved containers
-          Resources.subtractFrom(pendingResource, container.getResource());
-        }
-      }
-    }
-    // containers released/preemption from scheduler
-    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
-    if (allocation.getContainerPreemptions() != null) {
-      preemptionContainers.addAll(allocation.getContainerPreemptions());
-    }
-    if (allocation.getStrictContainerPreemptions() != null) {
-      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
-    }
-    if (! preemptionContainers.isEmpty()) {
-      for (ContainerId containerId : preemptionContainers) {
-        if (! preemptionContainerMap.containsKey(containerId)) {
-          Container container = null;
-          for (RMContainer c : report.getLiveContainers()) {
-            if (c.getContainerId().equals(containerId)) {
-              container = c.getContainer();
-              break;
-            }
-          }
-          if (container != null) {
-            preemptionContainerMap.put(containerId, container.getResource());
-          }
-        }
-
-      }
-    }
-
-    // update metrics
-    schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource,
-        queueName);
+  @Override
+  public void propagatedHandle(SchedulerEvent schedulerEvent) {
+    super.handle(schedulerEvent);
   }
 
   @Override
   public void serviceStop() throws Exception {
-    try {
-      if (metricsON) {
-        schedulerMetrics.tearDown();
-      }
-    } catch (Exception e) {
-      LOG.error("Caught exception while stopping service", e);
-    }
+    schedulerCommons.stopMetrics();
     super.serviceStop();
   }
 
 
+  public String getRealQueueName(String queue) throws YarnException {
+    if (getQueue(queue) == null) {
+      throw new YarnException("Can't find the queue by the given name: " + queue
+          + "! Please check if queue " + queue + " is in the allocation file.");
+    }
+    return getQueue(queue).getQueuePath();
+  }
+
   public SchedulerMetrics getSchedulerMetrics() {
-    return schedulerMetrics;
+    return schedulerCommons.getSchedulerMetrics();
   }
 
   @Override
@@ -390,11 +145,18 @@ public Configuration getConf() {
     return conf;
   }
 
-  public String getRealQueueName(String queue) throws YarnException {
-    if (getQueue(queue) == null) {
-      throw new YarnException("Can't find the queue by the given name: " + queue
-          + "! Please check if queue " + queue + " is in the allocation file.");
-    }
-    return getQueue(queue).getQueuePath();
+  public Tracker getTracker() {
+    return schedulerCommons.getTracker();
+  }
+
+  @Override
+  public void setSLSRunner(SLSRunner runner) {
+    this.runner = runner;
   }
-}
\ No newline at end of file
+
+  @Override
+  public SLSRunner getSLSRunner() {
+    return this.runner;
+  }
+}
+
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 84549bc492205..beb411025f853 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -17,84 +17,41 @@
  */
 package org.apache.hadoop.yarn.sls.scheduler;
 
-import com.codahale.metrics.Timer;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.sls.SLSRunner;
-import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
-import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
 @Private
 @Unstable
 public class SLSFairScheduler extends FairScheduler
     implements SchedulerWrapper, Configurable {
-  private SchedulerMetrics schedulerMetrics;
-  private boolean metricsON;
-  private Tracker tracker;
-
-  private Map<ContainerId, Resource> preemptionContainerMap =
-      new ConcurrentHashMap<>();
-
-  // logger
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SLSFairScheduler.class);
-
-  public SchedulerMetrics getSchedulerMetrics() {
-    return schedulerMetrics;
-  }
-
-  public Tracker getTracker() {
-    return tracker;
-  }
+  private final SLSSchedulerCommons schedulerCommons;
+  private SLSRunner runner;
+  private static final Logger LOG = LoggerFactory.getLogger(SLSFairScheduler.class);
 
   public SLSFairScheduler() {
-    tracker = new Tracker();
+    schedulerCommons = new SLSSchedulerCommons(this);
   }
 
   @Override
   public void setConf(Configuration conf) {
     super.setConfig(conf);
-
-    metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
-    if (metricsON) {
-      try {
-        schedulerMetrics = SchedulerMetrics.getInstance(conf,
-            FairScheduler.class);
-        schedulerMetrics.init(this, conf);
-      } catch (Exception e) {
-        LOG.error("Caught exception while initializing schedulerMetrics", e);
-      }
-    }
+    schedulerCommons.initMetrics(FairScheduler.class, conf);
   }
 
   @Override
@@ -103,237 +60,38 @@ public Allocation allocate(ApplicationAttemptId attemptId,
       List<SchedulingRequest> schedulingRequests, List<ContainerId> containerIds,
       List<String> blacklistAdditions, List<String> blacklistRemovals,
       ContainerUpdates updateRequests) {
-    if (metricsON) {
-      final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer()
-          .time();
-      Allocation allocation = null;
-      try {
-        allocation = super.allocate(attemptId, resourceRequests,
-            schedulingRequests, containerIds,
-            blacklistAdditions, blacklistRemovals, updateRequests);
-        return allocation;
-      } catch (Exception e) {
-        LOG.error("Caught exception from allocate", e);
-        throw e;
-      } finally {
-        context.stop();
-        schedulerMetrics.increaseSchedulerAllocationCounter();
-        try {
-          updateQueueWithAllocateRequest(allocation, attemptId,
-              resourceRequests, containerIds);
-        } catch (IOException e) {
-          LOG.error("Caught exception while executing finally block", e);
-        }
-      }
-    } else {
-      return super.allocate(attemptId, resourceRequests, schedulingRequests,
-          containerIds,
-          blacklistAdditions, blacklistRemovals, updateRequests);
-    }
+    return schedulerCommons.allocate(attemptId, resourceRequests, schedulingRequests,
+        containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
   }
 
   @Override
   public void handle(SchedulerEvent schedulerEvent) {
-    // metrics off
-    if (!metricsON) {
-      super.handle(schedulerEvent);
-      return;
-    }
-
-    // metrics on
-    if(!schedulerMetrics.isRunning()) {
-      schedulerMetrics.setRunning(true);
-    }
-
-    Timer.Context handlerTimer = null;
-    Timer.Context operationTimer = null;
-
-    NodeUpdateSchedulerEventWrapper eventWrapper;
     try {
-      if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
-          && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
-        eventWrapper = new NodeUpdateSchedulerEventWrapper(
-            (NodeUpdateSchedulerEvent)schedulerEvent);
-        schedulerEvent = eventWrapper;
-        updateQueueWithNodeUpdate(eventWrapper);
-      } else if (
-          schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
-          && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
-        // check if having AM Container, update resource usage information
-        AppAttemptRemovedSchedulerEvent appRemoveEvent =
-            (AppAttemptRemovedSchedulerEvent) schedulerEvent;
-        ApplicationAttemptId appAttemptId =
-            appRemoveEvent.getApplicationAttemptID();
-        String queueName = getSchedulerApp(appAttemptId).getQueue().getName();
-        SchedulerAppReport app = getSchedulerAppInfo(appAttemptId);
-        if (!app.getLiveContainers().isEmpty()) {  // have 0 or 1
-          // should have one container which is AM container
-          RMContainer rmc = app.getLiveContainers().iterator().next();
-          schedulerMetrics.updateQueueMetricsByRelease(
-              rmc.getContainer().getResource(), queueName);
-        }
-      }
-
-      handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time();
-      operationTimer = schedulerMetrics.getSchedulerHandleTimer(
-          schedulerEvent.getType()).time();
-
-      super.handle(schedulerEvent);
-    } finally {
-      if (handlerTimer != null) {
-        handlerTimer.stop();
-      }
-      if (operationTimer != null) {
-        operationTimer.stop();
-      }
-      schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType());
-
-      if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
-          && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
-        SLSRunner.decreaseRemainingApps();
-        if (SLSRunner.getRemainingApps() == 0) {
-          try {
-            getSchedulerMetrics().tearDown();
-            SLSRunner.exitSLSRunner();
-          } catch (Exception e) {
-            LOG.error("Scheduler Metrics failed to tear down.", e);
-          }
-        }
-      }
+      schedulerCommons.handle(schedulerEvent);
+    } catch (Exception e){
+      LOG.error("Caught exception while handling scheduler event", e);
+      throw e;
     }
   }
 
-  private void updateQueueWithNodeUpdate(
-      NodeUpdateSchedulerEventWrapper eventWrapper) {
-    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
-    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
-    for (UpdatedContainerInfo info : containerList) {
-      for (ContainerStatus status : info.getCompletedContainers()) {
-        ContainerId containerId = status.getContainerId();
-        SchedulerAppReport app = super.getSchedulerAppInfo(
-            containerId.getApplicationAttemptId());
-
-        if (app == null) {
-          // this happens for the AM container
-          // The app have already removed when the NM sends the release
-          // information.
-          continue;
-        }
-
-        int releasedMemory = 0, releasedVCores = 0;
-        if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
-          for (RMContainer rmc : app.getLiveContainers()) {
-            if (rmc.getContainerId() == containerId) {
-              Resource resource = rmc.getContainer().getResource();
-              releasedMemory += resource.getMemorySize();
-              releasedVCores += resource.getVirtualCores();
-              break;
-            }
-          }
-        } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
-          if (preemptionContainerMap.containsKey(containerId)) {
-            Resource preResource = preemptionContainerMap.get(containerId);
-            releasedMemory += preResource.getMemorySize();
-            releasedVCores += preResource.getVirtualCores();
-            preemptionContainerMap.remove(containerId);
-          }
-        }
-        // update queue counters
-        String queue = getSchedulerApp(containerId.getApplicationAttemptId()).
-            getQueueName();
-        schedulerMetrics.updateQueueMetricsByRelease(
-            Resource.newInstance(releasedMemory, releasedVCores), queue);
-      }
-    }
+  @Override
+  public void propagatedHandle(SchedulerEvent schedulerEvent) {
+    super.handle(schedulerEvent);
   }
 
-  private void updateQueueWithAllocateRequest(Allocation allocation,
-      ApplicationAttemptId attemptId,
+  @Override
+  public Allocation allocatePropagated(ApplicationAttemptId attemptId,
       List<ResourceRequest> resourceRequests,
-      List<ContainerId> containerIds) throws IOException {
-    // update queue information
-    Resource pendingResource = Resources.createResource(0, 0);
-    Resource allocatedResource = Resources.createResource(0, 0);
-    // container requested
-    for (ResourceRequest request : resourceRequests) {
-      if (request.getResourceName().equals(ResourceRequest.ANY)) {
-        Resources.addTo(pendingResource,
-            Resources.multiply(request.getCapability(),
-                request.getNumContainers()));
-      }
-    }
-    // container allocated
-    for (Container container : allocation.getContainers()) {
-      Resources.addTo(allocatedResource, container.getResource());
-      Resources.subtractFrom(pendingResource, container.getResource());
-    }
-    // container released from AM
-    SchedulerAppReport report = super.getSchedulerAppInfo(attemptId);
-    for (ContainerId containerId : containerIds) {
-      Container container = null;
-      for (RMContainer c : report.getLiveContainers()) {
-        if (c.getContainerId().equals(containerId)) {
-          container = c.getContainer();
-          break;
-        }
-      }
-      if (container != null) {
-        // released allocated containers
-        Resources.subtractFrom(allocatedResource, container.getResource());
-      } else {
-        for (RMContainer c : report.getReservedContainers()) {
-          if (c.getContainerId().equals(containerId)) {
-            container = c.getContainer();
-            break;
-          }
-        }
-        if (container != null) {
-          // released reserved containers
-          Resources.subtractFrom(pendingResource, container.getResource());
-        }
-      }
-    }
-    // containers released/preemption from scheduler
-    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
-    if (allocation.getContainerPreemptions() != null) {
-      preemptionContainers.addAll(allocation.getContainerPreemptions());
-    }
-    if (allocation.getStrictContainerPreemptions() != null) {
-      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
-    }
-    if (!preemptionContainers.isEmpty()) {
-      for (ContainerId containerId : preemptionContainers) {
-        if (!preemptionContainerMap.containsKey(containerId)) {
-          Container container = null;
-          for (RMContainer c : report.getLiveContainers()) {
-            if (c.getContainerId().equals(containerId)) {
-              container = c.getContainer();
-              break;
-            }
-          }
-          if (container != null) {
-            preemptionContainerMap.put(containerId, container.getResource());
-          }
-        }
-
-      }
-    }
-
-    // update metrics
-    String queueName = getSchedulerApp(attemptId).getQueueName();
-    schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource,
-        queueName);
+      List<SchedulingRequest> schedulingRequests,
+      List<ContainerId> containerIds, List<String> blacklistAdditions,
+      List<String> blacklistRemovals, ContainerUpdates updateRequests) {
+    return super.allocate(attemptId, resourceRequests, schedulingRequests,
+        containerIds, blacklistAdditions, blacklistRemovals, updateRequests);
   }
 
   @Override
   public void serviceStop() throws Exception {
-    try {
-      if (metricsON) {
-        schedulerMetrics.tearDown();
-      }
-    } catch (Exception e) {
-      LOG.error("Caught exception while stopping service", e);
-    }
+    schedulerCommons.stopMetrics();
     super.serviceStop();
   }
 
@@ -344,5 +102,22 @@ public String getRealQueueName(String queue) throws YarnException {
     }
     return getQueueManager().getQueue(queue).getQueueName();
   }
-}
 
+  public SchedulerMetrics getSchedulerMetrics() {
+    return schedulerCommons.getSchedulerMetrics();
+  }
+
+  public Tracker getTracker() {
+    return schedulerCommons.getTracker();
+  }
+
+  @Override
+  public void setSLSRunner(SLSRunner runner) {
+    this.runner = runner;
+  }
+
+  @Override
+  public SLSRunner getSLSRunner() {
+    return this.runner;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java
new file mode 100644
index 0000000000000..a99fe53eff91a
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSSchedulerCommons.java
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sls.scheduler;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.sls.SLSRunner;
+import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class SLSSchedulerCommons {
+  private static final Logger LOG = LoggerFactory.getLogger(SLSSchedulerCommons.class);
+
+  private final AbstractYarnScheduler<?, ?> scheduler;
+  private boolean metricsON;
+  private SchedulerMetrics schedulerMetrics;
+  private final Map<ContainerId, Resource> preemptionContainerMap = new ConcurrentHashMap<>();
+  private final Map<ApplicationAttemptId, String> appQueueMap = new ConcurrentHashMap<>();
+  private final Tracker tracker;
+  
+  public SLSSchedulerCommons(AbstractYarnScheduler<?, ?> scheduler) {
+    this.scheduler = scheduler;
+    this.tracker = new Tracker();
+  }
+
+  public void initMetrics(Class<? extends AbstractYarnScheduler<?, ?>> schedulerClass, Configuration conf) {
+    metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true);
+    if (metricsON) {
+      try {
+        schedulerMetrics = SchedulerMetrics.getInstance(conf, schedulerClass);
+        schedulerMetrics.init(scheduler, conf);
+      } catch (Exception e) {
+        LOG.error("Caught exception while initializing schedulerMetrics", e);
+      }
+    }
+  }
+
+  void stopMetrics() {
+    try {
+      if (metricsON) {
+        schedulerMetrics.tearDown();
+      }
+    } catch (Exception e) {
+      LOG.error("Caught exception while stopping service", e);
+    }
+  }
+  
+  public Allocation allocate(ApplicationAttemptId attemptId,
+      List<ResourceRequest> resourceRequests,
+      List<SchedulingRequest> schedulingRequests,
+      List<ContainerId> containerIds,
+      List<String> blacklistAdditions,
+      List<String> blacklistRemovals,
+      ContainerUpdates updateRequests) {
+    if (metricsON) {
+      final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer()
+          .time();
+      Allocation allocation = null;
+      try {
+        allocation = ((SchedulerWrapper)scheduler).allocatePropagated(
+            attemptId, resourceRequests,
+            schedulingRequests, containerIds,
+            blacklistAdditions, blacklistRemovals, updateRequests);
+        return allocation;
+      } catch (Exception e) {
+        LOG.error("Caught exception from allocate", e);
+        throw e;
+      } finally {
+        context.stop();
+        schedulerMetrics.increaseSchedulerAllocationCounter();
+        try {
+          updateQueueWithAllocateRequest(allocation, attemptId,
+              resourceRequests, containerIds);
+        } catch (IOException e) {
+          LOG.error("Caught exception while executing finally block", e);
+        }
+      }
+    } else {
+      return ((SchedulerWrapper)scheduler).allocatePropagated(
+          attemptId, resourceRequests, schedulingRequests,
+          containerIds,
+          blacklistAdditions, blacklistRemovals, updateRequests);
+    }
+  }
+
+  private void updateQueueWithAllocateRequest(Allocation allocation,
+      ApplicationAttemptId attemptId,
+      List<ResourceRequest> resourceRequests,
+      List<ContainerId> containerIds) throws IOException {
+    // update queue information
+    Resource pendingResource = Resources.createResource(0, 0);
+    Resource allocatedResource = Resources.createResource(0, 0);
+    String queueName = appQueueMap.get(attemptId);
+    // container requested
+    for (ResourceRequest request : resourceRequests) {
+      if (request.getResourceName().equals(ResourceRequest.ANY)) {
+        Resources.addTo(pendingResource,
+            Resources.multiply(request.getCapability(),
+                request.getNumContainers()));
+      }
+    }
+    // container allocated
+    for (Container container : allocation.getContainers()) {
+      Resources.addTo(allocatedResource, container.getResource());
+      Resources.subtractFrom(pendingResource, container.getResource());
+    }
+    // container released from AM
+    SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
+    for (ContainerId containerId : containerIds) {
+      Container container = null;
+      for (RMContainer c : report.getLiveContainers()) {
+        if (c.getContainerId().equals(containerId)) {
+          container = c.getContainer();
+          break;
+        }
+      }
+      if (container != null) {
+        // released allocated containers
+        Resources.subtractFrom(allocatedResource, container.getResource());
+      } else {
+        for (RMContainer c : report.getReservedContainers()) {
+          if (c.getContainerId().equals(containerId)) {
+            container = c.getContainer();
+            break;
+          }
+        }
+        if (container != null) {
+          // released reserved containers
+          Resources.subtractFrom(pendingResource, container.getResource());
+        }
+      }
+    }
+    // containers released/preemption from scheduler
+    Set<ContainerId> preemptionContainers = new HashSet<>();
+    if (allocation.getContainerPreemptions() != null) {
+      preemptionContainers.addAll(allocation.getContainerPreemptions());
+    }
+    if (allocation.getStrictContainerPreemptions() != null) {
+      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
+    }
+    if (!preemptionContainers.isEmpty()) {
+      for (ContainerId containerId : preemptionContainers) {
+        if (!preemptionContainerMap.containsKey(containerId)) {
+          Container container = null;
+          for (RMContainer c : report.getLiveContainers()) {
+            if (c.getContainerId().equals(containerId)) {
+              container = c.getContainer();
+              break;
+            }
+          }
+          if (container != null) {
+            preemptionContainerMap.put(containerId, container.getResource());
+          }
+        }
+
+      }
+    }
+
+    // update metrics
+    schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource,
+        queueName);
+  }
+
+  public void handle(SchedulerEvent schedulerEvent) {
+    SchedulerWrapper wrapper = (SchedulerWrapper) scheduler;
+    if (!metricsON) {
+      ((SchedulerWrapper)scheduler).propagatedHandle(schedulerEvent);
+      return;
+    }
+
+    if (!schedulerMetrics.isRunning()) {
+      schedulerMetrics.setRunning(true);
+    }
+
+    Timer.Context handlerTimer = null;
+    Timer.Context operationTimer = null;
+
+    NodeUpdateSchedulerEventWrapper eventWrapper;
+    try {
+      if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE
+          && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
+        eventWrapper = new NodeUpdateSchedulerEventWrapper(
+            (NodeUpdateSchedulerEvent) schedulerEvent);
+        schedulerEvent = eventWrapper;
+        updateQueueWithNodeUpdate(eventWrapper);
+      } else if (
+          schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
+              && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+        // check if having AM Container, update resource usage information
+        AppAttemptRemovedSchedulerEvent appRemoveEvent =
+            (AppAttemptRemovedSchedulerEvent) schedulerEvent;
+        ApplicationAttemptId appAttemptId =
+            appRemoveEvent.getApplicationAttemptID();
+        String queue = appQueueMap.get(appAttemptId);
+        SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId);
+        if (!app.getLiveContainers().isEmpty()) {  // have 0 or 1
+          // should have one container which is AM container
+          RMContainer rmc = app.getLiveContainers().iterator().next();
+          schedulerMetrics.updateQueueMetricsByRelease(
+              rmc.getContainer().getResource(), queue);
+        }
+      }
+
+      handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time();
+      operationTimer = schedulerMetrics.getSchedulerHandleTimer(
+          schedulerEvent.getType()).time();
+
+      ((SchedulerWrapper)scheduler).propagatedHandle(schedulerEvent);
+    } finally {
+      if (handlerTimer != null) {
+        handlerTimer.stop();
+      }
+      if (operationTimer != null) {
+        operationTimer.stop();
+      }
+      schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType());
+
+      if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
+          && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
+        wrapper.getSLSRunner().decreaseRemainingApps();
+        AppAttemptRemovedSchedulerEvent appRemoveEvent =
+            (AppAttemptRemovedSchedulerEvent) schedulerEvent;
+        appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
+        if (wrapper.getSLSRunner().getRemainingApps() == 0) {
+          try {
+            schedulerMetrics.tearDown();
+            SLSRunner.exitSLSRunner();
+          } catch (Exception e) {
+            LOG.error("Scheduler Metrics failed to tear down.", e);
+          }
+        }
+      } else if (schedulerEvent.getType() ==
+          SchedulerEventType.APP_ATTEMPT_ADDED
+          && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
+        AppAttemptAddedSchedulerEvent appAddEvent =
+            (AppAttemptAddedSchedulerEvent) schedulerEvent;
+        SchedulerApplication app =
+            scheduler.getSchedulerApplications()
+                .get(appAddEvent.getApplicationAttemptId().getApplicationId());
+        appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue()
+            .getQueueName());
+      }
+    }
+  }
+
+  private void updateQueueWithNodeUpdate(
+      NodeUpdateSchedulerEventWrapper eventWrapper) {
+    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
+    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
+    for (UpdatedContainerInfo info : containerList) {
+      for (ContainerStatus status : info.getCompletedContainers()) {
+        ContainerId containerId = status.getContainerId();
+        SchedulerAppReport app = scheduler.getSchedulerAppInfo(
+            containerId.getApplicationAttemptId());
+
+        if (app == null) {
+          // this happens for the AM container
+          // The app have already removed when the NM sends the release
+          // information.
+          continue;
+        }
+
+        int releasedMemory = 0, releasedVCores = 0;
+        if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
+          for (RMContainer rmc : app.getLiveContainers()) {
+            if (rmc.getContainerId() == containerId) {
+              Resource resource = rmc.getContainer().getResource();
+              releasedMemory += resource.getMemorySize();
+              releasedVCores += resource.getVirtualCores();
+              break;
+            }
+          }
+        } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
+          if (preemptionContainerMap.containsKey(containerId)) {
+            Resource preResource = preemptionContainerMap.get(containerId);
+            releasedMemory += preResource.getMemorySize();
+            releasedVCores += preResource.getVirtualCores();
+            preemptionContainerMap.remove(containerId);
+          }
+        }
+        // update queue counters
+        String queue = appQueueMap.get(containerId.getApplicationAttemptId());
+        schedulerMetrics.updateQueueMetricsByRelease(
+            Resource.newInstance(releasedMemory, releasedVCores), queue);
+      }
+    }
+  }
+  
+  public SchedulerMetrics getSchedulerMetrics() {
+    return schedulerMetrics;
+  }
+
+  public boolean isMetricsON() {
+    return metricsON;
+  }
+
+  public Tracker getTracker() {
+    return tracker;
+  }
+}
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
index 26a9da4cd8bc8..f66cf4384d974 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
@@ -20,6 +20,7 @@
 
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -177,14 +178,14 @@ void init(ResourceScheduler resourceScheduler, Configuration config)
     pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
         TimeUnit.MILLISECONDS);
 
-    // a thread to output metrics for real-tiem tracking
+    // a thread to output metrics for real-time tracking
     pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
         TimeUnit.MILLISECONDS);
 
     // application running information
     jobRuntimeLogBW =
         new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
-            metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
+            metricsOutputDir + "/jobruntime.csv"), StandardCharsets.UTF_8));
     jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
         "simulate_start_time,simulate_end_time" + EOL);
     jobRuntimeLogBW.flush();
@@ -316,7 +317,7 @@ private void registerClusterResourceMetrics() {
         new Gauge<Long>() {
           @Override
           public Long getValue() {
-            if (scheduler.getRootQueueMetrics() == null) {
+            if (isMetricsAvailable()) {
               return 0L;
             } else {
               return scheduler.getRootQueueMetrics().getAllocatedMB();
@@ -328,7 +329,7 @@ public Long getValue() {
         new Gauge<Integer>() {
           @Override
           public Integer getValue() {
-            if (scheduler.getRootQueueMetrics() == null) {
+            if (isMetricsAvailable()) {
               return 0;
             } else {
               return scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
@@ -340,7 +341,7 @@ public Integer getValue() {
         new Gauge<Long>() {
           @Override
           public Long getValue() {
-            if (scheduler.getRootQueueMetrics() == null) {
+            if (isMetricsAvailable()) {
               return 0L;
             } else {
               return scheduler.getRootQueueMetrics().getAvailableMB();
@@ -352,7 +353,7 @@ public Long getValue() {
         new Gauge<Integer>() {
           @Override
           public Integer getValue() {
-            if (scheduler.getRootQueueMetrics() == null) {
+            if (isMetricsAvailable()) {
               return 0;
             } else {
               return scheduler.getRootQueueMetrics().getAvailableVirtualCores();
@@ -362,6 +363,10 @@ public Integer getValue() {
     );
   }
 
+  private boolean isMetricsAvailable() {
+    return scheduler.getRootQueueMetrics() == null;
+  }
+
   private void registerContainerAppNumMetrics() {
     metrics.register("variable.running.application",
         new Gauge<Integer>() {
@@ -462,6 +467,9 @@ private void registerSchedulerMetrics() {
         schedulerHistogramList.add(histogram);
         histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e));
       }
+    } catch (Exception e) {
+      LOG.error("Caught exception while registering scheduler metrics", e);
+      throw e;
     } finally {
       samplerLock.unlock();
     }
@@ -505,6 +513,9 @@ public Integer getValue() {
             }
         );
       }
+    } catch (Exception e) {
+      LOG.error("Caught exception while registering nodes usage metrics", e);
+      throw e;
     } finally {
       samplerLock.unlock();
     }
@@ -556,7 +567,7 @@ class MetricsLogRunnable implements Runnable {
       try {
         metricsLogBW =
             new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
-                metricsOutputDir + "/realtimetrack.json"), "UTF-8"));
+                metricsOutputDir + "/realtimetrack.json"), StandardCharsets.UTF_8));
         metricsLogBW.write("[");
       } catch (IOException e) {
         LOG.info(e.getMessage());
@@ -713,11 +724,10 @@ public void addAMRuntime(ApplicationId appId, long traceStartTimeMS,
       long traceEndTimeMS, long simulateStartTimeMS, long simulateEndTimeMS) {
     try {
       // write job runtime information
-      StringBuilder sb = new StringBuilder();
-      sb.append(appId).append(",").append(traceStartTimeMS).append(",")
-          .append(traceEndTimeMS).append(",").append(simulateStartTimeMS)
-          .append(",").append(simulateEndTimeMS);
-      jobRuntimeLogBW.write(sb.toString() + EOL);
+      String runtimeInfo = appId + "," + traceStartTimeMS + "," +
+          traceEndTimeMS + "," + simulateStartTimeMS +
+          "," + simulateEndTimeMS;
+      jobRuntimeLogBW.write(runtimeInfo + EOL);
       jobRuntimeLogBW.flush();
     } catch (IOException e) {
       LOG.info(e.getMessage());
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
index 7112b1a6fd57b..cc9844a8f2af9 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java
@@ -19,7 +19,17 @@
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.sls.SLSRunner;
+
+import java.util.List;
 
 @Private
 @Unstable
@@ -29,4 +39,18 @@ public interface SchedulerWrapper {
   Tracker getTracker();
 
   String getRealQueueName(String queue) throws YarnException;
+
+  void propagatedHandle(SchedulerEvent schedulerEvent);
+
+  Allocation allocatePropagated(ApplicationAttemptId attemptId,
+      List<ResourceRequest> resourceRequests,
+      List<SchedulingRequest> schedulingRequests,
+      List<ContainerId> containerIds,
+      List<String> blacklistAdditions,
+      List<String> blacklistRemovals,
+      ContainerUpdates updateRequests);
+
+  void setSLSRunner(SLSRunner runner);
+
+  SLSRunner getSLSRunner();
 }
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthJob.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthJob.java
index 86a1dcf8b5351..eea0a8b64266f 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthJob.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthJob.java
@@ -184,9 +184,9 @@ protected SynthJob(JDKRandomGenerator rand, Configuration conf,
       int num = task.count.getInt();
       String taskType = task.type;
       long memory = task.max_memory.getLong();
-      memory = memory < MIN_MEMORY ? MIN_MEMORY: memory;
+      memory = Math.max(memory, MIN_MEMORY);
       long vcores = task.max_vcores.getLong();
-      vcores = vcores < MIN_VCORES ? MIN_VCORES  : vcores;
+      vcores = Math.max(vcores, MIN_VCORES);
       int priority = task.priority;
       ExecutionType executionType = task.executionType == null
           ? ExecutionType.GUARANTEED
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
index 7a3e22bd4cb5b..18b1c034bdf3a 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
@@ -17,6 +17,13 @@
  */
 package org.apache.hadoop.yarn.sls.synthetic;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.commons.math3.distribution.AbstractRealDistribution;
@@ -30,18 +37,13 @@
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.sls.appmaster.MRAMSimulator;
-import org.codehaus.jackson.annotate.JsonCreator;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import javax.xml.bind.annotation.XmlRootElement;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES;
-import static org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES;
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
 
 /**
  * This is a JobStoryProducer that operates from distribution of different
@@ -84,15 +86,16 @@ public SynthTraceJobProducer(Configuration conf, Path path)
     this.conf = conf;
     this.rand = new JDKRandomGenerator();
 
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.configure(INTERN_FIELD_NAMES, true);
+    JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder();
+    jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true);
+    ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build());
     mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     FileSystem ifs = path.getFileSystem(conf);
     FSDataInputStream fileIn = ifs.open(path);
 
     // Initialize the random generator and the seed
-    this.trace = mapper.readValue(fileIn, Trace.class);
+    this.trace = mapper.readValue(fileIn.getWrappedStream(), Trace.class);
     this.seed = trace.rand_seed;
     this.rand.setSeed(seed);
     // Initialize the trace
@@ -307,7 +310,6 @@ public void init(JDKRandomGenerator random){
 
       // Initialize job weights
       job_weights = new ArrayList<>();
-      job_weights = new ArrayList<>();
       for(JobDefinition j : job_classes){
         job_weights.add(j.class_weight);
       }
@@ -538,9 +540,9 @@ public Sample(@JsonProperty("val") Double val,
       if(val!=null){
         if(std==null){
           // Constant
-          if(dist!=null || discrete!=null || weights!=null){
-            throw new JsonMappingException("Instantiation of " + Sample.class
-                + " failed");
+          if (dist != null || discrete != null || weights != null) {
+            throw JsonMappingException
+                .from((JsonParser) null, "Instantiation of " + Sample.class + " failed");
           }
           mode = Mode.CONST;
           this.val = val;
@@ -550,9 +552,9 @@ public Sample(@JsonProperty("val") Double val,
           this.weights = null;
         } else {
           // Distribution
-          if(discrete!=null || weights != null){
-            throw new JsonMappingException("Instantiation of " + Sample.class
-                + " failed");
+          if (discrete != null || weights != null) {
+            throw JsonMappingException
+                .from((JsonParser) null, "Instantiation of " + Sample.class + " failed");
           }
           mode = Mode.DIST;
           this.val = val;
@@ -563,9 +565,9 @@ public Sample(@JsonProperty("val") Double val,
         }
       } else {
         // Discrete
-        if(discrete==null){
-          throw new JsonMappingException("Instantiation of " + Sample.class
-              + " failed");
+        if (discrete == null) {
+          throw JsonMappingException
+              .from((JsonParser) null, "Instantiation of " + Sample.class + " failed");
         }
         mode = Mode.DISC;
         this.val = 0;
@@ -576,9 +578,9 @@ public Sample(@JsonProperty("val") Double val,
           weights = new ArrayList<>(Collections.nCopies(
               discrete.size(), 1.0));
         }
-        if(weights.size() != discrete.size()){
-          throw new JsonMappingException("Instantiation of " + Sample.class
-              + " failed");
+        if (weights.size() != discrete.size()) {
+          throw JsonMappingException
+              .from((JsonParser) null, "Instantiation of " + Sample.class + " failed");
         }
         this.weights = weights;
       }
@@ -635,7 +637,7 @@ public String getString(){
     public String toString(){
       switch(mode){
       case CONST:
-        return "value: " + Double.toString(val);
+        return "value: " + val;
       case DIST:
         return "value: " + this.val + " std: " + this.std + " dist: "
             + this.dist.name();
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
index 256dcf4629188..af0b4f6caf3ab 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
@@ -57,8 +57,6 @@
 @Private
 @Unstable
 public class SLSUtils {
-  public final static String DEFAULT_JOB_TYPE = "mapreduce";
-
   private static final String LABEL_FORMAT_ERR_MSG =
       "Input format for adding node-labels is not correct, it should be "
           + "labelName1[(exclusive=true/false)],labelName2[] ..";
@@ -87,7 +85,7 @@ public static Set<NodeDetails> parseNodesFromRumenTrace(
     JobTraceReader reader = new JobTraceReader(
             new Path(fin.getAbsolutePath()), conf);
     try {
-      LoggedJob job = null;
+      LoggedJob job;
       while ((job = reader.getNext()) != null) {
         for(LoggedTask mapTask : job.getMapTasks()) {
           // select the last attempt
@@ -125,7 +123,7 @@ public static Set<NodeDetails> parseNodesFromSLSTrace(
     JsonFactory jsonF = new JsonFactory();
     ObjectMapper mapper = new ObjectMapper();
     Reader input =
-        new InputStreamReader(new FileInputStream(jobTrace), "UTF-8");
+        new InputStreamReader(new FileInputStream(jobTrace), StandardCharsets.UTF_8);
     try {
       Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
       while (i.hasNext()) {
@@ -172,7 +170,7 @@ public static Set<NodeDetails> parseNodesFromNodeFile(
     JsonFactory jsonF = new JsonFactory();
     ObjectMapper mapper = new ObjectMapper();
     Reader input =
-        new InputStreamReader(new FileInputStream(nodeFile), "UTF-8");
+        new InputStreamReader(new FileInputStream(nodeFile), StandardCharsets.UTF_8);
     try {
       Iterator<Map> i = mapper.readValues(jsonF.createParser(input), Map.class);
       while (i.hasNext()) {
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
index 1bb4710d56d16..513dbe5a436b6 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
@@ -103,7 +103,7 @@ public void uncaughtException(Thread t, Throwable e) {
     }
 
     if (nodeFile != null) {
-      args = ArrayUtils.addAll(args, new String[] {"-nodes", nodeFile });
+      args = ArrayUtils.addAll(args, "-nodes", nodeFile);
     }
 
     // enable continuous invariant checks
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestDagAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestDagAMSimulator.java
index 8ac7fff75cb82..e458b860e4d4e 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestDagAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestDagAMSimulator.java
@@ -26,6 +26,8 @@
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Tests for DagAMSimulator.
@@ -74,7 +76,17 @@ public void testGetToBeScheduledContainers() throws Exception {
 
   private ContainerSimulator createContainerSim(long allocationId,
       long requestDelay) {
-    return new ContainerSimulator(null, 1000, "*", 1, "Map",
-        null, allocationId, requestDelay);
+    TaskContainerDefinition taskContainerDef =
+        mock(TaskContainerDefinition.class);
+    when(taskContainerDef.getResource()).thenReturn(null);
+    when(taskContainerDef.getDuration()).thenReturn(1000L);
+    when(taskContainerDef.getHostname()).thenReturn("*");
+    when(taskContainerDef.getPriority()).thenReturn(1);
+    when(taskContainerDef.getType()).thenReturn("Map");
+    when(taskContainerDef.getExecutionType()).thenReturn(null);
+    when(taskContainerDef.getAllocationId()).thenReturn(allocationId);
+    when(taskContainerDef.getRequestDelay()).thenReturn(requestDelay);
+    return ContainerSimulator.createFromTaskContainerDefinition(
+        taskContainerDef);
   }
 }
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java
index 0792eece51e8c..dd12a10f94612 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSynthJobGeneration.java
@@ -19,11 +19,14 @@
 
 import org.apache.commons.math3.random.JDKRandomGenerator;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
 import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -32,12 +35,11 @@
 import java.io.IOException;
 import java.util.Arrays;
 
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
-import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES;
-import static org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES;
-
 /**
  * Simple test class driving the {@code SynthTraceJobProducer}, and validating
  * jobs produce are within expected range.
@@ -56,8 +58,9 @@ public void testWorkloadGenerateTime()
         + "{\"time\": 60, \"weight\": 2}," + "{\"time\": 90, \"weight\": 1}"
         + "]}";
 
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.configure(INTERN_FIELD_NAMES, true);
+    JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder();
+    jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true);
+    ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build());
     mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
     SynthTraceJobProducer.Workload wl =
         mapper.readValue(workloadJson, SynthTraceJobProducer.Workload.class);
@@ -86,7 +89,7 @@ public void testWorkloadGenerateTime()
     }
 
     Assert.assertTrue(bucket0 > 0);
-    Assert.assertTrue(bucket1 == 0);
+    assertEquals(0, bucket1);
     Assert.assertTrue(bucket2 > 0);
     Assert.assertTrue(bucket3 > 0);
     Assert.assertTrue(bucket2 > bucket0);
@@ -176,8 +179,9 @@ public void testStream() throws IllegalArgumentException, IOException {
   @Test
   public void testSample() throws IOException {
 
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.configure(INTERN_FIELD_NAMES, true);
+    JsonFactoryBuilder jsonFactoryBuilder = new JsonFactoryBuilder();
+    jsonFactoryBuilder.configure(JsonFactory.Feature.INTERN_FIELD_NAMES, true);
+    ObjectMapper mapper = new ObjectMapper(jsonFactoryBuilder.build());
     mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     JDKRandomGenerator rand = new JDKRandomGenerator();
@@ -235,7 +239,7 @@ public void testSample() throws IOException {
       mapper.readValue(invalidDistJson, SynthTraceJobProducer.Sample.class);
       Assert.fail();
     } catch (JsonMappingException e) {
-      Assert.assertTrue(e.getMessage().startsWith("Instantiation of"));
+      Assert.assertTrue(e.getMessage().startsWith("Cannot construct instance of"));
     }
   }
 
@@ -252,7 +256,7 @@ private void validateJob(SynthJob js) {
     assertTrue(js.getTasks().size() > 0);
 
     for (SynthJob.SynthTask t : js.getTasks()) {
-      assertTrue(t.getType() != null);
+      assertNotNull(t.getType());
       assertTrue(t.getTime() > 0);
       assertTrue(t.getMemory() > 0);
       assertTrue(t.getVcores() > 0);
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
index 50ac700d9c666..f5db1684c7164 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java
@@ -20,6 +20,7 @@
 import com.codahale.metrics.MetricRegistry;
 import java.util.HashMap;
 import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.tools.rumen.datatypes.UserName;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -33,6 +34,8 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.sls.AMDefinitionRumen;
+import org.apache.hadoop.yarn.sls.TaskContainerDefinition;
 import org.apache.hadoop.yarn.sls.SLSRunner;
 import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
 import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator;
@@ -57,6 +60,7 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentMap;
 
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 @RunWith(Parameterized.class)
@@ -157,9 +161,20 @@ public void testAMSimulator() throws Exception {
     String queue = "default";
     List<ContainerSimulator> containers = new ArrayList<>();
     HashMap<ApplicationId, AMSimulator> map = new HashMap<>();
-    app.init(1000, containers, rm, null, 0, 1000000L, "user1", queue, true,
-        appId, 0, SLSConfiguration.getAMContainerResource(conf), null, null,
-        map);
+
+    UserName mockUser = mock(UserName.class);
+    when(mockUser.getValue()).thenReturn("user1");
+    AMDefinitionRumen amDef =
+        AMDefinitionRumen.Builder.create()
+        .withUser(mockUser)
+        .withQueue(queue)
+        .withJobId(appId)
+        .withJobStartTime(0)
+        .withJobFinishTime(1000000L)
+        .withAmResource(SLSConfiguration.getAMContainerResource(conf))
+        .withTaskContainers(containers)
+        .build();
+    app.init(amDef, rm, null, true, 0, 1000, map);
     app.firstStep();
 
     verifySchedulerMetrics(appId);
@@ -184,9 +199,21 @@ public void testAMSimulatorWithNodeLabels() throws Exception {
       String queue = "default";
       List<ContainerSimulator> containers = new ArrayList<>();
       HashMap<ApplicationId, AMSimulator> map = new HashMap<>();
-      app.init(1000, containers, rm, null, 0, 1000000L, "user1", queue, true,
-          appId, 0, SLSConfiguration.getAMContainerResource(conf), "label1",
-          null, map);
+
+      UserName mockUser = mock(UserName.class);
+      when(mockUser.getValue()).thenReturn("user1");
+      AMDefinitionRumen amDef =
+          AMDefinitionRumen.Builder.create()
+              .withUser(mockUser)
+              .withQueue(queue)
+              .withJobId(appId)
+              .withJobStartTime(0)
+              .withJobFinishTime(1000000L)
+              .withAmResource(SLSConfiguration.getAMContainerResource(conf))
+              .withTaskContainers(containers)
+              .withLabelExpression("label1")
+              .build();
+      app.init(amDef, rm, null, true, 0, 1000, map);
       app.firstStep();
 
       verifySchedulerMetrics(appId);
@@ -201,7 +228,7 @@ public void testAMSimulatorWithNodeLabels() throws Exception {
   }
 
   @Test
-  public void testPackageRequests() {
+  public void testPackageRequests() throws YarnException {
     MockAMSimulator app = new MockAMSimulator();
     List<ContainerSimulator> containerSimulators = new ArrayList<>();
     Resource resource = Resources.createResource(1024);
@@ -209,12 +236,25 @@ public void testPackageRequests() {
     ExecutionType execType = ExecutionType.GUARANTEED;
     String type = "map";
 
-    ContainerSimulator s1 = new ContainerSimulator(resource, 100,
-        "/default-rack/h1", priority, type, execType);
-    ContainerSimulator s2 = new ContainerSimulator(resource, 100,
-        "/default-rack/h1", priority, type, execType);
-    ContainerSimulator s3 = new ContainerSimulator(resource, 100,
-        "/default-rack/h2", priority, type, execType);
+    TaskContainerDefinition.Builder builder =
+        TaskContainerDefinition.Builder.create()
+        .withResource(resource)
+        .withDuration(100)
+        .withPriority(1)
+        .withType(type)
+        .withExecutionType(execType)
+        .withAllocationId(-1)
+        .withRequestDelay(0);
+
+    ContainerSimulator s1 = ContainerSimulator
+        .createFromTaskContainerDefinition(
+            builder.withHostname("/default-rack/h1").build());
+    ContainerSimulator s2 = ContainerSimulator
+        .createFromTaskContainerDefinition(
+            builder.withHostname("/default-rack/h1").build());
+    ContainerSimulator s3 = ContainerSimulator
+        .createFromTaskContainerDefinition(
+            builder.withHostname("/default-rack/h2").build());
 
     containerSimulators.add(s1);
     containerSimulators.add(s2);
@@ -250,12 +290,15 @@ public void testPackageRequests() {
     Assert.assertEquals(2, nodeRequestCount);
 
     containerSimulators.clear();
-    s1 = new ContainerSimulator(resource, 100,
-        "/default-rack/h1", priority, type, execType, 1, 0);
-    s2 = new ContainerSimulator(resource, 100,
-        "/default-rack/h1", priority, type, execType, 2, 0);
-    s3 = new ContainerSimulator(resource, 100,
-        "/default-rack/h2", priority, type, execType, 1, 0);
+    s1 = ContainerSimulator.createFromTaskContainerDefinition(
+        createDefaultTaskContainerDefMock(resource, priority, execType, type,
+            "/default-rack/h1", 1));
+    s2 = ContainerSimulator.createFromTaskContainerDefinition(
+        createDefaultTaskContainerDefMock(resource, priority, execType, type,
+            "/default-rack/h1", 2));
+    s3 = ContainerSimulator.createFromTaskContainerDefinition(
+        createDefaultTaskContainerDefMock(resource, priority, execType, type,
+            "/default-rack/h2", 1));
 
     containerSimulators.add(s1);
     containerSimulators.add(s2);
@@ -317,6 +360,20 @@ public void testAMSimulatorRanNodesCleared() throws Exception {
     Assert.assertFalse(nm.getNode().getRunningApps().contains(app.appId));
     Assert.assertTrue(nm.getNode().getRunningApps().isEmpty());
   }
+  private TaskContainerDefinition createDefaultTaskContainerDefMock(
+      Resource resource, int priority, ExecutionType execType, String type,
+      String hostname, long allocationId) {
+    TaskContainerDefinition taskContainerDef =
+        mock(TaskContainerDefinition.class);
+    when(taskContainerDef.getResource()).thenReturn(resource);
+    when(taskContainerDef.getDuration()).thenReturn(100L);
+    when(taskContainerDef.getPriority()).thenReturn(priority);
+    when(taskContainerDef.getType()).thenReturn(type);
+    when(taskContainerDef.getExecutionType()).thenReturn(execType);
+    when(taskContainerDef.getHostname()).thenReturn(hostname);
+    when(taskContainerDef.getAllocationId()).thenReturn(allocationId);
+    return taskContainerDef;
+  }
 
   @After
   public void tearDown() {
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
index 423d6b2c7fe84..5376a55cc639b 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
@@ -36,7 +36,7 @@ public class TestSLSUtils {
   @Test
   public void testGetRackHostname() {
     String str = "/rack1/node1";
-    String rackHostname[] = SLSUtils.getRackHostName(str);
+    String[] rackHostname = SLSUtils.getRackHostName(str);
     Assert.assertEquals("rack1", rackHostname[0]);
     Assert.assertEquals("node1", rackHostname[1]);
 
@@ -81,7 +81,7 @@ public void testParseNodesFromNodeFile() throws Exception {
           } else if(nodeLabel.getName().equals("label2")) {
             Assert.assertFalse(nodeLabel.isExclusive());
           } else {
-            Assert.assertTrue("Unexepected label", false);
+            Assert.fail("Unexpected label");
           }
         }
       } else if (nodeDetail.getHostname().equals("/rack1/node4")) {
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index f03eb001450b4..33e6ca8ff7491 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -96,6 +96,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
index 551a5f2dc407b..4f9d820c1240f 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
@@ -40,6 +40,7 @@
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
@@ -969,10 +970,12 @@ protected void setJobConf() throws IOException {
         fail(LINK_URI);
     }
     // set the jobconf for the caching parameters
-    if (cacheArchives != null)
-      DistributedCache.setCacheArchives(archiveURIs, jobConf_);
-    if (cacheFiles != null)
-      DistributedCache.setCacheFiles(fileURIs, jobConf_);
+    if (cacheArchives != null) {
+      Job.setCacheArchives(archiveURIs, jobConf_);
+    }
+    if (cacheFiles != null) {
+      Job.setCacheFiles(fileURIs, jobConf_);
+    }
 
     if (verbose_) {
       listJobConfProperties();
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java
index 77f4e041d5f09..f44488c7c0202 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/mapreduce/StreamInputFormat.java
@@ -26,7 +26,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.impl.FutureIOSupport;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -35,6 +34,7 @@
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
 import org.apache.hadoop.streaming.StreamUtil;
+import org.apache.hadoop.util.functional.FutureIO;
 
 /**
  * An input format that selects a RecordReader based on a JobConf property. This
@@ -66,10 +66,10 @@ public RecordReader<Text, Text> createRecordReader(InputSplit genericSplit,
     FileSystem fs = path.getFileSystem(conf);
     // open the file
     final FutureDataInputStreamBuilder builder = fs.openFile(path);
-    FutureIOSupport.propagateOptions(builder, conf,
+    FutureIO.propagateOptions(builder, conf,
         MRJobConfig.INPUT_FILE_OPTION_PREFIX,
         MRJobConfig.INPUT_FILE_MANDATORY_PREFIX);
-    FSDataInputStream in = FutureIOSupport.awaitFuture(builder.build());
+    FSDataInputStream in = FutureIO.awaitFuture(builder.build());
 
     // Factory dispatch based on available params..
     Class readerClass;
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index cc811fca6949d..8a3e93c1037d3 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -38,6 +38,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-streaming</artifactId>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -86,12 +92,6 @@
       <type>pom</type>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-openstack</artifactId>
-      <scope>compile</scope>
-      <version>${project.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-aws</artifactId>
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index f026bc261e00b..4e934cd101f85 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -51,6 +51,7 @@
     <module>hadoop-azure-datalake</module>
     <module>hadoop-aliyun</module>
     <module>hadoop-fs2img</module>
+    <module>hadoop-benchmark</module>
   </modules>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/README b/hadoop-yarn-project/hadoop-yarn/README
deleted file mode 100644
index 13835fee222bc..0000000000000
--- a/hadoop-yarn-project/hadoop-yarn/README
+++ /dev/null
@@ -1,78 +0,0 @@
-YARN (YET ANOTHER RESOURCE NEGOTIATOR or YARN Application Resource Negotiator)
-------------------------------------------------------------------------------
-
-Requirements
--------------
-Java: JDK 1.6
-Maven: Maven 3
-
-Setup
------
-Install protobuf 2.5.0 (Download from http://code.google.com/p/protobuf/downloads/list)
- - install the protoc executable (configure, make, make install)
- - install the maven artifact (cd java; mvn install)
-
-
-Quick Maven Tips
-----------------
-clean workspace: mvn clean
-compile and test: mvn install
-skip tests: mvn install -DskipTests
-skip test execution but compile: mvn install -Dmaven.test.skip.exec=true
-clean and test: mvn clean install
-run selected test after compile: mvn test -Dtest=TestClassName (combined: mvn clean install -Dtest=TestClassName)
-create runnable binaries after install: mvn assembly:assembly -Pnative (combined: mvn clean install assembly:assembly -Pnative)
-
-Eclipse Projects
-----------------
-http://maven.apache.org/guides/mini/guide-ide-eclipse.html
-
-1. Generate .project and .classpath files in all maven modules
-mvn eclipse:eclipse
-CAUTION: If the project structure has changed from your previous workspace, clean up all .project and .classpath files recursively. Then run:
-mvn eclipse:eclipse
-
-2. Import the projects in eclipse.
-
-3. Set the environment variable M2_REPO to point to your .m2/repository location.
-
-NetBeans Projects
------------------
-
-NetBeans has builtin support of maven projects. Just "Open Project..."
-and everything is setup automatically. Verified with NetBeans 6.9.1.
-
-
-Custom Hadoop Dependencies
---------------------------
-
-By default Hadoop dependencies are specified in the top-level pom.xml
-properties section. One can override them via -Dhadoop-common.version=...
-on the command line. ~/.m2/settings.xml can also be used to specify
-these properties in different profiles, which is useful for IDEs.
-
-Modules
--------
-YARN consists of multiple modules. The modules are listed below as per the directory structure:
-
-hadoop-yarn-api - YARN's cross platform external interface
-
-hadoop-yarn-common - Utilities which can be used by yarn clients and server
-
-hadoop-yarn-server - Implementation of the hadoop-yarn-api
-	hadoop-yarn-server-common - APIs shared between resourcemanager and nodemanager
-	hadoop-yarn-server-nodemanager (TaskTracker replacement)
-	hadoop-yarn-server-resourcemanager (JobTracker replacement)
-
-Utilities for understanding the code
-------------------------------------
-Almost all of the yarn components as well as the mapreduce framework use
-state-machines for all the data objects. To understand those central pieces of
-the code, a visual representation of the state-machines helps much. You can first
-convert the state-machines into graphviz(.gv) format by
-running:
-   mvn compile -Pvisualize
-Then you can use the dot program for generating directed graphs and convert the above
-.gv files to images. The graphviz package has the needed dot program and related
-utilites.For e.g., to generate png files you can run:
-   dot -Tpng NodeManager.gv > NodeManager.png
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql
index eae882e4a48dd..6461cf2bd75e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql
@@ -122,10 +122,21 @@ BEGIN
    WHERE applicationId = applicationID_IN;
 END //
 
-CREATE PROCEDURE sp_getApplicationsHomeSubCluster()
+CREATE PROCEDURE sp_getApplicationsHomeSubCluster(IN limit_IN int, IN homeSubCluster_IN varchar(256))
 BEGIN
-   SELECT applicationId, homeSubCluster
-   FROM applicationsHomeSubCluster;
+   SELECT
+       applicationId,
+       homeSubCluster,
+       createTime
+   FROM (SELECT
+             applicationId,
+             homeSubCluster,
+             createTime,
+             @rownum := 0
+         FROM applicationshomesubcluster
+         ORDER BY createTime DESC) AS applicationshomesubcluster
+   WHERE (homeSubCluster_IN = '' OR homeSubCluster = homeSubCluster_IN)
+     AND (@rownum := @rownum + 1) <= limit_IN;
 END //
 
 CREATE PROCEDURE sp_deleteApplicationHomeSubCluster(
@@ -159,4 +170,53 @@ BEGIN
    FROM policies WHERE queue = queue_IN;
 END //
 
+CREATE PROCEDURE sp_addReservationHomeSubCluster(
+   IN reservationId_IN varchar(128), IN homeSubCluster_IN varchar(256),
+   OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)
+BEGIN
+   INSERT INTO reservationsHomeSubCluster
+      (reservationId,homeSubCluster)
+      (SELECT reservationId_IN, homeSubCluster_IN
+       FROM applicationsHomeSubCluster
+       WHERE reservationId = reservationId_IN
+       HAVING COUNT(*) = 0 );
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+   SELECT homeSubCluster INTO storedHomeSubCluster_OUT
+   FROM reservationsHomeSubCluster
+   WHERE applicationId = reservationId_IN;
+END //
+
+CREATE PROCEDURE sp_getReservationHomeSubCluster(
+   IN reservationId_IN varchar(128),
+   OUT homeSubCluster_OUT varchar(256))
+BEGIN
+   SELECT homeSubCluster INTO homeSubCluster_OUT
+   FROM reservationsHomeSubCluster
+   WHERE reservationId = reservationId_IN;
+END //
+
+CREATE PROCEDURE sp_getReservationsHomeSubCluster()
+BEGIN
+   SELECT reservationId, homeSubCluster
+   FROM reservationsHomeSubCluster;
+END //
+
+CREATE PROCEDURE sp_updateReservationHomeSubCluster(
+   IN reservationId_IN varchar(128),
+   IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)
+BEGIN
+   UPDATE reservationsHomeSubCluster
+     SET homeSubCluster = homeSubCluster_IN
+   WHERE reservationId = reservationId_IN;
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_deleteReservationHomeSubCluster(
+   IN reservationId_IN varchar(128), OUT rowCount_OUT int)
+BEGIN
+   DELETE FROM reservationsHomeSubCluster
+   WHERE reservationId = reservationId_IN;
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
 DELIMITER ;
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql
index 3a255f06bc3f8..d61a10f998b3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql
@@ -22,7 +22,8 @@ USE FederationStateStore
 
 CREATE TABLE applicationsHomeSubCluster(
    applicationId varchar(64) NOT NULL,
-   homeSubCluster varchar(256) NULL,
+   homeSubCluster varchar(256) NOT NULL,
+   createTime datetime NOT NULL,
    CONSTRAINT pk_applicationId PRIMARY KEY (applicationId)
 );
 
@@ -36,7 +37,7 @@ CREATE TABLE membership(
    state varchar(32) NOT NULL,
    lastStartTime bigint NULL,
    capability varchar(6000),
-   CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId)
+   CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId),
    UNIQUE(lastStartTime)
 );
 
@@ -46,3 +47,9 @@ CREATE TABLE policies(
    params varbinary(32768),
    CONSTRAINT pk_queue PRIMARY KEY (queue)
 );
+
+CREATE TABLE reservationsHomeSubCluster (
+   reservationId varchar(128) NOT NULL,
+   homeSubCluster varchar(256) NOT NULL,
+   CONSTRAINT pk_reservationId PRIMARY KEY (reservationId)
+);
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql
index f24f3fb22b55c..a2f0b882b3ff6 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql
@@ -45,3 +45,13 @@ DROP PROCEDURE sp_setPolicyConfiguration;
 DROP PROCEDURE sp_getPolicyConfiguration;
 
 DROP PROCEDURE sp_getPoliciesConfigurations;
+
+DROP PROCEDURE sp_addReservationHomeSubCluster;
+
+DROP PROCEDURE sp_getReservationHomeSubCluster;
+
+DROP PROCEDURE sp_getReservationsHomeSubCluster;
+
+DROP PROCEDURE sp_deleteReservationHomeSubCluster;
+
+DROP PROCEDURE sp_updateReservationHomeSubCluster;
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql
index ea6567b028b81..d29f8652c153e 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql
@@ -25,3 +25,5 @@ DROP TABLE applicationsHomeSubCluster;
 DROP TABLE membership;
 
 DROP TABLE policies;
+
+DROP TABLE reservationsHomeSubCluster;
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql
new file mode 100644
index 0000000000000..ae0a1c5b77fc3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to create a new Database in SQLServer for the Federation StateStore
+
+IF DB_ID ( '[FederationStateStore]') IS NOT NULL
+  DROP DATABASE [FederationStateStore];
+GO
+
+CREATE database FederationStateStore;
+GO
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoredProcs.sql
similarity index 52%
rename from hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql
rename to hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoredProcs.sql
index 66d6f0e203558..cc8a79d627367 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoredProcs.sql
@@ -24,10 +24,10 @@ IF OBJECT_ID ( '[sp_addApplicationHomeSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_addApplicationHomeSubCluster]
-    @applicationId VARCHAR(64),
-    @homeSubCluster VARCHAR(256),
-    @storedHomeSubCluster VARCHAR(256) OUTPUT,
-    @rowCount int OUTPUT
+    @applicationId_IN VARCHAR(64),
+    @homeSubCluster_IN VARCHAR(256),
+    @storedHomeSubCluster_OUT VARCHAR(256) OUTPUT,
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -37,21 +37,21 @@ AS BEGIN
             -- Otherwise don't change the current mapping.
             IF NOT EXISTS (SELECT TOP 1 *
                        FROM [dbo].[applicationsHomeSubCluster]
-                       WHERE [applicationId] = @applicationId)
+                       WHERE [applicationId] = @applicationId_IN)
 
                 INSERT INTO [dbo].[applicationsHomeSubCluster] (
                     [applicationId],
                     [homeSubCluster])
                 VALUES (
-                    @applicationId,
-                    @homeSubCluster);
+                    @applicationId_IN,
+                    @homeSubCluster_IN);
             -- End of the IF block
 
-            SELECT @rowCount = @@ROWCOUNT;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
-            SELECT @storedHomeSubCluster = [homeSubCluster]
+            SELECT @storedHomeSubCluster_OUT = [homeSubCluster]
             FROM [dbo].[applicationsHomeSubCluster]
-            WHERE [applicationId] = @applicationId;
+            WHERE [applicationId] = @applicationId_IN;
 
         COMMIT TRAN
     END TRY
@@ -75,9 +75,9 @@ IF OBJECT_ID ( '[sp_updateApplicationHomeSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_updateApplicationHomeSubCluster]
-    @applicationId VARCHAR(64),
-    @homeSubCluster VARCHAR(256),
-    @rowCount int OUTPUT
+    @applicationId_IN VARCHAR(64),
+    @homeSubCluster_IN VARCHAR(256),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -85,9 +85,9 @@ AS BEGIN
         BEGIN TRAN
 
             UPDATE [dbo].[applicationsHomeSubCluster]
-            SET [homeSubCluster] = @homeSubCluster
-            WHERE [applicationId] = @applicationid;
-            SELECT @rowCount = @@ROWCOUNT;
+            SET [homeSubCluster] = @homeSubCluster_IN
+            WHERE [applicationId] = @applicationId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -111,12 +111,26 @@ IF OBJECT_ID ( '[sp_getApplicationsHomeSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_getApplicationsHomeSubCluster]
+    @limit_IN int,
+    @homeSubCluster_IN VARCHAR(256)
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
     BEGIN TRY
-        SELECT [applicationId], [homeSubCluster], [createTime]
-        FROM [dbo].[applicationsHomeSubCluster]
+
+        SELECT
+            [applicationId],
+            [homeSubCluster],
+            [createTime]
+        FROM(SELECT
+                 [applicationId],
+                 [homeSubCluster],
+                 [createTime],
+                 row_number() over(order by [createTime] desc) AS app_rank
+             FROM [dbo].[applicationsHomeSubCluster]
+             WHERE [homeSubCluster] = @homeSubCluster_IN OR @homeSubCluster = '') AS applicationsHomeSubCluster
+        WHERE app_rank <= @limit_IN;
+
     END TRY
 
     BEGIN CATCH
@@ -136,16 +150,16 @@ IF OBJECT_ID ( '[sp_getApplicationHomeSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_getApplicationHomeSubCluster]
-    @applicationId VARCHAR(64),
-    @homeSubCluster VARCHAR(256) OUTPUT
+    @applicationId_IN VARCHAR(64),
+    @homeSubCluster_OUT VARCHAR(256) OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
     BEGIN TRY
 
-        SELECT @homeSubCluster = [homeSubCluster]
+        SELECT @homeSubCluster_OUT = [homeSubCluster]
         FROM [dbo].[applicationsHomeSubCluster]
-        WHERE [applicationId] = @applicationid;
+        WHERE [applicationId] = @applicationId_IN;
 
     END TRY
 
@@ -167,8 +181,8 @@ IF OBJECT_ID ( '[sp_deleteApplicationHomeSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_deleteApplicationHomeSubCluster]
-    @applicationId VARCHAR(64),
-    @rowCount int OUTPUT
+    @applicationId_IN VARCHAR(64),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -176,8 +190,8 @@ AS BEGIN
         BEGIN TRAN
 
             DELETE FROM [dbo].[applicationsHomeSubCluster]
-            WHERE [applicationId] = @applicationId;
-            SELECT @rowCount = @@ROWCOUNT;
+            WHERE [applicationId] = @applicationId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -201,15 +215,15 @@ IF OBJECT_ID ( '[sp_registerSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_registerSubCluster]
-    @subClusterId VARCHAR(256),
-    @amRMServiceAddress VARCHAR(256),
-    @clientRMServiceAddress VARCHAR(256),
-    @rmAdminServiceAddress VARCHAR(256),
-    @rmWebServiceAddress VARCHAR(256),
-    @state VARCHAR(32),
-    @lastStartTime BIGINT,
-    @capability VARCHAR(6000),
-    @rowCount int OUTPUT
+    @subClusterId_IN VARCHAR(256),
+    @amRMServiceAddress_IN VARCHAR(256),
+    @clientRMServiceAddress_IN VARCHAR(256),
+    @rmAdminServiceAddress_IN VARCHAR(256),
+    @rmWebServiceAddress_IN VARCHAR(256),
+    @state_IN VARCHAR(32),
+    @lastStartTime_IN BIGINT,
+    @capability_IN VARCHAR(6000),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -217,7 +231,7 @@ AS BEGIN
         BEGIN TRAN
 
             DELETE FROM [dbo].[membership]
-            WHERE [subClusterId] = @subClusterId;
+            WHERE [subClusterId] = @subClusterId_IN;
             INSERT INTO [dbo].[membership] (
                 [subClusterId],
                 [amRMServiceAddress],
@@ -229,16 +243,16 @@ AS BEGIN
                 [lastStartTime],
                 [capability] )
             VALUES (
-                @subClusterId,
-                @amRMServiceAddress,
-                @clientRMServiceAddress,
-                @rmAdminServiceAddress,
-                @rmWebServiceAddress,
+                @subClusterId_IN,
+                @amRMServiceAddress_IN,
+                @clientRMServiceAddress_IN,
+                @rmAdminServiceAddress_IN,
+                @rmWebServiceAddress_IN,
                 GETUTCDATE(),
-                @state,
-                @lastStartTime,
-                @capability);
-            SELECT @rowCount = @@ROWCOUNT;
+                @state_IN,
+                @lastStartTime_IN,
+                @capability_IN);
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -289,32 +303,32 @@ IF OBJECT_ID ( '[sp_getSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_getSubCluster]
-    @subClusterId VARCHAR(256),
-    @amRMServiceAddress VARCHAR(256) OUTPUT,
-    @clientRMServiceAddress VARCHAR(256) OUTPUT,
-    @rmAdminServiceAddress VARCHAR(256) OUTPUT,
-    @rmWebServiceAddress VARCHAR(256) OUTPUT,
-    @lastHeartbeat DATETIME2 OUTPUT,
-    @state VARCHAR(256) OUTPUT,
-    @lastStartTime BIGINT OUTPUT,
-    @capability VARCHAR(6000) OUTPUT
+    @subClusterId_IN VARCHAR(256),
+    @amRMServiceAddress_OUT VARCHAR(256) OUTPUT,
+    @clientRMServiceAddress_OUT VARCHAR(256) OUTPUT,
+    @rmAdminServiceAddress_OUT VARCHAR(256) OUTPUT,
+    @rmWebServiceAddress_OUT VARCHAR(256) OUTPUT,
+    @lastHeartBeat_OUT DATETIME2 OUTPUT,
+    @state_OUT VARCHAR(256) OUTPUT,
+    @lastStartTime_OUT BIGINT OUTPUT,
+    @capability_OUT VARCHAR(6000) OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
     BEGIN TRY
         BEGIN TRAN
 
-            SELECT @subClusterId = [subClusterId],
-                   @amRMServiceAddress = [amRMServiceAddress],
-                   @clientRMServiceAddress = [clientRMServiceAddress],
-                   @rmAdminServiceAddress = [rmAdminServiceAddress],
-                   @rmWebServiceAddress = [rmWebServiceAddress],
-                   @lastHeartBeat = [lastHeartBeat],
-                   @state = [state],
-                   @lastStartTime = [lastStartTime],
-                   @capability = [capability]
+            SELECT @subClusterId_IN = [subClusterId],
+                   @amRMServiceAddress_OUT = [amRMServiceAddress],
+                   @clientRMServiceAddress_OUT = [clientRMServiceAddress],
+                   @rmAdminServiceAddress_OUT = [rmAdminServiceAddress],
+                   @rmWebServiceAddress_OUT = [rmWebServiceAddress],
+                   @lastHeartBeat_OUT = [lastHeartBeat],
+                   @state_OUT = [state],
+                   @lastStartTime_OUT = [lastStartTime],
+                   @capability_OUT = [capability]
             FROM [dbo].[membership]
-            WHERE [subClusterId] = @subClusterId
+            WHERE [subClusterId] = @subClusterId_IN
 
         COMMIT TRAN
     END TRY
@@ -339,10 +353,10 @@ IF OBJECT_ID ( '[sp_subClusterHeartbeat]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_subClusterHeartbeat]
-    @subClusterId VARCHAR(256),
-    @state VARCHAR(256),
-    @capability VARCHAR(6000),
-    @rowCount int OUTPUT
+    @subClusterId_IN VARCHAR(256),
+    @state_IN VARCHAR(256),
+    @capability_IN VARCHAR(6000),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -350,11 +364,11 @@ AS BEGIN
         BEGIN TRAN
 
             UPDATE [dbo].[membership]
-            SET [state] = @state,
+            SET [state] = @state_IN,
                 [lastHeartbeat] = GETUTCDATE(),
-                [capability] = @capability
-            WHERE [subClusterId] = @subClusterId;
-            SELECT @rowCount = @@ROWCOUNT;
+                [capability] = @capability_IN
+            WHERE [subClusterId] = @subClusterId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -378,9 +392,9 @@ IF OBJECT_ID ( '[sp_deregisterSubCluster]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_deregisterSubCluster]
-    @subClusterId VARCHAR(256),
-    @state VARCHAR(256),
-    @rowCount int OUTPUT
+    @subClusterId_IN VARCHAR(256),
+    @state_IN VARCHAR(256),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -388,9 +402,9 @@ AS BEGIN
         BEGIN TRAN
 
             UPDATE [dbo].[membership]
-            SET [state] = @state
-            WHERE [subClusterId] = @subClusterId;
-            SELECT @rowCount = @@ROWCOUNT;
+            SET [state] = @state_IN
+            WHERE [subClusterId] = @subClusterId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -414,10 +428,10 @@ IF OBJECT_ID ( '[sp_setPolicyConfiguration]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_setPolicyConfiguration]
-    @queue VARCHAR(256),
-    @policyType VARCHAR(256),
-    @params VARBINARY(512),
-    @rowCount int OUTPUT
+    @queue_IN VARCHAR(256),
+    @policyType_IN VARCHAR(256),
+    @params_IN VARBINARY(512),
+    @rowCount_OUT int OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
@@ -425,16 +439,16 @@ AS BEGIN
         BEGIN TRAN
 
             DELETE FROM [dbo].[policies]
-            WHERE [queue] = @queue;
+            WHERE [queue] = @queue_IN;
             INSERT INTO [dbo].[policies] (
                 [queue],
                 [policyType],
                 [params])
             VALUES (
-                @queue,
-                @policyType,
-                @params);
-            SELECT @rowCount = @@ROWCOUNT;
+                @queue_IN,
+                @policyType_IN,
+                @params_IN);
+            SELECT @rowCount_OUT = @@ROWCOUNT;
 
         COMMIT TRAN
     END TRY
@@ -458,18 +472,18 @@ IF OBJECT_ID ( '[sp_getPolicyConfiguration]', 'P' ) IS NOT NULL
 GO
 
 CREATE PROCEDURE [dbo].[sp_getPolicyConfiguration]
-    @queue VARCHAR(256),
-    @policyType VARCHAR(256) OUTPUT,
-    @params VARBINARY(6000) OUTPUT
+    @queue_IN VARCHAR(256),
+    @policyType_OUT VARCHAR(256) OUTPUT,
+    @params_OUT VARBINARY(6000) OUTPUT
 AS BEGIN
     DECLARE @errorMessage nvarchar(4000)
 
     BEGIN TRY
 
-        SELECT @policyType = [policyType],
-               @params = [params]
+        SELECT @policyType_OUT = [policyType],
+               @params_OUT = [params]
         FROM [dbo].[policies]
-        WHERE [queue] = @queue
+        WHERE [queue] = @queue_IN
 
     END TRY
 
@@ -508,4 +522,181 @@ AS BEGIN
         ) WITH log
     END CATCH
 END;
+GO
+
+IF OBJECT_ID ( '[sp_addReservationHomeSubCluster]', 'P' ) IS NOT NULL
+    DROP PROCEDURE [sp_addReservationHomeSubCluster];
+GO
+
+CREATE PROCEDURE [dbo].[sp_addReservationHomeSubCluster]
+    @reservationId_IN VARCHAR(128),
+    @homeSubCluster_IN VARCHAR(256),
+    @storedHomeSubCluster_OUT VARCHAR(256) OUTPUT,
+    @rowCount_OUT int OUTPUT
+AS BEGIN
+    DECLARE @errorMessage nvarchar(4000)
+
+    BEGIN TRY
+        BEGIN TRAN
+            -- If application to sub-cluster map doesn't exist, insert it.
+            -- Otherwise don't change the current mapping.
+            IF NOT EXISTS (SELECT TOP 1 *
+                       FROM [dbo].[reservationsHomeSubCluster]
+                       WHERE [reservationId] = @reservationId_IN)
+
+                INSERT INTO [dbo].[reservationsHomeSubCluster] (
+                    [reservationId],
+                    [homeSubCluster])
+                VALUES (
+                    @reservationId_IN,
+                    @homeSubCluster_IN);
+            -- End of the IF block
+
+            SELECT @rowCount_OUT = @@ROWCOUNT;
+
+            SELECT @storedHomeSubCluster_OUT = [homeSubCluster]
+            FROM [dbo].[reservationsHomeSubCluster]
+            WHERE [reservationId] = @reservationId_IN;
+
+        COMMIT TRAN
+    END TRY
+
+    BEGIN CATCH
+        ROLLBACK TRAN
+
+        SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE())
+
+        /*  raise error and terminate the execution */
+        RAISERROR(@errorMessage, --- Error Message
+            1, -- Severity
+            -1 -- State
+        ) WITH log
+    END CATCH
+END;
+GO
+
+IF OBJECT_ID ( '[sp_updateReservationHomeSubCluster]', 'P' ) IS NOT NULL
+    DROP PROCEDURE [sp_updateReservationHomeSubCluster];
+GO
+
+CREATE PROCEDURE [dbo].[sp_updateReservationHomeSubCluster]
+    @reservationId_IN VARCHAR(128),
+    @homeSubCluster_IN VARCHAR(256),
+    @rowCount_OUT int OUTPUT
+AS BEGIN
+    DECLARE @errorMessage nvarchar(4000)
+
+    BEGIN TRY
+        BEGIN TRAN
+
+            UPDATE [dbo].[reservationsHomeSubCluster]
+            SET [homeSubCluster] = @homeSubCluster_IN
+            WHERE [reservationId] = @reservationId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
+
+        COMMIT TRAN
+    END TRY
+
+    BEGIN CATCH
+        ROLLBACK TRAN
+
+        SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE())
+
+        /*  raise error and terminate the execution */
+        RAISERROR(@errorMessage, --- Error Message
+            1, -- Severity
+            -1 -- State
+        ) WITH log
+    END CATCH
+END;
+GO
+
+IF OBJECT_ID ( '[sp_getReservationsHomeSubCluster]', 'P' ) IS NOT NULL
+    DROP PROCEDURE [sp_getReservationsHomeSubCluster];
+GO
+
+CREATE PROCEDURE [dbo].[sp_getReservationsHomeSubCluster]
+AS BEGIN
+    DECLARE @errorMessage nvarchar(4000)
+
+    BEGIN TRY
+        SELECT [reservationId], [homeSubCluster], [createTime]
+        FROM [dbo].[reservationsHomeSubCluster]
+    END TRY
+
+    BEGIN CATCH
+        SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE())
+
+        /*  raise error and terminate the execution */
+        RAISERROR(@errorMessage, --- Error Message
+            1, -- Severity
+            -1 -- State
+        ) WITH log
+    END CATCH
+END;
+GO
+
+IF OBJECT_ID ( '[sp_getReservationHomeSubCluster]', 'P' ) IS NOT NULL
+    DROP PROCEDURE [sp_getReservationHomeSubCluster];
+GO
+
+CREATE PROCEDURE [dbo].[sp_getReservationHomeSubCluster]
+    @reservationId_IN VARCHAR(128),
+    @homeSubCluster_OUT VARCHAR(256) OUTPUT
+AS BEGIN
+    DECLARE @errorMessage nvarchar(4000)
+
+    BEGIN TRY
+
+        SELECT @homeSubCluster_OUT = [homeSubCluster]
+        FROM [dbo].[reservationsHomeSubCluster]
+        WHERE [reservationId] = @reservationId_IN;
+
+    END TRY
+
+    BEGIN CATCH
+
+        SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE())
+
+        /*  raise error and terminate the execution */
+        RAISERROR(@errorMessage, --- Error Message
+            1, -- Severity
+            -1 -- State
+        ) WITH log
+    END CATCH
+END;
+GO
+
+IF OBJECT_ID ( '[sp_deleteReservationHomeSubCluster]', 'P' ) IS NOT NULL
+    DROP PROCEDURE [sp_deleteReservationHomeSubCluster];
+GO
+
+CREATE PROCEDURE [dbo].[sp_deleteReservationHomeSubCluster]
+    @reservationId_IN VARCHAR(128),
+    @rowCount_OUT int OUTPUT
+AS BEGIN
+    DECLARE @errorMessage nvarchar(4000)
+
+    BEGIN TRY
+        BEGIN TRAN
+
+            DELETE FROM [dbo].[reservationsHomeSubCluster]
+            WHERE [reservationId] = @reservationId_IN;
+            SELECT @rowCount_OUT = @@ROWCOUNT;
+
+        COMMIT TRAN
+    END TRY
+
+    BEGIN CATCH
+        ROLLBACK TRAN
+
+        SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE())
+
+        /*  raise error and terminate the execution */
+        RAISERROR(@errorMessage, --- Error Message
+            1, -- Severity
+            -1 -- State
+        ) WITH log
+    END CATCH
+END;
 GO
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql
index c16091ccf1461..fb8a1bff554b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql
@@ -77,7 +77,7 @@ IF NOT EXISTS ( SELECT * FROM [FederationStateStore].sys.tables
             CONSTRAINT [pk_subClusterId] PRIMARY KEY
             (
                 [subClusterId]
-            )
+            ),
             CONSTRAINT [uc_lastStartTime] UNIQUE
             (
                 [lastStartTime]
@@ -124,3 +124,35 @@ ELSE
     PRINT 'Table policies exists, no operation required...'
     GO
 GO
+
+IF NOT EXISTS ( SELECT * FROM [FederationStateStore].sys.tables
+    WHERE name = 'reservationsHomeSubCluster'
+    AND schema_id = SCHEMA_ID('dbo'))
+    BEGIN
+        PRINT 'Table reservationsHomeSubCluster does not exist, create it...'
+
+        SET ANSI_NULLS ON
+
+        SET QUOTED_IDENTIFIER ON
+
+        SET ANSI_PADDING ON
+
+        CREATE TABLE [dbo].[reservationsHomeSubCluster](
+            reservationId   VARCHAR(128) COLLATE Latin1_General_100_BIN2 NOT NULL,
+            homeSubCluster  VARCHAR(256) NOT NULL,
+            createTime      DATETIME2 NOT NULL CONSTRAINT ts_createResTime DEFAULT GETUTCDATE(),
+
+            CONSTRAINT [pk_reservationId] PRIMARY KEY
+            (
+                [reservationId]
+            )
+        )
+
+        SET ANSI_PADDING OFF
+
+        PRINT 'Table reservationsHomeSubCluster created.'
+    END
+ELSE
+    PRINT 'Table reservationsHomeSubCluster exists, no operation required...'
+    GO
+GO
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql
new file mode 100644
index 0000000000000..3f9553fbe3286
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to create a new User in SQLServer for the Federation StateStore
+
+USE [FederationStateStore]
+GO
+
+CREATE LOGIN 'FederationUser' with password = 'FederationPassword', default_database=[FederationStateStore] ;
+GO
+
+CREATE USER 'FederationUser' FOR LOGIN 'FederationUser' WITH default_schema=dbo;
+GO
+
+EXEC sp_addrolemember 'db_owner', 'FederationUser';
+GO
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql
new file mode 100644
index 0000000000000..8d434ab4d7b73
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop the Federation StateStore in SQLServer
+
+IF DB_ID ( '[FederationStateStore]') IS NOT NULL
+  DROP DATABASE [FederationStateStore];
+GO
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql
new file mode 100644
index 0000000000000..6204df2f418f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop all the stored procedures for the Federation StateStore in SQLServer
+
+USE [FederationStateStore]
+GO
+
+DROP PROCEDURE IF EXISTS [sp_addApplicationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_updateApplicationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getApplicationsHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getApplicationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_deleteApplicationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_registerSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getSubClusters];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_subClusterHeartbeat];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_deregisterSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_setPolicyConfiguration];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getPolicyConfiguration];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getPoliciesConfigurations];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_addApplicationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_updateReservationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getReservationsHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_getReservationHomeSubCluster];
+GO
+
+DROP PROCEDURE IF EXISTS [sp_deleteReservationHomeSubCluster];
+GO
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql
new file mode 100644
index 0000000000000..9bcacb7f8851a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop all the tables from the Federation StateStore in SQLServer
+
+USE [FederationStateStore]
+GO
+
+DROP TABLE [applicationsHomeSubCluster];
+GO
+
+DROP TABLE [membership];
+GO
+
+DROP TABLE [policies];
+GO
+
+DROP TABLE [reservationsHomeSubCluster];
+GO
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql
new file mode 100644
index 0000000000000..6d5203a52e519
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop the user from Federation StateStore in MySQL
+
+DROP USER 'FederationUser';
+GO
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 54daeaafbe96b..f305c2744efdf 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -124,7 +124,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
     nodemanager)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
-      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
+      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
       # Backwards compatibility
       if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
@@ -151,7 +151,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
     resourcemanager)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
-      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
+      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
       # Backwards compatibility
       if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
@@ -175,6 +175,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
     router)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
+      # Backwards compatibility
+      if [[ -n "${YARN_ROUTER_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${YARN_ROUTER_HEAPSIZE}"
+      fi
     ;;
     schedulerconf)
     HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
@@ -189,7 +193,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
     timelinereader)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
-      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
+      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
       HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
     ;;
 	nodeattributes)
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index e1ac11287fe49..ed0294c2edf97 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -220,7 +220,7 @@ goto :eof
 :resourcemanager
   set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
   set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
-  set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
+  set CLASSPATH=%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH%
   set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
   set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS%
   if defined YARN_RESOURCEMANAGER_HEAPSIZE (
@@ -251,7 +251,7 @@ goto :eof
 :timelinereader
   set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
   set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
-  set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
+  set CLASSPATH=%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH%
   set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer
   set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS%
   goto :eof
@@ -260,12 +260,15 @@ goto :eof
   set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\router-config\log4j.properties
   set CLASS=org.apache.hadoop.yarn.server.router.Router
   set YARN_OPTS=%YARN_OPTS% %HADOOP_ROUTER_OPTS%
+  if defined YARN_ROUTER_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%YARN_ROUTER_HEAPSIZE%m
+  )
   goto :eof
 
 :nodemanager
   set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
   set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
-  set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
+  set CLASSPATH=%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*;%CLASSPATH%
   set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
   set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS%
   if defined YARN_NODEMANAGER_HEAPSIZE (
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 7cd1390ca948c..b3650a6120cdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -147,6 +147,11 @@
 # Router specific parameters
 ###
 
+# Specify the max heapsize for the Router.  If no units are
+# given, it will be assumed to be in MB.
+# Default is the same as HADOOP_HEAPSIZE_MAX
+#export YARN_ROUTER_HEAPSIZE=
+
 # Specify the JVM options to be used when starting the Router.
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_2.10.2.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_2.10.2.xml
new file mode 100644
index 0000000000000..150443f14a097
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_2.10.2.xml
@@ -0,0 +1,22541 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue May 24 22:57:04 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN API 2.10.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/classes:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.10.2.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.5/commons-io-2.5.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.10.2.jar:/maven/com/nimbusds/nimbus-jose-jwt/7.9/nimbus-jose-jwt-7.9.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.3/json-smart-1.3.3.jar:/maven/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.10.2.jar:/usr/lib/jvm/zulu-7-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/site/jdiff/xml -apiname Apache Hadoop YARN API 2.10.2 -->
+<package name="org.apache.hadoop.yarn.ams">
+</package>
+<package name="org.apache.hadoop.yarn.api">
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <interface name="ApplicationClientProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <method name="getNewApplication" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ApplicationId} for 
+ submitting new applications.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a new, monotonically
+ increasing, {@link ApplicationId} which is used by the client to submit
+ a new application.</p>
+
+ <p>The <code>ResourceManager</code> also responds with details such 
+ as maximum resource capabilities in the cluster as specified in
+ {@link GetNewApplicationResponse}.</p>
+
+ @param request request to get a new <code>ApplicationId</code>
+ @return response containing the new <code>ApplicationId</code> to be used
+ to submit an application
+ @throws YarnException
+ @throws IOException
+ @see #submitApplication(SubmitApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to submit a new application to the
+ <code>ResourceManager.</code></p>
+ 
+ <p>The client is required to provide details such as queue, 
+ {@link Resource} required to run the <code>ApplicationMaster</code>, 
+ the equivalent of {@link ContainerLaunchContext} for launching
+ the <code>ApplicationMaster</code> etc. via the 
+ {@link SubmitApplicationRequest}.</p>
+ 
+ <p>Currently the <code>ResourceManager</code> sends an immediate (empty) 
+ {@link SubmitApplicationResponse} on accepting the submission and throws 
+ an exception if it rejects the submission. However, this call needs to be
+ followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ to make sure that the application gets properly submitted - obtaining a
+ {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee
+ that RM 'remembers' this application beyond failover or restart. If RM
+ failover or RM restart happens before ResourceManager saves the
+ application's state successfully, the subsequent
+ {@link #getApplicationReport(GetApplicationReportRequest)} will throw
+ a {@link ApplicationNotFoundException}. The Clients need to re-submit
+ the application with the same {@link ApplicationSubmissionContext} when
+ it encounters the {@link ApplicationNotFoundException} on the
+ {@link #getApplicationReport(GetApplicationReportRequest)} call.</p>
+ 
+ <p>During the submission process, it checks whether the application
+ already exists. If the application exists, it will simply return
+ SubmitApplicationResponse</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to
+ queues etc. before accepting the application submission.</p>
+ 
+ @param request request to submit a new application
+ @return (empty) response on accepting the submission
+ @throws YarnException
+ @throws IOException
+ @see #getNewApplication(GetNewApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the 
+ <code>ResourceManager</code> to fail an application attempt.</p>
+
+ <p>The client, via {@link FailApplicationAttemptRequest} provides the
+ {@link ApplicationAttemptId} of the attempt to be failed.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before failing the attempt.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to fail an attempt
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to abort submitted application.</p>
+ 
+ <p>The client, via {@link KillApplicationRequest} provides the
+ {@link ApplicationId} of the application to be aborted.</p>
+ 
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before terminating the application.</p> 
+ 
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+ 
+ @param request request to abort a submitted application
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get metrics about the cluster from
+ the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a
+ {@link GetClusterMetricsResponse} which includes the 
+ {@link YarnClusterMetrics} with details such as number of current
+ nodes in the cluster.</p>
+ 
+ @param request request for cluster metrics
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a 
+ {@link GetClusterNodesResponse} which includes the 
+ {@link NodeReport} for all the nodes in the cluster.</p>
+ 
+ @param request request for report on all nodes
+ @return report on all nodes
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queues</em>
+ from the <code>ResourceManager</code>.</p>
+ 
+ <p>The client, via {@link GetQueueInfoRequest}, can ask for details such
+ as used/total resources, child queues, running applications etc.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access before
+ providing the information.</p> 
+ 
+ @param request request to get queue information
+ @return queue information
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueUserAcls" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queue 
+ acls</em> for <em>current user</em> from the <code>ResourceManager</code>.
+ </p>
+ 
+ <p>The <code>ResourceManager</code> responds with queue acls for all
+ existing queues.</p>
+ 
+ @param request request to get queue acls for <em>current user</em>
+ @return queue acls for <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move an application to a new queue.
+ 
+ @param request the application ID and the target queue
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNewReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ReservationId} for
+ submitting new reservations.</p>
+
+ <p>The <code>ResourceManager</code> responds with a new, unique,
+ {@link ReservationId} which is used by the client to submit
+ a new reservation.</p>
+
+ @param request to get a new <code>ReservationId</code>
+ @return response containing the new <code>ReservationId</code> to be used
+ to submit a new reservation
+ @throws YarnException if the reservation system is not enabled.
+ @throws IOException on IO failures.
+ @see #submitReservation(ReservationSubmissionRequest)]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and concurrency needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationSubmissionRequest is satisfiable the
+ {@code ResourceManager} answers with a
+ {@link ReservationSubmissionResponse} that include a non-null
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the reason.
+ 
+ On application submission the client can use this {@link ReservationId} to
+ obtain access to the reserved resources.
+ </p>
+ 
+ <p>
+ The system guarantees that during the time-range specified by the user, the
+ reservationID will be corresponding to a valid reservation. The amount of
+ capacity dedicated to such queue can vary overtime, depending of the
+ allocation that has been determined. But it is guaranteed to satisfy all
+ the constraint expressed by the user in the
+ {@link ReservationSubmissionRequest}.
+ </p>
+ 
+ @param request the request to submit a new Reservation
+ @return response the {@link ReservationId} on accepting the submission
+ @throws YarnException if the request is invalid or reservation cannot be
+           created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationUpdateRequest}. Upon success the previous allocation is
+ substituted by the new one, and on failure (i.e., if the system cannot find
+ a valid allocation for the updated request), the previous allocation
+ remains valid.
+ 
+ The {@link ReservationId} is not changed, and applications currently
+ running within this reservation will automatically receive the resources
+ based on the new allocation.
+ </p>
+ 
+ @param request to update an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ 
+ Upon deletion of a reservation applications running with this reservation,
+ are automatically downgraded to normal jobs running without any dedicated
+ reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+
+ @param request
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mappings
+ in existing cluster
+ </p>
+
+ @param request
+ @return labels to nodes mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @param request to get node labels collection of this cluster
+ @return node labels collection of this cluster
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application.
+ </p>
+ @param request to set priority of an application
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to signal a container. For example,
+ the client can send command OUTPUT_THREAD_DUMP to dump threads of the
+ container.</p>
+
+ <p>The client, via {@link SignalContainerRequest} provides the
+ id of the container and the signal command. </p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application before signaling the container.
+ The user needs to have <code>MODIFY_APP</code> permission.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to signal a container
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set ApplicationTimeouts of an application.
+ The UpdateApplicationTimeoutsRequest should have timeout value with
+ absolute time with ISO8601 format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ </p>
+ <b>Note:</b> If application timeout value is less than or equal to current
+ time then update application throws YarnException.
+ @param request to set ApplicationTimeouts of an application
+ @return a response with updated timeouts.
+ @throws YarnException if update request has empty values or application is
+           in completing states.
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between clients and the <code>ResourceManager</code>
+ to submit/abort jobs and to get information on applications, cluster metrics,
+ nodes, queues and ACLs.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <interface name="ApplicationConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="APP_SUBMIT_TIME_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
+ only]]>
+      </doc>
+    </field>
+    <field name="CONTAINER_TOKEN_FILE_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cache file into which container token is written]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_WEB_PROXY_BASE_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in 
+ ApplicationMaster's environment only. This states that for all non-relative
+ web URLs in the app masters web UI what base should they have.]]>
+      </doc>
+    </field>
+    <field name="LOG_DIR_EXPANSION_VAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The temporary environmental variable for container log directory. This
+ should be replaced by real container log directory on container launch.]]>
+      </doc>
+    </field>
+    <field name="CLASS_PATH_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constant is used to construct class path and it will be replaced with
+ real class path separator(':' for Linux and ';' for Windows) by
+ NodeManager on container launch. User has to use this constant to construct
+ class path if user wants cross-platform practice i.e. submit an application
+ from a Windows client to a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_LEFT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The following two constants are used to expand parameter and it will be
+ replaced with real parameter expansion marker ('%' for Windows and '$' for
+ Linux) by NodeManager on container launch. For example: {{VAR}} will be
+ replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+ constant to construct class path if user wants cross-platform practice i.e.
+ submit an application from a Windows client to a Linux/Unix server or vice
+ versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_RIGHT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User has to use this constant to construct class path if user wants
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="STDERR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STDOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_APP_ATTEMPTS_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment
+ only]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[This is the API for the applications comprising of constants that YARN sets
+ up for the applications and the containers.
+ 
+ TODO: Investigate the semantics and security of each cross-boundary refs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <interface name="ApplicationHistoryProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>ApplicationHistoryServer</code> to
+ get the information of completed applications etc.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <interface name="ApplicationMasterProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by a new <code>ApplicationMaster</code> to register with
+ the <code>ResourceManager</code>.
+ </p>
+
+ <p>
+ The <code>ApplicationMaster</code> needs to provide details such as RPC
+ Port, HTTP tracking url etc. as specified in
+ {@link RegisterApplicationMasterRequest}.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code> responds with critical details such as
+ maximum resource capabilities in the cluster as specified in
+ {@link RegisterApplicationMasterResponse}.
+ </p>
+
+ <p>
+ Re-register is only allowed for <code>Unmanaged Application Master</code>
+ (UAM) HA, with
+ {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ set to true.
+ </p>
+
+ @param request registration request
+ @return registration respose
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException The exception is thrown
+           when an ApplicationMaster tries to register more then once.
+ @see RegisterApplicationMasterRequest
+ @see RegisterApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by an <code>ApplicationMaster</code> to notify the 
+ <code>ResourceManager</code> about its completion (success or failed).</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as 
+ final state, diagnostics (in case of failures) etc. as specified in 
+ {@link FinishApplicationMasterRequest}.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with 
+ {@link FinishApplicationMasterResponse}.</p>
+ 
+ @param request completion request
+ @return completion response
+ @throws YarnException
+ @throws IOException
+ @see FinishApplicationMasterRequest
+ @see FinishApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The main interface between an <code>ApplicationMaster</code> and the
+ <code>ResourceManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> uses this interface to provide a list of
+ {@link ResourceRequest} and returns unused {@link Container} allocated to
+ it via {@link AllocateRequest}. Optionally, the
+ <code>ApplicationMaster</code> can also <em>blacklist</em> resources which
+ it doesn't want to use.
+ </p>
+ 
+ <p>
+ This also doubles up as a <em>heartbeat</em> to let the
+ <code>ResourceManager</code> know that the <code>ApplicationMaster</code>
+ is alive. Thus, applications should periodically make this call to be kept
+ alive. The frequency depends on
+ {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
+ </p>
+ 
+ <p>
+ The <code>ResourceManager</code> responds with list of allocated
+ {@link Container}, status of completed containers and headroom information
+ for the application.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> can use the available headroom
+ (resources) to decide how to utilized allocated resources and make informed
+ decisions about future resource requests.
+ </p>
+ 
+ @param request
+          allocation request
+ @return allocation response
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException
+           This exception is thrown when an ApplicationMaster calls allocate
+           without registering first.
+ @throws InvalidResourceBlacklistRequestException
+           This exception is thrown when an application provides an invalid
+           specification for blacklist of resources.
+ @throws InvalidResourceRequestException
+           This exception is thrown when a {@link ResourceRequest} is out of
+           the range of the configured lower and upper limits on the
+           resources.
+ @see AllocateRequest
+ @see AllocateResponse]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between a live instance of <code>ApplicationMaster</code> 
+ and the <code>ResourceManager</code>.</p>
+ 
+ <p>This is used by the <code>ApplicationMaster</code> to register/unregister
+ and to request and obtain resources in the cluster from the
+ <code>ResourceManager</code>.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <interface name="ClientSCMProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="use" return="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to claim a resource with the
+ <code>SharedCacheManager.</code> The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application will be
+ using the resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>Path</code>
+ to the resource in the shared cache is returned. If the resource does not
+ exist, the response is empty.
+ </p>
+
+ @param request request to claim a resource in the shared cache
+ @return response indicating if the resource is already in the cache
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="release" return="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to release a resource with the
+ <code>SharedCacheManager.</code> This method is called once an application
+ is no longer using a claimed resource in the shared cache. The client uses
+ a checksum to identify the resource and an {@link ApplicationId} to
+ identify which application is releasing the resource.
+ </p>
+
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+
+ <p>
+ Currently the <code>SharedCacheManager</code> sends an empty response.
+ </p>
+
+ @param request request to release a resource in the shared cache
+ @return (empty) response on releasing the resource
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>SharedCacheManager</code> to claim
+ and release resources in the shared cache.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+  <interface name="ContainerManagementProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="startContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> provides a list of
+ {@link StartContainerRequest}s to a <code>NodeManager</code> to
+ <em>start</em> {@link Container}s allocated to it using this interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> has to provide details such as allocated
+ resource capability, security tokens (if enabled), command to be executed
+ to start the container, environment for the process, necessary
+ binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in
+ the {@link StartContainerRequest}.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StartContainersResponse} which includes a list of
+ {@link Container}s of successfully launched {@link Container}s, a
+ containerId-to-exception map for each failed {@link StartContainerRequest} in
+ which the exception indicates errors from per container and a
+ allServicesMetaData map between the names of auxiliary services and their
+ corresponding meta-data. Note: None-container-specific exceptions will
+ still be thrown by the API method itself.
+ </p>
+ <p>
+ The <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the to-be-launched or launched containers.
+ </p>
+ 
+ @param request
+          request to start a list of containers
+ @return response including conatinerIds of all successfully launched
+         containers, a containerId-to-exception map for failed requests and
+         a allServicesMetaData map.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="stopContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> requests a <code>NodeManager</code> to
+ <em>stop</em> a list of {@link Container}s allocated to it using this
+ interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a {@link StopContainersRequest}
+ which includes the {@link ContainerId}s of the containers to be stopped.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StopContainersResponse} which includes a list of {@link ContainerId}
+ s of successfully stopped containers, a containerId-to-exception map for
+ each failed request in which the exception indicates errors from per
+ container. Note: None-container-specific exceptions will still be thrown by
+ the API method itself. <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the containers.
+ </p>
+ 
+ @param request
+          request to stop a list of containers
+ @return response which includes a list of containerIds of successfully
+         stopped containers, a containerId-to-exception map for failed
+         requests.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for current
+ statuses of <code>Container</code>s from the <code>NodeManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a
+ {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s
+ of all containers whose statuses are needed.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> responds with
+ {@link GetContainerStatusesResponse} which includes a list of
+ {@link ContainerStatus} of the successfully queried containers and a
+ containerId-to-exception map for each failed request in which the exception
+ indicates errors from per container. Note: None-container-specific
+ exceptions will still be thrown by the API method itself.
+ </p>
+ 
+ @param request
+          request to get <code>ContainerStatus</code>es of containers with
+          the specified <code>ContainerId</code>s
+ @return response containing the list of <code>ContainerStatus</code> of the
+         successfully queried containers and a containerId-to-exception map
+         for failed requests.
+ 
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="increaseContainersResource" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource increase of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to increase resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully increased and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource update of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to update resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully updated and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from NodeManager]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="localize" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources required by the container.
+ Currently, this API only works for running containers.
+
+ @param request Specify the resources to be localized.
+ @return Response that the localize request is accepted.
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[ReInitialize the Container with a new Launch Context.
+ @param request Specify the new ContainerLaunchContext.
+ @return Response that the ReInitialize request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="restartContainer" return="org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restart the container.
+ @param containerId Container Id.
+ @return Response that the restart request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rollback the Last ReInitialization if possible.
+ @param containerId Container Id.
+ @return Response that the rollback request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.CommitResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Commit the Last ReInitialization if possible. Once the reinitialization
+ has been committed, It cannot be rolled back.
+ @param containerId Container Id.
+ @return Response that the commit request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between an <code>ApplicationMaster</code> and a 
+ <code>NodeManager</code> to start/stop and increase resource of containers
+ and to get status of running containers.</p>
+
+ <p>If security is enabled the <code>NodeManager</code> verifies that the
+ <code>ApplicationMaster</code> has truly been allocated the container
+ by the <code>ResourceManager</code> and also verifies all interactions such 
+ as stopping the container or obtaining status information for the container.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.api.protocolrecords">
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <class name="AllocateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <param name="increaseRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Use {@link AllocateRequest#newInstance(int, float, List, List,
+ ResourceBlacklistRequest, List)} instead
+ @param responseID responseId
+ @param appProgress appProgress
+ @param resourceAsk resourceAsk
+ @param containersToBeReleased containersToBeReleased
+ @param resourceBlacklistRequest resourceBlacklistRequest
+ @param increaseRequests increaseRequests
+ @return AllocateRequest]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="updateRequests" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>response id</em> used to track duplicate responses.
+ @return <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="setResponseId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>response id</em> used to track duplicate responses.
+ @param id <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current progress</em> of application. 
+ @return <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="setProgress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <em>current progress</em> of application
+ @param progress <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="getAskList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ResourceRequest</code> to update the 
+ <code>ResourceManager</code> about the application's resource requirements.
+ @return the list of <code>ResourceRequest</code>
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="setAskList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of <code>ResourceRequest</code> to update the
+ <code>ResourceManager</code> about the application's resource requirements.
+ @param resourceRequests list of <code>ResourceRequest</code> to update the 
+                        <code>ResourceManager</code> about the application's 
+                        resource requirements
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getReleaseList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> of containers being 
+ released by the <code>ApplicationMaster</code>.
+ @return list of <code>ContainerId</code> of containers being 
+         released by the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setReleaseList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseContainers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of <code>ContainerId</code> of containers being
+ released by the <code>ApplicationMaster</code>
+ @param releaseContainers list of <code>ContainerId</code> of 
+                          containers being released by the 
+                          <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceBlacklistRequest" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ResourceBlacklistRequest</code> being sent by the 
+ <code>ApplicationMaster</code>.
+ @return the <code>ResourceBlacklistRequest</code> being sent by the 
+         <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="setResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ResourceBlacklistRequest</code> to inform the 
+ <code>ResourceManager</code> about the blacklist additions and removals
+ per the <code>ApplicationMaster</code>.
+ 
+ @param resourceBlacklistRequest the <code>ResourceBlacklistRequest</code>  
+                         to inform the <code>ResourceManager</code> about  
+                         the blacklist additions and removals
+                         per the <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="getIncreaseRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use {@link AllocateRequest#getUpdateRequests()} instead
+ @return ContainerResourceIncreaseRequests]]>
+      </doc>
+    </method>
+    <method name="setIncreaseRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="increaseRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Use {@link AllocateRequest#setUpdateRequests(List)} instead
+ @param increaseRequests increaseRequests]]>
+      </doc>
+    </method>
+    <method name="getUpdateRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link UpdateContainerRequest}
+         being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setUpdateRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update requests to inform the
+ <code>ResourceManager</code> about the containers that need to be
+ updated.
+ @param updateRequests list of <code>UpdateContainerRequest</code> for
+                       containers to be updated]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tracking url update for this heartbeat.
+ @return tracking url to update this application with]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the new tracking url for this application.
+ @param trackingUrl the new tracking url]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The core request sent by the <code>ApplicationMaster</code> to the 
+ <code>ResourceManager</code> to obtain resources in the cluster.</p> 
+
+ <p>The request includes:
+ <ul>
+   <li>A response id to track duplicate responses.</li>
+   <li>Progress information.</li>
+   <li>
+     A list of {@link ResourceRequest} to inform the
+     <code>ResourceManager</code> about the application's
+     resource requirements.
+   </li>
+   <li>
+     A list of unused {@link Container} which are being returned.
+   </li>
+   <li>
+     A list of {@link UpdateContainerRequest} to inform
+     the <code>ResourceManager</code> about the change in
+     requirements of running containers.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <class name="AllocateRequest.AllocateRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="responseId" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>responseId</code> of the request.
+ @see AllocateRequest#setResponseId(int)
+ @param responseId <code>responseId</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="progress" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <code>progress</code> of the request.
+ @see AllocateRequest#setProgress(float)
+ @param progress <code>progress</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="askList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="askList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>askList</code> of the request.
+ @see AllocateRequest#setAskList(List)
+ @param askList <code>askList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="releaseList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>releaseList</code> of the request.
+ @see AllocateRequest#setReleaseList(List)
+ @param releaseList <code>releaseList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceBlacklistRequest" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>resourceBlacklistRequest</code> of the request.
+ @see AllocateRequest#setResourceBlacklistRequest(
+ ResourceBlacklistRequest)
+ @param resourceBlacklistRequest
+     <code>resourceBlacklistRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="updateRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>updateRequests</code> of the request.
+ @see AllocateRequest#setUpdateRequests(List)
+ @param updateRequests <code>updateRequests</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="trackingUrl" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>trackingUrl</code> of the request.
+ @see AllocateRequest#setTrackingUrl(String)
+ @param trackingUrl new tracking url
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link AllocateRequest} object.
+ @return {@link AllocateRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link AllocateRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <class name="AllocateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+      <param name="increasedContainers" type="java.util.List"/>
+      <param name="decreasedContainers" type="java.util.List"/>
+      <param name="collectorInfo" type="org.apache.hadoop.yarn.api.records.CollectorInfo"/>
+      <doc>
+      <![CDATA[Use {@link AllocateResponse#newInstance(int, List, List, List, Resource,
+ AMCommand, int, PreemptionMessage, List, Token, List, CollectorInfo)}
+ instead.
+ @param responseId responseId
+ @param completedContainers completedContainers
+ @param allocatedContainers allocatedContainers
+ @param updatedNodes updatedNodes
+ @param availResources availResources
+ @param command command
+ @param numClusterNodes numClusterNodes
+ @param preempt preempt
+ @param nmTokens nmTokens
+ @param increasedContainers increasedContainers
+ @param decreasedContainers decreasedContainers
+ @return AllocateResponse]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+      <param name="updatedContainers" type="java.util.List"/>
+    </method>
+    <method name="getAMCommand" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the <code>ResourceManager</code> needs the
+ <code>ApplicationMaster</code> to take some action then it will send an
+ AMCommand to the <code>ApplicationMaster</code>. See <code>AMCommand</code> 
+ for details on commands and actions for them.
+ @return <code>AMCommand</code> if the <code>ApplicationMaster</code> should
+         take action, <code>null</code> otherwise
+ @see AMCommand]]>
+      </doc>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last response id</em>.
+ @return <em>last response id</em>]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>newly allocated</em> <code>Container</code> by the
+ <code>ResourceManager</code>.
+ @return list of <em>newly allocated</em> <code>Container</code>]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>available headroom</em> for resources in the cluster for the
+ application.
+ @return limit of available headroom for resources in the cluster for the
+ application]]>
+      </doc>
+    </method>
+    <method name="getCompletedContainersStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>completed containers' statuses</em>.
+ @return the list of <em>completed containers' statuses</em>]]>
+      </doc>
+    </method>
+    <method name="getUpdatedNodes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>updated <code>NodeReport</code>s</em>. Updates could
+ be changes in health, availability etc of the nodes.
+ @return The delta of updated nodes since the last response]]>
+      </doc>
+    </method>
+    <method name="getNumClusterNodes" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of hosts available on the cluster.
+ @return the available host count.]]>
+      </doc>
+    </method>
+    <method name="getPreemptionMessage" return="org.apache.hadoop.yarn.api.records.PreemptionMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the description of containers owned by the AM, but requested back by
+ the cluster. Note that the RM may have an inconsistent view of the
+ resources owned by the AM. These messages are advisory, and the AM may
+ elect to ignore them.
+ <p>
+ The message is a snapshot of the resources the RM wants back from the AM.
+ While demand persists, the RM will repeat its request; applications should
+ not interpret each message as a request for <em>additional</em>
+ resources on top of previous messages. Resources requested consistently
+ over some duration may be forcibly killed by the RM.
+
+ @return A specification of the resources to reclaim from this AM.]]>
+      </doc>
+    </method>
+    <method name="getNMTokens" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens required for communicating with NM. New NMTokens
+ issued only if
+ <p>
+ 1) AM is receiving first container on underlying NodeManager.<br>
+ OR<br>
+ 2) NMToken master key rolled over in ResourceManager and AM is getting new
+ container on the same underlying NodeManager.
+ <p>
+ AM will receive one NMToken per NM irrespective of the number of containers
+ issued on same NM. AM is expected to store these tokens until issued a
+ new token for the same NM.
+ @return list of NMTokens required for communicating with NM]]>
+      </doc>
+    </method>
+    <method name="getUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of newly updated containers by
+ <code>ResourceManager</code>.
+ @return list of newly increased containers]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The AMRMToken that belong to this attempt
+
+ @return The AMRMToken that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the application
+
+ @return get application priority]]>
+      </doc>
+    </method>
+    <method name="getCollectorInfo" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The data associated with the collector that belongs to this app. Contains
+ address and token alongwith identification information.
+
+ @return The data of collector that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getUpdateErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+
+ @return List of Update Container Errors.]]>
+      </doc>
+    </method>
+    <method name="setUpdateErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+ @param updateErrors list of <code>UpdateContainerError</code> for
+                       containers updates requests that were in error]]>
+      </doc>
+    </method>
+    <method name="getIncreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use {@link AllocateResponse#getUpdatedContainers()} instead
+ @return null]]>
+      </doc>
+    </method>
+    <method name="getDecreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use {@link AllocateResponse#getUpdatedContainers()} instead
+ @return null]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> the
+ <code>ApplicationMaster</code> during resource negotiation.
+ <p>
+ The response, includes:
+ <ul>
+   <li>Response ID to track duplicate responses.</li>
+   <li>
+     An AMCommand sent by ResourceManager to let the
+     {@code ApplicationMaster} take some actions (resync, shutdown etc.).
+   </li>
+   <li>A list of newly allocated {@link Container}.</li>
+   <li>A list of completed {@link Container}s' statuses.</li>
+   <li>
+     The available headroom for resources in the cluster for the
+     application.
+   </li>
+   <li>A list of nodes whose status has been updated.</li>
+   <li>The number of available nodes in a cluster.</li>
+   <li>A description of resources requested back by the cluster</li>
+   <li>AMRMToken, if AMRMToken has been rolled over</li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been increased.
+   </li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been decreased.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <class name="ApplicationsRequestScope" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration that controls the scope of applications fetched]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <class name="CommitResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommitResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to Commit Container Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <class name="ContainerUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToUpdate" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource update.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource update.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToUpdate" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getUpdatedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource update.
+ @param containersToUpdate the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <class name="ContainerUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="successfullyUpdatedContainers" type="java.util.List"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getSuccessfullyUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully update.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully updated.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to update container resource.
+ </p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <class name="FailApplicationAttemptRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the attempt to be failed.
+ @return <code>ApplicationAttemptId</code> of the attempt.]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to fail an application attempt.</p>
+
+ <p>The request includes the {@link ApplicationAttemptId} of the attempt to
+ be failed.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <class name="FailApplicationAttemptResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ failing an application attempt.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <class name="FinishApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="url" type="java.lang.String"/>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>final state</em> of the <code>ApplicationMaster</code>.
+ @return <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setFinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalState" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <doc>
+      <![CDATA[Set the <em>final state</em> of the <code>ApplicationMaster</code>
+ @param finalState <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic information</em> on application failure.
+ @return <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <em>diagnostic information</em> on application failure.
+ @param diagnostics <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ This is the web-URL to which ResourceManager or web-application proxy will
+ redirect client/users once the application is finished and the
+ <code>ApplicationMaster</code> is gone.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param url
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The finalization request sent by the {@code ApplicationMaster} to
+ inform the {@code ResourceManager} about its completion.
+ <p>
+ The final request includes details such:
+ <ul>
+   <li>Final state of the {@code ApplicationMaster}</li>
+   <li>
+     Diagnostic information in case of failure of the
+     {@code ApplicationMaster}
+   </li>
+   <li>Tracking URL</li>
+ </ul>
+
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <class name="FinishApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsUnregistered" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the application has successfully
+ unregistered with the RM and the application can safely stop.
+ @return true if the application has unregistered with the RM,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a
+ <code>ApplicationMaster</code> on it's completion.
+ <p>
+ The response, includes:
+ <ul>
+ <li>A flag which indicates that the application has successfully unregistered
+ with the RM and the application can safely stop.</li>
+ </ul>
+ <p>
+ Note: The flag indicates whether the application has successfully
+ unregistered and is safe to stop. The application may stop after the flag is
+ true. If the application stops before the flag is true then the RM may retry
+ the application.
+ 
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <class name="GetAllResourceTypeInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <class name="GetAllResourceTypeInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceTypeInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceTypes" type="java.util.List"/>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <class name="GetApplicationAttemptReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ApplicationAttemptReport} for an application attempt.
+ </p>
+ 
+ <p>
+ The request should include the {@link ApplicationAttemptId} of the
+ application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <class name="GetApplicationAttemptReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ApplicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @return <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @param applicationAttemptReport
+          <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ an application attempt report.
+ </p>
+ 
+ <p>
+ The response includes an {@link ApplicationAttemptReport} which has the
+ details about the particular application attempt
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <class name="GetApplicationAttemptsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of an application
+ 
+ @return <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of an application
+ 
+ @param applicationId
+          <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of application attempt reports of an
+ application from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <class name="GetApplicationAttemptsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+    </method>
+    <method name="getApplicationAttemptList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @return a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @param applicationAttempts
+          a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ApplicationAttemptReport} for application attempts.
+ </p>
+ 
+ <p>
+ The <code>ApplicationAttemptReport</code> for each application includes the
+ details of an application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <class name="GetApplicationReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to the <code>ResourceManager</code> to 
+ get an {@link ApplicationReport} for an application.</p>
+ 
+ <p>The request should include the {@link ApplicationId} of the 
+ application.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)
+ @see ApplicationReport]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <class name="GetApplicationReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationReport</code> for the application.
+ @return <code>ApplicationReport</code> for the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an application report.</p>
+ 
+ <p>The response includes an {@link ApplicationReport} which has details such 
+ as user, queue, name, host on which the <code>ApplicationMaster</code> is 
+ running, RPC port, tracking URL, diagnostics, start time etc.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <class name="GetApplicationsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="queues" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="startRange" type="org.apache.commons.lang.math.LongRange"/>
+      <param name="finishRange" type="org.apache.commons.lang.math.LongRange"/>
+      <param name="limit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+
+ <p>Setting any of the parameters to null, would just disable that
+ filter</p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @param users list of users to filter by
+ @param queues list of scheduler queues to filter by
+ @param applicationTypes types of applications
+ @param applicationTags application tags to filter by
+ @param applicationStates application states to filter by
+ @param startRange range of application start times to filter by
+ @param finishRange range of application finish times to filter by
+ @param limit number of applications to limit to
+ @return {@link GetApplicationsRequest} to be used with
+ {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application states in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving and application types and application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in <code>GetApplicationsRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getApplicationTypes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application types to filter applications on
+
+ @return Set of Application Types to filter on]]>
+      </doc>
+    </method>
+    <method name="getApplicationStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application states to filter applications on
+
+ @return Set of Application states to filter on]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of Applications
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <class name="GetApplicationsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationReport</code> for applications.
+ @return <code>ApplicationReport</code> for applications]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an {@link ApplicationReport} for applications.</p>
+
+ <p>The <code>ApplicationReport</code> for each application includes details
+ such as user, queue, name, host on which the <code>ApplicationMaster</code>
+ is running, RPC port, tracking URL, diagnostics, start time etc.</p>
+
+ @see ApplicationReport
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <class name="GetClusterMetricsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get cluster metrics from the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>Currently, this is empty.</p>
+
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <class name="GetClusterMetricsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnClusterMetrics</code> for the cluster.
+ @return <code>YarnClusterMetrics</code> for the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a client
+ requesting cluster metrics.
+ 
+ @see YarnClusterMetrics
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <class name="GetClusterNodeLabelsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <class name="GetClusterNodeLabelsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #newInstance(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Creates a new instance.
+
+ @param labels Node labels
+ @return response
+ @deprecated Use {@link #newInstance(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="setNodeLabelList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="getNodeLabelList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNodeLabels"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setNodeLabelList(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node labels to the response.
+
+ @param labels Node labels
+ @deprecated Use {@link #setNodeLabelList(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getNodeLabelList()} instead.">
+      <doc>
+      <![CDATA[Get node labels of the response.
+
+ @return Node labels
+ @deprecated Use {@link #getNodeLabelList()} instead.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <class name="GetClusterNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @return the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <method name="setNodeStates"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @param states the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ The request will ask for all nodes in the given {@link NodeState}s.
+
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <class name="GetClusterNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>NodeReport</code> for all nodes in the cluster.
+ @return <code>NodeReport</code> for all nodes in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting a {@link NodeReport} for all nodes.</p>
+ 
+ <p>The <code>NodeReport</code> contains per-node information such as 
+ available resources, number of containers, tracking url, rack name, health
+ status etc.
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <class name="GetContainerReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the Container.
+ 
+ @return <code>ContainerId</code> of the Container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container
+ 
+ @param containerId
+          <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ContainerReport} for a container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <class name="GetContainerReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerReport</code> for the container.
+ 
+ @return <code>ContainerReport</code> for the container]]>
+      </doc>
+    </method>
+    <method name="setContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a container report.
+ </p>
+ 
+ <p>
+ The response includes a {@link ContainerReport} which has details of a
+ container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <class name="GetContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of container reports, which belong to
+ an application attempt from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <class name="GetContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+    </method>
+    <method name="getContainerList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @return a list of <code>ContainerReport</code> for all the containers of an
+         application attempt]]>
+      </doc>
+    </method>
+    <method name="setContainerList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @param containers
+          a list of <code>ContainerReport</code> for all the containers of
+          an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ContainerReport} for containers.
+ </p>
+ 
+ <p>
+ The <code>ContainerReport</code> for each container includes the container
+ details.
+ </p>
+ 
+ @see ContainerReport
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <class name="GetContainerStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>.
+ 
+ @return the list of <code>ContainerId</code>s of containers for which to
+         obtain the <code>ContainerStatus</code>.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>
+ 
+ @param containerIds
+          a list of <code>ContainerId</code>s of containers for which to
+          obtain the <code>ContainerStatus</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to get {@link ContainerStatus} of requested
+ containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <class name="GetContainerStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerStatus</code>es of the requested containers.
+ 
+ @return <code>ContainerStatus</code>es of the requested containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to obtain the
+ <code>ContainerStatus</code> of requested containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <class name="GetDelegationTokenRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to get a delegation token from
+ the {@code ResourceManager}.
+ for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <class name="GetDelegationTokenResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Delegation tokens have a identifier which maps to
+ {@link AbstractDelegationTokenIdentifier}.
+ @return the delegation tokens]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Response to a {@link GetDelegationTokenRequest} request 
+ from the client. The response contains the token that 
+ can be used by the containers to talk to  ClientRMService.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <class name="GetNewApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@link ApplicationId} for
+ submitting an application.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <class name="GetNewApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>new</em> <code>ApplicationId</code> allocated by the 
+ <code>ResourceManager</code>.
+ @return <em>new</em> <code>ApplicationId</code> allocated by the 
+          <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for 
+ a request to get a new {@link ApplicationId} for submitting applications.</p>
+ 
+ <p>Clients can submit an application with the returned
+ {@link ApplicationId}.</p>
+
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <class name="GetNewReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@code ReservationId} for
+ submitting an reservation.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <class name="GetNewReservationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a new {@link ReservationId} to be used to submit a reservation.
+
+ @return a {@link ReservationId} representing the unique id to identify
+ a reservation with which it was submitted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for
+ a request to get a new {@link ReservationId} for submitting reservations.</p>
+
+ <p>Clients can submit an reservation with the returned
+ {@link ReservationId}.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <class name="GetQueueInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <param name="includeApplications" type="boolean"/>
+      <param name="includeChildQueues" type="boolean"/>
+      <param name="recursive" type="boolean"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> for which to get queue information.
+ @return <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue name</em> for which to get queue information
+ @param queueName <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="getIncludeApplications" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>active applications</em> required?
+ @return <code>true</code> if applications' information is to be included,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeApplications"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeApplications" type="boolean"/>
+      <doc>
+      <![CDATA[Should we get fetch information about <em>active applications</em>?
+ @param includeApplications fetch information about <em>active 
+                            applications</em>?]]>
+      </doc>
+    </method>
+    <method name="getIncludeChildQueues" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>child queues</em> required?
+ @return <code>true</code> if information about child queues is required,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeChildQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeChildQueues" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information about <em>child queues</em>?
+ @param includeChildQueues fetch information about <em>child queues</em>?]]>
+      </doc>
+    </method>
+    <method name="getRecursive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information on the entire <em>child queue hierarchy</em> required?
+ @return <code>true</code> if information about entire hierarchy is 
+         required, <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setRecursive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information on the entire <em>child queue hierarchy</em>?
+ @param recursive fetch information on the entire <em>child queue 
+                  hierarchy</em>?]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get <em>queue information</em>
+ from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <class name="GetQueueInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueInfo</code> for the specified queue.
+ @return <code>QueueInfo</code> for the specified queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a client
+ requesting information about queues in the system.
+ <p>
+ The response includes a {@link QueueInfo} which has details such as
+ queue name, used/total capacities, running applications, child queues etc.
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <class name="GetQueueUserAclsInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to the <code>ResourceManager</code> to 
+ get queue acls for the <em>current user</em>.</p>
+
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <class name="GetQueueUserAclsInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUserAclsInfoList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueUserACLInfo</code> per queue for the user.
+ @return <code>QueueUserACLInfo</code> per queue for the user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to clients
+ seeking queue acls for the user.</p>
+
+ <p>The response contains a list of {@link QueueUserACLInfo} which
+ provides information about {@link QueueACL} per queue.</p>
+ 
+ @see QueueACL
+ @see QueueUserACLInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <class name="IncreaseContainersResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToIncrease" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource increase.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource increase.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToIncrease"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getIncreasedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource increase.
+ @param containersToIncrease the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <class name="IncreaseContainersResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyIncreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully increased.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully increased.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to increase container resource.
+ </p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <class name="KillApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be aborted.
+ @return <code>ApplicationId</code> of the application to be aborted]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostics</em> to which the application is being killed.
+ @return <em>diagnostics</em> to which the application is being killed]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>diagnostics</em> to which the application is being killed.
+ @param diagnostics <em>diagnostics</em> to which the application is being
+          killed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to abort a submitted application.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ aborted.</p>
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <class name="KillApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsKillCompleted" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the process of killing application is completed or not.
+ @return true if the process of killing application has completed,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to the client aborting
+ a submitted application.
+ <p>
+ The response, includes:
+ <ul>
+   <li>
+     A flag which indicates that the process of killing the application is
+     completed or not.
+   </li>
+ </ul>
+ Note: user is recommended to wait until this flag becomes true, otherwise if
+ the <code>ResourceManager</code> crashes before the process of killing the
+ application is completed, the <code>ResourceManager</code> may retry this
+ application on recovery.
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <class name="MoveApplicationAcrossQueuesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be moved.
+ @return <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application to be moved.
+ @param appId <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="getTargetQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @return the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <method name="setTargetQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @param queue the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to move a submitted application to a different queue.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ moved and the queue to place it in.</p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <class name="MoveApplicationAcrossQueuesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client moving
+ a submitted application to a different queue.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <class name="RegisterApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of <code>RegisterApplicationMasterRequest</code>.
+ If <em>port, trackingUrl</em> is not used, use the following default value:
+ <ul>
+  <li>port: -1</li>
+  <li>trackingUrl: null</li>
+ </ul>
+ The port is allowed to be any integer larger than or equal to -1.
+ @return the new instance of <code>RegisterApplicationMasterRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @param host <em>host</em> on which the <code>ApplicationMaster</code> 
+             is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @return the <em>RPC port</em> on which the {@code ApplicationMaster}
+         is responding]]>
+      </doc>
+    </method>
+    <method name="setRpcPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @param port <em>RPC port</em> on which the {@code ApplicationMaster}
+             is responding]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em> for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ it is running. This is the web-URL to which ResourceManager or
+ web-application proxy will redirect client/users while the application and
+ the <code>ApplicationMaster</code> are still running.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param trackingUrl
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
+ on registration.
+ <p>
+ The registration includes details such as:
+ <ul>
+   <li>Hostname on which the AM is running.</li>
+   <li>RPC Port</li>
+   <li>Tracking URL</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <class name="RegisterApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="getClientToAMTokenMasterKey" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get ClientToAMToken master key.</p>
+ <p>The ClientToAMToken master key is sent to <code>ApplicationMaster</code>
+ by <code>ResourceManager</code> via {@link RegisterApplicationMasterResponse}
+ , used to verify corresponding ClientToAMToken.</p>
+ @return ClientToAMToken master key]]>
+      </doc>
+    </method>
+    <method name="setClientToAMTokenMasterKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set ClientToAMToken master key.]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the queue that the application was placed in.<p>
+ @return the queue that the application was placed in.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>Set the queue that the application was placed in.<p>]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts.
+ </p>
+ 
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts
+ @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getNMTokensFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens for communicating with the NMs where the
+ containers of previous application attempts are running.
+ 
+ @return the list of NMTokens for communicating with the NMs where the
+         containers of previous application attempts are running.
+ 
+ @see RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getSchedulerResourceTypes" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a set of the resource types considered by the scheduler.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <method name="getResourceTypes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get available resource types supported by RM.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a new
+ {@code ApplicationMaster} on registration.
+ <p>
+ The response contains critical details such as:
+ <ul>
+   <li>Maximum capability for allocated resources in the cluster.</li>
+   <li>{@code ApplicationACL}s for the application.</li>
+   <li>ClientToAMToken master key.</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <class name="ReInitializeContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[Creates a new instance of the ReInitializationContainerRequest.
+ @param containerId Container Id.
+ @param containerLaunchContext Container Launch Context.
+ @param autoCommit AutoCommit.
+ @return ReInitializationContainerRequest.]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to re-initialize.
+
+ @return <code>ContainerId</code> of the container to re-initialize.]]>
+      </doc>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to re-initialize the container
+ with.
+
+ @return <code>ContainerLaunchContext</code> of to re-initialize the
+ container with.]]>
+      </doc>
+    </method>
+    <method name="getAutoCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if AutoCommit is set for this ReInitialization.
+ @return If AutoCommit is set for this ReInitialization.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates all the required fields needed for a Container
+ ReInitialization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <class name="ReInitializeContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ReInitializeContainerRequest}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <class name="ReleaseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be released.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be released.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be released.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be released.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to release a resource in the shared cache.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <class name="ReleaseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response to clients from the <code>SharedCacheManager</code> when
+ releasing a resource in the shared cache.
+ </p>
+
+ <p>
+ Currently, this is empty.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <class name="ReservationDeleteRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDeleteRequest} captures the set of requirements the user
+ has to delete an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <class name="ReservationDeleteResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationDeleteResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation delete
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <class name="ReservationListRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+ @param endTime Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via  {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name to use to find reservations.
+
+ @return the queue name to use to find reservations.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name to use to find resource allocations.
+
+ @param queue Required. Cannot be null or empty.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id to use to find a reservation.
+
+ @return the reservation id of the reservation.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reservation id to use to find a reservation.
+
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @return the start time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setEndTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+      <doc>
+      <![CDATA[Set the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @param endTime Optional. If provided, only reservations that
+                start before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getIncludeResourceAllocations" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setIncludeResourceAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[Set the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire list of
+                {@code ResourceAllocationRequest} will be returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListRequest} captures the set of requirements the
+ user has to list reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <class name="ReservationListResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationAllocationState" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationAllocationState}, that corresponds
+ to a reservation in the scheduler.
+
+ @return the list of {@link ReservationAllocationState} which holds
+ information of a particular reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListResponse} captures the list of reservations that the
+ user has queried.
+
+ The resulting list of {@link ReservationAllocationState} contains a list of
+ {@code ResourceAllocationRequest} representing the current state of the
+ reservation resource allocations will be returned. This is subject to change
+ in the event of re-planning a described by {@code ReservationDefinition}
+
+ @see ReservationAllocationState]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <class name="ReservationSubmissionRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to.
+ 
+ @return the name of the {@code Plan} that corresponds to the name of the
+         {@link QueueInfo} in the scheduler to which the reservation will be
+         submitted to]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to
+ 
+ @param queueName the name of the parent {@code Plan} that corresponds to
+          the name of the {@link QueueInfo} in the scheduler to which the
+          reservation will be submitted to]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id that corresponds to the reservation submission.
+
+ @return reservation id that will be used to identify the reservation
+ submission.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id that corresponds to the reservation submission.
+
+ @param reservationId reservation id that will be used to identify the
+                      reservation submission.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationSubmissionRequest} captures the set of requirements the
+ user has to create a reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <class name="ReservationSubmissionResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ reservation submission.</p>
+
+ <p>Currently, this is empty.</p>
+
+ {@code ApplicationClientProtocol#submitReservation(
+ ReservationSubmissionRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <class name="ReservationUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationUpdateRequest} captures the set of requirements the user
+ has to update an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <class name="ReservationUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationUpdateResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation update
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <class name="ResourceLocalizationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="localResources" type="java.util.Map"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to localize resources.
+
+ @return <code>ContainerId</code> of the container to localize resources.]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the ApplicationMaster to ask for localizing resources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <class name="ResourceLocalizationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ResourceLocalizationRequest}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <class name="RestartContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RestartContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to a restart Container request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <class name="RollbackResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RollbackResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to a Rollback request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <class name="SignalContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="signalContainerCommand" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to signal.
+ @return <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="getCommand" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>SignalContainerCommand</code> of the signal request.
+ @return <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <method name="setCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <doc>
+      <![CDATA[Set the <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ or by the <code>ApplicationMaster</code> to the <code>NodeManager</code>
+ to signal a container.
+ @see SignalContainerCommand </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <class name="SignalContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ signalling a container.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <class name="StartContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>.
+ 
+ @return <code>ContainerLaunchContext</code> for the container to be started
+         by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>
+ @param context <code>ContainerLaunchContext</code> for the container to be 
+                started by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the container token to be used for authorization during starting
+ container.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the container token to be used for authorization during starting
+ container.
+ @see NMToken
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="setContainerToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>start</em> a container.</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as
+ allocated resource capability, security tokens (if enabled), command
+ to be executed to start the container, environment for the process, 
+ necessary binaries/jar/shared-objects etc. via the 
+ {@link ContainerLaunchContext}.</p>
+
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <class name="StartContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+    </method>
+    <method name="getStartContainerRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link StartContainerRequest} to start containers.
+ @return a list of {@link StartContainerRequest} to start containers.]]>
+      </doc>
+    </method>
+    <method name="setStartContainerRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of {@link StartContainerRequest} to start containers.
+ @param request a list of {@link StartContainerRequest} to start containers]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request which contains a list of {@link StartContainerRequest} sent by
+ the <code>ApplicationMaster</code> to the <code>NodeManager</code> to
+ <em>start</em> containers.
+ </p>
+ 
+ <p>
+ In each {@link StartContainerRequest}, the <code>ApplicationMaster</code> has
+ to provide details such as allocated resource capability, security tokens (if
+ enabled), command to be executed to start the container, environment for the
+ process, necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <class name="StartContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStartedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> s of the containers that are
+ started successfully.
+ 
+ @return the list of <code>ContainerId</code> s of the containers that are
+         started successfully.
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <method name="getAllServicesMetaData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the meta-data from all auxiliary services running on the
+ <code>NodeManager</code>.
+ </p>
+ <p>
+ The meta-data is returned as a Map between the auxiliary service names and
+ their corresponding per service meta-data as an opaque blob
+ <code>ByteBuffer</code>
+ </p>
+ 
+ <p>
+ To be able to interpret the per-service meta-data, you should consult the
+ documentation for the Auxiliary-service configured on the NodeManager
+ </p>
+ 
+ @return a Map between the names of auxiliary services and their
+         corresponding meta-data]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>start</em> an allocated
+ container.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <class name="StopContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code>s of the containers to be stopped.
+ @return <code>ContainerId</code>s of containers to be stopped]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code>s of the containers to be stopped.
+ @param containerIds <code>ContainerId</code>s of the containers to be stopped]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>stop</em> containers.</p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <class name="StopContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStoppedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of successfully stopped containers.
+ 
+ @return the list of containerIds of successfully stopped containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>stop</em> allocated
+ containers.
+ </p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <class name="SubmitApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationSubmissionContext</code> for the application.
+ @return <code>ApplicationSubmissionContext</code> for the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationSubmissionContext</code> for the application.
+ @param context <code>ApplicationSubmissionContext</code> for the 
+                application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to <em>submit an application</em> to the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>The request, via {@link ApplicationSubmissionContext}, contains
+ details such as queue, {@link Resource} required to run the 
+ <code>ApplicationMaster</code>, the equivalent of 
+ {@link ContainerLaunchContext} for launching the 
+ <code>ApplicationMaster</code> etc.
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <class name="SubmitApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ application submission.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <class name="UpdateApplicationPriorityRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ 
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ 
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ 
+ @return <code>Priority</code> of the application to be set.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+ 
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application priority.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and
+ {@link Priority} to be set for an application
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <class name="UpdateApplicationPriorityResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ @return Updated <code>Priority</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ the application priority.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <class name="UpdateApplicationTimeoutsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationTimeouts" type="java.util.Map"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application timeout.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and timeout
+ to be set for an application
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <class name="UpdateApplicationTimeoutsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ application timeout.
+ </p>
+ <p>
+ A response without exception means that the update has completed
+ successfully.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <class name="UseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be used.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be used.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be used.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be used.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to the <code>SharedCacheManager</code> that claims a
+ resource in the shared cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+  <class name="UseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Path</code> corresponding to the requested resource in the
+ shared cache.
+
+ @return String A <code>Path</code> if the resource exists in the shared
+         cache, <code>null</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setPath"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>Path</code> corresponding to a resource in the shared cache.
+
+ @param p A <code>Path</code> corresponding to a resource in the shared
+          cache]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response from the SharedCacheManager to the client that indicates whether
+ a requested resource exists in the cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records">
+  <!-- start class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <class name="AMCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.AMCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Command sent by the Resource Manager to the Application Master in the 
+ AllocateResponse 
+ @see AllocateResponse]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <class name="ApplicationAccessType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application access types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <class name="ApplicationAttemptId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationAttemptId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="attemptId" type="int"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the <code>ApplicationAttempId</code>. 
+ @return <code>ApplicationId</code> of the <code>ApplicationAttempId</code>]]>
+      </doc>
+    </method>
+    <method name="getAttemptId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>attempt id</code> of the <code>Application</code>.
+ @return <code>attempt id</code> of the <code>Application</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <field name="appAttemptIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
+ of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
+ 
+ <p>Multiple attempts might be needed to run an application to completion due
+ to temporal failures of the <code>ApplicationMaster</code> such as hardware
+ failures, connectivity issues etc. on the node on which it was scheduled.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <class name="ApplicationAttemptReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="rpcPort" type="int"/>
+      <param name="url" type="java.lang.String"/>
+      <param name="oUrl" type="java.lang.String"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"/>
+      <param name="amContainerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getYarnApplicationAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>YarnApplicationAttemptState</em> of the application attempt.
+ 
+ @return <em>YarnApplicationAttemptState</em> of the application attempt]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of this attempt <code>ApplicationMaster</code>.
+ 
+ @return <em>RPC port</em> of this attempt <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which this attempt of
+ <code>ApplicationMaster</code> is running.
+ 
+ @return <em>host</em> on which this attempt of
+         <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnositic information</em> of the application attempt in case
+ of errors.
+ 
+ @return <em>diagnositic information</em> of the application attempt in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application attempt.
+ 
+ @return <em>tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>original tracking url</em> for the application attempt.
+ 
+ @return <em>original tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of this attempt of the
+ application
+ 
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getAMContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of AMContainer for this attempt
+ 
+ @return <code>ContainerId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ 
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationAttemptReport} is a report of an application attempt.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationAttemptId} of the application.</li>
+   <li>Host on which the <code>ApplicationMaster</code> of this attempt is
+   running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code> of this attempt.</li>
+   <li>Tracking URL.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>{@link YarnApplicationAttemptState} of the application attempt.</li>
+   <li>{@link ContainerId} of the master Container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <class name="ApplicationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clusterTimestamp" type="long"/>
+      <param name="id" type="int"/>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the short integer identifier of the <code>ApplicationId</code>
+ which is unique for all applications started by a particular instance
+ of the <code>ResourceManager</code>.
+ @return short integer identifier of the <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the <code>ResourceManager</code> which is 
+ used to generate globally unique <code>ApplicationId</code>.
+ @return <em>start time</em> of the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="appIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationId</code> represents the <em>globally unique</em> 
+ identifier for an application.</p>
+ 
+ <p>The globally unique nature of the identifier is achieved by using the 
+ <em>cluster timestamp</em> i.e. start-time of the 
+ <code>ResourceManager</code> along with a monotonically increasing counter
+ for the application.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <class name="ApplicationReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getCurrentApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the current
+ attempt of the application
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>user</em> who submitted the application.
+ @return <em>user</em> who submitted the application]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application was submitted.
+ @return <em>queue</em> to which the application was submitted]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <em>name</em> of the application.
+ @return <em>name</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ is running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code>
+         is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of the <code>ApplicationMaster</code>.
+ @return <em>RPC port</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getClientToAMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>.
+ <p>
+ <em>ClientToAMToken</em> is the security token used by the AMs to verify
+ authenticity of any <code>client</code>.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code>, provides a secure token (via
+ {@link ApplicationReport#getClientToAMToken()}) which is verified by the
+ ApplicationMaster when the client directly talks to an AM.
+ </p>
+ @return <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getYarnApplicationState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnApplicationState</code> of the application.
+ @return <code>YarnApplicationState</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get  the <em>diagnositic information</em> of the application in case of
+ errors.
+ @return <em>diagnositic information</em> of the application in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application.
+ @return <em>tracking url</em> for the application]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the application.
+ @return <em>start time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getSubmitTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>final finish status</em> of the application.
+ @return <em>final finish status</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationResourceUsageReport" return="org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve the structure containing the job resources for this application
+ @return the job resources structure for this application]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's progress ( range 0.0 to 1.0 )
+ @return application's progress]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's Type 
+ @return application's Type]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all tags corresponding to the application
+ @return Application's tags]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters Yarn takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+ 
+ @return the AM to RM token if available.]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationStatus" return="org.apache.hadoop.yarn.api.records.LogAggregationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get log aggregation status for the application
+ @return Application's log aggregation status]]>
+      </doc>
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedApp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="unmanagedApplication" type="boolean"/>
+      <doc>
+      <![CDATA[@param unmanagedApplication true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get priority of the application
+
+ @return Application's priority]]>
+      </doc>
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAppNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAmNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationReport} is a report of an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Applications user.</li>
+   <li>Application queue.</li>
+   <li>Application name.</li>
+   <li>Host on which the <code>ApplicationMaster</code> is running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code>.</li>
+   <li>Tracking URL.</li>
+   <li>{@link YarnApplicationState} of the application.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Start time of the application.</li>
+   <li>Client {@link Token} of the application (if security is enabled).</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <class name="ApplicationResourceUsageReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationResourceUsageReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumUsedContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of used containers.  -1 for invalid/inaccessible reports.
+ @return the number of used containers]]>
+      </doc>
+    </method>
+    <method name="getUsedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the used <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the used <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getReservedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the reserved <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getNeededResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the needed <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the needed <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory (in megabytes) the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores that the application has allocated
+ times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getQueueUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the queue that the app is using.
+ @return the percentage of resources of the queue that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getClusterUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the cluster that the app is using.
+ @return the percentage of resources of the cluster that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getPreemptedMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory preempted(in megabytes)
+ the application has allocated times the number of
+ seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated resource-seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated preempted
+ resource-seconds]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Contains various scheduling metrics to be reported by UI and CLI.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <class name="ApplicationSubmissionContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="amContainerLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="resourceRequest" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="attemptFailuresValidityInterval" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the submitted application.
+ @return <code>ApplicationId</code> of the submitted application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the submitted application.
+ @param applicationId <code>ApplicationId</code> of the submitted
+                      application]]>
+      </doc>
+    </method>
+    <method name="getApplicationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application <em>name</em>.
+ @return application name]]>
+      </doc>
+    </method>
+    <method name="setApplicationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application <em>name</em>.
+ @param applicationName application name]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application is being submitted.
+ @return <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue</em> to which the application is being submitted
+ @param queue <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application.
+ @return <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getAMContainerSpec" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @return <code>ContainerLaunchContext</code> for the 
+         <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerSpec"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @param amContainer <code>ContainerLaunchContext</code> for the 
+                    <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="getUnmanagedAM" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get if the RM should manage the execution of the AM. 
+ If true, then the RM 
+ will not allocate a container for the AM and start it. It will expect the 
+ AM to be launched and connect to the RM within the AM liveliness period and 
+ fail the app otherwise. The client should launch the AM only after the RM 
+ has ACCEPTED the application and changed the <code>YarnApplicationState</code>.
+ Such apps will not be retried by the RM on app attempt failure.
+ The default value is false.
+ @return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedAM"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[@param value true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getMaxAppAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of max attempts of the application to be submitted]]>
+      </doc>
+    </method>
+    <method name="setMaxAppAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxAppAttempts" type="int"/>
+      <doc>
+      <![CDATA[Set the number of max attempts of the application to be submitted. WARNING:
+ it should be no larger than the global number of max attempts in the Yarn
+ configuration.
+ @param maxAppAttempts the number of max attempts of the application
+ to be submitted.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource required by the <code>ApplicationMaster</code> for this
+ application. Please note this will be DEPRECATED, use <em>getResource</em>
+ in <em>getAMContainerResourceRequest</em> instead.
+ 
+ @return the resource required by the <code>ApplicationMaster</code> for
+         this application.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resource required by the <code>ApplicationMaster</code> for this
+ application.
+
+ @param resource the resource required by the <code>ApplicationMaster</code>
+ for this application.]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application type
+ 
+ @return the application type]]>
+      </doc>
+    </method>
+    <method name="setApplicationType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application type
+ 
+ @param applicationType the application type]]>
+      </doc>
+    </method>
+    <method name="getKeepContainersAcrossApplicationAttempts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates whether to keep containers across application
+ attempts or not.
+ 
+ @return the flag which indicates whether to keep containers across
+         application attempts or not.]]>
+      </doc>
+    </method>
+    <method name="setKeepContainersAcrossApplicationAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keepContainers" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which indicates whether to keep containers across application
+ attempts.
+ <p>
+ For managed AM, if the flag is true, running containers will not be killed
+ when application attempt fails and these containers will be retrieved by
+ the new application attempt on registration via
+ {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
+ </p>
+ <p>
+ For unmanaged AM, if the flag is true, RM allows re-register and returns
+ the running containers in the same attempt back to the UAM for HA.
+ </p>
+
+ @param keepContainers the flag which indicates whether to keep containers
+          across application attempts.]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get tags for the application
+
+ @return the application tags]]>
+      </doc>
+    </method>
+    <method name="setApplicationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set tags for the application. A maximum of
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
+ per application. Each tag can be at most
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
+ characters, and can contain only ASCII characters.
+
+ @param tags tags to set]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this app. If this is set, all containers of
+ this application without setting node-label-expression in ResurceRequest
+ will get allocated resources on only those nodes that satisfy this
+ node-label-expression.
+ 
+ If different node-label-expression of this app and ResourceRequest are set
+ at the same time, the one set in ResourceRequest will be used when
+ allocating container
+ 
+ @return node-label-expression for this app]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node-label-expression for this app
+ @param nodeLabelExpression node-label-expression of this app]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #getAMContainerResourceRequests()}">
+      <doc>
+      <![CDATA[Get the ResourceRequest of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will assemble a ResourceRequest by using
+ <em>getResource</em> and <em>getPriority</em> of
+ <em>ApplicationSubmissionContext</em>.
+
+ Number of containers and Priority will be ignored.
+
+ @return ResourceRequest of the AM container
+ @deprecated See {@link #getAMContainerResourceRequests()}]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #setAMContainerResourceRequests(List)}">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Set ResourceRequest of the AM container
+ @param request of the AM container
+ @deprecated See {@link #setAMContainerResourceRequests(List)}]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ResourceRequests of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will use the ResourceRequest as determined by
+ <em>getAMContainerResourceRequest</em> and its behavior.
+
+ Number of containers and Priority will be ignored.
+
+ @return List of ResourceRequests of the AM container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set ResourceRequests of the AM container.
+ @param requests of the AM container]]>
+      </doc>
+    </method>
+    <method name="getAttemptFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attemptFailuresValidityInterval in milliseconds for the application
+
+ @return the attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="setAttemptFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptFailuresValidityInterval" type="long"/>
+      <doc>
+      <![CDATA[Set the attemptFailuresValidityInterval in milliseconds for the application
+ @param attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LogAggregationContext</code> of the application
+
+ @return <code>LogAggregationContext</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+      <doc>
+      <![CDATA[Set <code>LogAggregationContext</code> for the application
+
+ @param logAggregationContext
+          for the application]]>
+      </doc>
+    </method>
+    <method name="getReservationID" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id, that corresponds to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @return the reservation id representing the unique id of the corresponding
+         reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationID" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id, that correspond to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @param reservationID representing the unique id of the
+          corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in seconds.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application in seconds.
+ All pre-existing Map entries are cleared before adding the new Map.
+ <p>
+ <b>Note:</b> If application timeout value is less than or equal to zero
+ then application submission will throw an exception.
+ </p>
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationSubmissionContext} represents all of the
+ information needed by the {@code ResourceManager} to launch
+ the {@code ApplicationMaster} for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Application user.</li>
+   <li>Application name.</li>
+   <li>{@link Priority} of the application.</li>
+   <li>
+     {@link ContainerLaunchContext} of the container in which the
+     <code>ApplicationMaster</code> is executed.
+   </li>
+   <li>
+     maxAppAttempts. The maximum number of application attempts.
+     It should be no larger than the global number of max attempts in the
+     Yarn configuration.
+   </li>
+   <li>
+     attemptFailuresValidityInterval. The default value is -1.
+     when attemptFailuresValidityInterval in milliseconds is set to
+     {@literal >} 0, the failure number will no take failures which happen
+     out of the validityInterval into failure count. If failure count
+     reaches to maxAppAttempts, the application will be failed.
+   </li>
+   <li>Optional, application-specific {@link LogAggregationContext}</li>
+ </ul>
+ 
+ @see ContainerLaunchContext
+ @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <class name="ApplicationTimeout" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTimeout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <param name="expiryTime" type="java.lang.String"/>
+      <param name="remainingTime" type="long"/>
+    </method>
+    <method name="getTimeoutType" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application timeout type.
+ @return timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="setTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeoutType" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <doc>
+      <![CDATA[Set the application timeout type.
+ @param timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="getExpiryTime" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>expiryTime</code> for given timeout type.
+ @return expiryTime in ISO8601 standard with format
+         <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="setExpiryTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expiryTime" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <code>expiryTime</code> for given timeout type.
+ @param expiryTime in ISO8601 standard with format
+          <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="getRemainingTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>Remaining Time</code> of an application for given timeout type.
+ @return Remaining Time in seconds.]]>
+      </doc>
+    </method>
+    <method name="setRemainingTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="remainingTime" type="long"/>
+      <doc>
+      <![CDATA[Set <code>Remaining Time</code> of an application for given timeout type.
+ @param remainingTime in seconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationTimeout} is a report for configured application timeouts.
+ It includes details such as:
+ <ul>
+ <li>{@link ApplicationTimeoutType} of the timeout type.</li>
+ <li>Expiry time in ISO8601 standard with format
+ <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b> or "UNLIMITED".</li>
+ <li>Remaining time in seconds.</li>
+ </ul>
+ The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ <ul>
+ <li>{UNLIMITED,-1} : Timeout is not configured for given timeout type
+ (LIFETIME).</li>
+ <li>{ISO8601 date string, 0} : Timeout is configured and application has
+ completed.</li>
+ <li>{ISO8601 date string, greater than zero} : Timeout is configured and
+ application is RUNNING. Application will be timed out after configured
+ value.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <class name="ApplicationTimeoutType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application timeout type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <class name="CollectorInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CollectorInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getCollectorAddr" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCollectorAddr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.lang.String"/>
+    </method>
+    <method name="getCollectorToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get delegation token for app collector which AM will use to publish
+ entities.
+ @return the delegation token for app collector.]]>
+      </doc>
+    </method>
+    <method name="setCollectorToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <field name="DEFAULT_TIMESTAMP_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Collector info containing collector address and collector token passed from
+ RM to AM in Allocate Response.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Container -->
+  <class name="Container" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Container"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the globally unique identifier for the container.
+ @return globally unique identifier for the container]]>
+      </doc>
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the node on which the container is allocated.
+ @return identifier of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the http uri of the node on which the container is allocated.
+ @return http uri of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> at which the <code>Container</code> was
+ allocated.
+ @return <code>Priority</code> at which the <code>Container</code> was
+         allocated]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerToken</code> for the container.
+ <p><code>ContainerToken</code> is the security token used by the framework
+ to verify authenticity of any <code>Container</code>.</p>
+
+ <p>The <code>ResourceManager</code>, on container allocation provides a
+ secure token which is verified by the <code>NodeManager</code> on
+ container launch.</p>
+
+ <p>Applications do not need to care about <code>ContainerToken</code>, they
+ are transparently handled by the framework - the allocated
+ <code>Container</code> includes the <code>ContainerToken</code>.</p>
+
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+
+ @return <code>ContainerToken</code> for the container]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to the original {@code
+ ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+ this allocated {@code Container}.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}s.
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+
+ @return the <em>ID</em> corresponding to the original  allocation request
+ which is satisfied by this allocation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code Container} represents an allocated resource in the cluster.
+ <p>
+ The {@code ResourceManager} is the sole authority to allocate any
+ {@code Container} to applications. The allocated {@code Container}
+ is always on a single node and has a unique {@link ContainerId}. It has
+ a specific amount of {@link Resource} allocated.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} for the container, which is globally unique.</li>
+   <li>
+     {@link NodeId} of the node on which it is allocated.
+   </li>
+   <li>HTTP uri of the node.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>{@link Priority} at which the container was allocated.</li>
+   <li>
+     Container {@link Token} of the container, used to securely verify
+     authenticity of the allocation.
+   </li>
+ </ul>
+ 
+ Typically, an {@code ApplicationMaster} receives the {@code Container}
+ from the {@code ResourceManager} during resource-negotiation and then
+ talks to the {@code NodeManager} to start/stop containers.
+ 
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Container -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <class name="ContainerExitStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerExitStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ABORTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc.]]>
+      </doc>
+    </field>
+    <field name="DISKS_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad.]]>
+      </doc>
+    </field>
+    <field name="PREEMPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers preempted by the framework.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_VMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated virtual memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_PMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated physical memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_APPMASTER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by stop request by the app master.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_RESOURCEMANAGER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the resource manager.]]>
+      </doc>
+    </field>
+    <field name="KILLED_AFTER_APP_COMPLETION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated after the application finished.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_CONTAINER_SCHEDULER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the ContainerScheduler to make room
+ for another container...]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Container exit statuses indicating special exit circumstances.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <class name="ContainerId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerId" type="long"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the application to which the
+ <code>Container</code> was assigned.
+ <p>
+ Note: If containers are kept alive across application attempts via
+ {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ the <code>ContainerId</code> does not necessarily contain the current
+ running application attempt's <code>ApplicationAttemptId</code> This
+ container can be allocated by previously exited application attempt and
+ managed by the current running attempt thus have the previous application
+ attempt's <code>ApplicationAttemptId</code>.
+ </p>
+ 
+ @return <code>ApplicationAttemptId</code> of the application to which the
+         <code>Container</code> was assigned]]>
+      </doc>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower 32 bits of identifier of the <code>ContainerId</code>,
+ which doesn't include epoch. Note that this method will be marked as
+ deprecated, so please use <code>getContainerId</code> instead.
+ @return lower 32 bits of identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the <code>ContainerId</code>. Upper 24 bits are
+ reserved as epoch of cluster, and lower 40 bits are reserved as
+ sequential number of containers.
+ @return identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return A string representation of containerId. The format is
+ container_e*epoch*_*clusterTimestamp*_*appId*_*attemptId*_*containerId*
+ when epoch is larger than 0
+ (e.g. container_e17_1410901177871_0001_01_000005).
+ *epoch* is increased when RM restarts or fails over.
+ When epoch is 0, epoch is omitted
+ (e.g. container_1410901177871_0001_01_000005).]]>
+      </doc>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="CONTAINER_ID_BITMASK" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ContainerId</code> represents a globally unique identifier
+ for a {@link Container} in the cluster.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <class name="ContainerLaunchContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLaunchContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+    </method>
+    <method name="getTokens" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the tokens needed by this container. It may include file-system
+ tokens, ApplicationMaster related tokens if this container is an
+ ApplicationMaster or framework level tokens needed by this container to
+ communicate to various services in a secure manner.
+ 
+ @return tokens needed by this container.]]>
+      </doc>
+    </method>
+    <method name="setTokens"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set security tokens needed by this container.
+ @param tokens security tokens]]>
+      </doc>
+    </method>
+    <method name="getTokensConf" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration used by RM to renew tokens.
+ @return The configuration used by RM to renew the tokens.]]>
+      </doc>
+    </method>
+    <method name="setTokensConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokensConf" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set the configuration used by RM to renew the tokens.
+ @param tokensConf The configuration used by RM to renew the tokens]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="setLocalResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set <code>LocalResource</code> required by the container. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param localResources <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="getServiceData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}.
+ </p>
+ 
+ <p>
+ This will be used to initialize this application on the specific
+ {@link AuxiliaryService} running on the NodeManager by calling
+ {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)}
+ </p>
+ 
+ @return application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="setServiceData"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceData" type="java.util.Map"/>
+      <doc>
+      <![CDATA[<p>
+ Set application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}. All pre-existing Map entries are
+ preserved.
+ </p>
+ 
+ @param serviceData
+          application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>environment variables</em> for the container.
+ @return <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="setEnvironment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="environment" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add <em>environment variables</em> for the container. All pre-existing Map
+ entries are cleared before adding the new Map
+ @param environment <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getCommands" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>commands</em> for launching the container.
+ @return the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="setCommands"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add the list of <em>commands</em> for launching the container. All
+ pre-existing List entries are cleared before adding the new List
+ @param commands the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="setApplicationACLs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationACL</code>s for the application. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param acls <code>ApplicationACL</code>s for the application]]>
+      </doc>
+    </method>
+    <method name="getContainerRetryContext" return="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerRetryContext</code> to relaunch container.
+ @return <code>ContainerRetryContext</code> to relaunch container.]]>
+      </doc>
+    </method>
+    <method name="setContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerRetryContext</code> to relaunch container.
+ @param containerRetryContext <code>ContainerRetryContext</code> to
+                              relaunch container.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerLaunchContext} represents all of the information
+ needed by the {@code NodeManager} to launch a container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>User to whom the container is allocated.</li>
+   <li>Security tokens (if security is enabled).</li>
+   <li>
+     {@link LocalResource} necessary for running the container such
+     as binaries, jar, shared-objects, side-files etc.
+   </li>
+   <li>Optional, application-specific binary service data.</li>
+   <li>Environment variables for the launched process.</li>
+   <li>Command to launch the container.</li>
+   <li>Retry strategy when container exits with failure.</li>
+ </ul>
+ 
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <class name="ContainerReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ 
+ @return <code>ContainerId</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getAllocatedResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Resource</code> of the container.
+ 
+ @return allocated <code>Resource</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getAssignedNode" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>NodeId</code> where container is running.
+ 
+ @return allocated <code>NodeId</code> where container is running.]]>
+      </doc>
+    </method>
+    <method name="setAssignedNode"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Priority</code> of the container.
+ 
+ @return allocated <code>Priority</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the creation time of the container.
+ 
+ @return creation time of the container]]>
+      </doc>
+    </method>
+    <method name="setCreationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="creationTime" type="long"/>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Finish time of the container.
+ 
+ @return Finish time of the container]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the DiagnosticsInfo of the container.
+ 
+ @return DiagnosticsInfo of the container]]>
+      </doc>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the LogURL of the container.
+ 
+ @return LogURL of the container]]>
+      </doc>
+    </method>
+    <method name="setLogUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logUrl" type="java.lang.String"/>
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>ContainerState</code> of the container.
+ 
+ @return final <code>ContainerState</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"/>
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>exit status</code> of the container.
+ 
+ @return final <code>exit status</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerExitStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerExitStatus" type="int"/>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Node Http address of the container
+ 
+ @return the node http address of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the execution type of the container.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerReport} is a report of an container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>Allocated Resources to the container.</li>
+   <li>Assigned Node id.</li>
+   <li>Assigned Priority.</li>
+   <li>Creation Time.</li>
+   <li>Finish Time.</li>
+   <li>Container Exit Status.</li>
+   <li>{@link ContainerState} of the container.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Log URL.</li>
+   <li>nodeHttpAddress</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <class name="ContainerRetryContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerRetryContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRetryPolicy" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryPolicy" type="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"/>
+    </method>
+    <method name="getErrorCodes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setErrorCodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCodes" type="java.util.Set"/>
+    </method>
+    <method name="getMaxRetries" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxRetries"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxRetries" type="int"/>
+    </method>
+    <method name="getRetryInterval" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryInterval" type="int"/>
+    </method>
+    <field name="RETRY_FOREVER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RETRY_INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NEVER_RETRY_CONTEXT" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@code ContainerRetryContext} indicates how container retry after it fails
+ to run.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>
+     {@link ContainerRetryPolicy} :
+     - NEVER_RETRY(DEFAULT value): no matter what error code is when container
+       fails to run, just do not retry.
+     - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails
+       to run, just retry.
+     - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if
+       the error code is one of <em>errorCodes</em>, otherwise do not retry.
+
+     Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry
+     because it is usually killed on purpose.
+   </li>
+   <li>
+     <em>maxRetries</em> specifies how many times to retry if need to retry.
+     If the value is -1, it means retry forever.
+   </li>
+   <li><em>retryInterval</em> specifies delaying some time before relaunch
+   container, the unit is millisecond.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <class name="ContainerRetryPolicy" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>Retry policy for relaunching a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <class name="ContainerState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <class name="ContainerStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerState</code> of the container.
+ @return <code>ContainerState</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the <em>exit status</em> for the container.</p>
+  
+ <p>Note: This is valid only for completed containers i.e. containers
+ with state {@link ContainerState#COMPLETE}. 
+ Otherwise, it returns an ContainerExitStatus.INVALID.
+ </p>
+ 
+ <p>Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc. have a special
+ exit code of ContainerExitStatus.ABORTED.</p>
+ 
+ <p>When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad, then
+ container is not launched and is exited with ContainersExitStatus.DISKS_FAILED.
+ </p>
+  
+ @return <em>exit status</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic messages</em> for failed containers.
+ @return <em>diagnostic messages</em> for failed containers]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getIPs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the IP addresses with which the container run.
+ @return The IP address where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the hostname where the container runs.
+ @return The hostname where the container runs.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerStatus} represents the current status of a
+ {@code Container}.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>{@code ContainerId} of the container.</li>
+   <li>{@code ExecutionType} of the container.</li>
+   <li>{@code ContainerState} of the container.</li>
+   <li><em>Exit status</em> of a completed container.</li>
+   <li><em>Diagnostic</em> message for a failed container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <class name="ContainerSubState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerSubState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerSubState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container Sub-State.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <class name="ContainerUpdateType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Encodes the type of Container Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <class name="ExecutionType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ExecutionType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container property encoding execution semantics.
+
+ <p>
+ The execution types are the following:
+ <ul>
+   <li>{@link #GUARANTEED} - this container is guaranteed to start its
+   execution, once the corresponding start container request is received by
+   an NM.
+   <li>{@link #OPPORTUNISTIC} - the execution of this container may not start
+   immediately at the NM that receives the corresponding start container
+   request (depending on the NM's available resources). Moreover, it may be
+   preempted if it blocks a GUARANTEED container from being executed.
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <class name="ExecutionTypeRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ExecutionTypeRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="ensureExecutionType" type="boolean"/>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionType</code> of the requested container.
+
+ @param execType
+          ExecutionType of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ExecutionType</code>.
+
+ @return <code>ExecutionType</code>.]]>
+      </doc>
+    </method>
+    <method name="setEnforceExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enforceExecutionType" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to explicitly ask that the Scheduling Authority return
+ Containers of exactly the Execution Type requested.
+ @param enforceExecutionType whether ExecutionType request should be
+                            strictly honored.]]>
+      </doc>
+    </method>
+    <method name="getEnforceExecutionType" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether Scheduling Authority should return Containers of exactly the
+ Execution Type requested for this <code>ResourceRequest</code>.
+ Defaults to false.
+ @return whether ExecutionType request should be strictly honored]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object of this class represents a specification of the execution
+ guarantee of the Containers associated with a ResourceRequest. It consists
+ of an <code>ExecutionType</code> as well as flag that explicitly asks the
+ configuredScheduler to return Containers of exactly the Execution Type
+ requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <class name="FinalApplicationStatus" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various final states of an <code>Application</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <class name="LocalResource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalResource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>location</em> of the resource to be localized.
+ @return <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <doc>
+      <![CDATA[Set <em>location</em> of the resource to be localized.
+ @param resource <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getSize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>size</em> of the resource to be localized.
+ @return <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>size</em> of the resource to be localized.
+ @param size <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @return <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @param timestamp <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceType</code> of the resource to be localized.
+ @return <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceType</code> of the resource to be localized.
+ @param type <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getVisibility" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @return <code>LocalResourceVisibility</code> of the resource to be 
+         localized]]>
+      </doc>
+    </method>
+    <method name="setVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @param visibility <code>LocalResourceVisibility</code> of the resource to be 
+                   localized]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @return <em>pattern</em> that should be used to extract entries from the 
+ archive.]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @param pattern <em>pattern</em> that should be used to extract entries 
+ from the archive.]]>
+      </doc>
+    </method>
+    <method name="getShouldBeUploadedToSharedCache" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM uses it to decide whether if it is necessary to upload the resource to
+ the shared cache
+ @return true if it is necessary to upload the resource
+                 to the shared cache,
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="setShouldBeUploadedToSharedCache"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+      <doc>
+      <![CDATA[Inform NM whether upload to SCM is needed.
+
+ @param shouldBeUploadedToSharedCache <em>shouldBeUploadedToSharedCache</em>
+          of this request]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>LocalResource</code> represents a local resource required to
+ run a container.</p>
+ 
+ <p>The <code>NodeManager</code> is responsible for localizing the resource 
+ prior to launching the container.</p>
+ 
+ <p>Applications can specify {@link LocalResourceType} and 
+ {@link LocalResourceVisibility}.</p>
+ 
+ @see LocalResourceType
+ @see LocalResourceVisibility
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <class name="LocalResourceType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceType} specifies the <em>type</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>type</em> can be one of:
+ <ul>
+   <li>
+     {@link #FILE} - Regular file i.e. uninterpreted bytes.
+   </li>
+   <li>
+     {@link #ARCHIVE} - Archive, which is automatically unarchived by the
+     <code>NodeManager</code>.
+   </li>
+   <li>
+     {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}.
+   </li>
+ </ul>
+
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <class name="LocalResourceVisibility" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceVisibility} specifies the <em>visibility</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>visibility</em> can be one of:
+ <ul>
+   <li>{@link #PUBLIC} - Shared by all users on the node.</li>
+   <li>
+     {@link #PRIVATE} - Shared among all applications of the
+     <em>same user</em> on the node.
+   </li>
+   <li>
+     {@link #APPLICATION} - Shared only among containers of the
+     <em>same application</em> on the node.
+   </li>
+ </ul>
+ 
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <class name="LogAggregationContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <param name="policyClassName" type="java.lang.String"/>
+      <param name="policyParameters" type="java.lang.String"/>
+    </method>
+    <method name="getIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param includePattern]]>
+      </doc>
+    </method>
+    <method name="getExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="excludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param excludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern in a rolling fashion.
+ 
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern in a rolling fashion.
+ 
+ @param rolledLogsIncludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern for aggregation in a rolling fashion.
+ 
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern for in a rolling fashion.
+ 
+ @param rolledLogsExcludePattern]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyClassName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy class.
+
+ @return log aggregation policy class]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyClassName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy class.
+
+ @param className]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyParameters" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy parameters.
+
+ @return log aggregation policy parameters]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy parameters.
+ There is no schema defined for the parameters string.
+ It is up to the log aggregation policy class to decide how to parse
+ the parameters string.
+
+ @param parameters]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code LogAggregationContext} represents all of the
+ information needed by the {@code NodeManager} to handle
+ the logs for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>
+     includePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be uploaded when the application finishes.
+   </li>
+   <li>
+     excludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be uploaded when application finishes. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     rolledLogsIncludePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be aggregated in a rolling fashion.
+   </li>
+   <li>
+     rolledLogsExcludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be aggregated in a rolling fashion. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     policyClassName. The policy class name that implements
+     ContainerLogAggregationPolicy. At runtime, nodemanager will the policy
+     if a given container's log should be aggregated based on the
+     ContainerType and other runtime state such as exit code by calling
+     ContainerLogAggregationPolicy#shouldDoLogAggregation.
+     This is useful when the app only wants to aggregate logs of a subset of
+     containers. Here are the available policies. Please make sure to specify
+     the canonical name by prefixing org.apache.hadoop.yarn.server.
+     nodemanager.containermanager.logaggregation.
+     to the class simple name below.
+     NoneContainerLogAggregationPolicy: skip aggregation for all containers.
+     AllContainerLogAggregationPolicy: aggregate all containers.
+     AMOrFailedContainerLogAggregationPolicy: aggregate application master
+         or failed containers.
+     FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed
+         containers
+     FailedContainerLogAggregationPolicy: aggregate failed containers
+     AMOnlyLogAggregationPolicy: aggregate application master containers
+     SampleContainerLogAggregationPolicy: sample logs of successful worker
+         containers, in addition to application master and failed/killed
+         containers.
+     If it isn't specified, it will use the cluster-wide default policy
+     defined by configuration yarn.nodemanager.log-aggregation.policy.class.
+     The default value of yarn.nodemanager.log-aggregation.policy.class is
+     AllContainerLogAggregationPolicy.
+   </li>
+   <li>
+     policyParameters. The parameters passed to the policy class via
+     ContainerLogAggregationPolicy#parseParameters during the policy object
+     initialization. This is optional. Some policy class might use parameters
+     to adjust its settings. It is up to policy class to define the scheme of
+     parameters.
+     For example, SampleContainerLogAggregationPolicy supports the format of
+     "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50
+     successful worker containers.
+   </li>
+ </ul>
+
+ @see ApplicationSubmissionContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NMToken -->
+  <class name="NMToken" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMToken"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link NodeId} of the <code>NodeManager</code> for which the NMToken
+ is used to authenticate.
+ @return the {@link NodeId} of the <code>NodeManager</code> for which the
+ NMToken is used to authenticate.]]>
+      </doc>
+    </method>
+    <method name="setNodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Token} used for authenticating with <code>NodeManager</code>
+ @return the {@link Token} used for authenticating with <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The NMToken is used for authenticating communication with
+ <code>NodeManager</code></p>
+ <p>It is issued by <code>ResourceMananger</code> when <code>ApplicationMaster</code>
+ negotiates resource with <code>ResourceManager</code> and
+ validated on <code>NodeManager</code> side.</p>
+ @see  AllocateResponse#getNMTokens()]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NMToken -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeId -->
+  <class name="NodeId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>hostname</em> of the node.
+ @return <em>hostname</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>port</em> for communicating with the node.
+ @return <em>port</em> for communicating with the node]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p><code>NodeId</code> is the unique identifier for a node.</p>
+ 
+ <p>It includes the <em>hostname</em> and <em>port</em> to uniquely 
+ identify the node. Thus, it is unique across restarts of any 
+ <code>NodeManager</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <class name="NodeLabel" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeLabel"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isExclusive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeLabel"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DEFAULT_NODE_LABEL_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default node label partition used for displaying.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABEL_EXPRESSION_NOT_SET" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node Label expression not set .]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABEL_EXCLUSIVITY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, node label is exclusive or not]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <class name="NodeReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeId</code> of the node.
+ @return <code>NodeId</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getNodeState" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeState</code> of the node.
+ @return <code>NodeState</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>http address</em> of the node.
+ @return <em>http address</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getRackName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>rack name</em> for the node.
+ @return <em>rack name</em> for the node]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>used</em> <code>Resource</code> on the node.
+ @return <em>used</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>total</em> <code>Resource</code> on the node.
+ @return <em>total</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get labels of this node.
+ @return labels of this node.]]>
+      </doc>
+    </method>
+    <method name="getAggregatedContainersUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get containers aggregated resource utilization in a node.
+ @return containers resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getNodeUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node resource utilization.
+ @return node resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getDecommissioningTimeout" return="java.lang.Integer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional decommissioning timeout in seconds (null indicates absent
+ timeout).
+ @return the decommissioning timeout in second.]]>
+      </doc>
+    </method>
+    <method name="setDecommissioningTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decommissioningTimeout" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Set the decommissioning timeout in seconds (null indicates absent timeout).]]>
+      </doc>
+    </method>
+    <method name="getNodeUpdateType" return="org.apache.hadoop.yarn.api.records.NodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional node update type (null indicates absent update type).
+ @return the node update.]]>
+      </doc>
+    </method>
+    <method name="setNodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeUpdateType" type="org.apache.hadoop.yarn.api.records.NodeUpdateType"/>
+      <doc>
+      <![CDATA[Set the node update type (null indicates absent node update type).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeReport} is a summary of runtime information of a node
+ in the cluster.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link NodeId} of the node.</li>
+   <li>HTTP Tracking URL of the node.</li>
+   <li>Rack name for the node.</li>
+   <li>Used {@link Resource} on the node.</li>
+   <li>Total available {@link Resource} of the node.</li>
+   <li>Number of running containers on the node.</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeState -->
+  <class name="NodeState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isUnusable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInactiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isActiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Node</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <class name="PreemptionContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Container referenced by this handle.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specific container requested back by the <code>ResourceManager</code>.
+ @see PreemptionContract
+ @see StrictPreemptionContract]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <class name="PreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the AM releases resources matching these requests, then the {@link
+ PreemptionContainer}s enumerated in {@link #getContainers()} should not be
+ evicted from the cluster. Due to delays in propagating cluster state and
+ sending these messages, there are conditions where satisfied contracts may
+ not prevent the platform from killing containers.
+ @return List of {@link PreemptionResourceRequest} to update the
+ <code>ApplicationMaster</code> about resources requested back by the
+ <code>ResourceManager</code>.
+ @see AllocateRequest#setAskList(List)]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Assign the set of {@link PreemptionContainer} specifying which containers
+ owned by the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>. If the AM prefers a different set of
+ containers, then it may checkpoint or kill containers matching the
+ description in {@link #getResourceRequest}.
+ @return Set of containers at risk if the contract is not met.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the <code>ResourceManager</code>.
+ The <code>ApplicationMaster</code> (AM) can satisfy this request according
+ to its own priorities to prevent containers from being forcibly killed by
+ the platform.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <class name="PreemptionMessage" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionMessage"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStrictContract" return="org.apache.hadoop.yarn.api.records.StrictPreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Specific resources that may be killed by the
+ <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContract" return="org.apache.hadoop.yarn.api.records.PreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Contract describing resources to return to the cluster.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link PreemptionMessage} is part of the RM-AM protocol, and it is used by
+ the RM to specify resources that the RM wants to reclaim from this
+ {@code ApplicationMaster} (AM). The AM receives a {@link
+ StrictPreemptionContract} message encoding which containers the platform may
+ forcibly kill, granting it an opportunity to checkpoint state or adjust its
+ execution plan. The message may also include a {@link PreemptionContract}
+ granting the AM more latitude in selecting which resources to return to the
+ cluster.
+ <p>
+ The AM should decode both parts of the message. The {@link
+ StrictPreemptionContract} specifies particular allocations that the RM
+ requires back. The AM can checkpoint containers' state, adjust its execution
+ plan to move the computation, or take no action and hope that conditions that
+ caused the RM to ask for the container will change.
+ <p>
+ In contrast, the {@link PreemptionContract} also includes a description of
+ resources with a set of containers. If the AM releases containers matching
+ that profile, then the containers enumerated in {@link
+ PreemptionContract#getContainers()} may not be killed.
+ <p>
+ Each preemption message reflects the RM's current understanding of the
+ cluster state, so a request to return <em>N</em> containers may not
+ reflect containers the AM is releasing, recently exited containers the RM has
+ yet to learn about, or new containers allocated before the message was
+ generated. Conversely, an RM may request a different profile of containers in
+ subsequent requests.
+ <p>
+ The policy enforced by the RM is part of the scheduler. Generally, only
+ containers that have been requested consistently should be killed, but the
+ details are not specified.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <class name="PreemptionResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Resource described in this request, to be matched against running
+ containers.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the cluster.
+ @see PreemptionContract
+ @see AllocateRequest#setAskList(java.util.List)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Priority -->
+  <class name="Priority" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Priority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="int"/>
+    </method>
+    <method name="getPriority" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the assigned priority
+ @return the assigned priority]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Set the assigned priority
+ @param priority the assigned priority]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="UNDEFINED" type="org.apache.hadoop.yarn.api.records.Priority"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The priority assigned to a ResourceRequest or Application or Container 
+ allocation]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Priority -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <class name="QueueACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code QueueACL} enumerates the various ACLs for queues.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue.
+   </li>
+   <li>{@link #ADMINISTER_QUEUE} - ACL to administer the queue.</li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>name</em> of the queue.
+ @return <em>name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>configured capacity</em> of the queue.
+ @return <em>configured capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getMaximumCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>maximum capacity</em> of the queue.
+ @return <em>maximum capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCurrentCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current capacity</em> of the queue.
+ @return <em>current capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>child queues</em> of the queue.
+ @return <em>child queues</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>running applications</em> of the queue.
+ @return <em>running applications</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueState" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueState</code> of the queue.
+ @return <code>QueueState</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getAccessibleNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>accessible node labels</code> of the queue.
+ @return <code>accessible node labels</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getDefaultNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>default node label expression</code> of the queue, this takes
+ affect only when the <code>ApplicationSubmissionContext</code> and
+ <code>ResourceRequest</code> don't specify their
+ <code>NodeLabelExpression</code>.
+ 
+ @return <code>default node label expression</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setDefaultNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getQueueStatistics" return="org.apache.hadoop.yarn.api.records.QueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>queue stats</code> for the queue
+
+ @return <code>queue stats</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueStatistics" type="org.apache.hadoop.yarn.api.records.QueueStatistics"/>
+      <doc>
+      <![CDATA[Set the queue statistics for the queue
+ 
+ @param queueStatistics
+          the queue statistics]]>
+      </doc>
+    </method>
+    <method name="getPreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>preemption status</em> of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return <em>preemption status</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueConfigurations" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the per-node-label queue configurations of the queue.
+
+ @return the per-node-label queue configurations of the queue.]]>
+      </doc>
+    </method>
+    <method name="getIntraQueuePreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the intra-queue preemption status of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return intra-queue preemption status of the queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[QueueInfo is a report of the runtime information of the queue.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>Queue name.</li>
+   <li>Capacity of the queue.</li>
+   <li>Maximum capacity of the queue.</li>
+   <li>Current capacity of the queue.</li>
+   <li>Child queues.</li>
+   <li>Running applications.</li>
+   <li>{@link QueueState} of the queue.</li>
+   <li>{@link QueueConfigurations} of the queue.</li>
+ </ul>
+
+ @see QueueState
+ @see QueueConfigurations
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of a Queue.
+ <p>
+ A queue is in one of:
+ <ul>
+   <li>{@link #RUNNING} - normal state.</li>
+   <li>{@link #STOPPED} - not accepting new application submissions.</li>
+   <li>
+     {@link #DRAINING} - not accepting new application submissions
+     and waiting for applications finish.
+   </li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <class name="QueueStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumAppsSubmitted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of apps submitted
+ 
+ @return the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="setNumAppsSubmitted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsSubmitted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of apps submitted
+ 
+ @param numAppsSubmitted
+          the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="getNumAppsRunning" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running apps
+ 
+ @return the number of running apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsRunning"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsRunning" type="long"/>
+      <doc>
+      <![CDATA[Set the number of running apps
+ 
+ @param numAppsRunning
+          the number of running apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsPending" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending apps
+ 
+ @return the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsPending"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsPending" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending apps
+ 
+ @param numAppsPending
+          the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsCompleted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of completed apps
+ 
+ @return the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsCompleted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsCompleted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of completed apps
+ 
+ @param numAppsCompleted
+          the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsKilled" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of killed apps
+ 
+ @return the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsKilled"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsKilled" type="long"/>
+      <doc>
+      <![CDATA[Set the number of killed apps
+ 
+ @param numAppsKilled
+          the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsFailed" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of failed apps
+ 
+ @return the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsFailed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsFailed" type="long"/>
+      <doc>
+      <![CDATA[Set the number of failed apps
+ 
+ @param numAppsFailed
+          the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="getNumActiveUsers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active users
+ 
+ @return the number of active users]]>
+      </doc>
+    </method>
+    <method name="setNumActiveUsers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numActiveUsers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of active users
+ 
+ @param numActiveUsers
+          the number of active users]]>
+      </doc>
+    </method>
+    <method name="getAvailableMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available memory in MB
+ 
+ @return the available memory]]>
+      </doc>
+    </method>
+    <method name="setAvailableMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the available memory in MB
+ 
+ @param availableMemoryMB
+          the available memory]]>
+      </doc>
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated memory in MB
+ 
+ @return the allocated memory]]>
+      </doc>
+    </method>
+    <method name="setAllocatedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated memory in MB
+ 
+ @param allocatedMemoryMB
+          the allocate memory]]>
+      </doc>
+    </method>
+    <method name="getPendingMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending memory in MB
+ 
+ @return the pending memory]]>
+      </doc>
+    </method>
+    <method name="setPendingMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the pending memory in MB
+ 
+ @param pendingMemoryMB
+          the pending memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved memory in MB
+ 
+ @return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved memory in MB
+ 
+ @param reservedMemoryMB
+          the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getAvailableVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available vcores
+ 
+ @return the available vcores]]>
+      </doc>
+    </method>
+    <method name="setAvailableVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the available vcores
+ 
+ @param availableVCores
+          the available vcores]]>
+      </doc>
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated vcores
+ 
+ @return the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="setAllocatedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated vcores
+ 
+ @param allocatedVCores
+          the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending vcores
+ 
+ @return the pending vcores]]>
+      </doc>
+    </method>
+    <method name="setPendingVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the pending vcores
+ 
+ @param pendingVCores
+          the pending vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending containers.
+ @return the number of pending containers.]]>
+      </doc>
+    </method>
+    <method name="setPendingContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending containers.
+ @param pendingContainers the pending containers.]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of allocated containers.
+ @return the number of allocated containers.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of allocated containers.
+ @param allocatedContainers the allocated containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved containers.
+ @return the number of reserved containers.]]>
+      </doc>
+    </method>
+    <method name="setReservedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of reserved containers.
+ @param reservedContainers the reserved containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved vcores
+ 
+ @return the reserved vcores]]>
+      </doc>
+    </method>
+    <method name="setReservedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved vcores
+ 
+ @param reservedVCores
+          the reserved vcores]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <class name="QueueUserACLInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueUserACLInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> of the queue.
+ @return <em>queue name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getUserAcls" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>QueueACL</code> for the given user.
+ @return list of <code>QueueACL</code> for the given user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>QueueUserACLInfo</code> provides information {@link QueueACL} for
+ the given user.</p>
+ 
+ @see QueueACL
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <class name="ReservationACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationACL} enumerates the various ACLs for reservations.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and
+     delete reservations.
+   </li>
+   <li> {@link #LIST_RESERVATIONS} - ACL to list reservations. </li>
+   <li> {@link #SUBMIT_RESERVATIONS} - ACL to create reservations. </li>
+ </ul>
+ Users can always list, update and delete their own reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <class name="ReservationAllocationState" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationAllocationState"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationAllocationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acceptanceTime" type="long"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="resourceAllocations" type="java.util.List"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[@param acceptanceTime The acceptance time of the reservation.
+ @param user The username of the user who made the reservation.
+ @param resourceAllocations List of {@link ResourceAllocationRequest}
+                            representing the current state of the
+                            reservation resource allocations. This is
+                            subject to change in the event of re-planning.
+ @param reservationId {@link ReservationId } of the reservation being
+                                            listed.
+ @param reservationDefinition {@link ReservationDefinition} used to make
+                              the reservation.
+ @return {@code ReservationAllocationState} that represents the state of
+ the reservation.]]>
+      </doc>
+    </method>
+    <method name="getAcceptanceTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the acceptance time of the reservation.
+
+ @return the time that the reservation was accepted.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user who made the reservation.
+
+ @return the name of the user who made the reservation.]]>
+      </doc>
+    </method>
+    <method name="getResourceAllocationRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Resource allocations of the reservation based on the current state
+ of the plan. This is subject to change in the event of re-planning.
+ The allocations will be constraint to the user contract as described by
+ the {@link ReservationDefinition}
+
+ @return a list of resource allocations for the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of the reservation.
+
+ @return the reservation id corresponding to the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation definition used to make the reservation.
+
+ @return the reservation definition used to make the reservation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationAllocationState} represents the reservation that is
+ made by a user.
+ <p>
+ It includes:
+ <ul>
+   <li>Duration of the reservation.</li>
+   <li>Acceptance time of the duration.</li>
+   <li>
+       List of {@link ResourceAllocationRequest}, which includes the time
+       interval, and capability of the allocation.
+       {@code ResourceAllocationRequest} represents an allocation
+       made for a reservation for the current state of the queue. This can be
+       changed for reasons such as re-planning, but will always be subject to
+       the constraints of the user contract as described by
+       {@link ReservationDefinition}
+   </li>
+   <li>{@link ReservationId} of the reservation.</li>
+   <li>{@link ReservationDefinition} used to make the reservation.</li>
+ </ul>
+
+ @see ResourceAllocationRequest
+ @see ReservationId
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <class name="ReservationDefinition" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDefinition"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getArrival" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @return the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="setArrival"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="earliestStartTime" type="long"/>
+      <doc>
+      <![CDATA[Set the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @param earliestStartTime the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="getDeadline" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @return the deadline or the latest time by when the resource(s) must be
+         allocated]]>
+      </doc>
+    </method>
+    <method name="setDeadline"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="latestEndTime" type="long"/>
+      <doc>
+      <![CDATA[Set the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @param latestEndTime the deadline or the latest time by when the
+          resource(s) should be allocated]]>
+      </doc>
+    </method>
+    <method name="getReservationRequests" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="setReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @param reservationRequests the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="getReservationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @return string representing the name of the corresponding reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @param name representing the name of the corresponding reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getRecurrenceExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @return recurrence of this reservation]]>
+      </doc>
+    </method>
+    <method name="setRecurrenceExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @param recurrenceExpression recurrence interval of this reservation]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @return int representing the priority of the reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @param priority representing the priority of the reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDefinition} captures the set of resource and time
+ constraints the user cares about regarding a reservation.
+ 
+ @see ResourceRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <class name="ReservationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the long identifier of the {@link ReservationId} which is unique for
+ all Reservations started by a particular instance of the
+ {@code ResourceManager}.
+
+ @return long identifier of the {@link ReservationId}]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the {@code ResourceManager} which is used to
+ generate globally unique {@link ReservationId}.
+
+ @return <em>start time</em> of the {@code ResourceManager}]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the string argument as a {@link ReservationId}
+
+ @param reservationId the string representation of the {@link ReservationId}
+ @return the {@link ReservationId} corresponding to the input string if
+         valid, null if input is null
+ @throws IOException if unable to parse the input string]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="reserveIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="clusterTimestamp" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ {@link ReservationId} represents the <em>globally unique</em> identifier for
+ a reservation.
+ </p>
+
+ <p>
+ The globally unique nature of the identifier is achieved by using the
+ <em>cluster timestamp</em> i.e. start-time of the {@code ResourceManager}
+ along with a monotonically increasing counter for the reservation.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <class name="ReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="concurrency" type="int"/>
+      <param name="duration" type="long"/>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Resource} capability of the request.
+ 
+ @return {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the {@link Resource} capability of the request
+ 
+ @param capability {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ 
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ 
+ @param numContainers number of containers required with the given
+          specifications]]>
+      </doc>
+    </method>
+    <method name="getConcurrency" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @return the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="setConcurrency"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @param numContainers the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="getDuration" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds for which the resource is required. A
+ default value of -1, indicates an unspecified lease duration, and fallback
+ to current behavior.
+ 
+ @return the duration in milliseconds for which the resource is required]]>
+      </doc>
+    </method>
+    <method name="setDuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="duration" type="long"/>
+      <doc>
+      <![CDATA[Set the duration in milliseconds for which the resource is required.
+ 
+ @param duration the duration in milliseconds for which the resource is
+          required]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequest} represents the request made by an application to
+ the {@code ResourceManager} to reserve {@link Resource}s.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required by the
+     application.
+   </li>
+   <li>Concurrency that indicates the gang size of the request.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <class name="ReservationRequest.ReservationRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ReservationRequest.ReservationRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <class name="ReservationRequestInterpreter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various types of dependencies among multiple
+ {@link ReservationRequests} within one {@link ReservationDefinition} (from
+ least constraining to most constraining).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <class name="ReservationRequests" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationRequests"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+    </method>
+    <method name="getReservationResources" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="setReservationResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @param reservationResources the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="getInterpreter" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @return the list of {@link ReservationRequestInterpreter}]]>
+      </doc>
+    </method>
+    <method name="setInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interpreter" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @param interpreter the {@link ReservationRequestInterpreter} for this
+          reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequests} captures the set of resource and constraints the
+ user cares about regarding a reservation.
+ 
+ @see ReservationRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Resource -->
+  <class name="Resource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+      <param name="others" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Create a new {@link Resource} instance with the given CPU and memory
+ values and additional resource values as set in the {@code others}
+ parameter. Note that the CPU and memory settings in the {@code others}
+ parameter will be ignored.
+
+ @param memory the memory value
+ @param vCores the CPU value
+ @param others a map of other resource values indexed by resource name
+ @return a {@link Resource} instance with the given resource values]]>
+      </doc>
+    </method>
+    <method name="getMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is DEPRECATED:
+ Use {@link Resource#getMemorySize()} instead
+
+ Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="getMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to set
+ memory but the assumption is that the value passed is in units of MB.
+
+ @param memory <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemorySize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource.
+ @param memory <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getVirtualCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @return <em>num of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setVirtualCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vCores" type="int"/>
+      <doc>
+      <![CDATA[Set <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @param vCores <em>number of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <exception name="ResourceNotFoundException" type="org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"/>
+      <doc>
+      <![CDATA[Get ResourceInformation for a specified resource.
+
+ @param resource name of the resource
+ @return the ResourceInformation object for the resource
+ @throws ResourceNotFoundException if the resource can't be found]]>
+      </doc>
+    </method>
+    <method name="getResourceValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <exception name="ResourceNotFoundException" type="org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"/>
+      <doc>
+      <![CDATA[Get the value for a specified resource. No information about the units is
+ returned.
+
+ @param resource name of the resource
+ @return the value for the resource
+ @throws ResourceNotFoundException if the resource can't be found]]>
+      </doc>
+    </method>
+    <method name="setResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="resourceInformation" type="org.apache.hadoop.yarn.api.records.ResourceInformation"/>
+      <exception name="ResourceNotFoundException" type="org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"/>
+      <doc>
+      <![CDATA[Set the ResourceInformation object for a particular resource.
+
+ @param resource the resource for which the ResourceInformation is provided
+ @param resourceInformation ResourceInformation object
+ @throws ResourceNotFoundException if the resource is not found]]>
+      </doc>
+    </method>
+    <method name="setResourceValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <exception name="ResourceNotFoundException" type="org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"/>
+      <doc>
+      <![CDATA[Set the value of a resource in the ResourceInformation object. The unit of
+ the value is assumed to be the one in the ResourceInformation object.
+
+ @param resource the resource for which the value is provided.
+ @param value    the value to set
+ @throws ResourceNotFoundException if the resource is not found]]>
+      </doc>
+    </method>
+    <method name="throwExceptionWhenArrayOutOfBound"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="index" type="int"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFormattedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is to get memory in terms of KB|MB|GB.
+ @return string containing all resources]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="castToIntSafely" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Convert long to int for a resource value safely. This method assumes
+ resource value is positive.
+
+ @param value long resource value
+ @return int resource value]]>
+      </doc>
+    </method>
+    <method name="newDefaultInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Create ResourceInformation with basic fields.
+ @param name Resource Type Name
+ @param unit Default unit of provided resource type
+ @param value Value associated with giveb resource
+ @return ResourceInformation object]]>
+      </doc>
+    </method>
+    <field name="resources" type="org.apache.hadoop.yarn.api.records.ResourceInformation[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_MANDATORY_RESOURCES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="VCORES_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>Resource</code> models a set of computer resources in the 
+ cluster.</p>
+ 
+ <p>Currently it models both <em>memory</em> and <em>CPU</em>.</p>
+ 
+ <p>The unit for memory is megabytes. CPU is modeled with virtual cores
+ (vcores), a unit for expressing parallelism. A node's capacity should
+ be configured with virtual cores equal to its number of physical cores. A
+ container should be requested with the number of cores it can saturate, i.e.
+ the average number of threads it expects to have runnable at a time.</p>
+ 
+ <p>Virtual cores take integer values and thus currently CPU-scheduling is
+ very coarse.  A complementary axis for CPU requests that represents
+ processing power will likely be added in the future to enable finer-grained
+ resource configuration.</p>
+
+ <p>Typically, applications request <code>Resource</code> of suitable
+ capability to run their component tasks.</p>
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Resource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <class name="ResourceAllocationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceAllocationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceAllocationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[@param startTime The start time that the capability is reserved for.
+ @param endTime The end time that the capability is reserved for.
+ @param capability {@link Resource} representing the capability of the
+                                   resource allocation.
+ @return {ResourceAllocationRequest} which represents the capability of
+ the resource allocation for a time interval.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time that the resource is allocated.
+
+ @return the start time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time that the resource is allocated.
+
+ @return the end time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated resource.
+
+ @return the allocated resource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceAllocationRequest} represents an allocation
+ made for a reservation for the current state of the plan. This can be
+ changed for reasons such as re-planning, but will always be subject to the
+ constraints of the user contract as described by
+ {@link ReservationDefinition}
+ {@link Resource}
+
+ <p>
+ It includes:
+ <ul>
+   <li>StartTime of the allocation.</li>
+   <li>EndTime of the allocation.</li>
+   <li>{@link Resource} reserved for the allocation.</li>
+ </ul>
+
+ @see Resource]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <class name="ResourceBlacklistRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceBlacklistRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="additions" type="java.util.List"/>
+      <param name="removals" type="java.util.List"/>
+    </method>
+    <method name="getBlacklistAdditions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be added to the 
+ application blacklist.
+ 
+ @return list of resource-names which should be added to the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistAdditions"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be added to the application blacklist.
+ 
+ @param resourceNames list of resource-names which should be added to the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <method name="getBlacklistRemovals" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @return list of resource-names which should be removed from the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistRemovals"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @param resourceNames list of resource-names which should be removed from the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ResourceBlacklistRequest} encapsulates the list of resource-names 
+ which should be added or removed from the <em>blacklist</em> of resources 
+ for the application.
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <class name="ResourceOption" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceOption"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="overCommitTimeout" type="int"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Negative value means no timeout.]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <class name="ResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Clone a ResourceRequest object (shallow copy). Please keep it loaded with
+ all (new) fields
+
+ @param rr the object to copy from
+ @return the copied object]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isAnyLocation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check whether the given <em>host/rack</em> string represents an arbitrary
+ host name.
+
+ @param hostName <em>host/rack</em> on which the allocation is desired
+ @return whether the given <em>host/rack</em> string represents an arbitrary
+ host name]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the request.
+ @return <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the request
+ @param priority <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="getResourceName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource 
+ (host/rack) is acceptable.
+ 
+ @return resource (e.g. <em>host/rack</em>) on which the allocation 
+                  is desired]]>
+      </doc>
+    </method>
+    <method name="setResourceName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the resource name (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource name
+ (e.g. host/rack) is acceptable. 
+ 
+ @param resourceName (e.g. <em>host/rack</em>) on which the 
+                     allocation is desired]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ @param numContainers number of containers required with the given 
+                      specifications]]>
+      </doc>
+    </method>
+    <method name="getRelaxLocality" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+ 
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execSpec" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionTypeRequest</code> of the requested container.
+
+ @param execSpec
+          ExecutionTypeRequest of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionTypeRequest" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setRelaxLocality"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[<p>For a request at a network hierarchy level, set whether locality can be relaxed
+ to that level and beyond.<p>
+ 
+ <p>If the flag is off on a rack-level <code>ResourceRequest</code>,
+ containers at that request's priority will not be assigned to nodes on that
+ request's rack unless requests specifically for those nodes have also been
+ submitted.<p>
+ 
+ <p>If the flag is off on an {@link ResourceRequest#ANY}-level
+ <code>ResourceRequest</code>, containers at that request's priority will
+ only be assigned on racks for which specific requests have also been
+ submitted.<p>
+ 
+ <p>For example, to request a container strictly on a specific node, the
+ corresponding rack-level and any-level requests should have locality
+ relaxation set to false.  Similarly, to request a container strictly on a
+ specific rack, the corresponding any-level request should have locality
+ relaxation set to false.<p>
+ 
+ @param relaxLocality whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this Resource Request. If this is set, all
+ containers allocated to satisfy this resource-request will be only on those
+ nodes that satisfy this node-label-expression.
+  
+ Please note that node label expression now can only take effect when the
+ resource request has resourceName = ANY
+ 
+ @return node-label-expression]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodelabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node label expression of this resource request. Now only support
+ specifying a single node label. In the future we will support more complex
+ node label expression specification like {@code AND(&&), OR(||)}, etc.
+ 
+ Any please note that node label expression now can only take effect when
+ the resource request has resourceName = ANY
+ 
+ @param nodelabelExpression
+          node-label-expression of this ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+
+ @return the <em>ID</em> corresponding to this allocation request.]]>
+      </doc>
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestID" type="long"/>
+      <doc>
+      <![CDATA[Set the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+ <p>
+ If the ID is not set, scheduler will continue to work as previously and all
+ allocated {@code Container}(s) will have the default ID, -1.
+
+ @param allocationRequestID the <em>ID</em> corresponding to this allocation
+                            request.]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <field name="ANY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The constant string representing no locality.
+ It should be used by all references that want to pass an arbitrary host
+ name in.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[{@code ResourceRequest} represents the request made
+ by an application to the {@code ResourceManager}
+ to obtain various {@code Container} allocations.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Priority} of the request.</li>
+   <li>
+     The <em>name</em> of the host or rack on which the allocation is
+     desired. A special value of <em>*</em> signifies that
+     <em>any</em> host/rack is acceptable to the application.
+   </li>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required
+     by the application.
+   </li>
+   <li>
+     A boolean <em>relaxLocality</em> flag, defaulting to {@code true},
+     which tells the {@code ResourceManager} if the application wants
+     locality to be loose (i.e. allows fall-through to rack or <em>any</em>)
+     or strict (i.e. specify hard constraint on resource allocation).
+   </li>
+ </ul>
+ 
+ @see Resource
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <class name="ResourceRequest.ResourceRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+ @see ResourceRequest#setPriority(Priority)
+ @param priority <code>priority</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceName" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>resourceName</code> of the request.
+ @see ResourceRequest#setResourceName(String)
+ @param resourceName <code>resourceName</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="capability" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>capability</code> of the request.
+ @see ResourceRequest#setCapability(Resource)
+ @param capability <code>capability</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="numContainers" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>numContainers</code> of the request.
+ @see ResourceRequest#setNumContainers(int)
+ @param numContainers <code>numContainers</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="relaxLocality" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[Set the <code>relaxLocality</code> of the request.
+ @see ResourceRequest#setRelaxLocality(boolean)
+ @param relaxLocality <code>relaxLocality</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="nodeLabelExpression" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>nodeLabelExpression</code> of the request.
+ @see ResourceRequest#setNodeLabelExpression(String)
+ @param nodeLabelExpression
+          <code>nodeLabelExpression</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionTypeRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionTypeRequest
+          <code>executionTypeRequest</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request with 'ensure
+ execution type' flag set to true.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request.
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+ @see ResourceRequest#setAllocationRequestId(long)
+ @param allocationRequestId
+          <code>allocationRequestId</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link ResourceRequest} object.
+ @return {@link ResourceRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link ResourceRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <class name="ResourceRequest.ResourceRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ResourceRequest.ResourceRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <class name="ResourceUtilization" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceUtilization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceUtil" type="org.apache.hadoop.yarn.api.records.ResourceUtilization"/>
+    </method>
+    <method name="getVirtualMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get used <em>virtual memory</em>.
+
+ @return <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setVirtualMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vmem" type="int"/>
+      <doc>
+      <![CDATA[Set used <em>virtual memory</em>.
+
+ @param vmem <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>physical memory</em>.
+
+ @return <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setPhysicalMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <doc>
+      <![CDATA[Set <em>physical memory</em>.
+
+ @param pmem <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getCPU" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>CPU</em> utilization.
+
+ @return <em>CPU utilization</em> normalized to 1 CPU]]>
+      </doc>
+    </method>
+    <method name="setCPU"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Set <em>CPU</em> utilization.
+
+ @param cpu <em>CPU utilization</em> normalized to 1 CPU]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ <code>ResourceUtilization</code> models the utilization of a set of computer
+ resources in the cluster.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <class name="SignalContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <class name="StrictPreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrictPreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of {@link PreemptionContainer} specifying containers owned by
+ the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>.
+ @return the set of {@link ContainerId} to be preempted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of particular allocations to be reclaimed. The platform will
+ reclaim exactly these resources, so the <code>ApplicationMaster</code> (AM)
+ may attempt to checkpoint work or adjust its execution plan to accommodate
+ it. In contrast to {@link PreemptionContract}, the AM has no flexibility in
+ selecting which resources to return to the cluster.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIdentifier" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier.
+ @return token identifier]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password
+ @return token password]]>
+      </doc>
+    </method>
+    <method name="getKind" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return token kind]]>
+      </doc>
+    </method>
+    <method name="getService" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service to which the token is allocated.
+ @return service to which the token is allocated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>Token</code> is the security entity used by the framework
+ to verify authenticity of any resource.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Token -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <class name="UpdateContainerError" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerError"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+    </method>
+    <method name="getReason" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get reason why the update request was not satisfiable.
+ @return Reason]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set reason why the update request was not satisfiable.
+ @param reason Reason]]>
+      </doc>
+    </method>
+    <method name="getCurrentContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current container version.
+ @return Current container Version.]]>
+      </doc>
+    </method>
+    <method name="setCurrentContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="currentVersion" type="int"/>
+      <doc>
+      <![CDATA[Set current container version.
+ @param currentVersion Current container version.]]>
+      </doc>
+    </method>
+    <method name="getUpdateContainerRequest" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@code UpdateContainerRequest} that was not satisfiable.
+ @return UpdateContainerRequest]]>
+      </doc>
+    </method>
+    <method name="setUpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Set the {@code UpdateContainerRequest} that was not satisfiable.
+ @param updateContainerRequest Update Container Request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerError} is used by the Scheduler to notify the
+ ApplicationMaster of an UpdateContainerRequest it cannot satisfy due to
+ an error in the request. It includes the update request as well as
+ a reason for why the request was not satisfiable.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <class name="UpdateContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="targetCapability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="targetExecutionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="getContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerVersion" type="int"/>
+      <doc>
+      <![CDATA[Set the current version of the container.
+ @param containerVersion of the container]]>
+      </doc>
+    </method>
+    <method name="getContainerUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code> of the container.
+ @return <code>ContainerUpdateType</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code> of the container.
+ @param updateType of the Container]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container.
+ @param containerId <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the target <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the target <code>ExecutionType</code> of the container.
+ @param executionType <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerRequest} represents the request made by an
+ application to the {@code ResourceManager} to update an attribute of a
+ {@code Container} such as its Resource allocation or (@code ExecutionType}
+ <p>
+ It includes:
+ <ul>
+   <li>version for the container.</li>
+   <li>{@link ContainerId} for the container.</li>
+   <li>
+     {@link Resource} capability of the container after the update request
+     is completed.
+   </li>
+   <li>
+     {@link ExecutionType} of the container after the update request is
+     completed.
+   </li>
+ </ul>
+
+ Update rules:
+ <ul>
+   <li>
+     Currently only ONE aspect of the container can be updated per request
+     (user can either update Capability OR ExecutionType in one request..
+     not both).
+   </li>
+   <li>
+     There must be only 1 update request per container in an allocate call.
+   </li>
+   <li>
+     If a new update request is sent for a container (in a subsequent allocate
+     call) before the first one is satisfied by the Scheduler, it will
+     overwrite the previous request.
+   </li>
+ </ul>
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <class name="UpdatedContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdatedContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdatedContainer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Static Factory method.
+
+ @param updateType ContainerUpdateType
+ @param container Container
+ @return UpdatedContainer]]>
+      </doc>
+    </method>
+    <method name="getUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code>.
+ @return ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="setUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code>.
+ @param updateType ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="getContainer" return="org.apache.hadoop.yarn.api.records.Container"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Container</code>.
+ @return Container]]>
+      </doc>
+    </method>
+    <method name="setContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Set the <code>Container</code>.
+ @param container Container]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object that encapsulates an updated container and the
+ type of Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.URL -->
+  <class name="URL" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="URL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="file" type="java.lang.String"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the scheme of the URL.
+ @return scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="setScheme"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheme of the URL
+ @param scheme scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="getUserInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user info of the URL.
+ @return user info of the URL]]>
+      </doc>
+    </method>
+    <method name="setUserInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user info of the URL.
+ @param userInfo user info of the URL]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the host of the URL.
+ @return host of the URL]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the host of the URL.
+ @param host host of the URL]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the port of the URL.
+ @return port of the URL]]>
+      </doc>
+    </method>
+    <method name="setPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the port of the URL
+ @param port port of the URL]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file of the URL.
+ @return file of the URL]]>
+      </doc>
+    </method>
+    <method name="setFile"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the file of the URL.
+ @param file file of the URL]]>
+      </doc>
+    </method>
+    <method name="toPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+    </method>
+    <method name="fromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="fromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[<p><code>URL</code> represents a serializable {@link java.net.URL}.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.URL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <class name="YarnApplicationAttemptState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of a <code>RMAppAttempt</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <class name="YarnApplicationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of an <code>ApplicationMaster</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+  <class name="YarnClusterMetrics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>NodeManager</code>s in the cluster.
+ @return number of <code>NodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumDecommissionedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>DecommissionedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>DecommissionedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumActiveNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>ActiveNodeManager</code>s in the cluster.
+ 
+ @return number of <code>ActiveNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumLostNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>LostNodeManager</code>s in the cluster.
+ 
+ @return number of <code>LostNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumUnhealthyNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>UnhealthyNodeManager</code>s in the cluster.
+ 
+ @return number of <code>UnhealthyNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumRebootedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>RebootedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>RebootedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>YarnClusterMetrics</code> represents cluster metrics.</p>
+ 
+ <p>Currently only number of <code>NodeManager</code>s is provided.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.impl">
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timeline">
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <class name="TimelineAbout" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineAbout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineAbout" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAbout" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAbout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersionBuiltOn" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersionBuiltOn" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <class name="TimelineDelegationTokenResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getContent" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="content" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[The response of delegation token related request]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID
+ 
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description
+ 
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description
+ 
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner
+ 
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ 
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string
+ 
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string
+ 
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string
+ 
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string
+ 
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain
+ 
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain
+ 
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain
+ 
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain
+ 
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline domain, which is used
+ to a user to host a number of timeline entities, isolating them from others'.
+ The user can also define the reader and writer users/groups for the the
+ domain, which is used to control the access to its entities.
+ </p>
+ 
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <class name="TimelineDomains" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomains"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDomains" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of domains
+ 
+ @return a list of domains]]>
+      </doc>
+    </method>
+    <method name="addDomain"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <doc>
+      <![CDATA[Add a single domain into the existing domain list
+ 
+ @param domain
+          a single domain]]>
+      </doc>
+    </method>
+    <method name="addDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of domains into the existing domain list
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <method name="setDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the domain list to the given list of domains
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline domains.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of entities
+ 
+ @return a list of entities]]>
+      </doc>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+      <doc>
+      <![CDATA[Add a single entity into the existing entity list
+ 
+ @param entity
+          a single entity]]>
+      </doc>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of entities into the existing entity list
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the entity list to the given list of entities
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time of the entity
+ 
+ @return the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the start time of the entity
+ 
+ @param startTime
+          the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events related to the entity
+ 
+ @return a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event related to the entity to the existing event list
+ 
+ @param event
+          a single event related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of events related to the entity to the existing event list
+ 
+ @param events
+          a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events related to the entity
+ 
+ @param events
+          events a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="getRelatedEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the related entities
+ 
+ @return the related entities]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add an entity to the existing related entity map
+ 
+ @param entityType
+          the entity type
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of related entities to the existing related entity map
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="setRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the related entity map to the given map of related entities
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="getPrimaryFilters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the primary filters
+ 
+ @return the primary filters]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add a single piece of primary filter to the existing primary filter map
+ 
+ @param key
+          the primary filter key
+ @param value
+          the primary filter value]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of primary filters to the existing primary filter map
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="setPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the primary filter map to the given map of primary filters
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="getOtherInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the other information of the entity
+ 
+ @return the other information of the entity]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of other information of the entity to the existing other info
+ map
+ 
+ @param key
+          the other information key
+ @param value
+          the other information value]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of other information of the entity to the existing other info map
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="setOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the other info map to the given map of other information
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="getDomainId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ID of the domain that the entity is to be put
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setDomainId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domainId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ID of the domain that the entity is to be put
+ 
+ @param domainId
+          the name space ID]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The class that contains the the meta information of some conceptual entity
+ and its related events. The entity can be an application, an application
+ attempt, a container or whatever the user-defined object.
+ </p>
+ 
+ <p>
+ Primary filters will be used to index the entities in
+ <code>TimelineStore</code>, such that users should carefully choose the
+ information they want to store as the primary filters. The remaining can be
+ stored as other information.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <class name="TimelineEntityGroupId" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntityGroupId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the
+ <code>TimelineEntityGroupId</code>.
+
+ @return <code>ApplicationId</code> of the
+         <code>TimelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getTimelineEntityGroupId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>timelineEntityGroupId</code>.
+
+ @return <code>timelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntityGroupIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TIMELINE_ENTITY_GROUPID_STR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>TimelineEntityGroupId</code> is an abstract way for
+ timeline service users to represent #a group of related timeline data.
+ For example, all entities that represents one data flow DAG execution
+ can be grouped into one timeline entity group. </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the event
+ 
+ @return the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the timestamp of the event
+ 
+ @param timestamp
+          the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="getEventType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the event type
+ 
+ @return the event type]]>
+      </doc>
+    </method>
+    <method name="setEventType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the event type
+ 
+ @param eventType
+          the event type]]>
+      </doc>
+    </method>
+    <method name="getEventInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the information of the event
+ 
+ @return the information of the event]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of the information of the event to the existing information
+ map
+ 
+ @param key
+          the information key
+ @param value
+          the information value]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of the information of the event to the existing information map
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="setEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the information map to the given map of the information of the event
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The class that contains the information of an event that is related to some
+ conceptual entity of an application. Users are free to define what the event
+ means, such as starting an application, getting allocated a container and
+ etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <class name="TimelineEvents" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEvents"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAllEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link EventsOfOneEntity} instances
+ 
+ @return a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventsOfOneEntity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity"/>
+      <doc>
+      <![CDATA[Add a single {@link EventsOfOneEntity} instance into the existing list
+ 
+ @param eventsOfOneEntity
+          a single {@link EventsOfOneEntity} instance]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link EventsOfOneEntity} instances into the existing list
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link EventsOfOneEntity} instances
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events, which are categorized according to
+ their related entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <class name="TimelineEvents.EventsOfOneEntity" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEvents.EventsOfOneEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events
+ 
+ @return a list of events]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event to the existing event list
+ 
+ @param event
+          a single event]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of event to the existing event list
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events that are only related to one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <class name="TimelinePutResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelinePutError} instances
+ 
+ @return a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelinePutError} instance into the existing list
+ 
+ @param error
+          a single {@link TimelinePutError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelinePutError} instances into the existing list
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelinePutError} instances
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are errors
+ in storing individual entity objects, they will be indicated in the list of
+ errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+  <class name="TimelinePutResponse.TimelinePutError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutResponse.TimelinePutError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code
+ 
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCode" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code
+ 
+ @param errorCode
+          an error code]]>
+      </doc>
+    </method>
+    <field name="NO_START_TIME" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned when no start time can be found when putting an
+ entity. This occurs when the entity does not already exist in the store
+ and it is put with no start time or events specified.]]>
+      </doc>
+    </field>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when putting an
+ entity.]]>
+      </doc>
+    </field>
+    <field name="SYSTEM_FILTER_CONFLICT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user specifies the timeline system reserved
+ filter key]]>
+      </doc>
+    </field>
+    <field name="ACCESS_DENIED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to access the timeline data]]>
+      </doc>
+    </field>
+    <field name="NO_DOMAIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity doesn't have an valid domain ID]]>
+      </doc>
+    </field>
+    <field name="FORBIDDEN_RELATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to relate the entity to another
+ one in different domain]]>
+      </doc>
+    </field>
+    <field name="EXPIRED_ENTITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity start time is before the eviction
+ period of old data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timelineservice">
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <class name="ApplicationAttemptEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <class name="ApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="isApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an ApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an ApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getApplicationEvent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <param name="eventId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param te TimelineEntity object.
+ @param eventId event with this id needs to be fetched
+ @return TimelineEvent if TimelineEntity contains the desired event.]]>
+      </doc>
+    </method>
+    <field name="QUEUE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents an application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <class name="ClusterEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClusterEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a YARN cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <class name="ContainerEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a container belonging to an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <class name="FlowActivityEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowActivityEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="java.lang.String, long, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reuse the base class equals method.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reuse the base class hashCode method.]]>
+      </doc>
+    </method>
+    <method name="getCluster" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cluster" type="java.lang.String"/>
+    </method>
+    <method name="getDate" return="java.util.Date"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFlowName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+    </method>
+    <method name="addFlowRun"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="run" type="org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity"/>
+    </method>
+    <method name="addFlowRuns"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runs" type="java.util.Collection"/>
+    </method>
+    <method name="getFlowRuns" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNumberOfRuns" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLUSTER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Entity that represents a record for flow activity. It's essentially a
+ container entity for flow runs with limited information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <class name="FlowRunEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowRunEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowRunEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="java.lang.String"/>
+    </method>
+    <method name="getRunId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRunId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runId" type="long"/>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+    </method>
+    <method name="getMaxEndTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxEndTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+    </method>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_END_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a flow run.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <class name="HierarchicalTimelineEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getParent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getChildren" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="child" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <field name="PARENT_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CHILDREN_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends timeline entity and defines parent-child relationships
+ with other entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <class name="QueueEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QueueEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a queue.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <class name="SubApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSubApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an SubApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an SubApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="java.lang.String"/>
+    </method>
+    <field name="YARN_APPLICATION_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a user defined entities to be stored under sub
+ application table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[This class hosts a set of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ The constuctor is used to construct a proxy {@link TimelineEntity} or its
+ subclass object from the real entity object that carries information.
+ </p>
+
+ <p>
+ It is usually used in the case where we want to recover class polymorphism
+ after deserializing the entity from its JSON form.
+ </p>
+ @param entity the real entity that carries information]]>
+      </doc>
+    </constructor>
+    <constructor name="TimelineEntity" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getIdentifier" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdentifier" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getConfigs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="getMetrics" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+    </method>
+    <method name="getEvents" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.NavigableSet"/>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.Set"/>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <method name="getIsRelatedToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getRelatesToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="addRelatesToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="setRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTs" type="java.lang.Long"/>
+    </method>
+    <method name="setUID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uidKey" type="java.lang.String"/>
+      <param name="uId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set UID in info which will be then used for query by UI.
+ @param uidKey key for UID in info.
+ @param uId UID to be set for the key.]]>
+      </doc>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="getReal" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdPrefix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdPrefix"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdPrefix" type="long"/>
+      <doc>
+      <![CDATA[Sets idPrefix for an entity.
+ <p>
+ <b>Note</b>: Entities will be stored in the order of idPrefix specified.
+ If users decide to set idPrefix for an entity, they <b>MUST</b> provide
+ the same prefix for every update of this entity.
+ </p>
+ Example: <blockquote><pre>
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+ </pre></blockquote>
+ Users can use {@link TimelineServiceHelper#invertLong(long)} to invert
+ the prefix if necessary.
+
+ @param entityIdPrefix prefix for an entity.]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_INFO_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENTITY_PREFIX" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The basic timeline entity data structure for timeline service v2. Timeline
+ entity objects are not thread safe and should not be accessed concurrently.
+ All collection members will be initialized into empty collections. Two
+ timeline entities are equal iff. their type and id are identical.
+
+ All non-primitive type, non-collection members will be initialized into null.
+ User should set the type and id of a timeline entity to make it valid (can be
+ checked by using the {@link #isValid()} method). Callers to the getters
+ should perform null checks for non-primitive type, non-collection members.
+
+ Callers are recommended not to alter the returned collection objects from the
+ getters.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <class name="TimelineEntityType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isParent" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a parent of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is parent of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="isChild" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a child of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is child of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="matches" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="typeString" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether the type of this entity matches the type indicated by the input
+ argument.
+
+ @param typeString entity type represented as a string.
+ @return true, if string representation of this entity type matches the
+     entity type passed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Defines type of entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventId" type="java.lang.String"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ts" type="long"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <field name="INVALID_TIMESTAMP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains the information of an event that belongs to an entity.
+ Users are free to define what the event means, such as starting an
+ application, container being allocated, etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <class name="TimelineMetric" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricType" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricId" type="java.lang.String"/>
+    </method>
+    <method name="getRealtimeAggregationOp" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the real time aggregation operation of this metric.
+
+ @return Real time aggregation operation]]>
+      </doc>
+    </method>
+    <method name="setRealtimeAggregationOp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"/>
+      <doc>
+      <![CDATA[Set the real time aggregation operation of this metric.
+
+ @param op A timeline metric operation that the metric should perform on
+           real time aggregations]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <param name="value" type="java.lang.Number"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLatestSingleValueMetric" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Get the latest timeline metric as single value type.
+
+ @param metric Incoming timeline metric
+ @return The latest metric in the incoming metric]]>
+      </doc>
+    </method>
+    <method name="getSingleDataTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data timestamp of the metric.
+
+ @return the single data timestamp]]>
+      </doc>
+    </method>
+    <method name="getSingleDataValue" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data value of the metric.
+
+ @return the single data value]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state in a stateless fashion. The assumption here is
+ baseAggregatedMetric and latestMetric should be single value data if not
+ null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="state" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state. The assumption here is baseAggregatedMetric and
+ latestMetric should be single value data if not null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @param state Operation state
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains the information of a metric that is related to some
+ entity. Metric can either be a time series or single value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <class name="TimelineWriteResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelineWriteError} instances.
+
+ @return a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelineWriteError} instance into the existing list.
+
+ @param error
+          a single {@link TimelineWriteError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelineWriteError} instances into the existing list.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelineWriteError} instances.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are
+ errors in storing individual entity objects, they will be indicated in the
+ list of errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <class name="TimelineWriteResponse.TimelineWriteError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteResponse.TimelineWriteError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id.
+
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id.
+
+ @param id the entity Id.]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type.
+
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type.
+
+ @param type the entity type.]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code.
+
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code.
+
+ @param code an error code.]]>
+      </doc>
+    </method>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when storing an
+ entity.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+  <class name="UserEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UserEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="UserEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+</package>
+<package name="org.apache.hadoop.yarn.conf">
+  <!-- start class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+  <class name="YarnConfiguration" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnConfiguration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isAclEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>. On an HA cluster,
+ this fetches the address corresponding to the RM identified by
+ {@link #RM_HA_ID}.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+    </method>
+    <method name="useHttps" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="shouldRMFailFast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isDistSchedulingEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOpportunisticContainerAllocationEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV2Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.2 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.2 is enabled. V.2 refers to a
+ version greater than equal to 2 but smaller than 3.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1 is enabled. V.1 refers to a
+ version greater than equal to 1 but smaller than 2.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV15Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1,5 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="systemMetricsPublisherEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the system publisher is enabled.
+
+ @param conf the configuration
+ @return whether the system publisher is enabled.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="DR_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CS_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_POLICY_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CORE_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_RESOURCES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEBUG_NM_DELETE_DELAY_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay before deleting resource to ease debugging of NM issues]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_CONTAINER_DEBUG_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create client IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create server IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create serializeable records.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RPC class implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HOSTNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_EPOCH" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the applications manager interface in the RM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of application tags.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAG_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of each application tag.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MASTER_SERVICE_PROCESSORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AUTO_UPDATE_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AUTO_UPDATE_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the RM.]]>
+      </doc>
+    </field>
+    <field name="RM_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle applications manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AMLAUNCHER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to launch/cleanup AM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AMLAUNCHER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_CONNECT_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retry times to connect with NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_CONNECT_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Miniumum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the port should be included or not in the node name. The node name
+ is used by the scheduler for resource requests allocation location 
+ matching. Typically this is just the hostname, using the port is needed
+ when using minicluster and specific NM are required.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUEUE_PLACEMENT_RULES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configured scheduler queue placement rules.]]>
+      </doc>
+    </field>
+    <field name="USER_GROUP_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[UserGroupMappingPlacementRule configuration string.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_UI_ACTIONS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Resource Manager webapp ui actions]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the RM should enable Reservation System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The PlanFollower for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The step size of the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum periodicity for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_ENABLE_MONITORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable periodic monitor threads.
+ @see #RM_SCHEDULER_MONITOR_POLICIES]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ENABLE_MONITORS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MONITOR_POLICIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of SchedulingEditPolicy classes affecting the scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SSL_SERVER_RESOURCE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable YARN WebApp V2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_WEBAPP_UI2_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_WARFILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for application master reporting.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a node manager is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_RESERVATION_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are reservation acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_RESERVATION_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL of who can be admin of YARN cluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_APP_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL used in case none is found. Allows nothing.]]>
+      </doc>
+    </field>
+    <field name="OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether opportunistic container allocation
+  is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of nodes to be used by the Opportunistic Container allocator for
+ dispatching containers during container allocation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency for computing least loaded NMs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comparator for determining node load for scheduling of opportunistic
+ containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LIMIT_STDEV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value of standard deviation used for calculation of queue limit
+ thresholds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the lower-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the upper-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min queue wait time for a container at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue wait time for a container queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_INTERMEDIATE_DATA_ENCRYPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable intermediate-data encryption at YARN level. For now, this
+ only is used by the FileSystemRMStateStore to setup right file-system
+ security attributes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts for
+ an application, if unset by user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GLOBAL_RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts.
+ It's a global setting for all application masters.]]>
+      </doc>
+    </field>
+    <field name="RM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The keytab for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing). Only applicable
+ when the http authentication type is kerberos.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cross origin (CORS) support.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a container is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to include.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable submission pre-processor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with hosts for the submission processor to handle.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submission processor refresh interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to exclude.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle resource tracker calls.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the resource scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM set next Heartbeat interval for NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of worker threads that write the history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM. This configuration setting is for ATS V1.
+  This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM and NM. This configuration setting is for
+  ATS v2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by RM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_CONF_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper interaction configs]]>
+      </doc>
+    </field>
+    <field name="RM_ZK_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_RM_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper znode limit]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_RETRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_TIMEOUT_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_APPID_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_APPID_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Index at which the RM Delegation Token ids will be split so that the
+ delegation token znodes stored in the zookeeper RM state store will be
+ stored as two different znodes (parent-child).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_AUTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_ROOT_NODE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Root node ACLs for fencing]]>
+      </doc>
+    </field>
+    <field name="RM_HA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[HA related configs]]>
+      </doc>
+    </field>
+    <field name="RM_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Store the related configuration files in File System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILE_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LEVELDB_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_MAX_LOGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKConfigurationStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_AUTHORIZATION_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_EMBEDDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This property should never be set to {@code false}.">
+      <doc>
+      <![CDATA[This property controls whether {@link ActiveStandbyElector} leader
+ election should be used when {@link #CURATOR_LEADER_ELECTOR} is
+ {@code false}.
+
+ @deprecated This property should never be set to {@code false}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_EMBEDDED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.">
+      <doc>
+      <![CDATA[The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
+
+ @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.]]>
+      </doc>
+    </field>
+    <field name="AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_BASE_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_FC_ELECTOR_ZK_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of zookeeper operation retry times in ActiveStandbyElector]]>
+      </doc>
+    </field>
+    <field name="CURATOR_LEADER_ELECTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.">
+      <doc>
+      <![CDATA[Whether to use the Curator-based elector for leader election.
+
+ @deprecated Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the persistent store.]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemRMStateStore]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_RM_STATE_STORE_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the leveldb database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM keeps.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM state store keeps, by
+ default equals to DEFAULT_RM_MAX_COMPLETED_APPLICATIONS]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APPLICATION_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application name]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_TYPE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type length]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default queue name]]>
+      </doc>
+    </field>
+    <field name="RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Buckets (in minutes) for the number of apps running in each queue.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default sizes of the runtime metric buckets in minutes.]]>
+      </doc>
+    </field>
+    <field name="RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout(msec) for an untracked node to remain in shutdown or decommissioned
+ state.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM proxy users' prefix]]>
+      </doc>
+    </field>
+    <field name="YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable loading resource-types.xml at client side.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications
+ to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Period in seconds of the poll timer task inside DecommissioningNodesWatcher
+ to identify and take care of DECOMMISSIONING nodes missing regular heart beat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for all node manager configs.]]>
+      </doc>
+    </field>
+    <field name="NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIST_SCHEDULING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether distributed scheduling is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DIST_SCHEDULING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that will be sent to containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that containers may override rather than use NodeManager's default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[address of node manager IPC.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address or the NM.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[who will execute(launch) the containers.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_STATE_TRANSITION_LISTENERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of container state transition listeners.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adjustment to make to the container os scheduling priority.
+ The valid values for this could vary depending on the platform.
+ On Linux, higher values mean run the containers at a less 
+ favorable priority than the NM. 
+ The value specified is an int.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MGR_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DELETE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in cleanup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of directories to store localized files in.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of files in each localized directories
+ Avoid tuning this too low.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the localizer IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the collector service IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval in between cache cleanups.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target size of localizer cache in MB, per nodemanager. It is a target
+ retention size that only includes resources with PUBLIC and PRIVATE
+ visibility and excludes resources with APPLICATION visibility]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle localization requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_FETCH_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to use for localization fetching.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to store container logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle log aggregation in node manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permissions for container logs.]]>
+      </doc>
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk Validator.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum size of contain's diagnostics to keep for relaunching container
+ case.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum container restart interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which the delayed token removal thread runs]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegation Token renewer thread count]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PRIVILEGES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for node IP caching. -1 disables the caching]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How many diagnostics/failure messages can be saved in RM for
+ log aggregation. It also defines the number of diagnostics/failure
+ messages can be shown in log aggregation web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to enable log aggregation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_CONTROLLER_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait before deleting aggregated logs, -1 disables.
+ Be careful set this too small and you will spam the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait between aggregated log retention checks. If set to
+ a value {@literal <=} 0 then the value is computed as one-tenth of the
+ log retention setting. Be careful set this too small and you will spam
+ the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation status
+ is reported from NodeManager exceeds the configured value, RM will report
+ log aggregation status for this NodeManager as TIME_OUT]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seconds to retain logs on the NodeManager. Only applicable if Log
+ aggregation is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how often NMs wake up and upload log files]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DELETION_THREADS_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in log cleanup. Only applicable if Log aggregation
+ is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to aggregate logs to.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The remote log dir will be created at
+ NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_WEBSERVICE_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_TRACKING_URL_GENERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SYSTEM_RESERVED_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that has been reserved for non-yarn use.]]>
+      </doc>
+    </field>
+    <field name="NM_PMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_PMEM_RATIO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Conversion ratio for physical memory to virtual memory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_PMEM_RATIO" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of Virtual CPU Cores which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count logical processors(like hyperthreads) as cores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PCORES_VCORES_MULTIPLIER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Multiplier to convert physical cores to vcores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PCORES_VCORES_MULTIPLIER" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Percentage of overall CPU which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable or disable node hardware capability detection.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cpu isolation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CPU_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for disk configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for disk operations is enabled.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for network bandwidth is
+ enabled. Work in progress: This configuration parameter may be
+ changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Network as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the interface to be used for applying network throttling rules.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total available outbound bandwidth on the node. Work in
+ progress: This configuration parameter may be changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total outbound bandwidth available to YARN containers.
+ defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for computation resources, example of computation resources like
+ GPU / FPGA / TPU, etc.]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for gpu configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTOMATICALLY_DISCOVER_GPU_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_GPU_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls where to how to invoke GPU binaries]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_GPU_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp https address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable CORS filter.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor resource in a node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates containers current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_PROCESS_TREE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates process tree resource utilization.]]>
+      </doc>
+    </field>
+    <field name="PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable container metrics.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_PERIOD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container metrics flush period. -1 for flush on completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The delay time ms to unregister container metrics after completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable disks' health checker. Default is true. An expert level
+ configuration property.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running disks' health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, disks' health is checked every 2 minutes.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_HEALTHY_DISKS_FRACTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum fraction of number of disks to be healthy for the nodemanager
+ to launch new containers. This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, at least 25% of disks are to be healthy to say that the node is
+ healthy in terms of disks.]]>
+      </doc>
+    </field>
+    <field name="NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum percentage of disk space that can be used after which a disk is
+ marked as offline. Values can range from 0.0 to 100.0. If the value is
+ greater than or equal to 100, NM will check for full disk. This applies to
+ nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, 90% of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The low threshold percentage of disk space used when an offline disk is
+ marked as online. Values can range from 0.0 to 100.0. The value shouldn't
+ be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is
+ more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be
+ set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on a local dir for it to be used.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="NM_WM_HIGH_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on an offline
+ disk for it to be marked as online.  The value should not be less
+ than NM_MIN_PER_DISK_FREE_SPACE_MB.  If its value is less than
+ NM_MIN_PER_DISK_FREE_SPACE_MB or is not set, it will be set to the
+ same value as NM_MIN_PER_DISK_FREE_SPACE_MB.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, all of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check script time out period.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health check script to run.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The arguments to pass to the health check script.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JVM options used on forking ContainerLocalizer process
+      by container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log level of container localizer process.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Docker image name(For DockerContainerExecutor).]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the docker executor (For DockerContainerExecutor).]]>
+      </doc>
+    </field>
+    <field name="NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default docker executor (For DockerContainerExecutor).]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for runtime configuration constants.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The allowed values are:
+ <ul>
+   <li>default</li>
+   <li>docker</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of allowed runtimes when using LinuxContainerExecutor.]]>
+      </doc>
+    </field>
+    <field name="DOCKER_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Capabilities allowed (and added by default) for docker containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[These are the default capabilities added by docker. We'll use the same
+ set here. While these may not be case-sensitive from a docker
+ perspective, it is best to keep these uppercase.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ENABLE_USER_REMAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[enable user remapping.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set enable user remapping as false by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable uids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower uid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable gids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower gid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request. This
+  network must be one of the (configurable) set of allowed container
+  networks.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request and
+ no default network is configured.
+ .]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_EXECUTOR_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the Linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The UNIX group that the linux-container-executor should run as.
+ This is intended to be set as part of container-executor.cfg.]]>
+      </doc>
+    </field>
+    <field name="NM_NONSECURE_MODE_LIMIT_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If linux-container-executor should limit itself to one user
+ when running in non-secure mode.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_LOCAL_USER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The UNIX user that containers will run as when Linux-container-executor
+ is used in nonsecure mode (a use case for this is using cgroups).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LOCAL_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_USER_PATTERN_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed pattern for UNIX user names enforced by 
+ Linux-container-executor when used in nonsecure mode (use case for this 
+ is using cgroups). The default value is taken from /usr/sbin/adduser]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_USER_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_RESOURCES_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The type of resource enforcement to use with the
+  linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_HIERARCHY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path the linux container executor should use for cgroups]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the apps should run in strict resource usage mode(not allowed to
+ use spare CPU)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_TIMEOUT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFALUT_RM_PROXY_TIMEOUT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_CONNECTION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_CONNECTION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container. This is required due to what 
+ it seems a race condition because the SIGTERM/SIGKILL is asynch.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay between attempts to remove linux cgroup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if memory and CPU limits will be set for the Windows Job
+ Object for the containers launched by the default container executor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_SECURE_CONTAINER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[/* The Windows group that the windows-secure-container-executor should run as.]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress aggregated logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the node manager.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_REMOTE_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the NM state database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_SUPERVISED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_SUPERVISED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS_PARAMETERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for Proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[YARN Service Level Authorization]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[No. of milliseconds to wait between sending a SIGTERM and SIGKILL
+ to a running container]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROCESS_KILL_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for a process to come up when trying to cleanup
+ container resources]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PROCESS_KILL_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APPLICATION_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
+ entries]]>
+      </doc>
+    </field>
+    <field name="AMRM_PROXY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether AMRMProxy is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default platform-agnostic CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries. The parameter expansion marker
+ will be replaced with real parameter expansion marker ('%' for Windows and
+ '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+ will be replaced as $VAR on Linux, and %VAR% on Windows.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Default platform-specific CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries constructed based on the client
+ OS environment expansion syntax.
+ </p>
+ <p>
+ Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.
+ </p>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CONTAINER_TEMP_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container temp directory]]>
+      </doc>
+    </field>
+    <field name="IS_MINI_YARN_CLUSTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_FIXED_PORTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to use fixed ports with the minicluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_FIXED_PORTS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default is false to be able to run tests concurrently without port
+ conflicts.]]>
+      </doc>
+    </field>
+    <field name="YARN_MINICLUSTER_USE_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the NM should use RPC to connect to the RM. Default is false.
+ Can be set to true only when using fixed ports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_USE_RPC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether users are explicitly trying to control resource monitoring
+ configuration for the MiniYARNCluster. Disabled by default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow changing the memory for the NodeManager in the MiniYARNCluster]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log directory for the containers]]>
+      </doc>
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_BACKUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_VERSION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_UI_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma seperated list of names for UIs hosted in the timeline server
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_WEB_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Relative web path that will serve up this UI (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client settings.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to war file or static content directory for this UI
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting for timeline service v1.5]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline service v2.0.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default schema prefix for hbase tables.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[config param name to override schema prefix.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how often the timeline collector flushes the
+ timeline writer.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_FINAL_VALUE_RETENTION_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how long the final value of
+ a metric of a completed app is retained before merging
+ into the flow sum.]]>
+      </doc>
+    </field>
+    <field name="FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the setting for the location of the coprocessor
+ jar on hdfs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default hdfs location for flowrun coprocessor jar.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that points to an optional HBase configuration
+ (hbase-site.xml file) with settings that will override the ones found on
+ the classpath.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that enables or disables authentication checks
+ for reading timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.Boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default setting for authentication checks for reading timeline
+ service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that lists the users and groups who are allowed
+ to read timeline service v2 data. It is a comma separated list of
+ user, followed by space, then comma separated list of groups.
+ It will allow this list of users and groups to read the data
+ and reject everyone else.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for list of the users who are allowed to read
+ timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how long the final value of a metric of a
+ completed app is retained before merging into the flow sum. Up to this time
+ after an application is completed out-of-order values that arrive can be
+ recognized and discarded at the cost of increased storage.]]>
+      </doc>
+    </field>
+    <field name="ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default version for any flow.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time period for which timeline v2 client will wait for draining
+ leftover entities after stop.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether application history service is
+  enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Application history store class]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Save container meta-info in the application history store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemApplicationHistoryStore]]>
+      </doc>
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether timeline service is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[host:port address for timeline service RPC APIs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The listening endpoint for the timeline service application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_HANDLER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle client RPC API requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_MAX_APPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defines the max number of applications could be fetched using
+ REST API or application history protocol and shown in timeline
+ server web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_MAX_APPS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service store class.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service enable data age off]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service length of time to retain data]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service rolling period. Valid values are daily, half_daily,
+ quarter_daily, and hourly.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Roll a new database each hour.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Implementation specific configuration prefix for Timeline Service
+ leveldb.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb read cache (uncompressed blocks). This is
+ per rolling instance so should be tuned if using rolling leveldb
+ timeline store]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb read cache size if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write buffer size.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write buffer size if no configuration is specified. This
+ is per rolling instance so should be tuned if using rolling leveldb
+ timeline store.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write batch size. This value can be tuned down
+ to reduce lock time for ttl eviction.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write batch size is no configuration is specified]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time read cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time write cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb interval to wait between deletion rounds]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb number of concurrent open files. Tuned this
+ configuration to stay within system limits. This is per rolling instance
+ so should be tuned if using rolling leveldb timeline store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb max open files if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos keytab for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enables cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, max retries (-1 means no limit)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, retry interval]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client policy for whether connections are fatal]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable recovery of timeline service]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STATE_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store class]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_OFFLINE_AGGREGATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[whether the shared cache is enabled/disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the shared cache root directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NESTED_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the level of nested directories before getting to the
+ checksum directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NESTED_LEVEL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STALENESS_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A resource in the InMemorySCMStore is considered stale if the time since
+ the last reference exceeds the staleness period. This value is specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the in-memory store runs its first check to remove
+ dead initial applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_CHECK_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which the in-memory store checks to remove dead initial
+ applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which a cleaner task runs. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the first cleaner task is scheduled. Specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_RESOURCE_SLEEP_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time to sleep between processing each shared cache resource. Specified
+ in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the node manager interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of SCM threads used to handle notify requests from the node
+ manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the client interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle shared cache manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the checksum algorithm implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The replication factor for the node manager uploader for the shared cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_MACHINE_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_USERNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_PASSWORD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PIPELINE_CACHE_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_SUBMIT_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of retries for GetNewApplication and SubmitApplication in
+ {@code FederationClientInterceptor}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST to communicate with
+ each SubCluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST should return
+ partial AppReports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
+ The interval of the yarn client's querying application state after
+ application submission. The unit is millisecond.]]>
+      </doc>
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval that the yarn client library uses to poll the completion
+ status of the asynchronous API of application client protocol.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The duration that the yarn client library waits, cumulatively across polls,
+ for an expected state change to occur. Defaults to -1, which indicates no
+ limit.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of threads in NMClientAsync to process container management
+ events]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_MAX_NM_PROXIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of proxy connections to cache for node managers. If set
+ to a value greater than zero then the cache is enabled and the NMClient
+ and MRAppMaster will cache the specified number of node manager proxies.
+ There will be at max one proxy per node manager. Ex. configuring it to a
+ value of 5 will make sure that client will at max have 5 proxies cached
+ with 5 different node managers. These connections for these proxies will
+ be timed out if idle for more than the system wide idle timeout period.
+ Note that this could cause issues on large clusters as many connections
+ could linger simultaneously and lead to a large number of connection
+ threads. The token used for authentication will be used only at
+ connection creation time. If a new token is received then the earlier
+ connection should be closed in order to use the new token. This and
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related
+ and should be in sync (no need for them to be equal).
+ If the value of this property is zero then the connection cache is
+ disabled and connections will use a zero idle timeout to prevent too
+ many connection threads on large clusters.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_MAX_NM_PROXIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for NM to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each NM attempt to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-labels configurations]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node label store implementation class]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_NODE_LABELS_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for NodeLabelManager]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_NODE_LABELS_STORE_RETRY_POLICY_SPEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_LABELS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to indicate if the node labels feature enabled, by default it's
+ disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABELS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CENTRALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONFIG_NODE_LABELS_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCRIPT_NODE_LABELS_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPLAY_APPS_FOR_LOGGED_IN_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENABLE_REST_APP_SUBMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENABLE_REST_APP_SUBMISSIONS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REST_CSRF" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="XFS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_XFS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline reader.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Marked collector properties as Private since it run as auxillary service.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINERS_LAUNCHER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers launcher implementation to use.]]>
+      </doc>
+    </field>
+    <field name="YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+</package>
+<package name="org.apache.hadoop.yarn.exceptions">
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <class name="ApplicationAttemptNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getApplicationAttemptReport (GetApplicationAttemptReportRequest)}
+ API when the Application Attempt doesn't exist in Application History Server or
+ {@link ApplicationMasterProtocol#allocate(AllocateRequest)} if application
+ doesn't exist in RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <class name="ApplicationIdNotProvidedException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception to be thrown when Client submit an application without
+ providing {@link ApplicationId} in {@link ApplicationSubmissionContext}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <class name="ApplicationNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} API
+ when the Application doesn't exist in RM and AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <class name="ConfigurationException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConfigurationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on unrecoverable configuration errors.
+ An example is container launch error due to configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <class name="ContainerNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getContainerReport (GetContainerReportRequest)}
+ API when the container doesn't exist in AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <class name="ResourceNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when details of an unknown resource type
+ are requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <class name="YarnException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[YarnException indicates exceptions from yarn servers. On the other hand,
+ IOExceptions indicates exceptions from RPC layer.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+  <class name="YARNFeatureNotEnabledException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when a feature is being used which is not enabled
+ yet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <class name="ApplicationInitializationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ApplicationId, java.nio.ByteBuffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-name of the application-submitter
+ 
+ @return user-name]]>
+      </doc>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <method name="getApplicationDataForService" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data sent to the NodeManager via
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ as part of {@link ContainerLaunchContext#getServiceData()}
+ 
+ @return the servicesData for this application.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <class name="ApplicationTerminationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTerminationContext" type="org.apache.hadoop.yarn.api.records.ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application being stopped.
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when stopping an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <interface name="AuxiliaryLocalPathHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getLocalPathForRead" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="size" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing a file of an estimated size
+ for a given Auxiliary Service.
+ @param path the requested path
+ @param size the size of the file that is going to be written
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getAllLocalPathsForRead" return="java.lang.Iterable"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all paths from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path list to the file on a local disk as an Iterable
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An Interface that can retrieve local directories to read from or write to.
+  Components can implement this interface to link it to
+  their own Directory Handler Service]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <!-- start class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <class name="AuxiliaryService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuxiliaryService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecoveryPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path specific to this auxiliary service to use for recovery.
+
+ @return state storage path or null if recovery is not enabled]]>
+      </doc>
+    </method>
+    <method name="initializeApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationInitializationContext"/>
+      <doc>
+      <![CDATA[A new application is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the application initialization.
+ 
+ @param initAppContext context for the application's initialization]]>
+      </doc>
+    </method>
+    <method name="stopApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationTerminationContext"/>
+      <doc>
+      <![CDATA[An application is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+ 
+ @param stopAppContext context for the application termination]]>
+      </doc>
+    </method>
+    <method name="getMetaData" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve meta-data for this {@link AuxiliaryService}. Applications using
+ this {@link AuxiliaryService} SHOULD know the format of the meta-data -
+ ideally each service should provide a method to parse out the information
+ to the applications. One example of meta-data is contact information so
+ that applications can access the service remotely. This will only be called
+ after the service's {@link #start()} method has finished. the result may be
+ cached.
+ 
+ <p>
+ The information is passed along to applications via
+ {@link StartContainersResponse#getAllServicesMetaData()} that is returned by
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ </p>
+ 
+ @return meta-data for this service that should be made available to
+         applications.]]>
+      </doc>
+    </method>
+    <method name="initializeContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerInitializationContext"/>
+      <doc>
+      <![CDATA[A new container is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the container initialization.
+ This method is called when the NodeManager receives the container launch
+ command from the ApplicationMaster and before the container process is 
+ launched.
+
+ @param initContainerContext context for the container's initialization]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerTerminationContext"/>
+      <doc>
+      <![CDATA[A container is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+
+ @param stopContainerContext context for the container termination]]>
+      </doc>
+    </method>
+    <method name="setRecoveryPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recoveryPath" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path for this auxiliary service to use for storing state
+ that will be used during recovery.
+
+ @param recoveryPath where recoverable state should be stored]]>
+      </doc>
+    </method>
+    <method name="getAuxiliaryLocalPathHandler" return="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Method that gets the local dirs path handler for this Auxiliary Service.
+
+ @return auxiliaryPathHandler object that is used to read from and write to
+ valid local Dirs.]]>
+      </doc>
+    </method>
+    <method name="setAuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="auxiliaryLocalPathHandler" type="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"/>
+      <doc>
+      <![CDATA[Method that sets the local dirs path handler for this Auxiliary Service.
+
+ @param auxiliaryLocalPathHandler the pathHandler for this auxiliary service]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A generic service that will be started by the NodeManager. This is a service
+ that administrators have to configure on each node by setting
+ {@link YarnConfiguration#NM_AUX_SERVICES}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <class name="ContainerContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get user of the container being initialized or stopped.
+
+ @return the user]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container being initialized or stopped.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link Resource} the resource capability allocated to the container
+ being initialized or stopped.
+
+ @return the resource capability.]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container
+ being initialized or stopped.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ExecutionType} the execution type of the container
+ being initialized or stopped.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base context class for {@link AuxiliaryService} initializing and stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <class name="ContainerInitializationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <interface name="ContainerLogAggregationPolicy"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseParameters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to initial the policy object with parameters specified by the application
+ or the cluster-wide setting.
+ </p>
+
+ @param parameters parameters with scheme defined by the policy class.]]>
+      </doc>
+    </method>
+    <method name="shouldDoLogAggregation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logContext" type="org.apache.hadoop.yarn.server.api.ContainerLogContext"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to ask the policy object if a given container's logs should be aggregated.
+ </p>
+
+ @param logContext ContainerLogContext
+ @return Whether or not the container's logs should be aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This API is used by NodeManager to decide if a given container's logs
+ should be aggregated at run time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <class name="ContainerLogContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLogContext" type="org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.server.api.ContainerType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exit code of the container.
+
+ @return the exit code]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Context class for {@link ContainerLogAggregationPolicy}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <class name="ContainerTerminationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Termination context for {@link AuxiliaryService} when stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+  <interface name="SCMAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="runCleanerTask" return="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by administrators to ask SCM to run cleaner task right away
+ </p>
+
+ @param request request <code>SharedCacheManager</code> to run a cleaner task
+ @return <code>SharedCacheManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between administrators and the <code>SharedCacheManager</code>
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml
new file mode 100644
index 0000000000000..98c7645154996
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml
@@ -0,0 +1,25332 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:53:54 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN API 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/classes:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/site/jdiff/xml -apiname Apache Hadoop YARN API 3.2.4 -->
+<package name="org.apache.hadoop.yarn.ams">
+</package>
+<package name="org.apache.hadoop.yarn.api">
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <interface name="ApplicationClientProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <method name="getNewApplication" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ApplicationId} for 
+ submitting new applications.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a new, monotonically
+ increasing, {@link ApplicationId} which is used by the client to submit
+ a new application.</p>
+
+ <p>The <code>ResourceManager</code> also responds with details such 
+ as maximum resource capabilities in the cluster as specified in
+ {@link GetNewApplicationResponse}.</p>
+
+ @param request request to get a new <code>ApplicationId</code>
+ @return response containing the new <code>ApplicationId</code> to be used
+ to submit an application
+ @throws YarnException
+ @throws IOException
+ @see #submitApplication(SubmitApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to submit a new application to the
+ <code>ResourceManager.</code></p>
+ 
+ <p>The client is required to provide details such as queue, 
+ {@link Resource} required to run the <code>ApplicationMaster</code>, 
+ the equivalent of {@link ContainerLaunchContext} for launching
+ the <code>ApplicationMaster</code> etc. via the 
+ {@link SubmitApplicationRequest}.</p>
+ 
+ <p>Currently the <code>ResourceManager</code> sends an immediate (empty) 
+ {@link SubmitApplicationResponse} on accepting the submission and throws 
+ an exception if it rejects the submission. However, this call needs to be
+ followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ to make sure that the application gets properly submitted - obtaining a
+ {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee
+ that RM 'remembers' this application beyond failover or restart. If RM
+ failover or RM restart happens before ResourceManager saves the
+ application's state successfully, the subsequent
+ {@link #getApplicationReport(GetApplicationReportRequest)} will throw
+ a {@link ApplicationNotFoundException}. The Clients need to re-submit
+ the application with the same {@link ApplicationSubmissionContext} when
+ it encounters the {@link ApplicationNotFoundException} on the
+ {@link #getApplicationReport(GetApplicationReportRequest)} call.</p>
+ 
+ <p>During the submission process, it checks whether the application
+ already exists. If the application exists, it will simply return
+ SubmitApplicationResponse</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to
+ queues etc. before accepting the application submission.</p>
+ 
+ @param request request to submit a new application
+ @return (empty) response on accepting the submission
+ @throws YarnException
+ @throws IOException
+ @see #getNewApplication(GetNewApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the 
+ <code>ResourceManager</code> to fail an application attempt.</p>
+
+ <p>The client, via {@link FailApplicationAttemptRequest} provides the
+ {@link ApplicationAttemptId} of the attempt to be failed.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before failing the attempt.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to fail an attempt
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to abort submitted application.</p>
+ 
+ <p>The client, via {@link KillApplicationRequest} provides the
+ {@link ApplicationId} of the application to be aborted.</p>
+ 
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before terminating the application.</p> 
+ 
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+ 
+ @param request request to abort a submitted application
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get metrics about the cluster from
+ the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a
+ {@link GetClusterMetricsResponse} which includes the 
+ {@link YarnClusterMetrics} with details such as number of current
+ nodes in the cluster.</p>
+ 
+ @param request request for cluster metrics
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a 
+ {@link GetClusterNodesResponse} which includes the 
+ {@link NodeReport} for all the nodes in the cluster.</p>
+ 
+ @param request request for report on all nodes
+ @return report on all nodes
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queues</em>
+ from the <code>ResourceManager</code>.</p>
+ 
+ <p>The client, via {@link GetQueueInfoRequest}, can ask for details such
+ as used/total resources, child queues, running applications etc.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access before
+ providing the information.</p> 
+ 
+ @param request request to get queue information
+ @return queue information
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueUserAcls" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queue 
+ acls</em> for <em>current user</em> from the <code>ResourceManager</code>.
+ </p>
+ 
+ <p>The <code>ResourceManager</code> responds with queue acls for all
+ existing queues.</p>
+ 
+ @param request request to get queue acls for <em>current user</em>
+ @return queue acls for <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move an application to a new queue.
+ 
+ @param request the application ID and the target queue
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNewReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ReservationId} for
+ submitting new reservations.</p>
+
+ <p>The <code>ResourceManager</code> responds with a new, unique,
+ {@link ReservationId} which is used by the client to submit
+ a new reservation.</p>
+
+ @param request to get a new <code>ReservationId</code>
+ @return response containing the new <code>ReservationId</code> to be used
+ to submit a new reservation
+ @throws YarnException if the reservation system is not enabled.
+ @throws IOException on IO failures.
+ @see #submitReservation(ReservationSubmissionRequest)]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and concurrency needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationSubmissionRequest is satisfiable the
+ {@code ResourceManager} answers with a
+ {@link ReservationSubmissionResponse} that include a non-null
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the reason.
+ 
+ On application submission the client can use this {@link ReservationId} to
+ obtain access to the reserved resources.
+ </p>
+ 
+ <p>
+ The system guarantees that during the time-range specified by the user, the
+ reservationID will be corresponding to a valid reservation. The amount of
+ capacity dedicated to such queue can vary overtime, depending of the
+ allocation that has been determined. But it is guaranteed to satisfy all
+ the constraint expressed by the user in the
+ {@link ReservationSubmissionRequest}.
+ </p>
+ 
+ @param request the request to submit a new Reservation
+ @return response the {@link ReservationId} on accepting the submission
+ @throws YarnException if the request is invalid or reservation cannot be
+           created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationUpdateRequest}. Upon success the previous allocation is
+ substituted by the new one, and on failure (i.e., if the system cannot find
+ a valid allocation for the updated request), the previous allocation
+ remains valid.
+ 
+ The {@link ReservationId} is not changed, and applications currently
+ running within this reservation will automatically receive the resources
+ based on the new allocation.
+ </p>
+ 
+ @param request to update an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ 
+ Upon deletion of a reservation applications running with this reservation,
+ are automatically downgraded to normal jobs running without any dedicated
+ reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+
+ @param request
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mappings
+ in existing cluster
+ </p>
+
+ @param request
+ @return labels to nodes mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @param request to get node labels collection of this cluster
+ @return node labels collection of this cluster
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application.
+ </p>
+ @param request to set priority of an application
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to signal a container. For example,
+ the client can send command OUTPUT_THREAD_DUMP to dump threads of the
+ container.</p>
+
+ <p>The client, via {@link SignalContainerRequest} provides the
+ id of the container and the signal command. </p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application before signaling the container.
+ The user needs to have <code>MODIFY_APP</code> permission.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to signal a container
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set ApplicationTimeouts of an application.
+ The UpdateApplicationTimeoutsRequest should have timeout value with
+ absolute time with ISO8601 format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ </p>
+ <b>Note:</b> If application timeout value is less than or equal to current
+ time then update application throws YarnException.
+ @param request to set ApplicationTimeouts of an application
+ @return a response with updated timeouts.
+ @throws YarnException if update request has empty values or application is
+           in completing states.
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get all the resource profiles that are
+ available on the ResourceManager.
+ </p>
+ @param request request to get all the resource profiles
+ @return Response containing a map of the profile name to Resource
+         capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get attributes to nodes mappings
+ available in ResourceManager.
+ </p>
+
+ @param request request to get details of attributes to nodes mapping.
+ @return Response containing the details of attributes to nodes mappings.
+ @throws YarnException if any error happens inside YARN
+ @throws IOException   incase of other errors]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes available in
+ ResourceManager.
+ </p>
+
+ @param request request to get node attributes collection of this cluster.
+ @return Response containing node attributes collection.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException   incase of other errors.]]>
+      </doc>
+    </method>
+    <method name="getNodesToAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to attributes mappings.
+ in existing cluster.
+ </p>
+
+ @param request request to get nodes to attributes mapping.
+ @return nodes to attributes mappings.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between clients and the <code>ResourceManager</code>
+ to submit/abort jobs and to get information on applications, cluster metrics,
+ nodes, queues and ACLs.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <interface name="ApplicationConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="APP_SUBMIT_TIME_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
+ only]]>
+      </doc>
+    </field>
+    <field name="CONTAINER_TOKEN_FILE_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cache file into which container token is written]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_WEB_PROXY_BASE_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ ApplicationMaster's environment only. This states that for all non-relative
+ web URLs in the app masters web UI what base should they have.]]>
+      </doc>
+    </field>
+    <field name="LOG_DIR_EXPANSION_VAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The temporary environmental variable for container log directory. This
+ should be replaced by real container log directory on container launch.]]>
+      </doc>
+    </field>
+    <field name="CLASS_PATH_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constant is used to construct class path and it will be replaced with
+ real class path separator(':' for Linux and ';' for Windows) by
+ NodeManager on container launch. User has to use this constant to construct
+ class path if user wants cross-platform practice i.e. submit an application
+ from a Windows client to a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_LEFT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The following two constants are used to expand parameter and it will be
+ replaced with real parameter expansion marker ('%' for Windows and '$' for
+ Linux) by NodeManager on container launch. For example: {{VAR}} will be
+ replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+ constant to construct class path if user wants cross-platform practice i.e.
+ submit an application from a Windows client to a Linux/Unix server or vice
+ versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_RIGHT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User has to use this constant to construct class path if user wants
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="STDERR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STDOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is the API for the applications comprising of constants that YARN sets
+ up for the applications and the containers.
+
+ TODO: Investigate the semantics and security of each cross-boundary refs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <!-- start class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <class name="ApplicationConstants.ContainerLaunchType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The type of launch for the container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <interface name="ApplicationHistoryProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>ApplicationHistoryServer</code> to
+ get the information of completed applications etc.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <interface name="ApplicationMasterProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by a new <code>ApplicationMaster</code> to register with
+ the <code>ResourceManager</code>.
+ </p>
+
+ <p>
+ The <code>ApplicationMaster</code> needs to provide details such as RPC
+ Port, HTTP tracking url etc. as specified in
+ {@link RegisterApplicationMasterRequest}.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code> responds with critical details such as
+ maximum resource capabilities in the cluster as specified in
+ {@link RegisterApplicationMasterResponse}.
+ </p>
+
+ <p>
+ Re-register is only allowed for <code>Unmanaged Application Master</code>
+ (UAM) HA, with
+ {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ set to true.
+ </p>
+
+ @param request registration request
+ @return registration respose
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException The exception is thrown
+           when an ApplicationMaster tries to register more then once.
+ @see RegisterApplicationMasterRequest
+ @see RegisterApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by an <code>ApplicationMaster</code> to notify the 
+ <code>ResourceManager</code> about its completion (success or failed).</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as 
+ final state, diagnostics (in case of failures) etc. as specified in 
+ {@link FinishApplicationMasterRequest}.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with 
+ {@link FinishApplicationMasterResponse}.</p>
+ 
+ @param request completion request
+ @return completion response
+ @throws YarnException
+ @throws IOException
+ @see FinishApplicationMasterRequest
+ @see FinishApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The main interface between an <code>ApplicationMaster</code> and the
+ <code>ResourceManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> uses this interface to provide a list of
+ {@link ResourceRequest} and returns unused {@link Container} allocated to
+ it via {@link AllocateRequest}. Optionally, the
+ <code>ApplicationMaster</code> can also <em>blacklist</em> resources which
+ it doesn't want to use.
+ </p>
+ 
+ <p>
+ This also doubles up as a <em>heartbeat</em> to let the
+ <code>ResourceManager</code> know that the <code>ApplicationMaster</code>
+ is alive. Thus, applications should periodically make this call to be kept
+ alive. The frequency depends on
+ {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
+ </p>
+ 
+ <p>
+ The <code>ResourceManager</code> responds with list of allocated
+ {@link Container}, status of completed containers and headroom information
+ for the application.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> can use the available headroom
+ (resources) to decide how to utilized allocated resources and make informed
+ decisions about future resource requests.
+ </p>
+ 
+ @param request
+          allocation request
+ @return allocation response
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException
+           This exception is thrown when an ApplicationMaster calls allocate
+           without registering first.
+ @throws InvalidResourceBlacklistRequestException
+           This exception is thrown when an application provides an invalid
+           specification for blacklist of resources.
+ @throws InvalidResourceRequestException
+           This exception is thrown when a {@link ResourceRequest} is out of
+           the range of the configured lower and upper limits on the
+           resources.
+ @see AllocateRequest
+ @see AllocateResponse]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between a live instance of <code>ApplicationMaster</code> 
+ and the <code>ResourceManager</code>.</p>
+ 
+ <p>This is used by the <code>ApplicationMaster</code> to register/unregister
+ and to request and obtain resources in the cluster from the
+ <code>ResourceManager</code>.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <interface name="ClientSCMProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="use" return="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to claim a resource with the
+ <code>SharedCacheManager.</code> The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application will be
+ using the resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>Path</code>
+ to the resource in the shared cache is returned. If the resource does not
+ exist, the response is empty.
+ </p>
+
+ @param request request to claim a resource in the shared cache
+ @return response indicating if the resource is already in the cache
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="release" return="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to release a resource with the
+ <code>SharedCacheManager.</code> This method is called once an application
+ is no longer using a claimed resource in the shared cache. The client uses
+ a checksum to identify the resource and an {@link ApplicationId} to
+ identify which application is releasing the resource.
+ </p>
+
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+
+ <p>
+ Currently the <code>SharedCacheManager</code> sends an empty response.
+ </p>
+
+ @param request request to release a resource in the shared cache
+ @return (empty) response on releasing the resource
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>SharedCacheManager</code> to claim
+ and release resources in the shared cache.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+  <interface name="ContainerManagementProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="startContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> provides a list of
+ {@link StartContainerRequest}s to a <code>NodeManager</code> to
+ <em>start</em> {@link Container}s allocated to it using this interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> has to provide details such as allocated
+ resource capability, security tokens (if enabled), command to be executed
+ to start the container, environment for the process, necessary
+ binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in
+ the {@link StartContainerRequest}.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StartContainersResponse} which includes a list of
+ {@link Container}s of successfully launched {@link Container}s, a
+ containerId-to-exception map for each failed {@link StartContainerRequest} in
+ which the exception indicates errors from per container and a
+ allServicesMetaData map between the names of auxiliary services and their
+ corresponding meta-data. Note: None-container-specific exceptions will
+ still be thrown by the API method itself.
+ </p>
+ <p>
+ The <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the to-be-launched or launched containers.
+ </p>
+ 
+ @param request
+          request to start a list of containers
+ @return response including conatinerIds of all successfully launched
+         containers, a containerId-to-exception map for failed requests and
+         a allServicesMetaData map.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="stopContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> requests a <code>NodeManager</code> to
+ <em>stop</em> a list of {@link Container}s allocated to it using this
+ interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a {@link StopContainersRequest}
+ which includes the {@link ContainerId}s of the containers to be stopped.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StopContainersResponse} which includes a list of {@link ContainerId}
+ s of successfully stopped containers, a containerId-to-exception map for
+ each failed request in which the exception indicates errors from per
+ container. Note: None-container-specific exceptions will still be thrown by
+ the API method itself. <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the containers.
+ </p>
+ 
+ @param request
+          request to stop a list of containers
+ @return response which includes a list of containerIds of successfully
+         stopped containers, a containerId-to-exception map for failed
+         requests.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for current
+ statuses of <code>Container</code>s from the <code>NodeManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a
+ {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s
+ of all containers whose statuses are needed.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> responds with
+ {@link GetContainerStatusesResponse} which includes a list of
+ {@link ContainerStatus} of the successfully queried containers and a
+ containerId-to-exception map for each failed request in which the exception
+ indicates errors from per container. Note: None-container-specific
+ exceptions will still be thrown by the API method itself.
+ </p>
+ 
+ @param request
+          request to get <code>ContainerStatus</code>es of containers with
+          the specified <code>ContainerId</code>s
+ @return response containing the list of <code>ContainerStatus</code> of the
+         successfully queried containers and a containerId-to-exception map
+         for failed requests.
+ 
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="increaseContainersResource" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource increase of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to increase resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully increased and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource update of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to update resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully updated and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from NodeManager]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="localize" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources required by the container.
+ Currently, this API only works for running containers.
+
+ @param request Specify the resources to be localized.
+ @return Response that the localize request is accepted.
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[ReInitialize the Container with a new Launch Context.
+ @param request Specify the new ContainerLaunchContext.
+ @return Response that the ReInitialize request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="restartContainer" return="org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restart the container.
+ @param containerId Container Id.
+ @return Response that the restart request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rollback the Last ReInitialization if possible.
+ @param containerId Container Id.
+ @return Response that the rollback request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.CommitResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Commit the Last ReInitialization if possible. Once the reinitialization
+ has been committed, It cannot be rolled back.
+ @param containerId Container Id.
+ @return Response that the commit request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between an <code>ApplicationMaster</code> and a 
+ <code>NodeManager</code> to start/stop and increase resource of containers
+ and to get status of running containers.</p>
+
+ <p>If security is enabled the <code>NodeManager</code> verifies that the
+ <code>ApplicationMaster</code> has truly been allocated the container
+ by the <code>ResourceManager</code> and also verifies all interactions such 
+ as stopping the container or obtaining status information for the container.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.api.protocolrecords">
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <class name="AllocateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="updateRequests" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>response id</em> used to track duplicate responses.
+ @return <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="setResponseId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>response id</em> used to track duplicate responses.
+ @param id <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current progress</em> of application. 
+ @return <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="setProgress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <em>current progress</em> of application
+ @param progress <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="getAskList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ResourceRequest</code> to update the 
+ <code>ResourceManager</code> about the application's resource requirements.
+ @return the list of <code>ResourceRequest</code>
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="setAskList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of <code>ResourceRequest</code> to update the
+ <code>ResourceManager</code> about the application's resource requirements.
+ @param resourceRequests list of <code>ResourceRequest</code> to update the 
+                        <code>ResourceManager</code> about the application's 
+                        resource requirements
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getReleaseList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> of containers being 
+ released by the <code>ApplicationMaster</code>.
+ @return list of <code>ContainerId</code> of containers being 
+         released by the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setReleaseList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseContainers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of <code>ContainerId</code> of containers being
+ released by the <code>ApplicationMaster</code>
+ @param releaseContainers list of <code>ContainerId</code> of 
+                          containers being released by the 
+                          <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceBlacklistRequest" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ResourceBlacklistRequest</code> being sent by the 
+ <code>ApplicationMaster</code>.
+ @return the <code>ResourceBlacklistRequest</code> being sent by the 
+         <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="setResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ResourceBlacklistRequest</code> to inform the 
+ <code>ResourceManager</code> about the blacklist additions and removals
+ per the <code>ApplicationMaster</code>.
+ 
+ @param resourceBlacklistRequest the <code>ResourceBlacklistRequest</code>  
+                         to inform the <code>ResourceManager</code> about  
+                         the blacklist additions and removals
+                         per the <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="getUpdateRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link UpdateContainerRequest}
+         being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setUpdateRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update requests to inform the
+ <code>ResourceManager</code> about the containers that need to be
+ updated.
+ @param updateRequests list of <code>UpdateContainerRequest</code> for
+                       containers to be updated]]>
+      </doc>
+    </method>
+    <method name="getSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of Scheduling requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link SchedulingRequest} being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of Scheduling requests to inform the
+ <code>ResourceManager</code> about the application's resource requirements
+ (potentially including allocation tags and placement constraints).
+ @param schedulingRequests list of {@link SchedulingRequest} to update
+          the <code>ResourceManager</code> about the application's resource
+          requirements.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tracking url update for this heartbeat.
+ @return tracking url to update this application with]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the new tracking url for this application.
+ @param trackingUrl the new tracking url]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The core request sent by the <code>ApplicationMaster</code> to the 
+ <code>ResourceManager</code> to obtain resources in the cluster.</p> 
+
+ <p>The request includes:
+ <ul>
+   <li>A response id to track duplicate responses.</li>
+   <li>Progress information.</li>
+   <li>
+     A list of {@link ResourceRequest} to inform the
+     <code>ResourceManager</code> about the application's
+     resource requirements.
+   </li>
+   <li>
+     A list of unused {@link Container} which are being returned.
+   </li>
+   <li>
+     A list of {@link UpdateContainerRequest} to inform
+     the <code>ResourceManager</code> about the change in
+     requirements of running containers.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <class name="AllocateRequest.AllocateRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="responseId" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>responseId</code> of the request.
+ @see AllocateRequest#setResponseId(int)
+ @param responseId <code>responseId</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="progress" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <code>progress</code> of the request.
+ @see AllocateRequest#setProgress(float)
+ @param progress <code>progress</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="askList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="askList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>askList</code> of the request.
+ @see AllocateRequest#setAskList(List)
+ @param askList <code>askList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="releaseList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>releaseList</code> of the request.
+ @see AllocateRequest#setReleaseList(List)
+ @param releaseList <code>releaseList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceBlacklistRequest" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>resourceBlacklistRequest</code> of the request.
+ @see AllocateRequest#setResourceBlacklistRequest(
+ ResourceBlacklistRequest)
+ @param resourceBlacklistRequest
+     <code>resourceBlacklistRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="updateRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>updateRequests</code> of the request.
+ @see AllocateRequest#setUpdateRequests(List)
+ @param updateRequests <code>updateRequests</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="schedulingRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>schedulingRequests</code> of the request.
+ @see AllocateRequest#setSchedulingRequests(List)
+ @param schedulingRequests <code>SchedulingRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="trackingUrl" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>trackingUrl</code> of the request.
+ @see AllocateRequest#setTrackingUrl(String)
+ @param trackingUrl new tracking url
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link AllocateRequest} object.
+ @return {@link AllocateRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link AllocateRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <class name="AllocateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+      <param name="amRMToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="updatedContainers" type="java.util.List"/>
+      <param name="collectorInfo" type="org.apache.hadoop.yarn.api.records.CollectorInfo"/>
+    </method>
+    <method name="getAMCommand" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the <code>ResourceManager</code> needs the
+ <code>ApplicationMaster</code> to take some action then it will send an
+ AMCommand to the <code>ApplicationMaster</code>. See <code>AMCommand</code> 
+ for details on commands and actions for them.
+ @return <code>AMCommand</code> if the <code>ApplicationMaster</code> should
+         take action, <code>null</code> otherwise
+ @see AMCommand]]>
+      </doc>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last response id</em>.
+ @return <em>last response id</em>]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>newly allocated</em> <code>Container</code> by the
+ <code>ResourceManager</code>.
+ @return list of <em>newly allocated</em> <code>Container</code>]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>available headroom</em> for resources in the cluster for the
+ application.
+ @return limit of available headroom for resources in the cluster for the
+ application]]>
+      </doc>
+    </method>
+    <method name="getCompletedContainersStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>completed containers' statuses</em>.
+ @return the list of <em>completed containers' statuses</em>]]>
+      </doc>
+    </method>
+    <method name="getUpdatedNodes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>updated <code>NodeReport</code>s</em>. Updates could
+ be changes in health, availability etc of the nodes.
+ @return The delta of updated nodes since the last response]]>
+      </doc>
+    </method>
+    <method name="getNumClusterNodes" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of hosts available on the cluster.
+ @return the available host count.]]>
+      </doc>
+    </method>
+    <method name="getPreemptionMessage" return="org.apache.hadoop.yarn.api.records.PreemptionMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the description of containers owned by the AM, but requested back by
+ the cluster. Note that the RM may have an inconsistent view of the
+ resources owned by the AM. These messages are advisory, and the AM may
+ elect to ignore them.
+ <p>
+ The message is a snapshot of the resources the RM wants back from the AM.
+ While demand persists, the RM will repeat its request; applications should
+ not interpret each message as a request for <em>additional</em>
+ resources on top of previous messages. Resources requested consistently
+ over some duration may be forcibly killed by the RM.
+
+ @return A specification of the resources to reclaim from this AM.]]>
+      </doc>
+    </method>
+    <method name="getNMTokens" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens required for communicating with NM. New NMTokens
+ issued only if
+ <p>
+ 1) AM is receiving first container on underlying NodeManager.<br>
+ OR<br>
+ 2) NMToken master key rolled over in ResourceManager and AM is getting new
+ container on the same underlying NodeManager.
+ <p>
+ AM will receive one NMToken per NM irrespective of the number of containers
+ issued on same NM. AM is expected to store these tokens until issued a
+ new token for the same NM.
+ @return list of NMTokens required for communicating with NM]]>
+      </doc>
+    </method>
+    <method name="getUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of newly updated containers by
+ <code>ResourceManager</code>.
+ @return list of newly increased containers]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The AMRMToken that belong to this attempt
+
+ @return The AMRMToken that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the application
+
+ @return get application priority]]>
+      </doc>
+    </method>
+    <method name="getCollectorInfo" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The data associated with the collector that belongs to this app. Contains
+ address and token alongwith identification information.
+
+ @return The data of collector that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getUpdateErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+
+ @return List of Update Container Errors.]]>
+      </doc>
+    </method>
+    <method name="setUpdateErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+ @param updateErrors list of <code>UpdateContainerError</code> for
+                       containers updates requests that were in error]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts which
+ have not been reported to the Application Master yet.
+ <br>
+ These containers were recovered by the RM after the application master
+ had already registered. This may happen after RM restart when some NMs get
+ delayed in connecting to the RM and reporting the active containers.
+ Since they were not reported in the registration
+ response, they are reported in the response to the AM heartbeat.
+
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts.]]>
+      </doc>
+    </method>
+    <method name="getRejectedSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of all SchedulingRequests that the RM has rejected between
+ this allocate call and the previous one.
+ @return List of RejectedSchedulingRequests.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> the
+ <code>ApplicationMaster</code> during resource negotiation.
+ <p>
+ The response, includes:
+ <ul>
+   <li>Response ID to track duplicate responses.</li>
+   <li>
+     An AMCommand sent by ResourceManager to let the
+     {@code ApplicationMaster} take some actions (resync, shutdown etc.).
+   </li>
+   <li>A list of newly allocated {@link Container}.</li>
+   <li>A list of completed {@link Container}s' statuses.</li>
+   <li>
+     The available headroom for resources in the cluster for the
+     application.
+   </li>
+   <li>A list of nodes whose status has been updated.</li>
+   <li>The number of available nodes in a cluster.</li>
+   <li>A description of resources requested back by the cluster</li>
+   <li>AMRMToken, if AMRMToken has been rolled over</li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been increased.
+   </li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been decreased.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <class name="ApplicationsRequestScope" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration that controls the scope of applications fetched]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <class name="CommitResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommitResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to Commit Container Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <class name="ContainerUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToUpdate" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource update.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource update.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToUpdate" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getUpdatedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource update.
+ @param containersToUpdate the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <class name="ContainerUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="successfullyUpdatedContainers" type="java.util.List"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getSuccessfullyUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully update.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully updated.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to update container resource.
+ </p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <class name="FailApplicationAttemptRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the attempt to be failed.
+ @return <code>ApplicationAttemptId</code> of the attempt.]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to fail an application attempt.</p>
+
+ <p>The request includes the {@link ApplicationAttemptId} of the attempt to
+ be failed.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <class name="FailApplicationAttemptResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ failing an application attempt.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <class name="FinishApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="url" type="java.lang.String"/>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>final state</em> of the <code>ApplicationMaster</code>.
+ @return <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setFinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalState" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <doc>
+      <![CDATA[Set the <em>final state</em> of the <code>ApplicationMaster</code>
+ @param finalState <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic information</em> on application failure.
+ @return <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <em>diagnostic information</em> on application failure.
+ @param diagnostics <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ This is the web-URL to which ResourceManager or web-application proxy will
+ redirect client/users once the application is finished and the
+ <code>ApplicationMaster</code> is gone.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param url
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The finalization request sent by the {@code ApplicationMaster} to
+ inform the {@code ResourceManager} about its completion.
+ <p>
+ The final request includes details such:
+ <ul>
+   <li>Final state of the {@code ApplicationMaster}</li>
+   <li>
+     Diagnostic information in case of failure of the
+     {@code ApplicationMaster}
+   </li>
+   <li>Tracking URL</li>
+ </ul>
+
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <class name="FinishApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsUnregistered" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the application has successfully
+ unregistered with the RM and the application can safely stop.
+ @return true if the application has unregistered with the RM,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a
+ <code>ApplicationMaster</code> on it's completion.
+ <p>
+ The response, includes:
+ <ul>
+ <li>A flag which indicates that the application has successfully unregistered
+ with the RM and the application can safely stop.</li>
+ </ul>
+ <p>
+ Note: The flag indicates whether the application has successfully
+ unregistered and is safe to stop. The application may stop after the flag is
+ true. If the application stops before the flag is true then the RM may retry
+ the application.
+ 
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <class name="GetAllResourceProfilesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <class name="GetAllResourceProfilesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceProfiles"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profiles" type="java.util.Map"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <class name="GetAllResourceTypeInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <class name="GetAllResourceTypeInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceTypeInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceTypes" type="java.util.List"/>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <class name="GetApplicationAttemptReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ApplicationAttemptReport} for an application attempt.
+ </p>
+ 
+ <p>
+ The request should include the {@link ApplicationAttemptId} of the
+ application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <class name="GetApplicationAttemptReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ApplicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @return <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @param applicationAttemptReport
+          <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ an application attempt report.
+ </p>
+ 
+ <p>
+ The response includes an {@link ApplicationAttemptReport} which has the
+ details about the particular application attempt
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <class name="GetApplicationAttemptsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of an application
+ 
+ @return <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of an application
+ 
+ @param applicationId
+          <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of application attempt reports of an
+ application from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <class name="GetApplicationAttemptsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+    </method>
+    <method name="getApplicationAttemptList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @return a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @param applicationAttempts
+          a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ApplicationAttemptReport} for application attempts.
+ </p>
+ 
+ <p>
+ The <code>ApplicationAttemptReport</code> for each application includes the
+ details of an application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <class name="GetApplicationReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to the <code>ResourceManager</code> to 
+ get an {@link ApplicationReport} for an application.</p>
+ 
+ <p>The request should include the {@link ApplicationId} of the 
+ application.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)
+ @see ApplicationReport]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <class name="GetApplicationReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationReport</code> for the application.
+ @return <code>ApplicationReport</code> for the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an application report.</p>
+ 
+ <p>The response includes an {@link ApplicationReport} which has details such 
+ as user, queue, name, host on which the <code>ApplicationMaster</code> is 
+ running, RPC port, tracking URL, diagnostics, start time etc.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <class name="GetApplicationsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="queues" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="startRange" type="org.apache.commons.lang3.Range"/>
+      <param name="finishRange" type="org.apache.commons.lang3.Range"/>
+      <param name="limit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+
+ <p>Setting any of the parameters to null, would just disable that
+ filter</p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @param users list of users to filter by
+ @param queues list of scheduler queues to filter by
+ @param applicationTypes types of applications
+ @param applicationTags application tags to filter by
+ @param applicationStates application states to filter by
+ @param startRange range of application start times to filter by
+ @param finishRange range of application finish times to filter by
+ @param limit number of applications to limit to
+ @return {@link GetApplicationsRequest} to be used with
+ {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application states in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving and application types and application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in <code>GetApplicationsRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getApplicationTypes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application types to filter applications on
+
+ @return Set of Application Types to filter on]]>
+      </doc>
+    </method>
+    <method name="getApplicationStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application states to filter applications on
+
+ @return Set of Application states to filter on]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of Applications
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <class name="GetApplicationsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationReport</code> for applications.
+ @return <code>ApplicationReport</code> for applications]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an {@link ApplicationReport} for applications.</p>
+
+ <p>The <code>ApplicationReport</code> for each application includes details
+ such as user, queue, name, host on which the <code>ApplicationMaster</code>
+ is running, RPC port, tracking URL, diagnostics, start time etc.</p>
+
+ @see ApplicationReport
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <class name="GetAttributesToNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributeKeys for which the mapping of hostname to attribute value
+ is required.
+
+ @param attributes Set of NodeAttributeKey provided.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributeKeys for which mapping of hostname to attribute value is
+ required.
+
+ @return Set of NodeAttributeKey]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node to attribute value mapping for all or
+ given set of Node AttributeKey's in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <class name="GetAttributesToNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setAttributeToNodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get mapping of NodeAttributeKey to its associated mapping of list of
+ NodeToAttributeValue associated with attribute.
+
+ @return Map of node attributes to list of NodeToAttributeValue.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ node to attribute value mapping for all or given set of Node AttributeKey's.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <class name="GetClusterMetricsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get cluster metrics from the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>Currently, this is empty.</p>
+
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <class name="GetClusterMetricsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnClusterMetrics</code> for the cluster.
+ @return <code>YarnClusterMetrics</code> for the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a client
+ requesting cluster metrics.
+ 
+ @see YarnClusterMetrics
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <class name="GetClusterNodeAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create new instance of GetClusterNodeAttributesRequest.
+
+ @return GetClusterNodeAttributesRequest is returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node attributes in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <class name="GetClusterNodeAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Create instance of GetClusterNodeAttributesResponse.
+
+ @param attributes
+ @return GetClusterNodeAttributesResponse.]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributes to the response.
+
+ @param attributes Map of Node attributeKey to Type.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes from the response.
+
+ @return Node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a node attributes in cluster.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <class name="GetClusterNodeLabelsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <class name="GetClusterNodeLabelsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #newInstance(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Creates a new instance.
+
+ @param labels Node labels
+ @return response
+ @deprecated Use {@link #newInstance(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="setNodeLabelList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="getNodeLabelList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNodeLabels"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setNodeLabelList(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node labels to the response.
+
+ @param labels Node labels
+ @deprecated Use {@link #setNodeLabelList(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getNodeLabelList()} instead.">
+      <doc>
+      <![CDATA[Get node labels of the response.
+
+ @return Node labels
+ @deprecated Use {@link #getNodeLabelList()} instead.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <class name="GetClusterNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @return the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <method name="setNodeStates"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @param states the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ The request will ask for all nodes in the given {@link NodeState}s.
+
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <class name="GetClusterNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>NodeReport</code> for all nodes in the cluster.
+ @return <code>NodeReport</code> for all nodes in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting a {@link NodeReport} for all nodes.</p>
+ 
+ <p>The <code>NodeReport</code> contains per-node information such as 
+ available resources, number of containers, tracking url, rack name, health
+ status etc.
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <class name="GetContainerReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the Container.
+ 
+ @return <code>ContainerId</code> of the Container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container
+ 
+ @param containerId
+          <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ContainerReport} for a container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <class name="GetContainerReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerReport</code> for the container.
+ 
+ @return <code>ContainerReport</code> for the container]]>
+      </doc>
+    </method>
+    <method name="setContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a container report.
+ </p>
+ 
+ <p>
+ The response includes a {@link ContainerReport} which has details of a
+ container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <class name="GetContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of container reports, which belong to
+ an application attempt from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <class name="GetContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+    </method>
+    <method name="getContainerList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @return a list of <code>ContainerReport</code> for all the containers of an
+         application attempt]]>
+      </doc>
+    </method>
+    <method name="setContainerList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @param containers
+          a list of <code>ContainerReport</code> for all the containers of
+          an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ContainerReport} for containers.
+ </p>
+ 
+ <p>
+ The <code>ContainerReport</code> for each container includes the container
+ details.
+ </p>
+ 
+ @see ContainerReport
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <class name="GetContainerStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>.
+ 
+ @return the list of <code>ContainerId</code>s of containers for which to
+         obtain the <code>ContainerStatus</code>.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>
+ 
+ @param containerIds
+          a list of <code>ContainerId</code>s of containers for which to
+          obtain the <code>ContainerStatus</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to get {@link ContainerStatus} of requested
+ containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <class name="GetContainerStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerStatus</code>es of the requested containers.
+ 
+ @return <code>ContainerStatus</code>es of the requested containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to obtain the
+ <code>ContainerStatus</code> of requested containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <class name="GetDelegationTokenRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to get a delegation token from
+ the {@code ResourceManager}.
+ for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <class name="GetDelegationTokenResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Delegation tokens have a identifier which maps to
+ {@link AbstractDelegationTokenIdentifier}.
+ @return the delegation tokens]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Response to a {@link GetDelegationTokenRequest} request 
+ from the client. The response contains the token that 
+ can be used by the containers to talk to  ClientRMService.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <class name="GetNewApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@link ApplicationId} for
+ submitting an application.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <class name="GetNewApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>new</em> <code>ApplicationId</code> allocated by the 
+ <code>ResourceManager</code>.
+ @return <em>new</em> <code>ApplicationId</code> allocated by the 
+          <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for 
+ a request to get a new {@link ApplicationId} for submitting applications.</p>
+ 
+ <p>Clients can submit an application with the returned
+ {@link ApplicationId}.</p>
+
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <class name="GetNewReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@code ReservationId} for
+ submitting an reservation.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <class name="GetNewReservationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a new {@link ReservationId} to be used to submit a reservation.
+
+ @return a {@link ReservationId} representing the unique id to identify
+ a reservation with which it was submitted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for
+ a request to get a new {@link ReservationId} for submitting reservations.</p>
+
+ <p>Clients can submit an reservation with the returned
+ {@link ReservationId}.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <class name="GetNodesToAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+    </method>
+    <method name="setHostNames"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostnames" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set hostnames for which mapping is required.
+
+ @param hostnames]]>
+      </doc>
+    </method>
+    <method name="getHostNames" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames for which mapping is required.
+
+ @return Set of hostnames.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get nodes to attributes mapping
+ in the cluster from the <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <class name="GetNodesToAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setNodeToAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames to NodeAttributes mapping.
+
+ @return Map of host to attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ nodes to attributes mapping.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <class name="GetQueueInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <param name="includeApplications" type="boolean"/>
+      <param name="includeChildQueues" type="boolean"/>
+      <param name="recursive" type="boolean"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> for which to get queue information.
+ @return <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue name</em> for which to get queue information
+ @param queueName <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="getIncludeApplications" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>active applications</em> required?
+ @return <code>true</code> if applications' information is to be included,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeApplications"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeApplications" type="boolean"/>
+      <doc>
+      <![CDATA[Should we get fetch information about <em>active applications</em>?
+ @param includeApplications fetch information about <em>active 
+                            applications</em>?]]>
+      </doc>
+    </method>
+    <method name="getIncludeChildQueues" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>child queues</em> required?
+ @return <code>true</code> if information about child queues is required,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeChildQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeChildQueues" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information about <em>child queues</em>?
+ @param includeChildQueues fetch information about <em>child queues</em>?]]>
+      </doc>
+    </method>
+    <method name="getRecursive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information on the entire <em>child queue hierarchy</em> required?
+ @return <code>true</code> if information about entire hierarchy is 
+         required, <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setRecursive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information on the entire <em>child queue hierarchy</em>?
+ @param recursive fetch information on the entire <em>child queue 
+                  hierarchy</em>?]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get <em>queue information</em>
+ from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <class name="GetQueueInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueInfo</code> for the specified queue.
+ @return <code>QueueInfo</code> for the specified queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a client
+ requesting information about queues in the system.
+ <p>
+ The response includes a {@link QueueInfo} which has details such as
+ queue name, used/total capacities, running applications, child queues etc.
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <class name="GetQueueUserAclsInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to the <code>ResourceManager</code> to 
+ get queue acls for the <em>current user</em>.</p>
+
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <class name="GetQueueUserAclsInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUserAclsInfoList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueUserACLInfo</code> per queue for the user.
+ @return <code>QueueUserACLInfo</code> per queue for the user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to clients
+ seeking queue acls for the user.</p>
+
+ <p>The response contains a list of {@link QueueUserACLInfo} which
+ provides information about {@link QueueACL} per queue.</p>
+ 
+ @see QueueACL
+ @see QueueUserACLInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <class name="GetResourceProfileRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="setProfileName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profileName" type="java.lang.String"/>
+    </method>
+    <method name="getProfileName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <class name="GetResourceProfileResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resources that will be allocated if the profile was used.
+
+ @return the resources that will be allocated if the profile was used.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resources that will be allocated if the profile is used.
+
+ @param r Set the resources that will be allocated if the profile is used.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <class name="IncreaseContainersResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToIncrease" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource increase.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource increase.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToIncrease"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getIncreasedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource increase.
+ @param containersToIncrease the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <class name="IncreaseContainersResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyIncreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully increased.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully increased.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to increase container resource.
+ </p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <class name="KillApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be aborted.
+ @return <code>ApplicationId</code> of the application to be aborted]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostics</em> to which the application is being killed.
+ @return <em>diagnostics</em> to which the application is being killed]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>diagnostics</em> to which the application is being killed.
+ @param diagnostics <em>diagnostics</em> to which the application is being
+          killed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to abort a submitted application.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ aborted.</p>
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <class name="KillApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsKillCompleted" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the process of killing application is completed or not.
+ @return true if the process of killing application has completed,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to the client aborting
+ a submitted application.
+ <p>
+ The response, includes:
+ <ul>
+   <li>
+     A flag which indicates that the process of killing the application is
+     completed or not.
+   </li>
+ </ul>
+ Note: user is recommended to wait until this flag becomes true, otherwise if
+ the <code>ResourceManager</code> crashes before the process of killing the
+ application is completed, the <code>ResourceManager</code> may retry this
+ application on recovery.
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <class name="MoveApplicationAcrossQueuesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be moved.
+ @return <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application to be moved.
+ @param appId <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="getTargetQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @return the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <method name="setTargetQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @param queue the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to move a submitted application to a different queue.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ moved and the queue to place it in.</p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <class name="MoveApplicationAcrossQueuesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client moving
+ a submitted application to a different queue.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <class name="RegisterApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of <code>RegisterApplicationMasterRequest</code>.
+ If <em>port, trackingUrl</em> is not used, use the following default value:
+ <ul>
+  <li>port: -1</li>
+  <li>trackingUrl: null</li>
+ </ul>
+ The port is allowed to be any integer larger than or equal to -1.
+ @return the new instance of <code>RegisterApplicationMasterRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @param host <em>host</em> on which the <code>ApplicationMaster</code> 
+             is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @return the <em>RPC port</em> on which the {@code ApplicationMaster}
+         is responding]]>
+      </doc>
+    </method>
+    <method name="setRpcPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @param port <em>RPC port</em> on which the {@code ApplicationMaster}
+             is responding]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em> for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ it is running. This is the web-URL to which ResourceManager or
+ web-application proxy will redirect client/users while the application and
+ the <code>ApplicationMaster</code> are still running.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param trackingUrl
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getPlacementConstraints" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all Placement Constraints specified at the Application level. The
+ mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags, i.e., each
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those
+ tags will be placed taking into account the corresponding constraint.
+
+ @return A map of Placement Constraints.]]>
+      </doc>
+    </method>
+    <method name="setPlacementConstraints"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraints" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set Placement Constraints applicable to the
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+ of this application.
+ The mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags.
+ For example:
+  Map &lt;
+   &lt;hb_regionserver&gt; -&gt; node_anti_affinity,
+   &lt;hb_regionserver, hb_master&gt; -&gt; rack_affinity,
+   ...
+  &gt;
+ @param placementConstraints Placement Constraint Mapping.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
+ on registration.
+ <p>
+ The registration includes details such as:
+ <ul>
+   <li>Hostname on which the AM is running.</li>
+   <li>RPC Port</li>
+   <li>Tracking URL</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <class name="RegisterApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="getClientToAMTokenMasterKey" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get ClientToAMToken master key.</p>
+ <p>The ClientToAMToken master key is sent to <code>ApplicationMaster</code>
+ by <code>ResourceManager</code> via {@link RegisterApplicationMasterResponse}
+ , used to verify corresponding ClientToAMToken.</p>
+ @return ClientToAMToken master key]]>
+      </doc>
+    </method>
+    <method name="setClientToAMTokenMasterKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set ClientToAMToken master key.]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the queue that the application was placed in.<p>
+ @return the queue that the application was placed in.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>Set the queue that the application was placed in.<p>]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts.
+ </p>
+ 
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts
+ @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getNMTokensFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens for communicating with the NMs where the
+ containers of previous application attempts are running.
+ 
+ @return the list of NMTokens for communicating with the NMs where the
+         containers of previous application attempts are running.
+ 
+ @see RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getSchedulerResourceTypes" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a set of the resource types considered by the scheduler.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of supported resource profiles from RM.
+
+ @return a map of resource profiles and its capabilities.]]>
+      </doc>
+    </method>
+    <method name="getResourceTypes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get available resource types supported by RM.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a new
+ {@code ApplicationMaster} on registration.
+ <p>
+ The response contains critical details such as:
+ <ul>
+   <li>Maximum capability for allocated resources in the cluster.</li>
+   <li>{@code ApplicationACL}s for the application.</li>
+   <li>ClientToAMToken master key.</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <class name="ReInitializeContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[Creates a new instance of the ReInitializationContainerRequest.
+ @param containerId Container Id.
+ @param containerLaunchContext Container Launch Context.
+ @param autoCommit AutoCommit.
+ @return ReInitializationContainerRequest.]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to re-initialize.
+
+ @return <code>ContainerId</code> of the container to re-initialize.]]>
+      </doc>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to re-initialize the container
+ with.
+
+ @return <code>ContainerLaunchContext</code> of to re-initialize the
+ container with.]]>
+      </doc>
+    </method>
+    <method name="getAutoCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if AutoCommit is set for this ReInitialization.
+ @return If AutoCommit is set for this ReInitialization.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates all the required fields needed for a Container
+ ReInitialization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <class name="ReInitializeContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ReInitializeContainerRequest}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <class name="ReleaseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be released.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be released.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be released.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be released.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to release a resource in the shared cache.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <class name="ReleaseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response to clients from the <code>SharedCacheManager</code> when
+ releasing a resource in the shared cache.
+ </p>
+
+ <p>
+ Currently, this is empty.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <class name="ReservationDeleteRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDeleteRequest} captures the set of requirements the user
+ has to delete an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <class name="ReservationDeleteResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationDeleteResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation delete
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <class name="ReservationListRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+ @param endTime Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via  {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name to use to find reservations.
+
+ @return the queue name to use to find reservations.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name to use to find resource allocations.
+
+ @param queue Required. Cannot be null or empty.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id to use to find a reservation.
+
+ @return the reservation id of the reservation.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reservation id to use to find a reservation.
+
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @return the start time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setEndTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+      <doc>
+      <![CDATA[Set the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @param endTime Optional. If provided, only reservations that
+                start before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getIncludeResourceAllocations" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setIncludeResourceAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[Set the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire list of
+                {@code ResourceAllocationRequest} will be returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListRequest} captures the set of requirements the
+ user has to list reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <class name="ReservationListResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationAllocationState" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationAllocationState}, that corresponds
+ to a reservation in the scheduler.
+
+ @return the list of {@link ReservationAllocationState} which holds
+ information of a particular reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListResponse} captures the list of reservations that the
+ user has queried.
+
+ The resulting list of {@link ReservationAllocationState} contains a list of
+ {@code ResourceAllocationRequest} representing the current state of the
+ reservation resource allocations will be returned. This is subject to change
+ in the event of re-planning a described by {@code ReservationDefinition}
+
+ @see ReservationAllocationState]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <class name="ReservationSubmissionRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to.
+ 
+ @return the name of the {@code Plan} that corresponds to the name of the
+         {@link QueueInfo} in the scheduler to which the reservation will be
+         submitted to]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to
+ 
+ @param queueName the name of the parent {@code Plan} that corresponds to
+          the name of the {@link QueueInfo} in the scheduler to which the
+          reservation will be submitted to]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id that corresponds to the reservation submission.
+
+ @return reservation id that will be used to identify the reservation
+ submission.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id that corresponds to the reservation submission.
+
+ @param reservationId reservation id that will be used to identify the
+                      reservation submission.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationSubmissionRequest} captures the set of requirements the
+ user has to create a reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <class name="ReservationSubmissionResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ reservation submission.</p>
+
+ <p>Currently, this is empty.</p>
+
+ {@code ApplicationClientProtocol#submitReservation(
+ ReservationSubmissionRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <class name="ReservationUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationUpdateRequest} captures the set of requirements the user
+ has to update an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <class name="ReservationUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationUpdateResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation update
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <class name="ResourceLocalizationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="localResources" type="java.util.Map"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to localize resources.
+
+ @return <code>ContainerId</code> of the container to localize resources.]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the ApplicationMaster to ask for localizing resources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <class name="ResourceLocalizationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ResourceLocalizationRequest}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <class name="RestartContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RestartContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to a restart Container request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <class name="RollbackResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RollbackResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to a Rollback request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <class name="SignalContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="signalContainerCommand" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to signal.
+ @return <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="getCommand" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>SignalContainerCommand</code> of the signal request.
+ @return <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <method name="setCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <doc>
+      <![CDATA[Set the <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ or by the <code>ApplicationMaster</code> to the <code>NodeManager</code>
+ to signal a container.
+ @see SignalContainerCommand </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <class name="SignalContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ signalling a container.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <class name="StartContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>.
+ 
+ @return <code>ContainerLaunchContext</code> for the container to be started
+         by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>
+ @param context <code>ContainerLaunchContext</code> for the container to be 
+                started by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the container token to be used for authorization during starting
+ container.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the container token to be used for authorization during starting
+ container.
+ @see NMToken
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="setContainerToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>start</em> a container.</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as
+ allocated resource capability, security tokens (if enabled), command
+ to be executed to start the container, environment for the process, 
+ necessary binaries/jar/shared-objects etc. via the 
+ {@link ContainerLaunchContext}.</p>
+
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <class name="StartContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+    </method>
+    <method name="getStartContainerRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link StartContainerRequest} to start containers.
+ @return a list of {@link StartContainerRequest} to start containers.]]>
+      </doc>
+    </method>
+    <method name="setStartContainerRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of {@link StartContainerRequest} to start containers.
+ @param request a list of {@link StartContainerRequest} to start containers]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request which contains a list of {@link StartContainerRequest} sent by
+ the <code>ApplicationMaster</code> to the <code>NodeManager</code> to
+ <em>start</em> containers.
+ </p>
+ 
+ <p>
+ In each {@link StartContainerRequest}, the <code>ApplicationMaster</code> has
+ to provide details such as allocated resource capability, security tokens (if
+ enabled), command to be executed to start the container, environment for the
+ process, necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <class name="StartContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStartedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> s of the containers that are
+ started successfully.
+ 
+ @return the list of <code>ContainerId</code> s of the containers that are
+         started successfully.
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <method name="getAllServicesMetaData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the meta-data from all auxiliary services running on the
+ <code>NodeManager</code>.
+ </p>
+ <p>
+ The meta-data is returned as a Map between the auxiliary service names and
+ their corresponding per service meta-data as an opaque blob
+ <code>ByteBuffer</code>
+ </p>
+ 
+ <p>
+ To be able to interpret the per-service meta-data, you should consult the
+ documentation for the Auxiliary-service configured on the NodeManager
+ </p>
+ 
+ @return a Map between the names of auxiliary services and their
+         corresponding meta-data]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>start</em> an allocated
+ container.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <class name="StopContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code>s of the containers to be stopped.
+ @return <code>ContainerId</code>s of containers to be stopped]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code>s of the containers to be stopped.
+ @param containerIds <code>ContainerId</code>s of the containers to be stopped]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>stop</em> containers.</p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <class name="StopContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStoppedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of successfully stopped containers.
+ 
+ @return the list of containerIds of successfully stopped containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>stop</em> allocated
+ containers.
+ </p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <class name="SubmitApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationSubmissionContext</code> for the application.
+ @return <code>ApplicationSubmissionContext</code> for the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationSubmissionContext</code> for the application.
+ @param context <code>ApplicationSubmissionContext</code> for the 
+                application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to <em>submit an application</em> to the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>The request, via {@link ApplicationSubmissionContext}, contains
+ details such as queue, {@link Resource} required to run the 
+ <code>ApplicationMaster</code>, the equivalent of 
+ {@link ContainerLaunchContext} for launching the 
+ <code>ApplicationMaster</code> etc.
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <class name="SubmitApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ application submission.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <class name="UpdateApplicationPriorityRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ 
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ 
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ 
+ @return <code>Priority</code> of the application to be set.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+ 
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application priority.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and
+ {@link Priority} to be set for an application
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <class name="UpdateApplicationPriorityResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ @return Updated <code>Priority</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ the application priority.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <class name="UpdateApplicationTimeoutsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationTimeouts" type="java.util.Map"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application timeout.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and timeout
+ to be set for an application
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <class name="UpdateApplicationTimeoutsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ application timeout.
+ </p>
+ <p>
+ A response without exception means that the update has completed
+ successfully.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <class name="UseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be used.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be used.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be used.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be used.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to the <code>SharedCacheManager</code> that claims a
+ resource in the shared cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+  <class name="UseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Path</code> corresponding to the requested resource in the
+ shared cache.
+
+ @return String A <code>Path</code> if the resource exists in the shared
+         cache, <code>null</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setPath"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>Path</code> corresponding to a resource in the shared cache.
+
+ @param p A <code>Path</code> corresponding to a resource in the shared
+          cache]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response from the SharedCacheManager to the client that indicates whether
+ a requested resource exists in the cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records">
+  <!-- start class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <class name="AMCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.AMCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Command sent by the Resource Manager to the Application Master in the 
+ AllocateResponse 
+ @see AllocateResponse]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <class name="ApplicationAccessType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application access types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <class name="ApplicationAttemptId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationAttemptId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="attemptId" type="int"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the <code>ApplicationAttempId</code>. 
+ @return <code>ApplicationId</code> of the <code>ApplicationAttempId</code>]]>
+      </doc>
+    </method>
+    <method name="getAttemptId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>attempt id</code> of the <code>Application</code>.
+ @return <code>attempt id</code> of the <code>Application</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <field name="appAttemptIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
+ of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
+ 
+ <p>Multiple attempts might be needed to run an application to completion due
+ to temporal failures of the <code>ApplicationMaster</code> such as hardware
+ failures, connectivity issues etc. on the node on which it was scheduled.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <class name="ApplicationAttemptReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="rpcPort" type="int"/>
+      <param name="url" type="java.lang.String"/>
+      <param name="oUrl" type="java.lang.String"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"/>
+      <param name="amContainerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getYarnApplicationAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>YarnApplicationAttemptState</em> of the application attempt.
+ 
+ @return <em>YarnApplicationAttemptState</em> of the application attempt]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of this attempt <code>ApplicationMaster</code>.
+ 
+ @return <em>RPC port</em> of this attempt <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which this attempt of
+ <code>ApplicationMaster</code> is running.
+ 
+ @return <em>host</em> on which this attempt of
+         <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnositic information</em> of the application attempt in case
+ of errors.
+ 
+ @return <em>diagnositic information</em> of the application attempt in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application attempt.
+ 
+ @return <em>tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>original tracking url</em> for the application attempt.
+ 
+ @return <em>original tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of this attempt of the
+ application
+ 
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getAMContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of AMContainer for this attempt
+ 
+ @return <code>ContainerId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ 
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationAttemptReport} is a report of an application attempt.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationAttemptId} of the application.</li>
+   <li>Host on which the <code>ApplicationMaster</code> of this attempt is
+   running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code> of this attempt.</li>
+   <li>Tracking URL.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>{@link YarnApplicationAttemptState} of the application attempt.</li>
+   <li>{@link ContainerId} of the master Container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <class name="ApplicationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clusterTimestamp" type="long"/>
+      <param name="id" type="int"/>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the short integer identifier of the <code>ApplicationId</code>
+ which is unique for all applications started by a particular instance
+ of the <code>ResourceManager</code>.
+ @return short integer identifier of the <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the <code>ResourceManager</code> which is 
+ used to generate globally unique <code>ApplicationId</code>.
+ @return <em>start time</em> of the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="appIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationId</code> represents the <em>globally unique</em> 
+ identifier for an application.</p>
+ 
+ <p>The globally unique nature of the identifier is achieved by using the 
+ <em>cluster timestamp</em> i.e. start-time of the 
+ <code>ResourceManager</code> along with a monotonically increasing counter
+ for the application.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <class name="ApplicationReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getCurrentApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the current
+ attempt of the application
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>user</em> who submitted the application.
+ @return <em>user</em> who submitted the application]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application was submitted.
+ @return <em>queue</em> to which the application was submitted]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <em>name</em> of the application.
+ @return <em>name</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ is running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code>
+         is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of the <code>ApplicationMaster</code>.
+ @return <em>RPC port</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getClientToAMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>.
+ <p>
+ <em>ClientToAMToken</em> is the security token used by the AMs to verify
+ authenticity of any <code>client</code>.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code>, provides a secure token (via
+ {@link ApplicationReport#getClientToAMToken()}) which is verified by the
+ ApplicationMaster when the client directly talks to an AM.
+ </p>
+ @return <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getYarnApplicationState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnApplicationState</code> of the application.
+ @return <code>YarnApplicationState</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get  the <em>diagnositic information</em> of the application in case of
+ errors.
+ @return <em>diagnositic information</em> of the application in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application.
+ @return <em>tracking url</em> for the application]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the application.
+ @return <em>start time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getSubmitTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>final finish status</em> of the application.
+ @return <em>final finish status</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationResourceUsageReport" return="org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve the structure containing the job resources for this application
+ @return the job resources structure for this application]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's progress ( range 0.0 to 1.0 )
+ @return application's progress]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's Type 
+ @return application's Type]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all tags corresponding to the application
+ @return Application's tags]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+ 
+ @return the AM to RM token if available.]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationStatus" return="org.apache.hadoop.yarn.api.records.LogAggregationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get log aggregation status for the application
+ @return Application's log aggregation status]]>
+      </doc>
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedApp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="unmanagedApplication" type="boolean"/>
+      <doc>
+      <![CDATA[@param unmanagedApplication true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get priority of the application
+
+ @return Application's priority]]>
+      </doc>
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAppNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAmNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationReport} is a report of an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Applications user.</li>
+   <li>Application queue.</li>
+   <li>Application name.</li>
+   <li>Host on which the <code>ApplicationMaster</code> is running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code>.</li>
+   <li>Tracking URL.</li>
+   <li>{@link YarnApplicationState} of the application.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Start time of the application.</li>
+   <li>Client {@link Token} of the application (if security is enabled).</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <class name="ApplicationResourceUsageReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationResourceUsageReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumUsedContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of used containers.  -1 for invalid/inaccessible reports.
+ @return the number of used containers]]>
+      </doc>
+    </method>
+    <method name="getUsedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the used <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the used <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getReservedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the reserved <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getNeededResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the needed <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the needed <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory (in megabytes) the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores that the application has allocated
+ times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getQueueUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the queue that the app is using.
+ @return the percentage of resources of the queue that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getClusterUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the cluster that the app is using.
+ @return the percentage of resources of the cluster that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getPreemptedMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory preempted(in megabytes)
+ the application has allocated times the number of
+ seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated resource-seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated preempted
+ resource-seconds]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Contains various scheduling metrics to be reported by UI and CLI.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <class name="ApplicationSubmissionContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="amContainerLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="resourceRequest" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="attemptFailuresValidityInterval" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the submitted application.
+ @return <code>ApplicationId</code> of the submitted application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the submitted application.
+ @param applicationId <code>ApplicationId</code> of the submitted
+                      application]]>
+      </doc>
+    </method>
+    <method name="getApplicationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application <em>name</em>.
+ @return application name]]>
+      </doc>
+    </method>
+    <method name="setApplicationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application <em>name</em>.
+ @param applicationName application name]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application is being submitted.
+ @return <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue</em> to which the application is being submitted
+ @param queue <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application.
+ @return <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getAMContainerSpec" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @return <code>ContainerLaunchContext</code> for the 
+         <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerSpec"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @param amContainer <code>ContainerLaunchContext</code> for the 
+                    <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="getUnmanagedAM" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get if the RM should manage the execution of the AM. 
+ If true, then the RM 
+ will not allocate a container for the AM and start it. It will expect the 
+ AM to be launched and connect to the RM within the AM liveliness period and 
+ fail the app otherwise. The client should launch the AM only after the RM 
+ has ACCEPTED the application and changed the <code>YarnApplicationState</code>.
+ Such apps will not be retried by the RM on app attempt failure.
+ The default value is false.
+ @return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedAM"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[@param value true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getMaxAppAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of max attempts of the application to be submitted]]>
+      </doc>
+    </method>
+    <method name="setMaxAppAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxAppAttempts" type="int"/>
+      <doc>
+      <![CDATA[Set the number of max attempts of the application to be submitted. WARNING:
+ it should be no larger than the global number of max attempts in the Yarn
+ configuration.
+ @param maxAppAttempts the number of max attempts of the application
+ to be submitted.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource required by the <code>ApplicationMaster</code> for this
+ application. Please note this will be DEPRECATED, use <em>getResource</em>
+ in <em>getAMContainerResourceRequest</em> instead.
+ 
+ @return the resource required by the <code>ApplicationMaster</code> for
+         this application.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resource required by the <code>ApplicationMaster</code> for this
+ application.
+
+ @param resource the resource required by the <code>ApplicationMaster</code>
+ for this application.]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application type
+ 
+ @return the application type]]>
+      </doc>
+    </method>
+    <method name="setApplicationType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application type
+ 
+ @param applicationType the application type]]>
+      </doc>
+    </method>
+    <method name="getKeepContainersAcrossApplicationAttempts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates whether to keep containers across application
+ attempts or not.
+ 
+ @return the flag which indicates whether to keep containers across
+         application attempts or not.]]>
+      </doc>
+    </method>
+    <method name="setKeepContainersAcrossApplicationAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keepContainers" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which indicates whether to keep containers across application
+ attempts.
+ <p>
+ For managed AM, if the flag is true, running containers will not be killed
+ when application attempt fails and these containers will be retrieved by
+ the new application attempt on registration via
+ {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
+ </p>
+ <p>
+ For unmanaged AM, if the flag is true, RM allows re-register and returns
+ the running containers in the same attempt back to the UAM for HA.
+ </p>
+
+ @param keepContainers the flag which indicates whether to keep containers
+          across application attempts.]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get tags for the application
+
+ @return the application tags]]>
+      </doc>
+    </method>
+    <method name="setApplicationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set tags for the application. A maximum of
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
+ per application. Each tag can be at most
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
+ characters, and can contain only ASCII characters.
+
+ @param tags tags to set]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this app. If this is set, all containers of
+ this application without setting node-label-expression in ResurceRequest
+ will get allocated resources on only those nodes that satisfy this
+ node-label-expression.
+ 
+ If different node-label-expression of this app and ResourceRequest are set
+ at the same time, the one set in ResourceRequest will be used when
+ allocating container
+ 
+ @return node-label-expression for this app]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node-label-expression for this app
+ @param nodeLabelExpression node-label-expression of this app]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #getAMContainerResourceRequests()}">
+      <doc>
+      <![CDATA[Get the ResourceRequest of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will assemble a ResourceRequest by using
+ <em>getResource</em> and <em>getPriority</em> of
+ <em>ApplicationSubmissionContext</em>.
+
+ Number of containers and Priority will be ignored.
+
+ @return ResourceRequest of the AM container
+ @deprecated See {@link #getAMContainerResourceRequests()}]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #setAMContainerResourceRequests(List)}">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Set ResourceRequest of the AM container
+ @param request of the AM container
+ @deprecated See {@link #setAMContainerResourceRequests(List)}]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ResourceRequests of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will use the ResourceRequest as determined by
+ <em>getAMContainerResourceRequest</em> and its behavior.
+
+ Number of containers and Priority will be ignored.
+
+ @return List of ResourceRequests of the AM container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set ResourceRequests of the AM container.
+ @param requests of the AM container]]>
+      </doc>
+    </method>
+    <method name="getAttemptFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attemptFailuresValidityInterval in milliseconds for the application
+
+ @return the attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="setAttemptFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptFailuresValidityInterval" type="long"/>
+      <doc>
+      <![CDATA[Set the attemptFailuresValidityInterval in milliseconds for the application
+ @param attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LogAggregationContext</code> of the application
+
+ @return <code>LogAggregationContext</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+      <doc>
+      <![CDATA[Set <code>LogAggregationContext</code> for the application
+
+ @param logAggregationContext
+          for the application]]>
+      </doc>
+    </method>
+    <method name="getReservationID" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id, that corresponds to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @return the reservation id representing the unique id of the corresponding
+         reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationID" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id, that correspond to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @param reservationID representing the unique id of the
+          corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in seconds.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application in seconds.
+ All pre-existing Map entries are cleared before adding the new Map.
+ <p>
+ <b>Note:</b> If application timeout value is less than or equal to zero
+ then application submission will throw an exception.
+ </p>
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <method name="getApplicationSchedulingPropertiesMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get application scheduling environment variables stored as a key value
+ pair map for application.
+
+ @return placement envs for application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationSchedulingPropertiesMap"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingEnvMap" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the scheduling envs for the application.
+
+ @param schedulingEnvMap
+          A map of env's for the application scheduling preferences.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationSubmissionContext} represents all of the
+ information needed by the {@code ResourceManager} to launch
+ the {@code ApplicationMaster} for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Application user.</li>
+   <li>Application name.</li>
+   <li>{@link Priority} of the application.</li>
+   <li>
+     {@link ContainerLaunchContext} of the container in which the
+     <code>ApplicationMaster</code> is executed.
+   </li>
+   <li>
+     maxAppAttempts. The maximum number of application attempts.
+     It should be no larger than the global number of max attempts in the
+     YARN configuration.
+   </li>
+   <li>
+     attemptFailuresValidityInterval. The default value is -1.
+     when attemptFailuresValidityInterval in milliseconds is set to
+     {@literal >} 0, the failure number will no take failures which happen
+     out of the validityInterval into failure count. If failure count
+     reaches to maxAppAttempts, the application will be failed.
+   </li>
+   <li>Optional, application-specific {@link LogAggregationContext}</li>
+ </ul>
+ 
+ @see ContainerLaunchContext
+ @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <class name="ApplicationTimeout" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTimeout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <param name="expiryTime" type="java.lang.String"/>
+      <param name="remainingTime" type="long"/>
+    </method>
+    <method name="getTimeoutType" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application timeout type.
+ @return timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="setTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeoutType" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <doc>
+      <![CDATA[Set the application timeout type.
+ @param timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="getExpiryTime" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>expiryTime</code> for given timeout type.
+ @return expiryTime in ISO8601 standard with format
+         <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="setExpiryTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expiryTime" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <code>expiryTime</code> for given timeout type.
+ @param expiryTime in ISO8601 standard with format
+          <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="getRemainingTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>Remaining Time</code> of an application for given timeout type.
+ @return Remaining Time in seconds.]]>
+      </doc>
+    </method>
+    <method name="setRemainingTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="remainingTime" type="long"/>
+      <doc>
+      <![CDATA[Set <code>Remaining Time</code> of an application for given timeout type.
+ @param remainingTime in seconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationTimeout} is a report for configured application timeouts.
+ It includes details such as:
+ <ul>
+ <li>{@link ApplicationTimeoutType} of the timeout type.</li>
+ <li>Expiry time in ISO8601 standard with format
+ <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b> or "UNLIMITED".</li>
+ <li>Remaining time in seconds.</li>
+ </ul>
+ The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ <ul>
+ <li>{UNLIMITED,-1} : Timeout is not configured for given timeout type
+ (LIFETIME).</li>
+ <li>{ISO8601 date string, 0} : Timeout is configured and application has
+ completed.</li>
+ <li>{ISO8601 date string, greater than zero} : Timeout is configured and
+ application is RUNNING. Application will be timed out after configured
+ value.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <class name="ApplicationTimeoutType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application timeout type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <class name="CollectorInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CollectorInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getCollectorAddr" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCollectorAddr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.lang.String"/>
+    </method>
+    <method name="getCollectorToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get delegation token for app collector which AM will use to publish
+ entities.
+ @return the delegation token for app collector.]]>
+      </doc>
+    </method>
+    <method name="setCollectorToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <field name="DEFAULT_TIMESTAMP_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Collector info containing collector address and collector token passed from
+ RM to AM in Allocate Response.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Container -->
+  <class name="Container" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Container"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the globally unique identifier for the container.
+ @return globally unique identifier for the container]]>
+      </doc>
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the node on which the container is allocated.
+ @return identifier of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the http uri of the node on which the container is allocated.
+ @return http uri of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> at which the <code>Container</code> was
+ allocated.
+ @return <code>Priority</code> at which the <code>Container</code> was
+         allocated]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerToken</code> for the container.
+ <p><code>ContainerToken</code> is the security token used by the framework
+ to verify authenticity of any <code>Container</code>.</p>
+
+ <p>The <code>ResourceManager</code>, on container allocation provides a
+ secure token which is verified by the <code>NodeManager</code> on
+ container launch.</p>
+
+ <p>Applications do not need to care about <code>ContainerToken</code>, they
+ are transparently handled by the framework - the allocated
+ <code>Container</code> includes the <code>ContainerToken</code>.</p>
+
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+
+ @return <code>ContainerToken</code> for the container]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to the original {@code
+ ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+ this allocated {@code Container}.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}s.
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+
+ @return the <em>ID</em> corresponding to the original  allocation request
+ which is satisfied by this allocation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code Container} represents an allocated resource in the cluster.
+ <p>
+ The {@code ResourceManager} is the sole authority to allocate any
+ {@code Container} to applications. The allocated {@code Container}
+ is always on a single node and has a unique {@link ContainerId}. It has
+ a specific amount of {@link Resource} allocated.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} for the container, which is globally unique.</li>
+   <li>
+     {@link NodeId} of the node on which it is allocated.
+   </li>
+   <li>HTTP uri of the node.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>{@link Priority} at which the container was allocated.</li>
+   <li>
+     Container {@link Token} of the container, used to securely verify
+     authenticity of the allocation.
+   </li>
+ </ul>
+ 
+ Typically, an {@code ApplicationMaster} receives the {@code Container}
+ from the {@code ResourceManager} during resource-negotiation and then
+ talks to the {@code NodeManager} to start/stop containers.
+ 
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Container -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <class name="ContainerExitStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerExitStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ABORTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc.]]>
+      </doc>
+    </field>
+    <field name="DISKS_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad.]]>
+      </doc>
+    </field>
+    <field name="PREEMPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers preempted by the framework.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_VMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated virtual memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_PMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated physical memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_APPMASTER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by stop request by the app master.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_RESOURCEMANAGER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the resource manager.]]>
+      </doc>
+    </field>
+    <field name="KILLED_AFTER_APP_COMPLETION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated after the application finished.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_CONTAINER_SCHEDULER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the ContainerScheduler to make room
+ for another container...]]>
+      </doc>
+    </field>
+    <field name="KILLED_FOR_EXCESS_LOGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated for generating excess log data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Container exit statuses indicating special exit circumstances.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <class name="ContainerId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerId" type="long"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the application to which the
+ <code>Container</code> was assigned.
+ <p>
+ Note: If containers are kept alive across application attempts via
+ {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ the <code>ContainerId</code> does not necessarily contain the current
+ running application attempt's <code>ApplicationAttemptId</code> This
+ container can be allocated by previously exited application attempt and
+ managed by the current running attempt thus have the previous application
+ attempt's <code>ApplicationAttemptId</code>.
+ </p>
+ 
+ @return <code>ApplicationAttemptId</code> of the application to which the
+         <code>Container</code> was assigned]]>
+      </doc>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower 32 bits of identifier of the <code>ContainerId</code>,
+ which doesn't include epoch. Note that this method will be marked as
+ deprecated, so please use <code>getContainerId</code> instead.
+ @return lower 32 bits of identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the <code>ContainerId</code>. Upper 24 bits are
+ reserved as epoch of cluster, and lower 40 bits are reserved as
+ sequential number of containers.
+ @return identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return A string representation of containerId. The format is
+ container_e*epoch*_*clusterTimestamp*_*appId*_*attemptId*_*containerId*
+ when epoch is larger than 0
+ (e.g. container_e17_1410901177871_0001_01_000005).
+ *epoch* is increased when RM restarts or fails over.
+ When epoch is 0, epoch is omitted
+ (e.g. container_1410901177871_0001_01_000005).]]>
+      </doc>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="CONTAINER_ID_BITMASK" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ContainerId</code> represents a globally unique identifier
+ for a {@link Container} in the cluster.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <class name="ContainerLaunchContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLaunchContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+    </method>
+    <method name="getTokens" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the tokens needed by this container. It may include file-system
+ tokens, ApplicationMaster related tokens if this container is an
+ ApplicationMaster or framework level tokens needed by this container to
+ communicate to various services in a secure manner.
+ 
+ @return tokens needed by this container.]]>
+      </doc>
+    </method>
+    <method name="setTokens"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set security tokens needed by this container.
+ @param tokens security tokens]]>
+      </doc>
+    </method>
+    <method name="getTokensConf" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration used by RM to renew tokens.
+ @return The configuration used by RM to renew the tokens.]]>
+      </doc>
+    </method>
+    <method name="setTokensConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokensConf" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set the configuration used by RM to renew the tokens.
+ @param tokensConf The configuration used by RM to renew the tokens]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="setLocalResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set <code>LocalResource</code> required by the container. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param localResources <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="getServiceData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}.
+ </p>
+ 
+ <p>
+ This will be used to initialize this application on the specific
+ {@link AuxiliaryService} running on the NodeManager by calling
+ {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)}
+ </p>
+ 
+ @return application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="setServiceData"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceData" type="java.util.Map"/>
+      <doc>
+      <![CDATA[<p>
+ Set application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}. All pre-existing Map entries are
+ preserved.
+ </p>
+ 
+ @param serviceData
+          application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>environment variables</em> for the container.
+ @return <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="setEnvironment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="environment" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add <em>environment variables</em> for the container. All pre-existing Map
+ entries are cleared before adding the new Map
+ @param environment <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getCommands" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>commands</em> for launching the container.
+ @return the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="setCommands"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add the list of <em>commands</em> for launching the container. All
+ pre-existing List entries are cleared before adding the new List
+ @param commands the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="setApplicationACLs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationACL</code>s for the application. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param acls <code>ApplicationACL</code>s for the application]]>
+      </doc>
+    </method>
+    <method name="getContainerRetryContext" return="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerRetryContext</code> to relaunch container.
+ @return <code>ContainerRetryContext</code> to relaunch container.]]>
+      </doc>
+    </method>
+    <method name="setContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerRetryContext</code> to relaunch container.
+ @param containerRetryContext <code>ContainerRetryContext</code> to
+                              relaunch container.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerLaunchContext} represents all of the information
+ needed by the {@code NodeManager} to launch a container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>User to whom the container is allocated.</li>
+   <li>Security tokens (if security is enabled).</li>
+   <li>
+     {@link LocalResource} necessary for running the container such
+     as binaries, jar, shared-objects, side-files etc.
+   </li>
+   <li>Optional, application-specific binary service data.</li>
+   <li>Environment variables for the launched process.</li>
+   <li>Command to launch the container.</li>
+   <li>Retry strategy when container exits with failure.</li>
+ </ul>
+ 
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <class name="ContainerReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ 
+ @return <code>ContainerId</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getAllocatedResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Resource</code> of the container.
+ 
+ @return allocated <code>Resource</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getAssignedNode" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>NodeId</code> where container is running.
+ 
+ @return allocated <code>NodeId</code> where container is running.]]>
+      </doc>
+    </method>
+    <method name="setAssignedNode"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Priority</code> of the container.
+ 
+ @return allocated <code>Priority</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the creation time of the container.
+ 
+ @return creation time of the container]]>
+      </doc>
+    </method>
+    <method name="setCreationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="creationTime" type="long"/>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Finish time of the container.
+ 
+ @return Finish time of the container]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the DiagnosticsInfo of the container.
+ 
+ @return DiagnosticsInfo of the container]]>
+      </doc>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the LogURL of the container.
+ 
+ @return LogURL of the container]]>
+      </doc>
+    </method>
+    <method name="setLogUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logUrl" type="java.lang.String"/>
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>ContainerState</code> of the container.
+ 
+ @return final <code>ContainerState</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"/>
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>exit status</code> of the container.
+ 
+ @return final <code>exit status</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerExitStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerExitStatus" type="int"/>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Node Http address of the container
+ 
+ @return the node http address of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the execution type of the container.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerReport} is a report of an container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>Allocated Resources to the container.</li>
+   <li>Assigned Node id.</li>
+   <li>Assigned Priority.</li>
+   <li>Creation Time.</li>
+   <li>Finish Time.</li>
+   <li>Container Exit Status.</li>
+   <li>{@link ContainerState} of the container.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Log URL.</li>
+   <li>nodeHttpAddress</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <class name="ContainerRetryContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerRetryContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRetryPolicy" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryPolicy" type="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"/>
+    </method>
+    <method name="getErrorCodes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setErrorCodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCodes" type="java.util.Set"/>
+    </method>
+    <method name="getMaxRetries" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxRetries"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxRetries" type="int"/>
+    </method>
+    <method name="getRetryInterval" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryInterval" type="int"/>
+    </method>
+    <method name="getFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failuresValidityInterval" type="long"/>
+    </method>
+    <field name="RETRY_FOREVER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RETRY_INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NEVER_RETRY_CONTEXT" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@code ContainerRetryContext} indicates how container retry after it fails
+ to run.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>
+     {@link ContainerRetryPolicy} :
+     - NEVER_RETRY(DEFAULT value): no matter what error code is when container
+       fails to run, just do not retry.
+     - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails
+       to run, just retry.
+     - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if
+       the error code is one of <em>errorCodes</em>, otherwise do not retry.
+
+     Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry
+     because it is usually killed on purpose.
+   </li>
+   <li>
+     <em>maxRetries</em> specifies how many times to retry if need to retry.
+     If the value is -1, it means retry forever.
+   </li>
+   <li><em>retryInterval</em> specifies delaying some time before relaunch
+   container, the unit is millisecond.</li>
+   <li>
+     <em>failuresValidityInterval</em>: default value is -1.
+     When failuresValidityInterval in milliseconds is set to {@literal >} 0,
+     the failure number will not take failures which happen out of the
+     failuresValidityInterval into failure count. If failure count
+     reaches to <em>maxRetries</em>, the container will be failed.
+   </li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <class name="ContainerRetryPolicy" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>Retry policy for relaunching a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <class name="ContainerState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <class name="ContainerStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerState</code> of the container.
+ @return <code>ContainerState</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the <em>exit status</em> for the container.</p>
+  
+ <p>Note: This is valid only for completed containers i.e. containers
+ with state {@link ContainerState#COMPLETE}. 
+ Otherwise, it returns an ContainerExitStatus.INVALID.
+ </p>
+ 
+ <p>Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc. have a special
+ exit code of ContainerExitStatus.ABORTED.</p>
+ 
+ <p>When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad, then
+ container is not launched and is exited with ContainersExitStatus.DISKS_FAILED.
+ </p>
+  
+ @return <em>exit status</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic messages</em> for failed containers.
+ @return <em>diagnostic messages</em> for failed containers]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getIPs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the IP addresses with which the container run.
+ @return The IP address where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the hostname where the container runs.
+ @return The hostname where the container runs.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerStatus} represents the current status of a
+ {@code Container}.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>{@code ContainerId} of the container.</li>
+   <li>{@code ExecutionType} of the container.</li>
+   <li>{@code ContainerState} of the container.</li>
+   <li><em>Exit status</em> of a completed container.</li>
+   <li><em>Diagnostic</em> message for a failed container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <class name="ContainerSubState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerSubState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerSubState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container Sub-State.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <class name="ContainerUpdateType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Encodes the type of Container Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <class name="ExecutionType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ExecutionType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container property encoding execution semantics.
+
+ <p>
+ The execution types are the following:
+ <ul>
+   <li>{@link #GUARANTEED} - this container is guaranteed to start its
+   execution, once the corresponding start container request is received by
+   an NM.
+   <li>{@link #OPPORTUNISTIC} - the execution of this container may not start
+   immediately at the NM that receives the corresponding start container
+   request (depending on the NM's available resources). Moreover, it may be
+   preempted if it blocks a GUARANTEED container from being executed.
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <class name="ExecutionTypeRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ExecutionTypeRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="ensureExecutionType" type="boolean"/>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionType</code> of the requested container.
+
+ @param execType
+          ExecutionType of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ExecutionType</code>.
+
+ @return <code>ExecutionType</code>.]]>
+      </doc>
+    </method>
+    <method name="setEnforceExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enforceExecutionType" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to explicitly ask that the Scheduling Authority return
+ Containers of exactly the Execution Type requested.
+ @param enforceExecutionType whether ExecutionType request should be
+                            strictly honored.]]>
+      </doc>
+    </method>
+    <method name="getEnforceExecutionType" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether Scheduling Authority should return Containers of exactly the
+ Execution Type requested for this <code>ResourceRequest</code>.
+ Defaults to false.
+ @return whether ExecutionType request should be strictly honored]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object of this class represents a specification of the execution
+ guarantee of the Containers associated with a ResourceRequest. It consists
+ of an <code>ExecutionType</code> as well as flag that explicitly asks the
+ configuredScheduler to return Containers of exactly the Execution Type
+ requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <class name="FinalApplicationStatus" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various final states of an <code>Application</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <class name="LocalResource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalResource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>location</em> of the resource to be localized.
+ @return <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <doc>
+      <![CDATA[Set <em>location</em> of the resource to be localized.
+ @param resource <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getSize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>size</em> of the resource to be localized.
+ @return <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>size</em> of the resource to be localized.
+ @param size <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @return <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @param timestamp <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceType</code> of the resource to be localized.
+ @return <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceType</code> of the resource to be localized.
+ @param type <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getVisibility" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @return <code>LocalResourceVisibility</code> of the resource to be 
+         localized]]>
+      </doc>
+    </method>
+    <method name="setVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @param visibility <code>LocalResourceVisibility</code> of the resource to be 
+                   localized]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @return <em>pattern</em> that should be used to extract entries from the 
+ archive.]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @param pattern <em>pattern</em> that should be used to extract entries 
+ from the archive.]]>
+      </doc>
+    </method>
+    <method name="getShouldBeUploadedToSharedCache" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM uses it to decide whether if it is necessary to upload the resource to
+ the shared cache
+ @return true if it is necessary to upload the resource
+                 to the shared cache,
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="setShouldBeUploadedToSharedCache"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+      <doc>
+      <![CDATA[Inform NM whether upload to SCM is needed.
+
+ @param shouldBeUploadedToSharedCache <em>shouldBeUploadedToSharedCache</em>
+          of this request]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>LocalResource</code> represents a local resource required to
+ run a container.</p>
+ 
+ <p>The <code>NodeManager</code> is responsible for localizing the resource 
+ prior to launching the container.</p>
+ 
+ <p>Applications can specify {@link LocalResourceType} and 
+ {@link LocalResourceVisibility}.</p>
+ 
+ @see LocalResourceType
+ @see LocalResourceVisibility
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <class name="LocalResourceType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceType} specifies the <em>type</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>type</em> can be one of:
+ <ul>
+   <li>
+     {@link #FILE} - Regular file i.e. uninterpreted bytes.
+   </li>
+   <li>
+     {@link #ARCHIVE} - Archive, which is automatically unarchived by the
+     <code>NodeManager</code>.
+   </li>
+   <li>
+     {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}.
+   </li>
+ </ul>
+
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <class name="LocalResourceVisibility" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceVisibility} specifies the <em>visibility</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>visibility</em> can be one of:
+ <ul>
+   <li>{@link #PUBLIC} - Shared by all users on the node.</li>
+   <li>
+     {@link #PRIVATE} - Shared among all applications of the
+     <em>same user</em> on the node.
+   </li>
+   <li>
+     {@link #APPLICATION} - Shared only among containers of the
+     <em>same application</em> on the node.
+   </li>
+ </ul>
+ 
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <class name="LogAggregationContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <param name="policyClassName" type="java.lang.String"/>
+      <param name="policyParameters" type="java.lang.String"/>
+    </method>
+    <method name="getIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param includePattern]]>
+      </doc>
+    </method>
+    <method name="getExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="excludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param excludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern in a rolling fashion.
+ 
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern in a rolling fashion.
+ 
+ @param rolledLogsIncludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern for aggregation in a rolling fashion.
+ 
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern for in a rolling fashion.
+ 
+ @param rolledLogsExcludePattern]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyClassName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy class.
+
+ @return log aggregation policy class]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyClassName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy class.
+
+ @param className]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyParameters" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy parameters.
+
+ @return log aggregation policy parameters]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy parameters.
+ There is no schema defined for the parameters string.
+ It is up to the log aggregation policy class to decide how to parse
+ the parameters string.
+
+ @param parameters]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code LogAggregationContext} represents all of the
+ information needed by the {@code NodeManager} to handle
+ the logs for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>
+     includePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be uploaded when the application finishes.
+   </li>
+   <li>
+     excludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be uploaded when application finishes. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     rolledLogsIncludePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be aggregated in a rolling fashion.
+   </li>
+   <li>
+     rolledLogsExcludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be aggregated in a rolling fashion. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     policyClassName. The policy class name that implements
+     ContainerLogAggregationPolicy. At runtime, nodemanager will the policy
+     if a given container's log should be aggregated based on the
+     ContainerType and other runtime state such as exit code by calling
+     ContainerLogAggregationPolicy#shouldDoLogAggregation.
+     This is useful when the app only wants to aggregate logs of a subset of
+     containers. Here are the available policies. Please make sure to specify
+     the canonical name by prefixing org.apache.hadoop.yarn.server.
+     nodemanager.containermanager.logaggregation.
+     to the class simple name below.
+     NoneContainerLogAggregationPolicy: skip aggregation for all containers.
+     AllContainerLogAggregationPolicy: aggregate all containers.
+     AMOrFailedContainerLogAggregationPolicy: aggregate application master
+         or failed containers.
+     FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed
+         containers
+     FailedContainerLogAggregationPolicy: aggregate failed containers
+     AMOnlyLogAggregationPolicy: aggregate application master containers
+     SampleContainerLogAggregationPolicy: sample logs of successful worker
+         containers, in addition to application master and failed/killed
+         containers.
+     If it isn't specified, it will use the cluster-wide default policy
+     defined by configuration yarn.nodemanager.log-aggregation.policy.class.
+     The default value of yarn.nodemanager.log-aggregation.policy.class is
+     AllContainerLogAggregationPolicy.
+   </li>
+   <li>
+     policyParameters. The parameters passed to the policy class via
+     ContainerLogAggregationPolicy#parseParameters during the policy object
+     initialization. This is optional. Some policy class might use parameters
+     to adjust its settings. It is up to policy class to define the scheme of
+     parameters.
+     For example, SampleContainerLogAggregationPolicy supports the format of
+     "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50
+     successful worker containers.
+   </li>
+ </ul>
+
+ @see ApplicationSubmissionContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NMToken -->
+  <class name="NMToken" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMToken"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link NodeId} of the <code>NodeManager</code> for which the NMToken
+ is used to authenticate.
+ @return the {@link NodeId} of the <code>NodeManager</code> for which the
+ NMToken is used to authenticate.]]>
+      </doc>
+    </method>
+    <method name="setNodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Token} used for authenticating with <code>NodeManager</code>
+ @return the {@link Token} used for authenticating with <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The NMToken is used for authenticating communication with
+ <code>NodeManager</code></p>
+ <p>It is issued by <code>ResourceMananger</code> when <code>ApplicationMaster</code>
+ negotiates resource with <code>ResourceManager</code> and
+ validated on <code>NodeManager</code> side.</p>
+ @see  AllocateResponse#getNMTokens()]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NMToken -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <class name="NodeAttribute" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttribute"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <field name="PREFIX_DISTRIBUTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREFIX_CENTRALIZED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute is a kind of a label which represents one of the
+ attribute/feature of a Node. Its different from node partition label as
+ resource guarantees across the queues will not be maintained for these type
+ of labels.
+ </p>
+ <p>
+ A given Node can be mapped with any kind of attribute, few examples are
+ HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ </p>
+ <p>
+ Its not compulsory for all the attributes to have value, empty string is the
+ default value of the <code>NodeAttributeType.STRING</code>
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <class name="NodeAttributeInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttribute" type="org.apache.hadoop.yarn.api.records.NodeAttribute"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute Info describes a NodeAttribute.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <class name="NodeAttributeKey" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="getAttributePrefix" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributePrefix"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute
+ is identified based on attribute prefix and name.
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <class name="NodeAttributeOpCode" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various node attribute op codes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <class name="NodeAttributeType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Type of a <code>node Attribute</code>.
+ </p>
+ Based on this attribute expressions and values will be evaluated.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeId -->
+  <class name="NodeId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>hostname</em> of the node.
+ @return <em>hostname</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>port</em> for communicating with the node.
+ @return <em>port</em> for communicating with the node]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p><code>NodeId</code> is the unique identifier for a node.</p>
+ 
+ <p>It includes the <em>hostname</em> and <em>port</em> to uniquely 
+ identify the node. Thus, it is unique across restarts of any 
+ <code>NodeManager</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <class name="NodeLabel" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeLabel"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isExclusive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeLabel"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DEFAULT_NODE_LABEL_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default node label partition used for displaying.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABEL_EXPRESSION_NOT_SET" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node Label expression not set .]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABEL_EXCLUSIVITY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, node label is exclusive or not]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <class name="NodeReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeId</code> of the node.
+ @return <code>NodeId</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getNodeState" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeState</code> of the node.
+ @return <code>NodeState</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>http address</em> of the node.
+ @return <em>http address</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getRackName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>rack name</em> for the node.
+ @return <em>rack name</em> for the node]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>used</em> <code>Resource</code> on the node.
+ @return <em>used</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>total</em> <code>Resource</code> on the node.
+ @return <em>total</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get labels of this node.
+ @return labels of this node.]]>
+      </doc>
+    </method>
+    <method name="getAggregatedContainersUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get containers aggregated resource utilization in a node.
+ @return containers resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getNodeUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node resource utilization.
+ @return node resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getDecommissioningTimeout" return="java.lang.Integer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional decommissioning timeout in seconds (null indicates absent
+ timeout).
+ @return the decommissioning timeout in second.]]>
+      </doc>
+    </method>
+    <method name="setDecommissioningTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decommissioningTimeout" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Set the decommissioning timeout in seconds (null indicates absent timeout).]]>
+      </doc>
+    </method>
+    <method name="getNodeUpdateType" return="org.apache.hadoop.yarn.api.records.NodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional node update type (null indicates absent update type).
+ @return the node update.]]>
+      </doc>
+    </method>
+    <method name="setNodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeUpdateType" type="org.apache.hadoop.yarn.api.records.NodeUpdateType"/>
+      <doc>
+      <![CDATA[Set the node update type (null indicates absent node update type).]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the node attributes of node.
+
+ @param nodeAttributes set of node attributes.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes of node.
+ @return the set of node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeReport} is a summary of runtime information of a node
+ in the cluster.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link NodeId} of the node.</li>
+   <li>HTTP Tracking URL of the node.</li>
+   <li>Rack name for the node.</li>
+   <li>Used {@link Resource} on the node.</li>
+   <li>Total available {@link Resource} of the node.</li>
+   <li>Number of running containers on the node.</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeState -->
+  <class name="NodeState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isUnusable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInactiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isActiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Node</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <class name="NodeToAttributeValue" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeToAttributeValue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeToAttributeValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getHostname" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHostname"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Mapping of Attribute Value to a Node.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <class name="PreemptionContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Container referenced by this handle.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specific container requested back by the <code>ResourceManager</code>.
+ @see PreemptionContract
+ @see StrictPreemptionContract]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <class name="PreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the AM releases resources matching these requests, then the {@link
+ PreemptionContainer}s enumerated in {@link #getContainers()} should not be
+ evicted from the cluster. Due to delays in propagating cluster state and
+ sending these messages, there are conditions where satisfied contracts may
+ not prevent the platform from killing containers.
+ @return List of {@link PreemptionResourceRequest} to update the
+ <code>ApplicationMaster</code> about resources requested back by the
+ <code>ResourceManager</code>.
+ @see AllocateRequest#setAskList(List)]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Assign the set of {@link PreemptionContainer} specifying which containers
+ owned by the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>. If the AM prefers a different set of
+ containers, then it may checkpoint or kill containers matching the
+ description in {@link #getResourceRequest}.
+ @return Set of containers at risk if the contract is not met.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the <code>ResourceManager</code>.
+ The <code>ApplicationMaster</code> (AM) can satisfy this request according
+ to its own priorities to prevent containers from being forcibly killed by
+ the platform.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <class name="PreemptionMessage" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionMessage"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStrictContract" return="org.apache.hadoop.yarn.api.records.StrictPreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Specific resources that may be killed by the
+ <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContract" return="org.apache.hadoop.yarn.api.records.PreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Contract describing resources to return to the cluster.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link PreemptionMessage} is part of the RM-AM protocol, and it is used by
+ the RM to specify resources that the RM wants to reclaim from this
+ {@code ApplicationMaster} (AM). The AM receives a {@link
+ StrictPreemptionContract} message encoding which containers the platform may
+ forcibly kill, granting it an opportunity to checkpoint state or adjust its
+ execution plan. The message may also include a {@link PreemptionContract}
+ granting the AM more latitude in selecting which resources to return to the
+ cluster.
+ <p>
+ The AM should decode both parts of the message. The {@link
+ StrictPreemptionContract} specifies particular allocations that the RM
+ requires back. The AM can checkpoint containers' state, adjust its execution
+ plan to move the computation, or take no action and hope that conditions that
+ caused the RM to ask for the container will change.
+ <p>
+ In contrast, the {@link PreemptionContract} also includes a description of
+ resources with a set of containers. If the AM releases containers matching
+ that profile, then the containers enumerated in {@link
+ PreemptionContract#getContainers()} may not be killed.
+ <p>
+ Each preemption message reflects the RM's current understanding of the
+ cluster state, so a request to return <em>N</em> containers may not
+ reflect containers the AM is releasing, recently exited containers the RM has
+ yet to learn about, or new containers allocated before the message was
+ generated. Conversely, an RM may request a different profile of containers in
+ subsequent requests.
+ <p>
+ The policy enforced by the RM is part of the scheduler. Generally, only
+ containers that have been requested consistently should be killed, but the
+ details are not specified.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <class name="PreemptionResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Resource described in this request, to be matched against running
+ containers.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the cluster.
+ @see PreemptionContract
+ @see AllocateRequest#setAskList(java.util.List)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Priority -->
+  <class name="Priority" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Priority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="int"/>
+    </method>
+    <method name="getPriority" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the assigned priority
+ @return the assigned priority]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Set the assigned priority
+ @param priority the assigned priority]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="UNDEFINED" type="org.apache.hadoop.yarn.api.records.Priority"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The priority assigned to a ResourceRequest or Application or Container 
+ allocation]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Priority -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <class name="QueueACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code QueueACL} enumerates the various ACLs for queues.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue.
+   </li>
+   <li>{@link #ADMINISTER_QUEUE} - ACL to administer the queue.</li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>name</em> of the queue.
+ @return <em>name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>configured capacity</em> of the queue.
+ @return <em>configured capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getMaximumCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>maximum capacity</em> of the queue.
+ @return <em>maximum capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCurrentCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current capacity</em> of the queue.
+ @return <em>current capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>child queues</em> of the queue.
+ @return <em>child queues</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>running applications</em> of the queue.
+ @return <em>running applications</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueState" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueState</code> of the queue.
+ @return <code>QueueState</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getAccessibleNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>accessible node labels</code> of the queue.
+ @return <code>accessible node labels</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getDefaultNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>default node label expression</code> of the queue, this takes
+ affect only when the <code>ApplicationSubmissionContext</code> and
+ <code>ResourceRequest</code> don't specify their
+ <code>NodeLabelExpression</code>.
+ 
+ @return <code>default node label expression</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setDefaultNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getQueueStatistics" return="org.apache.hadoop.yarn.api.records.QueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>queue stats</code> for the queue
+
+ @return <code>queue stats</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueStatistics" type="org.apache.hadoop.yarn.api.records.QueueStatistics"/>
+      <doc>
+      <![CDATA[Set the queue statistics for the queue
+ 
+ @param queueStatistics
+          the queue statistics]]>
+      </doc>
+    </method>
+    <method name="getPreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>preemption status</em> of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return <em>preemption status</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueConfigurations" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the per-node-label queue configurations of the queue.
+
+ @return the per-node-label queue configurations of the queue.]]>
+      </doc>
+    </method>
+    <method name="getIntraQueuePreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the intra-queue preemption status of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return intra-queue preemption status of the queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[QueueInfo is a report of the runtime information of the queue.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>Queue name.</li>
+   <li>Capacity of the queue.</li>
+   <li>Maximum capacity of the queue.</li>
+   <li>Current capacity of the queue.</li>
+   <li>Child queues.</li>
+   <li>Running applications.</li>
+   <li>{@link QueueState} of the queue.</li>
+   <li>{@link QueueConfigurations} of the queue.</li>
+ </ul>
+
+ @see QueueState
+ @see QueueConfigurations
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of a Queue.
+ <p>
+ A queue is in one of:
+ <ul>
+   <li>{@link #RUNNING} - normal state.</li>
+   <li>{@link #STOPPED} - not accepting new application submissions.</li>
+   <li>
+     {@link #DRAINING} - not accepting new application submissions
+     and waiting for applications finish.
+   </li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <class name="QueueStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumAppsSubmitted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of apps submitted
+ 
+ @return the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="setNumAppsSubmitted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsSubmitted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of apps submitted
+ 
+ @param numAppsSubmitted
+          the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="getNumAppsRunning" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running apps
+ 
+ @return the number of running apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsRunning"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsRunning" type="long"/>
+      <doc>
+      <![CDATA[Set the number of running apps
+ 
+ @param numAppsRunning
+          the number of running apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsPending" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending apps
+ 
+ @return the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsPending"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsPending" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending apps
+ 
+ @param numAppsPending
+          the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsCompleted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of completed apps
+ 
+ @return the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsCompleted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsCompleted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of completed apps
+ 
+ @param numAppsCompleted
+          the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsKilled" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of killed apps
+ 
+ @return the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsKilled"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsKilled" type="long"/>
+      <doc>
+      <![CDATA[Set the number of killed apps
+ 
+ @param numAppsKilled
+          the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsFailed" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of failed apps
+ 
+ @return the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsFailed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsFailed" type="long"/>
+      <doc>
+      <![CDATA[Set the number of failed apps
+ 
+ @param numAppsFailed
+          the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="getNumActiveUsers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active users
+ 
+ @return the number of active users]]>
+      </doc>
+    </method>
+    <method name="setNumActiveUsers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numActiveUsers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of active users
+ 
+ @param numActiveUsers
+          the number of active users]]>
+      </doc>
+    </method>
+    <method name="getAvailableMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available memory in MB
+ 
+ @return the available memory]]>
+      </doc>
+    </method>
+    <method name="setAvailableMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the available memory in MB
+ 
+ @param availableMemoryMB
+          the available memory]]>
+      </doc>
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated memory in MB
+ 
+ @return the allocated memory]]>
+      </doc>
+    </method>
+    <method name="setAllocatedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated memory in MB
+ 
+ @param allocatedMemoryMB
+          the allocate memory]]>
+      </doc>
+    </method>
+    <method name="getPendingMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending memory in MB
+ 
+ @return the pending memory]]>
+      </doc>
+    </method>
+    <method name="setPendingMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the pending memory in MB
+ 
+ @param pendingMemoryMB
+          the pending memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved memory in MB
+ 
+ @return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved memory in MB
+ 
+ @param reservedMemoryMB
+          the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getAvailableVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available vcores
+ 
+ @return the available vcores]]>
+      </doc>
+    </method>
+    <method name="setAvailableVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the available vcores
+ 
+ @param availableVCores
+          the available vcores]]>
+      </doc>
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated vcores
+ 
+ @return the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="setAllocatedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated vcores
+ 
+ @param allocatedVCores
+          the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending vcores
+ 
+ @return the pending vcores]]>
+      </doc>
+    </method>
+    <method name="setPendingVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the pending vcores
+ 
+ @param pendingVCores
+          the pending vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending containers.
+ @return the number of pending containers.]]>
+      </doc>
+    </method>
+    <method name="setPendingContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending containers.
+ @param pendingContainers the pending containers.]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of allocated containers.
+ @return the number of allocated containers.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of allocated containers.
+ @param allocatedContainers the allocated containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved containers.
+ @return the number of reserved containers.]]>
+      </doc>
+    </method>
+    <method name="setReservedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of reserved containers.
+ @param reservedContainers the reserved containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved vcores
+ 
+ @return the reserved vcores]]>
+      </doc>
+    </method>
+    <method name="setReservedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved vcores
+ 
+ @param reservedVCores
+          the reserved vcores]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <class name="QueueUserACLInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueUserACLInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> of the queue.
+ @return <em>queue name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getUserAcls" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>QueueACL</code> for the given user.
+ @return list of <code>QueueACL</code> for the given user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>QueueUserACLInfo</code> provides information {@link QueueACL} for
+ the given user.</p>
+ 
+ @see QueueACL
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <class name="RejectedSchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RejectedSchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Create new RejectedSchedulingRequest.
+ @param reason Rejection Reason.
+ @param request Rejected Scheduling Request.
+ @return RejectedSchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="getReason" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get Rejection Reason.
+ @return Rejection reason.]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <doc>
+      <![CDATA[Set Rejection Reason.
+ @param reason Rejection Reason.]]>
+      </doc>
+    </method>
+    <method name="getRequest" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Rejected Scheduling Request.
+ @return SchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="setRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Set the SchedulingRequest.
+ @param request SchedulingRequest.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates a Rejected SchedulingRequest. It contains the offending
+ Scheduling Request along with the reason for rejection.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <class name="RejectionReason" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.RejectionReason[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Reason for rejecting a Scheduling Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <class name="ReservationACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationACL} enumerates the various ACLs for reservations.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and
+     delete reservations.
+   </li>
+   <li> {@link #LIST_RESERVATIONS} - ACL to list reservations. </li>
+   <li> {@link #SUBMIT_RESERVATIONS} - ACL to create reservations. </li>
+ </ul>
+ Users can always list, update and delete their own reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <class name="ReservationAllocationState" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationAllocationState"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationAllocationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acceptanceTime" type="long"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="resourceAllocations" type="java.util.List"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[@param acceptanceTime The acceptance time of the reservation.
+ @param user The username of the user who made the reservation.
+ @param resourceAllocations List of {@link ResourceAllocationRequest}
+                            representing the current state of the
+                            reservation resource allocations. This is
+                            subject to change in the event of re-planning.
+ @param reservationId {@link ReservationId } of the reservation being
+                                            listed.
+ @param reservationDefinition {@link ReservationDefinition} used to make
+                              the reservation.
+ @return {@code ReservationAllocationState} that represents the state of
+ the reservation.]]>
+      </doc>
+    </method>
+    <method name="getAcceptanceTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the acceptance time of the reservation.
+
+ @return the time that the reservation was accepted.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user who made the reservation.
+
+ @return the name of the user who made the reservation.]]>
+      </doc>
+    </method>
+    <method name="getResourceAllocationRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Resource allocations of the reservation based on the current state
+ of the plan. This is subject to change in the event of re-planning.
+ The allocations will be constraint to the user contract as described by
+ the {@link ReservationDefinition}
+
+ @return a list of resource allocations for the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of the reservation.
+
+ @return the reservation id corresponding to the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation definition used to make the reservation.
+
+ @return the reservation definition used to make the reservation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationAllocationState} represents the reservation that is
+ made by a user.
+ <p>
+ It includes:
+ <ul>
+   <li>Duration of the reservation.</li>
+   <li>Acceptance time of the duration.</li>
+   <li>
+       List of {@link ResourceAllocationRequest}, which includes the time
+       interval, and capability of the allocation.
+       {@code ResourceAllocationRequest} represents an allocation
+       made for a reservation for the current state of the queue. This can be
+       changed for reasons such as re-planning, but will always be subject to
+       the constraints of the user contract as described by
+       {@link ReservationDefinition}
+   </li>
+   <li>{@link ReservationId} of the reservation.</li>
+   <li>{@link ReservationDefinition} used to make the reservation.</li>
+ </ul>
+
+ @see ResourceAllocationRequest
+ @see ReservationId
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <class name="ReservationDefinition" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDefinition"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getArrival" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @return the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="setArrival"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="earliestStartTime" type="long"/>
+      <doc>
+      <![CDATA[Set the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @param earliestStartTime the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="getDeadline" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @return the deadline or the latest time by when the resource(s) must be
+         allocated]]>
+      </doc>
+    </method>
+    <method name="setDeadline"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="latestEndTime" type="long"/>
+      <doc>
+      <![CDATA[Set the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @param latestEndTime the deadline or the latest time by when the
+          resource(s) should be allocated]]>
+      </doc>
+    </method>
+    <method name="getReservationRequests" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="setReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @param reservationRequests the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="getReservationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @return string representing the name of the corresponding reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @param name representing the name of the corresponding reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getRecurrenceExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @return recurrence of this reservation]]>
+      </doc>
+    </method>
+    <method name="setRecurrenceExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @param recurrenceExpression recurrence interval of this reservation]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @return int representing the priority of the reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @param priority representing the priority of the reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDefinition} captures the set of resource and time
+ constraints the user cares about regarding a reservation.
+ 
+ @see ResourceRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <class name="ReservationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the long identifier of the {@link ReservationId} which is unique for
+ all Reservations started by a particular instance of the
+ {@code ResourceManager}.
+
+ @return long identifier of the {@link ReservationId}]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the {@code ResourceManager} which is used to
+ generate globally unique {@link ReservationId}.
+
+ @return <em>start time</em> of the {@code ResourceManager}]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the string argument as a {@link ReservationId}
+
+ @param reservationId the string representation of the {@link ReservationId}
+ @return the {@link ReservationId} corresponding to the input string if
+         valid, null if input is null
+ @throws IOException if unable to parse the input string]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="reserveIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="clusterTimestamp" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ {@link ReservationId} represents the <em>globally unique</em> identifier for
+ a reservation.
+ </p>
+
+ <p>
+ The globally unique nature of the identifier is achieved by using the
+ <em>cluster timestamp</em> i.e. start-time of the {@code ResourceManager}
+ along with a monotonically increasing counter for the reservation.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <class name="ReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="concurrency" type="int"/>
+      <param name="duration" type="long"/>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Resource} capability of the request.
+ 
+ @return {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the {@link Resource} capability of the request
+ 
+ @param capability {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ 
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ 
+ @param numContainers number of containers required with the given
+          specifications]]>
+      </doc>
+    </method>
+    <method name="getConcurrency" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @return the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="setConcurrency"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @param numContainers the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="getDuration" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds for which the resource is required. A
+ default value of -1, indicates an unspecified lease duration, and fallback
+ to current behavior.
+ 
+ @return the duration in milliseconds for which the resource is required]]>
+      </doc>
+    </method>
+    <method name="setDuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="duration" type="long"/>
+      <doc>
+      <![CDATA[Set the duration in milliseconds for which the resource is required.
+ 
+ @param duration the duration in milliseconds for which the resource is
+          required]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequest} represents the request made by an application to
+ the {@code ResourceManager} to reserve {@link Resource}s.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required by the
+     application.
+   </li>
+   <li>Concurrency that indicates the gang size of the request.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <class name="ReservationRequest.ReservationRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ReservationRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <class name="ReservationRequestInterpreter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various types of dependencies among multiple
+ {@link ReservationRequests} within one {@link ReservationDefinition} (from
+ least constraining to most constraining).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <class name="ReservationRequests" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationRequests"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+    </method>
+    <method name="getReservationResources" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="setReservationResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @param reservationResources the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="getInterpreter" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @return the list of {@link ReservationRequestInterpreter}]]>
+      </doc>
+    </method>
+    <method name="setInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interpreter" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @param interpreter the {@link ReservationRequestInterpreter} for this
+          reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequests} captures the set of resource and constraints the
+ user cares about regarding a reservation.
+ 
+ @see ReservationRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Resource -->
+  <class name="Resource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+      <param name="others" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Create a new {@link Resource} instance with the given CPU and memory
+ values and additional resource values as set in the {@code others}
+ parameter. Note that the CPU and memory settings in the {@code others}
+ parameter will be ignored.
+
+ @param memory the memory value
+ @param vCores the CPU value
+ @param others a map of other resource values indexed by resource name
+ @return a {@link Resource} instance with the given resource values]]>
+      </doc>
+    </method>
+    <method name="getMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is DEPRECATED:
+ Use {@link Resource#getMemorySize()} instead
+
+ Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="getMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to set
+ memory but the assumption is that the value passed is in units of MB.
+
+ @param memory <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemorySize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource.
+ @param memory <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getVirtualCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @return <em>num of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setVirtualCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vCores" type="int"/>
+      <doc>
+      <![CDATA[Set <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @param vCores <em>number of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get ResourceInformation for a specified resource.
+
+ @param resource name of the resource
+ @return the ResourceInformation object for the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value for a specified resource. No information about the units is
+ returned.
+
+ @param resource name of the resource
+ @return the value for the resource]]>
+      </doc>
+    </method>
+    <method name="setResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="resourceInformation" type="org.apache.hadoop.yarn.api.records.ResourceInformation"/>
+      <doc>
+      <![CDATA[Set the ResourceInformation object for a particular resource.
+
+ @param resource the resource for which the ResourceInformation is provided
+ @param resourceInformation ResourceInformation object]]>
+      </doc>
+    </method>
+    <method name="setResourceValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of a resource in the ResourceInformation object. The unit of
+ the value is assumed to be the one in the ResourceInformation object.
+
+ @param resource the resource for which the value is provided.
+ @param value    the value to set]]>
+      </doc>
+    </method>
+    <method name="throwExceptionWhenArrayOutOfBound"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="index" type="int"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFormattedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is to get memory in terms of KB|MB|GB.
+ @return string containing all resources]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="castToIntSafely" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Convert long to int for a resource value safely. This method assumes
+ resource value is positive.
+
+ @param value long resource value
+ @return int resource value]]>
+      </doc>
+    </method>
+    <method name="newDefaultInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Create ResourceInformation with basic fields.
+ @param name Resource Type Name
+ @param unit Default unit of provided resource type
+ @param value Value associated with giveb resource
+ @return ResourceInformation object]]>
+      </doc>
+    </method>
+    <field name="resources" type="org.apache.hadoop.yarn.api.records.ResourceInformation[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUM_MANDATORY_RESOURCES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VCORES_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>Resource</code> models a set of computer resources in the 
+ cluster.</p>
+ 
+ <p>Currently it models both <em>memory</em> and <em>CPU</em>.</p>
+ 
+ <p>The unit for memory is megabytes. CPU is modeled with virtual cores
+ (vcores), a unit for expressing parallelism. A node's capacity should
+ be configured with virtual cores equal to its number of physical cores. A
+ container should be requested with the number of cores it can saturate, i.e.
+ the average number of threads it expects to have runnable at a time.</p>
+ 
+ <p>Virtual cores take integer values and thus currently CPU-scheduling is
+ very coarse.  A complementary axis for CPU requests that represents
+ processing power will likely be added in the future to enable finer-grained
+ resource configuration.</p>
+
+ <p>Typically, applications request <code>Resource</code> of suitable
+ capability to run their component tasks.</p>
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Resource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <class name="ResourceAllocationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceAllocationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceAllocationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[@param startTime The start time that the capability is reserved for.
+ @param endTime The end time that the capability is reserved for.
+ @param capability {@link Resource} representing the capability of the
+                                   resource allocation.
+ @return {ResourceAllocationRequest} which represents the capability of
+ the resource allocation for a time interval.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time that the resource is allocated.
+
+ @return the start time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time that the resource is allocated.
+
+ @return the end time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated resource.
+
+ @return the allocated resource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceAllocationRequest} represents an allocation
+ made for a reservation for the current state of the plan. This can be
+ changed for reasons such as re-planning, but will always be subject to the
+ constraints of the user contract as described by
+ {@link ReservationDefinition}
+ {@link Resource}
+
+ <p>
+ It includes:
+ <ul>
+   <li>StartTime of the allocation.</li>
+   <li>EndTime of the allocation.</li>
+   <li>{@link Resource} reserved for the allocation.</li>
+ </ul>
+
+ @see Resource]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <class name="ResourceBlacklistRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceBlacklistRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="additions" type="java.util.List"/>
+      <param name="removals" type="java.util.List"/>
+    </method>
+    <method name="getBlacklistAdditions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be added to the 
+ application blacklist.
+ 
+ @return list of resource-names which should be added to the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistAdditions"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be added to the application blacklist.
+ 
+ @param resourceNames list of resource-names which should be added to the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <method name="getBlacklistRemovals" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @return list of resource-names which should be removed from the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistRemovals"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @param resourceNames list of resource-names which should be removed from the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ResourceBlacklistRequest} encapsulates the list of resource-names 
+ which should be added or removed from the <em>blacklist</em> of resources 
+ for the application.
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <class name="ResourceOption" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceOption"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="overCommitTimeout" type="int"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Negative value means no timeout.]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <class name="ResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Clone a ResourceRequest object (shallow copy). Please keep it loaded with
+ all (new) fields
+
+ @param rr the object to copy from
+ @return the copied object]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isAnyLocation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check whether the given <em>host/rack</em> string represents an arbitrary
+ host name.
+
+ @param hostName <em>host/rack</em> on which the allocation is desired
+ @return whether the given <em>host/rack</em> string represents an arbitrary
+ host name]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the request.
+ @return <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the request
+ @param priority <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="getResourceName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource 
+ (host/rack) is acceptable.
+ 
+ @return resource (e.g. <em>host/rack</em>) on which the allocation 
+                  is desired]]>
+      </doc>
+    </method>
+    <method name="setResourceName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the resource name (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource name
+ (e.g. host/rack) is acceptable. 
+ 
+ @param resourceName (e.g. <em>host/rack</em>) on which the 
+                     allocation is desired]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ @param numContainers number of containers required with the given 
+                      specifications]]>
+      </doc>
+    </method>
+    <method name="getRelaxLocality" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+ 
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execSpec" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionTypeRequest</code> of the requested container.
+
+ @param execSpec
+          ExecutionTypeRequest of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionTypeRequest" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setRelaxLocality"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[<p>For a request at a network hierarchy level, set whether locality can be relaxed
+ to that level and beyond.<p>
+ 
+ <p>If the flag is off on a rack-level <code>ResourceRequest</code>,
+ containers at that request's priority will not be assigned to nodes on that
+ request's rack unless requests specifically for those nodes have also been
+ submitted.<p>
+ 
+ <p>If the flag is off on an {@link ResourceRequest#ANY}-level
+ <code>ResourceRequest</code>, containers at that request's priority will
+ only be assigned on racks for which specific requests have also been
+ submitted.<p>
+ 
+ <p>For example, to request a container strictly on a specific node, the
+ corresponding rack-level and any-level requests should have locality
+ relaxation set to false.  Similarly, to request a container strictly on a
+ specific rack, the corresponding any-level request should have locality
+ relaxation set to false.<p>
+ 
+ @param relaxLocality whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this Resource Request. If this is set, all
+ containers allocated to satisfy this resource-request will be only on those
+ nodes that satisfy this node-label-expression.
+  
+ Please note that node label expression now can only take effect when the
+ resource request has resourceName = ANY
+ 
+ @return node-label-expression]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodelabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node label expression of this resource request. Now only support
+ specifying a single node label. In the future we will support more complex
+ node label expression specification like {@code AND(&&), OR(||)}, etc.
+ 
+ Any please note that node label expression now can only take effect when
+ the resource request has resourceName = ANY
+ 
+ @param nodelabelExpression
+          node-label-expression of this ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+
+ @return the <em>ID</em> corresponding to this allocation request.]]>
+      </doc>
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestID" type="long"/>
+      <doc>
+      <![CDATA[Set the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+ <p>
+ If the ID is not set, scheduler will continue to work as previously and all
+ allocated {@code Container}(s) will have the default ID, -1.
+
+ @param allocationRequestID the <em>ID</em> corresponding to this allocation
+                            request.]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <field name="ANY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The constant string representing no locality.
+ It should be used by all references that want to pass an arbitrary host
+ name in.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[{@code ResourceRequest} represents the request made
+ by an application to the {@code ResourceManager}
+ to obtain various {@code Container} allocations.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Priority} of the request.</li>
+   <li>
+     The <em>name</em> of the host or rack on which the allocation is
+     desired. A special value of <em>*</em> signifies that
+     <em>any</em> host/rack is acceptable to the application.
+   </li>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required
+     by the application.
+   </li>
+   <li>
+     A boolean <em>relaxLocality</em> flag, defaulting to {@code true},
+     which tells the {@code ResourceManager} if the application wants
+     locality to be loose (i.e. allows fall-through to rack or <em>any</em>)
+     or strict (i.e. specify hard constraint on resource allocation).
+   </li>
+ </ul>
+ 
+ @see Resource
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <class name="ResourceRequest.ResourceRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+ @see ResourceRequest#setPriority(Priority)
+ @param priority <code>priority</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceName" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>resourceName</code> of the request.
+ @see ResourceRequest#setResourceName(String)
+ @param resourceName <code>resourceName</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="capability" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>capability</code> of the request.
+ @see ResourceRequest#setCapability(Resource)
+ @param capability <code>capability</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="numContainers" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>numContainers</code> of the request.
+ @see ResourceRequest#setNumContainers(int)
+ @param numContainers <code>numContainers</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="relaxLocality" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[Set the <code>relaxLocality</code> of the request.
+ @see ResourceRequest#setRelaxLocality(boolean)
+ @param relaxLocality <code>relaxLocality</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="nodeLabelExpression" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>nodeLabelExpression</code> of the request.
+ @see ResourceRequest#setNodeLabelExpression(String)
+ @param nodeLabelExpression
+          <code>nodeLabelExpression</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionTypeRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionTypeRequest
+          <code>executionTypeRequest</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request with 'ensure
+ execution type' flag set to true.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request.
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+ @see ResourceRequest#setAllocationRequestId(long)
+ @param allocationRequestId
+          <code>allocationRequestId</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link ResourceRequest} object.
+ @return {@link ResourceRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link ResourceRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <class name="ResourceRequest.ResourceRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ResourceRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <class name="ResourceSizing" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceSizing"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getNumAllocations" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNumAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+    </method>
+    <method name="getResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceSizing} contains information for the size of a
+ {@link SchedulingRequest}, such as the number of requested allocations and
+ the resources for each allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <class name="ResourceUtilization" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceUtilization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceUtil" type="org.apache.hadoop.yarn.api.records.ResourceUtilization"/>
+    </method>
+    <method name="getVirtualMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get used <em>virtual memory</em>.
+
+ @return <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setVirtualMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vmem" type="int"/>
+      <doc>
+      <![CDATA[Set used <em>virtual memory</em>.
+
+ @param vmem <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>physical memory</em>.
+
+ @return <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setPhysicalMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <doc>
+      <![CDATA[Set <em>physical memory</em>.
+
+ @param pmem <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getCPU" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>CPU</em> utilization.
+
+ @return <em>CPU utilization</em> normalized to 1 CPU]]>
+      </doc>
+    </method>
+    <method name="setCPU"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Set <em>CPU</em> utilization.
+
+ @param cpu <em>CPU utilization</em> normalized to 1 CPU]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ <code>ResourceUtilization</code> models the utilization of a set of computer
+ resources in the cluster.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <class name="SchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <param name="allocationTags" type="java.util.Set"/>
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="getAllocationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+    </method>
+    <method name="getResourceSizing" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+    </method>
+    <method name="getPlacementConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <doc>
+    <![CDATA[{@code SchedulingRequest} represents a request made by an application to the
+ {@code ResourceManager} to obtain an allocation. It is similar to the
+ {@link ResourceRequest}. However, it is more complete than the latter, as it
+ allows applications to specify allocation tags (e.g., to express that an
+ allocation belongs to {@code Spark} or is an {@code HBase-master}), as well
+ as involved {@link PlacementConstraint}s (e.g., anti-affinity between Spark
+ and HBase allocations).
+
+ The size specification of the allocation is in {@code ResourceSizing}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <class name="SchedulingRequest.SchedulingRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+
+ @see SchedulingRequest#setAllocationRequestId(long)
+ @param allocationRequestId <code>allocationRequestId</code> of the
+          request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+
+ @param priority <code>priority</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ @see SchedulingRequest#setPriority(Priority)]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationTags" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the <code>allocationTags</code> of the request.
+
+ @see SchedulingRequest#setAllocationTags(Set)
+ @param allocationTags <code>allocationsTags</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceSizing" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setResourceSizing(ResourceSizing)
+ @param resourceSizing <code>resourceSizing</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="placementConstraintExpression" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+      <doc>
+      <![CDATA[Set the <code>placementConstraintExpression</code> of the request.
+
+ @see SchedulingRequest#setPlacementConstraint(
+      PlacementConstraint)
+ @param placementConstraintExpression <code>placementConstraints</code> of
+          the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link SchedulingRequest} object.
+
+ @return {@link SchedulingRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link SchedulingRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <class name="SignalContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <class name="StrictPreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrictPreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of {@link PreemptionContainer} specifying containers owned by
+ the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>.
+ @return the set of {@link ContainerId} to be preempted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of particular allocations to be reclaimed. The platform will
+ reclaim exactly these resources, so the <code>ApplicationMaster</code> (AM)
+ may attempt to checkpoint work or adjust its execution plan to accommodate
+ it. In contrast to {@link PreemptionContract}, the AM has no flexibility in
+ selecting which resources to return to the cluster.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIdentifier" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier.
+ @return token identifier]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password
+ @return token password]]>
+      </doc>
+    </method>
+    <method name="getKind" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return token kind]]>
+      </doc>
+    </method>
+    <method name="getService" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service to which the token is allocated.
+ @return service to which the token is allocated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>Token</code> is the security entity used by the framework
+ to verify authenticity of any resource.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Token -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <class name="UpdateContainerError" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerError"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+    </method>
+    <method name="getReason" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get reason why the update request was not satisfiable.
+ @return Reason]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set reason why the update request was not satisfiable.
+ @param reason Reason]]>
+      </doc>
+    </method>
+    <method name="getCurrentContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current container version.
+ @return Current container Version.]]>
+      </doc>
+    </method>
+    <method name="setCurrentContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="currentVersion" type="int"/>
+      <doc>
+      <![CDATA[Set current container version.
+ @param currentVersion Current container version.]]>
+      </doc>
+    </method>
+    <method name="getUpdateContainerRequest" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@code UpdateContainerRequest} that was not satisfiable.
+ @return UpdateContainerRequest]]>
+      </doc>
+    </method>
+    <method name="setUpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Set the {@code UpdateContainerRequest} that was not satisfiable.
+ @param updateContainerRequest Update Container Request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerError} is used by the Scheduler to notify the
+ ApplicationMaster of an UpdateContainerRequest it cannot satisfy due to
+ an error in the request. It includes the update request as well as
+ a reason for why the request was not satisfiable.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <class name="UpdateContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="targetCapability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="targetExecutionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="getContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerVersion" type="int"/>
+      <doc>
+      <![CDATA[Set the current version of the container.
+ @param containerVersion of the container]]>
+      </doc>
+    </method>
+    <method name="getContainerUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code> of the container.
+ @return <code>ContainerUpdateType</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code> of the container.
+ @param updateType of the Container]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container.
+ @param containerId <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the target <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the target <code>ExecutionType</code> of the container.
+ @param executionType <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerRequest} represents the request made by an
+ application to the {@code ResourceManager} to update an attribute of a
+ {@code Container} such as its Resource allocation or (@code ExecutionType}
+ <p>
+ It includes:
+ <ul>
+   <li>version for the container.</li>
+   <li>{@link ContainerId} for the container.</li>
+   <li>
+     {@link Resource} capability of the container after the update request
+     is completed.
+   </li>
+   <li>
+     {@link ExecutionType} of the container after the update request is
+     completed.
+   </li>
+ </ul>
+
+ Update rules:
+ <ul>
+   <li>
+     Currently only ONE aspect of the container can be updated per request
+     (user can either update Capability OR ExecutionType in one request..
+     not both).
+   </li>
+   <li>
+     There must be only 1 update request per container in an allocate call.
+   </li>
+   <li>
+     If a new update request is sent for a container (in a subsequent allocate
+     call) before the first one is satisfied by the Scheduler, it will
+     overwrite the previous request.
+   </li>
+ </ul>
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <class name="UpdatedContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdatedContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdatedContainer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Static Factory method.
+
+ @param updateType ContainerUpdateType
+ @param container Container
+ @return UpdatedContainer]]>
+      </doc>
+    </method>
+    <method name="getUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code>.
+ @return ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="setUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code>.
+ @param updateType ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="getContainer" return="org.apache.hadoop.yarn.api.records.Container"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Container</code>.
+ @return Container]]>
+      </doc>
+    </method>
+    <method name="setContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Set the <code>Container</code>.
+ @param container Container]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object that encapsulates an updated container and the
+ type of Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.URL -->
+  <class name="URL" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="URL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="file" type="java.lang.String"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the scheme of the URL.
+ @return scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="setScheme"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheme of the URL
+ @param scheme scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="getUserInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user info of the URL.
+ @return user info of the URL]]>
+      </doc>
+    </method>
+    <method name="setUserInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user info of the URL.
+ @param userInfo user info of the URL]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the host of the URL.
+ @return host of the URL]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the host of the URL.
+ @param host host of the URL]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the port of the URL.
+ @return port of the URL]]>
+      </doc>
+    </method>
+    <method name="setPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the port of the URL
+ @param port port of the URL]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file of the URL.
+ @return file of the URL]]>
+      </doc>
+    </method>
+    <method name="setFile"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the file of the URL.
+ @param file file of the URL]]>
+      </doc>
+    </method>
+    <method name="toPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+    </method>
+    <method name="fromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="fromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[<p><code>URL</code> represents a serializable {@link java.net.URL}.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.URL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <class name="YarnApplicationAttemptState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of a <code>RMAppAttempt</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <class name="YarnApplicationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of an <code>ApplicationMaster</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+  <class name="YarnClusterMetrics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>NodeManager</code>s in the cluster.
+ @return number of <code>NodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumDecommissionedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>DecommissionedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>DecommissionedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumActiveNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>ActiveNodeManager</code>s in the cluster.
+ 
+ @return number of <code>ActiveNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumLostNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>LostNodeManager</code>s in the cluster.
+ 
+ @return number of <code>LostNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumUnhealthyNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>UnhealthyNodeManager</code>s in the cluster.
+ 
+ @return number of <code>UnhealthyNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumRebootedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>RebootedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>RebootedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>YarnClusterMetrics</code> represents cluster metrics.</p>
+ 
+ <p>Currently only number of <code>NodeManager</code>s is provided.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.impl">
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timeline">
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <class name="TimelineAbout" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineAbout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineAbout" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAbout" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAbout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersionBuiltOn" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersionBuiltOn" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <class name="TimelineDelegationTokenResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getContent" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="content" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[The response of delegation token related request]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID
+ 
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description
+ 
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description
+ 
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner
+ 
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ 
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string
+ 
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string
+ 
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string
+ 
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string
+ 
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain
+ 
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain
+ 
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain
+ 
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain
+ 
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline domain, which is used
+ to a user to host a number of timeline entities, isolating them from others'.
+ The user can also define the reader and writer users/groups for the the
+ domain, which is used to control the access to its entities.
+ </p>
+ 
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <class name="TimelineDomains" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomains"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDomains" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of domains
+ 
+ @return a list of domains]]>
+      </doc>
+    </method>
+    <method name="addDomain"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <doc>
+      <![CDATA[Add a single domain into the existing domain list
+ 
+ @param domain
+          a single domain]]>
+      </doc>
+    </method>
+    <method name="addDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of domains into the existing domain list
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <method name="setDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the domain list to the given list of domains
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline domains.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of entities
+ 
+ @return a list of entities]]>
+      </doc>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+      <doc>
+      <![CDATA[Add a single entity into the existing entity list
+ 
+ @param entity
+          a single entity]]>
+      </doc>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of entities into the existing entity list
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the entity list to the given list of entities
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time of the entity
+ 
+ @return the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the start time of the entity
+ 
+ @param startTime
+          the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events related to the entity
+ 
+ @return a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event related to the entity to the existing event list
+ 
+ @param event
+          a single event related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of events related to the entity to the existing event list
+ 
+ @param events
+          a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events related to the entity
+ 
+ @param events
+          events a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="getRelatedEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the related entities
+ 
+ @return the related entities]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add an entity to the existing related entity map
+ 
+ @param entityType
+          the entity type
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of related entities to the existing related entity map
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="setRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the related entity map to the given map of related entities
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="getPrimaryFilters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the primary filters
+ 
+ @return the primary filters]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add a single piece of primary filter to the existing primary filter map
+ 
+ @param key
+          the primary filter key
+ @param value
+          the primary filter value]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of primary filters to the existing primary filter map
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="setPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the primary filter map to the given map of primary filters
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="getOtherInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the other information of the entity
+ 
+ @return the other information of the entity]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of other information of the entity to the existing other info
+ map
+ 
+ @param key
+          the other information key
+ @param value
+          the other information value]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of other information of the entity to the existing other info map
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="setOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the other info map to the given map of other information
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="getDomainId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ID of the domain that the entity is to be put
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setDomainId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domainId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ID of the domain that the entity is to be put
+ 
+ @param domainId
+          the name space ID]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The class that contains the the meta information of some conceptual entity
+ and its related events. The entity can be an application, an application
+ attempt, a container or whatever the user-defined object.
+ </p>
+ 
+ <p>
+ Primary filters will be used to index the entities in
+ <code>TimelineStore</code>, such that users should carefully choose the
+ information they want to store as the primary filters. The remaining can be
+ stored as other information.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <class name="TimelineEntityGroupId" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntityGroupId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the
+ <code>TimelineEntityGroupId</code>.
+
+ @return <code>ApplicationId</code> of the
+         <code>TimelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getTimelineEntityGroupId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>timelineEntityGroupId</code>.
+
+ @return <code>timelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntityGroupIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TIMELINE_ENTITY_GROUPID_STR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>TimelineEntityGroupId</code> is an abstract way for
+ timeline service users to represent #a group of related timeline data.
+ For example, all entities that represents one data flow DAG execution
+ can be grouped into one timeline entity group. </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the event
+ 
+ @return the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the timestamp of the event
+ 
+ @param timestamp
+          the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="getEventType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the event type
+ 
+ @return the event type]]>
+      </doc>
+    </method>
+    <method name="setEventType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the event type
+ 
+ @param eventType
+          the event type]]>
+      </doc>
+    </method>
+    <method name="getEventInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the information of the event
+ 
+ @return the information of the event]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of the information of the event to the existing information
+ map
+ 
+ @param key
+          the information key
+ @param value
+          the information value]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of the information of the event to the existing information map
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="setEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the information map to the given map of the information of the event
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The class that contains the information of an event that is related to some
+ conceptual entity of an application. Users are free to define what the event
+ means, such as starting an application, getting allocated a container and
+ etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <class name="TimelineEvents" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEvents"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAllEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link EventsOfOneEntity} instances
+ 
+ @return a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventsOfOneEntity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity"/>
+      <doc>
+      <![CDATA[Add a single {@link EventsOfOneEntity} instance into the existing list
+ 
+ @param eventsOfOneEntity
+          a single {@link EventsOfOneEntity} instance]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link EventsOfOneEntity} instances into the existing list
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link EventsOfOneEntity} instances
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events, which are categorized according to
+ their related entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <class name="TimelineEvents.EventsOfOneEntity" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventsOfOneEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events
+ 
+ @return a list of events]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event to the existing event list
+ 
+ @param event
+          a single event]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of event to the existing event list
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events that are only related to one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <class name="TimelineHealth" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineHealth" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineHealth"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHealthStatus" return="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="healthStatus" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"/>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[This class holds health information for ATS.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <class name="TimelinePutResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelinePutError} instances
+ 
+ @return a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelinePutError} instance into the existing list
+ 
+ @param error
+          a single {@link TimelinePutError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelinePutError} instances into the existing list
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelinePutError} instances
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are errors
+ in storing individual entity objects, they will be indicated in the list of
+ errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+  <class name="TimelinePutResponse.TimelinePutError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code
+ 
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCode" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code
+ 
+ @param errorCode
+          an error code]]>
+      </doc>
+    </method>
+    <field name="NO_START_TIME" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned when no start time can be found when putting an
+ entity. This occurs when the entity does not already exist in the store
+ and it is put with no start time or events specified.]]>
+      </doc>
+    </field>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when putting an
+ entity.]]>
+      </doc>
+    </field>
+    <field name="SYSTEM_FILTER_CONFLICT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user specifies the timeline system reserved
+ filter key]]>
+      </doc>
+    </field>
+    <field name="ACCESS_DENIED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to access the timeline data]]>
+      </doc>
+    </field>
+    <field name="NO_DOMAIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity doesn't have an valid domain ID]]>
+      </doc>
+    </field>
+    <field name="FORBIDDEN_RELATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to relate the entity to another
+ one in different domain]]>
+      </doc>
+    </field>
+    <field name="EXPIRED_ENTITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity start time is before the eviction
+ period of old data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timelineservice">
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <class name="ApplicationAttemptEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <class name="ApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="isApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an ApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an ApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getApplicationEvent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <param name="eventId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param te TimelineEntity object.
+ @param eventId event with this id needs to be fetched
+ @return TimelineEvent if TimelineEntity contains the desired event.]]>
+      </doc>
+    </method>
+    <field name="QUEUE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents an application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <class name="ClusterEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClusterEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a YARN cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <class name="ContainerEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a container belonging to an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <class name="FlowActivityEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowActivityEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="java.lang.String, long, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reuse the base class equals method.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reuse the base class hashCode method.]]>
+      </doc>
+    </method>
+    <method name="getCluster" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cluster" type="java.lang.String"/>
+    </method>
+    <method name="getDate" return="java.util.Date"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFlowName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+    </method>
+    <method name="addFlowRun"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="run" type="org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity"/>
+    </method>
+    <method name="addFlowRuns"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runs" type="java.util.Collection"/>
+    </method>
+    <method name="getFlowRuns" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNumberOfRuns" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLUSTER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Entity that represents a record for flow activity. It's essentially a
+ container entity for flow runs with limited information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <class name="FlowRunEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowRunEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowRunEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="java.lang.String"/>
+    </method>
+    <method name="getRunId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRunId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runId" type="long"/>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+    </method>
+    <method name="getMaxEndTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxEndTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+    </method>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_END_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a flow run.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <class name="HierarchicalTimelineEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getParent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getChildren" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="child" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <field name="PARENT_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CHILDREN_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends timeline entity and defines parent-child relationships
+ with other entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <class name="QueueEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QueueEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a queue.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <class name="SubApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSubApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an SubApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an SubApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="java.lang.String"/>
+    </method>
+    <field name="YARN_APPLICATION_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a user defined entities to be stored under sub
+ application table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID.
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID.
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description.
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description.
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner.
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string.
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string.
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string.
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string.
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain.
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain.
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain.
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain.
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline service domain, which is
+ used to a user to host a number of timeline entities, isolating them from
+ others'. The user can also define the reader and writer users/groups for
+ the domain, which is used to control the access to its entities.
+ </p>
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[This class hosts a set of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ The constuctor is used to construct a proxy {@link TimelineEntity} or its
+ subclass object from the real entity object that carries information.
+ </p>
+
+ <p>
+ It is usually used in the case where we want to recover class polymorphism
+ after deserializing the entity from its JSON form.
+ </p>
+ @param entity the real entity that carries information]]>
+      </doc>
+    </constructor>
+    <constructor name="TimelineEntity" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getIdentifier" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdentifier" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getConfigs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="getMetrics" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+    </method>
+    <method name="getEvents" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.NavigableSet"/>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.Set"/>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <method name="getIsRelatedToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getRelatesToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="addRelatesToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="setRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTs" type="java.lang.Long"/>
+    </method>
+    <method name="setUID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uidKey" type="java.lang.String"/>
+      <param name="uId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set UID in info which will be then used for query by UI.
+ @param uidKey key for UID in info.
+ @param uId UID to be set for the key.]]>
+      </doc>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="getReal" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdPrefix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdPrefix"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdPrefix" type="long"/>
+      <doc>
+      <![CDATA[Sets idPrefix for an entity.
+ <p>
+ <b>Note</b>: Entities will be stored in the order of idPrefix specified.
+ If users decide to set idPrefix for an entity, they <b>MUST</b> provide
+ the same prefix for every update of this entity.
+ </p>
+ Example: <blockquote><pre>
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+ </pre></blockquote>
+ Users can use {@link TimelineServiceHelper#invertLong(long)} to invert
+ the prefix if necessary.
+
+ @param entityIdPrefix prefix for an entity.]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_INFO_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENTITY_PREFIX" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The basic timeline entity data structure for timeline service v2. Timeline
+ entity objects are not thread safe and should not be accessed concurrently.
+ All collection members will be initialized into empty collections. Two
+ timeline entities are equal iff. their type and id are identical.
+
+ All non-primitive type, non-collection members will be initialized into null.
+ User should set the type and id of a timeline entity to make it valid (can be
+ checked by using the {@link #isValid()} method). Callers to the getters
+ should perform null checks for non-primitive type, non-collection members.
+
+ Callers are recommended not to alter the returned collection objects from the
+ getters.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <class name="TimelineEntityType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isParent" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a parent of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is parent of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="isChild" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a child of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is child of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="matches" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="typeString" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether the type of this entity matches the type indicated by the input
+ argument.
+
+ @param typeString entity type represented as a string.
+ @return true, if string representation of this entity type matches the
+     entity type passed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Defines type of entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventId" type="java.lang.String"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ts" type="long"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <field name="INVALID_TIMESTAMP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains the information of an event that belongs to an entity.
+ Users are free to define what the event means, such as starting an
+ application, container being allocated, etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <class name="TimelineMetric" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricType" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricId" type="java.lang.String"/>
+    </method>
+    <method name="getRealtimeAggregationOp" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the real time aggregation operation of this metric.
+
+ @return Real time aggregation operation]]>
+      </doc>
+    </method>
+    <method name="setRealtimeAggregationOp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"/>
+      <doc>
+      <![CDATA[Set the real time aggregation operation of this metric.
+
+ @param op A timeline metric operation that the metric should perform on
+           real time aggregations]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <param name="value" type="java.lang.Number"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLatestSingleValueMetric" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Get the latest timeline metric as single value type.
+
+ @param metric Incoming timeline metric
+ @return The latest metric in the incoming metric]]>
+      </doc>
+    </method>
+    <method name="getSingleDataTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data timestamp of the metric.
+
+ @return the single data timestamp]]>
+      </doc>
+    </method>
+    <method name="getSingleDataValue" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data value of the metric.
+
+ @return the single data value]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state in a stateless fashion. The assumption here is
+ baseAggregatedMetric and latestMetric should be single value data if not
+ null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="state" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state. The assumption here is baseAggregatedMetric and
+ latestMetric should be single value data if not null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @param state Operation state
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains the information of a metric that is related to some
+ entity. Metric can either be a time series or single value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <class name="TimelineWriteResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelineWriteError} instances.
+
+ @return a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelineWriteError} instance into the existing list.
+
+ @param error
+          a single {@link TimelineWriteError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelineWriteError} instances into the existing list.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelineWriteError} instances.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are
+ errors in storing individual entity objects, they will be indicated in the
+ list of errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <class name="TimelineWriteResponse.TimelineWriteError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id.
+
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id.
+
+ @param id the entity Id.]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type.
+
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type.
+
+ @param type the entity type.]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code.
+
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code.
+
+ @param code an error code.]]>
+      </doc>
+    </method>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when storing an
+ entity.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+  <class name="UserEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UserEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="UserEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+</package>
+<package name="org.apache.hadoop.yarn.api.resource">
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <class name="PlacementConstraint" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PlacementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConstraintExpr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the constraint expression of the placement constraint.
+
+ @return the constraint expression]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code PlacementConstraint} represents a placement constraint for a resource
+ allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+  <class name="PlacementConstraints" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="targetIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ satisfy all target expressions within the given scope (e.g., node or rack).
+
+ For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows
+ allocations on nodes that belong to a rack that has at least one tag with
+ value "hbase-m".
+
+ @param scope the scope within which the target expressions should be
+          satisfied
+ @param targetExpressions the expressions that need to be satisfied within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNotIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that does not satisfy any of the
+ target expressions.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNodeAttribute" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="opCode" type="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that satisfy any of the
+ target expressions based on node attribute op code.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param opCode Node Attribute code which could be equals, not equals.
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that restricts the number of allocations within a
+ given scope (e.g., node or rack).
+
+ For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes
+ where there are no less than 3 allocations with tag "zk" and no more than
+ 10.
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but let you
+ attach a namespace to the given allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of the allocation tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param maxCardinality determines the maximum number of allocations within
+                       the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the minimum cardinality (the maximum cardinality is
+ unbound).
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #minCardinality(String, int, String...)}, but let you
+ attach a namespace to the allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of these tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the maximum cardinality (the minimum cardinality is 0).
+
+ @param scope the scope of the constraint
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="tagNamespace" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #maxCardinality(String, int, String...)}, but let you
+ specify a namespace for the tags, see supported namespaces in
+ {@link AllocationTagNamespaceType}.
+
+ @param scope the scope of the constraint
+ @param tagNamespace the namespace of these tags
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[This constraint generalizes the cardinality and target constraints.
+
+ Consider a set of nodes N that belongs to the scope specified in the
+ constraint. If the target expressions are satisfied at least minCardinality
+ times and at most maxCardinality times in the node set N, then the
+ constraint is satisfied.
+
+ For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
+ requires an allocation to be placed within a rack that has at least 2 and
+ at most 10 other allocations with tag "zk".
+
+ @param scope the scope of the constraint
+ @param minCardinality the minimum number of times the target expressions
+          have to be satisfied with the given scope
+ @param maxCardinality the maximum number of times the target expressions
+          have to be satisfied with the given scope
+ @param targetExpressions the target expressions
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="and" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.And"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A conjunction of constraints.
+
+ @param children the children constraints that should all be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="or" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A disjunction of constraints.
+
+ @param children the children constraints, one of which should be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="delayedOr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint[]"/>
+      <doc>
+      <![CDATA[Creates a composite constraint that includes a list of timed placement
+ constraints. The scheduler should try to satisfy first the first timed
+ child constraint within the specified time window. If this is not possible,
+ it should attempt to satisfy the second, and so on.
+
+ @param children the timed children constraints
+ @return the resulting composite constraint]]>
+      </doc>
+    </method>
+    <method name="timedClockConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <param name="timeUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a time
+ window.
+
+ @param constraint the placement constraint
+ @param delay the length of the time window within which the constraint has
+          to be satisfied
+ @param timeUnit the unit of time of the time window
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="timedOpportunitiesConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a number of
+ placement opportunities (invocations of the scheduler).
+
+ @param constraint the placement constraint
+ @param delay the number of scheduling opportunities within which the
+          constraint has to be satisfied
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintExpr" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <doc>
+      <![CDATA[Creates a {@link PlacementConstraint} given a constraint expression.
+
+ @param constraintExpr the constraint expression
+ @return the placement constraint]]>
+      </doc>
+    </method>
+    <field name="NODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains various static methods for the applications to create
+ placement constraints (see also {@link PlacementConstraint}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+</package>
+<package name="org.apache.hadoop.yarn.conf">
+  <!-- start class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+  <class name="YarnConfiguration" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnConfiguration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isAclEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>. On an HA cluster,
+ this fetches the address corresponding to the RM identified by
+ {@link #RM_HA_ID}.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+    </method>
+    <method name="useHttps" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="shouldRMFailFast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isDistSchedulingEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOpportunisticContainerAllocationEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV2Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.2 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.2 is enabled. V.2 refers to a
+ version greater than equal to 2 but smaller than 3.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1 is enabled. V.1 refers to a
+ version greater than equal to 1 but smaller than 2.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV15Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1,5 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="systemMetricsPublisherEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the system publisher is enabled.
+
+ @param conf the configuration
+ @return whether the system publisher is enabled.]]>
+      </doc>
+    </method>
+    <method name="numaAwarenessEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the NUMA awareness is enabled.
+
+ @param conf the configuration
+ @return whether the NUMA awareness is enabled.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="DR_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CS_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_POLICY_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CORE_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_RESOURCES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEBUG_NM_DELETE_DELAY_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay before deleting resource to ease debugging of NM issues]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_CONTAINER_DEBUG_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create client IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create server IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create serializable records.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RPC class implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HOSTNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_EPOCH" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH_RANGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The epoch range before wrap around. 0 disables wrap around]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_EPOCH_RANGE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the applications manager interface in the RM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of application tags.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAG_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of each application tag.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MASTER_SERVICE_PROCESSORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AUTO_UPDATE_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AUTO_UPDATE_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the RM.]]>
+      </doc>
+    </field>
+    <field name="RM_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle applications manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AMLAUNCHER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to launch/cleanup AM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AMLAUNCHER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_CONNECT_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retry times to connect with NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_CONNECT_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the port should be included or not in the node name. The node name
+ is used by the scheduler for resource requests allocation location 
+ matching. Typically this is just the hostname, using the port is needed
+ when using minicluster and specific NM are required.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUEUE_PLACEMENT_RULES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configured scheduler queue placement rules.]]>
+      </doc>
+    </field>
+    <field name="USER_GROUP_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[UserGroupMappingPlacementRule configuration string.]]>
+      </doc>
+    </field>
+    <field name="APP_NAME_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_UI_ACTIONS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Resource Manager webapp ui actions]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the RM should enable Reservation System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The PlanFollower for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The step size of the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum periodicity for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_ENABLE_MONITORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable periodic monitor threads.
+ @see #RM_SCHEDULER_MONITOR_POLICIES]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ENABLE_MONITORS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MONITOR_POLICIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of SchedulingEditPolicy classes affecting the scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SSL_SERVER_RESOURCE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable YARN WebApp V2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_WEBAPP_UI2_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_WARFILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_API_SERVICES_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_API_SYSTEM_SERVICES_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for application master reporting.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a node manager is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_RESERVATION_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are reservation acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_RESERVATION_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL of who can be admin of YARN cluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_APP_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL used in case none is found. Allows nothing.]]>
+      </doc>
+    </field>
+    <field name="OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether opportunistic container allocation
+  is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of nodes to be used by the Opportunistic Container allocator for
+ dispatching containers during container allocation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency for computing least loaded NMs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comparator for determining node load for scheduling of opportunistic
+ containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LIMIT_STDEV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value of standard deviation used for calculation of queue limit
+ thresholds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the lower-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the upper-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min queue wait time for a container at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue wait time for a container queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_INTERMEDIATE_DATA_ENCRYPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable intermediate-data encryption at YARN level. For now, this
+ only is used by the FileSystemRMStateStore to setup right file-system
+ security attributes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts for
+ an application, if unset by user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GLOBAL_RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts.
+ It's a global setting for all application masters.]]>
+      </doc>
+    </field>
+    <field name="RM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The keytab for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing). Only applicable
+ when the http authentication type is kerberos.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cross origin (CORS) support.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a container is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to include.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable submission pre-processor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with hosts for the submission processor to handle.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submission processor refresh interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to exclude.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle resource tracker calls.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the resource scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specify which handler will be used to process PlacementConstraints.
+ For details on PlacementConstraints, please refer to
+ {@link org.apache.hadoop.yarn.api.resource.PlacementConstraint}]]>
+      </doc>
+    </field>
+    <field name="DISABLED_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This handler rejects all allocate calls made by an application, if they
+ contain a {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}.]]>
+      </doc>
+    </field>
+    <field name="PROCESSOR_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, the placement of containers with constraints is
+ determined as a pre-processing step before the capacity or the fair
+ scheduler is called. Once the placement is decided, the capacity/fair
+ scheduler is invoked to perform the actual allocation. The advantage of
+ this approach is that it supports all constraint types (affinity,
+ anti-affinity, cardinality). Moreover, it considers multiple containers at
+ a time, which allows to satisfy more constraints than a container-at-a-time
+ approach can achieve. As it sits outside the main scheduler, it can be used
+ by both the capacity and fair schedulers. Note that at the moment it does
+ not account for task priorities within an application, given that such
+ priorities might be conflicting with the placement constraints.]]>
+      </doc>
+    </field>
+    <field name="SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, containers with constraints will be placed by the main
+ scheduler. If the configured RM scheduler
+ <pre>yarn.resourcemanager.scheduler.class</pre>
+ cannot handle placement constraints, the corresponding SchedulingRequests
+ will be rejected. As of now, only the capacity scheduler supports
+ SchedulingRequests. In particular, it currently supports anti-affinity
+ constraints (no affinity or cardinality) and places one container at a
+ time. The advantage of this handler compared to the placement-processor is
+ that it follows the same ordering rules for queues (sorted by utilization,
+ priority) and apps (sorted by FIFO/fairness/priority) as the ones followed
+ by the main scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Placement Algorithm.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Used for BasicPlacementAlgorithm - default SERIAL.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM set next Heartbeat interval for NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Heartbeat Interval Scaling based on cpu utilization.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of worker threads that write the history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM. This configuration setting is for ATS V1.
+  This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM and NM. This configuration setting is for
+  ATS v2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by RM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_CONF_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper interaction configs]]>
+      </doc>
+    </field>
+    <field name="RM_ZK_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_RM_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper znode limit]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_RETRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_TIMEOUT_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_APPID_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_APPID_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Index at which the RM Delegation Token ids will be split so that the
+ delegation token znodes stored in the zookeeper RM state store will be
+ stored as two different znodes (parent-child).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_AUTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_ROOT_NODE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Root node ACLs for fencing]]>
+      </doc>
+    </field>
+    <field name="RM_HA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[HA related configs]]>
+      </doc>
+    </field>
+    <field name="RM_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Store the related configuration files in File System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILE_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LEVELDB_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_MAX_LOGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKConfigurationStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_AUTHORIZATION_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_EMBEDDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This property should never be set to {@code false}.">
+      <doc>
+      <![CDATA[This property controls whether {@link ActiveStandbyElector} leader
+ election should be used when {@link #CURATOR_LEADER_ELECTOR} is
+ {@code false}.
+
+ @deprecated This property should never be set to {@code false}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_EMBEDDED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.">
+      <doc>
+      <![CDATA[The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
+
+ @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.]]>
+      </doc>
+    </field>
+    <field name="AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_BASE_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_FC_ELECTOR_ZK_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of zookeeper operation retry times in ActiveStandbyElector]]>
+      </doc>
+    </field>
+    <field name="CURATOR_LEADER_ELECTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.">
+      <doc>
+      <![CDATA[Whether to use the Curator-based elector for leader election.
+
+ @deprecated Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the persistent store.]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemRMStateStore]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the leveldb database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM keeps. By default equals
+ to {@link #DEFAULT_RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM state store keeps. By
+ default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.">
+      <doc>
+      <![CDATA[The default value for
+ {@code yarn.resourcemanager.state-store.max-completed-applications}.
+ @deprecated This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application name]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_TYPE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type length]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default queue name]]>
+      </doc>
+    </field>
+    <field name="RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Buckets (in minutes) for the number of apps running in each queue.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default sizes of the runtime metric buckets in minutes.]]>
+      </doc>
+    </field>
+    <field name="RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout(msec) for an untracked node to remain in shutdown or decommissioned
+ state.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM proxy users' prefix]]>
+      </doc>
+    </field>
+    <field name="RM_RESOURCE_PROFILES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File containing resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable loading resource-types.xml at client side.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications
+ to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Period in seconds of the poll timer task inside DecommissioningNodesWatcher
+ to identify and take care of DECOMMISSIONING nodes missing regular heart beat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for all node manager configs.]]>
+      </doc>
+    </field>
+    <field name="NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIST_SCHEDULING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether distributed scheduling is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DIST_SCHEDULING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that will be sent to containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[PATH components that will be prepended to the user's path.
+ If this is defined and the user does not define PATH, NM will also
+ append ":$PATH" to prevent this from eclipsing the PATH defined in
+ the container. This feature is only available for Linux.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that containers may override rather than use NodeManager's default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[address of node manager IPC.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the NM.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[who will execute(launch) the containers.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_STATE_TRANSITION_LISTENERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of container state transition listeners.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adjustment to make to the container os scheduling priority.
+ The valid values for this could vary depending on the platform.
+ On Linux, higher values mean run the containers at a less 
+ favorable priority than the NM. 
+ The value specified is an int.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MGR_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DELETE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in cleanup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of directories to store localized files in.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of files in each localized directories
+ Avoid tuning this too low.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the localizer IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the collector service IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by NM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval in between cache cleanups.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target size of localizer cache in MB, per nodemanager. It is a target
+ retention size that only includes resources with PUBLIC and PRIVATE
+ visibility and excludes resources with APPLICATION visibility]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle localization requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_FETCH_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to use for localization fetching.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to store container logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle log aggregation in node manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permissions for container logs.]]>
+      </doc>
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk Validator.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum size of contain's diagnostics to keep for relaunching container
+ case.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum container restart interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which the delayed token removal thread runs]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegation Token renewer thread count]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PRIVILEGES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for node IP caching. -1 disables the caching]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How many diagnostics/failure messages can be saved in RM for
+ log aggregation. It also defines the number of diagnostics/failure
+ messages can be shown in log aggregation web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to enable log aggregation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_CONTROLLER_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait before deleting aggregated logs, -1 disables.
+ Be careful set this too small and you will spam the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_DEBUG_FILESIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait between aggregated log retention checks. If set to
+ a value {@literal <=} 0 then the value is computed as one-tenth of the
+ log retention setting. Be careful set this too small and you will spam
+ the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation status
+ is reported from NodeManager exceeds the configured value, RM will report
+ log aggregation status for this NodeManager as TIME_OUT.
+
+ This configuration will be used in NodeManager as well to decide
+ whether and when to delete the cached log aggregation status.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seconds to retain logs on the NodeManager. Only applicable if Log
+ aggregation is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how often NMs wake up and upload log files]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed hard minimum limit for {@link
+ YarnConfiguration#NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS}.]]>
+      </doc>
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how many aggregated log files per application per NM we can have
+ in remote file system.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DELETION_THREADS_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in log cleanup. Only applicable if Log aggregation
+ is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to aggregate logs to.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The remote log dir will be created at
+ NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_WEBSERVICE_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_TRACKING_URL_GENERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SYSTEM_RESERVED_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that has been reserved for non-yarn use.]]>
+      </doc>
+    </field>
+    <field name="NM_PMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_PMEM_RATIO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Conversion ratio for physical memory to virtual memory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_PMEM_RATIO" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether to do memory check on overall usage.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the OOM handler code.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the OOM listener.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum time in seconds to resolve an OOM situation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.Integer"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of Virtual CPU Cores which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count logical processors(like hyperthreads) as cores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PCORES_VCORES_MULTIPLIER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Multiplier to convert physical cores to vcores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PCORES_VCORES_MULTIPLIER" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Percentage of overall CPU which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable or disable node hardware capability detection.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENFORCED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENFORCED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cpu isolation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CPU_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for disk configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for disk operations is enabled.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for network bandwidth is
+ enabled. Work in progress: This configuration parameter may be
+ changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Network as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the interface to be used for applying network throttling rules.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total available outbound bandwidth on the node. Work in
+ progress: This configuration parameter may be changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total outbound bandwidth available to YARN containers.
+ defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for computation resources, example of computation resources like
+ GPU / FPGA / TPU, etc.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether the initialization of the Node Manager should continue
+ if a certain device (GPU, FPGA, etc) was not found in the system. If set
+ to "true", then an exception will be thrown if a device is missing or
+ an error occurred during discovery.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PLUGINS_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_GPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for gpu configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTOMATICALLY_DISCOVER_GPU_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_GPU_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls where to how to invoke GPU binaries]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings to control which implementation of docker plugin for GPU will be
+ used.
+
+ By default uses NVIDIA docker v1.]]>
+      </doc>
+    </field>
+    <field name="NVIDIA_DOCKER_V1" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_V2" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls end point of nvidia-docker-v1 plugin]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for FPGA configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_FPGA_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_HANDLER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_TAG_HANDLER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp https address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable CORS filter.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor resource in a node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates containers current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_PROCESS_TREE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates process tree resource utilization.]]>
+      </doc>
+    </field>
+    <field name="PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable switch for container log monitoring.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor logs generated by containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for a single container log directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for all of a container's logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable container metrics.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_PERIOD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container metrics flush period. -1 for flush on completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The delay time ms to unregister container metrics after completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable disks' health checker. Default is true. An expert level
+ configuration property.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running disks' health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, disks' health is checked every 2 minutes.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_HEALTHY_DISKS_FRACTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum fraction of number of disks to be healthy for the nodemanager
+ to launch new containers. This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, at least 25% of disks are to be healthy to say that the node is
+ healthy in terms of disks.]]>
+      </doc>
+    </field>
+    <field name="NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum percentage of disk space that can be used after which a disk is
+ marked as offline. Values can range from 0.0 to 100.0. If the value is
+ greater than or equal to 100, NM will check for full disk. This applies to
+ nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-utilization-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, 90% of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the disk utilisation percentage
+ threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The low threshold percentage of disk space used when an offline disk is
+ marked as online. Values can range from 0.0 to 100.0. The value shouldn't
+ be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is
+ more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be
+ set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on a local dir for it to be used.
+ This applies to nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-free-space-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the minimum disk free
+ space threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_HIGH_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on an offline
+ disk for it to be marked as online.  The value should not be less
+ than NM_MIN_PER_DISK_FREE_SPACE_MB.  If its value is less than
+ NM_MIN_PER_DISK_FREE_SPACE_MB or is not set, it will be set to the
+ same value as NM_MIN_PER_DISK_FREE_SPACE_MB.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, all of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check script time out period.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not to run the node health script before the NM
+  starts up.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health check script to run.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The arguments to pass to the health check script.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JVM options used on forking ContainerLocalizer process
+      by container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log level of container localizer process.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for runtime configuration constants.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The standard values are:
+ <ul>
+   <li>default</li>
+   <li>docker</li>
+   <li>javasandbox</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_CLASS_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of allowed runtimes when using LinuxContainerExecutor.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runtime to be used.]]>
+      </doc>
+    </field>
+    <field name="DOCKER_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default docker image to be used.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Capabilities allowed (and added by default) for docker containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[These are the default capabilities added by docker. We'll use the same
+ set here. While these may not be case-sensitive from a docker
+ perspective, it is best to keep these uppercase.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ENABLE_USER_REMAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[enable user remapping.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set enable user remapping as false by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable uids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower uid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable gids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower gid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request. This
+  network must be one of the (configurable) set of allowed container
+  networks.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request and
+ no default network is configured.
+ .]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow host pid namespace for containers. Use with care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Host pid namespace for containers is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not users are allowed to request that Docker containers honor
+ the debug deletion delay. This is useful for troubleshooting Docker
+ container related launch failures.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value on whether or not a user can request that Docker
+ containers honor the debug deletion delay.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_STOP_GRACE_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}">
+      <doc>
+      <![CDATA[A configurable value to pass to the Docker Stop command. This value
+ defines the number of seconds between the docker stop command sending
+ a SIGTERM and a SIGKILL.
+
+ @deprecated use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for the grace period between the SIGTERM and the
+ SIGKILL in the Docker Stop command.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RO_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-only mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RW_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-write mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_TMPFS_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of tmpfs mounts to be mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The mode in which the Java Container Sandbox should run detailed by
+  the JavaSandboxLinuxContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for application local directories.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Location for non-default java policy file.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for group to policy file mapping.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_WHITELIST_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The group which will run by default without the java security manager.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_EXECUTOR_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the Linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_NONSECURE_MODE_LIMIT_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if linux-container-executor should limit itself to one user
+ when running in non-secure mode.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_LOCAL_USER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The UNIX user that containers will run as when Linux-container-executor
+ is used in nonsecure mode (a use case for this is using cgroups).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LOCAL_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_USER_PATTERN_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed pattern for UNIX user names enforced by 
+ Linux-container-executor when used in nonsecure mode (use case for this 
+ is using cgroups). The default value is taken from /usr/sbin/adduser]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_USER_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_RESOURCES_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The type of resource enforcement to use with the
+  linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_HIERARCHY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path the linux container executor should use for cgroups]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the apps should run in strict resource usage mode(not allowed to
+ use spare CPU)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_TIMEOUT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFALUT_RM_PROXY_TIMEOUT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_CONNECTION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_CONNECTION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container. This is required due to what 
+ it seems a race condition because the SIGTERM/SIGKILL is asynch.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay between attempts to remove linux cgroup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if memory and CPU limits will be set for the Windows Job
+ Object for the containers launched by the default container executor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_SECURE_CONTAINER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[/* The Windows group that the windows-secure-container-executor should run as.]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress aggregated logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the node manager.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_REMOTE_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the NM state database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_SUPERVISED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_SUPERVISED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS_PARAMETERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for Proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[YARN Service Level Authorization]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[No. of milliseconds to wait between sending a SIGTERM and SIGKILL
+ to a running container]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROCESS_KILL_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for a process to come up when trying to cleanup
+ container resources]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PROCESS_KILL_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The threshold used to trigger the logging of event types and counts
+  in RM's main event dispatcher. Default value is 5000,
+  which means RM will print events info when the queue size cumulatively
+  reaches 5000 every time. Such info can be used to reveal what
+  kind of events that RM is stuck at processing mostly,
+  it can help to narrow down certain performance issues.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resource manager dispatcher thread monitor sampling rate.
+ Units are samples per minute.  This controls how often to sample
+ the cpu utilization of the resource manager dispatcher thread.
+ The cpu utilization is displayed on the RM UI as scheduler busy %.
+ Set to zero to disable the dispatcher thread monitor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APPLICATION_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
+ entries]]>
+      </doc>
+    </field>
+    <field name="AMRM_PROXY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether AMRMProxy is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default platform-agnostic CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries. The parameter expansion marker
+ will be replaced with real parameter expansion marker ('%' for Windows and
+ '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+ will be replaced as $VAR on Linux, and %VAR% on Windows.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Default platform-specific CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries constructed based on the client
+ OS environment expansion syntax.
+ </p>
+ <p>
+ Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.
+ </p>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CONTAINER_TEMP_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container temp directory]]>
+      </doc>
+    </field>
+    <field name="IS_MINI_YARN_CLUSTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_FIXED_PORTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to use fixed ports with the minicluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_FIXED_PORTS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default is false to be able to run tests concurrently without port
+ conflicts.]]>
+      </doc>
+    </field>
+    <field name="YARN_MINICLUSTER_USE_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the NM should use RPC to connect to the RM. Default is false.
+ Can be set to true only when using fixed ports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_USE_RPC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether users are explicitly trying to control resource monitoring
+ configuration for the MiniYARNCluster. Disabled by default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow changing the memory for the NodeManager in the MiniYARNCluster]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log directory for the containers]]>
+      </doc>
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_BACKUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_VERSION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_UI_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of names for UIs hosted in the timeline server
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_WEB_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Relative web path that will serve up this UI (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client settings.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to war file or static content directory for this UI
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting for timeline service v1.5]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline service v2.0.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default schema prefix for hbase tables.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[config param name to override schema prefix.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how often the timeline collector flushes the
+ timeline writer.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls the capacity of the queue for async writes
+ to timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_FINAL_VALUE_RETENTION_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how long the final value of
+ a metric of a completed app is retained before merging
+ into the flow sum.]]>
+      </doc>
+    </field>
+    <field name="FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the setting for the location of the coprocessor
+ jar on hdfs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default hdfs location for flowrun coprocessor jar.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls the max size of the flow name getting generated
+ in ATSv2 after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_DEFAULT_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default setting for flow name size has no size restriction
+ after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that points to an optional HBase configuration
+ (hbase-site.xml file) with settings that will override the ones found on
+ the classpath.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that enables or disables authentication checks
+ for reading timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how often in-memory app level
+ aggregation is kicked off in timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.Boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default setting for authentication checks for reading timeline
+ service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that lists the users and groups who are allowed
+ to read timeline service v2 data. It is a comma separated list of
+ user, followed by space, then comma separated list of groups.
+ It will allow this list of users and groups to read the data
+ and reject everyone else.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for list of the users who are allowed to read
+ timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how long the final value of a metric of a
+ completed app is retained before merging into the flow sum. Up to this time
+ after an application is completed out-of-order values that arrive can be
+ recognized and discarded at the cost of increased storage.]]>
+      </doc>
+    </field>
+    <field name="ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default version for any flow.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time period for which timeline v2 client will wait for draining
+ leftover entities after stop.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether application history service is
+  enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Application history store class]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Save container meta-info in the application history store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemApplicationHistoryStore]]>
+      </doc>
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether timeline service is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[host:port address for timeline service RPC APIs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The listening endpoint for the timeline service application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_HANDLER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle client RPC API requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_MAX_APPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defines the max number of applications could be fetched using
+ REST API or application history protocol and shown in timeline
+ server web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_MAX_APPS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service store class.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service enable data age off]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service length of time to retain data]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service rolling period. Valid values are daily, half_daily,
+ quarter_daily, and hourly.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Roll a new database each hour.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Implementation specific configuration prefix for Timeline Service
+ leveldb.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb read cache (uncompressed blocks). This is
+ per rolling instance so should be tuned if using rolling leveldb
+ timeline store]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb read cache size if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write buffer size.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write buffer size if no configuration is specified. This
+ is per rolling instance so should be tuned if using rolling leveldb
+ timeline store.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write batch size. This value can be tuned down
+ to reduce lock time for ttl eviction.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write batch size is no configuration is specified]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time read cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time write cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb interval to wait between deletion rounds]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb number of concurrent open files. Tuned this
+ configuration to stay within system limits. This is per rolling instance
+ so should be tuned if using rolling leveldb timeline store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb max open files if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos keytab for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enables cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, max retries (-1 means no limit)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, retry interval]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client policy for whether connections are fatal]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable recovery of timeline service]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STATE_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store class]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_OFFLINE_AGGREGATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[whether the shared cache is enabled/disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the shared cache root directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NESTED_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the level of nested directories before getting to the
+ checksum directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NESTED_LEVEL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STALENESS_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A resource in the InMemorySCMStore is considered stale if the time since
+ the last reference exceeds the staleness period. This value is specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the in-memory store runs its first check to remove
+ dead initial applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_CHECK_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which the in-memory store checks to remove dead initial
+ applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which a cleaner task runs. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the first cleaner task is scheduled. Specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_RESOURCE_SLEEP_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time to sleep between processing each shared cache resource. Specified
+ in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the node manager interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of SCM threads used to handle notify requests from the node
+ manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the client interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle shared cache manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the checksum algorithm implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The replication factor for the node manager uploader for the shared cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_MACHINE_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_USERNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_PASSWORD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PIPELINE_CACHE_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_SUBMIT_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of retries for GetNewApplication and SubmitApplication in
+ {@code FederationClientInterceptor}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_USER_CLIENT_THREADS_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST to communicate with
+ each SubCluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST should return
+ partial AppReports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
+ The interval of the yarn client's querying application state after
+ application submission. The unit is millisecond.]]>
+      </doc>
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval that the yarn client library uses to poll the completion
+ status of the asynchronous API of application client protocol.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The duration that the yarn client library waits, cumulatively across polls,
+ for an expected state change to occur. Defaults to -1, which indicates no
+ limit.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of threads in NMClientAsync to process container management
+ events]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_MAX_NM_PROXIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of proxy connections to cache for node managers. If set
+ to a value greater than zero then the cache is enabled and the NMClient
+ and MRAppMaster will cache the specified number of node manager proxies.
+ There will be at max one proxy per node manager. Ex. configuring it to a
+ value of 5 will make sure that client will at max have 5 proxies cached
+ with 5 different node managers. These connections for these proxies will
+ be timed out if idle for more than the system wide idle timeout period.
+ Note that this could cause issues on large clusters as many connections
+ could linger simultaneously and lead to a large number of connection
+ threads. The token used for authentication will be used only at
+ connection creation time. If a new token is received then the earlier
+ connection should be closed in order to use the new token. This and
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related
+ and should be in sync (no need for them to be equal).
+ If the value of this property is zero then the connection cache is
+ disabled and connections will use a zero idle timeout to prevent too
+ many connection threads on large clusters.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_MAX_NM_PROXIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for NM to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each NM attempt to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-labels configurations]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node label store implementation class]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_NODE_LABELS_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for NodeLabelManager]]>
+      </doc>
+    </field>
+    <field name="NODE_ATTRIBUTE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-attribute configurations.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute store implementation class.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File system node attribute store directory.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to indicate if the node labels feature enabled, by default it's
+ disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABELS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CENTRALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONFIG_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCRIPT_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute provider fetch attributes interval and timeout.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPLAY_APPS_FOR_LOGGED_IN_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_ENTITY_LIST_BY_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENABLE_REST_APP_SUBMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENABLE_REST_APP_SUBMISSIONS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REST_CSRF" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="XFS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_XFS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline reader.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_STORAGE_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Marked collector properties as Private since it run as auxillary service.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for NUMA awareness.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_READ_TOPOLOGY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_READ_TOPOLOGY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NODE_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINERS_LAUNCHER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers launcher implementation to use.]]>
+      </doc>
+    </field>
+    <field name="YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+</package>
+<package name="org.apache.hadoop.yarn.exceptions">
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <class name="ApplicationAttemptNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getApplicationAttemptReport (GetApplicationAttemptReportRequest)}
+ API when the Application Attempt doesn't exist in Application History Server or
+ {@link ApplicationMasterProtocol#allocate(AllocateRequest)} if application
+ doesn't exist in RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <class name="ApplicationIdNotProvidedException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception to be thrown when Client submit an application without
+ providing {@link ApplicationId} in {@link ApplicationSubmissionContext}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <class name="ApplicationNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} API
+ when the Application doesn't exist in RM and AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <class name="ConfigurationException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConfigurationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on unrecoverable configuration errors.
+ An example is container launch error due to configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <class name="ContainerNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getContainerReport (GetContainerReportRequest)}
+ API when the container doesn't exist in AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <class name="ResourceNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when details of an unknown resource type
+ are requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <class name="YarnException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[YarnException indicates exceptions from yarn servers. On the other hand,
+ IOExceptions indicates exceptions from RPC layer.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+  <class name="YARNFeatureNotEnabledException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when a feature is being used which is not enabled
+ yet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <class name="ApplicationInitializationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ApplicationId, java.nio.ByteBuffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-name of the application-submitter
+ 
+ @return user-name]]>
+      </doc>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <method name="getApplicationDataForService" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data sent to the NodeManager via
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ as part of {@link ContainerLaunchContext#getServiceData()}
+ 
+ @return the servicesData for this application.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <class name="ApplicationTerminationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTerminationContext" type="org.apache.hadoop.yarn.api.records.ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application being stopped.
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when stopping an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <interface name="AuxiliaryLocalPathHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getLocalPathForRead" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="size" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing a file of an estimated size
+ for a given Auxiliary Service.
+ @param path the requested path
+ @param size the size of the file that is going to be written
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getAllLocalPathsForRead" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all paths from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path list to the file on a local disk as an Iterable
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An Interface that can retrieve local directories to read from or write to.
+  Components can implement this interface to link it to
+  their own Directory Handler Service]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <!-- start class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <class name="AuxiliaryService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuxiliaryService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecoveryPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path specific to this auxiliary service to use for recovery.
+
+ @return state storage path or null if recovery is not enabled]]>
+      </doc>
+    </method>
+    <method name="initializeApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationInitializationContext"/>
+      <doc>
+      <![CDATA[A new application is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the application initialization.
+ 
+ @param initAppContext context for the application's initialization]]>
+      </doc>
+    </method>
+    <method name="stopApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationTerminationContext"/>
+      <doc>
+      <![CDATA[An application is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+ 
+ @param stopAppContext context for the application termination]]>
+      </doc>
+    </method>
+    <method name="getMetaData" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve meta-data for this {@link AuxiliaryService}. Applications using
+ this {@link AuxiliaryService} SHOULD know the format of the meta-data -
+ ideally each service should provide a method to parse out the information
+ to the applications. One example of meta-data is contact information so
+ that applications can access the service remotely. This will only be called
+ after the service's {@link #start()} method has finished. the result may be
+ cached.
+ 
+ <p>
+ The information is passed along to applications via
+ {@link StartContainersResponse#getAllServicesMetaData()} that is returned by
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ </p>
+ 
+ @return meta-data for this service that should be made available to
+         applications.]]>
+      </doc>
+    </method>
+    <method name="initializeContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerInitializationContext"/>
+      <doc>
+      <![CDATA[A new container is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the container initialization.
+ This method is called when the NodeManager receives the container launch
+ command from the ApplicationMaster and before the container process is 
+ launched.
+
+ @param initContainerContext context for the container's initialization]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerTerminationContext"/>
+      <doc>
+      <![CDATA[A container is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+
+ @param stopContainerContext context for the container termination]]>
+      </doc>
+    </method>
+    <method name="setRecoveryPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recoveryPath" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path for this auxiliary service to use for storing state
+ that will be used during recovery.
+
+ @param recoveryPath where recoverable state should be stored]]>
+      </doc>
+    </method>
+    <method name="getAuxiliaryLocalPathHandler" return="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Method that gets the local dirs path handler for this Auxiliary Service.
+
+ @return auxiliaryPathHandler object that is used to read from and write to
+ valid local Dirs.]]>
+      </doc>
+    </method>
+    <method name="setAuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="auxiliaryLocalPathHandler" type="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"/>
+      <doc>
+      <![CDATA[Method that sets the local dirs path handler for this Auxiliary Service.
+
+ @param auxiliaryLocalPathHandler the pathHandler for this auxiliary service]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A generic service that will be started by the NodeManager. This is a service
+ that administrators have to configure on each node by setting
+ {@link YarnConfiguration#NM_AUX_SERVICES}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <class name="ContainerContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get user of the container being initialized or stopped.
+
+ @return the user]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container being initialized or stopped.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link Resource} the resource capability allocated to the container
+ being initialized or stopped.
+
+ @return the resource capability.]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container
+ being initialized or stopped.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ExecutionType} the execution type of the container
+ being initialized or stopped.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base context class for {@link AuxiliaryService} initializing and stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <class name="ContainerInitializationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <interface name="ContainerLogAggregationPolicy"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to initial the policy object with parameters specified by the application
+ or the cluster-wide setting.
+ </p>
+
+ @param parameters parameters with scheme defined by the policy class.]]>
+      </doc>
+    </method>
+    <method name="shouldDoLogAggregation" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logContext" type="org.apache.hadoop.yarn.server.api.ContainerLogContext"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to ask the policy object if a given container's logs should be aggregated.
+ </p>
+
+ @param logContext ContainerLogContext
+ @return Whether or not the container's logs should be aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This API is used by NodeManager to decide if a given container's logs
+ should be aggregated at run time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <class name="ContainerLogContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLogContext" type="org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.server.api.ContainerType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exit code of the container.
+
+ @return the exit code]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Context class for {@link ContainerLogAggregationPolicy}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <class name="ContainerTerminationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Termination context for {@link AuxiliaryService} when stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+  <interface name="SCMAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="runCleanerTask" return="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by administrators to ask SCM to run cleaner task right away
+ </p>
+
+ @param request request <code>SharedCacheManager</code> to run a cleaner task
+ @return <code>SharedCacheManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between administrators and the <code>SharedCacheManager</code>
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+</package>
+<package name="org.apache.hadoop.yarn.util.constraint">
+  <!-- start class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+  <class name="PlacementConstraintParser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseExpression" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintStr" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a given constraint expression to a {@link AbstractConstraint},
+ this expression can be any valid form of constraint expressions.
+
+ @param constraintStr expression string
+ @return a parsed {@link AbstractConstraint}
+ @throws PlacementConstraintParseException when given expression
+ is malformed]]>
+      </doc>
+    </method>
+    <method name="parsePlacementSpec" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expression" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a placement constraint specification. A placement constraint spec
+ is a composite expression which is composed by multiple sub constraint
+ expressions delimited by ":". With following syntax:
+
+ <p>Tag1(N1),P1:Tag2(N2),P2:...:TagN(Nn),Pn</p>
+
+ where <b>TagN(Nn)</b> is a key value pair to determine the source
+ allocation tag and the number of allocations, such as:
+
+ <p>foo(3)</p>
+
+ Optional when using NodeAttribute Constraint.
+
+ and where <b>Pn</b> can be any form of a valid constraint expression,
+ such as:
+
+ <ul>
+   <li>in,node,foo,bar</li>
+   <li>notin,node,foo,bar,1,2</li>
+   <li>and(notin,node,foo:notin,node,bar)</li>
+ </ul>
+
+ and NodeAttribute Constraint such as
+
+ <ul>
+   <li>yarn.rm.io/foo=true</li>
+   <li>java=1.7,1.8</li>
+ </ul>
+ @param expression expression string.
+ @return a map of source tags to placement constraint mapping.
+ @throws PlacementConstraintParseException]]>
+      </doc>
+    </method>
+    <field name="EXPRESSION_VAL_DELIM" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Placement constraint expression parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.3.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.3.xml
new file mode 100644
index 0000000000000..a994be99e7be4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.3.xml
@@ -0,0 +1,26407 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:07:25 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN API 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/classes:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/site/jdiff/xml -apiname Apache Hadoop YARN API 3.3.3 -->
+<package name="org.apache.hadoop.yarn.ams">
+</package>
+<package name="org.apache.hadoop.yarn.api">
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <interface name="ApplicationClientProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <method name="getNewApplication" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ApplicationId} for 
+ submitting new applications.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a new, monotonically
+ increasing, {@link ApplicationId} which is used by the client to submit
+ a new application.</p>
+
+ <p>The <code>ResourceManager</code> also responds with details such 
+ as maximum resource capabilities in the cluster as specified in
+ {@link GetNewApplicationResponse}.</p>
+
+ @param request request to get a new <code>ApplicationId</code>
+ @return response containing the new <code>ApplicationId</code> to be used
+ to submit an application
+ @throws YarnException
+ @throws IOException
+ @see #submitApplication(SubmitApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to submit a new application to the
+ <code>ResourceManager.</code></p>
+ 
+ <p>The client is required to provide details such as queue, 
+ {@link Resource} required to run the <code>ApplicationMaster</code>, 
+ the equivalent of {@link ContainerLaunchContext} for launching
+ the <code>ApplicationMaster</code> etc. via the 
+ {@link SubmitApplicationRequest}.</p>
+ 
+ <p>Currently the <code>ResourceManager</code> sends an immediate (empty) 
+ {@link SubmitApplicationResponse} on accepting the submission and throws 
+ an exception if it rejects the submission. However, this call needs to be
+ followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ to make sure that the application gets properly submitted - obtaining a
+ {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee
+ that RM 'remembers' this application beyond failover or restart. If RM
+ failover or RM restart happens before ResourceManager saves the
+ application's state successfully, the subsequent
+ {@link #getApplicationReport(GetApplicationReportRequest)} will throw
+ a {@link ApplicationNotFoundException}. The Clients need to re-submit
+ the application with the same {@link ApplicationSubmissionContext} when
+ it encounters the {@link ApplicationNotFoundException} on the
+ {@link #getApplicationReport(GetApplicationReportRequest)} call.</p>
+ 
+ <p>During the submission process, it checks whether the application
+ already exists. If the application exists, it will simply return
+ SubmitApplicationResponse</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to
+ queues etc. before accepting the application submission.</p>
+ 
+ @param request request to submit a new application
+ @return (empty) response on accepting the submission
+ @throws YarnException
+ @throws IOException
+ @see #getNewApplication(GetNewApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the 
+ <code>ResourceManager</code> to fail an application attempt.</p>
+
+ <p>The client, via {@link FailApplicationAttemptRequest} provides the
+ {@link ApplicationAttemptId} of the attempt to be failed.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before failing the attempt.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to fail an attempt
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to abort submitted application.</p>
+ 
+ <p>The client, via {@link KillApplicationRequest} provides the
+ {@link ApplicationId} of the application to be aborted.</p>
+ 
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before terminating the application.</p> 
+ 
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+ 
+ @param request request to abort a submitted application
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get metrics about the cluster from
+ the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a
+ {@link GetClusterMetricsResponse} which includes the 
+ {@link YarnClusterMetrics} with details such as number of current
+ nodes in the cluster.</p>
+ 
+ @param request request for cluster metrics
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a 
+ {@link GetClusterNodesResponse} which includes the 
+ {@link NodeReport} for all the nodes in the cluster.</p>
+ 
+ @param request request for report on all nodes
+ @return report on all nodes
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queues</em>
+ from the <code>ResourceManager</code>.</p>
+ 
+ <p>The client, via {@link GetQueueInfoRequest}, can ask for details such
+ as used/total resources, child queues, running applications etc.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access before
+ providing the information.</p> 
+ 
+ @param request request to get queue information
+ @return queue information
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueUserAcls" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queue 
+ acls</em> for <em>current user</em> from the <code>ResourceManager</code>.
+ </p>
+ 
+ <p>The <code>ResourceManager</code> responds with queue acls for all
+ existing queues.</p>
+ 
+ @param request request to get queue acls for <em>current user</em>
+ @return queue acls for <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move an application to a new queue.
+ 
+ @param request the application ID and the target queue
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNewReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ReservationId} for
+ submitting new reservations.</p>
+
+ <p>The <code>ResourceManager</code> responds with a new, unique,
+ {@link ReservationId} which is used by the client to submit
+ a new reservation.</p>
+
+ @param request to get a new <code>ReservationId</code>
+ @return response containing the new <code>ReservationId</code> to be used
+ to submit a new reservation
+ @throws YarnException if the reservation system is not enabled.
+ @throws IOException on IO failures.
+ @see #submitReservation(ReservationSubmissionRequest)]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and concurrency needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationSubmissionRequest is satisfiable the
+ {@code ResourceManager} answers with a
+ {@link ReservationSubmissionResponse} that include a non-null
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the reason.
+ 
+ On application submission the client can use this {@link ReservationId} to
+ obtain access to the reserved resources.
+ </p>
+ 
+ <p>
+ The system guarantees that during the time-range specified by the user, the
+ reservationID will be corresponding to a valid reservation. The amount of
+ capacity dedicated to such queue can vary overtime, depending of the
+ allocation that has been determined. But it is guaranteed to satisfy all
+ the constraint expressed by the user in the
+ {@link ReservationSubmissionRequest}.
+ </p>
+ 
+ @param request the request to submit a new Reservation
+ @return response the {@link ReservationId} on accepting the submission
+ @throws YarnException if the request is invalid or reservation cannot be
+           created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationUpdateRequest}. Upon success the previous allocation is
+ substituted by the new one, and on failure (i.e., if the system cannot find
+ a valid allocation for the updated request), the previous allocation
+ remains valid.
+ 
+ The {@link ReservationId} is not changed, and applications currently
+ running within this reservation will automatically receive the resources
+ based on the new allocation.
+ </p>
+ 
+ @param request to update an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ 
+ Upon deletion of a reservation applications running with this reservation,
+ are automatically downgraded to normal jobs running without any dedicated
+ reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+
+ @param request
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mappings
+ in existing cluster
+ </p>
+
+ @param request
+ @return labels to nodes mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @param request to get node labels collection of this cluster
+ @return node labels collection of this cluster
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application.
+ </p>
+ @param request to set priority of an application
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to signal a container. For example,
+ the client can send command OUTPUT_THREAD_DUMP to dump threads of the
+ container.</p>
+
+ <p>The client, via {@link SignalContainerRequest} provides the
+ id of the container and the signal command. </p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application before signaling the container.
+ The user needs to have <code>MODIFY_APP</code> permission.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to signal a container
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set ApplicationTimeouts of an application.
+ The UpdateApplicationTimeoutsRequest should have timeout value with
+ absolute time with ISO8601 format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ </p>
+ <b>Note:</b> If application timeout value is less than or equal to current
+ time then update application throws YarnException.
+ @param request to set ApplicationTimeouts of an application
+ @return a response with updated timeouts.
+ @throws YarnException if update request has empty values or application is
+           in completing states.
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get all the resource profiles that are
+ available on the ResourceManager.
+ </p>
+ @param request request to get all the resource profiles
+ @return Response containing a map of the profile name to Resource
+         capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get attributes to nodes mappings
+ available in ResourceManager.
+ </p>
+
+ @param request request to get details of attributes to nodes mapping.
+ @return Response containing the details of attributes to nodes mappings.
+ @throws YarnException if any error happens inside YARN
+ @throws IOException   incase of other errors]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes available in
+ ResourceManager.
+ </p>
+
+ @param request request to get node attributes collection of this cluster.
+ @return Response containing node attributes collection.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException   incase of other errors.]]>
+      </doc>
+    </method>
+    <method name="getNodesToAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to attributes mappings.
+ in existing cluster.
+ </p>
+
+ @param request request to get nodes to attributes mapping.
+ @return nodes to attributes mappings.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between clients and the <code>ResourceManager</code>
+ to submit/abort jobs and to get information on applications, cluster metrics,
+ nodes, queues and ACLs.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <interface name="ApplicationConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="APP_SUBMIT_TIME_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
+ only]]>
+      </doc>
+    </field>
+    <field name="CONTAINER_TOKEN_FILE_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cache file into which container token is written]]>
+      </doc>
+    </field>
+    <field name="KEYSTORE_FILE_LOCATION_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file into which the keystore containing the AM's certificate is
+ written.]]>
+      </doc>
+    </field>
+    <field name="KEYSTORE_PASSWORD_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The password for the AM's keystore.]]>
+      </doc>
+    </field>
+    <field name="TRUSTSTORE_FILE_LOCATION_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file into which the truststore containing the AM's certificate is
+ written.]]>
+      </doc>
+    </field>
+    <field name="TRUSTSTORE_PASSWORD_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The password for the AM's truststore.]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_WEB_PROXY_BASE_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ ApplicationMaster's environment only. This states that for all non-relative
+ web URLs in the app masters web UI what base should they have.]]>
+      </doc>
+    </field>
+    <field name="LOG_DIR_EXPANSION_VAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The temporary environmental variable for container log directory. This
+ should be replaced by real container log directory on container launch.]]>
+      </doc>
+    </field>
+    <field name="CLASS_PATH_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constant is used to construct class path and it will be replaced with
+ real class path separator(':' for Linux and ';' for Windows) by
+ NodeManager on container launch. User has to use this constant to construct
+ class path if user wants cross-platform practice i.e. submit an application
+ from a Windows client to a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_LEFT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The following two constants are used to expand parameter and it will be
+ replaced with real parameter expansion marker ('%' for Windows and '$' for
+ Linux) by NodeManager on container launch. For example: {{VAR}} will be
+ replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+ constant to construct class path if user wants cross-platform practice i.e.
+ submit an application from a Windows client to a Linux/Unix server or vice
+ versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_RIGHT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User has to use this constant to construct class path if user wants
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="STDERR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STDOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is the API for the applications comprising of constants that YARN sets
+ up for the applications and the containers.
+
+ TODO: Investigate the semantics and security of each cross-boundary refs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <!-- start class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <class name="ApplicationConstants.ContainerLaunchType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The type of launch for the container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <interface name="ApplicationHistoryProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>ApplicationHistoryServer</code> to
+ get the information of completed applications etc.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <interface name="ApplicationMasterProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by a new <code>ApplicationMaster</code> to register with
+ the <code>ResourceManager</code>.
+ </p>
+
+ <p>
+ The <code>ApplicationMaster</code> needs to provide details such as RPC
+ Port, HTTP tracking url etc. as specified in
+ {@link RegisterApplicationMasterRequest}.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code> responds with critical details such as
+ maximum resource capabilities in the cluster as specified in
+ {@link RegisterApplicationMasterResponse}.
+ </p>
+
+ <p>
+ Re-register is only allowed for <code>Unmanaged Application Master</code>
+ (UAM) HA, with
+ {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ set to true.
+ </p>
+
+ @param request registration request
+ @return registration respose
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException The exception is thrown
+           when an ApplicationMaster tries to register more then once.
+ @see RegisterApplicationMasterRequest
+ @see RegisterApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by an <code>ApplicationMaster</code> to notify the 
+ <code>ResourceManager</code> about its completion (success or failed).</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as 
+ final state, diagnostics (in case of failures) etc. as specified in 
+ {@link FinishApplicationMasterRequest}.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with 
+ {@link FinishApplicationMasterResponse}.</p>
+ 
+ @param request completion request
+ @return completion response
+ @throws YarnException
+ @throws IOException
+ @see FinishApplicationMasterRequest
+ @see FinishApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The main interface between an <code>ApplicationMaster</code> and the
+ <code>ResourceManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> uses this interface to provide a list of
+ {@link ResourceRequest} and returns unused {@link Container} allocated to
+ it via {@link AllocateRequest}. Optionally, the
+ <code>ApplicationMaster</code> can also <em>blacklist</em> resources which
+ it doesn't want to use.
+ </p>
+ 
+ <p>
+ This also doubles up as a <em>heartbeat</em> to let the
+ <code>ResourceManager</code> know that the <code>ApplicationMaster</code>
+ is alive. Thus, applications should periodically make this call to be kept
+ alive. The frequency depends on
+ {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
+ </p>
+ 
+ <p>
+ The <code>ResourceManager</code> responds with list of allocated
+ {@link Container}, status of completed containers and headroom information
+ for the application.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> can use the available headroom
+ (resources) to decide how to utilized allocated resources and make informed
+ decisions about future resource requests.
+ </p>
+ 
+ @param request
+          allocation request
+ @return allocation response
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException
+           This exception is thrown when an ApplicationMaster calls allocate
+           without registering first.
+ @throws InvalidResourceBlacklistRequestException
+           This exception is thrown when an application provides an invalid
+           specification for blacklist of resources.
+ @throws InvalidResourceRequestException
+           This exception is thrown when a {@link ResourceRequest} is out of
+           the range of the configured lower and upper limits on the
+           resources.
+ @see AllocateRequest
+ @see AllocateResponse]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between a live instance of <code>ApplicationMaster</code> 
+ and the <code>ResourceManager</code>.</p>
+ 
+ <p>This is used by the <code>ApplicationMaster</code> to register/unregister
+ and to request and obtain resources in the cluster from the
+ <code>ResourceManager</code>.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <interface name="ClientSCMProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="use" return="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to claim a resource with the
+ <code>SharedCacheManager.</code> The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application will be
+ using the resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>Path</code>
+ to the resource in the shared cache is returned. If the resource does not
+ exist, the response is empty.
+ </p>
+
+ @param request request to claim a resource in the shared cache
+ @return response indicating if the resource is already in the cache
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="release" return="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to release a resource with the
+ <code>SharedCacheManager.</code> This method is called once an application
+ is no longer using a claimed resource in the shared cache. The client uses
+ a checksum to identify the resource and an {@link ApplicationId} to
+ identify which application is releasing the resource.
+ </p>
+
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+
+ <p>
+ Currently the <code>SharedCacheManager</code> sends an empty response.
+ </p>
+
+ @param request request to release a resource in the shared cache
+ @return (empty) response on releasing the resource
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>SharedCacheManager</code> to claim
+ and release resources in the shared cache.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+  <interface name="ContainerManagementProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="startContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> provides a list of
+ {@link StartContainerRequest}s to a <code>NodeManager</code> to
+ <em>start</em> {@link Container}s allocated to it using this interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> has to provide details such as allocated
+ resource capability, security tokens (if enabled), command to be executed
+ to start the container, environment for the process, necessary
+ binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in
+ the {@link StartContainerRequest}.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StartContainersResponse} which includes a list of
+ {@link Container}s of successfully launched {@link Container}s, a
+ containerId-to-exception map for each failed {@link StartContainerRequest} in
+ which the exception indicates errors from per container and a
+ allServicesMetaData map between the names of auxiliary services and their
+ corresponding meta-data. Note: None-container-specific exceptions will
+ still be thrown by the API method itself.
+ </p>
+ <p>
+ The <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the to-be-launched or launched containers.
+ </p>
+ 
+ @param request
+          request to start a list of containers
+ @return response including conatinerIds of all successfully launched
+         containers, a containerId-to-exception map for failed requests and
+         a allServicesMetaData map.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="stopContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> requests a <code>NodeManager</code> to
+ <em>stop</em> a list of {@link Container}s allocated to it using this
+ interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a {@link StopContainersRequest}
+ which includes the {@link ContainerId}s of the containers to be stopped.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StopContainersResponse} which includes a list of {@link ContainerId}
+ s of successfully stopped containers, a containerId-to-exception map for
+ each failed request in which the exception indicates errors from per
+ container. Note: None-container-specific exceptions will still be thrown by
+ the API method itself. <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the containers.
+ </p>
+ 
+ @param request
+          request to stop a list of containers
+ @return response which includes a list of containerIds of successfully
+         stopped containers, a containerId-to-exception map for failed
+         requests.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for current
+ statuses of <code>Container</code>s from the <code>NodeManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a
+ {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s
+ of all containers whose statuses are needed.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> responds with
+ {@link GetContainerStatusesResponse} which includes a list of
+ {@link ContainerStatus} of the successfully queried containers and a
+ containerId-to-exception map for each failed request in which the exception
+ indicates errors from per container. Note: None-container-specific
+ exceptions will still be thrown by the API method itself.
+ </p>
+ 
+ @param request
+          request to get <code>ContainerStatus</code>es of containers with
+          the specified <code>ContainerId</code>s
+ @return response containing the list of <code>ContainerStatus</code> of the
+         successfully queried containers and a containerId-to-exception map
+         for failed requests.
+ 
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="increaseContainersResource" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource increase of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to increase resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully increased and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource update of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to update resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully updated and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from NodeManager]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="localize" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources required by the container.
+ Currently, this API only works for running containers.
+
+ @param request Specify the resources to be localized.
+ @return Response that the localize request is accepted.
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[ReInitialize the Container with a new Launch Context.
+ @param request Specify the new ContainerLaunchContext.
+ @return Response that the ReInitialize request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="restartContainer" return="org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restart the container.
+ @param containerId Container Id.
+ @return Response that the restart request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rollback the Last ReInitialization if possible.
+ @param containerId Container Id.
+ @return Response that the rollback request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.CommitResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Commit the Last ReInitialization if possible. Once the reinitialization
+ has been committed, It cannot be rolled back.
+ @param containerId Container Id.
+ @return Response that the commit request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="getLocalizationStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[API to request for the localization statuses of requested containers from
+ the Node Manager.
+ @param request {@link GetLocalizationStatusesRequest} which includes the
+                container ids of all the containers whose localization
+                statuses are needed.
+ @return {@link GetLocalizationStatusesResponse} which contains the
+         localization statuses of all the requested containers.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between an <code>ApplicationMaster</code> and a 
+ <code>NodeManager</code> to start/stop and increase resource of containers
+ and to get status of running containers.</p>
+
+ <p>If security is enabled the <code>NodeManager</code> verifies that the
+ <code>ApplicationMaster</code> has truly been allocated the container
+ by the <code>ResourceManager</code> and also verifies all interactions such 
+ as stopping the container or obtaining status information for the container.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.api.protocolrecords">
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <class name="AllocateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="updateRequests" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>response id</em> used to track duplicate responses.
+ @return <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="setResponseId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>response id</em> used to track duplicate responses.
+ @param id <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current progress</em> of application. 
+ @return <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="setProgress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <em>current progress</em> of application
+ @param progress <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="getAskList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ResourceRequest</code> to update the 
+ <code>ResourceManager</code> about the application's resource requirements.
+ @return the list of <code>ResourceRequest</code>
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="setAskList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of <code>ResourceRequest</code> to update the
+ <code>ResourceManager</code> about the application's resource requirements.
+ @param resourceRequests list of <code>ResourceRequest</code> to update the 
+                        <code>ResourceManager</code> about the application's 
+                        resource requirements
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getReleaseList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> of containers being 
+ released by the <code>ApplicationMaster</code>.
+ @return list of <code>ContainerId</code> of containers being 
+         released by the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setReleaseList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseContainers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of <code>ContainerId</code> of containers being
+ released by the <code>ApplicationMaster</code>
+ @param releaseContainers list of <code>ContainerId</code> of 
+                          containers being released by the 
+                          <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceBlacklistRequest" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ResourceBlacklistRequest</code> being sent by the 
+ <code>ApplicationMaster</code>.
+ @return the <code>ResourceBlacklistRequest</code> being sent by the 
+         <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="setResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ResourceBlacklistRequest</code> to inform the 
+ <code>ResourceManager</code> about the blacklist additions and removals
+ per the <code>ApplicationMaster</code>.
+ 
+ @param resourceBlacklistRequest the <code>ResourceBlacklistRequest</code>  
+                         to inform the <code>ResourceManager</code> about  
+                         the blacklist additions and removals
+                         per the <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="getUpdateRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link UpdateContainerRequest}
+         being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setUpdateRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update requests to inform the
+ <code>ResourceManager</code> about the containers that need to be
+ updated.
+ @param updateRequests list of <code>UpdateContainerRequest</code> for
+                       containers to be updated]]>
+      </doc>
+    </method>
+    <method name="getSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of Scheduling requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link SchedulingRequest} being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of Scheduling requests to inform the
+ <code>ResourceManager</code> about the application's resource requirements
+ (potentially including allocation tags and placement constraints).
+ @param schedulingRequests list of {@link SchedulingRequest} to update
+          the <code>ResourceManager</code> about the application's resource
+          requirements.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tracking url update for this heartbeat.
+ @return tracking url to update this application with]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the new tracking url for this application.
+ @param trackingUrl the new tracking url]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The core request sent by the <code>ApplicationMaster</code> to the 
+ <code>ResourceManager</code> to obtain resources in the cluster.</p> 
+
+ <p>The request includes:
+ <ul>
+   <li>A response id to track duplicate responses.</li>
+   <li>Progress information.</li>
+   <li>
+     A list of {@link ResourceRequest} to inform the
+     <code>ResourceManager</code> about the application's
+     resource requirements.
+   </li>
+   <li>
+     A list of unused {@link Container} which are being returned.
+   </li>
+   <li>
+     A list of {@link UpdateContainerRequest} to inform
+     the <code>ResourceManager</code> about the change in
+     requirements of running containers.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <class name="AllocateRequest.AllocateRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="responseId" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>responseId</code> of the request.
+ @see AllocateRequest#setResponseId(int)
+ @param responseId <code>responseId</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="progress" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <code>progress</code> of the request.
+ @see AllocateRequest#setProgress(float)
+ @param progress <code>progress</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="askList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="askList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>askList</code> of the request.
+ @see AllocateRequest#setAskList(List)
+ @param askList <code>askList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="releaseList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>releaseList</code> of the request.
+ @see AllocateRequest#setReleaseList(List)
+ @param releaseList <code>releaseList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceBlacklistRequest" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>resourceBlacklistRequest</code> of the request.
+ @see AllocateRequest#setResourceBlacklistRequest(
+ ResourceBlacklistRequest)
+ @param resourceBlacklistRequest
+     <code>resourceBlacklistRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="updateRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>updateRequests</code> of the request.
+ @see AllocateRequest#setUpdateRequests(List)
+ @param updateRequests <code>updateRequests</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="schedulingRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>schedulingRequests</code> of the request.
+ @see AllocateRequest#setSchedulingRequests(List)
+ @param schedulingRequests <code>SchedulingRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="trackingUrl" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>trackingUrl</code> of the request.
+ @see AllocateRequest#setTrackingUrl(String)
+ @param trackingUrl new tracking url
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link AllocateRequest} object.
+ @return {@link AllocateRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link AllocateRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <class name="AllocateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+      <param name="amRMToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="updatedContainers" type="java.util.List"/>
+      <param name="collectorInfo" type="org.apache.hadoop.yarn.api.records.CollectorInfo"/>
+    </method>
+    <method name="getAMCommand" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the <code>ResourceManager</code> needs the
+ <code>ApplicationMaster</code> to take some action then it will send an
+ AMCommand to the <code>ApplicationMaster</code>. See <code>AMCommand</code> 
+ for details on commands and actions for them.
+ @return <code>AMCommand</code> if the <code>ApplicationMaster</code> should
+         take action, <code>null</code> otherwise
+ @see AMCommand]]>
+      </doc>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last response id</em>.
+ @return <em>last response id</em>]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>newly allocated</em> <code>Container</code> by the
+ <code>ResourceManager</code>.
+ @return list of <em>newly allocated</em> <code>Container</code>]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>available headroom</em> for resources in the cluster for the
+ application.
+ @return limit of available headroom for resources in the cluster for the
+ application]]>
+      </doc>
+    </method>
+    <method name="getCompletedContainersStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>completed containers' statuses</em>.
+ @return the list of <em>completed containers' statuses</em>]]>
+      </doc>
+    </method>
+    <method name="getUpdatedNodes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>updated <code>NodeReport</code>s</em>. Updates could
+ be changes in health, availability etc of the nodes.
+ @return The delta of updated nodes since the last response]]>
+      </doc>
+    </method>
+    <method name="getNumClusterNodes" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of hosts available on the cluster.
+ @return the available host count.]]>
+      </doc>
+    </method>
+    <method name="getPreemptionMessage" return="org.apache.hadoop.yarn.api.records.PreemptionMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the description of containers owned by the AM, but requested back by
+ the cluster. Note that the RM may have an inconsistent view of the
+ resources owned by the AM. These messages are advisory, and the AM may
+ elect to ignore them.
+ <p>
+ The message is a snapshot of the resources the RM wants back from the AM.
+ While demand persists, the RM will repeat its request; applications should
+ not interpret each message as a request for <em>additional</em>
+ resources on top of previous messages. Resources requested consistently
+ over some duration may be forcibly killed by the RM.
+
+ @return A specification of the resources to reclaim from this AM.]]>
+      </doc>
+    </method>
+    <method name="getNMTokens" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens required for communicating with NM. New NMTokens
+ issued only if
+ <p>
+ 1) AM is receiving first container on underlying NodeManager.<br>
+ OR<br>
+ 2) NMToken master key rolled over in ResourceManager and AM is getting new
+ container on the same underlying NodeManager.
+ <p>
+ AM will receive one NMToken per NM irrespective of the number of containers
+ issued on same NM. AM is expected to store these tokens until issued a
+ new token for the same NM.
+ @return list of NMTokens required for communicating with NM]]>
+      </doc>
+    </method>
+    <method name="getUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of newly updated containers by
+ <code>ResourceManager</code>.
+ @return list of newly increased containers]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The AMRMToken that belong to this attempt
+
+ @return The AMRMToken that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the application
+
+ @return get application priority]]>
+      </doc>
+    </method>
+    <method name="getCollectorInfo" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The data associated with the collector that belongs to this app. Contains
+ address and token alongwith identification information.
+
+ @return The data of collector that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getUpdateErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+
+ @return List of Update Container Errors.]]>
+      </doc>
+    </method>
+    <method name="setUpdateErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+ @param updateErrors list of <code>UpdateContainerError</code> for
+                       containers updates requests that were in error]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts which
+ have not been reported to the Application Master yet.
+ <br>
+ These containers were recovered by the RM after the application master
+ had already registered. This may happen after RM restart when some NMs get
+ delayed in connecting to the RM and reporting the active containers.
+ Since they were not reported in the registration
+ response, they are reported in the response to the AM heartbeat.
+
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts.]]>
+      </doc>
+    </method>
+    <method name="getRejectedSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of all SchedulingRequests that the RM has rejected between
+ this allocate call and the previous one.
+ @return List of RejectedSchedulingRequests.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> the
+ <code>ApplicationMaster</code> during resource negotiation.
+ <p>
+ The response, includes:
+ <ul>
+   <li>Response ID to track duplicate responses.</li>
+   <li>
+     An AMCommand sent by ResourceManager to let the
+     {@code ApplicationMaster} take some actions (resync, shutdown etc.).
+   </li>
+   <li>A list of newly allocated {@link Container}.</li>
+   <li>A list of completed {@link Container}s' statuses.</li>
+   <li>
+     The available headroom for resources in the cluster for the
+     application.
+   </li>
+   <li>A list of nodes whose status has been updated.</li>
+   <li>The number of available nodes in a cluster.</li>
+   <li>A description of resources requested back by the cluster</li>
+   <li>AMRMToken, if AMRMToken has been rolled over</li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been increased.
+   </li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been decreased.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <class name="ApplicationsRequestScope" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration that controls the scope of applications fetched]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <class name="CommitResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommitResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to Commit Container Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <class name="ContainerUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToUpdate" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource update.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource update.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToUpdate" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getUpdatedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource update.
+ @param containersToUpdate the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <class name="ContainerUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="successfullyUpdatedContainers" type="java.util.List"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getSuccessfullyUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully update.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully updated.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to update container resource.
+ </p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <class name="FailApplicationAttemptRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the attempt to be failed.
+ @return <code>ApplicationAttemptId</code> of the attempt.]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to fail an application attempt.</p>
+
+ <p>The request includes the {@link ApplicationAttemptId} of the attempt to
+ be failed.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <class name="FailApplicationAttemptResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ failing an application attempt.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <class name="FinishApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="url" type="java.lang.String"/>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>final state</em> of the <code>ApplicationMaster</code>.
+ @return <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setFinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalState" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <doc>
+      <![CDATA[Set the <em>final state</em> of the <code>ApplicationMaster</code>
+ @param finalState <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic information</em> on application failure.
+ @return <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <em>diagnostic information</em> on application failure.
+ @param diagnostics <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ This is the web-URL to which ResourceManager or web-application proxy will
+ redirect client/users once the application is finished and the
+ <code>ApplicationMaster</code> is gone.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param url
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The finalization request sent by the {@code ApplicationMaster} to
+ inform the {@code ResourceManager} about its completion.
+ <p>
+ The final request includes details such:
+ <ul>
+   <li>Final state of the {@code ApplicationMaster}</li>
+   <li>
+     Diagnostic information in case of failure of the
+     {@code ApplicationMaster}
+   </li>
+   <li>Tracking URL</li>
+ </ul>
+
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <class name="FinishApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsUnregistered" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the application has successfully
+ unregistered with the RM and the application can safely stop.
+ @return true if the application has unregistered with the RM,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a
+ <code>ApplicationMaster</code> on it's completion.
+ <p>
+ The response, includes:
+ <ul>
+ <li>A flag which indicates that the application has successfully unregistered
+ with the RM and the application can safely stop.</li>
+ </ul>
+ <p>
+ Note: The flag indicates whether the application has successfully
+ unregistered and is safe to stop. The application may stop after the flag is
+ true. If the application stops before the flag is true then the RM may retry
+ the application.
+ 
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <class name="GetAllResourceProfilesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <class name="GetAllResourceProfilesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceProfiles"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profiles" type="java.util.Map"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <class name="GetAllResourceTypeInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <class name="GetAllResourceTypeInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceTypeInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceTypes" type="java.util.List"/>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <class name="GetApplicationAttemptReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ApplicationAttemptReport} for an application attempt.
+ </p>
+ 
+ <p>
+ The request should include the {@link ApplicationAttemptId} of the
+ application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <class name="GetApplicationAttemptReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ApplicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @return <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @param applicationAttemptReport
+          <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ an application attempt report.
+ </p>
+ 
+ <p>
+ The response includes an {@link ApplicationAttemptReport} which has the
+ details about the particular application attempt
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <class name="GetApplicationAttemptsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of an application
+ 
+ @return <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of an application
+ 
+ @param applicationId
+          <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of application attempt reports of an
+ application from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <class name="GetApplicationAttemptsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+    </method>
+    <method name="getApplicationAttemptList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @return a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @param applicationAttempts
+          a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ApplicationAttemptReport} for application attempts.
+ </p>
+ 
+ <p>
+ The <code>ApplicationAttemptReport</code> for each application includes the
+ details of an application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <class name="GetApplicationReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to the <code>ResourceManager</code> to 
+ get an {@link ApplicationReport} for an application.</p>
+ 
+ <p>The request should include the {@link ApplicationId} of the 
+ application.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)
+ @see ApplicationReport]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <class name="GetApplicationReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationReport</code> for the application.
+ @return <code>ApplicationReport</code> for the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an application report.</p>
+ 
+ <p>The response includes an {@link ApplicationReport} which has details such 
+ as user, queue, name, host on which the <code>ApplicationMaster</code> is 
+ running, RPC port, tracking URL, diagnostics, start time etc.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <class name="GetApplicationsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="queues" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="startRange" type="org.apache.commons.lang3.Range"/>
+      <param name="finishRange" type="org.apache.commons.lang3.Range"/>
+      <param name="limit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+
+ <p>Setting any of the parameters to null, would just disable that
+ filter</p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @param users list of users to filter by
+ @param queues list of scheduler queues to filter by
+ @param applicationTypes types of applications
+ @param applicationTags application tags to filter by
+ @param applicationStates application states to filter by
+ @param startRange range of application start times to filter by
+ @param finishRange range of application finish times to filter by
+ @param limit number of applications to limit to
+ @return {@link GetApplicationsRequest} to be used with
+ {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application states in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving and application types and application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in <code>GetApplicationsRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getApplicationTypes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application types to filter applications on
+
+ @return Set of Application Types to filter on]]>
+      </doc>
+    </method>
+    <method name="getApplicationStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application states to filter applications on
+
+ @return Set of Application states to filter on]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of Applications
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <class name="GetApplicationsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationReport</code> for applications.
+ @return <code>ApplicationReport</code> for applications]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an {@link ApplicationReport} for applications.</p>
+
+ <p>The <code>ApplicationReport</code> for each application includes details
+ such as user, queue, name, host on which the <code>ApplicationMaster</code>
+ is running, RPC port, tracking URL, diagnostics, start time etc.</p>
+
+ @see ApplicationReport
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <class name="GetAttributesToNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributeKeys for which the mapping of hostname to attribute value
+ is required.
+
+ @param attributes Set of NodeAttributeKey provided.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributeKeys for which mapping of hostname to attribute value is
+ required.
+
+ @return Set of NodeAttributeKey]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node to attribute value mapping for all or
+ given set of Node AttributeKey's in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <class name="GetAttributesToNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setAttributeToNodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get mapping of NodeAttributeKey to its associated mapping of list of
+ NodeToAttributeValue associated with attribute.
+
+ @return Map of node attributes to list of NodeToAttributeValue.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ node to attribute value mapping for all or given set of Node AttributeKey's.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <class name="GetClusterMetricsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get cluster metrics from the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>Currently, this is empty.</p>
+
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <class name="GetClusterMetricsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnClusterMetrics</code> for the cluster.
+ @return <code>YarnClusterMetrics</code> for the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a client
+ requesting cluster metrics.
+ 
+ @see YarnClusterMetrics
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <class name="GetClusterNodeAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create new instance of GetClusterNodeAttributesRequest.
+
+ @return GetClusterNodeAttributesRequest is returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node attributes in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <class name="GetClusterNodeAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Create instance of GetClusterNodeAttributesResponse.
+
+ @param attributes
+ @return GetClusterNodeAttributesResponse.]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributes to the response.
+
+ @param attributes Map of Node attributeKey to Type.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes from the response.
+
+ @return Node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a node attributes in cluster.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <class name="GetClusterNodeLabelsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <class name="GetClusterNodeLabelsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #newInstance(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Creates a new instance.
+
+ @param labels Node labels
+ @return response
+ @deprecated Use {@link #newInstance(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="setNodeLabelList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="getNodeLabelList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNodeLabels"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setNodeLabelList(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node labels to the response.
+
+ @param labels Node labels
+ @deprecated Use {@link #setNodeLabelList(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getNodeLabelList()} instead.">
+      <doc>
+      <![CDATA[Get node labels of the response.
+
+ @return Node labels
+ @deprecated Use {@link #getNodeLabelList()} instead.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <class name="GetClusterNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @return the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <method name="setNodeStates"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @param states the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ The request will ask for all nodes in the given {@link NodeState}s.
+
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <class name="GetClusterNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>NodeReport</code> for all nodes in the cluster.
+ @return <code>NodeReport</code> for all nodes in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting a {@link NodeReport} for all nodes.</p>
+ 
+ <p>The <code>NodeReport</code> contains per-node information such as 
+ available resources, number of containers, tracking url, rack name, health
+ status etc.
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <class name="GetContainerReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the Container.
+ 
+ @return <code>ContainerId</code> of the Container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container
+ 
+ @param containerId
+          <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ContainerReport} for a container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <class name="GetContainerReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerReport</code> for the container.
+ 
+ @return <code>ContainerReport</code> for the container]]>
+      </doc>
+    </method>
+    <method name="setContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a container report.
+ </p>
+ 
+ <p>
+ The response includes a {@link ContainerReport} which has details of a
+ container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <class name="GetContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of container reports, which belong to
+ an application attempt from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <class name="GetContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+    </method>
+    <method name="getContainerList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @return a list of <code>ContainerReport</code> for all the containers of an
+         application attempt]]>
+      </doc>
+    </method>
+    <method name="setContainerList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @param containers
+          a list of <code>ContainerReport</code> for all the containers of
+          an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ContainerReport} for containers.
+ </p>
+ 
+ <p>
+ The <code>ContainerReport</code> for each container includes the container
+ details.
+ </p>
+ 
+ @see ContainerReport
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <class name="GetContainerStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>.
+ 
+ @return the list of <code>ContainerId</code>s of containers for which to
+         obtain the <code>ContainerStatus</code>.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>
+ 
+ @param containerIds
+          a list of <code>ContainerId</code>s of containers for which to
+          obtain the <code>ContainerStatus</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to get {@link ContainerStatus} of requested
+ containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <class name="GetContainerStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerStatus</code>es of the requested containers.
+ 
+ @return <code>ContainerStatus</code>es of the requested containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to obtain the
+ <code>ContainerStatus</code> of requested containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <class name="GetDelegationTokenRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to get a delegation token from
+ the {@code ResourceManager}.
+ for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <class name="GetDelegationTokenResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Delegation tokens have a identifier which maps to
+ {@link AbstractDelegationTokenIdentifier}.
+ @return the delegation tokens]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Response to a {@link GetDelegationTokenRequest} request 
+ from the client. The response contains the token that 
+ can be used by the containers to talk to  ClientRMService.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest -->
+  <class name="GetLocalizationStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetLocalizationStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container IDs of the containers for which the localization
+ statuses are needed.
+
+ @return the list of container IDs.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets the list of container IDs of containers for which the localization
+ statuses are needed.
+ @param containerIds the list of container IDs.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by an application master to the node manager to get
+ {@link LocalizationStatus}es of containers.
+
+ @see ContainerManagementProtocol#getLocalizationStatuses(
+        GetLocalizationStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse -->
+  <class name="GetLocalizationStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetLocalizationStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statuses" type="java.util.Map"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getLocalizationStatuses" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the container localization statuses.
+
+ @return container localization statuses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the node manager to an application master when
+ localization statuses are requested.
+
+ @see ContainerManagementProtocol#getLocalizationStatuses(
+        GetLocalizationStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <class name="GetNewApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@link ApplicationId} for
+ submitting an application.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <class name="GetNewApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>new</em> <code>ApplicationId</code> allocated by the 
+ <code>ResourceManager</code>.
+ @return <em>new</em> <code>ApplicationId</code> allocated by the 
+          <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for 
+ a request to get a new {@link ApplicationId} for submitting applications.</p>
+ 
+ <p>Clients can submit an application with the returned
+ {@link ApplicationId}.</p>
+
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <class name="GetNewReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@code ReservationId} for
+ submitting an reservation.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <class name="GetNewReservationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a new {@link ReservationId} to be used to submit a reservation.
+
+ @return a {@link ReservationId} representing the unique id to identify
+ a reservation with which it was submitted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for
+ a request to get a new {@link ReservationId} for submitting reservations.</p>
+
+ <p>Clients can submit an reservation with the returned
+ {@link ReservationId}.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <class name="GetNodesToAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+    </method>
+    <method name="setHostNames"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostnames" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set hostnames for which mapping is required.
+
+ @param hostnames]]>
+      </doc>
+    </method>
+    <method name="getHostNames" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames for which mapping is required.
+
+ @return Set of hostnames.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get nodes to attributes mapping
+ in the cluster from the <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <class name="GetNodesToAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setNodeToAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames to NodeAttributes mapping.
+
+ @return Map of host to attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ nodes to attributes mapping.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <class name="GetQueueInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <param name="includeApplications" type="boolean"/>
+      <param name="includeChildQueues" type="boolean"/>
+      <param name="recursive" type="boolean"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> for which to get queue information.
+ @return <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue name</em> for which to get queue information
+ @param queueName <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="getIncludeApplications" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>active applications</em> required?
+ @return <code>true</code> if applications' information is to be included,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeApplications"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeApplications" type="boolean"/>
+      <doc>
+      <![CDATA[Should we get fetch information about <em>active applications</em>?
+ @param includeApplications fetch information about <em>active 
+                            applications</em>?]]>
+      </doc>
+    </method>
+    <method name="getIncludeChildQueues" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>child queues</em> required?
+ @return <code>true</code> if information about child queues is required,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeChildQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeChildQueues" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information about <em>child queues</em>?
+ @param includeChildQueues fetch information about <em>child queues</em>?]]>
+      </doc>
+    </method>
+    <method name="getRecursive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information on the entire <em>child queue hierarchy</em> required?
+ @return <code>true</code> if information about entire hierarchy is 
+         required, <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setRecursive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information on the entire <em>child queue hierarchy</em>?
+ @param recursive fetch information on the entire <em>child queue 
+                  hierarchy</em>?]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get <em>queue information</em>
+ from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <class name="GetQueueInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueInfo</code> for the specified queue.
+ @return <code>QueueInfo</code> for the specified queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a client
+ requesting information about queues in the system.
+ <p>
+ The response includes a {@link QueueInfo} which has details such as
+ queue name, used/total capacities, running applications, child queues etc.
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <class name="GetQueueUserAclsInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to the <code>ResourceManager</code> to 
+ get queue acls for the <em>current user</em>.</p>
+
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <class name="GetQueueUserAclsInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUserAclsInfoList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueUserACLInfo</code> per queue for the user.
+ @return <code>QueueUserACLInfo</code> per queue for the user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to clients
+ seeking queue acls for the user.</p>
+
+ <p>The response contains a list of {@link QueueUserACLInfo} which
+ provides information about {@link QueueACL} per queue.</p>
+ 
+ @see QueueACL
+ @see QueueUserACLInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <class name="GetResourceProfileRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="setProfileName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profileName" type="java.lang.String"/>
+    </method>
+    <method name="getProfileName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <class name="GetResourceProfileResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resources that will be allocated if the profile was used.
+
+ @return the resources that will be allocated if the profile was used.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resources that will be allocated if the profile is used.
+
+ @param r Set the resources that will be allocated if the profile is used.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <class name="IncreaseContainersResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToIncrease" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource increase.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource increase.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToIncrease"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getIncreasedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource increase.
+ @param containersToIncrease the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <class name="IncreaseContainersResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyIncreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully increased.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully increased.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to increase container resource.
+ </p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <class name="KillApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be aborted.
+ @return <code>ApplicationId</code> of the application to be aborted]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostics</em> to which the application is being killed.
+ @return <em>diagnostics</em> to which the application is being killed]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>diagnostics</em> to which the application is being killed.
+ @param diagnostics <em>diagnostics</em> to which the application is being
+          killed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to abort a submitted application.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ aborted.</p>
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <class name="KillApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsKillCompleted" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the process of killing application is completed or not.
+ @return true if the process of killing application has completed,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to the client aborting
+ a submitted application.
+ <p>
+ The response, includes:
+ <ul>
+   <li>
+     A flag which indicates that the process of killing the application is
+     completed or not.
+   </li>
+ </ul>
+ Note: user is recommended to wait until this flag becomes true, otherwise if
+ the <code>ResourceManager</code> crashes before the process of killing the
+ application is completed, the <code>ResourceManager</code> may retry this
+ application on recovery.
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <class name="MoveApplicationAcrossQueuesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be moved.
+ @return <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application to be moved.
+ @param appId <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="getTargetQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @return the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <method name="setTargetQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @param queue the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to move a submitted application to a different queue.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ moved and the queue to place it in.</p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <class name="MoveApplicationAcrossQueuesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client moving
+ a submitted application to a different queue.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <class name="RegisterApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of <code>RegisterApplicationMasterRequest</code>.
+ If <em>port, trackingUrl</em> is not used, use the following default value:
+ <ul>
+  <li>port: -1</li>
+  <li>trackingUrl: null</li>
+ </ul>
+ The port is allowed to be any integer larger than or equal to -1.
+ @return the new instance of <code>RegisterApplicationMasterRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @param host <em>host</em> on which the <code>ApplicationMaster</code> 
+             is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @return the <em>RPC port</em> on which the {@code ApplicationMaster}
+         is responding]]>
+      </doc>
+    </method>
+    <method name="setRpcPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @param port <em>RPC port</em> on which the {@code ApplicationMaster}
+             is responding]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em> for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ it is running. This is the web-URL to which ResourceManager or
+ web-application proxy will redirect client/users while the application and
+ the <code>ApplicationMaster</code> are still running.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param trackingUrl
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getPlacementConstraints" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all Placement Constraints specified at the Application level. The
+ mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags, i.e., each
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those
+ tags will be placed taking into account the corresponding constraint.
+
+ @return A map of Placement Constraints.]]>
+      </doc>
+    </method>
+    <method name="setPlacementConstraints"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraints" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set Placement Constraints applicable to the
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+ of this application.
+ The mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags.
+ For example:
+  Map &lt;
+   &lt;hb_regionserver&gt; -&gt; node_anti_affinity,
+   &lt;hb_regionserver, hb_master&gt; -&gt; rack_affinity,
+   ...
+  &gt;
+ @param placementConstraints Placement Constraint Mapping.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
+ on registration.
+ <p>
+ The registration includes details such as:
+ <ul>
+   <li>Hostname on which the AM is running.</li>
+   <li>RPC Port</li>
+   <li>Tracking URL</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <class name="RegisterApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="getClientToAMTokenMasterKey" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get ClientToAMToken master key.</p>
+ <p>The ClientToAMToken master key is sent to <code>ApplicationMaster</code>
+ by <code>ResourceManager</code> via {@link RegisterApplicationMasterResponse}
+ , used to verify corresponding ClientToAMToken.</p>
+ @return ClientToAMToken master key]]>
+      </doc>
+    </method>
+    <method name="setClientToAMTokenMasterKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set ClientToAMToken master key.]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the queue that the application was placed in.<p>
+ @return the queue that the application was placed in.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>Set the queue that the application was placed in.<p>]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts.
+ </p>
+ 
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts
+ @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getNMTokensFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens for communicating with the NMs where the
+ containers of previous application attempts are running.
+ 
+ @return the list of NMTokens for communicating with the NMs where the
+         containers of previous application attempts are running.
+ 
+ @see RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getSchedulerResourceTypes" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a set of the resource types considered by the scheduler.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of supported resource profiles from RM.
+
+ @return a map of resource profiles and its capabilities.]]>
+      </doc>
+    </method>
+    <method name="getResourceTypes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get available resource types supported by RM.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a new
+ {@code ApplicationMaster} on registration.
+ <p>
+ The response contains critical details such as:
+ <ul>
+   <li>Maximum capability for allocated resources in the cluster.</li>
+   <li>{@code ApplicationACL}s for the application.</li>
+   <li>ClientToAMToken master key.</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <class name="ReInitializeContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[Creates a new instance of the ReInitializationContainerRequest.
+ @param containerId Container Id.
+ @param containerLaunchContext Container Launch Context.
+ @param autoCommit AutoCommit.
+ @return ReInitializationContainerRequest.]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to re-initialize.
+
+ @return <code>ContainerId</code> of the container to re-initialize.]]>
+      </doc>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to re-initialize the container
+ with.
+
+ @return <code>ContainerLaunchContext</code> of to re-initialize the
+ container with.]]>
+      </doc>
+    </method>
+    <method name="getAutoCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if AutoCommit is set for this ReInitialization.
+ @return If AutoCommit is set for this ReInitialization.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates all the required fields needed for a Container
+ ReInitialization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <class name="ReInitializeContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ReInitializeContainerRequest}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <class name="ReleaseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be released.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be released.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be released.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be released.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to release a resource in the shared cache.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <class name="ReleaseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response to clients from the <code>SharedCacheManager</code> when
+ releasing a resource in the shared cache.
+ </p>
+
+ <p>
+ Currently, this is empty.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <class name="ReservationDeleteRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDeleteRequest} captures the set of requirements the user
+ has to delete an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <class name="ReservationDeleteResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationDeleteResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation delete
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <class name="ReservationListRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+ @param endTime Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via  {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name to use to find reservations.
+
+ @return the queue name to use to find reservations.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name to use to find resource allocations.
+
+ @param queue Required. Cannot be null or empty.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id to use to find a reservation.
+
+ @return the reservation id of the reservation.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reservation id to use to find a reservation.
+
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @return the start time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setEndTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+      <doc>
+      <![CDATA[Set the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @param endTime Optional. If provided, only reservations that
+                start before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getIncludeResourceAllocations" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setIncludeResourceAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[Set the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire list of
+                {@code ResourceAllocationRequest} will be returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListRequest} captures the set of requirements the
+ user has to list reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <class name="ReservationListResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationAllocationState" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationAllocationState}, that corresponds
+ to a reservation in the scheduler.
+
+ @return the list of {@link ReservationAllocationState} which holds
+ information of a particular reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListResponse} captures the list of reservations that the
+ user has queried.
+
+ The resulting list of {@link ReservationAllocationState} contains a list of
+ {@code ResourceAllocationRequest} representing the current state of the
+ reservation resource allocations will be returned. This is subject to change
+ in the event of re-planning a described by {@code ReservationDefinition}
+
+ @see ReservationAllocationState]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <class name="ReservationSubmissionRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to.
+ 
+ @return the name of the {@code Plan} that corresponds to the name of the
+         {@link QueueInfo} in the scheduler to which the reservation will be
+         submitted to]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to
+ 
+ @param queueName the name of the parent {@code Plan} that corresponds to
+          the name of the {@link QueueInfo} in the scheduler to which the
+          reservation will be submitted to]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id that corresponds to the reservation submission.
+
+ @return reservation id that will be used to identify the reservation
+ submission.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id that corresponds to the reservation submission.
+
+ @param reservationId reservation id that will be used to identify the
+                      reservation submission.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationSubmissionRequest} captures the set of requirements the
+ user has to create a reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <class name="ReservationSubmissionResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ reservation submission.</p>
+
+ <p>Currently, this is empty.</p>
+
+ {@code ApplicationClientProtocol#submitReservation(
+ ReservationSubmissionRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <class name="ReservationUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationUpdateRequest} captures the set of requirements the user
+ has to update an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <class name="ReservationUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationUpdateResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation update
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <class name="ResourceLocalizationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="localResources" type="java.util.Map"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to localize resources.
+
+ @return <code>ContainerId</code> of the container to localize resources.]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the ApplicationMaster to ask for localizing resources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <class name="ResourceLocalizationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ResourceLocalizationRequest}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <class name="RestartContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RestartContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to a restart Container request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <class name="RollbackResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RollbackResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to a Rollback request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <class name="SignalContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="signalContainerCommand" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to signal.
+ @return <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="getCommand" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>SignalContainerCommand</code> of the signal request.
+ @return <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <method name="setCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <doc>
+      <![CDATA[Set the <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ or by the <code>ApplicationMaster</code> to the <code>NodeManager</code>
+ to signal a container.
+ @see SignalContainerCommand </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <class name="SignalContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ signalling a container.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <class name="StartContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>.
+ 
+ @return <code>ContainerLaunchContext</code> for the container to be started
+         by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>
+ @param context <code>ContainerLaunchContext</code> for the container to be 
+                started by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the container token to be used for authorization during starting
+ container.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the container token to be used for authorization during starting
+ container.
+ @see NMToken
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="setContainerToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>start</em> a container.</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as
+ allocated resource capability, security tokens (if enabled), command
+ to be executed to start the container, environment for the process, 
+ necessary binaries/jar/shared-objects etc. via the 
+ {@link ContainerLaunchContext}.</p>
+
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <class name="StartContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+    </method>
+    <method name="getStartContainerRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link StartContainerRequest} to start containers.
+ @return a list of {@link StartContainerRequest} to start containers.]]>
+      </doc>
+    </method>
+    <method name="setStartContainerRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of {@link StartContainerRequest} to start containers.
+ @param request a list of {@link StartContainerRequest} to start containers]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request which contains a list of {@link StartContainerRequest} sent by
+ the <code>ApplicationMaster</code> to the <code>NodeManager</code> to
+ <em>start</em> containers.
+ </p>
+ 
+ <p>
+ In each {@link StartContainerRequest}, the <code>ApplicationMaster</code> has
+ to provide details such as allocated resource capability, security tokens (if
+ enabled), command to be executed to start the container, environment for the
+ process, necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <class name="StartContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStartedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> s of the containers that are
+ started successfully.
+ 
+ @return the list of <code>ContainerId</code> s of the containers that are
+         started successfully.
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <method name="getAllServicesMetaData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the meta-data from all auxiliary services running on the
+ <code>NodeManager</code>.
+ </p>
+ <p>
+ The meta-data is returned as a Map between the auxiliary service names and
+ their corresponding per service meta-data as an opaque blob
+ <code>ByteBuffer</code>
+ </p>
+ 
+ <p>
+ To be able to interpret the per-service meta-data, you should consult the
+ documentation for the Auxiliary-service configured on the NodeManager
+ </p>
+ 
+ @return a Map between the names of auxiliary services and their
+         corresponding meta-data]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>start</em> an allocated
+ container.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <class name="StopContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code>s of the containers to be stopped.
+ @return <code>ContainerId</code>s of containers to be stopped]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code>s of the containers to be stopped.
+ @param containerIds <code>ContainerId</code>s of the containers to be stopped]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>stop</em> containers.</p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <class name="StopContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStoppedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of successfully stopped containers.
+ 
+ @return the list of containerIds of successfully stopped containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>stop</em> allocated
+ containers.
+ </p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <class name="SubmitApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationSubmissionContext</code> for the application.
+ @return <code>ApplicationSubmissionContext</code> for the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationSubmissionContext</code> for the application.
+ @param context <code>ApplicationSubmissionContext</code> for the 
+                application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to <em>submit an application</em> to the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>The request, via {@link ApplicationSubmissionContext}, contains
+ details such as queue, {@link Resource} required to run the 
+ <code>ApplicationMaster</code>, the equivalent of 
+ {@link ContainerLaunchContext} for launching the 
+ <code>ApplicationMaster</code> etc.
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <class name="SubmitApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ application submission.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <class name="UpdateApplicationPriorityRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ 
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ 
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ 
+ @return <code>Priority</code> of the application to be set.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+ 
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application priority.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and
+ {@link Priority} to be set for an application
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <class name="UpdateApplicationPriorityResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ @return Updated <code>Priority</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ the application priority.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <class name="UpdateApplicationTimeoutsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationTimeouts" type="java.util.Map"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application timeout.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and timeout
+ to be set for an application
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <class name="UpdateApplicationTimeoutsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ application timeout.
+ </p>
+ <p>
+ A response without exception means that the update has completed
+ successfully.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <class name="UseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be used.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be used.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be used.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be used.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to the <code>SharedCacheManager</code> that claims a
+ resource in the shared cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+  <class name="UseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Path</code> corresponding to the requested resource in the
+ shared cache.
+
+ @return String A <code>Path</code> if the resource exists in the shared
+         cache, <code>null</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setPath"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>Path</code> corresponding to a resource in the shared cache.
+
+ @param p A <code>Path</code> corresponding to a resource in the shared
+          cache]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response from the SharedCacheManager to the client that indicates whether
+ a requested resource exists in the cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records">
+  <!-- start class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <class name="AMCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.AMCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Command sent by the Resource Manager to the Application Master in the 
+ AllocateResponse 
+ @see AllocateResponse]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <class name="ApplicationAccessType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application access types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <class name="ApplicationAttemptId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationAttemptId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="attemptId" type="int"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the <code>ApplicationAttempId</code>. 
+ @return <code>ApplicationId</code> of the <code>ApplicationAttempId</code>]]>
+      </doc>
+    </method>
+    <method name="getAttemptId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>attempt id</code> of the <code>Application</code>.
+ @return <code>attempt id</code> of the <code>Application</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <field name="appAttemptIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
+ of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
+ 
+ <p>Multiple attempts might be needed to run an application to completion due
+ to temporal failures of the <code>ApplicationMaster</code> such as hardware
+ failures, connectivity issues etc. on the node on which it was scheduled.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <class name="ApplicationAttemptReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="rpcPort" type="int"/>
+      <param name="url" type="java.lang.String"/>
+      <param name="oUrl" type="java.lang.String"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"/>
+      <param name="amContainerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getYarnApplicationAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>YarnApplicationAttemptState</em> of the application attempt.
+ 
+ @return <em>YarnApplicationAttemptState</em> of the application attempt]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of this attempt <code>ApplicationMaster</code>.
+ 
+ @return <em>RPC port</em> of this attempt <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which this attempt of
+ <code>ApplicationMaster</code> is running.
+ 
+ @return <em>host</em> on which this attempt of
+         <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnositic information</em> of the application attempt in case
+ of errors.
+ 
+ @return <em>diagnositic information</em> of the application attempt in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application attempt.
+ 
+ @return <em>tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>original tracking url</em> for the application attempt.
+ 
+ @return <em>original tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of this attempt of the
+ application
+ 
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getAMContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of AMContainer for this attempt
+ 
+ @return <code>ContainerId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ 
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationAttemptReport} is a report of an application attempt.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationAttemptId} of the application.</li>
+   <li>Host on which the <code>ApplicationMaster</code> of this attempt is
+   running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code> of this attempt.</li>
+   <li>Tracking URL.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>{@link YarnApplicationAttemptState} of the application attempt.</li>
+   <li>{@link ContainerId} of the master Container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <class name="ApplicationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clusterTimestamp" type="long"/>
+      <param name="id" type="int"/>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the short integer identifier of the <code>ApplicationId</code>
+ which is unique for all applications started by a particular instance
+ of the <code>ResourceManager</code>.
+ @return short integer identifier of the <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the <code>ResourceManager</code> which is 
+ used to generate globally unique <code>ApplicationId</code>.
+ @return <em>start time</em> of the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="appIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationId</code> represents the <em>globally unique</em> 
+ identifier for an application.</p>
+ 
+ <p>The globally unique nature of the identifier is achieved by using the 
+ <em>cluster timestamp</em> i.e. start-time of the 
+ <code>ResourceManager</code> along with a monotonically increasing counter
+ for the application.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <class name="ApplicationReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getCurrentApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the current
+ attempt of the application
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>user</em> who submitted the application.
+ @return <em>user</em> who submitted the application]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application was submitted.
+ @return <em>queue</em> to which the application was submitted]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <em>name</em> of the application.
+ @return <em>name</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ is running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code>
+         is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of the <code>ApplicationMaster</code>.
+ @return <em>RPC port</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getClientToAMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>.
+ <p>
+ <em>ClientToAMToken</em> is the security token used by the AMs to verify
+ authenticity of any <code>client</code>.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code>, provides a secure token (via
+ {@link ApplicationReport#getClientToAMToken()}) which is verified by the
+ ApplicationMaster when the client directly talks to an AM.
+ </p>
+ @return <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getYarnApplicationState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnApplicationState</code> of the application.
+ @return <code>YarnApplicationState</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get  the <em>diagnositic information</em> of the application in case of
+ errors.
+ @return <em>diagnositic information</em> of the application in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application.
+ @return <em>tracking url</em> for the application]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the application.
+ @return <em>start time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getSubmitTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>final finish status</em> of the application.
+ @return <em>final finish status</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationResourceUsageReport" return="org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve the structure containing the job resources for this application
+ @return the job resources structure for this application]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's progress ( range 0.0 to 1.0 )
+ @return application's progress]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's Type 
+ @return application's Type]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all tags corresponding to the application
+ @return Application's tags]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+ 
+ @return the AM to RM token if available.]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationStatus" return="org.apache.hadoop.yarn.api.records.LogAggregationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get log aggregation status for the application
+ @return Application's log aggregation status]]>
+      </doc>
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedApp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="unmanagedApplication" type="boolean"/>
+      <doc>
+      <![CDATA[@param unmanagedApplication true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get priority of the application
+
+ @return Application's priority]]>
+      </doc>
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAppNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAmNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationReport} is a report of an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Applications user.</li>
+   <li>Application queue.</li>
+   <li>Application name.</li>
+   <li>Host on which the <code>ApplicationMaster</code> is running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code>.</li>
+   <li>Tracking URL.</li>
+   <li>{@link YarnApplicationState} of the application.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Start time of the application.</li>
+   <li>Client {@link Token} of the application (if security is enabled).</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <class name="ApplicationResourceUsageReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationResourceUsageReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumUsedContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of used containers.  -1 for invalid/inaccessible reports.
+ @return the number of used containers]]>
+      </doc>
+    </method>
+    <method name="getUsedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the used <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the used <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getReservedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the reserved <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getNeededResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the needed <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the needed <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory (in megabytes) the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores that the application has allocated
+ times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getQueueUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the queue that the app is using.
+ @return the percentage of resources of the queue that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getClusterUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the cluster that the app is using.
+ @return the percentage of resources of the cluster that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getPreemptedMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory preempted(in megabytes)
+ the application has allocated times the number of
+ seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated resource-seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated preempted
+ resource-seconds]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Contains various scheduling metrics to be reported by UI and CLI.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <class name="ApplicationSubmissionContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="amContainerLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="resourceRequest" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="attemptFailuresValidityInterval" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the submitted application.
+ @return <code>ApplicationId</code> of the submitted application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the submitted application.
+ @param applicationId <code>ApplicationId</code> of the submitted
+                      application]]>
+      </doc>
+    </method>
+    <method name="getApplicationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application <em>name</em>.
+ @return application name]]>
+      </doc>
+    </method>
+    <method name="setApplicationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application <em>name</em>.
+ @param applicationName application name]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application is being submitted.
+ @return <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue</em> to which the application is being submitted
+ @param queue <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application.
+ @return <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getAMContainerSpec" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @return <code>ContainerLaunchContext</code> for the 
+         <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerSpec"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @param amContainer <code>ContainerLaunchContext</code> for the 
+                    <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="getUnmanagedAM" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get if the RM should manage the execution of the AM. 
+ If true, then the RM 
+ will not allocate a container for the AM and start it. It will expect the 
+ AM to be launched and connect to the RM within the AM liveliness period and 
+ fail the app otherwise. The client should launch the AM only after the RM 
+ has ACCEPTED the application and changed the <code>YarnApplicationState</code>.
+ Such apps will not be retried by the RM on app attempt failure.
+ The default value is false.
+ @return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedAM"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[@param value true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getMaxAppAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of max attempts of the application to be submitted]]>
+      </doc>
+    </method>
+    <method name="setMaxAppAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxAppAttempts" type="int"/>
+      <doc>
+      <![CDATA[Set the number of max attempts of the application to be submitted. WARNING:
+ it should be no larger than the global number of max attempts in the Yarn
+ configuration.
+ @param maxAppAttempts the number of max attempts of the application
+ to be submitted.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource required by the <code>ApplicationMaster</code> for this
+ application. Please note this will be DEPRECATED, use <em>getResource</em>
+ in <em>getAMContainerResourceRequest</em> instead.
+ 
+ @return the resource required by the <code>ApplicationMaster</code> for
+         this application.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resource required by the <code>ApplicationMaster</code> for this
+ application.
+
+ @param resource the resource required by the <code>ApplicationMaster</code>
+ for this application.]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application type
+ 
+ @return the application type]]>
+      </doc>
+    </method>
+    <method name="setApplicationType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application type
+ 
+ @param applicationType the application type]]>
+      </doc>
+    </method>
+    <method name="getKeepContainersAcrossApplicationAttempts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates whether to keep containers across application
+ attempts or not.
+ 
+ @return the flag which indicates whether to keep containers across
+         application attempts or not.]]>
+      </doc>
+    </method>
+    <method name="setKeepContainersAcrossApplicationAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keepContainers" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which indicates whether to keep containers across application
+ attempts.
+ <p>
+ For managed AM, if the flag is true, running containers will not be killed
+ when application attempt fails and these containers will be retrieved by
+ the new application attempt on registration via
+ {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
+ </p>
+ <p>
+ For unmanaged AM, if the flag is true, RM allows re-register and returns
+ the running containers in the same attempt back to the UAM for HA.
+ </p>
+
+ @param keepContainers the flag which indicates whether to keep containers
+          across application attempts.]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get tags for the application
+
+ @return the application tags]]>
+      </doc>
+    </method>
+    <method name="setApplicationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set tags for the application. A maximum of
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
+ per application. Each tag can be at most
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
+ characters, and can contain only ASCII characters.
+
+ @param tags tags to set]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this app. If this is set, all containers of
+ this application without setting node-label-expression in ResurceRequest
+ will get allocated resources on only those nodes that satisfy this
+ node-label-expression.
+ 
+ If different node-label-expression of this app and ResourceRequest are set
+ at the same time, the one set in ResourceRequest will be used when
+ allocating container
+ 
+ @return node-label-expression for this app]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node-label-expression for this app
+ @param nodeLabelExpression node-label-expression of this app]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #getAMContainerResourceRequests()}">
+      <doc>
+      <![CDATA[Get the ResourceRequest of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will assemble a ResourceRequest by using
+ <em>getResource</em> and <em>getPriority</em> of
+ <em>ApplicationSubmissionContext</em>.
+
+ Number of containers and Priority will be ignored.
+
+ @return ResourceRequest of the AM container
+ @deprecated See {@link #getAMContainerResourceRequests()}]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #setAMContainerResourceRequests(List)}">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Set ResourceRequest of the AM container
+ @param request of the AM container
+ @deprecated See {@link #setAMContainerResourceRequests(List)}]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ResourceRequests of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will use the ResourceRequest as determined by
+ <em>getAMContainerResourceRequest</em> and its behavior.
+
+ Number of containers and Priority will be ignored.
+
+ @return List of ResourceRequests of the AM container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set ResourceRequests of the AM container.
+ @param requests of the AM container]]>
+      </doc>
+    </method>
+    <method name="getAttemptFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attemptFailuresValidityInterval in milliseconds for the application
+
+ @return the attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="setAttemptFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptFailuresValidityInterval" type="long"/>
+      <doc>
+      <![CDATA[Set the attemptFailuresValidityInterval in milliseconds for the application
+ @param attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LogAggregationContext</code> of the application
+
+ @return <code>LogAggregationContext</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+      <doc>
+      <![CDATA[Set <code>LogAggregationContext</code> for the application
+
+ @param logAggregationContext
+          for the application]]>
+      </doc>
+    </method>
+    <method name="getReservationID" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id, that corresponds to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @return the reservation id representing the unique id of the corresponding
+         reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationID" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id, that correspond to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @param reservationID representing the unique id of the
+          corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in seconds.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application in seconds.
+ All pre-existing Map entries are cleared before adding the new Map.
+ <p>
+ <b>Note:</b> If application timeout value is less than or equal to zero
+ then application submission will throw an exception.
+ </p>
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <method name="getApplicationSchedulingPropertiesMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get application scheduling environment variables stored as a key value
+ pair map for application.
+
+ @return placement envs for application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationSchedulingPropertiesMap"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingEnvMap" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the scheduling envs for the application.
+
+ @param schedulingEnvMap
+          A map of env's for the application scheduling preferences.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationSubmissionContext} represents all of the
+ information needed by the {@code ResourceManager} to launch
+ the {@code ApplicationMaster} for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Application user.</li>
+   <li>Application name.</li>
+   <li>{@link Priority} of the application.</li>
+   <li>
+     {@link ContainerLaunchContext} of the container in which the
+     <code>ApplicationMaster</code> is executed.
+   </li>
+   <li>
+     maxAppAttempts. The maximum number of application attempts.
+     It should be no larger than the global number of max attempts in the
+     YARN configuration.
+   </li>
+   <li>
+     attemptFailuresValidityInterval. The default value is -1.
+     when attemptFailuresValidityInterval in milliseconds is set to
+     {@literal >} 0, the failure number will no take failures which happen
+     out of the validityInterval into failure count. If failure count
+     reaches to maxAppAttempts, the application will be failed.
+   </li>
+   <li>Optional, application-specific {@link LogAggregationContext}</li>
+ </ul>
+ 
+ @see ContainerLaunchContext
+ @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <class name="ApplicationTimeout" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTimeout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <param name="expiryTime" type="java.lang.String"/>
+      <param name="remainingTime" type="long"/>
+    </method>
+    <method name="getTimeoutType" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application timeout type.
+ @return timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="setTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeoutType" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <doc>
+      <![CDATA[Set the application timeout type.
+ @param timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="getExpiryTime" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>expiryTime</code> for given timeout type.
+ @return expiryTime in ISO8601 standard with format
+         <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="setExpiryTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expiryTime" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <code>expiryTime</code> for given timeout type.
+ @param expiryTime in ISO8601 standard with format
+          <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="getRemainingTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>Remaining Time</code> of an application for given timeout type.
+ @return Remaining Time in seconds.]]>
+      </doc>
+    </method>
+    <method name="setRemainingTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="remainingTime" type="long"/>
+      <doc>
+      <![CDATA[Set <code>Remaining Time</code> of an application for given timeout type.
+ @param remainingTime in seconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationTimeout} is a report for configured application timeouts.
+ It includes details such as:
+ <ul>
+ <li>{@link ApplicationTimeoutType} of the timeout type.</li>
+ <li>Expiry time in ISO8601 standard with format
+ <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b> or "UNLIMITED".</li>
+ <li>Remaining time in seconds.</li>
+ </ul>
+ The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ <ul>
+ <li>{UNLIMITED,-1} : Timeout is not configured for given timeout type
+ (LIFETIME).</li>
+ <li>{ISO8601 date string, 0} : Timeout is configured and application has
+ completed.</li>
+ <li>{ISO8601 date string, greater than zero} : Timeout is configured and
+ application is RUNNING. Application will be timed out after configured
+ value.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <class name="ApplicationTimeoutType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application timeout type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <class name="CollectorInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CollectorInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getCollectorAddr" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCollectorAddr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.lang.String"/>
+    </method>
+    <method name="getCollectorToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get delegation token for app collector which AM will use to publish
+ entities.
+ @return the delegation token for app collector.]]>
+      </doc>
+    </method>
+    <method name="setCollectorToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <field name="DEFAULT_TIMESTAMP_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Collector info containing collector address and collector token passed from
+ RM to AM in Allocate Response.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Container -->
+  <class name="Container" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Container"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the globally unique identifier for the container.
+ @return globally unique identifier for the container]]>
+      </doc>
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the node on which the container is allocated.
+ @return identifier of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the http uri of the node on which the container is allocated.
+ @return http uri of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exposed ports of the node on which the container is allocated.
+ @return exposed ports of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> at which the <code>Container</code> was
+ allocated.
+ @return <code>Priority</code> at which the <code>Container</code> was
+         allocated]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerToken</code> for the container.
+ <p><code>ContainerToken</code> is the security token used by the framework
+ to verify authenticity of any <code>Container</code>.</p>
+
+ <p>The <code>ResourceManager</code>, on container allocation provides a
+ secure token which is verified by the <code>NodeManager</code> on
+ container launch.</p>
+
+ <p>Applications do not need to care about <code>ContainerToken</code>, they
+ are transparently handled by the framework - the allocated
+ <code>Container</code> includes the <code>ContainerToken</code>.</p>
+
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+
+ @return <code>ContainerToken</code> for the container]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to the original {@code
+ ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+ this allocated {@code Container}.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}s.
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+
+ @return the <em>ID</em> corresponding to the original  allocation request
+ which is satisfied by this allocation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code Container} represents an allocated resource in the cluster.
+ <p>
+ The {@code ResourceManager} is the sole authority to allocate any
+ {@code Container} to applications. The allocated {@code Container}
+ is always on a single node and has a unique {@link ContainerId}. It has
+ a specific amount of {@link Resource} allocated.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} for the container, which is globally unique.</li>
+   <li>
+     {@link NodeId} of the node on which it is allocated.
+   </li>
+   <li>HTTP uri of the node.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>{@link Priority} at which the container was allocated.</li>
+   <li>
+     Container {@link Token} of the container, used to securely verify
+     authenticity of the allocation.
+   </li>
+ </ul>
+ 
+ Typically, an {@code ApplicationMaster} receives the {@code Container}
+ from the {@code ResourceManager} during resource-negotiation and then
+ talks to the {@code NodeManager} to start/stop containers.
+ 
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Container -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <class name="ContainerExitStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerExitStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ABORTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc.]]>
+      </doc>
+    </field>
+    <field name="DISKS_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad.]]>
+      </doc>
+    </field>
+    <field name="PREEMPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers preempted by the framework.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_VMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated virtual memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_PMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated physical memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_APPMASTER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by stop request by the app master.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_RESOURCEMANAGER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the resource manager.]]>
+      </doc>
+    </field>
+    <field name="KILLED_AFTER_APP_COMPLETION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated after the application finished.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_CONTAINER_SCHEDULER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the ContainerScheduler to make room
+ for another container...]]>
+      </doc>
+    </field>
+    <field name="KILLED_FOR_EXCESS_LOGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated for generating excess log data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Container exit statuses indicating special exit circumstances.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <class name="ContainerId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerId" type="long"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the application to which the
+ <code>Container</code> was assigned.
+ <p>
+ Note: If containers are kept alive across application attempts via
+ {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ the <code>ContainerId</code> does not necessarily contain the current
+ running application attempt's <code>ApplicationAttemptId</code> This
+ container can be allocated by previously exited application attempt and
+ managed by the current running attempt thus have the previous application
+ attempt's <code>ApplicationAttemptId</code>.
+ </p>
+ 
+ @return <code>ApplicationAttemptId</code> of the application to which the
+         <code>Container</code> was assigned]]>
+      </doc>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower 32 bits of identifier of the <code>ContainerId</code>,
+ which doesn't include epoch. Note that this method will be marked as
+ deprecated, so please use <code>getContainerId</code> instead.
+ @return lower 32 bits of identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the <code>ContainerId</code>. Upper 24 bits are
+ reserved as epoch of cluster, and lower 40 bits are reserved as
+ sequential number of containers.
+ @return identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return A string representation of containerId. The format is
+ container_e*epoch*_*clusterTimestamp*_*appId*_*attemptId*_*containerId*
+ when epoch is larger than 0
+ (e.g. container_e17_1410901177871_0001_01_000005).
+ *epoch* is increased when RM restarts or fails over.
+ When epoch is 0, epoch is omitted
+ (e.g. container_1410901177871_0001_01_000005).]]>
+      </doc>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="CONTAINER_ID_BITMASK" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ContainerId</code> represents a globally unique identifier
+ for a {@link Container} in the cluster.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <class name="ContainerLaunchContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLaunchContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+    </method>
+    <method name="getTokens" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the tokens needed by this container. It may include file-system
+ tokens, ApplicationMaster related tokens if this container is an
+ ApplicationMaster or framework level tokens needed by this container to
+ communicate to various services in a secure manner.
+ 
+ @return tokens needed by this container.]]>
+      </doc>
+    </method>
+    <method name="setTokens"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set security tokens needed by this container.
+ @param tokens security tokens]]>
+      </doc>
+    </method>
+    <method name="getTokensConf" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration used by RM to renew tokens.
+ @return The configuration used by RM to renew the tokens.]]>
+      </doc>
+    </method>
+    <method name="setTokensConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokensConf" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set the configuration used by RM to renew the tokens.
+ @param tokensConf The configuration used by RM to renew the tokens]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="setLocalResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set <code>LocalResource</code> required by the container. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param localResources <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="getServiceData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}.
+ </p>
+ 
+ <p>
+ This will be used to initialize this application on the specific
+ {@link AuxiliaryService} running on the NodeManager by calling
+ {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)}
+ </p>
+ 
+ @return application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="setServiceData"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceData" type="java.util.Map"/>
+      <doc>
+      <![CDATA[<p>
+ Set application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}. All pre-existing Map entries are
+ preserved.
+ </p>
+ 
+ @param serviceData
+          application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>environment variables</em> for the container.
+ @return <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="setEnvironment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="environment" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add <em>environment variables</em> for the container. All pre-existing Map
+ entries are cleared before adding the new Map
+ @param environment <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getCommands" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>commands</em> for launching the container.
+ @return the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="setCommands"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add the list of <em>commands</em> for launching the container. All
+ pre-existing List entries are cleared before adding the new List
+ @param commands the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="setApplicationACLs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationACL</code>s for the application. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param acls <code>ApplicationACL</code>s for the application]]>
+      </doc>
+    </method>
+    <method name="getContainerRetryContext" return="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerRetryContext</code> to relaunch container.
+ @return <code>ContainerRetryContext</code> to relaunch container.]]>
+      </doc>
+    </method>
+    <method name="setContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerRetryContext</code> to relaunch container.
+ @param containerRetryContext <code>ContainerRetryContext</code> to
+                              relaunch container.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerLaunchContext} represents all of the information
+ needed by the {@code NodeManager} to launch a container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>User to whom the container is allocated.</li>
+   <li>Security tokens (if security is enabled).</li>
+   <li>
+     {@link LocalResource} necessary for running the container such
+     as binaries, jar, shared-objects, side-files etc.
+   </li>
+   <li>Optional, application-specific binary service data.</li>
+   <li>Environment variables for the launched process.</li>
+   <li>Command to launch the container.</li>
+   <li>Retry strategy when container exits with failure.</li>
+ </ul>
+ 
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <class name="ContainerReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ 
+ @return <code>ContainerId</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getAllocatedResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Resource</code> of the container.
+ 
+ @return allocated <code>Resource</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getAssignedNode" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>NodeId</code> where container is running.
+ 
+ @return allocated <code>NodeId</code> where container is running.]]>
+      </doc>
+    </method>
+    <method name="setAssignedNode"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Priority</code> of the container.
+ 
+ @return allocated <code>Priority</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the creation time of the container.
+ 
+ @return creation time of the container]]>
+      </doc>
+    </method>
+    <method name="setCreationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="creationTime" type="long"/>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Finish time of the container.
+ 
+ @return Finish time of the container]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the DiagnosticsInfo of the container.
+ 
+ @return DiagnosticsInfo of the container]]>
+      </doc>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the LogURL of the container.
+ 
+ @return LogURL of the container]]>
+      </doc>
+    </method>
+    <method name="setLogUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logUrl" type="java.lang.String"/>
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>ContainerState</code> of the container.
+ 
+ @return final <code>ContainerState</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"/>
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>exit status</code> of the container.
+ 
+ @return final <code>exit status</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerExitStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerExitStatus" type="int"/>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exposed ports of the container.
+ 
+ @return the node exposed ports of the container]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Node Http address of the container.
+
+ @return the node http address of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the execution type of the container.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerReport} is a report of an container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>Allocated Resources to the container.</li>
+   <li>Assigned Node id.</li>
+   <li>Assigned Priority.</li>
+   <li>Creation Time.</li>
+   <li>Finish Time.</li>
+   <li>Container Exit Status.</li>
+   <li>{@link ContainerState} of the container.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Log URL.</li>
+   <li>nodeHttpAddress</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <class name="ContainerRetryContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerRetryContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRetryPolicy" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryPolicy" type="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"/>
+    </method>
+    <method name="getErrorCodes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setErrorCodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCodes" type="java.util.Set"/>
+    </method>
+    <method name="getMaxRetries" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxRetries"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxRetries" type="int"/>
+    </method>
+    <method name="getRetryInterval" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryInterval" type="int"/>
+    </method>
+    <method name="getFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failuresValidityInterval" type="long"/>
+    </method>
+    <field name="RETRY_FOREVER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RETRY_INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NEVER_RETRY_CONTEXT" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@code ContainerRetryContext} indicates how container retry after it fails
+ to run.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>
+     {@link ContainerRetryPolicy} :
+     - NEVER_RETRY(DEFAULT value): no matter what error code is when container
+       fails to run, just do not retry.
+     - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails
+       to run, just retry.
+     - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if
+       the error code is one of <em>errorCodes</em>, otherwise do not retry.
+
+     Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry
+     because it is usually killed on purpose.
+   </li>
+   <li>
+     <em>maxRetries</em> specifies how many times to retry if need to retry.
+     If the value is -1, it means retry forever.
+   </li>
+   <li><em>retryInterval</em> specifies delaying some time before relaunch
+   container, the unit is millisecond.</li>
+   <li>
+     <em>failuresValidityInterval</em>: default value is -1.
+     When failuresValidityInterval in milliseconds is set to {@literal >} 0,
+     the failure number will not take failures which happen out of the
+     failuresValidityInterval into failure count. If failure count
+     reaches to <em>maxRetries</em>, the container will be failed.
+   </li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <class name="ContainerRetryPolicy" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>Retry policy for relaunching a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <class name="ContainerState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <class name="ContainerStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerState</code> of the container.
+ @return <code>ContainerState</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the <em>exit status</em> for the container.</p>
+  
+ <p>Note: This is valid only for completed containers i.e. containers
+ with state {@link ContainerState#COMPLETE}. 
+ Otherwise, it returns an ContainerExitStatus.INVALID.
+ </p>
+ 
+ <p>Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc. have a special
+ exit code of ContainerExitStatus.ABORTED.</p>
+ 
+ <p>When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad, then
+ container is not launched and is exited with ContainersExitStatus.DISKS_FAILED.
+ </p>
+  
+ @return <em>exit status</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic messages</em> for failed containers.
+ @return <em>diagnostic messages</em> for failed containers]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getIPs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the IP addresses with which the container run.
+ @return The IP address where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the hostname where the container runs.
+ @return The hostname where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exposed ports of the container.
+ @return List of exposed ports]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerStatus} represents the current status of a
+ {@code Container}.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>{@code ContainerId} of the container.</li>
+   <li>{@code ExecutionType} of the container.</li>
+   <li>{@code ContainerState} of the container.</li>
+   <li><em>Exit status</em> of a completed container.</li>
+   <li><em>Diagnostic</em> message for a failed container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <class name="ContainerSubState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerSubState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerSubState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container Sub-State.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <class name="ContainerUpdateType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Encodes the type of Container Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <class name="ExecutionType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ExecutionType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container property encoding execution semantics.
+
+ <p>
+ The execution types are the following:
+ <ul>
+   <li>{@link #GUARANTEED} - this container is guaranteed to start its
+   execution, once the corresponding start container request is received by
+   an NM.
+   <li>{@link #OPPORTUNISTIC} - the execution of this container may not start
+   immediately at the NM that receives the corresponding start container
+   request (depending on the NM's available resources). Moreover, it may be
+   preempted if it blocks a GUARANTEED container from being executed.
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <class name="ExecutionTypeRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ExecutionTypeRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="ensureExecutionType" type="boolean"/>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionType</code> of the requested container.
+
+ @param execType
+          ExecutionType of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ExecutionType</code>.
+
+ @return <code>ExecutionType</code>.]]>
+      </doc>
+    </method>
+    <method name="setEnforceExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enforceExecutionType" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to explicitly ask that the Scheduling Authority return
+ Containers of exactly the Execution Type requested.
+ @param enforceExecutionType whether ExecutionType request should be
+                            strictly honored.]]>
+      </doc>
+    </method>
+    <method name="getEnforceExecutionType" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether Scheduling Authority should return Containers of exactly the
+ Execution Type requested for this <code>ResourceRequest</code>.
+ Defaults to false.
+ @return whether ExecutionType request should be strictly honored]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object of this class represents a specification of the execution
+ guarantee of the Containers associated with a ResourceRequest. It consists
+ of an <code>ExecutionType</code> as well as flag that explicitly asks the
+ configuredScheduler to return Containers of exactly the Execution Type
+ requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <class name="FinalApplicationStatus" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various final states of an <code>Application</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalizationState -->
+  <class name="LocalizationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalizationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalizationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of localization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalizationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalizationStatus -->
+  <class name="LocalizationStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalizationStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalizationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceKey" type="java.lang.String"/>
+      <param name="localizationState" type="org.apache.hadoop.yarn.api.records.LocalizationState"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalizationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceKey" type="java.lang.String"/>
+      <param name="localizationState" type="org.apache.hadoop.yarn.api.records.LocalizationState"/>
+      <param name="diagnostics" type="java.lang.String"/>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource key.
+
+ @return resource key.]]>
+      </doc>
+    </method>
+    <method name="getLocalizationState" return="org.apache.hadoop.yarn.api.records.LocalizationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the localization sate.
+
+ @return localization state.]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the diagnostics.
+
+ @return diagnostics.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents the localization status of a resource.
+ The status of the localization includes:
+ <ul>
+   <li>resource key</li>
+   <li>{@link LocalizationState} of the resource</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalizationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <class name="LocalResource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalResource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>location</em> of the resource to be localized.
+ @return <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <doc>
+      <![CDATA[Set <em>location</em> of the resource to be localized.
+ @param resource <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getSize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>size</em> of the resource to be localized.
+ @return <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>size</em> of the resource to be localized.
+ @param size <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @return <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @param timestamp <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceType</code> of the resource to be localized.
+ @return <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceType</code> of the resource to be localized.
+ @param type <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getVisibility" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @return <code>LocalResourceVisibility</code> of the resource to be 
+         localized]]>
+      </doc>
+    </method>
+    <method name="setVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @param visibility <code>LocalResourceVisibility</code> of the resource to be 
+                   localized]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @return <em>pattern</em> that should be used to extract entries from the 
+ archive.]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @param pattern <em>pattern</em> that should be used to extract entries 
+ from the archive.]]>
+      </doc>
+    </method>
+    <method name="getShouldBeUploadedToSharedCache" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM uses it to decide whether if it is necessary to upload the resource to
+ the shared cache
+ @return true if it is necessary to upload the resource
+                 to the shared cache,
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="setShouldBeUploadedToSharedCache"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+      <doc>
+      <![CDATA[Inform NM whether upload to SCM is needed.
+
+ @param shouldBeUploadedToSharedCache <em>shouldBeUploadedToSharedCache</em>
+          of this request]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>LocalResource</code> represents a local resource required to
+ run a container.</p>
+ 
+ <p>The <code>NodeManager</code> is responsible for localizing the resource 
+ prior to launching the container.</p>
+ 
+ <p>Applications can specify {@link LocalResourceType} and 
+ {@link LocalResourceVisibility}.</p>
+ 
+ @see LocalResourceType
+ @see LocalResourceVisibility
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <class name="LocalResourceType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceType} specifies the <em>type</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>type</em> can be one of:
+ <ul>
+   <li>
+     {@link #FILE} - Regular file i.e. uninterpreted bytes.
+   </li>
+   <li>
+     {@link #ARCHIVE} - Archive, which is automatically unarchived by the
+     <code>NodeManager</code>.
+   </li>
+   <li>
+     {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}.
+   </li>
+ </ul>
+
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <class name="LocalResourceVisibility" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceVisibility} specifies the <em>visibility</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>visibility</em> can be one of:
+ <ul>
+   <li>{@link #PUBLIC} - Shared by all users on the node.</li>
+   <li>
+     {@link #PRIVATE} - Shared among all applications of the
+     <em>same user</em> on the node.
+   </li>
+   <li>
+     {@link #APPLICATION} - Shared only among containers of the
+     <em>same application</em> on the node.
+   </li>
+ </ul>
+ 
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <class name="LogAggregationContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <param name="policyClassName" type="java.lang.String"/>
+      <param name="policyParameters" type="java.lang.String"/>
+    </method>
+    <method name="getIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param includePattern]]>
+      </doc>
+    </method>
+    <method name="getExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="excludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param excludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern in a rolling fashion.
+ 
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern in a rolling fashion.
+ 
+ @param rolledLogsIncludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern for aggregation in a rolling fashion.
+ 
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern for in a rolling fashion.
+ 
+ @param rolledLogsExcludePattern]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyClassName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy class.
+
+ @return log aggregation policy class]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyClassName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy class.
+
+ @param className]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyParameters" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy parameters.
+
+ @return log aggregation policy parameters]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy parameters.
+ There is no schema defined for the parameters string.
+ It is up to the log aggregation policy class to decide how to parse
+ the parameters string.
+
+ @param parameters]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code LogAggregationContext} represents all of the
+ information needed by the {@code NodeManager} to handle
+ the logs for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>
+     includePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be uploaded when the application finishes.
+   </li>
+   <li>
+     excludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be uploaded when application finishes. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     rolledLogsIncludePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be aggregated in a rolling fashion.
+   </li>
+   <li>
+     rolledLogsExcludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be aggregated in a rolling fashion. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     policyClassName. The policy class name that implements
+     ContainerLogAggregationPolicy. At runtime, nodemanager will the policy
+     if a given container's log should be aggregated based on the
+     ContainerType and other runtime state such as exit code by calling
+     ContainerLogAggregationPolicy#shouldDoLogAggregation.
+     This is useful when the app only wants to aggregate logs of a subset of
+     containers. Here are the available policies. Please make sure to specify
+     the canonical name by prefixing org.apache.hadoop.yarn.server.
+     nodemanager.containermanager.logaggregation.
+     to the class simple name below.
+     NoneContainerLogAggregationPolicy: skip aggregation for all containers.
+     AllContainerLogAggregationPolicy: aggregate all containers.
+     AMOrFailedContainerLogAggregationPolicy: aggregate application master
+         or failed containers.
+     FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed
+         containers
+     FailedContainerLogAggregationPolicy: aggregate failed containers
+     AMOnlyLogAggregationPolicy: aggregate application master containers
+     SampleContainerLogAggregationPolicy: sample logs of successful worker
+         containers, in addition to application master and failed/killed
+         containers.
+     LimitSizeContainerLogAggregationPolicy: skip aggregation for killed
+         containers whose log size exceeds the limit of container log size.
+     If it isn't specified, it will use the cluster-wide default policy
+     defined by configuration yarn.nodemanager.log-aggregation.policy.class.
+     The default value of yarn.nodemanager.log-aggregation.policy.class is
+     AllContainerLogAggregationPolicy.
+   </li>
+   <li>
+     policyParameters. The parameters passed to the policy class via
+     ContainerLogAggregationPolicy#parseParameters during the policy object
+     initialization. This is optional. Some policy class might use parameters
+     to adjust its settings. It is up to policy class to define the scheme of
+     parameters.
+     For example, SampleContainerLogAggregationPolicy supports the format of
+     "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50
+     successful worker containers.
+   </li>
+ </ul>
+
+ @see ApplicationSubmissionContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NMToken -->
+  <class name="NMToken" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMToken"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link NodeId} of the <code>NodeManager</code> for which the NMToken
+ is used to authenticate.
+ @return the {@link NodeId} of the <code>NodeManager</code> for which the
+ NMToken is used to authenticate.]]>
+      </doc>
+    </method>
+    <method name="setNodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Token} used for authenticating with <code>NodeManager</code>
+ @return the {@link Token} used for authenticating with <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The NMToken is used for authenticating communication with
+ <code>NodeManager</code></p>
+ <p>It is issued by <code>ResourceMananger</code> when <code>ApplicationMaster</code>
+ negotiates resource with <code>ResourceManager</code> and
+ validated on <code>NodeManager</code> side.</p>
+ @see  AllocateResponse#getNMTokens()]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NMToken -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <class name="NodeAttribute" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttribute"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <field name="PREFIX_DISTRIBUTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREFIX_CENTRALIZED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute is a kind of a label which represents one of the
+ attribute/feature of a Node. Its different from node partition label as
+ resource guarantees across the queues will not be maintained for these type
+ of labels.
+ </p>
+ <p>
+ A given Node can be mapped with any kind of attribute, few examples are
+ HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ </p>
+ <p>
+ Its not compulsory for all the attributes to have value, empty string is the
+ default value of the <code>NodeAttributeType.STRING</code>
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <class name="NodeAttributeInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttribute" type="org.apache.hadoop.yarn.api.records.NodeAttribute"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute Info describes a NodeAttribute.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <class name="NodeAttributeKey" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="getAttributePrefix" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributePrefix"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute
+ is identified based on attribute prefix and name.
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <class name="NodeAttributeOpCode" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various node attribute op codes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <class name="NodeAttributeType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Type of a <code>node Attribute</code>.
+ </p>
+ Based on this attribute expressions and values will be evaluated.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeId -->
+  <class name="NodeId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>hostname</em> of the node.
+ @return <em>hostname</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>port</em> for communicating with the node.
+ @return <em>port</em> for communicating with the node]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p><code>NodeId</code> is the unique identifier for a node.</p>
+ 
+ <p>It includes the <em>hostname</em> and <em>port</em> to uniquely 
+ identify the node. Thus, it is unique across restarts of any 
+ <code>NodeManager</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <class name="NodeLabel" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeLabel"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isExclusive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeLabel"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DEFAULT_NODE_LABEL_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default node label partition used for displaying.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABEL_EXPRESSION_NOT_SET" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node Label expression not set .]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABEL_EXCLUSIVITY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, node label is exclusive or not]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <class name="NodeReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeId</code> of the node.
+ @return <code>NodeId</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getNodeState" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeState</code> of the node.
+ @return <code>NodeState</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>http address</em> of the node.
+ @return <em>http address</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getRackName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>rack name</em> for the node.
+ @return <em>rack name</em> for the node]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>used</em> <code>Resource</code> on the node.
+ @return <em>used</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>total</em> <code>Resource</code> on the node.
+ @return <em>total</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get labels of this node.
+ @return labels of this node.]]>
+      </doc>
+    </method>
+    <method name="getAggregatedContainersUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get containers aggregated resource utilization in a node.
+ @return containers resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getNodeUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node resource utilization.
+ @return node resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getDecommissioningTimeout" return="java.lang.Integer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional decommissioning timeout in seconds (null indicates absent
+ timeout).
+ @return the decommissioning timeout in second.]]>
+      </doc>
+    </method>
+    <method name="setDecommissioningTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decommissioningTimeout" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Set the decommissioning timeout in seconds (null indicates absent timeout).]]>
+      </doc>
+    </method>
+    <method name="getNodeUpdateType" return="org.apache.hadoop.yarn.api.records.NodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional node update type (null indicates absent update type).
+ @return the node update.]]>
+      </doc>
+    </method>
+    <method name="setNodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeUpdateType" type="org.apache.hadoop.yarn.api.records.NodeUpdateType"/>
+      <doc>
+      <![CDATA[Set the node update type (null indicates absent node update type).]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the node attributes of node.
+
+ @param nodeAttributes set of node attributes.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes of node.
+ @return the set of node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeReport} is a summary of runtime information of a node
+ in the cluster.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link NodeId} of the node.</li>
+   <li>HTTP Tracking URL of the node.</li>
+   <li>Rack name for the node.</li>
+   <li>Used {@link Resource} on the node.</li>
+   <li>Total available {@link Resource} of the node.</li>
+   <li>Number of running containers on the node.</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeState -->
+  <class name="NodeState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isUnusable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInactiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isActiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Node</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <class name="NodeToAttributeValue" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeToAttributeValue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeToAttributeValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getHostname" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHostname"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Mapping of Attribute Value to a Node.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <class name="PreemptionContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Container referenced by this handle.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specific container requested back by the <code>ResourceManager</code>.
+ @see PreemptionContract
+ @see StrictPreemptionContract]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <class name="PreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the AM releases resources matching these requests, then the {@link
+ PreemptionContainer}s enumerated in {@link #getContainers()} should not be
+ evicted from the cluster. Due to delays in propagating cluster state and
+ sending these messages, there are conditions where satisfied contracts may
+ not prevent the platform from killing containers.
+ @return List of {@link PreemptionResourceRequest} to update the
+ <code>ApplicationMaster</code> about resources requested back by the
+ <code>ResourceManager</code>.
+ @see AllocateRequest#setAskList(List)]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Assign the set of {@link PreemptionContainer} specifying which containers
+ owned by the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>. If the AM prefers a different set of
+ containers, then it may checkpoint or kill containers matching the
+ description in {@link #getResourceRequest}.
+ @return Set of containers at risk if the contract is not met.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the <code>ResourceManager</code>.
+ The <code>ApplicationMaster</code> (AM) can satisfy this request according
+ to its own priorities to prevent containers from being forcibly killed by
+ the platform.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <class name="PreemptionMessage" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionMessage"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStrictContract" return="org.apache.hadoop.yarn.api.records.StrictPreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Specific resources that may be killed by the
+ <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContract" return="org.apache.hadoop.yarn.api.records.PreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Contract describing resources to return to the cluster.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link PreemptionMessage} is part of the RM-AM protocol, and it is used by
+ the RM to specify resources that the RM wants to reclaim from this
+ {@code ApplicationMaster} (AM). The AM receives a {@link
+ StrictPreemptionContract} message encoding which containers the platform may
+ forcibly kill, granting it an opportunity to checkpoint state or adjust its
+ execution plan. The message may also include a {@link PreemptionContract}
+ granting the AM more latitude in selecting which resources to return to the
+ cluster.
+ <p>
+ The AM should decode both parts of the message. The {@link
+ StrictPreemptionContract} specifies particular allocations that the RM
+ requires back. The AM can checkpoint containers' state, adjust its execution
+ plan to move the computation, or take no action and hope that conditions that
+ caused the RM to ask for the container will change.
+ <p>
+ In contrast, the {@link PreemptionContract} also includes a description of
+ resources with a set of containers. If the AM releases containers matching
+ that profile, then the containers enumerated in {@link
+ PreemptionContract#getContainers()} may not be killed.
+ <p>
+ Each preemption message reflects the RM's current understanding of the
+ cluster state, so a request to return <em>N</em> containers may not
+ reflect containers the AM is releasing, recently exited containers the RM has
+ yet to learn about, or new containers allocated before the message was
+ generated. Conversely, an RM may request a different profile of containers in
+ subsequent requests.
+ <p>
+ The policy enforced by the RM is part of the scheduler. Generally, only
+ containers that have been requested consistently should be killed, but the
+ details are not specified.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <class name="PreemptionResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Resource described in this request, to be matched against running
+ containers.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the cluster.
+ @see PreemptionContract
+ @see AllocateRequest#setAskList(java.util.List)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Priority -->
+  <class name="Priority" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Priority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="int"/>
+    </method>
+    <method name="getPriority" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the assigned priority
+ @return the assigned priority]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Set the assigned priority
+ @param priority the assigned priority]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="UNDEFINED" type="org.apache.hadoop.yarn.api.records.Priority"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The priority assigned to a ResourceRequest or Application or Container 
+ allocation]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Priority -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <class name="QueueACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code QueueACL} enumerates the various ACLs for queues.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue.
+   </li>
+   <li>{@link #ADMINISTER_QUEUE} - ACL to administer the queue.</li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>name</em> of the queue.
+ @return <em>name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>configured capacity</em> of the queue.
+ @return <em>configured capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getMaximumCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>maximum capacity</em> of the queue.
+ @return <em>maximum capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCurrentCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current capacity</em> of the queue.
+ @return <em>current capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>child queues</em> of the queue.
+ @return <em>child queues</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>running applications</em> of the queue.
+ @return <em>running applications</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueState" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueState</code> of the queue.
+ @return <code>QueueState</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getAccessibleNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>accessible node labels</code> of the queue.
+ @return <code>accessible node labels</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getDefaultNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>default node label expression</code> of the queue, this takes
+ affect only when the <code>ApplicationSubmissionContext</code> and
+ <code>ResourceRequest</code> don't specify their
+ <code>NodeLabelExpression</code>.
+ 
+ @return <code>default node label expression</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setDefaultNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getQueueStatistics" return="org.apache.hadoop.yarn.api.records.QueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>queue stats</code> for the queue
+
+ @return <code>queue stats</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueStatistics" type="org.apache.hadoop.yarn.api.records.QueueStatistics"/>
+      <doc>
+      <![CDATA[Set the queue statistics for the queue
+ 
+ @param queueStatistics
+          the queue statistics]]>
+      </doc>
+    </method>
+    <method name="getPreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>preemption status</em> of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return <em>preemption status</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueConfigurations" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the per-node-label queue configurations of the queue.
+
+ @return the per-node-label queue configurations of the queue.]]>
+      </doc>
+    </method>
+    <method name="getIntraQueuePreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the intra-queue preemption status of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return intra-queue preemption status of the queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[QueueInfo is a report of the runtime information of the queue.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>Queue name.</li>
+   <li>Capacity of the queue.</li>
+   <li>Maximum capacity of the queue.</li>
+   <li>Current capacity of the queue.</li>
+   <li>Child queues.</li>
+   <li>Running applications.</li>
+   <li>{@link QueueState} of the queue.</li>
+   <li>{@link QueueConfigurations} of the queue.</li>
+ </ul>
+
+ @see QueueState
+ @see QueueConfigurations
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of a Queue.
+ <p>
+ A queue is in one of:
+ <ul>
+   <li>{@link #RUNNING} - normal state.</li>
+   <li>{@link #STOPPED} - not accepting new application submissions.</li>
+   <li>
+     {@link #DRAINING} - not accepting new application submissions
+     and waiting for applications finish.
+   </li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <class name="QueueStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumAppsSubmitted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of apps submitted
+ 
+ @return the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="setNumAppsSubmitted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsSubmitted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of apps submitted
+ 
+ @param numAppsSubmitted
+          the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="getNumAppsRunning" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running apps
+ 
+ @return the number of running apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsRunning"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsRunning" type="long"/>
+      <doc>
+      <![CDATA[Set the number of running apps
+ 
+ @param numAppsRunning
+          the number of running apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsPending" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending apps
+ 
+ @return the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsPending"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsPending" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending apps
+ 
+ @param numAppsPending
+          the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsCompleted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of completed apps
+ 
+ @return the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsCompleted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsCompleted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of completed apps
+ 
+ @param numAppsCompleted
+          the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsKilled" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of killed apps
+ 
+ @return the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsKilled"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsKilled" type="long"/>
+      <doc>
+      <![CDATA[Set the number of killed apps
+ 
+ @param numAppsKilled
+          the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsFailed" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of failed apps
+ 
+ @return the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsFailed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsFailed" type="long"/>
+      <doc>
+      <![CDATA[Set the number of failed apps
+ 
+ @param numAppsFailed
+          the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="getNumActiveUsers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active users
+ 
+ @return the number of active users]]>
+      </doc>
+    </method>
+    <method name="setNumActiveUsers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numActiveUsers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of active users
+ 
+ @param numActiveUsers
+          the number of active users]]>
+      </doc>
+    </method>
+    <method name="getAvailableMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available memory in MB
+ 
+ @return the available memory]]>
+      </doc>
+    </method>
+    <method name="setAvailableMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the available memory in MB
+ 
+ @param availableMemoryMB
+          the available memory]]>
+      </doc>
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated memory in MB
+ 
+ @return the allocated memory]]>
+      </doc>
+    </method>
+    <method name="setAllocatedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated memory in MB
+ 
+ @param allocatedMemoryMB
+          the allocate memory]]>
+      </doc>
+    </method>
+    <method name="getPendingMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending memory in MB
+ 
+ @return the pending memory]]>
+      </doc>
+    </method>
+    <method name="setPendingMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the pending memory in MB
+ 
+ @param pendingMemoryMB
+          the pending memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved memory in MB
+ 
+ @return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved memory in MB
+ 
+ @param reservedMemoryMB
+          the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getAvailableVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available vcores
+ 
+ @return the available vcores]]>
+      </doc>
+    </method>
+    <method name="setAvailableVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the available vcores
+ 
+ @param availableVCores
+          the available vcores]]>
+      </doc>
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated vcores
+ 
+ @return the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="setAllocatedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated vcores
+ 
+ @param allocatedVCores
+          the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending vcores
+ 
+ @return the pending vcores]]>
+      </doc>
+    </method>
+    <method name="setPendingVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the pending vcores
+ 
+ @param pendingVCores
+          the pending vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending containers.
+ @return the number of pending containers.]]>
+      </doc>
+    </method>
+    <method name="setPendingContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending containers.
+ @param pendingContainers the pending containers.]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of allocated containers.
+ @return the number of allocated containers.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of allocated containers.
+ @param allocatedContainers the allocated containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved containers.
+ @return the number of reserved containers.]]>
+      </doc>
+    </method>
+    <method name="setReservedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of reserved containers.
+ @param reservedContainers the reserved containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved vcores
+ 
+ @return the reserved vcores]]>
+      </doc>
+    </method>
+    <method name="setReservedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved vcores
+ 
+ @param reservedVCores
+          the reserved vcores]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <class name="QueueUserACLInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueUserACLInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> of the queue.
+ @return <em>queue name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getUserAcls" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>QueueACL</code> for the given user.
+ @return list of <code>QueueACL</code> for the given user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>QueueUserACLInfo</code> provides information {@link QueueACL} for
+ the given user.</p>
+ 
+ @see QueueACL
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <class name="RejectedSchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RejectedSchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Create new RejectedSchedulingRequest.
+ @param reason Rejection Reason.
+ @param request Rejected Scheduling Request.
+ @return RejectedSchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="getReason" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get Rejection Reason.
+ @return Rejection reason.]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <doc>
+      <![CDATA[Set Rejection Reason.
+ @param reason Rejection Reason.]]>
+      </doc>
+    </method>
+    <method name="getRequest" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Rejected Scheduling Request.
+ @return SchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="setRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Set the SchedulingRequest.
+ @param request SchedulingRequest.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates a Rejected SchedulingRequest. It contains the offending
+ Scheduling Request along with the reason for rejection.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <class name="RejectionReason" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.RejectionReason[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Reason for rejecting a Scheduling Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <class name="ReservationACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationACL} enumerates the various ACLs for reservations.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and
+     delete reservations.
+   </li>
+   <li> {@link #LIST_RESERVATIONS} - ACL to list reservations. </li>
+   <li> {@link #SUBMIT_RESERVATIONS} - ACL to create reservations. </li>
+ </ul>
+ Users can always list, update and delete their own reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <class name="ReservationAllocationState" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationAllocationState"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationAllocationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acceptanceTime" type="long"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="resourceAllocations" type="java.util.List"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[@param acceptanceTime The acceptance time of the reservation.
+ @param user The username of the user who made the reservation.
+ @param resourceAllocations List of {@link ResourceAllocationRequest}
+                            representing the current state of the
+                            reservation resource allocations. This is
+                            subject to change in the event of re-planning.
+ @param reservationId {@link ReservationId } of the reservation being
+                                            listed.
+ @param reservationDefinition {@link ReservationDefinition} used to make
+                              the reservation.
+ @return {@code ReservationAllocationState} that represents the state of
+ the reservation.]]>
+      </doc>
+    </method>
+    <method name="getAcceptanceTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the acceptance time of the reservation.
+
+ @return the time that the reservation was accepted.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user who made the reservation.
+
+ @return the name of the user who made the reservation.]]>
+      </doc>
+    </method>
+    <method name="getResourceAllocationRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Resource allocations of the reservation based on the current state
+ of the plan. This is subject to change in the event of re-planning.
+ The allocations will be constraint to the user contract as described by
+ the {@link ReservationDefinition}
+
+ @return a list of resource allocations for the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of the reservation.
+
+ @return the reservation id corresponding to the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation definition used to make the reservation.
+
+ @return the reservation definition used to make the reservation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationAllocationState} represents the reservation that is
+ made by a user.
+ <p>
+ It includes:
+ <ul>
+   <li>Duration of the reservation.</li>
+   <li>Acceptance time of the duration.</li>
+   <li>
+       List of {@link ResourceAllocationRequest}, which includes the time
+       interval, and capability of the allocation.
+       {@code ResourceAllocationRequest} represents an allocation
+       made for a reservation for the current state of the queue. This can be
+       changed for reasons such as re-planning, but will always be subject to
+       the constraints of the user contract as described by
+       {@link ReservationDefinition}
+   </li>
+   <li>{@link ReservationId} of the reservation.</li>
+   <li>{@link ReservationDefinition} used to make the reservation.</li>
+ </ul>
+
+ @see ResourceAllocationRequest
+ @see ReservationId
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <class name="ReservationDefinition" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDefinition"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getArrival" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @return the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="setArrival"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="earliestStartTime" type="long"/>
+      <doc>
+      <![CDATA[Set the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @param earliestStartTime the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="getDeadline" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @return the deadline or the latest time by when the resource(s) must be
+         allocated]]>
+      </doc>
+    </method>
+    <method name="setDeadline"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="latestEndTime" type="long"/>
+      <doc>
+      <![CDATA[Set the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @param latestEndTime the deadline or the latest time by when the
+          resource(s) should be allocated]]>
+      </doc>
+    </method>
+    <method name="getReservationRequests" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="setReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @param reservationRequests the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="getReservationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @return string representing the name of the corresponding reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @param name representing the name of the corresponding reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getRecurrenceExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @return recurrence of this reservation]]>
+      </doc>
+    </method>
+    <method name="setRecurrenceExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @param recurrenceExpression recurrence interval of this reservation]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @return int representing the priority of the reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @param priority representing the priority of the reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDefinition} captures the set of resource and time
+ constraints the user cares about regarding a reservation.
+ 
+ @see ResourceRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <class name="ReservationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the long identifier of the {@link ReservationId} which is unique for
+ all Reservations started by a particular instance of the
+ {@code ResourceManager}.
+
+ @return long identifier of the {@link ReservationId}]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the {@code ResourceManager} which is used to
+ generate globally unique {@link ReservationId}.
+
+ @return <em>start time</em> of the {@code ResourceManager}]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the string argument as a {@link ReservationId}
+
+ @param reservationId the string representation of the {@link ReservationId}
+ @return the {@link ReservationId} corresponding to the input string if
+         valid, null if input is null
+ @throws IOException if unable to parse the input string]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="reserveIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="clusterTimestamp" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ {@link ReservationId} represents the <em>globally unique</em> identifier for
+ a reservation.
+ </p>
+
+ <p>
+ The globally unique nature of the identifier is achieved by using the
+ <em>cluster timestamp</em> i.e. start-time of the {@code ResourceManager}
+ along with a monotonically increasing counter for the reservation.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <class name="ReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="concurrency" type="int"/>
+      <param name="duration" type="long"/>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Resource} capability of the request.
+ 
+ @return {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the {@link Resource} capability of the request
+ 
+ @param capability {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ 
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ 
+ @param numContainers number of containers required with the given
+          specifications]]>
+      </doc>
+    </method>
+    <method name="getConcurrency" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @return the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="setConcurrency"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @param numContainers the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="getDuration" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds for which the resource is required. A
+ default value of -1, indicates an unspecified lease duration, and fallback
+ to current behavior.
+ 
+ @return the duration in milliseconds for which the resource is required]]>
+      </doc>
+    </method>
+    <method name="setDuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="duration" type="long"/>
+      <doc>
+      <![CDATA[Set the duration in milliseconds for which the resource is required.
+ 
+ @param duration the duration in milliseconds for which the resource is
+          required]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequest} represents the request made by an application to
+ the {@code ResourceManager} to reserve {@link Resource}s.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required by the
+     application.
+   </li>
+   <li>Concurrency that indicates the gang size of the request.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <class name="ReservationRequest.ReservationRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ReservationRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <class name="ReservationRequestInterpreter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various types of dependencies among multiple
+ {@link ReservationRequests} within one {@link ReservationDefinition} (from
+ least constraining to most constraining).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <class name="ReservationRequests" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationRequests"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+    </method>
+    <method name="getReservationResources" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="setReservationResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @param reservationResources the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="getInterpreter" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @return the list of {@link ReservationRequestInterpreter}]]>
+      </doc>
+    </method>
+    <method name="setInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interpreter" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @param interpreter the {@link ReservationRequestInterpreter} for this
+          reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequests} captures the set of resource and constraints the
+ user cares about regarding a reservation.
+ 
+ @see ReservationRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Resource -->
+  <class name="Resource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+      <param name="others" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Create a new {@link Resource} instance with the given CPU and memory
+ values and additional resource values as set in the {@code others}
+ parameter. Note that the CPU and memory settings in the {@code others}
+ parameter will be ignored.
+
+ @param memory the memory value
+ @param vCores the CPU value
+ @param others a map of other resource values indexed by resource name
+ @return a {@link Resource} instance with the given resource values]]>
+      </doc>
+    </method>
+    <method name="getMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is DEPRECATED:
+ Use {@link Resource#getMemorySize()} instead
+
+ Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="getMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to set
+ memory but the assumption is that the value passed is in units of MB.
+
+ @param memory <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemorySize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource.
+ @param memory <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getVirtualCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @return <em>num of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setVirtualCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vCores" type="int"/>
+      <doc>
+      <![CDATA[Set <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @param vCores <em>number of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get ResourceInformation for a specified resource.
+
+ @param resource name of the resource
+ @return the ResourceInformation object for the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value for a specified resource. No information about the units is
+ returned.
+
+ @param resource name of the resource
+ @return the value for the resource]]>
+      </doc>
+    </method>
+    <method name="setResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="resourceInformation" type="org.apache.hadoop.yarn.api.records.ResourceInformation"/>
+      <doc>
+      <![CDATA[Set the ResourceInformation object for a particular resource.
+
+ @param resource the resource for which the ResourceInformation is provided
+ @param resourceInformation ResourceInformation object]]>
+      </doc>
+    </method>
+    <method name="setResourceValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of a resource in the ResourceInformation object. The unit of
+ the value is assumed to be the one in the ResourceInformation object.
+
+ @param resource the resource for which the value is provided.
+ @param value    the value to set]]>
+      </doc>
+    </method>
+    <method name="throwExceptionWhenArrayOutOfBound"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="index" type="int"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFormattedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is to get memory in terms of KB|MB|GB.
+ @return string containing all resources]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="castToIntSafely" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Convert long to int for a resource value safely. This method assumes
+ resource value is positive.
+
+ @param value long resource value
+ @return int resource value]]>
+      </doc>
+    </method>
+    <method name="newDefaultInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Create ResourceInformation with basic fields.
+ @param name Resource Type Name
+ @param unit Default unit of provided resource type
+ @param value Value associated with giveb resource
+ @return ResourceInformation object]]>
+      </doc>
+    </method>
+    <field name="resources" type="org.apache.hadoop.yarn.api.records.ResourceInformation[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VCORES_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>Resource</code> models a set of computer resources in the 
+ cluster.</p>
+ 
+ <p>Currently it models both <em>memory</em> and <em>CPU</em>.</p>
+ 
+ <p>The unit for memory is megabytes. CPU is modeled with virtual cores
+ (vcores), a unit for expressing parallelism. A node's capacity should
+ be configured with virtual cores equal to its number of physical cores. A
+ container should be requested with the number of cores it can saturate, i.e.
+ the average number of threads it expects to have runnable at a time.</p>
+ 
+ <p>Virtual cores take integer values and thus currently CPU-scheduling is
+ very coarse.  A complementary axis for CPU requests that represents
+ processing power will likely be added in the future to enable finer-grained
+ resource configuration.</p>
+
+ <p>Typically, applications request <code>Resource</code> of suitable
+ capability to run their component tasks.</p>
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Resource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <class name="ResourceAllocationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceAllocationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceAllocationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[@param startTime The start time that the capability is reserved for.
+ @param endTime The end time that the capability is reserved for.
+ @param capability {@link Resource} representing the capability of the
+                                   resource allocation.
+ @return {ResourceAllocationRequest} which represents the capability of
+ the resource allocation for a time interval.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time that the resource is allocated.
+
+ @return the start time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time that the resource is allocated.
+
+ @return the end time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated resource.
+
+ @return the allocated resource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceAllocationRequest} represents an allocation
+ made for a reservation for the current state of the plan. This can be
+ changed for reasons such as re-planning, but will always be subject to the
+ constraints of the user contract as described by
+ {@link ReservationDefinition}
+ {@link Resource}
+
+ <p>
+ It includes:
+ <ul>
+   <li>StartTime of the allocation.</li>
+   <li>EndTime of the allocation.</li>
+   <li>{@link Resource} reserved for the allocation.</li>
+ </ul>
+
+ @see Resource]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <class name="ResourceBlacklistRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceBlacklistRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="additions" type="java.util.List"/>
+      <param name="removals" type="java.util.List"/>
+    </method>
+    <method name="getBlacklistAdditions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be added to the 
+ application blacklist.
+ 
+ @return list of resource-names which should be added to the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistAdditions"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be added to the application blacklist.
+ 
+ @param resourceNames list of resource-names which should be added to the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <method name="getBlacklistRemovals" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @return list of resource-names which should be removed from the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistRemovals"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @param resourceNames list of resource-names which should be removed from the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ResourceBlacklistRequest} encapsulates the list of resource-names 
+ which should be added or removed from the <em>blacklist</em> of resources 
+ for the application.
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <class name="ResourceOption" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceOption"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="overCommitTimeout" type="int"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Negative value means no timeout.]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <class name="ResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Clone a ResourceRequest object (shallow copy). Please keep it loaded with
+ all (new) fields
+
+ @param rr the object to copy from
+ @return the copied object]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isAnyLocation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check whether the given <em>host/rack</em> string represents an arbitrary
+ host name.
+
+ @param hostName <em>host/rack</em> on which the allocation is desired
+ @return whether the given <em>host/rack</em> string represents an arbitrary
+ host name]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the request.
+ @return <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the request
+ @param priority <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="getResourceName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource 
+ (host/rack) is acceptable.
+ 
+ @return resource (e.g. <em>host/rack</em>) on which the allocation 
+                  is desired]]>
+      </doc>
+    </method>
+    <method name="setResourceName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the resource name (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource name
+ (e.g. host/rack) is acceptable. 
+ 
+ @param resourceName (e.g. <em>host/rack</em>) on which the 
+                     allocation is desired]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ @param numContainers number of containers required with the given 
+                      specifications]]>
+      </doc>
+    </method>
+    <method name="getRelaxLocality" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+ 
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execSpec" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionTypeRequest</code> of the requested container.
+
+ @param execSpec
+          ExecutionTypeRequest of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionTypeRequest" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setRelaxLocality"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[<p>For a request at a network hierarchy level, set whether locality can be relaxed
+ to that level and beyond.<p>
+ 
+ <p>If the flag is off on a rack-level <code>ResourceRequest</code>,
+ containers at that request's priority will not be assigned to nodes on that
+ request's rack unless requests specifically for those nodes have also been
+ submitted.<p>
+ 
+ <p>If the flag is off on an {@link ResourceRequest#ANY}-level
+ <code>ResourceRequest</code>, containers at that request's priority will
+ only be assigned on racks for which specific requests have also been
+ submitted.<p>
+ 
+ <p>For example, to request a container strictly on a specific node, the
+ corresponding rack-level and any-level requests should have locality
+ relaxation set to false.  Similarly, to request a container strictly on a
+ specific rack, the corresponding any-level request should have locality
+ relaxation set to false.<p>
+ 
+ @param relaxLocality whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this Resource Request. If this is set, all
+ containers allocated to satisfy this resource-request will be only on those
+ nodes that satisfy this node-label-expression.
+  
+ Please note that node label expression now can only take effect when the
+ resource request has resourceName = ANY
+ 
+ @return node-label-expression]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodelabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node label expression of this resource request. Now only support
+ specifying a single node label. In the future we will support more complex
+ node label expression specification like {@code AND(&&), OR(||)}, etc.
+ 
+ Any please note that node label expression now can only take effect when
+ the resource request has resourceName = ANY
+ 
+ @param nodelabelExpression
+          node-label-expression of this ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+
+ @return the <em>ID</em> corresponding to this allocation request.]]>
+      </doc>
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestID" type="long"/>
+      <doc>
+      <![CDATA[Set the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+ <p>
+ If the ID is not set, scheduler will continue to work as previously and all
+ allocated {@code Container}(s) will have the default ID, -1.
+
+ @param allocationRequestID the <em>ID</em> corresponding to this allocation
+                            request.]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <field name="ANY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The constant string representing no locality.
+ It should be used by all references that want to pass an arbitrary host
+ name in.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[{@code ResourceRequest} represents the request made
+ by an application to the {@code ResourceManager}
+ to obtain various {@code Container} allocations.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Priority} of the request.</li>
+   <li>
+     The <em>name</em> of the host or rack on which the allocation is
+     desired. A special value of <em>*</em> signifies that
+     <em>any</em> host/rack is acceptable to the application.
+   </li>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required
+     by the application.
+   </li>
+   <li>
+     A boolean <em>relaxLocality</em> flag, defaulting to {@code true},
+     which tells the {@code ResourceManager} if the application wants
+     locality to be loose (i.e. allows fall-through to rack or <em>any</em>)
+     or strict (i.e. specify hard constraint on resource allocation).
+   </li>
+ </ul>
+ 
+ @see Resource
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <class name="ResourceRequest.ResourceRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+ @see ResourceRequest#setPriority(Priority)
+ @param priority <code>priority</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceName" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>resourceName</code> of the request.
+ @see ResourceRequest#setResourceName(String)
+ @param resourceName <code>resourceName</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="capability" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>capability</code> of the request.
+ @see ResourceRequest#setCapability(Resource)
+ @param capability <code>capability</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="numContainers" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>numContainers</code> of the request.
+ @see ResourceRequest#setNumContainers(int)
+ @param numContainers <code>numContainers</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="relaxLocality" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[Set the <code>relaxLocality</code> of the request.
+ @see ResourceRequest#setRelaxLocality(boolean)
+ @param relaxLocality <code>relaxLocality</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="nodeLabelExpression" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>nodeLabelExpression</code> of the request.
+ @see ResourceRequest#setNodeLabelExpression(String)
+ @param nodeLabelExpression
+          <code>nodeLabelExpression</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionTypeRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionTypeRequest
+          <code>executionTypeRequest</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request with 'ensure
+ execution type' flag set to true.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request.
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+ @see ResourceRequest#setAllocationRequestId(long)
+ @param allocationRequestId
+          <code>allocationRequestId</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link ResourceRequest} object.
+ @return {@link ResourceRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link ResourceRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <class name="ResourceRequest.ResourceRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ResourceRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <class name="ResourceSizing" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceSizing"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getNumAllocations" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNumAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+    </method>
+    <method name="getResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceSizing} contains information for the size of a
+ {@link SchedulingRequest}, such as the number of requested allocations and
+ the resources for each allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <class name="ResourceUtilization" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceUtilization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="customResources" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceUtil" type="org.apache.hadoop.yarn.api.records.ResourceUtilization"/>
+    </method>
+    <method name="getVirtualMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get used <em>virtual memory</em>.
+
+ @return <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setVirtualMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vmem" type="int"/>
+      <doc>
+      <![CDATA[Set used <em>virtual memory</em>.
+
+ @param vmem <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>physical memory</em>.
+
+ @return <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setPhysicalMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <doc>
+      <![CDATA[Set <em>physical memory</em>.
+
+ @param pmem <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getCPU" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>CPU</em> utilization (The amount of vcores used).
+
+ @return <em>CPU utilization</em>]]>
+      </doc>
+    </method>
+    <method name="setCPU"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Set <em>CPU</em> utilization (The amount of vcores used).
+
+ @param cpu <em>CPU utilization</em>]]>
+      </doc>
+    </method>
+    <method name="getCustomResource" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get <em>custom resource</em> utilization
+ (The amount of custom resource used).
+
+ @param resourceName <em>resourceName of custom resource</em>
+ @return <em>resourceName utilization</em>]]>
+      </doc>
+    </method>
+    <method name="getCustomResources" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCustomResources"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="customResources" type="java.util.Map"/>
+    </method>
+    <method name="setCustomResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Set <em>custom resource</em> utilization
+ (The amount of custom resource used).
+ @param resourceName <em>resourceName</em>
+ @param utilization <em>utilization of custom resource</em>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.]]>
+      </doc>
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.
+ @param resourceName of custom resource to add.
+ @param utilization of custom resource to add.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.
+ @param resourceName of custom resource to be subtracted.
+ @param utilization of custom resource to be subtracted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ <code>ResourceUtilization</code> models the utilization of a set of computer
+ resources in the cluster.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <class name="SchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <param name="allocationTags" type="java.util.Set"/>
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="getAllocationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+    </method>
+    <method name="getResourceSizing" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+    </method>
+    <method name="getPlacementConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <doc>
+    <![CDATA[{@code SchedulingRequest} represents a request made by an application to the
+ {@code ResourceManager} to obtain an allocation. It is similar to the
+ {@link ResourceRequest}. However, it is more complete than the latter, as it
+ allows applications to specify allocation tags (e.g., to express that an
+ allocation belongs to {@code Spark} or is an {@code HBase-master}), as well
+ as involved {@link PlacementConstraint}s (e.g., anti-affinity between Spark
+ and HBase allocations).
+
+ The size specification of the allocation is in {@code ResourceSizing}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <class name="SchedulingRequest.SchedulingRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+
+ @see SchedulingRequest#setAllocationRequestId(long)
+ @param allocationRequestId <code>allocationRequestId</code> of the
+          request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+
+ @param priority <code>priority</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ @see SchedulingRequest#setPriority(Priority)]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationTags" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the <code>allocationTags</code> of the request.
+
+ @see SchedulingRequest#setAllocationTags(Set)
+ @param allocationTags <code>allocationsTags</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceSizing" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setResourceSizing(ResourceSizing)
+ @param resourceSizing <code>resourceSizing</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="placementConstraintExpression" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+      <doc>
+      <![CDATA[Set the <code>placementConstraintExpression</code> of the request.
+
+ @see SchedulingRequest#setPlacementConstraint(
+      PlacementConstraint)
+ @param placementConstraintExpression <code>placementConstraints</code> of
+          the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link SchedulingRequest} object.
+
+ @return {@link SchedulingRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link SchedulingRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ShellContainerCommand -->
+  <class name="ShellContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ShellContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ShellContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ShellContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <class name="SignalContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <class name="StrictPreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrictPreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of {@link PreemptionContainer} specifying containers owned by
+ the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>.
+ @return the set of {@link ContainerId} to be preempted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of particular allocations to be reclaimed. The platform will
+ reclaim exactly these resources, so the <code>ApplicationMaster</code> (AM)
+ may attempt to checkpoint work or adjust its execution plan to accommodate
+ it. In contrast to {@link PreemptionContract}, the AM has no flexibility in
+ selecting which resources to return to the cluster.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIdentifier" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier.
+ @return token identifier]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password
+ @return token password]]>
+      </doc>
+    </method>
+    <method name="getKind" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return token kind]]>
+      </doc>
+    </method>
+    <method name="getService" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service to which the token is allocated.
+ @return service to which the token is allocated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>Token</code> is the security entity used by the framework
+ to verify authenticity of any resource.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Token -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <class name="UpdateContainerError" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerError"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+    </method>
+    <method name="getReason" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get reason why the update request was not satisfiable.
+ @return Reason]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set reason why the update request was not satisfiable.
+ @param reason Reason]]>
+      </doc>
+    </method>
+    <method name="getCurrentContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current container version.
+ @return Current container Version.]]>
+      </doc>
+    </method>
+    <method name="setCurrentContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="currentVersion" type="int"/>
+      <doc>
+      <![CDATA[Set current container version.
+ @param currentVersion Current container version.]]>
+      </doc>
+    </method>
+    <method name="getUpdateContainerRequest" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@code UpdateContainerRequest} that was not satisfiable.
+ @return UpdateContainerRequest]]>
+      </doc>
+    </method>
+    <method name="setUpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Set the {@code UpdateContainerRequest} that was not satisfiable.
+ @param updateContainerRequest Update Container Request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerError} is used by the Scheduler to notify the
+ ApplicationMaster of an UpdateContainerRequest it cannot satisfy due to
+ an error in the request. It includes the update request as well as
+ a reason for why the request was not satisfiable.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <class name="UpdateContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="targetCapability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="targetExecutionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="getContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerVersion" type="int"/>
+      <doc>
+      <![CDATA[Set the current version of the container.
+ @param containerVersion of the container]]>
+      </doc>
+    </method>
+    <method name="getContainerUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code> of the container.
+ @return <code>ContainerUpdateType</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code> of the container.
+ @param updateType of the Container]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container.
+ @param containerId <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the target <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the target <code>ExecutionType</code> of the container.
+ @param executionType <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerRequest} represents the request made by an
+ application to the {@code ResourceManager} to update an attribute of a
+ {@code Container} such as its Resource allocation or (@code ExecutionType}
+ <p>
+ It includes:
+ <ul>
+   <li>version for the container.</li>
+   <li>{@link ContainerId} for the container.</li>
+   <li>
+     {@link Resource} capability of the container after the update request
+     is completed.
+   </li>
+   <li>
+     {@link ExecutionType} of the container after the update request is
+     completed.
+   </li>
+ </ul>
+
+ Update rules:
+ <ul>
+   <li>
+     Currently only ONE aspect of the container can be updated per request
+     (user can either update Capability OR ExecutionType in one request..
+     not both).
+   </li>
+   <li>
+     There must be only 1 update request per container in an allocate call.
+   </li>
+   <li>
+     If a new update request is sent for a container (in a subsequent allocate
+     call) before the first one is satisfied by the Scheduler, it will
+     overwrite the previous request.
+   </li>
+ </ul>
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <class name="UpdatedContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdatedContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdatedContainer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Static Factory method.
+
+ @param updateType ContainerUpdateType
+ @param container Container
+ @return UpdatedContainer]]>
+      </doc>
+    </method>
+    <method name="getUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code>.
+ @return ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="setUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code>.
+ @param updateType ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="getContainer" return="org.apache.hadoop.yarn.api.records.Container"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Container</code>.
+ @return Container]]>
+      </doc>
+    </method>
+    <method name="setContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Set the <code>Container</code>.
+ @param container Container]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object that encapsulates an updated container and the
+ type of Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.URL -->
+  <class name="URL" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="URL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="file" type="java.lang.String"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the scheme of the URL.
+ @return scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="setScheme"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheme of the URL
+ @param scheme scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="getUserInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user info of the URL.
+ @return user info of the URL]]>
+      </doc>
+    </method>
+    <method name="setUserInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user info of the URL.
+ @param userInfo user info of the URL]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the host of the URL.
+ @return host of the URL]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the host of the URL.
+ @param host host of the URL]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the port of the URL.
+ @return port of the URL]]>
+      </doc>
+    </method>
+    <method name="setPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the port of the URL
+ @param port port of the URL]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file of the URL.
+ @return file of the URL]]>
+      </doc>
+    </method>
+    <method name="setFile"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the file of the URL.
+ @param file file of the URL]]>
+      </doc>
+    </method>
+    <method name="toPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+    </method>
+    <method name="fromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="fromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[<p><code>URL</code> represents a serializable {@link java.net.URL}.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.URL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <class name="YarnApplicationAttemptState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of a <code>RMAppAttempt</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <class name="YarnApplicationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of an <code>ApplicationMaster</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+  <class name="YarnClusterMetrics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>NodeManager</code>s in the cluster.
+ @return number of <code>NodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumDecommissionedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>DecommissionedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>DecommissionedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumActiveNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>ActiveNodeManager</code>s in the cluster.
+ 
+ @return number of <code>ActiveNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumLostNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>LostNodeManager</code>s in the cluster.
+ 
+ @return number of <code>LostNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumUnhealthyNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>UnhealthyNodeManager</code>s in the cluster.
+ 
+ @return number of <code>UnhealthyNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumRebootedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>RebootedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>RebootedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>YarnClusterMetrics</code> represents cluster metrics.</p>
+ 
+ <p>Currently only number of <code>NodeManager</code>s is provided.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.impl">
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timeline">
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <class name="TimelineAbout" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineAbout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineAbout" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAbout" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAbout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersionBuiltOn" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersionBuiltOn" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <class name="TimelineDelegationTokenResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getContent" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="content" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[The response of delegation token related request]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID
+ 
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description
+ 
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description
+ 
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner
+ 
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ 
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string
+ 
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string
+ 
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string
+ 
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string
+ 
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain
+ 
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain
+ 
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain
+ 
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain
+ 
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline domain, which is used
+ to a user to host a number of timeline entities, isolating them from others'.
+ The user can also define the reader and writer users/groups for the the
+ domain, which is used to control the access to its entities.
+ </p>
+ 
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <class name="TimelineDomains" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomains"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDomains" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of domains
+ 
+ @return a list of domains]]>
+      </doc>
+    </method>
+    <method name="addDomain"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <doc>
+      <![CDATA[Add a single domain into the existing domain list
+ 
+ @param domain
+          a single domain]]>
+      </doc>
+    </method>
+    <method name="addDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of domains into the existing domain list
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <method name="setDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the domain list to the given list of domains
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline domains.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of entities
+ 
+ @return a list of entities]]>
+      </doc>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+      <doc>
+      <![CDATA[Add a single entity into the existing entity list
+ 
+ @param entity
+          a single entity]]>
+      </doc>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of entities into the existing entity list
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the entity list to the given list of entities
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time of the entity
+ 
+ @return the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the start time of the entity
+ 
+ @param startTime
+          the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events related to the entity
+ 
+ @return a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event related to the entity to the existing event list
+ 
+ @param event
+          a single event related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of events related to the entity to the existing event list
+ 
+ @param events
+          a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events related to the entity
+ 
+ @param events
+          events a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="getRelatedEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the related entities
+ 
+ @return the related entities]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add an entity to the existing related entity map
+ 
+ @param entityType
+          the entity type
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of related entities to the existing related entity map
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="setRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the related entity map to the given map of related entities
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="getPrimaryFilters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the primary filters
+ 
+ @return the primary filters]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add a single piece of primary filter to the existing primary filter map
+ 
+ @param key
+          the primary filter key
+ @param value
+          the primary filter value]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of primary filters to the existing primary filter map
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="setPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the primary filter map to the given map of primary filters
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="getOtherInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the other information of the entity
+ 
+ @return the other information of the entity]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of other information of the entity to the existing other info
+ map
+ 
+ @param key
+          the other information key
+ @param value
+          the other information value]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of other information of the entity to the existing other info map
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="setOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the other info map to the given map of other information
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="getDomainId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ID of the domain that the entity is to be put
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setDomainId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domainId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ID of the domain that the entity is to be put
+ 
+ @param domainId
+          the name space ID]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The class that contains the the meta information of some conceptual entity
+ and its related events. The entity can be an application, an application
+ attempt, a container or whatever the user-defined object.
+ </p>
+ 
+ <p>
+ Primary filters will be used to index the entities in
+ <code>TimelineStore</code>, such that users should carefully choose the
+ information they want to store as the primary filters. The remaining can be
+ stored as other information.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <class name="TimelineEntityGroupId" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntityGroupId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the
+ <code>TimelineEntityGroupId</code>.
+
+ @return <code>ApplicationId</code> of the
+         <code>TimelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getTimelineEntityGroupId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>timelineEntityGroupId</code>.
+
+ @return <code>timelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntityGroupIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TIMELINE_ENTITY_GROUPID_STR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>TimelineEntityGroupId</code> is an abstract way for
+ timeline service users to represent #a group of related timeline data.
+ For example, all entities that represents one data flow DAG execution
+ can be grouped into one timeline entity group. </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the event
+ 
+ @return the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the timestamp of the event
+ 
+ @param timestamp
+          the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="getEventType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the event type
+ 
+ @return the event type]]>
+      </doc>
+    </method>
+    <method name="setEventType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the event type
+ 
+ @param eventType
+          the event type]]>
+      </doc>
+    </method>
+    <method name="getEventInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the information of the event
+ 
+ @return the information of the event]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of the information of the event to the existing information
+ map
+ 
+ @param key
+          the information key
+ @param value
+          the information value]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of the information of the event to the existing information map
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="setEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the information map to the given map of the information of the event
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The class that contains the information of an event that is related to some
+ conceptual entity of an application. Users are free to define what the event
+ means, such as starting an application, getting allocated a container and
+ etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <class name="TimelineEvents" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEvents"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAllEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link EventsOfOneEntity} instances
+ 
+ @return a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventsOfOneEntity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity"/>
+      <doc>
+      <![CDATA[Add a single {@link EventsOfOneEntity} instance into the existing list
+ 
+ @param eventsOfOneEntity
+          a single {@link EventsOfOneEntity} instance]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link EventsOfOneEntity} instances into the existing list
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link EventsOfOneEntity} instances
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events, which are categorized according to
+ their related entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <class name="TimelineEvents.EventsOfOneEntity" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventsOfOneEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events
+ 
+ @return a list of events]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event to the existing event list
+ 
+ @param event
+          a single event]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of event to the existing event list
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events that are only related to one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <class name="TimelineHealth" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineHealth" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineHealth"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHealthStatus" return="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="healthStatus" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"/>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[This class holds health information for ATS.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <class name="TimelinePutResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelinePutError} instances
+ 
+ @return a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelinePutError} instance into the existing list
+ 
+ @param error
+          a single {@link TimelinePutError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelinePutError} instances into the existing list
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelinePutError} instances
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are errors
+ in storing individual entity objects, they will be indicated in the list of
+ errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+  <class name="TimelinePutResponse.TimelinePutError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code
+ 
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCode" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code
+ 
+ @param errorCode
+          an error code]]>
+      </doc>
+    </method>
+    <field name="NO_START_TIME" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned when no start time can be found when putting an
+ entity. This occurs when the entity does not already exist in the store
+ and it is put with no start time or events specified.]]>
+      </doc>
+    </field>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when putting an
+ entity.]]>
+      </doc>
+    </field>
+    <field name="SYSTEM_FILTER_CONFLICT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user specifies the timeline system reserved
+ filter key]]>
+      </doc>
+    </field>
+    <field name="ACCESS_DENIED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to access the timeline data]]>
+      </doc>
+    </field>
+    <field name="NO_DOMAIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity doesn't have an valid domain ID]]>
+      </doc>
+    </field>
+    <field name="FORBIDDEN_RELATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to relate the entity to another
+ one in different domain]]>
+      </doc>
+    </field>
+    <field name="EXPIRED_ENTITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity start time is before the eviction
+ period of old data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timelineservice">
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <class name="ApplicationAttemptEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <class name="ApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="isApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an ApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an ApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getApplicationEvent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <param name="eventId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param te TimelineEntity object.
+ @param eventId event with this id needs to be fetched
+ @return TimelineEvent if TimelineEntity contains the desired event.]]>
+      </doc>
+    </method>
+    <field name="QUEUE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents an application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <class name="ClusterEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClusterEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a YARN cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <class name="ContainerEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a container belonging to an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <class name="FlowActivityEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowActivityEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="java.lang.String, long, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reuse the base class equals method.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reuse the base class hashCode method.]]>
+      </doc>
+    </method>
+    <method name="getCluster" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cluster" type="java.lang.String"/>
+    </method>
+    <method name="getDate" return="java.util.Date"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFlowName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+    </method>
+    <method name="addFlowRun"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="run" type="org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity"/>
+    </method>
+    <method name="addFlowRuns"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runs" type="java.util.Collection"/>
+    </method>
+    <method name="getFlowRuns" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNumberOfRuns" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLUSTER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Entity that represents a record for flow activity. It's essentially a
+ container entity for flow runs with limited information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <class name="FlowRunEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowRunEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowRunEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="java.lang.String"/>
+    </method>
+    <method name="getRunId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRunId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runId" type="long"/>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+    </method>
+    <method name="getMaxEndTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxEndTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+    </method>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_END_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a flow run.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <class name="HierarchicalTimelineEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getParent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getChildren" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="child" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <field name="PARENT_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CHILDREN_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends timeline entity and defines parent-child relationships
+ with other entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <class name="QueueEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QueueEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a queue.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <class name="SubApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSubApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an SubApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an SubApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="java.lang.String"/>
+    </method>
+    <field name="YARN_APPLICATION_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a user defined entities to be stored under sub
+ application table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID.
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID.
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description.
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description.
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner.
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string.
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string.
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string.
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string.
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain.
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain.
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain.
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain.
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline service domain, which is
+ used to a user to host a number of timeline entities, isolating them from
+ others'. The user can also define the reader and writer users/groups for
+ the domain, which is used to control the access to its entities.
+ </p>
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[This class hosts a set of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ The constuctor is used to construct a proxy {@link TimelineEntity} or its
+ subclass object from the real entity object that carries information.
+ </p>
+
+ <p>
+ It is usually used in the case where we want to recover class polymorphism
+ after deserializing the entity from its JSON form.
+ </p>
+ @param entity the real entity that carries information]]>
+      </doc>
+    </constructor>
+    <constructor name="TimelineEntity" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getIdentifier" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdentifier" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getConfigs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="getMetrics" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+    </method>
+    <method name="getEvents" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.NavigableSet"/>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.Set"/>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <method name="getIsRelatedToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getRelatesToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="addRelatesToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="setRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTs" type="java.lang.Long"/>
+    </method>
+    <method name="setUID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uidKey" type="java.lang.String"/>
+      <param name="uId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set UID in info which will be then used for query by UI.
+ @param uidKey key for UID in info.
+ @param uId UID to be set for the key.]]>
+      </doc>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="getReal" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdPrefix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdPrefix"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdPrefix" type="long"/>
+      <doc>
+      <![CDATA[Sets idPrefix for an entity.
+ <p>
+ <b>Note</b>: Entities will be stored in the order of idPrefix specified.
+ If users decide to set idPrefix for an entity, they <b>MUST</b> provide
+ the same prefix for every update of this entity.
+ </p>
+ Example: <blockquote><pre>
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+ </pre></blockquote>
+ Users can use {@link TimelineServiceHelper#invertLong(long)} to invert
+ the prefix if necessary.
+
+ @param entityIdPrefix prefix for an entity.]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_INFO_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENTITY_PREFIX" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The basic timeline entity data structure for timeline service v2. Timeline
+ entity objects are not thread safe and should not be accessed concurrently.
+ All collection members will be initialized into empty collections. Two
+ timeline entities are equal iff. their type and id are identical.
+
+ All non-primitive type, non-collection members will be initialized into null.
+ User should set the type and id of a timeline entity to make it valid (can be
+ checked by using the {@link #isValid()} method). Callers to the getters
+ should perform null checks for non-primitive type, non-collection members.
+
+ Callers are recommended not to alter the returned collection objects from the
+ getters.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <class name="TimelineEntityType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isParent" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a parent of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is parent of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="isChild" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a child of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is child of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="matches" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="typeString" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether the type of this entity matches the type indicated by the input
+ argument.
+
+ @param typeString entity type represented as a string.
+ @return true, if string representation of this entity type matches the
+     entity type passed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Defines type of entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventId" type="java.lang.String"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ts" type="long"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <field name="INVALID_TIMESTAMP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains the information of an event that belongs to an entity.
+ Users are free to define what the event means, such as starting an
+ application, container being allocated, etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <class name="TimelineMetric" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricType" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricId" type="java.lang.String"/>
+    </method>
+    <method name="getRealtimeAggregationOp" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the real time aggregation operation of this metric.
+
+ @return Real time aggregation operation]]>
+      </doc>
+    </method>
+    <method name="setRealtimeAggregationOp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"/>
+      <doc>
+      <![CDATA[Set the real time aggregation operation of this metric.
+
+ @param op A timeline metric operation that the metric should perform on
+           real time aggregations]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <param name="value" type="java.lang.Number"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLatestSingleValueMetric" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Get the latest timeline metric as single value type.
+
+ @param metric Incoming timeline metric
+ @return The latest metric in the incoming metric]]>
+      </doc>
+    </method>
+    <method name="getSingleDataTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data timestamp of the metric.
+
+ @return the single data timestamp]]>
+      </doc>
+    </method>
+    <method name="getSingleDataValue" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data value of the metric.
+
+ @return the single data value]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state in a stateless fashion. The assumption here is
+ baseAggregatedMetric and latestMetric should be single value data if not
+ null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="state" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state. The assumption here is baseAggregatedMetric and
+ latestMetric should be single value data if not null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @param state Operation state
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains the information of a metric that is related to some
+ entity. Metric can either be a time series or single value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <class name="TimelineWriteResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelineWriteError} instances.
+
+ @return a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelineWriteError} instance into the existing list.
+
+ @param error
+          a single {@link TimelineWriteError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelineWriteError} instances into the existing list.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelineWriteError} instances.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are
+ errors in storing individual entity objects, they will be indicated in the
+ list of errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <class name="TimelineWriteResponse.TimelineWriteError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id.
+
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id.
+
+ @param id the entity Id.]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type.
+
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type.
+
+ @param type the entity type.]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code.
+
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code.
+
+ @param code an error code.]]>
+      </doc>
+    </method>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when storing an
+ entity.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+  <class name="UserEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UserEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="UserEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+</package>
+<package name="org.apache.hadoop.yarn.api.resource">
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <class name="PlacementConstraint" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PlacementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConstraintExpr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the constraint expression of the placement constraint.
+
+ @return the constraint expression]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code PlacementConstraint} represents a placement constraint for a resource
+ allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+  <class name="PlacementConstraints" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="targetIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ satisfy all target expressions within the given scope (e.g., node or rack).
+
+ For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows
+ allocations on nodes that belong to a rack that has at least one tag with
+ value "hbase-m".
+
+ @param scope the scope within which the target expressions should be
+          satisfied
+ @param targetExpressions the expressions that need to be satisfied within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNotIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that does not satisfy any of the
+ target expressions.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNodeAttribute" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="opCode" type="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that satisfy any of the
+ target expressions based on node attribute op code.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param opCode Node Attribute code which could be equals, not equals.
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that restricts the number of allocations within a
+ given scope (e.g., node or rack).
+
+ For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes
+ where there are no less than 3 allocations with tag "zk" and no more than
+ 10.
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but let you
+ attach a namespace to the given allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of the allocation tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param maxCardinality determines the maximum number of allocations within
+                       the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the minimum cardinality (the maximum cardinality is
+ unbound).
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #minCardinality(String, int, String...)}, but let you
+ attach a namespace to the allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of these tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the maximum cardinality (the minimum cardinality is 0).
+
+ @param scope the scope of the constraint
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="tagNamespace" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #maxCardinality(String, int, String...)}, but let you
+ specify a namespace for the tags, see supported namespaces in
+ {@link AllocationTagNamespaceType}.
+
+ @param scope the scope of the constraint
+ @param tagNamespace the namespace of these tags
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[This constraint generalizes the cardinality and target constraints.
+
+ Consider a set of nodes N that belongs to the scope specified in the
+ constraint. If the target expressions are satisfied at least minCardinality
+ times and at most maxCardinality times in the node set N, then the
+ constraint is satisfied.
+
+ For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
+ requires an allocation to be placed within a rack that has at least 2 and
+ at most 10 other allocations with tag "zk".
+
+ @param scope the scope of the constraint
+ @param minCardinality the minimum number of times the target expressions
+          have to be satisfied with the given scope
+ @param maxCardinality the maximum number of times the target expressions
+          have to be satisfied with the given scope
+ @param targetExpressions the target expressions
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="and" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.And"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A conjunction of constraints.
+
+ @param children the children constraints that should all be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="or" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A disjunction of constraints.
+
+ @param children the children constraints, one of which should be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="delayedOr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint[]"/>
+      <doc>
+      <![CDATA[Creates a composite constraint that includes a list of timed placement
+ constraints. The scheduler should try to satisfy first the first timed
+ child constraint within the specified time window. If this is not possible,
+ it should attempt to satisfy the second, and so on.
+
+ @param children the timed children constraints
+ @return the resulting composite constraint]]>
+      </doc>
+    </method>
+    <method name="timedClockConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <param name="timeUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a time
+ window.
+
+ @param constraint the placement constraint
+ @param delay the length of the time window within which the constraint has
+          to be satisfied
+ @param timeUnit the unit of time of the time window
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="timedOpportunitiesConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a number of
+ placement opportunities (invocations of the scheduler).
+
+ @param constraint the placement constraint
+ @param delay the number of scheduling opportunities within which the
+          constraint has to be satisfied
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintExpr" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <doc>
+      <![CDATA[Creates a {@link PlacementConstraint} given a constraint expression.
+
+ @param constraintExpr the constraint expression
+ @return the placement constraint]]>
+      </doc>
+    </method>
+    <field name="NODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains various static methods for the applications to create
+ placement constraints (see also {@link PlacementConstraint}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+</package>
+<package name="org.apache.hadoop.yarn.conf">
+  <!-- start class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+  <class name="YarnConfiguration" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnConfiguration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isAclEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>. On an HA cluster,
+ this fetches the address corresponding to the RM identified by
+ {@link #RM_HA_ID}.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+    </method>
+    <method name="useHttps" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="shouldRMFailFast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isDistSchedulingEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOpportunisticContainerAllocationEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV2Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.2 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.2 is enabled. V.2 refers to a
+ version greater than equal to 2 but smaller than 3.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1 is enabled. V.1 refers to a
+ version greater than equal to 1 but smaller than 2.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV15Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1,5 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="systemMetricsPublisherEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the system publisher is enabled.
+
+ @param conf the configuration
+ @return whether the system publisher is enabled.]]>
+      </doc>
+    </method>
+    <method name="numaAwarenessEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the NUMA awareness is enabled.
+
+ @param conf the configuration
+ @return whether the NUMA awareness is enabled.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="DR_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CS_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_POLICY_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CORE_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_RESOURCES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEBUG_NM_DELETE_DELAY_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay before deleting resource to ease debugging of NM issues]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_CONTAINER_DEBUG_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create client IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create server IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create serializable records.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RPC class implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HOSTNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_EPOCH" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH_RANGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The epoch range before wrap around. 0 disables wrap around]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_EPOCH_RANGE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the applications manager interface in the RM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of application tags.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAG_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of each application tag.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MASTER_SERVICE_PROCESSORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AUTO_UPDATE_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AUTO_UPDATE_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the RM.]]>
+      </doc>
+    </field>
+    <field name="RM_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle applications manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AMLAUNCHER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to launch/cleanup AM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AMLAUNCHER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_CONNECT_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retry times to connect with NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_CONNECT_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the port should be included or not in the node name. The node name
+ is used by the scheduler for resource requests allocation location 
+ matching. Typically this is just the hostname, using the port is needed
+ when using minicluster and specific NM are required.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUEUE_PLACEMENT_RULES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configured scheduler queue placement rules.]]>
+      </doc>
+    </field>
+    <field name="USER_GROUP_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[UserGroupMappingPlacementRule configuration string.]]>
+      </doc>
+    </field>
+    <field name="APP_NAME_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_UI_ACTIONS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Resource Manager webapp ui actions]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the RM should enable Reservation System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The PlanFollower for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The step size of the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum periodicity for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_ENABLE_MONITORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable periodic monitor threads.
+ @see #RM_SCHEDULER_MONITOR_POLICIES]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ENABLE_MONITORS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MONITOR_POLICIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of SchedulingEditPolicy classes affecting the scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SSL_SERVER_RESOURCE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable YARN WebApp V2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_WEBAPP_UI2_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_WARFILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_API_SERVICES_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_API_SYSTEM_SERVICES_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for application master reporting.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a node manager is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_RESERVATION_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are reservation acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_RESERVATION_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL of who can be admin of YARN cluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_APP_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL used in case none is found. Allows nothing.]]>
+      </doc>
+    </field>
+    <field name="OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether opportunistic container allocation
+  is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_MAX_ALLOCATIONS_PER_AM_HEARTBEAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of opportunistic containers to be allocated in
+ AM heartbeat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_MAX_ALLOCATIONS_PER_AM_HEARTBEAT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of nodes to be used by the Opportunistic Container allocator for
+ dispatching containers during container allocation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency for computing least loaded NMs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comparator for determining node load for scheduling of opportunistic
+ containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LIMIT_STDEV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value of standard deviation used for calculation of queue limit
+ thresholds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the lower-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the upper-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min queue wait time for a container at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue wait time for a container queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_INTERMEDIATE_DATA_ENCRYPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable intermediate-data encryption at YARN level. For now, this
+ only is used by the FileSystemRMStateStore to setup right file-system
+ security attributes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts for
+ an application, if unset by user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GLOBAL_RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts.
+ It's a global setting for all application masters.]]>
+      </doc>
+    </field>
+    <field name="RM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The keytab for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing). Only applicable
+ when the http authentication type is kerberos.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cross origin (CORS) support.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a container is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to include.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable submission pre-processor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with hosts for the submission processor to handle.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submission processor refresh interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to exclude.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle resource tracker calls.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_REGISTRATION_IP_HOSTNAME_CHECK_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check IP and hostname resolution during nodemanager registration.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_REGISTRATION_IP_HOSTNAME_CHECK_KEY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the resource scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specify which handler will be used to process PlacementConstraints.
+ For details on PlacementConstraints, please refer to
+ {@link org.apache.hadoop.yarn.api.resource.PlacementConstraint}]]>
+      </doc>
+    </field>
+    <field name="DISABLED_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This handler rejects all allocate calls made by an application, if they
+ contain a {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}.]]>
+      </doc>
+    </field>
+    <field name="PROCESSOR_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, the placement of containers with constraints is
+ determined as a pre-processing step before the capacity or the fair
+ scheduler is called. Once the placement is decided, the capacity/fair
+ scheduler is invoked to perform the actual allocation. The advantage of
+ this approach is that it supports all constraint types (affinity,
+ anti-affinity, cardinality). Moreover, it considers multiple containers at
+ a time, which allows to satisfy more constraints than a container-at-a-time
+ approach can achieve. As it sits outside the main scheduler, it can be used
+ by both the capacity and fair schedulers. Note that at the moment it does
+ not account for task priorities within an application, given that such
+ priorities might be conflicting with the placement constraints.]]>
+      </doc>
+    </field>
+    <field name="SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, containers with constraints will be placed by the main
+ scheduler. If the configured RM scheduler
+ <pre>yarn.resourcemanager.scheduler.class</pre>
+ cannot handle placement constraints, the corresponding SchedulingRequests
+ will be rejected. As of now, only the capacity scheduler supports
+ SchedulingRequests. In particular, it currently supports anti-affinity
+ constraints (no affinity or cardinality) and places one container at a
+ time. The advantage of this handler compared to the placement-processor is
+ that it follows the same ordering rules for queues (sorted by utilization,
+ priority) and apps (sorted by FIFO/fairness/priority) as the ones followed
+ by the main scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Placement Algorithm.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Used for BasicPlacementAlgorithm - default SERIAL.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM set next Heartbeat interval for NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Heartbeat Interval Scaling based on cpu utilization.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of worker threads that write the history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM. This configuration setting is for ATS V1.
+  This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM and NM. This configuration setting is for
+  ATS v2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by RM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_CONF_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_RETRY_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_RETRY_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper interaction configs]]>
+      </doc>
+    </field>
+    <field name="RM_ZK_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_RM_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper znode limit]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_RETRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_TIMEOUT_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_APPID_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_APPID_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Index at which the RM Delegation Token ids will be split so that the
+ delegation token znodes stored in the zookeeper RM state store will be
+ stored as two different znodes (parent-child).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_AUTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_ROOT_NODE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Root node ACLs for fencing]]>
+      </doc>
+    </field>
+    <field name="RM_HA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[HA related configs]]>
+      </doc>
+    </field>
+    <field name="RM_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Store the related configuration files in File System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILE_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LEVELDB_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_MAX_LOGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKConfigurationStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_AUTHORIZATION_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_EMBEDDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This property should never be set to {@code false}.">
+      <doc>
+      <![CDATA[This property controls whether {@link ActiveStandbyElector} leader
+ election should be used when {@link #CURATOR_LEADER_ELECTOR} is
+ {@code false}.
+
+ @deprecated This property should never be set to {@code false}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_EMBEDDED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.">
+      <doc>
+      <![CDATA[The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
+
+ @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.]]>
+      </doc>
+    </field>
+    <field name="AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_NO_HA_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_NO_HA_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_BASE_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_FC_ELECTOR_ZK_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of zookeeper operation retry times in ActiveStandbyElector]]>
+      </doc>
+    </field>
+    <field name="CURATOR_LEADER_ELECTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.">
+      <doc>
+      <![CDATA[Whether to use the Curator-based elector for leader election.
+
+ @deprecated Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the persistent store.]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemRMStateStore]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the leveldb database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM keeps. By default equals
+ to {@link #DEFAULT_RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM state store keeps. By
+ default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.">
+      <doc>
+      <![CDATA[The default value for
+ {@code yarn.resourcemanager.state-store.max-completed-applications}.
+ @deprecated This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application name]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_TYPE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type length]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default queue name]]>
+      </doc>
+    </field>
+    <field name="RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Buckets (in minutes) for the number of apps running in each queue.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default sizes of the runtime metric buckets in minutes.]]>
+      </doc>
+    </field>
+    <field name="RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout(msec) for an untracked node to remain in shutdown or decommissioned
+ state.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM proxy users' prefix]]>
+      </doc>
+    </field>
+    <field name="RM_RESOURCE_PROFILES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File containing resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable loading resource-types.xml at client side.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications
+ to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Period in seconds of the poll timer task inside DecommissioningNodesWatcher
+ to identify and take care of DECOMMISSIONING nodes missing regular heart beat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for all node manager configs.]]>
+      </doc>
+    </field>
+    <field name="NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIST_SCHEDULING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether distributed scheduling is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DIST_SCHEDULING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that will be sent to containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[PATH components that will be prepended to the user's path.
+ If this is defined and the user does not define PATH, NM will also
+ append ":$PATH" to prevent this from eclipsing the PATH defined in
+ the container. This feature is only available for Linux.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that containers may override rather than use NodeManager's default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[address of node manager IPC.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the NM.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[who will execute(launch) the containers.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_STATE_TRANSITION_LISTENERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of container state transition listeners.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adjustment to make to the container os scheduling priority.
+ The valid values for this could vary depending on the platform.
+ On Linux, higher values mean run the containers at a less 
+ favorable priority than the NM. 
+ The value specified is an int.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MGR_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DELETE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in cleanup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_EXIT_FILE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_EXIT_FILE_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of directories to store localized files in.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of files in each localized directories
+ Avoid tuning this too low.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the localizer IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the collector service IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by NM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval in between cache cleanups.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target size of localizer cache in MB, per nodemanager. It is a target
+ retention size that only includes resources with PUBLIC and PRIVATE
+ visibility and excludes resources with APPLICATION visibility]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle localization requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_FETCH_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to use for localization fetching.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to store container logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle log aggregation in node manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permissions for container logs.]]>
+      </doc>
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk Validator.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum size of contain's diagnostics to keep for relaunching container
+ case.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum container restart interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which the delayed token removal thread runs]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegation Token renewer thread count]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PRIVILEGES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for node IP caching. -1 disables the caching]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How many diagnostics/failure messages can be saved in RM for
+ log aggregation. It also defines the number of diagnostics/failure
+ messages can be shown in log aggregation web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to enable log aggregation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_CONTROLLER_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait before deleting aggregated logs, -1 disables.
+ Be careful set this too small and you will spam the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_DEBUG_FILESIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait between aggregated log retention checks. If set to
+ a value {@literal <=} 0 then the value is computed as one-tenth of the
+ log retention setting. Be careful set this too small and you will spam
+ the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation status
+ is reported from NodeManager exceeds the configured value, RM will report
+ log aggregation status for this NodeManager as TIME_OUT.
+
+ This configuration will be used in NodeManager as well to decide
+ whether and when to delete the cached log aggregation status.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seconds to retain logs on the NodeManager. Only applicable if Log
+ aggregation is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how often NMs wake up and upload log files]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed hard minimum limit for {@link
+ YarnConfiguration#NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS}.]]>
+      </doc>
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how many aggregated log files per application per NM we can have
+ in remote file system.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DELETION_THREADS_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in log cleanup. Only applicable if Log aggregation
+ is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to aggregate logs to.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The remote log dir will be created at below location.
+ NM_REMOTE_APP_LOG_DIR/${user}/bucket_{NM_REMOTE_APP_LOG_DIR_SUFFIX}
+ /${bucketDir}/${appId}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether Older Application Log Directory is included.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_GROUPNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the group of the aggregated log directory.]]>
+      </doc>
+    </field>
+    <field name="YARN_LOG_SERVER_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_WEBSERVICE_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_TRACKING_URL_GENERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SYSTEM_RESERVED_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that has been reserved for non-yarn use.]]>
+      </doc>
+    </field>
+    <field name="NM_PMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether virtual memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_PMEM_RATIO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Conversion ratio for physical memory to virtual memory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_PMEM_RATIO" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether to do memory check on overall usage.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the OOM handler code.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the OOM listener.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum time in seconds to resolve an OOM situation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.Integer"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of Virtual CPU Cores which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count logical processors(like hyperthreads) as cores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PCORES_VCORES_MULTIPLIER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Multiplier to convert physical cores to vcores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PCORES_VCORES_MULTIPLIER" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Percentage of overall CPU which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable or disable node hardware capability detection.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENFORCED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENFORCED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cpu isolation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CPU_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for disk configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for disk operations is enabled.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for network bandwidth is
+ enabled. Work in progress: This configuration parameter may be
+ changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Network as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the interface to be used for applying network throttling rules.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total available outbound bandwidth on the node. Work in
+ progress: This configuration parameter may be changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total outbound bandwidth available to YARN containers.
+ defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for computation resources, example of computation resources like
+ GPU / FPGA / TPU, etc.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether the initialization of the Node Manager should continue
+ if a certain device (GPU, FPGA, etc) was not found in the system. If set
+ to "true", then an exception will be thrown if a device is missing or
+ an error occurred during discovery.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PLUGINS_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if pluggable device plugin framework is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The pluggable device plugin framework is disabled by default]]>
+      </doc>
+    </field>
+    <field name="NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting contains vendor plugin class names for
+ device plugin framework to load. Split by comma]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for gpu configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTOMATICALLY_DISCOVER_GPU_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_GPU_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls where to how to invoke GPU binaries]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings to control which implementation of docker plugin for GPU will be
+ used.
+
+ By default uses NVIDIA docker v1.]]>
+      </doc>
+    </field>
+    <field name="NVIDIA_DOCKER_V1" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_V2" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls end point of nvidia-docker-v1 plugin.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for FPGA configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_FPGA_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_DEVICE_DISCOVERY_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_AVAILABLE_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_HANDLER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_TAG_HANDLER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp https address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable CORS filter.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor resource in a node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates containers current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_PROCESS_TREE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates process tree resource utilization.]]>
+      </doc>
+    </field>
+    <field name="PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_TAG_BASED_PLACEMENT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable switch for container log monitoring.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor logs generated by containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for a single container log directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for all of a container's logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable container metrics.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_PERIOD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container metrics flush period. -1 for flush on completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The delay time ms to unregister container metrics after completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable disks' health checker. Default is true. An expert level
+ configuration property.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running disks' health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, disks' health is checked every 2 minutes.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_HEALTHY_DISKS_FRACTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum fraction of number of disks to be healthy for the nodemanager
+ to launch new containers. This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, at least 25% of disks are to be healthy to say that the node is
+ healthy in terms of disks.]]>
+      </doc>
+    </field>
+    <field name="NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum percentage of disk space that can be used after which a disk is
+ marked as offline. Values can range from 0.0 to 100.0. If the value is
+ greater than or equal to 100, NM will check for full disk. This applies to
+ nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-utilization-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, 90% of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the disk utilisation percentage
+ threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The low threshold percentage of disk space used when an offline disk is
+ marked as online. Values can range from 0.0 to 100.0. The value shouldn't
+ be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is
+ more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be
+ set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on a local dir for it to be used.
+ This applies to nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-free-space-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the minimum disk free
+ space threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_HIGH_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on an offline
+ disk for it to be marked as online.  The value should not be less
+ than NM_MIN_PER_DISK_FREE_SPACE_MB.  If its value is less than
+ NM_MIN_PER_DISK_FREE_SPACE_MB or is not set, it will be set to the
+ same value as NM_MIN_PER_DISK_FREE_SPACE_MB.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, all of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health checker scripts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_SCRIPTS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not to run the node health script before the NM
+  starts up.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check time out period for all scripts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check script time out period.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_PATH_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health check script to run.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_OPTS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The arguments to pass to the health check script.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_INTERVAL_MS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JVM options used on forking ContainerLocalizer process
+      by container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log level of container localizer process.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for runtime configuration constants.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The standard values are:
+ <ul>
+   <li>default</li>
+   <li>docker</li>
+   <li>javasandbox</li>
+   <li>runc</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_CLASS_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of allowed runtimes when using LinuxContainerExecutor.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runtime to be used.]]>
+      </doc>
+    </field>
+    <field name="RUNC_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The runc image tag to manifest plugin class that should be used.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc image tag to manifest plugin class.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The runc manifest to resources plugin class that should be used.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc manifest to resources plugin plugin class.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_IMAGE_TOPLEVEL_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The HDFS location under which the oci image manifests, layers,
+ and configs directories exist.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_IMAGE_TOPLEVEL_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default HDFS location under which the oci image manifests, layers,
+ and configs directories exist.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_LAYER_MOUNTS_TO_KEEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target count of layer mounts that we should keep on disk at one time.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval in seconds between executions of reaping layer mounts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RUNC_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc image to be used.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default set of networks allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default set of runtimes allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOW_HOST_PID_NAMESPACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow host pid namespace for containers. Use with care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOW_HOST_PID_NAMESPACE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Host pid namespace for containers is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_DEFAULT_RO_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-only mounts to be bind-mounted into all
+  runC containers that use RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_DEFAULT_RW_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-write mounts to be bind-mounted into all
+  runC containers that use RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_SECCOMP_PROFILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to the seccomp profile to use with Runc containers.]]>
+      </doc>
+    </field>
+    <field name="NM_HDFS_RUNC_IMAGE_TAG_TO_HASH_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The HDFS location where the runC image tag to hash file exists.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_RUNC_IMAGE_TAG_TO_HASH_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The local file system location where the runC image tag to hash file exists.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_CACHE_REFRESH_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval in seconds between refreshing the hdfs image tag to
+ hash cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_CACHE_REFRESH_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default interval in seconds between refreshing the hdfs image tag to
+ hash cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_NUM_MANIFESTS_TO_CACHE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of manifests to cache in the image tag to hash cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NUM_MANIFESTS_TO_CACHE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default number of manifests to cache in the image tag to hash cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_STAT_CACHE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The timeout value in seconds for the values in the stat cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_STAT_CACHE_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default timeout value in seconds for the values in the stat cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_STAT_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The size of the stat cache which stores stats of the layers and config.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RUNC_STAT_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default size of the stat cache which stores stats of the
+ layers and config.]]>
+      </doc>
+    </field>
+    <field name="DOCKER_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default docker image to be used.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_IMAGE_UPDATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default option to decide whether to pull the latest image or not.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Capabilities allowed (and added by default) for docker containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[These are the default capabilities added by docker. We'll use the same
+ set here. While these may not be case-sensitive from a docker
+ perspective, it is best to keep these uppercase.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ENABLE_USER_REMAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[enable user remapping.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set enable user remapping as false by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable uids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower uid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable gids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower gid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request. This
+  network must be one of the (configurable) set of allowed container
+  networks.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request and
+ no default network is configured.
+ .]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow host pid namespace for containers. Use with care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Host pid namespace for containers is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="YARN_HTTP_WEBAPP_EXTERNAL_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_WEBAPP_SCHEDULER_PAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This field is deprecated for
+ {@link #YARN_HTTP_WEBAPP_SCHEDULER_PAGE}">
+      <doc>
+      <![CDATA[@deprecated This field is deprecated for
+ {@link #YARN_HTTP_WEBAPP_SCHEDULER_PAGE}]]>
+      </doc>
+    </field>
+    <field name="YARN_HTTP_WEBAPP_SCHEDULER_PAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not users are allowed to request that Docker containers honor
+ the debug deletion delay. This is useful for troubleshooting Docker
+ container related launch failures.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value on whether or not a user can request that Docker
+ containers honor the debug deletion delay.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_STOP_GRACE_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}">
+      <doc>
+      <![CDATA[A configurable value to pass to the Docker Stop command. This value
+ defines the number of seconds between the docker stop command sending
+ a SIGTERM and a SIGKILL.
+
+ @deprecated use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for the grace period between the SIGTERM and the
+ SIGKILL in the Docker Stop command.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RO_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-only mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RW_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-write mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_TMPFS_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of tmpfs mounts to be mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The mode in which the Java Container Sandbox should run detailed by
+  the JavaSandboxLinuxContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for application local directories.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Location for non-default java policy file.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for group to policy file mapping.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_WHITELIST_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The group which will run by default without the java security manager.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_EXECUTOR_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the Linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_NONSECURE_MODE_LIMIT_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if linux-container-executor should limit itself to one user
+ when running in non-secure mode.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_LOCAL_USER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The UNIX user that containers will run as when Linux-container-executor
+ is used in nonsecure mode (a use case for this is using cgroups).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LOCAL_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_USER_PATTERN_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed pattern for UNIX user names enforced by 
+ Linux-container-executor when used in nonsecure mode (use case for this 
+ is using cgroups). The default value is taken from /usr/sbin/adduser]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_USER_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_RESOURCES_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The type of resource enforcement to use with the
+  linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_HIERARCHY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path the linux container executor should use for cgroups]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the apps should run in strict resource usage mode(not allowed to
+ use spare CPU)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_HTTPS_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies what the RM does regarding HTTPS enforcement for communication
+ with AM Web Servers, as well as generating and providing certificates.
+ Possible values are:
+ <ul>
+   <li>NONE - the RM will do nothing special.</li>
+   <li>LENIENT - the RM will generate and provide a keystore and truststore
+   to the AM, which it is free to use for HTTPS in its tracking URL web
+   server.  The RM proxy will still allow HTTP connections to AMs that opt
+   not to use HTTPS.</li>
+   <li>STRICT - this is the same as LENIENT, except that the RM proxy will
+   only allow HTTPS connections to AMs; HTTP connections will be blocked
+   and result in a warning page to the user.</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_HTTPS_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_TIMEOUT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFALUT_RM_PROXY_TIMEOUT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_CONNECTION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_CONNECTION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container. This is required due to what 
+ it seems a race condition because the SIGTERM/SIGKILL is asynch.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay between attempts to remove linux cgroup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if memory and CPU limits will be set for the Windows Job
+ Object for the containers launched by the default container executor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_SECURE_CONTAINER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[/* The Windows group that the windows-secure-container-executor should run as.]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress aggregated logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the node manager.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Boolean indicating whether loading aux services from a manifest is
+ enabled. If enabled, aux services may be dynamically modified through
+ reloading the manifest via filesystem changes or a REST API. When
+ enabled, aux services configuration properties unrelated to the manifest
+ will be ignored.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File containing auxiliary service specifications.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST_RELOAD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which manifest file will be reloaded when modifications are
+ found (0 or less means that the file will not be checked for modifications
+ and reloaded).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_REMOTE_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the NM state database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_SUPERVISED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_SUPERVISED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS_PARAMETERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for Proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Binding address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[YARN Service Level Authorization]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[No. of milliseconds to wait between sending a SIGTERM and SIGKILL
+ to a running container]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROCESS_KILL_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for a process to come up when trying to cleanup
+ container resources]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PROCESS_KILL_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The threshold used to trigger the logging of event types and counts
+  in RM's main event dispatcher. Default value is 5000,
+  which means RM will print events info when the queue size cumulatively
+  reaches 5000 every time. Such info can be used to reveal what
+  kind of events that RM is stuck at processing mostly,
+  it can help to narrow down certain performance issues.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resource manager dispatcher thread monitor sampling rate.
+ Units are samples per minute.  This controls how often to sample
+ the cpu utilization of the resource manager dispatcher thread.
+ The cpu utilization is displayed on the RM UI as scheduler busy %.
+ Set to zero to disable the dispatcher thread monitor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APPLICATION_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
+ entries]]>
+      </doc>
+    </field>
+    <field name="AMRM_PROXY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether AMRMProxy is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default platform-agnostic CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries. The parameter expansion marker
+ will be replaced with real parameter expansion marker ('%' for Windows and
+ '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+ will be replaced as $VAR on Linux, and %VAR% on Windows.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Default platform-specific CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries constructed based on the client
+ OS environment expansion syntax.
+ </p>
+ <p>
+ Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.
+ </p>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CONTAINER_TEMP_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container temp directory]]>
+      </doc>
+    </field>
+    <field name="IS_MINI_YARN_CLUSTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_FIXED_PORTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to use fixed ports with the minicluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_FIXED_PORTS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default is false to be able to run tests concurrently without port
+ conflicts.]]>
+      </doc>
+    </field>
+    <field name="YARN_MINICLUSTER_USE_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the NM should use RPC to connect to the RM. Default is false.
+ Can be set to true only when using fixed ports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_USE_RPC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether users are explicitly trying to control resource monitoring
+ configuration for the MiniYARNCluster. Disabled by default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow changing the memory for the NodeManager in the MiniYARNCluster]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log directory for the containers]]>
+      </doc>
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_BACKUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_VERSION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_UI_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of names for UIs hosted in the timeline server
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_WEB_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Relative web path that will serve up this UI (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client settings.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to war file or static content directory for this UI
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting for timeline service v1.5]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline service v2.0.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default schema prefix for hbase tables.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[config param name to override schema prefix.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how often the timeline collector flushes the
+ timeline writer.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls the capacity of the queue for async writes
+ to timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_FINAL_VALUE_RETENTION_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how long the final value of
+ a metric of a completed app is retained before merging
+ into the flow sum.]]>
+      </doc>
+    </field>
+    <field name="FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the setting for the location of the coprocessor
+ jar on hdfs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default hdfs location for flowrun coprocessor jar.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls the max size of the flow name getting generated
+ in ATSv2 after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_DEFAULT_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default setting for flow name size has no size restriction
+ after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that points to an optional HBase configuration
+ (hbase-site.xml file) with settings that will override the ones found on
+ the classpath.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that enables or disables authentication checks
+ for reading timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how often in-memory app level
+ aggregation is kicked off in timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.Boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default setting for authentication checks for reading timeline
+ service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that lists the users and groups who are allowed
+ to read timeline service v2 data. It is a comma separated list of
+ user, followed by space, then comma separated list of groups.
+ It will allow this list of users and groups to read the data
+ and reject everyone else.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for list of the users who are allowed to read
+ timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how long the final value of a metric of a
+ completed app is retained before merging into the flow sum. Up to this time
+ after an application is completed out-of-order values that arrive can be
+ recognized and discarded at the cost of increased storage.]]>
+      </doc>
+    </field>
+    <field name="ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default version for any flow.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time period for which timeline v2 client will wait for draining
+ leftover entities after stop.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether application history service is
+  enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Application history store class]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Save container meta-info in the application history store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemApplicationHistoryStore]]>
+      </doc>
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether timeline service is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[host:port address for timeline service RPC APIs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The listening endpoint for the timeline service application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_HANDLER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle client RPC API requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_MAX_APPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defines the max number of applications could be fetched using
+ REST API or application history protocol and shown in timeline
+ server web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_MAX_APPS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service store class.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service enable data age off]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service length of time to retain data]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service rolling period. Valid values are daily, half_daily,
+ quarter_daily, and hourly.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Roll a new database each hour.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Implementation specific configuration prefix for Timeline Service
+ leveldb.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb read cache (uncompressed blocks). This is
+ per rolling instance so should be tuned if using rolling leveldb
+ timeline store]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb read cache size if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write buffer size.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write buffer size if no configuration is specified. This
+ is per rolling instance so should be tuned if using rolling leveldb
+ timeline store.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write batch size. This value can be tuned down
+ to reduce lock time for ttl eviction.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write batch size is no configuration is specified]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time read cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time write cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb interval to wait between deletion rounds]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb number of concurrent open files. Tuned this
+ configuration to stay within system limits. This is per rolling instance
+ so should be tuned if using rolling leveldb timeline store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb max open files if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos keytab for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enables cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, max retries (-1 means no limit)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, retry interval]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client policy for whether connections are fatal]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable recovery of timeline service]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STATE_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store class]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_OFFLINE_AGGREGATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[whether the shared cache is enabled/disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the shared cache root directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NESTED_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the level of nested directories before getting to the
+ checksum directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NESTED_LEVEL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STALENESS_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A resource in the InMemorySCMStore is considered stale if the time since
+ the last reference exceeds the staleness period. This value is specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the in-memory store runs its first check to remove
+ dead initial applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_CHECK_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which the in-memory store checks to remove dead initial
+ applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which a cleaner task runs. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the first cleaner task is scheduled. Specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_RESOURCE_SLEEP_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time to sleep between processing each shared cache resource. Specified
+ in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the node manager interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of SCM threads used to handle notify requests from the node
+ manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the client interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle shared cache manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the checksum algorithm implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The replication factor for the node manager uploader for the shared cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_MACHINE_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_USERNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_PASSWORD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PIPELINE_CACHE_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_SUBMIT_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of retries for GetNewApplication and SubmitApplication in
+ {@code FederationClientInterceptor}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_USER_CLIENT_THREADS_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST to communicate with
+ each SubCluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST should return
+ partial AppReports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[TERMS:
+ csi-driver: a 3rd party CSI driver which implements the CSI protocol.
+   It is provided by the storage system.
+ csi-driver-adaptor: this is an internal RPC service working
+   as a bridge between YARN and a csi-driver.]]>
+      </doc>
+    </field>
+    <field name="NM_CSI_DRIVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_DRIVER_ENDPOINT_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_ADDRESS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_ADDRESSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[One or more socket addresses for csi-adaptor.
+ Multiple addresses are delimited by ",".]]>
+      </doc>
+    </field>
+    <field name="NM_CSI_DRIVER_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
+ The interval of the yarn client's querying application state after
+ application submission. The unit is millisecond.]]>
+      </doc>
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval that the yarn client library uses to poll the completion
+ status of the asynchronous API of application client protocol.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The duration that the yarn client library waits, cumulatively across polls,
+ for an expected state change to occur. Defaults to -1, which indicates no
+ limit.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of threads in NMClientAsync to process container management
+ events]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_MAX_NM_PROXIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of proxy connections to cache for node managers. If set
+ to a value greater than zero then the cache is enabled and the NMClient
+ and MRAppMaster will cache the specified number of node manager proxies.
+ There will be at max one proxy per node manager. Ex. configuring it to a
+ value of 5 will make sure that client will at max have 5 proxies cached
+ with 5 different node managers. These connections for these proxies will
+ be timed out if idle for more than the system wide idle timeout period.
+ Note that this could cause issues on large clusters as many connections
+ could linger simultaneously and lead to a large number of connection
+ threads. The token used for authentication will be used only at
+ connection creation time. If a new token is received then the earlier
+ connection should be closed in order to use the new token. This and
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related
+ and should be in sync (no need for them to be equal).
+ If the value of this property is zero then the connection cache is
+ disabled and connections will use a zero idle timeout to prevent too
+ many connection threads on large clusters.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_MAX_NM_PROXIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for NM to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each NM attempt to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-labels configurations]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node label store implementation class]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_NODE_LABELS_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for NodeLabelManager]]>
+      </doc>
+    </field>
+    <field name="NODE_ATTRIBUTE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-attribute configurations.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute store implementation class.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File system node attribute store directory.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to indicate if the node labels feature enabled, by default it's
+ disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABELS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CENTRALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONFIG_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCRIPT_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute provider fetch attributes interval and timeout.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPLAY_APPS_FOR_LOGGED_IN_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_ENTITY_LIST_BY_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_INVALID_XML_CHARS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FILTER_INVALID_XML_CHARS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENABLE_REST_APP_SUBMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENABLE_REST_APP_SUBMISSIONS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REST_CSRF" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="XFS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_XFS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline reader.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_STORAGE_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Marked collector properties as Private since it run as auxillary service.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for NUMA awareness.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_READ_TOPOLOGY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_READ_TOPOLOGY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NODE_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for activities manager.]]>
+      </doc>
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cleanup interval for activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time to live for scheduler activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time to live for app activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue length for app activities.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINERS_LAUNCHER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers launcher implementation to use.]]>
+      </doc>
+    </field>
+    <field name="YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+</package>
+<package name="org.apache.hadoop.yarn.exceptions">
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <class name="ApplicationAttemptNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getApplicationAttemptReport (GetApplicationAttemptReportRequest)}
+ API when the Application Attempt doesn't exist in Application History Server or
+ {@link ApplicationMasterProtocol#allocate(AllocateRequest)} if application
+ doesn't exist in RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <class name="ApplicationIdNotProvidedException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception to be thrown when Client submit an application without
+ providing {@link ApplicationId} in {@link ApplicationSubmissionContext}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <class name="ApplicationNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} API
+ when the Application doesn't exist in RM and AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <class name="ConfigurationException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConfigurationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on unrecoverable configuration errors.
+ An example is container launch error due to configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <class name="ContainerNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getContainerReport (GetContainerReportRequest)}
+ API when the container doesn't exist in AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <class name="ResourceNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when details of an unknown resource type
+ are requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <class name="YarnException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[YarnException indicates exceptions from yarn servers. On the other hand,
+ IOExceptions indicates exceptions from RPC layer.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+  <class name="YARNFeatureNotEnabledException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when a feature is being used which is not enabled
+ yet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <class name="ApplicationInitializationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ApplicationId, java.nio.ByteBuffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-name of the application-submitter
+ 
+ @return user-name]]>
+      </doc>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <method name="getApplicationDataForService" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data sent to the NodeManager via
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ as part of {@link ContainerLaunchContext#getServiceData()}
+ 
+ @return the servicesData for this application.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <class name="ApplicationTerminationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTerminationContext" type="org.apache.hadoop.yarn.api.records.ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application being stopped.
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when stopping an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <interface name="AuxiliaryLocalPathHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getLocalPathForRead" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="size" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing a file of an estimated size
+ for a given Auxiliary Service.
+ @param path the requested path
+ @param size the size of the file that is going to be written
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getAllLocalPathsForRead" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all paths from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path list to the file on a local disk as an Iterable
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An Interface that can retrieve local directories to read from or write to.
+  Components can implement this interface to link it to
+  their own Directory Handler Service]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <!-- start class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <class name="AuxiliaryService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuxiliaryService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecoveryPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path specific to this auxiliary service to use for recovery.
+
+ @return state storage path or null if recovery is not enabled]]>
+      </doc>
+    </method>
+    <method name="initializeApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationInitializationContext"/>
+      <doc>
+      <![CDATA[A new application is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the application initialization.
+ 
+ @param initAppContext context for the application's initialization]]>
+      </doc>
+    </method>
+    <method name="stopApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationTerminationContext"/>
+      <doc>
+      <![CDATA[An application is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+ 
+ @param stopAppContext context for the application termination]]>
+      </doc>
+    </method>
+    <method name="getMetaData" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve meta-data for this {@link AuxiliaryService}. Applications using
+ this {@link AuxiliaryService} SHOULD know the format of the meta-data -
+ ideally each service should provide a method to parse out the information
+ to the applications. One example of meta-data is contact information so
+ that applications can access the service remotely. This will only be called
+ after the service's {@link #start()} method has finished. the result may be
+ cached.
+ 
+ <p>
+ The information is passed along to applications via
+ {@link StartContainersResponse#getAllServicesMetaData()} that is returned by
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ </p>
+ 
+ @return meta-data for this service that should be made available to
+         applications.]]>
+      </doc>
+    </method>
+    <method name="initializeContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerInitializationContext"/>
+      <doc>
+      <![CDATA[A new container is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the container initialization.
+ This method is called when the NodeManager receives the container launch
+ command from the ApplicationMaster and before the container process is 
+ launched.
+
+ @param initContainerContext context for the container's initialization]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerTerminationContext"/>
+      <doc>
+      <![CDATA[A container is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+
+ @param stopContainerContext context for the container termination]]>
+      </doc>
+    </method>
+    <method name="setRecoveryPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recoveryPath" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path for this auxiliary service to use for storing state
+ that will be used during recovery.
+
+ @param recoveryPath where recoverable state should be stored]]>
+      </doc>
+    </method>
+    <method name="getAuxiliaryLocalPathHandler" return="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Method that gets the local dirs path handler for this Auxiliary Service.
+
+ @return auxiliaryPathHandler object that is used to read from and write to
+ valid local Dirs.]]>
+      </doc>
+    </method>
+    <method name="setAuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="auxiliaryLocalPathHandler" type="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"/>
+      <doc>
+      <![CDATA[Method that sets the local dirs path handler for this Auxiliary Service.
+
+ @param auxiliaryLocalPathHandler the pathHandler for this auxiliary service]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A generic service that will be started by the NodeManager. This is a service
+ that administrators have to configure on each node by setting
+ {@link YarnConfiguration#NM_AUX_SERVICES}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <class name="ContainerContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get user of the container being initialized or stopped.
+
+ @return the user]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container being initialized or stopped.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link Resource} the resource capability allocated to the container
+ being initialized or stopped.
+
+ @return the resource capability.]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container
+ being initialized or stopped.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ExecutionType} the execution type of the container
+ being initialized or stopped.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base context class for {@link AuxiliaryService} initializing and stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <class name="ContainerInitializationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <interface name="ContainerLogAggregationPolicy"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to initial the policy object with parameters specified by the application
+ or the cluster-wide setting.
+ </p>
+
+ @param parameters parameters with scheme defined by the policy class.]]>
+      </doc>
+    </method>
+    <method name="shouldDoLogAggregation" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logContext" type="org.apache.hadoop.yarn.server.api.ContainerLogContext"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to ask the policy object if a given container's logs should be aggregated.
+ </p>
+
+ @param logContext ContainerLogContext
+ @return Whether or not the container's logs should be aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This API is used by NodeManager to decide if a given container's logs
+ should be aggregated at run time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <class name="ContainerLogContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLogContext" type="org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.server.api.ContainerType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exit code of the container.
+
+ @return the exit code]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Context class for {@link ContainerLogAggregationPolicy}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <class name="ContainerTerminationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Termination context for {@link AuxiliaryService} when stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+  <interface name="SCMAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="runCleanerTask" return="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by administrators to ask SCM to run cleaner task right away
+ </p>
+
+ @param request request <code>SharedCacheManager</code> to run a cleaner task
+ @return <code>SharedCacheManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between administrators and the <code>SharedCacheManager</code>
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+</package>
+<package name="org.apache.hadoop.yarn.util.constraint">
+  <!-- start class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+  <class name="PlacementConstraintParser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseExpression" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintStr" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a given constraint expression to a {@link AbstractConstraint},
+ this expression can be any valid form of constraint expressions.
+
+ @param constraintStr expression string
+ @return a parsed {@link AbstractConstraint}
+ @throws PlacementConstraintParseException when given expression
+ is malformed]]>
+      </doc>
+    </method>
+    <method name="parsePlacementSpec" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expression" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a placement constraint specification. A placement constraint spec
+ is a composite expression which is composed by multiple sub constraint
+ expressions delimited by ":". With following syntax:
+
+ <p>Tag1(N1),P1:Tag2(N2),P2:...:TagN(Nn),Pn</p>
+
+ where <b>TagN(Nn)</b> is a key value pair to determine the source
+ allocation tag and the number of allocations, such as:
+
+ <p>foo(3)</p>
+
+ Optional when using NodeAttribute Constraint.
+
+ and where <b>Pn</b> can be any form of a valid constraint expression,
+ such as:
+
+ <ul>
+   <li>in,node,foo,bar</li>
+   <li>notin,node,foo,bar,1,2</li>
+   <li>and(notin,node,foo:notin,node,bar)</li>
+ </ul>
+
+ and NodeAttribute Constraint such as
+
+ <ul>
+   <li>yarn.rm.io/foo=true</li>
+   <li>java=1.7,1.8</li>
+ </ul>
+ @param expression expression string.
+ @return a map of source tags to placement constraint mapping.
+ @throws PlacementConstraintParseException]]>
+      </doc>
+    </method>
+    <field name="EXPRESSION_VAL_DELIM" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Placement constraint expression parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+</package>
+<package name="org.apache.hadoop.yarn.util.csi">
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml
new file mode 100644
index 0000000000000..f96c8d559ddf1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml
@@ -0,0 +1,26407 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:08:01 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN API 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/classes:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/site/jdiff/xml -apiname Apache Hadoop YARN API 3.3.4 -->
+<package name="org.apache.hadoop.yarn.ams">
+</package>
+<package name="org.apache.hadoop.yarn.api">
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <interface name="ApplicationClientProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <method name="getNewApplication" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ApplicationId} for 
+ submitting new applications.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a new, monotonically
+ increasing, {@link ApplicationId} which is used by the client to submit
+ a new application.</p>
+
+ <p>The <code>ResourceManager</code> also responds with details such 
+ as maximum resource capabilities in the cluster as specified in
+ {@link GetNewApplicationResponse}.</p>
+
+ @param request request to get a new <code>ApplicationId</code>
+ @return response containing the new <code>ApplicationId</code> to be used
+ to submit an application
+ @throws YarnException
+ @throws IOException
+ @see #submitApplication(SubmitApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to submit a new application to the
+ <code>ResourceManager.</code></p>
+ 
+ <p>The client is required to provide details such as queue, 
+ {@link Resource} required to run the <code>ApplicationMaster</code>, 
+ the equivalent of {@link ContainerLaunchContext} for launching
+ the <code>ApplicationMaster</code> etc. via the 
+ {@link SubmitApplicationRequest}.</p>
+ 
+ <p>Currently the <code>ResourceManager</code> sends an immediate (empty) 
+ {@link SubmitApplicationResponse} on accepting the submission and throws 
+ an exception if it rejects the submission. However, this call needs to be
+ followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ to make sure that the application gets properly submitted - obtaining a
+ {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee
+ that RM 'remembers' this application beyond failover or restart. If RM
+ failover or RM restart happens before ResourceManager saves the
+ application's state successfully, the subsequent
+ {@link #getApplicationReport(GetApplicationReportRequest)} will throw
+ a {@link ApplicationNotFoundException}. The Clients need to re-submit
+ the application with the same {@link ApplicationSubmissionContext} when
+ it encounters the {@link ApplicationNotFoundException} on the
+ {@link #getApplicationReport(GetApplicationReportRequest)} call.</p>
+ 
+ <p>During the submission process, it checks whether the application
+ already exists. If the application exists, it will simply return
+ SubmitApplicationResponse</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to
+ queues etc. before accepting the application submission.</p>
+ 
+ @param request request to submit a new application
+ @return (empty) response on accepting the submission
+ @throws YarnException
+ @throws IOException
+ @see #getNewApplication(GetNewApplicationRequest)]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the 
+ <code>ResourceManager</code> to fail an application attempt.</p>
+
+ <p>The client, via {@link FailApplicationAttemptRequest} provides the
+ {@link ApplicationAttemptId} of the attempt to be failed.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before failing the attempt.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to fail an attempt
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to abort submitted application.</p>
+ 
+ <p>The client, via {@link KillApplicationRequest} provides the
+ {@link ApplicationId} of the application to be aborted.</p>
+ 
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application, queue etc. before terminating the application.</p> 
+ 
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+ 
+ @param request request to abort a submitted application
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+      </doc>
+    </method>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get metrics about the cluster from
+ the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a
+ {@link GetClusterMetricsResponse} which includes the 
+ {@link YarnClusterMetrics} with details such as number of current
+ nodes in the cluster.</p>
+ 
+ @param request request for cluster metrics
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with a 
+ {@link GetClusterNodesResponse} which includes the 
+ {@link NodeReport} for all the nodes in the cluster.</p>
+ 
+ @param request request for report on all nodes
+ @return report on all nodes
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queues</em>
+ from the <code>ResourceManager</code>.</p>
+ 
+ <p>The client, via {@link GetQueueInfoRequest}, can ask for details such
+ as used/total resources, child queues, running applications etc.</p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access before
+ providing the information.</p> 
+ 
+ @param request request to get queue information
+ @return queue information
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueUserAcls" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to get information about <em>queue 
+ acls</em> for <em>current user</em> from the <code>ResourceManager</code>.
+ </p>
+ 
+ <p>The <code>ResourceManager</code> responds with queue acls for all
+ existing queues.</p>
+ 
+ @param request request to get queue acls for <em>current user</em>
+ @return queue acls for <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Move an application to a new queue.
+ 
+ @param request the application ID and the target queue
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNewReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to obtain a new {@link ReservationId} for
+ submitting new reservations.</p>
+
+ <p>The <code>ResourceManager</code> responds with a new, unique,
+ {@link ReservationId} which is used by the client to submit
+ a new reservation.</p>
+
+ @param request to get a new <code>ReservationId</code>
+ @return response containing the new <code>ReservationId</code> to be used
+ to submit a new reservation
+ @throws YarnException if the reservation system is not enabled.
+ @throws IOException on IO failures.
+ @see #submitReservation(ReservationSubmissionRequest)]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and concurrency needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationSubmissionRequest is satisfiable the
+ {@code ResourceManager} answers with a
+ {@link ReservationSubmissionResponse} that include a non-null
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the reason.
+ 
+ On application submission the client can use this {@link ReservationId} to
+ obtain access to the reserved resources.
+ </p>
+ 
+ <p>
+ The system guarantees that during the time-range specified by the user, the
+ reservationID will be corresponding to a valid reservation. The amount of
+ capacity dedicated to such queue can vary overtime, depending of the
+ allocation that has been determined. But it is guaranteed to satisfy all
+ the constraint expressed by the user in the
+ {@link ReservationSubmissionRequest}.
+ </p>
+ 
+ @param request the request to submit a new Reservation
+ @return response the {@link ReservationId} on accepting the submission
+ @throws YarnException if the request is invalid or reservation cannot be
+           created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationUpdateRequest}. Upon success the previous allocation is
+ substituted by the new one, and on failure (i.e., if the system cannot find
+ a valid allocation for the updated request), the previous allocation
+ remains valid.
+ 
+ The {@link ReservationId} is not changed, and applications currently
+ running within this reservation will automatically receive the resources
+ based on the new allocation.
+ </p>
+ 
+ @param request to update an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ 
+ Upon deletion of a reservation applications running with this reservation,
+ are automatically downgraded to normal jobs running without any dedicated
+ reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the ReservationRequest
+          should refer to an existing valid {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+
+ @param request
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mappings
+ in existing cluster
+ </p>
+
+ @param request
+ @return labels to nodes mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @param request to get node labels collection of this cluster
+ @return node labels collection of this cluster
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application.
+ </p>
+ @param request to set priority of an application
+ @return an empty response
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by clients to request the
+ <code>ResourceManager</code> to signal a container. For example,
+ the client can send command OUTPUT_THREAD_DUMP to dump threads of the
+ container.</p>
+
+ <p>The client, via {@link SignalContainerRequest} provides the
+ id of the container and the signal command. </p>
+
+ <p> In secure mode,the <code>ResourceManager</code> verifies access to the
+ application before signaling the container.
+ The user needs to have <code>MODIFY_APP</code> permission.</p>
+
+ <p>Currently, the <code>ResourceManager</code> returns an empty response
+ on success and throws an exception on rejecting the request.</p>
+
+ @param request request to signal a container
+ @return <code>ResourceManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set ApplicationTimeouts of an application.
+ The UpdateApplicationTimeoutsRequest should have timeout value with
+ absolute time with ISO8601 format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ </p>
+ <b>Note:</b> If application timeout value is less than or equal to current
+ time then update application throws YarnException.
+ @param request to set ApplicationTimeouts of an application
+ @return a response with updated timeouts.
+ @throws YarnException if update request has empty values or application is
+           in completing states.
+ @throws IOException on IO failures]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get all the resource profiles that are
+ available on the ResourceManager.
+ </p>
+ @param request request to get all the resource profiles
+ @return Response containing a map of the profile name to Resource
+         capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface to get the details for a specific resource profile.
+ </p>
+ @param request request to get the details of a resource profile
+ @return Response containing the details for a particular resource profile
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get attributes to nodes mappings
+ available in ResourceManager.
+ </p>
+
+ @param request request to get details of attributes to nodes mapping.
+ @return Response containing the details of attributes to nodes mappings.
+ @throws YarnException if any error happens inside YARN
+ @throws IOException   incase of other errors]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes available in
+ ResourceManager.
+ </p>
+
+ @param request request to get node attributes collection of this cluster.
+ @return Response containing node attributes collection.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException   incase of other errors.]]>
+      </doc>
+    </method>
+    <method name="getNodesToAttributes" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to attributes mappings.
+ in existing cluster.
+ </p>
+
+ @param request request to get nodes to attributes mapping.
+ @return nodes to attributes mappings.
+ @throws YarnException if any error happens inside YARN.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between clients and the <code>ResourceManager</code>
+ to submit/abort jobs and to get information on applications, cluster metrics,
+ nodes, queues and ACLs.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationClientProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <interface name="ApplicationConstants"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <field name="APP_SUBMIT_TIME_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
+ only]]>
+      </doc>
+    </field>
+    <field name="CONTAINER_TOKEN_FILE_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cache file into which container token is written]]>
+      </doc>
+    </field>
+    <field name="KEYSTORE_FILE_LOCATION_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file into which the keystore containing the AM's certificate is
+ written.]]>
+      </doc>
+    </field>
+    <field name="KEYSTORE_PASSWORD_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The password for the AM's keystore.]]>
+      </doc>
+    </field>
+    <field name="TRUSTSTORE_FILE_LOCATION_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The file into which the truststore containing the AM's certificate is
+ written.]]>
+      </doc>
+    </field>
+    <field name="TRUSTSTORE_PASSWORD_ENV_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The password for the AM's truststore.]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_WEB_PROXY_BASE_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ ApplicationMaster's environment only. This states that for all non-relative
+ web URLs in the app masters web UI what base should they have.]]>
+      </doc>
+    </field>
+    <field name="LOG_DIR_EXPANSION_VAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The temporary environmental variable for container log directory. This
+ should be replaced by real container log directory on container launch.]]>
+      </doc>
+    </field>
+    <field name="CLASS_PATH_SEPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This constant is used to construct class path and it will be replaced with
+ real class path separator(':' for Linux and ';' for Windows) by
+ NodeManager on container launch. User has to use this constant to construct
+ class path if user wants cross-platform practice i.e. submit an application
+ from a Windows client to a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_LEFT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The following two constants are used to expand parameter and it will be
+ replaced with real parameter expansion marker ('%' for Windows and '$' for
+ Linux) by NodeManager on container launch. For example: {{VAR}} will be
+ replaced as $VAR on Linux, and %VAR% on Windows. User has to use this
+ constant to construct class path if user wants cross-platform practice i.e.
+ submit an application from a Windows client to a Linux/Unix server or vice
+ versa.]]>
+      </doc>
+    </field>
+    <field name="PARAMETER_EXPANSION_RIGHT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[User has to use this constant to construct class path if user wants
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.]]>
+      </doc>
+    </field>
+    <field name="STDERR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="STDOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This is the API for the applications comprising of constants that YARN sets
+ up for the applications and the containers.
+
+ TODO: Investigate the semantics and security of each cross-boundary refs.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationConstants -->
+  <!-- start class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <class name="ApplicationConstants.ContainerLaunchType" extends="java.lang.Enum"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The type of launch for the container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.ApplicationConstants.ContainerLaunchType -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <interface name="ApplicationHistoryProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.api.ApplicationBaseProtocol"/>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>ApplicationHistoryServer</code> to
+ get the information of completed applications etc.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationHistoryProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <interface name="ApplicationMasterProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by a new <code>ApplicationMaster</code> to register with
+ the <code>ResourceManager</code>.
+ </p>
+
+ <p>
+ The <code>ApplicationMaster</code> needs to provide details such as RPC
+ Port, HTTP tracking url etc. as specified in
+ {@link RegisterApplicationMasterRequest}.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code> responds with critical details such as
+ maximum resource capabilities in the cluster as specified in
+ {@link RegisterApplicationMasterResponse}.
+ </p>
+
+ <p>
+ Re-register is only allowed for <code>Unmanaged Application Master</code>
+ (UAM) HA, with
+ {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ set to true.
+ </p>
+
+ @param request registration request
+ @return registration respose
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException The exception is thrown
+           when an ApplicationMaster tries to register more then once.
+ @see RegisterApplicationMasterRequest
+ @see RegisterApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>The interface used by an <code>ApplicationMaster</code> to notify the 
+ <code>ResourceManager</code> about its completion (success or failed).</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as 
+ final state, diagnostics (in case of failures) etc. as specified in 
+ {@link FinishApplicationMasterRequest}.</p>
+ 
+ <p>The <code>ResourceManager</code> responds with 
+ {@link FinishApplicationMasterResponse}.</p>
+ 
+ @param request completion request
+ @return completion response
+ @throws YarnException
+ @throws IOException
+ @see FinishApplicationMasterRequest
+ @see FinishApplicationMasterResponse]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The main interface between an <code>ApplicationMaster</code> and the
+ <code>ResourceManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> uses this interface to provide a list of
+ {@link ResourceRequest} and returns unused {@link Container} allocated to
+ it via {@link AllocateRequest}. Optionally, the
+ <code>ApplicationMaster</code> can also <em>blacklist</em> resources which
+ it doesn't want to use.
+ </p>
+ 
+ <p>
+ This also doubles up as a <em>heartbeat</em> to let the
+ <code>ResourceManager</code> know that the <code>ApplicationMaster</code>
+ is alive. Thus, applications should periodically make this call to be kept
+ alive. The frequency depends on
+ {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
+ </p>
+ 
+ <p>
+ The <code>ResourceManager</code> responds with list of allocated
+ {@link Container}, status of completed containers and headroom information
+ for the application.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> can use the available headroom
+ (resources) to decide how to utilized allocated resources and make informed
+ decisions about future resource requests.
+ </p>
+ 
+ @param request
+          allocation request
+ @return allocation response
+ @throws YarnException
+ @throws IOException
+ @throws InvalidApplicationMasterRequestException
+           This exception is thrown when an ApplicationMaster calls allocate
+           without registering first.
+ @throws InvalidResourceBlacklistRequestException
+           This exception is thrown when an application provides an invalid
+           specification for blacklist of resources.
+ @throws InvalidResourceRequestException
+           This exception is thrown when a {@link ResourceRequest} is out of
+           the range of the configured lower and upper limits on the
+           resources.
+ @see AllocateRequest
+ @see AllocateResponse]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between a live instance of <code>ApplicationMaster</code> 
+ and the <code>ResourceManager</code>.</p>
+ 
+ <p>This is used by the <code>ApplicationMaster</code> to register/unregister
+ and to request and obtain resources in the cluster from the
+ <code>ResourceManager</code>.</p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ApplicationMasterProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <interface name="ClientSCMProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="use" return="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to claim a resource with the
+ <code>SharedCacheManager.</code> The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application will be
+ using the resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>Path</code>
+ to the resource in the shared cache is returned. If the resource does not
+ exist, the response is empty.
+ </p>
+
+ @param request request to claim a resource in the shared cache
+ @return response indicating if the resource is already in the cache
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="release" return="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to release a resource with the
+ <code>SharedCacheManager.</code> This method is called once an application
+ is no longer using a claimed resource in the shared cache. The client uses
+ a checksum to identify the resource and an {@link ApplicationId} to
+ identify which application is releasing the resource.
+ </p>
+
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+
+ <p>
+ Currently the <code>SharedCacheManager</code> sends an empty response.
+ </p>
+
+ @param request request to release a resource in the shared cache
+ @return (empty) response on releasing the resource
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between clients and the <code>SharedCacheManager</code> to claim
+ and release resources in the shared cache.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ClientSCMProtocol -->
+  <!-- start interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+  <interface name="ContainerManagementProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="startContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> provides a list of
+ {@link StartContainerRequest}s to a <code>NodeManager</code> to
+ <em>start</em> {@link Container}s allocated to it using this interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> has to provide details such as allocated
+ resource capability, security tokens (if enabled), command to be executed
+ to start the container, environment for the process, necessary
+ binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in
+ the {@link StartContainerRequest}.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StartContainersResponse} which includes a list of
+ {@link Container}s of successfully launched {@link Container}s, a
+ containerId-to-exception map for each failed {@link StartContainerRequest} in
+ which the exception indicates errors from per container and a
+ allServicesMetaData map between the names of auxiliary services and their
+ corresponding meta-data. Note: None-container-specific exceptions will
+ still be thrown by the API method itself.
+ </p>
+ <p>
+ The <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the to-be-launched or launched containers.
+ </p>
+ 
+ @param request
+          request to start a list of containers
+ @return response including conatinerIds of all successfully launched
+         containers, a containerId-to-exception map for failed requests and
+         a allServicesMetaData map.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="stopContainers" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The <code>ApplicationMaster</code> requests a <code>NodeManager</code> to
+ <em>stop</em> a list of {@link Container}s allocated to it using this
+ interface.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a {@link StopContainersRequest}
+ which includes the {@link ContainerId}s of the containers to be stopped.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> sends a response via
+ {@link StopContainersResponse} which includes a list of {@link ContainerId}
+ s of successfully stopped containers, a containerId-to-exception map for
+ each failed request in which the exception indicates errors from per
+ container. Note: None-container-specific exceptions will still be thrown by
+ the API method itself. <code>ApplicationMaster</code> can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the containers.
+ </p>
+ 
+ @param request
+          request to stop a list of containers
+ @return response which includes a list of containerIds of successfully
+         stopped containers, a containerId-to-exception map for failed
+         requests.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for current
+ statuses of <code>Container</code>s from the <code>NodeManager</code>.
+ </p>
+ 
+ <p>
+ The <code>ApplicationMaster</code> sends a
+ {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s
+ of all containers whose statuses are needed.
+ </p>
+ 
+ <p>
+ The <code>NodeManager</code> responds with
+ {@link GetContainerStatusesResponse} which includes a list of
+ {@link ContainerStatus} of the successfully queried containers and a
+ containerId-to-exception map for each failed request in which the exception
+ indicates errors from per container. Note: None-container-specific
+ exceptions will still be thrown by the API method itself.
+ </p>
+ 
+ @param request
+          request to get <code>ContainerStatus</code>es of containers with
+          the specified <code>ContainerId</code>s
+ @return response containing the list of <code>ContainerStatus</code> of the
+         successfully queried containers and a containerId-to-exception map
+         for failed requests.
+ 
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="increaseContainersResource" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource increase of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to increase resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully increased and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The API used by the <code>ApplicationMaster</code> to request for
+ resource update of running containers on the <code>NodeManager</code>.
+ </p>
+
+ @param request
+         request to update resource of a list of containers
+ @return response which includes a list of containerIds of containers
+         whose resource has been successfully updated and a
+         containerId-to-exception map for failed requests.
+
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from NodeManager]]>
+      </doc>
+    </method>
+    <method name="signalToContainer" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="localize" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources required by the container.
+ Currently, this API only works for running containers.
+
+ @param request Specify the resources to be localized.
+ @return Response that the localize request is accepted.
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[ReInitialize the Container with a new Launch Context.
+ @param request Specify the new ContainerLaunchContext.
+ @return Response that the ReInitialize request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="restartContainer" return="org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Restart the container.
+ @param containerId Container Id.
+ @return Response that the restart request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Rollback the Last ReInitialization if possible.
+ @param containerId Container Id.
+ @return Response that the rollback request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization" return="org.apache.hadoop.yarn.api.protocolrecords.CommitResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Commit the Last ReInitialization if possible. Once the reinitialization
+ has been committed, It cannot be rolled back.
+ @param containerId Container Id.
+ @return Response that the commit request is accepted.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <method name="getLocalizationStatuses" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[API to request for the localization statuses of requested containers from
+ the Node Manager.
+ @param request {@link GetLocalizationStatusesRequest} which includes the
+                container ids of all the containers whose localization
+                statuses are needed.
+ @return {@link GetLocalizationStatusesResponse} which contains the
+         localization statuses of all the requested containers.
+ @throws YarnException Exception specific to YARN.
+ @throws IOException IOException thrown from the RPC layer.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The protocol between an <code>ApplicationMaster</code> and a 
+ <code>NodeManager</code> to start/stop and increase resource of containers
+ and to get status of running containers.</p>
+
+ <p>If security is enabled the <code>NodeManager</code> verifies that the
+ <code>ApplicationMaster</code> has truly been allocated the container
+ by the <code>ResourceManager</code> and also verifies all interactions such 
+ as stopping the container or obtaining status information for the container.
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.api.ContainerManagementProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.api.protocolrecords">
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <class name="AllocateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseID" type="int"/>
+      <param name="appProgress" type="float"/>
+      <param name="resourceAsk" type="java.util.List"/>
+      <param name="containersToBeReleased" type="java.util.List"/>
+      <param name="updateRequests" type="java.util.List"/>
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>response id</em> used to track duplicate responses.
+ @return <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="setResponseId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>response id</em> used to track duplicate responses.
+ @param id <em>response id</em>]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current progress</em> of application. 
+ @return <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="setProgress"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <em>current progress</em> of application
+ @param progress <em>current progress</em> of application]]>
+      </doc>
+    </method>
+    <method name="getAskList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ResourceRequest</code> to update the 
+ <code>ResourceManager</code> about the application's resource requirements.
+ @return the list of <code>ResourceRequest</code>
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="setAskList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of <code>ResourceRequest</code> to update the
+ <code>ResourceManager</code> about the application's resource requirements.
+ @param resourceRequests list of <code>ResourceRequest</code> to update the 
+                        <code>ResourceManager</code> about the application's 
+                        resource requirements
+ @see ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getReleaseList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> of containers being 
+ released by the <code>ApplicationMaster</code>.
+ @return list of <code>ContainerId</code> of containers being 
+         released by the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setReleaseList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseContainers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of <code>ContainerId</code> of containers being
+ released by the <code>ApplicationMaster</code>
+ @param releaseContainers list of <code>ContainerId</code> of 
+                          containers being released by the 
+                          <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceBlacklistRequest" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ResourceBlacklistRequest</code> being sent by the 
+ <code>ApplicationMaster</code>.
+ @return the <code>ResourceBlacklistRequest</code> being sent by the 
+         <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="setResourceBlacklistRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ResourceBlacklistRequest</code> to inform the 
+ <code>ResourceManager</code> about the blacklist additions and removals
+ per the <code>ApplicationMaster</code>.
+ 
+ @param resourceBlacklistRequest the <code>ResourceBlacklistRequest</code>  
+                         to inform the <code>ResourceManager</code> about  
+                         the blacklist additions and removals
+                         per the <code>ApplicationMaster</code>
+ @see ResourceBlacklistRequest]]>
+      </doc>
+    </method>
+    <method name="getUpdateRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link UpdateContainerRequest}
+         being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setUpdateRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update requests to inform the
+ <code>ResourceManager</code> about the containers that need to be
+ updated.
+ @param updateRequests list of <code>UpdateContainerRequest</code> for
+                       containers to be updated]]>
+      </doc>
+    </method>
+    <method name="getSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of Scheduling requests being sent by the
+ <code>ApplicationMaster</code>.
+ @return list of {@link SchedulingRequest} being sent by the
+         <code>ApplicationMaster</code>.]]>
+      </doc>
+    </method>
+    <method name="setSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of Scheduling requests to inform the
+ <code>ResourceManager</code> about the application's resource requirements
+ (potentially including allocation tags and placement constraints).
+ @param schedulingRequests list of {@link SchedulingRequest} to update
+          the <code>ResourceManager</code> about the application's resource
+          requirements.]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the tracking url update for this heartbeat.
+ @return tracking url to update this application with]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the new tracking url for this application.
+ @param trackingUrl the new tracking url]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The core request sent by the <code>ApplicationMaster</code> to the 
+ <code>ResourceManager</code> to obtain resources in the cluster.</p> 
+
+ <p>The request includes:
+ <ul>
+   <li>A response id to track duplicate responses.</li>
+   <li>Progress information.</li>
+   <li>
+     A list of {@link ResourceRequest} to inform the
+     <code>ResourceManager</code> about the application's
+     resource requirements.
+   </li>
+   <li>
+     A list of unused {@link Container} which are being returned.
+   </li>
+   <li>
+     A list of {@link UpdateContainerRequest} to inform
+     the <code>ResourceManager</code> about the change in
+     requirements of running containers.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <class name="AllocateRequest.AllocateRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="responseId" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>responseId</code> of the request.
+ @see AllocateRequest#setResponseId(int)
+ @param responseId <code>responseId</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="progress" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progress" type="float"/>
+      <doc>
+      <![CDATA[Set the <code>progress</code> of the request.
+ @see AllocateRequest#setProgress(float)
+ @param progress <code>progress</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="askList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="askList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>askList</code> of the request.
+ @see AllocateRequest#setAskList(List)
+ @param askList <code>askList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="releaseList" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="releaseList" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>releaseList</code> of the request.
+ @see AllocateRequest#setReleaseList(List)
+ @param releaseList <code>releaseList</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceBlacklistRequest" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceBlacklistRequest" type="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"/>
+      <doc>
+      <![CDATA[Set the <code>resourceBlacklistRequest</code> of the request.
+ @see AllocateRequest#setResourceBlacklistRequest(
+ ResourceBlacklistRequest)
+ @param resourceBlacklistRequest
+     <code>resourceBlacklistRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="updateRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>updateRequests</code> of the request.
+ @see AllocateRequest#setUpdateRequests(List)
+ @param updateRequests <code>updateRequests</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="schedulingRequests" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>schedulingRequests</code> of the request.
+ @see AllocateRequest#setSchedulingRequests(List)
+ @param schedulingRequests <code>SchedulingRequest</code> of the request
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="trackingUrl" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>trackingUrl</code> of the request.
+ @see AllocateRequest#setTrackingUrl(String)
+ @param trackingUrl new tracking url
+ @return {@link AllocateRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link AllocateRequest} object.
+ @return {@link AllocateRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link AllocateRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest.AllocateRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <class name="AllocateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AllocateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="responseId" type="int"/>
+      <param name="completedContainers" type="java.util.List"/>
+      <param name="allocatedContainers" type="java.util.List"/>
+      <param name="updatedNodes" type="java.util.List"/>
+      <param name="availResources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.AMCommand"/>
+      <param name="numClusterNodes" type="int"/>
+      <param name="preempt" type="org.apache.hadoop.yarn.api.records.PreemptionMessage"/>
+      <param name="nmTokens" type="java.util.List"/>
+      <param name="amRMToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="updatedContainers" type="java.util.List"/>
+      <param name="collectorInfo" type="org.apache.hadoop.yarn.api.records.CollectorInfo"/>
+    </method>
+    <method name="getAMCommand" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the <code>ResourceManager</code> needs the
+ <code>ApplicationMaster</code> to take some action then it will send an
+ AMCommand to the <code>ApplicationMaster</code>. See <code>AMCommand</code> 
+ for details on commands and actions for them.
+ @return <code>AMCommand</code> if the <code>ApplicationMaster</code> should
+         take action, <code>null</code> otherwise
+ @see AMCommand]]>
+      </doc>
+    </method>
+    <method name="getResponseId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last response id</em>.
+ @return <em>last response id</em>]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>newly allocated</em> <code>Container</code> by the
+ <code>ResourceManager</code>.
+ @return list of <em>newly allocated</em> <code>Container</code>]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>available headroom</em> for resources in the cluster for the
+ application.
+ @return limit of available headroom for resources in the cluster for the
+ application]]>
+      </doc>
+    </method>
+    <method name="getCompletedContainersStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>completed containers' statuses</em>.
+ @return the list of <em>completed containers' statuses</em>]]>
+      </doc>
+    </method>
+    <method name="getUpdatedNodes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>updated <code>NodeReport</code>s</em>. Updates could
+ be changes in health, availability etc of the nodes.
+ @return The delta of updated nodes since the last response]]>
+      </doc>
+    </method>
+    <method name="getNumClusterNodes" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of hosts available on the cluster.
+ @return the available host count.]]>
+      </doc>
+    </method>
+    <method name="getPreemptionMessage" return="org.apache.hadoop.yarn.api.records.PreemptionMessage"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the description of containers owned by the AM, but requested back by
+ the cluster. Note that the RM may have an inconsistent view of the
+ resources owned by the AM. These messages are advisory, and the AM may
+ elect to ignore them.
+ <p>
+ The message is a snapshot of the resources the RM wants back from the AM.
+ While demand persists, the RM will repeat its request; applications should
+ not interpret each message as a request for <em>additional</em>
+ resources on top of previous messages. Resources requested consistently
+ over some duration may be forcibly killed by the RM.
+
+ @return A specification of the resources to reclaim from this AM.]]>
+      </doc>
+    </method>
+    <method name="getNMTokens" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens required for communicating with NM. New NMTokens
+ issued only if
+ <p>
+ 1) AM is receiving first container on underlying NodeManager.<br>
+ OR<br>
+ 2) NMToken master key rolled over in ResourceManager and AM is getting new
+ container on the same underlying NodeManager.
+ <p>
+ AM will receive one NMToken per NM irrespective of the number of containers
+ issued on same NM. AM is expected to store these tokens until issued a
+ new token for the same NM.
+ @return list of NMTokens required for communicating with NM]]>
+      </doc>
+    </method>
+    <method name="getUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of newly updated containers by
+ <code>ResourceManager</code>.
+ @return list of newly increased containers]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The AMRMToken that belong to this attempt
+
+ @return The AMRMToken that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Priority of the application
+
+ @return get application priority]]>
+      </doc>
+    </method>
+    <method name="getCollectorInfo" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The data associated with the collector that belongs to this app. Contains
+ address and token alongwith identification information.
+
+ @return The data of collector that belong to this attempt]]>
+      </doc>
+    </method>
+    <method name="getUpdateErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+
+ @return List of Update Container Errors.]]>
+      </doc>
+    </method>
+    <method name="setUpdateErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of container update errors to inform the
+ Application Master about the container updates that could not be
+ satisfied due to error.
+ @param updateErrors list of <code>UpdateContainerError</code> for
+                       containers updates requests that were in error]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts which
+ have not been reported to the Application Master yet.
+ <br>
+ These containers were recovered by the RM after the application master
+ had already registered. This may happen after RM restart when some NMs get
+ delayed in connecting to the RM and reporting the active containers.
+ Since they were not reported in the registration
+ response, they are reported in the response to the AM heartbeat.
+
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts.]]>
+      </doc>
+    </method>
+    <method name="getRejectedSchedulingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of all SchedulingRequests that the RM has rejected between
+ this allocate call and the previous one.
+ @return List of RejectedSchedulingRequests.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> the
+ <code>ApplicationMaster</code> during resource negotiation.
+ <p>
+ The response, includes:
+ <ul>
+   <li>Response ID to track duplicate responses.</li>
+   <li>
+     An AMCommand sent by ResourceManager to let the
+     {@code ApplicationMaster} take some actions (resync, shutdown etc.).
+   </li>
+   <li>A list of newly allocated {@link Container}.</li>
+   <li>A list of completed {@link Container}s' statuses.</li>
+   <li>
+     The available headroom for resources in the cluster for the
+     application.
+   </li>
+   <li>A list of nodes whose status has been updated.</li>
+   <li>The number of available nodes in a cluster.</li>
+   <li>A description of resources requested back by the cluster</li>
+   <li>AMRMToken, if AMRMToken has been rolled over</li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been increased.
+   </li>
+   <li>
+     A list of {@link Container} representing the containers
+     whose resource has been decreased.
+   </li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#allocate(AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <class name="ApplicationsRequestScope" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration that controls the scope of applications fetched]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <class name="CommitResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CommitResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to Commit Container Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.CommitResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <class name="ContainerUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToUpdate" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource update.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource update.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToUpdate" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getUpdatedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource update.
+ @param containersToUpdate the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <class name="ContainerUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="successfullyUpdatedContainers" type="java.util.List"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getSuccessfullyUpdatedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully update.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully updated.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to update container resource.
+ </p>
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <class name="FailApplicationAttemptRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the attempt to be failed.
+ @return <code>ApplicationAttemptId</code> of the attempt.]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to fail an application attempt.</p>
+
+ <p>The request includes the {@link ApplicationAttemptId} of the attempt to
+ be failed.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <class name="FailApplicationAttemptResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FailApplicationAttemptResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ failing an application attempt.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <class name="FinishApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="url" type="java.lang.String"/>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>final state</em> of the <code>ApplicationMaster</code>.
+ @return <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setFinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finalState" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <doc>
+      <![CDATA[Set the <em>final state</em> of the <code>ApplicationMaster</code>
+ @param finalState <em>final state</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic information</em> on application failure.
+ @return <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <em>diagnostic information</em> on application failure.
+ @param diagnostics <em>diagnostic information</em> on application failure]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ This is the web-URL to which ResourceManager or web-application proxy will
+ redirect client/users once the application is finished and the
+ <code>ApplicationMaster</code> is gone.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param url
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The finalization request sent by the {@code ApplicationMaster} to
+ inform the {@code ResourceManager} about its completion.
+ <p>
+ The final request includes details such:
+ <ul>
+   <li>Final state of the {@code ApplicationMaster}</li>
+   <li>
+     Diagnostic information in case of failure of the
+     {@code ApplicationMaster}
+   </li>
+   <li>Tracking URL</li>
+ </ul>
+
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <class name="FinishApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FinishApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsUnregistered" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the application has successfully
+ unregistered with the RM and the application can safely stop.
+ @return true if the application has unregistered with the RM,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a
+ <code>ApplicationMaster</code> on it's completion.
+ <p>
+ The response, includes:
+ <ul>
+ <li>A flag which indicates that the application has successfully unregistered
+ with the RM and the application can safely stop.</li>
+ </ul>
+ <p>
+ Note: The flag indicates whether the application has successfully
+ unregistered and is safe to stop. The application may stop after the flag is
+ true. If the application stops before the flag is true then the RM may retry
+ the application.
+ 
+ @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <class name="GetAllResourceProfilesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <class name="GetAllResourceProfilesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceProfilesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceProfiles"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profiles" type="java.util.Map"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <class name="GetAllResourceTypeInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <class name="GetAllResourceTypeInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAllResourceTypeInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceTypeInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceTypes" type="java.util.List"/>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting all the resource profiles from the RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <class name="GetApplicationAttemptReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ApplicationAttemptReport} for an application attempt.
+ </p>
+ 
+ <p>
+ The request should include the {@link ApplicationAttemptId} of the
+ application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <class name="GetApplicationAttemptReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ApplicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @return <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptReport" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"/>
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptReport</code> for the application attempt.
+ 
+ @param applicationAttemptReport
+          <code>ApplicationAttemptReport</code> for the application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ an application attempt report.
+ </p>
+ 
+ <p>
+ The response includes an {@link ApplicationAttemptReport} which has the
+ details about the particular application attempt
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <class name="GetApplicationAttemptsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of an application
+ 
+ @return <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of an application
+ 
+ @param applicationId
+          <code>ApplicationId</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of application attempt reports of an
+ application from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <class name="GetApplicationAttemptsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationAttemptsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+    </method>
+    <method name="getApplicationAttemptList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @return a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttempts" type="java.util.List"/>
+      <doc>
+      <![CDATA[Get a list of <code>ApplicationReport</code> of an application.
+ 
+ @param applicationAttempts
+          a list of <code>ApplicationReport</code> of an application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ApplicationAttemptReport} for application attempts.
+ </p>
+ 
+ <p>
+ The <code>ApplicationAttemptReport</code> for each application includes the
+ details of an application attempt.
+ </p>
+ 
+ @see ApplicationAttemptReport
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <class name="GetApplicationReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to the <code>ResourceManager</code> to 
+ get an {@link ApplicationReport} for an application.</p>
+ 
+ <p>The request should include the {@link ApplicationId} of the 
+ application.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)
+ @see ApplicationReport]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <class name="GetApplicationReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationReport</code> for the application.
+ @return <code>ApplicationReport</code> for the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an application report.</p>
+ 
+ <p>The response includes an {@link ApplicationReport} which has details such 
+ as user, queue, name, host on which the <code>ApplicationMaster</code> is 
+ running, RPC port, tracking URL, diagnostics, start time etc.</p>
+ 
+ @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <class name="GetApplicationsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="queues" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="startRange" type="org.apache.commons.lang3.Range"/>
+      <param name="finishRange" type="org.apache.commons.lang3.Range"/>
+      <param name="limit" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+
+ <p>Setting any of the parameters to null, would just disable that
+ filter</p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @param users list of users to filter by
+ @param queues list of scheduler queues to filter by
+ @param applicationTypes types of applications
+ @param applicationTags application tags to filter by
+ @param applicationStates application states to filter by
+ @param startRange range of application start times to filter by
+ @param finishRange range of application finish times to filter by
+ @param limit number of applications to limit to
+ @return {@link GetApplicationsRequest} to be used with
+ {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving application states in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in {@link GetApplicationsRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[<p>
+ The request from clients to get a report of Applications matching the
+ giving and application types and application types in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return  a report of Applications in <code>GetApplicationsRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getApplicationTypes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application types to filter applications on
+
+ @return Set of Application Types to filter on]]>
+      </doc>
+    </method>
+    <method name="getApplicationStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application states to filter applications on
+
+ @return Set of Application states to filter on]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of Applications
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <class name="GetApplicationsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetApplicationsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationReport</code> for applications.
+ @return <code>ApplicationReport</code> for applications]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting an {@link ApplicationReport} for applications.</p>
+
+ <p>The <code>ApplicationReport</code> for each application includes details
+ such as user, queue, name, host on which the <code>ApplicationMaster</code>
+ is running, RPC port, tracking URL, diagnostics, start time etc.</p>
+
+ @see ApplicationReport
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <class name="GetAttributesToNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributeKeys for which the mapping of hostname to attribute value
+ is required.
+
+ @param attributes Set of NodeAttributeKey provided.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributeKeys for which mapping of hostname to attribute value is
+ required.
+
+ @return Set of NodeAttributeKey]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node to attribute value mapping for all or
+ given set of Node AttributeKey's in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <class name="GetAttributesToNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetAttributesToNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setAttributeToNodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get mapping of NodeAttributeKey to its associated mapping of list of
+ NodeToAttributeValue associated with attribute.
+
+ @return Map of node attributes to list of NodeToAttributeValue.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ node to attribute value mapping for all or given set of Node AttributeKey's.
+ </p>
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+      (GetAttributesToNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <class name="GetClusterMetricsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get cluster metrics from the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>Currently, this is empty.</p>
+
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <class name="GetClusterMetricsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterMetricsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnClusterMetrics</code> for the cluster.
+ @return <code>YarnClusterMetrics</code> for the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to a client
+ requesting cluster metrics.
+ 
+ @see YarnClusterMetrics
+ @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <class name="GetClusterNodeAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create new instance of GetClusterNodeAttributesRequest.
+
+ @return GetClusterNodeAttributesRequest is returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get node attributes in the cluster from the
+ <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <class name="GetClusterNodeAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Create instance of GetClusterNodeAttributesResponse.
+
+ @param attributes
+ @return GetClusterNodeAttributesResponse.]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node attributes to the response.
+
+ @param attributes Map of Node attributeKey to Type.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes from the response.
+
+ @return Node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a node attributes in cluster.
+ </p>
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <class name="GetClusterNodeLabelsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <class name="GetClusterNodeLabelsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodeLabelsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #newInstance(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Creates a new instance.
+
+ @param labels Node labels
+ @return response
+ @deprecated Use {@link #newInstance(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="setNodeLabelList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.List"/>
+    </method>
+    <method name="getNodeLabelList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNodeLabels"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #setNodeLabelList(List)} instead.">
+      <param name="labels" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set node labels to the response.
+
+ @param labels Node labels
+ @deprecated Use {@link #setNodeLabelList(List)} instead.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="Use {@link #getNodeLabelList()} instead.">
+      <doc>
+      <![CDATA[Get node labels of the response.
+
+ @return Node labels
+ @deprecated Use {@link #getNodeLabelList()} instead.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <class name="GetClusterNodesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeStates" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @return the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <method name="setNodeStates"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="java.util.EnumSet"/>
+      <doc>
+      <![CDATA[The state to filter the cluster nodes with.
+ @param states the set of {@link NodeState}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to get a report of all nodes
+ in the cluster from the <code>ResourceManager</code>.</p>
+
+ The request will ask for all nodes in the given {@link NodeState}s.
+
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <class name="GetClusterNodesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetClusterNodesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>NodeReport</code> for all nodes in the cluster.
+ @return <code>NodeReport</code> for all nodes in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client
+ requesting a {@link NodeReport} for all nodes.</p>
+ 
+ <p>The <code>NodeReport</code> contains per-node information such as 
+ available resources, number of containers, tracking url, rack name, health
+ status etc.
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <class name="GetContainerReportRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the Container.
+ 
+ @return <code>ContainerId</code> of the Container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container
+ 
+ @param containerId
+          <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by a client to the <code>ResourceManager</code> to get an
+ {@link ContainerReport} for a container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <class name="GetContainerReportResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerReportResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerReport</code> for the container.
+ 
+ @return <code>ContainerReport</code> for the container]]>
+      </doc>
+    </method>
+    <method name="setContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerReport" type="org.apache.hadoop.yarn.api.records.ContainerReport"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a container report.
+ </p>
+ 
+ <p>
+ The response includes a {@link ContainerReport} which has details of a
+ container.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <class name="GetContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of an application attempt.
+ 
+ @return <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <method name="setApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationAttemptId</code> of an application attempt
+ 
+ @param applicationAttemptId
+          <code>ApplicationAttemptId</code> of an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get a list of container reports, which belong to
+ an application attempt from the <code>ResourceManager</code>.
+ </p>
+ 
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <class name="GetContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+    </method>
+    <method name="getContainerList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @return a list of <code>ContainerReport</code> for all the containers of an
+         application attempt]]>
+      </doc>
+    </method>
+    <method name="setContainerList"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containers" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerReport</code> for all the containers of an
+ application attempt.
+ 
+ @param containers
+          a list of <code>ContainerReport</code> for all the containers of
+          an application attempt]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ a list of {@link ContainerReport} for containers.
+ </p>
+ 
+ <p>
+ The <code>ContainerReport</code> for each container includes the container
+ details.
+ </p>
+ 
+ @see ContainerReport
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <class name="GetContainerStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>.
+ 
+ @return the list of <code>ContainerId</code>s of containers for which to
+         obtain the <code>ContainerStatus</code>.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of <code>ContainerId</code>s of containers for which to obtain
+ the <code>ContainerStatus</code>
+ 
+ @param containerIds
+          a list of <code>ContainerId</code>s of containers for which to
+          obtain the <code>ContainerStatus</code>]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to get {@link ContainerStatus} of requested
+ containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <class name="GetContainerStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetContainerStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerStatuses" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerStatus</code>es of the requested containers.
+ 
+ @return <code>ContainerStatus</code>es of the requested containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to obtain the
+ <code>ContainerStatus</code> of requested containers.
+ 
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <class name="GetDelegationTokenRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <method name="getRenewer" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRenewer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[The request issued by the client to get a delegation token from
+ the {@code ResourceManager}.
+ for more information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <class name="GetDelegationTokenResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Delegation tokens have a identifier which maps to
+ {@link AbstractDelegationTokenIdentifier}.
+ @return the delegation tokens]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Response to a {@link GetDelegationTokenRequest} request 
+ from the client. The response contains the token that 
+ can be used by the containers to talk to  ClientRMService.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest -->
+  <class name="GetLocalizationStatusesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetLocalizationStatusesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of container IDs of the containers for which the localization
+ statuses are needed.
+
+ @return the list of container IDs.]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Sets the list of container IDs of containers for which the localization
+ statuses are needed.
+ @param containerIds the list of container IDs.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by an application master to the node manager to get
+ {@link LocalizationStatus}es of containers.
+
+ @see ContainerManagementProtocol#getLocalizationStatuses(
+        GetLocalizationStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse -->
+  <class name="GetLocalizationStatusesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetLocalizationStatusesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="statuses" type="java.util.Map"/>
+      <param name="failedRequests" type="java.util.Map"/>
+    </method>
+    <method name="getLocalizationStatuses" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the container localization statuses.
+
+ @return container localization statuses.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the node manager to an application master when
+ localization statuses are requested.
+
+ @see ContainerManagementProtocol#getLocalizationStatuses(
+        GetLocalizationStatusesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <class name="GetNewApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@link ApplicationId} for
+ submitting an application.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <class name="GetNewApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>new</em> <code>ApplicationId</code> allocated by the 
+ <code>ResourceManager</code>.
+ @return <em>new</em> <code>ApplicationId</code> allocated by the 
+          <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for 
+ a request to get a new {@link ApplicationId} for submitting applications.</p>
+ 
+ <p>Clients can submit an application with the returned
+ {@link ApplicationId}.</p>
+
+ @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <class name="GetNewReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get a new {@code ReservationId} for
+ submitting an reservation.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <class name="GetNewReservationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNewReservationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a new {@link ReservationId} to be used to submit a reservation.
+
+ @return a {@link ReservationId} representing the unique id to identify
+ a reservation with which it was submitted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client for
+ a request to get a new {@link ReservationId} for submitting reservations.</p>
+
+ <p>Clients can submit an reservation with the returned
+ {@link ReservationId}.</p>
+
+ {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <class name="GetNodesToAttributesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+    </method>
+    <method name="setHostNames"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostnames" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set hostnames for which mapping is required.
+
+ @param hostnames]]>
+      </doc>
+    </method>
+    <method name="getHostNames" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames for which mapping is required.
+
+ @return Set of hostnames.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to get nodes to attributes mapping
+ in the cluster from the <code>ResourceManager</code>.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <class name="GetNodesToAttributesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetNodesToAttributesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="setNodeToAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="map" type="java.util.Map"/>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get hostnames to NodeAttributes mapping.
+
+ @return Map of host to attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to a client requesting
+ nodes to attributes mapping.
+ </p>
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <class name="GetQueueInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <param name="includeApplications" type="boolean"/>
+      <param name="includeChildQueues" type="boolean"/>
+      <param name="recursive" type="boolean"/>
+    </method>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> for which to get queue information.
+ @return <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="setQueueName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue name</em> for which to get queue information
+ @param queueName <em>queue name</em> for which to get queue information]]>
+      </doc>
+    </method>
+    <method name="getIncludeApplications" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>active applications</em> required?
+ @return <code>true</code> if applications' information is to be included,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeApplications"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeApplications" type="boolean"/>
+      <doc>
+      <![CDATA[Should we get fetch information about <em>active applications</em>?
+ @param includeApplications fetch information about <em>active 
+                            applications</em>?]]>
+      </doc>
+    </method>
+    <method name="getIncludeChildQueues" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information about <em>child queues</em> required?
+ @return <code>true</code> if information about child queues is required,
+         else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="setIncludeChildQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeChildQueues" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information about <em>child queues</em>?
+ @param includeChildQueues fetch information about <em>child queues</em>?]]>
+      </doc>
+    </method>
+    <method name="getRecursive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is information on the entire <em>child queue hierarchy</em> required?
+ @return <code>true</code> if information about entire hierarchy is 
+         required, <code>false</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setRecursive"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recursive" type="boolean"/>
+      <doc>
+      <![CDATA[Should we fetch information on the entire <em>child queue hierarchy</em>?
+ @param recursive fetch information on the entire <em>child queue 
+                  hierarchy</em>?]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to get <em>queue information</em>
+ from the <code>ResourceManager</code>.</p>
+
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <class name="GetQueueInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueInfo</code> for the specified queue.
+ @return <code>QueueInfo</code> for the specified queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a client
+ requesting information about queues in the system.
+ <p>
+ The response includes a {@link QueueInfo} which has details such as
+ queue name, used/total capacities, running applications, child queues etc.
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <class name="GetQueueUserAclsInfoRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by clients to the <code>ResourceManager</code> to 
+ get queue acls for the <em>current user</em>.</p>
+
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <class name="GetQueueUserAclsInfoResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetQueueUserAclsInfoResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUserAclsInfoList" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueUserACLInfo</code> per queue for the user.
+ @return <code>QueueUserACLInfo</code> per queue for the user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to clients
+ seeking queue acls for the user.</p>
+
+ <p>The response contains a list of {@link QueueUserACLInfo} which
+ provides information about {@link QueueACL} per queue.</p>
+ 
+ @see QueueACL
+ @see QueueUserACLInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <class name="GetResourceProfileRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="setProfileName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profileName" type="java.lang.String"/>
+    </method>
+    <method name="getProfileName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Request class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <class name="GetResourceProfileResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="GetResourceProfileResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resources that will be allocated if the profile was used.
+
+ @return the resources that will be allocated if the profile was used.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resources that will be allocated if the profile is used.
+
+ @param r Set the resources that will be allocated if the profile is used.]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Response class for getting the details for a particular resource profile.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <class name="IncreaseContainersResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+    </method>
+    <method name="getContainersToIncrease" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of container tokens to be used for authorization during
+ container resource increase.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the list of container tokens to be used for authorization during
+ container resource increase.
+ @see NMToken]]>
+      </doc>
+    </method>
+    <method name="setContainersToIncrease"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containersToIncrease" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set container tokens to be used during container resource increase.
+ The token is acquired from
+ <code>AllocateResponse.getIncreasedContainers</code>.
+ The token contains the container id and resource capability required for
+ container resource increase.
+ @param containersToIncrease the list of container tokens to be used
+                             for container resource increase.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by <code>Application Master</code> to the
+ <code>Node Manager</code> to change the resource quota of a container.</p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <class name="IncreaseContainersResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="IncreaseContainersResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyIncreasedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of containers whose resource
+ have been successfully increased.
+
+ @return the list of containerIds of containers whose resource have
+ been successfully increased.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates
+ error from each container for failed requests.
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to increase container resource.
+ </p>
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <class name="KillApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be aborted.
+ @return <code>ApplicationId</code> of the application to be aborted]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostics</em> to which the application is being killed.
+ @return <em>diagnostics</em> to which the application is being killed]]>
+      </doc>
+    </method>
+    <method name="setDiagnostics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnostics" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>diagnostics</em> to which the application is being killed.
+ @param diagnostics <em>diagnostics</em> to which the application is being
+          killed]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to abort a submitted application.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ aborted.</p>
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <class name="KillApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="KillApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsKillCompleted" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates that the process of killing application is completed or not.
+ @return true if the process of killing application has completed,
+         false otherwise]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the <code>ResourceManager</code> to the client aborting
+ a submitted application.
+ <p>
+ The response, includes:
+ <ul>
+   <li>
+     A flag which indicates that the process of killing the application is
+     completed or not.
+   </li>
+ </ul>
+ Note: user is recommended to wait until this flag becomes true, otherwise if
+ the <code>ResourceManager</code> crashes before the process of killing the
+ application is completed, the <code>ResourceManager</code> may retry this
+ application on recovery.
+ 
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <class name="MoveApplicationAcrossQueuesRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application to be moved.
+ @return <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application to be moved.
+ @param appId <code>ApplicationId</code> of the application to be moved]]>
+      </doc>
+    </method>
+    <method name="getTargetQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @return the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <method name="setTargetQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the queue to place the application in.
+ @param queue the name of the queue to place the application in]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ to move a submitted application to a different queue.</p>
+ 
+ <p>The request includes the {@link ApplicationId} of the application to be
+ moved and the queue to place it in.</p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <class name="MoveApplicationAcrossQueuesResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="MoveApplicationAcrossQueuesResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client moving
+ a submitted application to a different queue.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <class name="RegisterApplicationMasterRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of <code>RegisterApplicationMasterRequest</code>.
+ If <em>port, trackingUrl</em> is not used, use the following default value:
+ <ul>
+  <li>port: -1</li>
+  <li>trackingUrl: null</li>
+ </ul>
+ The port is allowed to be any integer larger than or equal to -1.
+ @return the new instance of <code>RegisterApplicationMasterRequest</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>host</em> on which the <code>ApplicationMaster</code> is 
+ running.
+ @param host <em>host</em> on which the <code>ApplicationMaster</code> 
+             is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @return the <em>RPC port</em> on which the {@code ApplicationMaster}
+         is responding]]>
+      </doc>
+    </method>
+    <method name="setRpcPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the <em>RPC port</em> on which the {@code ApplicationMaster} is
+ responding.
+ @param port <em>RPC port</em> on which the {@code ApplicationMaster}
+             is responding]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return <em>tracking URL</em> for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="setTrackingUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ it is running. This is the web-URL to which ResourceManager or
+ web-application proxy will redirect client/users while the application and
+ the <code>ApplicationMaster</code> are still running.
+ <p>
+ If the passed url has a scheme then that will be used by the
+ ResourceManager and web-application proxy, otherwise the scheme will
+ default to http.
+ </p>
+ <p>
+ Empty, null, "N/A" strings are all valid besides a real URL. In case an url
+ isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ <p>
+
+ @param trackingUrl
+          <em>tracking URL</em>for the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getPlacementConstraints" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return all Placement Constraints specified at the Application level. The
+ mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags, i.e., each
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those
+ tags will be placed taking into account the corresponding constraint.
+
+ @return A map of Placement Constraints.]]>
+      </doc>
+    </method>
+    <method name="setPlacementConstraints"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraints" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set Placement Constraints applicable to the
+ {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+ of this application.
+ The mapping is from a set of allocation tags to a
+ <code>PlacementConstraint</code> associated with the tags.
+ For example:
+  Map &lt;
+   &lt;hb_regionserver&gt; -&gt; node_anti_affinity,
+   &lt;hb_regionserver, hb_master&gt; -&gt; rack_affinity,
+   ...
+  &gt;
+ @param placementConstraints Placement Constraint Mapping.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
+ on registration.
+ <p>
+ The registration includes details such as:
+ <ul>
+   <li>Hostname on which the AM is running.</li>
+   <li>RPC Port</li>
+   <li>Tracking URL</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <class name="RegisterApplicationMasterResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RegisterApplicationMasterResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getMaximumResourceCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the maximum capability for any {@link Resource} allocated by the 
+ <code>ResourceManager</code> in the cluster.
+ @return maximum capability of allocated resources in the cluster]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="getClientToAMTokenMasterKey" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get ClientToAMToken master key.</p>
+ <p>The ClientToAMToken master key is sent to <code>ApplicationMaster</code>
+ by <code>ResourceManager</code> via {@link RegisterApplicationMasterResponse}
+ , used to verify corresponding ClientToAMToken.</p>
+ @return ClientToAMToken master key]]>
+      </doc>
+    </method>
+    <method name="setClientToAMTokenMasterKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set ClientToAMToken master key.]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the queue that the application was placed in.<p>
+ @return the queue that the application was placed in.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>Set the queue that the application was placed in.<p>]]>
+      </doc>
+    </method>
+    <method name="getContainersFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the list of running containers as viewed by
+ <code>ResourceManager</code> from previous application attempts.
+ </p>
+ 
+ @return the list of running containers as viewed by
+         <code>ResourceManager</code> from previous application attempts
+ @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getNMTokensFromPreviousAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of NMTokens for communicating with the NMs where the
+ containers of previous application attempts are running.
+ 
+ @return the list of NMTokens for communicating with the NMs where the
+         containers of previous application attempts are running.
+ 
+ @see RegisterApplicationMasterResponse#getContainersFromPreviousAttempts()]]>
+      </doc>
+    </method>
+    <method name="getSchedulerResourceTypes" return="java.util.EnumSet"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a set of the resource types considered by the scheduler.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get list of supported resource profiles from RM.
+
+ @return a map of resource profiles and its capabilities.]]>
+      </doc>
+    </method>
+    <method name="getResourceTypes" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get available resource types supported by RM.
+
+ @return a Map of RM settings]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The response sent by the {@code ResourceManager} to a new
+ {@code ApplicationMaster} on registration.
+ <p>
+ The response contains critical details such as:
+ <ul>
+   <li>Maximum capability for allocated resources in the cluster.</li>
+   <li>{@code ApplicationACL}s for the application.</li>
+   <li>ClientToAMToken master key.</li>
+ </ul>
+ 
+ @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <class name="ReInitializeContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[Creates a new instance of the ReInitializationContainerRequest.
+ @param containerId Container Id.
+ @param containerLaunchContext Container Launch Context.
+ @param autoCommit AutoCommit.
+ @return ReInitializationContainerRequest.]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to re-initialize.
+
+ @return <code>ContainerId</code> of the container to re-initialize.]]>
+      </doc>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to re-initialize the container
+ with.
+
+ @return <code>ContainerLaunchContext</code> of to re-initialize the
+ container with.]]>
+      </doc>
+    </method>
+    <method name="getAutoCommit" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check if AutoCommit is set for this ReInitialization.
+ @return If AutoCommit is set for this ReInitialization.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates all the required fields needed for a Container
+ ReInitialization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <class name="ReInitializeContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReInitializeContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ReInitializeContainerRequest}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <class name="ReleaseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be released.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be released.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be released.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be released.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request from clients to release a resource in the shared cache.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <class name="ReleaseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReleaseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>
+ The response to clients from the <code>SharedCacheManager</code> when
+ releasing a resource in the shared cache.
+ </p>
+
+ <p>
+ Currently, this is empty.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReleaseSharedCacheResourceResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <class name="ReservationDeleteRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDeleteRequest} captures the set of requirements the user
+ has to delete an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <class name="ReservationDeleteResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDeleteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationDeleteResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation delete
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <class name="ReservationListRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+ @param endTime Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via  {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@code ReservationDefinition}.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[The {@link ReservationListRequest} will use the reservationId to search for
+ reservations to list if it is provided. Otherwise, it will select active
+ reservations within the startTime and endTime (inclusive).
+
+ @param queue Required. Cannot be null or empty. Refers to the reservable
+              queue in the scheduler that was selected when creating a
+              reservation submission {@link ReservationSubmissionRequest}.
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.
+ @return the list of reservations via {@link ReservationListRequest}]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get queue name to use to find reservations.
+
+ @return the queue name to use to find reservations.]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set queue name to use to find resource allocations.
+
+ @param queue Required. Cannot be null or empty.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id to use to find a reservation.
+
+ @return the reservation id of the reservation.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reservation id to use to find a reservation.
+
+ @param reservationId Optional. String representation of
+                     {@code ReservationId} If provided, other fields will
+                     be ignored.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @return the start time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <doc>
+      <![CDATA[Set the start time to use to search for reservations.
+ When this is set, reservations that start before this start
+ time are ignored.
+
+ @param startTime Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setEndTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+      <doc>
+      <![CDATA[Set the end time to use to search for reservations.
+ When this is set, reservations that start after this end
+ time are ignored.
+
+ @param endTime Optional. If provided, only reservations that
+                start before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.]]>
+      </doc>
+    </method>
+    <method name="getIncludeResourceAllocations" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @return the end time to use to search for reservations.]]>
+      </doc>
+    </method>
+    <method name="setIncludeResourceAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includeReservationAllocations" type="boolean"/>
+      <doc>
+      <![CDATA[Set the boolean representing whether or not the user
+ is requesting the full resource allocation.
+ If this is true, the full resource allocation will
+ be included in the response.
+
+ @param includeReservationAllocations Optional. Flag that
+                determines whether the entire list of
+                {@code ResourceAllocationRequest} will be returned.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListRequest} captures the set of requirements the
+ user has to list reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <class name="ReservationListResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationListResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getReservationAllocationState" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationAllocationState}, that corresponds
+ to a reservation in the scheduler.
+
+ @return the list of {@link ReservationAllocationState} which holds
+ information of a particular reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationListResponse} captures the list of reservations that the
+ user has queried.
+
+ The resulting list of {@link ReservationAllocationState} contains a list of
+ {@code ResourceAllocationRequest} representing the current state of the
+ reservation resource allocations will be returned. This is subject to change
+ in the event of re-planning a described by {@code ReservationDefinition}
+
+ @see ReservationAllocationState]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <class name="ReservationSubmissionRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the user constraints for
+ this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to.
+ 
+ @return the name of the {@code Plan} that corresponds to the name of the
+         {@link QueueInfo} in the scheduler to which the reservation will be
+         submitted to]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name of the {@code Plan} that corresponds to the name of the
+ {@link QueueInfo} in the scheduler to which the reservation will be
+ submitted to
+ 
+ @param queueName the name of the parent {@code Plan} that corresponds to
+          the name of the {@link QueueInfo} in the scheduler to which the
+          reservation will be submitted to]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id that corresponds to the reservation submission.
+
+ @return reservation id that will be used to identify the reservation
+ submission.]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id that corresponds to the reservation submission.
+
+ @param reservationId reservation id that will be used to identify the
+                      reservation submission.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationSubmissionRequest} captures the set of requirements the
+ user has to create a reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <class name="ReservationSubmissionResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationSubmissionResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ reservation submission.</p>
+
+ <p>Currently, this is empty.</p>
+
+ {@code ApplicationClientProtocol#submitReservation(
+ ReservationSubmissionRequest)}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <class name="ReservationUpdateRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @return the reservation definition representing user constraints]]>
+      </doc>
+    </method>
+    <method name="setReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationDefinition} representing the updated user
+ constraints for this reservation
+ 
+ @param reservationDefinition the reservation request representing the
+          reservation]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationId}, that corresponds to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @return the {@link ReservationId} representing the unique id of the
+         corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationId}, that correspond to a valid resource
+ allocation in the scheduler (between start and end time of this
+ reservation)
+ 
+ @param reservationId the {@link ReservationId} representing the the unique
+          id of the corresponding reserved resource allocation in the
+          scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationUpdateRequest} captures the set of requirements the user
+ has to update an existing reservation.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <class name="ReservationUpdateResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationUpdateResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[{@link ReservationUpdateResponse} contains the answer of the admission
+ control system in the {@code ResourceManager} to a reservation update
+ operation. Currently response is empty if the operation was successful, if
+ not an exception reporting reason for a failure.
+ 
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <class name="ResourceLocalizationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="localResources" type="java.util.Map"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to localize resources.
+
+ @return <code>ContainerId</code> of the container to localize resources.]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The request sent by the ApplicationMaster to ask for localizing resources.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <class name="ResourceLocalizationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceLocalizationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to the {@link ResourceLocalizationRequest}]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <class name="RestartContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RestartContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The response to a restart Container request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <class name="RollbackResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RollbackResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Response to a Rollback request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <class name="SignalContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="signalContainerCommand" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container to signal.
+ @return <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container to signal.]]>
+      </doc>
+    </method>
+    <method name="getCommand" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>SignalContainerCommand</code> of the signal request.
+ @return <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <method name="setCommand"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <doc>
+      <![CDATA[Set the <code>SignalContainerCommand</code> of the signal request.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the client to the <code>ResourceManager</code>
+ or by the <code>ApplicationMaster</code> to the <code>NodeManager</code>
+ to signal a container.
+ @see SignalContainerCommand </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <class name="SignalContainerResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SignalContainerResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to the client
+ signalling a container.</p>
+
+ <p>Currently it's empty.</p>
+
+ @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <class name="StartContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getContainerLaunchContext" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>.
+ 
+ @return <code>ContainerLaunchContext</code> for the container to be started
+         by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> for the container to be started
+ by the <code>NodeManager</code>
+ @param context <code>ContainerLaunchContext</code> for the container to be 
+                started by the <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the container token to be used for authorization during starting
+ container.
+ <p>
+ Note: {@link NMToken} will be used for authenticating communication with
+ {@code NodeManager}.
+ @return the container token to be used for authorization during starting
+ container.
+ @see NMToken
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="setContainerToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>start</em> a container.</p>
+ 
+ <p>The <code>ApplicationMaster</code> has to provide details such as
+ allocated resource capability, security tokens (if enabled), command
+ to be executed to start the container, environment for the process, 
+ necessary binaries/jar/shared-objects etc. via the 
+ {@link ContainerLaunchContext}.</p>
+
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <class name="StartContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+    </method>
+    <method name="getStartContainerRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link StartContainerRequest} to start containers.
+ @return a list of {@link StartContainerRequest} to start containers.]]>
+      </doc>
+    </method>
+    <method name="setStartContainerRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set a list of {@link StartContainerRequest} to start containers.
+ @param request a list of {@link StartContainerRequest} to start containers]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request which contains a list of {@link StartContainerRequest} sent by
+ the <code>ApplicationMaster</code> to the <code>NodeManager</code> to
+ <em>start</em> containers.
+ </p>
+ 
+ <p>
+ In each {@link StartContainerRequest}, the <code>ApplicationMaster</code> has
+ to provide details such as allocated resource capability, security tokens (if
+ enabled), command to be executed to start the container, environment for the
+ process, necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <class name="StartContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StartContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStartedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>ContainerId</code> s of the containers that are
+ started successfully.
+ 
+ @return the list of <code>ContainerId</code> s of the containers that are
+         started successfully.
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <method name="getAllServicesMetaData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get the meta-data from all auxiliary services running on the
+ <code>NodeManager</code>.
+ </p>
+ <p>
+ The meta-data is returned as a Map between the auxiliary service names and
+ their corresponding per service meta-data as an opaque blob
+ <code>ByteBuffer</code>
+ </p>
+ 
+ <p>
+ To be able to interpret the per-service meta-data, you should consult the
+ documentation for the Auxiliary-service configured on the NodeManager
+ </p>
+ 
+ @return a Map between the names of auxiliary services and their
+         corresponding meta-data]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>start</em> an allocated
+ container.
+ </p>
+ 
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <class name="StopContainersRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+    </method>
+    <method name="getContainerIds" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code>s of the containers to be stopped.
+ @return <code>ContainerId</code>s of containers to be stopped]]>
+      </doc>
+    </method>
+    <method name="setContainerIds"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIds" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code>s of the containers to be stopped.
+ @param containerIds <code>ContainerId</code>s of the containers to be stopped]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by the <code>ApplicationMaster</code> to the
+ <code>NodeManager</code> to <em>stop</em> containers.</p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <class name="StopContainersResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StopContainersResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getSuccessfullyStoppedContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of containerIds of successfully stopped containers.
+ 
+ @return the list of containerIds of successfully stopped containers.]]>
+      </doc>
+    </method>
+    <method name="getFailedRequests" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the containerId-to-exception map in which the exception indicates error
+ from per container for failed requests
+ @return map of containerId-to-exception]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>NodeManager</code> to the
+ <code>ApplicationMaster</code> when asked to <em>stop</em> allocated
+ containers.
+ </p>
+ 
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <class name="SubmitApplicationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationSubmissionContext</code> for the application.
+ @return <code>ApplicationSubmissionContext</code> for the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationSubmissionContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationSubmissionContext</code> for the application.
+ @param context <code>ApplicationSubmissionContext</code> for the 
+                application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>The request sent by a client to <em>submit an application</em> to the 
+ <code>ResourceManager</code>.</p>
+ 
+ <p>The request, via {@link ApplicationSubmissionContext}, contains
+ details such as queue, {@link Resource} required to run the 
+ <code>ApplicationMaster</code>, the equivalent of 
+ {@link ContainerLaunchContext} for launching the 
+ <code>ApplicationMaster</code> etc.
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <class name="SubmitApplicationResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubmitApplicationResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[<p>The response sent by the <code>ResourceManager</code> to a client on
+ application submission.</p>
+ 
+ <p>Currently, this is empty.</p>
+ 
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <class name="UpdateApplicationPriorityRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ 
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ 
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ 
+ @return <code>Priority</code> of the application to be set.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+ 
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application priority.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and
+ {@link Priority} to be set for an application
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <class name="UpdateApplicationPriorityResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationPriorityResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application to be set.
+ @return Updated <code>Priority</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the application.
+
+ @param priority <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ the application priority.
+ </p>
+ <p>
+ A response without exception means that the move has completed successfully.
+ </p>
+ 
+ @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <class name="UpdateApplicationTimeoutsRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationTimeouts" type="java.util.Map"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the application.
+ @param applicationId <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request sent by the client to the <code>ResourceManager</code> to set or
+ update the application timeout.
+ </p>
+ <p>
+ The request includes the {@link ApplicationId} of the application and timeout
+ to be set for an application
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <class name="UpdateApplicationTimeoutsResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateApplicationTimeoutsResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in ISO8601 standard with format <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application. Timeout value
+ is absolute. Timeout value should meet ISO8601 format. Support ISO8601
+ format is <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>. All pre-existing Map entries
+ are cleared before adding the new Map.
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response sent by the <code>ResourceManager</code> to the client on update
+ application timeout.
+ </p>
+ <p>
+ A response without exception means that the update has completed
+ successfully.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <class name="UseSharedCacheResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the resource to be used.
+
+ @return <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="setAppId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the resource to be used.
+
+ @param id <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>key</code> of the resource to be used.
+
+ @return <code>key</code>]]>
+      </doc>
+    </method>
+    <method name="setResourceKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>key</code> of the resource to be used.
+
+ @param key unique identifier for the resource]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The request from clients to the <code>SharedCacheManager</code> that claims a
+ resource in the shared cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+  <class name="UseSharedCacheResourceResponse" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UseSharedCacheResourceResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPath" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Path</code> corresponding to the requested resource in the
+ shared cache.
+
+ @return String A <code>Path</code> if the resource exists in the shared
+         cache, <code>null</code> otherwise]]>
+      </doc>
+    </method>
+    <method name="setPath"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>Path</code> corresponding to a resource in the shared cache.
+
+ @param p A <code>Path</code> corresponding to a resource in the shared
+          cache]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The response from the SharedCacheManager to the client that indicates whether
+ a requested resource exists in the cache.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.protocolrecords.UseSharedCacheResourceResponse -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records">
+  <!-- start class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <class name="AMCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.AMCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.AMCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Command sent by the Resource Manager to the Application Master in the 
+ AllocateResponse 
+ @see AllocateResponse]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.AMCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <class name="ApplicationAccessType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationAccessType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application access types.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAccessType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <class name="ApplicationAttemptId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationAttemptId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="attemptId" type="int"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the <code>ApplicationAttempId</code>. 
+ @return <code>ApplicationId</code> of the <code>ApplicationAttempId</code>]]>
+      </doc>
+    </method>
+    <method name="getAttemptId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>attempt id</code> of the <code>Application</code>.
+ @return <code>attempt id</code> of the <code>Application</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <field name="appAttemptIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
+ of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
+ 
+ <p>Multiple attempts might be needed to run an application to completion due
+ to temporal failures of the <code>ApplicationMaster</code> such as hardware
+ failures, connectivity issues etc. on the node on which it was scheduled.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <class name="ApplicationAttemptReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="rpcPort" type="int"/>
+      <param name="url" type="java.lang.String"/>
+      <param name="oUrl" type="java.lang.String"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <param name="state" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"/>
+      <param name="amContainerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getYarnApplicationAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>YarnApplicationAttemptState</em> of the application attempt.
+ 
+ @return <em>YarnApplicationAttemptState</em> of the application attempt]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of this attempt <code>ApplicationMaster</code>.
+ 
+ @return <em>RPC port</em> of this attempt <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which this attempt of
+ <code>ApplicationMaster</code> is running.
+ 
+ @return <em>host</em> on which this attempt of
+         <code>ApplicationMaster</code> is running]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnositic information</em> of the application attempt in case
+ of errors.
+ 
+ @return <em>diagnositic information</em> of the application attempt in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application attempt.
+ 
+ @return <em>tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>original tracking url</em> for the application attempt.
+ 
+ @return <em>original tracking url</em> for the application attempt]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of this attempt of the
+ application
+ 
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getAMContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of AMContainer for this attempt
+ 
+ @return <code>ContainerId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ 
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationAttemptReport} is a report of an application attempt.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationAttemptId} of the application.</li>
+   <li>Host on which the <code>ApplicationMaster</code> of this attempt is
+   running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code> of this attempt.</li>
+   <li>Tracking URL.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>{@link YarnApplicationAttemptState} of the application attempt.</li>
+   <li>{@link ContainerId} of the master Container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationAttemptReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <class name="ApplicationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="clusterTimestamp" type="long"/>
+      <param name="id" type="int"/>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the short integer identifier of the <code>ApplicationId</code>
+ which is unique for all applications started by a particular instance
+ of the <code>ResourceManager</code>.
+ @return short integer identifier of the <code>ApplicationId</code>]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the <code>ResourceManager</code> which is 
+ used to generate globally unique <code>ApplicationId</code>.
+ @return <em>start time</em> of the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="appIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ApplicationId</code> represents the <em>globally unique</em> 
+ identifier for an application.</p>
+ 
+ <p>The globally unique nature of the identifier is achieved by using the 
+ <em>cluster timestamp</em> i.e. start-time of the 
+ <code>ResourceManager</code> along with a monotonically increasing counter
+ for the application.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <class name="ApplicationReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the application.
+ @return <code>ApplicationId</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getCurrentApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the current
+ attempt of the application
+ @return <code>ApplicationAttemptId</code> of the attempt]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>user</em> who submitted the application.
+ @return <em>user</em> who submitted the application]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application was submitted.
+ @return <em>queue</em> to which the application was submitted]]>
+      </doc>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-defined <em>name</em> of the application.
+ @return <em>name</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ is running.
+ @return <em>host</em> on which the <code>ApplicationMaster</code>
+         is running]]>
+      </doc>
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>RPC port</em> of the <code>ApplicationMaster</code>.
+ @return <em>RPC port</em> of the <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getClientToAMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>.
+ <p>
+ <em>ClientToAMToken</em> is the security token used by the AMs to verify
+ authenticity of any <code>client</code>.
+ </p>
+
+ <p>
+ The <code>ResourceManager</code>, provides a secure token (via
+ {@link ApplicationReport#getClientToAMToken()}) which is verified by the
+ ApplicationMaster when the client directly talks to an AM.
+ </p>
+ @return <em>client token</em> for communicating with the
+ <code>ApplicationMaster</code>]]>
+      </doc>
+    </method>
+    <method name="getYarnApplicationState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>YarnApplicationState</code> of the application.
+ @return <code>YarnApplicationState</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get  the <em>diagnositic information</em> of the application in case of
+ errors.
+ @return <em>diagnositic information</em> of the application in case
+         of errors]]>
+      </doc>
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>tracking url</em> for the application.
+ @return <em>tracking url</em> for the application]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the application.
+ @return <em>start time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getSubmitTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>finish time</em> of the application.
+ @return <em>finish time</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getFinalApplicationStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>final finish status</em> of the application.
+ @return <em>final finish status</em> of the application]]>
+      </doc>
+    </method>
+    <method name="getApplicationResourceUsageReport" return="org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve the structure containing the job resources for this application
+ @return the job resources structure for this application]]>
+      </doc>
+    </method>
+    <method name="getProgress" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's progress ( range 0.0 to 1.0 )
+ @return application's progress]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application's Type 
+ @return application's Type]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all tags corresponding to the application
+ @return Application's tags]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+ 
+ @return the AM to RM token if available.]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationStatus" return="org.apache.hadoop.yarn.api.records.LogAggregationStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get log aggregation status for the application
+ @return Application's log aggregation status]]>
+      </doc>
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedApp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="unmanagedApplication" type="boolean"/>
+      <doc>
+      <![CDATA[@param unmanagedApplication true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get priority of the application
+
+ @return Application's priority]]>
+      </doc>
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAppNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the default Node Label expression for all the application's containers
+
+ @return Application's NodeLabelExpression]]>
+      </doc>
+    </method>
+    <method name="setAmNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amNodeLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationReport} is a report of an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Applications user.</li>
+   <li>Application queue.</li>
+   <li>Application name.</li>
+   <li>Host on which the <code>ApplicationMaster</code> is running.</li>
+   <li>RPC port of the <code>ApplicationMaster</code>.</li>
+   <li>Tracking URL.</li>
+   <li>{@link YarnApplicationState} of the application.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Start time of the application.</li>
+   <li>Client {@link Token} of the application (if security is enabled).</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <class name="ApplicationResourceUsageReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationResourceUsageReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumUsedContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of used containers.  -1 for invalid/inaccessible reports.
+ @return the number of used containers]]>
+      </doc>
+    </method>
+    <method name="getUsedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the used <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the used <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getReservedResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the reserved <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getNeededResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the needed <code>Resource</code>.  -1 for invalid/inaccessible reports.
+ @return the needed <code>Resource</code>]]>
+      </doc>
+    </method>
+    <method name="getMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory (in megabytes) the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores that the application has allocated
+ times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getQueueUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the queue that the app is using.
+ @return the percentage of resources of the queue that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getClusterUsagePercentage" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the percentage of resources of the cluster that the app is using.
+ @return the percentage of resources of the cluster that the app is using.]]>
+      </doc>
+    </method>
+    <method name="getPreemptedMemorySeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated amount of memory preempted(in megabytes)
+ the application has allocated times the number of
+ seconds the application has been running.
+ @return the aggregated amount of memory seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedVcoreSeconds" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of vcores preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return the aggregated number of vcore seconds]]>
+      </doc>
+    </method>
+    <method name="getResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated resource-seconds]]>
+      </doc>
+    </method>
+    <method name="getPreemptedResourceSecondsMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the aggregated number of resources preempted that the application has
+ allocated times the number of seconds the application has been running.
+ @return map containing the resource name and aggregated preempted
+ resource-seconds]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Contains various scheduling metrics to be reported by UI and CLI.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <class name="ApplicationSubmissionContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="amContainerLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="appLabelExpression" type="java.lang.String"/>
+      <param name="resourceRequest" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="attemptFailuresValidityInterval" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="applicationName" type="java.lang.String"/>
+      <param name="queue" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="isUnmanagedAM" type="boolean"/>
+      <param name="cancelTokensWhenComplete" type="boolean"/>
+      <param name="maxAppAttempts" type="int"/>
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="applicationType" type="java.lang.String"/>
+      <param name="keepContainers" type="boolean"/>
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the submitted application.
+ @return <code>ApplicationId</code> of the submitted application]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationId</code> of the submitted application.
+ @param applicationId <code>ApplicationId</code> of the submitted
+                      application]]>
+      </doc>
+    </method>
+    <method name="getApplicationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application <em>name</em>.
+ @return application name]]>
+      </doc>
+    </method>
+    <method name="setApplicationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application <em>name</em>.
+ @param applicationName application name]]>
+      </doc>
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue</em> to which the application is being submitted.
+ @return <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="setQueue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>queue</em> to which the application is being submitted
+ @param queue <em>queue</em> to which the application is being submitted]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the application.
+ @return <code>Priority</code> of the application]]>
+      </doc>
+    </method>
+    <method name="getAMContainerSpec" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @return <code>ContainerLaunchContext</code> for the 
+         <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerSpec"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amContainer" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerLaunchContext</code> to describe the 
+ <code>Container</code> with which the <code>ApplicationMaster</code> is
+ launched.
+ @param amContainer <code>ContainerLaunchContext</code> for the 
+                    <code>ApplicationMaster</code> container]]>
+      </doc>
+    </method>
+    <method name="getUnmanagedAM" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get if the RM should manage the execution of the AM. 
+ If true, then the RM 
+ will not allocate a container for the AM and start it. It will expect the 
+ AM to be launched and connect to the RM within the AM liveliness period and 
+ fail the app otherwise. The client should launch the AM only after the RM 
+ has ACCEPTED the application and changed the <code>YarnApplicationState</code>.
+ Such apps will not be retried by the RM on app attempt failure.
+ The default value is false.
+ @return true if the AM is not managed by the RM]]>
+      </doc>
+    </method>
+    <method name="setUnmanagedAM"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="value" type="boolean"/>
+      <doc>
+      <![CDATA[@param value true if RM should not manage the AM]]>
+      </doc>
+    </method>
+    <method name="getMaxAppAttempts" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return the number of max attempts of the application to be submitted]]>
+      </doc>
+    </method>
+    <method name="setMaxAppAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxAppAttempts" type="int"/>
+      <doc>
+      <![CDATA[Set the number of max attempts of the application to be submitted. WARNING:
+ it should be no larger than the global number of max attempts in the Yarn
+ configuration.
+ @param maxAppAttempts the number of max attempts of the application
+ to be submitted.]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource required by the <code>ApplicationMaster</code> for this
+ application. Please note this will be DEPRECATED, use <em>getResource</em>
+ in <em>getAMContainerResourceRequest</em> instead.
+ 
+ @return the resource required by the <code>ApplicationMaster</code> for
+         this application.]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the resource required by the <code>ApplicationMaster</code> for this
+ application.
+
+ @param resource the resource required by the <code>ApplicationMaster</code>
+ for this application.]]>
+      </doc>
+    </method>
+    <method name="getApplicationType" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application type
+ 
+ @return the application type]]>
+      </doc>
+    </method>
+    <method name="setApplicationType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the application type
+ 
+ @param applicationType the application type]]>
+      </doc>
+    </method>
+    <method name="getKeepContainersAcrossApplicationAttempts" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the flag which indicates whether to keep containers across application
+ attempts or not.
+ 
+ @return the flag which indicates whether to keep containers across
+         application attempts or not.]]>
+      </doc>
+    </method>
+    <method name="setKeepContainersAcrossApplicationAttempts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="keepContainers" type="boolean"/>
+      <doc>
+      <![CDATA[Set the flag which indicates whether to keep containers across application
+ attempts.
+ <p>
+ For managed AM, if the flag is true, running containers will not be killed
+ when application attempt fails and these containers will be retrieved by
+ the new application attempt on registration via
+ {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}.
+ </p>
+ <p>
+ For unmanaged AM, if the flag is true, RM allows re-register and returns
+ the running containers in the same attempt back to the UAM for HA.
+ </p>
+
+ @param keepContainers the flag which indicates whether to keep containers
+          across application attempts.]]>
+      </doc>
+    </method>
+    <method name="getApplicationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get tags for the application
+
+ @return the application tags]]>
+      </doc>
+    </method>
+    <method name="setApplicationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set tags for the application. A maximum of
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAGS} are allowed
+ per application. Each tag can be at most
+ {@link YarnConfiguration#RM_APPLICATION_MAX_TAG_LENGTH}
+ characters, and can contain only ASCII characters.
+
+ @param tags tags to set]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this app. If this is set, all containers of
+ this application without setting node-label-expression in ResurceRequest
+ will get allocated resources on only those nodes that satisfy this
+ node-label-expression.
+ 
+ If different node-label-expression of this app and ResourceRequest are set
+ at the same time, the one set in ResourceRequest will be used when
+ allocating container
+ 
+ @return node-label-expression for this app]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node-label-expression for this app
+ @param nodeLabelExpression node-label-expression of this app]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #getAMContainerResourceRequests()}">
+      <doc>
+      <![CDATA[Get the ResourceRequest of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will assemble a ResourceRequest by using
+ <em>getResource</em> and <em>getPriority</em> of
+ <em>ApplicationSubmissionContext</em>.
+
+ Number of containers and Priority will be ignored.
+
+ @return ResourceRequest of the AM container
+ @deprecated See {@link #getAMContainerResourceRequests()}]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="See {@link #setAMContainerResourceRequests(List)}">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Set ResourceRequest of the AM container
+ @param request of the AM container
+ @deprecated See {@link #setAMContainerResourceRequests(List)}]]>
+      </doc>
+    </method>
+    <method name="getAMContainerResourceRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ResourceRequests of the AM container.
+
+ If this is not null, scheduler will use this to acquire resource for AM
+ container.
+
+ If this is null, scheduler will use the ResourceRequest as determined by
+ <em>getAMContainerResourceRequest</em> and its behavior.
+
+ Number of containers and Priority will be ignored.
+
+ @return List of ResourceRequests of the AM container]]>
+      </doc>
+    </method>
+    <method name="setAMContainerResourceRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="requests" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set ResourceRequests of the AM container.
+ @param requests of the AM container]]>
+      </doc>
+    </method>
+    <method name="getAttemptFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the attemptFailuresValidityInterval in milliseconds for the application
+
+ @return the attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="setAttemptFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attemptFailuresValidityInterval" type="long"/>
+      <doc>
+      <![CDATA[Set the attemptFailuresValidityInterval in milliseconds for the application
+ @param attemptFailuresValidityInterval]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LogAggregationContext</code> of the application
+
+ @return <code>LogAggregationContext</code> of the application]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logAggregationContext" type="org.apache.hadoop.yarn.api.records.LogAggregationContext"/>
+      <doc>
+      <![CDATA[Set <code>LogAggregationContext</code> for the application
+
+ @param logAggregationContext
+          for the application]]>
+      </doc>
+    </method>
+    <method name="getReservationID" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation id, that corresponds to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @return the reservation id representing the unique id of the corresponding
+         reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationID"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationID" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <doc>
+      <![CDATA[Set the reservation id, that correspond to a valid resource allocation in
+ the scheduler (between start and end time of the corresponding reservation)
+ 
+ @param reservationID representing the unique id of the
+          corresponding reserved resource allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getApplicationTimeouts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ApplicationTimeouts</code> of the application. Timeout value is
+ in seconds.
+ @return all <code>ApplicationTimeouts</code> of the application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationTimeouts"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTimeouts" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationTimeouts</code> for the application in seconds.
+ All pre-existing Map entries are cleared before adding the new Map.
+ <p>
+ <b>Note:</b> If application timeout value is less than or equal to zero
+ then application submission will throw an exception.
+ </p>
+ @param applicationTimeouts <code>ApplicationTimeouts</code>s for the
+          application]]>
+      </doc>
+    </method>
+    <method name="getApplicationSchedulingPropertiesMap" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get application scheduling environment variables stored as a key value
+ pair map for application.
+
+ @return placement envs for application.]]>
+      </doc>
+    </method>
+    <method name="setApplicationSchedulingPropertiesMap"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingEnvMap" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the scheduling envs for the application.
+
+ @param schedulingEnvMap
+          A map of env's for the application scheduling preferences.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationSubmissionContext} represents all of the
+ information needed by the {@code ResourceManager} to launch
+ the {@code ApplicationMaster} for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ApplicationId} of the application.</li>
+   <li>Application user.</li>
+   <li>Application name.</li>
+   <li>{@link Priority} of the application.</li>
+   <li>
+     {@link ContainerLaunchContext} of the container in which the
+     <code>ApplicationMaster</code> is executed.
+   </li>
+   <li>
+     maxAppAttempts. The maximum number of application attempts.
+     It should be no larger than the global number of max attempts in the
+     YARN configuration.
+   </li>
+   <li>
+     attemptFailuresValidityInterval. The default value is -1.
+     when attemptFailuresValidityInterval in milliseconds is set to
+     {@literal >} 0, the failure number will no take failures which happen
+     out of the validityInterval into failure count. If failure count
+     reaches to maxAppAttempts, the application will be failed.
+   </li>
+   <li>Optional, application-specific {@link LogAggregationContext}</li>
+ </ul>
+ 
+ @see ContainerLaunchContext
+ @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <class name="ApplicationTimeout" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTimeout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ApplicationTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <param name="expiryTime" type="java.lang.String"/>
+      <param name="remainingTime" type="long"/>
+    </method>
+    <method name="getTimeoutType" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the application timeout type.
+ @return timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="setTimeoutType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timeoutType" type="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"/>
+      <doc>
+      <![CDATA[Set the application timeout type.
+ @param timeoutType of an application timeout.]]>
+      </doc>
+    </method>
+    <method name="getExpiryTime" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>expiryTime</code> for given timeout type.
+ @return expiryTime in ISO8601 standard with format
+         <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="setExpiryTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expiryTime" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set <code>expiryTime</code> for given timeout type.
+ @param expiryTime in ISO8601 standard with format
+          <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b>.]]>
+      </doc>
+    </method>
+    <method name="getRemainingTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>Remaining Time</code> of an application for given timeout type.
+ @return Remaining Time in seconds.]]>
+      </doc>
+    </method>
+    <method name="setRemainingTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="remainingTime" type="long"/>
+      <doc>
+      <![CDATA[Set <code>Remaining Time</code> of an application for given timeout type.
+ @param remainingTime in seconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ApplicationTimeout} is a report for configured application timeouts.
+ It includes details such as:
+ <ul>
+ <li>{@link ApplicationTimeoutType} of the timeout type.</li>
+ <li>Expiry time in ISO8601 standard with format
+ <b>yyyy-MM-dd'T'HH:mm:ss.SSSZ</b> or "UNLIMITED".</li>
+ <li>Remaining time in seconds.</li>
+ </ul>
+ The possible values for {ExpiryTime, RemainingTimeInSeconds} are
+ <ul>
+ <li>{UNLIMITED,-1} : Timeout is not configured for given timeout type
+ (LIFETIME).</li>
+ <li>{ISO8601 date string, 0} : Timeout is configured and application has
+ completed.</li>
+ <li>{ISO8601 date string, greater than zero} : Timeout is configured and
+ application is RUNNING. Application will be timed out after configured
+ value.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <class name="ApplicationTimeoutType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ApplicationTimeoutType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Application timeout type.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ApplicationTimeoutType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <class name="CollectorInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="CollectorInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.CollectorInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="collectorAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="getCollectorAddr" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCollectorAddr"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="addr" type="java.lang.String"/>
+    </method>
+    <method name="getCollectorToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get delegation token for app collector which AM will use to publish
+ entities.
+ @return the delegation token for app collector.]]>
+      </doc>
+    </method>
+    <method name="setCollectorToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <field name="DEFAULT_TIMESTAMP_VALUE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Collector info containing collector address and collector token passed from
+ RM to AM in Allocate Response.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.CollectorInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Container -->
+  <class name="Container" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Container"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the globally unique identifier for the container.
+ @return globally unique identifier for the container]]>
+      </doc>
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the node on which the container is allocated.
+ @return identifier of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the http uri of the node on which the container is allocated.
+ @return http uri of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exposed ports of the node on which the container is allocated.
+ @return exposed ports of the node on which the container is allocated]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> at which the <code>Container</code> was
+ allocated.
+ @return <code>Priority</code> at which the <code>Container</code> was
+         allocated]]>
+      </doc>
+    </method>
+    <method name="getContainerToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerToken</code> for the container.
+ <p><code>ContainerToken</code> is the security token used by the framework
+ to verify authenticity of any <code>Container</code>.</p>
+
+ <p>The <code>ResourceManager</code>, on container allocation provides a
+ secure token which is verified by the <code>NodeManager</code> on
+ container launch.</p>
+
+ <p>Applications do not need to care about <code>ContainerToken</code>, they
+ are transparently handled by the framework - the allocated
+ <code>Container</code> includes the <code>ContainerToken</code>.</p>
+
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+
+ @return <code>ContainerToken</code> for the container]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to the original {@code
+ ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by
+ this allocated {@code Container}.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}s.
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+
+ @return the <em>ID</em> corresponding to the original  allocation request
+ which is satisfied by this allocation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code Container} represents an allocated resource in the cluster.
+ <p>
+ The {@code ResourceManager} is the sole authority to allocate any
+ {@code Container} to applications. The allocated {@code Container}
+ is always on a single node and has a unique {@link ContainerId}. It has
+ a specific amount of {@link Resource} allocated.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} for the container, which is globally unique.</li>
+   <li>
+     {@link NodeId} of the node on which it is allocated.
+   </li>
+   <li>HTTP uri of the node.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>{@link Priority} at which the container was allocated.</li>
+   <li>
+     Container {@link Token} of the container, used to securely verify
+     authenticity of the allocation.
+   </li>
+ </ul>
+ 
+ Typically, an {@code ApplicationMaster} receives the {@code Container}
+ from the {@code ResourceManager} during resource-negotiation and then
+ talks to the {@code NodeManager} to start/stop containers.
+ 
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Container -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <class name="ContainerExitStatus" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerExitStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <field name="SUCCESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ABORTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc.]]>
+      </doc>
+    </field>
+    <field name="DISKS_FAILED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad.]]>
+      </doc>
+    </field>
+    <field name="PREEMPTED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers preempted by the framework.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_VMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated virtual memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_EXCEEDED_PMEM" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container terminated because of exceeding allocated physical memory.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_APPMASTER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by stop request by the app master.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_RESOURCEMANAGER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the resource manager.]]>
+      </doc>
+    </field>
+    <field name="KILLED_AFTER_APP_COMPLETION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated after the application finished.]]>
+      </doc>
+    </field>
+    <field name="KILLED_BY_CONTAINER_SCHEDULER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated by the ContainerScheduler to make room
+ for another container...]]>
+      </doc>
+    </field>
+    <field name="KILLED_FOR_EXCESS_LOGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container was terminated for generating excess log data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[Container exit statuses indicating special exit circumstances.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerExitStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <class name="ContainerId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerId" type="long"/>
+    </method>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationAttemptId</code> of the application to which the
+ <code>Container</code> was assigned.
+ <p>
+ Note: If containers are kept alive across application attempts via
+ {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ the <code>ContainerId</code> does not necessarily contain the current
+ running application attempt's <code>ApplicationAttemptId</code> This
+ container can be allocated by previously exited application attempt and
+ managed by the current running attempt thus have the previous application
+ attempt's <code>ApplicationAttemptId</code>.
+ </p>
+ 
+ @return <code>ApplicationAttemptId</code> of the application to which the
+         <code>Container</code> was assigned]]>
+      </doc>
+    </method>
+    <method name="getId" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the lower 32 bits of identifier of the <code>ContainerId</code>,
+ which doesn't include epoch. Note that this method will be marked as
+ deprecated, so please use <code>getContainerId</code> instead.
+ @return lower 32 bits of identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the identifier of the <code>ContainerId</code>. Upper 24 bits are
+ reserved as epoch of cluster, and lower 40 bits are reserved as
+ sequential number of containers.
+ @return identifier of the <code>ContainerId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return A string representation of containerId. The format is
+ container_e*epoch*_*clusterTimestamp*_*appId*_*attemptId*_*containerId*
+ when epoch is larger than 0
+ (e.g. container_e17_1410901177871_0001_01_000005).
+ *epoch* is increased when RM restarts or fails over.
+ When epoch is 0, epoch is omitted
+ (e.g. container_1410901177871_0001_01_000005).]]>
+      </doc>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="CONTAINER_ID_BITMASK" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>ContainerId</code> represents a globally unique identifier
+ for a {@link Container} in the cluster.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <class name="ContainerLaunchContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLaunchContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <param name="environment" type="java.util.Map"/>
+      <param name="commands" type="java.util.List"/>
+      <param name="serviceData" type="java.util.Map"/>
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <param name="acls" type="java.util.Map"/>
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+    </method>
+    <method name="getTokens" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the tokens needed by this container. It may include file-system
+ tokens, ApplicationMaster related tokens if this container is an
+ ApplicationMaster or framework level tokens needed by this container to
+ communicate to various services in a secure manner.
+ 
+ @return tokens needed by this container.]]>
+      </doc>
+    </method>
+    <method name="setTokens"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokens" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set security tokens needed by this container.
+ @param tokens security tokens]]>
+      </doc>
+    </method>
+    <method name="getTokensConf" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the configuration used by RM to renew tokens.
+ @return The configuration used by RM to renew the tokens.]]>
+      </doc>
+    </method>
+    <method name="setTokensConf"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="tokensConf" type="java.nio.ByteBuffer"/>
+      <doc>
+      <![CDATA[Set the configuration used by RM to renew the tokens.
+ @param tokensConf The configuration used by RM to renew the tokens]]>
+      </doc>
+    </method>
+    <method name="getLocalResources" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>LocalResource</code> required by the container.
+ @return all <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="setLocalResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="localResources" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set <code>LocalResource</code> required by the container. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param localResources <code>LocalResource</code> required by the container]]>
+      </doc>
+    </method>
+    <method name="getServiceData" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Get application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}.
+ </p>
+ 
+ <p>
+ This will be used to initialize this application on the specific
+ {@link AuxiliaryService} running on the NodeManager by calling
+ {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)}
+ </p>
+ 
+ @return application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="setServiceData"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="serviceData" type="java.util.Map"/>
+      <doc>
+      <![CDATA[<p>
+ Set application-specific binary <em>service data</em>. This is a map keyed
+ by the name of each {@link AuxiliaryService} that is configured on a
+ NodeManager and value correspond to the application specific data targeted
+ for the keyed {@link AuxiliaryService}. All pre-existing Map entries are
+ preserved.
+ </p>
+ 
+ @param serviceData
+          application-specific binary <em>service data</em>]]>
+      </doc>
+    </method>
+    <method name="getEnvironment" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>environment variables</em> for the container.
+ @return <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="setEnvironment"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="environment" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add <em>environment variables</em> for the container. All pre-existing Map
+ entries are cleared before adding the new Map
+ @param environment <em>environment variables</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getCommands" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <em>commands</em> for launching the container.
+ @return the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="setCommands"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="commands" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add the list of <em>commands</em> for launching the container. All
+ pre-existing List entries are cleared before adding the new List
+ @param commands the list of <em>commands</em> for launching the container]]>
+      </doc>
+    </method>
+    <method name="getApplicationACLs" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationACL</code>s for the application. 
+ @return all the <code>ApplicationACL</code>s]]>
+      </doc>
+    </method>
+    <method name="setApplicationACLs"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acls" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the <code>ApplicationACL</code>s for the application. All pre-existing
+ Map entries are cleared before adding the new Map
+ @param acls <code>ApplicationACL</code>s for the application]]>
+      </doc>
+    </method>
+    <method name="getContainerRetryContext" return="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerRetryContext</code> to relaunch container.
+ @return <code>ContainerRetryContext</code> to relaunch container.]]>
+      </doc>
+    </method>
+    <method name="setContainerRetryContext"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerRetryContext" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerRetryContext</code> to relaunch container.
+ @param containerRetryContext <code>ContainerRetryContext</code> to
+                              relaunch container.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerLaunchContext} represents all of the information
+ needed by the {@code NodeManager} to launch a container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+   <li>User to whom the container is allocated.</li>
+   <li>Security tokens (if security is enabled).</li>
+   <li>
+     {@link LocalResource} necessary for running the container such
+     as binaries, jar, shared-objects, side-files etc.
+   </li>
+   <li>Optional, application-specific binary service data.</li>
+   <li>Environment variables for the launched process.</li>
+   <li>Command to launch the container.</li>
+   <li>Retry strategy when container exits with failure.</li>
+ </ul>
+ 
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerLaunchContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <class name="ContainerReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ 
+ @return <code>ContainerId</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="getAllocatedResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Resource</code> of the container.
+ 
+ @return allocated <code>Resource</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getAssignedNode" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>NodeId</code> where container is running.
+ 
+ @return allocated <code>NodeId</code> where container is running.]]>
+      </doc>
+    </method>
+    <method name="setAssignedNode"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated <code>Priority</code> of the container.
+ 
+ @return allocated <code>Priority</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the creation time of the container.
+ 
+ @return creation time of the container]]>
+      </doc>
+    </method>
+    <method name="setCreationTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="creationTime" type="long"/>
+    </method>
+    <method name="getFinishTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Finish time of the container.
+ 
+ @return Finish time of the container]]>
+      </doc>
+    </method>
+    <method name="setFinishTime"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="finishTime" type="long"/>
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the DiagnosticsInfo of the container.
+ 
+ @return DiagnosticsInfo of the container]]>
+      </doc>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the LogURL of the container.
+ 
+ @return LogURL of the container]]>
+      </doc>
+    </method>
+    <method name="setLogUrl"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logUrl" type="java.lang.String"/>
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>ContainerState</code> of the container.
+ 
+ @return final <code>ContainerState</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"/>
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the final <code>exit status</code> of the container.
+ 
+ @return final <code>exit status</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerExitStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerExitStatus" type="int"/>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exposed ports of the container.
+ 
+ @return the node exposed ports of the container]]>
+      </doc>
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Node Http address of the container.
+
+ @return the node http address of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the execution type of the container.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerReport} is a report of an container.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link ContainerId} of the container.</li>
+   <li>Allocated Resources to the container.</li>
+   <li>Assigned Node id.</li>
+   <li>Assigned Priority.</li>
+   <li>Creation Time.</li>
+   <li>Finish Time.</li>
+   <li>Container Exit Status.</li>
+   <li>{@link ContainerState} of the container.</li>
+   <li>Diagnostic information in case of errors.</li>
+   <li>Log URL.</li>
+   <li>nodeHttpAddress</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <class name="ContainerRetryContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerRetryContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRetryPolicy" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryPolicy"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryPolicy" type="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"/>
+    </method>
+    <method name="getErrorCodes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setErrorCodes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCodes" type="java.util.Set"/>
+    </method>
+    <method name="getMaxRetries" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxRetries"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="maxRetries" type="int"/>
+    </method>
+    <method name="getRetryInterval" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRetryInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="retryInterval" type="int"/>
+    </method>
+    <method name="getFailuresValidityInterval" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFailuresValidityInterval"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="failuresValidityInterval" type="long"/>
+    </method>
+    <field name="RETRY_FOREVER" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RETRY_INVALID" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NEVER_RETRY_CONTEXT" type="org.apache.hadoop.yarn.api.records.ContainerRetryContext"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[{@code ContainerRetryContext} indicates how container retry after it fails
+ to run.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>
+     {@link ContainerRetryPolicy} :
+     - NEVER_RETRY(DEFAULT value): no matter what error code is when container
+       fails to run, just do not retry.
+     - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails
+       to run, just retry.
+     - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if
+       the error code is one of <em>errorCodes</em>, otherwise do not retry.
+
+     Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry
+     because it is usually killed on purpose.
+   </li>
+   <li>
+     <em>maxRetries</em> specifies how many times to retry if need to retry.
+     If the value is -1, it means retry forever.
+   </li>
+   <li><em>retryInterval</em> specifies delaying some time before relaunch
+   container, the unit is millisecond.</li>
+   <li>
+     <em>failuresValidityInterval</em>: default value is -1.
+     When failuresValidityInterval in milliseconds is set to {@literal >} 0,
+     the failure number will not take failures which happen out of the
+     failuresValidityInterval into failure count. If failure count
+     reaches to <em>maxRetries</em>, the container will be failed.
+   </li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <class name="ContainerRetryPolicy" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerRetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>Retry policy for relaunching a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerRetryPolicy -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <class name="ContainerState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Container</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <class name="ContainerStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerState</code> of the container.
+ @return <code>ContainerState</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExitStatus" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>Get the <em>exit status</em> for the container.</p>
+  
+ <p>Note: This is valid only for completed containers i.e. containers
+ with state {@link ContainerState#COMPLETE}. 
+ Otherwise, it returns an ContainerExitStatus.INVALID.
+ </p>
+ 
+ <p>Containers killed by the framework, either due to being released by
+ the application or being 'lost' due to node failures etc. have a special
+ exit code of ContainerExitStatus.ABORTED.</p>
+ 
+ <p>When threshold number of the nodemanager-local-directories or
+ threshold number of the nodemanager-log-directories become bad, then
+ container is not launched and is exited with ContainersExitStatus.DISKS_FAILED.
+ </p>
+  
+ @return <em>exit status</em> for the container]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>diagnostic messages</em> for failed containers.
+ @return <em>diagnostic messages</em> for failed containers]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> allocated to the container.
+ @return <code>Resource</code> allocated to the container]]>
+      </doc>
+    </method>
+    <method name="getIPs" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get all the IP addresses with which the container run.
+ @return The IP address where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the hostname where the container runs.
+ @return The hostname where the container runs.]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exposed ports of the container.
+ @return List of exposed ports]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ContainerStatus} represents the current status of a
+ {@code Container}.
+ <p>
+ It provides details such as:
+ <ul>
+   <li>{@code ContainerId} of the container.</li>
+   <li>{@code ExecutionType} of the container.</li>
+   <li>{@code ContainerState} of the container.</li>
+   <li><em>Exit status</em> of a completed container.</li>
+   <li><em>Diagnostic</em> message for a failed container.</li>
+   <li>{@link Resource} allocated to the container.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <class name="ContainerSubState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerSubState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerSubState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container Sub-State.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerSubState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <class name="ContainerUpdateType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Encodes the type of Container Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ContainerUpdateType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <class name="ExecutionType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ExecutionType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Container property encoding execution semantics.
+
+ <p>
+ The execution types are the following:
+ <ul>
+   <li>{@link #GUARANTEED} - this container is guaranteed to start its
+   execution, once the corresponding start container request is received by
+   an NM.
+   <li>{@link #OPPORTUNISTIC} - the execution of this container may not start
+   immediately at the NM that receives the corresponding start container
+   request (depending on the NM's available resources). Moreover, it may be
+   preempted if it blocks a GUARANTEED container from being executed.
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <class name="ExecutionTypeRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ExecutionTypeRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="ensureExecutionType" type="boolean"/>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionType</code> of the requested container.
+
+ @param execType
+          ExecutionType of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <code>ExecutionType</code>.
+
+ @return <code>ExecutionType</code>.]]>
+      </doc>
+    </method>
+    <method name="setEnforceExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enforceExecutionType" type="boolean"/>
+      <doc>
+      <![CDATA[Set to true to explicitly ask that the Scheduling Authority return
+ Containers of exactly the Execution Type requested.
+ @param enforceExecutionType whether ExecutionType request should be
+                            strictly honored.]]>
+      </doc>
+    </method>
+    <method name="getEnforceExecutionType" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether Scheduling Authority should return Containers of exactly the
+ Execution Type requested for this <code>ResourceRequest</code>.
+ Defaults to false.
+ @return whether ExecutionType request should be strictly honored]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object of this class represents a specification of the execution
+ guarantee of the Containers associated with a ResourceRequest. It consists
+ of an <code>ExecutionType</code> as well as flag that explicitly asks the
+ configuredScheduler to return Containers of exactly the Execution Type
+ requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ExecutionTypeRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <class name="FinalApplicationStatus" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various final states of an <code>Application</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.FinalApplicationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalizationState -->
+  <class name="LocalizationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalizationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalizationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of localization.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalizationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalizationStatus -->
+  <class name="LocalizationStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalizationStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalizationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceKey" type="java.lang.String"/>
+      <param name="localizationState" type="org.apache.hadoop.yarn.api.records.LocalizationState"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalizationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceKey" type="java.lang.String"/>
+      <param name="localizationState" type="org.apache.hadoop.yarn.api.records.LocalizationState"/>
+      <param name="diagnostics" type="java.lang.String"/>
+    </method>
+    <method name="getResourceKey" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource key.
+
+ @return resource key.]]>
+      </doc>
+    </method>
+    <method name="getLocalizationState" return="org.apache.hadoop.yarn.api.records.LocalizationState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the localization sate.
+
+ @return localization state.]]>
+      </doc>
+    </method>
+    <method name="getDiagnostics" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the diagnostics.
+
+ @return diagnostics.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Represents the localization status of a resource.
+ The status of the localization includes:
+ <ul>
+   <li>resource key</li>
+   <li>{@link LocalizationState} of the resource</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalizationStatus -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <class name="LocalResource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LocalResource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="pattern" type="java.lang.String"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LocalResource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <param name="size" type="long"/>
+      <param name="timestamp" type="long"/>
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>location</em> of the resource to be localized.
+ @return <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <doc>
+      <![CDATA[Set <em>location</em> of the resource to be localized.
+ @param resource <em>location</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getSize" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>size</em> of the resource to be localized.
+ @return <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setSize"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="size" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>size</em> of the resource to be localized.
+ @param size <em>size</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the original <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @return <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the <em>timestamp</em> of the resource to be localized, used
+ for verification.
+ @param timestamp <em>timestamp</em> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceType</code> of the resource to be localized.
+ @return <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="setType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.LocalResourceType"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceType</code> of the resource to be localized.
+ @param type <code>LocalResourceType</code> of the resource to be localized]]>
+      </doc>
+    </method>
+    <method name="getVisibility" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @return <code>LocalResourceVisibility</code> of the resource to be 
+         localized]]>
+      </doc>
+    </method>
+    <method name="setVisibility"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="visibility" type="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"/>
+      <doc>
+      <![CDATA[Set the <code>LocalResourceVisibility</code> of the resource to be 
+ localized.
+ @param visibility <code>LocalResourceVisibility</code> of the resource to be 
+                   localized]]>
+      </doc>
+    </method>
+    <method name="getPattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @return <em>pattern</em> that should be used to extract entries from the 
+ archive.]]>
+      </doc>
+    </method>
+    <method name="setPattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <em>pattern</em> that should be used to extract entries from the
+ archive (only used when type is <code>PATTERN</code>).
+ @param pattern <em>pattern</em> that should be used to extract entries 
+ from the archive.]]>
+      </doc>
+    </method>
+    <method name="getShouldBeUploadedToSharedCache" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM uses it to decide whether if it is necessary to upload the resource to
+ the shared cache
+ @return true if it is necessary to upload the resource
+                 to the shared cache,
+         false otherwise]]>
+      </doc>
+    </method>
+    <method name="setShouldBeUploadedToSharedCache"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="shouldBeUploadedToSharedCache" type="boolean"/>
+      <doc>
+      <![CDATA[Inform NM whether upload to SCM is needed.
+
+ @param shouldBeUploadedToSharedCache <em>shouldBeUploadedToSharedCache</em>
+          of this request]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>LocalResource</code> represents a local resource required to
+ run a container.</p>
+ 
+ <p>The <code>NodeManager</code> is responsible for localizing the resource 
+ prior to launching the container.</p>
+ 
+ <p>Applications can specify {@link LocalResourceType} and 
+ {@link LocalResourceVisibility}.</p>
+ 
+ @see LocalResourceType
+ @see LocalResourceVisibility
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <class name="LocalResourceType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceType} specifies the <em>type</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>type</em> can be one of:
+ <ul>
+   <li>
+     {@link #FILE} - Regular file i.e. uninterpreted bytes.
+   </li>
+   <li>
+     {@link #ARCHIVE} - Archive, which is automatically unarchived by the
+     <code>NodeManager</code>.
+   </li>
+   <li>
+     {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}.
+   </li>
+ </ul>
+
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <class name="LocalResourceVisibility" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.LocalResourceVisibility"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code LocalResourceVisibility} specifies the <em>visibility</em>
+ of a resource localized by the {@code NodeManager}.
+ <p>
+ The <em>visibility</em> can be one of:
+ <ul>
+   <li>{@link #PUBLIC} - Shared by all users on the node.</li>
+   <li>
+     {@link #PRIVATE} - Shared among all applications of the
+     <em>same user</em> on the node.
+   </li>
+   <li>
+     {@link #APPLICATION} - Shared only among containers of the
+     <em>same application</em> on the node.
+   </li>
+ </ul>
+ 
+ @see LocalResource
+ @see ContainerLaunchContext
+ @see ApplicationSubmissionContext
+ @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LocalResourceVisibility -->
+  <!-- start class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <class name="LogAggregationContext" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <param name="excludePattern" type="java.lang.String"/>
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <param name="policyClassName" type="java.lang.String"/>
+      <param name="policyParameters" type="java.lang.String"/>
+    </method>
+    <method name="getIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="includePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern. This includePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param includePattern]]>
+      </doc>
+    </method>
+    <method name="getExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="excludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern. This excludePattern only takes affect
+ on logs that exist at the time of application finish.
+
+ @param excludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsIncludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get include pattern in a rolling fashion.
+ 
+ @return include pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsIncludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsIncludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set include pattern in a rolling fashion.
+ 
+ @param rolledLogsIncludePattern]]>
+      </doc>
+    </method>
+    <method name="getRolledLogsExcludePattern" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get exclude pattern for aggregation in a rolling fashion.
+ 
+ @return exclude pattern]]>
+      </doc>
+    </method>
+    <method name="setRolledLogsExcludePattern"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rolledLogsExcludePattern" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set exclude pattern for in a rolling fashion.
+ 
+ @param rolledLogsExcludePattern]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyClassName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy class.
+
+ @return log aggregation policy class]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyClassName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="className" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy class.
+
+ @param className]]>
+      </doc>
+    </method>
+    <method name="getLogAggregationPolicyParameters" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation policy parameters.
+
+ @return log aggregation policy parameters]]>
+      </doc>
+    </method>
+    <method name="setLogAggregationPolicyParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the log aggregation policy parameters.
+ There is no schema defined for the parameters string.
+ It is up to the log aggregation policy class to decide how to parse
+ the parameters string.
+
+ @param parameters]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code LogAggregationContext} represents all of the
+ information needed by the {@code NodeManager} to handle
+ the logs for an application.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>
+     includePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be uploaded when the application finishes.
+   </li>
+   <li>
+     excludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be uploaded when application finishes. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     rolledLogsIncludePattern. It uses Java Regex to filter the log files
+     which match the defined include pattern and those log files
+     will be aggregated in a rolling fashion.
+   </li>
+   <li>
+     rolledLogsExcludePattern. It uses Java Regex to filter the log files
+     which match the defined exclude pattern and those log files
+     will not be aggregated in a rolling fashion. If the log file
+     name matches both the include and the exclude pattern, this file
+     will be excluded eventually.
+   </li>
+   <li>
+     policyClassName. The policy class name that implements
+     ContainerLogAggregationPolicy. At runtime, nodemanager will the policy
+     if a given container's log should be aggregated based on the
+     ContainerType and other runtime state such as exit code by calling
+     ContainerLogAggregationPolicy#shouldDoLogAggregation.
+     This is useful when the app only wants to aggregate logs of a subset of
+     containers. Here are the available policies. Please make sure to specify
+     the canonical name by prefixing org.apache.hadoop.yarn.server.
+     nodemanager.containermanager.logaggregation.
+     to the class simple name below.
+     NoneContainerLogAggregationPolicy: skip aggregation for all containers.
+     AllContainerLogAggregationPolicy: aggregate all containers.
+     AMOrFailedContainerLogAggregationPolicy: aggregate application master
+         or failed containers.
+     FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed
+         containers
+     FailedContainerLogAggregationPolicy: aggregate failed containers
+     AMOnlyLogAggregationPolicy: aggregate application master containers
+     SampleContainerLogAggregationPolicy: sample logs of successful worker
+         containers, in addition to application master and failed/killed
+         containers.
+     LimitSizeContainerLogAggregationPolicy: skip aggregation for killed
+         containers whose log size exceeds the limit of container log size.
+     If it isn't specified, it will use the cluster-wide default policy
+     defined by configuration yarn.nodemanager.log-aggregation.policy.class.
+     The default value of yarn.nodemanager.log-aggregation.policy.class is
+     AllContainerLogAggregationPolicy.
+   </li>
+   <li>
+     policyParameters. The parameters passed to the policy class via
+     ContainerLogAggregationPolicy#parseParameters during the policy object
+     initialization. This is optional. Some policy class might use parameters
+     to adjust its settings. It is up to policy class to define the scheme of
+     parameters.
+     For example, SampleContainerLogAggregationPolicy supports the format of
+     "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50
+     successful worker containers.
+   </li>
+ </ul>
+
+ @see ApplicationSubmissionContext]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.LogAggregationContext -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NMToken -->
+  <class name="NMToken" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMToken"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link NodeId} of the <code>NodeManager</code> for which the NMToken
+ is used to authenticate.
+ @return the {@link NodeId} of the <code>NodeManager</code> for which the
+ NMToken is used to authenticate.]]>
+      </doc>
+    </method>
+    <method name="setNodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Token} used for authenticating with <code>NodeManager</code>
+ @return the {@link Token} used for authenticating with <code>NodeManager</code>]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[<p>The NMToken is used for authenticating communication with
+ <code>NodeManager</code></p>
+ <p>It is issued by <code>ResourceMananger</code> when <code>ApplicationMaster</code>
+ negotiates resource with <code>ResourceManager</code> and
+ validated on <code>NodeManager</code> side.</p>
+ @see  AllocateResponse#getNMTokens()]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NMToken -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <class name="NodeAttribute" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttribute"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttribute"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <field name="PREFIX_DISTRIBUTED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PREFIX_CENTRALIZED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute is a kind of a label which represents one of the
+ attribute/feature of a Node. Its different from node partition label as
+ resource guarantees across the queues will not be maintained for these type
+ of labels.
+ </p>
+ <p>
+ A given Node can be mapped with any kind of attribute, few examples are
+ HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ </p>
+ <p>
+ Its not compulsory for all the attributes to have value, empty string is the
+ default value of the <code>NodeAttributeType.STRING</code>
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttribute -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <class name="NodeAttributeInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttribute" type="org.apache.hadoop.yarn.api.records.NodeAttribute"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeInfo"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <method name="getAttributeKey" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeKey"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeKey" type="org.apache.hadoop.yarn.api.records.NodeAttributeKey"/>
+    </method>
+    <method name="getAttributeType" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeType" type="org.apache.hadoop.yarn.api.records.NodeAttributeType"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node Attribute Info describes a NodeAttribute.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <class name="NodeAttributeKey" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeAttributeKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeAttributeKey"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <method name="getAttributePrefix" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributePrefix"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributePrefix" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeName" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute
+ is identified based on attribute prefix and name.
+ </p>
+ <p>
+ Node Attribute Prefix is used as namespace to segregate the attributes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeKey -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <class name="NodeAttributeOpCode" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various node attribute op codes.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeOpCode -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <class name="NodeAttributeType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeAttributeType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeAttributeType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Type of a <code>node Attribute</code>.
+ </p>
+ Based on this attribute expressions and values will be evaluated.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeAttributeType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeId -->
+  <class name="NodeId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>hostname</em> of the node.
+ @return <em>hostname</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>port</em> for communicating with the node.
+ @return <em>port</em> for communicating with the node]]>
+      </doc>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p><code>NodeId</code> is the unique identifier for a node.</p>
+ 
+ <p>It includes the <em>hostname</em> and <em>port</em> to uniquely 
+ identify the node. Thus, it is unique across restarts of any 
+ <code>NodeManager</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <class name="NodeLabel" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="NodeLabel"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isExclusive" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.NodeLabel"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="DEFAULT_NODE_LABEL_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default node label partition used for displaying.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABEL_EXPRESSION_NOT_SET" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node Label expression not set .]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABEL_EXCLUSIVITY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, node label is exclusive or not]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeLabel -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <class name="NodeReport" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeId</code> of the node.
+ @return <code>NodeId</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getNodeState" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>NodeState</code> of the node.
+ @return <code>NodeState</code> of the node]]>
+      </doc>
+    </method>
+    <method name="getHttpAddress" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>http address</em> of the node.
+ @return <em>http address</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getRackName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>rack name</em> for the node.
+ @return <em>rack name</em> for the node]]>
+      </doc>
+    </method>
+    <method name="getUsed" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>used</em> <code>Resource</code> on the node.
+ @return <em>used</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>total</em> <code>Resource</code> on the node.
+ @return <em>total</em> <code>Resource</code> on the node]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <method name="getNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get labels of this node.
+ @return labels of this node.]]>
+      </doc>
+    </method>
+    <method name="getAggregatedContainersUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get containers aggregated resource utilization in a node.
+ @return containers resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getNodeUtilization" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node resource utilization.
+ @return node resource utilization.]]>
+      </doc>
+    </method>
+    <method name="getDecommissioningTimeout" return="java.lang.Integer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional decommissioning timeout in seconds (null indicates absent
+ timeout).
+ @return the decommissioning timeout in second.]]>
+      </doc>
+    </method>
+    <method name="setDecommissioningTimeout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="decommissioningTimeout" type="java.lang.Integer"/>
+      <doc>
+      <![CDATA[Set the decommissioning timeout in seconds (null indicates absent timeout).]]>
+      </doc>
+    </method>
+    <method name="getNodeUpdateType" return="org.apache.hadoop.yarn.api.records.NodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Optional node update type (null indicates absent update type).
+ @return the node update.]]>
+      </doc>
+    </method>
+    <method name="setNodeUpdateType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeUpdateType" type="org.apache.hadoop.yarn.api.records.NodeUpdateType"/>
+      <doc>
+      <![CDATA[Set the node update type (null indicates absent node update type).]]>
+      </doc>
+    </method>
+    <method name="setNodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAttributes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the node attributes of node.
+
+ @param nodeAttributes set of node attributes.]]>
+      </doc>
+    </method>
+    <method name="getNodeAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node attributes of node.
+ @return the set of node attributes.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeReport} is a summary of runtime information of a node
+ in the cluster.
+ <p>
+ It includes details such as:
+ <ul>
+   <li>{@link NodeId} of the node.</li>
+   <li>HTTP Tracking URL of the node.</li>
+   <li>Rack name for the node.</li>
+   <li>Used {@link Resource} on the node.</li>
+   <li>Total available {@link Resource} of the node.</li>
+   <li>Number of running containers on the node.</li>
+ </ul>
+
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeReport -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeState -->
+  <class name="NodeState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.NodeState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.NodeState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isUnusable" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isInactiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isActiveState" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[<p>State of a <code>Node</code>.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <class name="NodeToAttributeValue" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeToAttributeValue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.NodeToAttributeValue"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getAttributeValue" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAttributeValue"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributeValue" type="java.lang.String"/>
+    </method>
+    <method name="getHostname" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHostname"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostname" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ Mapping of Attribute Value to a Node.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.NodeToAttributeValue -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <class name="PreemptionContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Container referenced by this handle.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Specific container requested back by the <code>ResourceManager</code>.
+ @see PreemptionContract
+ @see StrictPreemptionContract]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <class name="PreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the AM releases resources matching these requests, then the {@link
+ PreemptionContainer}s enumerated in {@link #getContainers()} should not be
+ evicted from the cluster. Due to delays in propagating cluster state and
+ sending these messages, there are conditions where satisfied contracts may
+ not prevent the platform from killing containers.
+ @return List of {@link PreemptionResourceRequest} to update the
+ <code>ApplicationMaster</code> about resources requested back by the
+ <code>ResourceManager</code>.
+ @see AllocateRequest#setAskList(List)]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Assign the set of {@link PreemptionContainer} specifying which containers
+ owned by the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>. If the AM prefers a different set of
+ containers, then it may checkpoint or kill containers matching the
+ description in {@link #getResourceRequest}.
+ @return Set of containers at risk if the contract is not met.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the <code>ResourceManager</code>.
+ The <code>ApplicationMaster</code> (AM) can satisfy this request according
+ to its own priorities to prevent containers from being forcibly killed by
+ the platform.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <class name="PreemptionMessage" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionMessage"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getStrictContract" return="org.apache.hadoop.yarn.api.records.StrictPreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Specific resources that may be killed by the
+ <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="getContract" return="org.apache.hadoop.yarn.api.records.PreemptionContract"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Contract describing resources to return to the cluster.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A {@link PreemptionMessage} is part of the RM-AM protocol, and it is used by
+ the RM to specify resources that the RM wants to reclaim from this
+ {@code ApplicationMaster} (AM). The AM receives a {@link
+ StrictPreemptionContract} message encoding which containers the platform may
+ forcibly kill, granting it an opportunity to checkpoint state or adjust its
+ execution plan. The message may also include a {@link PreemptionContract}
+ granting the AM more latitude in selecting which resources to return to the
+ cluster.
+ <p>
+ The AM should decode both parts of the message. The {@link
+ StrictPreemptionContract} specifies particular allocations that the RM
+ requires back. The AM can checkpoint containers' state, adjust its execution
+ plan to move the computation, or take no action and hope that conditions that
+ caused the RM to ask for the container will change.
+ <p>
+ In contrast, the {@link PreemptionContract} also includes a description of
+ resources with a set of containers. If the AM releases containers matching
+ that profile, then the containers enumerated in {@link
+ PreemptionContract#getContainers()} may not be killed.
+ <p>
+ Each preemption message reflects the RM's current understanding of the
+ cluster state, so a request to return <em>N</em> containers may not
+ reflect containers the AM is releasing, recently exited containers the RM has
+ yet to learn about, or new containers allocated before the message was
+ generated. Conversely, an RM may request a different profile of containers in
+ subsequent requests.
+ <p>
+ The policy enforced by the RM is part of the scheduler. Generally, only
+ containers that have been requested consistently should be killed, but the
+ details are not specified.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionMessage -->
+  <!-- start class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <class name="PreemptionResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PreemptionResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getResourceRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return Resource described in this request, to be matched against running
+ containers.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Description of resources requested back by the cluster.
+ @see PreemptionContract
+ @see AllocateRequest#setAskList(java.util.List)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.PreemptionResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Priority -->
+  <class name="Priority" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Priority"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="p" type="int"/>
+    </method>
+    <method name="getPriority" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the assigned priority
+ @return the assigned priority]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="int"/>
+      <doc>
+      <![CDATA[Set the assigned priority
+ @param priority the assigned priority]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="UNDEFINED" type="org.apache.hadoop.yarn.api.records.Priority"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The priority assigned to a ResourceRequest or Application or Container 
+ allocation]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Priority -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <class name="QueueACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code QueueACL} enumerates the various ACLs for queues.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue.
+   </li>
+   <li>{@link #ADMINISTER_QUEUE} - ACL to administer the queue.</li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <class name="QueueInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>name</em> of the queue.
+ @return <em>name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>configured capacity</em> of the queue.
+ @return <em>configured capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getMaximumCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>maximum capacity</em> of the queue.
+ @return <em>maximum capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getCurrentCapacity" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>current capacity</em> of the queue.
+ @return <em>current capacity</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getChildQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>child queues</em> of the queue.
+ @return <em>child queues</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>running applications</em> of the queue.
+ @return <em>running applications</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueState" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>QueueState</code> of the queue.
+ @return <code>QueueState</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getAccessibleNodeLabels" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>accessible node labels</code> of the queue.
+ @return <code>accessible node labels</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="getDefaultNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>default node label expression</code> of the queue, this takes
+ affect only when the <code>ApplicationSubmissionContext</code> and
+ <code>ResourceRequest</code> don't specify their
+ <code>NodeLabelExpression</code>.
+ 
+ @return <code>default node label expression</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setDefaultNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultLabelExpression" type="java.lang.String"/>
+    </method>
+    <method name="getQueueStatistics" return="org.apache.hadoop.yarn.api.records.QueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>queue stats</code> for the queue
+
+ @return <code>queue stats</code> of the queue]]>
+      </doc>
+    </method>
+    <method name="setQueueStatistics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueStatistics" type="org.apache.hadoop.yarn.api.records.QueueStatistics"/>
+      <doc>
+      <![CDATA[Set the queue statistics for the queue
+ 
+ @param queueStatistics
+          the queue statistics]]>
+      </doc>
+    </method>
+    <method name="getPreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>preemption status</em> of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return <em>preemption status</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getQueueConfigurations" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the per-node-label queue configurations of the queue.
+
+ @return the per-node-label queue configurations of the queue.]]>
+      </doc>
+    </method>
+    <method name="getIntraQueuePreemptionDisabled" return="java.lang.Boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the intra-queue preemption status of the queue.
+ @return if property is not in proto, return null;
+        otherwise, return intra-queue preemption status of the queue]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[QueueInfo is a report of the runtime information of the queue.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>Queue name.</li>
+   <li>Capacity of the queue.</li>
+   <li>Maximum capacity of the queue.</li>
+   <li>Current capacity of the queue.</li>
+   <li>Child queues.</li>
+   <li>Running applications.</li>
+   <li>{@link QueueState} of the queue.</li>
+   <li>{@link QueueConfigurations} of the queue.</li>
+ </ul>
+
+ @see QueueState
+ @see QueueConfigurations
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueState -->
+  <class name="QueueState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.QueueState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.QueueState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[State of a Queue.
+ <p>
+ A queue is in one of:
+ <ul>
+   <li>{@link #RUNNING} - normal state.</li>
+   <li>{@link #STOPPED} - not accepting new application submissions.</li>
+   <li>
+     {@link #DRAINING} - not accepting new application submissions
+     and waiting for applications finish.
+   </li>
+ </ul>
+ 
+ @see QueueInfo
+ @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <class name="QueueStatistics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueStatistics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumAppsSubmitted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of apps submitted
+ 
+ @return the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="setNumAppsSubmitted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsSubmitted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of apps submitted
+ 
+ @param numAppsSubmitted
+          the number of apps submitted]]>
+      </doc>
+    </method>
+    <method name="getNumAppsRunning" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of running apps
+ 
+ @return the number of running apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsRunning"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsRunning" type="long"/>
+      <doc>
+      <![CDATA[Set the number of running apps
+ 
+ @param numAppsRunning
+          the number of running apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsPending" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending apps
+ 
+ @return the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsPending"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsPending" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending apps
+ 
+ @param numAppsPending
+          the number of pending apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsCompleted" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of completed apps
+ 
+ @return the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsCompleted"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsCompleted" type="long"/>
+      <doc>
+      <![CDATA[Set the number of completed apps
+ 
+ @param numAppsCompleted
+          the number of completed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsKilled" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of killed apps
+ 
+ @return the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsKilled"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsKilled" type="long"/>
+      <doc>
+      <![CDATA[Set the number of killed apps
+ 
+ @param numAppsKilled
+          the number of killed apps]]>
+      </doc>
+    </method>
+    <method name="getNumAppsFailed" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of failed apps
+ 
+ @return the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="setNumAppsFailed"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAppsFailed" type="long"/>
+      <doc>
+      <![CDATA[Set the number of failed apps
+ 
+ @param numAppsFailed
+          the number of failed apps]]>
+      </doc>
+    </method>
+    <method name="getNumActiveUsers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of active users
+ 
+ @return the number of active users]]>
+      </doc>
+    </method>
+    <method name="setNumActiveUsers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numActiveUsers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of active users
+ 
+ @param numActiveUsers
+          the number of active users]]>
+      </doc>
+    </method>
+    <method name="getAvailableMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available memory in MB
+ 
+ @return the available memory]]>
+      </doc>
+    </method>
+    <method name="setAvailableMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the available memory in MB
+ 
+ @param availableMemoryMB
+          the available memory]]>
+      </doc>
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated memory in MB
+ 
+ @return the allocated memory]]>
+      </doc>
+    </method>
+    <method name="setAllocatedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated memory in MB
+ 
+ @param allocatedMemoryMB
+          the allocate memory]]>
+      </doc>
+    </method>
+    <method name="getPendingMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending memory in MB
+ 
+ @return the pending memory]]>
+      </doc>
+    </method>
+    <method name="setPendingMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the pending memory in MB
+ 
+ @param pendingMemoryMB
+          the pending memory]]>
+      </doc>
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved memory in MB
+ 
+ @return the reserved memory]]>
+      </doc>
+    </method>
+    <method name="setReservedMemoryMB"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedMemoryMB" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved memory in MB
+ 
+ @param reservedMemoryMB
+          the reserved memory]]>
+      </doc>
+    </method>
+    <method name="getAvailableVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the available vcores
+ 
+ @return the available vcores]]>
+      </doc>
+    </method>
+    <method name="setAvailableVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="availableVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the available vcores
+ 
+ @param availableVCores
+          the available vcores]]>
+      </doc>
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated vcores
+ 
+ @return the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="setAllocatedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the allocated vcores
+ 
+ @param allocatedVCores
+          the allocated vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the pending vcores
+ 
+ @return the pending vcores]]>
+      </doc>
+    </method>
+    <method name="setPendingVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the pending vcores
+ 
+ @param pendingVCores
+          the pending vcores]]>
+      </doc>
+    </method>
+    <method name="getPendingContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of pending containers.
+ @return the number of pending containers.]]>
+      </doc>
+    </method>
+    <method name="setPendingContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pendingContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of pending containers.
+ @param pendingContainers the pending containers.]]>
+      </doc>
+    </method>
+    <method name="getAllocatedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of allocated containers.
+ @return the number of allocated containers.]]>
+      </doc>
+    </method>
+    <method name="setAllocatedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocatedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of allocated containers.
+ @param allocatedContainers the allocated containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedContainers" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of reserved containers.
+ @return the number of reserved containers.]]>
+      </doc>
+    </method>
+    <method name="setReservedContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedContainers" type="long"/>
+      <doc>
+      <![CDATA[Set the number of reserved containers.
+ @param reservedContainers the reserved containers.]]>
+      </doc>
+    </method>
+    <method name="getReservedVCores" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reserved vcores
+ 
+ @return the reserved vcores]]>
+      </doc>
+    </method>
+    <method name="setReservedVCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservedVCores" type="long"/>
+      <doc>
+      <![CDATA[Set the reserved vcores
+ 
+ @param reservedVCores
+          the reserved vcores]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueStatistics -->
+  <!-- start class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <class name="QueueUserACLInfo" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueUserACLInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueueName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>queue name</em> of the queue.
+ @return <em>queue name</em> of the queue]]>
+      </doc>
+    </method>
+    <method name="getUserAcls" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of <code>QueueACL</code> for the given user.
+ @return list of <code>QueueACL</code> for the given user]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>QueueUserACLInfo</code> provides information {@link QueueACL} for
+ the given user.</p>
+ 
+ @see QueueACL
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.QueueUserACLInfo -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <class name="RejectedSchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RejectedSchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Create new RejectedSchedulingRequest.
+ @param reason Rejection Reason.
+ @param request Rejected Scheduling Request.
+ @return RejectedSchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="getReason" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get Rejection Reason.
+ @return Rejection reason.]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="org.apache.hadoop.yarn.api.records.RejectionReason"/>
+      <doc>
+      <![CDATA[Set Rejection Reason.
+ @param reason Rejection Reason.]]>
+      </doc>
+    </method>
+    <method name="getRequest" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Rejected Scheduling Request.
+ @return SchedulingRequest.]]>
+      </doc>
+    </method>
+    <method name="setRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.records.SchedulingRequest"/>
+      <doc>
+      <![CDATA[Set the SchedulingRequest.
+ @param request SchedulingRequest.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This encapsulates a Rejected SchedulingRequest. It contains the offending
+ Scheduling Request along with the reason for rejection.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <class name="RejectionReason" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.RejectionReason[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.RejectionReason"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Reason for rejecting a Scheduling Request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.RejectionReason -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <class name="ReservationACL" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationACL[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationACL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationACL} enumerates the various ACLs for reservations.
+ <p>
+ The ACL is one of:
+ <ul>
+   <li>
+     {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and
+     delete reservations.
+   </li>
+   <li> {@link #LIST_RESERVATIONS} - ACL to list reservations. </li>
+   <li> {@link #SUBMIT_RESERVATIONS} - ACL to create reservations. </li>
+ </ul>
+ Users can always list, update and delete their own reservations.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationACL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <class name="ReservationAllocationState" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationAllocationState"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationAllocationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="acceptanceTime" type="long"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="resourceAllocations" type="java.util.List"/>
+      <param name="reservationId" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+      <param name="reservationDefinition" type="org.apache.hadoop.yarn.api.records.ReservationDefinition"/>
+      <doc>
+      <![CDATA[@param acceptanceTime The acceptance time of the reservation.
+ @param user The username of the user who made the reservation.
+ @param resourceAllocations List of {@link ResourceAllocationRequest}
+                            representing the current state of the
+                            reservation resource allocations. This is
+                            subject to change in the event of re-planning.
+ @param reservationId {@link ReservationId } of the reservation being
+                                            listed.
+ @param reservationDefinition {@link ReservationDefinition} used to make
+                              the reservation.
+ @return {@code ReservationAllocationState} that represents the state of
+ the reservation.]]>
+      </doc>
+    </method>
+    <method name="getAcceptanceTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the acceptance time of the reservation.
+
+ @return the time that the reservation was accepted.]]>
+      </doc>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user who made the reservation.
+
+ @return the name of the user who made the reservation.]]>
+      </doc>
+    </method>
+    <method name="getResourceAllocationRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Resource allocations of the reservation based on the current state
+ of the plan. This is subject to change in the event of re-planning.
+ The allocations will be constraint to the user contract as described by
+ the {@link ReservationDefinition}
+
+ @return a list of resource allocations for the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of the reservation.
+
+ @return the reservation id corresponding to the reservation.]]>
+      </doc>
+    </method>
+    <method name="getReservationDefinition" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reservation definition used to make the reservation.
+
+ @return the reservation definition used to make the reservation.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ReservationAllocationState} represents the reservation that is
+ made by a user.
+ <p>
+ It includes:
+ <ul>
+   <li>Duration of the reservation.</li>
+   <li>Acceptance time of the duration.</li>
+   <li>
+       List of {@link ResourceAllocationRequest}, which includes the time
+       interval, and capability of the allocation.
+       {@code ResourceAllocationRequest} represents an allocation
+       made for a reservation for the current state of the queue. This can be
+       changed for reasons such as re-planning, but will always be subject to
+       the constraints of the user contract as described by
+       {@link ReservationDefinition}
+   </li>
+   <li>{@link ReservationId} of the reservation.</li>
+   <li>{@link ReservationDefinition} used to make the reservation.</li>
+ </ul>
+
+ @see ResourceAllocationRequest
+ @see ReservationId
+ @see ReservationDefinition]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationAllocationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <class name="ReservationDefinition" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationDefinition"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationDefinition"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="arrival" type="long"/>
+      <param name="deadline" type="long"/>
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getArrival" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @return the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="setArrival"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="earliestStartTime" type="long"/>
+      <doc>
+      <![CDATA[Set the arrival time or the earliest time from which the resource(s) can be
+ allocated. Time expressed as UTC.
+ 
+ @param earliestStartTime the earliest valid time for this reservation]]>
+      </doc>
+    </method>
+    <method name="getDeadline" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @return the deadline or the latest time by when the resource(s) must be
+         allocated]]>
+      </doc>
+    </method>
+    <method name="setDeadline"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="latestEndTime" type="long"/>
+      <doc>
+      <![CDATA[Set the deadline or the latest time by when the resource(s) must be
+ allocated. Time expressed as UTC.
+ 
+ @param latestEndTime the deadline or the latest time by when the
+          resource(s) should be allocated]]>
+      </doc>
+    </method>
+    <method name="getReservationRequests" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="setReservationRequests"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationRequests" type="org.apache.hadoop.yarn.api.records.ReservationRequests"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequests} representing the resources
+ required by the application
+ 
+ @param reservationRequests the list of {@link ReservationRequests}]]>
+      </doc>
+    </method>
+    <method name="getReservationName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @return string representing the name of the corresponding reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setReservationName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the name for this reservation. The name need-not be unique, and it is
+ just a mnemonic for the user (akin to job names). Accepted reservations are
+ uniquely identified by a system-generated ReservationId.
+ 
+ @param name representing the name of the corresponding reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="getRecurrenceExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @return recurrence of this reservation]]>
+      </doc>
+    </method>
+    <method name="setRecurrenceExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recurrenceExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the recurrence of this reservation representing the time period of
+ the periodic job. Currently, only long values are supported. Later,
+ support for regular expressions denoting arbitrary recurrence patterns
+ (e.g., every Tuesday and Thursday) will be added.
+ Recurrence is represented in milliseconds for periodic jobs.
+ Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ are explicitly cancelled and have higher priority than non-periodic jobs
+ (during initial placement and replanning). Periodic job allocations are
+ consistent across runs (flexibility in allocation is leveraged only during
+ initial placement, allocations remain consistent thereafter). Note that
+ as a long, the recurrence expression must be greater than the duration of
+ the reservation (deadline - arrival). Also note that the configured max
+ period must be divisible by the recurrence expression if expressed as a
+ long.
+
+ @param recurrenceExpression recurrence interval of this reservation]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @return int representing the priority of the reserved resource
+         allocation in the scheduler]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the priority for this reservation. A lower number for priority
+ indicates a higher priority reservation. Recurring reservations are
+ always higher priority than non-recurring reservations. Priority for
+ non-recurring reservations are only compared with non-recurring
+ reservations. Likewise for recurring reservations.
+
+ @param priority representing the priority of the reserved resource
+          allocation in the scheduler]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationDefinition} captures the set of resource and time
+ constraints the user cares about regarding a reservation.
+ 
+ @see ResourceRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationDefinition -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <class name="ReservationId" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the long identifier of the {@link ReservationId} which is unique for
+ all Reservations started by a particular instance of the
+ {@code ResourceManager}.
+
+ @return long identifier of the {@link ReservationId}]]>
+      </doc>
+    </method>
+    <method name="getClusterTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>start time</em> of the {@code ResourceManager} which is used to
+ generate globally unique {@link ReservationId}.
+
+ @return <em>start time</em> of the {@code ResourceManager}]]>
+      </doc>
+    </method>
+    <method name="build"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="parseReservationId" return="org.apache.hadoop.yarn.api.records.ReservationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Parse the string argument as a {@link ReservationId}
+
+ @param reservationId the string representation of the {@link ReservationId}
+ @return the {@link ReservationId} corresponding to the input string if
+         valid, null if input is null
+ @throws IOException if unable to parse the input string]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <field name="reserveIdStrPrefix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="clusterTimestamp" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="id" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p>
+ {@link ReservationId} represents the <em>globally unique</em> identifier for
+ a reservation.
+ </p>
+
+ <p>
+ The globally unique nature of the identifier is achieved by using the
+ <em>cluster timestamp</em> i.e. start-time of the {@code ResourceManager}
+ along with a monotonically increasing counter for the reservation.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <class name="ReservationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ReservationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="concurrency" type="int"/>
+      <param name="duration" type="long"/>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link Resource} capability of the request.
+ 
+ @return {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the {@link Resource} capability of the request
+ 
+ @param capability {@link Resource} capability of the request]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ 
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ 
+ @param numContainers number of containers required with the given
+          specifications]]>
+      </doc>
+    </method>
+    <method name="getConcurrency" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @return the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="setConcurrency"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers that need to be scheduled concurrently. The
+ default value of 1 would fall back to the current non concurrency
+ constraints on the scheduling behavior.
+ 
+ @param numContainers the number of containers to be concurrently scheduled]]>
+      </doc>
+    </method>
+    <method name="getDuration" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the duration in milliseconds for which the resource is required. A
+ default value of -1, indicates an unspecified lease duration, and fallback
+ to current behavior.
+ 
+ @return the duration in milliseconds for which the resource is required]]>
+      </doc>
+    </method>
+    <method name="setDuration"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="duration" type="long"/>
+      <doc>
+      <![CDATA[Set the duration in milliseconds for which the resource is required.
+ 
+ @param duration the duration in milliseconds for which the resource is
+          required]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequest} represents the request made by an application to
+ the {@code ResourceManager} to reserve {@link Resource}s.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required by the
+     application.
+   </li>
+   <li>Concurrency that indicates the gang size of the request.</li>
+ </ul>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <class name="ReservationRequest.ReservationRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ReservationRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ReservationRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequest.ReservationRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <class name="ReservationRequestInterpreter" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various types of dependencies among multiple
+ {@link ReservationRequests} within one {@link ReservationDefinition} (from
+ least constraining to most constraining).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <class name="ReservationRequests" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ReservationRequests"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ReservationRequests"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <param name="type" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+    </method>
+    <method name="getReservationResources" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @return the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="setReservationResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reservationResources" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list of {@link ReservationRequest} representing the resources
+ required by the application
+ 
+ @param reservationResources the list of {@link ReservationRequest}]]>
+      </doc>
+    </method>
+    <method name="getInterpreter" return="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @return the list of {@link ReservationRequestInterpreter}]]>
+      </doc>
+    </method>
+    <method name="setInterpreter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interpreter" type="org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter"/>
+      <doc>
+      <![CDATA[Set the {@link ReservationRequestInterpreter}, representing how the list of
+ resources should be allocated, this captures temporal ordering and other
+ constraints.
+ 
+ @param interpreter the {@link ReservationRequestInterpreter} for this
+          reservation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ReservationRequests} captures the set of resource and constraints the
+ user cares about regarding a reservation.
+ 
+ @see ReservationRequest]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ReservationRequests -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Resource -->
+  <class name="Resource" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <param name="vCores" type="int"/>
+      <param name="others" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Create a new {@link Resource} instance with the given CPU and memory
+ values and additional resource values as set in the {@code others}
+ parameter. Note that the CPU and memory settings in the {@code others}
+ parameter will be ignored.
+
+ @param memory the memory value
+ @param vCores the CPU value
+ @param others a map of other resource values indexed by resource name
+ @return a {@link Resource} instance with the given resource values]]>
+      </doc>
+    </method>
+    <method name="getMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is DEPRECATED:
+ Use {@link Resource#getMemorySize()} instead
+
+ Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="getMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to return
+ memory but in the units of MB
+
+ @return <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="int"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource. Note - while memory has
+ never had a unit specified, all YARN configurations have specified memory
+ in MB. The assumption has been that the daemons and applications are always
+ using the same units. With the introduction of the ResourceInformation
+ class we have support for units - so this function will continue to set
+ memory but the assumption is that the value passed is in units of MB.
+
+ @param memory <em>memory</em>(in MB) of the resource]]>
+      </doc>
+    </method>
+    <method name="setMemorySize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="memory" type="long"/>
+      <doc>
+      <![CDATA[Set <em>memory</em> of the resource.
+ @param memory <em>memory</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getVirtualCores" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @return <em>num of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="setVirtualCores"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vCores" type="int"/>
+      <doc>
+      <![CDATA[Set <em>number of virtual cpu cores</em> of the resource.
+ 
+ Virtual cores are a unit for expressing CPU parallelism. A node's capacity
+ should be configured with virtual cores equal to its number of physical
+ cores. A container should be requested with the number of cores it can
+ saturate, i.e. the average number of threads it expects to have runnable
+ at a time.
+
+ @param vCores <em>number of virtual cpu cores</em> of the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get ResourceInformation for a specified resource.
+
+ @param resource name of the resource
+ @return the ResourceInformation object for the resource]]>
+      </doc>
+    </method>
+    <method name="getResourceValue" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get the value for a specified resource. No information about the units is
+ returned.
+
+ @param resource name of the resource
+ @return the value for the resource]]>
+      </doc>
+    </method>
+    <method name="setResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="resourceInformation" type="org.apache.hadoop.yarn.api.records.ResourceInformation"/>
+      <doc>
+      <![CDATA[Set the ResourceInformation object for a particular resource.
+
+ @param resource the resource for which the ResourceInformation is provided
+ @param resourceInformation ResourceInformation object]]>
+      </doc>
+    </method>
+    <method name="setResourceValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Set the value of a resource in the ResourceInformation object. The unit of
+ the value is assumed to be the one in the ResourceInformation object.
+
+ @param resource the resource for which the value is provided.
+ @param value    the value to set]]>
+      </doc>
+    </method>
+    <method name="throwExceptionWhenArrayOutOfBound"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="index" type="int"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFormattedString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This method is to get memory in terms of KB|MB|GB.
+ @return string containing all resources]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="castToIntSafely" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Convert long to int for a resource value safely. This method assumes
+ resource value is positive.
+
+ @param value long resource value
+ @return int resource value]]>
+      </doc>
+    </method>
+    <method name="newDefaultInformation" return="org.apache.hadoop.yarn.api.records.ResourceInformation"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="unit" type="java.lang.String"/>
+      <param name="value" type="long"/>
+      <doc>
+      <![CDATA[Create ResourceInformation with basic fields.
+ @param name Resource Type Name
+ @param unit Default unit of provided resource type
+ @param value Value associated with giveb resource
+ @return ResourceInformation object]]>
+      </doc>
+    </method>
+    <field name="resources" type="org.apache.hadoop.yarn.api.records.ResourceInformation[]"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="VCORES_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>Resource</code> models a set of computer resources in the 
+ cluster.</p>
+ 
+ <p>Currently it models both <em>memory</em> and <em>CPU</em>.</p>
+ 
+ <p>The unit for memory is megabytes. CPU is modeled with virtual cores
+ (vcores), a unit for expressing parallelism. A node's capacity should
+ be configured with virtual cores equal to its number of physical cores. A
+ container should be requested with the number of cores it can saturate, i.e.
+ the average number of threads it expects to have runnable at a time.</p>
+ 
+ <p>Virtual cores take integer values and thus currently CPU-scheduling is
+ very coarse.  A complementary axis for CPU requests that represents
+ processing power will likely be added in the future to enable finer-grained
+ resource configuration.</p>
+
+ <p>Typically, applications request <code>Resource</code> of suitable
+ capability to run their component tasks.</p>
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Resource -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <class name="ResourceAllocationRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceAllocationRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceAllocationRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+      <param name="endTime" type="long"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[@param startTime The start time that the capability is reserved for.
+ @param endTime The end time that the capability is reserved for.
+ @param capability {@link Resource} representing the capability of the
+                                   resource allocation.
+ @return {ResourceAllocationRequest} which represents the capability of
+ the resource allocation for a time interval.]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time that the resource is allocated.
+
+ @return the start time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getEndTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the end time that the resource is allocated.
+
+ @return the end time that the resource is allocated.]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the allocated resource.
+
+ @return the allocated resource.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceAllocationRequest} represents an allocation
+ made for a reservation for the current state of the plan. This can be
+ changed for reasons such as re-planning, but will always be subject to the
+ constraints of the user contract as described by
+ {@link ReservationDefinition}
+ {@link Resource}
+
+ <p>
+ It includes:
+ <ul>
+   <li>StartTime of the allocation.</li>
+   <li>EndTime of the allocation.</li>
+   <li>{@link Resource} reserved for the allocation.</li>
+ </ul>
+
+ @see Resource]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceAllocationRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <class name="ResourceBlacklistRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceBlacklistRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="additions" type="java.util.List"/>
+      <param name="removals" type="java.util.List"/>
+    </method>
+    <method name="getBlacklistAdditions" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be added to the 
+ application blacklist.
+ 
+ @return list of resource-names which should be added to the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistAdditions"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be added to the application blacklist.
+ 
+ @param resourceNames list of resource-names which should be added to the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <method name="getBlacklistRemovals" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @return list of resource-names which should be removed from the 
+         application blacklist]]>
+      </doc>
+    </method>
+    <method name="setBlacklistRemovals"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceNames" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set list of resource-names which should be removed from the 
+ application blacklist.
+ 
+ @param resourceNames list of resource-names which should be removed from the 
+                  application blacklist]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@link ResourceBlacklistRequest} encapsulates the list of resource-names 
+ which should be added or removed from the <em>blacklist</em> of resources 
+ for the application.
+ 
+ @see ResourceRequest
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <class name="ResourceOption" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceOption"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceOption"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resource" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="overCommitTimeout" type="int"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Negative value means no timeout.]]>
+      </doc>
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceOption -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <class name="ResourceRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="hostName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="numContainers" type="int"/>
+      <param name="relaxLocality" type="boolean"/>
+      <param name="labelExpression" type="java.lang.String"/>
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="clone" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="rr" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <doc>
+      <![CDATA[Clone a ResourceRequest object (shallow copy). Please keep it loaded with
+ all (new) fields
+
+ @param rr the object to copy from
+ @return the copied object]]>
+      </doc>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isAnyLocation" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Check whether the given <em>host/rack</em> string represents an arbitrary
+ host name.
+
+ @param hostName <em>host/rack</em> on which the allocation is desired
+ @return whether the given <em>host/rack</em> string represents an arbitrary
+ host name]]>
+      </doc>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Priority</code> of the request.
+ @return <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>Priority</code> of the request
+ @param priority <code>Priority</code> of the request]]>
+      </doc>
+    </method>
+    <method name="getResourceName" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resource (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource 
+ (host/rack) is acceptable.
+ 
+ @return resource (e.g. <em>host/rack</em>) on which the allocation 
+                  is desired]]>
+      </doc>
+    </method>
+    <method name="setResourceName"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the resource name (e.g. <em>host/rack</em>) on which the allocation 
+ is desired.
+ 
+ A special value of <em>*</em> signifies that <em>any</em> resource name
+ (e.g. host/rack) is acceptable. 
+ 
+ @param resourceName (e.g. <em>host/rack</em>) on which the 
+                     allocation is desired]]>
+      </doc>
+    </method>
+    <method name="getNumContainers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of containers required with the given specifications.
+ @return number of containers required with the given specifications]]>
+      </doc>
+    </method>
+    <method name="setNumContainers"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the number of containers required with the given specifications
+ @param numContainers number of containers required with the given 
+                      specifications]]>
+      </doc>
+    </method>
+    <method name="getRelaxLocality" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+ 
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="execSpec" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>ExecutionTypeRequest</code> of the requested container.
+
+ @param execSpec
+          ExecutionTypeRequest of the requested container]]>
+      </doc>
+    </method>
+    <method name="getExecutionTypeRequest" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>. Defaults to true.
+
+ @return whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="setRelaxLocality"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[<p>For a request at a network hierarchy level, set whether locality can be relaxed
+ to that level and beyond.<p>
+ 
+ <p>If the flag is off on a rack-level <code>ResourceRequest</code>,
+ containers at that request's priority will not be assigned to nodes on that
+ request's rack unless requests specifically for those nodes have also been
+ submitted.<p>
+ 
+ <p>If the flag is off on an {@link ResourceRequest#ANY}-level
+ <code>ResourceRequest</code>, containers at that request's priority will
+ only be assigned on racks for which specific requests have also been
+ submitted.<p>
+ 
+ <p>For example, to request a container strictly on a specific node, the
+ corresponding rack-level and any-level requests should have locality
+ relaxation set to false.  Similarly, to request a container strictly on a
+ specific rack, the corresponding any-level request should have locality
+ relaxation set to false.<p>
+ 
+ @param relaxLocality whether locality relaxation is enabled with this
+ <code>ResourceRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get node-label-expression for this Resource Request. If this is set, all
+ containers allocated to satisfy this resource-request will be only on those
+ nodes that satisfy this node-label-expression.
+  
+ Please note that node label expression now can only take effect when the
+ resource request has resourceName = ANY
+ 
+ @return node-label-expression]]>
+      </doc>
+    </method>
+    <method name="setNodeLabelExpression"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodelabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set node label expression of this resource request. Now only support
+ specifying a single node label. In the future we will support more complex
+ node label expression specification like {@code AND(&&), OR(||)}, etc.
+ 
+ Any please note that node label expression now can only take effect when
+ the resource request has resourceName = ANY
+ 
+ @param nodelabelExpression
+          node-label-expression of this ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+
+ @return the <em>ID</em> corresponding to this allocation request.]]>
+      </doc>
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestID" type="long"/>
+      <doc>
+      <![CDATA[Set the optional <em>ID</em> corresponding to this allocation request. This
+ ID is an identifier for different {@code ResourceRequest}s from the <b>same
+ application</b>. The allocated {@code Container}(s) received as part of the
+ {@code AllocateResponse} response will have the ID corresponding to the
+ original {@code ResourceRequest} for which the RM made the allocation.
+ <p>
+ The scheduler may return multiple {@code AllocateResponse}s corresponding
+ to the same ID as and when scheduler allocates {@code Container}(s).
+ <b>Applications</b> can continue to completely ignore the returned ID in
+ the response and use the allocation for any of their outstanding requests.
+ <p>
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to
+ a specific ID, they can simply cancel the corresponding {@code
+ ResourceRequest} and submit a new one afresh.
+ <p>
+ If the ID is not set, scheduler will continue to work as previously and all
+ allocated {@code Container}(s) will have the default ID, -1.
+
+ @param allocationRequestID the <em>ID</em> corresponding to this allocation
+                            request.]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+    <field name="ANY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The constant string representing no locality.
+ It should be used by all references that want to pass an arbitrary host
+ name in.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[{@code ResourceRequest} represents the request made
+ by an application to the {@code ResourceManager}
+ to obtain various {@code Container} allocations.
+ <p>
+ It includes:
+ <ul>
+   <li>{@link Priority} of the request.</li>
+   <li>
+     The <em>name</em> of the host or rack on which the allocation is
+     desired. A special value of <em>*</em> signifies that
+     <em>any</em> host/rack is acceptable to the application.
+   </li>
+   <li>{@link Resource} required for each request.</li>
+   <li>
+     Number of containers, of above specifications, which are required
+     by the application.
+   </li>
+   <li>
+     A boolean <em>relaxLocality</em> flag, defaulting to {@code true},
+     which tells the {@code ResourceManager} if the application wants
+     locality to be loose (i.e. allows fall-through to rack or <em>any</em>)
+     or strict (i.e. specify hard constraint on resource allocation).
+   </li>
+ </ul>
+ 
+ @see Resource
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <class name="ResourceRequest.ResourceRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+ @see ResourceRequest#setPriority(Priority)
+ @param priority <code>priority</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceName" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>resourceName</code> of the request.
+ @see ResourceRequest#setResourceName(String)
+ @param resourceName <code>resourceName</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="capability" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>capability</code> of the request.
+ @see ResourceRequest#setCapability(Resource)
+ @param capability <code>capability</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="numContainers" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numContainers" type="int"/>
+      <doc>
+      <![CDATA[Set the <code>numContainers</code> of the request.
+ @see ResourceRequest#setNumContainers(int)
+ @param numContainers <code>numContainers</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="relaxLocality" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relaxLocality" type="boolean"/>
+      <doc>
+      <![CDATA[Set the <code>relaxLocality</code> of the request.
+ @see ResourceRequest#setRelaxLocality(boolean)
+ @param relaxLocality <code>relaxLocality</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="nodeLabelExpression" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeLabelExpression" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the <code>nodeLabelExpression</code> of the request.
+ @see ResourceRequest#setNodeLabelExpression(String)
+ @param nodeLabelExpression
+          <code>nodeLabelExpression</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionTypeRequest" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionTypeRequest" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionTypeRequest
+          <code>executionTypeRequest</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the <code>executionTypeRequest</code> of the request with 'ensure
+ execution type' flag set to true.
+ @see ResourceRequest#setExecutionTypeRequest(
+ ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request.
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+ @see ResourceRequest#setAllocationRequestId(long)
+ @param allocationRequestId
+          <code>allocationRequestId</code> of the request
+ @return {@link ResourceRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.ResourceRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link ResourceRequest} object.
+ @return {@link ResourceRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link ResourceRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <class name="ResourceRequest.ResourceRequestComparator" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Comparator"/>
+    <implements name="java.io.Serializable"/>
+    <constructor name="ResourceRequestComparator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="compare" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="r1" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+      <param name="r2" type="org.apache.hadoop.yarn.api.records.ResourceRequest"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceRequest.ResourceRequestComparator -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <class name="ResourceSizing" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceSizing"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getNumAllocations" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setNumAllocations"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="numAllocations" type="int"/>
+    </method>
+    <method name="getResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResources"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resources" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code ResourceSizing} contains information for the size of a
+ {@link SchedulingRequest}, such as the number of requested allocations and
+ the resources for each allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceSizing -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <class name="ResourceUtilization" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="ResourceUtilization"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="customResources" type="java.util.Map"/>
+    </method>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.ResourceUtilization"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceUtil" type="org.apache.hadoop.yarn.api.records.ResourceUtilization"/>
+    </method>
+    <method name="getVirtualMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get used <em>virtual memory</em>.
+
+ @return <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setVirtualMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vmem" type="int"/>
+      <doc>
+      <![CDATA[Set used <em>virtual memory</em>.
+
+ @param vmem <em>virtual memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getPhysicalMemory" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>physical memory</em>.
+
+ @return <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="setPhysicalMemory"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <doc>
+      <![CDATA[Set <em>physical memory</em>.
+
+ @param pmem <em>physical memory</em> in MB]]>
+      </doc>
+    </method>
+    <method name="getCPU" return="float"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get <em>CPU</em> utilization (The amount of vcores used).
+
+ @return <em>CPU utilization</em>]]>
+      </doc>
+    </method>
+    <method name="setCPU"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Set <em>CPU</em> utilization (The amount of vcores used).
+
+ @param cpu <em>CPU utilization</em>]]>
+      </doc>
+    </method>
+    <method name="getCustomResource" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Get <em>custom resource</em> utilization
+ (The amount of custom resource used).
+
+ @param resourceName <em>resourceName of custom resource</em>
+ @return <em>resourceName utilization</em>]]>
+      </doc>
+    </method>
+    <method name="getCustomResources" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCustomResources"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="customResources" type="java.util.Map"/>
+    </method>
+    <method name="setCustomResource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Set <em>custom resource</em> utilization
+ (The amount of custom resource used).
+ @param resourceName <em>resourceName</em>
+ @param utilization <em>utilization of custom resource</em>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.]]>
+      </doc>
+    </method>
+    <method name="addTo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Add utilization to the current one.
+ @param pmem Physical memory used to add.
+ @param vmem Virtual memory used to add.
+ @param cpu CPU utilization to add.
+ @param resourceName of custom resource to add.
+ @param utilization of custom resource to add.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.]]>
+      </doc>
+    </method>
+    <method name="subtractFrom"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pmem" type="int"/>
+      <param name="vmem" type="int"/>
+      <param name="cpu" type="float"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="utilization" type="float"/>
+      <doc>
+      <![CDATA[Subtract utilization from the current one.
+ @param pmem Physical memory to be subtracted.
+ @param vmem Virtual memory to be subtracted.
+ @param cpu CPU utilization to be subtracted.
+ @param resourceName of custom resource to be subtracted.
+ @param utilization of custom resource to be subtracted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ <code>ResourceUtilization</code> models the utilization of a set of computer
+ resources in the cluster.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ResourceUtilization -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <class name="SchedulingRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulingRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <param name="allocationTags" type="java.util.Set"/>
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <method name="newBuilder" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationRequestId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPriority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+    </method>
+    <method name="getAllocationTags" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAllocationTags"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+    </method>
+    <method name="getResourceSizing" return="org.apache.hadoop.yarn.api.records.ResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResourceSizing"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+    </method>
+    <method name="getPlacementConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setPlacementConstraint"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+    </method>
+    <doc>
+    <![CDATA[{@code SchedulingRequest} represents a request made by an application to the
+ {@code ResourceManager} to obtain an allocation. It is similar to the
+ {@link ResourceRequest}. However, it is more complete than the latter, as it
+ allows applications to specify allocation tags (e.g., to express that an
+ allocation belongs to {@code Spark} or is an {@code HBase-master}), as well
+ as involved {@link PlacementConstraint}s (e.g., anti-affinity between Spark
+ and HBase allocations).
+
+ The size specification of the allocation is in {@code ResourceSizing}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <class name="SchedulingRequest.SchedulingRequestBuilder" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="allocationRequestId" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Set the <code>allocationRequestId</code> of the request.
+
+ @see SchedulingRequest#setAllocationRequestId(long)
+ @param allocationRequestId <code>allocationRequestId</code> of the
+          request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="priority" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <doc>
+      <![CDATA[Set the <code>priority</code> of the request.
+
+ @param priority <code>priority</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ @see SchedulingRequest#setPriority(Priority)]]>
+      </doc>
+    </method>
+    <method name="executionType" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionTypeRequest"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
+ @param executionType <code>executionType</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="allocationTags" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationTags" type="java.util.Set"/>
+      <doc>
+      <![CDATA[Set the <code>allocationTags</code> of the request.
+
+ @see SchedulingRequest#setAllocationTags(Set)
+ @param allocationTags <code>allocationsTags</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="resourceSizing" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="resourceSizing" type="org.apache.hadoop.yarn.api.records.ResourceSizing"/>
+      <doc>
+      <![CDATA[Set the <code>executionType</code> of the request.
+
+ @see SchedulingRequest#setResourceSizing(ResourceSizing)
+ @param resourceSizing <code>resourceSizing</code> of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="placementConstraintExpression" return="org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="placementConstraintExpression" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint"/>
+      <doc>
+      <![CDATA[Set the <code>placementConstraintExpression</code> of the request.
+
+ @see SchedulingRequest#setPlacementConstraint(
+      PlacementConstraint)
+ @param placementConstraintExpression <code>placementConstraints</code> of
+          the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.records.SchedulingRequest"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return generated {@link SchedulingRequest} object.
+
+ @return {@link SchedulingRequest}]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Class to construct instances of {@link SchedulingRequest} with specific
+ options.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SchedulingRequest.SchedulingRequestBuilder -->
+  <!-- start class org.apache.hadoop.yarn.api.records.ShellContainerCommand -->
+  <class name="ShellContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.ShellContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.ShellContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.ShellContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <class name="SignalContainerCommand" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.SignalContainerCommand"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various signal container commands.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.SignalContainerCommand -->
+  <!-- start class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <class name="StrictPreemptionContract" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StrictPreemptionContract"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainers" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the set of {@link PreemptionContainer} specifying containers owned by
+ the <code>ApplicationMaster</code> that may be reclaimed by the
+ <code>ResourceManager</code>.
+ @return the set of {@link ContainerId} to be preempted.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of particular allocations to be reclaimed. The platform will
+ reclaim exactly these resources, so the <code>ApplicationMaster</code> (AM)
+ may attempt to checkpoint work or adjust its execution plan to accommodate
+ it. In contrast to {@link PreemptionContract}, the AM has no flexibility in
+ selecting which resources to return to the cluster.
+ @see PreemptionMessage]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.StrictPreemptionContract -->
+  <!-- start class org.apache.hadoop.yarn.api.records.Token -->
+  <class name="Token" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="Token"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIdentifier" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token identifier.
+ @return token identifier]]>
+      </doc>
+    </method>
+    <method name="getPassword" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token password
+ @return token password]]>
+      </doc>
+    </method>
+    <method name="getKind" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the token kind.
+ @return token kind]]>
+      </doc>
+    </method>
+    <method name="getService" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the service to which the token is allocated.
+ @return service to which the token is allocated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>Token</code> is the security entity used by the framework
+ to verify authenticity of any resource.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.Token -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <class name="UpdateContainerError" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerError"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+    </method>
+    <method name="getReason" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get reason why the update request was not satisfiable.
+ @return Reason]]>
+      </doc>
+    </method>
+    <method name="setReason"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="reason" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set reason why the update request was not satisfiable.
+ @param reason Reason]]>
+      </doc>
+    </method>
+    <method name="getCurrentContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current container version.
+ @return Current container Version.]]>
+      </doc>
+    </method>
+    <method name="setCurrentContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="currentVersion" type="int"/>
+      <doc>
+      <![CDATA[Set current container version.
+ @param currentVersion Current container version.]]>
+      </doc>
+    </method>
+    <method name="getUpdateContainerRequest" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the {@code UpdateContainerRequest} that was not satisfiable.
+ @return UpdateContainerRequest]]>
+      </doc>
+    </method>
+    <method name="setUpdateContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Set the {@code UpdateContainerRequest} that was not satisfiable.
+ @param updateContainerRequest Update Container Request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerError} is used by the Scheduler to notify the
+ ApplicationMaster of an UpdateContainerRequest it cannot satisfy due to
+ an error in the request. It includes the update request as well as
+ a reason for why the request was not satisfiable.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <class name="UpdateContainerRequest" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdateContainerRequest"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="int"/>
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="targetCapability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="targetExecutionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+    </method>
+    <method name="getContainerVersion" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerVersion"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerVersion" type="int"/>
+      <doc>
+      <![CDATA[Set the current version of the container.
+ @param containerVersion of the container]]>
+      </doc>
+    </method>
+    <method name="getContainerUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code> of the container.
+ @return <code>ContainerUpdateType</code> of the container.]]>
+      </doc>
+    </method>
+    <method name="setContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code> of the container.
+ @param updateType of the Container]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerId</code> of the container.
+ @return <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setContainerId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerId</code> of the container.
+ @param containerId <code>ContainerId</code> of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the target <code>ExecutionType</code> of the container.
+ @return <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setExecutionType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <doc>
+      <![CDATA[Set the target <code>ExecutionType</code> of the container.
+ @param executionType <code>ExecutionType</code> of the container]]>
+      </doc>
+    </method>
+    <method name="setCapability"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Set the <code>Resource</code> capability of the request.
+ @param capability <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="getCapability" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Resource</code> capability of the request.
+ @return <code>Resource</code> capability of the request]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[{@code UpdateContainerRequest} represents the request made by an
+ application to the {@code ResourceManager} to update an attribute of a
+ {@code Container} such as its Resource allocation or (@code ExecutionType}
+ <p>
+ It includes:
+ <ul>
+   <li>version for the container.</li>
+   <li>{@link ContainerId} for the container.</li>
+   <li>
+     {@link Resource} capability of the container after the update request
+     is completed.
+   </li>
+   <li>
+     {@link ExecutionType} of the container after the update request is
+     completed.
+   </li>
+ </ul>
+
+ Update rules:
+ <ul>
+   <li>
+     Currently only ONE aspect of the container can be updated per request
+     (user can either update Capability OR ExecutionType in one request..
+     not both).
+   </li>
+   <li>
+     There must be only 1 update request per container in an allocate call.
+   </li>
+   <li>
+     If a new update request is sent for a container (in a subsequent allocate
+     call) before the first one is satisfied by the Scheduler, it will
+     overwrite the previous request.
+   </li>
+ </ul>
+ @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdateContainerRequest -->
+  <!-- start class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <class name="UpdatedContainer" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UpdatedContainer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.UpdatedContainer"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Static Factory method.
+
+ @param updateType ContainerUpdateType
+ @param container Container
+ @return UpdatedContainer]]>
+      </doc>
+    </method>
+    <method name="getUpdateType" return="org.apache.hadoop.yarn.api.records.ContainerUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ContainerUpdateType</code>.
+ @return ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="setUpdateType"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="updateType" type="org.apache.hadoop.yarn.api.records.ContainerUpdateType"/>
+      <doc>
+      <![CDATA[Set the <code>ContainerUpdateType</code>.
+ @param updateType ContainerUpdateType]]>
+      </doc>
+    </method>
+    <method name="getContainer" return="org.apache.hadoop.yarn.api.records.Container"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>Container</code>.
+ @return Container]]>
+      </doc>
+    </method>
+    <method name="setContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[Set the <code>Container</code>.
+ @param container Container]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[An object that encapsulates an updated container and the
+ type of Update.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.UpdatedContainer -->
+  <!-- start class org.apache.hadoop.yarn.api.records.URL -->
+  <class name="URL" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="URL"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <param name="host" type="java.lang.String"/>
+      <param name="port" type="int"/>
+      <param name="file" type="java.lang.String"/>
+    </method>
+    <method name="getScheme" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the scheme of the URL.
+ @return scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="setScheme"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scheme" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the scheme of the URL
+ @param scheme scheme of the URL]]>
+      </doc>
+    </method>
+    <method name="getUserInfo" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user info of the URL.
+ @return user info of the URL]]>
+      </doc>
+    </method>
+    <method name="setUserInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="userInfo" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the user info of the URL.
+ @param userInfo user info of the URL]]>
+      </doc>
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the host of the URL.
+ @return host of the URL]]>
+      </doc>
+    </method>
+    <method name="setHost"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="host" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the host of the URL.
+ @param host host of the URL]]>
+      </doc>
+    </method>
+    <method name="getPort" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the port of the URL.
+ @return port of the URL]]>
+      </doc>
+    </method>
+    <method name="setPort"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="port" type="int"/>
+      <doc>
+      <![CDATA[Set the port of the URL
+ @param port port of the URL]]>
+      </doc>
+    </method>
+    <method name="getFile" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the file of the URL.
+ @return file of the URL]]>
+      </doc>
+    </method>
+    <method name="setFile"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="file" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the file of the URL.
+ @param file file of the URL]]>
+      </doc>
+    </method>
+    <method name="toPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+    </method>
+    <method name="fromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="fromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <doc>
+    <![CDATA[<p><code>URL</code> represents a serializable {@link java.net.URL}.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.URL -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <class name="YarnApplicationAttemptState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of a <code>RMAppAttempt</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <class name="YarnApplicationState" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.YarnApplicationState[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[Enumeration of various states of an <code>ApplicationMaster</code>.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnApplicationState -->
+  <!-- start class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+  <class name="YarnClusterMetrics" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClusterMetrics"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNumNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>NodeManager</code>s in the cluster.
+ @return number of <code>NodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumDecommissionedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>DecommissionedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>DecommissionedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumActiveNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>ActiveNodeManager</code>s in the cluster.
+ 
+ @return number of <code>ActiveNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumLostNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>LostNodeManager</code>s in the cluster.
+ 
+ @return number of <code>LostNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumUnhealthyNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>UnhealthyNodeManager</code>s in the cluster.
+ 
+ @return number of <code>UnhealthyNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <method name="getNumRebootedNodeManagers" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the number of <code>RebootedNodeManager</code>s in the cluster.
+ 
+ @return number of <code>RebootedNodeManager</code>s in the cluster]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p><code>YarnClusterMetrics</code> represents cluster metrics.</p>
+ 
+ <p>Currently only number of <code>NodeManager</code>s is provided.</p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.YarnClusterMetrics -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.impl">
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timeline">
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <class name="TimelineAbout" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineAbout"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineAbout" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAbout" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setAbout"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineServiceVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimelineServiceVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineServiceVersionBuiltOn" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopBuildVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopBuildVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopBuildVersion" type="java.lang.String"/>
+    </method>
+    <method name="getHadoopVersionBuiltOn" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHadoopVersionBuiltOn"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hadoopVersionBuiltOn" type="java.lang.String"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineAbout -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <class name="TimelineDelegationTokenResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getContent" return="java.lang.Object"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="content" type="java.lang.Object"/>
+    </method>
+    <doc>
+    <![CDATA[The response of delegation token related request]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID
+ 
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description
+ 
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description
+ 
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner
+ 
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ 
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string
+ 
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string
+ 
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string
+ 
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string
+ 
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain
+ 
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain
+ 
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain
+ 
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain
+ 
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline domain, which is used
+ to a user to host a number of timeline entities, isolating them from others'.
+ The user can also define the reader and writer users/groups for the the
+ domain, which is used to control the access to its entities.
+ </p>
+ 
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <class name="TimelineDomains" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomains"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getDomains" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of domains
+ 
+ @return a list of domains]]>
+      </doc>
+    </method>
+    <method name="addDomain"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <doc>
+      <![CDATA[Add a single domain into the existing domain list
+ 
+ @param domain
+          a single domain]]>
+      </doc>
+    </method>
+    <method name="addDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of domains into the existing domain list
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <method name="setDomains"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domains" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the domain list to the given list of domains
+ 
+ @param domains
+          a list of domains]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline domains.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineDomains -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of entities
+ 
+ @return a list of entities]]>
+      </doc>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+      <doc>
+      <![CDATA[Add a single entity into the existing entity list
+ 
+ @param entity
+          a single entity]]>
+      </doc>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[All a list of entities into the existing entity list
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the entity list to the given list of entities
+ 
+ @param entities
+          a list of entities]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getStartTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the start time of the entity
+ 
+ @return the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the start time of the entity
+ 
+ @param startTime
+          the start time of the entity]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events related to the entity
+ 
+ @return a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event related to the entity to the existing event list
+ 
+ @param event
+          a single event related to the entity]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of events related to the entity to the existing event list
+ 
+ @param events
+          a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events related to the entity
+ 
+ @param events
+          events a list of events related to the entity]]>
+      </doc>
+    </method>
+    <method name="getRelatedEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the related entities
+ 
+ @return the related entities]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Add an entity to the existing related entity map
+ 
+ @param entityType
+          the entity type
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="addRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of related entities to the existing related entity map
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="setRelatedEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatedEntities" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the related entity map to the given map of related entities
+ 
+ @param relatedEntities
+          a map of related entities]]>
+      </doc>
+    </method>
+    <method name="getPrimaryFilters" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the primary filters
+ 
+ @return the primary filters]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilter"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add a single piece of primary filter to the existing primary filter map
+ 
+ @param key
+          the primary filter key
+ @param value
+          the primary filter value]]>
+      </doc>
+    </method>
+    <method name="addPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of primary filters to the existing primary filter map
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="setPrimaryFilters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="primaryFilters" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the primary filter map to the given map of primary filters
+ 
+ @param primaryFilters
+          a map of primary filters]]>
+      </doc>
+    </method>
+    <method name="getOtherInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the other information of the entity
+ 
+ @return the other information of the entity]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of other information of the entity to the existing other info
+ map
+ 
+ @param key
+          the other information key
+ @param value
+          the other information value]]>
+      </doc>
+    </method>
+    <method name="addOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of other information of the entity to the existing other info map
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="setOtherInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="otherInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the other info map to the given map of other information
+ 
+ @param otherInfo
+          a map of other information]]>
+      </doc>
+    </method>
+    <method name="getDomainId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ID of the domain that the entity is to be put
+ 
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setDomainId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domainId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the ID of the domain that the entity is to be put
+ 
+ @param domainId
+          the name space ID]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The class that contains the the meta information of some conceptual entity
+ and its related events. The entity can be an application, an application
+ attempt, a container or whatever the user-defined object.
+ </p>
+ 
+ <p>
+ Primary filters will be used to index the entities in
+ <code>TimelineStore</code>, such that users should carefully choose the
+ information they want to store as the primary filters. The remaining can be
+ stored as other information.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <class name="TimelineEntityGroupId" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntityGroupId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="newInstance" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>ApplicationId</code> of the
+ <code>TimelineEntityGroupId</code>.
+
+ @return <code>ApplicationId</code> of the
+         <code>TimelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="getTimelineEntityGroupId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <code>timelineEntityGroupId</code>.
+
+ @return <code>timelineEntityGroupId</code>]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="fromString" return="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntityGroupIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TIMELINE_ENTITY_GROUPID_STR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<p><code>TimelineEntityGroupId</code> is an abstract way for
+ timeline service users to represent #a group of related timeline data.
+ For example, all entities that represents one data flow DAG execution
+ can be grouped into one timeline entity group. </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the timestamp of the event
+ 
+ @return the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <doc>
+      <![CDATA[Set the timestamp of the event
+ 
+ @param timestamp
+          the timestamp of the event]]>
+      </doc>
+    </method>
+    <method name="getEventType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the event type
+ 
+ @return the event type]]>
+      </doc>
+    </method>
+    <method name="setEventType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the event type
+ 
+ @param eventType
+          the event type]]>
+      </doc>
+    </method>
+    <method name="getEventInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set the information of the event
+ 
+ @return the information of the event]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Add one piece of the information of the event to the existing information
+ map
+ 
+ @param key
+          the information key
+ @param value
+          the information value]]>
+      </doc>
+    </method>
+    <method name="addEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Add a map of the information of the event to the existing information map
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="setEventInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventInfo" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Set the information map to the given map of the information of the event
+ 
+ @param eventInfo
+          a map of of the information of the event]]>
+      </doc>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[The class that contains the information of an event that is related to some
+ conceptual entity of an application. Users are free to define what the event
+ means, such as starting an application, getting allocated a container and
+ etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <class name="TimelineEvents" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEvents"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAllEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link EventsOfOneEntity} instances
+ 
+ @return a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventsOfOneEntity" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity"/>
+      <doc>
+      <![CDATA[Add a single {@link EventsOfOneEntity} instance into the existing list
+ 
+ @param eventsOfOneEntity
+          a single {@link EventsOfOneEntity} instance]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link EventsOfOneEntity} instances into the existing list
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allEvents" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link EventsOfOneEntity} instances
+ 
+ @param allEvents
+          a list of {@link EventsOfOneEntity} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events, which are categorized according to
+ their related entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <class name="TimelineEvents.EventsOfOneEntity" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="EventsOfOneEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getEvents" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of events
+ 
+ @return a list of events]]>
+      </doc>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEvent"/>
+      <doc>
+      <![CDATA[Add a single event to the existing event list
+ 
+ @param event
+          a single event]]>
+      </doc>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of event to the existing event list
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="events" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the event list to the given list of events
+ 
+ @param events
+          a list of events]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[The class that hosts a list of events that are only related to one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <class name="TimelineHealth" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineHealth" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineHealth"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getHealthStatus" return="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHealthStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="healthStatus" type="org.apache.hadoop.yarn.api.records.timeline.TimelineHealth.TimelineHealthStatus"/>
+    </method>
+    <method name="setDiagnosticsInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="diagnosticsInfo" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[This class holds health information for ATS.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelineHealth -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <class name="TimelinePutResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelinePutError} instances
+ 
+ @return a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelinePutError} instance into the existing list
+ 
+ @param error
+          a single {@link TimelinePutError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelinePutError} instances into the existing list
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelinePutError} instances
+ 
+ @param errors
+          a list of {@link TimelinePutError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are errors
+ in storing individual entity objects, they will be indicated in the list of
+ errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+  <class name="TimelinePutResponse.TimelinePutError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelinePutError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id
+ 
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id
+ 
+ @param entityId
+          the entity Id]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type
+ 
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityType" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type
+ 
+ @param entityType
+          the entity type]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code
+ 
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="errorCode" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code
+ 
+ @param errorCode
+          an error code]]>
+      </doc>
+    </method>
+    <field name="NO_START_TIME" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned when no start time can be found when putting an
+ entity. This occurs when the entity does not already exist in the store
+ and it is put with no start time or events specified.]]>
+      </doc>
+    </field>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when putting an
+ entity.]]>
+      </doc>
+    </field>
+    <field name="SYSTEM_FILTER_CONFLICT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user specifies the timeline system reserved
+ filter key]]>
+      </doc>
+    </field>
+    <field name="ACCESS_DENIED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to access the timeline data]]>
+      </doc>
+    </field>
+    <field name="NO_DOMAIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity doesn't have an valid domain ID]]>
+      </doc>
+    </field>
+    <field name="FORBIDDEN_RELATION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the user is denied to relate the entity to another
+ one in different domain]]>
+      </doc>
+    </field>
+    <field name="EXPIRED_ENTITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if the entity start time is before the eviction
+ period of old data.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError -->
+</package>
+<package name="org.apache.hadoop.yarn.api.records.timelineservice">
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <class name="ApplicationAttemptEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <class name="ApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setQueue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queue" type="java.lang.String"/>
+    </method>
+    <method name="isApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an ApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an ApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="getApplicationEvent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <param name="eventId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[@param te TimelineEntity object.
+ @param eventId event with this id needs to be fetched
+ @return TimelineEvent if TimelineEntity contains the desired event.]]>
+      </doc>
+    </method>
+    <field name="QUEUE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents an application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <class name="ClusterEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClusterEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClusterEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a YARN cluster.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <class name="ContainerEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a container belonging to an application attempt.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <class name="FlowActivityEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowActivityEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="java.lang.String, long, java.lang.String, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowActivityEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+      <doc>
+      <![CDATA[Reuse the base class equals method.]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Reuse the base class hashCode method.]]>
+      </doc>
+    </method>
+    <method name="getCluster" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCluster"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cluster" type="java.lang.String"/>
+    </method>
+    <method name="getDate" return="java.util.Date"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setDate"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="time" type="long"/>
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setFlowName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+    </method>
+    <method name="addFlowRun"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="run" type="org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity"/>
+    </method>
+    <method name="addFlowRuns"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runs" type="java.util.Collection"/>
+    </method>
+    <method name="getFlowRuns" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNumberOfRuns" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="CLUSTER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DATE_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Entity that represents a record for flow activity. It's essentially a
+ container entity for flow runs with limited information.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <class name="FlowRunEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="FlowRunEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="FlowRunEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setUser"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setName"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="getVersion" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setVersion"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="version" type="java.lang.String"/>
+    </method>
+    <method name="getRunId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setRunId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="runId" type="long"/>
+    </method>
+    <method name="getStartTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setStartTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="startTime" type="long"/>
+    </method>
+    <method name="getMaxEndTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMaxEndTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="endTime" type="long"/>
+    </method>
+    <field name="USER_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_NAME_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_END_TIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a flow run.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <class name="HierarchicalTimelineEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getParent" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="setParent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getChildren" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChildren"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="java.util.Set"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="child" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="addChild"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <field name="PARENT_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CHILDREN_INFO_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class extends timeline entity and defines parent-child relationships
+ with other entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <class name="QueueEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="QueueEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="QueueEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a queue.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <class name="SubApplicationEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.HierarchicalTimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SubApplicationEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isSubApplicationEntity" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="te" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+      <doc>
+      <![CDATA[Checks if the input TimelineEntity object is an SubApplicationEntity.
+
+ @param te TimelineEntity object.
+ @return true if input is an SubApplicationEntity, false otherwise]]>
+      </doc>
+    </method>
+    <method name="setApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="java.lang.String"/>
+    </method>
+    <field name="YARN_APPLICATION_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This entity represents a user defined entities to be stored under sub
+ application table.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <class name="TimelineDomain" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDomain"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain ID.
+ @return the domain ID]]>
+      </doc>
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain ID.
+ @param id the domain ID]]>
+      </doc>
+    </method>
+    <method name="getDescription" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain description.
+ @return the domain description]]>
+      </doc>
+    </method>
+    <method name="setDescription"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="description" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain description.
+ @param description the domain description]]>
+      </doc>
+    </method>
+    <method name="getOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the domain owner.
+ @return the domain owner]]>
+      </doc>
+    </method>
+    <method name="setOwner"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="owner" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the domain owner. The user doesn't need to set it, which will
+ automatically set to the user who puts the domain.
+ @param owner the domain owner]]>
+      </doc>
+    </method>
+    <method name="getReaders" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the reader (and/or reader group) list string.
+ @return the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="setReaders"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="readers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the reader (and/or reader group) list string.
+ @param readers the reader (and/or reader group) list string]]>
+      </doc>
+    </method>
+    <method name="getWriters" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the writer (and/or writer group) list string.
+ @return the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="setWriters"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writers" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the writer (and/or writer group) list string.
+ @param writers the writer (and/or writer group) list string]]>
+      </doc>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the created time of the domain.
+ @return the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the created time of the domain.
+ @param createdTime the created time of the domain]]>
+      </doc>
+    </method>
+    <method name="getModifiedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the modified time of the domain.
+ @return the modified time of the domain]]>
+      </doc>
+    </method>
+    <method name="setModifiedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="modifiedTime" type="java.lang.Long"/>
+      <doc>
+      <![CDATA[Set the modified time of the domain.
+ @param modifiedTime the modified time of the domain]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ This class contains the information about a timeline service domain, which is
+ used to a user to host a number of timeline entities, isolating them from
+ others'. The user can also define the reader and writer users/groups for
+ the domain, which is used to control the access to its entities.
+ </p>
+ <p>
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what <code>AccessControlList</code> takes.
+ </p>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <class name="TimelineEntities" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineEntities"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineEntities" type="java.util.List"/>
+    </method>
+    <method name="addEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <doc>
+    <![CDATA[This class hosts a set of timeline entities.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <class name="TimelineEntity" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ The constuctor is used to construct a proxy {@link TimelineEntity} or its
+ subclass object from the real entity object that carries information.
+ </p>
+
+ <p>
+ It is usually used in the case where we want to recover class polymorphism
+ after deserializing the entity from its JSON form.
+ </p>
+ @param entity the real entity that carries information]]>
+      </doc>
+    </constructor>
+    <constructor name="TimelineEntity" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getIdentifier" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdentifier"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdentifier" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityInfos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getConfigs" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfigs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityConfigs" type="java.util.Map"/>
+    </method>
+    <method name="addConfig"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.String"/>
+    </method>
+    <method name="getMetrics" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityMetrics" type="java.util.Set"/>
+    </method>
+    <method name="addMetric"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+    </method>
+    <method name="getEvents" return="java.util.NavigableSet"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.NavigableSet"/>
+    </method>
+    <method name="addEvents"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityEvents" type="java.util.Set"/>
+    </method>
+    <method name="addEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <method name="getIsRelatedToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="isRelatedTo" type="java.util.Map"/>
+    </method>
+    <method name="addIsRelatedToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="getRelatesToEntities" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="addRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="addRelatesToEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <param name="id" type="java.lang.String"/>
+    </method>
+    <method name="setRelatesToEntities"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="relatesTo" type="java.util.Map"/>
+    </method>
+    <method name="getCreatedTime" return="java.lang.Long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCreatedTime"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="createdTs" type="java.lang.Long"/>
+    </method>
+    <method name="setUID"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uidKey" type="java.lang.String"/>
+      <param name="uId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set UID in info which will be then used for query by UI.
+ @param uidKey key for UID in info.
+ @param uId UID to be set for the key.]]>
+      </doc>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"/>
+    </method>
+    <method name="getReal" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getIdPrefix" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setIdPrefix"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entityIdPrefix" type="long"/>
+      <doc>
+      <![CDATA[Sets idPrefix for an entity.
+ <p>
+ <b>Note</b>: Entities will be stored in the order of idPrefix specified.
+ If users decide to set idPrefix for an entity, they <b>MUST</b> provide
+ the same prefix for every update of this entity.
+ </p>
+ Example: <blockquote><pre>
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+ </pre></blockquote>
+ Users can use {@link TimelineServiceHelper#invertLong(long)} to invert
+ the prefix if necessary.
+
+ @param entityIdPrefix prefix for an entity.]]>
+      </doc>
+    </method>
+    <field name="SYSTEM_INFO_KEY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENTITY_PREFIX" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The basic timeline entity data structure for timeline service v2. Timeline
+ entity objects are not thread safe and should not be accessed concurrently.
+ All collection members will be initialized into empty collections. Two
+ timeline entities are equal iff. their type and id are identical.
+
+ All non-primitive type, non-collection members will be initialized into null.
+ User should set the type and id of a timeline entity to make it valid (can be
+ checked by using the {@link #isValid()} method). Callers to the getters
+ should perform null checks for non-primitive type, non-collection members.
+
+ Callers are recommended not to alter the returned collection objects from the
+ getters.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <class name="TimelineEntityType" extends="java.lang.Enum"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="values" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType[]"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="valueOf" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+    </method>
+    <method name="isParent" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a parent of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is parent of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="isChild" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType"/>
+      <doc>
+      <![CDATA[Whether the input type can be a child of this entity.
+
+ @param type entity type.
+ @return true, if this entity type is child of passed entity type, false
+     otherwise.]]>
+      </doc>
+    </method>
+    <method name="matches" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="typeString" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Whether the type of this entity matches the type indicated by the input
+ argument.
+
+ @param typeString entity type represented as a string.
+ @return true, if string representation of this entity type matches the
+     entity type passed.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Defines type of entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <class name="TimelineEvent" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Comparable"/>
+    <constructor name="TimelineEvent"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventId" type="java.lang.String"/>
+    </method>
+    <method name="getInfo" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="infos" type="java.util.Map"/>
+    </method>
+    <method name="addInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="java.lang.String"/>
+      <param name="value" type="java.lang.Object"/>
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTimestamp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ts" type="long"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="compareTo" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent"/>
+    </method>
+    <field name="INVALID_TIMESTAMP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains the information of an event that belongs to an entity.
+ Users are free to define what the event means, such as starting an
+ application, container being allocated, etc.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <class name="TimelineMetric" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineMetric"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getType" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricType" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type"/>
+    </method>
+    <method name="getId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metricId" type="java.lang.String"/>
+    </method>
+    <method name="getRealtimeAggregationOp" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the real time aggregation operation of this metric.
+
+ @return Real time aggregation operation]]>
+      </doc>
+    </method>
+    <method name="setRealtimeAggregationOp"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation"/>
+      <doc>
+      <![CDATA[Set the real time aggregation operation of this metric.
+
+ @param op A timeline metric operation that the metric should perform on
+           real time aggregations]]>
+      </doc>
+    </method>
+    <method name="getValues" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValues"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="vals" type="java.util.Map"/>
+    </method>
+    <method name="addValue"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timestamp" type="long"/>
+      <param name="value" type="java.lang.Number"/>
+    </method>
+    <method name="isValid" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLatestSingleValueMetric" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Get the latest timeline metric as single value type.
+
+ @param metric Incoming timeline metric
+ @return The latest metric in the incoming metric]]>
+      </doc>
+    </method>
+    <method name="getSingleDataTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data timestamp of the metric.
+
+ @return the single data timestamp]]>
+      </doc>
+    </method>
+    <method name="getSingleDataValue" return="java.lang.Number"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get single data value of the metric.
+
+ @return the single data value]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state in a stateless fashion. The assumption here is
+ baseAggregatedMetric and latestMetric should be single value data if not
+ null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <method name="aggregateTo" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="incomingMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="baseAggregatedMetric" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric"/>
+      <param name="state" type="java.util.Map"/>
+      <doc>
+      <![CDATA[Aggregate an incoming metric to the base aggregated metric with the given
+ operation state. The assumption here is baseAggregatedMetric and
+ latestMetric should be single value data if not null.
+
+ @param incomingMetric Incoming timeline metric to aggregate
+ @param baseAggregatedMetric Base timeline metric
+ @param state Operation state
+ @return Result metric after aggregation]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This class contains the information of a metric that is related to some
+ entity. Metric can either be a time series or single value.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <class name="TimelineWriteResponse" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteResponse"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getErrors" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a list of {@link TimelineWriteError} instances.
+
+ @return a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="addError"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="error" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError"/>
+      <doc>
+      <![CDATA[Add a single {@link TimelineWriteError} instance into the existing list.
+
+ @param error
+          a single {@link TimelineWriteError} instance]]>
+      </doc>
+    </method>
+    <method name="addErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Add a list of {@link TimelineWriteError} instances into the existing list.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <method name="setErrors"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="writeErrors" type="java.util.List"/>
+      <doc>
+      <![CDATA[Set the list to the given list of {@link TimelineWriteError} instances.
+
+ @param writeErrors
+          a list of {@link TimelineWriteError} instances]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A class that holds a list of put errors. This is the response returned when a
+ list of {@link TimelineEntity} objects is added to the timeline. If there are
+ errors in storing individual entity objects, they will be indicated in the
+ list of errors.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <class name="TimelineWriteResponse.TimelineWriteError" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineWriteError"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getEntityId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity Id.
+
+ @return the entity Id]]>
+      </doc>
+    </method>
+    <method name="setEntityId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="id" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity Id.
+
+ @param id the entity Id.]]>
+      </doc>
+    </method>
+    <method name="getEntityType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the entity type.
+
+ @return the entity type]]>
+      </doc>
+    </method>
+    <method name="setEntityType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="type" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Set the entity type.
+
+ @param type the entity type.]]>
+      </doc>
+    </method>
+    <method name="getErrorCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the error code.
+
+ @return an error code]]>
+      </doc>
+    </method>
+    <method name="setErrorCode"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="code" type="int"/>
+      <doc>
+      <![CDATA[Set the error code to the given error code.
+
+ @param code an error code.]]>
+      </doc>
+    </method>
+    <field name="IO_EXCEPTION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Error code returned if an IOException is encountered when storing an
+ entity.]]>
+      </doc>
+    </field>
+    <doc>
+    <![CDATA[A class that holds the error code for one entity.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse.TimelineWriteError -->
+  <!-- start class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+  <class name="UserEntity" extends="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UserEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="UserEntity" type="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This entity represents a user.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.records.timelineservice.UserEntity -->
+</package>
+<package name="org.apache.hadoop.yarn.api.resource">
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <class name="PlacementConstraint" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="PlacementConstraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getConstraintExpr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the constraint expression of the placement constraint.
+
+ @return the constraint expression]]>
+      </doc>
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[{@code PlacementConstraint} represents a placement constraint for a resource
+ allocation.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraint -->
+  <!-- start class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+  <class name="PlacementConstraints" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="targetIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ satisfy all target expressions within the given scope (e.g., node or rack).
+
+ For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows
+ allocations on nodes that belong to a rack that has at least one tag with
+ value "hbase-m".
+
+ @param scope the scope within which the target expressions should be
+          satisfied
+ @param targetExpressions the expressions that need to be satisfied within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNotIn" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that does not satisfy any of the
+ target expressions.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetNodeAttribute" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="opCode" type="org.apache.hadoop.yarn.api.records.NodeAttributeOpCode"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that requires allocations to be placed on nodes that
+ belong to a scope (e.g., node or rack) that satisfy any of the
+ target expressions based on node attribute op code.
+
+ @param scope the scope within which the target expressions should not be
+          true
+ @param opCode Node Attribute code which could be equals, not equals.
+ @param targetExpressions the expressions that need to not be true within
+          the scope
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Creates a constraint that restricts the number of allocations within a
+ given scope (e.g., node or rack).
+
+ For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes
+ where there are no less than 3 allocations with tag "zk" and no more than
+ 10.
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="cardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but let you
+ attach a namespace to the given allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of the allocation tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param maxCardinality determines the maximum number of allocations within
+                       the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the minimum cardinality (the maximum cardinality is
+ unbound).
+
+ @param scope the scope of the constraint
+ @param minCardinality determines the minimum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="minCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="namespace" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #minCardinality(String, int, String...)}, but let you
+ attach a namespace to the allocation tags.
+
+ @param scope the scope of the constraint
+ @param namespace the namespace of these tags
+ @param minCardinality determines the minimum number of allocations within
+                       the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #cardinality(String, int, int, String...)}, but
+ determines only the maximum cardinality (the minimum cardinality is 0).
+
+ @param scope the scope of the constraint
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags the constraint targets allocations with these tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="maxCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="tagNamespace" type="java.lang.String"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="allocationTags" type="java.lang.String[]"/>
+      <doc>
+      <![CDATA[Similar to {@link #maxCardinality(String, int, String...)}, but let you
+ specify a namespace for the tags, see supported namespaces in
+ {@link AllocationTagNamespaceType}.
+
+ @param scope the scope of the constraint
+ @param tagNamespace the namespace of these tags
+ @param maxCardinality determines the maximum number of allocations within
+          the scope
+ @param allocationTags allocation tags
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="targetCardinality" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="scope" type="java.lang.String"/>
+      <param name="minCardinality" type="int"/>
+      <param name="maxCardinality" type="int"/>
+      <param name="targetExpressions" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression[]"/>
+      <doc>
+      <![CDATA[This constraint generalizes the cardinality and target constraints.
+
+ Consider a set of nodes N that belongs to the scope specified in the
+ constraint. If the target expressions are satisfied at least minCardinality
+ times and at most maxCardinality times in the node set N, then the
+ constraint is satisfied.
+
+ For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
+ requires an allocation to be placed within a rack that has at least 2 and
+ at most 10 other allocations with tag "zk".
+
+ @param scope the scope of the constraint
+ @param minCardinality the minimum number of times the target expressions
+          have to be satisfied with the given scope
+ @param maxCardinality the maximum number of times the target expressions
+          have to be satisfied with the given scope
+ @param targetExpressions the target expressions
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="and" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.And"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A conjunction of constraints.
+
+ @param children the children constraints that should all be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="or" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint[]"/>
+      <doc>
+      <![CDATA[A disjunction of constraints.
+
+ @param children the children constraints, one of which should be satisfied
+ @return the resulting placement constraint]]>
+      </doc>
+    </method>
+    <method name="delayedOr" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="children" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint[]"/>
+      <doc>
+      <![CDATA[Creates a composite constraint that includes a list of timed placement
+ constraints. The scheduler should try to satisfy first the first timed
+ child constraint within the specified time window. If this is not possible,
+ it should attempt to satisfy the second, and so on.
+
+ @param children the timed children constraints
+ @return the resulting composite constraint]]>
+      </doc>
+    </method>
+    <method name="timedClockConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <param name="timeUnit" type="java.util.concurrent.TimeUnit"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a time
+ window.
+
+ @param constraint the placement constraint
+ @param delay the length of the time window within which the constraint has
+          to be satisfied
+ @param timeUnit the unit of time of the time window
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="timedOpportunitiesConstraint" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraint" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <param name="delay" type="long"/>
+      <doc>
+      <![CDATA[Creates a placement constraint that has to be satisfied within a number of
+ placement opportunities (invocations of the scheduler).
+
+ @param constraint the placement constraint
+ @param delay the number of scheduling opportunities within which the
+          constraint has to be satisfied
+ @return the resulting timed placement constraint]]>
+      </doc>
+    </method>
+    <method name="build" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintExpr" type="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"/>
+      <doc>
+      <![CDATA[Creates a {@link PlacementConstraint} given a constraint expression.
+
+ @param constraintExpr the constraint expression
+ @return the placement constraint]]>
+      </doc>
+    </method>
+    <field name="NODE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RACK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains various static methods for the applications to create
+ placement constraints (see also {@link PlacementConstraint}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.api.resource.PlacementConstraints -->
+</package>
+<package name="org.apache.hadoop.yarn.conf">
+  <!-- start class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+  <class name="YarnConfiguration" extends="org.apache.hadoop.conf.Configuration"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnConfiguration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnConfiguration" type="org.apache.hadoop.conf.Configuration"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="isAclEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getSocketAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="defaultAddress" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+      <doc>
+      <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>. On an HA cluster,
+ this fetches the address corresponding to the RM identified by
+ {@link #RM_HA_ID}.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+      </doc>
+    </method>
+    <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetSocketAddress"/>
+    </method>
+    <method name="useHttps" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="shouldRMFailFast" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isDistSchedulingEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="isOpportunisticContainerAllocationEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV2Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.2 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.2 is enabled. V.2 refers to a
+ version greater than equal to 2 but smaller than 3.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1 is enabled. V.1 refers to a
+ version greater than equal to 1 but smaller than 2.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV15Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1,5 is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="systemMetricsPublisherEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the system publisher is enabled.
+
+ @param conf the configuration
+ @return whether the system publisher is enabled.]]>
+      </doc>
+    </method>
+    <method name="numaAwarenessEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the NUMA awareness is enabled.
+
+ @param conf the configuration
+ @return whether the NUMA awareness is enabled.]]>
+      </doc>
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <field name="DR_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CS_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_POLICY_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CORE_SITE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODE_RESOURCES_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONFIGURATION_FILES" type="java.util.List"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCE_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEBUG_NM_DELETE_DELAY_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay before deleting resource to ease debugging of NM issues]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_CONTAINER_DEBUG_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create client IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_CLIENT_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create server IPC classes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_SERVER_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Factory to create serializable records.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RECORD_FACTORY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RPC class implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IPC_RPC_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CLUSTER_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HOSTNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_EPOCH" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_EPOCH_RANGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The epoch range before wrap around. 0 disables wrap around]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_EPOCH_RANGE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the applications manager interface in the RM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of application tags.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAGS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MAX_TAG_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of each application tag.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MAX_TAG_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MASTER_SERVICE_PROCESSORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AUTO_UPDATE_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AUTO_UPDATE_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the RM.]]>
+      </doc>
+    </field>
+    <field name="RM_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle applications manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AMLAUNCHER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to launch/cleanup AM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AMLAUNCHER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_CONNECT_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retry times to connect with NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_CONNECT_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum request grant-able by the RM scheduler.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle scheduler interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[If the port should be included or not in the node name. The node name
+ is used by the scheduler for resource requests allocation location 
+ matching. Typically this is just the hostname, using the port is needed
+ when using minicluster and specific NM are required.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="QUEUE_PLACEMENT_RULES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Configured scheduler queue placement rules.]]>
+      </doc>
+    </field>
+    <field name="USER_GROUP_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[UserGroupMappingPlacementRule configuration string.]]>
+      </doc>
+    </field>
+    <field name="APP_NAME_PLACEMENT_RULE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_UI_ACTIONS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Resource Manager webapp ui actions]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the RM should enable Reservation System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The PlanFollower for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The step size of the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum periodicity for the Reservation System.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_ENABLE_MONITORS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable periodic monitor threads.
+ @see #RM_SCHEDULER_MONITOR_POLICIES]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER_ENABLE_MONITORS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MONITOR_POLICIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of SchedulingEditPolicy classes affecting the scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the RM web application.]]>
+      </doc>
+    </field>
+    <field name="YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SSL_SERVER_RESOURCE_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable YARN WebApp V2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_WEBAPP_UI2_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_WEBAPP_UI2_WARFILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_API_SERVICES_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_API_SYSTEM_SERVICES_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for application master reporting.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a node manager is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_RESERVATION_ACL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Are reservation acls enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_RESERVATION_ACL_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL of who can be admin of YARN cluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_ADMIN_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_APP_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL used in case none is found. Allows nothing.]]>
+      </doc>
+    </field>
+    <field name="OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether opportunistic container allocation
+  is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_MAX_ALLOCATIONS_PER_AM_HEARTBEAT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of opportunistic containers to be allocated in
+ AM heartbeat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_MAX_ALLOCATIONS_PER_AM_HEARTBEAT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of nodes to be used by the Opportunistic Container allocator for
+ dispatching containers during container allocation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency for computing least loaded NMs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comparator for determining node load for scheduling of opportunistic
+ containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_LIMIT_STDEV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Value of standard deviation used for calculation of queue limit
+ thresholds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the lower-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max length of container queue at NodeManager. This is a cluster-wide
+ configuration that acts as the upper-bound of optimal queue length
+ calculated by the NodeQueueLoadMonitor]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Min queue wait time for a container at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue wait time for a container queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_INTERMEDIATE_DATA_ENCRYPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable intermediate-data encryption at YARN level. For now, this
+ only is used by the FileSystemRMStateStore to setup right file-system
+ security attributes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle RM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts for
+ an application, if unset by user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_AM_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="GLOBAL_RM_AM_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of application attempts.
+ It's a global setting for all application masters.]]>
+      </doc>
+    </field>
+    <field name="RM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The keytab for the resource manager.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for RM.]]>
+      </doc>
+    </field>
+    <field name="RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing). Only applicable
+ when the http authentication type is kerberos.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cross origin (CORS) support.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait until a container is considered dead.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to include.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_INCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable submission pre-processor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with hosts for the submission processor to handle.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Submission processor refresh interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SUBMISSION_PREPROCESSOR_REFRESH_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to file with nodes to exclude.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODES_EXCLUDE_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle resource tracker calls.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_REGISTRATION_IP_HOSTNAME_CHECK_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Check IP and hostname resolution during nodemanager registration.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_REGISTRATION_IP_HOSTNAME_CHECK_KEY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the resource scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specify which handler will be used to process PlacementConstraints.
+ For details on PlacementConstraints, please refer to
+ {@link org.apache.hadoop.yarn.api.resource.PlacementConstraint}]]>
+      </doc>
+    </field>
+    <field name="DISABLED_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This handler rejects all allocate calls made by an application, if they
+ contain a {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}.]]>
+      </doc>
+    </field>
+    <field name="PROCESSOR_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, the placement of containers with constraints is
+ determined as a pre-processing step before the capacity or the fair
+ scheduler is called. Once the placement is decided, the capacity/fair
+ scheduler is invoked to perform the actual allocation. The advantage of
+ this approach is that it supports all constraint types (affinity,
+ anti-affinity, cardinality). Moreover, it considers multiple containers at
+ a time, which allows to satisfy more constraints than a container-at-a-time
+ approach can achieve. As it sits outside the main scheduler, it can be used
+ by both the capacity and fair schedulers. Note that at the moment it does
+ not account for task priorities within an application, given that such
+ priorities might be conflicting with the placement constraints.]]>
+      </doc>
+    </field>
+    <field name="SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Using this handler, containers with constraints will be placed by the main
+ scheduler. If the configured RM scheduler
+ <pre>yarn.resourcemanager.scheduler.class</pre>
+ cannot handle placement constraints, the corresponding SchedulingRequests
+ will be rejected. As of now, only the capacity scheduler supports
+ SchedulingRequests. In particular, it currently supports anti-affinity
+ constraints (no affinity or cardinality) and places one container at a
+ time. The advantage of this handler compared to the placement-processor is
+ that it follows the same ordering rules for queues (sorted by utilization,
+ priority) and apps (sorted by FIFO/fairness/priority) as the ones followed
+ by the main scheduler.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Placement Algorithm.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Used for BasicPlacementAlgorithm - default SERIAL.]]>
+      </doc>
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDULER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM set next Heartbeat interval for NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable Heartbeat Interval Scaling based on cpu utilization.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SCALING_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MIN_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MAX_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SPEEDUP_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NM_HEARTBEAT_INTERVAL_SLOWDOWN_FACTOR" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of worker threads that write the history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM. This configuration setting is for ATS V1.
+  This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SYSTEM_METRICS_PUBLISHER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn system metrics is published on the
+  timeline server or not by RM and NM. This configuration setting is for
+  ATS v2.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by RM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_MAX_CONF_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_ALWAYS_CANCEL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_RETRY_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_RETRY_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_DT_RENEWER_THREAD_RETRY_MAX_ATTEMPTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper interaction configs]]>
+      </doc>
+    </field>
+    <field name="RM_ZK_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_RM_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Zookeeper znode limit]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ZK_ZNODE_SIZE_LIMIT_BYTES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_RETRY_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_TIMEOUT_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_APPID_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ZK_APPID_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Index at which the RM Delegation Token ids will be split so that the
+ delegation token znodes stored in the zookeeper RM state store will be
+ stored as two different znodes (parent-child).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_ZK_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ZK_AUTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_RM_STATE_STORE_ROOT_NODE_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Root node ACLs for fencing]]>
+      </doc>
+    </field>
+    <field name="RM_HA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[HA related configs]]>
+      </doc>
+    </field>
+    <field name="RM_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_ID" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Store the related configuration files in File System]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_BASED_RM_CONF_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILE_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MEMORY_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ZK_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LEVELDB_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CONFIGURATION_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_MAX_LOGS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKConfigurationStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_AUTHORIZATION_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTO_FAILOVER_EMBEDDED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This property should never be set to {@code false}.">
+      <doc>
+      <![CDATA[This property controls whether {@link ActiveStandbyElector} leader
+ election should be used when {@link #CURATOR_LEADER_ELECTOR} is
+ {@code false}.
+
+ @deprecated This property should never be set to {@code false}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_EMBEDDED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.">
+      <doc>
+      <![CDATA[The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
+
+ @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.]]>
+      </doc>
+    </field>
+    <field name="AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_NO_HA_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_NO_HA_PROXY_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_MAX_ATTEMPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_BASE_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_SLEEPTIME_MAX_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_HA_FC_ELECTOR_ZK_RETRIES_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[number of zookeeper operation retry times in ActiveStandbyElector]]>
+      </doc>
+    </field>
+    <field name="CURATOR_LEADER_ELECTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.">
+      <doc>
+      <![CDATA[Whether to use the Curator-based elector for leader election.
+
+ @deprecated Eventually, we want to default to the curator-based
+ implementation and remove the {@link ActiveStandbyElector} based
+ implementation. We should remove this config then.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The class to use as the persistent store.]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemRMStateStore]]>
+      </doc>
+    </field>
+    <field name="FS_RM_STATE_STORE_NUM_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the leveldb database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM keeps. By default equals
+ to {@link #DEFAULT_RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum number of completed applications RM state store keeps. By
+ default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.">
+      <doc>
+      <![CDATA[The default value for
+ {@code yarn.resourcemanager.state-store.max-completed-applications}.
+ @deprecated This default value is ignored and will be removed in a future
+ release. The default value of
+ {@code yarn.resourcemanager.state-store.max-completed-applications} is the
+ value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application name]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_TYPE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default application type length]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_QUEUE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default queue name]]>
+      </doc>
+    </field>
+    <field name="RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Buckets (in minutes) for the number of apps running in each queue.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_METRICS_RUNTIME_BUCKETS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default sizes of the runtime metric buckets in minutes.]]>
+      </doc>
+    </field>
+    <field name="RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout(msec) for an untracked node to remain in shutdown or decommissioned
+ state.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[RM proxy users' prefix]]>
+      </doc>
+    </field>
+    <field name="RM_RESOURCE_PROFILES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File containing resource profiles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable loading resource-types.xml at client side.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications
+ to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Period in seconds of the poll timer task inside DecommissioningNodesWatcher
+ to identify and take care of DECOMMISSIONING nodes missing regular heart beat.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for all node manager configs.]]>
+      </doc>
+    </field>
+    <field name="NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DIST_SCHEDULING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Setting that controls whether distributed scheduling is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DIST_SCHEDULING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that will be sent to containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_USER_ENV" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[PATH components that will be prepended to the user's path.
+ If this is defined and the user does not define PATH, NM will also
+ append ":$PATH" to prevent this from eclipsing the PATH defined in
+ the container. This feature is only available for Linux.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ADMIN_FORCE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Environment variables that containers may override rather than use NodeManager's default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENV_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[address of node manager IPC.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The actual bind address for the NM.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[who will execute(launch) the containers.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_STATE_TRANSITION_LISTENERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of container state transition listeners.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Adjustment to make to the container os scheduling priority.
+ The valid values for this could vary depending on the platform.
+ On Linux, higher values mean run the containers at a less 
+ favorable priority than the NM. 
+ The value specified is an int.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MGR_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads container manager uses.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DELETE_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in cleanup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_EXECUTOR_EXIT_FILE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_EXECUTOR_EXIT_FILE_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[List of directories to store localized files in.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of files in each localized directories
+ Avoid tuning this too low.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the localizer IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Address where the collector service IPC is.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether yarn container events are published to
+ the timeline service or not by NM. This configuration setting is for ATS
+ V2]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PUBLISH_CONTAINER_EVENTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval in between cache cleanups.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target size of localizer cache in MB, per nodemanager. It is a target
+ retention size that only includes resources with PUBLIC and PRIVATE
+ visibility and excludes resources with APPLICATION visibility]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to handle localization requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOCALIZER_FETCH_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads to use for localization fetching.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to store container logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DIRS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle log aggregation in node manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default permissions for container logs.]]>
+      </doc>
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk Validator.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_DISK_VALIDATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum size of contain's diagnostics to keep for relaunching container
+ case.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Minimum container restart interval.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which the delayed token removal thread runs]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delegation Token renewer thread count]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_USER_PRIVILEGES_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The expiry interval for node IP caching. -1 disables the caching]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How many diagnostics/failure messages can be saved in RM for
+ log aggregation. It also defines the number of diagnostics/failure
+ messages can be shown in log aggregation web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to enable log aggregation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_FORMATS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_FILE_CONTROLLER_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait before deleting aggregated logs, -1 disables.
+ Be careful set this too small and you will spam the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_DEBUG_FILESIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long to wait between aggregated log retention checks. If set to
+ a value {@literal <=} 0 then the value is computed as one-tenth of the
+ log retention setting. Be careful set this too small and you will spam
+ the name node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How long for ResourceManager to wait for NodeManager to report its
+ log aggregation status. If waiting time of which the log aggregation status
+ is reported from NodeManager exceeds the configured value, RM will report
+ log aggregation status for this NodeManager as TIME_OUT.
+
+ This configuration will be used in NodeManager as well to decide
+ whether and when to delete the cached log aggregation status.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of seconds to retain logs on the NodeManager. Only applicable if Log
+ aggregation is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_RETAIN_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how often NMs wake up and upload log files]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed hard minimum limit for {@link
+ YarnConfiguration#NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS}.]]>
+      </doc>
+    </field>
+    <field name="MIN_LOG_ROLLING_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Define how many aggregated log files per application per NM we can have
+ in remote file system.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_DELETION_THREADS_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used in log cleanup. Only applicable if Log aggregation
+ is disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_DELETE_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where to aggregate logs to.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The remote log dir will be created at below location.
+ NM_REMOTE_APP_LOG_DIR/${user}/bucket_{NM_REMOTE_APP_LOG_DIR_SUFFIX}
+ /${bucketDir}/${appId}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether Older Application Log Directory is included.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REMOTE_APP_LOG_DIR_GROUPNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the group of the aggregated log directory.]]>
+      </doc>
+    </field>
+    <field name="YARN_LOG_SERVER_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_LOG_SERVER_WEBSERVICE_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_TRACKING_URL_GENERATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SYSTEM_RESERVED_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Amount of memory in MB that has been reserved for non-yarn use.]]>
+      </doc>
+    </field>
+    <field name="NM_PMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether physical memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_CHECK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether virtual memory check is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_CHECK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VMEM_PMEM_RATIO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Conversion ratio for physical memory to virtual memory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VMEM_PMEM_RATIO" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether to do memory check on overall usage.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the OOM handler code.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the OOM listener.]]>
+      </doc>
+    </field>
+    <field name="NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum time in seconds to resolve an OOM situation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ELASTIC_MEMORY_CONTROL_OOM_TIMEOUT_SEC" type="java.lang.Integer"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_VCORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of Virtual CPU Cores which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_VCORES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Count logical processors(like hyperthreads) as cores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_COUNT_LOGICAL_PROCESSORS_AS_CORES" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PCORES_VCORES_MULTIPLIER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Multiplier to convert physical cores to vcores.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PCORES_VCORES_MULTIPLIER" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Percentage of overall CPU which can be allocated for containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable or disable node hardware capability detection.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_ENFORCED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_ENFORCED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CPU_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable cpu isolation.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CPU_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for disk configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for disk operations is enabled.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Disk as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if resource handling for network bandwidth is
+ enabled. Work in progress: This configuration parameter may be
+ changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Network as a resource is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the interface to be used for applying network throttling rules.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_INTERFACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total available outbound bandwidth on the node. Work in
+ progress: This configuration parameter may be changed/removed in the future]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies the total outbound bandwidth available to YARN containers.
+ defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified.
+ Work in progress: This configuration parameter may be changed/removed in
+ the future]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for computation resources, example of computation resources like
+ GPU / FPGA / TPU, etc.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCE_PLUGINS_FAIL_FAST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies whether the initialization of the Node Manager should continue
+ if a certain device (GPU, FPGA, etc) was not found in the system. If set
+ to "true", then an exception will be thrown if a device is missing or
+ an error occurred during discovery.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_PLUGINS_FAIL_FAST" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls if pluggable device plugin framework is enabled.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The pluggable device plugin framework is disabled by default]]>
+      </doc>
+    </field>
+    <field name="NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting contains vendor plugin class names for
+ device plugin framework to load. Split by comma]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for gpu configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AUTOMATICALLY_DISCOVER_GPU_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_GPU_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls where to how to invoke GPU binaries]]>
+      </doc>
+    </field>
+    <field name="NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings to control which implementation of docker plugin for GPU will be
+ used.
+
+ By default uses NVIDIA docker v1.]]>
+      </doc>
+    </field>
+    <field name="NVIDIA_DOCKER_V1" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_V2" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls end point of nvidia-docker-v1 plugin.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_RESOURCE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for FPGA configurations. Work in progress: This configuration
+ parameter may be changed/removed in the future.]]>
+      </doc>
+    </field>
+    <field name="NM_FPGA_ALLOWED_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_PATH_TO_EXEC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_FPGA_VENDOR_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_DEVICE_DISCOVERY_SCRIPT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_FPGA_AVAILABLE_DEVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_HANDLER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_TAG_HANDLER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NETWORK_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NETWORK_RESOURCE_TAG_MAPPING_FILE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[NM Webapp https address.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_ENABLE_CORS_FILTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable CORS filter.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WEBAPP_ENABLE_CORS_FILTER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCE_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor resource in a node.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RESOURCE_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_RESOURCE_CALCULATOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates containers current resource utilization.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_MON_PROCESS_TREE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Class that calculates process tree resource utilization.]]>
+      </doc>
+    </field>
+    <field name="PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_TAG_BASED_PLACEMENT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MONITOR_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable switch for container log monitoring.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MONITOR_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_MON_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[How often to monitor logs generated by containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_MON_INTERVAL_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for a single container log directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_DIR_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The disk space limit for all of a container's logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_LOG_TOTAL_SIZE_LIMIT_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/disable container metrics.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_ENABLE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_PERIOD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container metrics flush period. -1 for flush on completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The delay time ms to unregister container metrics after completion.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable disks' health checker. Default is true. An expert level
+ configuration property.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running disks' health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, disks' health is checked every 2 minutes.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_HEALTHY_DISKS_FRACTION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum fraction of number of disks to be healthy for the nodemanager
+ to launch new containers. This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, at least 25% of disks are to be healthy to say that the node is
+ healthy in terms of disks.]]>
+      </doc>
+    </field>
+    <field name="NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The maximum percentage of disk space that can be used after which a disk is
+ marked as offline. Values can range from 0.0 to 100.0. If the value is
+ greater than or equal to 100, NM will check for full disk. This applies to
+ nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-utilization-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, 90% of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the disk utilisation percentage
+ threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_UTILIZATION_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The low threshold percentage of disk space used when an offline disk is
+ marked as online. Values can range from 0.0 to 100.0. The value shouldn't
+ be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is
+ more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be
+ set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="NM_MIN_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on a local dir for it to be used.
+ This applies to nm-local-dirs and nm-log-dirs.
+
+ This applies when disk-free-space-threshold.enabled is true.]]>
+      </doc>
+    </field>
+    <field name="NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enable/Disable the minimum disk free
+ space threshold for disk health checker.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DISK_FREE_SPACE_THRESHOLD_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WM_HIGH_PER_DISK_FREE_SPACE_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The minimum space that must be available on an offline
+ disk for it to be marked as online.  The value should not be less
+ than NM_MIN_PER_DISK_FREE_SPACE_MB.  If its value is less than
+ NM_MIN_PER_DISK_FREE_SPACE_MB or is not set, it will be set to the
+ same value as NM_MIN_PER_DISK_FREE_SPACE_MB.
+ This applies to nm-local-dirs and nm-log-dirs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[By default, all of the disk can be used before it is marked as offline.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health checker scripts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_SCRIPTS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not to run the node health script before the NM
+  starts up.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_RUN_BEFORE_STARTUP" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check time out period for all scripts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_HEALTH_CHECK_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Health check script time out period.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_PATH_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The health check script to run.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_OPTS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The arguments to pass to the health check script.]]>
+      </doc>
+    </field>
+    <field name="NM_HEALTH_CHECK_SCRIPT_INTERVAL_MS_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Frequency of running node health script.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The JVM options used on forking ContainerLocalizer process
+      by container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log level of container localizer process.]]>
+      </doc>
+    </field>
+    <field name="NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for runtime configuration constants.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The standard values are:
+ <ul>
+   <li>default</li>
+   <li>docker</li>
+   <li>javasandbox</li>
+   <li>runc</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_CLASS_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of allowed runtimes when using LinuxContainerExecutor.]]>
+      </doc>
+    </field>
+    <field name="LINUX_CONTAINER_RUNTIME_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runtime to be used.]]>
+      </doc>
+    </field>
+    <field name="RUNC_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The runc image tag to manifest plugin class that should be used.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc image tag to manifest plugin class.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The runc manifest to resources plugin class that should be used.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_MANIFEST_TO_RESOURCES_PLUGIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc manifest to resources plugin plugin class.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_IMAGE_TOPLEVEL_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The HDFS location under which the oci image manifests, layers,
+ and configs directories exist.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_IMAGE_TOPLEVEL_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default HDFS location under which the oci image manifests, layers,
+ and configs directories exist.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_LAYER_MOUNTS_TO_KEEP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Target count of layer mounts that we should keep on disk at one time.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval in seconds between executions of reaping layer mounts.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RUNC_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default runc image to be used.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default set of networks allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default set of runtimes allowed when launching containers using the
+ RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_ALLOW_HOST_PID_NAMESPACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow host pid namespace for containers. Use with care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_ALLOW_HOST_PID_NAMESPACE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Host pid namespace for containers is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_DEFAULT_RO_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-only mounts to be bind-mounted into all
+  runC containers that use RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_DEFAULT_RW_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-write mounts to be bind-mounted into all
+  runC containers that use RuncContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_SECCOMP_PROFILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to the seccomp profile to use with Runc containers.]]>
+      </doc>
+    </field>
+    <field name="NM_HDFS_RUNC_IMAGE_TAG_TO_HASH_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The HDFS location where the runC image tag to hash file exists.]]>
+      </doc>
+    </field>
+    <field name="NM_LOCAL_RUNC_IMAGE_TAG_TO_HASH_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The local file system location where the runC image tag to hash file exists.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_CACHE_REFRESH_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval in seconds between refreshing the hdfs image tag to
+ hash cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_CACHE_REFRESH_INTERVAL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default interval in seconds between refreshing the hdfs image tag to
+ hash cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_NUM_MANIFESTS_TO_CACHE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of manifests to cache in the image tag to hash cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NUM_MANIFESTS_TO_CACHE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default number of manifests to cache in the image tag to hash cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_STAT_CACHE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The timeout value in seconds for the values in the stat cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RUNC_STAT_CACHE_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default timeout value in seconds for the values in the stat cache.]]>
+      </doc>
+    </field>
+    <field name="NM_RUNC_STAT_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The size of the stat cache which stores stats of the layers and config.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RUNC_STAT_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default size of the stat cache which stores stats of the
+ layers and config.]]>
+      </doc>
+    </field>
+    <field name="DOCKER_CONTAINER_RUNTIME_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_IMAGE_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default docker image to be used.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_IMAGE_UPDATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default option to decide whether to pull the latest image or not.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Capabilities allowed (and added by default) for docker containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[These are the default capabilities added by docker. We'll use the same
+ set here. While these may not be case-sensitive from a docker
+ perspective, it is best to keep these uppercase.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow privileged containers. Use with extreme care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Privileged containers are disabled by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ENABLE_USER_REMAPPING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[enable user remapping.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set enable user remapping as false by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable uids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower uid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[lower limit for acceptable gids of user remapped user.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set user remapping lower gid limit to 1 by default.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[ACL list for users allowed to run privileged containers.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default list for users allowed to run privileged containers is empty.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of networks allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request. This
+  network must be one of the (configurable) set of allowed container
+  networks.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The network used when launching containers using the
+ DockerContainerRuntime when no network is specified in the request and
+ no default network is configured.
+ .]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The set of runtimes allowed when launching containers using the
+ DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow host pid namespace for containers. Use with care.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Host pid namespace for containers is disabled by default.]]>
+      </doc>
+    </field>
+    <field name="YARN_HTTP_WEBAPP_EXTERNAL_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="HADOOP_HTTP_WEBAPP_SCHEDULER_PAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="This field is deprecated for
+ {@link #YARN_HTTP_WEBAPP_SCHEDULER_PAGE}">
+      <doc>
+      <![CDATA[@deprecated This field is deprecated for
+ {@link #YARN_HTTP_WEBAPP_SCHEDULER_PAGE}]]>
+      </doc>
+    </field>
+    <field name="YARN_HTTP_WEBAPP_SCHEDULER_PAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_WEBAPP_CUSTOM_DAO_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_WEBAPP_CUSTOM_UNWRAPPED_DAO_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether or not users are allowed to request that Docker containers honor
+ the debug deletion delay. This is useful for troubleshooting Docker
+ container related launch failures.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value on whether or not a user can request that Docker
+ containers honor the debug deletion delay.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_STOP_GRACE_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}">
+      <doc>
+      <![CDATA[A configurable value to pass to the Docker Stop command. This value
+ defines the number of seconds between the docker stop command sending
+ a SIGTERM and a SIGKILL.
+
+ @deprecated use {@link YarnConfiguration#NM_SLEEP_DELAY_BEFORE_SIGKILL_MS}]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for the grace period between the SIGTERM and the
+ SIGKILL in the Docker Stop command.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RO_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-only mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_RW_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of read-write mounts to be bind-mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="NM_DOCKER_DEFAULT_TMPFS_MOUNTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default list of tmpfs mounts to be mounted into all
+  Docker containers that use DockerContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The mode in which the Java Container Sandbox should run detailed by
+  the JavaSandboxLinuxContainerRuntime.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for application local directories.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Location for non-default java policy file.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Prefix for group to policy file mapping.]]>
+      </doc>
+    </field>
+    <field name="YARN_CONTAINER_SANDBOX_WHITELIST_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The group which will run by default without the java security manager.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_EXECUTOR_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path to the Linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_NONSECURE_MODE_LIMIT_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[True if linux-container-executor should limit itself to one user
+ when running in non-secure mode.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_LOCAL_USER_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The UNIX user that containers will run as when Linux-container-executor
+ is used in nonsecure mode (a use case for this is using cgroups).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_LOCAL_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NONSECURE_MODE_USER_PATTERN_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The allowed pattern for UNIX user names enforced by 
+ Linux-container-executor when used in nonsecure mode (use case for this 
+ is using cgroups). The default value is taken from /usr/sbin/adduser]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NONSECURE_MODE_USER_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_RESOURCES_HANDLER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The type of resource enforcement to use with the
+  linux container executor.]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_HIERARCHY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The path the linux container executor should use for cgroups]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Where the linux container executor should mount cgroups if not found]]>
+      </doc>
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the apps should run in strict resource usage mode(not allowed to
+ use spare CPU)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_APPLICATION_HTTPS_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Specifies what the RM does regarding HTTPS enforcement for communication
+ with AM Web Servers, as well as generating and providing certificates.
+ Possible values are:
+ <ul>
+   <li>NONE - the RM will do nothing special.</li>
+   <li>LENIENT - the RM will generate and provide a keystore and truststore
+   to the AM, which it is free to use for HTTPS in its tracking URL web
+   server.  The RM proxy will still allow HTTP connections to AMs that opt
+   not to use HTTPS.</li>
+   <li>STRICT - this is the same as LENIENT, except that the RM proxy will
+   only allow HTTPS connections to AMs; HTTP connections will be blocked
+   and result in a warning page to the user.</li>
+ </ul>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_APPLICATION_HTTPS_POLICY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_TIMEOUT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFALUT_RM_PROXY_TIMEOUT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_PROXY_CONNECTION_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_PROXY_CONNECTION_TIMEOUT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval of time the linux container executor should try cleaning up
+ cgroups entry when cleaning up a container. This is required due to what 
+ it seems a race condition because the SIGTERM/SIGKILL is asynch.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Delay between attempts to remove linux cgroup.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Indicates if memory and CPU limits will be set for the Windows Job
+ Object for the containers launched by the default container executor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WINDOWS_SECURE_CONTAINER_GROUP" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[/* The Windows group that the windows-secure-container-executor should run as.]]>
+      </doc>
+    </field>
+    <field name="NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress aggregated logs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the node manager.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Boolean indicating whether loading aux services from a manifest is
+ enabled. If enabled, aux services may be dynamically modified through
+ reloading the manifest via filesystem changes or a REST API. When
+ enabled, aux services configuration properties unrelated to the manifest
+ will be ignored.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File containing auxiliary service specifications.]]>
+      </doc>
+    </field>
+    <field name="NM_AUX_SERVICES_MANIFEST_RELOAD_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Interval at which manifest file will be reloaded when modifications are
+ found (0 or less means that the file will not be checked for modifications
+ and reloaded).]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_FMT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICE_REMOTE_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_AUX_SERVICES_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_PATTERN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINER_STDERR_BYTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_CONTAINER_STDERR_BYTES" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_USER_NAME_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos keytab to be used for spnego filter for NM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_USER_HOME_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time in seconds between full compactions of the NM state database.
+  Setting the interval to zero disables the full compaction cycles.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RECOVERY_SUPERVISED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_RECOVERY_SUPERVISED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_LOG_AGG_POLICY_CLASS_PARAMETERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The kerberos principal for the proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Keytab for Proxy.]]>
+      </doc>
+    </field>
+    <field name="PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PROXY_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Binding address for the web proxy.]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[YARN Service Level Authorization]]>
+      </doc>
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[No. of milliseconds to wait between sending a SIGTERM and SIGKILL
+ to a running container]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROCESS_KILL_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for a process to come up when trying to cleanup
+ container resources]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_PROCESS_KILL_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to RM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The threshold used to trigger the logging of event types and counts
+  in RM's main event dispatcher. Default value is 5000,
+  which means RM will print events info when the queue size cumulatively
+  reaches 5000 every time. Such info can be used to reveal what
+  kind of events that RM is stuck at processing mostly,
+  it can help to narrow down certain performance issues.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Resource manager dispatcher thread monitor sampling rate.
+ Units are samples per minute.  This controls how often to sample
+ the cpu utilization of the resource manager dispatcher thread.
+ The cpu utilization is displayed on the RM UI as scheduler busy %.
+ Set to zero to disable the dispatcher thread monitor.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APPLICATION_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
+ entries]]>
+      </doc>
+    </field>
+    <field name="AMRM_PROXY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether AMRMProxy is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AMRM_PROXY_HA_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AMRM_PROXY_HA_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default platform-agnostic CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries. The parameter expansion marker
+ will be replaced with real parameter expansion marker ('%' for Windows and
+ '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
+ will be replaced as $VAR on Linux, and %VAR% on Windows.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_APPLICATION_CLASSPATH" type="java.lang.String[]"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[<p>
+ Default platform-specific CLASSPATH for YARN applications. A
+ comma-separated list of CLASSPATH entries constructed based on the client
+ OS environment expansion syntax.
+ </p>
+ <p>
+ Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
+ cross-platform practice i.e. submit an application from a Windows client to
+ a Linux/Unix server or vice versa.
+ </p>]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CONTAINER_TEMP_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Container temp directory]]>
+      </doc>
+    </field>
+    <field name="IS_MINI_YARN_CLUSTER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MC_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_FIXED_PORTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether to use fixed ports with the minicluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_FIXED_PORTS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default is false to be able to run tests concurrently without port
+ conflicts.]]>
+      </doc>
+    </field>
+    <field name="YARN_MINICLUSTER_USE_RPC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether the NM should use RPC to connect to the RM. Default is false.
+ Can be set to true only when using fixed ports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_USE_RPC" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Whether users are explicitly trying to control resource monitoring
+ configuration for the MiniYARNCluster. Disabled by default.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_MINICLUSTER_NM_PMEM_MB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Allow changing the memory for the NodeManager in the MiniYARNCluster]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The log directory for the containers]]>
+      </doc>
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_APP_CONTAINER_LOG_BACKUPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_VERSION" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_VERSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_UI_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Comma separated list of names for UIs hosted in the timeline server
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_WEB_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Relative web path that will serve up this UI (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client settings.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Path to war file or static content directory for this UI
+ (For pluggable UIs).]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting for timeline service v1.5]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline service v2.0.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default schema prefix for hbase tables.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[config param name to override schema prefix.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how often the timeline collector flushes the
+ timeline writer.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls the capacity of the queue for async writes
+ to timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WRITER_ASYNC_QUEUE_CAPACITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_FINAL_VALUE_RETENTION_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how long the final value of
+ a metric of a completed app is retained before merging
+ into the flow sum.]]>
+      </doc>
+    </field>
+    <field name="FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name of the setting for the location of the coprocessor
+ jar on hdfs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default hdfs location for flowrun coprocessor jar.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[This setting controls the max size of the flow name getting generated
+ in ATSv2 after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="FLOW_NAME_DEFAULT_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default setting for flow name size has no size restriction
+ after removing UUID if present.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that points to an optional HBase configuration
+ (hbase-site.xml file) with settings that will override the ones found on
+ the classpath.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that enables or disables authentication checks
+ for reading timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that controls how often in-memory app level
+ aggregation is kicked off in timeline collector.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED" type="java.lang.Boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default setting for authentication checks for reading timeline
+ service v2 data.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The name for setting that lists the users and groups who are allowed
+ to read timeline service v2 data. It is a comma separated list of
+ user, followed by space, then comma separated list of groups.
+ It will allow this list of users and groups to read the data
+ and reject everyone else.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The default value for list of the users who are allowed to read
+ timeline service v2 data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls how long the final value of a metric of a
+ completed app is retained before merging into the flow sum. Up to this time
+ after an application is completed out-of-order values that arrive can be
+ recognized and discarded at the cost of increased storage.]]>
+      </doc>
+    </field>
+    <field name="ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[default version for any flow.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time period for which timeline v2 client will wait for draining
+ leftover entities after stop.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether application history service is
+  enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Application history store class]]>
+      </doc>
+    </field>
+    <field name="APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Save container meta-info in the application history store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_URI" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for FileSystemApplicationHistoryStore]]>
+      </doc>
+    </field>
+    <field name="FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[T-file compression types used to compress history data.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The setting that controls whether timeline service is enabled or not.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[host:port address for timeline service RPC APIs.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The listening endpoint for the timeline service application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_HANDLER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads to handle client RPC API requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the timeline service web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_HISTORY_MAX_APPS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Defines the max number of applications could be fetched using
+ REST API or application history protocol and shown in timeline
+ server web ui.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_APPLICATION_HISTORY_MAX_APPS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service store class.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_ENABLE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service enable data age off]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service length of time to retain data]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service rolling period. Valid values are daily, half_daily,
+ quarter_daily, and hourly.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_ROLLING_PERIOD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Roll a new database each hour.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Implementation specific configuration prefix for Timeline Service
+ leveldb.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb read cache (uncompressed blocks). This is
+ per rolling instance so should be tuned if using rolling leveldb
+ timeline store]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb read cache size if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write buffer size.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write buffer size if no configuration is specified. This
+ is per rolling instance so should be tuned if using rolling leveldb
+ timeline store.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb write batch size. This value can be tuned down
+ to reduce lock time for ttl eviction.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb write batch size is no configuration is specified]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time read cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb start time write cache (number of entities)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb interval to wait between deletion rounds]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service leveldb number of concurrent open files. Tuned this
+ configuration to stay within system limits. This is per rolling instance
+ so should be tuned if using rolling leveldb timeline store.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default leveldb max open files if no configuration is specified.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_PRINCIPAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos principal for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_KEYTAB" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The Kerberos keytab for the timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Enables cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default value for cross origin support for timeline server.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, max retries (-1 means no limit)]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client call, retry interval]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline client policy for whether connections are fatal]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_RECOVERY_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to enable recovery of timeline service]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_STATE_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store class]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Timeline service state store leveldb path]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_OFFLINE_AGGREGATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[whether the shared cache is enabled/disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the shared cache root directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_ROOT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NESTED_LEVEL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The config key for the level of nested directories before getting to the
+ checksum directory.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NESTED_LEVEL" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_STORE_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_APP_CHECKER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_ADMIN_CLIENT_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Number of threads used to handle SCM admin interface.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the SCM web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STORE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_STALENESS_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[A resource in the InMemorySCMStore is considered stale if the time since
+ the last reference exceeds the staleness period. This value is specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the in-memory store runs its first check to remove
+ dead initial applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="IN_MEMORY_CHECK_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which the in-memory store checks to remove dead initial
+ applications. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_PERIOD_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The frequency at which a cleaner task runs. Specified in minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_PERIOD_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_INITIAL_DELAY_MINS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initial delay before the first cleaner task is scheduled. Specified in
+ minutes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLEANER_RESOURCE_SLEEP_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The time to sleep between processing each shared cache resource. Specified
+ in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the node manager interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_UPLOADER_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of SCM threads used to handle notify requests from the node
+ manager.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the client interface in the SCM.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCM_CLIENT_SERVER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of threads used to handle shared cache manager requests.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[the checksum algorithm implementation]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The replication factor for the node manager uploader for the shared cache.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FAILOVER_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FAILOVER_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_REGISTRY_BASE_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_MACHINE_LIST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Parent znode path under which ZKRMStateStore will create znodes.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_USERNAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_PASSWORD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_URL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_JDBC_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_PIPELINE_CACHE_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_CLIENTRM_SUBMIT_RETRY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The number of retries for GetNewApplication and SubmitApplication in
+ {@code FederationClientInterceptor}.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_USER_CLIENT_THREADS_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The https address of the Router web application.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_PORT" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST to communicate with
+ each SubCluster.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_DEFAULT_INTERCEPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interceptor class used in FederationInterceptorREST should return
+ partial AppReports.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[TERMS:
+ csi-driver: a 3rd party CSI driver which implements the CSI protocol.
+   It is provided by the storage system.
+ csi-driver-adaptor: this is an internal RPC service working
+   as a bridge between YARN and a csi-driver.]]>
+      </doc>
+    </field>
+    <field name="NM_CSI_DRIVER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_DRIVER_ENDPOINT_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_ADDRESS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSI_ADAPTOR_ADDRESSES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[One or more socket addresses for csi-adaptor.
+ Multiple addresses are delimited by ",".]]>
+      </doc>
+    </field>
+    <field name="NM_CSI_DRIVER_NAMES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_REGISTRY_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
+ The interval of the yarn client's querying application state after
+ application submission. The unit is millisecond.]]>
+      </doc>
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The interval that the yarn client library uses to poll the completion
+ status of the asynchronous API of application client protocol.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The duration that the yarn client library waits, cumulatively across polls,
+ for an expected state change to occur. Defaults to -1, which indicates no
+ limit.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max number of threads in NMClientAsync to process container management
+ events]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CLIENT_MAX_NM_PROXIES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Maximum number of proxy connections to cache for node managers. If set
+ to a value greater than zero then the cache is enabled and the NMClient
+ and MRAppMaster will cache the specified number of node manager proxies.
+ There will be at max one proxy per node manager. Ex. configuring it to a
+ value of 5 will make sure that client will at max have 5 proxies cached
+ with 5 different node managers. These connections for these proxies will
+ be timed out if idle for more than the system wide idle timeout period.
+ Note that this could cause issues on large clusters as many connections
+ could linger simultaneously and lead to a large number of connection
+ threads. The token used for authentication will be used only at
+ connection creation time. If a new token is received then the earlier
+ connection should be closed in order to use the new token. This and
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related
+ and should be in sync (no need for them to be equal).
+ If the value of this property is zero then the connection cache is
+ disabled and connections will use a zero idle timeout to prevent too
+ many connection threads on large clusters.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_CLIENT_MAX_NM_PROXIES" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait to establish a connection to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_MAX_WAIT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each attempt to connect to NM]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_CLIENT_NM_CONNECT_RETRY_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_HTTP_POLICY_DEFAULT" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max time to wait for NM to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.]]>
+      </doc>
+    </field>
+    <field name="NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time interval between each NM attempt to connection to RM.
+ When not set, proxy will fall back to use value of
+ RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-labels configurations]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node label store implementation class]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FS_NODE_LABELS_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[URI for NodeLabelManager]]>
+      </doc>
+    </field>
+    <field name="NODE_ATTRIBUTE_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node-attribute configurations.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute store implementation class.]]>
+      </doc>
+    </field>
+    <field name="FS_NODE_ATTRIBUTE_STORE_ROOT_DIR" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[File system node attribute store directory.]]>
+      </doc>
+    </field>
+    <field name="NODE_LABELS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Flag to indicate if the node labels feature enabled, by default it's
+ disabled]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NODE_LABELS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CENTRALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NODELABEL_CONFIGURATION_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="EXCLUSIVE_ENFORCED_PARTITIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_APP_ATTEMPT_DIAGNOSTICS_LIMIT_KC" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONFIG_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="SCRIPT_NODE_DESCRIPTOR_PROVIDER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_RESYNC_INTERVAL" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_PARTITION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_CONFIG" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD" type="float"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Node attribute provider fetch attributes interval and timeout.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DISPLAY_APPS_FOR_LOGGED_IN_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_ENTITY_LIST_BY_USER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FILTER_INVALID_XML_CHARS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FILTER_INVALID_XML_CHARS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="ENABLE_REST_APP_SUBMISSIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_ENABLE_REST_APP_SUBMISSIONS" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="REST_CSRF" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_CUSTOM_HEADER" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_CSRF_METHODS_TO_IGNORE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="XFS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="YARN_XFS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_XFS_OPTIONS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for timeline reader.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_STORAGE_MONITOR_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Marked collector properties as Private since it run as auxillary service.]]>
+      </doc>
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_HOST" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_ENABLED" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for NUMA awareness.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_ENABLED" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_READ_TOPOLOGY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_READ_TOPOLOGY" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NODE_IDS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Settings for activities manager.]]>
+      </doc>
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_CLEANUP_INTERVAL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[The cleanup interval for activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_CLEANUP_INTERVAL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time to live for scheduler activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_SCHEDULER_ACTIVITIES_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_TTL_MS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Time to live for app activities in milliseconds.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_TTL_MS" type="long"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_MAX_QUEUE_LENGTH" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Max queue length for app activities.]]>
+      </doc>
+    </field>
+    <field name="DEFAULT_RM_ACTIVITIES_MANAGER_APP_ACTIVITIES_MAX_QUEUE_LENGTH" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="NM_CONTAINERS_LAUNCHER_CLASS" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Containers launcher implementation to use.]]>
+      </doc>
+    </field>
+    <field name="YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.conf.YarnConfiguration -->
+</package>
+<package name="org.apache.hadoop.yarn.exceptions">
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <class name="ApplicationAttemptNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationAttemptNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getApplicationAttemptReport (GetApplicationAttemptReportRequest)}
+ API when the Application Attempt doesn't exist in Application History Server or
+ {@link ApplicationMasterProtocol#allocate(AllocateRequest)} if application
+ doesn't exist in RM.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <class name="ApplicationIdNotProvidedException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationIdNotProvidedException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Exception to be thrown when Client submit an application without
+ providing {@link ApplicationId} in {@link ApplicationSubmissionContext}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <class name="ApplicationNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} API
+ when the Application doesn't exist in RM and AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <class name="ConfigurationException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConfigurationException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ConfigurationException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on unrecoverable configuration errors.
+ An example is container launch error due to configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ConfigurationException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <class name="ContainerNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerNotFoundException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerNotFoundException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown on
+ {@link ApplicationHistoryProtocol#getContainerReport (GetContainerReportRequest)}
+ API when the container doesn't exist in AHS]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ContainerNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <class name="ResourceNotFoundException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="org.apache.hadoop.yarn.api.records.Resource, java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ResourceNotFoundException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when details of an unknown resource type
+ are requested.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.ResourceNotFoundException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <class name="YarnException" extends="java.lang.Exception"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnException"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YarnException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[YarnException indicates exceptions from yarn servers. On the other hand,
+ IOExceptions indicates exceptions from RPC layer.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YarnException -->
+  <!-- start class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+  <class name="YARNFeatureNotEnabledException" extends="org.apache.hadoop.yarn.exceptions.YarnException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="YARNFeatureNotEnabledException" type="java.lang.String, java.lang.Throwable"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[This exception is thrown when a feature is being used which is not enabled
+ yet.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <class name="ApplicationInitializationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ApplicationId, java.nio.ByteBuffer"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the user-name of the application-submitter
+ 
+ @return user-name]]>
+      </doc>
+    </method>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <method name="getApplicationDataForService" return="java.nio.ByteBuffer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the data sent to the NodeManager via
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ as part of {@link ContainerLaunchContext#getServiceData()}
+ 
+ @return the servicesData for this application.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationInitializationContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <class name="ApplicationTerminationContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationTerminationContext" type="org.apache.hadoop.yarn.api.records.ApplicationId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ApplicationId} of the application being stopped.
+ 
+ @return applications ID]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when stopping an
+ application.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ApplicationTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <interface name="AuxiliaryLocalPathHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getLocalPathForRead" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getLocalPathForWrite" return="org.apache.hadoop.fs.Path"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <param name="size" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a path from the local FS for writing a file of an estimated size
+ for a given Auxiliary Service.
+ @param path the requested path
+ @param size the size of the file that is going to be written
+ @return the complete path to the file on a local disk
+ @throws IOException if the path creations fails]]>
+      </doc>
+    </method>
+    <method name="getAllLocalPathsForRead" return="java.lang.Iterable"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get all paths from the local FS for reading for a given Auxiliary Service.
+ @param path the requested path
+ @return the complete path list to the file on a local disk as an Iterable
+ @throws IOException if the file read encounters a problem]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[An Interface that can retrieve local directories to read from or write to.
+  Components can implement this interface to link it to
+  their own Directory Handler Service]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler -->
+  <!-- start class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <class name="AuxiliaryService" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AuxiliaryService" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getRecoveryPath" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the path specific to this auxiliary service to use for recovery.
+
+ @return state storage path or null if recovery is not enabled]]>
+      </doc>
+    </method>
+    <method name="initializeApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationInitializationContext"/>
+      <doc>
+      <![CDATA[A new application is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the application initialization.
+ 
+ @param initAppContext context for the application's initialization]]>
+      </doc>
+    </method>
+    <method name="stopApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopAppContext" type="org.apache.hadoop.yarn.server.api.ApplicationTerminationContext"/>
+      <doc>
+      <![CDATA[An application is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+ 
+ @param stopAppContext context for the application termination]]>
+      </doc>
+    </method>
+    <method name="getMetaData" return="java.nio.ByteBuffer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Retrieve meta-data for this {@link AuxiliaryService}. Applications using
+ this {@link AuxiliaryService} SHOULD know the format of the meta-data -
+ ideally each service should provide a method to parse out the information
+ to the applications. One example of meta-data is contact information so
+ that applications can access the service remotely. This will only be called
+ after the service's {@link #start()} method has finished. the result may be
+ cached.
+ 
+ <p>
+ The information is passed along to applications via
+ {@link StartContainersResponse#getAllServicesMetaData()} that is returned by
+ {@link ContainerManagementProtocol#startContainers(StartContainersRequest)}
+ </p>
+ 
+ @return meta-data for this service that should be made available to
+         applications.]]>
+      </doc>
+    </method>
+    <method name="initializeContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="initContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerInitializationContext"/>
+      <doc>
+      <![CDATA[A new container is started on this NodeManager. This is a signal to
+ this {@link AuxiliaryService} about the container initialization.
+ This method is called when the NodeManager receives the container launch
+ command from the ApplicationMaster and before the container process is 
+ launched.
+
+ @param initContainerContext context for the container's initialization]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="stopContainerContext" type="org.apache.hadoop.yarn.server.api.ContainerTerminationContext"/>
+      <doc>
+      <![CDATA[A container is finishing on this NodeManager. This is a signal to this
+ {@link AuxiliaryService} about the same.
+
+ @param stopContainerContext context for the container termination]]>
+      </doc>
+    </method>
+    <method name="setRecoveryPath"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recoveryPath" type="org.apache.hadoop.fs.Path"/>
+      <doc>
+      <![CDATA[Set the path for this auxiliary service to use for storing state
+ that will be used during recovery.
+
+ @param recoveryPath where recoverable state should be stored]]>
+      </doc>
+    </method>
+    <method name="getAuxiliaryLocalPathHandler" return="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Method that gets the local dirs path handler for this Auxiliary Service.
+
+ @return auxiliaryPathHandler object that is used to read from and write to
+ valid local Dirs.]]>
+      </doc>
+    </method>
+    <method name="setAuxiliaryLocalPathHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="auxiliaryLocalPathHandler" type="org.apache.hadoop.yarn.server.api.AuxiliaryLocalPathHandler"/>
+      <doc>
+      <![CDATA[Method that sets the local dirs path handler for this Auxiliary Service.
+
+ @param auxiliaryLocalPathHandler the pathHandler for this auxiliary service]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A generic service that will be started by the NodeManager. This is a service
+ that administrators have to configure on each node by setting
+ {@link YarnConfiguration#NM_AUX_SERVICES}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.AuxiliaryService -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <class name="ContainerContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get user of the container being initialized or stopped.
+
+ @return the user]]>
+      </doc>
+    </method>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container being initialized or stopped.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link Resource} the resource capability allocated to the container
+ being initialized or stopped.
+
+ @return the resource capability.]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container
+ being initialized or stopped.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ExecutionType} the execution type of the container
+ being initialized or stopped.
+
+ @return the execution type of the container]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Base context class for {@link AuxiliaryService} initializing and stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <class name="ContainerInitializationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInitializationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Initialization context for {@link AuxiliaryService} when starting a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerInitializationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <interface name="ContainerLogAggregationPolicy"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseParameters"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parameters" type="java.lang.String"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to initial the policy object with parameters specified by the application
+ or the cluster-wide setting.
+ </p>
+
+ @param parameters parameters with scheme defined by the policy class.]]>
+      </doc>
+    </method>
+    <method name="shouldDoLogAggregation" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logContext" type="org.apache.hadoop.yarn.server.api.ContainerLogContext"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by the NodeManager log aggregation service
+ to ask the policy object if a given container's logs should be aggregated.
+ </p>
+
+ @param logContext ContainerLogContext
+ @return Whether or not the container's logs should be aggregated.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This API is used by NodeManager to decide if a given container's logs
+ should be aggregated at run time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.ContainerLogAggregationPolicy -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <class name="ContainerLogContext" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerLogContext" type="org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.server.api.ContainerType, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerId} of the container.
+
+ @return the container ID]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get {@link ContainerType} the type of the container.
+
+ @return the type of the container]]>
+      </doc>
+    </method>
+    <method name="getExitCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the exit code of the container.
+
+ @return the exit code]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Context class for {@link ContainerLogAggregationPolicy}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerLogContext -->
+  <!-- start class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <class name="ContainerTerminationContext" extends="org.apache.hadoop.yarn.server.api.ContainerContext"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTerminationContext" type="java.lang.String, org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.Resource, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[Termination context for {@link AuxiliaryService} when stopping a
+ container.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.ContainerTerminationContext -->
+  <!-- start interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+  <interface name="SCMAdminProtocol"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="runCleanerTask" return="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The method used by administrators to ask SCM to run cleaner task right away
+ </p>
+
+ @param request request <code>SharedCacheManager</code> to run a cleaner task
+ @return <code>SharedCacheManager</code> returns an empty response
+         on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[<p>
+ The protocol between administrators and the <code>SharedCacheManager</code>
+ </p>]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.server.api.SCMAdminProtocol -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+</package>
+<package name="org.apache.hadoop.yarn.util.constraint">
+  <!-- start class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+  <class name="PlacementConstraintParser" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <method name="parseExpression" return="org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="constraintStr" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a given constraint expression to a {@link AbstractConstraint},
+ this expression can be any valid form of constraint expressions.
+
+ @param constraintStr expression string
+ @return a parsed {@link AbstractConstraint}
+ @throws PlacementConstraintParseException when given expression
+ is malformed]]>
+      </doc>
+    </method>
+    <method name="parsePlacementSpec" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="expression" type="java.lang.String"/>
+      <exception name="PlacementConstraintParseException" type="org.apache.hadoop.yarn.util.constraint.PlacementConstraintParseException"/>
+      <doc>
+      <![CDATA[Parses a placement constraint specification. A placement constraint spec
+ is a composite expression which is composed by multiple sub constraint
+ expressions delimited by ":". With following syntax:
+
+ <p>Tag1(N1),P1:Tag2(N2),P2:...:TagN(Nn),Pn</p>
+
+ where <b>TagN(Nn)</b> is a key value pair to determine the source
+ allocation tag and the number of allocations, such as:
+
+ <p>foo(3)</p>
+
+ Optional when using NodeAttribute Constraint.
+
+ and where <b>Pn</b> can be any form of a valid constraint expression,
+ such as:
+
+ <ul>
+   <li>in,node,foo,bar</li>
+   <li>notin,node,foo,bar,1,2</li>
+   <li>and(notin,node,foo:notin,node,bar)</li>
+ </ul>
+
+ and NodeAttribute Constraint such as
+
+ <ul>
+   <li>yarn.rm.io/foo=true</li>
+   <li>java=1.7,1.8</li>
+ </ul>
+ @param expression expression string.
+ @return a map of source tags to placement constraint mapping.
+ @throws PlacementConstraintParseException]]>
+      </doc>
+    </method>
+    <field name="EXPRESSION_VAL_DELIM" type="char"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Placement constraint expression parser.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.constraint.PlacementConstraintParser -->
+</package>
+<package name="org.apache.hadoop.yarn.util.csi">
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.10.2.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.10.2.xml
new file mode 100644
index 0000000000000..9dea8f4450715
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.10.2.xml
@@ -0,0 +1,2832 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue May 24 22:59:29 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Client 2.10.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.10.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.5/commons-io-2.5.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.10.2.jar:/maven/com/nimbusds/nimbus-jose-jwt/7.9/nimbus-jose-jwt-7.9.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.3/json-smart-1.3.3.jar:/maven/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.10.2.jar:/usr/lib/jvm/zulu-7-amd64/lib/tools.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.10.2.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.10.2.jar:/maven/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/maven/com/google/inject/guice/3.0/guice-3.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/site/jdiff/xml -apiname Apache Hadoop YARN Client 2.10.2 -->
+<package name="org.apache.hadoop.yarn.client">
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <class name="AHSClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSClient" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AHSClient.]]>
+      </doc>
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a report of the given Application.
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+   <li>host - set to "N/A"</li>
+   <li>RPC port - set to -1</li>
+   <li>client token - set to "N/A"</li>
+   <li>diagnostics - set to "N/A"</li>
+   <li>tracking URL - set to "N/A"</li>
+   <li>original tracking URL - set to "N/A"</li>
+   <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+ 
+ @return a list of reports for all applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId
+ @return a list of reports for all application attempts for specified
+         application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId
+ @return a list of reports of all containers for specified application
+         attempt
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <class name="AMRMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClient" return="org.apache.hadoop.yarn.client.api.AMRMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AMRMClient.
+ For usage:
+ <pre>
+ {@code
+ AMRMClient.<T>createAMRMClientContainerRequest()
+ }</pre>
+ @return the newly create AMRMClient instance.]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any 
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progressIndicator" type="float"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request additional containers and receive new container allocations.
+ Requests made via <code>addContainerRequest</code> are sent to the
+ <code>ResourceManager</code>. New containers assigned to the master are
+ retrieved. Status of completed containers and node health updates are also
+ retrieved. This also doubles up as a heartbeat to the ResourceManager and
+ must be made periodically. The call may not always return any new
+ allocations of containers. App should not make concurrent allocate
+ requests. May cause request loss.
+ 
+ <p>
+ Note : If the user has not removed container requests that have already
+ been satisfied, then the re-register may end up sending the entire
+ container requests to the RM (including matched requests). Which would mean
+ the RM could end up giving it a lot of new allocated containers.
+ </p>
+ 
+ @param progressIndicator Indicates progress made by the master
+ @return the response of the allocate request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given 
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating 
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical 
+ <code>Resource</code> size that fit in the given capability. In a 
+ collection, requests will be returned in the same order as they were added.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ <code>Resource</code> size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+ specify an <code>ExecutionType</code>.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @param priority Priority
+ @param resourceName Location
+ @param executionType ExecutionType
+ @param capability Capability
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ allocationRequestId. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITH the allocationRequestId being set to a non-default value.
+
+ @param allocationRequestId Allocation Request Id
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+ 
+ @param blacklistAdditions list of resources which should be added to the 
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the 
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ be shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>.
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache.]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <doc>
+      <![CDATA[Register TimelineV2Client to AMRMClient. Writer's address for the timeline
+ V2 client will be updated dynamically if registered.
+
+ @param client the timeline v2 client to register]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline v2 client.
+ @return the registered timeline v2 client]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(com.google.common.base.Supplier, int)}
+ and {@link #waitFor(com.google.common.base.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(com.google.common.base.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMClient -->
+  <class name="NMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="startContainer" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Start an allocated container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the allocated container, including the
+ Id, the assigned node's Id and the token via {@link Container}. In
+ addition, the AM needs to provide the {@link ContainerLaunchContext} as
+ well.</p>
+
+ @param container the allocated container
+ @param containerLaunchContext the context information needed by the
+                               <code>NodeManager</code> to launch the
+                               container
+ @return a map between the auxiliary service names and their outputs
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="increaseContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Increase the resource of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="updateContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Stop an started container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="getContainerStatus" return="org.apache.hadoop.yarn.api.records.ContainerStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Query the status of a container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+ 
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="restartContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="cleanupRunningContainersOnStop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Set whether the containers that are started by this client, and are
+ still running should be stopped when the client stops. By default, the
+ feature should be enabled.</p> However, containers will be stopped only  
+ when service is stopped. i.e. after {@link NMClient#stop()}. 
+
+ @param enabled whether the feature is enabled or not]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache]]>
+      </doc>
+    </method>
+    <method name="getNodeIdOfStartedContainer" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Get the NodeId of the node on which container is running. It returns
+ null if the container if container is not found or if it is not running.
+
+ @param containerId Container Id of the container.
+ @return NodeId of the container on which it is running.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <class name="NMTokenCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a NM token cache instance.]]>
+      </doc>
+    </constructor>
+    <method name="getSingleton" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the singleton NM token cache.
+
+ @return the singleton NM token cache.]]>
+      </doc>
+    </method>
+    <method name="getNMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent. Only the singleton obtained from
+ {@link #getSingleton()} is looked at for the tokens. If you are using your
+ own NMTokenCache that is different from the singleton, use
+ {@link #getToken(String) }
+ 
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node manager]]>
+      </doc>
+    </method>
+    <method name="setNMToken"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address only in the singleton obtained from
+ {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ different from the singleton, use {@link #setToken(String, Token) }
+ 
+ @param nodeAddr
+          node address (host:port)
+ @param token
+          NMToken]]>
+      </doc>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node
+         manager]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address
+ @param nodeAddr node address (host:port)
+ @param token NMToken]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NMTokenCache manages NMTokens required for an Application Master
+ communicating with individual NodeManagers.
+ <p>
+ By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use
+ {@link #getSingleton()} instance of the cache.
+ <ul>
+   <li>
+     Using the singleton instance of the cache is appropriate when running a
+     single ApplicationMaster in the same JVM.
+   </li>
+   <li>
+     When using the singleton, users don't need to do anything special,
+     {@link AMRMClient} and {@link NMClient} are already set up to use the
+     default singleton {@link NMTokenCache}
+     </li>
+ </ul>
+ If running multiple Application Masters in the same JVM, a different cache
+ instance should be used for each Application Master.
+ <ul>
+   <li>
+     If using the {@link AMRMClient} and the {@link NMClient}, setting up
+     and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using the {@link AMRMClientAsync} and the {@link NMClientAsync},
+     setting up and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using {@link ApplicationMasterProtocol} and
+     {@link ContainerManagementProtocol} directly, setting up and using an
+     instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+ </pre>
+   </li>
+ </ul>
+ It is also possible to mix the usage of a client ({@code AMRMClient} or
+ {@code NMClient}, or the async versions of them) with a protocol proxy
+ ({@code ContainerManagementProtocolProxy} or
+ {@code ApplicationMasterProtocol}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <!-- start class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <class name="SharedCacheClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createSharedCacheClient" return="org.apache.hadoop.yarn.client.api.SharedCacheClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="use" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to claim a resource with the <code>SharedCacheManager.</code>
+ The client uses a checksum to identify the resource and an
+ {@link ApplicationId} to identify which application will be using the
+ resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>URL</code> to
+ the resource in the shared cache is returned. If the resource does not
+ exist, null is returned instead.
+ </p>
+
+ <p>
+ Once a URL has been returned for a resource, that URL is safe to use for
+ the lifetime of the application that corresponds to the provided
+ ApplicationId.
+ </p>
+
+ @param applicationId ApplicationId of the application using the resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource
+ @return URL to the resource, or null if it does not exist]]>
+      </doc>
+    </method>
+    <method name="release"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to release a resource with the <code>SharedCacheManager.</code>
+ This method is called once an application is no longer using a claimed
+ resource in the shared cache. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application is
+ releasing the resource.
+ </p>
+ 
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+ 
+ @param applicationId ApplicationId of the application releasing the
+          resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sourceFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A convenience method to calculate the checksum of a specified file.
+ 
+ @param sourceFile A path to the input file
+ @return A hex string containing the checksum digest
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the client for YARN's shared cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <class name="YarnClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of YarnClient.]]>
+      </doc>
+    </method>
+    <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link YarnClientApplication} for a new application,
+ which in turn contains the {@link ApplicationSubmissionContext} and
+ {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
+ objects.
+ </p>
+
+ @return {@link YarnClientApplication} built for a new application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Submit a new application to <code>YARN.</code> It is a blocking call - it
+ will not return {@link ApplicationId} until the submitted application is
+ submitted successfully and accepted by the ResourceManager.
+ </p>
+ 
+ <p>
+ Users should provide an {@link ApplicationId} as part of the parameter
+ {@link ApplicationSubmissionContext} when submitting a new application,
+ otherwise it will throw the {@link ApplicationIdNotProvidedException}.
+ </p>
+
+ <p>This internally calls {@link ApplicationClientProtocol#submitApplication
+ (SubmitApplicationRequest)}, and after that, it internally invokes
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} and waits till it can make sure that the
+ application gets properly submitted. If RM fails over or RM restart
+ happens before ResourceManager saves the application's state,
+ {@link ApplicationClientProtocol
+ #getApplicationReport(GetApplicationReportRequest)} will throw
+ the {@link ApplicationNotFoundException}. This API automatically resubmits
+ the application with the same {@link ApplicationSubmissionContext} when it
+ catches the {@link ApplicationNotFoundException}</p>
+
+ @param appContext
+          {@link ApplicationSubmissionContext} containing all the details
+          needed to submit a new application
+ @return {@link ApplicationId} of the accepted application
+ @throws YarnException
+ @throws IOException
+ @see #createApplication()]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Fail an application attempt identified by given ID.
+ </p>
+
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the attempt to fail.
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ 
+ @param applicationId
+          {@link ApplicationId} of the application that needs to be killed
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ @param applicationId {@link ApplicationId} of the application that needs to
+          be killed
+ @param diagnostics for killing an application.
+ @throws YarnException in case of errors or if YARN rejects the request due
+           to access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Application.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+ <li>host - set to "N/A"</li>
+ <li>RPC port - set to -1</li>
+ <li>client token - set to "N/A"</li>
+ <li>diagnostics - set to "N/A"</li>
+ <li>tracking URL - set to "N/A"</li>
+ <li>original tracking URL - set to "N/A"</li>
+ <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters Yarn takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+
+ @param appId {@link ApplicationId} of the application to get the AMRM token
+ @return the AMRM token if available
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @return a list of reports of all running applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications
+ matching the given application types in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types and application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types, application states and application tags in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @param applicationTags set of application tags you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queues" type="java.util.Set"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given users,
+ queues, application types and application states in the cluster. If any of
+ the params is set to null, it is not used when filtering.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param queues set of queues you are interested in
+ @param users set of users you are interested in
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a list of ApplicationReports that match the given
+ {@link GetApplicationsRequest}.
+</p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param request the request object to get the list of applications.
+ @return The list of ApplicationReports that match the request
+ @throws YarnException Exception specific to YARN.
+ @throws IOException Exception mostly related to connection errors.]]>
+      </doc>
+    </method>
+    <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get metrics ({@link YarnClusterMetrics}) about the cluster.
+ </p>
+ 
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of nodes ({@link NodeReport}) in the cluster.
+ </p>
+ 
+ @param states The {@link NodeState}s to filter on. If no filter states are
+          given, nodes in all states will be returned.
+ @return A list of node reports
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to YARN using those tokens.
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to YARN.
+ @return a delegation token ({@link Token}) that can be used to
+         talk to YARN
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about a given <em>queue</em>.
+ </p>
+ 
+ @param queueName
+          Name of the queue whose information is needed
+ @return queue information
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all queues, recursively if there
+ is a hierarchy
+ </p>
+ 
+ @return a list of queue-information for all queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about top level queues.
+ </p>
+ 
+ @return a list of queue-information for all the top-level queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all the immediate children queues
+ of the given queue
+ </p>
+ 
+ @param parent
+          Name of the queue whose child-queues' information is needed
+ @return a list of queue-information for all queues who are direct children
+         of the given parent queue.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information about <em>acls</em> for <em>current user</em> on all the
+ existing queues.
+ </p>
+ 
+ @return a list of queue acls ({@link QueueUserACLInfo}) for
+         <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId application id of the app
+ @return a list of reports for all application attempts for specified
+         application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId application attempt id
+ @return a list of reports of all containers for specified application
+         attempts
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Attempts to move the given application to the given queue.
+ </p>
+ 
+ @param appId
+    Application to move.
+ @param queue
+    Queue to place it in to.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link GetNewReservationResponse} for a new reservation,
+ which contains the {@link ReservationId} object.
+ </p>
+
+ @return The {@link GetNewReservationResponse} containing a new
+         {@link ReservationId} object.
+ @throws YarnException if reservation cannot be created.
+ @throws IOException if reservation cannot be created.]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and gang needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationRequest is satisfiable the {@code ResourceManager}
+ answers with a {@link ReservationSubmissionResponse} that includes a
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the message detailing the reason of failure.
+ </p>
+ 
+ <p>
+ The semantics guarantees that the {@link ReservationId} returned,
+ corresponds to a valid reservation existing in the time-range request by
+ the user. The amount of capacity dedicated to such reservation can vary
+ overtime, depending of the allocation that has been determined. But it is
+ guaranteed to satisfy all the constraint expressed by the user in the
+ {@link ReservationDefinition}
+ </p>
+ 
+ @param request request to submit a new Reservation
+ @return response contains the {@link ReservationId} on accepting the
+         submission
+ @throws YarnException if the reservation cannot be created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationDefinition}. Upon success the previous allocation is
+ atomically substituted by the new one, and on failure (i.e., if the system
+ cannot find a valid allocation for the updated request), the previous
+ allocation remains valid.
+ </p>
+ 
+ @param request to update an existing Reservation (the
+          {@link ReservationUpdateRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the
+          {@link ReservationDeleteRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@link ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException if the request failed otherwise]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+ 
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ in existing cluster
+ </p>
+
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ for specified labels in existing cluster
+ </p>
+
+ @param labels labels for which labels to nodes mapping has to be retrieved
+ @return labels to nodes mappings for specific labels
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @return cluster node labels collection
+ @throws YarnException when there is a failure in
+           {@link ApplicationClientProtocol}
+ @throws IOException when there is a failure in
+           {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application
+ </p>
+ @param applicationId
+ @param priority
+ @return updated priority of an application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Signal a container identified by given ID.
+ </p>
+
+ @param containerId
+          {@link ContainerId} of the container that needs to be signaled
+ @param command the signal container command
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get available resource types supported by RM.
+ </p>
+ @return list of supported resource types with detailed information
+ @throws YarnException if any issue happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+  <class name="YarnClientApplication" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClientApplication" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse, org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNewApplicationResponse" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async">
+  <!-- start class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <class name="AMRMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param client the AMRMClient instance
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="setHeartbeatInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interval" type="int"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given Priority,
+ ResourceName, ExecutionType and Capability.
+
+ NOTE: This matches only requests that were made by the client WITHOUT the
+ allocationRequestId specified.
+
+ @param priority Priority.
+ @param resourceName Location.
+ @param executionType ExecutionType.
+ @param capability Capability.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given
+ AllocationRequestId.
+
+ NOTE: This matches only requests that were made by the client WITH the
+ allocationRequestId specified.
+
+ @param allocationRequestId AllocationRequestId.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this application master with the resource manager. On successful
+ registration, starts the heartbeating thread.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineClient" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Register TimelineClient to AMRMClient.
+ @param timelineClient
+ @throws YarnException when this method is invoked even when ATS V2 is not
+           configured.]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline client.
+ @return the registered timeline client]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+
+ @param blacklistAdditions list of resources which should be added to the
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(com.google.common.base.Supplier, int)}
+ and {@link #waitFor(com.google.common.base.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(com.google.common.base.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="com.google.common.base.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="handler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="heartbeatIntervalMs" type="java.util.concurrent.atomic.AtomicInteger"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>AMRMClientAsync</code> handles communication with the ResourceManager
+ and provides asynchronous updates on events such as container allocations and
+ completions.  It contains a thread that sends periodic heartbeats to the
+ ResourceManager.
+ 
+ It should be used by implementing a CallbackHandler:
+ <pre>
+ {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List<Container> containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List<Container> containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List<ContainerStatus> statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List<NodeReport> updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+ </pre>
+ 
+ The client's lifecycle should be managed similarly to the following:
+ 
+ <pre>
+ {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <!-- start class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+  <class name="NMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"/>
+    </method>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="startContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+    </method>
+    <method name="increaseContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+    </method>
+    <method name="updateContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?]]>
+      </doc>
+    </method>
+    <method name="restartContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.]]>
+      </doc>
+    </method>
+    <method name="stopContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getContainerStatusAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.NMClient"/>
+    </method>
+    <method name="getCallbackHandler" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.NMClient"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>NMClientAsync</code> handles communication with all the NodeManagers
+ and provides asynchronous updates on getting responses from them. It
+ maintains a thread pool to communicate with individual NMs where a number of
+ worker threads process requests to NMs by using {@link NMClientImpl}. The max
+ size of the thread pool is configurable through
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}.
+
+ It should be used in conjunction with a CallbackHandler. For example
+
+ <pre>
+ {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map<String, ByteBuffer> allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+ </pre>
+
+ The client's life-cycle should be managed like the following:
+
+ <pre>
+ {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.cli">
+  <!-- start class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <class name="LogsCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="LogsCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getAMContainerInfoForRMWebService" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="ClientHandlerException" type="com.sun.jersey.api.client.ClientHandlerException"/>
+      <exception name="UniformInterfaceException" type="com.sun.jersey.api.client.UniformInterfaceException"/>
+      <exception name="JSONException" type="org.codehaus.jettison.json.JSONException"/>
+    </method>
+    <method name="getMatchedContainerLogFiles" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="useRegex" type="boolean"/>
+      <param name="ignoreSizeLimit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResponeFromNMWebService" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="webServiceClient" type="com.sun.jersey.api.client.Client"/>
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="logFile" type="java.lang.String"/>
+    </method>
+    <method name="getNodeHttpAddressFromRMWebString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="ClientHandlerException" type="com.sun.jersey.api.client.ClientHandlerException"/>
+      <exception name="UniformInterfaceException" type="com.sun.jersey.api.client.UniformInterfaceException"/>
+      <exception name="JSONException" type="org.codehaus.jettison.json.JSONException"/>
+    </method>
+    <field name="HELP_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <!-- start class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+  <class name="SchedConfCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="SchedConfCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[CLI for modifying scheduler configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+</package>
+<package name="org.apache.hadoop.yarn.client.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml
new file mode 100644
index 0000000000000..b8f2a2e529c2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml
@@ -0,0 +1,3006 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:56:07 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Client 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.2.4.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.4.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/site/jdiff/xml -apiname Apache Hadoop YARN Client 3.2.4 -->
+<package name="org.apache.hadoop.yarn.client">
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <class name="AHSClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSClient" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AHSClient.]]>
+      </doc>
+    </method>
+    <method name="createAHSv2Client" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a report of the given Application.
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+   <li>host - set to "N/A"</li>
+   <li>RPC port - set to -1</li>
+   <li>client token - set to "N/A"</li>
+   <li>diagnostics - set to "N/A"</li>
+   <li>tracking URL - set to "N/A"</li>
+   <li>original tracking URL - set to "N/A"</li>
+   <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+ 
+ @return a list of reports for all applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId
+ @return a list of reports for all application attempts for specified
+         application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId
+ @return a list of reports of all containers for specified application
+         attempt
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <class name="AMRMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClient" return="org.apache.hadoop.yarn.client.api.AMRMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AMRMClient.
+ For usage:
+ <pre>
+ {@code
+ AMRMClient.<T>createAMRMClientContainerRequest()
+ }</pre>
+ @return the newly create AMRMClient instance.]]>
+      </doc>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any 
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progressIndicator" type="float"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request additional containers and receive new container allocations.
+ Requests made via <code>addContainerRequest</code> are sent to the
+ <code>ResourceManager</code>. New containers assigned to the master are
+ retrieved. Status of completed containers and node health updates are also
+ retrieved. This also doubles up as a heartbeat to the ResourceManager and
+ must be made periodically. The call may not always return any new
+ allocations of containers. App should not make concurrent allocate
+ requests. May cause request loss.
+ 
+ <p>
+ Note : If the user has not removed container requests that have already
+ been satisfied, then the re-register may end up sending the entire
+ container requests to the RM (including matched requests). Which would mean
+ the RM could end up giving it a lot of new allocated containers.
+ </p>
+ 
+ @param progressIndicator Indicates progress made by the master
+ @return the response of the allocate request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given 
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating 
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical 
+ <code>Resource</code> size that fit in the given capability. In a 
+ collection, requests will be returned in the same order as they were added.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ <code>Resource</code> size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+ specify an <code>ExecutionType</code>.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @param priority Priority
+ @param resourceName Location
+ @param executionType ExecutionType
+ @param capability Capability
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ allocationRequestId. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITH the allocationRequestId being set to a non-default value.
+
+ @param allocationRequestId Allocation Request Id
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+ 
+ @param blacklistAdditions list of resources which should be added to the 
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the 
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ be shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>.
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache.]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <doc>
+      <![CDATA[Register TimelineV2Client to AMRMClient. Writer's address for the timeline
+ V2 client will be updated dynamically if registered.
+
+ @param client the timeline v2 client to register]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline v2 client.
+ @return the registered timeline v2 client]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMClient -->
+  <class name="NMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="startContainer" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Start an allocated container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the allocated container, including the
+ Id, the assigned node's Id and the token via {@link Container}. In
+ addition, the AM needs to provide the {@link ContainerLaunchContext} as
+ well.</p>
+
+ @param container the allocated container
+ @param containerLaunchContext the context information needed by the
+                               <code>NodeManager</code> to launch the
+                               container
+ @return a map between the auxiliary service names and their outputs
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="increaseContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Increase the resource of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="updateContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Stop an started container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="getContainerStatus" return="org.apache.hadoop.yarn.api.records.ContainerStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Query the status of a container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+ 
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="restartContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="cleanupRunningContainersOnStop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Set whether the containers that are started by this client, and are
+ still running should be stopped when the client stops. By default, the
+ feature should be enabled.</p> However, containers will be stopped only  
+ when service is stopped. i.e. after {@link NMClient#stop()}. 
+
+ @param enabled whether the feature is enabled or not]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache]]>
+      </doc>
+    </method>
+    <method name="getNodeIdOfStartedContainer" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Get the NodeId of the node on which container is running. It returns
+ null if the container if container is not found or if it is not running.
+
+ @param containerId Container Id of the container.
+ @return NodeId of the container on which it is running.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <class name="NMTokenCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a NM token cache instance.]]>
+      </doc>
+    </constructor>
+    <method name="getSingleton" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the singleton NM token cache.
+
+ @return the singleton NM token cache.]]>
+      </doc>
+    </method>
+    <method name="getNMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent. Only the singleton obtained from
+ {@link #getSingleton()} is looked at for the tokens. If you are using your
+ own NMTokenCache that is different from the singleton, use
+ {@link #getToken(String) }
+ 
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node manager]]>
+      </doc>
+    </method>
+    <method name="setNMToken"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address only in the singleton obtained from
+ {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ different from the singleton, use {@link #setToken(String, Token) }
+ 
+ @param nodeAddr
+          node address (host:port)
+ @param token
+          NMToken]]>
+      </doc>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node
+         manager]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address
+ @param nodeAddr node address (host:port)
+ @param token NMToken]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NMTokenCache manages NMTokens required for an Application Master
+ communicating with individual NodeManagers.
+ <p>
+ By default YARN client libraries {@link AMRMClient} and {@link NMClient} use
+ {@link #getSingleton()} instance of the cache.
+ <ul>
+   <li>
+     Using the singleton instance of the cache is appropriate when running a
+     single ApplicationMaster in the same JVM.
+   </li>
+   <li>
+     When using the singleton, users don't need to do anything special,
+     {@link AMRMClient} and {@link NMClient} are already set up to use the
+     default singleton {@link NMTokenCache}
+     </li>
+ </ul>
+ If running multiple Application Masters in the same JVM, a different cache
+ instance should be used for each Application Master.
+ <ul>
+   <li>
+     If using the {@link AMRMClient} and the {@link NMClient}, setting up
+     and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using the {@link AMRMClientAsync} and the {@link NMClientAsync},
+     setting up and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using {@link ApplicationMasterProtocol} and
+     {@link ContainerManagementProtocol} directly, setting up and using an
+     instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+ </pre>
+   </li>
+ </ul>
+ It is also possible to mix the usage of a client ({@code AMRMClient} or
+ {@code NMClient}, or the async versions of them) with a protocol proxy
+ ({@code ContainerManagementProtocolProxy} or
+ {@code ApplicationMasterProtocol}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <!-- start class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <class name="SharedCacheClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createSharedCacheClient" return="org.apache.hadoop.yarn.client.api.SharedCacheClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="use" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to claim a resource with the <code>SharedCacheManager.</code>
+ The client uses a checksum to identify the resource and an
+ {@link ApplicationId} to identify which application will be using the
+ resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>URL</code> to
+ the resource in the shared cache is returned. If the resource does not
+ exist, null is returned instead.
+ </p>
+
+ <p>
+ Once a URL has been returned for a resource, that URL is safe to use for
+ the lifetime of the application that corresponds to the provided
+ ApplicationId.
+ </p>
+
+ @param applicationId ApplicationId of the application using the resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource
+ @return URL to the resource, or null if it does not exist]]>
+      </doc>
+    </method>
+    <method name="release"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to release a resource with the <code>SharedCacheManager.</code>
+ This method is called once an application is no longer using a claimed
+ resource in the shared cache. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application is
+ releasing the resource.
+ </p>
+ 
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+ 
+ @param applicationId ApplicationId of the application releasing the
+          resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sourceFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A convenience method to calculate the checksum of a specified file.
+ 
+ @param sourceFile A path to the input file
+ @return A hex string containing the checksum digest
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the client for YARN's shared cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <class name="YarnClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of YarnClient.]]>
+      </doc>
+    </method>
+    <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link YarnClientApplication} for a new application,
+ which in turn contains the {@link ApplicationSubmissionContext} and
+ {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
+ objects.
+ </p>
+
+ @return {@link YarnClientApplication} built for a new application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Submit a new application to <code>YARN.</code> It is a blocking call - it
+ will not return {@link ApplicationId} until the submitted application is
+ submitted successfully and accepted by the ResourceManager.
+ </p>
+ 
+ <p>
+ Users should provide an {@link ApplicationId} as part of the parameter
+ {@link ApplicationSubmissionContext} when submitting a new application,
+ otherwise it will throw the {@link ApplicationIdNotProvidedException}.
+ </p>
+
+ <p>This internally calls {@link ApplicationClientProtocol#submitApplication
+ (SubmitApplicationRequest)}, and after that, it internally invokes
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} and waits till it can make sure that the
+ application gets properly submitted. If RM fails over or RM restart
+ happens before ResourceManager saves the application's state,
+ {@link ApplicationClientProtocol
+ #getApplicationReport(GetApplicationReportRequest)} will throw
+ the {@link ApplicationNotFoundException}. This API automatically resubmits
+ the application with the same {@link ApplicationSubmissionContext} when it
+ catches the {@link ApplicationNotFoundException}</p>
+
+ @param appContext
+          {@link ApplicationSubmissionContext} containing all the details
+          needed to submit a new application
+ @return {@link ApplicationId} of the accepted application
+ @throws YarnException
+ @throws IOException
+ @see #createApplication()]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Fail an application attempt identified by given ID.
+ </p>
+
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the attempt to fail.
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ 
+ @param applicationId
+          {@link ApplicationId} of the application that needs to be killed
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ @param applicationId {@link ApplicationId} of the application that needs to
+          be killed
+ @param diagnostics for killing an application.
+ @throws YarnException in case of errors or if YARN rejects the request due
+           to access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Application.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+ <li>host - set to "N/A"</li>
+ <li>RPC port - set to -1</li>
+ <li>client token - set to "N/A"</li>
+ <li>diagnostics - set to "N/A"</li>
+ <li>tracking URL - set to "N/A"</li>
+ <li>original tracking URL - set to "N/A"</li>
+ <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+
+ @param appId {@link ApplicationId} of the application to get the AMRM token
+ @return the AMRM token if available
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @return a list of reports of all running applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications
+ matching the given application types in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types and application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types, application states and application tags in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @param applicationTags set of application tags you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queues" type="java.util.Set"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given users,
+ queues, application types and application states in the cluster. If any of
+ the params is set to null, it is not used when filtering.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param queues set of queues you are interested in
+ @param users set of users you are interested in
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a list of ApplicationReports that match the given
+ {@link GetApplicationsRequest}.
+</p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param request the request object to get the list of applications.
+ @return The list of ApplicationReports that match the request
+ @throws YarnException Exception specific to YARN.
+ @throws IOException Exception mostly related to connection errors.]]>
+      </doc>
+    </method>
+    <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get metrics ({@link YarnClusterMetrics}) about the cluster.
+ </p>
+ 
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of nodes ({@link NodeReport}) in the cluster.
+ </p>
+ 
+ @param states The {@link NodeState}s to filter on. If no filter states are
+          given, nodes in all states will be returned.
+ @return A list of node reports
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to YARN using those tokens.
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to YARN.
+ @return a delegation token ({@link Token}) that can be used to
+         talk to YARN
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about a given <em>queue</em>.
+ </p>
+ 
+ @param queueName
+          Name of the queue whose information is needed
+ @return queue information
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all queues, recursively if there
+ is a hierarchy
+ </p>
+ 
+ @return a list of queue-information for all queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about top level queues.
+ </p>
+ 
+ @return a list of queue-information for all the top-level queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all the immediate children queues
+ of the given queue
+ </p>
+ 
+ @param parent
+          Name of the queue whose child-queues' information is needed
+ @return a list of queue-information for all queues who are direct children
+         of the given parent queue.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information about <em>acls</em> for <em>current user</em> on all the
+ existing queues.
+ </p>
+ 
+ @return a list of queue acls ({@link QueueUserACLInfo}) for
+         <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId application id of the app
+ @return a list of reports for all application attempts for specified
+         application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId application attempt id
+ @return a list of reports of all containers for specified application
+         attempts
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Attempts to move the given application to the given queue.
+ </p>
+ 
+ @param appId
+    Application to move.
+ @param queue
+    Queue to place it in to.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link GetNewReservationResponse} for a new reservation,
+ which contains the {@link ReservationId} object.
+ </p>
+
+ @return The {@link GetNewReservationResponse} containing a new
+         {@link ReservationId} object.
+ @throws YarnException if reservation cannot be created.
+ @throws IOException if reservation cannot be created.]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and gang needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationRequest is satisfiable the {@code ResourceManager}
+ answers with a {@link ReservationSubmissionResponse} that includes a
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the message detailing the reason of failure.
+ </p>
+ 
+ <p>
+ The semantics guarantees that the {@link ReservationId} returned,
+ corresponds to a valid reservation existing in the time-range request by
+ the user. The amount of capacity dedicated to such reservation can vary
+ overtime, depending of the allocation that has been determined. But it is
+ guaranteed to satisfy all the constraint expressed by the user in the
+ {@link ReservationDefinition}
+ </p>
+ 
+ @param request request to submit a new Reservation
+ @return response contains the {@link ReservationId} on accepting the
+         submission
+ @throws YarnException if the reservation cannot be created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationDefinition}. Upon success the previous allocation is
+ atomically substituted by the new one, and on failure (i.e., if the system
+ cannot find a valid allocation for the updated request), the previous
+ allocation remains valid.
+ </p>
+ 
+ @param request to update an existing Reservation (the
+          {@link ReservationUpdateRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the
+          {@link ReservationDeleteRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@link ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException if the request failed otherwise]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+ 
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ in existing cluster
+ </p>
+
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ for specified labels in existing cluster
+ </p>
+
+ @param labels labels for which labels to nodes mapping has to be retrieved
+ @return labels to nodes mappings for specific labels
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @return cluster node labels collection
+ @throws YarnException when there is a failure in
+           {@link ApplicationClientProtocol}
+ @throws IOException when there is a failure in
+           {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application
+ </p>
+ @param applicationId
+ @param priority
+ @return updated priority of an application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Signal a container identified by given ID.
+ </p>
+
+ @param containerId
+          {@link ContainerId} of the container that needs to be signaled
+ @param command the signal container command
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the resource profiles available in the RM.
+ </p>
+ @return a Map of the resource profile names to their capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the details of a specific resource profile from the RM.
+ </p>
+ @param profile the profile name
+ @return resource profile name with its capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get available resource types supported by RM.
+ </p>
+ @return list of supported resource types with detailed information
+ @throws YarnException if any issue happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getClusterAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes in the cluster.
+ </p>
+
+ @return cluster node attributes collection
+ @throws YarnException when there is a failure in
+                       {@link ApplicationClientProtocol}
+ @throws IOException   when there is a failure in
+                       {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get mapping of AttributeKey to associated
+ NodeToAttributeValue list for specified node attributeKeys in the cluster.
+ </p>
+
+ @param attributes AttributeKeys for which associated NodeToAttributeValue
+          mapping value has to be retrieved. If empty or null is set then
+          will return mapping for all attributeKeys in the cluster
+ @return mapping of AttributeKey to List of associated
+         NodeToAttributeValue's.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get all node to attribute mapping in
+ existing cluster.
+ </p>
+
+ @param hostNames HostNames for which host to attributes mapping has to
+                  be retrived.If empty or null is set then will return
+                  all nodes to attributes mapping in cluster.
+ @return Node to attribute mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+  <class name="YarnClientApplication" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClientApplication" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse, org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNewApplicationResponse" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async">
+  <!-- start class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <class name="AMRMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param client the AMRMClient instance
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="setHeartbeatInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interval" type="int"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given Priority,
+ ResourceName, ExecutionType and Capability.
+
+ NOTE: This matches only requests that were made by the client WITHOUT the
+ allocationRequestId specified.
+
+ @param priority Priority.
+ @param resourceName Location.
+ @param executionType ExecutionType.
+ @param capability Capability.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given
+ AllocationRequestId.
+
+ NOTE: This matches only requests that were made by the client WITH the
+ allocationRequestId specified.
+
+ @param allocationRequestId AllocationRequestId.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this application master with the resource manager. On successful
+ registration, starts the heartbeating thread.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineClient" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Register TimelineClient to AMRMClient.
+ @param timelineClient
+ @throws YarnException when this method is invoked even when ATS V2 is not
+           configured.]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline client.
+ @return the registered timeline client]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+
+ @param blacklistAdditions list of resources which should be added to the
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="handler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="heartbeatIntervalMs" type="java.util.concurrent.atomic.AtomicInteger"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>AMRMClientAsync</code> handles communication with the ResourceManager
+ and provides asynchronous updates on events such as container allocations and
+ completions.  It contains a thread that sends periodic heartbeats to the
+ ResourceManager.
+ 
+ It should be used by implementing a CallbackHandler:
+ <pre>
+ {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List<Container> containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List<Container> containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List<ContainerStatus> statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List<NodeReport> updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+ </pre>
+ 
+ The client's lifecycle should be managed similarly to the following:
+ 
+ <pre>
+ {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <!-- start class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+  <class name="NMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"/>
+    </method>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="startContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+    </method>
+    <method name="increaseContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+    </method>
+    <method name="updateContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?]]>
+      </doc>
+    </method>
+    <method name="restartContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.]]>
+      </doc>
+    </method>
+    <method name="stopContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getContainerStatusAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.NMClient"/>
+    </method>
+    <method name="getCallbackHandler" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.NMClient"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>NMClientAsync</code> handles communication with all the NodeManagers
+ and provides asynchronous updates on getting responses from them. It
+ maintains a thread pool to communicate with individual NMs where a number of
+ worker threads process requests to NMs by using {@link NMClientImpl}. The max
+ size of the thread pool is configurable through
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}.
+
+ It should be used in conjunction with a CallbackHandler. For example
+
+ <pre>
+ {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map<String, ByteBuffer> allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+ </pre>
+
+ The client's life-cycle should be managed like the following:
+
+ <pre>
+ {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.cli">
+  <!-- start class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <class name="LogsCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="LogsCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getAMContainerInfoForRMWebService" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getMatchedContainerLogFiles" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="useRegex" type="boolean"/>
+      <param name="ignoreSizeLimit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResponeFromNMWebService" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="webServiceClient" type="com.sun.jersey.api.client.Client"/>
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="logFile" type="java.lang.String"/>
+    </method>
+    <method name="getNodeHttpAddressFromRMWebString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="ClientHandlerException" type="com.sun.jersey.api.client.ClientHandlerException"/>
+      <exception name="UniformInterfaceException" type="com.sun.jersey.api.client.UniformInterfaceException"/>
+      <exception name="JSONException" type="org.codehaus.jettison.json.JSONException"/>
+    </method>
+    <field name="HELP_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <!-- start class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+  <class name="SchedConfCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="SchedConfCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[CLI for modifying scheduler configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+</package>
+<package name="org.apache.hadoop.yarn.client.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.3.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.3.xml
new file mode 100644
index 0000000000000..745b4c6a2730e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.3.xml
@@ -0,0 +1,3067 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:12:09 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Client 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.3.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/site/jdiff/xml -apiname Apache Hadoop YARN Client 3.3.3 -->
+<package name="org.apache.hadoop.yarn.client">
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <class name="AHSClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSClient" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AHSClient.]]>
+      </doc>
+    </method>
+    <method name="createAHSv2Client" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a report of the given Application.
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+   <li>host - set to "N/A"</li>
+   <li>RPC port - set to -1</li>
+   <li>client token - set to "N/A"</li>
+   <li>diagnostics - set to "N/A"</li>
+   <li>tracking URL - set to "N/A"</li>
+   <li>original tracking URL - set to "N/A"</li>
+   <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+ 
+ @return a list of reports for all applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId
+ @return a list of reports for all application attempts for specified
+         application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId
+ @return a list of reports of all containers for specified application
+         attempt
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <class name="AMRMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClient" return="org.apache.hadoop.yarn.client.api.AMRMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AMRMClient.
+ For usage:
+ <pre>
+ {@code
+ AMRMClient.<T>createAMRMClientContainerRequest()
+ }</pre>
+ @return the newly create AMRMClient instance.]]>
+      </doc>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any 
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progressIndicator" type="float"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request additional containers and receive new container allocations.
+ Requests made via <code>addContainerRequest</code> are sent to the
+ <code>ResourceManager</code>. New containers assigned to the master are
+ retrieved. Status of completed containers and node health updates are also
+ retrieved. This also doubles up as a heartbeat to the ResourceManager and
+ must be made periodically. The call may not always return any new
+ allocations of containers. App should not make concurrent allocate
+ requests. May cause request loss.
+ 
+ <p>
+ Note : If the user has not removed container requests that have already
+ been satisfied, then the re-register may end up sending the entire
+ container requests to the RM (including matched requests). Which would mean
+ the RM could end up giving it a lot of new allocated containers.
+ </p>
+ 
+ @param progressIndicator Indicates progress made by the master
+ @return the response of the allocate request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given 
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating 
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical 
+ <code>Resource</code> size that fit in the given capability. In a 
+ collection, requests will be returned in the same order as they were added.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ <code>Resource</code> size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+ specify an <code>ExecutionType</code>.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @param priority Priority
+ @param resourceName Location
+ @param executionType ExecutionType
+ @param capability Capability
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ allocationRequestId. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITH the allocationRequestId being set to a non-default value.
+
+ @param allocationRequestId Allocation Request Id
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+ 
+ @param blacklistAdditions list of resources which should be added to the 
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the 
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ be shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>.
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache.]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <doc>
+      <![CDATA[Register TimelineV2Client to AMRMClient. Writer's address for the timeline
+ V2 client will be updated dynamically if registered.
+
+ @param client the timeline v2 client to register]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline v2 client.
+ @return the registered timeline v2 client]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMClient -->
+  <class name="NMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="startContainer" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Start an allocated container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the allocated container, including the
+ Id, the assigned node's Id and the token via {@link Container}. In
+ addition, the AM needs to provide the {@link ContainerLaunchContext} as
+ well.</p>
+
+ @param container the allocated container
+ @param containerLaunchContext the context information needed by the
+                               <code>NodeManager</code> to launch the
+                               container
+ @return a map between the auxiliary service names and their outputs
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="increaseContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Increase the resource of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="updateContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Stop an started container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="getContainerStatus" return="org.apache.hadoop.yarn.api.records.ContainerStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Query the status of a container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+ 
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="restartContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="cleanupRunningContainersOnStop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Set whether the containers that are started by this client, and are
+ still running should be stopped when the client stops. By default, the
+ feature should be enabled.</p> However, containers will be stopped only  
+ when service is stopped. i.e. after {@link NMClient#stop()}. 
+
+ @param enabled whether the feature is enabled or not]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache]]>
+      </doc>
+    </method>
+    <method name="getNodeIdOfStartedContainer" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Get the NodeId of the node on which container is running. It returns
+ null if the container if container is not found or if it is not running.
+
+ @param containerId Container Id of the container.
+ @return NodeId of the container on which it is running.]]>
+      </doc>
+    </method>
+    <method name="localize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <param name="localResources" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources for a container.
+ @param containerId     the ID of the container
+ @param nodeId          node Id of the container
+ @param localResources  resources to localize]]>
+      </doc>
+    </method>
+    <method name="getLocalizationStatuses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the localization statuses of a container.
+
+ @param containerId   the Id of the container
+ @param nodeId        node Id of the container
+
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <class name="NMTokenCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a NM token cache instance.]]>
+      </doc>
+    </constructor>
+    <method name="getSingleton" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the singleton NM token cache.
+
+ @return the singleton NM token cache.]]>
+      </doc>
+    </method>
+    <method name="getNMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent. Only the singleton obtained from
+ {@link #getSingleton()} is looked at for the tokens. If you are using your
+ own NMTokenCache that is different from the singleton, use
+ {@link #getToken(String) }
+ 
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node manager]]>
+      </doc>
+    </method>
+    <method name="setNMToken"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address only in the singleton obtained from
+ {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ different from the singleton, use {@link #setToken(String, Token) }
+ 
+ @param nodeAddr
+          node address (host:port)
+ @param token
+          NMToken]]>
+      </doc>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node
+         manager]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address
+ @param nodeAddr node address (host:port)
+ @param token NMToken]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NMTokenCache manages NMTokens required for an Application Master
+ communicating with individual NodeManagers.
+ <p>
+ By default YARN client libraries {@link AMRMClient} and {@link NMClient} use
+ {@link #getSingleton()} instance of the cache.
+ <ul>
+   <li>
+     Using the singleton instance of the cache is appropriate when running a
+     single ApplicationMaster in the same JVM.
+   </li>
+   <li>
+     When using the singleton, users don't need to do anything special,
+     {@link AMRMClient} and {@link NMClient} are already set up to use the
+     default singleton {@link NMTokenCache}
+     </li>
+ </ul>
+ If running multiple Application Masters in the same JVM, a different cache
+ instance should be used for each Application Master.
+ <ul>
+   <li>
+     If using the {@link AMRMClient} and the {@link NMClient}, setting up
+     and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using the {@link AMRMClientAsync} and the {@link NMClientAsync},
+     setting up and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using {@link ApplicationMasterProtocol} and
+     {@link ContainerManagementProtocol} directly, setting up and using an
+     instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+ </pre>
+   </li>
+ </ul>
+ It is also possible to mix the usage of a client ({@code AMRMClient} or
+ {@code NMClient}, or the async versions of them) with a protocol proxy
+ ({@code ContainerManagementProtocolProxy} or
+ {@code ApplicationMasterProtocol}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <!-- start class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <class name="SharedCacheClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createSharedCacheClient" return="org.apache.hadoop.yarn.client.api.SharedCacheClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="use" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to claim a resource with the <code>SharedCacheManager.</code>
+ The client uses a checksum to identify the resource and an
+ {@link ApplicationId} to identify which application will be using the
+ resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>URL</code> to
+ the resource in the shared cache is returned. If the resource does not
+ exist, null is returned instead.
+ </p>
+
+ <p>
+ Once a URL has been returned for a resource, that URL is safe to use for
+ the lifetime of the application that corresponds to the provided
+ ApplicationId.
+ </p>
+
+ @param applicationId ApplicationId of the application using the resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource
+ @return URL to the resource, or null if it does not exist]]>
+      </doc>
+    </method>
+    <method name="release"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to release a resource with the <code>SharedCacheManager.</code>
+ This method is called once an application is no longer using a claimed
+ resource in the shared cache. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application is
+ releasing the resource.
+ </p>
+ 
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+ 
+ @param applicationId ApplicationId of the application releasing the
+          resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sourceFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A convenience method to calculate the checksum of a specified file.
+ 
+ @param sourceFile A path to the input file
+ @return A hex string containing the checksum digest
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the client for YARN's shared cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <class name="YarnClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of YarnClient.]]>
+      </doc>
+    </method>
+    <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link YarnClientApplication} for a new application,
+ which in turn contains the {@link ApplicationSubmissionContext} and
+ {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
+ objects.
+ </p>
+
+ @return {@link YarnClientApplication} built for a new application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Submit a new application to <code>YARN.</code> It is a blocking call - it
+ will not return {@link ApplicationId} until the submitted application is
+ submitted successfully and accepted by the ResourceManager.
+ </p>
+ 
+ <p>
+ Users should provide an {@link ApplicationId} as part of the parameter
+ {@link ApplicationSubmissionContext} when submitting a new application,
+ otherwise it will throw the {@link ApplicationIdNotProvidedException}.
+ </p>
+
+ <p>This internally calls {@link ApplicationClientProtocol#submitApplication
+ (SubmitApplicationRequest)}, and after that, it internally invokes
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} and waits till it can make sure that the
+ application gets properly submitted. If RM fails over or RM restart
+ happens before ResourceManager saves the application's state,
+ {@link ApplicationClientProtocol
+ #getApplicationReport(GetApplicationReportRequest)} will throw
+ the {@link ApplicationNotFoundException}. This API automatically resubmits
+ the application with the same {@link ApplicationSubmissionContext} when it
+ catches the {@link ApplicationNotFoundException}</p>
+
+ @param appContext
+          {@link ApplicationSubmissionContext} containing all the details
+          needed to submit a new application
+ @return {@link ApplicationId} of the accepted application
+ @throws YarnException
+ @throws IOException
+ @see #createApplication()]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Fail an application attempt identified by given ID.
+ </p>
+
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the attempt to fail.
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ 
+ @param applicationId
+          {@link ApplicationId} of the application that needs to be killed
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ @param applicationId {@link ApplicationId} of the application that needs to
+          be killed
+ @param diagnostics for killing an application.
+ @throws YarnException in case of errors or if YARN rejects the request due
+           to access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Application.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+ <li>host - set to "N/A"</li>
+ <li>RPC port - set to -1</li>
+ <li>client token - set to "N/A"</li>
+ <li>diagnostics - set to "N/A"</li>
+ <li>tracking URL - set to "N/A"</li>
+ <li>original tracking URL - set to "N/A"</li>
+ <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+
+ @param appId {@link ApplicationId} of the application to get the AMRM token
+ @return the AMRM token if available
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @return a list of reports of all running applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications
+ matching the given application types in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types and application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types, application states and application tags in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @param applicationTags set of application tags you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queues" type="java.util.Set"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given users,
+ queues, application types and application states in the cluster. If any of
+ the params is set to null, it is not used when filtering.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param queues set of queues you are interested in
+ @param users set of users you are interested in
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a list of ApplicationReports that match the given
+ {@link GetApplicationsRequest}.
+</p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param request the request object to get the list of applications.
+ @return The list of ApplicationReports that match the request
+ @throws YarnException Exception specific to YARN.
+ @throws IOException Exception mostly related to connection errors.]]>
+      </doc>
+    </method>
+    <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get metrics ({@link YarnClusterMetrics}) about the cluster.
+ </p>
+ 
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of nodes ({@link NodeReport}) in the cluster.
+ </p>
+ 
+ @param states The {@link NodeState}s to filter on. If no filter states are
+          given, nodes in all states will be returned.
+ @return A list of node reports
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to YARN using those tokens.
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to YARN.
+ @return a delegation token ({@link Token}) that can be used to
+         talk to YARN
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about a given <em>queue</em>.
+ </p>
+ 
+ @param queueName
+          Name of the queue whose information is needed
+ @return queue information
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all queues, recursively if there
+ is a hierarchy
+ </p>
+ 
+ @return a list of queue-information for all queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about top level queues.
+ </p>
+ 
+ @return a list of queue-information for all the top-level queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all the immediate children queues
+ of the given queue
+ </p>
+ 
+ @param parent
+          Name of the queue whose child-queues' information is needed
+ @return a list of queue-information for all queues who are direct children
+         of the given parent queue.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information about <em>acls</em> for <em>current user</em> on all the
+ existing queues.
+ </p>
+ 
+ @return a list of queue acls ({@link QueueUserACLInfo}) for
+         <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId application id of the app
+ @return a list of reports for all application attempts for specified
+         application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId application attempt id
+ @return a list of reports of all containers for specified application
+         attempts
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Attempts to move the given application to the given queue.
+ </p>
+ 
+ @param appId
+    Application to move.
+ @param queue
+    Queue to place it in to.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link GetNewReservationResponse} for a new reservation,
+ which contains the {@link ReservationId} object.
+ </p>
+
+ @return The {@link GetNewReservationResponse} containing a new
+         {@link ReservationId} object.
+ @throws YarnException if reservation cannot be created.
+ @throws IOException if reservation cannot be created.]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and gang needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationRequest is satisfiable the {@code ResourceManager}
+ answers with a {@link ReservationSubmissionResponse} that includes a
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the message detailing the reason of failure.
+ </p>
+ 
+ <p>
+ The semantics guarantees that the {@link ReservationId} returned,
+ corresponds to a valid reservation existing in the time-range request by
+ the user. The amount of capacity dedicated to such reservation can vary
+ overtime, depending of the allocation that has been determined. But it is
+ guaranteed to satisfy all the constraint expressed by the user in the
+ {@link ReservationDefinition}
+ </p>
+ 
+ @param request request to submit a new Reservation
+ @return response contains the {@link ReservationId} on accepting the
+         submission
+ @throws YarnException if the reservation cannot be created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationDefinition}. Upon success the previous allocation is
+ atomically substituted by the new one, and on failure (i.e., if the system
+ cannot find a valid allocation for the updated request), the previous
+ allocation remains valid.
+ </p>
+ 
+ @param request to update an existing Reservation (the
+          {@link ReservationUpdateRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the
+          {@link ReservationDeleteRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@link ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException if the request failed otherwise]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+ 
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ in existing cluster
+ </p>
+
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ for specified labels in existing cluster
+ </p>
+
+ @param labels labels for which labels to nodes mapping has to be retrieved
+ @return labels to nodes mappings for specific labels
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @return cluster node labels collection
+ @throws YarnException when there is a failure in
+           {@link ApplicationClientProtocol}
+ @throws IOException when there is a failure in
+           {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application
+ </p>
+ @param applicationId
+ @param priority
+ @return updated priority of an application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Signal a container identified by given ID.
+ </p>
+
+ @param containerId
+          {@link ContainerId} of the container that needs to be signaled
+ @param command the signal container command
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the resource profiles available in the RM.
+ </p>
+ @return a Map of the resource profile names to their capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the details of a specific resource profile from the RM.
+ </p>
+ @param profile the profile name
+ @return resource profile name with its capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get available resource types supported by RM.
+ </p>
+ @return list of supported resource types with detailed information
+ @throws YarnException if any issue happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getClusterAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes in the cluster.
+ </p>
+
+ @return cluster node attributes collection
+ @throws YarnException when there is a failure in
+                       {@link ApplicationClientProtocol}
+ @throws IOException   when there is a failure in
+                       {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get mapping of AttributeKey to associated
+ NodeToAttributeValue list for specified node attributeKeys in the cluster.
+ </p>
+
+ @param attributes AttributeKeys for which associated NodeToAttributeValue
+          mapping value has to be retrieved. If empty or null is set then
+          will return mapping for all attributeKeys in the cluster
+ @return mapping of AttributeKey to List of associated
+         NodeToAttributeValue's.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get all node to attribute mapping in
+ existing cluster.
+ </p>
+
+ @param hostNames HostNames for which host to attributes mapping has to
+                  be retrived.If empty or null is set then will return
+                  all nodes to attributes mapping in cluster.
+ @return Node to attribute mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="shellToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.ShellContainerCommand"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get a shell to a container.
+ </p>
+
+ @param containerId Container ID
+ @param command Shell type
+ @throws IOException if connection fails.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+  <class name="YarnClientApplication" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClientApplication" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse, org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNewApplicationResponse" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async">
+  <!-- start class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <class name="AMRMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param client the AMRMClient instance
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="setHeartbeatInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interval" type="int"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given Priority,
+ ResourceName, ExecutionType and Capability.
+
+ NOTE: This matches only requests that were made by the client WITHOUT the
+ allocationRequestId specified.
+
+ @param priority Priority.
+ @param resourceName Location.
+ @param executionType ExecutionType.
+ @param capability Capability.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given
+ AllocationRequestId.
+
+ NOTE: This matches only requests that were made by the client WITH the
+ allocationRequestId specified.
+
+ @param allocationRequestId AllocationRequestId.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this application master with the resource manager. On successful
+ registration, starts the heartbeating thread.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineClient" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Register TimelineClient to AMRMClient.
+ @param timelineClient
+ @throws YarnException when this method is invoked even when ATS V2 is not
+           configured.]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline client.
+ @return the registered timeline client]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+
+ @param blacklistAdditions list of resources which should be added to the
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="handler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="heartbeatIntervalMs" type="java.util.concurrent.atomic.AtomicInteger"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>AMRMClientAsync</code> handles communication with the ResourceManager
+ and provides asynchronous updates on events such as container allocations and
+ completions.  It contains a thread that sends periodic heartbeats to the
+ ResourceManager.
+ 
+ It should be used by implementing a CallbackHandler:
+ <pre>
+ {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List<Container> containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List<Container> containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List<ContainerStatus> statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List<NodeReport> updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+ </pre>
+ 
+ The client's lifecycle should be managed similarly to the following:
+ 
+ <pre>
+ {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <!-- start class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+  <class name="NMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"/>
+    </method>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="startContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+    </method>
+    <method name="increaseContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+    </method>
+    <method name="updateContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?]]>
+      </doc>
+    </method>
+    <method name="restartContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.]]>
+      </doc>
+    </method>
+    <method name="stopContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getContainerStatusAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.NMClient"/>
+    </method>
+    <method name="getCallbackHandler" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.NMClient"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>NMClientAsync</code> handles communication with all the NodeManagers
+ and provides asynchronous updates on getting responses from them. It
+ maintains a thread pool to communicate with individual NMs where a number of
+ worker threads process requests to NMs by using {@link NMClientImpl}. The max
+ size of the thread pool is configurable through
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}.
+
+ It should be used in conjunction with a CallbackHandler. For example
+
+ <pre>
+ {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map<String, ByteBuffer> allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+ </pre>
+
+ The client's life-cycle should be managed like the following:
+
+ <pre>
+ {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.cli">
+  <!-- start class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <class name="LogsCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="LogsCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getAMContainerInfoForRMWebService" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getClientResponseFromTimelineReader" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMatchedContainerLogFiles" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="useRegex" type="boolean"/>
+      <param name="ignoreSizeLimit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResponseFromNMWebService" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="webServiceClient" type="com.sun.jersey.api.client.Client"/>
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="logFile" type="java.lang.String"/>
+    </method>
+    <method name="getNodeHttpAddressFromRMWebString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="ClientHandlerException" type="com.sun.jersey.api.client.ClientHandlerException"/>
+      <exception name="UniformInterfaceException" type="com.sun.jersey.api.client.UniformInterfaceException"/>
+      <exception name="JSONException" type="org.codehaus.jettison.json.JSONException"/>
+    </method>
+    <field name="HELP_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <!-- start class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+  <class name="SchedConfCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="SchedConfCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[CLI for modifying scheduler configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+</package>
+<package name="org.apache.hadoop.yarn.client.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml
new file mode 100644
index 0000000000000..aa23d55a710ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml
@@ -0,0 +1,3067 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:13:01 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Client 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/org/eclipse/jetty/websocket/websocket-client/9.4.43.v20210629/websocket-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-client/9.4.43.v20210629/jetty-client-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-common/9.4.43.v20210629/websocket-common-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/websocket/websocket-api/9.4.43.v20210629/websocket-api-9.4.43.v20210629.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.4.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.4.jar:/maven/com/squareup/okhttp3/okhttp/4.9.3/okhttp-4.9.3.jar:/maven/com/squareup/okio/okio/2.8.0/okio-2.8.0.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib/1.4.10/kotlin-stdlib-1.4.10.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib-common/1.4.10/kotlin-stdlib-common-1.4.10.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/jline/jline/3.9.0/jline-3.9.0.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/site/jdiff/xml -apiname Apache Hadoop YARN Client 3.3.4 -->
+<package name="org.apache.hadoop.yarn.client">
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <class name="AHSClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSClient" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AHSClient.]]>
+      </doc>
+    </method>
+    <method name="createAHSv2Client" return="org.apache.hadoop.yarn.client.api.AHSClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get a report of the given Application.
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+   <li>host - set to "N/A"</li>
+   <li>RPC port - set to -1</li>
+   <li>client token - set to "N/A"</li>
+   <li>diagnostics - set to "N/A"</li>
+   <li>tracking URL - set to "N/A"</li>
+   <li>original tracking URL - set to "N/A"</li>
+   <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+ 
+ @return a list of reports for all applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId
+ @return a list of reports for all application attempts for specified
+         application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId
+ @return a list of reports of all containers for specified application
+         attempt
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AHSClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <class name="AMRMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClient" return="org.apache.hadoop.yarn.client.api.AMRMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of AMRMClient.
+ For usage:
+ <pre>
+ {@code
+ AMRMClient.<T>createAMRMClientContainerRequest()
+ }</pre>
+ @return the newly create AMRMClient instance.]]>
+      </doc>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any 
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="allocate" return="org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="progressIndicator" type="float"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Request additional containers and receive new container allocations.
+ Requests made via <code>addContainerRequest</code> are sent to the
+ <code>ResourceManager</code>. New containers assigned to the master are
+ retrieved. Status of completed containers and node health updates are also
+ retrieved. This also doubles up as a heartbeat to the ResourceManager and
+ must be made periodically. The call may not always return any new
+ allocations of containers. App should not make concurrent allocate
+ requests. May cause request loss.
+ 
+ <p>
+ Note : If the user has not removed container requests that have already
+ been satisfied, then the re-register may end up sending the entire
+ container requests to the RM (including matched requests). Which would mean
+ the RM could end up giving it a lot of new allocated containers.
+ </p>
+ 
+ @param progressIndicator Indicates progress made by the master
+ @return the response of the allocate request
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given 
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating 
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical 
+ <code>Resource</code> size that fit in the given capability. In a 
+ collection, requests will be returned in the same order as they were added.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ parameters. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ <code>Resource</code> size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+ specify an <code>ExecutionType</code>.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @param priority Priority
+ @param resourceName Location
+ @param executionType ExecutionType
+ @param capability Capability
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <param name="profile" type="java.lang.String"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Get outstanding <code>ContainerRequest</code>s matching the given
+ allocationRequestId. These ContainerRequests should have been added via
+ <code>addContainerRequest</code> earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITH the allocationRequestId being set to a non-default value.
+
+ @param allocationRequestId Allocation Request Id
+ @return Collection of request matching the parameters]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+ 
+ @param blacklistAdditions list of resources which should be added to the 
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the 
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ be shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ shared with the {@link NMClient} used to manage containers for the
+ <code>AMRMClient</code>.
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache.]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <doc>
+      <![CDATA[Register TimelineV2Client to AMRMClient. Writer's address for the timeline
+ V2 client will be updated dynamically if registered.
+
+ @param client the timeline v2 client to register]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline v2 client.
+ @return the registered timeline v2 client]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AMRMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMClient -->
+  <class name="NMClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="createNMClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create a new instance of NMClient.]]>
+      </doc>
+    </method>
+    <method name="startContainer" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Start an allocated container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the allocated container, including the
+ Id, the assigned node's Id and the token via {@link Container}. In
+ addition, the AM needs to provide the {@link ContainerLaunchContext} as
+ well.</p>
+
+ @param container the allocated container
+ @param containerLaunchContext the context information needed by the
+                               <code>NodeManager</code> to launch the
+                               container
+ @return a map between the auxiliary service names and their outputs
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="increaseContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Increase the resource of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="updateContainerResource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="stopContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Stop an started container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="getContainerStatus" return="org.apache.hadoop.yarn.api.records.ContainerStatus"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Query the status of a container.</p>
+
+ @param containerId the Id of the started container
+ @param nodeId the Id of the <code>NodeManager</code>
+ 
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="restartContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitialization"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+    <method name="cleanupRunningContainersOnStop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="enabled" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Set whether the containers that are started by this client, and are
+ still running should be stopped when the client stops. By default, the
+ feature should be enabled.</p> However, containers will be stopped only  
+ when service is stopped. i.e. after {@link NMClient#stop()}. 
+
+ @param enabled whether the feature is enabled or not]]>
+      </doc>
+    </method>
+    <method name="setNMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nmTokenCache" type="org.apache.hadoop.yarn.client.api.NMTokenCache"/>
+      <doc>
+      <![CDATA[Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @param nmTokenCache the NM token cache to use.]]>
+      </doc>
+    </method>
+    <method name="getNMTokenCache" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the NM token cache of the <code>NMClient</code>. This cache must be
+ shared with the {@link AMRMClient} that requested the containers managed
+ by this <code>NMClient</code>
+ <p>
+ If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ singleton instance will be used.
+
+ @return the NM token cache]]>
+      </doc>
+    </method>
+    <method name="getNodeIdOfStartedContainer" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Get the NodeId of the node on which container is running. It returns
+ null if the container if container is not found or if it is not running.
+
+ @param containerId Container Id of the container.
+ @return NodeId of the container on which it is running.]]>
+      </doc>
+    </method>
+    <method name="localize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <param name="localResources" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Localize resources for a container.
+ @param containerId     the ID of the container
+ @param nodeId          node Id of the container
+ @param localResources  resources to localize]]>
+      </doc>
+    </method>
+    <method name="getLocalizationStatuses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the localization statuses of a container.
+
+ @param containerId   the Id of the container
+ @param nodeId        node Id of the container
+
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <class name="NMTokenCache" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenCache"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates a NM token cache instance.]]>
+      </doc>
+    </constructor>
+    <method name="getSingleton" return="org.apache.hadoop.yarn.client.api.NMTokenCache"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the singleton NM token cache.
+
+ @return the singleton NM token cache.]]>
+      </doc>
+    </method>
+    <method name="getNMToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent. Only the singleton obtained from
+ {@link #getSingleton()} is looked at for the tokens. If you are using your
+ own NMTokenCache that is different from the singleton, use
+ {@link #getToken(String) }
+ 
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node manager]]>
+      </doc>
+    </method>
+    <method name="setNMToken"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address only in the singleton obtained from
+ {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ different from the singleton, use {@link #setToken(String, Token) }
+ 
+ @param nodeAddr
+          node address (host:port)
+ @param token
+          NMToken]]>
+      </doc>
+    </method>
+    <method name="getToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Returns NMToken, null if absent
+ @param nodeAddr
+ @return {@link Token} NMToken required for communicating with node
+         manager]]>
+      </doc>
+    </method>
+    <method name="setToken"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeAddr" type="java.lang.String"/>
+      <param name="token" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <doc>
+      <![CDATA[Sets the NMToken for node address
+ @param nodeAddr node address (host:port)
+ @param token NMToken]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[NMTokenCache manages NMTokens required for an Application Master
+ communicating with individual NodeManagers.
+ <p>
+ By default YARN client libraries {@link AMRMClient} and {@link NMClient} use
+ {@link #getSingleton()} instance of the cache.
+ <ul>
+   <li>
+     Using the singleton instance of the cache is appropriate when running a
+     single ApplicationMaster in the same JVM.
+   </li>
+   <li>
+     When using the singleton, users don't need to do anything special,
+     {@link AMRMClient} and {@link NMClient} are already set up to use the
+     default singleton {@link NMTokenCache}
+     </li>
+ </ul>
+ If running multiple Application Masters in the same JVM, a different cache
+ instance should be used for each Application Master.
+ <ul>
+   <li>
+     If using the {@link AMRMClient} and the {@link NMClient}, setting up
+     and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using the {@link AMRMClientAsync} and the {@link NMClientAsync},
+     setting up and using an instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+ </pre>
+   </li>
+   <li>
+     If using {@link ApplicationMasterProtocol} and
+     {@link ContainerManagementProtocol} directly, setting up and using an
+     instance cache is as follows:
+ <pre>
+   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+ </pre>
+   </li>
+ </ul>
+ It is also possible to mix the usage of a client ({@code AMRMClient} or
+ {@code NMClient}, or the async versions of them) with a protocol proxy
+ ({@code ContainerManagementProtocolProxy} or
+ {@code ApplicationMasterProtocol}).]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.NMTokenCache -->
+  <!-- start class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <class name="SharedCacheClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createSharedCacheClient" return="org.apache.hadoop.yarn.client.api.SharedCacheClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="use" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to claim a resource with the <code>SharedCacheManager.</code>
+ The client uses a checksum to identify the resource and an
+ {@link ApplicationId} to identify which application will be using the
+ resource.
+ </p>
+
+ <p>
+ The <code>SharedCacheManager</code> responds with whether or not the
+ resource exists in the cache. If the resource exists, a <code>URL</code> to
+ the resource in the shared cache is returned. If the resource does not
+ exist, null is returned instead.
+ </p>
+
+ <p>
+ Once a URL has been returned for a resource, that URL is safe to use for
+ the lifetime of the application that corresponds to the provided
+ ApplicationId.
+ </p>
+
+ @param applicationId ApplicationId of the application using the resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource
+ @return URL to the resource, or null if it does not exist]]>
+      </doc>
+    </method>
+    <method name="release"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="resourceKey" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ The method to release a resource with the <code>SharedCacheManager.</code>
+ This method is called once an application is no longer using a claimed
+ resource in the shared cache. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application is
+ releasing the resource.
+ </p>
+ 
+ <p>
+ Note: This method is an optimization and the client is not required to call
+ it for correctness.
+ </p>
+ 
+ @param applicationId ApplicationId of the application releasing the
+          resource
+ @param resourceKey the key (i.e. checksum) that identifies the resource]]>
+      </doc>
+    </method>
+    <method name="getFileChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="sourceFile" type="org.apache.hadoop.fs.Path"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[A convenience method to calculate the checksum of a specified file.
+ 
+ @param sourceFile A path to the input file
+ @return A hex string containing the checksum digest
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the client for YARN's shared cache.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.SharedCacheClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <class name="YarnClient" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of YarnClient.]]>
+      </doc>
+    </method>
+    <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link YarnClientApplication} for a new application,
+ which in turn contains the {@link ApplicationSubmissionContext} and
+ {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
+ objects.
+ </p>
+
+ @return {@link YarnClientApplication} built for a new application
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Submit a new application to <code>YARN.</code> It is a blocking call - it
+ will not return {@link ApplicationId} until the submitted application is
+ submitted successfully and accepted by the ResourceManager.
+ </p>
+ 
+ <p>
+ Users should provide an {@link ApplicationId} as part of the parameter
+ {@link ApplicationSubmissionContext} when submitting a new application,
+ otherwise it will throw the {@link ApplicationIdNotProvidedException}.
+ </p>
+
+ <p>This internally calls {@link ApplicationClientProtocol#submitApplication
+ (SubmitApplicationRequest)}, and after that, it internally invokes
+ {@link ApplicationClientProtocol#getApplicationReport
+ (GetApplicationReportRequest)} and waits till it can make sure that the
+ application gets properly submitted. If RM fails over or RM restart
+ happens before ResourceManager saves the application's state,
+ {@link ApplicationClientProtocol
+ #getApplicationReport(GetApplicationReportRequest)} will throw
+ the {@link ApplicationNotFoundException}. This API automatically resubmits
+ the application with the same {@link ApplicationSubmissionContext} when it
+ catches the {@link ApplicationNotFoundException}</p>
+
+ @param appContext
+          {@link ApplicationSubmissionContext} containing all the details
+          needed to submit a new application
+ @return {@link ApplicationId} of the accepted application
+ @throws YarnException
+ @throws IOException
+ @see #createApplication()]]>
+      </doc>
+    </method>
+    <method name="failApplicationAttempt"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Fail an application attempt identified by given ID.
+ </p>
+
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the attempt to fail.
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ 
+ @param applicationId
+          {@link ApplicationId} of the application that needs to be killed
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException
+ @see #getQueueAclsInfo()]]>
+      </doc>
+    </method>
+    <method name="killApplication"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="diagnostics" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Kill an application identified by given ID.
+ </p>
+ @param applicationId {@link ApplicationId} of the application that needs to
+          be killed
+ @param diagnostics for killing an application.
+ @throws YarnException in case of errors or if YARN rejects the request due
+           to access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Application.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ <p>
+ If the user does not have <code>VIEW_APP</code> access then the following
+ fields in the report will be set to stubbed values:
+ <ul>
+ <li>host - set to "N/A"</li>
+ <li>RPC port - set to -1</li>
+ <li>client token - set to "N/A"</li>
+ <li>diagnostics - set to "N/A"</li>
+ <li>tracking URL - set to "N/A"</li>
+ <li>original tracking URL - set to "N/A"</li>
+ <li>resource usage report - all values are -1</li>
+ </ul>
+ 
+ @param appId
+          {@link ApplicationId} of the application that needs a report
+ @return application report
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the AMRM token of the application.
+ <p>
+ The AMRM token is required for AM to RM scheduling operations. For 
+ managed Application Masters YARN takes care of injecting it. For unmanaged
+ Applications Masters, the token must be obtained via this method and set
+ in the {@link org.apache.hadoop.security.UserGroupInformation} of the
+ current user.
+ <p>
+ The AMRM token will be returned only if all the following conditions are
+ met:
+ <ul>
+   <li>the requester is the owner of the ApplicationMaster</li>
+   <li>the application master is an unmanaged ApplicationMaster</li>
+   <li>the application master is in ACCEPTED state</li>
+ </ul>
+ Else this method returns NULL.
+
+ @param appId {@link ApplicationId} of the application to get the AMRM token
+ @return the AMRM token if available
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of all Applications in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @return a list of reports of all running applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications
+ matching the given application types in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types and application states in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <param name="applicationTags" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given
+ application types, application states and application tags in the cluster.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @param applicationTags set of application tags you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queues" type="java.util.Set"/>
+      <param name="users" type="java.util.Set"/>
+      <param name="applicationTypes" type="java.util.Set"/>
+      <param name="applicationStates" type="java.util.EnumSet"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report (ApplicationReport) of Applications matching the given users,
+ queues, application types and application states in the cluster. If any of
+ the params is set to null, it is not used when filtering.
+ </p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param queues set of queues you are interested in
+ @param users set of users you are interested in
+ @param applicationTypes set of application types you are interested in
+ @param applicationStates set of application states you are interested in
+ @return a list of reports of applications
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplications" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a list of ApplicationReports that match the given
+ {@link GetApplicationsRequest}.
+</p>
+
+ <p>
+ If the user does not have <code>VIEW_APP</code> access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+ </p>
+
+ @param request the request object to get the list of applications.
+ @return The list of ApplicationReports that match the request
+ @throws YarnException Exception specific to YARN.
+ @throws IOException Exception mostly related to connection errors.]]>
+      </doc>
+    </method>
+    <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get metrics ({@link YarnClusterMetrics}) about the cluster.
+ </p>
+ 
+ @return cluster metrics
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeReports" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of nodes ({@link NodeReport}) in the cluster.
+ </p>
+ 
+ @param states The {@link NodeState}s to filter on. If no filter states are
+          given, nodes in all states will be returned.
+ @return A list of node reports
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="org.apache.hadoop.io.Text"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to YARN using those tokens.
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to YARN.
+ @return a delegation token ({@link Token}) that can be used to
+         talk to YARN
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="queueName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about a given <em>queue</em>.
+ </p>
+ 
+ @param queueName
+          Name of the queue whose information is needed
+ @return queue information
+ @throws YarnException
+           in case of errors or if YARN rejects the request due to
+           access-control restrictions.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getAllQueues" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all queues, recursively if there
+ is a hierarchy
+ </p>
+ 
+ @return a list of queue-information for all queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRootQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about top level queues.
+ </p>
+ 
+ @return a list of queue-information for all the top-level queues
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getChildQueueInfos" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="parent" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information ({@link QueueInfo}) about all the immediate children queues
+ of the given queue
+ </p>
+ 
+ @param parent
+          Name of the queue whose child-queues' information is needed
+ @return a list of queue-information for all queues who are direct children
+         of the given parent queue.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getQueueAclsInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get information about <em>acls</em> for <em>current user</em> on all the
+ existing queues.
+ </p>
+ 
+ @return a list of queue acls ({@link QueueUserACLInfo}) for
+         <em>current user</em>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given ApplicationAttempt.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param applicationAttemptId
+          {@link ApplicationAttemptId} of the application attempt that needs
+          a report
+ @return application attempt report
+ @throws YarnException
+ @throws ApplicationAttemptNotFoundException if application attempt
+         not found
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttempts" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (ApplicationAttempts) of Application in the cluster.
+ </p>
+ 
+ @param applicationId application id of the app
+ @return a list of reports for all application attempts for specified
+         application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of the given Container.
+ </p>
+ 
+ <p>
+ In secure mode, <code>YARN</code> verifies access to the application, queue
+ etc. before accepting the request.
+ </p>
+ 
+ @param containerId
+          {@link ContainerId} of the container that needs a report
+ @return container report
+ @throws YarnException
+ @throws ContainerNotFoundException if container not found.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainers" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ </p>
+ 
+ @param applicationAttemptId application attempt id
+ @return a list of reports of all containers for specified application
+         attempts
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="moveApplicationAcrossQueues"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Attempts to move the given application to the given queue.
+ </p>
+ 
+ @param appId
+    Application to move.
+ @param queue
+    Queue to place it in to.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="createReservation" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Obtain a {@link GetNewReservationResponse} for a new reservation,
+ which contains the {@link ReservationId} object.
+ </p>
+
+ @return The {@link GetNewReservationResponse} containing a new
+         {@link ReservationId} object.
+ @throws YarnException if reservation cannot be created.
+ @throws IOException if reservation cannot be created.]]>
+      </doc>
+    </method>
+    <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to submit a new reservation to the
+ {@code ResourceManager}.
+ </p>
+ 
+ <p>
+ The client packages all details of its request in a
+ {@link ReservationSubmissionRequest} object. This contains information
+ about the amount of capacity, temporal constraints, and gang needs.
+ Furthermore, the reservation might be composed of multiple stages, with
+ ordering dependencies among them.
+ </p>
+ 
+ <p>
+ In order to respond, a new admission control component in the
+ {@code ResourceManager} performs an analysis of the resources that have
+ been committed over the period of time the user is requesting, verify that
+ the user requests can be fulfilled, and that it respect a sharing policy
+ (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined
+ that the ReservationRequest is satisfiable the {@code ResourceManager}
+ answers with a {@link ReservationSubmissionResponse} that includes a
+ {@link ReservationId}. Upon failure to find a valid allocation the response
+ is an exception with the message detailing the reason of failure.
+ </p>
+ 
+ <p>
+ The semantics guarantees that the {@link ReservationId} returned,
+ corresponds to a valid reservation existing in the time-range request by
+ the user. The amount of capacity dedicated to such reservation can vary
+ overtime, depending of the allocation that has been determined. But it is
+ guaranteed to satisfy all the constraint expressed by the user in the
+ {@link ReservationDefinition}
+ </p>
+ 
+ @param request request to submit a new Reservation
+ @return response contains the {@link ReservationId} on accepting the
+         submission
+ @throws YarnException if the reservation cannot be created successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to update an existing Reservation. This is
+ referred to as a re-negotiation process, in which a user that has
+ previously submitted a Reservation.
+ </p>
+ 
+ <p>
+ The allocation is attempted by virtually substituting all previous
+ allocations related to this Reservation with new ones, that satisfy the new
+ {@link ReservationDefinition}. Upon success the previous allocation is
+ atomically substituted by the new one, and on failure (i.e., if the system
+ cannot find a valid allocation for the updated request), the previous
+ allocation remains valid.
+ </p>
+ 
+ @param request to update an existing Reservation (the
+          {@link ReservationUpdateRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully updating the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           updated successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to remove an existing Reservation.
+ </p>
+ 
+ @param request to remove an existing Reservation (the
+          {@link ReservationDeleteRequest} should refer to an existing valid
+          {@link ReservationId})
+ @return response empty on successfully deleting the existing reservation
+ @throws YarnException if the request is invalid or reservation cannot be
+           deleted successfully
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="listReservations" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by clients to get the list of reservations in a plan.
+ The reservationId will be used to search for reservations to list if it is
+ provided. Otherwise, it will select active reservations within the
+ startTime and endTime (inclusive).
+ </p>
+
+ @param request to list reservations in a plan. Contains fields to select
+                String queue, ReservationId reservationId, long startTime,
+                long endTime, and a bool includeReservationAllocations.
+
+                queue: Required. Cannot be null or empty. Refers to the
+                reservable queue in the scheduler that was selected when
+                creating a reservation submission
+                {@link ReservationSubmissionRequest}.
+
+                reservationId: Optional. If provided, other fields will
+                be ignored.
+
+                startTime: Optional. If provided, only reservations that
+                end after the startTime will be selected. This defaults
+                to 0 if an invalid number is used.
+
+                endTime: Optional. If provided, only reservations that
+                start on or before endTime will be selected. This defaults
+                to Long.MAX_VALUE if an invalid number is used.
+
+                includeReservationAllocations: Optional. Flag that
+                determines whether the entire reservation allocations are
+                to be returned. Reservation allocations are subject to
+                change in the event of re-planning as described by
+                {@link ReservationDefinition}.
+
+ @return response that contains information about reservations that are
+                being searched for.
+ @throws YarnException if the request is invalid
+ @throws IOException if the request failed otherwise]]>
+      </doc>
+    </method>
+    <method name="getNodeToLabels" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node to labels mappings in existing cluster
+ </p>
+ 
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ in existing cluster
+ </p>
+
+ @return node to labels mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getLabelsToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="labels" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get labels to nodes mapping
+ for specified labels in existing cluster
+ </p>
+
+ @param labels labels for which labels to nodes mapping has to be retrieved
+ @return labels to nodes mappings for specific labels
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeLabels" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node labels in the cluster
+ </p>
+
+ @return cluster node labels collection
+ @throws YarnException when there is a failure in
+           {@link ApplicationClientProtocol}
+ @throws IOException when there is a failure in
+           {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="updateApplicationPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to set priority of an application
+ </p>
+ @param applicationId
+ @param priority
+ @return updated priority of an application.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="signalToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.SignalContainerCommand"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Signal a container identified by given ID.
+ </p>
+
+ @param containerId
+          {@link ContainerId} of the container that needs to be signaled
+ @param command the signal container command
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="updateApplicationTimeouts" return="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResourceProfiles" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the resource profiles available in the RM.
+ </p>
+ @return a Map of the resource profile names to their capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other errors]]>
+      </doc>
+    </method>
+    <method name="getResourceProfile" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="profile" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get the details of a specific resource profile from the RM.
+ </p>
+ @param profile the profile name
+ @return resource profile name with its capabilities
+ @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ @throws YarnException if any error happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getResourceTypeInfo" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ Get available resource types supported by RM.
+ </p>
+ @return list of supported resource types with detailed information
+ @throws YarnException if any issue happens inside YARN
+ @throws IOException in case of other others]]>
+      </doc>
+    </method>
+    <method name="getClusterAttributes" return="java.util.Set"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get node attributes in the cluster.
+ </p>
+
+ @return cluster node attributes collection
+ @throws YarnException when there is a failure in
+                       {@link ApplicationClientProtocol}
+ @throws IOException   when there is a failure in
+                       {@link ApplicationClientProtocol}]]>
+      </doc>
+    </method>
+    <method name="getAttributesToNodes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="attributes" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get mapping of AttributeKey to associated
+ NodeToAttributeValue list for specified node attributeKeys in the cluster.
+ </p>
+
+ @param attributes AttributeKeys for which associated NodeToAttributeValue
+          mapping value has to be retrieved. If empty or null is set then
+          will return mapping for all attributeKeys in the cluster
+ @return mapping of AttributeKey to List of associated
+         NodeToAttributeValue's.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getNodeToAttributes" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="hostNames" type="java.util.Set"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get all node to attribute mapping in
+ existing cluster.
+ </p>
+
+ @param hostNames HostNames for which host to attributes mapping has to
+                  be retrived.If empty or null is set then will return
+                  all nodes to attributes mapping in cluster.
+ @return Node to attribute mappings
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="shellToContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="command" type="org.apache.hadoop.yarn.api.records.ShellContainerCommand"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[<p>
+ The interface used by client to get a shell to a container.
+ </p>
+
+ @param containerId Container ID
+ @param command Shell type
+ @throws IOException if connection fails.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+  <class name="YarnClientApplication" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="YarnClientApplication" type="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse, org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getNewApplicationResponse" return="org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.YarnClientApplication -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async">
+  <!-- start class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <class name="AMRMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMClientAsync" type="org.apache.hadoop.yarn.client.api.AMRMClient, int, org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.AbstractCallbackHandler"/>
+      <doc>
+      <![CDATA[<p>Create a new instance of AMRMClientAsync.</p>
+
+ @param client the AMRMClient instance
+ @param intervalMs heartbeat interval in milliseconds between AM and RM
+ @param callbackHandler callback handler that processes responses from
+                        the <code>ResourceManager</code>]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(int,
+             AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="createAMRMClientAsync" return="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"/>
+      <param name="intervalMs" type="int"/>
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createAMRMClientAsync(AMRMClient,
+             int, AMRMClientAsync.AbstractCallbackHandler)} instead.]]>
+      </doc>
+    </method>
+    <method name="setHeartbeatInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="interval" type="int"/>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+    </method>
+    <method name="addSchedulingRequests"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="schedulingRequests" type="java.util.Collection"/>
+      <doc>
+      <![CDATA[Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ all requests in the same batch are sent in the same allocate call.
+ @param schedulingRequests Collection of Scheduling Requests.]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="priority" type="org.apache.hadoop.yarn.api.records.Priority"/>
+      <param name="resourceName" type="java.lang.String"/>
+      <param name="executionType" type="org.apache.hadoop.yarn.api.records.ExecutionType"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given Priority,
+ ResourceName, ExecutionType and Capability.
+
+ NOTE: This matches only requests that were made by the client WITHOUT the
+ allocationRequestId specified.
+
+ @param priority Priority.
+ @param resourceName Location.
+ @param executionType ExecutionType.
+ @param capability Capability.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="getMatchingRequests" return="java.util.Collection"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="allocationRequestId" type="long"/>
+      <doc>
+      <![CDATA[Returns all matching ContainerRequests that match the given
+ AllocationRequestId.
+
+ NOTE: This matches only requests that were made by the client WITH the
+ allocationRequestId specified.
+
+ @param allocationRequestId AllocationRequestId.
+ @return All matching ContainerRequests]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this application master with the resource manager. On successful
+ registration, starts the heartbeating thread.
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appHostName" type="java.lang.String"/>
+      <param name="appHostPort" type="int"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <param name="placementConstraints" type="java.util.Map"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register the application master. This must be called before any
+ other interaction
+ @param appHostName Name of the host on which master is running
+ @param appHostPort Port master is listening on
+ @param appTrackingUrl URL at which the master info can be seen
+ @param placementConstraints Placement Constraints mappings.
+ @return <code>RegisterApplicationMasterResponse</code>
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="unregisterApplicationMaster"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+      <param name="appMessage" type="java.lang.String"/>
+      <param name="appTrackingUrl" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregister the application master. This must be called in the end.
+ @param appStatus Success/Failure status of the master
+ @param appMessage Diagnostics message on failure
+ @param appTrackingUrl New URL to get master info
+ @throws YarnException
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="addContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Request containers for resources before calling <code>allocate</code>
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="removeContainerRequest"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="req" type="T"/>
+      <doc>
+      <![CDATA[Remove previous container request. The previous container request may have 
+ already been sent to the ResourceManager. So even after the remove request 
+ the app must be prepared to receive an allocation for the previous request 
+ even after the remove request
+ @param req Resource request]]>
+      </doc>
+    </method>
+    <method name="requestContainerResourceChange"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="capability" type="org.apache.hadoop.yarn.api.records.Resource"/>
+      <doc>
+      <![CDATA[Request container resource change before calling <code>allocate</code>.
+ Any previous pending resource change request of the same container will be
+ removed.
+
+ Application that calls this method is expected to maintain the
+ <code>Container</code>s that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+                  allocation or resource change
+ @param capability  The target resource capability of the container]]>
+      </doc>
+    </method>
+    <method name="requestContainerUpdate"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="updateContainerRequest" type="org.apache.hadoop.yarn.api.records.UpdateContainerRequest"/>
+      <doc>
+      <![CDATA[Request a container update before calling <code>allocate</code>.
+ Any previous pending update request of the same container will be
+ removed.
+
+ @param container The container returned from the last successful resource
+                  allocation or update
+ @param updateContainerRequest The <code>UpdateContainerRequest</code>.]]>
+      </doc>
+    </method>
+    <method name="releaseAssignedContainer"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[Release containers assigned by the Resource Manager. If the app cannot use
+ the container or wants to give up the container then it can release them.
+ The app needs to make new requests for the released resource capability if
+ it still needs it. eg. it released non-local resources
+ @param containerId]]>
+      </doc>
+    </method>
+    <method name="getAvailableResources" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the currently available resources in the cluster.
+ A valid value is available after a call to allocate has been made
+ @return Currently available resources]]>
+      </doc>
+    </method>
+    <method name="getClusterNodeCount" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the current number of nodes in the cluster.
+ A valid values is available after a call to allocate has been made
+ @return Current number of nodes in the cluster]]>
+      </doc>
+    </method>
+    <method name="registerTimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineClient" type="org.apache.hadoop.yarn.client.api.TimelineV2Client"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Register TimelineClient to AMRMClient.
+ @param timelineClient
+ @throws YarnException when this method is invoked even when ATS V2 is not
+           configured.]]>
+      </doc>
+    </method>
+    <method name="getRegisteredTimelineV2Client" return="org.apache.hadoop.yarn.client.api.TimelineV2Client"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get registered timeline client.
+ @return the registered timeline client]]>
+      </doc>
+    </method>
+    <method name="updateBlacklist"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="blacklistAdditions" type="java.util.List"/>
+      <param name="blacklistRemovals" type="java.util.List"/>
+      <doc>
+      <![CDATA[Update application's blacklist with addition or removal resources.
+
+ @param blacklistAdditions list of resources which should be added to the
+        application blacklist
+ @param blacklistRemovals list of resources which should be removed from the
+        application blacklist]]>
+      </doc>
+    </method>
+    <method name="updateTrackingUrl"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="trackingUrl" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Update application's tracking url on next heartbeat.
+
+ @param trackingUrl new tracking url for this application]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each 1000 ms.
+ See also {@link #waitFor(java.util.function.Supplier, int)}
+ and {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check the condition for which it should wait]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>]]>
+      </doc>
+    </method>
+    <method name="waitFor"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="check" type="java.util.function.Supplier"/>
+      <param name="checkEveryMillis" type="int"/>
+      <param name="logInterval" type="int"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Wait for <code>check</code> to return true for each
+ <code>checkEveryMillis</code> ms. In the main loop, this method will log
+ the message "waiting in main loop" for each <code>logInterval</code> times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call <code>check</code>
+ @param logInterval interval to log for each]]>
+      </doc>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.AMRMClient"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="handler" type="org.apache.hadoop.yarn.client.api.async.AMRMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="heartbeatIntervalMs" type="java.util.concurrent.atomic.AtomicInteger"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>AMRMClientAsync</code> handles communication with the ResourceManager
+ and provides asynchronous updates on events such as container allocations and
+ completions.  It contains a thread that sends periodic heartbeats to the
+ ResourceManager.
+ 
+ It should be used by implementing a CallbackHandler:
+ <pre>
+ {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List<Container> containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List<Container> containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List<ContainerStatus> statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List<NodeReport> updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+ </pre>
+ 
+ The client's lifecycle should be managed similarly to the following:
+ 
+ <pre>
+ {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.AMRMClientAsync -->
+  <!-- start class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+  <class name="NMClientAsync" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMClientAsync" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.">
+      <doc>
+      <![CDATA[@deprecated Use {@link #NMClientAsync(String, AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="NMClientAsync" type="java.lang.String, org.apache.hadoop.yarn.client.api.NMClient, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.AbstractCallbackHandler"/>
+    </method>
+    <method name="createNMClientAsync" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+      <doc>
+      <![CDATA[@deprecated Use {@link #createNMClientAsync(AbstractCallbackHandler)}
+             instead.]]>
+      </doc>
+    </method>
+    <method name="startContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <param name="containerLaunchContext" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+    </method>
+    <method name="increaseContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+    </method>
+    <method name="updateContainerResourceAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="container" type="org.apache.hadoop.yarn.api.records.Container"/>
+      <doc>
+      <![CDATA[<p>Update the resources of a container.</p>
+
+ <p>The <code>ApplicationMaster</code> or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+ </p>
+
+ @param container the container with updated token.]]>
+      </doc>
+    </method>
+    <method name="reInitializeContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="containerLaunchContex" type="org.apache.hadoop.yarn.api.records.ContainerLaunchContext"/>
+      <param name="autoCommit" type="boolean"/>
+      <doc>
+      <![CDATA[<p>Re-Initialize the Container.</p>
+
+ @param containerId the Id of the container to Re-Initialize.
+ @param containerLaunchContex the updated ContainerLaunchContext.
+ @param autoCommit commit re-initialization automatically ?]]>
+      </doc>
+    </method>
+    <method name="restartContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Restart the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="rollbackLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Rollback last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to restart.]]>
+      </doc>
+    </method>
+    <method name="commitLastReInitializationAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <doc>
+      <![CDATA[<p>Commit last reInitialization of the specified container.</p>
+
+ @param containerId the Id of the container to commit reInitialize.]]>
+      </doc>
+    </method>
+    <method name="stopContainerAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getContainerStatusAsync"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+    </method>
+    <method name="getClient" return="org.apache.hadoop.yarn.client.api.NMClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="client" type="org.apache.hadoop.yarn.client.api.NMClient"/>
+    </method>
+    <method name="getCallbackHandler" return="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setCallbackHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"/>
+    </method>
+    <field name="client" type="org.apache.hadoop.yarn.client.api.NMClient"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="callbackHandler" type="org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[<code>NMClientAsync</code> handles communication with all the NodeManagers
+ and provides asynchronous updates on getting responses from them. It
+ maintains a thread pool to communicate with individual NMs where a number of
+ worker threads process requests to NMs by using {@link NMClientImpl}. The max
+ size of the thread pool is configurable through
+ {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}.
+
+ It should be used in conjunction with a CallbackHandler. For example
+
+ <pre>
+ {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map<String, ByteBuffer> allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+ </pre>
+
+ The client's life-cycle should be managed like the following:
+
+ <pre>
+ {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+ </pre>]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.async.NMClientAsync -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.async.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+</package>
+<package name="org.apache.hadoop.yarn.client.cli">
+  <!-- start class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <class name="LogsCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="LogsCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="createYarnClient" return="org.apache.hadoop.yarn.client.api.YarnClient"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getAMContainerInfoForRMWebService" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getClientResponseFromTimelineReader" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getMatchedContainerLogFiles" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="useRegex" type="boolean"/>
+      <param name="ignoreSizeLimit" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getResponseFromNMWebService" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="webServiceClient" type="com.sun.jersey.api.client.Client"/>
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="logFile" type="java.lang.String"/>
+    </method>
+    <method name="getNodeHttpAddressFromRMWebString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="ClientHandlerException" type="com.sun.jersey.api.client.ClientHandlerException"/>
+      <exception name="UniformInterfaceException" type="com.sun.jersey.api.client.UniformInterfaceException"/>
+      <exception name="JSONException" type="org.codehaus.jettison.json.JSONException"/>
+    </method>
+    <field name="HELP_CMD" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.LogsCLI -->
+  <!-- start class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+  <class name="SchedConfCLI" extends="org.apache.hadoop.conf.Configured"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.util.Tool"/>
+    <constructor name="SchedConfCLI"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="main"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="run" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="args" type="java.lang.String[]"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <doc>
+    <![CDATA[CLI for modifying scheduler configuration.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.cli.SchedConfCLI -->
+</package>
+<package name="org.apache.hadoop.yarn.client.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.10.2.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.10.2.xml
new file mode 100644
index 0000000000000..2dfd16a259037
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.10.2.xml
@@ -0,0 +1,2936 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue May 24 22:57:32 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Common 2.10.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.10.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/slf4j/slf4j-reload4j/1.7.36/slf4j-reload4j-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.10.2.jar:/maven/com/nimbusds/nimbus-jose-jwt/7.9/nimbus-jose-jwt-7.9.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.3/json-smart-1.3.3.jar:/maven/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.10.2.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.10.2.jar:/usr/lib/jvm/zulu-7-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/commons-io/commons-io/2.5/commons-io-2.5.jar:/maven/com/google/inject/guice/3.0/guice-3.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Common 2.10.2 -->
+<package name="org.apache.hadoop.yarn">
+  <!-- start class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <class name="ContainerLogAppender" extends="org.apache.log4j.FileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <method name="getTotalLogFileSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTotalLogFileSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logSize" type="long"/>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <class name="ContainerRollingLogAppender" extends="org.apache.log4j.RollingFileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerRollingLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+  <class name="YarnUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="YarnUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.Thread"/>
+      <param name="e" type="java.lang.Throwable"/>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ In the main entry point.  It is intended to try and cleanly shut down
+ programs using the Yarn Event framework.
+ 
+ Note: Right now it only will shut down the program if a Error is caught, but
+ not any other exception.  Anything else is just logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.api">
+</package>
+<package name="org.apache.hadoop.yarn.client">
+  <!-- start class org.apache.hadoop.yarn.client.AHSProxy -->
+  <class name="AHSProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ahsAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="rmAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.AHSProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <class name="ClientRMProxy" extends="org.apache.hadoop.yarn.client.RMProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configuration" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a proxy to the ResourceManager for the specified protocol.
+ @param configuration Configuration with all the required information.
+ @param protocol Client protocol for which proxy is being requested.
+ @param <T> Type of proxy.
+ @return Proxy to the ResourceManager for the specified client protocol.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the token service name to be used for RMDelegationToken. Depending
+ on whether HA is enabled or not, this method generates the appropriate
+ service name as a comma-separated list of service addresses.
+
+ @param conf Configuration corresponding to the cluster we need the
+             RMDelegationToken for
+ @return - Service name for RMDelegationToken]]>
+      </doc>
+    </method>
+    <method name="getAMRMTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="address" type="java.lang.String"/>
+      <param name="defaultAddr" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.NMProxy -->
+  <class name="NMProxy" extends="org.apache.hadoop.yarn.client.ServerProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.NMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.RMProxy -->
+  <class name="RMProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMProxy"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="retryTime" type="long"/>
+      <param name="retryInterval" type="long"/>
+      <param name="isHAEnabled" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch retry policy from Configuration and create the
+ retry policy with specified retryTime and retry interval.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.RMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ServerProxy -->
+  <class name="ServerProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServerProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxWaitTimeStr" type="java.lang.String"/>
+      <param name="defMaxWaitTime" type="long"/>
+      <param name="connectRetryIntervalStr" type="java.lang.String"/>
+      <param name="defRetryInterval" type="long"/>
+    </method>
+    <method name="createRetriableProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+      <param name="retryPolicy" type="org.apache.hadoop.io.retry.RetryPolicy"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ServerProxy -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <class name="TimelineClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="TimelineClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineClient" return="org.apache.hadoop.yarn.client.api.TimelineClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an instance of the timeline v.1.x client.
+ The current UGI when the user initialize the client will be used to do the
+ put and the delegation token operations. The current user may use
+ {@link UserGroupInformation#doAs} another user to construct and initialize
+ a timeline client if the following operations are supposed to be conducted
+ by that user.
+
+ @return the created timeline client instance]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+ </p>
+ 
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="groupId" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param appAttemptId {@link ApplicationAttemptId}
+ @param groupId {@link TimelineEntityGroupId}
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+ </p>
+ 
+ @param domain
+          an {@link TimelineDomain} object
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param domain
+          an {@link TimelineDomain} object
+ @param appAttemptId {@link ApplicationAttemptId}
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to the timeline server in a
+ secure way.
+ </p>
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to the timeline server
+ @return a delegation token ({@link Token}) that can be used to talk to the
+         timeline server
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Renew a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to renew
+ @return the new expiration time
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Cancel a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to cancel
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to post some information in terms of a
+ number of conceptual entities. This client library needs to be used along
+ with Timeline V.1.x server versions.
+ Refer {@link TimelineV2Client} for ATS V2 interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineClient -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+</package>
+<package name="org.apache.hadoop.yarn.event">
+  <!-- start class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <class name="AbstractEvent" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Event"/>
+    <constructor name="AbstractEvent" type="TYPE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractEvent" type="TYPE, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="TYPE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Parent class of all the events. All events extend this class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <!-- start class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <class name="AsyncDispatcher" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Dispatcher"/>
+    <constructor name="AsyncDispatcher"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.util.concurrent.BlockingQueue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set a name for this dispatcher thread.
+ @param dispatcherName name of the dispatcher thread]]>
+      </doc>
+    </constructor>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="setDrainEventsOnStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="dispatch"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.event.Event"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEventThreadWaiting" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDrained" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isStopped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="eventDispatchers" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Dispatches {@link Event}s in a separate thread. Currently only single thread
+ does that. Potentially there could be multiple channels for each event type
+ class and a thread pool can be used to dispatch the events.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <interface name="Dispatcher"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <field name="DISPATCHER_EXIT_ON_ERROR_KEY" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_DISPATCHER_EXIT_ON_ERROR" type="boolean"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Event Dispatcher interface. It dispatches events to registered 
+ event handlers based on event types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Event -->
+  <interface name="Event"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="TYPE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface defining events api.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Event -->
+  <!-- start interface org.apache.hadoop.yarn.event.EventHandler -->
+  <interface name="EventHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="handle"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="T"/>
+    </method>
+    <doc>
+    <![CDATA[Interface for handling events of type T
+
+ @param <T> parameterized event of type T]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.EventHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.factories">
+</package>
+<package name="org.apache.hadoop.yarn.factory.providers">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <class name="AggregatedLogFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AggregatedLogFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <class name="AggregatedLogFormat.LogKey" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="AggregatedLogFormat.LogKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AggregatedLogFormat.LogKey" type="org.apache.hadoop.yarn.api.records.ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AggregatedLogFormat.LogKey" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+  <class name="AggregatedLogFormat.LogReader" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AggregatedLogFormat.LogReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @return the application owner.
+ @throws IOException if we can not get the application owner.]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.io.DataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next key and return the value-stream.
+ 
+ @param key the log key
+ @return the valueStream if there are more keys or null otherwise
+ @throws IOException if we can not get the dataInputStream
+ for the next key]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the valueStream
+ @param writer the log writer
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the value stream
+ @param writer the log writer
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the print stream
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container log by specifying
+ the container log type.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container for the specific bytes.
+
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log upload time stamp
+ @param bytes the output size of the log
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the output print stream
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.ifile">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.tfile">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels.event">
+</package>
+<package name="org.apache.hadoop.yarn.security">
+  <!-- start class org.apache.hadoop.yarn.security.AccessRequest -->
+  <class name="AccessRequest" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessRequest" type="org.apache.hadoop.yarn.security.PrivilegedEntity, org.apache.hadoop.security.UserGroupInformation, org.apache.hadoop.yarn.security.AccessType, java.lang.String, java.lang.String, java.lang.String, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAccessType" return="org.apache.hadoop.yarn.security.AccessType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEntity" return="org.apache.hadoop.yarn.security.PrivilegedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getForwardedAddresses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRemoteAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This request object contains all the context information to determine whether
+ a user has permission to access the target entity.
+ user       : the user who's currently accessing
+ accessType : the access type against the entity.
+ entity     : the target object user is accessing.
+ appId      : the associated app Id for current access. This could be null
+              if no app is associated.
+ appName    : the associated app name for current access. This could be null if
+              no app is associated.
+ remoteAddress : The caller's remote ip address.
+ forwardedAddresses : In case this is an http request, this contains the
+                    originating IP address of a client connecting to a web
+                    server through an HTTP proxy or load balancer. This
+                    parameter is null, if it's a RPC request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AccessRequest -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <class name="AMRMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.AMRMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[AMRMTokenIdentifier is the TokenIdentifier to be used by
+ ApplicationMasters to authenticate to the ResourceManager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <class name="AMRMTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="AMRMTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <class name="ContainerManagerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerManagerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <class name="ContainerTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="Use one of the other constructors instead.">
+      <doc>
+      <![CDATA[Creates a instance.
+
+ @param appSubmitter appSubmitter
+ @param containerID container ID
+ @param creationTime creation time
+ @param expiryTimeStamp expiry timestamp
+ @param hostName hostname
+ @param logAggregationContext log aggregation context
+ @param masterKeyId master key ID
+ @param priority priority
+ @param r resource needed by the container
+ @param rmIdentifier ResourceManager identifier
+ @deprecated Use one of the other constructors instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC layer/SecretManager.]]>
+      </doc>
+    </constructor>
+    <method name="getContainerID" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNmHostAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getExpiryTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMasterKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRMIdentifier" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the RMIdentifier of RM in which containers are allocated.
+ @return RMIdentifier]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ContainerType of container to allocate.
+ @return ContainerType]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ExecutionType of container to allocate
+ @return ExecutionType]]>
+      </doc>
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Container version
+ @return container version]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the node-label-expression in the original ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TokenIdentifier for a container. Encodes {@link ContainerId},
+ {@link Resource} needed by the container and the target NMs host-address.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <class name="ContainerTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="ContainerTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <class name="NMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, org.apache.hadoop.yarn.api.records.NodeId, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC/Secret manager]]>
+      </doc>
+    </constructor>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.NMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+  <class name="SchedulerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.admin">
+  <!-- start class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+  <class name="AdminSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AdminSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.client">
+  <!-- start class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <class name="BaseClientToAMTokenSecretManager" extends="org.apache.hadoop.security.token.SecretManager"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BaseClientToAMTokenSecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A base {@link SecretManager} for AMs to extend and validate Client-RM tokens
+ issued to clients by the RM using the underlying master-key shared by RM to
+ the AMs on their launch. All the methods are called by either Hadoop RPC or
+ YARN, so this class is strictly for the purpose of inherit/extend and
+ register with Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <class name="ClientRMSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientRMSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <class name="ClientTimelineSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientTimelineSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <class name="ClientToAMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClientToAMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationAttemptID" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClientName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ClientToAMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <class name="ClientToAMTokenSecretManager" extends="org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenSecretManager" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="retrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+    </method>
+    <method name="getMasterKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptID" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="setMasterKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+    </method>
+    <doc>
+    <![CDATA[A simple {@link SecretManager} for AMs to validate Client-RM tokens issued to
+ clients by the RM using the underlying master-key shared by RM to the AMs on
+ their launch. All the methods are called by either Hadoop RPC or YARN, so
+ this class is strictly for the purpose of inherit/extend and register with
+ Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <class name="RMDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="RMDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new delegation token identifier
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Delegation Token Identifier that identifies the delegation tokens from the 
+ Resource Manager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <class name="RMDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="RMDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <class name="TimelineDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new timeline delegation token identifier
+
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+  <class name="TimelineDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="TimelineDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.sharedcache">
+  <!-- start interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <interface name="SharedCacheChecksum"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="computeChecksum" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Calculate the checksum of the passed input stream.
+
+ @param in <code>InputStream</code> to be checksumed
+ @return the message digest of the input stream
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <!-- start class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+  <class name="SharedCacheChecksumFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheChecksumFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getChecksum" return="org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a new <code>SharedCacheChecksum</code> object based on the configurable
+ algorithm implementation
+ (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+
+ @return <code>SharedCacheChecksum</code> object]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+</package>
+<package name="org.apache.hadoop.yarn.state">
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <class name="InvalidStateTransitionException" extends="org.apache.hadoop.yarn.state.InvalidStateTransitonException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidStateTransitionException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The exception that happens when you call invalid state transition.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <class name="InvalidStateTransitonException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link InvalidStateTransitionException} instead.">
+    <constructor name="InvalidStateTransitonException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCurrentState" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEvent" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[@deprecated Use {@link InvalidStateTransitionException} instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <!-- start interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <interface name="MultipleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition" return="STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ @return the postState. Post state must be one of the 
+                      valid post states registered in StateMachine.
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. 
+ Post state is decided by Transition hook. Post state must be one of the 
+ valid post states registered in StateMachine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <interface name="SingleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ 
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. This lead to state machine to move to 
+ the post state as registered in the state machine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateMachine -->
+  <interface name="StateMachine"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getCurrentState" return="STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doTransition" return="STATE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="event" type="EVENT"/>
+      <exception name="InvalidStateTransitionException" type="org.apache.hadoop.yarn.state.InvalidStateTransitionException"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateMachine -->
+  <!-- start class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <class name="StateMachineFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StateMachineFactory" type="STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+
+ This is the only constructor in the API.]]>
+      </doc>
+    </constructor>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postStates" type="java.util.Set"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.MultipleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postStates valid post-transition states
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="installTopology" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a StateMachineFactory just like {@code this}, except that if
+         you won't need any synchronization to build a state machine
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+         The only way you could distinguish the returned
+         StateMachineFactory from {@code this} would be by
+         measuring the performance of the derived 
+         {@code StateMachine} you can get from it.
+
+ Calling this is optional.  It doesn't change the semantics of the factory,
+   if you call it then when you use the factory there is no synchronization.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+      <param name="listener" type="org.apache.hadoop.yarn.state.StateTransitionListener"/>
+      <doc>
+      <![CDATA[A StateMachine that accepts a transition listener.
+ @param operand the object upon which the returned
+                {@link StateMachine} will operate.
+ @param initialState the state in which the returned
+                {@link StateMachine} will start.
+ @param listener An implementation of a {@link StateTransitionListener}.
+ @return A (@link StateMachine}.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+    </method>
+    <method name="generateStateGraph" return="org.apache.hadoop.yarn.state.Graph"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a graph represents the state graph of this StateMachine
+ @param name graph name
+ @return Graph object generated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[State machine topology.
+ This object is semantically immutable.  If you have a
+ StateMachineFactory there's no operation in the API that changes
+ its semantic properties.
+
+ @param <OPERAND> The object type on which this state machine operates.
+ @param <STATE> The state of the entity.
+ @param <EVENTTYPE> The external eventType to be handled.
+ @param <EVENT> The event object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+  <interface name="StateTransitionListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="preTransition"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="eventToBeProcessed" type="EVENT"/>
+      <doc>
+      <![CDATA[Pre Transition Hook. This will be called before transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param eventToBeProcessed Incoming Event.]]>
+      </doc>
+    </method>
+    <method name="postTransition"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="afterState" type="STATE"/>
+      <param name="processedEvent" type="EVENT"/>
+      <doc>
+      <![CDATA[Post Transition Hook. This will be called after the transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param afterState State after transition.
+ @param processedEvent Processed Event.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A State Transition Listener.
+ It exposes a pre and post transition hook called before and
+ after the transition.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+  <!-- start class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <class name="AbstractLivelinessMonitor" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String, org.apache.hadoop.yarn.util.Clock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="expire"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="setExpireInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="expireInterval" type="int"/>
+    </method>
+    <method name="getExpireInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="o" type="O"/>
+    </method>
+    <method name="setMonitorInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="monitorInterval" type="long"/>
+    </method>
+    <method name="receivedPing"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+      <param name="expireTime" type="long"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="resetTimer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResetTimeOnStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="resetTimeOnStart" type="boolean"/>
+    </method>
+    <field name="DEFAULT_EXPIRE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A simple liveliness monitor with which clients can register, trust the
+ component to monitor liveliness, get a call-back on expiry and then finally
+ unregister.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <!-- start class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="org.apache.hadoop.util.ApplicationClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[This type has been deprecated in favor of
+ {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ ApplicationClassLoader should use that type instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <!-- start interface org.apache.hadoop.yarn.util.Clock -->
+  <interface name="Clock"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A simple clock interface that gives you time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.util.Clock -->
+  <!-- start class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <class name="ConverterUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConverterUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPathFromYarnURL" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[return a hadoop path from a given url
+ This method is deprecated, use {@link URL#toPath()} instead.
+ 
+ @param url
+          url to convert
+ @return path from {@link URL}
+ @throws URISyntaxException]]>
+      </doc>
+    </method>
+    <method name="getYarnUrlFromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getYarnUrlFromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recordFactory" type="org.apache.hadoop.yarn.factories.RecordFactory"/>
+      <param name="applicationIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="serviceAddr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service. Supposed
+ to be used for tokens other than RMDelegationToken. For
+ RMDelegationToken, use
+ {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
+ org.apache.hadoop.io.Text)} instead.
+
+ @param protoToken the yarn token
+ @param serviceAddr the connect address for the service
+ @return rpc token]]>
+      </doc>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service.
+
+ @param protoToken the yarn token
+ @param service the service for the token]]>
+      </doc>
+    </method>
+    <field name="APPLICATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONTAINER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_ATTEMPT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains a set of utilities which help converting data structures
+ from/to 'serializableFormat' to/from hadoop/nativejava data structures.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <!-- start class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <class name="MonotonicClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="MonotonicClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current time from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ @return a monotonic clock that counts in milliseconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A monotonic clock from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ This is appropriate to use when computing how much longer to wait for an
+ interval to expire.
+ This function can return a negative value and it must be handled correctly
+ by callers. See the documentation of System#nanoTime for caveats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <!-- start class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <class name="ResourceCalculatorProcessTree" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceCalculatorProcessTree" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create process-tree instance with specified root process.
+
+ Subclass must override this.
+ @param root process-tree root-process]]>
+      </doc>
+    </constructor>
+    <method name="updateProcessTree"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the process-tree with latest state.
+
+ Each call to this function should increment the age of the running
+ processes that already exist in the process tree. Age is used other API's
+ of the interface.]]>
+      </doc>
+    </method>
+    <method name="getProcessTreeDump" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a dump of the process-tree.
+
+ @return a string concatenating the dump of information of all the processes
+         in the process-tree]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree.
+
+ @return virtual memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeVmem" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree.
+
+ @return virtual memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree.
+
+ @return rss memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeRssmem" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree.
+
+ @return rss memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return virtual memory used by the process-tree in bytes for
+ processes older than the specified age, {@link #UNAVAILABLE} if it
+ cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeVmem" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return virtual memory used by the process-tree in bytes for
+ processes older than the specified age, {@link #UNAVAILABLE} if it
+ cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return rss memory used by the process-tree in bytes for
+ processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeRssmem" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return rss memory used by the process-tree in bytes for
+ processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU time in millisecond used by all the processes in the
+ process-tree since the process-tree was created
+
+ @return cumulative CPU time in millisecond since the process-tree
+ created, {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercent" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU usage by all the processes in the process-tree based on
+ average between samples as a ratio of overall CPU cycles similar to top.
+ Thus, if 2 out of 4 cores are used this should return 200.0.
+ Note: UNAVAILABLE will be returned in case when CPU usage is not
+ available. It is NOT advised to return any other error code.
+
+ @return percentage CPU usage since the process-tree was created,
+ {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available.]]>
+      </doc>
+    </method>
+    <method name="checkPidPgrpidForMatch" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify that the tree process id is same as its process group id.
+ @return true if the process id matches else return false.]]>
+      </doc>
+    </method>
+    <method name="getResourceCalculatorProcessTree" return="org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <param name="clazz" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create the ResourceCalculatorProcessTree rooted to specified process 
+ from the class name and configure it. If class name is null, this method
+ will try and return a process tree plugin available for this system.
+
+ @param pid process pid of the root of the process tree
+ @param clazz class-name
+ @param conf configure the plugin with this.
+
+ @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
+         is not available for this system.]]>
+      </doc>
+    </method>
+    <field name="UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interface class to obtain process resource usage
+ NOTE: This class should not be used by external users, but only by external
+ developers to extend and include their own process-tree implementation, 
+ especially for platforms other than Linux and Windows.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <!-- start class org.apache.hadoop.yarn.util.SystemClock -->
+  <class name="SystemClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="SystemClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.yarn.util.SystemClock"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current time from the system
+ clock in milliseconds.
+ 
+ NOTE: Do not use this to calculate a duration of expire or interval to sleep,
+ because it will be broken by settimeofday. Please use {@link MonotonicClock}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.SystemClock -->
+  <!-- start class org.apache.hadoop.yarn.util.UTCClock -->
+  <class name="UTCClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="UTCClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current UTC time in
+ milliseconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.UTCClock -->
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+<package name="org.apache.hadoop.yarn.util.timeline">
+  <!-- start class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+  <class name="TimelineUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <exception name="JsonGenerationException" type="org.codehaus.jackson.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="org.codehaus.jackson.map.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string not in a pretty format
+ 
+ @param o
+          an object to serialize
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <param name="pretty" type="boolean"/>
+      <exception name="JsonGenerationException" type="org.codehaus.jackson.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="org.codehaus.jackson.map.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string
+ 
+ @param o
+          an object to serialize
+ @param pretty
+          whether in a pretty format or not
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1_5Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1.5 is enabled by default via
+ configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="createTimelineAbout" return="org.apache.hadoop.yarn.api.records.timeline.TimelineAbout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineTokenServiceAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="buildTimelineTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="generateDefaultFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="generateFlowNameTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow name tag.
+
+ @param flowName flow name that identifies a distinct flow application which
+                 can be run repeatedly over time
+ @return flow name tag.]]>
+      </doc>
+    </method>
+    <method name="generateFlowVersionTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowVersion" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow version tag.
+
+ @param flowVersion flow version that keeps track of the changes made to the
+                    flow
+ @return flow version tag.]]>
+      </doc>
+    </method>
+    <method name="generateFlowRunIdTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowRunId" type="long"/>
+      <doc>
+      <![CDATA[Generate flow run ID tag.
+
+ @param flowRunId flow run ID that identifies one instance (or specific
+                  execution) of that flow
+ @return flow run id tag.]]>
+      </doc>
+    </method>
+    <field name="FLOW_NAME_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The helper class for the timeline module.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+</package>
+<package name="org.apache.hadoop.yarn.webapp.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml
new file mode 100644
index 0000000000000..5ca1716cca2d7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml
@@ -0,0 +1,3964 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:54:25 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Common 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.4.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Common 3.2.4 -->
+<package name="org.apache.hadoop.yarn">
+  <!-- start class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <class name="ContainerLogAppender" extends="org.apache.log4j.FileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <method name="getTotalLogFileSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTotalLogFileSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logSize" type="long"/>
+      <doc>
+      <![CDATA[Setter so that log4j can configure it from the
+  configuration(log4j.properties).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <class name="ContainerRollingLogAppender" extends="org.apache.log4j.RollingFileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerRollingLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+  <class name="YarnUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="YarnUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.Thread"/>
+      <param name="e" type="java.lang.Throwable"/>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ In the main entry point.  It is intended to try and cleanly shut down
+ programs using the YARN Event framework.
+ 
+ Note: Right now it only will shut down the program if a Error is caught, but
+ not any other exception.  Anything else is just logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.api">
+</package>
+<package name="org.apache.hadoop.yarn.client">
+  <!-- start class org.apache.hadoop.yarn.client.AHSProxy -->
+  <class name="AHSProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ahsAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="rmAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.AHSProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <class name="ClientRMProxy" extends="org.apache.hadoop.yarn.client.RMProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configuration" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a proxy to the ResourceManager for the specified protocol.
+ @param configuration Configuration with all the required information.
+ @param protocol Client protocol for which proxy is being requested.
+ @param <T> Type of proxy.
+ @return Proxy to the ResourceManager for the specified client protocol.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the token service name to be used for RMDelegationToken. Depending
+ on whether HA is enabled or not, this method generates the appropriate
+ service name as a comma-separated list of service addresses.
+
+ @param conf Configuration corresponding to the cluster we need the
+             RMDelegationToken for
+ @return - Service name for RMDelegationToken]]>
+      </doc>
+    </method>
+    <method name="getAMRMTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="address" type="java.lang.String"/>
+      <param name="defaultAddr" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.NMProxy -->
+  <class name="NMProxy" extends="org.apache.hadoop.yarn.client.ServerProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.NMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.RMProxy -->
+  <class name="RMProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMProxy"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="retryTime" type="long"/>
+      <param name="retryInterval" type="long"/>
+      <param name="isHAEnabled" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch retry policy from Configuration and create the
+ retry policy with specified retryTime and retry interval.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.RMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ServerProxy -->
+  <class name="ServerProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServerProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxWaitTimeStr" type="java.lang.String"/>
+      <param name="defMaxWaitTime" type="long"/>
+      <param name="connectRetryIntervalStr" type="java.lang.String"/>
+      <param name="defRetryInterval" type="long"/>
+    </method>
+    <method name="createRetriableProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+      <param name="retryPolicy" type="org.apache.hadoop.io.retry.RetryPolicy"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ServerProxy -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <class name="AppAdminClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAdminClient"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAppAdminClient" return="org.apache.hadoop.yarn.client.api.AppAdminClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appType" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[<p>
+ Create a new instance of AppAdminClient.
+ </p>
+
+ @param appType application type
+ @param conf configuration
+ @return app admin client]]>
+      </doc>
+    </method>
+    <method name="actionLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Launch a new YARN application.
+ </p>
+
+ @param fileName specification of application
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStop" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Stop a YARN application (attempt to stop gracefully before killing the
+ application). In the case of a long-running service, the service may be
+ restarted later.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStart" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Start a YARN application from a previously saved specification. In the
+ case of a long-running service, the service must have been previously
+ launched/started and then stopped, or previously saved but not started.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionSave" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Save the specification for a YARN application / long-running service.
+ The application may be started later.
+ </p>
+
+ @param fileName specification of application to save
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionDestroy" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Remove the specification and all application data for a YARN application.
+ The application cannot be running.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionFlex" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentCounts" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Change the number of running containers for a component of a YARN
+ application / long-running service.
+ </p>
+
+ @param appName the name of the application
+ @param componentCounts map of component name to new component count or
+                        amount to change existing component count (e.g.
+                        5, +5, -5)
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="enableFastLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="destinationFolder" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Upload AM dependencies to HDFS. This makes future application launches
+ faster since the dependencies do not have to be uploaded on each launch.
+ </p>
+
+ @param destinationFolder
+          an optional HDFS folder where dependency tarball will be uploaded
+ @return exit code
+ @throws IOException
+           IOException
+ @throws YarnException
+           exception in client or server]]>
+      </doc>
+    </method>
+    <method name="getStatusString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdOrName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get detailed app specific status string for a YARN application.
+ </p>
+
+ @param appIdOrName appId or appName
+ @return status string
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="initiateUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.lang.String"/>
+      <param name="autoFinalize" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initiate upgrade of a long running service.
+
+ @param appName      the name of the application.
+ @param fileName     specification of application upgrade to save.
+ @param autoFinalize when true, finalization of upgrade will be done
+                     automatically.
+ @return exit code
+ @throws IOException   IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeComponents" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade components of a long running service.
+
+ @param appName    the name of the application.
+ @param components the name of the components.]]>
+      </doc>
+    </method>
+    <method name="actionCleanUp" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Operation to be performed by the RM after an application has completed.
+
+ @param appName  the name of the application.
+ @param userName the name of the user.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <param name="version" type="java.lang.String"/>
+      <param name="containerStates" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="actionUpgradeExpress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Express upgrade a long running service.
+
+ @param appName  the name of the application
+ @param fileName specification of application upgrade to save.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="actionCancelUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Cancels the upgrade of the service.
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="actionDecommissionInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Decommission component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <field name="YARN_APP_ADMIN_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Client for managing applications.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <class name="TimelineClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="TimelineClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineClient" return="org.apache.hadoop.yarn.client.api.TimelineClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an instance of the timeline v.1.x client.
+ The current UGI when the user initialize the client will be used to do the
+ put and the delegation token operations. The current user may use
+ {@link UserGroupInformation#doAs} another user to construct and initialize
+ a timeline client if the following operations are supposed to be conducted
+ by that user.
+
+ @return the created timeline client instance]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+ </p>
+ 
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="groupId" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param appAttemptId {@link ApplicationAttemptId}
+ @param groupId {@link TimelineEntityGroupId}
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+ </p>
+ 
+ @param domain
+          an {@link TimelineDomain} object
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param domain
+          an {@link TimelineDomain} object
+ @param appAttemptId {@link ApplicationAttemptId}
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to the timeline server in a
+ secure way.
+ </p>
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to the timeline server
+ @return a delegation token ({@link Token}) that can be used to talk to the
+         timeline server
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Renew a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to renew
+ @return the new expiration time
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Cancel a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to cancel
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to post some information in terms of a
+ number of conceptual entities. This client library needs to be used along
+ with Timeline V.1.x server versions.
+ Refer {@link TimelineV2Client} for ATS V2 interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+  <class name="TimelineReaderClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineReaderClient" return="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of Timeline Reader Client.]]>
+      </doc>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application entity.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity of the application
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entity.
+ @param appAttemptId application attempt id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity associated with application attempt
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entities.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of application attempt entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets Timeline entity for the container.
+ @param containerId container id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return timeline entity for container
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets container entities for an application.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to get Timeline Entities associated with
+ application, application attempt or containers. This client library needs to
+ be used along with time line v.2 server version.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+  <!-- start class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+  <class name="TimelineReaderClientImpl" extends="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClientImpl"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="encodeValue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="value" type="java.lang.String"/>
+      <exception name="UnsupportedEncodingException" type="java.io.UnsupportedEncodingException"/>
+    </method>
+    <method name="doGetUri" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="base" type="java.net.URI"/>
+      <param name="path" type="java.lang.String"/>
+      <param name="params" type="javax.ws.rs.core.MultivaluedMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implementation of TimelineReaderClient interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+</package>
+<package name="org.apache.hadoop.yarn.event">
+  <!-- start class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <class name="AbstractEvent" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Event"/>
+    <constructor name="AbstractEvent" type="TYPE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractEvent" type="TYPE, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="TYPE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Parent class of all the events. All events extend this class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <!-- start class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <class name="AsyncDispatcher" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Dispatcher"/>
+    <constructor name="AsyncDispatcher"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.util.concurrent.BlockingQueue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set a name for this dispatcher thread.
+ @param dispatcherName name of the dispatcher thread]]>
+      </doc>
+    </constructor>
+    <method name="disableExitOnDispatchException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="setDrainEventsOnStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="dispatch"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.event.Event"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEventThreadWaiting" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDrained" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isStopped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="eventDispatchers" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Dispatches {@link Event}s in a separate thread. Currently only single thread
+ does that. Potentially there could be multiple channels for each event type
+ class and a thread pool can be used to dispatch the events.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <interface name="Dispatcher"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <doc>
+    <![CDATA[Event Dispatcher interface. It dispatches events to registered 
+ event handlers based on event types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Event -->
+  <interface name="Event"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="TYPE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface defining events api.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Event -->
+  <!-- start interface org.apache.hadoop.yarn.event.EventHandler -->
+  <interface name="EventHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="handle"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="T"/>
+    </method>
+    <doc>
+    <![CDATA[Interface for handling events of type T
+
+ @param <T> parameterized event of type T]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.EventHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.factories">
+</package>
+<package name="org.apache.hadoop.yarn.factory.providers">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <class name="AggregatedLogFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AggregatedLogFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <class name="AggregatedLogFormat.LogKey" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="LogKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="org.apache.hadoop.yarn.api.records.ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+  <class name="AggregatedLogFormat.LogReader" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @return the application owner.
+ @throws IOException if we can not get the application owner.]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.io.DataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next key and return the value-stream.
+ 
+ @param key the log key
+ @return the valueStream if there are more keys or null otherwise
+ @throws IOException if we can not get the dataInputStream
+ for the next key]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the valueStream
+ @param writer the log writer
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the value stream
+ @param writer the log writer
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the print stream
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container log by specifying
+ the container log type.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container for the specific bytes.
+
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log upload time stamp
+ @param bytes the output size of the log
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the output print stream
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+  <class name="LogAggregationFileController" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationFileController"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="controllerName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initialize the log file controller.
+ @param conf the Configuration
+ @param controllerName the log controller class name]]>
+      </doc>
+    </method>
+    <method name="initInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Derived classes initialize themselves using this method.
+ @param conf the Configuration]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the remote root log directory.
+ @return the remote root log directory path]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDirSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation directory suffix.
+ @return the log aggregation directory suffix]]>
+      </doc>
+    </method>
+    <method name="initializeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize the writer.
+ @param context the {@link LogAggregationFileControllerContext}
+ @throws IOException if fails to initialize the writer]]>
+      </doc>
+    </method>
+    <method name="closeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="LogAggregationDFSException" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationDFSException"/>
+      <doc>
+      <![CDATA[Close the writer.
+ @throws LogAggregationDFSException if the closing of the writer fails
+         (for example due to HDFS quota being exceeded)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logKey" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <param name="logValue" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the log content.
+ @param logKey the log key
+ @param logValue the log content
+ @throws IOException if fails to write the logs]]>
+      </doc>
+    </method>
+    <method name="postWrite"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Operations needed after write the log content.
+ @param record the {@link LogAggregationFileControllerContext}
+ @throws Exception if anything fails]]>
+      </doc>
+    </method>
+    <method name="closePrintStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+    </method>
+    <method name="readAggregatedLogs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="os" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Output container log.
+ @param logRequest {@link ContainerLogsRequest}
+ @param os the output stream
+ @return true if we can read the aggregated logs successfully
+ @throws IOException if we can not access the log file.]]>
+      </doc>
+    </method>
+    <method name="readAggregatedLogsMeta" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a list of {@link ContainerLogMeta} for an application
+ from Remote FileSystem.
+
+ @param logRequest {@link ContainerLogsRequest}
+ @return a list of {@link ContainerLogMeta}
+ @throws IOException if there is no available log file]]>
+      </doc>
+    </method>
+    <method name="renderAggregatedLogsBlock"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="html" type="org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block"/>
+      <param name="context" type="org.apache.hadoop.yarn.webapp.View.ViewContext"/>
+      <doc>
+      <![CDATA[Render Aggregated Logs block.
+ @param html the html
+ @param context the ViewContext]]>
+      </doc>
+    </method>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @param aggregatedLogPath the aggregatedLog path
+ @param appId the ApplicationId
+ @return the application owner
+ @throws IOException if we can not get the application owner]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @param aggregatedLogPath the aggregatedLog path.
+ @param appId the ApplicationId
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls]]>
+      </doc>
+    </method>
+    <method name="verifyAndCreateRemoteLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify and create the remote log directory.]]>
+      </doc>
+    </method>
+    <method name="createAppDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create remote Application directory for log aggregation.
+ @param user the user
+ @param appId the application ID
+ @param userUgi the UGI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRemoteNodeLogFileForApp" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <doc>
+      <![CDATA[Get the remote aggregated log path.
+ @param appId the ApplicationId
+ @param user the Application Owner
+ @param nodeId the NodeManager Id
+ @return the remote aggregated log path]]>
+      </doc>
+    </method>
+    <method name="getRemoteAppLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="appOwner" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the remote application directory for log aggregation.
+ @param appId the Application ID
+ @param appOwner the Application Owner
+ @return the remote application directory
+ @throws IOException if can not find the remote application directory]]>
+      </doc>
+    </method>
+    <method name="cleanOldLogs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="remoteNodeLogFileForApp" type="org.apache.hadoop.fs.Path"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="aggregatedLogSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create the aggregated log suffix. The LogAggregationFileController
+ should call this to get the suffix and append the suffix to the end
+ of each log. This would keep the aggregated log format consistent.
+
+ @param fileName the File Name
+ @return the aggregated log suffix String]]>
+      </doc>
+    </method>
+    <method name="isFsSupportsChmod" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="belongsToAppAttempt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TLDIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the top level directory under which app directories will be
+ created.]]>
+      </doc>
+    </field>
+    <field name="APP_DIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the Application directory.]]>
+      </doc>
+    </field>
+    <field name="APP_LOG_FILE_UMASK" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Umask for the log file.]]>
+      </doc>
+    </field>
+    <field name="LOG_AGGREGATION_FS_SUPPORT_APPEND" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDir" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDirSuffix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="retentionSize" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fileControllerName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fsSupportsChmod" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class to implement Log Aggregation File Controller.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.ifile">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.tfile">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels.event">
+</package>
+<package name="org.apache.hadoop.yarn.security">
+  <!-- start class org.apache.hadoop.yarn.security.AccessRequest -->
+  <class name="AccessRequest" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessRequest" type="org.apache.hadoop.yarn.security.PrivilegedEntity, org.apache.hadoop.security.UserGroupInformation, org.apache.hadoop.yarn.security.AccessType, java.lang.String, java.lang.String, java.lang.String, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAccessType" return="org.apache.hadoop.yarn.security.AccessType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEntity" return="org.apache.hadoop.yarn.security.PrivilegedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getForwardedAddresses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRemoteAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This request object contains all the context information to determine whether
+ a user has permission to access the target entity.
+ user       : the user who's currently accessing
+ accessType : the access type against the entity.
+ entity     : the target object user is accessing.
+ appId      : the associated app Id for current access. This could be null
+              if no app is associated.
+ appName    : the associated app name for current access. This could be null if
+              no app is associated.
+ remoteAddress : The caller's remote ip address.
+ forwardedAddresses : In case this is an http request, this contains the
+                    originating IP address of a client connecting to a web
+                    server through an HTTP proxy or load balancer. This
+                    parameter is null, if it's a RPC request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AccessRequest -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <class name="AMRMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.AMRMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[AMRMTokenIdentifier is the TokenIdentifier to be used by
+ ApplicationMasters to authenticate to the ResourceManager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <class name="AMRMTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="AMRMTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <class name="ContainerManagerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerManagerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <class name="ContainerTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="Use one of the other constructors instead.">
+      <doc>
+      <![CDATA[Creates a instance.
+
+ @param appSubmitter appSubmitter
+ @param containerID container ID
+ @param creationTime creation time
+ @param expiryTimeStamp expiry timestamp
+ @param hostName hostname
+ @param logAggregationContext log aggregation context
+ @param masterKeyId master key ID
+ @param priority priority
+ @param r resource needed by the container
+ @param rmIdentifier ResourceManager identifier
+ @deprecated Use one of the other constructors instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convenience Constructor for existing clients.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a Container Token Identifier.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId
+ @param allocationTags Set of allocation Tags.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC layer/SecretManager.]]>
+      </doc>
+    </constructor>
+    <method name="getContainerID" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNmHostAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getExpiryTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMasterKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRMIdentifier" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the RMIdentifier of RM in which containers are allocated.
+ @return RMIdentifier]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ContainerType of container to allocate.
+ @return ContainerType]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ExecutionType of container to allocate
+ @return ExecutionType]]>
+      </doc>
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Container version
+ @return container version]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the node-label-expression in the original ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllcationTags" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TokenIdentifier for a container. Encodes {@link ContainerId},
+ {@link Resource} needed by the container and the target NMs host-address.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <class name="ContainerTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="ContainerTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <class name="NMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, org.apache.hadoop.yarn.api.records.NodeId, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC/Secret manager]]>
+      </doc>
+    </constructor>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.NMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+  <class name="SchedulerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.admin">
+  <!-- start class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+  <class name="AdminSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AdminSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.client">
+  <!-- start class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <class name="BaseClientToAMTokenSecretManager" extends="org.apache.hadoop.security.token.SecretManager"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BaseClientToAMTokenSecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A base {@link SecretManager} for AMs to extend and validate Client-RM tokens
+ issued to clients by the RM using the underlying master-key shared by RM to
+ the AMs on their launch. All the methods are called by either Hadoop RPC or
+ YARN, so this class is strictly for the purpose of inherit/extend and
+ register with Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <class name="ClientRMSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientRMSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <class name="ClientTimelineSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientTimelineSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <class name="ClientToAMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClientToAMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationAttemptID" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClientName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ClientToAMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <class name="ClientToAMTokenSecretManager" extends="org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenSecretManager" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="retrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+    </method>
+    <method name="getMasterKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptID" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="setMasterKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+    </method>
+    <doc>
+    <![CDATA[A simple {@link SecretManager} for AMs to validate Client-RM tokens issued to
+ clients by the RM using the underlying master-key shared by RM to the AMs on
+ their launch. All the methods are called by either Hadoop RPC or YARN, so
+ this class is strictly for the purpose of inherit/extend and register with
+ Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <class name="RMDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="RMDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new delegation token identifier
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Delegation Token Identifier that identifies the delegation tokens from the 
+ Resource Manager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <class name="RMDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="RMDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <class name="TimelineDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new timeline delegation token identifier
+
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+  <class name="TimelineDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="TimelineDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.sharedcache">
+  <!-- start interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <interface name="SharedCacheChecksum"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="computeChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Calculate the checksum of the passed input stream.
+
+ @param in <code>InputStream</code> to be checksumed
+ @return the message digest of the input stream
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <!-- start class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+  <class name="SharedCacheChecksumFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheChecksumFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getChecksum" return="org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a new <code>SharedCacheChecksum</code> object based on the configurable
+ algorithm implementation
+ (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+
+ @return <code>SharedCacheChecksum</code> object]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+</package>
+<package name="org.apache.hadoop.yarn.state">
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <class name="InvalidStateTransitionException" extends="org.apache.hadoop.yarn.state.InvalidStateTransitonException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidStateTransitionException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The exception that happens when you call invalid state transition.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <class name="InvalidStateTransitonException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link InvalidStateTransitionException} instead.">
+    <constructor name="InvalidStateTransitonException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCurrentState" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEvent" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[@deprecated Use {@link InvalidStateTransitionException} instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <!-- start interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <interface name="MultipleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ @return the postState. Post state must be one of the 
+                      valid post states registered in StateMachine.
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. 
+ Post state is decided by Transition hook. Post state must be one of the 
+ valid post states registered in StateMachine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <interface name="SingleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ 
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. This lead to state machine to move to 
+ the post state as registered in the state machine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateMachine -->
+  <interface name="StateMachine"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getCurrentState" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doTransition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="event" type="EVENT"/>
+      <exception name="InvalidStateTransitionException" type="org.apache.hadoop.yarn.state.InvalidStateTransitionException"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateMachine -->
+  <!-- start class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <class name="StateMachineFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StateMachineFactory" type="STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+
+ This is the only constructor in the API.]]>
+      </doc>
+    </constructor>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postStates" type="java.util.Set"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.MultipleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postStates valid post-transition states
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="installTopology" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a StateMachineFactory just like {@code this}, except that if
+         you won't need any synchronization to build a state machine
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+         The only way you could distinguish the returned
+         StateMachineFactory from {@code this} would be by
+         measuring the performance of the derived 
+         {@code StateMachine} you can get from it.
+
+ Calling this is optional.  It doesn't change the semantics of the factory,
+   if you call it then when you use the factory there is no synchronization.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+      <param name="listener" type="org.apache.hadoop.yarn.state.StateTransitionListener"/>
+      <doc>
+      <![CDATA[A StateMachine that accepts a transition listener.
+ @param operand the object upon which the returned
+                {@link StateMachine} will operate.
+ @param initialState the state in which the returned
+                {@link StateMachine} will start.
+ @param listener An implementation of a {@link StateTransitionListener}.
+ @return A (@link StateMachine}.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+    </method>
+    <method name="generateStateGraph" return="org.apache.hadoop.yarn.state.Graph"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a graph represents the state graph of this StateMachine
+ @param name graph name
+ @return Graph object generated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[State machine topology.
+ This object is semantically immutable.  If you have a
+ StateMachineFactory there's no operation in the API that changes
+ its semantic properties.
+
+ @param <OPERAND> The object type on which this state machine operates.
+ @param <STATE> The state of the entity.
+ @param <EVENTTYPE> The external eventType to be handled.
+ @param <EVENT> The event object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+  <interface name="StateTransitionListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="preTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="eventToBeProcessed" type="EVENT"/>
+      <doc>
+      <![CDATA[Pre Transition Hook. This will be called before transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param eventToBeProcessed Incoming Event.]]>
+      </doc>
+    </method>
+    <method name="postTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="afterState" type="STATE"/>
+      <param name="processedEvent" type="EVENT"/>
+      <doc>
+      <![CDATA[Post Transition Hook. This will be called after the transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param afterState State after transition.
+ @param processedEvent Processed Event.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A State Transition Listener.
+ It exposes a pre and post transition hook called before and
+ after the transition.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+  <!-- start class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <class name="AbstractLivelinessMonitor" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String, org.apache.hadoop.yarn.util.Clock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="expire"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="setExpireInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="expireInterval" type="int"/>
+    </method>
+    <method name="getExpireInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="o" type="O"/>
+    </method>
+    <method name="setMonitorInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="monitorInterval" type="long"/>
+    </method>
+    <method name="receivedPing"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+      <param name="expireTime" type="long"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="resetTimer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResetTimeOnStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="resetTimeOnStart" type="boolean"/>
+    </method>
+    <field name="DEFAULT_EXPIRE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A simple liveliness monitor with which clients can register, trust the
+ component to monitor liveliness, get a call-back on expiry and then finally
+ unregister.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <!-- start class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="org.apache.hadoop.util.ApplicationClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[This type has been deprecated in favor of
+ {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ ApplicationClassLoader should use that type instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <!-- start class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <class name="BoundedAppender" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BoundedAppender" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append" return="org.apache.hadoop.yarn.util.BoundedAppender"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="csq" type="java.lang.CharSequence"/>
+      <doc>
+      <![CDATA[Append a {@link CharSequence} considering {@link #limit}, truncating
+ from the head of {@code csq} or {@link #messages} when necessary.
+
+ @param csq the {@link CharSequence} to append
+ @return this]]>
+      </doc>
+    </method>
+    <method name="length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current length of messages considering truncates
+ without header and ellipses.
+
+ @return current length]]>
+      </doc>
+    </method>
+    <method name="getLimit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a string representation of the actual contents, displaying also a
+ header and ellipses when there was a truncate.
+
+ @return String representation of the {@link #messages}]]>
+      </doc>
+    </method>
+    <field name="TRUNCATED_MESSAGES_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link CharSequence} appender that considers its {@link #limit} as upper
+ bound.
+ <p>
+ When {@link #limit} would be reached on append, past messages will be
+ truncated from head, and a header telling the user about truncation will be
+ prepended, with ellipses in between header and messages.
+ <p>
+ Note that header and ellipses are not counted against {@link #limit}.
+ <p>
+ An example:
+
+ <pre>
+ {@code
+   // At the beginning it's an empty string
+   final Appendable shortAppender = new BoundedAppender(80);
+   // The whole message fits into limit
+   shortAppender.append(
+       "message1 this is a very long message but fitting into limit\n");
+   // The first message is truncated, the second not
+   shortAppender.append("message2 this is shorter than the previous one\n");
+   // The first message is deleted, the second truncated, the third
+   // preserved
+   shortAppender.append("message3 this is even shorter message, maybe.\n");
+   // The first two are deleted, the third one truncated, the last preserved
+   shortAppender.append("message4 the shortest one, yet the greatest :)");
+   // Current contents are like this:
+   // Diagnostic messages truncated, showing last 80 chars out of 199:
+   // ...s is even shorter message, maybe.
+   // message4 the shortest one, yet the greatest :)
+ }
+ </pre>
+ <p>
+ Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
+ just like in {@link StringBuilder#append(CharSequence) original
+ implementation}.
+ <p>
+ Note that this class is not thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <!-- start interface org.apache.hadoop.yarn.util.Clock -->
+  <interface name="Clock"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A simple clock interface that gives you time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.util.Clock -->
+  <!-- start class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <class name="ConverterUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConverterUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPathFromYarnURL" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[return a hadoop path from a given url
+ This method is deprecated, use {@link URL#toPath()} instead.
+ 
+ @param url
+          url to convert
+ @return path from {@link URL}
+ @throws URISyntaxException]]>
+      </doc>
+    </method>
+    <method name="getYarnUrlFromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getYarnUrlFromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recordFactory" type="org.apache.hadoop.yarn.factories.RecordFactory"/>
+      <param name="applicationIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="serviceAddr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service. Supposed
+ to be used for tokens other than RMDelegationToken. For
+ RMDelegationToken, use
+ {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
+ org.apache.hadoop.io.Text)} instead.
+
+ @param protoToken the yarn token
+ @param serviceAddr the connect address for the service
+ @return rpc token]]>
+      </doc>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service.
+
+ @param protoToken the yarn token
+ @param service the service for the token]]>
+      </doc>
+    </method>
+    <field name="APPLICATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONTAINER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_ATTEMPT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains a set of utilities which help converting data structures
+ from/to 'serializableFormat' to/from hadoop/nativejava data structures.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <!-- start class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <class name="MonotonicClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="MonotonicClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current time from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ @return a monotonic clock that counts in milliseconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A monotonic clock from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ This is appropriate to use when computing how much longer to wait for an
+ interval to expire.
+ This function can return a negative value and it must be handled correctly
+ by callers. See the documentation of System#nanoTime for caveats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <!-- start class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <class name="ResourceCalculatorProcessTree" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceCalculatorProcessTree" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create process-tree instance with specified root process.
+
+ Subclass must override this.
+ @param root process-tree root-process]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initialize the object.
+ @throws YarnException Throws an exception on error.]]>
+      </doc>
+    </method>
+    <method name="updateProcessTree"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the process-tree with latest state.
+
+ Each call to this function should increment the age of the running
+ processes that already exist in the process tree. Age is used other API's
+ of the interface.]]>
+      </doc>
+    </method>
+    <method name="getProcessTreeDump" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a dump of the process-tree.
+
+ @return a string concatenating the dump of information of all the processes
+         in the process-tree]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree.
+
+ @return virtual memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree.
+
+ @return rss memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return virtual memory used by the process-tree in bytes for
+ processes older than the specified age, {@link #UNAVAILABLE} if it
+ cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return rss memory used by the process-tree in bytes for
+ processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU time in millisecond used by all the processes in the
+ process-tree since the process-tree was created
+
+ @return cumulative CPU time in millisecond since the process-tree
+ created, {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercent" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU usage by all the processes in the process-tree based on
+ average between samples as a ratio of overall CPU cycles similar to top.
+ Thus, if 2 out of 4 cores are used this should return 200.0.
+ Note: UNAVAILABLE will be returned in case when CPU usage is not
+ available. It is NOT advised to return any other error code.
+
+ @return percentage CPU usage since the process-tree was created,
+ {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available.]]>
+      </doc>
+    </method>
+    <method name="checkPidPgrpidForMatch" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify that the tree process id is same as its process group id.
+ @return true if the process id matches else return false.]]>
+      </doc>
+    </method>
+    <method name="getResourceCalculatorProcessTree" return="org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <param name="clazz" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create the ResourceCalculatorProcessTree rooted to specified process 
+ from the class name and configure it. If class name is null, this method
+ will try and return a process tree plugin available for this system.
+
+ @param pid process pid of the root of the process tree
+ @param clazz class-name
+ @param conf configure the plugin with this.
+
+ @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
+         is not available for this system.]]>
+      </doc>
+    </method>
+    <field name="UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interface class to obtain process resource usage
+ NOTE: This class should not be used by external users, but only by external
+ developers to extend and include their own process-tree implementation, 
+ especially for platforms other than Linux and Windows.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <!-- start class org.apache.hadoop.yarn.util.SystemClock -->
+  <class name="SystemClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="SystemClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.yarn.util.SystemClock"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current time from the system
+ clock in milliseconds.
+ 
+ NOTE: Do not use this to calculate a duration of expire or interval to sleep,
+ because it will be broken by settimeofday. Please use {@link MonotonicClock}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.SystemClock -->
+  <!-- start class org.apache.hadoop.yarn.util.UTCClock -->
+  <class name="UTCClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="UTCClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current UTC time in
+ milliseconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.UTCClock -->
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+<package name="org.apache.hadoop.yarn.util.timeline">
+  <!-- start class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+  <class name="TimelineUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string not in a pretty format
+ 
+ @param o
+          an object to serialize
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <param name="pretty" type="boolean"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string
+ 
+ @param o
+          an object to serialize
+ @param pretty
+          whether in a pretty format or not
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1_5Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1.5 is enabled by default via
+ configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="createTimelineAbout" return="org.apache.hadoop.yarn.api.records.timeline.TimelineAbout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineTokenServiceAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="buildTimelineTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="generateDefaultFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="generateFlowNameTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow name tag.
+
+ @param flowName flow name that identifies a distinct flow application which
+                 can be run repeatedly over time
+ @return flow name tag.]]>
+      </doc>
+    </method>
+    <method name="shortenFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Shortens the flow name for the configured size by removing UUID if present.
+
+ @param flowName which has to be shortened
+ @param conf to resize the flow name
+ @return shortened flowName]]>
+      </doc>
+    </method>
+    <method name="generateFlowVersionTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowVersion" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow version tag.
+
+ @param flowVersion flow version that keeps track of the changes made to the
+                    flow
+ @return flow version tag.]]>
+      </doc>
+    </method>
+    <method name="generateFlowRunIdTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowRunId" type="long"/>
+      <doc>
+      <![CDATA[Generate flow run ID tag.
+
+ @param flowRunId flow run ID that identifies one instance (or specific
+                  execution) of that flow
+ @return flow run id tag.]]>
+      </doc>
+    </method>
+    <field name="FLOW_NAME_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The helper class for the timeline module.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+</package>
+<package name="org.apache.hadoop.yarn.webapp.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.3.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.3.xml
new file mode 100644
index 0000000000000..4797c30f5c32a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.3.xml
@@ -0,0 +1,3975 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:08:36 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Common 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Common 3.3.3 -->
+<package name="org.apache.hadoop.yarn">
+  <!-- start class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <class name="ContainerLogAppender" extends="org.apache.log4j.FileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <method name="getTotalLogFileSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTotalLogFileSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logSize" type="long"/>
+      <doc>
+      <![CDATA[Setter so that log4j can configure it from the
+  configuration(log4j.properties).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <class name="ContainerRollingLogAppender" extends="org.apache.log4j.RollingFileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerRollingLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+  <class name="YarnUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="YarnUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.Thread"/>
+      <param name="e" type="java.lang.Throwable"/>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ In the main entry point.  It is intended to try and cleanly shut down
+ programs using the YARN Event framework.
+ 
+ Note: Right now it only will shut down the program if a Error is caught, but
+ not any other exception.  Anything else is just logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.api">
+</package>
+<package name="org.apache.hadoop.yarn.client">
+  <!-- start class org.apache.hadoop.yarn.client.AHSProxy -->
+  <class name="AHSProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ahsAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="rmAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.AHSProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <class name="ClientRMProxy" extends="org.apache.hadoop.yarn.client.RMProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configuration" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a proxy to the ResourceManager for the specified protocol.
+ @param configuration Configuration with all the required information.
+ @param protocol Client protocol for which proxy is being requested.
+ @param <T> Type of proxy.
+ @return Proxy to the ResourceManager for the specified client protocol.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the token service name to be used for RMDelegationToken. Depending
+ on whether HA is enabled or not, this method generates the appropriate
+ service name as a comma-separated list of service addresses.
+
+ @param conf Configuration corresponding to the cluster we need the
+             RMDelegationToken for
+ @return - Service name for RMDelegationToken]]>
+      </doc>
+    </method>
+    <method name="getAMRMTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="address" type="java.lang.String"/>
+      <param name="defaultAddr" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.NMProxy -->
+  <class name="NMProxy" extends="org.apache.hadoop.yarn.client.ServerProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.NMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.RMProxy -->
+  <class name="RMProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMProxy"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="retryTime" type="long"/>
+      <param name="retryInterval" type="long"/>
+      <param name="isHAEnabled" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch retry policy from Configuration and create the
+ retry policy with specified retryTime and retry interval.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.RMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ServerProxy -->
+  <class name="ServerProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServerProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxWaitTimeStr" type="java.lang.String"/>
+      <param name="defMaxWaitTime" type="long"/>
+      <param name="connectRetryIntervalStr" type="java.lang.String"/>
+      <param name="defRetryInterval" type="long"/>
+    </method>
+    <method name="createRetriableProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+      <param name="retryPolicy" type="org.apache.hadoop.io.retry.RetryPolicy"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ServerProxy -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <class name="AppAdminClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAdminClient"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAppAdminClient" return="org.apache.hadoop.yarn.client.api.AppAdminClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appType" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[<p>
+ Create a new instance of AppAdminClient.
+ </p>
+
+ @param appType application type
+ @param conf configuration
+ @return app admin client]]>
+      </doc>
+    </method>
+    <method name="actionLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Launch a new YARN application.
+ </p>
+
+ @param fileName specification of application
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStop" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Stop a YARN application (attempt to stop gracefully before killing the
+ application). In the case of a long-running service, the service may be
+ restarted later.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStart" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Start a YARN application from a previously saved specification. In the
+ case of a long-running service, the service must have been previously
+ launched/started and then stopped, or previously saved but not started.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionSave" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Save the specification for a YARN application / long-running service.
+ The application may be started later.
+ </p>
+
+ @param fileName specification of application to save
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionDestroy" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Remove the specification and all application data for a YARN application.
+ The application cannot be running.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionFlex" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentCounts" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Change the number of running containers for a component of a YARN
+ application / long-running service.
+ </p>
+
+ @param appName the name of the application
+ @param componentCounts map of component name to new component count or
+                        amount to change existing component count (e.g.
+                        5, +5, -5)
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="enableFastLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="destinationFolder" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Upload AM dependencies to HDFS. This makes future application launches
+ faster since the dependencies do not have to be uploaded on each launch.
+ </p>
+
+ @param destinationFolder
+          an optional HDFS folder where dependency tarball will be uploaded
+ @return exit code
+ @throws IOException
+           IOException
+ @throws YarnException
+           exception in client or server]]>
+      </doc>
+    </method>
+    <method name="getStatusString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdOrName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get detailed app specific status string for a YARN application.
+ </p>
+
+ @param appIdOrName appId or appName
+ @return status string
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="initiateUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.lang.String"/>
+      <param name="autoFinalize" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initiate upgrade of a long running service.
+
+ @param appName      the name of the application.
+ @param fileName     specification of application upgrade to save.
+ @param autoFinalize when true, finalization of upgrade will be done
+                     automatically.
+ @return exit code
+ @throws IOException   IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeComponents" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade components of a long running service.
+
+ @param appName    the name of the application.
+ @param components the name of the components.]]>
+      </doc>
+    </method>
+    <method name="actionCleanUp" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Operation to be performed by the RM after an application has completed.
+
+ @param appName  the name of the application.
+ @param userName the name of the user.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <param name="version" type="java.lang.String"/>
+      <param name="containerStates" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="actionUpgradeExpress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Express upgrade a long running service.
+
+ @param appName  the name of the application
+ @param fileName specification of application upgrade to save.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="actionCancelUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Cancels the upgrade of the service.
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="actionDecommissionInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Decommission component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <field name="YARN_APP_ADMIN_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Client for managing applications.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <class name="TimelineClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="TimelineClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineClient" return="org.apache.hadoop.yarn.client.api.TimelineClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an instance of the timeline v.1.x client.
+ The current UGI when the user initialize the client will be used to do the
+ put and the delegation token operations. The current user may use
+ {@link UserGroupInformation#doAs} another user to construct and initialize
+ a timeline client if the following operations are supposed to be conducted
+ by that user.
+
+ @return the created timeline client instance]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+ </p>
+ 
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="groupId" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param appAttemptId {@link ApplicationAttemptId}
+ @param groupId {@link TimelineEntityGroupId}
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+ </p>
+ 
+ @param domain
+          an {@link TimelineDomain} object
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param domain
+          an {@link TimelineDomain} object
+ @param appAttemptId {@link ApplicationAttemptId}
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to the timeline server in a
+ secure way.
+ </p>
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to the timeline server
+ @return a delegation token ({@link Token}) that can be used to talk to the
+         timeline server
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Renew a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to renew
+ @return the new expiration time
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Cancel a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to cancel
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to post some information in terms of a
+ number of conceptual entities. This client library needs to be used along
+ with Timeline V.1.x server versions.
+ Refer {@link TimelineV2Client} for ATS V2 interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+  <class name="TimelineReaderClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineReaderClient" return="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of Timeline Reader Client.]]>
+      </doc>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application entity.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity of the application
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entity.
+ @param appAttemptId application attempt id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity associated with application attempt
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entities.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of application attempt entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets Timeline entity for the container.
+ @param containerId container id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return timeline entity for container
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets container entities for an application.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to get Timeline Entities associated with
+ application, application attempt or containers. This client library needs to
+ be used along with time line v.2 server version.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+  <!-- start class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+  <class name="TimelineReaderClientImpl" extends="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClientImpl"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="doGetUri" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="base" type="java.net.URI"/>
+      <param name="path" type="java.lang.String"/>
+      <param name="params" type="javax.ws.rs.core.MultivaluedMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implementation of TimelineReaderClient interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+</package>
+<package name="org.apache.hadoop.yarn.event">
+  <!-- start class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <class name="AbstractEvent" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Event"/>
+    <constructor name="AbstractEvent" type="TYPE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractEvent" type="TYPE, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="TYPE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Parent class of all the events. All events extend this class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <!-- start class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <class name="AsyncDispatcher" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Dispatcher"/>
+    <constructor name="AsyncDispatcher"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.util.concurrent.BlockingQueue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set a name for this dispatcher thread.
+ @param dispatcherName name of the dispatcher thread]]>
+      </doc>
+    </constructor>
+    <method name="disableExitOnDispatchException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="setDrainEventsOnStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="dispatch"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.event.Event"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEventThreadWaiting" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDrained" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isStopped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics" type="org.apache.hadoop.yarn.metrics.EventTypeMetrics"/>
+      <param name="eventClass" type="java.lang.Class"/>
+    </method>
+    <field name="eventDispatchers" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Dispatches {@link Event}s in a separate thread. Currently only single thread
+ does that. Potentially there could be multiple channels for each event type
+ class and a thread pool can be used to dispatch the events.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <interface name="Dispatcher"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <doc>
+    <![CDATA[Event Dispatcher interface. It dispatches events to registered 
+ event handlers based on event types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Event -->
+  <interface name="Event"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="TYPE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface defining events api.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Event -->
+  <!-- start interface org.apache.hadoop.yarn.event.EventHandler -->
+  <interface name="EventHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="handle"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="T"/>
+    </method>
+    <doc>
+    <![CDATA[Interface for handling events of type T
+
+ @param <T> parameterized event of type T]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.EventHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.factories">
+</package>
+<package name="org.apache.hadoop.yarn.factory.providers">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <class name="AggregatedLogFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AggregatedLogFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <class name="AggregatedLogFormat.LogKey" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="LogKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="org.apache.hadoop.yarn.api.records.ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+  <class name="AggregatedLogFormat.LogReader" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @return the application owner.
+ @throws IOException if we can not get the application owner.]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.io.DataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next key and return the value-stream.
+ 
+ @param key the log key
+ @return the valueStream if there are more keys or null otherwise
+ @throws IOException if we can not get the dataInputStream
+ for the next key]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the valueStream
+ @param writer the log writer
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the value stream
+ @param writer the log writer
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the print stream
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container log by specifying
+ the container log type.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container for the specific bytes.
+
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log upload time stamp
+ @param bytes the output size of the log
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the output print stream
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+  <class name="LogAggregationFileController" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationFileController"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="controllerName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initialize the log file controller.
+ @param conf the Configuration
+ @param controllerName the log controller class name]]>
+      </doc>
+    </method>
+    <method name="initInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Derived classes initialize themselves using this method.
+ @param conf the Configuration]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the remote root log directory.
+ @return the remote root log directory path]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDirSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation directory suffix.
+ @return the log aggregation directory suffix]]>
+      </doc>
+    </method>
+    <method name="initializeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize the writer.
+ @param context the {@link LogAggregationFileControllerContext}
+ @throws IOException if fails to initialize the writer]]>
+      </doc>
+    </method>
+    <method name="closeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="LogAggregationDFSException" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationDFSException"/>
+      <doc>
+      <![CDATA[Close the writer.
+ @throws LogAggregationDFSException if the closing of the writer fails
+         (for example due to HDFS quota being exceeded)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logKey" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <param name="logValue" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the log content.
+ @param logKey the log key
+ @param logValue the log content
+ @throws IOException if fails to write the logs]]>
+      </doc>
+    </method>
+    <method name="postWrite"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Operations needed after write the log content.
+ @param record the {@link LogAggregationFileControllerContext}
+ @throws Exception if anything fails]]>
+      </doc>
+    </method>
+    <method name="closePrintStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+    </method>
+    <method name="readAggregatedLogs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="os" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Output container log.
+ @param logRequest {@link ContainerLogsRequest}
+ @param os the output stream
+ @return true if we can read the aggregated logs successfully
+ @throws IOException if we can not access the log file.]]>
+      </doc>
+    </method>
+    <method name="readAggregatedLogsMeta" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a list of {@link ContainerLogMeta} for an application
+ from Remote FileSystem.
+
+ @param logRequest {@link ContainerLogsRequest}
+ @return a list of {@link ContainerLogMeta}
+ @throws IOException if there is no available log file]]>
+      </doc>
+    </method>
+    <method name="renderAggregatedLogsBlock"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="html" type="org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block"/>
+      <param name="context" type="org.apache.hadoop.yarn.webapp.View.ViewContext"/>
+      <doc>
+      <![CDATA[Render Aggregated Logs block.
+ @param html the html
+ @param context the ViewContext]]>
+      </doc>
+    </method>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @param aggregatedLogPath the aggregatedLog path
+ @param appId the ApplicationId
+ @return the application owner
+ @throws IOException if we can not get the application owner]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @param aggregatedLogPath the aggregatedLog path.
+ @param appId the ApplicationId
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls]]>
+      </doc>
+    </method>
+    <method name="verifyAndCreateRemoteLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify and create the remote log directory.]]>
+      </doc>
+    </method>
+    <method name="createAppDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create remote Application directory for log aggregation.
+ @param user the user
+ @param appId the application ID
+ @param userUgi the UGI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRemoteNodeLogFileForApp" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <doc>
+      <![CDATA[Get the remote aggregated log path.
+ @param appId the ApplicationId
+ @param user the Application Owner
+ @param nodeId the NodeManager Id
+ @return the remote aggregated log path]]>
+      </doc>
+    </method>
+    <method name="getRemoteAppLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="appOwner" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the remote application directory for log aggregation.
+ @param appId the Application ID
+ @param appOwner the Application Owner
+ @return the remote application directory
+ @throws IOException if can not find the remote application directory]]>
+      </doc>
+    </method>
+    <method name="getOlderRemoteAppLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="appOwner" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the older remote application directory for log aggregation.
+ @param appId the Application ID
+ @param appOwner the Application Owner
+ @return the older remote application directory
+ @throws IOException if can not find the remote application directory]]>
+      </doc>
+    </method>
+    <method name="cleanOldLogs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="remoteNodeLogFileForApp" type="org.apache.hadoop.fs.Path"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="aggregatedLogSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create the aggregated log suffix. The LogAggregationFileController
+ should call this to get the suffix and append the suffix to the end
+ of each log. This would keep the aggregated log format consistent.
+
+ @param fileName the File Name
+ @return the aggregated log suffix String]]>
+      </doc>
+    </method>
+    <method name="isFsSupportsChmod" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="belongsToAppAttempt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TLDIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the top level directory under which app directories will be
+ created.]]>
+      </doc>
+    </field>
+    <field name="APP_DIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the Application directory.]]>
+      </doc>
+    </field>
+    <field name="APP_LOG_FILE_UMASK" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Umask for the log file.]]>
+      </doc>
+    </field>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDir" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDirSuffix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="retentionSize" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fileControllerName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fsSupportsChmod" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class to implement Log Aggregation File Controller.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.ifile">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.tfile">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels.event">
+</package>
+<package name="org.apache.hadoop.yarn.security">
+  <!-- start class org.apache.hadoop.yarn.security.AccessRequest -->
+  <class name="AccessRequest" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessRequest" type="org.apache.hadoop.yarn.security.PrivilegedEntity, org.apache.hadoop.security.UserGroupInformation, org.apache.hadoop.yarn.security.AccessType, java.lang.String, java.lang.String, java.lang.String, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAccessType" return="org.apache.hadoop.yarn.security.AccessType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEntity" return="org.apache.hadoop.yarn.security.PrivilegedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getForwardedAddresses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRemoteAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This request object contains all the context information to determine whether
+ a user has permission to access the target entity.
+ user       : the user who's currently accessing
+ accessType : the access type against the entity.
+ entity     : the target object user is accessing.
+ appId      : the associated app Id for current access. This could be null
+              if no app is associated.
+ appName    : the associated app name for current access. This could be null if
+              no app is associated.
+ remoteAddress : The caller's remote ip address.
+ forwardedAddresses : In case this is an http request, this contains the
+                    originating IP address of a client connecting to a web
+                    server through an HTTP proxy or load balancer. This
+                    parameter is null, if it's a RPC request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AccessRequest -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <class name="AMRMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.AMRMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[AMRMTokenIdentifier is the TokenIdentifier to be used by
+ ApplicationMasters to authenticate to the ResourceManager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <class name="AMRMTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="AMRMTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <class name="ContainerManagerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerManagerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <class name="ContainerTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="Use one of the other constructors instead.">
+      <doc>
+      <![CDATA[Creates a instance.
+
+ @param appSubmitter appSubmitter
+ @param containerID container ID
+ @param creationTime creation time
+ @param expiryTimeStamp expiry timestamp
+ @param hostName hostname
+ @param logAggregationContext log aggregation context
+ @param masterKeyId master key ID
+ @param priority priority
+ @param r resource needed by the container
+ @param rmIdentifier ResourceManager identifier
+ @deprecated Use one of the other constructors instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convenience Constructor for existing clients.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a Container Token Identifier.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId
+ @param allocationTags Set of allocation Tags.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC layer/SecretManager.]]>
+      </doc>
+    </constructor>
+    <method name="getContainerID" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNmHostAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getExpiryTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMasterKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRMIdentifier" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the RMIdentifier of RM in which containers are allocated.
+ @return RMIdentifier]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ContainerType of container to allocate.
+ @return ContainerType]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ExecutionType of container to allocate
+ @return ExecutionType]]>
+      </doc>
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Container version
+ @return container version]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the node-label-expression in the original ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllcationTags" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TokenIdentifier for a container. Encodes {@link ContainerId},
+ {@link Resource} needed by the container and the target NMs host-address.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <class name="ContainerTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="ContainerTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <class name="NMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, org.apache.hadoop.yarn.api.records.NodeId, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC/Secret manager]]>
+      </doc>
+    </constructor>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.NMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+  <class name="SchedulerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.admin">
+  <!-- start class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+  <class name="AdminSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AdminSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.client">
+  <!-- start class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <class name="BaseClientToAMTokenSecretManager" extends="org.apache.hadoop.security.token.SecretManager"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BaseClientToAMTokenSecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A base {@link SecretManager} for AMs to extend and validate Client-RM tokens
+ issued to clients by the RM using the underlying master-key shared by RM to
+ the AMs on their launch. All the methods are called by either Hadoop RPC or
+ YARN, so this class is strictly for the purpose of inherit/extend and
+ register with Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <class name="ClientRMSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientRMSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <class name="ClientTimelineSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientTimelineSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <class name="ClientToAMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClientToAMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationAttemptID" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClientName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ClientToAMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <class name="ClientToAMTokenSecretManager" extends="org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenSecretManager" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="retrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+    </method>
+    <method name="getMasterKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptID" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="setMasterKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+    </method>
+    <doc>
+    <![CDATA[A simple {@link SecretManager} for AMs to validate Client-RM tokens issued to
+ clients by the RM using the underlying master-key shared by RM to the AMs on
+ their launch. All the methods are called by either Hadoop RPC or YARN, so
+ this class is strictly for the purpose of inherit/extend and register with
+ Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <class name="RMDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="RMDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new delegation token identifier
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Delegation Token Identifier that identifies the delegation tokens from the 
+ Resource Manager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <class name="RMDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="RMDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <class name="TimelineDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new timeline delegation token identifier
+
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+  <class name="TimelineDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="TimelineDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.sharedcache">
+  <!-- start interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <interface name="SharedCacheChecksum"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="computeChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Calculate the checksum of the passed input stream.
+
+ @param in <code>InputStream</code> to be checksumed
+ @return the message digest of the input stream
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <!-- start class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+  <class name="SharedCacheChecksumFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheChecksumFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getChecksum" return="org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a new <code>SharedCacheChecksum</code> object based on the configurable
+ algorithm implementation
+ (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+
+ @return <code>SharedCacheChecksum</code> object]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+</package>
+<package name="org.apache.hadoop.yarn.state">
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <class name="InvalidStateTransitionException" extends="org.apache.hadoop.yarn.state.InvalidStateTransitonException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidStateTransitionException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The exception that happens when you call invalid state transition.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <class name="InvalidStateTransitonException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link InvalidStateTransitionException} instead.">
+    <constructor name="InvalidStateTransitonException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCurrentState" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEvent" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[@deprecated Use {@link InvalidStateTransitionException} instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <!-- start interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <interface name="MultipleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ @return the postState. Post state must be one of the 
+                      valid post states registered in StateMachine.
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. 
+ Post state is decided by Transition hook. Post state must be one of the 
+ valid post states registered in StateMachine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <interface name="SingleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ 
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. This lead to state machine to move to 
+ the post state as registered in the state machine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateMachine -->
+  <interface name="StateMachine"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getCurrentState" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doTransition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="event" type="EVENT"/>
+      <exception name="InvalidStateTransitionException" type="org.apache.hadoop.yarn.state.InvalidStateTransitionException"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateMachine -->
+  <!-- start class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <class name="StateMachineFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StateMachineFactory" type="STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+
+ This is the only constructor in the API.]]>
+      </doc>
+    </constructor>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postStates" type="java.util.Set"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.MultipleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postStates valid post-transition states
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="installTopology" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a StateMachineFactory just like {@code this}, except that if
+         you won't need any synchronization to build a state machine
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+         The only way you could distinguish the returned
+         StateMachineFactory from {@code this} would be by
+         measuring the performance of the derived 
+         {@code StateMachine} you can get from it.
+
+ Calling this is optional.  It doesn't change the semantics of the factory,
+   if you call it then when you use the factory there is no synchronization.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+      <param name="listener" type="org.apache.hadoop.yarn.state.StateTransitionListener"/>
+      <doc>
+      <![CDATA[A StateMachine that accepts a transition listener.
+ @param operand the object upon which the returned
+                {@link StateMachine} will operate.
+ @param initialState the state in which the returned
+                {@link StateMachine} will start.
+ @param listener An implementation of a {@link StateTransitionListener}.
+ @return A (@link StateMachine}.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+    </method>
+    <method name="generateStateGraph" return="org.apache.hadoop.yarn.state.Graph"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a graph represents the state graph of this StateMachine
+ @param name graph name
+ @return Graph object generated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[State machine topology.
+ This object is semantically immutable.  If you have a
+ StateMachineFactory there's no operation in the API that changes
+ its semantic properties.
+
+ @param <OPERAND> The object type on which this state machine operates.
+ @param <STATE> The state of the entity.
+ @param <EVENTTYPE> The external eventType to be handled.
+ @param <EVENT> The event object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+  <interface name="StateTransitionListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="preTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="eventToBeProcessed" type="EVENT"/>
+      <doc>
+      <![CDATA[Pre Transition Hook. This will be called before transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param eventToBeProcessed Incoming Event.]]>
+      </doc>
+    </method>
+    <method name="postTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="afterState" type="STATE"/>
+      <param name="processedEvent" type="EVENT"/>
+      <doc>
+      <![CDATA[Post Transition Hook. This will be called after the transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param afterState State after transition.
+ @param processedEvent Processed Event.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A State Transition Listener.
+ It exposes a pre and post transition hook called before and
+ after the transition.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+  <!-- start class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <class name="AbstractLivelinessMonitor" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String, org.apache.hadoop.yarn.util.Clock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="expire"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="setExpireInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="expireInterval" type="int"/>
+    </method>
+    <method name="getExpireInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="o" type="O"/>
+    </method>
+    <method name="setMonitorInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="monitorInterval" type="long"/>
+    </method>
+    <method name="receivedPing"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+      <param name="expireTime" type="long"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="resetTimer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResetTimeOnStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="resetTimeOnStart" type="boolean"/>
+    </method>
+    <field name="DEFAULT_EXPIRE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A simple liveliness monitor with which clients can register, trust the
+ component to monitor liveliness, get a call-back on expiry and then finally
+ unregister.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <!-- start class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="org.apache.hadoop.util.ApplicationClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[This type has been deprecated in favor of
+ {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ ApplicationClassLoader should use that type instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <!-- start class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <class name="BoundedAppender" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BoundedAppender" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append" return="org.apache.hadoop.yarn.util.BoundedAppender"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="csq" type="java.lang.CharSequence"/>
+      <doc>
+      <![CDATA[Append a {@link CharSequence} considering {@link #limit}, truncating
+ from the head of {@code csq} or {@link #messages} when necessary.
+
+ @param csq the {@link CharSequence} to append
+ @return this]]>
+      </doc>
+    </method>
+    <method name="length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current length of messages considering truncates
+ without header and ellipses.
+
+ @return current length]]>
+      </doc>
+    </method>
+    <method name="getLimit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a string representation of the actual contents, displaying also a
+ header and ellipses when there was a truncate.
+
+ @return String representation of the {@link #messages}]]>
+      </doc>
+    </method>
+    <field name="TRUNCATED_MESSAGES_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link CharSequence} appender that considers its {@link #limit} as upper
+ bound.
+ <p>
+ When {@link #limit} would be reached on append, past messages will be
+ truncated from head, and a header telling the user about truncation will be
+ prepended, with ellipses in between header and messages.
+ <p>
+ Note that header and ellipses are not counted against {@link #limit}.
+ <p>
+ An example:
+
+ <pre>
+ {@code
+   // At the beginning it's an empty string
+   final Appendable shortAppender = new BoundedAppender(80);
+   // The whole message fits into limit
+   shortAppender.append(
+       "message1 this is a very long message but fitting into limit\n");
+   // The first message is truncated, the second not
+   shortAppender.append("message2 this is shorter than the previous one\n");
+   // The first message is deleted, the second truncated, the third
+   // preserved
+   shortAppender.append("message3 this is even shorter message, maybe.\n");
+   // The first two are deleted, the third one truncated, the last preserved
+   shortAppender.append("message4 the shortest one, yet the greatest :)");
+   // Current contents are like this:
+   // Diagnostic messages truncated, showing last 80 chars out of 199:
+   // ...s is even shorter message, maybe.
+   // message4 the shortest one, yet the greatest :)
+ }
+ </pre>
+ <p>
+ Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
+ just like in {@link StringBuilder#append(CharSequence) original
+ implementation}.
+ <p>
+ Note that this class is not thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <!-- start interface org.apache.hadoop.yarn.util.Clock -->
+  <interface name="Clock"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A simple clock interface that gives you time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.util.Clock -->
+  <!-- start class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <class name="ConverterUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConverterUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPathFromYarnURL" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[return a hadoop path from a given url
+ This method is deprecated, use {@link URL#toPath()} instead.
+ 
+ @param url
+          url to convert
+ @return path from {@link URL}
+ @throws URISyntaxException]]>
+      </doc>
+    </method>
+    <method name="getYarnUrlFromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getYarnUrlFromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recordFactory" type="org.apache.hadoop.yarn.factories.RecordFactory"/>
+      <param name="applicationIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="serviceAddr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service. Supposed
+ to be used for tokens other than RMDelegationToken. For
+ RMDelegationToken, use
+ {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
+ org.apache.hadoop.io.Text)} instead.
+
+ @param protoToken the yarn token
+ @param serviceAddr the connect address for the service
+ @return rpc token]]>
+      </doc>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service.
+
+ @param protoToken the yarn token
+ @param service the service for the token]]>
+      </doc>
+    </method>
+    <field name="APPLICATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONTAINER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_ATTEMPT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains a set of utilities which help converting data structures
+ from/to 'serializableFormat' to/from hadoop/nativejava data structures.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <!-- start class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <class name="MonotonicClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="MonotonicClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current time from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ @return a monotonic clock that counts in milliseconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A monotonic clock from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ This is appropriate to use when computing how much longer to wait for an
+ interval to expire.
+ This function can return a negative value and it must be handled correctly
+ by callers. See the documentation of System#nanoTime for caveats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <!-- start class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <class name="ResourceCalculatorProcessTree" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceCalculatorProcessTree" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create process-tree instance with specified root process.
+
+ Subclass must override this.
+ @param root process-tree root-process]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initialize the object.
+ @throws YarnException Throws an exception on error.]]>
+      </doc>
+    </method>
+    <method name="updateProcessTree"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the process-tree with latest state.
+
+ Each call to this function should increment the age of the running
+ processes that already exist in the process tree. Age is used other API's
+ of the interface.]]>
+      </doc>
+    </method>
+    <method name="getProcessTreeDump" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a dump of the process-tree.
+
+ @return a string concatenating the dump of information of all the processes
+         in the process-tree]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree.
+
+ @return virtual memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree.
+
+ @return rss memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return virtual memory used by the process-tree in bytes for
+ processes older than the specified age, {@link #UNAVAILABLE} if it
+ cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return rss memory used by the process-tree in bytes for
+ processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU time in millisecond used by all the processes in the
+ process-tree since the process-tree was created
+
+ @return cumulative CPU time in millisecond since the process-tree
+ created, {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercent" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU usage by all the processes in the process-tree based on
+ average between samples as a ratio of overall CPU cycles similar to top.
+ Thus, if 2 out of 4 cores are used this should return 200.0.
+ Note: UNAVAILABLE will be returned in case when CPU usage is not
+ available. It is NOT advised to return any other error code.
+
+ @return percentage CPU usage since the process-tree was created,
+ {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available.]]>
+      </doc>
+    </method>
+    <method name="checkPidPgrpidForMatch" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify that the tree process id is same as its process group id.
+ @return true if the process id matches else return false.]]>
+      </doc>
+    </method>
+    <method name="getResourceCalculatorProcessTree" return="org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <param name="clazz" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create the ResourceCalculatorProcessTree rooted to specified process 
+ from the class name and configure it. If class name is null, this method
+ will try and return a process tree plugin available for this system.
+
+ @param pid process pid of the root of the process tree
+ @param clazz class-name
+ @param conf configure the plugin with this.
+
+ @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
+         is not available for this system.]]>
+      </doc>
+    </method>
+    <field name="UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interface class to obtain process resource usage
+ NOTE: This class should not be used by external users, but only by external
+ developers to extend and include their own process-tree implementation, 
+ especially for platforms other than Linux and Windows.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <!-- start class org.apache.hadoop.yarn.util.SystemClock -->
+  <class name="SystemClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="SystemClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.yarn.util.SystemClock"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current time from the system
+ clock in milliseconds.
+ 
+ NOTE: Do not use this to calculate a duration of expire or interval to sleep,
+ because it will be broken by settimeofday. Please use {@link MonotonicClock}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.SystemClock -->
+  <!-- start class org.apache.hadoop.yarn.util.UTCClock -->
+  <class name="UTCClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="UTCClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current UTC time in
+ milliseconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.UTCClock -->
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+<package name="org.apache.hadoop.yarn.util.timeline">
+  <!-- start class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+  <class name="TimelineUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string not in a pretty format
+ 
+ @param o
+          an object to serialize
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <param name="pretty" type="boolean"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string
+ 
+ @param o
+          an object to serialize
+ @param pretty
+          whether in a pretty format or not
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1_5Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1.5 is enabled by default via
+ configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="createTimelineAbout" return="org.apache.hadoop.yarn.api.records.timeline.TimelineAbout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineTokenServiceAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="buildTimelineTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="generateDefaultFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="generateFlowNameTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow name tag.
+
+ @param flowName flow name that identifies a distinct flow application which
+                 can be run repeatedly over time
+ @return flow name tag.]]>
+      </doc>
+    </method>
+    <method name="shortenFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Shortens the flow name for the configured size by removing UUID if present.
+
+ @param flowName which has to be shortened
+ @param conf to resize the flow name
+ @return shortened flowName]]>
+      </doc>
+    </method>
+    <method name="generateFlowVersionTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowVersion" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow version tag.
+
+ @param flowVersion flow version that keeps track of the changes made to the
+                    flow
+ @return flow version tag.]]>
+      </doc>
+    </method>
+    <method name="generateFlowRunIdTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowRunId" type="long"/>
+      <doc>
+      <![CDATA[Generate flow run ID tag.
+
+ @param flowRunId flow run ID that identifies one instance (or specific
+                  execution) of that flow
+ @return flow run id tag.]]>
+      </doc>
+    </method>
+    <field name="FLOW_NAME_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The helper class for the timeline module.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+</package>
+<package name="org.apache.hadoop.yarn.webapp.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml
new file mode 100644
index 0000000000000..3ec09cfb059b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml
@@ -0,0 +1,3975 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:09:19 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Common 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.4.jar:/maven/com/squareup/okhttp3/okhttp/4.9.3/okhttp-4.9.3.jar:/maven/com/squareup/okio/okio/2.8.0/okio-2.8.0.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib/1.4.10/kotlin-stdlib-1.4.10.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib-common/1.4.10/kotlin-stdlib-common-1.4.10.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Common 3.3.4 -->
+<package name="org.apache.hadoop.yarn">
+  <!-- start class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <class name="ContainerLogAppender" extends="org.apache.log4j.FileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="append"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.log4j.spi.LoggingEvent"/>
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <method name="getTotalLogFileSize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setTotalLogFileSize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logSize" type="long"/>
+      <doc>
+      <![CDATA[Setter so that log4j can configure it from the
+  configuration(log4j.properties).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <class name="ContainerRollingLogAppender" extends="org.apache.log4j.RollingFileAppender"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="ContainerRollingLogAppender"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="activateOptions"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="flush"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerLogDir" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Getter/Setter methods for log4j.]]>
+      </doc>
+    </method>
+    <method name="setContainerLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogDir" type="java.lang.String"/>
+    </method>
+    <method name="getContainerLogFile" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setContainerLogFile"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerLogFile" type="java.lang.String"/>
+    </method>
+    <doc>
+    <![CDATA[A simple log4j-appender for container's logs.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.ContainerRollingLogAppender -->
+  <!-- start class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+  <class name="YarnUncaughtExceptionHandler" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.lang.Thread.UncaughtExceptionHandler"/>
+    <constructor name="YarnUncaughtExceptionHandler"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="uncaughtException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="t" type="java.lang.Thread"/>
+      <param name="e" type="java.lang.Throwable"/>
+    </method>
+    <doc>
+    <![CDATA[This class is intended to be installed by calling 
+ {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
+ In the main entry point.  It is intended to try and cleanly shut down
+ programs using the YARN Event framework.
+ 
+ Note: Right now it only will shut down the program if a Error is caught, but
+ not any other exception.  Anything else is just logged.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.YarnUncaughtExceptionHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.api">
+</package>
+<package name="org.apache.hadoop.yarn.client">
+  <!-- start class org.apache.hadoop.yarn.client.AHSProxy -->
+  <class name="AHSProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AHSProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAHSProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ahsAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="rmAddress" type="java.net.InetSocketAddress"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.AHSProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <class name="ClientRMProxy" extends="org.apache.hadoop.yarn.client.RMProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="configuration" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a proxy to the ResourceManager for the specified protocol.
+ @param configuration Configuration with all the required information.
+ @param protocol Client protocol for which proxy is being requested.
+ @param <T> Type of proxy.
+ @return Proxy to the ResourceManager for the specified client protocol.
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get the token service name to be used for RMDelegationToken. Depending
+ on whether HA is enabled or not, this method generates the appropriate
+ service name as a comma-separated list of service addresses.
+
+ @param conf Configuration corresponding to the cluster we need the
+             RMDelegationToken for
+ @return - Service name for RMDelegationToken]]>
+      </doc>
+    </method>
+    <method name="getAMRMTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="address" type="java.lang.String"/>
+      <param name="defaultAddr" type="java.lang.String"/>
+      <param name="defaultPort" type="int"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ClientRMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.NMProxy -->
+  <class name="NMProxy" extends="org.apache.hadoop.yarn.client.ServerProxy"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createNMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.NMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.RMProxy -->
+  <class name="RMProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMProxy"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="retryTime" type="long"/>
+      <param name="retryInterval" type="long"/>
+      <param name="isHAEnabled" type="boolean"/>
+      <doc>
+      <![CDATA[Fetch retry policy from Configuration and create the
+ retry policy with specified retryTime and retry interval.]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.RMProxy -->
+  <!-- start class org.apache.hadoop.yarn.client.ServerProxy -->
+  <class name="ServerProxy" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ServerProxy"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createRetryPolicy" return="org.apache.hadoop.io.retry.RetryPolicy"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="maxWaitTimeStr" type="java.lang.String"/>
+      <param name="defMaxWaitTime" type="long"/>
+      <param name="connectRetryIntervalStr" type="java.lang.String"/>
+      <param name="defRetryInterval" type="long"/>
+    </method>
+    <method name="createRetriableProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="rpc" type="org.apache.hadoop.yarn.ipc.YarnRPC"/>
+      <param name="serverAddress" type="java.net.InetSocketAddress"/>
+      <param name="retryPolicy" type="org.apache.hadoop.io.retry.RetryPolicy"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.ServerProxy -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api">
+  <!-- start class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <class name="AppAdminClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAdminClient"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createAppAdminClient" return="org.apache.hadoop.yarn.client.api.AppAdminClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appType" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[<p>
+ Create a new instance of AppAdminClient.
+ </p>
+
+ @param appType application type
+ @param conf configuration
+ @return app admin client]]>
+      </doc>
+    </method>
+    <method name="actionLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Launch a new YARN application.
+ </p>
+
+ @param fileName specification of application
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStop" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Stop a YARN application (attempt to stop gracefully before killing the
+ application). In the case of a long-running service, the service may be
+ restarted later.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionStart" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Start a YARN application from a previously saved specification. In the
+ case of a long-running service, the service must have been previously
+ launched/started and then stopped, or previously saved but not started.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionSave" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <param name="appName" type="java.lang.String"/>
+      <param name="lifetime" type="java.lang.Long"/>
+      <param name="queue" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Save the specification for a YARN application / long-running service.
+ The application may be started later.
+ </p>
+
+ @param fileName specification of application to save
+ @param appName name of the application
+ @param lifetime lifetime of the application
+ @param queue queue of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionDestroy" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Remove the specification and all application data for a YARN application.
+ The application cannot be running.
+ </p>
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionFlex" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentCounts" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Change the number of running containers for a component of a YARN
+ application / long-running service.
+ </p>
+
+ @param appName the name of the application
+ @param componentCounts map of component name to new component count or
+                        amount to change existing component count (e.g.
+                        5, +5, -5)
+ @return exit code
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="enableFastLaunch" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="destinationFolder" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Upload AM dependencies to HDFS. This makes future application launches
+ faster since the dependencies do not have to be uploaded on each launch.
+ </p>
+
+ @param destinationFolder
+          an optional HDFS folder where dependency tarball will be uploaded
+ @return exit code
+ @throws IOException
+           IOException
+ @throws YarnException
+           exception in client or server]]>
+      </doc>
+    </method>
+    <method name="getStatusString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdOrName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get detailed app specific status string for a YARN application.
+ </p>
+
+ @param appIdOrName appId or appName
+ @return status string
+ @throws IOException IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="initiateUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.lang.String"/>
+      <param name="autoFinalize" type="boolean"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initiate upgrade of a long running service.
+
+ @param appName      the name of the application.
+ @param fileName     specification of application upgrade to save.
+ @param autoFinalize when true, finalization of upgrade will be done
+                     automatically.
+ @return exit code
+ @throws IOException   IOException
+ @throws YarnException exception in client or server]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <method name="actionUpgradeComponents" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Upgrade components of a long running service.
+
+ @param appName    the name of the application.
+ @param components the name of the components.]]>
+      </doc>
+    </method>
+    <method name="actionCleanUp" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="userName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+      <doc>
+      <![CDATA[Operation to be performed by the RM after an application has completed.
+
+ @param appName  the name of the application.
+ @param userName the name of the user.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="getInstances" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="components" type="java.util.List"/>
+      <param name="version" type="java.lang.String"/>
+      <param name="containerStates" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="actionUpgradeExpress" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="fileName" type="java.io.File"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Express upgrade a long running service.
+
+ @param appName  the name of the application
+ @param fileName specification of application upgrade to save.
+ @return exit code]]>
+      </doc>
+    </method>
+    <method name="actionCancelUpgrade" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Cancels the upgrade of the service.
+
+ @param appName the name of the application
+ @return exit code
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="actionDecommissionInstances" return="int"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="componentInstances" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Decommission component instances of a long running service.
+
+ @param appName            the name of the application.
+ @param componentInstances the name of the component instances.]]>
+      </doc>
+    </method>
+    <field name="YARN_APP_ADMIN_CLIENT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_TYPE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="UNIT_TEST_CLASS_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Client for managing applications.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.AppAdminClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <class name="TimelineClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.io.Flushable"/>
+    <constructor name="TimelineClient" type="java.lang.String"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineClient" return="org.apache.hadoop.yarn.client.api.TimelineClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Creates an instance of the timeline v.1.x client.
+ The current UGI when the user initialize the client will be used to do the
+ put and the delegation token operations. The current user may use
+ {@link UserGroupInformation#doAs} another user to construct and initialize
+ a timeline client if the following operations are supposed to be conducted
+ by that user.
+
+ @return the created timeline client instance]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+ </p>
+ 
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putEntities" return="org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="groupId" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId"/>
+      <param name="entities" type="org.apache.hadoop.yarn.api.records.timeline.TimelineEntity[]"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a number of conceptual entities to the timeline
+ server. It is a blocking API. The method will not return until it gets the
+ response from the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param appAttemptId {@link ApplicationAttemptId}
+ @param groupId {@link TimelineEntityGroupId}
+ @param entities
+          the collection of {@link TimelineEntity}
+ @return the error information if the sent entities are not correctly stored
+ @throws IOException if there are I/O errors
+ @throws YarnException if entities are incomplete/invalid]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+ </p>
+ 
+ @param domain
+          an {@link TimelineDomain} object
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="putDomain"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="domain" type="org.apache.hadoop.yarn.api.records.timeline.TimelineDomain"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Send the information of a domain to the timeline server. It is a
+ blocking API. The method will not return until it gets the response from
+ the timeline server.
+
+ This API is only for timeline service v1.5
+ </p>
+
+ @param domain
+          an {@link TimelineDomain} object
+ @param appAttemptId {@link ApplicationAttemptId}
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="renewer" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Get a delegation token so as to be able to talk to the timeline server in a
+ secure way.
+ </p>
+ 
+ @param renewer
+          Address of the renewer who can renew these tokens when needed by
+          securely talking to the timeline server
+ @return a delegation token ({@link Token}) that can be used to talk to the
+         timeline server
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="renewDelegationToken" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Renew a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to renew
+ @return the new expiration time
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <method name="cancelDelegationToken"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="timelineDT" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[<p>
+ Cancel a timeline delegation token.
+ </p>
+ 
+ @param timelineDT
+          the delegation token to cancel
+ @throws IOException
+ @throws YarnException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to post some information in terms of a
+ number of conceptual entities. This client library needs to be used along
+ with Timeline V.1.x server versions.
+ Refer {@link TimelineV2Client} for ATS V2 interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineClient -->
+  <!-- start class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+  <class name="TimelineReaderClient" extends="org.apache.hadoop.service.CompositeService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClient" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="createTimelineReaderClient" return="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new instance of Timeline Reader Client.]]>
+      </doc>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application entity.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity of the application
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entity.
+ @param appAttemptId application attempt id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return entity associated with application attempt
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets application attempt entities.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of application attempt entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets Timeline entity for the container.
+ @param containerId container id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @return timeline entity for container
+ @throws IOException]]>
+      </doc>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Gets container entities for an application.
+ @param appId application id
+ @param fields Fields to be fetched. Defaults to INFO.
+ @param filters Filters to be applied while fetching entities.
+ @param limit Number of entities to return.
+ @param fromId Retrieve next set of generic ids from given fromId
+ @return list of entities
+ @throws IOException]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A client library that can be used to get Timeline Entities associated with
+ application, application attempt or containers. This client library needs to
+ be used along with time line v.2 server version.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.TimelineReaderClient -->
+</package>
+<package name="org.apache.hadoop.yarn.client.api.impl">
+  <!-- start class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+  <class name="TimelineReaderClientImpl" extends="org.apache.hadoop.yarn.client.api.TimelineReaderClient"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineReaderClientImpl"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="getApplicationEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getApplicationAttemptEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntity" return="org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getContainerEntities" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="fields" type="java.lang.String"/>
+      <param name="filters" type="java.util.Map"/>
+      <param name="limit" type="long"/>
+      <param name="fromId" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="doGetUri" return="com.sun.jersey.api.client.ClientResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="base" type="java.net.URI"/>
+      <param name="path" type="java.lang.String"/>
+      <param name="params" type="javax.ws.rs.core.MultivaluedMap"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <doc>
+    <![CDATA[Implementation of TimelineReaderClient interface.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.client.api.impl.TimelineReaderClientImpl -->
+</package>
+<package name="org.apache.hadoop.yarn.event">
+  <!-- start class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <class name="AbstractEvent" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Event"/>
+    <constructor name="AbstractEvent" type="TYPE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractEvent" type="TYPE, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTimestamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="TYPE"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Parent class of all the events. All events extend this class.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AbstractEvent -->
+  <!-- start class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <class name="AsyncDispatcher" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.event.Dispatcher"/>
+    <constructor name="AsyncDispatcher"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.util.concurrent.BlockingQueue"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AsyncDispatcher" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Set a name for this dispatcher thread.
+ @param dispatcherName name of the dispatcher thread]]>
+      </doc>
+    </constructor>
+    <method name="disableExitOnDispatchException"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceInit"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="setDrainEventsOnStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="dispatch"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="event" type="org.apache.hadoop.yarn.event.Event"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isEventThreadWaiting" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isDrained" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isStopped" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="addMetrics"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="metrics" type="org.apache.hadoop.yarn.metrics.EventTypeMetrics"/>
+      <param name="eventClass" type="java.lang.Class"/>
+    </method>
+    <field name="eventDispatchers" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="true" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Dispatches {@link Event}s in a separate thread. Currently only single thread
+ does that. Potentially there could be multiple channels for each event type
+ class and a thread pool can be used to dispatch the events.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.event.AsyncDispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <interface name="Dispatcher"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getEventHandler" return="org.apache.hadoop.yarn.event.EventHandler"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="register"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="java.lang.Class"/>
+      <param name="handler" type="org.apache.hadoop.yarn.event.EventHandler"/>
+    </method>
+    <doc>
+    <![CDATA[Event Dispatcher interface. It dispatches events to registered 
+ event handlers based on event types.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Dispatcher -->
+  <!-- start interface org.apache.hadoop.yarn.event.Event -->
+  <interface name="Event"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getType" return="TYPE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTimestamp" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Interface defining events api.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.Event -->
+  <!-- start interface org.apache.hadoop.yarn.event.EventHandler -->
+  <interface name="EventHandler"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="handle"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="event" type="T"/>
+    </method>
+    <doc>
+    <![CDATA[Interface for handling events of type T
+
+ @param <T> parameterized event of type T]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.event.EventHandler -->
+</package>
+<package name="org.apache.hadoop.yarn.factories">
+</package>
+<package name="org.apache.hadoop.yarn.factory.providers">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <class name="AggregatedLogFormat" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AggregatedLogFormat"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <class name="AggregatedLogFormat.LogKey" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.io.Writable"/>
+    <constructor name="LogKey"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="org.apache.hadoop.yarn.api.records.ContainerId"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="LogKey" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="obj" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey -->
+  <!-- start class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+  <class name="AggregatedLogFormat.LogReader" extends="java.lang.Object"
+    abstract="false"
+    static="true" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogReader" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.fs.Path"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+    </constructor>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @return the application owner.
+ @throws IOException if we can not get the application owner.]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.io.DataInputStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Read the next key and return the value-stream.
+ 
+ @param key the log key
+ @return the valueStream if there are more keys or null otherwise
+ @throws IOException if we can not get the dataInputStream
+ for the next key]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the valueStream
+ @param writer the log writer
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAcontainerLogs"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="writer" type="java.io.Writer"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Writes all logs for a single container to the provided writer.
+ @param valueStream the value stream
+ @param writer the log writer
+ @throws IOException if we can not read the container logs.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the print stream
+ @param logUploadedTime the time stamp
+ @throws IOException if we can not read the container log by specifying
+ the container log type.]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container for the specific bytes.
+
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log upload time stamp
+ @param bytes the output size of the log
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readAContainerLogsForALogType"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ all types for a single container.
+ 
+ @param valueStream the value stream
+ @param out the output print stream
+ @throws IOException if we can not read the container log]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="readContainerLogsForALogType" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="valueStream" type="java.io.DataInputStream"/>
+      <param name="out" type="java.io.PrintStream"/>
+      <param name="logUploadedTime" type="long"/>
+      <param name="logType" type="java.util.List"/>
+      <param name="bytes" type="long"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Keep calling this till you get a {@link EOFException} for getting logs of
+ the specific types for a single container.
+ @param valueStream the value stream
+ @param out the output print stream
+ @param logUploadedTime the log uploaded time stamp
+ @param logType the given log type
+ @throws IOException if we can not read the container logs]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller">
+  <!-- start class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+  <class name="LogAggregationFileController" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="LogAggregationFileController"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="controllerName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Initialize the log file controller.
+ @param conf the Configuration
+ @param controllerName the log controller class name]]>
+      </doc>
+    </method>
+    <method name="initInternal"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Derived classes initialize themselves using this method.
+ @param conf the Configuration]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the remote root log directory.
+ @return the remote root log directory path]]>
+      </doc>
+    </method>
+    <method name="getRemoteRootLogDirSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the log aggregation directory suffix.
+ @return the log aggregation directory suffix]]>
+      </doc>
+    </method>
+    <method name="initializeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="context" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Initialize the writer.
+ @param context the {@link LogAggregationFileControllerContext}
+ @throws IOException if fails to initialize the writer]]>
+      </doc>
+    </method>
+    <method name="closeWriter"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="LogAggregationDFSException" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationDFSException"/>
+      <doc>
+      <![CDATA[Close the writer.
+ @throws LogAggregationDFSException if the closing of the writer fails
+         (for example due to HDFS quota being exceeded)]]>
+      </doc>
+    </method>
+    <method name="write"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logKey" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey"/>
+      <param name="logValue" type="org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Write the log content.
+ @param logKey the log key
+ @param logValue the log content
+ @throws IOException if fails to write the logs]]>
+      </doc>
+    </method>
+    <method name="postWrite"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="record" type="org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext"/>
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Operations needed after write the log content.
+ @param record the {@link LogAggregationFileControllerContext}
+ @throws Exception if anything fails]]>
+      </doc>
+    </method>
+    <method name="closePrintStream"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.OutputStream"/>
+    </method>
+    <method name="readAggregatedLogs" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <param name="os" type="java.io.OutputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Output container log.
+ @param logRequest {@link ContainerLogsRequest}
+ @param os the output stream
+ @return true if we can read the aggregated logs successfully
+ @throws IOException if we can not access the log file.]]>
+      </doc>
+    </method>
+    <method name="readAggregatedLogsMeta" return="java.util.List"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="logRequest" type="org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Return a list of {@link ContainerLogMeta} for an application
+ from Remote FileSystem.
+
+ @param logRequest {@link ContainerLogsRequest}
+ @return a list of {@link ContainerLogMeta}
+ @throws IOException if there is no available log file]]>
+      </doc>
+    </method>
+    <method name="renderAggregatedLogsBlock"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="html" type="org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block"/>
+      <param name="context" type="org.apache.hadoop.yarn.webapp.View.ViewContext"/>
+      <doc>
+      <![CDATA[Render Aggregated Logs block.
+ @param html the html
+ @param context the ViewContext]]>
+      </doc>
+    </method>
+    <method name="getApplicationOwner" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns the owner of the application.
+
+ @param aggregatedLogPath the aggregatedLog path
+ @param appId the ApplicationId
+ @return the application owner
+ @throws IOException if we can not get the application owner]]>
+      </doc>
+    </method>
+    <method name="getApplicationAcls" return="java.util.Map"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="aggregatedLogPath" type="org.apache.hadoop.fs.Path"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns ACLs for the application. An empty map is returned if no ACLs are
+ found.
+
+ @param aggregatedLogPath the aggregatedLog path.
+ @param appId the ApplicationId
+ @return a map of the Application ACLs.
+ @throws IOException if we can not get the application acls]]>
+      </doc>
+    </method>
+    <method name="verifyAndCreateRemoteLogDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify and create the remote log directory.]]>
+      </doc>
+    </method>
+    <method name="createAppDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="user" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <doc>
+      <![CDATA[Create remote Application directory for log aggregation.
+ @param user the user
+ @param appId the application ID
+ @param userUgi the UGI]]>
+      </doc>
+    </method>
+    <method name="getFileSystem" return="org.apache.hadoop.fs.FileSystem"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="createDir"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="checkExists" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+      <param name="fsPerm" type="org.apache.hadoop.fs.permission.FsPermission"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getRemoteNodeLogFileForApp" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="user" type="java.lang.String"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <doc>
+      <![CDATA[Get the remote aggregated log path.
+ @param appId the ApplicationId
+ @param user the Application Owner
+ @param nodeId the NodeManager Id
+ @return the remote aggregated log path]]>
+      </doc>
+    </method>
+    <method name="getRemoteAppLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="appOwner" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the remote application directory for log aggregation.
+ @param appId the Application ID
+ @param appOwner the Application Owner
+ @return the remote application directory
+ @throws IOException if can not find the remote application directory]]>
+      </doc>
+    </method>
+    <method name="getOlderRemoteAppLogDir" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="appOwner" type="java.lang.String"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Get the older remote application directory for log aggregation.
+ @param appId the Application ID
+ @param appOwner the Application Owner
+ @return the older remote application directory
+ @throws IOException if can not find the remote application directory]]>
+      </doc>
+    </method>
+    <method name="cleanOldLogs"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="remoteNodeLogFileForApp" type="org.apache.hadoop.fs.Path"/>
+      <param name="nodeId" type="org.apache.hadoop.yarn.api.records.NodeId"/>
+      <param name="userUgi" type="org.apache.hadoop.security.UserGroupInformation"/>
+    </method>
+    <method name="aggregatedLogSuffix" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="fileName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Create the aggregated log suffix. The LogAggregationFileController
+ should call this to get the suffix and append the suffix to the end
+ of each log. This would keep the aggregated log format consistent.
+
+ @param fileName the File Name
+ @return the aggregated log suffix String]]>
+      </doc>
+    </method>
+    <method name="isFsSupportsChmod" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="belongsToAppAttempt" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <field name="TLDIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the top level directory under which app directories will be
+ created.]]>
+      </doc>
+    </field>
+    <field name="APP_DIR_PERMISSIONS" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Permissions for the Application directory.]]>
+      </doc>
+    </field>
+    <field name="APP_LOG_FILE_UMASK" type="org.apache.hadoop.fs.permission.FsPermission"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="protected"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Umask for the log file.]]>
+      </doc>
+    </field>
+    <field name="conf" type="org.apache.hadoop.conf.Configuration"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDir" type="org.apache.hadoop.fs.Path"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="remoteRootLogDirSuffix" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="retentionSize" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fileControllerName" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="fsSupportsChmod" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Base class to implement Log Aggregation File Controller.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController -->
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.ifile">
+</package>
+<package name="org.apache.hadoop.yarn.logaggregation.filecontroller.tfile">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels">
+</package>
+<package name="org.apache.hadoop.yarn.nodelabels.event">
+</package>
+<package name="org.apache.hadoop.yarn.security">
+  <!-- start class org.apache.hadoop.yarn.security.AccessRequest -->
+  <class name="AccessRequest" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AccessRequest" type="org.apache.hadoop.yarn.security.PrivilegedEntity, org.apache.hadoop.security.UserGroupInformation, org.apache.hadoop.yarn.security.AccessType, java.lang.String, java.lang.String, java.lang.String, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAccessType" return="org.apache.hadoop.yarn.security.AccessType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEntity" return="org.apache.hadoop.yarn.security.PrivilegedEntity"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getForwardedAddresses" return="java.util.List"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRemoteAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[This request object contains all the context information to determine whether
+ a user has permission to access the target entity.
+ user       : the user who's currently accessing
+ accessType : the access type against the entity.
+ entity     : the target object user is accessing.
+ appId      : the associated app Id for current access. This could be null
+              if no app is associated.
+ appName    : the associated app name for current access. This could be null if
+              no app is associated.
+ remoteAddress : The caller's remote ip address.
+ forwardedAddresses : In case this is an http request, this contains the
+                    originating IP address of a client connecting to a web
+                    server through an HTTP proxy or load balancer. This
+                    parameter is null, if it's a RPC request.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AccessRequest -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <class name="AMRMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AMRMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AMRMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.AMRMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[AMRMTokenIdentifier is the TokenIdentifier to be used by
+ ApplicationMasters to authenticate to the ResourceManager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <class name="AMRMTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="AMRMTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.AMRMTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <class name="ContainerManagerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerManagerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <class name="ContainerTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      static="false" final="false" visibility="public"
+      deprecated="Use one of the other constructors instead.">
+      <doc>
+      <![CDATA[Creates a instance.
+
+ @param appSubmitter appSubmitter
+ @param containerID container ID
+ @param creationTime creation time
+ @param expiryTimeStamp expiry timestamp
+ @param hostName hostname
+ @param logAggregationContext log aggregation context
+ @param masterKeyId master key ID
+ @param priority priority
+ @param r resource needed by the container
+ @param rmIdentifier ResourceManager identifier
+ @deprecated Use one of the other constructors instead.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Convenience Constructor for existing clients.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ContainerId, int, java.lang.String, java.lang.String, org.apache.hadoop.yarn.api.records.Resource, long, int, long, org.apache.hadoop.yarn.api.records.Priority, long, org.apache.hadoop.yarn.api.records.LogAggregationContext, java.lang.String, org.apache.hadoop.yarn.server.api.ContainerType, org.apache.hadoop.yarn.api.records.ExecutionType, long, java.util.Set"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a Container Token Identifier.
+
+ @param containerID containerID
+ @param containerVersion containerVersion
+ @param hostName hostName
+ @param appSubmitter appSubmitter
+ @param r resource
+ @param expiryTimeStamp expiryTimeStamp
+ @param masterKeyId masterKeyId
+ @param rmIdentifier rmIdentifier
+ @param priority priority
+ @param creationTime creationTime
+ @param logAggregationContext logAggregationContext
+ @param nodeLabelExpression nodeLabelExpression
+ @param containerType containerType
+ @param executionType executionType
+ @param allocationRequestId allocationRequestId
+ @param allocationTags Set of allocation Tags.]]>
+      </doc>
+    </constructor>
+    <constructor name="ContainerTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC layer/SecretManager.]]>
+      </doc>
+    </constructor>
+    <method name="getContainerID" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNmHostAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getResource" return="org.apache.hadoop.yarn.api.records.Resource"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getExpiryTimeStamp" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getMasterKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="org.apache.hadoop.yarn.api.records.Priority"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCreationTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRMIdentifier" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the RMIdentifier of RM in which containers are allocated.
+ @return RMIdentifier]]>
+      </doc>
+    </method>
+    <method name="getContainerType" return="org.apache.hadoop.yarn.server.api.ContainerType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ContainerType of container to allocate.
+ @return ContainerType]]>
+      </doc>
+    </method>
+    <method name="getExecutionType" return="org.apache.hadoop.yarn.api.records.ExecutionType"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the ExecutionType of container to allocate
+ @return ExecutionType]]>
+      </doc>
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogAggregationContext" return="org.apache.hadoop.yarn.api.records.LogAggregationContext"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocationRequestId" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getVersion" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the Container version
+ @return container version]]>
+      </doc>
+    </method>
+    <method name="getNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the node-label-expression in the original ResourceRequest]]>
+      </doc>
+    </method>
+    <method name="getAllcationTags" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[TokenIdentifier for a container. Encodes {@link ContainerId},
+ {@link Resource} needed by the container and the target NMs host-address.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <class name="ContainerTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="ContainerTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.ContainerTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <class name="NMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, org.apache.hadoop.yarn.api.records.NodeId, java.lang.String, int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="NMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Default constructor needed by RPC/Secret manager]]>
+      </doc>
+    </constructor>
+    <method name="getApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationSubmitter" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getKeyId" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.NMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.NMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+  <class name="SchedulerSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SchedulerSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.SchedulerSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.admin">
+  <!-- start class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+  <class name="AdminSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AdminSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.admin.AdminSecurityInfo -->
+</package>
+<package name="org.apache.hadoop.yarn.security.client">
+  <!-- start class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <class name="BaseClientToAMTokenSecretManager" extends="org.apache.hadoop.security.token.SecretManager"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BaseClientToAMTokenSecretManager"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[A base {@link SecretManager} for AMs to extend and validate Client-RM tokens
+ issued to clients by the RM using the underlying master-key shared by RM to
+ the AMs on their launch. All the methods are called by either Hadoop RPC or
+ YARN, so this class is strictly for the purpose of inherit/extend and
+ register with Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <class name="ClientRMSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientRMSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <class name="ClientTimelineSecurityInfo" extends="org.apache.hadoop.security.SecurityInfo"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientTimelineSecurityInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getKerberosInfo" return="org.apache.hadoop.security.KerberosInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="getTokenInfo" return="org.apache.hadoop.security.token.TokenInfo"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <class name="ClientToAMTokenIdentifier" extends="org.apache.hadoop.security.token.TokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ClientToAMTokenIdentifier" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getApplicationAttemptID" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getClientName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProto" return="org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ClientToAMTokenIdentifierProto"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="write"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="out" type="java.io.DataOutput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="readFields"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.DataInput"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="org.apache.hadoop.security.UserGroupInformation"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hashCode" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="equals" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="other" type="java.lang.Object"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <class name="ClientToAMTokenSecretManager" extends="org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ClientToAMTokenSecretManager" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId, byte[]"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="retrievePassword" return="byte[]"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="identifier" type="org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier"/>
+      <exception name="SecretManager.InvalidToken" type="org.apache.hadoop.security.token.SecretManager.InvalidToken"/>
+    </method>
+    <method name="getMasterKey" return="javax.crypto.SecretKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptID" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+    </method>
+    <method name="setMasterKey"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+    </method>
+    <doc>
+    <![CDATA[A simple {@link SecretManager} for AMs to validate Client-RM tokens issued to
+ clients by the RM using the underlying master-key shared by RM to the AMs on
+ their launch. All the methods are called by either Hadoop RPC or YARN, so
+ this class is strictly for the purpose of inherit/extend and register with
+ Hadoop RPC.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <class name="RMDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="RMDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="RMDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new delegation token identifier
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Delegation Token Identifier that identifies the delegation tokens from the 
+ Resource Manager.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <class name="RMDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="RMDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <class name="TimelineDelegationTokenIdentifier" extends="org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineDelegationTokenIdentifier"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="TimelineDelegationTokenIdentifier" type="org.apache.hadoop.io.Text, org.apache.hadoop.io.Text, org.apache.hadoop.io.Text"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create a new timeline delegation token identifier
+
+ @param owner the effective username of the token owner
+ @param renewer the username of the renewer
+ @param realUser the real username of the token owner]]>
+      </doc>
+    </constructor>
+    <method name="getKind" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="KIND_NAME" type="org.apache.hadoop.io.Text"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier -->
+  <!-- start class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+  <class name="TimelineDelegationTokenSelector" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.security.token.TokenSelector"/>
+    <constructor name="TimelineDelegationTokenSelector"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="selectToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <param name="tokens" type="java.util.Collection"/>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.sharedcache">
+  <!-- start interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <interface name="SharedCacheChecksum"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="computeChecksum" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="in" type="java.io.InputStream"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Calculate the checksum of the passed input stream.
+
+ @param in <code>InputStream</code> to be checksumed
+ @return the message digest of the input stream
+ @throws IOException]]>
+      </doc>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum -->
+  <!-- start class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+  <class name="SharedCacheChecksumFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="SharedCacheChecksumFactory"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getChecksum" return="org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Get a new <code>SharedCacheChecksum</code> object based on the configurable
+ algorithm implementation
+ (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+
+ @return <code>SharedCacheChecksum</code> object]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory -->
+</package>
+<package name="org.apache.hadoop.yarn.state">
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <class name="InvalidStateTransitionException" extends="org.apache.hadoop.yarn.state.InvalidStateTransitonException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="InvalidStateTransitionException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <doc>
+    <![CDATA[The exception that happens when you call invalid state transition.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitionException -->
+  <!-- start class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <class name="InvalidStateTransitonException" extends="org.apache.hadoop.yarn.exceptions.YarnRuntimeException"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="Use {@link InvalidStateTransitionException} instead.">
+    <constructor name="InvalidStateTransitonException" type="java.lang.Enum, java.lang.Enum"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getCurrentState" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getEvent" return="java.lang.Enum"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[@deprecated Use {@link InvalidStateTransitionException} instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.InvalidStateTransitonException -->
+  <!-- start interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <interface name="MultipleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ @return the postState. Post state must be one of the 
+                      valid post states registered in StateMachine.
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. 
+ Post state is decided by Transition hook. Post state must be one of the 
+ valid post states registered in StateMachine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.MultipleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <interface name="SingleArcTransition"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="transition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="event" type="EVENT"/>
+      <doc>
+      <![CDATA[Transition hook.
+ 
+ @param operand the entity attached to the FSM, whose internal 
+                state may change.
+ @param event causal event]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Hook for Transition. This lead to state machine to move to 
+ the post state as registered in the state machine.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.SingleArcTransition -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateMachine -->
+  <interface name="StateMachine"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getCurrentState" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="doTransition" return="STATE"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="event" type="EVENT"/>
+      <exception name="InvalidStateTransitionException" type="org.apache.hadoop.yarn.state.InvalidStateTransitionException"/>
+    </method>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateMachine -->
+  <!-- start class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <class name="StateMachineFactory" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="StateMachineFactory" type="STATE"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor
+
+ This is the only constructor in the API.]]>
+      </doc>
+    </constructor>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition.  This overload
+          has no hook object.
+
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventTypes" type="java.util.Set"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct
+         object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventTypes List of stimuli for the transitions
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postState" type="STATE"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.SingleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postState post-transition state
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="addTransition" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="preState" type="STATE"/>
+      <param name="postStates" type="java.util.Set"/>
+      <param name="eventType" type="EVENTTYPE"/>
+      <param name="hook" type="org.apache.hadoop.yarn.state.MultipleArcTransition"/>
+      <doc>
+      <![CDATA[@return a NEW StateMachineFactory just like {@code this} with the current
+          transition added as a new legal transition
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+ @param preState pre-transition state
+ @param postStates valid post-transition states
+ @param eventType stimulus for the transition
+ @param hook transition hook]]>
+      </doc>
+    </method>
+    <method name="installTopology" return="org.apache.hadoop.yarn.state.StateMachineFactory"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[@return a StateMachineFactory just like {@code this}, except that if
+         you won't need any synchronization to build a state machine
+
+         Note that the returned StateMachineFactory is a distinct object.
+
+         This method is part of the API.
+
+         The only way you could distinguish the returned
+         StateMachineFactory from {@code this} would be by
+         measuring the performance of the derived 
+         {@code StateMachine} you can get from it.
+
+ Calling this is optional.  It doesn't change the semantics of the factory,
+   if you call it then when you use the factory there is no synchronization.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+      <param name="listener" type="org.apache.hadoop.yarn.state.StateTransitionListener"/>
+      <doc>
+      <![CDATA[A StateMachine that accepts a transition listener.
+ @param operand the object upon which the returned
+                {@link StateMachine} will operate.
+ @param initialState the state in which the returned
+                {@link StateMachine} will start.
+ @param listener An implementation of a {@link StateTransitionListener}.
+ @return A (@link StateMachine}.]]>
+      </doc>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+      <param name="initialState" type="STATE"/>
+    </method>
+    <method name="make" return="org.apache.hadoop.yarn.state.StateMachine"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="operand" type="OPERAND"/>
+    </method>
+    <method name="generateStateGraph" return="org.apache.hadoop.yarn.state.Graph"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="name" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate a graph represents the state graph of this StateMachine
+ @param name graph name
+ @return Graph object generated]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[State machine topology.
+ This object is semantically immutable.  If you have a
+ StateMachineFactory there's no operation in the API that changes
+ its semantic properties.
+
+ @param <OPERAND> The object type on which this state machine operates.
+ @param <STATE> The state of the entity.
+ @param <EVENTTYPE> The external eventType to be handled.
+ @param <EVENT> The event object.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.state.StateMachineFactory -->
+  <!-- start interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+  <interface name="StateTransitionListener"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="preTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="eventToBeProcessed" type="EVENT"/>
+      <doc>
+      <![CDATA[Pre Transition Hook. This will be called before transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param eventToBeProcessed Incoming Event.]]>
+      </doc>
+    </method>
+    <method name="postTransition"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="op" type="OPERAND"/>
+      <param name="beforeState" type="STATE"/>
+      <param name="afterState" type="STATE"/>
+      <param name="processedEvent" type="EVENT"/>
+      <doc>
+      <![CDATA[Post Transition Hook. This will be called after the transition.
+ @param op Operand.
+ @param beforeState State before transition.
+ @param afterState State after transition.
+ @param processedEvent Processed Event.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A State Transition Listener.
+ It exposes a pre and post transition hook called before and
+ after the transition.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.state.StateTransitionListener -->
+</package>
+<package name="org.apache.hadoop.yarn.util">
+  <!-- start class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <class name="AbstractLivelinessMonitor" extends="org.apache.hadoop.service.AbstractService"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String, org.apache.hadoop.yarn.util.Clock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AbstractLivelinessMonitor" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="expire"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="setExpireInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="expireInterval" type="int"/>
+    </method>
+    <method name="getExpireInterval" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="o" type="O"/>
+    </method>
+    <method name="setMonitorInterval"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="monitorInterval" type="long"/>
+    </method>
+    <method name="receivedPing"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="register"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+      <param name="expireTime" type="long"/>
+    </method>
+    <method name="unregister"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="ob" type="O"/>
+    </method>
+    <method name="resetTimer"
+      abstract="false" native="false" synchronized="true"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setResetTimeOnStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="resetTimeOnStart" type="boolean"/>
+    </method>
+    <field name="DEFAULT_EXPIRE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A simple liveliness monitor with which clients can register, trust the
+ component to monitor liveliness, get a call-back on expiry and then finally
+ unregister.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.AbstractLivelinessMonitor -->
+  <!-- start class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <class name="ApplicationClassLoader" extends="org.apache.hadoop.util.ApplicationClassLoader"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ApplicationClassLoader" type="java.net.URL[], java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ApplicationClassLoader" type="java.lang.String, java.lang.ClassLoader, java.util.List"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="MalformedURLException" type="java.net.MalformedURLException"/>
+    </constructor>
+    <doc>
+    <![CDATA[This type has been deprecated in favor of
+ {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ ApplicationClassLoader should use that type instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ApplicationClassLoader -->
+  <!-- start class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <class name="BoundedAppender" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="BoundedAppender" type="int"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="append" return="org.apache.hadoop.yarn.util.BoundedAppender"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="csq" type="java.lang.CharSequence"/>
+      <doc>
+      <![CDATA[Append a {@link CharSequence} considering {@link #limit}, truncating
+ from the head of {@code csq} or {@link #messages} when necessary.
+
+ @param csq the {@link CharSequence} to append
+ @return this]]>
+      </doc>
+    </method>
+    <method name="length" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current length of messages considering truncates
+ without header and ellipses.
+
+ @return current length]]>
+      </doc>
+    </method>
+    <method name="getLimit" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a string representation of the actual contents, displaying also a
+ header and ellipses when there was a truncate.
+
+ @return String representation of the {@link #messages}]]>
+      </doc>
+    </method>
+    <field name="TRUNCATED_MESSAGES_TEMPLATE" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A {@link CharSequence} appender that considers its {@link #limit} as upper
+ bound.
+ <p>
+ When {@link #limit} would be reached on append, past messages will be
+ truncated from head, and a header telling the user about truncation will be
+ prepended, with ellipses in between header and messages.
+ <p>
+ Note that header and ellipses are not counted against {@link #limit}.
+ <p>
+ An example:
+
+ <pre>
+ {@code
+   // At the beginning it's an empty string
+   final Appendable shortAppender = new BoundedAppender(80);
+   // The whole message fits into limit
+   shortAppender.append(
+       "message1 this is a very long message but fitting into limit\n");
+   // The first message is truncated, the second not
+   shortAppender.append("message2 this is shorter than the previous one\n");
+   // The first message is deleted, the second truncated, the third
+   // preserved
+   shortAppender.append("message3 this is even shorter message, maybe.\n");
+   // The first two are deleted, the third one truncated, the last preserved
+   shortAppender.append("message4 the shortest one, yet the greatest :)");
+   // Current contents are like this:
+   // Diagnostic messages truncated, showing last 80 chars out of 199:
+   // ...s is even shorter message, maybe.
+   // message4 the shortest one, yet the greatest :)
+ }
+ </pre>
+ <p>
+ Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
+ just like in {@link StringBuilder#append(CharSequence) original
+ implementation}.
+ <p>
+ Note that this class is not thread safe.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.BoundedAppender -->
+  <!-- start interface org.apache.hadoop.yarn.util.Clock -->
+  <interface name="Clock"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[A simple clock interface that gives you time.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.yarn.util.Clock -->
+  <!-- start class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <class name="ConverterUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ConverterUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getPathFromYarnURL" return="org.apache.hadoop.fs.Path"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="url" type="org.apache.hadoop.yarn.api.records.URL"/>
+      <exception name="URISyntaxException" type="java.net.URISyntaxException"/>
+      <doc>
+      <![CDATA[return a hadoop path from a given url
+ This method is deprecated, use {@link URL#toPath()} instead.
+ 
+ @param url
+          url to convert
+ @return path from {@link URL}
+ @throws URISyntaxException]]>
+      </doc>
+    </method>
+    <method name="getYarnUrlFromPath" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="path" type="org.apache.hadoop.fs.Path"/>
+    </method>
+    <method name="getYarnUrlFromURI" return="org.apache.hadoop.yarn.api.records.URL"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uri" type="java.net.URI"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="recordFactory" type="org.apache.hadoop.yarn.factories.RecordFactory"/>
+      <param name="applicationIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toString" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="cId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+    </method>
+    <method name="toNodeId" return="org.apache.hadoop.yarn.api.records.NodeId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="nodeIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toContainerId" return="org.apache.hadoop.yarn.api.records.ContainerId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationAttemptId" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="applicationAttemptIdStr" type="java.lang.String"/>
+    </method>
+    <method name="toApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appIdStr" type="java.lang.String"/>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="serviceAddr" type="java.net.InetSocketAddress"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service. Supposed
+ to be used for tokens other than RMDelegationToken. For
+ RMDelegationToken, use
+ {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
+ org.apache.hadoop.io.Text)} instead.
+
+ @param protoToken the yarn token
+ @param serviceAddr the connect address for the service
+ @return rpc token]]>
+      </doc>
+    </method>
+    <method name="convertFromYarn" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="protoToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+      <param name="service" type="org.apache.hadoop.io.Text"/>
+      <doc>
+      <![CDATA[Convert a protobuf token into a rpc token and set its service.
+
+ @param protoToken the yarn token
+ @param service the service for the token]]>
+      </doc>
+    </method>
+    <field name="APPLICATION_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="CONTAINER_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="APPLICATION_ATTEMPT_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[This class contains a set of utilities which help converting data structures
+ from/to 'serializableFormat' to/from hadoop/nativejava data structures.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ConverterUtils -->
+  <!-- start class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <class name="MonotonicClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="MonotonicClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get current time from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ @return a monotonic clock that counts in milliseconds.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A monotonic clock from some arbitrary time base in the past, counting in
+ milliseconds, and not affected by settimeofday or similar system clock
+ changes.
+ This is appropriate to use when computing how much longer to wait for an
+ interval to expire.
+ This function can return a negative value and it must be handled correctly
+ by callers. See the documentation of System#nanoTime for caveats.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.MonotonicClock -->
+  <!-- start class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <class name="ResourceCalculatorProcessTree" extends="org.apache.hadoop.conf.Configured"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ResourceCalculatorProcessTree" type="java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create process-tree instance with specified root process.
+
+ Subclass must override this.
+ @param root process-tree root-process]]>
+      </doc>
+    </constructor>
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Initialize the object.
+ @throws YarnException Throws an exception on error.]]>
+      </doc>
+    </method>
+    <method name="updateProcessTree"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Update the process-tree with latest state.
+
+ Each call to this function should increment the age of the running
+ processes that already exist in the process tree. Age is used other API's
+ of the interface.]]>
+      </doc>
+    </method>
+    <method name="getProcessTreeDump" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get a dump of the process-tree.
+
+ @return a string concatenating the dump of information of all the processes
+         in the process-tree]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree.
+
+ @return virtual memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree.
+
+ @return rss memory used by the process-tree in bytes,
+ {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getVirtualMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the virtual memory used by all the processes in the
+ process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return virtual memory used by the process-tree in bytes for
+ processes older than the specified age, {@link #UNAVAILABLE} if it
+ cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getRssMemorySize" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="olderThanAge" type="int"/>
+      <doc>
+      <![CDATA[Get the resident set size (rss) memory used by all the processes
+ in the process-tree that are older than the passed in age.
+
+ @param olderThanAge processes above this age are included in the
+                     memory addition
+ @return rss memory used by the process-tree in bytes for
+ processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ calculated.]]>
+      </doc>
+    </method>
+    <method name="getCumulativeCpuTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU time in millisecond used by all the processes in the
+ process-tree since the process-tree was created
+
+ @return cumulative CPU time in millisecond since the process-tree
+ created, {@link #UNAVAILABLE} if it cannot be calculated.]]>
+      </doc>
+    </method>
+    <method name="getCpuUsagePercent" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the CPU usage by all the processes in the process-tree based on
+ average between samples as a ratio of overall CPU cycles similar to top.
+ Thus, if 2 out of 4 cores are used this should return 200.0.
+ Note: UNAVAILABLE will be returned in case when CPU usage is not
+ available. It is NOT advised to return any other error code.
+
+ @return percentage CPU usage since the process-tree was created,
+ {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available.]]>
+      </doc>
+    </method>
+    <method name="checkPidPgrpidForMatch" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Verify that the tree process id is same as its process group id.
+ @return true if the process id matches else return false.]]>
+      </doc>
+    </method>
+    <method name="getResourceCalculatorProcessTree" return="org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pid" type="java.lang.String"/>
+      <param name="clazz" type="java.lang.Class"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Create the ResourceCalculatorProcessTree rooted to specified process 
+ from the class name and configure it. If class name is null, this method
+ will try and return a process tree plugin available for this system.
+
+ @param pid process pid of the root of the process tree
+ @param clazz class-name
+ @param conf configure the plugin with this.
+
+ @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
+         is not available for this system.]]>
+      </doc>
+    </method>
+    <field name="UNAVAILABLE" type="int"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[Interface class to obtain process resource usage
+ NOTE: This class should not be used by external users, but only by external
+ developers to extend and include their own process-tree implementation, 
+ especially for platforms other than Linux and Windows.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree -->
+  <!-- start class org.apache.hadoop.yarn.util.SystemClock -->
+  <class name="SystemClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="true" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="SystemClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getInstance" return="org.apache.hadoop.yarn.util.SystemClock"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current time from the system
+ clock in milliseconds.
+ 
+ NOTE: Do not use this to calculate a duration of expire or interval to sleep,
+ because it will be broken by settimeofday. Please use {@link MonotonicClock}
+ instead.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.SystemClock -->
+  <!-- start class org.apache.hadoop.yarn.util.UTCClock -->
+  <class name="UTCClock" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.yarn.util.Clock"/>
+    <constructor name="UTCClock"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <doc>
+    <![CDATA[Implementation of {@link Clock} that gives the current UTC time in
+ milliseconds.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.UTCClock -->
+</package>
+<package name="org.apache.hadoop.yarn.util.resource">
+</package>
+<package name="org.apache.hadoop.yarn.util.timeline">
+  <!-- start class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+  <class name="TimelineUtils" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="TimelineUtils"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string not in a pretty format
+ 
+ @param o
+          an object to serialize
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="dumpTimelineRecordtoJSON" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="o" type="java.lang.Object"/>
+      <param name="pretty" type="boolean"/>
+      <exception name="JsonGenerationException" type="com.fasterxml.jackson.core.JsonGenerationException"/>
+      <exception name="JsonMappingException" type="com.fasterxml.jackson.databind.JsonMappingException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Serialize a POJO object into a JSON string
+ 
+ @param o
+          an object to serialize
+ @param pretty
+          whether in a pretty format or not
+ @return a JSON string
+ @throws IOException
+ @throws JsonMappingException
+ @throws JsonGenerationException]]>
+      </doc>
+    </method>
+    <method name="timelineServiceEnabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service is enabled via configuration.
+
+ @param conf the configuration
+ @return whether the timeline service is enabled.]]>
+      </doc>
+    </method>
+    <method name="getTimelineServiceVersion" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns the timeline service version. It does not check whether the
+ timeline service itself is enabled.
+
+ @param conf the configuration
+ @return the timeline service version as a float.]]>
+      </doc>
+    </method>
+    <method name="timelineServiceV1_5Enabled" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Returns whether the timeline service v.1.5 is enabled by default via
+ configuration.
+
+ @param conf the configuration
+ @return whether the timeline service v.1.5 is enabled. V.1.5 refers to a
+ version equal to 1.5.]]>
+      </doc>
+    </method>
+    <method name="createTimelineAbout" return="org.apache.hadoop.yarn.api.records.timeline.TimelineAbout"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="about" type="java.lang.String"/>
+    </method>
+    <method name="getTimelineTokenServiceAddress" return="java.net.InetSocketAddress"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="buildTimelineTokenService" return="org.apache.hadoop.io.Text"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+    </method>
+    <method name="generateDefaultFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appName" type="java.lang.String"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+    </method>
+    <method name="generateFlowNameTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow name tag.
+
+ @param flowName flow name that identifies a distinct flow application which
+                 can be run repeatedly over time
+ @return flow name tag.]]>
+      </doc>
+    </method>
+    <method name="shortenFlowName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowName" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Shortens the flow name for the configured size by removing UUID if present.
+
+ @param flowName which has to be shortened
+ @param conf to resize the flow name
+ @return shortened flowName]]>
+      </doc>
+    </method>
+    <method name="generateFlowVersionTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowVersion" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Generate flow version tag.
+
+ @param flowVersion flow version that keeps track of the changes made to the
+                    flow
+ @return flow version tag.]]>
+      </doc>
+    </method>
+    <method name="generateFlowRunIdTag" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="true" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="flowRunId" type="long"/>
+      <doc>
+      <![CDATA[Generate flow run ID tag.
+
+ @param flowRunId flow run ID that identifies one instance (or specific
+                  execution) of that flow
+ @return flow run id tag.]]>
+      </doc>
+    </method>
+    <field name="FLOW_NAME_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_VERSION_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="FLOW_RUN_ID_TAG_PREFIX" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <field name="DEFAULT_FLOW_VERSION" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[The helper class for the timeline module.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.util.timeline.TimelineUtils -->
+</package>
+<package name="org.apache.hadoop.yarn.webapp.util">
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml
new file mode 100644
index 0000000000000..d06cc2fe00f6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml
@@ -0,0 +1,1412 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Jul 12 12:55:00 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Server Common 3.2.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.2.4.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.11/commons-codec-1.11.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.7/commons-lang3-3.7.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.35/slf4j-api-1.7.35.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.2.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/2.13.0/curator-client-2.13.0.jar:/maven/org/apache/curator/curator-recipes/2.13.0/curator-recipes-2.13.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.10.5.1/jackson-databind-2.10.5.1.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.2.4.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.10.5/jackson-annotations-2.10.5.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.2.4.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.2.4.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.10.5/jackson-core-2.10.5.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.10.5/jackson-module-jaxb-annotations-2.10.5.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.10.5/jackson-jaxrs-json-provider-2.10.5.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.10.5/jackson-jaxrs-base-2.10.5.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/target/hadoop-yarn-registry-3.2.4.jar:/maven/org/apache/curator/curator-framework/2.13.0/curator-framework-2.13.0.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/errorprone/error_prone_annotations/2.2.0/error_prone_annotations-2.2.0.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.2.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/zookeeper/zookeeper/3.4.14/zookeeper-3.4.14.jar:/maven/com/github/spotbugs/spotbugs-annotations/3.1.9/spotbugs-annotations-3.1.9.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/io/netty/netty/3.10.6.Final/netty-3.10.6.Final.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/org/apache/geronimo/specs/geronimo-jcache_1.0_spec/1.0-alpha-1/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/maven/org/ehcache/ehcache/3.3.1/ehcache-3.3.1.jar:/maven/com/zaxxer/HikariCP-java7/2.4.12/HikariCP-java7-2.4.12.jar:/maven/xerces/xercesImpl/2.11.0/xercesImpl-2.11.0.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Server Common 3.2.4 -->
+<package name="org.apache.hadoop.yarn.server">
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records">
+  <!-- start class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+  <class name="NodeHealthStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeHealthStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsNodeHealthy" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is the node healthy?
+ @return <code>true</code> if the node is healthy, else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeHealthStatus} is a summary of the health status of the node.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>
+     An indicator of whether the node is healthy, as determined by the
+     health-check script.
+   </li>
+   <li>The previous time at which the health status was reported.</li>
+   <li>A diagnostic report on the health status.</li>
+ </ul>
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.failover">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.amrmproxy">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.dao">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.exceptions">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.manager">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.router">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.resolver">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.exception">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.impl">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.scheduler">
+</package>
+<package name="org.apache.hadoop.yarn.server.security.http">
+</package>
+<package name="org.apache.hadoop.yarn.server.sharedcache">
+</package>
+<package name="org.apache.hadoop.yarn.server.uam">
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <class name="UnmanagedAMPoolManager" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedAMPoolManager" type="java.util.concurrent.ExecutorService"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Normally we should finish all applications before stop. If there are still
+ UAMs running, force kill all of them. Do parallel kill because of
+ performance reasons.
+
+ TODO: move waiting for the kill to finish into a separate thread, without
+ blocking the serviceStop.]]>
+      </doc>
+    </method>
+    <method name="createAndRegisterNewUAM" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new UAM and register the application, without specifying uamId and
+ appId. We will ask for an appId from RM and use it as the uamId.
+
+ @param registerRequest RegisterApplicationMasterRequest
+ @param conf configuration for this UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return uamId for the UAM
+ @throws YarnException if registerApplicationMaster fails
+ @throws IOException if registerApplicationMaster fails]]>
+      </doc>
+    </method>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM, using the provided uamId and appId.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return UAM token
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="uamToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM, using the provided uamIdentifier.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param uamToken UAM token
+ @param rmName name of the YarnRM
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="createUAM" return="org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Creates the UAM instance. Pull out to make unit test easy.
+
+ @param conf Configuration
+ @param appId application id
+ @param queueName queue of the application
+ @param submitter submitter name of the application
+ @param appNameSuffix application name suffix
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ @param rmName name of the YarnRM
+ @return the UAM instance]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register application master for the UAM.
+
+ @param uamId uam Id
+ @param registerRequest RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[AllocateAsync to an UAM.
+
+ @param uamId uam Id
+ @param request AllocateRequest
+ @param callback callback for response
+ @throws YarnException if allocate fails
+ @throws IOException if allocate fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finish an UAM/application.
+
+ @param uamId uam Id
+ @param request FinishApplicationMasterRequest
+ @return FinishApplicationMasterResponse
+ @throws YarnException if finishApplicationMaster call fails
+ @throws IOException if finishApplicationMaster call fails]]>
+      </doc>
+    </method>
+    <method name="getAllUAMIds" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of all running UAMs.
+
+ @return uamId set]]>
+      </doc>
+    </method>
+    <method name="hasUAMId" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return whether an UAM exists.
+
+ @param uamId uam Id
+ @return UAM exists or not]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Return the rmProxy relayer of an UAM.
+
+ @param uamId uam Id
+ @return the rmProxy relayer
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="drainUAMHeartbeats"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A service that manages a pool of UAM managers in
+ {@link UnmanagedApplicationManager}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+  <class name="UnmanagedApplicationManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedApplicationManager" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.yarn.api.records.ApplicationId, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param conf configuration
+ @param appId application Id to use for this UAM
+ @param queueName the queue of the UAM
+ @param submitter user name of the app
+ @param appNameSuffix the app name suffix to use
+ @param rmName name of the YarnRM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery. See {@link ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)}]]>
+      </doc>
+    </constructor>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM in the resource manager.
+
+ @return identifier uam identifier
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM in the resource manager.
+
+ @param amrmToken the UAM token
+ @throws IOException if re-attach fails
+ @throws YarnException if re-attach fails]]>
+      </doc>
+    </method>
+    <method name="createUAMProxy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this {@link UnmanagedApplicationManager} with the resource
+ manager.
+
+ @param request RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregisters from the resource manager and stops the request handler thread.
+
+ @param request the finishApplicationMaster request
+ @return the response
+ @throws YarnException if finishAM call fails
+ @throws IOException if finishAM call fails]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Force kill the UAM.
+
+ @return kill response
+ @throws IOException if fails to create rmProxy
+ @throws YarnException if force kill fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Sends the specified heart beat request to the resource manager and invokes
+ the callback asynchronously with the response.
+
+ @param request the allocate request
+ @param callback the callback method for the request
+ @throws YarnException if registerAM is not called yet]]>
+      </doc>
+    </method>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the application id of the UAM.
+
+ @return application id of the UAM]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the rmProxy relayer of this UAM.
+
+ @return rmProxy relayer of the UAM]]>
+      </doc>
+    </method>
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns RM proxy for the specified protocol type. Unit test cases can
+ override this method and return mock proxy instances.
+
+ @param protocol protocal of the proxy
+ @param config configuration
+ @param user ugi for the proxy connection
+ @param token token for the connection
+ @param <T> type of the proxy
+ @return the proxy instance
+ @throws IOException if fails to create the proxy]]>
+      </doc>
+    </method>
+    <method name="initializeUnmanagedAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Launch and initialize an unmanaged AM. First, it creates a new application
+ on the RM and negotiates a new attempt id. Then it waits for the RM
+ application attempt state to reach YarnApplicationAttemptState.LAUNCHED
+ after which it returns the AM-RM token.
+
+ @param appId application id
+ @return the UAM token
+ @throws IOException if initialize fails
+ @throws YarnException if initialize fails]]>
+      </doc>
+    </method>
+    <method name="getUAMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Gets the amrmToken of the unmanaged AM.
+
+ @return the amrmToken of the unmanaged AM.
+ @throws IOException if getApplicationReport fails
+ @throws YarnException if getApplicationReport fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="setHandlerThread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="thread" type="org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler"/>
+    </method>
+    <method name="drainHeartbeatThread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="APP_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[UnmanagedApplicationManager is used to register unmanaged application and
+ negotiate for resources from resource managers. An unmanagedAM is an AM that
+ is not launched and managed by the RM. Allocate calls are handled
+ asynchronously using {@link AsyncCallback}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+</package>
+<package name="org.apache.hadoop.yarn.server.utils">
+  <!-- start class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+  <class name="LeveldbIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Iterator"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB, org.iq80.leveldb.ReadOptions"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DBIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator using the specified underlying DBIterator]]>
+      </doc>
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so the key of the next BlockElement
+ returned greater than or equal to the specified targetKey.]]>
+      </doc>
+    </method>
+    <method name="seekToFirst"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so is is at the beginning of the Database.]]>
+      </doc>
+    </method>
+    <method name="seekToLast"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so it is at the end of of the Database.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns <tt>true</tt> if the iteration has more elements.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekNext" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration, without advancing the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="hasPrev" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return true if there is a previous entry in the iteration.]]>
+      </doc>
+    </method>
+    <method name="prev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration and rewinds the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekPrev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration, without rewinding the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Removes from the database the last element returned by the iterator.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes the iterator.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for a DBIterator to translate the raw RuntimeExceptions that
+ can be thrown into DBExceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp">
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp.dao">
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <class name="AppAttemptInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppAttemptInfo" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appAttemptState" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="amContainerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <class name="AppAttemptsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo"/>
+    </method>
+    <method name="getAttempts" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="attempt" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <class name="AppInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppInfo" type="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRunningContainers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinalAppStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSubmittedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTags" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregateResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregatePreemptedResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="currentAppAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="user" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="name" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="queue" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="type" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appState" type="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="runningContainers" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="float"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="applicationTags" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="unmanagedApplication" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <class name="AppsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appinfo" type="org.apache.hadoop.yarn.server.webapp.dao.AppInfo"/>
+    </method>
+    <method name="getApps" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="app" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <class name="ContainerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInfo" type="org.apache.hadoop.yarn.api.records.ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAssignedNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedResources" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a map of the allocated resources. The map key is the resource name,
+ and the value is the resource value.
+
+ @return the allocated resources map]]>
+      </doc>
+    </method>
+    <method name="hasCustomResources" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="containerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedMB" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedVCores" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="assignedNodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerExitStatus" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeHttpAddress" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedResources" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+  <class name="ContainersInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainersInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerInfo" type="org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo"/>
+    </method>
+    <method name="getContainers" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="container" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.3.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.3.xml
new file mode 100644
index 0000000000000..342de75310203
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.3.xml
@@ -0,0 +1,1456 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon May 09 18:09:38 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Server Common 3.3.3"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.3.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.18.3/reload4j-1.2.18.3.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.3.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.13.2.2/jackson-databind-2.13.2.2.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.3.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.13.2/jackson-annotations-2.13.2.jar:/maven/javax/ws/rs/javax.ws.rs-api/2.1.1/javax.ws.rs-api-2.1.1.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.3.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.3.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.13.2/jackson-core-2.13.2.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.13.2/jackson-module-jaxb-annotations-2.13.2.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.13.2/jackson-jaxrs-json-provider-2.13.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.13.2/jackson-jaxrs-base-2.13.2.jar:/build/source/hadoop-common-project/hadoop-registry/target/hadoop-registry-3.3.3.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.3.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/org/apache/geronimo/specs/geronimo-jcache_1.0_spec/1.0-alpha-1/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/maven/org/ehcache/ehcache/3.3.1/ehcache-3.3.1.jar:/maven/com/zaxxer/HikariCP-java7/2.4.12/HikariCP-java7-2.4.12.jar:/maven/xerces/xercesImpl/2.12.1/xercesImpl-2.12.1.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Server Common 3.3.3 -->
+<package name="org.apache.hadoop.yarn.server">
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records">
+  <!-- start class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+  <class name="NodeHealthStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeHealthStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsNodeHealthy" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is the node healthy?
+ @return <code>true</code> if the node is healthy, else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeHealthStatus} is a summary of the health status of the node.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>
+     An indicator of whether the node is healthy, as determined by the
+     health-check script.
+   </li>
+   <li>The previous time at which the health status was reported.</li>
+   <li>A diagnostic report on the health status.</li>
+ </ul>
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.failover">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.amrmproxy">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.dao">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.exceptions">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.manager">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.router">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.resolver">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.exception">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.impl">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.scheduler">
+</package>
+<package name="org.apache.hadoop.yarn.server.security.http">
+</package>
+<package name="org.apache.hadoop.yarn.server.sharedcache">
+</package>
+<package name="org.apache.hadoop.yarn.server.uam">
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <class name="UnmanagedAMPoolManager" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedAMPoolManager" type="java.util.concurrent.ExecutorService"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Normally we should finish all applications before stop. If there are still
+ UAMs running, force kill all of them. Do parallel kill because of
+ performance reasons.
+
+ TODO: move waiting for the kill to finish into a separate thread, without
+ blocking the serviceStop.]]>
+      </doc>
+    </method>
+    <method name="createAndRegisterNewUAM" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new UAM and register the application, without specifying uamId and
+ appId. We will ask for an appId from RM and use it as the uamId.
+
+ @param registerRequest RegisterApplicationMasterRequest
+ @param conf configuration for this UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return uamId for the UAM
+ @throws YarnException if registerApplicationMaster fails
+ @throws IOException if registerApplicationMaster fails]]>
+      </doc>
+    </method>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM, using the provided uamId and appId.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return UAM token
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="uamToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM, using the provided uamIdentifier.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param uamToken UAM token
+ @param rmName name of the YarnRM
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="createUAM" return="org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Creates the UAM instance. Pull out to make unit test easy.
+
+ @param conf Configuration
+ @param appId application id
+ @param queueName queue of the application
+ @param submitter submitter name of the application
+ @param appNameSuffix application name suffix
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ @param rmName name of the YarnRM
+ @return the UAM instance]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register application master for the UAM.
+
+ @param uamId uam Id
+ @param registerRequest RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[AllocateAsync to an UAM.
+
+ @param uamId uam Id
+ @param request AllocateRequest
+ @param callback callback for response
+ @throws YarnException if allocate fails
+ @throws IOException if allocate fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finish an UAM/application.
+
+ @param uamId uam Id
+ @param request FinishApplicationMasterRequest
+ @return FinishApplicationMasterResponse
+ @throws YarnException if finishApplicationMaster call fails
+ @throws IOException if finishApplicationMaster call fails]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Shutdown an UAM client without killing it in YarnRM.
+
+ @param uamId uam Id
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Shutdown all UAM clients without killing them in YarnRM.
+
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="getAllUAMIds" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of all running UAMs.
+
+ @return uamId set]]>
+      </doc>
+    </method>
+    <method name="hasUAMId" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return whether an UAM exists.
+
+ @param uamId uam Id
+ @return UAM exists or not]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Return the rmProxy relayer of an UAM.
+
+ @param uamId uam Id
+ @return the rmProxy relayer
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="drainUAMHeartbeats"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A service that manages a pool of UAM managers in
+ {@link UnmanagedApplicationManager}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+  <class name="UnmanagedApplicationManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedApplicationManager" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.yarn.api.records.ApplicationId, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param conf configuration
+ @param appId application Id to use for this UAM
+ @param queueName the queue of the UAM
+ @param submitter user name of the app
+ @param appNameSuffix the app name suffix to use
+ @param rmName name of the YarnRM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery. See {@link ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)}]]>
+      </doc>
+    </constructor>
+    <method name="createAMHeartbeatRequestHandler" return="org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="relayer" type="org.apache.hadoop.yarn.server.AMRMClientRelayer"/>
+    </method>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM in the resource manager.
+
+ @return identifier uam identifier
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM in the resource manager.
+
+ @param amrmToken the UAM token
+ @throws IOException if re-attach fails
+ @throws YarnException if re-attach fails]]>
+      </doc>
+    </method>
+    <method name="createUAMProxy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this {@link UnmanagedApplicationManager} with the resource
+ manager.
+
+ @param request RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregisters from the resource manager and stops the request handler thread.
+
+ @param request the finishApplicationMaster request
+ @return the response
+ @throws YarnException if finishAM call fails
+ @throws IOException if finishAM call fails]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Force kill the UAM.
+
+ @return kill response
+ @throws IOException if fails to create rmProxy
+ @throws YarnException if force kill fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Sends the specified heart beat request to the resource manager and invokes
+ the callback asynchronously with the response.
+
+ @param request the allocate request
+ @param callback the callback method for the request
+ @throws YarnException if registerAM is not called yet]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown this UAM client, without killing the UAM in the YarnRM side.]]>
+      </doc>
+    </method>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the application id of the UAM.
+
+ @return application id of the UAM]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the rmProxy relayer of this UAM.
+
+ @return rmProxy relayer of the UAM]]>
+      </doc>
+    </method>
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns RM proxy for the specified protocol type. Unit test cases can
+ override this method and return mock proxy instances.
+
+ @param protocol protocal of the proxy
+ @param config configuration
+ @param user ugi for the proxy connection
+ @param token token for the connection
+ @param <T> type of the proxy
+ @return the proxy instance
+ @throws IOException if fails to create the proxy]]>
+      </doc>
+    </method>
+    <method name="initializeUnmanagedAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Launch and initialize an unmanaged AM. First, it creates a new application
+ on the RM and negotiates a new attempt id. Then it waits for the RM
+ application attempt state to reach YarnApplicationAttemptState.LAUNCHED
+ after which it returns the AM-RM token.
+
+ @param appId application id
+ @return the UAM token
+ @throws IOException if initialize fails
+ @throws YarnException if initialize fails]]>
+      </doc>
+    </method>
+    <method name="getUAMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Gets the amrmToken of the unmanaged AM.
+
+ @return the amrmToken of the unmanaged AM.
+ @throws IOException if getApplicationReport fails
+ @throws YarnException if getApplicationReport fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="drainHeartbeatThread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isHeartbeatThreadAlive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="APP_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[UnmanagedApplicationManager is used to register unmanaged application and
+ negotiate for resources from resource managers. An unmanagedAM is an AM that
+ is not launched and managed by the RM. Allocate calls are handled
+ asynchronously using {@link AsyncCallback}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+</package>
+<package name="org.apache.hadoop.yarn.server.utils">
+  <!-- start class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+  <class name="LeveldbIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Iterator"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB, org.iq80.leveldb.ReadOptions"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DBIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator using the specified underlying DBIterator]]>
+      </doc>
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so the key of the next BlockElement
+ returned greater than or equal to the specified targetKey.]]>
+      </doc>
+    </method>
+    <method name="seekToFirst"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so is is at the beginning of the Database.]]>
+      </doc>
+    </method>
+    <method name="seekToLast"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so it is at the end of of the Database.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns <tt>true</tt> if the iteration has more elements.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekNext" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration, without advancing the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="hasPrev" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return true if there is a previous entry in the iteration.]]>
+      </doc>
+    </method>
+    <method name="prev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration and rewinds the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekPrev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration, without rewinding the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Removes from the database the last element returned by the iterator.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes the iterator.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for a DBIterator to translate the raw RuntimeExceptions that
+ can be thrown into DBExceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp">
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp.dao">
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <class name="AppAttemptInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppAttemptInfo" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appAttemptState" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="amContainerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <class name="AppAttemptsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo"/>
+    </method>
+    <method name="getAttempts" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="attempt" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <class name="AppInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppInfo" type="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRunningContainers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinalAppStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSubmittedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTags" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregateResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregatePreemptedResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="currentAppAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="user" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="name" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="queue" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="type" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appState" type="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="runningContainers" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="float"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="applicationTags" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="unmanagedApplication" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <class name="AppsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appinfo" type="org.apache.hadoop.yarn.server.webapp.dao.AppInfo"/>
+    </method>
+    <method name="getApps" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="app" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <class name="ContainerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInfo" type="org.apache.hadoop.yarn.api.records.ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAssignedNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedResources" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a map of the allocated resources. The map key is the resource name,
+ and the value is the resource value.
+
+ @return the allocated resources map]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCustomResources" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="containerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedMB" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedVCores" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="assignedNodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerExitStatus" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeHttpAddress" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedResources" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+  <class name="ContainersInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainersInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerInfo" type="org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo"/>
+    </method>
+    <method name="getContainers" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="container" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml
new file mode 100644
index 0000000000000..1f525eb844ab8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml
@@ -0,0 +1,1456 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Fri Jul 29 14:10:26 GMT 2022 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop YARN Server Common 3.3.4"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -verbose -classpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/classes:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-3.3.4.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-protobuf_3_7/1.1.1/hadoop-shaded-protobuf_3_7-1.1.1.jar:/maven/com/google/guava/guava/27.0-jre/guava-27.0-jre.jar:/maven/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar:/maven/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:/maven/org/checkerframework/checker-qual/2.5.2/checker-qual-2.5.2.jar:/maven/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar:/maven/org/codehaus/mojo/animal-sniffer-annotations/1.17/animal-sniffer-annotations-1.17.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar:/maven/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar:/maven/commons-codec/commons-codec/1.15/commons-codec-1.15.jar:/maven/commons-io/commons-io/2.8.0/commons-io-2.8.0.jar:/maven/commons-net/commons-net/3.6/commons-net-3.6.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar:/maven/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar:/maven/org/eclipse/jetty/jetty-server/9.4.43.v20210629/jetty-server-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-http/9.4.43.v20210629/jetty-http-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-io/9.4.43.v20210629/jetty-io-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util/9.4.43.v20210629/jetty-util-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-servlet/9.4.43.v20210629/jetty-servlet-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-security/9.4.43.v20210629/jetty-security-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-util-ajax/9.4.43.v20210629/jetty-util-ajax-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-webapp/9.4.43.v20210629/jetty-webapp-9.4.43.v20210629.jar:/maven/org/eclipse/jetty/jetty-xml/9.4.43.v20210629/jetty-xml-9.4.43.v20210629.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-core/1.19/jersey-core-1.19.jar:/maven/javax/ws/rs/jsr311-api/1.1.1/jsr311-api-1.1.1.jar:/maven/com/sun/jersey/jersey-servlet/1.19/jersey-servlet-1.19.jar:/maven/com/sun/jersey/jersey-json/1.19/jersey-json-1.19.jar:/maven/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/com/sun/jersey/jersey-server/1.19/jersey-server-1.19.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/ch/qos/reload4j/reload4j/1.2.22/reload4j-1.2.22.jar:/maven/commons-beanutils/commons-beanutils/1.9.4/commons-beanutils-1.9.4.jar:/maven/org/apache/commons/commons-configuration2/2.1.1/commons-configuration2-2.1.1.jar:/maven/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar:/maven/org/apache/commons/commons-text/1.4/commons-text-1.4.jar:/maven/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/com/google/re2j/re2j/1.1/re2j-1.1.jar:/maven/com/google/code/gson/gson/2.8.9/gson-2.8.9.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-3.3.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/9.8.1/nimbus-jose-jwt-9.8.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar:/maven/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar:/maven/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar:/maven/org/apache/kerby/kerb-simplekdc/1.0.1/kerb-simplekdc-1.0.1.jar:/maven/org/apache/kerby/kerb-client/1.0.1/kerb-client-1.0.1.jar:/maven/org/apache/kerby/kerby-config/1.0.1/kerby-config-1.0.1.jar:/maven/org/apache/kerby/kerb-common/1.0.1/kerb-common-1.0.1.jar:/maven/org/apache/kerby/kerb-crypto/1.0.1/kerb-crypto-1.0.1.jar:/maven/org/apache/kerby/kerb-util/1.0.1/kerb-util-1.0.1.jar:/maven/org/apache/kerby/token-provider/1.0.1/token-provider-1.0.1.jar:/maven/org/apache/kerby/kerb-admin/1.0.1/kerb-admin-1.0.1.jar:/maven/org/apache/kerby/kerb-server/1.0.1/kerb-server-1.0.1.jar:/maven/org/apache/kerby/kerb-identity/1.0.1/kerb-identity-1.0.1.jar:/maven/org/apache/kerby/kerby-xdr/1.0.1/kerby-xdr-1.0.1.jar:/maven/com/jcraft/jsch/0.1.55/jsch-0.1.55.jar:/maven/org/apache/curator/curator-client/4.2.0/curator-client-4.2.0.jar:/maven/org/apache/curator/curator-recipes/4.2.0/curator-recipes-4.2.0.jar:/maven/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar:/maven/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar:/maven/org/apache/kerby/kerb-core/1.0.1/kerb-core-1.0.1.jar:/maven/org/apache/kerby/kerby-pkix/1.0.1/kerby-pkix-1.0.1.jar:/maven/org/apache/kerby/kerby-asn1/1.0.1/kerby-asn1-1.0.1.jar:/maven/org/apache/kerby/kerby-util/1.0.1/kerby-util-1.0.1.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.12.7/jackson-databind-2.12.7.jar:/maven/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.3.0/woodstox-core-5.3.0.jar:/maven/dnsjava/dnsjava/2.1.7/dnsjava-2.1.7.jar:/maven/org/xerial/snappy/snappy-java/1.1.8.2/snappy-java-1.1.8.2.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-3.3.4.jar:/maven/javax/xml/bind/jaxb-api/2.2.11/jaxb-api-2.2.11.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.12.7/jackson-annotations-2.12.7.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-3.3.4.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-3.3.4.jar:/maven/com/squareup/okhttp3/okhttp/4.9.3/okhttp-4.9.3.jar:/maven/com/squareup/okio/okio/2.8.0/okio-2.8.0.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib/1.4.10/kotlin-stdlib-1.4.10.jar:/maven/org/jetbrains/kotlin/kotlin-stdlib-common/1.4.10/kotlin-stdlib-common-1.4.10.jar:/maven/com/sun/jersey/jersey-client/1.19/jersey-client-1.19.jar:/maven/com/google/inject/extensions/guice-servlet/4.0/guice-servlet-4.0.jar:/maven/com/google/inject/guice/4.0/guice-4.0.jar:/maven/javax/inject/javax.inject/1/javax.inject-1.jar:/maven/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/maven/com/sun/jersey/contribs/jersey-guice/1.19/jersey-guice-1.19.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.12.7/jackson-core-2.12.7.jar:/maven/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.7/jackson-module-jaxb-annotations-2.12.7.jar:/maven/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-json-provider/2.12.7/jackson-jaxrs-json-provider-2.12.7.jar:/maven/com/fasterxml/jackson/jaxrs/jackson-jaxrs-base/2.12.7/jackson-jaxrs-base-2.12.7.jar:/build/source/hadoop-common-project/hadoop-registry/target/hadoop-registry-3.3.4.jar:/maven/org/apache/curator/curator-framework/4.2.0/curator-framework-4.2.0.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/org/apache/hadoop/thirdparty/hadoop-shaded-guava/1.1.1/hadoop-shaded-guava-1.1.1.jar:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-3.3.4.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/org/apache/zookeeper/zookeeper/3.5.6/zookeeper-3.5.6.jar:/maven/org/apache/zookeeper/zookeeper-jute/3.5.6/zookeeper-jute-3.5.6.jar:/maven/org/apache/yetus/audience-annotations/0.5.0/audience-annotations-0.5.0.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/org/apache/geronimo/specs/geronimo-jcache_1.0_spec/1.0-alpha-1/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/maven/org/ehcache/ehcache/3.3.1/ehcache-3.3.1.jar:/maven/com/zaxxer/HikariCP-java7/2.4.12/HikariCP-java7-2.4.12.jar:/maven/xerces/xercesImpl/2.12.2/xercesImpl-2.12.2.jar:/maven/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar -sourcepath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-annotations.jar:/build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/jdiff.jar -apidir /build/source/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/site/jdiff/xml -apiname Apache Hadoop YARN Server Common 3.3.4 -->
+<package name="org.apache.hadoop.yarn.server">
+</package>
+<package name="org.apache.hadoop.yarn.server.api">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.client">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.impl.pb.service">
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records">
+  <!-- start class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+  <class name="NodeHealthStatus" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="NodeHealthStatus"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getIsNodeHealthy" return="boolean"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Is the node healthy?
+ @return <code>true</code> if the node is healthy, else <code>false</code>]]>
+      </doc>
+    </method>
+    <method name="getHealthReport" return="java.lang.String"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>diagnostic health report</em> of the node.
+ @return <em>diagnostic health report</em> of the node]]>
+      </doc>
+    </method>
+    <method name="getLastHealthReportTime" return="long"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the <em>last timestamp</em> at which the health report was received.
+ @return <em>last timestamp</em> at which the health report was received]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[{@code NodeHealthStatus} is a summary of the health status of the node.
+ <p>
+ It includes information such as:
+ <ul>
+   <li>
+     An indicator of whether the node is healthy, as determined by the
+     health-check script.
+   </li>
+   <li>The previous time at which the health status was reported.</li>
+   <li>A diagnostic report on the health status.</li>
+ </ul>
+ 
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.api.records.NodeHealthStatus -->
+</package>
+<package name="org.apache.hadoop.yarn.server.api.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.failover">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.amrmproxy">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.dao">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.exceptions">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.manager">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.policies.router">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.resolver">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.exception">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.impl">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.store.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.federation.utils">
+</package>
+<package name="org.apache.hadoop.yarn.server.metrics">
+</package>
+<package name="org.apache.hadoop.yarn.server.records">
+</package>
+<package name="org.apache.hadoop.yarn.server.records.impl.pb">
+</package>
+<package name="org.apache.hadoop.yarn.server.scheduler">
+</package>
+<package name="org.apache.hadoop.yarn.server.security.http">
+</package>
+<package name="org.apache.hadoop.yarn.server.sharedcache">
+</package>
+<package name="org.apache.hadoop.yarn.server.uam">
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <class name="UnmanagedAMPoolManager" extends="org.apache.hadoop.service.AbstractService"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedAMPoolManager" type="java.util.concurrent.ExecutorService"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="serviceStart"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+    </method>
+    <method name="serviceStop"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="Exception" type="java.lang.Exception"/>
+      <doc>
+      <![CDATA[Normally we should finish all applications before stop. If there are still
+ UAMs running, force kill all of them. Do parallel kill because of
+ performance reasons.
+
+ TODO: move waiting for the kill to finish into a separate thread, without
+ blocking the serviceStop.]]>
+      </doc>
+    </method>
+    <method name="createAndRegisterNewUAM" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Create a new UAM and register the application, without specifying uamId and
+ appId. We will ask for an appId from RM and use it as the uamId.
+
+ @param registerRequest RegisterApplicationMasterRequest
+ @param conf configuration for this UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return uamId for the UAM
+ @throws YarnException if registerApplicationMaster fails
+ @throws IOException if registerApplicationMaster fails]]>
+      </doc>
+    </method>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM, using the provided uamId and appId.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery.
+ @param rmName name of the YarnRM
+ @see ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)
+ @return UAM token
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="uamToken" type="org.apache.hadoop.security.token.Token"/>
+      <param name="rmName" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM, using the provided uamIdentifier.
+
+ @param uamId uam Id
+ @param conf configuration for this UAM
+ @param appId application id for the UAM
+ @param queueName queue of the application
+ @param submitter submitter name of the UAM
+ @param appNameSuffix application name suffix for the UAM
+ @param uamToken UAM token
+ @param rmName name of the YarnRM
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="createUAM" return="org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="queueName" type="java.lang.String"/>
+      <param name="submitter" type="java.lang.String"/>
+      <param name="appNameSuffix" type="java.lang.String"/>
+      <param name="keepContainersAcrossApplicationAttempts" type="boolean"/>
+      <param name="rmName" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Creates the UAM instance. Pull out to make unit test easy.
+
+ @param conf Configuration
+ @param appId application id
+ @param queueName queue of the application
+ @param submitter submitter name of the application
+ @param appNameSuffix application name suffix
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ @param rmName name of the YarnRM
+ @return the UAM instance]]>
+      </doc>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="registerRequest" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Register application master for the UAM.
+
+ @param uamId uam Id
+ @param registerRequest RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[AllocateAsync to an UAM.
+
+ @param uamId uam Id
+ @param request AllocateRequest
+ @param callback callback for response
+ @throws YarnException if allocate fails
+ @throws IOException if allocate fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Finish an UAM/application.
+
+ @param uamId uam Id
+ @param request FinishApplicationMasterRequest
+ @return FinishApplicationMasterResponse
+ @throws YarnException if finishApplicationMaster call fails
+ @throws IOException if finishApplicationMaster call fails]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Shutdown an UAM client without killing it in YarnRM.
+
+ @param uamId uam Id
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Shutdown all UAM clients without killing them in YarnRM.
+
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="getAllUAMIds" return="java.util.Set"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get the id of all running UAMs.
+
+ @return uamId set]]>
+      </doc>
+    </method>
+    <method name="hasUAMId" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <doc>
+      <![CDATA[Return whether an UAM exists.
+
+ @param uamId uam Id
+ @return UAM exists or not]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Return the rmProxy relayer of an UAM.
+
+ @param uamId uam Id
+ @return the rmProxy relayer
+ @throws YarnException if fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="uamId" type="java.lang.String"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+    </method>
+    <method name="drainUAMHeartbeats"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="LOG" type="org.slf4j.Logger"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[A service that manages a pool of UAM managers in
+ {@link UnmanagedApplicationManager}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager -->
+  <!-- start class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+  <class name="UnmanagedApplicationManager" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="UnmanagedApplicationManager" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.yarn.api.records.ApplicationId, java.lang.String, java.lang.String, java.lang.String, boolean, java.lang.String"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Constructor.
+
+ @param conf configuration
+ @param appId application Id to use for this UAM
+ @param queueName the queue of the UAM
+ @param submitter user name of the app
+ @param appNameSuffix the app name suffix to use
+ @param rmName name of the YarnRM
+ @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+          recovery. See {@link ApplicationSubmissionContext
+          #setKeepContainersAcrossApplicationAttempts(boolean)}]]>
+      </doc>
+    </constructor>
+    <method name="createAMHeartbeatRequestHandler" return="org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <param name="relayer" type="org.apache.hadoop.yarn.server.AMRMClientRelayer"/>
+    </method>
+    <method name="launchUAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Launch a new UAM in the resource manager.
+
+ @return identifier uam identifier
+ @throws YarnException if fails
+ @throws IOException if fails]]>
+      </doc>
+    </method>
+    <method name="reAttachUAM"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Re-attach to an existing UAM in the resource manager.
+
+ @param amrmToken the UAM token
+ @throws IOException if re-attach fails
+ @throws YarnException if re-attach fails]]>
+      </doc>
+    </method>
+    <method name="createUAMProxy"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="amrmToken" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+    </method>
+    <method name="registerApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Registers this {@link UnmanagedApplicationManager} with the resource
+ manager.
+
+ @param request RegisterApplicationMasterRequest
+ @return register response
+ @throws YarnException if register fails
+ @throws IOException if register fails]]>
+      </doc>
+    </method>
+    <method name="finishApplicationMaster" return="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Unregisters from the resource manager and stops the request handler thread.
+
+ @param request the finishApplicationMaster request
+ @return the response
+ @throws YarnException if finishAM call fails
+ @throws IOException if finishAM call fails]]>
+      </doc>
+    </method>
+    <method name="forceKillApplication" return="org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Force kill the UAM.
+
+ @return kill response
+ @throws IOException if fails to create rmProxy
+ @throws YarnException if force kill fails]]>
+      </doc>
+    </method>
+    <method name="allocateAsync"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest"/>
+      <param name="callback" type="org.apache.hadoop.yarn.util.AsyncCallback"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Sends the specified heart beat request to the resource manager and invokes
+ the callback asynchronously with the response.
+
+ @param request the allocate request
+ @param callback the callback method for the request
+ @throws YarnException if registerAM is not called yet]]>
+      </doc>
+    </method>
+    <method name="shutDownConnections"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown this UAM client, without killing the UAM in the YarnRM side.]]>
+      </doc>
+    </method>
+    <method name="getAppId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the application id of the UAM.
+
+ @return application id of the UAM]]>
+      </doc>
+    </method>
+    <method name="getAMRMClientRelayer" return="org.apache.hadoop.yarn.server.AMRMClientRelayer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Returns the rmProxy relayer of this UAM.
+
+ @return rmProxy relayer of the UAM]]>
+      </doc>
+    </method>
+    <method name="createRMProxy" return="T"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="protocol" type="java.lang.Class"/>
+      <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+      <param name="user" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="token" type="org.apache.hadoop.security.token.Token"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Returns RM proxy for the specified protocol type. Unit test cases can
+ override this method and return mock proxy instances.
+
+ @param protocol protocal of the proxy
+ @param config configuration
+ @param user ugi for the proxy connection
+ @param token token for the connection
+ @param <T> type of the proxy
+ @return the proxy instance
+ @throws IOException if fails to create the proxy]]>
+      </doc>
+    </method>
+    <method name="initializeUnmanagedAM" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Launch and initialize an unmanaged AM. First, it creates a new application
+ on the RM and negotiates a new attempt id. Then it waits for the RM
+ application attempt state to reach YarnApplicationAttemptState.LAUNCHED
+ after which it returns the AM-RM token.
+
+ @param appId application id
+ @return the UAM token
+ @throws IOException if initialize fails
+ @throws YarnException if initialize fails]]>
+      </doc>
+    </method>
+    <method name="getUAMToken" return="org.apache.hadoop.security.token.Token"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+      <doc>
+      <![CDATA[Gets the amrmToken of the unmanaged AM.
+
+ @return the amrmToken of the unmanaged AM.
+ @throws IOException if getApplicationReport fails
+ @throws YarnException if getApplicationReport fails]]>
+      </doc>
+    </method>
+    <method name="getRequestQueueSize" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="drainHeartbeatThread"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <method name="isHeartbeatThreadAlive" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </method>
+    <field name="APP_NAME" type="java.lang.String"
+      transient="false" volatile="false"
+      static="true" final="true" visibility="public"
+      deprecated="not deprecated">
+    </field>
+    <doc>
+    <![CDATA[UnmanagedApplicationManager is used to register unmanaged application and
+ negotiate for resources from resource managers. An unmanagedAM is an AM that
+ is not launched and managed by the RM. Allocate calls are handled
+ asynchronously using {@link AsyncCallback}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager -->
+</package>
+<package name="org.apache.hadoop.yarn.server.utils">
+  <!-- start class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+  <class name="LeveldbIterator" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="java.util.Iterator"/>
+    <implements name="java.io.Closeable"/>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DB, org.iq80.leveldb.ReadOptions"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator for the specified database]]>
+      </doc>
+    </constructor>
+    <constructor name="LeveldbIterator" type="org.iq80.leveldb.DBIterator"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Create an iterator using the specified underlying DBIterator]]>
+      </doc>
+    </constructor>
+    <method name="seek"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="key" type="byte[]"/>
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so the key of the next BlockElement
+ returned greater than or equal to the specified targetKey.]]>
+      </doc>
+    </method>
+    <method name="seekToFirst"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so is is at the beginning of the Database.]]>
+      </doc>
+    </method>
+    <method name="seekToLast"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Repositions the iterator so it is at the end of of the Database.]]>
+      </doc>
+    </method>
+    <method name="hasNext" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns <tt>true</tt> if the iteration has more elements.]]>
+      </doc>
+    </method>
+    <method name="next" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekNext" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Returns the next element in the iteration, without advancing the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="hasPrev" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return true if there is a previous entry in the iteration.]]>
+      </doc>
+    </method>
+    <method name="prev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration and rewinds the iteration.]]>
+      </doc>
+    </method>
+    <method name="peekPrev" return="java.util.Map.Entry"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[@return the previous element in the iteration, without rewinding the
+ iteration.]]>
+      </doc>
+    </method>
+    <method name="remove"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="DBException" type="org.iq80.leveldb.DBException"/>
+      <doc>
+      <![CDATA[Removes from the database the last element returned by the iterator.]]>
+      </doc>
+    </method>
+    <method name="close"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <exception name="IOException" type="java.io.IOException"/>
+      <doc>
+      <![CDATA[Closes the iterator.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[A wrapper for a DBIterator to translate the raw RuntimeExceptions that
+ can be thrown into DBExceptions.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.utils.LeveldbIterator -->
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp">
+</package>
+<package name="org.apache.hadoop.yarn.server.webapp.dao">
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <class name="AppAttemptInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppAttemptInfo" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppAttemptState" return="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appAttemptState" type="org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="amContainerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <class name="AppAttemptsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppAttemptsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="info" type="org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo"/>
+    </method>
+    <method name="getAttempts" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="attempt" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <class name="AppInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="AppInfo" type="org.apache.hadoop.yarn.api.records.ApplicationReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getAppId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getCurrentAppAttemptId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getUser" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getName" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getQueue" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getType" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getHost" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRpcPort" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppState" return="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getRunningContainers" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedCpuVcores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedMemoryMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getReservedGpus" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getProgress" return="float"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getOriginalTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getTrackingUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinalAppStatus" return="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getSubmittedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLaunchTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getApplicationTags" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="isUnmanagedApp" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAppNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAmNodeLabelExpression" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregateResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAggregatePreemptedResourceAllocation" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="appId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="currentAppAttemptId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="user" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="name" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="queue" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="type" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="host" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="rpcPort" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="appState" type="org.apache.hadoop.yarn.api.records.YarnApplicationState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="runningContainers" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="progress" type="float"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="originalTrackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="trackingUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finalAppStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="applicationTags" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="unmanagedApplication" type="boolean"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <class name="AppsInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="AppsInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="appinfo" type="org.apache.hadoop.yarn.server.webapp.dao.AppInfo"/>
+    </method>
+    <method name="getApps" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="app" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.AppsInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <class name="ContainerInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainerInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <constructor name="ContainerInfo" type="org.apache.hadoop.yarn.api.records.ContainerReport"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="getContainerId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedMB" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedVCores" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAssignedNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getPriority" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getStartedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getFinishedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getElapsedTime" return="long"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getDiagnosticsInfo" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getLogUrl" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerExitStatus" return="int"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getContainerState" return="org.apache.hadoop.yarn.api.records.ContainerState"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeHttpAddress" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getNodeId" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="getAllocatedResources" return="java.util.Map"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Return a map of the allocated resources. The map key is the resource name,
+ and the value is the resource value.
+
+ @return the allocated resources map]]>
+      </doc>
+    </method>
+    <method name="getExposedPorts" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <method name="hasCustomResources" return="boolean"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="containerId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedMB" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedVCores" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="assignedNodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="priority" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="startedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="finishedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="elapsedTime" type="long"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="diagnosticsInfo" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="logUrl" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerExitStatus" type="int"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="containerState" type="org.apache.hadoop.yarn.api.records.ContainerState"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeHttpAddress" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="nodeId" type="java.lang.String"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+    <field name="allocatedResources" type="java.util.Map"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo -->
+  <!-- start class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+  <class name="ContainersInfo" extends="java.lang.Object"
+    abstract="false"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="ContainersInfo"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="add"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="containerInfo" type="org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo"/>
+    </method>
+    <method name="getContainers" return="java.util.ArrayList"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </method>
+    <field name="container" type="java.util.ArrayList"
+      transient="false" volatile="false"
+      static="false" final="false" visibility="protected"
+      deprecated="not deprecated">
+    </field>
+  </class>
+  <!-- end class org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo -->
+</package>
+
+</api>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationBaseProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationBaseProtocol.java
index 8234c2fb80a01..3e8c0d69648e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationBaseProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationBaseProtocol.java
@@ -94,8 +94,8 @@ public interface ApplicationBaseProtocol {
    * @param request
    *          request for an application report
    * @return application report
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -126,8 +126,8 @@ public GetApplicationReportResponse getApplicationReport(
    *          request for report on applications
    * @return report on applications matching the given application types defined
    *         in the request
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see GetApplicationsRequest
    */
   @Public
@@ -166,8 +166,8 @@ public GetApplicationReportResponse getApplicationReport(
    * @param request
    *          request for an application attempt report
    * @return application attempt report
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -199,8 +199,8 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport(
    * @param request
    *          request for reports on all application attempts of an application
    * @return reports on all application attempts of an application
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -234,8 +234,8 @@ public GetApplicationAttemptsResponse getApplicationAttempts(
    * @param request
    *          request for a container report
    * @return container report
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -271,8 +271,8 @@ public GetContainerReportResponse getContainerReport(
    * @param request
    *          request for a list of container reports of an application attempt.
    * @return reports on all containers of an application attempt
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -293,8 +293,8 @@ public GetContainersResponse getContainers(GetContainersRequest request)
    * @param request
    *          request to get a delegation token for the client.
    * @return delegation token that can be used to talk to this service
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -308,8 +308,8 @@ public GetDelegationTokenResponse getDelegationToken(
    * @param request
    *          the delegation token to be renewed.
    * @return the new expiry time for the delegation token.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Private
   @Unstable
@@ -323,8 +323,8 @@ public RenewDelegationTokenResponse renewDelegationToken(
    * @param request
    *          the delegation token to be cancelled.
    * @return an empty response.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
index 941a688134f17..fa742915adf7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
@@ -112,8 +112,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
    * @param request request to get a new <code>ApplicationId</code>
    * @return response containing the new <code>ApplicationId</code> to be used
    * to submit an application
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see #submitApplication(SubmitApplicationRequest)
    */
   @Public
@@ -157,8 +157,8 @@ public GetNewApplicationResponse getNewApplication(
    * 
    * @param request request to submit a new application
    * @return (empty) response on accepting the submission
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see #getNewApplication(GetNewApplicationRequest)
    */
   @Public
@@ -184,8 +184,8 @@ public SubmitApplicationResponse submitApplication(
    * @param request request to fail an attempt
    * @return <code>ResourceManager</code> returns an empty response
    *         on success and throws an exception on rejecting the request
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)
    */
   @Public
@@ -210,8 +210,8 @@ public FailApplicationAttemptResponse failApplicationAttempt(
    * @param request request to abort a submitted application
    * @return <code>ResourceManager</code> returns an empty response
    *         on success and throws an exception on rejecting the request
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see #getQueueUserAcls(GetQueueUserAclsInfoRequest) 
    */
   @Public
@@ -232,8 +232,8 @@ public KillApplicationResponse forceKillApplication(
    * 
    * @param request request for cluster metrics
    * @return cluster metrics
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -252,8 +252,8 @@ public GetClusterMetricsResponse getClusterMetrics(
    * 
    * @param request request for report on all nodes
    * @return report on all nodes
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -274,8 +274,8 @@ public GetClusterNodesResponse getClusterNodes(
    * 
    * @param request request to get queue information
    * @return queue information
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -294,8 +294,8 @@ public GetQueueInfoResponse getQueueInfo(
    * 
    * @param request request to get queue acls for <em>current user</em>
    * @return queue acls for <em>current user</em>
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -309,8 +309,8 @@ public GetQueueUserAclsInfoResponse getQueueUserAcls(
    * 
    * @param request the application ID and the target queue
    * @return an empty response
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -383,7 +383,7 @@ GetNewReservationResponse getNewReservation(
    * @return response the {@link ReservationId} on accepting the submission
    * @throws YarnException if the request is invalid or reservation cannot be
    *           created successfully
-   * @throws IOException
+   * @throws IOException io error occur.
    * 
    */
   @Public
@@ -417,7 +417,7 @@ public ReservationSubmissionResponse submitReservation(
    * @return response empty on successfully updating the existing reservation
    * @throws YarnException if the request is invalid or reservation cannot be
    *           updated successfully
-   * @throws IOException
+   * @throws IOException io error occur.
    * 
    */
   @Public
@@ -439,7 +439,7 @@ public ReservationUpdateResponse updateReservation(
    * @return response empty on successfully deleting the existing reservation
    * @throws YarnException if the request is invalid or reservation cannot be
    *           deleted successfully
-   * @throws IOException
+   * @throws IOException io error occur.
    *
    */
   @Public
@@ -494,13 +494,13 @@ ReservationListResponse listReservations(
 
   /**
    * <p>
-   * The interface used by client to get node to labels mappings in existing cluster
+   * The interface used by client to get node to labels mappings in existing cluster.
    * </p>
    *
-   * @param request
+   * @param request get node to labels request.
    * @return node to labels mappings
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -510,13 +510,13 @@ public GetNodesToLabelsResponse getNodeToLabels(
   /**
    * <p>
    * The interface used by client to get labels to nodes mappings
-   * in existing cluster
+   * in existing cluster.
    * </p>
    *
-   * @param request
+   * @param request get label to nodes request.
    * @return labels to nodes mappings
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -530,8 +530,8 @@ public GetLabelsToNodesResponse getLabelsToNodes(
    *
    * @param request to get node labels collection of this cluster
    * @return node labels collection of this cluster
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -544,8 +544,8 @@ public GetClusterNodeLabelsResponse getClusterNodeLabels(
    * </p>
    * @param request to set priority of an application
    * @return an empty response
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -573,8 +573,8 @@ public UpdateApplicationPriorityResponse updateApplicationPriority(
    * @param request request to signal a container
    * @return <code>ResourceManager</code> returns an empty response
    *         on success and throws an exception on rejecting the request
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
@@ -691,7 +691,7 @@ GetClusterNodeAttributesResponse getClusterNodeAttributes(
    * @param request request to get nodes to attributes mapping.
    * @return nodes to attributes mappings.
    * @throws YarnException if any error happens inside YARN.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
index eb40fc7f3e0ad..cc92239e5c5f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationMasterProtocol.java
@@ -76,9 +76,9 @@ public interface ApplicationMasterProtocol {
    * </p>
    *
    * @param request registration request
-   * @return registration respose
-   * @throws YarnException
-   * @throws IOException
+   * @return registration response
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @throws InvalidApplicationMasterRequestException The exception is thrown
    *           when an ApplicationMaster tries to register more then once.
    * @see RegisterApplicationMasterRequest
@@ -104,8 +104,8 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
    * 
    * @param request completion request
    * @return completion response
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @see FinishApplicationMasterRequest
    * @see FinishApplicationMasterResponse
    */
@@ -154,8 +154,8 @@ public FinishApplicationMasterResponse finishApplicationMaster(
    * @param request
    *          allocation request
    * @return allocation response
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    * @throws InvalidApplicationMasterRequestException
    *           This exception is thrown when an ApplicationMaster calls allocate
    *           without registering first.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java
index d63fa117f91c6..4fa524e932fee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientSCMProtocol.java
@@ -55,8 +55,8 @@ public interface ClientSCMProtocol {
    *
    * @param request request to claim a resource in the shared cache
    * @return response indicating if the resource is already in the cache
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   public UseSharedCacheResourceResponse use(
       UseSharedCacheResourceRequest request) throws YarnException, IOException;
@@ -81,8 +81,8 @@ public UseSharedCacheResourceResponse use(
    *
    * @param request request to release a resource in the shared cache
    * @return (empty) response on releasing the resource
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   public ReleaseSharedCacheResourceResponse release(
       ReleaseSharedCacheResourceRequest request) throws YarnException, IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
index 0444440ebad16..9991248e666bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
@@ -102,8 +102,8 @@ public interface ContainerManagementProtocol {
    * @return response including conatinerIds of all successfully launched
    *         containers, a containerId-to-exception map for failed requests and
    *         a allServicesMetaData map.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -138,8 +138,8 @@ StartContainersResponse startContainers(StartContainersRequest request)
    * @return response which includes a list of containerIds of successfully
    *         stopped containers, a containerId-to-exception map for failed
    *         requests.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -174,8 +174,8 @@ StopContainersResponse stopContainers(StopContainersRequest request)
    *         successfully queried containers and a containerId-to-exception map
    *         for failed requests.
    * 
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Stable
@@ -195,8 +195,8 @@ GetContainerStatusesResponse getContainerStatuses(
    *         whose resource has been successfully increased and a
    *         containerId-to-exception map for failed requests.
    *
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPlugin.java
index 26b45f7c31fea..6974dda318b92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorPlugin.java
@@ -36,7 +36,7 @@ public interface CsiAdaptorPlugin extends CsiAdaptorProtocol {
    * customized configuration from yarn-site.xml.
    * @param driverName the name of the csi-driver.
    * @param conf configuration.
-   * @throws YarnException
+   * @throws YarnException exceptions from yarn servers.
    */
   void init(String driverName, Configuration conf) throws YarnException;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
index 4e064eb2196df..f78345a5d41e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/CsiAdaptorProtocol.java
@@ -39,8 +39,8 @@ public interface CsiAdaptorProtocol {
    * the name of the driver and its version.
    * @param request get plugin info request.
    * @return response that contains driver name and its version.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   GetPluginInfoResponse getPluginInfo(GetPluginInfoRequest request)
       throws YarnException, IOException;
@@ -51,8 +51,8 @@ GetPluginInfoResponse getPluginInfo(GetPluginInfoRequest request)
    * or not, with a detailed message.
    * @param request validate volume capability request.
    * @return validation response.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
       ValidateVolumeCapabilitiesRequest request) throws YarnException,
@@ -63,8 +63,8 @@ ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
    * to the local file system and become visible for clients.
    * @param request publish volume request.
    * @return publish volume response.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   NodePublishVolumeResponse nodePublishVolume(
       NodePublishVolumeRequest request) throws YarnException, IOException;
@@ -75,8 +75,8 @@ NodePublishVolumeResponse nodePublishVolume(
    * volume from given node.
    * @param request un-publish volume request.
    * @return un-publish volume response.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   NodeUnpublishVolumeResponse nodeUnpublishVolume(
       NodeUnpublishVolumeRequest request) throws YarnException, IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/package-info.java
index d1f4ea75b7db5..9fe3696e52eed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.api;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index 3b9c30ff114c7..788ff90ff3e33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.yarn.api.records.AMCommand;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.PreemptionMessage;
@@ -147,6 +148,23 @@ public static AllocateResponse newInstance(int responseId,
         .collectorInfo(collectorInfo).build();
   }
 
+  @Private
+  @Unstable
+  public static AllocateResponse newInstance(int responseId,
+      List<ContainerStatus> completedContainers,
+      List<Container> allocatedContainers, List<NodeReport> updatedNodes,
+      Resource availResources, AMCommand command, int numClusterNodes,
+      PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
+      List<UpdatedContainer> updatedContainers, CollectorInfo collectorInfo,
+      EnhancedHeadroom enhancedHeadroom) {
+    AllocateResponse response =
+        newInstance(responseId, completedContainers, allocatedContainers,
+            updatedNodes, availResources, command, numClusterNodes, preempt,
+            nmTokens, amRMToken, updatedContainers, collectorInfo);
+    response.setEnhancedHeadroom(enhancedHeadroom);
+    return response;
+  }
+
   /**
    * If the <code>ResourceManager</code> needs the
    * <code>ApplicationMaster</code> to take some action then it will send an
@@ -439,6 +457,14 @@ public static AllocateResponseBuilder newBuilder() {
     return new AllocateResponseBuilder();
   }
 
+  @Public
+  @Unstable
+  public abstract EnhancedHeadroom getEnhancedHeadroom();
+
+  @Private
+  @Unstable
+  public abstract void setEnhancedHeadroom(EnhancedHeadroom enhancedHeadroom);
+
   /**
    * Class to construct instances of {@link AllocateResponse} with specific
    * options.
@@ -666,6 +692,18 @@ public AllocateResponseBuilder containersFromPreviousAttempt(
       return this;
     }
 
+    @Public
+    @Unstable
+    public EnhancedHeadroom getEnhancedHeadroom() {
+      return allocateResponse.getEnhancedHeadroom();
+    }
+
+    @Private
+    @Unstable
+    public void setEnhancedHeadroom(EnhancedHeadroom enhancedHeadroom){
+      allocateResponse.setEnhancedHeadroom(enhancedHeadroom);
+    }
+
     /**
      * Return generated {@link AllocateResponse} object.
      * @return {@link AllocateResponse}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
index 5a00fa93f0c6c..37c7d8787cd3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
@@ -137,6 +137,7 @@ public static GetApplicationsRequest newInstance(
    *
    *
    * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   * @param applicationTypes application types.
    * @return a report of Applications in {@link GetApplicationsRequest}
    */
   @Public
@@ -158,6 +159,7 @@ public static GetApplicationsRequest newInstance(
    *
    *
    * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   * @param applicationStates application states.
    * @return  a report of Applications in {@link GetApplicationsRequest}
    */
   @Public
@@ -173,12 +175,14 @@ public static GetApplicationsRequest newInstance(
   /**
    * <p>
    * The request from clients to get a report of Applications matching the
-   * giving and application types and application types in the cluster from the
+   * giving and application types and application states in the cluster from the
    * <code>ResourceManager</code>.
    * </p>
    *
    *
    * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+   * @param applicationStates application states.
+   * @param applicationTypes application types.
    * @return  a report of Applications in <code>GetApplicationsRequest</code>
    */
   @Public
@@ -309,20 +313,20 @@ public static GetApplicationsRequest newInstance(
   public abstract Range<Long> getStartRange();
 
   /**
-   * Set the range of start times to filter applications on
+   * Set the range of start times to filter applications.
    *
-   * @param range
+   * @param range range of start times.
    */
   @Private
   @Unstable
   public abstract void setStartRange(Range<Long> range);
 
   /**
-   * Set the range of start times to filter applications on
+   * Set the range of start times to filter applications.
    *
    * @param begin beginning of the range
    * @param end end of the range
-   * @throws IllegalArgumentException
+   * @throws IllegalArgumentException if an argument is invalid.
    */
   @Private
   @Unstable
@@ -330,7 +334,7 @@ public abstract void setStartRange(long begin, long end)
       throws IllegalArgumentException;
 
   /**
-   * Get the range of finish times to filter applications on
+   * Get the range of finish times to filter applications.
    *
    * @return {@link Range} of finish times to filter applications on
    */
@@ -339,38 +343,38 @@ public abstract void setStartRange(long begin, long end)
   public abstract Range<Long> getFinishRange();
 
   /**
-   * Set the range of finish times to filter applications on
+   * Set the range of finish times to filter applications.
    *
-   * @param range
+   * @param range range of finish times.
    */
   @Private
   @Unstable
   public abstract void setFinishRange(Range<Long> range);
 
   /**
-   * Set the range of finish times to filter applications on
+   * Set the range of finish times to filter applications.
    *
    * @param begin beginning of the range
    * @param end end of the range
-   * @throws IllegalArgumentException
+   * @throws IllegalArgumentException if an argument is invalid.
    */
   @Private
   @Unstable
   public abstract void setFinishRange(long begin, long end);
 
   /**
-   * Get the tags to filter applications on
+   * Get the tags to filter applications.
    *
-   * @return list of tags to filter on
+   * @return list of tags to filter.
    */
   @Private
   @Unstable
   public abstract Set<String> getApplicationTags();
 
   /**
-   * Set the list of tags to filter applications on
+   * Set the list of tags to filter applications.
    *
-   * @param tags list of tags to filter on
+   * @param tags list of tags to filter.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java
index b0ccd906a321f..5a3f5c9ec34bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java
@@ -42,7 +42,7 @@ public abstract class GetClusterNodeAttributesResponse {
   /**
    * Create instance of GetClusterNodeAttributesResponse.
    *
-   * @param attributes
+   * @param attributes Map of Node attributeKey to Type.
    * @return GetClusterNodeAttributesResponse.
    */
   public static GetClusterNodeAttributesResponse newInstance(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java
index 75baf6e9ff558..34d9fe83fd61c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java
@@ -65,6 +65,7 @@ public static GetContainerStatusesResponse newInstance(
 
   /**
    * Set the <code>ContainerStatus</code>es of the requested containers.
+   * @param statuses <code>ContainerStatus</code>es of the requested containers.
    */
   @Private
   @Unstable
@@ -72,7 +73,7 @@ public static GetContainerStatusesResponse newInstance(
 
   /**
    * Get the containerId-to-exception map in which the exception indicates error
-   * from per container for failed requests
+   * from per container for failed requests.
    * @return map of containerId-to-exception
    */
   @Public
@@ -81,7 +82,9 @@ public static GetContainerStatusesResponse newInstance(
 
   /**
    * Set the containerId-to-exception map in which the exception indicates error
-   * from per container for failed requests
+   * from per container for failed requests.
+   *
+   * @param failedContainers containerId-to-exception map.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
index ef0bf60fa9840..f105359110418 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
@@ -24,7 +24,6 @@
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.util.Records;
 
 public abstract class GetLabelsToNodesResponse {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java
index 89fca9fbbddf0..3f29cbfa86b97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java
@@ -80,6 +80,8 @@ public abstract void setLocalizationStatuses(
   /**
    * Set the containerId-to-exception map in which the exception indicates error
    * from per container for failed request.
+   *
+   * @param failedContainers containerId-to-exception map.
    */
   @InterfaceAudience.Private
   public abstract void setFailedRequests(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java
index d7d80bff4b308..ccb22a4672264 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNewReservationResponse.java
@@ -20,7 +20,6 @@
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.util.Records;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java
index 4fcd8da693690..0da4bd8cb9aaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToAttributesRequest.java
@@ -48,7 +48,7 @@ public static GetNodesToAttributesRequest newInstance(Set<String> hostNames) {
   /**
    * Set hostnames for which mapping is required.
    *
-   * @param hostnames
+   * @param hostnames Set of hostnames.
    */
   @InterfaceAudience.Public
   @InterfaceStability.Evolving
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
index 1d821203a9128..ea2b64207e402 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/IncreaseContainersResourceResponse.java
@@ -68,6 +68,9 @@ public static IncreaseContainersResourceResponse newInstance(
   /**
    * Set the list of containerIds of containers whose resource have
    * been successfully increased.
+   *
+   * @param succeedIncreasedContainers list of containerIds of containers whose resource have
+   * been successfully increased.
    */
   @Private
   @Unstable
@@ -86,6 +89,8 @@ public abstract void setSuccessfullyIncreasedContainers(
   /**
    * Set the containerId-to-exception map in which the exception indicates
    * error from each container for failed requests.
+   *
+   * @param failedRequests map of containerId-to-exception.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java
index 83495760be07e..15c63fb11cb3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationResponse.java
@@ -66,6 +66,8 @@ public static KillApplicationResponse newInstance(boolean isKillCompleted) {
 
   /**
    * Set the flag which indicates that the process of killing application is completed or not.
+   * @param isKillCompleted true if the process of killing application has completed,
+   *     false otherwise.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
index f2d537ae07293..6a6d781b91072 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
@@ -53,6 +53,9 @@ public abstract class RegisterApplicationMasterRequest {
    *  <li>trackingUrl: null</li>
    * </ul>
    * The port is allowed to be any integer larger than or equal to -1.
+   * @param host host on which the ApplicationMaster is running.
+   * @param port the RPC port  on which the ApplicationMaster is responding.
+   * @param trackingUrl tracking URL for the ApplicationMaster.
    * @return the new instance of <code>RegisterApplicationMasterRequest</code>
    */
   @Public
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 1f8a151a4c54e..894e8c46d69de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -94,7 +94,7 @@ public static RegisterApplicationMasterResponse newInstance(
 
   /**
    * Set the <code>ApplicationACL</code>s for the application. 
-   * @param acls
+   * @param acls ApplicationACLs for the application.
    */
   @Private
   @Unstable
@@ -113,6 +113,8 @@ public static RegisterApplicationMasterResponse newInstance(
 
   /**
    * Set ClientToAMToken master key.
+   *
+   * @param key ClientToAMToken master key.
    */
   @Public
   @Stable
@@ -127,7 +129,8 @@ public static RegisterApplicationMasterResponse newInstance(
   public abstract String getQueue();
   
   /**
-   * <p>Set the queue that the application was placed in.<p>
+   * <p>Set the queue that the application was placed in.</p>
+   * @param queue queue.
    */
   @Public
   @Stable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/SignalContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/SignalContainerRequest.java
index 2a3861a773098..28cc8ea5b4c50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/SignalContainerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/SignalContainerRequest.java
@@ -56,6 +56,7 @@ public static SignalContainerRequest newInstance(ContainerId containerId,
 
   /**
    * Set the <code>ContainerId</code> of the container to signal.
+   * @param containerId containerId.
    */
   @Public
   @Unstable
@@ -71,6 +72,7 @@ public static SignalContainerRequest newInstance(ContainerId containerId,
 
   /**
    * Set the <code>SignalContainerCommand</code> of the signal request.
+   * @param command signal container command.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java
index 8905ae2ae4d96..c40250a36d7bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java
@@ -86,7 +86,8 @@ public abstract void setSuccessfullyStartedContainers(
 
   /**
    * Set the containerId-to-exception map in which the exception indicates error
-   * from per container for failed requests
+   * from per container for failed requests.
+   * @param failedContainers container for failed requests.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java
index 2763bc94b4c0a..3fe9eaa12f18c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java
@@ -65,6 +65,7 @@ public static StopContainersResponse newInstance(
 
   /**
    * Set the list of containerIds of successfully stopped containers.
+   * @param succeededRequests the list of containerIds of successfully stopped containers.
    */
   @Private
   @Unstable
@@ -73,8 +74,8 @@ public abstract void setSuccessfullyStoppedContainers(
 
   /**
    * Get the containerId-to-exception map in which the exception indicates error
-   * from per container for failed requests
-   * @return map of containerId-to-exception
+   * from per container for failed requests.
+   * @return map of containerId-to-exception.
    */
   @Public
   @Stable
@@ -82,7 +83,8 @@ public abstract void setSuccessfullyStoppedContainers(
 
   /**
    * Set the containerId-to-exception map in which the exception indicates error
-   * from per container for failed requests
+   * from per container for failed requests.
+   * @param failedRequests map of containerId-to-exception.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/package-info.java
index 503c4f0ed259b..c20779b1274bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.api.protocolrecords;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 1d4ffc9b5b9ee..71f28a165abdb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -516,7 +516,7 @@ public abstract void setAMContainerResourceRequests(
       List<ResourceRequest> requests);
 
   /**
-   * Get the attemptFailuresValidityInterval in milliseconds for the application
+   * Get the attemptFailuresValidityInterval in milliseconds for the application.
    *
    * @return the attemptFailuresValidityInterval
    */
@@ -525,8 +525,8 @@ public abstract void setAMContainerResourceRequests(
   public abstract long getAttemptFailuresValidityInterval();
 
   /**
-   * Set the attemptFailuresValidityInterval in milliseconds for the application
-   * @param attemptFailuresValidityInterval
+   * Set the attemptFailuresValidityInterval in milliseconds for the application.
+   * @param attemptFailuresValidityInterval attempt failures validity interval.
    */
   @Public
   @Stable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java
new file mode 100644
index 0000000000000..7a5ff6adcc5ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/EnhancedHeadroom.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Enhanced head room in AllocateResponse.
+ * This provides a channel for RMs to return load information for AMRMProxy
+ * decision making when rerouting resource requests.
+ *
+ * Contains total pending container count and active cores for a cluster.
+ */
+public abstract class EnhancedHeadroom {
+  public static EnhancedHeadroom newInstance(int totalPendingCount,
+      int totalActiveCores) {
+    EnhancedHeadroom enhancedHeadroom =
+        Records.newRecord(EnhancedHeadroom.class);
+    enhancedHeadroom.setTotalPendingCount(totalPendingCount);
+    enhancedHeadroom.setTotalActiveCores(totalActiveCores);
+    return enhancedHeadroom;
+  }
+
+  /**
+   * Set total pending container count.
+   * @param totalPendingCount the pending container count
+   */
+  public abstract void setTotalPendingCount(int totalPendingCount);
+
+  /**
+   * Get total pending container count.
+   * @return the pending container count
+   */
+  public abstract int getTotalPendingCount();
+
+  /**
+   * Set total active cores for the cluster.
+   * @param totalActiveCores the total active cores for the cluster
+   */
+  public abstract void setTotalActiveCores(int totalActiveCores);
+
+  /**
+   * Get total active cores for the cluster.
+   * @return totalActiveCores the total active cores for the cluster
+   */
+  public abstract int getTotalActiveCores();
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("<pendingCount:").append(this.getTotalPendingCount());
+    sb.append(", activeCores:").append(this.getTotalActiveCores());
+    sb.append(">");
+    return sb.toString();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java
index bca95b7fef10a..ca161173e25b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java
@@ -59,7 +59,7 @@ public static LocalizationStatus newInstance(String resourceKey,
 
   /**
    * Sets the resource key.
-   * @param resourceKey
+   * @param resourceKey resource key.
    */
   @InterfaceAudience.Private
   public abstract void setResourceKey(String resourceKey);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
index e58012d68b850..0e45b37e3b9d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
@@ -157,7 +157,7 @@ public static LogAggregationContext newInstance(String includePattern,
    * Set include pattern. This includePattern only takes affect
    * on logs that exist at the time of application finish.
    *
-   * @param includePattern
+   * @param includePattern include pattern.
    */
   @Public
   @Unstable
@@ -177,7 +177,7 @@ public static LogAggregationContext newInstance(String includePattern,
    * Set exclude pattern. This excludePattern only takes affect
    * on logs that exist at the time of application finish.
    *
-   * @param excludePattern
+   * @param excludePattern exclude pattern.
    */
   @Public
   @Unstable
@@ -195,7 +195,9 @@ public static LogAggregationContext newInstance(String includePattern,
   /**
    * Set include pattern in a rolling fashion.
    * 
-   * @param rolledLogsIncludePattern
+   * @param rolledLogsIncludePattern It uses Java Regex to filter the log files
+   * which match the defined include pattern and those log files
+   * will be aggregated in a rolling fashion.
    */
   @Public
   @Unstable
@@ -214,7 +216,7 @@ public abstract void setRolledLogsIncludePattern(
   /**
    * Set exclude pattern for in a rolling fashion.
    * 
-   * @param rolledLogsExcludePattern
+   * @param rolledLogsExcludePattern rolled logs exclude pattern.
    */
   @Public
   @Unstable
@@ -233,7 +235,7 @@ public abstract void setRolledLogsExcludePattern(
   /**
    * Set the log aggregation policy class.
    *
-   * @param className
+   * @param className log aggregation policy class name.
    */
   @Public
   @Unstable
@@ -255,7 +257,7 @@ public abstract void setLogAggregationPolicyClassName(
    * It is up to the log aggregation policy class to decide how to parse
    * the parameters string.
    *
-   * @param parameters
+   * @param parameters log aggregation policy parameters.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 625ad234081fb..1c6ab43d33aa7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -243,6 +243,7 @@ public Integer getDecommissioningTimeout() {
 
   /**
    * Set the decommissioning timeout in seconds (null indicates absent timeout).
+   * @param decommissioningTimeout decommissioning time out.
    * */
   public void setDecommissioningTimeout(Integer decommissioningTimeout) {}
 
@@ -256,7 +257,8 @@ public NodeUpdateType getNodeUpdateType() {
 
   /**
    * Set the node update type (null indicates absent node update type).
-   * */
+   * @param nodeUpdateType node update type.
+   */
   public void setNodeUpdateType(NodeUpdateType nodeUpdateType) {}
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
index 707554cd1c47c..c4d78f0c7ae22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
@@ -251,6 +251,7 @@ public static QueueInfo newInstance(String queueName,
   
   /**
    * Set the <code>accessible node labels</code> of the queue.
+   * @param labels node label expression of the queue.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index b91658f759c89..0c10e01768522 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.api.records;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -30,7 +29,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
@@ -469,6 +467,10 @@ public String toString() {
     return getFormattedString(String.valueOf(getMemorySize()));
   }
 
+  public String toFormattedString() {
+    return getFormattedString();
+  }
+
   private String getFormattedString(String memory) {
     StringBuilder sb = new StringBuilder();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index a8639108615f9..5ffc05e3c7c4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -437,17 +437,17 @@ public ExecutionTypeRequest getExecutionTypeRequest() {
 
   /**
    * <p>For a request at a network hierarchy level, set whether locality can be relaxed
-   * to that level and beyond.<p>
+   * to that level and beyond.</p>
    * 
    * <p>If the flag is off on a rack-level <code>ResourceRequest</code>,
    * containers at that request's priority will not be assigned to nodes on that
    * request's rack unless requests specifically for those nodes have also been
-   * submitted.<p>
+   * submitted.</p>
    * 
    * <p>If the flag is off on an {@link ResourceRequest#ANY}-level
    * <code>ResourceRequest</code>, containers at that request's priority will
    * only be assigned on racks for which specific requests have also been
-   * submitted.<p>
+   * submitted.</p>
    * 
    * <p>For example, to request a container strictly on a specific node, the
    * corresponding rack-level and any-level requests should have locality
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java
index 9355a234ebbb0..fc76269d1b7a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java
@@ -38,6 +38,8 @@ public static SerializedException newInstance(Throwable e) {
   /**
    * Constructs a new <code>SerializedException</code> with the specified detail
    * message and cause.
+   * @param message exception detail message.
+   * @param cause cause of the exception.
    */
   @Private
   @Unstable
@@ -46,6 +48,7 @@ public static SerializedException newInstance(Throwable e) {
   /**
    * Constructs a new <code>SerializedException</code> with the specified detail
    * message.
+   * @param message exception detail message.
    */
   @Private
   @Unstable
@@ -53,6 +56,7 @@ public static SerializedException newInstance(Throwable e) {
 
   /**
    * Constructs a new <code>SerializedException</code> with the specified cause.
+   * @param cause cause of the exception.
    */
   @Private
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java
index fc3edf7fb74b6..f460e60f483b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnClusterMetrics.java
@@ -53,6 +53,20 @@ public static YarnClusterMetrics newInstance(int numNodeManagers) {
   @Unstable
   public abstract void setNumNodeManagers(int numNodeManagers);
 
+  /**
+   * Get the number of <code>DecommissioningNodeManager</code>s in the cluster.
+   *
+   * @return number of <code>DecommissioningNodeManager</code>s in the cluster
+   */
+  @Public
+  @Unstable
+  public abstract int getNumDecommissioningNodeManagers();
+
+  @Private
+  @Unstable
+  public abstract void setNumDecommissioningNodeManagers(
+      int numDecommissioningNodeManagers);
+
   /**
    * Get the number of <code>DecommissionedNodeManager</code>s in the cluster.
    * 
@@ -119,4 +133,16 @@ public abstract void setNumDecommissionedNodeManagers(
   @Unstable
   public abstract void setNumRebootedNodeManagers(int numRebootedNodeManagers);
 
+  /**
+   * Get the number of <code>ShutdownNodeManager</code>s in the cluster.
+   *
+   * @return number of <code>ShutdownNodeManager</code>s in the cluster
+   */
+  @Public
+  @Unstable
+  public abstract int getNumShutdownNodeManagers();
+
+  @Private
+  @Unstable
+  public abstract void setNumShutdownNodeManagers(int numShutdownNodeManagers);
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/package-info.java
index b6db5403f0d58..94fed33e2c18d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.api.records;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
index d592167b862cb..3462441161fc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -39,12 +39,12 @@ public class TimelineHealth {
    * Timline health status.
    *
    * RUNNING - Service is up and running
-   * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+   * CONNECTION_FAULURE - isConnectionAlive() of reader / writer implementation
    *    reported an error
    */
   public enum TimelineHealthStatus {
     RUNNING,
-    READER_CONNECTION_FAILURE
+    CONNECTION_FAILURE
   }
 
   private TimelineHealthStatus healthStatus;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
index ae85e82bf37a1..d888e3c50edad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
@@ -44,12 +44,12 @@ public void close() throws Exception {
   }
 
   /**
-   * Opens an InputStream at the indicated file
-   * @param bootstrapConf Configuration
-   * @param name The configuration file name
+   * Opens an InputStream at the indicated file.
+   * @param bootstrapConf Configuration.
+   * @param name The configuration file name.
    * @return configuration
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   public abstract InputStream getConfigurationInputStream(
       Configuration bootstrapConf, String name) throws YarnException,
@@ -57,12 +57,15 @@ public abstract InputStream getConfigurationInputStream(
 
   /**
    * Derived classes initialize themselves using this method.
+   * @param bootstrapConf bootstrap configuration.
+   * @throws Exception exception occur.
    */
   public abstract void initInternal(Configuration bootstrapConf)
       throws Exception;
 
   /**
    * Derived classes close themselves using this method.
+   * @throws Exception exception occur.
    */
   public abstract void closeInternal() throws Exception;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
index 3562f173acbc8..e936f0d845c63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
@@ -33,8 +33,8 @@ public class ConfigurationProviderFactory {
   /**
    * Creates an instance of {@link ConfigurationProvider} using given
    * configuration.
-   * @param bootstrapConf
-   * @return configurationProvider
+   * @param bootstrapConf bootstrap configuration.
+   * @return configurationProvider configuration provider.
    */
   @SuppressWarnings("unchecked")
   public static ConfigurationProvider
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index d256252601e0d..c85e211bd31c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -64,8 +64,12 @@ public static boolean isFederationEnabled(Configuration conf) {
    *         configuration; else false.
    */
   public static boolean isFederationFailoverEnabled(Configuration conf) {
-    return conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
-        YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+    // Federation failover is not enabled unless federation is enabled. This previously caused
+    // YARN RMProxy to use the HA Retry policy in a non-HA & non-federation environments because
+    // the default federation failover enabled value is true.
+    return isFederationEnabled(conf) &&
+        conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+            YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
   }
 
   /**
@@ -98,7 +102,7 @@ public static boolean isAutomaticFailoverEmbedded(Configuration conf) {
   /**
    * Verify configuration for Resource Manager HA.
    * @param conf Configuration
-   * @throws YarnRuntimeException
+   * @throws YarnRuntimeException thrown by a remote service.
    */
   public static void verifyAndSetConfiguration(Configuration conf)
     throws YarnRuntimeException {
@@ -316,7 +320,10 @@ public static String getConfValueForRMInstance(
   }
 
   /**
-   * Add non empty and non null suffix to a key.
+   * Add non-empty and non-null suffix to a key.
+   *
+   * @param key key.
+   * @param suffix suffix.
    * @return the suffixed key
    */
   public static String addSuffix(String key, String suffix) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6cc4400017062..25513fec90ce9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -284,6 +284,10 @@ private static void addDeprecatedKeys() {
   public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME = 
       false;
 
+  /** Configure default application placement allocator. */
+  public static final String APPLICATION_PLACEMENT_TYPE_CLASS =
+      YARN_PREFIX + "scheduler.app-placement-allocator.class";
+
   /** Configured scheduler queue placement rules. */
   public static final String QUEUE_PLACEMENT_RULES = YARN_PREFIX
       + "scheduler.queue-placement-rules";
@@ -762,6 +766,20 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final int
       DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
 
+  public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE =
+      RM_PREFIX + "system-metrics-publisher.timeline-server-v1.batch-size";
+  public static final int
+      DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE =
+      1000;
+  public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL =
+      RM_PREFIX + "system-metrics-publisher.timeline-server-v1.interval-seconds";
+  public static final int DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL =
+      60;
+  public static final String RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED =
+      RM_PREFIX + "system-metrics-publisher.timeline-server-v1.enable-batch";
+  public static final boolean DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED =
+      false;
+
   //RM delegation token related keys
   public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
     RM_PREFIX + "delegation.key.update-interval";
@@ -775,6 +793,10 @@ public static boolean isAclEnabled(Configuration conf) {
      RM_PREFIX + "delegation.token.max-lifetime";
   public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
     7*24*60*60*1000; // 7 days
+  public static final String RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY =
+      RM_PREFIX + "delegation.token.remove-scan-interval";
+  public static final long RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT =
+      60*60*1000; // 1 hour
 
   public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE =
       RM_PREFIX + "delegation-token.max-conf-size-bytes";
@@ -1241,7 +1263,21 @@ public static boolean isAclEnabled(Configuration conf) {
   /** Prefix for all node manager configs.*/
   public static final String NM_PREFIX = "yarn.nodemanager.";
 
-  /** Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM. */
+  /**
+   * At the NM, the policy to determine whether to queue an
+   * <code>OPPORTUNISTIC</code> container or not.
+   * If set to <code>BY_QUEUE_LEN</code>, uses the queue capacity, as set by
+   * {@link YarnConfiguration#NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH},
+   * to limit how many containers to accept/queue.
+   * If set to <code>BY_RESOURCES</code>, limits the number of containers
+   * accepted based on the resource capacity of the node.
+   */
+  public static final String NM_OPPORTUNISTIC_CONTAINERS_QUEUE_POLICY =
+      NM_PREFIX + "opportunistic-containers-queue-policy";
+
+  /** Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM.
+   *  If set to 0, NM does not accept any <code>OPPORTUNISTIC</code> containers.
+   *  If set to {@literal > 0}, enforces the queue capacity. */
   public static final String NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
       NM_PREFIX + "opportunistic-containers-max-queue-length";
   public static final int DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
@@ -1516,6 +1552,13 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS
       = 10 * 60 * 1000;
 
+  /**
+   * Whether to clean up nodemanager logs when log aggregation is enabled.
+   */
+  public static final String LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP =
+      YARN_PREFIX + "log-aggregation.enable-local-cleanup";
+  public static final boolean DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = true;
+
   /**
    * Number of seconds to retain logs on the NodeManager. Only applicable if Log
    * aggregation is disabled
@@ -2951,6 +2994,51 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final int
       DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN = 60;
 
+  /**
+   * Resource manager dispatcher has a thread pool that prints EventQueue,
+   * configure the corePoolSize of this thread pool,
+   * the meaning of corePoolSize is the number of threads to keep in the pool.
+   */
+  public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE =
+      YARN_PREFIX + "dispatcher.print-thread-pool.core-pool-size";
+
+  /**
+   * The minimum number of core threads for the
+   * Resource manager dispatcher is 1.
+   */
+  public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = 1;
+
+  /**
+   * Resource manager dispatcher has a thread pool that prints EventQueue,
+   * configure the maximumPoolSize of this thread pool,
+   * the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
+   */
+  public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE =
+      YARN_PREFIX + "dispatcher.print-thread-pool.maximum-pool-size";
+
+  /**
+   * The maximum number of core threads for the
+   * Resource manager dispatcher is 5.
+   */
+  public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5;
+
+  /**
+   * Resource manager dispatcher has a thread pool that prints EventQueue,
+   * configure the keepAliveTime of this thread pool,
+   * The meaning of keepAliveTime is as follows
+   * when the number of threads is greater than the core,
+   * this is the maximum time that excess idle threads will wait for new tasks before terminating.
+   */
+  public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
+      YARN_PREFIX + "dispatcher.print-thread-pool.keep-alive-time";
+
+  /**
+   * The keep alive time of core threads for the
+   * Resource manager dispatcher is 10s.
+   */
+  public static final long DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
+      10*1000; // 10s
+
   /**
    * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
    * entries
@@ -3189,6 +3277,12 @@ public static boolean isAclEnabled(Configuration conf) {
       TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT =
         7 * 24 * 60 * 60;
 
+  public static final String
+      TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED =
+      TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "recovery-enabled";
+  public static final boolean
+      TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED_DEFAULT = true;
+
   // how old the most recent log of an UNKNOWN app needs to be in the active
   // directory before we treat it as COMPLETED
   public static final String
@@ -3882,6 +3976,13 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
       "yarnfederation/";
 
+  public static final String FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY =
+      FEDERATION_PREFIX + "state-store.heartbeat.initial-delay";
+
+  // 30 secs
+  public static final int
+      DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY = 30;
+
   public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
       FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
 
@@ -3955,6 +4056,22 @@ public static boolean isAclEnabled(Configuration conf) {
 
   public static final int DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS = 1;
 
+  public static final String FEDERATION_STATESTORE_MAX_APPLICATIONS =
+      FEDERATION_PREFIX + "state-store.max-applications";
+
+  public static final int DEFAULT_FEDERATION_STATESTORE_MAX_APPLICATIONS = 1000;
+
+  public static final String FEDERATION_STATESTORE_CLEANUP_RETRY_COUNT =
+      FEDERATION_PREFIX + "state-store.clean-up-retry-count";
+
+  public static final int DEFAULT_FEDERATION_STATESTORE_CLEANUP_RETRY_COUNT = 1;
+
+  public static final String FEDERATION_STATESTORE_CLEANUP_RETRY_SLEEP_TIME =
+      FEDERATION_PREFIX + "state-store.clean-up-retry-sleep-time";
+
+  public static final long DEFAULT_FEDERATION_STATESTORE_CLEANUP_RETRY_SLEEP_TIME =
+      TimeUnit.SECONDS.toMillis(1);
+
   public static final String ROUTER_PREFIX = YARN_PREFIX + "router.";
 
   public static final String ROUTER_BIND_HOST = ROUTER_PREFIX + "bind-host";
@@ -4000,6 +4117,14 @@ public static boolean isAclEnabled(Configuration conf) {
       ROUTER_PREFIX + "submit.retry";
   public static final int DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY = 3;
 
+  /**
+   * GetNewApplication and SubmitApplication request retry interval time.
+   */
+  public static final String ROUTER_CLIENTRM_SUBMIT_INTERVAL_TIME =
+      ROUTER_PREFIX + "submit.interval.time";
+  public static final long DEFAULT_CLIENTRM_SUBMIT_INTERVAL_TIME =
+      TimeUnit.MILLISECONDS.toMillis(10);
+
   /**
    * The interceptor class used in FederationClientInterceptor should return
    * partial ApplicationReports.
@@ -4011,11 +4136,87 @@ public static boolean isAclEnabled(Configuration conf) {
 
   public static final String ROUTER_WEBAPP_PREFIX = ROUTER_PREFIX + "webapp.";
 
+  /**
+   * This configurable that controls the thread pool size of the threadpool of the interceptor.
+   * The corePoolSize(minimumPoolSize) and maximumPoolSize of the thread pool
+   * are controlled by this configurable.
+   * In order to control the thread pool more accurately, this parameter is deprecated.
+   *
+   * corePoolSize(minimumPoolSize) use
+   * {@link YarnConfiguration#ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE}
+   *
+   * maximumPoolSize use
+   * {@link YarnConfiguration#ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE}
+   *
+   * This configurable will be deprecated.
+   */
   public static final String ROUTER_USER_CLIENT_THREADS_SIZE =
       ROUTER_PREFIX + "interceptor.user.threadpool-size";
 
+  /**
+   * The default value is 5.
+   * which means that the corePoolSize(minimumPoolSize) and maximumPoolSize
+   * of the thread pool are both 5s.
+   *
+   * corePoolSize(minimumPoolSize) default value use
+   * {@link YarnConfiguration#DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE}
+   *
+   * maximumPoolSize default value use
+   * {@link YarnConfiguration#DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE}
+   */
   public static final int DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE = 5;
 
+  /**
+   * This configurable is used to set the corePoolSize(minimumPoolSize)
+   * of the thread pool of the interceptor.
+   *
+   * corePoolSize the number of threads to keep in the pool, even if they are idle.
+   */
+  public static final String ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE =
+      ROUTER_PREFIX + "interceptor.user-thread-pool.minimum-pool-size";
+
+  /**
+   * This configuration is used to set the default value of corePoolSize (minimumPoolSize)
+   * of the thread pool of the interceptor.
+   *
+   * Default is 5.
+   */
+  public static final int DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE = 5;
+
+  /**
+   * This configurable is used to set the maximumPoolSize of the thread pool of the interceptor.
+   *
+   * maximumPoolSize the maximum number of threads to allow in the pool.
+   */
+  public static final String ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE =
+      ROUTER_PREFIX + "interceptor.user-thread-pool.maximum-pool-size";
+
+  /**
+   * This configuration is used to set the default value of maximumPoolSize
+   * of the thread pool of the interceptor.
+   *
+   * Default is 5.
+   */
+  public static final int DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5;
+
+  /**
+   * This configurable is used to set the keepAliveTime of the thread pool of the interceptor.
+   *
+   * keepAliveTime when the number of threads is greater than the core,
+   * this is the maximum time that excess idle threads will wait for new tasks before terminating.
+   */
+  public static final String ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME =
+      ROUTER_PREFIX + "interceptor.user-thread-pool.keep-alive-time";
+
+  /**
+   * This configurable is used to set the default time of keepAliveTime
+   * of the thread pool of the interceptor.
+   *
+   * the default value is 0s.
+   */
+  public static final long DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME =
+      TimeUnit.SECONDS.toMillis(0); // 0s
+
   /** The address of the Router web application. */
   public static final String ROUTER_WEBAPP_ADDRESS =
       ROUTER_WEBAPP_PREFIX + "address";
@@ -4069,6 +4270,30 @@ public static boolean isAclEnabled(Configuration conf) {
   public static final long DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT =
       TimeUnit.SECONDS.toMillis(30);
 
+  /** The Kerberos keytab for the yarn router.*/
+  public static final String ROUTER_KEYTAB = ROUTER_PREFIX + "keytab.file";
+
+  /** The Kerberos principal for the yarn router.*/
+  public static final String ROUTER_PRINCIPAL = ROUTER_PREFIX + "kerberos.principal";
+
+  /** The Kerberos principal hostname for the yarn router.*/
+  public static final String ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY = ROUTER_PREFIX +
+      "kerberos.principal.hostname";
+
+  /** Router enable AppsInfo Cache. **/
+  public static final String ROUTER_APPSINFO_ENABLED = ROUTER_WEBAPP_PREFIX + "appsinfo-enabled";
+  public static final boolean DEFAULT_ROUTER_APPSINFO_ENABLED = false;
+
+  /** Router AppsInfo Cache Count. **/
+  public static final String ROUTER_APPSINFO_CACHED_COUNT =
+      ROUTER_WEBAPP_PREFIX + "appsinfo-cached-count";
+  public static final int DEFAULT_ROUTER_APPSINFO_CACHED_COUNT = 100;
+
+  /** Enable cross origin (CORS) support. **/
+  public static final String ROUTER_WEBAPP_ENABLE_CORS_FILTER =
+      ROUTER_PREFIX + "webapp.cross-origin.enabled";
+  public static final boolean DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER = false;
+
   ////////////////////////////////
   // CSI Volume configs
   ////////////////////////////////
@@ -4371,6 +4596,12 @@ public static boolean areNodeLabelsEnabled(
   private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX
       + "node-labels.";
 
+  public static final String AM_DEFAULT_NODE_LABEL =
+      RM_NODE_LABELS_PREFIX + "am.default-node-label-expression";
+
+  public static final String AM_ALLOW_NON_EXCLUSIVE_ALLOCATION =
+      RM_NODE_LABELS_PREFIX + "am.allow-non-exclusive-allocation";
+
   public static final String RM_NODE_LABELS_PROVIDER_CONFIG =
       RM_NODE_LABELS_PREFIX + "provider";
 
@@ -4385,6 +4616,12 @@ public static boolean areNodeLabelsEnabled(
   public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS =
       30 * 60 * 1000;
 
+  public static final String RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
+      RM_NODE_LABELS_PROVIDER_PREFIX + "update-newly-registered-nodes-interval-ms";
+
+  public static final long DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS =
+      30 * 1000;
+
   @Private
   /**
    * This is a private feature that isn't supposed to be used by end-users.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/package-info.java
index ca0aa31426b54..8acd6a2a93f90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.conf;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java
index f2ec38b022826..bbbd2a8bc7c00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationAttemptNotFoundException.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 
 /**
  * This exception is thrown on
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java
index da83c397e8f36..939280862ec50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ApplicationNotFoundException.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 
 /**
  * This exception is thrown on
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java
index d5ef27b4e9b76..03c2001ab8e32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ContainerNotFoundException.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 
 /**
  * This exception is thrown on
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/package-info.java
index 89b73f0578d65..5ed6c2c9a7e3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,6 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.exceptions;
-import org.apache.hadoop.classification.InterfaceAudience;
\ No newline at end of file
+import org.apache.hadoop.classification.InterfaceAudience.Public;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/package-info.java
index 66bf20f3c8a15..90720e91d0647 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factories/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,6 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.factories;
-import org.apache.hadoop.classification.InterfaceAudience;
\ No newline at end of file
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/package-info.java
index 4f397f3987a23..f25bd41bac285 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/factory/providers/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.factory.providers;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
index 4777cf8b62a6b..6bd1a39eea414 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
@@ -106,8 +106,8 @@ public RefreshServiceAclsResponse refreshServiceAcls(
    * 
    * @param request request to update resource for a node in cluster.
    * @return (empty) response on accepting update.
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Private
   @Idempotent
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/SCMAdminProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/SCMAdminProtocol.java
index 5c791fab4e34d..5e37259e06729 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/SCMAdminProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/SCMAdminProtocol.java
@@ -42,8 +42,8 @@ public interface SCMAdminProtocol {
    * @param request request <code>SharedCacheManager</code> to run a cleaner task
    * @return <code>SharedCacheManager</code> returns an empty response
    *         on success and throws an exception on rejecting the request
-   * @throws YarnException
-   * @throws IOException
+   * @throws YarnException exceptions from yarn servers.
+   * @throws IOException io error occur.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
index 732d98ebe447d..fcbef039f2703 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
@@ -54,16 +54,16 @@ public static RefreshNodesRequest newInstance(
   }
 
   /**
-   * Set the DecommissionType
+   * Set the DecommissionType.
    * 
-   * @param decommissionType
+   * @param decommissionType decommission type.
    */
   public abstract void setDecommissionType(DecommissionType decommissionType);
 
   /**
-   * Get the DecommissionType
+   * Get the DecommissionType.
    * 
-   * @return decommissionType
+   * @return decommissionType decommission type.
    */
   public abstract DecommissionType getDecommissionType();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/package-info.java
index a1a78c802a204..56aaa15816fc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.server.api.protocolrecords;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
index 85352b3f5d10c..d0bf888ea36b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/constraint/PlacementConstraintParser.java
@@ -165,7 +165,8 @@ public interface ConstraintTokenizer extends Enumeration<String> {
 
     /**
      * Validate the schema before actual parsing the expression.
-     * @throws PlacementConstraintParseException
+     * @throws PlacementConstraintParseException when the placement constraint parser
+     * fails to parse an expression.
      */
     default void validate() throws PlacementConstraintParseException {
       // do nothing
@@ -633,9 +634,10 @@ public int getNumOfAllocations() {
 
     /**
      * Parses source tags from expression "sourceTags(numOfAllocations)".
-     * @param expr
+     * @param expr expression string.
      * @return source tags, see {@link SourceTags}
-     * @throws PlacementConstraintParseException
+     * @throws PlacementConstraintParseException when the placement constraint parser
+     * fails to parse an expression.
      */
     public static SourceTags parseFrom(String expr)
         throws PlacementConstraintParseException {
@@ -718,7 +720,8 @@ public static AbstractConstraint parseExpression(String constraintStr)
    * </ul>
    * @param expression expression string.
    * @return a map of source tags to placement constraint mapping.
-   * @throws PlacementConstraintParseException
+   * @throws PlacementConstraintParseException when the placement constraint parser
+   * fails to parse an expression.
    */
   public static Map<SourceTags, PlacementConstraint> parsePlacementSpec(
       String expression) throws PlacementConstraintParseException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/csi/CsiConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/csi/CsiConfigUtils.java
index 428fedbc1f502..6987741be51bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/csi/CsiConfigUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/csi/CsiConfigUtils.java
@@ -53,10 +53,10 @@ public static String getCsiDriverEndpoint(String driverName,
    * Resolve the CSI adaptor address for a CSI driver from configuration.
    * Expected configuration property name is
    * yarn.nodemanager.csi-driver-adaptor.${driverName}.address.
-   * @param driverName
-   * @param conf
+   * @param driverName driver name.
+   * @param conf configuration.
    * @return adaptor service address
-   * @throws YarnException
+   * @throws YarnException exceptions from yarn servers.
    */
   public static InetSocketAddress getCsiAdaptorAddressForDriver(
       String driverName, Configuration conf) throws YarnException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index d4d14d3b3ddda..1433c24e9e10c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -115,8 +115,7 @@ private static void checkSpecialResources(
      * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
      * names, in addition to 'MEMORY' for historical reasons
      */
-    String[] keys = { "memory", ResourceInformation.MEMORY_URI,
-        ResourceInformation.VCORES_URI };
+    String[] keys = {"memory", ResourceInformation.MEMORY_URI, ResourceInformation.VCORES_URI};
     for(String key : keys) {
       if (resourceInformationMap.containsKey(key)) {
         LOG.warn("Attempt to define resource '" + key + "', but it is not allowed.");
@@ -234,7 +233,8 @@ static void validateNameOfResourceNameAndThrowException(String resourceName)
   }
 
   /**
-   * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA*
+   * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA.
+   *
    * @param conf config
    * @return maximum allocation
    */
@@ -379,7 +379,7 @@ public static Map<String, Integer> getResourceTypeIndex() {
 
   /**
    * Get the resource types to be supported by the system.
-   * @return A map of the resource name to a ResouceInformation object
+   * @return A map of the resource name to a ResourceInformation object
    *         which contains details such as the unit.
    */
   public static Map<String, ResourceInformation> getResourceTypes() {
@@ -473,10 +473,10 @@ private static void addResourcesFileToConf(String resourceFile,
       LOG.debug("Found {}, adding to configuration", resourceFile);
       conf.addResource(ris);
     } catch (FileNotFoundException fe) {
-      LOG.info("Unable to find '" + resourceFile + "'.");
+      LOG.info("Unable to find '{}'.", resourceFile);
     } catch (IOException | YarnException ex) {
-      LOG.error("Exception trying to read resource types configuration '"
-          + resourceFile + "'.", ex);
+      LOG.error("Exception trying to read resource types configuration '{}'.",
+          resourceFile, ex);
       throw new YarnRuntimeException(ex);
     }
   }
@@ -668,7 +668,7 @@ public static List<ResourceTypeInfo> getResourcesTypeInfo() {
   /**
    * Reinitialize all resource types from external source (in case of client,
    * server will send the updated list and local resourceutils cache will be
-   * updated as per server's list of resources)
+   * updated as per server's list of resources).
    *
    * @param resourceTypeInfo
    *          List of resource types
@@ -857,6 +857,7 @@ private static Map<String, Long> parseResourcesString(String resourcesStr) {
         units = "Gi";
       } else if (units.isEmpty()) {
         // do nothing;
+        LOG.debug("units is empty.");
       } else {
         throw new IllegalArgumentException("Acceptable units are M/G or empty");
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
index 9ea2bc630dd29..96c83c8a48362 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationmaster_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationmaster_protocol.proto
index a48f66f4d9813..9c3cea55a5cf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationmaster_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationmaster_protocol.proto
@@ -19,7 +19,7 @@
 syntax = "proto2";
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
index 2570278cbb45d..ab7de8d2b067d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
index d06fd61ba6f45..2ec08aa1b268e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index b28bf74d0a9d4..3f9913b9896d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are private and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 95827ce56dd01..300bee24cbef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
@@ -572,6 +572,8 @@ message YarnClusterMetricsProto {
   optional int32 num_lost_nms = 4;
   optional int32 num_unhealthy_nms = 5;
   optional int32 num_rebooted_nms = 6;
+  optional int32 num_decommissioning_nms = 7;
+  optional int32 num_shutdown_nms = 8;
 }
 
 enum QueueStateProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 8a0273d7a792e..a86cbbae1169c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
@@ -106,6 +106,11 @@ message UpdatedContainerProto {
   required ContainerProto container = 2;
 }
 
+message EnhancedHeadroomProto {
+  optional int32 total_pending_count = 1;
+  optional int32 total_active_cores = 2;
+}
+
 message AllocateResponseProto {
   optional AMCommandProto a_m_command = 1;
   optional int32 response_id = 2;
@@ -123,6 +128,7 @@ message AllocateResponseProto {
   repeated UpdatedContainerProto updated_containers = 16;
   repeated ContainerProto containers_from_previous_attempts = 17;
   repeated RejectedSchedulingRequestProto rejected_scheduling_requests = 18;
+  optional EnhancedHeadroomProto enhanced_headroom = 19;
 }
 
 enum SchedulerResourceTypes {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
index e790cc8cefc70..3acd9ce0ea888 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml
@@ -39,64 +39,93 @@
         <javascript.test>*Spec</javascript.test>
     </properties>
 
-    <dependencies>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>io.swagger</groupId>
-          <artifactId>swagger-annotations</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-all</artifactId>
-            <version>1.9.5</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-module-junit4</artifactId>
-            <version>${powermock.version}</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-api-mockito</artifactId>
-            <version>${powermock.version}</version>
+  <dependencies>
+    <dependency>
+      <groupId>io.swagger</groupId>
+      <artifactId>swagger-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.9.5</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-junit4</artifactId>
+      <version>${powermock.version}</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>junit</groupId>
+          <artifactId>junit</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-api-mockito</artifactId>
+      <version>${powermock.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-all</artifactId>
+        </exclusion>
+      </exclusions>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.glassfish.jersey.media</groupId>
+      <artifactId>jersey-media-json-jackson</artifactId>
+      <version>2.12</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.javassist</groupId>
+          <artifactId>javassist</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-base</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+      <dependency>
+          <groupId>org.junit.jupiter</groupId>
+          <artifactId>junit-jupiter-api</artifactId>
+          <scope>test</scope>
+      </dependency>
+      <dependency>
+          <groupId>org.junit.jupiter</groupId>
+          <artifactId>junit-jupiter-engine</artifactId>
+          <scope>test</scope>
+      </dependency>
+      <dependency>
+          <groupId>org.junit.platform</groupId>
+          <artifactId>junit-platform-launcher</artifactId>
+          <scope>test</scope>
+      </dependency>
+
+      <dependency>
+            <groupId>com.github.pjfanning</groupId>
+            <artifactId>jersey-json</artifactId>
             <exclusions>
                 <exclusion>
-                    <groupId>org.mockito</groupId>
-                    <artifactId>mockito-all</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-core</artifactId>
                 </exclusion>
-            </exclusions>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.glassfish.jersey.media</groupId>
-            <artifactId>jersey-media-json-jackson</artifactId>
-            <version>2.12</version>
-            <scope>test</scope>
-            <exclusions>
                 <exclusion>
-                    <groupId>org.javassist</groupId>
-                    <artifactId>javassist</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.jaxrs</groupId>
-                    <artifactId>jackson-jaxrs-base</artifactId>
+                    <artifactId>jackson-jaxrs-json-provider</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
 
-        <dependency>
-            <groupId>com.sun.jersey</groupId>
-            <artifactId>jersey-json</artifactId>
-            <version>${jersey.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
@@ -276,8 +305,8 @@
                         </goals>
                         <phase>generate-resources</phase>
                         <configuration>
-                            <nodeVersion>v8.11.3</nodeVersion>
-                            <yarnVersion>v1.7.0</yarnVersion>
+                            <nodeVersion>${nodejs.version}</nodeVersion>
+                            <yarnVersion>${yarnpkg.version}</yarnVersion>
                         </configuration>
                     </execution>
                     <execution>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java
index 3a6c67d73700f..79838a9e4fb61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java
@@ -61,7 +61,7 @@ public YarnServiceClient() {
     try {
       asc = new ApiServiceClient(conf);
     } catch (Exception e) {
-      LOG.error("Error initialize YARN Service Client: {}", e);
+      LOG.error("Error initialize YARN Service Client.", e);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java
index eb78266695988..fceca7955ce9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/EmbeddedSolrServerFactory.java
@@ -22,7 +22,6 @@
 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
 import org.apache.solr.client.solrj.request.CoreAdminRequest;
 import org.apache.solr.core.NodeConfig;
-import org.apache.solr.core.SolrResourceLoader;
 
 import java.io.File;
 import java.io.IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
index 37a382021ea81..72e891513022a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
@@ -22,14 +22,15 @@
 import org.apache.hadoop.yarn.appcatalog.model.AppStoreEntry;
 import org.apache.hadoop.yarn.appcatalog.model.Application;
 import org.apache.solr.client.solrj.SolrClient;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.powermock.api.mockito.PowerMockito;
 import static org.powermock.api.mockito.PowerMockito.when;
 import static org.powermock.api.support.membermodification.MemberMatcher.method;
 
-import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.List;
 
@@ -42,7 +43,7 @@ public class TestAppCatalogSolrClient {
   private static SolrClient solrClient;
   private static AppCatalogSolrClient spy;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     String targetLocation = EmbeddedSolrServerFactory.class
         .getProtectionDomain().getCodeSource().getLocation().getFile() + "/..";
@@ -55,7 +56,7 @@ public void setup() throws Exception {
         .withNoArguments().thenReturn(solrClient);
   }
 
-  @After
+  @AfterEach
   public void teardown() throws Exception {
     try {
       solrClient.close();
@@ -64,7 +65,7 @@ public void teardown() throws Exception {
   }
 
   @Test
-  public void testRegister() throws Exception {
+  void testRegister() throws Exception {
     Application example = new Application();
     example.setOrganization("jenkins-ci.org");
     example.setName("jenkins");
@@ -76,7 +77,7 @@ public void testRegister() throws Exception {
   }
 
   @Test
-  public void testSearch() throws Exception {
+  void testSearch() throws Exception {
     Application example = new Application();
     example.setOrganization("jenkins-ci.org");
     example.setName("jenkins");
@@ -90,7 +91,7 @@ public void testSearch() throws Exception {
   }
 
   @Test
-  public void testNotFoundSearch() throws Exception {
+  void testNotFoundSearch() throws Exception {
     Application example = new Application();
     example.setOrganization("jenkins-ci.org");
     example.setName("jenkins");
@@ -104,7 +105,7 @@ public void testNotFoundSearch() throws Exception {
   }
 
   @Test
-  public void testGetRecommendedApps() throws Exception {
+  void testGetRecommendedApps() throws Exception {
     AppStoreEntry example = new AppStoreEntry();
     example.setOrg("jenkins-ci.org");
     example.setName("jenkins");
@@ -121,15 +122,15 @@ public void testGetRecommendedApps() throws Exception {
     spy.register(example2);
     List<AppStoreEntry> actual = spy.getRecommendedApps();
     long previous = 1000L;
-    for (AppStoreEntry app: actual) {
-      assertTrue("Recommend app is not sort by download count.",
-          previous > app.getDownload());
+    for (AppStoreEntry app : actual) {
+      assertTrue(previous > app.getDownload(),
+          "Recommend app is not sort by download count.");
       previous = app.getDownload();
     }
   }
 
   @Test
-  public void testUpgradeApp() throws Exception {
+  void testUpgradeApp() throws Exception {
     Application example = new Application();
     String expected = "2.0";
     String actual = "";
@@ -143,7 +144,7 @@ public void testUpgradeApp() throws Exception {
     example.setVersion("2.0");
     spy.upgradeApp(example);
     List<AppEntry> appEntries = spy.listAppEntries();
-    actual = appEntries.get(appEntries.size() -1).getYarnfile().getVersion();
+    actual = appEntries.get(appEntries.size() - 1).getYarnfile().getVersion();
     assertEquals(expected, actual);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsControllerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsControllerTest.java
index 437d50b737b18..1ceab0a69a9aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsControllerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppDetailsControllerTest.java
@@ -22,8 +22,8 @@
 import org.apache.hadoop.yarn.appcatalog.model.AppEntry;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Container;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
 import javax.ws.rs.Path;
@@ -31,8 +31,9 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.Mockito.when;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -44,14 +45,14 @@ public class AppDetailsControllerTest {
 
   private AppDetailsController controller;
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     this.controller = new AppDetailsController();
 
   }
 
   @Test
-  public void testGetDetails() throws Exception {
+  void testGetDetails() throws Exception {
     String id = "application 1";
     AppDetailsController ac = Mockito.mock(AppDetailsController.class);
 
@@ -63,7 +64,7 @@ public void testGetDetails() throws Exception {
   }
 
   @Test
-  public void testGetStatus() throws Exception {
+  void testGetStatus() throws Exception {
     String id = "application 1";
     AppDetailsController ac = Mockito.mock(AppDetailsController.class);
 
@@ -84,7 +85,7 @@ public void testGetStatus() throws Exception {
   }
 
   @Test
-  public void testStopApp() throws Exception {
+  void testStopApp() throws Exception {
     String id = "application 1";
     AppDetailsController ac = Mockito.mock(AppDetailsController.class);
 
@@ -103,7 +104,7 @@ public void testStopApp() throws Exception {
   }
 
   @Test
-  public void testRestartApp() throws Exception {
+  void testRestartApp() throws Exception {
     String id = "application 1";
     AppDetailsController ac = Mockito.mock(AppDetailsController.class);
 
@@ -122,12 +123,12 @@ public void testRestartApp() throws Exception {
   }
 
   @Test
-  public void testPathAnnotation() throws Exception {
+  void testPathAnnotation() throws Exception {
     assertNotNull(this.controller.getClass()
         .getAnnotations());
     assertThat("The controller has the annotation Path",
         this.controller.getClass()
-        .isAnnotationPresent(Path.class));
+            .isAnnotationPresent(Path.class));
 
     final Path path = this.controller.getClass()
         .getAnnotation(Path.class);
@@ -136,7 +137,7 @@ public void testPathAnnotation() throws Exception {
   }
 
   @Test
-  public void testUpgradeApp() throws Exception {
+  void testUpgradeApp() throws Exception {
     String id = "application1";
     AppDetailsController ac = Mockito.mock(AppDetailsController.class);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppListControllerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppListControllerTest.java
index 97f288e2476e0..d788de618a656 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppListControllerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppListControllerTest.java
@@ -20,8 +20,8 @@
 
 import org.apache.hadoop.yarn.appcatalog.model.AppEntry;
 import org.apache.hadoop.yarn.service.api.records.Service;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
 import javax.ws.rs.Path;
@@ -29,8 +29,9 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.Mockito.when;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -42,14 +43,14 @@ public class AppListControllerTest {
 
   private AppListController controller;
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     this.controller = new AppListController();
 
   }
 
   @Test
-  public void testGetList() throws Exception {
+  void testGetList() throws Exception {
     AppListController ac = Mockito.mock(AppListController.class);
 
     List<AppEntry> actual = new ArrayList<AppEntry>();
@@ -59,7 +60,7 @@ public void testGetList() throws Exception {
   }
 
   @Test
-  public void testDelete() throws Exception {
+  void testDelete() throws Exception {
     String id = "application 1";
     AppListController ac = Mockito.mock(AppListController.class);
 
@@ -70,7 +71,7 @@ public void testDelete() throws Exception {
   }
 
   @Test
-  public void testDeploy() throws Exception {
+  void testDeploy() throws Exception {
     String id = "application 1";
     AppListController ac = Mockito.mock(AppListController.class);
     Service service = new Service();
@@ -81,7 +82,7 @@ public void testDeploy() throws Exception {
   }
 
   @Test
-  public void testPathAnnotation() throws Exception {
+  void testPathAnnotation() throws Exception {
     assertNotNull(this.controller.getClass()
         .getAnnotations());
     assertThat("The controller has the annotation Path",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppStoreControllerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppStoreControllerTest.java
index d09952b1395b0..df0c6802b5a24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppStoreControllerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/controller/AppStoreControllerTest.java
@@ -20,8 +20,8 @@
 
 import org.apache.hadoop.yarn.appcatalog.model.AppStoreEntry;
 import org.apache.hadoop.yarn.appcatalog.model.Application;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
 import javax.ws.rs.Path;
@@ -29,8 +29,9 @@
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.Mockito.when;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -42,14 +43,14 @@ public class AppStoreControllerTest {
 
   private AppStoreController controller;
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     this.controller = new AppStoreController();
 
   }
 
   @Test
-  public void testGetRecommended() throws Exception {
+  void testGetRecommended() throws Exception {
     AppStoreController ac = Mockito.mock(AppStoreController.class);
     List<AppStoreEntry> actual = new ArrayList<AppStoreEntry>();
     when(ac.get()).thenReturn(actual);
@@ -58,7 +59,7 @@ public void testGetRecommended() throws Exception {
   }
 
   @Test
-  public void testSearch() throws Exception {
+  void testSearch() throws Exception {
     String keyword = "jenkins";
     AppStoreController ac = Mockito.mock(AppStoreController.class);
     List<AppStoreEntry> expected = new ArrayList<AppStoreEntry>();
@@ -68,7 +69,7 @@ public void testSearch() throws Exception {
   }
 
   @Test
-  public void testRegister() throws Exception {
+  void testRegister() throws Exception {
     AppStoreController ac = Mockito.mock(AppStoreController.class);
     Application app = new Application();
     app.setName("jenkins");
@@ -82,12 +83,12 @@ public void testRegister() throws Exception {
   }
 
   @Test
-  public void testPathAnnotation() throws Exception {
+  void testPathAnnotation() throws Exception {
     assertNotNull(this.controller.getClass()
         .getAnnotations());
     assertThat("The controller has the annotation Path",
         this.controller.getClass()
-        .isAnnotationPresent(Path.class));
+            .isAnnotationPresent(Path.class));
 
     final Path path = this.controller.getClass()
         .getAnnotation(Path.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 318831126b189..d1cd362c7d65d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -142,6 +142,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -153,6 +159,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
       <type>test-jar</type>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 4617184525aa6..098f3981cfd4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -1373,7 +1373,7 @@ static Map<String, Long> parseResourcesString(String resourcesStr) {
       resourcesStr = resourcesStr.substring(1);
     }
     if (resourcesStr.endsWith("]")) {
-      resourcesStr = resourcesStr.substring(0, resourcesStr.length());
+      resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1);
     }
 
     for (String resource : resourcesStr.trim().split(",")) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java
index cf5bc839da1a6..89eea4b67a95d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/DistributedShellTimelinePlugin.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
 import org.apache.hadoop.yarn.server.timeline.NameValuePair;
 import org.apache.hadoop.yarn.server.timeline.TimelineEntityGroupPlugin;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import java.util.Collection;
 import java.util.HashSet;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java
similarity index 59%
rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java
index e84bee06e6e08..074f2e8422bc5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpRequestLogAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java
@@ -6,32 +6,37 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.http;
+
+package org.apache.hadoop.yarn.applications.distributedshell;
 
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.Map;
 
-public class TestHttpRequestLogAppender {
+import static org.junit.Assert.assertEquals;
 
+public class TestClient {
   @Test
-  public void testParameterPropagation() {
+  public void testParseResourcesString() {
+    // Setup
+    final Map<String, Long> expectedResult = new HashMap<>();
+    expectedResult.put("memory-mb", 3072L);
+    expectedResult.put("vcores", 1L);
+
+    // Run the test
+    final Map<String, Long> result = Client.parseResourcesString("[memory-mb=3072,vcores=1]");
 
-    HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
-    requestLogAppender.setFilename("jetty-namenode-yyyy_mm_dd.log");
-    requestLogAppender.setRetainDays(17);
-    assertEquals("Filename mismatch", "jetty-namenode-yyyy_mm_dd.log",
-        requestLogAppender.getFilename());
-    assertEquals("Retain days mismatch", 17,
-        requestLogAppender.getRetainDays());
+    // Verify the results
+    assertEquals(expectedResult, result);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
index 2809c75ffd84f..770fceaaa3678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/pom.xml
@@ -31,23 +31,33 @@
   </properties>
 
   <dependencies>
+
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
     </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
     </dependency>
-
-      <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-          <type>test-jar</type>
-          <scope>test</scope>
-      </dependency>
 
     <dependency>
       <groupId>com.google.inject</groupId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java
index e189bcb8f435c..edfed8db1d7b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-mawo/hadoop-yarn-applications-mawo-core/src/test/java/org/apache/hadoop/applications/mawo/server/common/TestMaWoConfiguration.java
@@ -19,8 +19,10 @@
 package org.apache.hadoop.applications.mawo.server.common;
 
 
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test MaWo configuration.
@@ -31,29 +33,29 @@ public class TestMaWoConfiguration {
    * Validate default MaWo Configurations.
    */
   @Test
-  public void testMaWoConfiguration() {
+  void testMaWoConfiguration() {
 
     MawoConfiguration mawoConf = new MawoConfiguration();
 
     // validate Rpc server port
-    Assert.assertEquals(mawoConf.getRpcServerPort(), 5120);
+    assertEquals(5120, mawoConf.getRpcServerPort());
 
     // validate Rpc hostname
-    Assert.assertTrue("localhost".equals(mawoConf.getRpcHostName()));
+    assertEquals("localhost", mawoConf.getRpcHostName());
 
     // validate job queue storage conf
     boolean jobQueueStorage = mawoConf.getJobQueueStorageEnabled();
-    Assert.assertTrue(jobQueueStorage);
+    assertTrue(jobQueueStorage);
 
     // validate default teardownWorkerValidity Interval
-    Assert.assertEquals(mawoConf.getTeardownWorkerValidityInterval(), 120000);
+    assertEquals(120000, mawoConf.getTeardownWorkerValidityInterval());
 
     // validate Zk related configs
-    Assert.assertTrue("/tmp/mawoRoot".equals(mawoConf.getZKParentPath()));
-    Assert.assertTrue("localhost:2181".equals(mawoConf.getZKAddress()));
-    Assert.assertEquals(1000, mawoConf.getZKRetryIntervalMS());
-    Assert.assertEquals(10000, mawoConf.getZKSessionTimeoutMS());
-    Assert.assertEquals(1000, mawoConf.getZKRetriesNum());
+    assertEquals("/tmp/mawoRoot", mawoConf.getZKParentPath());
+    assertEquals("localhost:2181", mawoConf.getZKAddress());
+    assertEquals(1000, mawoConf.getZKRetryIntervalMS());
+    assertEquals(10000, mawoConf.getZKSessionTimeoutMS());
+    assertEquals(1000, mawoConf.getZKRetriesNum());
   }
 
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index 405e44188dc77..67be3758a5b55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -72,6 +72,21 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
index 134cba80ddc29..ed903f7f9b8d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
@@ -18,8 +18,10 @@
 
 package org.apache.hadoop.yarn.applications.unmanagedamlauncher;
 
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -41,10 +43,10 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,7 +57,7 @@ public class TestUnmanagedAMLauncher {
   protected static MiniYARNCluster yarnCluster = null;
   protected static Configuration conf = new YarnConfiguration();
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws InterruptedException, IOException {
     LOG.info("Starting up YARN cluster");
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
@@ -71,9 +73,9 @@ public static void setup() throws InterruptedException, IOException {
       LOG.info("MiniYARN ResourceManager published web address: " +
                yarnClusterConfig.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
       String webapp = yarnClusterConfig.get(YarnConfiguration.RM_WEBAPP_ADDRESS);
-      assertTrue("Web app address still unbound to a host at " + webapp,
-        !webapp.startsWith("0.0.0.0"));
-      LOG.info("Yarn webapp is at "+ webapp);
+      assertFalse(webapp.startsWith("0.0.0.0"),
+          "Web app address still unbound to a host at " + webapp);
+      LOG.info("Yarn webapp is at " + webapp);
       URL url = Thread.currentThread().getContextClassLoader()
           .getResource("yarn-site.xml");
       if (url == null) {
@@ -97,7 +99,7 @@ public static void setup() throws InterruptedException, IOException {
     }
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws IOException {
     if (yarnCluster != null) {
       try {
@@ -123,8 +125,9 @@ private static String getTestRuntimeClasspath() {
     return envClassPath;
   }
 
-  @Test(timeout=30000)
-  public void testUMALauncher() throws Exception {
+  @Test
+  @Timeout(30000)
+  void testUMALauncher() throws Exception {
     String classpath = getTestRuntimeClasspath();
     String javaHome = System.getenv("JAVA_HOME");
     if (javaHome == null) {
@@ -140,7 +143,7 @@ public void testUMALauncher() throws Exception {
         javaHome
             + "/bin/java -Xmx512m "
             + TestUnmanagedAMLauncher.class.getCanonicalName()
-            + " success" };
+            + " success"};
 
     LOG.info("Initializing Launcher");
     UnmanagedAMLauncher launcher =
@@ -149,24 +152,24 @@ public void launchAM(ApplicationAttemptId attemptId)
               throws IOException, YarnException {
             YarnApplicationAttemptState attemptState =
                 rmClient.getApplicationAttemptReport(attemptId)
-                  .getYarnApplicationAttemptState();
-            Assert.assertTrue(attemptState
-              .equals(YarnApplicationAttemptState.LAUNCHED));
+                    .getYarnApplicationAttemptState();
+            assertEquals(YarnApplicationAttemptState.LAUNCHED, attemptState);
             super.launchAM(attemptId);
           }
         };
     boolean initSuccess = launcher.init(args);
-    Assert.assertTrue(initSuccess);
+    assertTrue(initSuccess);
     LOG.info("Running Launcher");
     boolean result = launcher.run();
 
     LOG.info("Launcher run completed. Result=" + result);
-    Assert.assertTrue(result);
+    assertTrue(result);
 
   }
 
-  @Test(timeout=30000)
-  public void testUMALauncherError() throws Exception {
+  @Test
+  @Timeout(30000)
+  void testUMALauncherError() throws Exception {
     String classpath = getTestRuntimeClasspath();
     String javaHome = System.getenv("JAVA_HOME");
     if (javaHome == null) {
@@ -182,13 +185,13 @@ public void testUMALauncherError() throws Exception {
         javaHome
             + "/bin/java -Xmx512m "
             + TestUnmanagedAMLauncher.class.getCanonicalName()
-            + " failure" };
+            + " failure"};
 
     LOG.info("Initializing Launcher");
     UnmanagedAMLauncher launcher = new UnmanagedAMLauncher(new Configuration(
         yarnCluster.getConfig()));
     boolean initSuccess = launcher.init(args);
-    Assert.assertTrue(initSuccess);
+    assertTrue(initSuccess);
     LOG.info("Running Launcher");
 
     try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
index 2da2cdd42a82a..dbe0c69d5508b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/pom.xml
@@ -197,6 +197,16 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minicluster</artifactId>
@@ -211,7 +221,17 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-minikdc</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>junit</groupId>
+          <artifactId>junit</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
     </dependency>
-
   </dependencies>
 </project>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 0ccc149098c15..c0d5272a124e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -30,7 +30,7 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.UriBuilder;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
index 7d895d1cdd6c0..db2cffc93455c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.yarn.service;
 
-import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -47,9 +47,9 @@
 import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
 import org.apache.hadoop.yarn.service.conf.RestApiConstants;
 import org.apache.hadoop.yarn.service.webapp.ApiServer;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
 /**
@@ -61,7 +61,7 @@ public class TestApiServer {
   private HttpServletRequest request;
   private ServiceClientTest mockServerClient;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     request = Mockito.mock(HttpServletRequest.class);
     Mockito.when(request.getRemoteUser())
@@ -74,40 +74,41 @@ public void setup() throws Exception {
     apiServer.setServiceClient(mockServerClient);
   }
 
-  @After
+  @AfterEach
   public void teardown() {
     mockServerClient.forceStop();
   }
 
   @Test
-  public void testPathAnnotation() {
+  void testPathAnnotation() {
     assertNotNull(this.apiServer.getClass().getAnnotation(Path.class));
-    assertTrue("The controller has the annotation Path",
-        this.apiServer.getClass().isAnnotationPresent(Path.class));
+    assertTrue(this.apiServer.getClass().isAnnotationPresent(Path.class),
+        "The controller has the annotation Path");
     final Path path = this.apiServer.getClass()
         .getAnnotation(Path.class);
-    assertEquals("The path has /v1 annotation", "/v1", path.value());
+    assertEquals("/v1", path.value(), "The path has /v1 annotation");
   }
 
   @Test
-  public void testGetVersion() {
+  void testGetVersion() {
     final Response actual = apiServer.getVersion();
-    assertEquals("Version number is", Response.ok().build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.ok().build().getStatus(),
+        actual.getStatus(),
+        "Version number is");
   }
 
   @Test
-  public void testBadCreateService() {
+  void testBadCreateService() {
     Service service = new Service();
     // Test for invalid argument
     final Response actual = apiServer.createService(request, service);
-    assertEquals("Create service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Create service is ");
   }
 
   @Test
-  public void testGoodCreateService() throws Exception {
+  void testGoodCreateService() throws Exception {
     String json = "{\"auths\": "
         + "{\"https://index.docker.io/v1/\": "
         + "{\"auth\": \"foobarbaz\"},"
@@ -122,13 +123,13 @@ public void testGoodCreateService() throws Exception {
     bw.close();
     Service service = ServiceClientTest.buildGoodService();
     final Response actual = apiServer.createService(request, service);
-    assertEquals("Create service is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Create service is ");
   }
 
   @Test
-  public void testInternalServerErrorDockerClientConfigMissingCreateService() {
+  void testInternalServerErrorDockerClientConfigMissingCreateService() {
     Service service = new Service();
     service.setName("jenkins");
     service.setVersion("v1");
@@ -149,97 +150,94 @@ public void testInternalServerErrorDockerClientConfigMissingCreateService() {
     components.add(c);
     service.setComponents(components);
     final Response actual = apiServer.createService(request, service);
-    assertEquals("Create service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Create service is ");
   }
 
   @Test
-  public void testBadGetService() {
+  void testBadGetService() {
     final String serviceName = "nonexistent-jenkins";
     final Response actual = apiServer.getService(request, serviceName);
-    assertEquals("Get service is ",
-        Response.status(Status.NOT_FOUND).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.NOT_FOUND).build().getStatus(),
+        actual.getStatus(),
+        "Get service is ");
     ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Response code don't match",
-        RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
-    assertEquals("Response diagnostics don't match",
-        "Service " + serviceName + " not found",
-        serviceStatus.getDiagnostics());
+    assertEquals(RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode(),
+        "Response code don't match");
+    assertEquals("Service " + serviceName + " not found", serviceStatus.getDiagnostics(),
+        "Response diagnostics don't match");
   }
 
   @Test
-  public void testBadGetService2() {
+  void testBadGetService2() {
     final Response actual = apiServer.getService(request, null);
-    assertEquals("Get service is ",
-        Response.status(Status.NOT_FOUND).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.NOT_FOUND).build().getStatus(), actual.getStatus(),
+        "Get service is ");
     ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Response code don't match",
-        RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode());
-    assertEquals("Response diagnostics don't match",
-        "Service name cannot be null.", serviceStatus.getDiagnostics());
+    assertEquals(RestApiConstants.ERROR_CODE_APP_NAME_INVALID, serviceStatus.getCode(),
+        "Response code don't match");
+    assertEquals("Service name cannot be null.", serviceStatus.getDiagnostics(),
+        "Response diagnostics don't match");
   }
 
   @Test
-  public void testGoodGetService() {
+  void testGoodGetService() {
     final Response actual = apiServer.getService(request, "jenkins");
-    assertEquals("Get service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "Get service is ");
   }
 
   @Test
-  public void testBadDeleteService() {
+  void testBadDeleteService() {
     final Response actual = apiServer.deleteService(request, "no-jenkins");
-    assertEquals("Delete service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testBadDeleteService2() {
+  void testBadDeleteService2() {
     final Response actual = apiServer.deleteService(request, null);
-    assertEquals("Delete service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testBadDeleteService3() {
+  void testBadDeleteService3() {
     final Response actual = apiServer.deleteService(request,
         "jenkins-doesn't-exist");
-    assertEquals("Delete service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testBadDeleteService4() {
+  void testBadDeleteService4() {
     final Response actual = apiServer.deleteService(request,
         "jenkins-error-cleaning-registry");
-    assertEquals("Delete service is ",
-        Response.status(Status.INTERNAL_SERVER_ERROR).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.INTERNAL_SERVER_ERROR).build().getStatus(),
+        actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testGoodDeleteService() {
+  void testGoodDeleteService() {
     final Response actual = apiServer.deleteService(request, "jenkins");
-    assertEquals("Delete service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testDeleteStoppedService() {
-    final Response actual = apiServer.deleteService(request,
-        "jenkins-already-stopped");
-    assertEquals("Delete service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+  void testDeleteStoppedService() {
+    final Response actual = apiServer.deleteService(request, "jenkins-already-stopped");
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "Delete service is ");
   }
 
   @Test
-  public void testDecreaseContainerAndStop() {
+  void testDecreaseContainerAndStop() {
     Service service = new Service();
     service.setState(ServiceState.STOPPED);
     service.setName("jenkins");
@@ -260,12 +258,12 @@ public void testDecreaseContainerAndStop() {
     service.setComponents(components);
     final Response actual = apiServer.updateService(request, "jenkins",
         service);
-    assertEquals("update service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "update service is ");
   }
 
   @Test
-  public void testBadDecreaseContainerAndStop() {
+  void testBadDecreaseContainerAndStop() {
     Service service = new Service();
     service.setState(ServiceState.STOPPED);
     service.setName("no-jenkins");
@@ -287,13 +285,13 @@ public void testBadDecreaseContainerAndStop() {
     System.out.println("before stop");
     final Response actual = apiServer.updateService(request, "no-jenkins",
         service);
-    assertEquals("flex service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "flex service is ");
   }
 
   @Test
-  public void testIncreaseContainersAndStart() {
+  void testIncreaseContainersAndStart() {
     Service service = new Service();
     service.setState(ServiceState.STARTED);
     service.setName("jenkins");
@@ -314,12 +312,12 @@ public void testIncreaseContainersAndStart() {
     service.setComponents(components);
     final Response actual = apiServer.updateService(request, "jenkins",
         service);
-    assertEquals("flex service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "flex service is ");
   }
 
   @Test
-  public void testBadStartServices() {
+  void testBadStartServices() {
     Service service = new Service();
     service.setState(ServiceState.STARTED);
     service.setName("no-jenkins");
@@ -340,13 +338,13 @@ public void testBadStartServices() {
     service.setComponents(components);
     final Response actual = apiServer.updateService(request, "no-jenkins",
         service);
-    assertEquals("start service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "start service is ");
   }
 
   @Test
-  public void testGoodStartServices() {
+  void testGoodStartServices() {
     Service service = new Service();
     service.setState(ServiceState.STARTED);
     service.setName("jenkins");
@@ -367,12 +365,12 @@ public void testGoodStartServices() {
     service.setComponents(components);
     final Response actual = apiServer.updateService(request, "jenkins",
         service);
-    assertEquals("start service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "start service is ");
   }
 
   @Test
-  public void testBadStopServices() {
+  void testBadStopServices() {
     Service service = new Service();
     service.setState(ServiceState.STOPPED);
     service.setName("no-jenkins");
@@ -394,25 +392,25 @@ public void testBadStopServices() {
     System.out.println("before stop");
     final Response actual = apiServer.updateService(request, "no-jenkins",
         service);
-    assertEquals("stop service is ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "stop service is ");
   }
 
   @Test
-  public void testGoodStopServices() {
+  void testGoodStopServices() {
     Service service = new Service();
     service.setState(ServiceState.STOPPED);
     service.setName("jenkins");
     System.out.println("before stop");
     final Response actual = apiServer.updateService(request, "jenkins",
         service);
-    assertEquals("stop service is ",
-        Response.status(Status.OK).build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.OK).build().getStatus(), actual.getStatus(),
+        "stop service is ");
   }
 
   @Test
-  public void testBadSecondStopServices() throws Exception {
+  void testBadSecondStopServices() throws Exception {
     Service service = new Service();
     service.setState(ServiceState.STOPPED);
     service.setName("jenkins-second-stop");
@@ -420,17 +418,17 @@ public void testBadSecondStopServices() throws Exception {
     System.out.println("before second stop");
     final Response actual = apiServer.updateService(request,
         "jenkins-second-stop", service);
-    assertEquals("stop service should have thrown 400 Bad Request: ",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "stop service should have thrown 400 Bad Request: ");
     ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Stop service should have failed with service already stopped",
-        "Service jenkins-second-stop is already stopped",
-        serviceStatus.getDiagnostics());
+    assertEquals("Service jenkins-second-stop is already stopped",
+        serviceStatus.getDiagnostics(),
+        "Stop service should have failed with service already stopped");
   }
 
   @Test
-  public void testUpdateService() {
+  void testUpdateService() {
     Service service = new Service();
     service.setState(ServiceState.STARTED);
     service.setName("no-jenkins");
@@ -452,72 +450,71 @@ public void testUpdateService() {
     System.out.println("before stop");
     final Response actual = apiServer.updateService(request, "no-jenkins",
         service);
-    assertEquals("update service is ",
-        Response.status(Status.BAD_REQUEST)
-            .build().getStatus(), actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST)
+        .build().getStatus(), actual.getStatus(), "update service is ");
   }
 
   @Test
-  public void testUpdateComponent() {
+  void testUpdateComponent() {
     Response actual = apiServer.updateComponent(request, "jenkins",
         "jenkins-master", null);
     ServiceStatus serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Update component should have failed with 400 bad request",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
-    assertEquals("Update component should have failed with no data error",
-        "No component data provided", serviceStatus.getDiagnostics());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Update component should have failed with 400 bad request");
+    assertEquals("No component data provided", serviceStatus.getDiagnostics(),
+        "Update component should have failed with no data error");
 
     Component comp = new Component();
     actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
         comp);
     serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Update component should have failed with 400 bad request",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
-    assertEquals("Update component should have failed with no count error",
-        "No container count provided", serviceStatus.getDiagnostics());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Update component should have failed with 400 bad request");
+    assertEquals("No container count provided", serviceStatus.getDiagnostics(),
+        "Update component should have failed with no count error");
 
     comp.setNumberOfContainers(-1L);
     actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
         comp);
     serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Update component should have failed with 400 bad request",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
-    assertEquals("Update component should have failed with no count error",
-        "Invalid number of containers specified -1", serviceStatus.getDiagnostics());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Update component should have failed with 400 bad request");
+    assertEquals("Invalid number of containers specified -1", serviceStatus.getDiagnostics(),
+        "Update component should have failed with no count error");
 
     comp.setName("jenkins-slave");
     comp.setNumberOfContainers(1L);
     actual = apiServer.updateComponent(request, "jenkins", "jenkins-master",
         comp);
     serviceStatus = (ServiceStatus) actual.getEntity();
-    assertEquals("Update component should have failed with 400 bad request",
-        Response.status(Status.BAD_REQUEST).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.BAD_REQUEST).build().getStatus(),
+        actual.getStatus(),
+        "Update component should have failed with 400 bad request");
     assertEquals(
-        "Update component should have failed with component name mismatch "
-            + "error",
         "Component name in the request object (jenkins-slave) does not match "
             + "that in the URI path (jenkins-master)",
-        serviceStatus.getDiagnostics());
+        serviceStatus.getDiagnostics(),
+        "Update component should have failed with component name mismatch "
+            + "error");
   }
 
   @Test
-  public void testInitiateUpgrade() {
+  void testInitiateUpgrade() {
     Service goodService = ServiceClientTest.buildLiveGoodService();
     goodService.setVersion("v2");
     goodService.setState(ServiceState.UPGRADING);
     final Response actual = apiServer.updateService(request,
         goodService.getName(), goodService);
-    assertEquals("Initiate upgrade is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Initiate upgrade is ");
   }
 
   @Test
-  public void testUpgradeSingleInstance() {
+  void testUpgradeSingleInstance() {
     Service goodService = ServiceClientTest.buildLiveGoodService();
     Component comp = goodService.getComponents().iterator().next();
     Container container = comp.getContainers().iterator().next();
@@ -536,13 +533,13 @@ public void testUpgradeSingleInstance() {
     final Response actual = apiServer.updateComponentInstance(request,
         goodService.getName(), comp.getName(),
         container.getComponentInstanceName(), container);
-    assertEquals("Instance upgrade is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Instance upgrade is ");
   }
 
   @Test
-  public void testUpgradeMultipleInstances() {
+  void testUpgradeMultipleInstances() {
     Service goodService = ServiceClientTest.buildLiveGoodService();
     Component comp = goodService.getComponents().iterator().next();
     comp.getContainers().forEach(container ->
@@ -563,13 +560,13 @@ public void testUpgradeMultipleInstances() {
 
     final Response actual = apiServer.updateComponentInstances(request,
         goodService.getName(), comp.getContainers());
-    assertEquals("Instance upgrade is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Instance upgrade is ");
   }
 
   @Test
-  public void testUpgradeComponent() {
+  void testUpgradeComponent() {
     Service goodService = ServiceClientTest.buildLiveGoodService();
     Component comp = goodService.getComponents().iterator().next();
     comp.setState(ComponentState.UPGRADING);
@@ -589,13 +586,13 @@ public void testUpgradeComponent() {
 
     final Response actual = apiServer.updateComponent(request,
         goodService.getName(), comp.getName(), comp);
-    assertEquals("Component upgrade is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Component upgrade is ");
   }
 
   @Test
-  public void testUpgradeMultipleComps() {
+  void testUpgradeMultipleComps() {
     Service goodService = ServiceClientTest.buildLiveGoodService();
     goodService.getComponents().forEach(comp ->
         comp.setState(ComponentState.UPGRADING));
@@ -616,8 +613,8 @@ public void testUpgradeMultipleComps() {
 
     final Response actual = apiServer.updateComponents(request,
         goodService.getName(), goodService.getComponents());
-    assertEquals("Component upgrade is ",
-        Response.status(Status.ACCEPTED).build().getStatus(),
-        actual.getStatus());
+    assertEquals(Response.status(Status.ACCEPTED).build().getStatus(),
+        actual.getStatus(),
+        "Component upgrade is ");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestCleanupAfterKill.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestCleanupAfterKill.java
index 51e834a34d9a1..c2f3c689e238b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestCleanupAfterKill.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestCleanupAfterKill.java
@@ -27,16 +27,19 @@
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import java.io.IOException;
 
 /**
@@ -47,22 +50,20 @@ public class TestCleanupAfterKill extends ServiceTestUtils {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestCleanupAfterKill.class);
 
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
-
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     File tmpYarnDir = new File("target", "tmp");
     FileUtils.deleteQuietly(tmpYarnDir);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws IOException {
     shutdown();
   }
 
-  @Test(timeout = 200000)
-  public void testRegistryCleanedOnLifetimeExceeded() throws Exception {
+  @Test
+  @Timeout(200000)
+  void testRegistryCleanedOnLifetimeExceeded() throws Exception {
     setupInternal(NUM_NMS);
     ServiceClient client = createClient(getConf());
     Service exampleApp = createExampleApplication();
@@ -71,8 +72,8 @@ public void testRegistryCleanedOnLifetimeExceeded() throws Exception {
     waitForServiceToBeStable(client, exampleApp);
     String serviceZKPath = RegistryUtils.servicePath(RegistryUtils
         .currentUser(), YarnServiceConstants.APP_TYPE, exampleApp.getName());
-    Assert.assertTrue("Registry ZK service path doesn't exist",
-        getCuratorService().zkPathExists(serviceZKPath));
+    assertTrue(getCuratorService().zkPathExists(serviceZKPath),
+        "Registry ZK service path doesn't exist");
 
     // wait for app to be killed by RM
     ApplicationId exampleAppId = ApplicationId.fromString(exampleApp.getId());
@@ -85,10 +86,10 @@ public void testRegistryCleanedOnLifetimeExceeded() throws Exception {
         throw new RuntimeException("while waiting", e);
       }
     }, 2000, 200000);
-    Assert.assertFalse("Registry ZK service path still exists after killed",
-        getCuratorService().zkPathExists(serviceZKPath));
+    assertFalse(getCuratorService().zkPathExists(serviceZKPath),
+        "Registry ZK service path still exists after killed");
 
     LOG.info("Destroy the service");
-    Assert.assertEquals(0, client.actionDestroy(exampleApp.getName()));
+    assertEquals(0, client.actionDestroy(exampleApp.getName()));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
index 1d08b82fff1c8..fe9c081ed64a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
@@ -16,7 +16,7 @@
  */
 package org.apache.hadoop.yarn.service.client;
 
-import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -35,9 +35,10 @@
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
 import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.*;
 
 /**
@@ -88,7 +89,7 @@ protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
 
   }
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     server = new Server(8088);
     ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(20);
@@ -112,13 +113,13 @@ public static void setup() throws Exception {
     badAsc.serviceInit(conf2);
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     server.stop();
   }
 
   @Test
-  public void testGetRMWebAddress() throws Exception {
+  void testGetRMWebAddress() throws Exception {
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
     conf.set(YarnConfiguration.RM_HA_IDS, "rm1");
@@ -129,17 +130,17 @@ public void testGetRMWebAddress() throws Exception {
     String diagnosticsMsg = null;
     try {
       String rmWebAddress = asc1.getRMWebAddress();
-    } catch (IOException e){
+    } catch (IOException e) {
       exceptionCaught = true;
       diagnosticsMsg = e.getMessage();
     }
-    assertTrue("ApiServiceClient failed to throw exception", exceptionCaught);
-    assertTrue("Exception Message does not match",
-        diagnosticsMsg.contains("Error connecting to localhost:0"));
+    assertTrue(exceptionCaught, "ApiServiceClient failed to throw exception");
+    assertTrue(diagnosticsMsg.contains("Error connecting to localhost:0"),
+        "Exception Message does not match");
   }
 
   @Test
-  public void testLaunch() {
+  void testLaunch() {
     String fileName = "target/test-classes/example-app.json";
     String appName = "example-app";
     long lifetime = 3600L;
@@ -153,7 +154,7 @@ public void testLaunch() {
   }
 
   @Test
-  public void testBadLaunch() {
+  void testBadLaunch() {
     String fileName = "unknown_file";
     String appName = "unknown_app";
     long lifetime = 3600L;
@@ -167,19 +168,18 @@ public void testBadLaunch() {
   }
 
   @Test
-  public void testStatus() {
+  void testStatus() {
     String appName = "nonexistent-app";
     try {
       String result = asc.getStatusString(appName);
-      assertEquals("Status reponse don't match",
-          " Service " + appName + " not found", result);
+      assertEquals(" Service " + appName + " not found", result, "Status reponse don't match");
     } catch (IOException | YarnException e) {
       fail();
     }
   }
 
   @Test
-  public void testStop() {
+  void testStop() {
     String appName = "example-app";
     try {
       int result = asc.actionStop(appName);
@@ -190,7 +190,7 @@ public void testStop() {
   }
 
   @Test
-  public void testBadStop() {
+  void testBadStop() {
     String appName = "unknown_app";
     try {
       int result = badAsc.actionStop(appName);
@@ -201,7 +201,7 @@ public void testBadStop() {
   }
 
   @Test
-  public void testStart() {
+  void testStart() {
     String appName = "example-app";
     try {
       int result = asc.actionStart(appName);
@@ -212,7 +212,7 @@ public void testStart() {
   }
 
   @Test
-  public void testBadStart() {
+  void testBadStart() {
     String appName = "unknown_app";
     try {
       int result = badAsc.actionStart(appName);
@@ -223,7 +223,7 @@ public void testBadStart() {
   }
 
   @Test
-  public void testSave() {
+  void testSave() {
     String fileName = "target/test-classes/example-app.json";
     String appName = "example-app";
     long lifetime = 3600L;
@@ -237,7 +237,7 @@ public void testSave() {
   }
 
   @Test
-  public void testBadSave() {
+  void testBadSave() {
     String fileName = "unknown_file";
     String appName = "unknown_app";
     long lifetime = 3600L;
@@ -251,7 +251,7 @@ public void testBadSave() {
   }
 
   @Test
-  public void testFlex() {
+  void testFlex() {
     String appName = "example-app";
     HashMap<String, String> componentCounts = new HashMap<String, String>();
     try {
@@ -263,7 +263,7 @@ public void testFlex() {
   }
 
   @Test
-  public void testBadFlex() {
+  void testBadFlex() {
     String appName = "unknown_app";
     HashMap<String, String> componentCounts = new HashMap<String, String>();
     try {
@@ -275,7 +275,7 @@ public void testBadFlex() {
   }
 
   @Test
-  public void testDestroy() {
+  void testDestroy() {
     String appName = "example-app";
     try {
       int result = asc.actionDestroy(appName);
@@ -286,7 +286,7 @@ public void testDestroy() {
   }
 
   @Test
-  public void testBadDestroy() {
+  void testBadDestroy() {
     String appName = "unknown_app";
     try {
       int result = badAsc.actionDestroy(appName);
@@ -297,7 +297,7 @@ public void testBadDestroy() {
   }
 
   @Test
-  public void testInitiateServiceUpgrade() {
+  void testInitiateServiceUpgrade() {
     String appName = "example-app";
     String upgradeFileName = "target/test-classes/example-app.json";
     try {
@@ -309,7 +309,7 @@ public void testInitiateServiceUpgrade() {
   }
 
   @Test
-  public void testInstancesUpgrade() {
+  void testInstancesUpgrade() {
     String appName = "example-app";
     try {
       int result = asc.actionUpgradeInstances(appName, Lists.newArrayList(
@@ -321,7 +321,7 @@ public void testInstancesUpgrade() {
   }
 
   @Test
-  public void testComponentsUpgrade() {
+  void testComponentsUpgrade() {
     String appName = "example-app";
     try {
       int result = asc.actionUpgradeComponents(appName, Lists.newArrayList(
@@ -333,12 +333,12 @@ public void testComponentsUpgrade() {
   }
 
   @Test
-  public void testNoneSecureApiClient() throws IOException {
+  void testNoneSecureApiClient() throws IOException {
     String url = asc.getServicePath("/foobar");
-    assertTrue("User.name flag is missing in service path.",
-        url.contains("user.name"));
-    assertTrue("User.name flag is not matching JVM user.",
-        url.contains(System.getProperty("user.name")));
+    assertTrue(url.contains("user.name"),
+        "User.name flag is missing in service path.");
+    assertTrue(url.contains(System.getProperty("user.name")),
+        "User.name flag is not matching JVM user.");
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
index e2d613a305902..60c06e9aa75f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.client;
 
-import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -49,9 +49,9 @@
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 /**
  * Test Spnego Client Login.
@@ -129,8 +129,9 @@ public static boolean isHeaderExist() {
     }
   }
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
+    startMiniKdc();
     keytabFile = new File(getWorkDir(), "keytab");
     getKdc().createPrincipal(keytabFile, clientPrincipal, server1Principal,
         server2Principal);
@@ -163,13 +164,14 @@ List<String> getRMHAWebAddresses(Configuration conf) {
     asc.serviceInit(testConf);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     server.stop();
+    stopMiniKdc();
   }
 
   @Test
-  public void testHttpSpnegoChallenge() throws Exception {
+  void testHttpSpnegoChallenge() throws Exception {
     UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile
         .getCanonicalPath());
     String challenge = YarnClientUtils.generateToken("localhost");
@@ -177,7 +179,7 @@ public void testHttpSpnegoChallenge() throws Exception {
   }
 
   @Test
-  public void testAuthorizationHeader() throws Exception {
+  void testAuthorizationHeader() throws Exception {
     UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile
         .getCanonicalPath());
     String rmAddress = asc.getRMWebAddress();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
index 4954b478a529d..a2f698fd31f4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
@@ -29,14 +29,16 @@
 import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+
+import static org.junit.jupiter.api.Assertions.*;
+
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -44,8 +46,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.junit.Assert.fail;
-
 /**
  * Test class for system service manager.
  */
@@ -62,7 +62,7 @@ public class TestSystemServiceManagerImpl {
   private static Map<String, Set<String>> savedServices = new HashMap<>();
   private static Map<String, Set<String>> submittedServices = new HashMap<>();
 
-  @Before
+  @BeforeEach
   public void setup() {
     File file = new File(
         getClass().getClassLoader().getResource(resourcePath).getFile());
@@ -80,30 +80,30 @@ public void setup() {
     constructUserService(users[1], "example-app1", "example-app2");
   }
 
-  @After
+  @AfterEach
   public void tearDown() {
     systemService.stop();
   }
 
   @Test
-  public void testSystemServiceSubmission() throws Exception {
+  void testSystemServiceSubmission() throws Exception {
     systemService.start();
 
     /* verify for ignored sevices count */
     Map<String, Integer> ignoredUserServices =
         systemService.getIgnoredUserServices();
-    Assert.assertEquals(1, ignoredUserServices.size());
-    Assert.assertTrue("User user1 doesn't exist.",
-        ignoredUserServices.containsKey(users[0]));
+    assertEquals(1, ignoredUserServices.size());
+    assertTrue(ignoredUserServices.containsKey(users[0]),
+        "User user1 doesn't exist.");
     int count = ignoredUserServices.get(users[0]);
-    Assert.assertEquals(1, count);
-    Assert.assertEquals(1,
+    assertEquals(1, count);
+    assertEquals(1,
         systemService.getBadFileNameExtensionSkipCounter());
-    Assert.assertEquals(1, systemService.getBadDirSkipCounter());
+    assertEquals(1, systemService.getBadDirSkipCounter());
 
     Map<String, Set<Service>> userServices =
         systemService.getSyncUserServices();
-    Assert.assertEquals(loadedServices.size(), userServices.size());
+    assertEquals(loadedServices.size(), userServices.size());
     verifyForScannedUserServices(userServices);
 
     verifyForLaunchedUserServices();
@@ -123,13 +123,12 @@ private void verifyForScannedUserServices(
     for (String user : users) {
       Set<Service> services = userServices.get(user);
       Set<String> serviceNames = loadedServices.get(user);
-      Assert.assertEquals(serviceNames.size(), services.size());
+      assertEquals(serviceNames.size(), services.size());
       Iterator<Service> iterator = services.iterator();
       while (iterator.hasNext()) {
         Service next = iterator.next();
-        Assert.assertTrue(
-            "Service name doesn't exist in expected userService "
-                + serviceNames, serviceNames.contains(next.getName()));
+        assertTrue(serviceNames.contains(next.getName()),
+            "Service name doesn't exist in expected userService " + serviceNames);
       }
     }
   }
@@ -203,19 +202,19 @@ public ApplicationId actionStartAndGetId(String serviceName)
   }
 
   private void verifyForLaunchedUserServices() {
-    Assert.assertEquals(loadedServices.size(), submittedServices.size());
+    assertEquals(loadedServices.size(), submittedServices.size());
     for (Map.Entry<String, Set<String>> entry : submittedServices.entrySet()) {
       String user = entry.getKey();
       Set<String> serviceSet = entry.getValue();
-      Assert.assertTrue(loadedServices.containsKey(user));
+      assertTrue(loadedServices.containsKey(user));
       Set<String> services = loadedServices.get(user);
-      Assert.assertEquals(services.size(), serviceSet.size());
-      Assert.assertTrue(services.containsAll(serviceSet));
+      assertEquals(services.size(), serviceSet.size());
+      assertTrue(services.containsAll(serviceSet));
     }
   }
 
   @Test
-  public void testFileSystemCloseWhenCleanUpService() throws Exception {
+  void testFileSystemCloseWhenCleanUpService() throws Exception {
     FileSystem fs = null;
     Path path = new Path("/tmp/servicedir");
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index b671c71659d9e..746abf385e6bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -180,6 +180,12 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -190,6 +196,12 @@
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-configuration2</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
index eb999cb9a5b5f..2deffb8674f7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
index 5536b41008927..d37e5f0e205a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.service;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 6d143a3bd4aa0..9da8f31fe4b8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -1139,7 +1139,7 @@ public void syncSysFs(Service yarnApp) {
         LOG.info("YARN sysfs synchronized.");
       }
     } catch (IOException | URISyntaxException | InterruptedException e) {
-      LOG.error("Fail to sync service spec: {}", e);
+      LOG.error("Fail to sync service spec.", e);
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 915ee37f07b26..4e9932a9e2810 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.service.component;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
index 9ecfd3c41dbbe..47a833a4068f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.component;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
index 932f97c4797ca..f88252618c32f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstanceEvent.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.component.instance;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.event.AbstractEvent;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index fc6e2f240a81e..7db7894923011 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.service.containerlaunch;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
index 0f9aebe354370..70e3fc23abbd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.containerlaunch;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
index ac79aa4e74f7c..3debc45f670a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.containerlaunch;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.Container;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index 69e16281cd78e..5578ea3624ee0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.service.containerlaunch;
 
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.service.utils.ServiceUtils;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
index 492a11b2c67a3..40a87937629d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -88,8 +88,9 @@ public ProbeStatus ping(ComponentInstance instance) {
     }
     String ip = instance.getContainerStatus().getIPs().get(0);
     HttpURLConnection connection = null;
+    String hostString = urlString.replace(HOST_TOKEN, ip);
     try {
-      URL url = new URL(urlString.replace(HOST_TOKEN, ip));
+      URL url = new URL(hostString);
       connection = getConnection(url, this.timeout);
       int rc = connection.getResponseCode();
       if (rc < min || rc > max) {
@@ -101,7 +102,8 @@ public ProbeStatus ping(ComponentInstance instance) {
         status.succeed(this);
       }
     } catch (Throwable e) {
-      String error = "Probe " + urlString + " failed for IP " + ip + ": " + e;
+      String error =
+          "Probe " + hostString + " failed for IP " + ip + ": " + e;
       log.info(error, e);
       status.fail(this,
           new IOException(error, e));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
index 00e072ddb5bdb..6a7464f41f60c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.provider;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.service.api.records.Service;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index db7474d3f545c..2b13cf0cae8a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.registry;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
index ba9ebc2ffaa74..25cab87d231bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ClientRegistryBinder.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.utils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
index 5d103a98855fb..c5ff20a609018 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.utils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
 import org.slf4j.Logger;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 1026092f54bb7..6978a02a09809 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.utils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataInputStream;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
index 8e8c2ea929e00..447a3e967abe0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.service.utils;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index b8ce493aea408..b96d04a98e03b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -18,9 +18,9 @@
 
 package org.apache.hadoop.yarn.service.utils;
 
-import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ArrayListMultimap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap;
 import org.apache.commons.lang3.StringUtils;
@@ -75,20 +75,20 @@ public class ServiceApiUtil {
       LoggerFactory.getLogger(ServiceApiUtil.class);
   public static JsonSerDeser<Service> jsonSerDeser =
       new JsonSerDeser<>(Service.class,
-          PropertyNamingStrategy.SNAKE_CASE);
+          PropertyNamingStrategies.SNAKE_CASE);
 
   public static final JsonSerDeser<Container[]> CONTAINER_JSON_SERDE =
       new JsonSerDeser<>(Container[].class,
-          PropertyNamingStrategy.SNAKE_CASE);
+          PropertyNamingStrategies.SNAKE_CASE);
 
   public static final JsonSerDeser<ComponentContainers[]>
       COMP_CONTAINERS_JSON_SERDE = new JsonSerDeser<>(
           ComponentContainers[].class,
-          PropertyNamingStrategy.SNAKE_CASE);
+          PropertyNamingStrategies.SNAKE_CASE);
 
   public static final JsonSerDeser<Component[]> COMP_JSON_SERDE =
       new JsonSerDeser<>(Component[].class,
-          PropertyNamingStrategy.SNAKE_CASE);
+          PropertyNamingStrategies.SNAKE_CASE);
 
   private static final PatternValidator namePattern
       = new PatternValidator("[a-z][a-z0-9-]*");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
index 2de2223cec133..2034cc3b5d5aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service.utils;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.lang3.ArrayUtils;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index cd148ced48a86..b5ce0f17ac9b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.service;
 
-import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
 import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap;
@@ -98,7 +98,7 @@ public class ServiceTestUtils {
 
   public static final JsonSerDeser<Service> JSON_SER_DESER =
       new JsonSerDeser<>(Service.class,
-          PropertyNamingStrategy.SNAKE_CASE);
+          PropertyNamingStrategies.SNAKE_CASE);
 
   // Example service definition
   // 2 components, each of which has 2 containers.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
index c9886584bf27b..69c0c2cee57f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
@@ -553,7 +553,7 @@ public void testSyncSysFS() {
       am.stop();
       am.close();
     } catch (Exception e) {
-      LOG.error("Fail to sync sysfs: {}", e);
+      LOG.error("Fail to sync sysfs.", e);
       Assert.fail("Fail to sync sysfs.");
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
index 108078ca7f010..31ca38297c856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java
@@ -42,7 +42,6 @@
     .DEFAULT_CONTAINER_RETRY_MAX;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 import static org.mockito.Mockito.when;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index b7ec64934c420..dbe4e9048b2df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -42,6 +42,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.ow2.asm</groupId>
+          <artifactId>asm-commons</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 13b39b5c99673..f0e689d8d2830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -48,7 +48,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index faacf61ed8cd8..76e3dd2b07a54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -56,7 +56,7 @@
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
index e3653baa2b785..26f45baaee019 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
@@ -137,9 +137,8 @@ public List<ContainerReport> getContainers(ApplicationAttemptId
     ApplicationId appId = applicationAttemptId.getApplicationId();
     ApplicationReport appReport = getApplicationReport(appId);
     Map<String, String> filters = new HashMap<>();
-    filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
-        applicationAttemptId.toString() +
-        "\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}");
+    filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq "
+        + "{\"type\":\"YARN_APPLICATION_ATTEMPT\",\"id\":\"" + applicationAttemptId + "\"}");
     List<TimelineEntity> entities = readerClient.getContainerEntities(
         appId, "ALL", filters, 0, null);
     List<ContainerReport> containers =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index aaa586e043d0f..274920f7e1bed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -81,7 +81,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
index 90b6500a7ad7a..505eca8dcf3b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -50,7 +50,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 81848d469dd59..465f1ad7c3b7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -81,7 +81,7 @@
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 
 import static org.apache.hadoop.yarn.client.util.YarnClientUtils.NO_LABEL_ERR_MSG;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index a9f1c542ab217..9823a1afb6881 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -37,6 +37,7 @@
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
 import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -190,7 +191,7 @@ private static void prettyFormatWithIndent(String input, int indent)
     Source xmlInput = new StreamSource(new StringReader(input));
     StringWriter sw = new StringWriter();
     StreamResult xmlOutput = new StreamResult(sw);
-    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    TransformerFactory transformerFactory = XMLUtils.newSecureTransformerFactory();
     transformerFactory.setAttribute("indent-number", indent);
     Transformer transformer = transformerFactory.newTransformer();
     transformer.setOutputProperty(OutputKeys.INDENT, "yes");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index b1ec48f0a47c1..c16fe03b82a43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -339,9 +339,11 @@ private static class NodesInformation {
     int totalNodes;
     int runningNodes;
     int unhealthyNodes;
+    int decommissioningNodes;
     int decommissionedNodes;
     int lostNodes;
     int rebootedNodes;
+    int shutdownNodes;
   }
 
   private static class QueueMetrics {
@@ -696,6 +698,8 @@ protected NodesInformation getNodesInfo() {
       return nodeInfo;
     }
 
+    nodeInfo.decommissioningNodes =
+        yarnClusterMetrics.getNumDecommissioningNodeManagers();
     nodeInfo.decommissionedNodes =
         yarnClusterMetrics.getNumDecommissionedNodeManagers();
     nodeInfo.totalNodes = yarnClusterMetrics.getNumNodeManagers();
@@ -703,6 +707,7 @@ protected NodesInformation getNodesInfo() {
     nodeInfo.lostNodes = yarnClusterMetrics.getNumLostNodeManagers();
     nodeInfo.unhealthyNodes = yarnClusterMetrics.getNumUnhealthyNodeManagers();
     nodeInfo.rebootedNodes = yarnClusterMetrics.getNumRebootedNodeManagers();
+    nodeInfo.shutdownNodes = yarnClusterMetrics.getNumShutdownNodeManagers();
     return nodeInfo;
   }
 
@@ -880,11 +885,11 @@ String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) {
     ret.append(CLEAR_LINE)
         .append(limitLineLength(String.format(
             "NodeManager(s)"
-                + ": %d total, %d active, %d unhealthy, %d decommissioned,"
-                + " %d lost, %d rebooted%n",
+                + ": %d total, %d active, %d unhealthy, %d decommissioning,"
+                + " %d decommissioned, %d lost, %d rebooted, %d shutdown%n",
             nodes.totalNodes, nodes.runningNodes, nodes.unhealthyNodes,
-            nodes.decommissionedNodes, nodes.lostNodes,
-            nodes.rebootedNodes), terminalWidth, true));
+            nodes.decommissioningNodes, nodes.decommissionedNodes, nodes.lostNodes,
+            nodes.rebootedNodes, nodes.shutdownNodes), terminalWidth, true));
 
     ret.append(CLEAR_LINE)
         .append(limitLineLength(String.format(
@@ -1039,7 +1044,8 @@ protected void showFieldsScreen() {
     }
   }
 
-  protected void showTopScreen() {
+  @VisibleForTesting
+  void showTopScreen() {
     List<ApplicationInformation> appsInfo = new ArrayList<>();
     List<ApplicationReport> apps;
     try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
index 0a7ee3ff734c1..74cf5152d5a88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -107,7 +107,8 @@ private void testProxyProvider(boolean facadeFlushCache) throws Exception {
     conf.setBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR,
         facadeFlushCache);
 
-    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, true);
     conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
     conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
     conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/AMRMTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/AMRMTokenIdentifierForTest.java
index 782dc6d644cdd..c493bd2821663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/AMRMTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/AMRMTokenIdentifierForTest.java
@@ -25,11 +25,8 @@
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
 import org.apache.hadoop.yarn.proto.YarnSecurityTestAMRMTokenProtos.AMRMTokenIdentifierForTestProto;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
index 706400f80d7bc..63ebffaca4409 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
@@ -18,7 +18,12 @@
 
 package org.apache.hadoop.yarn.client.cli;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -27,9 +32,13 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -47,6 +56,9 @@ public class TestTopCLI {
 
   private static Map<String, String> savedStaticResolution = new HashMap<>();
 
+  private PrintStream stdout;
+  private PrintStream stderr;
+
   @BeforeClass
   public static void initializeDummyHostnameResolution() throws Exception {
     String previousIpAddress;
@@ -68,6 +80,18 @@ public static void restoreDummyHostnameResolution() throws Exception {
     }
   }
 
+  @Before
+  public void before() {
+    this.stdout = System.out;
+    this.stderr = System.err;
+  }
+
+  @After
+  public void after() {
+    System.setOut(this.stdout);
+    System.setErr(this.stderr);
+  }
+
   @Test
   public void testHAClusterInfoURL() throws IOException, InterruptedException {
     TopCLI topcli = new TopCLI();
@@ -103,4 +127,44 @@ public void testHAClusterInfoURL() throws IOException, InterruptedException {
     Assert.assertEquals("https", clusterUrl.getProtocol());
     Assert.assertEquals(rm1Address, clusterUrl.getAuthority());
   }
-}
\ No newline at end of file
+
+  @Test
+  public void testHeaderNodeManagers() throws Exception {
+    YarnClusterMetrics ymetrics = mock(YarnClusterMetrics.class);
+    when(ymetrics.getNumNodeManagers()).thenReturn(0);
+    when(ymetrics.getNumDecommissioningNodeManagers()).thenReturn(1);
+    when(ymetrics.getNumDecommissionedNodeManagers()).thenReturn(2);
+    when(ymetrics.getNumActiveNodeManagers()).thenReturn(3);
+    when(ymetrics.getNumLostNodeManagers()).thenReturn(4);
+    when(ymetrics.getNumUnhealthyNodeManagers()).thenReturn(5);
+    when(ymetrics.getNumRebootedNodeManagers()).thenReturn(6);
+    when(ymetrics.getNumShutdownNodeManagers()).thenReturn(7);
+
+    YarnClient client = mock(YarnClient.class);
+    when(client.getYarnClusterMetrics()).thenReturn(ymetrics);
+
+    TopCLI topcli = new TopCLI() {
+      @Override protected void createAndStartYarnClient() {
+      }
+    };
+    topcli.setClient(client);
+    topcli.terminalWidth = 200;
+
+    String actual;
+    try (ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+        PrintStream out = new PrintStream(outStream)) {
+      System.setOut(out);
+      System.setErr(out);
+      topcli.showTopScreen();
+      out.flush();
+      actual = outStream.toString("UTF-8");
+    }
+
+    String expected = "NodeManager(s)"
+        + ": 0 total, 3 active, 5 unhealthy, 1 decommissioning,"
+        + " 2 decommissioned, 4 lost, 6 rebooted, 7 shutdown";
+    Assert.assertTrue(
+        String.format("Expected output to contain [%s], actual output was [%s].", expected, actual),
+        actual.contains(expected));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 621540677cf04..22c7cc34bcf2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1928,7 +1928,7 @@ public void testGetQueueInfoWithNonExistedQueue() throws Exception {
     QueueCLI cli = createAndGetQueueCLI();
     when(client.getQueueInfo(any(String.class))).thenReturn(null);
     int result = cli.run(new String[] { "-status", queueName });
-    assertEquals(-1, result);;
+    assertEquals(-1, result);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
     pw.println("Cannot get queue from RM by queueName = " + queueName
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 195999fe1ff38..f1731751c479f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -138,6 +138,26 @@
       <artifactId>bcprov-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.sun.jersey.jersey-test-framework</groupId>
       <artifactId>jersey-test-framework-grizzly2</artifactId>
@@ -156,8 +176,22 @@
       <artifactId>jersey-server</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey.contribs</groupId>
@@ -233,15 +267,15 @@
             <exclude>src/main/resources/webapps/test/.keep</exclude>
             <exclude>src/main/resources/webapps/proxy/.keep</exclude>
             <exclude>src/main/resources/webapps/node/.keep</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd</exclude>
-            <exclude>src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/css/demo_table.css</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/images/Sorting icons.psd</exclude>
+            <exclude>src/main/resources/webapps/static/dt-1.11.5/js/jquery.dataTables.min.js</exclude>
             <exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
-            <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js</exclude>
-            <exclude>src/main/resources/webapps/static/jquery/jquery-3.5.1.min.js</exclude>
+            <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.13.2.custom.min.js</exclude>
+            <exclude>src/main/resources/webapps/static/jquery/jquery-3.6.0.min.js</exclude>
             <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
             <exclude>src/test/resources/application_1440536969523_0001.har/_index</exclude>
             <exclude>src/test/resources/application_1440536969523_0001.har/part-0</exclude>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
index 751d9af093f46..09efe41e0c22f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
@@ -91,6 +91,8 @@ public synchronized void close() {
 
   /**
    * Getter/Setter methods for log4j.
+   *
+   * @return containerLogDir.
    */
   
   public String getContainerLogDir() {
@@ -118,6 +120,8 @@ public long getTotalLogFileSize() {
   /**
    *  Setter so that log4j can configure it from the
    *  configuration(log4j.properties).
+   *
+   * @param logSize log size.
    */
   public void setTotalLogFileSize(long logSize) {
     maxEvents = (int)(logSize / EVENT_SIZE);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java
index 7dd712e156beb..f0e00fc194064 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java
@@ -54,6 +54,8 @@ public void flush() {
 
   /**
    * Getter/Setter methods for log4j.
+   *
+   * @return containerLogDir.
    */
 
   public String getContainerLogDir() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/package-info.java
index a4349b2295768..6ded96f0e7e54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,6 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.impl.pb.client;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/package-info.java
index 1d3d435385f30..730ac9c628b40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,6 +15,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.impl.pb.service;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
index 18da80f4869b1..167e078643354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,6 +18,6 @@
 /**
  * API related to protobuf objects that are not backed by PBImpl classes.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.pb;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index a5705d275fe01..4dc3d462ccb4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.yarn.api.records.AMCommand;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.PreemptionMessage;
@@ -43,6 +44,7 @@
 import org.apache.hadoop.yarn.api.records.impl.pb.CollectorInfoPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.EnhancedHeadroomPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
@@ -89,6 +91,7 @@ public class AllocateResponsePBImpl extends AllocateResponse {
   private Token amrmToken = null;
   private Priority appPriority = null;
   private CollectorInfo collectorInfo = null;
+  private EnhancedHeadroom enhancedHeadroom = null;
 
   public AllocateResponsePBImpl() {
     builder = AllocateResponseProto.newBuilder();
@@ -190,6 +193,9 @@ private synchronized void mergeLocalToBuilder() {
           getContainerProtoIterable(this.containersFromPreviousAttempts);
       builder.addAllContainersFromPreviousAttempts(iterable);
     }
+    if (this.enhancedHeadroom != null) {
+      builder.setEnhancedHeadroom(convertToProtoFormat(this.enhancedHeadroom));
+    }
   }
 
   private synchronized void mergeLocalToProto() {
@@ -422,6 +428,28 @@ public synchronized void setAMRMToken(Token amRMToken) {
     this.amrmToken = amRMToken;
   }
 
+  @Override
+  public synchronized EnhancedHeadroom getEnhancedHeadroom() {
+    AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (enhancedHeadroom != null) {
+      return enhancedHeadroom;
+    }
+    if (!p.hasEnhancedHeadroom()) {
+      return null;
+    }
+    this.enhancedHeadroom = convertFromProtoFormat(p.getEnhancedHeadroom());
+    return enhancedHeadroom;
+  }
+
+  @Override
+  public synchronized void setEnhancedHeadroom(
+      EnhancedHeadroom enhancedHeadroom) {
+    maybeInitBuilder();
+    if (enhancedHeadroom == null) {
+      builder.clearEnhancedHeadroom();
+    }
+    this.enhancedHeadroom = enhancedHeadroom;
+  }
 
   @Override
   public synchronized CollectorInfo getCollectorInfo() {
@@ -933,4 +961,14 @@ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
   private PriorityProto convertToProtoFormat(Priority t) {
     return ((PriorityPBImpl)t).getProto();
   }
+
+  private EnhancedHeadroomPBImpl convertFromProtoFormat(
+      YarnServiceProtos.EnhancedHeadroomProto p) {
+    return new EnhancedHeadroomPBImpl(p);
+  }
+
+  private YarnServiceProtos.EnhancedHeadroomProto convertToProtoFormat(
+      EnhancedHeadroom t) {
+    return ((EnhancedHeadroomPBImpl) t).getProto();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
index 961b9f1eec5ff..5f4f0394e3fe4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
@@ -29,9 +29,7 @@
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
 import org.apache.hadoop.yarn.proto.YarnProtos.LabelsToNodeIdsProto;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
index 6b7c45afaccaa..c9535cacf9adb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -28,9 +27,7 @@
 
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
index 24661e2054701..7212dd55133cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoRequestPBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoRequest;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
index ece903f78936a..b094fa2621d83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetPluginInfoResponsePBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.GetPluginInfoResponse;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
index 82bb6e314393c..72854ad5497cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeRequestPBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 import org.apache.hadoop.yarn.api.protocolrecords.NodePublishVolumeRequest;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeResponsePBImpl.java
index 43c4ada4f0f4f..48742cc2587fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodePublishVolumeResponsePBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.NodePublishVolumeResponse;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeRequestPBImpl.java
index 46a59b46fd739..2d64650e9c574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeRequestPBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 import org.apache.hadoop.yarn.api.protocolrecords.NodeUnpublishVolumeRequest;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeResponsePBImpl.java
index 8b170ea88e072..2ad583718970c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/NodeUnpublishVolumeResponsePBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.NodeUnpublishVolumeResponse;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
index 09aabe05ffa5f..660b24bc66fee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesRequestPBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
index 2d5421d7d0775..3653371e7de66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ValidateVolumeCapabilitiesResponsePBImpl.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/package-info.java
index 4b29e4f740ea2..e783448d2b40e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java
index 607d83245e7c0..e7b2078568e7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptIdPBImpl.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @Private
 @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java
index 1141ef31834bc..2b5c1d14d10e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationIdPBImpl.java
@@ -24,8 +24,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 @Private
 @Unstable
 public class ApplicationIdPBImpl extends ApplicationId {
@@ -46,24 +44,32 @@ public ApplicationIdProto getProto() {
 
   @Override
   public int getId() {
-    Preconditions.checkNotNull(proto);
+    if (proto == null) {
+      throw new NullPointerException("The argument object is NULL");
+    }
     return proto.getId();
   }
 
   @Override
   protected void setId(int id) {
-    Preconditions.checkNotNull(builder);
+    if (builder == null) {
+      throw new NullPointerException("The argument object is NULL");
+    }
     builder.setId(id);
   }
   @Override
   public long getClusterTimestamp() {
-    Preconditions.checkNotNull(proto);
+    if (proto == null) {
+      throw new NullPointerException("The argument object is NULL");
+    }
     return proto.getClusterTimestamp();
   }
 
   @Override
   protected void setClusterTimestamp(long clusterTimestamp) {
-    Preconditions.checkNotNull(builder);
+    if (builder == null) {
+      throw new NullPointerException("The argument object is NULL");
+    }
     builder.setClusterTimestamp((clusterTimestamp));
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
index 2a0f0eef8f3fa..134ce6617b653 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @Private
 @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/EnhancedHeadroomPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/EnhancedHeadroomPBImpl.java
new file mode 100644
index 0000000000000..65b5b9e11a9b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/EnhancedHeadroomPBImpl.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.EnhancedHeadroomProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.EnhancedHeadroomProtoOrBuilder;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+public class EnhancedHeadroomPBImpl extends EnhancedHeadroom {
+
+  private EnhancedHeadroomProto proto =
+      EnhancedHeadroomProto.getDefaultInstance();
+  private EnhancedHeadroomProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public EnhancedHeadroomPBImpl() {
+    builder = EnhancedHeadroomProto.newBuilder();
+  }
+
+  public EnhancedHeadroomPBImpl(EnhancedHeadroomProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public EnhancedHeadroomProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private void mergeLocalToBuilder() {
+    // No local content yet
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = EnhancedHeadroomProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public void setTotalPendingCount(int totalPendingCount) {
+    maybeInitBuilder();
+    if (totalPendingCount == 0) {
+      builder.clearTotalPendingCount();
+      return;
+    }
+    builder.setTotalPendingCount(totalPendingCount);
+  }
+
+  @Override
+  public int getTotalPendingCount() {
+    EnhancedHeadroomProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasTotalPendingCount()) ? p.getTotalPendingCount() : 0;
+  }
+
+  @Override
+  public void setTotalActiveCores(int totalActiveCores) {
+    maybeInitBuilder();
+    if (totalActiveCores == 0) {
+      builder.clearTotalActiveCores();
+      return;
+    }
+    builder.setTotalActiveCores(totalActiveCores);
+  }
+
+  @Override
+  public int getTotalActiveCores() {
+    EnhancedHeadroomProtoOrBuilder p = viaProto ? proto : builder;
+    return (p.hasTotalActiveCores()) ? p.getTotalActiveCores() : 0;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
index 95bc29b2e5c0a..8c426c4902bba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeIdPBImpl.java
@@ -24,7 +24,7 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @Private
 @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index 1b14f9d700c09..1d213b3059208 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -72,11 +72,11 @@ private void mergeLocalToBuilder() {
           .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
     }
     if (this.configuredMinResource != null) {
-      builder.setEffectiveMinCapacity(
+      builder.setConfiguredMinCapacity(
           convertToProtoFormat(this.configuredMinResource));
     }
     if (this.configuredMaxResource != null) {
-      builder.setEffectiveMaxCapacity(
+      builder.setConfiguredMaxCapacity(
           convertToProtoFormat(this.configuredMaxResource));
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationAllocationStatePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationAllocationStatePBImpl.java
index 88e39ec994fa5..48f62c1bae721 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationAllocationStatePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationAllocationStatePBImpl.java
@@ -62,7 +62,7 @@
 public class ReservationAllocationStatePBImpl extends
         ReservationAllocationState {
   private ReservationAllocationStateProto proto =
-          ReservationAllocationStateProto.getDefaultInstance();;
+          ReservationAllocationStateProto.getDefaultInstance();
   private ReservationAllocationStateProto.Builder builder = null;
   private boolean viaProto = false;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
index a87c7d90e45fd..271ae9402254f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationIdPBImpl.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @Private
 @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
index 35fe945a07fab..125bda9c57a28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
@@ -24,8 +24,6 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceOptionProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceOptionProtoOrBuilder;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 public class ResourceOptionPBImpl extends ResourceOption {
 
   ResourceOptionProto proto = ResourceOptionProto.getDefaultInstance();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 75f87541603b1..850c36b20addc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/YarnClusterMetricsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/YarnClusterMetricsPBImpl.java
index 14f8bff58190f..608d3c28af410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/YarnClusterMetricsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/YarnClusterMetricsPBImpl.java
@@ -89,6 +89,22 @@ public void setNumNodeManagers(int numNodeManagers) {
     builder.setNumNodeManagers((numNodeManagers));
   }
 
+  @Override
+  public int getNumDecommissioningNodeManagers() {
+    YarnClusterMetricsProtoOrBuilder p = viaProto ? proto : builder;
+    if (p.hasNumDecommissioningNms()) {
+      return (p.getNumDecommissioningNms());
+    }
+    return 0;
+  }
+
+  @Override
+  public void
+      setNumDecommissioningNodeManagers(int numDecommissioningNodeManagers) {
+    maybeInitBuilder();
+    builder.setNumDecommissioningNms(numDecommissioningNodeManagers);
+  }
+
   @Override
   public int getNumDecommissionedNodeManagers() {
     YarnClusterMetricsProtoOrBuilder p = viaProto ? proto : builder;
@@ -165,4 +181,19 @@ public void setNumRebootedNodeManagers(int numRebootedNodeManagers) {
     maybeInitBuilder();
     builder.setNumRebootedNms((numRebootedNodeManagers));
   }
-}
\ No newline at end of file
+
+  @Override
+  public int getNumShutdownNodeManagers() {
+    YarnClusterMetricsProtoOrBuilder p = viaProto ? proto : builder;
+    if (p.hasNumShutdownNms()) {
+      return (p.getNumShutdownNms());
+    }
+    return 0;
+  }
+
+  @Override
+  public void setNumShutdownNodeManagers(int numShutdownNodeManagers) {
+    maybeInitBuilder();
+    builder.setNumShutdownNms(numShutdownNodeManagers);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/package-info.java
index 2571db8e8dc5e..81334c0591dc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.records.impl.pb;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
index 660dc02d45c41..e1bd6dc17378e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,6 +18,6 @@
 /**
  * API related to resources.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.api.resource;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AutoRefreshNoHARMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AutoRefreshNoHARMFailoverProxyProvider.java
index f81fe1649b08c..cc20214169dee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AutoRefreshNoHARMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AutoRefreshNoHARMFailoverProxyProvider.java
@@ -73,7 +73,7 @@ protected T getProxyInternal() {
 
   /**
    * Stop the current proxy when performFailover.
-   * @param currentProxy
+   * @param currentProxy currentProxy.
    */
   @Override
   public synchronized void performFailover(T currentProxy) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
index c69a9df561bb5..35b1906698b7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
@@ -42,7 +42,7 @@
 import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @InterfaceAudience.Public
 @InterfaceStability.Stable
@@ -65,7 +65,7 @@ private ClientRMProxy(){
    * @param protocol Client protocol for which proxy is being requested.
    * @param <T> Type of proxy.
    * @return Proxy to the ResourceManager for the specified client protocol.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public static <T> T createRMProxy(final Configuration configuration,
       final Class<T> protocol) throws IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/DefaultNoHARMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/DefaultNoHARMFailoverProxyProvider.java
index e5197cfd1a44e..d64d1a37913b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/DefaultNoHARMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/DefaultNoHARMFailoverProxyProvider.java
@@ -80,7 +80,7 @@ public ProxyInfo<T> getProxy() {
 
   /**
    * PerformFailover does nothing in this class.
-   * @param currentProxy
+   * @param currentProxy currentProxy.
    */
   @Override
   public void performFailover(T currentProxy) {
@@ -89,7 +89,7 @@ public void performFailover(T currentProxy) {
 
   /**
    * Close the current proxy.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   @Override
   public void close() throws IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index e3be31c10a1c3..a2ea992aeea79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -70,6 +70,8 @@ protected RMProxy() {
 
   /**
    * Verify the passed protocol is supported.
+   *
+   * @param protocol protocol.
    */
   @Private
   public void checkAllowedProtocols(Class<?> protocol) {}
@@ -77,6 +79,11 @@ public void checkAllowedProtocols(Class<?> protocol) {}
   /**
    * Get the ResourceManager address from the provided Configuration for the
    * given protocol.
+   *
+   * @param conf configuration.
+   * @param protocol protocol.
+   * @return inet socket address.
+   * @throws IOException io error occur.
    */
   @Private
   public InetSocketAddress getRMAddress(
@@ -91,6 +98,13 @@ public InetSocketAddress getRMAddress(
    * this is a direct connection to the ResourceManager address. When HA is
    * enabled, the proxy handles the failover between the ResourceManagers as
    * well.
+   *
+   * @param configuration configuration.
+   * @param protocol protocol.
+   * @param instance RMProxy instance.
+   * @param <T> Generic T.
+   * @return RMProxy.
+   * @throws IOException io error occur.
    */
   @Private
   protected static <T> T createRMProxy(final Configuration configuration,
@@ -98,8 +112,7 @@ protected static <T> T createRMProxy(final Configuration configuration,
     YarnConfiguration conf = (configuration instanceof YarnConfiguration)
         ? (YarnConfiguration) configuration
         : new YarnConfiguration(configuration);
-    RetryPolicy retryPolicy = createRetryPolicy(conf,
-        (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf)));
+    RetryPolicy retryPolicy = createRetryPolicy(conf, isFailoverEnabled(conf));
     return newProxyInstance(conf, protocol, instance, retryPolicy);
   }
 
@@ -109,6 +122,15 @@ protected static <T> T createRMProxy(final Configuration configuration,
    * this is a direct connection to the ResourceManager address. When HA is
    * enabled, the proxy handles the failover between the ResourceManagers as
    * well.
+   *
+   * @param configuration configuration.
+   * @param protocol protocol.
+   * @param instance RMProxy instance.
+   * @param retryTime retry Time.
+   * @param retryInterval retry Interval.
+   * @param <T> Generic T.
+   * @return RMProxy.
+   * @throws IOException io error occur.
    */
   @Private
   protected static <T> T createRMProxy(final Configuration configuration,
@@ -126,7 +148,7 @@ private static <T> T newProxyInstance(final YarnConfiguration conf,
       final Class<T> protocol, RMProxy<T> instance, RetryPolicy retryPolicy)
           throws IOException{
     RMFailoverProxyProvider<T> provider;
-    if (HAUtil.isHAEnabled(conf) || HAUtil.isFederationEnabled(conf)) {
+    if (isFailoverEnabled(conf)) {
       provider = instance.createRMFailoverProxyProvider(conf, protocol);
     } else {
       provider = instance.createNonHaRMFailoverProxyProvider(conf, protocol);
@@ -137,6 +159,13 @@ private static <T> T newProxyInstance(final YarnConfiguration conf,
   /**
    * Get a proxy to the RM at the specified address. To be used to create a
    * RetryProxy.
+   *
+   * @param conf configuration.
+   * @param protocol protocol.
+   * @param rmAddress rmAddress.
+   * @param <T> Generic T.
+   * @return RM proxy.
+   * @throws IOException io error occur.
    */
   @Private
   public <T> T getProxy(final Configuration conf,
@@ -196,7 +225,11 @@ private <T> RMFailoverProxyProvider<T> createRMFailoverProxyProvider(
   }
 
   /**
-   * Fetch retry policy from Configuration
+   * Fetch retry policy from Configuration.
+   *
+   * @param conf configuration.
+   * @param isHAEnabled is HA enabled.
+   * @return RetryPolicy.
    */
   @Private
   @VisibleForTesting
@@ -219,6 +252,12 @@ public static RetryPolicy createRetryPolicy(Configuration conf,
   /**
    * Fetch retry policy from Configuration and create the
    * retry policy with specified retryTime and retry interval.
+   *
+   * @param conf configuration.
+   * @param retryTime retry time.
+   * @param retryInterval retry interval.
+   * @param isHAEnabled is HA enabled.
+   * @return RetryPolicy.
    */
   protected static RetryPolicy createRetryPolicy(Configuration conf,
       long retryTime, long retryInterval, boolean isHAEnabled) {
@@ -300,7 +339,20 @@ protected static RetryPolicy createRetryPolicy(Configuration conf,
     // YARN-4288: local IOException is also possible.
     exceptionToPolicyMap.put(IOException.class, retryPolicy);
     // Not retry on remote IO exception.
-    return RetryPolicies.retryOtherThanRemoteException(
+    return RetryPolicies.retryOtherThanRemoteAndSaslException(
         RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
   }
+
+  private static boolean isFailoverEnabled(YarnConfiguration conf) {
+    if (HAUtil.isHAEnabled(conf)) {
+      // Considering Resource Manager HA is enabled.
+      return true;
+    }
+    if (HAUtil.isFederationEnabled(conf) && HAUtil.isFederationFailoverEnabled(conf)) {
+      // Considering both federation and federation failover are enabled.
+      return true;
+    }
+    return false;
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index b7775b6fbd59d..878362e83aef2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -41,7 +41,7 @@
 import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 @Public
 @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index b9f72484a5d0b..38c31599b16b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -255,6 +255,9 @@ public abstract int initiateUpgrade(String appName, String fileName,
    *
    * @param appName            the name of the application.
    * @param componentInstances the name of the component instances.
+   * @return exit code.
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   @Unstable
@@ -267,6 +270,9 @@ public abstract int actionUpgradeInstances(String appName,
    *
    * @param appName    the name of the application.
    * @param components the name of the components.
+   * @return exit code.
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   @Unstable
@@ -279,6 +285,9 @@ public abstract int actionUpgradeComponents(String appName,
    * @param appName  the name of the application.
    * @param userName the name of the user.
    * @return exit code
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
+   * @throws InterruptedException  if interrupted.
    */
   @Public
   @Unstable
@@ -297,6 +306,8 @@ public abstract String getInstances(String appName,
    * @param appName  the name of the application
    * @param fileName specification of application upgrade to save.
    * @return exit code
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   @Unstable
@@ -308,8 +319,8 @@ public abstract int actionUpgradeExpress(String appName, File fileName)
    *
    * @param appName the name of the application
    * @return exit code
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   @Unstable
@@ -321,6 +332,9 @@ public abstract int actionCancelUpgrade(String appName) throws IOException,
    *
    * @param appName            the name of the application.
    * @param componentInstances the name of the component instances.
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
+   * @return exit code.
    */
   @Public
   @Unstable
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 4835239a92053..68e44ffc81573 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -114,8 +114,8 @@ public abstract TimelinePutResponse putEntities(
    * 
    * @param domain
    *          an {@link TimelineDomain} object
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   public abstract void putDomain(
@@ -133,8 +133,8 @@ public abstract void putDomain(
    * @param domain
    *          an {@link TimelineDomain} object
    * @param appAttemptId {@link ApplicationAttemptId}
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   public abstract void putDomain(ApplicationAttemptId appAttemptId,
@@ -151,8 +151,8 @@ public abstract void putDomain(ApplicationAttemptId appAttemptId,
    *          securely talking to the timeline server
    * @return a delegation token ({@link Token}) that can be used to talk to the
    *         timeline server
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   public abstract Token<TimelineDelegationTokenIdentifier> getDelegationToken(
@@ -166,8 +166,8 @@ public abstract Token<TimelineDelegationTokenIdentifier> getDelegationToken(
    * @param timelineDT
    *          the delegation token to renew
    * @return the new expiration time
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   public abstract long renewDelegationToken(
@@ -181,8 +181,8 @@ public abstract long renewDelegationToken(
    * 
    * @param timelineDT
    *          the delegation token to cancel
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   @Public
   public abstract void cancelDelegationToken(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
index f73c2d3733071..3c450f46e6bf1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineReaderClient.java
@@ -42,6 +42,8 @@ public abstract class TimelineReaderClient extends CompositeService {
 
   /**
    * Create a new instance of Timeline Reader Client.
+   *
+   * @return instance of Timeline Reader Client.
    */
   @InterfaceAudience.Public
   public static TimelineReaderClient createTimelineReaderClient() {
@@ -59,7 +61,7 @@ public TimelineReaderClient(String name) {
    * @param fields Fields to be fetched. Defaults to INFO.
    * @param filters Filters to be applied while fetching entities.
    * @return entity of the application
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public abstract  TimelineEntity getApplicationEntity(
       ApplicationId appId, String fields, Map<String, String> filters)
@@ -71,7 +73,7 @@ public abstract  TimelineEntity getApplicationEntity(
    * @param fields Fields to be fetched. Defaults to INFO.
    * @param filters Filters to be applied while fetching entities.
    * @return entity associated with application attempt
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public abstract  TimelineEntity getApplicationAttemptEntity(
       ApplicationAttemptId appAttemptId, String fields,
@@ -85,7 +87,7 @@ public abstract  TimelineEntity getApplicationAttemptEntity(
    * @param limit Number of entities to return.
    * @param fromId Retrieve next set of generic ids from given fromId
    * @return list of application attempt entities
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public abstract  List<TimelineEntity> getApplicationAttemptEntities(
       ApplicationId appId, String fields, Map<String, String> filters,
@@ -97,7 +99,7 @@ public abstract  List<TimelineEntity> getApplicationAttemptEntities(
    * @param fields Fields to be fetched. Defaults to INFO.
    * @param filters Filters to be applied while fetching entities.
    * @return timeline entity for container
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public abstract  TimelineEntity getContainerEntity(
       ContainerId containerId, String fields, Map<String, String> filters)
@@ -111,7 +113,7 @@ public abstract  TimelineEntity getContainerEntity(
    * @param limit Number of entities to return.
    * @param fromId Retrieve next set of generic ids from given fromId
    * @return list of entities
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public abstract List<TimelineEntity> getContainerEntities(
       ApplicationId appId, String fields,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
index b6cbab87a3827..5a216d20d35d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
@@ -57,7 +57,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientHandlerException;
 import com.sun.jersey.api.client.ClientRequest;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
index 29609b955fbec..71bf13220b016 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java
@@ -38,7 +38,10 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.MultivaluedMap;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -208,12 +211,21 @@ public List<TimelineEntity> getContainerEntities(
     return Arrays.asList(entity);
   }
 
+  @VisibleForTesting
+  protected String encodeValue(String value) throws UnsupportedEncodingException {
+    // Since URLEncoder doesn't use and doesn't have an option for percent-encoding
+    // (as specified in RFC 3986) the spaces are encoded to + signs, which need to be replaced
+    // manually
+    return URLEncoder.encode(value, StandardCharsets.UTF_8.toString())
+        .replaceAll("\\+", "%20");
+  }
+
   private void mergeFilters(MultivaluedMap<String, String> defaults,
-      Map<String, String> filters) {
+                            Map<String, String> filters) throws UnsupportedEncodingException {
     if (filters != null && !filters.isEmpty()) {
       for (Map.Entry<String, String> entry : filters.entrySet()) {
         if (!defaults.containsKey(entry.getKey())) {
-          defaults.add(entry.getKey(), filters.get(entry.getValue()));
+          defaults.add(entry.getKey(), encodeValue(entry.getValue()));
         }
       }
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 1c4ed24b47d78..eee927c8c7ce5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -191,8 +191,21 @@ protected void serviceInit(Configuration conf) throws Exception{
         .build();
     // Thread pool for async print event details,
     // to prevent wasting too much time for RM.
+    int numCorePoolSizeThreads = getConfig().getInt(
+        YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE,
+        YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE);
+
+    int numMaximumPoolSizeThreads = getConfig().getInt(
+        YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE,
+        YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE);
+
+    long keepAliveTime =
+         conf.getTimeDuration(YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
+         YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
+         TimeUnit.SECONDS);
+
     printEventDetailsExecutor = new ThreadPoolExecutor(
-        1, 5, 10, TimeUnit.SECONDS,
+        numCorePoolSizeThreads, numMaximumPoolSizeThreads, keepAliveTime, TimeUnit.SECONDS,
         new LinkedBlockingQueue<>(), threadFactory);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/package-info.java
index c410160f84133..a9129dc145612 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.event;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/package-info.java
index aae0b4896fc37..ea47367e88a5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.factories.impl.pb;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
index 088c2fc95bd81..b9f77510ab8bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
@@ -22,24 +22,28 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 
 import org.apache.hadoop.thirdparty.protobuf.ServiceException;
 
-@InterfaceAudience.LimitedPrivate({ "MapReduce", "YARN" })
+@LimitedPrivate({ "MapReduce", "YARN" })
 public class RPCUtil {
 
   /**
-   * Returns an instance of {@link YarnException}
+   * Returns an instance of {@link YarnException}.
+   * @param t instance of Throwable.
+   * @return instance of YarnException.
    */
   public static YarnException getRemoteException(Throwable t) {
     return new YarnException(t);
   }
 
   /**
-   * Returns an instance of {@link YarnException}
+   * Returns an instance of {@link YarnException}.
+   * @param message yarn exception message.
+   * @return instance of YarnException.
    */
   public static YarnException getRemoteException(String message) {
     return new YarnException(message);
@@ -92,6 +96,8 @@ private static <T extends RuntimeException> T instantiateRuntimeException(
    *          ServiceException
    * @return An instance of the actual exception, which will be a subclass of
    *         {@link YarnException} or {@link IOException}
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   public static Void unwrapAndThrowException(ServiceException se)
       throws IOException, YarnException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/package-info.java
index eec93feb53858..62d5e02797c1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.LimitedPrivate({ "MapReduce", "YARN" })
+@LimitedPrivate({ "MapReduce", "YARN" })
 package org.apache.hadoop.yarn.ipc;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
index 9427068cfc505..c7e871d0f5034 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
@@ -19,6 +19,8 @@
 package org.apache.hadoop.yarn.logaggregation;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
 
@@ -57,23 +59,21 @@ public class AggregatedLogDeletionService extends AbstractService {
   
   private Timer timer = null;
   private long checkIntervalMsecs;
-  private LogDeletionTask task;
+  private List<LogDeletionTask> tasks = new ArrayList<>();
   
-  static class LogDeletionTask extends TimerTask {
+  public static class LogDeletionTask extends TimerTask {
     private Configuration conf;
     private long retentionMillis;
     private String suffix = null;
     private Path remoteRootLogDir = null;
     private ApplicationClientProtocol rmClient = null;
     
-    public LogDeletionTask(Configuration conf, long retentionSecs, ApplicationClientProtocol rmClient) {
+    public LogDeletionTask(Configuration conf, long retentionSecs,
+                           ApplicationClientProtocol rmClient,
+                           LogAggregationFileController fileController) {
       this.conf = conf;
       this.retentionMillis = retentionSecs * 1000;
       this.suffix = LogAggregationUtils.getBucketSuffix();
-      LogAggregationFileControllerFactory factory =
-          new LogAggregationFileControllerFactory(conf);
-      LogAggregationFileController fileController =
-          factory.getFileControllerForWrite();
       this.remoteRootLogDir = fileController.getRemoteRootLogDir();
       this.rmClient = rmClient;
     }
@@ -101,7 +101,7 @@ public void run() {
           }
         }
       } catch (Throwable t) {
-        logException("Error reading root log dir this deletion " +
+        logException("Error reading root log dir, this deletion " +
             "attempt is being aborted", t);
       }
       LOG.info("aggregated log deletion finished.");
@@ -220,7 +220,7 @@ public AggregatedLogDeletionService() {
 
   @Override
   protected void serviceStart() throws Exception {
-    scheduleLogDeletionTask();
+    scheduleLogDeletionTasks();
     super.serviceStart();
   }
 
@@ -249,13 +249,13 @@ public void refreshLogRetentionSettings() throws IOException {
       setConfig(conf);
       stopRMClient();
       stopTimer();
-      scheduleLogDeletionTask();
+      scheduleLogDeletionTasks();
     } else {
       LOG.warn("Failed to execute refreshLogRetentionSettings : Aggregated Log Deletion Service is not started");
     }
   }
   
-  private void scheduleLogDeletionTask() throws IOException {
+  private void scheduleLogDeletionTasks() throws IOException {
     Configuration conf = getConfig();
     if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
         YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
@@ -271,9 +271,28 @@ private void scheduleLogDeletionTask() throws IOException {
       return;
     }
     setLogAggCheckIntervalMsecs(retentionSecs);
-    task = new LogDeletionTask(conf, retentionSecs, createRMClient());
-    timer = new Timer();
-    timer.scheduleAtFixedRate(task, 0, checkIntervalMsecs);
+
+    tasks = createLogDeletionTasks(conf, retentionSecs, createRMClient());
+    for (LogDeletionTask task : tasks) {
+      timer = new Timer();
+      timer.scheduleAtFixedRate(task, 0, checkIntervalMsecs);
+    }
+  }
+
+  @VisibleForTesting
+  public List<LogDeletionTask> createLogDeletionTasks(Configuration conf, long retentionSecs,
+                                                      ApplicationClientProtocol rmClient)
+          throws IOException {
+    List<LogDeletionTask> tasks = new ArrayList<>();
+    LogAggregationFileControllerFactory factory = new LogAggregationFileControllerFactory(conf);
+    List<LogAggregationFileController> fileControllers =
+            factory.getConfiguredLogAggregationFileControllerList();
+    for (LogAggregationFileController fileController : fileControllers) {
+      LogDeletionTask task = new LogDeletionTask(conf, retentionSecs, rmClient,
+              fileController);
+      tasks.add(task);
+    }
+    return tasks;
   }
 
   private void stopTimer() {
@@ -295,14 +314,18 @@ protected Configuration createConf() {
   // as @Idempotent, it will automatically take care of RM restart/failover.
   @VisibleForTesting
   protected ApplicationClientProtocol createRMClient() throws IOException {
-    return ClientRMProxy.createRMProxy(getConfig(),
-      ApplicationClientProtocol.class);
+    return ClientRMProxy.createRMProxy(getConfig(), ApplicationClientProtocol.class);
   }
 
   @VisibleForTesting
   protected void stopRMClient() {
-    if (task != null && task.getRMClient() != null) {
-      RPC.stopProxy(task.getRMClient());
+    for (LogDeletionTask task : tasks) {
+      if (task != null && task.getRMClient() != null) {
+        RPC.stopProxy(task.getRMClient());
+        //The RMClient instance is the same for all deletion tasks.
+        //It is enough to close the RM client once
+        break;
+      }
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 3f251c74bea62..fc0b71e6a42a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -77,6 +78,11 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
 
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
 @Public
 @Evolving
 public class AggregatedLogFormat {
@@ -576,9 +582,16 @@ public LogReader(Configuration conf, Path remoteAppLogFile)
       try {
         FileContext fileContext =
             FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
-        this.fsDataIStream = fileContext.open(remoteAppLogFile);
+        FileStatus status = fileContext.getFileStatus(remoteAppLogFile);
+        this.fsDataIStream = awaitFuture(
+            fileContext.openFile(remoteAppLogFile)
+                .opt(FS_OPTION_OPENFILE_READ_POLICY,
+                    FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
+                .opt(FS_OPTION_OPENFILE_LENGTH,
+                    status.getLen())   // file length hint for object stores
+                .build());
         reader = new TFile.Reader(this.fsDataIStream,
-            fileContext.getFileStatus(remoteAppLogFile).getLen(), conf);
+            status.getLen(), conf);
         this.scanner = reader.createScanner();
       } catch (IOException ioe) {
         close();
@@ -906,6 +919,7 @@ public static void readAContainerLogsForALogType(
      * @param logUploadedTime the log uploaded time stamp
      * @param logType the given log type
      * @throws IOException if we can not read the container logs
+     * @return If logType contains fileType, return 1, otherwise return 0.
      */
     public static int readContainerLogsForALogType(
         DataInputStream valueStream, PrintStream out, long logUploadedTime,
@@ -921,7 +935,9 @@ public static int readContainerLogsForALogType(
      * @param out the output print stream
      * @param logUploadedTime the log uploaded time stamp
      * @param logType the given log type
+     * @param bytes log bytes.
      * @throws IOException if we can not read the container logs
+     * @return If logType contains fileType, return 1, otherwise return 0.
      */
     public static int readContainerLogsForALogType(
         DataInputStream valueStream, PrintStream out, long logUploadedTime,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java
index 4268c095d3915..3ded57fe2fb50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileControllerFactory.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.logaggregation.filecontroller;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -53,9 +53,8 @@ public class LogAggregationFileControllerFactory {
       LogAggregationFileControllerFactory.class);
   private final Pattern p = Pattern.compile(
       "^[A-Za-z_]+[A-Za-z0-9_]*$");
-  private LinkedList<LogAggregationFileController> controllers
-      = new LinkedList<>();
-  private Configuration conf;
+  private final LinkedList<LogAggregationFileController> controllers = new LinkedList<>();
+  private final Configuration conf;
 
   /**
    * Construct the LogAggregationFileControllerFactory object.
@@ -65,77 +64,59 @@ public LogAggregationFileControllerFactory(Configuration conf) {
     this.conf = conf;
     Collection<String> fileControllers = conf.getStringCollection(
         YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS);
-    List<String> controllerClassName = new ArrayList<>();
-
     Map<String, String> controllerChecker = new HashMap<>();
 
-    for (String fileController : fileControllers) {
-      Preconditions.checkArgument(validateAggregatedFileControllerName(
-          fileController), "The FileControllerName: " + fileController
-          + " set in " + YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS
-          +" is invalid." + "The valid File Controller name should only "
-          + "contain a-zA-Z0-9_ and can not start with numbers");
+    for (String controllerName : fileControllers) {
+      validateAggregatedFileControllerName(controllerName);
 
-      String remoteDirStr = String.format(
-          YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT,
-          fileController);
-      String remoteDir = conf.get(remoteDirStr);
-      boolean defaultRemoteDir = false;
-      if (remoteDir == null || remoteDir.isEmpty()) {
-        remoteDir = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR);
-        defaultRemoteDir = true;
-      }
-      String suffixStr = String.format(
-          YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT,
-          fileController);
-      String suffix = conf.get(suffixStr);
-      boolean defaultSuffix = false;
-      if (suffix == null || suffix.isEmpty()) {
-        suffix = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
-            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX);
-        defaultSuffix = true;
-      }
-      String dirSuffix = remoteDir + "-" + suffix;
-      if (controllerChecker.containsKey(dirSuffix)) {
-        if (defaultRemoteDir && defaultSuffix) {
-          String fileControllerStr = controllerChecker.get(dirSuffix);
-          List<String> controllersList = new ArrayList<>();
-          controllersList.add(fileControllerStr);
-          controllersList.add(fileController);
-          fileControllerStr = StringUtils.join(controllersList, ",");
-          controllerChecker.put(dirSuffix, fileControllerStr);
-        } else {
-          String conflictController = controllerChecker.get(dirSuffix);
-          throw new RuntimeException("The combined value of " + remoteDirStr
-              + " and " + suffixStr + " should not be the same as the value"
-              + " set for " + conflictController);
-        }
+      validateConflictingControllers(conf, controllerChecker, controllerName);
+      DeterminedControllerClassName className =
+          new DeterminedControllerClassName(conf, controllerName);
+      LogAggregationFileController controller = createFileControllerInstance(conf,
+          controllerName, className);
+      controller.initialize(conf, controllerName);
+      controllers.add(controller);
+    }
+  }
+
+  private LogAggregationFileController createFileControllerInstance(
+      Configuration conf,
+      String fileController, DeterminedControllerClassName className) {
+    Class<? extends LogAggregationFileController> clazz = conf.getClass(
+        className.configKey, null, LogAggregationFileController.class);
+    if (clazz == null) {
+      throw new RuntimeException("No class defined for " + fileController);
+    }
+    LogAggregationFileController instance = ReflectionUtils.newInstance(clazz, conf);
+    if (instance == null) {
+      throw new RuntimeException("No object created for " + className.value);
+    }
+    return instance;
+  }
+
+  private void validateConflictingControllers(
+      Configuration conf, Map<String, String> controllerChecker, String fileController) {
+    DeterminedLogAggregationRemoteDir remoteDir =
+        new DeterminedLogAggregationRemoteDir(conf, fileController);
+    DeterminedLogAggregationSuffix suffix =
+        new DeterminedLogAggregationSuffix(conf, fileController);
+    String dirSuffix = remoteDir.value + "-" + suffix.value;
+    if (controllerChecker.containsKey(dirSuffix)) {
+      if (remoteDir.usingDefault && suffix.usingDefault) {
+        String fileControllerStr = controllerChecker.get(dirSuffix);
+        List<String> controllersList = new ArrayList<>();
+        controllersList.add(fileControllerStr);
+        controllersList.add(fileController);
+        fileControllerStr = StringUtils.join(controllersList, ",");
+        controllerChecker.put(dirSuffix, fileControllerStr);
       } else {
-        controllerChecker.put(dirSuffix, fileController);
-      }
-      String classKey = String.format(
-          YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT,
-          fileController);
-      String className = conf.get(classKey);
-      if (className == null || className.isEmpty()) {
-        throw new RuntimeException("No class configured for "
-            + fileController);
-      }
-      controllerClassName.add(className);
-      Class<? extends LogAggregationFileController> sClass = conf.getClass(
-          classKey, null, LogAggregationFileController.class);
-      if (sClass == null) {
-        throw new RuntimeException("No class defined for " + fileController);
-      }
-      LogAggregationFileController s = ReflectionUtils.newInstance(
-          sClass, conf);
-      if (s == null) {
-        throw new RuntimeException("No object created for "
-            + controllerClassName);
+        String conflictController = controllerChecker.get(dirSuffix);
+        throw new RuntimeException(String.format("The combined value of %s " +
+            "and %s should not be the same as the value set for %s",
+            remoteDir.configKey, suffix.configKey, conflictController));
       }
-      s.initialize(conf, fileController);
-      controllers.add(s);
+    } else {
+      controllerChecker.put(dirSuffix, fileController);
     }
   }
 
@@ -169,8 +150,7 @@ public LogAggregationFileController getFileControllerForRead(
             return fileController;
           }
         } catch (Exception ex) {
-          diagnosticsMsg.append(ex.getMessage() + "\n");
-          continue;
+          diagnosticsMsg.append(ex.getMessage()).append("\n");
         }
       }
     }
@@ -184,19 +164,26 @@ public LogAggregationFileController getFileControllerForRead(
           return fileController;
         }
       } catch (Exception ex) {
-        diagnosticsMsg.append(ex.getMessage() + "\n");
-        continue;
+        diagnosticsMsg.append(ex.getMessage()).append("\n");
       }
     }
 
     throw new IOException(diagnosticsMsg.toString());
   }
 
-  private boolean validateAggregatedFileControllerName(String name) {
+  private void validateAggregatedFileControllerName(String name) {
+    boolean valid;
     if (name == null || name.trim().isEmpty()) {
-      return false;
+      valid = false;
+    } else {
+      valid = p.matcher(name).matches();
     }
-    return p.matcher(name).matches();
+
+    Preconditions.checkArgument(valid,
+            String.format("The FileControllerName: %s set in " +
+                            "%s is invalid.The valid File Controller name should only contain " +
+                            "a-zA-Z0-9_ and cannot start with numbers", name,
+                    YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS));
   }
 
   @Private
@@ -205,4 +192,64 @@ private boolean validateAggregatedFileControllerName(String name) {
       getConfiguredLogAggregationFileControllerList() {
     return this.controllers;
   }
+
+  private static class DeterminedLogAggregationRemoteDir {
+    private String value;
+    private boolean usingDefault = false;
+    private final String configKey;
+
+    DeterminedLogAggregationRemoteDir(Configuration conf,
+        String fileController) {
+      configKey = String.format(
+          YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT,
+          fileController);
+      String remoteDir = conf.get(configKey);
+
+      if (remoteDir == null || remoteDir.isEmpty()) {
+        this.value = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR);
+        this.usingDefault = true;
+      } else {
+        this.value = remoteDir;
+      }
+    }
+  }
+
+  private static class DeterminedLogAggregationSuffix {
+    private String value;
+    private boolean usingDefault = false;
+    private final String configKey;
+
+    DeterminedLogAggregationSuffix(Configuration conf,
+        String fileController) {
+      configKey = String.format(
+          YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT,
+          fileController);
+      String suffix = conf.get(configKey);
+      if (suffix == null || suffix.isEmpty()) {
+        this.value = conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
+            YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX);
+        this.usingDefault = true;
+      } else {
+        this.value = suffix;
+      }
+    }
+  }
+
+  private static class DeterminedControllerClassName {
+    private final String configKey;
+    private final String value;
+
+    DeterminedControllerClassName(Configuration conf,
+        String fileController) {
+      this.configKey = String.format(
+          YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT,
+          fileController);
+      this.value = conf.get(configKey);
+      if (value == null || value.isEmpty()) {
+        throw new RuntimeException("No class configured for "
+            + fileController);
+      }
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
index 8390f99daed11..817f53eb1f977 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/IndexedFileAggregatedLogsBlock.java
@@ -23,7 +23,6 @@
 import com.google.inject.Inject;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -49,7 +48,6 @@
 import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController.IndexedLogsMeta;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController.IndexedPerAggregationLogMeta;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 
 /**
  * The Aggregated Logs Block implementation for Indexed File.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
index 6cd1f53d4ac8e..d4431d56b39a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/LogAggregationIndexedFileController.java
@@ -282,7 +282,8 @@ private Pair<Path, Boolean> initializeWriterInRolling(
           checksumFileInputStream = fc.open(remoteLogCheckSumFile);
           int nameLength = checksumFileInputStream.readInt();
           byte[] b = new byte[nameLength];
-          int actualLength = checksumFileInputStream.read(b);
+          checksumFileInputStream.readFully(b);
+          int actualLength = b.length;
           if (actualLength == nameLength) {
             String recoveredLogFile = new String(
                 b, Charset.forName("UTF-8"));
@@ -765,7 +766,8 @@ public Map<String, Long> parseCheckSumFiles(
         checksumFileInputStream = fc.open(file.getPath());
         int nameLength = checksumFileInputStream.readInt();
         byte[] b = new byte[nameLength];
-        int actualLength = checksumFileInputStream.read(b);
+        checksumFileInputStream.readFully(b);
+        int actualLength = b.length;
         if (actualLength == nameLength) {
           nodeName = new String(b, Charset.forName("UTF-8"));
           index = checksumFileInputStream.readLong();
@@ -799,7 +801,8 @@ private Long parseChecksum(FileStatus file) {
       checksumFileInputStream = fileContext.open(file.getPath());
       int nameLength = checksumFileInputStream.readInt();
       byte[] b = new byte[nameLength];
-      int actualLength = checksumFileInputStream.read(b);
+      checksumFileInputStream.readFully(b);
+      int actualLength = b.length;
       if (actualLength == nameLength) {
         nodeName = new String(b, StandardCharsets.UTF_8);
         index = checksumFileInputStream.readLong();
@@ -938,7 +941,8 @@ public IndexedLogsMeta loadIndexedLogsMeta(Path remoteLogPath, long end,
 
       // Load UUID and make sure the UUID is correct.
       byte[] uuidRead = new byte[UUID_LENGTH];
-      int uuidReadLen = fsDataIStream.read(uuidRead);
+      fsDataIStream.readFully(uuidRead);
+      int uuidReadLen = uuidRead.length;
       if (this.uuid == null) {
         this.uuid = createUUID(appId);
       }
@@ -1322,7 +1326,8 @@ private byte[] loadUUIDFromLogFile(final FileContext fc,
                 .endsWith(CHECK_SUM_FILE_SUFFIX)) {
           fsDataInputStream = fc.open(checkPath);
           byte[] b = new byte[uuid.length];
-          int actual = fsDataInputStream.read(b);
+          fsDataInputStream.readFully(b);
+          int actual = b.length;
           if (actual != uuid.length || Arrays.equals(b, uuid)) {
             deleteFileWithRetries(fc, checkPath);
           } else if (id == null){
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/package-info.java
index cad238a9a4205..1b53de6d1d65c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.logaggregation.filecontroller;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java
index 2c2604b1390a6..d7a44336ef97f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java
@@ -22,7 +22,6 @@
 
 import com.google.inject.Inject;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Map;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
@@ -38,8 +37,6 @@
 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationHtmlBlock;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.PRE;
 
 /**
  * The Aggregated Logs Block implementation for TFile.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/package-info.java
index 90dce80e63ca9..c1f1379ee80eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.logaggregation;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
index 5df20b1bf88d1..7497e43331579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/metrics/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,6 +18,6 @@
 /**
  *  Provides common metrics (available, allocated) for custom resources.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.metrics;
-import org.apache.hadoop.classification.InterfaceAudience;
\ No newline at end of file
+import org.apache.hadoop.classification.InterfaceAudience.Private;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
index d1d75cf1e9268..9e95a2e642663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
@@ -34,8 +34,8 @@ public interface AttributeValue {
    * validate the value based on the type and initialize for further compare
    * operations.
    *
-   * @param value
-   * @throws IOException
+   * @param value value.
+   * @throws IOException io error occur.
    */
   void validateAndInitializeValue(String value) throws IOException;
 
@@ -43,8 +43,8 @@ public interface AttributeValue {
    * compare the value against the other based on the
    * AttributeExpressionOperation.
    *
-   * @param other
-   * @param op
+   * @param other attribute value.
+   * @param op attribute expression operation.
    * @return true if value <code>other</code> matches the current value for the
    *         operation <code>op</code>.
    */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 7417b9a1c5ec2..a3dfcafba6dae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -337,10 +337,11 @@ public void addToCluserNodeLabels(Collection<NodeLabel> labels)
   }
 
   /**
-   * Add multiple node labels to repository
+   * Add multiple node labels to repository.
    *
    * @param labels
    *          new node labels added
+   * @throws IOException io error occur.
    */
   @VisibleForTesting
   public void addToCluserNodeLabelsWithDefaultExclusivity(Set<String> labels)
@@ -394,9 +395,10 @@ protected void checkAddLabelsToNode(
   }
   
   /**
-   * add more labels to nodes
+   * add more labels to nodes.
    * 
    * @param addedLabelsToNode node {@literal ->} labels map
+   * @throws IOException io error occur.
    */
   public void addLabelsToNode(Map<NodeId, Set<String>> addedLabelsToNode)
       throws IOException {
@@ -466,7 +468,7 @@ protected void internalRemoveFromClusterNodeLabels(Collection<String> labelsToRe
    * 
    * @param labelsToRemove
    *          node labels to remove
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public void removeFromClusterNodeLabels(Collection<String> labelsToRemove)
       throws IOException {
@@ -707,7 +709,7 @@ protected void internalUpdateLabelsOnNodes(
     
     if (null != dispatcher && isCentralizedNodeLabelConfiguration) {
       // In case of DistributedNodeLabelConfiguration or
-      // DelegatedCentralizedNodeLabelConfiguration, no need to save the the
+      // DelegatedCentralizedNodeLabelConfiguration, no need to save the
       // NodeLabels Mapping to the back-end store, as on RM restart/failover
       // NodeLabels are collected from NM through Register/Heartbeat again
       // in case of DistributedNodeLabelConfiguration and collected from
@@ -727,9 +729,10 @@ protected void internalUpdateLabelsOnNodes(
   
   /**
    * remove labels from nodes, labels being removed most be contained by these
-   * nodes
+   * nodes.
    * 
    * @param removeLabelsFromNode node {@literal ->} labels map
+   * @throws IOException io error occur.
    */
   public void
       removeLabelsFromNode(Map<NodeId, Set<String>> removeLabelsFromNode)
@@ -784,6 +787,7 @@ protected void checkReplaceLabelsOnNode(
    * replace labels to nodes
    * 
    * @param replaceLabelsToNode node {@literal ->} labels map
+   * @throws IOException io error occur.
    */
   public void replaceLabelsOnNode(Map<NodeId, Set<String>> replaceLabelsToNode)
       throws IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
index 8e9f9ff9f0ff8..f55c66abac3c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
@@ -35,7 +35,7 @@ public interface NodeAttributeStore extends Closeable {
    * Replace labels on node.
    *
    * @param nodeToAttribute node to attribute list.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   void replaceNodeAttributes(List<NodeToAttributes> nodeToAttribute)
       throws IOException;
@@ -44,7 +44,7 @@ void replaceNodeAttributes(List<NodeToAttributes> nodeToAttribute)
    * Add attribute to node.
    *
    * @param nodeToAttribute node to attribute list.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   void addNodeAttributes(List<NodeToAttributes> nodeToAttribute)
       throws IOException;
@@ -53,7 +53,7 @@ void addNodeAttributes(List<NodeToAttributes> nodeToAttribute)
    * Remove attribute from node.
    *
    * @param nodeToAttribute node to attribute list.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   void removeNodeAttributes(List<NodeToAttributes> nodeToAttribute)
       throws IOException;
@@ -62,16 +62,16 @@ void removeNodeAttributes(List<NodeToAttributes> nodeToAttribute)
    * Initialize based on configuration and NodeAttributesManager.
    *
    * @param configuration configuration instance.
-   * @param mgr nodeattributemanager instance.
-   * @throws Exception
+   * @param mgr node attribute manager instance.
+   * @throws Exception exception occurs.
    */
   void init(Configuration configuration, NodeAttributesManager mgr)
       throws Exception;
 
   /**
-   * Recover store on resourcemanager startup.
-   * @throws IOException
-   * @throws YarnException
+   * Recover store on resource manager startup.
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   void recover() throws IOException, YarnException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index a4c90a420a940..23675794ced0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -58,8 +58,8 @@ public abstract void replaceNodeAttributes(String prefix,
    * impacting other existing attribute mapping. Key would be name of the node
    * and value would be set of Attributes to be mapped.
    *
-   * @param nodeAttributeMapping
-   * @throws IOException
+   * @param nodeAttributeMapping host name to a set of node attributes mapping.
+   * @throws IOException io error occur.
    */
   public abstract void addNodeAttributes(
       Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
@@ -69,8 +69,8 @@ public abstract void addNodeAttributes(
    * impacting other existing attribute mapping. Key would be name of the node
    * and value would be set of Attributes to be removed.
    *
-   * @param nodeAttributeMapping
-   * @throws IOException
+   * @param nodeAttributeMapping host name to a set of node attributes mapping.
+   * @throws IOException io error occur.
    */
   public abstract void removeNodeAttributes(
       Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
@@ -93,6 +93,7 @@ public abstract Set<NodeAttribute> getClusterNodeAttributes(
    * If the attributeKeys set is null or empty, then mapping for all attributes
    * are returned.
    *
+   * @param attributes attributes set.
    * @return a Map of attributeKeys to a map of hostnames to its attribute
    *         values.
    */
@@ -103,6 +104,7 @@ Map<String, AttributeValue>> getAttributesToNodes(
   /**
    * NodeAttribute to AttributeValue Map.
    *
+   * @param hostName host name.
    * @return Map of NodeAttribute to AttributeValue.
    */
   public abstract Map<NodeAttribute, AttributeValue> getAttributesForNode(
@@ -111,6 +113,7 @@ public abstract Map<NodeAttribute, AttributeValue> getAttributesForNode(
   /**
    * Get All node to Attributes list based on filter.
    *
+   * @param prefix filter prefix set.
    * @return List of NodeToAttributes matching filter. If empty
    * or null is passed as argument will return all.
    */
@@ -120,6 +123,7 @@ public abstract List<NodeToAttributes> getNodeToAttributes(
   /**
    * Get all node to Attributes mapping.
    *
+   * @param hostNames host names.
    * @return Map of String to Set of nodesToAttributes matching
    * filter. If empty or null is passed as argument will return all.
    */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index 02f2188bfc8e1..f33a30e0f2b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -133,8 +133,8 @@ public static void checkAndThrowAttributePrefix(String prefix)
    *   <li>Missing prefix: the attribute doesn't have prefix defined</li>
    *   <li>Malformed attribute prefix: the prefix is not in valid format</li>
    * </ul>
-   * @param attributeSet
-   * @throws IOException
+   * @param attributeSet node attribute set.
+   * @throws IOException io error occur.
    */
   public static void validateNodeAttributes(Set<NodeAttribute> attributeSet)
       throws IOException {
@@ -179,6 +179,9 @@ public static Set<NodeAttribute> filterAttributesByPrefix(
 
   /**
    * Are these two input node attributes the same.
+   *
+   * @param leftNodeAttributes left node attribute.
+   * @param rightNodeAttributes right node attribute.
    * @return true if they are the same
    */
   public static boolean isNodeAttributesEquals(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
index e4efd68f92b9c..aac3a767d0777 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
@@ -37,18 +37,24 @@ public interface NodeLabelsStore extends Closeable {
 
   /**
    * Store node {@literal ->} label.
+   * @param nodeToLabels node to labels mapping.
+   * @throws IOException io error occur.
    */
   void updateNodeToLabelsMappings(
       Map<NodeId, Set<String>> nodeToLabels) throws IOException;
 
   /**
    * Store new labels.
+   * @param labels labels.
+   * @throws IOException io error occur.
    */
-  void storeNewClusterNodeLabels(List<NodeLabel> label)
+  void storeNewClusterNodeLabels(List<NodeLabel> labels)
       throws IOException;
 
   /**
    * Remove labels.
+   * @param labels labels.
+   * @throws IOException io error occur.
    */
   void removeClusterNodeLabels(Collection<String> labels)
       throws IOException;
@@ -60,8 +66,8 @@ void removeClusterNodeLabels(Collection<String> labels)
    * ignoreNodeToLabelsMappings will be set to true and recover will be invoked
    * as RM will collect the node labels from NM through registration/HB.
    *
-   * @throws IOException
-   * @throws YarnException
+   * @throws IOException io error occur.
+   * @throws YarnException exceptions from yarn servers.
    */
   void recover() throws IOException, YarnException;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
index 59a1860e31533..15d4efc03e6df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
@@ -100,7 +100,7 @@ private static void registerLog(StoreType type, int opcode,
   /**
    * Get mirror operation of store Type.
    *
-   * @param storeType
+   * @param storeType storeType.
    * @return instance of FSNodeStoreLogOp.
    */
   public static FSNodeStoreLogOp getMirrorOp(StoreType storeType) {
@@ -108,9 +108,9 @@ public static FSNodeStoreLogOp getMirrorOp(StoreType storeType) {
   }
 
   /**
-   * Will return StoreOp instance basead on opCode and StoreType.
-   * @param opCode
-   * @param storeType
+   * Will return StoreOp instance based on opCode and StoreType.
+   * @param opCode opCode.
+   * @param storeType storeType.
    * @return instance of FSNodeStoreLogOp.
    */
   public static FSNodeStoreLogOp get(int opCode, StoreType storeType) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
index e0b26da82e76f..44c2225559981 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
@@ -34,7 +34,7 @@ public interface StoreOp<W, R, M> {
    *
    * @param write write to be done to
    * @param mgr manager used by store
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   void write(W write, M mgr) throws IOException;
 
@@ -43,7 +43,7 @@ public interface StoreOp<W, R, M> {
    *
    * @param read read to be done from
    * @param mgr  manager used by store
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   void recover(R read, M mgr) throws IOException;
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
index 95004866458ab..64400fe6e6cba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
@@ -27,6 +27,8 @@
     .AddToClusterNodeLabelsRequestPBImpl;
 import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
     .ReplaceLabelsOnNodeRequestPBImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -40,6 +42,8 @@
  */
 public class NodeLabelMirrorOp
     extends FSNodeStoreLogOp<CommonNodeLabelsManager> {
+  private static final Logger LOG = LoggerFactory.getLogger(
+          NodeLabelMirrorOp.class);
 
   public NodeLabelMirrorOp() {
     super();
@@ -66,15 +70,20 @@ public void recover(InputStream is, CommonNodeLabelsManager mgr)
             .parseDelimitedFrom(is)).getNodeLabels();
     mgr.addToCluserNodeLabels(labels);
 
-    if (mgr.isCentralizedConfiguration()) {
-      // Only load node to labels mapping while using centralized
-      // configuration
-      Map<NodeId, Set<String>> nodeToLabels =
-          new ReplaceLabelsOnNodeRequestPBImpl(
-              YarnServerResourceManagerServiceProtos
-                  .ReplaceLabelsOnNodeRequestProto
-                  .parseDelimitedFrom(is)).getNodeToLabels();
-      mgr.replaceLabelsOnNode(nodeToLabels);
+    try {
+      if (mgr.isCentralizedConfiguration() && is.available() != 0) {
+        // Only load node to labels mapping while using centralized
+        // configuration
+        Map<NodeId, Set<String>> nodeToLabels =
+                new ReplaceLabelsOnNodeRequestPBImpl(
+                        YarnServerResourceManagerServiceProtos
+                                .ReplaceLabelsOnNodeRequestProto
+                                .parseDelimitedFrom(is)).getNodeToLabels();
+        mgr.replaceLabelsOnNode(nodeToLabels);
+      }
+    } catch (Exception e) {
+      LOG.error("Errors on loading node to labels mapping while using "
+              + "centralized configuration", e);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/package-info.java
index f6fb3d3ecaa74..3175b2c57888e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.nodelabels.store.op;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
index 0444807071ab5..d1546cf2b68ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.nodelabels.store;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/package-info.java
index 99ab44ce42d29..69aeda005df1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
index 615ecb0106e08..227f1c960a435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
@@ -43,7 +43,7 @@ public class ConfiguredYarnAuthorizer extends YarnAuthorizationProvider {
   private final ConcurrentMap<PrivilegedEntity, Map<AccessType, AccessControlList>>
       allAcls = new ConcurrentHashMap<>();
   private volatile AccessControlList adminAcl = null;
-  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();;
+  private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
   private final ReentrantReadWriteLock.ReadLock readLock = lock.readLock();
   private final ReentrantReadWriteLock.WriteLock writeLock =  lock.writeLock();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index f425e8dfe40f3..b29e7e6fe08fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -418,7 +418,8 @@ public int getVersion() {
     }
   }
   /**
-   * Get the node-label-expression in the original ResourceRequest
+   * Get the node-label-expression in the original ResourceRequest.
+   * @return node label expression.
    */
   public String getNodeLabelExpression() {
     if (proto.hasNodeLabelExpression()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
index 580bdf490a3f2..3539c8f0bb595 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
@@ -45,6 +45,11 @@ public PrivilegedEntity(EntityType type, String name) {
     this.name = name;
   }
 
+  public PrivilegedEntity(String name) {
+    this.type = EntityType.QUEUE;
+    this.name = name;
+  }
+
   public EntityType getType() {
     return type;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
index c6cabdbff74c4..d69f9abfc5ea7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
@@ -79,6 +79,7 @@ public static void destroy() {
   /**
    * Initialize the provider. Invoked on daemon startup. DefaultYarnAuthorizer is
    * initialized based on configurations.
+   * @param conf configuration.
    */
   public abstract void init(Configuration conf);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
index 6d8bc4bc1da88..95fe4bbc64e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java
@@ -24,9 +24,11 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
+import org.apache.hadoop.yarn.util.Records;
 
 @Private
 public abstract class YARNDelegationTokenIdentifier extends
@@ -112,4 +114,14 @@ public YARNDelegationTokenIdentifierProto getProto() {
     setBuilderFields();
     return builder.build();
   }
+
+  @Private
+  @Unstable
+  public static YARNDelegationTokenIdentifier newInstance(Text owner, Text renewer, Text realUser) {
+    YARNDelegationTokenIdentifier policy = Records.newRecord(YARNDelegationTokenIdentifier.class);
+    policy.setOwner(owner);
+    policy.setRenewer(renewer);
+    policy.setRenewer(realUser);
+    return policy;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java
new file mode 100644
index 0000000000000..f977a3391fe2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.security.client.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProtoOrBuilder;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+
+@Private
+@Unstable
+public class YARNDelegationTokenIdentifierPBImpl extends YARNDelegationTokenIdentifier {
+
+  private YARNDelegationTokenIdentifierProto proto =
+      YARNDelegationTokenIdentifierProto.getDefaultInstance();
+  private YARNDelegationTokenIdentifierProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public YARNDelegationTokenIdentifierPBImpl() {
+    builder = YARNDelegationTokenIdentifierProto.newBuilder();
+  }
+
+  public YARNDelegationTokenIdentifierPBImpl(YARNDelegationTokenIdentifierProto identifierProto) {
+    this.proto = identifierProto;
+    viaProto = true;
+  }
+
+  public YARNDelegationTokenIdentifierProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    // mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      if (proto == null) {
+        proto = YARNDelegationTokenIdentifierProto.getDefaultInstance();
+      }
+      builder = YARNDelegationTokenIdentifierProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public Text getOwner() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return new Text(p.getOwner());
+  }
+
+  @Override
+  public void setOwner(Text owner) {
+    super.setOwner(owner);
+    maybeInitBuilder();
+    if (owner == null) {
+      builder.clearOwner();
+      return;
+    }
+    builder.setOwner(owner.toString());
+  }
+
+  @Override
+  public Text getRenewer() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return new Text(p.getRenewer());
+  }
+
+  @Override
+  public void setRenewer(Text renewer) {
+    super.setRenewer(renewer);
+    maybeInitBuilder();
+    if (renewer == null) {
+      builder.clearRenewer();
+      return;
+    }
+    builder.setOwner(renewer.toString());
+  }
+
+  @Override
+  public Text getRealUser() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return new Text(p.getRealUser());
+  }
+
+  @Override
+  public void setRealUser(Text realUser) {
+    super.setRealUser(realUser);
+    maybeInitBuilder();
+    if (realUser == null) {
+      builder.clearRealUser();
+      return;
+    }
+    builder.setRealUser(realUser.toString());
+  }
+
+  @Override
+  public void setIssueDate(long issueDate) {
+    super.setIssueDate(issueDate);
+    maybeInitBuilder();
+    builder.setIssueDate(issueDate);
+  }
+
+  @Override
+  public long getIssueDate() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getIssueDate();
+  }
+
+  @Override
+  public void setMaxDate(long maxDate) {
+    super.setMaxDate(maxDate);
+    maybeInitBuilder();
+    builder.setMaxDate(maxDate);
+  }
+
+  @Override
+  public long getMaxDate() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getMaxDate();
+  }
+
+  @Override
+  public void setSequenceNumber(int seqNum) {
+    super.setSequenceNumber(seqNum);
+    maybeInitBuilder();
+    builder.setSequenceNumber(seqNum);
+  }
+
+  @Override
+  public int getSequenceNumber() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getSequenceNumber();
+  }
+
+  @Override
+  public void setMasterKeyId(int newId) {
+    super.setMasterKeyId(newId);
+    maybeInitBuilder();
+    builder.setMasterKeyId(newId);
+  }
+
+  @Override
+  public int getMasterKeyId() {
+    YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getMasterKeyId();
+  }
+
+  @Override
+  public Text getKind() {
+    return null;
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java
new file mode 100644
index 0000000000000..cb6b1be8807c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@Public
+package org.apache.hadoop.yarn.security.client.impl.pb;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/package-info.java
index 7aa12fd75fd8a..e44f8a270c0ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.security.client;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/package-info.java
index f09a1f32011bd..38a174db369cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.security;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
index d44599664a28b..0e05e731ad263 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/package-info.java
index e9c394bbd78c4..72e1d777658cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java
index b7ef6fb924fee..c293580f1968e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 /** Yarn Common Metrics package. **/
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.server.metrics;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index ddbcdae07e512..af7867ca8981d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -27,7 +27,6 @@
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -89,10 +88,11 @@ public void removeApplication(ApplicationId appId) {
    * <li>For all other users/groups application-acls are checked</li>
    * </ul>
    * 
-   * @param callerUGI
-   * @param applicationAccessType
-   * @param applicationOwner
-   * @param applicationId
+   * @param callerUGI UserGroupInformation for the user.
+   * @param applicationAccessType Application Access Type.
+   * @param applicationOwner Application Owner.
+   * @param applicationId ApplicationId.
+   * @return true if the user has permission, false otherwise.
    */
   public boolean checkAccess(UserGroupInformation callerUGI,
       ApplicationAccessType applicationAccessType, String applicationOwner,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/package-info.java
index 8aa55050c4d91..4fc7a7df40da3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Private
+@Private
 package org.apache.hadoop.yarn.server.security;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
index 7e6fddaa2635d..ca442873c44b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
@@ -37,7 +37,7 @@ public interface SharedCacheChecksum {
    *
    * @param in <code>InputStream</code> to be checksumed
    * @return the message digest of the input stream
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public String computeChecksum(InputStream in) throws IOException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
index cbfd95db5b91b..4397634c8f735 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
@@ -58,8 +58,9 @@ public class SharedCacheChecksumFactory {
   /**
    * Get a new <code>SharedCacheChecksum</code> object based on the configurable
    * algorithm implementation
-   * (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+   * (see <code>yarn.sharedcache.checksum.algo.impl</code>).
    *
+   * @param conf configuration.
    * @return <code>SharedCacheChecksum</code> object
    */
   public static SharedCacheChecksum getChecksum(Configuration conf) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
index 4bb005c053618..e7660c14b7322 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
@@ -60,7 +60,7 @@ final public class StateMachineFactory
    * Constructor
    *
    * This is the only constructor in the API.
-   *
+   * @param defaultInitialState default initial state.
    */
   public StateMachineFactory(STATE defaultInitialState) {
     this.transitionsListNode = null;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
index 16203976e3375..fa3831d341e84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/VisualizeStateMachine.java
@@ -27,9 +27,13 @@
 public class VisualizeStateMachine {
 
   /**
+   * get Graph From Classes.
+   *
+   * @param graphName graphName.
    * @param classes list of classes which have static field
    *                stateMachineFactory of type StateMachineFactory
    * @return graph represent this StateMachine
+   * @throws Exception exception occurs.
    */
   public static Graph getGraphFromClasses(String graphName, List<String> classes)
       throws Exception {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/package-info.java
index c62c5015e938e..4cdee806bbd6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.state;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index a53ae039781cb..3075ac626ab1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -229,6 +229,8 @@ public static Set<String> getEnvVarsFromInputProperty(
    * This older version of this method is kept around for compatibility
    * because downstream frameworks like Spark and Tez have been using it.
    * Downstream frameworks are expected to move off of it.
+   * @param env the environment to update.
+   * @param envString String containing env variable definitions.
    */
   @Deprecated
   public static void setEnvFromInputString(Map<String, String> env,
@@ -255,6 +257,10 @@ public static void addToEnvironment(
    * This older version of this method is kept around for compatibility
    * because downstream frameworks like Spark and Tez have been using it.
    * Downstream frameworks are expected to move off of it.
+   *
+   * @param environment map of environment variable.
+   * @param variable variable.
+   * @param value value.
    */
   @Deprecated
   public static void addToEnvironment(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
index db7103e1d6e5b..6d582ca1ec799 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.util;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index 67bc2b74fd14a..d4647a14a27da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -57,7 +57,8 @@ public class ConverterUtils {
    * @param url
    *          url to convert
    * @return path from {@link URL}
-   * @throws URISyntaxException
+   * @throws URISyntaxException exception thrown to indicate that a string could not be parsed as a
+   * URI reference.
    */
   @Public
   @Deprecated
@@ -171,6 +172,7 @@ public static ApplicationId toApplicationId(
    *
    * @param protoToken the yarn token
    * @param serviceAddr the connect address for the service
+   * @param <T> Generic Type T.
    * @return rpc token
    */
   public static <T extends TokenIdentifier> Token<T> convertFromYarn(
@@ -191,6 +193,8 @@ public static <T extends TokenIdentifier> Token<T> convertFromYarn(
    *
    * @param protoToken the yarn token
    * @param service the service for the token
+   * @param <T> Generic Type T.
+   * @return rpc token
    */
   public static <T extends TokenIdentifier> Token<T> convertFromYarn(
       org.apache.hadoop.yarn.api.records.Token protoToken,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
index c996225c9a75e..b1a186e8a8767 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
@@ -28,11 +28,12 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.security.DockerCredentialTokenIdentifier;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ObjectNode;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
@@ -97,13 +98,13 @@ public static Credentials readCredentialsFromConfigFile(Path configFile,
 
     // Parse the JSON and create the Tokens/Credentials.
     ObjectMapper mapper = new ObjectMapper();
-    JsonFactory factory = mapper.getJsonFactory();
-    JsonParser parser = factory.createJsonParser(contents);
+    JsonFactory factory = mapper.getFactory();
+    JsonParser parser = factory.createParser(contents);
     JsonNode rootNode = mapper.readTree(parser);
 
     Credentials credentials = new Credentials();
     if (rootNode.has(CONFIG_AUTHS_KEY)) {
-      Iterator<String> iter = rootNode.get(CONFIG_AUTHS_KEY).getFieldNames();
+      Iterator<String> iter = rootNode.get(CONFIG_AUTHS_KEY).fieldNames();
       for (; iter.hasNext();) {
         String registryUrl = iter.next();
         String registryCred = rootNode.get(CONFIG_AUTHS_KEY)
@@ -130,6 +131,7 @@ public static Credentials readCredentialsFromConfigFile(Path configFile,
    *
    * @param tokens the Tokens from the ContainerLaunchContext.
    * @return the Credentials object populated from the Tokens.
+   * @throws IOException io error occur.
    */
   public static Credentials getCredentialsFromTokensByteBuffer(
       ByteBuffer tokens) throws IOException {
@@ -169,14 +171,14 @@ public static boolean writeDockerCredentialsToPath(File outConfigFile,
           DockerCredentialTokenIdentifier ti =
               (DockerCredentialTokenIdentifier) tk.decodeIdentifier();
           ObjectNode registryCredNode = mapper.createObjectNode();
-          registryUrlNode.put(ti.getRegistryUrl(), registryCredNode);
+          registryUrlNode.set(ti.getRegistryUrl(), registryCredNode);
           registryCredNode.put(CONFIG_AUTH_KEY,
               new String(tk.getPassword(), Charset.forName("UTF-8")));
           LOG.debug("Prepared token for write: {}", tk);
         }
       }
       if (foundDockerCred) {
-        rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode);
+        rootNode.set(CONFIG_AUTHS_KEY, registryUrlNode);
         String json = mapper.writerWithDefaultPrettyPrinter()
             .writeValueAsString(rootNode);
         FileUtils.writeStringToFile(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index e24b96cd32f78..fe4a844619273 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -60,7 +60,11 @@
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 
-/**
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
+ /**
  * Download a single URL to the local disk.
  *
  */
@@ -119,10 +123,13 @@ private void createDir(Path path, FsPermission perm) throws IOException {
    * Creates the cache loader for the status loading cache. This should be used
    * to create an instance of the status cache that is passed into the
    * FSDownload constructor.
+   *
+   * @param conf configuration.
+   * @return cache loader for the status loading cache.
    */
-  public static CacheLoader<Path,Future<FileStatus>>
+  public static CacheLoader<Path, Future<FileStatus>>
       createStatusCacheLoader(final Configuration conf) {
-    return new CacheLoader<Path,Future<FileStatus>>() {
+    return new CacheLoader<Path, Future<FileStatus>>() {
       public Future<FileStatus> load(Path path) {
         try {
           FileSystem fs = path.getFileSystem(conf);
@@ -137,14 +144,19 @@ public Future<FileStatus> load(Path path) {
 
   /**
    * Returns a boolean to denote whether a cache file is visible to all (public)
-   * or not
+   * or not.
    *
+   * @param fs fileSystem.
+   * @param current current path.
+   * @param sStat file status.
+   * @param statCache stat cache.
    * @return true if the path in the current path is visible to all, false
    * otherwise
+   * @throws IOException io error occur.
    */
   @Private
   public static boolean isPublic(FileSystem fs, Path current, FileStatus sStat,
-      LoadingCache<Path,Future<FileStatus>> statCache) throws IOException {
+      LoadingCache<Path, Future<FileStatus>> statCache) throws IOException {
     current = fs.makeQualified(current);
     //the leaf level file should be readable by others
     if (!checkPublicPermsForAll(fs, sStat, FsAction.READ_EXECUTE, FsAction.READ)) {
@@ -285,23 +297,25 @@ private void verifyAndCopy(Path destination)
       }
     }
 
-    downloadAndUnpack(sCopy, destination);
+    downloadAndUnpack(sCopy, sStat, destination);
   }
 
   /**
    * Copy source path to destination with localization rules.
-   * @param source source path to copy. Typically HDFS
+   * @param source source path to copy. Typically HDFS or an object store.
+   * @param sourceStatus status of source
    * @param destination destination path. Typically local filesystem
    * @exception YarnException Any error has occurred
    */
-  private void downloadAndUnpack(Path source, Path destination)
+  private void downloadAndUnpack(Path source,
+      FileStatus sourceStatus,  Path destination)
       throws YarnException {
     try {
       FileSystem sourceFileSystem = source.getFileSystem(conf);
       FileSystem destinationFileSystem = destination.getFileSystem(conf);
-      if (sourceFileSystem.getFileStatus(source).isDirectory()) {
+      if (sourceStatus.isDirectory()) {
         FileUtil.copy(
-            sourceFileSystem, source,
+            sourceFileSystem, sourceStatus,
             destinationFileSystem, destination, false,
             true, conf);
       } else {
@@ -329,7 +343,11 @@ private void unpack(Path source, Path destination,
                       FileSystem sourceFileSystem,
                       FileSystem destinationFileSystem)
       throws IOException, InterruptedException, ExecutionException {
-    try (InputStream inputStream = sourceFileSystem.open(source)) {
+    try (InputStream inputStream = awaitFuture(
+        sourceFileSystem.openFile(source)
+            .opt(FS_OPTION_OPENFILE_READ_POLICY,
+                FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+            .build())) {
       File dst = new File(destination.toUri());
       String lowerDst = StringUtils.toLowerCase(dst.getName());
       switch (resource.getType()) {
@@ -445,7 +463,7 @@ public Void run() throws Exception {
    * Change to 755 or 700 for dirs, 555 or 500 for files.
    * @param fs FileSystem
    * @param path Path to modify perms for
-   * @throws IOException
+   * @throws IOException io error occur.
    * @throws InterruptedException 
    */
   private void changePermissions(FileSystem fs, final Path path)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 470594d853bec..5a518dff7e537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -579,6 +579,9 @@ public String toString() {
 /**
  * Returns boolean indicating whether pid
  * is in process tree.
+ *
+ * @param pid pid.
+ * @return if true, processTree contains pid, false, processTree does not contain pid.
  */
   public boolean contains(String pid) {
     return processTree.containsKey(pid);
@@ -1000,9 +1003,9 @@ public String toString() {
   }
 
   /**
-   * Test the {@link ProcfsBasedProcessTree}
+   * Test the {@link ProcfsBasedProcessTree}.
    *
-   * @param args
+   * @param args the pid arg.
    */
   public static void main(String[] args) {
     if (args.length != 1) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java
index 65f0fa62cd827..1ba18d212d416 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java
@@ -79,8 +79,8 @@ public synchronized static void init(Configuration conf) {
    * Utility method for getting a hostname resolved to a node in the
    * network topology. This method initializes the class with the
    * right resolver implementation.
-   * @param conf
-   * @param hostName
+   * @param conf configuration.
+   * @param hostName hostname.
    * @return node {@link Node} after resolving the hostname
    */
   public static Node resolve(Configuration conf, String hostName) {
@@ -92,8 +92,8 @@ public static Node resolve(Configuration conf, String hostName) {
    * Utility method for getting a list of hostname resolved to a list of node
    *  in the network topology. This method initializes the class with the
    * right resolver implementation.
-   * @param conf
-   * @param hostNames
+   * @param conf configuration.
+   * @param hostNames list of hostName.
    * @return nodes {@link Node} after resolving the hostnames
    */
   public static List<Node> resolve(
@@ -106,7 +106,7 @@ public static List<Node> resolve(
    * Utility method for getting a hostname resolved to a node in the
    * network topology. This method doesn't initialize the class.
    * Call {@link #init(Configuration)} explicitly.
-   * @param hostName
+   * @param hostName host name.
    * @return node {@link Node} after resolving the hostname
    */
   public static Node resolve(String hostName) {
@@ -120,7 +120,7 @@ public static Node resolve(String hostName) {
    * Utility method for getting a list of hostname resolved to a list of node
    *  in the network topology. This method doesn't initialize the class.
    * Call {@link #init(Configuration)} explicitly.
-   * @param hostNames
+   * @param hostNames list of hostNames.
    * @return nodes {@link Node} after resolving the hostnames
    */
   public static List<Node> resolve(List<String> hostNames) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
index 9bbec5f9d7be3..7209f9264747e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
@@ -23,7 +23,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
index 54ba886eaed01..462ae634912f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
@@ -67,7 +67,7 @@ public static String cjoin(Object... args) {
   }
 
   /**
-   * Join on dot
+   * Join on dot.
    * @param args to join
    * @return args joined by dot
    */
@@ -76,7 +76,7 @@ public static String djoin(Object... args) {
   }
 
   /**
-   * Join on underscore
+   * Join on underscore.
    * @param args to join
    * @return args joined underscore
    */
@@ -85,7 +85,7 @@ public static String _join(Object... args) {
   }
 
   /**
-   * Join on slash
+   * Join on slash.
    * @param args to join
    * @return args joined with slash
    */
@@ -103,8 +103,8 @@ public static String pajoin(Object... args) {
   }
 
   /**
-   * Join without separator
-   * @param args
+   * Join without separator.
+   * @param args to join.
    * @return joined args with no separator
    */
   public static String join(Object... args) {
@@ -131,7 +131,7 @@ public static Iterable<String> split(CharSequence s) {
   }
 
   /**
-   * Split on _ and trim results
+   * Split on _ and trim results.
    * @param s the string to split
    * @return an iterable of strings
    */
@@ -140,7 +140,7 @@ public static Iterable<String> _split(CharSequence s) {
   }
 
   /**
-   * Check whether a url is absolute or note
+   * Check whether a url is absolute or note.
    * @param url to check
    * @return true if url starts with scheme:// or //
    */
@@ -149,7 +149,7 @@ public static boolean isAbsUrl(CharSequence url) {
   }
 
   /**
-   * Join url components
+   * Join url components.
    * @param pathPrefix for relative urls
    * @param args url components to join
    * @return an url string
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java
index b6604eee12581..27550263983f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java
@@ -19,8 +19,9 @@
 package org.apache.hadoop.yarn.util;
 
 import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,23 +32,16 @@ public class Times {
   private static final Logger LOG =
       LoggerFactory.getLogger(Times.class);
 
-  static final String ISO8601DATEFORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
+  static final String ISO8601_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
 
   // This format should match the one used in yarn.dt.plugins.js
-  static final ThreadLocal<SimpleDateFormat> dateFormat =
-      new ThreadLocal<SimpleDateFormat>() {
-        @Override protected SimpleDateFormat initialValue() {
-          return new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy");
-        }
-      };
+  static final DateTimeFormatter DATE_FORMAT =
+      DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss Z yyyy").withZone(
+          ZoneId.systemDefault());
 
-  static final ThreadLocal<SimpleDateFormat> isoFormat =
-      new ThreadLocal<SimpleDateFormat>() {
-        @Override
-        protected SimpleDateFormat initialValue() {
-          return new SimpleDateFormat(ISO8601DATEFORMAT);
-        }
-      };
+  static final DateTimeFormatter ISO_OFFSET_DATE_TIME =
+      DateTimeFormatter.ofPattern(ISO8601_DATE_FORMAT).withZone(
+          ZoneId.systemDefault());
 
   public static long elapsed(long started, long finished) {
     return Times.elapsed(started, finished, true);
@@ -83,8 +77,7 @@ public static long elapsed(long started, long finished, boolean isRunning) {
   }
 
   public static String format(long ts) {
-    return ts > 0 ? String.valueOf(dateFormat.get().format(new Date(ts)))
-                  : "N/A";
+    return ts > 0 ? DATE_FORMAT.format(Instant.ofEpochMilli(ts)) : "N/A";
   }
 
   /**
@@ -94,7 +87,7 @@ public static String format(long ts) {
    * @return ISO 8601 formatted string.
    */
   public static String formatISO8601(long ts) {
-    return isoFormat.get().format(new Date(ts));
+    return ISO_OFFSET_DATE_TIME.format(Instant.ofEpochMilli(ts));
   }
 
   /**
@@ -109,6 +102,6 @@ public static long parseISO8601ToLocalTimeInMillis(String isoString)
     if (isoString == null) {
       throw new ParseException("Invalid input.", -1);
     }
-    return isoFormat.get().parse(isoString).getTime();
+    return Instant.from(ISO_OFFSET_DATE_TIME.parse(isoString)).toEpochMilli();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
index d29e52fee8000..1199a5de9f31f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
@@ -38,7 +38,8 @@ public abstract class TrackingUriPlugin extends Configured {
    * Given an application ID, return a tracking URI.
    * @param id the ID for which a URI is returned
    * @return the tracking URI
-   * @throws URISyntaxException
+   * @throws URISyntaxException exception thrown to indicate that a string could not be parsed as a
+   * URI reference.
    */
   public abstract URI getTrackingUri(ApplicationId id)
       throws URISyntaxException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
index 18654920216e6..c2dccb3dc5796 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
@@ -80,6 +80,8 @@ public static String getUser() {
   
   /**
    * Get the subversion URL for the root YARN directory.
+   *
+   * @return URL for the root YARN directory.
    */
   public static String getUrl() {
     return YARN_VERSION_INFO._getUrl();
@@ -88,14 +90,18 @@ public static String getUrl() {
   /**
    * Get the checksum of the source files from which YARN was
    * built.
-   **/
+   *
+   * @return srcChecksum.
+   */
   public static String getSrcChecksum() {
     return YARN_VERSION_INFO._getSrcChecksum();
   }
 
   /**
    * Returns the buildVersion which includes version, 
-   * revision, user and date. 
+   * revision, user and date.
+   *
+   * @return buildVersion.
    */
   public static String getBuildVersion(){
     return YARN_VERSION_INFO._getBuildVersion();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/package-info.java
index 8a22bfc1c5bed..0e9d95b9ff58e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.util;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 135f3486cebae..8fe9c7931a289 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -404,18 +404,51 @@ public boolean isAllInvalidDivisor(Resource r) {
 
   @Override
   public float ratio(Resource a, Resource b) {
-    float ratio = 0.0f;
+    return ratio(a, b, true);
+  }
+
+  /**
+   * Computes the ratio of resource a over resource b,
+   * where the boolean flag {@literal isDominantShare} allows
+   * specification of whether the max- or min-share should be computed.
+   * @param a the numerator resource.
+   * @param b the denominator resource.
+   * @param isDominantShare whether the dominant (max) share should be computed,
+   *                        computes the min-share if false.
+   * @return the max- or min-share ratio of the resources.
+   */
+  private float ratio(Resource a, Resource b, boolean isDominantShare) {
+    float ratio = isDominantShare ? 0.0f : 1.0f;
     int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
     for (int i = 0; i < maxLength; i++) {
       ResourceInformation aResourceInformation = a.getResourceInformation(i);
       ResourceInformation bResourceInformation = b.getResourceInformation(i);
       final float tmp = divideSafelyAsFloat(aResourceInformation.getValue(),
           bResourceInformation.getValue());
-      ratio = ratio > tmp ? ratio : tmp;
+      if (isDominantShare) {
+        ratio = Math.max(ratio, tmp);
+      } else {
+        ratio = Math.min(ratio, tmp);
+      }
     }
     return ratio;
   }
 
+  /**
+   * Computes the ratio of resource a over resource b.
+   * However, different from ratio(Resource, Resource),
+   * this returns the min-share of the resources.
+   * For example, ratio(Resource(10, 50), Resource(100, 100)) would return 0.5,
+   * whereas minRatio(Resource(10, 50), Resource(100, 100)) would return 0.1.
+   * @param a the numerator resource.
+   * @param b the denominator resource.
+   * @return the min-share ratio of the resources.
+   */
+  @Unstable
+  public float minRatio(Resource a, Resource b) {
+    return ratio(a, b, false);
+  }
+
   @Override
   public Resource divideAndCeil(Resource numerator, int denominator) {
     return divideAndCeil(numerator, (long) denominator);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 05850137c747e..089d19339786e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -91,13 +91,13 @@ public static long divideAndCeil(long a, float b) {
 
   /**
    * Divides lhs by rhs.
-   * If both lhs and rhs are having a value of 0, then we return 0.
+   *
+   * @param lhs left number.
+   * @param rhs right number.
+   * @return If both lhs and rhs are having a value of 0, then we return 0.
    * This is to avoid division by zero and return NaN as a result.
    * If lhs is zero but rhs is not, Float.infinity will be returned
    * as the result.
-   * @param lhs
-   * @param rhs
-   * @return
    */
   public static float divideSafelyAsFloat(long lhs, long rhs) {
     if (lhs == 0 && rhs == 0) {
@@ -263,6 +263,9 @@ public abstract float divide(
   
   /**
    * Check if a smaller resource can be contained by bigger resource.
+   * @param smaller smaller resource.
+   * @param bigger bigger resource.
+   * @return if true, smaller resource can be contained by bigger resource; false otherwise.
    */
   public abstract boolean fitsIn(Resource smaller, Resource bigger);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 9b96fd72b9a99..c39490909c4e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -20,7 +20,7 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -31,7 +31,7 @@
  * Resources is a computation class which provides a set of apis to do
  * mathematical operations on Resource object.
  */
-@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
+@LimitedPrivate({ "YARN", "MapReduce" })
 @Unstable
 public class Resources {
 
@@ -316,7 +316,11 @@ public static Resource multiply(Resource lhs, double by) {
 
   /**
    * Multiply {@code rhs} by {@code by}, and add the result to {@code lhs}
-   * without creating any new {@link Resource} object
+   * without creating any new {@link Resource} object.
+   * @param lhs {@link Resource} to subtract from.
+   * @param rhs {@link Resource} to subtract.
+   * @param by multiplier.
+   * @return instance of Resource.
    */
   public static Resource multiplyAndAddTo(
       Resource lhs, Resource rhs, double by) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index 46b3741271679..14b9b0ceb7d12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -66,9 +66,9 @@ public class TimelineUtils {
    * @param o
    *          an object to serialize
    * @return a JSON string
-   * @throws IOException
-   * @throws JsonMappingException
-   * @throws JsonGenerationException
+   * @throws IOException io error occur.
+   * @throws JsonMappingException exception used to signal fatal problems with mapping of content.
+   * @throws JsonGenerationException exception type for exceptions during JSON writing.
    */
   public static String dumpTimelineRecordtoJSON(Object o)
       throws JsonGenerationException, JsonMappingException, IOException {
@@ -83,9 +83,9 @@ public static String dumpTimelineRecordtoJSON(Object o)
    * @param pretty
    *          whether in a pretty format or not
    * @return a JSON string
-   * @throws IOException
-   * @throws JsonMappingException
-   * @throws JsonGenerationException
+   * @throws IOException io error occur.
+   * @throws JsonMappingException exception used to signal fatal problems with mapping of content.
+   * @throws JsonGenerationException exception type for exceptions during JSON writing.
    */
   public static String dumpTimelineRecordtoJSON(Object o, boolean pretty)
       throws JsonGenerationException, JsonMappingException, IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/package-info.java
index 5c18a55518d43..98fe0783694cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.Public
+@Public
 package org.apache.hadoop.yarn.util.timeline;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
index d0503876b879a..a859ffbc1f235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.util.Preconditions.checkState;
 
 import java.io.IOException;
 import java.util.Timer;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
index 2bad02c9f088e..b8fc9e00541d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
@@ -93,8 +93,8 @@ public Response toResponse(Exception e) {
         && e.getCause() instanceof UnmarshalException) {
       s = Response.Status.BAD_REQUEST;
     } else {
-      LOG.warn("INTERNAL_SERVER_ERROR", e);
-      s = Response.Status.INTERNAL_SERVER_ERROR;
+      LOG.warn("SERVICE_UNAVAILABLE", e);
+      s = Response.Status.SERVICE_UNAVAILABLE;
     }
 
     // let jaxb handle marshalling data out in the same format requested
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
index 5e7aa68b948c9..45e42ea3a5d86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkState;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkState;
 import static org.apache.hadoop.yarn.util.StringHelper.djoin;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
index 54adb8208c095..6ef1c50cc6df5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
@@ -28,6 +28,7 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.util.Lists;
@@ -299,4 +300,8 @@ static String getPrefix(String pathSpec) {
 
   public abstract void setup();
 
+  @VisibleForTesting
+  public HttpServer2 getHttpServer() {
+    return httpServer;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java
index dbede94f29589..10d947392192f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebAppException.java
@@ -25,6 +25,11 @@
 public class WebAppException extends YarnRuntimeException {
 
   private static final long serialVersionUID = 1L;
+  private WebApp webApp;
+
+  public WebApp getWebApp() {
+    return webApp;
+  }
 
   public WebAppException(String msg) {
     super(msg);
@@ -35,6 +40,11 @@ public WebAppException(Throwable cause) {
   }
 
   public WebAppException(String msg, Throwable cause) {
+    this(msg, cause, null);
+  }
+
+  public WebAppException(String msg, Throwable cause, WebApp webApp) {
     super(msg, cause);
+    this.webApp = webApp;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index a088f4bf39f12..9fef076196e4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.util.Preconditions.checkNotNull;
 
 import java.io.IOException;
 import java.net.ConnectException;
@@ -476,7 +476,7 @@ public WebApp start(WebApp webapp, WebAppContext ui2Context) {
         LOG.info("Web app " + name + " started at "
             + httpServer.getConnectorAddress(0).getPort());
       } catch (IOException e) {
-        throw new WebAppException("Error starting http server", e);
+        throw new WebAppException("Error starting http server", e, webApp);
       }
       return webApp;
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index ee9100f8e7866..f084adbd9a2b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -35,10 +35,12 @@ public interface YarnWebParams {
   String APP_STATE = "app.state";
   String APP_START_TIME_BEGIN = "app.started-time.begin";
   String APP_START_TIME_END = "app.started-time.end";
+  String APP_SC = "app.subcluster";
   String APPS_NUM = "apps.num";
   String QUEUE_NAME = "queue.name";
   String NODE_STATE = "node.state";
   String NODE_LABEL = "node.label";
+  String NODE_SC = "node.subcluster";
   String WEB_UI_TYPE = "web.ui.type";
   String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
   String ERROR_MESSAGE = "error.message";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
index aec6762772992..9e2c2316c77e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,9 +19,9 @@
 /**
  * Data structures for scheduler configuration mutation info.
  */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
+@Private
+@Unstable
 package org.apache.hadoop.yarn.webapp.dao;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/example/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/example/package-info.java
index 0f0936b7cd3ef..e164a0219fbb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/example/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/example/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@LimitedPrivate({"YARN", "MapReduce"})
 package org.apache.hadoop.yarn.webapp.example;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletGen.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletGen.java
index 0a8f016ac328a..ff8094c9780c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletGen.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletGen.java
@@ -73,7 +73,7 @@ public class HamletGen {
    * @param implClass a generic hamlet implementation. e.g. {@link HamletImpl}
    * @param outputName name of the output class. e.g. {@link Hamlet}
    * @param outputPkg package name of the output class.
-   * @throws IOException
+   * @throws IOException io error occur.
    */
   public void generate(Class<?> specClass, Class<?> implClass,
                        String outputName, String outputPkg) throws IOException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletImpl.java
index c1f5195d6d7a3..ae07ccb245ced 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletImpl.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.webapp.hamlet2;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.util.Preconditions.*;
 import org.apache.hadoop.thirdparty.com.google.common.base.Splitter;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletSpec.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletSpec.java
index 8aeba93f0989b..518a22c154b36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletSpec.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/HamletSpec.java
@@ -1251,7 +1251,7 @@ public interface _FormCtrl {
 
     /**
      * Add a TEXTAREA element.
-     * @param selector
+     * @param selector the css selector in the form of (#id)*(.class)*
      * @return a new TEXTAREA element builder
      */
     TEXTAREA textarea(String selector);
@@ -2080,13 +2080,13 @@ public interface BLOCKQUOTE extends Attrs, Block, _Script, _Child {
    */
   public interface INS extends Attrs, Flow, _Child {
     /** info on reason for change
-     * @param uri
+     * @param uri the URI.
      * @return the current element builder
      */
     INS $cite(String uri);
 
     /** date and time of change
-     * @param datetime
+     * @param datetime the time.
      * @return the current element builder
      */
     INS $datetime(String datetime);
@@ -2103,7 +2103,7 @@ public interface DEL extends Attrs, Flow, _Child {
     DEL $cite(String uri);
 
     /** date and time of change
-     * @param datetime the time
+     * @param datetime the time.
      * @return the current element builder
      */
     DEL $datetime(String datetime);
@@ -2205,13 +2205,13 @@ public interface LI extends Attrs, Flow, _Child {
   public interface FORM extends Attrs, _Child, /* (%block;|SCRIPT)+ -(FORM) */
                                 _Script, _Block, _FieldSet {
     /** server-side form handler
-     * @param uri
+     * @param uri the URI.
      * @return the current element builder
      */
     FORM $action(String uri);
 
     /** HTTP method used to submit the form
-     * @param method
+     * @param method method.
      * @return the current element builder
      */
     FORM $method(Method method);
@@ -2220,37 +2220,37 @@ public interface FORM extends Attrs, _Child, /* (%block;|SCRIPT)+ -(FORM) */
      * contentype for "POST" method.
      * The default is "application/x-www-form-urlencoded".
      * Use "multipart/form-data" for input type=file
-     * @param enctype
+     * @param enctype enctype.
      * @return the current element builder
      */
     FORM $enctype(String enctype);
 
     /** list of MIME types for file upload
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     FORM $accept(String cdata);
 
     /** name of form for scripting
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     FORM $name(String cdata);
 
     /** the form was submitted
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     FORM $onsubmit(String script);
 
     /** the form was reset
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     FORM $onreset(String script);
 
     /** (space and/or comma separated) list of supported charsets
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     FORM $accept_charset(String cdata);
@@ -2262,25 +2262,25 @@ public interface FORM extends Attrs, _Child, /* (%block;|SCRIPT)+ -(FORM) */
   public interface LABEL extends Attrs, _Child, /* (%inline;)* -(LABEL) */
                                  PCData, FontStyle, Phrase, Special, _FormCtrl {
     /** matches field ID value
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     LABEL $for(String cdata);
 
     /** accessibility key character
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     LABEL $accesskey(String cdata);
 
     /** the element got the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     LABEL $onfocus(String script);
 
     /** the element lost the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     LABEL $onblur(String script);
@@ -2292,19 +2292,19 @@ public interface LABEL extends Attrs, _Child, /* (%inline;)* -(LABEL) */
   @Element(endTag=false)
   public interface INPUT extends Attrs, _Child {
     /** what kind of widget is needed. default is "text".
-     * @param inputType
+     * @param inputType input value.
      * @return the current element builder
      */
     INPUT $type(InputType inputType);
 
     /** submit as part of form
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     INPUT $name(String cdata);
 
     /** Specify for radio buttons and checkboxes
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     INPUT $value(String cdata);
@@ -2325,25 +2325,25 @@ public interface INPUT extends Attrs, _Child {
     INPUT $readonly();
 
     /** specific to each type of field
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     INPUT $size(String cdata);
 
     /** max chars for text fields
-     * @param length
+     * @param length max chars length.
      * @return the current element builder
      */
     INPUT $maxlength(int length);
 
     /** for fields with images
-     * @param uri
+     * @param uri the URI.
      * @return the current element builder
      */
     INPUT $src(String uri);
 
     /** short description
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     INPUT $alt(String cdata);
@@ -2355,43 +2355,43 @@ public interface INPUT extends Attrs, _Child {
     INPUT $ismap();
 
     /** position in tabbing order
-     * @param index
+     * @param index the index
      * @return the current element builder
      */
     INPUT $tabindex(int index);
 
     /** accessibility key character
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     INPUT $accesskey(String cdata);
 
     /** the element got the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     INPUT $onfocus(String script);
 
     /** the element lost the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     INPUT $onblur(String script);
 
     /** some text was selected
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     INPUT $onselect(String script);
 
     /** the element value was changed
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     INPUT $onchange(String script);
 
     /** list of MIME types for file upload (csv)
-     * @param contentTypes
+     * @param contentTypes content types.
      * @return the current element builder
      */
     INPUT $accept(String contentTypes);
@@ -2426,13 +2426,13 @@ public interface SELECT extends Attrs, _Option, _Child {
     OPTGROUP optgroup();
 
     /** field name
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     SELECT $name(String cdata);
 
     /** rows visible
-     * @param rows
+     * @param rows number of rows.
      * @return the current element builder
      */
     SELECT $size(int rows);
@@ -2448,25 +2448,25 @@ public interface SELECT extends Attrs, _Option, _Child {
     SELECT $disabled();
 
     /** position in tabbing order
-     * @param index
+     * @param index the index
      * @return the current element builder
      */
     SELECT $tabindex(int index);
 
     /** the element got the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     SELECT $onfocus(String script);
 
     /** the element lost the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     SELECT $onblur(String script);
 
     /** the element value was changed
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     SELECT $onchange(String script);
@@ -2482,7 +2482,7 @@ public interface OPTGROUP extends Attrs, _Option, _Child {
     OPTGROUP $disabled();
 
     /** for use in hierarchical menus
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     OPTGROUP $label(String cdata);
@@ -2504,13 +2504,13 @@ public interface OPTION extends Attrs, PCData, _Child {
     OPTION $disabled();
 
     /** for use in hierarchical menus
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     OPTION $label(String cdata);
 
     /** defaults to element content
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     OPTION $value(String cdata);
@@ -2521,19 +2521,19 @@ public interface OPTION extends Attrs, PCData, _Child {
    */
   public interface TEXTAREA extends Attrs, PCData, _Child {
     /** variable name for the text
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     TEXTAREA $name(String cdata);
 
     /** visible rows
-     * @param rows
+     * @param rows number of rows.
      * @return the current element builder
      */
     TEXTAREA $rows(int rows);
 
     /** visible columns
-     * @param cols
+     * @param cols number of cols.
      * @return the current element builder
      */
     TEXTAREA $cols(int cols);
@@ -2549,37 +2549,37 @@ public interface TEXTAREA extends Attrs, PCData, _Child {
     TEXTAREA $readonly();
 
     /** position in tabbing order
-     * @param index
+     * @param index the index
      * @return the current element builder
      */
     TEXTAREA $tabindex(int index);
 
     /** accessibility key character
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     TEXTAREA $accesskey(String cdata);
 
     /** the element got the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     TEXTAREA $onfocus(String script);
 
     /** the element lost the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     TEXTAREA $onblur(String script);
 
     /** some text was selected
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     TEXTAREA $onselect(String script);
 
     /** the element value was changed
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     TEXTAREA $onchange(String script);
@@ -2597,7 +2597,7 @@ public interface _Legend extends _Child {
 
     /**
      * Add a LEGEND element.
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     _Legend legend(String cdata);
@@ -2614,7 +2614,7 @@ public interface FIELDSET extends Attrs, _Legend, PCData, Flow, _Child {
    */
   public interface LEGEND extends Attrs, Inline, _Child {
     /** accessibility key character
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     LEGEND $accesskey(String cdata);
@@ -2626,19 +2626,19 @@ public interface LEGEND extends Attrs, Inline, _Child {
   public interface BUTTON extends /* (%flow;)* -(A|%formctrl|FORM|FIELDSET) */
       _Block, PCData, FontStyle, Phrase, _Special, _ImgObject, _SubSup, Attrs {
     /** name of the value
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     BUTTON $name(String cdata);
 
     /** sent to server when submitted
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     BUTTON $value(String cdata);
 
     /** for use as form button
-     * @param type
+     * @param type button type.
      * @return the current element builder
      */
     BUTTON $type(ButtonType type);
@@ -2649,25 +2649,25 @@ public interface BUTTON extends /* (%flow;)* -(A|%formctrl|FORM|FIELDSET) */
     BUTTON $disabled();
 
     /** position in tabbing order
-     * @param index
+     * @param index the index
      * @return the current element builder
      */
     BUTTON $tabindex(int index);
 
     /** accessibility key character
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     BUTTON $accesskey(String cdata);
 
     /** the element got the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     BUTTON $onfocus(String script);
 
     /** the element lost the focus
-     * @param script
+     * @param script to invoke.
      * @return the current element builder
      */
     BUTTON $onblur(String script);
@@ -2721,7 +2721,7 @@ public interface _Table extends _TableRow, _TableCol {
 
     /**
      * Add a CAPTION element.
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     _Table caption(String cdata);
@@ -2813,7 +2813,7 @@ public interface TBODY extends Attrs, _TableRow, _Child {
   @Element(endTag=false)
   public interface COLGROUP extends Attrs, _TableCol, _Child {
     /** default number of columns in group. default: 1
-     * @param cols
+     * @param cols number of cols.
      * @return the current element builder
      */
     COLGROUP $span(int cols);
@@ -2827,7 +2827,7 @@ public interface COLGROUP extends Attrs, _TableCol, _Child {
   @Element(endTag=false)
   public interface COL extends Attrs, _Child {
     /** COL attributes affect N columns. default: 1
-     * @param cols
+     * @param cols number of cols.
      * @return the current element builder
      */
     COL $span(int cols);
@@ -2896,25 +2896,25 @@ public interface _Cell extends Attrs, Flow, _Child {
     // use $title for elaberation, when appropriate.
     // $axis omitted. use scope.
     /** space-separated list of id's for header cells
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     _Cell $headers(String cdata);
 
     /** scope covered by header cells
-     * @param scope
+     * @param scope scope.
      * @return the current element builder
      */
     _Cell $scope(Scope scope);
 
     /** number of rows spanned by cell. default: 1
-     * @param rows
+     * @param rows number of rows.
      * @return the current element builder
      */
     _Cell $rowspan(int rows);
 
     /** number of cols spanned by cell. default: 1
-     * @param cols
+     * @param cols number of cols.
      * @return the current element builder
      */
     _Cell $colspan(int cols);
@@ -2959,7 +2959,7 @@ public interface _Head extends HeadMisc {
 
     /**
      * Add a complete BASE element.
-     * @param uri
+     * @param uri the URI.
      * @return the current element builder
      */
     _Head base(String uri);
@@ -2984,7 +2984,7 @@ public interface TITLE extends I18nAttrs, PCData, _Child {
   @Element(endTag=false)
   public interface BASE extends _Child {
     /** URI that acts as base URI
-     * @param uri
+     * @param uri the URI.
      * @return the current element builder
      */
     BASE $href(String uri);
@@ -2996,19 +2996,19 @@ public interface BASE extends _Child {
   @Element(endTag=false)
   public interface META extends I18nAttrs, _Child {
     /** HTTP response header name
-     * @param header
+     * @param header for the http-equiv attribute
      * @return the current element builder
      */
     META $http_equiv(String header);
 
     /** metainformation name
-     * @param name
+     * @param name of the meta element
      * @return the current element builder
      */
     META $name(String name);
 
     /** associated information
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     META $content(String cdata);
@@ -3021,19 +3021,19 @@ public interface META extends I18nAttrs, _Child {
    */
   public interface STYLE extends I18nAttrs, _Content, _Child {
     /** content type of style language
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     STYLE $type(String cdata);
 
     /** designed for use with these media
-     * @param media
+     * @param media set of media.
      * @return the current element builder
      */
     STYLE $media(EnumSet<Media> media);
 
     /** advisory title
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     STYLE $title(String cdata);
@@ -3044,25 +3044,25 @@ public interface STYLE extends I18nAttrs, _Content, _Child {
    */
   public interface SCRIPT extends _Content, _Child {
     /** char encoding of linked resource
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     SCRIPT $charset(String cdata);
 
     /** content type of script language
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     SCRIPT $type(String cdata);
 
     /** URI for an external script
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     SCRIPT $src(String cdata);
 
     /** UA may defer execution of script
-     * @param cdata
+     * @param cdata the content of the element.
      * @return the current element builder
      */
     SCRIPT $defer(String cdata);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/package-info.java
index 64a8447e024e7..54aa474bf016a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet2/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -21,7 +21,7 @@
  * The old package is using _ as a one-character identifier,
  * which is banned from JDK9.
  */
-@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@LimitedPrivate({"YARN", "MapReduce"})
 package org.apache.hadoop.yarn.webapp.hamlet2;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/package-info.java
index 1d64404cc3623..23c4328ad8d17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@LimitedPrivate({"YARN", "MapReduce"})
 package org.apache.hadoop.yarn.webapp.log;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/package-info.java
index 342f28df283eb..4d179099eb7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@LimitedPrivate({"YARN", "MapReduce"})
 package org.apache.hadoop.yarn.webapp;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 5b578197100a1..02fec11505990 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -95,6 +95,13 @@ public static void setNMWebAppHostNameAndPort(Configuration conf,
    * Runs a certain function against the active RM. The function's first
    * argument is expected to be a string which contains the address of
    * the RM being tried.
+   * @param conf configuration.
+   * @param func throwing bi function.
+   * @param arg T arg.
+   * @param <T> Generic T.
+   * @param <R> Generic R.
+   * @throws Exception exception occurs.
+   * @return instance of Generic R.
    */
   public static <T, R> R execOnActiveRM(Configuration conf,
       ThrowingBiFunction<String, T, R> func, T arg) throws Exception {
@@ -389,7 +396,7 @@ public static String getTimelineCollectorWebAppURLWithoutScheme(
    * if url has scheme then it will be returned as it is else it will return
    * url with scheme.
    * @param schemePrefix eg. http:// or https://
-   * @param url
+   * @param url url.
    * @return url with scheme
    */
   public static String getURLWithScheme(String schemePrefix, String url) {
@@ -428,7 +435,8 @@ public static String getAggregatedLogURL(String serverHttpAddress,
   /**
    * Choose which scheme (HTTP or HTTPS) to use when generating a URL based on
    * the configuration.
-   * 
+   *
+   * @param conf configuration.
    * @return the scheme (HTTP / HTTPS)
    */
   public static String getHttpSchemePrefix(Configuration conf) {
@@ -438,6 +446,8 @@ public static String getHttpSchemePrefix(Configuration conf) {
   /**
    * Load the SSL keystore / truststore into the HttpServer builder.
    * @param builder the HttpServer2.Builder to populate with ssl config
+   * @return HttpServer2.Builder instance (passed in as the first parameter)
+   *         after loading SSL stores
    */
   public static HttpServer2.Builder loadSslConfiguration(
       HttpServer2.Builder builder) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java
index 3f61645461b8b..39cc2e361f1b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebServiceClient.java
@@ -46,8 +46,8 @@ public class WebServiceClient {
    * Construct a new WebServiceClient based on the configuration. It will try to
    * load SSL certificates when it is specified.
    *
-   * @param conf
-   * @throws Exception
+   * @param conf configuration.
+   * @throws Exception exception occur.
    */
   public static void initialize(Configuration conf) throws Exception {
     if (instance == null) {
@@ -75,9 +75,9 @@ SSLFactory getSSLFactory() {
   /**
    * Start SSL factory.
    *
-   * @param conf
-   * @return
-   * @throws Exception
+   * @param conf configuration.
+   * @return SSL factory.
+   * @throws Exception exception occur.
    */
   private static SSLFactory createSSLFactory(Configuration conf)
       throws Exception {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
index acf040ee0147e..c07a76617eef8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
@@ -31,6 +31,8 @@
 public abstract class HtmlBlock extends TextView implements SubView {
 
   protected static final String UNAVAILABLE = "N/A";
+  protected static final long BYTES_IN_MB = 1024 * 1024;
+  protected static final String DATE_PATTERN = "yyyy-MM-dd HH:mm:ss";
 
   public class Block extends Hamlet {
     Block(PrintWriter out, int level, boolean wasInline) {
@@ -95,4 +97,28 @@ protected UserGroupInformation getCallerUGI() {
     return callerUGI;
   }
 
+  /**
+   * Initialize User Help Information Div.
+   * When the user does not configure the Yarn Federation function, prompt the user.
+   *
+   * @param html HTML page.
+   * @param isEnabled If federation is enabled.
+   */
+  protected void initUserHelpInformationDiv(Block html, boolean isEnabled) {
+    if (!isEnabled) {
+      html.style(".alert {padding: 15px; margin-bottom: 20px; " +
+          " border: 1px solid transparent; border-radius: 4px;}");
+      html.style(".alert-dismissable {padding-right: 35px;}");
+      html.style(".alert-info {color: #856404;background-color: #fff3cd;border-color: #ffeeba;}");
+
+      Hamlet.DIV<Hamlet> div = html.div("#div_id").$class("alert alert-dismissable alert-info");
+      div.p().$style("color:red").__("Federation is not Enabled.").__()
+          .p().__()
+          .p().__("We can refer to the following documents to configure Yarn Federation. ").__()
+          .p().__()
+          .a("https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/Federation.html",
+          "Hadoop: YARN Federation").
+          __();
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
index fa5a36884840c..56d9f25710eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
@@ -20,15 +20,15 @@
 
 import java.util.List;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.HTML;
+
 import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
 import static org.apache.hadoop.yarn.util.StringHelper.djoin;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.util.StringHelper.split;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.HTML;
-
 @InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
 public class JQueryUI extends HtmlBlock {
 
@@ -65,12 +65,12 @@ public class JQueryUI extends HtmlBlock {
   @Override
   protected void render(Block html) {
     html.link(root_url("static/jquery/themes-1.9.1/base/jquery-ui.css"))
-        .link(root_url("static/dt-1.10.18/css/jquery.dataTables.css"))
-        .link(root_url("static/dt-1.10.18/css/jui-dt.css"))
-        .link(root_url("static/dt-1.10.18/css/custom_datatable.css"))
-        .script(root_url("static/jquery/jquery-3.5.1.min.js"))
-        .script(root_url("static/jquery/jquery-ui-1.12.1.custom.min.js"))
-        .script(root_url("static/dt-1.10.18/js/jquery.dataTables.min.js"))
+        .link(root_url("static/dt-1.11.5/css/jquery.dataTables.css"))
+        .link(root_url("static/dt-1.11.5/css/jui-dt.css"))
+        .link(root_url("static/dt-1.11.5/css/custom_datatable.css"))
+        .script(root_url("static/jquery/jquery-3.6.0.min.js"))
+        .script(root_url("static/jquery/jquery-ui-1.13.2.custom.min.js"))
+        .script(root_url("static/dt-1.11.5/js/jquery.dataTables.min.js"))
         .script(root_url("static/yarn.dt.plugins.js"))
         .script(root_url("static/dt-sorting/natural.js"))
         .style("#jsnotice { padding: 0.2em; text-align: center; }",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/package-info.java
index c1f92eb7075f9..a22c1b3601c34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/package-info.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@LimitedPrivate({"YARN", "MapReduce"})
 package org.apache.hadoop.yarn.webapp.view;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png
deleted file mode 100644
index a56d0e21902ce..0000000000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc.png and /dev/null differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png
deleted file mode 100644
index b7e621ef1c68d..0000000000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_asc_disabled.png and /dev/null differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png
deleted file mode 100644
index 839ac4bb5b0b4..0000000000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_both.png and /dev/null differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png
deleted file mode 100644
index 90b295159df99..0000000000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc.png and /dev/null differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png
deleted file mode 100644
index 2409653dc94cd..0000000000000
Binary files a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/sort_desc_disabled.png and /dev/null differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
deleted file mode 100644
index 3a79ccc169c22..0000000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-/*!
- DataTables 1.10.18
- ©2008-2018 SpryMedia Ltd - datatables.net/license
-*/
-(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Z(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()),
-d[c]=e,"o"===b[1]&&Z(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Z(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Ca(a){var b=n.defaults.oLanguage,c=b.sDecimal;c&&Da(c);if(a){var d=a.sZeroRecords;!a.sEmptyTable&&(d&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(d&&"Loading..."===b.sLoadingRecords)&&F(a,
-a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&c!==a&&Da(a)}}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":
-"");"boolean"===typeof a.scrollX&&(a.scrollX=a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&J(n.models.oSearch,a[b])}function fb(a){A(a,"orderable","bSortable");A(a,"orderData","aDataSort");A(a,"orderSequence","asSorting");A(a,"orderDataType","sortDataType");var b=a.aDataSort;"number"===typeof b&&!h.isArray(b)&&(a.aDataSort=[b])}function gb(a){if(!n.__browser){var b={};n.__browser=b;var c=h("<div/>").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1,
-overflow:"hidden"}).append(h("<div/>").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(h("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,n.__browser);a.oScroll.iBarWidth=n.__browser.barWidth}
-function hb(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ea(a,b){var c=n.defaults.column,d=a.aoColumns.length,c=h.extend({},n.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},n.models.oSearch,c[d]);ka(a,d,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b],
-d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),J(n.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),c.sClass&&e.addClass(c.sClass),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=S(g),i=b.mRender?
-S(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return N(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone,
-b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function $(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Fa(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;(""!==b.sY||""!==b.sX)&&la(a);r(a,null,"column-sizing",[a])}function aa(a,b){var c=ma(a,"bVisible");return"number"===
-typeof c[b]?c[b]:null}function ba(a,b){var c=ma(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function V(a){var b=0;h.each(a.aoColumns,function(a,d){d.bVisible&&"none"!==h(d.nTh).css("display")&&b++});return b}function ma(a,b){var c=[];h.map(a.aoColumns,function(a,e){a[b]&&c.push(e)});return c}function Ga(a){var b=a.aoColumns,c=a.aoData,d=n.ext.type.detect,e,f,g,j,i,h,l,q,t;e=0;for(f=b.length;e<f;e++)if(l=b[e],t=[],!l.sType&&l._sManualType)l.sType=l._sManualType;else if(!l.sType){g=0;for(j=d.length;g<
-j;g++){i=0;for(h=c.length;i<h;i++){t[i]===k&&(t[i]=B(a,i,e,"type"));q=d[g](t[i],a);if(!q&&g!==d.length-1)break;if("html"===q)break}if(q){l.sType=q;break}}l.sType||(l.sType="string")}}function ib(a,b,c,d){var e,f,g,j,i,m,l=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){m=b[e];var q=m.targets!==k?m.targets:m.aTargets;h.isArray(q)||(q=[q]);f=0;for(g=q.length;f<g;f++)if("number"===typeof q[f]&&0<=q[f]){for(;l.length<=q[f];)Ea(a);d(q[f],m)}else if("number"===typeof q[f]&&0>q[f])d(l.length+q[f],m);else if("string"===
-typeof q[f]){j=0;for(i=l.length;j<i;j++)("_all"==q[f]||h(l[j].nTh).hasClass(q[f]))&&d(j,m)}}if(c){e=0;for(a=c.length;e<a;e++)d(e,c[e])}}function O(a,b,c,d){var e=a.aoData.length,f=h.extend(!0,{},n.models.oRow,{src:c?"dom":"data",idx:e});f._aData=b;a.aoData.push(f);for(var g=a.aoColumns,j=0,i=g.length;j<i;j++)g[j].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==k&&(a.aIds[b]=f);(c||!a.oFeatures.bDeferRender)&&Ha(a,e,c,d);return e}function na(a,b){var c;b instanceof h||(b=h(b));return b.map(function(b,
-e){c=Ia(a,e);return O(a,c.data,e,c.cells)})}function B(a,b,c,d){var e=a.iDraw,f=a.aoColumns[c],g=a.aoData[b]._aData,j=f.sDefaultContent,i=f.fnGetData(g,d,{settings:a,row:b,col:c});if(i===k)return a.iDrawError!=e&&null===j&&(K(a,0,"Requested unknown parameter "+("function"==typeof f.mData?"{function}":"'"+f.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),j;if((i===g||null===i)&&null!==j&&d!==k)i=j;else if("function"===typeof i)return i.call(g);return null===i&&"display"==d?"":i}function jb(a,
-b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}function Ja(a){return h.map(a.match(/(\\.|[^\.])+/g)||[""],function(a){return a.replace(/\\\./g,".")})}function S(a){if(h.isPlainObject(a)){var b={};h.each(a,function(a,c){c&&(b[a]=S(c))});return function(a,c,f,g){var j=b[c]||b._;return j!==k?j(a,c,f,g):a}}if(null===a)return function(a){return a};if("function"===typeof a)return function(b,c,f,g){return a(b,c,f,g)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||
--1!==a.indexOf("("))){var c=function(a,b,f){var g,j;if(""!==f){j=Ja(f);for(var i=0,m=j.length;i<m;i++){f=j[i].match(ca);g=j[i].match(W);if(f){j[i]=j[i].replace(ca,"");""!==j[i]&&(a=a[j[i]]);g=[];j.splice(0,i+1);j=j.join(".");if(h.isArray(a)){i=0;for(m=a.length;i<m;i++)g.push(c(a[i],b,j))}a=f[0].substring(1,f[0].length-1);a=""===a?g:g.join(a);break}else if(g){j[i]=j[i].replace(W,"");a=a[j[i]]();continue}if(null===a||a[j[i]]===k)return k;a=a[j[i]]}}return a};return function(b,e){return c(b,e,a)}}return function(b){return b[a]}}
-function N(a){if(h.isPlainObject(a))return N(a._);if(null===a)return function(){};if("function"===typeof a)return function(b,d,e){a(b,"set",d,e)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("[")||-1!==a.indexOf("("))){var b=function(a,d,e){var e=Ja(e),f;f=e[e.length-1];for(var g,j,i=0,m=e.length-1;i<m;i++){g=e[i].match(ca);j=e[i].match(W);if(g){e[i]=e[i].replace(ca,"");a[e[i]]=[];f=e.slice();f.splice(0,i+1);g=f.join(".");if(h.isArray(d)){j=0;for(m=d.length;j<m;j++)f={},b(f,d[j],g),
-a[e[i]].push(f)}else a[e[i]]=d;return}j&&(e[i]=e[i].replace(W,""),a=a[e[i]](d));if(null===a[e[i]]||a[e[i]]===k)a[e[i]]={};a=a[e[i]]}if(f.match(W))a[f.replace(W,"")](d);else a[f.replace(ca,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Ka(a){return D(a.aoData,"_aData")}function oa(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}function pa(a,b,c){for(var d=-1,e=0,f=a.length;e<f;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===k&&a.splice(d,
-1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ia(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c<f;c++)g(j[c],c)}}e._aSortData=null;e._aFilterData=null;g=a.aoColumns;if(d!==k)g[d].sType=null;else{c=0;for(f=g.length;c<f;c++)g[c].sType=null;La(a,e)}}function Ia(a,b,c,d){var e=[],f=b.firstChild,g,
-j,i=0,m,l=a.aoColumns,q=a._rowReadObject,d=d!==k?d:q?{}:[],t=function(a,b){if("string"===typeof a){var c=a.indexOf("@");-1!==c&&(c=a.substring(c+1),N(a)(d,b.getAttribute(c)))}},G=function(a){if(c===k||c===i)j=l[i],m=h.trim(a.innerHTML),j&&j._bAttrSrc?(N(j.mData._)(d,m),t(j.mData.sort,a),t(j.mData.type,a),t(j.mData.filter,a)):q?(j._setter||(j._setter=N(j.mData)),j._setter(d,m)):d[i]=m;i++};if(f)for(;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)G(f),e.push(f);f=f.nextSibling}else{e=b.anCells;
-f=0;for(g=e.length;f<g;f++)G(e[f])}if(b=b.firstChild?b:b.nTr)(b=b.getAttribute("id"))&&N(a.rowId)(d,b);return{data:d,cells:e}}function Ha(a,b,c,d){var e=a.aoData[b],f=e._aData,g=[],j,i,m,l,q;if(null===e.nTr){j=c||H.createElement("tr");e.nTr=j;e.anCells=g;j._DT_RowIndex=b;La(a,e);l=0;for(q=a.aoColumns.length;l<q;l++){m=a.aoColumns[l];i=c?d[l]:H.createElement(m.sCellType);i._DT_CellIndex={row:b,column:l};g.push(i);if((!c||m.mRender||m.mData!==l)&&(!h.isPlainObject(m.mData)||m.mData._!==l+".display"))i.innerHTML=
-B(a,b,l,"display");m.sClass&&(i.className+=" "+m.sClass);m.bVisible&&!c?j.appendChild(i):!m.bVisible&&c&&i.parentNode.removeChild(i);m.fnCreatedCell&&m.fnCreatedCell.call(a.oInstance,i,B(a,b,l),f,b,l)}r(a,"aoRowCreatedCallback",null,[j,f,b,g])}e.nTr.setAttribute("role","row")}function La(a,b){var c=b.nTr,d=b._aData;if(c){var e=a.rowIdFn(d);e&&(c.id=e);d.DT_RowClass&&(e=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?qa(b.__rowc.concat(e)):e,h(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));
-d.DT_RowAttr&&h(c).attr(d.DT_RowAttr);d.DT_RowData&&h(c).data(d.DT_RowData)}}function kb(a){var b,c,d,e,f,g=a.nTHead,j=a.nTFoot,i=0===h("th, td",g).length,m=a.oClasses,l=a.aoColumns;i&&(e=h("<tr/>").appendTo(g));b=0;for(c=l.length;b<c;b++)f=l[b],d=h(f.nTh).addClass(f.sClass),i&&d.appendTo(e),a.oFeatures.bSort&&(d.addClass(f.sSortingClass),!1!==f.bSortable&&(d.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),Ma(a,f.nTh,b))),f.sTitle!=d[0].innerHTML&&d.html(f.sTitle),Na(a,"header")(a,d,
-f,m);i&&ea(a.aoHeader,g);h(g).find(">tr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(m.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(m.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b<c;b++)f=l[b],f.nTf=a[b].cell,f.sClass&&h(f.nTf).addClass(f.sClass)}}function fa(a,b,c){var d,e,f,g=[],j=[],i=a.aoColumns.length,m;if(b){c===k&&(c=!1);d=0;for(e=b.length;d<e;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=i-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);j.push([])}d=
-0;for(e=g.length;d<e;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(m=i=1,j[d][f]===k){a.appendChild(g[d][f].cell);for(j[d][f]=1;g[d+i]!==k&&g[d][f].cell==g[d+i][f].cell;)j[d+i][f]=1,i++;for(;g[d][f+m]!==k&&g[d][f].cell==g[d][f+m].cell;){for(c=0;c<i;c++)j[d+c][f+m]=1;m++}h(g[d][f].cell).attr("rowspan",i).attr("colspan",m)}}}}function P(a){var b=r(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))C(a,!1);else{var b=[],c=0,d=a.asStripeClasses,e=
-d.length,f=a.oLanguage,g=a.iInitDisplayStart,j="ssp"==y(a),i=a.aiDisplay;a.bDrawing=!0;g!==k&&-1!==g&&(a._iDisplayStart=j?g:g>=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,m=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!lb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:m;for(j=j?0:g;j<f;j++){var l=i[j],q=a.aoData[l];null===q.nTr&&Ha(a,l);var t=q.nTr;if(0!==e){var G=d[c%e];q._sRowStripe!=G&&(h(t).removeClass(q._sRowStripe).addClass(G),
-q._sRowStripe=G)}r(a,"aoRowCallback",null,[t,q._aData,c,j,l]);b.push(t);c++}}else c=f.sZeroRecords,1==a.iDraw&&"ajax"==y(a)?c=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(c=f.sEmptyTable),b[0]=h("<tr/>",{"class":e?d[0]:""}).append(h("<td />",{valign:"top",colSpan:V(a),"class":a.oClasses.sRowEmpty}).html(c))[0];r(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ka(a),g,m,i]);r(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ka(a),g,m,i]);d=h(a.nTBody);d.children().detach();
-d.append(h(b));r(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&mb(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;P(a);a._drawHold=!1}function nb(a){var b=a.oClasses,c=h(a.nTable),c=h("<div/>").insertBefore(c),d=a.oFeatures,e=h("<div/>",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=
-a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,m,l,q,k=0;k<f.length;k++){g=null;j=f[k];if("<"==j){i=h("<div/>")[0];m=f[k+1];if("'"==m||'"'==m){l="";for(q=2;f[k+q]!=m;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(m=l.split("."),i.id=m[0].substr(1,m[0].length-1),i.className=m[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=ob(a);else if("f"==j&&
-d.bFilter)g=pb(a);else if("r"==j&&d.bProcessing)g=qb(a);else if("t"==j)g=rb(a);else if("i"==j&&d.bInfo)g=sb(a);else if("p"==j&&d.bPaginate)g=tb(a);else if(0!==n.ext.feature.length){i=n.ext.feature;q=0;for(m=i.length;q<m;q++)if(j==i[q].cFeature){g=i[q].fnInit(a);break}}g&&(i=a.aanFeatures,i[j]||(i[j]=[]),i[j].push(g),e.append(g))}c.replaceWith(e);a.nHolding=null}function ea(a,b){var c=h(b).children("tr"),d,e,f,g,j,i,m,l,q,k;a.splice(0,a.length);f=0;for(i=c.length;f<i;f++)a.push([]);f=0;for(i=c.length;f<
-i;f++){d=c[f];for(e=d.firstChild;e;){if("TD"==e.nodeName.toUpperCase()||"TH"==e.nodeName.toUpperCase()){l=1*e.getAttribute("colspan");q=1*e.getAttribute("rowspan");l=!l||0===l||1===l?1:l;q=!q||0===q||1===q?1:q;g=0;for(j=a[f];j[g];)g++;m=g;k=1===l?!0:!1;for(j=0;j<l;j++)for(g=0;g<q;g++)a[f+g][m+j]={cell:e,unique:k},a[f+g].nTr=d}e=e.nextSibling}}}function ra(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],ea(c,b)));for(var b=0,e=c.length;b<e;b++)for(var f=0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||
-!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function sa(a,b,c){r(a,"aoServerParams","serverParams",[b]);if(b&&h.isArray(b)){var d={},e=/(.*?)\[\]$/;h.each(b,function(a,b){var c=b.name.match(e);c?(c=c[0],d[c]||(d[c]=[]),d[c].push(b.value)):d[b.name]=b.value});b=d}var f,g=a.ajax,j=a.oInstance,i=function(b){r(a,null,"xhr",[a,b,a.jqXHR]);c(b)};if(h.isPlainObject(g)&&g.data){f=g.data;var m="function"===typeof f?f(b,a):f,b="function"===typeof f&&m?m:h.extend(!0,b,m);delete g.data}m={data:b,success:function(b){var c=
-b.error||b.sError;c&&K(a,0,c);a.json=b;i(b)},dataType:"json",cache:!1,type:a.sServerMethod,error:function(b,c){var d=r(a,null,"xhr",[a,null,a.jqXHR]);-1===h.inArray(!0,d)&&("parsererror"==c?K(a,0,"Invalid JSON response",1):4===b.readyState&&K(a,0,"Ajax error",7));C(a,!1)}};a.oAjaxData=b;r(a,null,"preXhr",[a,b]);a.fnServerData?a.fnServerData.call(j,a.sAjaxSource,h.map(b,function(a,b){return{name:b,value:a}}),i,a):a.sAjaxSource||"string"===typeof g?a.jqXHR=h.ajax(h.extend(m,{url:g||a.sAjaxSource})):
-"function"===typeof g?a.jqXHR=g.call(j,b,i,a):(a.jqXHR=h.ajax(h.extend(m,g)),g.data=f)}function lb(a){return a.bAjaxDataGet?(a.iDraw++,C(a,!0),sa(a,ub(a),function(b){vb(a,b)}),!1):!0}function ub(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,f=a.aoPreSearchCols,g,j=[],i,m,l,k=X(a);g=a._iDisplayStart;i=!1!==d.bPaginate?a._iDisplayLength:-1;var t=function(a,b){j.push({name:a,value:b})};t("sEcho",a.iDraw);t("iColumns",c);t("sColumns",D(b,"sName").join(","));t("iDisplayStart",g);t("iDisplayLength",
-i);var G={draw:a.iDraw,columns:[],order:[],start:g,length:i,search:{value:e.sSearch,regex:e.bRegex}};for(g=0;g<c;g++)m=b[g],l=f[g],i="function"==typeof m.mData?"function":m.mData,G.columns.push({data:i,name:m.sName,searchable:m.bSearchable,orderable:m.bSortable,search:{value:l.sSearch,regex:l.bRegex}}),t("mDataProp_"+g,i),d.bFilter&&(t("sSearch_"+g,l.sSearch),t("bRegex_"+g,l.bRegex),t("bSearchable_"+g,m.bSearchable)),d.bSort&&t("bSortable_"+g,m.bSortable);d.bFilter&&(t("sSearch",e.sSearch),t("bRegex",
-e.bRegex));d.bSort&&(h.each(k,function(a,b){G.order.push({column:b.col,dir:b.dir});t("iSortCol_"+a,b.col);t("sSortDir_"+a,b.dir)}),t("iSortingCols",k.length));b=n.ext.legacy.ajax;return null===b?a.sAjaxSource?j:G:b?j:G}function vb(a,b){var c=ta(a,b),d=b.sEcho!==k?b.sEcho:b.draw,e=b.iTotalRecords!==k?b.iTotalRecords:b.recordsTotal,f=b.iTotalDisplayRecords!==k?b.iTotalDisplayRecords:b.recordsFiltered;if(d){if(1*d<a.iDraw)return;a.iDraw=1*d}oa(a);a._iRecordsTotal=parseInt(e,10);a._iRecordsDisplay=parseInt(f,
-10);d=0;for(e=c.length;d<e;d++)O(a,c[d]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;P(a);a._bInitComplete||ua(a,b);a.bAjaxDataGet=!0;C(a,!1)}function ta(a,b){var c=h.isPlainObject(a.ajax)&&a.ajax.dataSrc!==k?a.ajax.dataSrc:a.sAjaxDataProp;return"data"===c?b.aaData||b[c]:""!==c?S(c)(b):b}function pb(a){var b=a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,f=a.aanFeatures,g='<input type="search" class="'+b.sFilterInput+'"/>',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",
-g):j+g,b=h("<div/>",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("<label/>").append(j)),f=function(){var b=!this.value?"":this.value;b!=e.sSearch&&(ga(a,{sSearch:b,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive}),a._iDisplayStart=0,P(a))},g=null!==a.searchDelay?a.searchDelay:"ssp"===y(a)?400:0,i=h("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",g?Oa(f,g):f).on("keypress.DT",function(a){if(13==a.keyCode)return!1}).attr("aria-controls",
-c);h(a.nTable).on("search.dt.DT",function(b,c){if(a===c)try{i[0]!==H.activeElement&&i.val(e.sSearch)}catch(d){}});return b[0]}function ga(a,b,c){var d=a.oPreviousSearch,e=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};Ga(a);if("ssp"!=y(a)){wb(a,b.sSearch,c,b.bEscapeRegex!==k?!b.bEscapeRegex:b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<e.length;b++)xb(a,e[b].sSearch,b,e[b].bEscapeRegex!==k?!e[b].bEscapeRegex:e[b].bRegex,
-e[b].bSmart,e[b].bCaseInsensitive);yb(a)}else f(b);a.bFiltered=!0;r(a,null,"search",[a])}function yb(a){for(var b=n.ext.search,c=a.aiDisplay,d,e,f=0,g=b.length;f<g;f++){for(var j=[],i=0,m=c.length;i<m;i++)e=c[i],d=a.aoData[e],b[f](a,d._aFilterData,e,d._aData,i)&&j.push(e);c.length=0;h.merge(c,j)}}function xb(a,b,c,d,e,f){if(""!==b){for(var g=[],j=a.aiDisplay,d=Pa(b,d,e,f),e=0;e<j.length;e++)b=a.aoData[j[e]]._aFilterData[c],d.test(b)&&g.push(j[e]);a.aiDisplay=g}}function wb(a,b,c,d,e,f){var d=Pa(b,
-d,e,f),f=a.oPreviousSearch.sSearch,g=a.aiDisplayMaster,j,e=[];0!==n.ext.search.length&&(c=!0);j=zb(a);if(0>=b.length)a.aiDisplay=g.slice();else{if(j||c||f.length>b.length||0!==b.indexOf(f)||a.bSorted)a.aiDisplay=g.slice();b=a.aiDisplay;for(c=0;c<b.length;c++)d.test(a.aoData[b[c]]._sFilterRow)&&e.push(b[c]);a.aiDisplay=e}}function Pa(a,b,c,d){a=b?a:Qa(a);c&&(a="^(?=.*?"+h.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(a){if('"'===a.charAt(0))var b=a.match(/^"(.*)"$/),a=b?b[1]:a;return a.replace('"',
-"")}).join(")(?=.*?")+").*$");return RegExp(a,d?"i":"")}function zb(a){var b=a.aoColumns,c,d,e,f,g,j,i,h,l=n.ext.type.search;c=!1;d=0;for(f=a.aoData.length;d<f;d++)if(h=a.aoData[d],!h._aFilterData){j=[];e=0;for(g=b.length;e<g;e++)c=b[e],c.bSearchable?(i=B(a,d,e,"filter"),l[c.sType]&&(i=l[c.sType](i)),null===i&&(i=""),"string"!==typeof i&&i.toString&&(i=i.toString())):i="",i.indexOf&&-1!==i.indexOf("&")&&(va.innerHTML=i,i=Wb?va.textContent:va.innerText),i.replace&&(i=i.replace(/[\r\n]/g,"")),j.push(i);
-h._aFilterData=j;h._sFilterRow=j.join("  ");c=!0}return c}function Ab(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Bb(a){return{sSearch:a.search,bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function sb(a){var b=a.sTableId,c=a.aanFeatures.i,d=h("<div/>",{"class":a.oClasses.sInfo,id:!c?b+"_info":null});c||(a.aoDrawCallback.push({fn:Cb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),h(a.nTable).attr("aria-describedby",
-b+"_info"));return d[0]}function Cb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+1,e=a.fnDisplayEnd(),f=a.fnRecordsTotal(),g=a.fnRecordsDisplay(),j=g?c.sInfo:c.sInfoEmpty;g!==f&&(j+=" "+c.sInfoFiltered);j+=c.sInfoPostFix;j=Db(a,j);c=c.fnInfoCallback;null!==c&&(j=c.call(a.oInstance,a,d,e,f,g,j));h(b).html(j)}}function Db(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,f=a.fnRecordsDisplay(),g=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,
-c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,f)).replace(/_PAGE_/g,c.call(a,g?1:Math.ceil(d/e))).replace(/_PAGES_/g,c.call(a,g?1:Math.ceil(f/e)))}function ha(a){var b,c,d=a.iInitDisplayStart,e=a.aoColumns,f;c=a.oFeatures;var g=a.bDeferLoading;if(a.bInitialised){nb(a);kb(a);fa(a,a.aoHeader);fa(a,a.aoFooter);C(a,!0);c.bAutoWidth&&Fa(a);b=0;for(c=e.length;b<c;b++)f=e[b],f.sWidth&&(f.nTh.style.width=v(f.sWidth));r(a,null,"preInit",[a]);T(a);e=
-y(a);if("ssp"!=e||g)"ajax"==e?sa(a,[],function(c){var f=ta(a,c);for(b=0;b<f.length;b++)O(a,f[b]);a.iInitDisplayStart=d;T(a);C(a,!1);ua(a,c)},a):(C(a,!1),ua(a))}else setTimeout(function(){ha(a)},200)}function ua(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&$(a);r(a,null,"plugin-init",[a,b]);r(a,"aoInitComplete","init",[a,b])}function Ra(a,b){var c=parseInt(b,10);a._iDisplayLength=c;Sa(a);r(a,null,"length",[a,c])}function ob(a){for(var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=h.isArray(d[0]),f=
-e?d[0]:d,d=e?d[1]:d,e=h("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect}),g=0,j=f.length;g<j;g++)e[0][g]=new Option("number"===typeof d[g]?a.fnFormatNumber(d[g]):d[g],f[g]);var i=h("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ra(a,h(this).val());P(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a===
-c&&h("select",i).val(d)});return i[0]}function tb(a){var b=a.sPaginationType,c=n.ext.pager[b],d="function"===typeof c,e=function(a){P(a)},b=h("<div/>").addClass(a.oClasses.sPaging+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;l<k;l++)Na(a,"pageButton")(a,f.p[l],l,h,b,i)}else c.fnUpdate(a,
-e)},sName:"pagination"}));return b}function Ta(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,f=a.fnRecordsDisplay();0===f||-1===e?d=0:"number"===typeof b?(d=b*e,d>f&&(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e<f&&(d+=e):"last"==b?d=Math.floor((f-1)/e)*e:K(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(r(a,null,"page",[a]),c&&P(a));return b}function qb(a){return h("<div/>",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}
-function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");r(a,null,"processing",[a,b])}function rb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),m=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("<div/>",{"class":f.sScrollWrapper}).append(h("<div/>",{"class":f.sScrollHead}).css({overflow:"hidden",
-position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("<div/>",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("<div/>",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("<div/>",
-{"class":f.sScrollFootInner}).append(m.removeAttr("id").css("margin-left",0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],t=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(t.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=t;a.aoDrawCallback.push({fn:la,sName:"scrolling"});return i[0]}function la(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth,
-f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,m=j.children("table"),j=a.nScrollBody,l=h(j),q=j.style,t=h(a.nScrollFoot).children("div"),n=t.children("table"),o=h(a.nTHead),p=h(a.nTable),s=p[0],r=s.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,Xb=D(a.aoColumns,"nTh"),Q,L,R,w,Ua=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!==
-L&&a.scrollBarVis!==k)a.scrollBarVis=L,$(a);else{a.scrollBarVis=L;p.children("thead, tfoot").remove();u&&(R=u.clone().prependTo(p),Q=u.find("tr"),R=R.find("tr"));w=o.clone().prependTo(p);o=o.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ra(a,w),function(b,c){B=aa(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},R);f=p.outerWidth();if(""===c){r.width="100%";if(U&&(p.find("tbody").height()>j.offsetHeight||
-"scroll"==l.css("overflow-y")))r.width=v(p.outerWidth()-b);f=p.outerWidth()}else""!==d&&(r.width=v(d),f=p.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Ua.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,Xb)!==-1)a.style.width=Ua[b]},o);h(L).height(0);u&&(I(C,R),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},R),I(function(a,b){a.style.width=y[b]},Q),h(R).height(0));I(function(a,b){a.innerHTML='<div class="dataTables_sizing">'+z[b]+"</div>";a.childNodes[0].style.height=
-"0";a.childNodes[0].style.overflow="hidden";a.style.width=Ua[b]},L);u&&I(function(a,b){a.innerHTML='<div class="dataTables_sizing">'+A[b]+"</div>";a.childNodes[0].style.height="0";a.childNodes[0].style.overflow="hidden";a.style.width=y[b]},R);if(p.outerWidth()<f){Q=j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))r.width=v(Q-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else Q="100%";q.width=v(Q);
-g.width=v(Q);u&&(a.nScrollFoot.style.width=v(Q));!e&&U&&(q.height=v(s.offsetHeight+b));c=p.outerWidth();m[0].style.width=v(c);i.width=v(c);d=p.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(n[0].style.width=v(c),t[0].style.width=v(c),t[0].style[e]=d?b+"px":"0px");p.children("colgroup").insertBefore(p.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0,
-f=b.length,g,j;e<f;){g=b[e].firstChild;for(j=c?c[e].firstChild:null;g;)1===g.nodeType&&(c?a(g,j,d):a(g,d),d++),g=g.nextSibling,j=c?j.nextSibling:null;e++}}function Fa(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,e=d.sY,f=d.sX,g=d.sXInner,j=c.length,i=ma(a,"bVisible"),m=h("th",a.nTHead),l=b.getAttribute("width"),k=b.parentNode,t=!1,n,o,p=a.oBrowser,d=p.bScrollOversize;(n=b.style.width)&&-1!==n.indexOf("%")&&(l=n);for(n=0;n<i.length;n++)o=c[i[n]],null!==o.sWidth&&(o.sWidth=Eb(o.sWidthOrig,k),t=!0);if(d||
-!t&&!f&&!e&&j==V(a)&&j==m.length)for(n=0;n<j;n++)i=aa(a,n),null!==i&&(c[i].sWidth=v(m.eq(n).width()));else{j=h(b).clone().css("visibility","hidden").removeAttr("id");j.find("tbody tr").remove();var s=h("<tr/>").appendTo(j.find("tbody"));j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");m=ra(a,j.find("thead")[0]);for(n=0;n<i.length;n++)o=c[i[n]],m[n].style.width=null!==o.sWidthOrig&&""!==o.sWidthOrig?v(o.sWidthOrig):
-"",o.sWidthOrig&&f&&h(m[n]).append(h("<div/>").css({width:o.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(n=0;n<i.length;n++)t=i[n],o=c[t],h(Fb(a,t)).clone(!1).append(o.sContentPadding).appendTo(s);h("[name]",j).removeAttr("name");o=h("<div/>").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()<k.clientWidth&&l&&j.width(k.clientWidth)):e?j.width(k.clientWidth):
-l&&j.width(l);for(n=e=0;n<i.length;n++)k=h(m[n]),g=k.outerWidth()-k.width(),k=p.bBounding?Math.ceil(m[n].getBoundingClientRect().width):k.outerWidth(),e+=k,c[i[n]].sWidth=v(k-g);b.style.width=v(e);o.remove()}l&&(b.style.width=v(l));if((l||f)&&!a._reszEvt)b=function(){h(E).on("resize.DT-"+a.sInstance,Oa(function(){$(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0}function Eb(a,b){if(!a)return 0;var c=h("<div/>").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Fb(a,
-b){var c=Gb(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("<td/>").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Gb(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;f<g;f++)c=B(a,f,b,"display")+"",c=c.replace(Yb,""),c=c.replace(/&nbsp;/g," "),c.length>d&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function X(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var m=[];f=function(a){a.length&&
-!h.isArray(a[0])?m.push(a):h.merge(m,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;a<m.length;a++){i=m[a][0];f=e[i].aDataSort;b=0;for(c=f.length;b<c;b++)g=f[b],j=e[g].sType||"string",m[a]._idx===k&&(m[a]._idx=h.inArray(m[a][1],e[g].asSorting)),d.push({src:i,col:g,dir:m[a][1],index:m[a]._idx,type:j,formatter:n.ext.type.order[j+"-pre"]})}return d}function mb(a){var b,c,d=[],e=n.ext.type.order,f=a.aoData,g=0,j,i=a.aiDisplayMaster,h;Ga(a);h=X(a);b=0;for(c=h.length;b<
-c;b++)j=h[b],j.formatter&&g++,Hb(a,j.col);if("ssp"!=y(a)&&0!==h.length){b=0;for(c=i.length;b<c;b++)d[i[b]]=b;g===h.length?i.sort(function(a,b){var c,e,g,j,i=h.length,k=f[a]._aSortData,n=f[b]._aSortData;for(g=0;g<i;g++)if(j=h[g],c=k[j.col],e=n[j.col],c=c<e?-1:c>e?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return c<e?-1:c>e?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,n=f[a]._aSortData,o=f[b]._aSortData;for(j=0;j<k;j++)if(i=h[j],c=n[i.col],g=o[i.col],i=e[i.type+"-"+i.dir]||e["string-"+i.dir],
-c=i(c,g),0!==c)return c;c=d[a];g=d[b];return c<g?-1:c>g?1:0})}a.bSorted=!0}function Ib(a){for(var b,c,d=a.aoColumns,e=X(a),a=a.oLanguage.oAria,f=0,g=d.length;f<g;f++){c=d[f];var j=c.asSorting;b=c.sTitle.replace(/<.*?>/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0<e.length&&e[0].col==f?(i.setAttribute("aria-sort","asc"==e[0].dir?"ascending":"descending"),c=j[e[0].index+1]||j[0]):c=j[0],b+="asc"===c?a.sSortAscending:a.sSortDescending);i.setAttribute("aria-label",b)}}function Va(a,
-b,c,d){var e=a.aaSorting,f=a.aoColumns[b].asSorting,g=function(a,b){var c=a._idx;c===k&&(c=h.inArray(a[1],f));return c+1<f.length?c+1:b?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=h.inArray(b,D(e,"0")),-1!==c?(b=g(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=f[b],e[c]._idx=b)):(e.push([b,f[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=g(e[0]),e.length=1,e[0][1]=f[b],e[0]._idx=b):(e.length=0,e.push([b,f[0]]),e[0]._idx=0);T(a);"function"==
-typeof d&&d(a)}function Ma(a,b,c,d){var e=a.aoColumns[c];Wa(b,{},function(b){!1!==e.bSortable&&(a.oFeatures.bProcessing?(C(a,!0),setTimeout(function(){Va(a,c,b.shiftKey,d);"ssp"!==y(a)&&C(a,!1)},0)):Va(a,c,b.shiftKey,d))})}function wa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=X(a),e=a.oFeatures,f,g;if(e.bSort&&e.bSortClasses){e=0;for(f=b.length;e<f;e++)g=b[e].src,h(D(a.aoData,"anCells",g)).removeClass(c+(2>e?e+1:3));e=0;for(f=d.length;e<f;e++)g=d[e].src,h(D(a.aoData,"anCells",g)).addClass(c+
-(2>e?e+1:3))}a.aLastSort=d}function Hb(a,b){var c=a.aoColumns[b],d=n.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ba(a,b)));for(var f,g=n.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j<i;j++)if(c=a.aoData[j],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)f=d?e[j]:B(a,j,b,"sort"),c._aSortData[b]=g?g(f):f}function xa(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,order:h.extend(!0,[],a.aaSorting),search:Ab(a.oPreviousSearch),
-columns:h.map(a.aoColumns,function(b,d){return{visible:b.bVisible,search:Ab(a.aoPreSearchCols[d])}})};r(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oSavedState=b;a.fnStateSaveCallback.call(a.oInstance,a,b)}}function Jb(a,b,c){var d,e,f=a.aoColumns,b=function(b){if(b&&b.time){var g=r(a,"aoStateLoadParams","stateLoadParams",[a,b]);if(-1===h.inArray(!1,g)&&(g=a.iStateDuration,!(0<g&&b.time<+new Date-1E3*g)&&!(b.columns&&f.length!==b.columns.length))){a.oLoadedState=h.extend(!0,{},b);b.start!==k&&
-(a._iDisplayStart=b.start,a.iInitDisplayStart=b.start);b.length!==k&&(a._iDisplayLength=b.length);b.order!==k&&(a.aaSorting=[],h.each(b.order,function(b,c){a.aaSorting.push(c[0]>=f.length?[0,c[1]]:c)}));b.search!==k&&h.extend(a.oPreviousSearch,Bb(b.search));if(b.columns){d=0;for(e=b.columns.length;d<e;d++)g=b.columns[d],g.visible!==k&&(f[d].bVisible=g.visible),g.search!==k&&h.extend(a.aoPreSearchCols[d],Bb(g.search))}r(a,"aoStateLoaded","stateLoaded",[a,b])}}c()};if(a.oFeatures.bStateSave){var g=
-a.fnStateLoadCallback.call(a.oInstance,a,b);g!==k&&b(g)}else c()}function ya(a){var b=n.settings,a=h.inArray(a,D(b,"nTable"));return-1!==a?b[a]:null}function K(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)E.console&&console.log&&console.log(c);else if(b=n.ext,b=b.sErrMode||b.errMode,a&&r(a,null,"error",[a,d,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==
-typeof b&&b(a,d,c)}}function F(a,b,c,d){h.isArray(c)?h.each(c,function(c,d){h.isArray(d)?F(a,b,d[0],d[1]):F(a,b,d)}):(d===k&&(d=c),b[c]!==k&&(a[d]=b[c]))}function Xa(a,b,c){var d,e;for(e in b)b.hasOwnProperty(e)&&(d=b[e],h.isPlainObject(d)?(h.isPlainObject(a[e])||(a[e]={}),h.extend(!0,a[e],d)):a[e]=c&&"data"!==e&&"aaData"!==e&&h.isArray(d)?d.slice():d);return a}function Wa(a,b,c){h(a).on("click.DT",b,function(b){h(a).blur();c(b)}).on("keypress.DT",b,function(a){13===a.which&&(a.preventDefault(),c(a))}).on("selectstart.DT",
-function(){return!1})}function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function r(a,b,c,d){var e=[];b&&(e=h.map(a[b].slice().reverse(),function(b){return b.fn.apply(a.oInstance,d)}));null!==c&&(b=h.Event(c+".dt"),h(a.nTable).trigger(b,d),e.push(b.result));return e}function Sa(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Na(a,b){var c=a.renderer,d=n.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"===
-typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Kb.numbers_length,d=Math.floor(c/2);b<=c?c=Y(0,b):a<=d?(c=Y(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=Y(b-(c-2),b):(c=Y(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function Da(a){h.each({num:function(b){return za(b,a)},"num-fmt":function(b){return za(b,a,Ya)},"html-num":function(b){return za(b,
-a,Aa)},"html-num-fmt":function(b){return za(b,a,Aa,Ya)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Lb(a){return function(){var b=[ya(this[n.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return n.ext.internal[a].apply(this,b)}}var n=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new s(ya(this[x.iApiIndex])):new s(this)};
-this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&la(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,
-b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data():
-c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};
-this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return ya(this[x.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust();
-(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in n.ext.internal)e&&(this[e]=Lb(e));this.each(function(){var e={},g=1<d?Xa(e,a,!0):a,j=0,i,e=this.getAttribute("id"),m=!1,l=n.defaults,q=h(this);if("table"!=this.nodeName.toLowerCase())K(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{eb(l);fb(l.column);J(l,l,!0);J(l.column,l.column,!0);J(l,h.extend(g,q.data()));var t=n.settings,
-j=0;for(i=t.length;j<i;j++){var o=t[j];if(o.nTable==this||o.nTHead&&o.nTHead.parentNode==this||o.nTFoot&&o.nTFoot.parentNode==this){var s=g.bRetrieve!==k?g.bRetrieve:l.bRetrieve;if(c||s)return o.oInstance;if(g.bDestroy!==k?g.bDestroy:l.bDestroy){o.oInstance.fnDestroy();break}else{K(o,0,"Cannot reinitialise DataTable",3);return}}if(o.sTableId==this.id){t.splice(j,1);break}}if(null===e||""===e)this.id=e="DataTables_Table_"+n.ext._unique++;var p=h.extend(!0,{},n.models.oSettings,{sDestroyWidth:q[0].style.width,
-sInstance:e,sTableId:e});p.nTable=this;p.oApi=b.internal;p.oInit=g;t.push(p);p.oInstance=1===b.length?b:q.dataTable();eb(g);Ca(g.oLanguage);g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=h.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]);g=Xa(h.extend(!0,{},l),g);F(p.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));F(p,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod",
-"aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"]]);F(p.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);F(p.oLanguage,g,"fnInfoCallback");
-z(p,"aoDrawCallback",g.fnDrawCallback,"user");z(p,"aoServerParams",g.fnServerParams,"user");z(p,"aoStateSaveParams",g.fnStateSaveParams,"user");z(p,"aoStateLoadParams",g.fnStateLoadParams,"user");z(p,"aoStateLoaded",g.fnStateLoaded,"user");z(p,"aoRowCallback",g.fnRowCallback,"user");z(p,"aoRowCreatedCallback",g.fnCreatedRow,"user");z(p,"aoHeaderCallback",g.fnHeaderCallback,"user");z(p,"aoFooterCallback",g.fnFooterCallback,"user");z(p,"aoInitComplete",g.fnInitComplete,"user");z(p,"aoPreDrawCallback",
-g.fnPreDrawCallback,"user");p.rowIdFn=S(g.rowId);gb(p);var u=p.oClasses;h.extend(u,n.ext.classes,g.oClasses);q.addClass(u.sTable);p.iInitDisplayStart===k&&(p.iInitDisplayStart=g.iDisplayStart,p._iDisplayStart=g.iDisplayStart);null!==g.iDeferLoading&&(p.bDeferLoading=!0,e=h.isArray(g.iDeferLoading),p._iRecordsDisplay=e?g.iDeferLoading[0]:g.iDeferLoading,p._iRecordsTotal=e?g.iDeferLoading[1]:g.iDeferLoading);var v=p.oLanguage;h.extend(!0,v,g.oLanguage);v.sUrl&&(h.ajax({dataType:"json",url:v.sUrl,success:function(a){Ca(a);
-J(l.oLanguage,a);h.extend(true,v,a);ha(p)},error:function(){ha(p)}}),m=!0);null===g.asStripeClasses&&(p.asStripeClasses=[u.sStripeOdd,u.sStripeEven]);var e=p.asStripeClasses,x=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(e,function(a){return x.hasClass(a)}))&&(h("tbody tr",this).removeClass(e.join(" ")),p.asDestroyStripes=e.slice());e=[];t=this.getElementsByTagName("thead");0!==t.length&&(ea(p.aoHeader,t[0]),e=ra(p));if(null===g.aoColumns){t=[];j=0;for(i=e.length;j<i;j++)t.push(null)}else t=
-g.aoColumns;j=0;for(i=t.length;j<i;j++)Ea(p,e?e[j]:null);ib(p,g.aoColumnDefs,t,function(a,b){ka(p,a,b)});if(x.length){var w=function(a,b){return a.getAttribute("data-"+b)!==null?b:null};h(x[0]).children("th, td").each(function(a,b){var c=p.aoColumns[a];if(c.mData===a){var d=w(b,"sort")||w(b,"order"),e=w(b,"filter")||w(b,"search");if(d!==null||e!==null){c.mData={_:a+".display",sort:d!==null?a+".@data-"+d:k,type:d!==null?a+".@data-"+d:k,filter:e!==null?a+".@data-"+e:k};ka(p,a)}}})}var U=p.oFeatures,
-e=function(){if(g.aaSorting===k){var a=p.aaSorting;j=0;for(i=a.length;j<i;j++)a[j][1]=p.aoColumns[j].asSorting[0]}wa(p);U.bSort&&z(p,"aoDrawCallback",function(){if(p.bSorted){var a=X(p),b={};h.each(a,function(a,c){b[c.src]=c.dir});r(p,null,"order",[p,a,b]);Ib(p)}});z(p,"aoDrawCallback",function(){(p.bSorted||y(p)==="ssp"||U.bDeferRender)&&wa(p)},"sc");var a=q.children("caption").each(function(){this._captionSide=h(this).css("caption-side")}),b=q.children("thead");b.length===0&&(b=h("<thead/>").appendTo(q));
-p.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("<tbody/>").appendTo(q));p.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(p.oScroll.sX!==""||p.oScroll.sY!==""))b=h("<tfoot/>").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){p.nTFoot=b[0];ea(p.aoFooter,p.nTFoot)}if(g.aaData)for(j=0;j<g.aaData.length;j++)O(p,g.aaData[j]);else(p.bDeferLoading||y(p)=="dom")&&na(p,h(p.nTBody).children("tr"));p.aiDisplay=p.aiDisplayMaster.slice();
-p.bInitialised=true;m===false&&ha(p)};g.bStateSave?(U.bStateSave=!0,z(p,"aoDrawCallback",xa,"state_save"),Jb(p,g,e)):e()}});b=null;return this},x,s,o,u,Za={},Mb=/[\r\n]/g,Aa=/<.*?>/g,Zb=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,$b=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Ya=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Nb=function(a){var b=parseInt(a,10);return!isNaN(b)&&
-isFinite(a)?b:null},Ob=function(a,b){Za[b]||(Za[b]=RegExp(Qa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Za[b],"."):a},$a=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Ob(a,b));c&&d&&(a=a.replace(Ya,""));return!isNaN(parseFloat(a))&&isFinite(a)},Pb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:$a(a.replace(Aa,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;e<f;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<
-f;e++)a[e]&&d.push(a[e][b]);return d},ja=function(a,b,c,d){var e=[],f=0,g=b.length;if(d!==k)for(;f<g;f++)a[b[f]][c]&&e.push(a[b[f]][c][d]);else for(;f<g;f++)e.push(a[b[f]][c]);return e},Y=function(a,b){var c=[],d;b===k?(b=0,d=a):(d=b,b=a);for(var e=b;e<d;e++)c.push(e);return c},Qb=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);return b},qa=function(a){var b;a:{if(!(2>a.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d<e;d++){if(b[d]===c){b=!1;break a}c=b[d]}}b=!0}if(b)return a.slice();
-b=[];var e=a.length,f,g=0,d=0;a:for(;d<e;d++){c=a[d];for(f=0;f<g;f++)if(b[f]===c)continue a;b.push(c);g++}return b};n.util={throttle:function(a,b){var c=b!==k?b:200,d,e;return function(){var b=this,g=+new Date,j=arguments;d&&g<d+c?(clearTimeout(e),e=setTimeout(function(){d=k;a.apply(b,j)},c)):(d=g,a.apply(b,j))}},escapeRegex:function(a){return a.replace($b,"\\$1")}};var A=function(a,b,c){a[b]!==k&&(a[c]=a[b])},ca=/\[.*?\]$/,W=/\(\)$/,Qa=n.util.escapeRegex,va=h("<div>")[0],Wb=va.textContent!==k,Yb=
-/<.*?>/g,Oa=n.util.throttle,Rb=[],w=Array.prototype,ac=function(a){var b,c,d=n.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};s=function(a,b){if(!(this instanceof
-s))return new s(a,b);var c=[],d=function(a){(a=ac(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;e<f;e++)d(a[e]);else d(a);this.context=qa(c);b&&h.merge(this,b);this.selector={rows:null,cols:null,opts:null};s.extend(this,this,Rb)};n.Api=s;h.extend(s.prototype,{any:function(){return 0!==this.count()},concat:w.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=
-this.context;return b.length>a?new s(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);return new s(this.context,b)},flatten:function(){var a=[];return new s(this.context,a.concat.apply(a,this.toArray()))},join:w.join,indexOf:w.indexOf||function(a,b){for(var c=b||0,d=this.length;c<d;c++)if(this[c]===a)return c;return-1},iterator:function(a,b,c,d){var e=[],f,g,j,h,m,l=this.context,
-n,o,u=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);g=0;for(j=l.length;g<j;g++){var r=new s(l[g]);if("table"===b)f=c.call(r,l[g],g),f!==k&&e.push(f);else if("columns"===b||"rows"===b)f=c.call(r,l[g],this[g],g),f!==k&&e.push(f);else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){o=this[g];"column-rows"===b&&(n=Ba(l[g],u.opts));h=0;for(m=o.length;h<m;h++)f=o[h],f="cell"===b?c.call(r,l[g],f.row,f.column,g,h):c.call(r,l[g],f,g,h,n),f!==k&&e.push(f)}}return e.length||d?(a=new s(l,a?
-e.concat.apply([],e):e),b=a.selector,b.rows=u.rows,b.cols=u.cols,b.opts=u.opts,a):this},lastIndexOf:w.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(a){var b=[];if(w.map)b=w.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new s(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:w.pop,push:w.push,reduce:w.reduce||function(a,b){return hb(this,a,b,0,this.length,
-1)},reduceRight:w.reduceRight||function(a,b){return hb(this,a,b,this.length-1,-1,-1)},reverse:w.reverse,selector:null,shift:w.shift,slice:function(){return new s(this.context,this)},sort:w.sort,splice:w.splice,toArray:function(){return w.slice.call(this)},to$:function(){return h(this)},toJQuery:function(){return h(this)},unique:function(){return new s(this.context,qa(this))},unshift:w.unshift});s.extend=function(a,b,c){if(c.length&&b&&(b instanceof s||b.__dt_wrapper)){var d,e,f,g=function(a,b,c){return function(){var d=
-b.apply(a,arguments);s.extend(d,d,c.methodExt);return d}};d=0;for(e=c.length;d<e;d++)f=c[d],b[f.name]="function"===typeof f.val?g(a,f.val,f):h.isPlainObject(f.val)?{}:f.val,b[f.name].__dt_wrapper=!0,s.extend(a,b[f.name],f.propExt)}};s.register=o=function(a,b){if(h.isArray(a))for(var c=0,d=a.length;c<d;c++)s.register(a[c],b);else for(var e=a.split("."),f=Rb,g,j,c=0,d=e.length;c<d;c++){g=(j=-1!==e[c].indexOf("()"))?e[c].replace("()",""):e[c];var i;a:{i=0;for(var m=f.length;i<m;i++)if(f[i].name===g){i=
-f[i];break a}i=null}i||(i={name:g,val:{},methodExt:[],propExt:[]},f.push(i));c===d-1?i.val=b:f=j?i.methodExt:i.propExt}};s.registerPlural=u=function(a,b,c){s.register(a,c);s.register(b,function(){var a=c.apply(this,arguments);return a===this?this:a instanceof s?a.length?h.isArray(a[0])?new s(a.context,a[0]):a[0]:k:a})};o("tables()",function(a){var b;if(a){b=s;var c=this.context;if("number"===typeof a)a=[c[a]];else var d=h.map(c,function(a){return a.nTable}),a=h(d).filter(a).map(function(){var a=h.inArray(this,
-d);return c[a]}).toArray();b=new b(a)}else b=this;return b});o("table()",function(a){var a=this.tables(a),b=a.context;return b.length?new s(b[0]):a});u("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});u("tables().body()","table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});u("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});u("tables().footer()",
-"table().footer()",function(){return this.iterator("table",function(a){return a.nTFoot},1)});u("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});o("draw()",function(a){return this.iterator("table",function(b){"page"===a?P(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),T(b,!1===a))})});o("page()",function(a){return a===k?this.page.info().page:this.iterator("table",function(b){Ta(b,a)})});o("page.info()",function(){if(0===
-this.context.length)return k;var a=this.context[0],b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:d,serverSide:"ssp"===y(a)}});o("page.len()",function(a){return a===k?0!==this.context.length?this.context[0]._iDisplayLength:k:this.iterator("table",function(b){Ra(b,a)})});var Sb=function(a,b,c){if(c){var d=new s(a);
-d.one("draw",function(){c(d.ajax.json())})}if("ssp"==y(a))T(a,b);else{C(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();sa(a,[],function(c){oa(a);for(var c=ta(a,c),d=0,e=c.length;d<e;d++)O(a,c[d]);T(a,b);C(a,!1)})}};o("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});o("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});o("ajax.reload()",function(a,b){return this.iterator("table",function(c){Sb(c,!1===b,a)})});o("ajax.url()",function(a){var b=
-this.context;if(a===k){if(0===b.length)return k;b=b[0];return b.ajax?h.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(b){h.isPlainObject(b.ajax)?b.ajax.url=a:b.ajax=a})});o("ajax.url().load()",function(a,b){return this.iterator("table",function(c){Sb(c,!1===b,a)})});var ab=function(a,b,c,d,e){var f=[],g,j,i,m,l,n;i=typeof b;if(!b||"string"===i||"function"===i||b.length===k)b=[b];i=0;for(m=b.length;i<m;i++){j=b[i]&&b[i].split&&!b[i].match(/[\[\(:]/)?b[i].split(","):
-[b[i]];l=0;for(n=j.length;l<n;l++)(g=c("string"===typeof j[l]?h.trim(j[l]):j[l]))&&g.length&&(f=f.concat(g))}a=x.selector[a];if(a.length){i=0;for(m=a.length;i<m;i++)f=a[i](d,e,f)}return qa(f)},bb=function(a){a||(a={});a.filter&&a.search===k&&(a.search=a.filter);return h.extend({search:"none",order:"current",page:"all"},a)},cb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Ba=function(a,b){var c,
-d,e,f=[],g=a.aiDisplay;e=a.aiDisplayMaster;var j=b.search;c=b.order;d=b.page;if("ssp"==y(a))return"removed"===j?[]:Y(0,e.length);if("current"==d){c=a._iDisplayStart;for(d=a.fnDisplayEnd();c<d;c++)f.push(g[c])}else if("current"==c||"applied"==c)if("none"==j)f=e.slice();else if("applied"==j)f=g.slice();else{if("removed"==j){var i={};c=0;for(d=g.length;c<d;c++)i[g[c]]=null;f=h.map(e,function(a){return!i.hasOwnProperty(a)?a:null})}}else if("index"==c||"original"==c){c=0;for(d=a.aoData.length;c<d;c++)"none"==
-j?f.push(c):(e=h.inArray(c,g),(-1===e&&"removed"==j||0<=e&&"applied"==j)&&f.push(c))}return f};o("rows()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=bb(b),c=this.iterator("table",function(c){var e=b,f;return ab("row",a,function(a){var b=Nb(a),i=c.aoData;if(b!==null&&!e)return[b];f||(f=Ba(c,e));if(b!==null&&h.inArray(b,f)!==-1)return[b];if(a===null||a===k||a==="")return f;if(typeof a==="function")return h.map(f,function(b){var c=i[b];return a(b,c._aData,c.nTr)?b:null});if(a.nodeName){var b=
-a._DT_RowIndex,m=a._DT_CellIndex;if(b!==k)return i[b]&&i[b].nTr===a?[b]:[];if(m)return i[m.row]&&i[m.row].nTr===a?[m.row]:[];b=h(a).closest("*[data-dt-row]");return b.length?[b.data("dt-row")]:[]}if(typeof a==="string"&&a.charAt(0)==="#"){b=c.aIds[a.replace(/^#/,"")];if(b!==k)return[b.idx]}b=Qb(ja(c.aoData,f,"nTr"));return h(b).filter(a).map(function(){return this._DT_RowIndex}).toArray()},c,e)},1);c.selector.rows=a;c.selector.opts=b;return c});o("rows().nodes()",function(){return this.iterator("row",
-function(a,b){return a.aoData[b].nTr||k},1)});o("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return ja(a.aoData,b,"_aData")},1)});u("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){var d=b.aoData[c];return"search"===a?d._aFilterData:d._aSortData},1)});u("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){da(b,c,a)})});u("rows().indexes()","row().index()",function(){return this.iterator("row",
-function(a,b){return b},1)});u("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,d=0,e=c.length;d<e;d++)for(var f=0,g=this[d].length;f<g;f++){var h=c[d].rowIdFn(c[d].aoData[this[d][f]]._aData);b.push((!0===a?"#":"")+h)}return new s(c,b)});u("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,f=e[c],g,h,i,m,l;e.splice(c,1);g=0;for(h=e.length;g<h;g++)if(i=e[g],l=i.anCells,null!==i.nTr&&(i.nTr._DT_RowIndex=g),null!==l){i=0;for(m=
-l.length;i<m;i++)l[i]._DT_CellIndex.row=g}pa(b.aiDisplayMaster,c);pa(b.aiDisplay,c);pa(a[d],c,!1);0<b._iRecordsDisplay&&b._iRecordsDisplay--;Sa(b);c=b.rowIdFn(f._aData);c!==k&&delete b.aIds[c]});this.iterator("table",function(a){for(var c=0,d=a.aoData.length;c<d;c++)a.aoData[c].idx=c});return this});o("rows.add()",function(a){var b=this.iterator("table",function(b){var c,f,g,h=[];f=0;for(g=a.length;f<g;f++)c=a[f],c.nodeName&&"TR"===c.nodeName.toUpperCase()?h.push(na(b,c)[0]):h.push(O(b,c));return h},
-1),c=this.rows(-1);c.pop();h.merge(c,b);return c});o("row()",function(a,b){return cb(this.rows(a,b))});o("row().data()",function(a){var b=this.context;if(a===k)return b.length&&this.length?b[0].aoData[this[0]]._aData:k;var c=b[0].aoData[this[0]];c._aData=a;h.isArray(a)&&c.nTr.id&&N(b[0].rowId)(a,c.nTr.id);da(b[0],this[0],"data");return this});o("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});o("row.add()",function(a){a instanceof h&&
-a.length&&(a=a[0]);var b=this.iterator("table",function(b){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?na(b,a)[0]:O(b,a)});return this.row(b[0])});var db=function(a,b){var c=a.context;if(c.length&&(c=c[0].aoData[b!==k?b:a[0]])&&c._details)c._details.remove(),c._detailsShow=k,c._details=k},Tb=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];if(d._details){(d._detailsShow=b)?d._details.insertAfter(d.nTr):d._details.detach();var e=c[0],f=new s(e),g=e.aoData;f.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");
-0<D(g,"_details").length&&(f.on("draw.dt.DT_details",function(a,b){e===b&&f.rows({page:"current"}).eq(0).each(function(a){a=g[a];a._detailsShow&&a._details.insertAfter(a.nTr)})}),f.on("column-visibility.dt.DT_details",function(a,b){if(e===b)for(var c,d=V(b),f=0,h=g.length;f<h;f++)c=g[f],c._details&&c._details.children("td[colspan]").attr("colspan",d)}),f.on("destroy.dt.DT_details",function(a,b){if(e===b)for(var c=0,d=g.length;c<d;c++)g[c]._details&&db(f,c)}))}}};o("row().child()",function(a,b){var c=
-this.context;if(a===k)return c.length&&this.length?c[0].aoData[this[0]]._details:k;if(!0===a)this.child.show();else if(!1===a)db(this);else if(c.length&&this.length){var d=c[0],c=c[0].aoData[this[0]],e=[],f=function(a,b){if(h.isArray(a)||a instanceof h)for(var c=0,k=a.length;c<k;c++)f(a[c],b);else a.nodeName&&"tr"===a.nodeName.toLowerCase()?e.push(a):(c=h("<tr><td/></tr>").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=V(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e);
-c._detailsShow&&c._details.insertAfter(c.nTr)}return this});o(["row().child.show()","row().child().show()"],function(){Tb(this,!0);return this});o(["row().child.hide()","row().child().hide()"],function(){Tb(this,!1);return this});o(["row().child.remove()","row().child().remove()"],function(){db(this);return this});o("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var bc=/^([^:]+):(name|visIdx|visible)$/,Ub=function(a,b,
-c,d,e){for(var c=[],d=0,f=e.length;d<f;d++)c.push(B(a,e[d],b));return c};o("columns()",function(a,b){a===k?a="":h.isPlainObject(a)&&(b=a,a="");var b=bb(b),c=this.iterator("table",function(c){var e=a,f=b,g=c.aoColumns,j=D(g,"sName"),i=D(g,"nTh");return ab("column",e,function(a){var b=Nb(a);if(a==="")return Y(g.length);if(b!==null)return[b>=0?b:g.length+b];if(typeof a==="function"){var e=Ba(c,f);return h.map(g,function(b,f){return a(f,Ub(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(bc):
-"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var n=h.map(g,function(a,b){return a.bVisible?b:null});return[n[n.length+b]]}return[aa(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)},
-1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Ub,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},
-1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData,
-i,m,l;if(a!==k&&g.bVisible!==a){if(a){var n=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(m=j.length;i<m;i++)l=j[i].nTr,f=j[i].anCells,l&&l.insertBefore(f[c],f[n]||null)}else h(D(b.aoData,"anCells",c)).detach();g.bVisible=a;fa(b,b.aoHeader);fa(b,b.aoFooter);b.aiDisplay.length||h(b.nTBody).find("td[colspan]").attr("colspan",V(b));xa(b)}});a!==k&&(this.iterator("column",function(c,e){r(c,null,"column-visibility",[c,e,a,b])}),(b===k||b)&&this.columns.adjust());return c});u("columns().indexes()","column().index()",
-function(a){return this.iterator("column",function(b,c){return"visible"===a?ba(b,c):c},1)});o("columns.adjust()",function(){return this.iterator("table",function(a){$(a)},1)});o("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return aa(c,b);if("fromData"===a||"toVisible"===a)return ba(c,b)}});o("column()",function(a,b){return cb(this.columns(a,b))});o("cells()",function(a,b,c){h.isPlainObject(a)&&(a.row===k?(c=a,a=null):(c=b,b=null));
-h.isPlainObject(b)&&(c=b,b=null);if(null===b||b===k)return this.iterator("table",function(b){var d=a,e=bb(c),f=b.aoData,g=Ba(b,e),j=Qb(ja(f,g,"anCells")),i=h([].concat.apply([],j)),l,m=b.aoColumns.length,n,o,u,s,r,v;return ab("cell",d,function(a){var c=typeof a==="function";if(a===null||a===k||c){n=[];o=0;for(u=g.length;o<u;o++){l=g[o];for(s=0;s<m;s++){r={row:l,column:s};if(c){v=f[l];a(r,B(b,l,s),v.anCells?v.anCells[s]:null)&&n.push(r)}else n.push(r)}}return n}if(h.isPlainObject(a))return a.column!==
-k&&a.row!==k&&h.inArray(a.row,g)!==-1?[a]:[];c=i.filter(a).map(function(a,b){return{row:b._DT_CellIndex.row,column:b._DT_CellIndex.column}}).toArray();if(c.length||!a.nodeName)return c;v=h(a).closest("*[data-dt-row]");return v.length?[{row:v.data("dt-row"),column:v.data("dt-column")}]:[]},b,e)});var d=this.columns(b),e=this.rows(a),f,g,j,i,m;this.iterator("table",function(a,b){f=[];g=0;for(j=e[b].length;g<j;g++){i=0;for(m=d[b].length;i<m;i++)f.push({row:e[b][g],column:d[b][i]})}},1);var l=this.cells(f,
-c);h.extend(l.selector,{cols:b,rows:a,opts:c});return l});u("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b])&&a.anCells?a.anCells[c]:k},1)});o("cells().data()",function(){return this.iterator("cell",function(a,b,c){return B(a,b,c)},1)});u("cells().cache()","cell().cache()",function(a){a="search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});u("cells().render()","cell().render()",
-function(a){return this.iterator("cell",function(b,c,d){return B(b,c,d,a)},1)});u("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:ba(a,c)}},1)});u("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(b,c,d){da(b,c,a,d)})});o("cell()",function(a,b,c){return cb(this.cells(a,b,c))});o("cell().data()",function(a){var b=this.context,c=this[0];if(a===k)return b.length&&c.length?B(b[0],
-c[0].row,c[0].column):k;jb(b[0],c[0].row,c[0].column,a);da(b[0],c[0].row,"data",c[0].column);return this});o("order()",function(a,b){var c=this.context;if(a===k)return 0!==c.length?c[0].aaSorting:k;"number"===typeof a?a=[[a,b]]:a.length&&!h.isArray(a[0])&&(a=Array.prototype.slice.call(arguments));return this.iterator("table",function(b){b.aaSorting=a.slice()})});o("order.listener()",function(a,b,c){return this.iterator("table",function(d){Ma(d,a,b,c)})});o("order.fixed()",function(a){if(!a){var b=
-this.context,b=b.length?b[0].aaSortingFixed:k;return h.isArray(b)?{pre:b}:b}return this.iterator("table",function(b){b.aaSortingFixed=h.extend(!0,{},a)})});o(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];h.each(b[d],function(b,c){e.push([c,a])});c.aaSorting=e})});o("search()",function(a,b,c,d){var e=this.context;return a===k?0!==e.length?e[0].oPreviousSearch.sSearch:k:this.iterator("table",function(e){e.oFeatures.bFilter&&ga(e,
-h.extend({},e.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});u("columns().search()","column().search()",function(a,b,c,d){return this.iterator("column",function(e,f){var g=e.aoPreSearchCols;if(a===k)return g[f].sSearch;e.oFeatures.bFilter&&(h.extend(g[f],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),ga(e,e.oPreviousSearch,1))})});o("state()",function(){return this.context.length?this.context[0].oSavedState:
-null});o("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});o("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});o("state.save()",function(){return this.iterator("table",function(a){xa(a)})});n.versionCheck=n.fnVersionCheck=function(a){for(var b=n.version.split("."),a=a.split("."),c,d,e=0,f=a.length;e<f;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};n.isDataTable=
-n.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof n.Api)return!0;h.each(n.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};n.tables=n.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(n.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new s(c):c};n.camelToHungarian=J;o("$()",function(a,b){var c=
-this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){o(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});o("clear()",function(){return this.iterator("table",function(a){oa(a)})});o("settings()",function(){return new s(this.context,this.context)});o("init()",function(){var a=
-this.context;return a.length?a[0].oInit:null});o("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});o("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),o;b.bDestroying=!0;r(b,"aoDestroyCallback","destroy",[b]);a||(new s(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT");
-h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];wa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable),
-(o=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%o])}));c=h.inArray(b,n.settings);-1!==c&&n.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){o(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});o("i18n()",function(a,b,c){var d=this.context[0],a=S(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:
-a._);return a.replace("%d",c)});n.version="1.10.18";n.settings=[];n.models={};n.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};n.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};n.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,
-sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};n.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,
-bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+
-a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
-oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},
-n.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};Z(n.defaults);n.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};
-Z(n.defaults.column);n.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],
-aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",
-iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal:
-this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};n.ext=x={buttons:{},
-classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:n.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:n.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager});
-h.extend(n.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",
-sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
-sJUIHeader:"",sJUIFooter:""});var Kb=n.ext.pager;h.extend(Kb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,n.ext.renderer,{pageButton:{_:function(a,b,c,d,e,
-f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,n=0,o=function(b,d){var k,s,u,r,v=function(b){Ta(a,b.data.action,true)};k=0;for(s=d.length;k<s;k++){r=d[k];if(h.isArray(r)){u=h("<"+(r.DT_el||"div")+"/>").appendTo(b);o(u,r)}else{m=null;l="";switch(r){case "ellipsis":b.append('<span class="ellipsis">&#x2026;</span>');break;case "first":m=j.sFirst;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m=
-j.sNext;l=r+(e<f-1?"":" "+g.sPageButtonDisabled);break;case "last":m=j.sLast;l=r+(e<f-1?"":" "+g.sPageButtonDisabled);break;default:m=r+1;l=e===r?g.sPageButtonActive:""}if(m!==null){u=h("<a>",{"class":g.sPageButton+" "+l,"aria-controls":a.sTableId,"aria-label":i[r],"data-dt-idx":n,tabindex:a.iTabIndex,id:c===0&&typeof r==="string"?a.sTableId+"_"+r:null}).html(m).appendTo(b);Wa(u,{action:r},v);n++}}}},s;try{s=h(b).find(H.activeElement).data("dt-idx")}catch(u){}o(h(b).empty(),d);s!==k&&h(b).find("[data-dt-idx="+
-s+"]").focus()}}});h.extend(n.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!Zb.test(a))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Pb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Pb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)||
-"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(n.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Mb," ").replace(Aa,""):""},string:function(a){return M(a)?a:"string"===typeof a?a.replace(Mb," "):a}});var za=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Ob(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){a=Date.parse(a);return isNaN(a)?-Infinity:a},"html-pre":function(a){return M(a)?
-"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<b?1:a>b?-1:0}});Da("");h.extend(!0,n.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:
-c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]==
-"asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var Vb=function(a){return"string"===typeof a?a.replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;"):a};n.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return Vb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g,
-a)+f+(e||"")}}},text:function(){return{display:Vb}}};h.extend(n.ext.internal,{_fnExternApiFunc:Lb,_fnBuildAjax:sa,_fnAjaxUpdate:lb,_fnAjaxParameters:ub,_fnAjaxUpdateDraw:vb,_fnAjaxDataSrc:ta,_fnAddColumn:Ea,_fnColumnOptions:ka,_fnAdjustColumnSizing:$,_fnVisibleToColumnIndex:aa,_fnColumnIndexToVisible:ba,_fnVisbleColumns:V,_fnGetColumns:ma,_fnColumnTypes:Ga,_fnApplyColumnDefs:ib,_fnHungarianMap:Z,_fnCamelToHungarian:J,_fnLanguageCompat:Ca,_fnBrowserDetect:gb,_fnAddData:O,_fnAddTr:na,_fnNodeToDataIndex:function(a,
-b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:jb,_fnSplitObjNotation:Ja,_fnGetObjectDataFn:S,_fnSetObjectDataFn:N,_fnGetDataMaster:Ka,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:da,_fnGetRowElements:Ia,_fnCreateTr:Ha,_fnBuildHead:kb,_fnDrawHead:fa,_fnDraw:P,_fnReDraw:T,_fnAddOptionsHtml:nb,_fnDetectHeader:ea,_fnGetUniqueThs:ra,_fnFeatureHtmlFilter:pb,_fnFilterComplete:ga,_fnFilterCustom:yb,
-_fnFilterColumn:xb,_fnFilter:wb,_fnFilterCreateSearch:Pa,_fnEscapeRegex:Qa,_fnFilterData:zb,_fnFeatureHtmlInfo:sb,_fnUpdateInfo:Cb,_fnInfoMacros:Db,_fnInitialise:ha,_fnInitComplete:ua,_fnLengthChange:Ra,_fnFeatureHtmlLength:ob,_fnFeatureHtmlPaginate:tb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:qb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:rb,_fnScrollDraw:la,_fnApplyToChildren:I,_fnCalculateColumnWidths:Fa,_fnThrottle:Oa,_fnConvertToWidth:Eb,_fnGetWidestNode:Fb,_fnGetMaxLenString:Gb,_fnStringToCss:v,
-_fnSortFlatten:X,_fnSort:mb,_fnSortAria:Ib,_fnSortListener:Va,_fnSortAttachListener:Ma,_fnSortingClasses:wa,_fnSortData:Hb,_fnSaveState:xa,_fnLoadState:Jb,_fnSettingsFromNode:ya,_fnLog:K,_fnMap:F,_fnBindAction:Wa,_fnCallbackReg:z,_fnCallbackFire:r,_fnLengthOverflow:Sa,_fnRenderer:Na,_fnDataSource:y,_fnRowAttributes:La,_fnExtend:Xa,_fnCalculateEnd:function(){}});h.fn.dataTable=n;n.$=h;h.fn.dataTableSettings=n.settings;h.fn.dataTableExt=n.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};
-h.each(n,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/custom_datatable.css
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/custom_datatable.css
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/demo_page.css
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_page.css
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/demo_page.css
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/demo_table.css
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/demo_table.css
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/demo_table.css
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css
similarity index 85%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css
index 88bf2f14f70f2..0420d964e86bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jquery.dataTables.css
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/jquery.dataTables.css
@@ -39,7 +39,7 @@ table.dataTable tfoot th {
 table.dataTable thead th,
 table.dataTable thead td {
   padding: 10px 18px;
-  border-bottom: 1px solid #111;
+  border-bottom: 1px solid #111111;
 }
 table.dataTable thead th:active,
 table.dataTable thead td:active {
@@ -48,15 +48,19 @@ table.dataTable thead td:active {
 table.dataTable tfoot th,
 table.dataTable tfoot td {
   padding: 10px 18px 6px 18px;
-  border-top: 1px solid #111;
+  border-top: 1px solid #111111;
+}
+table.dataTable thead .sorting,
+table.dataTable thead .sorting_asc,
+table.dataTable thead .sorting_desc {
+  cursor: pointer;
+  *cursor: hand;
 }
 table.dataTable thead .sorting,
 table.dataTable thead .sorting_asc,
 table.dataTable thead .sorting_desc,
 table.dataTable thead .sorting_asc_disabled,
 table.dataTable thead .sorting_desc_disabled {
-  cursor: pointer;
-  *cursor: hand;
   background-repeat: no-repeat;
   background-position: center right;
 }
@@ -76,17 +80,17 @@ table.dataTable thead .sorting_desc_disabled {
   background-image: url("../images/sort_desc_disabled.png");
 }
 table.dataTable tbody tr {
-  background-color: #ffffff;
+  background-color: white;
 }
 table.dataTable tbody tr.selected {
-  background-color: #B0BED9;
+  background-color: #b0bed9;
 }
 table.dataTable tbody th,
 table.dataTable tbody td {
   padding: 8px 10px;
 }
 table.dataTable.row-border tbody th, table.dataTable.row-border tbody td, table.dataTable.display tbody th, table.dataTable.display tbody td {
-  border-top: 1px solid #ddd;
+  border-top: 1px solid #dddddd;
 }
 table.dataTable.row-border tbody tr:first-child th,
 table.dataTable.row-border tbody tr:first-child td, table.dataTable.display tbody tr:first-child th,
@@ -94,12 +98,12 @@ table.dataTable.display tbody tr:first-child td {
   border-top: none;
 }
 table.dataTable.cell-border tbody th, table.dataTable.cell-border tbody td {
-  border-top: 1px solid #ddd;
-  border-right: 1px solid #ddd;
+  border-top: 1px solid #dddddd;
+  border-right: 1px solid #dddddd;
 }
 table.dataTable.cell-border tbody tr th:first-child,
 table.dataTable.cell-border tbody tr td:first-child {
-  border-left: 1px solid #ddd;
+  border-left: 1px solid #dddddd;
 }
 table.dataTable.cell-border tbody tr:first-child th,
 table.dataTable.cell-border tbody tr:first-child td {
@@ -109,27 +113,27 @@ table.dataTable.stripe tbody tr.odd, table.dataTable.display tbody tr.odd {
   background-color: #f9f9f9;
 }
 table.dataTable.stripe tbody tr.odd.selected, table.dataTable.display tbody tr.odd.selected {
-  background-color: #acbad4;
+  background-color: #abb9d3;
 }
 table.dataTable.hover tbody tr:hover, table.dataTable.display tbody tr:hover {
-  background-color: #f6f6f6;
+  background-color: whitesmoke;
 }
 table.dataTable.hover tbody tr:hover.selected, table.dataTable.display tbody tr:hover.selected {
-  background-color: #aab7d1;
+  background-color: #a9b7d1;
 }
 table.dataTable.order-column tbody tr > .sorting_1,
 table.dataTable.order-column tbody tr > .sorting_2,
 table.dataTable.order-column tbody tr > .sorting_3, table.dataTable.display tbody tr > .sorting_1,
 table.dataTable.display tbody tr > .sorting_2,
 table.dataTable.display tbody tr > .sorting_3 {
-  background-color: #fafafa;
+  background-color: #f9f9f9;
 }
 table.dataTable.order-column tbody tr.selected > .sorting_1,
 table.dataTable.order-column tbody tr.selected > .sorting_2,
 table.dataTable.order-column tbody tr.selected > .sorting_3, table.dataTable.display tbody tr.selected > .sorting_1,
 table.dataTable.display tbody tr.selected > .sorting_2,
 table.dataTable.display tbody tr.selected > .sorting_3 {
-  background-color: #acbad5;
+  background-color: #acbad4;
 }
 table.dataTable.display tbody tr.odd > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd > .sorting_1 {
   background-color: #f1f1f1;
@@ -141,28 +145,28 @@ table.dataTable.display tbody tr.odd > .sorting_3, table.dataTable.order-column.
   background-color: whitesmoke;
 }
 table.dataTable.display tbody tr.odd.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_1 {
-  background-color: #a6b4cd;
+  background-color: #a6b3cd;
 }
 table.dataTable.display tbody tr.odd.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_2 {
-  background-color: #a8b5cf;
+  background-color: #a7b5ce;
 }
 table.dataTable.display tbody tr.odd.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.odd.selected > .sorting_3 {
-  background-color: #a9b7d1;
+  background-color: #a9b6d0;
 }
 table.dataTable.display tbody tr.even > .sorting_1, table.dataTable.order-column.stripe tbody tr.even > .sorting_1 {
-  background-color: #fafafa;
+  background-color: #f9f9f9;
 }
 table.dataTable.display tbody tr.even > .sorting_2, table.dataTable.order-column.stripe tbody tr.even > .sorting_2 {
-  background-color: #fcfcfc;
+  background-color: #fbfbfb;
 }
 table.dataTable.display tbody tr.even > .sorting_3, table.dataTable.order-column.stripe tbody tr.even > .sorting_3 {
-  background-color: #fefefe;
+  background-color: #fdfdfd;
 }
 table.dataTable.display tbody tr.even.selected > .sorting_1, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_1 {
-  background-color: #acbad5;
+  background-color: #acbad4;
 }
 table.dataTable.display tbody tr.even.selected > .sorting_2, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_2 {
-  background-color: #aebcd6;
+  background-color: #adbbd6;
 }
 table.dataTable.display tbody tr.even.selected > .sorting_3, table.dataTable.order-column.stripe tbody tr.even.selected > .sorting_3 {
   background-color: #afbdd8;
@@ -171,22 +175,22 @@ table.dataTable.display tbody tr:hover > .sorting_1, table.dataTable.order-colum
   background-color: #eaeaea;
 }
 table.dataTable.display tbody tr:hover > .sorting_2, table.dataTable.order-column.hover tbody tr:hover > .sorting_2 {
-  background-color: #ececec;
+  background-color: #ebebeb;
 }
 table.dataTable.display tbody tr:hover > .sorting_3, table.dataTable.order-column.hover tbody tr:hover > .sorting_3 {
-  background-color: #efefef;
+  background-color: #eeeeee;
 }
 table.dataTable.display tbody tr:hover.selected > .sorting_1, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_1 {
-  background-color: #a2aec7;
+  background-color: #a1aec7;
 }
 table.dataTable.display tbody tr:hover.selected > .sorting_2, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_2 {
-  background-color: #a3b0c9;
+  background-color: #a2afc8;
 }
 table.dataTable.display tbody tr:hover.selected > .sorting_3, table.dataTable.order-column.hover tbody tr:hover.selected > .sorting_3 {
-  background-color: #a5b2cb;
+  background-color: #a4b2cb;
 }
 table.dataTable.no-footer {
-  border-bottom: 1px solid #111;
+  border-bottom: 1px solid #111111;
 }
 table.dataTable.nowrap th, table.dataTable.nowrap td {
   white-space: nowrap;
@@ -278,6 +282,7 @@ table.dataTable tbody td.dt-body-nowrap {
 table.dataTable,
 table.dataTable th,
 table.dataTable td {
+  -webkit-box-sizing: content-box;
   box-sizing: content-box;
 }
 
@@ -320,25 +325,25 @@ table.dataTable td {
   text-decoration: none !important;
   cursor: pointer;
   *cursor: hand;
-  color: #333 !important;
+  color: #333333 !important;
   border: 1px solid transparent;
   border-radius: 2px;
 }
 .dataTables_wrapper .dataTables_paginate .paginate_button.current, .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover {
-  color: #333 !important;
+  color: #333333 !important;
   border: 1px solid #979797;
   background-color: white;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, #dcdcdc));
+  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, white), color-stop(100%, gainsboro));
   /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, white 0%, #dcdcdc 100%);
+  background: -webkit-linear-gradient(top, white 0%, gainsboro 100%);
   /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, white 0%, #dcdcdc 100%);
+  background: -moz-linear-gradient(top, white 0%, gainsboro 100%);
   /* FF3.6+ */
-  background: -ms-linear-gradient(top, white 0%, #dcdcdc 100%);
+  background: -ms-linear-gradient(top, white 0%, gainsboro 100%);
   /* IE10+ */
-  background: -o-linear-gradient(top, white 0%, #dcdcdc 100%);
+  background: -o-linear-gradient(top, white 0%, gainsboro 100%);
   /* Opera 11.10+ */
-  background: linear-gradient(to bottom, white 0%, #dcdcdc 100%);
+  background: linear-gradient(to bottom, white 0%, gainsboro 100%);
   /* W3C */
 }
 .dataTables_wrapper .dataTables_paginate .paginate_button.disabled, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover, .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active {
@@ -350,19 +355,19 @@ table.dataTable td {
 }
 .dataTables_wrapper .dataTables_paginate .paginate_button:hover {
   color: white !important;
-  border: 1px solid #111;
+  border: 1px solid #111111;
   background-color: #585858;
-  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));
+  background: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111111));
   /* Chrome,Safari4+ */
-  background: -webkit-linear-gradient(top, #585858 0%, #111 100%);
+  background: -webkit-linear-gradient(top, #585858 0%, #111111 100%);
   /* Chrome10+,Safari5.1+ */
-  background: -moz-linear-gradient(top, #585858 0%, #111 100%);
+  background: -moz-linear-gradient(top, #585858 0%, #111111 100%);
   /* FF3.6+ */
-  background: -ms-linear-gradient(top, #585858 0%, #111 100%);
+  background: -ms-linear-gradient(top, #585858 0%, #111111 100%);
   /* IE10+ */
-  background: -o-linear-gradient(top, #585858 0%, #111 100%);
+  background: -o-linear-gradient(top, #585858 0%, #111111 100%);
   /* Opera 11.10+ */
-  background: linear-gradient(to bottom, #585858 0%, #111 100%);
+  background: linear-gradient(to bottom, #585858 0%, #111111 100%);
   /* W3C */
 }
 .dataTables_wrapper .dataTables_paginate .paginate_button:active {
@@ -409,7 +414,7 @@ table.dataTable td {
 .dataTables_wrapper .dataTables_info,
 .dataTables_wrapper .dataTables_processing,
 .dataTables_wrapper .dataTables_paginate {
-  color: #333;
+  color: #333333;
 }
 .dataTables_wrapper .dataTables_scroll {
   clear: both;
@@ -418,22 +423,21 @@ table.dataTable td {
   *margin-top: -1px;
   -webkit-overflow-scrolling: touch;
 }
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > td, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > td {
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td {
   vertical-align: middle;
 }
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > th > div.dataTables_sizing,
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > thead > tr > td > div.dataTables_sizing, .dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > th > div.dataTables_sizing,
-.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody > table > tbody > tr > td > div.dataTables_sizing {
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody th > div.dataTables_sizing,
+.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody td > div.dataTables_sizing {
   height: 0;
   overflow: hidden;
   margin: 0 !important;
   padding: 0 !important;
 }
 .dataTables_wrapper.no-footer .dataTables_scrollBody {
-  border-bottom: 1px solid #111;
+  border-bottom: 1px solid #111111;
 }
-.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,
-.dataTables_wrapper.no-footer div.dataTables_scrollBody > table {
+.dataTables_wrapper.no-footer div.dataTables_scrollHead table,
+.dataTables_wrapper.no-footer div.dataTables_scrollBody table {
   border-bottom: none;
 }
 .dataTables_wrapper:after {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/css/jui-dt.css
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/css/jui-dt.css
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/Sorting icons.psd
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/Sorting icons.psd
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/Sorting icons.psd
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/back_disabled.jpg
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_disabled.jpg
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/back_disabled.jpg
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/back_enabled.jpg
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/back_enabled.jpg
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/back_enabled.jpg
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/favicon.ico
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/favicon.ico
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/favicon.ico
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/forward_disabled.jpg
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_disabled.jpg
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/forward_disabled.jpg
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/forward_enabled.jpg
similarity index 100%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/images/forward_enabled.jpg
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/forward_enabled.jpg
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc.png
new file mode 100644
index 0000000000000..e1ba61a8055fc
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc.png differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc_disabled.png
new file mode 100644
index 0000000000000..fb11dfe24a6c5
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_asc_disabled.png differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_both.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_both.png
new file mode 100644
index 0000000000000..af5bc7c5a10b9
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_both.png differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc.png
new file mode 100644
index 0000000000000..0e156deb5f61d
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc.png differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc_disabled.png b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc_disabled.png
new file mode 100644
index 0000000000000..c9fdd8a1502fd
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/images/sort_desc_disabled.png differ
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/js/jquery.dataTables.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/js/jquery.dataTables.min.js
new file mode 100644
index 0000000000000..d61c9c99af0c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/js/jquery.dataTables.min.js
@@ -0,0 +1,187 @@
+/*!
+   Copyright 2008-2021 SpryMedia Ltd.
+
+ This source file is free software, available under the following license:
+   MIT license - http://datatables.net/license
+
+ This source file is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+
+ For details please refer to: http://www.datatables.net
+ DataTables 1.11.5
+ ©2008-2021 SpryMedia Ltd - datatables.net/license
+*/
+var $jscomp=$jscomp||{};$jscomp.scope={};$jscomp.findInternal=function(l,z,A){l instanceof String&&(l=String(l));for(var q=l.length,E=0;E<q;E++){var P=l[E];if(z.call(A,P,E,l))return{i:E,v:P}}return{i:-1,v:void 0}};$jscomp.ASSUME_ES5=!1;$jscomp.ASSUME_NO_NATIVE_MAP=!1;$jscomp.ASSUME_NO_NATIVE_SET=!1;$jscomp.SIMPLE_FROUND_POLYFILL=!1;$jscomp.ISOLATE_POLYFILLS=!1;
+$jscomp.defineProperty=$jscomp.ASSUME_ES5||"function"==typeof Object.defineProperties?Object.defineProperty:function(l,z,A){if(l==Array.prototype||l==Object.prototype)return l;l[z]=A.value;return l};$jscomp.getGlobal=function(l){l=["object"==typeof globalThis&&globalThis,l,"object"==typeof window&&window,"object"==typeof self&&self,"object"==typeof global&&global];for(var z=0;z<l.length;++z){var A=l[z];if(A&&A.Math==Math)return A}throw Error("Cannot find global object");};$jscomp.global=$jscomp.getGlobal(this);
+$jscomp.IS_SYMBOL_NATIVE="function"===typeof Symbol&&"symbol"===typeof Symbol("x");$jscomp.TRUST_ES6_POLYFILLS=!$jscomp.ISOLATE_POLYFILLS||$jscomp.IS_SYMBOL_NATIVE;$jscomp.polyfills={};$jscomp.propertyToPolyfillSymbol={};$jscomp.POLYFILL_PREFIX="$jscp$";var $jscomp$lookupPolyfilledValue=function(l,z){var A=$jscomp.propertyToPolyfillSymbol[z];if(null==A)return l[z];A=l[A];return void 0!==A?A:l[z]};
+$jscomp.polyfill=function(l,z,A,q){z&&($jscomp.ISOLATE_POLYFILLS?$jscomp.polyfillIsolated(l,z,A,q):$jscomp.polyfillUnisolated(l,z,A,q))};$jscomp.polyfillUnisolated=function(l,z,A,q){A=$jscomp.global;l=l.split(".");for(q=0;q<l.length-1;q++){var E=l[q];if(!(E in A))return;A=A[E]}l=l[l.length-1];q=A[l];z=z(q);z!=q&&null!=z&&$jscomp.defineProperty(A,l,{configurable:!0,writable:!0,value:z})};
+$jscomp.polyfillIsolated=function(l,z,A,q){var E=l.split(".");l=1===E.length;q=E[0];q=!l&&q in $jscomp.polyfills?$jscomp.polyfills:$jscomp.global;for(var P=0;P<E.length-1;P++){var ma=E[P];if(!(ma in q))return;q=q[ma]}E=E[E.length-1];A=$jscomp.IS_SYMBOL_NATIVE&&"es6"===A?q[E]:null;z=z(A);null!=z&&(l?$jscomp.defineProperty($jscomp.polyfills,E,{configurable:!0,writable:!0,value:z}):z!==A&&($jscomp.propertyToPolyfillSymbol[E]=$jscomp.IS_SYMBOL_NATIVE?$jscomp.global.Symbol(E):$jscomp.POLYFILL_PREFIX+E,
+E=$jscomp.propertyToPolyfillSymbol[E],$jscomp.defineProperty(q,E,{configurable:!0,writable:!0,value:z})))};$jscomp.polyfill("Array.prototype.find",function(l){return l?l:function(z,A){return $jscomp.findInternal(this,z,A).v}},"es6","es3");
+(function(l){"function"===typeof define&&define.amd?define(["jquery"],function(z){return l(z,window,document)}):"object"===typeof exports?module.exports=function(z,A){z||(z=window);A||(A="undefined"!==typeof window?require("jquery"):require("jquery")(z));return l(A,z,z.document)}:window.DataTable=l(jQuery,window,document)})(function(l,z,A,q){function E(a){var b,c,d={};l.each(a,function(e,h){(b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" ")&&(c=e.replace(b[0],
+b[2].toLowerCase()),d[c]=e,"o"===b[1]&&E(a[e]))});a._hungarianMap=d}function P(a,b,c){a._hungarianMap||E(a);var d;l.each(b,function(e,h){d=a._hungarianMap[e];d===q||!c&&b[d]!==q||("o"===d.charAt(0)?(b[d]||(b[d]={}),l.extend(!0,b[d],b[e]),P(a[d],b[d],c)):b[d]=b[e])})}function ma(a){var b=u.defaults.oLanguage,c=b.sDecimal;c&&Xa(c);if(a){var d=a.sZeroRecords;!a.sEmptyTable&&d&&"No data available in table"===b.sEmptyTable&&X(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&d&&"Loading..."===b.sLoadingRecords&&
+X(a,a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&c!==a&&Xa(a)}}function zb(a){S(a,"ordering","bSort");S(a,"orderMulti","bSortMulti");S(a,"orderClasses","bSortClasses");S(a,"orderCellsTop","bSortCellsTop");S(a,"order","aaSorting");S(a,"orderFixed","aaSortingFixed");S(a,"paging","bPaginate");S(a,"pagingType","sPaginationType");S(a,"pageLength","iDisplayLength");S(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%":
+"");"boolean"===typeof a.scrollX&&(a.scrollX=a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b<c;b++)a[b]&&P(u.models.oSearch,a[b])}function Ab(a){S(a,"orderable","bSortable");S(a,"orderData","aDataSort");S(a,"orderSequence","asSorting");S(a,"orderDataType","sortDataType");var b=a.aDataSort;"number"!==typeof b||Array.isArray(b)||(a.aDataSort=[b])}function Bb(a){if(!u.__browser){var b={};u.__browser=b;var c=l("<div/>").css({position:"fixed",top:0,left:-1*l(z).scrollLeft(),height:1,
+width:1,overflow:"hidden"}).append(l("<div/>").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(l("<div/>").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}l.extend(a.oBrowser,u.__browser);a.oScroll.iBarWidth=u.__browser.barWidth}
+function Cb(a,b,c,d,e,h){var f=!1;if(c!==q){var g=c;f=!0}for(;d!==e;)a.hasOwnProperty(d)&&(g=f?b(g,a[d],d,a):a[d],f=!0,d+=h);return g}function Ya(a,b){var c=u.defaults.column,d=a.aoColumns.length;c=l.extend({},u.models.oColumn,c,{nTh:b?b:A.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=l.extend({},u.models.oSearch,c[d]);Ga(a,d,l(b).data())}function Ga(a,b,c){b=a.aoColumns[b];
+var d=a.oClasses,e=l(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var h=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);h&&(b.sWidthOrig=h[1])}c!==q&&null!==c&&(Ab(c),P(u.defaults.column,c,!0),c.mDataProp===q||c.mData||(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),c.sClass&&e.addClass(c.sClass),l.extend(b,c),X(b,c,"sWidth","sWidthOrig"),c.iDataSort!==q&&(b.aDataSort=[c.iDataSort]),X(b,c,"aDataSort"));var f=b.mData,g=na(f),
+k=b.mRender?na(b.mRender):null;c=function(m){return"string"===typeof m&&-1!==m.indexOf("@")};b._bAttrSrc=l.isPlainObject(f)&&(c(f.sort)||c(f.type)||c(f.filter));b._setter=null;b.fnGetData=function(m,n,p){var t=g(m,n,q,p);return k&&n?k(t,n,m,p):t};b.fnSetData=function(m,n,p){return ha(f)(m,n,p)};"number"!==typeof f&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==l.inArray("asc",b.asSorting);c=-1!==l.inArray("desc",b.asSorting);b.bSortable&&(a||c)?a&&!c?
+(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI):(b.sSortingClass=d.sSortableNone,b.sSortingClassJUI="")}function sa(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Za(a);for(var c=0,d=b.length;c<d;c++)b[c].nTh.style.width=b[c].sWidth}b=a.oScroll;""===b.sY&&""===b.sX||Ha(a);F(a,null,"column-sizing",[a])}function ta(a,b){a=Ia(a,"bVisible");
+return"number"===typeof a[b]?a[b]:null}function ua(a,b){a=Ia(a,"bVisible");b=l.inArray(b,a);return-1!==b?b:null}function oa(a){var b=0;l.each(a.aoColumns,function(c,d){d.bVisible&&"none"!==l(d.nTh).css("display")&&b++});return b}function Ia(a,b){var c=[];l.map(a.aoColumns,function(d,e){d[b]&&c.push(e)});return c}function $a(a){var b=a.aoColumns,c=a.aoData,d=u.ext.type.detect,e,h,f;var g=0;for(e=b.length;g<e;g++){var k=b[g];var m=[];if(!k.sType&&k._sManualType)k.sType=k._sManualType;else if(!k.sType){var n=
+0;for(h=d.length;n<h;n++){var p=0;for(f=c.length;p<f;p++){m[p]===q&&(m[p]=T(a,p,g,"type"));var t=d[n](m[p],a);if(!t&&n!==d.length-1)break;if("html"===t&&!Z(m[p]))break}if(t){k.sType=t;break}}k.sType||(k.sType="string")}}}function Db(a,b,c,d){var e,h,f,g=a.aoColumns;if(b)for(e=b.length-1;0<=e;e--){var k=b[e];var m=k.targets!==q?k.targets:k.aTargets;Array.isArray(m)||(m=[m]);var n=0;for(h=m.length;n<h;n++)if("number"===typeof m[n]&&0<=m[n]){for(;g.length<=m[n];)Ya(a);d(m[n],k)}else if("number"===typeof m[n]&&
+0>m[n])d(g.length+m[n],k);else if("string"===typeof m[n]){var p=0;for(f=g.length;p<f;p++)("_all"==m[n]||l(g[p].nTh).hasClass(m[n]))&&d(p,k)}}if(c)for(e=0,a=c.length;e<a;e++)d(e,c[e])}function ia(a,b,c,d){var e=a.aoData.length,h=l.extend(!0,{},u.models.oRow,{src:c?"dom":"data",idx:e});h._aData=b;a.aoData.push(h);for(var f=a.aoColumns,g=0,k=f.length;g<k;g++)f[g].sType=null;a.aiDisplayMaster.push(e);b=a.rowIdFn(b);b!==q&&(a.aIds[b]=h);!c&&a.oFeatures.bDeferRender||ab(a,e,c,d);return e}function Ja(a,
+b){var c;b instanceof l||(b=l(b));return b.map(function(d,e){c=bb(a,e);return ia(a,c.data,e,c.cells)})}function T(a,b,c,d){"search"===d?d="filter":"order"===d&&(d="sort");var e=a.iDraw,h=a.aoColumns[c],f=a.aoData[b]._aData,g=h.sDefaultContent,k=h.fnGetData(f,d,{settings:a,row:b,col:c});if(k===q)return a.iDrawError!=e&&null===g&&(da(a,0,"Requested unknown parameter "+("function"==typeof h.mData?"{function}":"'"+h.mData+"'")+" for row "+b+", column "+c,4),a.iDrawError=e),g;if((k===f||null===k)&&null!==
+g&&d!==q)k=g;else if("function"===typeof k)return k.call(f);if(null===k&&"display"===d)return"";"filter"===d&&(a=u.ext.type.search,a[h.sType]&&(k=a[h.sType](k)));return k}function Eb(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d,{settings:a,row:b,col:c})}function cb(a){return l.map(a.match(/(\\.|[^\.])+/g)||[""],function(b){return b.replace(/\\\./g,".")})}function db(a){return U(a.aoData,"_aData")}function Ka(a){a.aoData.length=0;a.aiDisplayMaster.length=0;a.aiDisplay.length=0;a.aIds={}}
+function La(a,b,c){for(var d=-1,e=0,h=a.length;e<h;e++)a[e]==b?d=e:a[e]>b&&a[e]--; -1!=d&&c===q&&a.splice(d,1)}function va(a,b,c,d){var e=a.aoData[b],h,f=function(k,m){for(;k.childNodes.length;)k.removeChild(k.firstChild);k.innerHTML=T(a,b,m,"display")};if("dom"!==c&&(c&&"auto"!==c||"dom"!==e.src)){var g=e.anCells;if(g)if(d!==q)f(g[d],d);else for(c=0,h=g.length;c<h;c++)f(g[c],c)}else e._aData=bb(a,e,d,d===q?q:e._aData).data;e._aSortData=null;e._aFilterData=null;f=a.aoColumns;if(d!==q)f[d].sType=null;
+else{c=0;for(h=f.length;c<h;c++)f[c].sType=null;eb(a,e)}}function bb(a,b,c,d){var e=[],h=b.firstChild,f,g=0,k,m=a.aoColumns,n=a._rowReadObject;d=d!==q?d:n?{}:[];var p=function(x,w){if("string"===typeof x){var r=x.indexOf("@");-1!==r&&(r=x.substring(r+1),ha(x)(d,w.getAttribute(r)))}},t=function(x){if(c===q||c===g)f=m[g],k=x.innerHTML.trim(),f&&f._bAttrSrc?(ha(f.mData._)(d,k),p(f.mData.sort,x),p(f.mData.type,x),p(f.mData.filter,x)):n?(f._setter||(f._setter=ha(f.mData)),f._setter(d,k)):d[g]=k;g++};if(h)for(;h;){var v=
+h.nodeName.toUpperCase();if("TD"==v||"TH"==v)t(h),e.push(h);h=h.nextSibling}else for(e=b.anCells,h=0,v=e.length;h<v;h++)t(e[h]);(b=b.firstChild?b:b.nTr)&&(b=b.getAttribute("id"))&&ha(a.rowId)(d,b);return{data:d,cells:e}}function ab(a,b,c,d){var e=a.aoData[b],h=e._aData,f=[],g,k;if(null===e.nTr){var m=c||A.createElement("tr");e.nTr=m;e.anCells=f;m._DT_RowIndex=b;eb(a,e);var n=0;for(g=a.aoColumns.length;n<g;n++){var p=a.aoColumns[n];e=(k=c?!1:!0)?A.createElement(p.sCellType):d[n];e._DT_CellIndex={row:b,
+column:n};f.push(e);if(k||!(!p.mRender&&p.mData===n||l.isPlainObject(p.mData)&&p.mData._===n+".display"))e.innerHTML=T(a,b,n,"display");p.sClass&&(e.className+=" "+p.sClass);p.bVisible&&!c?m.appendChild(e):!p.bVisible&&c&&e.parentNode.removeChild(e);p.fnCreatedCell&&p.fnCreatedCell.call(a.oInstance,e,T(a,b,n),h,b,n)}F(a,"aoRowCreatedCallback",null,[m,h,b,f])}}function eb(a,b){var c=b.nTr,d=b._aData;if(c){if(a=a.rowIdFn(d))c.id=a;d.DT_RowClass&&(a=d.DT_RowClass.split(" "),b.__rowc=b.__rowc?Ma(b.__rowc.concat(a)):
+a,l(c).removeClass(b.__rowc.join(" ")).addClass(d.DT_RowClass));d.DT_RowAttr&&l(c).attr(d.DT_RowAttr);d.DT_RowData&&l(c).data(d.DT_RowData)}}function Fb(a){var b,c,d=a.nTHead,e=a.nTFoot,h=0===l("th, td",d).length,f=a.oClasses,g=a.aoColumns;h&&(c=l("<tr/>").appendTo(d));var k=0;for(b=g.length;k<b;k++){var m=g[k];var n=l(m.nTh).addClass(m.sClass);h&&n.appendTo(c);a.oFeatures.bSort&&(n.addClass(m.sSortingClass),!1!==m.bSortable&&(n.attr("tabindex",a.iTabIndex).attr("aria-controls",a.sTableId),fb(a,m.nTh,
+k)));m.sTitle!=n[0].innerHTML&&n.html(m.sTitle);gb(a,"header")(a,n,m,f)}h&&wa(a.aoHeader,d);l(d).children("tr").children("th, td").addClass(f.sHeaderTH);l(e).children("tr").children("th, td").addClass(f.sFooterTH);if(null!==e)for(a=a.aoFooter[0],k=0,b=a.length;k<b;k++)m=g[k],m.nTf=a[k].cell,m.sClass&&l(m.nTf).addClass(m.sClass)}function xa(a,b,c){var d,e,h=[],f=[],g=a.aoColumns.length;if(b){c===q&&(c=!1);var k=0;for(d=b.length;k<d;k++){h[k]=b[k].slice();h[k].nTr=b[k].nTr;for(e=g-1;0<=e;e--)a.aoColumns[e].bVisible||
+c||h[k].splice(e,1);f.push([])}k=0;for(d=h.length;k<d;k++){if(a=h[k].nTr)for(;e=a.firstChild;)a.removeChild(e);e=0;for(b=h[k].length;e<b;e++){var m=g=1;if(f[k][e]===q){a.appendChild(h[k][e].cell);for(f[k][e]=1;h[k+g]!==q&&h[k][e].cell==h[k+g][e].cell;)f[k+g][e]=1,g++;for(;h[k][e+m]!==q&&h[k][e].cell==h[k][e+m].cell;){for(c=0;c<g;c++)f[k+c][e+m]=1;m++}l(h[k][e].cell).attr("rowspan",g).attr("colspan",m)}}}}}function ja(a,b){var c="ssp"==Q(a),d=a.iInitDisplayStart;d!==q&&-1!==d&&(a._iDisplayStart=c?
+d:d>=a.fnRecordsDisplay()?0:d,a.iInitDisplayStart=-1);c=F(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==l.inArray(!1,c))V(a,!1);else{c=[];var e=0;d=a.asStripeClasses;var h=d.length,f=a.oLanguage,g="ssp"==Q(a),k=a.aiDisplay,m=a._iDisplayStart,n=a.fnDisplayEnd();a.bDrawing=!0;if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,V(a,!1);else if(!g)a.iDraw++;else if(!a.bDestroying&&!b){Gb(a);return}if(0!==k.length)for(b=g?a.aoData.length:n,f=g?0:m;f<b;f++){g=k[f];var p=a.aoData[g];null===p.nTr&&ab(a,g);var t=
+p.nTr;if(0!==h){var v=d[e%h];p._sRowStripe!=v&&(l(t).removeClass(p._sRowStripe).addClass(v),p._sRowStripe=v)}F(a,"aoRowCallback",null,[t,p._aData,e,f,g]);c.push(t);e++}else e=f.sZeroRecords,1==a.iDraw&&"ajax"==Q(a)?e=f.sLoadingRecords:f.sEmptyTable&&0===a.fnRecordsTotal()&&(e=f.sEmptyTable),c[0]=l("<tr/>",{"class":h?d[0]:""}).append(l("<td />",{valign:"top",colSpan:oa(a),"class":a.oClasses.sRowEmpty}).html(e))[0];F(a,"aoHeaderCallback","header",[l(a.nTHead).children("tr")[0],db(a),m,n,k]);F(a,"aoFooterCallback",
+"footer",[l(a.nTFoot).children("tr")[0],db(a),m,n,k]);d=l(a.nTBody);d.children().detach();d.append(l(c));F(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function ka(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&Hb(a);d?ya(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;ja(a);a._drawHold=!1}function Ib(a){var b=a.oClasses,c=l(a.nTable);c=l("<div/>").insertBefore(c);var d=a.oFeatures,e=l("<div/>",{id:a.sTableId+"_wrapper",
+"class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var h=a.sDom.split(""),f,g,k,m,n,p,t=0;t<h.length;t++){f=null;g=h[t];if("<"==g){k=l("<div/>")[0];m=h[t+1];if("'"==m||'"'==m){n="";for(p=2;h[t+p]!=m;)n+=h[t+p],p++;"H"==n?n=b.sJUIHeader:"F"==n&&(n=b.sJUIFooter);-1!=n.indexOf(".")?(m=n.split("."),k.id=m[0].substr(1,m[0].length-1),k.className=m[1]):"#"==n.charAt(0)?k.id=n.substr(1,n.length-1):k.className=n;t+=p}e.append(k);
+e=l(k)}else if(">"==g)e=e.parent();else if("l"==g&&d.bPaginate&&d.bLengthChange)f=Jb(a);else if("f"==g&&d.bFilter)f=Kb(a);else if("r"==g&&d.bProcessing)f=Lb(a);else if("t"==g)f=Mb(a);else if("i"==g&&d.bInfo)f=Nb(a);else if("p"==g&&d.bPaginate)f=Ob(a);else if(0!==u.ext.feature.length)for(k=u.ext.feature,p=0,m=k.length;p<m;p++)if(g==k[p].cFeature){f=k[p].fnInit(a);break}f&&(k=a.aanFeatures,k[g]||(k[g]=[]),k[g].push(f),e.append(f))}c.replaceWith(e);a.nHolding=null}function wa(a,b){b=l(b).children("tr");
+var c,d,e;a.splice(0,a.length);var h=0;for(e=b.length;h<e;h++)a.push([]);h=0;for(e=b.length;h<e;h++){var f=b[h];for(c=f.firstChild;c;){if("TD"==c.nodeName.toUpperCase()||"TH"==c.nodeName.toUpperCase()){var g=1*c.getAttribute("colspan");var k=1*c.getAttribute("rowspan");g=g&&0!==g&&1!==g?g:1;k=k&&0!==k&&1!==k?k:1;var m=0;for(d=a[h];d[m];)m++;var n=m;var p=1===g?!0:!1;for(d=0;d<g;d++)for(m=0;m<k;m++)a[h+m][n+d]={cell:c,unique:p},a[h+m].nTr=f}c=c.nextSibling}}}function Na(a,b,c){var d=[];c||(c=a.aoHeader,
+b&&(c=[],wa(c,b)));b=0;for(var e=c.length;b<e;b++)for(var h=0,f=c[b].length;h<f;h++)!c[b][h].unique||d[h]&&a.bSortCellsTop||(d[h]=c[b][h].cell);return d}function Oa(a,b,c){F(a,"aoServerParams","serverParams",[b]);if(b&&Array.isArray(b)){var d={},e=/(.*?)\[\]$/;l.each(b,function(n,p){(n=p.name.match(e))?(n=n[0],d[n]||(d[n]=[]),d[n].push(p.value)):d[p.name]=p.value});b=d}var h=a.ajax,f=a.oInstance,g=function(n){var p=a.jqXHR?a.jqXHR.status:null;if(null===n||"number"===typeof p&&204==p)n={},za(a,n,[]);
+(p=n.error||n.sError)&&da(a,0,p);a.json=n;F(a,null,"xhr",[a,n,a.jqXHR]);c(n)};if(l.isPlainObject(h)&&h.data){var k=h.data;var m="function"===typeof k?k(b,a):k;b="function"===typeof k&&m?m:l.extend(!0,b,m);delete h.data}m={data:b,success:g,dataType:"json",cache:!1,type:a.sServerMethod,error:function(n,p,t){t=F(a,null,"xhr",[a,null,a.jqXHR]);-1===l.inArray(!0,t)&&("parsererror"==p?da(a,0,"Invalid JSON response",1):4===n.readyState&&da(a,0,"Ajax error",7));V(a,!1)}};a.oAjaxData=b;F(a,null,"preXhr",[a,
+b]);a.fnServerData?a.fnServerData.call(f,a.sAjaxSource,l.map(b,function(n,p){return{name:p,value:n}}),g,a):a.sAjaxSource||"string"===typeof h?a.jqXHR=l.ajax(l.extend(m,{url:h||a.sAjaxSource})):"function"===typeof h?a.jqXHR=h.call(f,b,g,a):(a.jqXHR=l.ajax(l.extend(m,h)),h.data=k)}function Gb(a){a.iDraw++;V(a,!0);Oa(a,Pb(a),function(b){Qb(a,b)})}function Pb(a){var b=a.aoColumns,c=b.length,d=a.oFeatures,e=a.oPreviousSearch,h=a.aoPreSearchCols,f=[],g=pa(a);var k=a._iDisplayStart;var m=!1!==d.bPaginate?
+a._iDisplayLength:-1;var n=function(x,w){f.push({name:x,value:w})};n("sEcho",a.iDraw);n("iColumns",c);n("sColumns",U(b,"sName").join(","));n("iDisplayStart",k);n("iDisplayLength",m);var p={draw:a.iDraw,columns:[],order:[],start:k,length:m,search:{value:e.sSearch,regex:e.bRegex}};for(k=0;k<c;k++){var t=b[k];var v=h[k];m="function"==typeof t.mData?"function":t.mData;p.columns.push({data:m,name:t.sName,searchable:t.bSearchable,orderable:t.bSortable,search:{value:v.sSearch,regex:v.bRegex}});n("mDataProp_"+
+k,m);d.bFilter&&(n("sSearch_"+k,v.sSearch),n("bRegex_"+k,v.bRegex),n("bSearchable_"+k,t.bSearchable));d.bSort&&n("bSortable_"+k,t.bSortable)}d.bFilter&&(n("sSearch",e.sSearch),n("bRegex",e.bRegex));d.bSort&&(l.each(g,function(x,w){p.order.push({column:w.col,dir:w.dir});n("iSortCol_"+x,w.col);n("sSortDir_"+x,w.dir)}),n("iSortingCols",g.length));b=u.ext.legacy.ajax;return null===b?a.sAjaxSource?f:p:b?f:p}function Qb(a,b){var c=function(f,g){return b[f]!==q?b[f]:b[g]},d=za(a,b),e=c("sEcho","draw"),h=
+c("iTotalRecords","recordsTotal");c=c("iTotalDisplayRecords","recordsFiltered");if(e!==q){if(1*e<a.iDraw)return;a.iDraw=1*e}d||(d=[]);Ka(a);a._iRecordsTotal=parseInt(h,10);a._iRecordsDisplay=parseInt(c,10);e=0;for(h=d.length;e<h;e++)ia(a,d[e]);a.aiDisplay=a.aiDisplayMaster.slice();ja(a,!0);a._bInitComplete||Pa(a,b);V(a,!1)}function za(a,b,c){a=l.isPlainObject(a.ajax)&&a.ajax.dataSrc!==q?a.ajax.dataSrc:a.sAjaxDataProp;if(!c)return"data"===a?b.aaData||b[a]:""!==a?na(a)(b):b;ha(a)(b,c)}function Kb(a){var b=
+a.oClasses,c=a.sTableId,d=a.oLanguage,e=a.oPreviousSearch,h=a.aanFeatures,f='<input type="search" class="'+b.sFilterInput+'"/>',g=d.sSearch;g=g.match(/_INPUT_/)?g.replace("_INPUT_",f):g+f;b=l("<div/>",{id:h.f?null:c+"_filter","class":b.sFilter}).append(l("<label/>").append(g));var k=function(n){var p=this.value?this.value:"";e.return&&"Enter"!==n.key||p==e.sSearch||(ya(a,{sSearch:p,bRegex:e.bRegex,bSmart:e.bSmart,bCaseInsensitive:e.bCaseInsensitive,"return":e.return}),a._iDisplayStart=0,ja(a))};h=
+null!==a.searchDelay?a.searchDelay:"ssp"===Q(a)?400:0;var m=l("input",b).val(e.sSearch).attr("placeholder",d.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",h?hb(k,h):k).on("mouseup",function(n){setTimeout(function(){k.call(m[0],n)},10)}).on("keypress.DT",function(n){if(13==n.keyCode)return!1}).attr("aria-controls",c);l(a.nTable).on("search.dt.DT",function(n,p){if(a===p)try{m[0]!==A.activeElement&&m.val(e.sSearch)}catch(t){}});return b[0]}function ya(a,b,c){var d=a.oPreviousSearch,
+e=a.aoPreSearchCols,h=function(g){d.sSearch=g.sSearch;d.bRegex=g.bRegex;d.bSmart=g.bSmart;d.bCaseInsensitive=g.bCaseInsensitive;d.return=g.return},f=function(g){return g.bEscapeRegex!==q?!g.bEscapeRegex:g.bRegex};$a(a);if("ssp"!=Q(a)){Rb(a,b.sSearch,c,f(b),b.bSmart,b.bCaseInsensitive,b.return);h(b);for(b=0;b<e.length;b++)Sb(a,e[b].sSearch,b,f(e[b]),e[b].bSmart,e[b].bCaseInsensitive);Tb(a)}else h(b);a.bFiltered=!0;F(a,null,"search",[a])}function Tb(a){for(var b=u.ext.search,c=a.aiDisplay,d,e,h=0,f=
+b.length;h<f;h++){for(var g=[],k=0,m=c.length;k<m;k++)e=c[k],d=a.aoData[e],b[h](a,d._aFilterData,e,d._aData,k)&&g.push(e);c.length=0;l.merge(c,g)}}function Sb(a,b,c,d,e,h){if(""!==b){var f=[],g=a.aiDisplay;d=ib(b,d,e,h);for(e=0;e<g.length;e++)b=a.aoData[g[e]]._aFilterData[c],d.test(b)&&f.push(g[e]);a.aiDisplay=f}}function Rb(a,b,c,d,e,h){e=ib(b,d,e,h);var f=a.oPreviousSearch.sSearch,g=a.aiDisplayMaster;h=[];0!==u.ext.search.length&&(c=!0);var k=Ub(a);if(0>=b.length)a.aiDisplay=g.slice();else{if(k||
+c||d||f.length>b.length||0!==b.indexOf(f)||a.bSorted)a.aiDisplay=g.slice();b=a.aiDisplay;for(c=0;c<b.length;c++)e.test(a.aoData[b[c]]._sFilterRow)&&h.push(b[c]);a.aiDisplay=h}}function ib(a,b,c,d){a=b?a:jb(a);c&&(a="^(?=.*?"+l.map(a.match(/"[^"]+"|[^ ]+/g)||[""],function(e){if('"'===e.charAt(0)){var h=e.match(/^"(.*)"$/);e=h?h[1]:e}return e.replace('"',"")}).join(")(?=.*?")+").*$");return new RegExp(a,d?"i":"")}function Ub(a){var b=a.aoColumns,c,d;var e=!1;var h=0;for(c=a.aoData.length;h<c;h++){var f=
+a.aoData[h];if(!f._aFilterData){var g=[];e=0;for(d=b.length;e<d;e++){var k=b[e];k.bSearchable?(k=T(a,h,e,"filter"),null===k&&(k=""),"string"!==typeof k&&k.toString&&(k=k.toString())):k="";k.indexOf&&-1!==k.indexOf("&")&&(Qa.innerHTML=k,k=tc?Qa.textContent:Qa.innerText);k.replace&&(k=k.replace(/[\r\n\u2028]/g,""));g.push(k)}f._aFilterData=g;f._sFilterRow=g.join("  ");e=!0}}return e}function Vb(a){return{search:a.sSearch,smart:a.bSmart,regex:a.bRegex,caseInsensitive:a.bCaseInsensitive}}function Wb(a){return{sSearch:a.search,
+bSmart:a.smart,bRegex:a.regex,bCaseInsensitive:a.caseInsensitive}}function Nb(a){var b=a.sTableId,c=a.aanFeatures.i,d=l("<div/>",{"class":a.oClasses.sInfo,id:c?null:b+"_info"});c||(a.aoDrawCallback.push({fn:Xb,sName:"information"}),d.attr("role","status").attr("aria-live","polite"),l(a.nTable).attr("aria-describedby",b+"_info"));return d[0]}function Xb(a){var b=a.aanFeatures.i;if(0!==b.length){var c=a.oLanguage,d=a._iDisplayStart+1,e=a.fnDisplayEnd(),h=a.fnRecordsTotal(),f=a.fnRecordsDisplay(),g=
+f?c.sInfo:c.sInfoEmpty;f!==h&&(g+=" "+c.sInfoFiltered);g+=c.sInfoPostFix;g=Yb(a,g);c=c.fnInfoCallback;null!==c&&(g=c.call(a.oInstance,a,d,e,h,f,g));l(b).html(g)}}function Yb(a,b){var c=a.fnFormatNumber,d=a._iDisplayStart+1,e=a._iDisplayLength,h=a.fnRecordsDisplay(),f=-1===e;return b.replace(/_START_/g,c.call(a,d)).replace(/_END_/g,c.call(a,a.fnDisplayEnd())).replace(/_MAX_/g,c.call(a,a.fnRecordsTotal())).replace(/_TOTAL_/g,c.call(a,h)).replace(/_PAGE_/g,c.call(a,f?1:Math.ceil(d/e))).replace(/_PAGES_/g,
+c.call(a,f?1:Math.ceil(h/e)))}function Aa(a){var b=a.iInitDisplayStart,c=a.aoColumns;var d=a.oFeatures;var e=a.bDeferLoading;if(a.bInitialised){Ib(a);Fb(a);xa(a,a.aoHeader);xa(a,a.aoFooter);V(a,!0);d.bAutoWidth&&Za(a);var h=0;for(d=c.length;h<d;h++){var f=c[h];f.sWidth&&(f.nTh.style.width=K(f.sWidth))}F(a,null,"preInit",[a]);ka(a);c=Q(a);if("ssp"!=c||e)"ajax"==c?Oa(a,[],function(g){var k=za(a,g);for(h=0;h<k.length;h++)ia(a,k[h]);a.iInitDisplayStart=b;ka(a);V(a,!1);Pa(a,g)},a):(V(a,!1),Pa(a))}else setTimeout(function(){Aa(a)},
+200)}function Pa(a,b){a._bInitComplete=!0;(b||a.oInit.aaData)&&sa(a);F(a,null,"plugin-init",[a,b]);F(a,"aoInitComplete","init",[a,b])}function kb(a,b){b=parseInt(b,10);a._iDisplayLength=b;lb(a);F(a,null,"length",[a,b])}function Jb(a){var b=a.oClasses,c=a.sTableId,d=a.aLengthMenu,e=Array.isArray(d[0]),h=e?d[0]:d;d=e?d[1]:d;e=l("<select/>",{name:c+"_length","aria-controls":c,"class":b.sLengthSelect});for(var f=0,g=h.length;f<g;f++)e[0][f]=new Option("number"===typeof d[f]?a.fnFormatNumber(d[f]):d[f],
+h[f]);var k=l("<div><label/></div>").addClass(b.sLength);a.aanFeatures.l||(k[0].id=c+"_length");k.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));l("select",k).val(a._iDisplayLength).on("change.DT",function(m){kb(a,l(this).val());ja(a)});l(a.nTable).on("length.dt.DT",function(m,n,p){a===n&&l("select",k).val(p)});return k[0]}function Ob(a){var b=a.sPaginationType,c=u.ext.pager[b],d="function"===typeof c,e=function(f){ja(f)};b=l("<div/>").addClass(a.oClasses.sPaging+b)[0];
+var h=a.aanFeatures;d||c.fnInit(a,b,e);h.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(f){if(d){var g=f._iDisplayStart,k=f._iDisplayLength,m=f.fnRecordsDisplay(),n=-1===k;g=n?0:Math.ceil(g/k);k=n?1:Math.ceil(m/k);m=c(g,k);var p;n=0;for(p=h.p.length;n<p;n++)gb(f,"pageButton")(f,h.p[n],n,m,g,k)}else c.fnUpdate(f,e)},sName:"pagination"}));return b}function Ra(a,b,c){var d=a._iDisplayStart,e=a._iDisplayLength,h=a.fnRecordsDisplay();0===h||-1===e?d=0:"number"===typeof b?(d=b*e,d>h&&
+(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e<h&&(d+=e):"last"==b?d=Math.floor((h-1)/e)*e:da(a,0,"Unknown paging action: "+b,5);b=a._iDisplayStart!==d;a._iDisplayStart=d;b&&(F(a,null,"page",[a]),c&&ja(a));return b}function Lb(a){return l("<div/>",{id:a.aanFeatures.r?null:a.sTableId+"_processing","class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function V(a,b){a.oFeatures.bProcessing&&l(a.aanFeatures.r).css("display",b?"block":"none");
+F(a,null,"processing",[a,b])}function Mb(a){var b=l(a.nTable),c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,h=a.oClasses,f=b.children("caption"),g=f.length?f[0]._captionSide:null,k=l(b[0].cloneNode(!1)),m=l(b[0].cloneNode(!1)),n=b.children("tfoot");n.length||(n=null);k=l("<div/>",{"class":h.sScrollWrapper}).append(l("<div/>",{"class":h.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:d?d?K(d):null:"100%"}).append(l("<div/>",{"class":h.sScrollHeadInner}).css({"box-sizing":"content-box",
+width:c.sXInner||"100%"}).append(k.removeAttr("id").css("margin-left",0).append("top"===g?f:null).append(b.children("thead"))))).append(l("<div/>",{"class":h.sScrollBody}).css({position:"relative",overflow:"auto",width:d?K(d):null}).append(b));n&&k.append(l("<div/>",{"class":h.sScrollFoot}).css({overflow:"hidden",border:0,width:d?d?K(d):null:"100%"}).append(l("<div/>",{"class":h.sScrollFootInner}).append(m.removeAttr("id").css("margin-left",0).append("bottom"===g?f:null).append(b.children("tfoot")))));
+b=k.children();var p=b[0];h=b[1];var t=n?b[2]:null;if(d)l(h).on("scroll.DT",function(v){v=this.scrollLeft;p.scrollLeft=v;n&&(t.scrollLeft=v)});l(h).css("max-height",e);c.bCollapse||l(h).css("height",e);a.nScrollHead=p;a.nScrollBody=h;a.nScrollFoot=t;a.aoDrawCallback.push({fn:Ha,sName:"scrolling"});return k[0]}function Ha(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY;b=b.iBarWidth;var h=l(a.nScrollHead),f=h[0].style,g=h.children("div"),k=g[0].style,m=g.children("table");g=a.nScrollBody;var n=l(g),p=
+g.style,t=l(a.nScrollFoot).children("div"),v=t.children("table"),x=l(a.nTHead),w=l(a.nTable),r=w[0],C=r.style,G=a.nTFoot?l(a.nTFoot):null,aa=a.oBrowser,L=aa.bScrollOversize;U(a.aoColumns,"nTh");var O=[],I=[],H=[],ea=[],Y,Ba=function(D){D=D.style;D.paddingTop="0";D.paddingBottom="0";D.borderTopWidth="0";D.borderBottomWidth="0";D.height=0};var fa=g.scrollHeight>g.clientHeight;if(a.scrollBarVis!==fa&&a.scrollBarVis!==q)a.scrollBarVis=fa,sa(a);else{a.scrollBarVis=fa;w.children("thead, tfoot").remove();
+if(G){var ba=G.clone().prependTo(w);var la=G.find("tr");ba=ba.find("tr")}var mb=x.clone().prependTo(w);x=x.find("tr");fa=mb.find("tr");mb.find("th, td").removeAttr("tabindex");c||(p.width="100%",h[0].style.width="100%");l.each(Na(a,mb),function(D,W){Y=ta(a,D);W.style.width=a.aoColumns[Y].sWidth});G&&ca(function(D){D.style.width=""},ba);h=w.outerWidth();""===c?(C.width="100%",L&&(w.find("tbody").height()>g.offsetHeight||"scroll"==n.css("overflow-y"))&&(C.width=K(w.outerWidth()-b)),h=w.outerWidth()):
+""!==d&&(C.width=K(d),h=w.outerWidth());ca(Ba,fa);ca(function(D){var W=z.getComputedStyle?z.getComputedStyle(D).width:K(l(D).width());H.push(D.innerHTML);O.push(W)},fa);ca(function(D,W){D.style.width=O[W]},x);l(fa).css("height",0);G&&(ca(Ba,ba),ca(function(D){ea.push(D.innerHTML);I.push(K(l(D).css("width")))},ba),ca(function(D,W){D.style.width=I[W]},la),l(ba).height(0));ca(function(D,W){D.innerHTML='<div class="dataTables_sizing">'+H[W]+"</div>";D.childNodes[0].style.height="0";D.childNodes[0].style.overflow=
+"hidden";D.style.width=O[W]},fa);G&&ca(function(D,W){D.innerHTML='<div class="dataTables_sizing">'+ea[W]+"</div>";D.childNodes[0].style.height="0";D.childNodes[0].style.overflow="hidden";D.style.width=I[W]},ba);Math.round(w.outerWidth())<Math.round(h)?(la=g.scrollHeight>g.offsetHeight||"scroll"==n.css("overflow-y")?h+b:h,L&&(g.scrollHeight>g.offsetHeight||"scroll"==n.css("overflow-y"))&&(C.width=K(la-b)),""!==c&&""===d||da(a,1,"Possible column misalignment",6)):la="100%";p.width=K(la);f.width=K(la);
+G&&(a.nScrollFoot.style.width=K(la));!e&&L&&(p.height=K(r.offsetHeight+b));c=w.outerWidth();m[0].style.width=K(c);k.width=K(c);d=w.height()>g.clientHeight||"scroll"==n.css("overflow-y");e="padding"+(aa.bScrollbarLeft?"Left":"Right");k[e]=d?b+"px":"0px";G&&(v[0].style.width=K(c),t[0].style.width=K(c),t[0].style[e]=d?b+"px":"0px");w.children("colgroup").insertBefore(w.children("thead"));n.trigger("scroll");!a.bSorted&&!a.bFiltered||a._drawHold||(g.scrollTop=0)}}function ca(a,b,c){for(var d=0,e=0,h=
+b.length,f,g;e<h;){f=b[e].firstChild;for(g=c?c[e].firstChild:null;f;)1===f.nodeType&&(c?a(f,g,d):a(f,d),d++),f=f.nextSibling,g=c?g.nextSibling:null;e++}}function Za(a){var b=a.nTable,c=a.aoColumns,d=a.oScroll,e=d.sY,h=d.sX,f=d.sXInner,g=c.length,k=Ia(a,"bVisible"),m=l("th",a.nTHead),n=b.getAttribute("width"),p=b.parentNode,t=!1,v,x=a.oBrowser;d=x.bScrollOversize;(v=b.style.width)&&-1!==v.indexOf("%")&&(n=v);for(v=0;v<k.length;v++){var w=c[k[v]];null!==w.sWidth&&(w.sWidth=Zb(w.sWidthOrig,p),t=!0)}if(d||
+!t&&!h&&!e&&g==oa(a)&&g==m.length)for(v=0;v<g;v++)k=ta(a,v),null!==k&&(c[k].sWidth=K(m.eq(v).width()));else{g=l(b).clone().css("visibility","hidden").removeAttr("id");g.find("tbody tr").remove();var r=l("<tr/>").appendTo(g.find("tbody"));g.find("thead, tfoot").remove();g.append(l(a.nTHead).clone()).append(l(a.nTFoot).clone());g.find("tfoot th, tfoot td").css("width","");m=Na(a,g.find("thead")[0]);for(v=0;v<k.length;v++)w=c[k[v]],m[v].style.width=null!==w.sWidthOrig&&""!==w.sWidthOrig?K(w.sWidthOrig):
+"",w.sWidthOrig&&h&&l(m[v]).append(l("<div/>").css({width:w.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(v=0;v<k.length;v++)t=k[v],w=c[t],l($b(a,t)).clone(!1).append(w.sContentPadding).appendTo(r);l("[name]",g).removeAttr("name");w=l("<div/>").css(h||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(g).appendTo(p);h&&f?g.width(f):h?(g.css("width","auto"),g.removeAttr("width"),g.width()<p.clientWidth&&n&&g.width(p.clientWidth)):e?g.width(p.clientWidth):
+n&&g.width(n);for(v=e=0;v<k.length;v++)p=l(m[v]),f=p.outerWidth()-p.width(),p=x.bBounding?Math.ceil(m[v].getBoundingClientRect().width):p.outerWidth(),e+=p,c[k[v]].sWidth=K(p-f);b.style.width=K(e);w.remove()}n&&(b.style.width=K(n));!n&&!h||a._reszEvt||(b=function(){l(z).on("resize.DT-"+a.sInstance,hb(function(){sa(a)}))},d?setTimeout(b,1E3):b(),a._reszEvt=!0)}function Zb(a,b){if(!a)return 0;a=l("<div/>").css("width",K(a)).appendTo(b||A.body);b=a[0].offsetWidth;a.remove();return b}function $b(a,b){var c=
+ac(a,b);if(0>c)return null;var d=a.aoData[c];return d.nTr?d.anCells[b]:l("<td/>").html(T(a,c,b,"display"))[0]}function ac(a,b){for(var c,d=-1,e=-1,h=0,f=a.aoData.length;h<f;h++)c=T(a,h,b,"display")+"",c=c.replace(uc,""),c=c.replace(/&nbsp;/g," "),c.length>d&&(d=c.length,e=h);return e}function K(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function pa(a){var b=[],c=a.aoColumns;var d=a.aaSortingFixed;var e=l.isPlainObject(d);var h=[];var f=function(n){n.length&&
+!Array.isArray(n[0])?h.push(n):l.merge(h,n)};Array.isArray(d)&&f(d);e&&d.pre&&f(d.pre);f(a.aaSorting);e&&d.post&&f(d.post);for(a=0;a<h.length;a++){var g=h[a][0];f=c[g].aDataSort;d=0;for(e=f.length;d<e;d++){var k=f[d];var m=c[k].sType||"string";h[a]._idx===q&&(h[a]._idx=l.inArray(h[a][1],c[k].asSorting));b.push({src:g,col:k,dir:h[a][1],index:h[a]._idx,type:m,formatter:u.ext.type.order[m+"-pre"]})}}return b}function Hb(a){var b,c=[],d=u.ext.type.order,e=a.aoData,h=0,f=a.aiDisplayMaster;$a(a);var g=
+pa(a);var k=0;for(b=g.length;k<b;k++){var m=g[k];m.formatter&&h++;bc(a,m.col)}if("ssp"!=Q(a)&&0!==g.length){k=0;for(b=f.length;k<b;k++)c[f[k]]=k;h===g.length?f.sort(function(n,p){var t,v=g.length,x=e[n]._aSortData,w=e[p]._aSortData;for(t=0;t<v;t++){var r=g[t];var C=x[r.col];var G=w[r.col];C=C<G?-1:C>G?1:0;if(0!==C)return"asc"===r.dir?C:-C}C=c[n];G=c[p];return C<G?-1:C>G?1:0}):f.sort(function(n,p){var t,v=g.length,x=e[n]._aSortData,w=e[p]._aSortData;for(t=0;t<v;t++){var r=g[t];var C=x[r.col];var G=
+w[r.col];r=d[r.type+"-"+r.dir]||d["string-"+r.dir];C=r(C,G);if(0!==C)return C}C=c[n];G=c[p];return C<G?-1:C>G?1:0})}a.bSorted=!0}function cc(a){var b=a.aoColumns,c=pa(a);a=a.oLanguage.oAria;for(var d=0,e=b.length;d<e;d++){var h=b[d];var f=h.asSorting;var g=h.ariaTitle||h.sTitle.replace(/<.*?>/g,"");var k=h.nTh;k.removeAttribute("aria-sort");h.bSortable&&(0<c.length&&c[0].col==d?(k.setAttribute("aria-sort","asc"==c[0].dir?"ascending":"descending"),h=f[c[0].index+1]||f[0]):h=f[0],g+="asc"===h?a.sSortAscending:
+a.sSortDescending);k.setAttribute("aria-label",g)}}function nb(a,b,c,d){var e=a.aaSorting,h=a.aoColumns[b].asSorting,f=function(g,k){var m=g._idx;m===q&&(m=l.inArray(g[1],h));return m+1<h.length?m+1:k?null:0};"number"===typeof e[0]&&(e=a.aaSorting=[e]);c&&a.oFeatures.bSortMulti?(c=l.inArray(b,U(e,"0")),-1!==c?(b=f(e[c],!0),null===b&&1===e.length&&(b=0),null===b?e.splice(c,1):(e[c][1]=h[b],e[c]._idx=b)):(e.push([b,h[0],0]),e[e.length-1]._idx=0)):e.length&&e[0][0]==b?(b=f(e[0]),e.length=1,e[0][1]=h[b],
+e[0]._idx=b):(e.length=0,e.push([b,h[0]]),e[0]._idx=0);ka(a);"function"==typeof d&&d(a)}function fb(a,b,c,d){var e=a.aoColumns[c];ob(b,{},function(h){!1!==e.bSortable&&(a.oFeatures.bProcessing?(V(a,!0),setTimeout(function(){nb(a,c,h.shiftKey,d);"ssp"!==Q(a)&&V(a,!1)},0)):nb(a,c,h.shiftKey,d))})}function Sa(a){var b=a.aLastSort,c=a.oClasses.sSortColumn,d=pa(a),e=a.oFeatures,h;if(e.bSort&&e.bSortClasses){e=0;for(h=b.length;e<h;e++){var f=b[e].src;l(U(a.aoData,"anCells",f)).removeClass(c+(2>e?e+1:3))}e=
+0;for(h=d.length;e<h;e++)f=d[e].src,l(U(a.aoData,"anCells",f)).addClass(c+(2>e?e+1:3))}a.aLastSort=d}function bc(a,b){var c=a.aoColumns[b],d=u.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ua(a,b)));for(var h,f=u.ext.type.order[c.sType+"-pre"],g=0,k=a.aoData.length;g<k;g++)if(c=a.aoData[g],c._aSortData||(c._aSortData=[]),!c._aSortData[b]||d)h=d?e[g]:T(a,g,b,"sort"),c._aSortData[b]=f?f(h):h}function Ca(a){if(!a._bLoadingState){var b={time:+new Date,start:a._iDisplayStart,length:a._iDisplayLength,
+order:l.extend(!0,[],a.aaSorting),search:Vb(a.oPreviousSearch),columns:l.map(a.aoColumns,function(c,d){return{visible:c.bVisible,search:Vb(a.aoPreSearchCols[d])}})};a.oSavedState=b;F(a,"aoStateSaveParams","stateSaveParams",[a,b]);a.oFeatures.bStateSave&&!a.bDestroying&&a.fnStateSaveCallback.call(a.oInstance,a,b)}}function dc(a,b,c){if(a.oFeatures.bStateSave)return b=a.fnStateLoadCallback.call(a.oInstance,a,function(d){pb(a,d,c)}),b!==q&&pb(a,b,c),!0;c()}function pb(a,b,c){var d,e=a.aoColumns;a._bLoadingState=
+!0;var h=a._bInitComplete?new u.Api(a):null;if(b&&b.time){var f=F(a,"aoStateLoadParams","stateLoadParams",[a,b]);if(-1!==l.inArray(!1,f))a._bLoadingState=!1;else if(f=a.iStateDuration,0<f&&b.time<+new Date-1E3*f)a._bLoadingState=!1;else if(b.columns&&e.length!==b.columns.length)a._bLoadingState=!1;else{a.oLoadedState=l.extend(!0,{},b);b.start!==q&&(null===h?(a._iDisplayStart=b.start,a.iInitDisplayStart=b.start):Ra(a,b.start/b.length));b.length!==q&&(a._iDisplayLength=b.length);b.order!==q&&(a.aaSorting=
+[],l.each(b.order,function(k,m){a.aaSorting.push(m[0]>=e.length?[0,m[1]]:m)}));b.search!==q&&l.extend(a.oPreviousSearch,Wb(b.search));if(b.columns){f=0;for(d=b.columns.length;f<d;f++){var g=b.columns[f];g.visible!==q&&(h?h.column(f).visible(g.visible,!1):e[f].bVisible=g.visible);g.search!==q&&l.extend(a.aoPreSearchCols[f],Wb(g.search))}h&&h.columns.adjust()}a._bLoadingState=!1;F(a,"aoStateLoaded","stateLoaded",[a,b])}}else a._bLoadingState=!1;c()}function Ta(a){var b=u.settings;a=l.inArray(a,U(b,
+"nTable"));return-1!==a?b[a]:null}function da(a,b,c,d){c="DataTables warning: "+(a?"table id="+a.sTableId+" - ":"")+c;d&&(c+=". For more information about this error, please see http://datatables.net/tn/"+d);if(b)z.console&&console.log&&console.log(c);else if(b=u.ext,b=b.sErrMode||b.errMode,a&&F(a,null,"error",[a,d,c]),"alert"==b)alert(c);else{if("throw"==b)throw Error(c);"function"==typeof b&&b(a,d,c)}}function X(a,b,c,d){Array.isArray(c)?l.each(c,function(e,h){Array.isArray(h)?X(a,b,h[0],h[1]):
+X(a,b,h)}):(d===q&&(d=c),b[c]!==q&&(a[d]=b[c]))}function qb(a,b,c){var d;for(d in b)if(b.hasOwnProperty(d)){var e=b[d];l.isPlainObject(e)?(l.isPlainObject(a[d])||(a[d]={}),l.extend(!0,a[d],e)):c&&"data"!==d&&"aaData"!==d&&Array.isArray(e)?a[d]=e.slice():a[d]=e}return a}function ob(a,b,c){l(a).on("click.DT",b,function(d){l(a).trigger("blur");c(d)}).on("keypress.DT",b,function(d){13===d.which&&(d.preventDefault(),c(d))}).on("selectstart.DT",function(){return!1})}function R(a,b,c,d){c&&a[b].push({fn:c,
+sName:d})}function F(a,b,c,d){var e=[];b&&(e=l.map(a[b].slice().reverse(),function(h,f){return h.fn.apply(a.oInstance,d)}));null!==c&&(b=l.Event(c+".dt"),l(a.nTable).trigger(b,d),e.push(b.result));return e}function lb(a){var b=a._iDisplayStart,c=a.fnDisplayEnd(),d=a._iDisplayLength;b>=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function gb(a,b){a=a.renderer;var c=u.ext.renderer[b];return l.isPlainObject(a)&&a[b]?c[a[b]]||c._:"string"===typeof a?c[a]||c._:c._}function Q(a){return a.oFeatures.bServerSide?
+"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function Da(a,b){var c=ec.numbers_length,d=Math.floor(c/2);b<=c?a=qa(0,b):a<=d?(a=qa(0,c-2),a.push("ellipsis"),a.push(b-1)):(a>=b-1-d?a=qa(b-(c-2),b):(a=qa(a-d+2,a+d-1),a.push("ellipsis"),a.push(b-1)),a.splice(0,0,"ellipsis"),a.splice(0,0,0));a.DT_el="span";return a}function Xa(a){l.each({num:function(b){return Ua(b,a)},"num-fmt":function(b){return Ua(b,a,rb)},"html-num":function(b){return Ua(b,a,Va)},"html-num-fmt":function(b){return Ua(b,a,Va,rb)}},function(b,
+c){M.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(M.type.search[b+a]=M.type.search.html)})}function fc(a){return function(){var b=[Ta(this[u.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return u.ext.internal[a].apply(this,b)}}var u=function(a,b){if(this instanceof u)return l(a).DataTable(b);b=a;this.$=function(f,g){return this.api(!0).$(f,g)};this._=function(f,g){return this.api(!0).rows(f,g).data()};this.api=function(f){return f?new B(Ta(this[M.iApiIndex])):new B(this)};this.fnAddData=
+function(f,g){var k=this.api(!0);f=Array.isArray(f)&&(Array.isArray(f[0])||l.isPlainObject(f[0]))?k.rows.add(f):k.row.add(f);(g===q||g)&&k.draw();return f.flatten().toArray()};this.fnAdjustColumnSizing=function(f){var g=this.api(!0).columns.adjust(),k=g.settings()[0],m=k.oScroll;f===q||f?g.draw(!1):(""!==m.sX||""!==m.sY)&&Ha(k)};this.fnClearTable=function(f){var g=this.api(!0).clear();(f===q||f)&&g.draw()};this.fnClose=function(f){this.api(!0).row(f).child.hide()};this.fnDeleteRow=function(f,g,k){var m=
+this.api(!0);f=m.rows(f);var n=f.settings()[0],p=n.aoData[f[0][0]];f.remove();g&&g.call(this,n,p);(k===q||k)&&m.draw();return p};this.fnDestroy=function(f){this.api(!0).destroy(f)};this.fnDraw=function(f){this.api(!0).draw(f)};this.fnFilter=function(f,g,k,m,n,p){n=this.api(!0);null===g||g===q?n.search(f,k,m,p):n.column(g).search(f,k,m,p);n.draw()};this.fnGetData=function(f,g){var k=this.api(!0);if(f!==q){var m=f.nodeName?f.nodeName.toLowerCase():"";return g!==q||"td"==m||"th"==m?k.cell(f,g).data():
+k.row(f).data()||null}return k.data().toArray()};this.fnGetNodes=function(f){var g=this.api(!0);return f!==q?g.row(f).node():g.rows().nodes().flatten().toArray()};this.fnGetPosition=function(f){var g=this.api(!0),k=f.nodeName.toUpperCase();return"TR"==k?g.row(f).index():"TD"==k||"TH"==k?(f=g.cell(f).index(),[f.row,f.columnVisible,f.column]):null};this.fnIsOpen=function(f){return this.api(!0).row(f).child.isShown()};this.fnOpen=function(f,g,k){return this.api(!0).row(f).child(g,k).show().child()[0]};
+this.fnPageChange=function(f,g){f=this.api(!0).page(f);(g===q||g)&&f.draw(!1)};this.fnSetColumnVis=function(f,g,k){f=this.api(!0).column(f).visible(g);(k===q||k)&&f.columns.adjust().draw()};this.fnSettings=function(){return Ta(this[M.iApiIndex])};this.fnSort=function(f){this.api(!0).order(f).draw()};this.fnSortListener=function(f,g,k){this.api(!0).order.listener(f,g,k)};this.fnUpdate=function(f,g,k,m,n){var p=this.api(!0);k===q||null===k?p.row(g).data(f):p.cell(g,k).data(f);(n===q||n)&&p.columns.adjust();
+(m===q||m)&&p.draw();return 0};this.fnVersionCheck=M.fnVersionCheck;var c=this,d=b===q,e=this.length;d&&(b={});this.oApi=this.internal=M.internal;for(var h in u.ext.internal)h&&(this[h]=fc(h));this.each(function(){var f={},g=1<e?qb(f,b,!0):b,k=0,m;f=this.getAttribute("id");var n=!1,p=u.defaults,t=l(this);if("table"!=this.nodeName.toLowerCase())da(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{zb(p);Ab(p.column);P(p,p,!0);P(p.column,p.column,!0);P(p,l.extend(g,t.data()),!0);var v=
+u.settings;k=0;for(m=v.length;k<m;k++){var x=v[k];if(x.nTable==this||x.nTHead&&x.nTHead.parentNode==this||x.nTFoot&&x.nTFoot.parentNode==this){var w=g.bRetrieve!==q?g.bRetrieve:p.bRetrieve;if(d||w)return x.oInstance;if(g.bDestroy!==q?g.bDestroy:p.bDestroy){x.oInstance.fnDestroy();break}else{da(x,0,"Cannot reinitialise DataTable",3);return}}if(x.sTableId==this.id){v.splice(k,1);break}}if(null===f||""===f)this.id=f="DataTables_Table_"+u.ext._unique++;var r=l.extend(!0,{},u.models.oSettings,{sDestroyWidth:t[0].style.width,
+sInstance:f,sTableId:f});r.nTable=this;r.oApi=c.internal;r.oInit=g;v.push(r);r.oInstance=1===c.length?c:t.dataTable();zb(g);ma(g.oLanguage);g.aLengthMenu&&!g.iDisplayLength&&(g.iDisplayLength=Array.isArray(g.aLengthMenu[0])?g.aLengthMenu[0][0]:g.aLengthMenu[0]);g=qb(l.extend(!0,{},p),g);X(r.oFeatures,g,"bPaginate bLengthChange bFilter bSort bSortMulti bInfo bProcessing bAutoWidth bSortClasses bServerSide bDeferRender".split(" "));X(r,g,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod",
+"aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"]]);X(r.oScroll,g,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]);X(r.oLanguage,g,"fnInfoCallback");
+R(r,"aoDrawCallback",g.fnDrawCallback,"user");R(r,"aoServerParams",g.fnServerParams,"user");R(r,"aoStateSaveParams",g.fnStateSaveParams,"user");R(r,"aoStateLoadParams",g.fnStateLoadParams,"user");R(r,"aoStateLoaded",g.fnStateLoaded,"user");R(r,"aoRowCallback",g.fnRowCallback,"user");R(r,"aoRowCreatedCallback",g.fnCreatedRow,"user");R(r,"aoHeaderCallback",g.fnHeaderCallback,"user");R(r,"aoFooterCallback",g.fnFooterCallback,"user");R(r,"aoInitComplete",g.fnInitComplete,"user");R(r,"aoPreDrawCallback",
+g.fnPreDrawCallback,"user");r.rowIdFn=na(g.rowId);Bb(r);var C=r.oClasses;l.extend(C,u.ext.classes,g.oClasses);t.addClass(C.sTable);r.iInitDisplayStart===q&&(r.iInitDisplayStart=g.iDisplayStart,r._iDisplayStart=g.iDisplayStart);null!==g.iDeferLoading&&(r.bDeferLoading=!0,f=Array.isArray(g.iDeferLoading),r._iRecordsDisplay=f?g.iDeferLoading[0]:g.iDeferLoading,r._iRecordsTotal=f?g.iDeferLoading[1]:g.iDeferLoading);var G=r.oLanguage;l.extend(!0,G,g.oLanguage);G.sUrl?(l.ajax({dataType:"json",url:G.sUrl,
+success:function(I){P(p.oLanguage,I);ma(I);l.extend(!0,G,I);F(r,null,"i18n",[r]);Aa(r)},error:function(){Aa(r)}}),n=!0):F(r,null,"i18n",[r]);null===g.asStripeClasses&&(r.asStripeClasses=[C.sStripeOdd,C.sStripeEven]);f=r.asStripeClasses;var aa=t.children("tbody").find("tr").eq(0);-1!==l.inArray(!0,l.map(f,function(I,H){return aa.hasClass(I)}))&&(l("tbody tr",this).removeClass(f.join(" ")),r.asDestroyStripes=f.slice());f=[];v=this.getElementsByTagName("thead");0!==v.length&&(wa(r.aoHeader,v[0]),f=Na(r));
+if(null===g.aoColumns)for(v=[],k=0,m=f.length;k<m;k++)v.push(null);else v=g.aoColumns;k=0;for(m=v.length;k<m;k++)Ya(r,f?f[k]:null);Db(r,g.aoColumnDefs,v,function(I,H){Ga(r,I,H)});if(aa.length){var L=function(I,H){return null!==I.getAttribute("data-"+H)?H:null};l(aa[0]).children("th, td").each(function(I,H){var ea=r.aoColumns[I];if(ea.mData===I){var Y=L(H,"sort")||L(H,"order");H=L(H,"filter")||L(H,"search");if(null!==Y||null!==H)ea.mData={_:I+".display",sort:null!==Y?I+".@data-"+Y:q,type:null!==Y?
+I+".@data-"+Y:q,filter:null!==H?I+".@data-"+H:q},Ga(r,I)}})}var O=r.oFeatures;f=function(){if(g.aaSorting===q){var I=r.aaSorting;k=0;for(m=I.length;k<m;k++)I[k][1]=r.aoColumns[k].asSorting[0]}Sa(r);O.bSort&&R(r,"aoDrawCallback",function(){if(r.bSorted){var Y=pa(r),Ba={};l.each(Y,function(fa,ba){Ba[ba.src]=ba.dir});F(r,null,"order",[r,Y,Ba]);cc(r)}});R(r,"aoDrawCallback",function(){(r.bSorted||"ssp"===Q(r)||O.bDeferRender)&&Sa(r)},"sc");I=t.children("caption").each(function(){this._captionSide=l(this).css("caption-side")});
+var H=t.children("thead");0===H.length&&(H=l("<thead/>").appendTo(t));r.nTHead=H[0];var ea=t.children("tbody");0===ea.length&&(ea=l("<tbody/>").insertAfter(H));r.nTBody=ea[0];H=t.children("tfoot");0===H.length&&0<I.length&&(""!==r.oScroll.sX||""!==r.oScroll.sY)&&(H=l("<tfoot/>").appendTo(t));0===H.length||0===H.children().length?t.addClass(C.sNoFooter):0<H.length&&(r.nTFoot=H[0],wa(r.aoFooter,r.nTFoot));if(g.aaData)for(k=0;k<g.aaData.length;k++)ia(r,g.aaData[k]);else(r.bDeferLoading||"dom"==Q(r))&&
+Ja(r,l(r.nTBody).children("tr"));r.aiDisplay=r.aiDisplayMaster.slice();r.bInitialised=!0;!1===n&&Aa(r)};R(r,"aoDrawCallback",Ca,"state_save");g.bStateSave?(O.bStateSave=!0,dc(r,g,f)):f()}});c=null;return this},M,y,J,sb={},gc=/[\r\n\u2028]/g,Va=/<.*?>/g,vc=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,wc=/(\/|\.|\*|\+|\?|\||\(|\)|\[|\]|\{|\}|\\|\$|\^|\-)/g,rb=/['\u00A0,$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,Z=function(a){return a&&!0!==a&&"-"!==a?!1:!0},hc=
+function(a){var b=parseInt(a,10);return!isNaN(b)&&isFinite(a)?b:null},ic=function(a,b){sb[b]||(sb[b]=new RegExp(jb(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(sb[b],"."):a},tb=function(a,b,c){var d="string"===typeof a;if(Z(a))return!0;b&&d&&(a=ic(a,b));c&&d&&(a=a.replace(rb,""));return!isNaN(parseFloat(a))&&isFinite(a)},jc=function(a,b,c){return Z(a)?!0:Z(a)||"string"===typeof a?tb(a.replace(Va,""),b,c)?!0:null:null},U=function(a,b,c){var d=[],e=0,h=a.length;if(c!==q)for(;e<
+h;e++)a[e]&&a[e][b]&&d.push(a[e][b][c]);else for(;e<h;e++)a[e]&&d.push(a[e][b]);return d},Ea=function(a,b,c,d){var e=[],h=0,f=b.length;if(d!==q)for(;h<f;h++)a[b[h]][c]&&e.push(a[b[h]][c][d]);else for(;h<f;h++)e.push(a[b[h]][c]);return e},qa=function(a,b){var c=[];if(b===q){b=0;var d=a}else d=b,b=a;for(a=b;a<d;a++)c.push(a);return c},kc=function(a){for(var b=[],c=0,d=a.length;c<d;c++)a[c]&&b.push(a[c]);return b},Ma=function(a){a:{if(!(2>a.length)){var b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d<
+e;d++){if(b[d]===c){b=!1;break a}c=b[d]}}b=!0}if(b)return a.slice();b=[];e=a.length;var h,f=0;d=0;a:for(;d<e;d++){c=a[d];for(h=0;h<f;h++)if(b[h]===c)continue a;b.push(c);f++}return b},lc=function(a,b){if(Array.isArray(b))for(var c=0;c<b.length;c++)lc(a,b[c]);else a.push(b);return a},mc=function(a,b){b===q&&(b=0);return-1!==this.indexOf(a,b)};Array.isArray||(Array.isArray=function(a){return"[object Array]"===Object.prototype.toString.call(a)});Array.prototype.includes||(Array.prototype.includes=mc);
+String.prototype.trim||(String.prototype.trim=function(){return this.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,"")});String.prototype.includes||(String.prototype.includes=mc);u.util={throttle:function(a,b){var c=b!==q?b:200,d,e;return function(){var h=this,f=+new Date,g=arguments;d&&f<d+c?(clearTimeout(e),e=setTimeout(function(){d=q;a.apply(h,g)},c)):(d=f,a.apply(h,g))}},escapeRegex:function(a){return a.replace(wc,"\\$1")},set:function(a){if(l.isPlainObject(a))return u.util.set(a._);if(null===
+a)return function(){};if("function"===typeof a)return function(c,d,e){a(c,"set",d,e)};if("string"!==typeof a||-1===a.indexOf(".")&&-1===a.indexOf("[")&&-1===a.indexOf("("))return function(c,d){c[a]=d};var b=function(c,d,e){e=cb(e);var h=e[e.length-1];for(var f,g,k=0,m=e.length-1;k<m;k++){if("__proto__"===e[k]||"constructor"===e[k])throw Error("Cannot set prototype values");f=e[k].match(Fa);g=e[k].match(ra);if(f){e[k]=e[k].replace(Fa,"");c[e[k]]=[];h=e.slice();h.splice(0,k+1);f=h.join(".");if(Array.isArray(d))for(g=
+0,m=d.length;g<m;g++)h={},b(h,d[g],f),c[e[k]].push(h);else c[e[k]]=d;return}g&&(e[k]=e[k].replace(ra,""),c=c[e[k]](d));if(null===c[e[k]]||c[e[k]]===q)c[e[k]]={};c=c[e[k]]}if(h.match(ra))c[h.replace(ra,"")](d);else c[h.replace(Fa,"")]=d};return function(c,d){return b(c,d,a)}},get:function(a){if(l.isPlainObject(a)){var b={};l.each(a,function(d,e){e&&(b[d]=u.util.get(e))});return function(d,e,h,f){var g=b[e]||b._;return g!==q?g(d,e,h,f):d}}if(null===a)return function(d){return d};if("function"===typeof a)return function(d,
+e,h,f){return a(d,e,h,f)};if("string"!==typeof a||-1===a.indexOf(".")&&-1===a.indexOf("[")&&-1===a.indexOf("("))return function(d,e){return d[a]};var c=function(d,e,h){if(""!==h){var f=cb(h);for(var g=0,k=f.length;g<k;g++){h=f[g].match(Fa);var m=f[g].match(ra);if(h){f[g]=f[g].replace(Fa,"");""!==f[g]&&(d=d[f[g]]);m=[];f.splice(0,g+1);f=f.join(".");if(Array.isArray(d))for(g=0,k=d.length;g<k;g++)m.push(c(d[g],e,f));d=h[0].substring(1,h[0].length-1);d=""===d?m:m.join(d);break}else if(m){f[g]=f[g].replace(ra,
+"");d=d[f[g]]();continue}if(null===d||d[f[g]]===q)return q;d=d[f[g]]}}return d};return function(d,e){return c(d,e,a)}}};var S=function(a,b,c){a[b]!==q&&(a[c]=a[b])},Fa=/\[.*?\]$/,ra=/\(\)$/,na=u.util.get,ha=u.util.set,jb=u.util.escapeRegex,Qa=l("<div>")[0],tc=Qa.textContent!==q,uc=/<.*?>/g,hb=u.util.throttle,nc=[],N=Array.prototype,xc=function(a){var b,c=u.settings,d=l.map(c,function(h,f){return h.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase()){var e=
+l.inArray(a,d);return-1!==e?[c[e]]:null}if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?b=l(a):a instanceof l&&(b=a)}else return[];if(b)return b.map(function(h){e=l.inArray(this,d);return-1!==e?c[e]:null}).toArray()};var B=function(a,b){if(!(this instanceof B))return new B(a,b);var c=[],d=function(f){(f=xc(f))&&c.push.apply(c,f)};if(Array.isArray(a))for(var e=0,h=a.length;e<h;e++)d(a[e]);else d(a);this.context=Ma(c);b&&l.merge(this,b);this.selector={rows:null,
+cols:null,opts:null};B.extend(this,this,nc)};u.Api=B;l.extend(B.prototype,{any:function(){return 0!==this.count()},concat:N.concat,context:[],count:function(){return this.flatten().length},each:function(a){for(var b=0,c=this.length;b<c;b++)a.call(this,this[b],b,this);return this},eq:function(a){var b=this.context;return b.length>a?new B(b[a],this[a]):null},filter:function(a){var b=[];if(N.filter)b=N.filter.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)a.call(this,this[c],c,this)&&b.push(this[c]);
+return new B(this.context,b)},flatten:function(){var a=[];return new B(this.context,a.concat.apply(a,this.toArray()))},join:N.join,indexOf:N.indexOf||function(a,b){b=b||0;for(var c=this.length;b<c;b++)if(this[b]===a)return b;return-1},iterator:function(a,b,c,d){var e=[],h,f,g=this.context,k,m=this.selector;"string"===typeof a&&(d=c,c=b,b=a,a=!1);var n=0;for(h=g.length;n<h;n++){var p=new B(g[n]);if("table"===b){var t=c.call(p,g[n],n);t!==q&&e.push(t)}else if("columns"===b||"rows"===b)t=c.call(p,g[n],
+this[n],n),t!==q&&e.push(t);else if("column"===b||"column-rows"===b||"row"===b||"cell"===b){var v=this[n];"column-rows"===b&&(k=Wa(g[n],m.opts));var x=0;for(f=v.length;x<f;x++)t=v[x],t="cell"===b?c.call(p,g[n],t.row,t.column,n,x):c.call(p,g[n],t,n,x,k),t!==q&&e.push(t)}}return e.length||d?(a=new B(g,a?e.concat.apply([],e):e),b=a.selector,b.rows=m.rows,b.cols=m.cols,b.opts=m.opts,a):this},lastIndexOf:N.lastIndexOf||function(a,b){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,
+map:function(a){var b=[];if(N.map)b=N.map.call(this,a,this);else for(var c=0,d=this.length;c<d;c++)b.push(a.call(this,this[c],c));return new B(this.context,b)},pluck:function(a){return this.map(function(b){return b[a]})},pop:N.pop,push:N.push,reduce:N.reduce||function(a,b){return Cb(this,a,b,0,this.length,1)},reduceRight:N.reduceRight||function(a,b){return Cb(this,a,b,this.length-1,-1,-1)},reverse:N.reverse,selector:null,shift:N.shift,slice:function(){return new B(this.context,this)},sort:N.sort,
+splice:N.splice,toArray:function(){return N.slice.call(this)},to$:function(){return l(this)},toJQuery:function(){return l(this)},unique:function(){return new B(this.context,Ma(this))},unshift:N.unshift});B.extend=function(a,b,c){if(c.length&&b&&(b instanceof B||b.__dt_wrapper)){var d,e=function(g,k,m){return function(){var n=k.apply(g,arguments);B.extend(n,n,m.methodExt);return n}};var h=0;for(d=c.length;h<d;h++){var f=c[h];b[f.name]="function"===f.type?e(a,f.val,f):"object"===f.type?{}:f.val;b[f.name].__dt_wrapper=
+!0;B.extend(a,b[f.name],f.propExt)}}};B.register=y=function(a,b){if(Array.isArray(a))for(var c=0,d=a.length;c<d;c++)B.register(a[c],b);else{d=a.split(".");var e=nc,h;a=0;for(c=d.length;a<c;a++){var f=(h=-1!==d[a].indexOf("()"))?d[a].replace("()",""):d[a];a:{var g=0;for(var k=e.length;g<k;g++)if(e[g].name===f){g=e[g];break a}g=null}g||(g={name:f,val:{},methodExt:[],propExt:[],type:"object"},e.push(g));a===c-1?(g.val=b,g.type="function"===typeof b?"function":l.isPlainObject(b)?"object":"other"):e=h?
+g.methodExt:g.propExt}}};B.registerPlural=J=function(a,b,c){B.register(a,c);B.register(b,function(){var d=c.apply(this,arguments);return d===this?this:d instanceof B?d.length?Array.isArray(d[0])?new B(d.context,d[0]):d[0]:q:d})};var oc=function(a,b){if(Array.isArray(a))return l.map(a,function(d){return oc(d,b)});if("number"===typeof a)return[b[a]];var c=l.map(b,function(d,e){return d.nTable});return l(c).filter(a).map(function(d){d=l.inArray(this,c);return b[d]}).toArray()};y("tables()",function(a){return a!==
+q&&null!==a?new B(oc(a,this.context)):this});y("table()",function(a){a=this.tables(a);var b=a.context;return b.length?new B(b[0]):a});J("tables().nodes()","table().node()",function(){return this.iterator("table",function(a){return a.nTable},1)});J("tables().body()","table().body()",function(){return this.iterator("table",function(a){return a.nTBody},1)});J("tables().header()","table().header()",function(){return this.iterator("table",function(a){return a.nTHead},1)});J("tables().footer()","table().footer()",
+function(){return this.iterator("table",function(a){return a.nTFoot},1)});J("tables().containers()","table().container()",function(){return this.iterator("table",function(a){return a.nTableWrapper},1)});y("draw()",function(a){return this.iterator("table",function(b){"page"===a?ja(b):("string"===typeof a&&(a="full-hold"===a?!1:!0),ka(b,!1===a))})});y("page()",function(a){return a===q?this.page.info().page:this.iterator("table",function(b){Ra(b,a)})});y("page.info()",function(a){if(0===this.context.length)return q;
+a=this.context[0];var b=a._iDisplayStart,c=a.oFeatures.bPaginate?a._iDisplayLength:-1,d=a.fnRecordsDisplay(),e=-1===c;return{page:e?0:Math.floor(b/c),pages:e?1:Math.ceil(d/c),start:b,end:a.fnDisplayEnd(),length:c,recordsTotal:a.fnRecordsTotal(),recordsDisplay:d,serverSide:"ssp"===Q(a)}});y("page.len()",function(a){return a===q?0!==this.context.length?this.context[0]._iDisplayLength:q:this.iterator("table",function(b){kb(b,a)})});var pc=function(a,b,c){if(c){var d=new B(a);d.one("draw",function(){c(d.ajax.json())})}if("ssp"==
+Q(a))ka(a,b);else{V(a,!0);var e=a.jqXHR;e&&4!==e.readyState&&e.abort();Oa(a,[],function(h){Ka(a);h=za(a,h);for(var f=0,g=h.length;f<g;f++)ia(a,h[f]);ka(a,b);V(a,!1)})}};y("ajax.json()",function(){var a=this.context;if(0<a.length)return a[0].json});y("ajax.params()",function(){var a=this.context;if(0<a.length)return a[0].oAjaxData});y("ajax.reload()",function(a,b){return this.iterator("table",function(c){pc(c,!1===b,a)})});y("ajax.url()",function(a){var b=this.context;if(a===q){if(0===b.length)return q;
+b=b[0];return b.ajax?l.isPlainObject(b.ajax)?b.ajax.url:b.ajax:b.sAjaxSource}return this.iterator("table",function(c){l.isPlainObject(c.ajax)?c.ajax.url=a:c.ajax=a})});y("ajax.url().load()",function(a,b){return this.iterator("table",function(c){pc(c,!1===b,a)})});var ub=function(a,b,c,d,e){var h=[],f,g,k;var m=typeof b;b&&"string"!==m&&"function"!==m&&b.length!==q||(b=[b]);m=0;for(g=b.length;m<g;m++){var n=b[m]&&b[m].split&&!b[m].match(/[\[\(:]/)?b[m].split(","):[b[m]];var p=0;for(k=n.length;p<k;p++)(f=
+c("string"===typeof n[p]?n[p].trim():n[p]))&&f.length&&(h=h.concat(f))}a=M.selector[a];if(a.length)for(m=0,g=a.length;m<g;m++)h=a[m](d,e,h);return Ma(h)},vb=function(a){a||(a={});a.filter&&a.search===q&&(a.search=a.filter);return l.extend({search:"none",order:"current",page:"all"},a)},wb=function(a){for(var b=0,c=a.length;b<c;b++)if(0<a[b].length)return a[0]=a[b],a[0].length=1,a.length=1,a.context=[a.context[b]],a;a.length=0;return a},Wa=function(a,b){var c=[],d=a.aiDisplay;var e=a.aiDisplayMaster;
+var h=b.search;var f=b.order;b=b.page;if("ssp"==Q(a))return"removed"===h?[]:qa(0,e.length);if("current"==b)for(f=a._iDisplayStart,a=a.fnDisplayEnd();f<a;f++)c.push(d[f]);else if("current"==f||"applied"==f)if("none"==h)c=e.slice();else if("applied"==h)c=d.slice();else{if("removed"==h){var g={};f=0;for(a=d.length;f<a;f++)g[d[f]]=null;c=l.map(e,function(k){return g.hasOwnProperty(k)?null:k})}}else if("index"==f||"original"==f)for(f=0,a=a.aoData.length;f<a;f++)"none"==h?c.push(f):(e=l.inArray(f,d),(-1===
+e&&"removed"==h||0<=e&&"applied"==h)&&c.push(f));return c},yc=function(a,b,c){var d;return ub("row",b,function(e){var h=hc(e),f=a.aoData;if(null!==h&&!c)return[h];d||(d=Wa(a,c));if(null!==h&&-1!==l.inArray(h,d))return[h];if(null===e||e===q||""===e)return d;if("function"===typeof e)return l.map(d,function(k){var m=f[k];return e(k,m._aData,m.nTr)?k:null});if(e.nodeName){h=e._DT_RowIndex;var g=e._DT_CellIndex;if(h!==q)return f[h]&&f[h].nTr===e?[h]:[];if(g)return f[g.row]&&f[g.row].nTr===e.parentNode?
+[g.row]:[];h=l(e).closest("*[data-dt-row]");return h.length?[h.data("dt-row")]:[]}if("string"===typeof e&&"#"===e.charAt(0)&&(h=a.aIds[e.replace(/^#/,"")],h!==q))return[h.idx];h=kc(Ea(a.aoData,d,"nTr"));return l(h).filter(e).map(function(){return this._DT_RowIndex}).toArray()},a,c)};y("rows()",function(a,b){a===q?a="":l.isPlainObject(a)&&(b=a,a="");b=vb(b);var c=this.iterator("table",function(d){return yc(d,a,b)},1);c.selector.rows=a;c.selector.opts=b;return c});y("rows().nodes()",function(){return this.iterator("row",
+function(a,b){return a.aoData[b].nTr||q},1)});y("rows().data()",function(){return this.iterator(!0,"rows",function(a,b){return Ea(a.aoData,b,"_aData")},1)});J("rows().cache()","row().cache()",function(a){return this.iterator("row",function(b,c){b=b.aoData[c];return"search"===a?b._aFilterData:b._aSortData},1)});J("rows().invalidate()","row().invalidate()",function(a){return this.iterator("row",function(b,c){va(b,c,a)})});J("rows().indexes()","row().index()",function(){return this.iterator("row",function(a,
+b){return b},1)});J("rows().ids()","row().id()",function(a){for(var b=[],c=this.context,d=0,e=c.length;d<e;d++)for(var h=0,f=this[d].length;h<f;h++){var g=c[d].rowIdFn(c[d].aoData[this[d][h]]._aData);b.push((!0===a?"#":"")+g)}return new B(c,b)});J("rows().remove()","row().remove()",function(){var a=this;this.iterator("row",function(b,c,d){var e=b.aoData,h=e[c],f,g;e.splice(c,1);var k=0;for(f=e.length;k<f;k++){var m=e[k];var n=m.anCells;null!==m.nTr&&(m.nTr._DT_RowIndex=k);if(null!==n)for(m=0,g=n.length;m<
+g;m++)n[m]._DT_CellIndex.row=k}La(b.aiDisplayMaster,c);La(b.aiDisplay,c);La(a[d],c,!1);0<b._iRecordsDisplay&&b._iRecordsDisplay--;lb(b);c=b.rowIdFn(h._aData);c!==q&&delete b.aIds[c]});this.iterator("table",function(b){for(var c=0,d=b.aoData.length;c<d;c++)b.aoData[c].idx=c});return this});y("rows.add()",function(a){var b=this.iterator("table",function(d){var e,h=[];var f=0;for(e=a.length;f<e;f++){var g=a[f];g.nodeName&&"TR"===g.nodeName.toUpperCase()?h.push(Ja(d,g)[0]):h.push(ia(d,g))}return h},1),
+c=this.rows(-1);c.pop();l.merge(c,b);return c});y("row()",function(a,b){return wb(this.rows(a,b))});y("row().data()",function(a){var b=this.context;if(a===q)return b.length&&this.length?b[0].aoData[this[0]]._aData:q;var c=b[0].aoData[this[0]];c._aData=a;Array.isArray(a)&&c.nTr&&c.nTr.id&&ha(b[0].rowId)(a,c.nTr.id);va(b[0],this[0],"data");return this});y("row().node()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]].nTr||null:null});y("row.add()",function(a){a instanceof
+l&&a.length&&(a=a[0]);var b=this.iterator("table",function(c){return a.nodeName&&"TR"===a.nodeName.toUpperCase()?Ja(c,a)[0]:ia(c,a)});return this.row(b[0])});l(A).on("plugin-init.dt",function(a,b){a=new B(b);a.on("stateSaveParams",function(d,e,h){d=e.rowIdFn;e=e.aoData;for(var f=[],g=0;g<e.length;g++)e[g]._detailsShow&&f.push("#"+d(e[g]._aData));h.childRows=f});var c=a.state.loaded();c&&c.childRows&&a.rows(l.map(c.childRows,function(d){return d.replace(/:/g,"\\:")})).every(function(){F(b,null,"requestChild",
+[this])})});var zc=function(a,b,c,d){var e=[],h=function(f,g){if(Array.isArray(f)||f instanceof l)for(var k=0,m=f.length;k<m;k++)h(f[k],g);else f.nodeName&&"tr"===f.nodeName.toLowerCase()?e.push(f):(k=l("<tr><td></td></tr>").addClass(g),l("td",k).addClass(g).html(f)[0].colSpan=oa(a),e.push(k[0]))};h(c,d);b._details&&b._details.detach();b._details=l(e);b._detailsShow&&b._details.insertAfter(b.nTr)},qc=u.util.throttle(function(a){Ca(a[0])},500),xb=function(a,b){var c=a.context;c.length&&(a=c[0].aoData[b!==
+q?b:a[0]])&&a._details&&(a._details.remove(),a._detailsShow=q,a._details=q,l(a.nTr).removeClass("dt-hasChild"),qc(c))},rc=function(a,b){var c=a.context;if(c.length&&a.length){var d=c[0].aoData[a[0]];d._details&&((d._detailsShow=b)?(d._details.insertAfter(d.nTr),l(d.nTr).addClass("dt-hasChild")):(d._details.detach(),l(d.nTr).removeClass("dt-hasChild")),F(c[0],null,"childRow",[b,a.row(a[0])]),Ac(c[0]),qc(c))}},Ac=function(a){var b=new B(a),c=a.aoData;b.off("draw.dt.DT_details column-visibility.dt.DT_details destroy.dt.DT_details");
+0<U(c,"_details").length&&(b.on("draw.dt.DT_details",function(d,e){a===e&&b.rows({page:"current"}).eq(0).each(function(h){h=c[h];h._detailsShow&&h._details.insertAfter(h.nTr)})}),b.on("column-visibility.dt.DT_details",function(d,e,h,f){if(a===e)for(e=oa(e),h=0,f=c.length;h<f;h++)d=c[h],d._details&&d._details.children("td[colspan]").attr("colspan",e)}),b.on("destroy.dt.DT_details",function(d,e){if(a===e)for(d=0,e=c.length;d<e;d++)c[d]._details&&xb(b,d)}))};y("row().child()",function(a,b){var c=this.context;
+if(a===q)return c.length&&this.length?c[0].aoData[this[0]]._details:q;!0===a?this.child.show():!1===a?xb(this):c.length&&this.length&&zc(c[0],c[0].aoData[this[0]],a,b);return this});y(["row().child.show()","row().child().show()"],function(a){rc(this,!0);return this});y(["row().child.hide()","row().child().hide()"],function(){rc(this,!1);return this});y(["row().child.remove()","row().child().remove()"],function(){xb(this);return this});y("row().child.isShown()",function(){var a=this.context;return a.length&&
+this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var Bc=/^([^:]+):(name|visIdx|visible)$/,sc=function(a,b,c,d,e){c=[];d=0;for(var h=e.length;d<h;d++)c.push(T(a,e[d],b));return c},Cc=function(a,b,c){var d=a.aoColumns,e=U(d,"sName"),h=U(d,"nTh");return ub("column",b,function(f){var g=hc(f);if(""===f)return qa(d.length);if(null!==g)return[0<=g?g:d.length+g];if("function"===typeof f){var k=Wa(a,c);return l.map(d,function(p,t){return f(t,sc(a,t,0,0,k),h[t])?t:null})}var m="string"===typeof f?f.match(Bc):
+"";if(m)switch(m[2]){case "visIdx":case "visible":g=parseInt(m[1],10);if(0>g){var n=l.map(d,function(p,t){return p.bVisible?t:null});return[n[n.length+g]]}return[ta(a,g)];case "name":return l.map(e,function(p,t){return p===m[1]?t:null});default:return[]}if(f.nodeName&&f._DT_CellIndex)return[f._DT_CellIndex.column];g=l(h).filter(f).map(function(){return l.inArray(this,h)}).toArray();if(g.length||!f.nodeName)return g;g=l(f).closest("*[data-dt-column]");return g.length?[g.data("dt-column")]:[]},a,c)};
+y("columns()",function(a,b){a===q?a="":l.isPlainObject(a)&&(b=a,a="");b=vb(b);var c=this.iterator("table",function(d){return Cc(d,a,b)},1);c.selector.cols=a;c.selector.opts=b;return c});J("columns().header()","column().header()",function(a,b){return this.iterator("column",function(c,d){return c.aoColumns[d].nTh},1)});J("columns().footer()","column().footer()",function(a,b){return this.iterator("column",function(c,d){return c.aoColumns[d].nTf},1)});J("columns().data()","column().data()",function(){return this.iterator("column-rows",
+sc,1)});J("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});J("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,h){return Ea(b.aoData,h,"search"===a?"_aFilterData":"_aSortData",c)},1)});J("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return Ea(a.aoData,e,"anCells",b)},1)});J("columns().visible()","column().visible()",
+function(a,b){var c=this,d=this.iterator("column",function(e,h){if(a===q)return e.aoColumns[h].bVisible;var f=e.aoColumns,g=f[h],k=e.aoData,m;if(a!==q&&g.bVisible!==a){if(a){var n=l.inArray(!0,U(f,"bVisible"),h+1);f=0;for(m=k.length;f<m;f++){var p=k[f].nTr;e=k[f].anCells;p&&p.insertBefore(e[h],e[n]||null)}}else l(U(e.aoData,"anCells",h)).detach();g.bVisible=a}});a!==q&&this.iterator("table",function(e){xa(e,e.aoHeader);xa(e,e.aoFooter);e.aiDisplay.length||l(e.nTBody).find("td[colspan]").attr("colspan",
+oa(e));Ca(e);c.iterator("column",function(h,f){F(h,null,"column-visibility",[h,f,a,b])});(b===q||b)&&c.columns.adjust()});return d});J("columns().indexes()","column().index()",function(a){return this.iterator("column",function(b,c){return"visible"===a?ua(b,c):c},1)});y("columns.adjust()",function(){return this.iterator("table",function(a){sa(a)},1)});y("column.index()",function(a,b){if(0!==this.context.length){var c=this.context[0];if("fromVisible"===a||"toData"===a)return ta(c,b);if("fromData"===
+a||"toVisible"===a)return ua(c,b)}});y("column()",function(a,b){return wb(this.columns(a,b))});var Dc=function(a,b,c){var d=a.aoData,e=Wa(a,c),h=kc(Ea(d,e,"anCells")),f=l(lc([],h)),g,k=a.aoColumns.length,m,n,p,t,v,x;return ub("cell",b,function(w){var r="function"===typeof w;if(null===w||w===q||r){m=[];n=0;for(p=e.length;n<p;n++)for(g=e[n],t=0;t<k;t++)v={row:g,column:t},r?(x=d[g],w(v,T(a,g,t),x.anCells?x.anCells[t]:null)&&m.push(v)):m.push(v);return m}if(l.isPlainObject(w))return w.column!==q&&w.row!==
+q&&-1!==l.inArray(w.row,e)?[w]:[];r=f.filter(w).map(function(C,G){return{row:G._DT_CellIndex.row,column:G._DT_CellIndex.column}}).toArray();if(r.length||!w.nodeName)return r;x=l(w).closest("*[data-dt-row]");return x.length?[{row:x.data("dt-row"),column:x.data("dt-column")}]:[]},a,c)};y("cells()",function(a,b,c){l.isPlainObject(a)&&(a.row===q?(c=a,a=null):(c=b,b=null));l.isPlainObject(b)&&(c=b,b=null);if(null===b||b===q)return this.iterator("table",function(n){return Dc(n,a,vb(c))});var d=c?{page:c.page,
+order:c.order,search:c.search}:{},e=this.columns(b,d),h=this.rows(a,d),f,g,k,m;d=this.iterator("table",function(n,p){n=[];f=0;for(g=h[p].length;f<g;f++)for(k=0,m=e[p].length;k<m;k++)n.push({row:h[p][f],column:e[p][k]});return n},1);d=c&&c.selected?this.cells(d,c):d;l.extend(d.selector,{cols:b,rows:a,opts:c});return d});J("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(a,b,c){return(a=a.aoData[b])&&a.anCells?a.anCells[c]:q},1)});y("cells().data()",function(){return this.iterator("cell",
+function(a,b,c){return T(a,b,c)},1)});J("cells().cache()","cell().cache()",function(a){a="search"===a?"_aFilterData":"_aSortData";return this.iterator("cell",function(b,c,d){return b.aoData[c][a][d]},1)});J("cells().render()","cell().render()",function(a){return this.iterator("cell",function(b,c,d){return T(b,c,d,a)},1)});J("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(a,b,c){return{row:b,column:c,columnVisible:ua(a,c)}},1)});J("cells().invalidate()","cell().invalidate()",
+function(a){return this.iterator("cell",function(b,c,d){va(b,c,a,d)})});y("cell()",function(a,b,c){return wb(this.cells(a,b,c))});y("cell().data()",function(a){var b=this.context,c=this[0];if(a===q)return b.length&&c.length?T(b[0],c[0].row,c[0].column):q;Eb(b[0],c[0].row,c[0].column,a);va(b[0],c[0].row,"data",c[0].column);return this});y("order()",function(a,b){var c=this.context;if(a===q)return 0!==c.length?c[0].aaSorting:q;"number"===typeof a?a=[[a,b]]:a.length&&!Array.isArray(a[0])&&(a=Array.prototype.slice.call(arguments));
+return this.iterator("table",function(d){d.aaSorting=a.slice()})});y("order.listener()",function(a,b,c){return this.iterator("table",function(d){fb(d,a,b,c)})});y("order.fixed()",function(a){if(!a){var b=this.context;b=b.length?b[0].aaSortingFixed:q;return Array.isArray(b)?{pre:b}:b}return this.iterator("table",function(c){c.aaSortingFixed=l.extend(!0,{},a)})});y(["columns().order()","column().order()"],function(a){var b=this;return this.iterator("table",function(c,d){var e=[];l.each(b[d],function(h,
+f){e.push([f,a])});c.aaSorting=e})});y("search()",function(a,b,c,d){var e=this.context;return a===q?0!==e.length?e[0].oPreviousSearch.sSearch:q:this.iterator("table",function(h){h.oFeatures.bFilter&&ya(h,l.extend({},h.oPreviousSearch,{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),1)})});J("columns().search()","column().search()",function(a,b,c,d){return this.iterator("column",function(e,h){var f=e.aoPreSearchCols;if(a===q)return f[h].sSearch;e.oFeatures.bFilter&&
+(l.extend(f[h],{sSearch:a+"",bRegex:null===b?!1:b,bSmart:null===c?!0:c,bCaseInsensitive:null===d?!0:d}),ya(e,e.oPreviousSearch,1))})});y("state()",function(){return this.context.length?this.context[0].oSavedState:null});y("state.clear()",function(){return this.iterator("table",function(a){a.fnStateSaveCallback.call(a.oInstance,a,{})})});y("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null});y("state.save()",function(){return this.iterator("table",function(a){Ca(a)})});
+u.versionCheck=u.fnVersionCheck=function(a){var b=u.version.split(".");a=a.split(".");for(var c,d,e=0,h=a.length;e<h;e++)if(c=parseInt(b[e],10)||0,d=parseInt(a[e],10)||0,c!==d)return c>d;return!0};u.isDataTable=u.fnIsDataTable=function(a){var b=l(a).get(0),c=!1;if(a instanceof u.Api)return!0;l.each(u.settings,function(d,e){d=e.nScrollHead?l("table",e.nScrollHead)[0]:null;var h=e.nScrollFoot?l("table",e.nScrollFoot)[0]:null;if(e.nTable===b||d===b||h===b)c=!0});return c};u.tables=u.fnTables=function(a){var b=
+!1;l.isPlainObject(a)&&(b=a.api,a=a.visible);var c=l.map(u.settings,function(d){if(!a||a&&l(d.nTable).is(":visible"))return d.nTable});return b?new B(c):c};u.camelToHungarian=P;y("$()",function(a,b){b=this.rows(b).nodes();b=l(b);return l([].concat(b.filter(a).toArray(),b.find(a).toArray()))});l.each(["on","one","off"],function(a,b){y(b+"()",function(){var c=Array.prototype.slice.call(arguments);c[0]=l.map(c[0].split(/\s/),function(e){return e.match(/\.dt\b/)?e:e+".dt"}).join(" ");var d=l(this.tables().nodes());
+d[b].apply(d,c);return this})});y("clear()",function(){return this.iterator("table",function(a){Ka(a)})});y("settings()",function(){return new B(this.context,this.context)});y("init()",function(){var a=this.context;return a.length?a[0].oInit:null});y("data()",function(){return this.iterator("table",function(a){return U(a.aoData,"_aData")}).flatten()});y("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,h=b.nTBody,f=b.nTHead,
+g=b.nTFoot,k=l(e);h=l(h);var m=l(b.nTableWrapper),n=l.map(b.aoData,function(t){return t.nTr}),p;b.bDestroying=!0;F(b,"aoDestroyCallback","destroy",[b]);a||(new B(b)).columns().visible(!0);m.off(".DT").find(":not(tbody *)").off(".DT");l(z).off(".DT-"+b.sInstance);e!=f.parentNode&&(k.children("thead").detach(),k.append(f));g&&e!=g.parentNode&&(k.children("tfoot").detach(),k.append(g));b.aaSorting=[];b.aaSortingFixed=[];Sa(b);l(n).removeClass(b.asStripeClasses.join(" "));l("th, td",f).removeClass(d.sSortable+
+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);h.children().detach();h.append(n);f=a?"remove":"detach";k[f]();m[f]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),k.css("width",b.sDestroyWidth).removeClass(d.sTable),(p=b.asDestroyStripes.length)&&h.children().each(function(t){l(this).addClass(b.asDestroyStripes[t%p])}));c=l.inArray(b,u.settings);-1!==c&&u.settings.splice(c,1)})});l.each(["column","row","cell"],function(a,b){y(b+"s().every()",function(c){var d=this.selector.opts,e=
+this;return this.iterator(b,function(h,f,g,k,m){c.call(e[b](f,"cell"===b?g:d,"cell"===b?d:q),f,g,k,m)})})});y("i18n()",function(a,b,c){var d=this.context[0];a=na(a)(d.oLanguage);a===q&&(a=b);c!==q&&l.isPlainObject(a)&&(a=a[c]!==q?a[c]:a._);return a.replace("%d",c)});u.version="1.11.5";u.settings=[];u.models={};u.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0,"return":!1};u.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",
+src:null,idx:-1};u.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};u.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,
+25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,
+fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){return{}}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},
+fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",
+sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:l.extend({},u.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};E(u.defaults);u.defaults.column={aDataSort:null,
+iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};E(u.defaults.column);u.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,
+iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],
+aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,jqXHR:null,json:q,oAjaxData:q,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,
+bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==Q(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==Q(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,h=
+e.bPaginate;return e.bServerSide?!1===h||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!h||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};u.ext=M={buttons:{},classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:u.fnVersionCheck,
+iApiIndex:0,oJUIClasses:{},sVersion:u.version};l.extend(M,{afnFiltering:M.search,aTypes:M.type.detect,ofnSearch:M.type.search,oSort:M.type.order,afnSortData:M.order,aoFeatures:M.feature,oApi:M.internal,oStdClasses:M.classes,oPagination:M.pager});l.extend(u.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",
+sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_desc_disabled",sSortableDesc:"sorting_asc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",
+sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var ec=u.ext.pager;l.extend(ec,{simple:function(a,b){return["previous","next"]},full:function(a,b){return["first","previous","next","last"]},numbers:function(a,b){return[Da(a,b)]},simple_numbers:function(a,b){return["previous",Da(a,b),"next"]},
+full_numbers:function(a,b){return["first","previous",Da(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",Da(a,b),"last"]},_numbers:Da,numbers_length:7});l.extend(!0,u.ext.renderer,{pageButton:{_:function(a,b,c,d,e,h){var f=a.oClasses,g=a.oLanguage.oPaginate,k=a.oLanguage.oAria.paginate||{},m,n,p=0,t=function(x,w){var r,C=f.sPageButtonDisabled,G=function(I){Ra(a,I.data.action,!0)};var aa=0;for(r=w.length;aa<r;aa++){var L=w[aa];if(Array.isArray(L)){var O=l("<"+(L.DT_el||"div")+"/>").appendTo(x);
+t(O,L)}else{m=null;n=L;O=a.iTabIndex;switch(L){case "ellipsis":x.append('<span class="ellipsis">&#x2026;</span>');break;case "first":m=g.sFirst;0===e&&(O=-1,n+=" "+C);break;case "previous":m=g.sPrevious;0===e&&(O=-1,n+=" "+C);break;case "next":m=g.sNext;if(0===h||e===h-1)O=-1,n+=" "+C;break;case "last":m=g.sLast;if(0===h||e===h-1)O=-1,n+=" "+C;break;default:m=a.fnFormatNumber(L+1),n=e===L?f.sPageButtonActive:""}null!==m&&(O=l("<a>",{"class":f.sPageButton+" "+n,"aria-controls":a.sTableId,"aria-label":k[L],
+"data-dt-idx":p,tabindex:O,id:0===c&&"string"===typeof L?a.sTableId+"_"+L:null}).html(m).appendTo(x),ob(O,{action:L},G),p++)}}};try{var v=l(b).find(A.activeElement).data("dt-idx")}catch(x){}t(l(b).empty(),d);v!==q&&l(b).find("[data-dt-idx="+v+"]").trigger("focus")}}});l.extend(u.ext.type.detect,[function(a,b){b=b.oLanguage.sDecimal;return tb(a,b)?"num"+b:null},function(a,b){if(a&&!(a instanceof Date)&&!vc.test(a))return null;b=Date.parse(a);return null!==b&&!isNaN(b)||Z(a)?"date":null},function(a,
+b){b=b.oLanguage.sDecimal;return tb(a,b,!0)?"num-fmt"+b:null},function(a,b){b=b.oLanguage.sDecimal;return jc(a,b)?"html-num"+b:null},function(a,b){b=b.oLanguage.sDecimal;return jc(a,b,!0)?"html-num-fmt"+b:null},function(a,b){return Z(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);l.extend(u.ext.type.search,{html:function(a){return Z(a)?a:"string"===typeof a?a.replace(gc," ").replace(Va,""):""},string:function(a){return Z(a)?a:"string"===typeof a?a.replace(gc," "):a}});var Ua=function(a,
+b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=ic(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};l.extend(M.type.order,{"date-pre":function(a){a=Date.parse(a);return isNaN(a)?-Infinity:a},"html-pre":function(a){return Z(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return Z(a)?"":"string"===typeof a?a.toLowerCase():a.toString?a.toString():""},"string-asc":function(a,b){return a<b?-1:a>b?1:0},"string-desc":function(a,b){return a<
+b?1:a>b?-1:0}});Xa("");l.extend(!0,u.ext.renderer,{header:{_:function(a,b,c,d){l(a.nTable).on("order.dt.DT",function(e,h,f,g){a===h&&(e=c.idx,b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass("asc"==g[e]?d.sSortAsc:"desc"==g[e]?d.sSortDesc:c.sSortingClass))})},jqueryui:function(a,b,c,d){l("<div/>").addClass(d.sSortJUIWrapper).append(b.contents()).append(l("<span/>").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);l(a.nTable).on("order.dt.DT",function(e,h,f,g){a===h&&(e=c.idx,b.removeClass(d.sSortAsc+
+" "+d.sSortDesc).addClass("asc"==g[e]?d.sSortAsc:"desc"==g[e]?d.sSortDesc:c.sSortingClass),b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass("asc"==g[e]?d.sSortJUIAsc:"desc"==g[e]?d.sSortJUIDesc:c.sSortingClassJUI))})}}});var yb=function(a){Array.isArray(a)&&(a=a.join(","));return"string"===typeof a?a.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;"):a};u.render=
+{number:function(a,b,c,d,e){return{display:function(h){if("number"!==typeof h&&"string"!==typeof h)return h;var f=0>h?"-":"",g=parseFloat(h);if(isNaN(g))return yb(h);g=g.toFixed(c);h=Math.abs(g);g=parseInt(h,10);h=c?b+(h-g).toFixed(c).substring(2):"";0===g&&0===parseFloat(h)&&(f="");return f+(d||"")+g.toString().replace(/\B(?=(\d{3})+(?!\d))/g,a)+h+(e||"")}}},text:function(){return{display:yb,filter:yb}}};l.extend(u.ext.internal,{_fnExternApiFunc:fc,_fnBuildAjax:Oa,_fnAjaxUpdate:Gb,_fnAjaxParameters:Pb,
+_fnAjaxUpdateDraw:Qb,_fnAjaxDataSrc:za,_fnAddColumn:Ya,_fnColumnOptions:Ga,_fnAdjustColumnSizing:sa,_fnVisibleToColumnIndex:ta,_fnColumnIndexToVisible:ua,_fnVisbleColumns:oa,_fnGetColumns:Ia,_fnColumnTypes:$a,_fnApplyColumnDefs:Db,_fnHungarianMap:E,_fnCamelToHungarian:P,_fnLanguageCompat:ma,_fnBrowserDetect:Bb,_fnAddData:ia,_fnAddTr:Ja,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==q?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return l.inArray(c,a.aoData[b].anCells)},_fnGetCellData:T,
+_fnSetCellData:Eb,_fnSplitObjNotation:cb,_fnGetObjectDataFn:na,_fnSetObjectDataFn:ha,_fnGetDataMaster:db,_fnClearTable:Ka,_fnDeleteIndex:La,_fnInvalidate:va,_fnGetRowElements:bb,_fnCreateTr:ab,_fnBuildHead:Fb,_fnDrawHead:xa,_fnDraw:ja,_fnReDraw:ka,_fnAddOptionsHtml:Ib,_fnDetectHeader:wa,_fnGetUniqueThs:Na,_fnFeatureHtmlFilter:Kb,_fnFilterComplete:ya,_fnFilterCustom:Tb,_fnFilterColumn:Sb,_fnFilter:Rb,_fnFilterCreateSearch:ib,_fnEscapeRegex:jb,_fnFilterData:Ub,_fnFeatureHtmlInfo:Nb,_fnUpdateInfo:Xb,
+_fnInfoMacros:Yb,_fnInitialise:Aa,_fnInitComplete:Pa,_fnLengthChange:kb,_fnFeatureHtmlLength:Jb,_fnFeatureHtmlPaginate:Ob,_fnPageChange:Ra,_fnFeatureHtmlProcessing:Lb,_fnProcessingDisplay:V,_fnFeatureHtmlTable:Mb,_fnScrollDraw:Ha,_fnApplyToChildren:ca,_fnCalculateColumnWidths:Za,_fnThrottle:hb,_fnConvertToWidth:Zb,_fnGetWidestNode:$b,_fnGetMaxLenString:ac,_fnStringToCss:K,_fnSortFlatten:pa,_fnSort:Hb,_fnSortAria:cc,_fnSortListener:nb,_fnSortAttachListener:fb,_fnSortingClasses:Sa,_fnSortData:bc,_fnSaveState:Ca,
+_fnLoadState:dc,_fnImplementState:pb,_fnSettingsFromNode:Ta,_fnLog:da,_fnMap:X,_fnBindAction:ob,_fnCallbackReg:R,_fnCallbackFire:F,_fnLengthOverflow:lb,_fnRenderer:gb,_fnDataSource:Q,_fnRowAttributes:eb,_fnExtend:qb,_fnCalculateEnd:function(){}});l.fn.dataTable=u;u.$=l;l.fn.dataTableSettings=u.settings;l.fn.dataTableExt=u.ext;l.fn.DataTable=function(a){return l(this).dataTable(a).api()};l.each(u,function(a,b){l.fn.DataTable[a]=b});return u});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/federation/federation.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/federation/federation.js
new file mode 100644
index 0000000000000..ac8e3efe11e9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/federation/federation.js
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ready(function() {
+    var table = $('#rms').DataTable();
+    $('#rms tbody').on('click', 'td.details-control', function () {
+        var tr = $(this).closest('tr');
+        var row = table.row(tr);
+        if (row.child.isShown()) {
+            row.child.hide();
+            tr.removeClass('shown');
+        } else {
+            var capabilityArr = scTableData.filter(item => (item.subcluster === row.id()));
+            var capabilityObj = JSON.parse(capabilityArr[0].capability).clusterMetrics;
+            row.child(
+                '<table style="line-height:25px;" >' +
+                '   <tr>' +
+                '      <td>' +
+                '         <h3>Application Metrics</h3>' +
+                '         ApplicationSubmitted* : ' + capabilityObj.appsSubmitted + ' </p>' +
+                '         ApplicationCompleted* : ' + capabilityObj.appsCompleted + ' </p>' +
+                '         ApplicationPending*   : ' + capabilityObj.appsPending + ' </p>' +
+                '         ApplicationRunning*   : ' + capabilityObj.appsRunning + ' </p>' +
+                '         ApplicationFailed*    : ' + capabilityObj.appsFailed + ' </p>' +
+                '         ApplicationKilled*    : ' + capabilityObj.appsKilled + ' </p>' +
+                '      </td>' +
+                '      <td>' +
+                '        <h3>Resource Metrics</h3>' +
+                '        <h4>Memory</h4>' +
+                '        Total Memory : ' + capabilityArr[0].totalmemory + ' </p>' +
+                '        Reserved Memory : ' + capabilityArr[0].reservedmemory + ' </p>' +
+                '        Available Memory : ' + capabilityArr[0].availablememory + ' </p>' +
+                '        Allocated Memory : ' + capabilityArr[0].allocatedmemory + ' </p>' +
+                '        Pending Memory : ' + capabilityArr[0].pendingmemory + ' </p>' +
+                '        <hr />' +
+                '        <h4>VirtualCores</h4>' +
+                '        TotalVirtualCores : ' + capabilityObj.totalVirtualCores + ' </p>' +
+                '        ReservedVirtualCores : ' + capabilityObj.reservedVirtualCores + ' </p>' +
+                '        AvailableVirtualCore : ' + capabilityObj.availableVirtualCores + ' </p>' +
+                '        AllocatedVirtualCores : '+ capabilityObj.allocatedVirtualCores + ' </p>' +
+                '        PendingVirtualCores : ' + capabilityObj.pendingVirtualCores + ' </p>' +
+                '        <h4>Containers</h4>' +
+                '        ContainersAllocated : ' + capabilityObj.containersAllocated + ' </p>' +
+                '        ContainersReserved : ' + capabilityObj.containersReserved + ' </p>' +
+                '        ContainersPending : ' + capabilityObj.containersPending + ' </p>' +
+                '     </td>' +
+                '     <td>' +
+                '        <h3>Node Metrics</h3>' +
+                '         TotalNodes : ' + capabilityObj.totalNodes + ' </p>' +
+                '         LostNodes : ' + capabilityObj.lostNodes + ' </p>' +
+                '         UnhealthyNodes : ' + capabilityObj.unhealthyNodes + ' </p>' +
+                '         DecommissioningNodes : ' + capabilityObj.decommissioningNodes + ' </p>' +
+                '         DecommissionedNodes : ' + capabilityObj.decommissionedNodes + ' </p>' +
+                '         RebootedNodes : ' + capabilityObj.rebootedNodes + ' </p>' +
+                '         ActiveNodes : ' + capabilityObj.activeNodes + ' </p>' +
+                '         ShutdownNodes : ' + capabilityObj.shutdownNodes + ' </p>' +
+                '     </td>' +
+                '  </tr>' +
+                '</table>').show();
+            tr.addClass('shown');
+        }
+    });
+});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.5.1.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.5.1.min.js
deleted file mode 100644
index b0614034ad3a9..0000000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.5.1.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v3.5.1 | (c) JS Foundation and other contributors | jquery.org/license */
-!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),D=function(e,t){return e===t&&(l=!0),0},j={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&j.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(D),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(D).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(D(this,e||[],!1))},not:function(e){return this.pushStack(D(this,e||[],!0))},is:function(e){return!!D(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var j,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^key/,we=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Te=/^([^.]*)(?:\.(.+)|)/;function Ce(){return!0}function Ee(){return!1}function Se(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function ke(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)ke(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Ee;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Ae(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,Ce)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=Te.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=Te.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click",Ce),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Ae(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ce:Ee,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Ee,isPropagationStopped:Ee,isImmediatePropagationStopped:Ee,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ce,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ce,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ce,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:function(e){var t=e.button;return null==e.which&&be.test(e.type)?null!=e.charCode?e.charCode:e.keyCode:!e.which&&void 0!==t&&we.test(e.type)?1&t?1:2&t?3:4&t?2:0:e.which}},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Ae(this,e,Se),!1},trigger:function(){return Ae(this,e),!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return ke(this,e,t,n,r)},one:function(e,t,n,r){return ke(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Ee),this.each(function(){S.event.remove(this,e,n,t)})}});var Ne=/<script|<style|<link/i,De=/checked\s*(?:[^=]|=\s*.checked.)/i,je=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function Pe(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&De.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),Pe(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),Le)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,He),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(je,""),u,l))}return n}function Re(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Oe(o[r],a[r]);else Oe(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Re(this,e,!0)},remove:function(e){return Re(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return Pe(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||qe(this,e).appendChild(e)})},prepend:function(){return Pe(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=qe(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return Pe(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Ne.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return Pe(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Me=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Ie=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},We=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Fe=new RegExp(ne.join("|"),"i");function Be(e,t,n){var r,i,o,a,s=e.style;return(n=n||Ie(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Me.test(a)&&Fe.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function $e(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px",t.style.height="1px",n.style.height="9px",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=3<parseInt(r.height),re.removeChild(e)),a}}))}();var _e=["Webkit","Moz","ms"],ze=E.createElement("div").style,Ue={};function Xe(e){var t=S.cssProps[e]||Ue[e];return t||(e in ze?e:Ue[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=_e.length;while(n--)if((e=_e[n]+t)in ze)return e}(e)||e)}var Ve=/^(none|table(?!-c[ea]).+)/,Ge=/^--/,Ye={position:"absolute",visibility:"hidden",display:"block"},Qe={letterSpacing:"0",fontWeight:"400"};function Je(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Ke(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Ze(e,t,n){var r=Ie(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=Be(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Me.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Ke(e,t,n||(i?"border":"content"),o,r,a)+"px"}function et(e,t,n,r,i){return new et.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Be(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Ge.test(t),l=e.style;if(u||(t=Xe(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Ge.test(t)||(t=Xe(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Be(e,t,r)),"normal"===i&&t in Qe&&(i=Qe[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ve.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Ze(e,u,n):We(e,Ye,function(){return Ze(e,u,n)})},set:function(e,t,n){var r,i=Ie(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Ke(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Ke(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Je(0,t,s)}}}),S.cssHooks.marginLeft=$e(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Be(e,"marginLeft"))||e.getBoundingClientRect().left-We(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Je)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Ie(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=et).prototype={constructor:et,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=et.propHooks[this.prop];return e&&e.get?e.get(this):et.propHooks._default.get(this)},run:function(e){var t,n=et.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):et.propHooks._default.set(this),this}}).init.prototype=et.prototype,(et.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[Xe(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=et.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=et.prototype.init,S.fx.step={};var tt,nt,rt,it,ot=/^(?:toggle|show|hide)$/,at=/queueHooks$/;function st(){nt&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(st):C.setTimeout(st,S.fx.interval),S.fx.tick())}function ut(){return C.setTimeout(function(){tt=void 0}),tt=Date.now()}function lt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ct(e,t,n){for(var r,i=(ft.tweeners[t]||[]).concat(ft.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function ft(o,e,t){var n,a,r=0,i=ft.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=tt||ut(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:tt||ut(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=ft.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ct,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(ft,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],ft.tweeners[n]=ft.tweeners[n]||[],ft.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ot.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ct(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?ft.prefilters.unshift(e):ft.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=ft(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&at.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(lt(r,!0),e,t,n)}}),S.each({slideDown:lt("show"),slideUp:lt("hide"),slideToggle:lt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(tt=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),tt=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){nt||(nt=!0,st())},S.fx.stop=function(){nt=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},rt=E.createElement("input"),it=E.createElement("select").appendChild(E.createElement("option")),rt.type="checkbox",y.checkOn=""!==rt.value,y.optSelected=it.selected,(rt=E.createElement("input")).value="t",rt.type="radio",y.radioValue="t"===rt.value;var pt,dt=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?pt:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),pt={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=dt[t]||S.find.attr;dt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=dt[o],dt[o]=r,r=null!=a(e,t,n)?o:null,dt[o]=i),r}});var ht=/^(?:input|select|textarea|button)$/i,gt=/^(?:a|area)$/i;function vt(e){return(e.match(P)||[]).join(" ")}function yt(e){return e.getAttribute&&e.getAttribute("class")||""}function mt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):ht.test(e.nodeName)||gt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,yt(this)))});if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,yt(this)))});if(!arguments.length)return this.attr("class","");if((e=mt(t)).length)while(n=this[u++])if(i=yt(n),r=1===n.nodeType&&" "+vt(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=vt(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,yt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=mt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=yt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+vt(yt(n))+" ").indexOf(t))return!0;return!1}});var xt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(xt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:vt(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var bt=/^(?:focusinfocus|focusoutblur)$/,wt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!bt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,bt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,wt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,wt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var Tt=C.location,Ct={guid:Date.now()},Et=/\?/;S.parseXML=function(e){var t;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){t=void 0}return t&&!t.getElementsByTagName("parsererror").length||S.error("Invalid XML: "+e),t};var St=/\[\]$/,kt=/\r?\n/g,At=/^(?:submit|button|image|reset|file)$/i,Nt=/^(?:input|select|textarea|keygen)/i;function Dt(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||St.test(n)?i(n,t):Dt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)Dt(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)Dt(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&Nt.test(this.nodeName)&&!At.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(kt,"\r\n")}}):{name:t.name,value:n.replace(kt,"\r\n")}}).get()}});var jt=/%20/g,qt=/#.*$/,Lt=/([?&])_=[^&]*/,Ht=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ot=/^(?:GET|HEAD)$/,Pt=/^\/\//,Rt={},Mt={},It="*/".concat("*"),Wt=E.createElement("a");function Ft(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Bt(t,i,o,a){var s={},u=t===Mt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function $t(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Wt.href=Tt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Tt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Tt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":It,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?$t($t(e,S.ajaxSettings),t):$t(S.ajaxSettings,e)},ajaxPrefilter:Ft(Rt),ajaxTransport:Ft(Mt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Ht.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Tt.href)+"").replace(Pt,Tt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Wt.protocol+"//"+Wt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Bt(Rt,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ot.test(v.type),f=v.url.replace(qt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(jt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Et.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Lt,"$1"),o=(Et.test(f)?"&":"?")+"_="+Ct.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+It+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Bt(Mt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var _t={0:200,1223:204},zt=S.ajaxSettings.xhr();y.cors=!!zt&&"withCredentials"in zt,y.ajax=zt=!!zt,S.ajaxTransport(function(i){var o,a;if(y.cors||zt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(_t[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=vt(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Gt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Gt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Yt=C.jQuery,Qt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Qt),e&&C.jQuery===S&&(C.jQuery=Yt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.6.0.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.6.0.min.js
new file mode 100644
index 0000000000000..200b54e470a5d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.6.0.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}S.fn=S.prototype={jquery:f,constructor:S,length:0,toArray:function(){return s.call(this)},get:function(e){return null==e?s.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=S.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return S.each(this,e)},map:function(n){return this.pushStack(S.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(s.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(S.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(S.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:u,sort:t.sort,splice:t.splice},S.extend=S.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||m(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(S.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||S.isPlainObject(n)?n:{},i=!1,a[t]=S.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},S.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==o.call(e))&&(!(t=r(e))||"function"==typeof(n=v.call(t,"constructor")&&t.constructor)&&a.call(n)===l)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){b(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(p(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},makeArray:function(e,t){var n=t||[];return null!=e&&(p(Object(e))?S.merge(n,"string"==typeof e?[e]:e):u.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:i.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(p(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:y}),"function"==typeof Symbol&&(S.fn[Symbol.iterator]=t[Symbol.iterator]),S.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var d=function(n){var e,d,b,o,i,h,f,g,w,u,l,T,C,a,E,v,s,c,y,S="sizzle"+1*new Date,p=n.document,k=0,r=0,m=ue(),x=ue(),A=ue(),N=ue(),j=function(e,t){return e===t&&(l=!0),0},D={}.hasOwnProperty,t=[],q=t.pop,L=t.push,H=t.push,O=t.slice,P=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},R="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",I="(?:\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",W="\\["+M+"*("+I+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+I+"))|)"+M+"*\\]",F=":("+I+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+W+")*)|.*)\\)|)",B=new RegExp(M+"+","g"),$=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),_=new RegExp("^"+M+"*,"+M+"*"),z=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="<a id='"+S+"'></a><select id='"+S+"-\r\\' msallowcapture=''><option selected=''></option></select>",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="<a href='' disabled='disabled'></a><select disabled='disabled'><option/></select>";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0<se(t,C,null,[e]).length},se.contains=function(e,t){return(e.ownerDocument||e)!=C&&T(e),y(e,t)},se.attr=function(e,t){(e.ownerDocument||e)!=C&&T(e);var n=b.attrHandle[t.toLowerCase()],r=n&&D.call(b.attrHandle,t.toLowerCase())?n(e,t,!E):void 0;return void 0!==r?r:d.attributes||!E?e.getAttribute(t):(r=e.getAttributeNode(t))&&r.specified?r.value:null},se.escape=function(e){return(e+"").replace(re,ie)},se.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},se.uniqueSort=function(e){var t,n=[],r=0,i=0;if(l=!d.detectDuplicates,u=!d.sortStable&&e.slice(0),e.sort(j),l){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)e.splice(n[r],1)}return u=null,e},o=se.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=o(e)}else if(3===i||4===i)return e.nodeValue}else while(t=e[r++])n+=o(t);return n},(b=se.selectors={cacheLength:50,createPseudo:le,match:G,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(B," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(h,e,t,g,v){var y="nth"!==h.slice(0,3),m="last"!==h.slice(-4),x="of-type"===e;return 1===g&&0===v?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u,l=y!==m?"nextSibling":"previousSibling",c=e.parentNode,f=x&&e.nodeName.toLowerCase(),p=!n&&!x,d=!1;if(c){if(y){while(l){a=e;while(a=a[l])if(x?a.nodeName.toLowerCase()===f:1===a.nodeType)return!1;u=l="only"===h&&!u&&"nextSibling"}return!0}if(u=[m?c.firstChild:c.lastChild],m&&p){d=(s=(r=(i=(o=(a=c)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1])&&r[2],a=s&&c.childNodes[s];while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if(1===a.nodeType&&++d&&a===e){i[h]=[k,s,d];break}}else if(p&&(d=s=(r=(i=(o=(a=e)[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]||[])[0]===k&&r[1]),!1===d)while(a=++s&&a&&a[l]||(d=s=0)||u.pop())if((x?a.nodeName.toLowerCase()===f:1===a.nodeType)&&++d&&(p&&((i=(o=a[S]||(a[S]={}))[a.uniqueID]||(o[a.uniqueID]={}))[h]=[k,d]),a===e))break;return(d-=v)===g||d%g==0&&0<=d/g}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||se.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?le(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=P(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:le(function(e){var r=[],i=[],s=f(e.replace($,"$1"));return s[S]?le(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:le(function(t){return function(e){return 0<se(t,e).length}}),contains:le(function(t){return t=t.replace(te,ne),function(e){return-1<(e.textContent||o(e)).indexOf(t)}}),lang:le(function(n){return V.test(n||"")||se.error("unsupported lang: "+n),n=n.replace(te,ne).toLowerCase(),function(e){var t;do{if(t=E?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=n.location&&n.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===a},focus:function(e){return e===C.activeElement&&(!C.hasFocus||C.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:ge(!1),disabled:ge(!0),checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return J.test(e.nodeName)},input:function(e){return Q.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:ve(function(){return[0]}),last:ve(function(e,t){return[t-1]}),eq:ve(function(e,t,n){return[n<0?n+t:n]}),even:ve(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:ve(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:ve(function(e,t,n){for(var r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:ve(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=de(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=he(e);function me(){}function xe(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function be(s,e,t){var u=e.dir,l=e.next,c=l||u,f=t&&"parentNode"===c,p=r++;return e.first?function(e,t,n){while(e=e[u])if(1===e.nodeType||f)return s(e,t,n);return!1}:function(e,t,n){var r,i,o,a=[k,p];if(n){while(e=e[u])if((1===e.nodeType||f)&&s(e,t,n))return!0}else while(e=e[u])if(1===e.nodeType||f)if(i=(o=e[S]||(e[S]={}))[e.uniqueID]||(o[e.uniqueID]={}),l&&l===e.nodeName.toLowerCase())e=e[u]||e;else{if((r=i[c])&&r[0]===k&&r[1]===p)return a[2]=r[2];if((i[c]=a)[2]=s(e,t,n))return!0}return!1}}function we(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Te(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function Ce(d,h,g,v,y,e){return v&&!v[S]&&(v=Ce(v)),y&&!y[S]&&(y=Ce(y,e)),le(function(e,t,n,r){var i,o,a,s=[],u=[],l=t.length,c=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)se(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),f=!d||!e&&h?c:Te(c,s,d,n,r),p=g?y||(e?d:l||v)?[]:t:f;if(g&&g(f,p,n,r),v){i=Te(p,u),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(p[u[o]]=!(f[u[o]]=a))}if(e){if(y||d){if(y){i=[],o=p.length;while(o--)(a=p[o])&&i.push(f[o]=a);y(null,p=[],i,r)}o=p.length;while(o--)(a=p[o])&&-1<(i=y?P(e,a):s[o])&&(e[i]=!(t[i]=a))}}else p=Te(p===t?p.splice(l,p.length):p),y?y(null,t,p,r):H.apply(t,p)})}function Ee(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=be(function(e){return e===i},a,!0),l=be(function(e){return-1<P(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!==w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[be(we(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return Ce(1<s&&we(c),1<s&&xe(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace($,"$1"),t,s<n&&Ee(e.slice(s,n)),n<r&&Ee(e=e.slice(n)),n<r&&xe(e))}c.push(t)}return we(c)}return me.prototype=b.filters=b.pseudos,b.setFilters=new me,h=se.tokenize=function(e,t){var n,r,i,o,a,s,u,l=x[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=_.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=z.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace($," ")}),a=a.slice(n.length)),b.filter)!(r=G[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?se.error(e):x(e,s).slice(0)},f=se.compile=function(e,t){var n,v,y,m,x,r,i=[],o=[],a=A[e+" "];if(!a){t||(t=h(e)),n=t.length;while(n--)(a=Ee(t[n]))[S]?i.push(a):o.push(a);(a=A(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=k+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==C||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==C||(T(o),n=!E);while(s=v[a++])if(s(o,t||C,n)){r.push(o);break}i&&(k=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=q.call(r));f=Te(f)}H.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&se.uniqueSort(r)}return i&&(k=h,w=p),c},m?le(r):r))).selector=e}return a},g=se.select=function(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&h(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&E&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(te,ne),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=G.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(te,ne),ee.test(o[0].type)&&ye(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&xe(o)))return H.apply(n,r),n;break}}}return(l||f(e,c))(r,t,!E,n,!t||ee.test(e)&&ye(t.parentNode)||t),n},d.sortStable=S.split("").sort(j).join("")===S,d.detectDuplicates=!!l,T(),d.sortDetached=ce(function(e){return 1&e.compareDocumentPosition(C.createElement("fieldset"))}),ce(function(e){return e.innerHTML="<a href='#'></a>","#"===e.firstChild.getAttribute("href")})||fe("type|href|height|width",function(e,t,n){if(!n)return e.getAttribute(t,"type"===t.toLowerCase()?1:2)}),d.attributes&&ce(function(e){return e.innerHTML="<input/>",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||fe("value",function(e,t,n){if(!n&&"input"===e.nodeName.toLowerCase())return e.defaultValue}),ce(function(e){return null==e.getAttribute("disabled")})||fe(R,function(e,t,n){var r;if(!n)return!0===e[t]?t.toLowerCase():(r=e.getAttributeNode(t))&&r.specified?r.value:null}),se}(C);S.find=d,S.expr=d.selectors,S.expr[":"]=S.expr.pseudos,S.uniqueSort=S.unique=d.uniqueSort,S.text=d.getText,S.isXMLDoc=d.isXML,S.contains=d.contains,S.escapeSelector=d.escape;var h=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&S(e).is(n))break;r.push(e)}return r},T=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},k=S.expr.match.needsContext;function A(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}var N=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1<i.call(n,e)!==r}):S.filter(n,e,r)}S.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?S.find.matchesSelector(r,e)?[r]:[]:S.find.matches(e,S.grep(t,function(e){return 1===e.nodeType}))},S.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(S(e).filter(function(){for(t=0;t<r;t++)if(S.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)S.find(e,i[t],n);return 1<r?S.uniqueSort(n):n},filter:function(e){return this.pushStack(j(this,e||[],!1))},not:function(e){return this.pushStack(j(this,e||[],!0))},is:function(e){return!!j(this,"string"==typeof e&&k.test(e)?S(e):e||[],!1).length}});var D,q=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(S.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&S(e);if(!k.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&S.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?S.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?i.call(S(e),this[0]):i.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(S.uniqueSort(S.merge(this.get(),S(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),S.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return h(e,"parentNode")},parentsUntil:function(e,t,n){return h(e,"parentNode",n)},next:function(e){return O(e,"nextSibling")},prev:function(e){return O(e,"previousSibling")},nextAll:function(e){return h(e,"nextSibling")},prevAll:function(e){return h(e,"previousSibling")},nextUntil:function(e,t,n){return h(e,"nextSibling",n)},prevUntil:function(e,t,n){return h(e,"previousSibling",n)},siblings:function(e){return T((e.parentNode||{}).firstChild,e)},children:function(e){return T(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(A(e,"template")&&(e=e.content||e),S.merge([],e.childNodes))}},function(r,i){S.fn[r]=function(e,t){var n=S.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=S.filter(t,n)),1<this.length&&(H[r]||S.uniqueSort(n),L.test(r)&&n.reverse()),this.pushStack(n)}});var P=/[^\x20\t\r\n\f]+/g;function R(e){return e}function M(e){throw e}function I(e,t,n,r){var i;try{e&&m(i=e.promise)?i.call(e).done(t).fail(n):e&&m(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}S.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},S.each(e.match(P)||[],function(e,t){n[t]=!0}),n):S.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){S.each(e,function(e,t){m(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==w(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return S.each(arguments,function(e,t){var n;while(-1<(n=S.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<S.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},S.extend({Deferred:function(e){var o=[["notify","progress",S.Callbacks("memory"),S.Callbacks("memory"),2],["resolve","done",S.Callbacks("once memory"),S.Callbacks("once memory"),0,"resolved"],["reject","fail",S.Callbacks("once memory"),S.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return S.Deferred(function(r){S.each(o,function(e,t){var n=m(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&m(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,m(t)?s?t.call(e,l(u,o,R,s),l(u,o,M,s)):(u++,t.call(e,l(u,o,R,s),l(u,o,M,s),l(u,o,R,o.notifyWith))):(a!==R&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){S.Deferred.exceptionHook&&S.Deferred.exceptionHook(e,t.stackTrace),u<=i+1&&(a!==M&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(S.Deferred.getStackHook&&(t.stackTrace=S.Deferred.getStackHook()),C.setTimeout(t))}}return S.Deferred(function(e){o[0][3].add(l(0,e,m(r)?r:R,e.notifyWith)),o[1][3].add(l(0,e,m(t)?t:R)),o[2][3].add(l(0,e,m(n)?n:M))}).promise()},promise:function(e){return null!=e?S.extend(e,a):a}},s={};return S.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=s.call(arguments),o=S.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?s.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(I(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||m(i[t]&&i[t].then)))return o.then();while(t--)I(i[t],a(t),o.reject);return o.promise()}});var W=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;S.Deferred.exceptionHook=function(e,t){C.console&&C.console.warn&&e&&W.test(e.name)&&C.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},S.readyException=function(e){C.setTimeout(function(){throw e})};var F=S.Deferred();function B(){E.removeEventListener("DOMContentLoaded",B),C.removeEventListener("load",B),S.ready()}S.fn.ready=function(e){return F.then(e)["catch"](function(e){S.readyException(e)}),this},S.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--S.readyWait:S.isReady)||(S.isReady=!0)!==e&&0<--S.readyWait||F.resolveWith(E,[S])}}),S.ready.then=F.then,"complete"===E.readyState||"loading"!==E.readyState&&!E.documentElement.doScroll?C.setTimeout(S.ready):(E.addEventListener("DOMContentLoaded",B),C.addEventListener("load",B));var $=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===w(n))for(s in i=!0,n)$(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,m(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(S(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},_=/^-ms-/,z=/-([a-z])/g;function U(e,t){return t.toUpperCase()}function X(e){return e.replace(_,"ms-").replace(z,U)}var V=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function G(){this.expando=S.expando+G.uid++}G.uid=1,G.prototype={cache:function(e){var t=e[this.expando];return t||(t={},V(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[X(t)]=n;else for(r in t)i[X(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][X(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(X):(t=X(t))in r?[t]:t.match(P)||[]).length;while(n--)delete r[t[n]]}(void 0===t||S.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!S.isEmptyObject(t)}};var Y=new G,Q=new G,J=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,K=/[A-Z]/g;function Z(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(K,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:J.test(i)?JSON.parse(i):i)}catch(e){}Q.set(e,t,n)}else n=void 0;return n}S.extend({hasData:function(e){return Q.hasData(e)||Y.hasData(e)},data:function(e,t,n){return Q.access(e,t,n)},removeData:function(e,t){Q.remove(e,t)},_data:function(e,t,n){return Y.access(e,t,n)},_removeData:function(e,t){Y.remove(e,t)}}),S.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=Q.get(o),1===o.nodeType&&!Y.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=X(r.slice(5)),Z(o,r,i[r]));Y.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){Q.set(this,n)}):$(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=Q.get(o,n))?t:void 0!==(t=Z(o,n))?t:void 0;this.each(function(){Q.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){Q.remove(this,e)})}}),S.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=Y.get(e,t),n&&(!r||Array.isArray(n)?r=Y.access(e,t,S.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=S.queue(e,t),r=n.length,i=n.shift(),o=S._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){S.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return Y.get(e,n)||Y.access(e,n,{empty:S.Callbacks("once memory").add(function(){Y.remove(e,[t+"queue",n])})})}}),S.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?S.queue(this[0],t):void 0===n?this:this.each(function(){var e=S.queue(this,t,n);S._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&S.dequeue(this,t)})},dequeue:function(e){return this.each(function(){S.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=S.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=Y.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var ee=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,te=new RegExp("^(?:([+-])=|)("+ee+")([a-z%]*)$","i"),ne=["Top","Right","Bottom","Left"],re=E.documentElement,ie=function(e){return S.contains(e.ownerDocument,e)},oe={composed:!0};re.getRootNode&&(ie=function(e){return S.contains(e.ownerDocument,e)||e.getRootNode(oe)===e.ownerDocument});var ae=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&ie(e)&&"none"===S.css(e,"display")};function se(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return S.css(e,t,"")},u=s(),l=n&&n[3]||(S.cssNumber[t]?"":"px"),c=e.nodeType&&(S.cssNumber[t]||"px"!==l&&+u)&&te.exec(S.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)S.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,S.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ue={};function le(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=Y.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ae(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ue[s])||(o=a.body.appendChild(a.createElement(s)),u=S.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ue[s]=u)))):"none"!==n&&(l[c]="none",Y.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}S.fn.extend({show:function(){return le(this,!0)},hide:function(){return le(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ae(this)?S(this).show():S(this).hide()})}});var ce,fe,pe=/^(?:checkbox|radio)$/i,de=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="<textarea>x</textarea>",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="<option></option>",y.option=!!ce.lastChild;var ge={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n<r;n++)Y.set(e[n],"globalEval",!t||Y.get(t[n],"globalEval"))}ge.tbody=ge.tfoot=ge.colgroup=ge.caption=ge.thead,ge.th=ge.td,y.option||(ge.optgroup=ge.option=[1,"<select multiple='multiple'>","</select>"]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===w(o))S.merge(p,o.nodeType?[o]:o);else if(me.test(o)){a=a||f.appendChild(t.createElement("div")),s=(de.exec(o)||["",""])[1].toLowerCase(),u=ge[s]||ge._default,a.innerHTML=u[1]+S.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;S.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<S.inArray(o,r))i&&i.push(o);else if(l=ie(o),a=ve(f.appendChild(o),"script"),l&&ye(a),n){c=0;while(o=a[c++])he.test(o.type||"")&&n.push(o)}return f}var be=/^([^.]*)(?:\.(.+)|)/;function we(){return!0}function Te(){return!1}function Ce(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ee(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ee(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Te;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return S().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=S.guid++)),e.each(function(){S.event.add(this,t,i,r,n)})}function Se(e,i,o){o?(Y.set(e,i,!1),S.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Y.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(S.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Y.set(this,i,r),t=o(this,i),this[i](),r!==(n=Y.get(this,i))||t?Y.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n&&n.value}else r.length&&(Y.set(this,i,{value:S.event.trigger(S.extend(r[0],S.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Y.get(e,i)&&S.event.add(e,i,we)}S.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.get(t);if(V(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&S.find.matchesSelector(re,i),n.guid||(n.guid=S.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof S&&S.event.triggered!==e.type?S.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(P)||[""]).length;while(l--)d=g=(s=be.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=S.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=S.event.special[d]||{},c=S.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&S.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),S.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Y.hasData(e)&&Y.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(P)||[""]).length;while(l--)if(d=g=(s=be.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=S.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||S.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)S.event.remove(e,d+t[l],n,r,!0);S.isEmptyObject(u)&&Y.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=S.event.fix(e),l=(Y.get(this,"events")||Object.create(null))[u.type]||[],c=S.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=S.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((S.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<S(i,this).index(l):S.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(S.Event.prototype,t,{enumerable:!0,configurable:!0,get:m(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[S.expando]?e:new S.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click",we),!1},trigger:function(e){var t=this||e;return pe.test(t.type)&&t.click&&A(t,"input")&&Se(t,"click"),!0},_default:function(e){var t=e.target;return pe.test(t.type)&&t.click&&A(t,"input")&&Y.get(t,"click")||A(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},S.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},S.Event=function(e,t){if(!(this instanceof S.Event))return new S.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?we:Te,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&S.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[S.expando]=!0},S.Event.prototype={constructor:S.Event,isDefaultPrevented:Te,isPropagationStopped:Te,isImmediatePropagationStopped:Te,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=we,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=we,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=we,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},S.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},S.event.addProp),S.each({focus:"focusin",blur:"focusout"},function(e,t){S.event.special[e]={setup:function(){return Se(this,e,Ce),!1},trigger:function(){return Se(this,e),!0},_default:function(){return!0},delegateType:t}}),S.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){S.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||S.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),S.fn.extend({on:function(e,t,n,r){return Ee(this,e,t,n,r)},one:function(e,t,n,r){return Ee(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,S(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=Te),this.each(function(){S.event.remove(this,e,n,t)})}});var ke=/<script|<style|<link/i,Ae=/checked\s*(?:[^=]|=\s*.checked.)/i,Ne=/^\s*<!(?:\[CDATA\[|--)|(?:\]\]|--)>\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)S.event.add(t,i,s[i][n]);Q.hasData(e)&&(o=Q.access(e),a=S.extend({},o),Q.set(t,a))}}function He(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=m(d);if(h||1<f&&"string"==typeof d&&!y.checkClone&&Ae.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),He(t,r,i,o)});if(f&&(t=(e=xe(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=S.map(ve(e,"script"),De)).length;c<f;c++)u=e,c!==p&&(u=S.clone(u,!0,!0),s&&S.merge(a,ve(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,S.map(a,qe),c=0;c<s;c++)u=a[c],he.test(u.type||"")&&!Y.access(u,"globalEval")&&S.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?S._evalUrl&&!u.noModule&&S._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):b(u.textContent.replace(Ne,""),u,l))}return n}function Oe(e,t,n){for(var r,i=t?S.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||S.cleanData(ve(r)),r.parentNode&&(n&&ie(r)&&ye(ve(r,"script")),r.parentNode.removeChild(r));return e}S.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=ie(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||S.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&pe.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||ve(e),a=a||ve(c),r=0,i=o.length;r<i;r++)Le(o[r],a[r]);else Le(e,c);return 0<(a=ve(c,"script")).length&&ye(a,!f&&ve(e,"script")),c},cleanData:function(e){for(var t,n,r,i=S.event.special,o=0;void 0!==(n=e[o]);o++)if(V(n)){if(t=n[Y.expando]){if(t.events)for(r in t.events)i[r]?S.event.remove(n,r):S.removeEvent(n,r,t.handle);n[Y.expando]=void 0}n[Q.expando]&&(n[Q.expando]=void 0)}}}),S.fn.extend({detach:function(e){return Oe(this,e,!0)},remove:function(e){return Oe(this,e)},text:function(e){return $(this,function(e){return void 0===e?S.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return He(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||je(this,e).appendChild(e)})},prepend:function(){return He(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=je(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return He(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(S.cleanData(ve(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return S.clone(this,e,t)})},html:function(e){return $(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!ke.test(e)&&!ge[(de.exec(e)||["",""])[1].toLowerCase()]){e=S.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(S.cleanData(ve(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return He(this,arguments,function(e){var t=this.parentNode;S.inArray(this,n)<0&&(S.cleanData(ve(this)),t&&t.replaceChild(e,this))},n)}}),S.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){S.fn[e]=function(e){for(var t,n=[],r=S(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),S(r[o])[a](t),u.apply(n,t.get());return this.pushStack(n)}});var Pe=new RegExp("^("+ee+")(?!px)[a-z%]+$","i"),Re=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=C),t.getComputedStyle(e)},Me=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ie=new RegExp(ne.join("|"),"i");function We(e,t,n){var r,i,o,a,s=e.style;return(n=n||Re(e))&&(""!==(a=n.getPropertyValue(t)||n[t])||ie(e)||(a=S.style(e,t)),!y.pixelBoxStyles()&&Pe.test(a)&&Ie.test(t)&&(r=s.width,i=s.minWidth,o=s.maxWidth,s.minWidth=s.maxWidth=s.width=a,a=n.width,s.width=r,s.minWidth=i,s.maxWidth=o)),void 0!==a?a+"":a}function Fe(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",re.appendChild(u).appendChild(l);var e=C.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),re.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=E.createElement("div"),l=E.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",y.clearCloneStyle="content-box"===l.style.backgroundClip,S.extend(y,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=E.createElement("table"),t=E.createElement("tr"),n=E.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",re.appendChild(e).appendChild(t).appendChild(n),r=C.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,re.removeChild(e)),a}}))}();var Be=["Webkit","Moz","ms"],$e=E.createElement("div").style,_e={};function ze(e){var t=S.cssProps[e]||_e[e];return t||(e in $e?e:_e[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Be.length;while(n--)if((e=Be[n]+t)in $e)return e}(e)||e)}var Ue=/^(none|table(?!-c[ea]).+)/,Xe=/^--/,Ve={position:"absolute",visibility:"hidden",display:"block"},Ge={letterSpacing:"0",fontWeight:"400"};function Ye(e,t,n){var r=te.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function Qe(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(u+=S.css(e,n+ne[a],!0,i)),r?("content"===n&&(u-=S.css(e,"padding"+ne[a],!0,i)),"margin"!==n&&(u-=S.css(e,"border"+ne[a]+"Width",!0,i))):(u+=S.css(e,"padding"+ne[a],!0,i),"padding"!==n?u+=S.css(e,"border"+ne[a]+"Width",!0,i):s+=S.css(e,"border"+ne[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function Je(e,t,n){var r=Re(e),i=(!y.boxSizingReliable()||n)&&"border-box"===S.css(e,"boxSizing",!1,r),o=i,a=We(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(Pe.test(a)){if(!n)return a;a="auto"}return(!y.boxSizingReliable()&&i||!y.reliableTrDimensions()&&A(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===S.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===S.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Qe(e,t,n||(i?"border":"content"),o,r,a)+"px"}function Ke(e,t,n,r,i){return new Ke.prototype.init(e,t,n,r,i)}S.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=We(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=X(t),u=Xe.test(t),l=e.style;if(u||(t=ze(s)),a=S.cssHooks[t]||S.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=te.exec(n))&&i[1]&&(n=se(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(S.cssNumber[s]?"":"px")),y.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=X(t);return Xe.test(t)||(t=ze(s)),(a=S.cssHooks[t]||S.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=We(e,t,r)),"normal"===i&&t in Ge&&(i=Ge[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),S.each(["height","width"],function(e,u){S.cssHooks[u]={get:function(e,t,n){if(t)return!Ue.test(S.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?Je(e,u,n):Me(e,Ve,function(){return Je(e,u,n)})},set:function(e,t,n){var r,i=Re(e),o=!y.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===S.css(e,"boxSizing",!1,i),s=n?Qe(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-Qe(e,u,"border",!1,i)-.5)),s&&(r=te.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=S.css(e,u)),Ye(0,t,s)}}}),S.cssHooks.marginLeft=Fe(y.reliableMarginLeft,function(e,t){if(t)return(parseFloat(We(e,"marginLeft"))||e.getBoundingClientRect().left-Me(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),S.each({margin:"",padding:"",border:"Width"},function(i,o){S.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+ne[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(S.cssHooks[i+o].set=Ye)}),S.fn.extend({css:function(e,t){return $(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Re(e),i=t.length;a<i;a++)o[t[a]]=S.css(e,t[a],!1,r);return o}return void 0!==n?S.style(e,t,n):S.css(e,t)},e,t,1<arguments.length)}}),((S.Tween=Ke).prototype={constructor:Ke,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||S.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(S.cssNumber[n]?"":"px")},cur:function(){var e=Ke.propHooks[this.prop];return e&&e.get?e.get(this):Ke.propHooks._default.get(this)},run:function(e){var t,n=Ke.propHooks[this.prop];return this.options.duration?this.pos=t=S.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):Ke.propHooks._default.set(this),this}}).init.prototype=Ke.prototype,(Ke.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=S.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){S.fx.step[e.prop]?S.fx.step[e.prop](e):1!==e.elem.nodeType||!S.cssHooks[e.prop]&&null==e.elem.style[ze(e.prop)]?e.elem[e.prop]=e.now:S.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=Ke.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},S.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},S.fx=Ke.prototype.init,S.fx.step={};var Ze,et,tt,nt,rt=/^(?:toggle|show|hide)$/,it=/queueHooks$/;function ot(){et&&(!1===E.hidden&&C.requestAnimationFrame?C.requestAnimationFrame(ot):C.setTimeout(ot,S.fx.interval),S.fx.tick())}function at(){return C.setTimeout(function(){Ze=void 0}),Ze=Date.now()}function st(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=ne[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function ut(e,t,n){for(var r,i=(lt.tweeners[t]||[]).concat(lt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function lt(o,e,t){var n,a,r=0,i=lt.prefilters.length,s=S.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=Ze||at(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:S.extend({},e),opts:S.extend(!0,{specialEasing:{},easing:S.easing._default},t),originalProperties:e,originalOptions:t,startTime:Ze||at(),duration:t.duration,tweens:[],createTween:function(e,t){var n=S.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=X(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=S.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=lt.prefilters[r].call(l,o,c,l.opts))return m(n.stop)&&(S._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return S.map(c,ut,l),m(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),S.fx.timer(S.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}S.Animation=S.extend(lt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return se(n.elem,e,te.exec(t),n),n}]},tweener:function(e,t){m(e)?(t=e,e=["*"]):e=e.match(P);for(var n,r=0,i=e.length;r<i;r++)n=e[r],lt.tweeners[n]=lt.tweeners[n]||[],lt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ae(e),v=Y.get(e,"fxshow");for(r in n.queue||(null==(a=S._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,S.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],rt.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||S.style(e,r)}if((u=!S.isEmptyObject(t))||!S.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=Y.get(e,"display")),"none"===(c=S.css(e,"display"))&&(l?c=l:(le([e],!0),l=e.style.display||l,c=S.css(e,"display"),le([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===S.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=Y.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&le([e],!0),p.done(function(){for(r in g||le([e]),Y.remove(e,"fxshow"),d)S.style(e,r,d[r])})),u=ut(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?lt.prefilters.unshift(e):lt.prefilters.push(e)}}),S.speed=function(e,t,n){var r=e&&"object"==typeof e?S.extend({},e):{complete:n||!n&&t||m(e)&&e,duration:e,easing:n&&t||t&&!m(t)&&t};return S.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in S.fx.speeds?r.duration=S.fx.speeds[r.duration]:r.duration=S.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){m(r.old)&&r.old.call(this),r.queue&&S.dequeue(this,r.queue)},r},S.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ae).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=S.isEmptyObject(t),o=S.speed(e,n,r),a=function(){var e=lt(this,S.extend({},t),o);(i||Y.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=S.timers,r=Y.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&it.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||S.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=Y.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=S.timers,o=n?n.length:0;for(t.finish=!0,S.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),S.each(["toggle","show","hide"],function(e,r){var i=S.fn[r];S.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(st(r,!0),e,t,n)}}),S.each({slideDown:st("show"),slideUp:st("hide"),slideToggle:st("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){S.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),S.timers=[],S.fx.tick=function(){var e,t=0,n=S.timers;for(Ze=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||S.fx.stop(),Ze=void 0},S.fx.timer=function(e){S.timers.push(e),S.fx.start()},S.fx.interval=13,S.fx.start=function(){et||(et=!0,ot())},S.fx.stop=function(){et=null},S.fx.speeds={slow:600,fast:200,_default:400},S.fn.delay=function(r,e){return r=S.fx&&S.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=C.setTimeout(e,r);t.stop=function(){C.clearTimeout(n)}})},tt=E.createElement("input"),nt=E.createElement("select").appendChild(E.createElement("option")),tt.type="checkbox",y.checkOn=""!==tt.value,y.optSelected=nt.selected,(tt=E.createElement("input")).value="t",tt.type="radio",y.radioValue="t"===tt.value;var ct,ft=S.expr.attrHandle;S.fn.extend({attr:function(e,t){return $(this,S.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){S.removeAttr(this,e)})}}),S.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?S.prop(e,t,n):(1===o&&S.isXMLDoc(e)||(i=S.attrHooks[t.toLowerCase()]||(S.expr.match.bool.test(t)?ct:void 0)),void 0!==n?null===n?void S.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=S.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!y.radioValue&&"radio"===t&&A(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(P);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),ct={set:function(e,t,n){return!1===t?S.removeAttr(e,n):e.setAttribute(n,n),n}},S.each(S.expr.match.bool.source.match(/\w+/g),function(e,t){var a=ft[t]||S.find.attr;ft[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=ft[o],ft[o]=r,r=null!=a(e,t,n)?o:null,ft[o]=i),r}});var pt=/^(?:input|select|textarea|button)$/i,dt=/^(?:a|area)$/i;function ht(e){return(e.match(P)||[]).join(" ")}function gt(e){return e.getAttribute&&e.getAttribute("class")||""}function vt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(P)||[]}S.fn.extend({prop:function(e,t){return $(this,S.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[S.propFix[e]||e]})}}),S.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&S.isXMLDoc(e)||(t=S.propFix[t]||t,i=S.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=S.find.attr(e,"tabindex");return t?parseInt(t,10):pt.test(e.nodeName)||dt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),y.optSelected||(S.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),S.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){S.propFix[this.toLowerCase()]=this}),S.fn.extend({addClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).addClass(t.call(this,e,gt(this)))});if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])r.indexOf(" "+o+" ")<0&&(r+=o+" ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},removeClass:function(t){var e,n,r,i,o,a,s,u=0;if(m(t))return this.each(function(e){S(this).removeClass(t.call(this,e,gt(this)))});if(!arguments.length)return this.attr("class","");if((e=vt(t)).length)while(n=this[u++])if(i=gt(n),r=1===n.nodeType&&" "+ht(i)+" "){a=0;while(o=e[a++])while(-1<r.indexOf(" "+o+" "))r=r.replace(" "+o+" "," ");i!==(s=ht(r))&&n.setAttribute("class",s)}return this},toggleClass:function(i,t){var o=typeof i,a="string"===o||Array.isArray(i);return"boolean"==typeof t&&a?t?this.addClass(i):this.removeClass(i):m(i)?this.each(function(e){S(this).toggleClass(i.call(this,e,gt(this),t),t)}):this.each(function(){var e,t,n,r;if(a){t=0,n=S(this),r=vt(i);while(e=r[t++])n.hasClass(e)?n.removeClass(e):n.addClass(e)}else void 0!==i&&"boolean"!==o||((e=gt(this))&&Y.set(this,"__className__",e),this.setAttribute&&this.setAttribute("class",e||!1===i?"":Y.get(this,"__className__")||""))})},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+ht(gt(n))+" ").indexOf(t))return!0;return!1}});var yt=/\r/g;S.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=m(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,S(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=S.map(t,function(e){return null==e?"":e+""})),(r=S.valHooks[this.type]||S.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=S.valHooks[t.type]||S.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(yt,""):null==e?"":e:void 0}}),S.extend({valHooks:{option:{get:function(e){var t=S.find.attr(e,"value");return null!=t?t:ht(S.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!A(n.parentNode,"optgroup"))){if(t=S(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=S.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<S.inArray(S.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),S.each(["radio","checkbox"],function(){S.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<S.inArray(S(e).val(),t)}},y.checkOn||(S.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})}),y.focusin="onfocusin"in C;var mt=/^(?:focusinfocus|focusoutblur)$/,xt=function(e){e.stopPropagation()};S.extend(S.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||E],d=v.call(e,"type")?e.type:e,h=v.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||E,3!==n.nodeType&&8!==n.nodeType&&!mt.test(d+S.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[S.expando]?e:new S.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:S.makeArray(t,[e]),c=S.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!x(n)){for(s=c.delegateType||d,mt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||E)&&p.push(a.defaultView||a.parentWindow||C)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(Y.get(o,"events")||Object.create(null))[e.type]&&Y.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&V(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!V(n)||u&&m(n[d])&&!x(n)&&((a=n[u])&&(n[u]=null),S.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,xt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,xt),S.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=S.extend(new S.Event,n,{type:e,isSimulated:!0});S.event.trigger(r,null,t)}}),S.fn.extend({trigger:function(e,t){return this.each(function(){S.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return S.event.trigger(e,t,n,!0)}}),y.focusin||S.each({focus:"focusin",blur:"focusout"},function(n,r){var i=function(e){S.event.simulate(r,e.target,S.event.fix(e))};S.event.special[r]={setup:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r);t||e.addEventListener(n,i,!0),Y.access(e,r,(t||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=Y.access(e,r)-1;t?Y.access(e,r,t):(e.removeEventListener(n,i,!0),Y.remove(e,r))}}});var bt=C.location,wt={guid:Date.now()},Tt=/\?/;S.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new C.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||S.error("Invalid XML: "+(n?S.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Ct=/\[\]$/,Et=/\r?\n/g,St=/^(?:submit|button|image|reset|file)$/i,kt=/^(?:input|select|textarea|keygen)/i;function At(n,e,r,i){var t;if(Array.isArray(e))S.each(e,function(e,t){r||Ct.test(n)?i(n,t):At(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==w(e))i(n,e);else for(t in e)At(n+"["+t+"]",e[t],r,i)}S.param=function(e,t){var n,r=[],i=function(e,t){var n=m(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!S.isPlainObject(e))S.each(e,function(){i(this.name,this.value)});else for(n in e)At(n,e[n],t,i);return r.join("&")},S.fn.extend({serialize:function(){return S.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=S.prop(this,"elements");return e?S.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!S(this).is(":disabled")&&kt.test(this.nodeName)&&!St.test(e)&&(this.checked||!pe.test(e))}).map(function(e,t){var n=S(this).val();return null==n?null:Array.isArray(n)?S.map(n,function(e){return{name:t.name,value:e.replace(Et,"\r\n")}}):{name:t.name,value:n.replace(Et,"\r\n")}}).get()}});var Nt=/%20/g,jt=/#.*$/,Dt=/([?&])_=[^&]*/,qt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Lt=/^(?:GET|HEAD)$/,Ht=/^\/\//,Ot={},Pt={},Rt="*/".concat("*"),Mt=E.createElement("a");function It(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(P)||[];if(m(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Wt(t,i,o,a){var s={},u=t===Pt;function l(e){var r;return s[e]=!0,S.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Ft(e,t){var n,r,i=S.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&S.extend(!0,e,r),e}Mt.href=bt.href,S.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:bt.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(bt.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":Rt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":S.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Ft(Ft(e,S.ajaxSettings),t):Ft(S.ajaxSettings,e)},ajaxPrefilter:It(Ot),ajaxTransport:It(Pt),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=S.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?S(y):S.event,x=S.Deferred(),b=S.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=qt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||bt.href)+"").replace(Ht,bt.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(P)||[""],null==v.crossDomain){r=E.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Mt.protocol+"//"+Mt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=S.param(v.data,v.traditional)),Wt(Ot,v,t,T),h)return T;for(i in(g=S.event&&v.global)&&0==S.active++&&S.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Lt.test(v.type),f=v.url.replace(jt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Nt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(Tt.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(Dt,"$1"),o=(Tt.test(f)?"&":"?")+"_="+wt.guid+++o),v.url=f+o),v.ifModified&&(S.lastModified[f]&&T.setRequestHeader("If-Modified-Since",S.lastModified[f]),S.etag[f]&&T.setRequestHeader("If-None-Match",S.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+Rt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Wt(Pt,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=C.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&C.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<S.inArray("script",v.dataTypes)&&S.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(S.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(S.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--S.active||S.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return S.get(e,t,n,"json")},getScript:function(e,t){return S.get(e,void 0,t,"script")}}),S.each(["get","post"],function(e,i){S[i]=function(e,t,n,r){return m(t)&&(r=r||n,n=t,t=void 0),S.ajax(S.extend({url:e,type:i,dataType:r,data:t,success:n},S.isPlainObject(e)&&e))}}),S.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),S._evalUrl=function(e,t,n){return S.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){S.globalEval(e,t,n)}})},S.fn.extend({wrapAll:function(e){var t;return this[0]&&(m(e)&&(e=e.call(this[0])),t=S(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return m(n)?this.each(function(e){S(this).wrapInner(n.call(this,e))}):this.each(function(){var e=S(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=m(t);return this.each(function(e){S(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){S(this).replaceWith(this.childNodes)}),this}}),S.expr.pseudos.hidden=function(e){return!S.expr.pseudos.visible(e)},S.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},S.ajaxSettings.xhr=function(){try{return new C.XMLHttpRequest}catch(e){}};var Bt={0:200,1223:204},$t=S.ajaxSettings.xhr();y.cors=!!$t&&"withCredentials"in $t,y.ajax=$t=!!$t,S.ajaxTransport(function(i){var o,a;if(y.cors||$t&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Bt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&C.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),S.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),S.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return S.globalEval(e),e}}}),S.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),S.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=S("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=ht(e.slice(s)),e=e.slice(0,s)),m(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&S.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?S("<div>").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var Xt=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;S.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),m(e))return r=s.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(s.call(arguments)))}).guid=e.guid=e.guid||S.guid++,i},S.holdReady=function(e){e?S.readyWait++:S.ready(!0)},S.isArray=Array.isArray,S.parseJSON=JSON.parse,S.nodeName=A,S.isFunction=m,S.isWindow=x,S.camelCase=X,S.type=w,S.now=Date.now,S.isNumeric=function(e){var t=S.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},S.trim=function(e){return null==e?"":(e+"").replace(Xt,"")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return S});var Vt=C.jQuery,Gt=C.$;return S.noConflict=function(e){return C.$===S&&(C.$=Gt),e&&C.jQuery===S&&(C.jQuery=Vt),S},"undefined"==typeof e&&(C.jQuery=C.$=S),S});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js
deleted file mode 100644
index 25398a1674150..0000000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js
+++ /dev/null
@@ -1,13 +0,0 @@
-/*! jQuery UI - v1.12.1 - 2016-09-14
-* http://jqueryui.com
-* Includes: widget.js, position.js, data.js, disable-selection.js, effect.js, effects/effect-blind.js, effects/effect-bounce.js, effects/effect-clip.js, effects/effect-drop.js, effects/effect-explode.js, effects/effect-fade.js, effects/effect-fold.js, effects/effect-highlight.js, effects/effect-puff.js, effects/effect-pulsate.js, effects/effect-scale.js, effects/effect-shake.js, effects/effect-size.js, effects/effect-slide.js, effects/effect-transfer.js, focusable.js, form-reset-mixin.js, jquery-1-7.js, keycode.js, labels.js, scroll-parent.js, tabbable.js, unique-id.js, widgets/accordion.js, widgets/autocomplete.js, widgets/button.js, widgets/checkboxradio.js, widgets/controlgroup.js, widgets/datepicker.js, widgets/dialog.js, widgets/draggable.js, widgets/droppable.js, widgets/menu.js, widgets/mouse.js, widgets/progressbar.js, widgets/resizable.js, widgets/selectable.js, widgets/selectmenu.js, widgets/slider.js, widgets/sortable.js, widgets/spinner.js, widgets/tabs.js, widgets/tooltip.js
-* Copyright jQuery Foundation and other contributors; Licensed MIT */
-
-(function(t){"function"==typeof define&&define.amd?define(["jquery"],t):t(jQuery)})(function(t){function e(t){for(var e=t.css("visibility");"inherit"===e;)t=t.parent(),e=t.css("visibility");return"hidden"!==e}function i(t){for(var e,i;t.length&&t[0]!==document;){if(e=t.css("position"),("absolute"===e||"relative"===e||"fixed"===e)&&(i=parseInt(t.css("zIndex"),10),!isNaN(i)&&0!==i))return i;t=t.parent()}return 0}function s(){this._curInst=null,this._keyEvent=!1,this._disabledInputs=[],this._datepickerShowing=!1,this._inDialog=!1,this._mainDivId="ui-datepicker-div",this._inlineClass="ui-datepicker-inline",this._appendClass="ui-datepicker-append",this._triggerClass="ui-datepicker-trigger",this._dialogClass="ui-datepicker-dialog",this._disableClass="ui-datepicker-disabled",this._unselectableClass="ui-datepicker-unselectable",this._currentClass="ui-datepicker-current-day",this._dayOverClass="ui-datepicker-days-cell-over",this.regional=[],this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su","Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:!1,showMonthAfterYear:!1,yearSuffix:""},this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:!1,hideIfNoPrevNext:!1,navigationAsDateFormat:!1,gotoCurrent:!1,changeMonth:!1,changeYear:!1,yearRange:"c-10:c+10",showOtherMonths:!1,selectOtherMonths:!1,showWeek:!1,calculateWeek:this.iso8601Week,shortYearCutoff:"+10",minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:!0,showButtonPanel:!1,autoSize:!1,disabled:!1},t.extend(this._defaults,this.regional[""]),this.regional.en=t.extend(!0,{},this.regional[""]),this.regional["en-US"]=t.extend(!0,{},this.regional.en),this.dpDiv=n(t("<div id='"+this._mainDivId+"' class='ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all'></div>"))}function n(e){var i="button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a";return e.on("mouseout",i,function(){t(this).removeClass("ui-state-hover"),-1!==this.className.indexOf("ui-datepicker-prev")&&t(this).removeClass("ui-datepicker-prev-hover"),-1!==this.className.indexOf("ui-datepicker-next")&&t(this).removeClass("ui-datepicker-next-hover")}).on("mouseover",i,o)}function o(){t.datepicker._isDisabledDatepicker(m.inline?m.dpDiv.parent()[0]:m.input[0])||(t(this).parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover"),t(this).addClass("ui-state-hover"),-1!==this.className.indexOf("ui-datepicker-prev")&&t(this).addClass("ui-datepicker-prev-hover"),-1!==this.className.indexOf("ui-datepicker-next")&&t(this).addClass("ui-datepicker-next-hover"))}function a(e,i){t.extend(e,i);for(var s in i)null==i[s]&&(e[s]=i[s]);return e}function r(t){return function(){var e=this.element.val();t.apply(this,arguments),this._refresh(),e!==this.element.val()&&this._trigger("change")}}t.ui=t.ui||{},t.ui.version="1.12.1";var h=0,l=Array.prototype.slice;t.cleanData=function(e){return function(i){var s,n,o;for(o=0;null!=(n=i[o]);o++)try{s=t._data(n,"events"),s&&s.remove&&t(n).triggerHandler("remove")}catch(a){}e(i)}}(t.cleanData),t.widget=function(e,i,s){var n,o,a,r={},h=e.split(".")[0];e=e.split(".")[1];var l=h+"-"+e;return s||(s=i,i=t.Widget),t.isArray(s)&&(s=t.extend.apply(null,[{}].concat(s))),t.expr[":"][l.toLowerCase()]=function(e){return!!t.data(e,l)},t[h]=t[h]||{},n=t[h][e],o=t[h][e]=function(t,e){return this._createWidget?(arguments.length&&this._createWidget(t,e),void 0):new o(t,e)},t.extend(o,n,{version:s.version,_proto:t.extend({},s),_childConstructors:[]}),a=new i,a.options=t.widget.extend({},a.options),t.each(s,function(e,s){return t.isFunction(s)?(r[e]=function(){function t(){return i.prototype[e].apply(this,arguments)}function n(t){return i.prototype[e].apply(this,t)}return function(){var e,i=this._super,o=this._superApply;return this._super=t,this._superApply=n,e=s.apply(this,arguments),this._super=i,this._superApply=o,e}}(),void 0):(r[e]=s,void 0)}),o.prototype=t.widget.extend(a,{widgetEventPrefix:n?a.widgetEventPrefix||e:e},r,{constructor:o,namespace:h,widgetName:e,widgetFullName:l}),n?(t.each(n._childConstructors,function(e,i){var s=i.prototype;t.widget(s.namespace+"."+s.widgetName,o,i._proto)}),delete n._childConstructors):i._childConstructors.push(o),t.widget.bridge(e,o),o},t.widget.extend=function(e){for(var i,s,n=l.call(arguments,1),o=0,a=n.length;a>o;o++)for(i in n[o])s=n[o][i],n[o].hasOwnProperty(i)&&void 0!==s&&(e[i]=t.isPlainObject(s)?t.isPlainObject(e[i])?t.widget.extend({},e[i],s):t.widget.extend({},s):s);return e},t.widget.bridge=function(e,i){var s=i.prototype.widgetFullName||e;t.fn[e]=function(n){var o="string"==typeof n,a=l.call(arguments,1),r=this;return o?this.length||"instance"!==n?this.each(function(){var i,o=t.data(this,s);return"instance"===n?(r=o,!1):o?t.isFunction(o[n])&&"_"!==n.charAt(0)?(i=o[n].apply(o,a),i!==o&&void 0!==i?(r=i&&i.jquery?r.pushStack(i.get()):i,!1):void 0):t.error("no such method '"+n+"' for "+e+" widget instance"):t.error("cannot call methods on "+e+" prior to initialization; "+"attempted to call method '"+n+"'")}):r=void 0:(a.length&&(n=t.widget.extend.apply(null,[n].concat(a))),this.each(function(){var e=t.data(this,s);e?(e.option(n||{}),e._init&&e._init()):t.data(this,s,new i(n,this))})),r}},t.Widget=function(){},t.Widget._childConstructors=[],t.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",defaultElement:"<div>",options:{classes:{},disabled:!1,create:null},_createWidget:function(e,i){i=t(i||this.defaultElement||this)[0],this.element=t(i),this.uuid=h++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=t(),this.hoverable=t(),this.focusable=t(),this.classesElementLookup={},i!==this&&(t.data(i,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===i&&this.destroy()}}),this.document=t(i.style?i.ownerDocument:i.document||i),this.window=t(this.document[0].defaultView||this.document[0].parentWindow)),this.options=t.widget.extend({},this.options,this._getCreateOptions(),e),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:t.noop,_create:t.noop,_init:t.noop,destroy:function(){var e=this;this._destroy(),t.each(this.classesElementLookup,function(t,i){e._removeClass(i,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:t.noop,widget:function(){return this.element},option:function(e,i){var s,n,o,a=e;if(0===arguments.length)return t.widget.extend({},this.options);if("string"==typeof e)if(a={},s=e.split("."),e=s.shift(),s.length){for(n=a[e]=t.widget.extend({},this.options[e]),o=0;s.length-1>o;o++)n[s[o]]=n[s[o]]||{},n=n[s[o]];if(e=s.pop(),1===arguments.length)return void 0===n[e]?null:n[e];n[e]=i}else{if(1===arguments.length)return void 0===this.options[e]?null:this.options[e];a[e]=i}return this._setOptions(a),this},_setOptions:function(t){var e;for(e in t)this._setOption(e,t[e]);return this},_setOption:function(t,e){return"classes"===t&&this._setOptionClasses(e),this.options[t]=e,"disabled"===t&&this._setOptionDisabled(e),this},_setOptionClasses:function(e){var i,s,n;for(i in e)n=this.classesElementLookup[i],e[i]!==this.options.classes[i]&&n&&n.length&&(s=t(n.get()),this._removeClass(n,i),s.addClass(this._classes({element:s,keys:i,classes:e,add:!0})))},_setOptionDisabled:function(t){this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,!!t),t&&(this._removeClass(this.hoverable,null,"ui-state-hover"),this._removeClass(this.focusable,null,"ui-state-focus"))},enable:function(){return this._setOptions({disabled:!1})},disable:function(){return this._setOptions({disabled:!0})},_classes:function(e){function i(i,o){var a,r;for(r=0;i.length>r;r++)a=n.classesElementLookup[i[r]]||t(),a=e.add?t(t.unique(a.get().concat(e.element.get()))):t(a.not(e.element).get()),n.classesElementLookup[i[r]]=a,s.push(i[r]),o&&e.classes[i[r]]&&s.push(e.classes[i[r]])}var s=[],n=this;return e=t.extend({element:this.element,classes:this.options.classes||{}},e),this._on(e.element,{remove:"_untrackClassesElement"}),e.keys&&i(e.keys.match(/\S+/g)||[],!0),e.extra&&i(e.extra.match(/\S+/g)||[]),s.join(" ")},_untrackClassesElement:function(e){var i=this;t.each(i.classesElementLookup,function(s,n){-1!==t.inArray(e.target,n)&&(i.classesElementLookup[s]=t(n.not(e.target).get()))})},_removeClass:function(t,e,i){return this._toggleClass(t,e,i,!1)},_addClass:function(t,e,i){return this._toggleClass(t,e,i,!0)},_toggleClass:function(t,e,i,s){s="boolean"==typeof s?s:i;var n="string"==typeof t||null===t,o={extra:n?e:i,keys:n?t:e,element:n?this.element:t,add:s};return o.element.toggleClass(this._classes(o),s),this},_on:function(e,i,s){var n,o=this;"boolean"!=typeof e&&(s=i,i=e,e=!1),s?(i=n=t(i),this.bindings=this.bindings.add(i)):(s=i,i=this.element,n=this.widget()),t.each(s,function(s,a){function r(){return e||o.options.disabled!==!0&&!t(this).hasClass("ui-state-disabled")?("string"==typeof a?o[a]:a).apply(o,arguments):void 0}"string"!=typeof a&&(r.guid=a.guid=a.guid||r.guid||t.guid++);var h=s.match(/^([\w:-]*)\s*(.*)$/),l=h[1]+o.eventNamespace,c=h[2];c?n.on(l,c,r):i.on(l,r)})},_off:function(e,i){i=(i||"").split(" ").join(this.eventNamespace+" ")+this.eventNamespace,e.off(i).off(i),this.bindings=t(this.bindings.not(e).get()),this.focusable=t(this.focusable.not(e).get()),this.hoverable=t(this.hoverable.not(e).get())},_delay:function(t,e){function i(){return("string"==typeof t?s[t]:t).apply(s,arguments)}var s=this;return setTimeout(i,e||0)},_hoverable:function(e){this.hoverable=this.hoverable.add(e),this._on(e,{mouseenter:function(e){this._addClass(t(e.currentTarget),null,"ui-state-hover")},mouseleave:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-hover")}})},_focusable:function(e){this.focusable=this.focusable.add(e),this._on(e,{focusin:function(e){this._addClass(t(e.currentTarget),null,"ui-state-focus")},focusout:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-focus")}})},_trigger:function(e,i,s){var n,o,a=this.options[e];if(s=s||{},i=t.Event(i),i.type=(e===this.widgetEventPrefix?e:this.widgetEventPrefix+e).toLowerCase(),i.target=this.element[0],o=i.originalEvent)for(n in o)n in i||(i[n]=o[n]);return this.element.trigger(i,s),!(t.isFunction(a)&&a.apply(this.element[0],[i].concat(s))===!1||i.isDefaultPrevented())}},t.each({show:"fadeIn",hide:"fadeOut"},function(e,i){t.Widget.prototype["_"+e]=function(s,n,o){"string"==typeof n&&(n={effect:n});var a,r=n?n===!0||"number"==typeof n?i:n.effect||i:e;n=n||{},"number"==typeof n&&(n={duration:n}),a=!t.isEmptyObject(n),n.complete=o,n.delay&&s.delay(n.delay),a&&t.effects&&t.effects.effect[r]?s[e](n):r!==e&&s[r]?s[r](n.duration,n.easing,o):s.queue(function(i){t(this)[e](),o&&o.call(s[0]),i()})}}),t.widget,function(){function e(t,e,i){return[parseFloat(t[0])*(u.test(t[0])?e/100:1),parseFloat(t[1])*(u.test(t[1])?i/100:1)]}function i(e,i){return parseInt(t.css(e,i),10)||0}function s(e){var i=e[0];return 9===i.nodeType?{width:e.width(),height:e.height(),offset:{top:0,left:0}}:t.isWindow(i)?{width:e.width(),height:e.height(),offset:{top:e.scrollTop(),left:e.scrollLeft()}}:i.preventDefault?{width:0,height:0,offset:{top:i.pageY,left:i.pageX}}:{width:e.outerWidth(),height:e.outerHeight(),offset:e.offset()}}var n,o=Math.max,a=Math.abs,r=/left|center|right/,h=/top|center|bottom/,l=/[\+\-]\d+(\.[\d]+)?%?/,c=/^\w+/,u=/%$/,d=t.fn.position;t.position={scrollbarWidth:function(){if(void 0!==n)return n;var e,i,s=t("<div style='display:block;position:absolute;width:50px;height:50px;overflow:hidden;'><div style='height:100px;width:auto;'></div></div>"),o=s.children()[0];return t("body").append(s),e=o.offsetWidth,s.css("overflow","scroll"),i=o.offsetWidth,e===i&&(i=s[0].clientWidth),s.remove(),n=e-i},getScrollInfo:function(e){var i=e.isWindow||e.isDocument?"":e.element.css("overflow-x"),s=e.isWindow||e.isDocument?"":e.element.css("overflow-y"),n="scroll"===i||"auto"===i&&e.width<e.element[0].scrollWidth,o="scroll"===s||"auto"===s&&e.height<e.element[0].scrollHeight;return{width:o?t.position.scrollbarWidth():0,height:n?t.position.scrollbarWidth():0}},getWithinInfo:function(e){var i=t(e||window),s=t.isWindow(i[0]),n=!!i[0]&&9===i[0].nodeType,o=!s&&!n;return{element:i,isWindow:s,isDocument:n,offset:o?t(e).offset():{left:0,top:0},scrollLeft:i.scrollLeft(),scrollTop:i.scrollTop(),width:i.outerWidth(),height:i.outerHeight()}}},t.fn.position=function(n){if(!n||!n.of)return d.apply(this,arguments);n=t.extend({},n);var u,p,f,g,m,_,v=t(n.of),b=t.position.getWithinInfo(n.within),y=t.position.getScrollInfo(b),w=(n.collision||"flip").split(" "),k={};return _=s(v),v[0].preventDefault&&(n.at="left top"),p=_.width,f=_.height,g=_.offset,m=t.extend({},g),t.each(["my","at"],function(){var t,e,i=(n[this]||"").split(" ");1===i.length&&(i=r.test(i[0])?i.concat(["center"]):h.test(i[0])?["center"].concat(i):["center","center"]),i[0]=r.test(i[0])?i[0]:"center",i[1]=h.test(i[1])?i[1]:"center",t=l.exec(i[0]),e=l.exec(i[1]),k[this]=[t?t[0]:0,e?e[0]:0],n[this]=[c.exec(i[0])[0],c.exec(i[1])[0]]}),1===w.length&&(w[1]=w[0]),"right"===n.at[0]?m.left+=p:"center"===n.at[0]&&(m.left+=p/2),"bottom"===n.at[1]?m.top+=f:"center"===n.at[1]&&(m.top+=f/2),u=e(k.at,p,f),m.left+=u[0],m.top+=u[1],this.each(function(){var s,r,h=t(this),l=h.outerWidth(),c=h.outerHeight(),d=i(this,"marginLeft"),_=i(this,"marginTop"),x=l+d+i(this,"marginRight")+y.width,C=c+_+i(this,"marginBottom")+y.height,D=t.extend({},m),I=e(k.my,h.outerWidth(),h.outerHeight());"right"===n.my[0]?D.left-=l:"center"===n.my[0]&&(D.left-=l/2),"bottom"===n.my[1]?D.top-=c:"center"===n.my[1]&&(D.top-=c/2),D.left+=I[0],D.top+=I[1],s={marginLeft:d,marginTop:_},t.each(["left","top"],function(e,i){t.ui.position[w[e]]&&t.ui.position[w[e]][i](D,{targetWidth:p,targetHeight:f,elemWidth:l,elemHeight:c,collisionPosition:s,collisionWidth:x,collisionHeight:C,offset:[u[0]+I[0],u[1]+I[1]],my:n.my,at:n.at,within:b,elem:h})}),n.using&&(r=function(t){var e=g.left-D.left,i=e+p-l,s=g.top-D.top,r=s+f-c,u={target:{element:v,left:g.left,top:g.top,width:p,height:f},element:{element:h,left:D.left,top:D.top,width:l,height:c},horizontal:0>i?"left":e>0?"right":"center",vertical:0>r?"top":s>0?"bottom":"middle"};l>p&&p>a(e+i)&&(u.horizontal="center"),c>f&&f>a(s+r)&&(u.vertical="middle"),u.important=o(a(e),a(i))>o(a(s),a(r))?"horizontal":"vertical",n.using.call(this,t,u)}),h.offset(t.extend(D,{using:r}))})},t.ui.position={fit:{left:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollLeft:s.offset.left,a=s.width,r=t.left-e.collisionPosition.marginLeft,h=n-r,l=r+e.collisionWidth-a-n;e.collisionWidth>a?h>0&&0>=l?(i=t.left+h+e.collisionWidth-a-n,t.left+=h-i):t.left=l>0&&0>=h?n:h>l?n+a-e.collisionWidth:n:h>0?t.left+=h:l>0?t.left-=l:t.left=o(t.left-r,t.left)},top:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollTop:s.offset.top,a=e.within.height,r=t.top-e.collisionPosition.marginTop,h=n-r,l=r+e.collisionHeight-a-n;e.collisionHeight>a?h>0&&0>=l?(i=t.top+h+e.collisionHeight-a-n,t.top+=h-i):t.top=l>0&&0>=h?n:h>l?n+a-e.collisionHeight:n:h>0?t.top+=h:l>0?t.top-=l:t.top=o(t.top-r,t.top)}},flip:{left:function(t,e){var i,s,n=e.within,o=n.offset.left+n.scrollLeft,r=n.width,h=n.isWindow?n.scrollLeft:n.offset.left,l=t.left-e.collisionPosition.marginLeft,c=l-h,u=l+e.collisionWidth-r-h,d="left"===e.my[0]?-e.elemWidth:"right"===e.my[0]?e.elemWidth:0,p="left"===e.at[0]?e.targetWidth:"right"===e.at[0]?-e.targetWidth:0,f=-2*e.offset[0];0>c?(i=t.left+d+p+f+e.collisionWidth-r-o,(0>i||a(c)>i)&&(t.left+=d+p+f)):u>0&&(s=t.left-e.collisionPosition.marginLeft+d+p+f-h,(s>0||u>a(s))&&(t.left+=d+p+f))},top:function(t,e){var i,s,n=e.within,o=n.offset.top+n.scrollTop,r=n.height,h=n.isWindow?n.scrollTop:n.offset.top,l=t.top-e.collisionPosition.marginTop,c=l-h,u=l+e.collisionHeight-r-h,d="top"===e.my[1],p=d?-e.elemHeight:"bottom"===e.my[1]?e.elemHeight:0,f="top"===e.at[1]?e.targetHeight:"bottom"===e.at[1]?-e.targetHeight:0,g=-2*e.offset[1];0>c?(s=t.top+p+f+g+e.collisionHeight-r-o,(0>s||a(c)>s)&&(t.top+=p+f+g)):u>0&&(i=t.top-e.collisionPosition.marginTop+p+f+g-h,(i>0||u>a(i))&&(t.top+=p+f+g))}},flipfit:{left:function(){t.ui.position.flip.left.apply(this,arguments),t.ui.position.fit.left.apply(this,arguments)},top:function(){t.ui.position.flip.top.apply(this,arguments),t.ui.position.fit.top.apply(this,arguments)}}}}(),t.ui.position,t.extend(t.expr[":"],{data:t.expr.createPseudo?t.expr.createPseudo(function(e){return function(i){return!!t.data(i,e)}}):function(e,i,s){return!!t.data(e,s[3])}}),t.fn.extend({disableSelection:function(){var t="onselectstart"in document.createElement("div")?"selectstart":"mousedown";return function(){return this.on(t+".ui-disableSelection",function(t){t.preventDefault()})}}(),enableSelection:function(){return this.off(".ui-disableSelection")}});var c="ui-effects-",u="ui-effects-style",d="ui-effects-animated",p=t;t.effects={effect:{}},function(t,e){function i(t,e,i){var s=u[e.type]||{};return null==t?i||!e.def?null:e.def:(t=s.floor?~~t:parseFloat(t),isNaN(t)?e.def:s.mod?(t+s.mod)%s.mod:0>t?0:t>s.max?s.max:t)}function s(i){var s=l(),n=s._rgba=[];return i=i.toLowerCase(),f(h,function(t,o){var a,r=o.re.exec(i),h=r&&o.parse(r),l=o.space||"rgba";return h?(a=s[l](h),s[c[l].cache]=a[c[l].cache],n=s._rgba=a._rgba,!1):e}),n.length?("0,0,0,0"===n.join()&&t.extend(n,o.transparent),s):o[i]}function n(t,e,i){return i=(i+1)%1,1>6*i?t+6*(e-t)*i:1>2*i?e:2>3*i?t+6*(e-t)*(2/3-i):t}var o,a="backgroundColor borderBottomColor borderLeftColor borderRightColor borderTopColor color columnRuleColor outlineColor textDecorationColor textEmphasisColor",r=/^([\-+])=\s*(\d+\.?\d*)/,h=[{re:/rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,parse:function(t){return[t[1],t[2],t[3],t[4]]}},{re:/rgba?\(\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,parse:function(t){return[2.55*t[1],2.55*t[2],2.55*t[3],t[4]]}},{re:/#([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})/,parse:function(t){return[parseInt(t[1],16),parseInt(t[2],16),parseInt(t[3],16)]}},{re:/#([a-f0-9])([a-f0-9])([a-f0-9])/,parse:function(t){return[parseInt(t[1]+t[1],16),parseInt(t[2]+t[2],16),parseInt(t[3]+t[3],16)]}},{re:/hsla?\(\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,space:"hsla",parse:function(t){return[t[1],t[2]/100,t[3]/100,t[4]]}}],l=t.Color=function(e,i,s,n){return new t.Color.fn.parse(e,i,s,n)},c={rgba:{props:{red:{idx:0,type:"byte"},green:{idx:1,type:"byte"},blue:{idx:2,type:"byte"}}},hsla:{props:{hue:{idx:0,type:"degrees"},saturation:{idx:1,type:"percent"},lightness:{idx:2,type:"percent"}}}},u={"byte":{floor:!0,max:255},percent:{max:1},degrees:{mod:360,floor:!0}},d=l.support={},p=t("<p>")[0],f=t.each;p.style.cssText="background-color:rgba(1,1,1,.5)",d.rgba=p.style.backgroundColor.indexOf("rgba")>-1,f(c,function(t,e){e.cache="_"+t,e.props.alpha={idx:3,type:"percent",def:1}}),l.fn=t.extend(l.prototype,{parse:function(n,a,r,h){if(n===e)return this._rgba=[null,null,null,null],this;(n.jquery||n.nodeType)&&(n=t(n).css(a),a=e);var u=this,d=t.type(n),p=this._rgba=[];return a!==e&&(n=[n,a,r,h],d="array"),"string"===d?this.parse(s(n)||o._default):"array"===d?(f(c.rgba.props,function(t,e){p[e.idx]=i(n[e.idx],e)}),this):"object"===d?(n instanceof l?f(c,function(t,e){n[e.cache]&&(u[e.cache]=n[e.cache].slice())}):f(c,function(e,s){var o=s.cache;f(s.props,function(t,e){if(!u[o]&&s.to){if("alpha"===t||null==n[t])return;u[o]=s.to(u._rgba)}u[o][e.idx]=i(n[t],e,!0)}),u[o]&&0>t.inArray(null,u[o].slice(0,3))&&(u[o][3]=1,s.from&&(u._rgba=s.from(u[o])))}),this):e},is:function(t){var i=l(t),s=!0,n=this;return f(c,function(t,o){var a,r=i[o.cache];return r&&(a=n[o.cache]||o.to&&o.to(n._rgba)||[],f(o.props,function(t,i){return null!=r[i.idx]?s=r[i.idx]===a[i.idx]:e})),s}),s},_space:function(){var t=[],e=this;return f(c,function(i,s){e[s.cache]&&t.push(i)}),t.pop()},transition:function(t,e){var s=l(t),n=s._space(),o=c[n],a=0===this.alpha()?l("transparent"):this,r=a[o.cache]||o.to(a._rgba),h=r.slice();return s=s[o.cache],f(o.props,function(t,n){var o=n.idx,a=r[o],l=s[o],c=u[n.type]||{};null!==l&&(null===a?h[o]=l:(c.mod&&(l-a>c.mod/2?a+=c.mod:a-l>c.mod/2&&(a-=c.mod)),h[o]=i((l-a)*e+a,n)))}),this[n](h)},blend:function(e){if(1===this._rgba[3])return this;var i=this._rgba.slice(),s=i.pop(),n=l(e)._rgba;return l(t.map(i,function(t,e){return(1-s)*n[e]+s*t}))},toRgbaString:function(){var e="rgba(",i=t.map(this._rgba,function(t,e){return null==t?e>2?1:0:t});return 1===i[3]&&(i.pop(),e="rgb("),e+i.join()+")"},toHslaString:function(){var e="hsla(",i=t.map(this.hsla(),function(t,e){return null==t&&(t=e>2?1:0),e&&3>e&&(t=Math.round(100*t)+"%"),t});return 1===i[3]&&(i.pop(),e="hsl("),e+i.join()+")"},toHexString:function(e){var i=this._rgba.slice(),s=i.pop();return e&&i.push(~~(255*s)),"#"+t.map(i,function(t){return t=(t||0).toString(16),1===t.length?"0"+t:t}).join("")},toString:function(){return 0===this._rgba[3]?"transparent":this.toRgbaString()}}),l.fn.parse.prototype=l.fn,c.hsla.to=function(t){if(null==t[0]||null==t[1]||null==t[2])return[null,null,null,t[3]];var e,i,s=t[0]/255,n=t[1]/255,o=t[2]/255,a=t[3],r=Math.max(s,n,o),h=Math.min(s,n,o),l=r-h,c=r+h,u=.5*c;return e=h===r?0:s===r?60*(n-o)/l+360:n===r?60*(o-s)/l+120:60*(s-n)/l+240,i=0===l?0:.5>=u?l/c:l/(2-c),[Math.round(e)%360,i,u,null==a?1:a]},c.hsla.from=function(t){if(null==t[0]||null==t[1]||null==t[2])return[null,null,null,t[3]];var e=t[0]/360,i=t[1],s=t[2],o=t[3],a=.5>=s?s*(1+i):s+i-s*i,r=2*s-a;return[Math.round(255*n(r,a,e+1/3)),Math.round(255*n(r,a,e)),Math.round(255*n(r,a,e-1/3)),o]},f(c,function(s,n){var o=n.props,a=n.cache,h=n.to,c=n.from;l.fn[s]=function(s){if(h&&!this[a]&&(this[a]=h(this._rgba)),s===e)return this[a].slice();var n,r=t.type(s),u="array"===r||"object"===r?s:arguments,d=this[a].slice();return f(o,function(t,e){var s=u["object"===r?t:e.idx];null==s&&(s=d[e.idx]),d[e.idx]=i(s,e)}),c?(n=l(c(d)),n[a]=d,n):l(d)},f(o,function(e,i){l.fn[e]||(l.fn[e]=function(n){var o,a=t.type(n),h="alpha"===e?this._hsla?"hsla":"rgba":s,l=this[h](),c=l[i.idx];return"undefined"===a?c:("function"===a&&(n=n.call(this,c),a=t.type(n)),null==n&&i.empty?this:("string"===a&&(o=r.exec(n),o&&(n=c+parseFloat(o[2])*("+"===o[1]?1:-1))),l[i.idx]=n,this[h](l)))})})}),l.hook=function(e){var i=e.split(" ");f(i,function(e,i){t.cssHooks[i]={set:function(e,n){var o,a,r="";if("transparent"!==n&&("string"!==t.type(n)||(o=s(n)))){if(n=l(o||n),!d.rgba&&1!==n._rgba[3]){for(a="backgroundColor"===i?e.parentNode:e;(""===r||"transparent"===r)&&a&&a.style;)try{r=t.css(a,"backgroundColor"),a=a.parentNode}catch(h){}n=n.blend(r&&"transparent"!==r?r:"_default")}n=n.toRgbaString()}try{e.style[i]=n}catch(h){}}},t.fx.step[i]=function(e){e.colorInit||(e.start=l(e.elem,i),e.end=l(e.end),e.colorInit=!0),t.cssHooks[i].set(e.elem,e.start.transition(e.end,e.pos))}})},l.hook(a),t.cssHooks.borderColor={expand:function(t){var e={};return f(["Top","Right","Bottom","Left"],function(i,s){e["border"+s+"Color"]=t}),e}},o=t.Color.names={aqua:"#00ffff",black:"#000000",blue:"#0000ff",fuchsia:"#ff00ff",gray:"#808080",green:"#008000",lime:"#00ff00",maroon:"#800000",navy:"#000080",olive:"#808000",purple:"#800080",red:"#ff0000",silver:"#c0c0c0",teal:"#008080",white:"#ffffff",yellow:"#ffff00",transparent:[null,null,null,0],_default:"#ffffff"}}(p),function(){function e(e){var i,s,n=e.ownerDocument.defaultView?e.ownerDocument.defaultView.getComputedStyle(e,null):e.currentStyle,o={};if(n&&n.length&&n[0]&&n[n[0]])for(s=n.length;s--;)i=n[s],"string"==typeof n[i]&&(o[t.camelCase(i)]=n[i]);else for(i in n)"string"==typeof n[i]&&(o[i]=n[i]);return o}function i(e,i){var s,o,a={};for(s in i)o=i[s],e[s]!==o&&(n[s]||(t.fx.step[s]||!isNaN(parseFloat(o)))&&(a[s]=o));return a}var s=["add","remove","toggle"],n={border:1,borderBottom:1,borderColor:1,borderLeft:1,borderRight:1,borderTop:1,borderWidth:1,margin:1,padding:1};t.each(["borderLeftStyle","borderRightStyle","borderBottomStyle","borderTopStyle"],function(e,i){t.fx.step[i]=function(t){("none"!==t.end&&!t.setAttr||1===t.pos&&!t.setAttr)&&(p.style(t.elem,i,t.end),t.setAttr=!0)}}),t.fn.addBack||(t.fn.addBack=function(t){return this.add(null==t?this.prevObject:this.prevObject.filter(t))}),t.effects.animateClass=function(n,o,a,r){var h=t.speed(o,a,r);return this.queue(function(){var o,a=t(this),r=a.attr("class")||"",l=h.children?a.find("*").addBack():a;l=l.map(function(){var i=t(this);return{el:i,start:e(this)}}),o=function(){t.each(s,function(t,e){n[e]&&a[e+"Class"](n[e])})},o(),l=l.map(function(){return this.end=e(this.el[0]),this.diff=i(this.start,this.end),this}),a.attr("class",r),l=l.map(function(){var e=this,i=t.Deferred(),s=t.extend({},h,{queue:!1,complete:function(){i.resolve(e)}});return this.el.animate(this.diff,s),i.promise()}),t.when.apply(t,l.get()).done(function(){o(),t.each(arguments,function(){var e=this.el;t.each(this.diff,function(t){e.css(t,"")})}),h.complete.call(a[0])})})},t.fn.extend({addClass:function(e){return function(i,s,n,o){return s?t.effects.animateClass.call(this,{add:i},s,n,o):e.apply(this,arguments)}}(t.fn.addClass),removeClass:function(e){return function(i,s,n,o){return arguments.length>1?t.effects.animateClass.call(this,{remove:i},s,n,o):e.apply(this,arguments)}}(t.fn.removeClass),toggleClass:function(e){return function(i,s,n,o,a){return"boolean"==typeof s||void 0===s?n?t.effects.animateClass.call(this,s?{add:i}:{remove:i},n,o,a):e.apply(this,arguments):t.effects.animateClass.call(this,{toggle:i},s,n,o)}}(t.fn.toggleClass),switchClass:function(e,i,s,n,o){return t.effects.animateClass.call(this,{add:i,remove:e},s,n,o)}})}(),function(){function e(e,i,s,n){return t.isPlainObject(e)&&(i=e,e=e.effect),e={effect:e},null==i&&(i={}),t.isFunction(i)&&(n=i,s=null,i={}),("number"==typeof i||t.fx.speeds[i])&&(n=s,s=i,i={}),t.isFunction(s)&&(n=s,s=null),i&&t.extend(e,i),s=s||i.duration,e.duration=t.fx.off?0:"number"==typeof s?s:s in t.fx.speeds?t.fx.speeds[s]:t.fx.speeds._default,e.complete=n||i.complete,e}function i(e){return!e||"number"==typeof e||t.fx.speeds[e]?!0:"string"!=typeof e||t.effects.effect[e]?t.isFunction(e)?!0:"object"!=typeof e||e.effect?!1:!0:!0}function s(t,e){var i=e.outerWidth(),s=e.outerHeight(),n=/^rect\((-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto)\)$/,o=n.exec(t)||["",0,i,s,0];return{top:parseFloat(o[1])||0,right:"auto"===o[2]?i:parseFloat(o[2]),bottom:"auto"===o[3]?s:parseFloat(o[3]),left:parseFloat(o[4])||0}}t.expr&&t.expr.filters&&t.expr.filters.animated&&(t.expr.filters.animated=function(e){return function(i){return!!t(i).data(d)||e(i)}}(t.expr.filters.animated)),t.uiBackCompat!==!1&&t.extend(t.effects,{save:function(t,e){for(var i=0,s=e.length;s>i;i++)null!==e[i]&&t.data(c+e[i],t[0].style[e[i]])},restore:function(t,e){for(var i,s=0,n=e.length;n>s;s++)null!==e[s]&&(i=t.data(c+e[s]),t.css(e[s],i))},setMode:function(t,e){return"toggle"===e&&(e=t.is(":hidden")?"show":"hide"),e},createWrapper:function(e){if(e.parent().is(".ui-effects-wrapper"))return e.parent();var i={width:e.outerWidth(!0),height:e.outerHeight(!0),"float":e.css("float")},s=t("<div></div>").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),n={width:e.width(),height:e.height()},o=document.activeElement;try{o.id}catch(a){o=document.body}return e.wrap(s),(e[0]===o||t.contains(e[0],o))&&t(o).trigger("focus"),s=e.parent(),"static"===e.css("position")?(s.css({position:"relative"}),e.css({position:"relative"})):(t.extend(i,{position:e.css("position"),zIndex:e.css("z-index")}),t.each(["top","left","bottom","right"],function(t,s){i[s]=e.css(s),isNaN(parseInt(i[s],10))&&(i[s]="auto")}),e.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})),e.css(n),s.css(i).show()},removeWrapper:function(e){var i=document.activeElement;return e.parent().is(".ui-effects-wrapper")&&(e.parent().replaceWith(e),(e[0]===i||t.contains(e[0],i))&&t(i).trigger("focus")),e}}),t.extend(t.effects,{version:"1.12.1",define:function(e,i,s){return s||(s=i,i="effect"),t.effects.effect[e]=s,t.effects.effect[e].mode=i,s},scaledDimensions:function(t,e,i){if(0===e)return{height:0,width:0,outerHeight:0,outerWidth:0};var s="horizontal"!==i?(e||100)/100:1,n="vertical"!==i?(e||100)/100:1;return{height:t.height()*n,width:t.width()*s,outerHeight:t.outerHeight()*n,outerWidth:t.outerWidth()*s}},clipToBox:function(t){return{width:t.clip.right-t.clip.left,height:t.clip.bottom-t.clip.top,left:t.clip.left,top:t.clip.top}},unshift:function(t,e,i){var s=t.queue();e>1&&s.splice.apply(s,[1,0].concat(s.splice(e,i))),t.dequeue()},saveStyle:function(t){t.data(u,t[0].style.cssText)},restoreStyle:function(t){t[0].style.cssText=t.data(u)||"",t.removeData(u)},mode:function(t,e){var i=t.is(":hidden");return"toggle"===e&&(e=i?"show":"hide"),(i?"hide"===e:"show"===e)&&(e="none"),e},getBaseline:function(t,e){var i,s;switch(t[0]){case"top":i=0;break;case"middle":i=.5;break;case"bottom":i=1;break;default:i=t[0]/e.height}switch(t[1]){case"left":s=0;break;case"center":s=.5;break;case"right":s=1;break;default:s=t[1]/e.width}return{x:s,y:i}},createPlaceholder:function(e){var i,s=e.css("position"),n=e.position();return e.css({marginTop:e.css("marginTop"),marginBottom:e.css("marginBottom"),marginLeft:e.css("marginLeft"),marginRight:e.css("marginRight")}).outerWidth(e.outerWidth()).outerHeight(e.outerHeight()),/^(static|relative)/.test(s)&&(s="absolute",i=t("<"+e[0].nodeName+">").insertAfter(e).css({display:/^(inline|ruby)/.test(e.css("display"))?"inline-block":"block",visibility:"hidden",marginTop:e.css("marginTop"),marginBottom:e.css("marginBottom"),marginLeft:e.css("marginLeft"),marginRight:e.css("marginRight"),"float":e.css("float")}).outerWidth(e.outerWidth()).outerHeight(e.outerHeight()).addClass("ui-effects-placeholder"),e.data(c+"placeholder",i)),e.css({position:s,left:n.left,top:n.top}),i},removePlaceholder:function(t){var e=c+"placeholder",i=t.data(e);i&&(i.remove(),t.removeData(e))},cleanUp:function(e){t.effects.restoreStyle(e),t.effects.removePlaceholder(e)},setTransition:function(e,i,s,n){return n=n||{},t.each(i,function(t,i){var o=e.cssUnit(i);o[0]>0&&(n[i]=o[0]*s+o[1])}),n}}),t.fn.extend({effect:function(){function i(e){function i(){r.removeData(d),t.effects.cleanUp(r),"hide"===s.mode&&r.hide(),a()}function a(){t.isFunction(h)&&h.call(r[0]),t.isFunction(e)&&e()}var r=t(this);s.mode=c.shift(),t.uiBackCompat===!1||o?"none"===s.mode?(r[l](),a()):n.call(r[0],s,i):(r.is(":hidden")?"hide"===l:"show"===l)?(r[l](),a()):n.call(r[0],s,a)}var s=e.apply(this,arguments),n=t.effects.effect[s.effect],o=n.mode,a=s.queue,r=a||"fx",h=s.complete,l=s.mode,c=[],u=function(e){var i=t(this),s=t.effects.mode(i,l)||o;i.data(d,!0),c.push(s),o&&("show"===s||s===o&&"hide"===s)&&i.show(),o&&"none"===s||t.effects.saveStyle(i),t.isFunction(e)&&e()};return t.fx.off||!n?l?this[l](s.duration,h):this.each(function(){h&&h.call(this)}):a===!1?this.each(u).each(i):this.queue(r,u).queue(r,i)},show:function(t){return function(s){if(i(s))return t.apply(this,arguments);var n=e.apply(this,arguments);return n.mode="show",this.effect.call(this,n)
-}}(t.fn.show),hide:function(t){return function(s){if(i(s))return t.apply(this,arguments);var n=e.apply(this,arguments);return n.mode="hide",this.effect.call(this,n)}}(t.fn.hide),toggle:function(t){return function(s){if(i(s)||"boolean"==typeof s)return t.apply(this,arguments);var n=e.apply(this,arguments);return n.mode="toggle",this.effect.call(this,n)}}(t.fn.toggle),cssUnit:function(e){var i=this.css(e),s=[];return t.each(["em","px","%","pt"],function(t,e){i.indexOf(e)>0&&(s=[parseFloat(i),e])}),s},cssClip:function(t){return t?this.css("clip","rect("+t.top+"px "+t.right+"px "+t.bottom+"px "+t.left+"px)"):s(this.css("clip"),this)},transfer:function(e,i){var s=t(this),n=t(e.to),o="fixed"===n.css("position"),a=t("body"),r=o?a.scrollTop():0,h=o?a.scrollLeft():0,l=n.offset(),c={top:l.top-r,left:l.left-h,height:n.innerHeight(),width:n.innerWidth()},u=s.offset(),d=t("<div class='ui-effects-transfer'></div>").appendTo("body").addClass(e.className).css({top:u.top-r,left:u.left-h,height:s.innerHeight(),width:s.innerWidth(),position:o?"fixed":"absolute"}).animate(c,e.duration,e.easing,function(){d.remove(),t.isFunction(i)&&i()})}}),t.fx.step.clip=function(e){e.clipInit||(e.start=t(e.elem).cssClip(),"string"==typeof e.end&&(e.end=s(e.end,e.elem)),e.clipInit=!0),t(e.elem).cssClip({top:e.pos*(e.end.top-e.start.top)+e.start.top,right:e.pos*(e.end.right-e.start.right)+e.start.right,bottom:e.pos*(e.end.bottom-e.start.bottom)+e.start.bottom,left:e.pos*(e.end.left-e.start.left)+e.start.left})}}(),function(){var e={};t.each(["Quad","Cubic","Quart","Quint","Expo"],function(t,i){e[i]=function(e){return Math.pow(e,t+2)}}),t.extend(e,{Sine:function(t){return 1-Math.cos(t*Math.PI/2)},Circ:function(t){return 1-Math.sqrt(1-t*t)},Elastic:function(t){return 0===t||1===t?t:-Math.pow(2,8*(t-1))*Math.sin((80*(t-1)-7.5)*Math.PI/15)},Back:function(t){return t*t*(3*t-2)},Bounce:function(t){for(var e,i=4;((e=Math.pow(2,--i))-1)/11>t;);return 1/Math.pow(4,3-i)-7.5625*Math.pow((3*e-2)/22-t,2)}}),t.each(e,function(e,i){t.easing["easeIn"+e]=i,t.easing["easeOut"+e]=function(t){return 1-i(1-t)},t.easing["easeInOut"+e]=function(t){return.5>t?i(2*t)/2:1-i(-2*t+2)/2}})}();var f=t.effects;t.effects.define("blind","hide",function(e,i){var s={up:["bottom","top"],vertical:["bottom","top"],down:["top","bottom"],left:["right","left"],horizontal:["right","left"],right:["left","right"]},n=t(this),o=e.direction||"up",a=n.cssClip(),r={clip:t.extend({},a)},h=t.effects.createPlaceholder(n);r.clip[s[o][0]]=r.clip[s[o][1]],"show"===e.mode&&(n.cssClip(r.clip),h&&h.css(t.effects.clipToBox(r)),r.clip=a),h&&h.animate(t.effects.clipToBox(r),e.duration,e.easing),n.animate(r,{queue:!1,duration:e.duration,easing:e.easing,complete:i})}),t.effects.define("bounce",function(e,i){var s,n,o,a=t(this),r=e.mode,h="hide"===r,l="show"===r,c=e.direction||"up",u=e.distance,d=e.times||5,p=2*d+(l||h?1:0),f=e.duration/p,g=e.easing,m="up"===c||"down"===c?"top":"left",_="up"===c||"left"===c,v=0,b=a.queue().length;for(t.effects.createPlaceholder(a),o=a.css(m),u||(u=a["top"===m?"outerHeight":"outerWidth"]()/3),l&&(n={opacity:1},n[m]=o,a.css("opacity",0).css(m,_?2*-u:2*u).animate(n,f,g)),h&&(u/=Math.pow(2,d-1)),n={},n[m]=o;d>v;v++)s={},s[m]=(_?"-=":"+=")+u,a.animate(s,f,g).animate(n,f,g),u=h?2*u:u/2;h&&(s={opacity:0},s[m]=(_?"-=":"+=")+u,a.animate(s,f,g)),a.queue(i),t.effects.unshift(a,b,p+1)}),t.effects.define("clip","hide",function(e,i){var s,n={},o=t(this),a=e.direction||"vertical",r="both"===a,h=r||"horizontal"===a,l=r||"vertical"===a;s=o.cssClip(),n.clip={top:l?(s.bottom-s.top)/2:s.top,right:h?(s.right-s.left)/2:s.right,bottom:l?(s.bottom-s.top)/2:s.bottom,left:h?(s.right-s.left)/2:s.left},t.effects.createPlaceholder(o),"show"===e.mode&&(o.cssClip(n.clip),n.clip=s),o.animate(n,{queue:!1,duration:e.duration,easing:e.easing,complete:i})}),t.effects.define("drop","hide",function(e,i){var s,n=t(this),o=e.mode,a="show"===o,r=e.direction||"left",h="up"===r||"down"===r?"top":"left",l="up"===r||"left"===r?"-=":"+=",c="+="===l?"-=":"+=",u={opacity:0};t.effects.createPlaceholder(n),s=e.distance||n["top"===h?"outerHeight":"outerWidth"](!0)/2,u[h]=l+s,a&&(n.css(u),u[h]=c+s,u.opacity=1),n.animate(u,{queue:!1,duration:e.duration,easing:e.easing,complete:i})}),t.effects.define("explode","hide",function(e,i){function s(){b.push(this),b.length===u*d&&n()}function n(){p.css({visibility:"visible"}),t(b).remove(),i()}var o,a,r,h,l,c,u=e.pieces?Math.round(Math.sqrt(e.pieces)):3,d=u,p=t(this),f=e.mode,g="show"===f,m=p.show().css("visibility","hidden").offset(),_=Math.ceil(p.outerWidth()/d),v=Math.ceil(p.outerHeight()/u),b=[];for(o=0;u>o;o++)for(h=m.top+o*v,c=o-(u-1)/2,a=0;d>a;a++)r=m.left+a*_,l=a-(d-1)/2,p.clone().appendTo("body").wrap("<div></div>").css({position:"absolute",visibility:"visible",left:-a*_,top:-o*v}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:_,height:v,left:r+(g?l*_:0),top:h+(g?c*v:0),opacity:g?0:1}).animate({left:r+(g?0:l*_),top:h+(g?0:c*v),opacity:g?1:0},e.duration||500,e.easing,s)}),t.effects.define("fade","toggle",function(e,i){var s="show"===e.mode;t(this).css("opacity",s?0:1).animate({opacity:s?1:0},{queue:!1,duration:e.duration,easing:e.easing,complete:i})}),t.effects.define("fold","hide",function(e,i){var s=t(this),n=e.mode,o="show"===n,a="hide"===n,r=e.size||15,h=/([0-9]+)%/.exec(r),l=!!e.horizFirst,c=l?["right","bottom"]:["bottom","right"],u=e.duration/2,d=t.effects.createPlaceholder(s),p=s.cssClip(),f={clip:t.extend({},p)},g={clip:t.extend({},p)},m=[p[c[0]],p[c[1]]],_=s.queue().length;h&&(r=parseInt(h[1],10)/100*m[a?0:1]),f.clip[c[0]]=r,g.clip[c[0]]=r,g.clip[c[1]]=0,o&&(s.cssClip(g.clip),d&&d.css(t.effects.clipToBox(g)),g.clip=p),s.queue(function(i){d&&d.animate(t.effects.clipToBox(f),u,e.easing).animate(t.effects.clipToBox(g),u,e.easing),i()}).animate(f,u,e.easing).animate(g,u,e.easing).queue(i),t.effects.unshift(s,_,4)}),t.effects.define("highlight","show",function(e,i){var s=t(this),n={backgroundColor:s.css("backgroundColor")};"hide"===e.mode&&(n.opacity=0),t.effects.saveStyle(s),s.css({backgroundImage:"none",backgroundColor:e.color||"#ffff99"}).animate(n,{queue:!1,duration:e.duration,easing:e.easing,complete:i})}),t.effects.define("size",function(e,i){var s,n,o,a=t(this),r=["fontSize"],h=["borderTopWidth","borderBottomWidth","paddingTop","paddingBottom"],l=["borderLeftWidth","borderRightWidth","paddingLeft","paddingRight"],c=e.mode,u="effect"!==c,d=e.scale||"both",p=e.origin||["middle","center"],f=a.css("position"),g=a.position(),m=t.effects.scaledDimensions(a),_=e.from||m,v=e.to||t.effects.scaledDimensions(a,0);t.effects.createPlaceholder(a),"show"===c&&(o=_,_=v,v=o),n={from:{y:_.height/m.height,x:_.width/m.width},to:{y:v.height/m.height,x:v.width/m.width}},("box"===d||"both"===d)&&(n.from.y!==n.to.y&&(_=t.effects.setTransition(a,h,n.from.y,_),v=t.effects.setTransition(a,h,n.to.y,v)),n.from.x!==n.to.x&&(_=t.effects.setTransition(a,l,n.from.x,_),v=t.effects.setTransition(a,l,n.to.x,v))),("content"===d||"both"===d)&&n.from.y!==n.to.y&&(_=t.effects.setTransition(a,r,n.from.y,_),v=t.effects.setTransition(a,r,n.to.y,v)),p&&(s=t.effects.getBaseline(p,m),_.top=(m.outerHeight-_.outerHeight)*s.y+g.top,_.left=(m.outerWidth-_.outerWidth)*s.x+g.left,v.top=(m.outerHeight-v.outerHeight)*s.y+g.top,v.left=(m.outerWidth-v.outerWidth)*s.x+g.left),a.css(_),("content"===d||"both"===d)&&(h=h.concat(["marginTop","marginBottom"]).concat(r),l=l.concat(["marginLeft","marginRight"]),a.find("*[width]").each(function(){var i=t(this),s=t.effects.scaledDimensions(i),o={height:s.height*n.from.y,width:s.width*n.from.x,outerHeight:s.outerHeight*n.from.y,outerWidth:s.outerWidth*n.from.x},a={height:s.height*n.to.y,width:s.width*n.to.x,outerHeight:s.height*n.to.y,outerWidth:s.width*n.to.x};n.from.y!==n.to.y&&(o=t.effects.setTransition(i,h,n.from.y,o),a=t.effects.setTransition(i,h,n.to.y,a)),n.from.x!==n.to.x&&(o=t.effects.setTransition(i,l,n.from.x,o),a=t.effects.setTransition(i,l,n.to.x,a)),u&&t.effects.saveStyle(i),i.css(o),i.animate(a,e.duration,e.easing,function(){u&&t.effects.restoreStyle(i)})})),a.animate(v,{queue:!1,duration:e.duration,easing:e.easing,complete:function(){var e=a.offset();0===v.opacity&&a.css("opacity",_.opacity),u||(a.css("position","static"===f?"relative":f).offset(e),t.effects.saveStyle(a)),i()}})}),t.effects.define("scale",function(e,i){var s=t(this),n=e.mode,o=parseInt(e.percent,10)||(0===parseInt(e.percent,10)?0:"effect"!==n?0:100),a=t.extend(!0,{from:t.effects.scaledDimensions(s),to:t.effects.scaledDimensions(s,o,e.direction||"both"),origin:e.origin||["middle","center"]},e);e.fade&&(a.from.opacity=1,a.to.opacity=0),t.effects.effect.size.call(this,a,i)}),t.effects.define("puff","hide",function(e,i){var s=t.extend(!0,{},e,{fade:!0,percent:parseInt(e.percent,10)||150});t.effects.effect.scale.call(this,s,i)}),t.effects.define("pulsate","show",function(e,i){var s=t(this),n=e.mode,o="show"===n,a="hide"===n,r=o||a,h=2*(e.times||5)+(r?1:0),l=e.duration/h,c=0,u=1,d=s.queue().length;for((o||!s.is(":visible"))&&(s.css("opacity",0).show(),c=1);h>u;u++)s.animate({opacity:c},l,e.easing),c=1-c;s.animate({opacity:c},l,e.easing),s.queue(i),t.effects.unshift(s,d,h+1)}),t.effects.define("shake",function(e,i){var s=1,n=t(this),o=e.direction||"left",a=e.distance||20,r=e.times||3,h=2*r+1,l=Math.round(e.duration/h),c="up"===o||"down"===o?"top":"left",u="up"===o||"left"===o,d={},p={},f={},g=n.queue().length;for(t.effects.createPlaceholder(n),d[c]=(u?"-=":"+=")+a,p[c]=(u?"+=":"-=")+2*a,f[c]=(u?"-=":"+=")+2*a,n.animate(d,l,e.easing);r>s;s++)n.animate(p,l,e.easing).animate(f,l,e.easing);n.animate(p,l,e.easing).animate(d,l/2,e.easing).queue(i),t.effects.unshift(n,g,h+1)}),t.effects.define("slide","show",function(e,i){var s,n,o=t(this),a={up:["bottom","top"],down:["top","bottom"],left:["right","left"],right:["left","right"]},r=e.mode,h=e.direction||"left",l="up"===h||"down"===h?"top":"left",c="up"===h||"left"===h,u=e.distance||o["top"===l?"outerHeight":"outerWidth"](!0),d={};t.effects.createPlaceholder(o),s=o.cssClip(),n=o.position()[l],d[l]=(c?-1:1)*u+n,d.clip=o.cssClip(),d.clip[a[h][1]]=d.clip[a[h][0]],"show"===r&&(o.cssClip(d.clip),o.css(l,d[l]),d.clip=s,d[l]=n),o.animate(d,{queue:!1,duration:e.duration,easing:e.easing,complete:i})});var f;t.uiBackCompat!==!1&&(f=t.effects.define("transfer",function(e,i){t(this).transfer(e,i)})),t.ui.focusable=function(i,s){var n,o,a,r,h,l=i.nodeName.toLowerCase();return"area"===l?(n=i.parentNode,o=n.name,i.href&&o&&"map"===n.nodeName.toLowerCase()?(a=t("img[usemap='#"+o+"']"),a.length>0&&a.is(":visible")):!1):(/^(input|select|textarea|button|object)$/.test(l)?(r=!i.disabled,r&&(h=t(i).closest("fieldset")[0],h&&(r=!h.disabled))):r="a"===l?i.href||s:s,r&&t(i).is(":visible")&&e(t(i)))},t.extend(t.expr[":"],{focusable:function(e){return t.ui.focusable(e,null!=t.attr(e,"tabindex"))}}),t.ui.focusable,t.fn.form=function(){return"string"==typeof this[0].form?this.closest("form"):t(this[0].form)},t.ui.formResetMixin={_formResetHandler:function(){var e=t(this);setTimeout(function(){var i=e.data("ui-form-reset-instances");t.each(i,function(){this.refresh()})})},_bindFormResetHandler:function(){if(this.form=this.element.form(),this.form.length){var t=this.form.data("ui-form-reset-instances")||[];t.length||this.form.on("reset.ui-form-reset",this._formResetHandler),t.push(this),this.form.data("ui-form-reset-instances",t)}},_unbindFormResetHandler:function(){if(this.form.length){var e=this.form.data("ui-form-reset-instances");e.splice(t.inArray(this,e),1),e.length?this.form.data("ui-form-reset-instances",e):this.form.removeData("ui-form-reset-instances").off("reset.ui-form-reset")}}},"1.7"===t.fn.jquery.substring(0,3)&&(t.each(["Width","Height"],function(e,i){function s(e,i,s,o){return t.each(n,function(){i-=parseFloat(t.css(e,"padding"+this))||0,s&&(i-=parseFloat(t.css(e,"border"+this+"Width"))||0),o&&(i-=parseFloat(t.css(e,"margin"+this))||0)}),i}var n="Width"===i?["Left","Right"]:["Top","Bottom"],o=i.toLowerCase(),a={innerWidth:t.fn.innerWidth,innerHeight:t.fn.innerHeight,outerWidth:t.fn.outerWidth,outerHeight:t.fn.outerHeight};t.fn["inner"+i]=function(e){return void 0===e?a["inner"+i].call(this):this.each(function(){t(this).css(o,s(this,e)+"px")})},t.fn["outer"+i]=function(e,n){return"number"!=typeof e?a["outer"+i].call(this,e):this.each(function(){t(this).css(o,s(this,e,!0,n)+"px")})}}),t.fn.addBack=function(t){return this.add(null==t?this.prevObject:this.prevObject.filter(t))}),t.ui.keyCode={BACKSPACE:8,COMMA:188,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,LEFT:37,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SPACE:32,TAB:9,UP:38},t.ui.escapeSelector=function(){var t=/([!"#$%&'()*+,.\/:;<=>?@[\]^`{|}~])/g;return function(e){return e.replace(t,"\\$1")}}(),t.fn.labels=function(){var e,i,s,n,o;return this[0].labels&&this[0].labels.length?this.pushStack(this[0].labels):(n=this.eq(0).parents("label"),s=this.attr("id"),s&&(e=this.eq(0).parents().last(),o=e.add(e.length?e.siblings():this.siblings()),i="label[for='"+t.ui.escapeSelector(s)+"']",n=n.add(o.find(i).addBack(i))),this.pushStack(n))},t.fn.scrollParent=function(e){var i=this.css("position"),s="absolute"===i,n=e?/(auto|scroll|hidden)/:/(auto|scroll)/,o=this.parents().filter(function(){var e=t(this);return s&&"static"===e.css("position")?!1:n.test(e.css("overflow")+e.css("overflow-y")+e.css("overflow-x"))}).eq(0);return"fixed"!==i&&o.length?o:t(this[0].ownerDocument||document)},t.extend(t.expr[":"],{tabbable:function(e){var i=t.attr(e,"tabindex"),s=null!=i;return(!s||i>=0)&&t.ui.focusable(e,s)}}),t.fn.extend({uniqueId:function(){var t=0;return function(){return this.each(function(){this.id||(this.id="ui-id-"+ ++t)})}}(),removeUniqueId:function(){return this.each(function(){/^ui-id-\d+$/.test(this.id)&&t(this).removeAttr("id")})}}),t.widget("ui.accordion",{version:"1.12.1",options:{active:0,animate:{},classes:{"ui-accordion-header":"ui-corner-top","ui-accordion-header-collapsed":"ui-corner-all","ui-accordion-content":"ui-corner-bottom"},collapsible:!1,event:"click",header:"> li > :first-child, > :not(li):even",heightStyle:"auto",icons:{activeHeader:"ui-icon-triangle-1-s",header:"ui-icon-triangle-1-e"},activate:null,beforeActivate:null},hideProps:{borderTopWidth:"hide",borderBottomWidth:"hide",paddingTop:"hide",paddingBottom:"hide",height:"hide"},showProps:{borderTopWidth:"show",borderBottomWidth:"show",paddingTop:"show",paddingBottom:"show",height:"show"},_create:function(){var e=this.options;this.prevShow=this.prevHide=t(),this._addClass("ui-accordion","ui-widget ui-helper-reset"),this.element.attr("role","tablist"),e.collapsible||e.active!==!1&&null!=e.active||(e.active=0),this._processPanels(),0>e.active&&(e.active+=this.headers.length),this._refresh()},_getCreateEventData:function(){return{header:this.active,panel:this.active.length?this.active.next():t()}},_createIcons:function(){var e,i,s=this.options.icons;s&&(e=t("<span>"),this._addClass(e,"ui-accordion-header-icon","ui-icon "+s.header),e.prependTo(this.headers),i=this.active.children(".ui-accordion-header-icon"),this._removeClass(i,s.header)._addClass(i,null,s.activeHeader)._addClass(this.headers,"ui-accordion-icons"))},_destroyIcons:function(){this._removeClass(this.headers,"ui-accordion-icons"),this.headers.children(".ui-accordion-header-icon").remove()},_destroy:function(){var t;this.element.removeAttr("role"),this.headers.removeAttr("role aria-expanded aria-selected aria-controls tabIndex").removeUniqueId(),this._destroyIcons(),t=this.headers.next().css("display","").removeAttr("role aria-hidden aria-labelledby").removeUniqueId(),"content"!==this.options.heightStyle&&t.css("height","")},_setOption:function(t,e){return"active"===t?(this._activate(e),void 0):("event"===t&&(this.options.event&&this._off(this.headers,this.options.event),this._setupEvents(e)),this._super(t,e),"collapsible"!==t||e||this.options.active!==!1||this._activate(0),"icons"===t&&(this._destroyIcons(),e&&this._createIcons()),void 0)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t),this._toggleClass(null,"ui-state-disabled",!!t),this._toggleClass(this.headers.add(this.headers.next()),null,"ui-state-disabled",!!t)},_keydown:function(e){if(!e.altKey&&!e.ctrlKey){var i=t.ui.keyCode,s=this.headers.length,n=this.headers.index(e.target),o=!1;switch(e.keyCode){case i.RIGHT:case i.DOWN:o=this.headers[(n+1)%s];break;case i.LEFT:case i.UP:o=this.headers[(n-1+s)%s];break;case i.SPACE:case i.ENTER:this._eventHandler(e);break;case i.HOME:o=this.headers[0];break;case i.END:o=this.headers[s-1]}o&&(t(e.target).attr("tabIndex",-1),t(o).attr("tabIndex",0),t(o).trigger("focus"),e.preventDefault())}},_panelKeyDown:function(e){e.keyCode===t.ui.keyCode.UP&&e.ctrlKey&&t(e.currentTarget).prev().trigger("focus")},refresh:function(){var e=this.options;this._processPanels(),e.active===!1&&e.collapsible===!0||!this.headers.length?(e.active=!1,this.active=t()):e.active===!1?this._activate(0):this.active.length&&!t.contains(this.element[0],this.active[0])?this.headers.length===this.headers.find(".ui-state-disabled").length?(e.active=!1,this.active=t()):this._activate(Math.max(0,e.active-1)):e.active=this.headers.index(this.active),this._destroyIcons(),this._refresh()},_processPanels:function(){var t=this.headers,e=this.panels;this.headers=this.element.find(this.options.header),this._addClass(this.headers,"ui-accordion-header ui-accordion-header-collapsed","ui-state-default"),this.panels=this.headers.next().filter(":not(.ui-accordion-content-active)").hide(),this._addClass(this.panels,"ui-accordion-content","ui-helper-reset ui-widget-content"),e&&(this._off(t.not(this.headers)),this._off(e.not(this.panels)))},_refresh:function(){var e,i=this.options,s=i.heightStyle,n=this.element.parent();this.active=this._findActive(i.active),this._addClass(this.active,"ui-accordion-header-active","ui-state-active")._removeClass(this.active,"ui-accordion-header-collapsed"),this._addClass(this.active.next(),"ui-accordion-content-active"),this.active.next().show(),this.headers.attr("role","tab").each(function(){var e=t(this),i=e.uniqueId().attr("id"),s=e.next(),n=s.uniqueId().attr("id");e.attr("aria-controls",n),s.attr("aria-labelledby",i)}).next().attr("role","tabpanel"),this.headers.not(this.active).attr({"aria-selected":"false","aria-expanded":"false",tabIndex:-1}).next().attr({"aria-hidden":"true"}).hide(),this.active.length?this.active.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0}).next().attr({"aria-hidden":"false"}):this.headers.eq(0).attr("tabIndex",0),this._createIcons(),this._setupEvents(i.event),"fill"===s?(e=n.height(),this.element.siblings(":visible").each(function(){var i=t(this),s=i.css("position");"absolute"!==s&&"fixed"!==s&&(e-=i.outerHeight(!0))}),this.headers.each(function(){e-=t(this).outerHeight(!0)}),this.headers.next().each(function(){t(this).height(Math.max(0,e-t(this).innerHeight()+t(this).height()))}).css("overflow","auto")):"auto"===s&&(e=0,this.headers.next().each(function(){var i=t(this).is(":visible");i||t(this).show(),e=Math.max(e,t(this).css("height","").height()),i||t(this).hide()}).height(e))},_activate:function(e){var i=this._findActive(e)[0];i!==this.active[0]&&(i=i||this.active[0],this._eventHandler({target:i,currentTarget:i,preventDefault:t.noop}))},_findActive:function(e){return"number"==typeof e?this.headers.eq(e):t()},_setupEvents:function(e){var i={keydown:"_keydown"};e&&t.each(e.split(" "),function(t,e){i[e]="_eventHandler"}),this._off(this.headers.add(this.headers.next())),this._on(this.headers,i),this._on(this.headers.next(),{keydown:"_panelKeyDown"}),this._hoverable(this.headers),this._focusable(this.headers)},_eventHandler:function(e){var i,s,n=this.options,o=this.active,a=t(e.currentTarget),r=a[0]===o[0],h=r&&n.collapsible,l=h?t():a.next(),c=o.next(),u={oldHeader:o,oldPanel:c,newHeader:h?t():a,newPanel:l};e.preventDefault(),r&&!n.collapsible||this._trigger("beforeActivate",e,u)===!1||(n.active=h?!1:this.headers.index(a),this.active=r?t():a,this._toggle(u),this._removeClass(o,"ui-accordion-header-active","ui-state-active"),n.icons&&(i=o.children(".ui-accordion-header-icon"),this._removeClass(i,null,n.icons.activeHeader)._addClass(i,null,n.icons.header)),r||(this._removeClass(a,"ui-accordion-header-collapsed")._addClass(a,"ui-accordion-header-active","ui-state-active"),n.icons&&(s=a.children(".ui-accordion-header-icon"),this._removeClass(s,null,n.icons.header)._addClass(s,null,n.icons.activeHeader)),this._addClass(a.next(),"ui-accordion-content-active")))},_toggle:function(e){var i=e.newPanel,s=this.prevShow.length?this.prevShow:e.oldPanel;this.prevShow.add(this.prevHide).stop(!0,!0),this.prevShow=i,this.prevHide=s,this.options.animate?this._animate(i,s,e):(s.hide(),i.show(),this._toggleComplete(e)),s.attr({"aria-hidden":"true"}),s.prev().attr({"aria-selected":"false","aria-expanded":"false"}),i.length&&s.length?s.prev().attr({tabIndex:-1,"aria-expanded":"false"}):i.length&&this.headers.filter(function(){return 0===parseInt(t(this).attr("tabIndex"),10)}).attr("tabIndex",-1),i.attr("aria-hidden","false").prev().attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0})},_animate:function(t,e,i){var s,n,o,a=this,r=0,h=t.css("box-sizing"),l=t.length&&(!e.length||t.index()<e.index()),c=this.options.animate||{},u=l&&c.down||c,d=function(){a._toggleComplete(i)};return"number"==typeof u&&(o=u),"string"==typeof u&&(n=u),n=n||u.easing||c.easing,o=o||u.duration||c.duration,e.length?t.length?(s=t.show().outerHeight(),e.animate(this.hideProps,{duration:o,easing:n,step:function(t,e){e.now=Math.round(t)}}),t.hide().animate(this.showProps,{duration:o,easing:n,complete:d,step:function(t,i){i.now=Math.round(t),"height"!==i.prop?"content-box"===h&&(r+=i.now):"content"!==a.options.heightStyle&&(i.now=Math.round(s-e.outerHeight()-r),r=0)}}),void 0):e.animate(this.hideProps,o,n,d):t.animate(this.showProps,o,n,d)},_toggleComplete:function(t){var e=t.oldPanel,i=e.prev();this._removeClass(e,"ui-accordion-content-active"),this._removeClass(i,"ui-accordion-header-active")._addClass(i,"ui-accordion-header-collapsed"),e.length&&(e.parent()[0].className=e.parent()[0].className),this._trigger("activate",null,t)}}),t.ui.safeActiveElement=function(t){var e;try{e=t.activeElement}catch(i){e=t.body}return e||(e=t.body),e.nodeName||(e=t.body),e},t.widget("ui.menu",{version:"1.12.1",defaultElement:"<ul>",delay:300,options:{icons:{submenu:"ui-icon-caret-1-e"},items:"> *",menus:"ul",position:{my:"left top",at:"right top"},role:"menu",blur:null,focus:null,select:null},_create:function(){this.activeMenu=this.element,this.mouseHandled=!1,this.element.uniqueId().attr({role:this.options.role,tabIndex:0}),this._addClass("ui-menu","ui-widget ui-widget-content"),this._on({"mousedown .ui-menu-item":function(t){t.preventDefault()},"click .ui-menu-item":function(e){var i=t(e.target),s=t(t.ui.safeActiveElement(this.document[0]));!this.mouseHandled&&i.not(".ui-state-disabled").length&&(this.select(e),e.isPropagationStopped()||(this.mouseHandled=!0),i.has(".ui-menu").length?this.expand(e):!this.element.is(":focus")&&s.closest(".ui-menu").length&&(this.element.trigger("focus",[!0]),this.active&&1===this.active.parents(".ui-menu").length&&clearTimeout(this.timer)))},"mouseenter .ui-menu-item":function(e){if(!this.previousFilter){var i=t(e.target).closest(".ui-menu-item"),s=t(e.currentTarget);i[0]===s[0]&&(this._removeClass(s.siblings().children(".ui-state-active"),null,"ui-state-active"),this.focus(e,s))}},mouseleave:"collapseAll","mouseleave .ui-menu":"collapseAll",focus:function(t,e){var i=this.active||this.element.find(this.options.items).eq(0);e||this.focus(t,i)},blur:function(e){this._delay(function(){var i=!t.contains(this.element[0],t.ui.safeActiveElement(this.document[0]));i&&this.collapseAll(e)})},keydown:"_keydown"}),this.refresh(),this._on(this.document,{click:function(t){this._closeOnDocumentClick(t)&&this.collapseAll(t),this.mouseHandled=!1}})},_destroy:function(){var e=this.element.find(".ui-menu-item").removeAttr("role aria-disabled"),i=e.children(".ui-menu-item-wrapper").removeUniqueId().removeAttr("tabIndex role aria-haspopup");this.element.removeAttr("aria-activedescendant").find(".ui-menu").addBack().removeAttr("role aria-labelledby aria-expanded aria-hidden aria-disabled tabIndex").removeUniqueId().show(),i.children().each(function(){var e=t(this);e.data("ui-menu-submenu-caret")&&e.remove()})},_keydown:function(e){var i,s,n,o,a=!0;switch(e.keyCode){case t.ui.keyCode.PAGE_UP:this.previousPage(e);break;case t.ui.keyCode.PAGE_DOWN:this.nextPage(e);break;case t.ui.keyCode.HOME:this._move("first","first",e);break;case t.ui.keyCode.END:this._move("last","last",e);break;case t.ui.keyCode.UP:this.previous(e);break;case t.ui.keyCode.DOWN:this.next(e);break;case t.ui.keyCode.LEFT:this.collapse(e);break;case t.ui.keyCode.RIGHT:this.active&&!this.active.is(".ui-state-disabled")&&this.expand(e);break;case t.ui.keyCode.ENTER:case t.ui.keyCode.SPACE:this._activate(e);break;case t.ui.keyCode.ESCAPE:this.collapse(e);break;default:a=!1,s=this.previousFilter||"",o=!1,n=e.keyCode>=96&&105>=e.keyCode?""+(e.keyCode-96):String.fromCharCode(e.keyCode),clearTimeout(this.filterTimer),n===s?o=!0:n=s+n,i=this._filterMenuItems(n),i=o&&-1!==i.index(this.active.next())?this.active.nextAll(".ui-menu-item"):i,i.length||(n=String.fromCharCode(e.keyCode),i=this._filterMenuItems(n)),i.length?(this.focus(e,i),this.previousFilter=n,this.filterTimer=this._delay(function(){delete this.previousFilter},1e3)):delete this.previousFilter}a&&e.preventDefault()},_activate:function(t){this.active&&!this.active.is(".ui-state-disabled")&&(this.active.children("[aria-haspopup='true']").length?this.expand(t):this.select(t))},refresh:function(){var e,i,s,n,o,a=this,r=this.options.icons.submenu,h=this.element.find(this.options.menus);this._toggleClass("ui-menu-icons",null,!!this.element.find(".ui-icon").length),s=h.filter(":not(.ui-menu)").hide().attr({role:this.options.role,"aria-hidden":"true","aria-expanded":"false"}).each(function(){var e=t(this),i=e.prev(),s=t("<span>").data("ui-menu-submenu-caret",!0);a._addClass(s,"ui-menu-icon","ui-icon "+r),i.attr("aria-haspopup","true").prepend(s),e.attr("aria-labelledby",i.attr("id"))}),this._addClass(s,"ui-menu","ui-widget ui-widget-content ui-front"),e=h.add(this.element),i=e.find(this.options.items),i.not(".ui-menu-item").each(function(){var e=t(this);a._isDivider(e)&&a._addClass(e,"ui-menu-divider","ui-widget-content")}),n=i.not(".ui-menu-item, .ui-menu-divider"),o=n.children().not(".ui-menu").uniqueId().attr({tabIndex:-1,role:this._itemRole()}),this._addClass(n,"ui-menu-item")._addClass(o,"ui-menu-item-wrapper"),i.filter(".ui-state-disabled").attr("aria-disabled","true"),this.active&&!t.contains(this.element[0],this.active[0])&&this.blur()},_itemRole:function(){return{menu:"menuitem",listbox:"option"}[this.options.role]},_setOption:function(t,e){if("icons"===t){var i=this.element.find(".ui-menu-icon");this._removeClass(i,null,this.options.icons.submenu)._addClass(i,null,e.submenu)}this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t+""),this._toggleClass(null,"ui-state-disabled",!!t)},focus:function(t,e){var i,s,n;this.blur(t,t&&"focus"===t.type),this._scrollIntoView(e),this.active=e.first(),s=this.active.children(".ui-menu-item-wrapper"),this._addClass(s,null,"ui-state-active"),this.options.role&&this.element.attr("aria-activedescendant",s.attr("id")),n=this.active.parent().closest(".ui-menu-item").children(".ui-menu-item-wrapper"),this._addClass(n,null,"ui-state-active"),t&&"keydown"===t.type?this._close():this.timer=this._delay(function(){this._close()},this.delay),i=e.children(".ui-menu"),i.length&&t&&/^mouse/.test(t.type)&&this._startOpening(i),this.activeMenu=e.parent(),this._trigger("focus",t,{item:e})},_scrollIntoView:function(e){var i,s,n,o,a,r;this._hasScroll()&&(i=parseFloat(t.css(this.activeMenu[0],"borderTopWidth"))||0,s=parseFloat(t.css(this.activeMenu[0],"paddingTop"))||0,n=e.offset().top-this.activeMenu.offset().top-i-s,o=this.activeMenu.scrollTop(),a=this.activeMenu.height(),r=e.outerHeight(),0>n?this.activeMenu.scrollTop(o+n):n+r>a&&this.activeMenu.scrollTop(o+n-a+r))},blur:function(t,e){e||clearTimeout(this.timer),this.active&&(this._removeClass(this.active.children(".ui-menu-item-wrapper"),null,"ui-state-active"),this._trigger("blur",t,{item:this.active}),this.active=null)},_startOpening:function(t){clearTimeout(this.timer),"true"===t.attr("aria-hidden")&&(this.timer=this._delay(function(){this._close(),this._open(t)},this.delay))},_open:function(e){var i=t.extend({of:this.active},this.options.position);clearTimeout(this.timer),this.element.find(".ui-menu").not(e.parents(".ui-menu")).hide().attr("aria-hidden","true"),e.show().removeAttr("aria-hidden").attr("aria-expanded","true").position(i)},collapseAll:function(e,i){clearTimeout(this.timer),this.timer=this._delay(function(){var s=i?this.element:t(e&&e.target).closest(this.element.find(".ui-menu"));s.length||(s=this.element),this._close(s),this.blur(e),this._removeClass(s.find(".ui-state-active"),null,"ui-state-active"),this.activeMenu=s},this.delay)},_close:function(t){t||(t=this.active?this.active.parent():this.element),t.find(".ui-menu").hide().attr("aria-hidden","true").attr("aria-expanded","false")},_closeOnDocumentClick:function(e){return!t(e.target).closest(".ui-menu").length},_isDivider:function(t){return!/[^\-\u2014\u2013\s]/.test(t.text())},collapse:function(t){var e=this.active&&this.active.parent().closest(".ui-menu-item",this.element);e&&e.length&&(this._close(),this.focus(t,e))},expand:function(t){var e=this.active&&this.active.children(".ui-menu ").find(this.options.items).first();e&&e.length&&(this._open(e.parent()),this._delay(function(){this.focus(t,e)}))},next:function(t){this._move("next","first",t)},previous:function(t){this._move("prev","last",t)},isFirstItem:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},isLastItem:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},_move:function(t,e,i){var s;this.active&&(s="first"===t||"last"===t?this.active["first"===t?"prevAll":"nextAll"](".ui-menu-item").eq(-1):this.active[t+"All"](".ui-menu-item").eq(0)),s&&s.length&&this.active||(s=this.activeMenu.find(this.options.items)[e]()),this.focus(i,s)},nextPage:function(e){var i,s,n;return this.active?(this.isLastItem()||(this._hasScroll()?(s=this.active.offset().top,n=this.element.height(),this.active.nextAll(".ui-menu-item").each(function(){return i=t(this),0>i.offset().top-s-n}),this.focus(e,i)):this.focus(e,this.activeMenu.find(this.options.items)[this.active?"last":"first"]())),void 0):(this.next(e),void 0)},previousPage:function(e){var i,s,n;return this.active?(this.isFirstItem()||(this._hasScroll()?(s=this.active.offset().top,n=this.element.height(),this.active.prevAll(".ui-menu-item").each(function(){return i=t(this),i.offset().top-s+n>0}),this.focus(e,i)):this.focus(e,this.activeMenu.find(this.options.items).first())),void 0):(this.next(e),void 0)},_hasScroll:function(){return this.element.outerHeight()<this.element.prop("scrollHeight")},select:function(e){this.active=this.active||t(e.target).closest(".ui-menu-item");var i={item:this.active};this.active.has(".ui-menu").length||this.collapseAll(e,!0),this._trigger("select",e,i)},_filterMenuItems:function(e){var i=e.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&"),s=RegExp("^"+i,"i");return this.activeMenu.find(this.options.items).filter(".ui-menu-item").filter(function(){return s.test(t.trim(t(this).children(".ui-menu-item-wrapper").text()))})}}),t.widget("ui.autocomplete",{version:"1.12.1",defaultElement:"<input>",options:{appendTo:null,autoFocus:!1,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null,change:null,close:null,focus:null,open:null,response:null,search:null,select:null},requestIndex:0,pending:0,_create:function(){var e,i,s,n=this.element[0].nodeName.toLowerCase(),o="textarea"===n,a="input"===n;
-this.isMultiLine=o||!a&&this._isContentEditable(this.element),this.valueMethod=this.element[o||a?"val":"text"],this.isNewMenu=!0,this._addClass("ui-autocomplete-input"),this.element.attr("autocomplete","off"),this._on(this.element,{keydown:function(n){if(this.element.prop("readOnly"))return e=!0,s=!0,i=!0,void 0;e=!1,s=!1,i=!1;var o=t.ui.keyCode;switch(n.keyCode){case o.PAGE_UP:e=!0,this._move("previousPage",n);break;case o.PAGE_DOWN:e=!0,this._move("nextPage",n);break;case o.UP:e=!0,this._keyEvent("previous",n);break;case o.DOWN:e=!0,this._keyEvent("next",n);break;case o.ENTER:this.menu.active&&(e=!0,n.preventDefault(),this.menu.select(n));break;case o.TAB:this.menu.active&&this.menu.select(n);break;case o.ESCAPE:this.menu.element.is(":visible")&&(this.isMultiLine||this._value(this.term),this.close(n),n.preventDefault());break;default:i=!0,this._searchTimeout(n)}},keypress:function(s){if(e)return e=!1,(!this.isMultiLine||this.menu.element.is(":visible"))&&s.preventDefault(),void 0;if(!i){var n=t.ui.keyCode;switch(s.keyCode){case n.PAGE_UP:this._move("previousPage",s);break;case n.PAGE_DOWN:this._move("nextPage",s);break;case n.UP:this._keyEvent("previous",s);break;case n.DOWN:this._keyEvent("next",s)}}},input:function(t){return s?(s=!1,t.preventDefault(),void 0):(this._searchTimeout(t),void 0)},focus:function(){this.selectedItem=null,this.previous=this._value()},blur:function(t){return this.cancelBlur?(delete this.cancelBlur,void 0):(clearTimeout(this.searching),this.close(t),this._change(t),void 0)}}),this._initSource(),this.menu=t("<ul>").appendTo(this._appendTo()).menu({role:null}).hide().menu("instance"),this._addClass(this.menu.element,"ui-autocomplete","ui-front"),this._on(this.menu.element,{mousedown:function(e){e.preventDefault(),this.cancelBlur=!0,this._delay(function(){delete this.cancelBlur,this.element[0]!==t.ui.safeActiveElement(this.document[0])&&this.element.trigger("focus")})},menufocus:function(e,i){var s,n;return this.isNewMenu&&(this.isNewMenu=!1,e.originalEvent&&/^mouse/.test(e.originalEvent.type))?(this.menu.blur(),this.document.one("mousemove",function(){t(e.target).trigger(e.originalEvent)}),void 0):(n=i.item.data("ui-autocomplete-item"),!1!==this._trigger("focus",e,{item:n})&&e.originalEvent&&/^key/.test(e.originalEvent.type)&&this._value(n.value),s=i.item.attr("aria-label")||n.value,s&&t.trim(s).length&&(this.liveRegion.children().hide(),t("<div>").text(s).appendTo(this.liveRegion)),void 0)},menuselect:function(e,i){var s=i.item.data("ui-autocomplete-item"),n=this.previous;this.element[0]!==t.ui.safeActiveElement(this.document[0])&&(this.element.trigger("focus"),this.previous=n,this._delay(function(){this.previous=n,this.selectedItem=s})),!1!==this._trigger("select",e,{item:s})&&this._value(s.value),this.term=this._value(),this.close(e),this.selectedItem=s}}),this.liveRegion=t("<div>",{role:"status","aria-live":"assertive","aria-relevant":"additions"}).appendTo(this.document[0].body),this._addClass(this.liveRegion,null,"ui-helper-hidden-accessible"),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_destroy:function(){clearTimeout(this.searching),this.element.removeAttr("autocomplete"),this.menu.element.remove(),this.liveRegion.remove()},_setOption:function(t,e){this._super(t,e),"source"===t&&this._initSource(),"appendTo"===t&&this.menu.element.appendTo(this._appendTo()),"disabled"===t&&e&&this.xhr&&this.xhr.abort()},_isEventTargetInWidget:function(e){var i=this.menu.element[0];return e.target===this.element[0]||e.target===i||t.contains(i,e.target)},_closeOnClickOutside:function(t){this._isEventTargetInWidget(t)||this.close()},_appendTo:function(){var e=this.options.appendTo;return e&&(e=e.jquery||e.nodeType?t(e):this.document.find(e).eq(0)),e&&e[0]||(e=this.element.closest(".ui-front, dialog")),e.length||(e=this.document[0].body),e},_initSource:function(){var e,i,s=this;t.isArray(this.options.source)?(e=this.options.source,this.source=function(i,s){s(t.ui.autocomplete.filter(e,i.term))}):"string"==typeof this.options.source?(i=this.options.source,this.source=function(e,n){s.xhr&&s.xhr.abort(),s.xhr=t.ajax({url:i,data:e,dataType:"json",success:function(t){n(t)},error:function(){n([])}})}):this.source=this.options.source},_searchTimeout:function(t){clearTimeout(this.searching),this.searching=this._delay(function(){var e=this.term===this._value(),i=this.menu.element.is(":visible"),s=t.altKey||t.ctrlKey||t.metaKey||t.shiftKey;(!e||e&&!i&&!s)&&(this.selectedItem=null,this.search(null,t))},this.options.delay)},search:function(t,e){return t=null!=t?t:this._value(),this.term=this._value(),t.length<this.options.minLength?this.close(e):this._trigger("search",e)!==!1?this._search(t):void 0},_search:function(t){this.pending++,this._addClass("ui-autocomplete-loading"),this.cancelSearch=!1,this.source({term:t},this._response())},_response:function(){var e=++this.requestIndex;return t.proxy(function(t){e===this.requestIndex&&this.__response(t),this.pending--,this.pending||this._removeClass("ui-autocomplete-loading")},this)},__response:function(t){t&&(t=this._normalize(t)),this._trigger("response",null,{content:t}),!this.options.disabled&&t&&t.length&&!this.cancelSearch?(this._suggest(t),this._trigger("open")):this._close()},close:function(t){this.cancelSearch=!0,this._close(t)},_close:function(t){this._off(this.document,"mousedown"),this.menu.element.is(":visible")&&(this.menu.element.hide(),this.menu.blur(),this.isNewMenu=!0,this._trigger("close",t))},_change:function(t){this.previous!==this._value()&&this._trigger("change",t,{item:this.selectedItem})},_normalize:function(e){return e.length&&e[0].label&&e[0].value?e:t.map(e,function(e){return"string"==typeof e?{label:e,value:e}:t.extend({},e,{label:e.label||e.value,value:e.value||e.label})})},_suggest:function(e){var i=this.menu.element.empty();this._renderMenu(i,e),this.isNewMenu=!0,this.menu.refresh(),i.show(),this._resizeMenu(),i.position(t.extend({of:this.element},this.options.position)),this.options.autoFocus&&this.menu.next(),this._on(this.document,{mousedown:"_closeOnClickOutside"})},_resizeMenu:function(){var t=this.menu.element;t.outerWidth(Math.max(t.width("").outerWidth()+1,this.element.outerWidth()))},_renderMenu:function(e,i){var s=this;t.each(i,function(t,i){s._renderItemData(e,i)})},_renderItemData:function(t,e){return this._renderItem(t,e).data("ui-autocomplete-item",e)},_renderItem:function(e,i){return t("<li>").append(t("<div>").text(i.label)).appendTo(e)},_move:function(t,e){return this.menu.element.is(":visible")?this.menu.isFirstItem()&&/^previous/.test(t)||this.menu.isLastItem()&&/^next/.test(t)?(this.isMultiLine||this._value(this.term),this.menu.blur(),void 0):(this.menu[t](e),void 0):(this.search(null,e),void 0)},widget:function(){return this.menu.element},_value:function(){return this.valueMethod.apply(this.element,arguments)},_keyEvent:function(t,e){(!this.isMultiLine||this.menu.element.is(":visible"))&&(this._move(t,e),e.preventDefault())},_isContentEditable:function(t){if(!t.length)return!1;var e=t.prop("contentEditable");return"inherit"===e?this._isContentEditable(t.parent()):"true"===e}}),t.extend(t.ui.autocomplete,{escapeRegex:function(t){return t.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&")},filter:function(e,i){var s=RegExp(t.ui.autocomplete.escapeRegex(i),"i");return t.grep(e,function(t){return s.test(t.label||t.value||t)})}}),t.widget("ui.autocomplete",t.ui.autocomplete,{options:{messages:{noResults:"No search results.",results:function(t){return t+(t>1?" results are":" result is")+" available, use up and down arrow keys to navigate."}}},__response:function(e){var i;this._superApply(arguments),this.options.disabled||this.cancelSearch||(i=e&&e.length?this.options.messages.results(e.length):this.options.messages.noResults,this.liveRegion.children().hide(),t("<div>").text(i).appendTo(this.liveRegion))}}),t.ui.autocomplete;var g=/ui-corner-([a-z]){2,6}/g;t.widget("ui.controlgroup",{version:"1.12.1",defaultElement:"<div>",options:{direction:"horizontal",disabled:null,onlyVisible:!0,items:{button:"input[type=button], input[type=submit], input[type=reset], button, a",controlgroupLabel:".ui-controlgroup-label",checkboxradio:"input[type='checkbox'], input[type='radio']",selectmenu:"select",spinner:".ui-spinner-input"}},_create:function(){this._enhance()},_enhance:function(){this.element.attr("role","toolbar"),this.refresh()},_destroy:function(){this._callChildMethod("destroy"),this.childWidgets.removeData("ui-controlgroup-data"),this.element.removeAttr("role"),this.options.items.controlgroupLabel&&this.element.find(this.options.items.controlgroupLabel).find(".ui-controlgroup-label-contents").contents().unwrap()},_initWidgets:function(){var e=this,i=[];t.each(this.options.items,function(s,n){var o,a={};return n?"controlgroupLabel"===s?(o=e.element.find(n),o.each(function(){var e=t(this);e.children(".ui-controlgroup-label-contents").length||e.contents().wrapAll("<span class='ui-controlgroup-label-contents'></span>")}),e._addClass(o,null,"ui-widget ui-widget-content ui-state-default"),i=i.concat(o.get()),void 0):(t.fn[s]&&(a=e["_"+s+"Options"]?e["_"+s+"Options"]("middle"):{classes:{}},e.element.find(n).each(function(){var n=t(this),o=n[s]("instance"),r=t.widget.extend({},a);if("button"!==s||!n.parent(".ui-spinner").length){o||(o=n[s]()[s]("instance")),o&&(r.classes=e._resolveClassesValues(r.classes,o)),n[s](r);var h=n[s]("widget");t.data(h[0],"ui-controlgroup-data",o?o:n[s]("instance")),i.push(h[0])}})),void 0):void 0}),this.childWidgets=t(t.unique(i)),this._addClass(this.childWidgets,"ui-controlgroup-item")},_callChildMethod:function(e){this.childWidgets.each(function(){var i=t(this),s=i.data("ui-controlgroup-data");s&&s[e]&&s[e]()})},_updateCornerClass:function(t,e){var i="ui-corner-top ui-corner-bottom ui-corner-left ui-corner-right ui-corner-all",s=this._buildSimpleOptions(e,"label").classes.label;this._removeClass(t,null,i),this._addClass(t,null,s)},_buildSimpleOptions:function(t,e){var i="vertical"===this.options.direction,s={classes:{}};return s.classes[e]={middle:"",first:"ui-corner-"+(i?"top":"left"),last:"ui-corner-"+(i?"bottom":"right"),only:"ui-corner-all"}[t],s},_spinnerOptions:function(t){var e=this._buildSimpleOptions(t,"ui-spinner");return e.classes["ui-spinner-up"]="",e.classes["ui-spinner-down"]="",e},_buttonOptions:function(t){return this._buildSimpleOptions(t,"ui-button")},_checkboxradioOptions:function(t){return this._buildSimpleOptions(t,"ui-checkboxradio-label")},_selectmenuOptions:function(t){var e="vertical"===this.options.direction;return{width:e?"auto":!1,classes:{middle:{"ui-selectmenu-button-open":"","ui-selectmenu-button-closed":""},first:{"ui-selectmenu-button-open":"ui-corner-"+(e?"top":"tl"),"ui-selectmenu-button-closed":"ui-corner-"+(e?"top":"left")},last:{"ui-selectmenu-button-open":e?"":"ui-corner-tr","ui-selectmenu-button-closed":"ui-corner-"+(e?"bottom":"right")},only:{"ui-selectmenu-button-open":"ui-corner-top","ui-selectmenu-button-closed":"ui-corner-all"}}[t]}},_resolveClassesValues:function(e,i){var s={};return t.each(e,function(n){var o=i.options.classes[n]||"";o=t.trim(o.replace(g,"")),s[n]=(o+" "+e[n]).replace(/\s+/g," ")}),s},_setOption:function(t,e){return"direction"===t&&this._removeClass("ui-controlgroup-"+this.options.direction),this._super(t,e),"disabled"===t?(this._callChildMethod(e?"disable":"enable"),void 0):(this.refresh(),void 0)},refresh:function(){var e,i=this;this._addClass("ui-controlgroup ui-controlgroup-"+this.options.direction),"horizontal"===this.options.direction&&this._addClass(null,"ui-helper-clearfix"),this._initWidgets(),e=this.childWidgets,this.options.onlyVisible&&(e=e.filter(":visible")),e.length&&(t.each(["first","last"],function(t,s){var n=e[s]().data("ui-controlgroup-data");if(n&&i["_"+n.widgetName+"Options"]){var o=i["_"+n.widgetName+"Options"](1===e.length?"only":s);o.classes=i._resolveClassesValues(o.classes,n),n.element[n.widgetName](o)}else i._updateCornerClass(e[s](),s)}),this._callChildMethod("refresh"))}}),t.widget("ui.checkboxradio",[t.ui.formResetMixin,{version:"1.12.1",options:{disabled:null,label:null,icon:!0,classes:{"ui-checkboxradio-label":"ui-corner-all","ui-checkboxradio-icon":"ui-corner-all"}},_getCreateOptions:function(){var e,i,s=this,n=this._super()||{};return this._readType(),i=this.element.labels(),this.label=t(i[i.length-1]),this.label.length||t.error("No label found for checkboxradio widget"),this.originalLabel="",this.label.contents().not(this.element[0]).each(function(){s.originalLabel+=3===this.nodeType?t(this).text():this.outerHTML}),this.originalLabel&&(n.label=this.originalLabel),e=this.element[0].disabled,null!=e&&(n.disabled=e),n},_create:function(){var t=this.element[0].checked;this._bindFormResetHandler(),null==this.options.disabled&&(this.options.disabled=this.element[0].disabled),this._setOption("disabled",this.options.disabled),this._addClass("ui-checkboxradio","ui-helper-hidden-accessible"),this._addClass(this.label,"ui-checkboxradio-label","ui-button ui-widget"),"radio"===this.type&&this._addClass(this.label,"ui-checkboxradio-radio-label"),this.options.label&&this.options.label!==this.originalLabel?this._updateLabel():this.originalLabel&&(this.options.label=this.originalLabel),this._enhance(),t&&(this._addClass(this.label,"ui-checkboxradio-checked","ui-state-active"),this.icon&&this._addClass(this.icon,null,"ui-state-hover")),this._on({change:"_toggleClasses",focus:function(){this._addClass(this.label,null,"ui-state-focus ui-visual-focus")},blur:function(){this._removeClass(this.label,null,"ui-state-focus ui-visual-focus")}})},_readType:function(){var e=this.element[0].nodeName.toLowerCase();this.type=this.element[0].type,"input"===e&&/radio|checkbox/.test(this.type)||t.error("Can't create checkboxradio on element.nodeName="+e+" and element.type="+this.type)},_enhance:function(){this._updateIcon(this.element[0].checked)},widget:function(){return this.label},_getRadioGroup:function(){var e,i=this.element[0].name,s="input[name='"+t.ui.escapeSelector(i)+"']";return i?(e=this.form.length?t(this.form[0].elements).filter(s):t(s).filter(function(){return 0===t(this).form().length}),e.not(this.element)):t([])},_toggleClasses:function(){var e=this.element[0].checked;this._toggleClass(this.label,"ui-checkboxradio-checked","ui-state-active",e),this.options.icon&&"checkbox"===this.type&&this._toggleClass(this.icon,null,"ui-icon-check ui-state-checked",e)._toggleClass(this.icon,null,"ui-icon-blank",!e),"radio"===this.type&&this._getRadioGroup().each(function(){var e=t(this).checkboxradio("instance");e&&e._removeClass(e.label,"ui-checkboxradio-checked","ui-state-active")})},_destroy:function(){this._unbindFormResetHandler(),this.icon&&(this.icon.remove(),this.iconSpace.remove())},_setOption:function(t,e){return"label"!==t||e?(this._super(t,e),"disabled"===t?(this._toggleClass(this.label,null,"ui-state-disabled",e),this.element[0].disabled=e,void 0):(this.refresh(),void 0)):void 0},_updateIcon:function(e){var i="ui-icon ui-icon-background ";this.options.icon?(this.icon||(this.icon=t("<span>"),this.iconSpace=t("<span> </span>"),this._addClass(this.iconSpace,"ui-checkboxradio-icon-space")),"checkbox"===this.type?(i+=e?"ui-icon-check ui-state-checked":"ui-icon-blank",this._removeClass(this.icon,null,e?"ui-icon-blank":"ui-icon-check")):i+="ui-icon-blank",this._addClass(this.icon,"ui-checkboxradio-icon",i),e||this._removeClass(this.icon,null,"ui-icon-check ui-state-checked"),this.icon.prependTo(this.label).after(this.iconSpace)):void 0!==this.icon&&(this.icon.remove(),this.iconSpace.remove(),delete this.icon)},_updateLabel:function(){var t=this.label.contents().not(this.element[0]);this.icon&&(t=t.not(this.icon[0])),this.iconSpace&&(t=t.not(this.iconSpace[0])),t.remove(),this.label.append(this.options.label)},refresh:function(){var t=this.element[0].checked,e=this.element[0].disabled;this._updateIcon(t),this._toggleClass(this.label,"ui-checkboxradio-checked","ui-state-active",t),null!==this.options.label&&this._updateLabel(),e!==this.options.disabled&&this._setOptions({disabled:e})}}]),t.ui.checkboxradio,t.widget("ui.button",{version:"1.12.1",defaultElement:"<button>",options:{classes:{"ui-button":"ui-corner-all"},disabled:null,icon:null,iconPosition:"beginning",label:null,showLabel:!0},_getCreateOptions:function(){var t,e=this._super()||{};return this.isInput=this.element.is("input"),t=this.element[0].disabled,null!=t&&(e.disabled=t),this.originalLabel=this.isInput?this.element.val():this.element.html(),this.originalLabel&&(e.label=this.originalLabel),e},_create:function(){!this.option.showLabel&!this.options.icon&&(this.options.showLabel=!0),null==this.options.disabled&&(this.options.disabled=this.element[0].disabled||!1),this.hasTitle=!!this.element.attr("title"),this.options.label&&this.options.label!==this.originalLabel&&(this.isInput?this.element.val(this.options.label):this.element.html(this.options.label)),this._addClass("ui-button","ui-widget"),this._setOption("disabled",this.options.disabled),this._enhance(),this.element.is("a")&&this._on({keyup:function(e){e.keyCode===t.ui.keyCode.SPACE&&(e.preventDefault(),this.element[0].click?this.element[0].click():this.element.trigger("click"))}})},_enhance:function(){this.element.is("button")||this.element.attr("role","button"),this.options.icon&&(this._updateIcon("icon",this.options.icon),this._updateTooltip())},_updateTooltip:function(){this.title=this.element.attr("title"),this.options.showLabel||this.title||this.element.attr("title",this.options.label)},_updateIcon:function(e,i){var s="iconPosition"!==e,n=s?this.options.iconPosition:i,o="top"===n||"bottom"===n;this.icon?s&&this._removeClass(this.icon,null,this.options.icon):(this.icon=t("<span>"),this._addClass(this.icon,"ui-button-icon","ui-icon"),this.options.showLabel||this._addClass("ui-button-icon-only")),s&&this._addClass(this.icon,null,i),this._attachIcon(n),o?(this._addClass(this.icon,null,"ui-widget-icon-block"),this.iconSpace&&this.iconSpace.remove()):(this.iconSpace||(this.iconSpace=t("<span> </span>"),this._addClass(this.iconSpace,"ui-button-icon-space")),this._removeClass(this.icon,null,"ui-wiget-icon-block"),this._attachIconSpace(n))},_destroy:function(){this.element.removeAttr("role"),this.icon&&this.icon.remove(),this.iconSpace&&this.iconSpace.remove(),this.hasTitle||this.element.removeAttr("title")},_attachIconSpace:function(t){this.icon[/^(?:end|bottom)/.test(t)?"before":"after"](this.iconSpace)},_attachIcon:function(t){this.element[/^(?:end|bottom)/.test(t)?"append":"prepend"](this.icon)},_setOptions:function(t){var e=void 0===t.showLabel?this.options.showLabel:t.showLabel,i=void 0===t.icon?this.options.icon:t.icon;e||i||(t.showLabel=!0),this._super(t)},_setOption:function(t,e){"icon"===t&&(e?this._updateIcon(t,e):this.icon&&(this.icon.remove(),this.iconSpace&&this.iconSpace.remove())),"iconPosition"===t&&this._updateIcon(t,e),"showLabel"===t&&(this._toggleClass("ui-button-icon-only",null,!e),this._updateTooltip()),"label"===t&&(this.isInput?this.element.val(e):(this.element.html(e),this.icon&&(this._attachIcon(this.options.iconPosition),this._attachIconSpace(this.options.iconPosition)))),this._super(t,e),"disabled"===t&&(this._toggleClass(null,"ui-state-disabled",e),this.element[0].disabled=e,e&&this.element.blur())},refresh:function(){var t=this.element.is("input, button")?this.element[0].disabled:this.element.hasClass("ui-button-disabled");t!==this.options.disabled&&this._setOptions({disabled:t}),this._updateTooltip()}}),t.uiBackCompat!==!1&&(t.widget("ui.button",t.ui.button,{options:{text:!0,icons:{primary:null,secondary:null}},_create:function(){this.options.showLabel&&!this.options.text&&(this.options.showLabel=this.options.text),!this.options.showLabel&&this.options.text&&(this.options.text=this.options.showLabel),this.options.icon||!this.options.icons.primary&&!this.options.icons.secondary?this.options.icon&&(this.options.icons.primary=this.options.icon):this.options.icons.primary?this.options.icon=this.options.icons.primary:(this.options.icon=this.options.icons.secondary,this.options.iconPosition="end"),this._super()},_setOption:function(t,e){return"text"===t?(this._super("showLabel",e),void 0):("showLabel"===t&&(this.options.text=e),"icon"===t&&(this.options.icons.primary=e),"icons"===t&&(e.primary?(this._super("icon",e.primary),this._super("iconPosition","beginning")):e.secondary&&(this._super("icon",e.secondary),this._super("iconPosition","end"))),this._superApply(arguments),void 0)}}),t.fn.button=function(e){return function(){return!this.length||this.length&&"INPUT"!==this[0].tagName||this.length&&"INPUT"===this[0].tagName&&"checkbox"!==this.attr("type")&&"radio"!==this.attr("type")?e.apply(this,arguments):(t.ui.checkboxradio||t.error("Checkboxradio widget missing"),0===arguments.length?this.checkboxradio({icon:!1}):this.checkboxradio.apply(this,arguments))}}(t.fn.button),t.fn.buttonset=function(){return t.ui.controlgroup||t.error("Controlgroup widget missing"),"option"===arguments[0]&&"items"===arguments[1]&&arguments[2]?this.controlgroup.apply(this,[arguments[0],"items.button",arguments[2]]):"option"===arguments[0]&&"items"===arguments[1]?this.controlgroup.apply(this,[arguments[0],"items.button"]):("object"==typeof arguments[0]&&arguments[0].items&&(arguments[0].items={button:arguments[0].items}),this.controlgroup.apply(this,arguments))}),t.ui.button,t.extend(t.ui,{datepicker:{version:"1.12.1"}});var m;t.extend(s.prototype,{markerClassName:"hasDatepicker",maxRows:4,_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(t){return a(this._defaults,t||{}),this},_attachDatepicker:function(e,i){var s,n,o;s=e.nodeName.toLowerCase(),n="div"===s||"span"===s,e.id||(this.uuid+=1,e.id="dp"+this.uuid),o=this._newInst(t(e),n),o.settings=t.extend({},i||{}),"input"===s?this._connectDatepicker(e,o):n&&this._inlineDatepicker(e,o)},_newInst:function(e,i){var s=e[0].id.replace(/([^A-Za-z0-9_\-])/g,"\\\\$1");return{id:s,input:e,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:i,dpDiv:i?n(t("<div class='"+this._inlineClass+" ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all'></div>")):this.dpDiv}},_connectDatepicker:function(e,i){var s=t(e);i.append=t([]),i.trigger=t([]),s.hasClass(this.markerClassName)||(this._attachments(s,i),s.addClass(this.markerClassName).on("keydown",this._doKeyDown).on("keypress",this._doKeyPress).on("keyup",this._doKeyUp),this._autoSize(i),t.data(e,"datepicker",i),i.settings.disabled&&this._disableDatepicker(e))},_attachments:function(e,i){var s,n,o,a=this._get(i,"appendText"),r=this._get(i,"isRTL");i.append&&i.append.remove(),a&&(i.append=t("<span class='"+this._appendClass+"'>"+a+"</span>"),e[r?"before":"after"](i.append)),e.off("focus",this._showDatepicker),i.trigger&&i.trigger.remove(),s=this._get(i,"showOn"),("focus"===s||"both"===s)&&e.on("focus",this._showDatepicker),("button"===s||"both"===s)&&(n=this._get(i,"buttonText"),o=this._get(i,"buttonImage"),i.trigger=t(this._get(i,"buttonImageOnly")?t("<img/>").addClass(this._triggerClass).attr({src:o,alt:n,title:n}):t("<button type='button'></button>").addClass(this._triggerClass).html(o?t("<img/>").attr({src:o,alt:n,title:n}):n)),e[r?"before":"after"](i.trigger),i.trigger.on("click",function(){return t.datepicker._datepickerShowing&&t.datepicker._lastInput===e[0]?t.datepicker._hideDatepicker():t.datepicker._datepickerShowing&&t.datepicker._lastInput!==e[0]?(t.datepicker._hideDatepicker(),t.datepicker._showDatepicker(e[0])):t.datepicker._showDatepicker(e[0]),!1}))},_autoSize:function(t){if(this._get(t,"autoSize")&&!t.inline){var e,i,s,n,o=new Date(2009,11,20),a=this._get(t,"dateFormat");a.match(/[DM]/)&&(e=function(t){for(i=0,s=0,n=0;t.length>n;n++)t[n].length>i&&(i=t[n].length,s=n);return s},o.setMonth(e(this._get(t,a.match(/MM/)?"monthNames":"monthNamesShort"))),o.setDate(e(this._get(t,a.match(/DD/)?"dayNames":"dayNamesShort"))+20-o.getDay())),t.input.attr("size",this._formatDate(t,o).length)}},_inlineDatepicker:function(e,i){var s=t(e);s.hasClass(this.markerClassName)||(s.addClass(this.markerClassName).append(i.dpDiv),t.data(e,"datepicker",i),this._setDate(i,this._getDefaultDate(i),!0),this._updateDatepicker(i),this._updateAlternate(i),i.settings.disabled&&this._disableDatepicker(e),i.dpDiv.css("display","block"))},_dialogDatepicker:function(e,i,s,n,o){var r,h,l,c,u,d=this._dialogInst;return d||(this.uuid+=1,r="dp"+this.uuid,this._dialogInput=t("<input type='text' id='"+r+"' style='position: absolute; top: -100px; width: 0px;'/>"),this._dialogInput.on("keydown",this._doKeyDown),t("body").append(this._dialogInput),d=this._dialogInst=this._newInst(this._dialogInput,!1),d.settings={},t.data(this._dialogInput[0],"datepicker",d)),a(d.settings,n||{}),i=i&&i.constructor===Date?this._formatDate(d,i):i,this._dialogInput.val(i),this._pos=o?o.length?o:[o.pageX,o.pageY]:null,this._pos||(h=document.documentElement.clientWidth,l=document.documentElement.clientHeight,c=document.documentElement.scrollLeft||document.body.scrollLeft,u=document.documentElement.scrollTop||document.body.scrollTop,this._pos=[h/2-100+c,l/2-150+u]),this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px"),d.settings.onSelect=s,this._inDialog=!0,this.dpDiv.addClass(this._dialogClass),this._showDatepicker(this._dialogInput[0]),t.blockUI&&t.blockUI(this.dpDiv),t.data(this._dialogInput[0],"datepicker",d),this},_destroyDatepicker:function(e){var i,s=t(e),n=t.data(e,"datepicker");s.hasClass(this.markerClassName)&&(i=e.nodeName.toLowerCase(),t.removeData(e,"datepicker"),"input"===i?(n.append.remove(),n.trigger.remove(),s.removeClass(this.markerClassName).off("focus",this._showDatepicker).off("keydown",this._doKeyDown).off("keypress",this._doKeyPress).off("keyup",this._doKeyUp)):("div"===i||"span"===i)&&s.removeClass(this.markerClassName).empty(),m===n&&(m=null))},_enableDatepicker:function(e){var i,s,n=t(e),o=t.data(e,"datepicker");n.hasClass(this.markerClassName)&&(i=e.nodeName.toLowerCase(),"input"===i?(e.disabled=!1,o.trigger.filter("button").each(function(){this.disabled=!1}).end().filter("img").css({opacity:"1.0",cursor:""})):("div"===i||"span"===i)&&(s=n.children("."+this._inlineClass),s.children().removeClass("ui-state-disabled"),s.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!1)),this._disabledInputs=t.map(this._disabledInputs,function(t){return t===e?null:t}))},_disableDatepicker:function(e){var i,s,n=t(e),o=t.data(e,"datepicker");n.hasClass(this.markerClassName)&&(i=e.nodeName.toLowerCase(),"input"===i?(e.disabled=!0,o.trigger.filter("button").each(function(){this.disabled=!0}).end().filter("img").css({opacity:"0.5",cursor:"default"})):("div"===i||"span"===i)&&(s=n.children("."+this._inlineClass),s.children().addClass("ui-state-disabled"),s.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!0)),this._disabledInputs=t.map(this._disabledInputs,function(t){return t===e?null:t}),this._disabledInputs[this._disabledInputs.length]=e)},_isDisabledDatepicker:function(t){if(!t)return!1;for(var e=0;this._disabledInputs.length>e;e++)if(this._disabledInputs[e]===t)return!0;return!1},_getInst:function(e){try{return t.data(e,"datepicker")}catch(i){throw"Missing instance data for this datepicker"}},_optionDatepicker:function(e,i,s){var n,o,r,h,l=this._getInst(e);return 2===arguments.length&&"string"==typeof i?"defaults"===i?t.extend({},t.datepicker._defaults):l?"all"===i?t.extend({},l.settings):this._get(l,i):null:(n=i||{},"string"==typeof i&&(n={},n[i]=s),l&&(this._curInst===l&&this._hideDatepicker(),o=this._getDateDatepicker(e,!0),r=this._getMinMaxDate(l,"min"),h=this._getMinMaxDate(l,"max"),a(l.settings,n),null!==r&&void 0!==n.dateFormat&&void 0===n.minDate&&(l.settings.minDate=this._formatDate(l,r)),null!==h&&void 0!==n.dateFormat&&void 0===n.maxDate&&(l.settings.maxDate=this._formatDate(l,h)),"disabled"in n&&(n.disabled?this._disableDatepicker(e):this._enableDatepicker(e)),this._attachments(t(e),l),this._autoSize(l),this._setDate(l,o),this._updateAlternate(l),this._updateDatepicker(l)),void 0)},_changeDatepicker:function(t,e,i){this._optionDatepicker(t,e,i)},_refreshDatepicker:function(t){var e=this._getInst(t);e&&this._updateDatepicker(e)},_setDateDatepicker:function(t,e){var i=this._getInst(t);i&&(this._setDate(i,e),this._updateDatepicker(i),this._updateAlternate(i))},_getDateDatepicker:function(t,e){var i=this._getInst(t);return i&&!i.inline&&this._setDateFromField(i,e),i?this._getDate(i):null},_doKeyDown:function(e){var i,s,n,o=t.datepicker._getInst(e.target),a=!0,r=o.dpDiv.is(".ui-datepicker-rtl");if(o._keyEvent=!0,t.datepicker._datepickerShowing)switch(e.keyCode){case 9:t.datepicker._hideDatepicker(),a=!1;break;case 13:return n=t("td."+t.datepicker._dayOverClass+":not(."+t.datepicker._currentClass+")",o.dpDiv),n[0]&&t.datepicker._selectDay(e.target,o.selectedMonth,o.selectedYear,n[0]),i=t.datepicker._get(o,"onSelect"),i?(s=t.datepicker._formatDate(o),i.apply(o.input?o.input[0]:null,[s,o])):t.datepicker._hideDatepicker(),!1;case 27:t.datepicker._hideDatepicker();break;case 33:t.datepicker._adjustDate(e.target,e.ctrlKey?-t.datepicker._get(o,"stepBigMonths"):-t.datepicker._get(o,"stepMonths"),"M");break;case 34:t.datepicker._adjustDate(e.target,e.ctrlKey?+t.datepicker._get(o,"stepBigMonths"):+t.datepicker._get(o,"stepMonths"),"M");break;case 35:(e.ctrlKey||e.metaKey)&&t.datepicker._clearDate(e.target),a=e.ctrlKey||e.metaKey;break;case 36:(e.ctrlKey||e.metaKey)&&t.datepicker._gotoToday(e.target),a=e.ctrlKey||e.metaKey;break;case 37:(e.ctrlKey||e.metaKey)&&t.datepicker._adjustDate(e.target,r?1:-1,"D"),a=e.ctrlKey||e.metaKey,e.originalEvent.altKey&&t.datepicker._adjustDate(e.target,e.ctrlKey?-t.datepicker._get(o,"stepBigMonths"):-t.datepicker._get(o,"stepMonths"),"M");break;case 38:(e.ctrlKey||e.metaKey)&&t.datepicker._adjustDate(e.target,-7,"D"),a=e.ctrlKey||e.metaKey;break;case 39:(e.ctrlKey||e.metaKey)&&t.datepicker._adjustDate(e.target,r?-1:1,"D"),a=e.ctrlKey||e.metaKey,e.originalEvent.altKey&&t.datepicker._adjustDate(e.target,e.ctrlKey?+t.datepicker._get(o,"stepBigMonths"):+t.datepicker._get(o,"stepMonths"),"M");break;case 40:(e.ctrlKey||e.metaKey)&&t.datepicker._adjustDate(e.target,7,"D"),a=e.ctrlKey||e.metaKey;break;default:a=!1}else 36===e.keyCode&&e.ctrlKey?t.datepicker._showDatepicker(this):a=!1;a&&(e.preventDefault(),e.stopPropagation())},_doKeyPress:function(e){var i,s,n=t.datepicker._getInst(e.target);return t.datepicker._get(n,"constrainInput")?(i=t.datepicker._possibleChars(t.datepicker._get(n,"dateFormat")),s=String.fromCharCode(null==e.charCode?e.keyCode:e.charCode),e.ctrlKey||e.metaKey||" ">s||!i||i.indexOf(s)>-1):void 0},_doKeyUp:function(e){var i,s=t.datepicker._getInst(e.target);if(s.input.val()!==s.lastVal)try{i=t.datepicker.parseDate(t.datepicker._get(s,"dateFormat"),s.input?s.input.val():null,t.datepicker._getFormatConfig(s)),i&&(t.datepicker._setDateFromField(s),t.datepicker._updateAlternate(s),t.datepicker._updateDatepicker(s))}catch(n){}return!0},_showDatepicker:function(e){if(e=e.target||e,"input"!==e.nodeName.toLowerCase()&&(e=t("input",e.parentNode)[0]),!t.datepicker._isDisabledDatepicker(e)&&t.datepicker._lastInput!==e){var s,n,o,r,h,l,c;s=t.datepicker._getInst(e),t.datepicker._curInst&&t.datepicker._curInst!==s&&(t.datepicker._curInst.dpDiv.stop(!0,!0),s&&t.datepicker._datepickerShowing&&t.datepicker._hideDatepicker(t.datepicker._curInst.input[0])),n=t.datepicker._get(s,"beforeShow"),o=n?n.apply(e,[e,s]):{},o!==!1&&(a(s.settings,o),s.lastVal=null,t.datepicker._lastInput=e,t.datepicker._setDateFromField(s),t.datepicker._inDialog&&(e.value=""),t.datepicker._pos||(t.datepicker._pos=t.datepicker._findPos(e),t.datepicker._pos[1]+=e.offsetHeight),r=!1,t(e).parents().each(function(){return r|="fixed"===t(this).css("position"),!r}),h={left:t.datepicker._pos[0],top:t.datepicker._pos[1]},t.datepicker._pos=null,s.dpDiv.empty(),s.dpDiv.css({position:"absolute",display:"block",top:"-1000px"}),t.datepicker._updateDatepicker(s),h=t.datepicker._checkOffset(s,h,r),s.dpDiv.css({position:t.datepicker._inDialog&&t.blockUI?"static":r?"fixed":"absolute",display:"none",left:h.left+"px",top:h.top+"px"}),s.inline||(l=t.datepicker._get(s,"showAnim"),c=t.datepicker._get(s,"duration"),s.dpDiv.css("z-index",i(t(e))+1),t.datepicker._datepickerShowing=!0,t.effects&&t.effects.effect[l]?s.dpDiv.show(l,t.datepicker._get(s,"showOptions"),c):s.dpDiv[l||"show"](l?c:null),t.datepicker._shouldFocusInput(s)&&s.input.trigger("focus"),t.datepicker._curInst=s))
-}},_updateDatepicker:function(e){this.maxRows=4,m=e,e.dpDiv.empty().append(this._generateHTML(e)),this._attachHandlers(e);var i,s=this._getNumberOfMonths(e),n=s[1],a=17,r=e.dpDiv.find("."+this._dayOverClass+" a");r.length>0&&o.apply(r.get(0)),e.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width(""),n>1&&e.dpDiv.addClass("ui-datepicker-multi-"+n).css("width",a*n+"em"),e.dpDiv[(1!==s[0]||1!==s[1]?"add":"remove")+"Class"]("ui-datepicker-multi"),e.dpDiv[(this._get(e,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl"),e===t.datepicker._curInst&&t.datepicker._datepickerShowing&&t.datepicker._shouldFocusInput(e)&&e.input.trigger("focus"),e.yearshtml&&(i=e.yearshtml,setTimeout(function(){i===e.yearshtml&&e.yearshtml&&e.dpDiv.find("select.ui-datepicker-year:first").replaceWith(e.yearshtml),i=e.yearshtml=null},0))},_shouldFocusInput:function(t){return t.input&&t.input.is(":visible")&&!t.input.is(":disabled")&&!t.input.is(":focus")},_checkOffset:function(e,i,s){var n=e.dpDiv.outerWidth(),o=e.dpDiv.outerHeight(),a=e.input?e.input.outerWidth():0,r=e.input?e.input.outerHeight():0,h=document.documentElement.clientWidth+(s?0:t(document).scrollLeft()),l=document.documentElement.clientHeight+(s?0:t(document).scrollTop());return i.left-=this._get(e,"isRTL")?n-a:0,i.left-=s&&i.left===e.input.offset().left?t(document).scrollLeft():0,i.top-=s&&i.top===e.input.offset().top+r?t(document).scrollTop():0,i.left-=Math.min(i.left,i.left+n>h&&h>n?Math.abs(i.left+n-h):0),i.top-=Math.min(i.top,i.top+o>l&&l>o?Math.abs(o+r):0),i},_findPos:function(e){for(var i,s=this._getInst(e),n=this._get(s,"isRTL");e&&("hidden"===e.type||1!==e.nodeType||t.expr.filters.hidden(e));)e=e[n?"previousSibling":"nextSibling"];return i=t(e).offset(),[i.left,i.top]},_hideDatepicker:function(e){var i,s,n,o,a=this._curInst;!a||e&&a!==t.data(e,"datepicker")||this._datepickerShowing&&(i=this._get(a,"showAnim"),s=this._get(a,"duration"),n=function(){t.datepicker._tidyDialog(a)},t.effects&&(t.effects.effect[i]||t.effects[i])?a.dpDiv.hide(i,t.datepicker._get(a,"showOptions"),s,n):a.dpDiv["slideDown"===i?"slideUp":"fadeIn"===i?"fadeOut":"hide"](i?s:null,n),i||n(),this._datepickerShowing=!1,o=this._get(a,"onClose"),o&&o.apply(a.input?a.input[0]:null,[a.input?a.input.val():"",a]),this._lastInput=null,this._inDialog&&(this._dialogInput.css({position:"absolute",left:"0",top:"-100px"}),t.blockUI&&(t.unblockUI(),t("body").append(this.dpDiv))),this._inDialog=!1)},_tidyDialog:function(t){t.dpDiv.removeClass(this._dialogClass).off(".ui-datepicker-calendar")},_checkExternalClick:function(e){if(t.datepicker._curInst){var i=t(e.target),s=t.datepicker._getInst(i[0]);(i[0].id!==t.datepicker._mainDivId&&0===i.parents("#"+t.datepicker._mainDivId).length&&!i.hasClass(t.datepicker.markerClassName)&&!i.closest("."+t.datepicker._triggerClass).length&&t.datepicker._datepickerShowing&&(!t.datepicker._inDialog||!t.blockUI)||i.hasClass(t.datepicker.markerClassName)&&t.datepicker._curInst!==s)&&t.datepicker._hideDatepicker()}},_adjustDate:function(e,i,s){var n=t(e),o=this._getInst(n[0]);this._isDisabledDatepicker(n[0])||(this._adjustInstDate(o,i+("M"===s?this._get(o,"showCurrentAtPos"):0),s),this._updateDatepicker(o))},_gotoToday:function(e){var i,s=t(e),n=this._getInst(s[0]);this._get(n,"gotoCurrent")&&n.currentDay?(n.selectedDay=n.currentDay,n.drawMonth=n.selectedMonth=n.currentMonth,n.drawYear=n.selectedYear=n.currentYear):(i=new Date,n.selectedDay=i.getDate(),n.drawMonth=n.selectedMonth=i.getMonth(),n.drawYear=n.selectedYear=i.getFullYear()),this._notifyChange(n),this._adjustDate(s)},_selectMonthYear:function(e,i,s){var n=t(e),o=this._getInst(n[0]);o["selected"+("M"===s?"Month":"Year")]=o["draw"+("M"===s?"Month":"Year")]=parseInt(i.options[i.selectedIndex].value,10),this._notifyChange(o),this._adjustDate(n)},_selectDay:function(e,i,s,n){var o,a=t(e);t(n).hasClass(this._unselectableClass)||this._isDisabledDatepicker(a[0])||(o=this._getInst(a[0]),o.selectedDay=o.currentDay=t("a",n).html(),o.selectedMonth=o.currentMonth=i,o.selectedYear=o.currentYear=s,this._selectDate(e,this._formatDate(o,o.currentDay,o.currentMonth,o.currentYear)))},_clearDate:function(e){var i=t(e);this._selectDate(i,"")},_selectDate:function(e,i){var s,n=t(e),o=this._getInst(n[0]);i=null!=i?i:this._formatDate(o),o.input&&o.input.val(i),this._updateAlternate(o),s=this._get(o,"onSelect"),s?s.apply(o.input?o.input[0]:null,[i,o]):o.input&&o.input.trigger("change"),o.inline?this._updateDatepicker(o):(this._hideDatepicker(),this._lastInput=o.input[0],"object"!=typeof o.input[0]&&o.input.trigger("focus"),this._lastInput=null)},_updateAlternate:function(e){var i,s,n,o=this._get(e,"altField");o&&(i=this._get(e,"altFormat")||this._get(e,"dateFormat"),s=this._getDate(e),n=this.formatDate(i,s,this._getFormatConfig(e)),t(o).val(n))},noWeekends:function(t){var e=t.getDay();return[e>0&&6>e,""]},iso8601Week:function(t){var e,i=new Date(t.getTime());return i.setDate(i.getDate()+4-(i.getDay()||7)),e=i.getTime(),i.setMonth(0),i.setDate(1),Math.floor(Math.round((e-i)/864e5)/7)+1},parseDate:function(e,i,s){if(null==e||null==i)throw"Invalid arguments";if(i="object"==typeof i?""+i:i+"",""===i)return null;var n,o,a,r,h=0,l=(s?s.shortYearCutoff:null)||this._defaults.shortYearCutoff,c="string"!=typeof l?l:(new Date).getFullYear()%100+parseInt(l,10),u=(s?s.dayNamesShort:null)||this._defaults.dayNamesShort,d=(s?s.dayNames:null)||this._defaults.dayNames,p=(s?s.monthNamesShort:null)||this._defaults.monthNamesShort,f=(s?s.monthNames:null)||this._defaults.monthNames,g=-1,m=-1,_=-1,v=-1,b=!1,y=function(t){var i=e.length>n+1&&e.charAt(n+1)===t;return i&&n++,i},w=function(t){var e=y(t),s="@"===t?14:"!"===t?20:"y"===t&&e?4:"o"===t?3:2,n="y"===t?s:1,o=RegExp("^\\d{"+n+","+s+"}"),a=i.substring(h).match(o);if(!a)throw"Missing number at position "+h;return h+=a[0].length,parseInt(a[0],10)},k=function(e,s,n){var o=-1,a=t.map(y(e)?n:s,function(t,e){return[[e,t]]}).sort(function(t,e){return-(t[1].length-e[1].length)});if(t.each(a,function(t,e){var s=e[1];return i.substr(h,s.length).toLowerCase()===s.toLowerCase()?(o=e[0],h+=s.length,!1):void 0}),-1!==o)return o+1;throw"Unknown name at position "+h},x=function(){if(i.charAt(h)!==e.charAt(n))throw"Unexpected literal at position "+h;h++};for(n=0;e.length>n;n++)if(b)"'"!==e.charAt(n)||y("'")?x():b=!1;else switch(e.charAt(n)){case"d":_=w("d");break;case"D":k("D",u,d);break;case"o":v=w("o");break;case"m":m=w("m");break;case"M":m=k("M",p,f);break;case"y":g=w("y");break;case"@":r=new Date(w("@")),g=r.getFullYear(),m=r.getMonth()+1,_=r.getDate();break;case"!":r=new Date((w("!")-this._ticksTo1970)/1e4),g=r.getFullYear(),m=r.getMonth()+1,_=r.getDate();break;case"'":y("'")?x():b=!0;break;default:x()}if(i.length>h&&(a=i.substr(h),!/^\s+/.test(a)))throw"Extra/unparsed characters found in date: "+a;if(-1===g?g=(new Date).getFullYear():100>g&&(g+=(new Date).getFullYear()-(new Date).getFullYear()%100+(c>=g?0:-100)),v>-1)for(m=1,_=v;;){if(o=this._getDaysInMonth(g,m-1),o>=_)break;m++,_-=o}if(r=this._daylightSavingAdjust(new Date(g,m-1,_)),r.getFullYear()!==g||r.getMonth()+1!==m||r.getDate()!==_)throw"Invalid date";return r},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:1e7*60*60*24*(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925)),formatDate:function(t,e,i){if(!e)return"";var s,n=(i?i.dayNamesShort:null)||this._defaults.dayNamesShort,o=(i?i.dayNames:null)||this._defaults.dayNames,a=(i?i.monthNamesShort:null)||this._defaults.monthNamesShort,r=(i?i.monthNames:null)||this._defaults.monthNames,h=function(e){var i=t.length>s+1&&t.charAt(s+1)===e;return i&&s++,i},l=function(t,e,i){var s=""+e;if(h(t))for(;i>s.length;)s="0"+s;return s},c=function(t,e,i,s){return h(t)?s[e]:i[e]},u="",d=!1;if(e)for(s=0;t.length>s;s++)if(d)"'"!==t.charAt(s)||h("'")?u+=t.charAt(s):d=!1;else switch(t.charAt(s)){case"d":u+=l("d",e.getDate(),2);break;case"D":u+=c("D",e.getDay(),n,o);break;case"o":u+=l("o",Math.round((new Date(e.getFullYear(),e.getMonth(),e.getDate()).getTime()-new Date(e.getFullYear(),0,0).getTime())/864e5),3);break;case"m":u+=l("m",e.getMonth()+1,2);break;case"M":u+=c("M",e.getMonth(),a,r);break;case"y":u+=h("y")?e.getFullYear():(10>e.getFullYear()%100?"0":"")+e.getFullYear()%100;break;case"@":u+=e.getTime();break;case"!":u+=1e4*e.getTime()+this._ticksTo1970;break;case"'":h("'")?u+="'":d=!0;break;default:u+=t.charAt(s)}return u},_possibleChars:function(t){var e,i="",s=!1,n=function(i){var s=t.length>e+1&&t.charAt(e+1)===i;return s&&e++,s};for(e=0;t.length>e;e++)if(s)"'"!==t.charAt(e)||n("'")?i+=t.charAt(e):s=!1;else switch(t.charAt(e)){case"d":case"m":case"y":case"@":i+="0123456789";break;case"D":case"M":return null;case"'":n("'")?i+="'":s=!0;break;default:i+=t.charAt(e)}return i},_get:function(t,e){return void 0!==t.settings[e]?t.settings[e]:this._defaults[e]},_setDateFromField:function(t,e){if(t.input.val()!==t.lastVal){var i=this._get(t,"dateFormat"),s=t.lastVal=t.input?t.input.val():null,n=this._getDefaultDate(t),o=n,a=this._getFormatConfig(t);try{o=this.parseDate(i,s,a)||n}catch(r){s=e?"":s}t.selectedDay=o.getDate(),t.drawMonth=t.selectedMonth=o.getMonth(),t.drawYear=t.selectedYear=o.getFullYear(),t.currentDay=s?o.getDate():0,t.currentMonth=s?o.getMonth():0,t.currentYear=s?o.getFullYear():0,this._adjustInstDate(t)}},_getDefaultDate:function(t){return this._restrictMinMax(t,this._determineDate(t,this._get(t,"defaultDate"),new Date))},_determineDate:function(e,i,s){var n=function(t){var e=new Date;return e.setDate(e.getDate()+t),e},o=function(i){try{return t.datepicker.parseDate(t.datepicker._get(e,"dateFormat"),i,t.datepicker._getFormatConfig(e))}catch(s){}for(var n=(i.toLowerCase().match(/^c/)?t.datepicker._getDate(e):null)||new Date,o=n.getFullYear(),a=n.getMonth(),r=n.getDate(),h=/([+\-]?[0-9]+)\s*(d|D|w|W|m|M|y|Y)?/g,l=h.exec(i);l;){switch(l[2]||"d"){case"d":case"D":r+=parseInt(l[1],10);break;case"w":case"W":r+=7*parseInt(l[1],10);break;case"m":case"M":a+=parseInt(l[1],10),r=Math.min(r,t.datepicker._getDaysInMonth(o,a));break;case"y":case"Y":o+=parseInt(l[1],10),r=Math.min(r,t.datepicker._getDaysInMonth(o,a))}l=h.exec(i)}return new Date(o,a,r)},a=null==i||""===i?s:"string"==typeof i?o(i):"number"==typeof i?isNaN(i)?s:n(i):new Date(i.getTime());return a=a&&"Invalid Date"==""+a?s:a,a&&(a.setHours(0),a.setMinutes(0),a.setSeconds(0),a.setMilliseconds(0)),this._daylightSavingAdjust(a)},_daylightSavingAdjust:function(t){return t?(t.setHours(t.getHours()>12?t.getHours()+2:0),t):null},_setDate:function(t,e,i){var s=!e,n=t.selectedMonth,o=t.selectedYear,a=this._restrictMinMax(t,this._determineDate(t,e,new Date));t.selectedDay=t.currentDay=a.getDate(),t.drawMonth=t.selectedMonth=t.currentMonth=a.getMonth(),t.drawYear=t.selectedYear=t.currentYear=a.getFullYear(),n===t.selectedMonth&&o===t.selectedYear||i||this._notifyChange(t),this._adjustInstDate(t),t.input&&t.input.val(s?"":this._formatDate(t))},_getDate:function(t){var e=!t.currentYear||t.input&&""===t.input.val()?null:this._daylightSavingAdjust(new Date(t.currentYear,t.currentMonth,t.currentDay));return e},_attachHandlers:function(e){var i=this._get(e,"stepMonths"),s="#"+e.id.replace(/\\\\/g,"\\");e.dpDiv.find("[data-handler]").map(function(){var e={prev:function(){t.datepicker._adjustDate(s,-i,"M")},next:function(){t.datepicker._adjustDate(s,+i,"M")},hide:function(){t.datepicker._hideDatepicker()},today:function(){t.datepicker._gotoToday(s)},selectDay:function(){return t.datepicker._selectDay(s,+this.getAttribute("data-month"),+this.getAttribute("data-year"),this),!1},selectMonth:function(){return t.datepicker._selectMonthYear(s,this,"M"),!1},selectYear:function(){return t.datepicker._selectMonthYear(s,this,"Y"),!1}};t(this).on(this.getAttribute("data-event"),e[this.getAttribute("data-handler")])})},_generateHTML:function(t){var e,i,s,n,o,a,r,h,l,c,u,d,p,f,g,m,_,v,b,y,w,k,x,C,D,I,T,P,M,S,H,z,O,A,N,W,E,F,L,R=new Date,B=this._daylightSavingAdjust(new Date(R.getFullYear(),R.getMonth(),R.getDate())),Y=this._get(t,"isRTL"),j=this._get(t,"showButtonPanel"),q=this._get(t,"hideIfNoPrevNext"),K=this._get(t,"navigationAsDateFormat"),U=this._getNumberOfMonths(t),V=this._get(t,"showCurrentAtPos"),$=this._get(t,"stepMonths"),X=1!==U[0]||1!==U[1],G=this._daylightSavingAdjust(t.currentDay?new Date(t.currentYear,t.currentMonth,t.currentDay):new Date(9999,9,9)),Q=this._getMinMaxDate(t,"min"),J=this._getMinMaxDate(t,"max"),Z=t.drawMonth-V,te=t.drawYear;if(0>Z&&(Z+=12,te--),J)for(e=this._daylightSavingAdjust(new Date(J.getFullYear(),J.getMonth()-U[0]*U[1]+1,J.getDate())),e=Q&&Q>e?Q:e;this._daylightSavingAdjust(new Date(te,Z,1))>e;)Z--,0>Z&&(Z=11,te--);for(t.drawMonth=Z,t.drawYear=te,i=this._get(t,"prevText"),i=K?this.formatDate(i,this._daylightSavingAdjust(new Date(te,Z-$,1)),this._getFormatConfig(t)):i,s=this._canAdjustMonth(t,-1,te,Z)?"<a class='ui-datepicker-prev ui-corner-all' data-handler='prev' data-event='click' title='"+i+"'><span class='ui-icon ui-icon-circle-triangle-"+(Y?"e":"w")+"'>"+i+"</span></a>":q?"":"<a class='ui-datepicker-prev ui-corner-all ui-state-disabled' title='"+i+"'><span class='ui-icon ui-icon-circle-triangle-"+(Y?"e":"w")+"'>"+i+"</span></a>",n=this._get(t,"nextText"),n=K?this.formatDate(n,this._daylightSavingAdjust(new Date(te,Z+$,1)),this._getFormatConfig(t)):n,o=this._canAdjustMonth(t,1,te,Z)?"<a class='ui-datepicker-next ui-corner-all' data-handler='next' data-event='click' title='"+n+"'><span class='ui-icon ui-icon-circle-triangle-"+(Y?"w":"e")+"'>"+n+"</span></a>":q?"":"<a class='ui-datepicker-next ui-corner-all ui-state-disabled' title='"+n+"'><span class='ui-icon ui-icon-circle-triangle-"+(Y?"w":"e")+"'>"+n+"</span></a>",a=this._get(t,"currentText"),r=this._get(t,"gotoCurrent")&&t.currentDay?G:B,a=K?this.formatDate(a,r,this._getFormatConfig(t)):a,h=t.inline?"":"<button type='button' class='ui-datepicker-close ui-state-default ui-priority-primary ui-corner-all' data-handler='hide' data-event='click'>"+this._get(t,"closeText")+"</button>",l=j?"<div class='ui-datepicker-buttonpane ui-widget-content'>"+(Y?h:"")+(this._isInRange(t,r)?"<button type='button' class='ui-datepicker-current ui-state-default ui-priority-secondary ui-corner-all' data-handler='today' data-event='click'>"+a+"</button>":"")+(Y?"":h)+"</div>":"",c=parseInt(this._get(t,"firstDay"),10),c=isNaN(c)?0:c,u=this._get(t,"showWeek"),d=this._get(t,"dayNames"),p=this._get(t,"dayNamesMin"),f=this._get(t,"monthNames"),g=this._get(t,"monthNamesShort"),m=this._get(t,"beforeShowDay"),_=this._get(t,"showOtherMonths"),v=this._get(t,"selectOtherMonths"),b=this._getDefaultDate(t),y="",k=0;U[0]>k;k++){for(x="",this.maxRows=4,C=0;U[1]>C;C++){if(D=this._daylightSavingAdjust(new Date(te,Z,t.selectedDay)),I=" ui-corner-all",T="",X){if(T+="<div class='ui-datepicker-group",U[1]>1)switch(C){case 0:T+=" ui-datepicker-group-first",I=" ui-corner-"+(Y?"right":"left");break;case U[1]-1:T+=" ui-datepicker-group-last",I=" ui-corner-"+(Y?"left":"right");break;default:T+=" ui-datepicker-group-middle",I=""}T+="'>"}for(T+="<div class='ui-datepicker-header ui-widget-header ui-helper-clearfix"+I+"'>"+(/all|left/.test(I)&&0===k?Y?o:s:"")+(/all|right/.test(I)&&0===k?Y?s:o:"")+this._generateMonthYearHeader(t,Z,te,Q,J,k>0||C>0,f,g)+"</div><table class='ui-datepicker-calendar'><thead>"+"<tr>",P=u?"<th class='ui-datepicker-week-col'>"+this._get(t,"weekHeader")+"</th>":"",w=0;7>w;w++)M=(w+c)%7,P+="<th scope='col'"+((w+c+6)%7>=5?" class='ui-datepicker-week-end'":"")+">"+"<span title='"+d[M]+"'>"+p[M]+"</span></th>";for(T+=P+"</tr></thead><tbody>",S=this._getDaysInMonth(te,Z),te===t.selectedYear&&Z===t.selectedMonth&&(t.selectedDay=Math.min(t.selectedDay,S)),H=(this._getFirstDayOfMonth(te,Z)-c+7)%7,z=Math.ceil((H+S)/7),O=X?this.maxRows>z?this.maxRows:z:z,this.maxRows=O,A=this._daylightSavingAdjust(new Date(te,Z,1-H)),N=0;O>N;N++){for(T+="<tr>",W=u?"<td class='ui-datepicker-week-col'>"+this._get(t,"calculateWeek")(A)+"</td>":"",w=0;7>w;w++)E=m?m.apply(t.input?t.input[0]:null,[A]):[!0,""],F=A.getMonth()!==Z,L=F&&!v||!E[0]||Q&&Q>A||J&&A>J,W+="<td class='"+((w+c+6)%7>=5?" ui-datepicker-week-end":"")+(F?" ui-datepicker-other-month":"")+(A.getTime()===D.getTime()&&Z===t.selectedMonth&&t._keyEvent||b.getTime()===A.getTime()&&b.getTime()===D.getTime()?" "+this._dayOverClass:"")+(L?" "+this._unselectableClass+" ui-state-disabled":"")+(F&&!_?"":" "+E[1]+(A.getTime()===G.getTime()?" "+this._currentClass:"")+(A.getTime()===B.getTime()?" ui-datepicker-today":""))+"'"+(F&&!_||!E[2]?"":" title='"+E[2].replace(/'/g,"&#39;")+"'")+(L?"":" data-handler='selectDay' data-event='click' data-month='"+A.getMonth()+"' data-year='"+A.getFullYear()+"'")+">"+(F&&!_?"&#xa0;":L?"<span class='ui-state-default'>"+A.getDate()+"</span>":"<a class='ui-state-default"+(A.getTime()===B.getTime()?" ui-state-highlight":"")+(A.getTime()===G.getTime()?" ui-state-active":"")+(F?" ui-priority-secondary":"")+"' href='#'>"+A.getDate()+"</a>")+"</td>",A.setDate(A.getDate()+1),A=this._daylightSavingAdjust(A);T+=W+"</tr>"}Z++,Z>11&&(Z=0,te++),T+="</tbody></table>"+(X?"</div>"+(U[0]>0&&C===U[1]-1?"<div class='ui-datepicker-row-break'></div>":""):""),x+=T}y+=x}return y+=l,t._keyEvent=!1,y},_generateMonthYearHeader:function(t,e,i,s,n,o,a,r){var h,l,c,u,d,p,f,g,m=this._get(t,"changeMonth"),_=this._get(t,"changeYear"),v=this._get(t,"showMonthAfterYear"),b="<div class='ui-datepicker-title'>",y="";if(o||!m)y+="<span class='ui-datepicker-month'>"+a[e]+"</span>";else{for(h=s&&s.getFullYear()===i,l=n&&n.getFullYear()===i,y+="<select class='ui-datepicker-month' data-handler='selectMonth' data-event='change'>",c=0;12>c;c++)(!h||c>=s.getMonth())&&(!l||n.getMonth()>=c)&&(y+="<option value='"+c+"'"+(c===e?" selected='selected'":"")+">"+r[c]+"</option>");y+="</select>"}if(v||(b+=y+(!o&&m&&_?"":"&#xa0;")),!t.yearshtml)if(t.yearshtml="",o||!_)b+="<span class='ui-datepicker-year'>"+i+"</span>";else{for(u=this._get(t,"yearRange").split(":"),d=(new Date).getFullYear(),p=function(t){var e=t.match(/c[+\-].*/)?i+parseInt(t.substring(1),10):t.match(/[+\-].*/)?d+parseInt(t,10):parseInt(t,10);return isNaN(e)?d:e},f=p(u[0]),g=Math.max(f,p(u[1]||"")),f=s?Math.max(f,s.getFullYear()):f,g=n?Math.min(g,n.getFullYear()):g,t.yearshtml+="<select class='ui-datepicker-year' data-handler='selectYear' data-event='change'>";g>=f;f++)t.yearshtml+="<option value='"+f+"'"+(f===i?" selected='selected'":"")+">"+f+"</option>";t.yearshtml+="</select>",b+=t.yearshtml,t.yearshtml=null}return b+=this._get(t,"yearSuffix"),v&&(b+=(!o&&m&&_?"":"&#xa0;")+y),b+="</div>"},_adjustInstDate:function(t,e,i){var s=t.selectedYear+("Y"===i?e:0),n=t.selectedMonth+("M"===i?e:0),o=Math.min(t.selectedDay,this._getDaysInMonth(s,n))+("D"===i?e:0),a=this._restrictMinMax(t,this._daylightSavingAdjust(new Date(s,n,o)));t.selectedDay=a.getDate(),t.drawMonth=t.selectedMonth=a.getMonth(),t.drawYear=t.selectedYear=a.getFullYear(),("M"===i||"Y"===i)&&this._notifyChange(t)},_restrictMinMax:function(t,e){var i=this._getMinMaxDate(t,"min"),s=this._getMinMaxDate(t,"max"),n=i&&i>e?i:e;return s&&n>s?s:n},_notifyChange:function(t){var e=this._get(t,"onChangeMonthYear");e&&e.apply(t.input?t.input[0]:null,[t.selectedYear,t.selectedMonth+1,t])},_getNumberOfMonths:function(t){var e=this._get(t,"numberOfMonths");return null==e?[1,1]:"number"==typeof e?[1,e]:e},_getMinMaxDate:function(t,e){return this._determineDate(t,this._get(t,e+"Date"),null)},_getDaysInMonth:function(t,e){return 32-this._daylightSavingAdjust(new Date(t,e,32)).getDate()},_getFirstDayOfMonth:function(t,e){return new Date(t,e,1).getDay()},_canAdjustMonth:function(t,e,i,s){var n=this._getNumberOfMonths(t),o=this._daylightSavingAdjust(new Date(i,s+(0>e?e:n[0]*n[1]),1));return 0>e&&o.setDate(this._getDaysInMonth(o.getFullYear(),o.getMonth())),this._isInRange(t,o)},_isInRange:function(t,e){var i,s,n=this._getMinMaxDate(t,"min"),o=this._getMinMaxDate(t,"max"),a=null,r=null,h=this._get(t,"yearRange");return h&&(i=h.split(":"),s=(new Date).getFullYear(),a=parseInt(i[0],10),r=parseInt(i[1],10),i[0].match(/[+\-].*/)&&(a+=s),i[1].match(/[+\-].*/)&&(r+=s)),(!n||e.getTime()>=n.getTime())&&(!o||e.getTime()<=o.getTime())&&(!a||e.getFullYear()>=a)&&(!r||r>=e.getFullYear())},_getFormatConfig:function(t){var e=this._get(t,"shortYearCutoff");return e="string"!=typeof e?e:(new Date).getFullYear()%100+parseInt(e,10),{shortYearCutoff:e,dayNamesShort:this._get(t,"dayNamesShort"),dayNames:this._get(t,"dayNames"),monthNamesShort:this._get(t,"monthNamesShort"),monthNames:this._get(t,"monthNames")}},_formatDate:function(t,e,i,s){e||(t.currentDay=t.selectedDay,t.currentMonth=t.selectedMonth,t.currentYear=t.selectedYear);var n=e?"object"==typeof e?e:this._daylightSavingAdjust(new Date(s,i,e)):this._daylightSavingAdjust(new Date(t.currentYear,t.currentMonth,t.currentDay));return this.formatDate(this._get(t,"dateFormat"),n,this._getFormatConfig(t))}}),t.fn.datepicker=function(e){if(!this.length)return this;t.datepicker.initialized||(t(document).on("mousedown",t.datepicker._checkExternalClick),t.datepicker.initialized=!0),0===t("#"+t.datepicker._mainDivId).length&&t("body").append(t.datepicker.dpDiv);var i=Array.prototype.slice.call(arguments,1);return"string"!=typeof e||"isDisabled"!==e&&"getDate"!==e&&"widget"!==e?"option"===e&&2===arguments.length&&"string"==typeof arguments[1]?t.datepicker["_"+e+"Datepicker"].apply(t.datepicker,[this[0]].concat(i)):this.each(function(){"string"==typeof e?t.datepicker["_"+e+"Datepicker"].apply(t.datepicker,[this].concat(i)):t.datepicker._attachDatepicker(this,e)}):t.datepicker["_"+e+"Datepicker"].apply(t.datepicker,[this[0]].concat(i))},t.datepicker=new s,t.datepicker.initialized=!1,t.datepicker.uuid=(new Date).getTime(),t.datepicker.version="1.12.1",t.datepicker,t.ui.ie=!!/msie [\w.]+/.exec(navigator.userAgent.toLowerCase());var _=!1;t(document).on("mouseup",function(){_=!1}),t.widget("ui.mouse",{version:"1.12.1",options:{cancel:"input, textarea, button, select, option",distance:1,delay:0},_mouseInit:function(){var e=this;this.element.on("mousedown."+this.widgetName,function(t){return e._mouseDown(t)}).on("click."+this.widgetName,function(i){return!0===t.data(i.target,e.widgetName+".preventClickEvent")?(t.removeData(i.target,e.widgetName+".preventClickEvent"),i.stopImmediatePropagation(),!1):void 0}),this.started=!1},_mouseDestroy:function(){this.element.off("."+this.widgetName),this._mouseMoveDelegate&&this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate)},_mouseDown:function(e){if(!_){this._mouseMoved=!1,this._mouseStarted&&this._mouseUp(e),this._mouseDownEvent=e;var i=this,s=1===e.which,n="string"==typeof this.options.cancel&&e.target.nodeName?t(e.target).closest(this.options.cancel).length:!1;return s&&!n&&this._mouseCapture(e)?(this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){i.mouseDelayMet=!0},this.options.delay)),this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(e)!==!1,!this._mouseStarted)?(e.preventDefault(),!0):(!0===t.data(e.target,this.widgetName+".preventClickEvent")&&t.removeData(e.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(t){return i._mouseMove(t)},this._mouseUpDelegate=function(t){return i._mouseUp(t)},this.document.on("mousemove."+this.widgetName,this._mouseMoveDelegate).on("mouseup."+this.widgetName,this._mouseUpDelegate),e.preventDefault(),_=!0,!0)):!0}},_mouseMove:function(e){if(this._mouseMoved){if(t.ui.ie&&(!document.documentMode||9>document.documentMode)&&!e.button)return this._mouseUp(e);if(!e.which)if(e.originalEvent.altKey||e.originalEvent.ctrlKey||e.originalEvent.metaKey||e.originalEvent.shiftKey)this.ignoreMissingWhich=!0;else if(!this.ignoreMissingWhich)return this._mouseUp(e)}return(e.which||e.button)&&(this._mouseMoved=!0),this._mouseStarted?(this._mouseDrag(e),e.preventDefault()):(this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(this._mouseDownEvent,e)!==!1,this._mouseStarted?this._mouseDrag(e):this._mouseUp(e)),!this._mouseStarted)},_mouseUp:function(e){this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,e.target===this._mouseDownEvent.target&&t.data(e.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(e)),this._mouseDelayTimer&&(clearTimeout(this._mouseDelayTimer),delete this._mouseDelayTimer),this.ignoreMissingWhich=!1,_=!1,e.preventDefault()},_mouseDistanceMet:function(t){return Math.max(Math.abs(this._mouseDownEvent.pageX-t.pageX),Math.abs(this._mouseDownEvent.pageY-t.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return!0}}),t.ui.plugin={add:function(e,i,s){var n,o=t.ui[e].prototype;for(n in s)o.plugins[n]=o.plugins[n]||[],o.plugins[n].push([i,s[n]])},call:function(t,e,i,s){var n,o=t.plugins[e];if(o&&(s||t.element[0].parentNode&&11!==t.element[0].parentNode.nodeType))for(n=0;o.length>n;n++)t.options[o[n][0]]&&o[n][1].apply(t.element,i)}},t.ui.safeBlur=function(e){e&&"body"!==e.nodeName.toLowerCase()&&t(e).trigger("blur")},t.widget("ui.draggable",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"drag",options:{addClasses:!0,appendTo:"parent",axis:!1,connectToSortable:!1,containment:!1,cursor:"auto",cursorAt:!1,grid:!1,handle:!1,helper:"original",iframeFix:!1,opacity:!1,refreshPositions:!1,revert:!1,revertDuration:500,scope:"default",scroll:!0,scrollSensitivity:20,scrollSpeed:20,snap:!1,snapMode:"both",snapTolerance:20,stack:!1,zIndex:!1,drag:null,start:null,stop:null},_create:function(){"original"===this.options.helper&&this._setPositionRelative(),this.options.addClasses&&this._addClass("ui-draggable"),this._setHandleClassName(),this._mouseInit()},_setOption:function(t,e){this._super(t,e),"handle"===t&&(this._removeHandleClassName(),this._setHandleClassName())},_destroy:function(){return(this.helper||this.element).is(".ui-draggable-dragging")?(this.destroyOnClear=!0,void 0):(this._removeHandleClassName(),this._mouseDestroy(),void 0)},_mouseCapture:function(e){var i=this.options;return this.helper||i.disabled||t(e.target).closest(".ui-resizable-handle").length>0?!1:(this.handle=this._getHandle(e),this.handle?(this._blurActiveElement(e),this._blockFrames(i.iframeFix===!0?"iframe":i.iframeFix),!0):!1)},_blockFrames:function(e){this.iframeBlocks=this.document.find(e).map(function(){var e=t(this);return t("<div>").css("position","absolute").appendTo(e.parent()).outerWidth(e.outerWidth()).outerHeight(e.outerHeight()).offset(e.offset())[0]})},_unblockFrames:function(){this.iframeBlocks&&(this.iframeBlocks.remove(),delete this.iframeBlocks)},_blurActiveElement:function(e){var i=t.ui.safeActiveElement(this.document[0]),s=t(e.target);s.closest(i).length||t.ui.safeBlur(i)},_mouseStart:function(e){var i=this.options;return this.helper=this._createHelper(e),this._addClass(this.helper,"ui-draggable-dragging"),this._cacheHelperProportions(),t.ui.ddmanager&&(t.ui.ddmanager.current=this),this._cacheMargins(),this.cssPosition=this.helper.css("position"),this.scrollParent=this.helper.scrollParent(!0),this.offsetParent=this.helper.offsetParent(),this.hasFixedAncestor=this.helper.parents().filter(function(){return"fixed"===t(this).css("position")}).length>0,this.positionAbs=this.element.offset(),this._refreshOffsets(e),this.originalPosition=this.position=this._generatePosition(e,!1),this.originalPageX=e.pageX,this.originalPageY=e.pageY,i.cursorAt&&this._adjustOffsetFromHelper(i.cursorAt),this._setContainment(),this._trigger("start",e)===!1?(this._clear(),!1):(this._cacheHelperProportions(),t.ui.ddmanager&&!i.dropBehaviour&&t.ui.ddmanager.prepareOffsets(this,e),this._mouseDrag(e,!0),t.ui.ddmanager&&t.ui.ddmanager.dragStart(this,e),!0)},_refreshOffsets:function(t){this.offset={top:this.positionAbs.top-this.margins.top,left:this.positionAbs.left-this.margins.left,scroll:!1,parent:this._getParentOffset(),relative:this._getRelativeOffset()},this.offset.click={left:t.pageX-this.offset.left,top:t.pageY-this.offset.top}},_mouseDrag:function(e,i){if(this.hasFixedAncestor&&(this.offset.parent=this._getParentOffset()),this.position=this._generatePosition(e,!0),this.positionAbs=this._convertPositionTo("absolute"),!i){var s=this._uiHash();if(this._trigger("drag",e,s)===!1)return this._mouseUp(new t.Event("mouseup",e)),!1;this.position=s.position}return this.helper[0].style.left=this.position.left+"px",this.helper[0].style.top=this.position.top+"px",t.ui.ddmanager&&t.ui.ddmanager.drag(this,e),!1},_mouseStop:function(e){var i=this,s=!1;return t.ui.ddmanager&&!this.options.dropBehaviour&&(s=t.ui.ddmanager.drop(this,e)),this.dropped&&(s=this.dropped,this.dropped=!1),"invalid"===this.options.revert&&!s||"valid"===this.options.revert&&s||this.options.revert===!0||t.isFunction(this.options.revert)&&this.options.revert.call(this.element,s)?t(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){i._trigger("stop",e)!==!1&&i._clear()}):this._trigger("stop",e)!==!1&&this._clear(),!1},_mouseUp:function(e){return this._unblockFrames(),t.ui.ddmanager&&t.ui.ddmanager.dragStop(this,e),this.handleElement.is(e.target)&&this.element.trigger("focus"),t.ui.mouse.prototype._mouseUp.call(this,e)},cancel:function(){return this.helper.is(".ui-draggable-dragging")?this._mouseUp(new t.Event("mouseup",{target:this.element[0]})):this._clear(),this},_getHandle:function(e){return this.options.handle?!!t(e.target).closest(this.element.find(this.options.handle)).length:!0},_setHandleClassName:function(){this.handleElement=this.options.handle?this.element.find(this.options.handle):this.element,this._addClass(this.handleElement,"ui-draggable-handle")},_removeHandleClassName:function(){this._removeClass(this.handleElement,"ui-draggable-handle")},_createHelper:function(e){var i=this.options,s=t.isFunction(i.helper),n=s?t(i.helper.apply(this.element[0],[e])):"clone"===i.helper?this.element.clone().removeAttr("id"):this.element;return n.parents("body").length||n.appendTo("parent"===i.appendTo?this.element[0].parentNode:i.appendTo),s&&n[0]===this.element[0]&&this._setPositionRelative(),n[0]===this.element[0]||/(fixed|absolute)/.test(n.css("position"))||n.css("position","absolute"),n},_setPositionRelative:function(){/^(?:r|a|f)/.test(this.element.css("position"))||(this.element[0].style.position="relative")},_adjustOffsetFromHelper:function(e){"string"==typeof e&&(e=e.split(" ")),t.isArray(e)&&(e={left:+e[0],top:+e[1]||0}),"left"in e&&(this.offset.click.left=e.left+this.margins.left),"right"in e&&(this.offset.click.left=this.helperProportions.width-e.right+this.margins.left),"top"in e&&(this.offset.click.top=e.top+this.margins.top),"bottom"in e&&(this.offset.click.top=this.helperProportions.height-e.bottom+this.margins.top)},_isRootNode:function(t){return/(html|body)/i.test(t.tagName)||t===this.document[0]},_getParentOffset:function(){var e=this.offsetParent.offset(),i=this.document[0];return"absolute"===this.cssPosition&&this.scrollParent[0]!==i&&t.contains(this.scrollParent[0],this.offsetParent[0])&&(e.left+=this.scrollParent.scrollLeft(),e.top+=this.scrollParent.scrollTop()),this._isRootNode(this.offsetParent[0])&&(e={top:0,left:0}),{top:e.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:e.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if("relative"!==this.cssPosition)return{top:0,left:0};var t=this.element.position(),e=this._isRootNode(this.scrollParent[0]);return{top:t.top-(parseInt(this.helper.css("top"),10)||0)+(e?0:this.scrollParent.scrollTop()),left:t.left-(parseInt(this.helper.css("left"),10)||0)+(e?0:this.scrollParent.scrollLeft())}
-},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0,right:parseInt(this.element.css("marginRight"),10)||0,bottom:parseInt(this.element.css("marginBottom"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var e,i,s,n=this.options,o=this.document[0];return this.relativeContainer=null,n.containment?"window"===n.containment?(this.containment=[t(window).scrollLeft()-this.offset.relative.left-this.offset.parent.left,t(window).scrollTop()-this.offset.relative.top-this.offset.parent.top,t(window).scrollLeft()+t(window).width()-this.helperProportions.width-this.margins.left,t(window).scrollTop()+(t(window).height()||o.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top],void 0):"document"===n.containment?(this.containment=[0,0,t(o).width()-this.helperProportions.width-this.margins.left,(t(o).height()||o.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top],void 0):n.containment.constructor===Array?(this.containment=n.containment,void 0):("parent"===n.containment&&(n.containment=this.helper[0].parentNode),i=t(n.containment),s=i[0],s&&(e=/(scroll|auto)/.test(i.css("overflow")),this.containment=[(parseInt(i.css("borderLeftWidth"),10)||0)+(parseInt(i.css("paddingLeft"),10)||0),(parseInt(i.css("borderTopWidth"),10)||0)+(parseInt(i.css("paddingTop"),10)||0),(e?Math.max(s.scrollWidth,s.offsetWidth):s.offsetWidth)-(parseInt(i.css("borderRightWidth"),10)||0)-(parseInt(i.css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left-this.margins.right,(e?Math.max(s.scrollHeight,s.offsetHeight):s.offsetHeight)-(parseInt(i.css("borderBottomWidth"),10)||0)-(parseInt(i.css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top-this.margins.bottom],this.relativeContainer=i),void 0):(this.containment=null,void 0)},_convertPositionTo:function(t,e){e||(e=this.position);var i="absolute"===t?1:-1,s=this._isRootNode(this.scrollParent[0]);return{top:e.top+this.offset.relative.top*i+this.offset.parent.top*i-("fixed"===this.cssPosition?-this.offset.scroll.top:s?0:this.offset.scroll.top)*i,left:e.left+this.offset.relative.left*i+this.offset.parent.left*i-("fixed"===this.cssPosition?-this.offset.scroll.left:s?0:this.offset.scroll.left)*i}},_generatePosition:function(t,e){var i,s,n,o,a=this.options,r=this._isRootNode(this.scrollParent[0]),h=t.pageX,l=t.pageY;return r&&this.offset.scroll||(this.offset.scroll={top:this.scrollParent.scrollTop(),left:this.scrollParent.scrollLeft()}),e&&(this.containment&&(this.relativeContainer?(s=this.relativeContainer.offset(),i=[this.containment[0]+s.left,this.containment[1]+s.top,this.containment[2]+s.left,this.containment[3]+s.top]):i=this.containment,t.pageX-this.offset.click.left<i[0]&&(h=i[0]+this.offset.click.left),t.pageY-this.offset.click.top<i[1]&&(l=i[1]+this.offset.click.top),t.pageX-this.offset.click.left>i[2]&&(h=i[2]+this.offset.click.left),t.pageY-this.offset.click.top>i[3]&&(l=i[3]+this.offset.click.top)),a.grid&&(n=a.grid[1]?this.originalPageY+Math.round((l-this.originalPageY)/a.grid[1])*a.grid[1]:this.originalPageY,l=i?n-this.offset.click.top>=i[1]||n-this.offset.click.top>i[3]?n:n-this.offset.click.top>=i[1]?n-a.grid[1]:n+a.grid[1]:n,o=a.grid[0]?this.originalPageX+Math.round((h-this.originalPageX)/a.grid[0])*a.grid[0]:this.originalPageX,h=i?o-this.offset.click.left>=i[0]||o-this.offset.click.left>i[2]?o:o-this.offset.click.left>=i[0]?o-a.grid[0]:o+a.grid[0]:o),"y"===a.axis&&(h=this.originalPageX),"x"===a.axis&&(l=this.originalPageY)),{top:l-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+("fixed"===this.cssPosition?-this.offset.scroll.top:r?0:this.offset.scroll.top),left:h-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+("fixed"===this.cssPosition?-this.offset.scroll.left:r?0:this.offset.scroll.left)}},_clear:function(){this._removeClass(this.helper,"ui-draggable-dragging"),this.helper[0]===this.element[0]||this.cancelHelperRemoval||this.helper.remove(),this.helper=null,this.cancelHelperRemoval=!1,this.destroyOnClear&&this.destroy()},_trigger:function(e,i,s){return s=s||this._uiHash(),t.ui.plugin.call(this,e,[i,s,this],!0),/^(drag|start|stop)/.test(e)&&(this.positionAbs=this._convertPositionTo("absolute"),s.offset=this.positionAbs),t.Widget.prototype._trigger.call(this,e,i,s)},plugins:{},_uiHash:function(){return{helper:this.helper,position:this.position,originalPosition:this.originalPosition,offset:this.positionAbs}}}),t.ui.plugin.add("draggable","connectToSortable",{start:function(e,i,s){var n=t.extend({},i,{item:s.element});s.sortables=[],t(s.options.connectToSortable).each(function(){var i=t(this).sortable("instance");i&&!i.options.disabled&&(s.sortables.push(i),i.refreshPositions(),i._trigger("activate",e,n))})},stop:function(e,i,s){var n=t.extend({},i,{item:s.element});s.cancelHelperRemoval=!1,t.each(s.sortables,function(){var t=this;t.isOver?(t.isOver=0,s.cancelHelperRemoval=!0,t.cancelHelperRemoval=!1,t._storedCSS={position:t.placeholder.css("position"),top:t.placeholder.css("top"),left:t.placeholder.css("left")},t._mouseStop(e),t.options.helper=t.options._helper):(t.cancelHelperRemoval=!0,t._trigger("deactivate",e,n))})},drag:function(e,i,s){t.each(s.sortables,function(){var n=!1,o=this;o.positionAbs=s.positionAbs,o.helperProportions=s.helperProportions,o.offset.click=s.offset.click,o._intersectsWith(o.containerCache)&&(n=!0,t.each(s.sortables,function(){return this.positionAbs=s.positionAbs,this.helperProportions=s.helperProportions,this.offset.click=s.offset.click,this!==o&&this._intersectsWith(this.containerCache)&&t.contains(o.element[0],this.element[0])&&(n=!1),n})),n?(o.isOver||(o.isOver=1,s._parent=i.helper.parent(),o.currentItem=i.helper.appendTo(o.element).data("ui-sortable-item",!0),o.options._helper=o.options.helper,o.options.helper=function(){return i.helper[0]},e.target=o.currentItem[0],o._mouseCapture(e,!0),o._mouseStart(e,!0,!0),o.offset.click.top=s.offset.click.top,o.offset.click.left=s.offset.click.left,o.offset.parent.left-=s.offset.parent.left-o.offset.parent.left,o.offset.parent.top-=s.offset.parent.top-o.offset.parent.top,s._trigger("toSortable",e),s.dropped=o.element,t.each(s.sortables,function(){this.refreshPositions()}),s.currentItem=s.element,o.fromOutside=s),o.currentItem&&(o._mouseDrag(e),i.position=o.position)):o.isOver&&(o.isOver=0,o.cancelHelperRemoval=!0,o.options._revert=o.options.revert,o.options.revert=!1,o._trigger("out",e,o._uiHash(o)),o._mouseStop(e,!0),o.options.revert=o.options._revert,o.options.helper=o.options._helper,o.placeholder&&o.placeholder.remove(),i.helper.appendTo(s._parent),s._refreshOffsets(e),i.position=s._generatePosition(e,!0),s._trigger("fromSortable",e),s.dropped=!1,t.each(s.sortables,function(){this.refreshPositions()}))})}}),t.ui.plugin.add("draggable","cursor",{start:function(e,i,s){var n=t("body"),o=s.options;n.css("cursor")&&(o._cursor=n.css("cursor")),n.css("cursor",o.cursor)},stop:function(e,i,s){var n=s.options;n._cursor&&t("body").css("cursor",n._cursor)}}),t.ui.plugin.add("draggable","opacity",{start:function(e,i,s){var n=t(i.helper),o=s.options;n.css("opacity")&&(o._opacity=n.css("opacity")),n.css("opacity",o.opacity)},stop:function(e,i,s){var n=s.options;n._opacity&&t(i.helper).css("opacity",n._opacity)}}),t.ui.plugin.add("draggable","scroll",{start:function(t,e,i){i.scrollParentNotHidden||(i.scrollParentNotHidden=i.helper.scrollParent(!1)),i.scrollParentNotHidden[0]!==i.document[0]&&"HTML"!==i.scrollParentNotHidden[0].tagName&&(i.overflowOffset=i.scrollParentNotHidden.offset())},drag:function(e,i,s){var n=s.options,o=!1,a=s.scrollParentNotHidden[0],r=s.document[0];a!==r&&"HTML"!==a.tagName?(n.axis&&"x"===n.axis||(s.overflowOffset.top+a.offsetHeight-e.pageY<n.scrollSensitivity?a.scrollTop=o=a.scrollTop+n.scrollSpeed:e.pageY-s.overflowOffset.top<n.scrollSensitivity&&(a.scrollTop=o=a.scrollTop-n.scrollSpeed)),n.axis&&"y"===n.axis||(s.overflowOffset.left+a.offsetWidth-e.pageX<n.scrollSensitivity?a.scrollLeft=o=a.scrollLeft+n.scrollSpeed:e.pageX-s.overflowOffset.left<n.scrollSensitivity&&(a.scrollLeft=o=a.scrollLeft-n.scrollSpeed))):(n.axis&&"x"===n.axis||(e.pageY-t(r).scrollTop()<n.scrollSensitivity?o=t(r).scrollTop(t(r).scrollTop()-n.scrollSpeed):t(window).height()-(e.pageY-t(r).scrollTop())<n.scrollSensitivity&&(o=t(r).scrollTop(t(r).scrollTop()+n.scrollSpeed))),n.axis&&"y"===n.axis||(e.pageX-t(r).scrollLeft()<n.scrollSensitivity?o=t(r).scrollLeft(t(r).scrollLeft()-n.scrollSpeed):t(window).width()-(e.pageX-t(r).scrollLeft())<n.scrollSensitivity&&(o=t(r).scrollLeft(t(r).scrollLeft()+n.scrollSpeed)))),o!==!1&&t.ui.ddmanager&&!n.dropBehaviour&&t.ui.ddmanager.prepareOffsets(s,e)}}),t.ui.plugin.add("draggable","snap",{start:function(e,i,s){var n=s.options;s.snapElements=[],t(n.snap.constructor!==String?n.snap.items||":data(ui-draggable)":n.snap).each(function(){var e=t(this),i=e.offset();this!==s.element[0]&&s.snapElements.push({item:this,width:e.outerWidth(),height:e.outerHeight(),top:i.top,left:i.left})})},drag:function(e,i,s){var n,o,a,r,h,l,c,u,d,p,f=s.options,g=f.snapTolerance,m=i.offset.left,_=m+s.helperProportions.width,v=i.offset.top,b=v+s.helperProportions.height;for(d=s.snapElements.length-1;d>=0;d--)h=s.snapElements[d].left-s.margins.left,l=h+s.snapElements[d].width,c=s.snapElements[d].top-s.margins.top,u=c+s.snapElements[d].height,h-g>_||m>l+g||c-g>b||v>u+g||!t.contains(s.snapElements[d].item.ownerDocument,s.snapElements[d].item)?(s.snapElements[d].snapping&&s.options.snap.release&&s.options.snap.release.call(s.element,e,t.extend(s._uiHash(),{snapItem:s.snapElements[d].item})),s.snapElements[d].snapping=!1):("inner"!==f.snapMode&&(n=g>=Math.abs(c-b),o=g>=Math.abs(u-v),a=g>=Math.abs(h-_),r=g>=Math.abs(l-m),n&&(i.position.top=s._convertPositionTo("relative",{top:c-s.helperProportions.height,left:0}).top),o&&(i.position.top=s._convertPositionTo("relative",{top:u,left:0}).top),a&&(i.position.left=s._convertPositionTo("relative",{top:0,left:h-s.helperProportions.width}).left),r&&(i.position.left=s._convertPositionTo("relative",{top:0,left:l}).left)),p=n||o||a||r,"outer"!==f.snapMode&&(n=g>=Math.abs(c-v),o=g>=Math.abs(u-b),a=g>=Math.abs(h-m),r=g>=Math.abs(l-_),n&&(i.position.top=s._convertPositionTo("relative",{top:c,left:0}).top),o&&(i.position.top=s._convertPositionTo("relative",{top:u-s.helperProportions.height,left:0}).top),a&&(i.position.left=s._convertPositionTo("relative",{top:0,left:h}).left),r&&(i.position.left=s._convertPositionTo("relative",{top:0,left:l-s.helperProportions.width}).left)),!s.snapElements[d].snapping&&(n||o||a||r||p)&&s.options.snap.snap&&s.options.snap.snap.call(s.element,e,t.extend(s._uiHash(),{snapItem:s.snapElements[d].item})),s.snapElements[d].snapping=n||o||a||r||p)}}),t.ui.plugin.add("draggable","stack",{start:function(e,i,s){var n,o=s.options,a=t.makeArray(t(o.stack)).sort(function(e,i){return(parseInt(t(e).css("zIndex"),10)||0)-(parseInt(t(i).css("zIndex"),10)||0)});a.length&&(n=parseInt(t(a[0]).css("zIndex"),10)||0,t(a).each(function(e){t(this).css("zIndex",n+e)}),this.css("zIndex",n+a.length))}}),t.ui.plugin.add("draggable","zIndex",{start:function(e,i,s){var n=t(i.helper),o=s.options;n.css("zIndex")&&(o._zIndex=n.css("zIndex")),n.css("zIndex",o.zIndex)},stop:function(e,i,s){var n=s.options;n._zIndex&&t(i.helper).css("zIndex",n._zIndex)}}),t.ui.draggable,t.widget("ui.resizable",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,classes:{"ui-resizable-se":"ui-icon ui-icon-gripsmall-diagonal-se"},containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:90,resize:null,start:null,stop:null},_num:function(t){return parseFloat(t)||0},_isNumber:function(t){return!isNaN(parseFloat(t))},_hasScroll:function(e,i){if("hidden"===t(e).css("overflow"))return!1;var s=i&&"left"===i?"scrollLeft":"scrollTop",n=!1;return e[s]>0?!0:(e[s]=1,n=e[s]>0,e[s]=0,n)},_create:function(){var e,i=this.options,s=this;this._addClass("ui-resizable"),t.extend(this,{_aspectRatio:!!i.aspectRatio,aspectRatio:i.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:i.helper||i.ghost||i.animate?i.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/^(canvas|textarea|input|select|button|img)$/i)&&(this.element.wrap(t("<div class='ui-wrapper' style='overflow: hidden;'></div>").css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("ui-resizable",this.element.resizable("instance")),this.elementIsWrapper=!0,e={marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom"),marginLeft:this.originalElement.css("marginLeft")},this.element.css(e),this.originalElement.css("margin",0),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css(e),this._proportionallyResize()),this._setupHandles(),i.autoHide&&t(this.element).on("mouseenter",function(){i.disabled||(s._removeClass("ui-resizable-autohide"),s._handles.show())}).on("mouseleave",function(){i.disabled||s.resizing||(s._addClass("ui-resizable-autohide"),s._handles.hide())}),this._mouseInit()},_destroy:function(){this._mouseDestroy();var e,i=function(e){t(e).removeData("resizable").removeData("ui-resizable").off(".resizable").find(".ui-resizable-handle").remove()};return this.elementIsWrapper&&(i(this.element),e=this.element,this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")}).insertAfter(e),e.remove()),this.originalElement.css("resize",this.originalResizeStyle),i(this.originalElement),this},_setOption:function(t,e){switch(this._super(t,e),t){case"handles":this._removeHandles(),this._setupHandles();break;default:}},_setupHandles:function(){var e,i,s,n,o,a=this.options,r=this;if(this.handles=a.handles||(t(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se"),this._handles=t(),this.handles.constructor===String)for("all"===this.handles&&(this.handles="n,e,s,w,se,sw,ne,nw"),s=this.handles.split(","),this.handles={},i=0;s.length>i;i++)e=t.trim(s[i]),n="ui-resizable-"+e,o=t("<div>"),this._addClass(o,"ui-resizable-handle "+n),o.css({zIndex:a.zIndex}),this.handles[e]=".ui-resizable-"+e,this.element.append(o);this._renderAxis=function(e){var i,s,n,o;e=e||this.element;for(i in this.handles)this.handles[i].constructor===String?this.handles[i]=this.element.children(this.handles[i]).first().show():(this.handles[i].jquery||this.handles[i].nodeType)&&(this.handles[i]=t(this.handles[i]),this._on(this.handles[i],{mousedown:r._mouseDown})),this.elementIsWrapper&&this.originalElement[0].nodeName.match(/^(textarea|input|select|button)$/i)&&(s=t(this.handles[i],this.element),o=/sw|ne|nw|se|n|s/.test(i)?s.outerHeight():s.outerWidth(),n=["padding",/ne|nw|n/.test(i)?"Top":/se|sw|s/.test(i)?"Bottom":/^e$/.test(i)?"Right":"Left"].join(""),e.css(n,o),this._proportionallyResize()),this._handles=this._handles.add(this.handles[i])},this._renderAxis(this.element),this._handles=this._handles.add(this.element.find(".ui-resizable-handle")),this._handles.disableSelection(),this._handles.on("mouseover",function(){r.resizing||(this.className&&(o=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)),r.axis=o&&o[1]?o[1]:"se")}),a.autoHide&&(this._handles.hide(),this._addClass("ui-resizable-autohide"))},_removeHandles:function(){this._handles.remove()},_mouseCapture:function(e){var i,s,n=!1;for(i in this.handles)s=t(this.handles[i])[0],(s===e.target||t.contains(s,e.target))&&(n=!0);return!this.options.disabled&&n},_mouseStart:function(e){var i,s,n,o=this.options,a=this.element;return this.resizing=!0,this._renderProxy(),i=this._num(this.helper.css("left")),s=this._num(this.helper.css("top")),o.containment&&(i+=t(o.containment).scrollLeft()||0,s+=t(o.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:i,top:s},this.size=this._helper?{width:this.helper.width(),height:this.helper.height()}:{width:a.width(),height:a.height()},this.originalSize=this._helper?{width:a.outerWidth(),height:a.outerHeight()}:{width:a.width(),height:a.height()},this.sizeDiff={width:a.outerWidth()-a.width(),height:a.outerHeight()-a.height()},this.originalPosition={left:i,top:s},this.originalMousePosition={left:e.pageX,top:e.pageY},this.aspectRatio="number"==typeof o.aspectRatio?o.aspectRatio:this.originalSize.width/this.originalSize.height||1,n=t(".ui-resizable-"+this.axis).css("cursor"),t("body").css("cursor","auto"===n?this.axis+"-resize":n),this._addClass("ui-resizable-resizing"),this._propagate("start",e),!0},_mouseDrag:function(e){var i,s,n=this.originalMousePosition,o=this.axis,a=e.pageX-n.left||0,r=e.pageY-n.top||0,h=this._change[o];return this._updatePrevProperties(),h?(i=h.apply(this,[e,a,r]),this._updateVirtualBoundaries(e.shiftKey),(this._aspectRatio||e.shiftKey)&&(i=this._updateRatio(i,e)),i=this._respectSize(i,e),this._updateCache(i),this._propagate("resize",e),s=this._applyChanges(),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),t.isEmptyObject(s)||(this._updatePrevProperties(),this._trigger("resize",e,this.ui()),this._applyChanges()),!1):!1},_mouseStop:function(e){this.resizing=!1;var i,s,n,o,a,r,h,l=this.options,c=this;return this._helper&&(i=this._proportionallyResizeElements,s=i.length&&/textarea/i.test(i[0].nodeName),n=s&&this._hasScroll(i[0],"left")?0:c.sizeDiff.height,o=s?0:c.sizeDiff.width,a={width:c.helper.width()-o,height:c.helper.height()-n},r=parseFloat(c.element.css("left"))+(c.position.left-c.originalPosition.left)||null,h=parseFloat(c.element.css("top"))+(c.position.top-c.originalPosition.top)||null,l.animate||this.element.css(t.extend(a,{top:h,left:r})),c.helper.height(c.size.height),c.helper.width(c.size.width),this._helper&&!l.animate&&this._proportionallyResize()),t("body").css("cursor","auto"),this._removeClass("ui-resizable-resizing"),this._propagate("stop",e),this._helper&&this.helper.remove(),!1},_updatePrevProperties:function(){this.prevPosition={top:this.position.top,left:this.position.left},this.prevSize={width:this.size.width,height:this.size.height}},_applyChanges:function(){var t={};return this.position.top!==this.prevPosition.top&&(t.top=this.position.top+"px"),this.position.left!==this.prevPosition.left&&(t.left=this.position.left+"px"),this.size.width!==this.prevSize.width&&(t.width=this.size.width+"px"),this.size.height!==this.prevSize.height&&(t.height=this.size.height+"px"),this.helper.css(t),t},_updateVirtualBoundaries:function(t){var e,i,s,n,o,a=this.options;o={minWidth:this._isNumber(a.minWidth)?a.minWidth:0,maxWidth:this._isNumber(a.maxWidth)?a.maxWidth:1/0,minHeight:this._isNumber(a.minHeight)?a.minHeight:0,maxHeight:this._isNumber(a.maxHeight)?a.maxHeight:1/0},(this._aspectRatio||t)&&(e=o.minHeight*this.aspectRatio,s=o.minWidth/this.aspectRatio,i=o.maxHeight*this.aspectRatio,n=o.maxWidth/this.aspectRatio,e>o.minWidth&&(o.minWidth=e),s>o.minHeight&&(o.minHeight=s),o.maxWidth>i&&(o.maxWidth=i),o.maxHeight>n&&(o.maxHeight=n)),this._vBoundaries=o},_updateCache:function(t){this.offset=this.helper.offset(),this._isNumber(t.left)&&(this.position.left=t.left),this._isNumber(t.top)&&(this.position.top=t.top),this._isNumber(t.height)&&(this.size.height=t.height),this._isNumber(t.width)&&(this.size.width=t.width)},_updateRatio:function(t){var e=this.position,i=this.size,s=this.axis;return this._isNumber(t.height)?t.width=t.height*this.aspectRatio:this._isNumber(t.width)&&(t.height=t.width/this.aspectRatio),"sw"===s&&(t.left=e.left+(i.width-t.width),t.top=null),"nw"===s&&(t.top=e.top+(i.height-t.height),t.left=e.left+(i.width-t.width)),t},_respectSize:function(t){var e=this._vBoundaries,i=this.axis,s=this._isNumber(t.width)&&e.maxWidth&&e.maxWidth<t.width,n=this._isNumber(t.height)&&e.maxHeight&&e.maxHeight<t.height,o=this._isNumber(t.width)&&e.minWidth&&e.minWidth>t.width,a=this._isNumber(t.height)&&e.minHeight&&e.minHeight>t.height,r=this.originalPosition.left+this.originalSize.width,h=this.originalPosition.top+this.originalSize.height,l=/sw|nw|w/.test(i),c=/nw|ne|n/.test(i);return o&&(t.width=e.minWidth),a&&(t.height=e.minHeight),s&&(t.width=e.maxWidth),n&&(t.height=e.maxHeight),o&&l&&(t.left=r-e.minWidth),s&&l&&(t.left=r-e.maxWidth),a&&c&&(t.top=h-e.minHeight),n&&c&&(t.top=h-e.maxHeight),t.width||t.height||t.left||!t.top?t.width||t.height||t.top||!t.left||(t.left=null):t.top=null,t},_getPaddingPlusBorderDimensions:function(t){for(var e=0,i=[],s=[t.css("borderTopWidth"),t.css("borderRightWidth"),t.css("borderBottomWidth"),t.css("borderLeftWidth")],n=[t.css("paddingTop"),t.css("paddingRight"),t.css("paddingBottom"),t.css("paddingLeft")];4>e;e++)i[e]=parseFloat(s[e])||0,i[e]+=parseFloat(n[e])||0;return{height:i[0]+i[2],width:i[1]+i[3]}},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var t,e=0,i=this.helper||this.element;this._proportionallyResizeElements.length>e;e++)t=this._proportionallyResizeElements[e],this.outerDimensions||(this.outerDimensions=this._getPaddingPlusBorderDimensions(t)),t.css({height:i.height()-this.outerDimensions.height||0,width:i.width()-this.outerDimensions.width||0})},_renderProxy:function(){var e=this.element,i=this.options;this.elementOffset=e.offset(),this._helper?(this.helper=this.helper||t("<div style='overflow:hidden;'></div>"),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++i.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize,s=this.originalPosition;return{left:s.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize,n=this.originalPosition;return{top:n.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},sw:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[e,i,s]))},ne:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},nw:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[e,i,s]))}},_propagate:function(e,i){t.ui.plugin.call(this,e,[i,this.ui()]),"resize"!==e&&this._trigger(e,i,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),t.ui.plugin.add("resizable","animate",{stop:function(e){var i=t(this).resizable("instance"),s=i.options,n=i._proportionallyResizeElements,o=n.length&&/textarea/i.test(n[0].nodeName),a=o&&i._hasScroll(n[0],"left")?0:i.sizeDiff.height,r=o?0:i.sizeDiff.width,h={width:i.size.width-r,height:i.size.height-a},l=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,c=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(t.extend(h,c&&l?{top:c,left:l}:{}),{duration:s.animateDuration,easing:s.animateEasing,step:function(){var s={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};n&&n.length&&t(n[0]).css({width:s.width,height:s.height}),i._updateCache(s),i._propagate("resize",e)}})}}),t.ui.plugin.add("resizable","containment",{start:function(){var e,i,s,n,o,a,r,h=t(this).resizable("instance"),l=h.options,c=h.element,u=l.containment,d=u instanceof t?u.get(0):/parent/.test(u)?c.parent().get(0):u;d&&(h.containerElement=t(d),/document/.test(u)||u===document?(h.containerOffset={left:0,top:0},h.containerPosition={left:0,top:0},h.parentData={element:t(document),left:0,top:0,width:t(document).width(),height:t(document).height()||document.body.parentNode.scrollHeight}):(e=t(d),i=[],t(["Top","Right","Left","Bottom"]).each(function(t,s){i[t]=h._num(e.css("padding"+s))}),h.containerOffset=e.offset(),h.containerPosition=e.position(),h.containerSize={height:e.innerHeight()-i[3],width:e.innerWidth()-i[1]},s=h.containerOffset,n=h.containerSize.height,o=h.containerSize.width,a=h._hasScroll(d,"left")?d.scrollWidth:o,r=h._hasScroll(d)?d.scrollHeight:n,h.parentData={element:d,left:s.left,top:s.top,width:a,height:r}))},resize:function(e){var i,s,n,o,a=t(this).resizable("instance"),r=a.options,h=a.containerOffset,l=a.position,c=a._aspectRatio||e.shiftKey,u={top:0,left:0},d=a.containerElement,p=!0;d[0]!==document&&/static/.test(d.css("position"))&&(u=h),l.left<(a._helper?h.left:0)&&(a.size.width=a.size.width+(a._helper?a.position.left-h.left:a.position.left-u.left),c&&(a.size.height=a.size.width/a.aspectRatio,p=!1),a.position.left=r.helper?h.left:0),l.top<(a._helper?h.top:0)&&(a.size.height=a.size.height+(a._helper?a.position.top-h.top:a.position.top),c&&(a.size.width=a.size.height*a.aspectRatio,p=!1),a.position.top=a._helper?h.top:0),n=a.containerElement.get(0)===a.element.parent().get(0),o=/relative|absolute/.test(a.containerElement.css("position")),n&&o?(a.offset.left=a.parentData.left+a.position.left,a.offset.top=a.parentData.top+a.position.top):(a.offset.left=a.element.offset().left,a.offset.top=a.element.offset().top),i=Math.abs(a.sizeDiff.width+(a._helper?a.offset.left-u.left:a.offset.left-h.left)),s=Math.abs(a.sizeDiff.height+(a._helper?a.offset.top-u.top:a.offset.top-h.top)),i+a.size.width>=a.parentData.width&&(a.size.width=a.parentData.width-i,c&&(a.size.height=a.size.width/a.aspectRatio,p=!1)),s+a.size.height>=a.parentData.height&&(a.size.height=a.parentData.height-s,c&&(a.size.width=a.size.height*a.aspectRatio,p=!1)),p||(a.position.left=a.prevPosition.left,a.position.top=a.prevPosition.top,a.size.width=a.prevSize.width,a.size.height=a.prevSize.height)},stop:function(){var e=t(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.containerPosition,o=e.containerElement,a=t(e.helper),r=a.offset(),h=a.outerWidth()-e.sizeDiff.width,l=a.outerHeight()-e.sizeDiff.height;e._helper&&!i.animate&&/relative/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l}),e._helper&&!i.animate&&/static/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l})}}),t.ui.plugin.add("resizable","alsoResize",{start:function(){var e=t(this).resizable("instance"),i=e.options;t(i.alsoResize).each(function(){var e=t(this);e.data("ui-resizable-alsoresize",{width:parseFloat(e.width()),height:parseFloat(e.height()),left:parseFloat(e.css("left")),top:parseFloat(e.css("top"))})})},resize:function(e,i){var s=t(this).resizable("instance"),n=s.options,o=s.originalSize,a=s.originalPosition,r={height:s.size.height-o.height||0,width:s.size.width-o.width||0,top:s.position.top-a.top||0,left:s.position.left-a.left||0};t(n.alsoResize).each(function(){var e=t(this),s=t(this).data("ui-resizable-alsoresize"),n={},o=e.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];t.each(o,function(t,e){var i=(s[e]||0)+(r[e]||0);i&&i>=0&&(n[e]=i||null)}),e.css(n)})},stop:function(){t(this).removeData("ui-resizable-alsoresize")}}),t.ui.plugin.add("resizable","ghost",{start:function(){var e=t(this).resizable("instance"),i=e.size;e.ghost=e.originalElement.clone(),e.ghost.css({opacity:.25,display:"block",position:"relative",height:i.height,width:i.width,margin:0,left:0,top:0}),e._addClass(e.ghost,"ui-resizable-ghost"),t.uiBackCompat!==!1&&"string"==typeof e.options.ghost&&e.ghost.addClass(this.options.ghost),e.ghost.appendTo(e.helper)},resize:function(){var e=t(this).resizable("instance");e.ghost&&e.ghost.css({position:"relative",height:e.size.height,width:e.size.width})},stop:function(){var e=t(this).resizable("instance");e.ghost&&e.helper&&e.helper.get(0).removeChild(e.ghost.get(0))}}),t.ui.plugin.add("resizable","grid",{resize:function(){var e,i=t(this).resizable("instance"),s=i.options,n=i.size,o=i.originalSize,a=i.originalPosition,r=i.axis,h="number"==typeof s.grid?[s.grid,s.grid]:s.grid,l=h[0]||1,c=h[1]||1,u=Math.round((n.width-o.width)/l)*l,d=Math.round((n.height-o.height)/c)*c,p=o.width+u,f=o.height+d,g=s.maxWidth&&p>s.maxWidth,m=s.maxHeight&&f>s.maxHeight,_=s.minWidth&&s.minWidth>p,v=s.minHeight&&s.minHeight>f;s.grid=h,_&&(p+=l),v&&(f+=c),g&&(p-=l),m&&(f-=c),/^(se|s|e)$/.test(r)?(i.size.width=p,i.size.height=f):/^(ne)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.top=a.top-d):/^(sw)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.left=a.left-u):((0>=f-c||0>=p-l)&&(e=i._getPaddingPlusBorderDimensions(this)),f-c>0?(i.size.height=f,i.position.top=a.top-d):(f=c-e.height,i.size.height=f,i.position.top=a.top+o.height-f),p-l>0?(i.size.width=p,i.position.left=a.left-u):(p=l-e.width,i.size.width=p,i.position.left=a.left+o.width-p))}}),t.ui.resizable,t.widget("ui.dialog",{version:"1.12.1",options:{appendTo:"body",autoOpen:!0,buttons:[],classes:{"ui-dialog":"ui-corner-all","ui-dialog-titlebar":"ui-corner-all"},closeOnEscape:!0,closeText:"Close",draggable:!0,hide:null,height:"auto",maxHeight:null,maxWidth:null,minHeight:150,minWidth:150,modal:!1,position:{my:"center",at:"center",of:window,collision:"fit",using:function(e){var i=t(this).css(e).offset().top;0>i&&t(this).css("top",e.top-i)}},resizable:!0,show:null,title:null,width:300,beforeClose:null,close:null,drag:null,dragStart:null,dragStop:null,focus:null,open:null,resize:null,resizeStart:null,resizeStop:null},sizeRelatedOptions:{buttons:!0,height:!0,maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0,width:!0},resizableRelatedOptions:{maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0},_create:function(){this.originalCss={display:this.element[0].style.display,width:this.element[0].style.width,minHeight:this.element[0].style.minHeight,maxHeight:this.element[0].style.maxHeight,height:this.element[0].style.height},this.originalPosition={parent:this.element.parent(),index:this.element.parent().children().index(this.element)},this.originalTitle=this.element.attr("title"),null==this.options.title&&null!=this.originalTitle&&(this.options.title=this.originalTitle),this.options.disabled&&(this.options.disabled=!1),this._createWrapper(),this.element.show().removeAttr("title").appendTo(this.uiDialog),this._addClass("ui-dialog-content","ui-widget-content"),this._createTitlebar(),this._createButtonPane(),this.options.draggable&&t.fn.draggable&&this._makeDraggable(),this.options.resizable&&t.fn.resizable&&this._makeResizable(),this._isOpen=!1,this._trackFocus()},_init:function(){this.options.autoOpen&&this.open()},_appendTo:function(){var e=this.options.appendTo;return e&&(e.jquery||e.nodeType)?t(e):this.document.find(e||"body").eq(0)},_destroy:function(){var t,e=this.originalPosition;this._untrackInstance(),this._destroyOverlay(),this.element.removeUniqueId().css(this.originalCss).detach(),this.uiDialog.remove(),this.originalTitle&&this.element.attr("title",this.originalTitle),t=e.parent.children().eq(e.index),t.length&&t[0]!==this.element[0]?t.before(this.element):e.parent.append(this.element)},widget:function(){return this.uiDialog
-},disable:t.noop,enable:t.noop,close:function(e){var i=this;this._isOpen&&this._trigger("beforeClose",e)!==!1&&(this._isOpen=!1,this._focusedElement=null,this._destroyOverlay(),this._untrackInstance(),this.opener.filter(":focusable").trigger("focus").length||t.ui.safeBlur(t.ui.safeActiveElement(this.document[0])),this._hide(this.uiDialog,this.options.hide,function(){i._trigger("close",e)}))},isOpen:function(){return this._isOpen},moveToTop:function(){this._moveToTop()},_moveToTop:function(e,i){var s=!1,n=this.uiDialog.siblings(".ui-front:visible").map(function(){return+t(this).css("z-index")}).get(),o=Math.max.apply(null,n);return o>=+this.uiDialog.css("z-index")&&(this.uiDialog.css("z-index",o+1),s=!0),s&&!i&&this._trigger("focus",e),s},open:function(){var e=this;return this._isOpen?(this._moveToTop()&&this._focusTabbable(),void 0):(this._isOpen=!0,this.opener=t(t.ui.safeActiveElement(this.document[0])),this._size(),this._position(),this._createOverlay(),this._moveToTop(null,!0),this.overlay&&this.overlay.css("z-index",this.uiDialog.css("z-index")-1),this._show(this.uiDialog,this.options.show,function(){e._focusTabbable(),e._trigger("focus")}),this._makeFocusTarget(),this._trigger("open"),void 0)},_focusTabbable:function(){var t=this._focusedElement;t||(t=this.element.find("[autofocus]")),t.length||(t=this.element.find(":tabbable")),t.length||(t=this.uiDialogButtonPane.find(":tabbable")),t.length||(t=this.uiDialogTitlebarClose.filter(":tabbable")),t.length||(t=this.uiDialog),t.eq(0).trigger("focus")},_keepFocus:function(e){function i(){var e=t.ui.safeActiveElement(this.document[0]),i=this.uiDialog[0]===e||t.contains(this.uiDialog[0],e);i||this._focusTabbable()}e.preventDefault(),i.call(this),this._delay(i)},_createWrapper:function(){this.uiDialog=t("<div>").hide().attr({tabIndex:-1,role:"dialog"}).appendTo(this._appendTo()),this._addClass(this.uiDialog,"ui-dialog","ui-widget ui-widget-content ui-front"),this._on(this.uiDialog,{keydown:function(e){if(this.options.closeOnEscape&&!e.isDefaultPrevented()&&e.keyCode&&e.keyCode===t.ui.keyCode.ESCAPE)return e.preventDefault(),this.close(e),void 0;if(e.keyCode===t.ui.keyCode.TAB&&!e.isDefaultPrevented()){var i=this.uiDialog.find(":tabbable"),s=i.filter(":first"),n=i.filter(":last");e.target!==n[0]&&e.target!==this.uiDialog[0]||e.shiftKey?e.target!==s[0]&&e.target!==this.uiDialog[0]||!e.shiftKey||(this._delay(function(){n.trigger("focus")}),e.preventDefault()):(this._delay(function(){s.trigger("focus")}),e.preventDefault())}},mousedown:function(t){this._moveToTop(t)&&this._focusTabbable()}}),this.element.find("[aria-describedby]").length||this.uiDialog.attr({"aria-describedby":this.element.uniqueId().attr("id")})},_createTitlebar:function(){var e;this.uiDialogTitlebar=t("<div>"),this._addClass(this.uiDialogTitlebar,"ui-dialog-titlebar","ui-widget-header ui-helper-clearfix"),this._on(this.uiDialogTitlebar,{mousedown:function(e){t(e.target).closest(".ui-dialog-titlebar-close")||this.uiDialog.trigger("focus")}}),this.uiDialogTitlebarClose=t("<button type='button'></button>").button({label:t("<a>").text(this.options.closeText).html(),icon:"ui-icon-closethick",showLabel:!1}).appendTo(this.uiDialogTitlebar),this._addClass(this.uiDialogTitlebarClose,"ui-dialog-titlebar-close"),this._on(this.uiDialogTitlebarClose,{click:function(t){t.preventDefault(),this.close(t)}}),e=t("<span>").uniqueId().prependTo(this.uiDialogTitlebar),this._addClass(e,"ui-dialog-title"),this._title(e),this.uiDialogTitlebar.prependTo(this.uiDialog),this.uiDialog.attr({"aria-labelledby":e.attr("id")})},_title:function(t){this.options.title?t.text(this.options.title):t.html("&#160;")},_createButtonPane:function(){this.uiDialogButtonPane=t("<div>"),this._addClass(this.uiDialogButtonPane,"ui-dialog-buttonpane","ui-widget-content ui-helper-clearfix"),this.uiButtonSet=t("<div>").appendTo(this.uiDialogButtonPane),this._addClass(this.uiButtonSet,"ui-dialog-buttonset"),this._createButtons()},_createButtons:function(){var e=this,i=this.options.buttons;return this.uiDialogButtonPane.remove(),this.uiButtonSet.empty(),t.isEmptyObject(i)||t.isArray(i)&&!i.length?(this._removeClass(this.uiDialog,"ui-dialog-buttons"),void 0):(t.each(i,function(i,s){var n,o;s=t.isFunction(s)?{click:s,text:i}:s,s=t.extend({type:"button"},s),n=s.click,o={icon:s.icon,iconPosition:s.iconPosition,showLabel:s.showLabel,icons:s.icons,text:s.text},delete s.click,delete s.icon,delete s.iconPosition,delete s.showLabel,delete s.icons,"boolean"==typeof s.text&&delete s.text,t("<button></button>",s).button(o).appendTo(e.uiButtonSet).on("click",function(){n.apply(e.element[0],arguments)})}),this._addClass(this.uiDialog,"ui-dialog-buttons"),this.uiDialogButtonPane.appendTo(this.uiDialog),void 0)},_makeDraggable:function(){function e(t){return{position:t.position,offset:t.offset}}var i=this,s=this.options;this.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(s,n){i._addClass(t(this),"ui-dialog-dragging"),i._blockFrames(),i._trigger("dragStart",s,e(n))},drag:function(t,s){i._trigger("drag",t,e(s))},stop:function(n,o){var a=o.offset.left-i.document.scrollLeft(),r=o.offset.top-i.document.scrollTop();s.position={my:"left top",at:"left"+(a>=0?"+":"")+a+" "+"top"+(r>=0?"+":"")+r,of:i.window},i._removeClass(t(this),"ui-dialog-dragging"),i._unblockFrames(),i._trigger("dragStop",n,e(o))}})},_makeResizable:function(){function e(t){return{originalPosition:t.originalPosition,originalSize:t.originalSize,position:t.position,size:t.size}}var i=this,s=this.options,n=s.resizable,o=this.uiDialog.css("position"),a="string"==typeof n?n:"n,e,s,w,se,sw,ne,nw";this.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:this.element,maxWidth:s.maxWidth,maxHeight:s.maxHeight,minWidth:s.minWidth,minHeight:this._minHeight(),handles:a,start:function(s,n){i._addClass(t(this),"ui-dialog-resizing"),i._blockFrames(),i._trigger("resizeStart",s,e(n))},resize:function(t,s){i._trigger("resize",t,e(s))},stop:function(n,o){var a=i.uiDialog.offset(),r=a.left-i.document.scrollLeft(),h=a.top-i.document.scrollTop();s.height=i.uiDialog.height(),s.width=i.uiDialog.width(),s.position={my:"left top",at:"left"+(r>=0?"+":"")+r+" "+"top"+(h>=0?"+":"")+h,of:i.window},i._removeClass(t(this),"ui-dialog-resizing"),i._unblockFrames(),i._trigger("resizeStop",n,e(o))}}).css("position",o)},_trackFocus:function(){this._on(this.widget(),{focusin:function(e){this._makeFocusTarget(),this._focusedElement=t(e.target)}})},_makeFocusTarget:function(){this._untrackInstance(),this._trackingInstances().unshift(this)},_untrackInstance:function(){var e=this._trackingInstances(),i=t.inArray(this,e);-1!==i&&e.splice(i,1)},_trackingInstances:function(){var t=this.document.data("ui-dialog-instances");return t||(t=[],this.document.data("ui-dialog-instances",t)),t},_minHeight:function(){var t=this.options;return"auto"===t.height?t.minHeight:Math.min(t.minHeight,t.height)},_position:function(){var t=this.uiDialog.is(":visible");t||this.uiDialog.show(),this.uiDialog.position(this.options.position),t||this.uiDialog.hide()},_setOptions:function(e){var i=this,s=!1,n={};t.each(e,function(t,e){i._setOption(t,e),t in i.sizeRelatedOptions&&(s=!0),t in i.resizableRelatedOptions&&(n[t]=e)}),s&&(this._size(),this._position()),this.uiDialog.is(":data(ui-resizable)")&&this.uiDialog.resizable("option",n)},_setOption:function(e,i){var s,n,o=this.uiDialog;"disabled"!==e&&(this._super(e,i),"appendTo"===e&&this.uiDialog.appendTo(this._appendTo()),"buttons"===e&&this._createButtons(),"closeText"===e&&this.uiDialogTitlebarClose.button({label:t("<a>").text(""+this.options.closeText).html()}),"draggable"===e&&(s=o.is(":data(ui-draggable)"),s&&!i&&o.draggable("destroy"),!s&&i&&this._makeDraggable()),"position"===e&&this._position(),"resizable"===e&&(n=o.is(":data(ui-resizable)"),n&&!i&&o.resizable("destroy"),n&&"string"==typeof i&&o.resizable("option","handles",i),n||i===!1||this._makeResizable()),"title"===e&&this._title(this.uiDialogTitlebar.find(".ui-dialog-title")))},_size:function(){var t,e,i,s=this.options;this.element.show().css({width:"auto",minHeight:0,maxHeight:"none",height:0}),s.minWidth>s.width&&(s.width=s.minWidth),t=this.uiDialog.css({height:"auto",width:s.width}).outerHeight(),e=Math.max(0,s.minHeight-t),i="number"==typeof s.maxHeight?Math.max(0,s.maxHeight-t):"none","auto"===s.height?this.element.css({minHeight:e,maxHeight:i,height:"auto"}):this.element.height(Math.max(0,s.height-t)),this.uiDialog.is(":data(ui-resizable)")&&this.uiDialog.resizable("option","minHeight",this._minHeight())},_blockFrames:function(){this.iframeBlocks=this.document.find("iframe").map(function(){var e=t(this);return t("<div>").css({position:"absolute",width:e.outerWidth(),height:e.outerHeight()}).appendTo(e.parent()).offset(e.offset())[0]})},_unblockFrames:function(){this.iframeBlocks&&(this.iframeBlocks.remove(),delete this.iframeBlocks)},_allowInteraction:function(e){return t(e.target).closest(".ui-dialog").length?!0:!!t(e.target).closest(".ui-datepicker").length},_createOverlay:function(){if(this.options.modal){var e=!0;this._delay(function(){e=!1}),this.document.data("ui-dialog-overlays")||this._on(this.document,{focusin:function(t){e||this._allowInteraction(t)||(t.preventDefault(),this._trackingInstances()[0]._focusTabbable())}}),this.overlay=t("<div>").appendTo(this._appendTo()),this._addClass(this.overlay,null,"ui-widget-overlay ui-front"),this._on(this.overlay,{mousedown:"_keepFocus"}),this.document.data("ui-dialog-overlays",(this.document.data("ui-dialog-overlays")||0)+1)}},_destroyOverlay:function(){if(this.options.modal&&this.overlay){var t=this.document.data("ui-dialog-overlays")-1;t?this.document.data("ui-dialog-overlays",t):(this._off(this.document,"focusin"),this.document.removeData("ui-dialog-overlays")),this.overlay.remove(),this.overlay=null}}}),t.uiBackCompat!==!1&&t.widget("ui.dialog",t.ui.dialog,{options:{dialogClass:""},_createWrapper:function(){this._super(),this.uiDialog.addClass(this.options.dialogClass)},_setOption:function(t,e){"dialogClass"===t&&this.uiDialog.removeClass(this.options.dialogClass).addClass(e),this._superApply(arguments)}}),t.ui.dialog,t.widget("ui.droppable",{version:"1.12.1",widgetEventPrefix:"drop",options:{accept:"*",addClasses:!0,greedy:!1,scope:"default",tolerance:"intersect",activate:null,deactivate:null,drop:null,out:null,over:null},_create:function(){var e,i=this.options,s=i.accept;this.isover=!1,this.isout=!0,this.accept=t.isFunction(s)?s:function(t){return t.is(s)},this.proportions=function(){return arguments.length?(e=arguments[0],void 0):e?e:e={width:this.element[0].offsetWidth,height:this.element[0].offsetHeight}},this._addToManager(i.scope),i.addClasses&&this._addClass("ui-droppable")},_addToManager:function(e){t.ui.ddmanager.droppables[e]=t.ui.ddmanager.droppables[e]||[],t.ui.ddmanager.droppables[e].push(this)},_splice:function(t){for(var e=0;t.length>e;e++)t[e]===this&&t.splice(e,1)},_destroy:function(){var e=t.ui.ddmanager.droppables[this.options.scope];this._splice(e)},_setOption:function(e,i){if("accept"===e)this.accept=t.isFunction(i)?i:function(t){return t.is(i)};else if("scope"===e){var s=t.ui.ddmanager.droppables[this.options.scope];this._splice(s),this._addToManager(i)}this._super(e,i)},_activate:function(e){var i=t.ui.ddmanager.current;this._addActiveClass(),i&&this._trigger("activate",e,this.ui(i))},_deactivate:function(e){var i=t.ui.ddmanager.current;this._removeActiveClass(),i&&this._trigger("deactivate",e,this.ui(i))},_over:function(e){var i=t.ui.ddmanager.current;i&&(i.currentItem||i.element)[0]!==this.element[0]&&this.accept.call(this.element[0],i.currentItem||i.element)&&(this._addHoverClass(),this._trigger("over",e,this.ui(i)))},_out:function(e){var i=t.ui.ddmanager.current;i&&(i.currentItem||i.element)[0]!==this.element[0]&&this.accept.call(this.element[0],i.currentItem||i.element)&&(this._removeHoverClass(),this._trigger("out",e,this.ui(i)))},_drop:function(e,i){var s=i||t.ui.ddmanager.current,n=!1;return s&&(s.currentItem||s.element)[0]!==this.element[0]?(this.element.find(":data(ui-droppable)").not(".ui-draggable-dragging").each(function(){var i=t(this).droppable("instance");return i.options.greedy&&!i.options.disabled&&i.options.scope===s.options.scope&&i.accept.call(i.element[0],s.currentItem||s.element)&&v(s,t.extend(i,{offset:i.element.offset()}),i.options.tolerance,e)?(n=!0,!1):void 0}),n?!1:this.accept.call(this.element[0],s.currentItem||s.element)?(this._removeActiveClass(),this._removeHoverClass(),this._trigger("drop",e,this.ui(s)),this.element):!1):!1},ui:function(t){return{draggable:t.currentItem||t.element,helper:t.helper,position:t.position,offset:t.positionAbs}},_addHoverClass:function(){this._addClass("ui-droppable-hover")},_removeHoverClass:function(){this._removeClass("ui-droppable-hover")},_addActiveClass:function(){this._addClass("ui-droppable-active")},_removeActiveClass:function(){this._removeClass("ui-droppable-active")}});var v=t.ui.intersect=function(){function t(t,e,i){return t>=e&&e+i>t}return function(e,i,s,n){if(!i.offset)return!1;var o=(e.positionAbs||e.position.absolute).left+e.margins.left,a=(e.positionAbs||e.position.absolute).top+e.margins.top,r=o+e.helperProportions.width,h=a+e.helperProportions.height,l=i.offset.left,c=i.offset.top,u=l+i.proportions().width,d=c+i.proportions().height;switch(s){case"fit":return o>=l&&u>=r&&a>=c&&d>=h;case"intersect":return o+e.helperProportions.width/2>l&&u>r-e.helperProportions.width/2&&a+e.helperProportions.height/2>c&&d>h-e.helperProportions.height/2;case"pointer":return t(n.pageY,c,i.proportions().height)&&t(n.pageX,l,i.proportions().width);case"touch":return(a>=c&&d>=a||h>=c&&d>=h||c>a&&h>d)&&(o>=l&&u>=o||r>=l&&u>=r||l>o&&r>u);default:return!1}}}();t.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(e,i){var s,n,o=t.ui.ddmanager.droppables[e.options.scope]||[],a=i?i.type:null,r=(e.currentItem||e.element).find(":data(ui-droppable)").addBack();t:for(s=0;o.length>s;s++)if(!(o[s].options.disabled||e&&!o[s].accept.call(o[s].element[0],e.currentItem||e.element))){for(n=0;r.length>n;n++)if(r[n]===o[s].element[0]){o[s].proportions().height=0;continue t}o[s].visible="none"!==o[s].element.css("display"),o[s].visible&&("mousedown"===a&&o[s]._activate.call(o[s],i),o[s].offset=o[s].element.offset(),o[s].proportions({width:o[s].element[0].offsetWidth,height:o[s].element[0].offsetHeight}))}},drop:function(e,i){var s=!1;return t.each((t.ui.ddmanager.droppables[e.options.scope]||[]).slice(),function(){this.options&&(!this.options.disabled&&this.visible&&v(e,this,this.options.tolerance,i)&&(s=this._drop.call(this,i)||s),!this.options.disabled&&this.visible&&this.accept.call(this.element[0],e.currentItem||e.element)&&(this.isout=!0,this.isover=!1,this._deactivate.call(this,i)))}),s},dragStart:function(e,i){e.element.parentsUntil("body").on("scroll.droppable",function(){e.options.refreshPositions||t.ui.ddmanager.prepareOffsets(e,i)})},drag:function(e,i){e.options.refreshPositions&&t.ui.ddmanager.prepareOffsets(e,i),t.each(t.ui.ddmanager.droppables[e.options.scope]||[],function(){if(!this.options.disabled&&!this.greedyChild&&this.visible){var s,n,o,a=v(e,this,this.options.tolerance,i),r=!a&&this.isover?"isout":a&&!this.isover?"isover":null;r&&(this.options.greedy&&(n=this.options.scope,o=this.element.parents(":data(ui-droppable)").filter(function(){return t(this).droppable("instance").options.scope===n}),o.length&&(s=t(o[0]).droppable("instance"),s.greedyChild="isover"===r)),s&&"isover"===r&&(s.isover=!1,s.isout=!0,s._out.call(s,i)),this[r]=!0,this["isout"===r?"isover":"isout"]=!1,this["isover"===r?"_over":"_out"].call(this,i),s&&"isout"===r&&(s.isout=!1,s.isover=!0,s._over.call(s,i)))}})},dragStop:function(e,i){e.element.parentsUntil("body").off("scroll.droppable"),e.options.refreshPositions||t.ui.ddmanager.prepareOffsets(e,i)}},t.uiBackCompat!==!1&&t.widget("ui.droppable",t.ui.droppable,{options:{hoverClass:!1,activeClass:!1},_addActiveClass:function(){this._super(),this.options.activeClass&&this.element.addClass(this.options.activeClass)},_removeActiveClass:function(){this._super(),this.options.activeClass&&this.element.removeClass(this.options.activeClass)},_addHoverClass:function(){this._super(),this.options.hoverClass&&this.element.addClass(this.options.hoverClass)},_removeHoverClass:function(){this._super(),this.options.hoverClass&&this.element.removeClass(this.options.hoverClass)}}),t.ui.droppable,t.widget("ui.progressbar",{version:"1.12.1",options:{classes:{"ui-progressbar":"ui-corner-all","ui-progressbar-value":"ui-corner-left","ui-progressbar-complete":"ui-corner-right"},max:100,value:0,change:null,complete:null},min:0,_create:function(){this.oldValue=this.options.value=this._constrainedValue(),this.element.attr({role:"progressbar","aria-valuemin":this.min}),this._addClass("ui-progressbar","ui-widget ui-widget-content"),this.valueDiv=t("<div>").appendTo(this.element),this._addClass(this.valueDiv,"ui-progressbar-value","ui-widget-header"),this._refreshValue()},_destroy:function(){this.element.removeAttr("role aria-valuemin aria-valuemax aria-valuenow"),this.valueDiv.remove()},value:function(t){return void 0===t?this.options.value:(this.options.value=this._constrainedValue(t),this._refreshValue(),void 0)},_constrainedValue:function(t){return void 0===t&&(t=this.options.value),this.indeterminate=t===!1,"number"!=typeof t&&(t=0),this.indeterminate?!1:Math.min(this.options.max,Math.max(this.min,t))},_setOptions:function(t){var e=t.value;delete t.value,this._super(t),this.options.value=this._constrainedValue(e),this._refreshValue()},_setOption:function(t,e){"max"===t&&(e=Math.max(this.min,e)),this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t),this._toggleClass(null,"ui-state-disabled",!!t)},_percentage:function(){return this.indeterminate?100:100*(this.options.value-this.min)/(this.options.max-this.min)},_refreshValue:function(){var e=this.options.value,i=this._percentage();this.valueDiv.toggle(this.indeterminate||e>this.min).width(i.toFixed(0)+"%"),this._toggleClass(this.valueDiv,"ui-progressbar-complete",null,e===this.options.max)._toggleClass("ui-progressbar-indeterminate",null,this.indeterminate),this.indeterminate?(this.element.removeAttr("aria-valuenow"),this.overlayDiv||(this.overlayDiv=t("<div>").appendTo(this.valueDiv),this._addClass(this.overlayDiv,"ui-progressbar-overlay"))):(this.element.attr({"aria-valuemax":this.options.max,"aria-valuenow":e}),this.overlayDiv&&(this.overlayDiv.remove(),this.overlayDiv=null)),this.oldValue!==e&&(this.oldValue=e,this._trigger("change")),e===this.options.max&&this._trigger("complete")}}),t.widget("ui.selectable",t.ui.mouse,{version:"1.12.1",options:{appendTo:"body",autoRefresh:!0,distance:0,filter:"*",tolerance:"touch",selected:null,selecting:null,start:null,stop:null,unselected:null,unselecting:null},_create:function(){var e=this;this._addClass("ui-selectable"),this.dragged=!1,this.refresh=function(){e.elementPos=t(e.element[0]).offset(),e.selectees=t(e.options.filter,e.element[0]),e._addClass(e.selectees,"ui-selectee"),e.selectees.each(function(){var i=t(this),s=i.offset(),n={left:s.left-e.elementPos.left,top:s.top-e.elementPos.top};t.data(this,"selectable-item",{element:this,$element:i,left:n.left,top:n.top,right:n.left+i.outerWidth(),bottom:n.top+i.outerHeight(),startselected:!1,selected:i.hasClass("ui-selected"),selecting:i.hasClass("ui-selecting"),unselecting:i.hasClass("ui-unselecting")})})},this.refresh(),this._mouseInit(),this.helper=t("<div>"),this._addClass(this.helper,"ui-selectable-helper")},_destroy:function(){this.selectees.removeData("selectable-item"),this._mouseDestroy()},_mouseStart:function(e){var i=this,s=this.options;this.opos=[e.pageX,e.pageY],this.elementPos=t(this.element[0]).offset(),this.options.disabled||(this.selectees=t(s.filter,this.element[0]),this._trigger("start",e),t(s.appendTo).append(this.helper),this.helper.css({left:e.pageX,top:e.pageY,width:0,height:0}),s.autoRefresh&&this.refresh(),this.selectees.filter(".ui-selected").each(function(){var s=t.data(this,"selectable-item");s.startselected=!0,e.metaKey||e.ctrlKey||(i._removeClass(s.$element,"ui-selected"),s.selected=!1,i._addClass(s.$element,"ui-unselecting"),s.unselecting=!0,i._trigger("unselecting",e,{unselecting:s.element}))}),t(e.target).parents().addBack().each(function(){var s,n=t.data(this,"selectable-item");return n?(s=!e.metaKey&&!e.ctrlKey||!n.$element.hasClass("ui-selected"),i._removeClass(n.$element,s?"ui-unselecting":"ui-selected")._addClass(n.$element,s?"ui-selecting":"ui-unselecting"),n.unselecting=!s,n.selecting=s,n.selected=s,s?i._trigger("selecting",e,{selecting:n.element}):i._trigger("unselecting",e,{unselecting:n.element}),!1):void 0}))},_mouseDrag:function(e){if(this.dragged=!0,!this.options.disabled){var i,s=this,n=this.options,o=this.opos[0],a=this.opos[1],r=e.pageX,h=e.pageY;return o>r&&(i=r,r=o,o=i),a>h&&(i=h,h=a,a=i),this.helper.css({left:o,top:a,width:r-o,height:h-a}),this.selectees.each(function(){var i=t.data(this,"selectable-item"),l=!1,c={};i&&i.element!==s.element[0]&&(c.left=i.left+s.elementPos.left,c.right=i.right+s.elementPos.left,c.top=i.top+s.elementPos.top,c.bottom=i.bottom+s.elementPos.top,"touch"===n.tolerance?l=!(c.left>r||o>c.right||c.top>h||a>c.bottom):"fit"===n.tolerance&&(l=c.left>o&&r>c.right&&c.top>a&&h>c.bottom),l?(i.selected&&(s._removeClass(i.$element,"ui-selected"),i.selected=!1),i.unselecting&&(s._removeClass(i.$element,"ui-unselecting"),i.unselecting=!1),i.selecting||(s._addClass(i.$element,"ui-selecting"),i.selecting=!0,s._trigger("selecting",e,{selecting:i.element}))):(i.selecting&&((e.metaKey||e.ctrlKey)&&i.startselected?(s._removeClass(i.$element,"ui-selecting"),i.selecting=!1,s._addClass(i.$element,"ui-selected"),i.selected=!0):(s._removeClass(i.$element,"ui-selecting"),i.selecting=!1,i.startselected&&(s._addClass(i.$element,"ui-unselecting"),i.unselecting=!0),s._trigger("unselecting",e,{unselecting:i.element}))),i.selected&&(e.metaKey||e.ctrlKey||i.startselected||(s._removeClass(i.$element,"ui-selected"),i.selected=!1,s._addClass(i.$element,"ui-unselecting"),i.unselecting=!0,s._trigger("unselecting",e,{unselecting:i.element})))))}),!1}},_mouseStop:function(e){var i=this;return this.dragged=!1,t(".ui-unselecting",this.element[0]).each(function(){var s=t.data(this,"selectable-item");i._removeClass(s.$element,"ui-unselecting"),s.unselecting=!1,s.startselected=!1,i._trigger("unselected",e,{unselected:s.element})}),t(".ui-selecting",this.element[0]).each(function(){var s=t.data(this,"selectable-item");i._removeClass(s.$element,"ui-selecting")._addClass(s.$element,"ui-selected"),s.selecting=!1,s.selected=!0,s.startselected=!0,i._trigger("selected",e,{selected:s.element})}),this._trigger("stop",e),this.helper.remove(),!1}}),t.widget("ui.selectmenu",[t.ui.formResetMixin,{version:"1.12.1",defaultElement:"<select>",options:{appendTo:null,classes:{"ui-selectmenu-button-open":"ui-corner-top","ui-selectmenu-button-closed":"ui-corner-all"},disabled:null,icons:{button:"ui-icon-triangle-1-s"},position:{my:"left top",at:"left bottom",collision:"none"},width:!1,change:null,close:null,focus:null,open:null,select:null},_create:function(){var e=this.element.uniqueId().attr("id");this.ids={element:e,button:e+"-button",menu:e+"-menu"},this._drawButton(),this._drawMenu(),this._bindFormResetHandler(),this._rendered=!1,this.menuItems=t()},_drawButton:function(){var e,i=this,s=this._parseOption(this.element.find("option:selected"),this.element[0].selectedIndex);this.labels=this.element.labels().attr("for",this.ids.button),this._on(this.labels,{click:function(t){this.button.focus(),t.preventDefault()}}),this.element.hide(),this.button=t("<span>",{tabindex:this.options.disabled?-1:0,id:this.ids.button,role:"combobox","aria-expanded":"false","aria-autocomplete":"list","aria-owns":this.ids.menu,"aria-haspopup":"true",title:this.element.attr("title")}).insertAfter(this.element),this._addClass(this.button,"ui-selectmenu-button ui-selectmenu-button-closed","ui-button ui-widget"),e=t("<span>").appendTo(this.button),this._addClass(e,"ui-selectmenu-icon","ui-icon "+this.options.icons.button),this.buttonItem=this._renderButtonItem(s).appendTo(this.button),this.options.width!==!1&&this._resizeButton(),this._on(this.button,this._buttonEvents),this.button.one("focusin",function(){i._rendered||i._refreshMenu()})},_drawMenu:function(){var e=this;this.menu=t("<ul>",{"aria-hidden":"true","aria-labelledby":this.ids.button,id:this.ids.menu}),this.menuWrap=t("<div>").append(this.menu),this._addClass(this.menuWrap,"ui-selectmenu-menu","ui-front"),this.menuWrap.appendTo(this._appendTo()),this.menuInstance=this.menu.menu({classes:{"ui-menu":"ui-corner-bottom"},role:"listbox",select:function(t,i){t.preventDefault(),e._setSelection(),e._select(i.item.data("ui-selectmenu-item"),t)},focus:function(t,i){var s=i.item.data("ui-selectmenu-item");null!=e.focusIndex&&s.index!==e.focusIndex&&(e._trigger("focus",t,{item:s}),e.isOpen||e._select(s,t)),e.focusIndex=s.index,e.button.attr("aria-activedescendant",e.menuItems.eq(s.index).attr("id"))}}).menu("instance"),this.menuInstance._off(this.menu,"mouseleave"),this.menuInstance._closeOnDocumentClick=function(){return!1},this.menuInstance._isDivider=function(){return!1}},refresh:function(){this._refreshMenu(),this.buttonItem.replaceWith(this.buttonItem=this._renderButtonItem(this._getSelectedItem().data("ui-selectmenu-item")||{})),null===this.options.width&&this._resizeButton()},_refreshMenu:function(){var t,e=this.element.find("option");this.menu.empty(),this._parseOptions(e),this._renderMenu(this.menu,this.items),this.menuInstance.refresh(),this.menuItems=this.menu.find("li").not(".ui-selectmenu-optgroup").find(".ui-menu-item-wrapper"),this._rendered=!0,e.length&&(t=this._getSelectedItem(),this.menuInstance.focus(null,t),this._setAria(t.data("ui-selectmenu-item")),this._setOption("disabled",this.element.prop("disabled")))},open:function(t){this.options.disabled||(this._rendered?(this._removeClass(this.menu.find(".ui-state-active"),null,"ui-state-active"),this.menuInstance.focus(null,this._getSelectedItem())):this._refreshMenu(),this.menuItems.length&&(this.isOpen=!0,this._toggleAttr(),this._resizeMenu(),this._position(),this._on(this.document,this._documentClick),this._trigger("open",t)))},_position:function(){this.menuWrap.position(t.extend({of:this.button},this.options.position))},close:function(t){this.isOpen&&(this.isOpen=!1,this._toggleAttr(),this.range=null,this._off(this.document),this._trigger("close",t))},widget:function(){return this.button},menuWidget:function(){return this.menu},_renderButtonItem:function(e){var i=t("<span>");return this._setText(i,e.label),this._addClass(i,"ui-selectmenu-text"),i},_renderMenu:function(e,i){var s=this,n="";t.each(i,function(i,o){var a;o.optgroup!==n&&(a=t("<li>",{text:o.optgroup}),s._addClass(a,"ui-selectmenu-optgroup","ui-menu-divider"+(o.element.parent("optgroup").prop("disabled")?" ui-state-disabled":"")),a.appendTo(e),n=o.optgroup),s._renderItemData(e,o)})},_renderItemData:function(t,e){return this._renderItem(t,e).data("ui-selectmenu-item",e)},_renderItem:function(e,i){var s=t("<li>"),n=t("<div>",{title:i.element.attr("title")});return i.disabled&&this._addClass(s,null,"ui-state-disabled"),this._setText(n,i.label),s.append(n).appendTo(e)},_setText:function(t,e){e?t.text(e):t.html("&#160;")},_move:function(t,e){var i,s,n=".ui-menu-item";this.isOpen?i=this.menuItems.eq(this.focusIndex).parent("li"):(i=this.menuItems.eq(this.element[0].selectedIndex).parent("li"),n+=":not(.ui-state-disabled)"),s="first"===t||"last"===t?i["first"===t?"prevAll":"nextAll"](n).eq(-1):i[t+"All"](n).eq(0),s.length&&this.menuInstance.focus(e,s)},_getSelectedItem:function(){return this.menuItems.eq(this.element[0].selectedIndex).parent("li")},_toggle:function(t){this[this.isOpen?"close":"open"](t)},_setSelection:function(){var t;this.range&&(window.getSelection?(t=window.getSelection(),t.removeAllRanges(),t.addRange(this.range)):this.range.select(),this.button.focus())},_documentClick:{mousedown:function(e){this.isOpen&&(t(e.target).closest(".ui-selectmenu-menu, #"+t.ui.escapeSelector(this.ids.button)).length||this.close(e))}},_buttonEvents:{mousedown:function(){var t;window.getSelection?(t=window.getSelection(),t.rangeCount&&(this.range=t.getRangeAt(0))):this.range=document.selection.createRange()},click:function(t){this._setSelection(),this._toggle(t)},keydown:function(e){var i=!0;switch(e.keyCode){case t.ui.keyCode.TAB:case t.ui.keyCode.ESCAPE:this.close(e),i=!1;break;case t.ui.keyCode.ENTER:this.isOpen&&this._selectFocusedItem(e);break;case t.ui.keyCode.UP:e.altKey?this._toggle(e):this._move("prev",e);break;case t.ui.keyCode.DOWN:e.altKey?this._toggle(e):this._move("next",e);break;case t.ui.keyCode.SPACE:this.isOpen?this._selectFocusedItem(e):this._toggle(e);break;case t.ui.keyCode.LEFT:this._move("prev",e);break;case t.ui.keyCode.RIGHT:this._move("next",e);break;case t.ui.keyCode.HOME:case t.ui.keyCode.PAGE_UP:this._move("first",e);break;case t.ui.keyCode.END:case t.ui.keyCode.PAGE_DOWN:this._move("last",e);break;default:this.menu.trigger(e),i=!1}i&&e.preventDefault()}},_selectFocusedItem:function(t){var e=this.menuItems.eq(this.focusIndex).parent("li");e.hasClass("ui-state-disabled")||this._select(e.data("ui-selectmenu-item"),t)},_select:function(t,e){var i=this.element[0].selectedIndex;this.element[0].selectedIndex=t.index,this.buttonItem.replaceWith(this.buttonItem=this._renderButtonItem(t)),this._setAria(t),this._trigger("select",e,{item:t}),t.index!==i&&this._trigger("change",e,{item:t}),this.close(e)},_setAria:function(t){var e=this.menuItems.eq(t.index).attr("id");this.button.attr({"aria-labelledby":e,"aria-activedescendant":e}),this.menu.attr("aria-activedescendant",e)},_setOption:function(t,e){if("icons"===t){var i=this.button.find("span.ui-icon");this._removeClass(i,null,this.options.icons.button)._addClass(i,null,e.button)}this._super(t,e),"appendTo"===t&&this.menuWrap.appendTo(this._appendTo()),"width"===t&&this._resizeButton()},_setOptionDisabled:function(t){this._super(t),this.menuInstance.option("disabled",t),this.button.attr("aria-disabled",t),this._toggleClass(this.button,null,"ui-state-disabled",t),this.element.prop("disabled",t),t?(this.button.attr("tabindex",-1),this.close()):this.button.attr("tabindex",0)},_appendTo:function(){var e=this.options.appendTo;return e&&(e=e.jquery||e.nodeType?t(e):this.document.find(e).eq(0)),e&&e[0]||(e=this.element.closest(".ui-front, dialog")),e.length||(e=this.document[0].body),e},_toggleAttr:function(){this.button.attr("aria-expanded",this.isOpen),this._removeClass(this.button,"ui-selectmenu-button-"+(this.isOpen?"closed":"open"))._addClass(this.button,"ui-selectmenu-button-"+(this.isOpen?"open":"closed"))._toggleClass(this.menuWrap,"ui-selectmenu-open",null,this.isOpen),this.menu.attr("aria-hidden",!this.isOpen)},_resizeButton:function(){var t=this.options.width;return t===!1?(this.button.css("width",""),void 0):(null===t&&(t=this.element.show().outerWidth(),this.element.hide()),this.button.outerWidth(t),void 0)},_resizeMenu:function(){this.menu.outerWidth(Math.max(this.button.outerWidth(),this.menu.width("").outerWidth()+1))},_getCreateOptions:function(){var t=this._super();return t.disabled=this.element.prop("disabled"),t},_parseOptions:function(e){var i=this,s=[];e.each(function(e,n){s.push(i._parseOption(t(n),e))}),this.items=s},_parseOption:function(t,e){var i=t.parent("optgroup");return{element:t,index:e,value:t.val(),label:t.text(),optgroup:i.attr("label")||"",disabled:i.prop("disabled")||t.prop("disabled")}},_destroy:function(){this._unbindFormResetHandler(),this.menuWrap.remove(),this.button.remove(),this.element.show(),this.element.removeUniqueId(),this.labels.attr("for",this.ids.element)}}]),t.widget("ui.slider",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"slide",options:{animate:!1,classes:{"ui-slider":"ui-corner-all","ui-slider-handle":"ui-corner-all","ui-slider-range":"ui-corner-all ui-widget-header"},distance:0,max:100,min:0,orientation:"horizontal",range:!1,step:1,value:0,values:null,change:null,slide:null,start:null,stop:null},numPages:5,_create:function(){this._keySliding=!1,this._mouseSliding=!1,this._animateOff=!0,this._handleIndex=null,this._detectOrientation(),this._mouseInit(),this._calculateNewMax(),this._addClass("ui-slider ui-slider-"+this.orientation,"ui-widget ui-widget-content"),this._refresh(),this._animateOff=!1
-},_refresh:function(){this._createRange(),this._createHandles(),this._setupEvents(),this._refreshValue()},_createHandles:function(){var e,i,s=this.options,n=this.element.find(".ui-slider-handle"),o="<span tabindex='0'></span>",a=[];for(i=s.values&&s.values.length||1,n.length>i&&(n.slice(i).remove(),n=n.slice(0,i)),e=n.length;i>e;e++)a.push(o);this.handles=n.add(t(a.join("")).appendTo(this.element)),this._addClass(this.handles,"ui-slider-handle","ui-state-default"),this.handle=this.handles.eq(0),this.handles.each(function(e){t(this).data("ui-slider-handle-index",e).attr("tabIndex",0)})},_createRange:function(){var e=this.options;e.range?(e.range===!0&&(e.values?e.values.length&&2!==e.values.length?e.values=[e.values[0],e.values[0]]:t.isArray(e.values)&&(e.values=e.values.slice(0)):e.values=[this._valueMin(),this._valueMin()]),this.range&&this.range.length?(this._removeClass(this.range,"ui-slider-range-min ui-slider-range-max"),this.range.css({left:"",bottom:""})):(this.range=t("<div>").appendTo(this.element),this._addClass(this.range,"ui-slider-range")),("min"===e.range||"max"===e.range)&&this._addClass(this.range,"ui-slider-range-"+e.range)):(this.range&&this.range.remove(),this.range=null)},_setupEvents:function(){this._off(this.handles),this._on(this.handles,this._handleEvents),this._hoverable(this.handles),this._focusable(this.handles)},_destroy:function(){this.handles.remove(),this.range&&this.range.remove(),this._mouseDestroy()},_mouseCapture:function(e){var i,s,n,o,a,r,h,l,c=this,u=this.options;return u.disabled?!1:(this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()},this.elementOffset=this.element.offset(),i={x:e.pageX,y:e.pageY},s=this._normValueFromMouse(i),n=this._valueMax()-this._valueMin()+1,this.handles.each(function(e){var i=Math.abs(s-c.values(e));(n>i||n===i&&(e===c._lastChangedValue||c.values(e)===u.min))&&(n=i,o=t(this),a=e)}),r=this._start(e,a),r===!1?!1:(this._mouseSliding=!0,this._handleIndex=a,this._addClass(o,null,"ui-state-active"),o.trigger("focus"),h=o.offset(),l=!t(e.target).parents().addBack().is(".ui-slider-handle"),this._clickOffset=l?{left:0,top:0}:{left:e.pageX-h.left-o.width()/2,top:e.pageY-h.top-o.height()/2-(parseInt(o.css("borderTopWidth"),10)||0)-(parseInt(o.css("borderBottomWidth"),10)||0)+(parseInt(o.css("marginTop"),10)||0)},this.handles.hasClass("ui-state-hover")||this._slide(e,a,s),this._animateOff=!0,!0))},_mouseStart:function(){return!0},_mouseDrag:function(t){var e={x:t.pageX,y:t.pageY},i=this._normValueFromMouse(e);return this._slide(t,this._handleIndex,i),!1},_mouseStop:function(t){return this._removeClass(this.handles,null,"ui-state-active"),this._mouseSliding=!1,this._stop(t,this._handleIndex),this._change(t,this._handleIndex),this._handleIndex=null,this._clickOffset=null,this._animateOff=!1,!1},_detectOrientation:function(){this.orientation="vertical"===this.options.orientation?"vertical":"horizontal"},_normValueFromMouse:function(t){var e,i,s,n,o;return"horizontal"===this.orientation?(e=this.elementSize.width,i=t.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)):(e=this.elementSize.height,i=t.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)),s=i/e,s>1&&(s=1),0>s&&(s=0),"vertical"===this.orientation&&(s=1-s),n=this._valueMax()-this._valueMin(),o=this._valueMin()+s*n,this._trimAlignValue(o)},_uiHash:function(t,e,i){var s={handle:this.handles[t],handleIndex:t,value:void 0!==e?e:this.value()};return this._hasMultipleValues()&&(s.value=void 0!==e?e:this.values(t),s.values=i||this.values()),s},_hasMultipleValues:function(){return this.options.values&&this.options.values.length},_start:function(t,e){return this._trigger("start",t,this._uiHash(e))},_slide:function(t,e,i){var s,n,o=this.value(),a=this.values();this._hasMultipleValues()&&(n=this.values(e?0:1),o=this.values(e),2===this.options.values.length&&this.options.range===!0&&(i=0===e?Math.min(n,i):Math.max(n,i)),a[e]=i),i!==o&&(s=this._trigger("slide",t,this._uiHash(e,i,a)),s!==!1&&(this._hasMultipleValues()?this.values(e,i):this.value(i)))},_stop:function(t,e){this._trigger("stop",t,this._uiHash(e))},_change:function(t,e){this._keySliding||this._mouseSliding||(this._lastChangedValue=e,this._trigger("change",t,this._uiHash(e)))},value:function(t){return arguments.length?(this.options.value=this._trimAlignValue(t),this._refreshValue(),this._change(null,0),void 0):this._value()},values:function(e,i){var s,n,o;if(arguments.length>1)return this.options.values[e]=this._trimAlignValue(i),this._refreshValue(),this._change(null,e),void 0;if(!arguments.length)return this._values();if(!t.isArray(arguments[0]))return this._hasMultipleValues()?this._values(e):this.value();for(s=this.options.values,n=arguments[0],o=0;s.length>o;o+=1)s[o]=this._trimAlignValue(n[o]),this._change(null,o);this._refreshValue()},_setOption:function(e,i){var s,n=0;switch("range"===e&&this.options.range===!0&&("min"===i?(this.options.value=this._values(0),this.options.values=null):"max"===i&&(this.options.value=this._values(this.options.values.length-1),this.options.values=null)),t.isArray(this.options.values)&&(n=this.options.values.length),this._super(e,i),e){case"orientation":this._detectOrientation(),this._removeClass("ui-slider-horizontal ui-slider-vertical")._addClass("ui-slider-"+this.orientation),this._refreshValue(),this.options.range&&this._refreshRange(i),this.handles.css("horizontal"===i?"bottom":"left","");break;case"value":this._animateOff=!0,this._refreshValue(),this._change(null,0),this._animateOff=!1;break;case"values":for(this._animateOff=!0,this._refreshValue(),s=n-1;s>=0;s--)this._change(null,s);this._animateOff=!1;break;case"step":case"min":case"max":this._animateOff=!0,this._calculateNewMax(),this._refreshValue(),this._animateOff=!1;break;case"range":this._animateOff=!0,this._refresh(),this._animateOff=!1}},_setOptionDisabled:function(t){this._super(t),this._toggleClass(null,"ui-state-disabled",!!t)},_value:function(){var t=this.options.value;return t=this._trimAlignValue(t)},_values:function(t){var e,i,s;if(arguments.length)return e=this.options.values[t],e=this._trimAlignValue(e);if(this._hasMultipleValues()){for(i=this.options.values.slice(),s=0;i.length>s;s+=1)i[s]=this._trimAlignValue(i[s]);return i}return[]},_trimAlignValue:function(t){if(this._valueMin()>=t)return this._valueMin();if(t>=this._valueMax())return this._valueMax();var e=this.options.step>0?this.options.step:1,i=(t-this._valueMin())%e,s=t-i;return 2*Math.abs(i)>=e&&(s+=i>0?e:-e),parseFloat(s.toFixed(5))},_calculateNewMax:function(){var t=this.options.max,e=this._valueMin(),i=this.options.step,s=Math.round((t-e)/i)*i;t=s+e,t>this.options.max&&(t-=i),this.max=parseFloat(t.toFixed(this._precision()))},_precision:function(){var t=this._precisionOf(this.options.step);return null!==this.options.min&&(t=Math.max(t,this._precisionOf(this.options.min))),t},_precisionOf:function(t){var e=""+t,i=e.indexOf(".");return-1===i?0:e.length-i-1},_valueMin:function(){return this.options.min},_valueMax:function(){return this.max},_refreshRange:function(t){"vertical"===t&&this.range.css({width:"",left:""}),"horizontal"===t&&this.range.css({height:"",bottom:""})},_refreshValue:function(){var e,i,s,n,o,a=this.options.range,r=this.options,h=this,l=this._animateOff?!1:r.animate,c={};this._hasMultipleValues()?this.handles.each(function(s){i=100*((h.values(s)-h._valueMin())/(h._valueMax()-h._valueMin())),c["horizontal"===h.orientation?"left":"bottom"]=i+"%",t(this).stop(1,1)[l?"animate":"css"](c,r.animate),h.options.range===!0&&("horizontal"===h.orientation?(0===s&&h.range.stop(1,1)[l?"animate":"css"]({left:i+"%"},r.animate),1===s&&h.range[l?"animate":"css"]({width:i-e+"%"},{queue:!1,duration:r.animate})):(0===s&&h.range.stop(1,1)[l?"animate":"css"]({bottom:i+"%"},r.animate),1===s&&h.range[l?"animate":"css"]({height:i-e+"%"},{queue:!1,duration:r.animate}))),e=i}):(s=this.value(),n=this._valueMin(),o=this._valueMax(),i=o!==n?100*((s-n)/(o-n)):0,c["horizontal"===this.orientation?"left":"bottom"]=i+"%",this.handle.stop(1,1)[l?"animate":"css"](c,r.animate),"min"===a&&"horizontal"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({width:i+"%"},r.animate),"max"===a&&"horizontal"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({width:100-i+"%"},r.animate),"min"===a&&"vertical"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({height:i+"%"},r.animate),"max"===a&&"vertical"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({height:100-i+"%"},r.animate))},_handleEvents:{keydown:function(e){var i,s,n,o,a=t(e.target).data("ui-slider-handle-index");switch(e.keyCode){case t.ui.keyCode.HOME:case t.ui.keyCode.END:case t.ui.keyCode.PAGE_UP:case t.ui.keyCode.PAGE_DOWN:case t.ui.keyCode.UP:case t.ui.keyCode.RIGHT:case t.ui.keyCode.DOWN:case t.ui.keyCode.LEFT:if(e.preventDefault(),!this._keySliding&&(this._keySliding=!0,this._addClass(t(e.target),null,"ui-state-active"),i=this._start(e,a),i===!1))return}switch(o=this.options.step,s=n=this._hasMultipleValues()?this.values(a):this.value(),e.keyCode){case t.ui.keyCode.HOME:n=this._valueMin();break;case t.ui.keyCode.END:n=this._valueMax();break;case t.ui.keyCode.PAGE_UP:n=this._trimAlignValue(s+(this._valueMax()-this._valueMin())/this.numPages);break;case t.ui.keyCode.PAGE_DOWN:n=this._trimAlignValue(s-(this._valueMax()-this._valueMin())/this.numPages);break;case t.ui.keyCode.UP:case t.ui.keyCode.RIGHT:if(s===this._valueMax())return;n=this._trimAlignValue(s+o);break;case t.ui.keyCode.DOWN:case t.ui.keyCode.LEFT:if(s===this._valueMin())return;n=this._trimAlignValue(s-o)}this._slide(e,a,n)},keyup:function(e){var i=t(e.target).data("ui-slider-handle-index");this._keySliding&&(this._keySliding=!1,this._stop(e,i),this._change(e,i),this._removeClass(t(e.target),null,"ui-state-active"))}}}),t.widget("ui.sortable",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"sort",ready:!1,options:{appendTo:"parent",axis:!1,connectWith:!1,containment:!1,cursor:"auto",cursorAt:!1,dropOnEmpty:!0,forcePlaceholderSize:!1,forceHelperSize:!1,grid:!1,handle:!1,helper:"original",items:"> *",opacity:!1,placeholder:!1,revert:!1,scroll:!0,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1e3,activate:null,beforeStop:null,change:null,deactivate:null,out:null,over:null,receive:null,remove:null,sort:null,start:null,stop:null,update:null},_isOverAxis:function(t,e,i){return t>=e&&e+i>t},_isFloating:function(t){return/left|right/.test(t.css("float"))||/inline|table-cell/.test(t.css("display"))},_create:function(){this.containerCache={},this._addClass("ui-sortable"),this.refresh(),this.offset=this.element.offset(),this._mouseInit(),this._setHandleClassName(),this.ready=!0},_setOption:function(t,e){this._super(t,e),"handle"===t&&this._setHandleClassName()},_setHandleClassName:function(){var e=this;this._removeClass(this.element.find(".ui-sortable-handle"),"ui-sortable-handle"),t.each(this.items,function(){e._addClass(this.instance.options.handle?this.item.find(this.instance.options.handle):this.item,"ui-sortable-handle")})},_destroy:function(){this._mouseDestroy();for(var t=this.items.length-1;t>=0;t--)this.items[t].item.removeData(this.widgetName+"-item");return this},_mouseCapture:function(e,i){var s=null,n=!1,o=this;return this.reverting?!1:this.options.disabled||"static"===this.options.type?!1:(this._refreshItems(e),t(e.target).parents().each(function(){return t.data(this,o.widgetName+"-item")===o?(s=t(this),!1):void 0}),t.data(e.target,o.widgetName+"-item")===o&&(s=t(e.target)),s?!this.options.handle||i||(t(this.options.handle,s).find("*").addBack().each(function(){this===e.target&&(n=!0)}),n)?(this.currentItem=s,this._removeCurrentsFromItems(),!0):!1:!1)},_mouseStart:function(e,i,s){var n,o,a=this.options;if(this.currentContainer=this,this.refreshPositions(),this.helper=this._createHelper(e),this._cacheHelperProportions(),this._cacheMargins(),this.scrollParent=this.helper.scrollParent(),this.offset=this.currentItem.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},t.extend(this.offset,{click:{left:e.pageX-this.offset.left,top:e.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()}),this.helper.css("position","absolute"),this.cssPosition=this.helper.css("position"),this.originalPosition=this._generatePosition(e),this.originalPageX=e.pageX,this.originalPageY=e.pageY,a.cursorAt&&this._adjustOffsetFromHelper(a.cursorAt),this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]},this.helper[0]!==this.currentItem[0]&&this.currentItem.hide(),this._createPlaceholder(),a.containment&&this._setContainment(),a.cursor&&"auto"!==a.cursor&&(o=this.document.find("body"),this.storedCursor=o.css("cursor"),o.css("cursor",a.cursor),this.storedStylesheet=t("<style>*{ cursor: "+a.cursor+" !important; }</style>").appendTo(o)),a.opacity&&(this.helper.css("opacity")&&(this._storedOpacity=this.helper.css("opacity")),this.helper.css("opacity",a.opacity)),a.zIndex&&(this.helper.css("zIndex")&&(this._storedZIndex=this.helper.css("zIndex")),this.helper.css("zIndex",a.zIndex)),this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName&&(this.overflowOffset=this.scrollParent.offset()),this._trigger("start",e,this._uiHash()),this._preserveHelperProportions||this._cacheHelperProportions(),!s)for(n=this.containers.length-1;n>=0;n--)this.containers[n]._trigger("activate",e,this._uiHash(this));return t.ui.ddmanager&&(t.ui.ddmanager.current=this),t.ui.ddmanager&&!a.dropBehaviour&&t.ui.ddmanager.prepareOffsets(this,e),this.dragging=!0,this._addClass(this.helper,"ui-sortable-helper"),this._mouseDrag(e),!0},_mouseDrag:function(e){var i,s,n,o,a=this.options,r=!1;for(this.position=this._generatePosition(e),this.positionAbs=this._convertPositionTo("absolute"),this.lastPositionAbs||(this.lastPositionAbs=this.positionAbs),this.options.scroll&&(this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName?(this.overflowOffset.top+this.scrollParent[0].offsetHeight-e.pageY<a.scrollSensitivity?this.scrollParent[0].scrollTop=r=this.scrollParent[0].scrollTop+a.scrollSpeed:e.pageY-this.overflowOffset.top<a.scrollSensitivity&&(this.scrollParent[0].scrollTop=r=this.scrollParent[0].scrollTop-a.scrollSpeed),this.overflowOffset.left+this.scrollParent[0].offsetWidth-e.pageX<a.scrollSensitivity?this.scrollParent[0].scrollLeft=r=this.scrollParent[0].scrollLeft+a.scrollSpeed:e.pageX-this.overflowOffset.left<a.scrollSensitivity&&(this.scrollParent[0].scrollLeft=r=this.scrollParent[0].scrollLeft-a.scrollSpeed)):(e.pageY-this.document.scrollTop()<a.scrollSensitivity?r=this.document.scrollTop(this.document.scrollTop()-a.scrollSpeed):this.window.height()-(e.pageY-this.document.scrollTop())<a.scrollSensitivity&&(r=this.document.scrollTop(this.document.scrollTop()+a.scrollSpeed)),e.pageX-this.document.scrollLeft()<a.scrollSensitivity?r=this.document.scrollLeft(this.document.scrollLeft()-a.scrollSpeed):this.window.width()-(e.pageX-this.document.scrollLeft())<a.scrollSensitivity&&(r=this.document.scrollLeft(this.document.scrollLeft()+a.scrollSpeed))),r!==!1&&t.ui.ddmanager&&!a.dropBehaviour&&t.ui.ddmanager.prepareOffsets(this,e)),this.positionAbs=this._convertPositionTo("absolute"),this.options.axis&&"y"===this.options.axis||(this.helper[0].style.left=this.position.left+"px"),this.options.axis&&"x"===this.options.axis||(this.helper[0].style.top=this.position.top+"px"),i=this.items.length-1;i>=0;i--)if(s=this.items[i],n=s.item[0],o=this._intersectsWithPointer(s),o&&s.instance===this.currentContainer&&n!==this.currentItem[0]&&this.placeholder[1===o?"next":"prev"]()[0]!==n&&!t.contains(this.placeholder[0],n)&&("semi-dynamic"===this.options.type?!t.contains(this.element[0],n):!0)){if(this.direction=1===o?"down":"up","pointer"!==this.options.tolerance&&!this._intersectsWithSides(s))break;this._rearrange(e,s),this._trigger("change",e,this._uiHash());break}return this._contactContainers(e),t.ui.ddmanager&&t.ui.ddmanager.drag(this,e),this._trigger("sort",e,this._uiHash()),this.lastPositionAbs=this.positionAbs,!1},_mouseStop:function(e,i){if(e){if(t.ui.ddmanager&&!this.options.dropBehaviour&&t.ui.ddmanager.drop(this,e),this.options.revert){var s=this,n=this.placeholder.offset(),o=this.options.axis,a={};o&&"x"!==o||(a.left=n.left-this.offset.parent.left-this.margins.left+(this.offsetParent[0]===this.document[0].body?0:this.offsetParent[0].scrollLeft)),o&&"y"!==o||(a.top=n.top-this.offset.parent.top-this.margins.top+(this.offsetParent[0]===this.document[0].body?0:this.offsetParent[0].scrollTop)),this.reverting=!0,t(this.helper).animate(a,parseInt(this.options.revert,10)||500,function(){s._clear(e)})}else this._clear(e,i);return!1}},cancel:function(){if(this.dragging){this._mouseUp(new t.Event("mouseup",{target:null})),"original"===this.options.helper?(this.currentItem.css(this._storedCSS),this._removeClass(this.currentItem,"ui-sortable-helper")):this.currentItem.show();for(var e=this.containers.length-1;e>=0;e--)this.containers[e]._trigger("deactivate",null,this._uiHash(this)),this.containers[e].containerCache.over&&(this.containers[e]._trigger("out",null,this._uiHash(this)),this.containers[e].containerCache.over=0)}return this.placeholder&&(this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]),"original"!==this.options.helper&&this.helper&&this.helper[0].parentNode&&this.helper.remove(),t.extend(this,{helper:null,dragging:!1,reverting:!1,_noFinalSort:null}),this.domPosition.prev?t(this.domPosition.prev).after(this.currentItem):t(this.domPosition.parent).prepend(this.currentItem)),this},serialize:function(e){var i=this._getItemsAsjQuery(e&&e.connected),s=[];return e=e||{},t(i).each(function(){var i=(t(e.item||this).attr(e.attribute||"id")||"").match(e.expression||/(.+)[\-=_](.+)/);i&&s.push((e.key||i[1]+"[]")+"="+(e.key&&e.expression?i[1]:i[2]))}),!s.length&&e.key&&s.push(e.key+"="),s.join("&")},toArray:function(e){var i=this._getItemsAsjQuery(e&&e.connected),s=[];return e=e||{},i.each(function(){s.push(t(e.item||this).attr(e.attribute||"id")||"")}),s},_intersectsWith:function(t){var e=this.positionAbs.left,i=e+this.helperProportions.width,s=this.positionAbs.top,n=s+this.helperProportions.height,o=t.left,a=o+t.width,r=t.top,h=r+t.height,l=this.offset.click.top,c=this.offset.click.left,u="x"===this.options.axis||s+l>r&&h>s+l,d="y"===this.options.axis||e+c>o&&a>e+c,p=u&&d;return"pointer"===this.options.tolerance||this.options.forcePointerForContainers||"pointer"!==this.options.tolerance&&this.helperProportions[this.floating?"width":"height"]>t[this.floating?"width":"height"]?p:e+this.helperProportions.width/2>o&&a>i-this.helperProportions.width/2&&s+this.helperProportions.height/2>r&&h>n-this.helperProportions.height/2},_intersectsWithPointer:function(t){var e,i,s="x"===this.options.axis||this._isOverAxis(this.positionAbs.top+this.offset.click.top,t.top,t.height),n="y"===this.options.axis||this._isOverAxis(this.positionAbs.left+this.offset.click.left,t.left,t.width),o=s&&n;return o?(e=this._getDragVerticalDirection(),i=this._getDragHorizontalDirection(),this.floating?"right"===i||"down"===e?2:1:e&&("down"===e?2:1)):!1},_intersectsWithSides:function(t){var e=this._isOverAxis(this.positionAbs.top+this.offset.click.top,t.top+t.height/2,t.height),i=this._isOverAxis(this.positionAbs.left+this.offset.click.left,t.left+t.width/2,t.width),s=this._getDragVerticalDirection(),n=this._getDragHorizontalDirection();return this.floating&&n?"right"===n&&i||"left"===n&&!i:s&&("down"===s&&e||"up"===s&&!e)},_getDragVerticalDirection:function(){var t=this.positionAbs.top-this.lastPositionAbs.top;return 0!==t&&(t>0?"down":"up")},_getDragHorizontalDirection:function(){var t=this.positionAbs.left-this.lastPositionAbs.left;return 0!==t&&(t>0?"right":"left")},refresh:function(t){return this._refreshItems(t),this._setHandleClassName(),this.refreshPositions(),this},_connectWith:function(){var t=this.options;return t.connectWith.constructor===String?[t.connectWith]:t.connectWith},_getItemsAsjQuery:function(e){function i(){r.push(this)}var s,n,o,a,r=[],h=[],l=this._connectWith();if(l&&e)for(s=l.length-1;s>=0;s--)for(o=t(l[s],this.document[0]),n=o.length-1;n>=0;n--)a=t.data(o[n],this.widgetFullName),a&&a!==this&&!a.options.disabled&&h.push([t.isFunction(a.options.items)?a.options.items.call(a.element):t(a.options.items,a.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),a]);for(h.push([t.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):t(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),this]),s=h.length-1;s>=0;s--)h[s][0].each(i);return t(r)},_removeCurrentsFromItems:function(){var e=this.currentItem.find(":data("+this.widgetName+"-item)");this.items=t.grep(this.items,function(t){for(var i=0;e.length>i;i++)if(e[i]===t.item[0])return!1;return!0})},_refreshItems:function(e){this.items=[],this.containers=[this];var i,s,n,o,a,r,h,l,c=this.items,u=[[t.isFunction(this.options.items)?this.options.items.call(this.element[0],e,{item:this.currentItem}):t(this.options.items,this.element),this]],d=this._connectWith();if(d&&this.ready)for(i=d.length-1;i>=0;i--)for(n=t(d[i],this.document[0]),s=n.length-1;s>=0;s--)o=t.data(n[s],this.widgetFullName),o&&o!==this&&!o.options.disabled&&(u.push([t.isFunction(o.options.items)?o.options.items.call(o.element[0],e,{item:this.currentItem}):t(o.options.items,o.element),o]),this.containers.push(o));for(i=u.length-1;i>=0;i--)for(a=u[i][1],r=u[i][0],s=0,l=r.length;l>s;s++)h=t(r[s]),h.data(this.widgetName+"-item",a),c.push({item:h,instance:a,width:0,height:0,left:0,top:0})},refreshPositions:function(e){this.floating=this.items.length?"x"===this.options.axis||this._isFloating(this.items[0].item):!1,this.offsetParent&&this.helper&&(this.offset.parent=this._getParentOffset());var i,s,n,o;for(i=this.items.length-1;i>=0;i--)s=this.items[i],s.instance!==this.currentContainer&&this.currentContainer&&s.item[0]!==this.currentItem[0]||(n=this.options.toleranceElement?t(this.options.toleranceElement,s.item):s.item,e||(s.width=n.outerWidth(),s.height=n.outerHeight()),o=n.offset(),s.left=o.left,s.top=o.top);if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(i=this.containers.length-1;i>=0;i--)o=this.containers[i].element.offset(),this.containers[i].containerCache.left=o.left,this.containers[i].containerCache.top=o.top,this.containers[i].containerCache.width=this.containers[i].element.outerWidth(),this.containers[i].containerCache.height=this.containers[i].element.outerHeight();return this},_createPlaceholder:function(e){e=e||this;var i,s=e.options;s.placeholder&&s.placeholder.constructor!==String||(i=s.placeholder,s.placeholder={element:function(){var s=e.currentItem[0].nodeName.toLowerCase(),n=t("<"+s+">",e.document[0]);return e._addClass(n,"ui-sortable-placeholder",i||e.currentItem[0].className)._removeClass(n,"ui-sortable-helper"),"tbody"===s?e._createTrPlaceholder(e.currentItem.find("tr").eq(0),t("<tr>",e.document[0]).appendTo(n)):"tr"===s?e._createTrPlaceholder(e.currentItem,n):"img"===s&&n.attr("src",e.currentItem.attr("src")),i||n.css("visibility","hidden"),n},update:function(t,n){(!i||s.forcePlaceholderSize)&&(n.height()||n.height(e.currentItem.innerHeight()-parseInt(e.currentItem.css("paddingTop")||0,10)-parseInt(e.currentItem.css("paddingBottom")||0,10)),n.width()||n.width(e.currentItem.innerWidth()-parseInt(e.currentItem.css("paddingLeft")||0,10)-parseInt(e.currentItem.css("paddingRight")||0,10)))}}),e.placeholder=t(s.placeholder.element.call(e.element,e.currentItem)),e.currentItem.after(e.placeholder),s.placeholder.update(e,e.placeholder)},_createTrPlaceholder:function(e,i){var s=this;e.children().each(function(){t("<td>&#160;</td>",s.document[0]).attr("colspan",t(this).attr("colspan")||1).appendTo(i)})},_contactContainers:function(e){var i,s,n,o,a,r,h,l,c,u,d=null,p=null;for(i=this.containers.length-1;i>=0;i--)if(!t.contains(this.currentItem[0],this.containers[i].element[0]))if(this._intersectsWith(this.containers[i].containerCache)){if(d&&t.contains(this.containers[i].element[0],d.element[0]))continue;d=this.containers[i],p=i}else this.containers[i].containerCache.over&&(this.containers[i]._trigger("out",e,this._uiHash(this)),this.containers[i].containerCache.over=0);if(d)if(1===this.containers.length)this.containers[p].containerCache.over||(this.containers[p]._trigger("over",e,this._uiHash(this)),this.containers[p].containerCache.over=1);else{for(n=1e4,o=null,c=d.floating||this._isFloating(this.currentItem),a=c?"left":"top",r=c?"width":"height",u=c?"pageX":"pageY",s=this.items.length-1;s>=0;s--)t.contains(this.containers[p].element[0],this.items[s].item[0])&&this.items[s].item[0]!==this.currentItem[0]&&(h=this.items[s].item.offset()[a],l=!1,e[u]-h>this.items[s][r]/2&&(l=!0),n>Math.abs(e[u]-h)&&(n=Math.abs(e[u]-h),o=this.items[s],this.direction=l?"up":"down"));if(!o&&!this.options.dropOnEmpty)return;if(this.currentContainer===this.containers[p])return this.currentContainer.containerCache.over||(this.containers[p]._trigger("over",e,this._uiHash()),this.currentContainer.containerCache.over=1),void 0;o?this._rearrange(e,o,null,!0):this._rearrange(e,null,this.containers[p].element,!0),this._trigger("change",e,this._uiHash()),this.containers[p]._trigger("change",e,this._uiHash(this)),this.currentContainer=this.containers[p],this.options.placeholder.update(this.currentContainer,this.placeholder),this.containers[p]._trigger("over",e,this._uiHash(this)),this.containers[p].containerCache.over=1}},_createHelper:function(e){var i=this.options,s=t.isFunction(i.helper)?t(i.helper.apply(this.element[0],[e,this.currentItem])):"clone"===i.helper?this.currentItem.clone():this.currentItem;return s.parents("body").length||t("parent"!==i.appendTo?i.appendTo:this.currentItem[0].parentNode)[0].appendChild(s[0]),s[0]===this.currentItem[0]&&(this._storedCSS={width:this.currentItem[0].style.width,height:this.currentItem[0].style.height,position:this.currentItem.css("position"),top:this.currentItem.css("top"),left:this.currentItem.css("left")}),(!s[0].style.width||i.forceHelperSize)&&s.width(this.currentItem.width()),(!s[0].style.height||i.forceHelperSize)&&s.height(this.currentItem.height()),s},_adjustOffsetFromHelper:function(e){"string"==typeof e&&(e=e.split(" ")),t.isArray(e)&&(e={left:+e[0],top:+e[1]||0}),"left"in e&&(this.offset.click.left=e.left+this.margins.left),"right"in e&&(this.offset.click.left=this.helperProportions.width-e.right+this.margins.left),"top"in e&&(this.offset.click.top=e.top+this.margins.top),"bottom"in e&&(this.offset.click.top=this.helperProportions.height-e.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var e=this.offsetParent.offset();return"absolute"===this.cssPosition&&this.scrollParent[0]!==this.document[0]&&t.contains(this.scrollParent[0],this.offsetParent[0])&&(e.left+=this.scrollParent.scrollLeft(),e.top+=this.scrollParent.scrollTop()),(this.offsetParent[0]===this.document[0].body||this.offsetParent[0].tagName&&"html"===this.offsetParent[0].tagName.toLowerCase()&&t.ui.ie)&&(e={top:0,left:0}),{top:e.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:e.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if("relative"===this.cssPosition){var t=this.currentItem.position();return{top:t.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:t.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.currentItem.css("marginLeft"),10)||0,top:parseInt(this.currentItem.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var e,i,s,n=this.options;"parent"===n.containment&&(n.containment=this.helper[0].parentNode),("document"===n.containment||"window"===n.containment)&&(this.containment=[0-this.offset.relative.left-this.offset.parent.left,0-this.offset.relative.top-this.offset.parent.top,"document"===n.containment?this.document.width():this.window.width()-this.helperProportions.width-this.margins.left,("document"===n.containment?this.document.height()||document.body.parentNode.scrollHeight:this.window.height()||this.document[0].body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top]),/^(document|window|parent)$/.test(n.containment)||(e=t(n.containment)[0],i=t(n.containment).offset(),s="hidden"!==t(e).css("overflow"),this.containment=[i.left+(parseInt(t(e).css("borderLeftWidth"),10)||0)+(parseInt(t(e).css("paddingLeft"),10)||0)-this.margins.left,i.top+(parseInt(t(e).css("borderTopWidth"),10)||0)+(parseInt(t(e).css("paddingTop"),10)||0)-this.margins.top,i.left+(s?Math.max(e.scrollWidth,e.offsetWidth):e.offsetWidth)-(parseInt(t(e).css("borderLeftWidth"),10)||0)-(parseInt(t(e).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left,i.top+(s?Math.max(e.scrollHeight,e.offsetHeight):e.offsetHeight)-(parseInt(t(e).css("borderTopWidth"),10)||0)-(parseInt(t(e).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top])},_convertPositionTo:function(e,i){i||(i=this.position);var s="absolute"===e?1:-1,n="absolute"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&t.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,o=/(html|body)/i.test(n[0].tagName);return{top:i.top+this.offset.relative.top*s+this.offset.parent.top*s-("fixed"===this.cssPosition?-this.scrollParent.scrollTop():o?0:n.scrollTop())*s,left:i.left+this.offset.relative.left*s+this.offset.parent.left*s-("fixed"===this.cssPosition?-this.scrollParent.scrollLeft():o?0:n.scrollLeft())*s}},_generatePosition:function(e){var i,s,n=this.options,o=e.pageX,a=e.pageY,r="absolute"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&t.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,h=/(html|body)/i.test(r[0].tagName);return"relative"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&this.scrollParent[0]!==this.offsetParent[0]||(this.offset.relative=this._getRelativeOffset()),this.originalPosition&&(this.containment&&(e.pageX-this.offset.click.left<this.containment[0]&&(o=this.containment[0]+this.offset.click.left),e.pageY-this.offset.click.top<this.containment[1]&&(a=this.containment[1]+this.offset.click.top),e.pageX-this.offset.click.left>this.containment[2]&&(o=this.containment[2]+this.offset.click.left),e.pageY-this.offset.click.top>this.containment[3]&&(a=this.containment[3]+this.offset.click.top)),n.grid&&(i=this.originalPageY+Math.round((a-this.originalPageY)/n.grid[1])*n.grid[1],a=this.containment?i-this.offset.click.top>=this.containment[1]&&i-this.offset.click.top<=this.containment[3]?i:i-this.offset.click.top>=this.containment[1]?i-n.grid[1]:i+n.grid[1]:i,s=this.originalPageX+Math.round((o-this.originalPageX)/n.grid[0])*n.grid[0],o=this.containment?s-this.offset.click.left>=this.containment[0]&&s-this.offset.click.left<=this.containment[2]?s:s-this.offset.click.left>=this.containment[0]?s-n.grid[0]:s+n.grid[0]:s)),{top:a-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+("fixed"===this.cssPosition?-this.scrollParent.scrollTop():h?0:r.scrollTop()),left:o-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+("fixed"===this.cssPosition?-this.scrollParent.scrollLeft():h?0:r.scrollLeft())}},_rearrange:function(t,e,i,s){i?i[0].appendChild(this.placeholder[0]):e.item[0].parentNode.insertBefore(this.placeholder[0],"down"===this.direction?e.item[0]:e.item[0].nextSibling),this.counter=this.counter?++this.counter:1;var n=this.counter;
-this._delay(function(){n===this.counter&&this.refreshPositions(!s)})},_clear:function(t,e){function i(t,e,i){return function(s){i._trigger(t,s,e._uiHash(e))}}this.reverting=!1;var s,n=[];if(!this._noFinalSort&&this.currentItem.parent().length&&this.placeholder.before(this.currentItem),this._noFinalSort=null,this.helper[0]===this.currentItem[0]){for(s in this._storedCSS)("auto"===this._storedCSS[s]||"static"===this._storedCSS[s])&&(this._storedCSS[s]="");this.currentItem.css(this._storedCSS),this._removeClass(this.currentItem,"ui-sortable-helper")}else this.currentItem.show();for(this.fromOutside&&!e&&n.push(function(t){this._trigger("receive",t,this._uiHash(this.fromOutside))}),!this.fromOutside&&this.domPosition.prev===this.currentItem.prev().not(".ui-sortable-helper")[0]&&this.domPosition.parent===this.currentItem.parent()[0]||e||n.push(function(t){this._trigger("update",t,this._uiHash())}),this!==this.currentContainer&&(e||(n.push(function(t){this._trigger("remove",t,this._uiHash())}),n.push(function(t){return function(e){t._trigger("receive",e,this._uiHash(this))}}.call(this,this.currentContainer)),n.push(function(t){return function(e){t._trigger("update",e,this._uiHash(this))}}.call(this,this.currentContainer)))),s=this.containers.length-1;s>=0;s--)e||n.push(i("deactivate",this,this.containers[s])),this.containers[s].containerCache.over&&(n.push(i("out",this,this.containers[s])),this.containers[s].containerCache.over=0);if(this.storedCursor&&(this.document.find("body").css("cursor",this.storedCursor),this.storedStylesheet.remove()),this._storedOpacity&&this.helper.css("opacity",this._storedOpacity),this._storedZIndex&&this.helper.css("zIndex","auto"===this._storedZIndex?"":this._storedZIndex),this.dragging=!1,e||this._trigger("beforeStop",t,this._uiHash()),this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.cancelHelperRemoval||(this.helper[0]!==this.currentItem[0]&&this.helper.remove(),this.helper=null),!e){for(s=0;n.length>s;s++)n[s].call(this,t);this._trigger("stop",t,this._uiHash())}return this.fromOutside=!1,!this.cancelHelperRemoval},_trigger:function(){t.Widget.prototype._trigger.apply(this,arguments)===!1&&this.cancel()},_uiHash:function(e){var i=e||this;return{helper:i.helper,placeholder:i.placeholder||t([]),position:i.position,originalPosition:i.originalPosition,offset:i.positionAbs,item:i.currentItem,sender:e?e.element:null}}}),t.widget("ui.spinner",{version:"1.12.1",defaultElement:"<input>",widgetEventPrefix:"spin",options:{classes:{"ui-spinner":"ui-corner-all","ui-spinner-down":"ui-corner-br","ui-spinner-up":"ui-corner-tr"},culture:null,icons:{down:"ui-icon-triangle-1-s",up:"ui-icon-triangle-1-n"},incremental:!0,max:null,min:null,numberFormat:null,page:10,step:1,change:null,spin:null,start:null,stop:null},_create:function(){this._setOption("max",this.options.max),this._setOption("min",this.options.min),this._setOption("step",this.options.step),""!==this.value()&&this._value(this.element.val(),!0),this._draw(),this._on(this._events),this._refresh(),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_getCreateOptions:function(){var e=this._super(),i=this.element;return t.each(["min","max","step"],function(t,s){var n=i.attr(s);null!=n&&n.length&&(e[s]=n)}),e},_events:{keydown:function(t){this._start(t)&&this._keydown(t)&&t.preventDefault()},keyup:"_stop",focus:function(){this.previous=this.element.val()},blur:function(t){return this.cancelBlur?(delete this.cancelBlur,void 0):(this._stop(),this._refresh(),this.previous!==this.element.val()&&this._trigger("change",t),void 0)},mousewheel:function(t,e){if(e){if(!this.spinning&&!this._start(t))return!1;this._spin((e>0?1:-1)*this.options.step,t),clearTimeout(this.mousewheelTimer),this.mousewheelTimer=this._delay(function(){this.spinning&&this._stop(t)},100),t.preventDefault()}},"mousedown .ui-spinner-button":function(e){function i(){var e=this.element[0]===t.ui.safeActiveElement(this.document[0]);e||(this.element.trigger("focus"),this.previous=s,this._delay(function(){this.previous=s}))}var s;s=this.element[0]===t.ui.safeActiveElement(this.document[0])?this.previous:this.element.val(),e.preventDefault(),i.call(this),this.cancelBlur=!0,this._delay(function(){delete this.cancelBlur,i.call(this)}),this._start(e)!==!1&&this._repeat(null,t(e.currentTarget).hasClass("ui-spinner-up")?1:-1,e)},"mouseup .ui-spinner-button":"_stop","mouseenter .ui-spinner-button":function(e){return t(e.currentTarget).hasClass("ui-state-active")?this._start(e)===!1?!1:(this._repeat(null,t(e.currentTarget).hasClass("ui-spinner-up")?1:-1,e),void 0):void 0},"mouseleave .ui-spinner-button":"_stop"},_enhance:function(){this.uiSpinner=this.element.attr("autocomplete","off").wrap("<span>").parent().append("<a></a><a></a>")},_draw:function(){this._enhance(),this._addClass(this.uiSpinner,"ui-spinner","ui-widget ui-widget-content"),this._addClass("ui-spinner-input"),this.element.attr("role","spinbutton"),this.buttons=this.uiSpinner.children("a").attr("tabIndex",-1).attr("aria-hidden",!0).button({classes:{"ui-button":""}}),this._removeClass(this.buttons,"ui-corner-all"),this._addClass(this.buttons.first(),"ui-spinner-button ui-spinner-up"),this._addClass(this.buttons.last(),"ui-spinner-button ui-spinner-down"),this.buttons.first().button({icon:this.options.icons.up,showLabel:!1}),this.buttons.last().button({icon:this.options.icons.down,showLabel:!1}),this.buttons.height()>Math.ceil(.5*this.uiSpinner.height())&&this.uiSpinner.height()>0&&this.uiSpinner.height(this.uiSpinner.height())},_keydown:function(e){var i=this.options,s=t.ui.keyCode;switch(e.keyCode){case s.UP:return this._repeat(null,1,e),!0;case s.DOWN:return this._repeat(null,-1,e),!0;case s.PAGE_UP:return this._repeat(null,i.page,e),!0;case s.PAGE_DOWN:return this._repeat(null,-i.page,e),!0}return!1},_start:function(t){return this.spinning||this._trigger("start",t)!==!1?(this.counter||(this.counter=1),this.spinning=!0,!0):!1},_repeat:function(t,e,i){t=t||500,clearTimeout(this.timer),this.timer=this._delay(function(){this._repeat(40,e,i)},t),this._spin(e*this.options.step,i)},_spin:function(t,e){var i=this.value()||0;this.counter||(this.counter=1),i=this._adjustValue(i+t*this._increment(this.counter)),this.spinning&&this._trigger("spin",e,{value:i})===!1||(this._value(i),this.counter++)},_increment:function(e){var i=this.options.incremental;return i?t.isFunction(i)?i(e):Math.floor(e*e*e/5e4-e*e/500+17*e/200+1):1},_precision:function(){var t=this._precisionOf(this.options.step);return null!==this.options.min&&(t=Math.max(t,this._precisionOf(this.options.min))),t},_precisionOf:function(t){var e=""+t,i=e.indexOf(".");return-1===i?0:e.length-i-1},_adjustValue:function(t){var e,i,s=this.options;return e=null!==s.min?s.min:0,i=t-e,i=Math.round(i/s.step)*s.step,t=e+i,t=parseFloat(t.toFixed(this._precision())),null!==s.max&&t>s.max?s.max:null!==s.min&&s.min>t?s.min:t},_stop:function(t){this.spinning&&(clearTimeout(this.timer),clearTimeout(this.mousewheelTimer),this.counter=0,this.spinning=!1,this._trigger("stop",t))},_setOption:function(t,e){var i,s,n;return"culture"===t||"numberFormat"===t?(i=this._parse(this.element.val()),this.options[t]=e,this.element.val(this._format(i)),void 0):(("max"===t||"min"===t||"step"===t)&&"string"==typeof e&&(e=this._parse(e)),"icons"===t&&(s=this.buttons.first().find(".ui-icon"),this._removeClass(s,null,this.options.icons.up),this._addClass(s,null,e.up),n=this.buttons.last().find(".ui-icon"),this._removeClass(n,null,this.options.icons.down),this._addClass(n,null,e.down)),this._super(t,e),void 0)},_setOptionDisabled:function(t){this._super(t),this._toggleClass(this.uiSpinner,null,"ui-state-disabled",!!t),this.element.prop("disabled",!!t),this.buttons.button(t?"disable":"enable")},_setOptions:r(function(t){this._super(t)}),_parse:function(t){return"string"==typeof t&&""!==t&&(t=window.Globalize&&this.options.numberFormat?Globalize.parseFloat(t,10,this.options.culture):+t),""===t||isNaN(t)?null:t},_format:function(t){return""===t?"":window.Globalize&&this.options.numberFormat?Globalize.format(t,this.options.numberFormat,this.options.culture):t},_refresh:function(){this.element.attr({"aria-valuemin":this.options.min,"aria-valuemax":this.options.max,"aria-valuenow":this._parse(this.element.val())})},isValid:function(){var t=this.value();return null===t?!1:t===this._adjustValue(t)},_value:function(t,e){var i;""!==t&&(i=this._parse(t),null!==i&&(e||(i=this._adjustValue(i)),t=this._format(i))),this.element.val(t),this._refresh()},_destroy:function(){this.element.prop("disabled",!1).removeAttr("autocomplete role aria-valuemin aria-valuemax aria-valuenow"),this.uiSpinner.replaceWith(this.element)},stepUp:r(function(t){this._stepUp(t)}),_stepUp:function(t){this._start()&&(this._spin((t||1)*this.options.step),this._stop())},stepDown:r(function(t){this._stepDown(t)}),_stepDown:function(t){this._start()&&(this._spin((t||1)*-this.options.step),this._stop())},pageUp:r(function(t){this._stepUp((t||1)*this.options.page)}),pageDown:r(function(t){this._stepDown((t||1)*this.options.page)}),value:function(t){return arguments.length?(r(this._value).call(this,t),void 0):this._parse(this.element.val())},widget:function(){return this.uiSpinner}}),t.uiBackCompat!==!1&&t.widget("ui.spinner",t.ui.spinner,{_enhance:function(){this.uiSpinner=this.element.attr("autocomplete","off").wrap(this._uiSpinnerHtml()).parent().append(this._buttonHtml())},_uiSpinnerHtml:function(){return"<span>"},_buttonHtml:function(){return"<a></a><a></a>"}}),t.ui.spinner,t.widget("ui.tabs",{version:"1.12.1",delay:300,options:{active:null,classes:{"ui-tabs":"ui-corner-all","ui-tabs-nav":"ui-corner-all","ui-tabs-panel":"ui-corner-bottom","ui-tabs-tab":"ui-corner-top"},collapsible:!1,event:"click",heightStyle:"content",hide:null,show:null,activate:null,beforeActivate:null,beforeLoad:null,load:null},_isLocal:function(){var t=/#.*$/;return function(e){var i,s;i=e.href.replace(t,""),s=location.href.replace(t,"");try{i=decodeURIComponent(i)}catch(n){}try{s=decodeURIComponent(s)}catch(n){}return e.hash.length>1&&i===s}}(),_create:function(){var e=this,i=this.options;this.running=!1,this._addClass("ui-tabs","ui-widget ui-widget-content"),this._toggleClass("ui-tabs-collapsible",null,i.collapsible),this._processTabs(),i.active=this._initialActive(),t.isArray(i.disabled)&&(i.disabled=t.unique(i.disabled.concat(t.map(this.tabs.filter(".ui-state-disabled"),function(t){return e.tabs.index(t)}))).sort()),this.active=this.options.active!==!1&&this.anchors.length?this._findActive(i.active):t(),this._refresh(),this.active.length&&this.load(i.active)},_initialActive:function(){var e=this.options.active,i=this.options.collapsible,s=location.hash.substring(1);return null===e&&(s&&this.tabs.each(function(i,n){return t(n).attr("aria-controls")===s?(e=i,!1):void 0}),null===e&&(e=this.tabs.index(this.tabs.filter(".ui-tabs-active"))),(null===e||-1===e)&&(e=this.tabs.length?0:!1)),e!==!1&&(e=this.tabs.index(this.tabs.eq(e)),-1===e&&(e=i?!1:0)),!i&&e===!1&&this.anchors.length&&(e=0),e},_getCreateEventData:function(){return{tab:this.active,panel:this.active.length?this._getPanelForTab(this.active):t()}},_tabKeydown:function(e){var i=t(t.ui.safeActiveElement(this.document[0])).closest("li"),s=this.tabs.index(i),n=!0;if(!this._handlePageNav(e)){switch(e.keyCode){case t.ui.keyCode.RIGHT:case t.ui.keyCode.DOWN:s++;break;case t.ui.keyCode.UP:case t.ui.keyCode.LEFT:n=!1,s--;break;case t.ui.keyCode.END:s=this.anchors.length-1;break;case t.ui.keyCode.HOME:s=0;break;case t.ui.keyCode.SPACE:return e.preventDefault(),clearTimeout(this.activating),this._activate(s),void 0;case t.ui.keyCode.ENTER:return e.preventDefault(),clearTimeout(this.activating),this._activate(s===this.options.active?!1:s),void 0;default:return}e.preventDefault(),clearTimeout(this.activating),s=this._focusNextTab(s,n),e.ctrlKey||e.metaKey||(i.attr("aria-selected","false"),this.tabs.eq(s).attr("aria-selected","true"),this.activating=this._delay(function(){this.option("active",s)},this.delay))}},_panelKeydown:function(e){this._handlePageNav(e)||e.ctrlKey&&e.keyCode===t.ui.keyCode.UP&&(e.preventDefault(),this.active.trigger("focus"))},_handlePageNav:function(e){return e.altKey&&e.keyCode===t.ui.keyCode.PAGE_UP?(this._activate(this._focusNextTab(this.options.active-1,!1)),!0):e.altKey&&e.keyCode===t.ui.keyCode.PAGE_DOWN?(this._activate(this._focusNextTab(this.options.active+1,!0)),!0):void 0},_findNextTab:function(e,i){function s(){return e>n&&(e=0),0>e&&(e=n),e}for(var n=this.tabs.length-1;-1!==t.inArray(s(),this.options.disabled);)e=i?e+1:e-1;return e},_focusNextTab:function(t,e){return t=this._findNextTab(t,e),this.tabs.eq(t).trigger("focus"),t},_setOption:function(t,e){return"active"===t?(this._activate(e),void 0):(this._super(t,e),"collapsible"===t&&(this._toggleClass("ui-tabs-collapsible",null,e),e||this.options.active!==!1||this._activate(0)),"event"===t&&this._setupEvents(e),"heightStyle"===t&&this._setupHeightStyle(e),void 0)},_sanitizeSelector:function(t){return t?t.replace(/[!"$%&'()*+,.\/:;<=>?@\[\]\^`{|}~]/g,"\\$&"):""},refresh:function(){var e=this.options,i=this.tablist.children(":has(a[href])");e.disabled=t.map(i.filter(".ui-state-disabled"),function(t){return i.index(t)}),this._processTabs(),e.active!==!1&&this.anchors.length?this.active.length&&!t.contains(this.tablist[0],this.active[0])?this.tabs.length===e.disabled.length?(e.active=!1,this.active=t()):this._activate(this._findNextTab(Math.max(0,e.active-1),!1)):e.active=this.tabs.index(this.active):(e.active=!1,this.active=t()),this._refresh()},_refresh:function(){this._setOptionDisabled(this.options.disabled),this._setupEvents(this.options.event),this._setupHeightStyle(this.options.heightStyle),this.tabs.not(this.active).attr({"aria-selected":"false","aria-expanded":"false",tabIndex:-1}),this.panels.not(this._getPanelForTab(this.active)).hide().attr({"aria-hidden":"true"}),this.active.length?(this.active.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0}),this._addClass(this.active,"ui-tabs-active","ui-state-active"),this._getPanelForTab(this.active).show().attr({"aria-hidden":"false"})):this.tabs.eq(0).attr("tabIndex",0)},_processTabs:function(){var e=this,i=this.tabs,s=this.anchors,n=this.panels;this.tablist=this._getList().attr("role","tablist"),this._addClass(this.tablist,"ui-tabs-nav","ui-helper-reset ui-helper-clearfix ui-widget-header"),this.tablist.on("mousedown"+this.eventNamespace,"> li",function(e){t(this).is(".ui-state-disabled")&&e.preventDefault()}).on("focus"+this.eventNamespace,".ui-tabs-anchor",function(){t(this).closest("li").is(".ui-state-disabled")&&this.blur()}),this.tabs=this.tablist.find("> li:has(a[href])").attr({role:"tab",tabIndex:-1}),this._addClass(this.tabs,"ui-tabs-tab","ui-state-default"),this.anchors=this.tabs.map(function(){return t("a",this)[0]}).attr({role:"presentation",tabIndex:-1}),this._addClass(this.anchors,"ui-tabs-anchor"),this.panels=t(),this.anchors.each(function(i,s){var n,o,a,r=t(s).uniqueId().attr("id"),h=t(s).closest("li"),l=h.attr("aria-controls");e._isLocal(s)?(n=s.hash,a=n.substring(1),o=e.element.find(e._sanitizeSelector(n))):(a=h.attr("aria-controls")||t({}).uniqueId()[0].id,n="#"+a,o=e.element.find(n),o.length||(o=e._createPanel(a),o.insertAfter(e.panels[i-1]||e.tablist)),o.attr("aria-live","polite")),o.length&&(e.panels=e.panels.add(o)),l&&h.data("ui-tabs-aria-controls",l),h.attr({"aria-controls":a,"aria-labelledby":r}),o.attr("aria-labelledby",r)}),this.panels.attr("role","tabpanel"),this._addClass(this.panels,"ui-tabs-panel","ui-widget-content"),i&&(this._off(i.not(this.tabs)),this._off(s.not(this.anchors)),this._off(n.not(this.panels)))},_getList:function(){return this.tablist||this.element.find("ol, ul").eq(0)},_createPanel:function(e){return t("<div>").attr("id",e).data("ui-tabs-destroy",!0)},_setOptionDisabled:function(e){var i,s,n;for(t.isArray(e)&&(e.length?e.length===this.anchors.length&&(e=!0):e=!1),n=0;s=this.tabs[n];n++)i=t(s),e===!0||-1!==t.inArray(n,e)?(i.attr("aria-disabled","true"),this._addClass(i,null,"ui-state-disabled")):(i.removeAttr("aria-disabled"),this._removeClass(i,null,"ui-state-disabled"));this.options.disabled=e,this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,e===!0)},_setupEvents:function(e){var i={};e&&t.each(e.split(" "),function(t,e){i[e]="_eventHandler"}),this._off(this.anchors.add(this.tabs).add(this.panels)),this._on(!0,this.anchors,{click:function(t){t.preventDefault()}}),this._on(this.anchors,i),this._on(this.tabs,{keydown:"_tabKeydown"}),this._on(this.panels,{keydown:"_panelKeydown"}),this._focusable(this.tabs),this._hoverable(this.tabs)},_setupHeightStyle:function(e){var i,s=this.element.parent();"fill"===e?(i=s.height(),i-=this.element.outerHeight()-this.element.height(),this.element.siblings(":visible").each(function(){var e=t(this),s=e.css("position");"absolute"!==s&&"fixed"!==s&&(i-=e.outerHeight(!0))}),this.element.children().not(this.panels).each(function(){i-=t(this).outerHeight(!0)}),this.panels.each(function(){t(this).height(Math.max(0,i-t(this).innerHeight()+t(this).height()))}).css("overflow","auto")):"auto"===e&&(i=0,this.panels.each(function(){i=Math.max(i,t(this).height("").height())}).height(i))},_eventHandler:function(e){var i=this.options,s=this.active,n=t(e.currentTarget),o=n.closest("li"),a=o[0]===s[0],r=a&&i.collapsible,h=r?t():this._getPanelForTab(o),l=s.length?this._getPanelForTab(s):t(),c={oldTab:s,oldPanel:l,newTab:r?t():o,newPanel:h};e.preventDefault(),o.hasClass("ui-state-disabled")||o.hasClass("ui-tabs-loading")||this.running||a&&!i.collapsible||this._trigger("beforeActivate",e,c)===!1||(i.active=r?!1:this.tabs.index(o),this.active=a?t():o,this.xhr&&this.xhr.abort(),l.length||h.length||t.error("jQuery UI Tabs: Mismatching fragment identifier."),h.length&&this.load(this.tabs.index(o),e),this._toggle(e,c))},_toggle:function(e,i){function s(){o.running=!1,o._trigger("activate",e,i)}function n(){o._addClass(i.newTab.closest("li"),"ui-tabs-active","ui-state-active"),a.length&&o.options.show?o._show(a,o.options.show,s):(a.show(),s())}var o=this,a=i.newPanel,r=i.oldPanel;this.running=!0,r.length&&this.options.hide?this._hide(r,this.options.hide,function(){o._removeClass(i.oldTab.closest("li"),"ui-tabs-active","ui-state-active"),n()}):(this._removeClass(i.oldTab.closest("li"),"ui-tabs-active","ui-state-active"),r.hide(),n()),r.attr("aria-hidden","true"),i.oldTab.attr({"aria-selected":"false","aria-expanded":"false"}),a.length&&r.length?i.oldTab.attr("tabIndex",-1):a.length&&this.tabs.filter(function(){return 0===t(this).attr("tabIndex")}).attr("tabIndex",-1),a.attr("aria-hidden","false"),i.newTab.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0})},_activate:function(e){var i,s=this._findActive(e);s[0]!==this.active[0]&&(s.length||(s=this.active),i=s.find(".ui-tabs-anchor")[0],this._eventHandler({target:i,currentTarget:i,preventDefault:t.noop}))},_findActive:function(e){return e===!1?t():this.tabs.eq(e)},_getIndex:function(e){return"string"==typeof e&&(e=this.anchors.index(this.anchors.filter("[href$='"+t.ui.escapeSelector(e)+"']"))),e},_destroy:function(){this.xhr&&this.xhr.abort(),this.tablist.removeAttr("role").off(this.eventNamespace),this.anchors.removeAttr("role tabIndex").removeUniqueId(),this.tabs.add(this.panels).each(function(){t.data(this,"ui-tabs-destroy")?t(this).remove():t(this).removeAttr("role tabIndex aria-live aria-busy aria-selected aria-labelledby aria-hidden aria-expanded")}),this.tabs.each(function(){var e=t(this),i=e.data("ui-tabs-aria-controls");i?e.attr("aria-controls",i).removeData("ui-tabs-aria-controls"):e.removeAttr("aria-controls")}),this.panels.show(),"content"!==this.options.heightStyle&&this.panels.css("height","")},enable:function(e){var i=this.options.disabled;i!==!1&&(void 0===e?i=!1:(e=this._getIndex(e),i=t.isArray(i)?t.map(i,function(t){return t!==e?t:null}):t.map(this.tabs,function(t,i){return i!==e?i:null})),this._setOptionDisabled(i))},disable:function(e){var i=this.options.disabled;if(i!==!0){if(void 0===e)i=!0;else{if(e=this._getIndex(e),-1!==t.inArray(e,i))return;i=t.isArray(i)?t.merge([e],i).sort():[e]}this._setOptionDisabled(i)}},load:function(e,i){e=this._getIndex(e);var s=this,n=this.tabs.eq(e),o=n.find(".ui-tabs-anchor"),a=this._getPanelForTab(n),r={tab:n,panel:a},h=function(t,e){"abort"===e&&s.panels.stop(!1,!0),s._removeClass(n,"ui-tabs-loading"),a.removeAttr("aria-busy"),t===s.xhr&&delete s.xhr};this._isLocal(o[0])||(this.xhr=t.ajax(this._ajaxSettings(o,i,r)),this.xhr&&"canceled"!==this.xhr.statusText&&(this._addClass(n,"ui-tabs-loading"),a.attr("aria-busy","true"),this.xhr.done(function(t,e,n){setTimeout(function(){a.html(t),s._trigger("load",i,r),h(n,e)},1)}).fail(function(t,e){setTimeout(function(){h(t,e)},1)})))},_ajaxSettings:function(e,i,s){var n=this;return{url:e.attr("href").replace(/#.*$/,""),beforeSend:function(e,o){return n._trigger("beforeLoad",i,t.extend({jqXHR:e,ajaxSettings:o},s))}}},_getPanelForTab:function(e){var i=t(e).attr("aria-controls");return this.element.find(this._sanitizeSelector("#"+i))}}),t.uiBackCompat!==!1&&t.widget("ui.tabs",t.ui.tabs,{_processTabs:function(){this._superApply(arguments),this._addClass(this.tabs,"ui-tab")}}),t.ui.tabs,t.widget("ui.tooltip",{version:"1.12.1",options:{classes:{"ui-tooltip":"ui-corner-all ui-widget-shadow"},content:function(){var e=t(this).attr("title")||"";return t("<a>").text(e).html()},hide:!0,items:"[title]:not([disabled])",position:{my:"left top+15",at:"left bottom",collision:"flipfit flip"},show:!0,track:!1,close:null,open:null},_addDescribedBy:function(e,i){var s=(e.attr("aria-describedby")||"").split(/\s+/);s.push(i),e.data("ui-tooltip-id",i).attr("aria-describedby",t.trim(s.join(" ")))},_removeDescribedBy:function(e){var i=e.data("ui-tooltip-id"),s=(e.attr("aria-describedby")||"").split(/\s+/),n=t.inArray(i,s);-1!==n&&s.splice(n,1),e.removeData("ui-tooltip-id"),s=t.trim(s.join(" ")),s?e.attr("aria-describedby",s):e.removeAttr("aria-describedby")},_create:function(){this._on({mouseover:"open",focusin:"open"}),this.tooltips={},this.parents={},this.liveRegion=t("<div>").attr({role:"log","aria-live":"assertive","aria-relevant":"additions"}).appendTo(this.document[0].body),this._addClass(this.liveRegion,null,"ui-helper-hidden-accessible"),this.disabledTitles=t([])},_setOption:function(e,i){var s=this;this._super(e,i),"content"===e&&t.each(this.tooltips,function(t,e){s._updateContent(e.element)})},_setOptionDisabled:function(t){this[t?"_disable":"_enable"]()},_disable:function(){var e=this;t.each(this.tooltips,function(i,s){var n=t.Event("blur");n.target=n.currentTarget=s.element[0],e.close(n,!0)}),this.disabledTitles=this.disabledTitles.add(this.element.find(this.options.items).addBack().filter(function(){var e=t(this);return e.is("[title]")?e.data("ui-tooltip-title",e.attr("title")).removeAttr("title"):void 0}))},_enable:function(){this.disabledTitles.each(function(){var e=t(this);e.data("ui-tooltip-title")&&e.attr("title",e.data("ui-tooltip-title"))}),this.disabledTitles=t([])},open:function(e){var i=this,s=t(e?e.target:this.element).closest(this.options.items);s.length&&!s.data("ui-tooltip-id")&&(s.attr("title")&&s.data("ui-tooltip-title",s.attr("title")),s.data("ui-tooltip-open",!0),e&&"mouseover"===e.type&&s.parents().each(function(){var e,s=t(this);s.data("ui-tooltip-open")&&(e=t.Event("blur"),e.target=e.currentTarget=this,i.close(e,!0)),s.attr("title")&&(s.uniqueId(),i.parents[this.id]={element:this,title:s.attr("title")},s.attr("title",""))}),this._registerCloseHandlers(e,s),this._updateContent(s,e))},_updateContent:function(t,e){var i,s=this.options.content,n=this,o=e?e.type:null;return"string"==typeof s||s.nodeType||s.jquery?this._open(e,t,s):(i=s.call(t[0],function(i){n._delay(function(){t.data("ui-tooltip-open")&&(e&&(e.type=o),this._open(e,t,i))})}),i&&this._open(e,t,i),void 0)},_open:function(e,i,s){function n(t){l.of=t,a.is(":hidden")||a.position(l)}var o,a,r,h,l=t.extend({},this.options.position);if(s){if(o=this._find(i))return o.tooltip.find(".ui-tooltip-content").html(s),void 0;i.is("[title]")&&(e&&"mouseover"===e.type?i.attr("title",""):i.removeAttr("title")),o=this._tooltip(i),a=o.tooltip,this._addDescribedBy(i,a.attr("id")),a.find(".ui-tooltip-content").html(s),this.liveRegion.children().hide(),h=t("<div>").html(a.find(".ui-tooltip-content").html()),h.removeAttr("name").find("[name]").removeAttr("name"),h.removeAttr("id").find("[id]").removeAttr("id"),h.appendTo(this.liveRegion),this.options.track&&e&&/^mouse/.test(e.type)?(this._on(this.document,{mousemove:n}),n(e)):a.position(t.extend({of:i},this.options.position)),a.hide(),this._show(a,this.options.show),this.options.track&&this.options.show&&this.options.show.delay&&(r=this.delayedShow=setInterval(function(){a.is(":visible")&&(n(l.of),clearInterval(r))},t.fx.interval)),this._trigger("open",e,{tooltip:a})}},_registerCloseHandlers:function(e,i){var s={keyup:function(e){if(e.keyCode===t.ui.keyCode.ESCAPE){var s=t.Event(e);s.currentTarget=i[0],this.close(s,!0)}}};i[0]!==this.element[0]&&(s.remove=function(){this._removeTooltip(this._find(i).tooltip)}),e&&"mouseover"!==e.type||(s.mouseleave="close"),e&&"focusin"!==e.type||(s.focusout="close"),this._on(!0,i,s)},close:function(e){var i,s=this,n=t(e?e.currentTarget:this.element),o=this._find(n);return o?(i=o.tooltip,o.closing||(clearInterval(this.delayedShow),n.data("ui-tooltip-title")&&!n.attr("title")&&n.attr("title",n.data("ui-tooltip-title")),this._removeDescribedBy(n),o.hiding=!0,i.stop(!0),this._hide(i,this.options.hide,function(){s._removeTooltip(t(this))}),n.removeData("ui-tooltip-open"),this._off(n,"mouseleave focusout keyup"),n[0]!==this.element[0]&&this._off(n,"remove"),this._off(this.document,"mousemove"),e&&"mouseleave"===e.type&&t.each(this.parents,function(e,i){t(i.element).attr("title",i.title),delete s.parents[e]}),o.closing=!0,this._trigger("close",e,{tooltip:i}),o.hiding||(o.closing=!1)),void 0):(n.removeData("ui-tooltip-open"),void 0)},_tooltip:function(e){var i=t("<div>").attr("role","tooltip"),s=t("<div>").appendTo(i),n=i.uniqueId().attr("id");return this._addClass(s,"ui-tooltip-content"),this._addClass(i,"ui-tooltip","ui-widget ui-widget-content"),i.appendTo(this._appendTo(e)),this.tooltips[n]={element:e,tooltip:i}},_find:function(t){var e=t.data("ui-tooltip-id");return e?this.tooltips[e]:null},_removeTooltip:function(t){t.remove(),delete this.tooltips[t.attr("id")]},_appendTo:function(t){var e=t.closest(".ui-front, dialog");return e.length||(e=this.document[0].body),e},_destroy:function(){var e=this;t.each(this.tooltips,function(i,s){var n=t.Event("blur"),o=s.element;n.target=n.currentTarget=o[0],e.close(n,!0),t("#"+i).remove(),o.data("ui-tooltip-title")&&(o.attr("title")||o.attr("title",o.data("ui-tooltip-title")),o.removeData("ui-tooltip-title"))}),this.liveRegion.remove()}}),t.uiBackCompat!==!1&&t.widget("ui.tooltip",t.ui.tooltip,{options:{tooltipClass:null},_tooltip:function(){var t=this._superApply(arguments);return this.options.tooltipClass&&t.tooltip.addClass(this.options.tooltipClass),t}}),t.ui.tooltip});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.13.2.custom.min.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.13.2.custom.min.js
new file mode 100644
index 0000000000000..50b036f4d2131
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-ui-1.13.2.custom.min.js
@@ -0,0 +1,6 @@
+/*! jQuery UI - v1.13.2 - 2022-07-14
+* http://jqueryui.com
+* Includes: widget.js, position.js, data.js, disable-selection.js, effect.js, effects/effect-blind.js, effects/effect-bounce.js, effects/effect-clip.js, effects/effect-drop.js, effects/effect-explode.js, effects/effect-fade.js, effects/effect-fold.js, effects/effect-highlight.js, effects/effect-puff.js, effects/effect-pulsate.js, effects/effect-scale.js, effects/effect-shake.js, effects/effect-size.js, effects/effect-slide.js, effects/effect-transfer.js, focusable.js, form-reset-mixin.js, jquery-patch.js, keycode.js, labels.js, scroll-parent.js, tabbable.js, unique-id.js, widgets/accordion.js, widgets/autocomplete.js, widgets/button.js, widgets/checkboxradio.js, widgets/controlgroup.js, widgets/datepicker.js, widgets/dialog.js, widgets/draggable.js, widgets/droppable.js, widgets/menu.js, widgets/mouse.js, widgets/progressbar.js, widgets/resizable.js, widgets/selectable.js, widgets/selectmenu.js, widgets/slider.js, widgets/sortable.js, widgets/spinner.js, widgets/tabs.js, widgets/tooltip.js
+* Copyright jQuery Foundation and other contributors; Licensed MIT */
+
+!function(t){"use strict";"function"==typeof define&&define.amd?define(["jquery"],t):t(jQuery)}(function(V){"use strict";V.ui=V.ui||{};V.ui.version="1.13.2";var n,i=0,a=Array.prototype.hasOwnProperty,r=Array.prototype.slice;V.cleanData=(n=V.cleanData,function(t){for(var e,i,s=0;null!=(i=t[s]);s++)(e=V._data(i,"events"))&&e.remove&&V(i).triggerHandler("remove");n(t)}),V.widget=function(t,i,e){var s,n,o,a={},r=t.split(".")[0],l=r+"-"+(t=t.split(".")[1]);return e||(e=i,i=V.Widget),Array.isArray(e)&&(e=V.extend.apply(null,[{}].concat(e))),V.expr.pseudos[l.toLowerCase()]=function(t){return!!V.data(t,l)},V[r]=V[r]||{},s=V[r][t],n=V[r][t]=function(t,e){if(!this||!this._createWidget)return new n(t,e);arguments.length&&this._createWidget(t,e)},V.extend(n,s,{version:e.version,_proto:V.extend({},e),_childConstructors:[]}),(o=new i).options=V.widget.extend({},o.options),V.each(e,function(e,s){function n(){return i.prototype[e].apply(this,arguments)}function o(t){return i.prototype[e].apply(this,t)}a[e]="function"==typeof s?function(){var t,e=this._super,i=this._superApply;return this._super=n,this._superApply=o,t=s.apply(this,arguments),this._super=e,this._superApply=i,t}:s}),n.prototype=V.widget.extend(o,{widgetEventPrefix:s&&o.widgetEventPrefix||t},a,{constructor:n,namespace:r,widgetName:t,widgetFullName:l}),s?(V.each(s._childConstructors,function(t,e){var i=e.prototype;V.widget(i.namespace+"."+i.widgetName,n,e._proto)}),delete s._childConstructors):i._childConstructors.push(n),V.widget.bridge(t,n),n},V.widget.extend=function(t){for(var e,i,s=r.call(arguments,1),n=0,o=s.length;n<o;n++)for(e in s[n])i=s[n][e],a.call(s[n],e)&&void 0!==i&&(V.isPlainObject(i)?t[e]=V.isPlainObject(t[e])?V.widget.extend({},t[e],i):V.widget.extend({},i):t[e]=i);return t},V.widget.bridge=function(o,e){var a=e.prototype.widgetFullName||o;V.fn[o]=function(i){var t="string"==typeof i,s=r.call(arguments,1),n=this;return t?this.length||"instance"!==i?this.each(function(){var t,e=V.data(this,a);return"instance"===i?(n=e,!1):e?"function"!=typeof e[i]||"_"===i.charAt(0)?V.error("no such method '"+i+"' for "+o+" widget instance"):(t=e[i].apply(e,s))!==e&&void 0!==t?(n=t&&t.jquery?n.pushStack(t.get()):t,!1):void 0:V.error("cannot call methods on "+o+" prior to initialization; attempted to call method '"+i+"'")}):n=void 0:(s.length&&(i=V.widget.extend.apply(null,[i].concat(s))),this.each(function(){var t=V.data(this,a);t?(t.option(i||{}),t._init&&t._init()):V.data(this,a,new e(i,this))})),n}},V.Widget=function(){},V.Widget._childConstructors=[],V.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",defaultElement:"<div>",options:{classes:{},disabled:!1,create:null},_createWidget:function(t,e){e=V(e||this.defaultElement||this)[0],this.element=V(e),this.uuid=i++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=V(),this.hoverable=V(),this.focusable=V(),this.classesElementLookup={},e!==this&&(V.data(e,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===e&&this.destroy()}}),this.document=V(e.style?e.ownerDocument:e.document||e),this.window=V(this.document[0].defaultView||this.document[0].parentWindow)),this.options=V.widget.extend({},this.options,this._getCreateOptions(),t),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:V.noop,_create:V.noop,_init:V.noop,destroy:function(){var i=this;this._destroy(),V.each(this.classesElementLookup,function(t,e){i._removeClass(e,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:V.noop,widget:function(){return this.element},option:function(t,e){var i,s,n,o=t;if(0===arguments.length)return V.widget.extend({},this.options);if("string"==typeof t)if(o={},t=(i=t.split(".")).shift(),i.length){for(s=o[t]=V.widget.extend({},this.options[t]),n=0;n<i.length-1;n++)s[i[n]]=s[i[n]]||{},s=s[i[n]];if(t=i.pop(),1===arguments.length)return void 0===s[t]?null:s[t];s[t]=e}else{if(1===arguments.length)return void 0===this.options[t]?null:this.options[t];o[t]=e}return this._setOptions(o),this},_setOptions:function(t){for(var e in t)this._setOption(e,t[e]);return this},_setOption:function(t,e){return"classes"===t&&this._setOptionClasses(e),this.options[t]=e,"disabled"===t&&this._setOptionDisabled(e),this},_setOptionClasses:function(t){var e,i,s;for(e in t)s=this.classesElementLookup[e],t[e]!==this.options.classes[e]&&s&&s.length&&(i=V(s.get()),this._removeClass(s,e),i.addClass(this._classes({element:i,keys:e,classes:t,add:!0})))},_setOptionDisabled:function(t){this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,!!t),t&&(this._removeClass(this.hoverable,null,"ui-state-hover"),this._removeClass(this.focusable,null,"ui-state-focus"))},enable:function(){return this._setOptions({disabled:!1})},disable:function(){return this._setOptions({disabled:!0})},_classes:function(n){var o=[],a=this;function t(t,e){for(var i,s=0;s<t.length;s++)i=a.classesElementLookup[t[s]]||V(),i=n.add?(function(){var i=[];n.element.each(function(t,e){V.map(a.classesElementLookup,function(t){return t}).some(function(t){return t.is(e)})||i.push(e)}),a._on(V(i),{remove:"_untrackClassesElement"})}(),V(V.uniqueSort(i.get().concat(n.element.get())))):V(i.not(n.element).get()),a.classesElementLookup[t[s]]=i,o.push(t[s]),e&&n.classes[t[s]]&&o.push(n.classes[t[s]])}return(n=V.extend({element:this.element,classes:this.options.classes||{}},n)).keys&&t(n.keys.match(/\S+/g)||[],!0),n.extra&&t(n.extra.match(/\S+/g)||[]),o.join(" ")},_untrackClassesElement:function(i){var s=this;V.each(s.classesElementLookup,function(t,e){-1!==V.inArray(i.target,e)&&(s.classesElementLookup[t]=V(e.not(i.target).get()))}),this._off(V(i.target))},_removeClass:function(t,e,i){return this._toggleClass(t,e,i,!1)},_addClass:function(t,e,i){return this._toggleClass(t,e,i,!0)},_toggleClass:function(t,e,i,s){var n="string"==typeof t||null===t,i={extra:n?e:i,keys:n?t:e,element:n?this.element:t,add:s="boolean"==typeof s?s:i};return i.element.toggleClass(this._classes(i),s),this},_on:function(n,o,t){var a,r=this;"boolean"!=typeof n&&(t=o,o=n,n=!1),t?(o=a=V(o),this.bindings=this.bindings.add(o)):(t=o,o=this.element,a=this.widget()),V.each(t,function(t,e){function i(){if(n||!0!==r.options.disabled&&!V(this).hasClass("ui-state-disabled"))return("string"==typeof e?r[e]:e).apply(r,arguments)}"string"!=typeof e&&(i.guid=e.guid=e.guid||i.guid||V.guid++);var s=t.match(/^([\w:-]*)\s*(.*)$/),t=s[1]+r.eventNamespace,s=s[2];s?a.on(t,s,i):o.on(t,i)})},_off:function(t,e){e=(e||"").split(" ").join(this.eventNamespace+" ")+this.eventNamespace,t.off(e),this.bindings=V(this.bindings.not(t).get()),this.focusable=V(this.focusable.not(t).get()),this.hoverable=V(this.hoverable.not(t).get())},_delay:function(t,e){var i=this;return setTimeout(function(){return("string"==typeof t?i[t]:t).apply(i,arguments)},e||0)},_hoverable:function(t){this.hoverable=this.hoverable.add(t),this._on(t,{mouseenter:function(t){this._addClass(V(t.currentTarget),null,"ui-state-hover")},mouseleave:function(t){this._removeClass(V(t.currentTarget),null,"ui-state-hover")}})},_focusable:function(t){this.focusable=this.focusable.add(t),this._on(t,{focusin:function(t){this._addClass(V(t.currentTarget),null,"ui-state-focus")},focusout:function(t){this._removeClass(V(t.currentTarget),null,"ui-state-focus")}})},_trigger:function(t,e,i){var s,n,o=this.options[t];if(i=i||{},(e=V.Event(e)).type=(t===this.widgetEventPrefix?t:this.widgetEventPrefix+t).toLowerCase(),e.target=this.element[0],n=e.originalEvent)for(s in n)s in e||(e[s]=n[s]);return this.element.trigger(e,i),!("function"==typeof o&&!1===o.apply(this.element[0],[e].concat(i))||e.isDefaultPrevented())}},V.each({show:"fadeIn",hide:"fadeOut"},function(o,a){V.Widget.prototype["_"+o]=function(e,t,i){var s,n=(t="string"==typeof t?{effect:t}:t)?!0!==t&&"number"!=typeof t&&t.effect||a:o;"number"==typeof(t=t||{})?t={duration:t}:!0===t&&(t={}),s=!V.isEmptyObject(t),t.complete=i,t.delay&&e.delay(t.delay),s&&V.effects&&V.effects.effect[n]?e[o](t):n!==o&&e[n]?e[n](t.duration,t.easing,i):e.queue(function(t){V(this)[o](),i&&i.call(e[0]),t()})}});var s,x,k,o,l,h,c,u,C;V.widget;function D(t,e,i){return[parseFloat(t[0])*(u.test(t[0])?e/100:1),parseFloat(t[1])*(u.test(t[1])?i/100:1)]}function I(t,e){return parseInt(V.css(t,e),10)||0}function T(t){return null!=t&&t===t.window}x=Math.max,k=Math.abs,o=/left|center|right/,l=/top|center|bottom/,h=/[\+\-]\d+(\.[\d]+)?%?/,c=/^\w+/,u=/%$/,C=V.fn.position,V.position={scrollbarWidth:function(){if(void 0!==s)return s;var t,e=V("<div style='display:block;position:absolute;width:200px;height:200px;overflow:hidden;'><div style='height:300px;width:auto;'></div></div>"),i=e.children()[0];return V("body").append(e),t=i.offsetWidth,e.css("overflow","scroll"),t===(i=i.offsetWidth)&&(i=e[0].clientWidth),e.remove(),s=t-i},getScrollInfo:function(t){var e=t.isWindow||t.isDocument?"":t.element.css("overflow-x"),i=t.isWindow||t.isDocument?"":t.element.css("overflow-y"),e="scroll"===e||"auto"===e&&t.width<t.element[0].scrollWidth;return{width:"scroll"===i||"auto"===i&&t.height<t.element[0].scrollHeight?V.position.scrollbarWidth():0,height:e?V.position.scrollbarWidth():0}},getWithinInfo:function(t){var e=V(t||window),i=T(e[0]),s=!!e[0]&&9===e[0].nodeType;return{element:e,isWindow:i,isDocument:s,offset:!i&&!s?V(t).offset():{left:0,top:0},scrollLeft:e.scrollLeft(),scrollTop:e.scrollTop(),width:e.outerWidth(),height:e.outerHeight()}}},V.fn.position=function(u){if(!u||!u.of)return C.apply(this,arguments);var d,p,f,g,m,t,_="string"==typeof(u=V.extend({},u)).of?V(document).find(u.of):V(u.of),v=V.position.getWithinInfo(u.within),b=V.position.getScrollInfo(v),y=(u.collision||"flip").split(" "),w={},e=9===(t=(e=_)[0]).nodeType?{width:e.width(),height:e.height(),offset:{top:0,left:0}}:T(t)?{width:e.width(),height:e.height(),offset:{top:e.scrollTop(),left:e.scrollLeft()}}:t.preventDefault?{width:0,height:0,offset:{top:t.pageY,left:t.pageX}}:{width:e.outerWidth(),height:e.outerHeight(),offset:e.offset()};return _[0].preventDefault&&(u.at="left top"),p=e.width,f=e.height,m=V.extend({},g=e.offset),V.each(["my","at"],function(){var t,e,i=(u[this]||"").split(" ");(i=1===i.length?o.test(i[0])?i.concat(["center"]):l.test(i[0])?["center"].concat(i):["center","center"]:i)[0]=o.test(i[0])?i[0]:"center",i[1]=l.test(i[1])?i[1]:"center",t=h.exec(i[0]),e=h.exec(i[1]),w[this]=[t?t[0]:0,e?e[0]:0],u[this]=[c.exec(i[0])[0],c.exec(i[1])[0]]}),1===y.length&&(y[1]=y[0]),"right"===u.at[0]?m.left+=p:"center"===u.at[0]&&(m.left+=p/2),"bottom"===u.at[1]?m.top+=f:"center"===u.at[1]&&(m.top+=f/2),d=D(w.at,p,f),m.left+=d[0],m.top+=d[1],this.each(function(){var i,t,a=V(this),r=a.outerWidth(),l=a.outerHeight(),e=I(this,"marginLeft"),s=I(this,"marginTop"),n=r+e+I(this,"marginRight")+b.width,o=l+s+I(this,"marginBottom")+b.height,h=V.extend({},m),c=D(w.my,a.outerWidth(),a.outerHeight());"right"===u.my[0]?h.left-=r:"center"===u.my[0]&&(h.left-=r/2),"bottom"===u.my[1]?h.top-=l:"center"===u.my[1]&&(h.top-=l/2),h.left+=c[0],h.top+=c[1],i={marginLeft:e,marginTop:s},V.each(["left","top"],function(t,e){V.ui.position[y[t]]&&V.ui.position[y[t]][e](h,{targetWidth:p,targetHeight:f,elemWidth:r,elemHeight:l,collisionPosition:i,collisionWidth:n,collisionHeight:o,offset:[d[0]+c[0],d[1]+c[1]],my:u.my,at:u.at,within:v,elem:a})}),u.using&&(t=function(t){var e=g.left-h.left,i=e+p-r,s=g.top-h.top,n=s+f-l,o={target:{element:_,left:g.left,top:g.top,width:p,height:f},element:{element:a,left:h.left,top:h.top,width:r,height:l},horizontal:i<0?"left":0<e?"right":"center",vertical:n<0?"top":0<s?"bottom":"middle"};p<r&&k(e+i)<p&&(o.horizontal="center"),f<l&&k(s+n)<f&&(o.vertical="middle"),x(k(e),k(i))>x(k(s),k(n))?o.important="horizontal":o.important="vertical",u.using.call(this,t,o)}),a.offset(V.extend(h,{using:t}))})},V.ui.position={fit:{left:function(t,e){var i=e.within,s=i.isWindow?i.scrollLeft:i.offset.left,n=i.width,o=t.left-e.collisionPosition.marginLeft,a=s-o,r=o+e.collisionWidth-n-s;e.collisionWidth>n?0<a&&r<=0?(i=t.left+a+e.collisionWidth-n-s,t.left+=a-i):t.left=!(0<r&&a<=0)&&r<a?s+n-e.collisionWidth:s:0<a?t.left+=a:0<r?t.left-=r:t.left=x(t.left-o,t.left)},top:function(t,e){var i=e.within,s=i.isWindow?i.scrollTop:i.offset.top,n=e.within.height,o=t.top-e.collisionPosition.marginTop,a=s-o,r=o+e.collisionHeight-n-s;e.collisionHeight>n?0<a&&r<=0?(i=t.top+a+e.collisionHeight-n-s,t.top+=a-i):t.top=!(0<r&&a<=0)&&r<a?s+n-e.collisionHeight:s:0<a?t.top+=a:0<r?t.top-=r:t.top=x(t.top-o,t.top)}},flip:{left:function(t,e){var i=e.within,s=i.offset.left+i.scrollLeft,n=i.width,o=i.isWindow?i.scrollLeft:i.offset.left,a=t.left-e.collisionPosition.marginLeft,r=a-o,l=a+e.collisionWidth-n-o,h="left"===e.my[0]?-e.elemWidth:"right"===e.my[0]?e.elemWidth:0,i="left"===e.at[0]?e.targetWidth:"right"===e.at[0]?-e.targetWidth:0,a=-2*e.offset[0];r<0?((s=t.left+h+i+a+e.collisionWidth-n-s)<0||s<k(r))&&(t.left+=h+i+a):0<l&&(0<(o=t.left-e.collisionPosition.marginLeft+h+i+a-o)||k(o)<l)&&(t.left+=h+i+a)},top:function(t,e){var i=e.within,s=i.offset.top+i.scrollTop,n=i.height,o=i.isWindow?i.scrollTop:i.offset.top,a=t.top-e.collisionPosition.marginTop,r=a-o,l=a+e.collisionHeight-n-o,h="top"===e.my[1]?-e.elemHeight:"bottom"===e.my[1]?e.elemHeight:0,i="top"===e.at[1]?e.targetHeight:"bottom"===e.at[1]?-e.targetHeight:0,a=-2*e.offset[1];r<0?((s=t.top+h+i+a+e.collisionHeight-n-s)<0||s<k(r))&&(t.top+=h+i+a):0<l&&(0<(o=t.top-e.collisionPosition.marginTop+h+i+a-o)||k(o)<l)&&(t.top+=h+i+a)}},flipfit:{left:function(){V.ui.position.flip.left.apply(this,arguments),V.ui.position.fit.left.apply(this,arguments)},top:function(){V.ui.position.flip.top.apply(this,arguments),V.ui.position.fit.top.apply(this,arguments)}}};V.ui.position,V.extend(V.expr.pseudos,{data:V.expr.createPseudo?V.expr.createPseudo(function(e){return function(t){return!!V.data(t,e)}}):function(t,e,i){return!!V.data(t,i[3])}}),V.fn.extend({disableSelection:(t="onselectstart"in document.createElement("div")?"selectstart":"mousedown",function(){return this.on(t+".ui-disableSelection",function(t){t.preventDefault()})}),enableSelection:function(){return this.off(".ui-disableSelection")}});var t,d=V,p={},e=p.toString,f=/^([\-+])=\s*(\d+\.?\d*)/,g=[{re:/rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,parse:function(t){return[t[1],t[2],t[3],t[4]]}},{re:/rgba?\(\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,parse:function(t){return[2.55*t[1],2.55*t[2],2.55*t[3],t[4]]}},{re:/#([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})([a-f0-9]{2})?/,parse:function(t){return[parseInt(t[1],16),parseInt(t[2],16),parseInt(t[3],16),t[4]?(parseInt(t[4],16)/255).toFixed(2):1]}},{re:/#([a-f0-9])([a-f0-9])([a-f0-9])([a-f0-9])?/,parse:function(t){return[parseInt(t[1]+t[1],16),parseInt(t[2]+t[2],16),parseInt(t[3]+t[3],16),t[4]?(parseInt(t[4]+t[4],16)/255).toFixed(2):1]}},{re:/hsla?\(\s*(\d+(?:\.\d+)?)\s*,\s*(\d+(?:\.\d+)?)\%\s*,\s*(\d+(?:\.\d+)?)\%\s*(?:,\s*(\d?(?:\.\d+)?)\s*)?\)/,space:"hsla",parse:function(t){return[t[1],t[2]/100,t[3]/100,t[4]]}}],m=d.Color=function(t,e,i,s){return new d.Color.fn.parse(t,e,i,s)},_={rgba:{props:{red:{idx:0,type:"byte"},green:{idx:1,type:"byte"},blue:{idx:2,type:"byte"}}},hsla:{props:{hue:{idx:0,type:"degrees"},saturation:{idx:1,type:"percent"},lightness:{idx:2,type:"percent"}}}},v={byte:{floor:!0,max:255},percent:{max:1},degrees:{mod:360,floor:!0}},b=m.support={},y=d("<p>")[0],w=d.each;function P(t){return null==t?t+"":"object"==typeof t?p[e.call(t)]||"object":typeof t}function M(t,e,i){var s=v[e.type]||{};return null==t?i||!e.def?null:e.def:(t=s.floor?~~t:parseFloat(t),isNaN(t)?e.def:s.mod?(t+s.mod)%s.mod:Math.min(s.max,Math.max(0,t)))}function S(s){var n=m(),o=n._rgba=[];return s=s.toLowerCase(),w(g,function(t,e){var i=e.re.exec(s),i=i&&e.parse(i),e=e.space||"rgba";if(i)return i=n[e](i),n[_[e].cache]=i[_[e].cache],o=n._rgba=i._rgba,!1}),o.length?("0,0,0,0"===o.join()&&d.extend(o,B.transparent),n):B[s]}function H(t,e,i){return 6*(i=(i+1)%1)<1?t+(e-t)*i*6:2*i<1?e:3*i<2?t+(e-t)*(2/3-i)*6:t}y.style.cssText="background-color:rgba(1,1,1,.5)",b.rgba=-1<y.style.backgroundColor.indexOf("rgba"),w(_,function(t,e){e.cache="_"+t,e.props.alpha={idx:3,type:"percent",def:1}}),d.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(t,e){p["[object "+e+"]"]=e.toLowerCase()}),(m.fn=d.extend(m.prototype,{parse:function(n,t,e,i){if(void 0===n)return this._rgba=[null,null,null,null],this;(n.jquery||n.nodeType)&&(n=d(n).css(t),t=void 0);var o=this,s=P(n),a=this._rgba=[];return void 0!==t&&(n=[n,t,e,i],s="array"),"string"===s?this.parse(S(n)||B._default):"array"===s?(w(_.rgba.props,function(t,e){a[e.idx]=M(n[e.idx],e)}),this):"object"===s?(w(_,n instanceof m?function(t,e){n[e.cache]&&(o[e.cache]=n[e.cache].slice())}:function(t,i){var s=i.cache;w(i.props,function(t,e){if(!o[s]&&i.to){if("alpha"===t||null==n[t])return;o[s]=i.to(o._rgba)}o[s][e.idx]=M(n[t],e,!0)}),o[s]&&d.inArray(null,o[s].slice(0,3))<0&&(null==o[s][3]&&(o[s][3]=1),i.from&&(o._rgba=i.from(o[s])))}),this):void 0},is:function(t){var n=m(t),o=!0,a=this;return w(_,function(t,e){var i,s=n[e.cache];return s&&(i=a[e.cache]||e.to&&e.to(a._rgba)||[],w(e.props,function(t,e){if(null!=s[e.idx])return o=s[e.idx]===i[e.idx]})),o}),o},_space:function(){var i=[],s=this;return w(_,function(t,e){s[e.cache]&&i.push(t)}),i.pop()},transition:function(t,a){var e=(h=m(t))._space(),i=_[e],t=0===this.alpha()?m("transparent"):this,r=t[i.cache]||i.to(t._rgba),l=r.slice(),h=h[i.cache];return w(i.props,function(t,e){var i=e.idx,s=r[i],n=h[i],o=v[e.type]||{};null!==n&&(null===s?l[i]=n:(o.mod&&(n-s>o.mod/2?s+=o.mod:s-n>o.mod/2&&(s-=o.mod)),l[i]=M((n-s)*a+s,e)))}),this[e](l)},blend:function(t){if(1===this._rgba[3])return this;var e=this._rgba.slice(),i=e.pop(),s=m(t)._rgba;return m(d.map(e,function(t,e){return(1-i)*s[e]+i*t}))},toRgbaString:function(){var t="rgba(",e=d.map(this._rgba,function(t,e){return null!=t?t:2<e?1:0});return 1===e[3]&&(e.pop(),t="rgb("),t+e.join()+")"},toHslaString:function(){var t="hsla(",e=d.map(this.hsla(),function(t,e){return null==t&&(t=2<e?1:0),t=e&&e<3?Math.round(100*t)+"%":t});return 1===e[3]&&(e.pop(),t="hsl("),t+e.join()+")"},toHexString:function(t){var e=this._rgba.slice(),i=e.pop();return t&&e.push(~~(255*i)),"#"+d.map(e,function(t){return 1===(t=(t||0).toString(16)).length?"0"+t:t}).join("")},toString:function(){return 0===this._rgba[3]?"transparent":this.toRgbaString()}})).parse.prototype=m.fn,_.hsla.to=function(t){if(null==t[0]||null==t[1]||null==t[2])return[null,null,null,t[3]];var e=t[0]/255,i=t[1]/255,s=t[2]/255,n=t[3],o=Math.max(e,i,s),a=Math.min(e,i,s),r=o-a,l=o+a,t=.5*l,i=a===o?0:e===o?60*(i-s)/r+360:i===o?60*(s-e)/r+120:60*(e-i)/r+240,l=0==r?0:t<=.5?r/l:r/(2-l);return[Math.round(i)%360,l,t,null==n?1:n]},_.hsla.from=function(t){if(null==t[0]||null==t[1]||null==t[2])return[null,null,null,t[3]];var e=t[0]/360,i=t[1],s=t[2],t=t[3],i=s<=.5?s*(1+i):s+i-s*i,s=2*s-i;return[Math.round(255*H(s,i,e+1/3)),Math.round(255*H(s,i,e)),Math.round(255*H(s,i,e-1/3)),t]},w(_,function(l,t){var e=t.props,o=t.cache,a=t.to,r=t.from;m.fn[l]=function(t){if(a&&!this[o]&&(this[o]=a(this._rgba)),void 0===t)return this[o].slice();var i=P(t),s="array"===i||"object"===i?t:arguments,n=this[o].slice();return w(e,function(t,e){t=s["object"===i?t:e.idx];null==t&&(t=n[e.idx]),n[e.idx]=M(t,e)}),r?((t=m(r(n)))[o]=n,t):m(n)},w(e,function(a,r){m.fn[a]||(m.fn[a]=function(t){var e,i=P(t),s="alpha"===a?this._hsla?"hsla":"rgba":l,n=this[s](),o=n[r.idx];return"undefined"===i?o:("function"===i&&(i=P(t=t.call(this,o))),null==t&&r.empty?this:("string"===i&&(e=f.exec(t))&&(t=o+parseFloat(e[2])*("+"===e[1]?1:-1)),n[r.idx]=t,this[s](n)))})})}),(m.hook=function(t){t=t.split(" ");w(t,function(t,o){d.cssHooks[o]={set:function(t,e){var i,s,n="";if("transparent"!==e&&("string"!==P(e)||(i=S(e)))){if(e=m(i||e),!b.rgba&&1!==e._rgba[3]){for(s="backgroundColor"===o?t.parentNode:t;(""===n||"transparent"===n)&&s&&s.style;)try{n=d.css(s,"backgroundColor"),s=s.parentNode}catch(t){}e=e.blend(n&&"transparent"!==n?n:"_default")}e=e.toRgbaString()}try{t.style[o]=e}catch(t){}}},d.fx.step[o]=function(t){t.colorInit||(t.start=m(t.elem,o),t.end=m(t.end),t.colorInit=!0),d.cssHooks[o].set(t.elem,t.start.transition(t.end,t.pos))}})})("backgroundColor borderBottomColor borderLeftColor borderRightColor borderTopColor color columnRuleColor outlineColor textDecorationColor textEmphasisColor"),d.cssHooks.borderColor={expand:function(i){var s={};return w(["Top","Right","Bottom","Left"],function(t,e){s["border"+e+"Color"]=i}),s}};var z,A,O,N,E,W,F,L,R,Y,B=d.Color.names={aqua:"#00ffff",black:"#000000",blue:"#0000ff",fuchsia:"#ff00ff",gray:"#808080",green:"#008000",lime:"#00ff00",maroon:"#800000",navy:"#000080",olive:"#808000",purple:"#800080",red:"#ff0000",silver:"#c0c0c0",teal:"#008080",white:"#ffffff",yellow:"#ffff00",transparent:[null,null,null,0],_default:"#ffffff"},j="ui-effects-",q="ui-effects-style",K="ui-effects-animated";function U(t){var e,i,s=t.ownerDocument.defaultView?t.ownerDocument.defaultView.getComputedStyle(t,null):t.currentStyle,n={};if(s&&s.length&&s[0]&&s[s[0]])for(i=s.length;i--;)"string"==typeof s[e=s[i]]&&(n[e.replace(/-([\da-z])/gi,function(t,e){return e.toUpperCase()})]=s[e]);else for(e in s)"string"==typeof s[e]&&(n[e]=s[e]);return n}function X(t,e,i,s){return t={effect:t=V.isPlainObject(t)?(e=t).effect:t},"function"==typeof(e=null==e?{}:e)&&(s=e,i=null,e={}),"number"!=typeof e&&!V.fx.speeds[e]||(s=i,i=e,e={}),"function"==typeof i&&(s=i,i=null),e&&V.extend(t,e),i=i||e.duration,t.duration=V.fx.off?0:"number"==typeof i?i:i in V.fx.speeds?V.fx.speeds[i]:V.fx.speeds._default,t.complete=s||e.complete,t}function $(t){return!t||"number"==typeof t||V.fx.speeds[t]||("string"==typeof t&&!V.effects.effect[t]||("function"==typeof t||"object"==typeof t&&!t.effect))}function G(t,e){var i=e.outerWidth(),e=e.outerHeight(),t=/^rect\((-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto),?\s*(-?\d*\.?\d*px|-?\d+%|auto)\)$/.exec(t)||["",0,i,e,0];return{top:parseFloat(t[1])||0,right:"auto"===t[2]?i:parseFloat(t[2]),bottom:"auto"===t[3]?e:parseFloat(t[3]),left:parseFloat(t[4])||0}}V.effects={effect:{}},N=["add","remove","toggle"],E={border:1,borderBottom:1,borderColor:1,borderLeft:1,borderRight:1,borderTop:1,borderWidth:1,margin:1,padding:1},V.each(["borderLeftStyle","borderRightStyle","borderBottomStyle","borderTopStyle"],function(t,e){V.fx.step[e]=function(t){("none"!==t.end&&!t.setAttr||1===t.pos&&!t.setAttr)&&(d.style(t.elem,e,t.end),t.setAttr=!0)}}),V.fn.addBack||(V.fn.addBack=function(t){return this.add(null==t?this.prevObject:this.prevObject.filter(t))}),V.effects.animateClass=function(n,t,e,i){var o=V.speed(t,e,i);return this.queue(function(){var i=V(this),t=i.attr("class")||"",e=(e=o.children?i.find("*").addBack():i).map(function(){return{el:V(this),start:U(this)}}),s=function(){V.each(N,function(t,e){n[e]&&i[e+"Class"](n[e])})};s(),e=e.map(function(){return this.end=U(this.el[0]),this.diff=function(t,e){var i,s,n={};for(i in e)s=e[i],t[i]!==s&&(E[i]||!V.fx.step[i]&&isNaN(parseFloat(s))||(n[i]=s));return n}(this.start,this.end),this}),i.attr("class",t),e=e.map(function(){var t=this,e=V.Deferred(),i=V.extend({},o,{queue:!1,complete:function(){e.resolve(t)}});return this.el.animate(this.diff,i),e.promise()}),V.when.apply(V,e.get()).done(function(){s(),V.each(arguments,function(){var e=this.el;V.each(this.diff,function(t){e.css(t,"")})}),o.complete.call(i[0])})})},V.fn.extend({addClass:(O=V.fn.addClass,function(t,e,i,s){return e?V.effects.animateClass.call(this,{add:t},e,i,s):O.apply(this,arguments)}),removeClass:(A=V.fn.removeClass,function(t,e,i,s){return 1<arguments.length?V.effects.animateClass.call(this,{remove:t},e,i,s):A.apply(this,arguments)}),toggleClass:(z=V.fn.toggleClass,function(t,e,i,s,n){return"boolean"==typeof e||void 0===e?i?V.effects.animateClass.call(this,e?{add:t}:{remove:t},i,s,n):z.apply(this,arguments):V.effects.animateClass.call(this,{toggle:t},e,i,s)}),switchClass:function(t,e,i,s,n){return V.effects.animateClass.call(this,{add:e,remove:t},i,s,n)}}),V.expr&&V.expr.pseudos&&V.expr.pseudos.animated&&(V.expr.pseudos.animated=(W=V.expr.pseudos.animated,function(t){return!!V(t).data(K)||W(t)})),!1!==V.uiBackCompat&&V.extend(V.effects,{save:function(t,e){for(var i=0,s=e.length;i<s;i++)null!==e[i]&&t.data(j+e[i],t[0].style[e[i]])},restore:function(t,e){for(var i,s=0,n=e.length;s<n;s++)null!==e[s]&&(i=t.data(j+e[s]),t.css(e[s],i))},setMode:function(t,e){return e="toggle"===e?t.is(":hidden")?"show":"hide":e},createWrapper:function(i){if(i.parent().is(".ui-effects-wrapper"))return i.parent();var s={width:i.outerWidth(!0),height:i.outerHeight(!0),float:i.css("float")},t=V("<div></div>").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),e={width:i.width(),height:i.height()},n=document.activeElement;try{n.id}catch(t){n=document.body}return i.wrap(t),i[0]!==n&&!V.contains(i[0],n)||V(n).trigger("focus"),t=i.parent(),"static"===i.css("position")?(t.css({position:"relative"}),i.css({position:"relative"})):(V.extend(s,{position:i.css("position"),zIndex:i.css("z-index")}),V.each(["top","left","bottom","right"],function(t,e){s[e]=i.css(e),isNaN(parseInt(s[e],10))&&(s[e]="auto")}),i.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})),i.css(e),t.css(s).show()},removeWrapper:function(t){var e=document.activeElement;return t.parent().is(".ui-effects-wrapper")&&(t.parent().replaceWith(t),t[0]!==e&&!V.contains(t[0],e)||V(e).trigger("focus")),t}}),V.extend(V.effects,{version:"1.13.2",define:function(t,e,i){return i||(i=e,e="effect"),V.effects.effect[t]=i,V.effects.effect[t].mode=e,i},scaledDimensions:function(t,e,i){if(0===e)return{height:0,width:0,outerHeight:0,outerWidth:0};var s="horizontal"!==i?(e||100)/100:1,e="vertical"!==i?(e||100)/100:1;return{height:t.height()*e,width:t.width()*s,outerHeight:t.outerHeight()*e,outerWidth:t.outerWidth()*s}},clipToBox:function(t){return{width:t.clip.right-t.clip.left,height:t.clip.bottom-t.clip.top,left:t.clip.left,top:t.clip.top}},unshift:function(t,e,i){var s=t.queue();1<e&&s.splice.apply(s,[1,0].concat(s.splice(e,i))),t.dequeue()},saveStyle:function(t){t.data(q,t[0].style.cssText)},restoreStyle:function(t){t[0].style.cssText=t.data(q)||"",t.removeData(q)},mode:function(t,e){t=t.is(":hidden");return"toggle"===e&&(e=t?"show":"hide"),e=(t?"hide"===e:"show"===e)?"none":e},getBaseline:function(t,e){var i,s;switch(t[0]){case"top":i=0;break;case"middle":i=.5;break;case"bottom":i=1;break;default:i=t[0]/e.height}switch(t[1]){case"left":s=0;break;case"center":s=.5;break;case"right":s=1;break;default:s=t[1]/e.width}return{x:s,y:i}},createPlaceholder:function(t){var e,i=t.css("position"),s=t.position();return t.css({marginTop:t.css("marginTop"),marginBottom:t.css("marginBottom"),marginLeft:t.css("marginLeft"),marginRight:t.css("marginRight")}).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()),/^(static|relative)/.test(i)&&(i="absolute",e=V("<"+t[0].nodeName+">").insertAfter(t).css({display:/^(inline|ruby)/.test(t.css("display"))?"inline-block":"block",visibility:"hidden",marginTop:t.css("marginTop"),marginBottom:t.css("marginBottom"),marginLeft:t.css("marginLeft"),marginRight:t.css("marginRight"),float:t.css("float")}).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()).addClass("ui-effects-placeholder"),t.data(j+"placeholder",e)),t.css({position:i,left:s.left,top:s.top}),e},removePlaceholder:function(t){var e=j+"placeholder",i=t.data(e);i&&(i.remove(),t.removeData(e))},cleanUp:function(t){V.effects.restoreStyle(t),V.effects.removePlaceholder(t)},setTransition:function(s,t,n,o){return o=o||{},V.each(t,function(t,e){var i=s.cssUnit(e);0<i[0]&&(o[e]=i[0]*n+i[1])}),o}}),V.fn.extend({effect:function(){function t(t){var e=V(this),i=V.effects.mode(e,r)||o;e.data(K,!0),l.push(i),o&&("show"===i||i===o&&"hide"===i)&&e.show(),o&&"none"===i||V.effects.saveStyle(e),"function"==typeof t&&t()}var s=X.apply(this,arguments),n=V.effects.effect[s.effect],o=n.mode,e=s.queue,i=e||"fx",a=s.complete,r=s.mode,l=[];return V.fx.off||!n?r?this[r](s.duration,a):this.each(function(){a&&a.call(this)}):!1===e?this.each(t).each(h):this.queue(i,t).queue(i,h);function h(t){var e=V(this);function i(){"function"==typeof a&&a.call(e[0]),"function"==typeof t&&t()}s.mode=l.shift(),!1===V.uiBackCompat||o?"none"===s.mode?(e[r](),i()):n.call(e[0],s,function(){e.removeData(K),V.effects.cleanUp(e),"hide"===s.mode&&e.hide(),i()}):(e.is(":hidden")?"hide"===r:"show"===r)?(e[r](),i()):n.call(e[0],s,i)}},show:(R=V.fn.show,function(t){if($(t))return R.apply(this,arguments);t=X.apply(this,arguments);return t.mode="show",this.effect.call(this,t)}),hide:(L=V.fn.hide,function(t){if($(t))return L.apply(this,arguments);t=X.apply(this,arguments);return t.mode="hide",this.effect.call(this,t)}),toggle:(F=V.fn.toggle,function(t){if($(t)||"boolean"==typeof t)return F.apply(this,arguments);t=X.apply(this,arguments);return t.mode="toggle",this.effect.call(this,t)}),cssUnit:function(t){var i=this.css(t),s=[];return V.each(["em","px","%","pt"],function(t,e){0<i.indexOf(e)&&(s=[parseFloat(i),e])}),s},cssClip:function(t){return t?this.css("clip","rect("+t.top+"px "+t.right+"px "+t.bottom+"px "+t.left+"px)"):G(this.css("clip"),this)},transfer:function(t,e){var i=V(this),s=V(t.to),n="fixed"===s.css("position"),o=V("body"),a=n?o.scrollTop():0,r=n?o.scrollLeft():0,o=s.offset(),o={top:o.top-a,left:o.left-r,height:s.innerHeight(),width:s.innerWidth()},s=i.offset(),l=V("<div class='ui-effects-transfer'></div>");l.appendTo("body").addClass(t.className).css({top:s.top-a,left:s.left-r,height:i.innerHeight(),width:i.innerWidth(),position:n?"fixed":"absolute"}).animate(o,t.duration,t.easing,function(){l.remove(),"function"==typeof e&&e()})}}),V.fx.step.clip=function(t){t.clipInit||(t.start=V(t.elem).cssClip(),"string"==typeof t.end&&(t.end=G(t.end,t.elem)),t.clipInit=!0),V(t.elem).cssClip({top:t.pos*(t.end.top-t.start.top)+t.start.top,right:t.pos*(t.end.right-t.start.right)+t.start.right,bottom:t.pos*(t.end.bottom-t.start.bottom)+t.start.bottom,left:t.pos*(t.end.left-t.start.left)+t.start.left})},Y={},V.each(["Quad","Cubic","Quart","Quint","Expo"],function(e,t){Y[t]=function(t){return Math.pow(t,e+2)}}),V.extend(Y,{Sine:function(t){return 1-Math.cos(t*Math.PI/2)},Circ:function(t){return 1-Math.sqrt(1-t*t)},Elastic:function(t){return 0===t||1===t?t:-Math.pow(2,8*(t-1))*Math.sin((80*(t-1)-7.5)*Math.PI/15)},Back:function(t){return t*t*(3*t-2)},Bounce:function(t){for(var e,i=4;t<((e=Math.pow(2,--i))-1)/11;);return 1/Math.pow(4,3-i)-7.5625*Math.pow((3*e-2)/22-t,2)}}),V.each(Y,function(t,e){V.easing["easeIn"+t]=e,V.easing["easeOut"+t]=function(t){return 1-e(1-t)},V.easing["easeInOut"+t]=function(t){return t<.5?e(2*t)/2:1-e(-2*t+2)/2}});y=V.effects,V.effects.define("blind","hide",function(t,e){var i={up:["bottom","top"],vertical:["bottom","top"],down:["top","bottom"],left:["right","left"],horizontal:["right","left"],right:["left","right"]},s=V(this),n=t.direction||"up",o=s.cssClip(),a={clip:V.extend({},o)},r=V.effects.createPlaceholder(s);a.clip[i[n][0]]=a.clip[i[n][1]],"show"===t.mode&&(s.cssClip(a.clip),r&&r.css(V.effects.clipToBox(a)),a.clip=o),r&&r.animate(V.effects.clipToBox(a),t.duration,t.easing),s.animate(a,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("bounce",function(t,e){var i,s,n=V(this),o=t.mode,a="hide"===o,r="show"===o,l=t.direction||"up",h=t.distance,c=t.times||5,o=2*c+(r||a?1:0),u=t.duration/o,d=t.easing,p="up"===l||"down"===l?"top":"left",f="up"===l||"left"===l,g=0,t=n.queue().length;for(V.effects.createPlaceholder(n),l=n.css(p),h=h||n["top"==p?"outerHeight":"outerWidth"]()/3,r&&((s={opacity:1})[p]=l,n.css("opacity",0).css(p,f?2*-h:2*h).animate(s,u,d)),a&&(h/=Math.pow(2,c-1)),(s={})[p]=l;g<c;g++)(i={})[p]=(f?"-=":"+=")+h,n.animate(i,u,d).animate(s,u,d),h=a?2*h:h/2;a&&((i={opacity:0})[p]=(f?"-=":"+=")+h,n.animate(i,u,d)),n.queue(e),V.effects.unshift(n,t,1+o)}),V.effects.define("clip","hide",function(t,e){var i={},s=V(this),n=t.direction||"vertical",o="both"===n,a=o||"horizontal"===n,o=o||"vertical"===n,n=s.cssClip();i.clip={top:o?(n.bottom-n.top)/2:n.top,right:a?(n.right-n.left)/2:n.right,bottom:o?(n.bottom-n.top)/2:n.bottom,left:a?(n.right-n.left)/2:n.left},V.effects.createPlaceholder(s),"show"===t.mode&&(s.cssClip(i.clip),i.clip=n),s.animate(i,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("drop","hide",function(t,e){var i=V(this),s="show"===t.mode,n=t.direction||"left",o="up"===n||"down"===n?"top":"left",a="up"===n||"left"===n?"-=":"+=",r="+="==a?"-=":"+=",l={opacity:0};V.effects.createPlaceholder(i),n=t.distance||i["top"==o?"outerHeight":"outerWidth"](!0)/2,l[o]=a+n,s&&(i.css(l),l[o]=r+n,l.opacity=1),i.animate(l,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("explode","hide",function(t,e){var i,s,n,o,a,r,l=t.pieces?Math.round(Math.sqrt(t.pieces)):3,h=l,c=V(this),u="show"===t.mode,d=c.show().css("visibility","hidden").offset(),p=Math.ceil(c.outerWidth()/h),f=Math.ceil(c.outerHeight()/l),g=[];function m(){g.push(this),g.length===l*h&&(c.css({visibility:"visible"}),V(g).remove(),e())}for(i=0;i<l;i++)for(o=d.top+i*f,r=i-(l-1)/2,s=0;s<h;s++)n=d.left+s*p,a=s-(h-1)/2,c.clone().appendTo("body").wrap("<div></div>").css({position:"absolute",visibility:"visible",left:-s*p,top:-i*f}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:p,height:f,left:n+(u?a*p:0),top:o+(u?r*f:0),opacity:u?0:1}).animate({left:n+(u?0:a*p),top:o+(u?0:r*f),opacity:u?1:0},t.duration||500,t.easing,m)}),V.effects.define("fade","toggle",function(t,e){var i="show"===t.mode;V(this).css("opacity",i?0:1).animate({opacity:i?1:0},{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("fold","hide",function(e,t){var i=V(this),s=e.mode,n="show"===s,o="hide"===s,a=e.size||15,r=/([0-9]+)%/.exec(a),l=!!e.horizFirst?["right","bottom"]:["bottom","right"],h=e.duration/2,c=V.effects.createPlaceholder(i),u=i.cssClip(),d={clip:V.extend({},u)},p={clip:V.extend({},u)},f=[u[l[0]],u[l[1]]],s=i.queue().length;r&&(a=parseInt(r[1],10)/100*f[o?0:1]),d.clip[l[0]]=a,p.clip[l[0]]=a,p.clip[l[1]]=0,n&&(i.cssClip(p.clip),c&&c.css(V.effects.clipToBox(p)),p.clip=u),i.queue(function(t){c&&c.animate(V.effects.clipToBox(d),h,e.easing).animate(V.effects.clipToBox(p),h,e.easing),t()}).animate(d,h,e.easing).animate(p,h,e.easing).queue(t),V.effects.unshift(i,s,4)}),V.effects.define("highlight","show",function(t,e){var i=V(this),s={backgroundColor:i.css("backgroundColor")};"hide"===t.mode&&(s.opacity=0),V.effects.saveStyle(i),i.css({backgroundImage:"none",backgroundColor:t.color||"#ffff99"}).animate(s,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("size",function(s,e){var n,i=V(this),t=["fontSize"],o=["borderTopWidth","borderBottomWidth","paddingTop","paddingBottom"],a=["borderLeftWidth","borderRightWidth","paddingLeft","paddingRight"],r=s.mode,l="effect"!==r,h=s.scale||"both",c=s.origin||["middle","center"],u=i.css("position"),d=i.position(),p=V.effects.scaledDimensions(i),f=s.from||p,g=s.to||V.effects.scaledDimensions(i,0);V.effects.createPlaceholder(i),"show"===r&&(r=f,f=g,g=r),n={from:{y:f.height/p.height,x:f.width/p.width},to:{y:g.height/p.height,x:g.width/p.width}},"box"!==h&&"both"!==h||(n.from.y!==n.to.y&&(f=V.effects.setTransition(i,o,n.from.y,f),g=V.effects.setTransition(i,o,n.to.y,g)),n.from.x!==n.to.x&&(f=V.effects.setTransition(i,a,n.from.x,f),g=V.effects.setTransition(i,a,n.to.x,g))),"content"!==h&&"both"!==h||n.from.y!==n.to.y&&(f=V.effects.setTransition(i,t,n.from.y,f),g=V.effects.setTransition(i,t,n.to.y,g)),c&&(c=V.effects.getBaseline(c,p),f.top=(p.outerHeight-f.outerHeight)*c.y+d.top,f.left=(p.outerWidth-f.outerWidth)*c.x+d.left,g.top=(p.outerHeight-g.outerHeight)*c.y+d.top,g.left=(p.outerWidth-g.outerWidth)*c.x+d.left),delete f.outerHeight,delete f.outerWidth,i.css(f),"content"!==h&&"both"!==h||(o=o.concat(["marginTop","marginBottom"]).concat(t),a=a.concat(["marginLeft","marginRight"]),i.find("*[width]").each(function(){var t=V(this),e=V.effects.scaledDimensions(t),i={height:e.height*n.from.y,width:e.width*n.from.x,outerHeight:e.outerHeight*n.from.y,outerWidth:e.outerWidth*n.from.x},e={height:e.height*n.to.y,width:e.width*n.to.x,outerHeight:e.height*n.to.y,outerWidth:e.width*n.to.x};n.from.y!==n.to.y&&(i=V.effects.setTransition(t,o,n.from.y,i),e=V.effects.setTransition(t,o,n.to.y,e)),n.from.x!==n.to.x&&(i=V.effects.setTransition(t,a,n.from.x,i),e=V.effects.setTransition(t,a,n.to.x,e)),l&&V.effects.saveStyle(t),t.css(i),t.animate(e,s.duration,s.easing,function(){l&&V.effects.restoreStyle(t)})})),i.animate(g,{queue:!1,duration:s.duration,easing:s.easing,complete:function(){var t=i.offset();0===g.opacity&&i.css("opacity",f.opacity),l||(i.css("position","static"===u?"relative":u).offset(t),V.effects.saveStyle(i)),e()}})}),V.effects.define("scale",function(t,e){var i=V(this),s=t.mode,s=parseInt(t.percent,10)||(0===parseInt(t.percent,10)||"effect"!==s?0:100),s=V.extend(!0,{from:V.effects.scaledDimensions(i),to:V.effects.scaledDimensions(i,s,t.direction||"both"),origin:t.origin||["middle","center"]},t);t.fade&&(s.from.opacity=1,s.to.opacity=0),V.effects.effect.size.call(this,s,e)}),V.effects.define("puff","hide",function(t,e){t=V.extend(!0,{},t,{fade:!0,percent:parseInt(t.percent,10)||150});V.effects.effect.scale.call(this,t,e)}),V.effects.define("pulsate","show",function(t,e){var i=V(this),s=t.mode,n="show"===s,o=2*(t.times||5)+(n||"hide"===s?1:0),a=t.duration/o,r=0,l=1,s=i.queue().length;for(!n&&i.is(":visible")||(i.css("opacity",0).show(),r=1);l<o;l++)i.animate({opacity:r},a,t.easing),r=1-r;i.animate({opacity:r},a,t.easing),i.queue(e),V.effects.unshift(i,s,1+o)}),V.effects.define("shake",function(t,e){var i=1,s=V(this),n=t.direction||"left",o=t.distance||20,a=t.times||3,r=2*a+1,l=Math.round(t.duration/r),h="up"===n||"down"===n?"top":"left",c="up"===n||"left"===n,u={},d={},p={},n=s.queue().length;for(V.effects.createPlaceholder(s),u[h]=(c?"-=":"+=")+o,d[h]=(c?"+=":"-=")+2*o,p[h]=(c?"-=":"+=")+2*o,s.animate(u,l,t.easing);i<a;i++)s.animate(d,l,t.easing).animate(p,l,t.easing);s.animate(d,l,t.easing).animate(u,l/2,t.easing).queue(e),V.effects.unshift(s,n,1+r)}),V.effects.define("slide","show",function(t,e){var i,s,n=V(this),o={up:["bottom","top"],down:["top","bottom"],left:["right","left"],right:["left","right"]},a=t.mode,r=t.direction||"left",l="up"===r||"down"===r?"top":"left",h="up"===r||"left"===r,c=t.distance||n["top"==l?"outerHeight":"outerWidth"](!0),u={};V.effects.createPlaceholder(n),i=n.cssClip(),s=n.position()[l],u[l]=(h?-1:1)*c+s,u.clip=n.cssClip(),u.clip[o[r][1]]=u.clip[o[r][0]],"show"===a&&(n.cssClip(u.clip),n.css(l,u[l]),u.clip=i,u[l]=s),n.animate(u,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),y=!1!==V.uiBackCompat?V.effects.define("transfer",function(t,e){V(this).transfer(t,e)}):y;V.ui.focusable=function(t,e){var i,s,n,o,a=t.nodeName.toLowerCase();return"area"===a?(s=(i=t.parentNode).name,!(!t.href||!s||"map"!==i.nodeName.toLowerCase())&&(0<(s=V("img[usemap='#"+s+"']")).length&&s.is(":visible"))):(/^(input|select|textarea|button|object)$/.test(a)?(n=!t.disabled)&&(o=V(t).closest("fieldset")[0])&&(n=!o.disabled):n="a"===a&&t.href||e,n&&V(t).is(":visible")&&function(t){var e=t.css("visibility");for(;"inherit"===e;)t=t.parent(),e=t.css("visibility");return"visible"===e}(V(t)))},V.extend(V.expr.pseudos,{focusable:function(t){return V.ui.focusable(t,null!=V.attr(t,"tabindex"))}});var Q,J;V.ui.focusable,V.fn._form=function(){return"string"==typeof this[0].form?this.closest("form"):V(this[0].form)},V.ui.formResetMixin={_formResetHandler:function(){var e=V(this);setTimeout(function(){var t=e.data("ui-form-reset-instances");V.each(t,function(){this.refresh()})})},_bindFormResetHandler:function(){var t;this.form=this.element._form(),this.form.length&&((t=this.form.data("ui-form-reset-instances")||[]).length||this.form.on("reset.ui-form-reset",this._formResetHandler),t.push(this),this.form.data("ui-form-reset-instances",t))},_unbindFormResetHandler:function(){var t;this.form.length&&((t=this.form.data("ui-form-reset-instances")).splice(V.inArray(this,t),1),t.length?this.form.data("ui-form-reset-instances",t):this.form.removeData("ui-form-reset-instances").off("reset.ui-form-reset"))}};V.expr.pseudos||(V.expr.pseudos=V.expr[":"]),V.uniqueSort||(V.uniqueSort=V.unique),V.escapeSelector||(Q=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g,J=function(t,e){return e?"\0"===t?"�":t.slice(0,-1)+"\\"+t.charCodeAt(t.length-1).toString(16)+" ":"\\"+t},V.escapeSelector=function(t){return(t+"").replace(Q,J)}),V.fn.even&&V.fn.odd||V.fn.extend({even:function(){return this.filter(function(t){return t%2==0})},odd:function(){return this.filter(function(t){return t%2==1})}});var Z;V.ui.keyCode={BACKSPACE:8,COMMA:188,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,LEFT:37,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SPACE:32,TAB:9,UP:38},V.fn.labels=function(){var t,e,i;return this.length?this[0].labels&&this[0].labels.length?this.pushStack(this[0].labels):(e=this.eq(0).parents("label"),(t=this.attr("id"))&&(i=(i=this.eq(0).parents().last()).add((i.length?i:this).siblings()),t="label[for='"+V.escapeSelector(t)+"']",e=e.add(i.find(t).addBack(t))),this.pushStack(e)):this.pushStack([])},V.fn.scrollParent=function(t){var e=this.css("position"),i="absolute"===e,s=t?/(auto|scroll|hidden)/:/(auto|scroll)/,t=this.parents().filter(function(){var t=V(this);return(!i||"static"!==t.css("position"))&&s.test(t.css("overflow")+t.css("overflow-y")+t.css("overflow-x"))}).eq(0);return"fixed"!==e&&t.length?t:V(this[0].ownerDocument||document)},V.extend(V.expr.pseudos,{tabbable:function(t){var e=V.attr(t,"tabindex"),i=null!=e;return(!i||0<=e)&&V.ui.focusable(t,i)}}),V.fn.extend({uniqueId:(Z=0,function(){return this.each(function(){this.id||(this.id="ui-id-"+ ++Z)})}),removeUniqueId:function(){return this.each(function(){/^ui-id-\d+$/.test(this.id)&&V(this).removeAttr("id")})}}),V.widget("ui.accordion",{version:"1.13.2",options:{active:0,animate:{},classes:{"ui-accordion-header":"ui-corner-top","ui-accordion-header-collapsed":"ui-corner-all","ui-accordion-content":"ui-corner-bottom"},collapsible:!1,event:"click",header:function(t){return t.find("> li > :first-child").add(t.find("> :not(li)").even())},heightStyle:"auto",icons:{activeHeader:"ui-icon-triangle-1-s",header:"ui-icon-triangle-1-e"},activate:null,beforeActivate:null},hideProps:{borderTopWidth:"hide",borderBottomWidth:"hide",paddingTop:"hide",paddingBottom:"hide",height:"hide"},showProps:{borderTopWidth:"show",borderBottomWidth:"show",paddingTop:"show",paddingBottom:"show",height:"show"},_create:function(){var t=this.options;this.prevShow=this.prevHide=V(),this._addClass("ui-accordion","ui-widget ui-helper-reset"),this.element.attr("role","tablist"),t.collapsible||!1!==t.active&&null!=t.active||(t.active=0),this._processPanels(),t.active<0&&(t.active+=this.headers.length),this._refresh()},_getCreateEventData:function(){return{header:this.active,panel:this.active.length?this.active.next():V()}},_createIcons:function(){var t,e=this.options.icons;e&&(t=V("<span>"),this._addClass(t,"ui-accordion-header-icon","ui-icon "+e.header),t.prependTo(this.headers),t=this.active.children(".ui-accordion-header-icon"),this._removeClass(t,e.header)._addClass(t,null,e.activeHeader)._addClass(this.headers,"ui-accordion-icons"))},_destroyIcons:function(){this._removeClass(this.headers,"ui-accordion-icons"),this.headers.children(".ui-accordion-header-icon").remove()},_destroy:function(){var t;this.element.removeAttr("role"),this.headers.removeAttr("role aria-expanded aria-selected aria-controls tabIndex").removeUniqueId(),this._destroyIcons(),t=this.headers.next().css("display","").removeAttr("role aria-hidden aria-labelledby").removeUniqueId(),"content"!==this.options.heightStyle&&t.css("height","")},_setOption:function(t,e){"active"!==t?("event"===t&&(this.options.event&&this._off(this.headers,this.options.event),this._setupEvents(e)),this._super(t,e),"collapsible"!==t||e||!1!==this.options.active||this._activate(0),"icons"===t&&(this._destroyIcons(),e&&this._createIcons())):this._activate(e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t),this._toggleClass(null,"ui-state-disabled",!!t),this._toggleClass(this.headers.add(this.headers.next()),null,"ui-state-disabled",!!t)},_keydown:function(t){if(!t.altKey&&!t.ctrlKey){var e=V.ui.keyCode,i=this.headers.length,s=this.headers.index(t.target),n=!1;switch(t.keyCode){case e.RIGHT:case e.DOWN:n=this.headers[(s+1)%i];break;case e.LEFT:case e.UP:n=this.headers[(s-1+i)%i];break;case e.SPACE:case e.ENTER:this._eventHandler(t);break;case e.HOME:n=this.headers[0];break;case e.END:n=this.headers[i-1]}n&&(V(t.target).attr("tabIndex",-1),V(n).attr("tabIndex",0),V(n).trigger("focus"),t.preventDefault())}},_panelKeyDown:function(t){t.keyCode===V.ui.keyCode.UP&&t.ctrlKey&&V(t.currentTarget).prev().trigger("focus")},refresh:function(){var t=this.options;this._processPanels(),!1===t.active&&!0===t.collapsible||!this.headers.length?(t.active=!1,this.active=V()):!1===t.active?this._activate(0):this.active.length&&!V.contains(this.element[0],this.active[0])?this.headers.length===this.headers.find(".ui-state-disabled").length?(t.active=!1,this.active=V()):this._activate(Math.max(0,t.active-1)):t.active=this.headers.index(this.active),this._destroyIcons(),this._refresh()},_processPanels:function(){var t=this.headers,e=this.panels;"function"==typeof this.options.header?this.headers=this.options.header(this.element):this.headers=this.element.find(this.options.header),this._addClass(this.headers,"ui-accordion-header ui-accordion-header-collapsed","ui-state-default"),this.panels=this.headers.next().filter(":not(.ui-accordion-content-active)").hide(),this._addClass(this.panels,"ui-accordion-content","ui-helper-reset ui-widget-content"),e&&(this._off(t.not(this.headers)),this._off(e.not(this.panels)))},_refresh:function(){var i,t=this.options,e=t.heightStyle,s=this.element.parent();this.active=this._findActive(t.active),this._addClass(this.active,"ui-accordion-header-active","ui-state-active")._removeClass(this.active,"ui-accordion-header-collapsed"),this._addClass(this.active.next(),"ui-accordion-content-active"),this.active.next().show(),this.headers.attr("role","tab").each(function(){var t=V(this),e=t.uniqueId().attr("id"),i=t.next(),s=i.uniqueId().attr("id");t.attr("aria-controls",s),i.attr("aria-labelledby",e)}).next().attr("role","tabpanel"),this.headers.not(this.active).attr({"aria-selected":"false","aria-expanded":"false",tabIndex:-1}).next().attr({"aria-hidden":"true"}).hide(),this.active.length?this.active.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0}).next().attr({"aria-hidden":"false"}):this.headers.eq(0).attr("tabIndex",0),this._createIcons(),this._setupEvents(t.event),"fill"===e?(i=s.height(),this.element.siblings(":visible").each(function(){var t=V(this),e=t.css("position");"absolute"!==e&&"fixed"!==e&&(i-=t.outerHeight(!0))}),this.headers.each(function(){i-=V(this).outerHeight(!0)}),this.headers.next().each(function(){V(this).height(Math.max(0,i-V(this).innerHeight()+V(this).height()))}).css("overflow","auto")):"auto"===e&&(i=0,this.headers.next().each(function(){var t=V(this).is(":visible");t||V(this).show(),i=Math.max(i,V(this).css("height","").height()),t||V(this).hide()}).height(i))},_activate:function(t){t=this._findActive(t)[0];t!==this.active[0]&&(t=t||this.active[0],this._eventHandler({target:t,currentTarget:t,preventDefault:V.noop}))},_findActive:function(t){return"number"==typeof t?this.headers.eq(t):V()},_setupEvents:function(t){var i={keydown:"_keydown"};t&&V.each(t.split(" "),function(t,e){i[e]="_eventHandler"}),this._off(this.headers.add(this.headers.next())),this._on(this.headers,i),this._on(this.headers.next(),{keydown:"_panelKeyDown"}),this._hoverable(this.headers),this._focusable(this.headers)},_eventHandler:function(t){var e=this.options,i=this.active,s=V(t.currentTarget),n=s[0]===i[0],o=n&&e.collapsible,a=o?V():s.next(),r=i.next(),a={oldHeader:i,oldPanel:r,newHeader:o?V():s,newPanel:a};t.preventDefault(),n&&!e.collapsible||!1===this._trigger("beforeActivate",t,a)||(e.active=!o&&this.headers.index(s),this.active=n?V():s,this._toggle(a),this._removeClass(i,"ui-accordion-header-active","ui-state-active"),e.icons&&(i=i.children(".ui-accordion-header-icon"),this._removeClass(i,null,e.icons.activeHeader)._addClass(i,null,e.icons.header)),n||(this._removeClass(s,"ui-accordion-header-collapsed")._addClass(s,"ui-accordion-header-active","ui-state-active"),e.icons&&(n=s.children(".ui-accordion-header-icon"),this._removeClass(n,null,e.icons.header)._addClass(n,null,e.icons.activeHeader)),this._addClass(s.next(),"ui-accordion-content-active")))},_toggle:function(t){var e=t.newPanel,i=this.prevShow.length?this.prevShow:t.oldPanel;this.prevShow.add(this.prevHide).stop(!0,!0),this.prevShow=e,this.prevHide=i,this.options.animate?this._animate(e,i,t):(i.hide(),e.show(),this._toggleComplete(t)),i.attr({"aria-hidden":"true"}),i.prev().attr({"aria-selected":"false","aria-expanded":"false"}),e.length&&i.length?i.prev().attr({tabIndex:-1,"aria-expanded":"false"}):e.length&&this.headers.filter(function(){return 0===parseInt(V(this).attr("tabIndex"),10)}).attr("tabIndex",-1),e.attr("aria-hidden","false").prev().attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0})},_animate:function(t,i,e){var s,n,o,a=this,r=0,l=t.css("box-sizing"),h=t.length&&(!i.length||t.index()<i.index()),c=this.options.animate||{},u=h&&c.down||c,h=function(){a._toggleComplete(e)};return n=(n="string"==typeof u?u:n)||u.easing||c.easing,o=(o="number"==typeof u?u:o)||u.duration||c.duration,i.length?t.length?(s=t.show().outerHeight(),i.animate(this.hideProps,{duration:o,easing:n,step:function(t,e){e.now=Math.round(t)}}),void t.hide().animate(this.showProps,{duration:o,easing:n,complete:h,step:function(t,e){e.now=Math.round(t),"height"!==e.prop?"content-box"===l&&(r+=e.now):"content"!==a.options.heightStyle&&(e.now=Math.round(s-i.outerHeight()-r),r=0)}})):i.animate(this.hideProps,o,n,h):t.animate(this.showProps,o,n,h)},_toggleComplete:function(t){var e=t.oldPanel,i=e.prev();this._removeClass(e,"ui-accordion-content-active"),this._removeClass(i,"ui-accordion-header-active")._addClass(i,"ui-accordion-header-collapsed"),e.length&&(e.parent()[0].className=e.parent()[0].className),this._trigger("activate",null,t)}}),V.ui.safeActiveElement=function(e){var i;try{i=e.activeElement}catch(t){i=e.body}return i=!(i=i||e.body).nodeName?e.body:i},V.widget("ui.menu",{version:"1.13.2",defaultElement:"<ul>",delay:300,options:{icons:{submenu:"ui-icon-caret-1-e"},items:"> *",menus:"ul",position:{my:"left top",at:"right top"},role:"menu",blur:null,focus:null,select:null},_create:function(){this.activeMenu=this.element,this.mouseHandled=!1,this.lastMousePosition={x:null,y:null},this.element.uniqueId().attr({role:this.options.role,tabIndex:0}),this._addClass("ui-menu","ui-widget ui-widget-content"),this._on({"mousedown .ui-menu-item":function(t){t.preventDefault(),this._activateItem(t)},"click .ui-menu-item":function(t){var e=V(t.target),i=V(V.ui.safeActiveElement(this.document[0]));!this.mouseHandled&&e.not(".ui-state-disabled").length&&(this.select(t),t.isPropagationStopped()||(this.mouseHandled=!0),e.has(".ui-menu").length?this.expand(t):!this.element.is(":focus")&&i.closest(".ui-menu").length&&(this.element.trigger("focus",[!0]),this.active&&1===this.active.parents(".ui-menu").length&&clearTimeout(this.timer)))},"mouseenter .ui-menu-item":"_activateItem","mousemove .ui-menu-item":"_activateItem",mouseleave:"collapseAll","mouseleave .ui-menu":"collapseAll",focus:function(t,e){var i=this.active||this._menuItems().first();e||this.focus(t,i)},blur:function(t){this._delay(function(){V.contains(this.element[0],V.ui.safeActiveElement(this.document[0]))||this.collapseAll(t)})},keydown:"_keydown"}),this.refresh(),this._on(this.document,{click:function(t){this._closeOnDocumentClick(t)&&this.collapseAll(t,!0),this.mouseHandled=!1}})},_activateItem:function(t){var e,i;this.previousFilter||t.clientX===this.lastMousePosition.x&&t.clientY===this.lastMousePosition.y||(this.lastMousePosition={x:t.clientX,y:t.clientY},e=V(t.target).closest(".ui-menu-item"),i=V(t.currentTarget),e[0]===i[0]&&(i.is(".ui-state-active")||(this._removeClass(i.siblings().children(".ui-state-active"),null,"ui-state-active"),this.focus(t,i))))},_destroy:function(){var t=this.element.find(".ui-menu-item").removeAttr("role aria-disabled").children(".ui-menu-item-wrapper").removeUniqueId().removeAttr("tabIndex role aria-haspopup");this.element.removeAttr("aria-activedescendant").find(".ui-menu").addBack().removeAttr("role aria-labelledby aria-expanded aria-hidden aria-disabled tabIndex").removeUniqueId().show(),t.children().each(function(){var t=V(this);t.data("ui-menu-submenu-caret")&&t.remove()})},_keydown:function(t){var e,i,s,n=!0;switch(t.keyCode){case V.ui.keyCode.PAGE_UP:this.previousPage(t);break;case V.ui.keyCode.PAGE_DOWN:this.nextPage(t);break;case V.ui.keyCode.HOME:this._move("first","first",t);break;case V.ui.keyCode.END:this._move("last","last",t);break;case V.ui.keyCode.UP:this.previous(t);break;case V.ui.keyCode.DOWN:this.next(t);break;case V.ui.keyCode.LEFT:this.collapse(t);break;case V.ui.keyCode.RIGHT:this.active&&!this.active.is(".ui-state-disabled")&&this.expand(t);break;case V.ui.keyCode.ENTER:case V.ui.keyCode.SPACE:this._activate(t);break;case V.ui.keyCode.ESCAPE:this.collapse(t);break;default:e=this.previousFilter||"",s=n=!1,i=96<=t.keyCode&&t.keyCode<=105?(t.keyCode-96).toString():String.fromCharCode(t.keyCode),clearTimeout(this.filterTimer),i===e?s=!0:i=e+i,e=this._filterMenuItems(i),(e=s&&-1!==e.index(this.active.next())?this.active.nextAll(".ui-menu-item"):e).length||(i=String.fromCharCode(t.keyCode),e=this._filterMenuItems(i)),e.length?(this.focus(t,e),this.previousFilter=i,this.filterTimer=this._delay(function(){delete this.previousFilter},1e3)):delete this.previousFilter}n&&t.preventDefault()},_activate:function(t){this.active&&!this.active.is(".ui-state-disabled")&&(this.active.children("[aria-haspopup='true']").length?this.expand(t):this.select(t))},refresh:function(){var t,e,s=this,n=this.options.icons.submenu,i=this.element.find(this.options.menus);this._toggleClass("ui-menu-icons",null,!!this.element.find(".ui-icon").length),e=i.filter(":not(.ui-menu)").hide().attr({role:this.options.role,"aria-hidden":"true","aria-expanded":"false"}).each(function(){var t=V(this),e=t.prev(),i=V("<span>").data("ui-menu-submenu-caret",!0);s._addClass(i,"ui-menu-icon","ui-icon "+n),e.attr("aria-haspopup","true").prepend(i),t.attr("aria-labelledby",e.attr("id"))}),this._addClass(e,"ui-menu","ui-widget ui-widget-content ui-front"),(t=i.add(this.element).find(this.options.items)).not(".ui-menu-item").each(function(){var t=V(this);s._isDivider(t)&&s._addClass(t,"ui-menu-divider","ui-widget-content")}),i=(e=t.not(".ui-menu-item, .ui-menu-divider")).children().not(".ui-menu").uniqueId().attr({tabIndex:-1,role:this._itemRole()}),this._addClass(e,"ui-menu-item")._addClass(i,"ui-menu-item-wrapper"),t.filter(".ui-state-disabled").attr("aria-disabled","true"),this.active&&!V.contains(this.element[0],this.active[0])&&this.blur()},_itemRole:function(){return{menu:"menuitem",listbox:"option"}[this.options.role]},_setOption:function(t,e){var i;"icons"===t&&(i=this.element.find(".ui-menu-icon"),this._removeClass(i,null,this.options.icons.submenu)._addClass(i,null,e.submenu)),this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",String(t)),this._toggleClass(null,"ui-state-disabled",!!t)},focus:function(t,e){var i;this.blur(t,t&&"focus"===t.type),this._scrollIntoView(e),this.active=e.first(),i=this.active.children(".ui-menu-item-wrapper"),this._addClass(i,null,"ui-state-active"),this.options.role&&this.element.attr("aria-activedescendant",i.attr("id")),i=this.active.parent().closest(".ui-menu-item").children(".ui-menu-item-wrapper"),this._addClass(i,null,"ui-state-active"),t&&"keydown"===t.type?this._close():this.timer=this._delay(function(){this._close()},this.delay),(i=e.children(".ui-menu")).length&&t&&/^mouse/.test(t.type)&&this._startOpening(i),this.activeMenu=e.parent(),this._trigger("focus",t,{item:e})},_scrollIntoView:function(t){var e,i,s;this._hasScroll()&&(i=parseFloat(V.css(this.activeMenu[0],"borderTopWidth"))||0,s=parseFloat(V.css(this.activeMenu[0],"paddingTop"))||0,e=t.offset().top-this.activeMenu.offset().top-i-s,i=this.activeMenu.scrollTop(),s=this.activeMenu.height(),t=t.outerHeight(),e<0?this.activeMenu.scrollTop(i+e):s<e+t&&this.activeMenu.scrollTop(i+e-s+t))},blur:function(t,e){e||clearTimeout(this.timer),this.active&&(this._removeClass(this.active.children(".ui-menu-item-wrapper"),null,"ui-state-active"),this._trigger("blur",t,{item:this.active}),this.active=null)},_startOpening:function(t){clearTimeout(this.timer),"true"===t.attr("aria-hidden")&&(this.timer=this._delay(function(){this._close(),this._open(t)},this.delay))},_open:function(t){var e=V.extend({of:this.active},this.options.position);clearTimeout(this.timer),this.element.find(".ui-menu").not(t.parents(".ui-menu")).hide().attr("aria-hidden","true"),t.show().removeAttr("aria-hidden").attr("aria-expanded","true").position(e)},collapseAll:function(e,i){clearTimeout(this.timer),this.timer=this._delay(function(){var t=i?this.element:V(e&&e.target).closest(this.element.find(".ui-menu"));t.length||(t=this.element),this._close(t),this.blur(e),this._removeClass(t.find(".ui-state-active"),null,"ui-state-active"),this.activeMenu=t},i?0:this.delay)},_close:function(t){(t=t||(this.active?this.active.parent():this.element)).find(".ui-menu").hide().attr("aria-hidden","true").attr("aria-expanded","false")},_closeOnDocumentClick:function(t){return!V(t.target).closest(".ui-menu").length},_isDivider:function(t){return!/[^\-\u2014\u2013\s]/.test(t.text())},collapse:function(t){var e=this.active&&this.active.parent().closest(".ui-menu-item",this.element);e&&e.length&&(this._close(),this.focus(t,e))},expand:function(t){var e=this.active&&this._menuItems(this.active.children(".ui-menu")).first();e&&e.length&&(this._open(e.parent()),this._delay(function(){this.focus(t,e)}))},next:function(t){this._move("next","first",t)},previous:function(t){this._move("prev","last",t)},isFirstItem:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},isLastItem:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},_menuItems:function(t){return(t||this.element).find(this.options.items).filter(".ui-menu-item")},_move:function(t,e,i){var s;(s=this.active?"first"===t||"last"===t?this.active["first"===t?"prevAll":"nextAll"](".ui-menu-item").last():this.active[t+"All"](".ui-menu-item").first():s)&&s.length&&this.active||(s=this._menuItems(this.activeMenu)[e]()),this.focus(i,s)},nextPage:function(t){var e,i,s;this.active?this.isLastItem()||(this._hasScroll()?(i=this.active.offset().top,s=this.element.innerHeight(),0===V.fn.jquery.indexOf("3.2.")&&(s+=this.element[0].offsetHeight-this.element.outerHeight()),this.active.nextAll(".ui-menu-item").each(function(){return(e=V(this)).offset().top-i-s<0}),this.focus(t,e)):this.focus(t,this._menuItems(this.activeMenu)[this.active?"last":"first"]())):this.next(t)},previousPage:function(t){var e,i,s;this.active?this.isFirstItem()||(this._hasScroll()?(i=this.active.offset().top,s=this.element.innerHeight(),0===V.fn.jquery.indexOf("3.2.")&&(s+=this.element[0].offsetHeight-this.element.outerHeight()),this.active.prevAll(".ui-menu-item").each(function(){return 0<(e=V(this)).offset().top-i+s}),this.focus(t,e)):this.focus(t,this._menuItems(this.activeMenu).first())):this.next(t)},_hasScroll:function(){return this.element.outerHeight()<this.element.prop("scrollHeight")},select:function(t){this.active=this.active||V(t.target).closest(".ui-menu-item");var e={item:this.active};this.active.has(".ui-menu").length||this.collapseAll(t,!0),this._trigger("select",t,e)},_filterMenuItems:function(t){var t=t.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&"),e=new RegExp("^"+t,"i");return this.activeMenu.find(this.options.items).filter(".ui-menu-item").filter(function(){return e.test(String.prototype.trim.call(V(this).children(".ui-menu-item-wrapper").text()))})}});V.widget("ui.autocomplete",{version:"1.13.2",defaultElement:"<input>",options:{appendTo:null,autoFocus:!1,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null,change:null,close:null,focus:null,open:null,response:null,search:null,select:null},requestIndex:0,pending:0,liveRegionTimer:null,_create:function(){var i,s,n,t=this.element[0].nodeName.toLowerCase(),e="textarea"===t,t="input"===t;this.isMultiLine=e||!t&&this._isContentEditable(this.element),this.valueMethod=this.element[e||t?"val":"text"],this.isNewMenu=!0,this._addClass("ui-autocomplete-input"),this.element.attr("autocomplete","off"),this._on(this.element,{keydown:function(t){if(this.element.prop("readOnly"))s=n=i=!0;else{s=n=i=!1;var e=V.ui.keyCode;switch(t.keyCode){case e.PAGE_UP:i=!0,this._move("previousPage",t);break;case e.PAGE_DOWN:i=!0,this._move("nextPage",t);break;case e.UP:i=!0,this._keyEvent("previous",t);break;case e.DOWN:i=!0,this._keyEvent("next",t);break;case e.ENTER:this.menu.active&&(i=!0,t.preventDefault(),this.menu.select(t));break;case e.TAB:this.menu.active&&this.menu.select(t);break;case e.ESCAPE:this.menu.element.is(":visible")&&(this.isMultiLine||this._value(this.term),this.close(t),t.preventDefault());break;default:s=!0,this._searchTimeout(t)}}},keypress:function(t){if(i)return i=!1,void(this.isMultiLine&&!this.menu.element.is(":visible")||t.preventDefault());if(!s){var e=V.ui.keyCode;switch(t.keyCode){case e.PAGE_UP:this._move("previousPage",t);break;case e.PAGE_DOWN:this._move("nextPage",t);break;case e.UP:this._keyEvent("previous",t);break;case e.DOWN:this._keyEvent("next",t)}}},input:function(t){if(n)return n=!1,void t.preventDefault();this._searchTimeout(t)},focus:function(){this.selectedItem=null,this.previous=this._value()},blur:function(t){clearTimeout(this.searching),this.close(t),this._change(t)}}),this._initSource(),this.menu=V("<ul>").appendTo(this._appendTo()).menu({role:null}).hide().attr({unselectable:"on"}).menu("instance"),this._addClass(this.menu.element,"ui-autocomplete","ui-front"),this._on(this.menu.element,{mousedown:function(t){t.preventDefault()},menufocus:function(t,e){var i,s;if(this.isNewMenu&&(this.isNewMenu=!1,t.originalEvent&&/^mouse/.test(t.originalEvent.type)))return this.menu.blur(),void this.document.one("mousemove",function(){V(t.target).trigger(t.originalEvent)});s=e.item.data("ui-autocomplete-item"),!1!==this._trigger("focus",t,{item:s})&&t.originalEvent&&/^key/.test(t.originalEvent.type)&&this._value(s.value),(i=e.item.attr("aria-label")||s.value)&&String.prototype.trim.call(i).length&&(clearTimeout(this.liveRegionTimer),this.liveRegionTimer=this._delay(function(){this.liveRegion.html(V("<div>").text(i))},100))},menuselect:function(t,e){var i=e.item.data("ui-autocomplete-item"),s=this.previous;this.element[0]!==V.ui.safeActiveElement(this.document[0])&&(this.element.trigger("focus"),this.previous=s,this._delay(function(){this.previous=s,this.selectedItem=i})),!1!==this._trigger("select",t,{item:i})&&this._value(i.value),this.term=this._value(),this.close(t),this.selectedItem=i}}),this.liveRegion=V("<div>",{role:"status","aria-live":"assertive","aria-relevant":"additions"}).appendTo(this.document[0].body),this._addClass(this.liveRegion,null,"ui-helper-hidden-accessible"),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_destroy:function(){clearTimeout(this.searching),this.element.removeAttr("autocomplete"),this.menu.element.remove(),this.liveRegion.remove()},_setOption:function(t,e){this._super(t,e),"source"===t&&this._initSource(),"appendTo"===t&&this.menu.element.appendTo(this._appendTo()),"disabled"===t&&e&&this.xhr&&this.xhr.abort()},_isEventTargetInWidget:function(t){var e=this.menu.element[0];return t.target===this.element[0]||t.target===e||V.contains(e,t.target)},_closeOnClickOutside:function(t){this._isEventTargetInWidget(t)||this.close()},_appendTo:function(){var t=this.options.appendTo;return t=!(t=!(t=t&&(t.jquery||t.nodeType?V(t):this.document.find(t).eq(0)))||!t[0]?this.element.closest(".ui-front, dialog"):t).length?this.document[0].body:t},_initSource:function(){var i,s,n=this;Array.isArray(this.options.source)?(i=this.options.source,this.source=function(t,e){e(V.ui.autocomplete.filter(i,t.term))}):"string"==typeof this.options.source?(s=this.options.source,this.source=function(t,e){n.xhr&&n.xhr.abort(),n.xhr=V.ajax({url:s,data:t,dataType:"json",success:function(t){e(t)},error:function(){e([])}})}):this.source=this.options.source},_searchTimeout:function(s){clearTimeout(this.searching),this.searching=this._delay(function(){var t=this.term===this._value(),e=this.menu.element.is(":visible"),i=s.altKey||s.ctrlKey||s.metaKey||s.shiftKey;t&&(e||i)||(this.selectedItem=null,this.search(null,s))},this.options.delay)},search:function(t,e){return t=null!=t?t:this._value(),this.term=this._value(),t.length<this.options.minLength?this.close(e):!1!==this._trigger("search",e)?this._search(t):void 0},_search:function(t){this.pending++,this._addClass("ui-autocomplete-loading"),this.cancelSearch=!1,this.source({term:t},this._response())},_response:function(){var e=++this.requestIndex;return function(t){e===this.requestIndex&&this.__response(t),this.pending--,this.pending||this._removeClass("ui-autocomplete-loading")}.bind(this)},__response:function(t){t=t&&this._normalize(t),this._trigger("response",null,{content:t}),!this.options.disabled&&t&&t.length&&!this.cancelSearch?(this._suggest(t),this._trigger("open")):this._close()},close:function(t){this.cancelSearch=!0,this._close(t)},_close:function(t){this._off(this.document,"mousedown"),this.menu.element.is(":visible")&&(this.menu.element.hide(),this.menu.blur(),this.isNewMenu=!0,this._trigger("close",t))},_change:function(t){this.previous!==this._value()&&this._trigger("change",t,{item:this.selectedItem})},_normalize:function(t){return t.length&&t[0].label&&t[0].value?t:V.map(t,function(t){return"string"==typeof t?{label:t,value:t}:V.extend({},t,{label:t.label||t.value,value:t.value||t.label})})},_suggest:function(t){var e=this.menu.element.empty();this._renderMenu(e,t),this.isNewMenu=!0,this.menu.refresh(),e.show(),this._resizeMenu(),e.position(V.extend({of:this.element},this.options.position)),this.options.autoFocus&&this.menu.next(),this._on(this.document,{mousedown:"_closeOnClickOutside"})},_resizeMenu:function(){var t=this.menu.element;t.outerWidth(Math.max(t.width("").outerWidth()+1,this.element.outerWidth()))},_renderMenu:function(i,t){var s=this;V.each(t,function(t,e){s._renderItemData(i,e)})},_renderItemData:function(t,e){return this._renderItem(t,e).data("ui-autocomplete-item",e)},_renderItem:function(t,e){return V("<li>").append(V("<div>").text(e.label)).appendTo(t)},_move:function(t,e){if(this.menu.element.is(":visible"))return this.menu.isFirstItem()&&/^previous/.test(t)||this.menu.isLastItem()&&/^next/.test(t)?(this.isMultiLine||this._value(this.term),void this.menu.blur()):void this.menu[t](e);this.search(null,e)},widget:function(){return this.menu.element},_value:function(){return this.valueMethod.apply(this.element,arguments)},_keyEvent:function(t,e){this.isMultiLine&&!this.menu.element.is(":visible")||(this._move(t,e),e.preventDefault())},_isContentEditable:function(t){if(!t.length)return!1;var e=t.prop("contentEditable");return"inherit"===e?this._isContentEditable(t.parent()):"true"===e}}),V.extend(V.ui.autocomplete,{escapeRegex:function(t){return t.replace(/[\-\[\]{}()*+?.,\\\^$|#\s]/g,"\\$&")},filter:function(t,e){var i=new RegExp(V.ui.autocomplete.escapeRegex(e),"i");return V.grep(t,function(t){return i.test(t.label||t.value||t)})}}),V.widget("ui.autocomplete",V.ui.autocomplete,{options:{messages:{noResults:"No search results.",results:function(t){return t+(1<t?" results are":" result is")+" available, use up and down arrow keys to navigate."}}},__response:function(t){var e;this._superApply(arguments),this.options.disabled||this.cancelSearch||(e=t&&t.length?this.options.messages.results(t.length):this.options.messages.noResults,clearTimeout(this.liveRegionTimer),this.liveRegionTimer=this._delay(function(){this.liveRegion.html(V("<div>").text(e))},100))}});V.ui.autocomplete;var tt=/ui-corner-([a-z]){2,6}/g;V.widget("ui.controlgroup",{version:"1.13.2",defaultElement:"<div>",options:{direction:"horizontal",disabled:null,onlyVisible:!0,items:{button:"input[type=button], input[type=submit], input[type=reset], button, a",controlgroupLabel:".ui-controlgroup-label",checkboxradio:"input[type='checkbox'], input[type='radio']",selectmenu:"select",spinner:".ui-spinner-input"}},_create:function(){this._enhance()},_enhance:function(){this.element.attr("role","toolbar"),this.refresh()},_destroy:function(){this._callChildMethod("destroy"),this.childWidgets.removeData("ui-controlgroup-data"),this.element.removeAttr("role"),this.options.items.controlgroupLabel&&this.element.find(this.options.items.controlgroupLabel).find(".ui-controlgroup-label-contents").contents().unwrap()},_initWidgets:function(){var o=this,a=[];V.each(this.options.items,function(s,t){var e,n={};if(t)return"controlgroupLabel"===s?((e=o.element.find(t)).each(function(){var t=V(this);t.children(".ui-controlgroup-label-contents").length||t.contents().wrapAll("<span class='ui-controlgroup-label-contents'></span>")}),o._addClass(e,null,"ui-widget ui-widget-content ui-state-default"),void(a=a.concat(e.get()))):void(V.fn[s]&&(n=o["_"+s+"Options"]?o["_"+s+"Options"]("middle"):{classes:{}},o.element.find(t).each(function(){var t=V(this),e=t[s]("instance"),i=V.widget.extend({},n);"button"===s&&t.parent(".ui-spinner").length||((e=e||t[s]()[s]("instance"))&&(i.classes=o._resolveClassesValues(i.classes,e)),t[s](i),i=t[s]("widget"),V.data(i[0],"ui-controlgroup-data",e||t[s]("instance")),a.push(i[0]))})))}),this.childWidgets=V(V.uniqueSort(a)),this._addClass(this.childWidgets,"ui-controlgroup-item")},_callChildMethod:function(e){this.childWidgets.each(function(){var t=V(this).data("ui-controlgroup-data");t&&t[e]&&t[e]()})},_updateCornerClass:function(t,e){e=this._buildSimpleOptions(e,"label").classes.label;this._removeClass(t,null,"ui-corner-top ui-corner-bottom ui-corner-left ui-corner-right ui-corner-all"),this._addClass(t,null,e)},_buildSimpleOptions:function(t,e){var i="vertical"===this.options.direction,s={classes:{}};return s.classes[e]={middle:"",first:"ui-corner-"+(i?"top":"left"),last:"ui-corner-"+(i?"bottom":"right"),only:"ui-corner-all"}[t],s},_spinnerOptions:function(t){t=this._buildSimpleOptions(t,"ui-spinner");return t.classes["ui-spinner-up"]="",t.classes["ui-spinner-down"]="",t},_buttonOptions:function(t){return this._buildSimpleOptions(t,"ui-button")},_checkboxradioOptions:function(t){return this._buildSimpleOptions(t,"ui-checkboxradio-label")},_selectmenuOptions:function(t){var e="vertical"===this.options.direction;return{width:e&&"auto",classes:{middle:{"ui-selectmenu-button-open":"","ui-selectmenu-button-closed":""},first:{"ui-selectmenu-button-open":"ui-corner-"+(e?"top":"tl"),"ui-selectmenu-button-closed":"ui-corner-"+(e?"top":"left")},last:{"ui-selectmenu-button-open":e?"":"ui-corner-tr","ui-selectmenu-button-closed":"ui-corner-"+(e?"bottom":"right")},only:{"ui-selectmenu-button-open":"ui-corner-top","ui-selectmenu-button-closed":"ui-corner-all"}}[t]}},_resolveClassesValues:function(i,s){var n={};return V.each(i,function(t){var e=s.options.classes[t]||"",e=String.prototype.trim.call(e.replace(tt,""));n[t]=(e+" "+i[t]).replace(/\s+/g," ")}),n},_setOption:function(t,e){"direction"===t&&this._removeClass("ui-controlgroup-"+this.options.direction),this._super(t,e),"disabled"!==t?this.refresh():this._callChildMethod(e?"disable":"enable")},refresh:function(){var n,o=this;this._addClass("ui-controlgroup ui-controlgroup-"+this.options.direction),"horizontal"===this.options.direction&&this._addClass(null,"ui-helper-clearfix"),this._initWidgets(),n=this.childWidgets,(n=this.options.onlyVisible?n.filter(":visible"):n).length&&(V.each(["first","last"],function(t,e){var i,s=n[e]().data("ui-controlgroup-data");s&&o["_"+s.widgetName+"Options"]?((i=o["_"+s.widgetName+"Options"](1===n.length?"only":e)).classes=o._resolveClassesValues(i.classes,s),s.element[s.widgetName](i)):o._updateCornerClass(n[e](),e)}),this._callChildMethod("refresh"))}});V.widget("ui.checkboxradio",[V.ui.formResetMixin,{version:"1.13.2",options:{disabled:null,label:null,icon:!0,classes:{"ui-checkboxradio-label":"ui-corner-all","ui-checkboxradio-icon":"ui-corner-all"}},_getCreateOptions:function(){var t,e=this._super()||{};return this._readType(),t=this.element.labels(),this.label=V(t[t.length-1]),this.label.length||V.error("No label found for checkboxradio widget"),this.originalLabel="",(t=this.label.contents().not(this.element[0])).length&&(this.originalLabel+=t.clone().wrapAll("<div></div>").parent().html()),this.originalLabel&&(e.label=this.originalLabel),null!=(t=this.element[0].disabled)&&(e.disabled=t),e},_create:function(){var t=this.element[0].checked;this._bindFormResetHandler(),null==this.options.disabled&&(this.options.disabled=this.element[0].disabled),this._setOption("disabled",this.options.disabled),this._addClass("ui-checkboxradio","ui-helper-hidden-accessible"),this._addClass(this.label,"ui-checkboxradio-label","ui-button ui-widget"),"radio"===this.type&&this._addClass(this.label,"ui-checkboxradio-radio-label"),this.options.label&&this.options.label!==this.originalLabel?this._updateLabel():this.originalLabel&&(this.options.label=this.originalLabel),this._enhance(),t&&this._addClass(this.label,"ui-checkboxradio-checked","ui-state-active"),this._on({change:"_toggleClasses",focus:function(){this._addClass(this.label,null,"ui-state-focus ui-visual-focus")},blur:function(){this._removeClass(this.label,null,"ui-state-focus ui-visual-focus")}})},_readType:function(){var t=this.element[0].nodeName.toLowerCase();this.type=this.element[0].type,"input"===t&&/radio|checkbox/.test(this.type)||V.error("Can't create checkboxradio on element.nodeName="+t+" and element.type="+this.type)},_enhance:function(){this._updateIcon(this.element[0].checked)},widget:function(){return this.label},_getRadioGroup:function(){var t=this.element[0].name,e="input[name='"+V.escapeSelector(t)+"']";return t?(this.form.length?V(this.form[0].elements).filter(e):V(e).filter(function(){return 0===V(this)._form().length})).not(this.element):V([])},_toggleClasses:function(){var t=this.element[0].checked;this._toggleClass(this.label,"ui-checkboxradio-checked","ui-state-active",t),this.options.icon&&"checkbox"===this.type&&this._toggleClass(this.icon,null,"ui-icon-check ui-state-checked",t)._toggleClass(this.icon,null,"ui-icon-blank",!t),"radio"===this.type&&this._getRadioGroup().each(function(){var t=V(this).checkboxradio("instance");t&&t._removeClass(t.label,"ui-checkboxradio-checked","ui-state-active")})},_destroy:function(){this._unbindFormResetHandler(),this.icon&&(this.icon.remove(),this.iconSpace.remove())},_setOption:function(t,e){if("label"!==t||e){if(this._super(t,e),"disabled"===t)return this._toggleClass(this.label,null,"ui-state-disabled",e),void(this.element[0].disabled=e);this.refresh()}},_updateIcon:function(t){var e="ui-icon ui-icon-background ";this.options.icon?(this.icon||(this.icon=V("<span>"),this.iconSpace=V("<span> </span>"),this._addClass(this.iconSpace,"ui-checkboxradio-icon-space")),"checkbox"===this.type?(e+=t?"ui-icon-check ui-state-checked":"ui-icon-blank",this._removeClass(this.icon,null,t?"ui-icon-blank":"ui-icon-check")):e+="ui-icon-blank",this._addClass(this.icon,"ui-checkboxradio-icon",e),t||this._removeClass(this.icon,null,"ui-icon-check ui-state-checked"),this.icon.prependTo(this.label).after(this.iconSpace)):void 0!==this.icon&&(this.icon.remove(),this.iconSpace.remove(),delete this.icon)},_updateLabel:function(){var t=this.label.contents().not(this.element[0]);this.icon&&(t=t.not(this.icon[0])),(t=this.iconSpace?t.not(this.iconSpace[0]):t).remove(),this.label.append(this.options.label)},refresh:function(){var t=this.element[0].checked,e=this.element[0].disabled;this._updateIcon(t),this._toggleClass(this.label,"ui-checkboxradio-checked","ui-state-active",t),null!==this.options.label&&this._updateLabel(),e!==this.options.disabled&&this._setOptions({disabled:e})}}]);var et;V.ui.checkboxradio;V.widget("ui.button",{version:"1.13.2",defaultElement:"<button>",options:{classes:{"ui-button":"ui-corner-all"},disabled:null,icon:null,iconPosition:"beginning",label:null,showLabel:!0},_getCreateOptions:function(){var t,e=this._super()||{};return this.isInput=this.element.is("input"),null!=(t=this.element[0].disabled)&&(e.disabled=t),this.originalLabel=this.isInput?this.element.val():this.element.html(),this.originalLabel&&(e.label=this.originalLabel),e},_create:function(){!this.option.showLabel&!this.options.icon&&(this.options.showLabel=!0),null==this.options.disabled&&(this.options.disabled=this.element[0].disabled||!1),this.hasTitle=!!this.element.attr("title"),this.options.label&&this.options.label!==this.originalLabel&&(this.isInput?this.element.val(this.options.label):this.element.html(this.options.label)),this._addClass("ui-button","ui-widget"),this._setOption("disabled",this.options.disabled),this._enhance(),this.element.is("a")&&this._on({keyup:function(t){t.keyCode===V.ui.keyCode.SPACE&&(t.preventDefault(),this.element[0].click?this.element[0].click():this.element.trigger("click"))}})},_enhance:function(){this.element.is("button")||this.element.attr("role","button"),this.options.icon&&(this._updateIcon("icon",this.options.icon),this._updateTooltip())},_updateTooltip:function(){this.title=this.element.attr("title"),this.options.showLabel||this.title||this.element.attr("title",this.options.label)},_updateIcon:function(t,e){var i="iconPosition"!==t,s=i?this.options.iconPosition:e,t="top"===s||"bottom"===s;this.icon?i&&this._removeClass(this.icon,null,this.options.icon):(this.icon=V("<span>"),this._addClass(this.icon,"ui-button-icon","ui-icon"),this.options.showLabel||this._addClass("ui-button-icon-only")),i&&this._addClass(this.icon,null,e),this._attachIcon(s),t?(this._addClass(this.icon,null,"ui-widget-icon-block"),this.iconSpace&&this.iconSpace.remove()):(this.iconSpace||(this.iconSpace=V("<span> </span>"),this._addClass(this.iconSpace,"ui-button-icon-space")),this._removeClass(this.icon,null,"ui-wiget-icon-block"),this._attachIconSpace(s))},_destroy:function(){this.element.removeAttr("role"),this.icon&&this.icon.remove(),this.iconSpace&&this.iconSpace.remove(),this.hasTitle||this.element.removeAttr("title")},_attachIconSpace:function(t){this.icon[/^(?:end|bottom)/.test(t)?"before":"after"](this.iconSpace)},_attachIcon:function(t){this.element[/^(?:end|bottom)/.test(t)?"append":"prepend"](this.icon)},_setOptions:function(t){var e=(void 0===t.showLabel?this.options:t).showLabel,i=(void 0===t.icon?this.options:t).icon;e||i||(t.showLabel=!0),this._super(t)},_setOption:function(t,e){"icon"===t&&(e?this._updateIcon(t,e):this.icon&&(this.icon.remove(),this.iconSpace&&this.iconSpace.remove())),"iconPosition"===t&&this._updateIcon(t,e),"showLabel"===t&&(this._toggleClass("ui-button-icon-only",null,!e),this._updateTooltip()),"label"===t&&(this.isInput?this.element.val(e):(this.element.html(e),this.icon&&(this._attachIcon(this.options.iconPosition),this._attachIconSpace(this.options.iconPosition)))),this._super(t,e),"disabled"===t&&(this._toggleClass(null,"ui-state-disabled",e),(this.element[0].disabled=e)&&this.element.trigger("blur"))},refresh:function(){var t=this.element.is("input, button")?this.element[0].disabled:this.element.hasClass("ui-button-disabled");t!==this.options.disabled&&this._setOptions({disabled:t}),this._updateTooltip()}}),!1!==V.uiBackCompat&&(V.widget("ui.button",V.ui.button,{options:{text:!0,icons:{primary:null,secondary:null}},_create:function(){this.options.showLabel&&!this.options.text&&(this.options.showLabel=this.options.text),!this.options.showLabel&&this.options.text&&(this.options.text=this.options.showLabel),this.options.icon||!this.options.icons.primary&&!this.options.icons.secondary?this.options.icon&&(this.options.icons.primary=this.options.icon):this.options.icons.primary?this.options.icon=this.options.icons.primary:(this.options.icon=this.options.icons.secondary,this.options.iconPosition="end"),this._super()},_setOption:function(t,e){"text"!==t?("showLabel"===t&&(this.options.text=e),"icon"===t&&(this.options.icons.primary=e),"icons"===t&&(e.primary?(this._super("icon",e.primary),this._super("iconPosition","beginning")):e.secondary&&(this._super("icon",e.secondary),this._super("iconPosition","end"))),this._superApply(arguments)):this._super("showLabel",e)}}),V.fn.button=(et=V.fn.button,function(i){var t="string"==typeof i,s=Array.prototype.slice.call(arguments,1),n=this;return t?this.length||"instance"!==i?this.each(function(){var t=V(this).attr("type"),e=V.data(this,"ui-"+("checkbox"!==t&&"radio"!==t?"button":"checkboxradio"));return"instance"===i?(n=e,!1):e?"function"!=typeof e[i]||"_"===i.charAt(0)?V.error("no such method '"+i+"' for button widget instance"):(t=e[i].apply(e,s))!==e&&void 0!==t?(n=t&&t.jquery?n.pushStack(t.get()):t,!1):void 0:V.error("cannot call methods on button prior to initialization; attempted to call method '"+i+"'")}):n=void 0:(s.length&&(i=V.widget.extend.apply(null,[i].concat(s))),this.each(function(){var t=V(this).attr("type"),e="checkbox"!==t&&"radio"!==t?"button":"checkboxradio",t=V.data(this,"ui-"+e);t?(t.option(i||{}),t._init&&t._init()):"button"!=e?V(this).checkboxradio(V.extend({icon:!1},i)):et.call(V(this),i)})),n}),V.fn.buttonset=function(){return V.ui.controlgroup||V.error("Controlgroup widget missing"),"option"===arguments[0]&&"items"===arguments[1]&&arguments[2]?this.controlgroup.apply(this,[arguments[0],"items.button",arguments[2]]):"option"===arguments[0]&&"items"===arguments[1]?this.controlgroup.apply(this,[arguments[0],"items.button"]):("object"==typeof arguments[0]&&arguments[0].items&&(arguments[0].items={button:arguments[0].items}),this.controlgroup.apply(this,arguments))});var it;V.ui.button;function st(){this._curInst=null,this._keyEvent=!1,this._disabledInputs=[],this._datepickerShowing=!1,this._inDialog=!1,this._mainDivId="ui-datepicker-div",this._inlineClass="ui-datepicker-inline",this._appendClass="ui-datepicker-append",this._triggerClass="ui-datepicker-trigger",this._dialogClass="ui-datepicker-dialog",this._disableClass="ui-datepicker-disabled",this._unselectableClass="ui-datepicker-unselectable",this._currentClass="ui-datepicker-current-day",this._dayOverClass="ui-datepicker-days-cell-over",this.regional=[],this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su","Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:!1,showMonthAfterYear:!1,yearSuffix:"",selectMonthLabel:"Select month",selectYearLabel:"Select year"},this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:!1,hideIfNoPrevNext:!1,navigationAsDateFormat:!1,gotoCurrent:!1,changeMonth:!1,changeYear:!1,yearRange:"c-10:c+10",showOtherMonths:!1,selectOtherMonths:!1,showWeek:!1,calculateWeek:this.iso8601Week,shortYearCutoff:"+10",minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,onUpdateDatepicker:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:!0,showButtonPanel:!1,autoSize:!1,disabled:!1},V.extend(this._defaults,this.regional[""]),this.regional.en=V.extend(!0,{},this.regional[""]),this.regional["en-US"]=V.extend(!0,{},this.regional.en),this.dpDiv=nt(V("<div id='"+this._mainDivId+"' class='ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all'></div>"))}function nt(t){var e="button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a";return t.on("mouseout",e,function(){V(this).removeClass("ui-state-hover"),-1!==this.className.indexOf("ui-datepicker-prev")&&V(this).removeClass("ui-datepicker-prev-hover"),-1!==this.className.indexOf("ui-datepicker-next")&&V(this).removeClass("ui-datepicker-next-hover")}).on("mouseover",e,ot)}function ot(){V.datepicker._isDisabledDatepicker((it.inline?it.dpDiv.parent():it.input)[0])||(V(this).parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover"),V(this).addClass("ui-state-hover"),-1!==this.className.indexOf("ui-datepicker-prev")&&V(this).addClass("ui-datepicker-prev-hover"),-1!==this.className.indexOf("ui-datepicker-next")&&V(this).addClass("ui-datepicker-next-hover"))}function at(t,e){for(var i in V.extend(t,e),e)null==e[i]&&(t[i]=e[i]);return t}V.extend(V.ui,{datepicker:{version:"1.13.2"}}),V.extend(st.prototype,{markerClassName:"hasDatepicker",maxRows:4,_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(t){return at(this._defaults,t||{}),this},_attachDatepicker:function(t,e){var i,s=t.nodeName.toLowerCase(),n="div"===s||"span"===s;t.id||(this.uuid+=1,t.id="dp"+this.uuid),(i=this._newInst(V(t),n)).settings=V.extend({},e||{}),"input"===s?this._connectDatepicker(t,i):n&&this._inlineDatepicker(t,i)},_newInst:function(t,e){return{id:t[0].id.replace(/([^A-Za-z0-9_\-])/g,"\\\\$1"),input:t,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:e,dpDiv:e?nt(V("<div class='"+this._inlineClass+" ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all'></div>")):this.dpDiv}},_connectDatepicker:function(t,e){var i=V(t);e.append=V([]),e.trigger=V([]),i.hasClass(this.markerClassName)||(this._attachments(i,e),i.addClass(this.markerClassName).on("keydown",this._doKeyDown).on("keypress",this._doKeyPress).on("keyup",this._doKeyUp),this._autoSize(e),V.data(t,"datepicker",e),e.settings.disabled&&this._disableDatepicker(t))},_attachments:function(t,e){var i,s=this._get(e,"appendText"),n=this._get(e,"isRTL");e.append&&e.append.remove(),s&&(e.append=V("<span>").addClass(this._appendClass).text(s),t[n?"before":"after"](e.append)),t.off("focus",this._showDatepicker),e.trigger&&e.trigger.remove(),"focus"!==(i=this._get(e,"showOn"))&&"both"!==i||t.on("focus",this._showDatepicker),"button"!==i&&"both"!==i||(s=this._get(e,"buttonText"),i=this._get(e,"buttonImage"),this._get(e,"buttonImageOnly")?e.trigger=V("<img>").addClass(this._triggerClass).attr({src:i,alt:s,title:s}):(e.trigger=V("<button type='button'>").addClass(this._triggerClass),i?e.trigger.html(V("<img>").attr({src:i,alt:s,title:s})):e.trigger.text(s)),t[n?"before":"after"](e.trigger),e.trigger.on("click",function(){return V.datepicker._datepickerShowing&&V.datepicker._lastInput===t[0]?V.datepicker._hideDatepicker():(V.datepicker._datepickerShowing&&V.datepicker._lastInput!==t[0]&&V.datepicker._hideDatepicker(),V.datepicker._showDatepicker(t[0])),!1}))},_autoSize:function(t){var e,i,s,n,o,a;this._get(t,"autoSize")&&!t.inline&&(o=new Date(2009,11,20),(a=this._get(t,"dateFormat")).match(/[DM]/)&&(e=function(t){for(n=s=i=0;n<t.length;n++)t[n].length>i&&(i=t[n].length,s=n);return s},o.setMonth(e(this._get(t,a.match(/MM/)?"monthNames":"monthNamesShort"))),o.setDate(e(this._get(t,a.match(/DD/)?"dayNames":"dayNamesShort"))+20-o.getDay())),t.input.attr("size",this._formatDate(t,o).length))},_inlineDatepicker:function(t,e){var i=V(t);i.hasClass(this.markerClassName)||(i.addClass(this.markerClassName).append(e.dpDiv),V.data(t,"datepicker",e),this._setDate(e,this._getDefaultDate(e),!0),this._updateDatepicker(e),this._updateAlternate(e),e.settings.disabled&&this._disableDatepicker(t),e.dpDiv.css("display","block"))},_dialogDatepicker:function(t,e,i,s,n){var o,a=this._dialogInst;return a||(this.uuid+=1,o="dp"+this.uuid,this._dialogInput=V("<input type='text' id='"+o+"' style='position: absolute; top: -100px; width: 0px;'/>"),this._dialogInput.on("keydown",this._doKeyDown),V("body").append(this._dialogInput),(a=this._dialogInst=this._newInst(this._dialogInput,!1)).settings={},V.data(this._dialogInput[0],"datepicker",a)),at(a.settings,s||{}),e=e&&e.constructor===Date?this._formatDate(a,e):e,this._dialogInput.val(e),this._pos=n?n.length?n:[n.pageX,n.pageY]:null,this._pos||(o=document.documentElement.clientWidth,s=document.documentElement.clientHeight,e=document.documentElement.scrollLeft||document.body.scrollLeft,n=document.documentElement.scrollTop||document.body.scrollTop,this._pos=[o/2-100+e,s/2-150+n]),this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px"),a.settings.onSelect=i,this._inDialog=!0,this.dpDiv.addClass(this._dialogClass),this._showDatepicker(this._dialogInput[0]),V.blockUI&&V.blockUI(this.dpDiv),V.data(this._dialogInput[0],"datepicker",a),this},_destroyDatepicker:function(t){var e,i=V(t),s=V.data(t,"datepicker");i.hasClass(this.markerClassName)&&(e=t.nodeName.toLowerCase(),V.removeData(t,"datepicker"),"input"===e?(s.append.remove(),s.trigger.remove(),i.removeClass(this.markerClassName).off("focus",this._showDatepicker).off("keydown",this._doKeyDown).off("keypress",this._doKeyPress).off("keyup",this._doKeyUp)):"div"!==e&&"span"!==e||i.removeClass(this.markerClassName).empty(),it===s&&(it=null,this._curInst=null))},_enableDatepicker:function(e){var t,i=V(e),s=V.data(e,"datepicker");i.hasClass(this.markerClassName)&&("input"===(t=e.nodeName.toLowerCase())?(e.disabled=!1,s.trigger.filter("button").each(function(){this.disabled=!1}).end().filter("img").css({opacity:"1.0",cursor:""})):"div"!==t&&"span"!==t||((i=i.children("."+this._inlineClass)).children().removeClass("ui-state-disabled"),i.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!1)),this._disabledInputs=V.map(this._disabledInputs,function(t){return t===e?null:t}))},_disableDatepicker:function(e){var t,i=V(e),s=V.data(e,"datepicker");i.hasClass(this.markerClassName)&&("input"===(t=e.nodeName.toLowerCase())?(e.disabled=!0,s.trigger.filter("button").each(function(){this.disabled=!0}).end().filter("img").css({opacity:"0.5",cursor:"default"})):"div"!==t&&"span"!==t||((i=i.children("."+this._inlineClass)).children().addClass("ui-state-disabled"),i.find("select.ui-datepicker-month, select.ui-datepicker-year").prop("disabled",!0)),this._disabledInputs=V.map(this._disabledInputs,function(t){return t===e?null:t}),this._disabledInputs[this._disabledInputs.length]=e)},_isDisabledDatepicker:function(t){if(!t)return!1;for(var e=0;e<this._disabledInputs.length;e++)if(this._disabledInputs[e]===t)return!0;return!1},_getInst:function(t){try{return V.data(t,"datepicker")}catch(t){throw"Missing instance data for this datepicker"}},_optionDatepicker:function(t,e,i){var s,n,o=this._getInst(t);if(2===arguments.length&&"string"==typeof e)return"defaults"===e?V.extend({},V.datepicker._defaults):o?"all"===e?V.extend({},o.settings):this._get(o,e):null;s=e||{},"string"==typeof e&&((s={})[e]=i),o&&(this._curInst===o&&this._hideDatepicker(),n=this._getDateDatepicker(t,!0),e=this._getMinMaxDate(o,"min"),i=this._getMinMaxDate(o,"max"),at(o.settings,s),null!==e&&void 0!==s.dateFormat&&void 0===s.minDate&&(o.settings.minDate=this._formatDate(o,e)),null!==i&&void 0!==s.dateFormat&&void 0===s.maxDate&&(o.settings.maxDate=this._formatDate(o,i)),"disabled"in s&&(s.disabled?this._disableDatepicker(t):this._enableDatepicker(t)),this._attachments(V(t),o),this._autoSize(o),this._setDate(o,n),this._updateAlternate(o),this._updateDatepicker(o))},_changeDatepicker:function(t,e,i){this._optionDatepicker(t,e,i)},_refreshDatepicker:function(t){t=this._getInst(t);t&&this._updateDatepicker(t)},_setDateDatepicker:function(t,e){t=this._getInst(t);t&&(this._setDate(t,e),this._updateDatepicker(t),this._updateAlternate(t))},_getDateDatepicker:function(t,e){t=this._getInst(t);return t&&!t.inline&&this._setDateFromField(t,e),t?this._getDate(t):null},_doKeyDown:function(t){var e,i,s=V.datepicker._getInst(t.target),n=!0,o=s.dpDiv.is(".ui-datepicker-rtl");if(s._keyEvent=!0,V.datepicker._datepickerShowing)switch(t.keyCode){case 9:V.datepicker._hideDatepicker(),n=!1;break;case 13:return(i=V("td."+V.datepicker._dayOverClass+":not(."+V.datepicker._currentClass+")",s.dpDiv))[0]&&V.datepicker._selectDay(t.target,s.selectedMonth,s.selectedYear,i[0]),(e=V.datepicker._get(s,"onSelect"))?(i=V.datepicker._formatDate(s),e.apply(s.input?s.input[0]:null,[i,s])):V.datepicker._hideDatepicker(),!1;case 27:V.datepicker._hideDatepicker();break;case 33:V.datepicker._adjustDate(t.target,t.ctrlKey?-V.datepicker._get(s,"stepBigMonths"):-V.datepicker._get(s,"stepMonths"),"M");break;case 34:V.datepicker._adjustDate(t.target,t.ctrlKey?+V.datepicker._get(s,"stepBigMonths"):+V.datepicker._get(s,"stepMonths"),"M");break;case 35:(t.ctrlKey||t.metaKey)&&V.datepicker._clearDate(t.target),n=t.ctrlKey||t.metaKey;break;case 36:(t.ctrlKey||t.metaKey)&&V.datepicker._gotoToday(t.target),n=t.ctrlKey||t.metaKey;break;case 37:(t.ctrlKey||t.metaKey)&&V.datepicker._adjustDate(t.target,o?1:-1,"D"),n=t.ctrlKey||t.metaKey,t.originalEvent.altKey&&V.datepicker._adjustDate(t.target,t.ctrlKey?-V.datepicker._get(s,"stepBigMonths"):-V.datepicker._get(s,"stepMonths"),"M");break;case 38:(t.ctrlKey||t.metaKey)&&V.datepicker._adjustDate(t.target,-7,"D"),n=t.ctrlKey||t.metaKey;break;case 39:(t.ctrlKey||t.metaKey)&&V.datepicker._adjustDate(t.target,o?-1:1,"D"),n=t.ctrlKey||t.metaKey,t.originalEvent.altKey&&V.datepicker._adjustDate(t.target,t.ctrlKey?+V.datepicker._get(s,"stepBigMonths"):+V.datepicker._get(s,"stepMonths"),"M");break;case 40:(t.ctrlKey||t.metaKey)&&V.datepicker._adjustDate(t.target,7,"D"),n=t.ctrlKey||t.metaKey;break;default:n=!1}else 36===t.keyCode&&t.ctrlKey?V.datepicker._showDatepicker(this):n=!1;n&&(t.preventDefault(),t.stopPropagation())},_doKeyPress:function(t){var e,i=V.datepicker._getInst(t.target);if(V.datepicker._get(i,"constrainInput"))return e=V.datepicker._possibleChars(V.datepicker._get(i,"dateFormat")),i=String.fromCharCode(null==t.charCode?t.keyCode:t.charCode),t.ctrlKey||t.metaKey||i<" "||!e||-1<e.indexOf(i)},_doKeyUp:function(t){t=V.datepicker._getInst(t.target);if(t.input.val()!==t.lastVal)try{V.datepicker.parseDate(V.datepicker._get(t,"dateFormat"),t.input?t.input.val():null,V.datepicker._getFormatConfig(t))&&(V.datepicker._setDateFromField(t),V.datepicker._updateAlternate(t),V.datepicker._updateDatepicker(t))}catch(t){}return!0},_showDatepicker:function(t){var e,i,s,n;"input"!==(t=t.target||t).nodeName.toLowerCase()&&(t=V("input",t.parentNode)[0]),V.datepicker._isDisabledDatepicker(t)||V.datepicker._lastInput===t||(n=V.datepicker._getInst(t),V.datepicker._curInst&&V.datepicker._curInst!==n&&(V.datepicker._curInst.dpDiv.stop(!0,!0),n&&V.datepicker._datepickerShowing&&V.datepicker._hideDatepicker(V.datepicker._curInst.input[0])),!1!==(i=(s=V.datepicker._get(n,"beforeShow"))?s.apply(t,[t,n]):{})&&(at(n.settings,i),n.lastVal=null,V.datepicker._lastInput=t,V.datepicker._setDateFromField(n),V.datepicker._inDialog&&(t.value=""),V.datepicker._pos||(V.datepicker._pos=V.datepicker._findPos(t),V.datepicker._pos[1]+=t.offsetHeight),e=!1,V(t).parents().each(function(){return!(e|="fixed"===V(this).css("position"))}),s={left:V.datepicker._pos[0],top:V.datepicker._pos[1]},V.datepicker._pos=null,n.dpDiv.empty(),n.dpDiv.css({position:"absolute",display:"block",top:"-1000px"}),V.datepicker._updateDatepicker(n),s=V.datepicker._checkOffset(n,s,e),n.dpDiv.css({position:V.datepicker._inDialog&&V.blockUI?"static":e?"fixed":"absolute",display:"none",left:s.left+"px",top:s.top+"px"}),n.inline||(i=V.datepicker._get(n,"showAnim"),s=V.datepicker._get(n,"duration"),n.dpDiv.css("z-index",function(t){for(var e,i;t.length&&t[0]!==document;){if(("absolute"===(e=t.css("position"))||"relative"===e||"fixed"===e)&&(i=parseInt(t.css("zIndex"),10),!isNaN(i)&&0!==i))return i;t=t.parent()}return 0}(V(t))+1),V.datepicker._datepickerShowing=!0,V.effects&&V.effects.effect[i]?n.dpDiv.show(i,V.datepicker._get(n,"showOptions"),s):n.dpDiv[i||"show"](i?s:null),V.datepicker._shouldFocusInput(n)&&n.input.trigger("focus"),V.datepicker._curInst=n)))},_updateDatepicker:function(t){this.maxRows=4,(it=t).dpDiv.empty().append(this._generateHTML(t)),this._attachHandlers(t);var e,i=this._getNumberOfMonths(t),s=i[1],n=t.dpDiv.find("."+this._dayOverClass+" a"),o=V.datepicker._get(t,"onUpdateDatepicker");0<n.length&&ot.apply(n.get(0)),t.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width(""),1<s&&t.dpDiv.addClass("ui-datepicker-multi-"+s).css("width",17*s+"em"),t.dpDiv[(1!==i[0]||1!==i[1]?"add":"remove")+"Class"]("ui-datepicker-multi"),t.dpDiv[(this._get(t,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl"),t===V.datepicker._curInst&&V.datepicker._datepickerShowing&&V.datepicker._shouldFocusInput(t)&&t.input.trigger("focus"),t.yearshtml&&(e=t.yearshtml,setTimeout(function(){e===t.yearshtml&&t.yearshtml&&t.dpDiv.find("select.ui-datepicker-year").first().replaceWith(t.yearshtml),e=t.yearshtml=null},0)),o&&o.apply(t.input?t.input[0]:null,[t])},_shouldFocusInput:function(t){return t.input&&t.input.is(":visible")&&!t.input.is(":disabled")&&!t.input.is(":focus")},_checkOffset:function(t,e,i){var s=t.dpDiv.outerWidth(),n=t.dpDiv.outerHeight(),o=t.input?t.input.outerWidth():0,a=t.input?t.input.outerHeight():0,r=document.documentElement.clientWidth+(i?0:V(document).scrollLeft()),l=document.documentElement.clientHeight+(i?0:V(document).scrollTop());return e.left-=this._get(t,"isRTL")?s-o:0,e.left-=i&&e.left===t.input.offset().left?V(document).scrollLeft():0,e.top-=i&&e.top===t.input.offset().top+a?V(document).scrollTop():0,e.left-=Math.min(e.left,e.left+s>r&&s<r?Math.abs(e.left+s-r):0),e.top-=Math.min(e.top,e.top+n>l&&n<l?Math.abs(n+a):0),e},_findPos:function(t){for(var e=this._getInst(t),i=this._get(e,"isRTL");t&&("hidden"===t.type||1!==t.nodeType||V.expr.pseudos.hidden(t));)t=t[i?"previousSibling":"nextSibling"];return[(e=V(t).offset()).left,e.top]},_hideDatepicker:function(t){var e,i,s=this._curInst;!s||t&&s!==V.data(t,"datepicker")||this._datepickerShowing&&(e=this._get(s,"showAnim"),i=this._get(s,"duration"),t=function(){V.datepicker._tidyDialog(s)},V.effects&&(V.effects.effect[e]||V.effects[e])?s.dpDiv.hide(e,V.datepicker._get(s,"showOptions"),i,t):s.dpDiv["slideDown"===e?"slideUp":"fadeIn"===e?"fadeOut":"hide"](e?i:null,t),e||t(),this._datepickerShowing=!1,(t=this._get(s,"onClose"))&&t.apply(s.input?s.input[0]:null,[s.input?s.input.val():"",s]),this._lastInput=null,this._inDialog&&(this._dialogInput.css({position:"absolute",left:"0",top:"-100px"}),V.blockUI&&(V.unblockUI(),V("body").append(this.dpDiv))),this._inDialog=!1)},_tidyDialog:function(t){t.dpDiv.removeClass(this._dialogClass).off(".ui-datepicker-calendar")},_checkExternalClick:function(t){var e;V.datepicker._curInst&&(e=V(t.target),t=V.datepicker._getInst(e[0]),(e[0].id===V.datepicker._mainDivId||0!==e.parents("#"+V.datepicker._mainDivId).length||e.hasClass(V.datepicker.markerClassName)||e.closest("."+V.datepicker._triggerClass).length||!V.datepicker._datepickerShowing||V.datepicker._inDialog&&V.blockUI)&&(!e.hasClass(V.datepicker.markerClassName)||V.datepicker._curInst===t)||V.datepicker._hideDatepicker())},_adjustDate:function(t,e,i){var s=V(t),t=this._getInst(s[0]);this._isDisabledDatepicker(s[0])||(this._adjustInstDate(t,e,i),this._updateDatepicker(t))},_gotoToday:function(t){var e=V(t),i=this._getInst(e[0]);this._get(i,"gotoCurrent")&&i.currentDay?(i.selectedDay=i.currentDay,i.drawMonth=i.selectedMonth=i.currentMonth,i.drawYear=i.selectedYear=i.currentYear):(t=new Date,i.selectedDay=t.getDate(),i.drawMonth=i.selectedMonth=t.getMonth(),i.drawYear=i.selectedYear=t.getFullYear()),this._notifyChange(i),this._adjustDate(e)},_selectMonthYear:function(t,e,i){var s=V(t),t=this._getInst(s[0]);t["selected"+("M"===i?"Month":"Year")]=t["draw"+("M"===i?"Month":"Year")]=parseInt(e.options[e.selectedIndex].value,10),this._notifyChange(t),this._adjustDate(s)},_selectDay:function(t,e,i,s){var n=V(t);V(s).hasClass(this._unselectableClass)||this._isDisabledDatepicker(n[0])||((n=this._getInst(n[0])).selectedDay=n.currentDay=parseInt(V("a",s).attr("data-date")),n.selectedMonth=n.currentMonth=e,n.selectedYear=n.currentYear=i,this._selectDate(t,this._formatDate(n,n.currentDay,n.currentMonth,n.currentYear)))},_clearDate:function(t){t=V(t);this._selectDate(t,"")},_selectDate:function(t,e){var i=V(t),t=this._getInst(i[0]);e=null!=e?e:this._formatDate(t),t.input&&t.input.val(e),this._updateAlternate(t),(i=this._get(t,"onSelect"))?i.apply(t.input?t.input[0]:null,[e,t]):t.input&&t.input.trigger("change"),t.inline?this._updateDatepicker(t):(this._hideDatepicker(),this._lastInput=t.input[0],"object"!=typeof t.input[0]&&t.input.trigger("focus"),this._lastInput=null)},_updateAlternate:function(t){var e,i,s=this._get(t,"altField");s&&(e=this._get(t,"altFormat")||this._get(t,"dateFormat"),i=this._getDate(t),t=this.formatDate(e,i,this._getFormatConfig(t)),V(document).find(s).val(t))},noWeekends:function(t){t=t.getDay();return[0<t&&t<6,""]},iso8601Week:function(t){var e=new Date(t.getTime());return e.setDate(e.getDate()+4-(e.getDay()||7)),t=e.getTime(),e.setMonth(0),e.setDate(1),Math.floor(Math.round((t-e)/864e5)/7)+1},parseDate:function(e,n,t){if(null==e||null==n)throw"Invalid arguments";if(""===(n="object"==typeof n?n.toString():n+""))return null;for(var i,s,o,a=0,r=(t?t.shortYearCutoff:null)||this._defaults.shortYearCutoff,r="string"!=typeof r?r:(new Date).getFullYear()%100+parseInt(r,10),l=(t?t.dayNamesShort:null)||this._defaults.dayNamesShort,h=(t?t.dayNames:null)||this._defaults.dayNames,c=(t?t.monthNamesShort:null)||this._defaults.monthNamesShort,u=(t?t.monthNames:null)||this._defaults.monthNames,d=-1,p=-1,f=-1,g=-1,m=!1,_=function(t){t=w+1<e.length&&e.charAt(w+1)===t;return t&&w++,t},v=function(t){var e=_(t),e="@"===t?14:"!"===t?20:"y"===t&&e?4:"o"===t?3:2,e=new RegExp("^\\d{"+("y"===t?e:1)+","+e+"}"),e=n.substring(a).match(e);if(!e)throw"Missing number at position "+a;return a+=e[0].length,parseInt(e[0],10)},b=function(t,e,i){var s=-1,e=V.map(_(t)?i:e,function(t,e){return[[e,t]]}).sort(function(t,e){return-(t[1].length-e[1].length)});if(V.each(e,function(t,e){var i=e[1];if(n.substr(a,i.length).toLowerCase()===i.toLowerCase())return s=e[0],a+=i.length,!1}),-1!==s)return s+1;throw"Unknown name at position "+a},y=function(){if(n.charAt(a)!==e.charAt(w))throw"Unexpected literal at position "+a;a++},w=0;w<e.length;w++)if(m)"'"!==e.charAt(w)||_("'")?y():m=!1;else switch(e.charAt(w)){case"d":f=v("d");break;case"D":b("D",l,h);break;case"o":g=v("o");break;case"m":p=v("m");break;case"M":p=b("M",c,u);break;case"y":d=v("y");break;case"@":d=(o=new Date(v("@"))).getFullYear(),p=o.getMonth()+1,f=o.getDate();break;case"!":d=(o=new Date((v("!")-this._ticksTo1970)/1e4)).getFullYear(),p=o.getMonth()+1,f=o.getDate();break;case"'":_("'")?y():m=!0;break;default:y()}if(a<n.length&&(s=n.substr(a),!/^\s+/.test(s)))throw"Extra/unparsed characters found in date: "+s;if(-1===d?d=(new Date).getFullYear():d<100&&(d+=(new Date).getFullYear()-(new Date).getFullYear()%100+(d<=r?0:-100)),-1<g)for(p=1,f=g;;){if(f<=(i=this._getDaysInMonth(d,p-1)))break;p++,f-=i}if((o=this._daylightSavingAdjust(new Date(d,p-1,f))).getFullYear()!==d||o.getMonth()+1!==p||o.getDate()!==f)throw"Invalid date";return o},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:24*(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*60*60*1e7,formatDate:function(e,t,i){if(!t)return"";function s(t,e,i){var s=""+e;if(c(t))for(;s.length<i;)s="0"+s;return s}function n(t,e,i,s){return(c(t)?s:i)[e]}var o,a=(i?i.dayNamesShort:null)||this._defaults.dayNamesShort,r=(i?i.dayNames:null)||this._defaults.dayNames,l=(i?i.monthNamesShort:null)||this._defaults.monthNamesShort,h=(i?i.monthNames:null)||this._defaults.monthNames,c=function(t){t=o+1<e.length&&e.charAt(o+1)===t;return t&&o++,t},u="",d=!1;if(t)for(o=0;o<e.length;o++)if(d)"'"!==e.charAt(o)||c("'")?u+=e.charAt(o):d=!1;else switch(e.charAt(o)){case"d":u+=s("d",t.getDate(),2);break;case"D":u+=n("D",t.getDay(),a,r);break;case"o":u+=s("o",Math.round((new Date(t.getFullYear(),t.getMonth(),t.getDate()).getTime()-new Date(t.getFullYear(),0,0).getTime())/864e5),3);break;case"m":u+=s("m",t.getMonth()+1,2);break;case"M":u+=n("M",t.getMonth(),l,h);break;case"y":u+=c("y")?t.getFullYear():(t.getFullYear()%100<10?"0":"")+t.getFullYear()%100;break;case"@":u+=t.getTime();break;case"!":u+=1e4*t.getTime()+this._ticksTo1970;break;case"'":c("'")?u+="'":d=!0;break;default:u+=e.charAt(o)}return u},_possibleChars:function(e){for(var t="",i=!1,s=function(t){t=n+1<e.length&&e.charAt(n+1)===t;return t&&n++,t},n=0;n<e.length;n++)if(i)"'"!==e.charAt(n)||s("'")?t+=e.charAt(n):i=!1;else switch(e.charAt(n)){case"d":case"m":case"y":case"@":t+="0123456789";break;case"D":case"M":return null;case"'":s("'")?t+="'":i=!0;break;default:t+=e.charAt(n)}return t},_get:function(t,e){return(void 0!==t.settings[e]?t.settings:this._defaults)[e]},_setDateFromField:function(t,e){if(t.input.val()!==t.lastVal){var i=this._get(t,"dateFormat"),s=t.lastVal=t.input?t.input.val():null,n=this._getDefaultDate(t),o=n,a=this._getFormatConfig(t);try{o=this.parseDate(i,s,a)||n}catch(t){s=e?"":s}t.selectedDay=o.getDate(),t.drawMonth=t.selectedMonth=o.getMonth(),t.drawYear=t.selectedYear=o.getFullYear(),t.currentDay=s?o.getDate():0,t.currentMonth=s?o.getMonth():0,t.currentYear=s?o.getFullYear():0,this._adjustInstDate(t)}},_getDefaultDate:function(t){return this._restrictMinMax(t,this._determineDate(t,this._get(t,"defaultDate"),new Date))},_determineDate:function(r,t,e){var i,s,t=null==t||""===t?e:"string"==typeof t?function(t){try{return V.datepicker.parseDate(V.datepicker._get(r,"dateFormat"),t,V.datepicker._getFormatConfig(r))}catch(t){}for(var e=(t.toLowerCase().match(/^c/)?V.datepicker._getDate(r):null)||new Date,i=e.getFullYear(),s=e.getMonth(),n=e.getDate(),o=/([+\-]?[0-9]+)\s*(d|D|w|W|m|M|y|Y)?/g,a=o.exec(t);a;){switch(a[2]||"d"){case"d":case"D":n+=parseInt(a[1],10);break;case"w":case"W":n+=7*parseInt(a[1],10);break;case"m":case"M":s+=parseInt(a[1],10),n=Math.min(n,V.datepicker._getDaysInMonth(i,s));break;case"y":case"Y":i+=parseInt(a[1],10),n=Math.min(n,V.datepicker._getDaysInMonth(i,s))}a=o.exec(t)}return new Date(i,s,n)}(t):"number"==typeof t?isNaN(t)?e:(i=t,(s=new Date).setDate(s.getDate()+i),s):new Date(t.getTime());return(t=t&&"Invalid Date"===t.toString()?e:t)&&(t.setHours(0),t.setMinutes(0),t.setSeconds(0),t.setMilliseconds(0)),this._daylightSavingAdjust(t)},_daylightSavingAdjust:function(t){return t?(t.setHours(12<t.getHours()?t.getHours()+2:0),t):null},_setDate:function(t,e,i){var s=!e,n=t.selectedMonth,o=t.selectedYear,e=this._restrictMinMax(t,this._determineDate(t,e,new Date));t.selectedDay=t.currentDay=e.getDate(),t.drawMonth=t.selectedMonth=t.currentMonth=e.getMonth(),t.drawYear=t.selectedYear=t.currentYear=e.getFullYear(),n===t.selectedMonth&&o===t.selectedYear||i||this._notifyChange(t),this._adjustInstDate(t),t.input&&t.input.val(s?"":this._formatDate(t))},_getDate:function(t){return!t.currentYear||t.input&&""===t.input.val()?null:this._daylightSavingAdjust(new Date(t.currentYear,t.currentMonth,t.currentDay))},_attachHandlers:function(t){var e=this._get(t,"stepMonths"),i="#"+t.id.replace(/\\\\/g,"\\");t.dpDiv.find("[data-handler]").map(function(){var t={prev:function(){V.datepicker._adjustDate(i,-e,"M")},next:function(){V.datepicker._adjustDate(i,+e,"M")},hide:function(){V.datepicker._hideDatepicker()},today:function(){V.datepicker._gotoToday(i)},selectDay:function(){return V.datepicker._selectDay(i,+this.getAttribute("data-month"),+this.getAttribute("data-year"),this),!1},selectMonth:function(){return V.datepicker._selectMonthYear(i,this,"M"),!1},selectYear:function(){return V.datepicker._selectMonthYear(i,this,"Y"),!1}};V(this).on(this.getAttribute("data-event"),t[this.getAttribute("data-handler")])})},_generateHTML:function(t){var e,i,s,n,o,a,r,l,h,c,u,d,p,f,g,m,_,v,b,y,w,x,k,C,D,I,T,P,M,S,H,z,A=new Date,O=this._daylightSavingAdjust(new Date(A.getFullYear(),A.getMonth(),A.getDate())),N=this._get(t,"isRTL"),E=this._get(t,"showButtonPanel"),W=this._get(t,"hideIfNoPrevNext"),F=this._get(t,"navigationAsDateFormat"),L=this._getNumberOfMonths(t),R=this._get(t,"showCurrentAtPos"),A=this._get(t,"stepMonths"),Y=1!==L[0]||1!==L[1],B=this._daylightSavingAdjust(t.currentDay?new Date(t.currentYear,t.currentMonth,t.currentDay):new Date(9999,9,9)),j=this._getMinMaxDate(t,"min"),q=this._getMinMaxDate(t,"max"),K=t.drawMonth-R,U=t.drawYear;if(K<0&&(K+=12,U--),q)for(e=this._daylightSavingAdjust(new Date(q.getFullYear(),q.getMonth()-L[0]*L[1]+1,q.getDate())),e=j&&e<j?j:e;this._daylightSavingAdjust(new Date(U,K,1))>e;)--K<0&&(K=11,U--);for(t.drawMonth=K,t.drawYear=U,R=this._get(t,"prevText"),R=F?this.formatDate(R,this._daylightSavingAdjust(new Date(U,K-A,1)),this._getFormatConfig(t)):R,i=this._canAdjustMonth(t,-1,U,K)?V("<a>").attr({class:"ui-datepicker-prev ui-corner-all","data-handler":"prev","data-event":"click",title:R}).append(V("<span>").addClass("ui-icon ui-icon-circle-triangle-"+(N?"e":"w")).text(R))[0].outerHTML:W?"":V("<a>").attr({class:"ui-datepicker-prev ui-corner-all ui-state-disabled",title:R}).append(V("<span>").addClass("ui-icon ui-icon-circle-triangle-"+(N?"e":"w")).text(R))[0].outerHTML,R=this._get(t,"nextText"),R=F?this.formatDate(R,this._daylightSavingAdjust(new Date(U,K+A,1)),this._getFormatConfig(t)):R,s=this._canAdjustMonth(t,1,U,K)?V("<a>").attr({class:"ui-datepicker-next ui-corner-all","data-handler":"next","data-event":"click",title:R}).append(V("<span>").addClass("ui-icon ui-icon-circle-triangle-"+(N?"w":"e")).text(R))[0].outerHTML:W?"":V("<a>").attr({class:"ui-datepicker-next ui-corner-all ui-state-disabled",title:R}).append(V("<span>").attr("class","ui-icon ui-icon-circle-triangle-"+(N?"w":"e")).text(R))[0].outerHTML,A=this._get(t,"currentText"),W=this._get(t,"gotoCurrent")&&t.currentDay?B:O,A=F?this.formatDate(A,W,this._getFormatConfig(t)):A,R="",t.inline||(R=V("<button>").attr({type:"button",class:"ui-datepicker-close ui-state-default ui-priority-primary ui-corner-all","data-handler":"hide","data-event":"click"}).text(this._get(t,"closeText"))[0].outerHTML),F="",E&&(F=V("<div class='ui-datepicker-buttonpane ui-widget-content'>").append(N?R:"").append(this._isInRange(t,W)?V("<button>").attr({type:"button",class:"ui-datepicker-current ui-state-default ui-priority-secondary ui-corner-all","data-handler":"today","data-event":"click"}).text(A):"").append(N?"":R)[0].outerHTML),n=parseInt(this._get(t,"firstDay"),10),n=isNaN(n)?0:n,o=this._get(t,"showWeek"),a=this._get(t,"dayNames"),r=this._get(t,"dayNamesMin"),l=this._get(t,"monthNames"),h=this._get(t,"monthNamesShort"),c=this._get(t,"beforeShowDay"),u=this._get(t,"showOtherMonths"),d=this._get(t,"selectOtherMonths"),p=this._getDefaultDate(t),f="",m=0;m<L[0];m++){for(_="",this.maxRows=4,v=0;v<L[1];v++){if(b=this._daylightSavingAdjust(new Date(U,K,t.selectedDay)),y=" ui-corner-all",w="",Y){if(w+="<div class='ui-datepicker-group",1<L[1])switch(v){case 0:w+=" ui-datepicker-group-first",y=" ui-corner-"+(N?"right":"left");break;case L[1]-1:w+=" ui-datepicker-group-last",y=" ui-corner-"+(N?"left":"right");break;default:w+=" ui-datepicker-group-middle",y=""}w+="'>"}for(w+="<div class='ui-datepicker-header ui-widget-header ui-helper-clearfix"+y+"'>"+(/all|left/.test(y)&&0===m?N?s:i:"")+(/all|right/.test(y)&&0===m?N?i:s:"")+this._generateMonthYearHeader(t,K,U,j,q,0<m||0<v,l,h)+"</div><table class='ui-datepicker-calendar'><thead><tr>",x=o?"<th class='ui-datepicker-week-col'>"+this._get(t,"weekHeader")+"</th>":"",g=0;g<7;g++)x+="<th scope='col'"+(5<=(g+n+6)%7?" class='ui-datepicker-week-end'":"")+"><span title='"+a[k=(g+n)%7]+"'>"+r[k]+"</span></th>";for(w+=x+"</tr></thead><tbody>",D=this._getDaysInMonth(U,K),U===t.selectedYear&&K===t.selectedMonth&&(t.selectedDay=Math.min(t.selectedDay,D)),C=(this._getFirstDayOfMonth(U,K)-n+7)%7,D=Math.ceil((C+D)/7),I=Y&&this.maxRows>D?this.maxRows:D,this.maxRows=I,T=this._daylightSavingAdjust(new Date(U,K,1-C)),P=0;P<I;P++){for(w+="<tr>",M=o?"<td class='ui-datepicker-week-col'>"+this._get(t,"calculateWeek")(T)+"</td>":"",g=0;g<7;g++)S=c?c.apply(t.input?t.input[0]:null,[T]):[!0,""],z=(H=T.getMonth()!==K)&&!d||!S[0]||j&&T<j||q&&q<T,M+="<td class='"+(5<=(g+n+6)%7?" ui-datepicker-week-end":"")+(H?" ui-datepicker-other-month":"")+(T.getTime()===b.getTime()&&K===t.selectedMonth&&t._keyEvent||p.getTime()===T.getTime()&&p.getTime()===b.getTime()?" "+this._dayOverClass:"")+(z?" "+this._unselectableClass+" ui-state-disabled":"")+(H&&!u?"":" "+S[1]+(T.getTime()===B.getTime()?" "+this._currentClass:"")+(T.getTime()===O.getTime()?" ui-datepicker-today":""))+"'"+(H&&!u||!S[2]?"":" title='"+S[2].replace(/'/g,"&#39;")+"'")+(z?"":" data-handler='selectDay' data-event='click' data-month='"+T.getMonth()+"' data-year='"+T.getFullYear()+"'")+">"+(H&&!u?"&#xa0;":z?"<span class='ui-state-default'>"+T.getDate()+"</span>":"<a class='ui-state-default"+(T.getTime()===O.getTime()?" ui-state-highlight":"")+(T.getTime()===B.getTime()?" ui-state-active":"")+(H?" ui-priority-secondary":"")+"' href='#' aria-current='"+(T.getTime()===B.getTime()?"true":"false")+"' data-date='"+T.getDate()+"'>"+T.getDate()+"</a>")+"</td>",T.setDate(T.getDate()+1),T=this._daylightSavingAdjust(T);w+=M+"</tr>"}11<++K&&(K=0,U++),_+=w+="</tbody></table>"+(Y?"</div>"+(0<L[0]&&v===L[1]-1?"<div class='ui-datepicker-row-break'></div>":""):"")}f+=_}return f+=F,t._keyEvent=!1,f},_generateMonthYearHeader:function(t,e,i,s,n,o,a,r){var l,h,c,u,d,p,f=this._get(t,"changeMonth"),g=this._get(t,"changeYear"),m=this._get(t,"showMonthAfterYear"),_=this._get(t,"selectMonthLabel"),v=this._get(t,"selectYearLabel"),b="<div class='ui-datepicker-title'>",y="";if(o||!f)y+="<span class='ui-datepicker-month'>"+a[e]+"</span>";else{for(l=s&&s.getFullYear()===i,h=n&&n.getFullYear()===i,y+="<select class='ui-datepicker-month' aria-label='"+_+"' data-handler='selectMonth' data-event='change'>",c=0;c<12;c++)(!l||c>=s.getMonth())&&(!h||c<=n.getMonth())&&(y+="<option value='"+c+"'"+(c===e?" selected='selected'":"")+">"+r[c]+"</option>");y+="</select>"}if(m||(b+=y+(!o&&f&&g?"":"&#xa0;")),!t.yearshtml)if(t.yearshtml="",o||!g)b+="<span class='ui-datepicker-year'>"+i+"</span>";else{for(a=this._get(t,"yearRange").split(":"),u=(new Date).getFullYear(),d=(_=function(t){t=t.match(/c[+\-].*/)?i+parseInt(t.substring(1),10):t.match(/[+\-].*/)?u+parseInt(t,10):parseInt(t,10);return isNaN(t)?u:t})(a[0]),p=Math.max(d,_(a[1]||"")),d=s?Math.max(d,s.getFullYear()):d,p=n?Math.min(p,n.getFullYear()):p,t.yearshtml+="<select class='ui-datepicker-year' aria-label='"+v+"' data-handler='selectYear' data-event='change'>";d<=p;d++)t.yearshtml+="<option value='"+d+"'"+(d===i?" selected='selected'":"")+">"+d+"</option>";t.yearshtml+="</select>",b+=t.yearshtml,t.yearshtml=null}return b+=this._get(t,"yearSuffix"),m&&(b+=(!o&&f&&g?"":"&#xa0;")+y),b+="</div>"},_adjustInstDate:function(t,e,i){var s=t.selectedYear+("Y"===i?e:0),n=t.selectedMonth+("M"===i?e:0),e=Math.min(t.selectedDay,this._getDaysInMonth(s,n))+("D"===i?e:0),e=this._restrictMinMax(t,this._daylightSavingAdjust(new Date(s,n,e)));t.selectedDay=e.getDate(),t.drawMonth=t.selectedMonth=e.getMonth(),t.drawYear=t.selectedYear=e.getFullYear(),"M"!==i&&"Y"!==i||this._notifyChange(t)},_restrictMinMax:function(t,e){var i=this._getMinMaxDate(t,"min"),t=this._getMinMaxDate(t,"max"),e=i&&e<i?i:e;return t&&t<e?t:e},_notifyChange:function(t){var e=this._get(t,"onChangeMonthYear");e&&e.apply(t.input?t.input[0]:null,[t.selectedYear,t.selectedMonth+1,t])},_getNumberOfMonths:function(t){t=this._get(t,"numberOfMonths");return null==t?[1,1]:"number"==typeof t?[1,t]:t},_getMinMaxDate:function(t,e){return this._determineDate(t,this._get(t,e+"Date"),null)},_getDaysInMonth:function(t,e){return 32-this._daylightSavingAdjust(new Date(t,e,32)).getDate()},_getFirstDayOfMonth:function(t,e){return new Date(t,e,1).getDay()},_canAdjustMonth:function(t,e,i,s){var n=this._getNumberOfMonths(t),n=this._daylightSavingAdjust(new Date(i,s+(e<0?e:n[0]*n[1]),1));return e<0&&n.setDate(this._getDaysInMonth(n.getFullYear(),n.getMonth())),this._isInRange(t,n)},_isInRange:function(t,e){var i=this._getMinMaxDate(t,"min"),s=this._getMinMaxDate(t,"max"),n=null,o=null,a=this._get(t,"yearRange");return a&&(t=a.split(":"),a=(new Date).getFullYear(),n=parseInt(t[0],10),o=parseInt(t[1],10),t[0].match(/[+\-].*/)&&(n+=a),t[1].match(/[+\-].*/)&&(o+=a)),(!i||e.getTime()>=i.getTime())&&(!s||e.getTime()<=s.getTime())&&(!n||e.getFullYear()>=n)&&(!o||e.getFullYear()<=o)},_getFormatConfig:function(t){var e=this._get(t,"shortYearCutoff");return{shortYearCutoff:e="string"!=typeof e?e:(new Date).getFullYear()%100+parseInt(e,10),dayNamesShort:this._get(t,"dayNamesShort"),dayNames:this._get(t,"dayNames"),monthNamesShort:this._get(t,"monthNamesShort"),monthNames:this._get(t,"monthNames")}},_formatDate:function(t,e,i,s){e||(t.currentDay=t.selectedDay,t.currentMonth=t.selectedMonth,t.currentYear=t.selectedYear);e=e?"object"==typeof e?e:this._daylightSavingAdjust(new Date(s,i,e)):this._daylightSavingAdjust(new Date(t.currentYear,t.currentMonth,t.currentDay));return this.formatDate(this._get(t,"dateFormat"),e,this._getFormatConfig(t))}}),V.fn.datepicker=function(t){if(!this.length)return this;V.datepicker.initialized||(V(document).on("mousedown",V.datepicker._checkExternalClick),V.datepicker.initialized=!0),0===V("#"+V.datepicker._mainDivId).length&&V("body").append(V.datepicker.dpDiv);var e=Array.prototype.slice.call(arguments,1);return"string"==typeof t&&("isDisabled"===t||"getDate"===t||"widget"===t)||"option"===t&&2===arguments.length&&"string"==typeof arguments[1]?V.datepicker["_"+t+"Datepicker"].apply(V.datepicker,[this[0]].concat(e)):this.each(function(){"string"==typeof t?V.datepicker["_"+t+"Datepicker"].apply(V.datepicker,[this].concat(e)):V.datepicker._attachDatepicker(this,t)})},V.datepicker=new st,V.datepicker.initialized=!1,V.datepicker.uuid=(new Date).getTime(),V.datepicker.version="1.13.2";V.datepicker,V.ui.ie=!!/msie [\w.]+/.exec(navigator.userAgent.toLowerCase());var rt=!1;V(document).on("mouseup",function(){rt=!1});V.widget("ui.mouse",{version:"1.13.2",options:{cancel:"input, textarea, button, select, option",distance:1,delay:0},_mouseInit:function(){var e=this;this.element.on("mousedown."+this.widgetName,function(t){return e._mouseDown(t)}).on("click."+this.widgetName,function(t){if(!0===V.data(t.target,e.widgetName+".preventClickEvent"))return V.removeData(t.target,e.widgetName+".preventClickEvent"),t.stopImmediatePropagation(),!1}),this.started=!1},_mouseDestroy:function(){this.element.off("."+this.widgetName),this._mouseMoveDelegate&&this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate)},_mouseDown:function(t){if(!rt){this._mouseMoved=!1,this._mouseStarted&&this._mouseUp(t),this._mouseDownEvent=t;var e=this,i=1===t.which,s=!("string"!=typeof this.options.cancel||!t.target.nodeName)&&V(t.target).closest(this.options.cancel).length;return i&&!s&&this._mouseCapture(t)?(this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){e.mouseDelayMet=!0},this.options.delay)),this._mouseDistanceMet(t)&&this._mouseDelayMet(t)&&(this._mouseStarted=!1!==this._mouseStart(t),!this._mouseStarted)?(t.preventDefault(),!0):(!0===V.data(t.target,this.widgetName+".preventClickEvent")&&V.removeData(t.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(t){return e._mouseMove(t)},this._mouseUpDelegate=function(t){return e._mouseUp(t)},this.document.on("mousemove."+this.widgetName,this._mouseMoveDelegate).on("mouseup."+this.widgetName,this._mouseUpDelegate),t.preventDefault(),rt=!0)):!0}},_mouseMove:function(t){if(this._mouseMoved){if(V.ui.ie&&(!document.documentMode||document.documentMode<9)&&!t.button)return this._mouseUp(t);if(!t.which)if(t.originalEvent.altKey||t.originalEvent.ctrlKey||t.originalEvent.metaKey||t.originalEvent.shiftKey)this.ignoreMissingWhich=!0;else if(!this.ignoreMissingWhich)return this._mouseUp(t)}return(t.which||t.button)&&(this._mouseMoved=!0),this._mouseStarted?(this._mouseDrag(t),t.preventDefault()):(this._mouseDistanceMet(t)&&this._mouseDelayMet(t)&&(this._mouseStarted=!1!==this._mouseStart(this._mouseDownEvent,t),this._mouseStarted?this._mouseDrag(t):this._mouseUp(t)),!this._mouseStarted)},_mouseUp:function(t){this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,t.target===this._mouseDownEvent.target&&V.data(t.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(t)),this._mouseDelayTimer&&(clearTimeout(this._mouseDelayTimer),delete this._mouseDelayTimer),this.ignoreMissingWhich=!1,rt=!1,t.preventDefault()},_mouseDistanceMet:function(t){return Math.max(Math.abs(this._mouseDownEvent.pageX-t.pageX),Math.abs(this._mouseDownEvent.pageY-t.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return!0}}),V.ui.plugin={add:function(t,e,i){var s,n=V.ui[t].prototype;for(s in i)n.plugins[s]=n.plugins[s]||[],n.plugins[s].push([e,i[s]])},call:function(t,e,i,s){var n,o=t.plugins[e];if(o&&(s||t.element[0].parentNode&&11!==t.element[0].parentNode.nodeType))for(n=0;n<o.length;n++)t.options[o[n][0]]&&o[n][1].apply(t.element,i)}},V.ui.safeBlur=function(t){t&&"body"!==t.nodeName.toLowerCase()&&V(t).trigger("blur")};V.widget("ui.draggable",V.ui.mouse,{version:"1.13.2",widgetEventPrefix:"drag",options:{addClasses:!0,appendTo:"parent",axis:!1,connectToSortable:!1,containment:!1,cursor:"auto",cursorAt:!1,grid:!1,handle:!1,helper:"original",iframeFix:!1,opacity:!1,refreshPositions:!1,revert:!1,revertDuration:500,scope:"default",scroll:!0,scrollSensitivity:20,scrollSpeed:20,snap:!1,snapMode:"both",snapTolerance:20,stack:!1,zIndex:!1,drag:null,start:null,stop:null},_create:function(){"original"===this.options.helper&&this._setPositionRelative(),this.options.addClasses&&this._addClass("ui-draggable"),this._setHandleClassName(),this._mouseInit()},_setOption:function(t,e){this._super(t,e),"handle"===t&&(this._removeHandleClassName(),this._setHandleClassName())},_destroy:function(){(this.helper||this.element).is(".ui-draggable-dragging")?this.destroyOnClear=!0:(this._removeHandleClassName(),this._mouseDestroy())},_mouseCapture:function(t){var e=this.options;return!(this.helper||e.disabled||0<V(t.target).closest(".ui-resizable-handle").length)&&(this.handle=this._getHandle(t),!!this.handle&&(this._blurActiveElement(t),this._blockFrames(!0===e.iframeFix?"iframe":e.iframeFix),!0))},_blockFrames:function(t){this.iframeBlocks=this.document.find(t).map(function(){var t=V(this);return V("<div>").css("position","absolute").appendTo(t.parent()).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()).offset(t.offset())[0]})},_unblockFrames:function(){this.iframeBlocks&&(this.iframeBlocks.remove(),delete this.iframeBlocks)},_blurActiveElement:function(t){var e=V.ui.safeActiveElement(this.document[0]);V(t.target).closest(e).length||V.ui.safeBlur(e)},_mouseStart:function(t){var e=this.options;return this.helper=this._createHelper(t),this._addClass(this.helper,"ui-draggable-dragging"),this._cacheHelperProportions(),V.ui.ddmanager&&(V.ui.ddmanager.current=this),this._cacheMargins(),this.cssPosition=this.helper.css("position"),this.scrollParent=this.helper.scrollParent(!0),this.offsetParent=this.helper.offsetParent(),this.hasFixedAncestor=0<this.helper.parents().filter(function(){return"fixed"===V(this).css("position")}).length,this.positionAbs=this.element.offset(),this._refreshOffsets(t),this.originalPosition=this.position=this._generatePosition(t,!1),this.originalPageX=t.pageX,this.originalPageY=t.pageY,e.cursorAt&&this._adjustOffsetFromHelper(e.cursorAt),this._setContainment(),!1===this._trigger("start",t)?(this._clear(),!1):(this._cacheHelperProportions(),V.ui.ddmanager&&!e.dropBehaviour&&V.ui.ddmanager.prepareOffsets(this,t),this._mouseDrag(t,!0),V.ui.ddmanager&&V.ui.ddmanager.dragStart(this,t),!0)},_refreshOffsets:function(t){this.offset={top:this.positionAbs.top-this.margins.top,left:this.positionAbs.left-this.margins.left,scroll:!1,parent:this._getParentOffset(),relative:this._getRelativeOffset()},this.offset.click={left:t.pageX-this.offset.left,top:t.pageY-this.offset.top}},_mouseDrag:function(t,e){if(this.hasFixedAncestor&&(this.offset.parent=this._getParentOffset()),this.position=this._generatePosition(t,!0),this.positionAbs=this._convertPositionTo("absolute"),!e){e=this._uiHash();if(!1===this._trigger("drag",t,e))return this._mouseUp(new V.Event("mouseup",t)),!1;this.position=e.position}return this.helper[0].style.left=this.position.left+"px",this.helper[0].style.top=this.position.top+"px",V.ui.ddmanager&&V.ui.ddmanager.drag(this,t),!1},_mouseStop:function(t){var e=this,i=!1;return V.ui.ddmanager&&!this.options.dropBehaviour&&(i=V.ui.ddmanager.drop(this,t)),this.dropped&&(i=this.dropped,this.dropped=!1),"invalid"===this.options.revert&&!i||"valid"===this.options.revert&&i||!0===this.options.revert||"function"==typeof this.options.revert&&this.options.revert.call(this.element,i)?V(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){!1!==e._trigger("stop",t)&&e._clear()}):!1!==this._trigger("stop",t)&&this._clear(),!1},_mouseUp:function(t){return this._unblockFrames(),V.ui.ddmanager&&V.ui.ddmanager.dragStop(this,t),this.handleElement.is(t.target)&&this.element.trigger("focus"),V.ui.mouse.prototype._mouseUp.call(this,t)},cancel:function(){return this.helper.is(".ui-draggable-dragging")?this._mouseUp(new V.Event("mouseup",{target:this.element[0]})):this._clear(),this},_getHandle:function(t){return!this.options.handle||!!V(t.target).closest(this.element.find(this.options.handle)).length},_setHandleClassName:function(){this.handleElement=this.options.handle?this.element.find(this.options.handle):this.element,this._addClass(this.handleElement,"ui-draggable-handle")},_removeHandleClassName:function(){this._removeClass(this.handleElement,"ui-draggable-handle")},_createHelper:function(t){var e=this.options,i="function"==typeof e.helper,t=i?V(e.helper.apply(this.element[0],[t])):"clone"===e.helper?this.element.clone().removeAttr("id"):this.element;return t.parents("body").length||t.appendTo("parent"===e.appendTo?this.element[0].parentNode:e.appendTo),i&&t[0]===this.element[0]&&this._setPositionRelative(),t[0]===this.element[0]||/(fixed|absolute)/.test(t.css("position"))||t.css("position","absolute"),t},_setPositionRelative:function(){/^(?:r|a|f)/.test(this.element.css("position"))||(this.element[0].style.position="relative")},_adjustOffsetFromHelper:function(t){"string"==typeof t&&(t=t.split(" ")),"left"in(t=Array.isArray(t)?{left:+t[0],top:+t[1]||0}:t)&&(this.offset.click.left=t.left+this.margins.left),"right"in t&&(this.offset.click.left=this.helperProportions.width-t.right+this.margins.left),"top"in t&&(this.offset.click.top=t.top+this.margins.top),"bottom"in t&&(this.offset.click.top=this.helperProportions.height-t.bottom+this.margins.top)},_isRootNode:function(t){return/(html|body)/i.test(t.tagName)||t===this.document[0]},_getParentOffset:function(){var t=this.offsetParent.offset(),e=this.document[0];return"absolute"===this.cssPosition&&this.scrollParent[0]!==e&&V.contains(this.scrollParent[0],this.offsetParent[0])&&(t.left+=this.scrollParent.scrollLeft(),t.top+=this.scrollParent.scrollTop()),{top:(t=this._isRootNode(this.offsetParent[0])?{top:0,left:0}:t).top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:t.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if("relative"!==this.cssPosition)return{top:0,left:0};var t=this.element.position(),e=this._isRootNode(this.scrollParent[0]);return{top:t.top-(parseInt(this.helper.css("top"),10)||0)+(e?0:this.scrollParent.scrollTop()),left:t.left-(parseInt(this.helper.css("left"),10)||0)+(e?0:this.scrollParent.scrollLeft())}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0,right:parseInt(this.element.css("marginRight"),10)||0,bottom:parseInt(this.element.css("marginBottom"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var t,e,i,s=this.options,n=this.document[0];this.relativeContainer=null,s.containment?"window"!==s.containment?"document"!==s.containment?s.containment.constructor!==Array?("parent"===s.containment&&(s.containment=this.helper[0].parentNode),(i=(e=V(s.containment))[0])&&(t=/(scroll|auto)/.test(e.css("overflow")),this.containment=[(parseInt(e.css("borderLeftWidth"),10)||0)+(parseInt(e.css("paddingLeft"),10)||0),(parseInt(e.css("borderTopWidth"),10)||0)+(parseInt(e.css("paddingTop"),10)||0),(t?Math.max(i.scrollWidth,i.offsetWidth):i.offsetWidth)-(parseInt(e.css("borderRightWidth"),10)||0)-(parseInt(e.css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left-this.margins.right,(t?Math.max(i.scrollHeight,i.offsetHeight):i.offsetHeight)-(parseInt(e.css("borderBottomWidth"),10)||0)-(parseInt(e.css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top-this.margins.bottom],this.relativeContainer=e)):this.containment=s.containment:this.containment=[0,0,V(n).width()-this.helperProportions.width-this.margins.left,(V(n).height()||n.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top]:this.containment=[V(window).scrollLeft()-this.offset.relative.left-this.offset.parent.left,V(window).scrollTop()-this.offset.relative.top-this.offset.parent.top,V(window).scrollLeft()+V(window).width()-this.helperProportions.width-this.margins.left,V(window).scrollTop()+(V(window).height()||n.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top]:this.containment=null},_convertPositionTo:function(t,e){e=e||this.position;var i="absolute"===t?1:-1,t=this._isRootNode(this.scrollParent[0]);return{top:e.top+this.offset.relative.top*i+this.offset.parent.top*i-("fixed"===this.cssPosition?-this.offset.scroll.top:t?0:this.offset.scroll.top)*i,left:e.left+this.offset.relative.left*i+this.offset.parent.left*i-("fixed"===this.cssPosition?-this.offset.scroll.left:t?0:this.offset.scroll.left)*i}},_generatePosition:function(t,e){var i,s=this.options,n=this._isRootNode(this.scrollParent[0]),o=t.pageX,a=t.pageY;return n&&this.offset.scroll||(this.offset.scroll={top:this.scrollParent.scrollTop(),left:this.scrollParent.scrollLeft()}),e&&(this.containment&&(i=this.relativeContainer?(i=this.relativeContainer.offset(),[this.containment[0]+i.left,this.containment[1]+i.top,this.containment[2]+i.left,this.containment[3]+i.top]):this.containment,t.pageX-this.offset.click.left<i[0]&&(o=i[0]+this.offset.click.left),t.pageY-this.offset.click.top<i[1]&&(a=i[1]+this.offset.click.top),t.pageX-this.offset.click.left>i[2]&&(o=i[2]+this.offset.click.left),t.pageY-this.offset.click.top>i[3]&&(a=i[3]+this.offset.click.top)),s.grid&&(t=s.grid[1]?this.originalPageY+Math.round((a-this.originalPageY)/s.grid[1])*s.grid[1]:this.originalPageY,a=!i||t-this.offset.click.top>=i[1]||t-this.offset.click.top>i[3]?t:t-this.offset.click.top>=i[1]?t-s.grid[1]:t+s.grid[1],t=s.grid[0]?this.originalPageX+Math.round((o-this.originalPageX)/s.grid[0])*s.grid[0]:this.originalPageX,o=!i||t-this.offset.click.left>=i[0]||t-this.offset.click.left>i[2]?t:t-this.offset.click.left>=i[0]?t-s.grid[0]:t+s.grid[0]),"y"===s.axis&&(o=this.originalPageX),"x"===s.axis&&(a=this.originalPageY)),{top:a-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+("fixed"===this.cssPosition?-this.offset.scroll.top:n?0:this.offset.scroll.top),left:o-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+("fixed"===this.cssPosition?-this.offset.scroll.left:n?0:this.offset.scroll.left)}},_clear:function(){this._removeClass(this.helper,"ui-draggable-dragging"),this.helper[0]===this.element[0]||this.cancelHelperRemoval||this.helper.remove(),this.helper=null,this.cancelHelperRemoval=!1,this.destroyOnClear&&this.destroy()},_trigger:function(t,e,i){return i=i||this._uiHash(),V.ui.plugin.call(this,t,[e,i,this],!0),/^(drag|start|stop)/.test(t)&&(this.positionAbs=this._convertPositionTo("absolute"),i.offset=this.positionAbs),V.Widget.prototype._trigger.call(this,t,e,i)},plugins:{},_uiHash:function(){return{helper:this.helper,position:this.position,originalPosition:this.originalPosition,offset:this.positionAbs}}}),V.ui.plugin.add("draggable","connectToSortable",{start:function(e,t,i){var s=V.extend({},t,{item:i.element});i.sortables=[],V(i.options.connectToSortable).each(function(){var t=V(this).sortable("instance");t&&!t.options.disabled&&(i.sortables.push(t),t.refreshPositions(),t._trigger("activate",e,s))})},stop:function(e,t,i){var s=V.extend({},t,{item:i.element});i.cancelHelperRemoval=!1,V.each(i.sortables,function(){var t=this;t.isOver?(t.isOver=0,i.cancelHelperRemoval=!0,t.cancelHelperRemoval=!1,t._storedCSS={position:t.placeholder.css("position"),top:t.placeholder.css("top"),left:t.placeholder.css("left")},t._mouseStop(e),t.options.helper=t.options._helper):(t.cancelHelperRemoval=!0,t._trigger("deactivate",e,s))})},drag:function(i,s,n){V.each(n.sortables,function(){var t=!1,e=this;e.positionAbs=n.positionAbs,e.helperProportions=n.helperProportions,e.offset.click=n.offset.click,e._intersectsWith(e.containerCache)&&(t=!0,V.each(n.sortables,function(){return this.positionAbs=n.positionAbs,this.helperProportions=n.helperProportions,this.offset.click=n.offset.click,t=this!==e&&this._intersectsWith(this.containerCache)&&V.contains(e.element[0],this.element[0])?!1:t})),t?(e.isOver||(e.isOver=1,n._parent=s.helper.parent(),e.currentItem=s.helper.appendTo(e.element).data("ui-sortable-item",!0),e.options._helper=e.options.helper,e.options.helper=function(){return s.helper[0]},i.target=e.currentItem[0],e._mouseCapture(i,!0),e._mouseStart(i,!0,!0),e.offset.click.top=n.offset.click.top,e.offset.click.left=n.offset.click.left,e.offset.parent.left-=n.offset.parent.left-e.offset.parent.left,e.offset.parent.top-=n.offset.parent.top-e.offset.parent.top,n._trigger("toSortable",i),n.dropped=e.element,V.each(n.sortables,function(){this.refreshPositions()}),n.currentItem=n.element,e.fromOutside=n),e.currentItem&&(e._mouseDrag(i),s.position=e.position)):e.isOver&&(e.isOver=0,e.cancelHelperRemoval=!0,e.options._revert=e.options.revert,e.options.revert=!1,e._trigger("out",i,e._uiHash(e)),e._mouseStop(i,!0),e.options.revert=e.options._revert,e.options.helper=e.options._helper,e.placeholder&&e.placeholder.remove(),s.helper.appendTo(n._parent),n._refreshOffsets(i),s.position=n._generatePosition(i,!0),n._trigger("fromSortable",i),n.dropped=!1,V.each(n.sortables,function(){this.refreshPositions()}))})}}),V.ui.plugin.add("draggable","cursor",{start:function(t,e,i){var s=V("body"),i=i.options;s.css("cursor")&&(i._cursor=s.css("cursor")),s.css("cursor",i.cursor)},stop:function(t,e,i){i=i.options;i._cursor&&V("body").css("cursor",i._cursor)}}),V.ui.plugin.add("draggable","opacity",{start:function(t,e,i){e=V(e.helper),i=i.options;e.css("opacity")&&(i._opacity=e.css("opacity")),e.css("opacity",i.opacity)},stop:function(t,e,i){i=i.options;i._opacity&&V(e.helper).css("opacity",i._opacity)}}),V.ui.plugin.add("draggable","scroll",{start:function(t,e,i){i.scrollParentNotHidden||(i.scrollParentNotHidden=i.helper.scrollParent(!1)),i.scrollParentNotHidden[0]!==i.document[0]&&"HTML"!==i.scrollParentNotHidden[0].tagName&&(i.overflowOffset=i.scrollParentNotHidden.offset())},drag:function(t,e,i){var s=i.options,n=!1,o=i.scrollParentNotHidden[0],a=i.document[0];o!==a&&"HTML"!==o.tagName?(s.axis&&"x"===s.axis||(i.overflowOffset.top+o.offsetHeight-t.pageY<s.scrollSensitivity?o.scrollTop=n=o.scrollTop+s.scrollSpeed:t.pageY-i.overflowOffset.top<s.scrollSensitivity&&(o.scrollTop=n=o.scrollTop-s.scrollSpeed)),s.axis&&"y"===s.axis||(i.overflowOffset.left+o.offsetWidth-t.pageX<s.scrollSensitivity?o.scrollLeft=n=o.scrollLeft+s.scrollSpeed:t.pageX-i.overflowOffset.left<s.scrollSensitivity&&(o.scrollLeft=n=o.scrollLeft-s.scrollSpeed))):(s.axis&&"x"===s.axis||(t.pageY-V(a).scrollTop()<s.scrollSensitivity?n=V(a).scrollTop(V(a).scrollTop()-s.scrollSpeed):V(window).height()-(t.pageY-V(a).scrollTop())<s.scrollSensitivity&&(n=V(a).scrollTop(V(a).scrollTop()+s.scrollSpeed))),s.axis&&"y"===s.axis||(t.pageX-V(a).scrollLeft()<s.scrollSensitivity?n=V(a).scrollLeft(V(a).scrollLeft()-s.scrollSpeed):V(window).width()-(t.pageX-V(a).scrollLeft())<s.scrollSensitivity&&(n=V(a).scrollLeft(V(a).scrollLeft()+s.scrollSpeed)))),!1!==n&&V.ui.ddmanager&&!s.dropBehaviour&&V.ui.ddmanager.prepareOffsets(i,t)}}),V.ui.plugin.add("draggable","snap",{start:function(t,e,i){var s=i.options;i.snapElements=[],V(s.snap.constructor!==String?s.snap.items||":data(ui-draggable)":s.snap).each(function(){var t=V(this),e=t.offset();this!==i.element[0]&&i.snapElements.push({item:this,width:t.outerWidth(),height:t.outerHeight(),top:e.top,left:e.left})})},drag:function(t,e,i){for(var s,n,o,a,r,l,h,c,u,d=i.options,p=d.snapTolerance,f=e.offset.left,g=f+i.helperProportions.width,m=e.offset.top,_=m+i.helperProportions.height,v=i.snapElements.length-1;0<=v;v--)l=(r=i.snapElements[v].left-i.margins.left)+i.snapElements[v].width,c=(h=i.snapElements[v].top-i.margins.top)+i.snapElements[v].height,g<r-p||l+p<f||_<h-p||c+p<m||!V.contains(i.snapElements[v].item.ownerDocument,i.snapElements[v].item)?(i.snapElements[v].snapping&&i.options.snap.release&&i.options.snap.release.call(i.element,t,V.extend(i._uiHash(),{snapItem:i.snapElements[v].item})),i.snapElements[v].snapping=!1):("inner"!==d.snapMode&&(s=Math.abs(h-_)<=p,n=Math.abs(c-m)<=p,o=Math.abs(r-g)<=p,a=Math.abs(l-f)<=p,s&&(e.position.top=i._convertPositionTo("relative",{top:h-i.helperProportions.height,left:0}).top),n&&(e.position.top=i._convertPositionTo("relative",{top:c,left:0}).top),o&&(e.position.left=i._convertPositionTo("relative",{top:0,left:r-i.helperProportions.width}).left),a&&(e.position.left=i._convertPositionTo("relative",{top:0,left:l}).left)),u=s||n||o||a,"outer"!==d.snapMode&&(s=Math.abs(h-m)<=p,n=Math.abs(c-_)<=p,o=Math.abs(r-f)<=p,a=Math.abs(l-g)<=p,s&&(e.position.top=i._convertPositionTo("relative",{top:h,left:0}).top),n&&(e.position.top=i._convertPositionTo("relative",{top:c-i.helperProportions.height,left:0}).top),o&&(e.position.left=i._convertPositionTo("relative",{top:0,left:r}).left),a&&(e.position.left=i._convertPositionTo("relative",{top:0,left:l-i.helperProportions.width}).left)),!i.snapElements[v].snapping&&(s||n||o||a||u)&&i.options.snap.snap&&i.options.snap.snap.call(i.element,t,V.extend(i._uiHash(),{snapItem:i.snapElements[v].item})),i.snapElements[v].snapping=s||n||o||a||u)}}),V.ui.plugin.add("draggable","stack",{start:function(t,e,i){var s,i=i.options,i=V.makeArray(V(i.stack)).sort(function(t,e){return(parseInt(V(t).css("zIndex"),10)||0)-(parseInt(V(e).css("zIndex"),10)||0)});i.length&&(s=parseInt(V(i[0]).css("zIndex"),10)||0,V(i).each(function(t){V(this).css("zIndex",s+t)}),this.css("zIndex",s+i.length))}}),V.ui.plugin.add("draggable","zIndex",{start:function(t,e,i){e=V(e.helper),i=i.options;e.css("zIndex")&&(i._zIndex=e.css("zIndex")),e.css("zIndex",i.zIndex)},stop:function(t,e,i){i=i.options;i._zIndex&&V(e.helper).css("zIndex",i._zIndex)}});V.ui.draggable;V.widget("ui.resizable",V.ui.mouse,{version:"1.13.2",widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,classes:{"ui-resizable-se":"ui-icon ui-icon-gripsmall-diagonal-se"},containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:90,resize:null,start:null,stop:null},_num:function(t){return parseFloat(t)||0},_isNumber:function(t){return!isNaN(parseFloat(t))},_hasScroll:function(t,e){if("hidden"===V(t).css("overflow"))return!1;var i=e&&"left"===e?"scrollLeft":"scrollTop",e=!1;if(0<t[i])return!0;try{t[i]=1,e=0<t[i],t[i]=0}catch(t){}return e},_create:function(){var t,e=this.options,i=this;this._addClass("ui-resizable"),V.extend(this,{_aspectRatio:!!e.aspectRatio,aspectRatio:e.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:e.helper||e.ghost||e.animate?e.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/^(canvas|textarea|input|select|button|img)$/i)&&(this.element.wrap(V("<div class='ui-wrapper'></div>").css({overflow:"hidden",position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("ui-resizable",this.element.resizable("instance")),this.elementIsWrapper=!0,t={marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom"),marginLeft:this.originalElement.css("marginLeft")},this.element.css(t),this.originalElement.css("margin",0),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css(t),this._proportionallyResize()),this._setupHandles(),e.autoHide&&V(this.element).on("mouseenter",function(){e.disabled||(i._removeClass("ui-resizable-autohide"),i._handles.show())}).on("mouseleave",function(){e.disabled||i.resizing||(i._addClass("ui-resizable-autohide"),i._handles.hide())}),this._mouseInit()},_destroy:function(){this._mouseDestroy(),this._addedHandles.remove();function t(t){V(t).removeData("resizable").removeData("ui-resizable").off(".resizable")}var e;return this.elementIsWrapper&&(t(this.element),e=this.element,this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")}).insertAfter(e),e.remove()),this.originalElement.css("resize",this.originalResizeStyle),t(this.originalElement),this},_setOption:function(t,e){switch(this._super(t,e),t){case"handles":this._removeHandles(),this._setupHandles();break;case"aspectRatio":this._aspectRatio=!!e}},_setupHandles:function(){var t,e,i,s,n,o=this.options,a=this;if(this.handles=o.handles||(V(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se"),this._handles=V(),this._addedHandles=V(),this.handles.constructor===String)for("all"===this.handles&&(this.handles="n,e,s,w,se,sw,ne,nw"),i=this.handles.split(","),this.handles={},e=0;e<i.length;e++)s="ui-resizable-"+(t=String.prototype.trim.call(i[e])),n=V("<div>"),this._addClass(n,"ui-resizable-handle "+s),n.css({zIndex:o.zIndex}),this.handles[t]=".ui-resizable-"+t,this.element.children(this.handles[t]).length||(this.element.append(n),this._addedHandles=this._addedHandles.add(n));this._renderAxis=function(t){var e,i,s;for(e in t=t||this.element,this.handles)this.handles[e].constructor===String?this.handles[e]=this.element.children(this.handles[e]).first().show():(this.handles[e].jquery||this.handles[e].nodeType)&&(this.handles[e]=V(this.handles[e]),this._on(this.handles[e],{mousedown:a._mouseDown})),this.elementIsWrapper&&this.originalElement[0].nodeName.match(/^(textarea|input|select|button)$/i)&&(i=V(this.handles[e],this.element),s=/sw|ne|nw|se|n|s/.test(e)?i.outerHeight():i.outerWidth(),i=["padding",/ne|nw|n/.test(e)?"Top":/se|sw|s/.test(e)?"Bottom":/^e$/.test(e)?"Right":"Left"].join(""),t.css(i,s),this._proportionallyResize()),this._handles=this._handles.add(this.handles[e])},this._renderAxis(this.element),this._handles=this._handles.add(this.element.find(".ui-resizable-handle")),this._handles.disableSelection(),this._handles.on("mouseover",function(){a.resizing||(this.className&&(n=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)),a.axis=n&&n[1]?n[1]:"se")}),o.autoHide&&(this._handles.hide(),this._addClass("ui-resizable-autohide"))},_removeHandles:function(){this._addedHandles.remove()},_mouseCapture:function(t){var e,i,s=!1;for(e in this.handles)(i=V(this.handles[e])[0])!==t.target&&!V.contains(i,t.target)||(s=!0);return!this.options.disabled&&s},_mouseStart:function(t){var e,i,s=this.options,n=this.element;return this.resizing=!0,this._renderProxy(),e=this._num(this.helper.css("left")),i=this._num(this.helper.css("top")),s.containment&&(e+=V(s.containment).scrollLeft()||0,i+=V(s.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:e,top:i},this.size=this._helper?{width:this.helper.width(),height:this.helper.height()}:{width:n.width(),height:n.height()},this.originalSize=this._helper?{width:n.outerWidth(),height:n.outerHeight()}:{width:n.width(),height:n.height()},this.sizeDiff={width:n.outerWidth()-n.width(),height:n.outerHeight()-n.height()},this.originalPosition={left:e,top:i},this.originalMousePosition={left:t.pageX,top:t.pageY},this.aspectRatio="number"==typeof s.aspectRatio?s.aspectRatio:this.originalSize.width/this.originalSize.height||1,s=V(".ui-resizable-"+this.axis).css("cursor"),V("body").css("cursor","auto"===s?this.axis+"-resize":s),this._addClass("ui-resizable-resizing"),this._propagate("start",t),!0},_mouseDrag:function(t){var e=this.originalMousePosition,i=this.axis,s=t.pageX-e.left||0,e=t.pageY-e.top||0,i=this._change[i];return this._updatePrevProperties(),i&&(e=i.apply(this,[t,s,e]),this._updateVirtualBoundaries(t.shiftKey),(this._aspectRatio||t.shiftKey)&&(e=this._updateRatio(e,t)),e=this._respectSize(e,t),this._updateCache(e),this._propagate("resize",t),e=this._applyChanges(),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),V.isEmptyObject(e)||(this._updatePrevProperties(),this._trigger("resize",t,this.ui()),this._applyChanges())),!1},_mouseStop:function(t){this.resizing=!1;var e,i,s,n=this.options,o=this;return this._helper&&(s=(e=(i=this._proportionallyResizeElements).length&&/textarea/i.test(i[0].nodeName))&&this._hasScroll(i[0],"left")?0:o.sizeDiff.height,i=e?0:o.sizeDiff.width,e={width:o.helper.width()-i,height:o.helper.height()-s},i=parseFloat(o.element.css("left"))+(o.position.left-o.originalPosition.left)||null,s=parseFloat(o.element.css("top"))+(o.position.top-o.originalPosition.top)||null,n.animate||this.element.css(V.extend(e,{top:s,left:i})),o.helper.height(o.size.height),o.helper.width(o.size.width),this._helper&&!n.animate&&this._proportionallyResize()),V("body").css("cursor","auto"),this._removeClass("ui-resizable-resizing"),this._propagate("stop",t),this._helper&&this.helper.remove(),!1},_updatePrevProperties:function(){this.prevPosition={top:this.position.top,left:this.position.left},this.prevSize={width:this.size.width,height:this.size.height}},_applyChanges:function(){var t={};return this.position.top!==this.prevPosition.top&&(t.top=this.position.top+"px"),this.position.left!==this.prevPosition.left&&(t.left=this.position.left+"px"),this.size.width!==this.prevSize.width&&(t.width=this.size.width+"px"),this.size.height!==this.prevSize.height&&(t.height=this.size.height+"px"),this.helper.css(t),t},_updateVirtualBoundaries:function(t){var e,i,s=this.options,n={minWidth:this._isNumber(s.minWidth)?s.minWidth:0,maxWidth:this._isNumber(s.maxWidth)?s.maxWidth:1/0,minHeight:this._isNumber(s.minHeight)?s.minHeight:0,maxHeight:this._isNumber(s.maxHeight)?s.maxHeight:1/0};(this._aspectRatio||t)&&(e=n.minHeight*this.aspectRatio,i=n.minWidth/this.aspectRatio,s=n.maxHeight*this.aspectRatio,t=n.maxWidth/this.aspectRatio,e>n.minWidth&&(n.minWidth=e),i>n.minHeight&&(n.minHeight=i),s<n.maxWidth&&(n.maxWidth=s),t<n.maxHeight&&(n.maxHeight=t)),this._vBoundaries=n},_updateCache:function(t){this.offset=this.helper.offset(),this._isNumber(t.left)&&(this.position.left=t.left),this._isNumber(t.top)&&(this.position.top=t.top),this._isNumber(t.height)&&(this.size.height=t.height),this._isNumber(t.width)&&(this.size.width=t.width)},_updateRatio:function(t){var e=this.position,i=this.size,s=this.axis;return this._isNumber(t.height)?t.width=t.height*this.aspectRatio:this._isNumber(t.width)&&(t.height=t.width/this.aspectRatio),"sw"===s&&(t.left=e.left+(i.width-t.width),t.top=null),"nw"===s&&(t.top=e.top+(i.height-t.height),t.left=e.left+(i.width-t.width)),t},_respectSize:function(t){var e=this._vBoundaries,i=this.axis,s=this._isNumber(t.width)&&e.maxWidth&&e.maxWidth<t.width,n=this._isNumber(t.height)&&e.maxHeight&&e.maxHeight<t.height,o=this._isNumber(t.width)&&e.minWidth&&e.minWidth>t.width,a=this._isNumber(t.height)&&e.minHeight&&e.minHeight>t.height,r=this.originalPosition.left+this.originalSize.width,l=this.originalPosition.top+this.originalSize.height,h=/sw|nw|w/.test(i),i=/nw|ne|n/.test(i);return o&&(t.width=e.minWidth),a&&(t.height=e.minHeight),s&&(t.width=e.maxWidth),n&&(t.height=e.maxHeight),o&&h&&(t.left=r-e.minWidth),s&&h&&(t.left=r-e.maxWidth),a&&i&&(t.top=l-e.minHeight),n&&i&&(t.top=l-e.maxHeight),t.width||t.height||t.left||!t.top?t.width||t.height||t.top||!t.left||(t.left=null):t.top=null,t},_getPaddingPlusBorderDimensions:function(t){for(var e=0,i=[],s=[t.css("borderTopWidth"),t.css("borderRightWidth"),t.css("borderBottomWidth"),t.css("borderLeftWidth")],n=[t.css("paddingTop"),t.css("paddingRight"),t.css("paddingBottom"),t.css("paddingLeft")];e<4;e++)i[e]=parseFloat(s[e])||0,i[e]+=parseFloat(n[e])||0;return{height:i[0]+i[2],width:i[1]+i[3]}},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var t,e=0,i=this.helper||this.element;e<this._proportionallyResizeElements.length;e++)t=this._proportionallyResizeElements[e],this.outerDimensions||(this.outerDimensions=this._getPaddingPlusBorderDimensions(t)),t.css({height:i.height()-this.outerDimensions.height||0,width:i.width()-this.outerDimensions.width||0})},_renderProxy:function(){var t=this.element,e=this.options;this.elementOffset=t.offset(),this._helper?(this.helper=this.helper||V("<div></div>").css({overflow:"hidden"}),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++e.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize;return{left:this.originalPosition.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize;return{top:this.originalPosition.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(t,e,i){return V.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},sw:function(t,e,i){return V.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[t,e,i]))},ne:function(t,e,i){return V.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[t,e,i]))},nw:function(t,e,i){return V.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[t,e,i]))}},_propagate:function(t,e){V.ui.plugin.call(this,t,[e,this.ui()]),"resize"!==t&&this._trigger(t,e,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),V.ui.plugin.add("resizable","animate",{stop:function(e){var i=V(this).resizable("instance"),t=i.options,s=i._proportionallyResizeElements,n=s.length&&/textarea/i.test(s[0].nodeName),o=n&&i._hasScroll(s[0],"left")?0:i.sizeDiff.height,a=n?0:i.sizeDiff.width,n={width:i.size.width-a,height:i.size.height-o},a=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,o=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(V.extend(n,o&&a?{top:o,left:a}:{}),{duration:t.animateDuration,easing:t.animateEasing,step:function(){var t={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};s&&s.length&&V(s[0]).css({width:t.width,height:t.height}),i._updateCache(t),i._propagate("resize",e)}})}}),V.ui.plugin.add("resizable","containment",{start:function(){var i,s,n=V(this).resizable("instance"),t=n.options,e=n.element,o=t.containment,a=o instanceof V?o.get(0):/parent/.test(o)?e.parent().get(0):o;a&&(n.containerElement=V(a),/document/.test(o)||o===document?(n.containerOffset={left:0,top:0},n.containerPosition={left:0,top:0},n.parentData={element:V(document),left:0,top:0,width:V(document).width(),height:V(document).height()||document.body.parentNode.scrollHeight}):(i=V(a),s=[],V(["Top","Right","Left","Bottom"]).each(function(t,e){s[t]=n._num(i.css("padding"+e))}),n.containerOffset=i.offset(),n.containerPosition=i.position(),n.containerSize={height:i.innerHeight()-s[3],width:i.innerWidth()-s[1]},t=n.containerOffset,e=n.containerSize.height,o=n.containerSize.width,o=n._hasScroll(a,"left")?a.scrollWidth:o,e=n._hasScroll(a)?a.scrollHeight:e,n.parentData={element:a,left:t.left,top:t.top,width:o,height:e}))},resize:function(t){var e=V(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.position,o=e._aspectRatio||t.shiftKey,a={top:0,left:0},r=e.containerElement,t=!0;r[0]!==document&&/static/.test(r.css("position"))&&(a=s),n.left<(e._helper?s.left:0)&&(e.size.width=e.size.width+(e._helper?e.position.left-s.left:e.position.left-a.left),o&&(e.size.height=e.size.width/e.aspectRatio,t=!1),e.position.left=i.helper?s.left:0),n.top<(e._helper?s.top:0)&&(e.size.height=e.size.height+(e._helper?e.position.top-s.top:e.position.top),o&&(e.size.width=e.size.height*e.aspectRatio,t=!1),e.position.top=e._helper?s.top:0),i=e.containerElement.get(0)===e.element.parent().get(0),n=/relative|absolute/.test(e.containerElement.css("position")),i&&n?(e.offset.left=e.parentData.left+e.position.left,e.offset.top=e.parentData.top+e.position.top):(e.offset.left=e.element.offset().left,e.offset.top=e.element.offset().top),n=Math.abs(e.sizeDiff.width+(e._helper?e.offset.left-a.left:e.offset.left-s.left)),s=Math.abs(e.sizeDiff.height+(e._helper?e.offset.top-a.top:e.offset.top-s.top)),n+e.size.width>=e.parentData.width&&(e.size.width=e.parentData.width-n,o&&(e.size.height=e.size.width/e.aspectRatio,t=!1)),s+e.size.height>=e.parentData.height&&(e.size.height=e.parentData.height-s,o&&(e.size.width=e.size.height*e.aspectRatio,t=!1)),t||(e.position.left=e.prevPosition.left,e.position.top=e.prevPosition.top,e.size.width=e.prevSize.width,e.size.height=e.prevSize.height)},stop:function(){var t=V(this).resizable("instance"),e=t.options,i=t.containerOffset,s=t.containerPosition,n=t.containerElement,o=V(t.helper),a=o.offset(),r=o.outerWidth()-t.sizeDiff.width,o=o.outerHeight()-t.sizeDiff.height;t._helper&&!e.animate&&/relative/.test(n.css("position"))&&V(this).css({left:a.left-s.left-i.left,width:r,height:o}),t._helper&&!e.animate&&/static/.test(n.css("position"))&&V(this).css({left:a.left-s.left-i.left,width:r,height:o})}}),V.ui.plugin.add("resizable","alsoResize",{start:function(){var t=V(this).resizable("instance").options;V(t.alsoResize).each(function(){var t=V(this);t.data("ui-resizable-alsoresize",{width:parseFloat(t.width()),height:parseFloat(t.height()),left:parseFloat(t.css("left")),top:parseFloat(t.css("top"))})})},resize:function(t,i){var e=V(this).resizable("instance"),s=e.options,n=e.originalSize,o=e.originalPosition,a={height:e.size.height-n.height||0,width:e.size.width-n.width||0,top:e.position.top-o.top||0,left:e.position.left-o.left||0};V(s.alsoResize).each(function(){var t=V(this),s=V(this).data("ui-resizable-alsoresize"),n={},e=t.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];V.each(e,function(t,e){var i=(s[e]||0)+(a[e]||0);i&&0<=i&&(n[e]=i||null)}),t.css(n)})},stop:function(){V(this).removeData("ui-resizable-alsoresize")}}),V.ui.plugin.add("resizable","ghost",{start:function(){var t=V(this).resizable("instance"),e=t.size;t.ghost=t.originalElement.clone(),t.ghost.css({opacity:.25,display:"block",position:"relative",height:e.height,width:e.width,margin:0,left:0,top:0}),t._addClass(t.ghost,"ui-resizable-ghost"),!1!==V.uiBackCompat&&"string"==typeof t.options.ghost&&t.ghost.addClass(this.options.ghost),t.ghost.appendTo(t.helper)},resize:function(){var t=V(this).resizable("instance");t.ghost&&t.ghost.css({position:"relative",height:t.size.height,width:t.size.width})},stop:function(){var t=V(this).resizable("instance");t.ghost&&t.helper&&t.helper.get(0).removeChild(t.ghost.get(0))}}),V.ui.plugin.add("resizable","grid",{resize:function(){var t,e=V(this).resizable("instance"),i=e.options,s=e.size,n=e.originalSize,o=e.originalPosition,a=e.axis,r="number"==typeof i.grid?[i.grid,i.grid]:i.grid,l=r[0]||1,h=r[1]||1,c=Math.round((s.width-n.width)/l)*l,u=Math.round((s.height-n.height)/h)*h,d=n.width+c,p=n.height+u,f=i.maxWidth&&i.maxWidth<d,g=i.maxHeight&&i.maxHeight<p,m=i.minWidth&&i.minWidth>d,s=i.minHeight&&i.minHeight>p;i.grid=r,m&&(d+=l),s&&(p+=h),f&&(d-=l),g&&(p-=h),/^(se|s|e)$/.test(a)?(e.size.width=d,e.size.height=p):/^(ne)$/.test(a)?(e.size.width=d,e.size.height=p,e.position.top=o.top-u):/^(sw)$/.test(a)?(e.size.width=d,e.size.height=p,e.position.left=o.left-c):((p-h<=0||d-l<=0)&&(t=e._getPaddingPlusBorderDimensions(this)),0<p-h?(e.size.height=p,e.position.top=o.top-u):(p=h-t.height,e.size.height=p,e.position.top=o.top+n.height-p),0<d-l?(e.size.width=d,e.position.left=o.left-c):(d=l-t.width,e.size.width=d,e.position.left=o.left+n.width-d))}});V.ui.resizable;V.widget("ui.dialog",{version:"1.13.2",options:{appendTo:"body",autoOpen:!0,buttons:[],classes:{"ui-dialog":"ui-corner-all","ui-dialog-titlebar":"ui-corner-all"},closeOnEscape:!0,closeText:"Close",draggable:!0,hide:null,height:"auto",maxHeight:null,maxWidth:null,minHeight:150,minWidth:150,modal:!1,position:{my:"center",at:"center",of:window,collision:"fit",using:function(t){var e=V(this).css(t).offset().top;e<0&&V(this).css("top",t.top-e)}},resizable:!0,show:null,title:null,width:300,beforeClose:null,close:null,drag:null,dragStart:null,dragStop:null,focus:null,open:null,resize:null,resizeStart:null,resizeStop:null},sizeRelatedOptions:{buttons:!0,height:!0,maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0,width:!0},resizableRelatedOptions:{maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0},_create:function(){this.originalCss={display:this.element[0].style.display,width:this.element[0].style.width,minHeight:this.element[0].style.minHeight,maxHeight:this.element[0].style.maxHeight,height:this.element[0].style.height},this.originalPosition={parent:this.element.parent(),index:this.element.parent().children().index(this.element)},this.originalTitle=this.element.attr("title"),null==this.options.title&&null!=this.originalTitle&&(this.options.title=this.originalTitle),this.options.disabled&&(this.options.disabled=!1),this._createWrapper(),this.element.show().removeAttr("title").appendTo(this.uiDialog),this._addClass("ui-dialog-content","ui-widget-content"),this._createTitlebar(),this._createButtonPane(),this.options.draggable&&V.fn.draggable&&this._makeDraggable(),this.options.resizable&&V.fn.resizable&&this._makeResizable(),this._isOpen=!1,this._trackFocus()},_init:function(){this.options.autoOpen&&this.open()},_appendTo:function(){var t=this.options.appendTo;return t&&(t.jquery||t.nodeType)?V(t):this.document.find(t||"body").eq(0)},_destroy:function(){var t,e=this.originalPosition;this._untrackInstance(),this._destroyOverlay(),this.element.removeUniqueId().css(this.originalCss).detach(),this.uiDialog.remove(),this.originalTitle&&this.element.attr("title",this.originalTitle),(t=e.parent.children().eq(e.index)).length&&t[0]!==this.element[0]?t.before(this.element):e.parent.append(this.element)},widget:function(){return this.uiDialog},disable:V.noop,enable:V.noop,close:function(t){var e=this;this._isOpen&&!1!==this._trigger("beforeClose",t)&&(this._isOpen=!1,this._focusedElement=null,this._destroyOverlay(),this._untrackInstance(),this.opener.filter(":focusable").trigger("focus").length||V.ui.safeBlur(V.ui.safeActiveElement(this.document[0])),this._hide(this.uiDialog,this.options.hide,function(){e._trigger("close",t)}))},isOpen:function(){return this._isOpen},moveToTop:function(){this._moveToTop()},_moveToTop:function(t,e){var i=!1,s=this.uiDialog.siblings(".ui-front:visible").map(function(){return+V(this).css("z-index")}).get(),s=Math.max.apply(null,s);return s>=+this.uiDialog.css("z-index")&&(this.uiDialog.css("z-index",s+1),i=!0),i&&!e&&this._trigger("focus",t),i},open:function(){var t=this;this._isOpen?this._moveToTop()&&this._focusTabbable():(this._isOpen=!0,this.opener=V(V.ui.safeActiveElement(this.document[0])),this._size(),this._position(),this._createOverlay(),this._moveToTop(null,!0),this.overlay&&this.overlay.css("z-index",this.uiDialog.css("z-index")-1),this._show(this.uiDialog,this.options.show,function(){t._focusTabbable(),t._trigger("focus")}),this._makeFocusTarget(),this._trigger("open"))},_focusTabbable:function(){var t=this._focusedElement;(t=!(t=!(t=!(t=!(t=t||this.element.find("[autofocus]")).length?this.element.find(":tabbable"):t).length?this.uiDialogButtonPane.find(":tabbable"):t).length?this.uiDialogTitlebarClose.filter(":tabbable"):t).length?this.uiDialog:t).eq(0).trigger("focus")},_restoreTabbableFocus:function(){var t=V.ui.safeActiveElement(this.document[0]);this.uiDialog[0]===t||V.contains(this.uiDialog[0],t)||this._focusTabbable()},_keepFocus:function(t){t.preventDefault(),this._restoreTabbableFocus(),this._delay(this._restoreTabbableFocus)},_createWrapper:function(){this.uiDialog=V("<div>").hide().attr({tabIndex:-1,role:"dialog"}).appendTo(this._appendTo()),this._addClass(this.uiDialog,"ui-dialog","ui-widget ui-widget-content ui-front"),this._on(this.uiDialog,{keydown:function(t){if(this.options.closeOnEscape&&!t.isDefaultPrevented()&&t.keyCode&&t.keyCode===V.ui.keyCode.ESCAPE)return t.preventDefault(),void this.close(t);var e,i,s;t.keyCode!==V.ui.keyCode.TAB||t.isDefaultPrevented()||(e=this.uiDialog.find(":tabbable"),i=e.first(),s=e.last(),t.target!==s[0]&&t.target!==this.uiDialog[0]||t.shiftKey?t.target!==i[0]&&t.target!==this.uiDialog[0]||!t.shiftKey||(this._delay(function(){s.trigger("focus")}),t.preventDefault()):(this._delay(function(){i.trigger("focus")}),t.preventDefault()))},mousedown:function(t){this._moveToTop(t)&&this._focusTabbable()}}),this.element.find("[aria-describedby]").length||this.uiDialog.attr({"aria-describedby":this.element.uniqueId().attr("id")})},_createTitlebar:function(){var t;this.uiDialogTitlebar=V("<div>"),this._addClass(this.uiDialogTitlebar,"ui-dialog-titlebar","ui-widget-header ui-helper-clearfix"),this._on(this.uiDialogTitlebar,{mousedown:function(t){V(t.target).closest(".ui-dialog-titlebar-close")||this.uiDialog.trigger("focus")}}),this.uiDialogTitlebarClose=V("<button type='button'></button>").button({label:V("<a>").text(this.options.closeText).html(),icon:"ui-icon-closethick",showLabel:!1}).appendTo(this.uiDialogTitlebar),this._addClass(this.uiDialogTitlebarClose,"ui-dialog-titlebar-close"),this._on(this.uiDialogTitlebarClose,{click:function(t){t.preventDefault(),this.close(t)}}),t=V("<span>").uniqueId().prependTo(this.uiDialogTitlebar),this._addClass(t,"ui-dialog-title"),this._title(t),this.uiDialogTitlebar.prependTo(this.uiDialog),this.uiDialog.attr({"aria-labelledby":t.attr("id")})},_title:function(t){this.options.title?t.text(this.options.title):t.html("&#160;")},_createButtonPane:function(){this.uiDialogButtonPane=V("<div>"),this._addClass(this.uiDialogButtonPane,"ui-dialog-buttonpane","ui-widget-content ui-helper-clearfix"),this.uiButtonSet=V("<div>").appendTo(this.uiDialogButtonPane),this._addClass(this.uiButtonSet,"ui-dialog-buttonset"),this._createButtons()},_createButtons:function(){var s=this,t=this.options.buttons;this.uiDialogButtonPane.remove(),this.uiButtonSet.empty(),V.isEmptyObject(t)||Array.isArray(t)&&!t.length?this._removeClass(this.uiDialog,"ui-dialog-buttons"):(V.each(t,function(t,e){var i;e=V.extend({type:"button"},e="function"==typeof e?{click:e,text:t}:e),i=e.click,t={icon:e.icon,iconPosition:e.iconPosition,showLabel:e.showLabel,icons:e.icons,text:e.text},delete e.click,delete e.icon,delete e.iconPosition,delete e.showLabel,delete e.icons,"boolean"==typeof e.text&&delete e.text,V("<button></button>",e).button(t).appendTo(s.uiButtonSet).on("click",function(){i.apply(s.element[0],arguments)})}),this._addClass(this.uiDialog,"ui-dialog-buttons"),this.uiDialogButtonPane.appendTo(this.uiDialog))},_makeDraggable:function(){var n=this,o=this.options;function a(t){return{position:t.position,offset:t.offset}}this.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(t,e){n._addClass(V(this),"ui-dialog-dragging"),n._blockFrames(),n._trigger("dragStart",t,a(e))},drag:function(t,e){n._trigger("drag",t,a(e))},stop:function(t,e){var i=e.offset.left-n.document.scrollLeft(),s=e.offset.top-n.document.scrollTop();o.position={my:"left top",at:"left"+(0<=i?"+":"")+i+" top"+(0<=s?"+":"")+s,of:n.window},n._removeClass(V(this),"ui-dialog-dragging"),n._unblockFrames(),n._trigger("dragStop",t,a(e))}})},_makeResizable:function(){var n=this,o=this.options,t=o.resizable,e=this.uiDialog.css("position"),t="string"==typeof t?t:"n,e,s,w,se,sw,ne,nw";function a(t){return{originalPosition:t.originalPosition,originalSize:t.originalSize,position:t.position,size:t.size}}this.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:this.element,maxWidth:o.maxWidth,maxHeight:o.maxHeight,minWidth:o.minWidth,minHeight:this._minHeight(),handles:t,start:function(t,e){n._addClass(V(this),"ui-dialog-resizing"),n._blockFrames(),n._trigger("resizeStart",t,a(e))},resize:function(t,e){n._trigger("resize",t,a(e))},stop:function(t,e){var i=n.uiDialog.offset(),s=i.left-n.document.scrollLeft(),i=i.top-n.document.scrollTop();o.height=n.uiDialog.height(),o.width=n.uiDialog.width(),o.position={my:"left top",at:"left"+(0<=s?"+":"")+s+" top"+(0<=i?"+":"")+i,of:n.window},n._removeClass(V(this),"ui-dialog-resizing"),n._unblockFrames(),n._trigger("resizeStop",t,a(e))}}).css("position",e)},_trackFocus:function(){this._on(this.widget(),{focusin:function(t){this._makeFocusTarget(),this._focusedElement=V(t.target)}})},_makeFocusTarget:function(){this._untrackInstance(),this._trackingInstances().unshift(this)},_untrackInstance:function(){var t=this._trackingInstances(),e=V.inArray(this,t);-1!==e&&t.splice(e,1)},_trackingInstances:function(){var t=this.document.data("ui-dialog-instances");return t||this.document.data("ui-dialog-instances",t=[]),t},_minHeight:function(){var t=this.options;return"auto"===t.height?t.minHeight:Math.min(t.minHeight,t.height)},_position:function(){var t=this.uiDialog.is(":visible");t||this.uiDialog.show(),this.uiDialog.position(this.options.position),t||this.uiDialog.hide()},_setOptions:function(t){var i=this,s=!1,n={};V.each(t,function(t,e){i._setOption(t,e),t in i.sizeRelatedOptions&&(s=!0),t in i.resizableRelatedOptions&&(n[t]=e)}),s&&(this._size(),this._position()),this.uiDialog.is(":data(ui-resizable)")&&this.uiDialog.resizable("option",n)},_setOption:function(t,e){var i,s=this.uiDialog;"disabled"!==t&&(this._super(t,e),"appendTo"===t&&this.uiDialog.appendTo(this._appendTo()),"buttons"===t&&this._createButtons(),"closeText"===t&&this.uiDialogTitlebarClose.button({label:V("<a>").text(""+this.options.closeText).html()}),"draggable"===t&&((i=s.is(":data(ui-draggable)"))&&!e&&s.draggable("destroy"),!i&&e&&this._makeDraggable()),"position"===t&&this._position(),"resizable"===t&&((i=s.is(":data(ui-resizable)"))&&!e&&s.resizable("destroy"),i&&"string"==typeof e&&s.resizable("option","handles",e),i||!1===e||this._makeResizable()),"title"===t&&this._title(this.uiDialogTitlebar.find(".ui-dialog-title")))},_size:function(){var t,e,i,s=this.options;this.element.show().css({width:"auto",minHeight:0,maxHeight:"none",height:0}),s.minWidth>s.width&&(s.width=s.minWidth),t=this.uiDialog.css({height:"auto",width:s.width}).outerHeight(),e=Math.max(0,s.minHeight-t),i="number"==typeof s.maxHeight?Math.max(0,s.maxHeight-t):"none","auto"===s.height?this.element.css({minHeight:e,maxHeight:i,height:"auto"}):this.element.height(Math.max(0,s.height-t)),this.uiDialog.is(":data(ui-resizable)")&&this.uiDialog.resizable("option","minHeight",this._minHeight())},_blockFrames:function(){this.iframeBlocks=this.document.find("iframe").map(function(){var t=V(this);return V("<div>").css({position:"absolute",width:t.outerWidth(),height:t.outerHeight()}).appendTo(t.parent()).offset(t.offset())[0]})},_unblockFrames:function(){this.iframeBlocks&&(this.iframeBlocks.remove(),delete this.iframeBlocks)},_allowInteraction:function(t){return!!V(t.target).closest(".ui-dialog").length||!!V(t.target).closest(".ui-datepicker").length},_createOverlay:function(){var i,s;this.options.modal&&(i=V.fn.jquery.substring(0,4),s=!0,this._delay(function(){s=!1}),this.document.data("ui-dialog-overlays")||this.document.on("focusin.ui-dialog",function(t){var e;s||((e=this._trackingInstances()[0])._allowInteraction(t)||(t.preventDefault(),e._focusTabbable(),"3.4."!==i&&"3.5."!==i||e._delay(e._restoreTabbableFocus)))}.bind(this)),this.overlay=V("<div>").appendTo(this._appendTo()),this._addClass(this.overlay,null,"ui-widget-overlay ui-front"),this._on(this.overlay,{mousedown:"_keepFocus"}),this.document.data("ui-dialog-overlays",(this.document.data("ui-dialog-overlays")||0)+1))},_destroyOverlay:function(){var t;this.options.modal&&this.overlay&&((t=this.document.data("ui-dialog-overlays")-1)?this.document.data("ui-dialog-overlays",t):(this.document.off("focusin.ui-dialog"),this.document.removeData("ui-dialog-overlays")),this.overlay.remove(),this.overlay=null)}}),!1!==V.uiBackCompat&&V.widget("ui.dialog",V.ui.dialog,{options:{dialogClass:""},_createWrapper:function(){this._super(),this.uiDialog.addClass(this.options.dialogClass)},_setOption:function(t,e){"dialogClass"===t&&this.uiDialog.removeClass(this.options.dialogClass).addClass(e),this._superApply(arguments)}});V.ui.dialog;function lt(t,e,i){return e<=t&&t<e+i}V.widget("ui.droppable",{version:"1.13.2",widgetEventPrefix:"drop",options:{accept:"*",addClasses:!0,greedy:!1,scope:"default",tolerance:"intersect",activate:null,deactivate:null,drop:null,out:null,over:null},_create:function(){var t,e=this.options,i=e.accept;this.isover=!1,this.isout=!0,this.accept="function"==typeof i?i:function(t){return t.is(i)},this.proportions=function(){if(!arguments.length)return t=t||{width:this.element[0].offsetWidth,height:this.element[0].offsetHeight};t=arguments[0]},this._addToManager(e.scope),e.addClasses&&this._addClass("ui-droppable")},_addToManager:function(t){V.ui.ddmanager.droppables[t]=V.ui.ddmanager.droppables[t]||[],V.ui.ddmanager.droppables[t].push(this)},_splice:function(t){for(var e=0;e<t.length;e++)t[e]===this&&t.splice(e,1)},_destroy:function(){var t=V.ui.ddmanager.droppables[this.options.scope];this._splice(t)},_setOption:function(t,e){var i;"accept"===t?this.accept="function"==typeof e?e:function(t){return t.is(e)}:"scope"===t&&(i=V.ui.ddmanager.droppables[this.options.scope],this._splice(i),this._addToManager(e)),this._super(t,e)},_activate:function(t){var e=V.ui.ddmanager.current;this._addActiveClass(),e&&this._trigger("activate",t,this.ui(e))},_deactivate:function(t){var e=V.ui.ddmanager.current;this._removeActiveClass(),e&&this._trigger("deactivate",t,this.ui(e))},_over:function(t){var e=V.ui.ddmanager.current;e&&(e.currentItem||e.element)[0]!==this.element[0]&&this.accept.call(this.element[0],e.currentItem||e.element)&&(this._addHoverClass(),this._trigger("over",t,this.ui(e)))},_out:function(t){var e=V.ui.ddmanager.current;e&&(e.currentItem||e.element)[0]!==this.element[0]&&this.accept.call(this.element[0],e.currentItem||e.element)&&(this._removeHoverClass(),this._trigger("out",t,this.ui(e)))},_drop:function(e,t){var i=t||V.ui.ddmanager.current,s=!1;return!(!i||(i.currentItem||i.element)[0]===this.element[0])&&(this.element.find(":data(ui-droppable)").not(".ui-draggable-dragging").each(function(){var t=V(this).droppable("instance");if(t.options.greedy&&!t.options.disabled&&t.options.scope===i.options.scope&&t.accept.call(t.element[0],i.currentItem||i.element)&&V.ui.intersect(i,V.extend(t,{offset:t.element.offset()}),t.options.tolerance,e))return!(s=!0)}),!s&&(!!this.accept.call(this.element[0],i.currentItem||i.element)&&(this._removeActiveClass(),this._removeHoverClass(),this._trigger("drop",e,this.ui(i)),this.element)))},ui:function(t){return{draggable:t.currentItem||t.element,helper:t.helper,position:t.position,offset:t.positionAbs}},_addHoverClass:function(){this._addClass("ui-droppable-hover")},_removeHoverClass:function(){this._removeClass("ui-droppable-hover")},_addActiveClass:function(){this._addClass("ui-droppable-active")},_removeActiveClass:function(){this._removeClass("ui-droppable-active")}}),V.ui.intersect=function(t,e,i,s){if(!e.offset)return!1;var n=(t.positionAbs||t.position.absolute).left+t.margins.left,o=(t.positionAbs||t.position.absolute).top+t.margins.top,a=n+t.helperProportions.width,r=o+t.helperProportions.height,l=e.offset.left,h=e.offset.top,c=l+e.proportions().width,u=h+e.proportions().height;switch(i){case"fit":return l<=n&&a<=c&&h<=o&&r<=u;case"intersect":return l<n+t.helperProportions.width/2&&a-t.helperProportions.width/2<c&&h<o+t.helperProportions.height/2&&r-t.helperProportions.height/2<u;case"pointer":return lt(s.pageY,h,e.proportions().height)&&lt(s.pageX,l,e.proportions().width);case"touch":return(h<=o&&o<=u||h<=r&&r<=u||o<h&&u<r)&&(l<=n&&n<=c||l<=a&&a<=c||n<l&&c<a);default:return!1}},!(V.ui.ddmanager={current:null,droppables:{default:[]},prepareOffsets:function(t,e){var i,s,n=V.ui.ddmanager.droppables[t.options.scope]||[],o=e?e.type:null,a=(t.currentItem||t.element).find(":data(ui-droppable)").addBack();t:for(i=0;i<n.length;i++)if(!(n[i].options.disabled||t&&!n[i].accept.call(n[i].element[0],t.currentItem||t.element))){for(s=0;s<a.length;s++)if(a[s]===n[i].element[0]){n[i].proportions().height=0;continue t}n[i].visible="none"!==n[i].element.css("display"),n[i].visible&&("mousedown"===o&&n[i]._activate.call(n[i],e),n[i].offset=n[i].element.offset(),n[i].proportions({width:n[i].element[0].offsetWidth,height:n[i].element[0].offsetHeight}))}},drop:function(t,e){var i=!1;return V.each((V.ui.ddmanager.droppables[t.options.scope]||[]).slice(),function(){this.options&&(!this.options.disabled&&this.visible&&V.ui.intersect(t,this,this.options.tolerance,e)&&(i=this._drop.call(this,e)||i),!this.options.disabled&&this.visible&&this.accept.call(this.element[0],t.currentItem||t.element)&&(this.isout=!0,this.isover=!1,this._deactivate.call(this,e)))}),i},dragStart:function(t,e){t.element.parentsUntil("body").on("scroll.droppable",function(){t.options.refreshPositions||V.ui.ddmanager.prepareOffsets(t,e)})},drag:function(n,o){n.options.refreshPositions&&V.ui.ddmanager.prepareOffsets(n,o),V.each(V.ui.ddmanager.droppables[n.options.scope]||[],function(){var t,e,i,s;this.options.disabled||this.greedyChild||!this.visible||(s=!(i=V.ui.intersect(n,this,this.options.tolerance,o))&&this.isover?"isout":i&&!this.isover?"isover":null)&&(this.options.greedy&&(e=this.options.scope,(i=this.element.parents(":data(ui-droppable)").filter(function(){return V(this).droppable("instance").options.scope===e})).length&&((t=V(i[0]).droppable("instance")).greedyChild="isover"===s)),t&&"isover"===s&&(t.isover=!1,t.isout=!0,t._out.call(t,o)),this[s]=!0,this["isout"===s?"isover":"isout"]=!1,this["isover"===s?"_over":"_out"].call(this,o),t&&"isout"===s&&(t.isout=!1,t.isover=!0,t._over.call(t,o)))})},dragStop:function(t,e){t.element.parentsUntil("body").off("scroll.droppable"),t.options.refreshPositions||V.ui.ddmanager.prepareOffsets(t,e)}})!==V.uiBackCompat&&V.widget("ui.droppable",V.ui.droppable,{options:{hoverClass:!1,activeClass:!1},_addActiveClass:function(){this._super(),this.options.activeClass&&this.element.addClass(this.options.activeClass)},_removeActiveClass:function(){this._super(),this.options.activeClass&&this.element.removeClass(this.options.activeClass)},_addHoverClass:function(){this._super(),this.options.hoverClass&&this.element.addClass(this.options.hoverClass)},_removeHoverClass:function(){this._super(),this.options.hoverClass&&this.element.removeClass(this.options.hoverClass)}});V.ui.droppable,V.widget("ui.progressbar",{version:"1.13.2",options:{classes:{"ui-progressbar":"ui-corner-all","ui-progressbar-value":"ui-corner-left","ui-progressbar-complete":"ui-corner-right"},max:100,value:0,change:null,complete:null},min:0,_create:function(){this.oldValue=this.options.value=this._constrainedValue(),this.element.attr({role:"progressbar","aria-valuemin":this.min}),this._addClass("ui-progressbar","ui-widget ui-widget-content"),this.valueDiv=V("<div>").appendTo(this.element),this._addClass(this.valueDiv,"ui-progressbar-value","ui-widget-header"),this._refreshValue()},_destroy:function(){this.element.removeAttr("role aria-valuemin aria-valuemax aria-valuenow"),this.valueDiv.remove()},value:function(t){if(void 0===t)return this.options.value;this.options.value=this._constrainedValue(t),this._refreshValue()},_constrainedValue:function(t){return void 0===t&&(t=this.options.value),this.indeterminate=!1===t,"number"!=typeof t&&(t=0),!this.indeterminate&&Math.min(this.options.max,Math.max(this.min,t))},_setOptions:function(t){var e=t.value;delete t.value,this._super(t),this.options.value=this._constrainedValue(e),this._refreshValue()},_setOption:function(t,e){"max"===t&&(e=Math.max(this.min,e)),this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t),this._toggleClass(null,"ui-state-disabled",!!t)},_percentage:function(){return this.indeterminate?100:100*(this.options.value-this.min)/(this.options.max-this.min)},_refreshValue:function(){var t=this.options.value,e=this._percentage();this.valueDiv.toggle(this.indeterminate||t>this.min).width(e.toFixed(0)+"%"),this._toggleClass(this.valueDiv,"ui-progressbar-complete",null,t===this.options.max)._toggleClass("ui-progressbar-indeterminate",null,this.indeterminate),this.indeterminate?(this.element.removeAttr("aria-valuenow"),this.overlayDiv||(this.overlayDiv=V("<div>").appendTo(this.valueDiv),this._addClass(this.overlayDiv,"ui-progressbar-overlay"))):(this.element.attr({"aria-valuemax":this.options.max,"aria-valuenow":t}),this.overlayDiv&&(this.overlayDiv.remove(),this.overlayDiv=null)),this.oldValue!==t&&(this.oldValue=t,this._trigger("change")),t===this.options.max&&this._trigger("complete")}}),V.widget("ui.selectable",V.ui.mouse,{version:"1.13.2",options:{appendTo:"body",autoRefresh:!0,distance:0,filter:"*",tolerance:"touch",selected:null,selecting:null,start:null,stop:null,unselected:null,unselecting:null},_create:function(){var i=this;this._addClass("ui-selectable"),this.dragged=!1,this.refresh=function(){i.elementPos=V(i.element[0]).offset(),i.selectees=V(i.options.filter,i.element[0]),i._addClass(i.selectees,"ui-selectee"),i.selectees.each(function(){var t=V(this),e=t.offset(),e={left:e.left-i.elementPos.left,top:e.top-i.elementPos.top};V.data(this,"selectable-item",{element:this,$element:t,left:e.left,top:e.top,right:e.left+t.outerWidth(),bottom:e.top+t.outerHeight(),startselected:!1,selected:t.hasClass("ui-selected"),selecting:t.hasClass("ui-selecting"),unselecting:t.hasClass("ui-unselecting")})})},this.refresh(),this._mouseInit(),this.helper=V("<div>"),this._addClass(this.helper,"ui-selectable-helper")},_destroy:function(){this.selectees.removeData("selectable-item"),this._mouseDestroy()},_mouseStart:function(i){var s=this,t=this.options;this.opos=[i.pageX,i.pageY],this.elementPos=V(this.element[0]).offset(),this.options.disabled||(this.selectees=V(t.filter,this.element[0]),this._trigger("start",i),V(t.appendTo).append(this.helper),this.helper.css({left:i.pageX,top:i.pageY,width:0,height:0}),t.autoRefresh&&this.refresh(),this.selectees.filter(".ui-selected").each(function(){var t=V.data(this,"selectable-item");t.startselected=!0,i.metaKey||i.ctrlKey||(s._removeClass(t.$element,"ui-selected"),t.selected=!1,s._addClass(t.$element,"ui-unselecting"),t.unselecting=!0,s._trigger("unselecting",i,{unselecting:t.element}))}),V(i.target).parents().addBack().each(function(){var t,e=V.data(this,"selectable-item");if(e)return t=!i.metaKey&&!i.ctrlKey||!e.$element.hasClass("ui-selected"),s._removeClass(e.$element,t?"ui-unselecting":"ui-selected")._addClass(e.$element,t?"ui-selecting":"ui-unselecting"),e.unselecting=!t,e.selecting=t,(e.selected=t)?s._trigger("selecting",i,{selecting:e.element}):s._trigger("unselecting",i,{unselecting:e.element}),!1}))},_mouseDrag:function(s){if(this.dragged=!0,!this.options.disabled){var t,n=this,o=this.options,a=this.opos[0],r=this.opos[1],l=s.pageX,h=s.pageY;return l<a&&(t=l,l=a,a=t),h<r&&(t=h,h=r,r=t),this.helper.css({left:a,top:r,width:l-a,height:h-r}),this.selectees.each(function(){var t=V.data(this,"selectable-item"),e=!1,i={};t&&t.element!==n.element[0]&&(i.left=t.left+n.elementPos.left,i.right=t.right+n.elementPos.left,i.top=t.top+n.elementPos.top,i.bottom=t.bottom+n.elementPos.top,"touch"===o.tolerance?e=!(i.left>l||i.right<a||i.top>h||i.bottom<r):"fit"===o.tolerance&&(e=i.left>a&&i.right<l&&i.top>r&&i.bottom<h),e?(t.selected&&(n._removeClass(t.$element,"ui-selected"),t.selected=!1),t.unselecting&&(n._removeClass(t.$element,"ui-unselecting"),t.unselecting=!1),t.selecting||(n._addClass(t.$element,"ui-selecting"),t.selecting=!0,n._trigger("selecting",s,{selecting:t.element}))):(t.selecting&&((s.metaKey||s.ctrlKey)&&t.startselected?(n._removeClass(t.$element,"ui-selecting"),t.selecting=!1,n._addClass(t.$element,"ui-selected"),t.selected=!0):(n._removeClass(t.$element,"ui-selecting"),t.selecting=!1,t.startselected&&(n._addClass(t.$element,"ui-unselecting"),t.unselecting=!0),n._trigger("unselecting",s,{unselecting:t.element}))),t.selected&&(s.metaKey||s.ctrlKey||t.startselected||(n._removeClass(t.$element,"ui-selected"),t.selected=!1,n._addClass(t.$element,"ui-unselecting"),t.unselecting=!0,n._trigger("unselecting",s,{unselecting:t.element})))))}),!1}},_mouseStop:function(e){var i=this;return this.dragged=!1,V(".ui-unselecting",this.element[0]).each(function(){var t=V.data(this,"selectable-item");i._removeClass(t.$element,"ui-unselecting"),t.unselecting=!1,t.startselected=!1,i._trigger("unselected",e,{unselected:t.element})}),V(".ui-selecting",this.element[0]).each(function(){var t=V.data(this,"selectable-item");i._removeClass(t.$element,"ui-selecting")._addClass(t.$element,"ui-selected"),t.selecting=!1,t.selected=!0,t.startselected=!0,i._trigger("selected",e,{selected:t.element})}),this._trigger("stop",e),this.helper.remove(),!1}}),V.widget("ui.selectmenu",[V.ui.formResetMixin,{version:"1.13.2",defaultElement:"<select>",options:{appendTo:null,classes:{"ui-selectmenu-button-open":"ui-corner-top","ui-selectmenu-button-closed":"ui-corner-all"},disabled:null,icons:{button:"ui-icon-triangle-1-s"},position:{my:"left top",at:"left bottom",collision:"none"},width:!1,change:null,close:null,focus:null,open:null,select:null},_create:function(){var t=this.element.uniqueId().attr("id");this.ids={element:t,button:t+"-button",menu:t+"-menu"},this._drawButton(),this._drawMenu(),this._bindFormResetHandler(),this._rendered=!1,this.menuItems=V()},_drawButton:function(){var t,e=this,i=this._parseOption(this.element.find("option:selected"),this.element[0].selectedIndex);this.labels=this.element.labels().attr("for",this.ids.button),this._on(this.labels,{click:function(t){this.button.trigger("focus"),t.preventDefault()}}),this.element.hide(),this.button=V("<span>",{tabindex:this.options.disabled?-1:0,id:this.ids.button,role:"combobox","aria-expanded":"false","aria-autocomplete":"list","aria-owns":this.ids.menu,"aria-haspopup":"true",title:this.element.attr("title")}).insertAfter(this.element),this._addClass(this.button,"ui-selectmenu-button ui-selectmenu-button-closed","ui-button ui-widget"),t=V("<span>").appendTo(this.button),this._addClass(t,"ui-selectmenu-icon","ui-icon "+this.options.icons.button),this.buttonItem=this._renderButtonItem(i).appendTo(this.button),!1!==this.options.width&&this._resizeButton(),this._on(this.button,this._buttonEvents),this.button.one("focusin",function(){e._rendered||e._refreshMenu()})},_drawMenu:function(){var i=this;this.menu=V("<ul>",{"aria-hidden":"true","aria-labelledby":this.ids.button,id:this.ids.menu}),this.menuWrap=V("<div>").append(this.menu),this._addClass(this.menuWrap,"ui-selectmenu-menu","ui-front"),this.menuWrap.appendTo(this._appendTo()),this.menuInstance=this.menu.menu({classes:{"ui-menu":"ui-corner-bottom"},role:"listbox",select:function(t,e){t.preventDefault(),i._setSelection(),i._select(e.item.data("ui-selectmenu-item"),t)},focus:function(t,e){e=e.item.data("ui-selectmenu-item");null!=i.focusIndex&&e.index!==i.focusIndex&&(i._trigger("focus",t,{item:e}),i.isOpen||i._select(e,t)),i.focusIndex=e.index,i.button.attr("aria-activedescendant",i.menuItems.eq(e.index).attr("id"))}}).menu("instance"),this.menuInstance._off(this.menu,"mouseleave"),this.menuInstance._closeOnDocumentClick=function(){return!1},this.menuInstance._isDivider=function(){return!1}},refresh:function(){this._refreshMenu(),this.buttonItem.replaceWith(this.buttonItem=this._renderButtonItem(this._getSelectedItem().data("ui-selectmenu-item")||{})),null===this.options.width&&this._resizeButton()},_refreshMenu:function(){var t=this.element.find("option");this.menu.empty(),this._parseOptions(t),this._renderMenu(this.menu,this.items),this.menuInstance.refresh(),this.menuItems=this.menu.find("li").not(".ui-selectmenu-optgroup").find(".ui-menu-item-wrapper"),this._rendered=!0,t.length&&(t=this._getSelectedItem(),this.menuInstance.focus(null,t),this._setAria(t.data("ui-selectmenu-item")),this._setOption("disabled",this.element.prop("disabled")))},open:function(t){this.options.disabled||(this._rendered?(this._removeClass(this.menu.find(".ui-state-active"),null,"ui-state-active"),this.menuInstance.focus(null,this._getSelectedItem())):this._refreshMenu(),this.menuItems.length&&(this.isOpen=!0,this._toggleAttr(),this._resizeMenu(),this._position(),this._on(this.document,this._documentClick),this._trigger("open",t)))},_position:function(){this.menuWrap.position(V.extend({of:this.button},this.options.position))},close:function(t){this.isOpen&&(this.isOpen=!1,this._toggleAttr(),this.range=null,this._off(this.document),this._trigger("close",t))},widget:function(){return this.button},menuWidget:function(){return this.menu},_renderButtonItem:function(t){var e=V("<span>");return this._setText(e,t.label),this._addClass(e,"ui-selectmenu-text"),e},_renderMenu:function(s,t){var n=this,o="";V.each(t,function(t,e){var i;e.optgroup!==o&&(i=V("<li>",{text:e.optgroup}),n._addClass(i,"ui-selectmenu-optgroup","ui-menu-divider"+(e.element.parent("optgroup").prop("disabled")?" ui-state-disabled":"")),i.appendTo(s),o=e.optgroup),n._renderItemData(s,e)})},_renderItemData:function(t,e){return this._renderItem(t,e).data("ui-selectmenu-item",e)},_renderItem:function(t,e){var i=V("<li>"),s=V("<div>",{title:e.element.attr("title")});return e.disabled&&this._addClass(i,null,"ui-state-disabled"),this._setText(s,e.label),i.append(s).appendTo(t)},_setText:function(t,e){e?t.text(e):t.html("&#160;")},_move:function(t,e){var i,s=".ui-menu-item";this.isOpen?i=this.menuItems.eq(this.focusIndex).parent("li"):(i=this.menuItems.eq(this.element[0].selectedIndex).parent("li"),s+=":not(.ui-state-disabled)"),(s="first"===t||"last"===t?i["first"===t?"prevAll":"nextAll"](s).eq(-1):i[t+"All"](s).eq(0)).length&&this.menuInstance.focus(e,s)},_getSelectedItem:function(){return this.menuItems.eq(this.element[0].selectedIndex).parent("li")},_toggle:function(t){this[this.isOpen?"close":"open"](t)},_setSelection:function(){var t;this.range&&(window.getSelection?((t=window.getSelection()).removeAllRanges(),t.addRange(this.range)):this.range.select(),this.button.trigger("focus"))},_documentClick:{mousedown:function(t){this.isOpen&&(V(t.target).closest(".ui-selectmenu-menu, #"+V.escapeSelector(this.ids.button)).length||this.close(t))}},_buttonEvents:{mousedown:function(){var t;window.getSelection?(t=window.getSelection()).rangeCount&&(this.range=t.getRangeAt(0)):this.range=document.selection.createRange()},click:function(t){this._setSelection(),this._toggle(t)},keydown:function(t){var e=!0;switch(t.keyCode){case V.ui.keyCode.TAB:case V.ui.keyCode.ESCAPE:this.close(t),e=!1;break;case V.ui.keyCode.ENTER:this.isOpen&&this._selectFocusedItem(t);break;case V.ui.keyCode.UP:t.altKey?this._toggle(t):this._move("prev",t);break;case V.ui.keyCode.DOWN:t.altKey?this._toggle(t):this._move("next",t);break;case V.ui.keyCode.SPACE:this.isOpen?this._selectFocusedItem(t):this._toggle(t);break;case V.ui.keyCode.LEFT:this._move("prev",t);break;case V.ui.keyCode.RIGHT:this._move("next",t);break;case V.ui.keyCode.HOME:case V.ui.keyCode.PAGE_UP:this._move("first",t);break;case V.ui.keyCode.END:case V.ui.keyCode.PAGE_DOWN:this._move("last",t);break;default:this.menu.trigger(t),e=!1}e&&t.preventDefault()}},_selectFocusedItem:function(t){var e=this.menuItems.eq(this.focusIndex).parent("li");e.hasClass("ui-state-disabled")||this._select(e.data("ui-selectmenu-item"),t)},_select:function(t,e){var i=this.element[0].selectedIndex;this.element[0].selectedIndex=t.index,this.buttonItem.replaceWith(this.buttonItem=this._renderButtonItem(t)),this._setAria(t),this._trigger("select",e,{item:t}),t.index!==i&&this._trigger("change",e,{item:t}),this.close(e)},_setAria:function(t){t=this.menuItems.eq(t.index).attr("id");this.button.attr({"aria-labelledby":t,"aria-activedescendant":t}),this.menu.attr("aria-activedescendant",t)},_setOption:function(t,e){var i;"icons"===t&&(i=this.button.find("span.ui-icon"),this._removeClass(i,null,this.options.icons.button)._addClass(i,null,e.button)),this._super(t,e),"appendTo"===t&&this.menuWrap.appendTo(this._appendTo()),"width"===t&&this._resizeButton()},_setOptionDisabled:function(t){this._super(t),this.menuInstance.option("disabled",t),this.button.attr("aria-disabled",t),this._toggleClass(this.button,null,"ui-state-disabled",t),this.element.prop("disabled",t),t?(this.button.attr("tabindex",-1),this.close()):this.button.attr("tabindex",0)},_appendTo:function(){var t=this.options.appendTo;return t=!(t=!(t=t&&(t.jquery||t.nodeType?V(t):this.document.find(t).eq(0)))||!t[0]?this.element.closest(".ui-front, dialog"):t).length?this.document[0].body:t},_toggleAttr:function(){this.button.attr("aria-expanded",this.isOpen),this._removeClass(this.button,"ui-selectmenu-button-"+(this.isOpen?"closed":"open"))._addClass(this.button,"ui-selectmenu-button-"+(this.isOpen?"open":"closed"))._toggleClass(this.menuWrap,"ui-selectmenu-open",null,this.isOpen),this.menu.attr("aria-hidden",!this.isOpen)},_resizeButton:function(){var t=this.options.width;!1!==t?(null===t&&(t=this.element.show().outerWidth(),this.element.hide()),this.button.outerWidth(t)):this.button.css("width","")},_resizeMenu:function(){this.menu.outerWidth(Math.max(this.button.outerWidth(),this.menu.width("").outerWidth()+1))},_getCreateOptions:function(){var t=this._super();return t.disabled=this.element.prop("disabled"),t},_parseOptions:function(t){var i=this,s=[];t.each(function(t,e){e.hidden||s.push(i._parseOption(V(e),t))}),this.items=s},_parseOption:function(t,e){var i=t.parent("optgroup");return{element:t,index:e,value:t.val(),label:t.text(),optgroup:i.attr("label")||"",disabled:i.prop("disabled")||t.prop("disabled")}},_destroy:function(){this._unbindFormResetHandler(),this.menuWrap.remove(),this.button.remove(),this.element.show(),this.element.removeUniqueId(),this.labels.attr("for",this.ids.element)}}]),V.widget("ui.slider",V.ui.mouse,{version:"1.13.2",widgetEventPrefix:"slide",options:{animate:!1,classes:{"ui-slider":"ui-corner-all","ui-slider-handle":"ui-corner-all","ui-slider-range":"ui-corner-all ui-widget-header"},distance:0,max:100,min:0,orientation:"horizontal",range:!1,step:1,value:0,values:null,change:null,slide:null,start:null,stop:null},numPages:5,_create:function(){this._keySliding=!1,this._mouseSliding=!1,this._animateOff=!0,this._handleIndex=null,this._detectOrientation(),this._mouseInit(),this._calculateNewMax(),this._addClass("ui-slider ui-slider-"+this.orientation,"ui-widget ui-widget-content"),this._refresh(),this._animateOff=!1},_refresh:function(){this._createRange(),this._createHandles(),this._setupEvents(),this._refreshValue()},_createHandles:function(){var t,e=this.options,i=this.element.find(".ui-slider-handle"),s=[],n=e.values&&e.values.length||1;for(i.length>n&&(i.slice(n).remove(),i=i.slice(0,n)),t=i.length;t<n;t++)s.push("<span tabindex='0'></span>");this.handles=i.add(V(s.join("")).appendTo(this.element)),this._addClass(this.handles,"ui-slider-handle","ui-state-default"),this.handle=this.handles.eq(0),this.handles.each(function(t){V(this).data("ui-slider-handle-index",t).attr("tabIndex",0)})},_createRange:function(){var t=this.options;t.range?(!0===t.range&&(t.values?t.values.length&&2!==t.values.length?t.values=[t.values[0],t.values[0]]:Array.isArray(t.values)&&(t.values=t.values.slice(0)):t.values=[this._valueMin(),this._valueMin()]),this.range&&this.range.length?(this._removeClass(this.range,"ui-slider-range-min ui-slider-range-max"),this.range.css({left:"",bottom:""})):(this.range=V("<div>").appendTo(this.element),this._addClass(this.range,"ui-slider-range")),"min"!==t.range&&"max"!==t.range||this._addClass(this.range,"ui-slider-range-"+t.range)):(this.range&&this.range.remove(),this.range=null)},_setupEvents:function(){this._off(this.handles),this._on(this.handles,this._handleEvents),this._hoverable(this.handles),this._focusable(this.handles)},_destroy:function(){this.handles.remove(),this.range&&this.range.remove(),this._mouseDestroy()},_mouseCapture:function(t){var i,s,n,o,e,a,r=this,l=this.options;return!l.disabled&&(this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()},this.elementOffset=this.element.offset(),a={x:t.pageX,y:t.pageY},i=this._normValueFromMouse(a),s=this._valueMax()-this._valueMin()+1,this.handles.each(function(t){var e=Math.abs(i-r.values(t));(e<s||s===e&&(t===r._lastChangedValue||r.values(t)===l.min))&&(s=e,n=V(this),o=t)}),!1!==this._start(t,o)&&(this._mouseSliding=!0,this._handleIndex=o,this._addClass(n,null,"ui-state-active"),n.trigger("focus"),e=n.offset(),a=!V(t.target).parents().addBack().is(".ui-slider-handle"),this._clickOffset=a?{left:0,top:0}:{left:t.pageX-e.left-n.width()/2,top:t.pageY-e.top-n.height()/2-(parseInt(n.css("borderTopWidth"),10)||0)-(parseInt(n.css("borderBottomWidth"),10)||0)+(parseInt(n.css("marginTop"),10)||0)},this.handles.hasClass("ui-state-hover")||this._slide(t,o,i),this._animateOff=!0))},_mouseStart:function(){return!0},_mouseDrag:function(t){var e={x:t.pageX,y:t.pageY},e=this._normValueFromMouse(e);return this._slide(t,this._handleIndex,e),!1},_mouseStop:function(t){return this._removeClass(this.handles,null,"ui-state-active"),this._mouseSliding=!1,this._stop(t,this._handleIndex),this._change(t,this._handleIndex),this._handleIndex=null,this._clickOffset=null,this._animateOff=!1},_detectOrientation:function(){this.orientation="vertical"===this.options.orientation?"vertical":"horizontal"},_normValueFromMouse:function(t){var e,t="horizontal"===this.orientation?(e=this.elementSize.width,t.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)):(e=this.elementSize.height,t.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)),t=t/e;return(t=1<t?1:t)<0&&(t=0),"vertical"===this.orientation&&(t=1-t),e=this._valueMax()-this._valueMin(),e=this._valueMin()+t*e,this._trimAlignValue(e)},_uiHash:function(t,e,i){var s={handle:this.handles[t],handleIndex:t,value:void 0!==e?e:this.value()};return this._hasMultipleValues()&&(s.value=void 0!==e?e:this.values(t),s.values=i||this.values()),s},_hasMultipleValues:function(){return this.options.values&&this.options.values.length},_start:function(t,e){return this._trigger("start",t,this._uiHash(e))},_slide:function(t,e,i){var s,n=this.value(),o=this.values();this._hasMultipleValues()&&(s=this.values(e?0:1),n=this.values(e),2===this.options.values.length&&!0===this.options.range&&(i=0===e?Math.min(s,i):Math.max(s,i)),o[e]=i),i!==n&&!1!==this._trigger("slide",t,this._uiHash(e,i,o))&&(this._hasMultipleValues()?this.values(e,i):this.value(i))},_stop:function(t,e){this._trigger("stop",t,this._uiHash(e))},_change:function(t,e){this._keySliding||this._mouseSliding||(this._lastChangedValue=e,this._trigger("change",t,this._uiHash(e)))},value:function(t){return arguments.length?(this.options.value=this._trimAlignValue(t),this._refreshValue(),void this._change(null,0)):this._value()},values:function(t,e){var i,s,n;if(1<arguments.length)return this.options.values[t]=this._trimAlignValue(e),this._refreshValue(),void this._change(null,t);if(!arguments.length)return this._values();if(!Array.isArray(t))return this._hasMultipleValues()?this._values(t):this.value();for(i=this.options.values,s=t,n=0;n<i.length;n+=1)i[n]=this._trimAlignValue(s[n]),this._change(null,n);this._refreshValue()},_setOption:function(t,e){var i,s=0;switch("range"===t&&!0===this.options.range&&("min"===e?(this.options.value=this._values(0),this.options.values=null):"max"===e&&(this.options.value=this._values(this.options.values.length-1),this.options.values=null)),Array.isArray(this.options.values)&&(s=this.options.values.length),this._super(t,e),t){case"orientation":this._detectOrientation(),this._removeClass("ui-slider-horizontal ui-slider-vertical")._addClass("ui-slider-"+this.orientation),this._refreshValue(),this.options.range&&this._refreshRange(e),this.handles.css("horizontal"===e?"bottom":"left","");break;case"value":this._animateOff=!0,this._refreshValue(),this._change(null,0),this._animateOff=!1;break;case"values":for(this._animateOff=!0,this._refreshValue(),i=s-1;0<=i;i--)this._change(null,i);this._animateOff=!1;break;case"step":case"min":case"max":this._animateOff=!0,this._calculateNewMax(),this._refreshValue(),this._animateOff=!1;break;case"range":this._animateOff=!0,this._refresh(),this._animateOff=!1}},_setOptionDisabled:function(t){this._super(t),this._toggleClass(null,"ui-state-disabled",!!t)},_value:function(){var t=this.options.value;return t=this._trimAlignValue(t)},_values:function(t){var e,i;if(arguments.length)return t=this.options.values[t],t=this._trimAlignValue(t);if(this._hasMultipleValues()){for(e=this.options.values.slice(),i=0;i<e.length;i+=1)e[i]=this._trimAlignValue(e[i]);return e}return[]},_trimAlignValue:function(t){if(t<=this._valueMin())return this._valueMin();if(t>=this._valueMax())return this._valueMax();var e=0<this.options.step?this.options.step:1,i=(t-this._valueMin())%e,t=t-i;return 2*Math.abs(i)>=e&&(t+=0<i?e:-e),parseFloat(t.toFixed(5))},_calculateNewMax:function(){var t=this.options.max,e=this._valueMin(),i=this.options.step;(t=Math.round((t-e)/i)*i+e)>this.options.max&&(t-=i),this.max=parseFloat(t.toFixed(this._precision()))},_precision:function(){var t=this._precisionOf(this.options.step);return t=null!==this.options.min?Math.max(t,this._precisionOf(this.options.min)):t},_precisionOf:function(t){var e=t.toString(),t=e.indexOf(".");return-1===t?0:e.length-t-1},_valueMin:function(){return this.options.min},_valueMax:function(){return this.max},_refreshRange:function(t){"vertical"===t&&this.range.css({width:"",left:""}),"horizontal"===t&&this.range.css({height:"",bottom:""})},_refreshValue:function(){var e,i,t,s,n,o=this.options.range,a=this.options,r=this,l=!this._animateOff&&a.animate,h={};this._hasMultipleValues()?this.handles.each(function(t){i=(r.values(t)-r._valueMin())/(r._valueMax()-r._valueMin())*100,h["horizontal"===r.orientation?"left":"bottom"]=i+"%",V(this).stop(1,1)[l?"animate":"css"](h,a.animate),!0===r.options.range&&("horizontal"===r.orientation?(0===t&&r.range.stop(1,1)[l?"animate":"css"]({left:i+"%"},a.animate),1===t&&r.range[l?"animate":"css"]({width:i-e+"%"},{queue:!1,duration:a.animate})):(0===t&&r.range.stop(1,1)[l?"animate":"css"]({bottom:i+"%"},a.animate),1===t&&r.range[l?"animate":"css"]({height:i-e+"%"},{queue:!1,duration:a.animate}))),e=i}):(t=this.value(),s=this._valueMin(),n=this._valueMax(),i=n!==s?(t-s)/(n-s)*100:0,h["horizontal"===this.orientation?"left":"bottom"]=i+"%",this.handle.stop(1,1)[l?"animate":"css"](h,a.animate),"min"===o&&"horizontal"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({width:i+"%"},a.animate),"max"===o&&"horizontal"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({width:100-i+"%"},a.animate),"min"===o&&"vertical"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({height:i+"%"},a.animate),"max"===o&&"vertical"===this.orientation&&this.range.stop(1,1)[l?"animate":"css"]({height:100-i+"%"},a.animate))},_handleEvents:{keydown:function(t){var e,i,s,n=V(t.target).data("ui-slider-handle-index");switch(t.keyCode){case V.ui.keyCode.HOME:case V.ui.keyCode.END:case V.ui.keyCode.PAGE_UP:case V.ui.keyCode.PAGE_DOWN:case V.ui.keyCode.UP:case V.ui.keyCode.RIGHT:case V.ui.keyCode.DOWN:case V.ui.keyCode.LEFT:if(t.preventDefault(),!this._keySliding&&(this._keySliding=!0,this._addClass(V(t.target),null,"ui-state-active"),!1===this._start(t,n)))return}switch(s=this.options.step,e=i=this._hasMultipleValues()?this.values(n):this.value(),t.keyCode){case V.ui.keyCode.HOME:i=this._valueMin();break;case V.ui.keyCode.END:i=this._valueMax();break;case V.ui.keyCode.PAGE_UP:i=this._trimAlignValue(e+(this._valueMax()-this._valueMin())/this.numPages);break;case V.ui.keyCode.PAGE_DOWN:i=this._trimAlignValue(e-(this._valueMax()-this._valueMin())/this.numPages);break;case V.ui.keyCode.UP:case V.ui.keyCode.RIGHT:if(e===this._valueMax())return;i=this._trimAlignValue(e+s);break;case V.ui.keyCode.DOWN:case V.ui.keyCode.LEFT:if(e===this._valueMin())return;i=this._trimAlignValue(e-s)}this._slide(t,n,i)},keyup:function(t){var e=V(t.target).data("ui-slider-handle-index");this._keySliding&&(this._keySliding=!1,this._stop(t,e),this._change(t,e),this._removeClass(V(t.target),null,"ui-state-active"))}}}),V.widget("ui.sortable",V.ui.mouse,{version:"1.13.2",widgetEventPrefix:"sort",ready:!1,options:{appendTo:"parent",axis:!1,connectWith:!1,containment:!1,cursor:"auto",cursorAt:!1,dropOnEmpty:!0,forcePlaceholderSize:!1,forceHelperSize:!1,grid:!1,handle:!1,helper:"original",items:"> *",opacity:!1,placeholder:!1,revert:!1,scroll:!0,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1e3,activate:null,beforeStop:null,change:null,deactivate:null,out:null,over:null,receive:null,remove:null,sort:null,start:null,stop:null,update:null},_isOverAxis:function(t,e,i){return e<=t&&t<e+i},_isFloating:function(t){return/left|right/.test(t.css("float"))||/inline|table-cell/.test(t.css("display"))},_create:function(){this.containerCache={},this._addClass("ui-sortable"),this.refresh(),this.offset=this.element.offset(),this._mouseInit(),this._setHandleClassName(),this.ready=!0},_setOption:function(t,e){this._super(t,e),"handle"===t&&this._setHandleClassName()},_setHandleClassName:function(){var t=this;this._removeClass(this.element.find(".ui-sortable-handle"),"ui-sortable-handle"),V.each(this.items,function(){t._addClass(this.instance.options.handle?this.item.find(this.instance.options.handle):this.item,"ui-sortable-handle")})},_destroy:function(){this._mouseDestroy();for(var t=this.items.length-1;0<=t;t--)this.items[t].item.removeData(this.widgetName+"-item");return this},_mouseCapture:function(t,e){var i=null,s=!1,n=this;return!this.reverting&&(!this.options.disabled&&"static"!==this.options.type&&(this._refreshItems(t),V(t.target).parents().each(function(){if(V.data(this,n.widgetName+"-item")===n)return i=V(this),!1}),!!(i=V.data(t.target,n.widgetName+"-item")===n?V(t.target):i)&&(!(this.options.handle&&!e&&(V(this.options.handle,i).find("*").addBack().each(function(){this===t.target&&(s=!0)}),!s))&&(this.currentItem=i,this._removeCurrentsFromItems(),!0))))},_mouseStart:function(t,e,i){var s,n,o=this.options;if((this.currentContainer=this).refreshPositions(),this.appendTo=V("parent"!==o.appendTo?o.appendTo:this.currentItem.parent()),this.helper=this._createHelper(t),this._cacheHelperProportions(),this._cacheMargins(),this.offset=this.currentItem.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},V.extend(this.offset,{click:{left:t.pageX-this.offset.left,top:t.pageY-this.offset.top},relative:this._getRelativeOffset()}),this.helper.css("position","absolute"),this.cssPosition=this.helper.css("position"),o.cursorAt&&this._adjustOffsetFromHelper(o.cursorAt),this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]},this.helper[0]!==this.currentItem[0]&&this.currentItem.hide(),this._createPlaceholder(),this.scrollParent=this.placeholder.scrollParent(),V.extend(this.offset,{parent:this._getParentOffset()}),o.containment&&this._setContainment(),o.cursor&&"auto"!==o.cursor&&(n=this.document.find("body"),this.storedCursor=n.css("cursor"),n.css("cursor",o.cursor),this.storedStylesheet=V("<style>*{ cursor: "+o.cursor+" !important; }</style>").appendTo(n)),o.zIndex&&(this.helper.css("zIndex")&&(this._storedZIndex=this.helper.css("zIndex")),this.helper.css("zIndex",o.zIndex)),o.opacity&&(this.helper.css("opacity")&&(this._storedOpacity=this.helper.css("opacity")),this.helper.css("opacity",o.opacity)),this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName&&(this.overflowOffset=this.scrollParent.offset()),this._trigger("start",t,this._uiHash()),this._preserveHelperProportions||this._cacheHelperProportions(),!i)for(s=this.containers.length-1;0<=s;s--)this.containers[s]._trigger("activate",t,this._uiHash(this));return V.ui.ddmanager&&(V.ui.ddmanager.current=this),V.ui.ddmanager&&!o.dropBehaviour&&V.ui.ddmanager.prepareOffsets(this,t),this.dragging=!0,this._addClass(this.helper,"ui-sortable-helper"),this.helper.parent().is(this.appendTo)||(this.helper.detach().appendTo(this.appendTo),this.offset.parent=this._getParentOffset()),this.position=this.originalPosition=this._generatePosition(t),this.originalPageX=t.pageX,this.originalPageY=t.pageY,this.lastPositionAbs=this.positionAbs=this._convertPositionTo("absolute"),this._mouseDrag(t),!0},_scroll:function(t){var e=this.options,i=!1;return this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName?(this.overflowOffset.top+this.scrollParent[0].offsetHeight-t.pageY<e.scrollSensitivity?this.scrollParent[0].scrollTop=i=this.scrollParent[0].scrollTop+e.scrollSpeed:t.pageY-this.overflowOffset.top<e.scrollSensitivity&&(this.scrollParent[0].scrollTop=i=this.scrollParent[0].scrollTop-e.scrollSpeed),this.overflowOffset.left+this.scrollParent[0].offsetWidth-t.pageX<e.scrollSensitivity?this.scrollParent[0].scrollLeft=i=this.scrollParent[0].scrollLeft+e.scrollSpeed:t.pageX-this.overflowOffset.left<e.scrollSensitivity&&(this.scrollParent[0].scrollLeft=i=this.scrollParent[0].scrollLeft-e.scrollSpeed)):(t.pageY-this.document.scrollTop()<e.scrollSensitivity?i=this.document.scrollTop(this.document.scrollTop()-e.scrollSpeed):this.window.height()-(t.pageY-this.document.scrollTop())<e.scrollSensitivity&&(i=this.document.scrollTop(this.document.scrollTop()+e.scrollSpeed)),t.pageX-this.document.scrollLeft()<e.scrollSensitivity?i=this.document.scrollLeft(this.document.scrollLeft()-e.scrollSpeed):this.window.width()-(t.pageX-this.document.scrollLeft())<e.scrollSensitivity&&(i=this.document.scrollLeft(this.document.scrollLeft()+e.scrollSpeed))),i},_mouseDrag:function(t){var e,i,s,n,o=this.options;for(this.position=this._generatePosition(t),this.positionAbs=this._convertPositionTo("absolute"),this.options.axis&&"y"===this.options.axis||(this.helper[0].style.left=this.position.left+"px"),this.options.axis&&"x"===this.options.axis||(this.helper[0].style.top=this.position.top+"px"),o.scroll&&!1!==this._scroll(t)&&(this._refreshItemPositions(!0),V.ui.ddmanager&&!o.dropBehaviour&&V.ui.ddmanager.prepareOffsets(this,t)),this.dragDirection={vertical:this._getDragVerticalDirection(),horizontal:this._getDragHorizontalDirection()},e=this.items.length-1;0<=e;e--)if(s=(i=this.items[e]).item[0],(n=this._intersectsWithPointer(i))&&i.instance===this.currentContainer&&!(s===this.currentItem[0]||this.placeholder[1===n?"next":"prev"]()[0]===s||V.contains(this.placeholder[0],s)||"semi-dynamic"===this.options.type&&V.contains(this.element[0],s))){if(this.direction=1===n?"down":"up","pointer"!==this.options.tolerance&&!this._intersectsWithSides(i))break;this._rearrange(t,i),this._trigger("change",t,this._uiHash());break}return this._contactContainers(t),V.ui.ddmanager&&V.ui.ddmanager.drag(this,t),this._trigger("sort",t,this._uiHash()),this.lastPositionAbs=this.positionAbs,!1},_mouseStop:function(t,e){var i,s,n,o;if(t)return V.ui.ddmanager&&!this.options.dropBehaviour&&V.ui.ddmanager.drop(this,t),this.options.revert?(s=(i=this).placeholder.offset(),o={},(n=this.options.axis)&&"x"!==n||(o.left=s.left-this.offset.parent.left-this.margins.left+(this.offsetParent[0]===this.document[0].body?0:this.offsetParent[0].scrollLeft)),n&&"y"!==n||(o.top=s.top-this.offset.parent.top-this.margins.top+(this.offsetParent[0]===this.document[0].body?0:this.offsetParent[0].scrollTop)),this.reverting=!0,V(this.helper).animate(o,parseInt(this.options.revert,10)||500,function(){i._clear(t)})):this._clear(t,e),!1},cancel:function(){if(this.dragging){this._mouseUp(new V.Event("mouseup",{target:null})),"original"===this.options.helper?(this.currentItem.css(this._storedCSS),this._removeClass(this.currentItem,"ui-sortable-helper")):this.currentItem.show();for(var t=this.containers.length-1;0<=t;t--)this.containers[t]._trigger("deactivate",null,this._uiHash(this)),this.containers[t].containerCache.over&&(this.containers[t]._trigger("out",null,this._uiHash(this)),this.containers[t].containerCache.over=0)}return this.placeholder&&(this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]),"original"!==this.options.helper&&this.helper&&this.helper[0].parentNode&&this.helper.remove(),V.extend(this,{helper:null,dragging:!1,reverting:!1,_noFinalSort:null}),this.domPosition.prev?V(this.domPosition.prev).after(this.currentItem):V(this.domPosition.parent).prepend(this.currentItem)),this},serialize:function(e){var t=this._getItemsAsjQuery(e&&e.connected),i=[];return e=e||{},V(t).each(function(){var t=(V(e.item||this).attr(e.attribute||"id")||"").match(e.expression||/(.+)[\-=_](.+)/);t&&i.push((e.key||t[1]+"[]")+"="+(e.key&&e.expression?t[1]:t[2]))}),!i.length&&e.key&&i.push(e.key+"="),i.join("&")},toArray:function(t){var e=this._getItemsAsjQuery(t&&t.connected),i=[];return t=t||{},e.each(function(){i.push(V(t.item||this).attr(t.attribute||"id")||"")}),i},_intersectsWith:function(t){var e=this.positionAbs.left,i=e+this.helperProportions.width,s=this.positionAbs.top,n=s+this.helperProportions.height,o=t.left,a=o+t.width,r=t.top,l=r+t.height,h=this.offset.click.top,c=this.offset.click.left,h="x"===this.options.axis||r<s+h&&s+h<l,c="y"===this.options.axis||o<e+c&&e+c<a;return"pointer"===this.options.tolerance||this.options.forcePointerForContainers||"pointer"!==this.options.tolerance&&this.helperProportions[this.floating?"width":"height"]>t[this.floating?"width":"height"]?h&&c:o<e+this.helperProportions.width/2&&i-this.helperProportions.width/2<a&&r<s+this.helperProportions.height/2&&n-this.helperProportions.height/2<l},_intersectsWithPointer:function(t){var e="x"===this.options.axis||this._isOverAxis(this.positionAbs.top+this.offset.click.top,t.top,t.height),t="y"===this.options.axis||this._isOverAxis(this.positionAbs.left+this.offset.click.left,t.left,t.width);return!(!e||!t)&&(e=this.dragDirection.vertical,t=this.dragDirection.horizontal,this.floating?"right"===t||"down"===e?2:1:e&&("down"===e?2:1))},_intersectsWithSides:function(t){var e=this._isOverAxis(this.positionAbs.top+this.offset.click.top,t.top+t.height/2,t.height),i=this._isOverAxis(this.positionAbs.left+this.offset.click.left,t.left+t.width/2,t.width),s=this.dragDirection.vertical,t=this.dragDirection.horizontal;return this.floating&&t?"right"===t&&i||"left"===t&&!i:s&&("down"===s&&e||"up"===s&&!e)},_getDragVerticalDirection:function(){var t=this.positionAbs.top-this.lastPositionAbs.top;return 0!=t&&(0<t?"down":"up")},_getDragHorizontalDirection:function(){var t=this.positionAbs.left-this.lastPositionAbs.left;return 0!=t&&(0<t?"right":"left")},refresh:function(t){return this._refreshItems(t),this._setHandleClassName(),this.refreshPositions(),this},_connectWith:function(){var t=this.options;return t.connectWith.constructor===String?[t.connectWith]:t.connectWith},_getItemsAsjQuery:function(t){var e,i,s,n,o=[],a=[],r=this._connectWith();if(r&&t)for(e=r.length-1;0<=e;e--)for(i=(s=V(r[e],this.document[0])).length-1;0<=i;i--)(n=V.data(s[i],this.widgetFullName))&&n!==this&&!n.options.disabled&&a.push(["function"==typeof n.options.items?n.options.items.call(n.element):V(n.options.items,n.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),n]);function l(){o.push(this)}for(a.push(["function"==typeof this.options.items?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):V(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),this]),e=a.length-1;0<=e;e--)a[e][0].each(l);return V(o)},_removeCurrentsFromItems:function(){var i=this.currentItem.find(":data("+this.widgetName+"-item)");this.items=V.grep(this.items,function(t){for(var e=0;e<i.length;e++)if(i[e]===t.item[0])return!1;return!0})},_refreshItems:function(t){this.items=[],this.containers=[this];var e,i,s,n,o,a,r,l,h=this.items,c=[["function"==typeof this.options.items?this.options.items.call(this.element[0],t,{item:this.currentItem}):V(this.options.items,this.element),this]],u=this._connectWith();if(u&&this.ready)for(e=u.length-1;0<=e;e--)for(i=(s=V(u[e],this.document[0])).length-1;0<=i;i--)(n=V.data(s[i],this.widgetFullName))&&n!==this&&!n.options.disabled&&(c.push(["function"==typeof n.options.items?n.options.items.call(n.element[0],t,{item:this.currentItem}):V(n.options.items,n.element),n]),this.containers.push(n));for(e=c.length-1;0<=e;e--)for(o=c[e][1],l=(a=c[e][i=0]).length;i<l;i++)(r=V(a[i])).data(this.widgetName+"-item",o),h.push({item:r,instance:o,width:0,height:0,left:0,top:0})},_refreshItemPositions:function(t){for(var e,i,s=this.items.length-1;0<=s;s--)e=this.items[s],this.currentContainer&&e.instance!==this.currentContainer&&e.item[0]!==this.currentItem[0]||(i=this.options.toleranceElement?V(this.options.toleranceElement,e.item):e.item,t||(e.width=i.outerWidth(),e.height=i.outerHeight()),i=i.offset(),e.left=i.left,e.top=i.top)},refreshPositions:function(t){var e,i;if(this.floating=!!this.items.length&&("x"===this.options.axis||this._isFloating(this.items[0].item)),this.offsetParent&&this.helper&&(this.offset.parent=this._getParentOffset()),this._refreshItemPositions(t),this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(e=this.containers.length-1;0<=e;e--)i=this.containers[e].element.offset(),this.containers[e].containerCache.left=i.left,this.containers[e].containerCache.top=i.top,this.containers[e].containerCache.width=this.containers[e].element.outerWidth(),this.containers[e].containerCache.height=this.containers[e].element.outerHeight();return this},_createPlaceholder:function(i){var s,n,o=(i=i||this).options;o.placeholder&&o.placeholder.constructor!==String||(s=o.placeholder,n=i.currentItem[0].nodeName.toLowerCase(),o.placeholder={element:function(){var t=V("<"+n+">",i.document[0]);return i._addClass(t,"ui-sortable-placeholder",s||i.currentItem[0].className)._removeClass(t,"ui-sortable-helper"),"tbody"===n?i._createTrPlaceholder(i.currentItem.find("tr").eq(0),V("<tr>",i.document[0]).appendTo(t)):"tr"===n?i._createTrPlaceholder(i.currentItem,t):"img"===n&&t.attr("src",i.currentItem.attr("src")),s||t.css("visibility","hidden"),t},update:function(t,e){s&&!o.forcePlaceholderSize||(e.height()&&(!o.forcePlaceholderSize||"tbody"!==n&&"tr"!==n)||e.height(i.currentItem.innerHeight()-parseInt(i.currentItem.css("paddingTop")||0,10)-parseInt(i.currentItem.css("paddingBottom")||0,10)),e.width()||e.width(i.currentItem.innerWidth()-parseInt(i.currentItem.css("paddingLeft")||0,10)-parseInt(i.currentItem.css("paddingRight")||0,10)))}}),i.placeholder=V(o.placeholder.element.call(i.element,i.currentItem)),i.currentItem.after(i.placeholder),o.placeholder.update(i,i.placeholder)},_createTrPlaceholder:function(t,e){var i=this;t.children().each(function(){V("<td>&#160;</td>",i.document[0]).attr("colspan",V(this).attr("colspan")||1).appendTo(e)})},_contactContainers:function(t){for(var e,i,s,n,o,a,r,l,h,c=null,u=null,d=this.containers.length-1;0<=d;d--)V.contains(this.currentItem[0],this.containers[d].element[0])||(this._intersectsWith(this.containers[d].containerCache)?c&&V.contains(this.containers[d].element[0],c.element[0])||(c=this.containers[d],u=d):this.containers[d].containerCache.over&&(this.containers[d]._trigger("out",t,this._uiHash(this)),this.containers[d].containerCache.over=0));if(c)if(1===this.containers.length)this.containers[u].containerCache.over||(this.containers[u]._trigger("over",t,this._uiHash(this)),this.containers[u].containerCache.over=1);else{for(i=1e4,s=null,n=(l=c.floating||this._isFloating(this.currentItem))?"left":"top",o=l?"width":"height",h=l?"pageX":"pageY",e=this.items.length-1;0<=e;e--)V.contains(this.containers[u].element[0],this.items[e].item[0])&&this.items[e].item[0]!==this.currentItem[0]&&(a=this.items[e].item.offset()[n],r=!1,t[h]-a>this.items[e][o]/2&&(r=!0),Math.abs(t[h]-a)<i&&(i=Math.abs(t[h]-a),s=this.items[e],this.direction=r?"up":"down"));(s||this.options.dropOnEmpty)&&(this.currentContainer!==this.containers[u]?(s?this._rearrange(t,s,null,!0):this._rearrange(t,null,this.containers[u].element,!0),this._trigger("change",t,this._uiHash()),this.containers[u]._trigger("change",t,this._uiHash(this)),this.currentContainer=this.containers[u],this.options.placeholder.update(this.currentContainer,this.placeholder),this.scrollParent=this.placeholder.scrollParent(),this.scrollParent[0]!==this.document[0]&&"HTML"!==this.scrollParent[0].tagName&&(this.overflowOffset=this.scrollParent.offset()),this.containers[u]._trigger("over",t,this._uiHash(this)),this.containers[u].containerCache.over=1):this.currentContainer.containerCache.over||(this.containers[u]._trigger("over",t,this._uiHash()),this.currentContainer.containerCache.over=1))}},_createHelper:function(t){var e=this.options,t="function"==typeof e.helper?V(e.helper.apply(this.element[0],[t,this.currentItem])):"clone"===e.helper?this.currentItem.clone():this.currentItem;return t.parents("body").length||this.appendTo[0].appendChild(t[0]),t[0]===this.currentItem[0]&&(this._storedCSS={width:this.currentItem[0].style.width,height:this.currentItem[0].style.height,position:this.currentItem.css("position"),top:this.currentItem.css("top"),left:this.currentItem.css("left")}),t[0].style.width&&!e.forceHelperSize||t.width(this.currentItem.width()),t[0].style.height&&!e.forceHelperSize||t.height(this.currentItem.height()),t},_adjustOffsetFromHelper:function(t){"string"==typeof t&&(t=t.split(" ")),"left"in(t=Array.isArray(t)?{left:+t[0],top:+t[1]||0}:t)&&(this.offset.click.left=t.left+this.margins.left),"right"in t&&(this.offset.click.left=this.helperProportions.width-t.right+this.margins.left),"top"in t&&(this.offset.click.top=t.top+this.margins.top),"bottom"in t&&(this.offset.click.top=this.helperProportions.height-t.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var t=this.offsetParent.offset();return"absolute"===this.cssPosition&&this.scrollParent[0]!==this.document[0]&&V.contains(this.scrollParent[0],this.offsetParent[0])&&(t.left+=this.scrollParent.scrollLeft(),t.top+=this.scrollParent.scrollTop()),{top:(t=this.offsetParent[0]===this.document[0].body||this.offsetParent[0].tagName&&"html"===this.offsetParent[0].tagName.toLowerCase()&&V.ui.ie?{top:0,left:0}:t).top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:t.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if("relative"!==this.cssPosition)return{top:0,left:0};var t=this.currentItem.position();return{top:t.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:t.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}},_cacheMargins:function(){this.margins={left:parseInt(this.currentItem.css("marginLeft"),10)||0,top:parseInt(this.currentItem.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var t,e,i=this.options;"parent"===i.containment&&(i.containment=this.helper[0].parentNode),"document"!==i.containment&&"window"!==i.containment||(this.containment=[0-this.offset.relative.left-this.offset.parent.left,0-this.offset.relative.top-this.offset.parent.top,"document"===i.containment?this.document.width():this.window.width()-this.helperProportions.width-this.margins.left,("document"===i.containment?this.document.height()||document.body.parentNode.scrollHeight:this.window.height()||this.document[0].body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top]),/^(document|window|parent)$/.test(i.containment)||(t=V(i.containment)[0],e=V(i.containment).offset(),i="hidden"!==V(t).css("overflow"),this.containment=[e.left+(parseInt(V(t).css("borderLeftWidth"),10)||0)+(parseInt(V(t).css("paddingLeft"),10)||0)-this.margins.left,e.top+(parseInt(V(t).css("borderTopWidth"),10)||0)+(parseInt(V(t).css("paddingTop"),10)||0)-this.margins.top,e.left+(i?Math.max(t.scrollWidth,t.offsetWidth):t.offsetWidth)-(parseInt(V(t).css("borderLeftWidth"),10)||0)-(parseInt(V(t).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left,e.top+(i?Math.max(t.scrollHeight,t.offsetHeight):t.offsetHeight)-(parseInt(V(t).css("borderTopWidth"),10)||0)-(parseInt(V(t).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top])},_convertPositionTo:function(t,e){e=e||this.position;var i="absolute"===t?1:-1,s="absolute"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&V.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,t=/(html|body)/i.test(s[0].tagName);return{top:e.top+this.offset.relative.top*i+this.offset.parent.top*i-("fixed"===this.cssPosition?-this.scrollParent.scrollTop():t?0:s.scrollTop())*i,left:e.left+this.offset.relative.left*i+this.offset.parent.left*i-("fixed"===this.cssPosition?-this.scrollParent.scrollLeft():t?0:s.scrollLeft())*i}},_generatePosition:function(t){var e=this.options,i=t.pageX,s=t.pageY,n="absolute"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&V.contains(this.scrollParent[0],this.offsetParent[0])?this.scrollParent:this.offsetParent,o=/(html|body)/i.test(n[0].tagName);return"relative"!==this.cssPosition||this.scrollParent[0]!==this.document[0]&&this.scrollParent[0]!==this.offsetParent[0]||(this.offset.relative=this._getRelativeOffset()),this.originalPosition&&(this.containment&&(t.pageX-this.offset.click.left<this.containment[0]&&(i=this.containment[0]+this.offset.click.left),t.pageY-this.offset.click.top<this.containment[1]&&(s=this.containment[1]+this.offset.click.top),t.pageX-this.offset.click.left>this.containment[2]&&(i=this.containment[2]+this.offset.click.left),t.pageY-this.offset.click.top>this.containment[3]&&(s=this.containment[3]+this.offset.click.top)),e.grid&&(t=this.originalPageY+Math.round((s-this.originalPageY)/e.grid[1])*e.grid[1],s=!this.containment||t-this.offset.click.top>=this.containment[1]&&t-this.offset.click.top<=this.containment[3]?t:t-this.offset.click.top>=this.containment[1]?t-e.grid[1]:t+e.grid[1],t=this.originalPageX+Math.round((i-this.originalPageX)/e.grid[0])*e.grid[0],i=!this.containment||t-this.offset.click.left>=this.containment[0]&&t-this.offset.click.left<=this.containment[2]?t:t-this.offset.click.left>=this.containment[0]?t-e.grid[0]:t+e.grid[0])),{top:s-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+("fixed"===this.cssPosition?-this.scrollParent.scrollTop():o?0:n.scrollTop()),left:i-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+("fixed"===this.cssPosition?-this.scrollParent.scrollLeft():o?0:n.scrollLeft())}},_rearrange:function(t,e,i,s){i?i[0].appendChild(this.placeholder[0]):e.item[0].parentNode.insertBefore(this.placeholder[0],"down"===this.direction?e.item[0]:e.item[0].nextSibling),this.counter=this.counter?++this.counter:1;var n=this.counter;this._delay(function(){n===this.counter&&this.refreshPositions(!s)})},_clear:function(t,e){this.reverting=!1;var i,s=[];if(!this._noFinalSort&&this.currentItem.parent().length&&this.placeholder.before(this.currentItem),this._noFinalSort=null,this.helper[0]===this.currentItem[0]){for(i in this._storedCSS)"auto"!==this._storedCSS[i]&&"static"!==this._storedCSS[i]||(this._storedCSS[i]="");this.currentItem.css(this._storedCSS),this._removeClass(this.currentItem,"ui-sortable-helper")}else this.currentItem.show();function n(e,i,s){return function(t){s._trigger(e,t,i._uiHash(i))}}for(this.fromOutside&&!e&&s.push(function(t){this._trigger("receive",t,this._uiHash(this.fromOutside))}),!this.fromOutside&&this.domPosition.prev===this.currentItem.prev().not(".ui-sortable-helper")[0]&&this.domPosition.parent===this.currentItem.parent()[0]||e||s.push(function(t){this._trigger("update",t,this._uiHash())}),this!==this.currentContainer&&(e||(s.push(function(t){this._trigger("remove",t,this._uiHash())}),s.push(function(e){return function(t){e._trigger("receive",t,this._uiHash(this))}}.call(this,this.currentContainer)),s.push(function(e){return function(t){e._trigger("update",t,this._uiHash(this))}}.call(this,this.currentContainer)))),i=this.containers.length-1;0<=i;i--)e||s.push(n("deactivate",this,this.containers[i])),this.containers[i].containerCache.over&&(s.push(n("out",this,this.containers[i])),this.containers[i].containerCache.over=0);if(this.storedCursor&&(this.document.find("body").css("cursor",this.storedCursor),this.storedStylesheet.remove()),this._storedOpacity&&this.helper.css("opacity",this._storedOpacity),this._storedZIndex&&this.helper.css("zIndex","auto"===this._storedZIndex?"":this._storedZIndex),this.dragging=!1,e||this._trigger("beforeStop",t,this._uiHash()),this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.cancelHelperRemoval||(this.helper[0]!==this.currentItem[0]&&this.helper.remove(),this.helper=null),!e){for(i=0;i<s.length;i++)s[i].call(this,t);this._trigger("stop",t,this._uiHash())}return this.fromOutside=!1,!this.cancelHelperRemoval},_trigger:function(){!1===V.Widget.prototype._trigger.apply(this,arguments)&&this.cancel()},_uiHash:function(t){var e=t||this;return{helper:e.helper,placeholder:e.placeholder||V([]),position:e.position,originalPosition:e.originalPosition,offset:e.positionAbs,item:e.currentItem,sender:t?t.element:null}}});function ht(e){return function(){var t=this.element.val();e.apply(this,arguments),this._refresh(),t!==this.element.val()&&this._trigger("change")}}V.widget("ui.spinner",{version:"1.13.2",defaultElement:"<input>",widgetEventPrefix:"spin",options:{classes:{"ui-spinner":"ui-corner-all","ui-spinner-down":"ui-corner-br","ui-spinner-up":"ui-corner-tr"},culture:null,icons:{down:"ui-icon-triangle-1-s",up:"ui-icon-triangle-1-n"},incremental:!0,max:null,min:null,numberFormat:null,page:10,step:1,change:null,spin:null,start:null,stop:null},_create:function(){this._setOption("max",this.options.max),this._setOption("min",this.options.min),this._setOption("step",this.options.step),""!==this.value()&&this._value(this.element.val(),!0),this._draw(),this._on(this._events),this._refresh(),this._on(this.window,{beforeunload:function(){this.element.removeAttr("autocomplete")}})},_getCreateOptions:function(){var s=this._super(),n=this.element;return V.each(["min","max","step"],function(t,e){var i=n.attr(e);null!=i&&i.length&&(s[e]=i)}),s},_events:{keydown:function(t){this._start(t)&&this._keydown(t)&&t.preventDefault()},keyup:"_stop",focus:function(){this.previous=this.element.val()},blur:function(t){this.cancelBlur?delete this.cancelBlur:(this._stop(),this._refresh(),this.previous!==this.element.val()&&this._trigger("change",t))},mousewheel:function(t,e){var i=V.ui.safeActiveElement(this.document[0]);if(this.element[0]===i&&e){if(!this.spinning&&!this._start(t))return!1;this._spin((0<e?1:-1)*this.options.step,t),clearTimeout(this.mousewheelTimer),this.mousewheelTimer=this._delay(function(){this.spinning&&this._stop(t)},100),t.preventDefault()}},"mousedown .ui-spinner-button":function(t){var e;function i(){this.element[0]===V.ui.safeActiveElement(this.document[0])||(this.element.trigger("focus"),this.previous=e,this._delay(function(){this.previous=e}))}e=this.element[0]===V.ui.safeActiveElement(this.document[0])?this.previous:this.element.val(),t.preventDefault(),i.call(this),this.cancelBlur=!0,this._delay(function(){delete this.cancelBlur,i.call(this)}),!1!==this._start(t)&&this._repeat(null,V(t.currentTarget).hasClass("ui-spinner-up")?1:-1,t)},"mouseup .ui-spinner-button":"_stop","mouseenter .ui-spinner-button":function(t){if(V(t.currentTarget).hasClass("ui-state-active"))return!1!==this._start(t)&&void this._repeat(null,V(t.currentTarget).hasClass("ui-spinner-up")?1:-1,t)},"mouseleave .ui-spinner-button":"_stop"},_enhance:function(){this.uiSpinner=this.element.attr("autocomplete","off").wrap("<span>").parent().append("<a></a><a></a>")},_draw:function(){this._enhance(),this._addClass(this.uiSpinner,"ui-spinner","ui-widget ui-widget-content"),this._addClass("ui-spinner-input"),this.element.attr("role","spinbutton"),this.buttons=this.uiSpinner.children("a").attr("tabIndex",-1).attr("aria-hidden",!0).button({classes:{"ui-button":""}}),this._removeClass(this.buttons,"ui-corner-all"),this._addClass(this.buttons.first(),"ui-spinner-button ui-spinner-up"),this._addClass(this.buttons.last(),"ui-spinner-button ui-spinner-down"),this.buttons.first().button({icon:this.options.icons.up,showLabel:!1}),this.buttons.last().button({icon:this.options.icons.down,showLabel:!1}),this.buttons.height()>Math.ceil(.5*this.uiSpinner.height())&&0<this.uiSpinner.height()&&this.uiSpinner.height(this.uiSpinner.height())},_keydown:function(t){var e=this.options,i=V.ui.keyCode;switch(t.keyCode){case i.UP:return this._repeat(null,1,t),!0;case i.DOWN:return this._repeat(null,-1,t),!0;case i.PAGE_UP:return this._repeat(null,e.page,t),!0;case i.PAGE_DOWN:return this._repeat(null,-e.page,t),!0}return!1},_start:function(t){return!(!this.spinning&&!1===this._trigger("start",t))&&(this.counter||(this.counter=1),this.spinning=!0)},_repeat:function(t,e,i){t=t||500,clearTimeout(this.timer),this.timer=this._delay(function(){this._repeat(40,e,i)},t),this._spin(e*this.options.step,i)},_spin:function(t,e){var i=this.value()||0;this.counter||(this.counter=1),i=this._adjustValue(i+t*this._increment(this.counter)),this.spinning&&!1===this._trigger("spin",e,{value:i})||(this._value(i),this.counter++)},_increment:function(t){var e=this.options.incremental;return e?"function"==typeof e?e(t):Math.floor(t*t*t/5e4-t*t/500+17*t/200+1):1},_precision:function(){var t=this._precisionOf(this.options.step);return t=null!==this.options.min?Math.max(t,this._precisionOf(this.options.min)):t},_precisionOf:function(t){var e=t.toString(),t=e.indexOf(".");return-1===t?0:e.length-t-1},_adjustValue:function(t){var e=this.options,i=null!==e.min?e.min:0,s=t-i;return t=i+Math.round(s/e.step)*e.step,t=parseFloat(t.toFixed(this._precision())),null!==e.max&&t>e.max?e.max:null!==e.min&&t<e.min?e.min:t},_stop:function(t){this.spinning&&(clearTimeout(this.timer),clearTimeout(this.mousewheelTimer),this.counter=0,this.spinning=!1,this._trigger("stop",t))},_setOption:function(t,e){var i;if("culture"===t||"numberFormat"===t)return i=this._parse(this.element.val()),this.options[t]=e,void this.element.val(this._format(i));"max"!==t&&"min"!==t&&"step"!==t||"string"==typeof e&&(e=this._parse(e)),"icons"===t&&(i=this.buttons.first().find(".ui-icon"),this._removeClass(i,null,this.options.icons.up),this._addClass(i,null,e.up),i=this.buttons.last().find(".ui-icon"),this._removeClass(i,null,this.options.icons.down),this._addClass(i,null,e.down)),this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this._toggleClass(this.uiSpinner,null,"ui-state-disabled",!!t),this.element.prop("disabled",!!t),this.buttons.button(t?"disable":"enable")},_setOptions:ht(function(t){this._super(t)}),_parse:function(t){return""===(t="string"==typeof t&&""!==t?window.Globalize&&this.options.numberFormat?Globalize.parseFloat(t,10,this.options.culture):+t:t)||isNaN(t)?null:t},_format:function(t){return""===t?"":window.Globalize&&this.options.numberFormat?Globalize.format(t,this.options.numberFormat,this.options.culture):t},_refresh:function(){this.element.attr({"aria-valuemin":this.options.min,"aria-valuemax":this.options.max,"aria-valuenow":this._parse(this.element.val())})},isValid:function(){var t=this.value();return null!==t&&t===this._adjustValue(t)},_value:function(t,e){var i;""!==t&&null!==(i=this._parse(t))&&(e||(i=this._adjustValue(i)),t=this._format(i)),this.element.val(t),this._refresh()},_destroy:function(){this.element.prop("disabled",!1).removeAttr("autocomplete role aria-valuemin aria-valuemax aria-valuenow"),this.uiSpinner.replaceWith(this.element)},stepUp:ht(function(t){this._stepUp(t)}),_stepUp:function(t){this._start()&&(this._spin((t||1)*this.options.step),this._stop())},stepDown:ht(function(t){this._stepDown(t)}),_stepDown:function(t){this._start()&&(this._spin((t||1)*-this.options.step),this._stop())},pageUp:ht(function(t){this._stepUp((t||1)*this.options.page)}),pageDown:ht(function(t){this._stepDown((t||1)*this.options.page)}),value:function(t){if(!arguments.length)return this._parse(this.element.val());ht(this._value).call(this,t)},widget:function(){return this.uiSpinner}}),!1!==V.uiBackCompat&&V.widget("ui.spinner",V.ui.spinner,{_enhance:function(){this.uiSpinner=this.element.attr("autocomplete","off").wrap(this._uiSpinnerHtml()).parent().append(this._buttonHtml())},_uiSpinnerHtml:function(){return"<span>"},_buttonHtml:function(){return"<a></a><a></a>"}});var ct;V.ui.spinner;V.widget("ui.tabs",{version:"1.13.2",delay:300,options:{active:null,classes:{"ui-tabs":"ui-corner-all","ui-tabs-nav":"ui-corner-all","ui-tabs-panel":"ui-corner-bottom","ui-tabs-tab":"ui-corner-top"},collapsible:!1,event:"click",heightStyle:"content",hide:null,show:null,activate:null,beforeActivate:null,beforeLoad:null,load:null},_isLocal:(ct=/#.*$/,function(t){var e=t.href.replace(ct,""),i=location.href.replace(ct,"");try{e=decodeURIComponent(e)}catch(t){}try{i=decodeURIComponent(i)}catch(t){}return 1<t.hash.length&&e===i}),_create:function(){var e=this,t=this.options;this.running=!1,this._addClass("ui-tabs","ui-widget ui-widget-content"),this._toggleClass("ui-tabs-collapsible",null,t.collapsible),this._processTabs(),t.active=this._initialActive(),Array.isArray(t.disabled)&&(t.disabled=V.uniqueSort(t.disabled.concat(V.map(this.tabs.filter(".ui-state-disabled"),function(t){return e.tabs.index(t)}))).sort()),!1!==this.options.active&&this.anchors.length?this.active=this._findActive(t.active):this.active=V(),this._refresh(),this.active.length&&this.load(t.active)},_initialActive:function(){var i=this.options.active,t=this.options.collapsible,s=location.hash.substring(1);return null===i&&(s&&this.tabs.each(function(t,e){if(V(e).attr("aria-controls")===s)return i=t,!1}),null!==(i=null===i?this.tabs.index(this.tabs.filter(".ui-tabs-active")):i)&&-1!==i||(i=!!this.tabs.length&&0)),!1!==i&&-1===(i=this.tabs.index(this.tabs.eq(i)))&&(i=!t&&0),i=!t&&!1===i&&this.anchors.length?0:i},_getCreateEventData:function(){return{tab:this.active,panel:this.active.length?this._getPanelForTab(this.active):V()}},_tabKeydown:function(t){var e=V(V.ui.safeActiveElement(this.document[0])).closest("li"),i=this.tabs.index(e),s=!0;if(!this._handlePageNav(t)){switch(t.keyCode){case V.ui.keyCode.RIGHT:case V.ui.keyCode.DOWN:i++;break;case V.ui.keyCode.UP:case V.ui.keyCode.LEFT:s=!1,i--;break;case V.ui.keyCode.END:i=this.anchors.length-1;break;case V.ui.keyCode.HOME:i=0;break;case V.ui.keyCode.SPACE:return t.preventDefault(),clearTimeout(this.activating),void this._activate(i);case V.ui.keyCode.ENTER:return t.preventDefault(),clearTimeout(this.activating),void this._activate(i!==this.options.active&&i);default:return}t.preventDefault(),clearTimeout(this.activating),i=this._focusNextTab(i,s),t.ctrlKey||t.metaKey||(e.attr("aria-selected","false"),this.tabs.eq(i).attr("aria-selected","true"),this.activating=this._delay(function(){this.option("active",i)},this.delay))}},_panelKeydown:function(t){this._handlePageNav(t)||t.ctrlKey&&t.keyCode===V.ui.keyCode.UP&&(t.preventDefault(),this.active.trigger("focus"))},_handlePageNav:function(t){return t.altKey&&t.keyCode===V.ui.keyCode.PAGE_UP?(this._activate(this._focusNextTab(this.options.active-1,!1)),!0):t.altKey&&t.keyCode===V.ui.keyCode.PAGE_DOWN?(this._activate(this._focusNextTab(this.options.active+1,!0)),!0):void 0},_findNextTab:function(t,e){var i=this.tabs.length-1;for(;-1!==V.inArray(t=(t=i<t?0:t)<0?i:t,this.options.disabled);)t=e?t+1:t-1;return t},_focusNextTab:function(t,e){return t=this._findNextTab(t,e),this.tabs.eq(t).trigger("focus"),t},_setOption:function(t,e){"active"!==t?(this._super(t,e),"collapsible"===t&&(this._toggleClass("ui-tabs-collapsible",null,e),e||!1!==this.options.active||this._activate(0)),"event"===t&&this._setupEvents(e),"heightStyle"===t&&this._setupHeightStyle(e)):this._activate(e)},_sanitizeSelector:function(t){return t?t.replace(/[!"$%&'()*+,.\/:;<=>?@\[\]\^`{|}~]/g,"\\$&"):""},refresh:function(){var t=this.options,e=this.tablist.children(":has(a[href])");t.disabled=V.map(e.filter(".ui-state-disabled"),function(t){return e.index(t)}),this._processTabs(),!1!==t.active&&this.anchors.length?this.active.length&&!V.contains(this.tablist[0],this.active[0])?this.tabs.length===t.disabled.length?(t.active=!1,this.active=V()):this._activate(this._findNextTab(Math.max(0,t.active-1),!1)):t.active=this.tabs.index(this.active):(t.active=!1,this.active=V()),this._refresh()},_refresh:function(){this._setOptionDisabled(this.options.disabled),this._setupEvents(this.options.event),this._setupHeightStyle(this.options.heightStyle),this.tabs.not(this.active).attr({"aria-selected":"false","aria-expanded":"false",tabIndex:-1}),this.panels.not(this._getPanelForTab(this.active)).hide().attr({"aria-hidden":"true"}),this.active.length?(this.active.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0}),this._addClass(this.active,"ui-tabs-active","ui-state-active"),this._getPanelForTab(this.active).show().attr({"aria-hidden":"false"})):this.tabs.eq(0).attr("tabIndex",0)},_processTabs:function(){var l=this,t=this.tabs,e=this.anchors,i=this.panels;this.tablist=this._getList().attr("role","tablist"),this._addClass(this.tablist,"ui-tabs-nav","ui-helper-reset ui-helper-clearfix ui-widget-header"),this.tablist.on("mousedown"+this.eventNamespace,"> li",function(t){V(this).is(".ui-state-disabled")&&t.preventDefault()}).on("focus"+this.eventNamespace,".ui-tabs-anchor",function(){V(this).closest("li").is(".ui-state-disabled")&&this.blur()}),this.tabs=this.tablist.find("> li:has(a[href])").attr({role:"tab",tabIndex:-1}),this._addClass(this.tabs,"ui-tabs-tab","ui-state-default"),this.anchors=this.tabs.map(function(){return V("a",this)[0]}).attr({tabIndex:-1}),this._addClass(this.anchors,"ui-tabs-anchor"),this.panels=V(),this.anchors.each(function(t,e){var i,s,n,o=V(e).uniqueId().attr("id"),a=V(e).closest("li"),r=a.attr("aria-controls");l._isLocal(e)?(n=(i=e.hash).substring(1),s=l.element.find(l._sanitizeSelector(i))):(n=a.attr("aria-controls")||V({}).uniqueId()[0].id,(s=l.element.find(i="#"+n)).length||(s=l._createPanel(n)).insertAfter(l.panels[t-1]||l.tablist),s.attr("aria-live","polite")),s.length&&(l.panels=l.panels.add(s)),r&&a.data("ui-tabs-aria-controls",r),a.attr({"aria-controls":n,"aria-labelledby":o}),s.attr("aria-labelledby",o)}),this.panels.attr("role","tabpanel"),this._addClass(this.panels,"ui-tabs-panel","ui-widget-content"),t&&(this._off(t.not(this.tabs)),this._off(e.not(this.anchors)),this._off(i.not(this.panels)))},_getList:function(){return this.tablist||this.element.find("ol, ul").eq(0)},_createPanel:function(t){return V("<div>").attr("id",t).data("ui-tabs-destroy",!0)},_setOptionDisabled:function(t){var e,i;for(Array.isArray(t)&&(t.length?t.length===this.anchors.length&&(t=!0):t=!1),i=0;e=this.tabs[i];i++)e=V(e),!0===t||-1!==V.inArray(i,t)?(e.attr("aria-disabled","true"),this._addClass(e,null,"ui-state-disabled")):(e.removeAttr("aria-disabled"),this._removeClass(e,null,"ui-state-disabled"));this.options.disabled=t,this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,!0===t)},_setupEvents:function(t){var i={};t&&V.each(t.split(" "),function(t,e){i[e]="_eventHandler"}),this._off(this.anchors.add(this.tabs).add(this.panels)),this._on(!0,this.anchors,{click:function(t){t.preventDefault()}}),this._on(this.anchors,i),this._on(this.tabs,{keydown:"_tabKeydown"}),this._on(this.panels,{keydown:"_panelKeydown"}),this._focusable(this.tabs),this._hoverable(this.tabs)},_setupHeightStyle:function(t){var i,e=this.element.parent();"fill"===t?(i=e.height(),i-=this.element.outerHeight()-this.element.height(),this.element.siblings(":visible").each(function(){var t=V(this),e=t.css("position");"absolute"!==e&&"fixed"!==e&&(i-=t.outerHeight(!0))}),this.element.children().not(this.panels).each(function(){i-=V(this).outerHeight(!0)}),this.panels.each(function(){V(this).height(Math.max(0,i-V(this).innerHeight()+V(this).height()))}).css("overflow","auto")):"auto"===t&&(i=0,this.panels.each(function(){i=Math.max(i,V(this).height("").height())}).height(i))},_eventHandler:function(t){var e=this.options,i=this.active,s=V(t.currentTarget).closest("li"),n=s[0]===i[0],o=n&&e.collapsible,a=o?V():this._getPanelForTab(s),r=i.length?this._getPanelForTab(i):V(),i={oldTab:i,oldPanel:r,newTab:o?V():s,newPanel:a};t.preventDefault(),s.hasClass("ui-state-disabled")||s.hasClass("ui-tabs-loading")||this.running||n&&!e.collapsible||!1===this._trigger("beforeActivate",t,i)||(e.active=!o&&this.tabs.index(s),this.active=n?V():s,this.xhr&&this.xhr.abort(),r.length||a.length||V.error("jQuery UI Tabs: Mismatching fragment identifier."),a.length&&this.load(this.tabs.index(s),t),this._toggle(t,i))},_toggle:function(t,e){var i=this,s=e.newPanel,n=e.oldPanel;function o(){i.running=!1,i._trigger("activate",t,e)}function a(){i._addClass(e.newTab.closest("li"),"ui-tabs-active","ui-state-active"),s.length&&i.options.show?i._show(s,i.options.show,o):(s.show(),o())}this.running=!0,n.length&&this.options.hide?this._hide(n,this.options.hide,function(){i._removeClass(e.oldTab.closest("li"),"ui-tabs-active","ui-state-active"),a()}):(this._removeClass(e.oldTab.closest("li"),"ui-tabs-active","ui-state-active"),n.hide(),a()),n.attr("aria-hidden","true"),e.oldTab.attr({"aria-selected":"false","aria-expanded":"false"}),s.length&&n.length?e.oldTab.attr("tabIndex",-1):s.length&&this.tabs.filter(function(){return 0===V(this).attr("tabIndex")}).attr("tabIndex",-1),s.attr("aria-hidden","false"),e.newTab.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0})},_activate:function(t){var t=this._findActive(t);t[0]!==this.active[0]&&(t=(t=!t.length?this.active:t).find(".ui-tabs-anchor")[0],this._eventHandler({target:t,currentTarget:t,preventDefault:V.noop}))},_findActive:function(t){return!1===t?V():this.tabs.eq(t)},_getIndex:function(t){return t="string"==typeof t?this.anchors.index(this.anchors.filter("[href$='"+V.escapeSelector(t)+"']")):t},_destroy:function(){this.xhr&&this.xhr.abort(),this.tablist.removeAttr("role").off(this.eventNamespace),this.anchors.removeAttr("role tabIndex").removeUniqueId(),this.tabs.add(this.panels).each(function(){V.data(this,"ui-tabs-destroy")?V(this).remove():V(this).removeAttr("role tabIndex aria-live aria-busy aria-selected aria-labelledby aria-hidden aria-expanded")}),this.tabs.each(function(){var t=V(this),e=t.data("ui-tabs-aria-controls");e?t.attr("aria-controls",e).removeData("ui-tabs-aria-controls"):t.removeAttr("aria-controls")}),this.panels.show(),"content"!==this.options.heightStyle&&this.panels.css("height","")},enable:function(i){var t=this.options.disabled;!1!==t&&(t=void 0!==i&&(i=this._getIndex(i),Array.isArray(t)?V.map(t,function(t){return t!==i?t:null}):V.map(this.tabs,function(t,e){return e!==i?e:null})),this._setOptionDisabled(t))},disable:function(t){var e=this.options.disabled;if(!0!==e){if(void 0===t)e=!0;else{if(t=this._getIndex(t),-1!==V.inArray(t,e))return;e=Array.isArray(e)?V.merge([t],e).sort():[t]}this._setOptionDisabled(e)}},load:function(t,s){t=this._getIndex(t);function n(t,e){"abort"===e&&o.panels.stop(!1,!0),o._removeClass(i,"ui-tabs-loading"),a.removeAttr("aria-busy"),t===o.xhr&&delete o.xhr}var o=this,i=this.tabs.eq(t),t=i.find(".ui-tabs-anchor"),a=this._getPanelForTab(i),r={tab:i,panel:a};this._isLocal(t[0])||(this.xhr=V.ajax(this._ajaxSettings(t,s,r)),this.xhr&&"canceled"!==this.xhr.statusText&&(this._addClass(i,"ui-tabs-loading"),a.attr("aria-busy","true"),this.xhr.done(function(t,e,i){setTimeout(function(){a.html(t),o._trigger("load",s,r),n(i,e)},1)}).fail(function(t,e){setTimeout(function(){n(t,e)},1)})))},_ajaxSettings:function(t,i,s){var n=this;return{url:t.attr("href").replace(/#.*$/,""),beforeSend:function(t,e){return n._trigger("beforeLoad",i,V.extend({jqXHR:t,ajaxSettings:e},s))}}},_getPanelForTab:function(t){t=V(t).attr("aria-controls");return this.element.find(this._sanitizeSelector("#"+t))}}),!1!==V.uiBackCompat&&V.widget("ui.tabs",V.ui.tabs,{_processTabs:function(){this._superApply(arguments),this._addClass(this.tabs,"ui-tab")}});V.ui.tabs;V.widget("ui.tooltip",{version:"1.13.2",options:{classes:{"ui-tooltip":"ui-corner-all ui-widget-shadow"},content:function(){var t=V(this).attr("title");return V("<a>").text(t).html()},hide:!0,items:"[title]:not([disabled])",position:{my:"left top+15",at:"left bottom",collision:"flipfit flip"},show:!0,track:!1,close:null,open:null},_addDescribedBy:function(t,e){var i=(t.attr("aria-describedby")||"").split(/\s+/);i.push(e),t.data("ui-tooltip-id",e).attr("aria-describedby",String.prototype.trim.call(i.join(" ")))},_removeDescribedBy:function(t){var e=t.data("ui-tooltip-id"),i=(t.attr("aria-describedby")||"").split(/\s+/),e=V.inArray(e,i);-1!==e&&i.splice(e,1),t.removeData("ui-tooltip-id"),(i=String.prototype.trim.call(i.join(" ")))?t.attr("aria-describedby",i):t.removeAttr("aria-describedby")},_create:function(){this._on({mouseover:"open",focusin:"open"}),this.tooltips={},this.parents={},this.liveRegion=V("<div>").attr({role:"log","aria-live":"assertive","aria-relevant":"additions"}).appendTo(this.document[0].body),this._addClass(this.liveRegion,null,"ui-helper-hidden-accessible"),this.disabledTitles=V([])},_setOption:function(t,e){var i=this;this._super(t,e),"content"===t&&V.each(this.tooltips,function(t,e){i._updateContent(e.element)})},_setOptionDisabled:function(t){this[t?"_disable":"_enable"]()},_disable:function(){var s=this;V.each(this.tooltips,function(t,e){var i=V.Event("blur");i.target=i.currentTarget=e.element[0],s.close(i,!0)}),this.disabledTitles=this.disabledTitles.add(this.element.find(this.options.items).addBack().filter(function(){var t=V(this);if(t.is("[title]"))return t.data("ui-tooltip-title",t.attr("title")).removeAttr("title")}))},_enable:function(){this.disabledTitles.each(function(){var t=V(this);t.data("ui-tooltip-title")&&t.attr("title",t.data("ui-tooltip-title"))}),this.disabledTitles=V([])},open:function(t){var i=this,e=V(t?t.target:this.element).closest(this.options.items);e.length&&!e.data("ui-tooltip-id")&&(e.attr("title")&&e.data("ui-tooltip-title",e.attr("title")),e.data("ui-tooltip-open",!0),t&&"mouseover"===t.type&&e.parents().each(function(){var t,e=V(this);e.data("ui-tooltip-open")&&((t=V.Event("blur")).target=t.currentTarget=this,i.close(t,!0)),e.attr("title")&&(e.uniqueId(),i.parents[this.id]={element:this,title:e.attr("title")},e.attr("title",""))}),this._registerCloseHandlers(t,e),this._updateContent(e,t))},_updateContent:function(e,i){var t=this.options.content,s=this,n=i?i.type:null;if("string"==typeof t||t.nodeType||t.jquery)return this._open(i,e,t);(t=t.call(e[0],function(t){s._delay(function(){e.data("ui-tooltip-open")&&(i&&(i.type=n),this._open(i,e,t))})}))&&this._open(i,e,t)},_open:function(t,e,i){var s,n,o,a=V.extend({},this.options.position);function r(t){a.of=t,n.is(":hidden")||n.position(a)}i&&((s=this._find(e))?s.tooltip.find(".ui-tooltip-content").html(i):(e.is("[title]")&&(t&&"mouseover"===t.type?e.attr("title",""):e.removeAttr("title")),s=this._tooltip(e),n=s.tooltip,this._addDescribedBy(e,n.attr("id")),n.find(".ui-tooltip-content").html(i),this.liveRegion.children().hide(),(i=V("<div>").html(n.find(".ui-tooltip-content").html())).removeAttr("name").find("[name]").removeAttr("name"),i.removeAttr("id").find("[id]").removeAttr("id"),i.appendTo(this.liveRegion),this.options.track&&t&&/^mouse/.test(t.type)?(this._on(this.document,{mousemove:r}),r(t)):n.position(V.extend({of:e},this.options.position)),n.hide(),this._show(n,this.options.show),this.options.track&&this.options.show&&this.options.show.delay&&(o=this.delayedShow=setInterval(function(){n.is(":visible")&&(r(a.of),clearInterval(o))},13)),this._trigger("open",t,{tooltip:n})))},_registerCloseHandlers:function(t,e){var i={keyup:function(t){t.keyCode===V.ui.keyCode.ESCAPE&&((t=V.Event(t)).currentTarget=e[0],this.close(t,!0))}};e[0]!==this.element[0]&&(i.remove=function(){var t=this._find(e);t&&this._removeTooltip(t.tooltip)}),t&&"mouseover"!==t.type||(i.mouseleave="close"),t&&"focusin"!==t.type||(i.focusout="close"),this._on(!0,e,i)},close:function(t){var e,i=this,s=V(t?t.currentTarget:this.element),n=this._find(s);n?(e=n.tooltip,n.closing||(clearInterval(this.delayedShow),s.data("ui-tooltip-title")&&!s.attr("title")&&s.attr("title",s.data("ui-tooltip-title")),this._removeDescribedBy(s),n.hiding=!0,e.stop(!0),this._hide(e,this.options.hide,function(){i._removeTooltip(V(this))}),s.removeData("ui-tooltip-open"),this._off(s,"mouseleave focusout keyup"),s[0]!==this.element[0]&&this._off(s,"remove"),this._off(this.document,"mousemove"),t&&"mouseleave"===t.type&&V.each(this.parents,function(t,e){V(e.element).attr("title",e.title),delete i.parents[t]}),n.closing=!0,this._trigger("close",t,{tooltip:e}),n.hiding||(n.closing=!1))):s.removeData("ui-tooltip-open")},_tooltip:function(t){var e=V("<div>").attr("role","tooltip"),i=V("<div>").appendTo(e),s=e.uniqueId().attr("id");return this._addClass(i,"ui-tooltip-content"),this._addClass(e,"ui-tooltip","ui-widget ui-widget-content"),e.appendTo(this._appendTo(t)),this.tooltips[s]={element:t,tooltip:e}},_find:function(t){t=t.data("ui-tooltip-id");return t?this.tooltips[t]:null},_removeTooltip:function(t){clearInterval(this.delayedShow),t.remove(),delete this.tooltips[t.attr("id")]},_appendTo:function(t){t=t.closest(".ui-front, dialog");return t=!t.length?this.document[0].body:t},_destroy:function(){var s=this;V.each(this.tooltips,function(t,e){var i=V.Event("blur"),e=e.element;i.target=i.currentTarget=e[0],s.close(i,!0),V("#"+t).remove(),e.data("ui-tooltip-title")&&(e.attr("title")||e.attr("title",e.data("ui-tooltip-title")),e.removeData("ui-tooltip-title"))}),this.liveRegion.remove()}}),!1!==V.uiBackCompat&&V.widget("ui.tooltip",V.ui.tooltip,{options:{tooltipClass:null},_tooltip:function(){var t=this._superApply(arguments);return this.options.tooltipClass&&t.tooltip.addClass(this.options.tooltipClass),t}});V.ui.tooltip});
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b3d86d0760162..e1cc6adbe52ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -133,6 +133,41 @@
     <value>60</value>
   </property>
 
+  <property>
+    <description>
+      Resource manager dispatcher has a thread pool that prints EventQueue,
+      configure the corePoolSize of this thread pool,
+      the meaning of corePoolSize is the number of threads to keep in the pool.
+      the default value is 1.
+    </description>
+    <name>yarn.dispatcher.print-thread-pool.core-pool-size</name>
+    <value>1</value>
+  </property>
+
+  <property>
+    <description>
+      Resource manager dispatcher has a thread pool that prints EventQueue,
+      configure the maximumPoolSize of this thread pool,
+      the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
+      the default value is 5.
+    </description>
+    <name>yarn.dispatcher.print-thread-pool.maximum-pool-size</name>
+    <value>5</value>
+  </property>
+
+  <property>
+    <description>
+      Resource manager dispatcher has a thread pool that prints EventQueue,
+      configure the keepAliveTime of this thread pool,
+      The meaning of keepAliveTime is as follows
+      when the number of threads is greater than the core,
+      this is the maximum time that excess idle threads will wait for new tasks before terminating.
+      the default value is 10s.
+    </description>
+    <name>yarn.dispatcher.print-thread-pool.keep-alive-time</name>
+    <value>10s</value>
+  </property>
+
   <property>
     <description>The expiry interval for application master reporting.</description>
     <name>yarn.am.liveness-monitor.expiry-interval-ms</name>
@@ -1008,6 +1043,33 @@
     <value>10</value>
   </property>
 
+  <property>
+    <description>
+      This setting enables/disables timeline server v1 publisher to publish timeline events in batch.
+    </description>
+    <name>yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>
+      The size of timeline server v1 publisher sending events in one request.
+    </description>
+    <name>yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size</name>
+    <value>1000</value>
+  </property>
+
+  <property>
+    <description>
+      When enable batch publishing in timeline server v1, we must avoid that the
+      publisher waits for a batch to be filled up and hold events in buffer for long
+      time. So we add another thread which send event's in the buffer periodically.
+      This config sets the interval of the cyclical sending thread.
+    </description>
+    <name>yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds</name>
+    <value>60</value>
+  </property>
+
   <property>
     <description>Number of diagnostics/failure messages can be saved in RM for
     log aggregation. It also defines the number of diagnostics/failure
@@ -1050,6 +1112,18 @@
     <value>86400000</value>
   </property>
 
+  <property>
+    <description>
+      This configuration is used for
+      how often the tokens are scanned for expired tokens in milliseconds.
+      the background thread(delegation token remover thread)
+      will delete expired tokens after the configured time.
+      the default value is 1h.
+    </description>
+    <name>yarn.resourcemanager.delegation.token.remove-scan-interval</name>
+    <value>1h</value>
+  </property>
+
   <property>
     <description>
     RM DelegationTokenRenewer thread timeout
@@ -1265,8 +1339,27 @@
   </property>
 
   <property>
-    <description>Max number of OPPORTUNISTIC containers to queue at the
-      nodemanager.</description>
+    <description>
+      At the NM, the policy to determine whether to queue an
+      OPPORTUNISTIC container or not.
+      If set to BY_QUEUE_LEN, uses the queue capacity, as set by
+      yarn.nodemanager.opportunistic-containers-max-queue-length
+      to limit how many containers to accept/queue.
+      If set to BY_RESOURCES, limits the number of containers
+      accepted based on the resource capacity of the node.
+    </description>
+    <name>yarn.nodemanager.opportunistic-containers-queue-policy</name>
+    <value>BY_QUEUE_LEN</value>
+  </property>
+
+  <property>
+    <description>
+      Max number of OPPORTUNISTIC containers to queue at the
+      nodemanager (NM). If the value is 0 or negative,
+      NMs do not allow any OPPORTUNISTIC containers.
+      If the value is positive, the NM caps the number of OPPORTUNISTIC
+      containers that can be queued at the NM.
+    </description>
     <name>yarn.nodemanager.opportunistic-containers-max-queue-length</name>
     <value>0</value>
   </property>
@@ -1502,6 +1595,15 @@
     <value>600000</value>
   </property>
 
+  <property>
+    <description>Whether to clean up nodemanager logs when log aggregation is enabled. Setting to
+      false disables the cleanup nodemanager logging, and it causes disk full in the long run. Users
+      can set to false for test-only purpose.
+    </description>
+    <name>yarn.log-aggregation.enable-local-cleanup</name>
+    <value>true</value>
+  </property>
+
   <property>
     <description>Time in seconds to retain user logs. Only applicable if
     log aggregation is disabled
@@ -3385,15 +3487,44 @@
   <property>
     <description>
     When "yarn.node-labels.configuration-type" is configured with
-    "delegated-centralized", then periodically node labels are retrieved
-    from the node labels provider. This configuration is to define the
-    interval. If -1 is configured then node labels are retrieved from
-    provider only once for each node after it registers. Defaults to 30 mins.
+    "delegated-centralized", then node labels of all nodes
+    are updated by periodically retrieving node labels from the
+    provider. If -1 is configured then node labels are retrieved
+    from provider only once for each node after it registers.
+    Defaults to 30 mins.
     </description>
     <name>yarn.resourcemanager.node-labels.provider.fetch-interval-ms</name>
     <value>1800000</value>
   </property>
 
+  <property>
+    <description>
+    When "yarn.node-labels.configuration-type" is configured with
+    "delegated-centralized", then node labels of newly registered
+    nodes are updated by periodically retrieving node labels from
+    the provider. Defaults to 30 secs.
+    </description>
+    <name>yarn.resourcemanager.node-labels.provider.update-newly-registered-nodes-interval-ms</name>
+    <value>30000</value>
+  </property>
+
+  <property>
+    <description>
+      Overwrites default-node-label-expression only for the ApplicationMaster
+      container. It is disabled by default.
+    </description>
+    <name>yarn.resourcemanager.node-labels.am.default-node-label-expression</name>
+  </property>
+
+  <property>
+    <description>
+      Flag to indicate whether the AM can be allocated to non-exclusive nodes or not.
+      Default is false.
+    </description>
+    <name>yarn.resourcemanager.node-labels.am.allow-non-exclusive-allocation</name>
+    <value>false</value>
+  </property>
+
   <!-- Distributed Node Attributes Configuration -->
   <property>
     <description>
@@ -3549,6 +3680,16 @@
     <name>yarn.federation.enabled</name>
     <value>false</value>
   </property>
+  <property>
+    <description>
+      Initial delay for federation state-store heartbeat service. Value is followed by a unit
+      specifier: ns, us, ms, s, m, h, d for nanoseconds, microseconds, milliseconds, seconds,
+      minutes, hours, days respectively. Values should provide units,
+      but seconds are assumed
+    </description>
+    <name>yarn.federation.state-store.heartbeat.initial-delay</name>
+    <value>30s</value>
+  </property>
   <property>
     <description>
       Machine list file to be loaded by the FederationSubCluster Resolver
@@ -3586,6 +3727,26 @@
     <value>yarnfederation/</value>
   </property>
 
+  <property>
+    <description>
+      The number of retries to clear the app in the FederationStateStore,
+      the default value is 1, that is, after the app fails to clean up, it will retry the cleanup again.
+    </description>
+    <name>yarn.federation.state-store.clean-up-retry-count</name>
+    <value>1</value>
+  </property>
+
+  <property>
+    <description>
+      Clear the sleep time of App retry in FederationStateStore.
+      When the app fails to clean up,
+      it will sleep for a period of time and then try to clean up.
+      The default value is 1s.
+    </description>
+    <name>yarn.federation.state-store.clean-up-retry-sleep-time</name>
+    <value>1s</value>
+  </property>
+
   <!-- Other Configuration -->
 
   <property>
@@ -4800,4 +4961,128 @@
     <name>yarn.resourcemanager.enable-node-untracked-without-include-path</name>
     <value>false</value>
   </property>
+
+  <property>
+    <name>yarn.scheduler.app-placement-allocator.class</name>
+    <value></value>
+    <description>
+      In the absence of APPLICATION_PLACEMENT_TYPE_CLASS from the RM
+      application scheduling environments, the value of this config
+      is used to determine the default implementation of AppPlacementAllocator.
+      If APPLICATION_PLACEMENT_TYPE_CLASS is absent from the application
+      scheduling env and this config also has no value present, then
+      default implementation LocalityAppPlacementAllocator is used.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.keytab.file</name>
+    <value></value>
+    <description>
+       The keytab file used by router to login as its
+       service principal. The principal name is configured with
+       dfs.federation.router.kerberos.principal.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.kerberos.principal</name>
+    <value></value>
+    <description>
+      The Router service principal. This is typically set to
+      router/_HOST@REALM.TLD. Each Router will substitute _HOST with its
+      own fully qualified hostname at startup. The _HOST placeholder
+      allows using the same configuration setting on both Router setup.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.kerberos.principal.hostname</name>
+    <value></value>
+    <description>
+      Optional.
+      The hostname for the Router containing this
+      configuration file. Will be different for each machine.
+      Defaults to current hostname.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.webapp.appsinfo-enabled</name>
+    <value>false</value>
+    <description>
+      This configuration is used to enable the cache of AppsInfo.
+      If it is set to true, the cache is enabled.
+      If it is set to false, the cache is not enabled.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.webapp.appsinfo-cached-count</name>
+    <value>100</value>
+    <description>
+      When yarn.router.appsinfo-enabled is set to true,
+      the number of cached appsInfo.
+      Default is 100
+    </description>
+  </property>
+  
+  <property>
+    <name>yarn.router.webapp.cross-origin.enabled</name>
+    <value>false</value>
+    <description>
+      Flag to enable cross-origin (CORS) support for Yarn Router.
+      For Yarn Router, also add
+      org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the
+      configuration hadoop.http.filter.initializers in core-site.xml.
+    </description>
+  </property>
+  
+  <property>
+    <name>yarn.federation.state-store.max-applications</name>
+    <value>1000</value>
+    <description>
+      Yarn federation state-store supports querying the maximum number of apps.
+      Default is 1000.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.interceptor.user-thread-pool.minimum-pool-size</name>
+    <value>5</value>
+    <description>
+      This configurable is used to set the corePoolSize(minimumPoolSize)
+      of the thread pool of the interceptor.
+      Default is 5.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.interceptor.user-thread-pool.maximum-pool-size</name>
+    <value>5</value>
+    <description>
+      This configuration is used to set the default value of maximumPoolSize
+      of the thread pool of the interceptor.
+      Default is 5.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.interceptor.user-thread-pool.keep-alive-time</name>
+    <value>0s</value>
+    <description>
+      This configurable is used to set the keepAliveTime of the thread pool of the interceptor.
+      Default is 0s.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.router.submit.interval.time</name>
+    <value>10ms</value>
+    <description>
+      The interval Time between calling different subCluster requests.
+      Default is 10ms.
+    </description>
+  </property>
+
 </configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
index f59f00e6e853f..11d54d6e6aeb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/MockApps.java
@@ -20,11 +20,11 @@
 
 import java.util.Iterator;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
+
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
-
 /**
  * Utilities to generate fake test apps
  */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
index 279a37b7d19b2..94f1e520e4f31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
@@ -24,8 +24,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
@@ -35,12 +37,12 @@
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
@@ -70,8 +72,9 @@
 import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /*
  * Test that the container launcher rpc times out properly. This is used
@@ -86,7 +89,7 @@ public class TestContainerLaunchRPC {
       .getRecordFactory(null);
 
   @Test
-  public void testHadoopProtoRPCTimeout() throws Exception {
+  void testHadoopProtoRPCTimeout() throws Exception {
     testRPCTimeout(HadoopYarnProtoRPC.class.getName());
   }
 
@@ -136,16 +139,15 @@ private void testRPCTimeout(String rpcClass) throws Exception {
         proxy.startContainers(allRequests);
       } catch (Exception e) {
         LOG.info(StringUtils.stringifyException(e));
-        Assert.assertEquals("Error, exception is not: "
-            + SocketTimeoutException.class.getName(),
-            SocketTimeoutException.class.getName(), e.getClass().getName());
+        assertEquals(SocketTimeoutException.class.getName(), e.getClass().getName(),
+            "Error, exception is not: " + SocketTimeoutException.class.getName());
         return;
       }
     } finally {
       server.stop();
     }
 
-    Assert.fail("timeout exception should have occurred!");
+    fail("timeout exception should have occurred!");
   }
 
   public static Token newContainerToken(NodeId nodeId, byte[] password,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
index 6b8e537a4c559..26acfd7bad87b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
@@ -18,14 +18,15 @@
 
 package org.apache.hadoop.yarn;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.log4j.Logger;
 import org.apache.log4j.PatternLayout;
-import org.junit.Test;
 
 public class TestContainerLogAppender {
 
   @Test
-  public void testAppendInClose() throws Exception {
+  void testAppendInClose() throws Exception {
     final ContainerLogAppender claAppender = new ContainerLogAppender();
     claAppender.setName("testCLA");
     claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
index c3dac91467237..e615036d8139f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
@@ -18,8 +18,16 @@
 
 package org.apache.hadoop.yarn;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
@@ -59,14 +67,9 @@
 import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.junit.Assert;
-import org.junit.Test;
 
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.List;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /*
  * Test that the container resource increase rpc times out properly.
@@ -78,7 +81,7 @@ public class TestContainerResourceIncreaseRPC {
       TestContainerResourceIncreaseRPC.class);
 
   @Test
-  public void testHadoopProtoRPCTimeout() throws Exception {
+  void testHadoopProtoRPCTimeout() throws Exception {
     testRPCTimeout(HadoopYarnProtoRPC.class.getName());
   }
 
@@ -122,15 +125,14 @@ private void testRPCTimeout(String rpcClass) throws Exception {
         proxy.updateContainer(request);
       } catch (Exception e) {
         LOG.info(StringUtils.stringifyException(e));
-        Assert.assertEquals("Error, exception is not: "
-            + SocketTimeoutException.class.getName(),
-            SocketTimeoutException.class.getName(), e.getClass().getName());
+        assertEquals(SocketTimeoutException.class.getName(), e.getClass().getName(),
+            "Error, exception is not: " + SocketTimeoutException.class.getName());
         return;
       }
     } finally {
       server.stop();
     }
-    Assert.fail("timeout exception should have occurred!");
+    fail("timeout exception should have occurred!");
   }
 
   public static Token newContainerToken(NodeId nodeId, byte[] password,
@@ -157,11 +159,9 @@ public StartContainersResponse startContainers(
     }
 
     @Override
-    public StopContainersResponse
-    stopContainers(StopContainersRequest requests) throws YarnException,
-        IOException {
-      Exception e = new Exception("Dummy function", new Exception(
-          "Dummy function cause"));
+    public StopContainersResponse stopContainers(StopContainersRequest requests)
+        throws YarnException, IOException {
+      Exception e = new Exception("Dummy function", new Exception("Dummy function cause"));
       throw new YarnException(e);
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPCFactories.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPCFactories.java
index 765b165e90fa8..6c283c6674421 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPCFactories.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPCFactories.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.Server;
@@ -37,16 +37,16 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl;
 import org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestRPCFactories {
-  
-  
-  
+
+
   @Test
-  public void test() {
+  void test() {
     testPbServerFactory();
-    
+
     testPbClientFactory();
   }
   
@@ -64,7 +64,7 @@ private void testPbServerFactory() {
       server.start();
     } catch (YarnRuntimeException e) {
       e.printStackTrace();
-      Assert.fail("Failed to create server");
+      fail("Failed to create server");
     } finally {
       if (server != null) {
         server.stop();
@@ -92,12 +92,12 @@ private void testPbClientFactory() {
         amrmClient = (ApplicationMasterProtocol) RpcClientFactoryPBImpl.get().getClient(ApplicationMasterProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
       } catch (YarnRuntimeException e) {
         e.printStackTrace();
-        Assert.fail("Failed to create client");
+        fail("Failed to create client");
       }
       
     } catch (YarnRuntimeException e) {
       e.printStackTrace();
-      Assert.fail("Failed to create server");
+      fail("Failed to create server");
     } finally {
       if (server != null) {
         server.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java
index 9492988c946fc..8be77ba6288c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java
@@ -18,39 +18,41 @@
 
 package org.apache.hadoop.yarn;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl;
-import org.junit.Test;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestRecordFactory {
-  
+
   @Test
-  public void testPbRecordFactory() {
+  void testPbRecordFactory() {
     RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
-    
+
     try {
       AllocateResponse response =
           pbRecordFactory.newRecordInstance(AllocateResponse.class);
-      Assert.assertEquals(AllocateResponsePBImpl.class, response.getClass());
+      assertEquals(AllocateResponsePBImpl.class, response.getClass());
     } catch (YarnRuntimeException e) {
       e.printStackTrace();
-      Assert.fail("Failed to crete record");
+      fail("Failed to crete record");
     }
-    
+
     try {
       AllocateRequest response =
           pbRecordFactory.newRecordInstance(AllocateRequest.class);
-      Assert.assertEquals(AllocateRequestPBImpl.class, response.getClass());
+      assertEquals(AllocateRequestPBImpl.class, response.getClass());
     } catch (YarnRuntimeException e) {
       e.printStackTrace();
-      Assert.fail("Failed to crete record");
+      fail("Failed to crete record");
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRpcFactoryProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRpcFactoryProvider.java
index 005a71bbd97c5..a8cc98e4e453e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRpcFactoryProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRpcFactoryProvider.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -28,21 +28,23 @@
 import org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl;
 import org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl;
 import org.apache.hadoop.yarn.factory.providers.RpcFactoryProvider;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestRpcFactoryProvider {
 
   @Test
-  public void testFactoryProvider() {
+  void testFactoryProvider() {
     Configuration conf = new Configuration();
     RpcClientFactory clientFactory = null;
     RpcServerFactory serverFactory = null;
-    
-    
+
+
     clientFactory = RpcFactoryProvider.getClientFactory(conf);
     serverFactory = RpcFactoryProvider.getServerFactory(conf);
-    Assert.assertEquals(RpcClientFactoryPBImpl.class, clientFactory.getClass());
-    Assert.assertEquals(RpcServerFactoryPBImpl.class, serverFactory.getClass());
+    assertEquals(RpcClientFactoryPBImpl.class, clientFactory.getClass());
+    assertEquals(RpcServerFactoryPBImpl.class, serverFactory.getClass());
 
     conf.set(YarnConfiguration.IPC_CLIENT_FACTORY_CLASS, "unknown");
     conf.set(YarnConfiguration.IPC_SERVER_FACTORY_CLASS, "unknown");
@@ -50,28 +52,30 @@ public void testFactoryProvider() {
 
     try {
       clientFactory = RpcFactoryProvider.getClientFactory(conf);
-      Assert.fail("Expected an exception - unknown serializer");
+      fail("Expected an exception - unknown serializer");
     } catch (YarnRuntimeException e) {
     }
     try {
       serverFactory = RpcFactoryProvider.getServerFactory(conf);
-      Assert.fail("Expected an exception - unknown serializer");
+      fail("Expected an exception - unknown serializer");
     } catch (YarnRuntimeException e) {
     }
-    
+
     conf = new Configuration();
     conf.set(YarnConfiguration.IPC_CLIENT_FACTORY_CLASS, "NonExistantClass");
     conf.set(YarnConfiguration.IPC_SERVER_FACTORY_CLASS, RpcServerFactoryPBImpl.class.getName());
-    
+
     try {
       clientFactory = RpcFactoryProvider.getClientFactory(conf);
-      Assert.fail("Expected an exception - unknown class");
+      fail("Expected an exception - unknown class");
     } catch (YarnRuntimeException e) {
     }
     try {
       serverFactory = RpcFactoryProvider.getServerFactory(conf);
     } catch (YarnRuntimeException e) {
-      Assert.fail("Error while loading factory using reflection: [" + RpcServerFactoryPBImpl.class.getName() + "]");
+      fail(
+          "Error while loading factory using reflection: [" + RpcServerFactoryPBImpl.class.getName()
+              + "]");
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestYarnUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestYarnUncaughtExceptionHandler.java
index 05bcdb982c431..e0201cfcd1557 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestYarnUncaughtExceptionHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestYarnUncaughtExceptionHandler.java
@@ -18,18 +18,20 @@
 
 package org.apache.hadoop.yarn;
 
-import static org.junit.Assert.assertSame;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 
 public class TestYarnUncaughtExceptionHandler {
 
   private static final YarnUncaughtExceptionHandler exHandler =
         new YarnUncaughtExceptionHandler();
+
   /**
    * Throw {@code YarnRuntimeException} inside thread and
    * check {@code YarnUncaughtExceptionHandler} instance
@@ -37,7 +39,7 @@ public class TestYarnUncaughtExceptionHandler {
    * @throws InterruptedException
    */
   @Test
-  public void testUncaughtExceptionHandlerWithRuntimeException()
+  void testUncaughtExceptionHandlerWithRuntimeException()
       throws InterruptedException {
     final YarnUncaughtExceptionHandler spyYarnHandler = spy(exHandler);
     final YarnRuntimeException yarnException = new YarnRuntimeException(
@@ -67,7 +69,7 @@ public void run() {
    * @throws InterruptedException
    */
   @Test
-  public void testUncaughtExceptionHandlerWithError()
+  void testUncaughtExceptionHandlerWithError()
       throws InterruptedException {
     ExitUtil.disableSystemExit();
     final YarnUncaughtExceptionHandler spyErrorHandler = spy(exHandler);
@@ -96,7 +98,7 @@ public void run() {
    * @throws InterruptedException
    */
   @Test
-  public void testUncaughtExceptionHandlerWithOutOfMemoryError()
+  void testUncaughtExceptionHandlerWithOutOfMemoryError()
       throws InterruptedException {
     ExitUtil.disableSystemHalt();
     final YarnUncaughtExceptionHandler spyOomHandler = spy(exHandler);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
index 5697923c9974d..e9ac044affc93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
@@ -17,26 +17,38 @@
  */
 package org.apache.hadoop.yarn.api;
 
-import org.apache.commons.lang3.Range;
-import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.util.Sets;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import java.lang.reflect.Array;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.nio.ByteBuffer;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
-
-import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.lang.reflect.*;
-import java.nio.ByteBuffer;
-import java.util.*;
+import org.apache.commons.lang3.Range;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
 
 import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints
-    .PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
 import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Generic helper class to validate protocol records.
@@ -80,6 +92,8 @@ private static Object genTypeValue(Type type) {
           'a' + rand.nextInt(26));
     } else if (type.equals(Float.class)) {
       return rand.nextFloat();
+    } else if (type.equals(Text.class)) {
+      return new Text('a' + String.valueOf(rand.nextInt(1000000)));
     } else if (type instanceof Class) {
       Class clazz = (Class)type;
       if (clazz.isArray()) {
@@ -167,7 +181,7 @@ protected static Object generateByNewInstance(Class clazz) throws Exception {
           " does not have newInstance method");
     }
     Object [] args = new Object[paramTypes.length];
-    for (int i=0;i<args.length;i++) {
+    for (int i = 0; i < args.length; i++) {
       args[i] = genTypeValue(paramTypes[i]);
     }
     ret = newInstance.invoke(null, args);
@@ -220,7 +234,7 @@ private <R> Map<String, GetSetPair> getGetSetPairs(Class<R> recordClass)
             p.getMethod = m;
             ret.put(propertyName, p);
           } else {
-            Assert.fail("Multiple get method with same name: " + recordClass
+            fail("Multiple get method with same name: " + recordClass
                 + p.propertyName);
           }
         }
@@ -281,18 +295,17 @@ protected  <R, P> void validatePBImplRecord(Class<R> recordClass,
       gsp.setMethod.invoke(origRecord, gsp.testValue);
     }
     Object ret = getProto.invoke(origRecord);
-    Assert.assertNotNull(recordClass.getName() + "#getProto returns null", ret);
+    assertNotNull(ret, recordClass.getName() + "#getProto returns null");
     if (!(protoClass.isAssignableFrom(ret.getClass()))) {
-      Assert.fail("Illegal getProto method return type: " + ret.getClass());
+      fail("Illegal getProto method return type: " + ret.getClass());
     }
     R deserRecord = pbConstructor.newInstance(ret);
-    Assert.assertEquals("whole " + recordClass + " records should be equal",
-        origRecord, deserRecord);
+    assertEquals(origRecord, deserRecord, "whole " + recordClass + " records should be equal");
     for (GetSetPair gsp : getSetPairs.values()) {
       Object origValue = gsp.getMethod.invoke(origRecord);
       Object deserValue = gsp.getMethod.invoke(deserRecord);
-      Assert.assertEquals("property " + recordClass.getName() + "#"
-          + gsp.propertyName + " should be equal", origValue, deserValue);
+      assertEquals(origValue, deserValue, "property " + recordClass.getName() + "#"
+          + gsp.propertyName + " should be equal");
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationAttemptId.java
index f54ed78ac1ea4..b2ef3ef1a4498 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationAttemptId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationAttemptId.java
@@ -19,41 +19,45 @@
 
 package org.apache.hadoop.yarn.api;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestApplicationAttemptId {
 
   @Test
-  public void testApplicationAttemptId() {
+  void testApplicationAttemptId() {
     ApplicationAttemptId a1 = createAppAttemptId(10l, 1, 1);
     ApplicationAttemptId a2 = createAppAttemptId(10l, 1, 2);
     ApplicationAttemptId a3 = createAppAttemptId(10l, 2, 1);
     ApplicationAttemptId a4 = createAppAttemptId(8l, 1, 4);
     ApplicationAttemptId a5 = createAppAttemptId(10l, 1, 1);
-    
-    Assert.assertTrue(a1.equals(a5));
-    Assert.assertFalse(a1.equals(a2));
-    Assert.assertFalse(a1.equals(a3));
-    Assert.assertFalse(a1.equals(a4));
-    
-    Assert.assertTrue(a1.compareTo(a5) == 0);
-    Assert.assertTrue(a1.compareTo(a2) < 0);
-    Assert.assertTrue(a1.compareTo(a3) < 0);
-    Assert.assertTrue(a1.compareTo(a4) > 0);
-    
-    Assert.assertTrue(a1.hashCode() == a5.hashCode());
-    Assert.assertFalse(a1.hashCode() == a2.hashCode());
-    Assert.assertFalse(a1.hashCode() == a3.hashCode());
-    Assert.assertFalse(a1.hashCode() == a4.hashCode());
-    
+
+    assertEquals(a1, a5);
+    assertNotEquals(a1, a2);
+    assertNotEquals(a1, a3);
+    assertNotEquals(a1, a4);
+
+    assertTrue(a1.compareTo(a5) == 0);
+    assertTrue(a1.compareTo(a2) < 0);
+    assertTrue(a1.compareTo(a3) < 0);
+    assertTrue(a1.compareTo(a4) > 0);
+
+    assertTrue(a1.hashCode() == a5.hashCode());
+    assertFalse(a1.hashCode() == a2.hashCode());
+    assertFalse(a1.hashCode() == a3.hashCode());
+    assertFalse(a1.hashCode() == a4.hashCode());
+
     long ts = System.currentTimeMillis();
     ApplicationAttemptId a6 = createAppAttemptId(ts, 543627, 33492611);
-    Assert.assertEquals("appattempt_10_0001_000001", a1.toString());
-    Assert.assertEquals("appattempt_" + ts + "_543627_33492611", a6.toString());
+    assertEquals("appattempt_10_0001_000001", a1.toString());
+    assertEquals("appattempt_" + ts + "_543627_33492611", a6.toString());
   }
 
   private ApplicationAttemptId createAppAttemptId(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationId.java
index ea25a64c95dbf..d084468b3ac38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicationId.java
@@ -18,36 +18,40 @@
 
 package org.apache.hadoop.yarn.api;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestApplicationId {
 
   @Test
-  public void testApplicationId() {
+  void testApplicationId() {
     ApplicationId a1 = ApplicationId.newInstance(10l, 1);
     ApplicationId a2 = ApplicationId.newInstance(10l, 2);
     ApplicationId a3 = ApplicationId.newInstance(10l, 1);
     ApplicationId a4 = ApplicationId.newInstance(8l, 3);
 
-    Assert.assertFalse(a1.equals(a2));
-    Assert.assertFalse(a1.equals(a4));
-    Assert.assertTrue(a1.equals(a3));
+    assertNotEquals(a1, a2);
+    assertNotEquals(a1, a4);
+    assertEquals(a1, a3);
+
+    assertTrue(a1.compareTo(a2) < 0);
+    assertTrue(a1.compareTo(a3) == 0);
+    assertTrue(a1.compareTo(a4) > 0);
 
-    Assert.assertTrue(a1.compareTo(a2) < 0);
-    Assert.assertTrue(a1.compareTo(a3) == 0);
-    Assert.assertTrue(a1.compareTo(a4) > 0);
+    assertTrue(a1.hashCode() == a3.hashCode());
+    assertFalse(a1.hashCode() == a2.hashCode());
+    assertFalse(a2.hashCode() == a4.hashCode());
 
-    Assert.assertTrue(a1.hashCode() == a3.hashCode());
-    Assert.assertFalse(a1.hashCode() == a2.hashCode());
-    Assert.assertFalse(a2.hashCode() == a4.hashCode());
-    
     long ts = System.currentTimeMillis();
     ApplicationId a5 = ApplicationId.newInstance(ts, 45436343);
-    Assert.assertEquals("application_10_0001", a1.toString());
-    Assert.assertEquals("application_" + ts + "_45436343", a5.toString());
+    assertEquals("application_10_0001", a1.toString());
+    assertEquals("application_" + ts + "_45436343", a5.toString());
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
index ea39a4ccdba09..c0904ec09faef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.api;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -25,13 +27,15 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestApplicatonReport {
 
   @Test
-  public void testApplicationReport() {
+  void testApplicationReport() {
     long timestamp = System.currentTimeMillis();
     ApplicationReport appReport1 =
         createApplicationReport(1, 1, timestamp);
@@ -39,15 +43,15 @@ public void testApplicationReport() {
         createApplicationReport(1, 1, timestamp);
     ApplicationReport appReport3 =
         createApplicationReport(1, 1, timestamp);
-    Assert.assertEquals(appReport1, appReport2);
-    Assert.assertEquals(appReport2, appReport3);
+    assertEquals(appReport1, appReport2);
+    assertEquals(appReport2, appReport3);
     appReport1.setApplicationId(null);
-    Assert.assertNull(appReport1.getApplicationId());
-    Assert.assertNotSame(appReport1, appReport2);
+    assertNull(appReport1.getApplicationId());
+    assertNotSame(appReport1, appReport2);
     appReport2.setCurrentApplicationAttemptId(null);
-    Assert.assertNull(appReport2.getCurrentApplicationAttemptId());
-    Assert.assertNotSame(appReport2, appReport3);
-    Assert.assertNull(appReport1.getAMRMToken());
+    assertNull(appReport2.getCurrentApplicationAttemptId());
+    assertNotSame(appReport2, appReport3);
+    assertNull(appReport1.getAMRMToken());
   }
 
   protected static ApplicationReport createApplicationReport(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
index 1643301072b81..73d585d8f688d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerId.java
@@ -19,57 +19,61 @@
 
 package org.apache.hadoop.yarn.api;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestContainerId {
 
   @Test
-  public void testContainerId() {
+  void testContainerId() {
     ContainerId c1 = newContainerId(1, 1, 10l, 1);
     ContainerId c2 = newContainerId(1, 1, 10l, 2);
     ContainerId c3 = newContainerId(1, 1, 10l, 1);
     ContainerId c4 = newContainerId(1, 3, 10l, 1);
     ContainerId c5 = newContainerId(1, 3, 8l, 1);
 
-    Assert.assertTrue(c1.equals(c3));
-    Assert.assertFalse(c1.equals(c2));
-    Assert.assertFalse(c1.equals(c4));
-    Assert.assertFalse(c1.equals(c5));
+    assertEquals(c1, c3);
+    assertNotEquals(c1, c2);
+    assertNotEquals(c1, c4);
+    assertNotEquals(c1, c5);
+
+    assertTrue(c1.compareTo(c3) == 0);
+    assertTrue(c1.compareTo(c2) < 0);
+    assertTrue(c1.compareTo(c4) < 0);
+    assertTrue(c1.compareTo(c5) > 0);
 
-    Assert.assertTrue(c1.compareTo(c3) == 0);
-    Assert.assertTrue(c1.compareTo(c2) < 0);
-    Assert.assertTrue(c1.compareTo(c4) < 0);
-    Assert.assertTrue(c1.compareTo(c5) > 0);
+    assertTrue(c1.hashCode() == c3.hashCode());
+    assertFalse(c1.hashCode() == c2.hashCode());
+    assertFalse(c1.hashCode() == c4.hashCode());
+    assertFalse(c1.hashCode() == c5.hashCode());
 
-    Assert.assertTrue(c1.hashCode() == c3.hashCode());
-    Assert.assertFalse(c1.hashCode() == c2.hashCode());
-    Assert.assertFalse(c1.hashCode() == c4.hashCode());
-    Assert.assertFalse(c1.hashCode() == c5.hashCode());
-    
     long ts = System.currentTimeMillis();
     ContainerId c6 = newContainerId(36473, 4365472, ts, 25645811);
-    Assert.assertEquals("container_10_0001_01_000001", c1.toString());
-    Assert.assertEquals(25645811, 0xffffffffffL & c6.getContainerId());
-    Assert.assertEquals(0, c6.getContainerId() >> 40);
-    Assert.assertEquals("container_" + ts + "_36473_4365472_25645811",
+    assertEquals("container_10_0001_01_000001", c1.toString());
+    assertEquals(25645811, 0xffffffffffL & c6.getContainerId());
+    assertEquals(0, c6.getContainerId() >> 40);
+    assertEquals("container_" + ts + "_36473_4365472_25645811",
         c6.toString());
 
     ContainerId c7 = newContainerId(36473, 4365472, ts, 4298334883325L);
-    Assert.assertEquals(999799999997L, 0xffffffffffL & c7.getContainerId());
-    Assert.assertEquals(3, c7.getContainerId() >> 40);
-    Assert.assertEquals(
+    assertEquals(999799999997L, 0xffffffffffL & c7.getContainerId());
+    assertEquals(3, c7.getContainerId() >> 40);
+    assertEquals(
         "container_e03_" + ts + "_36473_4365472_999799999997",
         c7.toString());
 
     ContainerId c8 = newContainerId(36473, 4365472, ts, 844424930131965L);
-    Assert.assertEquals(1099511627773L, 0xffffffffffL & c8.getContainerId());
-    Assert.assertEquals(767, c8.getContainerId() >> 40);
-    Assert.assertEquals(
+    assertEquals(1099511627773L, 0xffffffffffL & c8.getContainerId());
+    assertEquals(767, c8.getContainerId() >> 40);
+    assertEquals(
         "container_e767_" + ts + "_36473_4365472_1099511627773",
         c8.toString());
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
index c46c2bc0a9b5e..b91df652693da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestGetApplicationsRequest.java
@@ -21,93 +21,86 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.lang3.Range;
 import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestGetApplicationsRequest {
 
   @Test
-  public void testGetApplicationsRequest(){
+  void testGetApplicationsRequest() {
     GetApplicationsRequest request = GetApplicationsRequest.newInstance();
-    
-    EnumSet<YarnApplicationState> appStates = 
-      EnumSet.of(YarnApplicationState.ACCEPTED);
+
+    EnumSet<YarnApplicationState> appStates =
+        EnumSet.of(YarnApplicationState.ACCEPTED);
     request.setApplicationStates(appStates);
-    
+
     Set<String> tags = new HashSet<String>();
     tags.add("tag1");
     request.setApplicationTags(tags);
-    
+
     Set<String> types = new HashSet<String>();
     types.add("type1");
     request.setApplicationTypes(types);
-    
+
     long startBegin = System.currentTimeMillis();
     long startEnd = System.currentTimeMillis() + 1;
     request.setStartRange(startBegin, startEnd);
     long finishBegin = System.currentTimeMillis() + 2;
     long finishEnd = System.currentTimeMillis() + 3;
     request.setFinishRange(finishBegin, finishEnd);
-    
+
     long limit = 100L;
     request.setLimit(limit);
-    
+
     Set<String> queues = new HashSet<String>();
     queues.add("queue1");
     request.setQueues(queues);
-    
-    
+
+
     Set<String> users = new HashSet<String>();
     users.add("user1");
     request.setUsers(users);
-    
+
     ApplicationsRequestScope scope = ApplicationsRequestScope.ALL;
     request.setScope(scope);
-    
+
     GetApplicationsRequest requestFromProto = new GetApplicationsRequestPBImpl(
-        ((GetApplicationsRequestPBImpl)request).getProto());
-    
+        ((GetApplicationsRequestPBImpl) request).getProto());
+
     // verify the whole record equals with original record
-    Assert.assertEquals(requestFromProto, request);
+    assertEquals(requestFromProto, request);
 
     // verify all properties are the same as original request
-    Assert.assertEquals(
-        "ApplicationStates from proto is not the same with original request",
-        requestFromProto.getApplicationStates(), appStates);
-    
-    Assert.assertEquals(
-        "ApplicationTags from proto is not the same with original request",
-        requestFromProto.getApplicationTags(), tags);
-    
-    Assert.assertEquals(
-        "ApplicationTypes from proto is not the same with original request",
-        requestFromProto.getApplicationTypes(), types);
-    
-    Assert.assertEquals(
-        "StartRange from proto is not the same with original request",
-        requestFromProto.getStartRange(), Range.between(startBegin, startEnd));
-    
-    Assert.assertEquals(
-        "FinishRange from proto is not the same with original request",
-        requestFromProto.getFinishRange(),
-          Range.between(finishBegin, finishEnd));
-    
-    Assert.assertEquals(
-        "Limit from proto is not the same with original request",
-        requestFromProto.getLimit(), limit);
-    
-    Assert.assertEquals(
-        "Queues from proto is not the same with original request",
-        requestFromProto.getQueues(), queues);
-    
-    Assert.assertEquals(
-        "Users from proto is not the same with original request",
-        requestFromProto.getUsers(), users);
+    assertEquals(requestFromProto.getApplicationStates(), appStates,
+        "ApplicationStates from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getApplicationTags(), tags,
+        "ApplicationTags from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getApplicationTypes(), types,
+        "ApplicationTypes from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getStartRange(), Range.between(startBegin, startEnd),
+        "StartRange from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getFinishRange(), Range.between(finishBegin, finishEnd),
+        "FinishRange from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getLimit(), limit,
+        "Limit from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getQueues(), queues,
+        "Queues from proto is not the same with original request");
+
+    assertEquals(requestFromProto.getUsers(), users,
+        "Users from proto is not the same with original request");
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestNodeId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestNodeId.java
index 32d31a30b021e..2a4092d915a36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestNodeId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestNodeId.java
@@ -18,32 +18,36 @@
 
 package org.apache.hadoop.yarn.api;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestNodeId {
   @Test
-  public void testNodeId() {
+  void testNodeId() {
     NodeId nodeId1 = NodeId.newInstance("10.18.52.124", 8041);
     NodeId nodeId2 = NodeId.newInstance("10.18.52.125", 8038);
     NodeId nodeId3 = NodeId.newInstance("10.18.52.124", 8041);
     NodeId nodeId4 = NodeId.newInstance("10.18.52.124", 8039);
 
-    Assert.assertTrue(nodeId1.equals(nodeId3));
-    Assert.assertFalse(nodeId1.equals(nodeId2));
-    Assert.assertFalse(nodeId3.equals(nodeId4));
+    assertEquals(nodeId1, nodeId3);
+    assertNotEquals(nodeId1, nodeId2);
+    assertNotEquals(nodeId3, nodeId4);
 
-    Assert.assertTrue(nodeId1.compareTo(nodeId3) == 0);
-    Assert.assertTrue(nodeId1.compareTo(nodeId2) < 0);
-    Assert.assertTrue(nodeId3.compareTo(nodeId4) > 0);
+    assertTrue(nodeId1.compareTo(nodeId3) == 0);
+    assertTrue(nodeId1.compareTo(nodeId2) < 0);
+    assertTrue(nodeId3.compareTo(nodeId4) > 0);
 
-    Assert.assertTrue(nodeId1.hashCode() == nodeId3.hashCode());
-    Assert.assertFalse(nodeId1.hashCode() == nodeId2.hashCode());
-    Assert.assertFalse(nodeId3.hashCode() == nodeId4.hashCode());
+    assertTrue(nodeId1.hashCode() == nodeId3.hashCode());
+    assertFalse(nodeId1.hashCode() == nodeId2.hashCode());
+    assertFalse(nodeId3.hashCode() == nodeId4.hashCode());
 
-    Assert.assertEquals("10.18.52.124:8041", nodeId1.toString());
+    assertEquals("10.18.52.124:8041", nodeId1.toString());
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index ad3725278a1b9..3bc76cbfe0518 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -16,9 +16,15 @@
  * limitations under the License.
  */
 package org.apache.hadoop.yarn.api;
+
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.lang3.Range;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
 import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
@@ -127,13 +133,14 @@
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LogAggregationContext;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
-import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
 import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -184,11 +191,12 @@
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerReportPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerRetryContextPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.EnhancedHeadroomPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ExecutionTypeRequestPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeInfoPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
@@ -199,6 +207,7 @@
 import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionResourceRequestPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.QueueConfigurationsPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.QueueInfoPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.QueueUserACLInfoPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl;
@@ -229,8 +238,8 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
@@ -242,6 +251,7 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionMessageProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionResourceRequestProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
@@ -370,19 +380,15 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl;
 import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl;
 import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * Test class for YARN API protocol records.
  */
 public class TestPBImplRecords extends BasePBImplRecordsTest {
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     typeValueCache.put(Range.class, Range.between(1000L, 2000L));
     typeValueCache.put(URL.class, URL.newInstance(
@@ -430,6 +436,7 @@ public static void setup() throws Exception {
     generateByNewInstance(UpdatedContainer.class);
     generateByNewInstance(ContainerUpdateRequest.class);
     generateByNewInstance(ContainerUpdateResponse.class);
+    generateByNewInstance(EnhancedHeadroom.class);
     // genByNewInstance does not apply to QueueInfo, cause
     // it is recursive(has sub queues)
     typeValueCache.put(QueueInfo.class, QueueInfo.
@@ -469,326 +476,326 @@ public static void setup() throws Exception {
   }
 
   @Test
-  public void testAllocateRequestPBImpl() throws Exception {
+  void testAllocateRequestPBImpl() throws Exception {
     validatePBImplRecord(AllocateRequestPBImpl.class, AllocateRequestProto.class);
   }
 
   @Test
-  public void testAllocateResponsePBImpl() throws Exception {
+  void testAllocateResponsePBImpl() throws Exception {
     validatePBImplRecord(AllocateResponsePBImpl.class, AllocateResponseProto.class);
   }
 
   @Test
-  public void testCancelDelegationTokenRequestPBImpl() throws Exception {
+  void testCancelDelegationTokenRequestPBImpl() throws Exception {
     validatePBImplRecord(CancelDelegationTokenRequestPBImpl.class,
         CancelDelegationTokenRequestProto.class);
   }
 
   @Test
-  public void testCancelDelegationTokenResponsePBImpl() throws Exception {
+  void testCancelDelegationTokenResponsePBImpl() throws Exception {
     validatePBImplRecord(CancelDelegationTokenResponsePBImpl.class,
         CancelDelegationTokenResponseProto.class);
   }
 
   @Test
-  public void testFinishApplicationMasterRequestPBImpl() throws Exception {
+  void testFinishApplicationMasterRequestPBImpl() throws Exception {
     validatePBImplRecord(FinishApplicationMasterRequestPBImpl.class,
         FinishApplicationMasterRequestProto.class);
   }
 
   @Test
-  public void testFinishApplicationMasterResponsePBImpl() throws Exception {
+  void testFinishApplicationMasterResponsePBImpl() throws Exception {
     validatePBImplRecord(FinishApplicationMasterResponsePBImpl.class,
         FinishApplicationMasterResponseProto.class);
   }
 
   @Test
-  public void testGetApplicationAttemptReportRequestPBImpl() throws Exception {
+  void testGetApplicationAttemptReportRequestPBImpl() throws Exception {
     validatePBImplRecord(GetApplicationAttemptReportRequestPBImpl.class,
         GetApplicationAttemptReportRequestProto.class);
   }
 
   @Test
-  public void testGetApplicationAttemptReportResponsePBImpl() throws Exception {
+  void testGetApplicationAttemptReportResponsePBImpl() throws Exception {
     validatePBImplRecord(GetApplicationAttemptReportResponsePBImpl.class,
         GetApplicationAttemptReportResponseProto.class);
   }
 
   @Test
-  public void testGetApplicationAttemptsRequestPBImpl() throws Exception {
+  void testGetApplicationAttemptsRequestPBImpl() throws Exception {
     validatePBImplRecord(GetApplicationAttemptsRequestPBImpl.class,
         GetApplicationAttemptsRequestProto.class);
   }
 
   @Test
-  public void testGetApplicationAttemptsResponsePBImpl() throws Exception {
+  void testGetApplicationAttemptsResponsePBImpl() throws Exception {
     validatePBImplRecord(GetApplicationAttemptsResponsePBImpl.class,
         GetApplicationAttemptsResponseProto.class);
   }
 
   @Test
-  public void testGetApplicationReportRequestPBImpl() throws Exception {
+  void testGetApplicationReportRequestPBImpl() throws Exception {
     validatePBImplRecord(GetApplicationReportRequestPBImpl.class,
         GetApplicationReportRequestProto.class);
   }
 
   @Test
-  public void testGetApplicationReportResponsePBImpl() throws Exception {
+  void testGetApplicationReportResponsePBImpl() throws Exception {
     validatePBImplRecord(GetApplicationReportResponsePBImpl.class,
         GetApplicationReportResponseProto.class);
   }
 
   @Test
-  public void testGetApplicationsRequestPBImpl() throws Exception {
+  void testGetApplicationsRequestPBImpl() throws Exception {
     validatePBImplRecord(GetApplicationsRequestPBImpl.class,
         GetApplicationsRequestProto.class);
   }
 
   @Test
-  public void testGetApplicationsResponsePBImpl() throws Exception {
+  void testGetApplicationsResponsePBImpl() throws Exception {
     validatePBImplRecord(GetApplicationsResponsePBImpl.class,
         GetApplicationsResponseProto.class);
   }
 
   @Test
-  public void testGetClusterMetricsRequestPBImpl() throws Exception {
+  void testGetClusterMetricsRequestPBImpl() throws Exception {
     validatePBImplRecord(GetClusterMetricsRequestPBImpl.class,
         GetClusterMetricsRequestProto.class);
   }
 
   @Test
-  public void testGetClusterMetricsResponsePBImpl() throws Exception {
+  void testGetClusterMetricsResponsePBImpl() throws Exception {
     validatePBImplRecord(GetClusterMetricsResponsePBImpl.class,
         GetClusterMetricsResponseProto.class);
   }
 
   @Test
-  public void testGetClusterNodesRequestPBImpl() throws Exception {
+  void testGetClusterNodesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodesRequestPBImpl.class,
         GetClusterNodesRequestProto.class);
   }
 
   @Test
-  public void testGetClusterNodesResponsePBImpl() throws Exception {
+  void testGetClusterNodesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodesResponsePBImpl.class,
         GetClusterNodesResponseProto.class);
   }
 
   @Test
-  public void testGetContainerReportRequestPBImpl() throws Exception {
+  void testGetContainerReportRequestPBImpl() throws Exception {
     validatePBImplRecord(GetContainerReportRequestPBImpl.class,
         GetContainerReportRequestProto.class);
   }
 
   @Test
-  public void testGetContainerReportResponsePBImpl() throws Exception {
+  void testGetContainerReportResponsePBImpl() throws Exception {
     validatePBImplRecord(GetContainerReportResponsePBImpl.class,
         GetContainerReportResponseProto.class);
   }
 
   @Test
-  public void testGetContainersRequestPBImpl() throws Exception {
+  void testGetContainersRequestPBImpl() throws Exception {
     validatePBImplRecord(GetContainersRequestPBImpl.class,
         GetContainersRequestProto.class);
   }
 
   @Test
-  public void testGetContainersResponsePBImpl() throws Exception {
+  void testGetContainersResponsePBImpl() throws Exception {
     validatePBImplRecord(GetContainersResponsePBImpl.class,
         GetContainersResponseProto.class);
   }
 
   @Test
-  public void testGetContainerStatusesRequestPBImpl() throws Exception {
+  void testGetContainerStatusesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetContainerStatusesRequestPBImpl.class,
         GetContainerStatusesRequestProto.class);
   }
 
   @Test
-  public void testGetContainerStatusesResponsePBImpl() throws Exception {
+  void testGetContainerStatusesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetContainerStatusesResponsePBImpl.class,
         GetContainerStatusesResponseProto.class);
   }
 
   @Test
-  public void testGetDelegationTokenRequestPBImpl() throws Exception {
+  void testGetDelegationTokenRequestPBImpl() throws Exception {
     validatePBImplRecord(GetDelegationTokenRequestPBImpl.class,
         GetDelegationTokenRequestProto.class);
   }
 
   @Test
-  public void testGetDelegationTokenResponsePBImpl() throws Exception {
+  void testGetDelegationTokenResponsePBImpl() throws Exception {
     validatePBImplRecord(GetDelegationTokenResponsePBImpl.class,
         GetDelegationTokenResponseProto.class);
   }
 
   @Test
-  public void testGetNewApplicationRequestPBImpl() throws Exception {
+  void testGetNewApplicationRequestPBImpl() throws Exception {
     validatePBImplRecord(GetNewApplicationRequestPBImpl.class,
         GetNewApplicationRequestProto.class);
   }
 
   @Test
-  public void testGetNewApplicationResponsePBImpl() throws Exception {
+  void testGetNewApplicationResponsePBImpl() throws Exception {
     validatePBImplRecord(GetNewApplicationResponsePBImpl.class,
         GetNewApplicationResponseProto.class);
   }
 
   @Test
-  public void testGetQueueInfoRequestPBImpl() throws Exception {
+  void testGetQueueInfoRequestPBImpl() throws Exception {
     validatePBImplRecord(GetQueueInfoRequestPBImpl.class,
         GetQueueInfoRequestProto.class);
   }
 
   @Test
-  public void testGetQueueInfoResponsePBImpl() throws Exception {
+  void testGetQueueInfoResponsePBImpl() throws Exception {
     validatePBImplRecord(GetQueueInfoResponsePBImpl.class,
         GetQueueInfoResponseProto.class);
   }
 
   @Test
-  public void testGetQueueUserAclsInfoRequestPBImpl() throws Exception {
+  void testGetQueueUserAclsInfoRequestPBImpl() throws Exception {
     validatePBImplRecord(GetQueueUserAclsInfoRequestPBImpl.class,
         GetQueueUserAclsInfoRequestProto.class);
   }
 
   @Test
-  public void testGetQueueUserAclsInfoResponsePBImpl() throws Exception {
+  void testGetQueueUserAclsInfoResponsePBImpl() throws Exception {
     validatePBImplRecord(GetQueueUserAclsInfoResponsePBImpl.class,
         GetQueueUserAclsInfoResponseProto.class);
   }
 
   @Test
-  public void testKillApplicationRequestPBImpl() throws Exception {
+  void testKillApplicationRequestPBImpl() throws Exception {
     validatePBImplRecord(KillApplicationRequestPBImpl.class,
         KillApplicationRequestProto.class);
   }
 
   @Test
-  public void testKillApplicationResponsePBImpl() throws Exception {
+  void testKillApplicationResponsePBImpl() throws Exception {
     validatePBImplRecord(KillApplicationResponsePBImpl.class,
         KillApplicationResponseProto.class);
   }
 
   @Test
-  public void testMoveApplicationAcrossQueuesRequestPBImpl() throws Exception {
+  void testMoveApplicationAcrossQueuesRequestPBImpl() throws Exception {
     validatePBImplRecord(MoveApplicationAcrossQueuesRequestPBImpl.class,
         MoveApplicationAcrossQueuesRequestProto.class);
   }
 
   @Test
-  public void testMoveApplicationAcrossQueuesResponsePBImpl() throws Exception {
+  void testMoveApplicationAcrossQueuesResponsePBImpl() throws Exception {
     validatePBImplRecord(MoveApplicationAcrossQueuesResponsePBImpl.class,
         MoveApplicationAcrossQueuesResponseProto.class);
   }
 
   @Test
-  public void testRegisterApplicationMasterRequestPBImpl() throws Exception {
+  void testRegisterApplicationMasterRequestPBImpl() throws Exception {
     validatePBImplRecord(RegisterApplicationMasterRequestPBImpl.class,
         RegisterApplicationMasterRequestProto.class);
   }
 
   @Test
-  public void testRegisterApplicationMasterResponsePBImpl() throws Exception {
+  void testRegisterApplicationMasterResponsePBImpl() throws Exception {
     validatePBImplRecord(RegisterApplicationMasterResponsePBImpl.class,
         RegisterApplicationMasterResponseProto.class);
   }
 
   @Test
-  public void testRenewDelegationTokenRequestPBImpl() throws Exception {
+  void testRenewDelegationTokenRequestPBImpl() throws Exception {
     validatePBImplRecord(RenewDelegationTokenRequestPBImpl.class,
         RenewDelegationTokenRequestProto.class);
   }
 
   @Test
-  public void testRenewDelegationTokenResponsePBImpl() throws Exception {
+  void testRenewDelegationTokenResponsePBImpl() throws Exception {
     validatePBImplRecord(RenewDelegationTokenResponsePBImpl.class,
         RenewDelegationTokenResponseProto.class);
   }
 
   @Test
-  public void testStartContainerRequestPBImpl() throws Exception {
+  void testStartContainerRequestPBImpl() throws Exception {
     validatePBImplRecord(StartContainerRequestPBImpl.class,
         StartContainerRequestProto.class);
   }
 
   @Test
-  public void testStartContainersRequestPBImpl() throws Exception {
+  void testStartContainersRequestPBImpl() throws Exception {
     validatePBImplRecord(StartContainersRequestPBImpl.class,
         StartContainersRequestProto.class);
   }
 
   @Test
-  public void testStartContainersResponsePBImpl() throws Exception {
+  void testStartContainersResponsePBImpl() throws Exception {
     validatePBImplRecord(StartContainersResponsePBImpl.class,
         StartContainersResponseProto.class);
   }
 
   @Test
-  public void testStopContainersRequestPBImpl() throws Exception {
+  void testStopContainersRequestPBImpl() throws Exception {
     validatePBImplRecord(StopContainersRequestPBImpl.class,
         StopContainersRequestProto.class);
   }
 
   @Test
-  public void testStopContainersResponsePBImpl() throws Exception {
+  void testStopContainersResponsePBImpl() throws Exception {
     validatePBImplRecord(StopContainersResponsePBImpl.class,
         StopContainersResponseProto.class);
   }
 
   @Test
-  public void testIncreaseContainersResourceRequestPBImpl() throws Exception {
+  void testIncreaseContainersResourceRequestPBImpl() throws Exception {
     validatePBImplRecord(IncreaseContainersResourceRequestPBImpl.class,
         IncreaseContainersResourceRequestProto.class);
   }
 
   @Test
-  public void testIncreaseContainersResourceResponsePBImpl() throws Exception {
+  void testIncreaseContainersResourceResponsePBImpl() throws Exception {
     validatePBImplRecord(IncreaseContainersResourceResponsePBImpl.class,
         IncreaseContainersResourceResponseProto.class);
   }
 
   @Test
-  public void testSubmitApplicationRequestPBImpl() throws Exception {
+  void testSubmitApplicationRequestPBImpl() throws Exception {
     validatePBImplRecord(SubmitApplicationRequestPBImpl.class,
         SubmitApplicationRequestProto.class);
   }
 
   @Test
-  public void testSubmitApplicationResponsePBImpl() throws Exception {
+  void testSubmitApplicationResponsePBImpl() throws Exception {
     validatePBImplRecord(SubmitApplicationResponsePBImpl.class,
         SubmitApplicationResponseProto.class);
   }
 
-  @Test
-  @Ignore
   // ignore cause ApplicationIdPBImpl is immutable
-  public void testApplicationAttemptIdPBImpl() throws Exception {
+  @Test
+  @Disabled
+  void testApplicationAttemptIdPBImpl() throws Exception {
     validatePBImplRecord(ApplicationAttemptIdPBImpl.class,
         ApplicationAttemptIdProto.class);
   }
 
   @Test
-  public void testApplicationAttemptReportPBImpl() throws Exception {
+  void testApplicationAttemptReportPBImpl() throws Exception {
     validatePBImplRecord(ApplicationAttemptReportPBImpl.class,
         ApplicationAttemptReportProto.class);
   }
 
-  @Test
-  @Ignore
   // ignore cause ApplicationIdPBImpl is immutable
-  public void testApplicationIdPBImpl() throws Exception {
+  @Test
+  @Disabled
+  void testApplicationIdPBImpl() throws Exception {
     validatePBImplRecord(ApplicationIdPBImpl.class, ApplicationIdProto.class);
   }
 
   @Test
-  public void testApplicationReportPBImpl() throws Exception {
+  void testApplicationReportPBImpl() throws Exception {
     validatePBImplRecord(ApplicationReportPBImpl.class,
         ApplicationReportProto.class);
   }
 
   @Test
-  public void testApplicationResourceUsageReportPBImpl() throws Exception {
+  void testApplicationResourceUsageReportPBImpl() throws Exception {
     excludedPropertiesMap.put(ApplicationResourceUsageReportPBImpl.class.getClass(),
         Arrays.asList("PreemptedResourceSecondsMap", "ResourceSecondsMap"));
     validatePBImplRecord(ApplicationResourceUsageReportPBImpl.class,
@@ -796,539 +803,551 @@ public void testApplicationResourceUsageReportPBImpl() throws Exception {
   }
 
   @Test
-  public void testApplicationSubmissionContextPBImpl() throws Exception {
+  void testApplicationSubmissionContextPBImpl() throws Exception {
     validatePBImplRecord(ApplicationSubmissionContextPBImpl.class,
         ApplicationSubmissionContextProto.class);
-    
+
     ApplicationSubmissionContext ctx =
         ApplicationSubmissionContext.newInstance(null, null, null, null, null,
             false, false, 0, Resources.none(), null, false, null, null);
-    
-    Assert.assertNotNull(ctx.getResource());
+
+    assertNotNull(ctx.getResource());
   }
 
-  @Test
-  @Ignore
   // ignore cause ApplicationIdPBImpl is immutable
-  public void testContainerIdPBImpl() throws Exception {
+  @Test
+  @Disabled
+  void testContainerIdPBImpl() throws Exception {
     validatePBImplRecord(ContainerIdPBImpl.class, ContainerIdProto.class);
   }
 
   @Test
-  public void testContainerRetryPBImpl() throws Exception {
+  void testContainerRetryPBImpl() throws Exception {
     validatePBImplRecord(ContainerRetryContextPBImpl.class,
         ContainerRetryContextProto.class);
   }
 
   @Test
-  public void testContainerLaunchContextPBImpl() throws Exception {
+  void testContainerLaunchContextPBImpl() throws Exception {
     validatePBImplRecord(ContainerLaunchContextPBImpl.class,
         ContainerLaunchContextProto.class);
   }
 
   @Test
-  public void testResourceLocalizationRequest() throws Exception {
+  void testResourceLocalizationRequest() throws Exception {
     validatePBImplRecord(ResourceLocalizationRequestPBImpl.class,
         YarnServiceProtos.ResourceLocalizationRequestProto.class);
   }
 
   @Test
-  public void testResourceLocalizationResponse() throws Exception {
+  void testResourceLocalizationResponse() throws Exception {
     validatePBImplRecord(ResourceLocalizationResponsePBImpl.class,
         YarnServiceProtos.ResourceLocalizationResponseProto.class);
   }
 
   @Test
-  public void testContainerPBImpl() throws Exception {
+  void testContainerPBImpl() throws Exception {
     validatePBImplRecord(ContainerPBImpl.class, ContainerProto.class);
   }
 
   @Test
-  public void testContainerReportPBImpl() throws Exception {
+  void testContainerReportPBImpl() throws Exception {
     validatePBImplRecord(ContainerReportPBImpl.class, ContainerReportProto.class);
   }
 
   @Test
-  public void testUpdateContainerRequestPBImpl() throws Exception {
+  void testUpdateContainerRequestPBImpl() throws Exception {
     validatePBImplRecord(UpdateContainerRequestPBImpl.class,
         YarnServiceProtos.UpdateContainerRequestProto.class);
   }
 
   @Test
-  public void testContainerStatusPBImpl() throws Exception {
+  void testContainerStatusPBImpl() throws Exception {
     validatePBImplRecord(ContainerStatusPBImpl.class, ContainerStatusProto.class);
   }
 
   @Test
-  public void testLocalResourcePBImpl() throws Exception {
+  void testLocalResourcePBImpl() throws Exception {
     validatePBImplRecord(LocalResourcePBImpl.class, LocalResourceProto.class);
   }
 
   @Test
-  public void testNMTokenPBImpl() throws Exception {
+  void testNMTokenPBImpl() throws Exception {
     validatePBImplRecord(NMTokenPBImpl.class, NMTokenProto.class);
   }
 
-  @Test
-  @Ignore
   // ignore cause ApplicationIdPBImpl is immutable
-  public void testNodeIdPBImpl() throws Exception {
+  @Test
+  @Disabled
+  void testNodeIdPBImpl() throws Exception {
     validatePBImplRecord(NodeIdPBImpl.class, NodeIdProto.class);
   }
 
   @Test
-  public void testNodeReportPBImpl() throws Exception {
+  void testNodeReportPBImpl() throws Exception {
     validatePBImplRecord(NodeReportPBImpl.class, NodeReportProto.class);
   }
 
   @Test
-  public void testPreemptionContainerPBImpl() throws Exception {
+  void testPreemptionContainerPBImpl() throws Exception {
     validatePBImplRecord(PreemptionContainerPBImpl.class,
         PreemptionContainerProto.class);
   }
 
   @Test
-  public void testPreemptionContractPBImpl() throws Exception {
+  void testPreemptionContractPBImpl() throws Exception {
     validatePBImplRecord(PreemptionContractPBImpl.class,
         PreemptionContractProto.class);
   }
 
   @Test
-  public void testPreemptionMessagePBImpl() throws Exception {
+  void testPreemptionMessagePBImpl() throws Exception {
     validatePBImplRecord(PreemptionMessagePBImpl.class,
         PreemptionMessageProto.class);
   }
 
   @Test
-  public void testPreemptionResourceRequestPBImpl() throws Exception {
+  void testPreemptionResourceRequestPBImpl() throws Exception {
     validatePBImplRecord(PreemptionResourceRequestPBImpl.class,
         PreemptionResourceRequestProto.class);
   }
 
   @Test
-  public void testPriorityPBImpl() throws Exception {
+  void testPriorityPBImpl() throws Exception {
     validatePBImplRecord(PriorityPBImpl.class, PriorityProto.class);
   }
 
   @Test
-  public void testQueueInfoPBImpl() throws Exception {
+  void testQueueInfoPBImpl() throws Exception {
     validatePBImplRecord(QueueInfoPBImpl.class, QueueInfoProto.class);
   }
 
   @Test
-  public void testQueueUserACLInfoPBImpl() throws Exception {
+  void testQueueConfigurationsPBImpl() throws Exception {
+    validatePBImplRecord(QueueConfigurationsPBImpl.class,
+        QueueConfigurationsProto.class);
+  }
+
+  @Test
+  void testQueueUserACLInfoPBImpl() throws Exception {
     validatePBImplRecord(QueueUserACLInfoPBImpl.class,
         QueueUserACLInfoProto.class);
   }
 
   @Test
-  public void testResourceBlacklistRequestPBImpl() throws Exception {
+  void testResourceBlacklistRequestPBImpl() throws Exception {
     validatePBImplRecord(ResourceBlacklistRequestPBImpl.class,
         ResourceBlacklistRequestProto.class);
   }
 
-  @Test
-  @Ignore
   // ignore as ResourceOptionPBImpl is immutable
-  public void testResourceOptionPBImpl() throws Exception {
+  @Test
+  @Disabled
+  void testResourceOptionPBImpl() throws Exception {
     validatePBImplRecord(ResourceOptionPBImpl.class, ResourceOptionProto.class);
   }
 
   @Test
-  public void testResourcePBImpl() throws Exception {
+  void testResourcePBImpl() throws Exception {
     validatePBImplRecord(ResourcePBImpl.class, ResourceProto.class);
   }
 
   @Test
-  public void testResourceRequestPBImpl() throws Exception {
+  void testResourceRequestPBImpl() throws Exception {
     validatePBImplRecord(ResourceRequestPBImpl.class, ResourceRequestProto.class);
   }
 
   @Test
-  public void testResourceSizingPBImpl() throws Exception {
+  void testResourceSizingPBImpl() throws Exception {
     validatePBImplRecord(ResourceSizingPBImpl.class, ResourceSizingProto.class);
   }
 
   @Test
-  public void testSchedulingRequestPBImpl() throws Exception {
+  void testSchedulingRequestPBImpl() throws Exception {
     validatePBImplRecord(SchedulingRequestPBImpl.class,
         SchedulingRequestProto.class);
   }
 
   @Test
-  public void testSerializedExceptionPBImpl() throws Exception {
+  void testSerializedExceptionPBImpl() throws Exception {
     validatePBImplRecord(SerializedExceptionPBImpl.class,
         SerializedExceptionProto.class);
   }
 
   @Test
-  public void testStrictPreemptionContractPBImpl() throws Exception {
+  void testStrictPreemptionContractPBImpl() throws Exception {
     validatePBImplRecord(StrictPreemptionContractPBImpl.class,
         StrictPreemptionContractProto.class);
   }
 
   @Test
-  public void testTokenPBImpl() throws Exception {
+  void testTokenPBImpl() throws Exception {
     validatePBImplRecord(TokenPBImpl.class, TokenProto.class);
   }
 
   @Test
-  public void testURLPBImpl() throws Exception {
+  void testURLPBImpl() throws Exception {
     validatePBImplRecord(URLPBImpl.class, URLProto.class);
   }
 
   @Test
-  public void testYarnClusterMetricsPBImpl() throws Exception {
+  void testYarnClusterMetricsPBImpl() throws Exception {
     validatePBImplRecord(YarnClusterMetricsPBImpl.class,
         YarnClusterMetricsProto.class);
   }
 
   @Test
-  public void testRefreshAdminAclsRequestPBImpl() throws Exception {
+  void testRefreshAdminAclsRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshAdminAclsRequestPBImpl.class,
         RefreshAdminAclsRequestProto.class);
   }
 
   @Test
-  public void testRefreshAdminAclsResponsePBImpl() throws Exception {
+  void testRefreshAdminAclsResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshAdminAclsResponsePBImpl.class,
         RefreshAdminAclsResponseProto.class);
   }
 
   @Test
-  public void testRefreshNodesRequestPBImpl() throws Exception {
+  void testRefreshNodesRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshNodesRequestPBImpl.class,
         RefreshNodesRequestProto.class);
   }
 
   @Test
-  public void testRefreshNodesResponsePBImpl() throws Exception {
+  void testRefreshNodesResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshNodesResponsePBImpl.class,
         RefreshNodesResponseProto.class);
   }
 
   @Test
-  public void testRefreshQueuesRequestPBImpl() throws Exception {
+  void testRefreshQueuesRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshQueuesRequestPBImpl.class,
         RefreshQueuesRequestProto.class);
   }
 
   @Test
-  public void testRefreshQueuesResponsePBImpl() throws Exception {
+  void testRefreshQueuesResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshQueuesResponsePBImpl.class,
         RefreshQueuesResponseProto.class);
   }
 
   @Test
-  public void testRefreshNodesResourcesRequestPBImpl() throws Exception {
+  void testRefreshNodesResourcesRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshNodesResourcesRequestPBImpl.class,
         RefreshNodesResourcesRequestProto.class);
   }
 
   @Test
-  public void testRefreshNodesResourcesResponsePBImpl() throws Exception {
+  void testRefreshNodesResourcesResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshNodesResourcesResponsePBImpl.class,
         RefreshNodesResourcesResponseProto.class);
   }
 
   @Test
-  public void testRefreshServiceAclsRequestPBImpl() throws Exception {
+  void testRefreshServiceAclsRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshServiceAclsRequestPBImpl.class,
         RefreshServiceAclsRequestProto.class);
   }
 
   @Test
-  public void testRefreshServiceAclsResponsePBImpl() throws Exception {
+  void testRefreshServiceAclsResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshServiceAclsResponsePBImpl.class,
         RefreshServiceAclsResponseProto.class);
   }
 
   @Test
-  public void testRefreshSuperUserGroupsConfigurationRequestPBImpl()
+  void testRefreshSuperUserGroupsConfigurationRequestPBImpl()
       throws Exception {
     validatePBImplRecord(RefreshSuperUserGroupsConfigurationRequestPBImpl.class,
         RefreshSuperUserGroupsConfigurationRequestProto.class);
   }
 
   @Test
-  public void testRefreshSuperUserGroupsConfigurationResponsePBImpl()
+  void testRefreshSuperUserGroupsConfigurationResponsePBImpl()
       throws Exception {
     validatePBImplRecord(RefreshSuperUserGroupsConfigurationResponsePBImpl.class,
         RefreshSuperUserGroupsConfigurationResponseProto.class);
   }
 
   @Test
-  public void testRefreshUserToGroupsMappingsRequestPBImpl() throws Exception {
+  void testRefreshUserToGroupsMappingsRequestPBImpl() throws Exception {
     validatePBImplRecord(RefreshUserToGroupsMappingsRequestPBImpl.class,
         RefreshUserToGroupsMappingsRequestProto.class);
   }
 
   @Test
-  public void testRefreshUserToGroupsMappingsResponsePBImpl() throws Exception {
+  void testRefreshUserToGroupsMappingsResponsePBImpl() throws Exception {
     validatePBImplRecord(RefreshUserToGroupsMappingsResponsePBImpl.class,
         RefreshUserToGroupsMappingsResponseProto.class);
   }
 
   @Test
-  public void testUpdateNodeResourceRequestPBImpl() throws Exception {
+  void testUpdateNodeResourceRequestPBImpl() throws Exception {
     validatePBImplRecord(UpdateNodeResourceRequestPBImpl.class,
         UpdateNodeResourceRequestProto.class);
   }
 
   @Test
-  public void testUpdateNodeResourceResponsePBImpl() throws Exception {
+  void testUpdateNodeResourceResponsePBImpl() throws Exception {
     validatePBImplRecord(UpdateNodeResourceResponsePBImpl.class,
         UpdateNodeResourceResponseProto.class);
   }
 
   @Test
-  public void testReservationSubmissionRequestPBImpl() throws Exception {
+  void testReservationSubmissionRequestPBImpl() throws Exception {
     validatePBImplRecord(ReservationSubmissionRequestPBImpl.class,
         ReservationSubmissionRequestProto.class);
   }
 
   @Test
-  public void testReservationSubmissionResponsePBImpl() throws Exception {
+  void testReservationSubmissionResponsePBImpl() throws Exception {
     validatePBImplRecord(ReservationSubmissionResponsePBImpl.class,
         ReservationSubmissionResponseProto.class);
   }
 
   @Test
-  public void testReservationUpdateRequestPBImpl() throws Exception {
+  void testReservationUpdateRequestPBImpl() throws Exception {
     validatePBImplRecord(ReservationUpdateRequestPBImpl.class,
         ReservationUpdateRequestProto.class);
   }
 
   @Test
-  public void testReservationUpdateResponsePBImpl() throws Exception {
+  void testReservationUpdateResponsePBImpl() throws Exception {
     validatePBImplRecord(ReservationUpdateResponsePBImpl.class,
         ReservationUpdateResponseProto.class);
   }
 
   @Test
-  public void testReservationDeleteRequestPBImpl() throws Exception {
+  void testReservationDeleteRequestPBImpl() throws Exception {
     validatePBImplRecord(ReservationDeleteRequestPBImpl.class,
         ReservationDeleteRequestProto.class);
   }
 
   @Test
-  public void testReservationDeleteResponsePBImpl() throws Exception {
+  void testReservationDeleteResponsePBImpl() throws Exception {
     validatePBImplRecord(ReservationDeleteResponsePBImpl.class,
         ReservationDeleteResponseProto.class);
   }
 
   @Test
-  public void testReservationListRequestPBImpl() throws Exception {
+  void testReservationListRequestPBImpl() throws Exception {
     validatePBImplRecord(ReservationListRequestPBImpl.class,
-            ReservationListRequestProto.class);
+        ReservationListRequestProto.class);
   }
 
   @Test
-  public void testReservationListResponsePBImpl() throws Exception {
+  void testReservationListResponsePBImpl() throws Exception {
     validatePBImplRecord(ReservationListResponsePBImpl.class,
-            ReservationListResponseProto.class);
+        ReservationListResponseProto.class);
   }
 
   @Test
-  public void testAddToClusterNodeLabelsRequestPBImpl() throws Exception {
+  void testAddToClusterNodeLabelsRequestPBImpl() throws Exception {
     validatePBImplRecord(AddToClusterNodeLabelsRequestPBImpl.class,
         AddToClusterNodeLabelsRequestProto.class);
   }
-  
+
   @Test
-  public void testAddToClusterNodeLabelsResponsePBImpl() throws Exception {
+  void testAddToClusterNodeLabelsResponsePBImpl() throws Exception {
     validatePBImplRecord(AddToClusterNodeLabelsResponsePBImpl.class,
         AddToClusterNodeLabelsResponseProto.class);
   }
-  
+
   @Test
-  public void testRemoveFromClusterNodeLabelsRequestPBImpl() throws Exception {
+  void testRemoveFromClusterNodeLabelsRequestPBImpl() throws Exception {
     validatePBImplRecord(RemoveFromClusterNodeLabelsRequestPBImpl.class,
         RemoveFromClusterNodeLabelsRequestProto.class);
   }
-  
+
   @Test
-  public void testRemoveFromClusterNodeLabelsResponsePBImpl() throws Exception {
+  void testRemoveFromClusterNodeLabelsResponsePBImpl() throws Exception {
     validatePBImplRecord(RemoveFromClusterNodeLabelsResponsePBImpl.class,
         RemoveFromClusterNodeLabelsResponseProto.class);
   }
-  
+
   @Test
-  public void testGetClusterNodeLabelsRequestPBImpl() throws Exception {
+  void testGetClusterNodeLabelsRequestPBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodeLabelsRequestPBImpl.class,
         GetClusterNodeLabelsRequestProto.class);
   }
 
   @Test
-  public void testGetClusterNodeLabelsResponsePBImpl() throws Exception {
+  void testGetClusterNodeLabelsResponsePBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodeLabelsResponsePBImpl.class,
         GetClusterNodeLabelsResponseProto.class);
   }
-  
+
   @Test
-  public void testReplaceLabelsOnNodeRequestPBImpl() throws Exception {
+  void testReplaceLabelsOnNodeRequestPBImpl() throws Exception {
     validatePBImplRecord(ReplaceLabelsOnNodeRequestPBImpl.class,
         ReplaceLabelsOnNodeRequestProto.class);
   }
 
   @Test
-  public void testReplaceLabelsOnNodeResponsePBImpl() throws Exception {
+  void testReplaceLabelsOnNodeResponsePBImpl() throws Exception {
     validatePBImplRecord(ReplaceLabelsOnNodeResponsePBImpl.class,
         ReplaceLabelsOnNodeResponseProto.class);
   }
-  
+
   @Test
-  public void testGetNodeToLabelsRequestPBImpl() throws Exception {
+  void testGetNodeToLabelsRequestPBImpl() throws Exception {
     validatePBImplRecord(GetNodesToLabelsRequestPBImpl.class,
         GetNodesToLabelsRequestProto.class);
   }
 
   @Test
-  public void testGetNodeToLabelsResponsePBImpl() throws Exception {
+  void testGetNodeToLabelsResponsePBImpl() throws Exception {
     validatePBImplRecord(GetNodesToLabelsResponsePBImpl.class,
         GetNodesToLabelsResponseProto.class);
   }
 
   @Test
-  public void testGetLabelsToNodesRequestPBImpl() throws Exception {
+  void testGetLabelsToNodesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetLabelsToNodesRequestPBImpl.class,
         GetLabelsToNodesRequestProto.class);
   }
 
   @Test
-  public void testGetLabelsToNodesResponsePBImpl() throws Exception {
+  void testGetLabelsToNodesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetLabelsToNodesResponsePBImpl.class,
         GetLabelsToNodesResponseProto.class);
   }
-  
+
   @Test
-  public void testNodeLabelAttributesPBImpl() throws Exception {
+  void testNodeLabelAttributesPBImpl() throws Exception {
     validatePBImplRecord(NodeLabelPBImpl.class,
         NodeLabelProto.class);
   }
-  
+
   @Test
-  public void testCheckForDecommissioningNodesRequestPBImpl() throws Exception {
+  void testCheckForDecommissioningNodesRequestPBImpl() throws Exception {
     validatePBImplRecord(CheckForDecommissioningNodesRequestPBImpl.class,
         CheckForDecommissioningNodesRequestProto.class);
   }
 
   @Test
-  public void testCheckForDecommissioningNodesResponsePBImpl() throws Exception {
+  void testCheckForDecommissioningNodesResponsePBImpl() throws Exception {
     validatePBImplRecord(CheckForDecommissioningNodesResponsePBImpl.class,
         CheckForDecommissioningNodesResponseProto.class);
   }
 
   @Test
-  public void testExecutionTypeRequestPBImpl() throws Exception {
+  void testExecutionTypeRequestPBImpl() throws Exception {
     validatePBImplRecord(ExecutionTypeRequestPBImpl.class,
         ExecutionTypeRequestProto.class);
   }
 
   @Test
-  public void testGetAllResourceProfilesResponsePBImpl() throws Exception {
+  void testGetAllResourceProfilesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetAllResourceProfilesResponsePBImpl.class,
         GetAllResourceProfilesResponseProto.class);
   }
 
   @Test
-  public void testGetResourceProfileRequestPBImpl() throws Exception {
+  void testGetResourceProfileRequestPBImpl() throws Exception {
     validatePBImplRecord(GetResourceProfileRequestPBImpl.class,
         GetResourceProfileRequestProto.class);
   }
 
   @Test
-  public void testGetResourceProfileResponsePBImpl() throws Exception {
+  void testGetResourceProfileResponsePBImpl() throws Exception {
     validatePBImplRecord(GetResourceProfileResponsePBImpl.class,
         GetResourceProfileResponseProto.class);
   }
 
   @Test
-  public void testResourceTypesInfoPBImpl() throws Exception {
+  void testResourceTypesInfoPBImpl() throws Exception {
     validatePBImplRecord(ResourceTypeInfoPBImpl.class,
         YarnProtos.ResourceTypeInfoProto.class);
   }
 
   @Test
-  public void testGetAllResourceTypesInfoRequestPBImpl() throws Exception {
+  void testGetAllResourceTypesInfoRequestPBImpl() throws Exception {
     validatePBImplRecord(GetAllResourceTypeInfoRequestPBImpl.class,
         YarnServiceProtos.GetAllResourceTypeInfoRequestProto.class);
   }
 
   @Test
-  public void testGetAllResourceTypesInfoResponsePBImpl() throws Exception {
+  void testGetAllResourceTypesInfoResponsePBImpl() throws Exception {
     validatePBImplRecord(GetAllResourceTypeInfoResponsePBImpl.class,
         YarnServiceProtos.GetAllResourceTypeInfoResponseProto.class);
   }
 
   @Test
-  public void testNodeAttributeKeyPBImpl() throws Exception {
+  void testNodeAttributeKeyPBImpl() throws Exception {
     validatePBImplRecord(NodeAttributeKeyPBImpl.class,
         NodeAttributeKeyProto.class);
   }
 
   @Test
-  public void testNodeToAttributeValuePBImpl() throws Exception {
+  void testNodeToAttributeValuePBImpl() throws Exception {
     validatePBImplRecord(NodeToAttributeValuePBImpl.class,
         NodeToAttributeValueProto.class);
   }
 
   @Test
-  public void testNodeAttributePBImpl() throws Exception {
+  void testNodeAttributePBImpl() throws Exception {
     validatePBImplRecord(NodeAttributePBImpl.class, NodeAttributeProto.class);
   }
 
   @Test
-  public void testNodeAttributeInfoPBImpl() throws Exception {
+  void testNodeAttributeInfoPBImpl() throws Exception {
     validatePBImplRecord(NodeAttributeInfoPBImpl.class,
         NodeAttributeInfoProto.class);
   }
 
   @Test
-  public void testNodeToAttributesPBImpl() throws Exception {
+  void testNodeToAttributesPBImpl() throws Exception {
     validatePBImplRecord(NodeToAttributesPBImpl.class,
         NodeToAttributesProto.class);
   }
 
   @Test
-  public void testNodesToAttributesMappingRequestPBImpl() throws Exception {
+  void testNodesToAttributesMappingRequestPBImpl() throws Exception {
     validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
         NodesToAttributesMappingRequestProto.class);
   }
 
   @Test
-  public void testGetAttributesToNodesRequestPBImpl() throws Exception {
+  void testGetAttributesToNodesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetAttributesToNodesRequestPBImpl.class,
         YarnServiceProtos.GetAttributesToNodesRequestProto.class);
   }
 
   @Test
-  public void testGetAttributesToNodesResponsePBImpl() throws Exception {
+  void testGetAttributesToNodesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetAttributesToNodesResponsePBImpl.class,
         YarnServiceProtos.GetAttributesToNodesResponseProto.class);
   }
 
   @Test
-  public void testGetClusterNodeAttributesRequestPBImpl() throws Exception {
+  void testGetClusterNodeAttributesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodeAttributesRequestPBImpl.class,
         YarnServiceProtos.GetClusterNodeAttributesRequestProto.class);
   }
 
   @Test
-  public void testGetClusterNodeAttributesResponsePBImpl() throws Exception {
+  void testGetClusterNodeAttributesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetClusterNodeAttributesResponsePBImpl.class,
         YarnServiceProtos.GetClusterNodeAttributesResponseProto.class);
   }
 
   @Test
-  public void testGetNodesToAttributesRequestPBImpl() throws Exception {
+  void testGetNodesToAttributesRequestPBImpl() throws Exception {
     validatePBImplRecord(GetNodesToAttributesRequestPBImpl.class,
         YarnServiceProtos.GetNodesToAttributesRequestProto.class);
   }
 
   @Test
-  public void testGetNodesToAttributesResponsePBImpl() throws Exception {
+  void testGetNodesToAttributesResponsePBImpl() throws Exception {
     validatePBImplRecord(GetNodesToAttributesResponsePBImpl.class,
         YarnServiceProtos.GetNodesToAttributesResponseProto.class);
   }
+
+  @Test
+  void testGetEnhancedHeadroomPBImpl() throws Exception {
+    validatePBImplRecord(EnhancedHeadroomPBImpl.class,
+        YarnServiceProtos.EnhancedHeadroomProto.class);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
index bd245e29ce9a2..300fc6a42b084 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
@@ -18,17 +18,10 @@
 
 package org.apache.hadoop.yarn.api;
 
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
-
 import java.util.Iterator;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter;
 import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
@@ -40,8 +33,18 @@
 import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType;
 import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test class for {@link PlacementConstraintToProtoConverter} and
@@ -50,10 +53,10 @@
 public class TestPlacementConstraintPBConversion {
 
   @Test
-  public void testTargetConstraintProtoConverter() {
+  void testTargetConstraintProtoConverter() {
     AbstractConstraint sConstraintExpr =
         targetIn(NODE, allocationTag("hbase-m"));
-    Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+    assertTrue(sConstraintExpr instanceof SingleConstraint);
     SingleConstraint single = (SingleConstraint) sConstraintExpr;
     PlacementConstraint sConstraint =
         PlacementConstraints.build(sConstraintExpr);
@@ -63,14 +66,14 @@ public void testTargetConstraintProtoConverter() {
         new PlacementConstraintToProtoConverter(sConstraint);
     PlacementConstraintProto protoConstraint = toProtoConverter.convert();
 
-    Assert.assertTrue(protoConstraint.hasSimpleConstraint());
-    Assert.assertFalse(protoConstraint.hasCompositeConstraint());
+    assertTrue(protoConstraint.hasSimpleConstraint());
+    assertFalse(protoConstraint.hasCompositeConstraint());
     SimplePlacementConstraintProto sProto =
         protoConstraint.getSimpleConstraint();
-    Assert.assertEquals(single.getScope(), sProto.getScope());
-    Assert.assertEquals(single.getMinCardinality(), sProto.getMinCardinality());
-    Assert.assertEquals(single.getMaxCardinality(), sProto.getMaxCardinality());
-    Assert.assertEquals(single.getTargetExpressions().size(),
+    assertEquals(single.getScope(), sProto.getScope());
+    assertEquals(single.getMinCardinality(), sProto.getMinCardinality());
+    assertEquals(single.getMaxCardinality(), sProto.getMaxCardinality());
+    assertEquals(single.getTargetExpressions().size(),
         sProto.getTargetExpressionsList().size());
 
     // Convert from proto.
@@ -79,21 +82,21 @@ public void testTargetConstraintProtoConverter() {
     PlacementConstraint newConstraint = fromProtoConverter.convert();
 
     AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
-    Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+    assertTrue(newConstraintExpr instanceof SingleConstraint);
     SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
-    Assert.assertEquals(single.getScope(), newSingle.getScope());
-    Assert.assertEquals(single.getMinCardinality(),
+    assertEquals(single.getScope(), newSingle.getScope());
+    assertEquals(single.getMinCardinality(),
         newSingle.getMinCardinality());
-    Assert.assertEquals(single.getMaxCardinality(),
+    assertEquals(single.getMaxCardinality(),
         newSingle.getMaxCardinality());
-    Assert.assertEquals(single.getTargetExpressions(),
+    assertEquals(single.getTargetExpressions(),
         newSingle.getTargetExpressions());
   }
 
   @Test
-  public void testCardinalityConstraintProtoConverter() {
+  void testCardinalityConstraintProtoConverter() {
     AbstractConstraint sConstraintExpr = cardinality(RACK, 3, 10);
-    Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+    assertTrue(sConstraintExpr instanceof SingleConstraint);
     SingleConstraint single = (SingleConstraint) sConstraintExpr;
     PlacementConstraint sConstraint =
         PlacementConstraints.build(sConstraintExpr);
@@ -111,17 +114,17 @@ public void testCardinalityConstraintProtoConverter() {
     PlacementConstraint newConstraint = fromProtoConverter.convert();
 
     AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
-    Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+    assertTrue(newConstraintExpr instanceof SingleConstraint);
     SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
     compareSimpleConstraints(single, newSingle);
   }
 
   @Test
-  public void testCompositeConstraintProtoConverter() {
+  void testCompositeConstraintProtoConverter() {
     AbstractConstraint constraintExpr =
         or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
             targetCardinality(RACK, 2, 10, allocationTag("zk")));
-    Assert.assertTrue(constraintExpr instanceof Or);
+    assertTrue(constraintExpr instanceof Or);
     PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
     Or orExpr = (Or) constraintExpr;
 
@@ -130,14 +133,14 @@ public void testCompositeConstraintProtoConverter() {
         new PlacementConstraintToProtoConverter(constraint);
     PlacementConstraintProto protoConstraint = toProtoConverter.convert();
 
-    Assert.assertFalse(protoConstraint.hasSimpleConstraint());
-    Assert.assertTrue(protoConstraint.hasCompositeConstraint());
+    assertFalse(protoConstraint.hasSimpleConstraint());
+    assertTrue(protoConstraint.hasCompositeConstraint());
     CompositePlacementConstraintProto cProto =
         protoConstraint.getCompositeConstraint();
 
-    Assert.assertEquals(CompositeType.OR, cProto.getCompositeType());
-    Assert.assertEquals(3, cProto.getChildConstraintsCount());
-    Assert.assertEquals(0, cProto.getTimedChildConstraintsCount());
+    assertEquals(CompositeType.OR, cProto.getCompositeType());
+    assertEquals(3, cProto.getChildConstraintsCount());
+    assertEquals(0, cProto.getTimedChildConstraintsCount());
     Iterator<AbstractConstraint> orChildren = orExpr.getChildren().iterator();
     Iterator<PlacementConstraintProto> orProtoChildren =
         cProto.getChildConstraintsList().iterator();
@@ -153,9 +156,9 @@ public void testCompositeConstraintProtoConverter() {
     PlacementConstraint newConstraint = fromProtoConverter.convert();
 
     AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
-    Assert.assertTrue(newConstraintExpr instanceof Or);
+    assertTrue(newConstraintExpr instanceof Or);
     Or newOrExpr = (Or) newConstraintExpr;
-    Assert.assertEquals(3, newOrExpr.getChildren().size());
+    assertEquals(3, newOrExpr.getChildren().size());
     orChildren = orExpr.getChildren().iterator();
     Iterator<AbstractConstraint> newOrChildren =
         newOrExpr.getChildren().iterator();
@@ -169,26 +172,26 @@ public void testCompositeConstraintProtoConverter() {
 
   private void compareSimpleConstraintToProto(SingleConstraint constraint,
       PlacementConstraintProto proto) {
-    Assert.assertTrue(proto.hasSimpleConstraint());
-    Assert.assertFalse(proto.hasCompositeConstraint());
+    assertTrue(proto.hasSimpleConstraint());
+    assertFalse(proto.hasCompositeConstraint());
     SimplePlacementConstraintProto sProto = proto.getSimpleConstraint();
-    Assert.assertEquals(constraint.getScope(), sProto.getScope());
-    Assert.assertEquals(constraint.getMinCardinality(),
+    assertEquals(constraint.getScope(), sProto.getScope());
+    assertEquals(constraint.getMinCardinality(),
         sProto.getMinCardinality());
-    Assert.assertEquals(constraint.getMaxCardinality(),
+    assertEquals(constraint.getMaxCardinality(),
         sProto.getMaxCardinality());
-    Assert.assertEquals(constraint.getTargetExpressions().size(),
+    assertEquals(constraint.getTargetExpressions().size(),
         sProto.getTargetExpressionsList().size());
   }
 
   private void compareSimpleConstraints(SingleConstraint single,
       SingleConstraint newSingle) {
-    Assert.assertEquals(single.getScope(), newSingle.getScope());
-    Assert.assertEquals(single.getMinCardinality(),
+    assertEquals(single.getScope(), newSingle.getScope());
+    assertEquals(single.getMinCardinality(),
         newSingle.getMinCardinality());
-    Assert.assertEquals(single.getMaxCardinality(),
+    assertEquals(single.getMaxCardinality(),
         newSingle.getMaxCardinality());
-    Assert.assertEquals(single.getTargetExpressions(),
+    assertEquals(single.getTargetExpressions(),
         newSingle.getTargetExpressions());
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
index c92e73f44fd40..f564932bd4023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
@@ -20,6 +20,10 @@
 
 import java.io.File;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -29,20 +33,19 @@
 import org.apache.hadoop.yarn.proto.YarnProtos;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test class to handle various proto related tests for resources.
  */
 public class TestResourcePBImpl {
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     ResourceUtils.resetResourceTypes();
 
@@ -51,7 +54,7 @@ public void setup() throws Exception {
     TestResourceUtils.setupResourceTypes(conf, resourceTypesFile);
   }
 
-  @After
+  @AfterEach
   public void teardown() {
     Configuration conf = new YarnConfiguration();
     File source = new File(
@@ -63,80 +66,80 @@ public void teardown() {
   }
 
   @Test
-  public void testEmptyResourcePBInit() throws Exception {
+  void testEmptyResourcePBInit() throws Exception {
     Resource res = new ResourcePBImpl();
     // Assert to check it sets resource value and unit to default.
-    Assert.assertEquals(0, res.getMemorySize());
-    Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(),
+    assertEquals(0, res.getMemorySize());
+    assertEquals(ResourceInformation.MEMORY_MB.getUnits(),
         res.getResourceInformation(ResourceInformation.MEMORY_MB.getName())
             .getUnits());
-    Assert.assertEquals(ResourceInformation.VCORES.getUnits(),
+    assertEquals(ResourceInformation.VCORES.getUnits(),
         res.getResourceInformation(ResourceInformation.VCORES.getName())
             .getUnits());
   }
 
   @Test
-  public void testResourcePBInitFromOldPB() throws Exception {
+  void testResourcePBInitFromOldPB() throws Exception {
     YarnProtos.ResourceProto proto =
         YarnProtos.ResourceProto.newBuilder().setMemory(1024).setVirtualCores(3)
             .build();
     // Assert to check it sets resource value and unit to default.
     Resource res = new ResourcePBImpl(proto);
-    Assert.assertEquals(1024, res.getMemorySize());
-    Assert.assertEquals(3, res.getVirtualCores());
-    Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(),
+    assertEquals(1024, res.getMemorySize());
+    assertEquals(3, res.getVirtualCores());
+    assertEquals(ResourceInformation.MEMORY_MB.getUnits(),
         res.getResourceInformation(ResourceInformation.MEMORY_MB.getName())
             .getUnits());
-    Assert.assertEquals(ResourceInformation.VCORES.getUnits(),
+    assertEquals(ResourceInformation.VCORES.getUnits(),
         res.getResourceInformation(ResourceInformation.VCORES.getName())
             .getUnits());
   }
 
   @Test
   @SuppressWarnings("deprecation")
-  public void testGetMemory() {
+  void testGetMemory() {
     Resource res = new ResourcePBImpl();
     long memorySize = Integer.MAX_VALUE + 1L;
     res.setMemorySize(memorySize);
 
-    assertEquals("No need to cast if both are long", memorySize,
-        res.getMemorySize());
-    assertEquals("Cast to Integer.MAX_VALUE if the long is greater than "
-            + "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getMemory());
+    assertEquals(memorySize, res.getMemorySize(), "No need to cast if both are long");
+    assertEquals(Integer.MAX_VALUE, res.getMemory(),
+        "Cast to Integer.MAX_VALUE if the long is greater than " + "Integer.MAX_VALUE");
   }
 
   @Test
-  public void testGetVirtualCores() {
+  void testGetVirtualCores() {
     Resource res = new ResourcePBImpl();
     long vcores = Integer.MAX_VALUE + 1L;
     res.getResourceInformation("vcores").setValue(vcores);
 
-    assertEquals("No need to cast if both are long", vcores,
-        res.getResourceInformation("vcores").getValue());
-    assertEquals("Cast to Integer.MAX_VALUE if the long is greater than "
-        + "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getVirtualCores());
+    assertEquals(vcores,
+        res.getResourceInformation("vcores").getValue(),
+        "No need to cast if both are long");
+    assertEquals(Integer.MAX_VALUE, res.getVirtualCores(),
+        "Cast to Integer.MAX_VALUE if the long is greater than " + "Integer.MAX_VALUE");
   }
 
   @Test
-  public void testResourcePBWithExtraResources() throws Exception {
+  void testResourcePBWithExtraResources() throws Exception {
 
     //Resource 'resource1' has been passed as 4T
     //4T should be converted to 4000G
     YarnProtos.ResourceInformationProto riProto =
         YarnProtos.ResourceInformationProto.newBuilder().setType(
             YarnProtos.ResourceTypeInfoProto.newBuilder().
-            setName("resource1").setType(
+                setName("resource1").setType(
                 YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
-        setValue(4).setUnits("T").setKey("resource1").build();
+            setValue(4).setUnits("T").setKey("resource1").build();
 
     YarnProtos.ResourceProto proto =
         YarnProtos.ResourceProto.newBuilder().setMemory(1024).
-        setVirtualCores(3).addResourceValueMap(riProto).build();
+            setVirtualCores(3).addResourceValueMap(riProto).build();
     Resource res = new ResourcePBImpl(proto);
 
-    Assert.assertEquals(4000,
+    assertEquals(4000,
         res.getResourceInformation("resource1").getValue());
-    Assert.assertEquals("G",
+    assertEquals("G",
         res.getResourceInformation("resource1").getUnits());
 
     //Resource 'resource2' has been passed as 4M
@@ -144,18 +147,18 @@ public void testResourcePBWithExtraResources() throws Exception {
     YarnProtos.ResourceInformationProto riProto1 =
         YarnProtos.ResourceInformationProto.newBuilder().setType(
             YarnProtos.ResourceTypeInfoProto.newBuilder().
-            setName("resource2").setType(
+                setName("resource2").setType(
                 YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
-        setValue(4).setUnits("M").setKey("resource2").build();
+            setValue(4).setUnits("M").setKey("resource2").build();
 
     YarnProtos.ResourceProto proto1 =
         YarnProtos.ResourceProto.newBuilder().setMemory(1024).
-        setVirtualCores(3).addResourceValueMap(riProto1).build();
+            setVirtualCores(3).addResourceValueMap(riProto1).build();
     Resource res1 = new ResourcePBImpl(proto1);
 
-    Assert.assertEquals(4000000000L,
+    assertEquals(4000000000L,
         res1.getResourceInformation("resource2").getValue());
-    Assert.assertEquals("m",
+    assertEquals("m",
         res1.getResourceInformation("resource2").getUnits());
 
     //Resource 'resource1' has been passed as 3M
@@ -163,23 +166,23 @@ public void testResourcePBWithExtraResources() throws Exception {
     YarnProtos.ResourceInformationProto riProto2 =
         YarnProtos.ResourceInformationProto.newBuilder().setType(
             YarnProtos.ResourceTypeInfoProto.newBuilder().
-            setName("resource1").setType(
+                setName("resource1").setType(
                 YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
-        setValue(3).setUnits("M").setKey("resource1").build();
+            setValue(3).setUnits("M").setKey("resource1").build();
 
     YarnProtos.ResourceProto proto2 =
         YarnProtos.ResourceProto.newBuilder().setMemory(1024).
-        setVirtualCores(3).addResourceValueMap(riProto2).build();
+            setVirtualCores(3).addResourceValueMap(riProto2).build();
     Resource res2 = new ResourcePBImpl(proto2);
 
-    Assert.assertEquals(0,
+    assertEquals(0,
         res2.getResourceInformation("resource1").getValue());
-    Assert.assertEquals("G",
+    assertEquals("G",
         res2.getResourceInformation("resource1").getUnits());
   }
 
   @Test
-  public void testResourceTags() {
+  void testResourceTags() {
     YarnProtos.ResourceInformationProto riProto =
         YarnProtos.ResourceInformationProto.newBuilder()
             .setType(
@@ -201,19 +204,19 @@ public void testResourceTags() {
             .build();
     Resource res = new ResourcePBImpl(proto);
 
-    Assert.assertNotNull(res.getResourceInformation("yarn.io/test-volume"));
-    Assert.assertEquals(10,
+    assertNotNull(res.getResourceInformation("yarn.io/test-volume"));
+    assertEquals(10,
         res.getResourceInformation("yarn.io/test-volume")
             .getValue());
-    Assert.assertEquals("G",
+    assertEquals("G",
         res.getResourceInformation("yarn.io/test-volume")
             .getUnits());
-    Assert.assertEquals(3,
+    assertEquals(3,
         res.getResourceInformation("yarn.io/test-volume")
             .getTags().size());
-    Assert.assertFalse(res.getResourceInformation("yarn.io/test-volume")
+    assertFalse(res.getResourceInformation("yarn.io/test-volume")
         .getTags().isEmpty());
-    Assert.assertTrue(res.getResourceInformation("yarn.io/test-volume")
+    assertTrue(res.getResourceInformation("yarn.io/test-volume")
         .getAttributes().isEmpty());
 
     boolean protoConvertExpected = false;
@@ -225,13 +228,13 @@ public void testResourceTags() {
             && pf.getTagsCount() == 3;
       }
     }
-    Assert.assertTrue("Expecting resource's protobuf message"
-        + " contains 0 attributes and 3 tags",
-        protoConvertExpected);
+    assertTrue(protoConvertExpected,
+        "Expecting resource's protobuf message"
+            + " contains 0 attributes and 3 tags");
   }
 
   @Test
-  public void testResourceAttributes() {
+  void testResourceAttributes() {
     YarnProtos.ResourceInformationProto riProto =
         YarnProtos.ResourceInformationProto.newBuilder()
             .setType(
@@ -260,19 +263,19 @@ public void testResourceAttributes() {
             .build();
     Resource res = new ResourcePBImpl(proto);
 
-    Assert.assertNotNull(res.getResourceInformation("yarn.io/test-volume"));
-    Assert.assertEquals(10,
+    assertNotNull(res.getResourceInformation("yarn.io/test-volume"));
+    assertEquals(10,
         res.getResourceInformation("yarn.io/test-volume")
             .getValue());
-    Assert.assertEquals("G",
+    assertEquals("G",
         res.getResourceInformation("yarn.io/test-volume")
             .getUnits());
-    Assert.assertEquals(2,
+    assertEquals(2,
         res.getResourceInformation("yarn.io/test-volume")
             .getAttributes().size());
-    Assert.assertTrue(res.getResourceInformation("yarn.io/test-volume")
+    assertTrue(res.getResourceInformation("yarn.io/test-volume")
         .getTags().isEmpty());
-    Assert.assertFalse(res.getResourceInformation("yarn.io/test-volume")
+    assertFalse(res.getResourceInformation("yarn.io/test-volume")
         .getAttributes().isEmpty());
 
     boolean protoConvertExpected = false;
@@ -284,20 +287,20 @@ public void testResourceAttributes() {
             && pf.getTagsCount() == 0;
       }
     }
-    Assert.assertTrue("Expecting resource's protobuf message"
-            + " contains 2 attributes and 0 tags",
-        protoConvertExpected);
+    assertTrue(protoConvertExpected,
+        "Expecting resource's protobuf message"
+            + " contains 2 attributes and 0 tags");
   }
 
   @Test
-  public void testParsingResourceTags() {
+  void testParsingResourceTags() {
     ResourceInformation info =
         ResourceUtils.getResourceTypes().get("resource3");
-    Assert.assertTrue(info.getAttributes().isEmpty());
-    Assert.assertFalse(info.getTags().isEmpty());
+    assertTrue(info.getAttributes().isEmpty());
+    assertFalse(info.getTags().isEmpty());
     assertThat(info.getTags()).hasSize(2);
     info.getTags().remove("resource3_tag_1");
     info.getTags().remove("resource3_tag_2");
-    Assert.assertTrue(info.getTags().isEmpty());
+    assertTrue(info.getTags().isEmpty());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
index aef838cd17ebf..d3f3fe90d48c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
@@ -17,13 +17,15 @@
  */
 package org.apache.hadoop.yarn.api;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 
 /**
  * The class to test {@link ResourceRequest}.
@@ -31,7 +33,7 @@
 public class TestResourceRequest {
 
   @Test
-  public void testEqualsOnExecutionTypeRequest() {
+  void testEqualsOnExecutionTypeRequest() {
     ResourceRequest resourceRequestA =
         ResourceRequest.newInstance(Priority.newInstance(0), "localhost",
             Resource.newInstance(1024, 1), 1, false, "",
@@ -42,6 +44,6 @@ public void testEqualsOnExecutionTypeRequest() {
             Resource.newInstance(1024, 1), 1, false, "",
             ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, false));
 
-    Assert.assertFalse(resourceRequestA.equals(resourceRequestB));
+    assertNotEquals(resourceRequestA, resourceRequestB);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestTimelineEntityGroupId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestTimelineEntityGroupId.java
index 55b149640d3b8..952b7fede764f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestTimelineEntityGroupId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestTimelineEntityGroupId.java
@@ -18,15 +18,20 @@
 
 package org.apache.hadoop.yarn.api;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestTimelineEntityGroupId {
 
   @Test
-  public void testTimelineEntityGroupId() {
+  void testTimelineEntityGroupId() {
     ApplicationId appId1 = ApplicationId.newInstance(1234, 1);
     ApplicationId appId2 = ApplicationId.newInstance(1234, 2);
     TimelineEntityGroupId group1 = TimelineEntityGroupId.newInstance(appId1, "1");
@@ -34,19 +39,19 @@ public void testTimelineEntityGroupId() {
     TimelineEntityGroupId group3 = TimelineEntityGroupId.newInstance(appId2, "1");
     TimelineEntityGroupId group4 = TimelineEntityGroupId.newInstance(appId1, "1");
 
-    Assert.assertTrue(group1.equals(group4));
-    Assert.assertFalse(group1.equals(group2));
-    Assert.assertFalse(group1.equals(group3));
+    assertEquals(group1, group4);
+    assertNotEquals(group1, group2);
+    assertNotEquals(group1, group3);
 
-    Assert.assertTrue(group1.compareTo(group4) == 0);
-    Assert.assertTrue(group1.compareTo(group2) < 0);
-    Assert.assertTrue(group1.compareTo(group3) < 0);
+    assertTrue(group1.compareTo(group4) == 0);
+    assertTrue(group1.compareTo(group2) < 0);
+    assertTrue(group1.compareTo(group3) < 0);
 
-    Assert.assertTrue(group1.hashCode() == group4.hashCode());
-    Assert.assertFalse(group1.hashCode() == group2.hashCode());
-    Assert.assertFalse(group1.hashCode() == group3.hashCode());
+    assertTrue(group1.hashCode() == group4.hashCode());
+    assertFalse(group1.hashCode() == group2.hashCode());
+    assertFalse(group1.hashCode() == group3.hashCode());
 
-    Assert.assertEquals("timelineEntityGroupId_1234_1_1", group1.toString());
-    Assert.assertEquals(TimelineEntityGroupId.fromString("timelineEntityGroupId_1234_1_1"), group1);
+    assertEquals("timelineEntityGroupId_1234_1_1", group1.toString());
+    assertEquals(TimelineEntityGroupId.fromString("timelineEntityGroupId_1234_1_1"), group1);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java
index 35f9aa54e6478..beea949b5e537 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java
@@ -18,57 +18,64 @@
 
 package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
 
-@RunWith(Parameterized.class)
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
 public class TestGetApplicationsRequestPBImpl {
-  @Parameter
   @SuppressWarnings("checkstyle:visibilitymodifier")
   public GetApplicationsRequestPBImpl impl;
 
-  @Test
-  public void testAppTagsLowerCaseConversionDefault() {
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertEquals(s, s.toLowerCase()));
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionDefault(
+      GetApplicationsRequestPBImpl applicationsRequestPBImpl) {
+    initTestGetApplicationsRequestPBImpl(applicationsRequestPBImpl);
+    applicationsRequestPBImpl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationsRequestPBImpl.getApplicationTags().forEach(s -> assertEquals(s, s.toLowerCase()));
   }
 
-  @Test
-  public void testAppTagsLowerCaseConversionDisabled() {
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionDisabled(
+      GetApplicationsRequestPBImpl applicationsRequestPBImpl) {
+    initTestGetApplicationsRequestPBImpl(applicationsRequestPBImpl);
     GetApplicationsRequestPBImpl.setForceLowerCaseTags(false);
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertNotEquals(s, s.toLowerCase()));
+    applicationsRequestPBImpl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationsRequestPBImpl.getApplicationTags()
+        .forEach(s -> assertNotEquals(s, s.toLowerCase()));
   }
 
-  @Test
-  public void testAppTagsLowerCaseConversionEnabled() {
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionEnabled(
+      GetApplicationsRequestPBImpl applicationsRequestPBImpl) {
+    initTestGetApplicationsRequestPBImpl(applicationsRequestPBImpl);
     GetApplicationsRequestPBImpl.setForceLowerCaseTags(true);
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertEquals(s, s.toLowerCase()));
+    applicationsRequestPBImpl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationsRequestPBImpl.getApplicationTags().forEach(s -> assertEquals(s, s.toLowerCase()));
   }
 
-  @Parameters
   public static Collection<Object[]> data() {
     List<Object[]> list = new ArrayList<>();
-    list.add(new Object[] {new GetApplicationsRequestPBImpl()});
-    list.add(new Object[] {new GetApplicationsRequestPBImpl(
+    list.add(new Object[]{new GetApplicationsRequestPBImpl()});
+    list.add(new Object[]{new GetApplicationsRequestPBImpl(
         GetApplicationsRequestProto.newBuilder().build())});
 
     return list;
   }
+
+  public void initTestGetApplicationsRequestPBImpl(
+      GetApplicationsRequestPBImpl applicationsRequestPBImpl) {
+    this.impl = impl;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
index a2b05708326b5..27c05e082e5b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/TestResourceUtilization.java
@@ -18,54 +18,58 @@
 
 package org.apache.hadoop.yarn.api.records;
 
-import org.junit.Assert;
-import org.junit.Test;
-
 import java.util.HashMap;
 import java.util.Map;
 
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 public class TestResourceUtilization {
 
   @Test
-  public void testResourceUtilization() {
+  void testResourceUtilization() {
     ResourceUtilization u1 = ResourceUtilization.newInstance(10, 20, 0.5f);
     ResourceUtilization u2 = ResourceUtilization.newInstance(u1);
     ResourceUtilization u3 = ResourceUtilization.newInstance(10, 20, 0.5f);
     ResourceUtilization u4 = ResourceUtilization.newInstance(20, 20, 0.5f);
     ResourceUtilization u5 = ResourceUtilization.newInstance(30, 40, 0.8f);
 
-    Assert.assertEquals(u1, u2);
-    Assert.assertEquals(u1, u3);
-    Assert.assertNotEquals(u1, u4);
-    Assert.assertNotEquals(u2, u5);
-    Assert.assertNotEquals(u4, u5);
+    assertEquals(u1, u2);
+    assertEquals(u1, u3);
+    assertNotEquals(u1, u4);
+    assertNotEquals(u2, u5);
+    assertNotEquals(u4, u5);
 
-    Assert.assertTrue(u1.hashCode() == u2.hashCode());
-    Assert.assertTrue(u1.hashCode() == u3.hashCode());
-    Assert.assertFalse(u1.hashCode() == u4.hashCode());
-    Assert.assertFalse(u2.hashCode() == u5.hashCode());
-    Assert.assertFalse(u4.hashCode() == u5.hashCode());
+    assertTrue(u1.hashCode() == u2.hashCode());
+    assertTrue(u1.hashCode() == u3.hashCode());
+    assertFalse(u1.hashCode() == u4.hashCode());
+    assertFalse(u2.hashCode() == u5.hashCode());
+    assertFalse(u4.hashCode() == u5.hashCode());
 
-    Assert.assertTrue(u1.getPhysicalMemory() == 10);
-    Assert.assertFalse(u1.getVirtualMemory() == 10);
-    Assert.assertTrue(u1.getCPU() == 0.5f);
+    assertTrue(u1.getPhysicalMemory() == 10);
+    assertFalse(u1.getVirtualMemory() == 10);
+    assertTrue(u1.getCPU() == 0.5f);
 
-    Assert.assertEquals("<pmem:10, vmem:" + u1.getVirtualMemory()
+    assertEquals("<pmem:10, vmem:" + u1.getVirtualMemory()
         + ", vCores:0.5>", u1.toString());
 
     u1.addTo(10, 0, 0.0f);
-    Assert.assertNotEquals(u1, u2);
-    Assert.assertEquals(u1, u4);
+    assertNotEquals(u1, u2);
+    assertEquals(u1, u4);
     u1.addTo(10, 20, 0.3f);
-    Assert.assertEquals(u1, u5);
+    assertEquals(u1, u5);
     u1.subtractFrom(10, 20, 0.3f);
-    Assert.assertEquals(u1, u4);
+    assertEquals(u1, u4);
     u1.subtractFrom(10, 0, 0.0f);
-    Assert.assertEquals(u1, u3);
+    assertEquals(u1, u3);
   }
 
   @Test
-  public void testResourceUtilizationWithCustomResource() {
+  void testResourceUtilizationWithCustomResource() {
     Map<String, Float> customResources = new HashMap<>();
     customResources.put(ResourceInformation.GPU_URI, 5.0f);
     ResourceUtilization u1 = ResourceUtilization.
@@ -78,35 +82,35 @@ public void testResourceUtilizationWithCustomResource() {
     ResourceUtilization u5 = ResourceUtilization.
         newInstance(30, 40, 0.8f, customResources);
 
-    Assert.assertEquals(u1, u2);
-    Assert.assertEquals(u1, u3);
-    Assert.assertNotEquals(u1, u4);
-    Assert.assertNotEquals(u2, u5);
-    Assert.assertNotEquals(u4, u5);
-
-    Assert.assertTrue(u1.hashCode() == u2.hashCode());
-    Assert.assertTrue(u1.hashCode() == u3.hashCode());
-    Assert.assertFalse(u1.hashCode() == u4.hashCode());
-    Assert.assertFalse(u2.hashCode() == u5.hashCode());
-    Assert.assertFalse(u4.hashCode() == u5.hashCode());
-
-    Assert.assertTrue(u1.getPhysicalMemory() == 10);
-    Assert.assertFalse(u1.getVirtualMemory() == 10);
-    Assert.assertTrue(u1.getCPU() == 0.5f);
-    Assert.assertTrue(u1.
+    assertEquals(u1, u2);
+    assertEquals(u1, u3);
+    assertNotEquals(u1, u4);
+    assertNotEquals(u2, u5);
+    assertNotEquals(u4, u5);
+
+    assertTrue(u1.hashCode() == u2.hashCode());
+    assertTrue(u1.hashCode() == u3.hashCode());
+    assertFalse(u1.hashCode() == u4.hashCode());
+    assertFalse(u2.hashCode() == u5.hashCode());
+    assertFalse(u4.hashCode() == u5.hashCode());
+
+    assertTrue(u1.getPhysicalMemory() == 10);
+    assertFalse(u1.getVirtualMemory() == 10);
+    assertTrue(u1.getCPU() == 0.5f);
+    assertTrue(u1.
         getCustomResource(ResourceInformation.GPU_URI) == 5.0f);
 
-    Assert.assertEquals("<pmem:10, vmem:" + u1.getVirtualMemory()
+    assertEquals("<pmem:10, vmem:" + u1.getVirtualMemory()
         + ", vCores:0.5, yarn.io/gpu:5.0>", u1.toString());
 
     u1.addTo(10, 0, 0.0f);
-    Assert.assertNotEquals(u1, u2);
-    Assert.assertEquals(u1, u4);
+    assertNotEquals(u1, u2);
+    assertEquals(u1, u4);
     u1.addTo(10, 20, 0.3f);
-    Assert.assertEquals(u1, u5);
+    assertEquals(u1, u5);
     u1.subtractFrom(10, 20, 0.3f);
-    Assert.assertEquals(u1, u4);
+    assertEquals(u1, u4);
     u1.subtractFrom(10, 0, 0.0f);
-    Assert.assertEquals(u1, u3);
+    assertEquals(u1, u3);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
index 6c51516434ebd..4ce6646a7161d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
@@ -25,6 +25,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.security.Credentials;
@@ -36,8 +38,10 @@
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestApplicationClientProtocolRecords {
 
@@ -47,7 +51,7 @@ public class TestApplicationClientProtocolRecords {
    *
    */
   @Test
-  public void testCLCPBImplNullEnv() throws IOException {
+  void testCLCPBImplNullEnv() throws IOException {
     Map<String, LocalResource> localResources = Collections.emptyMap();
     Map<String, String> environment = new HashMap<String, String>();
     List<String> commands = Collections.emptyList();
@@ -68,7 +72,7 @@ public void testCLCPBImplNullEnv() throws IOException {
     ContainerLaunchContext clcProto = new ContainerLaunchContextPBImpl(
         ((ContainerLaunchContextPBImpl) clc).getProto());
 
-    Assert.assertEquals("",
+    assertEquals("",
         clcProto.getEnvironment().get("testCLCPBImplNullEnv"));
 
   }
@@ -78,7 +82,7 @@ public void testCLCPBImplNullEnv() throws IOException {
    * local resource URL.
    */
   @Test
-  public void testCLCPBImplNullResourceURL() throws IOException {
+  void testCLCPBImplNullResourceURL() throws IOException {
     RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
     try {
       LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
@@ -92,9 +96,9 @@ public void testCLCPBImplNullResourceURL() throws IOException {
       localResources.put("null_url_resource", rsrc_alpha);
       ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
       containerLaunchContext.setLocalResources(localResources);
-      Assert.fail("Setting an invalid local resource should be an error!");
+      fail("Setting an invalid local resource should be an error!");
     } catch (NullPointerException e) {
-      Assert.assertTrue(e.getMessage().contains("Null resource URL for local resource"));
+      assertTrue(e.getMessage().contains("Null resource URL for local resource"));
     }
   }
 
@@ -103,7 +107,7 @@ public void testCLCPBImplNullResourceURL() throws IOException {
    * local resource type.
    */
   @Test
-  public void testCLCPBImplNullResourceType() throws IOException {
+  void testCLCPBImplNullResourceType() throws IOException {
     RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
     try {
       LocalResource resource = recordFactory.newRecordInstance(LocalResource.class);
@@ -117,9 +121,9 @@ public void testCLCPBImplNullResourceType() throws IOException {
       localResources.put("null_type_resource", resource);
       ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
       containerLaunchContext.setLocalResources(localResources);
-      Assert.fail("Setting an invalid local resource should be an error!");
+      fail("Setting an invalid local resource should be an error!");
     } catch (NullPointerException e) {
-      Assert.assertTrue(e.getMessage().contains("Null resource type for local resource"));
+      assertTrue(e.getMessage().contains("Null resource type for local resource"));
     }
   }
 
@@ -128,7 +132,7 @@ public void testCLCPBImplNullResourceType() throws IOException {
    * local resource type.
    */
   @Test
-  public void testCLCPBImplNullResourceVisibility() throws IOException {
+  void testCLCPBImplNullResourceVisibility() throws IOException {
     RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
     try {
       LocalResource resource = recordFactory.newRecordInstance(LocalResource.class);
@@ -142,9 +146,9 @@ public void testCLCPBImplNullResourceVisibility() throws IOException {
       localResources.put("null_visibility_resource", resource);
       ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
       containerLaunchContext.setLocalResources(localResources);
-      Assert.fail("Setting an invalid local resource should be an error!");
+      fail("Setting an invalid local resource should be an error!");
     } catch (NullPointerException e) {
-      Assert.assertTrue(e.getMessage().contains("Null resource visibility for local resource"));
+      assertTrue(e.getMessage().contains("Null resource visibility for local resource"));
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java
index b6b2dbb6fb339..806c0725bdbe7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java
@@ -18,57 +18,66 @@
 
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameter;
-import org.junit.runners.Parameterized.Parameters;
 
-@RunWith(Parameterized.class)
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
 public class TestApplicationSubmissionContextPBImpl {
-  @Parameter
   @SuppressWarnings("checkstyle:visibilitymodifier")
   public ApplicationSubmissionContextPBImpl impl;
 
-  @Test
-  public void testAppTagsLowerCaseConversionDefault() {
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertEquals(s, s.toLowerCase()));
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionDefault(
+      ApplicationSubmissionContextPBImpl applicationSubmissionContextPB) {
+    initTestApplicationSubmissionContextPBImpl(applicationSubmissionContextPB);
+    applicationSubmissionContextPB.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationSubmissionContextPB.getApplicationTags()
+        .forEach(s -> assertEquals(s, s.toLowerCase()));
   }
 
-  @Test
-  public void testAppTagsLowerCaseConversionDisabled() {
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionDisabled(
+      ApplicationSubmissionContextPBImpl applicationSubmissionContextPB) {
+    initTestApplicationSubmissionContextPBImpl(applicationSubmissionContextPB);
     ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(false);
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertNotEquals(s, s.toLowerCase()));
+    applicationSubmissionContextPB.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationSubmissionContextPB.getApplicationTags()
+        .forEach(s -> assertNotEquals(s, s.toLowerCase()));
   }
 
-  @Test
-  public void testAppTagsLowerCaseConversionEnabled() {
+  @MethodSource("data")
+  @ParameterizedTest
+  void testAppTagsLowerCaseConversionEnabled(
+      ApplicationSubmissionContextPBImpl applicationSubmissionContextPB) {
+    initTestApplicationSubmissionContextPBImpl(applicationSubmissionContextPB);
     ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(true);
-    impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
-    impl.getApplicationTags().forEach(s ->
-        assertEquals(s, s.toLowerCase()));
+    applicationSubmissionContextPB.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
+    applicationSubmissionContextPB.getApplicationTags()
+        .forEach(s -> assertEquals(s, s.toLowerCase()));
   }
 
-  @Parameters
   public static Collection<Object[]> data() {
     List<Object[]> list = new ArrayList<>();
-    list.add(new Object[] {new ApplicationSubmissionContextPBImpl()});
-    list.add(new Object[] {new ApplicationSubmissionContextPBImpl(
+    list.add(new Object[]{new ApplicationSubmissionContextPBImpl()});
+    list.add(new Object[]{new ApplicationSubmissionContextPBImpl(
         ApplicationSubmissionContextProto.newBuilder().build())});
 
     return list;
   }
+
+  public void initTestApplicationSubmissionContextPBImpl(
+      ApplicationSubmissionContextPBImpl applicationSubmissionContextPB) {
+    this.impl = applicationSubmissionContextPB;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestProtoUtils.java
index b3330694837d1..6d1f20f269e85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestProtoUtils.java
@@ -17,20 +17,21 @@
  */
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
-import static org.junit.Assert.*;
-
 import java.util.stream.Stream;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerSubState;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerSubStateProto;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestProtoUtils {
 
   @Test
-  public void testConvertFromOrToProtoFormat() {
+  void testConvertFromOrToProtoFormat() {
     // Check if utility has all enum values
     try {
       Stream.of(ContainerState.values())
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
index ecfa63e03293d..d4bfb318fed4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
@@ -20,71 +20,75 @@
 
 import java.nio.channels.ClosedChannelException;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestSerializedExceptionPBImpl {
   @Test
-  public void testSerializedException() throws Exception {
+  void testSerializedException() throws Exception {
     SerializedExceptionPBImpl orig = new SerializedExceptionPBImpl();
     orig.init(new Exception("test exception"));
     SerializedExceptionProto proto = orig.getProto();
     SerializedExceptionPBImpl deser = new SerializedExceptionPBImpl(proto);
-    Assert.assertEquals(orig, deser);
-    Assert.assertEquals(orig.getMessage(), deser.getMessage());
-    Assert.assertEquals(orig.getRemoteTrace(), deser.getRemoteTrace());
-    Assert.assertEquals(orig.getCause(), deser.getCause());
+    assertEquals(orig, deser);
+    assertEquals(orig.getMessage(), deser.getMessage());
+    assertEquals(orig.getRemoteTrace(), deser.getRemoteTrace());
+    assertEquals(orig.getCause(), deser.getCause());
   }
 
   @Test
-  public void testDeserialize() throws Exception {
+  void testDeserialize() throws Exception {
     Exception ex = new Exception("test exception");
     SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
 
     try {
       pb.deSerialize();
-      Assert.fail("deSerialze should throw YarnRuntimeException");
+      fail("deSerialze should throw YarnRuntimeException");
     } catch (YarnRuntimeException e) {
-      Assert.assertEquals(ClassNotFoundException.class,
+      assertEquals(ClassNotFoundException.class,
           e.getCause().getClass());
     }
 
     pb.init(ex);
-    Assert.assertEquals(ex.toString(), pb.deSerialize().toString());
+    assertEquals(ex.toString(), pb.deSerialize().toString());
   }
 
   @Test
-  public void testDeserializeWithDefaultConstructor() {
+  void testDeserializeWithDefaultConstructor() {
     // Init SerializedException with an Exception with default constructor.
     ClosedChannelException ex = new ClosedChannelException();
     SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
     pb.init(ex);
-    Assert.assertEquals(ex.getClass(), pb.deSerialize().getClass());
+    assertEquals(ex.getClass(), pb.deSerialize().getClass());
   }
 
   @Test
-  public void testBeforeInit() throws Exception {
+  void testBeforeInit() throws Exception {
     SerializedExceptionProto defaultProto =
         SerializedExceptionProto.newBuilder().build();
 
     SerializedExceptionPBImpl pb1 = new SerializedExceptionPBImpl();
-    Assert.assertNull(pb1.getCause());
+    assertNull(pb1.getCause());
 
     SerializedExceptionPBImpl pb2 = new SerializedExceptionPBImpl();
-    Assert.assertEquals(defaultProto, pb2.getProto());
+    assertEquals(defaultProto, pb2.getProto());
 
     SerializedExceptionPBImpl pb3 = new SerializedExceptionPBImpl();
-    Assert.assertEquals(defaultProto.getTrace(), pb3.getRemoteTrace());
+    assertEquals(defaultProto.getTrace(), pb3.getRemoteTrace());
   }
 
   @Test
-  public void testThrowableDeserialization() {
+  void testThrowableDeserialization() {
     // java.lang.Error should also be serializable
     Error ex = new Error();
     SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
     pb.init(ex);
-    Assert.assertEquals(ex.getClass(), pb.deSerialize().getClass());
+    assertEquals(ex.getClass(), pb.deSerialize().getClass());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
index 0de8200d63b77..7973f4ea3afe7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
@@ -27,12 +27,16 @@
 import java.util.TreeMap;
 import java.util.WeakHashMap;
 
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestTimelineRecords {
 
@@ -40,7 +44,7 @@ public class TestTimelineRecords {
       LoggerFactory.getLogger(TestTimelineRecords.class);
 
   @Test
-  public void testEntities() throws Exception {
+  void testEntities() throws Exception {
     TimelineEntities entities = new TimelineEntities();
     for (int j = 0; j < 2; ++j) {
       TimelineEntity entity = new TimelineEntity();
@@ -67,27 +71,27 @@ public void testEntities() throws Exception {
     LOG.info("Entities in JSON:");
     LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true));
 
-    Assert.assertEquals(2, entities.getEntities().size());
+    assertEquals(2, entities.getEntities().size());
     TimelineEntity entity1 = entities.getEntities().get(0);
-    Assert.assertEquals("entity id 0", entity1.getEntityId());
-    Assert.assertEquals("entity type 0", entity1.getEntityType());
-    Assert.assertEquals(2, entity1.getRelatedEntities().size());
-    Assert.assertEquals(2, entity1.getEvents().size());
-    Assert.assertEquals(2, entity1.getPrimaryFilters().size());
-    Assert.assertEquals(2, entity1.getOtherInfo().size());
-    Assert.assertEquals("domain id 0", entity1.getDomainId());
+    assertEquals("entity id 0", entity1.getEntityId());
+    assertEquals("entity type 0", entity1.getEntityType());
+    assertEquals(2, entity1.getRelatedEntities().size());
+    assertEquals(2, entity1.getEvents().size());
+    assertEquals(2, entity1.getPrimaryFilters().size());
+    assertEquals(2, entity1.getOtherInfo().size());
+    assertEquals("domain id 0", entity1.getDomainId());
     TimelineEntity entity2 = entities.getEntities().get(1);
-    Assert.assertEquals("entity id 1", entity2.getEntityId());
-    Assert.assertEquals("entity type 1", entity2.getEntityType());
-    Assert.assertEquals(2, entity2.getRelatedEntities().size());
-    Assert.assertEquals(2, entity2.getEvents().size());
-    Assert.assertEquals(2, entity2.getPrimaryFilters().size());
-    Assert.assertEquals(2, entity2.getOtherInfo().size());
-    Assert.assertEquals("domain id 1", entity2.getDomainId());
+    assertEquals("entity id 1", entity2.getEntityId());
+    assertEquals("entity type 1", entity2.getEntityType());
+    assertEquals(2, entity2.getRelatedEntities().size());
+    assertEquals(2, entity2.getEvents().size());
+    assertEquals(2, entity2.getPrimaryFilters().size());
+    assertEquals(2, entity2.getOtherInfo().size());
+    assertEquals("domain id 1", entity2.getDomainId());
   }
 
   @Test
-  public void testEvents() throws Exception {
+  void testEvents() throws Exception {
     TimelineEvents events = new TimelineEvents();
     for (int j = 0; j < 2; ++j) {
       TimelineEvents.EventsOfOneEntity partEvents =
@@ -107,31 +111,31 @@ public void testEvents() throws Exception {
     LOG.info("Events in JSON:");
     LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(events, true));
 
-    Assert.assertEquals(2, events.getAllEvents().size());
+    assertEquals(2, events.getAllEvents().size());
     TimelineEvents.EventsOfOneEntity partEvents1 = events.getAllEvents().get(0);
-    Assert.assertEquals("entity id 0", partEvents1.getEntityId());
-    Assert.assertEquals("entity type 0", partEvents1.getEntityType());
-    Assert.assertEquals(2, partEvents1.getEvents().size());
+    assertEquals("entity id 0", partEvents1.getEntityId());
+    assertEquals("entity type 0", partEvents1.getEntityType());
+    assertEquals(2, partEvents1.getEvents().size());
     TimelineEvent event11 = partEvents1.getEvents().get(0);
-    Assert.assertEquals("event type 0", event11.getEventType());
-    Assert.assertEquals(2, event11.getEventInfo().size());
+    assertEquals("event type 0", event11.getEventType());
+    assertEquals(2, event11.getEventInfo().size());
     TimelineEvent event12 = partEvents1.getEvents().get(1);
-    Assert.assertEquals("event type 1", event12.getEventType());
-    Assert.assertEquals(2, event12.getEventInfo().size());
+    assertEquals("event type 1", event12.getEventType());
+    assertEquals(2, event12.getEventInfo().size());
     TimelineEvents.EventsOfOneEntity partEvents2 = events.getAllEvents().get(1);
-    Assert.assertEquals("entity id 1", partEvents2.getEntityId());
-    Assert.assertEquals("entity type 1", partEvents2.getEntityType());
-    Assert.assertEquals(2, partEvents2.getEvents().size());
+    assertEquals("entity id 1", partEvents2.getEntityId());
+    assertEquals("entity type 1", partEvents2.getEntityType());
+    assertEquals(2, partEvents2.getEvents().size());
     TimelineEvent event21 = partEvents2.getEvents().get(0);
-    Assert.assertEquals("event type 0", event21.getEventType());
-    Assert.assertEquals(2, event21.getEventInfo().size());
+    assertEquals("event type 0", event21.getEventType());
+    assertEquals(2, event21.getEventInfo().size());
     TimelineEvent event22 = partEvents2.getEvents().get(1);
-    Assert.assertEquals("event type 1", event22.getEventType());
-    Assert.assertEquals(2, event22.getEventInfo().size());
+    assertEquals("event type 1", event22.getEventType());
+    assertEquals(2, event22.getEventInfo().size());
   }
 
   @Test
-  public void testTimelinePutErrors() throws Exception {
+  void testTimelinePutErrors() throws Exception {
     TimelinePutResponse TimelinePutErrors = new TimelinePutResponse();
     TimelinePutError error1 = new TimelinePutError();
     error1.setEntityId("entity id 1");
@@ -149,23 +153,23 @@ public void testTimelinePutErrors() throws Exception {
     LOG.info("Errors in JSON:");
     LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(TimelinePutErrors, true));
 
-    Assert.assertEquals(3, TimelinePutErrors.getErrors().size());
+    assertEquals(3, TimelinePutErrors.getErrors().size());
     TimelinePutError e = TimelinePutErrors.getErrors().get(0);
-    Assert.assertEquals(error1.getEntityId(), e.getEntityId());
-    Assert.assertEquals(error1.getEntityType(), e.getEntityType());
-    Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
+    assertEquals(error1.getEntityId(), e.getEntityId());
+    assertEquals(error1.getEntityType(), e.getEntityType());
+    assertEquals(error1.getErrorCode(), e.getErrorCode());
     e = TimelinePutErrors.getErrors().get(1);
-    Assert.assertEquals(error1.getEntityId(), e.getEntityId());
-    Assert.assertEquals(error1.getEntityType(), e.getEntityType());
-    Assert.assertEquals(error1.getErrorCode(), e.getErrorCode());
+    assertEquals(error1.getEntityId(), e.getEntityId());
+    assertEquals(error1.getEntityType(), e.getEntityType());
+    assertEquals(error1.getErrorCode(), e.getErrorCode());
     e = TimelinePutErrors.getErrors().get(2);
-    Assert.assertEquals(error2.getEntityId(), e.getEntityId());
-    Assert.assertEquals(error2.getEntityType(), e.getEntityType());
-    Assert.assertEquals(error2.getErrorCode(), e.getErrorCode());
+    assertEquals(error2.getEntityId(), e.getEntityId());
+    assertEquals(error2.getEntityType(), e.getEntityType());
+    assertEquals(error2.getErrorCode(), e.getErrorCode());
   }
 
   @Test
-  public void testTimelineDomain() throws Exception {
+  void testTimelineDomain() throws Exception {
     TimelineDomains domains = new TimelineDomains();
 
     TimelineDomain domain = null;
@@ -185,25 +189,25 @@ public void testTimelineDomain() throws Exception {
     LOG.info("Domain in JSON:");
     LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(domains, true));
 
-    Assert.assertEquals(2, domains.getDomains().size());
+    assertEquals(2, domains.getDomains().size());
 
     for (int i = 0; i < domains.getDomains().size(); ++i) {
       domain = domains.getDomains().get(i);
-      Assert.assertEquals("test id " + (i + 1), domain.getId());
-      Assert.assertEquals("test description " + (i + 1),
+      assertEquals("test id " + (i + 1), domain.getId());
+      assertEquals("test description " + (i + 1),
           domain.getDescription());
-      Assert.assertEquals("test owner " + (i + 1), domain.getOwner());
-      Assert.assertEquals("test_reader_user_" + (i + 1) +
+      assertEquals("test owner " + (i + 1), domain.getOwner());
+      assertEquals("test_reader_user_" + (i + 1) +
           " test_reader_group+" + (i + 1), domain.getReaders());
-      Assert.assertEquals("test_writer_user_" + (i + 1) +
+      assertEquals("test_writer_user_" + (i + 1) +
           " test_writer_group+" + (i + 1), domain.getWriters());
-      Assert.assertEquals(new Long(0L), domain.getCreatedTime());
-      Assert.assertEquals(new Long(1L), domain.getModifiedTime());
+      assertEquals(Long.valueOf(0L), domain.getCreatedTime());
+      assertEquals(Long.valueOf(1L), domain.getModifiedTime());
     }
   }
 
   @Test
-  public void testMapInterfaceOrTimelineRecords() throws Exception {
+  void testMapInterfaceOrTimelineRecords() throws Exception {
     TimelineEntity entity = new TimelineEntity();
     List<Map<String, Set<Object>>> primaryFiltersList =
         new ArrayList<Map<String, Set<Object>>>();
@@ -284,36 +288,36 @@ public void testMapInterfaceOrTimelineRecords() throws Exception {
   }
 
   private static void assertPrimaryFilters(TimelineEntity entity) {
-    Assert.assertNotNull(entity.getPrimaryFilters());
-    Assert.assertNotNull(entity.getPrimaryFiltersJAXB());
-    Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap);
-    Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap);
-    Assert.assertEquals(
+    assertNotNull(entity.getPrimaryFilters());
+    assertNotNull(entity.getPrimaryFiltersJAXB());
+    assertTrue(entity.getPrimaryFilters() instanceof HashMap);
+    assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap);
+    assertEquals(
         entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB());
   }
 
   private static void assertRelatedEntities(TimelineEntity entity) {
-    Assert.assertNotNull(entity.getRelatedEntities());
-    Assert.assertNotNull(entity.getRelatedEntitiesJAXB());
-    Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap);
-    Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap);
-    Assert.assertEquals(
+    assertNotNull(entity.getRelatedEntities());
+    assertNotNull(entity.getRelatedEntitiesJAXB());
+    assertTrue(entity.getRelatedEntities() instanceof HashMap);
+    assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap);
+    assertEquals(
         entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB());
   }
 
   private static void assertOtherInfo(TimelineEntity entity) {
-    Assert.assertNotNull(entity.getOtherInfo());
-    Assert.assertNotNull(entity.getOtherInfoJAXB());
-    Assert.assertTrue(entity.getOtherInfo() instanceof HashMap);
-    Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap);
-    Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB());
+    assertNotNull(entity.getOtherInfo());
+    assertNotNull(entity.getOtherInfoJAXB());
+    assertTrue(entity.getOtherInfo() instanceof HashMap);
+    assertTrue(entity.getOtherInfoJAXB() instanceof HashMap);
+    assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB());
   }
 
   private static void assertEventInfo(TimelineEvent event) {
-    Assert.assertNotNull(event);
-    Assert.assertNotNull(event.getEventInfoJAXB());
-    Assert.assertTrue(event.getEventInfo() instanceof HashMap);
-    Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap);
-    Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB());
+    assertNotNull(event);
+    assertNotNull(event.getEventInfoJAXB());
+    assertTrue(event.getEventInfo() instanceof HashMap);
+    assertTrue(event.getEventInfoJAXB() instanceof HashMap);
+    assertEquals(event.getEventInfo(), event.getEventInfoJAXB());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index b488a652ae261..b822835337a0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -17,22 +17,28 @@
  */
 package org.apache.hadoop.yarn.api.records.timelineservice;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.junit.Test;
-import org.junit.Assert;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 
 public class TestTimelineServiceRecords {
@@ -40,7 +46,7 @@ public class TestTimelineServiceRecords {
       LoggerFactory.getLogger(TestTimelineServiceRecords.class);
 
   @Test
-  public void testTimelineEntities() throws Exception {
+  void testTimelineEntities() throws Exception {
     TimelineEntity entity = new TimelineEntity();
     entity.setType("test type 1");
     entity.setId("test id 1");
@@ -48,7 +54,7 @@ public void testTimelineEntities() throws Exception {
     entity.addInfo("test info key 2",
         Arrays.asList("test info value 2", "test info value 3"));
     entity.addInfo("test info key 3", true);
-    Assert.assertTrue(
+    assertTrue(
         entity.getInfo().get("test info key 3") instanceof Boolean);
     entity.addConfig("test config key 1", "test config value 1");
     entity.addConfig("test config key 2", "test config value 2");
@@ -59,43 +65,43 @@ public void testTimelineEntities() throws Exception {
     metric1.addValue(1L, 1.0F);
     metric1.addValue(3L, 3.0D);
     metric1.addValue(2L, 2);
-    Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
+    assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
     Iterator<Map.Entry<Long, Number>> itr =
         metric1.getValues().entrySet().iterator();
     Map.Entry<Long, Number> entry = itr.next();
-    Assert.assertEquals(new Long(3L), entry.getKey());
-    Assert.assertEquals(3.0D, entry.getValue());
+    assertEquals(Long.valueOf(3L), entry.getKey());
+    assertEquals(3.0D, entry.getValue());
     entry = itr.next();
-    Assert.assertEquals(new Long(2L), entry.getKey());
-    Assert.assertEquals(2, entry.getValue());
+    assertEquals(Long.valueOf(2L), entry.getKey());
+    assertEquals(2, entry.getValue());
     entry = itr.next();
-    Assert.assertEquals(new Long(1L), entry.getKey());
-    Assert.assertEquals(1.0F, entry.getValue());
-    Assert.assertFalse(itr.hasNext());
+    assertEquals(Long.valueOf(1L), entry.getKey());
+    assertEquals(1.0F, entry.getValue());
+    assertFalse(itr.hasNext());
     entity.addMetric(metric1);
 
     TimelineMetric metric2 =
         new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
     metric2.setId("test metric id 1");
     metric2.addValue(3L, (short) 3);
-    Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
-    Assert.assertTrue(
+    assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
+    assertTrue(
         metric2.getValues().values().iterator().next() instanceof Short);
     Map<Long, Number> points = new HashMap<>();
     points.put(4L, 4.0D);
     points.put(5L, 5.0D);
     try {
       metric2.setValues(points);
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "Values cannot contain more than one point in"));
     }
     try {
       metric2.addValues(points);
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "Values cannot contain more than one point in"));
     }
     entity.addMetric(metric2);
@@ -104,9 +110,8 @@ public void testTimelineEntities() throws Exception {
         new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
     metric3.setId("test metric id 1");
     metric3.addValue(4L, (short) 4);
-    Assert.assertEquals("metric3 should equal to metric2! ", metric3, metric2);
-    Assert.assertNotEquals("metric1 should not equal to metric2! ",
-        metric1, metric2);
+    assertEquals(metric3, metric2, "metric3 should equal to metric2! ");
+    assertNotEquals(metric1, metric2, "metric1 should not equal to metric2! ");
 
     TimelineEvent event1 = new TimelineEvent();
     event1.setId("test event id 1");
@@ -114,7 +119,7 @@ public void testTimelineEntities() throws Exception {
     event1.addInfo("test info key 2",
         Arrays.asList("test info value 2", "test info value 3"));
     event1.addInfo("test info key 3", true);
-    Assert.assertTrue(
+    assertTrue(
         event1.getInfo().get("test info key 3") instanceof Boolean);
     event1.setTimestamp(1L);
     entity.addEvent(event1);
@@ -125,19 +130,17 @@ public void testTimelineEntities() throws Exception {
     event2.addInfo("test info key 2",
         Arrays.asList("test info value 2", "test info value 3"));
     event2.addInfo("test info key 3", true);
-    Assert.assertTrue(
+    assertTrue(
         event2.getInfo().get("test info key 3") instanceof Boolean);
     event2.setTimestamp(2L);
     entity.addEvent(event2);
 
-    Assert.assertFalse("event1 should not equal to event2! ",
-        event1.equals(event2));
+    assertNotEquals(event1, event2);
     TimelineEvent event3 = new TimelineEvent();
     event3.setId("test event id 1");
     event3.setTimestamp(1L);
-    Assert.assertEquals("event1 should equal to event3! ", event3, event1);
-    Assert.assertNotEquals("event1 should not equal to event2! ",
-        event1, event2);
+    assertEquals(event3, event1, "event1 should equal to event3! ");
+    assertNotEquals(event1, event2, "event1 should not equal to event2! ");
 
     entity.setCreatedTime(0L);
     entity.addRelatesToEntity("test type 2", "test id 2");
@@ -153,25 +156,22 @@ public void testTimelineEntities() throws Exception {
     entities.addEntity(entity2);
     LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true));
 
-    Assert.assertFalse("entity 1 should not be valid without type and id",
-        entity1.isValid());
+    assertFalse(entity1.isValid(),
+        "entity 1 should not be valid without type and id");
     entity1.setId("test id 2");
     entity1.setType("test type 2");
     entity2.setId("test id 1");
     entity2.setType("test type 1");
 
-    Assert.assertEquals("Timeline entity should equal to entity2! ",
-        entity, entity2);
-    Assert.assertNotEquals("entity1 should not equal to entity! ",
-        entity1, entity);
-    Assert.assertEquals("entity should be less than entity1! ",
-        entity1.compareTo(entity), 1);
-    Assert.assertEquals("entity's hash code should be -28727840 but not "
-        + entity.hashCode(), entity.hashCode(), -28727840);
+    assertEquals(entity, entity2, "Timeline entity should equal to entity2! ");
+    assertNotEquals(entity1, entity, "entity1 should not equal to entity! ");
+    assertEquals(entity1.compareTo(entity), 1, "entity should be less than entity1! ");
+    assertEquals(entity.hashCode(), -28727840, "entity's hash code should be -28727840 but not "
+        + entity.hashCode());
   }
 
   @Test
-  public void testFirstClassCitizenEntities() throws Exception {
+  void testFirstClassCitizenEntities() throws Exception {
     UserEntity user = new UserEntity();
     user.setId("test user id");
 
@@ -245,49 +245,49 @@ public void testFirstClassCitizenEntities() throws Exception {
 
 
     // Check parent/children APIs
-    Assert.assertNotNull(app1.getParent());
-    Assert.assertEquals(flow2.getType(), app1.getParent().getType());
-    Assert.assertEquals(flow2.getId(), app1.getParent().getId());
+    assertNotNull(app1.getParent());
+    assertEquals(flow2.getType(), app1.getParent().getType());
+    assertEquals(flow2.getId(), app1.getParent().getId());
     app1.addInfo(ApplicationEntity.PARENT_INFO_KEY, "invalid parent object");
     try {
       app1.getParent();
-      Assert.fail();
+      fail();
     } catch (Exception e) {
-      Assert.assertTrue(e instanceof YarnRuntimeException);
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e instanceof YarnRuntimeException);
+      assertTrue(e.getMessage().contains(
           "Parent info is invalid identifier object"));
     }
 
-    Assert.assertNotNull(app1.getChildren());
-    Assert.assertEquals(1, app1.getChildren().size());
-    Assert.assertEquals(
+    assertNotNull(app1.getChildren());
+    assertEquals(1, app1.getChildren().size());
+    assertEquals(
         appAttempt.getType(), app1.getChildren().iterator().next().getType());
-    Assert.assertEquals(
+    assertEquals(
         appAttempt.getId(), app1.getChildren().iterator().next().getId());
     app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY,
         Collections.singletonList("invalid children set"));
     try {
       app1.getChildren();
-      Assert.fail();
+      fail();
     } catch (Exception e) {
-      Assert.assertTrue(e instanceof YarnRuntimeException);
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e instanceof YarnRuntimeException);
+      assertTrue(e.getMessage().contains(
           "Children info is invalid identifier set"));
     }
     app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY,
         Collections.singleton("invalid child object"));
     try {
       app1.getChildren();
-      Assert.fail();
+      fail();
     } catch (Exception e) {
-      Assert.assertTrue(e instanceof YarnRuntimeException);
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e instanceof YarnRuntimeException);
+      assertTrue(e.getMessage().contains(
           "Children info contains invalid identifier object"));
     }
   }
 
   @Test
-  public void testUser() throws Exception {
+  void testUser() throws Exception {
     UserEntity user = new UserEntity();
     user.setId("test user id");
     user.addInfo("test info key 1", "test info value 1");
@@ -296,7 +296,7 @@ public void testUser() throws Exception {
   }
 
   @Test
-  public void testQueue() throws Exception {
+  void testQueue() throws Exception {
     QueueEntity queue = new QueueEntity();
     queue.setId("test queue id");
     queue.addInfo("test info key 1", "test info value 1");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
index 557b82fe257c3..0c40fa924a2bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -18,18 +18,12 @@
 
 package org.apache.hadoop.yarn.api.resource;
 
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
-import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
-
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
@@ -39,8 +33,16 @@
 import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
 import org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test class for {@link PlacementConstraintTransformations}.
@@ -48,10 +50,10 @@
 public class TestPlacementConstraintTransformations {
 
   @Test
-  public void testTargetConstraint() {
+  void testTargetConstraint() {
     AbstractConstraint sConstraintExpr =
         targetIn(NODE, allocationTag("hbase-m"));
-    Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+    assertTrue(sConstraintExpr instanceof SingleConstraint);
     PlacementConstraint sConstraint =
         PlacementConstraints.build(sConstraintExpr);
 
@@ -61,17 +63,17 @@ public void testTargetConstraint() {
     PlacementConstraint tConstraint = specTransformer.transform();
 
     AbstractConstraint tConstraintExpr = tConstraint.getConstraintExpr();
-    Assert.assertTrue(tConstraintExpr instanceof TargetConstraint);
+    assertTrue(tConstraintExpr instanceof TargetConstraint);
 
     SingleConstraint single = (SingleConstraint) sConstraintExpr;
     TargetConstraint target = (TargetConstraint) tConstraintExpr;
 
     // Make sure the expression string is consistent
     // before and after transforming
-    Assert.assertEquals(single.toString(), target.toString());
-    Assert.assertEquals(single.getScope(), target.getScope());
-    Assert.assertEquals(TargetOperator.IN, target.getOp());
-    Assert.assertEquals(single.getTargetExpressions(),
+    assertEquals(single.toString(), target.toString());
+    assertEquals(single.getScope(), target.getScope());
+    assertEquals(TargetOperator.IN, target.getOp());
+    assertEquals(single.getTargetExpressions(),
         target.getTargetExpressions());
 
     // Transform from specialized TargetConstraint to SimpleConstraint
@@ -80,18 +82,18 @@ public void testTargetConstraint() {
     sConstraint = singleTransformer.transform();
 
     sConstraintExpr = sConstraint.getConstraintExpr();
-    Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+    assertTrue(sConstraintExpr instanceof SingleConstraint);
 
     single = (SingleConstraint) sConstraintExpr;
-    Assert.assertEquals(target.getScope(), single.getScope());
-    Assert.assertEquals(1, single.getMinCardinality());
-    Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
-    Assert.assertEquals(single.getTargetExpressions(),
+    assertEquals(target.getScope(), single.getScope());
+    assertEquals(1, single.getMinCardinality());
+    assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
+    assertEquals(single.getTargetExpressions(),
         target.getTargetExpressions());
   }
 
   @Test
-  public void testCardinalityConstraint() {
+  void testCardinalityConstraint() {
     CardinalityConstraint cardinality = new CardinalityConstraint(RACK, 3, 10,
         new HashSet<>(Arrays.asList("hb")));
     PlacementConstraint cConstraint = PlacementConstraints.build(cardinality);
@@ -102,27 +104,27 @@ public void testCardinalityConstraint() {
     PlacementConstraint sConstraint = singleTransformer.transform();
 
     AbstractConstraint sConstraintExpr = sConstraint.getConstraintExpr();
-    Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+    assertTrue(sConstraintExpr instanceof SingleConstraint);
 
     SingleConstraint single = (SingleConstraint) sConstraintExpr;
     // Make sure the consistent expression string is consistent
     // before and after transforming
-    Assert.assertEquals(single.toString(), cardinality.toString());
-    Assert.assertEquals(cardinality.getScope(), single.getScope());
-    Assert.assertEquals(cardinality.getMinCardinality(),
+    assertEquals(single.toString(), cardinality.toString());
+    assertEquals(cardinality.getScope(), single.getScope());
+    assertEquals(cardinality.getMinCardinality(),
         single.getMinCardinality());
-    Assert.assertEquals(cardinality.getMaxCardinality(),
+    assertEquals(cardinality.getMaxCardinality(),
         single.getMaxCardinality());
-    Assert.assertEquals(
+    assertEquals(
         new HashSet<>(Arrays.asList(PlacementTargets.allocationTag("hb"))),
         single.getTargetExpressions());
   }
 
   @Test
-  public void testTargetCardinalityConstraint() {
+  void testTargetCardinalityConstraint() {
     AbstractConstraint constraintExpr =
         targetCardinality(RACK, 3, 10, allocationTag("zk"));
-    Assert.assertTrue(constraintExpr instanceof SingleConstraint);
+    assertTrue(constraintExpr instanceof SingleConstraint);
     PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
 
     // Apply transformation. Should be a no-op.
@@ -131,19 +133,19 @@ public void testTargetCardinalityConstraint() {
     PlacementConstraint newConstraint = specTransformer.transform();
 
     // The constraint expression should be the same.
-    Assert.assertEquals(constraintExpr, newConstraint.getConstraintExpr());
+    assertEquals(constraintExpr, newConstraint.getConstraintExpr());
   }
 
   @Test
-  public void testCompositeConstraint() {
+  void testCompositeConstraint() {
     AbstractConstraint constraintExpr =
         or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
             targetCardinality(RACK, 2, 10, allocationTag("zk")));
-    Assert.assertTrue(constraintExpr instanceof Or);
+    assertTrue(constraintExpr instanceof Or);
     PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
     Or orExpr = (Or) constraintExpr;
     for (AbstractConstraint child : orExpr.getChildren()) {
-      Assert.assertTrue(child instanceof SingleConstraint);
+      assertTrue(child instanceof SingleConstraint);
     }
 
     // Apply transformation. Should transform target and cardinality constraints
@@ -154,19 +156,19 @@ public void testCompositeConstraint() {
 
     Or specOrExpr = (Or) specConstraint.getConstraintExpr();
     List<AbstractConstraint> specChildren = specOrExpr.getChildren();
-    Assert.assertEquals(3, specChildren.size());
-    Assert.assertTrue(specChildren.get(0) instanceof TargetConstraint);
-    Assert.assertTrue(specChildren.get(1) instanceof SingleConstraint);
-    Assert.assertTrue(specChildren.get(2) instanceof SingleConstraint);
+    assertEquals(3, specChildren.size());
+    assertTrue(specChildren.get(0) instanceof TargetConstraint);
+    assertTrue(specChildren.get(1) instanceof SingleConstraint);
+    assertTrue(specChildren.get(2) instanceof SingleConstraint);
 
     // Transform from specialized TargetConstraint to SimpleConstraint
     SingleConstraintTransformer singleTransformer =
         new SingleConstraintTransformer(specConstraint);
     PlacementConstraint simConstraint = singleTransformer.transform();
-    Assert.assertTrue(simConstraint.getConstraintExpr() instanceof Or);
+    assertTrue(simConstraint.getConstraintExpr() instanceof Or);
     Or simOrExpr = (Or) specConstraint.getConstraintExpr();
     for (AbstractConstraint child : simOrExpr.getChildren()) {
-      Assert.assertTrue(child instanceof SingleConstraint);
+      assertTrue(child instanceof SingleConstraint);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
index 6c31fea7d563f..b5e86779cd270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/TestClientRMProxy.java
@@ -18,6 +18,12 @@
 
 package org.apache.hadoop.yarn.client;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -29,20 +35,15 @@
 import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.util.Records;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestClientRMProxy {
   @Test
-  public void testGetRMDelegationTokenService() {
+  void testGetRMDelegationTokenService() {
     String defaultRMAddress = YarnConfiguration.DEFAULT_RM_ADDRESS;
     YarnConfiguration conf = new YarnConfiguration();
 
@@ -51,8 +52,8 @@ public void testGetRMDelegationTokenService() {
     String[] services = tokenService.toString().split(",");
     assertEquals(1, services.length);
     for (String service : services) {
-      assertTrue("Incorrect token service name",
-          service.contains(defaultRMAddress));
+      assertTrue(service.contains(defaultRMAddress),
+          "Incorrect token service name");
     }
 
     // HA is enabled
@@ -66,13 +67,13 @@ public void testGetRMDelegationTokenService() {
     services = tokenService.toString().split(",");
     assertEquals(2, services.length);
     for (String service : services) {
-      assertTrue("Incorrect token service name",
-          service.contains(defaultRMAddress));
+      assertTrue(service.contains(defaultRMAddress),
+          "Incorrect token service name");
     }
   }
 
   @Test
-  public void testGetAMRMTokenService() {
+  void testGetAMRMTokenService() {
     String defaultRMAddress = YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS;
     YarnConfiguration conf = new YarnConfiguration();
 
@@ -81,8 +82,8 @@ public void testGetAMRMTokenService() {
     String[] services = tokenService.toString().split(",");
     assertEquals(1, services.length);
     for (String service : services) {
-      assertTrue("Incorrect token service name",
-          service.contains(defaultRMAddress));
+      assertTrue(service.contains(defaultRMAddress),
+          "Incorrect token service name");
     }
 
     // HA is enabled
@@ -96,8 +97,8 @@ public void testGetAMRMTokenService() {
     services = tokenService.toString().split(",");
     assertEquals(2, services.length);
     for (String service : services) {
-      assertTrue("Incorrect token service name",
-          service.contains(defaultRMAddress));
+      assertTrue(service.contains(defaultRMAddress),
+          "Incorrect token service name");
     }
   }
 
@@ -109,7 +110,7 @@ public void testGetAMRMTokenService() {
    * @throws Exception an Exception occurred
    */
   @Test
-  public void testProxyUserCorrectUGI() throws Exception {
+  void testProxyUserCorrectUGI() throws Exception {
     final YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
     conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
@@ -129,7 +130,7 @@ public void testProxyUserCorrectUGI() throws Exception {
     UserGroupInformation realUser = UserGroupInformation.getCurrentUser();
     UserGroupInformation proxyUser =
         UserGroupInformation.createProxyUserForTesting("proxy", realUser,
-        new String[] {"group1"});
+            new String[]{"group1"});
 
     // Create the RMProxy using the proxyUser
     ApplicationClientProtocol rmProxy = proxyUser.doAs(
@@ -163,7 +164,7 @@ private void assertUGI() throws IOException {
         UGICapturingHadoopYarnProtoRPC.lastCurrentUser;
     assertNotNull(lastCurrentUser);
     assertEquals("proxy", lastCurrentUser.getShortUserName());
-    Assert.assertEquals(UserGroupInformation.AuthenticationMethod.PROXY,
+    assertEquals(UserGroupInformation.AuthenticationMethod.PROXY,
         lastCurrentUser.getAuthenticationMethod());
     assertEquals(UserGroupInformation.getCurrentUser(),
         lastCurrentUser.getRealUser());
@@ -187,7 +188,7 @@ public Object getProxy(Class protocol, InetSocketAddress addr,
       try {
         currentUser = UserGroupInformation.getCurrentUser();
       } catch (IOException ioe) {
-        Assert.fail("Unable to get current user\n"
+        fail("Unable to get current user\n"
             + StringUtils.stringifyException(ioe));
       }
       lastCurrentUser = currentUser;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index e7110dda9ffcc..507cac61332d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -18,22 +18,19 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.times;
-
 import java.io.IOException;
 import java.net.ConnectException;
 import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.http.HttpConfig.Policy;
@@ -42,7 +39,6 @@
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
-import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_MONITORING_THREAD_NAME;
 import org.apache.hadoop.test.TestGenericTestUtils;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -52,14 +48,22 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_MONITORING_THREAD_NAME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TestTimelineClient {
 
@@ -68,7 +72,7 @@ public class TestTimelineClient {
   private String keystoresDir;
   private String sslConfDir;
 
-  @Before
+  @BeforeEach
   public void setup() {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -76,7 +80,7 @@ public void setup() {
     client = createTimelineClient(conf);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     if (client != null) {
       client.stop();
@@ -87,113 +91,113 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testPostEntities() throws Exception {
+  void testPostEntities() throws Exception {
     mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK,
-      false, false);
+        false, false);
     try {
       TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(0, response.getErrors().size());
+      assertEquals(0, response.getErrors().size());
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
   }
 
   @Test
-  public void testPostEntitiesWithError() throws Exception {
+  void testPostEntitiesWithError() throws Exception {
     mockEntityClientResponse(spyTimelineWriter, ClientResponse.Status.OK, true,
-      false);
+        false);
     try {
       TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(1, response.getErrors().size());
-      Assert.assertEquals("test entity id", response.getErrors().get(0)
+      assertEquals(1, response.getErrors().size());
+      assertEquals("test entity id", response.getErrors().get(0)
           .getEntityId());
-      Assert.assertEquals("test entity type", response.getErrors().get(0)
+      assertEquals("test entity type", response.getErrors().get(0)
           .getEntityType());
-      Assert.assertEquals(TimelinePutResponse.TimelinePutError.IO_EXCEPTION,
+      assertEquals(TimelinePutResponse.TimelinePutError.IO_EXCEPTION,
           response.getErrors().get(0).getErrorCode());
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
   }
 
   @Test
-  public void testPostIncompleteEntities() throws Exception {
+  void testPostIncompleteEntities() throws Exception {
     try {
       client.putEntities(new TimelineEntity());
-      Assert.fail("Exception should have been thrown");
+      fail("Exception should have been thrown");
     } catch (YarnException e) {
     }
   }
 
   @Test
-  public void testPostEntitiesNoResponse() throws Exception {
+  void testPostEntitiesNoResponse() throws Exception {
     mockEntityClientResponse(spyTimelineWriter,
-      ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
+        ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
     try {
       client.putEntities(generateEntity());
-      Assert.fail("Exception is expected");
+      fail("Exception is expected");
     } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "Failed to get the response from the timeline server."));
     }
   }
 
   @Test
-  public void testPostEntitiesConnectionRefused() throws Exception {
+  void testPostEntitiesConnectionRefused() throws Exception {
     mockEntityClientResponse(spyTimelineWriter, null, false, true);
     try {
       client.putEntities(generateEntity());
-      Assert.fail("RuntimeException is expected");
+      fail("RuntimeException is expected");
     } catch (RuntimeException re) {
-      Assert.assertTrue(re instanceof ClientHandlerException);
+      assertTrue(re instanceof ClientHandlerException);
     }
   }
 
   @Test
-  public void testPutDomain() throws Exception {
+  void testPutDomain() throws Exception {
     mockDomainClientResponse(spyTimelineWriter, ClientResponse.Status.OK, false);
     try {
       client.putDomain(generateDomain());
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
   }
 
   @Test
-  public void testPutDomainNoResponse() throws Exception {
+  void testPutDomainNoResponse() throws Exception {
     mockDomainClientResponse(spyTimelineWriter,
         ClientResponse.Status.FORBIDDEN, false);
     try {
       client.putDomain(generateDomain());
-      Assert.fail("Exception is expected");
+      fail("Exception is expected");
     } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "Failed to get the response from the timeline server."));
     }
   }
 
   @Test
-  public void testPutDomainConnectionRefused() throws Exception {
+  void testPutDomainConnectionRefused() throws Exception {
     mockDomainClientResponse(spyTimelineWriter, null, true);
     try {
       client.putDomain(generateDomain());
-      Assert.fail("RuntimeException is expected");
+      fail("RuntimeException is expected");
     } catch (RuntimeException re) {
-      Assert.assertTrue(re instanceof ClientHandlerException);
+      assertTrue(re instanceof ClientHandlerException);
     }
   }
 
   @Test
-  public void testCheckRetryCount() throws Exception {
+  void testCheckRetryCount() throws Exception {
     try {
       YarnConfiguration conf = new YarnConfiguration();
       conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
       conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
-        -2);
+          -2);
       createTimelineClient(conf);
-      Assert.fail();
-    } catch(IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES));
     }
 
@@ -201,46 +205,46 @@ public void testCheckRetryCount() throws Exception {
       YarnConfiguration conf = new YarnConfiguration();
       conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
       conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
-        0);
+          0);
       createTimelineClient(conf);
-      Assert.fail();
-    } catch(IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS));
     }
     int newMaxRetries = 5;
     long newIntervalMs = 500;
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
-      newMaxRetries);
+        newMaxRetries);
     conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
-      newIntervalMs);
+        newIntervalMs);
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     TimelineClientImpl client = createTimelineClient(conf);
     try {
       // This call should fail because there is no timeline server
       client.putEntities(generateEntity());
-      Assert.fail("Exception expected! "
+      fail("Exception expected! "
           + "Timeline server should be off to run this test. ");
     } catch (RuntimeException ce) {
-      Assert.assertTrue(
-          "Handler exception for reason other than retry: " + ce.getMessage(),
-          ce.getMessage().contains("Connection retries limit exceeded"));
+      assertTrue(
+          ce.getMessage().contains("Connection retries limit exceeded"),
+          "Handler exception for reason other than retry: " + ce.getMessage());
       // we would expect this exception here, check if the client has retried
-      Assert.assertTrue("Retry filter didn't perform any retries! ",
-          client.connector.connectionRetry.getRetired());
+      assertTrue(client.connector.connectionRetry.getRetired(),
+          "Retry filter didn't perform any retries! ");
     }
   }
 
   @Test
-  public void testDelegationTokenOperationsRetry() throws Exception {
+  void testDelegationTokenOperationsRetry() throws Exception {
     int newMaxRetries = 5;
     long newIntervalMs = 500;
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
-      newMaxRetries);
+        newMaxRetries);
     conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
-      newIntervalMs);
+        newIntervalMs);
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     // use kerberos to bypass the issue in HADOOP-11215
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -260,7 +264,7 @@ public void testDelegationTokenOperationsRetry() throws Exception {
       try {
         // try getting a delegation token
         client.getDelegationToken(
-          UserGroupInformation.getCurrentUser().getShortUserName());
+            UserGroupInformation.getCurrentUser().getShortUserName());
         assertFail();
       } catch (RuntimeException ce) {
         assertException(client, ce);
@@ -323,7 +327,7 @@ public void testDelegationTokenOperationsRetry() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testDelegationTokenDisabledOnSimpleAuth() throws Exception {
+  void testDelegationTokenDisabledOnSimpleAuth() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, "simple");
@@ -336,15 +340,15 @@ public void testDelegationTokenDisabledOnSimpleAuth() throws Exception {
       // try getting a delegation token
       Token<TimelineDelegationTokenIdentifier> identifierToken =
           tClient.getDelegationToken(
-          UserGroupInformation.getCurrentUser().getShortUserName());
+              UserGroupInformation.getCurrentUser().getShortUserName());
       // Get a null token when using simple auth
-      Assert.assertNull(identifierToken);
+      assertNull(identifierToken);
 
       // try renew a delegation token
       Token<TimelineDelegationTokenIdentifier> dummyToken = new Token<>();
       long renewTime = tClient.renewDelegationToken(dummyToken);
       // Get invalid expiration time so that RM skips renewal
-      Assert.assertEquals(renewTime, -1);
+      assertEquals(renewTime, -1);
 
       // try cancel a delegation token
       tClient.cancelDelegationToken(dummyToken);
@@ -356,17 +360,16 @@ public void testDelegationTokenDisabledOnSimpleAuth() throws Exception {
   }
 
   private static void assertFail() {
-    Assert.fail("Exception expected! "
+    fail("Exception expected! "
         + "Timeline server should be off to run this test.");
   }
 
   private void assertException(TimelineClientImpl client, RuntimeException ce) {
-    Assert.assertTrue(
-        "Handler exception for reason other than retry: " + ce.toString(), ce
-            .getMessage().contains("Connection retries limit exceeded"));
+    assertTrue(ce.getMessage().contains("Connection retries limit exceeded"),
+        "Handler exception for reason other than retry: " + ce.toString());
     // we would expect this exception here, check if the client has retried
-    Assert.assertTrue("Retry filter didn't perform any retries! ",
-        client.connector.connectionRetry.getRetired());
+    assertTrue(client.connector.connectionRetry.getRetired(),
+        "Retry filter didn't perform any retries! ");
   }
 
   public static ClientResponse mockEntityClientResponse(
@@ -495,7 +498,7 @@ protected TimelineConnector createTimelineConnector() {
   }
 
   @Test
-  public void testTimelineClientCleanup() throws Exception {
+  void testTimelineClientCleanup() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 0);
@@ -520,7 +523,7 @@ public void testTimelineClientCleanup() throws Exception {
         reloaderThread = thread;
       }
     }
-    Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
+    assertTrue(reloaderThread.isAlive(), "Reloader is not alive");
 
     client.close();
 
@@ -532,11 +535,11 @@ public void testTimelineClientCleanup() throws Exception {
       }
       Thread.sleep(1000);
     }
-    Assert.assertFalse("Reloader is still alive", reloaderStillAlive);
+    assertFalse(reloaderStillAlive, "Reloader is still alive");
   }
 
   @Test
-  public void testTimelineConnectorDestroy() {
+  void testTimelineConnectorDestroy() {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     TimelineClientImpl client = createTimelineClient(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java
index 26dd7f42bc5f8..4d4e412e732ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientForATS1_5.java
@@ -18,20 +18,18 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.reset;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -42,13 +40,16 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TestTimelineClientForATS1_5 {
 
@@ -61,7 +62,7 @@ public class TestTimelineClientForATS1_5 {
   private TimelineWriter spyTimelineWriter;
   private UserGroupInformation authUgi;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     localFS = FileContext.getLocalFSFileContext();
     localActiveDir =
@@ -85,7 +86,7 @@ private YarnConfiguration getConfigurations() {
     return conf;
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     if (client != null) {
       client.stop();
@@ -94,13 +95,13 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testPostEntities() throws Exception {
+  void testPostEntities() throws Exception {
     client = createTimelineClient(getConfigurations());
     verifyForPostEntities(false);
   }
 
   @Test
-  public void testPostEntitiesToKeepUnderUserDir() throws Exception {
+  void testPostEntitiesToKeepUnderUserDir() throws Exception {
     YarnConfiguration conf = getConfigurations();
     conf.setBoolean(
         YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR,
@@ -137,7 +138,7 @@ private void verifyForPostEntities(boolean storeInsideUserDir) {
       TimelineEntity[] entityTDB = new TimelineEntity[1];
       entityTDB[0] = entities[0];
       verify(spyTimelineWriter, times(1)).putEntities(entityTDB);
-      Assert.assertTrue(localFS.util().exists(
+      assertTrue(localFS.util().exists(
           new Path(getAppAttemptDir(attemptId1, storeInsideUserDir),
               "summarylog-"
             + attemptId1.toString())));
@@ -152,32 +153,32 @@ private void verifyForPostEntities(boolean storeInsideUserDir) {
       client.putEntities(attemptId2, groupId2, entities);
       verify(spyTimelineWriter, times(0)).putEntities(
         any(TimelineEntity[].class));
-      Assert.assertTrue(localFS.util().exists(
+      assertTrue(localFS.util().exists(
           new Path(getAppAttemptDir(attemptId2, storeInsideUserDir),
               "summarylog-"
             + attemptId2.toString())));
-      Assert.assertTrue(localFS.util().exists(
+      assertTrue(localFS.util().exists(
           new Path(getAppAttemptDir(attemptId2, storeInsideUserDir),
               "entitylog-"
             + groupId.toString())));
-      Assert.assertTrue(localFS.util().exists(
+      assertTrue(localFS.util().exists(
           new Path(getAppAttemptDir(attemptId2, storeInsideUserDir),
               "entitylog-"
             + groupId2.toString())));
       reset(spyTimelineWriter);
     } catch (Exception e) {
-      Assert.fail("Exception is not expected. " + e);
+      fail("Exception is not expected. " + e);
     }
   }
 
   @Test
-  public void testPutDomain() {
+  void testPutDomain() {
     client = createTimelineClient(getConfigurations());
     verifyForPutDomain(false);
   }
 
   @Test
-  public void testPutDomainToKeepUnderUserDir() {
+  void testPutDomainToKeepUnderUserDir() {
     YarnConfiguration conf = getConfigurations();
     conf.setBoolean(
         YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR,
@@ -200,12 +201,12 @@ private void verifyForPutDomain(boolean storeInsideUserDir) {
 
       client.putDomain(attemptId1, domain);
       verify(spyTimelineWriter, times(0)).putDomain(domain);
-      Assert.assertTrue(localFS.util()
+      assertTrue(localFS.util()
           .exists(new Path(getAppAttemptDir(attemptId1, storeInsideUserDir),
               "domainlog-" + attemptId1.toString())));
       reset(spyTimelineWriter);
     } catch (Exception e) {
-      Assert.fail("Exception is not expected." + e);
+      fail("Exception is not expected." + e);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
index 8d437af3076ea..a26b4bf0a6797 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
@@ -18,20 +18,19 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
-
 import javax.ws.rs.core.MultivaluedMap;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -42,12 +41,13 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineClientV2Impl {
   private static final Logger LOG =
@@ -56,20 +56,25 @@ public class TestTimelineClientV2Impl {
   private static final long TIME_TO_SLEEP = 150L;
   private static final String EXCEPTION_MSG = "Exception in the content";
 
-  @Before
-  public void setup() {
+  @BeforeEach
+  public void setup(TestInfo testInfo) {
     conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
     conf.setInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE, 3);
-    if (!currTestName.getMethodName()
+    if (!testInfo.getDisplayName()
         .contains("testRetryOnConnectionFailure")) {
       client = createTimelineClient(conf);
     }
   }
 
-  @Rule
-  public TestName currTestName = new TestName();
+  @Test
+  void getTestInfo(TestInfo testInfo) {
+    System.out.println(testInfo.getDisplayName());
+    System.out.println(testInfo.getTestMethod());
+    System.out.println(testInfo.getTestClass());
+    System.out.println(testInfo.getTags());
+  }
   private YarnConfiguration conf;
 
   private TestV2TimelineClient createTimelineClient(YarnConfiguration config) {
@@ -116,8 +121,8 @@ private class TestV2TimelineClient
     private List<TimelineEntities> publishedEntities;
 
     public TimelineEntities getPublishedEntities(int putIndex) {
-      Assert.assertTrue("Not So many entities Published",
-          putIndex < publishedEntities.size());
+      assertTrue(putIndex < publishedEntities.size(),
+          "Not So many entities Published");
       return publishedEntities.get(putIndex);
     }
 
@@ -152,7 +157,7 @@ protected void putObjects(String path,
   }
 
   @Test
-  public void testExceptionMultipleRetry() {
+  void testExceptionMultipleRetry() {
     TestV2TimelineClientForExceptionHandling c =
         new TestV2TimelineClientForExceptionHandling(
             ApplicationId.newInstance(0, 0));
@@ -165,42 +170,42 @@ public void testExceptionMultipleRetry() {
     try {
       c.putEntities(new TimelineEntity());
     } catch (IOException e) {
-      Assert.fail("YARN exception is expected");
+      fail("YARN exception is expected");
     } catch (YarnException e) {
       Throwable cause = e.getCause();
-      Assert.assertTrue("IOException is expected",
-          cause instanceof IOException);
-      Assert.assertTrue("YARN exception is expected",
-          cause.getMessage().contains(
-              "TimelineClient has reached to max retry times : " + maxRetries));
+      assertTrue(cause instanceof IOException,
+          "IOException is expected");
+      assertTrue(cause.getMessage().contains(
+              "TimelineClient has reached to max retry times : " + maxRetries),
+          "YARN exception is expected");
     }
 
     c.setThrowYarnException(true);
     try {
       c.putEntities(new TimelineEntity());
     } catch (IOException e) {
-      Assert.fail("YARN exception is expected");
+      fail("YARN exception is expected");
     } catch (YarnException e) {
       Throwable cause = e.getCause();
-      Assert.assertTrue("YARN exception is expected",
-          cause instanceof YarnException);
-      Assert.assertTrue("YARN exception is expected",
-          cause.getMessage().contains(EXCEPTION_MSG));
+      assertTrue(cause instanceof YarnException,
+          "YARN exception is expected");
+      assertTrue(cause.getMessage().contains(EXCEPTION_MSG),
+          "YARN exception is expected");
     }
     c.stop();
   }
 
   @Test
-  public void testPostEntities() throws Exception {
+  void testPostEntities() throws Exception {
     try {
       client.putEntities(generateEntity("1"));
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
   }
 
   @Test
-  public void testASyncCallMerge() throws Exception {
+  void testASyncCallMerge() throws Exception {
     client.setSleepBeforeReturn(true);
     try {
       client.putEntitiesAsync(generateEntity("1"));
@@ -209,7 +214,7 @@ public void testASyncCallMerge() throws Exception {
       client.putEntitiesAsync(generateEntity("2"));
       client.putEntitiesAsync(generateEntity("3"));
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
     for (int i = 0; i < 4; i++) {
       if (client.getNumOfTimelineEntitiesPublished() == 2) {
@@ -217,20 +222,24 @@ public void testASyncCallMerge() throws Exception {
       }
       Thread.sleep(TIME_TO_SLEEP);
     }
-    Assert.assertEquals("two merged TimelineEntities needs to be published", 2,
-        client.getNumOfTimelineEntitiesPublished());
+    assertEquals(2,
+        client.getNumOfTimelineEntitiesPublished(),
+        "two merged TimelineEntities needs to be published");
     TimelineEntities secondPublishedEntities = client.getPublishedEntities(1);
-    Assert.assertEquals(
-        "Merged TimelineEntities Object needs to 2 TimelineEntity Object", 2,
-        secondPublishedEntities.getEntities().size());
-    Assert.assertEquals("Order of Async Events Needs to be FIFO", "2",
-        secondPublishedEntities.getEntities().get(0).getId());
-    Assert.assertEquals("Order of Async Events Needs to be FIFO", "3",
-        secondPublishedEntities.getEntities().get(1).getId());
+    assertEquals(
+        2,
+        secondPublishedEntities.getEntities().size(),
+        "Merged TimelineEntities Object needs to 2 TimelineEntity Object");
+    assertEquals("2",
+        secondPublishedEntities.getEntities().get(0).getId(),
+        "Order of Async Events Needs to be FIFO");
+    assertEquals("3",
+        secondPublishedEntities.getEntities().get(1).getId(),
+        "Order of Async Events Needs to be FIFO");
   }
 
   @Test
-  public void testSyncCall() throws Exception {
+  void testSyncCall() throws Exception {
     try {
       // sync entity should not be be merged with Async
       client.putEntities(generateEntity("1"));
@@ -239,7 +248,7 @@ public void testSyncCall() throws Exception {
       // except for the sync call above 2 should be merged
       client.putEntities(generateEntity("4"));
     } catch (YarnException e) {
-      Assert.fail("Exception is not expected");
+      fail("Exception is not expected");
     }
     for (int i = 0; i < 4; i++) {
       if (client.getNumOfTimelineEntitiesPublished() == 3) {
@@ -253,57 +262,65 @@ public void testSyncCall() throws Exception {
     int lastPublishIndex = asyncPushesMerged ? 2 : 3;
 
     TimelineEntities firstPublishedEntities = client.getPublishedEntities(0);
-    Assert.assertEquals("sync entities should not be merged with async", 1,
-        firstPublishedEntities.getEntities().size());
+    assertEquals(1,
+        firstPublishedEntities.getEntities().size(),
+        "sync entities should not be merged with async");
 
     // async push does not guarantee a merge but is FIFO
     if (asyncPushesMerged) {
       TimelineEntities secondPublishedEntities = client.getPublishedEntities(1);
-      Assert.assertEquals(
-          "async entities should be merged before publishing sync", 2,
-          secondPublishedEntities.getEntities().size());
-      Assert.assertEquals("Order of Async Events Needs to be FIFO", "2",
-          secondPublishedEntities.getEntities().get(0).getId());
-      Assert.assertEquals("Order of Async Events Needs to be FIFO", "3",
-          secondPublishedEntities.getEntities().get(1).getId());
+      assertEquals(
+          2,
+          secondPublishedEntities.getEntities().size(),
+          "async entities should be merged before publishing sync");
+      assertEquals("2",
+          secondPublishedEntities.getEntities().get(0).getId(),
+          "Order of Async Events Needs to be FIFO");
+      assertEquals("3",
+          secondPublishedEntities.getEntities().get(1).getId(),
+          "Order of Async Events Needs to be FIFO");
     } else {
       TimelineEntities secondAsyncPublish = client.getPublishedEntities(1);
-      Assert.assertEquals("Order of Async Events Needs to be FIFO", "2",
-          secondAsyncPublish.getEntities().get(0).getId());
+      assertEquals("2",
+          secondAsyncPublish.getEntities().get(0).getId(),
+          "Order of Async Events Needs to be FIFO");
       TimelineEntities thirdAsyncPublish = client.getPublishedEntities(2);
-      Assert.assertEquals("Order of Async Events Needs to be FIFO", "3",
-          thirdAsyncPublish.getEntities().get(0).getId());
+      assertEquals("3",
+          thirdAsyncPublish.getEntities().get(0).getId(),
+          "Order of Async Events Needs to be FIFO");
     }
 
     // test the last entity published is sync put
     TimelineEntities thirdPublishedEntities =
         client.getPublishedEntities(lastPublishIndex);
-    Assert.assertEquals("sync entities had to be published at the last", 1,
-        thirdPublishedEntities.getEntities().size());
-    Assert.assertEquals("Expected last sync Event is not proper", "4",
-        thirdPublishedEntities.getEntities().get(0).getId());
+    assertEquals(1,
+        thirdPublishedEntities.getEntities().size(),
+        "sync entities had to be published at the last");
+    assertEquals("4",
+        thirdPublishedEntities.getEntities().get(0).getId(),
+        "Expected last sync Event is not proper");
   }
 
   @Test
-  public void testExceptionCalls() throws Exception {
+  void testExceptionCalls() throws Exception {
     client.setThrowYarnException(true);
     try {
       client.putEntitiesAsync(generateEntity("1"));
     } catch (YarnException e) {
-      Assert.fail("Async calls are not expected to throw exception");
+      fail("Async calls are not expected to throw exception");
     }
 
     try {
       client.putEntities(generateEntity("2"));
-      Assert.fail("Sync calls are expected to throw exception");
+      fail("Sync calls are expected to throw exception");
     } catch (YarnException e) {
-      Assert.assertEquals("Same exception needs to be thrown",
-          "ActualException", e.getCause().getMessage());
+      assertEquals("ActualException", e.getCause().getMessage(),
+          "Same exception needs to be thrown");
     }
   }
 
   @Test
-  public void testConfigurableNumberOfMerges() throws Exception {
+  void testConfigurableNumberOfMerges() throws Exception {
     client.setSleepBeforeReturn(true);
     try {
       // At max 3 entities need to be merged
@@ -318,52 +335,52 @@ public void testConfigurableNumberOfMerges() throws Exception {
       client.putEntitiesAsync(generateEntity("9"));
       client.putEntitiesAsync(generateEntity("10"));
     } catch (YarnException e) {
-      Assert.fail("No exception expected");
+      fail("No exception expected");
     }
     // not having the same logic here as it doesn't depend on how many times
     // events are published.
     Thread.sleep(2 * TIME_TO_SLEEP);
     printReceivedEntities();
     for (TimelineEntities publishedEntities : client.publishedEntities) {
-      Assert.assertTrue(
+      assertTrue(
+          publishedEntities.getEntities().size() <= 3,
           "Number of entities should not be greater than 3 for each publish,"
-              + " but was " + publishedEntities.getEntities().size(),
-          publishedEntities.getEntities().size() <= 3);
+              + " but was " + publishedEntities.getEntities().size());
     }
   }
 
   @Test
-  public void testSetTimelineToken() throws Exception {
+  void testSetTimelineToken() throws Exception {
     UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
     assertEquals(0, ugi.getTokens().size());
-    assertNull("Timeline token in v2 client should not be set",
-        client.currentTimelineToken);
+    assertNull(client.currentTimelineToken,
+        "Timeline token in v2 client should not be set");
 
     Token token = Token.newInstance(
         new byte[0], "kind", new byte[0], "service");
     client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token));
-    assertNull("Timeline token in v2 client should not be set as token kind " +
-        "is unexepcted.", client.currentTimelineToken);
+    assertNull(client.currentTimelineToken,
+        "Timeline token in v2 client should not be set as token kind " + "is unexepcted.");
     assertEquals(0, ugi.getTokens().size());
 
     token = Token.newInstance(new byte[0], TimelineDelegationTokenIdentifier.
         KIND_NAME.toString(), new byte[0], null);
     client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token));
-    assertNull("Timeline token in v2 client should not be set as serice is " +
-        "not set.", client.currentTimelineToken);
+    assertNull(client.currentTimelineToken,
+        "Timeline token in v2 client should not be set as serice is " + "not set.");
     assertEquals(0, ugi.getTokens().size());
 
     TimelineDelegationTokenIdentifier ident =
         new TimelineDelegationTokenIdentifier(new Text(ugi.getUserName()),
-        new Text("renewer"), null);
+            new Text("renewer"), null);
     ident.setSequenceNumber(1);
     token = Token.newInstance(ident.getBytes(),
         TimelineDelegationTokenIdentifier.KIND_NAME.toString(), new byte[0],
         "localhost:1234");
     client.setTimelineCollectorInfo(CollectorInfo.newInstance(null, token));
     assertEquals(1, ugi.getTokens().size());
-    assertNotNull("Timeline token should be set in v2 client.",
-        client.currentTimelineToken);
+    assertNotNull(client.currentTimelineToken,
+        "Timeline token should be set in v2 client.");
     assertEquals(token, client.currentTimelineToken);
 
     ident.setSequenceNumber(20);
@@ -377,7 +394,7 @@ public void testSetTimelineToken() throws Exception {
   }
 
   @Test
-  public void testAfterStop() throws Exception {
+  void testAfterStop() throws Exception {
     client.setSleepBeforeReturn(true);
     try {
       // At max 3 entities need to be merged
@@ -388,12 +405,12 @@ public void testAfterStop() throws Exception {
       client.stop();
       try {
         client.putEntitiesAsync(generateEntity("50"));
-        Assert.fail("Exception expected");
+        fail("Exception expected");
       } catch (YarnException e) {
         // expected
       }
     } catch (YarnException e) {
-      Assert.fail("No exception expected");
+      fail("No exception expected");
     }
     // not having the same logic here as it doesn't depend on how many times
     // events are published.
@@ -411,7 +428,7 @@ public void testAfterStop() throws Exception {
         client.publishedEntities.get(client.publishedEntities.size() - 1);
     TimelineEntity timelineEntity = publishedEntities.getEntities()
         .get(publishedEntities.getEntities().size() - 1);
-    Assert.assertEquals("", "19", timelineEntity.getId());
+    assertEquals("19", timelineEntity.getId(), "");
   }
 
   private void printReceivedEntities() {
@@ -435,7 +452,7 @@ private static TimelineEntity generateEntity(String id) {
     return entity;
   }
 
-  @After
+  @AfterEach
   public void tearDown() {
     if (client != null) {
       client.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
index f668472256a7b..975f9c74f4e7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineReaderClientImpl.java
@@ -18,12 +18,18 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
-import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import javax.ws.rs.core.MultivaluedMap;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import com.sun.jersey.api.client.ClientResponse;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -31,25 +37,23 @@
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import javax.ws.rs.core.MultivaluedMap;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_APPLICATION_ATTEMPT;
+import static org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType.YARN_CONTAINER;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Test class for Timeline Reader Client.
  */
 public class TestTimelineReaderClientImpl {
 
+  private final String appAttemptInfoFilter = "{\"type\":\"YARN_APPLICATION_ATTEMPT\"," +
+      "\"id\":\"appattempt_1234_0001_000001\"}";
   private TimelineReaderClient client;
 
-  @Before
+  @BeforeEach
   public void setup() {
     client = new MockTimelineReaderClient();
     Configuration conf = new YarnConfiguration();
@@ -60,54 +64,64 @@ public void setup() {
   }
 
   @Test
-  public void testGetApplication() throws Exception {
+  void testGetApplication() throws Exception {
     ApplicationId applicationId =
         ApplicationId.fromString("application_1234_0001");
     TimelineEntity entity = client.getApplicationEntity(applicationId,
         null, null);
-    Assert.assertEquals("mockApp1", entity.getId());
+    assertEquals("mockApp1", entity.getId());
   }
 
   @Test
-  public void getApplicationAttemptEntity() throws Exception {
+  void getApplicationAttemptEntity() throws Exception {
     ApplicationAttemptId attemptId =
         ApplicationAttemptId.fromString("appattempt_1234_0001_000001");
     TimelineEntity entity = client.getApplicationAttemptEntity(attemptId,
         null, null);
-    Assert.assertEquals("mockAppAttempt1", entity.getId());
+    assertEquals("mockAppAttempt1", entity.getId());
   }
 
   @Test
-  public void getApplicationAttemptEntities() throws Exception {
+  void getApplicationAttemptEntities() throws Exception {
     ApplicationId applicationId =
         ApplicationId.fromString("application_1234_0001");
     List<TimelineEntity> entities =
         client.getApplicationAttemptEntities(applicationId, null,
             null, 0, null);
-    Assert.assertEquals(2, entities.size());
-    Assert.assertEquals("mockAppAttempt2", entities.get(1).getId());
+    assertEquals(2, entities.size());
+    assertEquals("mockAppAttempt2", entities.get(1).getId());
   }
 
   @Test
-  public void testGetContainer() throws Exception {
+  void testGetContainer() throws Exception {
     ContainerId containerId =
         ContainerId.fromString("container_1234_0001_01_000001");
     TimelineEntity entity = client.getContainerEntity(containerId,
         null, null);
-    Assert.assertEquals("mockContainer1", entity.getId());
+    assertEquals("mockContainer1", entity.getId());
   }
 
   @Test
-  public void testGetContainers() throws Exception {
+  void testGetContainers() throws Exception {
     ApplicationId appId =
         ApplicationId.fromString("application_1234_0001");
     List<TimelineEntity> entities = client.getContainerEntities(appId,
         null, null, 0, null);
-    Assert.assertEquals(2, entities.size());
-    Assert.assertEquals("mockContainer2", entities.get(1).getId());
+    assertEquals(2, entities.size());
+    assertEquals("mockContainer2", entities.get(1).getId());
   }
 
-  @After
+  @Test
+  void testGetContainersForAppAttempt() throws Exception {
+    ApplicationId appId =
+        ApplicationId.fromString("application_1234_0001");
+    List<TimelineEntity> entities = client.getContainerEntities(appId,
+        null, ImmutableMap.of("infofilters", appAttemptInfoFilter), 0, null);
+    assertEquals(2, entities.size());
+    assertEquals("mockContainer4", entities.get(1).getId());
+  }
+
+  @AfterEach
   public void tearDown() {
     if (client != null) {
       client.stop();
@@ -135,11 +149,15 @@ private class MockTimelineReaderClient extends TimelineReaderClientImpl {
     protected ClientResponse doGetUri(URI base, String path,
         MultivaluedMap<String, String> params) throws IOException {
       ClientResponse mockClientResponse = mock(ClientResponse.class);
-      if (path.contains(YARN_CONTAINER.toString())) {
+      if (path.contains(YARN_CONTAINER.toString()) && !params.containsKey("infofilters")) {
         when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
             createTimelineEntity("mockContainer1"));
         when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
             createTimelineEntities("mockContainer1", "mockContainer2"));
+      } else if (path.contains(YARN_CONTAINER.toString()) && params.containsKey("infofilters")) {
+        assertEquals(encodeValue(appAttemptInfoFilter), params.get("infofilters").get(0));
+        when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
+            createTimelineEntities("mockContainer3", "mockContainer4"));
       } else if (path.contains(YARN_APPLICATION_ATTEMPT.toString())) {
         when(mockClientResponse.getEntity(TimelineEntity.class)).thenReturn(
             createTimelineEntity("mockAppAttempt1"));
@@ -151,6 +169,7 @@ protected ClientResponse doGetUri(URI base, String path,
         when(mockClientResponse.getEntity(TimelineEntity[].class)).thenReturn(
             createTimelineEntities("mockApp1", "mockApp2"));
       }
+
       return mockClientResponse;
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
index fc2c1d0d3350e..d7d3f10615685 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
@@ -18,20 +18,20 @@
 
 package org.apache.hadoop.yarn.conf;
 
-import org.apache.hadoop.conf.Configuration;
+import java.util.Collection;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.Collection;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestHAUtil {
   private Configuration conf;
@@ -48,7 +48,7 @@ public class TestHAUtil {
   private static final String RM_NODE_IDS_UNTRIMMED = RM1_NODE_ID_UNTRIMMED + "," + RM2_NODE_ID;
   private static final String RM_NODE_IDS = RM1_NODE_ID + "," + RM2_NODE_ID;
 
-  @Before
+  @BeforeEach
   public void setUp() {
     conf = new Configuration();
     conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED);
@@ -62,7 +62,7 @@ public void setUp() {
   }
 
   @Test
-  public void testGetRMServiceId() throws Exception {
+  void testGetRMServiceId() throws Exception {
     conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
     Collection<String> rmhaIds = HAUtil.getRMHAIds(conf);
     assertEquals(2, rmhaIds.size());
@@ -73,18 +73,18 @@ public void testGetRMServiceId() throws Exception {
   }
 
   @Test
-  public void testGetRMId() throws Exception {
+  void testGetRMId() throws Exception {
     conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
-    assertEquals("Does not honor " + YarnConfiguration.RM_HA_ID,
-      RM1_NODE_ID, HAUtil.getRMHAId(conf));
+    assertEquals(RM1_NODE_ID, HAUtil.getRMHAId(conf),
+        "Does not honor " + YarnConfiguration.RM_HA_ID);
 
     conf.clear();
-    assertNull("Return null when " + YarnConfiguration.RM_HA_ID
-        + " is not set", HAUtil.getRMHAId(conf));
+    assertNull(HAUtil.getRMHAId(conf), "Return null when " + YarnConfiguration.RM_HA_ID
+        + " is not set");
   }
 
   @Test
-  public void testVerifyAndSetConfiguration() throws Exception {
+  void testVerifyAndSetConfiguration() throws Exception {
     Configuration myConf = new Configuration(conf);
 
     try {
@@ -93,14 +93,12 @@ public void testVerifyAndSetConfiguration() throws Exception {
       fail("Should not throw any exceptions.");
     }
 
-    assertEquals("Should be saved as Trimmed collection",
-        StringUtils.getStringCollection(RM_NODE_IDS),
-        HAUtil.getRMHAIds(myConf));
-    assertEquals("Should be saved as Trimmed string",
-        RM1_NODE_ID, HAUtil.getRMHAId(myConf));
+    assertEquals(StringUtils.getStringCollection(RM_NODE_IDS),
+        HAUtil.getRMHAIds(myConf),
+        "Should be saved as Trimmed collection");
+    assertEquals(RM1_NODE_ID, HAUtil.getRMHAId(myConf), "Should be saved as Trimmed string");
     for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
-      assertEquals("RPC address not set for " + confKey,
-          RM1_ADDRESS, myConf.get(confKey));
+      assertEquals(RM1_ADDRESS, myConf.get(confKey), "RPC address not set for " + confKey);
     }
 
     myConf = new Configuration(conf);
@@ -108,12 +106,12 @@ public void testVerifyAndSetConfiguration() throws Exception {
     try {
       HAUtil.verifyAndSetConfiguration(myConf);
     } catch (YarnRuntimeException e) {
-      assertEquals("YarnRuntimeException by verifyAndSetRMHAIds()",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
           HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_IDS,
               myConf.get(YarnConfiguration.RM_HA_IDS) +
-              "\nHA mode requires atleast two RMs"),
-        e.getMessage());
+                  "\nHA mode requires atleast two RMs"),
+          e.getMessage(),
+          "YarnRuntimeException by verifyAndSetRMHAIds()");
     }
 
     myConf = new Configuration(conf);
@@ -127,10 +125,10 @@ public void testVerifyAndSetConfiguration() throws Exception {
     try {
       HAUtil.verifyAndSetConfiguration(myConf);
     } catch (YarnRuntimeException e) {
-      assertEquals("YarnRuntimeException by getRMId()",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
           HAUtil.getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID),
-        e.getMessage());
+          e.getMessage(),
+          "YarnRuntimeException by getRMId()");
     }
 
     myConf = new Configuration(conf);
@@ -144,11 +142,11 @@ public void testVerifyAndSetConfiguration() throws Exception {
     try {
       HAUtil.verifyAndSetConfiguration(myConf);
     } catch (YarnRuntimeException e) {
-      assertEquals("YarnRuntimeException by addSuffix()",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
           HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_ID,
-            RM_INVALID_NODE_ID),
-        e.getMessage());
+              RM_INVALID_NODE_ID),
+          e.getMessage(),
+          "YarnRuntimeException by addSuffix()");
     }
 
     myConf = new Configuration();
@@ -160,11 +158,10 @@ public void testVerifyAndSetConfiguration() throws Exception {
       fail("Should throw YarnRuntimeException. by Configuration#set()");
     } catch (YarnRuntimeException e) {
       String confKey =
-        HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
-      assertEquals("YarnRuntimeException by Configuration#set()",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(
-            HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID)
-            + " or " + confKey), e.getMessage());
+          HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(
+          HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID)
+              + " or " + confKey), e.getMessage(), "YarnRuntimeException by Configuration#set()");
     }
 
     // simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
@@ -180,10 +177,10 @@ public void testVerifyAndSetConfiguration() throws Exception {
     try {
       HAUtil.verifyAndSetConfiguration(myConf);
     } catch (YarnRuntimeException e) {
-      assertEquals("YarnRuntimeException by getRMId()'s validation",
-        HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
-        HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID),
-          e.getMessage());
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+          HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID),
+          e.getMessage(),
+          "YarnRuntimeException by getRMId()'s validation");
     }
 
     // simulate the case that no leader election is enabled
@@ -196,19 +193,19 @@ public void testVerifyAndSetConfiguration() throws Exception {
     try {
       HAUtil.verifyAndSetConfiguration(myConf);
     } catch (YarnRuntimeException e) {
-      assertEquals("YarnRuntimeException by getRMId()'s validation",
-          HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.NO_LEADER_ELECTION_MESSAGE,
-          e.getMessage());
+      assertEquals(HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.NO_LEADER_ELECTION_MESSAGE,
+          e.getMessage(),
+          "YarnRuntimeException by getRMId()'s validation");
     }
   }
 
   @Test
-  public void testGetConfKeyForRMInstance() {
-    assertTrue("RM instance id is not suffixed",
-        HAUtil.getConfKeyForRMInstance(YarnConfiguration.RM_ADDRESS, conf)
-            .contains(HAUtil.getRMHAId(conf)));
-    assertFalse("RM instance id is suffixed",
-        HAUtil.getConfKeyForRMInstance(YarnConfiguration.NM_ADDRESS, conf)
-            .contains(HAUtil.getRMHAId(conf)));
+  void testGetConfKeyForRMInstance() {
+    assertTrue(HAUtil.getConfKeyForRMInstance(YarnConfiguration.RM_ADDRESS, conf)
+            .contains(HAUtil.getRMHAId(conf)),
+        "RM instance id is not suffixed");
+    assertFalse(HAUtil.getConfKeyForRMInstance(YarnConfiguration.NM_ADDRESS, conf)
+            .contains(HAUtil.getRMHAId(conf)),
+        "RM instance id is suffixed");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
index 212e09c02e90f..b17c1806de39c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
@@ -18,28 +18,29 @@
 
 package org.apache.hadoop.yarn.conf;
 
-import org.junit.Assert;
+import java.net.InetSocketAddress;
 
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
-import java.net.InetSocketAddress;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestYarnConfiguration {
 
   @Test
-  public void testDefaultRMWebUrl() throws Exception {
+  void testDefaultRMWebUrl() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
     // shouldn't have a "/" on the end of the url as all the other uri routinnes
     // specifically add slashes and Jetty doesn't handle double slashes.
-    Assert.assertNotSame("RM Web Url is not correct", "http://0.0.0.0:8088",
-        rmWebUrl);
+    assertNotSame("http://0.0.0.0:8088",
+        rmWebUrl,
+        "RM Web Url is not correct");
 
     // test it in HA scenario
     conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
@@ -47,7 +48,7 @@ public void testDefaultRMWebUrl() throws Exception {
     conf.set("yarn.resourcemanager.webapp.address.rm1", "10.10.10.10:18088");
     conf.set("yarn.resourcemanager.webapp.address.rm2", "20.20.20.20:28088");
     String rmWebUrlinHA = WebAppUtils.getRMWebAppURLWithScheme(conf);
-    Assert.assertEquals("http://10.10.10.10:18088", rmWebUrlinHA);
+    assertEquals("http://10.10.10.10:18088", rmWebUrlinHA);
 
     YarnConfiguration conf2 = new YarnConfiguration();
     conf2.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
@@ -55,17 +56,17 @@ public void testDefaultRMWebUrl() throws Exception {
     conf2.set("yarn.resourcemanager.hostname.rm1", "30.30.30.30");
     conf2.set("yarn.resourcemanager.hostname.rm2", "40.40.40.40");
     String rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2);
-    Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2);
+    assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2);
 
     rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 0);
-    Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2);
+    assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2);
 
     rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2, 1);
-    Assert.assertEquals("http://40.40.40.40:8088", rmWebUrlinHA2);
+    assertEquals("http://40.40.40.40:8088", rmWebUrlinHA2);
   }
 
   @Test
-  public void testRMWebUrlSpecified() throws Exception {
+  void testRMWebUrlSpecified() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     // seems a bit odd but right now we are forcing webapp for RM to be
     // RM_ADDRESS
@@ -74,15 +75,15 @@ public void testRMWebUrlSpecified() throws Exception {
     conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
     String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
     String[] parts = rmWebUrl.split(":");
-    Assert.assertEquals("RM Web URL Port is incrrect", 24543,
-        Integer.parseInt(parts[parts.length - 1]));
-    Assert.assertNotSame(
-        "RM Web Url not resolved correctly. Should not be rmtesting",
-        "http://rmtesting:24543", rmWebUrl);
+    assertEquals(24543,
+        Integer.parseInt(parts[parts.length - 1]),
+        "RM Web URL Port is incrrect");
+    assertNotSame("http://rmtesting:24543", rmWebUrl,
+        "RM Web Url not resolved correctly. Should not be rmtesting");
   }
 
   @Test
-  public void testGetSocketAddressForNMWithHA() {
+  void testGetSocketAddressForNMWithHA() {
     YarnConfiguration conf = new YarnConfiguration();
 
     // Set NM address
@@ -100,7 +101,7 @@ public void testGetSocketAddressForNMWithHA() {
   }
 
   @Test
-  public void testGetSocketAddr() throws Exception {
+  void testGetSocketAddr() throws Exception {
 
     YarnConfiguration conf;
     InetSocketAddress resourceTrackerAddress;
@@ -113,9 +114,9 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
 
     //with address
@@ -126,9 +127,9 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.1",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "10.0.0.1",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
 
     //address and socket
@@ -139,9 +140,9 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.2",
-        5001),
+        new InetSocketAddress(
+            "10.0.0.2",
+            5001),
         resourceTrackerAddress);
 
     //bind host only
@@ -153,9 +154,9 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        "10.0.0.3",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "10.0.0.3",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
 
     //bind host and address no port
@@ -167,9 +168,9 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        "0.0.0.0",
-        YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+        new InetSocketAddress(
+            "0.0.0.0",
+            YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
         resourceTrackerAddress);
 
     //bind host and address with port
@@ -181,15 +182,15 @@ public void testGetSocketAddr() throws Exception {
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
         YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
     assertEquals(
-      new InetSocketAddress(
-        "0.0.0.0",
-        5003),
+        new InetSocketAddress(
+            "0.0.0.0",
+            5003),
         resourceTrackerAddress);
 
   }
 
   @Test
-  public void testUpdateConnectAddr() throws Exception {
+  void testUpdateConnectAddr() throws Exception {
     YarnConfiguration conf;
     InetSocketAddress resourceTrackerConnectAddress;
     InetSocketAddress serverAddress;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java
index cd6274afd0d23..e882631d8f78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/InlineDispatcher.java
@@ -20,9 +20,6 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
 
 @SuppressWarnings({"unchecked", "rawtypes"})
 public class InlineDispatcher extends AsyncDispatcher {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 8b2dfa08b0dd0..4119542164cff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -27,23 +27,30 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.slf4j.Logger;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.AbstractMetric;
 import org.apache.hadoop.metrics2.MetricsRecord;
 import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
-import org.slf4j.Logger;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.Assert;
-import org.junit.Test;
+import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
 
 import static org.apache.hadoop.metrics2.lib.Interns.info;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TestAsyncDispatcher {
 
@@ -52,9 +59,10 @@ public class TestAsyncDispatcher {
    * 1. A thread which was putting event to event queue is interrupted.
    * 2. Event queue is empty on close.
    */
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test(timeout=10000)
-  public void testDispatcherOnCloseIfQueueEmpty() throws Exception {
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @Test
+  @Timeout(10000)
+  void testDispatcherOnCloseIfQueueEmpty() throws Exception {
     BlockingQueue<Event> eventQueue = spy(new LinkedBlockingQueue<Event>());
     Event event = mock(Event.class);
     doThrow(new InterruptedException()).when(eventQueue).put(event);
@@ -66,19 +74,20 @@ public void testDispatcherOnCloseIfQueueEmpty() throws Exception {
     disp.waitForEventThreadToWait();
     try {
       disp.getEventHandler().handle(event);
-      Assert.fail("Expected YarnRuntimeException");
+      fail("Expected YarnRuntimeException");
     } catch (YarnRuntimeException e) {
-      Assert.assertTrue(e.getCause() instanceof InterruptedException);
+      assertTrue(e.getCause() instanceof InterruptedException);
     }
     // Queue should be empty and dispatcher should not hang on close
-    Assert.assertTrue("Event Queue should have been empty",
-        eventQueue.isEmpty());
+    assertTrue(eventQueue.isEmpty(),
+        "Event Queue should have been empty");
     disp.close();
   }
 
   // Test dispatcher should timeout on draining events.
-  @Test(timeout=10000)
-  public void testDispatchStopOnTimeout() throws Exception {
+  @Test
+  @Timeout(10000)
+  void testDispatchStopOnTimeout() throws Exception {
     BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>();
     eventQueue = spy(eventQueue);
     // simulate dispatcher is not drained.
@@ -143,9 +152,10 @@ private void dispatchDummyEvents(Dispatcher disp, int count) {
   }
 
   // Test if drain dispatcher drains events on stop.
-  @SuppressWarnings({ "rawtypes" })
-  @Test(timeout=10000)
-  public void testDrainDispatcherDrainEventsOnStop() throws Exception {
+  @SuppressWarnings({"rawtypes"})
+  @Test
+  @Timeout(10000)
+  void testDrainDispatcherDrainEventsOnStop() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 2000);
     BlockingQueue<Event> queue = new LinkedBlockingQueue<Event>();
@@ -161,11 +171,12 @@ public void testDrainDispatcherDrainEventsOnStop() throws Exception {
   }
 
   //Test print dispatcher details when the blocking queue is heavy
-  @Test(timeout = 10000)
-  public void testPrintDispatcherEventDetails() throws Exception {
+  @Test
+  @Timeout(10000)
+  void testPrintDispatcherEventDetails() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setInt(YarnConfiguration.
-            YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD, 5000);
+        YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD, 5000);
     Logger log = mock(Logger.class);
     AsyncDispatcher dispatcher = new AsyncDispatcher();
     dispatcher.init(conf);
@@ -190,7 +201,7 @@ public void testPrintDispatcherEventDetails() throws Exception {
       Thread.sleep(2000);
       //Make sure more than one event to take
       verify(log, atLeastOnce()).
-              info("Latest dispatch event type: TestEventType");
+          info("Latest dispatch event type: TestEventType");
     } finally {
       //... restore logger object
       logger.set(null, oldLog);
@@ -199,8 +210,9 @@ public void testPrintDispatcherEventDetails() throws Exception {
   }
 
   //Test print dispatcher details when the blocking queue is heavy
-  @Test(timeout = 60000)
-  public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
     for (int i = 0; i < 5; i++) {
       testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
     }
@@ -241,7 +253,7 @@ public void testPrintDispatcherEventDetailsAvoidDeadLoopInternal()
   }
 
   @Test
-  public void testMetricsForDispatcher() throws Exception {
+  void testMetricsForDispatcher() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     AsyncDispatcher dispatcher = null;
 
@@ -252,7 +264,7 @@ public void testMetricsForDispatcher() throws Exception {
           new GenericEventTypeMetrics.EventTypeMetricsBuilder()
               .setMs(DefaultMetricsSystem.instance())
               .setInfo(info("GenericEventTypeMetrics for "
-                      + TestEnum.class.getName(),
+                  + TestEnum.class.getName(),
                   "Metrics for " + dispatcher.getName()))
               .setEnumClass(TestEnum.class)
               .setEnums(TestEnum.class.getEnumConstants())
@@ -287,34 +299,34 @@ public void testMetricsForDispatcher() throws Exception {
           get(TestEnum.TestEventType2) == 2, 1000, 10000);
 
       // Check time spend.
-      Assert.assertTrue(genericEventTypeMetrics.
+      assertTrue(genericEventTypeMetrics.
           getTotalProcessingTime(TestEnum.TestEventType)
-          >= 1500*3);
-      Assert.assertTrue(genericEventTypeMetrics.
+          >= 1500 * 3);
+      assertTrue(genericEventTypeMetrics.
           getTotalProcessingTime(TestEnum.TestEventType)
-          < 1500*4);
+          < 1500 * 4);
 
-      Assert.assertTrue(genericEventTypeMetrics.
+      assertTrue(genericEventTypeMetrics.
           getTotalProcessingTime(TestEnum.TestEventType2)
-          >= 1500*2);
-      Assert.assertTrue(genericEventTypeMetrics.
+          >= 1500 * 2);
+      assertTrue(genericEventTypeMetrics.
           getTotalProcessingTime(TestEnum.TestEventType2)
-          < 1500*3);
+          < 1500 * 3);
 
       // Make sure metrics consistent.
-      Assert.assertEquals(Long.toString(genericEventTypeMetrics.
+      assertEquals(Long.toString(genericEventTypeMetrics.
               get(TestEnum.TestEventType)),
           genericEventTypeMetrics.
               getRegistry().get("TestEventType_event_count").toString());
-      Assert.assertEquals(Long.toString(genericEventTypeMetrics.
+      assertEquals(Long.toString(genericEventTypeMetrics.
               get(TestEnum.TestEventType2)),
           genericEventTypeMetrics.
               getRegistry().get("TestEventType2_event_count").toString());
-      Assert.assertEquals(Long.toString(genericEventTypeMetrics.
+      assertEquals(Long.toString(genericEventTypeMetrics.
               getTotalProcessingTime(TestEnum.TestEventType)),
           genericEventTypeMetrics.
               getRegistry().get("TestEventType_processing_time").toString());
-      Assert.assertEquals(Long.toString(genericEventTypeMetrics.
+      assertEquals(Long.toString(genericEventTypeMetrics.
               getTotalProcessingTime(TestEnum.TestEventType2)),
           genericEventTypeMetrics.
               getRegistry().get("TestEventType2_processing_time").toString());
@@ -326,7 +338,7 @@ public void testMetricsForDispatcher() throws Exception {
   }
 
   @Test
-  public void testDispatcherMetricsHistogram() throws Exception {
+  void testDispatcherMetricsHistogram() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     AsyncDispatcher dispatcher = null;
 
@@ -337,7 +349,7 @@ public void testDispatcherMetricsHistogram() throws Exception {
           new GenericEventTypeMetrics.EventTypeMetricsBuilder()
               .setMs(DefaultMetricsSystem.instance())
               .setInfo(info("GenericEventTypeMetrics for "
-                      + TestEnum.class.getName(),
+                  + TestEnum.class.getName(),
                   "Metrics for " + dispatcher.getName()))
               .setEnumClass(TestEnum.class)
               .setEnums(TestEnum.class.getEnumConstants())
@@ -393,14 +405,13 @@ public void testDispatcherMetricsHistogram() throws Exception {
           String metricName = metric.name();
           if (expectedValues.containsKey(metricName)) {
             Long expectedValue = expectedValues.get(metricName);
-            Assert.assertEquals(
-                "Metric " + metricName + " doesn't have expected value",
-                expectedValue, metric.value());
+            assertEquals(expectedValue, metric.value(),
+                "Metric " + metricName + " doesn't have expected value");
             testResults.add(metricName);
           }
         }
       }
-      Assert.assertEquals(expectedValues.keySet(), testResults);
+      assertEquals(expectedValues.keySet(), testResults);
 
     } finally {
       dispatcher.close();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcClientFactoryPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcClientFactoryPBImpl.java
index 7967c41765dba..ae807ec6e483c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcClientFactoryPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcClientFactoryPBImpl.java
@@ -18,11 +18,12 @@
 
 package org.apache.hadoop.yarn.factories.impl.pb;
 
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
 import java.net.InetSocketAddress;
 
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.mock;
@@ -33,7 +34,7 @@
  */
 public class TestRpcClientFactoryPBImpl {
   @Test
-  public void testToUseCustomClassloader() throws Exception {
+  void testToUseCustomClassloader() throws Exception {
     Configuration configuration = mock(Configuration.class);
     RpcClientFactoryPBImpl rpcClientFactoryPB = RpcClientFactoryPBImpl.get();
     try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcServerFactoryPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcServerFactoryPBImpl.java
index c8650e169a147..bc9e1d890c2f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcServerFactoryPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/factories/impl/pb/TestRpcServerFactoryPBImpl.java
@@ -18,11 +18,12 @@
 
 package org.apache.hadoop.yarn.factories.impl.pb;
 
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
 import java.net.InetSocketAddress;
 
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.conf.Configuration;
+
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.mock;
@@ -33,7 +34,7 @@
  */
 public class TestRpcServerFactoryPBImpl {
   @Test
-  public void testToUseCustomClassloader() throws Exception {
+  void testToUseCustomClassloader() throws Exception {
     Configuration configuration = mock(Configuration.class);
     RpcServerFactoryPBImpl rpcServerFactoryPB = RpcServerFactoryPBImpl.get();
     try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
index 73380741d2494..8ffb7deac0b53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/ipc/TestRPCUtil.java
@@ -21,71 +21,71 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 
-import org.junit.Assert;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.protobuf.ServiceException;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestRPCUtil {
 
   @Test
-  public void testUnknownExceptionUnwrapping() {
+  void testUnknownExceptionUnwrapping() {
     Class<? extends Throwable> exception = YarnException.class;
     String className = "UnknownException.class";
     verifyRemoteExceptionUnwrapping(exception, className);
   }
 
   @Test
-  public void testRemoteIOExceptionUnwrapping() {
+  void testRemoteIOExceptionUnwrapping() {
     Class<? extends Throwable> exception = IOException.class;
     verifyRemoteExceptionUnwrapping(exception, exception.getName());
   }
 
   @Test
-  public void testRemoteIOExceptionDerivativeUnwrapping() {
+  void testRemoteIOExceptionDerivativeUnwrapping() {
     // Test IOException sub-class
     Class<? extends Throwable> exception = FileNotFoundException.class;
     verifyRemoteExceptionUnwrapping(exception, exception.getName());
   }
 
   @Test
-  public void testRemoteYarnExceptionUnwrapping() {
+  void testRemoteYarnExceptionUnwrapping() {
     Class<? extends Throwable> exception = YarnException.class;
     verifyRemoteExceptionUnwrapping(exception, exception.getName());
 
   }
 
   @Test
-  public void testRemoteYarnExceptionDerivativeUnwrapping() {
+  void testRemoteYarnExceptionDerivativeUnwrapping() {
     Class<? extends Throwable> exception = YarnTestException.class;
     verifyRemoteExceptionUnwrapping(exception, exception.getName());
   }
 
   @Test
-  public void testRemoteRuntimeExceptionUnwrapping() {
+  void testRemoteRuntimeExceptionUnwrapping() {
     Class<? extends Throwable> exception = NullPointerException.class;
     verifyRemoteExceptionUnwrapping(exception, exception.getName());
   }
 
   @Test
-  public void testUnexpectedRemoteExceptionUnwrapping() {
+  void testUnexpectedRemoteExceptionUnwrapping() {
     // Non IOException, YarnException thrown by the remote side.
     Class<? extends Throwable> exception = Exception.class;
     verifyRemoteExceptionUnwrapping(RemoteException.class, exception.getName());
   }
-  
+
   @Test
-  public void testRemoteYarnExceptionWithoutStringConstructor() {
+  void testRemoteYarnExceptionWithoutStringConstructor() {
     // Derivatives of YarnException should always define a string constructor.
     Class<? extends Throwable> exception = YarnTestExceptionNoConstructor.class;
     verifyRemoteExceptionUnwrapping(RemoteException.class, exception.getName());
   }
-  
+
   @Test
-  public void testRPCServiceExceptionUnwrapping() {
+  void testRPCServiceExceptionUnwrapping() {
     String message = "ServiceExceptionMessage";
     ServiceException se = new ServiceException(message);
 
@@ -96,12 +96,12 @@ public void testRPCServiceExceptionUnwrapping() {
       t = thrown;
     }
 
-    Assert.assertTrue(IOException.class.isInstance(t));
-    Assert.assertTrue(t.getMessage().contains(message));
+    assertTrue(IOException.class.isInstance(t));
+    assertTrue(t.getMessage().contains(message));
   }
 
   @Test
-  public void testRPCIOExceptionUnwrapping() {
+  void testRPCIOExceptionUnwrapping() {
     String message = "DirectIOExceptionMessage";
     IOException ioException = new FileNotFoundException(message);
     ServiceException se = new ServiceException(ioException);
@@ -112,12 +112,12 @@ public void testRPCIOExceptionUnwrapping() {
     } catch (Throwable thrown) {
       t = thrown;
     }
-    Assert.assertTrue(FileNotFoundException.class.isInstance(t));
-    Assert.assertTrue(t.getMessage().contains(message));
+    assertTrue(FileNotFoundException.class.isInstance(t));
+    assertTrue(t.getMessage().contains(message));
   }
 
   @Test
-  public void testRPCRuntimeExceptionUnwrapping() {
+  void testRPCRuntimeExceptionUnwrapping() {
     String message = "RPCRuntimeExceptionUnwrapping";
     RuntimeException re = new NullPointerException(message);
     ServiceException se = new ServiceException(re);
@@ -129,8 +129,8 @@ public void testRPCRuntimeExceptionUnwrapping() {
       t = thrown;
     }
 
-    Assert.assertTrue(NullPointerException.class.isInstance(t));
-    Assert.assertTrue(t.getMessage().contains(message));
+    assertTrue(NullPointerException.class.isInstance(t));
+    assertTrue(t.getMessage().contains(message));
   }
 
   private void verifyRemoteExceptionUnwrapping(
@@ -147,11 +147,10 @@ private void verifyRemoteExceptionUnwrapping(
       t = thrown;
     }
 
-    Assert.assertTrue("Expected exception [" + expectedLocalException
-        + "] but found " + t, expectedLocalException.isInstance(t));
-    Assert.assertTrue(
-        "Expected message [" + message + "] but found " + t.getMessage(), t
-            .getMessage().contains(message));
+    assertTrue(expectedLocalException.isInstance(t), "Expected exception [" + expectedLocalException
+        + "] but found " + t);
+    assertTrue(t.getMessage().contains(message),
+        "Expected message [" + message + "] but found " + t.getMessage());
   }
 
   private static class YarnTestException extends YarnException {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/LogAggregationTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/LogAggregationTestUtils.java
new file mode 100644
index 0000000000000..e0010a373b697
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/LogAggregationTestUtils.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation;
+
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT;
+
+
+public class LogAggregationTestUtils {
+  public static final String REMOTE_LOG_ROOT = "target/app-logs/";
+
+  public static void enableFileControllers(Configuration conf,
+          List<Class<? extends LogAggregationFileController>> fileControllers,
+          List<String> fileControllerNames) {
+    enableFcs(conf, REMOTE_LOG_ROOT, fileControllers, fileControllerNames);
+  }
+
+  public static void enableFileControllers(Configuration conf,
+                                           String remoteLogRoot,
+                                           List<Class<? extends LogAggregationFileController>> fileControllers,
+                                           List<String> fileControllerNames) {
+    enableFcs(conf, remoteLogRoot, fileControllers, fileControllerNames);
+  }
+
+
+  private static void enableFcs(Configuration conf,
+                                String remoteLogRoot,
+                                List<Class<? extends LogAggregationFileController>> fileControllers,
+                                List<String> fileControllerNames) {
+    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS,
+            StringUtils.join(fileControllerNames, ","));
+    for (int i = 0; i < fileControllers.size(); i++) {
+      Class<? extends LogAggregationFileController> fileController = fileControllers.get(i);
+      String controllerName = fileControllerNames.get(i);
+
+      conf.setClass(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, controllerName),
+              fileController, LogAggregationFileController.class);
+      conf.set(String.format(LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT, controllerName),
+              remoteLogRoot + controllerName + "/");
+      conf.set(String.format(LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT, controllerName),
+              controllerName);
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
index daa2fc6b01cbc..fa5a5870c4ebe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
@@ -18,557 +18,295 @@
 
 package org.apache.hadoop.yarn.logaggregation;
 
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT;
-
 import java.io.IOException;
 import java.net.URI;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.Assert;
+import org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcase;
+import org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcaseBuilder;
+import org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcaseBuilder.AppDescriptor;
+import org.apache.log4j.Level;
 
-import static org.mockito.Mockito.*;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT;
+import static org.apache.hadoop.yarn.logaggregation.LogAggregationTestUtils.enableFileControllers;
+import static org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcaseBuilder.NO_TIMEOUT;
+import static org.mockito.Mockito.mock;
 
 public class TestAggregatedLogDeletionService {
-  
-  @Before
+  private static final String T_FILE = "TFile";
+  private static final String I_FILE = "IFile";
+  private static final String USER_ME = "me";
+  private static final String DIR_HOST1 = "host1";
+  private static final String DIR_HOST2 = "host2";
+
+  private static final String ROOT = "mockfs://foo/";
+  private static final String REMOTE_ROOT_LOG_DIR = ROOT + "tmp/logs/";
+  private static final String SUFFIX = "logs";
+  private static final int TEN_DAYS_IN_SECONDS = 10 * 24 * 3600;
+
+  private static final List<Class<? extends LogAggregationFileController>>
+          ALL_FILE_CONTROLLERS = Arrays.asList(
+          LogAggregationIndexedFileController.class,
+          LogAggregationTFileController.class);
+  public static final List<String> ALL_FILE_CONTROLLER_NAMES = Arrays.asList(I_FILE, T_FILE);
+
+  @BeforeAll
+  public static void beforeClass() {
+    org.apache.log4j.Logger.getRootLogger().setLevel(Level.DEBUG);
+  }
+
+  @BeforeEach
   public void closeFilesystems() throws IOException {
     // prevent the same mockfs instance from being reused due to FS cache
     FileSystem.closeAll();
   }
 
+  private Configuration setupConfiguration(int retainSeconds, int retainCheckIntervalSeconds) {
+    Configuration conf = new Configuration();
+    conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+    conf.setInt(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, retainSeconds);
+    conf.setInt(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS,
+            retainCheckIntervalSeconds);
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, REMOTE_ROOT_LOG_DIR);
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, SUFFIX);
+    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, T_FILE);
+    conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, T_FILE),
+            LogAggregationTFileController.class.getName());
+    return conf;
+  }
+
   @Test
-  public void testDeletion() throws Exception {
+  void testDeletion() throws Exception {
     long now = System.currentTimeMillis();
-    long toDeleteTime = now - (2000*1000);
-    long toKeepTime = now - (1500*1000);
-    String root = "mockfs://foo/";
-    String remoteRootLogDir = root+"tmp/logs";
-    String suffix = "logs";
-    String newSuffix = LogAggregationUtils.getBucketSuffix() + suffix;
-    final Configuration conf = new Configuration();
-    conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "1800");
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
-    conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"),
-         LogAggregationTFileController.class.getName());
-
-
-    Path rootPath = new Path(root);
-    FileSystem rootFs = rootPath.getFileSystem(conf);
-    FileSystem mockFs = ((FilterFileSystem)rootFs).getRawFileSystem();
-    
-    Path remoteRootLogPath = new Path(remoteRootLogDir);
-    
-    Path userDir = new Path(remoteRootLogPath, "me");
-    FileStatus userDirStatus = new FileStatus(0, true, 0, 0, toKeepTime, userDir); 
-    
-    when(mockFs.listStatus(remoteRootLogPath)).thenReturn(
-        new FileStatus[]{userDirStatus});
-
-    ApplicationId appId1 =
-        ApplicationId.newInstance(now, 1);
-    Path suffixDir = new Path(userDir, newSuffix);
-    FileStatus suffixDirStatus = new FileStatus(0, true,
-        0, 0, toDeleteTime, suffixDir);
-    Path bucketDir = LogAggregationUtils.getRemoteBucketDir(
-        remoteRootLogPath, "me", suffix, appId1);
-    FileStatus bucketDirStatus = new FileStatus(0, true, 0,
-        0, toDeleteTime, bucketDir);
-    Path app1Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId1, "me", suffix);
-    FileStatus app1DirStatus = new FileStatus(0, true, 0, 0,
-        toDeleteTime, app1Dir);
-    
-    ApplicationId appId2 =
-        ApplicationId.newInstance(now, 2);
-    Path app2Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId2, "me", suffix);
-    FileStatus app2DirStatus = new FileStatus(0, true, 0, 0,
-        toDeleteTime, app2Dir);
-    
-    ApplicationId appId3 =
-        ApplicationId.newInstance(now, 3);
-    Path app3Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId3, "me", suffix);
-    FileStatus app3DirStatus = new FileStatus(0, true, 0, 0,
-        toDeleteTime, app3Dir);
-    
-    ApplicationId appId4 =
-        ApplicationId.newInstance(now, 4);
-    Path app4Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId4, "me", suffix);
-    FileStatus app4DirStatus =
-        new FileStatus(0, true, 0, 0, toDeleteTime, app4Dir);
-
-    when(mockFs.listStatus(userDir)).thenReturn(
-        new FileStatus[] {suffixDirStatus});
-    when(mockFs.listStatus(suffixDir)).thenReturn(
-        new FileStatus[] {bucketDirStatus});
-    when(mockFs.listStatus(bucketDir)).thenReturn(
-        new FileStatus[] {app1DirStatus, app2DirStatus,
-            app3DirStatus, app4DirStatus});
-    
-    when(mockFs.listStatus(app1Dir)).thenReturn(
-        new FileStatus[]{});
-
-
-    Path app2Log1 = new Path(app2Dir, "host1");
-    FileStatus app2Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app2Log1);
-    
-    Path app2Log2 = new Path(app2Dir, "host2");
-    FileStatus app2Log2Status = new FileStatus(10, false, 1, 1, toKeepTime, app2Log2);
-    
-    when(mockFs.listStatus(app2Dir)).thenReturn(
-        new FileStatus[]{app2Log1Status, app2Log2Status});
-    
-    Path app3Log1 = new Path(app3Dir, "host1");
-    FileStatus app3Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app3Log1);
-    
-    Path app3Log2 = new Path(app3Dir, "host2");
-    FileStatus app3Log2Status = new FileStatus(10, false, 1, 1, toDeleteTime, app3Log2);
-    
-    when(mockFs.delete(app3Dir, true)).thenThrow(new AccessControlException("Injected Error\nStack Trace :("));
-    
-    when(mockFs.listStatus(app3Dir)).thenReturn(
-        new FileStatus[]{app3Log1Status, app3Log2Status});
-    
-    Path app4Log1 = new Path(app4Dir, "host1");
-    FileStatus app4Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app4Log1);
-    
-    Path app4Log2 = new Path(app4Dir, "host2");
-    FileStatus app4Log2Status = new FileStatus(10, false, 1, 1,
-        toKeepTime, app4Log2);
-
-    when(mockFs.listStatus(app4Dir)).thenReturn(
-        new FileStatus[]{app4Log1Status, app4Log2Status});
-
-    final List<ApplicationId> finishedApplications =
-        Collections.unmodifiableList(Arrays.asList(appId1, appId2, appId3));
-    final List<ApplicationId> runningApplications =
-        Collections.unmodifiableList(Arrays.asList(appId4));
-
-    AggregatedLogDeletionService deletionService =
-        new AggregatedLogDeletionService() {
-          @Override
-          protected ApplicationClientProtocol createRMClient()
-              throws IOException {
-            try {
-              return createMockRMClient(finishedApplications,
-                runningApplications);
-            } catch (Exception e) {
-              throw new IOException(e);
-            }
-          }
-          @Override
-          protected void stopRMClient() {
-            // DO NOTHING
-          }
-        };
-    deletionService.init(conf);
-    deletionService.start();
-
-    verify(mockFs, timeout(2000)).delete(app1Dir, true);
-    verify(mockFs, timeout(2000).times(0)).delete(app2Dir, true);
-    verify(mockFs, timeout(2000)).delete(app3Dir, true);
-    verify(mockFs, timeout(2000).times(0)).delete(app4Dir, true);
-    verify(mockFs, timeout(2000)).delete(app4Log1, true);
-    verify(mockFs, timeout(2000).times(0)).delete(app4Log2, true);
-
-    deletionService.stop();
+    long toDeleteTime = now - (2000 * 1000);
+    long toKeepTime = now - (1500 * 1000);
+
+    Configuration conf = setupConfiguration(1800, -1);
+    long timeout = 2000L;
+    LogAggregationTestcaseBuilder.create(conf)
+        .withRootPath(ROOT)
+        .withRemoteRootLogPath(REMOTE_ROOT_LOG_DIR)
+        .withUserDir(USER_ME, toKeepTime)
+        .withSuffixDir(SUFFIX, toDeleteTime)
+        .withBucketDir(toDeleteTime)
+        .withApps(Lists.newArrayList(
+            new AppDescriptor(toDeleteTime, Lists.newArrayList()),
+            new AppDescriptor(toDeleteTime, Lists.newArrayList(
+                Pair.of(DIR_HOST1, toDeleteTime),
+                Pair.of(DIR_HOST2, toKeepTime))),
+            new AppDescriptor(toDeleteTime, Lists.newArrayList(
+                Pair.of(DIR_HOST1, toDeleteTime),
+                Pair.of(DIR_HOST2, toDeleteTime))),
+            new AppDescriptor(toDeleteTime, Lists.newArrayList(
+                Pair.of(DIR_HOST1, toDeleteTime),
+                Pair.of(DIR_HOST2, toKeepTime)))))
+        .withFinishedApps(1, 2, 3)
+        .withRunningApps(4)
+        .injectExceptionForAppDirDeletion(3)
+        .build()
+        .startDeletionService()
+        .verifyAppDirsDeleted(timeout, 1, 3)
+        .verifyAppDirsNotDeleted(timeout, 2, 4)
+        .verifyAppFileDeleted(4, 1, timeout)
+        .verifyAppFileNotDeleted(4, 2, timeout)
+        .teardown(1);
   }
 
   @Test
-  public void testRefreshLogRetentionSettings() throws Exception {
+  void testRefreshLogRetentionSettings() throws Exception {
     long now = System.currentTimeMillis();
-    //time before 2000 sec
     long before2000Secs = now - (2000 * 1000);
-    //time before 50 sec
     long before50Secs = now - (50 * 1000);
-    String root = "mockfs://foo/";
-    String remoteRootLogDir = root + "tmp/logs";
-    String suffix = "logs";
-    String newSuffix = LogAggregationUtils.getBucketSuffix() + suffix;
-    final Configuration conf = new Configuration();
-    conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "1800");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS,
-        "1");
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
-    conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"),
-         LogAggregationTFileController.class.getName());
-
-
-    Path rootPath = new Path(root);
-    FileSystem rootFs = rootPath.getFileSystem(conf);
-    FileSystem mockFs = ((FilterFileSystem) rootFs).getRawFileSystem();
-
-    Path remoteRootLogPath = new Path(remoteRootLogDir);
-
-    Path userDir = new Path(remoteRootLogPath, "me");
-    FileStatus userDirStatus = new FileStatus(0, true, 0, 0, before50Secs,
-        userDir);
-
-    when(mockFs.listStatus(remoteRootLogPath)).thenReturn(
-        new FileStatus[] { userDirStatus });
-
-    Path suffixDir = new Path(userDir, newSuffix);
-    FileStatus suffixStatus = new FileStatus(0, true, 0, 0, before50Secs,
-        suffixDir);
-
-    ApplicationId appId1 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
-    //Set time last modified of app1Dir directory and its files to before2000Secs 
-    Path app1Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId1, "me", suffix);
-    Path bucketDir = LogAggregationUtils.getRemoteBucketDir(
-        remoteRootLogPath, "me", suffix, appId1);
-    FileStatus bucketDirStatus = new FileStatus(0, true, 0,
-        0, before50Secs, bucketDir);
-    FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, before2000Secs,
-        app1Dir);
-    
-    ApplicationId appId2 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 2);
-    //Set time last modified of app1Dir directory and its files to before50Secs 
-    Path app2Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId2, "me", suffix);
-    FileStatus app2DirStatus = new FileStatus(0, true, 0, 0, before50Secs,
-        app2Dir);
-
-    when(mockFs.listStatus(userDir)).thenReturn(
-        new FileStatus[] {suffixStatus });
-    when(mockFs.listStatus(suffixDir)).thenReturn(
-        new FileStatus[] {bucketDirStatus });
-    when(mockFs.listStatus(bucketDir)).thenReturn(
-        new FileStatus[] {app1DirStatus, app2DirStatus });
-
-    Path app1Log1 = new Path(app1Dir, "host1");
-    FileStatus app1Log1Status = new FileStatus(10, false, 1, 1, before2000Secs,
-        app1Log1);
-
-    when(mockFs.listStatus(app1Dir)).thenReturn(
-        new FileStatus[] { app1Log1Status });
-
-    Path app2Log1 = new Path(app2Dir, "host1");
-    FileStatus app2Log1Status = new FileStatus(10, false, 1, 1, before50Secs,
-        app2Log1);
-
-    when(mockFs.listStatus(app2Dir)).thenReturn(
-        new FileStatus[] { app2Log1Status });
-
-    final List<ApplicationId> finishedApplications =
-        Collections.unmodifiableList(Arrays.asList(appId1, appId2));
-
-    AggregatedLogDeletionService deletionSvc = new AggregatedLogDeletionService() {
-      @Override
-      protected Configuration createConf() {
-        return conf;
-      }
-      @Override
-      protected ApplicationClientProtocol createRMClient()
-          throws IOException {
-        try {
-          return createMockRMClient(finishedApplications, null);
-        } catch (Exception e) {
-          throw new IOException(e);
-        }
-      }
-      @Override
-      protected void stopRMClient() {
-        // DO NOTHING
-      }
-    };
-    
-    deletionSvc.init(conf);
-    deletionSvc.start();
-    
-    //app1Dir would be deleted since its done above log retention period
-    verify(mockFs, timeout(10000)).delete(app1Dir, true);
-    //app2Dir is not expected to be deleted since its below the threshold
-    verify(mockFs, timeout(3000).times(0)).delete(app2Dir, true);
-
-    //Now,lets change the confs
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "50");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS,
-        "2");
-    //We have not called refreshLogSettings,hence don't expect to see the changed conf values
-    Assert.assertTrue(2000l != deletionSvc.getCheckIntervalMsecs());
-    
-    //refresh the log settings
-    deletionSvc.refreshLogRetentionSettings();
-
-    //Check interval time should reflect the new value
-    Assert.assertTrue(2000l == deletionSvc.getCheckIntervalMsecs());
-    //app2Dir should be deleted since it falls above the threshold
-    verify(mockFs, timeout(10000)).delete(app2Dir, true);
-    deletionSvc.stop();
+    int checkIntervalSeconds = 2;
+    int checkIntervalMilliSeconds = checkIntervalSeconds * 1000;
+
+    Configuration conf = setupConfiguration(1800, 1);
+
+    LogAggregationTestcase testcase = LogAggregationTestcaseBuilder.create(conf)
+        .withRootPath(ROOT)
+        .withRemoteRootLogPath(REMOTE_ROOT_LOG_DIR)
+        .withUserDir(USER_ME, before50Secs)
+        .withSuffixDir(SUFFIX, before50Secs)
+        .withBucketDir(before50Secs)
+        .withApps(Lists.newArrayList(
+            //Set time last modified of app1Dir directory and its files to before2000Secs
+            new AppDescriptor(before2000Secs, Lists.newArrayList(
+                Pair.of(DIR_HOST1, before2000Secs))),
+            //Set time last modified of app1Dir directory and its files to before50Secs
+            new AppDescriptor(before50Secs, Lists.newArrayList(
+                Pair.of(DIR_HOST1, before50Secs))))
+        )
+        .withFinishedApps(1, 2)
+        .withRunningApps()
+        .build();
+
+    testcase
+        .startDeletionService()
+        //app1Dir would be deleted since it is done above log retention period
+        .verifyAppDirDeleted(1, 10000L)
+        //app2Dir is not expected to be deleted since it is below the threshold
+        .verifyAppDirNotDeleted(2, 3000L);
+
+    //Now, let's change the log aggregation retention configs
+    conf.setInt(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, 50);
+    conf.setInt(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS,
+        checkIntervalSeconds);
+
+    testcase
+        //We have not called refreshLogSettings, hence don't expect to see
+        // the changed conf values
+        .verifyCheckIntervalMilliSecondsNotEqualTo(checkIntervalMilliSeconds)
+        //refresh the log settings
+        .refreshLogRetentionSettings()
+        //Check interval time should reflect the new value
+        .verifyCheckIntervalMilliSecondsEqualTo(checkIntervalMilliSeconds)
+        //app2Dir should be deleted since it falls above the threshold
+        .verifyAppDirDeleted(2, 10000L)
+        //Close expected 2 times: once for refresh and once for stopping
+        .teardown(2);
   }
-  
+
   @Test
-  public void testCheckInterval() throws Exception {
-    long RETENTION_SECS = 10 * 24 * 3600;
+  void testCheckInterval() throws Exception {
     long now = System.currentTimeMillis();
-    long toDeleteTime = now - RETENTION_SECS*1000;
-
-    String root = "mockfs://foo/";
-    String remoteRootLogDir = root+"tmp/logs";
-    String suffix = "logs";
-    String newSuffix = LogAggregationUtils.getBucketSuffix() + suffix;
-    Configuration conf = new Configuration();
-    conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "864000");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, "1");
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
-    conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"),
-         LogAggregationTFileController.class.getName());
+    long toDeleteTime = now - TEN_DAYS_IN_SECONDS * 1000;
 
+    Configuration conf = setupConfiguration(TEN_DAYS_IN_SECONDS, 1);
 
     // prevent us from picking up the same mockfs instance from another test
     FileSystem.closeAll();
-    Path rootPath = new Path(root);
-    FileSystem rootFs = rootPath.getFileSystem(conf);
-    FileSystem mockFs = ((FilterFileSystem)rootFs).getRawFileSystem();
-
-    Path remoteRootLogPath = new Path(remoteRootLogDir);
-
-    Path userDir = new Path(remoteRootLogPath, "me");
-    FileStatus userDirStatus = new FileStatus(0, true, 0, 0, now, userDir);
-
-    when(mockFs.listStatus(remoteRootLogPath)).thenReturn(
-        new FileStatus[]{userDirStatus});
-
-    ApplicationId appId1 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
-    Path suffixDir = new Path(userDir, newSuffix);
-    FileStatus suffixDirStatus = new FileStatus(0, true, 0, 0, now,
-        suffixDir);
-    Path bucketDir = LogAggregationUtils.getRemoteBucketDir(
-        remoteRootLogPath, "me", suffix, appId1);
-    Path app1Dir = LogAggregationUtils.getRemoteAppLogDir(
-        remoteRootLogPath, appId1, "me", suffix);
-    FileStatus bucketDirStatus = new FileStatus(0, true, 0,
-        0, now, bucketDir);
-
-    FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, now, app1Dir);
-
-    when(mockFs.listStatus(userDir)).thenReturn(
-        new FileStatus[] {suffixDirStatus});
-    when(mockFs.listStatus(suffixDir)).thenReturn(
-        new FileStatus[] {bucketDirStatus});
-    when(mockFs.listStatus(bucketDir)).thenReturn(
-        new FileStatus[] {app1DirStatus});
-
-    Path app1Log1 = new Path(app1Dir, "host1");
-    FileStatus app1Log1Status = new FileStatus(10, false, 1, 1, now, app1Log1);
-
-    when(mockFs.listStatus(app1Dir)).thenReturn(
-        new FileStatus[]{app1Log1Status});
-
-    final List<ApplicationId> finishedApplications =
-        Collections.unmodifiableList(Arrays.asList(appId1));
-
-    AggregatedLogDeletionService deletionSvc =
-        new AggregatedLogDeletionService() {
-      @Override
-      protected ApplicationClientProtocol createRMClient()
-          throws IOException {
-        try {
-          return createMockRMClient(finishedApplications, null);
-        } catch (Exception e) {
-          throw new IOException(e);
-        }
-      }
-      @Override
-      protected void stopRMClient() {
-        // DO NOTHING
-      }
-    };
-    deletionSvc.init(conf);
-    deletionSvc.start();
- 
-    verify(mockFs, timeout(10000).atLeast(4)).listStatus(any(Path.class));
-    verify(mockFs, never()).delete(app1Dir, true);
-
-    // modify the timestamp of the logs and verify it's picked up quickly
-    bucketDirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, bucketDir);
-    app1DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app1Dir);
-    app1Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app1Log1);
-    when(mockFs.listStatus(userDir)).thenReturn(
-        new FileStatus[] {suffixDirStatus});
-    when(mockFs.listStatus(suffixDir)).thenReturn(
-        new FileStatus[] {bucketDirStatus });
-    when(mockFs.listStatus(bucketDir)).thenReturn(
-        new FileStatus[] {app1DirStatus });
-    when(mockFs.listStatus(app1Dir)).thenReturn(
-        new FileStatus[]{app1Log1Status});
-
-    verify(mockFs, timeout(10000)).delete(app1Dir, true);
-
-    deletionSvc.stop();
+
+    LogAggregationTestcaseBuilder.create(conf)
+        .withRootPath(ROOT)
+        .withRemoteRootLogPath(REMOTE_ROOT_LOG_DIR)
+        .withUserDir(USER_ME, now)
+        .withSuffixDir(SUFFIX, now)
+        .withBucketDir(now)
+        .withApps(Lists.newArrayList(
+            new AppDescriptor(now,
+                Lists.newArrayList(Pair.of(DIR_HOST1, now))),
+            new AppDescriptor(now)))
+        .withFinishedApps(1)
+        .withRunningApps()
+        .build()
+        .startDeletionService()
+        .verifyAnyPathListedAtLeast(4, 10000L)
+        .verifyAppDirNotDeleted(1, NO_TIMEOUT)
+        // modify the timestamp of the logs and verify if it is picked up quickly
+        .changeModTimeOfApp(1, toDeleteTime)
+        .changeModTimeOfAppLogDir(1, 1, toDeleteTime)
+        .changeModTimeOfBucketDir(toDeleteTime)
+        .reinitAllPaths()
+        .verifyAppDirDeleted(1, 10000L)
+        .teardown(1);
   }
 
   @Test
-  public void testRobustLogDeletion() throws Exception {
-    final long RETENTION_SECS = 10 * 24 * 3600;
-
-    String root = "mockfs://foo/";
-    String remoteRootLogDir = root+"tmp/logs";
-    String suffix = "logs";
-    String newSuffix = LogAggregationUtils.getBucketSuffix() + suffix;
-    Configuration conf = new Configuration();
-    conf.setClass("fs.mockfs.impl", MockFileSystem.class,
-        FileSystem.class);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "864000");
-    conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS,
-        "1");
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
-    conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"),
-         LogAggregationTFileController.class.getName());
+  void testRobustLogDeletion() throws Exception {
+    Configuration conf = setupConfiguration(TEN_DAYS_IN_SECONDS, 1);
 
     // prevent us from picking up the same mockfs instance from another test
     FileSystem.closeAll();
-    Path rootPath = new Path(root);
-    FileSystem rootFs = rootPath.getFileSystem(conf);
-    FileSystem mockFs = ((FilterFileSystem)rootFs).getRawFileSystem();
-
-    Path remoteRootLogPath = new Path(remoteRootLogDir);
-
-    Path userDir = new Path(remoteRootLogPath, "me");
-    Path suffixDir = new Path(userDir, newSuffix);
-    FileStatus userDirStatus = new FileStatus(0, true, 0, 0, 0, userDir);
-    FileStatus suffixStatus = new FileStatus(0, true, 0, 0, 0, suffixDir);
-    Path bucketDir = new Path(suffixDir, String.valueOf(0));
-    FileStatus bucketDirStatus = new FileStatus(0, true, 0, 0, 0, bucketDir);
-
-    when(mockFs.listStatus(remoteRootLogPath)).thenReturn(
-        new FileStatus[]{userDirStatus});
-    when(mockFs.listStatus(userDir)).thenReturn(
-        new FileStatus[]{suffixStatus});
-    when(mockFs.listStatus(suffixDir)).thenReturn(
-        new FileStatus[]{bucketDirStatus});
-
-    ApplicationId appId1 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
-    Path app1Dir = new Path(bucketDir, appId1.toString());
-    FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, 0, app1Dir);
-    ApplicationId appId2 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 2);
-    Path app2Dir = new Path(bucketDir, "application_a");
-    FileStatus app2DirStatus = new FileStatus(0, true, 0, 0, 0, app2Dir);
-    ApplicationId appId3 =
-        ApplicationId.newInstance(System.currentTimeMillis(), 3);
-    Path app3Dir = new Path(bucketDir, appId3.toString());
-    FileStatus app3DirStatus = new FileStatus(0, true, 0, 0, 0, app3Dir);
-
-    when(mockFs.listStatus(bucketDir)).thenReturn(
-        new FileStatus[]{app1DirStatus, app2DirStatus, app3DirStatus});
-    when(mockFs.listStatus(app2Dir)).thenReturn(
-            new FileStatus[]{});
-
-    when(mockFs.listStatus(app1Dir)).thenThrow(
-        new RuntimeException("Should Be Caught and Logged"));
-    Path app3Log3 = new Path(app3Dir, "host1");
-    FileStatus app3Log3Status = new FileStatus(10, false, 1, 1, 0, app3Log3);
-    when(mockFs.listStatus(app3Dir)).thenReturn(
-        new FileStatus[]{app3Log3Status});
-
-    final List<ApplicationId> finishedApplications =
-        Collections.unmodifiableList(Arrays.asList(appId1, appId3));
-
-    ApplicationClientProtocol rmClient =
-        createMockRMClient(finishedApplications, null);
-    AggregatedLogDeletionService.LogDeletionTask deletionTask =
-        new AggregatedLogDeletionService.LogDeletionTask(conf,
-            RETENTION_SECS,
-            rmClient);
-    deletionTask.run();
-    verify(mockFs).delete(app3Dir, true);
+    long modTime = 0L;
+
+    LogAggregationTestcaseBuilder.create(conf)
+        .withRootPath(ROOT)
+        .withRemoteRootLogPath(REMOTE_ROOT_LOG_DIR)
+        .withUserDir(USER_ME, modTime)
+        .withSuffixDir(SUFFIX, modTime)
+        .withBucketDir(modTime, "0")
+        .withApps(Lists.newArrayList(
+            new AppDescriptor(modTime),
+            new AppDescriptor(modTime),
+            new AppDescriptor(modTime, Lists.newArrayList(Pair.of(DIR_HOST1, modTime)))))
+        .withAdditionalAppDirs(Lists.newArrayList(Pair.of("application_a", modTime)))
+        .withFinishedApps(1, 3)
+        .withRunningApps()
+        .injectExceptionForAppDirDeletion(1)
+        .build()
+        .runDeletionTask(TEN_DAYS_IN_SECONDS)
+        .verifyAppDirDeleted(3, NO_TIMEOUT);
+  }
+
+  @Test
+  void testDeletionTwoControllers() throws IOException {
+    long now = System.currentTimeMillis();
+    long toDeleteTime = now - (2000 * 1000);
+    long toKeepTime = now - (1500 * 1000);
+
+
+    Configuration conf = setupConfiguration(1800, -1);
+    enableFileControllers(conf, REMOTE_ROOT_LOG_DIR, ALL_FILE_CONTROLLERS,
+        ALL_FILE_CONTROLLER_NAMES);
+    long timeout = 2000L;
+    LogAggregationTestcaseBuilder.create(conf)
+        .withRootPath(ROOT)
+        .withRemoteRootLogPath(REMOTE_ROOT_LOG_DIR)
+        .withBothFileControllers()
+        .withUserDir(USER_ME, toKeepTime)
+        .withSuffixDir(SUFFIX, toDeleteTime)
+        .withBucketDir(toDeleteTime)
+        .withApps(//Apps for TFile
+            Lists.newArrayList(
+                new AppDescriptor(T_FILE, toDeleteTime, Lists.newArrayList()),
+                new AppDescriptor(T_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toKeepTime))),
+                new AppDescriptor(T_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toDeleteTime))),
+                new AppDescriptor(T_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toKeepTime))),
+                //Apps for IFile
+                new AppDescriptor(I_FILE, toDeleteTime, Lists.newArrayList()),
+                new AppDescriptor(I_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toKeepTime))),
+                new AppDescriptor(I_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toDeleteTime))),
+                new AppDescriptor(I_FILE, toDeleteTime, Lists.newArrayList(
+                    Pair.of(DIR_HOST1, toDeleteTime),
+                    Pair.of(DIR_HOST2, toKeepTime)))))
+        .withFinishedApps(1, 2, 3, 5, 6, 7)
+        .withRunningApps(4, 8)
+        .injectExceptionForAppDirDeletion(3, 6)
+        .build()
+        .startDeletionService()
+        .verifyAppDirsDeleted(timeout, 1, 3, 5, 7)
+        .verifyAppDirsNotDeleted(timeout, 2, 4, 6, 8)
+        .verifyAppFilesDeleted(timeout, Lists.newArrayList(Pair.of(4, 1), Pair.of(8, 1)))
+        .verifyAppFilesNotDeleted(timeout, Lists.newArrayList(Pair.of(4, 2), Pair.of(8, 2)))
+        .teardown(1);
   }
 
   static class MockFileSystem extends FilterFileSystem {
     MockFileSystem() {
       super(mock(FileSystem.class));
     }
+
     public void initialize(URI name, Configuration conf) throws IOException {}
-  }
 
-  private static ApplicationClientProtocol createMockRMClient(
-      List<ApplicationId> finishedApplicaitons,
-      List<ApplicationId> runningApplications) throws Exception {
-    final ApplicationClientProtocol mockProtocol =
-        mock(ApplicationClientProtocol.class);
-    if (finishedApplicaitons != null && !finishedApplicaitons.isEmpty()) {
-      for (ApplicationId appId : finishedApplicaitons) {
-        GetApplicationReportRequest request =
-            GetApplicationReportRequest.newInstance(appId);
-        GetApplicationReportResponse response =
-            createApplicationReportWithFinishedApplication();
-        when(mockProtocol.getApplicationReport(request))
-          .thenReturn(response);
-      }
+    @Override
+    public boolean hasPathCapability(Path path, String capability) {
+      return true;
     }
-    if (runningApplications != null && !runningApplications.isEmpty()) {
-      for (ApplicationId appId : runningApplications) {
-        GetApplicationReportRequest request =
-            GetApplicationReportRequest.newInstance(appId);
-        GetApplicationReportResponse response =
-            createApplicationReportWithRunningApplication();
-        when(mockProtocol.getApplicationReport(request))
-          .thenReturn(response);
-      }
-    }
-    return mockProtocol;
-  }
-
-  private static GetApplicationReportResponse
-      createApplicationReportWithRunningApplication() {
-    ApplicationReport report = mock(ApplicationReport.class);
-    when(report.getYarnApplicationState()).thenReturn(
-      YarnApplicationState.RUNNING);
-    GetApplicationReportResponse response =
-        mock(GetApplicationReportResponse.class);
-    when(response.getApplicationReport()).thenReturn(report);
-    return response;
-  }
-
-  private static GetApplicationReportResponse
-      createApplicationReportWithFinishedApplication() {
-    ApplicationReport report = mock(ApplicationReport.class);
-    when(report.getYarnApplicationState()).thenReturn(
-      YarnApplicationState.FINISHED);
-    GetApplicationReportResponse response =
-        mock(GetApplicationReportResponse.class);
-    when(response.getApplicationReport()).thenReturn(report);
-    return response;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index bf20fb7429275..5a4beca990c3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -18,10 +18,6 @@
 
 package org.apache.hadoop.yarn.logaggregation;
 
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.doThrow;
-
 import java.io.BufferedReader;
 import java.io.DataInputStream;
 import java.io.File;
@@ -41,9 +37,14 @@
 import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileStatus;
@@ -62,10 +63,15 @@
 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogWriter;
 import org.apache.hadoop.yarn.util.Times;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class TestAggregatedLogFormat {
 
@@ -84,8 +90,8 @@ public class TestAggregatedLogFormat {
     }
   }
 
-  @Before
-  @After
+  @BeforeEach
+  @AfterEach
   public void cleanupTestDir() throws Exception {
     Path workDirPath = new Path(testWorkDir.getAbsolutePath());
     LOG.info("Cleaning test directory [" + workDirPath + "]");
@@ -97,7 +103,7 @@ public void cleanupTestDir() throws Exception {
   //appending to logs
 
   @Test
-  public void testForCorruptedAggregatedLogs() throws Exception {
+  void testForCorruptedAggregatedLogs() throws Exception {
     Configuration conf = new Configuration();
     File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
     Path remoteAppLogFile =
@@ -112,7 +118,7 @@ public void testForCorruptedAggregatedLogs() throws Exception {
     long numChars = 950000;
 
     writeSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile,
-       srcFileRoot, testContainerId);
+        srcFileRoot, testContainerId);
 
     LogReader logReader = new LogReader(conf, remoteAppLogFile);
     LogKey rLogKey = new LogKey();
@@ -121,8 +127,8 @@ public void testForCorruptedAggregatedLogs() throws Exception {
     try {
       LogReader.readAcontainerLogs(dis, writer);
     } catch (Exception e) {
-      if(e.toString().contains("NumberFormatException")) {
-        Assert.fail("Aggregated logs are corrupted.");
+      if (e.toString().contains("NumberFormatException")) {
+        fail("Aggregated logs are corrupted.");
       }
     }
 
@@ -134,10 +140,10 @@ public void testForCorruptedAggregatedLogs() throws Exception {
       // Trying to read a corrupted log file created above should cause
       // log reading to fail below with an IOException.
       logReader = new LogReader(conf, remoteAppLogFile);
-      Assert.fail("Expect IOException from reading corrupt aggregated logs.");
+      fail("Expect IOException from reading corrupt aggregated logs.");
     } catch (IOException ioe) {
       DataInputStream dIS = logReader.next(rLogKey);
-      Assert.assertNull("Input stream not available for reading", dIS);
+      assertNull(dIS, "Input stream not available for reading");
     }
   }
 
@@ -198,7 +204,7 @@ public void run() {
   }
 
   @Test
-  public void testReadAcontainerLogs1() throws Exception {
+  void testReadAcontainerLogs1() throws Exception {
     //Verify the output generated by readAContainerLogs(DataInputStream, Writer, logUploadedTime)
     testReadAcontainerLog(true);
 
@@ -250,12 +256,10 @@ private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
 
       logWriter.append(logKey, spyLogValue);
     }
-
     // make sure permission are correct on the file
-    FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
-    Assert.assertEquals("permissions on log aggregation file are wrong",  
-      FsPermission.createImmutable((short) 0640), fsStatus.getPermission()); 
-
+    FileStatus fsStatus = fs.getFileStatus(remoteAppLogFile);
+    assertEquals(FsPermission.createImmutable((short) 0640), fsStatus.getPermission(),
+        "permissions on log aggregation file are wrong");
     LogReader logReader = new LogReader(conf, remoteAppLogFile);
     LogKey rLogKey = new LogKey();
     DataInputStream dis = logReader.next(rLogKey);
@@ -283,24 +287,24 @@ private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
         + numChars + ("\n").length() + ("End of LogType:stdout"
             + System.lineSeparator() + System.lineSeparator()).length();
 
-    Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
-    Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
-    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
-    Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
-    Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
+    assertTrue(s.contains("LogType:stdout"), "LogType not matched");
+    assertTrue(!s.contains("LogType:stderr"), "log file:stderr should not be aggregated.");
+    assertTrue(!s.contains("LogType:logs"), "log file:logs should not be aggregated.");
+    assertTrue(s.contains("LogLength:" + numChars), "LogLength not matched");
+    assertTrue(s.contains("Log Contents"), "Log Contents not matched");
     
     StringBuilder sb = new StringBuilder();
     for (int i = 0 ; i < numChars ; i++) {
       sb.append(filler);
     }
     String expectedContent = sb.toString();
-    Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
+    assertTrue(s.contains(expectedContent), "Log content incorrect");
     
-    Assert.assertEquals(expectedLength, s.length());
+    assertEquals(expectedLength, s.length());
   }
 
   @Test
-  public void testZeroLengthLog() throws IOException {
+  void testZeroLengthLog() throws IOException {
     Configuration conf = new Configuration();
     File workDir = new File(testWorkDir, "testZeroLength");
     Path remoteAppLogFile = new Path(workDir.getAbsolutePath(),
@@ -332,17 +336,18 @@ public void testZeroLengthLog() throws IOException {
     Writer writer = new StringWriter();
     LogReader.readAcontainerLogs(dis, writer);
 
-    Assert.assertEquals("LogType:stdout\n" +
+    assertEquals("LogType:stdout\n" +
         "LogLength:0\n" +
         "Log Contents:\n\n" +
         "End of LogType:stdout\n\n", writer.toString());
   }
 
-  @Test(timeout=10000)
-  public void testContainerLogsFileAccess() throws IOException {
+  @Test
+  @Timeout(10000)
+  void testContainerLogsFileAccess() throws IOException {
     // This test will run only if NativeIO is enabled as SecureIOUtils 
     // require it to be enabled.
-    Assume.assumeTrue(NativeIO.isAvailable());
+    Assumptions.assumeTrue(NativeIO.isAvailable());
     Configuration conf = new Configuration();
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
@@ -415,28 +420,28 @@ public void testContainerLogsFileAccess() throws IOException {
     String stdoutFile1 =
         StringUtils.join(
             File.separator,
-            Arrays.asList(new String[] {
+            Arrays.asList(new String[]{
                 workDir.getAbsolutePath(), "srcFiles",
                 testContainerId1.getApplicationAttemptId().getApplicationId()
-                    .toString(), testContainerId1.toString(), stderr }));
+                    .toString(), testContainerId1.toString(), stderr}));
 
     // The file: stdout is expected to be aggregated.
     String stdoutFile2 =
         StringUtils.join(
             File.separator,
-            Arrays.asList(new String[] {
+            Arrays.asList(new String[]{
                 workDir.getAbsolutePath(), "srcFiles",
                 testContainerId1.getApplicationAttemptId().getApplicationId()
-                    .toString(), testContainerId1.toString(), stdout }));
+                    .toString(), testContainerId1.toString(), stdout}));
     String message2 =
         "Owner '" + expectedOwner + "' for path "
             + stdoutFile2 + " did not match expected owner '"
             + ugi.getShortUserName() + "'";
-    
-    Assert.assertFalse(line.contains(message2));
-    Assert.assertFalse(line.contains(data + testContainerId1.toString()
+
+    assertFalse(line.contains(message2));
+    assertFalse(line.contains(data + testContainerId1.toString()
         + stderr));
-    Assert.assertTrue(line.contains(data + testContainerId1.toString()
+    assertTrue(line.contains(data + testContainerId1.toString()
         + stdout));
   }
   
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
index 66008a82323e2..2712560230415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java
@@ -28,9 +28,11 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
 import javax.servlet.http.HttpServletRequest;
 
+import com.google.inject.Inject;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
@@ -49,19 +51,17 @@
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.TFileAggregatedLogsBlock;
-import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.View.ViewContext;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.log.AggregatedLogsBlockForTest;
 import org.apache.hadoop.yarn.webapp.view.BlockForTest;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlockForTest;
-import org.junit.Test;
-
-import static org.mockito.Mockito.*;
-
-import com.google.inject.Inject;
 
-import static org.junit.Assert.*;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Test AggregatedLogsBlock. AggregatedLogsBlock should check user, aggregate a
@@ -73,7 +73,7 @@ public class TestAggregatedLogsBlock {
    * Bad user. User 'owner' is trying to read logs without access
    */
   @Test
-  public void testAccessDenied() throws Exception {
+  void testAccessDenied() throws Exception {
 
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
@@ -89,7 +89,7 @@ public void testAccessDenied() throws Exception {
     HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
     TFileAggregatedLogsBlockForTest aggregatedBlock
         = getTFileAggregatedLogsBlockForTest(configuration, "owner",
-            "container_0_0001_01_000001", "localhost:1234");
+        "container_0_0001_01_000001", "localhost:1234");
     aggregatedBlock.render(block);
 
     block.getWriter().flush();
@@ -100,7 +100,7 @@ public void testAccessDenied() throws Exception {
   }
 
   @Test
-  public void testBlockContainsPortNumForUnavailableAppLog() {
+  void testBlockContainsPortNumForUnavailableAppLog() {
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
 
@@ -125,7 +125,7 @@ public void testBlockContainsPortNumForUnavailableAppLog() {
    * @throws Exception
    */
   @Test
-  public void testBadLogs() throws Exception {
+  void testBadLogs() throws Exception {
 
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
@@ -146,8 +146,8 @@ public void testBadLogs() throws Exception {
     String out = data.toString();
     assertTrue(out
         .contains("Logs not available for entity. Aggregation may not be "
-        + "complete, Check back later or try to find the container logs "
-        + "in the local directory of nodemanager localhost:1234"));
+            + "complete, Check back later or try to find the container logs "
+            + "in the local directory of nodemanager localhost:1234"));
     assertTrue(out
         .contains("Or see application log at http://localhost:8042"));
 
@@ -160,7 +160,7 @@ public void testBadLogs() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testAggregatedLogsBlock() throws Exception {
+  void testAggregatedLogsBlock() throws Exception {
 
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
@@ -175,7 +175,7 @@ public void testAggregatedLogsBlock() throws Exception {
     HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
     TFileAggregatedLogsBlockForTest aggregatedBlock
         = getTFileAggregatedLogsBlockForTest(configuration, "admin",
-            "container_0_0001_01_000001", "localhost:1234");
+        "container_0_0001_01_000001", "localhost:1234");
     aggregatedBlock.render(block);
 
     block.getWriter().flush();
@@ -192,7 +192,7 @@ public void testAggregatedLogsBlock() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testAggregatedLogsBlockHar() throws Exception {
+  void testAggregatedLogsBlockHar() throws Exception {
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
 
@@ -209,7 +209,7 @@ public void testAggregatedLogsBlockHar() throws Exception {
     HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
     TFileAggregatedLogsBlockForTest aggregatedBlock
         = getTFileAggregatedLogsBlockForTest(configuration, "admin",
-            "container_1440536969523_0001_01_000001", "host1:1111");
+        "container_1440536969523_0001_01_000001", "host1:1111");
     aggregatedBlock.render(block);
 
     block.getWriter().flush();
@@ -238,7 +238,7 @@ public void testAggregatedLogsBlockHar() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testNoLogs() throws Exception {
+  void testNoLogs() throws Exception {
 
     FileUtil.fullyDelete(new File("target/logs"));
     Configuration configuration = getConfiguration();
@@ -255,7 +255,7 @@ public void testNoLogs() throws Exception {
     HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
     TFileAggregatedLogsBlockForTest aggregatedBlock
         = getTFileAggregatedLogsBlockForTest(configuration, "admin",
-            "container_0_0001_01_000001", "localhost:1234");
+        "container_0_0001_01_000001", "localhost:1234");
     aggregatedBlock.render(block);
 
     block.getWriter().flush();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java
index 3687d023d6968..cce8a62ba1e00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.yarn.logaggregation;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -27,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -39,6 +38,8 @@
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
 
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 /**
  * This class contains several utility functions for log aggregation tests.
  */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogAggregationMetaCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogAggregationMetaCollector.java
index c60635b0e2ace..e7434f4b990a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogAggregationMetaCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogAggregationMetaCollector.java
@@ -18,19 +18,6 @@
 
 package org.apache.hadoop.yarn.logaggregation;
 
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.logaggregation.filecontroller.FakeLogAggregationFileController;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
 import java.io.IOException;
 import java.time.Clock;
 import java.util.ArrayList;
@@ -42,7 +29,23 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import static org.junit.Assert.*;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.filecontroller.FakeLogAggregationFileController;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestLogAggregationMetaCollector {
   private static final String TEST_NODE = "TEST_NODE_1";
@@ -133,17 +136,17 @@ public Map<String, List<ContainerLogFileInfo>> getLogMetaFilesOfNode(
     }
   }
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     fileController = createFileController();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
   }
 
   @Test
-  public void testAllNull() throws IOException {
+  void testAllNull() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     request.setAppId(null);
@@ -165,7 +168,7 @@ public void testAllNull() throws IOException {
   }
 
   @Test
-  public void testAllSet() throws IOException {
+  void testAllSet() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     Set<String> fileSizeExpressions = new HashSet<>();
@@ -191,7 +194,7 @@ public void testAllSet() throws IOException {
   }
 
   @Test
-  public void testSingleNodeRequest() throws IOException {
+  void testSingleNodeRequest() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     request.setAppId(null);
@@ -214,7 +217,7 @@ public void testSingleNodeRequest() throws IOException {
   }
 
   @Test
-  public void testMultipleNodeRegexRequest() throws IOException {
+  void testMultipleNodeRegexRequest() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     request.setAppId(null);
@@ -236,7 +239,7 @@ public void testMultipleNodeRegexRequest() throws IOException {
   }
 
   @Test
-  public void testMultipleFileRegex() throws IOException {
+  void testMultipleFileRegex() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     request.setAppId(null);
@@ -260,7 +263,7 @@ public void testMultipleFileRegex() throws IOException {
   }
 
   @Test
-  public void testContainerIdExactMatch() throws IOException {
+  void testContainerIdExactMatch() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     request.setAppId(null);
@@ -284,7 +287,7 @@ public void testContainerIdExactMatch() throws IOException {
   }
 
   @Test
-  public void testMultipleFileBetweenSize() throws IOException {
+  void testMultipleFileBetweenSize() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     Set<String> fileSizeExpressions = new HashSet<>();
@@ -311,7 +314,7 @@ public void testMultipleFileBetweenSize() throws IOException {
   }
 
   @Test
-  public void testInvalidQueryStrings() throws IOException {
+  void testInvalidQueryStrings() throws IOException {
     ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder request =
         new ExtendedLogMetaRequest.ExtendedLogMetaRequestBuilder();
     Set<String> fileSizeExpressions = new HashSet<>();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/FakeLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/FakeLogAggregationFileController.java
index c667d3b4fee35..d28cee3b1bdd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/FakeLogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/FakeLogAggregationFileController.java
@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.yarn.logaggregation.filecontroller;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -28,11 +33,6 @@
 import org.apache.hadoop.yarn.webapp.View;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-import java.util.Map;
-
 public class FakeLogAggregationFileController
     extends LogAggregationFileController {
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
index 818e01129fc60..fe1c5f2fa7328 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
@@ -18,6 +18,13 @@
 
 package org.apache.hadoop.yarn.logaggregation.filecontroller;
 
+import java.io.FileNotFoundException;
+import java.net.URI;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -25,15 +32,9 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.io.FileNotFoundException;
-import java.net.URI;
 
 import static org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController.TLDIR_PERMISSIONS;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.ArgumentMatchers.eq;
@@ -49,19 +50,19 @@
 public class TestLogAggregationFileController {
 
   @Test
-  public void testRemoteDirCreationDefault() throws Exception {
+  void testRemoteDirCreationDefault() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     doReturn(new URI("")).when(fs).getUri();
     doThrow(FileNotFoundException.class).when(fs)
-            .getFileStatus(any(Path.class));
+        .getFileStatus(any(Path.class));
 
     Configuration conf = new Configuration();
     LogAggregationFileController controller = mock(
-            LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
+        LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
     doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
 
     UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
-        "yarn_user", new String[] {"yarn_group", "other_group"});
+        "yarn_user", new String[]{"yarn_group", "other_group"});
     UserGroupInformation.setLoginUser(ugi);
 
     controller.initialize(conf, "TFile");
@@ -71,7 +72,7 @@ public void testRemoteDirCreationDefault() throws Exception {
   }
 
   @Test
-  public void testRemoteDirCreationWithCustomGroup() throws Exception {
+  void testRemoteDirCreationWithCustomGroup() throws Exception {
     String testGroupName = "testGroup";
 
     FileSystem fs = mock(FileSystem.class);
@@ -86,7 +87,7 @@ public void testRemoteDirCreationWithCustomGroup() throws Exception {
     doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
 
     UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
-        "yarn_user", new String[] {"yarn_group", "other_group"});
+        "yarn_user", new String[]{"yarn_group", "other_group"});
     UserGroupInformation.setLoginUser(ugi);
 
     controller.initialize(conf, "TFile");
@@ -114,7 +115,7 @@ public String toString() {
   }
 
   @Test
-  public void testRemoteDirCreationWithCustomUser() throws Exception {
+  void testRemoteDirCreationWithCustomUser() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     doReturn(new URI("")).when(fs).getUri();
     doReturn(new FileStatus(128, false, 0, 64, System.currentTimeMillis(),
@@ -139,6 +140,6 @@ public void testRemoteDirCreationWithCustomUser() throws Exception {
     verify(fs).setPermission(argThat(new PathContainsString(".permission_check")),
         eq(new FsPermission(TLDIR_PERMISSIONS)));
     verify(fs).delete(argThat(new PathContainsString(".permission_check")), eq(false));
-    Assert.assertTrue(controller.fsSupportsChmod);
+    assertTrue(controller.fsSupportsChmod);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
index 2d2fb49c0efc9..0f879616f3bef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileControllerFactory.java
@@ -18,13 +18,6 @@
 
 package org.apache.hadoop.yarn.logaggregation.filecontroller;
 
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -35,7 +28,11 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
@@ -51,10 +48,14 @@
 import org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController;
 import org.apache.hadoop.yarn.webapp.View.ViewContext;
 import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS;
+import static org.apache.hadoop.yarn.logaggregation.LogAggregationTestUtils.REMOTE_LOG_ROOT;
+import static org.apache.hadoop.yarn.logaggregation.LogAggregationTestUtils.enableFileControllers;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Test LogAggregationFileControllerFactory.
@@ -63,7 +64,6 @@ public class TestLogAggregationFileControllerFactory extends Configured {
   private static final Logger LOG = LoggerFactory.getLogger(
       TestLogAggregationFileControllerFactory.class);
 
-  private static final String REMOTE_LOG_ROOT = "target/app-logs/";
   private static final String REMOTE_DEFAULT_DIR = "default/";
   private static final String APP_OWNER = "test";
 
@@ -83,12 +83,11 @@ public class TestLogAggregationFileControllerFactory extends Configured {
   private ApplicationId appId = ApplicationId.newInstance(
       System.currentTimeMillis(), 1);
 
-  @Before
+  @BeforeEach
   public void setup() throws IOException {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
-    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, REMOTE_LOG_ROOT +
-        REMOTE_DEFAULT_DIR);
+    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, REMOTE_LOG_ROOT + REMOTE_DEFAULT_DIR);
     conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, "log");
     setConf(conf);
   }
@@ -112,135 +111,121 @@ private void verifyFileControllerInstance(
                new FileWriter(new File(logPath.toString(), "testLog"))) {
         writer.write("test");
       }
-      assertTrue("The used LogAggregationFileController is not instance of "
-          + className.getSimpleName(), className.isInstance(
-              factory.getFileControllerForRead(appId, APP_OWNER)));
+      assertTrue(className.isInstance(factory.getFileControllerForRead(appId, APP_OWNER)),
+          "The used LogAggregationFileController is not instance of " + className.getSimpleName());
     } finally {
       fs.delete(logPath, true);
     }
   }
 
   @Test
-  public void testDefaultLogAggregationFileControllerFactory()
+  void testDefaultLogAggregationFileControllerFactory()
       throws IOException {
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(getConf());
     List<LogAggregationFileController> list = factory
         .getConfiguredLogAggregationFileControllerList();
 
-    assertEquals("Only one LogAggregationFileController is expected!", 1,
-        list.size());
-    assertTrue("TFile format is expected to be the first " +
-        "LogAggregationFileController!", list.get(0) instanceof
-        LogAggregationTFileController);
-    assertTrue("TFile format is expected to be used for writing!",
-        factory.getFileControllerForWrite() instanceof
-            LogAggregationTFileController);
+    assertEquals(1,
+        list.size(),
+        "Only one LogAggregationFileController is expected!");
+    assertTrue(list.get(0) instanceof
+        LogAggregationTFileController, "TFile format is expected to be the first " +
+        "LogAggregationFileController!");
+    assertTrue(factory.getFileControllerForWrite() instanceof
+            LogAggregationTFileController,
+        "TFile format is expected to be used for writing!");
 
     verifyFileControllerInstance(factory, LogAggregationTFileController.class);
   }
 
-  @Test(expected = Exception.class)
-  public void testLogAggregationFileControllerFactoryClassNotSet() {
-    Configuration conf = getConf();
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS,
-        "TestLogAggregationFileController");
-    new LogAggregationFileControllerFactory(conf);
-    fail("TestLogAggregationFileController's class was not set, " +
-        "but the factory creation did not fail.");
-  }
-
-  private void enableFileControllers(
-      List<Class<? extends LogAggregationFileController>> fileControllers,
-      List<String> fileControllerNames) {
-    Configuration conf = getConf();
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS,
-        StringUtils.join(fileControllerNames, ","));
-    for (int i = 0; i < fileControllers.size(); i++) {
-      Class<? extends LogAggregationFileController> fileController =
-          fileControllers.get(i);
-      String controllerName = fileControllerNames.get(i);
-
-      conf.setClass(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT,
-          controllerName), fileController, LogAggregationFileController.class);
-      conf.set(String.format(LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT,
-          controllerName), REMOTE_LOG_ROOT + controllerName + "/");
-      conf.set(String.format(LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT,
-          controllerName), controllerName);
-    }
+  @Test
+  void testLogAggregationFileControllerFactoryClassNotSet() {
+    assertThrows(Exception.class, () -> {
+      Configuration conf = getConf();
+      conf.set(LOG_AGGREGATION_FILE_FORMATS, "TestLogAggregationFileController");
+      new LogAggregationFileControllerFactory(conf);
+      fail("TestLogAggregationFileController's class was not set, " +
+          "but the factory creation did not fail.");
+    });
   }
 
   @Test
-  public void testLogAggregationFileControllerFactory() throws Exception {
-    enableFileControllers(ALL_FILE_CONTROLLERS, ALL_FILE_CONTROLLER_NAMES);
+  void testLogAggregationFileControllerFactory() throws Exception {
+    enableFileControllers(getConf(), ALL_FILE_CONTROLLERS, ALL_FILE_CONTROLLER_NAMES);
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(getConf());
     List<LogAggregationFileController> list =
         factory.getConfiguredLogAggregationFileControllerList();
 
-    assertEquals("The expected number of LogAggregationFileController " +
-        "is not 3!", 3, list.size());
-    assertTrue("Test format is expected to be the first " +
-        "LogAggregationFileController!", list.get(0) instanceof
-        TestLogAggregationFileController);
-    assertTrue("IFile format is expected to be the second " +
-        "LogAggregationFileController!", list.get(1) instanceof
-        LogAggregationIndexedFileController);
-    assertTrue("TFile format is expected to be the first " +
-        "LogAggregationFileController!", list.get(2) instanceof
-        LogAggregationTFileController);
-    assertTrue("Test format is expected to be used for writing!",
-        factory.getFileControllerForWrite() instanceof
-            TestLogAggregationFileController);
+    assertEquals(3, list.size(), "The expected number of LogAggregationFileController " +
+        "is not 3!");
+    assertTrue(list.get(0) instanceof
+        TestLogAggregationFileController, "Test format is expected to be the first " +
+        "LogAggregationFileController!");
+    assertTrue(list.get(1) instanceof
+        LogAggregationIndexedFileController, "IFile format is expected to be the second " +
+        "LogAggregationFileController!");
+    assertTrue(list.get(2) instanceof
+        LogAggregationTFileController, "TFile format is expected to be the first " +
+        "LogAggregationFileController!");
+    assertTrue(factory.getFileControllerForWrite() instanceof
+            TestLogAggregationFileController,
+        "Test format is expected to be used for writing!");
 
     verifyFileControllerInstance(factory,
         TestLogAggregationFileController.class);
   }
 
   @Test
-  public void testClassConfUsed() {
-    enableFileControllers(Collections.singletonList(
-        LogAggregationTFileController.class),
+  void testClassConfUsed() {
+    enableFileControllers(getConf(), Collections.singletonList(LogAggregationTFileController.class),
         Collections.singletonList("TFile"));
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(getConf());
     LogAggregationFileController fc = factory.getFileControllerForWrite();
 
-    assertEquals(WRONG_ROOT_LOG_DIR_MSG, "target/app-logs/TFile",
-        fc.getRemoteRootLogDir().toString());
-    assertEquals(WRONG_ROOT_LOG_DIR_SUFFIX_MSG, "TFile",
-        fc.getRemoteRootLogDirSuffix());
+    assertEquals("target/app-logs/TFile",
+        fc.getRemoteRootLogDir().toString(),
+        WRONG_ROOT_LOG_DIR_MSG);
+    assertEquals("TFile",
+        fc.getRemoteRootLogDirSuffix(),
+        WRONG_ROOT_LOG_DIR_SUFFIX_MSG);
   }
 
   @Test
-  public void testNodemanagerConfigurationIsUsed() {
+  void testNodemanagerConfigurationIsUsed() {
     Configuration conf = getConf();
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
+    conf.set(LOG_AGGREGATION_FILE_FORMATS, "TFile");
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(conf);
     LogAggregationFileController fc = factory.getFileControllerForWrite();
 
-    assertEquals(WRONG_ROOT_LOG_DIR_MSG, "target/app-logs/default",
-        fc.getRemoteRootLogDir().toString());
-    assertEquals(WRONG_ROOT_LOG_DIR_SUFFIX_MSG, "log-tfile",
-        fc.getRemoteRootLogDirSuffix());
+    assertEquals("target/app-logs/default",
+        fc.getRemoteRootLogDir().toString(),
+        WRONG_ROOT_LOG_DIR_MSG);
+    assertEquals("log-tfile",
+        fc.getRemoteRootLogDirSuffix(),
+        WRONG_ROOT_LOG_DIR_SUFFIX_MSG);
   }
 
   @Test
-  public void testDefaultConfUsed() {
+  void testDefaultConfUsed() {
     Configuration conf = getConf();
     conf.unset(YarnConfiguration.NM_REMOTE_APP_LOG_DIR);
     conf.unset(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX);
-    conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile");
+    conf.set(LOG_AGGREGATION_FILE_FORMATS, "TFile");
 
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(getConf());
     LogAggregationFileController fc = factory.getFileControllerForWrite();
 
-    assertEquals(WRONG_ROOT_LOG_DIR_MSG, "/tmp/logs",
-        fc.getRemoteRootLogDir().toString());
-    assertEquals(WRONG_ROOT_LOG_DIR_SUFFIX_MSG, "logs-tfile",
-        fc.getRemoteRootLogDirSuffix());
+    assertEquals("/tmp/logs",
+        fc.getRemoteRootLogDir().toString(),
+        WRONG_ROOT_LOG_DIR_MSG);
+    assertEquals("logs-tfile",
+        fc.getRemoteRootLogDirSuffix(),
+        WRONG_ROOT_LOG_DIR_SUFFIX_MSG);
   }
 
   private static class TestLogAggregationFileController
@@ -268,20 +253,19 @@ public void postWrite(LogAggregationFileControllerContext record)
     }
 
     @Override
-    public void initializeWriter(LogAggregationFileControllerContext context)
-        throws IOException {
+    public void initializeWriter(LogAggregationFileControllerContext context) {
       // Do Nothing
     }
 
     @Override
     public boolean readAggregatedLogs(ContainerLogsRequest logRequest,
-        OutputStream os) throws IOException {
+        OutputStream os) {
       return false;
     }
 
     @Override
     public List<ContainerLogMeta> readAggregatedLogsMeta(
-        ContainerLogsRequest logRequest) throws IOException {
+        ContainerLogsRequest logRequest) {
       return null;
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java
index dfb19d490785d..cd178382b5290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexedFileController.java
@@ -32,6 +32,12 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -48,27 +54,24 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
+import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
 import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
 import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
 import org.apache.hadoop.yarn.logaggregation.ExtendedLogMetaRequest;
 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogValue;
-import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
 import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ControlledClock;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -104,7 +107,7 @@ private Configuration getTestConf() {
     return conf;
   }
 
-  @Before
+  @BeforeEach
   public void setUp() throws IOException {
     setConf(getTestConf());
     appId = ApplicationId.newInstance(123456, 1);
@@ -122,14 +125,15 @@ public void setUp() throws IOException {
     System.setErr(sysErr);
   }
 
-  @After
+  @AfterEach
   public void teardown() throws Exception {
     fs.delete(rootLocalLogDirPath, true);
     fs.delete(new Path(remoteLogDir), true);
   }
 
-  @Test(timeout = 15000)
-  public void testLogAggregationIndexFileFormat() throws Exception {
+  @Test
+  @Timeout(15000)
+  void testLogAggregationIndexFileFormat() throws Exception {
     if (fs.exists(rootLocalLogDirPath)) {
       fs.delete(rootLocalLogDirPath, true);
     }
@@ -150,7 +154,7 @@ public void testLogAggregationIndexFileFormat() throws Exception {
 
     LogKey key1 = new LogKey(containerId.toString());
 
-    for(String logType : logTypes) {
+    for (String logType : logTypes) {
       File file = createAndWriteLocalLogFile(containerId, appLogsDir,
           logType);
       files.add(file);
@@ -162,24 +166,23 @@ public void testLogAggregationIndexFileFormat() throws Exception {
 
     final ControlledClock clock = new ControlledClock();
     clock.setTime(System.currentTimeMillis());
-    LogAggregationIndexedFileController fileFormat
-        = new LogAggregationIndexedFileController() {
-          private int rollOverCheck = 0;
-          @Override
-          public Clock getSystemClock() {
-            return clock;
-          }
-
-          @Override
-          public boolean isRollover(final FileContext fc,
-              final Path candidate) throws IOException {
-            rollOverCheck++;
-            if (rollOverCheck >= 3) {
-              return true;
-            }
-            return false;
-          }
-        };
+    LogAggregationIndexedFileController fileFormat = new LogAggregationIndexedFileController() {
+      private int rollOverCheck = 0;
+
+      @Override
+      public Clock getSystemClock() {
+        return clock;
+      }
+
+      @Override
+      public boolean isRollover(final FileContext fc, final Path candidate) throws IOException {
+        rollOverCheck++;
+        if (rollOverCheck >= 3) {
+          return true;
+        }
+        return false;
+      }
+    };
 
     fileFormat.initialize(getConf(), "Indexed");
 
@@ -238,7 +241,7 @@ public boolean isRollover(final FileContext fc,
     factoryConf.set("yarn.log-aggregation.file-formats", "Indexed");
     factoryConf.set("yarn.log-aggregation.file-controller.Indexed.class",
         "org.apache.hadoop.yarn.logaggregation.filecontroller.ifile"
-        + ".LogAggregationIndexedFileController");
+            + ".LogAggregationIndexedFileController");
     LogAggregationFileControllerFactory factory =
         new LogAggregationFileControllerFactory(factoryConf);
     LogAggregationFileController fileController = factory
@@ -255,9 +258,9 @@ public boolean isRollover(final FileContext fc,
 
     // create a checksum file
     Path checksumFile = new Path(fileFormat.getRemoteAppLogDir(
-        appId, USER_UGI.getShortUserName()),
+            appId, USER_UGI.getShortUserName()),
         LogAggregationUtils.getNodeString(nodeId)
-        + LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
+            + LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
     FSDataOutputStream fInput = null;
     try {
       String nodeName = logPath.getName() + "_" + clock.getTime();
@@ -330,7 +333,7 @@ public boolean isRollover(final FileContext fc,
     fileFormat.postWrite(context);
     fileFormat.closeWriter();
     meta = fileFormat.readAggregatedLogsMeta(
-            logRequest);
+        logRequest);
     assertThat(meta.size()).isEqualTo(2);
     for (ContainerLogMeta log : meta) {
       assertEquals(containerId.toString(), log.getContainerId());
@@ -380,8 +383,9 @@ public boolean isRollover(final FileContext fc,
     sysOutStream.reset();
   }
 
-  @Test(timeout = 15000)
-  public void testFetchApplictionLogsHar() throws Exception {
+  @Test
+  @Timeout(15000)
+  void testFetchApplictionLogsHar() throws Exception {
     List<String> newLogTypes = new ArrayList<>();
     newLogTypes.add("syslog");
     newLogTypes.add("stdout");
@@ -472,7 +476,7 @@ private String logMessage(ContainerId containerId, String logType) {
   }
 
   @Test
-  public void testGetRollOverLogMaxSize() {
+  void testGetRollOverLogMaxSize() {
     String fileControllerName = "testController";
     String remoteDirConf = String.format(
         YarnConfiguration.LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT,
@@ -500,7 +504,7 @@ public void testGetRollOverLogMaxSize() {
   }
 
   @Test
-  public void testGetLogMetaFilesOfNode() throws Exception {
+  void testGetLogMetaFilesOfNode() throws Exception {
     if (fs.exists(rootLocalLogDirPath)) {
       fs.delete(rootLocalLogDirPath, true);
     }
@@ -521,7 +525,7 @@ public void testGetLogMetaFilesOfNode() throws Exception {
 
     LogKey key1 = new LogKey(containerId.toString());
 
-    for(String logType : logTypes) {
+    for (String logType : logTypes) {
       File file = createAndWriteLocalLogFile(containerId, appLogsDir,
           logType);
       files.add(file);
@@ -566,7 +570,7 @@ public void testGetLogMetaFilesOfNode() throws Exception {
     final ControlledClock clock = new ControlledClock();
     clock.setTime(System.currentTimeMillis());
     Path checksumFile = new Path(fileFormat.getRemoteAppLogDir(
-        appId, USER_UGI.getShortUserName()),
+            appId, USER_UGI.getShortUserName()),
         LogAggregationUtils.getNodeString(nodeId)
             + LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX);
     FSDataOutputStream fInput = null;
@@ -593,11 +597,11 @@ public void testGetLogMetaFilesOfNode() throws Exception {
 
       if (node.getPath().getName().contains(
           LogAggregationIndexedFileController.CHECK_SUM_FILE_SUFFIX)) {
-        assertTrue("Checksum node files should not contain any logs",
-            metas.isEmpty());
+        assertTrue(metas.isEmpty(),
+            "Checksum node files should not contain any logs");
       } else {
-        assertFalse("Non-checksum node files should contain log files",
-            metas.isEmpty());
+        assertFalse(metas.isEmpty(),
+            "Non-checksum node files should contain log files");
         assertEquals(4, metas.values().stream().findFirst().get().size());
       }
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/AggregatedLogDeletionServiceForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/AggregatedLogDeletionServiceForTest.java
new file mode 100644
index 0000000000000..88f2d42a11deb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/AggregatedLogDeletionServiceForTest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
+
+import static org.apache.hadoop.yarn.logaggregation.testutils.MockRMClientUtils.createMockRMClient;
+
+public class AggregatedLogDeletionServiceForTest extends AggregatedLogDeletionService {
+  private final List<ApplicationId> finishedApplications;
+  private final List<ApplicationId> runningApplications;
+  private final Configuration conf;
+  private ApplicationClientProtocol mockRMClient;
+
+  public AggregatedLogDeletionServiceForTest(List<ApplicationId> runningApplications,
+                                             List<ApplicationId> finishedApplications) {
+    this(runningApplications, finishedApplications, null);
+  }
+
+  public AggregatedLogDeletionServiceForTest(List<ApplicationId> runningApplications,
+                                             List<ApplicationId> finishedApplications,
+                                             Configuration conf) {
+    this.runningApplications = runningApplications;
+    this.finishedApplications = finishedApplications;
+    this.conf = conf;
+  }
+
+  @Override
+  protected ApplicationClientProtocol createRMClient() throws IOException {
+    if (mockRMClient != null) {
+      return mockRMClient;
+    }
+    try {
+      mockRMClient =
+          createMockRMClient(finishedApplications, runningApplications);
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+    return mockRMClient;
+  }
+
+  @Override
+  protected Configuration createConf() {
+    return conf;
+  }
+
+  public ApplicationClientProtocol getMockRMClient() {
+    return mockRMClient;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/FileStatusUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/FileStatusUtils.java
new file mode 100644
index 0000000000000..c4af8618614a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/FileStatusUtils.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
+
+public class FileStatusUtils {
+  public static PathWithFileStatus createPathWithFileStatusForAppId(Path remoteRootLogDir,
+                                                             ApplicationId appId,
+                                                             String user, String suffix,
+                                                             long modificationTime) {
+    Path path = LogAggregationUtils.getRemoteAppLogDir(
+            remoteRootLogDir, appId, user, suffix);
+    FileStatus fileStatus = createEmptyFileStatus(modificationTime, path);
+    return new PathWithFileStatus(path, fileStatus);
+  }
+
+  public static FileStatus createEmptyFileStatus(long modificationTime, Path path) {
+    return new FileStatus(0, true, 0, 0, modificationTime, path);
+  }
+
+  public static PathWithFileStatus createFileLogPathWithFileStatus(Path baseDir, String childDir,
+                                                            long modificationTime) {
+    Path logPath = new Path(baseDir, childDir);
+    FileStatus fStatus = createFileStatusWithLengthForFile(10, modificationTime, logPath);
+    return new PathWithFileStatus(logPath, fStatus);
+  }
+
+  public static PathWithFileStatus createDirLogPathWithFileStatus(Path baseDir, String childDir,
+                                                           long modificationTime) {
+    Path logPath = new Path(baseDir, childDir);
+    FileStatus fStatus = createFileStatusWithLengthForDir(10, modificationTime, logPath);
+    return new PathWithFileStatus(logPath, fStatus);
+  }
+
+  public static PathWithFileStatus createDirBucketDirLogPathWithFileStatus(Path remoteRootLogPath,
+                                                                    String user,
+                                                                    String suffix,
+                                                                    ApplicationId appId,
+                                                                    long modificationTime) {
+    Path bucketDir = LogAggregationUtils.getRemoteBucketDir(remoteRootLogPath, user, suffix, appId);
+    FileStatus fStatus = new FileStatus(0, true, 0, 0, modificationTime, bucketDir);
+    return new PathWithFileStatus(bucketDir, fStatus);
+  }
+
+  public static FileStatus createFileStatusWithLengthForFile(long length,
+                                                              long modificationTime,
+                                                              Path logPath) {
+    return new FileStatus(length, false, 1, 1, modificationTime, logPath);
+  }
+
+  public static FileStatus createFileStatusWithLengthForDir(long length,
+                                                             long modificationTime,
+                                                             Path logPath) {
+    return new FileStatus(length, true, 1, 1, modificationTime, logPath);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcase.java
new file mode 100644
index 0000000000000..2af74f8cbb870
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcase.java
@@ -0,0 +1,451 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService.LogDeletionTask;
+import org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcaseBuilder.AppDescriptor;
+
+import static org.apache.hadoop.yarn.logaggregation.testutils.FileStatusUtils.createDirBucketDirLogPathWithFileStatus;
+import static org.apache.hadoop.yarn.logaggregation.testutils.FileStatusUtils.createDirLogPathWithFileStatus;
+import static org.apache.hadoop.yarn.logaggregation.testutils.FileStatusUtils.createFileLogPathWithFileStatus;
+import static org.apache.hadoop.yarn.logaggregation.testutils.FileStatusUtils.createPathWithFileStatusForAppId;
+import static org.apache.hadoop.yarn.logaggregation.testutils.LogAggregationTestcaseBuilder.NO_TIMEOUT;
+import static org.apache.hadoop.yarn.logaggregation.testutils.MockRMClientUtils.createMockRMClient;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class LogAggregationTestcase {
+  private static final Logger LOG = LoggerFactory.getLogger(LogAggregationTestcase.class);
+
+  private final Configuration conf;
+  private final long now;
+  private PathWithFileStatus bucketDir;
+  private final long bucketDirModTime;
+  private PathWithFileStatus userDir;
+  private final String userDirName;
+  private final long userDirModTime;
+  private PathWithFileStatus suffixDir;
+  private final String suffix;
+  private final String suffixDirName;
+  private final long suffixDirModTime;
+  private final String bucketId;
+  private final Path remoteRootLogPath;
+  private final Map<Integer, Exception> injectedAppDirDeletionExceptions;
+  private final List<String> fileControllers;
+  private final List<Pair<String, Long>> additionalAppDirs;
+
+  private final List<ApplicationId> applicationIds = new ArrayList<>();
+  private final int[] runningAppIds;
+  private final int[] finishedAppIds;
+  private final List<List<PathWithFileStatus>> appFiles = new ArrayList<>();
+  private final FileSystem mockFs;
+  private List<PathWithFileStatus> appDirs;
+  private final List<AppDescriptor> appDescriptors;
+  private AggregatedLogDeletionServiceForTest deletionService;
+  private ApplicationClientProtocol rmClient;
+
+  public LogAggregationTestcase(LogAggregationTestcaseBuilder builder) throws IOException {
+    conf = builder.conf;
+    now = builder.now;
+    bucketDir = builder.bucketDir;
+    bucketDirModTime = builder.bucketDirModTime;
+    userDir = builder.userDir;
+    userDirName = builder.userDirName;
+    userDirModTime = builder.userDirModTime;
+    suffix = builder.suffix;
+    suffixDir = builder.suffixDir;
+    suffixDirName = builder.suffixDirName;
+    suffixDirModTime = builder.suffixDirModTime;
+    bucketId = builder.bucketId;
+    appDescriptors = builder.apps;
+    runningAppIds = builder.runningAppIds;
+    finishedAppIds = builder.finishedAppIds;
+    remoteRootLogPath = builder.remoteRootLogPath;
+    injectedAppDirDeletionExceptions = builder.injectedAppDirDeletionExceptions;
+    fileControllers = builder.fileControllers;
+    additionalAppDirs = builder.additionalAppDirs;
+
+    mockFs = ((FilterFileSystem) builder.rootFs).getRawFileSystem();
+    validateAppControllers();
+    setupMocks();
+
+    setupDeletionService();
+  }
+
+  private void validateAppControllers() {
+    Set<String> controllers = appDescriptors.stream()
+            .map(a -> a.fileController)
+            .filter(Objects::nonNull)
+            .collect(Collectors.toSet());
+    Set<String> availableControllers = fileControllers != null ?
+            new HashSet<>(this.fileControllers) : Sets.newHashSet();
+    Set<String> difference = Sets.difference(controllers, availableControllers);
+    if (!difference.isEmpty()) {
+      throw new IllegalStateException(String.format("Invalid controller defined!" +
+                      " Available: %s, Actual: %s", availableControllers, controllers));
+    }
+  }
+
+  private void setupMocks() throws IOException {
+    createApplicationsByDescriptors();
+
+    List<Path> rootPaths = determineRootPaths();
+    for (Path rootPath : rootPaths) {
+      String controllerName = rootPath.getName();
+      ApplicationId arbitraryAppIdForBucketDir = this.applicationIds.get(0);
+      userDir = createDirLogPathWithFileStatus(rootPath, userDirName, userDirModTime);
+      suffixDir = createDirLogPathWithFileStatus(userDir.path, suffixDirName, suffixDirModTime);
+      if (bucketId != null) {
+        bucketDir = createDirLogPathWithFileStatus(suffixDir.path, bucketId, bucketDirModTime);
+      } else {
+        bucketDir = createDirBucketDirLogPathWithFileStatus(rootPath, userDirName, suffix,
+                arbitraryAppIdForBucketDir, bucketDirModTime);
+      }
+      setupListStatusForPath(rootPath, userDir);
+      initFileSystemListings(controllerName);
+    }
+  }
+
+  private List<Path> determineRootPaths() {
+    List<Path> rootPaths = new ArrayList<>();
+    if (fileControllers != null && !fileControllers.isEmpty()) {
+      for (String fileController : fileControllers) {
+        //Generic path: <remote-app-log-dir>/<user>/bucket-<suffix>/<bucket id>/
+        // <application id>/<NodeManager id>
+
+        //remoteRootLogPath: <remote-app-log-dir>/
+        //example: mockfs://foo/tmp/logs/
+
+        //userDir: <remote-app-log-dir>/<user>/
+        //example: mockfs://foo/tmp/logs/me/
+
+        //suffixDir: <remote-app-log-dir>/<user>/bucket-<suffix>/
+        //example: mockfs://foo/tmp/logs/me/bucket-logs/
+
+        //bucketDir: <remote-app-log-dir>/<user>/bucket-<suffix>/<bucket id>/
+        //example: mockfs://foo/tmp/logs/me/bucket-logs/0001/
+
+        //remoteRootLogPath with controller: <remote-app-log-dir>/<controllerName>
+        //example: mockfs://foo/tmp/logs/IFile
+        rootPaths.add(new Path(remoteRootLogPath, fileController));
+      }
+    } else {
+      rootPaths.add(remoteRootLogPath);
+    }
+    return rootPaths;
+  }
+
+  private void initFileSystemListings(String controllerName) throws IOException {
+    setupListStatusForPath(userDir, suffixDir);
+    setupListStatusForPath(suffixDir, bucketDir);
+    setupListStatusForPath(bucketDir, appDirs.stream()
+            .filter(app -> app.path.toString().contains(controllerName))
+            .map(app -> app.fileStatus)
+            .toArray(FileStatus[]::new));
+
+    for (Pair<String, Long> appDirPair : additionalAppDirs) {
+      PathWithFileStatus appDir = createDirLogPathWithFileStatus(bucketDir.path,
+              appDirPair.getLeft(), appDirPair.getRight());
+      setupListStatusForPath(appDir, new FileStatus[] {});
+    }
+  }
+
+  private void createApplicationsByDescriptors() throws IOException {
+    int len = appDescriptors.size();
+    appDirs = new ArrayList<>(len);
+
+    for (int i = 0; i < len; i++) {
+      AppDescriptor appDesc = appDescriptors.get(i);
+      ApplicationId applicationId = appDesc.createApplicationId(now, i + 1);
+      applicationIds.add(applicationId);
+      Path basePath = this.remoteRootLogPath;
+      if (appDesc.fileController != null) {
+        basePath = new Path(basePath, appDesc.fileController);
+      }
+
+      PathWithFileStatus appDir = createPathWithFileStatusForAppId(
+              basePath, applicationId, userDirName, suffix, appDesc.modTimeOfAppDir);
+      LOG.debug("Created application with ID '{}' to path '{}'", applicationId, appDir.path);
+      appDirs.add(appDir);
+      addAppChildrenFiles(appDesc, appDir);
+    }
+
+    setupFsMocksForAppsAndChildrenFiles();
+
+    for (Map.Entry<Integer, Exception> e : injectedAppDirDeletionExceptions.entrySet()) {
+      when(mockFs.delete(this.appDirs.get(e.getKey()).path, true)).thenThrow(e.getValue());
+    }
+  }
+
+  private void setupFsMocksForAppsAndChildrenFiles() throws IOException {
+    for (int i = 0; i < appDirs.size(); i++) {
+      List<PathWithFileStatus> appChildren = appFiles.get(i);
+      Path appPath = appDirs.get(i).path;
+      setupListStatusForPath(appPath,
+              appChildren.stream()
+                      .map(child -> child.fileStatus)
+                      .toArray(FileStatus[]::new));
+    }
+  }
+
+  private void setupListStatusForPath(Path dir, PathWithFileStatus pathWithFileStatus)
+          throws IOException {
+    setupListStatusForPath(dir, new FileStatus[]{pathWithFileStatus.fileStatus});
+  }
+
+  private void setupListStatusForPath(PathWithFileStatus dir, PathWithFileStatus pathWithFileStatus)
+          throws IOException {
+    setupListStatusForPath(dir, new FileStatus[]{pathWithFileStatus.fileStatus});
+  }
+
+  private void setupListStatusForPath(Path dir, FileStatus[] fileStatuses) throws IOException {
+    LOG.debug("Setting up listStatus. Parent: {}, files: {}", dir, fileStatuses);
+    when(mockFs.listStatus(dir)).thenReturn(fileStatuses);
+  }
+
+  private void setupListStatusForPath(PathWithFileStatus dir, FileStatus[] fileStatuses)
+          throws IOException {
+    LOG.debug("Setting up listStatus. Parent: {}, files: {}", dir.path, fileStatuses);
+    when(mockFs.listStatus(dir.path)).thenReturn(fileStatuses);
+  }
+
+  private void setupDeletionService() {
+    List<ApplicationId> finishedApps = createFinishedAppsList();
+    List<ApplicationId> runningApps = createRunningAppsList();
+    deletionService = new AggregatedLogDeletionServiceForTest(runningApps, finishedApps, conf);
+  }
+
+  public LogAggregationTestcase startDeletionService() {
+    deletionService.init(conf);
+    deletionService.start();
+    return this;
+  }
+
+  private List<ApplicationId> createRunningAppsList() {
+    List<ApplicationId> runningApps = new ArrayList<>();
+    for (int i : runningAppIds) {
+      ApplicationId appId = this.applicationIds.get(i - 1);
+      runningApps.add(appId);
+    }
+    return runningApps;
+  }
+
+  private List<ApplicationId> createFinishedAppsList() {
+    List<ApplicationId> finishedApps = new ArrayList<>();
+    for (int i : finishedAppIds) {
+      ApplicationId appId = this.applicationIds.get(i - 1);
+      finishedApps.add(appId);
+    }
+    return finishedApps;
+  }
+
+  public LogAggregationTestcase runDeletionTask(long retentionSeconds) throws Exception {
+    List<ApplicationId> finishedApps = createFinishedAppsList();
+    List<ApplicationId> runningApps = createRunningAppsList();
+    rmClient = createMockRMClient(finishedApps, runningApps);
+    List<LogDeletionTask> tasks = deletionService.createLogDeletionTasks(conf, retentionSeconds,
+            rmClient);
+    for (LogDeletionTask deletionTask : tasks) {
+      deletionTask.run();
+    }
+
+    return this;
+  }
+
+  private void addAppChildrenFiles(AppDescriptor appDesc, PathWithFileStatus appDir) {
+    List<PathWithFileStatus> appChildren = new ArrayList<>();
+    for (Pair<String, Long> fileWithModDate : appDesc.filesWithModDate) {
+      PathWithFileStatus appChildFile = createFileLogPathWithFileStatus(appDir.path,
+              fileWithModDate.getLeft(),
+              fileWithModDate.getRight());
+      appChildren.add(appChildFile);
+    }
+    this.appFiles.add(appChildren);
+  }
+
+  public LogAggregationTestcase verifyAppDirsDeleted(long timeout, int... ids) throws IOException {
+    for (int id : ids) {
+      verifyAppDirDeleted(id, timeout);
+    }
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppDirsNotDeleted(long timeout, int... ids)
+          throws IOException {
+    for (int id : ids) {
+      verifyAppDirNotDeleted(id, timeout);
+    }
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppDirDeleted(int id, long timeout) throws IOException {
+    verifyAppDirDeletion(id, 1, timeout);
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppDirNotDeleted(int id, long timeout) throws IOException {
+    verifyAppDirDeletion(id, 0, timeout);
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppFilesDeleted(long timeout,
+                                                      List<Pair<Integer, Integer>> pairs)
+          throws IOException {
+    for (Pair<Integer, Integer> pair : pairs) {
+      verifyAppFileDeleted(pair.getLeft(), pair.getRight(), timeout);
+    }
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppFilesNotDeleted(long timeout,
+                                                         List<Pair<Integer, Integer>> pairs)
+          throws IOException {
+    for (Pair<Integer, Integer> pair : pairs) {
+      verifyAppFileNotDeleted(pair.getLeft(), pair.getRight(), timeout);
+    }
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppFileDeleted(int id, int fileNo, long timeout)
+          throws IOException {
+    verifyAppFileDeletion(id, fileNo, 1, timeout);
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAppFileNotDeleted(int id, int fileNo, long timeout)
+          throws IOException {
+    verifyAppFileDeletion(id, fileNo, 0, timeout);
+    return this;
+  }
+
+  private void verifyAppDirDeletion(int id, int times, long timeout) throws IOException {
+    if (timeout == NO_TIMEOUT) {
+      verify(mockFs, times(times)).delete(this.appDirs.get(id - 1).path, true);
+    } else {
+      verify(mockFs, timeout(timeout).times(times)).delete(this.appDirs.get(id - 1).path, true);
+    }
+  }
+
+  private void verifyAppFileDeletion(int appId, int fileNo, int times, long timeout)
+          throws IOException {
+    List<PathWithFileStatus> childrenFiles = this.appFiles.get(appId - 1);
+    PathWithFileStatus file = childrenFiles.get(fileNo - 1);
+    verify(mockFs, timeout(timeout).times(times)).delete(file.path, true);
+  }
+
+  private void verifyMockRmClientWasClosedNTimes(int expectedRmClientCloses)
+      throws IOException {
+    ApplicationClientProtocol mockRMClient;
+    if (deletionService != null) {
+      mockRMClient = deletionService.getMockRMClient();
+    } else {
+      mockRMClient = rmClient;
+    }
+    verify((Closeable)mockRMClient, times(expectedRmClientCloses)).close();
+  }
+
+  public void teardown(int expectedRmClientCloses) throws IOException {
+    deletionService.stop();
+    verifyMockRmClientWasClosedNTimes(expectedRmClientCloses);
+  }
+
+  public LogAggregationTestcase refreshLogRetentionSettings() throws IOException {
+    deletionService.refreshLogRetentionSettings();
+    return this;
+  }
+
+  public AggregatedLogDeletionService getDeletionService() {
+    return deletionService;
+  }
+
+  public LogAggregationTestcase verifyCheckIntervalMilliSecondsEqualTo(
+          int checkIntervalMilliSeconds) {
+    assertEquals(checkIntervalMilliSeconds, deletionService.getCheckIntervalMsecs());
+    return this;
+  }
+
+  public LogAggregationTestcase verifyCheckIntervalMilliSecondsNotEqualTo(
+          int checkIntervalMilliSeconds) {
+    assertTrue(checkIntervalMilliSeconds != deletionService.getCheckIntervalMsecs());
+    return this;
+  }
+
+  public LogAggregationTestcase verifyAnyPathListedAtLeast(int atLeast, long timeout)
+          throws IOException {
+    verify(mockFs, timeout(timeout).atLeast(atLeast)).listStatus(any(Path.class));
+    return this;
+  }
+
+  public LogAggregationTestcase changeModTimeOfApp(int appId, long modTime) {
+    PathWithFileStatus appDir = appDirs.get(appId - 1);
+    appDir.changeModificationTime(modTime);
+    return this;
+  }
+
+  public LogAggregationTestcase changeModTimeOfAppLogDir(int appId, int fileNo, long modTime) {
+    List<PathWithFileStatus> childrenFiles = this.appFiles.get(appId - 1);
+    PathWithFileStatus file = childrenFiles.get(fileNo - 1);
+    file.changeModificationTime(modTime);
+    return this;
+  }
+
+  public LogAggregationTestcase changeModTimeOfBucketDir(long modTime) {
+    bucketDir.changeModificationTime(modTime);
+    return this;
+  }
+
+  public LogAggregationTestcase reinitAllPaths() throws IOException {
+    List<Path> rootPaths = determineRootPaths();
+    for (Path rootPath : rootPaths) {
+      String controllerName = rootPath.getName();
+      initFileSystemListings(controllerName);
+    }
+    setupFsMocksForAppsAndChildrenFiles();
+    return this;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcaseBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcaseBuilder.java
new file mode 100644
index 0000000000000..3b56bcf37cf0a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/LogAggregationTestcaseBuilder.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
+
+import static org.apache.hadoop.yarn.logaggregation.TestAggregatedLogDeletionService.ALL_FILE_CONTROLLER_NAMES;
+
+public class LogAggregationTestcaseBuilder {
+  public static final long NO_TIMEOUT = -1;
+  final long now;
+  final Configuration conf;
+  Path remoteRootLogPath;
+  String suffix;
+  String userDirName;
+  long userDirModTime;
+  final Map<Integer, Exception> injectedAppDirDeletionExceptions = new HashMap<>();
+  List<String> fileControllers;
+  long suffixDirModTime;
+  long bucketDirModTime;
+  String suffixDirName;
+  List<AppDescriptor> apps = Lists.newArrayList();
+  int[] finishedAppIds;
+  int[] runningAppIds;
+  PathWithFileStatus userDir;
+  PathWithFileStatus suffixDir;
+  PathWithFileStatus bucketDir;
+  String bucketId;
+  List<Pair<String, Long>> additionalAppDirs = new ArrayList<>();
+  FileSystem rootFs;
+
+  public LogAggregationTestcaseBuilder(Configuration conf) {
+    this.conf = conf;
+    this.now = System.currentTimeMillis();
+  }
+
+  public static LogAggregationTestcaseBuilder create(Configuration conf) {
+    return new LogAggregationTestcaseBuilder(conf);
+  }
+
+  public LogAggregationTestcaseBuilder withRootPath(String root) throws IOException {
+    Path rootPath = new Path(root);
+    rootFs = rootPath.getFileSystem(conf);
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withRemoteRootLogPath(String remoteRootLogDir) {
+    remoteRootLogPath = new Path(remoteRootLogDir);
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withUserDir(String userDirName, long modTime) {
+    this.userDirName = userDirName;
+    this.userDirModTime = modTime;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withSuffixDir(String suffix, long modTime) {
+    this.suffix = suffix;
+    this.suffixDirName = LogAggregationUtils.getBucketSuffix() + suffix;
+    this.suffixDirModTime = modTime;
+    return this;
+  }
+
+  /**
+   * Bucket dir paths will be generated later.
+   * @param modTime The modification time
+   * @return The builder
+   */
+  public LogAggregationTestcaseBuilder withBucketDir(long modTime) {
+    this.bucketDirModTime = modTime;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withBucketDir(long modTime, String bucketId) {
+    this.bucketDirModTime = modTime;
+    this.bucketId = bucketId;
+    return this;
+  }
+
+  public final LogAggregationTestcaseBuilder withApps(List<AppDescriptor> apps) {
+    this.apps = apps;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withFinishedApps(int... apps) {
+    this.finishedAppIds = apps;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withRunningApps(int... apps) {
+    this.runningAppIds = apps;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withBothFileControllers() {
+    this.fileControllers = ALL_FILE_CONTROLLER_NAMES;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder withAdditionalAppDirs(List<Pair<String, Long>> appDirs) {
+    this.additionalAppDirs = appDirs;
+    return this;
+  }
+
+  public LogAggregationTestcaseBuilder injectExceptionForAppDirDeletion(int... indices) {
+    for (int i : indices) {
+      AccessControlException e = new AccessControlException("Injected Error\nStack Trace :(");
+      this.injectedAppDirDeletionExceptions.put(i, e);
+    }
+    return this;
+  }
+
+  public LogAggregationTestcase build() throws IOException {
+    return new LogAggregationTestcase(this);
+  }
+
+  public static final class AppDescriptor {
+    final long modTimeOfAppDir;
+    List<Pair<String, Long>> filesWithModDate = new ArrayList<>();
+    String fileController;
+
+    public AppDescriptor(long modTimeOfAppDir) {
+      this.modTimeOfAppDir = modTimeOfAppDir;
+    }
+
+    public AppDescriptor(long modTimeOfAppDir, List<Pair<String, Long>> filesWithModDate) {
+      this.modTimeOfAppDir = modTimeOfAppDir;
+      this.filesWithModDate = filesWithModDate;
+    }
+
+    public AppDescriptor(String fileController, long modTimeOfAppDir,
+                         List<Pair<String, Long>> filesWithModDate) {
+      this(modTimeOfAppDir, filesWithModDate);
+      this.fileController = fileController;
+    }
+
+
+    public ApplicationId createApplicationId(long now, int id) {
+      return ApplicationId.newInstance(now, id);
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/MockRMClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/MockRMClientUtils.java
new file mode 100644
index 0000000000000..33c5d8bd9b4db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/MockRMClientUtils.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import java.util.List;
+
+import org.apache.hadoop.test.MockitoUtil;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MockRMClientUtils {
+  public static ApplicationClientProtocol createMockRMClient(
+          List<ApplicationId> finishedApplications,
+          List<ApplicationId> runningApplications) throws Exception {
+    final ApplicationClientProtocol mockProtocol =
+        MockitoUtil.mockProtocol(ApplicationClientProtocol.class);
+    if (finishedApplications != null && !finishedApplications.isEmpty()) {
+      for (ApplicationId appId : finishedApplications) {
+        GetApplicationReportRequest request = GetApplicationReportRequest.newInstance(appId);
+        GetApplicationReportResponse response = createApplicationReportWithFinishedApplication();
+        when(mockProtocol.getApplicationReport(request)).thenReturn(response);
+      }
+    }
+    if (runningApplications != null && !runningApplications.isEmpty()) {
+      for (ApplicationId appId : runningApplications) {
+        GetApplicationReportRequest request = GetApplicationReportRequest.newInstance(appId);
+        GetApplicationReportResponse response = createApplicationReportWithRunningApplication();
+        when(mockProtocol.getApplicationReport(request)).thenReturn(response);
+      }
+    }
+    return mockProtocol;
+  }
+
+  public static GetApplicationReportResponse createApplicationReportWithRunningApplication() {
+    ApplicationReport report = mock(ApplicationReport.class);
+    when(report.getYarnApplicationState()).thenReturn(
+            YarnApplicationState.RUNNING);
+    GetApplicationReportResponse response =
+            mock(GetApplicationReportResponse.class);
+    when(response.getApplicationReport()).thenReturn(report);
+    return response;
+  }
+
+  public static GetApplicationReportResponse createApplicationReportWithFinishedApplication() {
+    ApplicationReport report = mock(ApplicationReport.class);
+    when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FINISHED);
+    GetApplicationReportResponse response = mock(GetApplicationReportResponse.class);
+    when(response.getApplicationReport()).thenReturn(report);
+    return response;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/PathWithFileStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/PathWithFileStatus.java
new file mode 100644
index 0000000000000..5e743f1138e4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/testutils/PathWithFileStatus.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.logaggregation.testutils;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+public class PathWithFileStatus {
+  public final Path path;
+  public FileStatus fileStatus;
+
+  public PathWithFileStatus(Path path, FileStatus fileStatus) {
+    this.path = path;
+    this.fileStatus = fileStatus;
+  }
+
+  public void changeModificationTime(long modTime) {
+    fileStatus = new FileStatus(fileStatus.getLen(), fileStatus.isDirectory(),
+            fileStatus.getReplication(),
+            fileStatus.getBlockSize(), modTime, fileStatus.getPath());
+  }
+
+  @Override
+  public String toString() {
+    return "PathWithFileStatus{" +
+            "path=" + path +
+            '}';
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
index 798d8835c81fd..de6bcf9af4009 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
@@ -25,19 +25,23 @@
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
-import org.junit.Assert;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class NodeLabelTestBase {
   public static void assertMapEquals(Map<NodeId, Set<String>> expected,
       ImmutableMap<NodeId, Set<String>> actual) {
-    Assert.assertEquals(expected.size(), actual.size());
+    assertEquals(expected.size(), actual.size());
     for (NodeId k : expected.keySet()) {
-      Assert.assertTrue(actual.containsKey(k));
+      assertTrue(actual.containsKey(k));
       assertCollectionEquals(expected.get(k), actual.get(k));
     }
   }
@@ -45,9 +49,9 @@ public static void assertMapEquals(Map<NodeId, Set<String>> expected,
   public static void assertLabelInfoMapEquals(
       Map<NodeId, Set<NodeLabel>> expected,
       ImmutableMap<NodeId, Set<NodeLabel>> actual) {
-    Assert.assertEquals(expected.size(), actual.size());
+    assertEquals(expected.size(), actual.size());
     for (NodeId k : expected.keySet()) {
-      Assert.assertTrue(actual.containsKey(k));
+      assertTrue(actual.containsKey(k));
       assertNLCollectionEquals(expected.get(k), actual.get(k));
     }
   }
@@ -55,13 +59,13 @@ public static void assertLabelInfoMapEquals(
   public static void assertLabelsToNodesEquals(
       Map<String, Set<NodeId>> expected,
       ImmutableMap<String, Set<NodeId>> actual) {
-    Assert.assertEquals(expected.size(), actual.size());
+    assertEquals(expected.size(), actual.size());
     for (String k : expected.keySet()) {
-      Assert.assertTrue(actual.containsKey(k));
+      assertTrue(actual.containsKey(k));
       Set<NodeId> expectedS1 = new HashSet<>(expected.get(k));
       Set<NodeId> actualS2 = new HashSet<>(actual.get(k));
-      Assert.assertEquals(expectedS1, actualS2);
-      Assert.assertTrue(expectedS1.containsAll(actualS2));
+      assertEquals(expectedS1, actualS2);
+      assertTrue(expectedS1.containsAll(actualS2));
     }
   }
 
@@ -86,7 +90,7 @@ public static ImmutableMap<String, Set<NodeId>> transposeNodeToLabels(
   public static void assertMapContains(Map<NodeId, Set<String>> expected,
       ImmutableMap<NodeId, Set<String>> actual) {
     for (NodeId k : actual.keySet()) {
-      Assert.assertTrue(expected.containsKey(k));
+      assertTrue(expected.containsKey(k));
       assertCollectionEquals(expected.get(k), actual.get(k));
     }
   }
@@ -94,28 +98,28 @@ public static void assertMapContains(Map<NodeId, Set<String>> expected,
   public static void assertCollectionEquals(Collection<String> expected,
       Collection<String> actual) {
     if (expected == null) {
-      Assert.assertNull(actual);
+      assertNull(actual);
     } else {
-      Assert.assertNotNull(actual);
+      assertNotNull(actual);
     }
     Set<String> expectedSet = new HashSet<>(expected);
     Set<String> actualSet = new HashSet<>(actual);
-    Assert.assertEquals(expectedSet, actualSet);
-    Assert.assertTrue(expectedSet.containsAll(actualSet));
+    assertEquals(expectedSet, actualSet);
+    assertTrue(expectedSet.containsAll(actualSet));
   }
 
   public static void assertNLCollectionEquals(Collection<NodeLabel> expected,
       Collection<NodeLabel> actual) {
     if (expected == null) {
-      Assert.assertNull(actual);
+      assertNull(actual);
     } else {
-      Assert.assertNotNull(actual);
+      assertNotNull(actual);
     }
 
     Set<NodeLabel> expectedSet = new HashSet<>(expected);
     Set<NodeLabel> actualSet = new HashSet<>(actual);
-    Assert.assertEquals(expectedSet, actualSet);
-    Assert.assertTrue(expectedSet.containsAll(actualSet));
+    assertEquals(expectedSet, actualSet);
+    assertTrue(expectedSet.containsAll(actualSet));
   }
 
   @SuppressWarnings("unchecked")
@@ -150,13 +154,13 @@ public NodeId toNodeId(String str) {
   public static void assertLabelsInfoToNodesEquals(
       Map<NodeLabel, Set<NodeId>> expected,
       ImmutableMap<NodeLabel, Set<NodeId>> actual) {
-    Assert.assertEquals(expected.size(), actual.size());
+    assertEquals(expected.size(), actual.size());
     for (NodeLabel k : expected.keySet()) {
-      Assert.assertTrue(actual.containsKey(k));
+      assertTrue(actual.containsKey(k));
       Set<NodeId> expectedS1 = new HashSet<>(expected.get(k));
       Set<NodeId> actualS2 = new HashSet<>(actual.get(k));
-      Assert.assertEquals(expectedS1, actualS2);
-      Assert.assertTrue(expectedS1.containsAll(actualS2));
+      assertEquals(expectedS1, actualS2);
+      assertTrue(expectedS1.containsAll(actualS2));
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
index 5a5cab85b47f0..ab4650fd189c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.nodelabels;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
@@ -27,24 +25,31 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
   DummyCommonNodeLabelsManager mgr = null;
 
-  @Before
+  @BeforeEach
   public void before() {
     mgr = new DummyCommonNodeLabelsManager();
     Configuration conf = new YarnConfiguration();
@@ -53,13 +58,14 @@ public void before() {
     mgr.start();
   }
 
-  @After
+  @AfterEach
   public void after() {
     mgr.stop();
   }
 
-  @Test(timeout = 5000)
-  public void testAddRemovelabel() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testAddRemovelabel() throws Exception {
     // Add some label
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("hello"));
     verifyNodeLabelAdded(Sets.newHashSet("hello"), mgr.lastAddedlabels);
@@ -68,23 +74,23 @@ public void testAddRemovelabel() throws Exception {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("hello1", "world1"));
     verifyNodeLabelAdded(Sets.newHashSet("hello1", "world1"), mgr.lastAddedlabels);
 
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Sets.newHashSet("hello", "world", "hello1", "world1")));
     try {
       mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("hello1",
           false)));
-      Assert.fail("IOException not thrown on exclusivity change of labels");
+      fail("IOException not thrown on exclusivity change of labels");
     } catch (Exception e) {
-      Assert.assertTrue("IOException is expected when exclusivity is modified",
-          e instanceof IOException);
+      assertTrue(e instanceof IOException,
+          "IOException is expected when exclusivity is modified");
     }
     try {
       mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("hello1",
           true)));
     } catch (Exception e) {
-      Assert.assertFalse(
-          "IOException not expected when no change in exclusivity",
-          e instanceof IOException);
+      assertFalse(
+          e instanceof IOException,
+          "IOException not expected when no change in exclusivity");
     }
     // try to remove null, empty and non-existed label, should fail
     for (String p : Arrays.asList(null, CommonNodeLabelsManager.NO_LABEL, "xx")) {
@@ -94,42 +100,45 @@ public void testAddRemovelabel() throws Exception {
       } catch (IOException e) {
         caught = true;
       }
-      Assert.assertTrue("remove label should fail "
-          + "when label is null/empty/non-existed", caught);
+      assertTrue(caught, "remove label should fail "
+          + "when label is null/empty/non-existed");
     }
 
     // Remove some label
     mgr.removeFromClusterNodeLabels(Arrays.asList("hello"));
     assertCollectionEquals(Sets.newHashSet("hello"), mgr.lastRemovedlabels);
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("world", "hello1", "world1")));
 
     mgr.removeFromClusterNodeLabels(Arrays
         .asList("hello1", "world1", "world"));
-    Assert.assertTrue(mgr.lastRemovedlabels.containsAll(Sets.newHashSet(
+    assertTrue(mgr.lastRemovedlabels.containsAll(Sets.newHashSet(
         "hello1", "world1", "world")));
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
+    assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
   }
 
-  @Test(timeout = 5000)
-  public void testAddlabelWithCase() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testAddlabelWithCase() throws Exception {
     // Add some label, case will not ignore here
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("HeLlO"));
     verifyNodeLabelAdded(Sets.newHashSet("HeLlO"), mgr.lastAddedlabels);
-    Assert.assertFalse(mgr.getClusterNodeLabelNames().containsAll(
+    assertFalse(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("hello")));
   }
 
-  @Test(timeout = 5000)
-  public void testAddlabelWithExclusivity() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testAddlabelWithExclusivity() throws Exception {
     // Add some label, case will not ignore here
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("a", false), NodeLabel.newInstance("b", true)));
-    Assert.assertFalse(mgr.isExclusiveNodeLabel("a"));
-    Assert.assertTrue(mgr.isExclusiveNodeLabel("b"));
+    assertFalse(mgr.isExclusiveNodeLabel("a"));
+    assertTrue(mgr.isExclusiveNodeLabel("b"));
   }
 
-  @Test(timeout = 5000)
-  public void testAddInvalidlabel() throws IOException {
+  @Test
+  @Timeout(5000)
+  void testAddInvalidlabel() throws IOException {
     boolean caught = false;
     try {
       Set<String> set = new HashSet<String>();
@@ -138,7 +147,7 @@ public void testAddInvalidlabel() throws IOException {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("null label should not add to repo", caught);
+    assertTrue(caught, "null label should not add to repo");
 
     caught = false;
     try {
@@ -147,7 +156,7 @@ public void testAddInvalidlabel() throws IOException {
       caught = true;
     }
 
-    Assert.assertTrue("empty label should not add to repo", caught);
+    assertTrue(caught, "empty label should not add to repo");
 
     caught = false;
     try {
@@ -155,7 +164,7 @@ public void testAddInvalidlabel() throws IOException {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("invalid label character should not add to repo", caught);
+    assertTrue(caught, "invalid label character should not add to repo");
 
     caught = false;
     try {
@@ -163,7 +172,7 @@ public void testAddInvalidlabel() throws IOException {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("too long label should not add to repo", caught);
+    assertTrue(caught, "too long label should not add to repo");
 
     caught = false;
     try {
@@ -171,7 +180,7 @@ public void testAddInvalidlabel() throws IOException {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("label cannot start with \"-\"", caught);
+    assertTrue(caught, "label cannot start with \"-\"");
 
     caught = false;
     try {
@@ -179,28 +188,29 @@ public void testAddInvalidlabel() throws IOException {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("label cannot start with \"_\"", caught);
-    
+    assertTrue(caught, "label cannot start with \"_\"");
+
     caught = false;
     try {
       mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("a^aabbb"));
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("label cannot contains other chars like ^[] ...", caught);
-    
+    assertTrue(caught, "label cannot contains other chars like ^[] ...");
+
     caught = false;
     try {
       mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("aa[a]bbb"));
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("label cannot contains other chars like ^[] ...", caught);
+    assertTrue(caught, "label cannot contains other chars like ^[] ...");
   }
 
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test(timeout = 5000)
-  public void testAddReplaceRemoveLabelsOnNodes() throws Exception {
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @Test
+  @Timeout(5000)
+  void testAddReplaceRemoveLabelsOnNodes() throws Exception {
     // set a label on a node, but label doesn't exist
     boolean caught = false;
     try {
@@ -208,8 +218,8 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("trying to set a label to a node but "
-        + "label doesn't exist in repository should fail", caught);
+    assertTrue(caught, "trying to set a label to a node but "
+        + "label doesn't exist in repository should fail");
 
     // set a label on a node, but node is null or empty
     try {
@@ -218,7 +228,7 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception {
     } catch (IOException e) {
       caught = true;
     }
-    Assert.assertTrue("trying to add a empty node but succeeded", caught);
+    assertTrue(caught, "trying to add a empty node but succeeded");
 
     // set node->label one by one
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
@@ -263,15 +273,16 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception {
     // remove labels on node
     mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
         toNodeId("n2"), toSet("p3"), toNodeId("n3"), toSet("p3")));
-    Assert.assertEquals(0, mgr.getNodeLabels().size());
+    assertEquals(0, mgr.getNodeLabels().size());
     assertMapEquals(mgr.lastNodeToLabels, ImmutableMap.of(toNodeId("n1"),
         CommonNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n2"),
         CommonNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n3"),
         CommonNodeLabelsManager.EMPTY_STRING_SET));
   }
 
-  @Test(timeout = 5000)
-  public void testRemovelabelWithNodes() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testRemovelabelWithNodes() throws Exception {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2"), toSet("p2")));
@@ -283,21 +294,23 @@ public void testRemovelabelWithNodes() throws Exception {
     assertCollectionEquals(Arrays.asList("p1"), mgr.lastRemovedlabels);
 
     mgr.removeFromClusterNodeLabels(ImmutableSet.of("p2", "p3"));
-    Assert.assertTrue(mgr.getNodeLabels().isEmpty());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
+    assertTrue(mgr.getNodeLabels().isEmpty());
+    assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
     assertCollectionEquals(Arrays.asList("p2", "p3"), mgr.lastRemovedlabels);
   }
-  
-  @Test(timeout = 5000) 
-  public void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException {
+
+  @Test
+  @Timeout(5000)
+  void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1"));
     assertCollectionEquals(toSet("p1"), mgr.getClusterNodeLabelNames());
     mgr.removeFromClusterNodeLabels(toSet("p1 "));
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
+    assertTrue(mgr.getClusterNodeLabelNames().isEmpty());
   }
-  
-  @Test(timeout = 5000) 
-  public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
+
+  @Test
+  @Timeout(5000)
+  void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1", "p2"));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ")));
     assertMapEquals(
@@ -308,49 +321,51 @@ public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
         mgr.getNodeLabels(),
         ImmutableMap.of(toNodeId("n1"), toSet("p2")));
     mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("  p2 ")));
-    Assert.assertTrue(mgr.getNodeLabels().isEmpty());
+    assertTrue(mgr.getNodeLabels().isEmpty());
   }
-  
-  @Test(timeout = 5000)
-  public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo()
+
+  @Test
+  @Timeout(5000)
+  void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo()
       throws IOException {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     assertMapEquals(
         mgr.getNodeLabels(),
         ImmutableMap.of(toNodeId("n1"), toSet("p1")));
-    
+
     // Replace labels on n1:1 to P2
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2"),
         toNodeId("n1:2"), toSet("p2")));
     assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"),
         toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"),
         toSet("p2")));
-    
+
     // Replace labels on n1 to P1, both n1:1/n1 will be P1 now
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"),
         toSet("p1"), toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"),
         toSet("p1")));
-    
+
     // Set labels on n1:1 to P2 again to verify if add/remove works
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2")));
   }
 
   private void assertNodeLabelsDisabledErrorMessage(IOException e) {
-    Assert.assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR,
+    assertEquals(CommonNodeLabelsManager.NODE_LABELS_NOT_ENABLED_ERR,
         e.getMessage());
   }
-  
-  @Test(timeout = 5000)
-  public void testNodeLabelsDisabled() throws IOException {
+
+  @Test
+  @Timeout(5000)
+  void testNodeLabelsDisabled() throws IOException {
     DummyCommonNodeLabelsManager mgr = new DummyCommonNodeLabelsManager();
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false);
     mgr.init(conf);
     mgr.start();
     boolean caught = false;
-    
+
     // add labels
     try {
       mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x"));
@@ -359,9 +374,9 @@ public void testNodeLabelsDisabled() throws IOException {
       caught = true;
     }
     // check exception caught
-    Assert.assertTrue(caught);
+    assertTrue(caught);
     caught = false;
-    
+
     // remove labels
     try {
       mgr.removeFromClusterNodeLabels(ImmutableSet.of("x"));
@@ -370,9 +385,9 @@ public void testNodeLabelsDisabled() throws IOException {
       caught = true;
     }
     // check exception caught
-    Assert.assertTrue(caught);
+    assertTrue(caught);
     caught = false;
-    
+
     // add labels to node
     try {
       mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host", 0),
@@ -382,9 +397,9 @@ public void testNodeLabelsDisabled() throws IOException {
       caught = true;
     }
     // check exception caught
-    Assert.assertTrue(caught);
+    assertTrue(caught);
     caught = false;
-    
+
     // remove labels from node
     try {
       mgr.removeLabelsFromNode(ImmutableMap.of(NodeId.newInstance("host", 0),
@@ -394,9 +409,9 @@ public void testNodeLabelsDisabled() throws IOException {
       caught = true;
     }
     // check exception caught
-    Assert.assertTrue(caught);
+    assertTrue(caught);
     caught = false;
-    
+
     // replace labels on node
     try {
       mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("host", 0),
@@ -406,14 +421,15 @@ public void testNodeLabelsDisabled() throws IOException {
       caught = true;
     }
     // check exception caught
-    Assert.assertTrue(caught);
+    assertTrue(caught);
     caught = false;
-    
+
     mgr.close();
-  }  
+  }
 
-  @Test(timeout = 5000)
-  public void testLabelsToNodes()
+  @Test
+  @Timeout(5000)
+  void testLabelsToNodes()
       throws IOException {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
@@ -421,7 +437,7 @@ public void testLabelsToNodes()
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n1"))));
+            "p1", toSet(toNodeId("n1"))));
     assertLabelsToNodesEquals(
         labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
 
@@ -432,8 +448,8 @@ public void testLabelsToNodes()
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n1")),
-        "p2", toSet(toNodeId("n1:1"),toNodeId("n1:2"))));
+            "p1", toSet(toNodeId("n1")),
+            "p2", toSet(toNodeId("n1:1"), toNodeId("n1:2"))));
     assertLabelsToNodesEquals(
         labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
 
@@ -443,7 +459,7 @@ public void testLabelsToNodes()
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n1"),toNodeId("n1:1"),toNodeId("n1:2"))));
+            "p1", toSet(toNodeId("n1"), toNodeId("n1:1"), toNodeId("n1:2"))));
     assertLabelsToNodesEquals(
         labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
 
@@ -455,9 +471,9 @@ public void testLabelsToNodes()
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n1"),toNodeId("n1:2")),
-        "p2", toSet(toNodeId("n1:1")),
-        "p3", toSet(toNodeId("n2"))));
+            "p1", toSet(toNodeId("n1"), toNodeId("n1:2")),
+            "p2", toSet(toNodeId("n1:1")),
+            "p3", toSet(toNodeId("n2"))));
     assertLabelsToNodesEquals(
         labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
 
@@ -467,20 +483,21 @@ public void testLabelsToNodes()
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n1"),toNodeId("n1:2")),
-        "p2", toSet(toNodeId("n1:1"))));
+            "p1", toSet(toNodeId("n1"), toNodeId("n1:2")),
+            "p2", toSet(toNodeId("n1:1"))));
     assertLabelsToNodesEquals(
         labelsToNodes, transposeNodeToLabels(mgr.getNodeLabels()));
   }
 
-  @Test(timeout = 5000)
-  public void testLabelsToNodesForSelectedLabels()
+  @Test
+  @Timeout(5000)
+  void testLabelsToNodesForSelectedLabels()
       throws IOException {
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addLabelsToNode(
         ImmutableMap.of(
-        toNodeId("n1:1"), toSet("p1"),
-        toNodeId("n1:2"), toSet("p2")));
+            toNodeId("n1:1"), toSet("p1"),
+            toNodeId("n1:2"), toSet("p2")));
     Set<String> setlabels =
         new HashSet<String>(Arrays.asList(new String[]{"p1"}));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(setlabels),
@@ -493,14 +510,14 @@ public void testLabelsToNodesForSelectedLabels()
     assertLabelsToNodesEquals(
         mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of(
-        "p3", toSet(toNodeId("n1"), toNodeId("n1:1"),toNodeId("n1:2"))));
+            "p3", toSet(toNodeId("n1"), toNodeId("n1:1"), toNodeId("n1:2"))));
 
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n2"), toSet("p2")));
     assertLabelsToNodesEquals(
         mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of(
-        "p2", toSet(toNodeId("n2")),
-        "p3", toSet(toNodeId("n1"), toNodeId("n1:1"),toNodeId("n1:2"))));
+            "p2", toSet(toNodeId("n2")),
+            "p3", toSet(toNodeId("n1"), toNodeId("n1:1"), toNodeId("n1:2"))));
 
     mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p3")));
     setlabels =
@@ -508,29 +525,30 @@ public void testLabelsToNodesForSelectedLabels()
     assertLabelsToNodesEquals(
         mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of(
-        "p2", toSet(toNodeId("n2"))));
+            "p2", toSet(toNodeId("n2"))));
 
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n3"), toSet("p1")));
     assertLabelsToNodesEquals(
         mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n3")),
-        "p2", toSet(toNodeId("n2"))));
+            "p1", toSet(toNodeId("n3")),
+            "p2", toSet(toNodeId("n2"))));
 
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2:2"), toSet("p3")));
     assertLabelsToNodesEquals(
         mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of(
-        "p1", toSet(toNodeId("n3")),
-        "p2", toSet(toNodeId("n2")),
-        "p3", toSet(toNodeId("n2:2"))));
+            "p1", toSet(toNodeId("n3")),
+            "p2", toSet(toNodeId("n2")),
+            "p3", toSet(toNodeId("n2:2"))));
     setlabels = new HashSet<String>(Arrays.asList(new String[]{"p1"}));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(setlabels),
         ImmutableMap.of("p1", toSet(toNodeId("n3"))));
   }
 
-  @Test(timeout = 5000)
-  public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
+  @Test
+  @Timeout(5000)
+  void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
     boolean failed = false;
     // As in YARN-2694, we temporarily disable no more than one label existed in
     // one host
@@ -540,14 +558,14 @@ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
     } catch (IOException e) {
       failed = true;
     }
-    Assert.assertTrue("Should failed when set > 1 labels on a host", failed);
+    assertTrue(failed, "Should failed when set > 1 labels on a host");
 
     try {
       mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
     } catch (IOException e) {
       failed = true;
     }
-    Assert.assertTrue("Should failed when add > 1 labels on a host", failed);
+    assertTrue(failed, "Should failed when add > 1 labels on a host");
 
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     // add a same label to a node, #labels in this node is still 1, shouldn't
@@ -558,20 +576,21 @@ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
     } catch (IOException e) {
       failed = true;
     }
-    Assert.assertTrue("Should failed when #labels > 1 on a host after add",
-        failed);
+    assertTrue(failed,
+        "Should failed when #labels > 1 on a host after add");
   }
 
   private void verifyNodeLabelAdded(Set<String> expectedAddedLabelNames,
       Collection<NodeLabel> addedNodeLabels) {
-    Assert.assertEquals(expectedAddedLabelNames.size(), addedNodeLabels.size());
+    assertEquals(expectedAddedLabelNames.size(), addedNodeLabels.size());
     for (NodeLabel label : addedNodeLabels) {
-      Assert.assertTrue(expectedAddedLabelNames.contains(label.getName()));
+      assertTrue(expectedAddedLabelNames.contains(label.getName()));
     }
   }
 
-  @Test(timeout = 5000)
-  public void testReplaceLabelsOnNodeInDistributedMode() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testReplaceLabelsOnNodeInDistributedMode() throws Exception {
     //create new DummyCommonNodeLabelsManager than the one got from @before
     mgr.stop();
     mgr = new DummyCommonNodeLabelsManager();
@@ -587,16 +606,17 @@ public void testReplaceLabelsOnNodeInDistributedMode() throws Exception {
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
     Set<String> labelsByNode = mgr.getLabelsByNode(toNodeId("n1"));
 
-    Assert.assertNull(
-        "Labels are not expected to be written to the NodeLabelStore",
-        mgr.lastNodeToLabels);
-    Assert.assertNotNull("Updated labels should be available from the Mgr",
-        labelsByNode);
-    Assert.assertTrue(labelsByNode.contains("p1"));
+    assertNull(
+        mgr.lastNodeToLabels,
+        "Labels are not expected to be written to the NodeLabelStore");
+    assertNotNull(labelsByNode,
+        "Updated labels should be available from the Mgr");
+    assertTrue(labelsByNode.contains("p1"));
   }
 
-  @Test(timeout = 5000)
-  public void testLabelsInfoToNodes() throws IOException {
+  @Test
+  @Timeout(5000)
+  void testLabelsInfoToNodes() throws IOException {
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false),
         NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", true)));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
@@ -605,8 +625,9 @@ public void testLabelsInfoToNodes() throws IOException {
         NodeLabel.newInstance("p1", false), toSet(toNodeId("n1"))));
   }
 
-  @Test(timeout = 5000)
-  public void testGetNodeLabelsInfo() throws IOException {
+  @Test
+  @Timeout(5000)
+  void testGetNodeLabelsInfo() throws IOException {
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false),
         NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", false)));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
@@ -617,8 +638,9 @@ public void testGetNodeLabelsInfo() throws IOException {
         toNodeId("n2"), toSet(NodeLabel.newInstance("p3", false))));
   }
 
-  @Test(timeout = 5000)
-  public void testRemoveNodeLabelsInfo() throws IOException {
+  @Test
+  @Timeout(5000)
+  void testRemoveNodeLabelsInfo() throws IOException {
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", true)));
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p2", true)));
     mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1")));
@@ -628,10 +650,10 @@ public void testRemoveNodeLabelsInfo() throws IOException {
     assertLabelsToNodesEquals(
         labelsToNodes,
         ImmutableMap.of(
-        "p2", toSet(toNodeId("n1:1"), toNodeId("n1:0"))));
+            "p2", toSet(toNodeId("n1:1"), toNodeId("n1:0"))));
 
     mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), new HashSet()));
     Map<String, Set<NodeId>> labelsToNodes2 = mgr.getLabelsToNodes();
-    Assert.assertEquals(labelsToNodes2.get("p2"), null);
+    assertNull(labelsToNodes2.get("p2"));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index f0885cd50f2c0..e769a21a75094 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -24,23 +24,24 @@
 import java.util.Collection;
 import java.util.Map;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.InlineDispatcher;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.Mockito;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
-@RunWith(Parameterized.class)
 public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
   MockNodeLabelManager mgr = null;
   Configuration conf = null;
@@ -57,26 +58,15 @@ protected void initDispatcher(Configuration conf) {
     protected void startDispatcher() {
       // do nothing
     }
-    
+
     @Override
     protected void stopDispatcher() {
       // do nothing
     }
   }
-  
-  public TestFileSystemNodeLabelsStore(String className) {
-    this.storeClassName = className;
-  }
-  
-  @Parameterized.Parameters
-  public static Collection<String[]> getParameters() {
-    return Arrays.asList(
-        new String[][] { { FileSystemNodeLabelsStore.class.getCanonicalName() },
-            { NonAppendableFSNodeLabelStore.class.getCanonicalName() } });
-  }
 
-  @Before
-  public void before() throws IOException {
+  public void initTestFileSystemNodeLabelsStore(String className) throws IOException {
+    this.storeClassName = className;
     mgr = new MockNodeLabelManager();
     conf = new Configuration();
     conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
@@ -91,7 +81,13 @@ public void before() throws IOException {
     mgr.start();
   }
 
-  @After
+  public static Collection<String[]> getParameters() {
+    return Arrays.asList(
+        new String[][]{{FileSystemNodeLabelsStore.class.getCanonicalName()},
+            {NonAppendableFSNodeLabelStore.class.getCanonicalName()}});
+  }
+
+  @AfterEach
   public void after() throws IOException {
     if (mgr.store instanceof FileSystemNodeLabelsStore) {
       FileSystemNodeLabelsStore fsStore =
@@ -101,9 +97,12 @@ public void after() throws IOException {
     mgr.stop();
   }
 
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test(timeout = 10000)
-  public void testRecoverWithMirror() throws Exception {
+  @MethodSource("getParameters")
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @ParameterizedTest
+  @Timeout(10000)
+  void testRecoverWithMirror(String className) throws Exception {
+    initTestFileSystemNodeLabelsStore(className);
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6"));
@@ -131,15 +130,15 @@ public void testRecoverWithMirror() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(3, mgr.getClusterNodeLabelNames().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6")));
 
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
-            ImmutableMap.of(
+        ImmutableMap.of(
             "p6", toSet(toNodeId("n6"), toNodeId("n7")),
             "p4", toSet(toNodeId("n4")),
             "p2", toSet(toNodeId("n2"))));
@@ -151,24 +150,27 @@ public void testRecoverWithMirror() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(3, mgr.getClusterNodeLabelNames().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6")));
 
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
-            ImmutableMap.of(
+        ImmutableMap.of(
             "p6", toSet(toNodeId("n6"), toNodeId("n7")),
             "p4", toSet(toNodeId("n4")),
             "p2", toSet(toNodeId("n2"))));
     mgr.stop();
   }
 
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test(timeout = 10000)
-  public void testRecoverWithDistributedNodeLabels() throws Exception {
+  @MethodSource("getParameters")
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @ParameterizedTest
+  @Timeout(10000)
+  void testRecoverWithDistributedNodeLabels(String className) throws Exception {
+    initTestFileSystemNodeLabelsStore(className);
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6"));
@@ -190,20 +192,23 @@ public void testRecoverWithDistributedNodeLabels() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(3, mgr.getClusterNodeLabels().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(3, mgr.getClusterNodeLabels().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6")));
 
-    Assert.assertTrue("During recovery in distributed node-labels setup, "
-        + "node to labels mapping should not be recovered ", mgr
-        .getNodeLabels().size() == 0);
+    assertTrue(mgr
+        .getNodeLabels().size() == 0, "During recovery in distributed node-labels setup, "
+        + "node to labels mapping should not be recovered ");
 
     mgr.stop();
   }
 
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test(timeout = 10000)
-  public void testEditlogRecover() throws Exception {
+  @MethodSource("getParameters")
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @ParameterizedTest
+  @Timeout(10000)
+  void testEditlogRecover(String className) throws Exception {
+    initTestFileSystemNodeLabelsStore(className);
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4"));
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6"));
@@ -231,24 +236,27 @@ public void testEditlogRecover() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(3, mgr.getClusterNodeLabelNames().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6")));
 
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
         toSet("p2"), toNodeId("n4"), toSet("p4"), toNodeId("n6"), toSet("p6"),
         toNodeId("n7"), toSet("p6")));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
-            ImmutableMap.of(
+        ImmutableMap.of(
             "p6", toSet(toNodeId("n6"), toNodeId("n7")),
             "p4", toSet(toNodeId("n4")),
             "p2", toSet(toNodeId("n2"))));
     mgr.stop();
   }
-  
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  @Test (timeout = 10000)
-  public void testSerilizationAfterRecovery() throws Exception {
+
+  @MethodSource("getParameters")
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  @ParameterizedTest
+  @Timeout(10000)
+  void testSerilizationAfterRecovery(String className) throws Exception {
+    initTestFileSystemNodeLabelsStore(className);
     // Add to cluster node labels, p2/p6 are non-exclusive.
     mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", true),
         NodeLabel.newInstance("p2", false), NodeLabel.newInstance("p3", true),
@@ -289,8 +297,8 @@ public void testSerilizationAfterRecovery() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(3, mgr.getClusterNodeLabelNames().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6")));
 
     assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"),
@@ -298,13 +306,13 @@ public void testSerilizationAfterRecovery() throws Exception {
         toNodeId("n7"), toSet("p6")));
     assertLabelsToNodesEquals(mgr.getLabelsToNodes(),
         ImmutableMap.of(
-        "p6", toSet(toNodeId("n6"), toNodeId("n7")),
-        "p4", toSet(toNodeId("n4")),
-        "p2", toSet(toNodeId("n2"))));
+            "p6", toSet(toNodeId("n6"), toNodeId("n7")),
+            "p4", toSet(toNodeId("n4")),
+            "p2", toSet(toNodeId("n2"))));
 
-    Assert.assertFalse(mgr.isExclusiveNodeLabel("p2"));
-    Assert.assertTrue(mgr.isExclusiveNodeLabel("p4"));
-    Assert.assertFalse(mgr.isExclusiveNodeLabel("p6"));
+    assertFalse(mgr.isExclusiveNodeLabel("p2"));
+    assertTrue(mgr.isExclusiveNodeLabel("p4"));
+    assertFalse(mgr.isExclusiveNodeLabel("p6"));
 
     /*
      * Add label p7,p8 then shutdown
@@ -314,7 +322,7 @@ public void testSerilizationAfterRecovery() throws Exception {
     mgr.start();
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p7", "p8"));
     mgr.stop();
-    
+
     /*
      * Restart, add label p9 and shutdown
      */
@@ -323,7 +331,7 @@ public void testSerilizationAfterRecovery() throws Exception {
     mgr.start();
     mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p9"));
     mgr.stop();
-    
+
     /*
      * Recovery, and see if p9 added
      */
@@ -332,14 +340,16 @@ public void testSerilizationAfterRecovery() throws Exception {
     mgr.start();
 
     // check variables
-    Assert.assertEquals(6, mgr.getClusterNodeLabelNames().size());
-    Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll(
+    assertEquals(6, mgr.getClusterNodeLabelNames().size());
+    assertTrue(mgr.getClusterNodeLabelNames().containsAll(
         Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9")));
     mgr.stop();
   }
 
-  @Test
-  public void testRootMkdirOnInitStore() throws Exception {
+  @MethodSource("getParameters")
+  @ParameterizedTest
+  void testRootMkdirOnInitStore(String className) throws Exception {
+    initTestFileSystemNodeLabelsStore(className);
     final FileSystem mockFs = Mockito.mock(FileSystem.class);
     FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() {
       public void initFileSystem(Configuration config) throws IOException {
@@ -355,7 +365,7 @@ public void initFileSystem(Configuration config) throws IOException {
   }
 
   private void verifyMkdirsCount(FileSystemNodeLabelsStore store,
-                                 boolean existsRetVal, int expectedNumOfCalls)
+      boolean existsRetVal, int expectedNumOfCalls)
       throws Exception {
     Mockito.when(store.getFs().exists(Mockito.any(
         Path.class))).thenReturn(existsRetVal);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
index 73e0bda91f6eb..52646b6596432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
@@ -17,13 +17,15 @@
  */
 package org.apache.hadoop.yarn.nodelabels;
 
-import static org.junit.Assert.fail;
-
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeAttributeType;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Test class to verify node label util ops.
@@ -31,7 +33,7 @@
 public class TestNodeLabelUtil {
 
   @Test
-  public void testAttributeValueAddition() {
+  void testAttributeValueAddition() {
     String[] values =
         new String[]{"1_8", "1.8", "ABZ", "ABZ", "az", "a-z", "a_z",
             "123456789"};
@@ -55,7 +57,7 @@ public void testAttributeValueAddition() {
   }
 
   @Test
-  public void testIsNodeAttributesEquals() {
+  void testIsNodeAttributesEquals() {
     NodeAttribute nodeAttributeCK1V1 = NodeAttribute
         .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "K1",
             NodeAttributeType.STRING, "V1");
@@ -77,45 +79,45 @@ public void testIsNodeAttributesEquals() {
     /*
      * equals if set size equals and items are all the same
      */
-    Assert.assertTrue(NodeLabelUtil.isNodeAttributesEquals(null, null));
-    Assert.assertTrue(NodeLabelUtil
+    assertTrue(NodeLabelUtil.isNodeAttributesEquals(null, null));
+    assertTrue(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(), ImmutableSet.of()));
-    Assert.assertTrue(NodeLabelUtil
+    assertTrue(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeCK1V1),
             ImmutableSet.of(nodeAttributeCK1V1Copy)));
-    Assert.assertTrue(NodeLabelUtil
+    assertTrue(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeDK1V1),
             ImmutableSet.of(nodeAttributeDK1V1Copy)));
-    Assert.assertTrue(NodeLabelUtil.isNodeAttributesEquals(
+    assertTrue(NodeLabelUtil.isNodeAttributesEquals(
         ImmutableSet.of(nodeAttributeCK1V1, nodeAttributeDK1V1),
         ImmutableSet.of(nodeAttributeCK1V1Copy, nodeAttributeDK1V1Copy)));
     /*
      * not equals if set size not equals or items are different
      */
-    Assert.assertFalse(
+    assertFalse(
         NodeLabelUtil.isNodeAttributesEquals(null, ImmutableSet.of()));
-    Assert.assertFalse(
+    assertFalse(
         NodeLabelUtil.isNodeAttributesEquals(ImmutableSet.of(), null));
     // different attribute prefix
-    Assert.assertFalse(NodeLabelUtil
+    assertFalse(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeCK1V1),
             ImmutableSet.of(nodeAttributeDK1V1)));
     // different attribute name
-    Assert.assertFalse(NodeLabelUtil
+    assertFalse(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeDK1V1),
             ImmutableSet.of(nodeAttributeDK2V1)));
     // different attribute value
-    Assert.assertFalse(NodeLabelUtil
+    assertFalse(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeDK2V1),
             ImmutableSet.of(nodeAttributeDK2V2)));
     // different set
-    Assert.assertFalse(NodeLabelUtil
+    assertFalse(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeCK1V1),
             ImmutableSet.of()));
-    Assert.assertFalse(NodeLabelUtil
+    assertFalse(NodeLabelUtil
         .isNodeAttributesEquals(ImmutableSet.of(nodeAttributeCK1V1),
             ImmutableSet.of(nodeAttributeCK1V1, nodeAttributeDK1V1)));
-    Assert.assertFalse(NodeLabelUtil.isNodeAttributesEquals(
+    assertFalse(NodeLabelUtil.isNodeAttributesEquals(
         ImmutableSet.of(nodeAttributeCK1V1, nodeAttributeDK1V1),
         ImmutableSet.of(nodeAttributeDK1V1)));
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
index f035da831d9b1..95b02c4966095 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
@@ -16,17 +16,18 @@
 
 package org.apache.hadoop.yarn.resourcetypes;
 
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
 /**
  * Contains helper methods to create Resource and ResourceInformation objects.
  * ResourceInformation can be created from a resource name
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
index cfe5a45569374..8613a8b9dc112 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
@@ -16,6 +16,14 @@
  */
 package org.apache.hadoop.yarn.security;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.nio.ByteBuffer;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -25,16 +33,9 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.util.DockerClientConfigHandler;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.nio.ByteBuffer;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test the functionality of the DockerClientConfigHandler.
@@ -51,7 +52,7 @@ public class TestDockerClientConfigHandler {
   private File file;
   private Configuration conf = new Configuration();
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     file = File.createTempFile("docker-client-config", "test");
     file.deleteOnExit();
@@ -61,7 +62,7 @@ public void setUp() throws Exception {
   }
 
   @Test
-  public void testReadCredentialsFromConfigFile() throws Exception {
+  void testReadCredentialsFromConfigFile() throws Exception {
     Credentials credentials =
         DockerClientConfigHandler.readCredentialsFromConfigFile(
             new Path(file.toURI()), conf, APPLICATION_ID);
@@ -85,7 +86,7 @@ public void testReadCredentialsFromConfigFile() throws Exception {
   }
 
   @Test
-  public void testGetCredentialsFromTokensByteBuffer() throws Exception {
+  void testGetCredentialsFromTokensByteBuffer() throws Exception {
     Credentials credentials =
         DockerClientConfigHandler.readCredentialsFromConfigFile(
             new Path(file.toURI()), conf, APPLICATION_ID);
@@ -110,7 +111,7 @@ public void testGetCredentialsFromTokensByteBuffer() throws Exception {
   }
 
   @Test
-  public void testWriteDockerCredentialsToPath() throws Exception {
+  void testWriteDockerCredentialsToPath() throws Exception {
     File outFile = File.createTempFile("docker-client-config", "out");
     outFile.deleteOnExit();
     Credentials credentials =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
index 8109b5ead4a05..69f58839c9226 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
@@ -21,6 +21,8 @@
 import java.io.DataOutputStream;
 import java.io.IOException;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -43,31 +45,33 @@
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.ContainerType;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestYARNTokenIdentifier {
 
   @Test
-  public void testNMTokenIdentifier() throws IOException {
+  void testNMTokenIdentifier() throws IOException {
     testNMTokenIdentifier(false);
   }
 
   @Test
-  public void testNMTokenIdentifierOldFormat() throws IOException {
+  void testNMTokenIdentifierOldFormat() throws IOException {
     testNMTokenIdentifier(true);
   }
 
   public void testNMTokenIdentifier(boolean oldFormat) throws IOException {
-    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
-        ApplicationId.newInstance(1, 1), 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
     NodeId nodeId = NodeId.newInstance("host0", 0);
     String applicationSubmitter = "usr0";
     int masterKeyId = 1;
-    
-    NMTokenIdentifier token = new NMTokenIdentifier(
-        appAttemptId, nodeId, applicationSubmitter, masterKeyId);
-    
+
+    NMTokenIdentifier token =
+        new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter, masterKeyId);
+
     NMTokenIdentifier anotherToken = new NMTokenIdentifier();
 
     byte[] tokenContent;
@@ -79,36 +83,32 @@ public void testNMTokenIdentifier(boolean oldFormat) throws IOException {
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
-    
+
     // verify the whole record equals with original record
-    Assert.assertEquals("Token is not the same after serialization " +
-        "and deserialization.", token, anotherToken);
-    
+    assertEquals(token, anotherToken,
+        "Token is not the same after serialization " + "and deserialization.");
+
     // verify all properties are the same as original
-    Assert.assertEquals(
-        "appAttemptId from proto is not the same with original token",
-        anotherToken.getApplicationAttemptId(), appAttemptId);
-    
-    Assert.assertEquals(
-        "NodeId from proto is not the same with original token",
-        anotherToken.getNodeId(), nodeId);
-    
-    Assert.assertEquals(
-        "applicationSubmitter from proto is not the same with original token",
-        anotherToken.getApplicationSubmitter(), applicationSubmitter);
-    
-    Assert.assertEquals(
-        "masterKeyId from proto is not the same with original token",
-        anotherToken.getKeyId(), masterKeyId);
+    assertEquals(anotherToken.getApplicationAttemptId(), appAttemptId,
+        "appAttemptId from proto is not the same with original token");
+
+    assertEquals(anotherToken.getNodeId(), nodeId,
+        "NodeId from proto is not the same with original token");
+
+    assertEquals(anotherToken.getApplicationSubmitter(), applicationSubmitter,
+        "applicationSubmitter from proto is not the same with original token");
+
+    assertEquals(anotherToken.getKeyId(), masterKeyId,
+        "masterKeyId from proto is not the same with original token");
   }
 
   @Test
-  public void testAMRMTokenIdentifier() throws IOException {
+  void testAMRMTokenIdentifier() throws IOException {
     testAMRMTokenIdentifier(false);
   }
 
   @Test
-  public void testAMRMTokenIdentifierOldFormat() throws IOException {
+  void testAMRMTokenIdentifierOldFormat() throws IOException {
     testAMRMTokenIdentifier(true);
   }
 
@@ -130,55 +130,55 @@ public void testAMRMTokenIdentifier(boolean oldFormat) throws IOException {
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
-        
+
     // verify the whole record equals with original record
-    Assert.assertEquals("Token is not the same after serialization " +
-        "and deserialization.", token, anotherToken);
-        
-    Assert.assertEquals("ApplicationAttemptId from proto is not the same with original token",
-        anotherToken.getApplicationAttemptId(), appAttemptId);
-    
-    Assert.assertEquals("masterKeyId from proto is not the same with original token",
-        anotherToken.getKeyId(), masterKeyId);
+    assertEquals(token, anotherToken,
+        "Token is not the same after serialization " + "and deserialization.");
+
+    assertEquals(anotherToken.getApplicationAttemptId(), appAttemptId,
+        "ApplicationAttemptId from proto is not the same with original token");
+
+    assertEquals(anotherToken.getKeyId(), masterKeyId,
+        "masterKeyId from proto is not the same with original token");
   }
-  
+
   @Test
-  public void testClientToAMTokenIdentifier() throws IOException {
+  void testClientToAMTokenIdentifier() throws IOException {
     ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
         ApplicationId.newInstance(1, 1), 1);
-    
+
     String clientName = "user";
-  
+
     ClientToAMTokenIdentifier token = new ClientToAMTokenIdentifier(
         appAttemptId, clientName);
-    
+
     ClientToAMTokenIdentifier anotherToken = new ClientToAMTokenIdentifier();
-    
+
     byte[] tokenContent = token.getBytes();
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
-        
+
     // verify the whole record equals with original record
-    Assert.assertEquals("Token is not the same after serialization " +
-        "and deserialization.", token, anotherToken);
-        
-    Assert.assertEquals("ApplicationAttemptId from proto is not the same with original token",
-        anotherToken.getApplicationAttemptID(), appAttemptId);
-    
-    Assert.assertEquals("clientName from proto is not the same with original token",
-        anotherToken.getClientName(), clientName);
+    assertEquals(token, anotherToken,
+        "Token is not the same after serialization " + "and deserialization.");
+
+    assertEquals(anotherToken.getApplicationAttemptID(), appAttemptId,
+        "ApplicationAttemptId from proto is not the same with original token");
+
+    assertEquals(anotherToken.getClientName(), clientName,
+        "clientName from proto is not the same with original token");
   }
 
   @Test
-  public void testContainerTokenIdentifierProtoMissingFields()
+  void testContainerTokenIdentifierProtoMissingFields()
       throws IOException {
     ContainerTokenIdentifierProto.Builder builder =
         ContainerTokenIdentifierProto.newBuilder();
     ContainerTokenIdentifierProto proto = builder.build();
-    Assert.assertFalse(proto.hasContainerType());
-    Assert.assertFalse(proto.hasExecutionType());
-    Assert.assertFalse(proto.hasNodeLabelExpression());
+    assertFalse(proto.hasContainerType());
+    assertFalse(proto.hasExecutionType());
+    assertFalse(proto.hasNodeLabelExpression());
 
     byte[] tokenData = proto.toByteArray();
     DataInputBuffer dib = new DataInputBuffer();
@@ -186,21 +186,19 @@ public void testContainerTokenIdentifierProtoMissingFields()
     ContainerTokenIdentifier tid = new ContainerTokenIdentifier();
     tid.readFields(dib);
 
-    Assert.assertEquals("container type",
-        ContainerType.TASK, tid.getContainerType());
-    Assert.assertEquals("execution type",
-        ExecutionType.GUARANTEED, tid.getExecutionType());
-    Assert.assertEquals("node label expression",
-        CommonNodeLabelsManager.NO_LABEL, tid.getNodeLabelExpression());
+    assertEquals(ContainerType.TASK, tid.getContainerType(), "container type");
+    assertEquals(ExecutionType.GUARANTEED, tid.getExecutionType(), "execution type");
+    assertEquals(CommonNodeLabelsManager.NO_LABEL, tid.getNodeLabelExpression(),
+        "node label expression");
   }
 
   @Test
-  public void testContainerTokenIdentifier() throws IOException {
+  void testContainerTokenIdentifier() throws IOException {
     testContainerTokenIdentifier(false, false);
   }
 
   @Test
-  public void testContainerTokenIdentifierOldFormat() throws IOException {
+  void testContainerTokenIdentifierOldFormat() throws IOException {
     testContainerTokenIdentifier(true, true);
     testContainerTokenIdentifier(true, false);
   }
@@ -236,64 +234,55 @@ public void testContainerTokenIdentifier(boolean oldFormat,
     anotherToken.readFields(dib);
     
     // verify the whole record equals with original record
-    Assert.assertEquals("Token is not the same after serialization " +
-        "and deserialization.", token, anotherToken);
-    
-    Assert.assertEquals(
-        "ContainerID from proto is not the same with original token",
-        anotherToken.getContainerID(), containerID);
-    
-    Assert.assertEquals(
-        "Hostname from proto is not the same with original token",
-        anotherToken.getNmHostAddress(), hostName);
-    
-    Assert.assertEquals(
-        "ApplicationSubmitter from proto is not the same with original token",
-        anotherToken.getApplicationSubmitter(), appSubmitter);
-    
-    Assert.assertEquals(
-        "Resource from proto is not the same with original token",
-        anotherToken.getResource(), r);
-    
-    Assert.assertEquals(
-        "expiryTimeStamp from proto is not the same with original token",
-        anotherToken.getExpiryTimeStamp(), expiryTimeStamp);
-    
-    Assert.assertEquals(
-        "KeyId from proto is not the same with original token",
-        anotherToken.getMasterKeyId(), masterKeyId);
-    
-    Assert.assertEquals(
-        "RMIdentifier from proto is not the same with original token",
-        anotherToken.getRMIdentifier(), rmIdentifier);
-    
-    Assert.assertEquals(
-        "Priority from proto is not the same with original token",
-        anotherToken.getPriority(), priority);
-    
-    Assert.assertEquals(
-        "CreationTime from proto is not the same with original token",
-        anotherToken.getCreationTime(), creationTime);
+    assertEquals(token, anotherToken,
+        "Token is not the same after serialization " + "and deserialization.");
+
+    assertEquals(anotherToken.getContainerID(), containerID,
+        "ContainerID from proto is not the same with original token");
+
+    assertEquals(anotherToken.getNmHostAddress(), hostName,
+        "Hostname from proto is not the same with original token");
+
+    assertEquals(anotherToken.getApplicationSubmitter(), appSubmitter,
+        "ApplicationSubmitter from proto is not the same with original token");
+
+    assertEquals(anotherToken.getResource(), r,
+        "Resource from proto is not the same with original token");
+
+    assertEquals(anotherToken.getExpiryTimeStamp(), expiryTimeStamp,
+        "expiryTimeStamp from proto is not the same with original token");
+
+    assertEquals(anotherToken.getMasterKeyId(), masterKeyId,
+        "KeyId from proto is not the same with original token");
+
+    assertEquals(anotherToken.getRMIdentifier(), rmIdentifier,
+        "RMIdentifier from proto is not the same with original token");
+
+    assertEquals(anotherToken.getPriority(), priority,
+        "Priority from proto is not the same with original token");
+
+    assertEquals(anotherToken.getCreationTime(), creationTime,
+        "CreationTime from proto is not the same with original token");
     
-    Assert.assertNull(anotherToken.getLogAggregationContext());
+    assertNull(anotherToken.getLogAggregationContext());
 
-    Assert.assertEquals(CommonNodeLabelsManager.NO_LABEL,
+    assertEquals(CommonNodeLabelsManager.NO_LABEL,
         anotherToken.getNodeLabelExpression());
 
-    Assert.assertEquals(ContainerType.TASK,
+    assertEquals(ContainerType.TASK,
         anotherToken.getContainerType());
 
-    Assert.assertEquals(ExecutionType.GUARANTEED,
+    assertEquals(ExecutionType.GUARANTEED,
         anotherToken.getExecutionType());
   }
 
   @Test
-  public void testRMDelegationTokenIdentifier() throws IOException {
+  void testRMDelegationTokenIdentifier() throws IOException {
     testRMDelegationTokenIdentifier(false);
   }
 
   @Test
-  public void testRMDelegationTokenIdentifierOldFormat() throws IOException {
+  void testRMDelegationTokenIdentifierOldFormat() throws IOException {
     testRMDelegationTokenIdentifier(true);
   }
 
@@ -333,30 +322,22 @@ public void testRMDelegationTokenIdentifier(boolean oldFormat)
       dib.close();
     }
     // verify the whole record equals with original record
-    Assert.assertEquals(
-        "Token is not the same after serialization and deserialization.",
-        originalToken, anotherToken);
-    Assert.assertEquals(
-        "owner from proto is not the same with original token",
-        owner, anotherToken.getOwner());
-    Assert.assertEquals(
-        "renewer from proto is not the same with original token",
-        renewer, anotherToken.getRenewer());
-    Assert.assertEquals(
-        "realUser from proto is not the same with original token",
-        realUser, anotherToken.getRealUser());
-    Assert.assertEquals(
-        "issueDate from proto is not the same with original token",
-        issueDate, anotherToken.getIssueDate());
-    Assert.assertEquals(
-        "maxDate from proto is not the same with original token",
-        maxDate, anotherToken.getMaxDate());
-    Assert.assertEquals(
-        "sequenceNumber from proto is not the same with original token",
-        sequenceNumber, anotherToken.getSequenceNumber());
-    Assert.assertEquals(
-        "masterKeyId from proto is not the same with original token",
-        masterKeyId, anotherToken.getMasterKeyId());
+    assertEquals(originalToken, anotherToken,
+        "Token is not the same after serialization and deserialization.");
+    assertEquals(owner, anotherToken.getOwner(),
+        "owner from proto is not the same with original token");
+    assertEquals(renewer, anotherToken.getRenewer(),
+        "renewer from proto is not the same with original token");
+    assertEquals(realUser, anotherToken.getRealUser(),
+        "realUser from proto is not the same with original token");
+    assertEquals(issueDate, anotherToken.getIssueDate(),
+        "issueDate from proto is not the same with original token");
+    assertEquals(maxDate, anotherToken.getMaxDate(),
+        "maxDate from proto is not the same with original token");
+    assertEquals(sequenceNumber, anotherToken.getSequenceNumber(),
+        "sequenceNumber from proto is not the same with original token");
+    assertEquals(masterKeyId, anotherToken.getMasterKeyId(),
+        "masterKeyId from proto is not the same with original token");
 
     // Test getProto
     YARNDelegationTokenIdentifierProto tokenProto = originalToken.getProto();
@@ -372,15 +353,15 @@ public void testRMDelegationTokenIdentifier(boolean oldFormat)
     readToken.readFields(db);
 
     // Verify if read token equals with original token
-    Assert.assertEquals("Token from getProto is not the same after " +
-        "serialization and deserialization.", originalToken, readToken);
+    assertEquals(originalToken, readToken, "Token from getProto is not the same after " +
+        "serialization and deserialization.");
     db.close();
     out.close();
   }
-  
+
   @Test
-  public void testTimelineDelegationTokenIdentifier() throws IOException {
-    
+  void testTimelineDelegationTokenIdentifier() throws IOException {
+
     Text owner = new Text("user1");
     Text renewer = new Text("user2");
     Text realUser = new Text("user3");
@@ -388,50 +369,50 @@ public void testTimelineDelegationTokenIdentifier() throws IOException {
     long maxDate = 2;
     int sequenceNumber = 3;
     int masterKeyId = 4;
-    
-    TimelineDelegationTokenIdentifier token = 
+
+    TimelineDelegationTokenIdentifier token =
         new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
     token.setIssueDate(issueDate);
     token.setMaxDate(maxDate);
     token.setSequenceNumber(sequenceNumber);
     token.setMasterKeyId(masterKeyId);
-    
-    TimelineDelegationTokenIdentifier anotherToken = 
+
+    TimelineDelegationTokenIdentifier anotherToken =
         new TimelineDelegationTokenIdentifier();
-    
+
     byte[] tokenContent = token.getBytes();
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
-        
+
     // verify the whole record equals with original record
-    Assert.assertEquals("Token is not the same after serialization " +
-        "and deserialization.", token, anotherToken);
-    
-    Assert.assertEquals("owner from proto is not the same with original token",
-        anotherToken.getOwner(), owner);
-    
-    Assert.assertEquals("renewer from proto is not the same with original token",
-        anotherToken.getRenewer(), renewer);
-    
-    Assert.assertEquals("realUser from proto is not the same with original token",
-        anotherToken.getRealUser(), realUser);
-    
-    Assert.assertEquals("issueDate from proto is not the same with original token",
-        anotherToken.getIssueDate(), issueDate);
-    
-    Assert.assertEquals("maxDate from proto is not the same with original token",
-        anotherToken.getMaxDate(), maxDate);
-    
-    Assert.assertEquals("sequenceNumber from proto is not the same with original token",
-        anotherToken.getSequenceNumber(), sequenceNumber);
-    
-    Assert.assertEquals("masterKeyId from proto is not the same with original token",
-        anotherToken.getMasterKeyId(), masterKeyId);
+    assertEquals(token, anotherToken,
+        "Token is not the same after serialization " + "and deserialization.");
+
+    assertEquals(anotherToken.getOwner(), owner,
+        "owner from proto is not the same with original token");
+
+    assertEquals(anotherToken.getRenewer(), renewer,
+        "renewer from proto is not the same with original token");
+
+    assertEquals(anotherToken.getRealUser(), realUser,
+        "realUser from proto is not the same with original token");
+
+    assertEquals(anotherToken.getIssueDate(), issueDate,
+        "issueDate from proto is not the same with original token");
+
+    assertEquals(anotherToken.getMaxDate(), maxDate,
+        "maxDate from proto is not the same with original token");
+
+    assertEquals(anotherToken.getSequenceNumber(), sequenceNumber,
+        "sequenceNumber from proto is not the same with original token");
+
+    assertEquals(anotherToken.getMasterKeyId(), masterKeyId,
+        "masterKeyId from proto is not the same with original token");
   }
 
   @Test
-  public void testParseTimelineDelegationTokenIdentifierRenewer() throws IOException {
+  void testParseTimelineDelegationTokenIdentifierRenewer() throws IOException {
     // Server side when generation a timeline DT
     Configuration conf = new YarnConfiguration();
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
@@ -442,11 +423,11 @@ public void testParseTimelineDelegationTokenIdentifierRenewer() throws IOExcepti
     Text realUser = new Text("realUser");
     TimelineDelegationTokenIdentifier token =
         new TimelineDelegationTokenIdentifier(owner, renewer, realUser);
-    Assert.assertEquals(new Text("yarn"), token.getRenewer());
+    assertEquals(new Text("yarn"), token.getRenewer());
   }
 
   @Test
-  public void testAMContainerTokenIdentifier() throws IOException {
+  void testAMContainerTokenIdentifier() throws IOException {
     ContainerId containerID = ContainerId.newContainerId(
         ApplicationAttemptId.newInstance(ApplicationId.newInstance(
             1, 1), 1), 1);
@@ -471,10 +452,10 @@ public void testAMContainerTokenIdentifier() throws IOException {
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
 
-    Assert.assertEquals(ContainerType.APPLICATION_MASTER,
+    assertEquals(ContainerType.APPLICATION_MASTER,
         anotherToken.getContainerType());
 
-    Assert.assertEquals(ExecutionType.GUARANTEED,
+    assertEquals(ExecutionType.GUARANTEED,
         anotherToken.getExecutionType());
 
     token =
@@ -490,10 +471,10 @@ public void testAMContainerTokenIdentifier() throws IOException {
     dib.reset(tokenContent, tokenContent.length);
     anotherToken.readFields(dib);
 
-    Assert.assertEquals(ContainerType.TASK,
+    assertEquals(ContainerType.TASK,
         anotherToken.getContainerType());
 
-    Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
+    assertEquals(ExecutionType.OPPORTUNISTIC,
         anotherToken.getExecutionType());
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
index 2db1da90416ab..0a32bb5145930 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
@@ -17,18 +17,19 @@
  */
 package org.apache.hadoop.yarn.server.security;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
 import java.util.HashMap;
 import java.util.Map;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestApplicationACLsManager {
 
@@ -39,14 +40,14 @@ public class TestApplicationACLsManager {
   private static final String TESTUSER3 = "testuser3";
 
   @Test
-  public void testCheckAccess() {
+  void testCheckAccess() {
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE,
         true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL,
         ADMIN_USER);
     ApplicationACLsManager aclManager = new ApplicationACLsManager(conf);
-    Map<ApplicationAccessType, String> aclMap = 
+    Map<ApplicationAccessType, String> aclMap =
         new HashMap<ApplicationAccessType, String>();
     aclMap.put(ApplicationAccessType.VIEW_APP, TESTUSER1 + "," + TESTUSER3);
     aclMap.put(ApplicationAccessType.MODIFY_APP, TESTUSER1);
@@ -56,46 +57,46 @@ public void testCheckAccess() {
     //User in ACL, should be allowed access
     UserGroupInformation testUser1 = UserGroupInformation
         .createRemoteUser(TESTUSER1);
-    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     //User NOT in ACL, should not be allowed access
     UserGroupInformation testUser2 = UserGroupInformation
         .createRemoteUser(TESTUSER2);
-    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.VIEW_APP, 
+    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.MODIFY_APP, 
+    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     //User has View access, but not modify access
     UserGroupInformation testUser3 = UserGroupInformation
         .createRemoteUser(TESTUSER3);
-    assertTrue(aclManager.checkAccess(testUser3, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(testUser3, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertFalse(aclManager.checkAccess(testUser3, ApplicationAccessType.MODIFY_APP, 
+    assertFalse(aclManager.checkAccess(testUser3, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     //Application Owner should have all access
     UserGroupInformation appOwner = UserGroupInformation
         .createRemoteUser(APP_OWNER);
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     //Admin should have all access
     UserGroupInformation adminUser = UserGroupInformation
         .createRemoteUser(ADMIN_USER);
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
   }
 
   @Test
-  public void testCheckAccessWithNullACLS() {
+  void testCheckAccessWithNullACLS() {
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE,
         true);
@@ -108,30 +109,30 @@ public void testCheckAccessWithNullACLS() {
     //Application ACL is not added
 
     //Application Owner should have all access even if Application ACL is not added
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
 
     //Admin should have all access
     UserGroupInformation adminUser = UserGroupInformation
         .createRemoteUser(ADMIN_USER);
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     // A regular user should Not have access
     UserGroupInformation testUser1 = UserGroupInformation
         .createRemoteUser(TESTUSER1);
-    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP, 
+    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP, 
+    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
   }
-  
+
   @Test
-  public void testCheckAccessWithPartialACLS() {
+  void testCheckAccessWithPartialACLS() {
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE,
         true);
@@ -141,40 +142,40 @@ public void testCheckAccessWithPartialACLS() {
     UserGroupInformation appOwner = UserGroupInformation
         .createRemoteUser(APP_OWNER);
     // Add only the VIEW ACLS
-    Map<ApplicationAccessType, String> aclMap = 
+    Map<ApplicationAccessType, String> aclMap =
         new HashMap<ApplicationAccessType, String>();
-    aclMap.put(ApplicationAccessType.VIEW_APP, TESTUSER1 );
+    aclMap.put(ApplicationAccessType.VIEW_APP, TESTUSER1);
     ApplicationId appId = ApplicationId.newInstance(1, 1);
     aclManager.addApplication(appId, aclMap);
 
     //Application Owner should have all access even if Application ACL is not added
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(appOwner, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
 
     //Admin should have all access
     UserGroupInformation adminUser = UserGroupInformation
         .createRemoteUser(ADMIN_USER);
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP, 
+    assertTrue(aclManager.checkAccess(adminUser, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
 
     // testuser1 should  have view access only
     UserGroupInformation testUser1 = UserGroupInformation
         .createRemoteUser(TESTUSER1);
-    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP, 
+    assertTrue(aclManager.checkAccess(testUser1, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP, 
+    assertFalse(aclManager.checkAccess(testUser1, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
-    
+
     // A testuser2 should Not have access
     UserGroupInformation testUser2 = UserGroupInformation
         .createRemoteUser(TESTUSER2);
-    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.VIEW_APP, 
+    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.VIEW_APP,
         APP_OWNER, appId));
-    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.MODIFY_APP, 
+    assertFalse(aclManager.checkAccess(testUser2, ApplicationAccessType.MODIFY_APP,
         APP_OWNER, appId));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestAdHocLogDumper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestAdHocLogDumper.java
index 4b2545e812c8f..2b7be0be28499 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestAdHocLogDumper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestAdHocLogDumper.java
@@ -18,23 +18,24 @@
 
 package org.apache.hadoop.yarn.util;
 
+import java.io.File;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
 
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.util.Time;
 import org.apache.log4j.Appender;
 import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Priority;
 import org.apache.log4j.LogManager;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.log4j.Priority;
 
 import static org.apache.hadoop.util.GenericsUtil.isLog4jLogger;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestAdHocLogDumper {
 
@@ -42,14 +43,14 @@ public class TestAdHocLogDumper {
       LoggerFactory.getLogger(TestAdHocLogDumper.class);
 
   @Test
-  public void testDumpingSchedulerLogs() throws Exception {
+  void testDumpingSchedulerLogs() throws Exception {
 
     Map<Appender, Priority> levels = new HashMap<>();
     String logFilename = "test.log";
     Logger logger = LoggerFactory.getLogger(TestAdHocLogDumper.class);
     if (isLog4jLogger(this.getClass())) {
-      for (Enumeration appenders = LogManager.getRootLogger().
-          getAllAppenders(); appenders.hasMoreElements();) {
+      for (Enumeration appenders =
+           LogManager.getRootLogger().getAllAppenders(); appenders.hasMoreElements();) {
         Object obj = appenders.nextElement();
         if (obj instanceof AppenderSkeleton) {
           AppenderSkeleton appender = (AppenderSkeleton) obj;
@@ -64,11 +65,11 @@ public void testDumpingSchedulerLogs() throws Exception {
     LOG.debug("test message 1");
     LOG.info("test message 2");
     File logFile = new File(logFilename);
-    Assert.assertTrue(logFile.exists());
+    assertTrue(logFile.exists());
     Thread.sleep(2000);
     long lastWrite = logFile.lastModified();
-    Assert.assertTrue(lastWrite < Time.now());
-    Assert.assertTrue(logFile.length() != 0);
+    assertTrue(lastWrite < Time.now());
+    assertTrue(logFile.length() != 0);
 
     // make sure levels are set back to their original values
     if (isLog4jLogger(this.getClass())) {
@@ -77,12 +78,12 @@ public void testDumpingSchedulerLogs() throws Exception {
         Object obj = appenders.nextElement();
         if (obj instanceof AppenderSkeleton) {
           AppenderSkeleton appender = (AppenderSkeleton) obj;
-          Assert.assertEquals(levels.get(appender), appender.getThreshold());
+          assertEquals(levels.get(appender), appender.getThreshold());
         }
       }
     }
     boolean del = logFile.delete();
-    if(!del) {
+    if (!del) {
       LOG.info("Couldn't clean up after test");
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
index 5db8046c15ccf..5866a3e6946a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
@@ -17,22 +17,23 @@
 */
 package org.apache.hadoop.yarn.util;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Shell;
-import org.junit.Test;
-
 import java.io.File;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
+
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestApps {
   @Test
-  public void testSetEnvFromInputString() {
+  void testSetEnvFromInputString() {
     Map<String, String> environment = new HashMap<String, String>();
     environment.put("JAVA_HOME", "/path/jdk");
     String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
@@ -64,7 +65,7 @@ public void testSetEnvFromInputString() {
   }
 
   @Test
-  public void testSetEnvFromInputProperty() {
+  void testSetEnvFromInputProperty() {
     Configuration conf = new Configuration(false);
     Map<String, String> env = new HashMap<>();
     String propName = "mapreduce.map.env";
@@ -91,7 +92,7 @@ public void testSetEnvFromInputProperty() {
   }
 
   @Test
-  public void testSetEnvFromInputPropertyDefault() {
+  void testSetEnvFromInputPropertyDefault() {
     Configuration conf = new Configuration(false);
     Map<String, String> env = new HashMap<>();
     String propName = "mapreduce.map.env";
@@ -122,7 +123,7 @@ public void testSetEnvFromInputPropertyDefault() {
   }
 
   @Test
-  public void testSetEnvFromInputPropertyOverrideDefault() {
+  void testSetEnvFromInputPropertyOverrideDefault() {
     Configuration conf = new Configuration(false);
     Map<String, String> env = new HashMap<>();
 
@@ -152,7 +153,7 @@ public void testSetEnvFromInputPropertyOverrideDefault() {
   }
 
   @Test
-  public void testSetEnvFromInputPropertyCommas() {
+  void testSetEnvFromInputPropertyCommas() {
     Configuration conf = new Configuration(false);
     Map<String, String> env = new HashMap<>();
     String propName = "mapreduce.reduce.env";
@@ -176,7 +177,7 @@ public void testSetEnvFromInputPropertyCommas() {
   }
 
   @Test
-  public void testSetEnvFromInputPropertyNull() {
+  void testSetEnvFromInputPropertyNull() {
     Configuration conf = new Configuration(false);
     Map<String, String> env = new HashMap<>();
     String propName = "mapreduce.map.env";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
index 2b9cfce25fe48..411be311974e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
@@ -18,48 +18,49 @@
 
 package org.apache.hadoop.yarn.util;
 
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.Test;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test class for {@link BoundedAppender}.
  */
 public class TestBoundedAppender {
-  @Rule
-  public ExpectedException expected = ExpectedException.none();
 
   @Test
-  public void initWithZeroLimitThrowsException() {
-    expected.expect(IllegalArgumentException.class);
-    expected.expectMessage("limit should be positive");
+  void initWithZeroLimitThrowsException() {
+    Throwable exception = assertThrows(IllegalArgumentException.class, () -> {
 
-    new BoundedAppender(0);
+      new BoundedAppender(0);
+    });
+    assertTrue(exception.getMessage().contains("limit should be positive"));
   }
 
   @Test
-  public void nullAppendedNullStringRead() {
+  void nullAppendedNullStringRead() {
     final BoundedAppender boundedAppender = new BoundedAppender(4);
     boundedAppender.append(null);
 
-    assertEquals("null appended, \"null\" read", "null",
-        boundedAppender.toString());
+    assertEquals("null",
+        boundedAppender.toString(),
+        "null appended, \"null\" read");
   }
 
   @Test
-  public void appendBelowLimitOnceValueIsReadCorrectly() {
+  void appendBelowLimitOnceValueIsReadCorrectly() {
     final BoundedAppender boundedAppender = new BoundedAppender(2);
 
     boundedAppender.append("ab");
 
-    assertEquals("value appended is read correctly", "ab",
-        boundedAppender.toString());
+    assertEquals("ab",
+        boundedAppender.toString(),
+        "value appended is read correctly");
   }
 
   @Test
-  public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() {
+  void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() {
     final BoundedAppender boundedAppender = new BoundedAppender(3);
 
     boundedAppender.append("ab");
@@ -67,13 +68,13 @@ public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() {
     boundedAppender.append("e");
     boundedAppender.append("fg");
 
-    assertEquals("last values appended fitting limit are read correctly",
-        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
-        boundedAppender.toString());
+    assertEquals(String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
+        boundedAppender.toString(),
+        "last values appended fitting limit are read correctly");
   }
 
   @Test
-  public void appendLastAboveLimitPreservesLastMessagePostfix() {
+  void appendLastAboveLimitPreservesLastMessagePostfix() {
     final BoundedAppender boundedAppender = new BoundedAppender(3);
 
     boundedAppender.append("ab");
@@ -81,35 +82,37 @@ public void appendLastAboveLimitPreservesLastMessagePostfix() {
     boundedAppender.append("fghij");
 
     assertEquals(
-        "last value appended above limit postfix is read correctly", String
+        String
             .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 10, "hij"),
-        boundedAppender.toString());
+        boundedAppender.toString(),
+        "last value appended above limit postfix is read correctly");
   }
 
   @Test
-  public void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() {
+  void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() {
     final BoundedAppender boundedAppender = new BoundedAppender(3);
 
     boundedAppender.append("ab");
     boundedAppender.append("cde");
 
-    assertEquals("last value appended above limit postfix is read correctly",
-        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"),
-        boundedAppender.toString());
+    assertEquals(String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"),
+        boundedAppender.toString(),
+        "last value appended above limit postfix is read correctly");
 
     boundedAppender.append("fg");
 
     assertEquals(
-        "middle value appended above limit postfix and last value are "
-            + "read correctly",
         String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
-        boundedAppender.toString());
+        boundedAppender.toString(),
+        "middle value appended above limit postfix and last value are "
+            + "read correctly");
 
     boundedAppender.append("hijkl");
 
     assertEquals(
-        "last value appended above limit postfix is read correctly", String
+        String
             .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 12, "jkl"),
-        boundedAppender.toString());
+        boundedAppender.toString(),
+        "last value appended above limit postfix is read correctly");
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
index 57542218aa216..954efda526415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
@@ -17,23 +17,25 @@
 */
 package org.apache.hadoop.yarn.util;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
 import java.net.URISyntaxException;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.TestContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.junit.Test;
+import org.apache.hadoop.yarn.api.records.URL;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class TestConverterUtils {
-  
+
   @Test
-  public void testConvertUrlWithNoPort() throws URISyntaxException {
+  void testConvertUrlWithNoPort() throws URISyntaxException {
     Path expectedPath = new Path("hdfs://foo.com");
     URL url = URL.fromPath(expectedPath);
     Path actualPath = url.toPath();
@@ -41,15 +43,15 @@ public void testConvertUrlWithNoPort() throws URISyntaxException {
   }
 
   @Test
-  public void testConvertUrlWithUserinfo() throws URISyntaxException {
+  void testConvertUrlWithUserinfo() throws URISyntaxException {
     Path expectedPath = new Path("foo://username:password@example.com:8042");
     URL url = URL.fromPath(expectedPath);
     Path actualPath = url.toPath();
     assertEquals(expectedPath, actualPath);
   }
-  
+
   @Test
-  public void testContainerId() throws URISyntaxException {
+  void testContainerId() throws URISyntaxException {
     ContainerId id = TestContainerId.newContainerId(0, 0, 0, 0);
     String cid = id.toString();
     assertEquals("container_0_0000_00_000000", cid);
@@ -58,7 +60,7 @@ public void testContainerId() throws URISyntaxException {
   }
 
   @Test
-  public void testContainerIdWithEpoch() throws URISyntaxException {
+  void testContainerIdWithEpoch() throws URISyntaxException {
     ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
     String cid = id.toString();
     assertEquals("container_0_0000_00_25645811", cid);
@@ -85,38 +87,44 @@ public void testContainerIdWithEpoch() throws URISyntaxException {
 
   @Test
   @SuppressWarnings("deprecation")
-  public void testContainerIdNull() throws URISyntaxException {
-    assertNull(ConverterUtils.toString((ContainerId)null));
-  }  
-  
+  void testContainerIdNull() throws URISyntaxException {
+    assertNull(ConverterUtils.toString((ContainerId) null));
+  }
+
   @Test
-  public void testNodeIdWithDefaultPort() throws URISyntaxException {
+  void testNodeIdWithDefaultPort() throws URISyntaxException {
     NodeId nid;
-    
+
     nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
     assertThat(nid.getPort()).isEqualTo(10);
     assertThat(nid.getHost()).isEqualTo("node");
-    
+
     nid = ConverterUtils.toNodeIdWithDefaultPort("node");
     assertThat(nid.getPort()).isEqualTo(0);
     assertThat(nid.getHost()).isEqualTo("node");
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test
   @SuppressWarnings("deprecation")
-  public void testInvalidContainerId() {
-    ContainerId.fromString("container_e20_1423221031460_0003_01");
+  void testInvalidContainerId() {
+    assertThrows(IllegalArgumentException.class, () -> {
+      ContainerId.fromString("container_e20_1423221031460_0003_01");
+    });
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test
   @SuppressWarnings("deprecation")
-  public void testInvalidAppattemptId() {
-    ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
+  void testInvalidAppattemptId() {
+    assertThrows(IllegalArgumentException.class, () -> {
+      ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
+    });
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test
   @SuppressWarnings("deprecation")
-  public void testApplicationId() {
-    ConverterUtils.toApplicationId("application_1423221031460");
+  void testApplicationId() {
+    assertThrows(IllegalArgumentException.class, () -> {
+      ConverterUtils.toApplicationId("application_1423221031460");
+    });
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 59b779c071df4..eb3e67bd5722c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -18,13 +18,6 @@
 
 package org.apache.hadoop.yarn.util;
 
-import static org.apache.hadoop.fs.CreateFlag.CREATE;
-import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -52,14 +45,17 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.apache.hadoop.yarn.api.records.URL;
-import org.junit.Assert;
+import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
+import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
+import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -70,17 +66,21 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.junit.AfterClass;
-import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
-import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader;
-import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache;
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /**
  * Unit test for the FSDownload class.
@@ -96,7 +96,7 @@ private enum TEST_FILE_TYPE {
   };
   private Configuration conf = new Configuration();
 
-  @AfterClass
+  @AfterAll
   public static void deleteTestDir() throws IOException {
     FileContext fs = FileContext.getLocalFSFileContext();
     fs.delete(new Path("target", TestFSDownload.class.getSimpleName()), true);
@@ -270,16 +270,17 @@ static LocalResource createZipFile(FileContext files, Path p, int len,
     return ret;
   }
 
-  @Test (timeout=10000)
-  public void testDownloadBadPublic() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownloadBadPublic() throws IOException, URISyntaxException,
       InterruptedException {
     conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
     FileContext files = FileContext.getLocalFSFileContext(conf);
     final Path basedir = files.makeQualified(new Path("target",
-      TestFSDownload.class.getSimpleName()));
+        TestFSDownload.class.getSimpleName()));
     files.mkdir(basedir, null, true);
     conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
-    
+
     Map<LocalResource, LocalResourceVisibility> rsrcVis =
         new HashMap<LocalResource, LocalResourceVisibility>();
 
@@ -288,11 +289,11 @@ public void testDownloadBadPublic() throws IOException, URISyntaxException,
     rand.setSeed(sharedSeed);
     System.out.println("SEED: " + sharedSeed);
 
-    Map<LocalResource,Future<Path>> pending =
-      new HashMap<LocalResource,Future<Path>>();
+    Map<LocalResource, Future<Path>> pending =
+        new HashMap<LocalResource, Future<Path>>();
     ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
     LocalDirAllocator dirs =
-      new LocalDirAllocator(TestFSDownload.class.getName());
+        new LocalDirAllocator(TestFSDownload.class.getName());
     int size = 512;
     LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
     Path path = new Path(basedir, "test-file");
@@ -300,32 +301,33 @@ public void testDownloadBadPublic() throws IOException, URISyntaxException,
     rsrcVis.put(rsrc, vis);
     Path destPath = dirs.getLocalPathForWrite(
         basedir.toString(), size, conf);
-    destPath = new Path (destPath,
-      Long.toString(uniqueNumberGenerator.incrementAndGet()));
+    destPath = new Path(destPath,
+        Long.toString(uniqueNumberGenerator.incrementAndGet()));
     FSDownload fsd =
-      new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
-          destPath, rsrc);
+        new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
+            destPath, rsrc);
     pending.put(rsrc, exec.submit(fsd));
     exec.shutdown();
-    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS));
-    Assert.assertTrue(pending.get(rsrc).isDone());
+    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
+    assertTrue(pending.get(rsrc).isDone());
 
     try {
-      for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
+      for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
         p.getValue().get();
-        Assert.fail("We localized a file that is not public.");
+        fail("We localized a file that is not public.");
       }
     } catch (ExecutionException e) {
-      Assert.assertTrue(e.getCause() instanceof IOException);
+      assertTrue(e.getCause() instanceof IOException);
     }
   }
 
-  @Test (timeout=60000)
-  public void testDownloadPublicWithStatCache() throws IOException,
+  @Test
+  @Timeout(60000)
+  void testDownloadPublicWithStatCache() throws IOException,
       URISyntaxException, InterruptedException, ExecutionException {
     FileContext files = FileContext.getLocalFSFileContext(conf);
     Path basedir = files.makeQualified(new Path("target",
-      TestFSDownload.class.getSimpleName()));
+        TestFSDownload.class.getSimpleName()));
 
     // if test directory doesn't have ancestor permission, skip this test
     FileSystem f = basedir.getFileSystem(conf);
@@ -336,28 +338,28 @@ public void testDownloadPublicWithStatCache() throws IOException,
 
     int size = 512;
 
-    final ConcurrentMap<Path,AtomicInteger> counts =
-        new ConcurrentHashMap<Path,AtomicInteger>();
-    final CacheLoader<Path,Future<FileStatus>> loader =
+    final ConcurrentMap<Path, AtomicInteger> counts =
+        new ConcurrentHashMap<Path, AtomicInteger>();
+    final CacheLoader<Path, Future<FileStatus>> loader =
         FSDownload.createStatusCacheLoader(conf);
-    final LoadingCache<Path,Future<FileStatus>> statCache =
-        CacheBuilder.newBuilder().build(new CacheLoader<Path,Future<FileStatus>>() {
-      public Future<FileStatus> load(Path path) throws Exception {
-        // increment the count
-        AtomicInteger count = counts.get(path);
-        if (count == null) {
-          count = new AtomicInteger(0);
-          AtomicInteger existing = counts.putIfAbsent(path, count);
-          if (existing != null) {
-            count = existing;
-          }
-        }
-        count.incrementAndGet();
+    final LoadingCache<Path, Future<FileStatus>> statCache =
+        CacheBuilder.newBuilder().build(new CacheLoader<Path, Future<FileStatus>>() {
+          public Future<FileStatus> load(Path path) throws Exception {
+            // increment the count
+            AtomicInteger count = counts.get(path);
+            if (count == null) {
+              count = new AtomicInteger(0);
+              AtomicInteger existing = counts.putIfAbsent(path, count);
+              if (existing != null) {
+                count = existing;
+              }
+            }
+            count.incrementAndGet();
 
-        // use the default loader
-        return loader.load(path);
-      }
-    });
+            // use the default loader
+            return loader.load(path);
+          }
+        });
 
     // test FSDownload.isPublic() concurrently
     final int fileCount = 3;
@@ -382,11 +384,11 @@ public Boolean call() throws IOException {
     try {
       List<Future<Boolean>> futures = exec.invokeAll(tasks);
       // files should be public
-      for (Future<Boolean> future: futures) {
+      for (Future<Boolean> future : futures) {
         assertTrue(future.get());
       }
       // for each path exactly one file status call should be made
-      for (AtomicInteger count: counts.values()) {
+      for (AtomicInteger count : counts.values()) {
         assertSame(count.get(), 1);
       }
     } finally {
@@ -394,16 +396,17 @@ public Boolean call() throws IOException {
     }
   }
 
-  @Test (timeout=10000)
-  public void testDownload() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownload() throws IOException, URISyntaxException,
       InterruptedException {
     conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
     FileContext files = FileContext.getLocalFSFileContext(conf);
     final Path basedir = files.makeQualified(new Path("target",
-      TestFSDownload.class.getSimpleName()));
+        TestFSDownload.class.getSimpleName()));
     files.mkdir(basedir, null, true);
     conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
-    
+
     Map<LocalResource, LocalResourceVisibility> rsrcVis =
         new HashMap<LocalResource, LocalResourceVisibility>();
 
@@ -412,16 +415,16 @@ public void testDownload() throws IOException, URISyntaxException,
     rand.setSeed(sharedSeed);
     System.out.println("SEED: " + sharedSeed);
 
-    Map<LocalResource,Future<Path>> pending =
-      new HashMap<LocalResource,Future<Path>>();
+    Map<LocalResource, Future<Path>> pending =
+        new HashMap<LocalResource, Future<Path>>();
     ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
     LocalDirAllocator dirs =
-      new LocalDirAllocator(TestFSDownload.class.getName());
+        new LocalDirAllocator(TestFSDownload.class.getName());
     int[] sizes = new int[10];
     for (int i = 0; i < 10; ++i) {
       sizes[i] = rand.nextInt(512) + 512;
       LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
-      if (i%2 == 1) {
+      if (i % 2 == 1) {
         vis = LocalResourceVisibility.APPLICATION;
       }
       Path p = new Path(basedir, "" + i);
@@ -429,7 +432,7 @@ public void testDownload() throws IOException, URISyntaxException,
       rsrcVis.put(rsrc, vis);
       Path destPath = dirs.getLocalPathForWrite(
           basedir.toString(), sizes[i], conf);
-      destPath = new Path (destPath,
+      destPath = new Path(destPath,
           Long.toString(uniqueNumberGenerator.incrementAndGet()));
       FSDownload fsd =
           new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
@@ -438,29 +441,29 @@ public void testDownload() throws IOException, URISyntaxException,
     }
 
     exec.shutdown();
-    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS));
-    for (Future<Path> path: pending.values()) {
-      Assert.assertTrue(path.isDone());
+    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
+    for (Future<Path> path : pending.values()) {
+      assertTrue(path.isDone());
     }
 
     try {
-      for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
+      for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
         Path localized = p.getValue().get();
         assertEquals(sizes[Integer.parseInt(localized.getName())], p.getKey()
             .getSize());
 
         FileStatus status = files.getFileStatus(localized.getParent());
         FsPermission perm = status.getPermission();
-        assertEquals("Cache directory permissions are incorrect",
-            new FsPermission((short)0755), perm);
+        assertEquals(new FsPermission((short) 0755), perm,
+            "Cache directory permissions are incorrect");
 
         status = files.getFileStatus(localized);
         perm = status.getPermission();
-        System.out.println("File permission " + perm + 
+        System.out.println("File permission " + perm +
             " for rsrc vis " + p.getKey().getVisibility().name());
         assert(rsrcVis.containsKey(p.getKey()));
-        Assert.assertTrue("Private file should be 500",
-            perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
+        assertTrue(perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort(),
+            "Private file should be 500");
       }
     } catch (ExecutionException e) {
       throw new IOException("Failed exec", e);
@@ -524,15 +527,14 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
           FileStatus[] childFiles = files.getDefaultFileSystem().listStatus(
               filestatus.getPath());
           for (FileStatus childfile : childFiles) {
-            if(strFileName.endsWith(".ZIP") &&
-               childfile.getPath().getName().equals(strFileName) &&
-               !childfile.isDirectory()) {
-               Assert.fail("Failure...After unzip, there should have been a" +
-                 " directory formed with zip file name but found a file. "
-                 + childfile.getPath());
+            if (strFileName.endsWith(".ZIP") && childfile.getPath().getName().equals(strFileName)
+                && !childfile.isDirectory()) {
+              fail("Failure...After unzip, there should have been a"
+                  + " directory formed with zip file name but found a file. "
+                  + childfile.getPath());
             }
             if (childfile.getPath().getName().startsWith("tmp")) {
-              Assert.fail("Tmp File should not have been there "
+              fail("Tmp File should not have been there "
                   + childfile.getPath());
             }
           }
@@ -543,20 +545,23 @@ private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException,
     }
   }
 
-  @Test (timeout=10000)
-  public void testDownloadArchive() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownloadArchive() throws IOException, URISyntaxException,
       InterruptedException {
     downloadWithFileType(TEST_FILE_TYPE.TAR);
   }
 
-  @Test (timeout=10000)
-  public void testDownloadPatternJar() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownloadPatternJar() throws IOException, URISyntaxException,
       InterruptedException {
     downloadWithFileType(TEST_FILE_TYPE.JAR);
   }
 
-  @Test (timeout=10000)
-  public void testDownloadArchiveZip() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownloadArchiveZip() throws IOException, URISyntaxException,
       InterruptedException {
     downloadWithFileType(TEST_FILE_TYPE.ZIP);
   }
@@ -564,8 +569,9 @@ public void testDownloadArchiveZip() throws IOException, URISyntaxException,
   /*
    * To test fix for YARN-3029
    */
-  @Test (timeout=10000)
-  public void testDownloadArchiveZipWithTurkishLocale() throws IOException,
+  @Test
+  @Timeout(10000)
+  void testDownloadArchiveZipWithTurkishLocale() throws IOException,
       URISyntaxException, InterruptedException {
     Locale defaultLocale = Locale.getDefault();
     // Set to Turkish
@@ -576,8 +582,9 @@ public void testDownloadArchiveZipWithTurkishLocale() throws IOException,
     Locale.setDefault(defaultLocale);
   }
 
-  @Test (timeout=10000)
-  public void testDownloadArchiveTgz() throws IOException, URISyntaxException,
+  @Test
+  @Timeout(10000)
+  void testDownloadArchiveTgz() throws IOException, URISyntaxException,
       InterruptedException {
     downloadWithFileType(TEST_FILE_TYPE.TGZ);
   }
@@ -588,11 +595,11 @@ private void verifyPermsRecursively(FileSystem fs,
     FileStatus status = files.getFileStatus(p);
     if (status.isDirectory()) {
       if (vis == LocalResourceVisibility.PUBLIC) {
-        Assert.assertTrue(status.getPermission().toShort() ==
+        assertTrue(status.getPermission().toShort() ==
           FSDownload.PUBLIC_DIR_PERMS.toShort());
       }
       else {
-        Assert.assertTrue(status.getPermission().toShort() ==
+        assertTrue(status.getPermission().toShort() ==
           FSDownload.PRIVATE_DIR_PERMS.toShort());
       }
       if (!status.isSymlink()) {
@@ -604,40 +611,41 @@ private void verifyPermsRecursively(FileSystem fs,
     }
     else {
       if (vis == LocalResourceVisibility.PUBLIC) {
-        Assert.assertTrue(status.getPermission().toShort() ==
+        assertTrue(status.getPermission().toShort() ==
           FSDownload.PUBLIC_FILE_PERMS.toShort());
       }
       else {
-        Assert.assertTrue(status.getPermission().toShort() ==
+        assertTrue(status.getPermission().toShort() ==
           FSDownload.PRIVATE_FILE_PERMS.toShort());
       }
     }      
   }
 
-  @Test (timeout=10000)
-  public void testDirDownload() throws IOException, InterruptedException {
+  @Test
+  @Timeout(10000)
+  void testDirDownload() throws IOException, InterruptedException {
     FileContext files = FileContext.getLocalFSFileContext(conf);
     final Path basedir = files.makeQualified(new Path("target",
-      TestFSDownload.class.getSimpleName()));
+        TestFSDownload.class.getSimpleName()));
     files.mkdir(basedir, null, true);
     conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
-    
+
     Map<LocalResource, LocalResourceVisibility> rsrcVis =
         new HashMap<LocalResource, LocalResourceVisibility>();
-  
+
     Random rand = new Random();
     long sharedSeed = rand.nextLong();
     rand.setSeed(sharedSeed);
     System.out.println("SEED: " + sharedSeed);
 
-    Map<LocalResource,Future<Path>> pending =
-      new HashMap<LocalResource,Future<Path>>();
+    Map<LocalResource, Future<Path>> pending =
+        new HashMap<LocalResource, Future<Path>>();
     ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
     LocalDirAllocator dirs =
-      new LocalDirAllocator(TestFSDownload.class.getName());
+        new LocalDirAllocator(TestFSDownload.class.getName());
     for (int i = 0; i < 5; ++i) {
       LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
-      if (i%2 == 1) {
+      if (i % 2 == 1) {
         vis = LocalResourceVisibility.APPLICATION;
       }
 
@@ -646,7 +654,7 @@ public void testDirDownload() throws IOException, InterruptedException {
       rsrcVis.put(rsrc, vis);
       Path destPath = dirs.getLocalPathForWrite(
           basedir.toString(), conf);
-      destPath = new Path (destPath,
+      destPath = new Path(destPath,
           Long.toString(uniqueNumberGenerator.incrementAndGet()));
       FSDownload fsd =
           new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
@@ -655,21 +663,21 @@ public void testDirDownload() throws IOException, InterruptedException {
     }
 
     exec.shutdown();
-    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS));
-    for (Future<Path> path: pending.values()) {
-      Assert.assertTrue(path.isDone());
+    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
+    for (Future<Path> path : pending.values()) {
+      assertTrue(path.isDone());
     }
 
     try {
-      
-      for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
+
+      for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
         Path localized = p.getValue().get();
         FileStatus status = files.getFileStatus(localized);
 
         System.out.println("Testing path " + localized);
         assert(status.isDirectory());
         assert(rsrcVis.containsKey(p.getKey()));
-        
+
         verifyPermsRecursively(localized.getFileSystem(conf),
             files, localized, rsrcVis.get(p.getKey()));
       }
@@ -678,8 +686,9 @@ public void testDirDownload() throws IOException, InterruptedException {
     }
   }
 
-  @Test (timeout=10000)
-  public void testUniqueDestinationPath() throws Exception {
+  @Test
+  @Timeout(10000)
+  void testUniqueDestinationPath() throws Exception {
     FileContext files = FileContext.getLocalFSFileContext(conf);
     final Path basedir = files.makeQualified(new Path("target",
         TestFSDownload.class.getSimpleName()));
@@ -704,12 +713,12 @@ public void testUniqueDestinationPath() throws Exception {
             destPath, rsrc);
     Future<Path> rPath = singleThreadedExec.submit(fsd);
     singleThreadedExec.shutdown();
-    while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS));
-    Assert.assertTrue(rPath.isDone());
+    while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
+    assertTrue(rPath.isDone());
     // Now FSDownload will not create a random directory to localize the
     // resource. Therefore the final localizedPath for the resource should be
     // destination directory (passed as an argument) + file name.
-    Assert.assertEquals(destPath, rPath.get().getParent());
+    assertEquals(destPath, rPath.get().getParent());
   }
 
   /**
@@ -717,8 +726,9 @@ public void testUniqueDestinationPath() throws Exception {
    * from modification to the local resource's timestamp on the source FS just
    * before the download of this local resource has started.
    */
-  @Test(timeout=10000)
-  public void testResourceTimestampChangeDuringDownload()
+  @Test
+  @Timeout(10000)
+  void testResourceTimestampChangeDuringDownload()
       throws IOException, InterruptedException {
     conf = new Configuration();
     FileContext files = FileContext.getLocalFSFileContext(conf);
@@ -759,7 +769,7 @@ public void testResourceTimestampChangeDuringDownload()
       FileSystem sourceFs = sourceFsPath.getFileSystem(conf);
       sourceFs.setTimes(sourceFsPath, modifiedFSTimestamp, modifiedFSTimestamp);
     } catch (URISyntaxException use) {
-      Assert.fail("No exception expected.");
+      fail("No exception expected.");
     }
 
     // Execute the FSDownload operation.
@@ -770,19 +780,19 @@ public void testResourceTimestampChangeDuringDownload()
     exec.shutdown();
 
     exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
-    Assert.assertTrue(pending.get(localResource).isDone());
+    assertTrue(pending.get(localResource).isDone());
 
     try {
       for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
         p.getValue().get();
       }
-      Assert.fail("Exception expected from timestamp update during download");
+      fail("Exception expected from timestamp update during download");
     } catch (ExecutionException ee) {
-      Assert.assertTrue(ee.getCause() instanceof IOException);
-      Assert.assertTrue("Exception contains original timestamp",
-          ee.getMessage().contains(Times.formatISO8601(origLRTimestamp)));
-      Assert.assertTrue("Exception contains modified timestamp",
-          ee.getMessage().contains(Times.formatISO8601(modifiedFSTimestamp)));
+      assertTrue(ee.getCause() instanceof IOException);
+      assertTrue(ee.getMessage().contains(Times.formatISO8601(origLRTimestamp)),
+          "Exception contains original timestamp");
+      assertTrue(ee.getMessage().contains(Times.formatISO8601(modifiedFSTimestamp)),
+          "Exception contains modified timestamp");
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCacheHashMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCacheHashMap.java
index 9d3ec32975ae4..5cf9a61dcadba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCacheHashMap.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCacheHashMap.java
@@ -19,9 +19,12 @@
 
 import java.io.IOException;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test class to validate the correctness of the {@code LRUCacheHashMap}.
@@ -34,7 +37,7 @@ public class TestLRUCacheHashMap {
    * expected.
    */
   @Test
-  public void testLRUCache()
+  void testLRUCache()
       throws YarnException, IOException, InterruptedException {
 
     int mapSize = 5;
@@ -48,11 +51,11 @@ public void testLRUCache()
     map.put("4", 4);
     map.put("5", 5);
 
-    Assert.assertEquals(mapSize, map.size());
+    assertEquals(mapSize, map.size());
 
     // Check if all the elements in the map are from 1 to 5
     for (int i = 1; i < mapSize; i++) {
-      Assert.assertTrue(map.containsKey(Integer.toString(i)));
+      assertTrue(map.containsKey(Integer.toString(i)));
     }
 
     map.put("6", 6);
@@ -60,14 +63,14 @@ public void testLRUCache()
     map.put("7", 7);
     map.put("8", 8);
 
-    Assert.assertEquals(mapSize, map.size());
+    assertEquals(mapSize, map.size());
 
     // Check if all the elements in the map are from 5 to 8 and the 3
     for (int i = 5; i < mapSize; i++) {
-      Assert.assertTrue(map.containsKey(Integer.toString(i)));
+      assertTrue(map.containsKey(Integer.toString(i)));
     }
 
-    Assert.assertTrue(map.containsKey("3"));
+    assertTrue(map.containsKey("3"));
 
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
index 46c891a55bdac..346239f8e1bbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLog4jWarningErrorMetricsAppender.java
@@ -18,20 +18,22 @@
 
 package org.apache.hadoop.yarn.util;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
 import org.slf4j.MarkerFactory;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Level;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Test;
 
+import org.apache.hadoop.util.Time;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestLog4jWarningErrorMetricsAppender {
 
@@ -81,186 +83,179 @@ void logMessages(Level level, String message, int count) {
   }
 
   @Test
-  public void testPurge() throws Exception {
+  void testPurge() throws Exception {
     setupAppender(2, 1, 1);
     logMessages(Level.ERROR, "test message 1", 1);
     cutoff.clear();
     cutoff.add(0L);
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).get(0)
-      .size());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(1, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).get(0)
+        .size());
     Thread.sleep(3000);
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
-      .size());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
+        .size());
 
     setupAppender(2, 1000, 2);
 
     logMessages(Level.ERROR, "test message 1", 3);
     logMessages(Level.ERROR, "test message 2", 2);
 
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
-      .size());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
+        .size());
     logMessages(Level.ERROR, "test message 3", 3);
     Thread.sleep(2000);
-    Assert.assertEquals(8, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
-      .size());
+    assertEquals(8, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).get(0)
+        .size());
   }
 
   @Test
-  public void testErrorCounts() throws Exception {
+  void testErrorCounts() throws Exception {
     cutoff.clear();
     setupAppender(100, 100, 100);
     cutoff.add(0L);
     logMessages(Level.ERROR, "test message 1", 2);
     logMessages(Level.ERROR, "test message 2", 3);
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
-    Assert.assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert
-      .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(1, appender.getWarningCounts(cutoff).size());
+    assertEquals(5, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
     Thread.sleep(1000);
     cutoff.add(Time.now() / 1000);
     logMessages(Level.ERROR, "test message 3", 2);
-    Assert.assertEquals(2, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(2, appender.getWarningCounts(cutoff).size());
-    Assert.assertEquals(7, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(2, appender.getErrorCounts(cutoff).get(1).longValue());
-    Assert
-      .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
-    Assert
-      .assertEquals(0, appender.getWarningCounts(cutoff).get(1).longValue());
+    assertEquals(2, appender.getErrorCounts(cutoff).size());
+    assertEquals(2, appender.getWarningCounts(cutoff).size());
+    assertEquals(7, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(2, appender.getErrorCounts(cutoff).get(1).longValue());
+    assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getWarningCounts(cutoff).get(1).longValue());
   }
 
   @Test
-  public void testWarningCounts() throws Exception {
+  void testWarningCounts() throws Exception {
     cutoff.clear();
     setupAppender(100, 100, 100);
     cutoff.add(0L);
     logMessages(Level.WARN, "test message 1", 2);
     logMessages(Level.WARN, "test message 2", 3);
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
-    Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert
-      .assertEquals(5, appender.getWarningCounts(cutoff).get(0).longValue());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(1, appender.getWarningCounts(cutoff).size());
+    assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(5, appender.getWarningCounts(cutoff).get(0).longValue());
     Thread.sleep(1000);
     cutoff.add(Time.now() / 1000);
     logMessages(Level.WARN, "test message 3", 2);
-    Assert.assertEquals(2, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(2, appender.getWarningCounts(cutoff).size());
-    Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(1).longValue());
-    Assert
-      .assertEquals(7, appender.getWarningCounts(cutoff).get(0).longValue());
-    Assert
-      .assertEquals(2, appender.getWarningCounts(cutoff).get(1).longValue());
+    assertEquals(2, appender.getErrorCounts(cutoff).size());
+    assertEquals(2, appender.getWarningCounts(cutoff).size());
+    assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getErrorCounts(cutoff).get(1).longValue());
+    assertEquals(7, appender.getWarningCounts(cutoff).get(0).longValue());
+    assertEquals(2, appender.getWarningCounts(cutoff).get(1).longValue());
   }
 
   @Test
-  public void testWarningMessages() throws Exception {
+  void testWarningMessages() throws Exception {
     cutoff.clear();
     setupAppender(100, 100, 100);
     cutoff.add(0L);
     logMessages(Level.WARN, "test message 1", 2);
     logMessages(Level.WARN, "test message 2", 3);
-    Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
     Map<String, Log4jWarningErrorMetricsAppender.Element> errorsMap =
         appender.getErrorMessagesAndCounts(cutoff).get(0);
     Map<String, Log4jWarningErrorMetricsAppender.Element> warningsMap =
         appender.getWarningMessagesAndCounts(cutoff).get(0);
-    Assert.assertEquals(0, errorsMap.size());
-    Assert.assertEquals(2, warningsMap.size());
-    Assert.assertTrue(warningsMap.containsKey("test message 1"));
-    Assert.assertTrue(warningsMap.containsKey("test message 2"));
+    assertEquals(0, errorsMap.size());
+    assertEquals(2, warningsMap.size());
+    assertTrue(warningsMap.containsKey("test message 1"));
+    assertTrue(warningsMap.containsKey("test message 2"));
     Log4jWarningErrorMetricsAppender.Element msg1Info = warningsMap.get("test message 1");
     Log4jWarningErrorMetricsAppender.Element msg2Info = warningsMap.get("test message 2");
-    Assert.assertEquals(2, msg1Info.count.intValue());
-    Assert.assertEquals(3, msg2Info.count.intValue());
+    assertEquals(2, msg1Info.count.intValue());
+    assertEquals(3, msg2Info.count.intValue());
     Thread.sleep(1000);
     cutoff.add(Time.now() / 1000);
     logMessages(Level.WARN, "test message 3", 2);
-    Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
+    assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
+    assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
     errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(0);
     warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(0);
-    Assert.assertEquals(0, errorsMap.size());
-    Assert.assertEquals(3, warningsMap.size());
-    Assert.assertTrue(warningsMap.containsKey("test message 3"));
+    assertEquals(0, errorsMap.size());
+    assertEquals(3, warningsMap.size());
+    assertTrue(warningsMap.containsKey("test message 3"));
     errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(1);
     warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(1);
-    Assert.assertEquals(0, errorsMap.size());
-    Assert.assertEquals(1, warningsMap.size());
-    Assert.assertTrue(warningsMap.containsKey("test message 3"));
+    assertEquals(0, errorsMap.size());
+    assertEquals(1, warningsMap.size());
+    assertTrue(warningsMap.containsKey("test message 3"));
     Log4jWarningErrorMetricsAppender.Element msg3Info = warningsMap.get("test message 3");
-    Assert.assertEquals(2, msg3Info.count.intValue());
+    assertEquals(2, msg3Info.count.intValue());
   }
 
   @Test
-  public void testErrorMessages() throws Exception {
+  void testErrorMessages() throws Exception {
     cutoff.clear();
     setupAppender(100, 100, 100);
     cutoff.add(0L);
     logMessages(Level.ERROR, "test message 1", 2);
     logMessages(Level.ERROR, "test message 2", 3);
-    Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
     Map<String, Log4jWarningErrorMetricsAppender.Element> errorsMap =
         appender.getErrorMessagesAndCounts(cutoff).get(0);
     Map<String, Log4jWarningErrorMetricsAppender.Element> warningsMap =
         appender.getWarningMessagesAndCounts(cutoff).get(0);
-    Assert.assertEquals(2, errorsMap.size());
-    Assert.assertEquals(0, warningsMap.size());
-    Assert.assertTrue(errorsMap.containsKey("test message 1"));
-    Assert.assertTrue(errorsMap.containsKey("test message 2"));
+    assertEquals(2, errorsMap.size());
+    assertEquals(0, warningsMap.size());
+    assertTrue(errorsMap.containsKey("test message 1"));
+    assertTrue(errorsMap.containsKey("test message 2"));
     Log4jWarningErrorMetricsAppender.Element msg1Info = errorsMap.get("test message 1");
     Log4jWarningErrorMetricsAppender.Element msg2Info = errorsMap.get("test message 2");
-    Assert.assertEquals(2, msg1Info.count.intValue());
-    Assert.assertEquals(3, msg2Info.count.intValue());
+    assertEquals(2, msg1Info.count.intValue());
+    assertEquals(3, msg2Info.count.intValue());
     Thread.sleep(1000);
     cutoff.add(Time.now() / 1000);
     logMessages(Level.ERROR, "test message 3", 2);
-    Assert.assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
+    assertEquals(2, appender.getErrorMessagesAndCounts(cutoff).size());
+    assertEquals(2, appender.getWarningMessagesAndCounts(cutoff).size());
     errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(0);
     warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(0);
-    Assert.assertEquals(3, errorsMap.size());
-    Assert.assertEquals(0, warningsMap.size());
-    Assert.assertTrue(errorsMap.containsKey("test message 3"));
+    assertEquals(3, errorsMap.size());
+    assertEquals(0, warningsMap.size());
+    assertTrue(errorsMap.containsKey("test message 3"));
     errorsMap = appender.getErrorMessagesAndCounts(cutoff).get(1);
     warningsMap = appender.getWarningMessagesAndCounts(cutoff).get(1);
-    Assert.assertEquals(1, errorsMap.size());
-    Assert.assertEquals(0, warningsMap.size());
-    Assert.assertTrue(errorsMap.containsKey("test message 3"));
+    assertEquals(1, errorsMap.size());
+    assertEquals(0, warningsMap.size());
+    assertTrue(errorsMap.containsKey("test message 3"));
     Log4jWarningErrorMetricsAppender.Element msg3Info = errorsMap.get("test message 3");
-    Assert.assertEquals(2, msg3Info.count.intValue());
+    assertEquals(2, msg3Info.count.intValue());
   }
 
   @Test
-  public void testInfoDebugTrace() {
+  void testInfoDebugTrace() {
     cutoff.clear();
     setupAppender(100, 100, 100);
     cutoff.add(0L);
     logMessages(Level.INFO, "test message 1", 2);
     logMessages(Level.DEBUG, "test message 2", 2);
     logMessages(Level.TRACE, "test message 3", 2);
-    Assert.assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getErrorCounts(cutoff).size());
-    Assert.assertEquals(1, appender.getWarningCounts(cutoff).size());
-    Assert.assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
-    Assert
-      .assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
-    Assert.assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
-      .size());
-    Assert.assertEquals(0, appender.getWarningMessagesAndCounts(cutoff).get(0)
-      .size());
+    assertEquals(1, appender.getErrorMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getWarningMessagesAndCounts(cutoff).size());
+    assertEquals(1, appender.getErrorCounts(cutoff).size());
+    assertEquals(1, appender.getWarningCounts(cutoff).size());
+    assertEquals(0, appender.getErrorCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getWarningCounts(cutoff).get(0).longValue());
+    assertEquals(0, appender.getErrorMessagesAndCounts(cutoff).get(0)
+        .size());
+    assertEquals(0, appender.getWarningMessagesAndCounts(cutoff).get(0)
+        .size());
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
index 8065f40eb43e4..09dfb92f1d0e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
@@ -18,11 +18,6 @@
 
 package org.apache.hadoop.yarn.util;
 
-import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
-import static org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.UNAVAILABLE;
-import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
-
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.File;
@@ -37,9 +32,13 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileUtil;
@@ -52,19 +51,23 @@
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.MemInfo;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.ProcessSmapMemoryInfo;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.ProcessTreeSmapMemInfo;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
+import static org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.UNAVAILABLE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /**
  * A JUnit test to test ProcfsBasedProcessTree.
  */
 public class TestProcfsBasedProcessTree {
 
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestProcfsBasedProcessTree.class);
-  protected static File TEST_ROOT_DIR = new File("target",
-    TestProcfsBasedProcessTree.class.getName() + "-localDir");
+  private static final Logger LOG = LoggerFactory.getLogger(TestProcfsBasedProcessTree.class);
+  protected static File TEST_ROOT_DIR =
+      new File("target", TestProcfsBasedProcessTree.class.getName() + "-localDir");
 
   private ShellCommandExecutor shexec = null;
   private String pidFile, lowestDescendant, lostDescendant;
@@ -111,21 +114,22 @@ private String getRogueTaskPID() {
     return getPidFromPidFile(pidFile);
   }
 
-  @Before
+  @BeforeEach
   public void setup() throws IOException {
     assumeTrue(Shell.LINUX);
     FileContext.getLocalFSFileContext().delete(
       new Path(TEST_ROOT_DIR.getAbsolutePath()), true);
   }
 
-  @Test(timeout = 30000)
-  public void testProcessTree() throws Exception {
+  @Test
+  @Timeout(30000)
+  void testProcessTree() throws Exception {
     try {
-      Assert.assertTrue(ProcfsBasedProcessTree.isAvailable());
+      assertTrue(ProcfsBasedProcessTree.isAvailable());
     } catch (Exception e) {
       LOG.info(StringUtils.stringifyException(e));
-      Assert.assertTrue("ProcfsBaseProcessTree should be available on Linux",
-        false);
+      assertTrue(false,
+          "ProcfsBaseProcessTree should be available on Linux");
       return;
     }
     // create shell script
@@ -156,7 +160,7 @@ public void testProcessTree() throws Exception {
         + "(sleep 300&\n"
         + "echo $! > " + lostDescendant + ")\n"
         + " while true\n do\n" + "  sleep 5\n" + " done\n" + "fi",
-            StandardCharsets.UTF_8);
+        StandardCharsets.UTF_8);
 
     Thread t = new RogueTaskThread();
     t.start();
@@ -182,8 +186,8 @@ public void testProcessTree() throws Exception {
     // Verify the orphaned pid is In process tree
     String lostpid = getPidFromPidFile(lostDescendant);
     LOG.info("Orphaned pid: " + lostpid);
-    Assert.assertTrue("Child process owned by init escaped process tree.",
-       p.contains(lostpid));
+    assertTrue(p.contains(lostpid),
+        "Child process owned by init escaped process tree.");
 
     // Get the process-tree dump
     String processTreeDump = p.getProcessTreeDump();
@@ -208,18 +212,18 @@ public void testProcessTree() throws Exception {
     }
 
     LOG.info("Process-tree dump follows: \n" + processTreeDump);
-    Assert.assertTrue("Process-tree dump doesn't start with a proper header",
-      processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME "
-          + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
-          + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
+    assertTrue(processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME "
+            + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
+            + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"),
+        "Process-tree dump doesn't start with a proper header");
     for (int i = N; i >= 0; i--) {
       String cmdLineDump =
           "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)"
               + " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i;
       Pattern pat = Pattern.compile(cmdLineDump);
       Matcher mat = pat.matcher(processTreeDump);
-      Assert.assertTrue("Process-tree dump doesn't contain the cmdLineDump of "
-          + i + "th process!", mat.find());
+      assertTrue(mat.find(), "Process-tree dump doesn't contain the cmdLineDump of "
+          + i + "th process!");
     }
 
     // Not able to join thread sometimes when forking with large N.
@@ -232,12 +236,12 @@ public void testProcessTree() throws Exception {
 
     // ProcessTree is gone now. Any further calls should be sane.
     p.updateProcessTree();
-    Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
-    Assert.assertTrue(
-      "vmem for the gone-process is " + p.getVirtualMemorySize()
-          + " . It should be UNAVAILABLE(-1).",
-          p.getVirtualMemorySize() == UNAVAILABLE);
-    Assert.assertEquals("[ ]", p.toString());
+    assertFalse(isAlive(pid), "ProcessTree must have been gone");
+    assertTrue(
+        p.getVirtualMemorySize() == UNAVAILABLE,
+        "vmem for the gone-process is " + p.getVirtualMemorySize()
+            + " . It should be UNAVAILABLE(-1).");
+    assertEquals("[ ]", p.toString());
   }
 
   protected ProcfsBasedProcessTree createProcessTree(String pid) {
@@ -395,11 +399,12 @@ public void createMemoryMappingInfo(ProcessTreeSmapMemInfo[] procMemInfo) {
    *           if there was a problem setting up the fake procfs directories or
    *           files.
    */
-  @Test(timeout = 30000)
-  public void testCpuAndMemoryForProcessTree() throws IOException {
+  @Test
+  @Timeout(30000)
+  void testCpuAndMemoryForProcessTree() throws IOException {
 
     // test processes
-    String[] pids = { "100", "200", "300", "400" };
+    String[] pids = {"100", "200", "300", "400"};
     ControlledClock testClock = new ControlledClock();
     testClock.setTime(0);
     // create the fake procfs root directory.
@@ -442,34 +447,35 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
       processTree.updateProcessTree();
 
       // verify virtual memory
-      Assert.assertEquals("Virtual memory does not match", 600000L,
-        processTree.getVirtualMemorySize());
+      assertEquals(600000L, processTree.getVirtualMemorySize(), "Virtual memory does not match");
 
       // verify rss memory
       long cumuRssMem =
           ProcfsBasedProcessTree.PAGE_SIZE > 0
-              ? 600L * ProcfsBasedProcessTree.PAGE_SIZE : 
-                  ResourceCalculatorProcessTree.UNAVAILABLE;
-      Assert.assertEquals("rss memory does not match", cumuRssMem,
-        processTree.getRssMemorySize());
+              ? 600L * ProcfsBasedProcessTree.PAGE_SIZE :
+              ResourceCalculatorProcessTree.UNAVAILABLE;
+      assertEquals(cumuRssMem,
+          processTree.getRssMemorySize(),
+          "rss memory does not match");
 
       // verify cumulative cpu time
       long cumuCpuTime =
           ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0
               ? 7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
-      Assert.assertEquals("Cumulative cpu time does not match", cumuCpuTime,
-        processTree.getCumulativeCpuTime());
+      assertEquals(cumuCpuTime,
+          processTree.getCumulativeCpuTime(),
+          "Cumulative cpu time does not match");
 
       // verify CPU usage
-      Assert.assertEquals("Percent CPU time should be set to -1 initially",
-          -1.0, processTree.getCpuUsagePercent(),
-          0.01);
+      assertEquals(-1.0, processTree.getCpuUsagePercent(),
+          0.01,
+          "Percent CPU time should be set to -1 initially");
 
       // Check by enabling smaps
       setSmapsInProceTree(processTree, true);
       // anon (exclude r-xs,r--s)
-      Assert.assertEquals("rss memory does not match",
-          (20 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
+      assertEquals((20 * KB_TO_BYTES * 3), processTree.getRssMemorySize(),
+          "rss memory does not match");
 
       // test the cpu time again to see if it cumulates
       procInfos[0] =
@@ -490,8 +496,9 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
       cumuCpuTime =
           ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0
               ? 9400L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
-      Assert.assertEquals("Cumulative cpu time does not match", cumuCpuTime,
-        processTree.getCumulativeCpuTime());
+      assertEquals(cumuCpuTime,
+          processTree.getCumulativeCpuTime(),
+          "Cumulative cpu time does not match");
 
       double expectedCpuUsagePercent =
           (ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0) ?
@@ -500,11 +507,12 @@ public void testCpuAndMemoryForProcessTree() throws IOException {
       // expectedCpuUsagePercent is given by (94000L - 72000) * 100/
       //    200000;
       // which in this case is 11. Lets verify that first
-      Assert.assertEquals(11, expectedCpuUsagePercent, 0.001);
-      Assert.assertEquals("Percent CPU time is not correct expected " +
-              expectedCpuUsagePercent, expectedCpuUsagePercent,
+      assertEquals(11, expectedCpuUsagePercent, 0.001);
+      assertEquals(expectedCpuUsagePercent,
           processTree.getCpuUsagePercent(),
-          0.01);
+          0.01,
+          "Percent CPU time is not correct expected " +
+              expectedCpuUsagePercent);
     } finally {
       FileUtil.fullyDelete(procfsRootDir);
     }
@@ -529,8 +537,9 @@ private void setSmapsInProceTree(ProcfsBasedProcessTree processTree,
    *           if there was a problem setting up the fake procfs directories or
    *           files.
    */
-  @Test(timeout = 30000)
-  public void testMemForOlderProcesses() throws IOException {
+  @Test
+  @Timeout(30000)
+  void testMemForOlderProcesses() throws IOException {
     testMemForOlderProcesses(false);
     testMemForOlderProcesses(true);
   }
@@ -576,8 +585,7 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
       setSmapsInProceTree(processTree, smapEnabled);
 
       // verify virtual memory
-      Assert.assertEquals("Virtual memory does not match", 700000L,
-        processTree.getVirtualMemorySize());
+      assertEquals(700000L, processTree.getVirtualMemorySize(), "Virtual memory does not match");
 
       // write one more process as child of 100.
       String[] newPids = { "500" };
@@ -594,36 +602,31 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
 
       // check memory includes the new process.
       processTree.updateProcessTree();
-      Assert.assertEquals("vmem does not include new process",
-        1200000L, processTree.getVirtualMemorySize());
+      assertEquals(1200000L, processTree.getVirtualMemorySize(),
+          "vmem does not include new process");
       if (!smapEnabled) {
-        long cumuRssMem =
-            ProcfsBasedProcessTree.PAGE_SIZE > 0
-                ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 
-                    ResourceCalculatorProcessTree.UNAVAILABLE;
-        Assert.assertEquals("rssmem does not include new process",
-          cumuRssMem, processTree.getRssMemorySize());
+        long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
+            1200L * ProcfsBasedProcessTree.PAGE_SIZE :
+            ResourceCalculatorProcessTree.UNAVAILABLE;
+        assertEquals(cumuRssMem, processTree.getRssMemorySize(),
+            "rssmem does not include new process");
       } else {
-        Assert.assertEquals("rssmem does not include new process",
-            20 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
+        assertEquals(20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(),
+            "rssmem does not include new process");
       }
 
       // however processes older than 1 iteration will retain the older value
-      Assert.assertEquals(
-        "vmem shouldn't have included new process", 700000L,
-        processTree.getVirtualMemorySize(1));
+      assertEquals(700000L, processTree.getVirtualMemorySize(1),
+          "vmem shouldn't have included new process");
       if (!smapEnabled) {
-        long cumuRssMem =
-            ProcfsBasedProcessTree.PAGE_SIZE > 0
-                ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 
-                    ResourceCalculatorProcessTree.UNAVAILABLE;
-        Assert.assertEquals(
-          "rssmem shouldn't have included new process", cumuRssMem,
-          processTree.getRssMemorySize(1));
+        long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
+            700L * ProcfsBasedProcessTree.PAGE_SIZE :
+            ResourceCalculatorProcessTree.UNAVAILABLE;
+        assertEquals(cumuRssMem, processTree.getRssMemorySize(1),
+            "rssmem shouldn't have included new process");
       } else {
-        Assert.assertEquals(
-          "rssmem shouldn't have included new process",
-            20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
+        assertEquals(20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1),
+            "rssmem shouldn't have included new process");
       }
 
       // one more process
@@ -643,49 +646,41 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
       processTree.updateProcessTree();
 
       // processes older than 2 iterations should be same as before.
-      Assert.assertEquals(
-        "vmem shouldn't have included new processes", 700000L,
-        processTree.getVirtualMemorySize(2));
+      assertEquals(700000L, processTree.getVirtualMemorySize(2),
+          "vmem shouldn't have included new processes");
       if (!smapEnabled) {
         long cumuRssMem =
             ProcfsBasedProcessTree.PAGE_SIZE > 0
                 ? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 
                     ResourceCalculatorProcessTree.UNAVAILABLE;
-        Assert.assertEquals(
-          "rssmem shouldn't have included new processes",
-          cumuRssMem, processTree.getRssMemorySize(2));
+        assertEquals(cumuRssMem, processTree.getRssMemorySize(2),
+            "rssmem shouldn't have included new processes");
       } else {
-        Assert.assertEquals(
-          "rssmem shouldn't have included new processes",
-            20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
+        assertEquals(20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2),
+            "rssmem shouldn't have included new processes");
       }
 
       // processes older than 1 iteration should not include new process,
       // but include process 500
-      Assert.assertEquals(
-        "vmem shouldn't have included new processes", 1200000L,
-        processTree.getVirtualMemorySize(1));
+      assertEquals(1200000L, processTree.getVirtualMemorySize(1),
+          "vmem shouldn't have included new processes");
       if (!smapEnabled) {
         long cumuRssMem =
             ProcfsBasedProcessTree.PAGE_SIZE > 0
                 ? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 
                     ResourceCalculatorProcessTree.UNAVAILABLE;
-        Assert.assertEquals(
-          "rssmem shouldn't have included new processes",
-          cumuRssMem, processTree.getRssMemorySize(1));
+        assertEquals(cumuRssMem, processTree.getRssMemorySize(1),
+            "rssmem shouldn't have included new processes");
       } else {
-        Assert.assertEquals(
-          "rssmem shouldn't have included new processes",
-            20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
+        assertEquals(20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1),
+            "rssmem shouldn't have included new processes");
       }
 
       // no processes older than 3 iterations
-      Assert.assertEquals(
-          "Getting non-zero vmem for processes older than 3 iterations",
-          0, processTree.getVirtualMemorySize(3));
-      Assert.assertEquals(
-          "Getting non-zero rssmem for processes older than 3 iterations",
-          0, processTree.getRssMemorySize(3));
+      assertEquals(0, processTree.getVirtualMemorySize(3),
+          "Getting non-zero vmem for processes older than 3 iterations");
+      assertEquals(0, processTree.getRssMemorySize(3),
+          "Getting non-zero rssmem for processes older than 3 iterations");
     } finally {
       FileUtil.fullyDelete(procfsRootDir);
     }
@@ -700,8 +695,9 @@ private void testMemForOlderProcesses(boolean smapEnabled) throws IOException {
    *           if there was a problem setting up the fake procfs directories or
    *           files.
    */
-  @Test(timeout = 30000)
-  public void testDestroyProcessTree() throws IOException {
+  @Test
+  @Timeout(30000)
+  void testDestroyProcessTree() throws IOException {
     // test process
     String pid = "100";
     // create the fake procfs root directory.
@@ -715,8 +711,8 @@ public void testDestroyProcessTree() throws IOException {
           SystemClock.getInstance());
 
       // Let us not create stat file for pid 100.
-      Assert.assertTrue(ProcfsBasedProcessTree.checkPidPgrpidForMatch(pid,
-        procfsRootDir.getAbsolutePath()));
+      assertTrue(ProcfsBasedProcessTree.checkPidPgrpidForMatch(pid,
+          procfsRootDir.getAbsolutePath()));
     } finally {
       FileUtil.fullyDelete(procfsRootDir);
     }
@@ -727,10 +723,11 @@ public void testDestroyProcessTree() throws IOException {
    *
    * @throws IOException
    */
-  @Test(timeout = 30000)
-  public void testProcessTreeDump() throws IOException {
+  @Test
+  @Timeout(30000)
+  void testProcessTreeDump() throws IOException {
 
-    String[] pids = { "100", "200", "300", "400", "500", "600" };
+    String[] pids = {"100", "200", "300", "400", "500", "600"};
 
     File procfsRootDir = new File(TEST_ROOT_DIR, "proc");
 
@@ -790,29 +787,29 @@ public void testProcessTreeDump() throws IOException {
       String processTreeDump = processTree.getProcessTreeDump();
 
       LOG.info("Process-tree dump follows: \n" + processTreeDump);
-      Assert.assertTrue("Process-tree dump doesn't start with a proper header",
-        processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME "
-            + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
-            + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
+      assertTrue(processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME "
+              + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
+              + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"),
+          "Process-tree dump doesn't start with a proper header");
       for (int i = 0; i < 5; i++) {
         ProcessStatInfo p = procInfos[i];
-        Assert.assertTrue(
-          "Process-tree dump doesn't contain the cmdLineDump of process "
-              + p.pid,
-          processTreeDump.contains("\t|- " + p.pid + " " + p.ppid + " "
-              + p.pgrpId + " " + p.session + " (" + p.name + ") " + p.utime
-              + " " + p.stime + " " + p.vmem + " " + p.rssmemPage + " "
-              + cmdLines[i]));
+        assertTrue(
+            processTreeDump.contains("\t|- " + p.pid + " " + p.ppid + " "
+                + p.pgrpId + " " + p.session + " (" + p.name + ") " + p.utime
+                + " " + p.stime + " " + p.vmem + " " + p.rssmemPage + " "
+                + cmdLines[i]),
+            "Process-tree dump doesn't contain the cmdLineDump of process "
+                + p.pid);
       }
 
       // 600 should not be in the dump
       ProcessStatInfo p = procInfos[5];
-      Assert.assertFalse(
-        "Process-tree dump shouldn't contain the cmdLineDump of process "
-            + p.pid,
-        processTreeDump.contains("\t|- " + p.pid + " " + p.ppid + " "
-            + p.pgrpId + " " + p.session + " (" + p.name + ") " + p.utime + " "
-            + p.stime + " " + p.vmem + " " + cmdLines[5]));
+      assertFalse(
+          processTreeDump.contains("\t|- " + p.pid + " " + p.ppid + " "
+              + p.pgrpId + " " + p.session + " (" + p.name + ") " + p.utime + " "
+              + p.stime + " " + p.vmem + " " + cmdLines[5]),
+          "Process-tree dump shouldn't contain the cmdLineDump of process "
+              + p.pid);
     } finally {
       FileUtil.fullyDelete(procfsRootDir);
     }
@@ -887,11 +884,11 @@ private static boolean isAnyProcessInTreeAlive(
   public static void setupProcfsRootDir(File procfsRootDir) throws IOException {
     // cleanup any existing process root dir.
     if (procfsRootDir.exists()) {
-      Assert.assertTrue(FileUtil.fullyDelete(procfsRootDir));
+      assertTrue(FileUtil.fullyDelete(procfsRootDir));
     }
 
     // create afresh
-    Assert.assertTrue(procfsRootDir.mkdirs());
+    assertTrue(procfsRootDir.mkdirs());
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
index 4b1d7921fdbb6..9079fa00573b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
@@ -24,16 +24,19 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.net.DNSToSwitchMapping;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.net.Node;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestRackResolver {
 
@@ -41,7 +44,7 @@ public class TestRackResolver {
       LoggerFactory.getLogger(TestRackResolver.class);
   private static final String invalidHost = "invalidHost";
 
-  @Before
+  @BeforeEach
   public void setUp() {
     RackResolver.reset();
   }
@@ -54,8 +57,7 @@ public static final class MyResolver implements DNSToSwitchMapping {
     @Override
     public List<String> resolve(List<String> hostList) {
       // Only one host at a time
-      Assert.assertTrue("hostList size is " + hostList.size(),
-        hostList.size() <= 1);
+      assertTrue(hostList.size() <= 1, "hostList size is " + hostList.size());
       List<String> returnList = new ArrayList<String>();
       if (hostList.isEmpty()) {
         return returnList;
@@ -74,7 +76,7 @@ public List<String> resolve(List<String> hostList) {
       }
       // I should not be reached again as RackResolver is supposed to do
       // caching.
-      Assert.assertTrue(numHost1 <= 1);
+      assertTrue(numHost1 <= 1);
       return returnList;
     }
 
@@ -112,7 +114,7 @@ public List<String> resolve(List<String> hostList) {
         // I should not be reached again as RackResolver is supposed to do
         // caching.
       }
-      Assert.assertEquals(returnList.size(), hostList.size());
+      assertEquals(returnList.size(), hostList.size());
       return returnList;
     }
 
@@ -127,11 +129,11 @@ public void reloadCachedMappings(List<String> names) {
   }
 
   @Test
-  public void testCaching() {
+  void testCaching() {
     Configuration conf = new Configuration();
     conf.setClass(
-      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
-      MyResolver.class, DNSToSwitchMapping.class);
+        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+        MyResolver.class, DNSToSwitchMapping.class);
     RackResolver.init(conf);
     try {
       InetAddress iaddr = InetAddress.getByName("host1");
@@ -140,15 +142,15 @@ public void testCaching() {
       // Ignore if not found
     }
     Node node = RackResolver.resolve("host1");
-    Assert.assertEquals("/rack1", node.getNetworkLocation());
+    assertEquals("/rack1", node.getNetworkLocation());
     node = RackResolver.resolve("host1");
-    Assert.assertEquals("/rack1", node.getNetworkLocation());
+    assertEquals("/rack1", node.getNetworkLocation());
     node = RackResolver.resolve(invalidHost);
-    Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
+    assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation());
   }
 
   @Test
-  public void testMultipleHosts() {
+  void testMultipleHosts() {
     Configuration conf = new Configuration();
     conf.setClass(
         CommonConfigurationKeysPublic
@@ -158,9 +160,9 @@ public void testMultipleHosts() {
     RackResolver.init(conf);
     List<Node> nodes = RackResolver.resolve(
         Arrays.asList("host1", invalidHost, "host2"));
-    Assert.assertEquals("/rack1", nodes.get(0).getNetworkLocation());
-    Assert.assertEquals(NetworkTopology.DEFAULT_RACK,
+    assertEquals("/rack1", nodes.get(0).getNetworkLocation());
+    assertEquals(NetworkTopology.DEFAULT_RACK,
         nodes.get(1).getNetworkLocation());
-    Assert.assertEquals("/rack2", nodes.get(2).getNetworkLocation());
+    assertEquals("/rack2", nodes.get(2).getNetworkLocation());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolverScriptBasedMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolverScriptBasedMapping.java
index e8e875978b57e..41c6d439f6517 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolverScriptBasedMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolverScriptBasedMapping.java
@@ -18,17 +18,19 @@
 
 package org.apache.hadoop.yarn.util;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.net.DNSToSwitchMapping;
 import org.apache.hadoop.net.ScriptBasedMapping;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestRackResolverScriptBasedMapping {
 
   @Test
-  public void testScriptName() {
+  void testScriptName() {
     Configuration conf = new Configuration();
     conf
         .setClass(
@@ -38,7 +40,7 @@ public void testScriptName() {
     conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
         "testScript");
     RackResolver.init(conf);
-    Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(),
+    assertEquals(RackResolver.getDnsToSwitchMapping().toString(),
         "script-based mapping with script testScript");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
index 28cee7f1028ed..15ac9a54ea0ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
@@ -17,11 +17,14 @@
  */
 package org.apache.hadoop.yarn.util;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import static org.junit.Assert.*;
-import static org.hamcrest.core.IsInstanceOf.*;
-import static org.hamcrest.core.IsSame.*;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+import static org.hamcrest.core.IsSame.sameInstance;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * A JUnit test to test {@link ResourceCalculatorPlugin}
@@ -73,7 +76,7 @@ public boolean checkPidPgrpidForMatch() {
   }
 
   @Test
-  public void testCreateInstance() {
+  void testCreateInstance() {
     ResourceCalculatorProcessTree tree;
     tree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("1", EmptyProcessTree.class, new Configuration());
     assertNotNull(tree);
@@ -81,7 +84,7 @@ public void testCreateInstance() {
   }
 
   @Test
-  public void testCreatedInstanceConfigured() {
+  void testCreatedInstanceConfigured() {
     ResourceCalculatorProcessTree tree;
     Configuration conf = new Configuration();
     tree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("1", EmptyProcessTree.class, conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
index 21a27baccd40d..480d2635fb750 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
@@ -17,34 +17,35 @@
  */
 package org.apache.hadoop.yarn.util;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestTimelineServiceHelper {
 
   @Test
-  public void testMapCastToHashMap() {
+  void testMapCastToHashMap() {
 
     // Test null map be casted to null
     Map<String, String> nullMap = null;
-    Assert.assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap));
+    assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap));
 
     // Test empty hashmap be casted to a empty hashmap
     Map<String, String> emptyHashMap = new HashMap<String, String>();
-    Assert.assertEquals(
+    assertEquals(
         TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0);
 
     // Test empty non-hashmap be casted to a empty hashmap
     Map<String, String> emptyTreeMap = new TreeMap<String, String>();
-    Assert.assertEquals(
+    assertEquals(
         TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0);
 
     // Test non-empty hashmap be casted to hashmap correctly
@@ -52,7 +53,7 @@ public void testMapCastToHashMap() {
     String key = "KEY";
     String value = "VALUE";
     firstHashMap.put(key, value);
-    Assert.assertEquals(
+    assertEquals(
         TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap);
 
     // Test non-empty non-hashmap is casted correctly.
@@ -60,7 +61,7 @@ public void testMapCastToHashMap() {
     firstTreeMap.put(key, value);
     HashMap<String, String> alternateHashMap =
         TimelineServiceHelper.mapCastToHashMap(firstTreeMap);
-    Assert.assertEquals(firstTreeMap.size(), alternateHashMap.size());
+    assertEquals(firstTreeMap.size(), alternateHashMap.size());
     assertThat(alternateHashMap.get(key)).isEqualTo(value);
 
     // Test complicated hashmap be casted correctly
@@ -69,7 +70,7 @@ public void testMapCastToHashMap() {
     Set<String> hashSet = new HashSet<String>();
     hashSet.add(value);
     complicatedHashMap.put(key, hashSet);
-    Assert.assertEquals(
+    assertEquals(
         TimelineServiceHelper.mapCastToHashMap(complicatedHashMap),
         complicatedHashMap);
 
@@ -77,7 +78,7 @@ public void testMapCastToHashMap() {
     Map<String, Set<String>> complicatedTreeMap =
         new TreeMap<String, Set<String>>();
     complicatedTreeMap.put(key, hashSet);
-    Assert.assertEquals(
+    assertEquals(
         TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key),
         hashSet);
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java
index 918743d02ff24..4c438b1bbf971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java
@@ -18,47 +18,64 @@
 
 package org.apache.hadoop.yarn.util;
 
-import org.junit.Assert;
-import org.junit.Test;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hadoop.yarn.util.Times.ISO8601_DATE_FORMAT;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestTimes {
 
   @Test
-  public void testNegativeStartTimes() {
+  void testNegativeStartTimes() {
     long elapsed = Times.elapsed(-5, 10, true);
-    Assert.assertEquals("Elapsed time is not 0", 0, elapsed);
+    assertEquals(0, elapsed, "Elapsed time is not 0");
     elapsed = Times.elapsed(-5, 10, false);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
   }
 
   @Test
-  public void testNegativeFinishTimes() {
+  void testNegativeFinishTimes() {
     long elapsed = Times.elapsed(5, -10, false);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
   }
 
   @Test
-  public void testNegativeStartandFinishTimes() {
+  void testNegativeStartandFinishTimes() {
     long elapsed = Times.elapsed(-5, -10, false);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
   }
 
   @Test
-  public void testPositiveStartandFinishTimes() {
+  void testPositiveStartandFinishTimes() {
     long elapsed = Times.elapsed(5, 10, true);
-    Assert.assertEquals("Elapsed time is not 5", 5, elapsed);
+    assertEquals(5, elapsed, "Elapsed time is not 5");
     elapsed = Times.elapsed(5, 10, false);
-    Assert.assertEquals("Elapsed time is not 5", 5, elapsed);
+    assertEquals(5, elapsed, "Elapsed time is not 5");
   }
 
   @Test
-  public void testFinishTimesAheadOfStartTimes() {
+  void testFinishTimesAheadOfStartTimes() {
     long elapsed = Times.elapsed(10, 5, true);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
     elapsed = Times.elapsed(10, 5, false);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
     // use Long.MAX_VALUE to ensure started time is after the current one
     elapsed = Times.elapsed(Long.MAX_VALUE, 0, true);
-    Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+    assertEquals(-1, elapsed, "Elapsed time is not -1");
+  }
+
+  @Test
+  void validateISO() throws IOException {
+    SimpleDateFormat isoFormat = new SimpleDateFormat(ISO8601_DATE_FORMAT);
+    for (int i = 0; i < 1000; i++) {
+      long now = System.currentTimeMillis();
+      String instant =  Times.formatISO8601(now);
+      String date = isoFormat.format(new Date(now));
+      assertEquals(date, instant);
+    }
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
index db5d4bea94018..a83bb19c11faa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
@@ -18,13 +18,14 @@
 
 package org.apache.hadoop.yarn.util;
 
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.junit.Assert;
-import org.junit.Test;
 
 import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestWindowsBasedProcessTree {
   private static final Logger LOG = LoggerFactory
@@ -42,12 +43,13 @@ String getAllProcessInfoFromShell() {
     }
   }
 
-  @Test (timeout = 30000)
+  @Test
+  @Timeout(30000)
   @SuppressWarnings("deprecation")
-  public void tree() {
+  void tree() {
     assumeWindows();
-    assertTrue("WindowsBasedProcessTree should be available on Windows", 
-               WindowsBasedProcessTree.isAvailable());
+    assertTrue(WindowsBasedProcessTree.isAvailable(),
+        "WindowsBasedProcessTree should be available on Windows");
     ControlledClock testClock = new ControlledClock();
     long elapsedTimeBetweenUpdatesMsec = 0;
     testClock.setTime(elapsedTimeBetweenUpdatesMsec);
@@ -72,8 +74,7 @@ public void tree() {
     assertTrue(pTree.getRssMemorySize(1) == 2048);
     assertTrue(pTree.getCumulativeCpuTime() == 3000);
     assertTrue(pTree.getCpuUsagePercent() == 200);
-    Assert.assertEquals("Percent CPU time is not correct",
-        pTree.getCpuUsagePercent(), 200, 0.01);
+    assertEquals(pTree.getCpuUsagePercent(), 200, 0.01, "Percent CPU time is not correct");
 
     pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
     elapsedTimeBetweenUpdatesMsec = 2000;
@@ -84,7 +85,6 @@ public void tree() {
     assertTrue(pTree.getRssMemorySize() == 2048);
     assertTrue(pTree.getRssMemorySize(2) == 2048);
     assertTrue(pTree.getCumulativeCpuTime() == 4000);
-    Assert.assertEquals("Percent CPU time is not correct",
-        pTree.getCpuUsagePercent(), 0, 0.01);
+    assertEquals(pTree.getCpuUsagePercent(), 0, 0.01, "Percent CPU time is not correct");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
index 7e41501aaf2f3..5f38e9ce573e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
@@ -20,39 +20,38 @@
 
 import java.io.IOException;
 
-import org.junit.Test;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 /**
  * A JUnit test to test {@link YarnVersionInfo}
  */
 public class TestYarnVersionInfo {
-  
+
   /**
    * Test the yarn version info routines.
    * @throws IOException
    */
   @Test
-  public void versionInfoGenerated() throws IOException {
+  void versionInfoGenerated() throws IOException {
 
     // can't easily know what the correct values are going to be so just
     // make sure they aren't Unknown
-    assertNotEquals("getVersion returned Unknown",
-        "Unknown", YarnVersionInfo.getVersion());
-    assertNotEquals("getUser returned Unknown",
-        "Unknown", YarnVersionInfo.getUser());
-    assertNotEquals("getSrcChecksum returned Unknown",
-        "Unknown", YarnVersionInfo.getSrcChecksum());
+    assertNotEquals("Unknown", YarnVersionInfo.getVersion(), "getVersion returned Unknown");
+    assertNotEquals("Unknown", YarnVersionInfo.getUser(), "getUser returned Unknown");
+    assertNotEquals("Unknown", YarnVersionInfo.getSrcChecksum(), "getSrcChecksum returned Unknown");
 
     // these could be Unknown if the VersionInfo generated from code not in svn or git
     // so just check that they return something
-    assertNotNull("getUrl returned null", YarnVersionInfo.getUrl());
-    assertNotNull("getRevision returned null", YarnVersionInfo.getRevision());
-    assertNotNull("getBranch returned null", YarnVersionInfo.getBranch());
+    assertNotNull(YarnVersionInfo.getUrl(), "getUrl returned null");
+    assertNotNull(YarnVersionInfo.getRevision(), "getRevision returned null");
+    assertNotNull(YarnVersionInfo.getBranch(), "getBranch returned null");
 
-    assertTrue("getBuildVersion check doesn't contain: source checksum",
-               YarnVersionInfo.getBuildVersion().contains("source checksum"));
+    assertTrue(YarnVersionInfo.getBuildVersion().contains("source checksum"),
+        "getBuildVersion check doesn't contain: source checksum");
 
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/CustomResourceTypesConfigurationProvider.java
index 2b26151ccfd0d..16cb8d1e9d8f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/CustomResourceTypesConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/CustomResourceTypesConfigurationProvider.java
@@ -16,14 +16,6 @@
 
 package org.apache.hadoop.yarn.util.resource;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.Lists;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -33,6 +25,14 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
 import static java.util.stream.Collectors.toList;
 
 /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index 39f33990270f3..561aaed97b79b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -22,32 +22,32 @@
 import java.util.Collection;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
-@RunWith(Parameterized.class)
 public class TestResourceCalculator {
   private static final String EXTRA_RESOURCE_NAME = "test";
 
-  private final ResourceCalculator resourceCalculator;
+  private ResourceCalculator resourceCalculator;
 
-  @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> getParameters() {
-    return Arrays.asList(new Object[][] {
-        { "DefaultResourceCalculator", new DefaultResourceCalculator() },
-        { "DominantResourceCalculator", new DominantResourceCalculator() } });
+    return Arrays.asList(new Object[][]{
+        {"DefaultResourceCalculator", new DefaultResourceCalculator()},
+        {"DominantResourceCalculator", new DominantResourceCalculator()}});
   }
 
-  @Before
+  @BeforeEach
   public void setupNoExtraResource() {
     // This has to run before each test because we don't know when
     // setupExtraResource() might be called
@@ -61,34 +61,38 @@ private static void setupExtraResource() {
     ResourceUtils.resetResourceTypes(conf);
   }
 
-  public TestResourceCalculator(String name, ResourceCalculator rs) {
+  public void initTestResourceCalculator(String name, ResourceCalculator rs) {
     this.resourceCalculator = rs;
   }
-  
-  @Test(timeout = 10000)
-  public void testFitsIn() {
+
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  @Timeout(10000)
+  void testFitsIn(String name, ResourceCalculator rs) {
+
+    initTestResourceCalculator(name, rs);
 
     if (resourceCalculator instanceof DefaultResourceCalculator) {
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(2, 1)));
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(2, 2)));
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(1, 2)));
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(1, 1)));
-      Assert.assertFalse(resourceCalculator.fitsIn(
+      assertFalse(resourceCalculator.fitsIn(
           Resource.newInstance(2, 1), Resource.newInstance(1, 2)));
     } else if (resourceCalculator instanceof DominantResourceCalculator) {
-      Assert.assertFalse(resourceCalculator.fitsIn(
+      assertFalse(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(2, 1)));
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(2, 2)));
-      Assert.assertTrue(resourceCalculator.fitsIn(
+      assertTrue(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(1, 2)));
-      Assert.assertFalse(resourceCalculator.fitsIn(
+      assertFalse(resourceCalculator.fitsIn(
           Resource.newInstance(1, 2), Resource.newInstance(1, 1)));
-      Assert.assertFalse(resourceCalculator.fitsIn(
+      assertFalse(resourceCalculator.fitsIn(
           Resource.newInstance(2, 1), Resource.newInstance(1, 2)));
     }
   }
@@ -121,22 +125,22 @@ private void assertComparison(Resource cluster, Resource res1, Resource res2,
       int expected) {
     int actual = resourceCalculator.compare(cluster, res1, res2);
 
-    assertEquals(String.format("Resource comparison did not give the expected "
-        + "result for %s v/s %s", res1.toString(), res2.toString()),
-        expected, actual);
+    assertEquals(expected, actual, String.format("Resource comparison did not give the expected "
+        + "result for %s v/s %s", res1.toString(), res2.toString()));
 
     if (expected != 0) {
       // Try again with args in the opposite order and the negative of the
       // expected result.
       actual = resourceCalculator.compare(cluster, res2, res1);
-      assertEquals(String.format("Resource comparison did not give the "
-          + "expected result for %s v/s %s", res2.toString(), res1.toString()),
-          expected * -1, actual);
+      assertEquals(expected * -1, actual, String.format("Resource comparison did not give the "
+          + "expected result for %s v/s %s", res2.toString(), res1.toString()));
     }
   }
 
-  @Test
-  public void testCompareWithOnlyMandatory() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testCompareWithOnlyMandatory(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     // This test is necessary because there are optimizations that are only
     // triggered when only the mandatory resources are configured.
 
@@ -173,8 +177,10 @@ private void testCompareDominantWithOnlyMandatory(Resource cluster) {
     assertComparison(cluster, newResource(3, 1), newResource(3, 0), 1);
   }
 
-  @Test
-  public void testCompare() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testCompare(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     // Test with 3 resources
     setupExtraResource();
 
@@ -209,7 +215,7 @@ private void testCompareDefault(Resource cluster) {
   /**
    * Verify compare when one or all the resource are zero.
    */
-  private void testCompareDominantZeroValueResource(){
+  private void testCompareDominantZeroValueResource() {
     Resource cluster = newResource(4L, 4, 0);
     assertComparison(cluster, newResource(2, 1, 1), newResource(1, 1, 2), 1);
     assertComparison(cluster, newResource(2, 2, 1), newResource(1, 2, 2), 1);
@@ -261,8 +267,11 @@ private void testCompareDominant(Resource cluster) {
     assertComparison(cluster, newResource(3, 1, 1), newResource(3, 0, 0), 1);
   }
 
-  @Test(timeout = 10000)
-  public void testCompareWithEmptyCluster() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  @Timeout(10000)
+  void testCompareWithEmptyCluster(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     Resource clusterResource = Resource.newInstance(0, 0);
 
     // For lhs == rhs
@@ -316,35 +325,39 @@ private void assertResourcesOperations(Resource clusterResource,
       boolean greaterThan, boolean greaterThanOrEqual, Resource max,
       Resource min) {
 
-    assertEquals("Less Than operation is wrongly calculated.", lessThan,
-        Resources.lessThan(resourceCalculator, clusterResource, lhs, rhs));
+    assertEquals(lessThan,
+        Resources.lessThan(resourceCalculator, clusterResource, lhs, rhs),
+        "Less Than operation is wrongly calculated.");
 
     assertEquals(
-        "Less Than Or Equal To operation is wrongly calculated.",
         lessThanOrEqual, Resources.lessThanOrEqual(resourceCalculator,
-            clusterResource, lhs, rhs));
+            clusterResource, lhs, rhs), "Less Than Or Equal To operation is wrongly calculated.");
 
-    assertEquals("Greater Than operation is wrongly calculated.",
-        greaterThan,
-        Resources.greaterThan(resourceCalculator, clusterResource, lhs, rhs));
+    assertEquals(greaterThan,
+        Resources.greaterThan(resourceCalculator, clusterResource, lhs, rhs),
+        "Greater Than operation is wrongly calculated.");
 
-    assertEquals(
-        "Greater Than Or Equal To operation is wrongly calculated.",
-        greaterThanOrEqual, Resources.greaterThanOrEqual(resourceCalculator,
-            clusterResource, lhs, rhs));
+    assertEquals(greaterThanOrEqual,
+        Resources.greaterThanOrEqual(resourceCalculator, clusterResource, lhs, rhs),
+        "Greater Than Or Equal To operation is wrongly calculated.");
 
-    assertEquals("Max(value) Operation wrongly calculated.", max,
-        Resources.max(resourceCalculator, clusterResource, lhs, rhs));
+    assertEquals(max,
+        Resources.max(resourceCalculator, clusterResource, lhs, rhs),
+        "Max(value) Operation wrongly calculated.");
 
-    assertEquals("Min(value) operation is wrongly calculated.", min,
-        Resources.min(resourceCalculator, clusterResource, lhs, rhs));
+    assertEquals(min,
+        Resources.min(resourceCalculator, clusterResource, lhs, rhs),
+        "Min(value) operation is wrongly calculated.");
   }
 
   /**
    * Test resource normalization.
    */
-  @Test(timeout = 10000)
-  public void testNormalize() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  @Timeout(10000)
+  void testNormalize(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     // requested resources value cannot be an arbitrary number.
     Resource ask = Resource.newInstance(1111, 2);
     Resource min = Resource.newInstance(1024, 1);
@@ -420,22 +433,28 @@ public void testNormalize() {
     }
   }
 
-  @Test
-  public void testDivisionByZeroRatioDenominatorIsZero() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testDivisionByZeroRatioDenominatorIsZero(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     float ratio = resourceCalculator.ratio(newResource(1, 1), newResource(0,
         0));
     assertEquals(Float.POSITIVE_INFINITY, ratio, 0.00001);
   }
 
-  @Test
-  public void testDivisionByZeroRatioNumeratorAndDenominatorIsZero() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testDivisionByZeroRatioNumeratorAndDenominatorIsZero(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     float ratio = resourceCalculator.ratio(newResource(0, 0), newResource(0,
         0));
     assertEquals(0.0, ratio, 0.00001);
   }
 
-  @Test
-  public void testFitsInDiagnosticsCollector() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testFitsInDiagnosticsCollector(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     if (resourceCalculator instanceof DefaultResourceCalculator) {
       // required-resource = (0, 0)
       assertEquals(ImmutableSet.of(),
@@ -551,8 +570,10 @@ public void testFitsInDiagnosticsCollector() {
     }
   }
 
-  @Test
-  public void testRatioWithNoExtraResource() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testRatioWithNoExtraResource(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     //setup
     Resource resource1 = newResource(1, 1);
     Resource resource2 = newResource(2, 1);
@@ -570,8 +591,10 @@ public void testRatioWithNoExtraResource() {
     }
   }
 
-  @Test
-  public void testRatioWithExtraResource() {
+  @MethodSource("getParameters")
+  @ParameterizedTest(name = "{0}")
+  void testRatioWithExtraResource(String name, ResourceCalculator rs) {
+    initTestResourceCalculator(name, rs);
     //setup
     setupExtraResource();
     Resource resource1 = newResource(1, 1, 2);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 8f742fa902fc1..cdd99d1f79fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -18,25 +18,6 @@
 
 package org.apache.hadoop.yarn.util.resource;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
@@ -48,6 +29,27 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
 /**
  * Test class to verify all resource utility methods.
  */
@@ -70,15 +72,12 @@ static class ResourceFileInformation {
     }
   }
 
-  @Rule
-  public ExpectedException expexted = ExpectedException.none();
-
-  @Before
+  @BeforeEach
   public void setup() {
     ResourceUtils.resetResourceTypes();
   }
 
-  @After
+  @AfterEach
   public void teardown() {
     if (nodeResourcesFile != null && nodeResourcesFile.exists()) {
       nodeResourcesFile.delete();
@@ -137,29 +136,28 @@ private Map<String, ResourceInformation> setupNodeResources(
   private void testMemoryAndVcores(Map<String, ResourceInformation> res) {
     String memory = ResourceInformation.MEMORY_MB.getName();
     String vcores = ResourceInformation.VCORES.getName();
-    Assert.assertTrue("Resource 'memory' missing", res.containsKey(memory));
-    Assert.assertEquals("'memory' units incorrect",
-        ResourceInformation.MEMORY_MB.getUnits(), res.get(memory).getUnits());
-    Assert.assertEquals("'memory' types incorrect",
-        ResourceInformation.MEMORY_MB.getResourceType(),
-        res.get(memory).getResourceType());
-    Assert.assertTrue("Resource 'vcores' missing", res.containsKey(vcores));
-    Assert.assertEquals("'vcores' units incorrect",
-        ResourceInformation.VCORES.getUnits(), res.get(vcores).getUnits());
-    Assert.assertEquals("'vcores' type incorrect",
-        ResourceInformation.VCORES.getResourceType(),
-        res.get(vcores).getResourceType());
+    assertTrue(res.containsKey(memory), "Resource 'memory' missing");
+    assertEquals(ResourceInformation.MEMORY_MB.getUnits(), res.get(memory).getUnits(),
+        "'memory' units incorrect");
+    assertEquals(ResourceInformation.MEMORY_MB.getResourceType(), res.get(memory).getResourceType(),
+        "'memory' types incorrect");
+    assertTrue(res.containsKey(vcores), "Resource 'vcores' missing");
+    assertEquals(ResourceInformation.VCORES.getUnits(), res.get(vcores).getUnits(),
+        "'vcores' units incorrect");
+    assertEquals(ResourceInformation.VCORES.getResourceType(),
+        res.get(vcores).getResourceType(),
+        "'vcores' type incorrect");
   }
 
   @Test
-  public void testGetResourceTypes() {
+  void testGetResourceTypes() {
     Map<String, ResourceInformation> res = ResourceUtils.getResourceTypes();
-    Assert.assertEquals(2, res.size());
+    assertEquals(2, res.size());
     testMemoryAndVcores(res);
   }
 
   @Test
-  public void testGetResourceTypesConfigs() throws Exception {
+  void testGetResourceTypesConfigs() throws Exception {
     Configuration conf = new YarnConfiguration();
 
     ResourceFileInformation testFile1 =
@@ -183,19 +181,19 @@ public void testGetResourceTypesConfigs() throws Exception {
       ResourceUtils.resetResourceTypes();
       res = setupResourceTypesInternal(conf, testInformation.filename);
       testMemoryAndVcores(res);
-      Assert.assertEquals(testInformation.resourceCount, res.size());
+      assertEquals(testInformation.resourceCount, res.size());
       for (Map.Entry<String, String> entry :
           testInformation.resourceNameUnitsMap.entrySet()) {
         String resourceName = entry.getKey();
-        Assert.assertTrue("Missing key " + resourceName,
-            res.containsKey(resourceName));
-        Assert.assertEquals(entry.getValue(), res.get(resourceName).getUnits());
+        assertTrue(res.containsKey(resourceName),
+            "Missing key " + resourceName);
+        assertEquals(entry.getValue(), res.get(resourceName).getUnits());
       }
     }
   }
 
   @Test
-  public void testGetRequestedResourcesFromConfig() {
+  void testGetRequestedResourcesFromConfig() {
     Configuration conf = new Configuration();
 
     //these resource type configurations should be recognised
@@ -229,14 +227,14 @@ public void testGetRequestedResourcesFromConfig() {
     Set<String> expectedSet =
         new HashSet<>(Arrays.asList(expectedKeys));
 
-    Assert.assertEquals(properList.size(), expectedKeys.length);
+    assertEquals(properList.size(), expectedKeys.length);
     properList.forEach(
-        item -> Assert.assertTrue(expectedSet.contains(item.getName())));
+        item -> assertTrue(expectedSet.contains(item.getName())));
 
   }
 
   @Test
-  public void testGetResourceTypesConfigErrors() throws IOException {
+  void testGetResourceTypesConfigErrors() throws IOException {
     Configuration conf = new YarnConfiguration();
 
     String[] resourceFiles = {"resource-types-error-1.xml",
@@ -246,7 +244,7 @@ public void testGetResourceTypesConfigErrors() throws IOException {
       ResourceUtils.resetResourceTypes();
       try {
         setupResourceTypesInternal(conf, resourceFile);
-        Assert.fail("Expected error with file " + resourceFile);
+        fail("Expected error with file " + resourceFile);
       } catch (YarnRuntimeException | IllegalArgumentException e) {
         //Test passed
       }
@@ -254,7 +252,7 @@ public void testGetResourceTypesConfigErrors() throws IOException {
   }
 
   @Test
-  public void testInitializeResourcesMap() {
+  void testInitializeResourcesMap() {
     String[] empty = {"", ""};
     String[] res1 = {"resource1", "m"};
     String[] res2 = {"resource2", "G"};
@@ -291,31 +289,30 @@ public void testInitializeResourcesMap() {
         len = 4;
       }
 
-      Assert.assertEquals(len, ret.size());
+      assertEquals(len, ret.size());
       for (String[] resources : test) {
         if (resources[0].length() == 0) {
           continue;
         }
-        Assert.assertTrue(ret.containsKey(resources[0]));
+        assertTrue(ret.containsKey(resources[0]));
         ResourceInformation resInfo = ret.get(resources[0]);
-        Assert.assertEquals(resources[1], resInfo.getUnits());
-        Assert.assertEquals(ResourceTypes.COUNTABLE, resInfo.getResourceType());
+        assertEquals(resources[1], resInfo.getUnits());
+        assertEquals(ResourceTypes.COUNTABLE, resInfo.getResourceType());
       }
       // we must always have memory and vcores with their fixed units
-      Assert.assertTrue(ret.containsKey("memory-mb"));
+      assertTrue(ret.containsKey("memory-mb"));
       ResourceInformation memInfo = ret.get("memory-mb");
-      Assert.assertEquals("Mi", memInfo.getUnits());
-      Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
-      Assert.assertTrue(ret.containsKey("vcores"));
+      assertEquals("Mi", memInfo.getUnits());
+      assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
+      assertTrue(ret.containsKey("vcores"));
       ResourceInformation vcoresInfo = ret.get("vcores");
-      Assert.assertEquals("", vcoresInfo.getUnits());
-      Assert
-          .assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType());
+      assertEquals("", vcoresInfo.getUnits());
+      assertEquals(ResourceTypes.COUNTABLE, vcoresInfo.getResourceType());
     }
   }
 
   @Test
-  public void testInitializeResourcesMapErrors() {
+  void testInitializeResourcesMapErrors() {
     String[] mem1 = {"memory-mb", ""};
     String[] vcores1 = {"vcores", "M"};
 
@@ -346,7 +343,7 @@ public void testInitializeResourcesMapErrors() {
       }
       try {
         ResourceUtils.initializeResourcesMap(conf);
-        Assert.fail("resource map initialization should fail");
+        fail("resource map initialization should fail");
       } catch (Exception e) {
         //Test passed
       }
@@ -354,7 +351,7 @@ public void testInitializeResourcesMapErrors() {
   }
 
   @Test
-  public void testGetResourceInformation() throws Exception {
+  void testGetResourceInformation() throws Exception {
     Configuration conf = new YarnConfiguration();
     Map<String, Resource> testRun = new HashMap<>();
     setupResourceTypesInternal(conf, "resource-types-4.xml");
@@ -372,16 +369,16 @@ public void testGetResourceInformation() throws Exception {
       ResourceUtils.resetNodeResources();
       Map<String, ResourceInformation> actual = setupNodeResources(conf,
           resourceFile);
-      Assert.assertEquals(actual.size(),
+      assertEquals(actual.size(),
           entry.getValue().getResources().length);
       for (ResourceInformation resInfo : entry.getValue().getResources()) {
-        Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+        assertEquals(resInfo, actual.get(resInfo.getName()));
       }
     }
   }
 
   @Test
-  public void testGetNodeResourcesConfigErrors() throws Exception {
+  void testGetNodeResourcesConfigErrors() throws Exception {
     Configuration conf = new YarnConfiguration();
     setupResourceTypesInternal(conf, "resource-types-4.xml");
     String[] invalidNodeResFiles = {"node-resources-error-1.xml"};
@@ -390,7 +387,7 @@ public void testGetNodeResourcesConfigErrors() throws Exception {
       ResourceUtils.resetNodeResources();
       try {
         setupNodeResources(conf, resourceFile);
-        Assert.fail("Expected error with file " + resourceFile);
+        fail("Expected error with file " + resourceFile);
       } catch (YarnRuntimeException e) {
         //Test passed
       }
@@ -398,26 +395,28 @@ public void testGetNodeResourcesConfigErrors() throws Exception {
   }
 
   @Test
-  public void testGetNodeResourcesRedefineFpgaErrors() throws Exception {
-    Configuration conf = new YarnConfiguration();
-    expexted.expect(YarnRuntimeException.class);
-    expexted.expectMessage("Defined mandatory resource type=yarn.io/fpga");
-    setupResourceTypesInternal(conf,
-        "resource-types-error-redefine-fpga-unit.xml");
+  void testGetNodeResourcesRedefineFpgaErrors() throws Exception {
+    Throwable exception = assertThrows(YarnRuntimeException.class, () -> {
+      Configuration conf = new YarnConfiguration();
+      setupResourceTypesInternal(conf,
+          "resource-types-error-redefine-fpga-unit.xml");
+    });
+    assertTrue(exception.getMessage().contains("Defined mandatory resource type=yarn.io/fpga"));
   }
 
   @Test
-  public void testGetNodeResourcesRedefineGpuErrors() throws Exception {
-    Configuration conf = new YarnConfiguration();
-    expexted.expect(YarnRuntimeException.class);
-    expexted.expectMessage("Defined mandatory resource type=yarn.io/gpu");
-    setupResourceTypesInternal(conf,
-        "resource-types-error-redefine-gpu-unit.xml");
+  void testGetNodeResourcesRedefineGpuErrors() throws Exception {
+    Throwable exception = assertThrows(YarnRuntimeException.class, () -> {
+      Configuration conf = new YarnConfiguration();
+      setupResourceTypesInternal(conf,
+          "resource-types-error-redefine-gpu-unit.xml");
+    });
+    assertTrue(exception.getMessage().contains("Defined mandatory resource type=yarn.io/gpu"));
   }
 
   @Test
-  public void testResourceNameFormatValidation() {
-    String[] validNames = new String[] {
+  void testResourceNameFormatValidation() {
+    String[] validNames = new String[]{
         "yarn.io/gpu",
         "gpu",
         "g_1_2",
@@ -427,7 +426,7 @@ public void testResourceNameFormatValidation() {
         "a....b",
     };
 
-    String[] invalidNames = new String[] {
+    String[] invalidNames = new String[]{
         "asd/resource/-name",
         "prefix/-resource_1",
         "prefix/0123resource",
@@ -443,7 +442,7 @@ public void testResourceNameFormatValidation() {
     for (String invalidName : invalidNames) {
       try {
         ResourceUtils.validateNameOfResourceNameAndThrowException(invalidName);
-        Assert.fail("Expected to fail name check, the name=" + invalidName
+        fail("Expected to fail name check, the name=" + invalidName
             + " is illegal.");
       } catch (YarnRuntimeException e) {
         // Expected
@@ -452,7 +451,7 @@ public void testResourceNameFormatValidation() {
   }
 
   @Test
-  public void testGetResourceInformationWithDiffUnits() throws Exception {
+  void testGetResourceInformationWithDiffUnits() throws Exception {
     Configuration conf = new YarnConfiguration();
     Map<String, Resource> testRun = new HashMap<>();
     setupResourceTypesInternal(conf, "resource-types-4.xml");
@@ -476,82 +475,82 @@ public void testGetResourceInformationWithDiffUnits() throws Exception {
       ResourceUtils.resetNodeResources();
       Map<String, ResourceInformation> actual = setupNodeResources(conf,
           resourceFile);
-      Assert.assertEquals(actual.size(),
+      assertEquals(actual.size(),
           entry.getValue().getResources().length);
       for (ResourceInformation resInfo : entry.getValue().getResources()) {
-        Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+        assertEquals(resInfo, actual.get(resInfo.getName()));
       }
     }
   }
 
   @Test
-  public void testResourceUnitParsing() throws Exception {
+  void testResourceUnitParsing() throws Exception {
     Resource res = ResourceUtils.createResourceFromString("memory=20g,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20 * 1024, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory=20G,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20 * 1024, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory=20M,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory=20m,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory-mb=20,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory-mb=20m,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20, 3), res);
 
     res = ResourceUtils.createResourceFromString("memory-mb=20G,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(20 * 1024, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(20 * 1024, 3), res);
 
     // W/o unit for memory means bits, and 20 bits will be rounded to 0
     res = ResourceUtils.createResourceFromString("memory=20,vcores=3",
-            ResourceUtils.getResourcesTypeInfo());
-    Assert.assertEquals(Resources.createResource(0, 3), res);
+        ResourceUtils.getResourcesTypeInfo());
+    assertEquals(Resources.createResource(0, 3), res);
 
     // Test multiple resources
     List<ResourceTypeInfo> resTypes = new ArrayList<>(
-            ResourceUtils.getResourcesTypeInfo());
+        ResourceUtils.getResourcesTypeInfo());
     resTypes.add(ResourceTypeInfo.newInstance(ResourceInformation.GPU_URI, ""));
     ResourceUtils.reinitializeResources(resTypes);
     res = ResourceUtils.createResourceFromString("memory=2G,vcores=3,gpu=0",
-            resTypes);
-    Assert.assertEquals(2 * 1024, res.getMemorySize());
-    Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
+        resTypes);
+    assertEquals(2 * 1024, res.getMemorySize());
+    assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
 
     res = ResourceUtils.createResourceFromString("memory=2G,vcores=3,gpu=3",
-            resTypes);
-    Assert.assertEquals(2 * 1024, res.getMemorySize());
-    Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
+        resTypes);
+    assertEquals(2 * 1024, res.getMemorySize());
+    assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
 
     res = ResourceUtils.createResourceFromString("memory=2G,vcores=3",
-            resTypes);
-    Assert.assertEquals(2 * 1024, res.getMemorySize());
-    Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
+        resTypes);
+    assertEquals(2 * 1024, res.getMemorySize());
+    assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
 
     res = ResourceUtils.createResourceFromString(
-            "memory=2G,vcores=3,yarn.io/gpu=0", resTypes);
-    Assert.assertEquals(2 * 1024, res.getMemorySize());
-    Assert.assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
+        "memory=2G,vcores=3,yarn.io/gpu=0", resTypes);
+    assertEquals(2 * 1024, res.getMemorySize());
+    assertEquals(0, res.getResourceValue(ResourceInformation.GPU_URI));
 
     res = ResourceUtils.createResourceFromString(
-            "memory=2G,vcores=3,yarn.io/gpu=3", resTypes);
-    Assert.assertEquals(2 * 1024, res.getMemorySize());
-    Assert.assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
+        "memory=2G,vcores=3,yarn.io/gpu=3", resTypes);
+    assertEquals(2 * 1024, res.getMemorySize());
+    assertEquals(3, res.getResourceValue(ResourceInformation.GPU_URI));
   }
 
   @Test
-  public void testMultipleOpsForResourcesWithTags() throws Exception {
+  void testMultipleOpsForResourcesWithTags() throws Exception {
 
     Configuration conf = new YarnConfiguration();
     setupResourceTypes(conf, "resource-types-6.xml");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 307e0d8e07dc4..3f724096277b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -18,28 +18,30 @@
 
 package org.apache.hadoop.yarn.util.resource;
 
+import java.io.File;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
 
-import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMin;
-import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMax;
 import static org.apache.hadoop.yarn.util.resource.Resources.add;
-import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundUp;
-import static org.apache.hadoop.yarn.util.resource.Resources.subtract;
+import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMax;
+import static org.apache.hadoop.yarn.util.resource.Resources.componentwiseMin;
+import static org.apache.hadoop.yarn.util.resource.Resources.fitsIn;
 import static org.apache.hadoop.yarn.util.resource.Resources.multiply;
 import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndAddTo;
 import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundDown;
-import static org.apache.hadoop.yarn.util.resource.Resources.fitsIn;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.yarn.util.resource.Resources.multiplyAndRoundUp;
+import static org.apache.hadoop.yarn.util.resource.Resources.subtract;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestResources {
   private static final String INVALID_RESOURCE_MSG = "Invalid resource value";
@@ -75,12 +77,12 @@ private void deleteResourceTypesFile() {
     }
   }
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     setupExtraResourceType();
   }
 
-  @After
+  @AfterEach
   public void teardown() {
     deleteResourceTypesFile();
   }
@@ -96,8 +98,9 @@ public Resource createResource(long memory, int vCores, long resource2) {
     return ret;
   }
 
-  @Test(timeout = 10000)
-  public void testCompareToWithUnboundedResource() {
+  @Test
+  @Timeout(10000)
+  void testCompareToWithUnboundedResource() {
     unsetExtraResourceType();
     Resource unboundedClone = Resources.clone(ExtendedResources.unbounded());
     assertTrue(unboundedClone
@@ -107,8 +110,9 @@ public void testCompareToWithUnboundedResource() {
         unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0);
   }
 
-  @Test(timeout = 10000)
-  public void testCompareToWithNoneResource() {
+  @Test
+  @Timeout(10000)
+  void testCompareToWithNoneResource() {
     assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0);
     assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0);
     assertTrue(Resources.none().compareTo(createResource(0, 1)) < 0);
@@ -118,8 +122,9 @@ public void testCompareToWithNoneResource() {
     assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
   }
 
-  @Test(timeout = 1000)
-  public void testFitsIn() {
+  @Test
+  @Timeout(1000)
+  void testFitsIn() {
     assertTrue(fitsIn(createResource(1, 1), createResource(2, 2)));
     assertTrue(fitsIn(createResource(2, 2), createResource(2, 2)));
     assertFalse(fitsIn(createResource(2, 2), createResource(1, 1)));
@@ -130,8 +135,9 @@ public void testFitsIn() {
     assertTrue(fitsIn(createResource(1, 1, 1), createResource(2, 2, 2)));
   }
 
-  @Test(timeout = 1000)
-  public void testComponentwiseMin() {
+  @Test
+  @Timeout(1000)
+  void testComponentwiseMin() {
     assertEquals(createResource(1, 1),
         componentwiseMin(createResource(1, 1), createResource(2, 2)));
     assertEquals(createResource(1, 1),
@@ -147,7 +153,7 @@ public void testComponentwiseMin() {
   }
 
   @Test
-  public void testComponentwiseMax() {
+  void testComponentwiseMax() {
     assertEquals(createResource(2, 2),
         componentwiseMax(createResource(1, 1), createResource(2, 2)));
     assertEquals(createResource(2, 2),
@@ -165,7 +171,7 @@ public void testComponentwiseMax() {
   }
 
   @Test
-  public void testAdd() {
+  void testAdd() {
     assertEquals(createResource(2, 3),
         add(createResource(1, 1), createResource(1, 2)));
     assertEquals(createResource(3, 2),
@@ -177,7 +183,7 @@ public void testAdd() {
   }
 
   @Test
-  public void testSubtract() {
+  void testSubtract() {
     assertEquals(createResource(1, 0),
         subtract(createResource(2, 1), createResource(1, 1)));
     assertEquals(createResource(0, 1),
@@ -189,7 +195,7 @@ public void testSubtract() {
   }
 
   @Test
-  public void testClone() {
+  void testClone() {
     assertEquals(createResource(1, 1), Resources.clone(createResource(1, 1)));
     assertEquals(createResource(1, 1, 0),
         Resources.clone(createResource(1, 1)));
@@ -200,7 +206,7 @@ public void testClone() {
   }
 
   @Test
-  public void testMultiply() {
+  void testMultiply() {
     assertEquals(createResource(4, 2), multiply(createResource(2, 1), 2));
     assertEquals(createResource(4, 2, 0), multiply(createResource(2, 1), 2));
     assertEquals(createResource(2, 4), multiply(createResource(1, 2), 2));
@@ -209,61 +215,74 @@ public void testMultiply() {
     assertEquals(createResource(4, 4, 6), multiply(createResource(2, 2, 3), 2));
   }
 
-  @Test(timeout=10000)
-  public void testMultiplyRoundUp() {
+  @Test
+  @Timeout(10000)
+  void testMultiplyRoundUp() {
     final double by = 0.5;
     final String memoryErrorMsg = "Invalid memory size.";
     final String vcoreErrorMsg = "Invalid virtual core number.";
     Resource resource = Resources.createResource(1, 1);
     Resource result = Resources.multiplyAndRoundUp(resource, by);
-    assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
-    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+    assertEquals(result.getMemorySize(), 1, memoryErrorMsg);
+    assertEquals(result.getVirtualCores(), 1, vcoreErrorMsg);
 
     resource = Resources.createResource(2, 2);
     result = Resources.multiplyAndRoundUp(resource, by);
-    assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
-    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
+    assertEquals(result.getMemorySize(), 1, memoryErrorMsg);
+    assertEquals(result.getVirtualCores(), 1, vcoreErrorMsg);
 
     resource = Resources.createResource(0, 0);
     result = Resources.multiplyAndRoundUp(resource, by);
-    assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
-    assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
+    assertEquals(result.getMemorySize(), 0, memoryErrorMsg);
+    assertEquals(result.getVirtualCores(), 0, vcoreErrorMsg);
   }
 
   @Test
-  public void testMultiplyAndRoundUpCustomResources() {
-    assertEquals(INVALID_RESOURCE_MSG, createResource(5, 2, 8),
-        multiplyAndRoundUp(createResource(3, 1, 5), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(5, 2, 0),
-        multiplyAndRoundUp(createResource(3, 1, 0), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(5, 5, 0),
-        multiplyAndRoundUp(createResource(3, 3, 0), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(8, 3, 13),
-        multiplyAndRoundUp(createResource(3, 1, 5), 2.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(8, 3, 0),
-        multiplyAndRoundUp(createResource(3, 1, 0), 2.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(8, 8, 0),
-        multiplyAndRoundUp(createResource(3, 3, 0), 2.5));
+  void testMultiplyAndRoundUpCustomResources() {
+    assertEquals(createResource(5, 2, 8),
+        multiplyAndRoundUp(createResource(3, 1, 5), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(5, 2, 0),
+        multiplyAndRoundUp(createResource(3, 1, 0), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(5, 5, 0),
+        multiplyAndRoundUp(createResource(3, 3, 0), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(8, 3, 13),
+        multiplyAndRoundUp(createResource(3, 1, 5), 2.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(8, 3, 0),
+        multiplyAndRoundUp(createResource(3, 1, 0), 2.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(8, 8, 0),
+        multiplyAndRoundUp(createResource(3, 3, 0), 2.5),
+        INVALID_RESOURCE_MSG);
   }
 
   @Test
-  public void testMultiplyAndRoundDown() {
-    assertEquals(INVALID_RESOURCE_MSG, createResource(4, 1),
-        multiplyAndRoundDown(createResource(3, 1), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(4, 1, 0),
-        multiplyAndRoundDown(createResource(3, 1), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(1, 4),
-        multiplyAndRoundDown(createResource(1, 3), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(1, 4, 0),
-        multiplyAndRoundDown(createResource(1, 3), 1.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(7, 7, 0),
-        multiplyAndRoundDown(createResource(3, 3, 0), 2.5));
-    assertEquals(INVALID_RESOURCE_MSG, createResource(2, 2, 7),
-        multiplyAndRoundDown(createResource(1, 1, 3), 2.5));
+  void testMultiplyAndRoundDown() {
+    assertEquals(createResource(4, 1),
+        multiplyAndRoundDown(createResource(3, 1), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(4, 1, 0),
+        multiplyAndRoundDown(createResource(3, 1), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(1, 4),
+        multiplyAndRoundDown(createResource(1, 3), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(1, 4, 0),
+        multiplyAndRoundDown(createResource(1, 3), 1.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(7, 7, 0),
+        multiplyAndRoundDown(createResource(3, 3, 0), 2.5),
+        INVALID_RESOURCE_MSG);
+    assertEquals(createResource(2, 2, 7),
+        multiplyAndRoundDown(createResource(1, 1, 3), 2.5),
+        INVALID_RESOURCE_MSG);
   }
 
   @Test
-  public void testMultiplyAndAddTo() throws Exception {
+  void testMultiplyAndAddTo() throws Exception {
     unsetExtraResourceType();
     setupExtraResourceType();
     assertEquals(createResource(6, 4),
@@ -283,7 +302,7 @@ public void testMultiplyAndAddTo() throws Exception {
   }
 
   @Test
-  public void testCreateResourceWithSameLongValue() throws Exception {
+  void testCreateResourceWithSameLongValue() throws Exception {
     unsetExtraResourceType();
     setupExtraResourceType();
 
@@ -294,7 +313,7 @@ public void testCreateResourceWithSameLongValue() throws Exception {
   }
 
   @Test
-  public void testCreateResourceWithSameIntValue() throws Exception {
+  void testCreateResourceWithSameIntValue() throws Exception {
     unsetExtraResourceType();
     setupExtraResourceType();
 
@@ -305,14 +324,14 @@ public void testCreateResourceWithSameIntValue() throws Exception {
   }
 
   @Test
-  public void testCreateSimpleResourceWithSameLongValue() {
+  void testCreateSimpleResourceWithSameLongValue() {
     Resource res = ResourceUtils.createResourceWithSameValue(11L);
     assertEquals(11L, res.getMemorySize());
     assertEquals(11, res.getVirtualCores());
   }
 
   @Test
-  public void testCreateSimpleResourceWithSameIntValue() {
+  void testCreateSimpleResourceWithSameIntValue() {
     Resource res = ResourceUtils.createResourceWithSameValue(11);
     assertEquals(11, res.getMemorySize());
     assertEquals(11, res.getVirtualCores());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/timeline/TestShortenedFlowName.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/timeline/TestShortenedFlowName.java
index f9295579ea20e..e69734199b621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/timeline/TestShortenedFlowName.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/timeline/TestShortenedFlowName.java
@@ -17,11 +17,13 @@
  */
 package org.apache.hadoop.yarn.util.timeline;
 
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
 
-import java.util.UUID;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * Test case for limiting flow name size.
@@ -31,22 +33,22 @@ public class TestShortenedFlowName {
   private static final String TEST_FLOW_NAME = "TestFlowName";
 
   @Test
-  public void testRemovingUUID() {
+  void testRemovingUUID() {
     String flowName = TEST_FLOW_NAME + "-" + UUID.randomUUID();
     flowName = TimelineUtils.removeUUID(flowName);
-    Assert.assertEquals(TEST_FLOW_NAME, flowName);
+    assertEquals(TEST_FLOW_NAME, flowName);
   }
 
   @Test
-  public void testShortenedFlowName() {
+  void testShortenedFlowName() {
     YarnConfiguration conf = new YarnConfiguration();
     String flowName = TEST_FLOW_NAME + UUID.randomUUID();
     conf.setInt(YarnConfiguration.FLOW_NAME_MAX_SIZE, 8);
     String shortenedFlowName = TimelineUtils.shortenFlowName(flowName, conf);
-    Assert.assertEquals("TestFlow", shortenedFlowName);
+    assertEquals("TestFlow", shortenedFlowName);
     conf.setInt(YarnConfiguration.FLOW_NAME_MAX_SIZE,
         YarnConfiguration.FLOW_NAME_DEFAULT_MAX_SIZE);
     shortenedFlowName = TimelineUtils.shortenFlowName(flowName, conf);
-    Assert.assertEquals(TEST_FLOW_NAME, shortenedFlowName);
+    assertEquals(TEST_FLOW_NAME, shortenedFlowName);
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java
index d537fa748f9bc..6578248cae0d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/JerseyTestBase.java
@@ -21,11 +21,11 @@
 import java.io.IOException;
 import java.util.Random;
 
-import org.apache.hadoop.net.ServerSocketUtil;
-
 import com.sun.jersey.test.framework.JerseyTest;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 
+import org.apache.hadoop.net.ServerSocketUtil;
+
 public abstract class JerseyTestBase extends JerseyTest {
   public JerseyTestBase(WebAppDescriptor appDescriptor) {
     super(appDescriptor);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
index 6f6ee5d4b53ab..242bf047805d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
@@ -21,17 +21,16 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
-
 import javax.ws.rs.ext.ContextResolver;
 import javax.ws.rs.ext.Provider;
 import javax.xml.bind.JAXBContext;
 
-import org.apache.hadoop.yarn.webapp.MyTestWebService.MyInfo;
-
 import com.google.inject.Singleton;
 import com.sun.jersey.api.json.JSONConfiguration;
 import com.sun.jersey.api.json.JSONJAXBContext;
 
+import org.apache.hadoop.yarn.webapp.MyTestWebService.MyInfo;
+
 @Singleton
 @Provider
 public class MyTestJAXBContextResolver implements ContextResolver<JAXBContext> {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
index 74623a4fed960..1d0a01ea53dea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
@@ -27,6 +27,7 @@
 import javax.xml.bind.annotation.XmlRootElement;
 
 import com.google.inject.Singleton;
+
 import org.apache.hadoop.http.JettyUtils;
 
 @Singleton
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestParseRoute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestParseRoute.java
index 87d62cc7e77a4..7d22aaea9d24b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestParseRoute.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestParseRoute.java
@@ -20,64 +20,76 @@
 
 import java.util.Arrays;
 
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebAppException;
-import org.junit.Test;
-import static org.junit.Assert.*;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class TestParseRoute {
 
-  @Test public void testNormalAction() {
+  @Test
+  void testNormalAction() {
     assertEquals(Arrays.asList("/foo/action", "foo", "action", ":a1", ":a2"),
-                 WebApp.parseRoute("/foo/action/:a1/:a2"));
+        WebApp.parseRoute("/foo/action/:a1/:a2"));
   }
 
-  @Test public void testDefaultController() {
+  @Test
+  void testDefaultController() {
     assertEquals(Arrays.asList("/", "default", "index"),
-                 WebApp.parseRoute("/"));
+        WebApp.parseRoute("/"));
   }
 
-  @Test public void testDefaultAction() {
+  @Test
+  void testDefaultAction() {
     assertEquals(Arrays.asList("/foo", "foo", "index"),
-                 WebApp.parseRoute("/foo"));
+        WebApp.parseRoute("/foo"));
     assertEquals(Arrays.asList("/foo", "foo", "index"),
-                 WebApp.parseRoute("/foo/"));
+        WebApp.parseRoute("/foo/"));
   }
 
-  @Test public void testMissingAction() {
+  @Test
+  void testMissingAction() {
     assertEquals(Arrays.asList("/foo", "foo", "index", ":a1"),
-                 WebApp.parseRoute("/foo/:a1"));
+        WebApp.parseRoute("/foo/:a1"));
   }
 
-  @Test public void testDefaultCapture() {
+  @Test
+  void testDefaultCapture() {
     assertEquals(Arrays.asList("/", "default", "index", ":a"),
-                 WebApp.parseRoute("/:a"));
+        WebApp.parseRoute("/:a"));
   }
 
-  @Test public void testPartialCapture1() {
+  @Test
+  void testPartialCapture1() {
     assertEquals(Arrays.asList("/foo/action/bar", "foo", "action", "bar", ":a"),
-                 WebApp.parseRoute("/foo/action/bar/:a"));
+        WebApp.parseRoute("/foo/action/bar/:a"));
   }
 
-  @Test public void testPartialCapture2() {
+  @Test
+  void testPartialCapture2() {
     assertEquals(Arrays.asList("/foo/action", "foo", "action", ":a1", "bar",
-                               ":a2", ":a3"),
-                 WebApp.parseRoute("/foo/action/:a1/bar/:a2/:a3"));
+            ":a2", ":a3"),
+        WebApp.parseRoute("/foo/action/:a1/bar/:a2/:a3"));
   }
 
-  @Test public void testLeadingPaddings() {
+  @Test
+  void testLeadingPaddings() {
     assertEquals(Arrays.asList("/foo/action", "foo", "action", ":a"),
-                 WebApp.parseRoute(" /foo/action/ :a"));
+        WebApp.parseRoute(" /foo/action/ :a"));
   }
 
-  @Test public void testTrailingPaddings() {
+  @Test
+  void testTrailingPaddings() {
     assertEquals(Arrays.asList("/foo/action", "foo", "action", ":a"),
-                 WebApp.parseRoute("/foo/action//:a / "));
+        WebApp.parseRoute("/foo/action//:a / "));
     assertEquals(Arrays.asList("/foo/action", "foo", "action"),
-                 WebApp.parseRoute("/foo/action / "));
+        WebApp.parseRoute("/foo/action / "));
   }
 
-  @Test(expected=WebAppException.class) public void testMissingLeadingSlash() {
-    WebApp.parseRoute("foo/bar");
+  @Test
+  void testMissingLeadingSlash() {
+    assertThrows(WebAppException.class, () -> {
+      WebApp.parseRoute("foo/bar");
+    });
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java
index 075bed216dcd5..fd7525167f88d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java
@@ -18,15 +18,18 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage;
 import java.io.PrintWriter;
 import javax.servlet.http.HttpServletResponse;
+
 import com.google.inject.Injector;
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
 
-import org.junit.Test;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class TestSubViews {
 
@@ -61,7 +64,8 @@ public void render(Block html) {
     }
   }
 
-  @Test public void testSubView() throws Exception {
+  @Test
+  void testSubView() throws Exception {
     Injector injector = WebAppTests.createMockInjector(this);
     injector.getInstance(MainView.class).render();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
index 98b7505426896..7d7a1575b4724 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
@@ -18,22 +18,16 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.apache.hadoop.yarn.util.StringHelper.join;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLEncoder;
 
+import com.google.inject.Inject;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.net.ServerSocketUtil;
@@ -42,11 +36,18 @@
 import org.apache.hadoop.yarn.webapp.view.JQueryUI;
 import org.apache.hadoop.yarn.webapp.view.RobotsTextPage;
 import org.apache.hadoop.yarn.webapp.view.TextPage;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestWebApp {
   static final Logger LOG = LoggerFactory.getLogger(TestWebApp.class);
@@ -150,12 +151,14 @@ public void render(Page.HTML<__> html) {
 
   String echo(String s) { return s; }
 
-  @Test public void testCreate() {
+  @Test
+  void testCreate() {
     WebApp app = WebApps.$for(this).start();
     app.stop();
   }
 
-  @Test public void testCreateWithPort() {
+  @Test
+  void testCreateWithPort() {
     // see if the ephemeral port is updated
     WebApp app = WebApps.$for(this).at(0).start();
     int port = app.getListenerAddress().getPort();
@@ -167,72 +170,80 @@ public void render(Page.HTML<__> html) {
     app.stop();
   }
 
-  @Test(expected=org.apache.hadoop.yarn.webapp.WebAppException.class)
-  public void testCreateWithBindAddressNonZeroPort() {
-    WebApp app = WebApps.$for(this).at("0.0.0.0:50000").start();
-    int port = app.getListenerAddress().getPort();
-    assertEquals(50000, port);
-    // start another WebApp with same NonZero port
-    WebApp app2 = WebApps.$for(this).at("0.0.0.0:50000").start();
-    // An exception occurs (findPort disabled)
-    app.stop();
-    app2.stop();
+  @Test
+  void testCreateWithBindAddressNonZeroPort() {
+    assertThrows(org.apache.hadoop.yarn.webapp.WebAppException.class, () -> {
+      WebApp app = WebApps.$for(this).at("0.0.0.0:50000").start();
+      int port = app.getListenerAddress().getPort();
+      assertEquals(50000, port);
+      // start another WebApp with same NonZero port
+      WebApp app2 = WebApps.$for(this).at("0.0.0.0:50000").start();
+      // An exception occurs (findPort disabled)
+      app.stop();
+      app2.stop();
+    });
   }
 
-  @Test(expected=org.apache.hadoop.yarn.webapp.WebAppException.class)
-  public void testCreateWithNonZeroPort() {
-    WebApp app = WebApps.$for(this).at(50000).start();
-    int port = app.getListenerAddress().getPort();
-    assertEquals(50000, port);
-    // start another WebApp with same NonZero port
-    WebApp app2 = WebApps.$for(this).at(50000).start();
-    // An exception occurs (findPort disabled)
-    app.stop();
-    app2.stop();
+  @Test
+  void testCreateWithNonZeroPort() {
+    assertThrows(org.apache.hadoop.yarn.webapp.WebAppException.class, () -> {
+      WebApp app = WebApps.$for(this).at(50000).start();
+      int port = app.getListenerAddress().getPort();
+      assertEquals(50000, port);
+      // start another WebApp with same NonZero port
+      WebApp app2 = WebApps.$for(this).at(50000).start();
+      // An exception occurs (findPort disabled)
+      app.stop();
+      app2.stop();
+    });
   }
 
-  @Test public void testServePaths() {
+  @Test
+  void testServePaths() {
     WebApp app = WebApps.$for("test", this).start();
     assertEquals("/test", app.getRedirectPath());
-    String[] expectedPaths = { "/test", "/test/*" };
+    String[] expectedPaths = {"/test", "/test/*"};
     String[] pathSpecs = app.getServePathSpecs();
-     
+
     assertEquals(2, pathSpecs.length);
-    for(int i = 0; i < expectedPaths.length; i++) {
+    for (int i = 0; i < expectedPaths.length; i++) {
       assertTrue(ArrayUtils.contains(pathSpecs, expectedPaths[i]));
     }
     app.stop();
   }
 
-  @Test public void testServePathsNoName() {
+  @Test
+  void testServePathsNoName() {
     WebApp app = WebApps.$for("", this).start();
     assertEquals("/", app.getRedirectPath());
-    String[] expectedPaths = { "/*" };
+    String[] expectedPaths = {"/*"};
     String[] pathSpecs = app.getServePathSpecs();
-     
+
     assertEquals(1, pathSpecs.length);
-    for(int i = 0; i < expectedPaths.length; i++) {
+    for (int i = 0; i < expectedPaths.length; i++) {
       assertTrue(ArrayUtils.contains(pathSpecs, expectedPaths[i]));
     }
     app.stop();
   }
 
-  @Test public void testDefaultRoutes() throws Exception {
+  @Test
+  void testDefaultRoutes() throws Exception {
     WebApp app = WebApps.$for("test", this).start();
     String baseUrl = baseUrl(app);
     try {
-      assertEquals("foo", getContent(baseUrl +"test/foo").trim());
-      assertEquals("foo", getContent(baseUrl +"test/foo/index").trim());
-      assertEquals("bar", getContent(baseUrl +"test/foo/bar").trim());
-      assertEquals("default", getContent(baseUrl +"test").trim());
-      assertEquals("default", getContent(baseUrl +"test/").trim());
+      assertEquals("foo", getContent(baseUrl + "test/foo").trim());
+      assertEquals("foo", getContent(baseUrl + "test/foo/index").trim());
+      assertEquals("bar", getContent(baseUrl + "test/foo/bar").trim());
+      assertEquals("default", getContent(baseUrl + "test").trim());
+      assertEquals("default", getContent(baseUrl + "test/").trim());
       assertEquals("default", getContent(baseUrl).trim());
     } finally {
       app.stop();
     }
   }
 
-  @Test public void testCustomRoutes() throws Exception {
+  @Test
+  void testCustomRoutes() throws Exception {
     WebApp app =
         WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
           @Override
@@ -249,21 +260,22 @@ public void setup() {
     String baseUrl = baseUrl(app);
     try {
       assertEquals("foo", getContent(baseUrl).trim());
-      assertEquals("foo", getContent(baseUrl +"test").trim());
-      assertEquals("foo1", getContent(baseUrl +"test/1").trim());
-      assertEquals("bar", getContent(baseUrl +"test/bar/foo").trim());
-      assertEquals("default", getContent(baseUrl +"test/foo/bar").trim());
-      assertEquals("default1", getContent(baseUrl +"test/foo/1").trim());
-      assertEquals("default2", getContent(baseUrl +"test/foo/bar/2").trim());
-      assertEquals(404, getResponseCode(baseUrl +"test/goo"));
-      assertEquals(200, getResponseCode(baseUrl +"ws/v1/test"));
-      assertTrue(getContent(baseUrl +"ws/v1/test").contains("myInfo"));
+      assertEquals("foo", getContent(baseUrl + "test").trim());
+      assertEquals("foo1", getContent(baseUrl + "test/1").trim());
+      assertEquals("bar", getContent(baseUrl + "test/bar/foo").trim());
+      assertEquals("default", getContent(baseUrl + "test/foo/bar").trim());
+      assertEquals("default1", getContent(baseUrl + "test/foo/1").trim());
+      assertEquals("default2", getContent(baseUrl + "test/foo/bar/2").trim());
+      assertEquals(404, getResponseCode(baseUrl + "test/goo"));
+      assertEquals(200, getResponseCode(baseUrl + "ws/v1/test"));
+      assertTrue(getContent(baseUrl + "ws/v1/test").contains("myInfo"));
     } finally {
       app.stop();
     }
   }
 
-  @Test public void testEncodedUrl() throws Exception {
+  @Test
+  void testEncodedUrl() throws Exception {
     WebApp app =
         WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
           @Override
@@ -292,7 +304,8 @@ public void setup() {
     }
   }
 
-  @Test public void testRobotsText() throws Exception {
+  @Test
+  void testRobotsText() throws Exception {
     WebApp app =
         WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
           @Override
@@ -319,18 +332,20 @@ public void setup() {
 
   // This is to test the GuiceFilter should only be applied to webAppContext,
   // not to logContext;
-  @Test public void testYARNWebAppContext() throws Exception {
+  @Test
+  void testYARNWebAppContext() throws Exception {
     // setting up the log context
     System.setProperty("hadoop.log.dir", "/Not/Existing/dir");
     WebApp app = WebApps.$for("test", this).start(new WebApp() {
-      @Override public void setup() {
+      @Override
+      public void setup() {
         route("/", FooController.class);
       }
     });
     String baseUrl = baseUrl(app);
     try {
       // Not able to access a non-existing dir, should not redirect to foo.
-      assertEquals(404, getResponseCode(baseUrl +"logs"));
+      assertEquals(404, getResponseCode(baseUrl + "logs"));
       // should be able to redirect to foo.
       assertEquals("foo", getContent(baseUrl).trim());
     } finally {
@@ -345,7 +360,7 @@ private static void stopWebApp(WebApp app) {
   }
 
   @Test
-  public void testPortRanges() throws Exception {
+  void testPortRanges() throws Exception {
     WebApp app = WebApps.$for("test", this).start();
     String baseUrl = baseUrl(app);
     WebApp app1 = null;
@@ -355,7 +370,7 @@ public void testPortRanges() throws Exception {
     WebApp app5 = null;
     try {
       int port =  ServerSocketUtil.waitForPort(48000, 60);
-      assertEquals("foo", getContent(baseUrl +"test/foo").trim());
+      assertEquals("foo", getContent(baseUrl + "test/foo").trim());
       app1 = WebApps.$for("test", this).at(port).start();
       assertEquals(port, app1.getListenerAddress().getPort());
       app2 = WebApps.$for("test", this).at("0.0.0.0", port, true).start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
index b4832eff42002..f803c5313ca57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.util.ArrayList;
 import java.util.List;
 import javax.ws.rs.core.Response.StatusType;
@@ -30,6 +27,9 @@
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 public class WebServicesTestUtils {
   public static long getXmlLong(Element element, String name) {
     String val = getXmlString(element, name);
@@ -122,20 +122,18 @@ public static String getXmlAttrString(Element element, String name) {
 
   public static void checkStringMatch(String print, String expected, String got) {
     assertTrue(
-        print + " doesn't match, got: " + got + " expected: " + expected,
-        got.matches(expected));
+        got.matches(expected),
+        print + " doesn't match, got: " + got + " expected: " + expected);
   }
 
   public static void checkStringContains(String print, String expected, String got) {
     assertTrue(
-        print + " doesn't contain expected string, got: " + got + " expected: " + expected,
-        got.contains(expected));
+        got.contains(expected),
+        print + " doesn't contain expected string, got: " + got + " expected: " + expected);
   }
 
   public static void checkStringEqual(String print, String expected, String got) {
-    assertTrue(
-        print + " is not equal, got: " + got + " expected: " + expected,
-        got.equals(expected));
+    assertEquals(got, expected);
   }
 
   public static void assertResponseStatusCode(StatusType expected,
@@ -145,6 +143,6 @@ public static void assertResponseStatusCode(StatusType expected,
 
   public static void assertResponseStatusCode(String errmsg,
       StatusType expected, StatusType actual) {
-    assertEquals(errmsg, expected.getStatusCode(), actual.getStatusCode());
+    assertEquals(expected.getStatusCode(), actual.getStatusCode(), errmsg);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamlet.java
index 275b64cb79d61..6aa499fad4f7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamlet.java
@@ -17,19 +17,27 @@
  */
 package org.apache.hadoop.yarn.webapp.hamlet2;
 
-import java.util.EnumSet;
 import java.io.PrintWriter;
-import org.junit.Test;
+import java.util.EnumSet;
+
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.webapp.SubView;
 
-import static org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.*;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.LinkType;
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.Media;
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.TABLE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class TestHamlet {
 
-  @Test public void testHamlet() {
+  @Test
+  void testHamlet() {
     Hamlet h = newHamlet().
         title("test").
         h1("heading 1").
@@ -69,7 +77,8 @@ public class TestHamlet {
     verify(out, never()).print("</p>");
   }
 
-  @Test public void testTable() {
+  @Test
+  void testTable() {
     Hamlet h = newHamlet().
         title("test table").
         link("style.css");
@@ -90,7 +99,8 @@ public class TestHamlet {
     verify(out, atLeast(1)).print("</tr>");
   }
 
-  @Test public void testEnumAttrs() {
+  @Test
+  void testEnumAttrs() {
     Hamlet h = newHamlet().
         meta_http("Content-type", "text/html; charset=utf-8").
         title("test enum attrs").
@@ -109,7 +119,8 @@ public class TestHamlet {
     verify(out).print(" rel=\"start index\"");
   }
 
-  @Test public void testScriptStyle() {
+  @Test
+  void testScriptStyle() {
     Hamlet h = newHamlet().
         script("a.js").script("b.js").
         style("h1 { font-size: 1.2em }");
@@ -121,7 +132,8 @@ public class TestHamlet {
     verify(out).print(" type=\"text/css\"");
   }
 
-  @Test public void testPreformatted() {
+  @Test
+  void testPreformatted() {
     Hamlet h = newHamlet().
         div().
         i("inline before pre").
@@ -144,7 +156,8 @@ static class TestView2 implements SubView {
     @Override public void renderPartial() {}
   }
 
-  @Test public void testSubViews() {
+  @Test
+  void testSubViews() {
     Hamlet h = newHamlet().
         title("test sub-views").
         div("#view1").__(TestView1.class).__().
@@ -153,8 +166,8 @@ static class TestView2 implements SubView {
     PrintWriter out = h.getWriter();
     out.flush();
     assertEquals(0, h.nestLevel);
-    verify(out).print("["+ TestView1.class.getName() +"]");
-    verify(out).print("["+ TestView2.class.getName() +"]");
+    verify(out).print("[" + TestView1.class.getName() + "]");
+    verify(out).print("[" + TestView2.class.getName() + "]");
   }
 
   static Hamlet newHamlet() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamletImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamletImpl.java
index 3b19aa30c3d11..6a3e90b568930 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamletImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestHamletImpl.java
@@ -18,19 +18,29 @@
 package org.apache.hadoop.yarn.webapp.hamlet2;
 
 import java.io.PrintWriter;
-import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
+import org.junit.jupiter.api.Test;
 
-import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.*;
+import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.CoreAttrs;
+import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.H1;
+import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.LINK;
+import org.apache.hadoop.yarn.webapp.hamlet2.HamletSpec.SCRIPT;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 public class TestHamletImpl {
   /**
    * Test the generic implementation methods
    * @see TestHamlet for Hamlet syntax
    */
-  @Test public void testGeneric() {
+  @Test
+  void testGeneric() {
     PrintWriter out = spy(new PrintWriter(System.out));
     HamletImpl hi = new HamletImpl(out, 0, false);
     hi.
@@ -66,7 +76,8 @@ public class TestHamletImpl {
     verify(out, never()).print("</sub4>");
   }
 
-  @Test public void testSetSelector() {
+  @Test
+  void testSetSelector() {
     CoreAttrs e = mock(CoreAttrs.class);
     HamletImpl.setSelector(e, "#id.class");
 
@@ -81,7 +92,8 @@ public class TestHamletImpl {
     verify(t).__("heading");
   }
 
-  @Test public void testSetLinkHref() {
+  @Test
+  void testSetLinkHref() {
     LINK link = mock(LINK.class);
     HamletImpl.setLinkHref(link, "uri");
     HamletImpl.setLinkHref(link, "style.css");
@@ -93,7 +105,8 @@ public class TestHamletImpl {
     verifyNoMoreInteractions(link);
   }
 
-  @Test public void testSetScriptSrc() {
+  @Test
+  void testSetScriptSrc() {
     SCRIPT script = mock(SCRIPT.class);
     HamletImpl.setScriptSrc(script, "uri");
     HamletImpl.setScriptSrc(script, "script.js");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestParseSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestParseSelector.java
index e2141e6942c09..a340f1491f352 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestParseSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestParseSelector.java
@@ -17,41 +17,51 @@
  */
 package org.apache.hadoop.yarn.webapp.hamlet2;
 
-import org.junit.Test;
-
-import static org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.webapp.WebAppException;
 
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl.S_CLASS;
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl.S_ID;
+import static org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl.parseSelector;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
 public class TestParseSelector {
 
-  @Test public void testNormal() {
+  @Test
+  void testNormal() {
     String[] res = parseSelector("#id.class");
     assertEquals("id", res[S_ID]);
     assertEquals("class", res[S_CLASS]);
   }
 
-  @Test public void testMultiClass() {
+  @Test
+  void testMultiClass() {
     String[] res = parseSelector("#id.class1.class2");
     assertEquals("id", res[S_ID]);
     assertEquals("class1 class2", res[S_CLASS]);
   }
 
-  @Test public void testMissingId() {
+  @Test
+  void testMissingId() {
     String[] res = parseSelector(".class");
     assertNull(res[S_ID]);
     assertEquals("class", res[S_CLASS]);
   }
 
-  @Test public void testMissingClass() {
+  @Test
+  void testMissingClass() {
     String[] res = parseSelector("#id");
     assertEquals("id", res[S_ID]);
     assertNull(res[S_CLASS]);
   }
 
-  @Test(expected=WebAppException.class) public void testMissingAll() {
-    parseSelector("");
+  @Test
+  void testMissingAll() {
+    assertThrows(WebAppException.class, () -> {
+      parseSelector("");
+    });
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java
index 57e6c81659b97..e171ee1fc5684 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlockForTest.java
@@ -20,7 +20,6 @@
 
 import java.util.HashMap;
 import java.util.Map;
-
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/TestWebAppTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/TestWebAppTests.java
index e2f2bfa1b2583..aef8ea463e645 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/TestWebAppTests.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/TestWebAppTests.java
@@ -18,23 +18,26 @@
 
 package org.apache.hadoop.yarn.webapp.test;
 
-import com.google.inject.AbstractModule;
-import com.google.inject.Injector;
-import com.google.inject.servlet.RequestScoped;
 import java.io.PrintWriter;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.junit.Test;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
 
-import org.slf4j.LoggerFactory;
+import com.google.inject.AbstractModule;
+import com.google.inject.Injector;
+import com.google.inject.servlet.RequestScoped;
+import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.Mockito.verify;
 
 public class TestWebAppTests {
   static final Logger LOG = LoggerFactory.getLogger(TestWebAppTests.class);
 
-  @Test public void testInstances() throws Exception {
+  @Test
+  void testInstances() throws Exception {
     Injector injector = WebAppTests.createMockInjector(this);
     HttpServletRequest req = injector.getInstance(HttpServletRequest.class);
     HttpServletResponse res = injector.getInstance(HttpServletResponse.class);
@@ -61,24 +64,27 @@ static class Bar implements Foo {
   static class FooBar extends Bar {
   }
 
-  @Test public void testCreateInjector() throws Exception {
+  @Test
+  void testCreateInjector() throws Exception {
     Bar bar = new Bar();
     Injector injector = WebAppTests.createMockInjector(Foo.class, bar);
     logInstances(injector.getInstance(HttpServletRequest.class),
-                 injector.getInstance(HttpServletResponse.class),
-                 injector.getInstance(HttpServletResponse.class).getWriter());
+        injector.getInstance(HttpServletResponse.class),
+        injector.getInstance(HttpServletResponse.class).getWriter());
     assertSame(bar, injector.getInstance(Foo.class));
   }
 
-  @Test public void testCreateInjector2() {
+  @Test
+  void testCreateInjector2() {
     final FooBar foobar = new FooBar();
     Bar bar = new Bar();
     Injector injector = WebAppTests.createMockInjector(Foo.class, bar,
         new AbstractModule() {
-      @Override protected void configure() {
-        bind(Bar.class).toInstance(foobar);
-      }
-    });
+          @Override
+          protected void configure() {
+            bind(Bar.class).toInstance(foobar);
+          }
+        });
     assertNotSame(bar, injector.getInstance(Bar.class));
     assertSame(foobar, injector.getInstance(Bar.class));
   }
@@ -87,11 +93,12 @@ static class FooBar extends Bar {
   static class ScopeTest {
   }
 
-  @Test public void testRequestScope() {
+  @Test
+  void testRequestScope() {
     Injector injector = WebAppTests.createMockInjector(this);
 
     assertSame(injector.getInstance(ScopeTest.class),
-               injector.getInstance(ScopeTest.class));
+        injector.getInstance(ScopeTest.class));
   }
 
   private void logInstances(HttpServletRequest req, HttpServletResponse res,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java
index da009d4d28e21..41c56ce1b61a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/test/WebAppTests.java
@@ -18,30 +18,29 @@
 
 package org.apache.hadoop.yarn.webapp.test;
 
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.SubView;
-import org.apache.hadoop.yarn.webapp.View;
-import org.apache.hadoop.yarn.webapp.WebAppException;
-
+import java.io.IOException;
+import java.io.PrintWriter;
 import java.lang.reflect.Method;
 import java.util.Map;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
-import com.google.inject.Module;
-import com.google.inject.Scopes;
-import com.google.inject.servlet.RequestScoped;
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
+import com.google.inject.Module;
 import com.google.inject.Provides;
+import com.google.inject.Scopes;
+import com.google.inject.servlet.RequestScoped;
 
-import java.io.IOException;
-import java.io.PrintWriter;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.servlet.http.HttpServletRequest;
-
+import org.apache.hadoop.yarn.webapp.Controller;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.WebAppException;
 
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class WebAppTests {
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index d6f78b18c1bc2..2d9c39aba8b8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -18,14 +18,18 @@
 
 package org.apache.hadoop.yarn.webapp.util;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.UnknownHostException;
 import java.util.HashMap;
 import java.util.Map;
+import javax.servlet.http.HttpServletRequest;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.http.HttpServer2;
@@ -35,13 +39,10 @@
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
 
-import javax.servlet.http.HttpServletRequest;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestWebAppUtils {
   private static final String RM1_NODE_ID = "rm1";
@@ -53,7 +54,7 @@ public class TestWebAppUtils {
   private static final String anyIpAddress = "1.2.3.4";
   private static Map<String, String> savedStaticResolution = new HashMap<>();
 
-  @BeforeClass
+  @BeforeAll
   public static void initializeDummyHostnameResolution() throws Exception {
     String previousIpAddress;
     for (String hostName : dummyHostNames) {
@@ -64,7 +65,7 @@ public static void initializeDummyHostnameResolution() throws Exception {
     }
   }
 
-  @AfterClass
+  @AfterAll
   public static void restoreDummyHostnameResolution() throws Exception {
     for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
       NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
@@ -72,7 +73,7 @@ public static void restoreDummyHostnameResolution() throws Exception {
   }
 
   @Test
-  public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
+  void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
     Configuration configuration = new Configuration();
     final String rmAddress = "host1:8088";
     configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
@@ -84,30 +85,32 @@ public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
     configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
 
     String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
-    Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
+    assertEquals(rm1Address, rmRemoteUrl,
+        "ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address");
 
     String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
-    Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
+    assertEquals(rmAddress, rmLocalUrl,
+        "ResolvedRMWebAppUrl should resolve to the default RM webapp address");
   }
 
   @Test
-  public void testGetPassword() throws Exception {
+  void testGetPassword() throws Exception {
     Configuration conf = provisionCredentialsForSSL();
 
     // use WebAppUtils as would be used by loadSslConfiguration
-    Assert.assertEquals("keypass",
+    assertEquals("keypass",
         WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_KEY_PASSWORD_KEY));
-    Assert.assertEquals("storepass",
+    assertEquals("storepass",
         WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY));
-    Assert.assertEquals("trustpass",
+    assertEquals("trustpass",
         WebAppUtils.getPassword(conf, WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY));
 
     // let's make sure that a password that doesn't exist returns null
-    Assert.assertEquals(null, WebAppUtils.getPassword(conf,"invalid-alias"));
+    assertNull(WebAppUtils.getPassword(conf, "invalid-alias"));
   }
 
   @Test
-  public void testLoadSslConfiguration() throws Exception {
+  void testLoadSslConfiguration() throws Exception {
     Configuration conf = provisionCredentialsForSSL();
     TestBuilder builder = (TestBuilder) new TestBuilder();
 
@@ -116,12 +119,12 @@ public void testLoadSslConfiguration() throws Exception {
 
     String keypass = "keypass";
     String storepass = "storepass";
-    String trustpass = "trustpass";    
+    String trustpass = "trustpass";
 
     // make sure we get the right passwords in the builder
-    assertEquals(keypass, ((TestBuilder)builder).keypass);
-    assertEquals(storepass, ((TestBuilder)builder).keystorePassword);
-    assertEquals(trustpass, ((TestBuilder)builder).truststorePassword);
+    assertEquals(keypass, ((TestBuilder) builder).keypass);
+    assertEquals(storepass, ((TestBuilder) builder).keystorePassword);
+    assertEquals(trustpass, ((TestBuilder) builder).truststorePassword);
   }
 
   protected Configuration provisionCredentialsForSSL() throws IOException,
@@ -145,11 +148,11 @@ protected Configuration provisionCredentialsForSSL() throws IOException,
     char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
 
     // ensure that we get nulls when the key isn't there
-    assertEquals(null, provider.getCredentialEntry(
+    assertNull(provider.getCredentialEntry(
         WebAppUtils.WEB_APP_KEY_PASSWORD_KEY));
-    assertEquals(null, provider.getCredentialEntry(
+    assertNull(provider.getCredentialEntry(
         WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY));
-    assertEquals(null, provider.getCredentialEntry(
+    assertNull(provider.getCredentialEntry(
         WebAppUtils.WEB_APP_TRUSTSTORE_PASSWORD_KEY));
 
     // create new aliases
@@ -180,7 +183,7 @@ protected Configuration provisionCredentialsForSSL() throws IOException,
   }
 
   @Test
-  public void testAppendQueryParams() throws Exception {
+  void testAppendQueryParams() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
     String targetUri = "/test/path";
     Mockito.when(request.getCharacterEncoding()).thenReturn(null);
@@ -194,12 +197,12 @@ public void testAppendQueryParams() throws Exception {
     for (Map.Entry<String, String> entry : paramResultMap.entrySet()) {
       Mockito.when(request.getQueryString()).thenReturn(entry.getKey());
       String uri = WebAppUtils.appendQueryParams(request, targetUri);
-      Assert.assertEquals(entry.getValue(), uri);
+      assertEquals(entry.getValue(), uri);
     }
   }
 
   @Test
-  public void testGetHtmlEscapedURIWithQueryString() throws Exception {
+  void testGetHtmlEscapedURIWithQueryString() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
     String targetUri = "/test/path";
     Mockito.when(request.getCharacterEncoding()).thenReturn(null);
@@ -214,7 +217,7 @@ public void testGetHtmlEscapedURIWithQueryString() throws Exception {
     for (Map.Entry<String, String> entry : paramResultMap.entrySet()) {
       Mockito.when(request.getQueryString()).thenReturn(entry.getKey());
       String uri = WebAppUtils.getHtmlEscapedURIWithQueryString(request);
-      Assert.assertEquals(entry.getValue(), uri);
+      assertEquals(entry.getValue(), uri);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java
index 99a5783628e9a..b51dcf88bcbf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebServiceClient.java
@@ -22,6 +22,8 @@
 import java.net.URI;
 import java.net.URL;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.http.HttpServer2;
@@ -29,8 +31,9 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 public class TestWebServiceClient {
 
@@ -43,17 +46,17 @@ public class TestWebServiceClient {
   static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
 
   @Test
-  public void testGetWebServiceClient() throws Exception {
+  void testGetWebServiceClient() throws Exception {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
     WebServiceClient.initialize(conf);
     WebServiceClient client = WebServiceClient.getWebServiceClient();
-    Assert.assertNotNull(client.getSSLFactory());
+    assertNotNull(client.getSSLFactory());
     WebServiceClient.destroy();
   }
 
   @Test
-  public void testCreateClient() throws Exception {
+  void testCreateClient() throws Exception {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
     File base = new File(BASEDIR);
@@ -91,7 +94,7 @@ public void testCreateClient() throws Exception {
     WebServiceClient client = WebServiceClient.getWebServiceClient();
     HttpURLConnection conn = client.getHttpURLConnectionFactory()
         .getHttpURLConnection(u);
-    Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+    assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
     WebServiceClient.destroy();
     server.stop();
     FileUtil.fullyDelete(new File(BASEDIR));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestCommonViews.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestCommonViews.java
index b533bce846dbe..028bc6ad18c20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestCommonViews.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestCommonViews.java
@@ -19,37 +19,36 @@
 package org.apache.hadoop.yarn.webapp.view;
 
 import com.google.inject.Injector;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.webapp.ResponseInfo;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.apache.hadoop.yarn.webapp.view.ErrorPage;
-import org.apache.hadoop.yarn.webapp.view.FooterBlock;
-import org.apache.hadoop.yarn.webapp.view.HeaderBlock;
-import org.apache.hadoop.yarn.webapp.view.JQueryUI;
-
-import org.junit.Test;
-import static org.mockito.Mockito.*;
 
 public class TestCommonViews {
 
-  @Test public void testErrorPage() {
+  @Test
+  void testErrorPage() {
     Injector injector = WebAppTests.testPage(ErrorPage.class);
 
   }
 
-  @Test public void testHeaderBlock() {
+  @Test
+  void testHeaderBlock() {
     WebAppTests.testBlock(HeaderBlock.class);
   }
 
-  @Test public void testFooterBlock() {
+  @Test
+  void testFooterBlock() {
     WebAppTests.testBlock(FooterBlock.class);
   }
 
-  @Test public void testJQueryUI() {
+  @Test
+  void testJQueryUI() {
     WebAppTests.testBlock(JQueryUI.class);
   }
 
-  @Test public void testInfoBlock() {
+  @Test
+  void testInfoBlock() {
     Injector injector = WebAppTests.createMockInjector(this);
     ResponseInfo info = injector.getInstance(ResponseInfo.class);
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlBlock.java
index e510dd57ba85f..de9bd454de54a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlBlock.java
@@ -18,15 +18,16 @@
 
 package org.apache.hadoop.yarn.webapp.view;
 
-import com.google.inject.Injector;
-
 import java.io.PrintWriter;
 
+import com.google.inject.Injector;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.webapp.WebAppException;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
 
-import org.junit.Test;
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.verify;
 
 public class TestHtmlBlock {
   public static class TestBlock extends HtmlBlock {
@@ -54,7 +55,8 @@ public void render(Page.HTML<__> html) {
     }
   }
 
-  @Test public void testUsual() {
+  @Test
+  void testUsual() {
     Injector injector = WebAppTests.testBlock(TestBlock.class);
     PrintWriter out = injector.getInstance(PrintWriter.class);
 
@@ -62,11 +64,17 @@ public void render(Page.HTML<__> html) {
     verify(out).print("test note");
   }
 
-  @Test(expected=WebAppException.class) public void testShortBlock() {
-    WebAppTests.testBlock(ShortBlock.class);
+  @Test
+  void testShortBlock() {
+    assertThrows(WebAppException.class, () -> {
+      WebAppTests.testBlock(ShortBlock.class);
+    });
   }
 
-  @Test(expected=WebAppException.class) public void testShortPage() {
-    WebAppTests.testPage(ShortPage.class);
+  @Test
+  void testShortPage() {
+    assertThrows(WebAppException.class, () -> {
+      WebAppTests.testPage(ShortPage.class);
+    });
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java
index beed31fb478bb..ce8bbc9d85162 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java
@@ -18,16 +18,17 @@
 
 package org.apache.hadoop.yarn.webapp.view;
 
-import com.google.inject.Injector;
-
 import java.io.PrintWriter;
 
+import com.google.inject.Injector;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.webapp.MimeType;
 import org.apache.hadoop.yarn.webapp.WebAppException;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.junit.Test;
 
-import static org.mockito.Mockito.*;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.verify;
 
 public class TestHtmlPage {
   
@@ -49,7 +50,8 @@ public void render(Page.HTML<__> html) {
     }
   }
 
-  @Test public void testUsual() {
+  @Test
+  void testUsual() {
     Injector injector = WebAppTests.testPage(TestView.class);
     PrintWriter out = injector.getInstance(PrintWriter.class);
 
@@ -64,7 +66,10 @@ public void render(Page.HTML<__> html) {
     verify(out).print("test note");
   }
 
-  @Test(expected=WebAppException.class) public void testShort() {
-    WebAppTests.testPage(ShortView.class);
+  @Test
+  void testShort() {
+    assertThrows(WebAppException.class, () -> {
+      WebAppTests.testPage(ShortView.class);
+    });
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java
index 751aa2cabe4c9..1f0a4a6467eb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java
@@ -21,13 +21,15 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 import org.apache.hadoop.yarn.webapp.ResponseInfo;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestInfoBlock {
 
@@ -86,30 +88,32 @@ public MultilineInfoBlock() {
     }
   }
 
-  @Before
+  @BeforeEach
   public void setup() {
     sw = new StringWriter();
     pw = new PrintWriter(sw);
   }
 
-  @Test(timeout=60000L)
-  public void testMultilineInfoBlock() throws Exception{
+  @Test
+  @Timeout(60000L)
+  void testMultilineInfoBlock() throws Exception {
 
     WebAppTests.testBlock(MultilineInfoBlock.class);
     TestInfoBlock.pw.flush();
     String output = TestInfoBlock.sw.toString().replaceAll(" +", " ");
     String expectedMultilineData1 = String.format("<tr class=\"odd\">%n"
-      + " <th>%n Multiple_line_value%n </th>%n"
-      + " <td>%n This is one line.%n </td>%n");
+        + " <th>%n Multiple_line_value%n </th>%n"
+        + " <td>%n This is one line.%n </td>%n");
     String expectedMultilineData2 = String.format("<tr class=\"even\">%n"
-      + " <th>%n Multiple_line_value%n </th>%n <td>%n <div>%n"
-      + " This is first line.%n </div>%n <div>%n"
-      + " This is second line.%n </div>%n");
+        + " <th>%n Multiple_line_value%n </th>%n <td>%n <div>%n"
+        + " This is first line.%n </div>%n <div>%n"
+        + " This is second line.%n </div>%n");
     assertTrue(output.contains(expectedMultilineData1) && output.contains(expectedMultilineData2));
   }
-  
-  @Test(timeout=60000L)
-  public void testJavaScriptInfoBlock() throws Exception{
+
+  @Test
+  @Timeout(60000L)
+  void testJavaScriptInfoBlock() throws Exception {
     WebAppTests.testBlock(JavaScriptInfoBlock.class);
     TestInfoBlock.pw.flush();
     String output = TestInfoBlock.sw.toString();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnCssPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnCssPage.java
index 20df4093ad355..a6ec759e8939f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnCssPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnCssPage.java
@@ -18,11 +18,12 @@
 
 package org.apache.hadoop.yarn.webapp.view;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.webapp.Controller;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.junit.Test;
 
 public class TestTwoColumnCssPage {
 
@@ -57,7 +58,8 @@ public void render(Page.HTML<__> html) {
     }
   }
 
-  @Test public void shouldNotThrow() {
+  @Test
+  void shouldNotThrow() {
     WebAppTests.testPage(TwoColumnCssLayout.class);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnLayout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnLayout.java
index 52ae6ae9bd53f..d2e4176d7726e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnLayout.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestTwoColumnLayout.java
@@ -18,10 +18,11 @@
 
 package org.apache.hadoop.yarn.webapp.view;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.webapp.Controller;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.junit.Test;
 
 public class TestTwoColumnLayout {
 
@@ -34,7 +35,8 @@ public void index() {
     }
   }
 
-  @Test public void shouldNotThrow() {
+  @Test
+  void shouldNotThrow() {
     WebAppTests.testPage(TwoColumnLayout.class);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
index 62707eadb9d22..953791e01e9ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml
@@ -27,7 +27,6 @@
 
     <properties>
         <grpc.version>1.26.0</grpc.version>
-        <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
     </properties>
 
     <dependencies>
@@ -92,6 +91,16 @@
             <artifactId>assertj-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
@@ -149,13 +158,6 @@
     </dependencies>
 
     <build>
-        <extensions>
-            <extension>
-                <groupId>kr.motd.maven</groupId>
-                <artifactId>os-maven-plugin</artifactId>
-                <version>${os-maven-plugin.version}</version>
-            </extension>
-        </extensions>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
index 206fb1433a1d6..cc14711397366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java
@@ -42,10 +42,9 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 import java.io.File;
 import java.io.IOException;
@@ -53,6 +52,9 @@
 import java.net.ServerSocket;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER;
 import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM;
 
@@ -64,7 +66,7 @@ public class TestCsiAdaptorService {
   private static File testRoot = null;
   private static String domainSocket = null;
 
-  @BeforeClass
+  @BeforeAll
   public static void setUp() throws IOException {
     testRoot = GenericTestUtils.getTestDir("csi-test");
     File socketPath = new File(testRoot, "csi.sock");
@@ -72,7 +74,7 @@ public static void setUp() throws IOException {
     domainSocket = "unix://" + socketPath.getAbsolutePath();
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws IOException {
     if (testRoot != null) {
       FileUtils.deleteDirectory(testRoot);
@@ -113,7 +115,7 @@ default NodeUnpublishVolumeResponse nodeUnpublishVolume(
   }
 
   @Test
-  public void testValidateVolume() throws IOException, YarnException {
+  void testValidateVolume() throws IOException, YarnException {
     ServerSocket ss = new ServerSocket(0);
     ss.close();
     InetSocketAddress address = new InetSocketAddress(ss.getLocalPort());
@@ -145,20 +147,20 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
           ValidateVolumeCapabilitiesRequest request) throws YarnException,
           IOException {
         // validate we get all info from the request
-        Assert.assertEquals("volume-id-0000123", request.getVolumeId());
-        Assert.assertEquals(1, request.getVolumeCapabilities().size());
-        Assert.assertEquals(Csi.VolumeCapability.AccessMode
+        assertEquals("volume-id-0000123", request.getVolumeId());
+        assertEquals(1, request.getVolumeCapabilities().size());
+        assertEquals(Csi.VolumeCapability.AccessMode
                 .newBuilder().setModeValue(5).build().getMode().name(),
             request.getVolumeCapabilities().get(0).getAccessMode().name());
-        Assert.assertEquals(2, request.getVolumeCapabilities().get(0)
+        assertEquals(2, request.getVolumeCapabilities().get(0)
             .getMountFlags().size());
-        Assert.assertTrue(request.getVolumeCapabilities().get(0)
+        assertTrue(request.getVolumeCapabilities().get(0)
             .getMountFlags().contains("mountFlag1"));
-        Assert.assertTrue(request.getVolumeCapabilities().get(0)
+        assertTrue(request.getVolumeCapabilities().get(0)
             .getMountFlags().contains("mountFlag2"));
-        Assert.assertEquals(2, request.getVolumeAttributes().size());
-        Assert.assertEquals("v1", request.getVolumeAttributes().get("k1"));
-        Assert.assertEquals("v2", request.getVolumeAttributes().get("k2"));
+        assertEquals(2, request.getVolumeAttributes().size());
+        assertEquals("v1", request.getVolumeAttributes().get("k1"));
+        assertEquals("v2", request.getVolumeAttributes().get("k2"));
         // return a fake result
         return ValidateVolumeCapabilitiesResponse
             .newInstance(false, "this is a test");
@@ -178,22 +180,22 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
                   ImmutableList.of(
                       new ValidateVolumeCapabilitiesRequest
                           .VolumeCapability(
-                              MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
+                          MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
                           ImmutableList.of("mountFlag1", "mountFlag2"))),
-              ImmutableMap.of("k1", "v1", "k2", "v2"));
+                  ImmutableMap.of("k1", "v1", "k2", "v2"));
 
       ValidateVolumeCapabilitiesResponse response = client
           .validateVolumeCapacity(request);
 
-      Assert.assertEquals(false, response.isSupported());
-      Assert.assertEquals("this is a test", response.getResponseMessage());
+      assertEquals(false, response.isSupported());
+      assertEquals("this is a test", response.getResponseMessage());
     } finally {
       service.stop();
     }
   }
 
   @Test
-  public void testValidateVolumeWithNMProxy() throws Exception {
+  void testValidateVolumeWithNMProxy() throws Exception {
     ServerSocket ss = new ServerSocket(0);
     ss.close();
     InetSocketAddress address = new InetSocketAddress(ss.getLocalPort());
@@ -225,21 +227,21 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
           ValidateVolumeCapabilitiesRequest request)
           throws YarnException, IOException {
         // validate we get all info from the request
-        Assert.assertEquals("volume-id-0000123", request.getVolumeId());
-        Assert.assertEquals(1, request.getVolumeCapabilities().size());
-        Assert.assertEquals(
+        assertEquals("volume-id-0000123", request.getVolumeId());
+        assertEquals(1, request.getVolumeCapabilities().size());
+        assertEquals(
             Csi.VolumeCapability.AccessMode.newBuilder().setModeValue(5)
                 .build().getMode().name(),
             request.getVolumeCapabilities().get(0).getAccessMode().name());
-        Assert.assertEquals(2,
+        assertEquals(2,
             request.getVolumeCapabilities().get(0).getMountFlags().size());
-        Assert.assertTrue(request.getVolumeCapabilities().get(0).getMountFlags()
+        assertTrue(request.getVolumeCapabilities().get(0).getMountFlags()
             .contains("mountFlag1"));
-        Assert.assertTrue(request.getVolumeCapabilities().get(0).getMountFlags()
+        assertTrue(request.getVolumeCapabilities().get(0).getMountFlags()
             .contains("mountFlag2"));
-        Assert.assertEquals(2, request.getVolumeAttributes().size());
-        Assert.assertEquals("v1", request.getVolumeAttributes().get("k1"));
-        Assert.assertEquals("v2", request.getVolumeAttributes().get("k2"));
+        assertEquals(2, request.getVolumeAttributes().size());
+        assertEquals("v1", request.getVolumeAttributes().get("k1"));
+        assertEquals("v2", request.getVolumeAttributes().get("k2"));
         // return a fake result
         return ValidateVolumeCapabilitiesResponse
             .newInstance(false, "this is a test");
@@ -261,50 +263,59 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity(
             .newInstance("volume-id-0000123",
                 ImmutableList.of(new ValidateVolumeCapabilitiesRequest
                     .VolumeCapability(
-                        MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
+                    MULTI_NODE_MULTI_WRITER, FILE_SYSTEM,
                     ImmutableList.of("mountFlag1", "mountFlag2"))),
                 ImmutableMap.of("k1", "v1", "k2", "v2"));
 
     ValidateVolumeCapabilitiesResponse response = adaptorClient
         .validateVolumeCapacity(request);
-    Assert.assertEquals(false, response.isSupported());
-    Assert.assertEquals("this is a test", response.getResponseMessage());
+    assertEquals(false, response.isSupported());
+    assertEquals("this is a test", response.getResponseMessage());
 
     service.stop();
   }
 
-  @Test (expected = ServiceStateException.class)
-  public void testMissingConfiguration() {
-    Configuration conf = new Configuration();
-    CsiAdaptorProtocolService service =
-        new CsiAdaptorProtocolService(new FakeCsiAdaptor() {});
-    service.init(conf);
+  @Test
+  void testMissingConfiguration() {
+    assertThrows(ServiceStateException.class, () -> {
+      Configuration conf = new Configuration();
+      CsiAdaptorProtocolService service =
+          new CsiAdaptorProtocolService(new FakeCsiAdaptor() {
+          });
+      service.init(conf);
+    });
   }
 
-  @Test (expected = ServiceStateException.class)
-  public void testInvalidServicePort() {
-    Configuration conf = new Configuration();
-    conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
-        + "test-driver-0001.address",
-        "0.0.0.0:-100"); // this is an invalid address
-    CsiAdaptorProtocolService service =
-        new CsiAdaptorProtocolService(new FakeCsiAdaptor() {});
-    service.init(conf);
+  @Test
+  void testInvalidServicePort() {
+    assertThrows(ServiceStateException.class, () -> {
+      Configuration conf = new Configuration();
+      conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
+          + "test-driver-0001.address",
+          "0.0.0.0:-100"); // this is an invalid address
+      CsiAdaptorProtocolService service =
+          new CsiAdaptorProtocolService(new FakeCsiAdaptor() {
+          });
+      service.init(conf);
+    });
   }
 
-  @Test (expected = ServiceStateException.class)
-  public void testInvalidHost() {
-    Configuration conf = new Configuration();
-    conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
-            + "test-driver-0001.address",
-        "192.0.1:8999"); // this is an invalid ip address
-    CsiAdaptorProtocolService service =
-        new CsiAdaptorProtocolService(new FakeCsiAdaptor() {});
-    service.init(conf);
+  @Test
+  void testInvalidHost() {
+    assertThrows(ServiceStateException.class, () -> {
+      Configuration conf = new Configuration();
+      conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
+          + "test-driver-0001.address",
+          "192.0.1:8999"); // this is an invalid ip address
+      CsiAdaptorProtocolService service =
+          new CsiAdaptorProtocolService(new FakeCsiAdaptor() {
+          });
+      service.init(conf);
+    });
   }
 
   @Test
-  public void testCustomizedAdaptor() throws IOException, YarnException {
+  void testCustomizedAdaptor() throws IOException, YarnException {
     ServerSocket ss = new ServerSocket(0);
     ss.close();
     InetSocketAddress address = new InetSocketAddress(ss.getLocalPort());
@@ -349,15 +360,15 @@ public void testCustomizedAdaptor() throws IOException, YarnException {
 
     ValidateVolumeCapabilitiesResponse response = adaptorClient
         .validateVolumeCapacity(request);
-    Assert.assertEquals(true, response.isSupported());
-    Assert.assertEquals("verified via MockCsiAdaptor",
+    assertEquals(true, response.isSupported());
+    assertEquals("verified via MockCsiAdaptor",
         response.getResponseMessage());
 
     services.stop();
   }
 
   @Test
-  public void testMultipleCsiAdaptors() throws IOException, YarnException {
+  void testMultipleCsiAdaptors() throws IOException, YarnException {
     ServerSocket driver1Addr = new ServerSocket(0);
     ServerSocket driver2Addr = new ServerSocket(0);
 
@@ -374,22 +385,22 @@ public void testMultipleCsiAdaptors() throws IOException, YarnException {
 
     // customized-driver-1
     conf.setSocketAddr(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
-            + "customized-driver-1.address", address1);
+        + "customized-driver-1.address", address1);
     conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
-            + "customized-driver-1.class",
+        + "customized-driver-1.class",
         "org.apache.hadoop.yarn.csi.adaptor.MockCsiAdaptor");
     conf.set(YarnConfiguration.NM_CSI_DRIVER_PREFIX
-            + "customized-driver-1.endpoint",
+        + "customized-driver-1.endpoint",
         "unix:///tmp/customized-driver-1.sock");
 
     // customized-driver-2
     conf.setSocketAddr(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
         + "customized-driver-2.address", address2);
     conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX
-            + "customized-driver-2.class",
+        + "customized-driver-2.class",
         "org.apache.hadoop.yarn.csi.adaptor.MockCsiAdaptor");
     conf.set(YarnConfiguration.NM_CSI_DRIVER_PREFIX
-            + "customized-driver-2.endpoint",
+        + "customized-driver-2.endpoint",
         "unix:///tmp/customized-driver-2.sock");
 
     driver1Addr.close();
@@ -427,8 +438,8 @@ public void testMultipleCsiAdaptors() throws IOException, YarnException {
 
     ValidateVolumeCapabilitiesResponse response = client1
         .validateVolumeCapacity(request);
-    Assert.assertEquals(true, response.isSupported());
-    Assert.assertEquals("verified via MockCsiAdaptor",
+    assertEquals(true, response.isSupported());
+    assertEquals("verified via MockCsiAdaptor",
         response.getResponseMessage());
 
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
index f1734c84b3774..a6591091e5a5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java
@@ -21,8 +21,10 @@
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * Verify the integrity of GetPluginInfoRequest and GetPluginInfoResponse.
@@ -30,37 +32,37 @@
 public class TestGetPluginInfoRequestResponse {
 
   @Test
-  public void testGetPluginInfoRequestPBRecord() {
+  void testGetPluginInfoRequestPBRecord() {
     CsiAdaptorProtos.GetPluginInfoRequest requestProto =
         CsiAdaptorProtos.GetPluginInfoRequest.newBuilder().build();
     GetPluginInfoRequestPBImpl pbImpl =
         new GetPluginInfoRequestPBImpl(requestProto);
-    Assert.assertNotNull(pbImpl);
-    Assert.assertEquals(requestProto, pbImpl.getProto());
+    assertNotNull(pbImpl);
+    assertEquals(requestProto, pbImpl.getProto());
   }
 
   @Test
-  public void testGetPluginInfoResponsePBRecord() {
+  void testGetPluginInfoResponsePBRecord() {
     CsiAdaptorProtos.GetPluginInfoResponse responseProto =
         CsiAdaptorProtos.GetPluginInfoResponse.newBuilder()
-        .setName("test-driver")
-        .setVendorVersion("1.0.1")
-        .build();
+            .setName("test-driver")
+            .setVendorVersion("1.0.1")
+            .build();
 
     GetPluginInfoResponsePBImpl pbImpl =
         new GetPluginInfoResponsePBImpl(responseProto);
-    Assert.assertEquals("test-driver", pbImpl.getDriverName());
-    Assert.assertEquals("1.0.1", pbImpl.getVersion());
-    Assert.assertEquals(responseProto, pbImpl.getProto());
+    assertEquals("test-driver", pbImpl.getDriverName());
+    assertEquals("1.0.1", pbImpl.getVersion());
+    assertEquals(responseProto, pbImpl.getProto());
 
     GetPluginInfoResponse pbImpl2 = GetPluginInfoResponsePBImpl
         .newInstance("test-driver", "1.0.1");
-    Assert.assertEquals("test-driver", pbImpl2.getDriverName());
-    Assert.assertEquals("1.0.1", pbImpl2.getVersion());
+    assertEquals("test-driver", pbImpl2.getDriverName());
+    assertEquals("1.0.1", pbImpl2.getVersion());
 
     CsiAdaptorProtos.GetPluginInfoResponse proto =
         ((GetPluginInfoResponsePBImpl) pbImpl2).getProto();
-    Assert.assertEquals("test-driver", proto.getName());
-    Assert.assertEquals("1.0.1", proto.getVendorVersion());
+    assertEquals("test-driver", proto.getName());
+    assertEquals("1.0.1", proto.getVendorVersion());
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java
index 5f7fa8774a4a8..77c60fd6d1102 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java
@@ -21,8 +21,10 @@
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.AccessMode;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.VolumeType;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 
 /**
  * UT for NodePublishVolumeRequest.
@@ -30,7 +32,7 @@
 public class TestNodePublishVolumeRequest {
 
   @Test
-  public void testPBRecord() {
+  void testPBRecord() {
     CsiAdaptorProtos.VolumeCapability capability =
         CsiAdaptorProtos.VolumeCapability.newBuilder()
             .setAccessMode(AccessMode.MULTI_NODE_READER_ONLY)
@@ -47,9 +49,9 @@ public void testPBRecord() {
 
     NodePublishVolumeRequestPBImpl pbImpl =
         new NodePublishVolumeRequestPBImpl(proto);
-    Assert.assertEquals("test-vol-000001", pbImpl.getVolumeId());
-    Assert.assertEquals("/mnt/data", pbImpl.getTargetPath());
-    Assert.assertEquals("/mnt/staging", pbImpl.getStagingPath());
-    Assert.assertFalse(pbImpl.getReadOnly());
+    assertEquals("test-vol-000001", pbImpl.getVolumeId());
+    assertEquals("/mnt/data", pbImpl.getTargetPath());
+    assertEquals("/mnt/staging", pbImpl.getStagingPath());
+    assertFalse(pbImpl.getReadOnly());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
index 127ddb3383d91..0c1d243a62552 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java
@@ -26,11 +26,11 @@
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.AccessMode;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.VolumeType;
 import org.apache.hadoop.yarn.proto.YarnProtos;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER;
 import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * UT for message exchanges.
@@ -38,7 +38,7 @@
 public class TestValidateVolumeCapabilityRequest {
 
   @Test
-  public void testPBRecord() {
+  void testPBRecord() {
     CsiAdaptorProtos.VolumeCapability vcProto =
         CsiAdaptorProtos.VolumeCapability.newBuilder()
             .setAccessMode(AccessMode.MULTI_NODE_MULTI_WRITER)
@@ -64,22 +64,22 @@ public void testPBRecord() {
     ValidateVolumeCapabilitiesRequestPBImpl request =
         new ValidateVolumeCapabilitiesRequestPBImpl(requestProto);
 
-    Assert.assertEquals("volume-id-0000001", request.getVolumeId());
-    Assert.assertEquals(2, request.getVolumeAttributes().size());
-    Assert.assertEquals("value0", request.getVolumeAttributes().get("attr0"));
-    Assert.assertEquals("value1", request.getVolumeAttributes().get("attr1"));
-    Assert.assertEquals(1, request.getVolumeCapabilities().size());
+    assertEquals("volume-id-0000001", request.getVolumeId());
+    assertEquals(2, request.getVolumeAttributes().size());
+    assertEquals("value0", request.getVolumeAttributes().get("attr0"));
+    assertEquals("value1", request.getVolumeAttributes().get("attr1"));
+    assertEquals(1, request.getVolumeCapabilities().size());
     VolumeCapability vc =
         request.getVolumeCapabilities().get(0);
-    Assert.assertEquals(MULTI_NODE_MULTI_WRITER, vc.getAccessMode());
-    Assert.assertEquals(FILE_SYSTEM, vc.getVolumeType());
-    Assert.assertEquals(2, vc.getMountFlags().size());
+    assertEquals(MULTI_NODE_MULTI_WRITER, vc.getAccessMode());
+    assertEquals(FILE_SYSTEM, vc.getVolumeType());
+    assertEquals(2, vc.getMountFlags().size());
 
-    Assert.assertEquals(requestProto, request.getProto());
+    assertEquals(requestProto, request.getProto());
   }
 
   @Test
-  public void testNewInstance() {
+  void testNewInstance() {
     ValidateVolumeCapabilitiesRequest pbImpl =
         ValidateVolumeCapabilitiesRequestPBImpl
             .newInstance("volume-id-0000123",
@@ -89,25 +89,25 @@ public void testNewInstance() {
                         ImmutableList.of("mountFlag1", "mountFlag2"))),
                 ImmutableMap.of("k1", "v1", "k2", "v2"));
 
-    Assert.assertEquals("volume-id-0000123", pbImpl.getVolumeId());
-    Assert.assertEquals(1, pbImpl.getVolumeCapabilities().size());
-    Assert.assertEquals(FILE_SYSTEM,
+    assertEquals("volume-id-0000123", pbImpl.getVolumeId());
+    assertEquals(1, pbImpl.getVolumeCapabilities().size());
+    assertEquals(FILE_SYSTEM,
         pbImpl.getVolumeCapabilities().get(0).getVolumeType());
-    Assert.assertEquals(MULTI_NODE_MULTI_WRITER,
+    assertEquals(MULTI_NODE_MULTI_WRITER,
         pbImpl.getVolumeCapabilities().get(0).getAccessMode());
-    Assert.assertEquals(2, pbImpl.getVolumeAttributes().size());
+    assertEquals(2, pbImpl.getVolumeAttributes().size());
 
     CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest proto =
         ((ValidateVolumeCapabilitiesRequestPBImpl) pbImpl).getProto();
-    Assert.assertEquals("volume-id-0000123", proto.getVolumeId());
-    Assert.assertEquals(1, proto.getVolumeCapabilitiesCount());
-    Assert.assertEquals(AccessMode.MULTI_NODE_MULTI_WRITER,
+    assertEquals("volume-id-0000123", proto.getVolumeId());
+    assertEquals(1, proto.getVolumeCapabilitiesCount());
+    assertEquals(AccessMode.MULTI_NODE_MULTI_WRITER,
         proto.getVolumeCapabilities(0).getAccessMode());
-    Assert.assertEquals(VolumeType.FILE_SYSTEM,
+    assertEquals(VolumeType.FILE_SYSTEM,
         proto.getVolumeCapabilities(0).getVolumeType());
-    Assert.assertEquals(2, proto.getVolumeCapabilities(0)
+    assertEquals(2, proto.getVolumeCapabilities(0)
         .getMountFlagsCount());
-    Assert.assertEquals(2, proto.getVolumeCapabilities(0)
+    assertEquals(2, proto.getVolumeCapabilities(0)
         .getMountFlagsList().size());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
index 97f116af0d5cc..ee80bf4d0c648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java
@@ -20,8 +20,9 @@
 import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl;
 import org.apache.hadoop.yarn.proto.CsiAdaptorProtos;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * UT for message exchanges.
@@ -29,33 +30,33 @@
 public class TestValidateVolumeCapabilityResponse {
 
   @Test
-  public void testPBRecord() {
+  void testPBRecord() {
     CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto =
         CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.newBuilder()
-        .setSupported(true)
-        .setMessage("capability is supported")
-        .build();
+            .setSupported(true)
+            .setMessage("capability is supported")
+            .build();
 
     ValidateVolumeCapabilitiesResponsePBImpl pbImpl =
         new ValidateVolumeCapabilitiesResponsePBImpl(proto);
 
-    Assert.assertEquals(true, pbImpl.isSupported());
-    Assert.assertEquals("capability is supported", pbImpl.getResponseMessage());
-    Assert.assertEquals(proto, pbImpl.getProto());
+    assertEquals(true, pbImpl.isSupported());
+    assertEquals("capability is supported", pbImpl.getResponseMessage());
+    assertEquals(proto, pbImpl.getProto());
   }
 
   @Test
-  public void testNewInstance() {
+  void testNewInstance() {
     ValidateVolumeCapabilitiesResponse pbImpl =
         ValidateVolumeCapabilitiesResponsePBImpl
             .newInstance(false, "capability not supported");
-    Assert.assertEquals(false, pbImpl.isSupported());
-    Assert.assertEquals("capability not supported",
+    assertEquals(false, pbImpl.isSupported());
+    assertEquals("capability not supported",
         pbImpl.getResponseMessage());
 
     CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto =
         ((ValidateVolumeCapabilitiesResponsePBImpl) pbImpl).getProto();
-    Assert.assertEquals(false, proto.getSupported());
-    Assert.assertEquals("capability not supported", proto.getMessage());
+    assertEquals(false, proto.getSupported());
+    assertEquals("capability not supported", proto.getMessage());
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java
index b60d5cdb86080..059cc90c4f36e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java
@@ -20,59 +20,63 @@
 
 import csi.v0.Csi;
 import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 /**
  * Test class for CSI client.
  */
 public class TestCsiClient {
-
+  private static final Logger LOG = LoggerFactory.getLogger(TestCsiClient.class);
   private static File testRoot = null;
   private static String domainSocket = null;
   private static FakeCsiDriver driver = null;
 
-  @BeforeClass
+  @BeforeAll
   public static void setUp() throws IOException {
-    File testDir = GenericTestUtils.getTestDir();
+    // Use /tmp to fix bind failure caused by the long file name
+    File tmpDir = new File(System.getProperty("java.io.tmpdir"));
     testRoot = Files
-        .createTempDirectory(testDir.toPath(), "test").toFile();
+        .createTempDirectory(tmpDir.toPath(), "test").toFile();
     File socketPath = new File(testRoot, "csi.sock");
     FileUtils.forceMkdirParent(socketPath);
     domainSocket = "unix://" + socketPath.getAbsolutePath();
+    LOG.info("Create unix domain socket: {}", domainSocket);
     driver = new FakeCsiDriver(domainSocket);
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws IOException {
     if (testRoot != null) {
       FileUtils.deleteDirectory(testRoot);
     }
   }
 
-  @Before
+  @BeforeEach
   public void beforeMethod() {
     // Skip tests on non-linux systems
     String osName = System.getProperty("os.name").toLowerCase();
-    Assume.assumeTrue(osName.contains("nix") || osName.contains("nux"));
+    Assumptions.assumeTrue(osName.contains("nix") || osName.contains("nux"));
   }
 
   @Test
-  public void testIdentityService() throws IOException {
+  void testIdentityService() throws IOException {
     try {
       driver.start();
       CsiClient client = new CsiClientImpl(domainSocket);
       Csi.GetPluginInfoResponse response = client.getPluginInfo();
-      Assert.assertEquals("fake-csi-identity-service", response.getName());
+      assertEquals("fake-csi-identity-service", response.getName());
     } finally {
       driver.stop();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index d73853d5ca932..d0fd79aaa78e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -34,6 +34,8 @@
     <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
   </properties>
 
+
+
   <dependencies>
     <dependency>
       <groupId>javax.servlet</groupId>
@@ -93,7 +95,29 @@
     </dependency>
     <dependency>
       <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey.contribs</groupId>
@@ -124,14 +148,6 @@
       <groupId>org.codehaus.jettison</groupId>
       <artifactId>jettison</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-client</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
       <artifactId>hadoop-shaded-guava</artifactId>
@@ -180,6 +196,26 @@
       <artifactId>bcprov-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>de.ruedigermoeller</groupId>
       <artifactId>fst</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 9502af469d1bc..5689ecf298f69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -58,7 +58,7 @@
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
index 10d3b52c7e93b..6b0570a32e19e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
@@ -21,7 +21,6 @@
 import com.google.inject.Inject;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index 2b562c6a340d2..c4d8d4da33d7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -19,9 +19,8 @@
 package org.apache.hadoop.yarn.server.timeline;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.commons.collections.map.LRUMap;
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -32,7 +31,6 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.timeline.*;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -41,6 +39,7 @@
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils;
 import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
 import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyParser;
 import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
@@ -48,7 +47,6 @@
 import org.iq80.leveldb.*;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.*;
@@ -242,19 +240,7 @@ protected void serviceInit(Configuration conf) throws Exception {
       IOUtils.cleanupWithLogger(LOG, localFS);
     }
     LOG.info("Using leveldb path " + dbPath);
-    try {
-      db = factory.open(new File(dbPath.toString()), options);
-    } catch (IOException ioe) {
-      File dbFile = new File(dbPath.toString());
-      File backupPath = new File(
-          dbPath.toString() + BACKUP_EXT + Time.monotonicNow());
-      LOG.warn("Incurred exception while loading LevelDb database. Backing " +
-          "up at "+ backupPath, ioe);
-      FileUtils.copyDirectory(dbFile, backupPath);
-      LOG.warn("Going to try repair");
-      factory.repair(dbFile, options);
-      db = factory.open(dbFile, options);
-    }
+    db = LeveldbUtils.loadOrRepairLevelDb(factory, dbPath, options);
     checkVersion();
     startTimeWriteCache =
         Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
index b421f954bfeb3..e93dcee3404e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
@@ -19,9 +19,8 @@
 package org.apache.hadoop.yarn.server.timeline;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
-import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -62,6 +61,7 @@
 import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.timeline.RollingLevelDB.RollingWriteBatch;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils;
 import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
 import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyParser;
 
@@ -199,6 +199,11 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
   static final String STARTTIME = "starttime-ldb";
   static final String OWNER = "owner-ldb";
 
+  @VisibleForTesting
+  //Extension to FILENAME where backup will be stored in case we need to
+  //call LevelDb recovery
+  static final String BACKUP_EXT = ".backup-";
+
   private static final byte[] DOMAIN_ID_COLUMN = "d".getBytes(UTF_8);
   private static final byte[] EVENTS_COLUMN = "e".getBytes(UTF_8);
   private static final byte[] PRIMARY_FILTERS_COLUMN = "f".getBytes(UTF_8);
@@ -240,6 +245,12 @@ public RollingLevelDBTimelineStore() {
     super(RollingLevelDBTimelineStore.class.getName());
   }
 
+  private JniDBFactory factory;
+  @VisibleForTesting
+  void setFactory(JniDBFactory fact) {
+    this.factory = fact;
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   protected void serviceInit(Configuration conf) throws Exception {
@@ -284,7 +295,9 @@ protected void serviceInit(Configuration conf) throws Exception {
     options.cacheSize(conf.getLong(
         TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE,
         DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
-    JniDBFactory factory = new JniDBFactory();
+    if(factory == null) {
+      factory = new JniDBFactory();
+    }
     Path dbPath = new Path(
         conf.get(TIMELINE_SERVICE_LEVELDB_PATH), FILENAME);
     Path domainDBPath = new Path(dbPath, DOMAIN);
@@ -327,13 +340,13 @@ protected void serviceInit(Configuration conf) throws Exception {
         TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE,
         DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE));
     LOG.info("Using leveldb path " + dbPath);
-    domaindb = factory.open(new File(domainDBPath.toString()), options);
+    domaindb = LeveldbUtils.loadOrRepairLevelDb(factory, domainDBPath, options);
     entitydb = new RollingLevelDB(ENTITY);
     entitydb.init(conf);
     indexdb = new RollingLevelDB(INDEX);
     indexdb.init(conf);
-    starttimedb = factory.open(new File(starttimeDBPath.toString()), options);
-    ownerdb = factory.open(new File(ownerDBPath.toString()), options);
+    starttimedb = LeveldbUtils.loadOrRepairLevelDb(factory, starttimeDBPath, options);
+    ownerdb = LeveldbUtils.loadOrRepairLevelDb(factory, ownerDBPath, options);
     checkVersion();
     startTimeWriteCache = Collections.synchronizedMap(new LRUMap(
         getStartTimeWriteCacheSize(conf)));
@@ -346,7 +359,7 @@ protected void serviceInit(Configuration conf) throws Exception {
 
     super.serviceInit(conf);
   }
-  
+
   @Override
   protected void serviceStart() throws Exception {
     if (getConfig().getBoolean(TIMELINE_SERVICE_TTL_ENABLE, true)) {
@@ -1816,4 +1829,4 @@ private static TimelineDomain getTimelineDomain(DBIterator iterator,
       return domain;
     }
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java
index 82c7f265e766a..fcdb2ac733720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/util/LeveldbUtils.java
@@ -19,16 +19,30 @@
 package org.apache.hadoop.yarn.server.timeline.util;
 
 
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.util.Time;
 
+import java.io.File;
 import java.io.IOException;
 
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
 
 public class LeveldbUtils {
 
+  private static final String BACKUP_EXT = ".backup-";
+  private static final Logger LOG = LoggerFactory
+      .getLogger(LeveldbUtils.class);
+
   /** A string builder utility for building timeline server leveldb keys. */
   public static class KeyBuilder {
     /** Maximum subkeys that can be added to construct a key. */
@@ -184,4 +198,22 @@ public static boolean prefixMatches(byte[] prefix, int prefixlen,
   public static final FsPermission LEVELDB_DIR_UMASK = FsPermission
       .createImmutable((short) 0700);
 
+  public static DB loadOrRepairLevelDb(JniDBFactory factory, Path dbPath, Options options)
+      throws IOException {
+    DB db;
+    try{
+      db = factory.open(new File(dbPath.toString()), options);
+    } catch (IOException ioe){
+      File dbFile = new File(dbPath.toString());
+      File dbBackupPath = new File(
+          dbPath.toString() + BACKUP_EXT + Time.monotonicNow());
+      LOG.warn("Incurred exception while loading LevelDb database. Backing " +
+          "up at "+ dbBackupPath, ioe);
+      FileUtils.copyDirectory(dbFile, dbBackupPath);
+      factory.repair(dbFile, options);
+      db = factory.open(dbFile, options);
+    }
+    return db;
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7ef6eca0dcad6..86b15b3412833 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -21,6 +21,9 @@
 import java.io.IOException;
 import java.util.List;
 
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -49,9 +52,11 @@
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestApplicationHistoryClientService {
 
@@ -59,7 +64,7 @@ public class TestApplicationHistoryClientService {
   private static TimelineDataManager dataManager;
   private final static int MAX_APPS = 2;
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     Configuration conf = new YarnConfiguration();
     TimelineStore store =
@@ -78,7 +83,7 @@ public static void setup() throws Exception {
   }
 
   @Test
-  public void testApplicationNotFound() throws IOException, YarnException {
+  void testApplicationNotFound() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, MAX_APPS + 1);
     GetApplicationReportRequest request =
@@ -87,18 +92,18 @@ public void testApplicationNotFound() throws IOException, YarnException {
       @SuppressWarnings("unused")
       GetApplicationReportResponse response =
           clientService.getApplicationReport(request);
-      Assert.fail("Exception should have been thrown before we reach here.");
+      fail("Exception should have been thrown before we reach here.");
     } catch (ApplicationNotFoundException e) {
       //This exception is expected.
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "doesn't exist in the timeline store"));
     } catch (Exception e) {
-      Assert.fail("Undesired exception caught");
+      fail("Undesired exception caught");
     }
   }
 
   @Test
-  public void testApplicationAttemptNotFound() throws IOException, YarnException {
+  void testApplicationAttemptNotFound() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
@@ -108,41 +113,41 @@ public void testApplicationAttemptNotFound() throws IOException, YarnException {
       @SuppressWarnings("unused")
       GetApplicationAttemptReportResponse response =
           clientService.getApplicationAttemptReport(request);
-      Assert.fail("Exception should have been thrown before we reach here.");
+      fail("Exception should have been thrown before we reach here.");
     } catch (ApplicationAttemptNotFoundException e) {
       //This Exception is expected
       System.out.println(e.getMessage());
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           "doesn't exist in the timeline store"));
     } catch (Exception e) {
-      Assert.fail("Undesired exception caught");
+      fail("Undesired exception caught");
     }
   }
 
   @Test
-  public void testContainerNotFound() throws IOException, YarnException {
-   ApplicationId appId = ApplicationId.newInstance(0, 1);
-   ApplicationAttemptId appAttemptId =
-       ApplicationAttemptId.newInstance(appId, 1);
-   ContainerId containerId = ContainerId.newContainerId(appAttemptId,
-       MAX_APPS + 1);
-   GetContainerReportRequest request =
-       GetContainerReportRequest.newInstance(containerId);
-   try {
-   @SuppressWarnings("unused")
-   GetContainerReportResponse response =
-       clientService.getContainerReport(request);
-   } catch (ContainerNotFoundException e) {
-     //This exception is expected
-     Assert.assertTrue(e.getMessage().contains(
-         "doesn't exist in the timeline store"));
-   }  catch (Exception e) {
-      Assert.fail("Undesired exception caught");
-   }
- }
+  void testContainerNotFound() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+        MAX_APPS + 1);
+    GetContainerReportRequest request =
+        GetContainerReportRequest.newInstance(containerId);
+    try {
+      @SuppressWarnings("unused")
+      GetContainerReportResponse response =
+          clientService.getContainerReport(request);
+    } catch (ContainerNotFoundException e) {
+      //This exception is expected
+      assertTrue(e.getMessage().contains(
+          "doesn't exist in the timeline store"));
+    }  catch (Exception e) {
+      fail("Undesired exception caught");
+    }
+  }
 
   @Test
-  public void testApplicationReport() throws IOException, YarnException {
+  void testApplicationReport() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);
     GetApplicationReportRequest request =
@@ -150,20 +155,20 @@ public void testApplicationReport() throws IOException, YarnException {
     GetApplicationReportResponse response =
         clientService.getApplicationReport(request);
     ApplicationReport appReport = response.getApplicationReport();
-    Assert.assertNotNull(appReport);
-    Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+    assertNotNull(appReport);
+    assertEquals(123, appReport.getApplicationResourceUsageReport()
         .getMemorySeconds());
-    Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+    assertEquals(345, appReport.getApplicationResourceUsageReport()
         .getVcoreSeconds());
-    Assert.assertEquals("application_0_0001", appReport.getApplicationId()
-      .toString());
-    Assert.assertEquals("test app type",
+    assertEquals("application_0_0001", appReport.getApplicationId()
+        .toString());
+    assertEquals("test app type",
         appReport.getApplicationType().toString());
-    Assert.assertEquals("test queue", appReport.getQueue().toString());
+    assertEquals("test queue", appReport.getQueue().toString());
   }
 
   @Test
-  public void testApplications() throws IOException, YarnException {
+  void testApplications() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);
     ApplicationId appId1 = ApplicationId.newInstance(0, 2);
@@ -171,9 +176,9 @@ public void testApplications() throws IOException, YarnException {
     GetApplicationsResponse response =
         clientService.getApplications(request);
     List<ApplicationReport> appReport = response.getApplicationList();
-    Assert.assertNotNull(appReport);
-    Assert.assertEquals(appId, appReport.get(1).getApplicationId());
-    Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+    assertNotNull(appReport);
+    assertEquals(appId, appReport.get(1).getApplicationId());
+    assertEquals(appId1, appReport.get(0).getApplicationId());
 
     // Create a historyManager, and set the max_apps can be loaded
     // as 1.
@@ -181,7 +186,7 @@ public void testApplications() throws IOException, YarnException {
     conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
     ApplicationHistoryManagerOnTimelineStore historyManager2 =
         new ApplicationHistoryManagerOnTimelineStore(dataManager,
-          new ApplicationACLsManager(conf));
+            new ApplicationACLsManager(conf));
     historyManager2.init(conf);
     historyManager2.start();
     @SuppressWarnings("resource")
@@ -189,14 +194,14 @@ public void testApplications() throws IOException, YarnException {
         new ApplicationHistoryClientService(historyManager2);
     response = clientService2.getApplications(request);
     appReport = response.getApplicationList();
-    Assert.assertNotNull(appReport);
-    Assert.assertTrue(appReport.size() == 1);
+    assertNotNull(appReport);
+    assertTrue(appReport.size() == 1);
     // Expected to get the appReport for application with appId1
-    Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+    assertEquals(appId1, appReport.get(0).getApplicationId());
   }
 
   @Test
-  public void testApplicationAttemptReport() throws IOException, YarnException {
+  void testApplicationAttemptReport() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -206,13 +211,13 @@ public void testApplicationAttemptReport() throws IOException, YarnException {
         clientService.getApplicationAttemptReport(request);
     ApplicationAttemptReport attemptReport =
         response.getApplicationAttemptReport();
-    Assert.assertNotNull(attemptReport);
-    Assert.assertEquals("appattempt_0_0001_000001", attemptReport
-      .getApplicationAttemptId().toString());
+    assertNotNull(attemptReport);
+    assertEquals("appattempt_0_0001_000001", attemptReport
+        .getApplicationAttemptId().toString());
   }
 
   @Test
-  public void testApplicationAttempts() throws IOException, YarnException {
+  void testApplicationAttempts() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -224,15 +229,15 @@ public void testApplicationAttempts() throws IOException, YarnException {
         clientService.getApplicationAttempts(request);
     List<ApplicationAttemptReport> attemptReports =
         response.getApplicationAttemptList();
-    Assert.assertNotNull(attemptReports);
-    Assert.assertEquals(appAttemptId, attemptReports.get(0)
-      .getApplicationAttemptId());
-    Assert.assertEquals(appAttemptId1, attemptReports.get(1)
-      .getApplicationAttemptId());
+    assertNotNull(attemptReports);
+    assertEquals(appAttemptId, attemptReports.get(0)
+        .getApplicationAttemptId());
+    assertEquals(appAttemptId1, attemptReports.get(1)
+        .getApplicationAttemptId());
   }
 
   @Test
-  public void testContainerReport() throws IOException, YarnException {
+  void testContainerReport() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -242,15 +247,15 @@ public void testContainerReport() throws IOException, YarnException {
     GetContainerReportResponse response =
         clientService.getContainerReport(request);
     ContainerReport container = response.getContainerReport();
-    Assert.assertNotNull(container);
-    Assert.assertEquals(containerId, container.getContainerId());
-    Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
+    assertNotNull(container);
+    assertEquals(containerId, container.getContainerId());
+    assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
         "test host:100/container_0_0001_01_000001/" +
         "container_0_0001_01_000001/user1", container.getLogUrl());
   }
 
   @Test
-  public void testContainers() throws IOException, YarnException {
+  void testContainers() throws IOException, YarnException {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -261,8 +266,8 @@ public void testContainers() throws IOException, YarnException {
     GetContainersResponse response =
         clientService.getContainers(request);
     List<ContainerReport> containers = response.getContainerList();
-    Assert.assertNotNull(containers);
-    Assert.assertEquals(containerId, containers.get(0).getContainerId());
-    Assert.assertEquals(containerId1, containers.get(1).getContainerId());
+    assertNotNull(containers);
+    assertEquals(containerId, containers.get(0).getContainerId());
+    assertEquals(containerId1, containers.get(1).getContainerId());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
index da5ddc183b173..87a191d643bff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
@@ -21,39 +21,43 @@
 import java.io.IOException;
 import java.util.Map;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestApplicationHistoryManagerImpl extends
     ApplicationHistoryStoreTestUtils {
-  ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null;
+  private ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     Configuration config = new Configuration();
     config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
-      MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
+        MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
     applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl();
     applicationHistoryManagerImpl.init(config);
     applicationHistoryManagerImpl.start();
     store = applicationHistoryManagerImpl.getHistoryStore();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     applicationHistoryManagerImpl.stop();
   }
 
   @Test
-  public void testApplicationReport() throws IOException, YarnException {
+  void testApplicationReport() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);
     writeApplicationStartData(appId);
@@ -64,17 +68,17 @@ public void testApplicationReport() throws IOException, YarnException {
     writeApplicationAttemptFinishData(appAttemptId);
     ApplicationReport appReport =
         applicationHistoryManagerImpl.getApplication(appId);
-    Assert.assertNotNull(appReport);
-    Assert.assertEquals(appId, appReport.getApplicationId());
-    Assert.assertEquals(appAttemptId,
-      appReport.getCurrentApplicationAttemptId());
-    Assert.assertEquals(appAttemptId.toString(), appReport.getHost());
-    Assert.assertEquals("test type", appReport.getApplicationType().toString());
-    Assert.assertEquals("test queue", appReport.getQueue().toString());
+    assertNotNull(appReport);
+    assertEquals(appId, appReport.getApplicationId());
+    assertEquals(appAttemptId,
+        appReport.getCurrentApplicationAttemptId());
+    assertEquals(appAttemptId.toString(), appReport.getHost());
+    assertEquals("test type", appReport.getApplicationType().toString());
+    assertEquals("test queue", appReport.getQueue().toString());
   }
 
   @Test
-  public void testApplications() throws IOException {
+  void testApplications() throws IOException {
     ApplicationId appId1 = ApplicationId.newInstance(0, 1);
     ApplicationId appId2 = ApplicationId.newInstance(0, 2);
     ApplicationId appId3 = ApplicationId.newInstance(0, 3);
@@ -86,10 +90,10 @@ public void testApplications() throws IOException {
     writeApplicationFinishData(appId3);
     Map<ApplicationId, ApplicationReport> reports =
         applicationHistoryManagerImpl.getApplications(2, 2000L, 5000L);
-    Assert.assertNotNull(reports);
-    Assert.assertEquals(2, reports.size());
-    Assert.assertNull(reports.get("1"));
-    Assert.assertNull(reports.get("2"));
-    Assert.assertNull(reports.get("3"));
+    assertNotNull(reports);
+    assertEquals(2, reports.size());
+    assertNull(reports.get("1"));
+    assertNull(reports.get("2"));
+    assertNull(reports.get("3"));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 96c81315704d8..a31d9a92419f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,6 +26,11 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -56,16 +61,13 @@
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
 
-@RunWith(Parameterized.class)
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
 public class TestApplicationHistoryManagerOnTimelineStore {
 
   private static final int SCALE = 5;
@@ -75,91 +77,78 @@ public class TestApplicationHistoryManagerOnTimelineStore {
   private UserGroupInformation callerUGI;
   private Configuration conf;
 
-  @BeforeClass
+  @BeforeAll
   public static void prepareStore() throws Exception {
     store = createStore(SCALE);
     TimelineEntities entities = new TimelineEntities();
-    entities.addEntity(createApplicationTimelineEntity(
-        ApplicationId.newInstance(0, SCALE + 1), true, true, false, false,
-        YarnApplicationState.FINISHED));
-    entities.addEntity(createApplicationTimelineEntity(
-        ApplicationId.newInstance(0, SCALE + 2), true, false, true, false,
-        YarnApplicationState.FINISHED));
+    entities.addEntity(
+        createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 1), true, true, false,
+            false, YarnApplicationState.FINISHED));
+    entities.addEntity(
+        createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 2), true, false, true,
+            false, YarnApplicationState.FINISHED));
     store.put(entities);
   }
 
   public static TimelineStore createStore(int scale) throws Exception {
-    TimelineStore store = new MemoryTimelineStore();
-    prepareTimelineStore(store, scale);
+    store = new MemoryTimelineStore();
+    prepareTimelineStore(scale);
     return store;
   }
 
-  @Before
-  public void setup() throws Exception {
-    // Only test the ACLs of the generic history
-    TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
-    aclsManager.setTimelineStore(store);
-    TimelineDataManager dataManager =
-        new TimelineDataManager(store, aclsManager);
-    dataManager.init(conf);
-    ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
-    historyManager =
-        new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
-    historyManager.init(conf);
-    historyManager.start();
-  }
-
-  @After
+  @AfterEach
   public void tearDown() {
     if (historyManager != null) {
       historyManager.stop();
     }
   }
 
-  @Parameters
   public static Collection<Object[]> callers() {
     // user1 is the owner
     // user2 is the authorized user
     // user3 is the unauthorized user
     // admin is the admin acl
-    return Arrays.asList(
-        new Object[][] { { "" }, { "user1" }, { "user2" }, { "user3" }, { "admin" } });
+    return Arrays.asList(new Object[][] {{""}, {"user1"}, {"user2"}, {"user3"}, {"admin"}});
   }
 
-  public TestApplicationHistoryManagerOnTimelineStore(String caller) {
+  public void initTestApplicationHistoryManagerOnTimelineStore(String caller) {
     conf = new YarnConfiguration();
     if (!caller.equals("")) {
       callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE);
       conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
       conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
     }
+    TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
+    aclsManager.setTimelineStore(store);
+    TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager);
+    dataManager.init(conf);
+    ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
+    historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
+    historyManager.init(conf);
+    historyManager.start();
   }
 
-  private static void prepareTimelineStore(TimelineStore store, int scale)
-      throws Exception {
+  private static void prepareTimelineStore(int scale) throws Exception {
     for (int i = 1; i <= scale; ++i) {
       TimelineEntities entities = new TimelineEntities();
       ApplicationId appId = ApplicationId.newInstance(0, i);
       if (i == 2) {
-        entities.addEntity(createApplicationTimelineEntity(
-            appId, true, false, false, true, YarnApplicationState.FINISHED));
+        entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, true,
+            YarnApplicationState.FINISHED));
       } else if (i == 3) {
-        entities.addEntity(createApplicationTimelineEntity(
-            appId, false, false, false, false, YarnApplicationState.FINISHED,
-            true, false));
+        entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
+            YarnApplicationState.FINISHED, true, false));
       } else if (i == SCALE + 1) {
-        entities.addEntity(createApplicationTimelineEntity(
-            appId, false, false, false, false, YarnApplicationState.FINISHED,
-            false, true));
+        entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
+            YarnApplicationState.FINISHED, false, true));
       } else {
-        entities.addEntity(createApplicationTimelineEntity(
-            appId, false, false, false, false, YarnApplicationState.FINISHED));
+        entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
+            YarnApplicationState.FINISHED));
       }
       store.put(entities);
       for (int j = 1; j <= scale; ++j) {
         entities = new TimelineEntities();
-        ApplicationAttemptId appAttemptId =
-            ApplicationAttemptId.newInstance(appId, j);
+        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
         entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
         store.put(entities);
         for (int k = 1; k <= scale; ++k) {
@@ -172,129 +161,119 @@ private static void prepareTimelineStore(TimelineStore store, int scale)
     }
     TimelineEntities entities = new TimelineEntities();
     ApplicationId appId = ApplicationId.newInstance(1234, 1);
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, 1);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
     ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
-    entities.addEntity(createApplicationTimelineEntity(
-        appId, true, false, false, false, YarnApplicationState.RUNNING));
+    entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, false,
+        YarnApplicationState.RUNNING));
     entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
     entities.addEntity(createContainerEntity(containerId));
     store.put(entities);
   }
 
-  @Test
-  public void testGetApplicationReport() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetApplicationReport(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     for (int i = 1; i <= 3; ++i) {
       final ApplicationId appId = ApplicationId.newInstance(0, i);
       ApplicationReport app;
       if (callerUGI == null) {
         app = historyManager.getApplication(appId);
       } else {
-        app =
-            callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
+        app = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport>() {
           @Override
           public ApplicationReport run() throws Exception {
             return historyManager.getApplication(appId);
           }
         });
       }
-      Assert.assertNotNull(app);
-      Assert.assertEquals(appId, app.getApplicationId());
-      Assert.assertEquals("test app", app.getName());
-      Assert.assertEquals("test app type", app.getApplicationType());
-      Assert.assertEquals("user1", app.getUser());
+      assertNotNull(app);
+      assertEquals(appId, app.getApplicationId());
+      assertEquals("test app", app.getName());
+      assertEquals("test app type", app.getApplicationType());
+      assertEquals("user1", app.getUser());
       if (i == 2) {
         // Change event is fired only in case of app with ID 2, hence verify
         // with updated changes. And make sure last updated change is accepted.
-        Assert.assertEquals("changed queue1", app.getQueue());
-        Assert.assertEquals(Priority.newInstance(6), app.getPriority());
+        assertEquals("changed queue1", app.getQueue());
+        assertEquals(Priority.newInstance(6), app.getPriority());
       } else {
-        Assert.assertEquals("test queue", app.getQueue());
-        Assert.assertEquals(Priority.newInstance(0), app.getPriority());
+        assertEquals("test queue", app.getQueue());
+        assertEquals(Priority.newInstance(0), app.getPriority());
       }
-      Assert.assertEquals(Integer.MAX_VALUE + 2L
-          + app.getApplicationId().getId(), app.getStartTime());
-      Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
-      Assert.assertEquals(Integer.MAX_VALUE + 3L
-          + +app.getApplicationId().getId(), app.getFinishTime());
-      Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
-      Assert.assertEquals(2, app.getApplicationTags().size());
-      Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
-      Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
+      assertEquals(Integer.MAX_VALUE + 2L + app.getApplicationId().getId(), app.getStartTime());
+      assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
+      assertEquals(Integer.MAX_VALUE + 3L + +app.getApplicationId().getId(), app.getFinishTime());
+      assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+      assertEquals(2, app.getApplicationTags().size());
+      assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
+      assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
       // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
       // Nobody except admin and owner has access to the details of the app.
-      if ((i != 2 && callerUGI != null &&
-          callerUGI.getShortUserName().equals("user3")) ||
-          (i ==  2 && callerUGI != null &&
-          (callerUGI.getShortUserName().equals("user2") ||
-              callerUGI.getShortUserName().equals("user3")))) {
-        Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
+      if ((i != 2 && callerUGI != null && callerUGI.getShortUserName().equals("user3")) || (i == 2
+          && callerUGI != null && (callerUGI.getShortUserName().equals("user2")
+          || callerUGI.getShortUserName().equals("user3")))) {
+        assertEquals(ApplicationAttemptId.newInstance(appId, -1),
             app.getCurrentApplicationAttemptId());
-        Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
-            app.getHost());
-        Assert.assertEquals(-1, app.getRpcPort());
-        Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
-            app.getTrackingUrl());
-        Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
+        assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getHost());
+        assertEquals(-1, app.getRpcPort());
+        assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getTrackingUrl());
+        assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
             app.getOriginalTrackingUrl());
-        Assert.assertEquals("", app.getDiagnostics());
+        assertEquals("", app.getDiagnostics());
       } else {
-        Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
+        assertEquals(ApplicationAttemptId.newInstance(appId, 1),
             app.getCurrentApplicationAttemptId());
-        Assert.assertEquals("test host", app.getHost());
-        Assert.assertEquals(100, app.getRpcPort());
-        Assert.assertEquals("test tracking url", app.getTrackingUrl());
-        Assert.assertEquals("test original tracking url",
-            app.getOriginalTrackingUrl());
-        Assert.assertEquals("test diagnostics info", app.getDiagnostics());
+        assertEquals("test host", app.getHost());
+        assertEquals(100, app.getRpcPort());
+        assertEquals("test tracking url", app.getTrackingUrl());
+        assertEquals("test original tracking url", app.getOriginalTrackingUrl());
+        assertEquals("test diagnostics info", app.getDiagnostics());
       }
       ApplicationResourceUsageReport applicationResourceUsageReport =
           app.getApplicationResourceUsageReport();
-      Assert.assertEquals(123,
-          applicationResourceUsageReport.getMemorySeconds());
-      Assert
-          .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+      assertEquals(123, applicationResourceUsageReport.getMemorySeconds());
+      assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
       long expectedPreemptMemSecs = 456;
       long expectedPreemptVcoreSecs = 789;
       if (i == 3) {
         expectedPreemptMemSecs = 0;
         expectedPreemptVcoreSecs = 0;
       }
-      Assert.assertEquals(expectedPreemptMemSecs,
+      assertEquals(expectedPreemptMemSecs,
           applicationResourceUsageReport.getPreemptedMemorySeconds());
-      Assert
-          .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport
-              .getPreemptedVcoreSeconds());
-      Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
-          app.getFinalApplicationStatus());
-      Assert.assertEquals(YarnApplicationState.FINISHED,
-          app.getYarnApplicationState());
+      assertEquals(expectedPreemptVcoreSecs,
+          applicationResourceUsageReport.getPreemptedVcoreSeconds());
+      assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus());
+      assertEquals(YarnApplicationState.FINISHED, app.getYarnApplicationState());
     }
   }
 
-  @Test
-  public void testGetApplicationReportWithNotAttempt() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetApplicationReportWithNotAttempt(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1);
     ApplicationReport app;
     if (callerUGI == null) {
       app = historyManager.getApplication(appId);
     } else {
-      app =
-          callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
-            @Override
-            public ApplicationReport run() throws Exception {
-              return historyManager.getApplication(appId);
-            }
-          });
+      app = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport>() {
+        @Override
+        public ApplicationReport run() throws Exception {
+          return historyManager.getApplication(appId);
+        }
+      });
     }
-    Assert.assertNotNull(app);
-    Assert.assertEquals(appId, app.getApplicationId());
-    Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
-        app.getCurrentApplicationAttemptId());
+    assertNotNull(app);
+    assertEquals(appId, app.getApplicationId());
+    assertEquals(ApplicationAttemptId.newInstance(appId, -1), app.getCurrentApplicationAttemptId());
   }
 
-  @Test
-  public void testGetApplicationAttemptReport() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetApplicationAttemptReport(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     final ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
     ApplicationAttemptReport appAttempt;
@@ -302,8 +281,7 @@ public void testGetApplicationAttemptReport() throws Exception {
       appAttempt = historyManager.getApplicationAttempt(appAttemptId);
     } else {
       try {
-        appAttempt =
-            callerUGI.doAs(new PrivilegedExceptionAction<ApplicationAttemptReport> () {
+        appAttempt = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationAttemptReport>() {
           @Override
           public ApplicationAttemptReport run() throws Exception {
             return historyManager.getApplicationAttempt(appAttemptId);
@@ -311,7 +289,7 @@ public ApplicationAttemptReport run() throws Exception {
         });
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
           // The exception is expected
-          Assert.fail();
+          fail();
         }
       } catch (AuthorizationException e) {
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@@ -321,32 +299,29 @@ public ApplicationAttemptReport run() throws Exception {
         throw e;
       }
     }
-    Assert.assertNotNull(appAttempt);
-    Assert.assertEquals(appAttemptId, appAttempt.getApplicationAttemptId());
-    Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1),
-        appAttempt.getAMContainerId());
-    Assert.assertEquals("test host", appAttempt.getHost());
-    Assert.assertEquals(100, appAttempt.getRpcPort());
-    Assert.assertEquals("test tracking url", appAttempt.getTrackingUrl());
-    Assert.assertEquals("test original tracking url",
-        appAttempt.getOriginalTrackingUrl());
-    Assert.assertEquals("test diagnostics info", appAttempt.getDiagnostics());
-    Assert.assertEquals(YarnApplicationAttemptState.FINISHED,
-        appAttempt.getYarnApplicationAttemptState());
+    assertNotNull(appAttempt);
+    assertEquals(appAttemptId, appAttempt.getApplicationAttemptId());
+    assertEquals(ContainerId.newContainerId(appAttemptId, 1), appAttempt.getAMContainerId());
+    assertEquals("test host", appAttempt.getHost());
+    assertEquals(100, appAttempt.getRpcPort());
+    assertEquals("test tracking url", appAttempt.getTrackingUrl());
+    assertEquals("test original tracking url", appAttempt.getOriginalTrackingUrl());
+    assertEquals("test diagnostics info", appAttempt.getDiagnostics());
+    assertEquals(YarnApplicationAttemptState.FINISHED, appAttempt.getYarnApplicationAttemptState());
   }
 
-  @Test
-  public void testGetContainerReport() throws Exception {
-    final ContainerId containerId =
-        ContainerId.newContainerId(ApplicationAttemptId.newInstance(
-            ApplicationId.newInstance(0, 1), 1), 1);
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetContainerReport(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
+    final ContainerId containerId = ContainerId.newContainerId(
+        ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
     ContainerReport container;
     if (callerUGI == null) {
       container = historyManager.getContainer(containerId);
     } else {
       try {
-        container =
-            callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport> () {
+        container = callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport>() {
           @Override
           public ContainerReport run() throws Exception {
             return historyManager.getContainer(containerId);
@@ -354,7 +329,7 @@ public ContainerReport run() throws Exception {
         });
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
           // The exception is expected
-          Assert.fail();
+          fail();
         }
       } catch (AuthorizationException e) {
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@@ -364,61 +339,59 @@ public ContainerReport run() throws Exception {
         throw e;
       }
     }
-    Assert.assertNotNull(container);
-    Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime());
-    Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
-    Assert.assertEquals(Resource.newInstance(-1, -1),
-        container.getAllocatedResource());
-    Assert.assertEquals(NodeId.newInstance("test host", 100),
-        container.getAssignedNode());
-    Assert.assertEquals(Priority.UNDEFINED, container.getPriority());
-    Assert
-        .assertEquals("test diagnostics info", container.getDiagnosticsInfo());
-    Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState());
-    Assert.assertEquals(-1, container.getContainerExitStatus());
-    Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
-        "test host:100/container_0_0001_01_000001/"
-        + "container_0_0001_01_000001/user1", container.getLogUrl());
+    assertNotNull(container);
+    assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime());
+    assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
+    assertEquals(Resource.newInstance(-1, -1), container.getAllocatedResource());
+    assertEquals(NodeId.newInstance("test host", 100), container.getAssignedNode());
+    assertEquals(Priority.UNDEFINED, container.getPriority());
+    assertEquals("test diagnostics info", container.getDiagnosticsInfo());
+    assertEquals(ContainerState.COMPLETE, container.getContainerState());
+    assertEquals(-1, container.getContainerExitStatus());
+    assertEquals(
+        "http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/"
+            + "container_0_0001_01_000001/user1", container.getLogUrl());
   }
 
-  @Test
-  public void testGetApplications() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetApplications(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     Collection<ApplicationReport> apps =
-        historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE)
-          .values();
-    Assert.assertNotNull(apps);
-    Assert.assertEquals(SCALE + 2, apps.size());
+        historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE).values();
+    assertNotNull(apps);
+    assertEquals(SCALE + 2, apps.size());
     ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
     for (ApplicationReport app : apps) {
-      Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
+      assertNotEquals(ignoredAppId, app.getApplicationId());
     }
 
     // Get apps by given appStartedTime period
-    apps =
-        historyManager.getApplications(Long.MAX_VALUE, 2147483653L,
-          Long.MAX_VALUE).values();
-    Assert.assertNotNull(apps);
-    Assert.assertEquals(2, apps.size());
+    apps = historyManager.getApplications(Long.MAX_VALUE, 2147483653L, Long.MAX_VALUE).values();
+    assertNotNull(apps);
+    assertEquals(2, apps.size());
   }
 
-  @Test
-  public void testGetApplicationAttempts() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetApplicationAttempts(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     final ApplicationId appId = ApplicationId.newInstance(0, 1);
     Collection<ApplicationAttemptReport> appAttempts;
     if (callerUGI == null) {
       appAttempts = historyManager.getApplicationAttempts(appId).values();
     } else {
       try {
-        appAttempts = callerUGI.doAs(
-            new PrivilegedExceptionAction<Collection<ApplicationAttemptReport>> () {
-          @Override
-          public Collection<ApplicationAttemptReport> run() throws Exception {
-            return historyManager.getApplicationAttempts(appId).values();
-          }
-        });
+        appAttempts =
+            callerUGI.doAs(new PrivilegedExceptionAction<Collection<ApplicationAttemptReport>>() {
+              @Override
+              public Collection<ApplicationAttemptReport> run() throws Exception {
+                return historyManager.getApplicationAttempts(appId).values();
+              }
+            });
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
           // The exception is expected
-          Assert.fail();
+          fail();
         }
       } catch (AuthorizationException e) {
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@@ -428,12 +401,14 @@ public Collection<ApplicationAttemptReport> run() throws Exception {
         throw e;
       }
     }
-    Assert.assertNotNull(appAttempts);
-    Assert.assertEquals(SCALE, appAttempts.size());
+    assertNotNull(appAttempts);
+    assertEquals(SCALE, appAttempts.size());
   }
 
-  @Test
-  public void testGetContainers() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetContainers(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     final ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
     Collection<ContainerReport> containers;
@@ -441,8 +416,7 @@ public void testGetContainers() throws Exception {
       containers = historyManager.getContainers(appAttemptId).values();
     } else {
       try {
-        containers = callerUGI.doAs(
-            new PrivilegedExceptionAction<Collection<ContainerReport>> () {
+        containers = callerUGI.doAs(new PrivilegedExceptionAction<Collection<ContainerReport>>() {
           @Override
           public Collection<ContainerReport> run() throws Exception {
             return historyManager.getContainers(appAttemptId).values();
@@ -450,7 +424,7 @@ public Collection<ContainerReport> run() throws Exception {
         });
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
           // The exception is expected
-          Assert.fail();
+          fail();
         }
       } catch (AuthorizationException e) {
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@@ -460,12 +434,14 @@ public Collection<ContainerReport> run() throws Exception {
         throw e;
       }
     }
-    Assert.assertNotNull(containers);
-    Assert.assertEquals(SCALE, containers.size());
+    assertNotNull(containers);
+    assertEquals(SCALE, containers.size());
   }
 
-  @Test
-  public void testGetAMContainer() throws Exception {
+  @MethodSource("callers")
+  @ParameterizedTest
+  void testGetAMContainer(String caller) throws Exception {
+    initTestApplicationHistoryManagerOnTimelineStore(caller);
     final ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
     ContainerReport container;
@@ -473,8 +449,7 @@ public void testGetAMContainer() throws Exception {
       container = historyManager.getAMContainer(appAttemptId);
     } else {
       try {
-        container =
-            callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport> () {
+        container = callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport>() {
           @Override
           public ContainerReport run() throws Exception {
             return historyManager.getAMContainer(appAttemptId);
@@ -482,7 +457,7 @@ public ContainerReport run() throws Exception {
         });
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
           // The exception is expected
-          Assert.fail();
+          fail();
         }
       } catch (AuthorizationException e) {
         if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@@ -492,24 +467,20 @@ public ContainerReport run() throws Exception {
         throw e;
       }
     }
-    Assert.assertNotNull(container);
-    Assert.assertEquals(appAttemptId, container.getContainerId()
-        .getApplicationAttemptId());
+    assertNotNull(container);
+    assertEquals(appAttemptId, container.getContainerId().getApplicationAttemptId());
   }
 
-  private static TimelineEntity createApplicationTimelineEntity(
-      ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
-      boolean wrongAppId, boolean enableUpdateEvent,
+  private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId,
+      boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent,
       YarnApplicationState state) {
-    return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
-        wrongAppId, enableUpdateEvent, state, false, false);
+    return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, wrongAppId,
+        enableUpdateEvent, state, false, false);
   }
 
-  private static TimelineEntity createApplicationTimelineEntity(
-      ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
-      boolean wrongAppId, boolean enableUpdateEvent,
-      YarnApplicationState state, boolean missingPreemptMetrics,
-      boolean missingQueue) {
+  private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId,
+      boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent,
+      YarnApplicationState state, boolean missingPreemptMetrics, boolean missingQueue) {
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
     if (wrongAppId) {
@@ -518,23 +489,17 @@ private static TimelineEntity createApplicationTimelineEntity(
       entity.setEntityId(appId.toString());
     }
     entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
-    entity.addPrimaryFilter(
-        TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
+    entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
     Map<String, Object> entityInfo = new HashMap<String, Object>();
     entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app");
-    entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
-        "test app type");
+    entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, "test app type");
     entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1");
     if (!missingQueue) {
-      entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
-          "test queue");
+      entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
     }
-    entityInfo.put(
-        ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
-    entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
-        Priority.newInstance(0));
-    entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
-        Integer.MAX_VALUE + 1L);
+    entityInfo.put(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
+    entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, Priority.newInstance(0));
+    entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L);
     entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
     entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
     if (!missingPreemptMetrics) {
@@ -544,8 +509,7 @@ private static TimelineEntity createApplicationTimelineEntity(
     if (emptyACLs) {
       entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
     } else {
-      entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
-          "user2");
+      entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "user2");
     }
     Set<String> appTags = new HashSet<String>();
     appTags.add("Test_APP_TAGS_1");
@@ -557,16 +521,13 @@ private static TimelineEntity createApplicationTimelineEntity(
     tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
     entity.addEvent(tEvent);
     tEvent = new TimelineEvent();
-    tEvent.setEventType(
-        ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+    tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
     Map<String, Object> eventInfo = new HashMap<String, Object>();
-    eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
-        "test diagnostics info");
+    eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
     eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
         FinalApplicationStatus.UNDEFINED.toString());
-    eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
-        state.toString());
+    eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, state.toString());
     if (!noAttemptId) {
       eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
           ApplicationAttemptId.newInstance(appId, 1));
@@ -577,60 +538,51 @@ private static TimelineEntity createApplicationTimelineEntity(
     // after YARN_APPLICATION_FINISHED
     // The final YarnApplicationState should not be changed
     tEvent = new TimelineEvent();
-    tEvent.setEventType(
-        ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE);
+    tEvent.setEventType(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId());
     eventInfo = new HashMap<String, Object>();
-    eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
-        YarnApplicationState.KILLED);
+    eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.KILLED);
     tEvent.setEventInfo(eventInfo);
     entity.addEvent(tEvent);
     if (enableUpdateEvent) {
       tEvent = new TimelineEvent();
       long updatedTimeIndex = 4L;
-      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue",
-          5);
+      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 5);
       entity.addEvent(tEvent);
       // Change priority alone
       tEvent = new TimelineEvent();
-      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue",
-          6);
+      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 6);
       // Now change queue
       tEvent = new TimelineEvent();
-      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++,
-          "changed queue1", 6);
+      createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue1", 6);
       entity.addEvent(tEvent);
     }
     return entity;
   }
 
-  private static void createAppModifiedEvent(ApplicationId appId,
-      TimelineEvent tEvent, long updatedTimeIndex, String queue, int priority) {
+  private static void createAppModifiedEvent(ApplicationId appId, TimelineEvent tEvent,
+      long updatedTimeIndex, String queue, int priority) {
     tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId());
     Map<String, Object> eventInfo = new HashMap<String, Object>();
     eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue);
-    eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
-        priority);
+    eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, priority);
     tEvent.setEventInfo(eventInfo);
   }
 
-  private static TimelineEntity createAppAttemptTimelineEntity(
-      ApplicationAttemptId appAttemptId) {
+  private static TimelineEntity createAppAttemptTimelineEntity(ApplicationAttemptId appAttemptId) {
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE);
     entity.setEntityId(appAttemptId.toString());
     entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
     entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
         appAttemptId.getApplicationId().toString());
-    entity.addPrimaryFilter(
-        TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
+    entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
     Map<String, Object> eventInfo = new HashMap<String, Object>();
-    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO,
-        "test tracking url");
+    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url");
     eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO,
         "test original tracking url");
     eventInfo.put(AppAttemptMetricsConstants.HOST_INFO, "test host");
@@ -643,12 +595,10 @@ private static TimelineEntity createAppAttemptTimelineEntity(
     tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
     eventInfo = new HashMap<String, Object>();
-    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO,
-        "test tracking url");
+    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url");
     eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO,
         "test original tracking url");
-    eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO,
-        "test diagnostics info");
+    eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info");
     eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO,
         FinalApplicationStatus.UNDEFINED.toString());
     eventInfo.put(AppAttemptMetricsConstants.STATE_INFO,
@@ -665,35 +615,28 @@ private static TimelineEntity createContainerEntity(ContainerId containerId) {
     entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
     entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
         containerId.getApplicationAttemptId().toString());
-    entity.addPrimaryFilter(
-        TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
+    entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
     Map<String, Object> entityInfo = new HashMap<String, Object>();
     entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, -1);
     entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, -1);
-    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO,
-        "test host");
+    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, "test host");
     entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, 100);
-    entityInfo
-        .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1);
-    entityInfo.put(ContainerMetricsConstants
-        .ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234");
+    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1);
+    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234");
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
     entity.addEvent(tEvent);
-    ;
     tEvent = new TimelineEvent();
     tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
     tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
     Map<String, Object> eventInfo = new HashMap<String, Object>();
-    eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO,
-        "test diagnostics info");
+    eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info");
     eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, -1);
-    eventInfo.put(ContainerMetricsConstants.STATE_INFO,
-        ContainerState.COMPLETE.toString());
+    eventInfo.put(ContainerMetricsConstants.STATE_INFO, ContainerState.COMPLETE.toString());
     tEvent.setEventInfo(eventInfo);
     entity.addEvent(tEvent);
     return entity;
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
index b9d1c5e492f12..eb3db5eee190c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
@@ -18,9 +18,17 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.lib.StaticUserWebFilter;
@@ -33,23 +41,18 @@
 import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
-import org.junit.Assert;
-import org.junit.Test;
 
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestApplicationHistoryServer {
 
   // simple test init/start/stop ApplicationHistoryServer. Status should change.
-  @Test(timeout = 60000)
-  public void testStartStopServer() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testStartStopServer() throws Exception {
     ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
     Configuration config = new YarnConfiguration();
     config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
@@ -65,7 +68,7 @@ public void testStartStopServer() throws Exception {
         historyServer.start();
         fail();
       } catch (IllegalArgumentException e) {
-        Assert.assertTrue(e.getMessage().contains(
+        assertTrue(e.getMessage().contains(
             YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT));
       }
       config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
@@ -89,8 +92,9 @@ public void testStartStopServer() throws Exception {
   }
 
   // test launch method
-  @Test(timeout = 60000)
-  public void testLaunch() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testLaunch() throws Exception {
     ExitUtil.disableSystemExit();
     ApplicationHistoryServer historyServer = null;
     try {
@@ -109,34 +113,37 @@ public void testLaunch() throws Exception {
     }
   }
 
- //test launch method with -D arguments
- @Test(timeout = 60000)
- public void testLaunchWithArguments() throws Exception {
-   ExitUtil.disableSystemExit();
-   ApplicationHistoryServer historyServer = null;
-   try {
-     // Not able to modify the config of this test case,
-     // but others have been customized to avoid conflicts
-     String[] args = new String[2];
-     args[0]="-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000";
-     args[1]="-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200";
-     historyServer =
-         ApplicationHistoryServer.launchAppHistoryServer(args);
-     Configuration conf = historyServer.getConfig();
-     assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
-     assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
-   } catch (ExitUtil.ExitException e) {
-     assertEquals(0, e.status);
-     ExitUtil.resetFirstExitException();
-     fail();
-   } finally {
-     if (historyServer != null) {
-       historyServer.stop();
-     }
-   }
- }
-  @Test(timeout = 240000)
-  public void testFilterOverrides() throws Exception {
+  //test launch method with -D arguments
+  @Test
+  @Timeout(60000)
+  void testLaunchWithArguments() throws Exception {
+    ExitUtil.disableSystemExit();
+    ApplicationHistoryServer historyServer = null;
+    try {
+      // Not able to modify the config of this test case,
+      // but others have been customized to avoid conflicts
+      String[] args = new String[2];
+      args[0] = "-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000";
+      args[1] = "-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200";
+      historyServer =
+          ApplicationHistoryServer.launchAppHistoryServer(args);
+      Configuration conf = historyServer.getConfig();
+      assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
+      assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
+    } catch (ExitUtil.ExitException e) {
+      assertEquals(0, e.status);
+      ExitUtil.resetFirstExitException();
+      fail();
+    } finally {
+      if (historyServer != null) {
+        historyServer.stop();
+      }
+    }
+  }
+
+  @Test
+  @Timeout(240000)
+  void testFilterOverrides() throws Exception {
 
     HashMap<String, String> driver = new HashMap<String, String>();
     driver.put("", TimelineAuthenticationFilterInitializer.class.getName());
@@ -144,15 +151,15 @@ public void testFilterOverrides() throws Exception {
         StaticUserWebFilter.class.getName() + "," +
             TimelineAuthenticationFilterInitializer.class.getName());
     driver.put(AuthenticationFilterInitializer.class.getName(),
-      TimelineAuthenticationFilterInitializer.class.getName());
+        TimelineAuthenticationFilterInitializer.class.getName());
     driver.put(TimelineAuthenticationFilterInitializer.class.getName(),
-      TimelineAuthenticationFilterInitializer.class.getName());
+        TimelineAuthenticationFilterInitializer.class.getName());
     driver.put(AuthenticationFilterInitializer.class.getName() + ","
         + TimelineAuthenticationFilterInitializer.class.getName(),
-      TimelineAuthenticationFilterInitializer.class.getName());
+        TimelineAuthenticationFilterInitializer.class.getName());
     driver.put(AuthenticationFilterInitializer.class.getName() + ", "
         + TimelineAuthenticationFilterInitializer.class.getName(),
-      TimelineAuthenticationFilterInitializer.class.getName());
+        TimelineAuthenticationFilterInitializer.class.getName());
 
     for (Map.Entry<String, String> entry : driver.entrySet()) {
       String filterInitializer = entry.getKey();
@@ -176,8 +183,9 @@ public void testFilterOverrides() throws Exception {
     }
   }
 
-  @Test(timeout = 240000)
-  public void testHostedUIs() throws Exception {
+  @Test
+  @Timeout(240000)
+  void testHostedUIs() throws Exception {
 
     ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
     Configuration config = new YarnConfiguration();
@@ -209,8 +217,8 @@ public void testHostedUIs() throws Exception {
     } finally {
       historyServer.stop();
     }
-    assertEquals("Web file contents should be the same as on disk contents",
-        diskFileStr, connFileStr);
+    assertEquals(diskFileStr, connFileStr,
+        "Web file contents should be the same as on disk contents");
   }
   private String readInputStream(InputStream input) throws Exception {
     ByteArrayOutputStream data = new ByteArrayOutputStream();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
index 6b068c132d4c7..64b11eb4f1f1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
@@ -22,15 +22,12 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 
-import org.junit.Assert;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -45,12 +42,19 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 public class TestFileSystemApplicationHistoryStore extends
     ApplicationHistoryStoreTestUtils {
@@ -61,7 +65,7 @@ public class TestFileSystemApplicationHistoryStore extends
   private FileSystem fs;
   private Path fsWorkingPath;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     fs = new RawLocalFileSystem();
     initAndStartStore(fs);
@@ -75,8 +79,7 @@ private void initAndStartStore(final FileSystem fs) throws IOException,
         new Path("target",
           TestFileSystemApplicationHistoryStore.class.getSimpleName());
     fs.delete(fsWorkingPath, true);
-    conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
-      fsWorkingPath.toString());
+    conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
     store = new FileSystemApplicationHistoryStore() {
       @Override
       protected FileSystem getFileSystem(Path path, Configuration conf) {
@@ -87,7 +90,7 @@ protected FileSystem getFileSystem(Path path, Configuration conf) {
     store.start();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
     fs.delete(fsWorkingPath, true);
@@ -95,7 +98,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testReadWriteHistoryData() throws IOException {
+  void testReadWriteHistoryData() throws IOException {
     LOG.info("Starting testReadWriteHistoryData");
     testWriteHistoryData(5);
     testReadHistoryData(5);
@@ -146,58 +149,56 @@ private void testReadHistoryData(
       int num, boolean missingContainer, boolean missingApplicationAttempt)
           throws IOException {
     // read application history data
-    Assert.assertEquals(num, store.getAllApplications().size());
+    assertEquals(num, store.getAllApplications().size());
     for (int i = 1; i <= num; ++i) {
       ApplicationId appId = ApplicationId.newInstance(0, i);
       ApplicationHistoryData appData = store.getApplication(appId);
-      Assert.assertNotNull(appData);
-      Assert.assertEquals(appId.toString(), appData.getApplicationName());
-      Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo());
+      assertNotNull(appData);
+      assertEquals(appId.toString(), appData.getApplicationName());
+      assertEquals(appId.toString(), appData.getDiagnosticsInfo());
 
       // read application attempt history data
-      Assert.assertEquals(num, store.getApplicationAttempts(appId).size());
+      assertEquals(num, store.getApplicationAttempts(appId).size());
       for (int j = 1; j <= num; ++j) {
         ApplicationAttemptId appAttemptId =
             ApplicationAttemptId.newInstance(appId, j);
         ApplicationAttemptHistoryData attemptData =
             store.getApplicationAttempt(appAttemptId);
-        Assert.assertNotNull(attemptData);
-        Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
+        assertNotNull(attemptData);
+        assertEquals(appAttemptId.toString(), attemptData.getHost());
         
         if (missingApplicationAttempt && j == num) {
-          Assert.assertNull(attemptData.getDiagnosticsInfo());
+          assertNull(attemptData.getDiagnosticsInfo());
           continue;
         } else {
-          Assert.assertEquals(appAttemptId.toString(),
+          assertEquals(appAttemptId.toString(),
               attemptData.getDiagnosticsInfo());
         }
 
         // read container history data
-        Assert.assertEquals(num, store.getContainers(appAttemptId).size());
+        assertEquals(num, store.getContainers(appAttemptId).size());
         for (int k = 1; k <= num; ++k) {
           ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
           ContainerHistoryData containerData = store.getContainer(containerId);
-          Assert.assertNotNull(containerData);
-          Assert.assertEquals(Priority.newInstance(containerId.getId()),
-            containerData.getPriority());
+          assertNotNull(containerData);
+          assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority());
           if (missingContainer && k == num) {
-            Assert.assertNull(containerData.getDiagnosticsInfo());
+            assertNull(containerData.getDiagnosticsInfo());
           } else {
-            Assert.assertEquals(containerId.toString(),
+            assertEquals(containerId.toString(),
                 containerData.getDiagnosticsInfo());
           }
         }
         ContainerHistoryData masterContainer =
             store.getAMContainer(appAttemptId);
-        Assert.assertNotNull(masterContainer);
-        Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1),
-          masterContainer.getContainerId());
+        assertNotNull(masterContainer);
+        assertEquals(ContainerId.newContainerId(appAttemptId, 1), masterContainer.getContainerId());
       }
     }
   }
 
   @Test
-  public void testWriteAfterApplicationFinish() throws IOException {
+  void testWriteAfterApplicationFinish() throws IOException {
     LOG.info("Starting testWriteAfterApplicationFinish");
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     writeApplicationStartData(appId);
@@ -207,34 +208,34 @@ public void testWriteAfterApplicationFinish() throws IOException {
         ApplicationAttemptId.newInstance(appId, 1);
     try {
       writeApplicationAttemptStartData(appAttemptId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is not opened"));
+      assertTrue(e.getMessage().contains("is not opened"));
     }
     try {
       writeApplicationAttemptFinishData(appAttemptId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is not opened"));
+      assertTrue(e.getMessage().contains("is not opened"));
     }
     // write container history data
     ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
     try {
       writeContainerStartData(containerId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is not opened"));
+      assertTrue(e.getMessage().contains("is not opened"));
     }
     try {
       writeContainerFinishData(containerId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is not opened"));
+      assertTrue(e.getMessage().contains("is not opened"));
     }
   }
 
   @Test
-  public void testMassiveWriteContainerHistoryData() throws IOException {
+  void testMassiveWriteContainerHistoryData() throws IOException {
     LOG.info("Starting testMassiveWriteContainerHistoryData");
     long mb = 1024 * 1024;
     long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
@@ -249,25 +250,25 @@ public void testMassiveWriteContainerHistoryData() throws IOException {
     }
     writeApplicationFinishData(appId);
     long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
-    Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20);
+    assertTrue((usedDiskAfter - usedDiskBefore) < 20);
   }
 
   @Test
-  public void testMissingContainerHistoryData() throws IOException {
+  void testMissingContainerHistoryData() throws IOException {
     LOG.info("Starting testMissingContainerHistoryData");
     testWriteHistoryData(3, true, false);
     testReadHistoryData(3, true, false);
   }
-  
+
   @Test
-  public void testMissingApplicationAttemptHistoryData() throws IOException {
+  void testMissingApplicationAttemptHistoryData() throws IOException {
     LOG.info("Starting testMissingApplicationAttemptHistoryData");
     testWriteHistoryData(3, false, true);
     testReadHistoryData(3, false, true);
   }
 
   @Test
-  public void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
+  void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
     LOG.info("Starting testInitExistingWorkingDirectoryInSafeMode");
     tearDown();
 
@@ -280,7 +281,7 @@ public void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
     try {
       initAndStartStore(fileSystem);
     } catch (Exception e) {
-      Assert.fail("Exception should not be thrown: " + e);
+      fail("Exception should not be thrown: " + e);
     }
 
     // Make sure that directory creation was not attempted
@@ -289,7 +290,7 @@ public void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
   }
 
   @Test
-  public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception {
+  void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception {
     LOG.info("Starting testInitNonExistingWorkingDirectoryInSafeMode");
     tearDown();
 
@@ -302,7 +303,7 @@ public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception {
 
     try {
       initAndStartStore(fileSystem);
-      Assert.fail("Exception should have been thrown");
+      fail("Exception should have been thrown");
     } catch (Exception e) {
       // Expected failure
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
index 556db2beaf4b8..8a1b980680925 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
@@ -20,7 +20,8 @@
 
 import java.io.IOException;
 
-import org.junit.Assert;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -29,27 +30,30 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestMemoryApplicationHistoryStore extends
     ApplicationHistoryStoreTestUtils {
 
-  @Before
+  @BeforeEach
   public void setup() {
     store = new MemoryApplicationHistoryStore();
   }
 
   @Test
-  public void testReadWriteApplicationHistory() throws Exception {
+  void testReadWriteApplicationHistory() throws Exception {
     // Out of order
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     try {
       writeApplicationFinishData(appId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains(
-        "is stored before the start information"));
+      assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
     }
     // Normal
     int numApps = 5;
@@ -58,42 +62,42 @@ public void testReadWriteApplicationHistory() throws Exception {
       writeApplicationStartData(appId);
       writeApplicationFinishData(appId);
     }
-    Assert.assertEquals(numApps, store.getAllApplications().size());
+    assertEquals(numApps, store.getAllApplications().size());
     for (int i = 1; i <= numApps; ++i) {
       appId = ApplicationId.newInstance(0, i);
       ApplicationHistoryData data = store.getApplication(appId);
-      Assert.assertNotNull(data);
-      Assert.assertEquals(appId.toString(), data.getApplicationName());
-      Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo());
+      assertNotNull(data);
+      assertEquals(appId.toString(), data.getApplicationName());
+      assertEquals(appId.toString(), data.getDiagnosticsInfo());
     }
     // Write again
     appId = ApplicationId.newInstance(0, 1);
     try {
       writeApplicationStartData(appId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
     try {
       writeApplicationFinishData(appId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
   }
 
   @Test
-  public void testReadWriteApplicationAttemptHistory() throws Exception {
+  void testReadWriteApplicationAttemptHistory() throws Exception {
     // Out of order
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
     try {
       writeApplicationAttemptFinishData(appAttemptId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains(
-        "is stored before the start information"));
+      assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
     }
     // Normal
     int numAppAttempts = 5;
@@ -103,36 +107,36 @@ public void testReadWriteApplicationAttemptHistory() throws Exception {
       writeApplicationAttemptStartData(appAttemptId);
       writeApplicationAttemptFinishData(appAttemptId);
     }
-    Assert.assertEquals(numAppAttempts, store.getApplicationAttempts(appId)
-      .size());
+    assertEquals(numAppAttempts, store.getApplicationAttempts(appId)
+        .size());
     for (int i = 1; i <= numAppAttempts; ++i) {
       appAttemptId = ApplicationAttemptId.newInstance(appId, i);
       ApplicationAttemptHistoryData data =
           store.getApplicationAttempt(appAttemptId);
-      Assert.assertNotNull(data);
-      Assert.assertEquals(appAttemptId.toString(), data.getHost());
-      Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
+      assertNotNull(data);
+      assertEquals(appAttemptId.toString(), data.getHost());
+      assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
     }
     writeApplicationFinishData(appId);
     // Write again
     appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
     try {
       writeApplicationAttemptStartData(appAttemptId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
     try {
       writeApplicationAttemptFinishData(appAttemptId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
   }
 
   @SuppressWarnings("deprecation")
   @Test
-  public void testReadWriteContainerHistory() throws Exception {
+  void testReadWriteContainerHistory() throws Exception {
     // Out of order
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
@@ -140,10 +144,10 @@ public void testReadWriteContainerHistory() throws Exception {
     ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
     try {
       writeContainerFinishData(containerId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains(
-        "is stored before the start information"));
+      assertTrue(e.getMessage().contains(
+          "is stored before the start information"));
     }
     // Normal
     writeApplicationAttemptStartData(appAttemptId);
@@ -153,39 +157,38 @@ public void testReadWriteContainerHistory() throws Exception {
       writeContainerStartData(containerId);
       writeContainerFinishData(containerId);
     }
-    Assert
-      .assertEquals(numContainers, store.getContainers(appAttemptId).size());
+    assertEquals(numContainers, store.getContainers(appAttemptId).size());
     for (int i = 1; i <= numContainers; ++i) {
       containerId = ContainerId.newContainerId(appAttemptId, i);
       ContainerHistoryData data = store.getContainer(containerId);
-      Assert.assertNotNull(data);
-      Assert.assertEquals(Priority.newInstance(containerId.getId()),
-        data.getPriority());
-      Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo());
+      assertNotNull(data);
+      assertEquals(Priority.newInstance(containerId.getId()),
+          data.getPriority());
+      assertEquals(containerId.toString(), data.getDiagnosticsInfo());
     }
     ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
-    Assert.assertNotNull(masterContainer);
-    Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1),
-      masterContainer.getContainerId());
+    assertNotNull(masterContainer);
+    assertEquals(ContainerId.newContainerId(appAttemptId, 1),
+        masterContainer.getContainerId());
     writeApplicationAttemptFinishData(appAttemptId);
     // Write again
     containerId = ContainerId.newContainerId(appAttemptId, 1);
     try {
       writeContainerStartData(containerId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
     try {
       writeContainerFinishData(containerId);
-      Assert.fail();
+      fail();
     } catch (IOException e) {
-      Assert.assertTrue(e.getMessage().contains("is already stored"));
+      assertTrue(e.getMessage().contains("is already stored"));
     }
   }
 
   @Test
-  public void testMassiveWriteContainerHistory() throws IOException {
+  void testMassiveWriteContainerHistory() throws IOException {
     long mb = 1024 * 1024;
     Runtime runtime = Runtime.getRuntime();
     long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
@@ -199,7 +202,7 @@ public void testMassiveWriteContainerHistory() throws IOException {
       writeContainerFinishData(containerId);
     }
     long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
-    Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
+    assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
index 49fcc58b2ae49..b5bd66937a309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
@@ -18,10 +18,12 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
 
-import static org.apache.hadoop.yarn.webapp.Params.TITLE;
-import static org.mockito.Mockito.mock;
-
 import java.util.Map;
+
+import com.google.inject.Injector;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -38,11 +40,11 @@
 import org.apache.hadoop.yarn.util.StringHelper;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.test.WebAppTests;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import com.google.inject.Injector;
+import static org.apache.hadoop.yarn.webapp.Params.TITLE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
 
 public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
 
@@ -50,46 +52,45 @@ public void setApplicationHistoryStore(ApplicationHistoryStore store) {
     this.store = store;
   }
 
-  @Before
+  @BeforeEach
   public void setup() {
     store = new MemoryApplicationHistoryStore();
   }
 
   @Test
-  public void testAppControllerIndex() throws Exception {
+  void testAppControllerIndex() throws Exception {
     ApplicationHistoryManager ahManager = mock(ApplicationHistoryManager.class);
     Injector injector =
         WebAppTests.createMockInjector(ApplicationHistoryManager.class,
-          ahManager);
+            ahManager);
     AHSController controller = injector.getInstance(AHSController.class);
     controller.index();
-    Assert
-      .assertEquals("Application History", controller.get(TITLE, "unknown"));
+    assertEquals("Application History", controller.get(TITLE, "unknown"));
   }
 
   @Test
-  public void testView() throws Exception {
+  void testView() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
-          mockApplicationHistoryClientService(5, 1, 1));
+            mockApplicationHistoryClientService(5, 1, 1));
     AHSView ahsViewInstance = injector.getInstance(AHSView.class);
 
     ahsViewInstance.render();
     WebAppTests.flushOutput(injector);
 
     ahsViewInstance.set(YarnWebParams.APP_STATE,
-      YarnApplicationState.FAILED.toString());
+        YarnApplicationState.FAILED.toString());
     ahsViewInstance.render();
     WebAppTests.flushOutput(injector);
 
     ahsViewInstance.set(YarnWebParams.APP_STATE, StringHelper.cjoin(
-      YarnApplicationState.FAILED.toString(), YarnApplicationState.KILLED));
+        YarnApplicationState.FAILED.toString(), YarnApplicationState.KILLED));
     ahsViewInstance.render();
     WebAppTests.flushOutput(injector);
   }
 
   @Test
-  public void testAPPViewNaturalSortType() throws Exception {
+  void testAPPViewNaturalSortType() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
             mockApplicationHistoryClientService(5, 1, 1));
@@ -100,11 +101,11 @@ public void testAPPViewNaturalSortType() throws Exception {
     Map<String, String> moreParams =
         ahsViewInstance.context().requestContext().moreParams();
     String appTableColumnsMeta = moreParams.get("ui.dataTables.apps.init");
-    Assert.assertTrue(appTableColumnsMeta.indexOf("natural") != -1);
+    assertTrue(appTableColumnsMeta.indexOf("natural") != -1);
   }
 
   @Test
-  public void testAboutPage() throws Exception {
+  void testAboutPage() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
             mockApplicationHistoryClientService(0, 0, 0));
@@ -118,23 +119,23 @@ public void testAboutPage() throws Exception {
   }
 
   @Test
-  public void testAppPage() throws Exception {
+  void testAppPage() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
-          mockApplicationHistoryClientService(1, 5, 1));
+            mockApplicationHistoryClientService(1, 5, 1));
     AppPage appPageInstance = injector.getInstance(AppPage.class);
 
     appPageInstance.render();
     WebAppTests.flushOutput(injector);
 
     appPageInstance.set(YarnWebParams.APPLICATION_ID, ApplicationId
-      .newInstance(0, 1).toString());
+        .newInstance(0, 1).toString());
     appPageInstance.render();
     WebAppTests.flushOutput(injector);
   }
 
   @Test
-  public void testAppPageNaturalSortType() throws Exception {
+  void testAppPageNaturalSortType() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
             mockApplicationHistoryClientService(1, 5, 1));
@@ -146,14 +147,14 @@ public void testAppPageNaturalSortType() throws Exception {
         appPageInstance.context().requestContext().moreParams();
     String attemptsTableColumnsMeta =
         moreParams.get("ui.dataTables.attempts.init");
-    Assert.assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1);
+    assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1);
   }
 
   @Test
-  public void testAppAttemptPage() throws Exception {
+  void testAppAttemptPage() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
-          mockApplicationHistoryClientService(1, 1, 5));
+            mockApplicationHistoryClientService(1, 1, 5));
     AppAttemptPage appAttemptPageInstance =
         injector.getInstance(AppAttemptPage.class);
 
@@ -161,14 +162,14 @@ public void testAppAttemptPage() throws Exception {
     WebAppTests.flushOutput(injector);
 
     appAttemptPageInstance.set(YarnWebParams.APPLICATION_ATTEMPT_ID,
-      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1)
-        .toString());
+        ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1)
+            .toString());
     appAttemptPageInstance.render();
     WebAppTests.flushOutput(injector);
   }
 
   @Test
-  public void testAppAttemptPageNaturalSortType() throws Exception {
+  void testAppAttemptPageNaturalSortType() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
             mockApplicationHistoryClientService(1, 1, 5));
@@ -179,14 +180,14 @@ public void testAppAttemptPageNaturalSortType() throws Exception {
     Map<String, String> moreParams =
         appAttemptPageInstance.context().requestContext().moreParams();
     String tableColumnsMeta = moreParams.get("ui.dataTables.containers.init");
-    Assert.assertTrue(tableColumnsMeta.indexOf("natural") != -1);
+    assertTrue(tableColumnsMeta.indexOf("natural") != -1);
   }
 
   @Test
-  public void testContainerPage() throws Exception {
+  void testContainerPage() throws Exception {
     Injector injector =
         WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
-          mockApplicationHistoryClientService(1, 1, 1));
+            mockApplicationHistoryClientService(1, 1, 1));
     ContainerPage containerPageInstance =
         injector.getInstance(ContainerPage.class);
 
@@ -194,11 +195,11 @@ public void testContainerPage() throws Exception {
     WebAppTests.flushOutput(injector);
 
     containerPageInstance.set(
-      YarnWebParams.CONTAINER_ID,
-      ContainerId
-        .newContainerId(
-          ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1),
-          1).toString());
+        YarnWebParams.CONTAINER_ID,
+        ContainerId
+            .newContainerId(
+                ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1),
+                1).toString());
     containerPageInstance.render();
     WebAppTests.flushOutput(injector);
   }
@@ -230,7 +231,7 @@ ApplicationHistoryClientService mockApplicationHistoryClientService(int numApps,
 
   class MockApplicationHistoryManagerImpl extends ApplicationHistoryManagerImpl {
 
-    public MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) {
+    MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) {
       super();
       init(new YarnConfiguration());
       start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 29890b2c7cd3f..fbcf9e9eb374e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -18,16 +18,6 @@
 
 package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-
 import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URL;
@@ -36,11 +26,31 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
-
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.core.MediaType;
+
+import com.google.inject.Guice;
+import com.google.inject.Singleton;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -57,6 +67,7 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
 import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
@@ -72,41 +83,28 @@
 import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
 import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-import com.google.inject.Guice;
-import com.google.inject.Singleton;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 
-@RunWith(Parameterized.class)
 public class TestAHSWebServices extends JerseyTestBase {
 
   private static ApplicationHistoryClientService historyClientService;
   private static AHSWebServices ahsWebservice;
-  private static final String[] USERS = new String[] { "foo" , "bar" };
+  private static final String[] USERS = new String[]{"foo", "bar"};
   private static final int MAX_APPS = 6;
   private static Configuration conf;
   private static FileSystem fs;
@@ -115,7 +113,7 @@ public class TestAHSWebServices extends JerseyTestBase {
   private static final String NM_WEBADDRESS = "test-nm-web-address:9999";
   private static final String NM_ID = "test:1234";
 
-  @BeforeClass
+  @BeforeAll
   public static void setupClass() throws Exception {
     conf = new YarnConfiguration();
     TimelineStore store =
@@ -153,7 +151,7 @@ protected void serviceStart() throws Exception {
         Guice.createInjector(new WebServletModule()));
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDownClass() throws Exception {
     if (historyClientService != null) {
       historyClientService.stop();
@@ -162,9 +160,8 @@ public static void tearDownClass() throws Exception {
     fs.delete(new Path(rootLogDir), true);
   }
 
-  @Parameterized.Parameters
   public static Collection<Object[]> rounds() {
-    return Arrays.asList(new Object[][] { { 0 }, { 1 } });
+    return Arrays.asList(new Object[][]{{0}, {1}});
   }
 
   private static class WebServletModule extends ServletModule {
@@ -179,7 +176,7 @@ protected void configureServlets() {
     }
   }
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     super.setUp();
     GuiceServletConfig.setInjector(
@@ -199,44 +196,43 @@ protected Properties getConfiguration(String configPrefix,
     }
   }
 
-  private int round;
-
-  public TestAHSWebServices(int round) {
+  public TestAHSWebServices() {
     super(new WebAppDescriptor.Builder(
-      "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
-      .contextListenerClass(GuiceServletConfig.class)
-      .filterClass(com.google.inject.servlet.GuiceFilter.class)
-      .contextPath("jersey-guice-filter").servletPath("/").build());
-    this.round = round;
+        "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
+        .contextListenerClass(GuiceServletConfig.class)
+        .filterClass(com.google.inject.servlet.GuiceFilter.class)
+        .contextPath("jersey-guice-filter").servletPath("/").build());
   }
 
-  @Test
-  public void testInvalidApp() {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidApp(int round) {
     ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     assertResponseStatusCode("404 not found expected",
         Status.NOT_FOUND, response.getStatusInfo());
   }
 
-  @Test
-  public void testInvalidAttempt() {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidAttempt(int round) {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .path(appAttemptId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .path(appAttemptId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -245,8 +241,9 @@ public void testInvalidAttempt() {
         Status.NOT_FOUND, response.getStatusInfo());
   }
 
-  @Test
-  public void testInvalidContainer() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidContainer(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -255,12 +252,12 @@ public void testInvalidContainer() throws Exception {
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .path(appAttemptId.toString()).path("containers")
-          .path(containerId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .path(appAttemptId.toString()).path("containers")
+            .path(containerId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -269,27 +266,29 @@ public void testInvalidContainer() throws Exception {
         Status.NOT_FOUND, response.getStatusInfo());
   }
 
-  @Test
-  public void testInvalidUri() throws JSONException, Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidUri(int round) throws JSONException, Exception {
     WebResource r = resource();
     String responseStr = "";
     try {
       responseStr =
           r.path("ws").path("v1").path("applicationhistory").path("bogus")
-            .queryParam("user.name", USERS[round])
-            .accept(MediaType.APPLICATION_JSON).get(String.class);
+              .queryParam("user.name", USERS[round])
+              .accept(MediaType.APPLICATION_JSON).get(String.class);
       fail("should have thrown exception on invalid uri");
     } catch (UniformInterfaceException ue) {
       ClientResponse response = ue.getResponse();
       assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
 
       WebServicesTestUtils.checkStringMatch(
-        "error string exists and shouldn't", "", responseStr);
+          "error string exists and shouldn't", "", responseStr);
     }
   }
 
-  @Test
-  public void testInvalidUri2() throws JSONException, Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidUri2(int round) throws JSONException, Exception {
     WebResource r = resource();
     String responseStr = "";
     try {
@@ -300,31 +299,33 @@ public void testInvalidUri2() throws JSONException, Exception {
       ClientResponse response = ue.getResponse();
       assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
       WebServicesTestUtils.checkStringMatch(
-        "error string exists and shouldn't", "", responseStr);
+          "error string exists and shouldn't", "", responseStr);
     }
   }
 
-  @Test
-  public void testInvalidAccept() throws JSONException, Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testInvalidAccept(int round) throws JSONException, Exception {
     WebResource r = resource();
     String responseStr = "";
     try {
       responseStr =
           r.path("ws").path("v1").path("applicationhistory")
-            .queryParam("user.name", USERS[round])
-            .accept(MediaType.TEXT_PLAIN).get(String.class);
+              .queryParam("user.name", USERS[round])
+              .accept(MediaType.TEXT_PLAIN).get(String.class);
       fail("should have thrown exception on invalid uri");
     } catch (UniformInterfaceException ue) {
       ClientResponse response = ue.getResponse();
       assertResponseStatusCode(Status.INTERNAL_SERVER_ERROR,
           response.getStatusInfo());
       WebServicesTestUtils.checkStringMatch(
-        "error string exists and shouldn't", "", responseStr);
+          "error string exists and shouldn't", "", responseStr);
     }
   }
 
-  @Test
-  public void testAbout() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testAbout(int round) throws Exception {
     WebResource r = resource();
     ClientResponse response = r
         .path("ws").path("v1").path("applicationhistory").path("about")
@@ -335,43 +336,45 @@ public void testAbout() throws Exception {
     TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
     TimelineAbout expectedAbout =
         TimelineUtils.createTimelineAbout("Generic History Service API");
-    Assert.assertNotNull(
-        "Timeline service about response is null", actualAbout);
-    Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
-    Assert.assertEquals(expectedAbout.getTimelineServiceVersion(),
+    assertNotNull(
+        actualAbout, "Timeline service about response is null");
+    assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
+    assertEquals(expectedAbout.getTimelineServiceVersion(),
         actualAbout.getTimelineServiceVersion());
-    Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
+    assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
         actualAbout.getTimelineServiceBuildVersion());
-    Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
+    assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
         actualAbout.getTimelineServiceVersionBuiltOn());
-    Assert.assertEquals(expectedAbout.getHadoopVersion(),
+    assertEquals(expectedAbout.getHadoopVersion(),
         actualAbout.getHadoopVersion());
-    Assert.assertEquals(expectedAbout.getHadoopBuildVersion(),
+    assertEquals(expectedAbout.getHadoopBuildVersion(),
         actualAbout.getHadoopBuildVersion());
-    Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
+    assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
         actualAbout.getHadoopVersionBuiltOn());
   }
 
-  @Test
-  public void testAppsQuery() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testAppsQuery(int round) throws Exception {
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .queryParam("state", YarnApplicationState.FINISHED.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+            .queryParam("state", YarnApplicationState.FINISHED.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject apps = json.getJSONObject("apps");
-    assertEquals("incorrect number of elements", 1, apps.length());
+    assertEquals(1, apps.length(), "incorrect number of elements");
     JSONArray array = apps.getJSONArray("app");
-    assertEquals("incorrect number of elements", MAX_APPS, array.length());
+    assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
   }
 
-  @Test
-  public void testQueueQuery() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testQueueQuery(int round) throws Exception {
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
@@ -382,28 +385,30 @@ public void testQueueQuery() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject apps = json.getJSONObject("apps");
-    assertEquals("incorrect number of elements", 1, apps.length());
+    assertEquals(1, apps.length(), "incorrect number of elements");
     JSONArray array = apps.getJSONArray("app");
-    assertEquals("incorrect number of elements", MAX_APPS - 1,
-        array.length());
+    assertEquals(MAX_APPS - 1,
+        array.length(),
+        "incorrect number of elements");
   }
 
-  @Test
-  public void testSingleApp() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testSingleApp(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject app = json.getJSONObject("app");
     assertEquals(appId.toString(), app.getString("appId"));
     assertEquals("test app", app.get("name"));
@@ -414,23 +419,24 @@ public void testSingleApp() throws Exception {
     assertEquals("user1", app.get("user"));
     assertEquals("test app type", app.get("type"));
     assertEquals(FinalApplicationStatus.UNDEFINED.toString(),
-      app.get("finalAppStatus"));
+        app.get("finalAppStatus"));
     assertEquals(YarnApplicationState.FINISHED.toString(), app.get("appState"));
-    assertNotNull("Aggregate resource allocation is null",
-        app.get("aggregateResourceAllocation"));
-    assertNotNull("Aggregate Preempted Resource Allocation is null",
-        app.get("aggregatePreemptedResourceAllocation"));
+    assertNotNull(app.get("aggregateResourceAllocation"),
+        "Aggregate resource allocation is null");
+    assertNotNull(app.get("aggregatePreemptedResourceAllocation"),
+        "Aggregate Preempted Resource Allocation is null");
   }
 
-  @Test
-  public void testMultipleAttempts() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testMultipleAttempts(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -438,26 +444,27 @@ public void testMultipleAttempts() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject appAttempts = json.getJSONObject("appAttempts");
-    assertEquals("incorrect number of elements", 1, appAttempts.length());
+    assertEquals(1, appAttempts.length(), "incorrect number of elements");
     JSONArray array = appAttempts.getJSONArray("appAttempt");
-    assertEquals("incorrect number of elements", MAX_APPS, array.length());
+    assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
   }
 
-  @Test
-  public void testSingleAttempt() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testSingleAttempt(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .path(appAttemptId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .path(appAttemptId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -465,29 +472,30 @@ public void testSingleAttempt() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject appAttempt = json.getJSONObject("appAttempt");
     assertEquals(appAttemptId.toString(), appAttempt.getString("appAttemptId"));
     assertEquals("test host", appAttempt.getString("host"));
     assertEquals("test diagnostics info",
-      appAttempt.getString("diagnosticsInfo"));
+        appAttempt.getString("diagnosticsInfo"));
     assertEquals("test tracking url", appAttempt.getString("trackingUrl"));
     assertEquals(YarnApplicationAttemptState.FINISHED.toString(),
-      appAttempt.get("appAttemptState"));
+        appAttempt.get("appAttemptState"));
   }
 
-  @Test
-  public void testMultipleContainers() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testMultipleContainers(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .path(appAttemptId.toString()).path("containers")
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .path(appAttemptId.toString()).path("containers")
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -495,15 +503,16 @@ public void testMultipleContainers() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject containers = json.getJSONObject("containers");
-    assertEquals("incorrect number of elements", 1, containers.length());
+    assertEquals(1, containers.length(), "incorrect number of elements");
     JSONArray array = containers.getJSONArray("container");
-    assertEquals("incorrect number of elements", MAX_APPS, array.length());
+    assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
   }
 
-  @Test
-  public void testSingleContainer() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  void testSingleContainer(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -511,12 +520,12 @@ public void testSingleContainer() throws Exception {
     WebResource r = resource();
     ClientResponse response =
         r.path("ws").path("v1").path("applicationhistory").path("apps")
-          .path(appId.toString()).path("appattempts")
-          .path(appAttemptId.toString()).path("containers")
-          .path(containerId.toString())
-          .queryParam("user.name", USERS[round])
-          .accept(MediaType.APPLICATION_JSON)
-          .get(ClientResponse.class);
+            .path(appId.toString()).path("appattempts")
+            .path(appAttemptId.toString()).path("containers")
+            .path(containerId.toString())
+            .queryParam("user.name", USERS[round])
+            .accept(MediaType.APPLICATION_JSON)
+            .get(ClientResponse.class);
     if (round == 1) {
       assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
       return;
@@ -524,14 +533,14 @@ public void testSingleContainer() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     JSONObject json = response.getEntity(JSONObject.class);
-    assertEquals("incorrect number of elements", 1, json.length());
+    assertEquals(1, json.length(), "incorrect number of elements");
     JSONObject container = json.getJSONObject("container");
     assertEquals(containerId.toString(), container.getString("containerId"));
     assertEquals("test diagnostics info", container.getString("diagnosticsInfo"));
     assertEquals("-1", container.getString("allocatedMB"));
     assertEquals("-1", container.getString("allocatedVCores"));
     assertEquals(NodeId.newInstance("test host", 100).toString(),
-      container.getString("assignedNodeId"));
+        container.getString("assignedNodeId"));
     assertEquals("-1", container.getString("priority"));
     Configuration conf = new YarnConfiguration();
     assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
@@ -539,11 +548,13 @@ public void testSingleContainer() throws Exception {
         "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" +
         "container_0_0001_01_000001/user1", container.getString("logUrl"));
     assertEquals(ContainerState.COMPLETE.toString(),
-      container.getString("containerState"));
+        container.getString("containerState"));
   }
 
-  @Test(timeout = 10000)
-  public void testContainerLogsForFinishedApps() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  @Timeout(10000)
+  void testContainerLogsForFinishedApps(int round) throws Exception {
     String fileName = "syslog";
     String user = "user1";
     NodeId nodeId = NodeId.newInstance("test host", 100);
@@ -553,7 +564,6 @@ public void testContainerLogsForFinishedApps() throws Exception {
         appId, 1);
     ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
     ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100);
-
     TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs,
         rootLogDir, appId, Collections.singletonMap(containerId1,
             "Hello." + containerId1),
@@ -573,7 +583,6 @@ public void testContainerLogsForFinishedApps() throws Exception {
         .get(ClientResponse.class);
     String responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Hello." + containerId1));
-
     // Do the same test with new API
     r = resource();
     response = r.path("ws").path("v1")
@@ -584,7 +593,6 @@ public void testContainerLogsForFinishedApps() throws Exception {
         .get(ClientResponse.class);
     responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Hello." + containerId1));
-
     // test whether we can find container log from remote diretory if
     // the containerInfo for this container could not be fetched from AHS.
     r = resource();
@@ -596,7 +604,6 @@ public void testContainerLogsForFinishedApps() throws Exception {
         .get(ClientResponse.class);
     responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Hello." + containerId100));
-
     // Do the same test with new API
     r = resource();
     response = r.path("ws").path("v1")
@@ -607,14 +614,12 @@ public void testContainerLogsForFinishedApps() throws Exception {
         .get(ClientResponse.class);
     responseText = response.getEntity(String.class);
     assertTrue(responseText.contains("Hello." + containerId100));
-
     // create an application which can not be found from AHS
     ApplicationId appId100 = ApplicationId.newInstance(0, 100);
     ApplicationAttemptId appAttemptId100 = ApplicationAttemptId.newInstance(
         appId100, 1);
     ContainerId containerId1ForApp100 = ContainerId.newContainerId(
         appAttemptId100, 1);
-
     TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs,
         rootLogDir, appId100,
         Collections.singletonMap(containerId1ForApp100,
@@ -634,7 +639,6 @@ public void testContainerLogsForFinishedApps() throws Exception {
         "End of LogType:syslog".length() + 50) + "\n\n";
     int tailTextSize = "\nEnd of LogType:syslog\n".getBytes().length
         + tailEndSeparator.getBytes().length;
-
     String logMessage = "Hello." + containerId1ForApp100;
     int fileContentSize = logMessage.getBytes().length;
     // specify how many bytes we should get from logs
@@ -653,9 +657,8 @@ public void testContainerLogsForFinishedApps() throws Exception {
         (fullTextSize - fileContentSize) + 5);
     assertTrue(fullTextSize >= responseText.getBytes().length);
     assertEquals(new String(responseText.getBytes(),
-        (fullTextSize - fileContentSize - tailTextSize), 5),
+            (fullTextSize - fileContentSize - tailTextSize), 5),
         new String(logMessage.getBytes(), 0, 5));
-
     // specify how many bytes we should get from logs
     // if we specify a negative number, it would get the last n bytes from
     // container log
@@ -672,9 +675,8 @@ public void testContainerLogsForFinishedApps() throws Exception {
         (fullTextSize - fileContentSize) + 5);
     assertTrue(fullTextSize >= responseText.getBytes().length);
     assertEquals(new String(responseText.getBytes(),
-        (fullTextSize - fileContentSize - tailTextSize), 5),
+            (fullTextSize - fileContentSize - tailTextSize), 5),
         new String(logMessage.getBytes(), fileContentSize - 5, 5));
-
     // specify the bytes which is larger than the actual file size,
     // we would get the full logs
     r = resource();
@@ -700,8 +702,10 @@ public void testContainerLogsForFinishedApps() throws Exception {
     assertThat(responseText.getBytes()).hasSize(fullTextSize);
   }
 
-  @Test(timeout = 10000)
-  public void testContainerLogsForRunningApps() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  @Timeout(10000)
+  void testContainerLogsForRunningApps(int round) throws Exception {
     String fileName = "syslog";
     String user = "user1";
     ApplicationId appId = ApplicationId.newInstance(
@@ -826,8 +830,10 @@ public void testContainerLogsForRunningApps() throws Exception {
         + ContainerLogAggregationType.AGGREGATED));
   }
 
-  @Test(timeout = 10000)
-  public void testContainerLogsMetaForRunningApps() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  @Timeout(10000)
+  void testContainerLogsMetaForRunningApps(int round) throws Exception {
     String user = "user1";
     ApplicationId appId = ApplicationId.newInstance(
         1234, 1);
@@ -883,10 +889,11 @@ public void testContainerLogsMetaForRunningApps() throws Exception {
         .get(ClientResponse.class);
 
     List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<
-        List<ContainerLogsInfo>>(){});
+        List<ContainerLogsInfo>>(){
+    });
     assertTrue(responseText.size() == 2);
     for (ContainerLogsInfo logInfo : responseText) {
-      if(logInfo.getLogType().equals(
+      if (logInfo.getLogType().equals(
           ContainerLogAggregationType.AGGREGATED.toString())) {
         List<ContainerLogFileInfo> logMeta = logInfo
             .getContainerLogsInfo();
@@ -911,10 +918,11 @@ public void testContainerLogsMetaForRunningApps() throws Exception {
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     responseText = response.getEntity(new GenericType<
-        List<ContainerLogsInfo>>(){});
+        List<ContainerLogsInfo>>(){
+    });
     assertTrue(responseText.size() == 2);
     for (ContainerLogsInfo logInfo : responseText) {
-      if(logInfo.getLogType().equals(
+      if (logInfo.getLogType().equals(
           ContainerLogAggregationType.AGGREGATED.toString())) {
         List<ContainerLogFileInfo> logMeta = logInfo
             .getContainerLogsInfo();
@@ -929,8 +937,10 @@ public void testContainerLogsMetaForRunningApps() throws Exception {
     }
   }
 
-  @Test(timeout = 10000)
-  public void testContainerLogsMetaForFinishedApps() throws Exception {
+  @MethodSource("rounds")
+  @ParameterizedTest
+  @Timeout(10000)
+  void testContainerLogsMetaForFinishedApps(int round) throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
@@ -951,7 +961,8 @@ public void testContainerLogsMetaForFinishedApps() throws Exception {
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<
-        List<ContainerLogsInfo>>(){});
+        List<ContainerLogsInfo>>(){
+    });
     assertTrue(responseText.size() == 1);
     assertEquals(responseText.get(0).getLogType(),
         ContainerLogAggregationType.AGGREGATED.toString());
@@ -971,7 +982,7 @@ private static String getRedirectURL(String url) {
       // do not automatically follow the redirection
       // otherwise we get too many redirections exception
       conn.setInstanceFollowRedirects(false);
-      if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
+      if (conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) {
         redirectUrl = conn.getHeaderField("Location");
       }
     } catch (Exception e) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java
index ab0923173b937..6c671f7476339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java
@@ -17,52 +17,56 @@
  */
 package org.apache.hadoop.yarn.server.timeline;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.junit.Test;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class TestGenericObjectMapper {
 
   @Test
-  public void testEncoding() {
+  void testEncoding() {
     testEncoding(Long.MAX_VALUE);
     testEncoding(Long.MIN_VALUE);
-    testEncoding(0l);
-    testEncoding(128l);
-    testEncoding(256l);
-    testEncoding(512l);
-    testEncoding(-256l);
+    testEncoding(0L);
+    testEncoding(128L);
+    testEncoding(256L);
+    testEncoding(512L);
+    testEncoding(-256L);
   }
 
   private static void testEncoding(long l) {
     byte[] b = GenericObjectMapper.writeReverseOrderedLong(l);
-    assertEquals("error decoding", l,
-        GenericObjectMapper.readReverseOrderedLong(b, 0));
+    assertEquals(l,
+        GenericObjectMapper.readReverseOrderedLong(b, 0),
+        "error decoding");
     byte[] buf = new byte[16];
     System.arraycopy(b, 0, buf, 5, 8);
-    assertEquals("error decoding at offset", l,
-        GenericObjectMapper.readReverseOrderedLong(buf, 5));
+    assertEquals(l,
+        GenericObjectMapper.readReverseOrderedLong(buf, 5),
+        "error decoding at offset");
     if (l > Long.MIN_VALUE) {
       byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1);
-      assertEquals("error preserving ordering", 1,
-          WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length));
+      assertEquals(1,
+          WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length),
+          "error preserving ordering");
     }
     if (l < Long.MAX_VALUE) {
       byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1);
-      assertEquals("error preserving ordering", 1,
-          WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length));
+      assertEquals(1,
+          WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length),
+          "error preserving ordering");
     }
   }
 
@@ -71,20 +75,20 @@ private static void verify(Object o) throws IOException {
   }
 
   @Test
-  public void testValueTypes() throws IOException {
+  void testValueTypes() throws IOException {
     verify(Integer.MAX_VALUE);
     verify(Integer.MIN_VALUE);
     assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read(
         GenericObjectMapper.write((long) Integer.MAX_VALUE)));
     assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read(
         GenericObjectMapper.write((long) Integer.MIN_VALUE)));
-    verify((long)Integer.MAX_VALUE + 1l);
-    verify((long)Integer.MIN_VALUE - 1l);
+    verify((long) Integer.MAX_VALUE + 1L);
+    verify((long) Integer.MIN_VALUE - 1L);
 
     verify(Long.MAX_VALUE);
     verify(Long.MIN_VALUE);
 
-    assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l)));
+    assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42L)));
     verify(42);
     verify(1.23);
     verify("abc");
@@ -93,9 +97,9 @@ public void testValueTypes() throws IOException {
     list.add("123");
     list.add("abc");
     verify(list);
-    Map<String,String> map = new HashMap<String,String>();
-    map.put("k1","v1");
-    map.put("k2","v2");
+    Map<String, String> map = new HashMap<String, String>();
+    map.put("k1", "v1");
+    map.put("k2", "v2");
     verify(map);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
index f68a1c41c612c..c7166a1268c55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
@@ -17,10 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.timeline;
 
-import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
@@ -29,6 +25,14 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.Options;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -46,14 +50,13 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
-import org.fusesource.leveldbjni.JniDBFactory;
-import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Options;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
+
+import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -62,7 +65,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
   private File fsPath;
   private Configuration config = new YarnConfiguration();
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     fsContext = FileContext.getLocalFSFileContext();
     fsPath = new File("target", this.getClass().getSimpleName() +
@@ -79,14 +82,14 @@ public void setup() throws Exception {
     loadTestDomainData();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
     fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
   }
 
   @Test
-  public void testRootDirPermission() throws IOException {
+  void testRootDirPermission() throws IOException {
     FileSystem fs = FileSystem.getLocal(new YarnConfiguration());
     FileStatus file = fs.getFileStatus(
         new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME));
@@ -133,7 +136,7 @@ public void testGetEvents() throws IOException {
   }
 
   @Test
-  public void testCacheSizes() {
+  void testCacheSizes() {
     Configuration conf = new Configuration();
     assertEquals(10000, LeveldbTimelineStore.getStartTimeReadCacheSize(conf));
     assertEquals(10000, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf));
@@ -165,8 +168,8 @@ private boolean deleteNextEntity(String entityType, byte[] ts)
   }
 
   @Test
-  public void testGetEntityTypes() throws IOException {
-    List<String> entityTypes = ((LeveldbTimelineStore)store).getEntityTypes();
+  void testGetEntityTypes() throws IOException {
+    List<String> entityTypes = ((LeveldbTimelineStore) store).getEntityTypes();
     assertEquals(7, entityTypes.size());
     assertEquals("ACL_ENTITY_TYPE_1", entityTypes.get(0));
     assertEquals("OLD_ENTITY_TYPE_1", entityTypes.get(1));
@@ -177,7 +180,7 @@ public void testGetEntityTypes() throws IOException {
   }
 
   @Test
-  public void testDeleteEntities() throws IOException, InterruptedException {
+  void testDeleteEntities() throws IOException, InterruptedException {
     assertEquals(3, getEntities("type_1").size());
     assertEquals(1, getEntities("type_2").size());
 
@@ -191,7 +194,7 @@ public void testDeleteEntities() throws IOException, InterruptedException {
     List<TimelineEntity> entities = getEntities("type_2");
     assertEquals(1, entities.size());
     verifyEntityInfo(entityId2, entityType2, events2, Collections.singletonMap(
-        entityType1, Collections.singleton(entityId1b)), EMPTY_PRIMARY_FILTERS,
+            entityType1, Collections.singleton(entityId1b)), EMPTY_PRIMARY_FILTERS,
         EMPTY_MAP, entities.get(0), domainId1);
     entities = getEntitiesWithPrimaryFilter("type_1", userFilter);
     assertEquals(2, entities.size());
@@ -201,20 +204,20 @@ public void testDeleteEntities() throws IOException, InterruptedException {
     verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES,
         primaryFilters, otherInfo, entities.get(1), domainId2);
 
-    ((LeveldbTimelineStore)store).discardOldEntities(0L);
+    ((LeveldbTimelineStore) store).discardOldEntities(0L);
     assertEquals(2, getEntities("type_1").size());
     assertEquals(0, getEntities("type_2").size());
-    assertEquals(6, ((LeveldbTimelineStore)store).getEntityTypes().size());
+    assertEquals(6, ((LeveldbTimelineStore) store).getEntityTypes().size());
 
-    ((LeveldbTimelineStore)store).discardOldEntities(123L);
+    ((LeveldbTimelineStore) store).discardOldEntities(123L);
     assertEquals(0, getEntities("type_1").size());
     assertEquals(0, getEntities("type_2").size());
-    assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size());
+    assertEquals(0, ((LeveldbTimelineStore) store).getEntityTypes().size());
     assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
   }
 
   @Test
-  public void testDeleteEntitiesPrimaryFilters()
+  void testDeleteEntitiesPrimaryFilters()
       throws IOException, InterruptedException {
     Map<String, Set<Object>> primaryFilter =
         Collections.singletonMap("user", Collections.singleton(
@@ -243,28 +246,28 @@ public void testDeleteEntitiesPrimaryFilters()
     verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES,
         primaryFilters, otherInfo, entities.get(2), domainId2);
 
-    ((LeveldbTimelineStore)store).discardOldEntities(-123L);
+    ((LeveldbTimelineStore) store).discardOldEntities(-123L);
     assertEquals(1, getEntitiesWithPrimaryFilter("type_1", pfPair).size());
     assertEquals(3, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
 
-    ((LeveldbTimelineStore)store).discardOldEntities(123L);
+    ((LeveldbTimelineStore) store).discardOldEntities(123L);
     assertEquals(0, getEntities("type_1").size());
     assertEquals(0, getEntities("type_2").size());
-    assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size());
+    assertEquals(0, ((LeveldbTimelineStore) store).getEntityTypes().size());
 
     assertEquals(0, getEntitiesWithPrimaryFilter("type_1", pfPair).size());
     assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
   }
 
   @Test
-  public void testFromTsWithDeletion()
+  void testFromTsWithDeletion()
       throws IOException, InterruptedException {
     long l = System.currentTimeMillis();
     assertEquals(3, getEntitiesFromTs("type_1", l).size());
     assertEquals(1, getEntitiesFromTs("type_2", l).size());
     assertEquals(3, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
         l).size());
-    ((LeveldbTimelineStore)store).discardOldEntities(123L);
+    ((LeveldbTimelineStore) store).discardOldEntities(123L);
     assertEquals(0, getEntitiesFromTs("type_1", l).size());
     assertEquals(0, getEntitiesFromTs("type_2", l).size());
     assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter,
@@ -284,22 +287,22 @@ public void testFromTsWithDeletion()
   }
 
   @Test
-  public void testCheckVersion() throws IOException {
+  void testCheckVersion() throws IOException {
     LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store;
     // default version
     Version defaultVersion = dbStore.getCurrentVersion();
-    Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+    assertEquals(defaultVersion, dbStore.loadVersion());
 
     // compatible version
     Version compatibleVersion =
         Version.newInstance(defaultVersion.getMajorVersion(),
-          defaultVersion.getMinorVersion() + 2);
+            defaultVersion.getMinorVersion() + 2);
     dbStore.storeVersion(compatibleVersion);
-    Assert.assertEquals(compatibleVersion, dbStore.loadVersion());
+    assertEquals(compatibleVersion, dbStore.loadVersion());
     restartTimelineStore();
     dbStore = (LeveldbTimelineStore) store;
     // overwrite the compatible version
-    Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+    assertEquals(defaultVersion, dbStore.loadVersion());
 
     // incompatible version
     Version incompatibleVersion = Version.newInstance(
@@ -307,24 +310,24 @@ public void testCheckVersion() throws IOException {
     dbStore.storeVersion(incompatibleVersion);
     try {
       restartTimelineStore();
-      Assert.fail("Incompatible version, should expect fail here.");
+      fail("Incompatible version, should expect fail here.");
     } catch (ServiceStateException e) {
-      Assert.assertTrue("Exception message mismatch",
-          e.getMessage().contains("Incompatible version for timeline store"));
+      assertTrue(e.getMessage().contains("Incompatible version for timeline store"),
+          "Exception message mismatch");
     }
   }
 
   @Test
-  public void testValidateConfig() throws IOException {
+  void testValidateConfig() throws IOException {
     Configuration copyConfig = new YarnConfiguration(config);
     try {
       Configuration newConfig = new YarnConfiguration(copyConfig);
       newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
     }
     try {
@@ -333,9 +336,9 @@ public void testValidateConfig() throws IOException {
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
     }
     try {
@@ -344,9 +347,9 @@ public void testValidateConfig() throws IOException {
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
     }
     try {
@@ -357,12 +360,11 @@ public void testValidateConfig() throws IOException {
               0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert
-          .assertTrue(e
+      assertTrue(e
               .getMessage().contains(
-                  YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
+              YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
     }
     try {
       Configuration newConfig = new YarnConfiguration(copyConfig);
@@ -372,10 +374,9 @@ public void testValidateConfig() throws IOException {
               0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert
-          .assertTrue(e
+      assertTrue(e
               .getMessage()
               .contains(
                   YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
@@ -405,7 +406,7 @@ public void testGetDomains() throws IOException {
   }
 
   @Test
-  public void testRelatingToNonExistingEntity() throws IOException {
+  void testRelatingToNonExistingEntity() throws IOException {
     TimelineEntity entityToStore = new TimelineEntity();
     entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
     entityToStore.setEntityId("TEST_ENTITY_ID_1");
@@ -416,17 +417,17 @@ public void testRelatingToNonExistingEntity() throws IOException {
     store.put(entities);
     TimelineEntity entityToGet =
         store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
-    Assert.assertEquals("TEST_ENTITY_TYPE_1",
+    assertNotNull(entityToGet);
+    assertEquals("DEFAULT", entityToGet.getDomainId());
+    assertEquals("TEST_ENTITY_TYPE_1",
         entityToGet.getRelatedEntities().keySet().iterator().next());
-    Assert.assertEquals("TEST_ENTITY_ID_1",
+    assertEquals("TEST_ENTITY_ID_1",
         entityToGet.getRelatedEntities().values().iterator().next()
             .iterator().next());
   }
 
   @Test
-  public void testRelatingToOldEntityWithoutDomainId() throws IOException {
+  void testRelatingToOldEntityWithoutDomainId() throws IOException {
     // New entity is put in the default domain
     TimelineEntity entityToStore = new TimelineEntity();
     entityToStore.setEntityType("NEW_ENTITY_TYPE_1");
@@ -439,11 +440,11 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException {
 
     TimelineEntity entityToGet =
         store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertNull(entityToGet.getDomainId());
-    Assert.assertEquals("NEW_ENTITY_TYPE_1",
+    assertNotNull(entityToGet);
+    assertNull(entityToGet.getDomainId());
+    assertEquals("NEW_ENTITY_TYPE_1",
         entityToGet.getRelatedEntities().keySet().iterator().next());
-    Assert.assertEquals("NEW_ENTITY_ID_1",
+    assertEquals("NEW_ENTITY_ID_1",
         entityToGet.getRelatedEntities().values().iterator().next()
             .iterator().next());
 
@@ -456,26 +457,26 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException {
     entities = new TimelineEntities();
     entities.addEntity(entityToStore);
     TimelinePutResponse response = store.put(entities);
-    Assert.assertEquals(1, response.getErrors().size());
-    Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION,
+    assertEquals(1, response.getErrors().size());
+    assertEquals(TimelinePutError.FORBIDDEN_RELATION,
         response.getErrors().get(0).getErrorCode());
     entityToGet =
         store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertNull(entityToGet.getDomainId());
+    assertNotNull(entityToGet);
+    assertNull(entityToGet.getDomainId());
     // Still have one related entity
-    Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
-    Assert.assertEquals(1, entityToGet.getRelatedEntities().values()
+    assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
+    assertEquals(1, entityToGet.getRelatedEntities().values()
         .iterator().next().size());
   }
 
-  @Test
   /**
    * Test that LevelDb repair is attempted at least once during
    * serviceInit for LeveldbTimelineStore in case open fails the
    * first time.
    */
-  public void testLevelDbRepair() throws IOException {
+  @Test
+  void testLevelDbRepair() throws IOException {
     LeveldbTimelineStore store = new LeveldbTimelineStore();
 
     JniDBFactory factory = Mockito.mock(JniDBFactory.class);
@@ -496,8 +497,8 @@ public void testLevelDbRepair() throws IOException {
       Mockito.verify(factory, Mockito.times(1))
           .repair(Mockito.any(File.class), Mockito.any(Options.class));
       FileFilter fileFilter = new WildcardFileFilter(
-          "*" + LeveldbTimelineStore.BACKUP_EXT +"*");
-      Assert.assertTrue(path.listFiles(fileFilter).length > 0);
+          "*" + LeveldbTimelineStore.BACKUP_EXT + "*");
+      assertTrue(path.listFiles(fileFilter).length > 0);
     } finally {
       store.close();
       fsContext.delete(new Path(path.getAbsolutePath()), true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
index 6ba1bfeca7e33..9a018dbad71a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
@@ -18,18 +18,17 @@
 
 package org.apache.hadoop.yarn.server.timeline;
 
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
-import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
 import java.io.IOException;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
 public class TestMemoryTimelineStore extends TimelineStoreTestUtils {
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     store = new MemoryTimelineStore();
     store.init(new YarnConfiguration());
@@ -39,7 +38,7 @@ public void setup() throws Exception {
     loadTestDomainData();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java
index d2d0860386cc8..44effe33a0d79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java
@@ -19,14 +19,16 @@
 
 import java.io.File;
 
+import org.iq80.leveldb.DB;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.iq80.leveldb.DB;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /** Test class for verification of RollingLevelDB. */
 public class TestRollingLevelDB {
@@ -53,7 +55,7 @@ public void setCurrentTimeMillis(long time) {
     }
   };
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     lfs = FileSystem.getLocal(conf);
     File fsPath = new File("target", this.getClass().getSimpleName() +
@@ -65,26 +67,26 @@ public void setup() throws Exception {
   }
 
   @Test
-  public void testInsertAfterRollPeriodRollsDB() throws Exception {
+  void testInsertAfterRollPeriodRollsDB() throws Exception {
 
     rollingLevelDB.init(conf);
     long now = rollingLevelDB.currentTimeMillis();
     DB db = rollingLevelDB.getDBForStartTime(now);
     long startTime = rollingLevelDB.getStartTimeFor(db);
-    Assert.assertEquals("Received level db for incorrect start time",
-        rollingLevelDB.computeCurrentCheckMillis(now),
-        startTime);
+    assertEquals(rollingLevelDB.computeCurrentCheckMillis(now),
+        startTime,
+        "Received level db for incorrect start time");
     now = rollingLevelDB.getNextRollingTimeMillis();
     rollingLevelDB.setCurrentTimeMillis(now);
     db = rollingLevelDB.getDBForStartTime(now);
     startTime = rollingLevelDB.getStartTimeFor(db);
-    Assert.assertEquals("Received level db for incorrect start time",
-        rollingLevelDB.computeCurrentCheckMillis(now),
-        startTime);
+    assertEquals(rollingLevelDB.computeCurrentCheckMillis(now),
+        startTime,
+        "Received level db for incorrect start time");
   }
 
   @Test
-  public void testInsertForPreviousPeriodAfterRollPeriodRollsDB()
+  void testInsertForPreviousPeriodAfterRollPeriodRollsDB()
       throws Exception {
 
     rollingLevelDB.init(conf);
@@ -93,8 +95,8 @@ public void testInsertForPreviousPeriodAfterRollPeriodRollsDB()
     rollingLevelDB.setCurrentTimeMillis(now);
     DB db = rollingLevelDB.getDBForStartTime(now - 1);
     long startTime = rollingLevelDB.getStartTimeFor(db);
-    Assert.assertEquals("Received level db for incorrect start time",
-        rollingLevelDB.computeCurrentCheckMillis(now - 1),
-        startTime);
+    assertEquals(rollingLevelDB.computeCurrentCheckMillis(now - 1),
+        startTime,
+        "Received level db for incorrect start time");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java
index ab5d7aea443dc..b20497728bab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java
@@ -17,12 +17,19 @@
  */
 package org.apache.hadoop.yarn.server.timeline;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 
+import org.eclipse.jetty.util.log.Log;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.iq80.leveldb.Options;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -38,11 +45,11 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.records.Version;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.eclipse.jetty.util.log.Log;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /** Test class to verify RollingLevelDBTimelineStore. */
 @InterfaceAudience.Private
@@ -52,7 +59,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
   private File fsPath;
   private Configuration config = new YarnConfiguration();
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     fsContext = FileContext.getLocalFSFileContext();
     fsPath = new File("target", this.getClass().getSimpleName() +
@@ -69,14 +76,14 @@ public void setup() throws Exception {
     loadTestDomainData();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
     fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
   }
 
   @Test
-  public void testRootDirPermission() throws IOException {
+  void testRootDirPermission() throws IOException {
     FileSystem fs = FileSystem.getLocal(new YarnConfiguration());
     FileStatus file = fs.getFileStatus(new Path(fsPath.getAbsolutePath(),
         RollingLevelDBTimelineStore.FILENAME));
@@ -124,7 +131,7 @@ public void testGetEvents() throws IOException {
   }
 
   @Test
-  public void testCacheSizes() {
+  void testCacheSizes() {
     Configuration conf = new Configuration();
     assertEquals(10000,
         RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf));
@@ -144,48 +151,48 @@ public void testCacheSizes() {
   }
 
   @Test
-  public void testCheckVersion() throws IOException {
+  void testCheckVersion() throws IOException {
     RollingLevelDBTimelineStore dbStore = (RollingLevelDBTimelineStore) store;
     // default version
     Version defaultVersion = dbStore.getCurrentVersion();
-    Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+    assertEquals(defaultVersion, dbStore.loadVersion());
 
     // compatible version
     Version compatibleVersion =
         Version.newInstance(defaultVersion.getMajorVersion(),
-          defaultVersion.getMinorVersion() + 2);
+            defaultVersion.getMinorVersion() + 2);
     dbStore.storeVersion(compatibleVersion);
-    Assert.assertEquals(compatibleVersion, dbStore.loadVersion());
+    assertEquals(compatibleVersion, dbStore.loadVersion());
     restartTimelineStore();
     dbStore = (RollingLevelDBTimelineStore) store;
     // overwrite the compatible version
-    Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+    assertEquals(defaultVersion, dbStore.loadVersion());
 
     // incompatible version
     Version incompatibleVersion =
         Version.newInstance(defaultVersion.getMajorVersion() + 1,
-          defaultVersion.getMinorVersion());
+            defaultVersion.getMinorVersion());
     dbStore.storeVersion(incompatibleVersion);
     try {
       restartTimelineStore();
-      Assert.fail("Incompatible version, should expect fail here.");
+      fail("Incompatible version, should expect fail here.");
     } catch (ServiceStateException e) {
-      Assert.assertTrue("Exception message mismatch",
-          e.getMessage().contains("Incompatible version for timeline store"));
+      assertTrue(e.getMessage().contains("Incompatible version for timeline store"),
+          "Exception message mismatch");
     }
   }
 
   @Test
-  public void testValidateConfig() throws IOException {
+  void testValidateConfig() throws IOException {
     Configuration copyConfig = new YarnConfiguration(config);
     try {
       Configuration newConfig = new YarnConfiguration(copyConfig);
       newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
     }
     try {
@@ -194,9 +201,9 @@ public void testValidateConfig() throws IOException {
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
     }
     try {
@@ -205,9 +212,9 @@ public void testValidateConfig() throws IOException {
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
     }
     try {
@@ -217,25 +224,25 @@ public void testValidateConfig() throws IOException {
           0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration
-          .TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
+              .TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
     }
     try {
       Configuration newConfig = new YarnConfiguration(copyConfig);
       newConfig.setLong(
           YarnConfiguration
-          .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
+              .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE,
           0);
       config = newConfig;
       restartTimelineStore();
-      Assert.fail();
+      fail();
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().contains(
+      assertTrue(e.getMessage().contains(
           YarnConfiguration
-          .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
+              .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
     }
     config = copyConfig;
     restartTimelineStore();
@@ -262,7 +269,7 @@ public void testGetDomains() throws IOException {
   }
 
   @Test
-  public void testRelatingToNonExistingEntity() throws IOException {
+  void testRelatingToNonExistingEntity() throws IOException {
     TimelineEntity entityToStore = new TimelineEntity();
     entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
     entityToStore.setEntityId("TEST_ENTITY_ID_1");
@@ -273,17 +280,17 @@ public void testRelatingToNonExistingEntity() throws IOException {
     store.put(entities);
     TimelineEntity entityToGet =
         store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
-    Assert.assertEquals("TEST_ENTITY_TYPE_1",
+    assertNotNull(entityToGet);
+    assertEquals("DEFAULT", entityToGet.getDomainId());
+    assertEquals("TEST_ENTITY_TYPE_1",
         entityToGet.getRelatedEntities().keySet().iterator().next());
-    Assert.assertEquals("TEST_ENTITY_ID_1",
+    assertEquals("TEST_ENTITY_ID_1",
         entityToGet.getRelatedEntities().values().iterator().next()
             .iterator().next());
   }
 
   @Test
-  public void testRelatingToEntityInSamePut() throws IOException {
+  void testRelatingToEntityInSamePut() throws IOException {
     TimelineEntity entityToRelate = new TimelineEntity();
     entityToRelate.setEntityType("TEST_ENTITY_TYPE_2");
     entityToRelate.setEntityId("TEST_ENTITY_ID_2");
@@ -299,17 +306,17 @@ public void testRelatingToEntityInSamePut() throws IOException {
     store.put(entities);
     TimelineEntity entityToGet =
         store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertEquals("TEST_DOMAIN", entityToGet.getDomainId());
-    Assert.assertEquals("TEST_ENTITY_TYPE_1",
+    assertNotNull(entityToGet);
+    assertEquals("TEST_DOMAIN", entityToGet.getDomainId());
+    assertEquals("TEST_ENTITY_TYPE_1",
         entityToGet.getRelatedEntities().keySet().iterator().next());
-    Assert.assertEquals("TEST_ENTITY_ID_1",
+    assertEquals("TEST_ENTITY_ID_1",
         entityToGet.getRelatedEntities().values().iterator().next()
             .iterator().next());
   }
 
   @Test
-  public void testRelatingToOldEntityWithoutDomainId() throws IOException {
+  void testRelatingToOldEntityWithoutDomainId() throws IOException {
     // New entity is put in the default domain
     TimelineEntity entityToStore = new TimelineEntity();
     entityToStore.setEntityType("NEW_ENTITY_TYPE_1");
@@ -322,11 +329,11 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException {
 
     TimelineEntity entityToGet =
         store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
-    Assert.assertEquals("NEW_ENTITY_TYPE_1",
+    assertNotNull(entityToGet);
+    assertEquals("DEFAULT", entityToGet.getDomainId());
+    assertEquals("NEW_ENTITY_TYPE_1",
         entityToGet.getRelatedEntities().keySet().iterator().next());
-    Assert.assertEquals("NEW_ENTITY_ID_1",
+    assertEquals("NEW_ENTITY_ID_1",
         entityToGet.getRelatedEntities().values().iterator().next()
             .iterator().next());
 
@@ -339,16 +346,16 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException {
     entities = new TimelineEntities();
     entities.addEntity(entityToStore);
     TimelinePutResponse response = store.put(entities);
-    Assert.assertEquals(1, response.getErrors().size());
-    Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION,
+    assertEquals(1, response.getErrors().size());
+    assertEquals(TimelinePutError.FORBIDDEN_RELATION,
         response.getErrors().get(0).getErrorCode());
     entityToGet =
         store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entityToGet);
-    Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
+    assertNotNull(entityToGet);
+    assertEquals("DEFAULT", entityToGet.getDomainId());
     // Still have one related entity
-    Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
-    Assert.assertEquals(1, entityToGet.getRelatedEntities().values()
+    assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
+    assertEquals(1, entityToGet.getRelatedEntities().values()
         .iterator().next().size());
   }
 
@@ -417,6 +424,36 @@ public void testStorePerformance() throws IOException {
     Log.getLog().info("Duration for " + num + ": " + duration);
   }
 
+  /**
+   * Test that RollingLevelDb repair is attempted at least once during
+   * serviceInit for RollingLeveldbTimelineStore in case open fails the
+   * first time.
+   */ @Test
+  void testLevelDbRepair() throws IOException {
+    RollingLevelDBTimelineStore store = new RollingLevelDBTimelineStore();
+    JniDBFactory factory = Mockito.mock(JniDBFactory.class);
+    Mockito.when(factory.open(Mockito.any(File.class), Mockito.any(Options.class)))
+        .thenThrow(new IOException()).thenCallRealMethod();
+    store.setFactory(factory);
+
+    //Create the LevelDb in a different location
+    File path = new File("target", this.getClass().getSimpleName() + "-tmpDir2").getAbsoluteFile();
+    Configuration conf = new Configuration(this.config);
+    conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH, path.getAbsolutePath());
+    try {
+      store.init(conf);
+      Mockito.verify(factory, Mockito.times(1))
+          .repair(Mockito.any(File.class), Mockito.any(Options.class));
+      FilenameFilter fileFilter =
+          new WildcardFileFilter("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*");
+      assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME)
+          .list(fileFilter).length > 0);
+    } finally {
+      store.close();
+      fsContext.delete(new Path(path.getAbsolutePath()), true);
+    }
+  }
+
   public static void main(String[] args) throws Exception {
     TestRollingLevelDBTimelineStore store =
         new TestRollingLevelDBTimelineStore();
@@ -424,4 +461,4 @@ public static void main(String[] args) throws Exception {
     store.testStorePerformance();
     store.tearDown();
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java
index 8fba54c9e8055..be57c84cd8c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java
@@ -20,6 +20,10 @@
 
 import java.io.File;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -30,10 +34,10 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AdminACLsManager;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 public class TestTimelineDataManager extends TimelineStoreTestUtils {
@@ -43,7 +47,7 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
   private TimelineDataManager dataManaer;
   private static TimelineACLsManager aclsManager;
   private static AdminACLsManager adminACLsManager;
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     fsPath = new File("target", this.getClass().getSimpleName() +
         "-tmpDir").getAbsoluteFile();
@@ -70,7 +74,7 @@ public void setup() throws Exception {
     adminACLsManager = new AdminACLsManager(conf);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     if (store != null) {
       store.stop();
@@ -81,55 +85,55 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testGetOldEntityWithOutDomainId() throws Exception {
+  void testGetOldEntityWithOutDomainId() throws Exception {
     TimelineEntity entity = dataManaer.getEntity(
         "OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1", null,
         UserGroupInformation.getCurrentUser());
-    Assert.assertNotNull(entity);
-    Assert.assertEquals("OLD_ENTITY_ID_1", entity.getEntityId());
-    Assert.assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType());
-    Assert.assertEquals(
+    assertNotNull(entity);
+    assertEquals("OLD_ENTITY_ID_1", entity.getEntityId());
+    assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType());
+    assertEquals(
         TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId());
   }
 
   @Test
-  public void testGetEntitiesAclEnabled() throws Exception {
+  void testGetEntitiesAclEnabled() throws Exception {
     AdminACLsManager oldAdminACLsManager =
-      aclsManager.setAdminACLsManager(adminACLsManager);
+        aclsManager.setAdminACLsManager(adminACLsManager);
     try {
       TimelineEntities entities = dataManaer.getEntities(
-        "ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1l, null,
-        UserGroupInformation.createUserForTesting("owner_1", new String[] {"group1"}));
-      Assert.assertEquals(1, entities.getEntities().size());
-      Assert.assertEquals("ACL_ENTITY_ID_11",
-        entities.getEntities().get(0).getEntityId());
+          "ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1L, null,
+          UserGroupInformation.createUserForTesting("owner_1", new String[]{"group1"}));
+      assertEquals(1, entities.getEntities().size());
+      assertEquals("ACL_ENTITY_ID_11",
+          entities.getEntities().get(0).getEntityId());
     } finally {
       aclsManager.setAdminACLsManager(oldAdminACLsManager);
     }
   }
 
   @Test
-  public void testGetOldEntitiesWithOutDomainId() throws Exception {
+  void testGetOldEntitiesWithOutDomainId() throws Exception {
     TimelineEntities entities = dataManaer.getEntities(
         "OLD_ENTITY_TYPE_1", null, null, null, null, null, null, null, null,
         UserGroupInformation.getCurrentUser());
-    Assert.assertEquals(2, entities.getEntities().size());
-    Assert.assertEquals("OLD_ENTITY_ID_2",
+    assertEquals(2, entities.getEntities().size());
+    assertEquals("OLD_ENTITY_ID_2",
         entities.getEntities().get(0).getEntityId());
-    Assert.assertEquals("OLD_ENTITY_TYPE_1",
+    assertEquals("OLD_ENTITY_TYPE_1",
         entities.getEntities().get(0).getEntityType());
-    Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
+    assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
         entities.getEntities().get(0).getDomainId());
-    Assert.assertEquals("OLD_ENTITY_ID_1",
+    assertEquals("OLD_ENTITY_ID_1",
         entities.getEntities().get(1).getEntityId());
-    Assert.assertEquals("OLD_ENTITY_TYPE_1",
+    assertEquals("OLD_ENTITY_TYPE_1",
         entities.getEntities().get(1).getEntityType());
-    Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
+    assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
         entities.getEntities().get(1).getDomainId());
   }
 
   @Test
-  public void testUpdatingOldEntityWithoutDomainId() throws Exception {
+  void testUpdatingOldEntityWithoutDomainId() throws Exception {
     // Set the domain to the default domain when updating
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityType("OLD_ENTITY_TYPE_1");
@@ -140,18 +144,18 @@ public void testUpdatingOldEntityWithoutDomainId() throws Exception {
     entities.addEntity(entity);
     TimelinePutResponse response = dataManaer.postEntities(
         entities, UserGroupInformation.getCurrentUser());
-    Assert.assertEquals(0, response.getErrors().size());
+    assertEquals(0, response.getErrors().size());
     entity = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entity);
+    assertNotNull(entity);
     // Even in leveldb, the domain is updated to the default domain Id
-    Assert.assertEquals(
+    assertEquals(
         TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId());
-    Assert.assertEquals(1, entity.getOtherInfo().size());
-    Assert.assertEquals("NEW_OTHER_INFO_KEY",
+    assertEquals(1, entity.getOtherInfo().size());
+    assertEquals("NEW_OTHER_INFO_KEY",
         entity.getOtherInfo().keySet().iterator().next());
-    Assert.assertEquals("NEW_OTHER_INFO_VALUE",
+    assertEquals("NEW_OTHER_INFO_VALUE",
         entity.getOtherInfo().values().iterator().next());
-    
+
     // Set the domain to the non-default domain when updating
     entity = new TimelineEntity();
     entity.setEntityType("OLD_ENTITY_TYPE_1");
@@ -162,15 +166,15 @@ public void testUpdatingOldEntityWithoutDomainId() throws Exception {
     entities.addEntity(entity);
     response = dataManaer.postEntities(
         entities, UserGroupInformation.getCurrentUser());
-    Assert.assertEquals(1, response.getErrors().size());
-    Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
+    assertEquals(1, response.getErrors().size());
+    assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
         response.getErrors().get(0).getErrorCode());
     entity = store.getEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
-    Assert.assertNotNull(entity);
+    assertNotNull(entity);
     // In leveldb, the domain Id is still null
-    Assert.assertNull(entity.getDomainId());
+    assertNull(entity.getDomainId());
     // Updating is not executed
-    Assert.assertEquals(0, entity.getOtherInfo().size());
+    assertEquals(0, entity.getOtherInfo().size());
   }
   
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
index 71e298c018654..f925b1104f503 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
@@ -17,10 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.timeline;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -28,7 +24,6 @@
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -36,16 +31,20 @@
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 public class TimelineStoreTestUtils {
 
   protected static final List<TimelineEvent> EMPTY_EVENTS =
@@ -504,18 +503,24 @@ protected List<TimelineEntity> getEntities(String entityType, Long limit,
 
   public void testGetEntities() throws IOException {
     // test getting entities
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntities("type_0").size());
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntities("type_3").size());
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntities("type_6").size());
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntitiesWithPrimaryFilter("type_0", userFilter).size());
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntitiesWithPrimaryFilter("type_3", userFilter).size());
-    assertEquals("nonzero entities size for nonexistent type", 0,
-        getEntitiesWithPrimaryFilter("type_6", userFilter).size());
+    assertEquals(0,
+        getEntities("type_0").size(),
+        "nonzero entities size for nonexistent type");
+    assertEquals(0,
+        getEntities("type_3").size(),
+        "nonzero entities size for nonexistent type");
+    assertEquals(0,
+        getEntities("type_6").size(),
+        "nonzero entities size for nonexistent type");
+    assertEquals(0,
+        getEntitiesWithPrimaryFilter("type_0", userFilter).size(),
+        "nonzero entities size for nonexistent type");
+    assertEquals(0,
+        getEntitiesWithPrimaryFilter("type_3", userFilter).size(),
+        "nonzero entities size for nonexistent type");
+    assertEquals(0,
+        getEntitiesWithPrimaryFilter("type_6", userFilter).size(),
+        "nonzero entities size for nonexistent type");
 
     List<TimelineEntity> entities = getEntities("type_1");
     assertEquals(3, entities.size());
@@ -682,15 +687,18 @@ public void testGetEntitiesWithFromTs() throws IOException {
 
   public void testGetEntitiesWithPrimaryFilters() throws IOException {
     // test using primary filter
-    assertEquals("nonzero entities size for primary filter", 0,
+    assertEquals(0,
         getEntitiesWithPrimaryFilter("type_1",
-            new NameValuePair("none", "none")).size());
-    assertEquals("nonzero entities size for primary filter", 0,
+            new NameValuePair("none", "none")).size(),
+        "nonzero entities size for primary filter");
+    assertEquals(0,
         getEntitiesWithPrimaryFilter("type_2",
-            new NameValuePair("none", "none")).size());
-    assertEquals("nonzero entities size for primary filter", 0,
+            new NameValuePair("none", "none")).size(),
+        "nonzero entities size for primary filter");
+    assertEquals(0,
         getEntitiesWithPrimaryFilter("type_3",
-            new NameValuePair("none", "none")).size());
+            new NameValuePair("none", "none")).size(),
+        "nonzero entities size for primary filter");
 
     List<TimelineEntity> entities = getEntitiesWithPrimaryFilter("type_1",
         userFilter);
@@ -928,13 +936,12 @@ protected static void verifyEntityInfo(String entityId, String entityType,
     if (primaryFilters == null) {
       assertNull(retrievedEntityInfo.getPrimaryFilters());
     } else {
-      assertTrue(primaryFilters.equals(
-          retrievedEntityInfo.getPrimaryFilters()));
+      assertEquals(primaryFilters, retrievedEntityInfo.getPrimaryFilters());
     }
     if (otherInfo == null) {
       assertNull(retrievedEntityInfo.getOtherInfo());
     } else {
-      assertTrue(otherInfo.equals(retrievedEntityInfo.getOtherInfo()));
+      assertEquals(otherInfo, retrievedEntityInfo.getOtherInfo());
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
index a35477dc70b99..3baf41425cb97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
@@ -18,14 +18,13 @@
 
 package org.apache.hadoop.yarn.server.timeline.recovery;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.io.File;
 import java.io.IOException;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -36,10 +35,11 @@
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestLeveldbTimelineStateStore {
 
@@ -48,7 +48,7 @@ public class TestLeveldbTimelineStateStore {
   private Configuration conf;
   private TimelineStateStore store;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     fsPath = new File("target", getClass().getSimpleName() +
         "-tmpDir").getAbsoluteFile();
@@ -63,7 +63,7 @@ public void setup() throws Exception {
         fsPath.getAbsolutePath());
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     if (store != null) {
       store.stop();
@@ -81,11 +81,11 @@ private LeveldbTimelineStateStore initAndStartTimelineServiceStateStoreService()
   }
 
   @Test
-  public void testTokenStore() throws Exception {
+  void testTokenStore() throws Exception {
     initAndStartTimelineServiceStateStoreService();
     TimelineServiceState state = store.loadState();
-    assertTrue("token state not empty", state.tokenState.isEmpty());
-    assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty());
+    assertTrue(state.tokenState.isEmpty(), "token state not empty");
+    assertTrue(state.tokenMasterKeyState.isEmpty(), "key state not empty");
 
     final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
     final TimelineDelegationTokenIdentifier token1 =
@@ -120,19 +120,23 @@ public void testTokenStore() throws Exception {
 
     initAndStartTimelineServiceStateStoreService();
     state = store.loadState();
-    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
-    assertTrue("missing token 1", state.tokenState.containsKey(token1));
-    assertEquals("incorrect token 1 date", tokenDate1,
-        state.tokenState.get(token1));
-    assertTrue("missing token 2", state.tokenState.containsKey(token2));
-    assertEquals("incorrect token 2 date", tokenDate2,
-        state.tokenState.get(token2));
-    assertEquals("incorrect master key count", 1,
-        state.tokenMasterKeyState.size());
-    assertTrue("missing master key 1",
-        state.tokenMasterKeyState.contains(key1));
-    assertEquals("incorrect latest sequence number", 12345678,
-        state.getLatestSequenceNumber());
+    assertEquals(2, state.tokenState.size(), "incorrect loaded token count");
+    assertTrue(state.tokenState.containsKey(token1), "missing token 1");
+    assertEquals(tokenDate1,
+        state.tokenState.get(token1),
+        "incorrect token 1 date");
+    assertTrue(state.tokenState.containsKey(token2), "missing token 2");
+    assertEquals(tokenDate2,
+        state.tokenState.get(token2),
+        "incorrect token 2 date");
+    assertEquals(1,
+        state.tokenMasterKeyState.size(),
+        "incorrect master key count");
+    assertTrue(state.tokenMasterKeyState.contains(key1),
+        "missing master key 1");
+    assertEquals(12345678,
+        state.getLatestSequenceNumber(),
+        "incorrect latest sequence number");
 
     final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
     final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
@@ -154,46 +158,50 @@ public void testTokenStore() throws Exception {
 
     initAndStartTimelineServiceStateStoreService();
     state = store.loadState();
-    assertEquals("incorrect loaded token count", 2, state.tokenState.size());
-    assertFalse("token 1 not removed", state.tokenState.containsKey(token1));
-    assertTrue("missing token 2", state.tokenState.containsKey(token2));
-    assertEquals("incorrect token 2 date", newTokenDate2,
-        state.tokenState.get(token2));
-    assertTrue("missing token 3", state.tokenState.containsKey(token3));
-    assertEquals("incorrect token 3 date", tokenDate3,
-        state.tokenState.get(token3));
-    assertEquals("incorrect master key count", 2,
-        state.tokenMasterKeyState.size());
-    assertFalse("master key 1 not removed",
-        state.tokenMasterKeyState.contains(key1));
-    assertTrue("missing master key 2",
-        state.tokenMasterKeyState.contains(key2));
-    assertTrue("missing master key 3",
-        state.tokenMasterKeyState.contains(key3));
-    assertEquals("incorrect latest sequence number", 12345679,
-        state.getLatestSequenceNumber());
+    assertEquals(2, state.tokenState.size(), "incorrect loaded token count");
+    assertFalse(state.tokenState.containsKey(token1), "token 1 not removed");
+    assertTrue(state.tokenState.containsKey(token2), "missing token 2");
+    assertEquals(newTokenDate2,
+        state.tokenState.get(token2),
+        "incorrect token 2 date");
+    assertTrue(state.tokenState.containsKey(token3), "missing token 3");
+    assertEquals(tokenDate3,
+        state.tokenState.get(token3),
+        "incorrect token 3 date");
+    assertEquals(2,
+        state.tokenMasterKeyState.size(),
+        "incorrect master key count");
+    assertFalse(state.tokenMasterKeyState.contains(key1),
+        "master key 1 not removed");
+    assertTrue(state.tokenMasterKeyState.contains(key2),
+        "missing master key 2");
+    assertTrue(state.tokenMasterKeyState.contains(key3),
+        "missing master key 3");
+    assertEquals(12345679,
+        state.getLatestSequenceNumber(),
+        "incorrect latest sequence number");
     store.close();
   }
 
   @Test
-  public void testCheckVersion() throws IOException {
+  void testCheckVersion() throws IOException {
     LeveldbTimelineStateStore store =
         initAndStartTimelineServiceStateStoreService();
     // default version
     Version defaultVersion = store.getCurrentVersion();
-    Assert.assertEquals(defaultVersion, store.loadVersion());
+    assertEquals(defaultVersion, store.loadVersion());
 
     // compatible version
     Version compatibleVersion =
         Version.newInstance(defaultVersion.getMajorVersion(),
             defaultVersion.getMinorVersion() + 2);
     store.storeVersion(compatibleVersion);
-    Assert.assertEquals(compatibleVersion, store.loadVersion());
+    assertEquals(compatibleVersion, store.loadVersion());
     store.stop();
 
     // overwrite the compatible version
     store = initAndStartTimelineServiceStateStoreService();
-    Assert.assertEquals(defaultVersion, store.loadVersion());
+    assertEquals(defaultVersion, store.loadVersion());
 
     // incompatible version
     Version incompatibleVersion =
@@ -204,10 +212,10 @@ public void testCheckVersion() throws IOException {
 
     try {
       initAndStartTimelineServiceStateStoreService();
-      Assert.fail("Incompatible version, should expect fail here.");
+      fail("Incompatible version, should expect fail here.");
     } catch (ServiceStateException e) {
-      Assert.assertTrue("Exception message mismatch",
-          e.getMessage().contains("Incompatible version for timeline state store"));
+      assertTrue(e.getMessage().contains("Incompatible version for timeline state store"),
+          "Exception message mismatch");
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
index 3bfb4b09576e4..da3a9e226bfd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
@@ -20,6 +20,8 @@
 
 import java.io.IOException;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -29,8 +31,10 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineACLsManager {
 
@@ -45,7 +49,7 @@ public class TestTimelineACLsManager {
   }
 
   @Test
-  public void testYarnACLsNotEnabledForEntity() throws Exception {
+  void testYarnACLsNotEnabledForEntity() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
     TimelineACLsManager timelineACLsManager =
@@ -56,20 +60,20 @@ public void testYarnACLsNotEnabledForEntity() throws Exception {
         TimelineStore.SystemFilter.ENTITY_OWNER
             .toString(), "owner");
     entity.setDomainId("domain_id_1");
-    Assert.assertTrue(
-        "Always true when ACLs are not enabled",
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("user"),
-            ApplicationAccessType.VIEW_APP, entity));
-    Assert.assertTrue(
-        "Always true when ACLs are not enabled",
+            ApplicationAccessType.VIEW_APP, entity),
+        "Always true when ACLs are not enabled");
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("user"),
-            ApplicationAccessType.MODIFY_APP, entity));
+            ApplicationAccessType.MODIFY_APP, entity),
+        "Always true when ACLs are not enabled");
   }
 
   @Test
-  public void testYarnACLsEnabledForEntity() throws Exception {
+  void testYarnACLsEnabledForEntity() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@@ -81,51 +85,51 @@ public void testYarnACLsEnabledForEntity() throws Exception {
         TimelineStore.SystemFilter.ENTITY_OWNER
             .toString(), "owner");
     entity.setDomainId("domain_id_1");
-    Assert.assertTrue(
-        "Owner should be allowed to view",
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("owner"),
-            ApplicationAccessType.VIEW_APP, entity));
-    Assert.assertTrue(
-        "Reader should be allowed to view",
+            ApplicationAccessType.VIEW_APP, entity),
+        "Owner should be allowed to view");
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("reader"),
-            ApplicationAccessType.VIEW_APP, entity));
-    Assert.assertFalse(
-        "Other shouldn't be allowed to view",
+            ApplicationAccessType.VIEW_APP, entity),
+        "Reader should be allowed to view");
+    assertFalse(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("other"),
-            ApplicationAccessType.VIEW_APP, entity));
-    Assert.assertTrue(
-        "Admin should be allowed to view",
+            ApplicationAccessType.VIEW_APP, entity),
+        "Other shouldn't be allowed to view");
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("admin"),
-            ApplicationAccessType.VIEW_APP, entity));
+            ApplicationAccessType.VIEW_APP, entity),
+        "Admin should be allowed to view");
 
-    Assert.assertTrue(
-        "Owner should be allowed to modify",
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("owner"),
-            ApplicationAccessType.MODIFY_APP, entity));
-    Assert.assertTrue(
-        "Writer should be allowed to modify",
+            ApplicationAccessType.MODIFY_APP, entity),
+        "Owner should be allowed to modify");
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("writer"),
-            ApplicationAccessType.MODIFY_APP, entity));
-    Assert.assertFalse(
-        "Other shouldn't be allowed to modify",
+            ApplicationAccessType.MODIFY_APP, entity),
+        "Writer should be allowed to modify");
+    assertFalse(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("other"),
-            ApplicationAccessType.MODIFY_APP, entity));
-    Assert.assertTrue(
-        "Admin should be allowed to modify",
+            ApplicationAccessType.MODIFY_APP, entity),
+        "Other shouldn't be allowed to modify");
+    assertTrue(
         timelineACLsManager.checkAccess(
             UserGroupInformation.createRemoteUser("admin"),
-            ApplicationAccessType.MODIFY_APP, entity));
+            ApplicationAccessType.MODIFY_APP, entity),
+        "Admin should be allowed to modify");
   }
 
   @Test
-  public void testCorruptedOwnerInfoForEntity() throws Exception {
+  void testCorruptedOwnerInfoForEntity() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@@ -137,29 +141,29 @@ public void testCorruptedOwnerInfoForEntity() throws Exception {
       timelineACLsManager.checkAccess(
           UserGroupInformation.createRemoteUser("owner"),
           ApplicationAccessType.VIEW_APP, entity);
-      Assert.fail("Exception is expected");
+      fail("Exception is expected");
     } catch (YarnException e) {
-      Assert.assertTrue("It's not the exact expected exception", e.getMessage()
-          .contains("doesn't exist."));
+      assertTrue(e.getMessage()
+          .contains("doesn't exist."), "It's not the exact expected exception");
     }
   }
 
   @Test
-  public void testYarnACLsNotEnabledForDomain() throws Exception {
+  void testYarnACLsNotEnabledForDomain() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
     TimelineACLsManager timelineACLsManager =
         new TimelineACLsManager(conf);
     TimelineDomain domain = new TimelineDomain();
     domain.setOwner("owner");
-    Assert.assertTrue(
-        "Always true when ACLs are not enabled",
+    assertTrue(
         timelineACLsManager.checkAccess(
-            UserGroupInformation.createRemoteUser("user"), domain));
+            UserGroupInformation.createRemoteUser("user"), domain),
+        "Always true when ACLs are not enabled");
   }
 
   @Test
-  public void testYarnACLsEnabledForDomain() throws Exception {
+  void testYarnACLsEnabledForDomain() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@@ -167,22 +171,22 @@ public void testYarnACLsEnabledForDomain() throws Exception {
         new TimelineACLsManager(conf);
     TimelineDomain domain = new TimelineDomain();
     domain.setOwner("owner");
-    Assert.assertTrue(
-        "Owner should be allowed to access",
+    assertTrue(
         timelineACLsManager.checkAccess(
-            UserGroupInformation.createRemoteUser("owner"), domain));
-    Assert.assertFalse(
-        "Other shouldn't be allowed to access",
+            UserGroupInformation.createRemoteUser("owner"), domain),
+        "Owner should be allowed to access");
+    assertFalse(
         timelineACLsManager.checkAccess(
-            UserGroupInformation.createRemoteUser("other"), domain));
-    Assert.assertTrue(
-        "Admin should be allowed to access",
+            UserGroupInformation.createRemoteUser("other"), domain),
+        "Other shouldn't be allowed to access");
+    assertTrue(
         timelineACLsManager.checkAccess(
-            UserGroupInformation.createRemoteUser("admin"), domain));
+            UserGroupInformation.createRemoteUser("admin"), domain),
+        "Admin should be allowed to access");
   }
 
   @Test
-  public void testCorruptedOwnerInfoForDomain() throws Exception {
+  void testCorruptedOwnerInfoForDomain() throws Exception {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@@ -192,10 +196,10 @@ public void testCorruptedOwnerInfoForDomain() throws Exception {
     try {
       timelineACLsManager.checkAccess(
           UserGroupInformation.createRemoteUser("owner"), domain);
-      Assert.fail("Exception is expected");
+      fail("Exception is expected");
     } catch (YarnException e) {
-      Assert.assertTrue("It's not the exact expected exception", e.getMessage()
-          .contains("is corrupted."));
+      assertTrue(e.getMessage()
+          .contains("is corrupted."), "It's not the exact expected exception");
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
index cf2db2f8c6fb4..01e787af605f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
@@ -24,6 +24,13 @@
 import java.util.Collection;
 import java.util.concurrent.Callable;
 
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileUtil;
@@ -46,23 +53,18 @@
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
-import static org.junit.Assert.fail;
 
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Test cases for authentication via TimelineAuthenticationFilter while
  * publishing entities for ATSv1.
  */
-@RunWith(Parameterized.class)
 public class TestTimelineAuthenticationFilterForV1 {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class);
@@ -74,7 +76,7 @@ public class TestTimelineAuthenticationFilterForV1 {
 
   private static final File TEST_ROOT_DIR = new File(
       System.getProperty("test.build.dir", "target/test-dir"),
-          TestTimelineAuthenticationFilterForV1.class.getName() + "-root");
+      TestTimelineAuthenticationFilterForV1.class.getName() + "-root");
   private static final File httpSpnegoKeytabFile = new File(
       KerberosTestUtils.getKeytabFile());
   private static final String httpSpnegoPrincipal =
@@ -83,9 +85,8 @@ public class TestTimelineAuthenticationFilterForV1 {
       System.getProperty("test.build.dir", "target/test-dir") + "/"
           + TestTimelineAuthenticationFilterForV1.class.getSimpleName();
 
-  @Parameterized.Parameters
   public static Collection<Object[]> withSsl() {
-    return Arrays.asList(new Object[][] {{false}, {true}});
+    return Arrays.asList(new Object[][]{{false}, {true}});
   }
 
   private static MiniKdc testMiniKDC;
@@ -95,11 +96,11 @@ public static Collection<Object[]> withSsl() {
   private static Configuration conf;
   private static boolean withSsl;
 
-  public TestTimelineAuthenticationFilterForV1(boolean withSsl) {
-    TestTimelineAuthenticationFilterForV1.withSsl = withSsl;
+  public void initTestTimelineAuthenticationFilterForV1(boolean isSslEnabled) {
+    TestTimelineAuthenticationFilterForV1.withSsl = isSslEnabled;
   }
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() {
     try {
       testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
@@ -167,7 +168,7 @@ private TimelineClient createTimelineClientForUGI() {
     return client;
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     if (testMiniKDC != null) {
       testMiniKDC.stop();
@@ -184,8 +185,10 @@ public static void tearDown() throws Exception {
     }
   }
 
-  @Test
-  public void testPutTimelineEntities() throws Exception {
+  @MethodSource("withSsl")
+  @ParameterizedTest
+  void testPutTimelineEntities(boolean isSslEnabled) throws Exception {
+    initTestTimelineAuthenticationFilterForV1(isSslEnabled);
     KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() {
       @Override
       public Void call() throws Exception {
@@ -199,20 +202,22 @@ public Void call() throws Exception {
         if (putResponse.getErrors().size() > 0) {
           LOG.error("putResponse errors: {}", putResponse.getErrors());
         }
-        Assert.assertTrue("There were some errors in the putResponse",
-            putResponse.getErrors().isEmpty());
+        assertTrue(putResponse.getErrors().isEmpty(),
+            "There were some errors in the putResponse");
         TimelineEntity entityToRead =
             testTimelineServer.getTimelineStore().getEntity("entity1",
                 TestTimelineAuthenticationFilterForV1.class.getName(), null);
-        Assert.assertNotNull("Timeline entity should not be null",
-            entityToRead);
+        assertNotNull(entityToRead,
+            "Timeline entity should not be null");
         return null;
       }
     });
   }
 
-  @Test
-  public void testPutDomains() throws Exception {
+  @MethodSource("withSsl")
+  @ParameterizedTest
+  void testPutDomains(boolean isSslEnabled) throws Exception {
+    initTestTimelineAuthenticationFilterForV1(isSslEnabled);
     KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() {
       @Override
       public Void call() throws Exception {
@@ -226,71 +231,73 @@ public Void call() throws Exception {
         TimelineDomain domainToRead =
             testTimelineServer.getTimelineStore().getDomain(
                 TestTimelineAuthenticationFilterForV1.class.getName());
-        Assert.assertNotNull("Timeline domain should not be null",
-            domainToRead);
+        assertNotNull(domainToRead,
+            "Timeline domain should not be null");
         return null;
       }
     });
   }
 
-  @Test
-  public void testDelegationTokenOperations() throws Exception {
+  @MethodSource("withSsl")
+  @ParameterizedTest
+  void testDelegationTokenOperations(boolean isSslEnabled) throws Exception {
+    initTestTimelineAuthenticationFilterForV1(isSslEnabled);
     TimelineClient httpUserClient =
         KerberosTestUtils.doAs(PRINCIPAL,
             new Callable<TimelineClient>() {
-            @Override
-            public TimelineClient call() throws Exception {
-              return createTimelineClientForUGI();
-            }
-          });
+              @Override
+              public TimelineClient call() throws Exception {
+                return createTimelineClientForUGI();
+              }
+            });
     UserGroupInformation httpUser =
         KerberosTestUtils.doAs(PRINCIPAL,
             new Callable<UserGroupInformation>() {
-            @Override
-            public UserGroupInformation call() throws Exception {
-              return UserGroupInformation.getCurrentUser();
-            }
-          });
+              @Override
+              public UserGroupInformation call() throws Exception {
+                return UserGroupInformation.getCurrentUser();
+              }
+            });
 
     // Let HTTP user to get the delegation for itself
     Token<TimelineDelegationTokenIdentifier> token =
         httpUserClient.getDelegationToken(httpUser.getShortUserName());
-    Assert.assertNotNull("Delegation token should not be null", token);
+    assertNotNull(token, "Delegation token should not be null");
     TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
-    Assert.assertNotNull("Delegation token identifier should not be null",
-        tDT);
-    Assert.assertEquals("Owner of delegation token identifier does not match",
-        new Text(HTTP_USER), tDT.getOwner());
+    assertNotNull(tDT,
+        "Delegation token identifier should not be null");
+    assertEquals(new Text(HTTP_USER), tDT.getOwner(),
+        "Owner of delegation token identifier does not match");
 
     // Renew token
-    Assert.assertFalse("Service field of token should not be empty",
-        token.getService().toString().isEmpty());
+    assertFalse(token.getService().toString().isEmpty(),
+        "Service field of token should not be empty");
     // Renew the token from the token service address
     long renewTime1 = httpUserClient.renewDelegationToken(token);
     Thread.sleep(100);
     token.setService(new Text());
-    Assert.assertTrue("Service field of token should be empty",
-        token.getService().toString().isEmpty());
+    assertTrue(token.getService().toString().isEmpty(),
+        "Service field of token should be empty");
     // If the token service address is not available, it still can be renewed
     // from the configured address
     long renewTime2 = httpUserClient.renewDelegationToken(token);
-    Assert.assertTrue("renewTime2 should be later than renewTime1",
-        renewTime1 < renewTime2);
+    assertTrue(renewTime1 < renewTime2,
+        "renewTime2 should be later than renewTime1");
 
     // Cancel token
-    Assert.assertTrue("Service field of token should be empty",
-        token.getService().toString().isEmpty());
+    assertTrue(token.getService().toString().isEmpty(),
+        "Service field of token should be empty");
     // If the token service address is not available, it still can be canceled
     // from the configured address
     httpUserClient.cancelDelegationToken(token);
     // Renew should not be successful because the token is canceled
     try {
       httpUserClient.renewDelegationToken(token);
-      Assert.fail("Renew of delegation token should not be successful");
+      fail("Renew of delegation token should not be successful");
     } catch (Exception e) {
       LOG.info("Exception while renewing delegation token", e);
-      Assert.assertTrue(e.getMessage().contains(
-            "Renewal request for unknown token"));
+      assertTrue(e.getMessage().contains(
+          "Renewal request for unknown token"));
     }
 
     // Let HTTP user to get the delegation token for FOO user
@@ -304,35 +311,35 @@ public TimelineClient run() {
           }
         });
     token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
-    Assert.assertNotNull("Delegation token should not be null", token);
+    assertNotNull(token, "Delegation token should not be null");
     tDT = token.decodeIdentifier();
-    Assert.assertNotNull("Delegation token identifier should not be null",
-        tDT);
-    Assert.assertEquals("Owner of delegation token is not the expected",
-        new Text(FOO_USER), tDT.getOwner());
-    Assert.assertEquals("Real user of delegation token is not the expected",
-        new Text(HTTP_USER), tDT.getRealUser());
+    assertNotNull(tDT,
+        "Delegation token identifier should not be null");
+    assertEquals(new Text(FOO_USER), tDT.getOwner(),
+        "Owner of delegation token is not the expected");
+    assertEquals(new Text(HTTP_USER), tDT.getRealUser(),
+        "Real user of delegation token is not the expected");
 
     // Renew token as the renewer
     final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
     renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
     renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
-    Assert.assertTrue("renewTime2 should be later than renewTime1",
-        renewTime1 < renewTime2);
+    assertTrue(renewTime1 < renewTime2,
+        "renewTime2 should be later than renewTime1");
 
     // Cancel token
-    Assert.assertFalse("Service field of token should not be empty",
-        tokenToRenew.getService().toString().isEmpty());
+    assertFalse(tokenToRenew.getService().toString().isEmpty(),
+        "Service field of token should not be empty");
     // Cancel the token from the token service address
     fooUserClient.cancelDelegationToken(tokenToRenew);
 
     // Renew should not be successful because the token is canceled
     try {
       httpUserClient.renewDelegationToken(tokenToRenew);
-      Assert.fail("Renew of delegation token should not be successful");
+      fail("Renew of delegation token should not be successful");
     } catch (Exception e) {
       LOG.info("Exception while renewing delegation token", e);
-      Assert.assertTrue(
+      assertTrue(
           e.getMessage().contains("Renewal request for unknown token"));
     }
 
@@ -349,10 +356,10 @@ public TimelineClient run() {
 
     try {
       barUserClient.getDelegationToken(httpUser.getShortUserName());
-      Assert.fail("Retrieval of delegation token should not be successful");
+      fail("Retrieval of delegation token should not be successful");
     } catch (Exception e) {
       LOG.info("Exception while retrieving delegation token", e);
-      Assert.assertTrue(e.getCause() instanceof AuthorizationException ||
+      assertTrue(e.getCause() instanceof AuthorizationException ||
           e.getCause() instanceof AuthenticationException);
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
index c8f9eb2d522cb..fd2d34f39cc8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
@@ -18,22 +18,12 @@
 
 package org.apache.hadoop.yarn.server.timeline.webapp;
 
-import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
@@ -42,17 +32,28 @@
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -64,23 +65,23 @@
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
 import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class TestTimelineWebServices extends JerseyTestBase {
 
@@ -99,7 +100,7 @@ protected void configureServlets() {
       try {
         store = mockTimelineStore();
       } catch (Exception e) {
-        Assert.fail();
+        fail();
       }
       Configuration conf = new YarnConfiguration();
       conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
@@ -138,7 +139,7 @@ protected void configureServlets() {
       try {
         taFilter.init(filterConfig);
       } catch (ServletException e) {
-        Assert.fail("Unable to initialize TimelineAuthenticationFilter: " +
+        fail("Unable to initialize TimelineAuthenticationFilter: " +
             e.getMessage());
       }
 
@@ -146,7 +147,7 @@ protected void configureServlets() {
       try {
         doNothing().when(taFilter).init(any(FilterConfig.class));
       } catch (ServletException e) {
-        Assert.fail("Unable to initialize TimelineAuthenticationFilter: " +
+        fail("Unable to initialize TimelineAuthenticationFilter: " +
             e.getMessage());
       }
       filter("/*").through(taFilter);
@@ -158,7 +159,7 @@ protected void configureServlets() {
         Guice.createInjector(new WebServletModule()));
   }
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     super.setUp();
     GuiceServletConfig.setInjector(
@@ -187,7 +188,7 @@ public TestTimelineWebServices() {
   }
 
   @Test
-  public void testAbout() throws Exception {
+  void testAbout() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .accept(MediaType.APPLICATION_JSON)
@@ -197,54 +198,54 @@ public void testAbout() throws Exception {
     TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
     TimelineAbout expectedAbout =
         TimelineUtils.createTimelineAbout("Timeline API");
-    Assert.assertNotNull(
-        "Timeline service about response is null", actualAbout);
-    Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
-    Assert.assertEquals(expectedAbout.getTimelineServiceVersion(),
+    assertNotNull(
+        actualAbout, "Timeline service about response is null");
+    assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
+    assertEquals(expectedAbout.getTimelineServiceVersion(),
         actualAbout.getTimelineServiceVersion());
-    Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
+    assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
         actualAbout.getTimelineServiceBuildVersion());
-    Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
+    assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
         actualAbout.getTimelineServiceVersionBuiltOn());
-    Assert.assertEquals(expectedAbout.getHadoopVersion(),
+    assertEquals(expectedAbout.getHadoopVersion(),
         actualAbout.getHadoopVersion());
-    Assert.assertEquals(expectedAbout.getHadoopBuildVersion(),
+    assertEquals(expectedAbout.getHadoopBuildVersion(),
         actualAbout.getHadoopBuildVersion());
-    Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
+    assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
         actualAbout.getHadoopVersionBuiltOn());
   }
 
   private static void verifyEntities(TimelineEntities entities) {
-    Assert.assertNotNull(entities);
-    Assert.assertEquals(3, entities.getEntities().size());
+    assertNotNull(entities);
+    assertEquals(3, entities.getEntities().size());
     TimelineEntity entity1 = entities.getEntities().get(0);
-    Assert.assertNotNull(entity1);
-    Assert.assertEquals("id_1", entity1.getEntityId());
-    Assert.assertEquals("type_1", entity1.getEntityType());
-    Assert.assertEquals(123l, entity1.getStartTime().longValue());
-    Assert.assertEquals(2, entity1.getEvents().size());
-    Assert.assertEquals(4, entity1.getPrimaryFilters().size());
-    Assert.assertEquals(4, entity1.getOtherInfo().size());
+    assertNotNull(entity1);
+    assertEquals("id_1", entity1.getEntityId());
+    assertEquals("type_1", entity1.getEntityType());
+    assertEquals(123L, entity1.getStartTime().longValue());
+    assertEquals(2, entity1.getEvents().size());
+    assertEquals(4, entity1.getPrimaryFilters().size());
+    assertEquals(4, entity1.getOtherInfo().size());
     TimelineEntity entity2 = entities.getEntities().get(1);
-    Assert.assertNotNull(entity2);
-    Assert.assertEquals("id_2", entity2.getEntityId());
-    Assert.assertEquals("type_1", entity2.getEntityType());
-    Assert.assertEquals(123l, entity2.getStartTime().longValue());
-    Assert.assertEquals(2, entity2.getEvents().size());
-    Assert.assertEquals(4, entity2.getPrimaryFilters().size());
-    Assert.assertEquals(4, entity2.getOtherInfo().size());
+    assertNotNull(entity2);
+    assertEquals("id_2", entity2.getEntityId());
+    assertEquals("type_1", entity2.getEntityType());
+    assertEquals(123L, entity2.getStartTime().longValue());
+    assertEquals(2, entity2.getEvents().size());
+    assertEquals(4, entity2.getPrimaryFilters().size());
+    assertEquals(4, entity2.getOtherInfo().size());
     TimelineEntity entity3 = entities.getEntities().get(2);
-    Assert.assertNotNull(entity2);
-    Assert.assertEquals("id_6", entity3.getEntityId());
-    Assert.assertEquals("type_1", entity3.getEntityType());
-    Assert.assertEquals(61l, entity3.getStartTime().longValue());
-    Assert.assertEquals(0, entity3.getEvents().size());
-    Assert.assertEquals(4, entity3.getPrimaryFilters().size());
-    Assert.assertEquals(4, entity3.getOtherInfo().size());
+    assertNotNull(entity2);
+    assertEquals("id_6", entity3.getEntityId());
+    assertEquals("type_1", entity3.getEntityType());
+    assertEquals(61L, entity3.getStartTime().longValue());
+    assertEquals(0, entity3.getEvents().size());
+    assertEquals(4, entity3.getPrimaryFilters().size());
+    assertEquals(4, entity3.getOtherInfo().size());
   }
 
   @Test
-  public void testGetEntities() throws Exception {
+  void testGetEntities() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1")
@@ -256,7 +257,7 @@ public void testGetEntities() throws Exception {
   }
 
   @Test
-  public void testFromId() throws Exception {
+  void testFromId() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("fromId", "id_2")
@@ -278,7 +279,7 @@ public void testFromId() throws Exception {
   }
 
   @Test
-  public void testFromTs() throws Exception {
+  void testFromTs() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("fromTs", Long.toString(beforeTime))
@@ -291,7 +292,7 @@ public void testFromTs() throws Exception {
 
     response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("fromTs", Long.toString(
-            System.currentTimeMillis()))
+        System.currentTimeMillis()))
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -301,7 +302,7 @@ public void testFromTs() throws Exception {
   }
 
   @Test
-  public void testPrimaryFilterString() {
+  void testPrimaryFilterString() {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("primaryFilter", "user:username")
@@ -313,11 +314,11 @@ public void testPrimaryFilterString() {
   }
 
   @Test
-  public void testPrimaryFilterInteger() {
+  void testPrimaryFilterInteger() {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("primaryFilter",
-            "appname:" + Integer.toString(Integer.MAX_VALUE))
+        "appname:" + Integer.toString(Integer.MAX_VALUE))
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -326,11 +327,11 @@ public void testPrimaryFilterInteger() {
   }
 
   @Test
-  public void testPrimaryFilterLong() {
+  void testPrimaryFilterLong() {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").queryParam("primaryFilter",
-            "long:" + Long.toString((long) Integer.MAX_VALUE + 1l))
+        "long:" + Long.toString((long) Integer.MAX_VALUE + 1L))
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -339,7 +340,7 @@ public void testPrimaryFilterLong() {
   }
 
   @Test
-  public void testSecondaryFilters() {
+  void testSecondaryFilters() {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1")
@@ -353,7 +354,7 @@ public void testSecondaryFilters() {
   }
 
   @Test
-  public void testGetEntity() throws Exception {
+  void testGetEntity() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").path("id_1")
@@ -362,17 +363,17 @@ public void testGetEntity() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     TimelineEntity entity = response.getEntity(TimelineEntity.class);
-    Assert.assertNotNull(entity);
-    Assert.assertEquals("id_1", entity.getEntityId());
-    Assert.assertEquals("type_1", entity.getEntityType());
-    Assert.assertEquals(123l, entity.getStartTime().longValue());
-    Assert.assertEquals(2, entity.getEvents().size());
-    Assert.assertEquals(4, entity.getPrimaryFilters().size());
-    Assert.assertEquals(4, entity.getOtherInfo().size());
+    assertNotNull(entity);
+    assertEquals("id_1", entity.getEntityId());
+    assertEquals("type_1", entity.getEntityType());
+    assertEquals(123L, entity.getStartTime().longValue());
+    assertEquals(2, entity.getEvents().size());
+    assertEquals(4, entity.getPrimaryFilters().size());
+    assertEquals(4, entity.getOtherInfo().size());
   }
 
   @Test
-  public void testGetEntityFields1() throws Exception {
+  void testGetEntityFields1() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
@@ -381,37 +382,37 @@ public void testGetEntityFields1() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     TimelineEntity entity = response.getEntity(TimelineEntity.class);
-    Assert.assertNotNull(entity);
-    Assert.assertEquals("id_1", entity.getEntityId());
-    Assert.assertEquals("type_1", entity.getEntityType());
-    Assert.assertEquals(123l, entity.getStartTime().longValue());
-    Assert.assertEquals(2, entity.getEvents().size());
-    Assert.assertEquals(0, entity.getPrimaryFilters().size());
-    Assert.assertEquals(4, entity.getOtherInfo().size());
+    assertNotNull(entity);
+    assertEquals("id_1", entity.getEntityId());
+    assertEquals("type_1", entity.getEntityType());
+    assertEquals(123L, entity.getStartTime().longValue());
+    assertEquals(2, entity.getEvents().size());
+    assertEquals(0, entity.getPrimaryFilters().size());
+    assertEquals(4, entity.getOtherInfo().size());
   }
 
   @Test
-  public void testGetEntityFields2() throws Exception {
+  void testGetEntityFields2() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").path("id_1").queryParam("fields", "lasteventonly," +
-            "primaryfilters,relatedentities")
+        "primaryfilters,relatedentities")
         .accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     TimelineEntity entity = response.getEntity(TimelineEntity.class);
-    Assert.assertNotNull(entity);
-    Assert.assertEquals("id_1", entity.getEntityId());
-    Assert.assertEquals("type_1", entity.getEntityType());
-    Assert.assertEquals(123l, entity.getStartTime().longValue());
-    Assert.assertEquals(1, entity.getEvents().size());
-    Assert.assertEquals(4, entity.getPrimaryFilters().size());
-    Assert.assertEquals(0, entity.getOtherInfo().size());
+    assertNotNull(entity);
+    assertEquals("id_1", entity.getEntityId());
+    assertEquals("type_1", entity.getEntityType());
+    assertEquals(123L, entity.getStartTime().longValue());
+    assertEquals(1, entity.getEvents().size());
+    assertEquals(4, entity.getPrimaryFilters().size());
+    assertEquals(0, entity.getOtherInfo().size());
   }
 
   @Test
-  public void testGetEvents() throws Exception {
+  void testGetEvents() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("type_1").path("events")
@@ -421,22 +422,22 @@ public void testGetEvents() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     TimelineEvents events = response.getEntity(TimelineEvents.class);
-    Assert.assertNotNull(events);
-    Assert.assertEquals(1, events.getAllEvents().size());
+    assertNotNull(events);
+    assertEquals(1, events.getAllEvents().size());
     TimelineEvents.EventsOfOneEntity partEvents = events.getAllEvents().get(0);
-    Assert.assertEquals(2, partEvents.getEvents().size());
+    assertEquals(2, partEvents.getEvents().size());
     TimelineEvent event1 = partEvents.getEvents().get(0);
-    Assert.assertEquals(456l, event1.getTimestamp());
-    Assert.assertEquals("end_event", event1.getEventType());
-    Assert.assertEquals(1, event1.getEventInfo().size());
+    assertEquals(456L, event1.getTimestamp());
+    assertEquals("end_event", event1.getEventType());
+    assertEquals(1, event1.getEventInfo().size());
     TimelineEvent event2 = partEvents.getEvents().get(1);
-    Assert.assertEquals(123l, event2.getTimestamp());
-    Assert.assertEquals("start_event", event2.getEventType());
-    Assert.assertEquals(0, event2.getEventInfo().size());
+    assertEquals(123L, event2.getTimestamp());
+    assertEquals("start_event", event2.getEventType());
+    assertEquals(0, event2.getEventInfo().size());
   }
 
   @Test
-  public void testPostEntitiesWithPrimaryFilter() throws Exception {
+  void testPostEntitiesWithPrimaryFilter() throws Exception {
     TimelineEntities entities = new TimelineEntities();
     TimelineEntity entity = new TimelineEntity();
     Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>();
@@ -455,11 +456,11 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception {
         .post(ClientResponse.class, entities);
     TimelinePutResponse putResposne =
         response.getEntity(TimelinePutResponse.class);
-    Assert.assertEquals(0, putResposne.getErrors().size());
+    assertEquals(0, putResposne.getErrors().size());
   }
 
   @Test
-  public void testPostEntities() throws Exception {
+  void testPostEntities() throws Exception {
     TimelineEntities entities = new TimelineEntities();
     TimelineEntity entity = new TimelineEntity();
     entity.setEntityId("test id 1");
@@ -486,8 +487,8 @@ public void testPostEntities() throws Exception {
         response.getType().toString());
     TimelinePutResponse putResposne =
         response.getEntity(TimelinePutResponse.class);
-    Assert.assertNotNull(putResposne);
-    Assert.assertEquals(0, putResposne.getErrors().size());
+    assertNotNull(putResposne);
+    assertEquals(0, putResposne.getErrors().size());
     // verify the entity exists in the store
     response = r.path("ws").path("v1").path("timeline")
         .path("test type 1").path("test id 1")
@@ -496,13 +497,13 @@ public void testPostEntities() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     entity = response.getEntity(TimelineEntity.class);
-    Assert.assertNotNull(entity);
-    Assert.assertEquals("test id 1", entity.getEntityId());
-    Assert.assertEquals("test type 1", entity.getEntityType());
+    assertNotNull(entity);
+    assertEquals("test id 1", entity.getEntityId());
+    assertEquals("test type 1", entity.getEntityType());
   }
 
   @Test
-  public void testPostIncompleteEntities() throws Exception {
+  void testPostIncompleteEntities() throws Exception {
     TimelineEntities entities = new TimelineEntities();
     TimelineEntity entity1 = new TimelineEntity();
     entity1.setEntityId("test id 1");
@@ -516,14 +517,14 @@ public void testPostIncompleteEntities() throws Exception {
     // One of the entities has no id or type. HTTP 400 will be returned
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .queryParam("user.name", "tester").accept(MediaType.APPLICATION_JSON)
-         .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
+        .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
   }
 
   @Test
-  public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
+  void testPostEntitiesWithYarnACLsEnabled() throws Exception {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -544,8 +545,8 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
           response.getType().toString());
       TimelinePutResponse putResponse =
           response.getEntity(TimelinePutResponse.class);
-      Assert.assertNotNull(putResponse);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertNotNull(putResponse);
+      assertEquals(0, putResponse.getErrors().size());
 
       // override/append timeline data in the same entity with different user
       response = r.path("ws").path("v1").path("timeline")
@@ -556,9 +557,9 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       putResponse = response.getEntity(TimelinePutResponse.class);
-      Assert.assertNotNull(putResponse);
-      Assert.assertEquals(1, putResponse.getErrors().size());
-      Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
+      assertNotNull(putResponse);
+      assertEquals(1, putResponse.getErrors().size());
+      assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
           putResponse.getErrors().get(0).getErrorCode());
 
       // Cross domain relationship will be rejected
@@ -580,9 +581,9 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       putResponse = response.getEntity(TimelinePutResponse.class);
-      Assert.assertNotNull(putResponse);
-      Assert.assertEquals(1, putResponse.getErrors().size());
-      Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION,
+      assertNotNull(putResponse);
+      assertEquals(1, putResponse.getErrors().size());
+      assertEquals(TimelinePutError.FORBIDDEN_RELATION,
           putResponse.getErrors().get(0).getErrorCode());
 
       // Make sure the entity has been added anyway even though the
@@ -595,16 +596,16 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       entity = response.getEntity(TimelineEntity.class);
-      Assert.assertNotNull(entity);
-      Assert.assertEquals("test id 3", entity.getEntityId());
-      Assert.assertEquals("test type 2", entity.getEntityType());
+      assertNotNull(entity);
+      assertEquals("test id 3", entity.getEntityId());
+      assertEquals("test type 2", entity.getEntityType());
     } finally {
       timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
     }
   }
 
   @Test
-  public void testPostEntitiesToDefaultDomain() throws Exception {
+  void testPostEntitiesToDefaultDomain() throws Exception {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -624,8 +625,8 @@ public void testPostEntitiesToDefaultDomain() throws Exception {
           response.getType().toString());
       TimelinePutResponse putResposne =
           response.getEntity(TimelinePutResponse.class);
-      Assert.assertNotNull(putResposne);
-      Assert.assertEquals(0, putResposne.getErrors().size());
+      assertNotNull(putResposne);
+      assertEquals(0, putResposne.getErrors().size());
       // verify the entity exists in the store
       response = r.path("ws").path("v1").path("timeline")
           .path("test type 7").path("test id 7")
@@ -635,10 +636,10 @@ public void testPostEntitiesToDefaultDomain() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       entity = response.getEntity(TimelineEntity.class);
-      Assert.assertNotNull(entity);
-      Assert.assertEquals("test id 7", entity.getEntityId());
-      Assert.assertEquals("test type 7", entity.getEntityType());
-      Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
+      assertNotNull(entity);
+      assertEquals("test id 7", entity.getEntityId());
+      assertEquals("test type 7", entity.getEntityType());
+      assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
           entity.getDomainId());
     } finally {
       timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
@@ -646,7 +647,7 @@ public void testPostEntitiesToDefaultDomain() throws Exception {
   }
 
   @Test
-  public void testGetEntityWithYarnACLsEnabled() throws Exception {
+  void testGetEntityWithYarnACLsEnabled() throws Exception {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -667,7 +668,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception {
           response.getType().toString());
       TimelinePutResponse putResponse =
           response.getEntity(TimelinePutResponse.class);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertEquals(0, putResponse.getErrors().size());
       // verify the system data will not be exposed
       // 1. No field specification
       response = r.path("ws").path("v1").path("timeline")
@@ -678,7 +679,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       entity = response.getEntity(TimelineEntity.class);
-      Assert.assertNull(entity.getPrimaryFilters().get(
+      assertNull(entity.getPrimaryFilters().get(
           TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
       // 2. other field
       response = r.path("ws").path("v1").path("timeline")
@@ -690,7 +691,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       entity = response.getEntity(TimelineEntity.class);
-      Assert.assertNull(entity.getPrimaryFilters().get(
+      assertNull(entity.getPrimaryFilters().get(
           TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
       // 3. primaryfilters field
       response = r.path("ws").path("v1").path("timeline")
@@ -702,7 +703,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       entity = response.getEntity(TimelineEntity.class);
-      Assert.assertNull(entity.getPrimaryFilters().get(
+      assertNull(entity.getPrimaryFilters().get(
           TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
 
       // get entity with other user
@@ -720,7 +721,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception {
   }
 
   @Test
-  public void testGetEntitiesWithYarnACLsEnabled() {
+  void testGetEntitiesWithYarnACLsEnabled() {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -742,7 +743,7 @@ public void testGetEntitiesWithYarnACLsEnabled() {
           response.getType().toString());
       TimelinePutResponse putResponse =
           response.getEntity(TimelinePutResponse.class);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertEquals(0, putResponse.getErrors().size());
 
       // Put entity [4, 5] in domain 2
       entities = new TimelineEntities();
@@ -761,7 +762,7 @@ public void testGetEntitiesWithYarnACLsEnabled() {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       putResponse = response.getEntity(TimelinePutResponse.class);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertEquals(0, putResponse.getErrors().size());
 
       // Query entities of type 4
       response = r.path("ws").path("v1").path("timeline")
@@ -782,7 +783,7 @@ public void testGetEntitiesWithYarnACLsEnabled() {
   }
 
   @Test
-  public void testGetEventsWithYarnACLsEnabled() {
+  void testGetEventsWithYarnACLsEnabled() {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -808,7 +809,7 @@ public void testGetEventsWithYarnACLsEnabled() {
           response.getType().toString());
       TimelinePutResponse putResponse =
           response.getEntity(TimelinePutResponse.class);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertEquals(0, putResponse.getErrors().size());
 
       // Put entity [5, 6] in domain 2
       entities = new TimelineEntities();
@@ -831,7 +832,7 @@ public void testGetEventsWithYarnACLsEnabled() {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       putResponse = response.getEntity(TimelinePutResponse.class);
-      Assert.assertEquals(0, putResponse.getErrors().size());
+      assertEquals(0, putResponse.getErrors().size());
 
       // Query events belonging to the entities of type 4
       response = r.path("ws").path("v1").path("timeline")
@@ -852,7 +853,7 @@ public void testGetEventsWithYarnACLsEnabled() {
   }
 
   @Test
-  public void testGetDomain() throws Exception {
+  void testGetDomain() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("domain").path("domain_id_1")
@@ -865,7 +866,7 @@ public void testGetDomain() throws Exception {
   }
 
   @Test
-  public void testGetDomainYarnACLsEnabled() {
+  void testGetDomainYarnACLsEnabled() {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -894,7 +895,7 @@ public void testGetDomainYarnACLsEnabled() {
   }
 
   @Test
-  public void testGetDomains() throws Exception {
+  void testGetDomains() throws Exception {
     WebResource r = resource();
     ClientResponse response = r.path("ws").path("v1").path("timeline")
         .path("domain")
@@ -904,7 +905,7 @@ public void testGetDomains() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     TimelineDomains domains = response.getEntity(TimelineDomains.class);
-    Assert.assertEquals(2, domains.getDomains().size());
+    assertEquals(2, domains.getDomains().size());
     for (int i = 0; i < domains.getDomains().size(); ++i) {
       verifyDomain(domains.getDomains().get(i),
           i == 0 ? "domain_id_4" : "domain_id_1");
@@ -912,7 +913,7 @@ public void testGetDomains() throws Exception {
   }
 
   @Test
-  public void testGetDomainsYarnACLsEnabled() throws Exception {
+  void testGetDomainsYarnACLsEnabled() throws Exception {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -925,7 +926,7 @@ public void testGetDomainsYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       TimelineDomains domains = response.getEntity(TimelineDomains.class);
-      Assert.assertEquals(2, domains.getDomains().size());
+      assertEquals(2, domains.getDomains().size());
       for (int i = 0; i < domains.getDomains().size(); ++i) {
         verifyDomain(domains.getDomains().get(i),
             i == 0 ? "domain_id_4" : "domain_id_1");
@@ -940,14 +941,14 @@ public void testGetDomainsYarnACLsEnabled() throws Exception {
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       domains = response.getEntity(TimelineDomains.class);
-      Assert.assertEquals(0, domains.getDomains().size());
+      assertEquals(0, domains.getDomains().size());
     } finally {
       timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
     }
   }
 
   @Test
-  public void testPutDomain() throws Exception {
+  void testPutDomain() throws Exception {
     TimelineDomain domain = new TimelineDomain();
     domain.setId("test_domain_id");
     WebResource r = resource();
@@ -977,10 +978,10 @@ public void testPutDomain() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     domain = response.getEntity(TimelineDomain.class);
-    Assert.assertNotNull(domain);
-    Assert.assertEquals("test_domain_id", domain.getId());
-    Assert.assertEquals("tester", domain.getOwner());
-    Assert.assertEquals(null, domain.getDescription());
+    assertNotNull(domain);
+    assertEquals("test_domain_id", domain.getId());
+    assertEquals("tester", domain.getOwner());
+    assertNull(domain.getDescription());
 
     // Update the domain
     domain.setDescription("test_description");
@@ -1000,13 +1001,13 @@ public void testPutDomain() throws Exception {
     assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     domain = response.getEntity(TimelineDomain.class);
-    Assert.assertNotNull(domain);
-    Assert.assertEquals("test_domain_id", domain.getId());
-    Assert.assertEquals("test_description", domain.getDescription());
+    assertNotNull(domain);
+    assertEquals("test_domain_id", domain.getId());
+    assertEquals("test_description", domain.getDescription());
   }
 
   @Test
-  public void testPutDomainYarnACLsEnabled() throws Exception {
+  void testPutDomainYarnACLsEnabled() throws Exception {
     AdminACLsManager oldAdminACLsManager =
         timelineACLsManager.setAdminACLsManager(adminACLsManager);
     try {
@@ -1035,7 +1036,7 @@ public void testPutDomainYarnACLsEnabled() throws Exception {
   }
 
   @Test
-  public void testContextFactory() throws Exception {
+  void testContextFactory() throws Exception {
     JAXBContext jaxbContext1 = ContextFactory.createContext(
         new Class[]{TimelineDomain.class}, Collections.EMPTY_MAP);
     JAXBContext jaxbContext2 = ContextFactory.createContext(
@@ -1045,21 +1046,21 @@ public void testContextFactory() throws Exception {
     try {
       ContextFactory.createContext(new Class[]{TimelineEntity.class},
           Collections.EMPTY_MAP);
-      Assert.fail("Expected JAXBException");
-    } catch(Exception e) {
+      fail("Expected JAXBException");
+    } catch (Exception e) {
       assertThat(e).isExactlyInstanceOf(JAXBException.class);
     }
   }
 
   private static void verifyDomain(TimelineDomain domain, String domainId) {
-    Assert.assertNotNull(domain);
-    Assert.assertEquals(domainId, domain.getId());
+    assertNotNull(domain);
+    assertEquals(domainId, domain.getId());
     // The specific values have been verified in TestMemoryTimelineStore
-    Assert.assertNotNull(domain.getDescription());
-    Assert.assertNotNull(domain.getOwner());
-    Assert.assertNotNull(domain.getReaders());
-    Assert.assertNotNull(domain.getWriters());
-    Assert.assertNotNull(domain.getCreatedTime());
-    Assert.assertNotNull(domain.getModifiedTime());
+    assertNotNull(domain.getDescription());
+    assertNotNull(domain.getOwner());
+    assertNotNull(domain.getReaders());
+    assertNotNull(domain.getWriters());
+    assertNotNull(domain.getCreatedTime());
+    assertNotNull(domain.getModifiedTime());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
index 46d5b6b873c74..70b58aaca01f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
@@ -23,6 +23,12 @@
 import java.net.URI;
 import java.util.EnumSet;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -35,17 +41,13 @@
 import org.apache.hadoop.yarn.client.api.impl.TimelineWriter;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
 
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestTimelineWebServicesWithSSL {
 
@@ -59,7 +61,7 @@ public class TestTimelineWebServicesWithSSL {
   private static TimelineStore store;
   private static Configuration conf;
 
-  @BeforeClass
+  @BeforeAll
   public static void setupServer() throws Exception {
     conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -85,7 +87,7 @@ public static void setupServer() throws Exception {
     store = timelineServer.getTimelineStore();
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDownServer() throws Exception {
     if (timelineServer != null) {
       timelineServer.stop();
@@ -93,7 +95,7 @@ public static void tearDownServer() throws Exception {
   }
 
   @Test
-  public void testPutEntities() throws Exception {
+  void testPutEntities() throws Exception {
     TestTimelineClient client = new TestTimelineClient();
     try {
       client.init(conf);
@@ -108,16 +110,16 @@ public void testPutEntities() throws Exception {
       expectedEntity.addEvent(event);
 
       TimelinePutResponse response = client.putEntities(expectedEntity);
-      Assert.assertEquals(0, response.getErrors().size());
-      Assert.assertTrue(client.resp.toString().contains("https"));
+      assertEquals(0, response.getErrors().size());
+      assertTrue(client.resp.toString().contains("https"));
 
       TimelineEntity actualEntity = store.getEntity(
           expectedEntity.getEntityId(), expectedEntity.getEntityType(),
           EnumSet.allOf(Field.class));
-      Assert.assertNotNull(actualEntity);
-      Assert.assertEquals(
+      assertNotNull(actualEntity);
+      assertEquals(
           expectedEntity.getEntityId(), actualEntity.getEntityId());
-      Assert.assertEquals(
+      assertEquals(
           expectedEntity.getEntityType(), actualEntity.getEntityType());
     } finally {
       client.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 9b61479528bf1..6837de8001464 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -125,6 +125,12 @@
     <dependency>
       <groupId>org.apache.geronimo.specs</groupId>
       <artifactId>geronimo-jcache_1.0_spec</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.osgi</groupId>
+          <artifactId>org.osgi.core</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.ehcache</groupId>
@@ -138,6 +144,7 @@
       <groupId>org.hsqldb</groupId>
       <artifactId>hsqldb</artifactId>
       <scope>test</scope>
+      <classifier>jdk8</classifier>
     </dependency>
     <dependency>
       <groupId>com.microsoft.sqlserver</groupId>
@@ -163,6 +170,7 @@
         <artifactId>maven-antrun-plugin</artifactId>
         <executions>
           <execution>
+            <id>copy-site</id>
             <phase>pre-site</phase>
             <goals>
               <goal>run</goal>
@@ -173,6 +181,21 @@
               </tasks>
             </configuration>
           </execution>
+          <execution>
+            <id>copy-sql</id>
+            <phase>test</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <tasks>
+                <copy file="${basedir}/../../bin/FederationStateStore/MySQL/FederationStateStoreTables.sql" todir="${project.build.directory}/test-classes/MySQL"/>
+                <copy file="${basedir}/../../bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql" todir="${project.build.directory}/test-classes/MySQL"/>
+                <copy file="${basedir}/../../bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql" todir="${project.build.directory}/test-classes/SQLServer"/>
+                <copy file="${basedir}/../../bin/FederationStateStore/SQLServer/FederationStateStoreStoredProcs.sql" todir="${project.build.directory}/test-classes/SQLServer"/>
+              </tasks>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
       <plugin>
@@ -190,6 +213,9 @@
                 <additionalProtoPathElement>
                   ${basedir}/../../hadoop-yarn-api/src/main/proto
                 </additionalProtoPathElement>
+                <additionalProtoPathElement>
+                  ${basedir}/../../hadoop-yarn-common/src/main/proto
+                </additionalProtoPathElement>
               </additionalProtoPathElements>
             </configuration>
           </execution>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
index 9542798b93a95..9a73fb308ce0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
@@ -33,7 +33,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Extends Thread and provides an implementation that is used for processing the
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index 9f1870188ece6..50eed3a75d511 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.yarn.client.RMProxy;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index e523ea1c5df35..e0912c8c811b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -43,7 +43,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * A FailoverProxyProvider implementation that uses the
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
index 9b795b0507bd2..fc8cc5bd464b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
@@ -20,7 +20,7 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.List;
+import java.util.Collection;
 import java.util.Random;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -188,8 +188,8 @@ public static FederationAMRMProxyPolicy loadAMRMPolicy(String queue,
    * @throws FederationPolicyException if there are no usable subclusters.
    */
   public static void validateSubClusterAvailability(
-      List<SubClusterId> activeSubClusters,
-      List<SubClusterId> blackListSubClusters)
+      Collection<SubClusterId> activeSubClusters,
+      Collection<SubClusterId> blackListSubClusters)
       throws FederationPolicyException {
     if (activeSubClusters != null && !activeSubClusters.isEmpty()) {
       if (blackListSubClusters == null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
index 0cbda314f287e..c4fc7322a1eca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java
@@ -25,6 +25,7 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
@@ -136,7 +137,7 @@ public SubClusterId getHomeSubcluster(
 
     if (appSubmissionContext == null) {
       throw new FederationPolicyException(
-          "The ApplicationSubmissionContext " + "cannot be null.");
+          "The ApplicationSubmissionContext cannot be null.");
     }
 
     String queue = appSubmissionContext.getQueue();
@@ -148,51 +149,7 @@ public SubClusterId getHomeSubcluster(
       queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
     }
 
-    // the facade might cache this request, based on its parameterization
-    SubClusterPolicyConfiguration configuration = null;
-
-    try {
-      configuration = federationFacade.getPolicyConfiguration(queue);
-    } catch (YarnException e) {
-      String errMsg = "There is no policy configured for the queue: " + queue
-          + ", falling back to defaults.";
-      LOG.warn(errMsg, e);
-    }
-
-    // If there is no policy configured for this queue, fallback to the baseline
-    // policy that is configured either in the store or via XML config (and
-    // cached)
-    if (configuration == null) {
-      LOG.warn("There is no policies configured for queue: " + queue + " we"
-          + " fallback to default policy for: "
-          + YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
-
-      queue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
-      try {
-        configuration = federationFacade.getPolicyConfiguration(queue);
-      } catch (YarnException e) {
-        String errMsg = "Cannot retrieve policy configured for the queue: "
-            + queue + ", falling back to defaults.";
-        LOG.warn(errMsg, e);
-
-      }
-    }
-
-    // the fallback is not configure via store, but via XML, using
-    // previously loaded configuration.
-    if (configuration == null) {
-      configuration =
-          cachedConfs.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
-    }
-
-    // if the configuration has changed since last loaded, reinit the policy
-    // based on current configuration
-    if (!cachedConfs.containsKey(queue)
-        || !cachedConfs.get(queue).equals(configuration)) {
-      singlePolicyReinit(policyMap, cachedConfs, queue, configuration);
-    }
-
-    FederationRouterPolicy policy = policyMap.get(queue);
+    FederationRouterPolicy policy = getFederationRouterPolicy(cachedConfs, policyMap, queue);
     if (policy == null) {
       // this should never happen, as the to maps are updated together
       throw new FederationPolicyException("No FederationRouterPolicy found "
@@ -262,4 +219,92 @@ public synchronized void reset() {
 
   }
 
+  /**
+   * This method provides a wrapper of all policy functionalities for routing a
+   * reservation. Internally it manages configuration changes, and policy
+   * init/reinit.
+   *
+   * @param request the reservation to route.
+   *
+   * @return the id of the subcluster that will be the "home" for this
+   *         reservation.
+   *
+   * @throws YarnException if there are issues initializing policies, or no
+   *           valid sub-cluster id could be found for this reservation.
+   */
+  public SubClusterId getReservationHomeSubCluster(
+      ReservationSubmissionRequest request) throws YarnException {
+
+    // the maps are concurrent, but we need to protect from reset()
+    // reinitialization mid-execution by creating a new reference local to this
+    // method.
+    Map<String, SubClusterPolicyConfiguration> cachedConfs = globalConfMap;
+    Map<String, FederationRouterPolicy> policyMap = globalPolicyMap;
+
+    if (request == null) {
+      throw new FederationPolicyException(
+          "The ReservationSubmissionRequest cannot be null.");
+    }
+
+    String queue = request.getQueue();
+    FederationRouterPolicy policy = getFederationRouterPolicy(cachedConfs, policyMap, queue);
+
+    if (policy == null) {
+      // this should never happen, as the to maps are updated together
+      throw new FederationPolicyException("No FederationRouterPolicy found "
+          + "for queue: " + request.getQueue() + " (while routing "
+          + "reservation: " + request.getReservationId() + ") "
+          + "and no default specified.");
+    }
+
+    return policy.getReservationHomeSubcluster(request);
+  }
+
+  private FederationRouterPolicy getFederationRouterPolicy(
+      Map<String, SubClusterPolicyConfiguration> cachedConfiguration,
+      Map<String, FederationRouterPolicy> policyMap, String queue)
+      throws FederationPolicyInitializationException {
+
+    // the facade might cache this request, based on its parameterization
+    SubClusterPolicyConfiguration configuration = null;
+    String copyQueue = queue;
+
+    try {
+      configuration = federationFacade.getPolicyConfiguration(copyQueue);
+    } catch (YarnException e) {
+      LOG.warn("There is no policy configured for the queue: {}, falling back to defaults.",
+          copyQueue, e);
+    }
+
+    // If there is no policy configured for this queue, fallback to the baseline
+    // policy that is configured either in the store or via XML config (and cached)
+    if (configuration == null) {
+      final String policyKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+      LOG.warn("There is no policies configured for queue: {} " +
+          "we fallback to default policy for: {}. ", copyQueue, policyKey);
+      copyQueue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY;
+      try {
+        configuration = federationFacade.getPolicyConfiguration(copyQueue);
+      } catch (YarnException e) {
+        LOG.warn("Cannot retrieve policy configured for the queue: {}, falling back to defaults.",
+            copyQueue, e);
+      }
+    }
+
+    // the fallback is not configure via store, but via XML, using
+    // previously loaded configuration.
+    if (configuration == null) {
+      configuration = cachedConfiguration.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+    }
+
+    // if the configuration has changed since last loaded, reinit the policy
+    // based on current configuration
+    SubClusterPolicyConfiguration policyConfiguration =
+        cachedConfiguration.getOrDefault(copyQueue, null);
+    if (policyConfiguration == null || !policyConfiguration.equals(configuration)) {
+      singlePolicyReinit(policyMap, cachedConfiguration, copyQueue, configuration);
+    }
+
+    return policyMap.get(copyQueue);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
index 2d23340adc6e3..c8f945896e4dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -32,6 +32,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -51,7 +52,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * An implementation of the {@link FederationAMRMProxyPolicy} interface that
@@ -131,6 +132,7 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
   private SubClusterResolver resolver;
 
   private Map<SubClusterId, Resource> headroom;
+  private Map<SubClusterId, EnhancedHeadroom> enhancedHeadroom;
   private float hrAlpha;
   private FederationStateStoreFacade federationFacade;
   private SubClusterId homeSubcluster;
@@ -182,6 +184,7 @@ public void reinitialize(
 
     if (headroom == null) {
       headroom = new ConcurrentHashMap<>();
+      enhancedHeadroom = new ConcurrentHashMap<>();
     }
     hrAlpha = policy.getHeadroomAlpha();
 
@@ -195,9 +198,14 @@ public void notifyOfResponse(SubClusterId subClusterId,
       AllocateResponse response) throws YarnException {
     if (response.getAvailableResources() != null) {
       headroom.put(subClusterId, response.getAvailableResources());
-      LOG.info("Subcluster {} updated with {} memory headroom", subClusterId,
-          response.getAvailableResources().getMemorySize());
     }
+    if (response.getEnhancedHeadroom() != null) {
+      this.enhancedHeadroom.put(subClusterId, response.getEnhancedHeadroom());
+    }
+    LOG.info(
+        "Subcluster {} updated with AvailableResource {}, EnhancedHeadRoom {}",
+        subClusterId, response.getAvailableResources(),
+        response.getEnhancedHeadroom());
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
index 730fb417f883d..3bab20cef1661 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java
@@ -18,15 +18,24 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
+import java.util.List;
 import java.util.Map;
+import java.util.Collections;
 
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 
 /**
  * Base abstract class for {@link FederationRouterPolicy} implementations, that
@@ -63,4 +72,116 @@ public void validate(ApplicationSubmissionContext appSubmissionContext)
     }
   }
 
+  /**
+   * This method is implemented by the specific policy, and it is used to route
+   * both reservations, and applications among a given set of
+   * sub-clusters.
+   *
+   * @param queue the queue for this application/reservation
+   * @param preSelectSubClusters a pre-filter set of sub-clusters
+   * @return the chosen sub-cluster
+   *
+   * @throws YarnException if the policy fails to choose a sub-cluster
+   */
+  protected abstract SubClusterId chooseSubCluster(String queue,
+      Map<SubClusterId, SubClusterInfo> preSelectSubClusters) throws YarnException;
+
+  /**
+   * Filter chosen SubCluster based on reservationId.
+   *
+   * @param reservationId the globally unique identifier for a reservation.
+   * @param activeSubClusters the map of ids to info for all active subclusters.
+   * @return the chosen sub-cluster
+   * @throws YarnException if the policy fails to choose a sub-cluster
+   */
+  protected Map<SubClusterId, SubClusterInfo> prefilterSubClusters(
+      ReservationId reservationId, Map<SubClusterId, SubClusterInfo> activeSubClusters)
+      throws YarnException {
+
+    // if a reservation exists limit scope to the sub-cluster this
+    // reservation is mapped to
+    if (reservationId != null) {
+      // note this might throw YarnException if the reservation is
+      // unknown. This is to be expected, and should be handled by
+      // policy invoker.
+      FederationStateStoreFacade stateStoreFacade =
+          getPolicyContext().getFederationStateStoreFacade();
+      SubClusterId resSubCluster = stateStoreFacade.getReservationHomeSubCluster(reservationId);
+      SubClusterInfo subClusterInfo = activeSubClusters.get(resSubCluster);
+      return Collections.singletonMap(resSubCluster, subClusterInfo);
+    }
+
+    return activeSubClusters;
+  }
+
+  /**
+   * Simply picks from alphabetically-sorted active subclusters based on the
+   * hash of quey name. Jobs of the same queue will all be routed to the same
+   * sub-cluster, as far as the number of active sub-cluster and their names
+   * remain the same.
+   *
+   * @param appContext the {@link ApplicationSubmissionContext} that
+   *          has to be routed to an appropriate subCluster for execution.
+   *
+   * @param blackLists the list of subClusters as identified by
+   *          {@link SubClusterId} to blackList from the selection of the home
+   *          subCluster.
+   *
+   * @return a hash-based chosen {@link SubClusterId} that will be the "home"
+   *         for this application.
+   *
+   * @throws YarnException if there are no active subclusters.
+   */
+  @Override
+  public SubClusterId getHomeSubcluster(ApplicationSubmissionContext appContext,
+      List<SubClusterId> blackLists) throws YarnException {
+
+    // null checks and default-queue behavior
+    validate(appContext);
+
+    // apply filtering based on reservation location and active sub-clusters
+    Map<SubClusterId, SubClusterInfo> filteredSubClusters = prefilterSubClusters(
+        appContext.getReservationID(), getActiveSubclusters());
+
+    FederationPolicyUtils.validateSubClusterAvailability(filteredSubClusters.keySet(), blackLists);
+
+    // remove black SubCluster
+    if (blackLists != null) {
+      blackLists.forEach(filteredSubClusters::remove);
+    }
+
+    // pick the chosen subCluster from the active ones
+    return chooseSubCluster(appContext.getQueue(), filteredSubClusters);
+  }
+
+  /**
+   * This method provides a wrapper of all policy functionalities for routing a
+   * reservation. Internally it manages configuration changes, and policy
+   * init/reinit.
+   *
+   * @param request the reservation to route.
+   *
+   * @return the id of the subcluster that will be the "home" for this
+   *         reservation.
+   *
+   * @throws YarnException if there are issues initializing policies, or no
+   *           valid sub-cluster id could be found for this reservation.
+   */
+  @Override
+  public SubClusterId getReservationHomeSubcluster(ReservationSubmissionRequest request)
+      throws YarnException {
+    if (request == null) {
+      throw new FederationPolicyException("The ReservationSubmissionRequest cannot be null.");
+    }
+
+    if (request.getQueue() == null) {
+      request.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
+    }
+
+    // apply filtering based on reservation location and active sub-clusters
+    Map<SubClusterId, SubClusterInfo> filteredSubClusters = getActiveSubclusters();
+
+    // pick the chosen subCluster from the active ones
+    return chooseSubCluster(request.getQueue(), filteredSubClusters);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
index 9325bd8ca2a15..af5810665913c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java
@@ -19,6 +19,7 @@
 
 import java.util.List;
 
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
@@ -49,4 +50,16 @@ public interface FederationRouterPolicy extends ConfigurableFederationPolicy {
   SubClusterId getHomeSubcluster(
       ApplicationSubmissionContext appSubmissionContext,
       List<SubClusterId> blackListSubClusters) throws YarnException;
+
+  /**
+   * Determines the sub-cluster where a ReservationSubmissionRequest should be
+   * sent to.
+   *
+   * @param request the original request
+   * @return a mapping of sub-clusters and the requests
+   *
+   * @throws YarnException if the policy fails to choose a sub-cluster
+   */
+  SubClusterId getReservationHomeSubcluster(
+      ReservationSubmissionRequest request) throws YarnException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
index cc11880665335..5ac2d1cce0720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java
@@ -22,11 +22,9 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -50,53 +48,12 @@ public void reinitialize(
     setPolicyContext(federationPolicyContext);
   }
 
-  /**
-   * Simply picks from alphabetically-sorted active subclusters based on the
-   * hash of quey name. Jobs of the same queue will all be routed to the same
-   * sub-cluster, as far as the number of active sub-cluster and their names
-   * remain the same.
-   *
-   * @param appSubmissionContext the {@link ApplicationSubmissionContext} that
-   *          has to be routed to an appropriate subCluster for execution.
-   *
-   * @param blackListSubClusters the list of subClusters as identified by
-   *          {@link SubClusterId} to blackList from the selection of the home
-   *          subCluster.
-   *
-   * @return a hash-based chosen {@link SubClusterId} that will be the "home"
-   *         for this application.
-   *
-   * @throws YarnException if there are no active subclusters.
-   */
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blackListSubClusters) throws YarnException {
-
-    // throws if no active subclusters available
-    Map<SubClusterId, SubClusterInfo> activeSubclusters =
-        getActiveSubclusters();
-
-    FederationPolicyUtils.validateSubClusterAvailability(
-        new ArrayList<SubClusterId>(activeSubclusters.keySet()),
-        blackListSubClusters);
-
-    if (blackListSubClusters != null) {
-
-      // Remove from the active SubClusters from StateStore the blacklisted ones
-      for (SubClusterId scId : blackListSubClusters) {
-        activeSubclusters.remove(scId);
-      }
-    }
-
-    validate(appSubmissionContext);
-
-    int chosenPosition = Math.abs(
-        appSubmissionContext.getQueue().hashCode() % activeSubclusters.size());
-
-    List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
+  protected SubClusterId chooseSubCluster(String queue,
+      Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
+    int chosenPosition = Math.abs(queue.hashCode() % preSelectSubclusters.size());
+    List<SubClusterId> list = new ArrayList<>(preSelectSubclusters.keySet());
     Collections.sort(list);
     return list.get(chosenPosition);
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index fa5eb4be2cfd5..a86a43a213de0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -17,14 +17,10 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
@@ -65,28 +61,12 @@ public void reinitialize(FederationPolicyInitializationContext policyContext)
   }
 
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blacklist) throws YarnException {
-
-    // null checks and default-queue behavior
-    validate(appSubmissionContext);
-
-    Map<SubClusterId, SubClusterInfo> activeSubclusters =
-        getActiveSubclusters();
-
-    FederationPolicyUtils.validateSubClusterAvailability(
-        new ArrayList<SubClusterId>(activeSubclusters.keySet()), blacklist);
-
-    Map<SubClusterIdInfo, Float> weights =
-        getPolicyInfo().getRouterPolicyWeights();
+  protected SubClusterId chooseSubCluster(
+      String queue, Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
+    Map<SubClusterIdInfo, Float> weights = getPolicyInfo().getRouterPolicyWeights();
     SubClusterIdInfo chosen = null;
     long currBestMem = -1;
-    for (Map.Entry<SubClusterId, SubClusterInfo> entry : activeSubclusters
-        .entrySet()) {
-      if (blacklist != null && blacklist.contains(entry.getKey())) {
-        continue;
-      }
+    for (Map.Entry<SubClusterId, SubClusterInfo> entry : preSelectSubclusters.entrySet()) {
       SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
       if (weights.containsKey(id) && weights.get(id) > 0) {
         long availableMemory = getAvailableMemory(entry.getValue());
@@ -110,7 +90,7 @@ private long getAvailableMemory(SubClusterInfo value) throws YarnException {
       mem = obj.getJSONObject("clusterMetrics").getLong("availableMB");
       return mem;
     } catch (JSONException j) {
-      throw new YarnException("FederationSubCluserInfo cannot be parsed", j);
+      throw new YarnException("FederationSubClusterInfo cannot be parsed", j);
     }
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java
index 469240af518d9..3abcf6fa378e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java
@@ -21,6 +21,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Collections;
 
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -78,7 +79,7 @@ public void reinitialize(FederationPolicyInitializationContext policyContext)
     resolver = policyContext.getFederationSubclusterResolver();
     Map<SubClusterIdInfo, Float> weights =
         getPolicyInfo().getRouterPolicyWeights();
-    enabledSCs = new ArrayList<SubClusterId>();
+    enabledSCs = new ArrayList<>();
     for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
       if (entry != null && entry.getValue() > 0) {
         enabledSCs.add(entry.getKey().toId());
@@ -100,8 +101,7 @@ public SubClusterId getHomeSubcluster(
     // Fast path for FailForward to WeightedRandomRouterPolicy
     if (rrList == null || rrList.isEmpty() || (rrList.size() == 1
         && ResourceRequest.isAnyLocation(rrList.get(0).getResourceName()))) {
-      return super
-          .getHomeSubcluster(appSubmissionContext, blackListSubClusters);
+      return super.getHomeSubcluster(appSubmissionContext, blackListSubClusters);
     }
 
     if (rrList.size() != 3) {
@@ -109,12 +109,11 @@ public SubClusterId getHomeSubcluster(
           "Invalid number of resource requests: " + rrList.size());
     }
 
-    Map<SubClusterId, SubClusterInfo> activeSubClusters =
-        getActiveSubclusters();
-    List<SubClusterId> validSubClusters =
-        new ArrayList<>(activeSubClusters.keySet());
-    FederationPolicyUtils
-        .validateSubClusterAvailability(validSubClusters, blackListSubClusters);
+    Map<SubClusterId, SubClusterInfo> activeSubClusters = getActiveSubclusters();
+    Set<SubClusterId> validSubClusters = activeSubClusters.keySet();
+    FederationPolicyUtils.validateSubClusterAvailability(activeSubClusters.keySet(),
+        blackListSubClusters);
+
     if (blackListSubClusters != null) {
       // Remove from the active SubClusters from StateStore the blacklisted ones
       validSubClusters.removeAll(blackListSubClusters);
@@ -128,20 +127,21 @@ public SubClusterId getHomeSubcluster(
       ResourceRequest nodeRequest = null;
       ResourceRequest rackRequest = null;
       ResourceRequest anyRequest = null;
+
       for (ResourceRequest rr : rrList) {
         // Handle "node" requests
         try {
           targetId = resolver.getSubClusterForNode(rr.getResourceName());
           nodeRequest = rr;
         } catch (YarnException e) {
-          LOG.error("Cannot resolve node : {}", e.getLocalizedMessage());
+          LOG.error("Cannot resolve node : {}.", e.getMessage());
         }
         // Handle "rack" requests
         try {
           resolver.getSubClustersForRack(rr.getResourceName());
           rackRequest = rr;
         } catch (YarnException e) {
-          LOG.error("Cannot resolve rack : {}", e.getLocalizedMessage());
+          LOG.error("Cannot resolve rack : {}.", e.getMessage());
         }
         // Handle "ANY" requests
         if (ResourceRequest.isAnyLocation(rr.getResourceName())) {
@@ -149,32 +149,33 @@ public SubClusterId getHomeSubcluster(
           continue;
         }
       }
+
       if (nodeRequest == null) {
-        throw new YarnException("Missing node request");
+        throw new YarnException("Missing node request.");
       }
       if (rackRequest == null) {
-        throw new YarnException("Missing rack request");
+        throw new YarnException("Missing rack request.");
       }
       if (anyRequest == null) {
-        throw new YarnException("Missing any request");
+        throw new YarnException("Missing any request.");
       }
-      LOG.info(
-          "Node request: " + nodeRequest.getResourceName() + ", Rack request: "
-              + rackRequest.getResourceName() + ", Any request: " + anyRequest
-              .getResourceName());
+
+      LOG.info("Node request: {} , Rack request: {} , Any request: {}.",
+          nodeRequest.getResourceName(), rackRequest.getResourceName(),
+          anyRequest.getResourceName());
+
       // Handle "node" requests
       if (validSubClusters.contains(targetId) && enabledSCs
           .contains(targetId)) {
-        LOG.info("Node {} is in SubCluster: {}", nodeRequest.getResourceName(),
-            targetId);
+        LOG.info("Node {} is in SubCluster: {}.", nodeRequest.getResourceName(), targetId);
         return targetId;
       } else {
         throw new YarnException("The node " + nodeRequest.getResourceName()
             + " is in a blacklist SubCluster or not active. ");
       }
     } catch (YarnException e) {
-      LOG.error("Validating resource requests failed, Falling back to "
-          + "WeightedRandomRouterPolicy placement: " + e.getMessage());
+      LOG.error("Validating resource requests failed, " +
+          "Falling back to WeightedRandomRouterPolicy placement : {}.", e.getMessage());
       // FailForward to WeightedRandomRouterPolicy
       // Overwrite request to use a default ANY
       ResourceRequest amReq = Records.newRecord(ResourceRequest.class);
@@ -183,14 +184,10 @@ public SubClusterId getHomeSubcluster(
       amReq.setCapability(appSubmissionContext.getResource());
       amReq.setNumContainers(1);
       amReq.setRelaxLocality(true);
-      amReq.setNodeLabelExpression(
-          appSubmissionContext.getNodeLabelExpression());
-      amReq.setExecutionTypeRequest(
-          ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
-      appSubmissionContext
-          .setAMContainerResourceRequests(Collections.singletonList(amReq));
-      return super
-          .getHomeSubcluster(appSubmissionContext, blackListSubClusters);
+      amReq.setNodeLabelExpression(appSubmissionContext.getNodeLabelExpression());
+      amReq.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+      appSubmissionContext.setAMContainerResourceRequests(Collections.singletonList(amReq));
+      return super.getHomeSubcluster(appSubmissionContext, blackListSubClusters);
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index b81ca07b42ad8..7d50d3814a0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
@@ -17,13 +17,9 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
@@ -37,30 +33,15 @@
 public class PriorityRouterPolicy extends AbstractRouterPolicy {
 
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blacklist) throws YarnException {
-
-    // null checks and default-queue behavior
-    validate(appSubmissionContext);
-
-    Map<SubClusterId, SubClusterInfo> activeSubclusters =
-        getActiveSubclusters();
-
-    FederationPolicyUtils.validateSubClusterAvailability(
-        new ArrayList<SubClusterId>(activeSubclusters.keySet()), blacklist);
-
+  protected SubClusterId chooseSubCluster(
+      String queue, Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
     // This finds the sub-cluster with the highest weight among the
     // currently active ones.
-    Map<SubClusterIdInfo, Float> weights =
-        getPolicyInfo().getRouterPolicyWeights();
+    Map<SubClusterIdInfo, Float> weights = getPolicyInfo().getRouterPolicyWeights();
     SubClusterId chosen = null;
     Float currentBest = Float.MIN_VALUE;
-    for (SubClusterId id : activeSubclusters.keySet()) {
+    for (SubClusterId id : preSelectSubclusters.keySet()) {
       SubClusterIdInfo idInfo = new SubClusterIdInfo(id);
-      if (blacklist != null && blacklist.contains(id)) {
-        continue;
-      }
       if (weights.containsKey(idInfo) && weights.get(idInfo) > currentBest) {
         currentBest = weights.get(idInfo);
         chosen = id;
@@ -68,10 +49,8 @@ public SubClusterId getHomeSubcluster(
     }
     if (chosen == null) {
       throw new FederationPolicyException(
-          "No Active Subcluster with weight vector greater than zero");
+          "No Active Subcluster with weight vector greater than zero.");
     }
-
     return chosen;
   }
-
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
index b4c019270249c..32e31ebfec712 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java
@@ -17,15 +17,15 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
-import java.util.List;
+import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 
 /**
  * This {@link FederationRouterPolicy} simply rejects all incoming requests.
@@ -43,34 +43,12 @@ public void reinitialize(
     setPolicyContext(federationPolicyContext);
   }
 
-  /**
-   * The policy always reject requests.
-   *
-   * @param appSubmissionContext the {@link ApplicationSubmissionContext} that
-   *          has to be routed to an appropriate subCluster for execution.
-   *
-   * @param blackListSubClusters the list of subClusters as identified by
-   *          {@link SubClusterId} to blackList from the selection of the home
-   *          subCluster.
-   *
-   * @return (never).
-   *
-   * @throws YarnException (always) to prevent applications in this queue to be
-   *           run anywhere in the federated cluster.
-   */
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blackListSubClusters) throws YarnException {
-
-    // run standard validation, as error might differ
-    validate(appSubmissionContext);
-
-    throw new FederationPolicyException("The policy configured for this queue"
-        + " (" + appSubmissionContext.getQueue() + ") reject all routing "
-        + "requests by construction. Application "
-        + appSubmissionContext.getApplicationId()
-        + " cannot be routed to any RM.");
+  protected SubClusterId chooseSubCluster(
+      String queue, Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
+    throw new FederationPolicyException(
+        "The policy configured for this queue (" + queue + ") "
+        + "reject all routing requests by construction. Application in "
+        + queue + " cannot be routed to any RM.");
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
index 7a8be91fcd0f6..353329613ab97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java
@@ -22,11 +22,10 @@
 import java.util.Map;
 import java.util.Random;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -55,50 +54,16 @@ public void reinitialize(FederationPolicyInitializationContext policyContext)
         this.getClass().getCanonicalName());
 
     // note: this overrides AbstractRouterPolicy and ignores the weights
-
     setPolicyContext(policyContext);
   }
 
-  /**
-   * Simply picks a random active subCluster to start the AM (this does NOT
-   * depend on the weights in the policy).
-   *
-   * @param appSubmissionContext the {@link ApplicationSubmissionContext} that
-   *          has to be routed to an appropriate subCluster for execution.
-   *
-   * @param blackListSubClusters the list of subClusters as identified by
-   *          {@link SubClusterId} to blackList from the selection of the home
-   *          subCluster.
-   *
-   * @return a randomly chosen subcluster.
-   *
-   * @throws YarnException if there are no active subclusters.
-   */
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blackListSubClusters) throws YarnException {
-
-    // null checks and default-queue behavior
-    validate(appSubmissionContext);
-
-    Map<SubClusterId, SubClusterInfo> activeSubclusters =
-        getActiveSubclusters();
-
-    List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
-
-    FederationPolicyUtils.validateSubClusterAvailability(list,
-        blackListSubClusters);
-
-    if (blackListSubClusters != null) {
-
-      // Remove from the active SubClusters from StateStore the blacklisted ones
-      for (SubClusterId scId : blackListSubClusters) {
-        list.remove(scId);
-      }
+  protected SubClusterId chooseSubCluster(
+      String queue, Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
+    if (preSelectSubclusters == null || preSelectSubclusters.isEmpty()) {
+      throw new FederationPolicyException("No available subcluster to choose from.");
     }
-
+    List<SubClusterId> list = new ArrayList<>(preSelectSubclusters.keySet());
     return list.get(rand.nextInt(list.size()));
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
index b1434104836c0..f2acf663603f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java
@@ -19,10 +19,8 @@
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
@@ -35,47 +33,30 @@
  * sub-clusters.
  */
 public class WeightedRandomRouterPolicy extends AbstractRouterPolicy {
-
   @Override
-  public SubClusterId getHomeSubcluster(
-      ApplicationSubmissionContext appSubmissionContext,
-      List<SubClusterId> blacklist) throws YarnException {
-
-    // null checks and default-queue behavior
-    validate(appSubmissionContext);
-
-    Map<SubClusterId, SubClusterInfo> activeSubclusters =
-        getActiveSubclusters();
+  protected SubClusterId chooseSubCluster(
+      String queue, Map<SubClusterId, SubClusterInfo> preSelectSubclusters) throws YarnException {
 
-    FederationPolicyUtils.validateSubClusterAvailability(
-        new ArrayList<SubClusterId>(activeSubclusters.keySet()), blacklist);
-
-    // note: we cannot pre-compute the weights, as the set of activeSubcluster
+    // note: we cannot pre-compute the weights, as the set of activeSubCluster
     // changes dynamically (and this would unfairly spread the load to
     // sub-clusters adjacent to an inactive one), hence we need to count/scan
     // the list and based on weight pick the next sub-cluster.
-    Map<SubClusterIdInfo, Float> weights =
-        getPolicyInfo().getRouterPolicyWeights();
+    Map<SubClusterIdInfo, Float> weights = getPolicyInfo().getRouterPolicyWeights();
 
     ArrayList<Float> weightList = new ArrayList<>();
     ArrayList<SubClusterId> scIdList = new ArrayList<>();
     for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
-      if (blacklist != null && blacklist.contains(entry.getKey().toId())) {
-        continue;
-      }
-      if (entry.getKey() != null
-          && activeSubclusters.containsKey(entry.getKey().toId())) {
+      SubClusterIdInfo key = entry.getKey();
+      if (key != null && preSelectSubclusters.containsKey(key.toId())) {
         weightList.add(entry.getValue());
-        scIdList.add(entry.getKey().toId());
+        scIdList.add(key.toId());
       }
     }
 
     int pickedIndex = FederationPolicyUtils.getWeightedRandom(weightList);
     if (pickedIndex == -1) {
-      throw new FederationPolicyException(
-          "No positive weight found on active subclusters");
+      throw new FederationPolicyException("No positive weight found on active subclusters");
     }
     return scIdList.get(pickedIndex);
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/FederationActionRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/FederationActionRetry.java
new file mode 100644
index 0000000000000..3068526c1eab1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/FederationActionRetry.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.retry;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public interface FederationActionRetry<T> {
+
+  Logger LOG = LoggerFactory.getLogger(FederationActionRetry.class);
+
+  T run(int retry) throws Exception;
+
+  default T runWithRetries(int retryCount, long retrySleepTime) throws Exception {
+    int retry = 0;
+    while (true) {
+      try {
+        return run(retry);
+      } catch (Exception e) {
+        LOG.info("Exception while executing an Federation operation.", e);
+        if (++retry > retryCount) {
+          LOG.info("Maxed out Federation retries. Giving up!");
+          throw e;
+        }
+        LOG.info("Retrying operation on Federation. Retry no. {}", retry);
+        Thread.sleep(retrySleepTime);
+      }
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/package-info.java
new file mode 100644
index 0000000000000..5d8477cfe599d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/retry/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Federation Retry Policies. **/
+package org.apache.hadoop.yarn.server.federation.retry;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java
new file mode 100644
index 0000000000000..384295fd09dcc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+
+import java.io.IOException;
+
+/**
+ * FederationDelegationTokenStateStore maintains the state of all
+ * <em>DelegationToken</em> that have been submitted to the federated cluster.
+ */
+@Private
+@Unstable
+public interface FederationDelegationTokenStateStore {
+
+  /**
+   * The Router Supports Store NewMasterKey.
+   * During this Process, Facade will call the specific StateStore to store the MasterKey.
+   *
+   * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+   * @return routerMasterKeyResponse
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports Remove MasterKey.
+   * During this Process, Facade will call the specific StateStore to remove the MasterKey.
+   *
+   * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+   * @return routerMasterKeyResponse
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports GetMasterKeyByDelegationKey.
+   *
+   * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+   * @return routerMasterKeyResponse
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports Store RMDelegationTokenIdentifier.
+   *
+   * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+   * @return routerRMTokenResponse.
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports Update RMDelegationTokenIdentifier.
+   *
+   * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+   * @return RouterRMTokenResponse.
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports Remove RMDelegationTokenIdentifier.
+   *
+   * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+   * @return RouterRMTokenResponse.
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException;
+
+  /**
+   * The Router Supports GetTokenByRouterStoreToken.
+   *
+   * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+   * @return RouterRMTokenResponse.
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
+      throws YarnException, IOException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java
new file mode 100644
index 0000000000000..8c2d5ffe5868f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+
+/**
+ * FederationReservationHomeSubClusterStore maintains the state of all
+ * <em>Reservations</em> that have been submitted to the federated cluster.
+ *
+ * *
+ * <p>
+ * The mapping details contains:
+ * <ul>
+ * <li>{@code ReservationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public interface FederationReservationHomeSubClusterStore {
+
+  /**
+   * Register the home {@code SubClusterId} of the newly submitted
+   * {@code ReservationId}. Currently response is empty if the operation was
+   * successful, if not an exception reporting reason for a failure. If a
+   * mapping for the Reservation already existed, the {@code SubClusterId} in
+   * this response will return the existing mapping which might be different
+   * from that in the {@code AddReservationHomeSubClusterRequest}.
+   *
+   * @param request the request to register a new Reservation with its home
+   *          sub-cluster
+   * @return upon successful registration of the Reservation in the StateStore,
+   *         {@code AddReservationHomeSubClusterRequest} containing the home
+   *         sub-cluster of the Reservation. Otherwise, an exception reporting
+   *         reason for a failure
+   * @throws YarnException if the request is invalid/fails
+   */
+  AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
+      AddReservationHomeSubClusterRequest request) throws YarnException;
+
+  /**
+   * Get information about the Reservation identified by the input
+   * {@code ReservationId}.
+   *
+   * @param request contains the Reservation queried
+   * @return {@code ReservationHomeSubCluster} containing the Reservation's home
+   *         subcluster
+   * @throws YarnException if the request is invalid/fails
+   */
+  GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
+      GetReservationHomeSubClusterRequest request) throws YarnException;
+
+  /**
+   * Get the {@code ReservationHomeSubCluster} list representing the mapping of
+   * all submitted Reservations to it's home sub-cluster.
+   *
+   * @param request empty representing all Reservations
+   * @return the mapping of all submitted Reservation to it's home sub-cluster
+   * @throws YarnException if the request is invalid/fails
+   */
+  GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
+      GetReservationsHomeSubClusterRequest request) throws YarnException;
+
+  /**
+   * Update the home {@code SubClusterId} of a previously submitted
+   * {@code ReservationId}. Currently response is empty if the operation was
+   * successful, if not an exception reporting reason for a failure.
+   *
+   * @param request the request to update the home sub-cluster of a reservation.
+   * @return empty on successful update of the Reservation in the StateStore, if
+   *         not an exception reporting reason for a failure
+   * @throws YarnException if the request is invalid/fails
+   */
+  UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
+      UpdateReservationHomeSubClusterRequest request) throws YarnException;
+
+
+  /**
+   * Delete the mapping of home {@code SubClusterId} of a previously submitted
+   * {@code ReservationId}. Currently response is empty if the operation was
+   * successful, if not an exception reporting reason for a failure.
+   *
+   * @param request the request to delete the home sub-cluster of a reservation.
+   * @return empty on successful update of the Reservation in the StateStore, if
+   *         not an exception reporting reason for a failure
+   * @throws YarnException if the request is invalid/fails
+   */
+  DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
+      DeleteReservationHomeSubClusterRequest request) throws YarnException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
index 9397e9c240473..3ca8ccc2bfb69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java
@@ -25,12 +25,14 @@
 /**
  * FederationStore extends the three interfaces used to coordinate the state of
  * a federated cluster: {@link FederationApplicationHomeSubClusterStore},
- * {@link FederationMembershipStateStore}, and {@link FederationPolicyStore}.
+ * {@link FederationMembershipStateStore}, {@link FederationPolicyStore}, and
+ * {@link FederationReservationHomeSubClusterStore}.
  *
  */
-public interface FederationStateStore
-    extends FederationApplicationHomeSubClusterStore,
-    FederationMembershipStateStore, FederationPolicyStore {
+public interface FederationStateStore extends
+    FederationApplicationHomeSubClusterStore, FederationMembershipStateStore,
+    FederationPolicyStore, FederationReservationHomeSubClusterStore,
+    FederationDelegationTokenStateStore {
 
   /**
    * Initialize the FederationStore.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 7c06256a41364..2d210ab37496c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -17,16 +17,26 @@
 
 package org.apache.hadoop.yarn.server.federation.store.impl;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.TimeZone;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.Comparator;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
@@ -59,7 +69,26 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
@@ -68,6 +97,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils.filterHomeSubCluster;
+
 /**
  * In-memory implementation of {@link FederationStateStore}.
  */
@@ -75,7 +106,10 @@ public class MemoryFederationStateStore implements FederationStateStore {
 
   private Map<SubClusterId, SubClusterInfo> membership;
   private Map<ApplicationId, SubClusterId> applications;
+  private Map<ReservationId, SubClusterId> reservations;
   private Map<String, SubClusterPolicyConfiguration> policies;
+  private RouterRMDTSecretManagerState routerRMSecretManagerState;
+  private int maxAppsInStateStore;
 
   private final MonotonicClock clock = new MonotonicClock();
 
@@ -86,13 +120,19 @@ public class MemoryFederationStateStore implements FederationStateStore {
   public void init(Configuration conf) {
     membership = new ConcurrentHashMap<SubClusterId, SubClusterInfo>();
     applications = new ConcurrentHashMap<ApplicationId, SubClusterId>();
+    reservations = new ConcurrentHashMap<ReservationId, SubClusterId>();
     policies = new ConcurrentHashMap<String, SubClusterPolicyConfiguration>();
+    routerRMSecretManagerState = new RouterRMDTSecretManagerState();
+    maxAppsInStateStore = conf.getInt(
+        YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_MAX_APPLICATIONS);
   }
 
   @Override
   public void close() {
     membership = null;
     applications = null;
+    reservations = null;
     policies = null;
   }
 
@@ -233,24 +273,34 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
 
-    return GetApplicationHomeSubClusterResponse.newInstance(
-        ApplicationHomeSubCluster.newInstance(appId, applications.get(appId)));
+    return GetApplicationHomeSubClusterResponse.newInstance(appId, applications.get(appId));
   }
 
   @Override
   public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       GetApplicationsHomeSubClusterRequest request) throws YarnException {
-    List<ApplicationHomeSubCluster> result =
-        new ArrayList<ApplicationHomeSubCluster>();
-    for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
-      result
-          .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+
+    if (request == null) {
+      throw new YarnException("Missing getApplicationsHomeSubCluster request");
     }
 
-    GetApplicationsHomeSubClusterResponse.newInstance(result);
+    SubClusterId requestSC = request.getSubClusterId();
+    List<ApplicationHomeSubCluster> result = applications.keySet().stream()
+        .map(applicationId -> generateAppHomeSC(applicationId))
+        .sorted(Comparator.comparing(ApplicationHomeSubCluster::getCreateTime).reversed())
+        .filter(appHomeSC -> filterHomeSubCluster(requestSC, appHomeSC.getHomeSubCluster()))
+        .limit(maxAppsInStateStore)
+        .collect(Collectors.toList());
+
+    LOG.info("filterSubClusterId = {}, appCount = {}.", requestSC, result.size());
     return GetApplicationsHomeSubClusterResponse.newInstance(result);
   }
 
+  private ApplicationHomeSubCluster generateAppHomeSC(ApplicationId applicationId) {
+    SubClusterId subClusterId = applications.get(applicationId);
+    return ApplicationHomeSubCluster.newInstance(applicationId, subClusterId);
+  }
+
   @Override
   public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
       DeleteApplicationHomeSubClusterRequest request) throws YarnException {
@@ -312,4 +362,217 @@ public Version loadVersion() {
     return null;
   }
 
+  @Override
+  public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
+      AddReservationHomeSubClusterRequest request) throws YarnException {
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationHomeSubCluster homeSubCluster = request.getReservationHomeSubCluster();
+    ReservationId reservationId = homeSubCluster.getReservationId();
+    if (!reservations.containsKey(reservationId)) {
+      reservations.put(reservationId, homeSubCluster.getHomeSubCluster());
+    }
+    return AddReservationHomeSubClusterResponse.newInstance(reservations.get(reservationId));
+  }
+
+  @Override
+  public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
+      GetReservationHomeSubClusterRequest request) throws YarnException {
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationId reservationId = request.getReservationId();
+    if (!reservations.containsKey(reservationId)) {
+      throw new YarnException("Reservation " + reservationId + " does not exist");
+    }
+    SubClusterId subClusterId = reservations.get(reservationId);
+    ReservationHomeSubCluster homeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+    return GetReservationHomeSubClusterResponse.newInstance(homeSubCluster);
+  }
+
+  @Override
+  public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
+      GetReservationsHomeSubClusterRequest request) throws YarnException {
+    List<ReservationHomeSubCluster> result = new ArrayList<>();
+
+    for (Entry<ReservationId, SubClusterId> entry : reservations.entrySet()) {
+      ReservationId reservationId = entry.getKey();
+      SubClusterId subClusterId = entry.getValue();
+      ReservationHomeSubCluster homeSubCluster =
+          ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+      result.add(homeSubCluster);
+    }
+
+    return GetReservationsHomeSubClusterResponse.newInstance(result);
+  }
+
+  @Override
+  public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
+      UpdateReservationHomeSubClusterRequest request) throws YarnException {
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationId reservationId = request.getReservationHomeSubCluster().getReservationId();
+
+    if (!reservations.containsKey(reservationId)) {
+      throw new YarnException("Reservation " + reservationId + " does not exist.");
+    }
+
+    SubClusterId subClusterId = request.getReservationHomeSubCluster().getHomeSubCluster();
+    reservations.put(reservationId, subClusterId);
+    return UpdateReservationHomeSubClusterResponse.newInstance();
+  }
+
+  @Override
+  public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
+      DeleteReservationHomeSubClusterRequest request) throws YarnException {
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationId reservationId = request.getReservationId();
+    if (!reservations.containsKey(reservationId)) {
+      throw new YarnException("Reservation " + reservationId + " does not exist");
+    }
+    reservations.remove(reservationId);
+    return DeleteReservationHomeSubClusterResponse.newInstance();
+  }
+
+  @Override
+  public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    // Restore the DelegationKey from the request
+    RouterMasterKey masterKey = request.getRouterMasterKey();
+    DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey);
+
+    Set<DelegationKey> rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState();
+    if (rmDTMasterKeyState.contains(delegationKey)) {
+      LOG.info("Error storing info for RMDTMasterKey with keyID: {}.", delegationKey.getKeyId());
+      throw new IOException("RMDTMasterKey with keyID: " + delegationKey.getKeyId() +
+          " is already stored");
+    }
+
+    routerRMSecretManagerState.getMasterKeyState().add(delegationKey);
+    LOG.info("Store Router-RMDT master key with key id: {}. Currently rmDTMasterKeyState size: {}",
+        delegationKey.getKeyId(), rmDTMasterKeyState.size());
+
+    return RouterMasterKeyResponse.newInstance(masterKey);
+  }
+
+  @Override
+  public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    // Restore the DelegationKey from the request
+    RouterMasterKey masterKey = request.getRouterMasterKey();
+    DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey);
+
+    LOG.info("Remove Router-RMDT master key with key id: {}.", delegationKey.getKeyId());
+    Set<DelegationKey> rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState();
+    rmDTMasterKeyState.remove(delegationKey);
+
+    return RouterMasterKeyResponse.newInstance(masterKey);
+  }
+
+  @Override
+  public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    // Restore the DelegationKey from the request
+    RouterMasterKey masterKey = request.getRouterMasterKey();
+    DelegationKey delegationKey = getDelegationKeyByMasterKey(masterKey);
+
+    Set<DelegationKey> rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState();
+    if (!rmDTMasterKeyState.contains(delegationKey)) {
+      throw new IOException("GetMasterKey with keyID: " + masterKey.getKeyId() +
+          " does not exist.");
+    }
+    RouterMasterKey resultRouterMasterKey = RouterMasterKey.newInstance(delegationKey.getKeyId(),
+        ByteBuffer.wrap(delegationKey.getEncodedKey()), delegationKey.getExpiryDate());
+    return RouterMasterKeyResponse.newInstance(resultRouterMasterKey);
+  }
+
+  @Override
+  public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    RouterStoreToken storeToken = request.getRouterStoreToken();
+    RMDelegationTokenIdentifier tokenIdentifier =
+        (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier();
+    Long renewDate = storeToken.getRenewDate();
+    storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, false);
+    return RouterRMTokenResponse.newInstance(storeToken);
+  }
+
+  @Override
+  public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    RouterStoreToken storeToken = request.getRouterStoreToken();
+    RMDelegationTokenIdentifier tokenIdentifier =
+        (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier();
+    Long renewDate = storeToken.getRenewDate();
+    Map<RMDelegationTokenIdentifier, Long> rmDTState = routerRMSecretManagerState.getTokenState();
+    rmDTState.remove(tokenIdentifier);
+    storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, true);
+    return RouterRMTokenResponse.newInstance(storeToken);
+  }
+
+  @Override
+  public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    RouterStoreToken storeToken = request.getRouterStoreToken();
+    RMDelegationTokenIdentifier tokenIdentifier =
+        (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier();
+    Map<RMDelegationTokenIdentifier, Long> rmDTState = routerRMSecretManagerState.getTokenState();
+    rmDTState.remove(tokenIdentifier);
+    return RouterRMTokenResponse.newInstance(storeToken);
+  }
+
+  @Override
+  public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    RouterStoreToken storeToken = request.getRouterStoreToken();
+    RMDelegationTokenIdentifier tokenIdentifier =
+        (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier();
+    Map<RMDelegationTokenIdentifier, Long> rmDTState = routerRMSecretManagerState.getTokenState();
+    if (!rmDTState.containsKey(tokenIdentifier)) {
+      LOG.info("Router RMDelegationToken: {} does not exist.", tokenIdentifier);
+      throw new IOException("Router RMDelegationToken: " + tokenIdentifier + " does not exist.");
+    }
+    RouterStoreToken resultToken =
+        RouterStoreToken.newInstance(tokenIdentifier, rmDTState.get(tokenIdentifier));
+    return RouterRMTokenResponse.newInstance(resultToken);
+  }
+
+  private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier,
+      Long renewDate, boolean isUpdate) throws IOException {
+    Map<RMDelegationTokenIdentifier, Long> rmDTState = routerRMSecretManagerState.getTokenState();
+    if (rmDTState.containsKey(rmDTIdentifier)) {
+      LOG.info("Error storing info for RMDelegationToken: {}.", rmDTIdentifier);
+      throw new IOException("Router RMDelegationToken: " + rmDTIdentifier + "is already stored.");
+    }
+    rmDTState.put(rmDTIdentifier, renewDate);
+    if (!isUpdate) {
+      routerRMSecretManagerState.setDtSequenceNumber(rmDTIdentifier.getSequenceNumber());
+    }
+    LOG.info("Store Router RM-RMDT with sequence number {}.", rmDTIdentifier.getSequenceNumber());
+  }
+
+  /**
+   * Get DelegationKey By based on MasterKey.
+   *
+   * @param masterKey masterKey
+   * @return DelegationKey
+   */
+  private static DelegationKey getDelegationKeyByMasterKey(RouterMasterKey masterKey) {
+    ByteBuffer keyByteBuf = masterKey.getKeyBytes();
+    byte[] keyBytes = new byte[keyByteBuf.remaining()];
+    keyByteBuf.get(keyBytes);
+    return new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes);
+  }
+
+  @VisibleForTesting
+  public RouterRMDTSecretManagerState getRouterRMSecretManagerState() {
+    return routerRMSecretManagerState;
+  }
+
+  @VisibleForTesting
+  public Map<SubClusterId, SubClusterInfo> getMembership() {
+    return membership;
+  }
+
+  @VisibleForTesting
+  public void setMembership(Map<SubClusterId, SubClusterInfo> membership) {
+    this.membership = membership;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
index 2b3fea5609b2e..2bf2658944e5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.federation.store.impl;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.sql.CallableStatement;
 import java.sql.Connection;
@@ -30,8 +31,10 @@
 import java.util.TimeZone;
 
 import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
@@ -68,10 +71,26 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -119,7 +138,7 @@ public class SQLFederationStateStore implements FederationStateStore {
       "{call sp_getApplicationHomeSubCluster(?, ?)}";
 
   private static final String CALL_SP_GET_APPLICATIONS_HOME_SUBCLUSTER =
-      "{call sp_getApplicationsHomeSubCluster()}";
+      "{call sp_getApplicationsHomeSubCluster(?, ?)}";
 
   private static final String CALL_SP_SET_POLICY_CONFIGURATION =
       "{call sp_setPolicyConfiguration(?, ?, ?, ?)}";
@@ -130,6 +149,21 @@ public class SQLFederationStateStore implements FederationStateStore {
   private static final String CALL_SP_GET_POLICIES_CONFIGURATIONS =
       "{call sp_getPoliciesConfigurations()}";
 
+  protected static final String CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER =
+      "{call sp_addReservationHomeSubCluster(?, ?, ?, ?)}";
+
+  protected static final String CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER =
+      "{call sp_getReservationHomeSubCluster(?, ?)}";
+
+  protected static final String CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER =
+      "{call sp_getReservationsHomeSubCluster()}";
+
+  protected static final String CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER =
+      "{call sp_deleteReservationHomeSubCluster(?, ?)}";
+
+  protected static final String CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER =
+      "{call sp_updateReservationHomeSubCluster(?, ?, ?)}";
+
   private Calendar utcCalendar =
       Calendar.getInstance(TimeZone.getTimeZone("UTC"));
 
@@ -143,7 +177,8 @@ public class SQLFederationStateStore implements FederationStateStore {
   private HikariDataSource dataSource = null;
   private final Clock clock = new MonotonicClock();
   @VisibleForTesting
-  Connection conn = null;
+  private Connection conn = null;
+  private int maxAppsInStateStore;
 
   @Override
   public void init(Configuration conf) throws YarnException {
@@ -162,8 +197,7 @@ public void init(Configuration conf) throws YarnException {
     try {
       Class.forName(driverClass);
     } catch (ClassNotFoundException e) {
-      FederationStateStoreUtils.logAndThrowException(LOG,
-          "Driver class not found.", e);
+      FederationStateStoreUtils.logAndThrowException(LOG, "Driver class not found.", e);
     }
 
     // Create the data source to pool connections in a thread-safe manner
@@ -174,45 +208,46 @@ public void init(Configuration conf) throws YarnException {
     FederationStateStoreUtils.setProperty(dataSource,
         FederationStateStoreUtils.FEDERATION_STORE_URL, url);
     dataSource.setMaximumPoolSize(maximumPoolSize);
-    LOG.info("Initialized connection pool to the Federation StateStore "
-        + "database at address: " + url);
+    LOG.info("Initialized connection pool to the Federation StateStore database at address: {}.",
+        url);
+
     try {
       conn = getConnection();
       LOG.debug("Connection created");
     } catch (SQLException e) {
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Not able to get Connection", e);
+      FederationStateStoreUtils.logAndThrowRetriableException(LOG, "Not able to get Connection", e);
     }
+
+    maxAppsInStateStore = conf.getInt(
+        YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_MAX_APPLICATIONS);
   }
 
   @Override
   public SubClusterRegisterResponse registerSubCluster(
-      SubClusterRegisterRequest registerSubClusterRequest)
-      throws YarnException {
+      SubClusterRegisterRequest registerSubClusterRequest) throws YarnException {
 
     // Input validator
-    FederationMembershipStateStoreInputValidator
-        .validate(registerSubClusterRequest);
+    FederationMembershipStateStoreInputValidator.validate(registerSubClusterRequest);
 
     CallableStatement cstmt = null;
 
-    SubClusterInfo subClusterInfo =
-        registerSubClusterRequest.getSubClusterInfo();
+    SubClusterInfo subClusterInfo = registerSubClusterRequest.getSubClusterInfo();
     SubClusterId subClusterId = subClusterInfo.getSubClusterId();
 
     try {
       cstmt = getCallableStatement(CALL_SP_REGISTER_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, subClusterId.getId());
-      cstmt.setString(2, subClusterInfo.getAMRMServiceAddress());
-      cstmt.setString(3, subClusterInfo.getClientRMServiceAddress());
-      cstmt.setString(4, subClusterInfo.getRMAdminServiceAddress());
-      cstmt.setString(5, subClusterInfo.getRMWebServiceAddress());
-      cstmt.setString(6, subClusterInfo.getState().toString());
-      cstmt.setLong(7, subClusterInfo.getLastStartTime());
-      cstmt.setString(8, subClusterInfo.getCapability());
-      cstmt.registerOutParameter(9, java.sql.Types.INTEGER);
+      cstmt.setString("subClusterId_IN", subClusterId.getId());
+      cstmt.setString("amRMServiceAddress_IN", subClusterInfo.getAMRMServiceAddress());
+      cstmt.setString("clientRMServiceAddress_IN", subClusterInfo.getClientRMServiceAddress());
+      cstmt.setString("rmAdminServiceAddress_IN", subClusterInfo.getRMAdminServiceAddress());
+      cstmt.setString("rmWebServiceAddress_IN", subClusterInfo.getRMWebServiceAddress());
+      cstmt.setString("state_IN", subClusterInfo.getState().toString());
+      cstmt.setLong("lastStartTime_IN", subClusterInfo.getLastStartTime());
+      cstmt.setString("capability_IN", subClusterInfo.getCapability());
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -221,30 +256,26 @@ public SubClusterRegisterResponse registerSubCluster(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not add a new subcluster into FederationStateStore
-      if (cstmt.getInt(9) == 0) {
-        String errMsg = "SubCluster " + subClusterId
-            + " was not registered into the StateStore";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "SubCluster %s was not registered into the StateStore.", subClusterId);
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(9) != 1) {
-        String errMsg = "Wrong behavior during registration of SubCluster "
-            + subClusterId + " into the StateStore";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (rowCount != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during registration of SubCluster %s into the StateStore",
+            subClusterId);
       }
 
-      LOG.info(
-          "Registered the SubCluster " + subClusterId + " into the StateStore");
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      LOG.info("Registered the SubCluster {} into the StateStore.", subClusterId);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to register the SubCluster " + subClusterId
-              + " into the StateStore",
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e,
+          LOG, "Unable to register the SubCluster %s into the StateStore.", subClusterId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -255,12 +286,10 @@ public SubClusterRegisterResponse registerSubCluster(
 
   @Override
   public SubClusterDeregisterResponse deregisterSubCluster(
-      SubClusterDeregisterRequest subClusterDeregisterRequest)
-      throws YarnException {
+      SubClusterDeregisterRequest subClusterDeregisterRequest) throws YarnException {
 
     // Input validator
-    FederationMembershipStateStoreInputValidator
-        .validate(subClusterDeregisterRequest);
+    FederationMembershipStateStoreInputValidator.validate(subClusterDeregisterRequest);
 
     CallableStatement cstmt = null;
 
@@ -271,9 +300,9 @@ public SubClusterDeregisterResponse deregisterSubCluster(
       cstmt = getCallableStatement(CALL_SP_DEREGISTER_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, subClusterId.getId());
-      cstmt.setString(2, state.toString());
-      cstmt.registerOutParameter(3, java.sql.Types.INTEGER);
+      cstmt.setString("subClusterId_IN", subClusterId.getId());
+      cstmt.setString("state_IN", state.toString());
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -282,29 +311,25 @@ public SubClusterDeregisterResponse deregisterSubCluster(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not deregister the subcluster into FederationStateStore
-      if (cstmt.getInt(3) == 0) {
-        String errMsg = "SubCluster " + subClusterId + " not found";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "SubCluster %s not found.", subClusterId);
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(3) != 1) {
-        String errMsg = "Wrong behavior during deregistration of SubCluster "
-            + subClusterId + " from the StateStore";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (rowCount != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during deregistration of SubCluster %s from the StateStore.",
+            subClusterId);
       }
-
-      LOG.info("Deregistered the SubCluster " + subClusterId + " state to "
-          + state.toString());
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      LOG.info("Deregistered the SubCluster {} state to {}.", subClusterId, state.toString());
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to deregister the sub-cluster " + subClusterId + " state to "
-              + state.toString(),
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to deregister the sub-cluster %s state to %s.", subClusterId, state.toString());
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -314,12 +339,10 @@ public SubClusterDeregisterResponse deregisterSubCluster(
 
   @Override
   public SubClusterHeartbeatResponse subClusterHeartbeat(
-      SubClusterHeartbeatRequest subClusterHeartbeatRequest)
-      throws YarnException {
+      SubClusterHeartbeatRequest subClusterHeartbeatRequest) throws YarnException {
 
     // Input validator
-    FederationMembershipStateStoreInputValidator
-        .validate(subClusterHeartbeatRequest);
+    FederationMembershipStateStoreInputValidator.validate(subClusterHeartbeatRequest);
 
     CallableStatement cstmt = null;
 
@@ -330,10 +353,10 @@ public SubClusterHeartbeatResponse subClusterHeartbeat(
       cstmt = getCallableStatement(CALL_SP_SUBCLUSTER_HEARTBEAT);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, subClusterId.getId());
-      cstmt.setString(2, state.toString());
-      cstmt.setString(3, subClusterHeartbeatRequest.getCapability());
-      cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
+      cstmt.setString("subClusterId_IN", subClusterId.getId());
+      cstmt.setString("state_IN", state.toString());
+      cstmt.setString("capability_IN", subClusterHeartbeatRequest.getCapability());
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -342,30 +365,25 @@ public SubClusterHeartbeatResponse subClusterHeartbeat(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not update the subcluster into FederationStateStore
-      if (cstmt.getInt(4) == 0) {
-        String errMsg = "SubCluster " + subClusterId.toString()
-            + " does not exist; cannot heartbeat";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "SubCluster %s does not exist; cannot heartbeat.", subClusterId);
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(4) != 1) {
-        String errMsg =
-            "Wrong behavior during the heartbeat of SubCluster " + subClusterId;
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (rowCount != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during the heartbeat of SubCluster %s.", subClusterId);
       }
 
-      LOG.info("Heartbeated the StateStore for the specified SubCluster "
-          + subClusterId);
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      LOG.info("Heartbeated the StateStore for the specified SubCluster {}.", subClusterId);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to heartbeat the StateStore for the specified SubCluster "
-              + subClusterId,
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to heartbeat the StateStore for the specified SubCluster %s.", subClusterId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -387,27 +405,27 @@ public GetSubClusterInfoResponse getSubCluster(
 
     try {
       cstmt = getCallableStatement(CALL_SP_GET_SUBCLUSTER);
-      cstmt.setString(1, subClusterId.getId());
+      cstmt.setString("subClusterId_IN", subClusterId.getId());
 
       // Set the parameters for the stored procedure
-      cstmt.registerOutParameter(2, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(3, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(4, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(5, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(6, java.sql.Types.TIMESTAMP);
-      cstmt.registerOutParameter(7, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(8, java.sql.Types.BIGINT);
-      cstmt.registerOutParameter(9, java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("amRMServiceAddress_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("clientRMServiceAddress_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("rmAdminServiceAddress_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("rmWebServiceAddress_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("lastHeartBeat_OUT", java.sql.Types.TIMESTAMP);
+      cstmt.registerOutParameter("state_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("lastStartTime_OUT", java.sql.Types.BIGINT);
+      cstmt.registerOutParameter("capability_OUT", java.sql.Types.VARCHAR);
 
       // Execute the query
       long startTime = clock.getTime();
       cstmt.execute();
       long stopTime = clock.getTime();
 
-      String amRMAddress = cstmt.getString(2);
-      String clientRMAddress = cstmt.getString(3);
-      String rmAdminAddress = cstmt.getString(4);
-      String webAppAddress = cstmt.getString(5);
+      String amRMAddress = cstmt.getString("amRMServiceAddress_OUT");
+      String clientRMAddress = cstmt.getString("clientRMServiceAddress_OUT");
+      String rmAdminAddress = cstmt.getString("rmAdminServiceAddress_OUT");
+      String webAppAddress = cstmt.getString("rmWebServiceAddress_OUT");
 
       // first check if the subCluster exists
       if((amRMAddress == null) || (clientRMAddress == null)) {
@@ -415,36 +433,31 @@ public GetSubClusterInfoResponse getSubCluster(
         return null;
       }
 
-      Timestamp heartBeatTimeStamp = cstmt.getTimestamp(6, utcCalendar);
-      long lastHeartBeat =
-          heartBeatTimeStamp != null ? heartBeatTimeStamp.getTime() : 0;
+      Timestamp heartBeatTimeStamp = cstmt.getTimestamp("lastHeartBeat_OUT", utcCalendar);
+      long lastHeartBeat = heartBeatTimeStamp != null ? heartBeatTimeStamp.getTime() : 0;
 
-      SubClusterState state = SubClusterState.fromString(cstmt.getString(7));
-      long lastStartTime = cstmt.getLong(8);
-      String capability = cstmt.getString(9);
+      SubClusterState state = SubClusterState.fromString(cstmt.getString("state_OUT"));
+      long lastStartTime = cstmt.getLong("lastStartTime_OUT");
+      String capability = cstmt.getString("capability_OUT");
 
       subClusterInfo = SubClusterInfo.newInstance(subClusterId, amRMAddress,
           clientRMAddress, rmAdminAddress, webAppAddress, lastHeartBeat, state,
           lastStartTime, capability);
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
       // Check if the output it is a valid subcluster
       try {
-        FederationMembershipStateStoreInputValidator
-            .checkSubClusterInfo(subClusterInfo);
+        FederationMembershipStateStoreInputValidator.checkSubClusterInfo(subClusterInfo);
       } catch (FederationStateStoreInvalidInputException e) {
-        String errMsg =
-            "SubCluster " + subClusterId.toString() + " does not exist";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+        FederationStateStoreUtils.logAndThrowStoreException(e, LOG,
+            "SubCluster %s does not exist.", subClusterId);
       }
-      LOG.debug("Got the information about the specified SubCluster {}",
-          subClusterInfo);
+      LOG.debug("Got the information about the specified SubCluster {}", subClusterInfo);
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to obtain the SubCluster information for " + subClusterId, e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to obtain the SubCluster information for %s.", subClusterId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -457,7 +470,7 @@ public GetSubClustersInfoResponse getSubClusters(
       GetSubClustersInfoRequest subClustersRequest) throws YarnException {
     CallableStatement cstmt = null;
     ResultSet rs = null;
-    List<SubClusterInfo> subClusters = new ArrayList<SubClusterInfo>();
+    List<SubClusterInfo> subClusters = new ArrayList<>();
 
     try {
       cstmt = getCallableStatement(CALL_SP_GET_SUBCLUSTERS);
@@ -470,15 +483,15 @@ public GetSubClustersInfoResponse getSubClusters(
       while (rs.next()) {
 
         // Extract the output for each tuple
-        String subClusterName = rs.getString(1);
-        String amRMAddress = rs.getString(2);
-        String clientRMAddress = rs.getString(3);
-        String rmAdminAddress = rs.getString(4);
-        String webAppAddress = rs.getString(5);
-        long lastHeartBeat = rs.getTimestamp(6, utcCalendar).getTime();
-        SubClusterState state = SubClusterState.fromString(rs.getString(7));
-        long lastStartTime = rs.getLong(8);
-        String capability = rs.getString(9);
+        String subClusterName = rs.getString("subClusterId");
+        String amRMAddress = rs.getString("amRMServiceAddress");
+        String clientRMAddress = rs.getString("clientRMServiceAddress");
+        String rmAdminAddress = rs.getString("rmAdminServiceAddress");
+        String webAppAddress = rs.getString("rmWebServiceAddress");
+        long lastHeartBeat = rs.getTimestamp("lastHeartBeat", utcCalendar).getTime();
+        SubClusterState state = SubClusterState.fromString(rs.getString("state"));
+        long lastStartTime = rs.getLong("lastStartTime");
+        String capability = rs.getString("capability");
 
         SubClusterId subClusterId = SubClusterId.newInstance(subClusterName);
         SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
@@ -488,15 +501,12 @@ public GetSubClustersInfoResponse getSubClusters(
         FederationStateStoreClientMetrics
             .succeededStateStoreCall(stopTime - startTime);
 
-
         // Check if the output it is a valid subcluster
         try {
-          FederationMembershipStateStoreInputValidator
-              .checkSubClusterInfo(subClusterInfo);
+          FederationMembershipStateStoreInputValidator.checkSubClusterInfo(subClusterInfo);
         } catch (FederationStateStoreInvalidInputException e) {
-          String errMsg =
-              "SubCluster " + subClusterId.toString() + " is not valid";
-          FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+          FederationStateStoreUtils.logAndThrowStoreException(e, LOG,
+              "SubCluster %s is not valid.", subClusterId);
         }
 
         // Filter the inactive
@@ -536,68 +546,61 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
       cstmt = getCallableStatement(CALL_SP_ADD_APPLICATION_HOME_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, appId.toString());
-      cstmt.setString(2, subClusterId.getId());
-      cstmt.registerOutParameter(3, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
+      cstmt.setString("applicationId_IN", appId.toString());
+      cstmt.setString("homeSubCluster_IN", subClusterId.getId());
+      cstmt.registerOutParameter("storedHomeSubCluster_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
       cstmt.executeUpdate();
       long stopTime = clock.getTime();
 
-      subClusterHome = cstmt.getString(3);
+      subClusterHome = cstmt.getString("storedHomeSubCluster_OUT");
       SubClusterId subClusterIdHome = SubClusterId.newInstance(subClusterHome);
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
       // For failover reason, we check the returned SubClusterId.
       // If it is equal to the subclusterId we sent, the call added the new
       // application into FederationStateStore. If the call returns a different
       // SubClusterId it means we already tried to insert this application but a
       // component (Router/StateStore/RM) failed during the submission.
+      int rowCount = cstmt.getInt("rowCount_OUT");
       if (subClusterId.equals(subClusterIdHome)) {
         // Check the ROWCOUNT value, if it is equal to 0 it means the call
         // did not add a new application into FederationStateStore
-        if (cstmt.getInt(4) == 0) {
-          LOG.info(
-              "The application {} was not inserted in the StateStore because it"
-                  + " was already present in SubCluster {}",
-              appId, subClusterHome);
-        } else if (cstmt.getInt(4) != 1) {
+        if (rowCount == 0) {
+          LOG.info("The application {} was not inserted in the StateStore because it"
+              + " was already present in SubCluster {}", appId, subClusterHome);
+        } else if (cstmt.getInt("rowCount_OUT") != 1) {
           // Check the ROWCOUNT value, if it is different from 1 it means the
           // call had a wrong behavior. Maybe the database is not set correctly.
-          String errMsg = "Wrong behavior during the insertion of SubCluster "
-              + subClusterId;
-          FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+          FederationStateStoreUtils.logAndThrowStoreException(LOG,
+              "Wrong behavior during the insertion of SubCluster %s.", subClusterId);
         }
 
-        LOG.info("Insert into the StateStore the application: " + appId
-            + " in SubCluster:  " + subClusterHome);
+        LOG.info("Insert into the StateStore the application: {} in SubCluster: {}.",
+            appId, subClusterHome);
       } else {
         // Check the ROWCOUNT value, if it is different from 0 it means the call
         // did edited the table
-        if (cstmt.getInt(4) != 0) {
-          String errMsg =
-              "The application " + appId + " does exist but was overwritten";
-          FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+        if (rowCount != 0) {
+          FederationStateStoreUtils.logAndThrowStoreException(LOG,
+              "The application %s does exist but was overwritten.", appId);
         }
-        LOG.info("Application: " + appId + " already present with SubCluster:  "
-            + subClusterHome);
+        LOG.info("Application: {} already present with SubCluster: {}.", appId, subClusterHome);
       }
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils
-          .logAndThrowRetriableException(LOG,
-              "Unable to insert the newly generated application "
-                  + request.getApplicationHomeSubCluster().getApplicationId(),
-              e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to insert the newly generated application %s.", appId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
     }
+
     return AddApplicationHomeSubClusterResponse
         .newInstance(SubClusterId.newInstance(subClusterHome));
   }
@@ -620,9 +623,9 @@ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
       cstmt = getCallableStatement(CALL_SP_UPDATE_APPLICATION_HOME_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, appId.toString());
-      cstmt.setString(2, subClusterId.getId());
-      cstmt.registerOutParameter(3, java.sql.Types.INTEGER);
+      cstmt.setString("applicationId_IN", appId.toString());
+      cstmt.setString("homeSubCluster_IN", subClusterId.getId());
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -631,31 +634,25 @@ public UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not update the application into FederationStateStore
-      if (cstmt.getInt(3) == 0) {
-        String errMsg = "Application " + appId + " does not exist";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Application %s does not exist.", appId);
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(3) != 1) {
-        String errMsg =
-            "Wrong behavior during the update of SubCluster " + subClusterId;
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (cstmt.getInt("rowCount_OUT") != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during the update of SubCluster %s.", subClusterId);
       }
 
-      LOG.info(
-          "Update the SubCluster to {} for application {} in the StateStore",
+      LOG.info("Update the SubCluster to {} for application {} in the StateStore",
           subClusterId, appId);
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
-
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils
-          .logAndThrowRetriableException(LOG,
-              "Unable to update the application "
-                  + request.getApplicationHomeSubCluster().getApplicationId(),
-              e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to update the application %s.", appId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -673,76 +670,84 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
 
     SubClusterId homeRM = null;
 
+    ApplicationId applicationId = request.getApplicationId();
+
     try {
       cstmt = getCallableStatement(CALL_SP_GET_APPLICATION_HOME_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, request.getApplicationId().toString());
-      cstmt.registerOutParameter(2, java.sql.Types.VARCHAR);
+      cstmt.setString("applicationId_IN", applicationId.toString());
+      cstmt.registerOutParameter("homeSubCluster_OUT", java.sql.Types.VARCHAR);
 
       // Execute the query
       long startTime = clock.getTime();
       cstmt.execute();
       long stopTime = clock.getTime();
 
-      if (cstmt.getString(2) != null) {
-        homeRM = SubClusterId.newInstance(cstmt.getString(2));
+      String homeSubCluster = cstmt.getString("homeSubCluster_OUT");
+      if (homeSubCluster != null) {
+        homeRM = SubClusterId.newInstance(homeSubCluster);
       } else {
-        String errMsg =
-            "Application " + request.getApplicationId() + " does not exist";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Application %s does not exist.", applicationId);
       }
 
       LOG.debug("Got the information about the specified application {}."
-          + " The AM is running in {}", request.getApplicationId(), homeRM);
+          + " The AM is running in {}", applicationId, homeRM);
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to obtain the application information "
-              + "for the specified application " + request.getApplicationId(),
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to obtain the application information for the specified application %s.",
+          applicationId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
     }
-    return GetApplicationHomeSubClusterResponse
-        .newInstance(ApplicationHomeSubCluster
-            .newInstance(request.getApplicationId(), homeRM));
+    return GetApplicationHomeSubClusterResponse.newInstance(request.getApplicationId(), homeRM);
   }
 
   @Override
   public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       GetApplicationsHomeSubClusterRequest request) throws YarnException {
+
+    if (request == null) {
+      throw new YarnException("Missing getApplicationsHomeSubCluster request");
+    }
+
     CallableStatement cstmt = null;
     ResultSet rs = null;
-    List<ApplicationHomeSubCluster> appsHomeSubClusters =
-        new ArrayList<ApplicationHomeSubCluster>();
+    List<ApplicationHomeSubCluster> appsHomeSubClusters = new ArrayList<>();
 
     try {
       cstmt = getCallableStatement(CALL_SP_GET_APPLICATIONS_HOME_SUBCLUSTER);
+      cstmt.setInt("limit_IN", maxAppsInStateStore);
+      String homeSubClusterIN = StringUtils.EMPTY;
+      SubClusterId subClusterId = request.getSubClusterId();
+      if (subClusterId != null) {
+        homeSubClusterIN = subClusterId.toString();
+      }
+      cstmt.setString("homeSubCluster_IN", homeSubClusterIN);
 
       // Execute the query
       long startTime = clock.getTime();
       rs = cstmt.executeQuery();
       long stopTime = clock.getTime();
 
-      while (rs.next()) {
+      while (rs.next() && appsHomeSubClusters.size() <= maxAppsInStateStore) {
 
         // Extract the output for each tuple
-        String applicationId = rs.getString(1);
-        String homeSubCluster = rs.getString(2);
+        String applicationId = rs.getString("applicationId");
+        String homeSubCluster = rs.getString("homeSubCluster");
 
         appsHomeSubClusters.add(ApplicationHomeSubCluster.newInstance(
             ApplicationId.fromString(applicationId),
             SubClusterId.newInstance(homeSubCluster)));
       }
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
@@ -752,8 +757,8 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt, null, rs);
     }
-    return GetApplicationsHomeSubClusterResponse
-        .newInstance(appsHomeSubClusters);
+
+    return GetApplicationsHomeSubClusterResponse.newInstance(appsHomeSubClusters);
   }
 
   @Override
@@ -764,13 +769,13 @@ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
     FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
 
     CallableStatement cstmt = null;
-
+    ApplicationId applicationId = request.getApplicationId();
     try {
       cstmt = getCallableStatement(CALL_SP_DELETE_APPLICATION_HOME_SUBCLUSTER);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, request.getApplicationId().toString());
-      cstmt.registerOutParameter(2, java.sql.Types.INTEGER);
+      cstmt.setString("applicationId_IN", applicationId.toString());
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -779,28 +784,25 @@ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not delete the application from FederationStateStore
-      if (cstmt.getInt(2) == 0) {
-        String errMsg =
-            "Application " + request.getApplicationId() + " does not exist";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Application %s does not exist.", applicationId);
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(2) != 1) {
-        String errMsg = "Wrong behavior during deleting the application "
-            + request.getApplicationId();
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (cstmt.getInt("rowCount_OUT") != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during deleting the application %s.", applicationId);
       }
 
-      LOG.info("Delete from the StateStore the application: {}",
-          request.getApplicationId());
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      LOG.info("Delete from the StateStore the application: {}", request.getApplicationId());
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to delete the application " + request.getApplicationId(), e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to delete the application %s.", applicationId);
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -822,9 +824,9 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
       cstmt = getCallableStatement(CALL_SP_GET_POLICY_CONFIGURATION);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, request.getQueue());
-      cstmt.registerOutParameter(2, java.sql.Types.VARCHAR);
-      cstmt.registerOutParameter(3, java.sql.Types.VARBINARY);
+      cstmt.setString("queue_IN", request.getQueue());
+      cstmt.registerOutParameter("policyType_OUT", java.sql.Types.VARCHAR);
+      cstmt.registerOutParameter("params_OUT", java.sql.Types.VARBINARY);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -832,10 +834,11 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
       long stopTime = clock.getTime();
 
       // Check if the output it is a valid policy
-      if (cstmt.getString(2) != null && cstmt.getBytes(3) != null) {
-        subClusterPolicyConfiguration =
-            SubClusterPolicyConfiguration.newInstance(request.getQueue(),
-                cstmt.getString(2), ByteBuffer.wrap(cstmt.getBytes(3)));
+      String policyType = cstmt.getString("policyType_OUT");
+      byte[] param = cstmt.getBytes("params_OUT");
+      if (policyType != null && param != null) {
+        subClusterPolicyConfiguration = SubClusterPolicyConfiguration.newInstance(
+            request.getQueue(), policyType, ByteBuffer.wrap(param));
         LOG.debug("Selected from StateStore the policy for the queue: {}",
             subClusterPolicyConfiguration);
       } else {
@@ -843,20 +846,17 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
         return null;
       }
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to select the policy for the queue :" + request.getQueue(),
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to select the policy for the queue : %s." + request.getQueue());
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
     }
-    return GetSubClusterPolicyConfigurationResponse
-        .newInstance(subClusterPolicyConfiguration);
+    return GetSubClusterPolicyConfigurationResponse.newInstance(subClusterPolicyConfiguration);
   }
 
   @Override
@@ -874,10 +874,10 @@ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
       cstmt = getCallableStatement(CALL_SP_SET_POLICY_CONFIGURATION);
 
       // Set the parameters for the stored procedure
-      cstmt.setString(1, policyConf.getQueue());
-      cstmt.setString(2, policyConf.getType());
-      cstmt.setBytes(3, getByteArray(policyConf.getParams()));
-      cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
+      cstmt.setString("queue_IN", policyConf.getQueue());
+      cstmt.setString("policyType_IN", policyConf.getType());
+      cstmt.setBytes("params_IN", getByteArray(policyConf.getParams()));
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
 
       // Execute the query
       long startTime = clock.getTime();
@@ -886,30 +886,25 @@ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
 
       // Check the ROWCOUNT value, if it is equal to 0 it means the call
       // did not add a new policy into FederationStateStore
-      if (cstmt.getInt(4) == 0) {
-        String errMsg = "The policy " + policyConf.getQueue()
-            + " was not insert into the StateStore";
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      int rowCount = cstmt.getInt("rowCount_OUT");
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "The policy %s was not insert into the StateStore.", policyConf.getQueue());
       }
       // Check the ROWCOUNT value, if it is different from 1 it means the call
       // had a wrong behavior. Maybe the database is not set correctly.
-      if (cstmt.getInt(4) != 1) {
-        String errMsg =
-            "Wrong behavior during insert the policy " + policyConf.getQueue();
-        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      if (rowCount != 1) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during insert the policy %s.", policyConf.getQueue());
       }
 
-      LOG.info("Insert into the state store the policy for the queue: "
-          + policyConf.getQueue());
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      LOG.info("Insert into the state store the policy for the queue: {}.", policyConf.getQueue());
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
-      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
-          "Unable to insert the newly generated policy for the queue :"
-              + policyConf.getQueue(),
-          e);
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to insert the newly generated policy for the queue : %s.", policyConf.getQueue());
     } finally {
       // Return to the pool the CallableStatement
       FederationStateStoreUtils.returnToPool(LOG, cstmt);
@@ -923,8 +918,7 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
 
     CallableStatement cstmt = null;
     ResultSet rs = null;
-    List<SubClusterPolicyConfiguration> policyConfigurations =
-        new ArrayList<SubClusterPolicyConfiguration>();
+    List<SubClusterPolicyConfiguration> policyConfigurations = new ArrayList<>();
 
     try {
       cstmt = getCallableStatement(CALL_SP_GET_POLICIES_CONFIGURATIONS);
@@ -935,20 +929,17 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
       long stopTime = clock.getTime();
 
       while (rs.next()) {
-
         // Extract the output for each tuple
-        String queue = rs.getString(1);
-        String type = rs.getString(2);
-        byte[] policyInfo = rs.getBytes(3);
+        String queue = rs.getString("queue");
+        String type = rs.getString("policyType");
+        byte[] policyInfo = rs.getBytes("params");
 
         SubClusterPolicyConfiguration subClusterPolicyConfiguration =
-            SubClusterPolicyConfiguration.newInstance(queue, type,
-                ByteBuffer.wrap(policyInfo));
+            SubClusterPolicyConfiguration.newInstance(queue, type, ByteBuffer.wrap(policyInfo));
         policyConfigurations.add(subClusterPolicyConfiguration);
       }
 
-      FederationStateStoreClientMetrics
-          .succeededStateStoreCall(stopTime - startTime);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
 
     } catch (SQLException e) {
       FederationStateStoreClientMetrics.failedStateStoreCall();
@@ -959,8 +950,7 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
       FederationStateStoreUtils.returnToPool(LOG, cstmt, null, rs);
     }
 
-    return GetSubClusterPoliciesConfigurationsResponse
-        .newInstance(policyConfigurations);
+    return GetSubClusterPoliciesConfigurationsResponse.newInstance(policyConfigurations);
   }
 
   @Override
@@ -988,12 +978,14 @@ public void close() throws Exception {
    * @return a connection from the DataSource pool.
    * @throws SQLException on failure
    */
-  public Connection getConnection() throws SQLException {
+  @VisibleForTesting
+  protected Connection getConnection() throws SQLException {
     FederationStateStoreClientMetrics.incrConnections();
     return dataSource.getConnection();
   }
 
-  private CallableStatement getCallableStatement(String procedure)
+  @VisibleForTesting
+  protected CallableStatement getCallableStatement(String procedure)
       throws SQLException {
     return conn.prepareCall(procedure);
   }
@@ -1004,4 +996,402 @@ private static byte[] getByteArray(ByteBuffer bb) {
     return ba;
   }
 
-}
+  @Override
+  public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
+      AddReservationHomeSubClusterRequest request) throws YarnException {
+
+    // validate
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    CallableStatement cstmt = null;
+
+    ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
+    ReservationId reservationId = reservationHomeSubCluster.getReservationId();
+    SubClusterId subClusterId = reservationHomeSubCluster.getHomeSubCluster();
+    SubClusterId subClusterHomeId = null;
+
+    try {
+
+      // Defined the sp_addReservationHomeSubCluster procedure
+      // this procedure requires 4 parameters
+      // Input parameters
+      // 1）IN reservationId_IN varchar(128)
+      // 2）IN homeSubCluster_IN varchar(256)
+      // Output parameters
+      // 3）OUT storedHomeSubCluster_OUT varchar(256)
+      // 4）OUT rowCount_OUT int
+
+      // Call procedure
+      cstmt = getCallableStatement(CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER);
+
+      // Set the parameters for the stored procedure
+      // 1）IN reservationId_IN varchar(128)
+      cstmt.setString("reservationId_IN", reservationId.toString());
+      // 2）IN homeSubCluster_IN varchar(256)
+      cstmt.setString("homeSubCluster_IN", subClusterId.getId());
+      // 3) OUT storedHomeSubCluster_OUT varchar(256)
+      cstmt.registerOutParameter("storedHomeSubCluster_OUT", java.sql.Types.VARCHAR);
+      // 4) OUT rowCount_OUT int
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+      // Execute the query
+      long startTime = clock.getTime();
+      cstmt.executeUpdate();
+      long stopTime = clock.getTime();
+
+      // Get SubClusterHome
+      String subClusterHomeIdString = cstmt.getString("storedHomeSubCluster_OUT");
+      subClusterHomeId = SubClusterId.newInstance(subClusterHomeIdString);
+
+      // Get rowCount
+      int rowCount = cstmt.getInt("rowCount_OUT");
+
+      // For failover reason, we check the returned subClusterId.
+      // 1.If it is equal to the subClusterId we sent, the call added the new
+      // reservation into FederationStateStore.
+      // 2.If the call returns a different subClusterId
+      // it means we already tried to insert this reservation
+      // but a component (Router/StateStore/RM) failed during the submission.
+      if (subClusterId.equals(subClusterHomeId)) {
+        // if it is equal to 0
+        // it means the call did not add a new reservation into FederationStateStore.
+        if (rowCount == 0) {
+          LOG.info("The reservation {} was not inserted in the StateStore because it" +
+              " was already present in subCluster {}", reservationId, subClusterHomeId);
+        } else if (rowCount != 1) {
+          // if it is different from 1
+          // it means the call had a wrong behavior. Maybe the database is not set correctly.
+          FederationStateStoreUtils.logAndThrowStoreException(LOG,
+              "Wrong behavior during the insertion of subCluster %s according to reservation %s. " +
+              "The database expects to insert 1 record, but the number of " +
+              "inserted changes is greater than 1, " +
+              "please check the records of the database.",
+              subClusterId, reservationId);
+        }
+      } else {
+        // If it is different from 0,
+        // it means that there is a data situation that does not meet the expectations,
+        // and an exception should be thrown at this time
+        if (rowCount != 0) {
+          FederationStateStoreUtils.logAndThrowStoreException(LOG,
+              "The reservation %s does exist but was overwritten.", reservationId);
+        }
+        LOG.info("Reservation: {} already present with subCluster: {}.",
+            reservationId, subClusterHomeId);
+      }
+
+      // Record successful call time
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
+    } catch (SQLException e) {
+      FederationStateStoreClientMetrics.failedStateStoreCall();
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to insert the newly generated reservation %s to subCluster %s.",
+          reservationId, subClusterId);
+    } finally {
+      // Return to the pool the CallableStatement
+      FederationStateStoreUtils.returnToPool(LOG, cstmt);
+    }
+
+    return AddReservationHomeSubClusterResponse.newInstance(subClusterHomeId);
+  }
+
+  @Override
+  public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
+      GetReservationHomeSubClusterRequest request) throws YarnException {
+    // validate
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+
+    CallableStatement cstmt = null;
+    ReservationId reservationId = request.getReservationId();
+    SubClusterId subClusterId = null;
+
+    try {
+
+      // Defined the sp_getReservationHomeSubCluster procedure
+      // this procedure requires 2 parameters
+      // Input parameters
+      // 1）IN reservationId_IN varchar(128)
+      // Output parameters
+      // 2）OUT homeSubCluster_OUT varchar(256)
+
+      cstmt = getCallableStatement(CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER);
+
+      // Set the parameters for the stored procedure
+      // 1）IN reservationId_IN varchar(128)
+      cstmt.setString("reservationId_IN", reservationId.toString());
+      // 2）OUT homeSubCluster_OUT varchar(256)
+      cstmt.registerOutParameter("homeSubCluster_OUT", java.sql.Types.VARCHAR);
+
+      // Execute the query
+      long startTime = clock.getTime();
+      cstmt.execute();
+      long stopTime = clock.getTime();
+
+      // Get Result
+      String subClusterHomeIdString = cstmt.getString("homeSubCluster_OUT");
+
+      if (StringUtils.isNotBlank(subClusterHomeIdString)) {
+        subClusterId = SubClusterId.newInstance(subClusterHomeIdString);
+      } else {
+        // If subClusterHomeIdString blank, we need to throw an exception
+        FederationStateStoreUtils.logAndThrowRetriableException(LOG,
+            "Reservation %s does not exist", reservationId);
+      }
+
+      LOG.info("Got the information about the specified reservation {} in subCluster = {}.",
+          reservationId, subClusterId);
+
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
+
+      ReservationHomeSubCluster homeSubCluster =
+          ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+      return GetReservationHomeSubClusterResponse.newInstance(homeSubCluster);
+
+    } catch (SQLException e) {
+      FederationStateStoreClientMetrics.failedStateStoreCall();
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to obtain the reservation information according to %s.", reservationId);
+    } finally {
+      // Return to the pool the CallableStatement
+      FederationStateStoreUtils.returnToPool(LOG, cstmt);
+    }
+
+    throw new YarnException(
+        "Unable to obtain the reservation information according to " + reservationId);
+  }
+
+  @Override
+  public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
+      GetReservationsHomeSubClusterRequest request) throws YarnException {
+    CallableStatement cstmt = null;
+    ResultSet rs = null;
+    List<ReservationHomeSubCluster> reservationsHomeSubClusters = new ArrayList<>();
+
+    try {
+
+      // Defined the sp_getReservationsHomeSubCluster procedure
+      // This procedure requires no input parameters, but will have 2 output parameters
+      // Output parameters
+      // 1）OUT reservationId
+      // 2）OUT homeSubCluster
+
+      cstmt = getCallableStatement(CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER);
+
+      // Execute the query
+      long startTime = clock.getTime();
+      rs = cstmt.executeQuery();
+      long stopTime = clock.getTime();
+
+      while (rs.next()) {
+        // Extract the output for each tuple
+        // 1）OUT reservationId
+        String dbReservationId = rs.getString("reservationId");
+        // 2）OUT homeSubCluster
+        String dbHomeSubCluster = rs.getString("homeSubCluster");
+
+        // Generate parameters
+        ReservationId reservationId = ReservationId.parseReservationId(dbReservationId);
+        SubClusterId homeSubCluster = SubClusterId.newInstance(dbHomeSubCluster);
+        ReservationHomeSubCluster reservationHomeSubCluster =
+            ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster);
+        reservationsHomeSubClusters.add(reservationHomeSubCluster);
+      }
+
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
+
+      return GetReservationsHomeSubClusterResponse.newInstance(
+          reservationsHomeSubClusters);
+    } catch (Exception e) {
+      FederationStateStoreClientMetrics.failedStateStoreCall();
+      FederationStateStoreUtils.logAndThrowRetriableException(LOG,
+          "Unable to obtain the information for all the reservations.", e);
+    } finally {
+      // Return to the pool the CallableStatement
+      FederationStateStoreUtils.returnToPool(LOG, cstmt, null, rs);
+    }
+
+    throw new YarnException("Unable to obtain the information for all the reservations.");
+  }
+
+  @Override
+  public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
+      DeleteReservationHomeSubClusterRequest request) throws YarnException {
+
+    // validate
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+
+    CallableStatement cstmt = null;
+    ReservationId reservationId = request.getReservationId();
+
+    try {
+
+      // Defined the sp_deleteReservationHomeSubCluster procedure
+      // This procedure requires 1 input parameters, 1 output parameters
+      // Input parameters
+      // 1）IN reservationId_IN varchar(128)
+      // Output parameters
+      // 2）OUT rowCount_OUT int
+
+      cstmt = getCallableStatement(CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER);
+
+      // Set the parameters for the stored procedure
+      // 1）IN reservationId_IN varchar(128)
+      cstmt.setString("reservationId_IN", reservationId.toString());
+      // 2）OUT rowCount_OUT int
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+      // Execute the query
+      long startTime = clock.getTime();
+      cstmt.executeUpdate();
+      long stopTime = clock.getTime();
+
+      int rowCount = cstmt.getInt("rowCount_OUT");
+
+      // if it is equal to 0 it means the call
+      // did not delete the reservation from FederationStateStore
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Reservation %s does not exist", reservationId);
+      } else if (rowCount != 1) {
+        // if it is different from 1 it means the call
+        // had a wrong behavior. Maybe the database is not set correctly.
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during deleting the reservation %s. " +
+            "The database is expected to delete 1 record, " +
+            "but the number of deleted records returned by the database is greater than 1, " +
+            "indicating that a duplicate reservationId occurred during the deletion process.",
+            reservationId);
+      }
+
+      LOG.info("Delete from the StateStore the reservation: {}.", reservationId);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
+      return DeleteReservationHomeSubClusterResponse.newInstance();
+    } catch (SQLException e) {
+      FederationStateStoreClientMetrics.failedStateStoreCall();
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to delete the reservation %s.", reservationId);
+    } finally {
+      // Return to the pool the CallableStatement
+      FederationStateStoreUtils.returnToPool(LOG, cstmt);
+    }
+    throw new YarnException("Unable to delete the reservation " + reservationId);
+  }
+
+  @Override
+  public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
+      UpdateReservationHomeSubClusterRequest request) throws YarnException {
+
+    // validate
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+
+    CallableStatement cstmt = null;
+    ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
+    ReservationId reservationId = reservationHomeSubCluster.getReservationId();
+    SubClusterId subClusterId = reservationHomeSubCluster.getHomeSubCluster();
+
+    try {
+
+      // Defined the sp_updateReservationHomeSubCluster procedure
+      // This procedure requires 2 input parameters, 1 output parameters
+      // Input parameters
+      // 1）IN reservationId_IN varchar(128)
+      // 2）IN homeSubCluster_IN varchar(256)
+      // Output parameters
+      // 3）OUT rowCount_OUT int
+
+      cstmt = getCallableStatement(CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER);
+
+      // Set the parameters for the stored procedure
+      // 1）IN reservationId_IN varchar(128)
+      cstmt.setString("reservationId_IN", reservationId.toString());
+      // 2）IN homeSubCluster_IN varchar(256)
+      cstmt.setString("homeSubCluster_IN", subClusterId.getId());
+      // 3）OUT rowCount_OUT int
+      cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+      // Execute the query
+      long startTime = clock.getTime();
+      cstmt.executeUpdate();
+      long stopTime = clock.getTime();
+
+      int rowCount = cstmt.getInt("rowCount_OUT");
+
+      // if it is equal to 0 it means the call
+      // did not update the reservation into FederationStateStore
+      if (rowCount == 0) {
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Reservation %s does not exist", reservationId);
+      } else if (rowCount != 1) {
+        // if it is different from 1 it means the call
+        // had a wrong behavior. Maybe the database is not set correctly.
+        FederationStateStoreUtils.logAndThrowStoreException(LOG,
+            "Wrong behavior during update the subCluster %s according to reservation %s. " +
+            "The database is expected to update 1 record, " +
+            "but the number of database update records is greater than 1, " +
+            "the records of the database should be checked.",
+            subClusterId, reservationId);
+      }
+      LOG.info("Update the subCluster to {} for reservation {} in the StateStore.",
+          subClusterId, reservationId);
+      FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime);
+      return UpdateReservationHomeSubClusterResponse.newInstance();
+    } catch (SQLException e) {
+      FederationStateStoreClientMetrics.failedStateStoreCall();
+      FederationStateStoreUtils.logAndThrowRetriableException(e, LOG,
+          "Unable to update the subCluster %s according to reservation %s.",
+          subClusterId, reservationId);
+    } finally {
+      // Return to the pool the CallableStatement
+      FederationStateStoreUtils.returnToPool(LOG, cstmt);
+    }
+    throw new YarnException(
+        "Unable to update the subCluster " + subClusterId +
+        " according to reservation" + reservationId);
+  }
+
+  @VisibleForTesting
+  public Connection getConn() {
+    return conn;
+  }
+
+  @Override
+  public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java
new file mode 100644
index 0000000000000..113e4850a5709
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@Metrics(context="ZKFederationStateStore-op-durations")
+public final class ZKFederationStateStoreOpDurations implements MetricsSource {
+
+  @Metric("Duration for a add application homeSubcluster call")
+  private MutableRate addAppHomeSubCluster;
+
+  @Metric("Duration for a update application homeSubcluster call")
+  private MutableRate updateAppHomeSubCluster;
+
+  @Metric("Duration for a get application homeSubcluster call")
+  private MutableRate getAppHomeSubCluster;
+
+  @Metric("Duration for a get applications homeSubcluster call")
+  private MutableRate getAppsHomeSubCluster;
+
+  @Metric("Duration for a delete applications homeSubcluster call")
+  private MutableRate deleteAppHomeSubCluster;
+
+  @Metric("Duration for a register subCluster call")
+  private MutableRate registerSubCluster;
+
+  @Metric("Duration for a deregister subCluster call")
+  private MutableRate deregisterSubCluster;
+
+  @Metric("Duration for a subCluster Heartbeat call")
+  private MutableRate subClusterHeartbeat;
+
+  @Metric("Duration for a get SubCluster call")
+  private MutableRate getSubCluster;
+
+  @Metric("Duration for a get SubClusters call")
+  private MutableRate getSubClusters;
+
+  @Metric("Duration for a get PolicyConfiguration call")
+  private MutableRate getPolicyConfiguration;
+
+  @Metric("Duration for a set PolicyConfiguration call")
+  private MutableRate setPolicyConfiguration;
+
+  @Metric("Duration for a get PolicyConfigurations call")
+  private MutableRate getPoliciesConfigurations;
+
+  @Metric("Duration for a add reservation homeSubCluster call")
+  private MutableRate addReservationHomeSubCluster;
+
+  @Metric("Duration for a get reservation homeSubCluster call")
+  private MutableRate getReservationHomeSubCluster;
+
+  @Metric("Duration for a get reservations homeSubCluster call")
+  private MutableRate getReservationsHomeSubCluster;
+
+  @Metric("Duration for a delete reservation homeSubCluster call")
+  private MutableRate deleteReservationHomeSubCluster;
+
+  @Metric("Duration for a update reservation homeSubCluster call")
+  private MutableRate updateReservationHomeSubCluster;
+
+  protected static final MetricsInfo RECORD_INFO =
+      info("ZKFederationStateStoreOpDurations", "Durations of ZKFederationStateStore calls");
+
+  private final MetricsRegistry registry;
+
+  private static final ZKFederationStateStoreOpDurations INSTANCE =
+      new ZKFederationStateStoreOpDurations();
+
+  public static ZKFederationStateStoreOpDurations getInstance() {
+    return INSTANCE;
+  }
+
+  private ZKFederationStateStoreOpDurations() {
+    registry = new MetricsRegistry(RECORD_INFO);
+    registry.tag(RECORD_INFO, "ZKFederationStateStoreOpDurations");
+
+    MetricsSystem ms = DefaultMetricsSystem.instance();
+    if (ms != null) {
+      ms.register(RECORD_INFO.name(), RECORD_INFO.description(), this);
+    }
+  }
+
+  @Override
+  public synchronized void getMetrics(MetricsCollector collector, boolean all) {
+    registry.snapshot(collector.addRecord(registry.info()), all);
+  }
+
+  public void addAppHomeSubClusterDuration(long startTime, long endTime) {
+    addAppHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addUpdateAppHomeSubClusterDuration(long startTime, long endTime) {
+    updateAppHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addGetAppHomeSubClusterDuration(long startTime, long endTime) {
+    getAppHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addGetAppsHomeSubClusterDuration(long startTime, long endTime) {
+    getAppsHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addDeleteAppHomeSubClusterDuration(long startTime, long endTime) {
+    deleteAppHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addRegisterSubClusterDuration(long startTime, long endTime) {
+    registerSubCluster.add(endTime - startTime);
+  }
+
+  public void addDeregisterSubClusterDuration(long startTime, long endTime) {
+    deregisterSubCluster.add(endTime - startTime);
+  }
+
+  public void addSubClusterHeartbeatDuration(long startTime, long endTime) {
+    subClusterHeartbeat.add(endTime - startTime);
+  }
+
+  public void addGetSubClusterDuration(long startTime, long endTime) {
+    getSubCluster.add(endTime - startTime);
+  }
+
+  public void addGetSubClustersDuration(long startTime, long endTime) {
+    getSubClusters.add(endTime - startTime);
+  }
+
+  public void addGetPolicyConfigurationDuration(long startTime, long endTime) {
+    getPolicyConfiguration.add(endTime - startTime);
+  }
+
+  public void addSetPolicyConfigurationDuration(long startTime, long endTime) {
+    setPolicyConfiguration.add(endTime - startTime);
+  }
+
+  public void addGetPoliciesConfigurationsDuration(long startTime, long endTime) {
+    getPoliciesConfigurations.add(endTime - startTime);
+  }
+
+  public void addReservationHomeSubClusterDuration(long startTime, long endTime) {
+    addReservationHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addGetReservationHomeSubClusterDuration(long startTime, long endTime) {
+    getReservationHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addGetReservationsHomeSubClusterDuration(long startTime, long endTime) {
+    getReservationsHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addDeleteReservationHomeSubClusterDuration(long startTime, long endTime) {
+    deleteReservationHomeSubCluster.add(endTime - startTime);
+  }
+
+  public void addUpdateReservationHomeSubClusterDuration(long startTime, long endTime) {
+    updateReservationHomeSubCluster.add(endTime - startTime);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
index c9b5849ad68d2..f4d45f5a723c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
@@ -24,7 +24,11 @@
 import java.util.Calendar;
 import java.util.List;
 import java.util.TimeZone;
+import java.util.Comparator;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.curator.ZKCuratorManager;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -63,8 +67,23 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterPolicyConfigurationPBImpl;
@@ -72,13 +91,19 @@
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
 import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator;
 import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
 import org.apache.zookeeper.data.ACL;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 
+import static org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils.filterHomeSubCluster;
+
 /**
  * ZooKeeper implementation of {@link FederationStateStore}.
  *
@@ -91,8 +116,11 @@
  * |     |----- APP1
  * |     |----- APP2
  * |--- POLICY
- *       |----- QUEUE1
- *       |----- QUEUE1
+ * |     |----- QUEUE1
+ * |     |----- QUEUE1
+ * |--- RESERVATION
+ * |     |----- RESERVATION1
+ * |     |----- RESERVATION2
  */
 public class ZookeeperFederationStateStore implements FederationStateStore {
 
@@ -102,6 +130,7 @@ public class ZookeeperFederationStateStore implements FederationStateStore {
   private final static String ROOT_ZNODE_NAME_MEMBERSHIP = "memberships";
   private final static String ROOT_ZNODE_NAME_APPLICATION = "applications";
   private final static String ROOT_ZNODE_NAME_POLICY = "policies";
+  private final static String ROOT_ZNODE_NAME_RESERVATION = "reservation";
 
   /** Interface to Zookeeper. */
   private ZKCuratorManager zkManager;
@@ -112,11 +141,23 @@ public class ZookeeperFederationStateStore implements FederationStateStore {
   private String appsZNode;
   private String membershipZNode;
   private String policiesZNode;
+  private String reservationsZNode;
+  private int maxAppsInStateStore;
+
+  private volatile Clock clock = SystemClock.getInstance();
+
+  @VisibleForTesting
+  private ZKFederationStateStoreOpDurations opDurations =
+      ZKFederationStateStoreOpDurations.getInstance();
 
   @Override
   public void init(Configuration conf) throws YarnException {
     LOG.info("Initializing ZooKeeper connection");
 
+    maxAppsInStateStore = conf.getInt(
+       YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS,
+       YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_MAX_APPLICATIONS);
+
     baseZNode = conf.get(
         YarnConfiguration.FEDERATION_STATESTORE_ZK_PARENT_PATH,
         YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH);
@@ -131,6 +172,7 @@ public void init(Configuration conf) throws YarnException {
     membershipZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_MEMBERSHIP);
     appsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_APPLICATION);
     policiesZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_POLICY);
+    reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION);
 
     // Create base znode for each entity
     try {
@@ -138,11 +180,11 @@ public void init(Configuration conf) throws YarnException {
       zkManager.createRootDirRecursively(membershipZNode, zkAcl);
       zkManager.createRootDirRecursively(appsZNode, zkAcl);
       zkManager.createRootDirRecursively(policiesZNode, zkAcl);
+      zkManager.createRootDirRecursively(reservationsZNode, zkAcl);
     } catch (Exception e) {
       String errMsg = "Cannot create base directories: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
-
   }
 
   @Override
@@ -156,6 +198,7 @@ public void close() throws Exception {
   public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
       AddApplicationHomeSubClusterRequest request) throws YarnException {
 
+    long start = clock.getTime();
     FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
     ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster();
     ApplicationId appId = app.getApplicationId();
@@ -176,7 +219,8 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
       String errMsg = "Cannot check app home subcluster for " + appId;
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
-
+    long end = clock.getTime();
+    opDurations.addAppHomeSubClusterDuration(start, end);
     return AddApplicationHomeSubClusterResponse
         .newInstance(homeSubCluster);
   }
@@ -187,6 +231,7 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
           UpdateApplicationHomeSubClusterRequest request)
               throws YarnException {
 
+    long start = clock.getTime();
     FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
     ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster();
     ApplicationId appId = app.getApplicationId();
@@ -198,6 +243,9 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
     SubClusterId newSubClusterId =
         request.getApplicationHomeSubCluster().getHomeSubCluster();
     putApp(appId, newSubClusterId, true);
+
+    long end = clock.getTime();
+    opDurations.addUpdateAppHomeSubClusterDuration(start, end);
     return UpdateApplicationHomeSubClusterResponse.newInstance();
   }
 
@@ -205,6 +253,7 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
   public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
       GetApplicationHomeSubClusterRequest request) throws YarnException {
 
+    long start = clock.getTime();
     FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
     ApplicationId appId = request.getApplicationId();
     SubClusterId homeSubCluster = getApp(appId);
@@ -212,29 +261,52 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
       String errMsg = "Application " + appId + " does not exist";
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
-    return GetApplicationHomeSubClusterResponse.newInstance(
-        ApplicationHomeSubCluster.newInstance(appId, homeSubCluster));
+    long end = clock.getTime();
+    opDurations.addGetAppHomeSubClusterDuration(start, end);
+    return GetApplicationHomeSubClusterResponse.newInstance(appId, homeSubCluster);
   }
 
   @Override
   public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       GetApplicationsHomeSubClusterRequest request) throws YarnException {
-    List<ApplicationHomeSubCluster> result = new ArrayList<>();
+
+    if (request == null) {
+      throw new YarnException("Missing getApplicationsHomeSubCluster request");
+    }
 
     try {
-      for (String child : zkManager.getChildren(appsZNode)) {
-        ApplicationId appId = ApplicationId.fromString(child);
-        SubClusterId homeSubCluster = getApp(appId);
-        ApplicationHomeSubCluster app =
-            ApplicationHomeSubCluster.newInstance(appId, homeSubCluster);
-        result.add(app);
-      }
+      long start = clock.getTime();
+      SubClusterId requestSC = request.getSubClusterId();
+      List<String> children = zkManager.getChildren(appsZNode);
+      List<ApplicationHomeSubCluster> result = children.stream()
+          .map(child -> generateAppHomeSC(child))
+          .sorted(Comparator.comparing(ApplicationHomeSubCluster::getCreateTime).reversed())
+          .filter(appHomeSC -> filterHomeSubCluster(requestSC, appHomeSC.getHomeSubCluster()))
+          .limit(maxAppsInStateStore)
+          .collect(Collectors.toList());
+      long end = clock.getTime();
+      opDurations.addGetAppsHomeSubClusterDuration(start, end);
+      LOG.info("filterSubClusterId = {}, appCount = {}.", requestSC, result.size());
+      return GetApplicationsHomeSubClusterResponse.newInstance(result);
     } catch (Exception e) {
       String errMsg = "Cannot get apps: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
 
-    return GetApplicationsHomeSubClusterResponse.newInstance(result);
+    throw new YarnException("Cannot get app by request");
+  }
+
+  private ApplicationHomeSubCluster generateAppHomeSC(String appId) {
+    try {
+      ApplicationId applicationId = ApplicationId.fromString(appId);
+      SubClusterId homeSubCluster = getApp(applicationId);
+      ApplicationHomeSubCluster app =
+          ApplicationHomeSubCluster.newInstance(applicationId, homeSubCluster);
+      return app;
+    } catch (Exception ex) {
+      LOG.error("get homeSubCluster by appId = {}.", appId);
+    }
+    return null;
   }
 
   @Override
@@ -242,7 +314,7 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       deleteApplicationHomeSubCluster(
           DeleteApplicationHomeSubClusterRequest request)
               throws YarnException {
-
+    long start = clock.getTime();
     FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
     ApplicationId appId = request.getApplicationId();
     String appZNode = getNodePath(appsZNode, appId.toString());
@@ -265,13 +337,15 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
       String errMsg = "Cannot delete app: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
-
+    long end = clock.getTime();
+    opDurations.addDeleteAppHomeSubClusterDuration(start, end);
     return DeleteApplicationHomeSubClusterResponse.newInstance();
   }
 
   @Override
   public SubClusterRegisterResponse registerSubCluster(
       SubClusterRegisterRequest request) throws YarnException {
+    long start = clock.getTime();
     FederationMembershipStateStoreInputValidator.validate(request);
     SubClusterInfo subClusterInfo = request.getSubClusterInfo();
     SubClusterId subclusterId = subClusterInfo.getSubClusterId();
@@ -286,12 +360,15 @@ public SubClusterRegisterResponse registerSubCluster(
       String errMsg = "Cannot register subcluster: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
+    long end = clock.getTime();
+    opDurations.addRegisterSubClusterDuration(start, end);
     return SubClusterRegisterResponse.newInstance();
   }
 
   @Override
   public SubClusterDeregisterResponse deregisterSubCluster(
       SubClusterDeregisterRequest request) throws YarnException {
+    long start = clock.getTime();
     FederationMembershipStateStoreInputValidator.validate(request);
     SubClusterId subClusterId = request.getSubClusterId();
     SubClusterState state = request.getState();
@@ -305,14 +382,15 @@ public SubClusterDeregisterResponse deregisterSubCluster(
       subClusterInfo.setState(state);
       putSubclusterInfo(subClusterId, subClusterInfo, true);
     }
-
+    long end = clock.getTime();
+    opDurations.addDeregisterSubClusterDuration(start, end);
     return SubClusterDeregisterResponse.newInstance();
   }
 
   @Override
   public SubClusterHeartbeatResponse subClusterHeartbeat(
       SubClusterHeartbeatRequest request) throws YarnException {
-
+    long start = clock.getTime();
     FederationMembershipStateStoreInputValidator.validate(request);
     SubClusterId subClusterId = request.getSubClusterId();
 
@@ -329,14 +407,15 @@ public SubClusterHeartbeatResponse subClusterHeartbeat(
     subClusterInfo.setCapability(request.getCapability());
 
     putSubclusterInfo(subClusterId, subClusterInfo, true);
-
+    long end = clock.getTime();
+    opDurations.addSubClusterHeartbeatDuration(start, end);
     return SubClusterHeartbeatResponse.newInstance();
   }
 
   @Override
   public GetSubClusterInfoResponse getSubCluster(
       GetSubClusterInfoRequest request) throws YarnException {
-
+    long start = clock.getTime();
     FederationMembershipStateStoreInputValidator.validate(request);
     SubClusterId subClusterId = request.getSubClusterId();
     SubClusterInfo subClusterInfo = null;
@@ -350,12 +429,15 @@ public GetSubClusterInfoResponse getSubCluster(
       String errMsg = "Cannot get subcluster: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
+    long end = clock.getTime();
+    opDurations.addGetSubClusterDuration(start, end);
     return GetSubClusterInfoResponse.newInstance(subClusterInfo);
   }
 
   @Override
   public GetSubClustersInfoResponse getSubClusters(
       GetSubClustersInfoRequest request) throws YarnException {
+    long start = clock.getTime();
     List<SubClusterInfo> result = new ArrayList<>();
 
     try {
@@ -371,6 +453,8 @@ public GetSubClustersInfoResponse getSubClusters(
       String errMsg = "Cannot get subclusters: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
+    long end = clock.getTime();
+    opDurations.addGetSubClustersDuration(start, end);
     return GetSubClustersInfoResponse.newInstance(result);
   }
 
@@ -378,7 +462,7 @@ public GetSubClustersInfoResponse getSubClusters(
   @Override
   public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
       GetSubClusterPolicyConfigurationRequest request) throws YarnException {
-
+    long start = clock.getTime();
     FederationPolicyStoreInputValidator.validate(request);
     String queue = request.getQueue();
     SubClusterPolicyConfiguration policy = null;
@@ -393,6 +477,8 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
       LOG.warn("Policy for queue: {} does not exist.", queue);
       return null;
     }
+    long end = clock.getTime();
+    opDurations.addGetPolicyConfigurationDuration(start, end);
     return GetSubClusterPolicyConfigurationResponse
         .newInstance(policy);
   }
@@ -400,7 +486,7 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
   @Override
   public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
       SetSubClusterPolicyConfigurationRequest request) throws YarnException {
-
+    long start = clock.getTime();
     FederationPolicyStoreInputValidator.validate(request);
     SubClusterPolicyConfiguration policy =
         request.getPolicyConfiguration();
@@ -411,12 +497,15 @@ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
       String errMsg = "Cannot set policy: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
+    long end = clock.getTime();
+    opDurations.addSetPolicyConfigurationDuration(start, end);
     return SetSubClusterPolicyConfigurationResponse.newInstance();
   }
 
   @Override
   public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
       GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+    long start = clock.getTime();
     List<SubClusterPolicyConfiguration> result = new ArrayList<>();
 
     try {
@@ -432,6 +521,8 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
       String errMsg = "Cannot get policies: " + e.getMessage();
       FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
     }
+    long end = clock.getTime();
+    opDurations.addGetPoliciesConfigurationsDuration(start, end);
     return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
   }
 
@@ -637,4 +728,203 @@ private static long getCurrentTime() {
     Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
     return cal.getTimeInMillis();
   }
-}
+
+  private void putReservation(final ReservationId reservationId,
+      final SubClusterId subClusterId, boolean update) throws YarnException {
+    String reservationZNode = getNodePath(reservationsZNode, reservationId.toString());
+    SubClusterIdProto proto = ((SubClusterIdPBImpl)subClusterId).getProto();
+    byte[] data = proto.toByteArray();
+    put(reservationZNode, data, update);
+  }
+
+  private SubClusterId getReservation(final ReservationId reservationId)
+      throws YarnException {
+    String reservationIdZNode = getNodePath(reservationsZNode, reservationId.toString());
+    SubClusterId subClusterId = null;
+    byte[] data = get(reservationIdZNode);
+    if (data != null) {
+      try {
+        subClusterId = new SubClusterIdPBImpl(SubClusterIdProto.parseFrom(data));
+      } catch (InvalidProtocolBufferException e) {
+        String errMsg = "Cannot parse reservation at " + reservationId;
+        FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+      }
+    }
+    return subClusterId;
+  }
+
+  @VisibleForTesting
+  public ZKFederationStateStoreOpDurations getOpDurations() {
+    return opDurations;
+  }
+
+  @Override
+  public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
+      AddReservationHomeSubClusterRequest request) throws YarnException {
+
+    long start = clock.getTime();
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
+    ReservationId reservationId = reservationHomeSubCluster.getReservationId();
+
+    // Try to write the subcluster
+    SubClusterId homeSubCluster = reservationHomeSubCluster.getHomeSubCluster();
+    try {
+      putReservation(reservationId, homeSubCluster, false);
+    } catch (Exception e) {
+      String errMsg = "Cannot add reservation home subcluster for " + reservationId;
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+
+    // Check for the actual subcluster
+    try {
+      homeSubCluster = getReservation(reservationId);
+    } catch (Exception e) {
+      String errMsg = "Cannot check app home subcluster for " + reservationId;
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+    long end = clock.getTime();
+    opDurations.addReservationHomeSubClusterDuration(start, end);
+    return AddReservationHomeSubClusterResponse.newInstance(homeSubCluster);
+  }
+
+  @Override
+  public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
+      GetReservationHomeSubClusterRequest request) throws YarnException {
+
+    long start = clock.getTime();
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationId reservationId = request.getReservationId();
+    SubClusterId homeSubCluster = getReservation(reservationId);
+
+    if (homeSubCluster == null) {
+      String errMsg = "Reservation " + reservationId + " does not exist";
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster);
+    long end = clock.getTime();
+    opDurations.addGetReservationHomeSubClusterDuration(start, end);
+    return GetReservationHomeSubClusterResponse.newInstance(reservationHomeSubCluster);
+  }
+
+  @Override
+  public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
+      GetReservationsHomeSubClusterRequest request) throws YarnException {
+    long start = clock.getTime();
+    List<ReservationHomeSubCluster> result = new ArrayList<>();
+
+    try {
+      for (String child : zkManager.getChildren(reservationsZNode)) {
+        ReservationId reservationId = ReservationId.parseReservationId(child);
+        SubClusterId homeSubCluster = getReservation(reservationId);
+        ReservationHomeSubCluster app =
+            ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster);
+        result.add(app);
+      }
+    } catch (Exception e) {
+      String errMsg = "Cannot get apps: " + e.getMessage();
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+    long end = clock.getTime();
+    opDurations.addGetReservationsHomeSubClusterDuration(start, end);
+    return GetReservationsHomeSubClusterResponse.newInstance(result);
+  }
+
+  @Override
+  public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
+      DeleteReservationHomeSubClusterRequest request) throws YarnException {
+    long start = clock.getTime();
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationId reservationId = request.getReservationId();
+    String reservationZNode = getNodePath(reservationsZNode, reservationId.toString());
+
+    boolean exists = false;
+    try {
+      exists = zkManager.exists(reservationZNode);
+    } catch (Exception e) {
+      String errMsg = "Cannot check reservation: " + e.getMessage();
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+
+    if (!exists) {
+      String errMsg = "Reservation " + reservationId + " does not exist";
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+
+    try {
+      zkManager.delete(reservationZNode);
+    } catch (Exception e) {
+      String errMsg = "Cannot delete reservation: " + e.getMessage();
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+    long end = clock.getTime();
+    opDurations.addDeleteReservationHomeSubClusterDuration(start, end);
+    return DeleteReservationHomeSubClusterResponse.newInstance();
+  }
+
+  @Override
+  public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
+      UpdateReservationHomeSubClusterRequest request) throws YarnException {
+
+    long start = clock.getTime();
+    FederationReservationHomeSubClusterStoreInputValidator.validate(request);
+    ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster();
+    ReservationId reservationId = reservationHomeSubCluster.getReservationId();
+    SubClusterId homeSubCluster = getReservation(reservationId);
+
+    if (homeSubCluster == null) {
+      String errMsg = "Reservation " + reservationId + " does not exist";
+      FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+    }
+
+    SubClusterId newSubClusterId = reservationHomeSubCluster.getHomeSubCluster();
+    putReservation(reservationId, newSubClusterId, true);
+    long end = clock.getTime();
+    opDurations.addUpdateReservationHomeSubClusterDuration(start, end);
+    return UpdateReservationHomeSubClusterResponse.newInstance();
+  }
+
+  @Override
+  public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java
new file mode 100644
index 0000000000000..e51bce342849c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to <code>Federation state
+ * store</code> to map the home subcluster of a newly submitted Reservation.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ReservationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class AddReservationHomeSubClusterRequest {
+
+  @Private
+  @Unstable
+  public static AddReservationHomeSubClusterRequest newInstance(
+      ReservationHomeSubCluster reservationHomeSubCluster) {
+    AddReservationHomeSubClusterRequest mapRequest =
+        Records.newRecord(AddReservationHomeSubClusterRequest.class);
+    mapRequest.setReservationHomeSubCluster(reservationHomeSubCluster);
+    return mapRequest;
+  }
+
+  /**
+   * Get the {@link ReservationHomeSubCluster} representing the mapping of the
+   * Reservation to it's home sub-cluster.
+   *
+   * @return the mapping of the Reservation to it's home sub-cluster.
+   */
+  @Public
+  @Unstable
+  public abstract ReservationHomeSubCluster getReservationHomeSubCluster();
+
+  /**
+   * Set the {@link ReservationHomeSubCluster} representing the mapping of the
+   * Reservation to it's home sub-cluster.
+   *
+   * @param reservationHomeSubCluster the mapping of the Reservation to it's
+   *          home sub-cluster.
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationHomeSubCluster);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java
new file mode 100644
index 0000000000000..0486d889ac19f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * AddReservationHomeSubClusterResponse contains the answer from the
+ * {@code FederationReservationHomeSubClusterStore} to a request to insert a
+ * newly generated ReservationId and its owner.
+ *
+ * The response contains reservation's home sub-cluster as it is stored in the
+ * {@code FederationReservationHomeSubClusterStore}. If a mapping for the
+ * reservation already existed, the {@code SubClusterId} in this response will
+ * return the existing mapping which might be different from that in the
+ * {@code AddReservationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public abstract class AddReservationHomeSubClusterResponse {
+
+  @Private
+  @Unstable
+  public static AddReservationHomeSubClusterResponse newInstance(
+      SubClusterId homeSubCluster) {
+    AddReservationHomeSubClusterResponse response =
+        Records.newRecord(AddReservationHomeSubClusterResponse.class);
+    response.setHomeSubCluster(homeSubCluster);
+    return response;
+  }
+
+  /**
+   * Set the home sub-cluster that this Reservation has been assigned to.
+   *
+   * @param homeSubCluster the {@link SubClusterId} of this reservation's home
+   *          sub-cluster
+   */
+  public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
+
+  /**
+   * Get the home sub-cluster that this Reservation has been assigned to. This
+   * may not match the {@link SubClusterId} in the corresponding response, if
+   * the mapping for the request's reservation already existed.
+   *
+   * @return the {@link SubClusterId} of this reservation's home sub-cluster
+   */
+  public abstract SubClusterId getHomeSubCluster();
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
index 5e4c7ccf4ef9e..e1ea302380a7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ApplicationHomeSubCluster.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -51,6 +53,17 @@ public static ApplicationHomeSubCluster newInstance(ApplicationId appId,
     return appMapping;
   }
 
+  @Private
+  @Unstable
+  public static ApplicationHomeSubCluster newInstance(ApplicationId appId, long createTime,
+      SubClusterId homeSubCluster) {
+    ApplicationHomeSubCluster appMapping = Records.newRecord(ApplicationHomeSubCluster.class);
+    appMapping.setApplicationId(appId);
+    appMapping.setHomeSubCluster(homeSubCluster);
+    appMapping.setCreateTime(createTime);
+    return appMapping;
+  }
+
   /**
    * Get the {@link ApplicationId} representing the unique identifier of the
    * application.
@@ -91,34 +104,63 @@ public static ApplicationHomeSubCluster newInstance(ApplicationId appId,
   @Unstable
   public abstract void setHomeSubCluster(SubClusterId homeSubCluster);
 
+  /**
+   * Get the create time of the subcluster.
+   *
+   * @return the state of the subcluster
+   */
+  @Public
+  @Unstable
+  public abstract long getCreateTime();
+
+  /**
+   * Set the create time of the subcluster.
+   *
+   * @param time the last heartbeat time of the subcluster
+   */
+  @Private
+  @Unstable
+  public abstract void setCreateTime(long time);
+
+
   @Override
   public boolean equals(Object obj) {
+
     if (this == obj) {
       return true;
     }
+
     if (obj == null) {
       return false;
     }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    ApplicationHomeSubCluster other = (ApplicationHomeSubCluster) obj;
-    if (!this.getApplicationId().equals(other.getApplicationId())) {
-      return false;
+
+    if (obj instanceof ApplicationHomeSubCluster) {
+      ApplicationHomeSubCluster other = (ApplicationHomeSubCluster) obj;
+      return new EqualsBuilder()
+          .append(this.getApplicationId(), other.getApplicationId())
+          .append(this.getHomeSubCluster(), other.getHomeSubCluster())
+          .isEquals();
     }
-    return this.getHomeSubCluster().equals(other.getHomeSubCluster());
+
+    return false;
   }
 
   @Override
   public int hashCode() {
-    return getApplicationId().hashCode() * 31 + getHomeSubCluster().hashCode();
+    return new HashCodeBuilder().
+        append(this.getApplicationId()).
+        append(this.getHomeSubCluster()).
+        append(this.getCreateTime()).toHashCode();
   }
 
   @Override
   public String toString() {
-    return "ApplicationHomeSubCluster [getApplicationId()="
-        + getApplicationId() + ", getHomeSubCluster()=" + getHomeSubCluster()
-        + "]";
+    StringBuilder sb = new StringBuilder();
+    sb.append("ApplicationHomeSubCluster: [")
+        .append("ApplicationId: ").append(getApplicationId()).append(", ")
+        .append("HomeSubCluster: ").append(getHomeSubCluster()).append(", ")
+        .append("CreateTime: ").append(getCreateTime()).append(", ")
+        .append("]");
+    return sb.toString();
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java
new file mode 100644
index 0000000000000..81d9fc7aa35bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The request to <code>Federation state store</code> to delete the mapping of
+ * home subcluster of a submitted reservation.
+ */
+@Private
+@Unstable
+public abstract class DeleteReservationHomeSubClusterRequest {
+
+  @Private
+  @Unstable
+  public static DeleteReservationHomeSubClusterRequest newInstance(
+      ReservationId reservationId) {
+    DeleteReservationHomeSubClusterRequest deleteReservationRequest =
+        Records.newRecord(DeleteReservationHomeSubClusterRequest.class);
+    deleteReservationRequest.setReservationId(reservationId);
+    return deleteReservationRequest;
+  }
+
+  /**
+   * Get the identifier of the {@link ReservationId} to be removed from
+   * <code>Federation state store</code> .
+   *
+   * @return the identifier of the Reservation to be removed from Federation
+   *         State Store.
+   */
+  @Public
+  @Unstable
+  public abstract ReservationId getReservationId();
+
+  /**
+   * Set the identifier of the {@link ReservationId} to be removed from
+   * <code>Federation state store</code> .
+   *
+   * @param reservationId the identifier of the Reservation to be removed from
+   *          Federation State Store.
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationId(ReservationId reservationId);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java
new file mode 100644
index 0000000000000..0b7116904be1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * DeleteReservationHomeSubClusterResponse contains the answer from the {@code
+ * FederationReservationHomeSubClusterStore} to a request to delete the mapping
+ * of home subcluster of a submitted reservation. Currently, response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class DeleteReservationHomeSubClusterResponse {
+
+  @Private
+  @Unstable
+  public static DeleteReservationHomeSubClusterResponse newInstance() {
+    DeleteReservationHomeSubClusterResponse response =
+        Records.newRecord(DeleteReservationHomeSubClusterResponse.class);
+    return response;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
index 60735b382f16a..6144b01e86060 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java
@@ -20,6 +20,7 @@
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.Records;
 
 /**
@@ -42,7 +43,9 @@ public abstract class GetApplicationHomeSubClusterResponse {
   @Private
   @Unstable
   public static GetApplicationHomeSubClusterResponse newInstance(
-      ApplicationHomeSubCluster applicationHomeSubCluster) {
+      ApplicationId appId, SubClusterId homeSubCluster) {
+    ApplicationHomeSubCluster applicationHomeSubCluster =
+        ApplicationHomeSubCluster.newInstance(appId, homeSubCluster);
     GetApplicationHomeSubClusterResponse mapResponse =
         Records.newRecord(GetApplicationHomeSubClusterResponse.class);
     mapResponse.setApplicationHomeSubCluster(applicationHomeSubCluster);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
index 60549722093db..06b6987dcbafb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationsHomeSubClusterRequest.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.util.Records;
@@ -37,4 +38,33 @@ public static GetApplicationsHomeSubClusterRequest newInstance() {
     return request;
   }
 
+  @Private
+  @Unstable
+  public static GetApplicationsHomeSubClusterRequest
+      newInstance(SubClusterId subClusterId) {
+    GetApplicationsHomeSubClusterRequest request =
+        Records.newRecord(GetApplicationsHomeSubClusterRequest.class);
+    request.setSubClusterId(subClusterId);
+    return request;
+  }
+
+  /**
+   * Get the {@link SubClusterId} representing the unique identifier of the
+   * subcluster.
+   *
+   * @return the subcluster identifier
+   */
+  @Public
+  @Unstable
+  public abstract SubClusterId getSubClusterId();
+
+  /**
+   * Set the {@link SubClusterId} representing the unique identifier of the
+   * subcluster.
+   *
+   * @param subClusterId the subcluster identifier
+   */
+  @Public
+  @Unstable
+  public abstract void setSubClusterId(SubClusterId subClusterId);
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java
new file mode 100644
index 0000000000000..f82fec6206440
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster for the specified
+ * {@link ReservationId}.
+ */
+@Private
+@Unstable
+public abstract class GetReservationHomeSubClusterRequest {
+
+  @Private
+  @Unstable
+  public static GetReservationHomeSubClusterRequest newInstance(
+      ReservationId reservationId) {
+    GetReservationHomeSubClusterRequest appMapping =
+        Records.newRecord(GetReservationHomeSubClusterRequest.class);
+    appMapping.setReservationId(reservationId);
+    return appMapping;
+  }
+
+  /**
+   * Get the {@link ReservationId} representing the unique identifier of the
+   * application.
+   *
+   * @return the application identifier
+   */
+  @Public
+  @Unstable
+  public abstract ReservationId getReservationId();
+
+  /**
+   * Set the {@link ReservationId} representing the unique identifier of the
+   * application.
+   *
+   * @param reservationId the reservatopm identifier
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationId(ReservationId reservationId);
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java
new file mode 100644
index 0000000000000..50328a176b100
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of a newly submitted
+ * reservation.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ReservationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetReservationHomeSubClusterResponse {
+
+  @Private
+  @Unstable
+  public static GetReservationHomeSubClusterResponse newInstance(
+      ReservationHomeSubCluster reservationHomeSubCluster) {
+    GetReservationHomeSubClusterResponse mapResponse =
+        Records.newRecord(GetReservationHomeSubClusterResponse.class);
+    mapResponse.setReservationHomeSubCluster(reservationHomeSubCluster);
+    return mapResponse;
+  }
+
+  /**
+   * Get the {@link ReservationHomeSubCluster} representing the mapping of the
+   * Reservation to it's home sub-cluster.
+   *
+   * @return the mapping of the reservation to it's home sub-cluster.
+   */
+  @Public
+  @Unstable
+  public abstract ReservationHomeSubCluster getReservationHomeSubCluster();
+
+  /**
+   * Set the {@link ReservationHomeSubCluster} representing the mapping of the
+   * Reservation to it's home sub-cluster.
+   *
+   * @param reservationHomeSubCluster the mapping of the reservation to it's home sub-cluster.
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationHomeSubCluster);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java
new file mode 100644
index 0000000000000..c91d25dcd4930
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Request class to obtain the home sub-cluster mapping of all active
+ * reservations.
+ */
+@Private
+@Unstable
+public abstract class GetReservationsHomeSubClusterRequest {
+
+  @Private
+  @Unstable
+  public static GetReservationsHomeSubClusterRequest newInstance() {
+    GetReservationsHomeSubClusterRequest request =
+        Records.newRecord(GetReservationsHomeSubClusterRequest.class);
+    return request;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java
new file mode 100644
index 0000000000000..f39430dbe6322
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The response sent by <code>Federation state
+ * store</code> to a query for the home subcluster of all submitted
+ * reservations.
+ *
+ * <p>
+ * The response includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ReservationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class GetReservationsHomeSubClusterResponse {
+
+  @Private
+  @Unstable
+  public static GetReservationsHomeSubClusterResponse newInstance(
+      List<ReservationHomeSubCluster> appsHomeSubClusters) {
+    GetReservationsHomeSubClusterResponse mapResponse =
+        Records.newRecord(GetReservationsHomeSubClusterResponse.class);
+    mapResponse.setAppsHomeSubClusters(appsHomeSubClusters);
+    return mapResponse;
+  }
+
+  /**
+   * Get the {@link ReservationHomeSubCluster} list representing the mapping of
+   * all submitted reservations to it's home sub-cluster.
+   *
+   * @return the mapping of all submitted reservation to it's home sub-cluster.
+   */
+  @Public
+  @Unstable
+  public abstract List<ReservationHomeSubCluster> getAppsHomeSubClusters();
+
+  /**
+   * Set the {@link ReservationHomeSubCluster} list representing the mapping of
+   * all submitted reservations to it's home sub-cluster.
+   *
+   * @param reservationsHomeSubClusters the mapping of all submitted reservation
+   *          to it's home sub-cluster.
+   */
+  @Private
+  @Unstable
+  public abstract void setAppsHomeSubClusters(
+      List<ReservationHomeSubCluster> reservationsHomeSubClusters);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
index bcf75aba1aef0..0ffe4ae28a355 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.federation.store.records;
 
 import java.util.List;
+import java.util.Collection;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -36,7 +37,7 @@ public abstract class GetSubClustersInfoResponse {
   @Public
   @Unstable
   public static GetSubClustersInfoResponse newInstance(
-      List<SubClusterInfo> subClusters) {
+      Collection<SubClusterInfo> subClusters) {
     GetSubClustersInfoResponse subClusterInfos =
         Records.newRecord(GetSubClustersInfoResponse.class);
     subClusterInfos.setSubClusters(subClusters);
@@ -61,6 +62,5 @@ public static GetSubClustersInfoResponse newInstance(
    */
   @Private
   @Unstable
-  public abstract void setSubClusters(List<SubClusterInfo> subClusters);
-
+  public abstract void setSubClusters(Collection<SubClusterInfo> subClusters);
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java
new file mode 100644
index 0000000000000..c1a15536d2688
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * ReservationHomeSubCluster is a report of the runtime information of the
+ * reservation that is running in the federated cluster.
+ *
+ * <p>
+ * It includes information such as:
+ * <ul>
+ * <li>{@link ReservationId}</li>
+ * <li>{@link SubClusterId}</li>
+ * </ul>
+ *
+ */
+@Private
+@Unstable
+public abstract class ReservationHomeSubCluster {
+
+  @Private
+  @Unstable
+  public static ReservationHomeSubCluster newInstance(ReservationId resId,
+      SubClusterId homeSubCluster) {
+    ReservationHomeSubCluster appMapping = Records.newRecord(ReservationHomeSubCluster.class);
+    appMapping.setReservationId(resId);
+    appMapping.setHomeSubCluster(homeSubCluster);
+    return appMapping;
+  }
+
+  /**
+   * Get the {@link ReservationId} representing the unique identifier of the
+   * Reservation.
+   *
+   * @return the reservation identifier
+   */
+  @Public
+  @Unstable
+  public abstract ReservationId getReservationId();
+
+  /**
+   * Set the {@link ReservationId} representing the unique identifier of the
+   * Reservation.
+   *
+   * @param resId the reservation identifier
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationId(ReservationId resId);
+
+  /**
+   * Get the {@link SubClusterId} representing the unique identifier of the home
+   * subcluster in which the reservation is mapped to.
+   *
+   * @return the home subcluster identifier
+   */
+  @Public
+  @Unstable
+  public abstract SubClusterId getHomeSubCluster();
+
+  /**
+   * Set the {@link SubClusterId} representing the unique identifier of the home
+   * subcluster in which the ReservationMaster of the reservation is running.
+   *
+   * @param subClusterId the home subcluster identifier
+   */
+  @Private
+  @Unstable
+  public abstract void setHomeSubCluster(SubClusterId subClusterId);
+
+  @Override
+  public boolean equals(Object obj) {
+
+    if (this == obj) {
+      return true;
+    }
+
+    if (obj == null) {
+      return false;
+    }
+
+    if (obj instanceof ReservationHomeSubCluster) {
+      ReservationHomeSubCluster other = (ReservationHomeSubCluster) obj;
+      return new EqualsBuilder()
+          .append(this.getReservationId(), other.getReservationId())
+          .append(this.getHomeSubCluster(), other.getHomeSubCluster())
+          .isEquals();
+    }
+
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder().
+        append(this.getReservationId()).
+        append(this.getHomeSubCluster()).
+        toHashCode();
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("ReservationHomeSubCluster: [")
+        .append("ReservationId: ").append(getReservationId()).append(", ")
+        .append("HomeSubCluster: ").append(getHomeSubCluster())
+        .append("]");
+    return sb.toString();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java
new file mode 100644
index 0000000000000..8cd80328c3fce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.nio.ByteBuffer;
+
+@Private
+@Unstable
+public abstract class RouterMasterKey {
+
+  @Private
+  @Unstable
+  public static RouterMasterKey newInstance(Integer keyId, ByteBuffer keyBytes, Long expiryDate) {
+    RouterMasterKey policy = Records.newRecord(RouterMasterKey.class);
+    policy.setKeyId(keyId);
+    policy.setKeyBytes(keyBytes);
+    policy.setExpiryDate(expiryDate);
+    return policy;
+  }
+
+  @Private
+  @Unstable
+  public static RouterMasterKey newInstance(RouterMasterKey masterKey) {
+    RouterMasterKey routerMasterKey = Records.newRecord(RouterMasterKey.class);
+    routerMasterKey.setKeyId(masterKey.getKeyId());
+    routerMasterKey.setKeyBytes(masterKey.getKeyBytes());
+    routerMasterKey.setExpiryDate(masterKey.getExpiryDate());
+    return routerMasterKey;
+  }
+
+  /**
+   * Get the keyId of the MasterKey.
+   *
+   * @return MasterKeyId.
+   */
+  @Public
+  @Unstable
+  public abstract Integer getKeyId();
+
+  /**
+   * Set the keyId of the MasterKey.
+   *
+   * @param keyId MasterKeyId.
+   */
+  @Private
+  @Unstable
+  public abstract void setKeyId(Integer keyId);
+
+  /**
+   * Get the keyBytes of the DelegationKey.
+   *
+   * @return KeyBytes of the DelegationKey.
+   */
+  @Public
+  @Unstable
+  public abstract ByteBuffer getKeyBytes();
+
+  /**
+   * Set the keyBytes of the DelegationKey.
+   *
+   * @param keyBytes KeyBytes of the DelegationKey.
+   */
+  @Private
+  @Unstable
+  public abstract void setKeyBytes(ByteBuffer keyBytes);
+
+  /**
+   * Get the ExpiryDate of the DelegationKey.
+   *
+   * @return ExpiryDate of the DelegationKey.
+   */
+  @Private
+  @Unstable
+  public abstract Long getExpiryDate();
+
+  /**
+   * Set the expiryDate of the DelegationKey.
+   *
+   * @param expiryDate expiryDate of the DelegationKey.
+   */
+  @Private
+  @Unstable
+  public abstract void setExpiryDate(Long expiryDate);
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder()
+        .append(this.getExpiryDate().longValue())
+        .append(this.getKeyId().intValue())
+        .append(getKeyBytes().array())
+        .hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+
+    if (this == obj) {
+      return true;
+    }
+
+    if (obj == null) {
+      return false;
+    }
+
+    if (obj instanceof RouterMasterKey) {
+      RouterMasterKey other = (RouterMasterKey) obj;
+      return new EqualsBuilder()
+          .append(this.getKeyId().intValue(), other.getKeyId().intValue())
+          .append(this.getExpiryDate().longValue(), other.getExpiryDate().longValue())
+          .append(this.getKeyBytes().array(), other.getKeyBytes())
+          .isEquals();
+    }
+
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("RouterMasterKey: [")
+        .append("KeyId: ").append(getKeyId()).append(", ")
+        .append("ExpiryDate: ").append(getExpiryDate()).append(", ")
+        .append("KeyBytes: ").append(getKeyBytes()).append(", ")
+        .append("]");
+    return sb.toString();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java
new file mode 100644
index 0000000000000..9be6ed01ef2b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+@Private
+@Unstable
+public abstract class RouterMasterKeyRequest {
+
+  @Private
+  @Unstable
+  public static RouterMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) {
+    RouterMasterKeyRequest request = Records.newRecord(RouterMasterKeyRequest.class);
+    request.setRouterMasterKey(routerMasterKey);
+    return request;
+  }
+
+  @Public
+  @Unstable
+  public abstract RouterMasterKey getRouterMasterKey();
+
+  @Private
+  @Unstable
+  public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java
new file mode 100644
index 0000000000000..27c0a93eef335
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+@Private
+@Unstable
+public abstract class RouterMasterKeyResponse {
+
+  @Private
+  @Unstable
+  public static RouterMasterKeyResponse newInstance(RouterMasterKey masterKey) {
+    RouterMasterKeyResponse request = Records.newRecord(RouterMasterKeyResponse.class);
+    request.setRouterMasterKey(masterKey);
+    return request;
+  }
+
+  @Public
+  @Unstable
+  public abstract RouterMasterKey getRouterMasterKey();
+
+  @Private
+  @Unstable
+  public abstract void setRouterMasterKey(RouterMasterKey masterKey);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java
new file mode 100644
index 0000000000000..85a8002c91c9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class RouterRMDTSecretManagerState {
+
+  // DTIdentifier -> renewDate
+  private Map<RMDelegationTokenIdentifier, Long> delegationTokenState = new HashMap<>();
+
+  private Set<DelegationKey> masterKeyState = new HashSet<>();
+
+  private int dtSequenceNumber = 0;
+
+  public Map<RMDelegationTokenIdentifier, Long> getTokenState() {
+    return delegationTokenState;
+  }
+
+  public Set<DelegationKey> getMasterKeyState() {
+    return masterKeyState;
+  }
+
+  public int getDTSequenceNumber() {
+    return dtSequenceNumber;
+  }
+
+  public void setDtSequenceNumber(int dtSequenceNumber) {
+    this.dtSequenceNumber = dtSequenceNumber;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java
new file mode 100644
index 0000000000000..790ee513bbd32
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+@Private
+@Unstable
+public abstract class RouterRMTokenRequest {
+
+  @Private
+  @Unstable
+  public static RouterRMTokenRequest newInstance(RouterStoreToken routerStoreToken) {
+    RouterRMTokenRequest request = Records.newRecord(RouterRMTokenRequest.class);
+    request.setRouterStoreToken(routerStoreToken);
+    return request;
+  }
+
+  @Public
+  @Unstable
+  public abstract RouterStoreToken getRouterStoreToken();
+
+  @Private
+  @Unstable
+  public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java
new file mode 100644
index 0000000000000..c629e46a048c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+@Private
+@Unstable
+public abstract class RouterRMTokenResponse {
+
+  @Private
+  @Unstable
+  public static RouterRMTokenResponse newInstance(RouterStoreToken routerStoreToken) {
+    RouterRMTokenResponse request = Records.newRecord(RouterRMTokenResponse.class);
+    request.setRouterStoreToken(routerStoreToken);
+    return request;
+  }
+
+  @Public
+  @Unstable
+  public abstract RouterStoreToken getRouterStoreToken();
+
+  @Private
+  @Unstable
+  public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java
new file mode 100644
index 0000000000000..d6eade878020f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.io.IOException;
+
+@Private
+@Unstable
+public abstract class RouterStoreToken {
+
+  @Private
+  @Unstable
+  public static RouterStoreToken newInstance(YARNDelegationTokenIdentifier identifier,
+      Long renewdate) {
+    RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class);
+    storeToken.setIdentifier(identifier);
+    storeToken.setRenewDate(renewdate);
+    return storeToken;
+  }
+
+  @Private
+  @Unstable
+  public abstract YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException;
+
+  @Private
+  @Unstable
+  public abstract void setIdentifier(YARNDelegationTokenIdentifier identifier);
+
+  @Private
+  @Unstable
+  public abstract Long getRenewDate();
+
+  @Private
+  @Unstable
+  public abstract void setRenewDate(Long renewDate);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
index fec967d86bdd9..db638c2fac53d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -43,6 +45,14 @@ public static SubClusterId newInstance(String subClusterId) {
     return id;
   }
 
+  @Private
+  @Unstable
+  public static SubClusterId newInstance(Integer subClusterId) {
+    SubClusterId id = Records.newRecord(SubClusterId.class);
+    id.setId(String.valueOf(subClusterId));
+    return id;
+  }
+
   /**
    * Get the string identifier of the <em>subcluster</em> which is unique across
    * the federated cluster. The identifier is static, i.e. preserved across
@@ -70,19 +80,26 @@ public boolean equals(Object obj) {
     if (this == obj) {
       return true;
     }
+
     if (obj == null) {
       return false;
     }
-    if (getClass() != obj.getClass()) {
-      return false;
+
+    if (obj instanceof SubClusterId) {
+      SubClusterId other = (SubClusterId) obj;
+      return new EqualsBuilder()
+          .append(this.getId(), other.getId())
+          .isEquals();
     }
-    SubClusterId other = (SubClusterId) obj;
-    return this.getId().equals(other.getId());
+
+    return false;
   }
 
   @Override
   public int hashCode() {
-    return getId().hashCode();
+    return new HashCodeBuilder().
+        append(this.getId()).
+        toHashCode();
   }
 
   @Override
@@ -96,5 +113,4 @@ public String toString() {
     sb.append(getId());
     return sb.toString();
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
index e2260a1f457fd..ad03fb09a4e85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterIdInfo.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -58,18 +60,28 @@ public SubClusterId toId() {
   }
 
   @Override
-  public boolean equals(Object other) {
-    if (other instanceof SubClusterIdInfo) {
-      if (((SubClusterIdInfo) other).id.equals(this.id)) {
-        return true;
-      }
+  public boolean equals(Object obj) {
+
+    if (this == obj) {
+      return true;
+    }
+
+    if (obj == null) {
+      return false;
     }
+
+    if (obj instanceof SubClusterIdInfo) {
+      SubClusterIdInfo other = (SubClusterIdInfo) obj;
+      return new EqualsBuilder()
+          .append(this.id, other.id)
+          .isEquals();
+    }
+
     return false;
   }
 
   @Override
   public int hashCode() {
-    return id.hashCode();
+    return new HashCodeBuilder().append(this.id).toHashCode();
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
index cbf64e6126b28..40b87c7eb094e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -43,6 +45,7 @@ public abstract class SubClusterInfo {
 
   @Private
   @Unstable
+  @SuppressWarnings("checkstyle:ParameterNumber")
   public static SubClusterInfo newInstance(SubClusterId subClusterId,
       String amRMServiceAddress, String clientRMServiceAddress,
       String rmAdminServiceAddress, String rmWebServiceAddress,
@@ -54,6 +57,7 @@ public static SubClusterInfo newInstance(SubClusterId subClusterId,
 
   @Private
   @Unstable
+  @SuppressWarnings("checkstyle:ParameterNumber")
   public static SubClusterInfo newInstance(SubClusterId subClusterId,
       String amRMServiceAddress, String clientRMServiceAddress,
       String rmAdminServiceAddress, String rmWebServiceAddress,
@@ -252,48 +256,49 @@ public static SubClusterInfo newInstance(SubClusterId subClusterId,
 
   @Override
   public String toString() {
-    return "SubClusterInfo [getSubClusterId() = " + getSubClusterId()
-        + ", getAMRMServiceAddress() = " + getAMRMServiceAddress()
-        + ", getClientRMServiceAddress() = " + getClientRMServiceAddress()
-        + ", getRMAdminServiceAddress() = " + getRMAdminServiceAddress()
-        + ", getRMWebServiceAddress() = " + getRMWebServiceAddress()
-        + ", getState() = " + getState() + ", getLastStartTime() = "
-        + getLastStartTime() + ", getCapability() = " + getCapability() + "]";
+    StringBuilder sb = new StringBuilder();
+    sb.append("SubClusterInfo: [")
+        .append("SubClusterId: ").append(getSubClusterId()).append(", ")
+        .append("AMRMServiceAddress: ").append(getAMRMServiceAddress()).append(", ")
+        .append("ClientRMServiceAddress: ").append(getClientRMServiceAddress()).append(", ")
+        .append("RMAdminServiceAddress: ").append(getRMAdminServiceAddress()).append(", ")
+        .append("RMWebServiceAddress: ").append(getRMWebServiceAddress()).append(", ")
+        .append("State: ").append(getState()).append(", ")
+        .append("LastStartTime: ").append(getLastStartTime()).append(", ")
+        .append("Capability: ").append(getCapability())
+        .append("]");
+    return sb.toString();
   }
 
   @Override
   public boolean equals(Object obj) {
+
     if (this == obj) {
       return true;
     }
+
     if (obj == null) {
       return false;
     }
+
     if (getClass() != obj.getClass()) {
       return false;
     }
-    SubClusterInfo other = (SubClusterInfo) obj;
-    if (!this.getSubClusterId().equals(other.getSubClusterId())) {
-      return false;
-    }
-    if (!this.getAMRMServiceAddress().equals(other.getAMRMServiceAddress())) {
-      return false;
-    }
-    if (!this.getClientRMServiceAddress()
-        .equals(other.getClientRMServiceAddress())) {
-      return false;
-    }
-    if (!this.getRMAdminServiceAddress()
-        .equals(other.getRMAdminServiceAddress())) {
-      return false;
-    }
-    if (!this.getRMWebServiceAddress().equals(other.getRMWebServiceAddress())) {
-      return false;
-    }
-    if (!this.getState().equals(other.getState())) {
-      return false;
+
+    if (obj instanceof SubClusterInfo) {
+      SubClusterInfo other = (SubClusterInfo) obj;
+      return new EqualsBuilder()
+          .append(this.getSubClusterId(), other.getSubClusterId())
+          .append(this.getAMRMServiceAddress(), other.getAMRMServiceAddress())
+          .append(this.getClientRMServiceAddress(), other.getClientRMServiceAddress())
+          .append(this.getRMAdminServiceAddress(), other.getRMAdminServiceAddress())
+          .append(this.getRMWebServiceAddress(), other.getRMWebServiceAddress())
+          .append(this.getState(), other.getState())
+          .append(this.getLastStartTime(), other.getLastStartTime())
+          .isEquals();
     }
-    return this.getLastStartTime() == other.getLastStartTime();
+
+    return false;
     // Capability and HeartBeat fields are not included as they are temporal
     // (i.e. timestamps), so they change during the lifetime of the same
     // sub-cluster
@@ -301,23 +306,16 @@ public boolean equals(Object obj) {
 
   @Override
   public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result
-        + ((getSubClusterId() == null) ? 0 : getSubClusterId().hashCode());
-    result = prime * result + ((getAMRMServiceAddress() == null) ? 0
-        : getAMRMServiceAddress().hashCode());
-    result = prime * result + ((getClientRMServiceAddress() == null) ? 0
-        : getClientRMServiceAddress().hashCode());
-    result = prime * result + ((getRMAdminServiceAddress() == null) ? 0
-        : getRMAdminServiceAddress().hashCode());
-    result = prime * result + ((getRMWebServiceAddress() == null) ? 0
-        : getRMWebServiceAddress().hashCode());
-    result =
-        prime * result + ((getState() == null) ? 0 : getState().hashCode());
-    result = prime * result
-        + (int) (getLastStartTime() ^ (getLastStartTime() >>> 32));
-    return result;
+
+    return new HashCodeBuilder()
+        .append(this.getSubClusterId())
+        .append(this.getAMRMServiceAddress())
+        .append(this.getClientRMServiceAddress())
+        .append(this.getRMAdminServiceAddress())
+        .append(this.getRMWebServiceAddress())
+        .append(this.getState())
+        .append(this.getLastStartTime())
+        .toHashCode();
     // Capability and HeartBeat fields are not included as they are temporal
     // (i.e. timestamps), so they change during the lifetime of the same
     // sub-cluster
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
index 817d270146fd1..822e40c384ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterPolicyConfiguration.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -127,36 +129,44 @@ public static SubClusterPolicyConfiguration newInstance(
 
   @Override
   public int hashCode() {
-    return 31 * getParams().hashCode() + getType().hashCode();
+    return new HashCodeBuilder()
+        .append(this.getType())
+        .append(this.getQueue())
+        .append(this.getParams()).
+        toHashCode();
   }
 
   @Override
   public boolean equals(Object obj) {
+
     if (this == obj) {
       return true;
     }
+
     if (obj == null) {
       return false;
     }
-    if (getClass() != obj.getClass()) {
-      return false;
-    }
-    SubClusterPolicyConfiguration other = (SubClusterPolicyConfiguration) obj;
-    if (!this.getType().equals(other.getType())) {
-      return false;
-    }
-    if (!this.getParams().equals(other.getParams())) {
-      return false;
+
+    if (obj instanceof SubClusterPolicyConfiguration) {
+      SubClusterPolicyConfiguration other = (SubClusterPolicyConfiguration) obj;
+      return new EqualsBuilder()
+          .append(this.getType(), other.getType())
+          .append(this.getQueue(), other.getQueue())
+          .append(this.getParams(), other.getParams())
+          .isEquals();
     }
-    return true;
+
+    return false;
   }
 
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
-    sb.append(getType())
-        .append(" : ")
-        .append(getParams());
+    sb.append("SubClusterPolicyConfiguration: [")
+        .append("Type: ").append(getType()).append(", ")
+        .append("Queue: ").append(getQueue()).append(", ")
+        .append("Params: ").append(getParams()).append(", ")
+        .append("]");
     return sb.toString();
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
index b30bd32fd0260..e05f17ad2a5c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterState.java
@@ -70,15 +70,15 @@ public boolean isFinal() {
   /**
    * Convert a string into {@code SubClusterState}.
    *
-   * @param x the string to convert in SubClusterState
+   * @param state the string to convert in SubClusterState
    * @return the respective {@code SubClusterState}
    */
-  public static SubClusterState fromString(String x) {
+  public static SubClusterState fromString(String state) {
     try {
-      return SubClusterState.valueOf(x);
+      return SubClusterState.valueOf(state);
     } catch (Exception e) {
-      LOG.error("Invalid SubCluster State value in the StateStore does not"
-          + " match with the YARN Federation standard.");
+      LOG.error("Invalid SubCluster State value({}) in the StateStore does not"
+          + " match with the YARN Federation standard.", state);
       return null;
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java
new file mode 100644
index 0000000000000..8410d65095537
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * The request sent by the <code>Router</code> to
+ * <code>Federation state store</code> to update the home subcluster of a newly
+ * submitted reservation.
+ *
+ * <p>
+ * The request includes the mapping details, i.e.:
+ * <ul>
+ * <li>{@code ReservationId}</li>
+ * <li>{@code SubClusterId}</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public abstract class UpdateReservationHomeSubClusterRequest {
+
+  @Private
+  @Unstable
+  public static UpdateReservationHomeSubClusterRequest newInstance(
+      ReservationHomeSubCluster reservationHomeSubCluster) {
+    UpdateReservationHomeSubClusterRequest updateReservationRequest =
+        Records.newRecord(UpdateReservationHomeSubClusterRequest.class);
+    updateReservationRequest
+        .setReservationHomeSubCluster(reservationHomeSubCluster);
+    return updateReservationRequest;
+  }
+
+  /**
+   * Get the {@link ReservationHomeSubCluster} representing the mapping of the
+   * reservation to it's home sub-cluster.
+   *
+   * @return the mapping of the reservation to it's home sub-cluster.
+   */
+  @Public
+  @Unstable
+  public abstract ReservationHomeSubCluster getReservationHomeSubCluster();
+
+  /**
+   * Set the {@link ReservationHomeSubCluster} representing the mapping of the
+   * reservation to it's home sub-cluster.
+   *
+   * @param reservationHomeSubCluster the mapping of the reservation to it's
+   *          home sub-cluster.
+   */
+  @Private
+  @Unstable
+  public abstract void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationHomeSubCluster);
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java
new file mode 100644
index 0000000000000..d6fd0a481e7ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * UpdateReservationHomeSubClusterResponse contains the answer from the
+ * {@code FederationReservationHomeSubClusterStore} to a request to register the
+ * home subcluster of a submitted reservation. Currently response is empty if
+ * the operation was successful, if not an exception reporting reason for a
+ * failure.
+ */
+@Private
+@Unstable
+public abstract class UpdateReservationHomeSubClusterResponse {
+
+  @Private
+  @Unstable
+  public static UpdateReservationHomeSubClusterResponse newInstance() {
+    UpdateReservationHomeSubClusterResponse response =
+        Records.newRecord(UpdateReservationHomeSubClusterResponse.class);
+    return response;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000000000..e6facbec9c937
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddReservationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class AddReservationHomeSubClusterRequestPBImpl
+    extends AddReservationHomeSubClusterRequest {
+
+  private AddReservationHomeSubClusterRequestProto proto =
+      AddReservationHomeSubClusterRequestProto.getDefaultInstance();
+  private AddReservationHomeSubClusterRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public AddReservationHomeSubClusterRequestPBImpl() {
+    builder = AddReservationHomeSubClusterRequestProto.newBuilder();
+  }
+
+  public AddReservationHomeSubClusterRequestPBImpl(
+      AddReservationHomeSubClusterRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public AddReservationHomeSubClusterRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = AddReservationHomeSubClusterRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationHomeSubCluster getReservationHomeSubCluster() {
+    AddReservationHomeSubClusterRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasAppSubclusterMap()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getAppSubclusterMap());
+  }
+
+  @Override
+  public void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationInfo) {
+    maybeInitBuilder();
+    if (reservationInfo == null) {
+      builder.clearAppSubclusterMap();
+      return;
+    }
+    builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo));
+  }
+
+  private ReservationHomeSubCluster convertFromProtoFormat(
+      ReservationHomeSubClusterProto sc) {
+    return new ReservationHomeSubClusterPBImpl(sc);
+  }
+
+  private ReservationHomeSubClusterProto convertToProtoFormat(
+      ReservationHomeSubCluster sc) {
+    return ((ReservationHomeSubClusterPBImpl) sc).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000000000..e64e740634dec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link AddReservationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class AddReservationHomeSubClusterResponsePBImpl
+    extends AddReservationHomeSubClusterResponse {
+
+  private AddReservationHomeSubClusterResponseProto proto =
+      AddReservationHomeSubClusterResponseProto.getDefaultInstance();
+  private AddReservationHomeSubClusterResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public AddReservationHomeSubClusterResponsePBImpl() {
+    builder = AddReservationHomeSubClusterResponseProto.newBuilder();
+  }
+
+  public AddReservationHomeSubClusterResponsePBImpl(
+      AddReservationHomeSubClusterResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = AddReservationHomeSubClusterResponseProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  public AddReservationHomeSubClusterResponseProto getProto() {
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  @Override
+  public void setHomeSubCluster(SubClusterId homeSubCluster) {
+    maybeInitBuilder();
+    if (homeSubCluster == null) {
+      builder.clearHomeSubCluster();
+      return;
+    }
+    builder.setHomeSubCluster(convertToProtoFormat(homeSubCluster));
+  }
+
+  @Override
+  public SubClusterId getHomeSubCluster() {
+    AddReservationHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder;
+
+    if (!p.hasHomeSubCluster()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getHomeSubCluster());
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) {
+    return new SubClusterIdPBImpl(sc);
+  }
+
+  private SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+    return ((SubClusterIdPBImpl) sc).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
index 05b0b62649f44..41d557ae3a9e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ApplicationHomeSubClusterPBImpl.java
@@ -44,6 +44,7 @@ public class ApplicationHomeSubClusterPBImpl extends ApplicationHomeSubCluster {
 
   private ApplicationId applicationId = null;
   private SubClusterId homeSubCluster = null;
+  private long createTime = 0L;
 
   public ApplicationHomeSubClusterPBImpl() {
     builder = ApplicationHomeSubClusterProto.newBuilder();
@@ -110,6 +111,9 @@ public String toString() {
   @Override
   public ApplicationId getApplicationId() {
     ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.applicationId != null) {
+      return this.applicationId;
+    }
     if (!p.hasApplicationId()) {
       return null;
     }
@@ -125,6 +129,7 @@ public void setApplicationId(ApplicationId applicationId) {
       return;
     }
     this.applicationId = applicationId;
+    builder.setApplicationId(convertToProtoFormat(applicationId));
   }
 
   @Override
@@ -141,12 +146,34 @@ public SubClusterId getHomeSubCluster() {
   }
 
   @Override
-  public void setHomeSubCluster(SubClusterId homeSubCluster) {
+  public void setHomeSubCluster(SubClusterId paramHomeSubCluster) {
     maybeInitBuilder();
-    if (homeSubCluster == null) {
+    if (paramHomeSubCluster == null) {
       builder.clearHomeSubCluster();
+      return;
     }
-    this.homeSubCluster = homeSubCluster;
+    this.homeSubCluster = paramHomeSubCluster;
+    builder.setHomeSubCluster(convertToProtoFormat(paramHomeSubCluster));
+  }
+
+  @Override
+  public long getCreateTime() {
+    ApplicationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.createTime != 0) {
+      return this.createTime;
+    }
+    if (!p.hasCreateTime()) {
+      return 0;
+    }
+    this.createTime = p.getCreateTime();
+    return this.createTime;
+  }
+
+  @Override
+  public void setCreateTime(long time) {
+    maybeInitBuilder();
+    this.createTime = time;
+    builder.setCreateTime(time);
   }
 
   private SubClusterId convertFromProtoFormat(SubClusterIdProto subClusterId) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000000000..b6970b04e7fca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteReservationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class DeleteReservationHomeSubClusterRequestPBImpl
+    extends DeleteReservationHomeSubClusterRequest {
+
+  private DeleteReservationHomeSubClusterRequestProto proto =
+      DeleteReservationHomeSubClusterRequestProto.getDefaultInstance();
+  private DeleteReservationHomeSubClusterRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public DeleteReservationHomeSubClusterRequestPBImpl() {
+    builder = DeleteReservationHomeSubClusterRequestProto.newBuilder();
+  }
+
+  public DeleteReservationHomeSubClusterRequestPBImpl(
+      DeleteReservationHomeSubClusterRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public DeleteReservationHomeSubClusterRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = DeleteReservationHomeSubClusterRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationId getReservationId() {
+    DeleteReservationHomeSubClusterRequestProtoOrBuilder p =
+        viaProto ? proto : builder;
+    if (!p.hasReservationId()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getReservationId());
+  }
+
+  @Override
+  public void setReservationId(ReservationId reservationId) {
+    maybeInitBuilder();
+    if (reservationId == null) {
+      builder.clearReservationId();
+      return;
+    }
+    builder.setReservationId(convertToProtoFormat(reservationId));
+  }
+
+  private ReservationId convertFromProtoFormat(ReservationIdProto appId) {
+    return new ReservationIdPBImpl(appId);
+  }
+
+  private ReservationIdProto convertToProtoFormat(ReservationId appId) {
+    return ((ReservationIdPBImpl) appId).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000000000..1bbbe40d63a6c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link DeleteReservationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class DeleteReservationHomeSubClusterResponsePBImpl
+    extends DeleteReservationHomeSubClusterResponse {
+  private DeleteReservationHomeSubClusterResponseProto proto =
+      DeleteReservationHomeSubClusterResponseProto.getDefaultInstance();
+  private DeleteReservationHomeSubClusterResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public DeleteReservationHomeSubClusterResponsePBImpl() {
+    builder = DeleteReservationHomeSubClusterResponseProto.newBuilder();
+  }
+
+  public DeleteReservationHomeSubClusterResponsePBImpl(
+      DeleteReservationHomeSubClusterResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public DeleteReservationHomeSubClusterResponseProto getProto() {
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
index 44c8ca8cc5409..857aeef0f31fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationHomeSubClusterRequestPBImpl.java
@@ -127,6 +127,7 @@ public void setApplicationId(ApplicationId applicationId) {
       return;
     }
     this.applicationId = applicationId;
+    builder.setApplicationId(convertToProtoFormat(applicationId));
   }
 
   private ApplicationId convertFromProtoFormat(ApplicationIdProto appId) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
index a3c1c1a6bb5a1..1a75044cff3b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterRequestPBImpl.java
@@ -19,10 +19,13 @@
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterRequestProtoOrBuilder;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
 
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 
 /**
  * Protocol buffer based implementation of
@@ -75,4 +78,37 @@ public String toString() {
     return TextFormat.shortDebugString(getProto());
   }
 
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = GetApplicationsHomeSubClusterRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public SubClusterId getSubClusterId() {
+    GetApplicationsHomeSubClusterRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasSubClusterId()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getSubClusterId());
+  }
+
+  @Override
+  public void setSubClusterId(SubClusterId subClusterId) {
+    maybeInitBuilder();
+    if (subClusterId == null) {
+      builder.clearSubClusterId();
+      return;
+    }
+    builder.setSubClusterId(convertToProtoFormat(subClusterId));
+  }
+
+  private SubClusterId convertFromProtoFormat(YarnServerFederationProtos.SubClusterIdProto sc) {
+    return new SubClusterIdPBImpl(sc);
+  }
+
+  private YarnServerFederationProtos.SubClusterIdProto convertToProtoFormat(SubClusterId sc) {
+    return ((SubClusterIdPBImpl) sc).getProto();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
index a53b381ae4ef6..9a58a6d57a9ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
@@ -122,6 +122,7 @@ public void setAppsHomeSubClusters(
       return;
     }
     this.appsHomeSubCluster = appsHomeSubClusters;
+    addSubClustersInfoToProto();
   }
 
   private void initSubClustersInfoList() {
@@ -132,7 +133,7 @@ private void initSubClustersInfoList() {
         viaProto ? proto : builder;
     List<ApplicationHomeSubClusterProto> subClusterInfosList =
         p.getAppSubclusterMapList();
-    appsHomeSubCluster = new ArrayList<ApplicationHomeSubCluster>();
+    appsHomeSubCluster = new ArrayList<>();
 
     for (ApplicationHomeSubClusterProto r : subClusterInfosList) {
       appsHomeSubCluster.add(convertFromProtoFormat(r));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000000000..02351fff85be2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetReservationHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetReservationHomeSubClusterRequestPBImpl
+    extends GetReservationHomeSubClusterRequest {
+
+  private GetReservationHomeSubClusterRequestProto proto =
+      GetReservationHomeSubClusterRequestProto.getDefaultInstance();
+  private GetReservationHomeSubClusterRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  private ReservationId reservationId = null;
+
+  public GetReservationHomeSubClusterRequestPBImpl() {
+    builder = GetReservationHomeSubClusterRequestProto.newBuilder();
+  }
+
+  public GetReservationHomeSubClusterRequestPBImpl(
+      GetReservationHomeSubClusterRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public GetReservationHomeSubClusterRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = GetReservationHomeSubClusterRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.reservationId != null) {
+      builder.setReservationId(convertToProtoFormat(this.reservationId));
+    }
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationId getReservationId() {
+    GetReservationHomeSubClusterRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (reservationId != null) {
+      return reservationId;
+    }
+
+    if (!p.hasReservationId()) {
+      return null;
+    }
+    this.reservationId = convertFromProtoFormat(p.getReservationId());
+    return reservationId;
+  }
+
+  @Override
+  public void setReservationId(ReservationId paramReservationId) {
+    maybeInitBuilder();
+    if (paramReservationId == null) {
+      builder.clearReservationId();
+      return;
+    }
+    this.reservationId = paramReservationId;
+    builder.setReservationId(convertToProtoFormat(paramReservationId));
+  }
+
+  private ReservationId convertFromProtoFormat(ReservationIdProto appId) {
+    return new ReservationIdPBImpl(appId);
+  }
+
+  private ReservationIdProto convertToProtoFormat(ReservationId appId) {
+    return ((ReservationIdPBImpl) appId).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000000000..f72b61b650dcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetReservationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetReservationHomeSubClusterResponsePBImpl
+    extends GetReservationHomeSubClusterResponse {
+
+  private GetReservationHomeSubClusterResponseProto proto =
+      GetReservationHomeSubClusterResponseProto.getDefaultInstance();
+  private GetReservationHomeSubClusterResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public GetReservationHomeSubClusterResponsePBImpl() {
+    builder = GetReservationHomeSubClusterResponseProto.newBuilder();
+  }
+
+  public GetReservationHomeSubClusterResponsePBImpl(
+      GetReservationHomeSubClusterResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public GetReservationHomeSubClusterResponseProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = GetReservationHomeSubClusterResponseProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationHomeSubCluster getReservationHomeSubCluster() {
+    GetReservationHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasAppSubclusterMap()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getAppSubclusterMap());
+  }
+
+  @Override
+  public void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationInfo) {
+    maybeInitBuilder();
+    if (reservationInfo == null) {
+      builder.clearAppSubclusterMap();
+      return;
+    }
+    builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo));
+  }
+
+  private ReservationHomeSubCluster convertFromProtoFormat(
+      ReservationHomeSubClusterProto sc) {
+    return new ReservationHomeSubClusterPBImpl(sc);
+  }
+
+  private ReservationHomeSubClusterProto convertToProtoFormat(
+      ReservationHomeSubCluster sc) {
+    return ((ReservationHomeSubClusterPBImpl) sc).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000000000..889941ede1b68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetReservationsHomeSubClusterRequest}.
+ */
+@Private
+@Unstable
+public class GetReservationsHomeSubClusterRequestPBImpl
+    extends GetReservationsHomeSubClusterRequest {
+
+  private GetReservationsHomeSubClusterRequestProto proto =
+      GetReservationsHomeSubClusterRequestProto.getDefaultInstance();
+  private GetReservationsHomeSubClusterRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public GetReservationsHomeSubClusterRequestPBImpl() {
+    builder = GetReservationsHomeSubClusterRequestProto.newBuilder();
+  }
+
+  public GetReservationsHomeSubClusterRequestPBImpl(
+      GetReservationsHomeSubClusterRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public GetReservationsHomeSubClusterRequestProto getProto() {
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000000000..e424c03f055b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetReservationsHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetReservationsHomeSubClusterResponsePBImpl
+    extends GetReservationsHomeSubClusterResponse {
+
+  private GetReservationsHomeSubClusterResponseProto proto =
+      GetReservationsHomeSubClusterResponseProto.getDefaultInstance();
+  private GetReservationsHomeSubClusterResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  private List<ReservationHomeSubCluster> appsHomeSubCluster;
+
+  public GetReservationsHomeSubClusterResponsePBImpl() {
+    builder = GetReservationsHomeSubClusterResponseProto.newBuilder();
+  }
+
+  public GetReservationsHomeSubClusterResponsePBImpl(
+      GetReservationsHomeSubClusterResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public GetReservationsHomeSubClusterResponseProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = GetReservationsHomeSubClusterResponseProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.appsHomeSubCluster != null) {
+      addSubClustersInfoToProto();
+    }
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public List<ReservationHomeSubCluster> getAppsHomeSubClusters() {
+    initSubClustersInfoList();
+    return appsHomeSubCluster;
+  }
+
+  @Override
+  public void setAppsHomeSubClusters(
+      List<ReservationHomeSubCluster> appsHomeSubClusters) {
+    maybeInitBuilder();
+    if (appsHomeSubClusters == null) {
+      builder.clearAppSubclusterMap();
+      return;
+    }
+    this.appsHomeSubCluster = appsHomeSubClusters;
+    addSubClustersInfoToProto();
+  }
+
+  private void initSubClustersInfoList() {
+    if (this.appsHomeSubCluster != null) {
+      return;
+    }
+    GetReservationsHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder;
+    List<ReservationHomeSubClusterProto> subClusterInfosList = p.getAppSubclusterMapList();
+    appsHomeSubCluster = new ArrayList<>();
+
+    for (ReservationHomeSubClusterProto r : subClusterInfosList) {
+      appsHomeSubCluster.add(convertFromProtoFormat(r));
+    }
+  }
+
+  private void addSubClustersInfoToProto() {
+    maybeInitBuilder();
+    builder.clearAppSubclusterMap();
+    if (appsHomeSubCluster == null) {
+      return;
+    }
+    Iterable<ReservationHomeSubClusterProto> iterable =
+        new Iterable<ReservationHomeSubClusterProto>() {
+          @Override
+          public Iterator<ReservationHomeSubClusterProto> iterator() {
+            return new Iterator<ReservationHomeSubClusterProto>() {
+
+              private Iterator<ReservationHomeSubCluster> iter = appsHomeSubCluster.iterator();
+
+              @Override
+              public boolean hasNext() {
+                return iter.hasNext();
+              }
+
+              @Override
+              public ReservationHomeSubClusterProto next() {
+                return convertToProtoFormat(iter.next());
+              }
+
+              @Override
+              public void remove() {
+                throw new UnsupportedOperationException();
+              }
+            };
+          }
+        };
+    builder.addAllAppSubclusterMap(iterable);
+  }
+
+  private ReservationHomeSubCluster convertFromProtoFormat(ReservationHomeSubClusterProto sc) {
+    return new ReservationHomeSubClusterPBImpl(sc);
+  }
+
+  private ReservationHomeSubClusterProto convertToProtoFormat(ReservationHomeSubCluster sc) {
+    return ((ReservationHomeSubClusterPBImpl) sc).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
index 1647d06aaa9b6..bedd295dc65a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterInfoResponsePBImpl.java
@@ -114,12 +114,14 @@ public SubClusterInfo getSubClusterInfo() {
   }
 
   @Override
-  public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
+  public void setSubClusterInfo(SubClusterInfo paramSubClusterInfo) {
     maybeInitBuilder();
-    if (subClusterInfo == null) {
+    if (paramSubClusterInfo == null) {
       builder.clearSubClusterInfo();
+      return;
     }
-    this.subClusterInfo = subClusterInfo;
+    this.subClusterInfo = paramSubClusterInfo;
+    builder.setSubClusterInfo(convertToProtoFormat(paramSubClusterInfo));
   }
 
   private SubClusterInfo convertFromProtoFormat(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
index 67d5995bba8e0..458a165632e5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPoliciesConfigurationsResponsePBImpl.java
@@ -123,6 +123,7 @@ public void setPoliciesConfigs(
       builder.clearPoliciesConfigurations();
     }
     this.subClusterPolicies = policyConfigurations;
+    addSubClusterPoliciesConfigurationsToProto();
   }
 
   private void initSubClusterPoliciesConfigurationsList() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
index 5b54f262cd492..6ccd4d9d7012b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClusterPolicyConfigurationResponsePBImpl.java
@@ -126,8 +126,10 @@ public void setPolicyConfiguration(
     maybeInitBuilder();
     if (policyConfiguration == null) {
       builder.clearPolicyConfiguration();
+      return;
     }
     this.subClusterPolicy = policyConfiguration;
+    this.builder.setPolicyConfiguration(convertToProtoFormat(policyConfiguration));
   }
 
   private SubClusterPolicyConfiguration convertFromProtoFormat(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
index 5ecc3e249b2fc..b6029315bfb1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java
@@ -19,8 +19,10 @@
 package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -93,12 +95,13 @@ public List<SubClusterInfo> getSubClusters() {
   }
 
   @Override
-  public void setSubClusters(List<SubClusterInfo> subClusters) {
+  public void setSubClusters(Collection<SubClusterInfo> subClusters) {
     if (subClusters == null) {
       builder.clearSubClusterInfos();
       return;
     }
-    this.subClusterInfos = subClusters;
+    this.subClusterInfos = subClusters.stream().collect(Collectors.toList());
+    addSubClusterInfosToProto();
   }
 
   private void initSubClustersInfoList() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java
new file mode 100644
index 0000000000000..5a04e73f8709b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link ReservationHomeSubCluster}.
+ */
+@Private
+@Unstable
+public class ReservationHomeSubClusterPBImpl extends ReservationHomeSubCluster {
+
+  private ReservationHomeSubClusterProto proto =
+      ReservationHomeSubClusterProto.getDefaultInstance();
+  private ReservationHomeSubClusterProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  private ReservationId reservationId = null;
+  private SubClusterId homeSubCluster = null;
+
+  public ReservationHomeSubClusterPBImpl() {
+    builder = ReservationHomeSubClusterProto.newBuilder();
+  }
+
+  public ReservationHomeSubClusterPBImpl(ReservationHomeSubClusterProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public ReservationHomeSubClusterProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = ReservationHomeSubClusterProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.reservationId != null) {
+      builder.setReservationId(convertToProtoFormat(this.reservationId));
+    }
+    if (this.homeSubCluster != null) {
+      builder.setHomeSubCluster(convertToProtoFormat(this.homeSubCluster));
+    }
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationId getReservationId() {
+    ReservationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasReservationId()) {
+      return null;
+    }
+    this.reservationId = convertFromProtoFormat(p.getReservationId());
+    return this.reservationId;
+  }
+
+  @Override
+  public void setReservationId(ReservationId resId) {
+    maybeInitBuilder();
+    if (resId == null) {
+      builder.clearReservationId();
+      return;
+    }
+    builder.setReservationId(convertToProtoFormat(resId));
+    this.reservationId = resId;
+  }
+
+  @Override
+  public SubClusterId getHomeSubCluster() {
+    ReservationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.homeSubCluster != null) {
+      return this.homeSubCluster;
+    }
+    if (!p.hasHomeSubCluster()) {
+      return null;
+    }
+    this.homeSubCluster = convertFromProtoFormat(p.getHomeSubCluster());
+    return this.homeSubCluster;
+  }
+
+  @Override
+  public void setHomeSubCluster(SubClusterId subClusterId) {
+    maybeInitBuilder();
+    if (subClusterId == null) {
+      builder.clearHomeSubCluster();
+      return;
+    }
+    this.homeSubCluster = subClusterId;
+  }
+
+  private SubClusterId convertFromProtoFormat(SubClusterIdProto subClusterId) {
+    return new SubClusterIdPBImpl(subClusterId);
+  }
+
+  private SubClusterIdProto convertToProtoFormat(SubClusterId subClusterId) {
+    return ((SubClusterIdPBImpl) subClusterId).getProto();
+  }
+
+  private ReservationId convertFromProtoFormat(ReservationIdProto appId) {
+    return new ReservationIdPBImpl(appId);
+  }
+
+  private ReservationIdProto convertToProtoFormat(ReservationId appId) {
+    return ((ReservationIdPBImpl) appId).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java
new file mode 100644
index 0000000000000..e2adccc955205
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+
+import java.nio.ByteBuffer;
+
+public class RouterMasterKeyPBImpl extends RouterMasterKey {
+
+  private RouterMasterKeyProto proto = RouterMasterKeyProto.getDefaultInstance();
+  private RouterMasterKeyProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public RouterMasterKeyPBImpl() {
+    builder = RouterMasterKeyProto.newBuilder();
+  }
+
+  public RouterMasterKeyPBImpl(RouterMasterKeyProto masterKeyProto) {
+    this.proto = masterKeyProto;
+    viaProto = true;
+  }
+
+  public RouterMasterKeyProto getProto() {
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterMasterKeyProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public Integer getKeyId() {
+    RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getKeyId();
+  }
+
+  @Override
+  public void setKeyId(Integer keyId) {
+    maybeInitBuilder();
+    if (keyId == null) {
+      builder.clearKeyId();
+      return;
+    }
+    builder.setKeyId(keyId);
+  }
+
+  @Override
+  public ByteBuffer getKeyBytes() {
+    RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder;
+    return convertFromProtoFormat(p.getKeyBytes());
+  }
+
+  @Override
+  public void setKeyBytes(ByteBuffer keyBytes) {
+    maybeInitBuilder();
+    if (keyBytes == null) {
+      builder.clearKeyBytes();
+      return;
+    }
+    builder.setKeyBytes(convertToProtoFormat(keyBytes));
+  }
+
+  @Override
+  public Long getExpiryDate() {
+    RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getExpiryDate();
+  }
+
+  @Override
+  public void setExpiryDate(Long expiryDate) {
+    maybeInitBuilder();
+    if (expiryDate == null) {
+      builder.clearExpiryDate();
+      return;
+    }
+    builder.setExpiryDate(expiryDate);
+  }
+
+  protected final ByteBuffer convertFromProtoFormat(ByteString byteString) {
+    return ProtoUtils.convertFromProtoFormat(byteString);
+  }
+
+  protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) {
+    return ProtoUtils.convertToProtoFormat(byteBuffer);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java
new file mode 100644
index 0000000000000..44d1e4ecb940f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProtoOrBuilder;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class RouterMasterKeyRequestPBImpl extends RouterMasterKeyRequest {
+
+  private RouterMasterKeyRequestProto proto = RouterMasterKeyRequestProto.getDefaultInstance();
+  private RouterMasterKeyRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+  private RouterMasterKey routerMasterKey = null;
+
+  public RouterMasterKeyRequestPBImpl() {
+    builder = RouterMasterKeyRequestProto.newBuilder();
+  }
+
+  public RouterMasterKeyRequestPBImpl(RouterMasterKeyRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public RouterMasterKeyRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterMasterKeyRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey;
+    RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey();
+    if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) {
+      builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey));
+    }
+  }
+
+  @Override
+  public RouterMasterKey getRouterMasterKey() {
+    RouterMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.routerMasterKey != null) {
+      return this.routerMasterKey;
+    }
+    if (!p.hasRouterMasterKey()) {
+      return null;
+    }
+    this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey());
+    return this.routerMasterKey;
+  }
+
+  @Override
+  public void setRouterMasterKey(RouterMasterKey masterKey) {
+    maybeInitBuilder();
+    if (masterKey == null) {
+      builder.clearRouterMasterKey();
+      return;
+    }
+    this.routerMasterKey = masterKey;
+    builder.setRouterMasterKey(convertToProtoFormat(masterKey));
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) {
+    return new RouterMasterKeyPBImpl(masterKeyProto);
+  }
+
+  private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) {
+    return ((RouterMasterKeyPBImpl) masterKey).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java
new file mode 100644
index 0000000000000..26ca8a0f8b3ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+
+@Private
+@Unstable
+public class RouterMasterKeyResponsePBImpl extends RouterMasterKeyResponse {
+
+  private RouterMasterKeyResponseProto proto = RouterMasterKeyResponseProto.getDefaultInstance();
+  private RouterMasterKeyResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+  private RouterMasterKey routerMasterKey = null;
+
+  public RouterMasterKeyResponsePBImpl() {
+    builder = RouterMasterKeyResponseProto.newBuilder();
+  }
+
+  public RouterMasterKeyResponsePBImpl(RouterMasterKeyResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public RouterMasterKeyResponseProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterMasterKeyResponseProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey;
+    RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey();
+    if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) {
+      builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey));
+    }
+  }
+
+  @Override
+  public RouterMasterKey getRouterMasterKey() {
+    RouterMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.routerMasterKey != null) {
+      return this.routerMasterKey;
+    }
+    if (!p.hasRouterMasterKey()) {
+      return null;
+    }
+    this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey());
+    return this.routerMasterKey;
+  }
+
+  @Override
+  public void setRouterMasterKey(RouterMasterKey masterKey) {
+    maybeInitBuilder();
+    if (masterKey == null) {
+      builder.clearRouterMasterKey();
+      return;
+    }
+    this.routerMasterKey = masterKey;
+    builder.setRouterMasterKey(convertToProtoFormat(masterKey));
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) {
+    return new RouterMasterKeyPBImpl(masterKeyProto);
+  }
+
+  private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) {
+    return ((RouterMasterKeyPBImpl) masterKey).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java
new file mode 100644
index 0000000000000..1358a78326ff8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+
+public class RouterRMTokenRequestPBImpl extends RouterRMTokenRequest {
+
+  private RouterRMTokenRequestProto proto = RouterRMTokenRequestProto.getDefaultInstance();
+  private RouterRMTokenRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+  private RouterStoreToken routerStoreToken = null;
+
+  public RouterRMTokenRequestPBImpl() {
+    builder = RouterRMTokenRequestProto.newBuilder();
+  }
+
+  public RouterRMTokenRequestPBImpl(RouterRMTokenRequestProto requestProto) {
+    this.proto = requestProto;
+    viaProto = true;
+  }
+
+  public RouterRMTokenRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterRMTokenRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.routerStoreToken != null) {
+      RouterStoreTokenPBImpl routerStoreTokenPBImpl =
+          (RouterStoreTokenPBImpl) this.routerStoreToken;
+      RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto();
+      if (!storeTokenProto.equals(builder.getRouterStoreToken())) {
+        builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken));
+      }
+    }
+  }
+
+  @Override
+  public RouterStoreToken getRouterStoreToken() {
+    RouterRMTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.routerStoreToken != null) {
+      return this.routerStoreToken;
+    }
+    if (!p.hasRouterStoreToken()) {
+      return null;
+    }
+    this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken());
+    return this.routerStoreToken;
+  }
+
+  @Override
+  public void setRouterStoreToken(RouterStoreToken storeToken) {
+    maybeInitBuilder();
+    if (storeToken == null) {
+      builder.clearRouterStoreToken();
+      return;
+    }
+    this.routerStoreToken = storeToken;
+    this.builder.setRouterStoreToken(convertToProtoFormat(storeToken));
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) {
+    return ((RouterStoreTokenPBImpl) storeToken).getProto();
+  }
+
+  private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) {
+    return new RouterStoreTokenPBImpl(storeTokenProto);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java
new file mode 100644
index 0000000000000..f50967d352be0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+
+@Private
+@Unstable
+public class RouterRMTokenResponsePBImpl extends RouterRMTokenResponse {
+
+  private RouterRMTokenResponseProto proto = RouterRMTokenResponseProto.getDefaultInstance();
+  private RouterRMTokenResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+  private RouterStoreToken routerStoreToken = null;
+
+  public RouterRMTokenResponsePBImpl() {
+    builder = RouterRMTokenResponseProto.newBuilder();
+  }
+
+  public RouterRMTokenResponsePBImpl(RouterRMTokenResponseProto requestProto) {
+    this.proto = requestProto;
+    viaProto = true;
+  }
+
+  public RouterRMTokenResponseProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterRMTokenResponseProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.routerStoreToken != null) {
+      RouterStoreTokenPBImpl routerStoreTokenPBImpl =
+          (RouterStoreTokenPBImpl) this.routerStoreToken;
+      RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto();
+      if (!storeTokenProto.equals(builder.getRouterStoreToken())) {
+        builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken));
+      }
+    }
+  }
+
+  @Override
+  public RouterStoreToken getRouterStoreToken() {
+    RouterRMTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.routerStoreToken != null) {
+      return this.routerStoreToken;
+    }
+    if (!p.hasRouterStoreToken()) {
+      return null;
+    }
+    this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken());
+    return this.routerStoreToken;
+  }
+
+  @Override
+  public void setRouterStoreToken(RouterStoreToken storeToken) {
+    maybeInitBuilder();
+    if (storeToken == null) {
+      builder.clearRouterStoreToken();
+    }
+    this.routerStoreToken = storeToken;
+  }
+
+  private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) {
+    return ((RouterStoreTokenPBImpl) storeToken).getProto();
+  }
+
+  private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) {
+    return new RouterStoreTokenPBImpl(storeTokenProto);
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java
new file mode 100644
index 0000000000000..32e148cb5b7ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProtoOrBuilder;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+/**
+ * Protocol buffer based implementation of {@link RouterStoreToken}.
+ */
+@Private
+@Unstable
+public class RouterStoreTokenPBImpl extends RouterStoreToken {
+
+  private RouterStoreTokenProto proto = RouterStoreTokenProto.getDefaultInstance();
+
+  private RouterStoreTokenProto.Builder builder = null;
+
+  private boolean viaProto = false;
+
+  private YARNDelegationTokenIdentifier rMDelegationTokenIdentifier = null;
+  private Long renewDate;
+
+  public RouterStoreTokenPBImpl() {
+    builder = RouterStoreTokenProto.newBuilder();
+  }
+
+  public RouterStoreTokenPBImpl(RouterStoreTokenProto storeTokenProto) {
+    this.proto = storeTokenProto;
+    viaProto = true;
+  }
+
+  public RouterStoreTokenProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.rMDelegationTokenIdentifier != null) {
+      YARNDelegationTokenIdentifierProto idProto = this.rMDelegationTokenIdentifier.getProto();
+      if (!idProto.equals(builder.getTokenIdentifier())) {
+        builder.setTokenIdentifier(convertToProtoFormat(this.rMDelegationTokenIdentifier));
+      }
+    }
+
+    if (this.renewDate != null) {
+      builder.setRenewDate(this.renewDate);
+    }
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = RouterStoreTokenProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException {
+    RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder;
+    if (rMDelegationTokenIdentifier != null) {
+      return rMDelegationTokenIdentifier;
+    }
+    if(!p.hasTokenIdentifier()){
+      return null;
+    }
+    YARNDelegationTokenIdentifierProto identifierProto = p.getTokenIdentifier();
+    ByteArrayInputStream in = new ByteArrayInputStream(identifierProto.toByteArray());
+    RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier();
+    identifier.readFields(new DataInputStream(in));
+    this.rMDelegationTokenIdentifier = identifier;
+    return identifier;
+  }
+
+  @Override
+  public Long getRenewDate() {
+    RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.renewDate != null) {
+      return this.renewDate;
+    }
+    if (!p.hasRenewDate()) {
+      return null;
+    }
+    this.renewDate = p.getRenewDate();
+    return this.renewDate;
+  }
+
+  @Override
+  public void setIdentifier(YARNDelegationTokenIdentifier identifier) {
+    maybeInitBuilder();
+    if(identifier == null) {
+      builder.clearTokenIdentifier();
+      return;
+    }
+    this.rMDelegationTokenIdentifier = identifier;
+    this.builder.setTokenIdentifier(identifier.getProto());
+  }
+
+  @Override
+  public void setRenewDate(Long renewDate) {
+    maybeInitBuilder();
+    if(renewDate == null) {
+      builder.clearRenewDate();
+      return;
+    }
+    this.renewDate = renewDate;
+    this.builder.setRenewDate(renewDate);
+  }
+
+  private YARNDelegationTokenIdentifierProto convertToProtoFormat(
+      YARNDelegationTokenIdentifier delegationTokenIdentifier) {
+    return delegationTokenIdentifier.getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
index 84fb0d6720ffc..3bdf12449da6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SetSubClusterPolicyConfigurationRequestPBImpl.java
@@ -126,8 +126,10 @@ public void setPolicyConfiguration(
     maybeInitBuilder();
     if (policyConfiguration == null) {
       builder.clearPolicyConfiguration();
+      return;
     }
     this.subClusterPolicy = policyConfiguration;
+    builder.setPolicyConfiguration(convertToProtoFormat(policyConfiguration));
   }
 
   private SubClusterPolicyConfiguration convertFromProtoFormat(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
index 3725652bdfcf0..573ee0297de49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterHeartbeatRequestPBImpl.java
@@ -122,8 +122,10 @@ public void setSubClusterId(SubClusterId subClusterId) {
     maybeInitBuilder();
     if (subClusterId == null) {
       builder.clearSubClusterId();
+      return;
     }
     this.subClusterId = subClusterId;
+    builder.setSubClusterId(convertToProtoFormat(subClusterId));
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
index 04feb4a7f2ebc..cc16097f62b2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterInfoPBImpl.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.protobuf.TextFormat;
 
 /**
@@ -104,8 +104,10 @@ public void setSubClusterId(SubClusterId subClusterId) {
     maybeInitBuilder();
     if (subClusterId == null) {
       builder.clearSubClusterId();
+      return;
     }
     this.subClusterId = subClusterId;
+    builder.setSubClusterId(convertToProtoFormat(subClusterId));
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
index 306ad6599cd42..2295947d9a43e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterRegisterRequestPBImpl.java
@@ -118,8 +118,10 @@ public void setSubClusterInfo(SubClusterInfo subClusterInfo) {
     maybeInitBuilder();
     if (subClusterInfo == null) {
       builder.clearSubClusterInfo();
+      return;
     }
     this.subClusterInfo = subClusterInfo;
+    builder.setSubClusterInfo(convertToProtoFormat(subClusterInfo));
   }
 
   private SubClusterInfo convertFromProtoFormat(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java
new file mode 100644
index 0000000000000..c9d9a5aa0a646
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateReservationHomeSubClusterRequest} .
+ */
+@Private
+@Unstable
+public class UpdateReservationHomeSubClusterRequestPBImpl
+    extends UpdateReservationHomeSubClusterRequest {
+
+  private UpdateReservationHomeSubClusterRequestProto proto =
+      UpdateReservationHomeSubClusterRequestProto.getDefaultInstance();
+  private UpdateReservationHomeSubClusterRequestProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public UpdateReservationHomeSubClusterRequestPBImpl() {
+    builder = UpdateReservationHomeSubClusterRequestProto.newBuilder();
+  }
+
+  public UpdateReservationHomeSubClusterRequestPBImpl(
+      UpdateReservationHomeSubClusterRequestProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public UpdateReservationHomeSubClusterRequestProto getProto() {
+    mergeLocalToProto();
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void maybeInitBuilder() {
+    if (viaProto || builder == null) {
+      builder = UpdateReservationHomeSubClusterRequestProto.newBuilder(proto);
+    }
+    viaProto = false;
+  }
+
+  private void mergeLocalToBuilder() {
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+
+  @Override
+  public ReservationHomeSubCluster getReservationHomeSubCluster() {
+    UpdateReservationHomeSubClusterRequestProtoOrBuilder p =
+        viaProto ? proto : builder;
+    if (!p.hasAppSubclusterMap()) {
+      return null;
+    }
+    return convertFromProtoFormat(p.getAppSubclusterMap());
+  }
+
+  @Override
+  public void setReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationInfo) {
+    maybeInitBuilder();
+    if (reservationInfo == null) {
+      builder.clearAppSubclusterMap();
+      return;
+    }
+    builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo));
+  }
+
+  private ReservationHomeSubCluster convertFromProtoFormat(
+      ReservationHomeSubClusterProto sc) {
+    return new ReservationHomeSubClusterPBImpl(sc);
+  }
+
+  private ReservationHomeSubClusterProto convertToProtoFormat(
+      ReservationHomeSubCluster sc) {
+    return ((ReservationHomeSubClusterPBImpl) sc).getProto();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java
new file mode 100644
index 0000000000000..16f75480a7065
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+
+import org.apache.hadoop.thirdparty.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link UpdateReservationHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class UpdateReservationHomeSubClusterResponsePBImpl
+    extends UpdateReservationHomeSubClusterResponse {
+
+  private UpdateReservationHomeSubClusterResponseProto proto =
+      UpdateReservationHomeSubClusterResponseProto.getDefaultInstance();
+  private UpdateReservationHomeSubClusterResponseProto.Builder builder = null;
+  private boolean viaProto = false;
+
+  public UpdateReservationHomeSubClusterResponsePBImpl() {
+    builder = UpdateReservationHomeSubClusterResponseProto.newBuilder();
+  }
+
+  public UpdateReservationHomeSubClusterResponsePBImpl(
+      UpdateReservationHomeSubClusterResponseProto proto) {
+    this.proto = proto;
+    viaProto = true;
+  }
+
+  public UpdateReservationHomeSubClusterResponseProto getProto() {
+    proto = viaProto ? proto : builder.build();
+    viaProto = true;
+    return proto;
+  }
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
+  @Override
+  public String toString() {
+    return TextFormat.shortDebugString(getProto());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java
new file mode 100644
index 0000000000000..d792ca6b668ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class to validate the inputs to
+ * {@code FederationReservationHomeSubClusterStore}, allows a fail fast
+ * mechanism for invalid user inputs.
+ *
+ */
+public final class FederationReservationHomeSubClusterStoreInputValidator {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(FederationReservationHomeSubClusterStoreInputValidator.class);
+
+  private FederationReservationHomeSubClusterStoreInputValidator() {
+  }
+
+  /**
+   * Quick validation on the input to check some obvious fail conditions (fail
+   * fast). Check if the provided {@link AddReservationHomeSubClusterRequest}
+   * for adding a new reservation is valid or not.
+   *
+   * @param request the {@link AddReservationHomeSubClusterRequest} to validate against
+   * @throws FederationStateStoreInvalidInputException if the request is invalid
+   */
+  public static void validate(AddReservationHomeSubClusterRequest request)
+      throws FederationStateStoreInvalidInputException {
+    if (request == null) {
+      String message = "Missing AddReservationHomeSubCluster Request."
+          + " Please try again by specifying"
+          + " an AddReservationHomeSubCluster information.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+
+    // validate ReservationHomeSubCluster info
+    checkReservationHomeSubCluster(request.getReservationHomeSubCluster());
+  }
+
+  /**
+   * Quick validation on the input to check some obvious fail conditions (fail
+   * fast). Check if the provided {@link GetReservationHomeSubClusterRequest}
+   * for querying reservation's information is valid or not.
+   *
+   * @param request the {@link GetReservationHomeSubClusterRequest} to validate against
+   * @throws FederationStateStoreInvalidInputException if the request is invalid
+   */
+  public static void validate(GetReservationHomeSubClusterRequest request)
+      throws FederationStateStoreInvalidInputException {
+    if (request == null) {
+      String message = "Missing GetReservationHomeSubCluster Request."
+          + " Please try again by specifying an Reservation Id information.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+
+    // validate Reservation Id
+    checkReservationId(request.getReservationId());
+  }
+
+  /**
+   * Validate if the ReservationHomeSubCluster info are present or not.
+   *
+   * @param reservationHomeSubCluster the information of the Reservation to be verified
+   * @throws FederationStateStoreInvalidInputException if the SubCluster Info are invalid
+   */
+  private static void checkReservationHomeSubCluster(
+      ReservationHomeSubCluster reservationHomeSubCluster)
+      throws FederationStateStoreInvalidInputException {
+    if (reservationHomeSubCluster == null) {
+      String message = "Missing ReservationHomeSubCluster Info."
+          + " Please try again by specifying"
+          + " an ReservationHomeSubCluster information.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+
+    // validate Reservation Id
+    checkReservationId(reservationHomeSubCluster.getReservationId());
+
+    // validate subcluster Id
+    FederationMembershipStateStoreInputValidator
+        .checkSubClusterId(reservationHomeSubCluster.getHomeSubCluster());
+  }
+
+  /**
+   * Validate if the Reservation id is present or not.
+   *
+   * @param reservationId the id of the Reservation to be verified
+   * @throws FederationStateStoreInvalidInputException if the Reservation Id is invalid
+   */
+  private static void checkReservationId(ReservationId reservationId)
+      throws FederationStateStoreInvalidInputException {
+    if (reservationId == null) {
+      String message = "Missing ReservationId. Please try again by specifying an ReservationId.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+  }
+
+  /**
+   * Quick validation on the input to check some obvious fail conditions (fail
+   * fast). Check if the provided {@link UpdateReservationHomeSubClusterRequest}
+   * for updating an reservation is valid or not.
+   *
+   * @param request the {@link UpdateReservationHomeSubClusterRequest} to
+   *          validate against
+   * @throws FederationStateStoreInvalidInputException if the request is invalid
+   */
+  public static void validate(UpdateReservationHomeSubClusterRequest request)
+      throws FederationStateStoreInvalidInputException {
+    if (request == null) {
+      String message = "Missing UpdateReservationHomeSubCluster Request." +
+          " Please try again by specifying an ReservationHomeSubCluster information.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+
+    // validate ReservationHomeSubCluster info
+    checkReservationHomeSubCluster(request.getReservationHomeSubCluster());
+  }
+
+  /**
+   * Quick validation on the input to check some obvious fail conditions (fail
+   * fast). Check if the provided {@link DeleteReservationHomeSubClusterRequest}
+   * for deleting an Reservation is valid or not.
+   *
+   * @param request the {@link DeleteReservationHomeSubClusterRequest} to
+   *          validate against
+   * @throws FederationStateStoreInvalidInputException if the request is invalid
+   */
+  public static void validate(DeleteReservationHomeSubClusterRequest request)
+      throws FederationStateStoreInvalidInputException {
+    if (request == null) {
+      String message = "Missing DeleteReservationHomeSubCluster Request." +
+          " Please try again by specifying an ReservationHomeSubCluster information.";
+      LOG.warn(message);
+      throw new FederationStateStoreInvalidInputException(message);
+    }
+
+    // validate Reservation Id
+    checkReservationId(request.getReservationId());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java
index 27a4f7dba5327..f14867a0e6563 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException;
 import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException;
 import org.apache.hadoop.yarn.server.federation.store.metrics.FederationStateStoreClientMetrics;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -145,6 +146,45 @@ public static void logAndThrowStoreException(Logger log, String errMsg)
     throw new FederationStateStoreException(errMsg);
   }
 
+  /**
+   * Throws an <code>FederationStateStoreException</code> due to an error in
+   * <code>FederationStateStore</code>.
+   *
+   * @param log the logger interface
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws YarnException on failure
+   */
+  public static void logAndThrowStoreException(Logger log, String errMsgFormat, Object... args)
+      throws YarnException {
+    String errMsg = String.format(errMsgFormat, args);
+    log.error(errMsg);
+    throw new FederationStateStoreException(errMsg);
+  }
+
+
+  /**
+   * Throws an <code>FederationStateStoreException</code> due to an error in
+   * <code>FederationStateStore</code>.
+   *
+   * @param t the throwable raised in the called class.
+   * @param log the logger interface.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws YarnException on failure
+   */
+  public static void logAndThrowStoreException(
+      Throwable t, Logger log, String errMsgFormat, Object... args) throws YarnException {
+    String errMsg = String.format(errMsgFormat, args);
+    if (t != null) {
+      log.error(errMsg, t);
+      throw new FederationStateStoreException(errMsg, t);
+    } else {
+      log.error(errMsg);
+      throw new FederationStateStoreException(errMsg);
+    }
+  }
+
   /**
    * Throws an <code>FederationStateStoreInvalidInputException</code> due to an
    * error in <code>FederationStateStore</code>.
@@ -179,6 +219,44 @@ public static void logAndThrowRetriableException(Logger log, String errMsg,
     }
   }
 
+  /**
+   * Throws an <code>FederationStateStoreRetriableException</code> due to an
+   * error in <code>FederationStateStore</code>.
+   *
+   * @param t the throwable raised in the called class.
+   * @param log the logger interface.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws YarnException on failure
+   */
+  public static void logAndThrowRetriableException(
+      Throwable t, Logger log, String errMsgFormat, Object... args) throws YarnException {
+    String errMsg = String.format(errMsgFormat, args);
+    if (t != null) {
+      log.error(errMsg, t);
+      throw new FederationStateStoreRetriableException(errMsg, t);
+    } else {
+      log.error(errMsg);
+      throw new FederationStateStoreRetriableException(errMsg);
+    }
+  }
+
+  /**
+   * Throws an <code>FederationStateStoreRetriableException</code> due to an
+   * error in <code>FederationStateStore</code>.
+   *
+   * @param log the logger interface.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws YarnException on failure
+   */
+  public static void logAndThrowRetriableException(
+      Logger log, String errMsgFormat, Object... args) throws YarnException {
+    String errMsg = String.format(errMsgFormat, args);
+    log.error(errMsg);
+    throw new FederationStateStoreRetriableException(errMsg);
+  }
+
   /**
    * Sets a specific value for a specific property of
    * <code>HikariDataSource</code> SQL connections.
@@ -225,4 +303,30 @@ public static void setPassword(HikariDataSource dataSource, String password) {
       LOG.debug("NULL Credentials specified for Store connection, so ignoring");
     }
   }
+
+  /**
+   * Filter HomeSubCluster based on Filter SubCluster.
+   *
+   * @param filterSubCluster filter query conditions
+   * @param homeSubCluster homeSubCluster
+   * @return return true, if match filter conditions,
+   *         return false, if not match filter conditions.
+   */
+  public static boolean filterHomeSubCluster(SubClusterId filterSubCluster,
+      SubClusterId homeSubCluster) {
+
+    // If the filter condition is empty,
+    // it means that homeSubCluster needs to be added
+    if (filterSubCluster == null) {
+      return true;
+    }
+
+    // If the filter condition filterSubCluster is not empty,
+    // and filterSubCluster is equal to homeSubCluster, it needs to be added
+    if (filterSubCluster.equals(homeSubCluster)) {
+      return true;
+    }
+
+    return false;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationMethodWrapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationMethodWrapper.java
new file mode 100644
index 0000000000000..9b8944049a6dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationMethodWrapper.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+
+public abstract class FederationMethodWrapper {
+
+  /**
+   * List of parameters: static and dynamic values, matchings types.
+   */
+  private Object[] params;
+
+  /**
+   * List of method parameters types, matches parameters.
+   */
+  private Class<?>[] types;
+
+  /**
+   * String name of the method.
+   */
+  private String methodName;
+
+  public FederationMethodWrapper(Class<?>[] pTypes, Object... pParams)
+      throws IOException {
+    if (pParams.length != pTypes.length) {
+      throw new IOException("Invalid parameters for method.");
+    }
+    this.params = pParams;
+    this.types = Arrays.copyOf(pTypes, pTypes.length);
+  }
+
+  public Object[] getParams() {
+    return Arrays.copyOf(this.params, this.params.length);
+  }
+
+  public String getMethodName() {
+    return methodName;
+  }
+
+  public void setMethodName(String methodName) {
+    this.methodName = methodName;
+  }
+
+  /**
+   * Get the calling types for this method.
+   *
+   * @return An array of calling types.
+   */
+  public Class<?>[] getTypes() {
+    return Arrays.copyOf(this.types, this.types.length);
+  }
+
+  protected abstract <R> Collection<R> invokeConcurrent(Class<R> clazz) throws YarnException;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index fa3c8b8d9819f..fc1e442ab9cc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -18,10 +18,14 @@
 
 package org.apache.hadoop.yarn.server.federation.utils;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.Random;
 
 import javax.cache.Cache;
 import javax.cache.CacheManager;
@@ -36,24 +40,34 @@
 import javax.cache.integration.CacheLoaderException;
 import javax.cache.spi.CachingProvider;
 
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.NotImplementedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.retry.RetryPolicies;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryProxy;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
@@ -62,10 +76,20 @@
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,10 +110,14 @@ public final class FederationStateStoreFacade {
   private static final String GET_SUBCLUSTERS_CACHEID = "getSubClusters";
   private static final String GET_POLICIES_CONFIGURATIONS_CACHEID =
       "getPoliciesConfigurations";
+  private static final String GET_APPLICATION_HOME_SUBCLUSTER_CACHEID =
+      "getApplicationHomeSubCluster";
 
   private static final FederationStateStoreFacade FACADE =
       new FederationStateStoreFacade();
 
+  private static Random rand = new Random(System.currentTimeMillis());
+
   private FederationStateStore stateStore;
   private int cacheTimeToLive;
   private Configuration conf;
@@ -376,10 +404,19 @@ public void updateApplicationHomeSubCluster(
    */
   public SubClusterId getApplicationHomeSubCluster(ApplicationId appId)
       throws YarnException {
-    GetApplicationHomeSubClusterResponse response =
-        stateStore.getApplicationHomeSubCluster(
+    try {
+      if (isCachingEnabled()) {
+        SubClusterId value = SubClusterId.class.cast(
+            cache.get(buildGetApplicationHomeSubClusterRequest(appId)));
+        return value;
+      } else {
+        GetApplicationHomeSubClusterResponse response = stateStore.getApplicationHomeSubCluster(
             GetApplicationHomeSubClusterRequest.newInstance(appId));
-    return response.getApplicationHomeSubCluster().getHomeSubCluster();
+        return response.getApplicationHomeSubCluster().getHomeSubCluster();
+      }
+    } catch (Throwable ex) {
+      throw new YarnException(ex);
+    }
   }
 
   /**
@@ -400,6 +437,63 @@ public Configuration getConf() {
     return this.conf;
   }
 
+  /**
+   * Adds the home {@link SubClusterId} for the specified {@link ReservationId}.
+   *
+   * @param appHomeSubCluster the mapping of the reservation to it's home
+   *          sub-cluster
+   * @return the stored subCluster from StateStore
+   * @throws YarnException if the call to the state store is unsuccessful
+   */
+  public SubClusterId addReservationHomeSubCluster(ReservationHomeSubCluster appHomeSubCluster)
+      throws YarnException {
+    AddReservationHomeSubClusterResponse response = stateStore.addReservationHomeSubCluster(
+        AddReservationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+    return response.getHomeSubCluster();
+  }
+
+  /**
+   * Returns the home {@link SubClusterId} for the specified {@link ReservationId}.
+   *
+   * @param reservationId the identifier of the reservation
+   * @return the home subCluster identifier
+   * @throws YarnException if the call to the state store is unsuccessful
+   */
+  public SubClusterId getReservationHomeSubCluster(ReservationId reservationId)
+      throws YarnException {
+    GetReservationHomeSubClusterResponse response = stateStore.getReservationHomeSubCluster(
+         GetReservationHomeSubClusterRequest.newInstance(reservationId));
+    return response.getReservationHomeSubCluster().getHomeSubCluster();
+  }
+
+  /**
+   * Updates the home {@link SubClusterId} for the specified
+   * {@link ReservationId}.
+   *
+   * @param appHomeSubCluster the mapping of the reservation to it's home
+   *          sub-cluster
+   * @throws YarnException if the call to the state store is unsuccessful
+   */
+  public void updateReservationHomeSubCluster(ReservationHomeSubCluster appHomeSubCluster)
+      throws YarnException {
+    UpdateReservationHomeSubClusterRequest request =
+        UpdateReservationHomeSubClusterRequest.newInstance(appHomeSubCluster);
+    stateStore.updateReservationHomeSubCluster(request);
+  }
+
+  /**
+   * Delete the home {@link SubClusterId} for the specified
+   * {@link ReservationId}.
+   *
+   * @param reservationId the identifier of the reservation
+   * @throws YarnException if the call to the state store is unsuccessful
+   */
+  public void deleteReservationHomeSubCluster(ReservationId reservationId) throws YarnException {
+    DeleteReservationHomeSubClusterRequest request =
+        DeleteReservationHomeSubClusterRequest.newInstance(reservationId);
+    stateStore.deleteReservationHomeSubCluster(request);
+  }
+
   /**
    * Helper method to create instances of Object using the class name defined in
    * the configuration object. The instances creates {@link RetryProxy} using
@@ -410,6 +504,7 @@ public Configuration getConf() {
    * @param defaultValue the default implementation for fallback
    * @param type the class for which a retry proxy is required
    * @param retryPolicy the policy for retrying method call failures
+   * @param <T> The type of the instance
    * @return a retry proxy for the specified interface
    */
   public static <T> Object createRetryInstance(Configuration conf,
@@ -513,6 +608,26 @@ public Map<String, SubClusterPolicyConfiguration> invoke(
     return cacheRequest;
   }
 
+  private Object buildGetApplicationHomeSubClusterRequest(ApplicationId applicationId) {
+    final String cacheKey = buildCacheKey(getClass().getSimpleName(),
+        GET_APPLICATION_HOME_SUBCLUSTER_CACHEID, applicationId.toString());
+    CacheRequest<String, SubClusterId> cacheRequest = new CacheRequest<>(
+        cacheKey,
+        input -> {
+
+          GetApplicationHomeSubClusterRequest request =
+              GetApplicationHomeSubClusterRequest.newInstance(applicationId);
+          GetApplicationHomeSubClusterResponse response =
+              stateStore.getApplicationHomeSubCluster(request);
+
+          ApplicationHomeSubCluster appHomeSubCluster = response.getApplicationHomeSubCluster();
+          SubClusterId subClusterId = appHomeSubCluster.getHomeSubCluster();
+
+          return subClusterId;
+        });
+    return cacheRequest;
+  }
+
   protected String buildCacheKey(String typeName, String methodName,
       String argName) {
     StringBuilder buffer = new StringBuilder();
@@ -560,7 +675,7 @@ private static class CacheRequest<K, V> {
     private K key;
     private Func<K, V> func;
 
-    public CacheRequest(K key, Func<K, V> func) {
+    CacheRequest(K key, Func<K, V> func) {
       this.key = key;
       this.func = func;
     }
@@ -609,4 +724,297 @@ public boolean equals(Object obj) {
   protected interface Func<T, TResult> {
     TResult invoke(T input) throws Exception;
   }
+
+  @VisibleForTesting
+  public Cache<Object, Object> getCache() {
+    return cache;
+  }
+
+  @VisibleForTesting
+  protected Object getAppHomeSubClusterCacheRequest(ApplicationId applicationId) {
+    return buildGetApplicationHomeSubClusterRequest(applicationId);
+  }
+
+  @VisibleForTesting
+  public FederationStateStore getStateStore() {
+    return stateStore;
+  }
+
+  /**
+   * The Router Supports Store NewMasterKey (RouterMasterKey{@link RouterMasterKey}).
+   *
+   * @param newKey Key used for generating and verifying delegation tokens
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   * @return RouterMasterKeyResponse
+   */
+  public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey)
+      throws YarnException, IOException {
+    LOG.info("Storing master key with keyID {}.", newKey.getKeyId());
+    ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey());
+    RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(),
+        keyBytes, newKey.getExpiryDate());
+    RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey);
+    return stateStore.storeNewMasterKey(keyRequest);
+  }
+
+  /**
+   * The Router Supports Remove MasterKey (RouterMasterKey{@link RouterMasterKey}).
+   *
+   * @param newKey Key used for generating and verifying delegation tokens
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IOException {
+    LOG.info("Removing master key with keyID {}.", newKey.getKeyId());
+    ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey());
+    RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(),
+        keyBytes, newKey.getExpiryDate());
+    RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey);
+    stateStore.removeStoredMasterKey(keyRequest);
+  }
+
+  /**
+   * The Router Supports GetMasterKeyByDelegationKey.
+   *
+   * @param newKey Key used for generating and verifying delegation tokens
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   * @return RouterMasterKeyResponse
+   */
+  public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey)
+      throws YarnException, IOException {
+    LOG.info("Storing master key with keyID {}.", newKey.getKeyId());
+    ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey());
+    RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(),
+        keyBytes, newKey.getExpiryDate());
+    RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey);
+    return stateStore.getMasterKeyByDelegationKey(keyRequest);
+  }
+
+  /**
+   * The Router Supports Store RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}.
+   *
+   * @param identifier delegation tokens from the RM
+   * @param renewDate renewDate
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  public void storeNewToken(RMDelegationTokenIdentifier identifier,
+      long renewDate) throws YarnException, IOException {
+    LOG.info("storing RMDelegation token with sequence number: {}.",
+        identifier.getSequenceNumber());
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    stateStore.storeNewToken(request);
+  }
+
+  /**
+   * The Router Supports Update RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}.
+   *
+   * @param identifier delegation tokens from the RM
+   * @param renewDate renewDate
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  public void updateStoredToken(RMDelegationTokenIdentifier identifier,
+      long renewDate) throws YarnException, IOException {
+    LOG.info("updating RMDelegation token with sequence number: {}.",
+        identifier.getSequenceNumber());
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    stateStore.updateStoredToken(request);
+  }
+
+  /**
+   * The Router Supports Remove RMDelegationTokenIdentifier{@link RMDelegationTokenIdentifier}.
+   *
+   * @param identifier delegation tokens from the RM
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  public void removeStoredToken(RMDelegationTokenIdentifier identifier)
+      throws YarnException, IOException{
+    LOG.info("removing RMDelegation token with sequence number: {}.",
+        identifier.getSequenceNumber());
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    stateStore.removeStoredToken(request);
+  }
+
+  /**
+   * The Router Supports GetTokenByRouterStoreToken{@link RMDelegationTokenIdentifier}.
+   *
+   * @param identifier delegation tokens from the RM
+   * @return RouterStoreToken
+   * @throws YarnException if the call to the state store is unsuccessful
+   * @throws IOException An IO Error occurred
+   */
+  public RouterRMTokenResponse getTokenByRouterStoreToken(RMDelegationTokenIdentifier identifier)
+      throws YarnException, IOException {
+    LOG.info("get RouterStoreToken token with sequence number: {}.",
+        identifier.getSequenceNumber());
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    return stateStore.getTokenByRouterStoreToken(request);
+  }
+
+  /**
+   * Get the number of active cluster nodes.
+   *
+   * @return number of active cluster nodes.
+   * @throws YarnException if the call to the state store is unsuccessful.
+   */
+  public int getActiveSubClustersCount() throws YarnException {
+    Map<SubClusterId, SubClusterInfo> activeSubClusters = getSubClusters(true);
+    if (activeSubClusters == null || activeSubClusters.isEmpty()) {
+      return 0;
+    } else {
+      return activeSubClusters.size();
+    }
+  }
+
+  /**
+   * Randomly pick ActiveSubCluster.
+   * During the selection process, we will exclude SubClusters from the blacklist.
+   *
+   * @param activeSubClusters List of active subClusters.
+   * @param blackList blacklist.
+   * @return Active SubClusterId.
+   * @throws YarnException When there is no Active SubCluster,
+   * an exception will be thrown (No active SubCluster available to submit the request.)
+   */
+  public static SubClusterId getRandomActiveSubCluster(
+      Map<SubClusterId, SubClusterInfo> activeSubClusters, List<SubClusterId> blackList)
+      throws YarnException {
+
+    // Check if activeSubClusters is empty, if it is empty, we need to throw an exception
+    if (MapUtils.isEmpty(activeSubClusters)) {
+      throw new FederationPolicyException(
+          FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE);
+    }
+
+    // Change activeSubClusters to List
+    List<SubClusterId> subClusterIds = new ArrayList<>(activeSubClusters.keySet());
+
+    // If the blacklist is not empty, we need to remove all the subClusters in the blacklist
+    if (CollectionUtils.isNotEmpty(blackList)) {
+      subClusterIds.removeAll(blackList);
+    }
+
+    // Check there are still active subcluster after removing the blacklist
+    if (CollectionUtils.isEmpty(subClusterIds)) {
+      throw new FederationPolicyException(
+          FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE);
+    }
+
+    // Randomly choose a SubCluster
+    return subClusterIds.get(rand.nextInt(subClusterIds.size()));
+  }
+
+  /**
+   * Get the number of retries.
+   *
+   * @param configRetries User-configured number of retries.
+   * @return number of retries.
+   * @throws YarnException yarn exception.
+   */
+  public int getRetryNumbers(int configRetries) throws YarnException {
+    int activeSubClustersCount = getActiveSubClustersCount();
+    int actualRetryNums = Math.min(activeSubClustersCount, configRetries);
+    // Normally, we don't set a negative number for the number of retries,
+    // but if the user sets a negative number for the number of retries,
+    // we will return 0
+    if (actualRetryNums < 0) {
+      return 0;
+    }
+    return actualRetryNums;
+  }
+
+  /**
+   * Query SubClusterId By applicationId.
+   *
+   * If SubClusterId is not empty, it means it exists and returns true;
+   * if SubClusterId is empty, it means it does not exist and returns false.
+   *
+   * @param applicationId applicationId
+   * @return true, SubClusterId exists; false, SubClusterId not exists.
+   */
+  public boolean existsApplicationHomeSubCluster(ApplicationId applicationId) {
+    try {
+      SubClusterId subClusterId = getApplicationHomeSubCluster(applicationId);
+      if (subClusterId != null) {
+        return true;
+      }
+    } catch (YarnException e) {
+      LOG.warn("get homeSubCluster by applicationId = {} error.", applicationId, e);
+    }
+    return false;
+  }
+
+  /**
+   * Add ApplicationHomeSubCluster to FederationStateStore.
+   *
+   * @param applicationId applicationId.
+   * @param homeSubCluster homeSubCluster, homeSubCluster selected according to policy.
+   * @throws YarnException yarn exception.
+   */
+  public void addApplicationHomeSubCluster(ApplicationId applicationId,
+      ApplicationHomeSubCluster homeSubCluster) throws YarnException {
+    try {
+      addApplicationHomeSubCluster(homeSubCluster);
+    } catch (YarnException e) {
+      String msg = String.format(
+          "Unable to insert the ApplicationId %s into the FederationStateStore.", applicationId);
+      throw new YarnException(msg, e);
+    }
+  }
+
+  /**
+   * Update ApplicationHomeSubCluster to FederationStateStore.
+   *
+   * @param subClusterId homeSubClusterId
+   * @param applicationId applicationId.
+   * @param homeSubCluster homeSubCluster, homeSubCluster selected according to policy.
+   * @throws YarnException yarn exception.
+   */
+  public void updateApplicationHomeSubCluster(SubClusterId subClusterId,
+      ApplicationId applicationId, ApplicationHomeSubCluster homeSubCluster) throws YarnException {
+    try {
+      updateApplicationHomeSubCluster(homeSubCluster);
+    } catch (YarnException e) {
+      SubClusterId subClusterIdInStateStore = getApplicationHomeSubCluster(applicationId);
+      if (subClusterId == subClusterIdInStateStore) {
+        LOG.info("Application {} already submitted on SubCluster {}.", applicationId, subClusterId);
+      } else {
+        String msg = String.format(
+            "Unable to update the ApplicationId %s into the FederationStateStore.", applicationId);
+        throw new YarnException(msg, e);
+      }
+    }
+  }
+
+  /**
+   * Add or Update ApplicationHomeSubCluster.
+   *
+   * @param applicationId applicationId, is the id of the application.
+   * @param subClusterId homeSubClusterId, this is selected by strategy.
+   * @param retryCount number of retries.
+   * @throws YarnException yarn exception.
+   */
+  public void addOrUpdateApplicationHomeSubCluster(ApplicationId applicationId,
+      SubClusterId subClusterId, int retryCount) throws YarnException {
+    Boolean exists = existsApplicationHomeSubCluster(applicationId);
+    ApplicationHomeSubCluster appHomeSubCluster =
+        ApplicationHomeSubCluster.newInstance(applicationId, subClusterId);
+    if (!exists || retryCount == 0) {
+      // persist the mapping of applicationId and the subClusterId which has
+      // been selected as its home.
+      addApplicationHomeSubCluster(applicationId, appHomeSubCluster);
+    } else {
+      // update the mapping of applicationId and the home subClusterId to
+      // the new subClusterId we have selected.
+      updateApplicationHomeSubCluster(subClusterId, applicationId, appHomeSubCluster);
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/OpportunisticSchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/OpportunisticSchedulerMetrics.java
index d78f8adf3f4a7..2e24da6874146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/OpportunisticSchedulerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/OpportunisticSchedulerMetrics.java
@@ -63,6 +63,18 @@ public static OpportunisticSchedulerMetrics getMetrics() {
     return INSTANCE;
   }
 
+  @VisibleForTesting
+  public static void resetMetrics() {
+    synchronized (OpportunisticSchedulerMetrics.class) {
+      isInitialized.set(false);
+      INSTANCE = null;
+      MetricsSystem ms = DefaultMetricsSystem.instance();
+      if (ms != null) {
+        ms.unregisterSource("OpportunisticSchedulerMetrics");
+      }
+    }
+  }
+
   private static void registerMetrics() {
     registry = new MetricsRegistry(RECORD_INFO);
     registry.tag(RECORD_INFO, "ResourceManager");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java
deleted file mode 100644
index a67cf9304c162..0000000000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.metrics;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
index 47203b257eb4d..78ad48d7f23cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
@@ -22,13 +22,15 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.Callable;
+import java.util.HashMap;
+import java.util.Collections;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -74,6 +76,10 @@ public class UnmanagedAMPoolManager extends AbstractService {
 
   private ExecutorService threadpool;
 
+  private String dispatcherThreadName = "UnmanagedAMPoolManager-Finish-Thread";
+
+  private Thread finishApplicationThread;
+
   public UnmanagedAMPoolManager(ExecutorService threadpool) {
     super(UnmanagedAMPoolManager.class.getName());
     this.threadpool = threadpool;
@@ -94,48 +100,16 @@ protected void serviceStart() throws Exception {
    * UAMs running, force kill all of them. Do parallel kill because of
    * performance reasons.
    *
-   * TODO: move waiting for the kill to finish into a separate thread, without
-   * blocking the serviceStop.
    */
   @Override
   protected void serviceStop() throws Exception {
-    ExecutorCompletionService<KillApplicationResponse> completionService =
-        new ExecutorCompletionService<>(this.threadpool);
-    if (this.unmanagedAppMasterMap.isEmpty()) {
-      return;
-    }
-
-    // Save a local copy of the key set so that it won't change with the map
-    Set<String> addressList =
-        new HashSet<>(this.unmanagedAppMasterMap.keySet());
-    LOG.warn("Abnormal shutdown of UAMPoolManager, still {} UAMs in map",
-        addressList.size());
 
-    for (final String uamId : addressList) {
-      completionService.submit(new Callable<KillApplicationResponse>() {
-        @Override
-        public KillApplicationResponse call() throws Exception {
-          try {
-            LOG.info("Force-killing UAM id " + uamId + " for application "
-                + appIdMap.get(uamId));
-            return unmanagedAppMasterMap.remove(uamId).forceKillApplication();
-          } catch (Exception e) {
-            LOG.error("Failed to kill unmanaged application master", e);
-            return null;
-          }
-        }
-      });
+    if (!this.unmanagedAppMasterMap.isEmpty()) {
+      finishApplicationThread = new Thread(createForceFinishApplicationThread());
+      finishApplicationThread.setName(dispatcherThreadName);
+      finishApplicationThread.start();
     }
 
-    for (int i = 0; i < addressList.size(); ++i) {
-      try {
-        Future<KillApplicationResponse> future = completionService.take();
-        future.get();
-      } catch (Exception e) {
-        LOG.error("Failed to kill unmanaged application master", e);
-      }
-    }
-    this.appIdMap.clear();
     super.serviceStop();
   }
 
@@ -450,4 +424,120 @@ public void drainUAMHeartbeats() {
       uam.drainHeartbeatThread();
     }
   }
+
+  /**
+   * Complete FinishApplicationMaster interface calls in batches.
+   *
+   * @param request FinishApplicationMasterRequest
+   * @param appId application Id
+   * @return Returns the Map map,
+   *         the key is subClusterId, the value is FinishApplicationMasterResponse
+   */
+  public Map<String, FinishApplicationMasterResponse> batchFinishApplicationMaster(
+      FinishApplicationMasterRequest request, String appId) {
+
+    Map<String, FinishApplicationMasterResponse> responseMap = new HashMap<>();
+    Set<String> subClusterIds = this.unmanagedAppMasterMap.keySet();
+
+    if (subClusterIds != null && !subClusterIds.isEmpty()) {
+      ExecutorCompletionService<Map<String, FinishApplicationMasterResponse>> finishAppService =
+          new ExecutorCompletionService<>(this.threadpool);
+      LOG.info("Sending finish application request to {} sub-cluster RMs", subClusterIds.size());
+
+      for (final String subClusterId : subClusterIds) {
+        finishAppService.submit(() -> {
+          LOG.info("Sending finish application request to RM {}", subClusterId);
+          try {
+            FinishApplicationMasterResponse uamResponse =
+                finishApplicationMaster(subClusterId, request);
+            return Collections.singletonMap(subClusterId, uamResponse);
+          } catch (Throwable e) {
+            LOG.warn("Failed to finish unmanaged application master: " +
+                " RM address: {} ApplicationId: {}", subClusterId, appId, e);
+            return Collections.singletonMap(subClusterId, null);
+          }
+        });
+      }
+
+      for (int i = 0; i < subClusterIds.size(); ++i) {
+        try {
+          Future<Map<String, FinishApplicationMasterResponse>> future = finishAppService.take();
+          Map<String, FinishApplicationMasterResponse> uamResponse = future.get();
+          LOG.debug("Received finish application response from RM: {}", uamResponse.keySet());
+          responseMap.putAll(uamResponse);
+        } catch (Throwable e) {
+          LOG.warn("Failed to finish unmanaged application master: ApplicationId: {}", appId, e);
+        }
+      }
+    }
+
+    return responseMap;
+  }
+
+  Runnable createForceFinishApplicationThread() {
+    return () -> {
+
+      ExecutorCompletionService<Pair<String, KillApplicationResponse>> completionService =
+          new ExecutorCompletionService<>(threadpool);
+
+      // Save a local copy of the key set so that it won't change with the map
+      Set<String> addressList = new HashSet<>(unmanagedAppMasterMap.keySet());
+
+      LOG.warn("Abnormal shutdown of UAMPoolManager, still {} UAMs in map", addressList.size());
+
+      for (final String uamId : addressList) {
+        completionService.submit(() -> {
+          try {
+            ApplicationId appId = appIdMap.get(uamId);
+            LOG.info("Force-killing UAM id {} for application {}", uamId, appId);
+            UnmanagedApplicationManager applicationManager = unmanagedAppMasterMap.remove(uamId);
+            KillApplicationResponse response = applicationManager.forceKillApplication();
+            return Pair.of(uamId, response);
+          } catch (Exception e) {
+            LOG.error("Failed to kill unmanaged application master", e);
+            return Pair.of(uamId, null);
+          }
+        });
+      }
+
+      for (int i = 0; i < addressList.size(); ++i) {
+        try {
+          Future<Pair<String, KillApplicationResponse>> future = completionService.take();
+          Pair<String, KillApplicationResponse> pairs = future.get();
+          String uamId = pairs.getLeft();
+          ApplicationId appId = appIdMap.get(uamId);
+          KillApplicationResponse response = pairs.getRight();
+          if (response == null) {
+            throw new YarnException(
+                "Failed Force-killing UAM id " + uamId + " for application " + appId);
+          }
+          LOG.info("Force-killing UAM id = {} for application {} KillCompleted {}.",
+              uamId, appId, response.getIsKillCompleted());
+        } catch (Exception e) {
+          LOG.error("Failed to kill unmanaged application master", e);
+        }
+      }
+
+      appIdMap.clear();
+    };
+  }
+
+  public void unAttachUAM(String uamId) {
+    if (this.unmanagedAppMasterMap.containsKey(uamId)) {
+      UnmanagedApplicationManager appManager = this.unmanagedAppMasterMap.get(uamId);
+      appManager.shutDownConnections();
+    }
+    this.unmanagedAppMasterMap.remove(uamId);
+    this.appIdMap.remove(uamId);
+  }
+
+  @VisibleForTesting
+  protected Map<String, UnmanagedApplicationManager> getUnmanagedAppMasterMap() {
+    return unmanagedAppMasterMap;
+  }
+
+  @VisibleForTesting
+  protected Thread getFinishApplicationThread() {
+    return finishApplicationThread;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
index 8034794041a5e..c70a2db25f161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
@@ -69,7 +69,7 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * UnmanagedApplicationManager is used to register unmanaged application and
@@ -382,8 +382,13 @@ protected <T> T createRMProxy(Class<T> protocol, Configuration config,
   protected Token<AMRMTokenIdentifier> initializeUnmanagedAM(
       ApplicationId appId) throws IOException, YarnException {
     try {
-      UserGroupInformation appSubmitter =
-          UserGroupInformation.createRemoteUser(this.submitter);
+      UserGroupInformation appSubmitter;
+      if (UserGroupInformation.isSecurityEnabled()) {
+        appSubmitter = UserGroupInformation.createProxyUser(this.submitter,
+            UserGroupInformation.getLoginUser());
+      } else {
+        appSubmitter = UserGroupInformation.createRemoteUser(this.submitter);
+      }
       this.rmClient = createRMProxy(ApplicationClientProtocol.class, this.conf,
           appSubmitter, null);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
index 9ee68d12c7797..6a5d22affae8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
@@ -79,7 +79,7 @@ public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId,
    *
    * @param applicationId Application ID
    * @param credentials HDFS Tokens
-   * @return systemCredentialsForAppsProto SystemCredentialsForAppsProto
+   * @return systemCredentialsForAppsProto
    */
   public static SystemCredentialsForAppsProto newSystemCredentialsForAppsProto(
       ApplicationId applicationId, ByteBuffer credentials) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java
index 545f84e3232cb..a1627152d5f9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.webapp.dao;
 
 import java.util.ArrayList;
+import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -46,4 +47,7 @@ public ArrayList<ContainerInfo> getContainers() {
     return container;
   }
 
+  public void addAll(Collection<ContainerInfo> containersInfo) {
+    container.addAll(containersInfo);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduling_am_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduling_am_protocol.proto
index 07efcf735c5d7..83d1fb2bcfc65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduling_am_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/distributed_scheduling_am_protocol.proto
@@ -18,7 +18,7 @@
 
 /**
  * These .proto interfaces are public and stable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
+ * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  * for what changes are allowed for a *stable* .proto interface.
  */
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
index 114a60df87b6a..0544a26e4c5a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto
@@ -25,6 +25,7 @@ package hadoop.yarn;
 
 import "yarn_protos.proto";
 import "yarn_server_common_protos.proto";
+import "yarn_security_token.proto";
 
 message SubClusterIdProto {
   optional string id = 1;
@@ -96,6 +97,7 @@ message GetSubClustersInfoResponseProto {
 message ApplicationHomeSubClusterProto {
   optional ApplicationIdProto application_id = 1;
   optional SubClusterIdProto home_sub_cluster = 2;
+  optional int64 create_time = 3;
 }
 
 message AddApplicationHomeSubClusterRequestProto {
@@ -122,7 +124,7 @@ message GetApplicationHomeSubClusterResponseProto {
 }
 
 message GetApplicationsHomeSubClusterRequestProto {
-
+  optional SubClusterIdProto sub_cluster_id = 1;
 }
 
 message GetApplicationsHomeSubClusterResponseProto {
@@ -137,6 +139,56 @@ message DeleteApplicationHomeSubClusterRequestProto {
 message DeleteApplicationHomeSubClusterResponseProto {
 }
 
+
+// ---- reservations ----
+
+message ReservationHomeSubClusterProto {
+  optional ReservationIdProto reservation_id = 1;
+  optional SubClusterIdProto home_sub_cluster = 2;
+}
+
+message AddReservationHomeSubClusterRequestProto {
+  optional ReservationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message AddReservationHomeSubClusterResponseProto {
+  optional SubClusterIdProto home_sub_cluster = 1;
+}
+
+message UpdateReservationHomeSubClusterRequestProto {
+  optional ReservationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message UpdateReservationHomeSubClusterResponseProto {
+}
+
+message GetReservationHomeSubClusterRequestProto {
+  optional ReservationIdProto reservation_id = 1;
+}
+
+message GetReservationHomeSubClusterResponseProto {
+  optional ReservationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+message GetReservationsHomeSubClusterRequestProto {
+
+}
+
+message GetReservationsHomeSubClusterResponseProto {
+  repeated ReservationHomeSubClusterProto app_subcluster_map = 1;
+}
+
+
+message DeleteReservationHomeSubClusterRequestProto {
+  optional ReservationIdProto reservation_id = 1;
+}
+
+message DeleteReservationHomeSubClusterResponseProto {
+}
+
+
+//----- configurations ---
+
 message SubClusterPolicyConfigurationProto {
   optional string queue = 1;
   optional string type = 2;
@@ -163,4 +215,31 @@ message GetSubClusterPoliciesConfigurationsRequestProto {
 
 message GetSubClusterPoliciesConfigurationsResponseProto {
   repeated SubClusterPolicyConfigurationProto policies_configurations = 1;
- }
+}
+
+message RouterMasterKeyProto {
+  optional int32 key_id = 1;
+  optional bytes key_bytes = 2;
+  optional uint64 expiry_date = 3;
+}
+
+message RouterMasterKeyRequestProto {
+  optional RouterMasterKeyProto router_master_key = 1;
+}
+
+message RouterMasterKeyResponseProto {
+  optional RouterMasterKeyProto router_master_key = 1;
+}
+
+message RouterStoreTokenProto {
+  optional YARNDelegationTokenIdentifierProto token_identifier = 1;
+  optional int64 renew_date = 2;
+}
+
+message RouterRMTokenRequestProto {
+  optional RouterStoreTokenProto router_store_token = 1;
+}
+
+message RouterRMTokenResponseProto {
+  optional RouterStoreTokenProto router_store_token = 1;
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 93ffd2aeed284..40ddbf6b56c29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -27,6 +27,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.StandbyException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -954,4 +955,9 @@ public NodesToAttributesMappingResponse mapAttributesToNodes(
       throws YarnException, IOException {
     return null;
   }
+
+  @VisibleForTesting
+  public HashMap<ApplicationId, List<ContainerId>> getApplicationContainerIdMap() {
+    return applicationContainerIdMap;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index 249efd324b4a0..d9ebd2f1c5e92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.federation.policies;
 
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.nio.ByteBuffer;
 import java.util.HashMap;
@@ -28,20 +27,26 @@
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
 import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.junit.Test;
 
 /**
@@ -58,6 +63,9 @@ public abstract class BaseFederationPoliciesTest {
   private Random rand = new Random();
   private SubClusterId homeSubCluster;
 
+  private ReservationSubmissionRequest reservationSubmissionRequest =
+      mock(ReservationSubmissionRequest.class);
+
   @Test
   public void testReinitilialize() throws YarnException {
     FederationPolicyInitializationContext fpc =
@@ -177,11 +185,60 @@ public void setHomeSubCluster(SubClusterId homeSubCluster) {
   public void setMockActiveSubclusters(int numSubclusters) {
     for (int i = 1; i <= numSubclusters; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
+      SubClusterInfo sci = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+          System.currentTimeMillis(), "something");
       getActiveSubclusters().put(sc.toId(), sci);
     }
   }
 
+  public String generateClusterMetricsInfo(int id) {
+    long mem = 1024 * getRand().nextInt(277 * 100 - 1);
+    // plant a best cluster
+    if (id == 5) {
+      mem = 1024 * 277 * 100;
+    }
+    String clusterMetrics =
+        "{\"clusterMetrics\":{\"appsSubmitted\":65, \"appsCompleted\":64,\"appsPending\":0,"
+        + "\"appsRunning\":0, \"appsFailed\":0, \"appsKilled\":1,\"reservedMB\":0,\"availableMB\":"
+        + mem + ", \"allocatedMB\":0,\"reservedVirtualCores\":0, \"availableVirtualCores\":2216,"
+        + "\"allocatedVirtualCores\":0, \"containersAllocated\":0,\"containersReserved\":0,"
+        + "\"containersPending\":0,\"totalMB\":28364800, \"totalVirtualCores\":2216,"
+        + "\"totalNodes\":278, \"lostNodes\":1,\"unhealthyNodes\":0,\"decommissionedNodes\":0, "
+        + "\"rebootedNodes\":0, \"activeNodes\":277}}";
+    return clusterMetrics;
+  }
+
+  public FederationStateStoreFacade getMemoryFacade() throws YarnException {
+
+    // setting up a store and its facade (with caching off)
+    FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance();
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+    FederationStateStore store = new MemoryFederationStateStore();
+    store.init(conf);
+    fedFacade.reinitialize(store, conf);
+
+    for (SubClusterInfo sinfo : getActiveSubclusters().values()) {
+      store.registerSubCluster(SubClusterRegisterRequest.newInstance(sinfo));
+    }
+
+    return fedFacade;
+  }
+
+  public ReservationSubmissionRequest getReservationSubmissionRequest() {
+    return reservationSubmissionRequest;
+  }
+
+  public void setReservationSubmissionRequest(
+      ReservationSubmissionRequest reservationSubmissionRequest) {
+    this.reservationSubmissionRequest = reservationSubmissionRequest;
+  }
+
+  public void setupContext() throws YarnException {
+    FederationPolicyInitializationContext context =
+        FederationPoliciesTestUtil.initializePolicyContext2(getPolicy(),
+        getPolicyInfo(), getActiveSubclusters(), getMemoryFacade());
+    this.setFederationPolicyContext(context);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
index 52f36a40baf56..64e6068c1cb35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.util.HashSet;
 import java.util.List;
@@ -52,9 +51,9 @@ public void setUp() throws Exception {
 
     for (int i = 1; i <= 2; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
+      SubClusterInfo sci = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+          System.currentTimeMillis(), "something");
       getActiveSubclusters().put(sc.toId(), sci);
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestHomeAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestHomeAMRMProxyPolicy.java
index 1f57c1fe5d4ad..a4998e6871997 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestHomeAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestHomeAMRMProxyPolicy.java
@@ -24,7 +24,6 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.util.HashSet;
 import java.util.List;
@@ -62,9 +61,9 @@ public void setUp() throws Exception {
 
     for (int i = 0; i < NUM_SUBCLUSTERS; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
+      SubClusterInfo sci = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+          System.currentTimeMillis(), "something");
       getActiveSubclusters().put(sc.toId(), sci);
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
index 10359e4445886..b12614066fe5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
@@ -19,8 +19,6 @@
 package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -79,9 +77,9 @@ public void setUp() throws Exception {
       SubClusterIdInfo sc = new SubClusterIdInfo("subcluster" + i);
       // sub-cluster 3 is not active
       if (i != 3) {
-        SubClusterInfo sci = mock(SubClusterInfo.class);
-        when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-        when(sci.getSubClusterId()).thenReturn(sc.toId());
+        SubClusterInfo sci = SubClusterInfo.newInstance(
+            sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+            System.currentTimeMillis(), "something");
         getActiveSubclusters().put(sc.toId(), sci);
       }
 
@@ -330,11 +328,14 @@ Collections.<NodeReport> emptyList(),
    * use as the default for when nodes or racks are unknown.
    */
   private void addHomeSubClusterAsActive() {
-    SubClusterInfo sci = mock(SubClusterInfo.class);
-    when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-    when(sci.getSubClusterId()).thenReturn(getHomeSubCluster());
-    getActiveSubclusters().put(getHomeSubCluster(), sci);
-    SubClusterIdInfo sc = new SubClusterIdInfo(getHomeSubCluster().getId());
+
+    SubClusterId homeSubCluster = getHomeSubCluster();
+    SubClusterInfo sci = SubClusterInfo.newInstance(
+        homeSubCluster, "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+        System.currentTimeMillis(), "something");
+
+    getActiveSubclusters().put(homeSubCluster, sci);
+    SubClusterIdInfo sc = new SubClusterIdInfo(homeSubCluster.getId());
 
     getPolicyInfo().getRouterPolicyWeights().put(sc, 0.1f);
     getPolicyInfo().getAMRMPolicyWeights().put(sc, 0.1f);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
index 655582941f78e..8bea41ca312ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.util.HashSet;
 import java.util.List;
@@ -51,9 +50,9 @@ public void setUp() throws Exception {
 
     for (int i = 1; i <= 2; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
+      SubClusterInfo sci = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+          System.currentTimeMillis(), "something");
       getActiveSubclusters().put(sc.toId(), sci);
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
index d09ba754d55a7..b73578c6deec7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java
@@ -19,23 +19,35 @@
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
+import static org.mockito.Mockito.when;
+
 /**
  * Base class for router policies tests, tests for null input cases.
  */
@@ -115,4 +127,143 @@ public void testAllBlacklistSubcluster() throws YarnException {
       }
     }
   }
+
+  @Test
+  public void testNullReservationContext() throws Exception {
+    FederationRouterPolicy policy = ((FederationRouterPolicy) getPolicy());
+
+    LambdaTestUtils.intercept(FederationPolicyException.class,
+        "The ReservationSubmissionRequest cannot be null.",
+        () -> policy.getReservationHomeSubcluster(null));
+  }
+
+  @Test
+  public void testUnknownReservation() throws Exception {
+
+    long now = Time.now();
+    ReservationSubmissionRequest resReq = getReservationSubmissionRequest();
+    ReservationId reservationId = ReservationId.newInstance(now, 1);
+    when(resReq.getQueue()).thenReturn("queue1");
+    when(resReq.getReservationId()).thenReturn(reservationId);
+
+    // route an application that uses this app
+    ApplicationSubmissionContext applicationSubmissionContext =
+        ApplicationSubmissionContext.newInstance(
+            ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1),
+                null, false, false, 1, null, null, false);
+
+    applicationSubmissionContext.setReservationID(resReq.getReservationId());
+    FederationRouterPolicy policy = (FederationRouterPolicy) getPolicy();
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + reservationId + " does not exist",
+        () -> policy.getHomeSubcluster(applicationSubmissionContext, new ArrayList<>()));
+  }
+
+  @Test
+  public void testFollowReservation() throws YarnException {
+
+    long now = Time.now();
+    ReservationSubmissionRequest resReq = getReservationSubmissionRequest();
+    when(resReq.getQueue()).thenReturn("queue1");
+    when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1));
+
+    FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy();
+    FederationPolicyInitializationContext fdContext = getFederationPolicyContext();
+    FederationStateStoreFacade storeFacade = fdContext.getFederationStateStoreFacade();
+
+    // first we invoke a reservation placement
+    SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq);
+
+    // add this to the store
+    ReservationHomeSubCluster homeSubCluster =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen);
+    storeFacade.addReservationHomeSubCluster(homeSubCluster);
+
+    // route an application that uses this app
+    ApplicationSubmissionContext applicationSubmissionContext =
+        ApplicationSubmissionContext.newInstance(
+            ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1),
+                null, false, false, 1, null, null, false);
+
+    applicationSubmissionContext.setReservationID(resReq.getReservationId());
+    SubClusterId chosen2 = routerPolicy.getHomeSubcluster(
+        applicationSubmissionContext, Collections.emptyList());
+
+    // application follows reservation
+    Assert.assertEquals(chosen, chosen2);
+  }
+
+  @Test
+  public void testUpdateReservation() throws YarnException {
+    long now = Time.now();
+    ReservationSubmissionRequest resReq = getReservationSubmissionRequest();
+    when(resReq.getQueue()).thenReturn("queue1");
+    when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1));
+
+    // first we invoke a reservation placement
+    FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy();
+    SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq);
+
+    // add this to the store
+    FederationStateStoreFacade facade =
+        getFederationPolicyContext().getFederationStateStoreFacade();
+    ReservationHomeSubCluster subCluster =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen);
+    facade.addReservationHomeSubCluster(subCluster);
+
+    // get all activeSubClusters
+    Map<SubClusterId, SubClusterInfo> activeSubClusters = getActiveSubclusters();
+
+    // Update ReservationHomeSubCluster
+    List<SubClusterId> subClusterIds = new ArrayList<>(activeSubClusters.keySet());
+    SubClusterId chosen2 = subClusterIds.get(this.getRand().nextInt(subClusterIds.size()));
+    ReservationHomeSubCluster subCluster2 =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen2);
+    facade.updateReservationHomeSubCluster(subCluster2);
+
+    // route an application that uses this app
+    ApplicationSubmissionContext applicationSubmissionContext =
+        ApplicationSubmissionContext.newInstance(
+            ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1),
+                 null, false, false, 1, null, null, false);
+
+    applicationSubmissionContext.setReservationID(resReq.getReservationId());
+    SubClusterId chosen3 = routerPolicy.getHomeSubcluster(
+        applicationSubmissionContext, new ArrayList<>());
+
+    Assert.assertEquals(chosen2, chosen3);
+  }
+
+  @Test
+  public void testDeleteReservation() throws Exception {
+    long now = Time.now();
+    ReservationSubmissionRequest resReq = getReservationSubmissionRequest();
+    when(resReq.getQueue()).thenReturn("queue1");
+    when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1));
+
+    // first we invoke a reservation placement
+    FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy();
+    SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq);
+
+    // add this to the store
+    FederationStateStoreFacade facade =
+        getFederationPolicyContext().getFederationStateStoreFacade();
+    ReservationHomeSubCluster subCluster =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen);
+    facade.addReservationHomeSubCluster(subCluster);
+
+    // delete this to the store
+    facade.deleteReservationHomeSubCluster(resReq.getReservationId());
+
+    ApplicationSubmissionContext applicationSubmissionContext =
+        ApplicationSubmissionContext.newInstance(
+            ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1),
+                null, false, false, 1, null, null, false);
+    applicationSubmissionContext.setReservationID(resReq.getReservationId());
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + resReq.getReservationId() + " does not exist",
+        () -> routerPolicy.getHomeSubcluster(applicationSubmissionContext, new ArrayList<>()));
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
index ee3e09d2b93ed..57f0b59ffe5fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,8 +49,7 @@ public void setUp() throws Exception {
     setMockActiveSubclusters(numSubclusters);
 
     // initialize policy with context
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        getPolicyInfo(), getActiveSubclusters());
+    setupContext();
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
index 58f1b9947bd81..d5acc5c9ee4f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java
@@ -17,22 +17,34 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
-import static org.junit.Assert.fail;
-
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.mockito.Mockito.when;
+
 /**
  * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load
  * is properly considered for allocation.
@@ -46,12 +58,14 @@ public void setUp() throws Exception {
     Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
     Map<SubClusterIdInfo, Float> amrmWeights = new HashMap<>();
 
+    long now = Time.now();
+
     // simulate 20 active subclusters
     for (int i = 0; i < 20; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", i));
-      SubClusterInfo federationSubClusterInfo =
-          SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1,
-              SubClusterState.SC_RUNNING, -1, generateClusterMetricsInfo(i));
+      SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83",
+          now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i));
       getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
       float weight = getRand().nextInt(2);
       if (i == 5) {
@@ -67,12 +81,11 @@ public void setUp() throws Exception {
     getPolicyInfo().setRouterPolicyWeights(routerWeights);
     getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
 
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        getPolicyInfo(), getActiveSubclusters());
-
+    // initialize policy with context
+    setupContext();
   }
 
-  private String generateClusterMetricsInfo(int id) {
+  public String generateClusterMetricsInfo(int id) {
 
     long mem = 1024 * getRand().nextInt(277 * 100 - 1);
     // plant a best cluster
@@ -106,7 +119,7 @@ public void testLoadIsRespected() throws YarnException {
   }
 
   @Test
-  public void testIfNoSubclustersWithWeightOne() {
+  public void testIfNoSubclustersWithWeightOne() throws Exception {
     setPolicy(new LoadBasedRouterPolicy());
     setPolicyInfo(new WeightedPolicyInfo());
     Map<SubClusterIdInfo, Float> routerWeights = new HashMap<>();
@@ -123,15 +136,66 @@ public void testIfNoSubclustersWithWeightOne() {
     getPolicyInfo().setRouterPolicyWeights(routerWeights);
     getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
 
-    try {
-      FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-          getPolicyInfo(), getActiveSubclusters());
-      ((FederationRouterPolicy) getPolicy())
-          .getHomeSubcluster(getApplicationSubmissionContext(), null);
-      fail();
-    } catch (YarnException ex) {
-      Assert.assertTrue(
-          ex.getMessage().contains("Zero Active Subcluster with weight 1"));
+
+    ConfigurableFederationPolicy policy = getPolicy();
+    FederationPoliciesTestUtil.initializePolicyContext(policy,
+        getPolicyInfo(), getActiveSubclusters());
+
+    LambdaTestUtils.intercept(YarnException.class, "Zero Active Subcluster with weight 1.",
+        () ->  ((FederationRouterPolicy) policy).
+        getHomeSubcluster(getApplicationSubmissionContext(), null));
+  }
+
+  @Test
+  public void testUpdateReservation() throws YarnException {
+    long now = Time.now();
+    ReservationSubmissionRequest resReq = getReservationSubmissionRequest();
+    when(resReq.getQueue()).thenReturn("queue1");
+    when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1));
+
+    // first we invoke a reservation placement
+    FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy();
+    SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq);
+
+    // add this to the store
+    FederationStateStoreFacade facade =
+        getFederationPolicyContext().getFederationStateStoreFacade();
+    ReservationHomeSubCluster subCluster =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen);
+    facade.addReservationHomeSubCluster(subCluster);
+
+    // get all activeSubClusters
+    Map<SubClusterId, SubClusterInfo> activeSubClusters = getActiveSubclusters();
+
+    // Update ReservationHomeSubCluster
+    // Cannot be randomly selected, SubCluster with Weight >= 1.0 needs to be selected
+    WeightedPolicyInfo weightedPolicyInfo = this.getPolicyInfo();
+    Map<SubClusterIdInfo, Float> routerPolicyWeights = weightedPolicyInfo.getRouterPolicyWeights();
+
+    List<SubClusterId> subClusterIds = new ArrayList<>();
+    for (Map.Entry<SubClusterIdInfo, Float> entry : routerPolicyWeights.entrySet()) {
+      SubClusterIdInfo subClusterIdInfo = entry.getKey();
+      Float subClusterWeight = entry.getValue();
+      if (subClusterWeight >= 1.0) {
+        subClusterIds.add(subClusterIdInfo.toId());
+      }
     }
+
+    SubClusterId chosen2 = subClusterIds.get(this.getRand().nextInt(subClusterIds.size()));
+    ReservationHomeSubCluster subCluster2 =
+        ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen2);
+    facade.updateReservationHomeSubCluster(subCluster2);
+
+    // route an application that uses this app
+    ApplicationSubmissionContext applicationSubmissionContext =
+        ApplicationSubmissionContext.newInstance(
+            ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1),
+                null, false, false, 1, null, null, false);
+
+    applicationSubmissionContext.setReservationID(resReq.getReservationId());
+    SubClusterId chosen3 = routerPolicy.getHomeSubcluster(
+        applicationSubmissionContext, new ArrayList<>());
+
+    Assert.assertEquals(chosen2, chosen3);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java
index 05939329a0681..3af0d037fcae6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java
@@ -73,6 +73,7 @@ public void setUp() throws Exception {
 
     configureWeights(4);
 
+    // initialize policy with context
     initializePolicy(new YarnConfiguration());
   }
 
@@ -86,9 +87,7 @@ private void initializePolicy(Configuration conf) throws YarnException {
             .newInstance("queue1", getPolicy().getClass().getCanonicalName(),
                 buf));
     getFederationPolicyContext().setHomeSubcluster(getHomeSubCluster());
-    FederationPoliciesTestUtil
-        .initializePolicyContext(getFederationPolicyContext(), getPolicy(),
-            getPolicyInfo(), getActiveSubclusters(), conf);
+    setupContext();
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
index e1799d321083c..cf2df6d560ef3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java
@@ -17,12 +17,11 @@
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
@@ -54,10 +53,11 @@ public void setUp() throws Exception {
 
       // with 5% omit a subcluster
       if (getRand().nextFloat() < 0.95f || i == 5) {
-        SubClusterInfo sci = mock(SubClusterInfo.class);
-        when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-        when(sci.getSubClusterId()).thenReturn(sc.toId());
-        getActiveSubclusters().put(sc.toId(), sci);
+        long now = Time.now();
+        SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+            sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83",
+            now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i));
+        getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
       }
       float weight = getRand().nextFloat();
       if (i == 5) {
@@ -72,9 +72,8 @@ public void setUp() throws Exception {
     }
     getPolicyInfo().setRouterPolicyWeights(routerWeights);
     getPolicyInfo().setAMRMPolicyWeights(amrmWeights);
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        getPolicyInfo(), getActiveSubclusters());
 
+    setupContext();
   }
 
   @Test
@@ -92,9 +91,9 @@ public void testZeroSubClustersWithPositiveWeight() throws Exception {
     for (int i = 0; i < 5; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
 
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
+      SubClusterInfo sci = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING,
+          System.currentTimeMillis(), "something");
       getActiveSubclusters().put(sc.toId(), sci);
       routerWeights.put(sc, 0.0f);
       amrmWeights.put(sc, -1.0f);
@@ -105,7 +104,7 @@ public void testZeroSubClustersWithPositiveWeight() throws Exception {
         getPolicyInfo(), getActiveSubclusters());
 
     intercept(FederationPolicyException.class,
-        "No Active Subcluster with weight vector greater than zero",
+        "No Active Subcluster with weight vector greater than zero.",
         () -> ((FederationRouterPolicy) getPolicy())
             .getHomeSubcluster(getApplicationSubmissionContext(), null));
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
index 1747f73715cda..4b3b3392b4908 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Before;
 import org.junit.Test;
@@ -39,8 +38,7 @@ public void setUp() throws Exception {
     setMockActiveSubclusters(2);
 
     // initialize policy with context
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        getPolicyInfo(), getActiveSubclusters());
+    setupContext();
 
   }
 
@@ -60,4 +58,21 @@ public void testNullQueueRouting() throws YarnException {
     localPolicy.getHomeSubcluster(applicationSubmissionContext, null);
   }
 
+  @Override
+  @Test(expected = FederationPolicyException.class)
+  public void testFollowReservation() throws YarnException {
+    super.testFollowReservation();
+  }
+
+  @Override
+  @Test(expected = FederationPolicyException.class)
+  public void testUpdateReservation() throws YarnException {
+    super.testUpdateReservation();
+  }
+
+  @Override
+  @Test(expected = FederationPolicyException.class)
+  public void testDeleteReservation() throws Exception {
+    super.testDeleteReservation();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
index 05490aba67247..8346277505ecb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java
@@ -18,15 +18,14 @@
 package org.apache.hadoop.yarn.server.federation.policies.router;
 
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -44,14 +43,14 @@ public void setUp() throws Exception {
     setPolicyInfo(mock(WeightedPolicyInfo.class));
     for (int i = 1; i <= 2; i++) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
-      SubClusterInfo sci = mock(SubClusterInfo.class);
-      when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-      when(sci.getSubClusterId()).thenReturn(sc.toId());
-      getActiveSubclusters().put(sc.toId(), sci);
+      long now = Time.now();
+      SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+          sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83",
+          now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i));
+      getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
     }
 
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        mock(WeightedPolicyInfo.class), getActiveSubclusters());
+    setupContext();
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index d549250f07256..8121ce282cd6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
@@ -32,7 +33,6 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
-import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -51,8 +51,7 @@ public void setUp() throws Exception {
 
     configureWeights(20);
 
-    FederationPoliciesTestUtil.initializePolicyContext(getPolicy(),
-        getPolicyInfo(), getActiveSubclusters());
+    setupContext();
   }
 
   public void configureWeights(float numSubClusters) {
@@ -68,10 +67,11 @@ public void configureWeights(float numSubClusters) {
       SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
       // with 5% omit a subcluster
       if (getRand().nextFloat() < 0.95f) {
-        SubClusterInfo sci = mock(SubClusterInfo.class);
-        when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING);
-        when(sci.getSubClusterId()).thenReturn(sc.toId());
-        getActiveSubclusters().put(sc.toId(), sci);
+        long now = Time.now();
+        SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance(
+            sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83",
+            now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i));
+        getActiveSubclusters().put(sc.toId(), federationSubClusterInfo);
       }
 
       // 80% of the weight is evenly spread, 20% is randomly generated
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
index d0e6485b02883..3786f7cccc0c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java
@@ -21,16 +21,25 @@
 import java.nio.ByteBuffer;
 import java.util.Calendar;
 import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
 import java.util.TimeZone;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreException;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
@@ -56,6 +65,20 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
 import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.junit.After;
 import org.junit.Assert;
@@ -69,6 +92,8 @@ public abstract class FederationStateStoreBaseTest {
 
   private static final MonotonicClock CLOCK = new MonotonicClock();
   private FederationStateStore stateStore;
+  private static final int NUM_APPS_10 = 10;
+  private static final int NUM_APPS_20 = 20;
 
   protected abstract FederationStateStore createStateStore();
 
@@ -398,6 +423,89 @@ public void testGetApplicationsHomeSubCluster() throws Exception {
     Assert.assertTrue(result.getAppsHomeSubClusters().contains(ahsc2));
   }
 
+  @Test
+  public void testGetApplicationsHomeSubClusterEmpty() throws Exception {
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getApplicationsHomeSubCluster request",
+        () -> stateStore.getApplicationsHomeSubCluster(null));
+  }
+
+  @Test
+  public void testGetApplicationsHomeSubClusterFilter() throws Exception {
+    // Add ApplicationHomeSC - SC1
+    long now = Time.now();
+
+    Set<ApplicationHomeSubCluster> appHomeSubClusters = new HashSet<>();
+
+    for (int i = 0; i < NUM_APPS_10; i++) {
+      ApplicationId appId = ApplicationId.newInstance(now, i);
+      SubClusterId subClusterId = SubClusterId.newInstance("SC1");
+      addApplicationHomeSC(appId, subClusterId);
+      ApplicationHomeSubCluster ahsc =
+          ApplicationHomeSubCluster.newInstance(appId, subClusterId);
+      appHomeSubClusters.add(ahsc);
+    }
+
+    // Add ApplicationHomeSC - SC2
+    for (int i = 10; i < NUM_APPS_20; i++) {
+      ApplicationId appId = ApplicationId.newInstance(now, i);
+      SubClusterId subClusterId = SubClusterId.newInstance("SC2");
+      addApplicationHomeSC(appId, subClusterId);
+    }
+
+    GetApplicationsHomeSubClusterRequest getRequest =
+        GetApplicationsHomeSubClusterRequest.newInstance();
+    getRequest.setSubClusterId(SubClusterId.newInstance("SC1"));
+
+    GetApplicationsHomeSubClusterResponse result =
+        stateStore.getApplicationsHomeSubCluster(getRequest);
+    Assert.assertNotNull(result);
+
+    List<ApplicationHomeSubCluster> items = result.getAppsHomeSubClusters();
+    Assert.assertNotNull(items);
+    Assert.assertEquals(10, items.size());
+
+    for (ApplicationHomeSubCluster item : items) {
+      Assert.assertTrue(appHomeSubClusters.contains(item));
+    }
+  }
+
+  @Test
+  public void testGetApplicationsHomeSubClusterLimit() throws Exception {
+    // Add ApplicationHomeSC - SC1
+    long now = Time.now();
+
+    for (int i = 0; i < 50; i++) {
+      ApplicationId appId = ApplicationId.newInstance(now, i);
+      SubClusterId subClusterId = SubClusterId.newInstance("SC1");
+      addApplicationHomeSC(appId, subClusterId);
+    }
+
+    GetApplicationsHomeSubClusterRequest getRequest =
+        GetApplicationsHomeSubClusterRequest.newInstance();
+    getRequest.setSubClusterId(SubClusterId.newInstance("SC1"));
+    GetApplicationsHomeSubClusterResponse result =
+        stateStore.getApplicationsHomeSubCluster(getRequest);
+    Assert.assertNotNull(result);
+
+    // Write 50 records, but get 10 records because the maximum number is limited to 10
+    List<ApplicationHomeSubCluster> items = result.getAppsHomeSubClusters();
+    Assert.assertNotNull(items);
+    Assert.assertEquals(10, items.size());
+
+    GetApplicationsHomeSubClusterRequest getRequest1 =
+        GetApplicationsHomeSubClusterRequest.newInstance();
+    getRequest1.setSubClusterId(SubClusterId.newInstance("SC2"));
+    GetApplicationsHomeSubClusterResponse result1 =
+        stateStore.getApplicationsHomeSubCluster(getRequest1);
+    Assert.assertNotNull(result1);
+
+    // SC2 data does not exist, so the number of returned records is 0
+    List<ApplicationHomeSubCluster> items1 = result1.getAppsHomeSubClusters();
+    Assert.assertNotNull(items1);
+    Assert.assertEquals(0, items1.size());
+  }
+
   @Test
   public void testUpdateApplicationHomeSubCluster() throws Exception {
     ApplicationId appId = ApplicationId.newInstance(1, 1);
@@ -599,4 +707,327 @@ protected FederationStateStore getStateStore() {
     return stateStore;
   }
 
+  SubClusterId queryReservationHomeSC(ReservationId reservationId)
+      throws YarnException {
+
+    GetReservationHomeSubClusterRequest request =
+        GetReservationHomeSubClusterRequest.newInstance(reservationId);
+
+    GetReservationHomeSubClusterResponse response =
+        stateStore.getReservationHomeSubCluster(request);
+
+    return response.getReservationHomeSubCluster().getHomeSubCluster();
+  }
+
+  @Test
+  public void testAddReservationHomeSubCluster() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId = SubClusterId.newInstance("SC");
+
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+
+    AddReservationHomeSubClusterRequest request =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+    AddReservationHomeSubClusterResponse response =
+        stateStore.addReservationHomeSubCluster(request);
+
+    Assert.assertEquals(subClusterId, response.getHomeSubCluster());
+    Assert.assertEquals(subClusterId, queryReservationHomeSC(reservationId));
+  }
+
+  private void addReservationHomeSC(ReservationId reservationId, SubClusterId subClusterId)
+      throws YarnException {
+
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+    AddReservationHomeSubClusterRequest request =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+    stateStore.addReservationHomeSubCluster(request);
+  }
+
+  @Test
+  public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+    addReservationHomeSC(reservationId, subClusterId1);
+
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+    ReservationHomeSubCluster reservationHomeSubCluster2 =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId2);
+    AddReservationHomeSubClusterRequest request2 =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2);
+    AddReservationHomeSubClusterResponse response =
+        stateStore.addReservationHomeSubCluster(request2);
+
+    Assert.assertNotNull(response);
+    Assert.assertEquals(subClusterId1, response.getHomeSubCluster());
+    Assert.assertEquals(subClusterId1, queryReservationHomeSC(reservationId));
+  }
+
+  @Test
+  public void testAddReservationHomeSubClusterAppAlreadyExistsInTheSameSC()
+      throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+    addReservationHomeSC(reservationId, subClusterId1);
+
+    ReservationHomeSubCluster reservationHomeSubCluster2 =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId1);
+    AddReservationHomeSubClusterRequest request2 =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2);
+    AddReservationHomeSubClusterResponse response =
+        stateStore.addReservationHomeSubCluster(request2);
+
+    Assert.assertNotNull(response);
+    Assert.assertEquals(subClusterId1, response.getHomeSubCluster());
+    Assert.assertEquals(subClusterId1, queryReservationHomeSC(reservationId));
+  }
+
+  @Test
+  public void testDeleteReservationHomeSubCluster() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC");
+    addReservationHomeSC(reservationId, subClusterId1);
+
+    DeleteReservationHomeSubClusterRequest delReservationRequest =
+        DeleteReservationHomeSubClusterRequest.newInstance(reservationId);
+    DeleteReservationHomeSubClusterResponse delReservationResponse =
+        stateStore.deleteReservationHomeSubCluster(delReservationRequest);
+
+    Assert.assertNotNull(delReservationResponse);
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + reservationId + " does not exist",
+        () -> queryReservationHomeSC(reservationId));
+  }
+
+  @Test
+  public void testDeleteReservationHomeSubClusterUnknownApp() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+
+    DeleteReservationHomeSubClusterRequest delReservationRequest =
+        DeleteReservationHomeSubClusterRequest.newInstance(reservationId);
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + reservationId + " does not exist",
+        () -> stateStore.deleteReservationHomeSubCluster(delReservationRequest));
+  }
+
+  @Test
+  public void testUpdateReservationHomeSubCluster() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC");
+    addReservationHomeSC(reservationId, subClusterId1);
+
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId2);
+
+    UpdateReservationHomeSubClusterRequest updateReservationRequest =
+        UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+
+    UpdateReservationHomeSubClusterResponse updateReservationResponse =
+        stateStore.updateReservationHomeSubCluster(updateReservationRequest);
+
+    Assert.assertNotNull(updateReservationResponse);
+    Assert.assertEquals(subClusterId2, queryReservationHomeSC(reservationId));
+  }
+
+  @Test
+  public void testUpdateReservationHomeSubClusterUnknownApp() throws Exception {
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC1");
+
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId1);
+
+    UpdateReservationHomeSubClusterRequest updateReservationRequest =
+        UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + reservationId + " does not exist",
+        () -> stateStore.updateReservationHomeSubCluster(updateReservationRequest));
+  }
+
+  @Test
+  public void testStoreNewMasterKey() throws Exception {
+    // store delegation key;
+    DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    Set<DelegationKey> keySet = new HashSet<>();
+    keySet.add(key);
+
+    RouterMasterKey routerMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+        ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+    RouterMasterKeyRequest routerMasterKeyRequest =
+        RouterMasterKeyRequest.newInstance(routerMasterKey);
+    RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+
+    Assert.assertNotNull(response);
+    RouterMasterKey routerMasterKeyResp = response.getRouterMasterKey();
+    Assert.assertNotNull(routerMasterKeyResp);
+    Assert.assertEquals(routerMasterKey.getKeyId(), routerMasterKeyResp.getKeyId());
+    Assert.assertEquals(routerMasterKey.getKeyBytes(), routerMasterKeyResp.getKeyBytes());
+    Assert.assertEquals(routerMasterKey.getExpiryDate(), routerMasterKeyResp.getExpiryDate());
+  }
+
+  @Test
+  public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
+    // store delegation key;
+    DelegationKey key = new DelegationKey(5678, 8765, "keyBytes".getBytes());
+    Set<DelegationKey> keySet = new HashSet<>();
+    keySet.add(key);
+
+    RouterMasterKey routerMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+        ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+    RouterMasterKeyRequest routerMasterKeyRequest =
+        RouterMasterKeyRequest.newInstance(routerMasterKey);
+    RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+    Assert.assertNotNull(response);
+
+    RouterMasterKeyResponse routerMasterKeyResponse =
+        stateStore.getMasterKeyByDelegationKey(routerMasterKeyRequest);
+
+    Assert.assertNotNull(routerMasterKeyResponse);
+
+    RouterMasterKey routerMasterKeyResp = routerMasterKeyResponse.getRouterMasterKey();
+    Assert.assertNotNull(routerMasterKeyResp);
+    Assert.assertEquals(routerMasterKey.getKeyId(), routerMasterKeyResp.getKeyId());
+    Assert.assertEquals(routerMasterKey.getKeyBytes(), routerMasterKeyResp.getKeyBytes());
+    Assert.assertEquals(routerMasterKey.getExpiryDate(), routerMasterKeyResp.getExpiryDate());
+  }
+
+  @Test
+  public void testRemoveStoredMasterKey() throws YarnException, IOException {
+    // store delegation key;
+    DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    Set<DelegationKey> keySet = new HashSet<>();
+    keySet.add(key);
+
+    RouterMasterKey routerMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+        ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+    RouterMasterKeyRequest routerMasterKeyRequest =
+        RouterMasterKeyRequest.newInstance(routerMasterKey);
+    RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+    Assert.assertNotNull(response);
+
+    RouterMasterKeyResponse masterKeyResponse =
+        stateStore.removeStoredMasterKey(routerMasterKeyRequest);
+    Assert.assertNotNull(masterKeyResponse);
+
+    RouterMasterKey routerMasterKeyResp = masterKeyResponse.getRouterMasterKey();
+    Assert.assertEquals(routerMasterKey.getKeyId(), routerMasterKeyResp.getKeyId());
+    Assert.assertEquals(routerMasterKey.getKeyBytes(), routerMasterKeyResp.getKeyBytes());
+    Assert.assertEquals(routerMasterKey.getExpiryDate(), routerMasterKeyResp.getExpiryDate());
+  }
+
+  @Test
+  public void testStoreNewToken() throws IOException, YarnException {
+    // prepare parameters
+    RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1;
+    identifier.setSequenceNumber(sequenceNumber);
+    Long renewDate = Time.now();
+
+    // store new rm-token
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+
+    // Verify the returned result to ensure that the returned Response is not empty
+    // and the returned result is consistent with the input parameters.
+    Assert.assertNotNull(routerRMTokenResponse);
+    RouterStoreToken storeTokenResp = routerRMTokenResponse.getRouterStoreToken();
+    Assert.assertNotNull(storeTokenResp);
+    Assert.assertEquals(storeToken.getRenewDate(), storeTokenResp.getRenewDate());
+    Assert.assertEquals(storeToken.getTokenIdentifier(), storeTokenResp.getTokenIdentifier());
+  }
+
+  @Test
+  public void testUpdateStoredToken() throws IOException, YarnException {
+    // prepare saveToken parameters
+    RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+        new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+    int sequenceNumber = 2;
+    identifier.setSequenceNumber(sequenceNumber);
+    Long renewDate = Time.now();
+
+    // store new rm-token
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+    Assert.assertNotNull(routerRMTokenResponse);
+
+    // prepare updateToken parameters
+    Long renewDate2 = Time.now();
+    int sequenceNumber2 = 3;
+    identifier.setSequenceNumber(sequenceNumber2);
+
+    // update rm-token
+    RouterStoreToken updateToken = RouterStoreToken.newInstance(identifier, renewDate2);
+    RouterRMTokenRequest updateTokenRequest = RouterRMTokenRequest.newInstance(updateToken);
+    RouterRMTokenResponse updateTokenResponse = stateStore.updateStoredToken(updateTokenRequest);
+
+    Assert.assertNotNull(updateTokenResponse);
+    RouterStoreToken updateTokenResp = updateTokenResponse.getRouterStoreToken();
+    Assert.assertNotNull(updateTokenResp);
+    Assert.assertEquals(updateToken.getRenewDate(), updateTokenResp.getRenewDate());
+    Assert.assertEquals(updateToken.getTokenIdentifier(), updateTokenResp.getTokenIdentifier());
+  }
+
+  @Test
+  public void testRemoveStoredToken() throws IOException, YarnException {
+    // prepare saveToken parameters
+    RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+        new Text("owner3"), new Text("renewer3"), new Text("realuser3"));
+    int sequenceNumber = 3;
+    identifier.setSequenceNumber(sequenceNumber);
+    Long renewDate = Time.now();
+
+    // store new rm-token
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+    Assert.assertNotNull(routerRMTokenResponse);
+
+    // remove rm-token
+    RouterRMTokenResponse removeTokenResponse = stateStore.removeStoredToken(request);
+    Assert.assertNotNull(removeTokenResponse);
+    RouterStoreToken removeTokenResp = removeTokenResponse.getRouterStoreToken();
+    Assert.assertNotNull(removeTokenResp);
+    Assert.assertEquals(removeTokenResp.getRenewDate(), storeToken.getRenewDate());
+    Assert.assertEquals(removeTokenResp.getTokenIdentifier(), storeToken.getTokenIdentifier());
+  }
+
+  @Test
+  public void testGetTokenByRouterStoreToken() throws IOException, YarnException {
+    // prepare saveToken parameters
+    RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+        new Text("owner4"), new Text("renewer4"), new Text("realuser4"));
+    int sequenceNumber = 4;
+    identifier.setSequenceNumber(sequenceNumber);
+    Long renewDate = Time.now();
+
+    // store new rm-token
+    RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+    RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+    RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+    Assert.assertNotNull(routerRMTokenResponse);
+
+    // getTokenByRouterStoreToken
+    RouterRMTokenResponse getRouterRMTokenResp = stateStore.getTokenByRouterStoreToken(request);
+    Assert.assertNotNull(getRouterRMTokenResp);
+    RouterStoreToken getStoreTokenResp = getRouterRMTokenResp.getRouterStoreToken();
+    Assert.assertNotNull(getStoreTokenResp);
+    Assert.assertEquals(getStoreTokenResp.getRenewDate(), storeToken.getRenewDate());
+    Assert.assertEquals(getStoreTokenResp.getTokenIdentifier(), storeToken.getTokenIdentifier());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java
index c3d0a9e1bbd53..b3bb0764dfa4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java
@@ -18,10 +18,20 @@
 
 package org.apache.hadoop.yarn.server.federation.store.impl;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.slf4j.Logger;
@@ -41,6 +51,7 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore {
       " CREATE TABLE applicationsHomeSubCluster ("
           + " applicationId varchar(64) NOT NULL,"
           + " homeSubCluster varchar(256) NOT NULL,"
+          + " createTime datetime NOT NULL,"
           + " CONSTRAINT pk_applicationId PRIMARY KEY (applicationId))";
 
   private static final String TABLE_MEMBERSHIP =
@@ -58,6 +69,12 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore {
           + " policyType varchar(256) NOT NULL, params varbinary(512),"
           + " CONSTRAINT pk_queue PRIMARY KEY (queue))";
 
+  private static final String TABLE_RESERVATIONSHOMESUBCLUSTER =
+      " CREATE TABLE reservationsHomeSubCluster ("
+           + " reservationId varchar(128) NOT NULL,"
+           + " homeSubCluster varchar(256) NOT NULL,"
+           + " CONSTRAINT pk_reservationId PRIMARY KEY (reservationId))";
+
   private static final String SP_REGISTERSUBCLUSTER =
       "CREATE PROCEDURE sp_registerSubCluster("
           + " IN subClusterId_IN varchar(256),"
@@ -134,8 +151,9 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore {
           + " OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)"
           + " MODIFIES SQL DATA BEGIN ATOMIC"
           + " INSERT INTO applicationsHomeSubCluster "
-          + " (applicationId,homeSubCluster) "
-          + " (SELECT applicationId_IN, homeSubCluster_IN"
+          + " (applicationId,homeSubCluster,createTime) "
+          + " (SELECT applicationId_IN, homeSubCluster_IN, "
+          + " NOW() AT TIME ZONE INTERVAL '0:00' HOUR TO MINUTE"
           + " FROM applicationsHomeSubCluster"
           + " WHERE applicationId = applicationId_IN"
           + " HAVING COUNT(*) = 0 );"
@@ -164,11 +182,16 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore {
           + " WHERE applicationId = applicationID_IN; END";
 
   private static final String SP_GETAPPLICATIONSHOMESUBCLUSTER =
-      "CREATE PROCEDURE sp_getApplicationsHomeSubCluster()"
+      "CREATE PROCEDURE sp_getApplicationsHomeSubCluster("
+          + "IN limit_IN int, IN homeSubCluster_IN varchar(256))"
           + " MODIFIES SQL DATA DYNAMIC RESULT SETS 1 BEGIN ATOMIC"
           + " DECLARE result CURSOR FOR"
-          + " SELECT applicationId, homeSubCluster"
-          + " FROM applicationsHomeSubCluster; OPEN result; END";
+          + " SELECT applicationId, homeSubCluster, createTime"
+          + " FROM applicationsHomeSubCluster "
+          + " WHERE ROWNUM() <= limit_IN AND "
+          + " (homeSubCluster_IN = '' OR homeSubCluster = homeSubCluster_IN) "
+          + " ORDER BY createTime desc; "
+          + " OPEN result; END";
 
   private static final String SP_DELETEAPPLICATIONHOMESUBCLUSTER =
       "CREATE PROCEDURE sp_deleteApplicationHomeSubCluster("
@@ -201,21 +224,115 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore {
           + " DECLARE result CURSOR FOR"
           + " SELECT * FROM policies; OPEN result; END";
 
+  private static final String SP_ADDRESERVATIONHOMESUBCLUSTER =
+      "CREATE PROCEDURE sp_addReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128),"
+          + " IN homeSubCluster_IN varchar(256),"
+          + " OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " INSERT INTO reservationsHomeSubCluster "
+          + " (reservationId,homeSubCluster) "
+          + " (SELECT reservationId_IN, homeSubCluster_IN"
+          + " FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN"
+          + " HAVING COUNT(*) = 0 );"
+          + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT;"
+          + " SELECT homeSubCluster INTO storedHomeSubCluster_OUT"
+          + " FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN; END";
+
+  private static final String SP_GETRESERVATIONHOMESUBCLUSTER =
+      "CREATE PROCEDURE sp_getReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128),"
+          + " OUT homeSubCluster_OUT varchar(256))"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " SELECT homeSubCluster INTO homeSubCluster_OUT"
+          + " FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN; END";
+
+  private static final String SP_GETRESERVATIONSHOMESUBCLUSTER =
+      "CREATE PROCEDURE sp_getReservationsHomeSubCluster()"
+          + " MODIFIES SQL DATA DYNAMIC RESULT SETS 1 BEGIN ATOMIC"
+          + " DECLARE result CURSOR FOR"
+          + " SELECT reservationId, homeSubCluster"
+          + " FROM reservationsHomeSubCluster; OPEN result; END";
+
+  private static final String SP_DELETERESERVATIONHOMESUBCLUSTER =
+      "CREATE PROCEDURE sp_deleteReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " DELETE FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN;"
+          + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT; END";
+
+  private static final String SP_UPDATERESERVATIONHOMESUBCLUSTER =
+      "CREATE PROCEDURE sp_updateReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128),"
+          + " IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " UPDATE reservationsHomeSubCluster"
+          + " SET homeSubCluster = homeSubCluster_IN"
+          + " WHERE reservationId = reservationId_IN;"
+          + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT; END";
+
+  protected static final String SP_DROP_ADDRESERVATIONHOMESUBCLUSTER =
+      "DROP PROCEDURE sp_addReservationHomeSubCluster";
+
+  protected static final String SP_ADDRESERVATIONHOMESUBCLUSTER2 =
+      "CREATE PROCEDURE sp_addReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128),"
+          + " IN homeSubCluster_IN varchar(256),"
+          + " OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " INSERT INTO reservationsHomeSubCluster "
+          + " (reservationId,homeSubCluster) "
+          + " (SELECT reservationId_IN, homeSubCluster_IN"
+          + " FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN"
+          + " HAVING COUNT(*) = 0 );"
+          + " SELECT homeSubCluster, 2 INTO storedHomeSubCluster_OUT, rowCount_OUT"
+          + " FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN; END";
+
+  protected static final String SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER =
+      "DROP PROCEDURE sp_updateReservationHomeSubCluster";
+
+  protected static final String SP_UPDATERESERVATIONHOMESUBCLUSTER2 =
+      "CREATE PROCEDURE sp_updateReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128),"
+          + " IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " UPDATE reservationsHomeSubCluster"
+          + " SET homeSubCluster = homeSubCluster_IN"
+          + " WHERE reservationId = reservationId_IN;"
+          + " SET rowCount_OUT = 2; END";
+
+  protected static final String SP_DROP_DELETERESERVATIONHOMESUBCLUSTER =
+      "DROP PROCEDURE sp_deleteReservationHomeSubCluster";
+
+  protected static final String SP_DELETERESERVATIONHOMESUBCLUSTER2 =
+      "CREATE PROCEDURE sp_deleteReservationHomeSubCluster("
+          + " IN reservationId_IN varchar(128), OUT rowCount_OUT int)"
+          + " MODIFIES SQL DATA BEGIN ATOMIC"
+          + " DELETE FROM reservationsHomeSubCluster"
+          + " WHERE reservationId = reservationId_IN;"
+          + " SET rowCount_OUT = 2; END";
+
+  private List<String> tables = new ArrayList<>();
+
   @Override
   public void init(Configuration conf) {
     try {
+      conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, 10);
       super.init(conf);
-    } catch (YarnException e1) {
-      LOG.error("ERROR: failed to init HSQLDB " + e1.getMessage());
-    }
-    try {
-      conn = super.conn;
+      conn = super.getConn();
 
       LOG.info("Database Init: Start");
 
       conn.prepareStatement(TABLE_APPLICATIONSHOMESUBCLUSTER).execute();
       conn.prepareStatement(TABLE_MEMBERSHIP).execute();
       conn.prepareStatement(TABLE_POLICIES).execute();
+      conn.prepareStatement(TABLE_RESERVATIONSHOMESUBCLUSTER).execute();
 
       conn.prepareStatement(SP_REGISTERSUBCLUSTER).execute();
       conn.prepareStatement(SP_DEREGISTERSUBCLUSTER).execute();
@@ -233,9 +350,24 @@ public void init(Configuration conf) {
       conn.prepareStatement(SP_GETPOLICYCONFIGURATION).execute();
       conn.prepareStatement(SP_GETPOLICIESCONFIGURATIONS).execute();
 
+      conn.prepareStatement(SP_ADDRESERVATIONHOMESUBCLUSTER).execute();
+      conn.prepareStatement(SP_GETRESERVATIONHOMESUBCLUSTER).execute();
+      conn.prepareStatement(SP_GETRESERVATIONSHOMESUBCLUSTER).execute();
+      conn.prepareStatement(SP_DELETERESERVATIONHOMESUBCLUSTER).execute();
+      conn.prepareStatement(SP_UPDATERESERVATIONHOMESUBCLUSTER).execute();
+
       LOG.info("Database Init: Complete");
-    } catch (SQLException e) {
-      LOG.error("ERROR: failed to inizialize HSQLDB " + e.getMessage());
+    } catch (Exception e) {
+      LOG.error("ERROR: failed to initialize HSQLDB {}.", e.getMessage());
+    }
+  }
+
+  public void initConnection(Configuration conf) {
+    try {
+      super.init(conf);
+      conn = super.getConn();
+    } catch (YarnException e1) {
+      LOG.error("ERROR: failed open connection to HSQLDB DB {}.", e1.getMessage());
     }
   }
 
@@ -243,9 +375,38 @@ public void closeConnection() {
     try {
       conn.close();
     } catch (SQLException e) {
-      LOG.error(
-          "ERROR: failed to close connection to HSQLDB DB " + e.getMessage());
+      LOG.error("ERROR: failed to close connection to HSQLDB DB {}.", e.getMessage());
+    }
+  }
+
+  /**
+   * Extract The Create Table Sql From The Script.
+   *
+   * @param dbIdentifier database identifier, Like Mysql / SqlServer
+   * @param regex the regex
+   * @throws IOException IO exception.
+   */
+  protected void extractCreateTableSQL(String dbIdentifier, String regex) throws IOException {
+
+    String[] createTableScriptPathItems = new String[] {
+        ".", "target", "test-classes", dbIdentifier, "FederationStateStoreTables.sql" };
+    String createTableScriptPath = StringUtils.join(createTableScriptPathItems, File.separator);
+
+    String createTableSQL =
+        FileUtils.readFileToString(new File(createTableScriptPath), StandardCharsets.UTF_8);
+    Pattern p = Pattern.compile(regex);
+    Matcher m = p.matcher(createTableSQL);
+    while (m != null && m.find()) {
+      String group = m.group();
+      tables.add(group);
     }
   }
 
+  public List<String> getTables() {
+    return tables;
+  }
+
+  public void setTables(List<String> tables) {
+    this.tables = tables;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/MySQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/MySQLFederationStateStore.java
new file mode 100644
index 0000000000000..ae60ba6b99af2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/MySQLFederationStateStore.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * MySQLFederationStateStore implementation of {@link FederationStateStore}.
+ */
+public class MySQLFederationStateStore extends HSQLDBFederationStateStore {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MySQLFederationStateStore.class);
+
+  @Override
+  public void init(Configuration conf) {
+    try {
+      super.initConnection(conf);
+      // get the sql that creates the table
+      extractCreateTableSQL("MySQL", "CREATE TABLE.*\\n(.*,\\n){1,10}.*\\n.*");
+      // print log
+      LOG.info("Mysql - tables = {}.", getTables().size());
+    } catch (IOException e) {
+      LOG.error("ERROR: failed to init HSQLDB {}", e.getMessage());
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLServerFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLServerFederationStateStore.java
new file mode 100644
index 0000000000000..bc54f74ae3f83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLServerFederationStateStore.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * SQLServerFederationStateStore implementation of {@link FederationStateStore}.
+ */
+public class SQLServerFederationStateStore extends HSQLDBFederationStateStore {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SQLServerFederationStateStore.class);
+
+  @Override
+  public void init(Configuration conf) {
+    try {
+      super.initConnection(conf);
+      // get the sql that creates the table
+      extractCreateTableSQL("SQLServer", "CREATE TABLE .*\\n(.*,\\n){1,5}.*(\\n.*){1,15}\\)");
+
+      List<String> tables = getTables();
+
+      // replacing some incompatible syntaxes
+      if (tables != null && !tables.isEmpty()) {
+        tables = tables.stream().map(table -> {
+          String newTable = table.replace("COLLATE Latin1_General_100_BIN2", "").
+                  replace("DEFAULT GETUTCDATE()", "").
+                  replace("[dbo].", "").
+                  replace("[", "").
+                  replace("]", "");
+          return newTable;
+        }).collect(Collectors.toList());
+        setTables(tables);
+      }
+
+      // print log
+      LOG.info("SqlServer - tables = {}.", tables.size());
+    } catch (IOException e) {
+      LOG.error("ERROR: failed to init HSQLDB {}.", e.getMessage());
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
index c29fc03c5b698..70dda2227d0d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestMemoryFederationStateStore.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.federation.store.impl;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 
 /**
@@ -29,6 +30,7 @@ public class TestMemoryFederationStateStore
   @Override
   protected FederationStateStore createStateStore() {
     Configuration conf = new Configuration();
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, 10);
     super.setConf(conf);
     return new MemoryFederationStateStore();
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java
index 3c1d327b393aa..cddcf29ffb079 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java
@@ -17,7 +17,11 @@
 
 package org.apache.hadoop.yarn.server.federation.store.impl;
 
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
@@ -25,14 +29,43 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
 import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.CallableStatement;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_ADDRESERVATIONHOMESUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_ADDRESERVATIONHOMESUBCLUSTER2;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_UPDATERESERVATIONHOMESUBCLUSTER2;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_DELETERESERVATIONHOMESUBCLUSTER;
+import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DELETERESERVATIONHOMESUBCLUSTER2;
 
 /**
  * Unit tests for SQLFederationStateStore.
  */
 public class TestSQLFederationStateStore extends FederationStateStoreBaseTest {
 
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestSQLFederationStateStore.class);
   private static final String HSQLDB_DRIVER = "org.hsqldb.jdbc.JDBCDataSource";
   private static final String DATABASE_URL = "jdbc:hsqldb:mem:state";
   private static final String DATABASE_USERNAME = "SA";
@@ -51,6 +84,7 @@ protected FederationStateStore createStateStore() {
         DATABASE_PASSWORD);
     conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_URL,
         DATABASE_URL + System.currentTimeMillis());
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, 10);
     super.setConf(conf);
     return new HSQLDBFederationStateStore();
   }
@@ -74,4 +108,488 @@ public void testSqlConnectionsCreatedCount() throws YarnException {
     Assert.assertEquals(1,
         FederationStateStoreClientMetrics.getNumConnections());
   }
+
+  class ReservationHomeSC {
+    private String reservationId;
+    private String subHomeClusterId;
+    private int dbUpdateCount;
+
+    ReservationHomeSC(String rId, String subHomeSCId, int dbUpdateCount) {
+      this.reservationId = rId;
+      this.subHomeClusterId = subHomeSCId;
+      this.dbUpdateCount = dbUpdateCount;
+    }
+  }
+
+  private ReservationHomeSC addReservationHomeSubCluster(
+      SQLFederationStateStore sqlFederationStateStore, String procedure,
+      String reservationId, String subHomeClusterId) throws SQLException, YarnException {
+    // procedure call parameter preparation
+    CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure);
+    cstmt.setString("reservationId_IN", reservationId);
+    cstmt.setString("homeSubCluster_IN", subHomeClusterId);
+    cstmt.registerOutParameter("storedHomeSubCluster_OUT", java.sql.Types.VARCHAR);
+    cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+    // execute procedure
+    cstmt.executeUpdate();
+
+    // get call result
+    String dbStoredHomeSubCluster = cstmt.getString("storedHomeSubCluster_OUT");
+    int dbRowCount = cstmt.getInt("rowCount_OUT");
+
+    // return cstmt to pool
+    FederationStateStoreUtils.returnToPool(LOG, cstmt);
+
+    return new ReservationHomeSC(reservationId, dbStoredHomeSubCluster, dbRowCount);
+  }
+
+  private ReservationHomeSC getReservationHomeSubCluster(
+      SQLFederationStateStore sqlFederationStateStore, String procedure,
+      String reservationId) throws SQLException, YarnException {
+
+    // procedure call parameter preparation
+    CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure);
+    cstmt.setString("reservationId_IN", reservationId.toString());
+    cstmt.registerOutParameter("homeSubCluster_OUT", java.sql.Types.VARCHAR);
+
+    // execute procedure
+    cstmt.execute();
+
+    // get call result
+    String dBSubClusterHomeId = cstmt.getString("homeSubCluster_OUT");
+
+    // return cstmt to pool
+    FederationStateStoreUtils.returnToPool(LOG, cstmt);
+
+    // returns the ReservationHomeSubCluster object
+    return new ReservationHomeSC(reservationId, dBSubClusterHomeId, 0);
+  }
+
+  private List<ReservationHomeSC> getReservationsHomeSubCluster(
+      SQLFederationStateStore sqlFederationStateStore, String procedure)
+      throws SQLException, IOException, YarnException {
+
+    List<ReservationHomeSC> results = new ArrayList<>();
+
+    // procedure call parameter preparation
+    CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure);
+
+    // execute procedure
+    ResultSet rs = cstmt.executeQuery();
+    while (rs.next()) {
+      // 1）OUT reservationId
+      String dbReservationId = rs.getString("reservationId");
+
+      // 2）OUT homeSubCluster
+      String dbHomeSubCluster = rs.getString("homeSubCluster");
+      results.add(new ReservationHomeSC(dbReservationId, dbHomeSubCluster, 0));
+    }
+
+    // return cstmt to pool
+    FederationStateStoreUtils.returnToPool(LOG, cstmt);
+
+    // return ReservationHomeSubCluster List
+    return results;
+  }
+
+  private ReservationHomeSC updateReservationHomeSubCluster(
+      SQLFederationStateStore sqlFederationStateStore, String procedure,
+      String reservationId, String subHomeClusterId)
+      throws SQLException, IOException {
+
+    // procedure call parameter preparation
+    CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure);
+
+    // 1）IN reservationId_IN varchar(128)
+    cstmt.setString("reservationId_IN", reservationId);
+    // 2）IN homeSubCluster_IN varchar(256)
+    cstmt.setString("homeSubCluster_IN", subHomeClusterId);
+    // 3）OUT rowCount_OUT int
+    cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+    // execute procedure
+    cstmt.executeUpdate();
+
+    // get rowcount
+    int rowCount = cstmt.getInt("rowCount_OUT");
+
+    // returns the ReservationHomeSubCluster object
+    return new ReservationHomeSC(reservationId, subHomeClusterId, rowCount);
+  }
+
+  private ReservationHomeSC deleteReservationHomeSubCluster(
+      SQLFederationStateStore sqlFederationStateStore, String procedure,
+      String reservationId) throws SQLException {
+    // procedure call parameter preparation
+    CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure);
+
+    // Set the parameters for the stored procedure
+    // 1）IN reservationId_IN varchar(128)
+    cstmt.setString("reservationId_IN", reservationId);
+    // 2）OUT rowCount_OUT int
+    cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER);
+
+    // execute procedure
+    cstmt.executeUpdate();
+
+    // get rowcount
+    int rowCount = cstmt.getInt("rowCount_OUT");
+
+    // returns the ReservationHomeSubCluster object
+    return new ReservationHomeSC(reservationId, "-", rowCount);
+  }
+
+  /**
+   * This test case is used to check whether the procedure
+   * sp_addReservationHomeSubCluster can be executed normally.
+   *
+   * This test case will write 1 record to the database, and check returns the result.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testCheckAddReservationHomeSubCluster() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    // procedure call parameter preparation
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    String subHomeClusterId = "SC-1";
+    ReservationHomeSC resultHC = addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId);
+
+    // validation results
+    Assert.assertNotNull(resultHC);
+    Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId);
+    Assert.assertEquals(1, resultHC.dbUpdateCount);
+  }
+
+  /**
+   * This test case is used to check whether the procedure
+   * sp_getReservationHomeSubCluster can be executed normally.
+   *
+   * Query according to reservationId, expect accurate query results,
+   * and check the homeSubCluster field.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testCheckGetReservationHomeSubCluster() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    // procedure call parameter preparation
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    String subHomeClusterId = "SC-1";
+    addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId);
+
+    // Call getReservationHomeSubCluster to get the result
+    ReservationHomeSC resultHC = getReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString());
+
+    Assert.assertNotNull(resultHC);
+    Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId);
+    Assert.assertEquals(reservationId.toString(), resultHC.reservationId);
+  }
+
+  /**
+   * This test case is used to check whether the procedure
+   * sp_getReservationsHomeSubCluster can be executed normally.
+   *
+   * This test case will write 2 record to the database, and check returns the result.
+   *
+   * The test case will compare the number of returned records from the database
+   * and whether the content of each returned record is accurate.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testCheckGetReservationsHomeSubCluster() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    // add 1st record
+    ReservationId reservationId1 = ReservationId.newInstance(Time.now(), 1);
+    String subHomeClusterId1 = "SC-1";
+    addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER,  reservationId1.toString(),  subHomeClusterId1);
+
+    // add 2nd record
+    ReservationId reservationId2 = ReservationId.newInstance(Time.now(), 2);
+    String subHomeClusterId2 = "SC-2";
+    addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER,  reservationId2.toString(),  subHomeClusterId2);
+
+    List<ReservationHomeSC> reservationHomeSubClusters = getReservationsHomeSubCluster(
+        sqlFederationStateStore, CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER);
+
+    Assert.assertNotNull(reservationHomeSubClusters);
+    Assert.assertEquals(2, reservationHomeSubClusters.size());
+
+    ReservationHomeSC resultHC1 = reservationHomeSubClusters.get(0);
+    Assert.assertNotNull(resultHC1);
+    Assert.assertEquals(reservationId1.toString(), resultHC1.reservationId);
+    Assert.assertEquals(subHomeClusterId1, resultHC1.subHomeClusterId);
+
+    ReservationHomeSC resultHC2 = reservationHomeSubClusters.get(1);
+    Assert.assertNotNull(resultHC2);
+    Assert.assertEquals(reservationId2.toString(), resultHC2.reservationId);
+    Assert.assertEquals(subHomeClusterId2, resultHC2.subHomeClusterId);
+  }
+
+  /**
+   * This test case is used to check whether the procedure
+   * sp_updateReservationHomeSubCluster can be executed normally.
+   *
+   * This test case will first insert 1 record into the database,
+   * and then update the new SubHomeClusterId according to the reservationId.
+   *
+   * It will check whether the SubHomeClusterId is as expected after the addition and update.
+   * For the first time, the HomeClusterId of the database should be SC-1,
+   * and for the second time, the HomeClusterId of the database should be SC-2.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testCheckUpdateReservationHomeSubCluster() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    // procedure call parameter preparation
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    String subHomeClusterId = "SC-1";
+    addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId);
+
+    // verify that the subHomeClusterId corresponding to reservationId is SC-1
+    ReservationHomeSC resultHC = getReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString());
+    Assert.assertNotNull(resultHC);
+    Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId);
+
+    // prepare to update parameters
+    String newSubHomeClusterId = "SC-2";
+    ReservationHomeSC reservationHomeSubCluster =
+        updateReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), newSubHomeClusterId);
+
+    Assert.assertNotNull(reservationHomeSubCluster);
+    Assert.assertEquals(1, reservationHomeSubCluster.dbUpdateCount);
+
+    // verify that the subHomeClusterId corresponding to reservationId is SC-2
+    ReservationHomeSC resultHC2 = getReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString());
+    Assert.assertNotNull(resultHC2);
+    Assert.assertEquals(newSubHomeClusterId, resultHC2.subHomeClusterId);
+  }
+
+  /**
+   * This test case is used to check whether the procedure
+   * sp_deleteReservationHomeSubCluster can be executed normally.
+   *
+   * This test case will first write 1 record to the database,
+   * and then delete the corresponding record according to reservationId.
+   *
+   * Query the corresponding homeSubCluster according to reservationId,
+   * we should get a NULL at this time.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testCheckDeleteReservationHomeSubCluster() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    // procedure call parameter preparation
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    String subHomeClusterId = "SC-1";
+    addReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId);
+
+    // call the delete method of the reservation
+    ReservationHomeSC resultHC = deleteReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER, reservationId.toString());
+
+    Assert.assertNotNull(resultHC);
+    Assert.assertEquals(1, resultHC.dbUpdateCount);
+
+    // call getReservationHomeSubCluster to get the result
+    ReservationHomeSC resultHC1 = getReservationHomeSubCluster(sqlFederationStateStore,
+        CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString());
+    Assert.assertNotNull(resultHC1);
+    Assert.assertEquals(null, resultHC1.subHomeClusterId);
+  }
+
+  /**
+   * This test case is used to verify the processing logic of the incorrect number of
+   * updated records returned by the database when AddReservationHomeSubCluster is used.
+   *
+   * The probability of the database returning an update record greater than 1 is very low.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testAddReservationHomeSubClusterAbnormalSituation() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    Connection conn =  sqlFederationStateStore.getConn();
+    conn.prepareStatement(SP_DROP_ADDRESERVATIONHOMESUBCLUSTER).execute();
+    conn.prepareStatement(SP_ADDRESERVATIONHOMESUBCLUSTER2).execute();
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId = SubClusterId.newInstance("SC");
+
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+    AddReservationHomeSubClusterRequest request =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+
+    String errorMsg = String.format(
+        "Wrong behavior during the insertion of subCluster %s according to reservation %s. " +
+        "The database expects to insert 1 record, but the number of " +
+        "inserted changes is greater than 1, " +
+        "please check the records of the database.", subClusterId, reservationId);
+
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> stateStore.addReservationHomeSubCluster(request));
+  }
+
+  /**
+   * This test case is used to verify the logic when calling the updateReservationHomeSubCluster
+   * method if the database returns an inaccurate result.
+   *
+   * The probability of the database returning an update record greater than 1 is very low.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testUpdateReservationHomeSubClusterAbnormalSituation() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    Connection conn =  sqlFederationStateStore.getConn();
+    conn.prepareStatement(SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER).execute();
+    conn.prepareStatement(SP_UPDATERESERVATIONHOMESUBCLUSTER2).execute();
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC");
+
+    // add Reservation data.
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId1);
+    AddReservationHomeSubClusterRequest addRequest =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+    stateStore.addReservationHomeSubCluster(addRequest);
+
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC2");
+    ReservationHomeSubCluster reservationHomeSubCluster2 =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId2);
+    UpdateReservationHomeSubClusterRequest updateRequest =
+        UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2);
+
+    String errorMsg = String.format(
+        "Wrong behavior during update the subCluster %s according to reservation %s. " +
+        "The database is expected to update 1 record, " +
+        "but the number of database update records is greater than 1, " +
+        "the records of the database should be checked.",
+        subClusterId2, reservationId);
+
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> stateStore.updateReservationHomeSubCluster(updateRequest));
+  }
+
+  /**
+   * This test case is used to verify the logic when calling the deleteReservationHomeSubCluster
+   * method if the database returns an inaccurate result.
+   *
+   * The probability of the database returning an update record greater than 1 is very low.
+   *
+   * @throws Exception when the error occurs
+   */
+  @Test
+  public void testDeleteReservationHomeSubClusterAbnormalSituation() throws Exception {
+    FederationStateStore stateStore = getStateStore();
+    Assert.assertTrue(stateStore instanceof SQLFederationStateStore);
+
+    SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore;
+
+    Connection conn =  sqlFederationStateStore.getConn();
+    conn.prepareStatement(SP_DROP_DELETERESERVATIONHOMESUBCLUSTER).execute();
+    conn.prepareStatement(SP_DELETERESERVATIONHOMESUBCLUSTER2).execute();
+
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC");
+
+    // add Reservation data.
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, subClusterId1);
+    AddReservationHomeSubClusterRequest addRequest =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+    stateStore.addReservationHomeSubCluster(addRequest);
+
+    DeleteReservationHomeSubClusterRequest delRequest =
+        DeleteReservationHomeSubClusterRequest.newInstance(reservationId);
+
+    String errorMsg = String.format(
+        "Wrong behavior during deleting the reservation %s. " +
+        "The database is expected to delete 1 record, " +
+        "but the number of deleted records returned by the database is greater than 1, " +
+        "indicating that a duplicate reservationId occurred during the deletion process.",
+        reservationId);
+
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> stateStore.deleteReservationHomeSubCluster(delRequest));
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testStoreNewMasterKey() throws Exception {
+    super.testStoreNewMasterKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
+    super.testGetMasterKeyByDelegationKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testRemoveStoredMasterKey() throws YarnException, IOException {
+    super.testRemoveStoredMasterKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testStoreNewToken() throws IOException, YarnException {
+    super.testStoreNewToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testUpdateStoredToken() throws IOException, YarnException {
+    super.testUpdateStoredToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testRemoveStoredToken() throws IOException, YarnException {
+    super.testRemoveStoredToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testGetTokenByRouterStoreToken() throws IOException, YarnException {
+    super.testGetTokenByRouterStoreToken();
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
index fe28641eb2560..18396cb821566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
@@ -19,20 +19,28 @@
 
 import java.io.IOException;
 
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.RetryNTimes;
 import org.apache.curator.test.TestingServer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
+import org.apache.hadoop.metrics2.impl.MetricsRecords;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.junit.Assert.assertEquals;
+
 /**
  * Unit tests for ZookeeperFederationStateStore.
  */
@@ -60,6 +68,7 @@ public void before() throws IOException, YarnException {
 
       Configuration conf = new YarnConfiguration();
       conf.set(CommonConfigurationKeys.ZK_ADDRESS, connectString);
+      conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_MAX_APPLICATIONS, 10);
       setConf(conf);
     } catch (Exception e) {
       LOG.error("Cannot initialize ZooKeeper store", e);
@@ -84,4 +93,116 @@ protected FederationStateStore createStateStore() {
     super.setConf(getConf());
     return new ZookeeperFederationStateStore();
   }
+
+  @Test
+  public void testMetricsInited() throws Exception {
+    ZookeeperFederationStateStore zkStateStore = (ZookeeperFederationStateStore) createStateStore();
+    ZKFederationStateStoreOpDurations zkStateStoreOpDurations = zkStateStore.getOpDurations();
+    MetricsCollectorImpl collector = new MetricsCollectorImpl();
+
+    long anyDuration = 10;
+    long start = Time.now();
+    long end = start + anyDuration;
+
+    zkStateStoreOpDurations.addAppHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addUpdateAppHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addGetAppHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addGetAppsHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addDeleteAppHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addRegisterSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addDeregisterSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addSubClusterHeartbeatDuration(start, end);
+    zkStateStoreOpDurations.addGetSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addGetSubClustersDuration(start, end);
+    zkStateStoreOpDurations.addGetPolicyConfigurationDuration(start, end);
+    zkStateStoreOpDurations.addSetPolicyConfigurationDuration(start, end);
+    zkStateStoreOpDurations.addGetPoliciesConfigurationsDuration(start, end);
+    zkStateStoreOpDurations.addReservationHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addGetReservationHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addGetReservationsHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addDeleteReservationHomeSubClusterDuration(start, end);
+    zkStateStoreOpDurations.addUpdateReservationHomeSubClusterDuration(start, end);
+
+    zkStateStoreOpDurations.getMetrics(collector, true);
+    assertEquals("Incorrect number of perf metrics", 1, collector.getRecords().size());
+
+    MetricsRecord record = collector.getRecords().get(0);
+    MetricsRecords.assertTag(record, ZKFederationStateStoreOpDurations.RECORD_INFO.name(),
+        "ZKFederationStateStoreOpDurations");
+
+    double expectAvgTime = anyDuration;
+    MetricsRecords.assertMetric(record, "AddAppHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "UpdateAppHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetAppHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetAppsHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "DeleteAppHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "RegisterSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "DeregisterSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "SubClusterHeartbeatAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetSubClustersAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetPolicyConfigurationAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "SetPolicyConfigurationAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterAvgTime",  expectAvgTime);
+    MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterAvgTime",  expectAvgTime);
+
+    long expectOps = 1;
+    MetricsRecords.assertMetric(record, "AddAppHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "UpdateAppHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetAppHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetAppsHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "DeleteAppHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "RegisterSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "DeregisterSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "SubClusterHeartbeatNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetSubClustersNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetPolicyConfigurationNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "SetPolicyConfigurationNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterNumOps",  expectOps);
+    MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps",  expectOps);
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testStoreNewMasterKey() throws Exception {
+    super.testStoreNewMasterKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
+    super.testGetMasterKeyByDelegationKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testRemoveStoredMasterKey() throws YarnException, IOException {
+    super.testRemoveStoredMasterKey();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testStoreNewToken() throws IOException, YarnException {
+    super.testStoreNewToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testUpdateStoredToken() throws IOException, YarnException {
+    super.testUpdateStoredToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testRemoveStoredToken() throws IOException, YarnException {
+    super.testRemoveStoredToken();
+  }
+
+  @Test(expected = NotImplementedException.class)
+  public void testGetTokenByRouterStoreToken() throws IOException, YarnException {
+    super.testGetTokenByRouterStoreToken();
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
index cf8cf719d01d5..bc20856e8c5d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java
@@ -17,8 +17,10 @@
 
 package org.apache.hadoop.yarn.server.federation.store.records;
 
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.BasePBImplRecordsTest;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto;
@@ -47,6 +49,15 @@
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto;
 import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl;
@@ -75,10 +86,24 @@
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl;
 import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyResponsePBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreTokenPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenRequestPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenResponsePBImpl;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.ApplicationHomeSubClusterPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.GetReservationHomeSubClusterRequestPBImpl;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
 /**
  * Test class for federation protocol records.
  */
@@ -92,6 +117,10 @@ public static void setup() throws Exception {
     generateByNewInstance(SubClusterInfo.class);
     generateByNewInstance(ApplicationHomeSubCluster.class);
     generateByNewInstance(SubClusterPolicyConfiguration.class);
+    generateByNewInstance(RouterMasterKey.class);
+    generateByNewInstance(YARNDelegationTokenIdentifier.class);
+    generateByNewInstance(RouterStoreToken.class);
+    generateByNewInstance(ReservationId.class);
   }
 
   @Test
@@ -262,4 +291,134 @@ public void testGetSubClusterPoliciesConfigurationsResponse()
         GetSubClusterPoliciesConfigurationsResponsePBImpl.class,
         GetSubClusterPoliciesConfigurationsResponseProto.class);
   }
+
+  @Test
+  public void testRouterMasterKey() throws Exception {
+    validatePBImplRecord(RouterMasterKeyPBImpl.class, RouterMasterKeyProto.class);
+  }
+
+  @Test
+  public void testRouterMasterKeyRequest() throws Exception {
+    validatePBImplRecord(RouterMasterKeyRequestPBImpl.class, RouterMasterKeyRequestProto.class);
+  }
+
+  @Test
+  public void testRouterMasterKeyResponse() throws Exception {
+    validatePBImplRecord(RouterMasterKeyResponsePBImpl.class, RouterMasterKeyResponseProto.class);
+  }
+
+  @Test
+  public void testRouterStoreToken() throws Exception {
+    validatePBImplRecord(RouterStoreTokenPBImpl.class, RouterStoreTokenProto.class);
+  }
+
+  @Test
+  public void testRouterRMTokenRequest() throws Exception {
+    validatePBImplRecord(RouterRMTokenRequestPBImpl.class, RouterRMTokenRequestProto.class);
+  }
+
+  @Test
+  public void testRouterRMTokenResponse() throws Exception {
+    validatePBImplRecord(RouterRMTokenResponsePBImpl.class, RouterRMTokenResponseProto.class);
+  }
+
+  @Test
+  public void testApplicationHomeSubCluster() throws Exception {
+    validatePBImplRecord(ApplicationHomeSubClusterPBImpl.class,
+        ApplicationHomeSubClusterProto.class);
+  }
+
+  @Test
+  public void testGetReservationHomeSubClusterRequest() throws Exception {
+    validatePBImplRecord(GetReservationHomeSubClusterRequestPBImpl.class,
+        GetReservationHomeSubClusterRequestProto.class);
+  }
+
+  @Test
+  public void testValidateApplicationHomeSubClusterEqual() throws Exception {
+    long now = Time.now();
+
+    ApplicationId appId1 = ApplicationId.newInstance(now, 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC-1");
+    ApplicationHomeSubCluster applicationHomeSubCluster1 =
+        ApplicationHomeSubCluster.newInstance(appId1, subClusterId1);
+
+    ApplicationId appId2 = ApplicationId.newInstance(now, 1);
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC-1");
+    ApplicationHomeSubCluster applicationHomeSubCluster2 =
+        ApplicationHomeSubCluster.newInstance(appId2, subClusterId2);
+
+    assertEquals(applicationHomeSubCluster1, applicationHomeSubCluster2);
+  }
+
+  @Test
+  public void testValidateReservationHomeSubClusterEqual() throws Exception {
+    long now = Time.now();
+
+    ReservationId reservationId1 = ReservationId.newInstance(now, 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC-1");
+    ReservationHomeSubCluster reservationHomeSubCluster1 =
+        ReservationHomeSubCluster.newInstance(reservationId1, subClusterId1);
+
+    ReservationId reservationId2 = ReservationId.newInstance(now, 1);
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC-1");
+    ReservationHomeSubCluster reservationHomeSubCluster2 =
+        ReservationHomeSubCluster.newInstance(reservationId2, subClusterId2);
+
+    assertEquals(reservationHomeSubCluster1, reservationHomeSubCluster2);
+  }
+
+  @Test
+  public void testSubClusterIdEqual() throws Exception {
+    SubClusterId subClusterId1 = SubClusterId.newInstance("SC-1");
+    SubClusterId subClusterId2 = SubClusterId.newInstance("SC-1");
+    assertEquals(subClusterId1, subClusterId2);
+  }
+
+  @Test
+  public void testSubClusterIdInfoEqual() throws Exception {
+    SubClusterIdInfo subClusterIdInfo1 = new SubClusterIdInfo("SC-1");
+    SubClusterIdInfo subClusterIdInfo2 = new SubClusterIdInfo("SC-1");
+    assertEquals(subClusterIdInfo1, subClusterIdInfo2);
+  }
+
+  @Test
+  public void testSubClusterPolicyConfigurationEqual() throws Exception {
+
+    String queue1 = "queue1";
+    WeightedPolicyInfo policyInfo1 = mock(WeightedPolicyInfo.class);
+    ByteBuffer buf1 = policyInfo1.toByteBuffer();
+    SubClusterPolicyConfiguration configuration1 = SubClusterPolicyConfiguration
+        .newInstance(queue1, policyInfo1.getClass().getCanonicalName(), buf1);
+
+    String queue2 = "queue1";
+    WeightedPolicyInfo policyInfo2 = mock(WeightedPolicyInfo.class);
+    ByteBuffer buf2 = policyInfo1.toByteBuffer();
+    SubClusterPolicyConfiguration configuration2 = SubClusterPolicyConfiguration
+        .newInstance(queue2, policyInfo2.getClass().getCanonicalName(), buf2);
+
+    assertEquals(configuration1, configuration2);
+  }
+
+  @Test
+  public void testSubClusterInfoEqual() throws Exception {
+
+    String scAmRMAddress = "5.6.7.8:5";
+    String scClientRMAddress = "5.6.7.8:6";
+    String scRmAdminAddress = "5.6.7.8:7";
+    String scWebAppAddress = "127.0.0.1:8080";
+    String capabilityJson = "-";
+
+    SubClusterInfo sc1 =
+        SubClusterInfo.newInstance(SubClusterId.newInstance("SC-1"),
+        scAmRMAddress, scClientRMAddress, scRmAdminAddress, scWebAppAddress,
+        SubClusterState.SC_RUNNING, Time.now(), capabilityJson);
+
+    SubClusterInfo sc2 =
+        SubClusterInfo.newInstance(SubClusterId.newInstance("SC-1"),
+        scAmRMAddress, scClientRMAddress, scRmAdminAddress, scWebAppAddress,
+        SubClusterState.SC_RUNNING, Time.now(), capabilityJson);
+
+    assertEquals(sc1, sc2);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationSQLAccuracyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationSQLAccuracyTest.java
new file mode 100644
index 0000000000000..bd1920e274982
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/FederationSQLAccuracyTest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.sql;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public abstract class FederationSQLAccuracyTest {
+
+  protected static final String HSQLDB_DRIVER = "org.hsqldb.jdbc.JDBCDataSource";
+  protected static final String DATABASE_URL = "jdbc:hsqldb:mem:state";
+  protected static final String DATABASE_USERNAME = "SA";
+  protected static final String DATABASE_PASSWORD = "";
+
+  private FederationStateStore stateStore;
+
+  protected abstract FederationStateStore createStateStore();
+
+  private Configuration conf;
+
+  @Before
+  public void before() throws IOException, YarnException {
+    stateStore = createStateStore();
+    conf = new YarnConfiguration();
+    conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_JDBC_CLASS, HSQLDB_DRIVER);
+    conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_USERNAME, DATABASE_USERNAME);
+    conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_PASSWORD, DATABASE_PASSWORD);
+    conf.set(YarnConfiguration.FEDERATION_STATESTORE_SQL_URL, getSQLURL());
+    stateStore.init(conf);
+  }
+
+  @After
+  public void after() throws Exception {
+    stateStore.close();
+  }
+
+  protected abstract String getSQLURL();
+
+  protected void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  protected Configuration getConf() {
+    return conf;
+  }
+
+  protected FederationStateStore getStateStore() {
+    return stateStore;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationMySQLScriptAccuracy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationMySQLScriptAccuracy.java
new file mode 100644
index 0000000000000..5c1541559078c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationMySQLScriptAccuracy.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.sql;
+
+import org.apache.hadoop.yarn.server.federation.store.impl.MySQLFederationStateStore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.List;
+
+public class TestFederationMySQLScriptAccuracy extends FederationSQLAccuracyTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFederationMySQLScriptAccuracy.class);
+
+  private static final String MYSQL_COMPATIBILITY = ";sql.syntax_mys=true";
+
+  @Override
+  protected MySQLFederationStateStore createStateStore() {
+    return new MySQLFederationStateStore();
+  }
+
+  @Override
+  protected String getSQLURL() {
+    return DATABASE_URL + System.currentTimeMillis() + MYSQL_COMPATIBILITY;
+  }
+
+  @Test
+  public void checkMysqlScriptAccuracy() throws SQLException {
+    MySQLFederationStateStore federationStateStore = this.createStateStore();
+    federationStateStore.initConnection(this.getConf());
+
+    // get a list of tables
+    List<String> tables = federationStateStore.getTables();
+    for (String table : tables) {
+      federationStateStore.getConn().prepareStatement(table).execute();
+    }
+
+    LOG.info("FederationStateStore create {} tables.", tables.size());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationSQLServerScriptAccuracy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationSQLServerScriptAccuracy.java
new file mode 100644
index 0000000000000..b845ae747b237
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/sql/TestFederationSQLServerScriptAccuracy.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.sql;
+
+import org.apache.hadoop.yarn.server.federation.store.impl.SQLServerFederationStateStore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.SQLException;
+import java.util.List;
+
+public class TestFederationSQLServerScriptAccuracy extends FederationSQLAccuracyTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFederationSQLServerScriptAccuracy.class);
+
+  private static final String SQLSERVER_COMPATIBILITY = ";sql.syntax_mss=true";
+
+  @Override
+  protected SQLServerFederationStateStore createStateStore() {
+    return new SQLServerFederationStateStore();
+  }
+
+  @Override
+  protected String getSQLURL() {
+    return DATABASE_URL + System.currentTimeMillis() + SQLSERVER_COMPATIBILITY;
+  }
+
+  @Test
+  public void checkSqlServerScriptAccuracy() throws SQLException {
+    SQLServerFederationStateStore federationStateStore = this.createStateStore();
+    federationStateStore.init(getConf());
+
+    // get a list of tables
+    List<String> tables = federationStateStore.getTables();
+    for (String table : tables) {
+      federationStateStore.getConn().prepareStatement(table).execute();
+    }
+
+    LOG.info("FederationStateStore create {} tables.", tables.size());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index a9b9029b25732..ad4daf945fc6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -159,6 +159,51 @@ public static void initializePolicyContext(
         new Configuration());
   }
 
+  public static FederationPolicyInitializationContext initializePolicyContext2(
+      ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo,
+      Map<SubClusterId, SubClusterInfo> activeSubClusters,
+      FederationStateStoreFacade facade) throws YarnException {
+    FederationPolicyInitializationContext context =
+        new FederationPolicyInitializationContext(null, initResolver(), facade,
+        SubClusterId.newInstance("homesubcluster"));
+    return initializePolicyContext2(context, policy, policyInfo, activeSubClusters);
+  }
+
+  public static FederationPolicyInitializationContext initializePolicyContext2(
+      ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo,
+      Map<SubClusterId, SubClusterInfo> activeSubClusters)
+      throws YarnException {
+    return initializePolicyContext2(policy, policyInfo, activeSubClusters, initFacade());
+  }
+
+  public static FederationPolicyInitializationContext initializePolicyContext2(
+      FederationPolicyInitializationContext fpc,
+      ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo,
+      Map<SubClusterId, SubClusterInfo> activeSubClusters)
+      throws YarnException {
+    ByteBuffer buf = policyInfo.toByteBuffer();
+    fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+        .newInstance("queue1", policy.getClass().getCanonicalName(), buf));
+
+    if (fpc.getFederationStateStoreFacade() == null) {
+      FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+      FederationStateStore fss = mock(FederationStateStore.class);
+
+      if (activeSubClusters == null) {
+        activeSubClusters = new HashMap<>();
+      }
+
+      GetSubClustersInfoResponse response = GetSubClustersInfoResponse.newInstance(
+          activeSubClusters.values());
+
+      when(fss.getSubClusters(any())).thenReturn(response);
+      facade.reinitialize(fss, new Configuration());
+      fpc.setFederationStateStoreFacade(facade);
+    }
+    policy.reinitialize(fpc);
+    return fpc;
+  }
+
   /**
    * Initialize a {@link SubClusterResolver}.
    *
@@ -231,8 +276,8 @@ public static FederationStateStoreFacade initFacade(
    * @throws YarnException in case the initialization is not successful.
    */
   public static FederationStateStoreFacade initFacade() throws YarnException {
-    return initFacade(new ArrayList<>(), mock(SubClusterPolicyConfiguration
-        .class));
+    SubClusterPolicyConfiguration policyConfiguration =
+        SubClusterPolicyConfiguration.newInstance(null, null, null);
+    return initFacade(new ArrayList<>(), policyConfiguration);
   }
-
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
index 46cf034468313..daf885a2171d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java
@@ -22,6 +22,7 @@
 import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
@@ -38,6 +39,8 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 import org.apache.hadoop.yarn.util.MonotonicClock;
 
@@ -178,4 +181,12 @@ public void deRegisterSubCluster(SubClusterId subClusterId)
         .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED));
   }
 
+  public SubClusterId queryReservationHomeSC(ReservationId reservationId)
+      throws YarnException {
+    GetReservationHomeSubClusterRequest request =
+        GetReservationHomeSubClusterRequest.newInstance(reservationId);
+    GetReservationHomeSubClusterResponse response =
+        stateStore.getReservationHomeSubCluster(request);
+    return response.getReservationHomeSubCluster().getHomeSubCluster();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
index 56fa0524a39dc..92dd426f51372 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -22,17 +22,29 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -41,6 +53,8 @@
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
+import javax.cache.Cache;
+
 /**
  * Unit tests for FederationStateStoreFacade.
  */
@@ -64,12 +78,14 @@ public static Collection<Boolean[]> getParameters() {
   private FederationStateStoreTestUtil stateStoreTestUtil;
   private FederationStateStoreFacade facade =
       FederationStateStoreFacade.getInstance();
+  private Boolean isCachingEnabled;
 
   public TestFederationStateStoreFacade(Boolean isCachingEnabled) {
     conf = new Configuration();
     if (!(isCachingEnabled.booleanValue())) {
       conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
     }
+    this.isCachingEnabled = isCachingEnabled;
   }
 
   @Before
@@ -206,4 +222,152 @@ public void testAddApplicationHomeSubCluster() throws YarnException {
     Assert.assertEquals(subClusterId1, result);
   }
 
+  @Test
+  public void testGetApplicationHomeSubClusterCache() throws YarnException {
+    ApplicationId appId = ApplicationId.newInstance(clusterTs, numApps + 1);
+    SubClusterId subClusterId1 = SubClusterId.newInstance("Home1");
+
+    ApplicationHomeSubCluster appHomeSubCluster =
+        ApplicationHomeSubCluster.newInstance(appId, subClusterId1);
+    SubClusterId subClusterIdAdd = facade.addApplicationHomeSubCluster(appHomeSubCluster);
+
+    SubClusterId subClusterIdByFacade = facade.getApplicationHomeSubCluster(appId);
+    Assert.assertEquals(subClusterIdByFacade, subClusterIdAdd);
+    Assert.assertEquals(subClusterId1, subClusterIdAdd);
+
+    if (isCachingEnabled.booleanValue()) {
+      Cache<Object, Object> cache = facade.getCache();
+      Object cacheKey = facade.getAppHomeSubClusterCacheRequest(appId);
+      Object subClusterIdByCache = cache.get(cacheKey);
+      Assert.assertEquals(subClusterIdByFacade, subClusterIdByCache);
+      Assert.assertEquals(subClusterId1, subClusterIdByCache);
+    }
+  }
+
+  @Test
+  public void testStoreNewMasterKey() throws YarnException, IOException {
+    // store delegation key;
+    DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    Set<DelegationKey> keySet = new HashSet<>();
+    keySet.add(key);
+    facade.storeNewMasterKey(key);
+
+    MemoryFederationStateStore federationStateStore =
+        (MemoryFederationStateStore) facade.getStateStore();
+    RouterRMDTSecretManagerState secretManagerState =
+        federationStateStore.getRouterRMSecretManagerState();
+    Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
+  }
+
+  @Test
+  public void testRemoveStoredMasterKey() throws YarnException, IOException {
+    // store delegation key;
+    DelegationKey key = new DelegationKey(4567, 7654, "keyBytes".getBytes());
+    Set<DelegationKey> keySet = new HashSet<>();
+    keySet.add(key);
+    facade.storeNewMasterKey(key);
+
+    // check to delete delegationKey
+    facade.removeStoredMasterKey(key);
+    keySet.clear();
+
+    MemoryFederationStateStore federationStateStore =
+        (MemoryFederationStateStore) facade.getStateStore();
+    RouterRMDTSecretManagerState secretManagerState =
+        federationStateStore.getRouterRMSecretManagerState();
+    Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
+  }
+
+  @Test
+  public void testStoreNewToken() throws YarnException, IOException {
+    // store new rm-token
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    facade.storeNewToken(dtId1, renewDate1);
+
+    // get RouterStoreToken from StateStore
+    RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1);
+    RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken);
+    RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest);
+    Assert.assertNotNull(rmTokenResponse);
+
+    RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken();
+    YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier();
+    Assert.assertNotNull(resultStoreToken);
+    Assert.assertNotNull(resultTokenIdentifier);
+    Assert.assertNotNull(resultStoreToken.getRenewDate());
+
+    Assert.assertEquals(dtId1, resultTokenIdentifier);
+    Assert.assertEquals(renewDate1, resultStoreToken.getRenewDate());
+    Assert.assertEquals(sequenceNumber, resultTokenIdentifier.getSequenceNumber());
+  }
+
+  @Test
+  public void testUpdateNewToken() throws YarnException, IOException {
+    // store new rm-token
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+    int sequenceNumber = 2;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    facade.storeNewToken(dtId1, renewDate1);
+
+    Long renewDate2 = Time.now();
+    int sequenceNumber2 = 3;
+    dtId1.setSequenceNumber(sequenceNumber2);
+    facade.updateStoredToken(dtId1, renewDate2);
+
+    // get RouterStoreToken from StateStore
+    RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1);
+    RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken);
+    RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest);
+    Assert.assertNotNull(rmTokenResponse);
+
+    RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken();
+    YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier();
+    Assert.assertNotNull(resultStoreToken);
+    Assert.assertNotNull(resultTokenIdentifier);
+    Assert.assertNotNull(resultStoreToken.getRenewDate());
+
+    Assert.assertEquals(dtId1, resultTokenIdentifier);
+    Assert.assertEquals(renewDate2, resultStoreToken.getRenewDate());
+    Assert.assertEquals(sequenceNumber2, resultTokenIdentifier.getSequenceNumber());
+  }
+
+  @Test
+  public void testRemoveStoredToken() throws Exception {
+    // store new rm-token
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner3"), new Text("renewer3"), new Text("realuser3"));
+    int sequenceNumber = 3;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    facade.storeNewToken(dtId1, renewDate1);
+
+    // get RouterStoreToken from StateStore
+    RouterStoreToken routerStoreToken = RouterStoreToken.newInstance(dtId1, renewDate1);
+    RouterRMTokenRequest rmTokenRequest = RouterRMTokenRequest.newInstance(routerStoreToken);
+    RouterRMTokenResponse rmTokenResponse = stateStore.getTokenByRouterStoreToken(rmTokenRequest);
+    Assert.assertNotNull(rmTokenResponse);
+
+    RouterStoreToken resultStoreToken = rmTokenResponse.getRouterStoreToken();
+    YARNDelegationTokenIdentifier resultTokenIdentifier = resultStoreToken.getTokenIdentifier();
+    Assert.assertNotNull(resultStoreToken);
+    Assert.assertNotNull(resultTokenIdentifier);
+    Assert.assertNotNull(resultStoreToken.getRenewDate());
+
+    Assert.assertEquals(dtId1, resultTokenIdentifier);
+    Assert.assertEquals(renewDate1, resultStoreToken.getRenewDate());
+    Assert.assertEquals(sequenceNumber, resultTokenIdentifier.getSequenceNumber());
+
+    // remove rm-token
+    facade.removeStoredToken(dtId1);
+
+    // Call again(getTokenByRouterStoreToken) after remove will throw IOException(not exist)
+    LambdaTestUtils.intercept(IOException.class, "RMDelegationToken: " + dtId1 + " does not exist.",
+        () -> stateStore.getTokenByRouterStoreToken(rmTokenRequest));
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
index abb1d93c3399e..ddc85bf3e6fac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
@@ -20,10 +20,16 @@
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -39,6 +45,7 @@
 import org.apache.hadoop.yarn.server.AMRMClientRelayer;
 import org.apache.hadoop.yarn.server.MockResourceManagerFacade;
 import org.apache.hadoop.yarn.util.AsyncCallback;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -58,6 +65,9 @@ public class TestUnmanagedApplicationManager {
 
   private ApplicationAttemptId attemptId;
 
+  private UnmanagedAMPoolManager uamPool;
+  private ExecutorService threadpool;
+
   @Before
   public void setup() {
     conf.set(YarnConfiguration.RM_CLUSTER_ID, "subclusterId");
@@ -69,6 +79,29 @@ public void setup() {
     uam = new TestableUnmanagedApplicationManager(conf,
         attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true,
         "rm");
+
+    threadpool = Executors.newCachedThreadPool();
+    uamPool = new TestableUnmanagedAMPoolManager(this.threadpool);
+    uamPool.init(conf);
+    uamPool.start();
+  }
+
+  @After
+  public void tearDown() throws IOException, InterruptedException {
+    if (uam != null) {
+      uam.shutDownConnections();
+      uam = null;
+    }
+    if (uamPool != null) {
+      if (uamPool.isInState(Service.STATE.STARTED)) {
+        uamPool.stop();
+      }
+      uamPool = null;
+    }
+    if (threadpool != null) {
+      threadpool.shutdownNow();
+      threadpool = null;
+    }
   }
 
   protected void waitForCallBackCountAndCheckZeroPending(
@@ -464,4 +497,49 @@ public Object run() {
     }
   }
 
+  protected class TestableUnmanagedAMPoolManager extends UnmanagedAMPoolManager {
+    public TestableUnmanagedAMPoolManager(ExecutorService threadpool) {
+      super(threadpool);
+    }
+
+    @Override
+    public UnmanagedApplicationManager createUAM(Configuration configuration,
+        ApplicationId appId, String queueName, String submitter, String appNameSuffix,
+        boolean keepContainersAcrossApplicationAttempts, String rmId) {
+      return new TestableUnmanagedApplicationManager(configuration, appId, queueName, submitter,
+          appNameSuffix, keepContainersAcrossApplicationAttempts, rmId);
+    }
+  }
+
+  @Test
+  public void testSeparateThreadWithoutBlockServiceStop() throws Exception {
+    ApplicationAttemptId attemptId1 =
+        ApplicationAttemptId.newInstance(ApplicationId.newInstance(Time.now(), 1), 1);
+    Token<AMRMTokenIdentifier> token1 = uamPool.launchUAM("SC-1", this.conf,
+        attemptId1.getApplicationId(), "default", "test-user", "SC-HOME", true, "SC-1");
+    Assert.assertNotNull(token1);
+
+    ApplicationAttemptId attemptId2 =
+        ApplicationAttemptId.newInstance(ApplicationId.newInstance(Time.now(), 2), 1);
+    Token<AMRMTokenIdentifier> token2 = uamPool.launchUAM("SC-2", this.conf,
+        attemptId2.getApplicationId(), "default", "test-user", "SC-HOME", true, "SC-2");
+    Assert.assertNotNull(token2);
+
+    Map<String, UnmanagedApplicationManager> unmanagedAppMasterMap =
+        uamPool.getUnmanagedAppMasterMap();
+    Assert.assertNotNull(unmanagedAppMasterMap);
+    Assert.assertEquals(2, unmanagedAppMasterMap.size());
+
+    // try to stop uamPool
+    uamPool.stop();
+    Assert.assertTrue(uamPool.waitForServiceToStop(2000));
+    // process force finish Application in a separate thread, not blocking the main thread
+    Assert.assertEquals(Service.STATE.STOPPED, uamPool.getServiceState());
+
+    // Wait for the thread to terminate, check if uamPool#unmanagedAppMasterMap is 0
+    Thread finishApplicationThread = uamPool.getFinishApplicationThread();
+    GenericTestUtils.waitFor(() -> !finishApplicationThread.isAlive(),
+        100, 2000);
+    Assert.assertEquals(0, unmanagedAppMasterMap.size());
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 6f84bfc59bfa9..24be1ebd2a96e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -152,8 +152,22 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey.contribs</groupId>
@@ -186,6 +200,11 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minikdc</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <profiles>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 7d18016d95ade..705ef88dee357 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -20,6 +20,7 @@
 
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
 import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.numaAwarenessEnabled;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import java.io.DataOutputStream;
@@ -28,12 +29,13 @@
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.fs.FileContext;
@@ -51,14 +53,19 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocator;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
@@ -86,6 +93,10 @@ public class DefaultContainerExecutor extends ContainerExecutor {
 
   private String logDirPermissions = null;
 
+  private NumaResourceAllocator numaResourceAllocator;
+
+
+  private String numactl;
   /**
    * Default constructor for use in testing.
    */
@@ -137,7 +148,19 @@ protected void setScriptExecutable(Path script, String owner)
 
   @Override
   public void init(Context nmContext) throws IOException {
-    // nothing to do or verify here
+    if(numaAwarenessEnabled(getConf())) {
+      numaResourceAllocator = new NumaResourceAllocator(nmContext);
+      numactl = this.getConf().get(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD,
+              YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD);
+      try {
+        numaResourceAllocator.init(this.getConf());
+        LOG.info("NUMA resources allocation is enabled in DefaultContainer Executor," +
+                " Successfully initialized NUMA resources allocator.");
+      } catch (YarnException e) {
+        LOG.warn("Improper NUMA configuration provided.", e);
+        throw new IOException("Failed to initialize configured numa subsystem!");
+      }
+    }
   }
 
   @Override
@@ -300,11 +323,28 @@ public int launchContainer(ContainerStartContext ctx)
       setScriptExecutable(launchDst, user);
       setScriptExecutable(sb.getWrapperScriptPath(), user);
 
+      // adding numa commands based on configuration
+      String[] numaCommands = new String[]{};
+
+      if (numaResourceAllocator != null) {
+        try {
+          NumaResourceAllocation numaResourceAllocation =
+                  numaResourceAllocator.allocateNumaNodes(container);
+          if (numaResourceAllocation != null) {
+            numaCommands = getNumaCommands(numaResourceAllocation);
+          }
+        } catch (ResourceHandlerException e) {
+          LOG.error("NumaResource Allocation failed!", e);
+          throw new IOException("NumaResource Allocation Error!", e);
+        }
+      }
+
       shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
-          containerIdStr, user, pidFile, container.getResource(),
-          new File(containerWorkDir.toUri().getPath()),
-          container.getLaunchContext().getEnvironment());
-      
+              containerIdStr, user, pidFile, container.getResource(),
+              new File(containerWorkDir.toUri().getPath()),
+              container.getLaunchContext().getEnvironment(),
+              numaCommands);
+
       if (isContainerActive(containerId)) {
         shExec.execute();
       } else {
@@ -350,6 +390,7 @@ public int launchContainer(ContainerStartContext ctx)
       return exitCode;
     } finally {
       if (shExec != null) shExec.close();
+      postComplete(containerId);
     }
     return 0;
   }
@@ -372,16 +413,22 @@ public int relaunchContainer(ContainerStartContext ctx)
    * as the current working directory for the command. If null,
    * the current working directory is not modified.
    * @param environment the container environment
+   * @param numaCommands list of prefix numa commands
    * @return the new {@link ShellCommandExecutor}
    * @see ShellCommandExecutor
    */
-  protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, 
-      String containerIdStr, String user, Path pidFile, Resource resource,
-      File workDir, Map<String, String> environment) {
-    
+  protected CommandExecutor buildCommandExecutor(String wrapperScriptPath,
+                            String containerIdStr, String user, Path pidFile, Resource resource,
+                            File workDir, Map<String, String> environment, String[] numaCommands) {
+
     String[] command = getRunCommand(wrapperScriptPath,
         containerIdStr, user, pidFile, this.getConf(), resource);
 
+    // check if numa commands are passed and append it as prefix commands
+    if(numaCommands != null && numaCommands.length!=0) {
+      command = concatStringCommands(numaCommands, command);
+    }
+
     LOG.info("launchContainer: {}", Arrays.toString(command));
     return new ShellCommandExecutor(
         command,
@@ -1040,4 +1087,92 @@ public void updateYarnSysFS(Context ctx, String user,
       String appId, String spec) throws IOException {
     throw new ServiceStateException("Implementation unavailable");
   }
+
+  @Override
+  public int reacquireContainer(ContainerReacquisitionContext ctx)
+          throws IOException, InterruptedException {
+    try {
+      if (numaResourceAllocator != null) {
+        numaResourceAllocator.recoverNumaResource(ctx.getContainerId());
+      }
+      return super.reacquireContainer(ctx);
+    } finally {
+      postComplete(ctx.getContainerId());
+    }
+  }
+
+  /**
+   * clean up and release of resources.
+   *
+   * @param containerId containerId of running container
+   */
+  public void postComplete(final ContainerId containerId) {
+    if (numaResourceAllocator != null) {
+      try {
+        numaResourceAllocator.releaseNumaResource(containerId);
+      } catch (ResourceHandlerException e) {
+        LOG.warn("NumaResource release failed for " +
+                "containerId: {}. Exception: ", containerId, e);
+      }
+    }
+  }
+
+  /**
+   * @param resourceAllocation NonNull NumaResourceAllocation object reference
+   * @return Array of numa specific commands
+   */
+  String[] getNumaCommands(NumaResourceAllocation resourceAllocation) {
+    String[] numaCommand = new String[3];
+    numaCommand[0] = numactl;
+    numaCommand[1] = "--interleave=" + String.join(",", resourceAllocation.getMemNodes());
+    numaCommand[2] = "--cpunodebind=" + String.join(",", resourceAllocation.getCpuNodes());
+    return numaCommand;
+
+  }
+
+  /**
+   * @param firstStringArray  Array of String
+   * @param secondStringArray Array of String
+   * @return combined array of string where first elements are from firstStringArray
+   * and later are the elements from secondStringArray
+   */
+  String[] concatStringCommands(String[] firstStringArray, String[] secondStringArray) {
+
+    if(firstStringArray == null && secondStringArray == null) {
+      return secondStringArray;
+    }
+
+    else if(firstStringArray == null || firstStringArray.length == 0) {
+      return secondStringArray;
+    }
+
+    else if(secondStringArray == null || secondStringArray.length == 0){
+      return firstStringArray;
+    }
+
+    int len = firstStringArray.length + secondStringArray.length;
+
+    String[] ret = new String[len];
+    int idx = 0;
+    for (String s : firstStringArray) {
+      ret[idx] = s;
+      idx++;
+    }
+    for (String s : secondStringArray) {
+      ret[idx] = s;
+      idx++;
+    }
+    return ret;
+  }
+
+  @VisibleForTesting
+  public void setNumaResourceAllocator(NumaResourceAllocator numaResourceAllocator) {
+    this.numaResourceAllocator = numaResourceAllocator;
+  }
+
+  @VisibleForTesting
+  public void setNumactl(String numactl) {
+    this.numactl = numactl;
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index e899215291b36..ea4595dffc48d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -175,8 +175,12 @@ public enum ExitCode {
     COULD_NOT_CREATE_WORK_DIRECTORIES(35),
     COULD_NOT_CREATE_APP_LOG_DIRECTORIES(36),
     COULD_NOT_CREATE_TMP_DIRECTORIES(37),
-    ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38);
-
+    ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS(38),
+    CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK(80),
+    TOO_LONG_EXECUTOR_PATH(81),
+    CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL(82),
+    CANNOT_GET_EXECUTABLE_NAME_FROM_PID(83),
+    WRONG_PATH_OF_EXECUTABLE(84);
     private final int code;
 
     ExitCode(int exitCode) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 4dadf9c62e4c5..be7d7e52ba9da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -201,7 +201,7 @@ protected void serviceInit(Configuration conf) throws Exception {
     // Update configured resources via plugins.
     updateConfiguredResourcesViaPlugins(totalResource);
 
-    LOG.info("Nodemanager resources is set to: " + totalResource);
+    LOG.info("Nodemanager resources is set to: {}.", totalResource);
 
     metrics.addResource(totalResource);
 
@@ -247,9 +247,8 @@ protected void serviceInit(Configuration conf) throws Exception {
     LOG.debug("{} :{}", YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
         durationToTrackStoppedContainers);
     super.serviceInit(conf);
-    LOG.info("Initialized nodemanager with :" +
-        " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
-        " virtual-cores=" + virtualCores);
+    LOG.info("Initialized nodemanager with : physical-memory={} virtual-memory={} " +
+        "virtual-cores={}.", memoryMb, virtualMemoryMb, virtualCores);
 
     this.logAggregationEnabled =
         conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
@@ -264,7 +263,7 @@ protected void serviceStart() throws Exception {
 
     // NodeManager is the last service to start, so NodeId is available.
     this.nodeId = this.context.getNodeId();
-    LOG.info("Node ID assigned is : " + this.nodeId);
+    LOG.info("Node ID assigned is : {}.", this.nodeId);
     this.httpPort = this.context.getHttpPort();
     this.nodeManagerVersionId = YarnVersionInfo.getVersion();
     try {
@@ -312,10 +311,9 @@ private void unRegisterNM() {
     request.setNodeId(this.nodeId);
     try {
       resourceTracker.unRegisterNodeManager(request);
-      LOG.info("Successfully Unregistered the Node " + this.nodeId
-          + " with ResourceManager.");
+      LOG.info("Successfully Unregistered the Node {} with ResourceManager.", this.nodeId);
     } catch (Exception e) {
-      LOG.warn("Unregistration of the Node " + this.nodeId + " failed.", e);
+      LOG.warn("Unregistration of the Node {} failed.", this.nodeId, e);
     }
   }
 
@@ -399,7 +397,7 @@ nodeManagerVersionId, containerReports, getRunningApplications(),
               nodeLabels, physicalResource, nodeAttributes, nodeStatus);
 
       if (containerReports != null && !containerReports.isEmpty()) {
-        LOG.info("Registering with RM using containers :" + containerReports);
+        LOG.info("Registering with RM using containers.size : {}." + containerReports.size());
       }
       if (logAggregationEnabled) {
         // pull log aggregation status for application running in this NM
@@ -641,6 +639,7 @@ private List<ApplicationId> getRunningApplications() {
         runningApplications.add(appEntry.getKey());
       }
     }
+    LOG.info("Running Applications Size : {}.", runningApplications.size());
     return runningApplications;
   }
 
@@ -667,8 +666,8 @@ private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
       }
     }
     if (!containerStatuses.isEmpty()) {
-      LOG.info("Sending out " + containerStatuses.size()
-          + " NM container statuses: " + containerStatuses);
+      LOG.info("Sending out {} container NM container statuses: {}.",
+          containerStatuses.size(), containerStatuses);
     }
     return containerStatuses;
   }
@@ -724,8 +723,7 @@ public void removeOrTrackCompletedContainersFromContext(
     }
 
     if (!removedContainers.isEmpty()) {
-      LOG.info("Removed completed containers from NM context: "
-          + removedContainers);
+      LOG.info("Removed completed containers from NM context: {}.", removedContainers);
     }
   }
 
@@ -792,7 +790,7 @@ public void removeVeryOldStoppedContainersFromCache() {
             try {
               context.getNMStateStore().removeContainer(cid);
             } catch (IOException e) {
-              LOG.error("Unable to remove container " + cid + " in store", e);
+              LOG.error("Unable to remove container {} in store.", cid, e);
             }
           }
         }
@@ -839,18 +837,15 @@ private boolean handleShutdownOrResyncCommand(
     if (response.getNodeAction() == NodeAction.SHUTDOWN) {
       LOG.warn("Received SHUTDOWN signal from Resourcemanager as part of"
           + " heartbeat, hence shutting down.");
-      LOG.warn("Message from ResourceManager: "
-          + response.getDiagnosticsMessage());
+      LOG.warn("Message from ResourceManager: {}.", response.getDiagnosticsMessage());
       context.setDecommissioned(true);
       dispatcher.getEventHandler().handle(
           new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
       return true;
     }
     if (response.getNodeAction() == NodeAction.RESYNC) {
-      LOG.warn("Node is out of sync with ResourceManager,"
-          + " hence resyncing.");
-      LOG.warn("Message from ResourceManager: "
-          + response.getDiagnosticsMessage());
+      LOG.warn("Node is out of sync with ResourceManager, hence resyncing.");
+      LOG.warn("Message from ResourceManager: {}.", response.getDiagnosticsMessage());
       // Invalidate the RMIdentifier while resync
       NodeStatusUpdaterImpl.this.rmIdentifier =
           ResourceManagerConstants.RM_INVALID_IDENTIFIER;
@@ -1095,8 +1090,7 @@ protected void validate(Set<NodeAttribute> nodeAttributes)
       try {
         NodeLabelUtil.validateNodeAttributes(nodeAttributes);
       } catch (IOException e) {
-        LOG.error(
-            "Invalid node attribute(s) from Provider : " + e.getMessage());
+        LOG.error("Invalid node attribute(s) from Provider : {}.", e.getMessage());
         throw e;
       }
     }
@@ -1136,9 +1130,8 @@ public void verifyRMHeartbeatResponseForNodeAttributes(
         } else {
           // case where updated node attributes from NodeAttributesProvider
           // is sent to RM and RM rejected the attributes
-          LOG.error("NM node attributes {" + getPreviousValue()
-              + "} were not accepted by RM and message from RM : " + response
-              .getDiagnosticsMessage());
+          LOG.error("NM node attributes [{}] were not accepted by RM and message from RM : {}.",
+              getPreviousValue(), response.getDiagnosticsMessage());
         }
       }
     }
@@ -1262,7 +1255,7 @@ protected void validate(Set<NodeLabel> nodeLabels)
         }
       }
       if (hasInvalidLabel) {
-        LOG.error("Invalid Node Label(s) from Provider : " + errorMsg);
+        LOG.error("Invalid Node Label(s) from Provider : {}.", errorMsg);
         throw new IOException(errorMsg.toString());
       }
     }
@@ -1287,10 +1280,8 @@ public void verifyRMHeartbeatResponseForNodeLabels(
         } else {
           // case where updated labels from NodeLabelsProvider is sent to RM and
           // RM rejected the labels
-          LOG.error(
-              "NM node labels {" + StringUtils.join(",", getPreviousValue())
-                  + "} were not accepted by RM and message from RM : "
-                  + response.getDiagnosticsMessage());
+          LOG.error("NM node labels [{}] were not accepted by RM and message from RM : {}.",
+              StringUtils.join(",", getPreviousValue()), response.getDiagnosticsMessage());
         }
       }
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index c13efadff1b33..9d57f8fff4fe5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -449,7 +449,7 @@ private enum State {
       COMPLETE
     };
     
-    private State state;;
+    private State state;
     
     private final String cwd;
     private final String jobName;
@@ -718,10 +718,10 @@ public void startLocalizer(LocalizerStartContext ctx) throws IOException,
   @Override
   protected CommandExecutor buildCommandExecutor(String wrapperScriptPath,
       String containerIdStr, String userName, Path pidFile, Resource resource,
-      File wordDir, Map<String, String> environment) {
+      File wordDir, Map<String, String> environment, String[] numaCommands) {
      return new WintuilsProcessStubExecutor(
          wordDir.toString(),
-         containerIdStr, userName, pidFile.toString(), 
+         containerIdStr, userName, pidFile.toString(),
          "cmd /c " + wrapperScriptPath);
    }
    
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
index 92afcb7ad12d6..86499c84b59c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
@@ -27,7 +27,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 
 /**
- * Interface that can be used by the intercepter plugins to get the information
+ * Interface that can be used by the interceptor plugins to get the information
  * about one application.
  *
  */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
index 8a0209507b420..881e9425f013c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
@@ -32,7 +32,7 @@
 
 /**
  * Encapsulates the information about one application that is needed by the
- * request intercepters.
+ * request interceptors.
  *
  */
 public class AMRMProxyApplicationContextImpl implements
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java
index 241eeeb62a85a..f5a30828a4dd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java
@@ -45,6 +45,16 @@ public final class AMRMProxyMetrics {
   private MutableGaugeLong failedAllocateRequests;
   @Metric("# of failed application recoveries")
   private MutableGaugeLong failedAppRecoveryCount;
+  @Metric("# of failed application stop")
+  private MutableGaugeLong failedAppStopRequests;
+  @Metric("# of failed update token")
+  private MutableGaugeLong failedUpdateAMRMTokenRequests;
+  @Metric("# all allocate requests count")
+  private MutableGaugeLong allocateCount;
+  @Metric("# all requests count")
+  private MutableGaugeLong requestCount;
+
+
   // Aggregate metrics are shared, and don't have to be looked up per call
   @Metric("Application start request latency(ms)")
   private MutableRate totalSucceededAppStartRequests;
@@ -54,11 +64,22 @@ public final class AMRMProxyMetrics {
   private MutableRate totalSucceededFinishAMRequests;
   @Metric("Allocate latency(ms)")
   private MutableRate totalSucceededAllocateRequests;
+  @Metric("Application stop request latency(ms)")
+  private MutableRate totalSucceededAppStopRequests;
+  @Metric("Recover latency(ms)")
+  private MutableRate totalSucceededRecoverRequests;
+  @Metric("UpdateAMRMToken latency(ms)")
+  private MutableRate totalSucceededUpdateAMRMTokenRequests;
+
   // Quantile latency in ms - this is needed for SLA (95%, 99%, etc)
   private MutableQuantiles applicationStartLatency;
   private MutableQuantiles registerAMLatency;
   private MutableQuantiles finishAMLatency;
   private MutableQuantiles allocateLatency;
+  private MutableQuantiles recoverLatency;
+  private MutableQuantiles applicationStopLatency;
+  private MutableQuantiles updateAMRMTokenLatency;
+
   private static volatile AMRMProxyMetrics instance = null;
   private MetricsRegistry registry;
 
@@ -78,6 +99,15 @@ private AMRMProxyMetrics() {
     allocateLatency = registry
         .newQuantiles("allocateLatency", "latency of allocate", "ops",
             "latency", 10);
+    applicationStopLatency = registry
+        .newQuantiles("applicationStopLatency", "latency of app stop", "ops",
+            "latency", 10);
+    recoverLatency = registry
+        .newQuantiles("recoverLatency", "latency of recover", "ops",
+            "latency", 10);
+    updateAMRMTokenLatency = registry
+        .newQuantiles("updateAMRMTokenLatency", "latency of update amrm token", "ops",
+            "latency", 10);
   }
 
   /**
@@ -146,16 +176,57 @@ long getNumSucceededAllocateRequests() {
     return totalSucceededAllocateRequests.lastStat().numSamples();
   }
 
+  @VisibleForTesting
+  long getNumSucceededAppStopRequests() {
+    return totalSucceededAppStopRequests.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  long getNumSucceededRecoverRequests() {
+    return totalSucceededRecoverRequests.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  long getNumSucceededUpdateAMRMTokenRequests() {
+    return totalSucceededUpdateAMRMTokenRequests.lastStat().numSamples();
+  }
+
+
   @VisibleForTesting
   double getLatencySucceededAllocateRequests() {
     return totalSucceededAllocateRequests.lastStat().mean();
   }
 
+  @VisibleForTesting
+  double getLatencySucceededAppStopRequests() {
+    return totalSucceededAppStopRequests.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  double getLatencySucceededRecoverRequests() {
+    return totalSucceededRecoverRequests.lastStat().mean();
+  }
+
   public void succeededAllocateRequests(long duration) {
     totalSucceededAllocateRequests.add(duration);
     allocateLatency.add(duration);
   }
 
+  public void succeededAppStopRequests(long duration) {
+    totalSucceededAppStopRequests.add(duration);
+    applicationStopLatency.add(duration);
+  }
+
+  public void succeededRecoverRequests(long duration) {
+    totalSucceededRecoverRequests.add(duration);
+    recoverLatency.add(duration);
+  }
+
+  public void succeededUpdateTokenRequests(long duration) {
+    totalSucceededUpdateAMRMTokenRequests.add(duration);
+    updateAMRMTokenLatency.add(duration);
+  }
+
   long getFailedAppStartRequests() {
     return failedAppStartRequests.value();
   }
@@ -195,4 +266,36 @@ long getFailedAppRecoveryCount() {
   public void incrFailedAppRecoveryCount() {
     failedAppRecoveryCount.incr();
   }
+
+  long getFailedAppStopRequests() {
+    return failedAppStopRequests.value();
+  }
+
+  public void incrFailedAppStopRequests() {
+    failedAppStopRequests.incr();
+  }
+
+  long getFailedUpdateAMRMTokenRequests() {
+    return failedUpdateAMRMTokenRequests.value();
+  }
+
+  public void incrFailedUpdateAMRMTokenRequests() {
+    failedUpdateAMRMTokenRequests.incr();
+  }
+
+  public void incrAllocateCount() {
+    allocateCount.incr();
+  }
+
+  public void incrRequestCount() {
+    requestCount.incr();
+  }
+
+  long getAllocateCount() {
+    return allocateCount.value();
+  }
+
+  long getRequestCount() {
+    return requestCount.value();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 821524dd6619d..82873e07cdb09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -53,6 +54,7 @@
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -81,7 +83,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * AMRMProxyService is a service that runs on each node manager that can be used
@@ -123,12 +125,9 @@ public AMRMProxyService(Context nmContext, AsyncDispatcher dispatcher) {
     Preconditions.checkArgument(dispatcher != null, "dispatcher is null");
     this.nmContext = nmContext;
     this.dispatcher = dispatcher;
-    this.applPipelineMap =
-        new ConcurrentHashMap<ApplicationId, RequestInterceptorChainWrapper>();
-
-    this.dispatcher.register(ApplicationEventType.class,
-        new ApplicationEventHandler());
+    this.applPipelineMap = new ConcurrentHashMap<>();
 
+    this.dispatcher.register(ApplicationEventType.class, new ApplicationEventHandler());
     metrics = AMRMProxyMetrics.getMetrics();
   }
 
@@ -155,7 +154,7 @@ protected void serviceInit(Configuration conf) throws Exception {
 
   @Override
   protected void serviceStart() throws Exception {
-    LOG.info("Starting AMRMProxyService");
+    LOG.info("Starting AMRMProxyService.");
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
     UserGroupInformation.setConfiguration(conf);
@@ -182,27 +181,22 @@ protected void serviceStart() throws Exception {
             listenerEndpoint, serverConf, this.secretManager,
             numWorkerThreads);
 
-    if (conf
-        .getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
-            false)) {
-        this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
+    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
+      this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
     }
 
     this.server.start();
-    LOG.info("AMRMProxyService listening on address: "
-        + this.server.getListenerAddress());
+    LOG.info("AMRMProxyService listening on address: {}.", this.server.getListenerAddress());
     super.serviceStart();
   }
 
   @Override
   protected void serviceStop() throws Exception {
-    LOG.info("Stopping AMRMProxyService");
+    LOG.info("Stopping AMRMProxyService.");
     if (this.server != null) {
       this.server.stop();
     }
-
     this.secretManager.stop();
-
     super.serviceStop();
   }
 
@@ -212,19 +206,21 @@ protected void serviceStop() throws Exception {
    * @throws IOException if recover fails
    */
   public void recover() throws IOException {
-    LOG.info("Recovering AMRMProxyService");
+    LOG.info("Recovering AMRMProxyService.");
 
     RecoveredAMRMProxyState state =
         this.nmContext.getNMStateStore().loadAMRMProxyState();
 
     this.secretManager.recover(state);
 
-    LOG.info("Recovering {} running applications for AMRMProxy",
+    LOG.info("Recovering {} running applications for AMRMProxy.",
         state.getAppContexts().size());
+
     for (Map.Entry<ApplicationAttemptId, Map<String, byte[]>> entry : state
         .getAppContexts().entrySet()) {
       ApplicationAttemptId attemptId = entry.getKey();
-      LOG.info("Recovering app attempt {}", attemptId);
+      LOG.info("Recovering app attempt {}.", attemptId);
+      long startTime = clock.getTime();
 
       // Try recover for the running application attempt
       try {
@@ -233,19 +229,18 @@ public void recover() throws IOException {
         for (Map.Entry<String, byte[]> contextEntry : entry.getValue()
             .entrySet()) {
           if (contextEntry.getKey().equals(NMSS_USER_KEY)) {
-            user = new String(contextEntry.getValue(), "UTF-8");
+            user = new String(contextEntry.getValue(), StandardCharsets.UTF_8);
           } else if (contextEntry.getKey().equals(NMSS_AMRMTOKEN_KEY)) {
             amrmToken = new Token<>();
             amrmToken.decodeFromUrlString(
-                new String(contextEntry.getValue(), "UTF-8"));
+                new String(contextEntry.getValue(), StandardCharsets.UTF_8));
             // Clear the service field, as if RM just issued the token
             amrmToken.setService(new Text());
           }
         }
 
         if (amrmToken == null) {
-          throw new IOException(
-              "No amrmToken found for app attempt " + attemptId);
+          throw new IOException("No amrmToken found for app attempt " + attemptId);
         }
         if (user == null) {
           throw new IOException("No user found for app attempt " + attemptId);
@@ -258,14 +253,14 @@ public void recover() throws IOException {
         // Retrieve the AM container credentials from NM context
         Credentials amCred = null;
         for (Container container : this.nmContext.getContainers().values()) {
-          LOG.debug("From NM Context container {}", container.getContainerId());
+          LOG.debug("From NM Context container {}.", container.getContainerId());
           if (container.getContainerId().getApplicationAttemptId().equals(
               attemptId) && container.getContainerTokenIdentifier() != null) {
-            LOG.debug("Container type {}",
+            LOG.debug("Container type {}.",
                 container.getContainerTokenIdentifier().getContainerType());
             if (container.getContainerTokenIdentifier()
                 .getContainerType() == ContainerType.APPLICATION_MASTER) {
-              LOG.info("AM container {} found in context, has credentials: {}",
+              LOG.info("AM container {} found in context, has credentials: {}.",
                   container.getContainerId(),
                   (container.getCredentials() != null));
               amCred = container.getCredentials();
@@ -274,15 +269,17 @@ public void recover() throws IOException {
         }
         if (amCred == null) {
           LOG.error("No credentials found for AM container of {}. "
-              + "Yarn registry access might not work", attemptId);
+              + "Yarn registry access might not work.", attemptId);
         }
 
-        // Create the intercepter pipeline for the AM
+        // Create the interceptor pipeline for the AM
         initializePipeline(attemptId, user, amrmToken, localToken,
             entry.getValue(), true, amCred);
+        long endTime = clock.getTime();
+        this.metrics.succeededRecoverRequests(endTime - startTime);
       } catch (Throwable e) {
-        LOG.error("Exception when recovering " + attemptId
-            + ", removing it from NMStateStore and move on", e);
+        LOG.error("Exception when recovering {}, removing it from NMStateStore and move on.",
+            attemptId, e);
         this.metrics.incrFailedAppRecoveryCount();
         this.nmContext.getNMStateStore().removeAMRMProxyAppContext(attemptId);
       }
@@ -292,26 +289,28 @@ public void recover() throws IOException {
   /**
    * This is called by the AMs started on this node to register with the RM.
    * This method does the initial authorization and then forwards the request to
-   * the application instance specific intercepter chain.
+   * the application instance specific interceptor chain.
    */
   @Override
   public RegisterApplicationMasterResponse registerApplicationMaster(
       RegisterApplicationMasterRequest request) throws YarnException,
       IOException {
+    this.metrics.incrRequestCount();
     long startTime = clock.getTime();
     try {
       RequestInterceptorChainWrapper pipeline =
           authorizeAndGetInterceptorChain();
-      LOG.info("Registering application master." + " Host:" + request.getHost()
-          + " Port:" + request.getRpcPort() + " Tracking Url:" + request
-          .getTrackingUrl() + " for application " + pipeline
-          .getApplicationAttemptId());
+
+      LOG.info("RegisteringAM Host: {}, Port: {}, Tracking Url: {} for application {}. ",
+          request.getHost(), request.getRpcPort(), request.getTrackingUrl(),
+          pipeline.getApplicationAttemptId());
+
       RegisterApplicationMasterResponse response =
           pipeline.getRootInterceptor().registerApplicationMaster(request);
 
       long endTime = clock.getTime();
       this.metrics.succeededRegisterAMRequests(endTime - startTime);
-      LOG.info("RegisterAM processing finished in {} ms for application {}",
+      LOG.info("RegisterAM processing finished in {} ms for application {}.",
           endTime - startTime, pipeline.getApplicationAttemptId());
       return response;
     } catch (Throwable t) {
@@ -323,24 +322,25 @@ public RegisterApplicationMasterResponse registerApplicationMaster(
   /**
    * This is called by the AMs started on this node to unregister from the RM.
    * This method does the initial authorization and then forwards the request to
-   * the application instance specific intercepter chain.
+   * the application instance specific interceptor chain.
    */
   @Override
   public FinishApplicationMasterResponse finishApplicationMaster(
       FinishApplicationMasterRequest request) throws YarnException,
       IOException {
+    this.metrics.incrRequestCount();
     long startTime = clock.getTime();
     try {
       RequestInterceptorChainWrapper pipeline =
           authorizeAndGetInterceptorChain();
-      LOG.info("Finishing application master for {}. Tracking Url: {}",
+      LOG.info("Finishing application master for {}. Tracking Url: {}.",
           pipeline.getApplicationAttemptId(), request.getTrackingUrl());
       FinishApplicationMasterResponse response =
           pipeline.getRootInterceptor().finishApplicationMaster(request);
 
       long endTime = clock.getTime();
       this.metrics.succeededFinishAMRequests(endTime - startTime);
-      LOG.info("FinishAM finished with isUnregistered = {} in {} ms for {}",
+      LOG.info("FinishAM finished with isUnregistered = {} in {} ms for {}.",
           response.getIsUnregistered(), endTime - startTime,
           pipeline.getApplicationAttemptId());
       return response;
@@ -354,12 +354,13 @@ public FinishApplicationMasterResponse finishApplicationMaster(
    * This is called by the AMs started on this node to send heart beat to RM.
    * This method does the initial authorization and then forwards the request to
    * the application instance specific pipeline, which is a chain of request
-   * intercepter objects. One application request processing pipeline is created
+   * interceptor objects. One application request processing pipeline is created
    * per AM instance.
    */
   @Override
   public AllocateResponse allocate(AllocateRequest request)
       throws YarnException, IOException {
+    this.metrics.incrAllocateCount();
     long startTime = clock.getTime();
     try {
       AMRMTokenIdentifier amrmTokenIdentifier =
@@ -373,7 +374,7 @@ public AllocateResponse allocate(AllocateRequest request)
 
       long endTime = clock.getTime();
       this.metrics.succeededAllocateRequests(endTime - startTime);
-      LOG.info("Allocate processing finished in {} ms for application {}",
+      LOG.info("Allocate processing finished in {} ms for application {}.",
           endTime - startTime, pipeline.getApplicationAttemptId());
       return allocateResponse;
     } catch (Throwable t) {
@@ -392,6 +393,7 @@ public AllocateResponse allocate(AllocateRequest request)
    */
   public void processApplicationStartRequest(StartContainerRequest request)
       throws IOException, YarnException {
+    this.metrics.incrRequestCount();
     long startTime = clock.getTime();
     try {
       ContainerTokenIdentifier containerTokenIdentifierForKey =
@@ -408,8 +410,7 @@ public void processApplicationStartRequest(StartContainerRequest request)
       if (!checkIfAppExistsInStateStore(applicationID)) {
         return;
       }
-      LOG.info("Callback received for initializing request "
-          + "processing pipeline for an AM");
+      LOG.info("Callback received for initializing request processing pipeline for an AM.");
       Credentials credentials = YarnServerSecurityUtils
           .parseCredentials(request.getContainerLaunchContext());
 
@@ -417,8 +418,7 @@ public void processApplicationStartRequest(StartContainerRequest request)
           getFirstAMRMToken(credentials.getAllTokens());
       if (amrmToken == null) {
         throw new YarnRuntimeException(
-            "AMRMToken not found in the start container request for "
-                + "application:" + appAttemptId.toString());
+            "AMRMToken not found in the start container request for application:" + appAttemptId);
       }
 
       // Substitute the existing AMRM Token with a local one. Keep the rest of
@@ -445,7 +445,7 @@ public void processApplicationStartRequest(StartContainerRequest request)
   }
 
   /**
-   * Initializes the request intercepter pipeline for the specified application.
+   * Initializes the request interceptor pipeline for the specified application.
    *
    * @param applicationAttemptId attempt id
    * @param user user name
@@ -465,8 +465,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
           .containsKey(applicationAttemptId.getApplicationId())) {
         LOG.warn("Request to start an already existing appId was received. "
             + " This can happen if an application failed and a new attempt "
-            + "was created on this machine.  ApplicationId: "
-            + applicationAttemptId.toString());
+            + "was created on this machine.  ApplicationId: {}.", applicationAttemptId);
 
         RequestInterceptorChainWrapper chainWrapperBackup =
             this.applPipelineMap.get(applicationAttemptId.getApplicationId());
@@ -476,8 +475,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
                 .equals(applicationAttemptId)) {
           // TODO: revisit in AMRMProxy HA in YARN-6128
           // Remove the existing pipeline
-          LOG.info("Remove the previous pipeline for ApplicationId: "
-              + applicationAttemptId.toString());
+          LOG.info("Remove the previous pipeline for ApplicationId: {}.", applicationAttemptId);
           RequestInterceptorChainWrapper pipeline =
               applPipelineMap.remove(applicationAttemptId.getApplicationId());
 
@@ -485,19 +483,17 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
             try {
               this.nmContext.getNMStateStore()
                   .removeAMRMProxyAppContext(applicationAttemptId);
-            } catch (IOException e) {
-              LOG.error("Error removing AMRMProxy application context for "
-                  + applicationAttemptId, e);
+            } catch (IOException ioe) {
+              LOG.error("Error removing AMRMProxy application context for {}.",
+                  applicationAttemptId, ioe);
             }
           }
 
           try {
             pipeline.getRootInterceptor().shutdown();
           } catch (Throwable ex) {
-            LOG.warn(
-                "Failed to shutdown the request processing pipeline for app:"
-                    + applicationAttemptId.getApplicationId(),
-                ex);
+            LOG.warn("Failed to shutdown the request processing pipeline for app: {}.",
+                applicationAttemptId.getApplicationId(), ex);
           }
         } else {
           return;
@@ -510,12 +506,11 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
     }
 
     // We register the pipeline instance in the map first and then initialize it
-    // later because chain initialization can be expensive and we would like to
+    // later because chain initialization can be expensive, and we would like to
     // release the lock as soon as possible to prevent other applications from
     // blocking when one application's chain is initializing
     LOG.info("Initializing request processing pipeline for application. "
-        + " ApplicationId:" + applicationAttemptId + " for the user: "
-        + user);
+        + " ApplicationId: {} for the user: {}.", applicationAttemptId, user);
 
     try {
       RequestInterceptor interceptorChain =
@@ -525,8 +520,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
               user, amrmToken, localToken, credentials, this.registry));
       if (isRecovery) {
         if (recoveredDataMap == null) {
-          throw new YarnRuntimeException(
-              "null recoveredDataMap recieved for recover");
+          throw new YarnRuntimeException("null recoveredDataMap received for recover");
         }
         interceptorChain.recover(recoveredDataMap);
       }
@@ -535,13 +529,13 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
       if (!isRecovery && this.nmContext.getNMStateStore() != null) {
         try {
           this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry(
-              applicationAttemptId, NMSS_USER_KEY, user.getBytes("UTF-8"));
+              applicationAttemptId, NMSS_USER_KEY, user.getBytes(StandardCharsets.UTF_8));
           this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry(
               applicationAttemptId, NMSS_AMRMTOKEN_KEY,
-              amrmToken.encodeToUrlString().getBytes("UTF-8"));
+              amrmToken.encodeToUrlString().getBytes(StandardCharsets.UTF_8));
         } catch (IOException e) {
-          LOG.error("Error storing AMRMProxy application context entry for "
-              + applicationAttemptId, e);
+          LOG.error("Error storing AMRMProxy application context entry for {}.",
+              applicationAttemptId, e);
         }
       }
     } catch (Exception e) {
@@ -557,29 +551,27 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
    * @param applicationId application id
    */
   protected void stopApplication(ApplicationId applicationId) {
-    Preconditions.checkArgument(applicationId != null,
-        "applicationId is null");
+    this.metrics.incrRequestCount();
+    Preconditions.checkArgument(applicationId != null, "applicationId is null");
     RequestInterceptorChainWrapper pipeline =
         this.applPipelineMap.remove(applicationId);
+    boolean isStopSuccess = true;
+    long startTime = clock.getTime();
 
     if (pipeline == null) {
-      LOG.info(
-          "No interceptor pipeline for application {},"
-              + " likely because its AM is not run in this node.",
-          applicationId);
+      LOG.info("No interceptor pipeline for application {},"
+          + " likely because its AM is not run in this node.", applicationId);
+      isStopSuccess = false;
     } else {
       // Remove the appAttempt in AMRMTokenSecretManager
-      this.secretManager
-          .applicationMasterFinished(pipeline.getApplicationAttemptId());
-
-      LOG.info("Stopping the request processing pipeline for application: "
-          + applicationId);
+      this.secretManager.applicationMasterFinished(pipeline.getApplicationAttemptId());
+      LOG.info("Stopping the request processing pipeline for application: {}.", applicationId);
       try {
         pipeline.getRootInterceptor().shutdown();
       } catch (Throwable ex) {
-        LOG.warn(
-            "Failed to shutdown the request processing pipeline for app:"
-                + applicationId, ex);
+        LOG.warn("Failed to shutdown the request processing pipeline for app: {}.",
+            applicationId, ex);
+        isStopSuccess = false;
       }
 
       // Remove the app context from NMSS after the interceptors are shutdown
@@ -588,74 +580,83 @@ protected void stopApplication(ApplicationId applicationId) {
           this.nmContext.getNMStateStore()
               .removeAMRMProxyAppContext(pipeline.getApplicationAttemptId());
         } catch (IOException e) {
-          LOG.error("Error removing AMRMProxy application context for "
-              + applicationId, e);
+          LOG.error("Error removing AMRMProxy application context for {}.",
+              applicationId, e);
+          isStopSuccess = false;
         }
       }
     }
+
+    if (isStopSuccess) {
+      long endTime = clock.getTime();
+      this.metrics.succeededAppStopRequests(endTime - startTime);
+    } else {
+      this.metrics.incrFailedAppStopRequests();
+    }
   }
 
   private void updateAMRMTokens(AMRMTokenIdentifier amrmTokenIdentifier,
       RequestInterceptorChainWrapper pipeline,
       AllocateResponse allocateResponse) {
+
     AMRMProxyApplicationContextImpl context =
-        (AMRMProxyApplicationContextImpl) pipeline.getRootInterceptor()
-            .getApplicationContext();
+        (AMRMProxyApplicationContextImpl) pipeline.getRootInterceptor().getApplicationContext();
+
+    try {
+      long startTime = clock.getTime();
 
-    // check to see if the RM has issued a new AMRMToken & accordingly update
-    // the real ARMRMToken in the current context
-    if (allocateResponse.getAMRMToken() != null) {
-      LOG.info("RM rolled master-key for amrm-tokens");
+      // check to see if the RM has issued a new AMRMToken & accordingly update
+      // the real ARMRMToken in the current context
+      if (allocateResponse.getAMRMToken() != null) {
+        LOG.info("RM rolled master-key for amrm-tokens.");
 
-      org.apache.hadoop.yarn.api.records.Token token =
-          allocateResponse.getAMRMToken();
+        org.apache.hadoop.yarn.api.records.Token token = allocateResponse.getAMRMToken();
 
-      // Do not propagate this info back to AM
-      allocateResponse.setAMRMToken(null);
+        // Do not propagate this info back to AM
+        allocateResponse.setAMRMToken(null);
 
-      org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken =
-          ConverterUtils.convertFromYarn(token, (Text) null);
+        org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken =
+                ConverterUtils.convertFromYarn(token, (Text) null);
 
-      // Update the AMRMToken in context map, and in NM state store if it is
-      // different
-      if (context.setAMRMToken(newToken)
-          && this.nmContext.getNMStateStore() != null) {
-        try {
+        // Update the AMRMToken in context map, and in NM state store if it is
+        // different
+        if (context.setAMRMToken(newToken) && this.nmContext.getNMStateStore() != null) {
           this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry(
               context.getApplicationAttemptId(), NMSS_AMRMTOKEN_KEY,
-              newToken.encodeToUrlString().getBytes("UTF-8"));
-        } catch (IOException e) {
-          LOG.error("Error storing AMRMProxy application context entry for "
-              + context.getApplicationAttemptId(), e);
+              newToken.encodeToUrlString().getBytes(StandardCharsets.UTF_8));
         }
       }
-    }
 
-    // Check if the local AMRMToken is rolled up and update the context and
-    // response accordingly
-    MasterKeyData nextMasterKey =
-        this.secretManager.getNextMasterKeyData();
-
-    if (nextMasterKey != null
-        && nextMasterKey.getMasterKey().getKeyId() != amrmTokenIdentifier
-            .getKeyId()) {
-      Token<AMRMTokenIdentifier> localToken = context.getLocalAMRMToken();
-      if (nextMasterKey.getMasterKey().getKeyId() != context
-          .getLocalAMRMTokenKeyId()) {
-        LOG.info("The local AMRMToken has been rolled-over."
-            + " Send new local AMRMToken back to application: "
-            + pipeline.getApplicationId());
-        localToken =
-            this.secretManager.createAndGetAMRMToken(pipeline
-                .getApplicationAttemptId());
-        context.setLocalAMRMToken(localToken);
-      }
+      // Check if the local AMRMToken is rolled up and update the context and
+      // response accordingly
+      MasterKeyData nextMasterKey = this.secretManager.getNextMasterKeyData();
+
+      if (nextMasterKey != null) {
+        MasterKey masterKey = nextMasterKey.getMasterKey();
+        if (masterKey.getKeyId() != amrmTokenIdentifier.getKeyId()) {
+          Token<AMRMTokenIdentifier> localToken = context.getLocalAMRMToken();
+          if (masterKey.getKeyId() != context.getLocalAMRMTokenKeyId()) {
+            LOG.info("The local AMRMToken has been rolled-over."
+                + " Send new local AMRMToken back to application: {}",
+                pipeline.getApplicationId());
+            localToken = this.secretManager.createAndGetAMRMToken(
+                pipeline.getApplicationAttemptId());
+            context.setLocalAMRMToken(localToken);
+          }
 
-      allocateResponse
-          .setAMRMToken(org.apache.hadoop.yarn.api.records.Token
-              .newInstance(localToken.getIdentifier(), localToken
-                  .getKind().toString(), localToken.getPassword(),
-                  localToken.getService().toString()));
+          allocateResponse
+              .setAMRMToken(org.apache.hadoop.yarn.api.records.Token
+                  .newInstance(localToken.getIdentifier(), localToken
+                      .getKind().toString(), localToken.getPassword(),
+                      localToken.getService().toString()));
+        }
+      }
+      long endTime = clock.getTime();
+      this.metrics.succeededUpdateTokenRequests(endTime - startTime);
+    } catch (IOException e) {
+      LOG.error("Error storing AMRMProxy application context entry for {}.",
+          context.getApplicationAttemptId(), e);
+      this.metrics.incrFailedUpdateAMRMTokenRequests();
     }
   }
 
@@ -672,19 +673,19 @@ private AMRMProxyApplicationContext createApplicationMasterContext(
   }
 
   /**
-   * Gets the Request intercepter chains for all the applications.
+   * Gets the Request interceptor chains for all the applications.
    *
-   * @return the request intercepter chains.
+   * @return the request interceptor chains.
    */
   protected Map<ApplicationId, RequestInterceptorChainWrapper> getPipelines() {
     return this.applPipelineMap;
   }
 
   /**
-   * This method creates and returns reference of the first intercepter in the
-   * chain of request intercepter instances.
+   * This method creates and returns reference of the first interceptor in the
+   * chain of request interceptor instances.
    *
-   * @return the reference of the first intercepter in the chain
+   * @return the reference of the first interceptor in the chain
    */
   protected RequestInterceptor createRequestInterceptorChain() {
     Configuration conf = getConfig();
@@ -717,7 +718,7 @@ protected RequestInterceptor createRequestInterceptorChain() {
       } catch (ClassNotFoundException e) {
         throw new YarnRuntimeException(
             "Could not instantiate ApplicationMasterRequestInterceptor: "
-                + interceptorClassName, e);
+            + interceptorClassName, e);
       }
     }
 
@@ -729,10 +730,10 @@ protected RequestInterceptor createRequestInterceptorChain() {
   }
 
   /**
-   * Returns the comma separated intercepter class names from the configuration.
+   * Returns the comma separated interceptor class names from the configuration.
    *
    * @param conf configuration
-   * @return the intercepter class names as an instance of ArrayList
+   * @return the interceptor class names as an instance of ArrayList
    */
   private List<String> getInterceptorClassNames(Configuration conf) {
     String configuredInterceptorClassNames =
@@ -759,7 +760,7 @@ private List<String> getInterceptorClassNames(Configuration conf) {
    * Authorizes the request and returns the application specific request
    * processing pipeline.
    *
-   * @return the the intercepter wrapper instance
+   * @return the interceptor wrapper instance
    * @throws YarnException if fails
    */
   private RequestInterceptorChainWrapper authorizeAndGetInterceptorChain()
@@ -775,11 +776,10 @@ private RequestInterceptorChainWrapper getInterceptorChain(
         tokenIdentifier.getApplicationAttemptId();
 
     synchronized (this.applPipelineMap) {
-      if (!this.applPipelineMap.containsKey(appAttemptId
-          .getApplicationId())) {
+      if (!this.applPipelineMap.containsKey(appAttemptId.getApplicationId())) {
         throw new YarnException(
             "The AM request processing pipeline is not initialized for app: "
-                + appAttemptId.getApplicationId().toString());
+            + appAttemptId.getApplicationId());
       }
 
       return this.applPipelineMap.get(appAttemptId.getApplicationId());
@@ -827,29 +827,26 @@ public AMRMProxyTokenSecretManager getSecretManager() {
 
   /**
    * Private class for handling application stop events.
-   *
    */
   class ApplicationEventHandler implements EventHandler<ApplicationEvent> {
 
     @Override
     public void handle(ApplicationEvent event) {
       Application app =
-          AMRMProxyService.this.nmContext.getApplications().get(
-              event.getApplicationID());
+          AMRMProxyService.this.nmContext.getApplications().get(event.getApplicationID());
       if (app != null) {
         switch (event.getType()) {
         case APPLICATION_RESOURCES_CLEANEDUP:
-          LOG.info("Application stop event received for stopping AppId:"
-              + event.getApplicationID().toString());
+          LOG.info("Application stop event received for stopping AppId: {}.",
+              event.getApplicationID().toString());
           AMRMProxyService.this.stopApplication(event.getApplicationID());
           break;
         default:
-          LOG.debug("AMRMProxy is ignoring event: {}", event.getType());
+          LOG.debug("AMRMProxy is ignoring event: {}.", event.getType());
           break;
         }
       } else {
-        LOG.warn("Event " + event + " sent to absent application "
-            + event.getApplicationID());
+        LOG.warn("Event {} sent to absent application {}.", event, event.getApplicationID());
       }
     }
   }
@@ -866,20 +863,20 @@ public static class RequestInterceptorChainWrapper {
 
     /**
      * Initializes the wrapper with the specified parameters.
-     * 
-     * @param rootInterceptor the root request intercepter
-     * @param applicationAttemptId attempt id
+     *
+     * @param interceptor the root request interceptor
+     * @param appAttemptId attempt id
      */
-    public synchronized void init(RequestInterceptor rootInterceptor,
-        ApplicationAttemptId applicationAttemptId) {
-      this.rootInterceptor = rootInterceptor;
-      this.applicationAttemptId = applicationAttemptId;
+    public synchronized void init(RequestInterceptor interceptor,
+        ApplicationAttemptId appAttemptId) {
+      rootInterceptor = interceptor;
+      applicationAttemptId = appAttemptId;
     }
 
     /**
-     * Gets the root request intercepter.
+     * Gets the root request interceptor.
      *
-     * @return the root request intercepter
+     * @return the root request interceptor
      */
     public synchronized RequestInterceptor getRootInterceptor() {
       return rootInterceptor;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
index cfc44f72afd5e..d1fc341614471 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AbstractRequestInterceptor.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.util.Map;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -32,7 +32,7 @@
 
 /**
  * Implements the RequestInterceptor interface and provides common functionality
- * which can can be used and/or extended by other concrete intercepter classes.
+ * which can can be used and/or extended by other concrete interceptor classes.
  *
  */
 public abstract class AbstractRequestInterceptor implements
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index ce61993123fab..d3dff95abc88b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -35,6 +35,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
@@ -66,6 +67,7 @@
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.client.AMRMClientUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
@@ -85,6 +87,7 @@
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager;
@@ -96,13 +99,13 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Extends the AbstractRequestInterceptor and provides an implementation for
  * federation of YARN RM and scaling an application across multiple YARN
  * sub-clusters. All the federation specific implementation is encapsulated in
- * this class. This is always the last intercepter in the chain.
+ * this class. This is always the last interceptor in the chain.
  */
 public class FederationInterceptor extends AbstractRequestInterceptor {
   private static final Logger LOG =
@@ -198,8 +201,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    * that all the {@link AMRMClientRelayer} will be re-populated with all
    * pending requests.
    *
-   * TODO: When split-merge is not idempotent, this can lead to some
-   * over-allocation without a full cancel to RM.
    */
   private volatile boolean justRecovered;
 
@@ -355,143 +356,196 @@ public void init(AMRMProxyApplicationContext appContext) {
   @Override
   public void recover(Map<String, byte[]> recoveredDataMap) {
     super.recover(recoveredDataMap);
-    LOG.info("Recovering data for FederationInterceptor for {}",
-        this.attemptId);
+    LOG.info("Recovering data for FederationInterceptor for {}.", this.attemptId);
     this.justRecovered = true;
 
-    if (recoveredDataMap == null) {
+    if (recoveredDataMap == null || recoveredDataMap.isEmpty()) {
+      LOG.warn("recoveredDataMap isNull Or isEmpty, FederationInterceptor can't recover.");
       return;
     }
+
+    if (!recoveredDataMap.containsKey(NMSS_REG_REQUEST_KEY)) {
+      return;
+    }
+
     try {
+
       if (recoveredDataMap.containsKey(NMSS_REG_REQUEST_KEY)) {
+        byte[] appMasterRequestBytes = recoveredDataMap.get(NMSS_REG_REQUEST_KEY);
         RegisterApplicationMasterRequestProto pb =
-            RegisterApplicationMasterRequestProto
-                .parseFrom(recoveredDataMap.get(NMSS_REG_REQUEST_KEY));
-        this.amRegistrationRequest =
-            new RegisterApplicationMasterRequestPBImpl(pb);
-        LOG.info("amRegistrationRequest recovered for {}", this.attemptId);
-
+            RegisterApplicationMasterRequestProto.parseFrom(appMasterRequestBytes);
+        this.amRegistrationRequest = new RegisterApplicationMasterRequestPBImpl(pb);
+        LOG.info("amRegistrationRequest recovered for {}.", this.attemptId);
         // Give the register request to homeRMRelayer for future re-registration
         this.homeRMRelayer.setAMRegistrationRequest(this.amRegistrationRequest);
       }
+
       if (recoveredDataMap.containsKey(NMSS_REG_RESPONSE_KEY)) {
+        byte[] appMasterResponseBytes = recoveredDataMap.get(NMSS_REG_RESPONSE_KEY);
         RegisterApplicationMasterResponseProto pb =
-            RegisterApplicationMasterResponseProto
-                .parseFrom(recoveredDataMap.get(NMSS_REG_RESPONSE_KEY));
-        this.amRegistrationResponse =
-            new RegisterApplicationMasterResponsePBImpl(pb);
-        LOG.info("amRegistrationResponse recovered for {}", this.attemptId);
+            RegisterApplicationMasterResponseProto.parseFrom(appMasterResponseBytes);
+        this.amRegistrationResponse = new RegisterApplicationMasterResponsePBImpl(pb);
+        LOG.info("amRegistrationResponse recovered for {}.", this.attemptId);
       }
 
       // Recover UAM amrmTokens from registry or NMSS
-      Map<String, Token<AMRMTokenIdentifier>> uamMap;
-      if (this.registryClient != null) {
-        uamMap = this.registryClient
-            .loadStateFromRegistry(this.attemptId.getApplicationId());
-        LOG.info("Found {} existing UAMs for application {} in Yarn Registry",
-            uamMap.size(), this.attemptId.getApplicationId());
-      } else {
-        uamMap = new HashMap<>();
-        for (Entry<String, byte[]> entry : recoveredDataMap.entrySet()) {
-          if (entry.getKey().startsWith(NMSS_SECONDARY_SC_PREFIX)) {
-            // entry for subClusterId -> UAM amrmToken
-            String scId =
-                entry.getKey().substring(NMSS_SECONDARY_SC_PREFIX.length());
-            Token<AMRMTokenIdentifier> amrmToken = new Token<>();
-            amrmToken.decodeFromUrlString(
-                new String(entry.getValue(), STRING_TO_BYTE_FORMAT));
-            uamMap.put(scId, amrmToken);
-            LOG.debug("Recovered UAM in {} from NMSS", scId);
-          }
-        }
-        LOG.info("Found {} existing UAMs for application {} in NMStateStore",
-            uamMap.size(), this.attemptId.getApplicationId());
-      }
+      Map<String, Token<AMRMTokenIdentifier>> uamMap =
+          recoverSubClusterAMRMTokenIdentifierMap(recoveredDataMap);
 
       // Re-attach the UAMs
       int containers = 0;
-      for (Map.Entry<String, Token<AMRMTokenIdentifier>> entry : uamMap
-          .entrySet()) {
-        SubClusterId subClusterId = SubClusterId.newInstance(entry.getKey());
-
-        // Create a config loaded with federation on and subclusterId
+      AMRMProxyApplicationContext applicationContext = getApplicationContext();
+      ApplicationId applicationId = this.attemptId.getApplicationId();
+      String queue = this.amRegistrationResponse.getQueue();
+      String homeSCId = this.homeSubClusterId.getId();
+      String user = applicationContext.getUser();
+
+      for (Map.Entry<String, Token<AMRMTokenIdentifier>> entry : uamMap.entrySet()) {
+        String keyScId = entry.getKey();
+        Token<AMRMTokenIdentifier> tokens = entry.getValue();
+        SubClusterId subClusterId = SubClusterId.newInstance(keyScId);
+
+        // Create a config loaded with federation on and subClusterId
         // for each UAM
         YarnConfiguration config = new YarnConfiguration(getConf());
-        FederationProxyProviderUtil.updateConfForFederation(config,
-            subClusterId.getId());
+        FederationProxyProviderUtil.updateConfForFederation(config, keyScId);
 
         try {
-          this.uamPool.reAttachUAM(subClusterId.getId(), config,
-              this.attemptId.getApplicationId(),
-              this.amRegistrationResponse.getQueue(),
-              getApplicationContext().getUser(), this.homeSubClusterId.getId(),
-              entry.getValue(), subClusterId.toString());
+          // ReAttachUAM
+          this.uamPool.reAttachUAM(keyScId, config, applicationId, queue, user, homeSCId,
+              tokens, keyScId);
 
-          this.secondaryRelayers.put(subClusterId.getId(),
-              this.uamPool.getAMRMClientRelayer(subClusterId.getId()));
+          // GetAMRMClientRelayer
+          this.secondaryRelayers.put(keyScId, this.uamPool.getAMRMClientRelayer(keyScId));
 
+          // RegisterApplicationMaster
           RegisterApplicationMasterResponse response =
-              this.uamPool.registerApplicationMaster(subClusterId.getId(),
-                  this.amRegistrationRequest);
+              this.uamPool.registerApplicationMaster(keyScId, this.amRegistrationRequest);
 
           // Set sub-cluster to be timed out initially
-          lastSCResponseTime.put(subClusterId,
-              clock.getTime() - subClusterTimeOut);
+          lastSCResponseTime.put(subClusterId, clock.getTime() - subClusterTimeOut);
 
           // Running containers from secondary RMs
-          for (Container container : response
-              .getContainersFromPreviousAttempts()) {
-            containerIdToSubClusterIdMap.put(container.getId(), subClusterId);
+          List<Container> previousAttempts = response.getContainersFromPreviousAttempts();
+          for (Container container : previousAttempts) {
+            ContainerId containerId = container.getId();
+            containerIdToSubClusterIdMap.put(containerId, subClusterId);
             containers++;
-            LOG.debug("  From subcluster {} running container {}",
-                subClusterId, container.getId());
+            LOG.info("From subCluster {} running container {}", subClusterId, containerId);
           }
-          LOG.info("Recovered {} running containers from UAM in {}",
-              response.getContainersFromPreviousAttempts().size(),
-              subClusterId);
+
+          LOG.info("Recovered {} running containers from UAM in {}.",
+              previousAttempts.size(), subClusterId);
 
         } catch (Exception e) {
-          LOG.error("Error reattaching UAM to " + subClusterId + " for "
-              + this.attemptId, e);
+          LOG.error("Error reattaching UAM to {} for {}.", subClusterId, this.attemptId, e);
+          // During recovery, we need to clean up the data of the bad SubCluster.
+          // This ensures that when the bad SubCluster is recovered,
+          // new Containers can still be allocated and new UAMs can be registered.
+          this.uamPool.unAttachUAM(keyScId);
+          this.secondaryRelayers.remove(keyScId);
+          this.lastSCResponseTime.remove(subClusterId);
+          List<ContainerId> containerIds =
+              containerIdToSubClusterIdMap.entrySet().stream()
+              .filter(item-> item.getValue().equals(subClusterId))
+              .map(Entry::getKey)
+              .collect(Collectors.toList());
+          for (ContainerId containerId : containerIds) {
+            containerIdToSubClusterIdMap.remove(containerId);
+          }
         }
       }
 
       // Get the running containers from home RM, note that we will also get the
       // AM container itself from here. We don't need it, but no harm to put the
       // map as well.
-      UserGroupInformation appSubmitter = UserGroupInformation
-          .createRemoteUser(getApplicationContext().getUser());
-      ApplicationClientProtocol rmClient =
-          createHomeRMProxy(getApplicationContext(),
-              ApplicationClientProtocol.class, appSubmitter);
-
-      GetContainersResponse response = rmClient
-          .getContainers(GetContainersRequest.newInstance(this.attemptId));
+      UserGroupInformation appSubmitter;
+      if (UserGroupInformation.isSecurityEnabled()) {
+        appSubmitter = UserGroupInformation.createProxyUser(user,
+            UserGroupInformation.getLoginUser());
+      } else {
+        appSubmitter = UserGroupInformation.createRemoteUser(user);
+      }
+
+      ApplicationClientProtocol rmClient = createHomeRMProxy(applicationContext,
+          ApplicationClientProtocol.class, appSubmitter);
+
+      GetContainersRequest request = GetContainersRequest.newInstance(this.attemptId);
+      GetContainersResponse response = rmClient.getContainers(request);
+
       for (ContainerReport container : response.getContainerList()) {
-        containerIdToSubClusterIdMap.put(container.getContainerId(),
-            this.homeSubClusterId);
+        ContainerId containerId = container.getContainerId();
+        containerIdToSubClusterIdMap.put(containerId, this.homeSubClusterId);
         containers++;
-        LOG.debug("  From home RM {} running container {}",
-            this.homeSubClusterId, container.getContainerId());
+        LOG.debug("From home RM {} running container {}.", this.homeSubClusterId, containerId);
       }
-      LOG.info("{} running containers including AM recovered from home RM {}",
+      LOG.info("{} running containers including AM recovered from home RM {}.",
           response.getContainerList().size(), this.homeSubClusterId);
 
-      LOG.info(
-          "In all {} UAMs {} running containers including AM recovered for {}",
+      LOG.info("In all {} UAMs {} running containers including AM recovered for {}.",
           uamMap.size(), containers, this.attemptId);
 
-      if (this.amRegistrationResponse != null) {
+      if (queue != null) {
         // Initialize the AMRMProxyPolicy
-        String queue = this.amRegistrationResponse.getQueue();
-        this.policyInterpreter =
-            FederationPolicyUtils.loadAMRMPolicy(queue, this.policyInterpreter,
-                getConf(), this.federationFacade, this.homeSubClusterId);
+        queue = this.amRegistrationResponse.getQueue();
+        this.policyInterpreter = FederationPolicyUtils.loadAMRMPolicy(queue, this.policyInterpreter,
+            getConf(), this.federationFacade, this.homeSubClusterId);
       }
     } catch (IOException | YarnException e) {
       throw new YarnRuntimeException(e);
     }
+  }
+
+  /**
+   * recover SubClusterAMRMTokenIdentifierMap.
+   *
+   * If registryClient is not empty, restore directly from registryClient,
+   * otherwise restore from NMSS.
+   *
+   * @param recoveredDataMap recoveredDataMap.
+   * @return subClusterAMRMTokenIdentifierMap.
+   * @throws IOException IO Exception occurs.
+   */
+  private Map<String, Token<AMRMTokenIdentifier>> recoverSubClusterAMRMTokenIdentifierMap(
+      Map<String, byte[]> recoveredDataMap) throws IOException {
+    Map<String, Token<AMRMTokenIdentifier>> uamMap;
+    ApplicationId applicationId = this.attemptId.getApplicationId();
+    if (this.registryClient != null) {
+      uamMap = this.registryClient.loadStateFromRegistry(applicationId);
+      LOG.info("Found {} existing UAMs for application {} in Yarn Registry.",
+          uamMap.size(), applicationId);
+    } else {
+      uamMap = recoverSubClusterAMRMTokenIdentifierMapFromNMSS(recoveredDataMap);
+      LOG.info("Found {} existing UAMs for application {} in NMStateStore.",
+          uamMap.size(), applicationId);
+    }
+    return uamMap;
+  }
 
+  /**
+   * recover SubClusterAMRMTokenIdentifierMap From NMSS.
+   *
+   * @param recoveredDataMap recoveredDataMap
+   * @return subClusterAMRMTokenIdentifierMap.
+   * @throws IOException IO Exception occurs.
+   */
+  private Map<String, Token<AMRMTokenIdentifier>> recoverSubClusterAMRMTokenIdentifierMapFromNMSS(
+      Map<String, byte[]> recoveredDataMap) throws IOException {
+    Map<String, Token<AMRMTokenIdentifier>> uamMap = new HashMap<>();
+    for (Entry<String, byte[]> entry : recoveredDataMap.entrySet()) {
+      String key = entry.getKey();
+      byte[] value = entry.getValue();
+
+      if (key.startsWith(NMSS_SECONDARY_SC_PREFIX)) {
+        // entry for subClusterId -> UAM AMRMTokenIdentifier
+        String scId = key.substring(NMSS_SECONDARY_SC_PREFIX.length());
+        Token<AMRMTokenIdentifier> aMRMTokenIdentifier = new Token<>();
+        aMRMTokenIdentifier.decodeFromUrlString(new String(value, STRING_TO_BYTE_FORMAT));
+        uamMap.put(scId, aMRMTokenIdentifier);
+        LOG.debug("Recovered UAM in {} from NMSS.", scId);
+      }
+    }
+    return uamMap;
   }
 
   /**
@@ -647,7 +701,7 @@ public AllocateResponse allocate(AllocateRequest request)
       if (AMRMClientUtils.getNextResponseId(
           request.getResponseId()) == this.lastAllocateResponse
               .getResponseId()) {
-        // heartbeat one step old, simply return lastReponse
+        // heartbeat one step old, simply return lastResponse
         return this.lastAllocateResponse;
       } else if (request.getResponseId() != this.lastAllocateResponse
           .getResponseId()) {
@@ -731,50 +785,26 @@ public FinishApplicationMasterResponse finishApplicationMaster(
 
     this.finishAMCalled = true;
 
-    // TODO: consider adding batchFinishApplicationMaster in UAMPoolManager
     boolean failedToUnRegister = false;
-    ExecutorCompletionService<FinishApplicationMasterResponseInfo> compSvc =
-        null;
 
     // Application master is completing operation. Send the finish
     // application master request to all the registered sub-cluster resource
     // managers in parallel, wait for the responses and aggregate the results.
-    Set<String> subClusterIds = this.uamPool.getAllUAMIds();
-    if (subClusterIds.size() > 0) {
-      final FinishApplicationMasterRequest finishRequest = request;
-      compSvc =
-          new ExecutorCompletionService<FinishApplicationMasterResponseInfo>(
-              this.threadpool);
-
-      LOG.info("Sending finish application request to {} sub-cluster RMs",
-          subClusterIds.size());
-      for (final String subClusterId : subClusterIds) {
-        compSvc.submit(new Callable<FinishApplicationMasterResponseInfo>() {
-          @Override
-          public FinishApplicationMasterResponseInfo call() throws Exception {
-            LOG.info("Sending finish application request to RM {}",
-                subClusterId);
-            FinishApplicationMasterResponse uamResponse = null;
-            try {
-              uamResponse =
-                  uamPool.finishApplicationMaster(subClusterId, finishRequest);
-
-              if (uamResponse.getIsUnregistered()) {
-                secondaryRelayers.remove(subClusterId);
-                if (getNMStateStore() != null) {
-                  getNMStateStore().removeAMRMProxyAppContextEntry(attemptId,
-                      NMSS_SECONDARY_SC_PREFIX + subClusterId);
-                }
-              }
-            } catch (Throwable e) {
-              LOG.warn("Failed to finish unmanaged application master: "
-                  + "RM address: " + subClusterId + " ApplicationId: "
-                  + attemptId, e);
-            }
-            return new FinishApplicationMasterResponseInfo(uamResponse,
-                subClusterId);
-          }
-        });
+    Map<String, FinishApplicationMasterResponse> responseMap =
+        this.uamPool.batchFinishApplicationMaster(request, attemptId.toString());
+
+    for (Map.Entry<String, FinishApplicationMasterResponse> entry : responseMap.entrySet()) {
+      String subClusterId = entry.getKey();
+      FinishApplicationMasterResponse response = entry.getValue();
+      if (response != null && response.getIsUnregistered()) {
+        secondaryRelayers.remove(subClusterId);
+        if (getNMStateStore() != null) {
+          getNMStateStore().removeAMRMProxyAppContextEntry(attemptId,
+              NMSS_SECONDARY_SC_PREFIX + subClusterId);
+        }
+      } else {
+        // response is null or response.getIsUnregistered() == false
+        failedToUnRegister = true;
       }
     }
 
@@ -787,44 +817,27 @@ public FinishApplicationMasterResponseInfo call() throws Exception {
     // Stop the home heartbeat thread
     this.homeHeartbeartHandler.shutdown();
 
-    if (subClusterIds.size() > 0) {
-      // Wait for other sub-cluster resource managers to return the
-      // response and merge it with the home response
-      LOG.info(
-          "Waiting for finish application response from {} sub-cluster RMs",
-          subClusterIds.size());
-      for (int i = 0; i < subClusterIds.size(); ++i) {
-        try {
-          Future<FinishApplicationMasterResponseInfo> future = compSvc.take();
-          FinishApplicationMasterResponseInfo uamResponse = future.get();
-          LOG.debug("Received finish application response from RM: {}",
-              uamResponse.getSubClusterId());
-          if (uamResponse.getResponse() == null
-              || !uamResponse.getResponse().getIsUnregistered()) {
-            failedToUnRegister = true;
-          }
-        } catch (Throwable e) {
-          failedToUnRegister = true;
-          LOG.warn("Failed to finish unmanaged application master: "
-              + " ApplicationId: " + this.attemptId, e);
-        }
-      }
-    }
-
     if (failedToUnRegister) {
       homeResponse.setIsUnregistered(false);
-    } else {
+    } else if (checkRequestFinalApplicationStatusSuccess(request)) {
       // Clean up UAMs only when the app finishes successfully, so that no more
       // attempt will be launched.
       this.uamPool.stop();
-      if (this.registryClient != null) {
-        this.registryClient
-            .removeAppFromRegistry(this.attemptId.getApplicationId());
-      }
+      removeAppFromRegistry();
     }
     return homeResponse;
   }
 
+  private boolean checkRequestFinalApplicationStatusSuccess(
+      FinishApplicationMasterRequest request) {
+    if (request != null && request.getFinalApplicationStatus() != null) {
+      if (request.getFinalApplicationStatus().equals(FinalApplicationStatus.SUCCEEDED)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   @Override
   public void setNextInterceptor(RequestInterceptor next) {
     throw new YarnRuntimeException(
@@ -860,9 +873,21 @@ public void shutdown() {
     this.homeHeartbeartHandler.shutdown();
     this.homeRMRelayer.shutdown();
 
+    // Shutdown needs to clean up app
+    removeAppFromRegistry();
+
     super.shutdown();
   }
 
+  private void removeAppFromRegistry() {
+    if (this.registryClient != null && this.attemptId != null) {
+      ApplicationId applicationId = this.attemptId.getApplicationId();
+      if (applicationId != null) {
+        this.registryClient.removeAppFromRegistry(applicationId);
+      }
+    }
+  }
+
   /**
    * Only for unit test cleanup.
    */
@@ -1518,6 +1543,7 @@ protected void mergeAllocateResponse(AllocateResponse homeResponse,
   private void cacheAllocatedContainers(List<Container> containers,
       SubClusterId subClusterId) {
     for (Container container : containers) {
+      SubClusterId chooseSubClusterId = SubClusterId.newInstance(subClusterId.toString());
       LOG.debug("Adding container {}", container);
 
       if (this.containerIdToSubClusterIdMap.containsKey(container.getId())) {
@@ -1540,29 +1566,60 @@ private void cacheAllocatedContainers(List<Container> containers,
                   + " from same sub-cluster: {}, so ignoring.",
               container.getId(), subClusterId);
         } else {
+
+          LOG.info("Duplicate containerID found in the allocated containers. " +
+              "try to re-pick the sub-cluster.");
+
           // The same container allocation from different sub-clusters,
           // something is wrong.
-          // TODO: YARN-6667 if some subcluster RM is configured wrong, we
-          // should not fail the entire heartbeat.
-          throw new YarnRuntimeException(
-              "Duplicate containerID found in the allocated containers. This"
-                  + " can happen if the RM epoch is not configured properly."
-                  + " ContainerId: " + container.getId().toString()
-                  + " ApplicationId: " + this.attemptId + " From RM: "
-                  + subClusterId
-                  + " . Previous container was from sub-cluster: "
-                  + existingSubClusterId);
+          try {
+
+            boolean existAllocatedScHealth = isSCHealth(existingSubClusterId);
+            boolean newAllocatedScHealth = isSCHealth(subClusterId);
+
+            if (existAllocatedScHealth) {
+              // If the previous RM which allocated Container is normal,
+              // the previous RM will be used first
+              LOG.info("Use Previous Allocated Container's subCluster. " +
+                  "ContainerId: {} ApplicationId: {} From RM: {}.", this.attemptId,
+                  container.getId(), existingSubClusterId);
+              chooseSubClusterId = existingSubClusterId;
+            } else if (newAllocatedScHealth) {
+              // If the previous RM which allocated Container is abnormal,
+              // but the RM of the newly allocated Container is normal, use the new RM
+              LOG.info("Use Newly Allocated Container's subCluster. " +
+                  "ApplicationId: {} ContainerId: {} From RM: {}.", this.attemptId,
+                  container.getId(), subClusterId);
+              chooseSubClusterId = subClusterId;
+            } else {
+              // There is a very small probability that an exception will be thrown.
+              // The RM of the previously allocated Container
+              // and the RM of the newly allocated Container are not normal.
+              throw new YarnRuntimeException(
+                  " Can't use any subCluster because an exception occurred" +
+                  " ContainerId: " + container.getId() + " ApplicationId: " + this.attemptId +
+                  " From RM: " + subClusterId + ". " +
+                  " Previous Container was From subCluster: " + existingSubClusterId);
+            }
+          } catch (Exception ex) {
+            // An exception occurred
+            throw new YarnRuntimeException(
+                " Can't use any subCluster because an exception occurred" +
+                " ContainerId: " + container.getId() + " ApplicationId: " + this.attemptId +
+                " From RM: " + subClusterId + ". " +
+                " Previous Container was From subCluster: " + existingSubClusterId, ex);
+          }
         }
       }
 
-      this.containerIdToSubClusterIdMap.put(container.getId(), subClusterId);
+      this.containerIdToSubClusterIdMap.put(container.getId(), chooseSubClusterId);
     }
   }
 
   /**
    * Check to see if an AllocateRequest exists in the Map for the specified sub
    * cluster. If not found, create a new one, copy the value of responseId and
-   * progress from the orignialAMRequest, save it in the specified Map and
+   * progress from the originalAMRequest, save it in the specified Map and
    * return the new instance. If found, just return the old instance.
    */
   private static AllocateRequest findOrCreateAllocateRequestForSubCluster(
@@ -1804,4 +1861,25 @@ public static <T> boolean isNullOrEmpty(Collection<T> c) {
   public static <T1, T2> boolean isNullOrEmpty(Map<T1, T2> c) {
     return (c == null || c.size() == 0);
   }
+
+  @VisibleForTesting
+  protected void cacheAllocatedContainersForSubClusterId(
+      List<Container> containers, SubClusterId subClusterId) {
+    cacheAllocatedContainers(containers, subClusterId);
+  }
+
+  @VisibleForTesting
+  protected Map<ContainerId, SubClusterId> getContainerIdToSubClusterIdMap() {
+    return containerIdToSubClusterIdMap;
+  }
+
+  private boolean isSCHealth(SubClusterId subClusterId) throws YarnException {
+    Set<SubClusterId> timeOutScs = getTimedOutSCs(true);
+    SubClusterInfo subClusterInfo = federationFacade.getSubCluster(subClusterId);
+    if (timeOutScs.contains(subClusterId) ||
+        subClusterInfo == null || subClusterInfo.getState().isUnusable()) {
+      return false;
+    }
+    return true;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
index 950bdcc043048..3c9adf922c122 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/RequestInterceptor.java
@@ -24,14 +24,14 @@
 import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocol;
 
 /**
- * Defines the contract to be implemented by the request intercepter classes,
+ * Defines the contract to be implemented by the request interceptor classes,
  * that can be used to intercept and inspect messages sent from the application
  * master to the resource manager.
  */
 public interface RequestInterceptor extends DistributedSchedulingAMProtocol,
     Configurable {
   /**
-   * This method is called for initializing the intercepter. This is guaranteed
+   * This method is called for initializing the interceptor. This is guaranteed
    * to be called only once in the lifetime of this instance.
    *
    * @param ctx AMRMProxy application context
@@ -39,42 +39,42 @@ public interface RequestInterceptor extends DistributedSchedulingAMProtocol,
   void init(AMRMProxyApplicationContext ctx);
 
   /**
-   * Recover intercepter state when NM recovery is enabled. AMRMProxy will
+   * Recover interceptor state when NM recovery is enabled. AMRMProxy will
    * recover the data map into
-   * AMRMProxyApplicationContext.getRecoveredDataMap(). All intercepters should
+   * AMRMProxyApplicationContext.getRecoveredDataMap(). All interceptors should
    * recover state from it.
    *
-   * For example, registerRequest has to be saved by the last intercepter (i.e.
+   * For example, registerRequest has to be saved by the last interceptor (i.e.
    * the one that actually connects to RM), in order to re-register when RM
    * fails over.
    *
-   * @param recoveredDataMap states for all intercepters recovered from NMSS
+   * @param recoveredDataMap states for all interceptors recovered from NMSS
    */
   void recover(Map<String, byte[]> recoveredDataMap);
 
   /**
-   * This method is called to release the resources held by the intercepter.
+   * This method is called to release the resources held by the interceptor.
    * This will be called when the application pipeline is being destroyed. The
    * concrete implementations should dispose the resources and forward the
-   * request to the next intercepter, if any.
+   * request to the next interceptor, if any.
    */
   void shutdown();
 
   /**
-   * Sets the next intercepter in the pipeline. The concrete implementation of
+   * Sets the next interceptor in the pipeline. The concrete implementation of
    * this interface should always pass the request to the nextInterceptor after
-   * inspecting the message. The last intercepter in the chain is responsible to
+   * inspecting the message. The last interceptor in the chain is responsible to
    * send the messages to the resource manager service and so the last
-   * intercepter will not receive this method call.
+   * interceptor will not receive this method call.
    *
-   * @param nextInterceptor the next intercepter to set
+   * @param nextInterceptor the next interceptor to set
    */
   void setNextInterceptor(RequestInterceptor nextInterceptor);
 
   /**
-   * Returns the next intercepter in the chain.
+   * Returns the next interceptor in the chain.
    * 
-   * @return the next intercepter in the chain
+   * @return the next interceptor in the chain
    */
   RequestInterceptor getNextInterceptor();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java
new file mode 100644
index 0000000000000..2f20d98426fef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.nodemanager.amrmproxy contains
+ * classes for handling federation amrm information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index 0bf3cdcf173c6..794ef9d9a4326 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -80,7 +80,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
 import org.apache.hadoop.yarn.util.FSDownload;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 public class AuxServices extends AbstractService
     implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
@@ -240,7 +240,7 @@ private AuxiliaryService createAuxServiceFromLocalClasspath(
     }
 
     return AuxiliaryServiceWithCustomClassLoader.getInstance(conf, className,
-        appLocalClassPath, getSystemClasses(service, className));
+        appLocalClassPath, getSystemClasses(service));
   }
 
   /**
@@ -292,7 +292,7 @@ private AuxiliaryService createAuxService(AuxServiceRecord service,
           + " is using the custom classloader with classpath " + destFiles);
       return AuxiliaryServiceWithCustomClassLoader.getInstance(conf,
           className, StringUtils.join(File.pathSeparatorChar, destFiles),
-          getSystemClasses(service, className));
+          getSystemClasses(service));
     } else {
       return createAuxServiceFromConfiguration(service);
     }
@@ -681,15 +681,12 @@ private static String getClassName(AuxServiceRecord service) {
     return serviceConf.getProperty(CLASS_NAME);
   }
 
-  private static String[] getSystemClasses(AuxServiceRecord service, String
-      className) {
-    AuxServiceConfiguration serviceConf =
-        service.getConfiguration();
-    if (serviceConf == null) {
-      return new String[]{className};
+  private static String[] getSystemClasses(AuxServiceRecord service) {
+    AuxServiceConfiguration serviceConf = service.getConfiguration();
+    if (serviceConf == null || serviceConf.getProperty(SYSTEM_CLASSES) == null) {
+      return new String[]{};
     }
-    return StringUtils.split(serviceConf.getProperty(SYSTEM_CLASSES,
-        className));
+    return StringUtils.split(serviceConf.getProperty(SYSTEM_CLASSES));
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
index 1e5a9a737f588..e47158041dd6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
@@ -21,7 +21,6 @@
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.AbstractEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
     .Container;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index ea0c91c289a6d..cdd9fc916e339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1187,10 +1187,7 @@ public ContainerState transition(ContainerImpl container,
       if (container.recoveredStatus == RecoveredContainerStatus.COMPLETED) {
         container.sendFinishedEvents();
         return ContainerState.DONE;
-      } else if (container.recoveredStatus == RecoveredContainerStatus.QUEUED) {
-        return ContainerState.SCHEDULED;
-      } else if (container.recoveredAsKilled &&
-          container.recoveredStatus == RecoveredContainerStatus.REQUESTED) {
+      } else if (isContainerRecoveredAsKilled(container)) {
         // container was killed but never launched
         container.metrics.killedContainer();
         NMAuditLogger.logSuccess(container.user,
@@ -1201,6 +1198,8 @@ public ContainerState transition(ContainerImpl container,
             container.containerTokenIdentifier.getResource());
         container.sendFinishedEvents();
         return ContainerState.DONE;
+      } else if (container.recoveredStatus == RecoveredContainerStatus.QUEUED) {
+        return ContainerState.SCHEDULED;
       }
 
       final ContainerLaunchContext ctxt = container.launchContext;
@@ -1264,6 +1263,16 @@ public ContainerState transition(ContainerImpl container,
         return ContainerState.LOCALIZATION_FAILED;
       }
     }
+
+    static boolean isContainerRecoveredAsKilled(ContainerImpl container) {
+      if (!container.recoveredAsKilled) {
+        return false;
+      }
+      // container was killed but never launched
+      RecoveredContainerStatus containerStatus = container.recoveredStatus;
+      return containerStatus == RecoveredContainerStatus.REQUESTED
+          || containerStatus == RecoveredContainerStatus.QUEUED;
+    }
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
index 70b28d4fcaa79..c7bdd81a76e4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
 import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
index 4aabff7fdad9a..33ae86d0497fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerCleanup.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -36,7 +36,6 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DockerContainerDeletionTask;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEvent.java
index 38bedf246b95e..e32ac370cb72e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEvent.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
 
 import org.apache.hadoop.yarn.event.AbstractEvent;
-import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 
 public class ContainersLauncherEvent
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
index 36cb5e3b08ca0..cc2ded4422b71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/NetworkTagMappingJsonManager.java
@@ -25,15 +25,17 @@
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.ObjectMapper;
 
 /**
  * The NetworkTagMapping JsonManager implementation.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
index 6bfa0a931d4bf..2deaf16880a0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
@@ -143,7 +143,7 @@ public void init(Configuration conf) throws YarnException {
   }
 
   @VisibleForTesting
-  String executeNGetCmdOutput(Configuration conf) throws YarnException {
+  public String executeNGetCmdOutput(Configuration conf) throws YarnException {
     String numaCtlCmd = conf.get(
         YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD,
         YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD);
@@ -231,7 +231,7 @@ public synchronized NumaResourceAllocation allocateNumaNodes(
   }
 
   private NumaResourceAllocation allocate(ContainerId containerId,
-      Resource resource) {
+      Resource resource) throws ResourceHandlerException {
     for (int index = 0; index < numaNodesList.size(); index++) {
       NumaNodeResource numaNode = numaNodesList
           .get((currentAssignNode + index) % numaNodesList.size());
@@ -306,12 +306,20 @@ private NumaResourceAllocation allocate(ContainerId containerId,
    * Release assigned NUMA resources for the container.
    *
    * @param containerId the container ID
+   * @throws ResourceHandlerException when failed to release numa resource
    */
-  public synchronized void releaseNumaResource(ContainerId containerId) {
+  public synchronized void releaseNumaResource(ContainerId containerId)
+      throws ResourceHandlerException {
     LOG.info("Releasing the assigned NUMA resources for " + containerId);
     for (NumaNodeResource numaNode : numaNodesList) {
       numaNode.releaseResources(containerId);
     }
+    // delete from NM State store
+    try {
+      context.getNMStateStore().releaseAssignedResources(containerId, NUMA_RESOURCE_TYPE);
+    } catch (IOException e){
+      throw new ResourceHandlerException(e);
+    }
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 72166236f5abc..14f5ffeefe0d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -40,8 +40,10 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher;
 import org.apache.hadoop.yarn.util.DockerClientConfigHandler;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -206,6 +208,8 @@ public class DockerLinuxContainerRuntime extends OCIContainerRuntime {
   private static final Pattern dockerImagePattern =
       Pattern.compile(DOCKER_IMAGE_PATTERN);
 
+  private static final Pattern DOCKER_DIGEST_PATTERN = Pattern.compile("^sha256:[a-z0-9]{12,64}$");
+
   private static final String DEFAULT_PROCFS = "/proc";
 
   @InterfaceAudience.Private
@@ -215,6 +219,9 @@ public class DockerLinuxContainerRuntime extends OCIContainerRuntime {
   public static final String ENV_DOCKER_CONTAINER_IMAGE =
       "YARN_CONTAINER_RUNTIME_DOCKER_IMAGE";
   @InterfaceAudience.Private
+  public static final String ENV_DOCKER_CONTAINER_CLIENT_CONFIG =
+      "YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG";
+  @InterfaceAudience.Private
   public static final String ENV_DOCKER_CONTAINER_NETWORK =
       "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
   @InterfaceAudience.Private
@@ -595,6 +602,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
     boolean serviceMode = Boolean.parseBoolean(environment.get(
         ENV_DOCKER_CONTAINER_DOCKER_SERVICE_MODE));
     boolean useEntryPoint = serviceMode || checkUseEntryPoint(environment);
+    String clientConfig = environment.get(ENV_DOCKER_CONTAINER_CLIENT_CONFIG);
 
     if (imageName == null || imageName.isEmpty()) {
       imageName = defaultImageName;
@@ -796,7 +804,8 @@ public void launchContainer(ContainerRuntimeContext ctx)
       runCommand.setPrivileged();
     }
 
-    addDockerClientConfigToRunCommand(ctx, runCommand);
+    addDockerClientConfigToRunCommand(ctx, runCommand,
+        getAdditionalDockerClientCredentials(clientConfig, containerIdStr));
 
     String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
 
@@ -891,6 +900,22 @@ public void launchContainer(ContainerRuntimeContext ctx)
     }
   }
 
+  private Credentials getAdditionalDockerClientCredentials(String clientConfig,
+      String containerIdStr) {
+    Credentials additionalDockerCredentials = null;
+    if (clientConfig != null && !clientConfig.isEmpty()) {
+      try {
+        additionalDockerCredentials =
+            DockerClientConfigHandler.readCredentialsFromConfigFile(new Path(clientConfig), conf,
+                containerIdStr);
+      } catch (IOException e) {
+        throw new RuntimeException(
+            "Fail to read additional docker client config file from " + clientConfig);
+      }
+    }
+    return additionalDockerCredentials;
+  }
+
   @Override
   public void relaunchContainer(ContainerRuntimeContext ctx)
       throws ContainerExecutionException {
@@ -1178,9 +1203,17 @@ public static void validateImageName(String imageName)
       throw new ContainerExecutionException(
           ENV_DOCKER_CONTAINER_IMAGE + " not set!");
     }
-    if (!dockerImagePattern.matcher(imageName).matches()) {
-      throw new ContainerExecutionException("Image name '" + imageName
-          + "' doesn't match docker image name pattern");
+    // check if digest is part of imageName, extract and validate it.
+    String digest = null;
+    if (imageName.contains("@sha256")) {
+      String[] digestParts = imageName.split("@");
+      digest = digestParts[1];
+      imageName = digestParts[0];
+    }
+    if (!dockerImagePattern.matcher(imageName).matches() || (digest != null
+        && !DOCKER_DIGEST_PATTERN.matcher(digest).matches())) {
+      throw new ContainerExecutionException(
+          "Image name '" + imageName + "' doesn't match docker image name pattern");
     }
   }
 
@@ -1366,36 +1399,41 @@ private void handleContainerRemove(String containerId,
   }
 
   private void addDockerClientConfigToRunCommand(ContainerRuntimeContext ctx,
-      DockerRunCommand dockerRunCommand) throws ContainerExecutionException {
+      DockerRunCommand dockerRunCommand, Credentials additionDockerCredentials)
+      throws ContainerExecutionException {
     ByteBuffer tokens = ctx.getContainer().getLaunchContext().getTokens();
-    Credentials credentials;
+    Credentials credentials = new Credentials();
     if (tokens != null) {
       tokens.rewind();
       if (tokens.hasRemaining()) {
         try {
-          credentials = DockerClientConfigHandler
-              .getCredentialsFromTokensByteBuffer(tokens);
+          credentials.addAll(DockerClientConfigHandler
+              .getCredentialsFromTokensByteBuffer(tokens));
         } catch (IOException e) {
           throw new ContainerExecutionException("Unable to read tokens.");
         }
-        if (credentials.numberOfTokens() > 0) {
-          Path nmPrivateDir =
-              ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH)
-                  .getParent();
-          File dockerConfigPath = new File(nmPrivateDir + "/config.json");
-          try {
-            if (DockerClientConfigHandler
-                .writeDockerCredentialsToPath(dockerConfigPath, credentials)) {
-              dockerRunCommand.setClientConfigDir(dockerConfigPath.getParent());
-            }
-          } catch (IOException e) {
-            throw new ContainerExecutionException(
-                "Unable to write Docker client credentials to "
-                    + dockerConfigPath);
-          }
-        }
       }
     }
+
+    if (additionDockerCredentials != null) {
+      credentials.addAll(additionDockerCredentials);
+    }
+
+    if (credentials.numberOfTokens() > 0) {
+      Path nmPrivateDir =
+          ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH)
+              .getParent();
+      File dockerConfigPath = new File(nmPrivateDir + "/config.json");
+      try {
+        DockerClientConfigHandler
+            .writeDockerCredentialsToPath(dockerConfigPath, credentials);
+      } catch (IOException e) {
+        throw new ContainerExecutionException(
+            "Unable to write Docker client credentials to "
+                + dockerConfigPath);
+      }
+      dockerRunCommand.setClientConfigDir(dockerConfigPath.getParent());
+    }
   }
 
   boolean getHostPidNamespaceEnabled() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java
index 78ed5a6b97767..e43f7788d78b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/RuncContainerRuntime.java
@@ -62,8 +62,6 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.volume.csi.ContainerVolumePublisher;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import java.io.File;
 import java.io.IOException;
@@ -81,6 +79,9 @@
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_IMAGE_TAG_TO_MANIFEST_PLUGIN;
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_LAYER_MOUNTS_TO_KEEP;
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_REAP_RUNC_LAYER_MOUNTS_INTERVAL;
@@ -642,7 +643,7 @@ protected List<String> extractImageEnv(File config) throws IOException {
     if (envNode.isMissingNode()) {
       return null;
     }
-    return mapper.readValue(envNode, List.class);
+    return mapper.readValue(envNode.traverse(), List.class);
   }
 
   @SuppressWarnings("unchecked")
@@ -653,7 +654,7 @@ protected List<String> extractImageEntrypoint(File config)
     if (entrypointNode.isMissingNode()) {
       return null;
     }
-    return mapper.readValue(entrypointNode, List.class);
+    return mapper.readValue(entrypointNode.traverse(), List.class);
   }
 
   private RuncContainerExecutorConfig createRuncContainerExecutorConfig(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerImagesCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerImagesCommand.java
index da18509921e5b..785f035947fa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerImagesCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerImagesCommand.java
@@ -17,7 +17,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /**
  * Encapsulates the docker images command and its command
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java
index 629785dfcd7f1..fbec3ee6f5ea6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/ImageTagToManifestPlugin.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -45,6 +44,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_CACHE_REFRESH_INTERVAL;
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NM_RUNC_IMAGE_TOPLEVEL_DIR;
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_NUM_MANIFESTS_TO_CACHE;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java
index 88a01a22e13ed..3333b820c893c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/runc/RuncContainerExecutorConfig.java
@@ -20,12 +20,13 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc;
 
 import org.apache.hadoop.classification.InterfaceStability;
-import org.codehaus.jackson.annotate.JsonRawValue;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
 
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonRawValue;
+
 /**
  *  This class is used by the
  *  {@link org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.RuncContainerRuntime}
@@ -35,7 +36,7 @@
  *  a JSON object named ociRuntimeConfig that mirrors the
  *  OCI runtime specification.
  */
-@JsonSerialize(include=JsonSerialize.Inclusion.NON_DEFAULT)
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
 @InterfaceStability.Unstable
 public class RuncContainerExecutorConfig {
   final private String version;
@@ -164,7 +165,7 @@ public RuncContainerExecutorConfig(String version, String runAsUser,
   /**
    * This class is a Java representation of an OCI image layer.
    */
-  @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+  @JsonInclude(JsonInclude.Include.NON_DEFAULT)
   @InterfaceStability.Unstable
   public static class OCILayer {
     final private String mediaType;
@@ -192,7 +193,7 @@ public OCILayer() {
    * This class is a Java representation of the OCI Runtime Specification.
    */
   @InterfaceStability.Unstable
-  @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+  @JsonInclude(JsonInclude.Include.NON_DEFAULT)
   public static class OCIRuntimeConfig {
     final private OCIRootConfig root;
     final private List<OCIMount> mounts;
@@ -254,7 +255,7 @@ public OCIRuntimeConfig(OCIRootConfig root, List<OCIMount> mounts,
      * This class is a Java representation of the oci root config section
      * of the OCI Runtime Specification.
      */
-    @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
     public static class OCIRootConfig {
       public String getPath() {
         return path;
@@ -281,7 +282,7 @@ public OCIRootConfig() {
      * This class is a Java representation of the oci mount section
      * of the OCI Runtime Specification.
      */
-    @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
     public static class OCIMount {
       final private String destination;
       final private String type;
@@ -329,7 +330,7 @@ public OCIMount() {
      * This class is a Java representation of the oci process section
      * of the OCI Runtime Specification.
      */
-    @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
     public static class OCIProcessConfig {
       final private boolean terminal;
       final private ConsoleSize consoleSize;
@@ -422,7 +423,7 @@ public OCIProcessConfig() {
        * This class is a Java representation of the console size section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class ConsoleSize {
         public int getHeight() {
           return height;
@@ -450,7 +451,7 @@ public ConsoleSize() {
        * This class is a Java representation of the rlimits section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class RLimits {
         public String getType() {
           return type;
@@ -484,7 +485,7 @@ public RLimits() {
        * This class is a Java representation of the capabilities section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class Capabilities {
         final private List<String> effective;
         final private List<String> bounding;
@@ -554,7 +555,7 @@ public User() {
      * This class is a Java representation of the oci hooks section
      * of the OCI Runtime Specification.
      */
-    @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
     public static class OCIHooksConfig {
       final private List<HookType> prestart;
       final private List<HookType> poststart;
@@ -587,7 +588,7 @@ public OCIHooksConfig() {
        * This class is a Java representation of the hook type section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class HookType {
         final private String path;
         final private List<String> args;
@@ -650,7 +651,7 @@ public OCIAnnotationsConfig() {
      * This class is a Java representation of the oci linux config section
      * of the OCI Runtime Specification.
      */
-    @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
     public static class OCILinuxConfig {
       final private List<Namespace> namespaces;
       final private List<IDMapping> uidMappings;
@@ -768,7 +769,7 @@ public Namespace() {
        * This class is a Java representation of the idmapping section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class IDMapping {
         final private int containerID;
         final private int hostID;
@@ -802,7 +803,7 @@ public IDMapping() {
        * This class is a Java representation of the device section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class Device {
         final private String type;
         final private String path;
@@ -861,7 +862,7 @@ public Device() {
        * This class is a Java representation of the resources section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class Resources {
         final private List<Device> device;
         final private Memory memory;
@@ -927,7 +928,7 @@ public Resources() {
          * This class is a Java representation of the device section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class Device {
           final private boolean allow;
           final private String type;
@@ -973,7 +974,7 @@ public Device() {
          * This class is a Java representation of the memory section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class Memory {
           final private long limit;
           final private long reservation;
@@ -1032,7 +1033,7 @@ public Memory() {
          * This class is a Java representation of the cpu section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class CPU {
           final private long quota;
           final private long period;
@@ -1092,7 +1093,7 @@ public CPU() {
          * This class is a Java representation of the blockio section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class BlockIO {
           final private int weight;
           final private int leafWeight;
@@ -1153,7 +1154,7 @@ public BlockIO() {
            * This class is a Java representation of the weight device section
            * of the OCI Runtime Specification.
            */
-          @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+          @JsonInclude(JsonInclude.Include.NON_DEFAULT)
           public static class WeightDevice {
             final private long major;
             final private long minor;
@@ -1193,7 +1194,7 @@ public WeightDevice() {
            * This class is a Java representation of the throttle device section
            * of the OCI Runtime Specification.
            */
-          @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+          @JsonInclude(JsonInclude.Include.NON_DEFAULT)
           public static class ThrottleDevice {
             final private long major;
             final private long minor;
@@ -1227,7 +1228,7 @@ public ThrottleDevice() {
          * This class is a Java representation of the huge page limits section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class HugePageLimits {
           final private String pageSize;
           final private long limit;
@@ -1254,7 +1255,7 @@ public HugePageLimits() {
          * This class is a Java representation of the network section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class Network {
           final private int classID;
           final private List<NetworkPriority> priorities;
@@ -1280,7 +1281,7 @@ public Network() {
            * This class is a Java representation of the network priority section
            * of the OCI Runtime Specification.
            */
-          @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+          @JsonInclude(JsonInclude.Include.NON_DEFAULT)
           public static class NetworkPriority {
             final private String name;
             final private int priority;
@@ -1308,7 +1309,7 @@ public NetworkPriority() {
          * This class is a Java representation of the pid section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class PID {
           final private long limit;
 
@@ -1329,7 +1330,7 @@ public PID() {
          * This class is a Java representation of the rdma section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class RDMA {
           final private int hcaHandles;
           final private int hcaObjects;
@@ -1357,7 +1358,7 @@ public RDMA() {
        * This class is a Java representation of the intelrdt section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class IntelRdt {
         final private String closID;
         final private String l3CacheSchema;
@@ -1391,7 +1392,7 @@ public IntelRdt() {
        * This class is a Java representation of the sysctl section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class Sysctl {
         // for kernel params
       }
@@ -1400,7 +1401,7 @@ public static class Sysctl {
        * This class is a Java representation of the seccomp section
        * of the OCI Runtime Specification.
        */
-      @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+      @JsonInclude(JsonInclude.Include.NON_DEFAULT)
       public static class Seccomp {
         final private String defaultAction;
         final private List<String> architectures;
@@ -1433,7 +1434,7 @@ public Seccomp() {
          * This class is a Java representation of the syscall section
          * of the OCI Runtime Specification.
          */
-        @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+        @JsonInclude(JsonInclude.Include.NON_DEFAULT)
         public static class Syscall {
           final private List<String> names;
           final private String action;
@@ -1466,7 +1467,7 @@ public Syscall() {
            * This class is a Java representation of the seccomp arguments
            * of the OCI Runtime Specification.
            */
-          @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+          @JsonInclude(JsonInclude.Include.NON_DEFAULT)
           public static class SeccompArg {
             final private int index;
             final private long value;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 5245226ea103e..604a810ec9468 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -19,7 +19,7 @@
 
 import static org.apache.hadoop.util.Shell.getAllShells;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -218,7 +218,7 @@ public LocalizationProtocol run() {
 
   ExecutorService createDownloadThreadPool() {
     return HadoopExecutors.newSingleThreadExecutor(new ThreadFactoryBuilder()
-      .setNameFormat("ContainerLocalizer Downloader").build());
+      .setNameFormat("ContainerLocalizer Downloader-" + localizerId).build());
   }
 
   CompletionService<Path> createCompletionService(ExecutorService exec) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
index d2e8e22d45964..bd161cc35f82a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 public class LocalResourceRequest
     extends LocalResource implements Comparable<LocalResourceRequest> {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
index 0e732a7ce58a2..857b3314e1adb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
@@ -21,7 +21,6 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizerContext;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 /**
  * Event indicating that the {@link ResourceLocalizationService}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
index ed75116391537..8fc2e1e91220f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
@@ -44,7 +44,6 @@
 import org.apache.hadoop.yarn.server.sharedcache.SharedCacheUtil;
 import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum;
 import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.FSDownload;
 
 import org.apache.hadoop.classification.VisibleForTesting;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 96e08c5ff38ee..1ba7353a1eef3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -86,6 +86,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
   private final Dispatcher dispatcher;
   private final ApplicationId appId;
   private final String applicationId;
+  private final boolean enableLocalCleanup;
   private boolean logAggregationDisabled = false;
   private final Configuration conf;
   private final DeletionService delService;
@@ -172,6 +173,13 @@ public AppLogAggregatorImpl(Dispatcher dispatcher,
     this.logAggregationContext = logAggregationContext;
     this.context = context;
     this.nodeId = nodeId;
+    this.enableLocalCleanup =
+        conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP,
+            YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP);
+    if (!this.enableLocalCleanup) {
+      LOG.warn("{} is only for testing and not for any production system ",
+          YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP);
+    }
     this.logAggPolicy = getLogAggPolicy(conf);
     this.recoveredLogInitedTime = recoveredLogInitedTime;
     this.logFileSizeThreshold =
@@ -337,26 +345,26 @@ private void uploadLogsForContainers(boolean appFinished)
             appFinished, finishedContainers.contains(container));
         if (uploadedFilePathsInThisCycle.size() > 0) {
           uploadedLogsInThisCycle = true;
-          LOG.trace("Uploaded the following files for {}: {}",
-              container, uploadedFilePathsInThisCycle.toString());
-          List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
-          uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
-          if (LOG.isDebugEnabled()) {
-            for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
-              try {
-                long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
-                if (fileSize >= logFileSizeThreshold) {
-                  LOG.debug("Log File " + uploadedFilePath
-                      + " size is " + fileSize + " bytes");
+          if (enableLocalCleanup) {
+            LOG.trace("Uploaded the following files for {}: {}", container,
+                uploadedFilePathsInThisCycle.toString());
+            List<Path> uploadedFilePathsInThisCycleList = new ArrayList<>();
+            uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle);
+            if (LOG.isDebugEnabled()) {
+              for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) {
+                try {
+                  long fileSize = lfs.getFileStatus(uploadedFilePath).getLen();
+                  if (fileSize >= logFileSizeThreshold) {
+                    LOG.debug("Log File " + uploadedFilePath + " size is " + fileSize + " bytes");
+                  }
+                } catch (Exception e1) {
+                  LOG.error("Failed to get log file size " + e1);
                 }
-              } catch (Exception e1) {
-                LOG.error("Failed to get log file size " + e1);
               }
             }
+            deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null,
+                uploadedFilePathsInThisCycleList);
           }
-          deletionTask = new FileDeletionTask(delService,
-              this.userUgi.getShortUserName(), null,
-              uploadedFilePathsInThisCycleList);
         }
 
         // This container is finished, and all its logs have been uploaded,
@@ -528,6 +536,9 @@ private void doAppLogAggregation() throws LogAggregationDFSException {
   }
 
   private void doAppLogAggregationPostCleanUp() {
+    if (!enableLocalCleanup) {
+      return;
+    }
     // Remove the local app-log-dirs
     List<Path> localAppLogDirs = new ArrayList<Path>();
     for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
index 7d55f2c108c5e..e8b2f472fb46f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
@@ -20,8 +20,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 1dd67959ca36f..eceb7b25e48ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java
index 4bc92f2711727..8d801198644e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java
@@ -27,6 +27,7 @@
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.hadoop.util.Shell;
@@ -58,11 +59,11 @@ class VEDeviceDiscoverer {
 
   public Set<Device> getDevicesFromPath(String path) throws IOException {
     MutableInt counter = new MutableInt(0);
-
-    return Files.walk(Paths.get(path), 1)
-      .filter(p -> p.toFile().getName().startsWith("veslot"))
-      .map(p -> toDevice(p, counter))
-      .collect(Collectors.toSet());
+    try (Stream<Path> stream = Files.walk(Paths.get(path), 1)) {
+      return stream.filter(p -> p.toFile().getName().startsWith("veslot"))
+            .map(p -> toDevice(p, counter))
+            .collect(Collectors.toSet());
+    }
   }
 
   private Device toDevice(Path p, MutableInt counter) {
@@ -140,4 +141,4 @@ void setCommandExecutorProvider(
       Function<String[], CommandExecutor> provider) {
     this.commandExecutorProvider = provider;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDevice.java
index 1dafd07cf4cb9..def45dff7f8e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDevice.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDevice.java
@@ -20,7 +20,7 @@
 
 import java.io.Serializable;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 /** A class that represents an FPGA card. */
 public class FpgaDevice implements Serializable {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
index 100676d27ddd8..879a5d9d0b97d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/AllocationBasedResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
 
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
@@ -34,6 +35,9 @@ public class AllocationBasedResourceUtilizationTracker implements
   private static final Logger LOG =
       LoggerFactory.getLogger(AllocationBasedResourceUtilizationTracker.class);
 
+  private static final long LEFT_SHIFT_MB_IN_BYTES = 20;
+  private static final int RIGHT_SHIFT_BYTES_IN_MB = 20;
+
   private ResourceUtilization containersAllocation;
   private ContainerScheduler scheduler;
 
@@ -80,10 +84,34 @@ public void subtractContainerResource(Container container) {
    */
   @Override
   public boolean hasResourcesAvailable(Container container) {
-    long pMemBytes = container.getResource().getMemorySize() * 1024 * 1024L;
-    return hasResourcesAvailable(pMemBytes,
-        (long) (getContainersMonitor().getVmemRatio()* pMemBytes),
-        container.getResource().getVirtualCores());
+    return hasResourcesAvailable(container.getResource());
+  }
+
+  /**
+   * Converts memory in megabytes to bytes by bitwise left-shifting 20 times.
+   * @param memMB the memory in megabytes
+   * @return the memory in bytes
+   */
+  private static long convertMBToBytes(final long memMB) {
+    return memMB << LEFT_SHIFT_MB_IN_BYTES;
+  }
+
+  /**
+   * Converts memory in bytes to megabytes by bitwise right-shifting 20 times.
+   * @param bytes the memory in bytes
+   * @return the memory in megabytes
+   */
+  private static long convertBytesToMB(final long bytes) {
+    return bytes >> RIGHT_SHIFT_BYTES_IN_MB;
+  }
+
+  @Override
+  public boolean hasResourcesAvailable(Resource resource) {
+    long pMemBytes = convertMBToBytes(resource.getMemorySize());
+    final long vmemBytes = (long)
+        (getContainersMonitor().getVmemRatio() * pMemBytes);
+    return hasResourcesAvailable(
+        pMemBytes, vmemBytes, resource.getVirtualCores());
   }
 
   private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
@@ -92,13 +120,13 @@ private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
     if (LOG.isDebugEnabled()) {
       LOG.debug("pMemCheck [current={} + asked={} > allowed={}]",
           this.containersAllocation.getPhysicalMemory(),
-          (pMemBytes >> 20),
-          (getContainersMonitor().getPmemAllocatedForContainers() >> 20));
+          convertBytesToMB(pMemBytes),
+          convertBytesToMB(
+              getContainersMonitor().getPmemAllocatedForContainers()));
     }
     if (this.containersAllocation.getPhysicalMemory() +
-        (int) (pMemBytes >> 20) >
-        (int) (getContainersMonitor()
-            .getPmemAllocatedForContainers() >> 20)) {
+        convertBytesToMB(pMemBytes) > convertBytesToMB(
+            getContainersMonitor().getPmemAllocatedForContainers())) {
       return false;
     }
 
@@ -106,15 +134,17 @@ private boolean hasResourcesAvailable(long pMemBytes, long vMemBytes,
       LOG.debug("before vMemCheck" +
               "[isEnabled={}, current={} + asked={} > allowed={}]",
           getContainersMonitor().isVmemCheckEnabled(),
-          this.containersAllocation.getVirtualMemory(), (vMemBytes >> 20),
-          (getContainersMonitor().getVmemAllocatedForContainers() >> 20));
+          this.containersAllocation.getVirtualMemory(),
+          convertBytesToMB(vMemBytes),
+          convertBytesToMB(
+              getContainersMonitor().getVmemAllocatedForContainers()));
     }
     // Check virtual memory.
     if (getContainersMonitor().isVmemCheckEnabled() &&
         this.containersAllocation.getVirtualMemory() +
-            (int) (vMemBytes >> 20) >
-            (int) (getContainersMonitor()
-                .getVmemAllocatedForContainers() >> 20)) {
+            convertBytesToMB(vMemBytes) >
+            convertBytesToMB(getContainersMonitor()
+                .getVmemAllocatedForContainers())) {
       return false;
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index b52e125367bca..4ae7936c83370 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -24,6 +24,7 @@
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -46,6 +47,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService
         .RecoveredContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -74,6 +76,7 @@ public class ContainerScheduler extends AbstractService implements
   private final Context context;
   // Capacity of the queue for opportunistic Containers.
   private final int maxOppQueueLength;
+  private final boolean forceStartGuaranteedContainers;
 
   // Queue of Guaranteed Containers waiting for resources to run
   private final LinkedHashMap<ContainerId, Container>
@@ -106,9 +109,37 @@ public class ContainerScheduler extends AbstractService implements
 
   private final AsyncDispatcher dispatcher;
   private final NodeManagerMetrics metrics;
+  private final OpportunisticContainersQueuePolicy oppContainersQueuePolicy;
 
   private Boolean usePauseEventForPreemption = false;
 
+  private static int getMaxOppQueueLengthFromConf(final Context context) {
+    if (context == null || context.getConf() == null) {
+      return YarnConfiguration
+          .DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH;
+    }
+
+    return context.getConf().getInt(
+        YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
+        YarnConfiguration.DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH
+    );
+  }
+
+  private static OpportunisticContainersQueuePolicy
+      getOppContainersQueuePolicyFromConf(final Context context) {
+    final OpportunisticContainersQueuePolicy queuePolicy;
+    if (context == null || context.getConf() == null) {
+      queuePolicy = OpportunisticContainersQueuePolicy.DEFAULT;
+    } else {
+      queuePolicy = context.getConf().getEnum(
+          YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_QUEUE_POLICY,
+          OpportunisticContainersQueuePolicy.DEFAULT
+      );
+    }
+
+    return queuePolicy;
+  }
+
   @VisibleForTesting
   ResourceHandlerChain resourceHandlerChain = null;
 
@@ -120,10 +151,9 @@ public class ContainerScheduler extends AbstractService implements
    */
   public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
       NodeManagerMetrics metrics) {
-    this(context, dispatcher, metrics, context.getConf().getInt(
-        YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
-        YarnConfiguration.
-            DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
+    this(context, dispatcher, metrics,
+        getOppContainersQueuePolicyFromConf(context),
+        getMaxOppQueueLengthFromConf(context));
   }
 
 
@@ -149,13 +179,35 @@ public void serviceInit(Configuration conf) throws Exception {
   @VisibleForTesting
   public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
       NodeManagerMetrics metrics, int qLength) {
+    this(context, dispatcher, metrics,
+        getOppContainersQueuePolicyFromConf(context), qLength);
+  }
+
+  @VisibleForTesting
+  public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
+      NodeManagerMetrics metrics,
+      OpportunisticContainersQueuePolicy oppContainersQueuePolicy,
+      int qLength) {
     super(ContainerScheduler.class.getName());
     this.context = context;
     this.dispatcher = dispatcher;
     this.metrics = metrics;
-    this.maxOppQueueLength = (qLength <= 0) ? 0 : qLength;
     this.utilizationTracker =
         new AllocationBasedResourceUtilizationTracker(this);
+    this.oppContainersQueuePolicy = oppContainersQueuePolicy;
+    switch (oppContainersQueuePolicy) {
+    case BY_RESOURCES:
+      this.maxOppQueueLength = 0;
+      this.forceStartGuaranteedContainers = false;
+      LOG.info("Setting max opportunistic queue length to 0,"
+              + " as {} is incompatible with queue length",
+          oppContainersQueuePolicy);
+      break;
+    case BY_QUEUE_LEN:
+    default:
+      this.maxOppQueueLength = qLength;
+      this.forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
+    }
     this.opportunisticContainersStatus =
         OpportunisticContainersStatus.newInstance();
   }
@@ -187,7 +239,7 @@ public void handle(ContainerSchedulerEvent event) {
       shedQueuedOpportunisticContainers();
       break;
     case RECOVERY_COMPLETED:
-      startPendingContainers(maxOppQueueLength <= 0);
+      startPendingContainers(forceStartGuaranteedContainers);
       metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
           queuedGuaranteedContainers.size());
       break;
@@ -243,7 +295,7 @@ private void onUpdateContainer(UpdateContainerSchedulerEvent updateEvent) {
         LOG.warn(String.format("Could not update resources on " +
             "continer update of %s", containerId), ex);
       }
-      startPendingContainers(maxOppQueueLength <= 0);
+      startPendingContainers(forceStartGuaranteedContainers);
       metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
           queuedGuaranteedContainers.size());
     }
@@ -371,7 +423,6 @@ private void onResourcesReclaimed(Container container) {
           ExecutionType.OPPORTUNISTIC) {
         this.metrics.completeOpportunisticContainer(container.getResource());
       }
-      boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
       startPendingContainers(forceStartGuaranteedContainers);
     }
     this.metrics.setQueuedContainers(queuedOpportunisticContainers.size(),
@@ -380,13 +431,13 @@ private void onResourcesReclaimed(Container container) {
 
   /**
    * Start pending containers in the queue.
-   * @param forceStartGuaranteedContaieners When this is true, start guaranteed
+   * @param forceStartGContainers When this is true, start guaranteed
    *        container without looking at available resource
    */
-  private void startPendingContainers(boolean forceStartGuaranteedContaieners) {
+  private void startPendingContainers(boolean forceStartGContainers) {
     // Start guaranteed containers that are paused, if resources available.
     boolean resourcesAvailable = startContainers(
-          queuedGuaranteedContainers.values(), forceStartGuaranteedContaieners);
+          queuedGuaranteedContainers.values(), forceStartGContainers);
     // Start opportunistic containers, if resources available.
     if (resourcesAvailable) {
       startContainers(queuedOpportunisticContainers.values(), false);
@@ -429,6 +480,21 @@ private boolean resourceAvailableToStartContainer(Container container) {
     return this.utilizationTracker.hasResourcesAvailable(container);
   }
 
+  private boolean resourceAvailableToQueueOppContainer(
+      Container newOppContainer) {
+    final Resource cumulativeResource = Resource.newInstance(Resources.none());
+    for (final Container container : queuedGuaranteedContainers.values()) {
+      Resources.addTo(cumulativeResource, container.getResource());
+    }
+
+    for (final Container container : queuedOpportunisticContainers.values()) {
+      Resources.addTo(cumulativeResource, container.getResource());
+    }
+
+    Resources.addTo(cumulativeResource, newOppContainer.getResource());
+    return this.utilizationTracker.hasResourcesAvailable(cumulativeResource);
+  }
+
   private boolean enqueueContainer(Container container) {
     boolean isGuaranteedContainer = container.getContainerTokenIdentifier().
         getExecutionType() == ExecutionType.GUARANTEED;
@@ -438,7 +504,21 @@ private boolean enqueueContainer(Container container) {
       queuedGuaranteedContainers.put(container.getContainerId(), container);
       isQueued = true;
     } else {
-      if (queuedOpportunisticContainers.size() < maxOppQueueLength) {
+      switch (oppContainersQueuePolicy) {
+      case BY_RESOURCES:
+        isQueued = resourceAvailableToQueueOppContainer(container);
+        break;
+      case BY_QUEUE_LEN:
+      default:
+        if (maxOppQueueLength <= 0) {
+          isQueued = false;
+        } else {
+          isQueued =
+              queuedOpportunisticContainers.size() < maxOppQueueLength;
+        }
+      }
+
+      if (isQueued) {
         LOG.info("Opportunistic container {} will be queued at the NM.",
             container.getContainerId());
         queuedOpportunisticContainers.put(
@@ -451,7 +531,6 @@ private boolean enqueueContainer(Container container) {
         container.sendKillEvent(
             ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
             "Opportunistic container queue is full.");
-        isQueued = false;
       }
     }
 
@@ -484,7 +563,6 @@ protected void scheduleContainer(Container container) {
       // When opportunistic container not allowed (which is determined by
       // max-queue length of pending opportunistic containers <= 0), start
       // guaranteed containers without looking at available resources.
-      boolean forceStartGuaranteedContainers = (maxOppQueueLength <= 0);
       startPendingContainers(forceStartGuaranteedContainers);
 
       // if the guaranteed container is queued, we need to preempt opportunistic
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/OpportunisticContainersQueuePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/OpportunisticContainersQueuePolicy.java
new file mode 100644
index 0000000000000..ce7aafbc4e98a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/OpportunisticContainersQueuePolicy.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Determines how to schedule opportunistic containers at the NodeManager,
+ * i.e., whether or not to accept, queue, or reject a container run request.
+ */
+public enum OpportunisticContainersQueuePolicy {
+  /**
+   * Determines whether or not to run a container by the queue capacity:
+   * {@link YarnConfiguration#NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH}.
+   * If there's enough capacity in the queue,
+   * queues the container, otherwise rejects it.
+   */
+  BY_QUEUE_LEN,
+  /**
+   * Determines whether or not to run a container based on the amount of
+   * resource capacity the node has.
+   * Sums up the resources running + already queued at the node, compares
+   * it with the total capacity of the node, and accepts the new container only
+   * if the computed resources above + resources used by the container
+   * is less than or equal to the node capacity.
+   */
+  BY_RESOURCES;
+
+  public static final OpportunisticContainersQueuePolicy DEFAULT = BY_QUEUE_LEN;
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
index 3c17ecaffef86..3d28de0204454 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ResourceUtilizationTracker.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
 
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 
@@ -56,4 +57,10 @@ public interface ResourceUtilizationTracker {
    */
   boolean hasResourcesAvailable(Container container);
 
+  /**
+   * Check if NM has resources available currently to run requested resources.
+   * @param resource the resources.
+   * @return True, if NM has enough available resources.
+   */
+  boolean hasResourcesAvailable(Resource resource);
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java
index eb2aaf54a3dee..c51d47a7217f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/logaggregation/tracker/NMLogAggregationStatusTracker.java
@@ -34,7 +34,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index cbdb424fc824b..81cfb2e743b7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -1813,4 +1813,21 @@ protected void checkVersion() throws IOException {
             + getCurrentVersion() + ", but loading version " + loadedVersion);
     }
   }
+  @Override
+  public void releaseAssignedResources(ContainerId containerId, String resourceType)
+      throws IOException {
+    LOG.debug("releaseAssignedResources: containerId=" + containerId + " resourceType="
+        + resourceType);
+    try {
+      try (WriteBatch batch = db.createWriteBatch()) {
+        String key = CONTAINERS_KEY_PREFIX + containerId
+            + CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType;
+        batch.delete(bytes(key));
+        db.write(batch);
+      }
+    }catch (DBException e){
+      markStoreUnHealthy(e);
+      throw new IOException(e);
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
index c6d8a1368aa33..aa56c6f247475 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
@@ -786,6 +786,15 @@ public abstract void storeAssignedResources(Container container,
       String resourceType, List<Serializable> assignedResources)
       throws IOException;
 
+  /**
+   * Delete the assigned resources of a container of specific resourceType.
+   * @param containerId Container Id
+   * @param resourceType resource Type
+   * @throws IOException while releasing resources
+   */
+  public void releaseAssignedResources(ContainerId containerId, String resourceType)
+      throws IOException {}
+
   protected abstract void initStorage(Configuration conf) throws IOException;
 
   protected abstract void startStorage() throws IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
index 86d9ae7687ae9..87d511b1725e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.nodemanager.webapp;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AppInfo.java
index f55ca810d8d07..3af7c9330fc62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/AppInfo.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 @XmlRootElement(name = "app")
 @XmlAccessorType(XmlAccessType.FIELD)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
index 5c166571ccc68..d21d6a0b72a6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformationParser.java
@@ -18,20 +18,27 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
 
+import java.io.StringReader;
+import javax.xml.XMLConstants;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.sax.SAXSource;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.InputSource;
 import org.xml.sax.XMLReader;
 
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.parsers.SAXParserFactory;
-import javax.xml.transform.sax.SAXSource;
-import java.io.StringReader;
+import static org.apache.hadoop.util.XMLUtils.EXTERNAL_GENERAL_ENTITIES;
+import static org.apache.hadoop.util.XMLUtils.EXTERNAL_PARAMETER_ENTITIES;
+import static org.apache.hadoop.util.XMLUtils.LOAD_EXTERNAL_DECL;
+import static org.apache.hadoop.util.XMLUtils.VALIDATION;
 
 /**
  * Parse XML and get GPU device information
@@ -68,10 +75,11 @@ public GpuDeviceInformationParser() throws YarnException {
    */
   private SAXParserFactory initSaxParserFactory() throws Exception {
     SAXParserFactory spf = SAXParserFactory.newInstance();
-    spf.setFeature(
-        "http://apache.org/xml/features/nonvalidating/load-external-dtd",
-        false);
-    spf.setFeature("http://xml.org/sax/features/validation", false);
+    spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+    spf.setFeature(LOAD_EXTERNAL_DECL, false);
+    spf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+    spf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+    spf.setFeature(VALIDATION, false);
     return spf;
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index baaa4dc4234d7..3a5109fba4b13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -546,8 +546,8 @@ int read_config(const char *file_path, struct configuration *cfg) {
 
   if (cfg->size == 0) {
     free_configuration(cfg);
-    fprintf(ERRORFILE, "Invalid configuration provided in %s\n", file_path);
-    return INVALID_CONFIG_FILE;
+    fprintf(ERRORFILE, "Empty configuration file provided %s\n", file_path);
+    exit(INVALID_CONFIG_FILE);
   }
   return 0;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
index e1ec293cd4702..b027e51bd311e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/get_executable.c
@@ -56,17 +56,17 @@ char *__get_exec_readproc(char *procfn) {
   filename = malloc(EXECUTOR_PATH_MAX);
   if (!filename) {
     fprintf(ERRORFILE,"cannot allocate memory for filename before readlink: %s\n",strerror(errno));
-    exit(-1);
+    exit(OUT_OF_MEMORY);
   }
   len = readlink(procfn, filename, EXECUTOR_PATH_MAX);
   if (len == -1) {
-    fprintf(ERRORFILE,"Can't get executable name from %s - %s\n", procfn,
+    fprintf(ERRORFILE,"Cannot get executable name from %s - %s\n", procfn,
             strerror(errno));
-    exit(-1);
+    exit(CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK);
   } else if (len >= EXECUTOR_PATH_MAX) {
     fprintf(ERRORFILE,"Resolved path for %s [%s] is longer than %d characters.\n",
             procfn, filename, EXECUTOR_PATH_MAX);
-    exit(-1);
+    exit(TOO_LONG_EXECUTOR_PATH);
   }
   filename[len] = '\0';
   return filename;
@@ -88,14 +88,14 @@ char *__get_exec_sysctl(int *mib)
 
   len = sizeof(buffer);
   if (sysctl(mib, 4, buffer, &len, NULL, 0) == -1) {
-    fprintf(ERRORFILE,"Can't get executable name from kernel: %s\n",
+    fprintf(ERRORFILE,"Cannot get executable name from kernel: %s\n",
       strerror(errno));
-    exit(-1);
+    exit(CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL);
   }
   filename=malloc(EXECUTOR_PATH_MAX);
   if (!filename) {
     fprintf(ERRORFILE,"cannot allocate memory for filename after sysctl: %s\n",strerror(errno));
-    exit(-1);
+    exit(OUT_OF_MEMORY);
   }
   snprintf(filename,EXECUTOR_PATH_MAX,"%s",buffer);
   return filename;
@@ -120,13 +120,13 @@ char* get_executable(char *argv0) {
   filename = malloc(PROC_PIDPATHINFO_MAXSIZE);
   if (!filename) {
     fprintf(ERRORFILE,"cannot allocate memory for filename before proc_pidpath: %s\n",strerror(errno));
-    exit(-1);
+    exit(OUT_OF_MEMORY);
   }
   pid = getpid();
   if (proc_pidpath(pid,filename,PROC_PIDPATHINFO_MAXSIZE) <= 0) {
-    fprintf(ERRORFILE,"Can't get executable name from pid %u - %s\n", pid,
+    fprintf(ERRORFILE,"Cannot get executable name from pid %u - %s\n", pid,
             strerror(errno));
-    exit(-1);
+    exit(CANNOT_GET_EXECUTABLE_NAME_FROM_PID);
   }
   return filename;
 }
@@ -194,7 +194,7 @@ char* get_executable (char *argv0) {
 
   if (!filename) {
     fprintf(ERRORFILE,"realpath of executable: %s\n",strerror(errno));
-    exit(-1);
+    exit(WRONG_PATH_OF_EXECUTABLE);
   }
   return filename;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
index ab1eab53248cf..b1c38f1873131 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
@@ -114,7 +114,7 @@ int update_cgroups_parameters(
 
   if (!full_path) {
     fprintf(ERRORFILE,
-      "Failed to get cgroups path to write, it should be a configuration issue");
+      "Failed to get cgroups path to write, it should be a configuration issue\n");
     failure = 1;
     goto cleanup;
   }
@@ -127,7 +127,7 @@ int update_cgroups_parameters(
   // Make sure file exists
   struct stat sb;
   if (stat(full_path, &sb) != 0) {
-    fprintf(ERRORFILE, "CGroups: Could not find file to write, %s", full_path);
+    fprintf(ERRORFILE, "CGroups: Could not find file to write, %s\n", full_path);
     failure = 1;
     goto cleanup;
   }
@@ -139,18 +139,18 @@ int update_cgroups_parameters(
   FILE *f;
   f = fopen(full_path, "a");
   if (!f) {
-    fprintf(ERRORFILE, "CGroups: Failed to open cgroups file, %s", full_path);
+    fprintf(ERRORFILE, "CGroups: Failed to open cgroups file, %s\n", full_path);
     failure = 1;
     goto cleanup;
   }
   if (fprintf(f, "%s", value) < 0) {
-    fprintf(ERRORFILE, "CGroups: Failed to write cgroups file, %s", full_path);
+    fprintf(ERRORFILE, "CGroups: Failed to write cgroups file, %s\n", full_path);
     fclose(f);
     failure = 1;
     goto cleanup;
   }
   if (fclose(f) != 0) {
-    fprintf(ERRORFILE, "CGroups: Failed to close cgroups file, %s", full_path);
+    fprintf(ERRORFILE, "CGroups: Failed to close cgroups file, %s\n", full_path);
     failure = 1;
     goto cleanup;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
index c8ee7b461e67b..33a388fc6465d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c
@@ -337,6 +337,16 @@ const char *get_error_message(const int error_code) {
         return "runC run failed";
       case ERROR_RUNC_REAP_LAYER_MOUNTS_FAILED:
         return "runC reap layer mounts failed";
+      case CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK:
+        return "Cannot get executable name from readlink";
+      case TOO_LONG_EXECUTOR_PATH:
+        return "Too long executor path";
+      case CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL:
+        return "Cannot get executable name from kernel";
+      case CANNOT_GET_EXECUTABLE_NAME_FROM_PID:
+        return "Cannot get executable name from pid";
+      case WRONG_PATH_OF_EXECUTABLE:
+        return "Wrong path of executable";
       default:
         return "Unknown error code";
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index 920888f1effbe..73dfeb629d7f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -104,7 +104,12 @@ enum errorcodes {
   ERROR_RUNC_SETUP_FAILED = 76,
   ERROR_RUNC_RUN_FAILED = 77,
   ERROR_RUNC_REAP_LAYER_MOUNTS_FAILED = 78,
-  ERROR_DOCKER_CONTAINER_EXEC_FAILED = 79
+  ERROR_DOCKER_CONTAINER_EXEC_FAILED = 79,
+  CANNOT_GET_EXECUTABLE_NAME_FROM_READLINK = 80,
+  TOO_LONG_EXECUTOR_PATH = 81,
+  CANNOT_GET_EXECUTABLE_NAME_FROM_KERNEL = 82,
+  CANNOT_GET_EXECUTABLE_NAME_FROM_PID = 83,
+  WRONG_PATH_OF_EXECUTABLE = 84
 };
 
 /* Macros for min/max. */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 7bf042f5ebf9d..c3d46bb8095f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
 import org.junit.After;
 import org.junit.Assume;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index a5c115283a021..473292f35f749 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -20,8 +20,8 @@
 
 import static org.apache.hadoop.fs.CreateFlag.CREATE;
 import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.isA;
@@ -39,10 +39,12 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -60,23 +62,32 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
 import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings.AssignedResources;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocator;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.MockLocalizerHeartbeatResponse;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
 import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -86,6 +97,14 @@ public class TestDefaultContainerExecutor {
   private static Path BASE_TMP_PATH = new Path("target",
       TestDefaultContainerExecutor.class.getSimpleName());
 
+  private YarnConfiguration yarnConfiguration;
+
+  private DefaultContainerExecutor containerExecutor;
+
+  private Container mockContainer;
+
+  private NumaResourceAllocator numaResourceAllocator;
+
   @AfterClass
   public static void deleteTmpFiles() throws IOException {
     FileContext lfs = FileContext.getLocalFSFileContext();
@@ -736,4 +755,204 @@ public void testPickDirectory() throws Exception {
 //        new FsPermission(ApplicationLocalizer.LOGDIR_PERM), true);
 //  }
 
+  @Before
+  public void setUp() throws IOException, YarnException {
+    yarnConfiguration = new YarnConfiguration();
+    setNumaConfig();
+    Context mockContext = createAndGetMockContext();
+    NMStateStoreService nmStateStoreService =
+            mock(NMStateStoreService.class);
+    when(mockContext.getNMStateStore()).thenReturn(nmStateStoreService);
+    numaResourceAllocator = new NumaResourceAllocator(mockContext) {
+      @Override
+      public String executeNGetCmdOutput(Configuration config)
+              throws YarnRuntimeException {
+        return getNumaCmdOutput();
+      }
+    };
+
+    numaResourceAllocator.init(yarnConfiguration);
+    FileContext lfs = FileContext.getLocalFSFileContext();
+    containerExecutor = new DefaultContainerExecutor(lfs) {
+      @Override
+      public Configuration getConf() {
+        return yarnConfiguration;
+      }
+    };
+    containerExecutor.setNumaResourceAllocator(numaResourceAllocator);
+    mockContainer = mock(Container.class);
+  }
+
+  private void setNumaConfig() {
+    yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_ENABLED, "true");
+    yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_READ_TOPOLOGY, "true");
+    yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD, "/usr/bin/numactl");
+  }
+
+
+  private String getNumaCmdOutput() {
+    // architecture of 8 cpu cores
+    // randomly picked size of memory
+    return "available: 2 nodes (0-1)\n\t"
+            + "node 0 cpus: 0 2 4 6\n\t"
+            + "node 0 size: 73717 MB\n\t"
+            + "node 0 free: 73717 MB\n\t"
+            + "node 1 cpus: 1 3 5 7\n\t"
+            + "node 1 size: 73717 MB\n\t"
+            + "node 1 free: 73717 MB\n\t"
+            + "node distances:\n\t"
+            + "node 0 1\n\t"
+            + "0: 10 20\n\t"
+            + "1: 20 10";
+  }
+
+  private Context createAndGetMockContext() {
+    Context mockContext = mock(Context.class);
+    @SuppressWarnings("unchecked")
+    ConcurrentHashMap<ContainerId, Container> mockContainers = mock(
+            ConcurrentHashMap.class);
+    mockContainer = mock(Container.class);
+    when(mockContainer.getResourceMappings())
+            .thenReturn(new ResourceMappings());
+    when(mockContainers.get(any())).thenReturn(mockContainer);
+    when(mockContext.getContainers()).thenReturn(mockContainers);
+    when(mockContainer.getResource()).thenReturn(Resource.newInstance(2048, 2));
+    return mockContext;
+  }
+
+  private void testAllocateNumaResource(String containerId, Resource resource,
+                                        String memNodes, String cpuNodes) throws Exception {
+    when(mockContainer.getContainerId())
+            .thenReturn(ContainerId.fromString(containerId));
+    when(mockContainer.getResource()).thenReturn(resource);
+    NumaResourceAllocation numaResourceAllocation =
+            numaResourceAllocator.allocateNumaNodes(mockContainer);
+    containerExecutor.setNumactl(containerExecutor.getConf().get(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD,
+            YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD));
+    String[] commands = containerExecutor.getNumaCommands(numaResourceAllocation);
+    assertEquals(Arrays.asList(commands), Arrays.asList("/usr/bin/numactl",
+            "--interleave=" + memNodes, "--cpunodebind=" + cpuNodes));
+  }
+
+  @Test
+  public void testAllocateNumaMemoryResource() throws Exception {
+    // keeping cores constant for testing memory resources
+
+    // allocates node 0 for memory and cpu
+    testAllocateNumaResource("container_1481156246874_0001_01_000001",
+            Resource.newInstance(2048, 2), "0", "0");
+
+    // allocates node 1 for memory and cpu since allocator uses round robin assignment
+    testAllocateNumaResource("container_1481156246874_0001_01_000002",
+            Resource.newInstance(60000, 2), "1", "1");
+
+    // allocates node 0,1 for memory since there is no sufficient memory in any one node
+    testAllocateNumaResource("container_1481156246874_0001_01_000003",
+            Resource.newInstance(80000, 2), "0,1", "0");
+
+    // returns null since there are no sufficient resources available for the request
+    when(mockContainer.getContainerId()).thenReturn(
+            ContainerId.fromString("container_1481156246874_0001_01_000004"));
+    when(mockContainer.getResource())
+            .thenReturn(Resource.newInstance(80000, 2));
+    Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer));
+
+    // allocates node 1 for memory and cpu
+    testAllocateNumaResource("container_1481156246874_0001_01_000005",
+            Resource.newInstance(1024, 2), "1", "1");
+  }
+
+  @Test
+  public void testAllocateNumaCpusResource() throws Exception {
+    // keeping memory constant
+
+    // allocates node 0 for memory and cpu
+    testAllocateNumaResource("container_1481156246874_0001_01_000001",
+            Resource.newInstance(2048, 2), "0", "0");
+
+    // allocates node 1 for memory and cpu since allocator uses round robin assignment
+    testAllocateNumaResource("container_1481156246874_0001_01_000002",
+            Resource.newInstance(2048, 2), "1", "1");
+
+    // allocates node 0,1 for cpus since there is are no sufficient cpus available in any one node
+    testAllocateNumaResource("container_1481156246874_0001_01_000003",
+            Resource.newInstance(2048, 3), "0", "0,1");
+
+    // returns null since there are no sufficient resources available for the request
+    when(mockContainer.getContainerId()).thenReturn(
+            ContainerId.fromString("container_1481156246874_0001_01_000004"));
+    when(mockContainer.getResource()).thenReturn(Resource.newInstance(2048, 2));
+    Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer));
+
+    // allocates node 1 for memory and cpu
+    testAllocateNumaResource("container_1481156246874_0001_01_000005",
+            Resource.newInstance(2048, 1), "1", "1");
+  }
+
+  @Test
+  public void testReacquireContainer() throws Exception {
+    @SuppressWarnings("unchecked")
+    ConcurrentHashMap<ContainerId, Container> mockContainers = mock(
+            ConcurrentHashMap.class);
+    Context mockContext = mock(Context.class);
+    NMStateStoreService mock = mock(NMStateStoreService.class);
+    when(mockContext.getNMStateStore()).thenReturn(mock);
+    ResourceMappings resourceMappings = new ResourceMappings();
+    AssignedResources assignedRscs = new AssignedResources();
+    when(mockContainer.getResource())
+            .thenReturn(Resource.newInstance(147434, 2));
+    ContainerId cid = ContainerId.fromString("container_1481156246874_0001_01_000001");
+    when(mockContainer.getContainerId()).thenReturn(cid);
+    NumaResourceAllocation numaResourceAllocation =
+            numaResourceAllocator.allocateNumaNodes(mockContainer);
+    assignedRscs.updateAssignedResources(Arrays.asList(numaResourceAllocation));
+    resourceMappings.addAssignedResources("numa", assignedRscs);
+    when(mockContainer.getResourceMappings()).thenReturn(resourceMappings);
+    when(mockContainers.get(any())).thenReturn(mockContainer);
+    when(mockContext.getContainers()).thenReturn(mockContainers);
+
+    // recovered numa resources should be added to the used resources and
+    // remaining will be available for further allocation.
+
+    ContainerReacquisitionContext containerReacquisitionContext =
+            new ContainerReacquisitionContext.Builder()
+                    .setContainerId(cid)
+                    .setUser("user")
+                    .setContainer(mockContainer)
+                    .build();
+
+    containerExecutor.reacquireContainer(containerReacquisitionContext);
+
+    // reacquireContainer recovers all the numa resources ,
+    // that should be free to use next
+    testAllocateNumaResource("container_1481156246874_0001_01_000001",
+            Resource.newInstance(147434, 2), "0,1", "1");
+    when(mockContainer.getContainerId()).thenReturn(
+            ContainerId.fromString("container_1481156246874_0001_01_000004"));
+    when(mockContainer.getResource())
+            .thenReturn(Resource.newInstance(1024, 2));
+
+    // returns null since there are no sufficient resources available for the request
+    Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer));
+  }
+
+  @Test
+  public void testConcatStringCommands() {
+    // test one array of string as null
+    assertEquals(containerExecutor.concatStringCommands(null, new String[]{"hello"})[0],
+            new String[]{"hello"}[0]);
+    // test both array of string as null
+    Assert.assertNull(containerExecutor.concatStringCommands(null, null));
+    // test case when both arrays are not null and of equal length
+    String[] res = containerExecutor.concatStringCommands(new String[]{"one"},
+            new String[]{"two"});
+    assertEquals(res[0]+res[1], "one" + "two");
+    // test both array of different length
+    res = containerExecutor.concatStringCommands(new String[]{"one"},
+            new String[]{"two", "three"});
+    assertEquals(res[0] + res[1] + res[2], "one" + "two" + "three");
+
+  }
+
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
index 59a30370283de..33bd4d92347ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
@@ -35,8 +35,6 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Before;
 import org.junit.Test;
 
 public class TestDirectoryCollection {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index b3c401453fd24..511013cad19ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -181,7 +181,7 @@ public RegisterNodeManagerResponse registerNodeManager(
         IOException {
       NodeId nodeId = request.getNodeId();
       Resource resource = request.getResource();
-      LOG.info("Registering " + nodeId.toString());
+      LOG.info("Registering {}.", nodeId.toString());
       // NOTE: this really should be checking against the config value
       InetSocketAddress expected = NetUtils.getConnectAddress(
           conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1));
@@ -217,7 +217,7 @@ private Map<ApplicationId, List<ContainerStatus>> getAppToContainerStatusMap(
     public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
         throws YarnException, IOException {
       NodeStatus nodeStatus = request.getNodeStatus();
-      LOG.info("Got heartbeat number " + heartBeatID);
+      LOG.info("Got heartbeat number {}.", heartBeatID);
       NodeManagerMetrics mockMetrics = mock(NodeManagerMetrics.class);
       Dispatcher mockDispatcher = mock(Dispatcher.class);
       @SuppressWarnings("unchecked")
@@ -625,7 +625,7 @@ public RegisterNodeManagerResponse registerNodeManager(
     @Override
     public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
         throws YarnException, IOException {
-      LOG.info("Got heartBeatId: [" + heartBeatID +"]");
+      LOG.info("Got heartBeatId: [{}]", heartBeatID);
       NodeStatus nodeStatus = request.getNodeStatus();
       nodeStatus.setResponseId(heartBeatID.getAndIncrement());
       NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils.
@@ -644,7 +644,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
         }
       }
       if (heartBeatID.get() == 2) {
-        LOG.info("Sending FINISH_APP for application: [" + appId + "]");
+        LOG.info("Sending FINISH_APP for application: [{}]", appId);
         this.context.getApplications().put(appId, mock(Application.class));
         nhResponse.addAllApplicationsToCleanup(Collections.singletonList(appId));
       }
@@ -1528,7 +1528,7 @@ public void testApplicationKeepAlive() throws Exception {
       rt.context.getApplications().remove(rt.appId);
       Assert.assertEquals(1, rt.keepAliveRequests.size());
       int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size();
-      LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]");
+      LOG.info("Number of Keep Alive Requests: [{}]", numKeepAliveRequests);
       Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3);
       GenericTestUtils.waitFor(
           () -> nm.getServiceState() != STATE.STARTED
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 2ecbf316eba60..62b59617fd4a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -134,7 +134,7 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
+    // Create a request interceptor pipeline for testing. The last one in the
     // chain will call the mock resource manager. The others in the chain will
     // simply forward it to the next one in the chain
     config.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE,
@@ -191,7 +191,7 @@ protected Context createContext() {
     return new NMContext(null, null, null, null, stateStore, false, this.conf);
   }
 
-  // A utility method for intercepter recover unit test
+  // A utility method for interceptor recover unit test
   protected Map<String, byte[]> recoverDataMapForAppAttempt(
       NMStateStoreService nmStateStore, ApplicationAttemptId attemptId)
       throws IOException {
@@ -341,7 +341,7 @@ public RegisterApplicationMasterResponseInfo<T> invoke(
                   Assert.assertEquals(Integer.toString(index), response
                       .getResponse().getQueue());
 
-                  LOG.info("Sucessfully registered application master with test context: "
+                  LOG.info("Successfully registered application master with test context: "
                       + testContext);
                 } catch (Throwable ex) {
                   response = null;
@@ -424,7 +424,7 @@ public FinishApplicationMasterResponseInfo<T> invoke(
                           testContext);
                   Assert.assertNotNull(response.getResponse());
 
-                  LOG.info("Sucessfully finished application master with test contexts: "
+                  LOG.info("Successfully finished application master with test contexts: "
                       + testContext);
                 } catch (Throwable ex) {
                   response = null;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/PassThroughRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/PassThroughRequestInterceptor.java
index 97a844eb77893..93a0d6ef27e7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/PassThroughRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/PassThroughRequestInterceptor.java
@@ -29,8 +29,8 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 
 /**
- * Mock intercepter that does not do anything other than forwarding it to the
- * next intercepter in the chain
+ * Mock interceptor that does not do anything other than forwarding it to the
+ * next interceptor in the chain
  *
  */
 public class PassThroughRequestInterceptor extends
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java
index 4621c4dce32ca..134edb7d6e4d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java
@@ -42,11 +42,19 @@ public static void init() {
     Assert.assertEquals(0, metrics.getFailedRegisterAMRequests());
     Assert.assertEquals(0, metrics.getFailedFinishAMRequests());
     Assert.assertEquals(0, metrics.getFailedAllocateRequests());
+    Assert.assertEquals(0, metrics.getFailedAppRecoveryCount());
+    Assert.assertEquals(0, metrics.getFailedAppStopRequests());
+    Assert.assertEquals(0, metrics.getFailedUpdateAMRMTokenRequests());
+    Assert.assertEquals(0, metrics.getAllocateCount());
+    Assert.assertEquals(0, metrics.getRequestCount());
 
     Assert.assertEquals(0, metrics.getNumSucceededAppStartRequests());
     Assert.assertEquals(0, metrics.getNumSucceededRegisterAMRequests());
     Assert.assertEquals(0, metrics.getNumSucceededFinishAMRequests());
     Assert.assertEquals(0, metrics.getNumSucceededAllocateRequests());
+    Assert.assertEquals(0, metrics.getNumSucceededRecoverRequests());
+    Assert.assertEquals(0, metrics.getNumSucceededAppStopRequests());
+    Assert.assertEquals(0, metrics.getNumSucceededUpdateAMRMTokenRequests());
 
     LOG.info("Test: aggregate metrics are updated correctly");
   }
@@ -57,37 +65,36 @@ public void testAllocateRequestWithNullValues() throws Exception {
     long failedRegisterAMRequests = metrics.getFailedRegisterAMRequests();
     long failedFinishAMRequests = metrics.getFailedFinishAMRequests();
     long failedAllocateRequests = metrics.getFailedAllocateRequests();
+    long failedAppRecoveryRequests = metrics.getFailedAppRecoveryCount();
+    long failedAppStopRequests = metrics.getFailedAppStopRequests();
+    long failedUpdateAMRMTokenRequests = metrics.getFailedUpdateAMRMTokenRequests();
 
     long succeededAppStartRequests = metrics.getNumSucceededAppStartRequests();
-    long succeededRegisterAMRequests =
-        metrics.getNumSucceededRegisterAMRequests();
+    long succeededRegisterAMRequests = metrics.getNumSucceededRegisterAMRequests();
     long succeededFinishAMRequests = metrics.getNumSucceededFinishAMRequests();
     long succeededAllocateRequests = metrics.getNumSucceededAllocateRequests();
 
     int testAppId = 1;
-    RegisterApplicationMasterResponse registerResponse =
-        registerApplicationMaster(testAppId);
+    RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(testAppId);
     Assert.assertNotNull(registerResponse);
-    Assert
-        .assertEquals(Integer.toString(testAppId), registerResponse.getQueue());
+    Assert.assertEquals(Integer.toString(testAppId), registerResponse.getQueue());
 
     AllocateResponse allocateResponse = allocate(testAppId);
     Assert.assertNotNull(allocateResponse);
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId, FinalApplicationStatus.SUCCEEDED);
 
-    Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(true, finshResponse.getIsUnregistered());
+    Assert.assertNotNull(finishResponse);
+    Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
-    Assert.assertEquals(failedAppStartRequests,
-        metrics.getFailedAppStartRequests());
-    Assert.assertEquals(failedRegisterAMRequests,
-        metrics.getFailedRegisterAMRequests());
-    Assert.assertEquals(failedFinishAMRequests,
-        metrics.getFailedFinishAMRequests());
-    Assert.assertEquals(failedAllocateRequests,
-        metrics.getFailedAllocateRequests());
+    Assert.assertEquals(failedAppStartRequests, metrics.getFailedAppStartRequests());
+    Assert.assertEquals(failedRegisterAMRequests, metrics.getFailedRegisterAMRequests());
+    Assert.assertEquals(failedFinishAMRequests, metrics.getFailedFinishAMRequests());
+    Assert.assertEquals(failedAllocateRequests, metrics.getFailedAllocateRequests());
+    Assert.assertEquals(failedAppRecoveryRequests, metrics.getFailedAppRecoveryCount());
+    Assert.assertEquals(failedAppStopRequests, metrics.getFailedAppStopRequests());
+    Assert.assertEquals(failedUpdateAMRMTokenRequests, metrics.getFailedUpdateAMRMTokenRequests());
 
     Assert.assertEquals(succeededAppStartRequests,
         metrics.getNumSucceededAppStartRequests());
@@ -119,10 +126,10 @@ public void testFinishOneApplicationMasterWithFailure() throws Exception {
     Assert
         .assertEquals(Integer.toString(testAppId), registerResponse.getQueue());
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId, FinalApplicationStatus.FAILED);
 
-    Assert.assertNotNull(finshResponse);
+    Assert.assertNotNull(finishResponse);
 
     try {
       // Try to finish an application master that is already finished.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 60e383870fe28..10a68cfb33a25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -128,7 +128,7 @@ public void testInterceptorInitFailure() throws IOException {
           getAMRMProxyService().getPipelines();
       ApplicationId id = getApplicationId(testAppId);
       Assert.assertTrue(
-          "The interceptor pipeline should be removed if initializtion fails",
+          "The interceptor pipeline should be removed if initialization fails",
           pipelines.get(id) == null);
     }
   }
@@ -140,7 +140,7 @@ public void testInterceptorInitFailure() throws IOException {
    * @throws Exception
    */
   @Test
-  public void testRegisterMulitpleApplicationMasters() throws Exception {
+  public void testRegisterMultipleApplicationMasters() throws Exception {
     for (int testAppId = 0; testAppId < 3; testAppId++) {
       RegisterApplicationMasterResponse response =
           registerApplicationMaster(testAppId);
@@ -157,7 +157,7 @@ public void testRegisterMulitpleApplicationMasters() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testRegisterMulitpleApplicationMastersInParallel()
+  public void testRegisterMultipleApplicationMastersInParallel()
       throws Exception {
     int numberOfRequests = 5;
     ArrayList<String> testContexts =
@@ -185,12 +185,12 @@ public void testFinishOneApplicationMasterWithSuccess() throws Exception {
     Assert.assertEquals(Integer.toString(testAppId),
         registerResponse.getQueue());
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId,
             FinalApplicationStatus.SUCCEEDED);
 
-    Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(true, finshResponse.getIsUnregistered());
+    Assert.assertNotNull(finishResponse);
+    Assert.assertEquals(true, finishResponse.getIsUnregistered());
   }
 
   @Test
@@ -202,10 +202,10 @@ public void testFinishOneApplicationMasterWithFailure() throws Exception {
     Assert.assertEquals(Integer.toString(testAppId),
         registerResponse.getQueue());
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId, FinalApplicationStatus.FAILED);
 
-    Assert.assertNotNull(finshResponse);
+    Assert.assertNotNull(finishResponse);
 
     try {
       // Try to finish an application master that is already finished.
@@ -232,7 +232,7 @@ public void testFinishInvalidApplicationMaster() throws Exception {
   }
 
   @Test
-  public void testFinishMulitpleApplicationMasters() throws Exception {
+  public void testFinishMultipleApplicationMasters() throws Exception {
     int numberOfRequests = 3;
     for (int index = 0; index < numberOfRequests; index++) {
       RegisterApplicationMasterResponse registerResponse =
@@ -244,11 +244,11 @@ public void testFinishMulitpleApplicationMasters() throws Exception {
 
     // Finish in reverse sequence
     for (int index = numberOfRequests - 1; index >= 0; index--) {
-      FinishApplicationMasterResponse finshResponse =
+      FinishApplicationMasterResponse finishResponse =
           finishApplicationMaster(index, FinalApplicationStatus.SUCCEEDED);
 
-      Assert.assertNotNull(finshResponse);
-      Assert.assertEquals(true, finshResponse.getIsUnregistered());
+      Assert.assertNotNull(finishResponse);
+      Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
       // Assert that the application has been removed from the collection
       Assert.assertTrue(this.getAMRMProxyService()
@@ -277,7 +277,7 @@ public void testFinishMulitpleApplicationMasters() throws Exception {
   }
 
   @Test
-  public void testFinishMulitpleApplicationMastersInParallel()
+  public void testFinishMultipleApplicationMastersInParallel()
       throws Exception {
     int numberOfRequests = 5;
     ArrayList<String> testContexts = new ArrayList<String>();
@@ -308,12 +308,12 @@ public void testAllocateRequestWithNullValues() throws Exception {
     AllocateResponse allocateResponse = allocate(testAppId);
     Assert.assertNotNull(allocateResponse);
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId,
             FinalApplicationStatus.SUCCEEDED);
 
-    Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(true, finshResponse.getIsUnregistered());
+    Assert.assertNotNull(finishResponse);
+    Assert.assertEquals(true, finishResponse.getIsUnregistered());
   }
 
   @Test
@@ -399,7 +399,7 @@ public Integer invoke(Integer testAppId) {
                   getContainersAndAssert(testAppId, 10);
               releaseContainersAndAssert(testAppId, containers);
 
-              LOG.info("Sucessfully registered application master with appId: "
+              LOG.info("Successfully registered application master with appId: "
                   + testAppId);
             } catch (Throwable ex) {
               LOG.error(
@@ -533,7 +533,7 @@ private void releaseContainersAndAssert(int appId,
 
     // We need to make sure all the resource managers received the
     // release list. The containers sent by the mock resource managers will be
-    // aggregated and returned back to us and we can assert if all the release
+    // aggregated and returned back to us, and we can assert if all the release
     // lists reached the sub-clusters
     List<ContainerId> containersForReleasedContainerIds = new ArrayList<>();
     List<ContainerId> newlyFinished = getCompletedContainerIds(
@@ -607,19 +607,19 @@ public void testRecovery() throws YarnException, Exception {
     allocateResponse = allocate(testAppId1);
     Assert.assertNotNull(allocateResponse);
 
-    FinishApplicationMasterResponse finshResponse =
+    FinishApplicationMasterResponse finishResponse =
         finishApplicationMaster(testAppId1, FinalApplicationStatus.SUCCEEDED);
-    Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(true, finshResponse.getIsUnregistered());
+    Assert.assertNotNull(finishResponse);
+    Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
     allocateResponse = allocate(testAppId2);
     Assert.assertNotNull(allocateResponse);
 
-    finshResponse =
+    finishResponse =
         finishApplicationMaster(testAppId2, FinalApplicationStatus.SUCCEEDED);
 
-    Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(true, finshResponse.getIsUnregistered());
+    Assert.assertNotNull(finishResponse);
+    Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
     int testAppId3 = 3;
     try {
@@ -684,7 +684,7 @@ public void testCheckIfAppExistsInStateStore()
   }
 
   /**
-   * A mock intercepter implementation that uses the same mockRM instance across
+   * A mock interceptor implementation that uses the same mockRM instance across
    * restart.
    */
   public static class MockRequestInterceptorAcrossRestart
@@ -723,7 +723,7 @@ public AllocateResponse allocate(AllocateRequest request)
   }
 
   /**
-   * A mock intercepter implementation that throws when recovering.
+   * A mock interceptor implementation that throws when recovering.
    */
   public static class BadRequestInterceptorAcrossRestart
       extends MockRequestInterceptorAcrossRestart {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index 3288382b26de0..51c1c5a096589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -26,16 +26,19 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.HashMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
 
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -57,10 +60,12 @@
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
 import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.MockResourceManagerFacade;
 import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
@@ -69,11 +74,13 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager;
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.Assert;
 import org.junit.Test;
@@ -84,7 +91,7 @@
  * Extends the TestAMRMProxyService and overrides methods in order to use the
  * AMRMProxyService's pipeline test cases for testing the FederationInterceptor
  * class. The tests for AMRMProxyService has been written cleverly so that it
- * can be reused to validate different request intercepter chains.
+ * can be reused to validate different request interceptor chains.
  */
 public class TestFederationInterceptor extends BaseAMRMProxyTest {
   private static final Logger LOG =
@@ -148,8 +155,8 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
-    // chain is the federation intercepter that calls the mock resource manager.
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
     // The others in the chain will simply forward it to the next one in the
     // chain
     conf.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE,
@@ -255,7 +262,7 @@ private void releaseContainersAndAssert(List<Container> containers)
 
     // The release request will be split and handled by the corresponding UAM.
     // The release containers returned by the mock resource managers will be
-    // aggregated and returned back to us and we can check if total request size
+    // aggregated and returned back to us, and we can check if total request size
     // and returned size are the same
     List<ContainerId> containersForReleasedContainerIds =
         new ArrayList<ContainerId>();
@@ -361,10 +368,10 @@ public Object run() throws Exception {
         finishReq.setTrackingUrl("");
         finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
 
-        FinishApplicationMasterResponse finshResponse =
+        FinishApplicationMasterResponse finishResponse =
             interceptor.finishApplicationMaster(finishReq);
-        Assert.assertNotNull(finshResponse);
-        Assert.assertEquals(true, finshResponse.getIsUnregistered());
+        Assert.assertNotNull(finishResponse);
+        Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
         return null;
       }
@@ -421,10 +428,10 @@ public Object run() throws Exception {
         finishReq.setTrackingUrl("");
         finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
 
-        FinishApplicationMasterResponse finshResponse =
+        FinishApplicationMasterResponse finishResponse =
             interceptor.finishApplicationMaster(finishReq);
-        Assert.assertNotNull(finshResponse);
-        Assert.assertEquals(true, finshResponse.getIsUnregistered());
+        Assert.assertNotNull(finishResponse);
+        Assert.assertEquals(true, finishResponse.getIsUnregistered());
         return null;
       }
     });
@@ -513,6 +520,16 @@ public void testRecoverWithoutAMRMProxyHA() throws Exception {
     testRecover(null);
   }
 
+  @Test
+  public void testRecoverBadSCWithAMRMProxyHA() throws Exception {
+    testRecoverWithBadSubCluster(registry);
+  }
+
+  @Test
+  public void testRecoverBadSCWithoutAMRMProxyHA() throws Exception {
+    testRecoverWithBadSubCluster(null);
+  }
+
   protected void testRecover(final RegistryOperations registryObj)
       throws Exception {
     UserGroupInformation ugi =
@@ -569,7 +586,7 @@ public Object run() throws Exception {
         ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
             interceptor.getSecondaryRMs();
 
-        // Create a new intercepter instance and recover
+        // Create a new interceptor instance and recover
         interceptor = new TestableFederationInterceptor(homeRM, secondaries);
         interceptor.init(new AMRMProxyApplicationContextImpl(nmContext,
             getConf(), attemptId, "test-user", null, null, null, registryObj));
@@ -604,10 +621,10 @@ public Object run() throws Exception {
         finishReq.setTrackingUrl("");
         finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
 
-        FinishApplicationMasterResponse finshResponse =
+        FinishApplicationMasterResponse finishResponse =
             interceptor.finishApplicationMaster(finishReq);
-        Assert.assertNotNull(finshResponse);
-        Assert.assertEquals(true, finshResponse.getIsUnregistered());
+        Assert.assertNotNull(finishResponse);
+        Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
         // After the application succeeds, the registry/NMSS entry should be
         // cleaned up
@@ -845,7 +862,7 @@ public Object run() throws Exception {
         ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
             interceptor.getSecondaryRMs();
 
-        // Increase the attemptId and create a new intercepter instance for it
+        // Increase the attemptId and create a new interceptor instance for it
         attemptId = ApplicationAttemptId.newInstance(
             attemptId.getApplicationId(), attemptId.getAttemptId() + 1);
 
@@ -884,10 +901,10 @@ public Object run() throws Exception {
         finishReq.setTrackingUrl("");
         finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
 
-        FinishApplicationMasterResponse finshResponse =
+        FinishApplicationMasterResponse finishResponse =
             interceptor.finishApplicationMaster(finishReq);
-        Assert.assertNotNull(finshResponse);
-        Assert.assertEquals(true, finshResponse.getIsUnregistered());
+        Assert.assertNotNull(finishResponse);
+        Assert.assertEquals(true, finishResponse.getIsUnregistered());
 
         // After the application succeeds, the registry entry should be deleted
         if (interceptor.getRegistryClient() != null) {
@@ -969,4 +986,450 @@ private PreemptionMessage createDummyPreemptionMessage(
     preemptionMessage.setContract(contract);
     return preemptionMessage;
   }
+
+  @Test
+  public void testSameContainerFromDiffRM() throws IOException, InterruptedException {
+
+    UserGroupInformation ugi =
+        interceptor.getUGIWithToken(interceptor.getAttemptId());
+
+    ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+
+      // Register the application
+      RegisterApplicationMasterRequest registerReq =
+          Records.newRecord(RegisterApplicationMasterRequest.class);
+      registerReq.setHost(Integer.toString(testAppId));
+      registerReq.setRpcPort(0);
+      registerReq.setTrackingUrl("");
+
+      RegisterApplicationMasterResponse registerResponse =
+          interceptor.registerApplicationMaster(registerReq);
+      Assert.assertNotNull(registerResponse);
+      lastResponseId = 0;
+
+      Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+      // Allocate the first batch of containers, with sc1 active
+      SubClusterId subClusterId1 = SubClusterId.newInstance("SC-1");
+      registerSubCluster(subClusterId1);
+
+      int numberOfContainers = 3;
+      List<Container> containers =
+          getContainersAndAssert(numberOfContainers, numberOfContainers);
+      Assert.assertNotNull(containers);
+      Assert.assertEquals(3, containers.size());
+
+      // with sc2 active
+      SubClusterId subClusterId2 = SubClusterId.newInstance("SC-2");
+      registerSubCluster(subClusterId2);
+
+      // 1.Container has been registered to SubCluster1, try to register the same Container
+      // to SubCluster2.
+      // Because SubCluster1 is in normal state at this time,
+      // So the SubCluster corresponding to Container should be SubCluster1
+      interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId2);
+      Map<ContainerId, SubClusterId> cIdToSCMap = interceptor.getContainerIdToSubClusterIdMap();
+      for (SubClusterId subClusterId : cIdToSCMap.values()) {
+        Assert.assertNotNull(subClusterId);
+        Assert.assertEquals(subClusterId1, subClusterId);
+      }
+
+      // 2.Deregister SubCluster1, Register the same Containers to SubCluster2
+      // So the SubCluster corresponding to Container should be SubCluster2
+      deRegisterSubCluster(subClusterId1);
+      interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId2);
+      Map<ContainerId, SubClusterId> cIdToSCMap2 = interceptor.getContainerIdToSubClusterIdMap();
+      for (SubClusterId subClusterId : cIdToSCMap2.values()) {
+        Assert.assertNotNull(subClusterId);
+        Assert.assertEquals(subClusterId2, subClusterId);
+      }
+
+      // 3.Deregister subClusterId2, Register the same Containers to SubCluster1
+      // Because both SubCluster1 and SubCluster2 are abnormal at this time,
+      // an exception will be thrown when registering the first Container.
+      deRegisterSubCluster(subClusterId2);
+      Container container1 = containers.get(0);
+      Assert.assertNotNull(container1);
+      String errMsg =
+          " Can't use any subCluster because an exception occurred" +
+          " ContainerId: " + container1.getId() +
+          " ApplicationId: " + interceptor.getAttemptId() +
+          " From RM: " + subClusterId1 + ". " +
+          " Previous Container was From subCluster: " + subClusterId2;
+
+      LambdaTestUtils.intercept(YarnRuntimeException.class, errMsg,
+          () -> interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId1));
+
+      // 4. register SubCluster1, re-register the Container,
+      // and try to finish application
+      registerSubCluster(subClusterId1);
+      interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId1);
+      releaseContainersAndAssert(containers);
+
+      // Finish the application
+      FinishApplicationMasterRequest finishReq =
+          Records.newRecord(FinishApplicationMasterRequest.class);
+      finishReq.setDiagnostics("");
+      finishReq.setTrackingUrl("");
+      finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+      FinishApplicationMasterResponse finishResponse =
+          interceptor.finishApplicationMaster(finishReq);
+      Assert.assertNotNull(finishResponse);
+      Assert.assertTrue(finishResponse.getIsUnregistered());
+
+      return null;
+    });
+  }
+
+  @Test
+  public void testBatchFinishApplicationMaster() throws IOException, InterruptedException {
+
+    final RegisterApplicationMasterRequest registerReq =
+        Records.newRecord(RegisterApplicationMasterRequest.class);
+    registerReq.setHost(Integer.toString(testAppId));
+    registerReq.setRpcPort(testAppId);
+    registerReq.setTrackingUrl("");
+
+    UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId());
+
+    ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+
+      // Register the application
+      RegisterApplicationMasterRequest registerReq1 =
+          Records.newRecord(RegisterApplicationMasterRequest.class);
+      registerReq1.setHost(Integer.toString(testAppId));
+      registerReq1.setRpcPort(0);
+      registerReq1.setTrackingUrl("");
+
+      // Register ApplicationMaster
+      RegisterApplicationMasterResponse registerResponse =
+          interceptor.registerApplicationMaster(registerReq1);
+      Assert.assertNotNull(registerResponse);
+      lastResponseId = 0;
+
+      Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+      // Allocate the first batch of containers, with sc1 and sc2 active
+      registerSubCluster(SubClusterId.newInstance("SC-1"));
+      registerSubCluster(SubClusterId.newInstance("SC-2"));
+
+      int numberOfContainers = 3;
+      List<Container> containers =
+          getContainersAndAssert(numberOfContainers, numberOfContainers * 2);
+      Assert.assertEquals(2, interceptor.getUnmanagedAMPoolSize());
+      Assert.assertEquals(numberOfContainers * 2, containers.size());
+
+      // Finish the application
+      FinishApplicationMasterRequest finishReq =
+          Records.newRecord(FinishApplicationMasterRequest.class);
+      finishReq.setDiagnostics("");
+      finishReq.setTrackingUrl("");
+      finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+      FinishApplicationMasterResponse finishResp = interceptor.finishApplicationMaster(finishReq);
+      Assert.assertNotNull(finishResp);
+      Assert.assertTrue(finishResp.getIsUnregistered());
+
+      return null;
+    });
+  }
+
+  @Test
+  public void testRemoveAppFromRegistryApplicationSuccess()
+      throws IOException, InterruptedException {
+
+    final RegisterApplicationMasterRequest registerReq =
+        Records.newRecord(RegisterApplicationMasterRequest.class);
+    registerReq.setHost(Integer.toString(testAppId));
+    registerReq.setRpcPort(testAppId);
+    registerReq.setTrackingUrl("");
+
+    UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId());
+
+    ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+
+      // Register the application
+      RegisterApplicationMasterRequest registerReq1 =
+          Records.newRecord(RegisterApplicationMasterRequest.class);
+      registerReq1.setHost(Integer.toString(testAppId));
+      registerReq1.setRpcPort(0);
+      registerReq1.setTrackingUrl("");
+
+      // Register ApplicationMaster
+      RegisterApplicationMasterResponse registerResponse =
+          interceptor.registerApplicationMaster(registerReq1);
+      Assert.assertNotNull(registerResponse);
+      lastResponseId = 0;
+
+      Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+      // Allocate the first batch of containers, with sc1 active
+      registerSubCluster(SubClusterId.newInstance("SC-1"));
+
+      int numberOfContainers = 3;
+      List<Container> containers =
+          getContainersAndAssert(numberOfContainers, numberOfContainers);
+      Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+      Assert.assertEquals(numberOfContainers, containers.size());
+
+      // Finish the application
+      FinishApplicationMasterRequest finishReq =
+          Records.newRecord(FinishApplicationMasterRequest.class);
+      finishReq.setDiagnostics("");
+      finishReq.setTrackingUrl("");
+      finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+      FinishApplicationMasterResponse finishResp = interceptor.finishApplicationMaster(finishReq);
+      Assert.assertNotNull(finishResp);
+      Assert.assertTrue(finishResp.getIsUnregistered());
+
+      FederationRegistryClient client = interceptor.getRegistryClient();
+      List<String> applications = client.getAllApplications();
+      Assert.assertNotNull(finishResp);
+      Assert.assertEquals(0, applications.size());
+      return null;
+    });
+  }
+
+  @Test
+  public void testRemoveAppFromRegistryApplicationFailed()
+      throws IOException, InterruptedException {
+
+    final RegisterApplicationMasterRequest registerReq =
+        Records.newRecord(RegisterApplicationMasterRequest.class);
+
+    registerReq.setHost(Integer.toString(testAppId));
+    registerReq.setRpcPort(testAppId);
+    registerReq.setTrackingUrl("");
+
+    UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId());
+
+    ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+
+      // Register the application
+      RegisterApplicationMasterRequest registerReq1 =
+          Records.newRecord(RegisterApplicationMasterRequest.class);
+      registerReq1.setHost(Integer.toString(testAppId));
+      registerReq1.setRpcPort(0);
+      registerReq1.setTrackingUrl("");
+
+      // Register ApplicationMaster
+      RegisterApplicationMasterResponse registerResponse =
+          interceptor.registerApplicationMaster(registerReq1);
+      Assert.assertNotNull(registerResponse);
+      lastResponseId = 0;
+
+      Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+      // Allocate the first batch of containers, with sc1 active
+      registerSubCluster(SubClusterId.newInstance("SC-1"));
+
+      int numberOfContainers = 3;
+      List<Container> containers =
+          getContainersAndAssert(numberOfContainers, numberOfContainers);
+      Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+      Assert.assertEquals(numberOfContainers, containers.size());
+
+      // Finish the application
+      FinishApplicationMasterRequest finishReq =
+          Records.newRecord(FinishApplicationMasterRequest.class);
+      finishReq.setDiagnostics("");
+      finishReq.setTrackingUrl("");
+      finishReq.setFinalApplicationStatus(FinalApplicationStatus.FAILED);
+
+      // Check Registry Applications
+      // At this time, the Application should not be cleaned up because the state is not SUCCESS.
+      FederationRegistryClient client = interceptor.getRegistryClient();
+      List<String> applications = client.getAllApplications();
+      Assert.assertNotNull(applications);
+      Assert.assertEquals(1, applications.size());
+
+      // interceptor cleanupRegistry
+      ApplicationId applicationId = interceptor.getAttemptId().getApplicationId();
+      client.removeAppFromRegistry(applicationId);
+      applications = client.getAllApplications();
+      Assert.assertNotNull(applications);
+      Assert.assertEquals(0, applications.size());
+
+      return null;
+    });
+  }
+
+  public void testRecoverWithBadSubCluster(final RegistryOperations registryObj)
+      throws IOException, InterruptedException {
+
+    UserGroupInformation ugi =
+        interceptor.getUGIWithToken(interceptor.getAttemptId());
+
+    // Prepare a list of subclusters
+    List<SubClusterId> subClusterIds = new ArrayList<>();
+    SubClusterId sc1 = SubClusterId.newInstance("SC-1");
+    SubClusterId sc2 = SubClusterId.newInstance("SC-2");
+    SubClusterId homeSC = SubClusterId.newInstance(HOME_SC_ID);
+    subClusterIds.add(sc1);
+    subClusterIds.add(sc2);
+    subClusterIds.add(homeSC);
+
+    // Prepare AMRMProxy Context
+    AMRMProxyApplicationContext appContext = new AMRMProxyApplicationContextImpl(nmContext,
+        getConf(), attemptId, "test-user", null, null, null, registryObj);
+
+    // Prepare RegisterApplicationMasterRequest
+    RegisterApplicationMasterRequest registerReq =
+        Records.newRecord(RegisterApplicationMasterRequest.class);
+    registerReq.setHost(Integer.toString(testAppId));
+    registerReq.setRpcPort(testAppId);
+    registerReq.setTrackingUrl("");
+
+    ugi.doAs((PrivilegedExceptionAction<Object>) () -> {
+
+      // Step1. Prepare subClusters SC-1, SC-2, HomeSC and Interceptor
+      initSubClusterAndInterceptor(subClusterIds, registryObj);
+
+      // Step2. Register Application And Assign Containers
+      List<Container> containers = registerApplicationAndAssignContainers(registerReq);
+
+      // Step3. Offline SC-1 cluster
+      offlineSubClusterSC1(sc1);
+
+      // Step4. Recover ApplicationMaster
+      recoverApplicationMaster(appContext);
+
+      // Step5. We recovered ApplicationMaster.
+      // SC-1 was offline, SC-2 was recovered at this time, UnmanagedAMPool.size=1 and only SC-2
+      UnmanagedAMPoolManager unmanagedAMPoolManager = interceptor.getUnmanagedAMPool();
+      Set<String> allUAMIds = unmanagedAMPoolManager.getAllUAMIds();
+      Assert.assertNotNull(allUAMIds);
+      Assert.assertEquals(1, allUAMIds.size());
+      Assert.assertTrue(allUAMIds.contains(sc2.getId()));
+
+      // Step6. The first allocate call expects a fail-over exception and re-register.
+      AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+      allocateRequest.setResponseId(0);
+      LambdaTestUtils.intercept(ApplicationMasterNotRegisteredException.class,
+          "AMRMProxy just restarted and recovered for " + this.attemptId +
+          ". AM should re-register and full re-send pending requests.",
+          () -> interceptor.allocate(allocateRequest));
+      interceptor.registerApplicationMaster(registerReq);
+
+      // Step7. release Containers
+      releaseContainers(containers, sc1);
+
+      // Step8. finish application
+      finishApplication();
+
+      return null;
+    });
+  }
+
+  private void initSubClusterAndInterceptor(List<SubClusterId> subClusterIds,
+      RegistryOperations registryObj) throws YarnException {
+    // Prepare subClusters SC-1, SC-2, HomeSC
+    for (SubClusterId subClusterId : subClusterIds) {
+      registerSubCluster(subClusterId);
+    }
+
+    // Prepare Interceptor
+    interceptor = new TestableFederationInterceptor();
+    AMRMProxyApplicationContext appContext = new AMRMProxyApplicationContextImpl(nmContext,
+        getConf(), attemptId, "test-user", null, null, null, registryObj);
+    interceptor.init(appContext);
+    interceptor.cleanupRegistry();
+  }
+
+  private List<Container> registerApplicationAndAssignContainers(
+      RegisterApplicationMasterRequest registerReq) throws Exception {
+
+    // Register HomeSC
+    RegisterApplicationMasterResponse registerResponse =
+        interceptor.registerApplicationMaster(registerReq);
+    Assert.assertNotNull(registerResponse);
+
+    // We only registered HomeSC, so UnmanagedAMPoolSize should be empty
+    Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+    // We assign 3 Containers to each cluster
+    int numberOfContainers = 3;
+    List<Container> containers =
+        getContainersAndAssert(numberOfContainers, numberOfContainers * 3);
+
+    // At this point, UnmanagedAMPoolSize should be equal to 2 and should contain SC-1, SC-2
+    Assert.assertEquals(2, interceptor.getUnmanagedAMPoolSize());
+    UnmanagedAMPoolManager unmanagedAMPoolManager = interceptor.getUnmanagedAMPool();
+    Set<String> allUAMIds = unmanagedAMPoolManager.getAllUAMIds();
+    Assert.assertNotNull(allUAMIds);
+    Assert.assertEquals(2, allUAMIds.size());
+    Assert.assertTrue(allUAMIds.contains("SC-1"));
+    Assert.assertTrue(allUAMIds.contains("SC-2"));
+
+    // Make sure all async hb threads are done
+    interceptor.drainAllAsyncQueue(true);
+
+    return containers;
+  }
+
+  private void offlineSubClusterSC1(SubClusterId subClusterId) throws YarnException {
+
+    ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
+        interceptor.getSecondaryRMs();
+
+    // SC-1 out of service
+    deRegisterSubCluster(subClusterId);
+    secondaries.get(subClusterId.getId()).setRunningMode(false);
+  }
+
+  private void recoverApplicationMaster(AMRMProxyApplicationContext appContext)
+      throws IOException {
+    // Prepare for Federation Interceptor restart and recover
+    Map<String, byte[]> recoveredDataMap =
+        recoverDataMapForAppAttempt(nmStateStore, attemptId);
+
+    // Preserve the mock RM instances
+    MockResourceManagerFacade homeRM = interceptor.getHomeRM();
+
+    // Create a new interceptor instance and recover
+    interceptor = new TestableFederationInterceptor(homeRM,
+        interceptor.getSecondaryRMs());
+    interceptor.init(appContext);
+    interceptor.recover(recoveredDataMap);
+  }
+
+  private void releaseContainers(List<Container> containers, SubClusterId subClusterId)
+      throws Exception {
+
+    ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
+        interceptor.getSecondaryRMs();
+    lastResponseId = 0;
+
+    // Get the Container list of SC-1
+    MockResourceManagerFacade sc1Facade = secondaries.get("SC-1");
+    HashMap<ApplicationId, List<ContainerId>> appContainerMap =
+        sc1Facade.getApplicationContainerIdMap();
+    Assert.assertNotNull(appContainerMap);
+    ApplicationId applicationId = attemptId.getApplicationId();
+    Assert.assertNotNull(applicationId);
+    List<ContainerId> sc1ContainerList = appContainerMap.get(applicationId);
+
+    // Release all containers,
+    // Because SC-1 is offline, it is necessary to clean up the Containers allocated by SC-1
+    containers = containers.stream()
+        .filter(container -> !sc1ContainerList.contains(container.getId()))
+        .collect(Collectors.toList());
+    releaseContainersAndAssert(containers);
+  }
+
+  private void finishApplication() throws IOException, YarnException {
+    // Finish the application
+    FinishApplicationMasterRequest finishReq =
+        Records.newRecord(FinishApplicationMasterRequest.class);
+    finishReq.setDiagnostics("");
+    finishReq.setTrackingUrl("");
+    finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+    FinishApplicationMasterResponse finishResponse =
+        interceptor.finishApplicationMaster(finishReq);
+    Assert.assertNotNull(finishResponse);
+    Assert.assertTrue(finishResponse.getIsUnregistered());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java
new file mode 100644
index 0000000000000..1069fc8e32b5a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java
@@ -0,0 +1,491 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.security.authorize.Service;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.MockResourceManagerFacade;
+import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestFederationInterceptorSecure extends BaseAMRMProxyTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class);
+
+  public static final String HOME_SC_ID = "SC-home";
+  public static final String SC_ID1 = "SC-1";
+  private static final String HOME_RM_ADDRESS = "127.0.0.1:28032";
+
+  private YarnConfiguration conf;
+  private TestableFederationInterceptor interceptor;
+  private MemoryFederationStateStore stateStore;
+  private NMStateStoreService nmStateStore;
+  private RegistryOperations registry;
+
+  private Context nmContext;
+  private int testAppId;
+  private ApplicationAttemptId attemptId;
+  private volatile int lastResponseId;
+
+  private MockResourceManagerFacade mockHomeRm = null;
+  private Server homeClientRMRpcServer;
+
+  private static File workDir;
+  private static final File TEST_ROOT_DIR =
+      new File(System.getProperty("test.build.dir", "target/test-dir"),
+          TestFederationInterceptorSecure.class.getName() + "-root");
+  private static MiniKdc miniKDC;
+  private static String serverUserName = "yarn";
+  private static String serverPrincipal;
+  private static File serverKeytab;
+
+  @ClassRule
+  public final static TemporaryFolder FOLDER = new TemporaryFolder();
+
+  @BeforeClass
+  public static void setUpCluster() throws Exception {
+    miniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
+    miniKDC.start();
+
+    workDir = FOLDER.getRoot();
+    serverKeytab = new File(workDir, "yarn.keytab");
+    serverPrincipal =
+        serverUserName + "/" + (Path.WINDOWS ? "127.0.0.1" : "localhost") + "@EXAMPLE.COM";
+    miniKDC.createPrincipal(serverKeytab, serverPrincipal);
+  }
+
+  @AfterClass
+  public static void tearDownCluster() {
+    if (miniKDC != null) {
+      miniKDC.stop();
+    }
+    if (FOLDER != null) {
+      FOLDER.delete();
+    }
+  }
+
+  @Override
+  public void setUp() throws IOException {
+    super.setUp();
+
+    stateStore = new MemoryFederationStateStore();
+    stateStore.init(conf);
+    FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf);
+
+    nmStateStore = new NMMemoryStateStoreService();
+    nmStateStore.init(conf);
+    nmStateStore.start();
+
+    registry = new FSRegistryOperationsService();
+    registry.init(conf);
+    registry.start();
+
+    testAppId = 1;
+    attemptId = getApplicationAttemptId(testAppId);
+    nmContext = new NMContext(null, null, null, null, nmStateStore, false, conf);
+    lastResponseId = 0;
+    this.mockHomeRm = new MockResourceManagerFacade(new YarnConfiguration(conf), 0);
+
+    // Login as NodeManager
+    conf.set(YarnConfiguration.NM_KEYTAB, this.serverKeytab.getAbsolutePath());
+    conf.set(YarnConfiguration.NM_PRINCIPAL, this.serverPrincipal);
+    SecurityUtil.login(conf, YarnConfiguration.NM_KEYTAB, YarnConfiguration.NM_PRINCIPAL);
+
+    // start rpc server
+    startRpcServer();
+  }
+
+  private void startRpcServer() {
+    YarnRPC rpc = YarnRPC.create(conf);
+    this.homeClientRMRpcServer = rpc.getServer(ApplicationClientProtocol.class, mockHomeRm,
+        NetUtils.createSocketAddr(HOME_RM_ADDRESS), conf, null, 2);
+    this.homeClientRMRpcServer.start();
+    this.homeClientRMRpcServer.refreshServiceAcl(conf, new MockRMPolicyProvider());
+  }
+
+  private void stopRpcServer() {
+    if (homeClientRMRpcServer != null) {
+      homeClientRMRpcServer.stop();
+    }
+  }
+
+  @Override
+  public void tearDown() {
+    interceptor.cleanupRegistry();
+    interceptor.shutdown();
+    registry.stop();
+    stopRpcServer();
+    super.tearDown();
+  }
+
+  @Override
+  protected YarnConfiguration createConfiguration() {
+    conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    String mockPassThroughInterceptorClass =
+        PassThroughRequestInterceptor.class.getName();
+
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
+    // The others in the chain will simply forward it to the next one in the
+    // chain
+    conf.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE,
+        mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass
+            + "," + TestableFederationInterceptor.class.getName());
+
+    conf.set(YarnConfiguration.FEDERATION_POLICY_MANAGER,
+        UniformBroadcastPolicyManager.class.getName());
+
+    conf.set(YarnConfiguration.RM_CLUSTER_ID, HOME_SC_ID);
+
+    // Disable StateStoreFacade cache
+    conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+
+    // Set sub-cluster timeout to 500ms
+    conf.setLong(YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT, 500);
+
+    // Enable security mode
+    conf.set(HADOOP_SECURITY_AUTHENTICATION, AuthMethod.KERBEROS.toString());
+    conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, true);
+    UserGroupInformation.setConfiguration(conf);
+    conf.set("hadoop.proxyuser.yarn.hosts", "*");
+    conf.set("hadoop.proxyuser.yarn.groups", "*");
+    ProxyUsers.refreshSuperUserGroupsConfiguration(conf, ProxyUsers.CONF_HADOOP_PROXYUSER);
+    conf.set(YarnConfiguration.RM_PRINCIPAL, serverPrincipal);
+
+    return conf;
+  }
+
+  private void registerSubCluster(SubClusterId subClusterId)
+      throws YarnException {
+    if (subClusterId.getId().equals(HOME_SC_ID)) {
+      stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(SubClusterInfo
+          .newInstance(subClusterId, "1.2.3.4:1", HOME_RM_ADDRESS, "1.2.3.4:3", "1.2.3.4:4",
+              SubClusterState.SC_RUNNING, 0, "capacity")));
+    } else {
+      stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(SubClusterInfo
+          .newInstance(subClusterId, "1.2.3.4:1", "1.2.3.4:2", "1.2.3.4:3", "1.2.3.4:4",
+              SubClusterState.SC_RUNNING, 0, "capacity")));
+    }
+  }
+
+  private List<Container> getContainersAndAssert(int numberOfResourceRequests,
+      int numberOfAllocationExcepted) throws Exception {
+    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+    List<Container> containers = new ArrayList<Container>(numberOfResourceRequests);
+    List<ResourceRequest> askList = new ArrayList<ResourceRequest>(numberOfResourceRequests);
+    for (int id = 0; id < numberOfResourceRequests; id++) {
+      askList.add(createResourceRequest("test-node-" + Integer.toString(id), 6000, 2, id % 5, 1));
+    }
+
+    allocateRequest.setAskList(askList);
+
+    allocateRequest.setResponseId(lastResponseId);
+    AllocateResponse allocateResponse = interceptor.allocate(allocateRequest);
+    Assert.assertNotNull("allocate() returned null response", allocateResponse);
+    checkAMRMToken(allocateResponse.getAMRMToken());
+    lastResponseId = allocateResponse.getResponseId();
+
+    containers.addAll(allocateResponse.getAllocatedContainers());
+    LOG.info("Number of allocated containers in the original request: {}",
+        allocateResponse.getAllocatedContainers().size());
+
+    // Send max 10 heart beats to receive all the containers. If not, we will
+    // fail the test
+    int numHeartbeat = 0;
+    while (containers.size() < numberOfAllocationExcepted && numHeartbeat++ < 10) {
+      allocateRequest = Records.newRecord(AllocateRequest.class);
+      allocateRequest.setResponseId(lastResponseId);
+      allocateResponse = interceptor.allocate(allocateRequest);
+      Assert.assertNotNull("allocate() returned null response", allocateResponse);
+      checkAMRMToken(allocateResponse.getAMRMToken());
+      lastResponseId = allocateResponse.getResponseId();
+
+      // Make sure this request is picked up by all async heartbeat handlers
+      interceptor.drainAllAsyncQueue(false);
+
+      containers.addAll(allocateResponse.getAllocatedContainers());
+      LOG.info("Number of allocated containers in this request: "
+          + Integer.toString(allocateResponse.getAllocatedContainers().size()));
+      LOG.info("Total number of allocated containers: {}", containers.size());
+      Thread.sleep(10);
+    }
+    Assert.assertEquals(numberOfAllocationExcepted, containers.size());
+    return containers;
+  }
+
+  private void releaseContainersAndAssert(List<Container> containers)
+      throws Exception {
+    Assert.assertTrue(containers.size() > 0);
+    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+    List<ContainerId> relList = new ArrayList<ContainerId>(containers.size());
+    for (Container container : containers) {
+      relList.add(container.getId());
+    }
+
+    allocateRequest.setReleaseList(relList);
+
+    allocateRequest.setResponseId(lastResponseId);
+    AllocateResponse allocateResponse = interceptor.allocate(allocateRequest);
+    Assert.assertNotNull(allocateResponse);
+    checkAMRMToken(allocateResponse.getAMRMToken());
+    lastResponseId = allocateResponse.getResponseId();
+
+    // The release request will be split and handled by the corresponding UAM.
+    // The release containers returned by the mock resource managers will be
+    // aggregated and returned back to us and we can check if total request size
+    // and returned size are the same
+    List<ContainerId> containersForReleasedContainerIds = new ArrayList<ContainerId>();
+    List<ContainerId> newlyFinished = getCompletedContainerIds(
+        allocateResponse.getCompletedContainersStatuses());
+    containersForReleasedContainerIds.addAll(newlyFinished);
+    LOG.info("Number of containers received in the original request: {}", newlyFinished.size());
+
+    // Send max 10 heart beats to receive all the containers. If not, we will fail the test
+    int numHeartbeat = 0;
+    while (containersForReleasedContainerIds.size() < relList.size() && numHeartbeat++ < 10) {
+      allocateRequest = Records.newRecord(AllocateRequest.class);
+      allocateRequest.setResponseId(lastResponseId);
+      allocateResponse = interceptor.allocate(allocateRequest);
+      Assert.assertNotNull(allocateResponse);
+      checkAMRMToken(allocateResponse.getAMRMToken());
+      lastResponseId = allocateResponse.getResponseId();
+
+      // Make sure this request is picked up by all async heartbeat handlers
+      interceptor.drainAllAsyncQueue(false);
+
+      newlyFinished = getCompletedContainerIds(allocateResponse.getCompletedContainersStatuses());
+      containersForReleasedContainerIds.addAll(newlyFinished);
+      LOG.info("Number of containers received in this request: ", newlyFinished.size());
+      LOG.info("Total number of containers received: ", containersForReleasedContainerIds.size());
+      Thread.sleep(10);
+    }
+
+    Assert.assertEquals(relList.size(), containersForReleasedContainerIds.size());
+  }
+
+  private void checkAMRMToken(Token amrmToken) {
+    if (amrmToken != null) {
+      // The token should be the one issued by home MockRM
+      Assert.assertTrue(amrmToken.getKind().equals(Integer.toString(0)));
+    }
+  }
+
+  @Test
+  public void testRecoverWithAMRMProxyHA() throws Exception {
+    testRecoverWithRPCClientRM(registry);
+  }
+
+  @Test
+  public void testRecoverWithoutAMRMProxyHA() throws Exception {
+    testRecoverWithRPCClientRM(null);
+  }
+
+  protected void testRecoverWithRPCClientRM(
+      final RegistryOperations registryObj)
+      throws Exception {
+    UserGroupInformation ugi = this.getUGIWithToken(attemptId);
+    ugi.doAs(new PrivilegedExceptionAction<Object>() {
+      @Override
+      public Object run() throws Exception {
+        interceptor = new TestableFederationInterceptor(mockHomeRm);
+        interceptor.init(new AMRMProxyApplicationContextImpl(nmContext, conf, attemptId,
+            "test-user", null, null, null, registryObj));
+        interceptor.cleanupRegistry();
+
+        // Register the application
+        RegisterApplicationMasterRequest registerReq =
+            Records.newRecord(RegisterApplicationMasterRequest.class);
+        registerReq.setHost(Integer.toString(testAppId));
+        registerReq.setRpcPort(testAppId);
+        registerReq.setTrackingUrl("");
+
+        RegisterApplicationMasterResponse registerResponse =
+            interceptor.registerApplicationMaster(registerReq);
+        Assert.assertNotNull(registerResponse);
+        lastResponseId = 0;
+
+        Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+        // Allocate one batch of containers
+        registerSubCluster(SubClusterId.newInstance(SC_ID1));
+        registerSubCluster(SubClusterId.newInstance(HOME_SC_ID));
+
+        int numberOfContainers = 3;
+        List<Container> containers =
+            getContainersAndAssert(numberOfContainers, numberOfContainers * 2);
+        Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+
+        // Make sure all async hb threads are done
+        interceptor.drainAllAsyncQueue(true);
+
+        // Prepare for Federation Interceptor restart and recover
+        Map<String, byte[]> recoveredDataMap = recoverDataMapForAppAttempt(nmStateStore, attemptId);
+        String scEntry = FederationInterceptor.NMSS_SECONDARY_SC_PREFIX + "SC-1";
+        if (registryObj == null) {
+          Assert.assertTrue(recoveredDataMap.containsKey(scEntry));
+        } else {
+          // When AMRMPRoxy HA is enabled, NMSS should not have the UAM token,
+          // it should be in Registry
+          Assert.assertFalse(recoveredDataMap.containsKey(scEntry));
+        }
+
+        // Preserve the mock RM instances
+        MockResourceManagerFacade homeRM = interceptor.getHomeRM();
+        ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
+            interceptor.getSecondaryRMs();
+
+        // Create a new interceptor instance and recover
+        interceptor = new TestableFederationInterceptor(homeRM, secondaries);
+        interceptor.init(new AMRMProxyApplicationContextImpl(nmContext, conf, attemptId,
+            "test-user", null, null, null, registryObj));
+        interceptor.setClientRPC(true);
+        interceptor.recover(recoveredDataMap);
+        interceptor.setClientRPC(false);
+
+        Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+        // SC-1 should be initialized to be timed out
+        Assert.assertEquals(1, interceptor.getTimedOutSCs(true).size());
+
+        // The first allocate call expects a fail-over exception and re-register
+        try {
+          AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
+          allocateRequest.setResponseId(lastResponseId);
+          AllocateResponse allocateResponse = interceptor.allocate(allocateRequest);
+          lastResponseId = allocateResponse.getResponseId();
+          Assert.fail("Expecting an ApplicationMasterNotRegisteredException  "
+              + " after FederationInterceptor restarts and recovers");
+        } catch (ApplicationMasterNotRegisteredException e) {
+        }
+        interceptor.registerApplicationMaster(registerReq);
+        lastResponseId = 0;
+
+        // Release all containers
+        releaseContainersAndAssert(containers);
+
+        // Finish the application
+        FinishApplicationMasterRequest finishReq =
+            Records.newRecord(FinishApplicationMasterRequest.class);
+        finishReq.setDiagnostics("");
+        finishReq.setTrackingUrl("");
+        finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+        FinishApplicationMasterResponse finishResponse =
+            interceptor.finishApplicationMaster(finishReq);
+        Assert.assertNotNull(finishResponse);
+        Assert.assertEquals(true, finishResponse.getIsUnregistered());
+
+        // After the application succeeds, the registry/NMSS entry should be
+        // cleaned up
+        if (registryObj != null) {
+          Assert.assertEquals(0, interceptor.getRegistryClient().getAllApplications().size());
+        } else {
+          recoveredDataMap = recoverDataMapForAppAttempt(nmStateStore, attemptId);
+          Assert.assertFalse(recoveredDataMap.containsKey(scEntry));
+        }
+        return null;
+      }
+    });
+  }
+
+  private UserGroupInformation getUGIWithToken(
+      ApplicationAttemptId appAttemptId) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+    AMRMTokenIdentifier token = new AMRMTokenIdentifier(appAttemptId, 1);
+    ugi.addTokenIdentifier(token);
+    return ugi;
+  }
+
+  private class MockRMPolicyProvider extends PolicyProvider {
+
+    private MockRMPolicyProvider() {
+    }
+
+    private final Service[] resourceManagerServices =
+        new Service[]{
+            new Service(
+                YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL,
+                ApplicationClientProtocolPB.class),
+        };
+
+    @Override
+    public Service[] getServices() {
+      return resourceManagerServices;
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
index 7c57ab1292068..5ec47f1b52e11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
@@ -53,9 +53,15 @@ public class TestableFederationInterceptor extends FederationInterceptor {
   private AtomicInteger runningIndex = new AtomicInteger(0);
   private MockResourceManagerFacade mockRm;
 
+  private boolean isClientRPC = false;
+
   public TestableFederationInterceptor() {
   }
 
+  public TestableFederationInterceptor(MockResourceManagerFacade homeRM) {
+    mockRm = homeRM;
+  }
+
   public TestableFederationInterceptor(MockResourceManagerFacade homeRM,
       ConcurrentHashMap<String, MockResourceManagerFacade> secondaries) {
     mockRm = homeRM;
@@ -79,6 +85,9 @@ protected AMHeartbeatRequestHandler createHomeHeartbeartHandler(
   @Override
   protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext,
       Class<T> protocol, UserGroupInformation user) {
+    if (isClientRPC) {
+      return super.createHomeRMProxy(appContext, protocol, user);
+    }
     synchronized (this) {
       if (mockRm == null) {
         mockRm = new MockResourceManagerFacade(
@@ -249,7 +258,7 @@ protected <T> T createRMProxy(Class<T> protocol, Configuration config,
   }
 
   /**
-   * Wrap the handler thread so it calls from the same user.
+   * Wrap the handler thread, so it calls from the same user.
    */
   protected class TestableAMRequestHandlerThread
       extends AMHeartbeatRequestHandler {
@@ -273,4 +282,8 @@ public Object run() {
       }
     }
   }
+
+  public void setClientRPC(boolean clientRPC) {
+    this.isClientRPC = clientRPC;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
index 05ead9df4ab1b..0fe8f0179bed9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
@@ -48,7 +48,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
 import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+
 import org.junit.Test;
 
 public class TestPBRecordImpl {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerSchedulerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerSchedulerTest.java
new file mode 100644
index 0000000000000..5a495d741377a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerSchedulerTest.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager;
+
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ConfigurationException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.TestContainerSchedulerQueuing;
+import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.mockito.Mockito.spy;
+
+/**
+ * Base test class that overrides the behavior of
+ * {@link ContainerStateTransitionListener} for testing
+ * the {@link ContainerScheduler}.
+ */
+public class BaseContainerSchedulerTest extends BaseContainerManagerTest {
+  private static final long TWO_GB = 2048 * 1024 * 1024L;
+
+  public BaseContainerSchedulerTest() throws UnsupportedFileSystemException {
+    super();
+  }
+
+  static {
+    LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
+  }
+
+  public static class Listener implements ContainerStateTransitionListener {
+
+    private final Map<ContainerId, List<ContainerState>> states =
+        new HashMap<>();
+    private final Map<ContainerId, List<ContainerEventType>> events =
+        new HashMap<>();
+
+    public Map<ContainerId, List<ContainerEventType>> getEvents() {
+      return events;
+    }
+
+    public Map<ContainerId, List<ContainerState>> getStates() {
+      return states;
+    }
+
+    @Override
+    public void init(Context context) {}
+
+    @Override
+    public void preTransition(ContainerImpl op,
+        ContainerState beforeState,
+        ContainerEvent eventToBeProcessed) {
+      if (!states.containsKey(op.getContainerId())) {
+        states.put(op.getContainerId(), new ArrayList<>());
+        states.get(op.getContainerId()).add(beforeState);
+        events.put(op.getContainerId(), new ArrayList<>());
+      }
+    }
+
+    @Override
+    public void postTransition(ContainerImpl op, ContainerState beforeState,
+        ContainerState afterState, ContainerEvent processedEvent) {
+      states.get(op.getContainerId()).add(afterState);
+      events.get(op.getContainerId()).add(processedEvent.getType());
+    }
+  }
+
+  private boolean delayContainers = true;
+
+  protected void setDelayContainers(final boolean delayContainersParam) {
+    this.delayContainers = delayContainersParam;
+  }
+
+  @Override
+  protected ContainerManagerImpl createContainerManager(
+      DeletionService delSrvc) {
+    return new ContainerManagerImpl(context, exec, delSrvc,
+        getNodeStatusUpdater(), metrics, dirsHandler) {
+
+      @Override
+      protected UserGroupInformation getRemoteUgi() throws YarnException {
+        ApplicationId appId = ApplicationId.newInstance(0, 0);
+        ApplicationAttemptId appAttemptId =
+            ApplicationAttemptId.newInstance(appId, 1);
+        UserGroupInformation ugi =
+            UserGroupInformation.createRemoteUser(appAttemptId.toString());
+        ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+            .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+            .getKeyId()));
+        return ugi;
+      }
+
+      @Override
+      protected ContainersMonitor createContainersMonitor(
+          ContainerExecutor exec) {
+        return new ContainersMonitorImpl(exec, dispatcher, this.context) {
+          // Define resources available for containers to be executed.
+          @Override
+          public long getPmemAllocatedForContainers() {
+            return TWO_GB;
+          }
+
+          @Override
+          public long getVmemAllocatedForContainers() {
+            float pmemRatio = getConfig().getFloat(
+                YarnConfiguration.NM_VMEM_PMEM_RATIO,
+                YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+            return (long) (pmemRatio * getPmemAllocatedForContainers());
+          }
+
+          @Override
+          public long getVCoresAllocatedForContainers() {
+            return 4;
+          }
+        };
+      }
+    };
+  }
+
+  @Override
+  protected ContainerExecutor createContainerExecutor() {
+    DefaultContainerExecutor exec = new DefaultContainerExecutor() {
+      ConcurrentMap<String, Boolean> oversleepMap =
+          new ConcurrentHashMap<String, Boolean>();
+
+      /**
+       * Launches the container.
+       * If delayContainers is turned on, then we sleep a while before
+       * starting the container.
+       */
+      @Override
+      public int launchContainer(ContainerStartContext ctx)
+          throws IOException, ConfigurationException {
+        final String containerId =
+            ctx.getContainer().getContainerId().toString();
+        oversleepMap.put(containerId, false);
+        if (delayContainers) {
+          try {
+            Thread.sleep(10000);
+            if (oversleepMap.get(containerId)) {
+              Thread.sleep(10000);
+            }
+          } catch (InterruptedException e) {
+            // Nothing..
+          }
+        }
+        return super.launchContainer(ctx);
+      }
+
+      @Override
+      public void pauseContainer(Container container) {
+        // To mimic pausing we force the container to be in the PAUSED state
+        // a little longer by oversleeping.
+        oversleepMap.put(container.getContainerId().toString(), true);
+        LOG.info("Container was paused");
+      }
+
+      @Override
+      public void resumeContainer(Container container) {
+        LOG.info("Container was resumed");
+      }
+    };
+    exec.setConf(conf);
+    return spy(exec);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index de29efd59fefb..b960f5dea1f7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -42,6 +42,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
@@ -682,7 +683,49 @@ public void testContainerCleanupOnShutdown() throws Exception {
     verify(cm, never()).handle(isA(CMgrCompletedAppsEvent.class));
   }
 
-  private void commonLaunchContainer(ApplicationId appId, ContainerId cid,
+  @Test
+  public void testKilledContainerInQueuedStateRecovery() throws Exception {
+    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+    NMStateStoreService stateStore = new NMMemoryStateStoreService();
+    stateStore.init(conf);
+    stateStore.start();
+    context = createContext(conf, stateStore);
+    ContainerManagerImpl cm = createContainerManager(context, delSrvc);
+    ((NMContext) context).setContainerManager(cm);
+    cm.init(conf);
+    cm.start();
+
+    // add an application by starting a container
+    ApplicationId appId = ApplicationId.newInstance(0, 0);
+    ApplicationAttemptId attemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+    createStartContainerRequest(appId, cid, cm);
+
+    Application app = context.getApplications().get(appId);
+    assertEquals(1, context.getApplications().size());
+    assertNotNull(app);
+
+    stateStore.storeContainerKilled(cid);
+    // restart and verify container scheduler has recovered correctly
+    cm.stop();
+    context = createContext(conf, stateStore);
+    cm = createContainerManager(context, delSrvc);
+    ((NMContext) context).setContainerManager(cm);
+    cm.init(conf);
+    cm.start();
+    assertEquals(1, context.getApplications().size());
+
+    ConcurrentMap<ContainerId, Container> containers = context.getContainers();
+    Container c = containers.get(cid);
+    assertEquals(ContainerState.DONE, c.getContainerState());
+    app = context.getApplications().get(appId);
+    assertNotNull(app);
+    cm.stop();
+  }
+
+  private void createStartContainerRequest(ApplicationId appId, ContainerId cid,
       ContainerManagerImpl cm) throws Exception {
     Map<String, String> containerEnv = new HashMap<>();
     setFlowContext(containerEnv, "app_name1", appId);
@@ -727,6 +770,11 @@ private void commonLaunchContainer(ApplicationId appId, ContainerId cid,
         context, cm, cid, clc, null, ContainerType.TASK);
     assertTrue(startResponse.getFailedRequests().isEmpty());
     assertEquals(1, context.getApplications().size());
+  }
+
+  private void commonLaunchContainer(ApplicationId appId, ContainerId cid,
+      ContainerManagerImpl cm) throws Exception {
+    createStartContainerRequest(appId, cid, cm);
     // make sure the container reaches RUNNING state
     waitForNMContainerState(cm, cid,
         org.apache.hadoop.yarn.server.nodemanager
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
index f48d785569512..d1b1650731727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainersLauncher.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.test.Whitebox;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -40,8 +39,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -123,10 +120,8 @@ public void setup() throws IllegalArgumentException, IllegalAccessException {
   @SuppressWarnings("unchecked")
   @Test
   public void testLaunchContainerEvent()
-      throws IllegalArgumentException, IllegalAccessException {
-    Map<ContainerId, ContainerLaunch> dummyMap =
-        (Map<ContainerId, ContainerLaunch>) Whitebox.getInternalState(spy,
-            "running");
+      throws IllegalArgumentException {
+    Map<ContainerId, ContainerLaunch> dummyMap = spy.running;
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.LAUNCH_CONTAINER);
     assertEquals(0, dummyMap.size());
@@ -139,10 +134,8 @@ public void testLaunchContainerEvent()
   @SuppressWarnings("unchecked")
   @Test
   public void testRelaunchContainerEvent()
-      throws IllegalArgumentException, IllegalAccessException {
-    Map<ContainerId, ContainerLaunch> dummyMap =
-        (Map<ContainerId, ContainerLaunch>) Whitebox.getInternalState(spy,
-            "running");
+      throws IllegalArgumentException {
+    Map<ContainerId, ContainerLaunch> dummyMap = spy.running;
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.RELAUNCH_CONTAINER);
     assertEquals(0, dummyMap.size());
@@ -159,10 +152,8 @@ public void testRelaunchContainerEvent()
   @SuppressWarnings("unchecked")
   @Test
   public void testRecoverContainerEvent()
-      throws IllegalArgumentException, IllegalAccessException {
-    Map<ContainerId, ContainerLaunch> dummyMap =
-        (Map<ContainerId, ContainerLaunch>) Whitebox.getInternalState(spy,
-            "running");
+      throws IllegalArgumentException {
+    Map<ContainerId, ContainerLaunch> dummyMap = spy.running;
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.RECOVER_CONTAINER);
     assertEquals(0, dummyMap.size());
@@ -178,7 +169,7 @@ public void testRecoverContainerEvent()
 
   @Test
   public void testRecoverPausedContainerEvent()
-      throws IllegalArgumentException, IllegalAccessException {
+      throws IllegalArgumentException {
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.RECOVER_PAUSED_CONTAINER);
     spy.handle(event);
@@ -189,16 +180,14 @@ public void testRecoverPausedContainerEvent()
   @Test
   public void testCleanupContainerEvent()
       throws IllegalArgumentException, IllegalAccessException, IOException {
-    Map<ContainerId, ContainerLaunch> dummyMap = Collections
-        .synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
-    dummyMap.put(containerId, containerLaunch);
-    Whitebox.setInternalState(spy, "running", dummyMap);
+    spy.running.clear();
+    spy.running.put(containerId, containerLaunch);
 
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER);
-    assertEquals(1, dummyMap.size());
+    assertEquals(1, spy.running.size());
     spy.handle(event);
-    assertEquals(0, dummyMap.size());
+    assertEquals(0, spy.running.size());
     Mockito.verify(containerLauncher, Mockito.times(1))
         .submit(Mockito.any(ContainerCleanup.class));
   }
@@ -206,10 +195,8 @@ public void testCleanupContainerEvent()
   @Test
   public void testCleanupContainerForReINITEvent()
       throws IllegalArgumentException, IllegalAccessException, IOException {
-    Map<ContainerId, ContainerLaunch> dummyMap = Collections
-        .synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
-    dummyMap.put(containerId, containerLaunch);
-    Whitebox.setInternalState(spy, "running", dummyMap);
+    spy.running.clear();
+    spy.running.put(containerId, containerLaunch);
 
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.CLEANUP_CONTAINER_FOR_REINIT);
@@ -226,9 +213,6 @@ public void testCleanupContainerForReINITEvent()
   @Test
   public void testSignalContainerEvent()
       throws IllegalArgumentException, IllegalAccessException, IOException {
-    Map<ContainerId, ContainerLaunch> dummyMap = Collections
-        .synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
-    dummyMap.put(containerId, containerLaunch);
 
     SignalContainersLauncherEvent dummyEvent =
         mock(SignalContainersLauncherEvent.class);
@@ -238,7 +222,8 @@ public void testSignalContainerEvent()
     when(containerId.getApplicationAttemptId().getApplicationId())
         .thenReturn(appId);
 
-    Whitebox.setInternalState(spy, "running", dummyMap);
+    spy.running.clear();
+    spy.running.put(containerId, containerLaunch);
     when(dummyEvent.getType())
         .thenReturn(ContainersLauncherEventType.SIGNAL_CONTAINER);
     when(dummyEvent.getCommand())
@@ -246,7 +231,7 @@ public void testSignalContainerEvent()
     doNothing().when(containerLaunch)
         .signalContainer(SignalContainerCommand.GRACEFUL_SHUTDOWN);
     spy.handle(dummyEvent);
-    assertEquals(1, dummyMap.size());
+    assertEquals(1, spy.running.size());
     Mockito.verify(containerLaunch, Mockito.times(1))
         .signalContainer(SignalContainerCommand.GRACEFUL_SHUTDOWN);
   }
@@ -254,30 +239,26 @@ public void testSignalContainerEvent()
   @Test
   public void testPauseContainerEvent()
       throws IllegalArgumentException, IllegalAccessException, IOException {
-    Map<ContainerId, ContainerLaunch> dummyMap = Collections
-        .synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
-    dummyMap.put(containerId, containerLaunch);
-    Whitebox.setInternalState(spy, "running", dummyMap);
+    spy.running.clear();
+    spy.running.put(containerId, containerLaunch);
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.PAUSE_CONTAINER);
     doNothing().when(containerLaunch).pauseContainer();
     spy.handle(event);
-    assertEquals(1, dummyMap.size());
+    assertEquals(1, spy.running.size());
     Mockito.verify(containerLaunch, Mockito.times(1)).pauseContainer();
   }
 
   @Test
   public void testResumeContainerEvent()
       throws IllegalArgumentException, IllegalAccessException, IOException {
-    Map<ContainerId, ContainerLaunch> dummyMap = Collections
-        .synchronizedMap(new HashMap<ContainerId, ContainerLaunch>());
-    dummyMap.put(containerId, containerLaunch);
-    Whitebox.setInternalState(spy, "running", dummyMap);
+    spy.running.clear();
+    spy.running.put(containerId, containerLaunch);
     when(event.getType())
         .thenReturn(ContainersLauncherEventType.RESUME_CONTAINER);
     doNothing().when(containerLaunch).resumeContainer();
     spy.handle(event);
-    assertEquals(1, dummyMap.size());
+    assertEquals(1, spy.running.size());
     Mockito.verify(containerLaunch, Mockito.times(1)).resumeContainer();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java
index c8072c327a9b6..42c495c88f8f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java
@@ -95,7 +95,7 @@ public void testReadNumaTopologyFromCmdOutput() throws Exception {
         + "1: 20 10";
     numaResourceAllocator = new NumaResourceAllocator(mock(Context.class)) {
       @Override
-      String executeNGetCmdOutput(Configuration config)
+      public String executeNGetCmdOutput(Configuration config)
           throws YarnRuntimeException {
         return cmdOutput;
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index bb1abf51df03f..ea7c213809330 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -57,7 +57,6 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
-import org.codehaus.jackson.map.ObjectMapper;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -2034,19 +2033,27 @@ public static Configuration enableMockContainerExecutor(Configuration conf) {
 
   @Test
   public void testDockerImageNamePattern() throws Exception {
-    String[] validNames =
-        { "ubuntu", "fedora/httpd:version1.0",
-            "fedora/httpd:version1.0.test",
-            "fedora/httpd:version1.0.TEST",
-            "myregistryhost:5000/ubuntu",
-            "myregistryhost:5000/fedora/httpd:version1.0",
-            "myregistryhost:5000/fedora/httpd:version1.0.test",
-            "myregistryhost:5000/fedora/httpd:version1.0.TEST"};
-
-    String[] invalidNames = { "Ubuntu", "ubuntu || fedora", "ubuntu#",
-        "myregistryhost:50AB0/ubuntu", "myregistry#host:50AB0/ubuntu",
-        ":8080/ubuntu"
-    };
+    String[] validNames = {"ubuntu", "fedora/httpd:version1.0", "fedora/httpd:version1.0.test",
+        "fedora/httpd:version1.0.TEST", "myregistryhost:5000/ubuntu",
+        "myregistryhost:5000/fedora/httpd:version1.0",
+        "myregistryhost:5000/fedora/httpd:version1.0.test",
+        "myregistryhost:5000/fedora/httpd:version1.0.TEST",
+        "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example"
+            + "@sha256:f1d4ae3f7261a72e98c6ebefe9985cf10a0ea5bd762585a43e0700ed99863807"};
+
+    String[] invalidNames = {"Ubuntu", "ubuntu || fedora", "ubuntu#", "myregistryhost:50AB0/ubuntu",
+        "myregistry#host:50AB0/ubuntu", ":8080/ubuntu",
+
+        // Invalid: contains "@sha256" but doesn't really contain a digest.
+        "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example@sha256",
+
+        // Invalid: digest is too short.
+        "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example"
+            + "@sha256:f1d4",
+
+        // Invalid: digest is too long
+        "123456789123.dkr.ecr.us-east-1.amazonaws.com/emr-docker-examples:pyspark-example"
+            + "@sha256:f1d4ae3f7261a72e98c6ebefe9985cf10a0ea5bd762585a43e0700ed99863807f"};
 
     for (String name : validNames) {
       DockerLinuxContainerRuntime.validateImageName(name);
@@ -2384,24 +2391,37 @@ public void testDockerCapabilities() throws ContainerExecutionException {
 
   @Test
   public void testLaunchContainerWithDockerTokens()
-      throws ContainerExecutionException, PrivilegedOperationException,
-      IOException {
-    // Write the JSOn to a temp file.
-    File file = File.createTempFile("docker-client-config", "runtime-test");
-    file.deleteOnExit();
-    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
-    bw.write(TestDockerClientConfigHandler.JSON);
-    bw.close();
+      throws ContainerExecutionException, PrivilegedOperationException, IOException {
 
     // Get the credentials object with the Tokens.
-    Credentials credentials = DockerClientConfigHandler
-        .readCredentialsFromConfigFile(new Path(file.toURI()), conf, appId);
+    Credentials credentials = DockerClientConfigHandler.readCredentialsFromConfigFile(
+        new Path(getDockerClientConfigFile().toURI()), conf, appId);
     DataOutputBuffer dob = new DataOutputBuffer();
     credentials.writeTokenStorageToStream(dob);
     ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
 
-    // Configure the runtime and launch the container
-    when(context.getTokens()).thenReturn(tokens);
+    testLaunchContainer(tokens, null);
+  }
+
+  @Test
+  public void testLaunchContainerWithAdditionalDockerClientConfig()
+      throws ContainerExecutionException, PrivilegedOperationException, IOException {
+    testLaunchContainer(null, getDockerClientConfigFile());
+  }
+
+  public void testLaunchContainer(ByteBuffer tokens, File dockerConfigFile)
+      throws ContainerExecutionException, PrivilegedOperationException,
+      IOException {
+    if (dockerConfigFile != null) {
+      // load the docker client config file from system environment
+      env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_CLIENT_CONFIG,
+          dockerConfigFile.getPath());
+    }
+
+    if (tokens != null) {
+      // Configure the runtime and launch the container
+      when(context.getTokens()).thenReturn(tokens);
+    }
     DockerLinuxContainerRuntime runtime =
         new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
     runtime.initialize(conf, nmContext);
@@ -2487,6 +2507,16 @@ public void testLaunchContainerWithDockerTokens()
         dockerCommands.get(counter++));
   }
 
+  private File getDockerClientConfigFile() throws IOException {
+    // Write the JSOn to a temp file.
+    File file = File.createTempFile("docker-client-config", "runtime-test");
+    file.deleteOnExit();
+    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
+    bw.write(TestDockerClientConfigHandler.JSON);
+    bw.close();
+    return file;
+  }
+
   @Test
   public void testDockerContainerRelaunch()
       throws ContainerExecutionException, PrivilegedOperationException,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java
index 73bfa026f145c..3c2a951597ec9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestImageTagToManifestPlugin.java
@@ -24,7 +24,8 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageManifest;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin;
-import org.codehaus.jackson.map.ObjectMapper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java
index 1e06d03dda4b2..8a541bbe1ae32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestRuncContainerRuntime.java
@@ -58,8 +58,9 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
 import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -501,7 +502,7 @@ private RuncContainerExecutorConfig verifyRuncConfig(File configFile)
     JsonNode configNode = mapper.readTree(configFile);
 
     RuncContainerExecutorConfig runcContainerExecutorConfig =
-        mapper.readValue(configNode, RuncContainerExecutorConfig.class);
+        mapper.readValue(configNode.traverse(), RuncContainerExecutorConfig.class);
     configSize = configNode.size();
 
     OCIRuntimeConfig ociRuntimeConfig =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
index 13310ad51009f..daa1033697f5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
@@ -27,7 +27,6 @@
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import static org.apache.hadoop.yarn.api.records.LocalResourceType.*;
 import static org.apache.hadoop.yarn.api.records.LocalResourceVisibility.*;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 4cc9ac1f3a7a4..4ec8f462f5117 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -56,6 +56,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
@@ -206,7 +207,9 @@ private void verifyLocalFileDeletion(
     logAggregationService.init(this.conf);
     logAggregationService.start();
 
-    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
+    Random random = new Random(System.currentTimeMillis());
+    long clusterTimeStamp = random.nextLong();
+    ApplicationId application1 = BuilderUtils.newApplicationId(clusterTimeStamp, 1);
 
     // AppLogDir should be created
     File app1LogDir =
@@ -234,31 +237,41 @@ private void verifyLocalFileDeletion(
     // ensure filesystems were closed
     verify(logAggregationService).closeFileSystems(
         any(UserGroupInformation.class));
-    List<Path> dirList = new ArrayList<>();
-    dirList.add(new Path(app1LogDir.toURI()));
-    verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher(
-        delSrvc, user, null, dirList)));
-    
-    String containerIdStr = container11.toString();
-    File containerLogDir = new File(app1LogDir, containerIdStr);
-    int count = 0;
-    int maxAttempts = 50;
-    for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
-      File f = new File(containerLogDir, fileType);
-      count = 0;
-      while ((f.exists()) && (count < maxAttempts)) {
-        count++;
-        Thread.sleep(100);
+    boolean filesShouldBeDeleted =
+        this.conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP,
+            YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP);
+    if (filesShouldBeDeleted) {
+      List<Path> dirList = new ArrayList<>();
+      dirList.add(new Path(app1LogDir.toURI()));
+      verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher(
+          delSrvc, user, null, dirList)));
+
+      String containerIdStr = container11.toString();
+      File containerLogDir = new File(app1LogDir, containerIdStr);
+      for (String fileType : new String[]{"stdout", "stderr", "syslog"}) {
+        File f = new File(containerLogDir, fileType);
+        GenericTestUtils.waitFor(() -> !f.exists(), 1000, 1000 * 50);
+        Assert.assertFalse("File [" + f + "] was not deleted", f.exists());
       }
-      Assert.assertFalse("File [" + f + "] was not deleted", f.exists());
-    }
-    count = 0;
-    while ((app1LogDir.exists()) && (count < maxAttempts)) {
-      count++;
-      Thread.sleep(100);
+      Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted",
+          app1LogDir.exists());
+    } else {
+      List<Path> dirList = new ArrayList<>();
+      dirList.add(new Path(app1LogDir.toURI()));
+      verify(delSrvc, never()).delete(argThat(new FileDeletionMatcher(
+          delSrvc, user, null, dirList)));
+
+      String containerIdStr = container11.toString();
+      File containerLogDir = new File(app1LogDir, containerIdStr);
+      Thread.sleep(5000);
+      for (String fileType : new String[]{"stdout", "stderr", "syslog"}) {
+        File f = new File(containerLogDir, fileType);
+        Assert.assertTrue("File [" + f + "] was not deleted", f.exists());
+      }
+      Assert.assertTrue("Directory [" + app1LogDir + "] was not deleted",
+          app1LogDir.exists());
     }
-    Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted",
-      app1LogDir.exists());
+    delSrvc.stop();
 
     Path logFilePath = logAggregationService
         .getLogAggregationFileController(conf)
@@ -297,6 +310,20 @@ public void testLocalFileDeletionAfterUpload() throws Exception {
     verifyLocalFileDeletion(logAggregationService);
   }
 
+  @Test
+  public void testLocalFileRemainsAfterUploadOnCleanupDisable() throws Exception {
+    this.delSrvc = new DeletionService(createContainerExecutor());
+    delSrvc = spy(delSrvc);
+    this.delSrvc.init(conf);
+    this.conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, false);
+    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+        this.remoteRootLogDir.getAbsolutePath());
+    LogAggregationService logAggregationService = spy(
+        new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler));
+    verifyLocalFileDeletion(logAggregationService);
+  }
+
   @Test
   public void testLocalFileDeletionOnDiskFull() throws Exception {
     this.delSrvc = new DeletionService(createContainerExecutor());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerOppContainersByResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerOppContainersByResources.java
new file mode 100644
index 0000000000000..30fbbde1299e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerOppContainersByResources.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerSchedulerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Tests the behavior of {@link ContainerScheduler} when its queueing policy
+ * is set to {@link OpportunisticContainersQueuePolicy#BY_RESOURCES}
+ * such that the NM only queues containers if there's enough resources
+ * on the node to start all queued containers.
+ */
+public class TestContainerSchedulerOppContainersByResources
+    extends BaseContainerSchedulerTest {
+  public TestContainerSchedulerOppContainersByResources()
+      throws UnsupportedFileSystemException {
+  }
+
+  @Override
+  public void setup() throws IOException {
+    conf.set(YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_QUEUE_POLICY,
+        OpportunisticContainersQueuePolicy.BY_RESOURCES.name());
+    super.setup();
+    containerManager.start();
+  }
+
+  /**
+   * Checks if a container is in a running or successfully run state.
+   * @param containerStatus the container status
+   * @return true if the container is running or completed
+   * with a successful state, false if the container has not started or failed
+   */
+  private static boolean isContainerInSuccessfulState(
+      final ContainerStatus containerStatus) {
+    final org.apache.hadoop.yarn.api.records.ContainerState state =
+        containerStatus.getState();
+    final ContainerSubState subState = containerStatus.getContainerSubState();
+    switch (subState) {
+    case RUNNING:
+    case COMPLETING:
+      return true;
+    case DONE:
+      // If the state is not COMPLETE, then the
+      // container is a failed container
+      return state ==
+          org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
+    default:
+      return false;
+    }
+  }
+
+  private void verifyRunAndKilledContainers(
+      final List<ContainerId> statList,
+      final int numExpectedContainers, final Set<ContainerId> runContainers,
+      final Set<ContainerId> killedContainers)
+      throws TimeoutException, InterruptedException {
+    GenericTestUtils.waitFor(
+        () -> {
+          GetContainerStatusesRequest statRequest =
+              GetContainerStatusesRequest.newInstance(statList);
+          final List<ContainerStatus> containerStatuses;
+          try {
+            containerStatuses = containerManager
+                .getContainerStatuses(statRequest).getContainerStatuses();
+          } catch (final Exception e) {
+            return false;
+          }
+
+          if (numExpectedContainers != containerStatuses.size()) {
+            return false;
+          }
+
+          for (final ContainerStatus status : containerStatuses) {
+            if (runContainers.contains(status.getContainerId())) {
+              if (!isContainerInSuccessfulState(status)) {
+                return false;
+              }
+            } else if (killedContainers.contains(status.getContainerId())) {
+              if (!status.getDiagnostics()
+                  .contains("Opportunistic container queue is full")) {
+                return false;
+              }
+            } else {
+              return false;
+            }
+          }
+
+          return true;
+        }, 1000, 10000);
+  }
+
+  /**
+   * Verifies that nothing is queued at the container scheduler.
+   */
+  private void verifyNothingQueued() {
+    // Check that nothing is queued
+    ContainerScheduler containerScheduler =
+        containerManager.getContainerScheduler();
+    Assert.assertEquals(0,
+        containerScheduler.getNumQueuedContainers());
+    Assert.assertEquals(0,
+        containerScheduler.getNumQueuedGuaranteedContainers());
+    Assert.assertEquals(0,
+        containerScheduler.getNumQueuedOpportunisticContainers());
+    Assert.assertEquals(0,
+        metrics.getQueuedOpportunisticContainers());
+    Assert.assertEquals(0, metrics.getQueuedGuaranteedContainers());
+  }
+
+  /**
+   * Tests that newly arrived containers after the resources are filled up
+   * get killed and never gets run.
+   */
+  @Test
+  public void testKillOpportunisticWhenNoResourcesAvailable() throws Exception {
+    List<StartContainerRequest> startContainerRequests = new ArrayList<>();
+
+    // GContainer that takes up the whole node
+    startContainerRequests.add(StartContainerRequest.newInstance(
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
+        createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
+            context.getNodeId(),
+            user, BuilderUtils.newResource(2048, 1),
+            context.getContainerTokenSecretManager(), null,
+            ExecutionType.GUARANTEED)));
+
+    // OContainer that should be killed
+    startContainerRequests.add(StartContainerRequest.newInstance(
+        recordFactory.newRecordInstance(ContainerLaunchContext.class),
+        createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
+            context.getNodeId(),
+            user, BuilderUtils.newResource(2048, 1),
+            context.getContainerTokenSecretManager(), null,
+            ExecutionType.OPPORTUNISTIC)));
+
+    StartContainersRequest allRequests =
+        StartContainersRequest.newInstance(startContainerRequests);
+    containerManager.startContainers(allRequests);
+
+    BaseContainerManagerTest.waitForNMContainerState(containerManager,
+        createContainerId(0), ContainerState.RUNNING, 40);
+
+    // Wait for the OContainer to get killed
+    BaseContainerManagerTest.waitForNMContainerState(containerManager,
+        createContainerId(1), ContainerState.DONE, 40);
+
+    // Get container statuses.
+    // Container 0 should be running and container 1 should be killed
+    List<ContainerId> statList = ImmutableList.of(createContainerId(0),
+        createContainerId(1));
+
+    verifyRunAndKilledContainers(
+        statList, 2,
+        Collections.singleton(createContainerId(0)),
+        Collections.singleton(createContainerId(1))
+    );
+
+    verifyNothingQueued();
+  }
+
+  /**
+   * Tests that newly arrived containers after the resources are filled up
+   * get killed and never gets run.
+   * This scenario is more granular and runs more small container compared to
+   * {@link #testKillOpportunisticWhenNoResourcesAvailable()}.
+   */
+  @Test
+  public void testOpportunisticRunsWhenResourcesAvailable() throws Exception {
+    List<StartContainerRequest> startContainerRequests = new ArrayList<>();
+    final int numContainers = 8;
+    final int numContainersQueued = 4;
+    final Set<ContainerId> runContainers = new HashSet<>();
+    final Set<ContainerId> killedContainers = new HashSet<>();
+
+    for (int i = 0; i < numContainers; i++) {
+      // OContainers that should be run
+      startContainerRequests.add(StartContainerRequest.newInstance(
+          recordFactory.newRecordInstance(ContainerLaunchContext.class),
+          createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER,
+              context.getNodeId(),
+              user, BuilderUtils.newResource(512, 1),
+              context.getContainerTokenSecretManager(), null,
+              ExecutionType.OPPORTUNISTIC)));
+    }
+
+    StartContainersRequest allRequests =
+        StartContainersRequest.newInstance(startContainerRequests);
+    containerManager.startContainers(allRequests);
+
+    // Wait for containers to start
+    for (int i = 0; i < numContainersQueued; i++) {
+      final ContainerId containerId = createContainerId(i);
+      BaseContainerManagerTest
+          .waitForNMContainerState(containerManager, containerId,
+              ContainerState.RUNNING, 40);
+      runContainers.add(containerId);
+    }
+
+    // Wait for containers to be killed
+    for (int i = numContainersQueued; i < numContainers; i++) {
+      final ContainerId containerId = createContainerId(i);
+      BaseContainerManagerTest
+          .waitForNMContainerState(containerManager, createContainerId(i),
+              ContainerState.DONE, 40);
+      killedContainers.add(containerId);
+    }
+
+    Thread.sleep(5000);
+
+    // Get container statuses.
+    List<ContainerId> statList = new ArrayList<>();
+    for (int i = 0; i < numContainers; i++) {
+      statList.add(createContainerId(i));
+    }
+
+
+    verifyRunAndKilledContainers(
+        statList, numContainers, runContainers, killedContainers);
+
+    verifyNothingQueued();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 508b8bd091505..218d03afe7eda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -18,18 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse;
@@ -37,8 +26,6 @@
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -46,35 +33,27 @@
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.security.NMTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerSchedulerTest;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
-import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.junit.Assert;
 import org.junit.Test;
-import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
@@ -82,137 +61,11 @@
  * Tests to verify that the {@link ContainerScheduler} is able to queue and
  * make room for containers.
  */
-public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
+public class TestContainerSchedulerQueuing extends BaseContainerSchedulerTest {
   public TestContainerSchedulerQueuing() throws UnsupportedFileSystemException {
     super();
   }
 
-  static {
-    LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
-  }
-
-  private static class Listener implements ContainerStateTransitionListener {
-
-    private final Map<ContainerId,
-        List<ContainerState>> states = new HashMap<>();
-    private final Map<ContainerId, List<ContainerEventType>> events =
-        new HashMap<>();
-
-    @Override
-    public void init(Context context) {}
-
-    @Override
-    public void preTransition(ContainerImpl op,
-        org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
-            ContainerState beforeState,
-        ContainerEvent eventToBeProcessed) {
-      if (!states.containsKey(op.getContainerId())) {
-        states.put(op.getContainerId(), new ArrayList<>());
-        states.get(op.getContainerId()).add(beforeState);
-        events.put(op.getContainerId(), new ArrayList<>());
-      }
-    }
-
-    @Override
-    public void postTransition(ContainerImpl op,
-        org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
-            ContainerState beforeState,
-        org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
-            ContainerState afterState,
-        ContainerEvent processedEvent) {
-      states.get(op.getContainerId()).add(afterState);
-      events.get(op.getContainerId()).add(processedEvent.getType());
-    }
-  }
-
-  private boolean delayContainers = true;
-
-  @Override
-  protected ContainerManagerImpl createContainerManager(
-      DeletionService delSrvc) {
-    return new ContainerManagerImpl(context, exec, delSrvc,
-        getNodeStatusUpdater(), metrics, dirsHandler) {
-
-      @Override
-      protected UserGroupInformation getRemoteUgi() throws YarnException {
-        ApplicationId appId = ApplicationId.newInstance(0, 0);
-        ApplicationAttemptId appAttemptId =
-            ApplicationAttemptId.newInstance(appId, 1);
-        UserGroupInformation ugi =
-            UserGroupInformation.createRemoteUser(appAttemptId.toString());
-        ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
-            .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
-            .getKeyId()));
-        return ugi;
-      }
-
-      @Override
-      protected ContainersMonitor createContainersMonitor(
-          ContainerExecutor exec) {
-        return new ContainersMonitorImpl(exec, dispatcher, this.context) {
-          // Define resources available for containers to be executed.
-          @Override
-          public long getPmemAllocatedForContainers() {
-            return 2048 * 1024 * 1024L;
-          }
-
-          @Override
-          public long getVmemAllocatedForContainers() {
-            float pmemRatio = getConfig().getFloat(
-                YarnConfiguration.NM_VMEM_PMEM_RATIO,
-                YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
-            return (long) (pmemRatio * getPmemAllocatedForContainers());
-          }
-
-          @Override
-          public long getVCoresAllocatedForContainers() {
-            return 4;
-          }
-        };
-      }
-    };
-  }
-
-  @Override
-  protected ContainerExecutor createContainerExecutor() {
-    DefaultContainerExecutor exec = new DefaultContainerExecutor() {
-      ConcurrentMap<String, Boolean> oversleepMap =
-          new ConcurrentHashMap<String, Boolean>();
-      @Override
-      public int launchContainer(ContainerStartContext ctx)
-          throws IOException, ConfigurationException {
-        oversleepMap.put(ctx.getContainer().getContainerId().toString(), false);
-        if (delayContainers) {
-          try {
-            Thread.sleep(10000);
-            if(oversleepMap.get(ctx.getContainer().getContainerId().toString())
-                == true) {
-              Thread.sleep(10000);
-            }
-          } catch (InterruptedException e) {
-            // Nothing..
-          }
-        }
-        return super.launchContainer(ctx);
-      }
-
-      @Override
-      public void pauseContainer(Container container) {
-        // To mimic pausing we force the container to be in the PAUSED state
-        // a little longer by oversleeping.
-        oversleepMap.put(container.getContainerId().toString(), true);
-        LOG.info("Container was paused");
-      }
-
-      @Override
-      public void resumeContainer(Container container) {
-        LOG.info("Container was resumed");
-      }
-    };
-    exec.setConf(conf);
-    return spy(exec);
-  }
-
   @Override
   public void setup() throws IOException {
     conf.setInt(
@@ -408,7 +261,7 @@ public void testStartAndQueueMultipleContainers() throws Exception {
    * @throws Exception
    */
   @Test
-  public void testStartOpportunistcsWhenOppQueueIsFull() throws Exception {
+  public void testStartOpportunisticsWhenOppQueueIsFull() throws Exception {
     containerManager.start();
 
     List<StartContainerRequest> list = new ArrayList<>();
@@ -655,7 +508,7 @@ public void testPauseOpportunisticForGuaranteedContainer() throws Exception {
 
     List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
         ContainerState> containerStates =
-        listener.states.get(createContainerId(0));
+        listener.getStates().get(createContainerId(0));
     Assert.assertEquals(Arrays.asList(
         org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
             ContainerState.NEW,
@@ -676,7 +529,7 @@ public void testPauseOpportunisticForGuaranteedContainer() throws Exception {
         org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
             ContainerState.DONE), containerStates);
     List<ContainerEventType> containerEventTypes =
-        listener.events.get(createContainerId(0));
+        listener.getEvents().get(createContainerId(0));
     Assert.assertEquals(Arrays.asList(ContainerEventType.INIT_CONTAINER,
         ContainerEventType.CONTAINER_LAUNCHED,
         ContainerEventType.PAUSE_CONTAINER,
@@ -1230,7 +1083,7 @@ public void testPromotionOfOpportunisticContainers() throws Exception {
 
     List<org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
         ContainerState> containerStates =
-        listener.states.get(createContainerId(1));
+        listener.getStates().get(createContainerId(1));
     Assert.assertEquals(Arrays.asList(
         org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
             ContainerState.NEW,
@@ -1241,7 +1094,7 @@ public void testPromotionOfOpportunisticContainers() throws Exception {
         org.apache.hadoop.yarn.server.nodemanager.containermanager.container.
             ContainerState.RUNNING), containerStates);
     List<ContainerEventType> containerEventTypes =
-        listener.events.get(createContainerId(1));
+        listener.getEvents().get(createContainerId(1));
     Assert.assertEquals(Arrays.asList(
         ContainerEventType.INIT_CONTAINER,
         ContainerEventType.UPDATE_CONTAINER_TOKEN,
@@ -1254,7 +1107,7 @@ public void testPromotionOfOpportunisticContainers() throws Exception {
   @Test
   public void testContainerUpdateExecTypeGuaranteedToOpportunistic()
       throws Exception {
-    delayContainers = true;
+    setDelayContainers(true);
     containerManager.start();
     // Construct the Container-id
     ContainerId cId = createContainerId(0);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
index b09a1f260e04f..103bfb3217e00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -40,7 +40,6 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
index 54842a91463dc..16abe46dca031 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
@@ -168,6 +168,7 @@ public synchronized void storeContainer(ContainerId containerId,
       int version, long startTime, StartContainerRequest startRequest) {
     RecoveredContainerState rcs = new RecoveredContainerState(containerId);
     rcs.startRequest = startRequest;
+    rcs.status = RecoveredContainerStatus.REQUESTED;
     rcs.version = version;
     try {
       ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 06ad727b2bea2..74968746bc136 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -1756,6 +1756,18 @@ public void testStateStoreForResourceMapping() throws IOException {
     resources = rcs.getResourceMappings().getAssignedResources("numa");
     Assert.assertEquals(numaRes, resources);
     Assert.assertEquals(numaRes, resourceMappings.getAssignedResources("numa"));
+    // test removing numa resources from state store
+    stateStore.releaseAssignedResources(containerId, "numa");
+    recoveredContainers = loadContainersState(stateStore.getContainerStateIterator());
+    resourceMappings = recoveredContainers.get(0).getResourceMappings();
+    assertTrue(resourceMappings.getAssignedResources("numa").isEmpty());
+
+    // testing calling deletion of non-existing key doesn't break anything
+    try {
+      stateStore.releaseAssignedResources(containerId, "numa");
+    }catch (RuntimeException e){
+      Assert.fail("Should not throw exception while deleting non existing key from statestore");
+    }
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebFilter.java
index 5dbe244e78662..266c638a89830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebFilter.java
@@ -27,7 +27,6 @@
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.net.HttpURLConnection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 30b73c1acc7ab..a7222f62e8a83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -432,10 +433,9 @@ public void testSingleNodesXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML+ "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
-    InputSource is = new InputSource();
-    is.setCharacterStream(new StringReader(xml));
+    InputSource is = new InputSource(new StringReader(xml));
     Document dom = db.parse(is);
     NodeList nodes = dom.getElementsByTagName("nodeInfo");
     assertEquals("incorrect number of elements", 1, nodes.getLength());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
index ab06c0f9f3396..204cd00dd9788 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -486,7 +487,7 @@ public void testNodeAppsStateInvalidXML() throws JSONException, Exception {
           response.getType().toString());
       String msg = response.getEntity(String.class);
 
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(msg));
@@ -651,7 +652,7 @@ public void testNodeAppsXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -676,7 +677,7 @@ public void testNodeSingleAppsXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
index 7ec8fcd47d30b..20e1fc98895b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -257,7 +258,7 @@ public void testNodeAuxServicesXML() throws Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 175a0b02470ba..e348b1559eac9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -39,6 +39,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -447,7 +448,7 @@ public void testNodeSingleContainerXML() throws JSONException, Exception {
       assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
           response.getType().toString());
       String xml = response.getEntity(String.class);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
@@ -476,7 +477,7 @@ public void testNodeContainerXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index deae04ad2b0a9..9d096d20c5fdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -107,8 +107,22 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
+      <groupId>com.github.pjfanning</groupId>
       <artifactId>jersey-json</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.jaxrs</groupId>
+          <artifactId>jackson-jaxrs-json-provider</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.sun.jersey.contribs</groupId>
@@ -451,6 +465,8 @@
                  during build time -->
             <exclude>src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema/MappingRulesDescription.java</exclude>
             <exclude>src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema/Rule.java</exclude>
+            <exclude>src/test/resources/webapp/*.json</exclude>
+            <exclude>src/test/resources/webapp/*.xml</exclude>
           </excludes>
         </configuration>
       </plugin>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index dbadc3ce87d7f..f0bc3d67a4663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -37,7 +37,6 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
-
 import org.apache.commons.cli.UnrecognizedOptionException;
 import org.apache.commons.lang3.Range;
 import org.slf4j.Logger;
@@ -600,10 +599,12 @@ public SubmitApplicationResponse submitApplication(
     // checked here, those that are dependent on RM configuration are validated
     // in RMAppManager.
 
+    UserGroupInformation userUgi = null;
     String user = null;
     try {
       // Safety
-      user = UserGroupInformation.getCurrentUser().getShortUserName();
+      userUgi = UserGroupInformation.getCurrentUser();
+      user = userUgi.getShortUserName();
     } catch (IOException ie) {
       LOG.warn("Unable to get the current user.", ie);
       RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@@ -694,7 +695,7 @@ public SubmitApplicationResponse submitApplication(
     try {
       // call RMAppManager to submit application directly
       rmAppManager.submitApplication(submissionContext,
-          System.currentTimeMillis(), user);
+          System.currentTimeMillis(), userUgi);
 
       LOG.info("Application with id " + applicationId.getId() + 
           " submitted by user " + user);
@@ -859,12 +860,14 @@ public GetClusterMetricsResponse getClusterMetrics(
         .newRecordInstance(YarnClusterMetrics.class);
     ymetrics.setNumNodeManagers(this.rmContext.getRMNodes().size());
     ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
+    ymetrics.setNumDecommissioningNodeManagers(clusterMetrics.getNumDecommissioningNMs());
     ymetrics.setNumDecommissionedNodeManagers(clusterMetrics
       .getNumDecommisionedNMs());
     ymetrics.setNumActiveNodeManagers(clusterMetrics.getNumActiveNMs());
     ymetrics.setNumLostNodeManagers(clusterMetrics.getNumLostNMs());
     ymetrics.setNumUnhealthyNodeManagers(clusterMetrics.getUnhealthyNMs());
     ymetrics.setNumRebootedNodeManagers(clusterMetrics.getNumRebootedNMs());
+    ymetrics.setNumShutdownNodeManagers(clusterMetrics.getNumShutdownNMs());
     response.setClusterMetrics(ymetrics);
     return response;
   }
@@ -898,6 +901,9 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
     String name = request.getName();
 
     final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
+    final Set<ApplicationId> runningAppsFilteredByQueues =
+        getRunningAppsFilteredByQueues(apps, queues);
+
     Iterator<RMApp> appsIter = apps.values().iterator();
     
     List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
@@ -911,7 +917,8 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
       }
 
       if (queues != null && !queues.isEmpty()) {
-        if (!queues.contains(application.getQueue())) {
+        if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) &&
+            !queues.contains(application.getQueue())) {
           continue;
         }
       }
@@ -990,6 +997,23 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
     return response;
   }
 
+  private Set<ApplicationId> getRunningAppsFilteredByQueues(
+      Map<ApplicationId, RMApp> apps, Set<String> queues) {
+    final Set<ApplicationId> runningApps = new HashSet<>();
+    for (String queue : queues) {
+      List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue);
+      if (appsInQueue != null) {
+        for (ApplicationAttemptId appAttemptId : appsInQueue) {
+          RMApp rmApp = apps.get(appAttemptId.getApplicationId());
+          if (rmApp != null) {
+            runningApps.add(rmApp.getApplicationId());
+          }
+        }
+      }
+    }
+    return runningApps;
+  }
+
   private Set<String> getLowerCasedAppTypes(GetApplicationsRequest request) {
     Set<String> applicationTypes = new HashSet<>();
     if (request.getApplicationTypes() != null && !request.getApplicationTypes()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
index c56a1e7b3d92d..b31d7c0e6f439 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
@@ -39,10 +39,8 @@
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.metrics.CustomResourceMetricValue;
 import org.apache.hadoop.yarn.metrics.CustomResourceMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetricsForCustomResources;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 @InterfaceAudience.Private
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index c8777f579d5fc..bab7e521dc0c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -36,6 +37,7 @@
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.EnhancedHeadroom;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -53,8 +55,6 @@
 import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
-        .InvalidResourceType;
 import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -334,11 +334,36 @@ public void allocate(ApplicationAttemptId appAttemptId,
         .pullJustFinishedContainers());
     response.setAvailableResources(allocation.getResourceLimit());
 
+    QueueMetrics queueMetrics =
+        this.rmContext.getScheduler().getRootQueueMetrics();
+    if (queueMetrics != null) {
+      int totalVirtualCores =
+          queueMetrics.getAllocatedVirtualCores() + queueMetrics
+              .getAvailableVirtualCores();
+      int pendingContainers = queueMetrics.getPendingContainers();
+      response.setEnhancedHeadroom(
+          EnhancedHeadroom.newInstance(pendingContainers, totalVirtualCores));
+    }
+
     addToContainerUpdates(response, allocation,
         ((AbstractYarnScheduler)getScheduler())
             .getApplicationAttempt(appAttemptId).pullUpdateContainerErrors());
 
-    response.setNumClusterNodes(getScheduler().getNumClusterNodes());
+    String label="";
+    try {
+      label = rmContext.getScheduler()
+          .getQueueInfo(app.getQueue(), false, false)
+          .getDefaultNodeLabelExpression();
+    } catch (Exception e){
+      //Queue may not exist since it could be auto-created in case of
+      // dynamic queues
+    }
+
+    if (label == null || label.equals("")) {
+      response.setNumClusterNodes(getScheduler().getNumClusterNodes());
+    } else {
+      response.setNumClusterNodes(rmContext.getNodeLabelManager().getActiveNMCountPerLabel(label));
+    }
 
     // add collector address for this application
     if (timelineServiceV2Enabled) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 01c9b93417e6b..15b5c7f765976 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -257,7 +257,9 @@ public OpportunisticContainerAllocatorAMService(RMContext rmContext,
 
     int limitMin, limitMax;
 
-    if (comparator == NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH) {
+    if (comparator == NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH ||
+        comparator ==
+            NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH_THEN_RESOURCES) {
       limitMin = rmContext.getYarnConfiguration()
           .getInt(YarnConfiguration.NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH,
               YarnConfiguration.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 762d043b1f6f1..87596e16ee6e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -28,6 +28,12 @@
 
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
+import org.apache.hadoop.yarn.security.Permission;
+import org.apache.hadoop.yarn.security.PrivilegedEntity;
+import org.apache.hadoop.yarn.server.resourcemanager.federation.FederationStateStoreService;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -83,6 +89,9 @@
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
 import org.apache.hadoop.yarn.util.StringHelper;
 
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static org.apache.commons.lang.StringUtils.isNotEmpty;
+
 /**
  * This class manages the list of applications for the resource manager.
  */
@@ -106,6 +115,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
   private boolean timelineServiceV2Enabled;
   private boolean nodeLabelsEnabled;
   private Set<String> exclusiveEnforcedPartitions;
+  private String amDefaultNodeLabel;
+  private FederationStateStoreService federationStateStoreService;
 
   private static final String USER_ID_PREFIX = "userid=";
 
@@ -134,6 +145,8 @@ public RMAppManager(RMContext context,
         .areNodeLabelsEnabled(rmContext.getYarnConfiguration());
     this.exclusiveEnforcedPartitions = YarnConfiguration
         .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
+    this.amDefaultNodeLabel = conf
+        .get(YarnConfiguration.AM_DEFAULT_NODE_LABEL, null);
   }
 
   /**
@@ -294,7 +307,7 @@ protected synchronized void finishApplication(ApplicationId applicationId) {
 
   protected void writeAuditLog(ApplicationId appId) {
     RMApp app = rmContext.getRMApps().get(appId);
-    String operation = "UNKONWN";
+    String operation = "UNKNOWN";
     boolean success = false;
     switch (app.getState()) {
       case FAILED:
@@ -337,6 +350,7 @@ protected synchronized void checkAppNumCompletedLimit() {
           + ", removing app " + removeApp.getApplicationId()
           + " from state store.");
       rmContext.getStateStore().removeApplication(removeApp);
+      removeApplicationIdFromStateStore(removeId);
       completedAppsInStateStore--;
     }
 
@@ -348,20 +362,31 @@ protected synchronized void checkAppNumCompletedLimit() {
           + this.maxCompletedAppsInMemory + ", removing app " + removeId
           + " from memory: ");
       rmContext.getRMApps().remove(removeId);
+      removeApplicationIdFromStateStore(removeId);
       this.applicationACLsManager.removeApplication(removeId);
     }
   }
 
-  @SuppressWarnings("unchecked")
+  @VisibleForTesting
+  @Deprecated
   protected void submitApplication(
       ApplicationSubmissionContext submissionContext, long submitTime,
       String user) throws YarnException {
+    submitApplication(submissionContext, submitTime,
+        UserGroupInformation.createRemoteUser(user));
+  }
+
+  @VisibleForTesting
+  @SuppressWarnings("unchecked")
+  protected void submitApplication(
+      ApplicationSubmissionContext submissionContext, long submitTime,
+      UserGroupInformation userUgi) throws YarnException {
     ApplicationId applicationId = submissionContext.getApplicationId();
 
     // Passing start time as -1. It will be eventually set in RMAppImpl
     // constructor.
     RMAppImpl application = createAndPopulateNewRMApp(
-        submissionContext, submitTime, user, false, -1, null);
+        submissionContext, submitTime, userUgi, false, -1, null);
     try {
       if (UserGroupInformation.isSecurityEnabled()) {
         this.rmContext.getDelegationTokenRenewer()
@@ -394,11 +419,21 @@ protected void recoverApplication(ApplicationStateData appState,
     ApplicationSubmissionContext appContext =
         appState.getApplicationSubmissionContext();
     ApplicationId appId = appContext.getApplicationId();
+    UserGroupInformation userUgi = null;
+    if (appState.getRealUser() != null) {
+      UserGroupInformation realUserUgi = null;
+      realUserUgi =
+          UserGroupInformation.createRemoteUser(appState.getRealUser());
+      userUgi = UserGroupInformation.createProxyUser(appState.getUser(),
+          realUserUgi);
+    } else {
+      userUgi = UserGroupInformation.createRemoteUser(appState.getUser());
+    }
 
     // create and recover app.
     RMAppImpl application =
         createAndPopulateNewRMApp(appContext, appState.getSubmitTime(),
-            appState.getUser(), true, appState.getStartTime(),
+            userUgi, true, appState.getStartTime(),
             appState.getState());
 
     application.handle(new RMAppRecoverEvent(appId, rmState));
@@ -406,8 +441,9 @@ protected void recoverApplication(ApplicationStateData appState,
 
   private RMAppImpl createAndPopulateNewRMApp(
       ApplicationSubmissionContext submissionContext, long submitTime,
-      String user, boolean isRecovery, long startTime,
+      UserGroupInformation userUgi, boolean isRecovery, long startTime,
       RMAppState recoveredFinalState) throws YarnException {
+    String user = userUgi.getShortUserName();
 
     ApplicationPlacementContext placementContext = null;
     if (recoveredFinalState == null) {
@@ -431,7 +467,6 @@ private RMAppImpl createAndPopulateNewRMApp(
 
     // Verify and get the update application priority and set back to
     // submissionContext
-    UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
 
     // Application priority needed to be validated only while submitting. During
     // recovery, validated priority could be recovered from submission context.
@@ -447,32 +482,33 @@ private RMAppImpl createAndPopulateNewRMApp(
       if (scheduler instanceof CapacityScheduler) {
         String queueName = placementContext == null ?
             submissionContext.getQueue() : placementContext.getFullQueuePath();
-
-        String appName = submissionContext.getApplicationName();
-        CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName);
-
-        if (csqueue == null && placementContext != null) {
-          //could be an auto created queue through queue mapping. Validate
-          // parent queue exists and has valid acls
-          String parentQueueName = placementContext.getParentQueue();
-          csqueue = ((CapacityScheduler) scheduler).getQueue(parentQueueName);
+        CapacityScheduler cs = (CapacityScheduler) scheduler;
+        CSQueue csqueue = cs.getQueue(queueName);
+        PrivilegedEntity privilegedEntity = new PrivilegedEntity(
+            csqueue == null ? queueName : csqueue.getQueuePath());
+
+        YarnAuthorizationProvider dynamicAuthorizer = null;
+        if (csqueue == null) {
+          List<Permission> permissions =
+              cs.getCapacitySchedulerQueueManager().getPermissionsForDynamicQueue(
+                  new QueuePath(queueName), cs.getConfiguration());
+          if (!permissions.isEmpty()) {
+            dynamicAuthorizer = new ConfiguredYarnAuthorizer();
+            dynamicAuthorizer.setPermission(permissions, userUgi);
+          }
         }
 
-        if (csqueue != null
-            && !authorizer.checkPermission(
-            new AccessRequest(csqueue.getPrivilegedEntity(), userUgi,
-                SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS),
-                applicationId.toString(), appName, Server.getRemoteAddress(),
-                null))
-            && !authorizer.checkPermission(
-            new AccessRequest(csqueue.getPrivilegedEntity(), userUgi,
-                SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE),
-                applicationId.toString(), appName, Server.getRemoteAddress(),
-                null))) {
-          throw RPCUtil.getRemoteException(new AccessControlException(
-              "User " + user + " does not have permission to submit "
-                  + applicationId + " to queue "
-                  + submissionContext.getQueue()));
+        if (csqueue != null || dynamicAuthorizer != null) {
+          String appName = submissionContext.getApplicationName();
+          if (!checkPermission(createAccessRequest(privilegedEntity, userUgi, applicationId,
+                  appName, QueueACL.SUBMIT_APPLICATIONS), dynamicAuthorizer) &&
+              !checkPermission(createAccessRequest(privilegedEntity, userUgi, applicationId,
+                  appName, QueueACL.ADMINISTER_QUEUE), dynamicAuthorizer)) {
+            throw RPCUtil.getRemoteException(new AccessControlException(
+                "User " + user + " does not have permission to submit "
+                    + applicationId + " to queue "
+                    + submissionContext.getQueue()));
+          }
         }
       }
       if (scheduler instanceof FairScheduler) {
@@ -519,7 +555,7 @@ private RMAppImpl createAndPopulateNewRMApp(
     // Create RMApp
     RMAppImpl application =
         new RMAppImpl(applicationId, rmContext, this.conf,
-            submissionContext.getApplicationName(), user,
+            submissionContext.getApplicationName(), userUgi,
             placementQueueName,
             submissionContext, this.scheduler, this.masterService,
             submitTime, submissionContext.getApplicationType(),
@@ -546,6 +582,23 @@ private RMAppImpl createAndPopulateNewRMApp(
     return application;
   }
 
+  private boolean checkPermission(AccessRequest accessRequest,
+                                  YarnAuthorizationProvider dynamicAuthorizer) {
+    return authorizer.checkPermission(accessRequest) ||
+        (dynamicAuthorizer != null && dynamicAuthorizer.checkPermission(accessRequest));
+  }
+
+  private static AccessRequest createAccessRequest(PrivilegedEntity privilegedEntity,
+                                                   UserGroupInformation userUgi,
+                                                   ApplicationId applicationId,
+                                                   String appName,
+                                                   QueueACL submitApplications) {
+    return new AccessRequest(privilegedEntity, userUgi,
+        SchedulerUtils.toAccessType(submitApplications),
+        applicationId.toString(), appName, Server.getRemoteAddress(),
+        null);
+  }
+
   private List<ResourceRequest> validateAndCreateResourceRequest(
       ApplicationSubmissionContext submissionContext, boolean isRecovery)
       throws InvalidResourceRequestException {
@@ -602,9 +655,12 @@ private List<ResourceRequest> validateAndCreateResourceRequest(
         }
 
         // set label expression for AM ANY request if not set
-        if (null == anyReq.getNodeLabelExpression()) {
-          anyReq.setNodeLabelExpression(submissionContext
-              .getNodeLabelExpression());
+        if (isEmpty(anyReq.getNodeLabelExpression())) {
+          if (isNotEmpty(amDefaultNodeLabel)) {
+            anyReq.setNodeLabelExpression(amDefaultNodeLabel);
+          } else {
+            anyReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
+          }
         }
 
         // Put ANY request at the front
@@ -1003,4 +1059,42 @@ private void copyPlacementQueueToSubmissionContext(
       context.setQueue(placementContext.getQueue());
     }
   }
+
+  @VisibleForTesting
+  public void setFederationStateStoreService(FederationStateStoreService stateStoreService) {
+    this.federationStateStoreService = stateStoreService;
+  }
+
+  /**
+   * Remove ApplicationId From StateStore.
+   *
+   * @param appId appId
+   */
+  private void removeApplicationIdFromStateStore(ApplicationId appId) {
+    if (HAUtil.isFederationEnabled(conf) && federationStateStoreService != null) {
+      try {
+        boolean cleanUpResult =
+            federationStateStoreService.cleanUpFinishApplicationsWithRetries(appId, true);
+        if(cleanUpResult){
+          LOG.info("applicationId = {} remove from state store success.", appId);
+        } else {
+          LOG.warn("applicationId = {} remove from state store failed.", appId);
+        }
+      } catch (Exception e) {
+        LOG.error("applicationId = {} remove from state store error.", appId, e);
+      }
+    }
+  }
+
+  // just test using
+  @VisibleForTesting
+  public void checkAppNumCompletedLimit4Test() {
+    checkAppNumCompletedLimit();
+  }
+
+  // just test using
+  @VisibleForTesting
+  public void finishApplication4Test(ApplicationId applicationId) {
+    finishApplication(applicationId);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
index 2933b40f4ac00..32369ba53ed5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
 
 import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 
 public class RMSecretManagerService extends AbstractService {
 
@@ -135,9 +136,13 @@ protected RMDelegationTokenSecretManager createRMDelegationTokenSecretManager(
     long tokenRenewInterval =
         conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
             YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+    long removeScanInterval =
+        conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
+        TimeUnit.MILLISECONDS);
 
     return new RMDelegationTokenSecretManager(secretKeyInterval,
-        tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext);
+        tokenMaxLifetime, tokenRenewInterval, removeScanInterval, rmContext);
   }
 
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 357cbf3f15291..1bcfdbbafa862 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -23,6 +23,7 @@
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
 import org.apache.hadoop.yarn.metrics.GenericEventTypeMetrics;
+import org.apache.hadoop.yarn.webapp.WebAppException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -916,6 +917,7 @@ protected void serviceInit(Configuration configuration) throws Exception {
         }
         federationStateStoreService = createFederationStateStoreService();
         addIfService(federationStateStoreService);
+        rmAppManager.setFederationStateStoreService(federationStateStoreService);
         LOG.info("Initialized Federation membership.");
       }
 
@@ -995,6 +997,13 @@ protected void serviceStart() throws Exception {
           RMState state = rmStore.loadState();
           recover(state);
           LOG.info("Recovery ended");
+
+          // Make sure that the App is cleaned up after the RM memory is restored.
+          if (HAUtil.isFederationEnabled(conf)) {
+            federationStateStoreService.
+                createCleanUpFinishApplicationThread("Recovery");
+          }
+
         } catch (Exception e) {
           // the Exception from loadState() needs to be handled for
           // HA and we need to give up master status if we got fenced
@@ -1440,7 +1449,12 @@ protected void startWepApp() {
         IsResourceManagerActiveServlet.PATH_SPEC,
         IsResourceManagerActiveServlet.class);
 
-    webApp = builder.start(new RMWebApp(this), uiWebAppContext);
+    try {
+      webApp = builder.start(new RMWebApp(this), uiWebAppContext);
+    } catch (WebAppException e) {
+      webApp = e.getWebApp();
+      throw e;
+    }
   }
 
   private String getWebAppsPath(String appName) {
@@ -1864,8 +1878,8 @@ static void deleteRMConfStore(Configuration conf) throws Exception {
       confStore.initialize(conf, conf, rmContext);
       confStore.format();
     } else {
-      System.out.println("Scheduler Configuration format only " +
-          "supported by MutableConfScheduler.");
+      System.out.println(String.format("Scheduler Configuration format only " +
+          "supported by %s.", MutableConfScheduler.class.getSimpleName()));
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
index a532a4ab9d975..a9e73b8e0b745 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
@@ -36,7 +36,6 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryWriter;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
 import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
index c4dae7d4f7d3f..3cc4dab238618 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -18,27 +18,41 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.federation;
 
+import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.retry.FederationActionRetry;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
@@ -60,10 +74,18 @@
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -86,7 +108,11 @@ public class FederationStateStoreService extends AbstractService
   private FederationStateStore stateStoreClient = null;
   private SubClusterId subClusterId;
   private long heartbeatInterval;
+  private long heartbeatInitialDelay;
   private RMContext rmContext;
+  private String cleanUpThreadNamePrefix = "FederationStateStoreService-Clean-Thread";
+  private int cleanUpRetryCountNum;
+  private long cleanUpRetrySleepTime;
 
   public FederationStateStoreService(RMContext rmContext) {
     super(FederationStateStoreService.class.getName());
@@ -116,10 +142,33 @@ protected void serviceInit(Configuration conf) throws Exception {
     heartbeatInterval = conf.getLong(
         YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS,
         YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+
     if (heartbeatInterval <= 0) {
       heartbeatInterval =
           YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS;
     }
+
+    heartbeatInitialDelay = conf.getTimeDuration(
+        YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY,
+        TimeUnit.SECONDS);
+
+    if (heartbeatInitialDelay <= 0) {
+      LOG.warn("{} configured value is wrong, must be > 0; using default value of {}",
+          YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY,
+          YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY);
+      heartbeatInitialDelay =
+          YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY;
+    }
+
+    cleanUpRetryCountNum = conf.getInt(YarnConfiguration.FEDERATION_STATESTORE_CLEANUP_RETRY_COUNT,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLEANUP_RETRY_COUNT);
+
+    cleanUpRetrySleepTime = conf.getTimeDuration(
+        YarnConfiguration.FEDERATION_STATESTORE_CLEANUP_RETRY_SLEEP_TIME,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLEANUP_RETRY_SLEEP_TIME,
+        TimeUnit.MILLISECONDS);
+
     LOG.info("Initialized federation membership service.");
 
     super.serviceInit(conf);
@@ -196,9 +245,9 @@ private void registerAndInitializeHeartbeat() {
     scheduledExecutorService =
         HadoopExecutors.newSingleThreadScheduledExecutor();
     scheduledExecutorService.scheduleWithFixedDelay(stateStoreHeartbeat,
-        heartbeatInterval, heartbeatInterval, TimeUnit.SECONDS);
-    LOG.info("Started federation membership heartbeat with interval: {}",
-        heartbeatInterval);
+        heartbeatInitialDelay, heartbeatInterval, TimeUnit.SECONDS);
+    LOG.info("Started federation membership heartbeat with interval: {} and initial delay: {}",
+        heartbeatInterval, heartbeatInitialDelay);
   }
 
   @VisibleForTesting
@@ -301,4 +350,232 @@ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster(
       DeleteApplicationHomeSubClusterRequest request) throws YarnException {
     return stateStoreClient.deleteApplicationHomeSubCluster(request);
   }
+
+  @Override
+  public AddReservationHomeSubClusterResponse addReservationHomeSubCluster(
+      AddReservationHomeSubClusterRequest request) throws YarnException {
+    return stateStoreClient.addReservationHomeSubCluster(request);
+  }
+
+  @Override
+  public GetReservationHomeSubClusterResponse getReservationHomeSubCluster(
+      GetReservationHomeSubClusterRequest request) throws YarnException {
+    return stateStoreClient.getReservationHomeSubCluster(request);
+  }
+
+  @Override
+  public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster(
+      GetReservationsHomeSubClusterRequest request) throws YarnException {
+    return stateStoreClient.getReservationsHomeSubCluster(request);
+  }
+
+  @Override
+  public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
+      UpdateReservationHomeSubClusterRequest request) throws YarnException {
+    return stateStoreClient.updateReservationHomeSubCluster(request);
+  }
+
+  @Override
+  public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster(
+      DeleteReservationHomeSubClusterRequest request) throws YarnException {
+    return stateStoreClient.deleteReservationHomeSubCluster(request);
+  }
+
+  @Override
+  public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.storeNewMasterKey(request);
+  }
+
+  @Override
+  public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.removeStoredMasterKey(request);
+  }
+
+  @Override
+  public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.getMasterKeyByDelegationKey(request);
+  }
+
+  @Override
+  public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.storeNewToken(request);
+  }
+
+  @Override
+  public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.updateStoredToken(request);
+  }
+
+  @Override
+  public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.removeStoredToken(request);
+  }
+
+  @Override
+  public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
+      throws YarnException, IOException {
+    return stateStoreClient.getTokenByRouterStoreToken(request);
+  }
+
+  /**
+   * Create a thread that cleans up the app.
+   * @param stage rm-start/rm-stop.
+   */
+  public void createCleanUpFinishApplicationThread(String stage) {
+    String threadName = cleanUpThreadNamePrefix + "-" + stage;
+    Thread finishApplicationThread = new Thread(createCleanUpFinishApplicationThread());
+    finishApplicationThread.setName(threadName);
+    finishApplicationThread.start();
+    LOG.info("CleanUpFinishApplicationThread has been started {}.", threadName);
+  }
+
+  /**
+   * Create a thread that cleans up the apps.
+   *
+   * @return thread object.
+   */
+  private Runnable createCleanUpFinishApplicationThread() {
+    return () -> {
+      createCleanUpFinishApplication();
+    };
+  }
+
+  /**
+   * cleans up the apps.
+   */
+  private void createCleanUpFinishApplication() {
+    try {
+      // Get the current RM's App list based on subClusterId
+      GetApplicationsHomeSubClusterRequest request =
+          GetApplicationsHomeSubClusterRequest.newInstance(subClusterId);
+      GetApplicationsHomeSubClusterResponse response =
+          getApplicationsHomeSubCluster(request);
+      List<ApplicationHomeSubCluster> applicationHomeSCs = response.getAppsHomeSubClusters();
+
+      // Traverse the app list and clean up the app.
+      long successCleanUpAppCount = 0;
+
+      // Save a local copy of the map so that it won't change with the map
+      Map<ApplicationId, RMApp> rmApps = new HashMap<>(this.rmContext.getRMApps());
+
+      // Need to make sure there is app list in RM memory.
+      if (rmApps != null && !rmApps.isEmpty()) {
+        for (ApplicationHomeSubCluster applicationHomeSC : applicationHomeSCs) {
+          ApplicationId applicationId = applicationHomeSC.getApplicationId();
+          if (!rmApps.containsKey(applicationId)) {
+            try {
+              Boolean cleanUpSuccess = cleanUpFinishApplicationsWithRetries(applicationId, false);
+              if (cleanUpSuccess) {
+                LOG.info("application = {} has been cleaned up successfully.", applicationId);
+                successCleanUpAppCount++;
+              }
+            } catch (Exception e) {
+              LOG.error("problem during application = {} cleanup.", applicationId, e);
+            }
+          }
+        }
+      }
+
+      // print app cleanup log
+      LOG.info("cleanup finished applications size = {}, number = {} successful cleanup.",
+          applicationHomeSCs.size(), successCleanUpAppCount);
+    } catch (Exception e) {
+      LOG.error("problem during cleanup applications.", e);
+    }
+  }
+
+  /**
+   * Clean up the federation completed Application.
+   *
+   * @param appId app id.
+   * @param isQuery true, need to query from statestore, false not query.
+   * @throws Exception exception occurs.
+   * @return true, successfully deleted; false, failed to delete or no need to delete
+   */
+  public boolean cleanUpFinishApplicationsWithRetries(ApplicationId appId, boolean isQuery)
+      throws Exception {
+
+    // Generate a request to delete data
+    DeleteApplicationHomeSubClusterRequest req =
+        DeleteApplicationHomeSubClusterRequest.newInstance(appId);
+
+    // CleanUp Finish App.
+    return ((FederationActionRetry<Boolean>) (retry) -> invokeCleanUpFinishApp(appId, isQuery, req))
+        .runWithRetries(cleanUpRetryCountNum, cleanUpRetrySleepTime);
+  }
+
+  /**
+   * CleanUp Finish App.
+   *
+   * @param applicationId app id.
+   * @param isQuery true, need to query from statestore, false not query.
+   * @param delRequest delete Application Request
+   * @return true, successfully deleted; false, failed to delete or no need to delete
+   * @throws YarnException
+   */
+  private boolean invokeCleanUpFinishApp(ApplicationId applicationId, boolean isQuery,
+      DeleteApplicationHomeSubClusterRequest delRequest) throws YarnException {
+    boolean isAppNeedClean = true;
+    // If we need to query the StateStore
+    if (isQuery) {
+      isAppNeedClean = isApplicationNeedClean(applicationId);
+    }
+    // When the App needs to be cleaned up, clean up the App.
+    if (isAppNeedClean) {
+      DeleteApplicationHomeSubClusterResponse response =
+          deleteApplicationHomeSubCluster(delRequest);
+      if (response != null) {
+        LOG.info("The applicationId = {} has been successfully cleaned up.", applicationId);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Used to determine whether the Application is cleaned up.
+   *
+   * When the app in the RM is completed,
+   * the HomeSC corresponding to the app will be queried in the StateStore.
+   * If the current RM is the HomeSC, the completed app will be cleaned up.
+   *
+   * @param applicationId applicationId
+   * @return true, app needs to be cleaned up;
+   *         false, app doesn't need to be cleaned up.
+   */
+  private boolean isApplicationNeedClean(ApplicationId applicationId) {
+    GetApplicationHomeSubClusterRequest queryRequest =
+            GetApplicationHomeSubClusterRequest.newInstance(applicationId);
+    // Here we need to use try...catch,
+    // because getApplicationHomeSubCluster may throw not exist exception
+    try {
+      GetApplicationHomeSubClusterResponse queryResp =
+          getApplicationHomeSubCluster(queryRequest);
+      if (queryResp != null) {
+        ApplicationHomeSubCluster appHomeSC = queryResp.getApplicationHomeSubCluster();
+        SubClusterId homeSubClusterId = appHomeSC.getHomeSubCluster();
+        if (!subClusterId.equals(homeSubClusterId)) {
+          LOG.warn("The homeSubCluster of applicationId = {} belong subCluster = {}, " +
+              " not belong subCluster = {} and is not allowed to delete.",
+              applicationId, homeSubClusterId, subClusterId);
+          return false;
+        }
+      } else {
+        LOG.warn("The applicationId = {} not belong subCluster = {} " +
+            " and is not allowed to delete.", applicationId, subClusterId);
+        return false;
+      }
+    } catch (Exception e) {
+      LOG.warn("query applicationId = {} error.", applicationId, e);
+      return false;
+    }
+    return true;
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 23aba4a23b2a0..f1b80a946a7d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -18,8 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.metrics;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,6 +37,7 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
@@ -59,9 +65,55 @@ public TimelineServiceV1Publisher() {
   }
 
   private TimelineClient client;
+  private LinkedBlockingQueue<TimelineEntity> entityQueue;
+  private ExecutorService sendEventThreadPool;
+  private int dispatcherPoolSize;
+  private int dispatcherBatchSize;
+  private int putEventInterval;
+  private boolean isTimeLineServerBatchEnabled;
+  private volatile boolean stopped = false;
+  private PutEventThread putEventThread;
+  private Object sendEntityLock;
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
+    isTimeLineServerBatchEnabled =
+        conf.getBoolean(
+            YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED,
+            YarnConfiguration.DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED);
+    if (isTimeLineServerBatchEnabled) {
+      putEventInterval =
+          conf.getInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL,
+              YarnConfiguration.DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL)
+              * 1000;
+      if (putEventInterval <= 0) {
+        throw new IllegalArgumentException(
+            "RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL should be greater than 0");
+      }
+      dispatcherPoolSize = conf.getInt(
+          YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
+          YarnConfiguration.
+              DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE);
+      if (dispatcherPoolSize <= 0) {
+        throw new IllegalArgumentException(
+            "RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE should be greater than 0");
+      }
+      dispatcherBatchSize = conf.getInt(
+          YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE,
+          YarnConfiguration.
+              DEFAULT_RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE);
+      if (dispatcherBatchSize <= 1) {
+        throw new IllegalArgumentException(
+            "RM_TIMELINE_SERVER_V1_PUBLISHER_DISPATCHER_BATCH_SIZE should be greater than 1");
+      }
+      putEventThread = new PutEventThread();
+      sendEventThreadPool = Executors.newFixedThreadPool(dispatcherPoolSize);
+      entityQueue = new LinkedBlockingQueue<>(dispatcherBatchSize + 1);
+      sendEntityLock = new Object();
+      LOG.info("Timeline service v1 batch publishing enabled");
+    } else {
+      LOG.info("Timeline service v1 batch publishing disabled");
+    }
     client = TimelineClient.createTimelineClient();
     addIfService(client);
     super.serviceInit(conf);
@@ -69,6 +121,36 @@ protected void serviceInit(Configuration conf) throws Exception {
         new TimelineV1EventHandler());
   }
 
+  protected void serviceStart() throws Exception {
+    if (isTimeLineServerBatchEnabled) {
+      stopped = false;
+      putEventThread.start();
+    }
+    super.serviceStart();
+  }
+
+  protected void serviceStop() throws Exception {
+    super.serviceStop();
+    if (isTimeLineServerBatchEnabled) {
+      stopped = true;
+      putEventThread.interrupt();
+      try {
+        putEventThread.join();
+        SendEntity task = new SendEntity();
+        if (!task.buffer.isEmpty()) {
+          LOG.info("Initiating final putEntities, remaining entities left in entityQueue: {}",
+              task.buffer.size());
+          sendEventThreadPool.submit(task);
+        }
+      } finally {
+        sendEventThreadPool.shutdown();
+        if (!sendEventThreadPool.awaitTermination(3, TimeUnit.SECONDS)) {
+          sendEventThreadPool.shutdownNow();
+        }
+      }
+    }
+  }
+
   @SuppressWarnings("unchecked")
   @Override
   public void appCreated(RMApp app, long createdTime) {
@@ -257,7 +339,7 @@ public void appAttemptRegistered(RMAppAttempt appAttempt,
   @SuppressWarnings("unchecked")
   @Override
   public void appAttemptFinished(RMAppAttempt appAttempt,
-      RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
+      RMAppAttemptState appAttemptState, RMApp app, long finishedTime) {
     TimelineEntity entity =
         createAppAttemptEntity(appAttempt.getAppAttemptId());
 
@@ -274,7 +356,7 @@ public void appAttemptFinished(RMAppAttempt appAttempt,
     eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO,
         app.getFinalApplicationStatus().toString());
     eventInfo.put(AppAttemptMetricsConstants.STATE_INFO, RMServerUtils
-        .createApplicationAttemptState(appAttemtpState).toString());
+        .createApplicationAttemptState(appAttemptState).toString());
     if (appAttempt.getMasterContainer() != null) {
       eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO,
           appAttempt.getMasterContainer().getId().toString());
@@ -374,23 +456,68 @@ private static TimelineEntity createContainerEntity(ContainerId containerId) {
   }
 
   private void putEntity(TimelineEntity entity) {
-    try {
+    if (isTimeLineServerBatchEnabled) {
+      try {
+        entityQueue.put(entity);
+        if (entityQueue.size() > dispatcherBatchSize) {
+          SendEntity task = null;
+          synchronized (sendEntityLock) {
+            if (entityQueue.size() > dispatcherBatchSize) {
+              task = new SendEntity();
+            }
+          }
+          if (task != null) {
+            sendEventThreadPool.submit(task);
+          }
+        }
+      } catch (Exception e) {
+        LOG.error("Error when publishing entity batch  [ " + entity.getEntityType() + ","
+            + entity.getEntityId() + " ] ", e);
+      }
+    } else {
+      try {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Publishing the entity " + entity.getEntityId()
+              + ", JSON-style content: "
+              + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+        }
+        client.putEntities(entity);
+      } catch (Exception e) {
+        LOG.error("Error when publishing entity [ " + entity.getEntityType() + ","
+            + entity.getEntityId() + " ] ", e);
+      }
+    }
+  }
+
+  private class SendEntity implements Runnable {
+
+    private ArrayList<TimelineEntity> buffer;
+
+    SendEntity() {
+      buffer = new ArrayList();
+      entityQueue.drainTo(buffer);
+    }
+
+    @Override
+    public void run() {
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Publishing the entity " + entity.getEntityId()
-            + ", JSON-style content: "
-            + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+        LOG.debug("Number of timeline entities being sent in batch: {}", buffer.size());
+      }
+      if (buffer.isEmpty()) {
+        return;
+      }
+      try {
+        client.putEntities(buffer.toArray(new TimelineEntity[0]));
+      } catch (Exception e) {
+        LOG.error("Error when publishing entity: ", e);
       }
-      client.putEntities(entity);
-    } catch (Exception e) {
-      LOG.error("Error when publishing entity [" + entity.getEntityType() + ","
-          + entity.getEntityId() + "]", e);
     }
   }
 
   private class TimelineV1PublishEvent extends TimelinePublishEvent {
     private TimelineEntity entity;
 
-    public TimelineV1PublishEvent(SystemMetricsEventType type,
+    TimelineV1PublishEvent(SystemMetricsEventType type,
         TimelineEntity entity, ApplicationId appId) {
       super(type, appId);
       this.entity = entity;
@@ -408,4 +535,46 @@ public void handle(TimelineV1PublishEvent event) {
       putEntity(event.getEntity());
     }
   }
-}
+
+  private class PutEventThread extends Thread {
+    PutEventThread() {
+      super("PutEventThread");
+    }
+
+    @Override
+    public void run() {
+      LOG.info("System metrics publisher will put events every " +
+          String.valueOf(putEventInterval) + " milliseconds");
+      while (!stopped && !Thread.currentThread().isInterrupted()) {
+        if (System.currentTimeMillis() % putEventInterval >= 1000) {
+          try {
+            Thread.sleep(500);
+          } catch (InterruptedException e) {
+            LOG.warn(SystemMetricsPublisher.class.getName()
+                + " is interrupted. Exiting.");
+            break;
+          }
+          continue;
+        }
+        SendEntity task = null;
+        synchronized (sendEntityLock) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Creating SendEntity task in PutEventThread");
+          }
+          task = new SendEntity();
+        }
+        if (task != null) {
+          sendEventThreadPool.submit(task);
+        }
+        try {
+          // sleep added to avoid multiple SendEntity task within a single interval.
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          LOG.warn(SystemMetricsPublisher.class.getName()
+              + " is interrupted. Exiting.");
+          break;
+        }
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
index c8f68a26a07a4..c3c594512b07c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
@@ -31,12 +31,16 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.PriorityQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Calculate how much resources need to be preempted for each queue,
  * will be used by {@link PreemptionCandidatesSelector}.
  */
 public class AbstractPreemptableResourceCalculator {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbstractPreemptableResourceCalculator.class);
 
   protected final CapacitySchedulerPreemptionContext context;
   protected final ResourceCalculator rc;
@@ -76,6 +80,34 @@ private double getIdealPctOfGuaranteed(TempQueuePerPartition q) {
     }
   }
 
+  private static class NormalizationTuple {
+    private Resource numerator;
+    private Resource denominator;
+
+    NormalizationTuple(Resource numer, Resource denom) {
+      this.numerator = numer;
+      this.denominator = denom;
+    }
+
+    long getNumeratorValue(int i) {
+      return numerator.getResourceInformation(i).getValue();
+    }
+
+    long getDenominatorValue(int i) {
+      String nUnits = numerator.getResourceInformation(i).getUnits();
+      ResourceInformation dResourceInformation = denominator
+          .getResourceInformation(i);
+      return UnitsConversionUtil.convert(
+          dResourceInformation.getUnits(), nUnits, dResourceInformation.getValue());
+    }
+
+    float getNormalizedValue(int i) {
+      long nValue = getNumeratorValue(i);
+      long dValue = getDenominatorValue(i);
+      return dValue == 0 ? 0.0f : (float) nValue / dValue;
+    }
+  }
+
   /**
    * PreemptableResourceCalculator constructor.
    *
@@ -175,7 +207,7 @@ protected void computeFixpointAllocation(Resource totGuarant,
         unassigned, Resources.none())) {
       // we compute normalizedGuarantees capacity based on currently active
       // queues
-      resetCapacity(unassigned, orderedByNeed, ignoreGuarantee);
+      resetCapacity(orderedByNeed, ignoreGuarantee);
 
       // For each underserved queue (or set of queues if multiple are equally
       // underserved), offer its share of the unassigned resources based on its
@@ -252,47 +284,146 @@ protected void initIdealAssignment(Resource totGuarant,
   /**
    * Computes a normalizedGuaranteed capacity based on active queues.
    *
-   * @param clusterResource
-   *          the total amount of resources in the cluster
    * @param queues
    *          the list of queues to consider
    * @param ignoreGuar
    *          ignore guarantee.
    */
-  private void resetCapacity(Resource clusterResource,
-      Collection<TempQueuePerPartition> queues, boolean ignoreGuar) {
+  private void resetCapacity(Collection<TempQueuePerPartition> queues,
+                             boolean ignoreGuar) {
     Resource activeCap = Resource.newInstance(0, 0);
+    float activeTotalAbsCap = 0.0f;
     int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
 
     if (ignoreGuar) {
-      for (TempQueuePerPartition q : queues) {
-        for (int i = 0; i < maxLength; i++) {
-          q.normalizedGuarantee[i] = 1.0f / queues.size();
+      for (int i = 0; i < maxLength; i++) {
+        for (TempQueuePerPartition q : queues) {
+          computeNormGuarEvenly(q, queues.size(), i);
         }
       }
     } else {
       for (TempQueuePerPartition q : queues) {
         Resources.addTo(activeCap, q.getGuaranteed());
+        activeTotalAbsCap += q.getAbsCapacity();
       }
-      for (TempQueuePerPartition q : queues) {
-        for (int i = 0; i < maxLength; i++) {
-          ResourceInformation nResourceInformation = q.getGuaranteed()
-              .getResourceInformation(i);
-          ResourceInformation dResourceInformation = activeCap
-              .getResourceInformation(i);
-
-          long nValue = nResourceInformation.getValue();
-          long dValue = UnitsConversionUtil.convert(
-              dResourceInformation.getUnits(), nResourceInformation.getUnits(),
-              dResourceInformation.getValue());
-          if (dValue != 0) {
-            q.normalizedGuarantee[i] = (float) nValue / dValue;
+
+      // loop through all resource types and normalize guaranteed capacity for all queues
+      for (int i = 0; i < maxLength; i++) {
+        boolean useAbsCapBasedNorm = false;
+        // if the sum of absolute capacity of all queues involved is 0,
+        // we should normalize evenly
+        boolean useEvenlyDistNorm = activeTotalAbsCap == 0;
+
+        // loop through all the queues once to determine the
+        // right normalization strategy for current processing resource type
+        for (TempQueuePerPartition q : queues) {
+          NormalizationTuple normTuple = new NormalizationTuple(
+              q.getGuaranteed(), activeCap);
+          long queueGuaranValue = normTuple.getNumeratorValue(i);
+          long totalActiveGuaranValue = normTuple.getDenominatorValue(i);
+
+          if (queueGuaranValue == 0 && q.getAbsCapacity() != 0 && totalActiveGuaranValue != 0) {
+            // when the rounded value of a resource type is 0 but its absolute capacity is not 0,
+            // we should consider taking the normalized guarantee based on absolute capacity
+            useAbsCapBasedNorm = true;
+            break;
+          }
+
+          if (totalActiveGuaranValue == 0) {
+            // If totalActiveGuaranValue from activeCap is zero, that means the guaranteed capacity
+            // of this resource dimension for all active queues is tiny (close to 0).
+            // For example, if a queue has 1% of minCapacity on a cluster with a totalVcores of 48,
+            // then the idealAssigned Vcores for this queue is (48 * 0.01)=0.48 which then
+            // get rounded/casted into 0 (double -> long)
+            // In this scenario where the denominator is 0, we can just spread resources across
+            // all tiny queues evenly since their absoluteCapacity are roughly the same
+            useEvenlyDistNorm = true;
+          }
+        }
+
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Queue normalization strategy: " +
+              "absoluteCapacityBasedNormalization(" + useAbsCapBasedNorm +
+              "), evenlyDistributedNormalization(" + useEvenlyDistNorm +
+              "), defaultNormalization(" + !(useAbsCapBasedNorm || useEvenlyDistNorm) + ")");
+        }
+
+        // loop through all the queues again to apply normalization strategy
+        for (TempQueuePerPartition q : queues) {
+          if (useAbsCapBasedNorm) {
+            computeNormGuarFromAbsCapacity(q, activeTotalAbsCap, i);
+          } else if (useEvenlyDistNorm) {
+            computeNormGuarEvenly(q, queues.size(), i);
+          } else {
+            computeDefaultNormGuar(q, activeCap, i);
           }
         }
       }
     }
   }
 
+  /**
+   * Computes the normalized guaranteed capacity based on the weight of a queue's abs capacity.
+   *
+   * Example:
+   *  There are two active queues: queueA & queueB, and
+   *  their configured absolute minimum capacity is 1% and 3% respectively.
+   *
+   *  Then their normalized guaranteed capacity are:
+   *    normalized_guar_queueA = 0.01 / (0.01 + 0.03) = 0.25
+   *    normalized_guar_queueB = 0.03 / (0.01 + 0.03) = 0.75
+   *
+   * @param q
+   *          the queue to consider
+   * @param activeTotalAbsCap
+   *          the sum of absolute capacity of all active queues
+   * @param resourceTypeIdx
+   *          index of the processing resource type
+   */
+  private static void computeNormGuarFromAbsCapacity(TempQueuePerPartition q,
+                                                     float activeTotalAbsCap,
+                                                     int resourceTypeIdx) {
+    if (activeTotalAbsCap != 0) {
+      q.normalizedGuarantee[resourceTypeIdx] = q.getAbsCapacity() / activeTotalAbsCap;
+    }
+  }
+
+  /**
+   * Computes the normalized guaranteed capacity evenly based on num of active queues.
+   *
+   * @param q
+   *          the queue to consider
+   * @param numOfActiveQueues
+   *          number of active queues
+   * @param resourceTypeIdx
+   *          index of the processing resource type
+   */
+  private static void computeNormGuarEvenly(TempQueuePerPartition q,
+                                            int numOfActiveQueues,
+                                            int resourceTypeIdx) {
+    q.normalizedGuarantee[resourceTypeIdx] = 1.0f / numOfActiveQueues;
+  }
+
+  /**
+   * The default way to compute a queue's normalized guaranteed capacity.
+   *
+   * For each resource type, divide a queue's configured guaranteed amount (MBs/Vcores) by
+   * the total amount of guaranteed resource of all active queues
+   *
+   * @param q
+   *          the queue to consider
+   * @param activeCap
+   *          total guaranteed resources of all active queues
+   * @param resourceTypeIdx
+   *          index of the processing resource type
+   */
+  private static void computeDefaultNormGuar(TempQueuePerPartition q,
+                                             Resource activeCap,
+                                             int resourceTypeIdx) {
+    NormalizationTuple normTuple = new NormalizationTuple(q.getGuaranteed(), activeCap);
+    q.normalizedGuarantee[resourceTypeIdx] = normTuple.getNormalizedValue(resourceTypeIdx);
+  }
+
   // Take the most underserved TempQueue (the one on the head). Collect and
   // return the list of all queues that have the same idealAssigned
   // percentage of guaranteed.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index d9e9091bc86f7..440066c5ca606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -24,7 +24,7 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
@@ -86,7 +86,7 @@ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
       }
 
       // compute resToObtainByPartition considered inter-queue preemption
-      LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName,
+      AbstractLeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName,
           RMNodeLabelsManager.NO_LABEL).leafQueue;
 
       Map<String, Resource> resToObtainByPartition =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index a9d3c058e6569..4cc1c8cee9dd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -23,13 +23,13 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.TreeSet;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +39,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.IntraQueuePreemptionOrderPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
@@ -401,8 +401,7 @@ private PriorityQueue<TempAppPerPartition> createTempAppForResCalculation(
       pending = (pending == null) ? Resources.createResource(0, 0) : pending;
       reserved = (reserved == null) ? Resources.createResource(0, 0) : reserved;
 
-      HashSet<String> partitions = new HashSet<String>(
-          app.getAppAttemptResourceUsage().getNodePartitionsSet());
+      Set<String> partitions = app.getAppAttemptResourceUsage().getExistingNodeLabels();
       partitions.addAll(app.getTotalPendingRequestsPerPartition().keySet());
 
       // Create TempAppPerQueue for further calculation.
@@ -419,8 +418,10 @@ private PriorityQueue<TempAppPerPartition> createTempAppForResCalculation(
       String userName = app.getUser();
       TempUserPerPartition tmpUser = usersPerPartition.get(userName);
       if (tmpUser == null) {
-        ResourceUsage userResourceUsage = tq.leafQueue.getUser(userName)
-            .getResourceUsage();
+        // User might have already been removed, but preemption still accounts for this app,
+        // therefore reinserting the user will not cause a memory leak
+        User  user = tq.leafQueue.getOrCreateUser(userName);
+        ResourceUsage userResourceUsage = user.getResourceUsage();
 
         // perUserAMUsed was populated with running apps, now we are looping
         // through both running and pending apps.
@@ -428,8 +429,7 @@ private PriorityQueue<TempAppPerPartition> createTempAppForResCalculation(
         amUsed = (userSpecificAmUsed == null)
             ? Resources.none() : userSpecificAmUsed;
 
-        tmpUser = new TempUserPerPartition(
-            tq.leafQueue.getUser(userName), tq.queueName,
+        tmpUser = new TempUserPerPartition(user, tq.queueName,
             Resources.clone(userResourceUsage.getUsed(partition)),
             Resources.clone(amUsed),
             Resources.clone(userResourceUsage.getReserved(partition)),
@@ -575,7 +575,7 @@ public void validateOutSameAppPriorityFromDemand(Resource cluster,
   }
 
   private Resource calculateUsedAMResourcesPerQueue(String partition,
-      LeafQueue leafQueue, Map<String, Resource> perUserAMUsed) {
+      AbstractLeafQueue leafQueue, Map<String, Resource> perUserAMUsed) {
     Collection<FiCaSchedulerApp> runningApps = leafQueue.getApplications();
     Resource amUsed = Resources.createResource(0, 0);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
index cea1bca773690..f0bd03b7d1330 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.IntraQueuePreemptionOrderPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.AbstractComparatorOrderingPolicy;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -146,7 +146,7 @@ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
 
       // 4. Iterate from most under-served queue in order.
       for (String queueName : queueNames) {
-        LeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName,
+        AbstractLeafQueue leafQueue = preemptionContext.getQueueByPartition(queueName,
             RMNodeLabelsManager.NO_LABEL).leafQueue;
 
         // skip if not a leafqueue
@@ -181,7 +181,7 @@ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
         leafQueue.getReadLock().lock();
         try {
           for (FiCaSchedulerApp app : apps) {
-            preemptFromLeastStarvedApp(leafQueue, app, selectedCandidates,
+            preemptFromLeastStarvedApp(app, selectedCandidates,
                 curCandidates, clusterResource, totalPreemptedResourceAllowed,
                 resToObtainByPartition, rollingResourceUsagePerUser);
           }
@@ -195,7 +195,7 @@ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
   }
 
   private void initializeUsageAndUserLimitForCompute(Resource clusterResource,
-      String partition, LeafQueue leafQueue,
+      String partition, AbstractLeafQueue leafQueue,
       Map<String, Resource> rollingResourceUsagePerUser) {
     for (String user : leafQueue.getAllUsers()) {
       // Initialize used resource of a given user for rolling computation.
@@ -206,8 +206,7 @@ private void initializeUsageAndUserLimitForCompute(Resource clusterResource,
     }
   }
 
-  private void preemptFromLeastStarvedApp(LeafQueue leafQueue,
-      FiCaSchedulerApp app,
+  private void preemptFromLeastStarvedApp(FiCaSchedulerApp app,
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
       Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       Resource clusterResource, Resource totalPreemptedResourceAllowed,
@@ -293,7 +292,7 @@ private void computeIntraQueuePreemptionDemand(Resource clusterResource,
       for (String queueName : queueNames) {
         TempQueuePerPartition tq = context.getQueueByPartition(queueName,
             partition);
-        LeafQueue leafQueue = tq.leafQueue;
+        AbstractLeafQueue leafQueue = tq.leafQueue;
 
         // skip if its parent queue
         if (null == leafQueue) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 9439c538af52f..1c4d60962eef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -496,7 +496,7 @@ private void containerBasedPreemptOrKill(CSQueue root,
     Map<ApplicationAttemptId, Set<RMContainer>> toPreempt =
         new HashMap<>();
     Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
-        Set<RMContainer>>> toPreemptPerSelector =  new HashMap<>();;
+        Set<RMContainer>>> toPreemptPerSelector =  new HashMap<>();
     for (PreemptionCandidatesSelector selector :
         candidatesSelectionPolicies) {
       long startTime = 0;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 57dc639570215..78075bb5c17c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -26,7 +26,7 @@
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -56,7 +56,7 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
 
   final ArrayList<TempQueuePerPartition> children;
   private Collection<TempAppPerPartition> apps;
-  LeafQueue leafQueue;
+  AbstractLeafQueue leafQueue;
   ParentQueue parentQueue;
   boolean preemptionDisabled;
 
@@ -81,8 +81,8 @@ public TempQueuePerPartition(String queueName, Resource current,
     super(queueName, current, Resource.newInstance(0, 0), reserved,
         Resource.newInstance(0, 0));
 
-    if (queue instanceof LeafQueue) {
-      LeafQueue l = (LeafQueue) queue;
+    if (queue instanceof AbstractLeafQueue) {
+      AbstractLeafQueue l = (AbstractLeafQueue) queue;
       pending = l.getTotalPendingResourcesConsideringUserLimit(
           totalPartitionResource, partition, false);
       pendingDeductReserved = l.getTotalPendingResourcesConsideringUserLimit(
@@ -113,7 +113,7 @@ public TempQueuePerPartition(String queueName, Resource current,
     this.effMaxRes = effMaxRes;
   }
 
-  public void setLeafQueue(LeafQueue l) {
+  public void setLeafQueue(AbstractLeafQueue l) {
     assert children.size() == 0;
     this.leafQueue = l;
   }
@@ -201,6 +201,10 @@ Resource offer(Resource avail, ResourceCalculator rc,
     return remain;
   }
 
+  public float getAbsCapacity() {
+    return absCapacity;
+  }
+
   public Resource getGuaranteed() {
     if(!effMinRes.equals(Resources.none())) {
       return Resources.clone(effMinRes);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index 6769e83669fd4..b0b98ad9b481a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -61,7 +61,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAttributesUpdateSchedulerEvent;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 
 /**
  * Manager holding the attributes to Labels.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java
index bb95cf98f40a9..ba0c430d77f93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMDelegatedNodeLabelsUpdater.java
@@ -56,7 +56,7 @@ public class RMDelegatedNodeLabelsUpdater extends CompositeService {
   private Timer nodeLabelsScheduler;
   // 30 seconds
   @VisibleForTesting
-  public long nodeLabelsUpdateInterval = 30 * 1000;
+  public long nodeLabelsUpdateInterval;
 
   private Set<NodeId> newlyRegisteredNodes = new HashSet<NodeId>();
   // Lock to protect newlyRegisteredNodes
@@ -78,6 +78,9 @@ protected void serviceInit(Configuration conf) throws Exception {
     allNodesLabelUpdateInterval = conf.getLong(
         YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
         YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
+    nodeLabelsUpdateInterval =
+        conf.getLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS,
+            YarnConfiguration.DEFAULT_RM_NODE_LABELS_PROVIDER_UPDATE_NEWLY_REGISTERED_INTERVAL_MS);
     rmNodeLabelsMappingProvider = createRMNodeLabelsMappingProvider(conf);
     addService(rmNodeLabelsMappingProvider);
     super.serviceInit(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index ef98c1a79f178..cefed1dd9fd85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.placement;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -32,7 +32,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -133,7 +133,7 @@ public boolean initialize(ResourceScheduler scheduler) throws IOException {
     overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
 
     if (groups == null) {
-      groups = Groups.getUserToGroupsMappingService(conf);
+      groups = Groups.getUserToGroupsMappingService(csContext.getConf());
     }
 
     MappingRuleValidationContext validationContext = buildValidationContext();
@@ -277,7 +277,7 @@ private String validateAndNormalizeQueue(
     }
 
     CSQueue queue = queueManager.getQueueByFullName(normalizedName);
-    if (queue != null && !(queue instanceof LeafQueue)) {
+    if (queue != null && !(queue instanceof AbstractLeafQueue)) {
       throw new YarnException("Mapping rule returned a non-leaf queue '" +
           normalizedName + "', cannot place application in it.");
     }
@@ -535,4 +535,9 @@ private String cleanName(String name) {
       return name;
     }
   }
+
+  @VisibleForTesting
+  public Groups getGroups() {
+    return groups;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
index 76e3e275fc9f7..f6381bdcc1f72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
@@ -21,13 +21,11 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
 
 import java.io.IOException;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-
 /**
  * Utility class for Capacity Scheduler queue PlacementRules.
  */
@@ -83,7 +81,7 @@ public static QueueMapping validateAndGetAutoCreatedQueueMapping(
   public static QueueMapping validateAndGetQueueMapping(
       CapacitySchedulerQueueManager queueManager, CSQueue queue,
       QueueMapping mapping) throws IOException {
-    if (!(queue instanceof LeafQueue)) {
+    if (!(queue instanceof AbstractLeafQueue)) {
       throw new IOException(
           "mapping contains invalid or non-leaf queue : " +
           mapping.getFullPath());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java
index 4218b6faa2478..cceb7e6bb2315 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/MappingRuleValidationContextImpl.java
@@ -22,7 +22,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
@@ -91,7 +91,7 @@ private boolean validateStaticQueuePath(QueuePath path)
           "' under it.");
     case QUEUE_EXISTS:
       CSQueue queue = queueManager.getQueue(normalizedPath);
-      if (!(queue instanceof LeafQueue)) {
+      if (!(queue instanceof AbstractLeafQueue)) {
         throw new YarnException("Target queue '" + path.getFullPath() +
             "' but it's not a leaf queue.");
       }
@@ -157,7 +157,7 @@ private boolean validateDynamicQueuePath(QueuePath path)
     //if the static part of our queue exists, and it's not a leaf queue,
     //we cannot do any deeper validation
     if (queue != null) {
-      if (queue instanceof LeafQueue) {
+      if (queue instanceof AbstractLeafQueue) {
         throw new YarnException("Queue path '" + path +"' is invalid " +
             "because '" + normalizedStaticPart + "' is a leaf queue, " +
             "which can have no other queues under it.");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 8a444911c81d3..a93ed1bc6b495 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -937,7 +937,8 @@ public void storeNewApplication(RMApp app) {
     assert context instanceof ApplicationSubmissionContextPBImpl;
     ApplicationStateData appState =
         ApplicationStateData.newInstance(app.getSubmitTime(),
-            app.getStartTime(), context, app.getUser(), app.getCallerContext());
+            app.getStartTime(), context, app.getUser(), app.getRealUser(),
+            app.getCallerContext());
     appState.setApplicationTimeouts(app.getApplicationTimeouts());
     getRMStateStoreEventHandler().handle(new RMStateStoreAppEvent(appState));
   }
@@ -1170,7 +1171,7 @@ public void removeApplication(RMApp app) {
     ApplicationStateData appState =
         ApplicationStateData.newInstance(app.getSubmitTime(),
             app.getStartTime(), app.getApplicationSubmissionContext(),
-            app.getUser(), app.getCallerContext());
+            app.getUser(), app.getRealUser(), app.getCallerContext());
     for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
       appState.attempts.put(appAttempt.getAppAttemptId(), null);
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
index 2b0bd2b5bad39..8b908e56d0a4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
@@ -92,9 +92,46 @@ public static ApplicationStateData newInstance(long submitTime,
   
   public static ApplicationStateData newInstance(long submitTime,
       long startTime, ApplicationSubmissionContext context, String user) {
-    return newInstance(submitTime, startTime, context, user, null);
+    return newInstance(submitTime, startTime, context, user,
+        (CallerContext) null);
   }
-  
+
+  public static ApplicationStateData newInstance(long submitTime,
+      long startTime, String user, String realUser,
+      ApplicationSubmissionContext submissionContext, RMAppState state,
+      String diagnostics, long launchTime, long finishTime,
+      CallerContext callerContext) {
+    ApplicationStateData appState =
+        newInstance(submitTime, startTime, user, submissionContext, state,
+            diagnostics, launchTime, finishTime, callerContext);
+    if (realUser != null) {
+      appState.setRealUser(realUser);
+    }
+    return appState;
+  }
+
+  public static ApplicationStateData newInstance(long submitTime,
+      long startTime, String user, String realUser,
+      ApplicationSubmissionContext submissionContext, RMAppState state,
+      String diagnostics, long launchTime, long finishTime,
+      CallerContext callerContext,
+      Map<ApplicationTimeoutType, Long> applicationTimeouts) {
+    ApplicationStateData appState =
+        newInstance(submitTime, startTime, user, submissionContext, state,
+            diagnostics, launchTime, finishTime, callerContext, applicationTimeouts);
+    if (realUser != null) {
+      appState.setRealUser(realUser);
+    }
+    return appState;
+  }
+
+  public static ApplicationStateData newInstance(long submitTime,
+      long startTime, ApplicationSubmissionContext context, String user,
+      String realUser, CallerContext callerContext) {
+    return newInstance(submitTime, startTime, user, realUser, context, null, "",
+        0, 0, callerContext);
+  }
+
   public int getAttemptCount() {
     return attempts.size();
   }
@@ -213,4 +250,8 @@ public abstract void setApplicationSubmissionContext(
   @Public
   public abstract void setApplicationTimeouts(
       Map<ApplicationTimeoutType, Long> applicationTimeouts);
+
+  public abstract String getRealUser();
+
+  public abstract void setRealUser(String realUser);
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
index a6a99ee7cb062..c0f7bb02627ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
@@ -148,7 +148,22 @@ public void setUser(String user) {
     maybeInitBuilder();
     builder.setUser(user);
   }
-  
+
+  @Override
+  public String getRealUser() {
+    ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasRealUser()) {
+      return null;
+    }
+    return (p.getRealUser());
+  }
+
+  @Override
+  public void setRealUser(String realUser) {
+    maybeInitBuilder();
+    builder.setRealUser(realUser);
+  }
+
   @Override
   public ApplicationSubmissionContext getApplicationSubmissionContext() {
     ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
index 7962d8e30f55f..910079171fa2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
@@ -95,7 +95,7 @@ protected void addReservationQueue(
     PlanQueue planQueue = (PlanQueue)queue;
     try {
       ReservationQueue resQueue =
-          new ReservationQueue(cs, currResId, planQueue);
+          new ReservationQueue(cs.getQueueContext(), currResId, planQueue);
       cs.addQueue(resQueue);
     } catch (SchedulerDynamicEditException e) {
       LOG.warn(
@@ -115,7 +115,7 @@ protected void createDefaultReservationQueue(
     if (cs.getQueue(defReservationId) == null) {
       try {
         ReservationQueue defQueue =
-            new ReservationQueue(cs, defReservationId, planQueue);
+            new ReservationQueue(cs.getQueueContext(), defReservationId, planQueue);
         cs.addQueue(defQueue);
       } catch (SchedulerDynamicEditException e) {
         LOG.warn(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java
index 3869a008d9add..e0c498661c5ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DynamicResourceConfiguration.java
@@ -32,7 +32,6 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 public class DynamicResourceConfiguration extends Configuration {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index 1f101ee3860a3..24cb34327b745 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -19,6 +19,8 @@
 package org.apache.hadoop.yarn.server.resourcemanager.resource;
 
 import org.apache.hadoop.classification.VisibleForTesting;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -29,7 +31,6 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
-import org.codehaus.jackson.map.ObjectMapper;
 
 import java.io.File;
 import java.io.IOException;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index d8323f5497b9a..949b34e8bb504 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -325,4 +325,6 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
    * @return Map of envs related to application scheduling preferences.
    */
   Map<String, String> getApplicationSchedulingEnvs();
+
+  String getRealUser();
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6e2db905a5959..ca88b8be3281c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -134,6 +134,7 @@ public class RMAppImpl implements RMApp, Recoverable {
   private final RMContext rmContext;
   private final Configuration conf;
   private final String user;
+  private final UserGroupInformation userUgi;
   private final String name;
   private final ApplicationSubmissionContext submissionContext;
   private final Dispatcher dispatcher;
@@ -421,7 +422,19 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
       String applicationType, Set<String> applicationTags,
       List<ResourceRequest> amReqs, ApplicationPlacementContext
       placementContext, long startTime) {
+    this(applicationId, rmContext, config, name,
+        (user != null ? UserGroupInformation.createRemoteUser(user) : null),
+        queue, submissionContext, scheduler, masterService, submitTime,
+        applicationType, applicationTags, amReqs, placementContext, startTime);
+  }
 
+  public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
+      Configuration config, String name, UserGroupInformation userUgi,
+      String queue, ApplicationSubmissionContext submissionContext,
+      YarnScheduler scheduler, ApplicationMasterService masterService,
+      long submitTime, String applicationType, Set<String> applicationTags,
+      List<ResourceRequest> amReqs, ApplicationPlacementContext
+      placementContext, long startTime) {
     this.systemClock = SystemClock.getInstance();
 
     this.applicationId = applicationId;
@@ -430,7 +443,9 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
     this.dispatcher = rmContext.getDispatcher();
     this.handler = dispatcher.getEventHandler();
     this.conf = config;
-    this.user = StringInterner.weakIntern(user);
+    this.user = StringInterner.weakIntern(
+        (userUgi != null) ? userUgi.getShortUserName() : null);
+    this.userUgi = userUgi;
     this.queue = queue;
     this.submissionContext = submissionContext;
     this.scheduler = scheduler;
@@ -1313,7 +1328,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppEvent event,
 
     ApplicationStateData appState =
         ApplicationStateData.newInstance(this.submitTime, this.startTime,
-            this.user, this.submissionContext,
+            this.getUser(), this.getRealUser(), this.submissionContext,
             stateToBeStored, diags, this.launchTime, this.storedFinishTime,
             this.callerContext);
     appState.setApplicationTimeouts(this.applicationTimeouts);
@@ -1902,4 +1917,10 @@ long getLogAggregationStartTime() {
   Clock getSystemClock() {
     return systemClock;
   }
+
+  @Override
+  public String getRealUser() {
+    UserGroupInformation realUserUgi = this.userUgi.getRealUser();
+    return (realUserUgi != null) ? realUserUgi.getShortUserName() : null;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index bcee51d65c330..8c6784b12bd04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -31,7 +31,6 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index e6205d2dac6a3..5d60b4fbe0678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -35,6 +35,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
 import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.util.resource.Resources;
 
 /**
  * Node managers information on available resources 
@@ -104,6 +105,17 @@ public interface RMNode {
    */
   public Resource getTotalCapability();
 
+  /**
+   * The total allocated resources to containers.
+   * This will include the sum of Guaranteed and Opportunistic
+   * containers queued + running + paused on the node.
+   * @return the total allocated resources, including all Guaranteed and
+   * Opportunistic containers in queued, running and paused states.
+   */
+  default Resource getAllocatedContainerResource() {
+    return Resources.none();
+  }
+
   /**
    * If the total available resources has been updated.
    * @return If the capability has been updated.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index e5f6f2c85bbc5..b8aaea5de330c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -128,6 +128,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
   /* Snapshot of total resources before receiving decommissioning command */
   private volatile Resource originalTotalCapability;
   private volatile Resource totalCapability;
+  private volatile Resource allocatedContainerResource =
+      Resource.newInstance(Resources.none());
   private volatile boolean updatedCapability = false;
   private final Node node;
 
@@ -464,6 +466,11 @@ public Resource getTotalCapability() {
     return this.totalCapability;
   }
 
+  @Override
+  public Resource getAllocatedContainerResource() {
+    return this.allocatedContainerResource;
+  }
+
   @Override
   public boolean isUpdatedCapability() {
     return this.updatedCapability;
@@ -952,13 +959,22 @@ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
           ClusterMetrics.getMetrics().decrDecommisionedNMs();
         }
         containers = startEvent.getNMContainerStatuses();
+        final Resource allocatedResource = Resource.newInstance(
+            Resources.none());
         if (containers != null && !containers.isEmpty()) {
           for (NMContainerStatus container : containers) {
-            if (container.getContainerState() == ContainerState.RUNNING) {
-              rmNode.launchedContainers.add(container.getContainerId());
+            if (container.getContainerState() == ContainerState.NEW ||
+                container.getContainerState() == ContainerState.RUNNING) {
+              Resources.addTo(allocatedResource,
+                  container.getAllocatedResource());
+              if (container.getContainerState() == ContainerState.RUNNING) {
+                rmNode.launchedContainers.add(container.getContainerId());
+              }
             }
           }
         }
+
+        rmNode.allocatedContainerResource = allocatedResource;
       }
 
       if (null != startEvent.getRunningApplications()) {
@@ -1554,6 +1570,8 @@ private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
     List<Map.Entry<ApplicationId, ContainerStatus>> needUpdateContainers =
         new ArrayList<Map.Entry<ApplicationId, ContainerStatus>>();
     int numRemoteRunningContainers = 0;
+    final Resource allocatedResource = Resource.newInstance(Resources.none());
+
     for (ContainerStatus remoteContainer : containerStatuses) {
       ContainerId containerId = remoteContainer.getContainerId();
 
@@ -1622,8 +1640,16 @@ private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
         containerAllocationExpirer
             .unregister(new AllocationExpirationInfo(containerId));
       }
+
+      if ((remoteContainer.getState() == ContainerState.RUNNING ||
+          remoteContainer.getState() == ContainerState.NEW) &&
+          remoteContainer.getCapability() != null) {
+        Resources.addTo(allocatedResource, remoteContainer.getCapability());
+      }
     }
 
+    allocatedContainerResource = allocatedResource;
+
     List<ContainerStatus> lostContainers =
         findLostContainers(numRemoteRunningContainers, containerStatuses);
     for (ContainerStatus remoteContainer : lostContainers) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
index b8f8e734f9b19..6bbe39e89fc12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
 
-import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.yarn.api.records.Container;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
index ad3bfb18ef3b8..b95f192febe53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReferenceArray;
@@ -200,10 +201,10 @@ public String toString() {
     }
   }
 
-  public Set<String> getNodePartitionsSet() {
+  public Set<String> getExistingNodeLabels() {
     readLock.lock();
     try {
-      return usages.keySet();
+      return new HashSet<>(usages.keySet());
     } finally {
       readLock.unlock();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 7ce7bfc96dc55..8772ddb44e6b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -584,8 +584,14 @@ public void recoverContainersOnNode(List<NMContainerStatus> containerReports,
             rmContainer);
 
         // recover scheduler attempt
-        schedulerAttempt.recoverContainer(schedulerNode, rmContainer);
+        final boolean recovered = schedulerAttempt.recoverContainer(
+            schedulerNode, rmContainer);
 
+        if (recovered && rmContainer.getExecutionType() ==
+            ExecutionType.OPPORTUNISTIC) {
+          OpportunisticSchedulerMetrics.getMetrics()
+              .incrAllocatedOppContainers(1);
+        }
         // set master container for the current running AMContainer for this
         // attempt.
         RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
@@ -720,7 +726,10 @@ public void completedContainer(RMContainer rmContainer,
       SchedulerApplicationAttempt schedulerAttempt =
           getCurrentAttemptForContainer(containerId);
       if (schedulerAttempt != null) {
-        schedulerAttempt.removeRMContainer(containerId);
+        if (schedulerAttempt.removeRMContainer(containerId)) {
+          OpportunisticSchedulerMetrics.getMetrics()
+              .incrReleasedOppContainers(1);
+        }
       }
       LOG.debug("Completed container: {} in state: {} event:{}",
           rmContainer.getContainerId(), rmContainer.getState(), event);
@@ -729,7 +738,6 @@ public void completedContainer(RMContainer rmContainer,
       if (node != null) {
         node.releaseContainer(rmContainer.getContainerId(), false);
       }
-      OpportunisticSchedulerMetrics.getMetrics().incrReleasedOppContainers(1);
     }
 
     // If the container is getting killed in ACQUIRED state, the requester (AM
@@ -1411,6 +1419,8 @@ private void handleDecreaseRequests(SchedulerApplicationAttempt appAttempt,
             RMContainer demotedRMContainer =
                 createDemotedRMContainer(appAttempt, oppCntxt, rmContainer);
             if (demotedRMContainer != null) {
+              OpportunisticSchedulerMetrics.getMetrics()
+                  .incrAllocatedOppContainers(1);
               appAttempt.addToNewlyDemotedContainers(
                       uReq.getContainerId(), demotedRMContainer);
             }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 2b2f8324655be..203709b4cc05b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -33,6 +33,7 @@
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -105,6 +106,8 @@ public class AppSchedulingInfo {
   private final int retryAttempts;
   private boolean unmanagedAM;
 
+  private final String defaultResourceRequestAppPlacementType;
+
   public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user,
       Queue queue, AbstractUsersManager abstractUsersManager, long epoch,
       ResourceUsage appResourceUsage,
@@ -129,6 +132,31 @@ public AppSchedulingInfo(ApplicationAttemptId appAttemptId, String user,
     updateContext = new ContainerUpdateContext(this);
     readLock = lock.readLock();
     writeLock = lock.writeLock();
+
+    this.defaultResourceRequestAppPlacementType =
+        getDefaultResourceRequestAppPlacementType();
+  }
+
+  /**
+   * Set default App Placement Allocator.
+   *
+   * @return app placement class.
+   */
+  public String getDefaultResourceRequestAppPlacementType() {
+    if (this.rmContext != null
+        && this.rmContext.getYarnConfiguration() != null) {
+
+      String appPlacementClass = applicationSchedulingEnvs.get(
+          ApplicationSchedulingConfig.ENV_APPLICATION_PLACEMENT_TYPE_CLASS);
+      if (null != appPlacementClass) {
+        return appPlacementClass;
+      } else {
+        Configuration conf = rmContext.getYarnConfiguration();
+        return conf.get(
+            YarnConfiguration.APPLICATION_PLACEMENT_TYPE_CLASS);
+      }
+    }
+    return null;
   }
 
   public ApplicationId getApplicationId() {
@@ -331,8 +359,7 @@ private boolean internalAddResourceRequests(
       SchedulerRequestKey schedulerRequestKey = entry.getKey();
       AppPlacementAllocator<SchedulerNode> appPlacementAllocator =
           getAndAddAppPlacementAllocatorIfNotExist(schedulerRequestKey,
-              applicationSchedulingEnvs.get(
-                  ApplicationSchedulingConfig.ENV_APPLICATION_PLACEMENT_TYPE_CLASS));
+              defaultResourceRequestAppPlacementType);
 
       // Update AppPlacementAllocator
       PendingAskUpdateResult pendingAmountChanges =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementAllocatorFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementAllocatorFactory.java
index a4e5484d81009..71d657debee69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementAllocatorFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ApplicationPlacementAllocatorFactory.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -45,7 +46,7 @@ public static AppPlacementAllocator<SchedulerNode> getAppPlacementAllocator(
       SchedulerRequestKey schedulerRequestKey, RMContext rmContext) {
     Class<?> policyClass;
     try {
-      if (appPlacementAllocatorName == null) {
+      if (StringUtils.isEmpty(appPlacementAllocatorName)) {
         policyClass = ApplicationSchedulingConfig.DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS;
       } else {
         policyClass = Class.forName(appPlacementAllocatorName);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 965a7293ad37f..40b4ae39467a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
index f43131809a0ca..150abdf51d06d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java
@@ -40,6 +40,7 @@ protected PartitionQueueMetrics(MetricsSystem ms, String queueName,
       String parentMetricName =
           partition + METRIC_NAME_DELIMITER + newQueueName;
       setParent(getQueueMetrics().get(parentMetricName));
+      storedPartitionMetrics = null;
     }
   }
 
@@ -77,7 +78,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) {
       metrics = new PartitionQueueMetrics(this.metricsSystem, this.queueName,
           null, false, this.conf, this.partition);
       users.put(userName, metrics);
-      metricsSystem.register(
+      registerMetrics(
           pSourceName(partitionJMXStr).append(qSourceName(queueName))
               .append(",user=").append(userName).toString(),
           "Metrics for user '" + userName + "' in queue '" + queueName + "'",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 6f9b1ab47b2b1..0bfee4d33500e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -40,6 +41,7 @@
 import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
 import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
 import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -133,7 +135,7 @@ public class QueueMetrics implements MetricsSource {
   protected final MetricsRegistry registry;
   protected final String queueName;
   private QueueMetrics parent;
-  private final Queue parentQueue;
+  private Queue parentQueue;
   protected final MetricsSystem metricsSystem;
   protected final Map<String, QueueMetrics> users;
   protected final Configuration conf;
@@ -177,6 +179,7 @@ public class QueueMetrics implements MetricsSource {
     "AggregatePreemptedSeconds.";
   private static final String AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC =
     "Aggregate Preempted Seconds for NAME";
+  protected Set<String> storedPartitionMetrics = Sets.newConcurrentHashSet();
 
   public QueueMetrics(MetricsSystem ms, String queueName, Queue parent,
       boolean enableUserMetrics, Configuration conf) {
@@ -286,7 +289,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) {
       metrics =
           new QueueMetrics(metricsSystem, queueName, null, false, conf);
       users.put(userName, metrics);
-      metricsSystem.register(
+      registerMetrics(
           sourceName(queueName).append(",user=").append(userName).toString(),
           "Metrics for user '"+ userName +"' in queue '"+ queueName +"'",
           metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName));
@@ -331,13 +334,14 @@ public synchronized QueueMetrics getPartitionQueueMetrics(String partition) {
       QueueMetrics queueMetrics =
           new PartitionQueueMetrics(metricsSystem, this.queueName, parentQueue,
               this.enableUserMetrics, this.conf, partition);
-      metricsSystem.register(
+      registerMetrics(
           pSourceName(partitionJMXStr).append(qSourceName(this.queueName))
               .toString(),
           "Metrics for queue: " + this.queueName,
           queueMetrics.tag(PARTITION_INFO, partitionJMXStr).tag(QUEUE_INFO,
               this.queueName));
       getQueueMetrics().put(metricName, queueMetrics);
+      registerPartitionMetricsCreation(metricName);
       return queueMetrics;
     } else {
       return metrics;
@@ -374,12 +378,13 @@ private QueueMetrics getPartitionMetrics(String partition) {
 
       // Register with the MetricsSystems
       if (metricsSystem != null) {
-        metricsSystem.register(pSourceName(partitionJMXStr).toString(),
+        registerMetrics(pSourceName(partitionJMXStr).toString(),
             "Metrics for partition: " + partitionJMXStr,
             (PartitionQueueMetrics) metrics.tag(PARTITION_INFO,
                 partitionJMXStr));
       }
       getQueueMetrics().put(metricName, metrics);
+      registerPartitionMetricsCreation(metricName);
     }
     return metrics;
   }
@@ -1332,4 +1337,37 @@ public void setParent(QueueMetrics parent) {
   public Queue getParentQueue() {
     return parentQueue;
   }
+
+  protected void registerPartitionMetricsCreation(String metricName) {
+    if (storedPartitionMetrics != null) {
+      storedPartitionMetrics.add(metricName);
+    }
+  }
+
+  public void setParentQueue(Queue parentQueue) {
+    this.parentQueue = parentQueue;
+
+    if (storedPartitionMetrics == null) {
+      return;
+    }
+
+    for (String partitionMetric : storedPartitionMetrics) {
+      QueueMetrics metric = getQueueMetrics().get(partitionMetric);
+
+      if (metric != null && metric.parentQueue != null) {
+        metric.parentQueue = parentQueue;
+      }
+    }
+  }
+
+  protected void registerMetrics(String sourceName, String desc, QueueMetrics metrics) {
+    MetricsSource source = metricsSystem.getSource(sourceName);
+    // Unregister metrics if a source is already present
+    if (source != null) {
+      LOG.info("Unregistering source " + sourceName);
+      metricsSystem.unregisterSource(sourceName);
+    }
+
+    metricsSystem.register(sourceName, desc, metrics);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
index 0127e1ecaae7a..ab0f445275ee1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
@@ -77,7 +77,7 @@ public synchronized void activateQueue(String queueName)
       throw new YarnException("The specified queue:" + queueName
           + " does not exist!");
     }
-    queue.activeQueue();
+    queue.activateQueue();
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 484ae0b94a5b8..2c84d399242a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -91,7 +91,7 @@
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ConcurrentHashMultiset;
 
 /**
@@ -402,7 +402,13 @@ public void addRMContainer(
     }
   }
 
-  public void removeRMContainer(ContainerId containerId) {
+  /**
+   * Removes an RM container from the map of live containers
+   * related to this application attempt.
+   * @param containerId The container ID of the RMContainer to remove
+   * @return true if the container is in the map
+   */
+  public boolean removeRMContainer(ContainerId containerId) {
     writeLock.lock();
     try {
       RMContainer rmContainer = liveContainers.remove(containerId);
@@ -415,7 +421,11 @@ public void removeRMContainer(ContainerId containerId) {
           this.attemptResourceUsageAllocatedRemotely
               .decUsed(rmContainer.getAllocatedResource());
         }
+
+        return true;
       }
+
+      return false;
     } finally {
       writeLock.unlock();
     }
@@ -646,7 +656,7 @@ public int getNumReservedContainers(
   @SuppressWarnings("unchecked")
   public void containerLaunchedOnNode(ContainerId containerId,
       NodeId nodeId) {
-    writeLock.lock();
+    readLock.lock();
     try {
       // Inform the container
       RMContainer rmContainer = getRMContainer(containerId);
@@ -660,7 +670,7 @@ public void containerLaunchedOnNode(ContainerId containerId,
       rmContainer.handle(
           new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
     } finally {
-      writeLock.unlock();
+      readLock.unlock();
     }
   }
   
@@ -1226,7 +1236,7 @@ public void move(Queue newQueue) {
     }
   }
 
-  public void recoverContainer(SchedulerNode node,
+  public boolean recoverContainer(SchedulerNode node,
       RMContainer rmContainer) {
     writeLock.lock();
     try {
@@ -1234,7 +1244,7 @@ public void recoverContainer(SchedulerNode node,
       appSchedulingInfo.recoverContainer(rmContainer, node.getPartition());
 
       if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
-        return;
+        return false;
       }
       LOG.info("SchedulerAttempt " + getApplicationAttemptId()
           + " is recovering container " + rmContainer.getContainerId());
@@ -1244,6 +1254,8 @@ public void recoverContainer(SchedulerNode node,
             rmContainer.getContainer().getResource());
       }
 
+      return true;
+
       // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
       // is called.
       // newlyAllocatedContainers.add(rmContainer);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java
index 9a67e0188aeae..33037fc43fba2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java
@@ -62,8 +62,8 @@ public interface SchedulerQueue<T extends SchedulerQueue> extends Queue {
   void stopQueue();
 
   /**
-   * Active the queue.
+   * Activate the queue.
    * @throws YarnException if the queue can not be activated.
    */
-  void activeQueue() throws YarnException;
+  void activateQueue() throws YarnException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java
index 6285cc941e92b..01a18db786f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesManager.java
@@ -50,10 +50,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.List;
-import java.util.Set;
 import java.util.*;
-import java.util.ArrayList;
 import java.util.stream.Collectors;
 
 /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
index 8d7733453f8eb..15960c8de8494 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
@@ -34,24 +34,16 @@
  * Abstract class for dynamic auto created queues managed by an implementation
  * of AbstractManagedParentQueue
  */
-public class AbstractAutoCreatedLeafQueue extends LeafQueue {
+public class AbstractAutoCreatedLeafQueue extends AbstractLeafQueue {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbstractAutoCreatedLeafQueue.class);
 
   protected AbstractManagedParentQueue parent;
 
-  public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs,
+  public AbstractAutoCreatedLeafQueue(CapacitySchedulerQueueContext queueContext,
       String queueName, AbstractManagedParentQueue parent, CSQueue old)
       throws IOException {
-    super(cs, queueName, parent, old);
-    this.parent = parent;
-  }
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      AbstractAutoCreatedLeafQueue.class);
-
-  public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration leafQueueConfigs, String queueName,
-      AbstractManagedParentQueue parent, CSQueue old) throws IOException {
-    super(cs, leafQueueConfigs, queueName, parent, old);
+    super(queueContext, queueName, parent, old);
     this.parent = parent;
   }
 
@@ -71,7 +63,7 @@ public void setEntitlement(QueueEntitlement entitlement)
   @Override
   protected Resource getMinimumAbsoluteResource(String queuePath,
       String label) {
-    return super.getMinimumAbsoluteResource(csContext.getConfiguration()
+    return super.getMinimumAbsoluteResource(queueContext.getConfiguration()
         .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
         label);
   }
@@ -79,7 +71,7 @@ protected Resource getMinimumAbsoluteResource(String queuePath,
   @Override
   protected Resource getMaximumAbsoluteResource(String queuePath,
       String label) {
-    return super.getMaximumAbsoluteResource(csContext.getConfiguration()
+    return super.getMaximumAbsoluteResource(queueContext.getConfiguration()
         .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
         label);
   }
@@ -87,7 +79,7 @@ protected Resource getMaximumAbsoluteResource(String queuePath,
   @Override
   protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
       String label) {
-    return super.checkConfigTypeIsAbsoluteResource(csContext.getConfiguration()
+    return super.checkConfigTypeIsAbsoluteResource(queueContext.getConfiguration()
         .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
         label);
   }
@@ -111,7 +103,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement)
       }
       setCapacity(nodeLabel, capacity);
       setAbsoluteCapacity(nodeLabel,
-          getParent().getQueueCapacities().
+          this.getParent().getQueueCapacities().
               getAbsoluteCapacity(nodeLabel)
               * getQueueCapacities().getCapacity(nodeLabel));
       // note: we currently set maxCapacity to capacity
@@ -122,7 +114,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement)
 
       //update queue used capacity etc
       CSQueueUtils.updateQueueStatistics(resourceCalculator,
-          csContext.getClusterResource(),
+          queueContext.getClusterResource(),
           this, labelManager, nodeLabel);
     } finally {
       writeLock.unlock();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 5c520b783b943..7a3ec4f6f4e5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.AccessControlException;
@@ -30,14 +29,11 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueConfigurations;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueState;
-import org.apache.hadoop.yarn.api.records.QueueStatistics;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.AccessRequest;
@@ -76,69 +72,53 @@
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED;
 
+/**
+ * Provides implementation of {@code CSQueue} methods common for every queue class in Capacity
+ * Scheduler.
+ */
 public abstract class AbstractCSQueue implements CSQueue {
-
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractCSQueue.class);
+  protected final QueueAllocationSettings queueAllocationSettings;
   volatile CSQueue parent;
-  final String queueName;
-  private final String queuePath;
-  volatile int numContainers;
+  protected final QueuePath queuePath;
+  protected QueueNodeLabelsSettings queueNodeLabelsSettings;
+  private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings;
+  private CSQueuePreemptionSettings preemptionSettings;
 
-  final Resource minimumAllocation;
-  volatile Resource maximumAllocation;
   private volatile QueueState state = null;
-  final CSQueueMetrics metrics;
   protected final PrivilegedEntity queueEntity;
 
   final ResourceCalculator resourceCalculator;
-  Set<String> accessibleLabels;
-  protected Set<String> configuredNodeLabels;
   Set<String> resourceTypes;
   final RMNodeLabelsManager labelManager;
-  String defaultLabelExpression;
   private String multiNodeSortingPolicyName = null;
 
   Map<AccessType, AccessControlList> acls =
       new HashMap<AccessType, AccessControlList>();
   volatile boolean reservationsContinueLooking;
 
-  // Track resource usage-by-label like used-resource/pending-resource, etc.
-  volatile ResourceUsage queueUsage;
-
   // Track capacities like
   // used-capacity/abs-used-capacity/capacity/abs-capacity,
   // etc.
   QueueCapacities queueCapacities;
-
-  QueueResourceQuotas queueResourceQuotas;
-
-  // -1 indicates lifetime is disabled
-  private volatile long maxApplicationLifetime = -1;
-
-  private volatile long defaultApplicationLifetime = -1;
-
-  // Indicates if this queue's default lifetime was set by a config property,
-  // either at this level or anywhere in the queue's hierarchy.
-  private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
-  private CSQueuePreemption preemptionSettings;
+  CSQueueUsageTracker usageTracker;
 
   public enum CapacityConfigType {
-    // FIXME, from what I can see, Percentage mode can almost apply to weighted
-    // and percentage mode at the same time, there's only small area need to be
-    // changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT"
     NONE, PERCENTAGE, ABSOLUTE_RESOURCE
   };
+
   protected CapacityConfigType capacityConfigType =
       CapacityConfigType.NONE;
 
+  protected Map<String, QueueCapacityVector> configuredCapacityVectors;
+
   private final RecordFactory recordFactory =
       RecordFactoryProvider.getRecordFactory(null);
-  protected CapacitySchedulerContext csContext;
+  protected CapacitySchedulerQueueContext queueContext;
   protected YarnAuthorizationProvider authorizer = null;
 
   protected ActivitiesManager activitiesManager;
@@ -148,70 +128,69 @@ public enum CapacityConfigType {
 
   volatile Priority priority = Priority.newInstance(0);
   private UserWeights userWeights = UserWeights.createEmpty();
-  private int maxParallelApps;
 
   // is it a dynamic queue?
   private boolean dynamicQueue = false;
 
-  // The timestamp of the last submitted application to this queue.
-  // Only applies to dynamic queues.
-  private long lastSubmittedTimestamp;
-
-  public AbstractCSQueue(CapacitySchedulerContext cs,
-      String queueName, CSQueue parent, CSQueue old) throws IOException {
-    this(cs, cs.getConfiguration(), queueName, parent, old);
-  }
-
-  public AbstractCSQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration configuration, String queueName,
+  public AbstractCSQueue(CapacitySchedulerQueueContext queueContext, String queueName,
       CSQueue parent, CSQueue old) {
-
-    this.labelManager = cs.getRMContext().getNodeLabelManager();
     this.parent = parent;
-    this.queueName = queueName;
-    this.queuePath = ((parent == null) ? "" : (parent.getQueuePath() + "."))
-        + this.queueName;
-    this.resourceCalculator = cs.getResourceCalculator();
-    this.activitiesManager = cs.getActivitiesManager();
+    this.queuePath = createQueuePath(parent, queueName);
+
+    this.queueContext = queueContext;
+    this.resourceCalculator = queueContext.getResourceCalculator();
+    this.activitiesManager = queueContext.getActivitiesManager();
+    this.labelManager = queueContext.getLabelManager();
 
     // must be called after parent and queueName is set
-    this.metrics = old != null ?
+    CSQueueMetrics metrics = old != null ?
         (CSQueueMetrics) old.getMetrics() :
         CSQueueMetrics.forQueue(getQueuePath(), parent,
-            cs.getConfiguration().getEnableUserMetrics(), configuration);
-
-    this.csContext = cs;
-    this.minimumAllocation = csContext.getMinimumResourceCapability();
+            queueContext.getConfiguration().getEnableUserMetrics(),
+            queueContext.getConfiguration());
+    this.usageTracker = new CSQueueUsageTracker(metrics);
 
-    // initialize ResourceUsage
-    queueUsage = new ResourceUsage();
-    queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
+    this.queueCapacities = new QueueCapacities(parent == null);
+    this.queueAllocationSettings = new QueueAllocationSettings(queueContext.getMinimumAllocation());
 
-    // initialize QueueCapacities
-    queueCapacities = new QueueCapacities(parent == null);
+    this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
 
-    // initialize queueResourceQuotas
-    queueResourceQuotas = new QueueResourceQuotas();
+    this.resourceTypes = new HashSet<>();
+    for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
+      this.resourceTypes.add(type.toString().toLowerCase());
+    }
 
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-    readLock = lock.readLock();
-    writeLock = lock.writeLock();
+    this.readLock = lock.readLock();
+    this.writeLock = lock.writeLock();
+
+    LOG.debug("Initialized {}: name={}, fullname={}", this.getClass().getSimpleName(),
+        queueName, getQueuePath());
   }
 
-  @VisibleForTesting
-  protected void setupConfigurableCapacities() {
-    setupConfigurableCapacities(csContext.getConfiguration());
+  private static QueuePath createQueuePath(CSQueue parent, String queueName) {
+    if (parent == null) {
+      return new QueuePath(null, queueName);
+    }
+    return new QueuePath(parent.getQueuePath(), queueName);
   }
 
-  protected void setupConfigurableCapacities(
-      CapacitySchedulerConfiguration configuration) {
-    CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities,
-        configuration, configuredNodeLabels);
+  /**
+   * Sets up capacity and weight values from configuration.
+   */
+  protected void setupConfigurableCapacities() {
+    CSQueueUtils.loadCapacitiesByLabelsFromConf(queuePath, queueCapacities,
+        queueContext.getConfiguration(), this.queueNodeLabelsSettings.getConfiguredNodeLabels());
   }
 
   @Override
   public String getQueuePath() {
-    return queuePath;
+    return queuePath.getFullPath();
+  }
+
+  @Override
+  public QueuePath getQueuePathObject() {
+    return this.queuePath;
   }
 
   @Override
@@ -246,11 +225,11 @@ public float getUsedCapacity() {
 
   @Override
   public Resource getUsedResources() {
-    return queueUsage.getUsed();
+    return usageTracker.getQueueUsage().getUsed();
   }
 
   public int getNumContainers() {
-    return numContainers;
+    return usageTracker.getNumContainers();
   }
 
   @Override
@@ -260,22 +239,17 @@ public QueueState getState() {
 
   @Override
   public CSQueueMetrics getMetrics() {
-    return metrics;
+    return usageTracker.getMetrics();
   }
 
   @Override
   public String getQueueShortName() {
-    return queueName;
+    return queuePath.getLeafName();
   }
 
   @Override
   public String getQueueName() {
-    return queueName;
-  }
-
-  @Override
-  public PrivilegedEntity getPrivilegedEntity() {
-    return queueEntity;
+    return this.queuePath.getLeafName();
   }
 
   @Override
@@ -286,12 +260,28 @@ public CSQueue getParent() {
   @Override
   public void setParent(CSQueue newParentQueue) {
     this.parent = newParentQueue;
+    getMetrics().setParentQueue(newParentQueue);
+  }
+
+  @Override
+  public PrivilegedEntity getPrivilegedEntity() {
+    return queueEntity;
+  }
+
+  public CapacitySchedulerQueueContext getQueueContext() {
+    return queueContext;
   }
 
   public Set<String> getAccessibleNodeLabels() {
-    return accessibleLabels;
+    return queueNodeLabelsSettings.getAccessibleNodeLabels();
   }
 
+  /**
+   * Checks whether the user has the required permission to execute the action of {@code QueueACL}.
+   * @param acl the access type the user is checked for
+   * @param user UGI of the user
+   * @return true, if the user has permission, false otherwise
+   */
   @Override
   public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
     return authorizer.checkPermission(
@@ -300,40 +290,31 @@ public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
   }
 
   /**
-   * Set maximum capacity - used only for testing.
+   * Set maximum capacity for empty node label.
    * @param maximumCapacity new max capacity
    */
+  @VisibleForTesting
   void setMaxCapacity(float maximumCapacity) {
-    writeLock.lock();
-    try {
-      // Sanity check
-      CSQueueUtils.checkMaxCapacity(getQueuePath(),
-          queueCapacities.getCapacity(), maximumCapacity);
-      float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
-          maximumCapacity, parent);
-      CSQueueUtils.checkAbsoluteCapacity(getQueuePath(),
-          queueCapacities.getAbsoluteCapacity(), absMaxCapacity);
-
-      queueCapacities.setMaximumCapacity(maximumCapacity);
-      queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
-    } finally {
-      writeLock.unlock();
-    }
+    internalSetMaximumCapacity(maximumCapacity, NO_LABEL);
   }
 
   /**
-   * Set maximum capacity
+   * Set maximum capacity.
    * @param maximumCapacity new max capacity
    */
   void setMaxCapacity(String nodeLabel, float maximumCapacity) {
+    internalSetMaximumCapacity(maximumCapacity, nodeLabel);
+  }
+
+  private void internalSetMaximumCapacity(float maximumCapacity, String nodeLabel) {
     writeLock.lock();
     try {
       // Sanity check
-      CSQueueUtils.checkMaxCapacity(getQueuePath(),
+      CSQueueUtils.checkMaxCapacity(this.queuePath,
           queueCapacities.getCapacity(nodeLabel), maximumCapacity);
       float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
           maximumCapacity, parent);
-      CSQueueUtils.checkAbsoluteCapacity(getQueuePath(),
+      CSQueueUtils.checkAbsoluteCapacity(this.queuePath,
           queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity);
 
       queueCapacities.setMaximumCapacity(maximumCapacity);
@@ -343,62 +324,68 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) {
     }
   }
 
-
   @Override
   public String getDefaultNodeLabelExpression() {
-    return defaultLabelExpression;
+    return this.queueNodeLabelsSettings.getDefaultLabelExpression();
   }
 
-  protected void setupQueueConfigs(Resource clusterResource,
-      CapacitySchedulerConfiguration configuration) throws
+  /**
+   * Initialize queue properties that are based on configuration.
+   * @param clusterResource overall resource of the cluster
+   * @throws IOException if configuration is set in a way that is inconsistent
+   */
+  protected void setupQueueConfigs(Resource clusterResource) throws
       IOException {
 
     writeLock.lock();
     try {
+      CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
+      this.acls = configuration.getAcls(getQueuePath());
+
       if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
-        setDynamicQueueProperties(configuration);
+        setDynamicQueueProperties();
+        setDynamicQueueACLProperties();
       }
 
       // Collect and set the Node label configuration
-      initializeNodeLabels(configuration);
+      this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
+          queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
 
       // Initialize the queue capacities
-      setupConfigurableCapacities(configuration);
+      setupConfigurableCapacities();
       updateAbsoluteCapacities();
-
       updateCapacityConfigType();
 
       // Fetch minimum/maximum resource limits for this queue if
       // configured
-      this.resourceTypes = new HashSet<>();
-      for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
-        resourceTypes.add(type.toString().toLowerCase());
-      }
       updateConfigurableResourceLimits(clusterResource);
 
       // Setup queue's maximumAllocation respecting the global
       // and the queue settings
-      setupMaximumAllocation(configuration);
+      this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(),
+          parent);
 
       // Initialize the queue state based on previous state, configured state
       // and its parent state
-      initializeQueueState(configuration);
+      QueueStateHelper.setQueueState(this);
 
       authorizer = YarnAuthorizationProvider.getInstance(configuration);
 
-      this.acls = configuration.getAcls(getQueuePath());
-
-      this.userWeights = getUserWeightsFromHierarchy(configuration);
+      this.userWeights = getUserWeightsFromHierarchy();
 
       this.reservationsContinueLooking =
           configuration.getReservationContinueLook();
 
+      this.configuredCapacityVectors = configuration
+          .parseConfiguredResourceVector(queuePath.getFullPath(),
+              this.queueNodeLabelsSettings.getConfiguredNodeLabels());
+
       // Update metrics
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
           this, labelManager, null);
 
       // Store preemption settings
-      this.preemptionSettings = new CSQueuePreemption(this, csContext, configuration);
+      this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration);
       this.priority = configuration.getQueuePriority(
           getQueuePath());
 
@@ -407,7 +394,8 @@ protected void setupQueueConfigs(Resource clusterResource,
           configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath()));
 
       // Setup application related limits
-      setupApplicationLimits(configuration);
+      this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
+          this, queuePath);
     } finally {
       writeLock.unlock();
     }
@@ -415,162 +403,35 @@ protected void setupQueueConfigs(Resource clusterResource,
 
   /**
    * Set properties specific to dynamic queues.
-   * @param configuration configuration on which the properties are set
    */
-  protected void setDynamicQueueProperties(
-      CapacitySchedulerConfiguration configuration) {
+  protected void setDynamicQueueProperties() {
     // Set properties from parent template
-    if (getParent() instanceof ParentQueue) {
-      ((ParentQueue) getParent()).getAutoCreatedQueueTemplate()
-          .setTemplateEntriesForChild(configuration, getQueuePath());
+    if (parent instanceof ParentQueue) {
+      ((ParentQueue) parent).getAutoCreatedQueueTemplate()
+          .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath());
 
-      String parentTemplate = String.format("%s.%s", getParent().getQueuePath(),
+      String parentTemplate = String.format("%s.%s", parent.getQueuePath(),
           AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX);
       parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf(
           DOT));
-      Set<String> parentNodeLabels = csContext
-          .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+      Set<String> parentNodeLabels = queueContext.getQueueManager()
+          .getConfiguredNodeLabelsForAllQueues()
           .getLabelsByQueue(parentTemplate);
 
       if (parentNodeLabels != null && parentNodeLabels.size() > 1) {
-        csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
-            .setLabelsByQueue(queuePath, new HashSet<>(parentNodeLabels));
+        queueContext.getQueueManager()
+            .getConfiguredNodeLabelsForAllQueues()
+            .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels));
       }
     }
   }
 
-  private void initializeNodeLabels(
-      CapacitySchedulerConfiguration configuration) throws IOException {
-    // Collect and store labels
-    this.accessibleLabels =
-        configuration.getAccessibleNodeLabels(getQueuePath());
-    this.defaultLabelExpression =
-        configuration.getDefaultNodeLabelExpression(
-            getQueuePath());
-
-    // Inherit labels from parent if not set
-    if (this.accessibleLabels == null && parent != null) {
-      this.accessibleLabels = parent.getAccessibleNodeLabels();
-    }
-
-    // If the accessible labels is not null and the queue has a parent with a
-    // similar set of labels copy the defaultNodeLabelExpression from the parent
-    if (this.accessibleLabels != null && parent != null
-        && this.defaultLabelExpression == null &&
-        this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
-      this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
-    }
-
-    if (csContext.getCapacitySchedulerQueueManager() != null
-        && csContext.getCapacitySchedulerQueueManager()
-        .getConfiguredNodeLabels() != null) {
-      if (getQueuePath().equals(ROOT)) {
-        this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager()
-            .getConfiguredNodeLabels().getAllConfiguredLabels();
-      } else {
-        this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager()
-            .getConfiguredNodeLabels().getLabelsByQueue(getQueuePath());
-      }
-    } else {
-      // Fallback to suboptimal but correct logic
-      this.configuredNodeLabels = csContext.getConfiguration()
-          .getConfiguredNodeLabels(queuePath);
-    }
-
-    // Validate the initialized settings
-    validateNodeLabels();
-  }
-
-  private void validateNodeLabels() throws IOException {
-    // Check if labels of this queue is a subset of parent queue, only do this
-    // when the queue in question is not root
-    if (parent != null && parent.getParent() != null) {
-      if (parent.getAccessibleNodeLabels() != null && !parent
-          .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
-        // if parent isn't "*", child shouldn't be "*" too
-        if (this.getAccessibleNodeLabels().contains(
-            RMNodeLabelsManager.ANY)) {
-          throw new IOException("Parent's accessible queue is not ANY(*), "
-              + "but child's accessible queue is *");
-        } else{
-          Set<String> diff = Sets.difference(this.getAccessibleNodeLabels(),
-              parent.getAccessibleNodeLabels());
-          if (!diff.isEmpty()) {
-            throw new IOException(
-                "Some labels of child queue is not a subset "
-                    + "of parent queue, these labels=[" + StringUtils
-                    .join(diff, ",") + "]");
-          }
-        }
-      }
-    }
-  }
-
-  private void setupApplicationLimits(CapacitySchedulerConfiguration configuration) {
-    // Store max parallel apps property
-    this.maxParallelApps = configuration.getMaxParallelAppsForQueue(getQueuePath());
-
-    maxApplicationLifetime = getInheritedMaxAppLifetime(this, configuration);
-    defaultApplicationLifetime =
-        getInheritedDefaultAppLifetime(this, configuration,
-            maxApplicationLifetime);
-  }
-
-  private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) {
-    String myQueuePath = getQueuePath();
-    /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration
-    * object is a cloned one containing only the template configs
-    * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual
-    * cluster maximum allocation is fetched the original config object should
-    * be used.
-    */
-    Resource clusterMax = ResourceUtils
-        .fetchMaximumAllocationFromConfig(this.csContext.getConfiguration());
-    Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath);
-
-    maximumAllocation = Resources.clone(
-        parent == null ? clusterMax : parent.getMaximumAllocation());
-
-    String errMsg =
-        "Queue maximum allocation cannot be larger than the cluster setting"
-            + " for queue " + myQueuePath
-            + " max allocation per queue: %s"
-            + " cluster setting: " + clusterMax;
-
-    if (queueMax == Resources.none()) {
-      // Handle backward compatibility
-      long queueMemory = csConf.getQueueMaximumAllocationMb(myQueuePath);
-      int queueVcores = csConf.getQueueMaximumAllocationVcores(myQueuePath);
-      if (queueMemory != UNDEFINED) {
-        maximumAllocation.setMemorySize(queueMemory);
-      }
-
-      if (queueVcores != UNDEFINED) {
-        maximumAllocation.setVirtualCores(queueVcores);
-      }
-
-      if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize()
-          || (queueVcores != UNDEFINED
-          && queueVcores > clusterMax.getVirtualCores()))) {
-        throw new IllegalArgumentException(
-            String.format(errMsg, maximumAllocation));
-      }
-    } else {
-      // Queue level maximum-allocation can't be larger than cluster setting
-      for (ResourceInformation ri : queueMax.getResources()) {
-        if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) {
-          throw new IllegalArgumentException(String.format(errMsg, queueMax));
-        }
-
-        maximumAllocation.setResourceInformation(ri.getName(), ri);
-      }
-    }
+  protected void setDynamicQueueACLProperties() {
   }
 
-  private UserWeights getUserWeightsFromHierarchy(
-      CapacitySchedulerConfiguration configuration) {
+  private UserWeights getUserWeightsFromHierarchy() {
     UserWeights unionInheritedWeights = UserWeights.createEmpty();
-    CSQueue parentQ = getParent();
+    CSQueue parentQ = parent;
     if (parentQ != null) {
       // Inherit all of parent's userWeights
       unionInheritedWeights.addFrom(parentQ.getUserWeights());
@@ -578,36 +439,34 @@ private UserWeights getUserWeightsFromHierarchy(
     // Insert this queue's userWeights, overriding parent's userWeights if
     // there is an overlap.
     unionInheritedWeights.addFrom(
-        configuration.getAllUserWeightsForQueue(getQueuePath()));
+        queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePath()));
     return unionInheritedWeights;
   }
 
   protected Resource getMinimumAbsoluteResource(String queuePath, String label) {
-    Resource minResource = csContext.getConfiguration()
+    return queueContext.getConfiguration()
         .getMinimumResourceRequirement(label, queuePath, resourceTypes);
-    return minResource;
   }
 
   protected Resource getMaximumAbsoluteResource(String queuePath, String label) {
-    Resource maxResource = csContext.getConfiguration()
+    return queueContext.getConfiguration()
         .getMaximumResourceRequirement(label, queuePath, resourceTypes);
-    return maxResource;
   }
 
   protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
       String label) {
-    return csContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label,
+    return queueContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label,
         queuePath, resourceTypes);
   }
 
   protected void updateCapacityConfigType() {
     this.capacityConfigType = CapacityConfigType.NONE;
-    for (String label : configuredNodeLabels) {
+    for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
       LOG.debug("capacityConfigType is '{}' for queue {}",
           capacityConfigType, getQueuePath());
 
       CapacityConfigType localType = checkConfigTypeIsAbsoluteResource(
-          queuePath, label) ? CapacityConfigType.ABSOLUTE_RESOURCE
+          getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE
           : CapacityConfigType.PERCENTAGE;
 
       if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
@@ -620,13 +479,19 @@ protected void updateCapacityConfigType() {
     }
   }
 
+  /**
+   * Initializes configured minimum and maximum capacity from configuration, if capacity is defined
+   * in ABSOLUTE node.
+   * @param clusterResource overall resource of the cluster
+   */
   protected void updateConfigurableResourceLimits(Resource clusterResource) {
-    for (String label : configuredNodeLabels) {
-      final Resource minResource = getMinimumAbsoluteResource(queuePath, label);
-      Resource maxResource = getMaximumAbsoluteResource(queuePath, label);
+    for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
+      final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label);
+      Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label);
 
       if (parent != null) {
-        final Resource parentMax = parent.getQueueResourceQuotas().getConfiguredMaxResource(label);
+        final Resource parentMax = parent.getQueueResourceQuotas()
+            .getConfiguredMaxResource(label);
         validateMinResourceIsNotGreaterThanMaxResource(maxResource, parentMax, clusterResource,
             "Max resource configuration "
                 + maxResource + " is greater than parents max value:"
@@ -650,8 +515,8 @@ protected void updateConfigurableResourceLimits(Resource clusterResource) {
               + " minResource={} and maxResource={}", getQueuePath(), minResource,
           maxResource);
 
-      queueResourceQuotas.setConfiguredMinResource(label, minResource);
-      queueResourceQuotas.setConfiguredMaxResource(label, maxResource);
+      usageTracker.getQueueResourceQuotas().setConfiguredMinResource(label, minResource);
+      usageTracker.getQueueResourceQuotas().setConfiguredMaxResource(label, maxResource);
     }
   }
 
@@ -667,7 +532,7 @@ private void validateMinResourceIsNotGreaterThanMaxResource(Resource minResource
 
   private void validateAbsoluteVsPercentageCapacityConfig(
       CapacityConfigType localType) {
-    if (!queuePath.equals("root")
+    if (!queuePath.isRoot()
         && !this.capacityConfigType.equals(localType)) {
       throw new IllegalArgumentException("Queue '" + getQueuePath()
           + "' should use either percentage based capacity"
@@ -690,7 +555,7 @@ public Resource getEffectiveCapacity(String label) {
   public Resource getEffectiveCapacityDown(String label, Resource factor) {
     return Resources.normalizeDown(resourceCalculator,
         getQueueResourceQuotas().getEffectiveMinResource(label),
-        minimumAllocation);
+        queueAllocationSettings.getMinimumAllocation());
   }
 
   @Override
@@ -703,61 +568,13 @@ public Resource getEffectiveMaxCapacity(String label) {
   public Resource getEffectiveMaxCapacityDown(String label, Resource factor) {
     return Resources.normalizeDown(resourceCalculator,
         getQueueResourceQuotas().getEffectiveMaxResource(label),
-        minimumAllocation);
+        queueAllocationSettings.getMinimumAllocation());
   }
 
-  private void initializeQueueState(CapacitySchedulerConfiguration configuration) {
-    QueueState previousState = getState();
-    QueueState configuredState = configuration
-        .getConfiguredState(getQueuePath());
-    QueueState parentState = (parent == null) ? null : parent.getState();
-
-    // verify that we can not any value for State other than RUNNING/STOPPED
-    if (configuredState != null && configuredState != QueueState.RUNNING
-        && configuredState != QueueState.STOPPED) {
-      throw new IllegalArgumentException("Invalid queue state configuration."
-          + " We can only use RUNNING or STOPPED.");
-    }
-    // If we did not set state in configuration, use Running as default state
-    QueueState defaultState = QueueState.RUNNING;
-
-    if (previousState == null) {
-      // If current state of the queue is null, we would inherit the state
-      // from its parent. If this queue does not has parent, such as root queue,
-      // we would use the configured state.
-      if (parentState == null) {
-        updateQueueState((configuredState == null) ? defaultState
-            : configuredState);
-      } else {
-        if (configuredState == null) {
-          updateQueueState((parentState == QueueState.DRAINING) ?
-              QueueState.STOPPED : parentState);
-        } else if (configuredState == QueueState.RUNNING
-            && parentState != QueueState.RUNNING) {
-          throw new IllegalArgumentException(
-              "The parent queue:" + parent.getQueuePath()
-                  + " cannot be STOPPED as the child queue:" + queuePath
-                  + " is in RUNNING state.");
-        } else {
-          updateQueueState(configuredState);
-        }
-      }
-    } else {
-      // when we get a refreshQueue request from AdminService,
-      if (previousState == QueueState.RUNNING) {
-        if (configuredState == QueueState.STOPPED) {
-          stopQueue();
-        }
-      } else {
-        if (configuredState == QueueState.RUNNING) {
-          try {
-            activeQueue();
-          } catch (YarnException ex) {
-            throw new IllegalArgumentException(ex.getMessage());
-          }
-        }
-      }
-    }
+  @Override
+  public QueueCapacityVector getConfiguredCapacityVector(
+      String label) {
+    return configuredCapacityVectors.get(label);
   }
 
   protected QueueInfo getQueueInfo() {
@@ -765,102 +582,32 @@ protected QueueInfo getQueueInfo() {
     // from schedulerApplicationAttempt, to avoid deadlock, sacrifice
     // consistency here.
     // TODO, improve this
-    QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
-    queueInfo.setQueueName(queueName);
-    queueInfo.setQueuePath(queuePath);
-    queueInfo.setAccessibleNodeLabels(accessibleLabels);
-    queueInfo.setCapacity(queueCapacities.getCapacity());
-    queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
-    queueInfo.setQueueState(getState());
-    queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
-    queueInfo.setCurrentCapacity(getUsedCapacity());
-    queueInfo.setQueueStatistics(getQueueStatistics());
-    queueInfo.setPreemptionDisabled(getPreemptionDisabled());
-    queueInfo.setIntraQueuePreemptionDisabled(
-        getIntraQueuePreemptionDisabled());
-    queueInfo.setQueueConfigurations(getQueueConfigurations());
-    queueInfo.setWeight(queueCapacities.getWeight());
-    queueInfo.setMaxParallelApps(maxParallelApps);
-    return queueInfo;
-  }
-
-  public QueueStatistics getQueueStatistics() {
-    // Deliberately doesn't use lock here, because this method will be invoked
-    // from schedulerApplicationAttempt, to avoid deadlock, sacrifice
-    // consistency here.
-    // TODO, improve this
-    QueueStatistics stats = recordFactory.newRecordInstance(
-        QueueStatistics.class);
-    stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted());
-    stats.setNumAppsRunning(getMetrics().getAppsRunning());
-    stats.setNumAppsPending(getMetrics().getAppsPending());
-    stats.setNumAppsCompleted(getMetrics().getAppsCompleted());
-    stats.setNumAppsKilled(getMetrics().getAppsKilled());
-    stats.setNumAppsFailed(getMetrics().getAppsFailed());
-    stats.setNumActiveUsers(getMetrics().getActiveUsers());
-    stats.setAvailableMemoryMB(getMetrics().getAvailableMB());
-    stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB());
-    stats.setPendingMemoryMB(getMetrics().getPendingMB());
-    stats.setReservedMemoryMB(getMetrics().getReservedMB());
-    stats.setAvailableVCores(getMetrics().getAvailableVirtualCores());
-    stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores());
-    stats.setPendingVCores(getMetrics().getPendingVirtualCores());
-    stats.setReservedVCores(getMetrics().getReservedVirtualCores());
-    stats.setPendingContainers(getMetrics().getPendingContainers());
-    stats.setAllocatedContainers(getMetrics().getAllocatedContainers());
-    stats.setReservedContainers(getMetrics().getReservedContainers());
-    return stats;
-  }
-
-  public Map<String, QueueConfigurations> getQueueConfigurations() {
-    Map<String, QueueConfigurations> queueConfigurations = new HashMap<>();
-    Set<String> nodeLabels = getNodeLabelsForQueue();
-    for (String nodeLabel : nodeLabels) {
-      QueueConfigurations queueConfiguration =
-          recordFactory.newRecordInstance(QueueConfigurations.class);
-      float capacity = queueCapacities.getCapacity(nodeLabel);
-      float absoluteCapacity = queueCapacities.getAbsoluteCapacity(nodeLabel);
-      float maxCapacity = queueCapacities.getMaximumCapacity(nodeLabel);
-      float absMaxCapacity =
-          queueCapacities.getAbsoluteMaximumCapacity(nodeLabel);
-      float maxAMPercentage =
-          queueCapacities.getMaxAMResourcePercentage(nodeLabel);
-      queueConfiguration.setCapacity(capacity);
-      queueConfiguration.setAbsoluteCapacity(absoluteCapacity);
-      queueConfiguration.setMaxCapacity(maxCapacity);
-      queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
-      queueConfiguration.setMaxAMPercentage(maxAMPercentage);
-      queueConfiguration.setConfiguredMinCapacity(
-          queueResourceQuotas.getConfiguredMinResource(nodeLabel));
-      queueConfiguration.setConfiguredMaxCapacity(
-          queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
-      queueConfiguration.setEffectiveMinCapacity(
-          queueResourceQuotas.getEffectiveMinResource(nodeLabel));
-      queueConfiguration.setEffectiveMaxCapacity(
-          queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
-      queueConfigurations.put(nodeLabel, queueConfiguration);
-    }
-    return queueConfigurations;
+    return CSQueueInfoProvider.getQueueInfo(this);
   }
 
   @Private
   public Resource getMaximumAllocation() {
-    return maximumAllocation;
+    return queueAllocationSettings.getMaximumAllocation();
   }
 
   @Private
   public Resource getMinimumAllocation() {
-    return minimumAllocation;
+    return queueAllocationSettings.getMinimumAllocation();
   }
 
+  /**
+   * Increments resource usage of the queue and all related statistics and metrics that depends on
+   * it.
+   * @param clusterResource overall cluster resource
+   * @param resource resource amount to increment
+   * @param nodePartition node label
+   */
   void allocateResource(Resource clusterResource,
       Resource resource, String nodePartition) {
     writeLock.lock();
     try {
-      queueUsage.incUsed(nodePartition, resource);
-
-      ++numContainers;
-
+      usageTracker.getQueueUsage().incUsed(nodePartition, resource);
+      usageTracker.increaseNumContainers();
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
           this, labelManager, nodePartition);
     } finally {
@@ -868,23 +615,34 @@ void allocateResource(Resource clusterResource,
     }
   }
 
+  /**
+   * Decrements resource usage of the queue and all related statistics and metrics that depends on
+   * it.
+   * @param clusterResource overall cluster resource
+   * @param resource resource amount to decrement
+   * @param nodePartition node label
+   */
   protected void releaseResource(Resource clusterResource,
       Resource resource, String nodePartition) {
     writeLock.lock();
     try {
-      queueUsage.decUsed(nodePartition, resource);
+      usageTracker.getQueueUsage().decUsed(nodePartition, resource);
 
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
           this, labelManager, nodePartition);
 
-      --numContainers;
+      usageTracker.decreaseNumContainers();
     } finally {
       writeLock.unlock();
     }
   }
 
+  /**
+   * Returns whether we should continue to look at all heart beating nodes even
+   * after the reservation limit was hit.
+   */
   @Private
-  public boolean getReservationContinueLooking() {
+  public boolean isReservationsContinueLooking() {
     return reservationsContinueLooking;
   }
 
@@ -905,8 +663,7 @@ public boolean getPreemptionDisabled() {
 
   @Private
   public boolean getIntraQueuePreemptionDisabled() {
-    return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() ||
-        preemptionSettings.isPreemptionDisabled();
+    return preemptionSettings.isIntraQueuePreemptionDisabled();
   }
 
   @Private
@@ -921,12 +678,12 @@ public QueueCapacities getQueueCapacities() {
 
   @Private
   public ResourceUsage getQueueResourceUsage() {
-    return queueUsage;
+    return usageTracker.getQueueUsage();
   }
 
   @Override
   public QueueResourceQuotas getQueueResourceQuotas() {
-    return queueResourceQuotas;
+    return usageTracker.getQueueResourceQuotas();
   }
 
   @Override
@@ -934,76 +691,6 @@ public ReentrantReadWriteLock.ReadLock getReadLock() {
     return readLock;
   }
 
-  private long getInheritedMaxAppLifetime(CSQueue q,
-      CapacitySchedulerConfiguration conf) {
-    CSQueue parentQ = q.getParent();
-    long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
-
-    // If q is the root queue, then get max app lifetime from conf.
-    if (parentQ == null) {
-      return maxAppLifetime;
-    }
-
-    // If this is not the root queue, get this queue's max app lifetime
-    // from the conf. The parent's max app lifetime will be used if it's
-    // not set for this queue.
-    // A value of 0 will override the parent's value and means no max lifetime.
-    // A negative value means that the parent's max should be used.
-    long parentsMaxAppLifetime = getParent().getMaximumApplicationLifetime();
-    return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime;
-  }
-
-  private long getInheritedDefaultAppLifetime(CSQueue q,
-      CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
-    CSQueue parentQ = q.getParent();
-    long defaultAppLifetime = conf.getDefaultLifetimePerQueue(getQueuePath());
-    defaultAppLifetimeWasSpecifiedInConfig =
-        (defaultAppLifetime >= 0
-            || (parentQ != null &&
-            parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
-
-    // If q is the root queue, then get default app lifetime from conf.
-    if (parentQ == null) {
-      return defaultAppLifetime;
-    }
-
-    // If this is not the root queue, get the parent's default app lifetime. The
-    // parent's default app lifetime will be used if not set for this queue.
-    long parentsDefaultAppLifetime =
-        getParent().getDefaultApplicationLifetime();
-
-    // Negative value indicates default lifetime was not set at this level.
-    // If default lifetime was not set at this level, calculate it based on
-    // parent's default lifetime or current queue's max lifetime.
-    if (defaultAppLifetime < 0) {
-      // If default lifetime was not set at this level but was set somewhere in
-      // the parent's hierarchy, set default lifetime to parent queue's default
-      // only if parent queue's lifetime is less than current queue's max
-      // lifetime. Otherwise, use current queue's max lifetime value for its
-      // default lifetime.
-      if (defaultAppLifetimeWasSpecifiedInConfig) {
-        defaultAppLifetime =
-            Math.min(parentsDefaultAppLifetime, myMaxAppLifetime);
-      } else {
-        // Default app lifetime value was not set anywhere in this queue's
-        // hierarchy. Use current queue's max lifetime as its default.
-        defaultAppLifetime = myMaxAppLifetime;
-      }
-    } // else if >= 0, default lifetime was set at this level. Just use it.
-
-    if (myMaxAppLifetime > 0 &&
-        defaultAppLifetime > myMaxAppLifetime) {
-      throw new YarnRuntimeException(
-          "Default lifetime " + defaultAppLifetime
-              + " can't exceed maximum lifetime " + myMaxAppLifetime);
-    }
-    
-    if (defaultAppLifetime <= 0) {
-      defaultAppLifetime = myMaxAppLifetime;
-    }
-    return defaultAppLifetime;
-  }
-
   private Resource getCurrentLimitResource(String nodePartition,
       Resource clusterResource, ResourceLimits currentResourceLimits,
       SchedulingMode schedulingMode) {
@@ -1032,11 +719,21 @@ Resource getQueueMaxResource(String nodePartition) {
     return getEffectiveMaxCapacity(nodePartition);
   }
 
-  public boolean hasChildQueues() {
+  @VisibleForTesting
+  boolean hasChildQueues() {
     List<CSQueue> childQueues = getChildQueues();
     return childQueues != null && !childQueues.isEmpty();
   }
 
+  /**
+   * Checks whether this queue has remaining resources left for further container assigment.
+   * @param clusterResource overall cluster resource
+   * @param nodePartition node label
+   * @param currentResourceLimits limit of the queue imposed by its maximum capacity
+   * @param resourceCouldBeUnreserved reserved resource that could potentially be unreserved
+   * @param schedulingMode scheduling strategy to handle node labels
+   * @return true if queue has remaining free resource, false otherwise
+   */
   boolean canAssignToThisQueue(Resource clusterResource,
       String nodePartition, ResourceLimits currentResourceLimits,
       Resource resourceCouldBeUnreserved, SchedulingMode schedulingMode) {
@@ -1056,7 +753,7 @@ boolean canAssignToThisQueue(Resource clusterResource,
       Resource currentLimitResource = getCurrentLimitResource(nodePartition,
           clusterResource, currentResourceLimits, schedulingMode);
 
-      Resource nowTotalUsed = queueUsage.getUsed(nodePartition);
+      Resource nowTotalUsed = usageTracker.getQueueUsage().getUsed(nodePartition);
 
       // Set headroom for currentResourceLimits:
       // When queue is a parent queue: Headroom = limit - used + killable
@@ -1088,7 +785,7 @@ boolean canAssignToThisQueue(Resource clusterResource,
               newTotalWithoutReservedResource, currentLimitResource)) {
             if (LOG.isDebugEnabled()) {
               LOG.debug("try to use reserved: " + getQueuePath()
-                  + " usedResources: " + queueUsage.getUsed()
+                  + " usedResources: " + usageTracker.getQueueUsage().getUsed()
                   + ", clusterResources: " + clusterResource
                   + ", reservedResources: " + resourceCouldBeUnreserved
                   + ", capacity-without-reserved: "
@@ -1103,7 +800,7 @@ boolean canAssignToThisQueue(Resource clusterResource,
         if (LOG.isDebugEnabled()) {
           LOG.debug("Failed to assign to queue: " + getQueuePath()
               + " nodePartition: " + nodePartition
-              + ", usedResources: " + queueUsage.getUsed(nodePartition)
+              + ", usedResources: " + usageTracker.getQueueUsage().getUsed(nodePartition)
               + ", clusterResources: " + clusterResource
               + ", reservedResources: " + resourceCouldBeUnreserved
               + ", maxLimitCapacity: " + currentLimitResource
@@ -1114,11 +811,11 @@ boolean canAssignToThisQueue(Resource clusterResource,
       if (LOG.isDebugEnabled()) {
         LOG.debug("Check assign to queue: " + getQueuePath()
             + " nodePartition: " + nodePartition
-            + ", usedResources: " + queueUsage.getUsed(nodePartition)
+            + ", usedResources: " + usageTracker.getQueueUsage().getUsed(nodePartition)
             + ", clusterResources: " + clusterResource
             + ", currentUsedCapacity: " + Resources
             .divide(resourceCalculator, clusterResource,
-                queueUsage.getUsed(nodePartition), labelManager
+                usageTracker.getQueueUsage().getUsed(nodePartition), labelManager
                     .getResourceByLabel(nodePartition, clusterResource))
             + ", max-capacity: " + queueCapacities
             .getAbsoluteMaximumCapacity(nodePartition));
@@ -1131,7 +828,7 @@ boolean canAssignToThisQueue(Resource clusterResource,
   }
 
   private static String ensurePartition(String partition) {
-    return Optional.ofNullable(partition).orElse(RMNodeLabelsManager.NO_LABEL);
+    return Optional.ofNullable(partition).orElse(NO_LABEL);
   }
 
   @FunctionalInterface
@@ -1162,39 +859,39 @@ private void countAndUpdate(String partition, Resource resource,
 
   @Override
   public void incReservedResource(String partition, Resource reservedRes) {
-    count(partition, reservedRes, queueUsage::incReserved,
+    count(partition, reservedRes, usageTracker.getQueueUsage()::incReserved,
         parent == null ? null : parent::incReservedResource);
   }
 
   @Override
   public void decReservedResource(String partition, Resource reservedRes) {
-    count(partition, reservedRes, queueUsage::decReserved,
+    count(partition, reservedRes, usageTracker.getQueueUsage()::decReserved,
         parent == null ? null : parent::decReservedResource);
   }
 
   @Override
   public void incPendingResource(String nodeLabel, Resource resourceToInc) {
-    count(nodeLabel, resourceToInc, queueUsage::incPending,
+    count(nodeLabel, resourceToInc, usageTracker.getQueueUsage()::incPending,
         parent == null ? null : parent::incPendingResource);
   }
 
   @Override
   public void decPendingResource(String nodeLabel, Resource resourceToDec) {
-    count(nodeLabel, resourceToDec, queueUsage::decPending,
+    count(nodeLabel, resourceToDec, usageTracker.getQueueUsage()::decPending,
         parent == null ? null : parent::decPendingResource);
   }
 
   @Override
   public void incUsedResource(String nodeLabel, Resource resourceToInc,
       SchedulerApplicationAttempt application) {
-    countAndUpdate(nodeLabel, resourceToInc, queueUsage::incUsed,
+    countAndUpdate(nodeLabel, resourceToInc, usageTracker.getQueueUsage()::incUsed,
         parent == null ? null : parent::incUsedResource);
   }
 
   @Override
   public void decUsedResource(String nodeLabel, Resource resourceToDec,
       SchedulerApplicationAttempt application) {
-    countAndUpdate(nodeLabel, resourceToDec, queueUsage::decUsed,
+    countAndUpdate(nodeLabel, resourceToDec, usageTracker.getQueueUsage()::decUsed,
         parent == null ? null : parent::decUsedResource);
   }
 
@@ -1205,34 +902,19 @@ public void decUsedResource(String nodeLabel, Resource resourceToDec,
   boolean hasPendingResourceRequest(String nodePartition,
       Resource cluster, SchedulingMode schedulingMode) {
     return SchedulerUtils.hasPendingResourceRequest(resourceCalculator,
-        queueUsage, nodePartition, cluster, schedulingMode);
-  }
-
-  public boolean accessibleToPartition(String nodePartition) {
-    // if queue's label is *, it can access any node
-    if (accessibleLabels != null
-        && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
-      return true;
-    }
-    // any queue can access to a node without label
-    if (nodePartition == null
-        || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
-      return true;
-    }
-    // a queue can access to a node only if it contains any label of the node
-    if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) {
-      return true;
-    }
-    // sorry, you cannot access
-    return false;
+        usageTracker.getQueueUsage(), nodePartition, cluster, schedulingMode);
   }
 
   @Override
   public Priority getDefaultApplicationPriority() {
-    // TODO add dummy implementation
     return null;
   }
 
+  /**
+   * Returns the union of all node labels that could be accessed by this queue based on accessible
+   * node labels and configured node labels properties.
+   * @return node labels this queue has access to
+   */
   @Override
   public Set<String> getNodeLabelsForQueue() {
     // if queue's label is *, queue can access any labels. Instead of
@@ -1241,27 +923,27 @@ public Set<String> getNodeLabelsForQueue() {
     Set<String> nodeLabels = new HashSet<String>();
     if (this.getAccessibleNodeLabels() != null && this.getAccessibleNodeLabels()
         .contains(RMNodeLabelsManager.ANY)) {
-      nodeLabels.addAll(Sets.union(this.getQueueCapacities().getNodePartitionsSet(),
-          this.getQueueResourceUsage().getNodePartitionsSet()));
+      nodeLabels.addAll(Sets.union(this.getQueueCapacities().getExistingNodeLabels(),
+          this.getQueueResourceUsage().getExistingNodeLabels()));
     } else {
       nodeLabels.addAll(this.getAccessibleNodeLabels());
     }
 
     // Add NO_LABEL also to this list as NO_LABEL also can be granted with
     // resource in many general cases.
-    if (!nodeLabels.contains(RMNodeLabelsManager.NO_LABEL)) {
-      nodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+    if (!nodeLabels.contains(NO_LABEL)) {
+      nodeLabels.add(NO_LABEL);
     }
     return nodeLabels;
   }
 
   public Resource getTotalKillableResource(String partition) {
-    return csContext.getPreemptionManager().getKillableResource(getQueuePath(),
+    return queueContext.getPreemptionManager().getKillableResource(getQueuePath(),
         partition);
   }
 
   public Iterator<RMContainer> getKillableContainers(String partition) {
-    return csContext.getPreemptionManager().getKillableContainers(
+    return queueContext.getPreemptionManager().getKillableContainers(
         getQueuePath(),
         partition);
   }
@@ -1275,6 +957,12 @@ public CSAssignment assignContainers(Resource clusterResource,
         resourceLimits, schedulingMode);
   }
 
+  /**
+   * Checks whether this queue could accept the container allocation request.
+   * @param cluster overall cluster resource
+   * @param request container allocation request
+   * @return true if queue could accept the container allocation request, false otherwise
+   */
   @Override
   public boolean accept(Resource cluster,
       ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
@@ -1304,10 +992,10 @@ public boolean accept(Resource cluster,
               schedulerContainer.getNodePartition(), cluster);
         }
         if (!Resources.fitsIn(resourceCalculator,
-            Resources.add(queueUsage.getUsed(partition), netAllocated),
+            Resources.add(usageTracker.getQueueUsage().getUsed(partition), netAllocated),
             maxResourceLimit)) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("Used resource=" + queueUsage.getUsed(partition)
+            LOG.debug("Used resource=" + usageTracker.getQueueUsage().getUsed(partition)
                 + " exceeded maxResourceLimit of the queue ="
                 + maxResourceLimit);
           }
@@ -1340,15 +1028,19 @@ public void updateQueueState(QueueState queueState) {
     this.state = queueState;
   }
 
+  /**
+   * Sets the state of this queue to RUNNING.
+   * @throws YarnException if its parent queue is not in RUNNING state
+   */
   @Override
-  public void activeQueue() throws YarnException {
+  public void activateQueue() throws YarnException {
     this.writeLock.lock();
     try {
       if (getState() == QueueState.RUNNING) {
         LOG.info("The specified queue:" + getQueuePath()
             + " is already in the RUNNING state.");
       } else {
-        CSQueue parentQueue = getParent();
+        CSQueue parentQueue = parent;
         if (parentQueue == null || parentQueue.getState() == QueueState.RUNNING) {
           updateQueueState(QueueState.RUNNING);
         } else {
@@ -1361,6 +1053,9 @@ public void activeQueue() throws YarnException {
     }
   }
 
+  /**
+   * Stops this queue if no application is currently running on the queue.
+   */
   protected void appFinished() {
     this.writeLock.lock();
     try {
@@ -1384,6 +1079,9 @@ public UserWeights getUserWeights() {
     return userWeights;
   }
 
+  /**
+   * Recursively sets the state of this queue and the state of its parent to DRAINING.
+   */
   public void recoverDrainingState() {
     this.writeLock.lock();
     try {
@@ -1391,8 +1089,8 @@ public void recoverDrainingState() {
         updateQueueState(QueueState.DRAINING);
       }
       LOG.info("Recover draining state for queue " + this.getQueuePath());
-      if (getParent() != null && getParent().getState() == QueueState.STOPPED) {
-        ((AbstractCSQueue) getParent()).recoverDrainingState();
+      if (parent != null && parent.getState() == QueueState.STOPPED) {
+        ((AbstractCSQueue) parent).recoverDrainingState();
       }
     } finally {
       this.writeLock.unlock();
@@ -1409,24 +1107,24 @@ public void setMultiNodeSortingPolicyName(String policyName) {
   }
 
   public long getMaximumApplicationLifetime() {
-    return maxApplicationLifetime;
+    return queueAppLifetimeSettings.getMaxApplicationLifetime();
   }
 
   public long getDefaultApplicationLifetime() {
-    return defaultApplicationLifetime;
+    return queueAppLifetimeSettings.getDefaultApplicationLifetime();
   }
 
   public boolean getDefaultAppLifetimeWasSpecifiedInConfig() {
-    return defaultAppLifetimeWasSpecifiedInConfig;
+    return queueAppLifetimeSettings.isDefaultAppLifetimeWasSpecifiedInConfig();
   }
 
   public void setMaxParallelApps(int maxParallelApps) {
-    this.maxParallelApps = maxParallelApps;
+    this.queueAppLifetimeSettings.setMaxParallelApps(maxParallelApps);
   }
 
   @Override
   public int getMaxParallelApps() {
-    return maxParallelApps;
+    return this.queueAppLifetimeSettings.getMaxParallelApps();
   }
 
   abstract int getNumRunnableApps();
@@ -1454,7 +1152,7 @@ private Resource createNormalizedMinResource(Resource minResource,
         ret.setResourceValue(i,
             (long) (nResourceInformation.getValue() * ratio));
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Updating min resource for Queue: " + queuePath + " as " + ret
+          LOG.debug("Updating min resource for Queue: " + getQueuePath() + " as " + ret
               .getResourceInformation(i) + ", Actual resource: "
               + nResourceInformation.getValue() + ", ratio: " + ratio);
         }
@@ -1481,60 +1179,13 @@ private Resource getOrInheritMaxResource(Resource resourceByLabel, String label)
         configuredMaxResource, parentMaxResource));
   }
 
-  void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf,
-      LeafQueue leafQueue) {
-    int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
-    int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
-    String maxLabel = RMNodeLabelsManager.NO_LABEL;
-
-    if (maxApplications < 0) {
-      for (String label : configuredNodeLabels) {
-        int maxApplicationsByLabel = 0;
-        if (maxGlobalPerQueueApps > 0) {
-          // In absolute mode, should
-          // shrink when change to corresponding label capacity.
-          maxApplicationsByLabel = this.capacityConfigType
-              != CapacityConfigType.ABSOLUTE_RESOURCE ?
-              maxGlobalPerQueueApps :
-              (int) (maxGlobalPerQueueApps * queueCapacities
-                  .getAbsoluteCapacity(label));
-        } else {
-          maxApplicationsByLabel = (int) (conf.getMaximumSystemApplications()
-              * queueCapacities.getAbsoluteCapacity(label));
-        }
-        if (maxApplicationsByLabel > maxApplications) {
-          maxApplications = maxApplicationsByLabel;
-          maxLabel = label;
-        }
-      }
-    }
-    leafQueue.setMaxApplications(maxApplications);
-
-    int maxApplicationsPerUser = Math.min(maxApplications,
-        (int) (maxApplications
-            * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
-            * leafQueue.getUsersManager().getUserLimitFactor()));
-    if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
-      maxApplicationsPerUser =  maxApplications;
-    }
-
-    leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
-    LOG.info("LeafQueue:" + leafQueue.getQueuePath() +
-        "update max app related, maxApplications="
-        + maxApplications + ", maxApplicationsPerUser="
-        + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
-        .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities
-        .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities
-        .getMaximumCapacity(maxLabel));
-  }
-
   void deriveCapacityFromAbsoluteConfigurations(String label,
       Resource clusterResource) {
     // Update capacity with a float calculated from the parent's minResources
     // and the recently changed queue minResources.
     // capacity = effectiveMinResource / {parent's effectiveMinResource}
     float result = resourceCalculator.divide(clusterResource,
-        queueResourceQuotas.getEffectiveMinResource(label),
+        usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label),
         parent.getQueueResourceQuotas().getEffectiveMinResource(label));
     queueCapacities.setCapacity(label,
         Float.isInfinite(result) ? 0 : result);
@@ -1543,7 +1194,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label,
     // and the recently changed queue maxResources.
     // maxCapacity = effectiveMaxResource / parent's effectiveMaxResource
     result = resourceCalculator.divide(clusterResource,
-        queueResourceQuotas.getEffectiveMaxResource(label),
+        usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(label),
         parent.getQueueResourceQuotas().getEffectiveMaxResource(label));
     queueCapacities.setMaximumCapacity(label,
         Float.isInfinite(result) ? 0 : result);
@@ -1567,7 +1218,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label,
   }
 
   void updateEffectiveResources(Resource clusterResource) {
-    for (String label : configuredNodeLabels) {
+    for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
       Resource resourceByLabel = labelManager.getResourceByLabel(label,
           clusterResource);
       Resource newEffectiveMinResource;
@@ -1577,8 +1228,8 @@ void updateEffectiveResources(Resource clusterResource) {
       if (getCapacityConfigType().equals(
           CapacityConfigType.ABSOLUTE_RESOURCE)) {
         newEffectiveMinResource = createNormalizedMinResource(
-            queueResourceQuotas.getConfiguredMinResource(label),
-            ((ParentQueue) parent).getEffectiveMinRatioPerResource());
+            usageTracker.getQueueResourceQuotas().getConfiguredMinResource(label),
+            ((ParentQueue) parent).getEffectiveMinRatio(label));
 
         // Max resource of a queue should be the minimum of {parent's maxResources,
         // this queue's maxResources}. Both parent's maxResources and this queue's
@@ -1597,13 +1248,13 @@ void updateEffectiveResources(Resource clusterResource) {
       }
 
       // Update the effective min
-      queueResourceQuotas.setEffectiveMinResource(label,
+      usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label,
           newEffectiveMinResource);
-      queueResourceQuotas.setEffectiveMaxResource(label,
+      usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label,
           newEffectiveMaxResource);
 
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Updating queue:" + queuePath
+        LOG.debug("Updating queue:" + getQueuePath()
             + " with effective minimum resource=" + newEffectiveMinResource
             + "and effective maximum resource="
             + newEffectiveMaxResource);
@@ -1652,42 +1303,52 @@ protected String getCapacityOrWeightString() {
     }
   }
 
+  /**
+   * Checks whether this queue is a dynamic queue and could be deleted.
+   * @return true if the dynamic queue could be deleted, false otherwise
+   */
   public boolean isEligibleForAutoDeletion() {
     return false;
   }
 
+  /**
+   * Checks whether this queue is a dynamic queue and there has not been an application submission
+   * on it for a configured period of time.
+   * @return true if queue has been idle for a configured period of time, false otherwise
+   */
   public boolean isInactiveDynamicQueue() {
     long idleDurationSeconds =
         (Time.monotonicNow() - getLastSubmittedTimestamp())/1000;
     return isDynamicQueue() && isEligibleForAutoDeletion() &&
-        (idleDurationSeconds > this.csContext.getConfiguration().
+        (idleDurationSeconds > queueContext.getConfiguration().
             getAutoExpiredDeletionTime());
   }
 
-  public void updateLastSubmittedTimeStamp() {
+  void updateLastSubmittedTimeStamp() {
     writeLock.lock();
     try {
-      this.lastSubmittedTimestamp = Time.monotonicNow();
+      usageTracker.setLastSubmittedTimestamp(Time.monotonicNow());
     } finally {
       writeLock.unlock();
     }
   }
 
-  public long getLastSubmittedTimestamp() {
+  @VisibleForTesting
+  long getLastSubmittedTimestamp() {
     readLock.lock();
 
     try {
-      return lastSubmittedTimestamp;
+      return usageTracker.getLastSubmittedTimestamp();
     } finally {
       readLock.unlock();
     }
   }
 
   @VisibleForTesting
-  public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
+  void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
     writeLock.lock();
     try {
-      this.lastSubmittedTimestamp = lastSubmittedTimestamp;
+      usageTracker.setLastSubmittedTimestamp(lastSubmittedTimestamp);
     } finally {
       writeLock.unlock();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
new file mode 100644
index 0000000000000..08fedb578cab9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
@@ -0,0 +1,2478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
+import org.apache.hadoop.yarn.server.utils.Lock;
+import org.apache.hadoop.yarn.server.utils.Lock.NoLock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getACLsForFlexibleAutoCreatedLeafQueue;
+
+public class AbstractLeafQueue extends AbstractCSQueue {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(AbstractLeafQueue.class);
+
+  private float absoluteUsedCapacity = 0.0f;
+
+  // TODO the max applications should consider label
+  protected int maxApplications;
+  protected volatile int maxApplicationsPerUser;
+
+  private float maxAMResourcePerQueuePercent;
+
+  private volatile int nodeLocalityDelay;
+  private volatile int rackLocalityAdditionalDelay;
+  private volatile boolean rackLocalityFullReset;
+
+  Map<ApplicationAttemptId, FiCaSchedulerApp> applicationAttemptMap =
+      new ConcurrentHashMap<>();
+
+  private Priority defaultAppPriorityPerQueue;
+
+  private final OrderingPolicy<FiCaSchedulerApp> pendingOrderingPolicy;
+
+  private volatile float minimumAllocationFactor;
+
+  private final RecordFactory recordFactory =
+      RecordFactoryProvider.getRecordFactory(null);
+
+  private final UsersManager usersManager;
+
+  // cache last cluster resource to compute actual capacity
+  private Resource lastClusterResource = Resources.none();
+
+  private final QueueResourceLimitsInfo queueResourceLimitsInfo =
+      new QueueResourceLimitsInfo();
+
+  private volatile ResourceLimits cachedResourceLimitsForHeadroom = null;
+
+  private volatile OrderingPolicy<FiCaSchedulerApp> orderingPolicy = null;
+
+  // Map<Partition, Map<SchedulingMode, Map<User, CachedUserLimit>>>
+  // Not thread safe: only the last level is a ConcurrentMap
+  @VisibleForTesting
+  Map<String, Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>>
+      userLimitsCache = new HashMap<>();
+
+  // Not thread safe
+  @VisibleForTesting
+  long currentUserLimitCacheVersion = 0;
+
+  // record all ignore partition exclusivityRMContainer, this will be used to do
+  // preemption, key is the partition of the RMContainer allocated on
+  private Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityRMContainers =
+      new ConcurrentHashMap<>();
+
+  List<AppPriorityACLGroup> priorityAcls =
+      new ArrayList<AppPriorityACLGroup>();
+
+  private final List<FiCaSchedulerApp> runnableApps = new ArrayList<>();
+  private final List<FiCaSchedulerApp> nonRunnableApps = new ArrayList<>();
+
+  public AbstractLeafQueue(CapacitySchedulerQueueContext queueContext,
+      String queueName, CSQueue parent, CSQueue old) throws IOException {
+    this(queueContext, queueName, parent, old, false);
+  }
+
+  public AbstractLeafQueue(CapacitySchedulerQueueContext queueContext,
+      String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws
+      IOException {
+    super(queueContext, queueName, parent, old);
+    setDynamicQueue(isDynamic);
+
+    this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager,
+        resourceCalculator);
+
+    // One time initialization is enough since it is static ordering policy
+    this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps();
+  }
+
+  @SuppressWarnings("checkstyle:nowhitespaceafter")
+  protected void setupQueueConfigs(Resource clusterResource) throws
+      IOException {
+    writeLock.lock();
+    try {
+      CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
+      super.setupQueueConfigs(clusterResource);
+
+      this.lastClusterResource = clusterResource;
+
+      this.cachedResourceLimitsForHeadroom = new ResourceLimits(
+          clusterResource);
+
+      // Initialize headroom info, also used for calculating application
+      // master resource limits.  Since this happens during queue initialization
+      // and all queues may not be realized yet, we'll use (optimistic)
+      // absoluteMaxCapacity (it will be replaced with the more accurate
+      // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
+      setQueueResourceLimitsInfo(clusterResource);
+
+      setOrderingPolicy(
+          configuration.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePath()));
+
+      usersManager.setUserLimit(configuration.getUserLimit(getQueuePath()));
+      usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePath()));
+
+      maxAMResourcePerQueuePercent =
+          configuration.getMaximumApplicationMasterResourcePerQueuePercent(
+              getQueuePath());
+
+      maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePath());
+      if (maxApplications < 0) {
+        int maxGlobalPerQueueApps =
+            configuration.getGlobalMaximumApplicationsPerQueue();
+        if (maxGlobalPerQueueApps > 0) {
+          maxApplications = maxGlobalPerQueueApps;
+        }
+      }
+
+      priorityAcls = configuration.getPriorityAcls(getQueuePath(),
+          configuration.getClusterLevelApplicationMaxPriority());
+
+      Set<String> accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels();
+      if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels,
+          this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) {
+        throw new IOException(
+            "Invalid default label expression of " + " queue=" + getQueuePath()
+                + " doesn't have permission to access all labels "
+                + "in default label expression. labelExpression of resource request="
+                + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + (
+                getAccessibleNodeLabels() == null ?
+                    "" :
+                    StringUtils
+                        .join(getAccessibleNodeLabels().iterator(), ',')));
+      }
+
+      nodeLocalityDelay = configuration.getNodeLocalityDelay();
+      rackLocalityAdditionalDelay = configuration
+          .getRackLocalityAdditionalDelay();
+      rackLocalityFullReset = configuration
+          .getRackLocalityFullReset();
+
+      // re-init this since max allocation could have changed
+      this.minimumAllocationFactor = Resources.ratio(resourceCalculator,
+          Resources.subtract(
+              queueAllocationSettings.getMaximumAllocation(),
+              queueAllocationSettings.getMinimumAllocation()),
+          queueAllocationSettings.getMaximumAllocation());
+
+      StringBuilder aclsString = new StringBuilder();
+      for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
+        aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
+      }
+
+      StringBuilder labelStrBuilder = new StringBuilder();
+      if (accessibleNodeLabels != null) {
+        for (String nodeLabel : accessibleNodeLabels) {
+          labelStrBuilder.append(nodeLabel).append(",");
+        }
+      }
+
+      defaultAppPriorityPerQueue = Priority.newInstance(
+          configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
+
+      // Validate leaf queue's user's weights.
+      float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePath()));
+      getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath());
+      usersManager.updateUserWeights();
+
+      LOG.info(
+          "Initializing " + getQueuePath() + "\n" +
+              getExtendedCapacityOrWeightString() + "\n"
+              + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
+              + " [= parentAbsoluteCapacity * capacity ]" + "\n"
+              + "maxCapacity = " + queueCapacities.getMaximumCapacity()
+              + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = "
+              + queueCapacities.getAbsoluteMaximumCapacity()
+              + " [= 1.0 maximumCapacity undefined, "
+              + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]"
+              + "\n" + "effectiveMinResource=" +
+              getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n"
+              + " , effectiveMaxResource=" +
+              getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL)
+              + "\n" + "userLimit = " + usersManager.getUserLimit()
+              + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = "
+              + usersManager.getUserLimitFactor()
+              + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = "
+              + maxApplications
+              + " [= configuredMaximumSystemApplicationsPerQueue or"
+              + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]"
+              + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser
+              + " [= (int)(maxApplications * (userLimit / 100.0f) * "
+              + "userLimitFactor) ]" + "\n"
+              + "maxParallelApps = " + getMaxParallelApps() + "\n"
+              + "usedCapacity = " +
+              + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / "
+              + "(clusterResourceMemory * absoluteCapacity)]" + "\n"
+              + "absoluteUsedCapacity = " + absoluteUsedCapacity
+              + " [= usedResourcesMemory / clusterResourceMemory]" + "\n"
+              + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent
+              + " [= configuredMaximumAMResourcePercent ]" + "\n"
+              + "minimumAllocationFactor = " + minimumAllocationFactor
+              + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / "
+              + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = "
+              + queueAllocationSettings.getMaximumAllocation() +
+              " [= configuredMaxAllocation ]" + "\n"
+              + "numContainers = " + usageTracker.getNumContainers()
+              + " [= currentNumContainers ]" + "\n" + "state = " + getState()
+              + " [= configuredState ]" + "\n" + "acls = " + aclsString
+              + " [= configuredAcls ]" + "\n"
+              + "nodeLocalityDelay = " + nodeLocalityDelay + "\n"
+              + "rackLocalityAdditionalDelay = "
+              + rackLocalityAdditionalDelay + "\n"
+              + "labels=" + labelStrBuilder.toString() + "\n"
+              + "reservationsContinueLooking = "
+              + reservationsContinueLooking + "\n" + "preemptionDisabled = "
+              + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = "
+              + defaultAppPriorityPerQueue + "\npriority = " + priority
+              + "\nmaxLifetime = " + getMaximumApplicationLifetime()
+              + " seconds" + "\ndefaultLifetime = "
+              + getDefaultApplicationLifetime() + " seconds");
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private String getDefaultNodeLabelExpressionStr() {
+    String defaultLabelExpression = queueNodeLabelsSettings.getDefaultLabelExpression();
+    return defaultLabelExpression == null ? "" : defaultLabelExpression;
+  }
+
+  /**
+   * Used only by tests.
+   */
+  @Private
+  public float getMinimumAllocationFactor() {
+    return minimumAllocationFactor;
+  }
+
+  /**
+   * Used only by tests.
+   */
+  @Private
+  public float getMaxAMResourcePerQueuePercent() {
+    return maxAMResourcePerQueuePercent;
+  }
+
+  public int getMaxApplications() {
+    return maxApplications;
+  }
+
+  public int getMaxApplicationsPerUser() {
+    return maxApplicationsPerUser;
+  }
+
+  /**
+   *
+   * @return UsersManager instance.
+   */
+  public UsersManager getUsersManager() {
+    return usersManager;
+  }
+
+  @Override
+  public AbstractUsersManager getAbstractUsersManager() {
+    return usersManager;
+  }
+
+  @Override
+  public List<CSQueue> getChildQueues() {
+    return null;
+  }
+
+  /**
+   * Set user limit.
+   * @param userLimit new user limit
+   */
+  @VisibleForTesting
+  void setUserLimit(float userLimit) {
+    usersManager.setUserLimit(userLimit);
+    usersManager.userLimitNeedsRecompute();
+  }
+
+  /**
+   * Set user limit factor.
+   * @param userLimitFactor new user limit factor
+   */
+  @VisibleForTesting
+  void setUserLimitFactor(float userLimitFactor) {
+    usersManager.setUserLimitFactor(userLimitFactor);
+    usersManager.userLimitNeedsRecompute();
+  }
+
+  @Override
+  public int getNumApplications() {
+    readLock.lock();
+    try {
+      return getNumPendingApplications() + getNumActiveApplications() +
+          getNumNonRunnableApps();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public int getNumPendingApplications() {
+    readLock.lock();
+    try {
+      return pendingOrderingPolicy.getNumSchedulableEntities();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public int getNumActiveApplications() {
+    readLock.lock();
+    try {
+      return orderingPolicy.getNumSchedulableEntities();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Private
+  public int getNumPendingApplications(String user) {
+    readLock.lock();
+    try {
+      User u = getUser(user);
+      if (null == u) {
+        return 0;
+      }
+      return u.getPendingApplications();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Private
+  public int getNumActiveApplications(String user) {
+    readLock.lock();
+    try {
+      User u = getUser(user);
+      if (null == u) {
+        return 0;
+      }
+      return u.getActiveApplications();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Private
+  public float getUserLimit() {
+    return usersManager.getUserLimit();
+  }
+
+  @Private
+  public float getUserLimitFactor() {
+    return usersManager.getUserLimitFactor();
+  }
+
+  @Override
+  public QueueInfo getQueueInfo(
+      boolean includeChildQueues, boolean recursive) {
+    QueueInfo queueInfo = getQueueInfo();
+    return queueInfo;
+  }
+
+  @Override
+  public List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user) {
+    readLock.lock();
+    try {
+      QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
+          QueueUserACLInfo.class);
+      List<QueueACL> operations = new ArrayList<>();
+      for (QueueACL operation : QueueACL.values()) {
+        if (hasAccess(operation, user)) {
+          operations.add(operation);
+        }
+      }
+
+      userAclInfo.setQueueName(getQueuePath());
+      userAclInfo.setUserAcls(operations);
+      return Collections.singletonList(userAclInfo);
+    } finally {
+      readLock.unlock();
+    }
+
+  }
+
+  public String toString() {
+    readLock.lock();
+    try {
+      return getQueuePath() + ": " + getCapacityOrWeightString()
+          + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
+          + ", " + "usedResources=" + usageTracker.getQueueUsage().getUsed() + ", "
+          + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity="
+          + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications()
+          + ", " + "numContainers=" + getNumContainers() + ", "
+          + "effectiveMinResource=" +
+          getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) +
+          " , effectiveMaxResource=" +
+          getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  protected String getExtendedCapacityOrWeightString() {
+    if (queueCapacities.getWeight() != -1) {
+      return "weight = " + queueCapacities.getWeight()
+          + " [= (float) configuredCapacity (with w suffix)] " + "\n"
+          + "normalizedWeight = " + queueCapacities.getNormalizedWeight()
+          + " [= (float) configuredCapacity / sum(configuredCapacity of " +
+          "all queues under the parent)]";
+    } else {
+      return "capacity = " + queueCapacities.getCapacity()
+          + " [= (float) configuredCapacity / 100 ]";
+    }
+  }
+
+  @VisibleForTesting
+  public User getUser(String userName) {
+    return usersManager.getUser(userName);
+  }
+
+  @VisibleForTesting
+  public User getOrCreateUser(String userName) {
+    return usersManager.getUserAndAddIfAbsent(userName);
+  }
+
+  @Private
+  public List<AppPriorityACLGroup> getPriorityACLs() {
+    readLock.lock();
+    try {
+      return new ArrayList<>(priorityAcls);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+      throws IOException {
+
+    writeLock.lock();
+    try {
+      // We skip reinitialize for dynamic queues, when this is called, and
+      // new queue is different from this queue, we will make this queue to be
+      // static queue.
+      if (newlyParsedQueue != this) {
+        this.setDynamicQueue(false);
+      }
+
+      // Sanity check
+      if (!(newlyParsedQueue instanceof AbstractLeafQueue) || !newlyParsedQueue
+          .getQueuePath().equals(getQueuePath())) {
+        throw new IOException(
+            "Trying to reinitialize " + getQueuePath() + " from "
+                + newlyParsedQueue.getQueuePath());
+      }
+
+      AbstractLeafQueue newlyParsedLeafQueue = (AbstractLeafQueue) newlyParsedQueue;
+
+      // don't allow the maximum allocation to be decreased in size
+      // since we have already told running AM's the size
+      Resource oldMax = getMaximumAllocation();
+      Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
+
+      if (!Resources.fitsIn(oldMax, newMax)) {
+        throw new IOException("Trying to reinitialize " + getQueuePath()
+            + " the maximum allocation size can not be decreased!"
+            + " Current setting: " + oldMax + ", trying to set it to: "
+            + newMax);
+      }
+
+      setupQueueConfigs(clusterResource);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void submitApplicationAttempt(FiCaSchedulerApp application,
+      String userName) {
+    submitApplicationAttempt(application, userName, false);
+  }
+
+  @Override
+  public void submitApplicationAttempt(FiCaSchedulerApp application,
+      String userName, boolean isMoveApp) {
+    // Careful! Locking order is important!
+    boolean isAppAlreadySubmitted = applicationAttemptMap.containsKey(
+        application.getApplicationAttemptId());
+    writeLock.lock();
+    try {
+      // TODO, should use getUser, use this method just to avoid UT failure
+      // which is caused by wrong invoking order, will fix UT separately
+      User user = usersManager.getUserAndAddIfAbsent(userName);
+
+      // Add the attempt to our data-structures
+      addApplicationAttempt(application, user);
+    } finally {
+      writeLock.unlock();
+    }
+
+    // We don't want to update metrics for move app
+    if (!isMoveApp && !isAppAlreadySubmitted) {
+      boolean unmanagedAM = application.getAppSchedulingInfo() != null &&
+          application.getAppSchedulingInfo().isUnmanagedAM();
+      usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM);
+    }
+
+    parent.submitApplicationAttempt(application, userName);
+  }
+
+  @Override
+  public void submitApplication(ApplicationId applicationId, String userName,
+      String queue)  throws AccessControlException {
+    // Careful! Locking order is important!
+    validateSubmitApplication(applicationId, userName, queue);
+
+    // Signal for expired auto deletion.
+    updateLastSubmittedTimeStamp();
+
+    // Inform the parent queue
+    try {
+      parent.submitApplication(applicationId, userName, queue);
+    } catch (AccessControlException ace) {
+      LOG.info("Failed to submit application to parent-queue: " +
+          parent.getQueuePath(), ace);
+      throw ace;
+    }
+
+  }
+
+  public void validateSubmitApplication(ApplicationId applicationId,
+      String userName, String queue) throws AccessControlException {
+    writeLock.lock();
+    try {
+      // Check if the queue is accepting jobs
+      if (getState() != QueueState.RUNNING) {
+        String msg = "Queue " + getQueuePath()
+            + " is STOPPED. Cannot accept submission of application: "
+            + applicationId;
+        LOG.info(msg);
+        throw new AccessControlException(msg);
+      }
+
+      // Check submission limits for queues
+      //TODO recalculate max applications because they can depend on capacity
+      if (getNumApplications() >= getMaxApplications() &&
+          !(this instanceof AutoCreatedLeafQueue)) {
+        String msg =
+            "Queue " + getQueuePath() + " already has " + getNumApplications()
+                + " applications,"
+                + " cannot accept submission of application: " + applicationId;
+        LOG.info(msg);
+        throw new AccessControlException(msg);
+      }
+
+      // Check submission limits for the user on this queue
+      User user = usersManager.getUserAndAddIfAbsent(userName);
+      //TODO recalculate max applications because they can depend on capacity
+      if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&
+          !(this instanceof AutoCreatedLeafQueue)) {
+        String msg = "Queue " + getQueuePath() + " already has " + user
+            .getTotalApplications() + " applications from user " + userName
+            + " cannot accept submission of application: " + applicationId;
+        LOG.info(msg);
+        throw new AccessControlException(msg);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+
+    try {
+      parent.validateSubmitApplication(applicationId, userName, queue);
+    } catch (AccessControlException ace) {
+      LOG.info("Failed to submit application to parent-queue: " +
+          parent.getQueuePath(), ace);
+      throw ace;
+    }
+  }
+
+  public Resource getAMResourceLimit() {
+    return usageTracker.getQueueUsage().getAMLimit();
+  }
+
+  public Resource getAMResourceLimitPerPartition(String nodePartition) {
+    return usageTracker.getQueueUsage().getAMLimit(nodePartition);
+  }
+
+  @VisibleForTesting
+  public Resource calculateAndGetAMResourceLimit() {
+    return calculateAndGetAMResourceLimitPerPartition(
+        RMNodeLabelsManager.NO_LABEL);
+  }
+
+  @VisibleForTesting
+  public Resource getUserAMResourceLimit() {
+    return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL,
+         null);
+  }
+
+  public Resource getUserAMResourceLimitPerPartition(
+      String nodePartition, String userName) {
+    float userWeight = 1.0f;
+    if (userName != null && getUser(userName) != null) {
+      userWeight = getUser(userName).getWeight();
+    }
+
+    readLock.lock();
+    try {
+      /*
+       * The user am resource limit is based on the same approach as the user
+       * limit (as it should represent a subset of that). This means that it uses
+       * the absolute queue capacity (per partition) instead of the max and is
+       * modified by the userlimit and the userlimit factor as is the userlimit
+       */
+      float effectiveUserLimit = Math.max(usersManager.getUserLimit() / 100.0f,
+          1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
+      float preWeightedUserLimit = effectiveUserLimit;
+      effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
+
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
+
+      Resource minimumAllocation = queueAllocationSettings.getMinimumAllocation();
+
+      Resource userAMLimit = Resources.multiplyAndNormalizeUp(
+          resourceCalculator, queuePartitionResource,
+          queueCapacities.getMaxAMResourcePercentage(nodePartition)
+              * effectiveUserLimit * usersManager.getUserLimitFactor(),
+          minimumAllocation);
+
+      if (getUserLimitFactor() == -1) {
+        userAMLimit = Resources.multiplyAndNormalizeUp(
+            resourceCalculator, queuePartitionResource,
+            queueCapacities.getMaxAMResourcePercentage(nodePartition),
+            minimumAllocation);
+      }
+
+      userAMLimit =
+          Resources.min(resourceCalculator, lastClusterResource,
+              userAMLimit,
+              Resources.clone(getAMResourceLimitPerPartition(nodePartition)));
+
+      Resource preWeighteduserAMLimit =
+          Resources.multiplyAndNormalizeUp(
+              resourceCalculator, queuePartitionResource,
+              queueCapacities.getMaxAMResourcePercentage(nodePartition)
+              * preWeightedUserLimit * usersManager.getUserLimitFactor(),
+              minimumAllocation);
+
+      if (getUserLimitFactor() == -1) {
+        preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp(
+            resourceCalculator, queuePartitionResource,
+            queueCapacities.getMaxAMResourcePercentage(nodePartition),
+            minimumAllocation);
+      }
+
+      preWeighteduserAMLimit =
+          Resources.min(resourceCalculator, lastClusterResource,
+              preWeighteduserAMLimit,
+              Resources.clone(getAMResourceLimitPerPartition(nodePartition)));
+      usageTracker.getQueueUsage().setUserAMLimit(nodePartition, preWeighteduserAMLimit);
+
+      LOG.debug("Effective user AM limit for \"{}\":{}. Effective weighted"
+          + " user AM limit: {}. User weight: {}", userName,
+          preWeighteduserAMLimit, userAMLimit, userWeight);
+      return userAMLimit;
+    } finally {
+      readLock.unlock();
+    }
+
+  }
+
+  public Resource calculateAndGetAMResourceLimitPerPartition(
+      String nodePartition) {
+    writeLock.lock();
+    try {
+      /*
+       * For non-labeled partition, get the max value from resources currently
+       * available to the queue and the absolute resources guaranteed for the
+       * partition in the queue. For labeled partition, consider only the absolute
+       * resources guaranteed. Multiply this value (based on labeled/
+       * non-labeled), * with per-partition am-resource-percent to get the max am
+       * resource limit for this queue and partition.
+       */
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
+
+      Resource queueCurrentLimit = Resources.none();
+      // For non-labeled partition, we need to consider the current queue
+      // usage limit.
+      if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
+        synchronized (queueResourceLimitsInfo){
+          queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
+        }
+      }
+
+      float amResourcePercent = queueCapacities.getMaxAMResourcePercentage(
+          nodePartition);
+
+      // Current usable resource for this queue and partition is the max of
+      // queueCurrentLimit and queuePartitionResource.
+      // If any of the resources available to this queue are less than queue's
+      // guarantee, use the guarantee as the queuePartitionUsableResource
+      // because nothing less than the queue's guarantee should be used when
+      // calculating the AM limit.
+      Resource queuePartitionUsableResource = (Resources.fitsIn(
+          resourceCalculator, queuePartitionResource, queueCurrentLimit)) ?
+          queueCurrentLimit : queuePartitionResource;
+
+      Resource amResouceLimit = Resources.multiplyAndNormalizeUp(
+          resourceCalculator, queuePartitionUsableResource, amResourcePercent,
+          queueAllocationSettings.getMinimumAllocation());
+
+      usageTracker.getMetrics().setAMResouceLimit(nodePartition, amResouceLimit);
+      usageTracker.getQueueUsage().setAMLimit(nodePartition, amResouceLimit);
+      LOG.debug("Queue: {}, node label : {}, queue partition resource : {},"
+          + " queue current limit : {}, queue partition usable resource : {},"
+          + " amResourceLimit : {}", getQueuePath(), nodePartition,
+          queuePartitionResource, queueCurrentLimit,
+          queuePartitionUsableResource, amResouceLimit);
+      return amResouceLimit;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  protected void activateApplications() {
+    writeLock.lock();
+    try {
+      // limit of allowed resource usage for application masters
+      Map<String, Resource> userAmPartitionLimit =
+          new HashMap<String, Resource>();
+
+      // AM Resource Limit for accessible labels can be pre-calculated.
+      // This will help in updating AMResourceLimit for all labels when queue
+      // is initialized for the first time (when no applications are present).
+      for (String nodePartition : getNodeLabelsForQueue()) {
+        calculateAndGetAMResourceLimitPerPartition(nodePartition);
+      }
+
+      for (Iterator<FiCaSchedulerApp> fsApp = getPendingAppsOrderingPolicy()
+               .getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR);
+           fsApp.hasNext();) {
+        FiCaSchedulerApp application = fsApp.next();
+        ApplicationId applicationId = application.getApplicationId();
+
+        // Get the am-node-partition associated with each application
+        // and calculate max-am resource limit for this partition.
+        String partitionName = application.getAppAMNodePartitionName();
+
+        Resource amLimit = getAMResourceLimitPerPartition(partitionName);
+        // Verify whether we already calculated am-limit for this label.
+        if (amLimit == null) {
+          amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
+        }
+        // Check am resource limit.
+        Resource amIfStarted = Resources.add(
+            application.getAMResource(partitionName),
+            usageTracker.getQueueUsage().getAMUsed(partitionName));
+
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("application " + application.getId() + " AMResource "
+              + application.getAMResource(partitionName)
+              + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent
+              + " amLimit " + amLimit + " lastClusterResource "
+              + lastClusterResource + " amIfStarted " + amIfStarted
+              + " AM node-partition name " + partitionName);
+        }
+
+        if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) {
+          if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
+              resourceCalculator, lastClusterResource,
+              usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) {
+            LOG.warn("maximum-am-resource-percent is insufficient to start a"
+                + " single application in queue, it is likely set too low."
+                + " skipping enforcement to allow at least one application"
+                + " to start");
+          } else{
+            application.updateAMContainerDiagnostics(
+                SchedulerApplicationAttempt.AMState.INACTIVATED,
+                CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
+            LOG.debug("Not activating application {} as  amIfStarted: {}"
+                + " exceeds amLimit: {}", applicationId, amIfStarted, amLimit);
+            continue;
+          }
+        }
+
+        // Check user am resource limit
+        User user = usersManager.getUserAndAddIfAbsent(application.getUser());
+        Resource userAMLimit = userAmPartitionLimit.get(partitionName);
+
+        // Verify whether we already calculated user-am-limit for this label.
+        if (userAMLimit == null) {
+          userAMLimit = getUserAMResourceLimitPerPartition(partitionName,
+              application.getUser());
+          userAmPartitionLimit.put(partitionName, userAMLimit);
+        }
+
+        Resource userAmIfStarted = Resources.add(
+            application.getAMResource(partitionName),
+            user.getConsumedAMResources(partitionName));
+
+        if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) {
+          if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
+              resourceCalculator, lastClusterResource,
+              usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) {
+            LOG.warn("maximum-am-resource-percent is insufficient to start a"
+                + " single application in queue for user, it is likely set too"
+                + " low. skipping enforcement to allow at least one application"
+                + " to start");
+          } else{
+            application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+                CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
+            LOG.debug("Not activating application {} for user: {} as"
+                + " userAmIfStarted: {} exceeds userAmLimit: {}",
+                applicationId, user, userAmIfStarted, userAMLimit);
+            continue;
+          }
+        }
+        user.activateApplication();
+        orderingPolicy.addSchedulableEntity(application);
+        application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
+
+        usageTracker.getQueueUsage().incAMUsed(partitionName,
+            application.getAMResource(partitionName));
+        user.getResourceUsage().incAMUsed(partitionName,
+            application.getAMResource(partitionName));
+        user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
+        usageTracker.getMetrics().incAMUsed(partitionName, application.getUser(),
+            application.getAMResource(partitionName));
+        usageTracker.getMetrics().setAMResouceLimitForUser(partitionName,
+            application.getUser(), userAMLimit);
+        fsApp.remove();
+        LOG.info("Application " + applicationId + " from user: " + application
+            .getUser() + " activated in queue: " + getQueuePath());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void addApplicationAttempt(FiCaSchedulerApp application,
+      User user) {
+    writeLock.lock();
+    try {
+      applicationAttemptMap.put(application.getApplicationAttemptId(),
+          application);
+
+      if (application.isRunnable()) {
+        runnableApps.add(application);
+        LOG.debug("Adding runnable application: {}",
+            application.getApplicationAttemptId());
+      } else {
+        nonRunnableApps.add(application);
+        LOG.info("Application attempt {} is not runnable,"
+            + " parallel limit reached", application.getApplicationAttemptId());
+        return;
+      }
+
+      // Accept
+      user.submitApplication();
+      getPendingAppsOrderingPolicy().addSchedulableEntity(application);
+
+      // Activate applications
+      if (Resources.greaterThan(resourceCalculator, lastClusterResource,
+          lastClusterResource, Resources.none())) {
+        activateApplications();
+      } else {
+        application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+            CSAMContainerLaunchDiagnosticsConstants.CLUSTER_RESOURCE_EMPTY);
+        LOG.info("Skipping activateApplications for "
+            + application.getApplicationAttemptId()
+            + " since cluster resource is " + Resources.none());
+      }
+
+      LOG.info(
+          "Application added -" + " appId: " + application.getApplicationId()
+              + " user: " + application.getUser() + "," + " leaf-queue: "
+              + getQueuePath() + " #user-pending-applications: " + user
+              .getPendingApplications() + " #user-active-applications: " + user
+              .getActiveApplications() + " #queue-pending-applications: "
+              + getNumPendingApplications() + " #queue-active-applications: "
+              + getNumActiveApplications()
+              + " #queue-nonrunnable-applications: "
+              + getNumNonRunnableApps());
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void finishApplication(ApplicationId application, String user) {
+    // Inform the activeUsersManager
+    usersManager.deactivateApplication(user, application);
+
+    appFinished();
+
+    // Inform the parent queue
+    parent.finishApplication(application, user);
+  }
+
+  @Override
+  public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) {
+    // Careful! Locking order is important!
+    removeApplicationAttempt(application, application.getUser());
+    parent.finishApplicationAttempt(application, queue);
+  }
+
+  private void removeApplicationAttempt(
+      FiCaSchedulerApp application, String userName) {
+
+    writeLock.lock();
+    try {
+      // TODO, should use getUser, use this method just to avoid UT failure
+      // which is caused by wrong invoking order, will fix UT separately
+      User user = usersManager.getUserAndAddIfAbsent(userName);
+
+      boolean runnable = runnableApps.remove(application);
+      if (!runnable) {
+        // removeNonRunnableApp acquires the write lock again, which is fine
+        if (!removeNonRunnableApp(application)) {
+          LOG.error("Given app to remove " + application +
+              " does not exist in queue " + getQueuePath());
+        }
+      }
+
+      String partitionName = application.getAppAMNodePartitionName();
+      boolean wasActive = orderingPolicy.removeSchedulableEntity(application);
+      if (!wasActive) {
+        pendingOrderingPolicy.removeSchedulableEntity(application);
+      } else{
+        usageTracker.getQueueUsage().decAMUsed(partitionName,
+            application.getAMResource(partitionName));
+        user.getResourceUsage().decAMUsed(partitionName,
+            application.getAMResource(partitionName));
+        usageTracker.getMetrics().decAMUsed(partitionName, application.getUser(),
+            application.getAMResource(partitionName));
+      }
+      applicationAttemptMap.remove(application.getApplicationAttemptId());
+
+      user.finishApplication(wasActive);
+      if (user.getTotalApplications() == 0) {
+        usersManager.removeUser(application.getUser());
+      }
+
+      // Check if we can activate more applications
+      activateApplications();
+
+      LOG.info(
+          "Application removed -" + " appId: " + application.getApplicationId()
+              + " user: " + application.getUser() + " queue: " + getQueuePath()
+              + " #user-pending-applications: " + user.getPendingApplications()
+              + " #user-active-applications: " + user.getActiveApplications()
+              + " #queue-pending-applications: " + getNumPendingApplications()
+              + " #queue-active-applications: " + getNumActiveApplications());
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private FiCaSchedulerApp getApplication(
+      ApplicationAttemptId applicationAttemptId) {
+    return applicationAttemptMap.get(applicationAttemptId);
+  }
+
+  private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
+    // Set preemption-allowed:
+    // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
+    if (!usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition)
+        .equals(Resources.none())) {
+      limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
+          queueContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition),
+          usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition)));
+      return;
+    }
+
+    float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
+    float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
+    limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
+  }
+
+  private CSAssignment allocateFromReservedContainer(Resource clusterResource,
+      CandidateNodeSet<FiCaSchedulerNode> candidates,
+      ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
+
+    // Irrespective of Single / Multi Node Placement, the allocate from
+    // Reserved Container has to happen only for the single node which
+    // CapacityScheduler#allocateFromReservedContainer invokes with.
+    // Else In Multi Node Placement, there won't be any Allocation or
+    // Reserve of new containers when there is a RESERVED container on
+    // a node which is full.
+    FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
+    if (node != null) {
+      RMContainer reservedContainer = node.getReservedContainer();
+      if (reservedContainer != null) {
+        FiCaSchedulerApp application = getApplication(
+            reservedContainer.getApplicationAttemptId());
+
+        if (null != application) {
+          ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
+              node, SystemClock.getInstance().getTime(), application);
+          CSAssignment assignment = application.assignContainers(
+              clusterResource, candidates, currentResourceLimits,
+              schedulingMode, reservedContainer);
+          return assignment;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private ConcurrentMap<String, CachedUserLimit> getUserLimitCache(
+      String partition,
+      SchedulingMode schedulingMode) {
+    synchronized (userLimitsCache) {
+      long latestVersion = usersManager.getLatestVersionOfUsersState();
+
+      if (latestVersion != this.currentUserLimitCacheVersion) {
+        // User limits cache needs invalidating
+        this.currentUserLimitCacheVersion = latestVersion;
+        userLimitsCache.clear();
+
+        Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>
+            uLCByPartition = new HashMap<>();
+        userLimitsCache.put(partition, uLCByPartition);
+
+        ConcurrentMap<String, CachedUserLimit> uLCBySchedulingMode =
+            new ConcurrentHashMap<>();
+        uLCByPartition.put(schedulingMode, uLCBySchedulingMode);
+
+        return uLCBySchedulingMode;
+      }
+
+      // User limits cache does not need invalidating
+      Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>
+          uLCByPartition = userLimitsCache.get(partition);
+      if (uLCByPartition == null) {
+        uLCByPartition = new HashMap<>();
+        userLimitsCache.put(partition, uLCByPartition);
+      }
+
+      ConcurrentMap<String, CachedUserLimit> uLCBySchedulingMode =
+          uLCByPartition.get(schedulingMode);
+      if (uLCBySchedulingMode == null) {
+        uLCBySchedulingMode = new ConcurrentHashMap<>();
+        uLCByPartition.put(schedulingMode, uLCBySchedulingMode);
+      }
+
+      return uLCBySchedulingMode;
+    }
+  }
+
+  @Override
+  public CSAssignment assignContainers(Resource clusterResource,
+      CandidateNodeSet<FiCaSchedulerNode> candidates,
+      ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
+    updateCurrentResourceLimits(currentResourceLimits, clusterResource);
+    FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("assignContainers: partition=" + candidates.getPartition()
+          + " #applications=" + orderingPolicy.getNumSchedulableEntities());
+    }
+
+    setPreemptionAllowed(currentResourceLimits, candidates.getPartition());
+
+    // Check for reserved resources, try to allocate reserved container first.
+    CSAssignment assignment = allocateFromReservedContainer(clusterResource,
+        candidates, currentResourceLimits, schedulingMode);
+    if (null != assignment) {
+      return assignment;
+    }
+
+    // if our queue cannot access this node, just return
+    if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
+        && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) {
+      ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+          parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED,
+          ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION);
+      return CSAssignment.NULL_ASSIGNMENT;
+    }
+
+    // Check if this queue need more resource, simply skip allocation if this
+    // queue doesn't need more resources.
+    if (!hasPendingResourceRequest(candidates.getPartition(), clusterResource,
+        schedulingMode)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Skip this queue=" + getQueuePath()
+            + ", because it doesn't need more resource, schedulingMode="
+            + schedulingMode.name() + " node-partition=" + candidates
+            .getPartition());
+      }
+      ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+          parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
+          ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
+      return CSAssignment.NULL_ASSIGNMENT;
+    }
+
+    ConcurrentMap<String, CachedUserLimit> userLimits =
+        this.getUserLimitCache(candidates.getPartition(), schedulingMode);
+    boolean needAssignToQueueCheck = true;
+    IteratorSelector sel = new IteratorSelector();
+    sel.setPartition(candidates.getPartition());
+    for (Iterator<FiCaSchedulerApp> assignmentIterator = orderingPolicy.getAssignmentIterator(sel);
+         assignmentIterator.hasNext();) {
+      FiCaSchedulerApp application = assignmentIterator.next();
+
+      ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
+          node, SystemClock.getInstance().getTime(), application);
+
+      // Check queue max-capacity limit
+      Resource appReserved = application.getCurrentReservation();
+      if (needAssignToQueueCheck) {
+        if (!super.canAssignToThisQueue(clusterResource,
+            candidates.getPartition(), currentResourceLimits, appReserved,
+            schedulingMode)) {
+          ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
+              activitiesManager, node, application, application.getPriority(),
+              ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT);
+          ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+              parent.getQueuePath(), getQueuePath(),
+              ActivityState.REJECTED,
+              ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT);
+          return CSAssignment.NULL_ASSIGNMENT;
+        }
+        // If there was no reservation and canAssignToThisQueue returned
+        // true, there is no reason to check further.
+        if (!this.reservationsContinueLooking
+            || appReserved.equals(Resources.none())) {
+          needAssignToQueueCheck = false;
+        }
+      }
+
+      CachedUserLimit cul = userLimits.get(application.getUser());
+      Resource cachedUserLimit = null;
+      if (cul != null) {
+        cachedUserLimit = cul.userLimit;
+      }
+      Resource userLimit = computeUserLimitAndSetHeadroom(application,
+          clusterResource, candidates.getPartition(), schedulingMode,
+          cachedUserLimit);
+      if (cul == null) {
+        cul = new CachedUserLimit(userLimit);
+        CachedUserLimit retVal =
+            userLimits.putIfAbsent(application.getUser(), cul);
+        if (retVal != null) {
+          // another thread updated the user limit cache before us
+          cul = retVal;
+          userLimit = cul.userLimit;
+        }
+      }
+      // Check user limit
+      boolean userAssignable = true;
+      if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) {
+        userAssignable = false;
+      } else {
+        userAssignable = canAssignToUser(clusterResource, application.getUser(),
+            userLimit, application, candidates.getPartition(),
+            currentResourceLimits);
+        if (!userAssignable && Resources.fitsIn(cul.reservation, appReserved)) {
+          cul.canAssign = false;
+          cul.reservation = appReserved;
+        }
+      }
+      if (!userAssignable) {
+        application.updateAMContainerDiagnostics(AMState.ACTIVATED,
+            "User capacity has reached its maximum limit.");
+        ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
+            activitiesManager, node, application, application.getPriority(),
+            ActivityDiagnosticConstant.QUEUE_HIT_USER_MAX_CAPACITY_LIMIT);
+        continue;
+      }
+
+      // Try to schedule
+      assignment = application.assignContainers(clusterResource,
+          candidates, currentResourceLimits, schedulingMode, null);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("post-assignContainers for application " + application
+            .getApplicationId());
+        application.showRequests();
+      }
+
+      // Did we schedule or reserve a container?
+      Resource assigned = assignment.getResource();
+
+      if (Resources.greaterThan(resourceCalculator, clusterResource, assigned,
+          Resources.none())) {
+        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+            parent.getQueuePath(), getQueuePath(),
+            ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
+        return assignment;
+      } else if (assignment.getSkippedType()
+          == CSAssignment.SkippedType.OTHER) {
+        ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
+            activitiesManager, application.getApplicationId(),
+            ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
+        application.updateNodeInfoForAMDiagnostics(node);
+      } else if (assignment.getSkippedType()
+          == CSAssignment.SkippedType.QUEUE_LIMIT) {
+        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+            parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED,
+            () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM
+                + " from " + application.getApplicationId());
+        return assignment;
+      } else{
+        // If we don't allocate anything, and it is not skipped by application,
+        // we will return to respect FIFO of applications
+        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+            parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
+            ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO);
+        ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
+            activitiesManager, application.getApplicationId(),
+            ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
+        return CSAssignment.NULL_ASSIGNMENT;
+      }
+    }
+    ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+        parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
+        ActivityDiagnosticConstant.EMPTY);
+
+    return CSAssignment.NULL_ASSIGNMENT;
+  }
+
+  @Override
+  public boolean accept(Resource cluster,
+      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
+    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation =
+        request.getFirstAllocatedOrReservedContainer();
+    SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer =
+        allocation.getAllocatedOrReservedContainer();
+
+    // Do not check limits when allocation from a reserved container
+    if (allocation.getAllocateFromReservedContainer() == null) {
+      readLock.lock();
+      try {
+        FiCaSchedulerApp app =
+            schedulerContainer.getSchedulerApplicationAttempt();
+        String username = app.getUser();
+        String p = schedulerContainer.getNodePartition();
+
+        // check user-limit
+        Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p,
+            allocation.getSchedulingMode(), null);
+
+        // Deduct resources that we can release
+        User user = getUser(username);
+        if (user == null) {
+          LOG.debug("User {} has been removed!", username);
+          return false;
+        }
+        Resource usedResource = Resources.clone(user.getUsed(p));
+        Resources.subtractFrom(usedResource,
+            request.getTotalReleasedResource());
+
+        if (Resources.greaterThan(resourceCalculator, cluster, usedResource,
+            userLimit)) {
+          LOG.debug("Used resource={} exceeded user-limit={}",
+              usedResource, userLimit);
+          return false;
+        }
+      } finally {
+        readLock.unlock();
+      }
+    }
+
+    return super.accept(cluster, request);
+  }
+
+  private void internalReleaseContainer(Resource clusterResource,
+      SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
+    RMContainer rmContainer = schedulerContainer.getRmContainer();
+
+    AbstractLeafQueue targetLeafQueue =
+        schedulerContainer.getSchedulerApplicationAttempt().getCSLeafQueue();
+
+    if (targetLeafQueue == this) {
+      // When trying to preempt containers from the same queue
+      if (rmContainer.getState() == RMContainerState.RESERVED) {
+        // For other reserved containers
+        // This is a reservation exchange, complete previous reserved container
+        completedContainer(clusterResource,
+            schedulerContainer.getSchedulerApplicationAttempt(),
+            schedulerContainer.getSchedulerNode(), rmContainer, SchedulerUtils
+                .createAbnormalContainerStatus(rmContainer.getContainerId(),
+                    SchedulerUtils.UNRESERVED_CONTAINER),
+            RMContainerEventType.RELEASED, null, false);
+      }
+    } else{
+      // When trying to preempt containers from different queue -- this
+      // is for lazy preemption feature (kill preemption candidate in scheduling
+      // cycle).
+      targetLeafQueue.completedContainer(clusterResource,
+          schedulerContainer.getSchedulerApplicationAttempt(),
+          schedulerContainer.getSchedulerNode(),
+          schedulerContainer.getRmContainer(), SchedulerUtils
+              .createPreemptedContainerStatus(rmContainer.getContainerId(),
+                  SchedulerUtils.PREEMPTED_CONTAINER),
+          RMContainerEventType.KILL, null, false);
+    }
+  }
+
+  private void releaseContainers(Resource clusterResource,
+      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
+    for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> c : request
+        .getContainersToRelease()) {
+      internalReleaseContainer(clusterResource, c);
+    }
+
+    // Handle container reservation looking, or lazy preemption case:
+    if (null != request.getContainersToAllocate() && !request
+        .getContainersToAllocate().isEmpty()) {
+      for (ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> context : request
+          .getContainersToAllocate()) {
+        if (null != context.getToRelease()) {
+          for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> c : context
+              .getToRelease()) {
+            internalReleaseContainer(clusterResource, c);
+          }
+        }
+      }
+    }
+  }
+
+  public void apply(Resource cluster,
+      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
+    // Do we need to call parent queue's apply?
+    boolean applyToParentQueue = false;
+
+    releaseContainers(cluster, request);
+
+    writeLock.lock();
+    try {
+      if (request.anythingAllocatedOrReserved()) {
+        ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>
+            allocation = request.getFirstAllocatedOrReservedContainer();
+        SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+            schedulerContainer = allocation.getAllocatedOrReservedContainer();
+
+        // Do not modify queue when allocation from reserved container
+        if (allocation.getAllocateFromReservedContainer() == null) {
+          // Only invoke apply() of ParentQueue when new allocation /
+          // reservation happen.
+          applyToParentQueue = true;
+          // Book-keeping
+          // Note: Update headroom to account for current allocation too...
+          allocateResource(cluster,
+              schedulerContainer.getSchedulerApplicationAttempt(),
+              allocation.getAllocatedOrReservedResource(),
+              schedulerContainer.getNodePartition(),
+              schedulerContainer.getRmContainer());
+          orderingPolicy.containerAllocated(
+              schedulerContainer.getSchedulerApplicationAttempt(),
+              schedulerContainer.getRmContainer());
+        }
+
+        // Update reserved resource
+        if (Resources.greaterThan(resourceCalculator, cluster,
+            request.getTotalReservedResource(), Resources.none())) {
+          incReservedResource(schedulerContainer.getNodePartition(),
+              request.getTotalReservedResource());
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+
+    if (parent != null && applyToParentQueue) {
+      parent.apply(cluster, request);
+    }
+  }
+
+
+  protected Resource getHeadroom(User user, Resource queueCurrentLimit,
+      Resource clusterResource, FiCaSchedulerApp application) {
+    return getHeadroom(user, queueCurrentLimit, clusterResource, application,
+        RMNodeLabelsManager.NO_LABEL);
+  }
+
+  protected Resource getHeadroom(User user, Resource queueCurrentLimit,
+      Resource clusterResource, FiCaSchedulerApp application,
+      String partition) {
+    return getHeadroom(user, queueCurrentLimit, clusterResource,
+        getResourceLimitForActiveUsers(application.getUser(), clusterResource,
+            partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+        partition);
+  }
+
+  private Resource getHeadroom(User user,
+      Resource currentPartitionResourceLimit, Resource clusterResource,
+      Resource userLimitResource, String partition) {
+    /**
+     * Headroom is:
+     *    min(
+     *        min(userLimit, queueMaxCap) - userConsumed,
+     *        queueMaxCap - queueUsedResources
+     *       )
+     *
+     * ( which can be expressed as,
+     *  min (userLimit - userConsumed, queuMaxCap - userConsumed,
+     *    queueMaxCap - queueUsedResources)
+     *  )
+     *
+     * given that queueUsedResources >= userConsumed, this simplifies to
+     *
+     * >> min (userlimit - userConsumed,   queueMaxCap - queueUsedResources) <<
+     *
+     * sum of queue max capacities of multiple queue's will be greater than the
+     * actual capacity of a given partition, hence we need to ensure that the
+     * headroom is not greater than the available resource for a given partition
+     *
+     * headroom = min (unused resourcelimit of a label, calculated headroom )
+     */
+    currentPartitionResourceLimit =
+        partition.equals(RMNodeLabelsManager.NO_LABEL)
+            ? currentPartitionResourceLimit
+            : getQueueMaxResource(partition);
+
+    Resource headroom = Resources.componentwiseMin(
+        Resources.subtractNonNegative(userLimitResource,
+            user.getUsed(partition)),
+        Resources.subtractNonNegative(currentPartitionResourceLimit,
+            usageTracker.getQueueUsage().getUsed(partition)));
+    // Normalize it before return
+    headroom =
+        Resources.roundDown(resourceCalculator, headroom,
+            queueAllocationSettings.getMinimumAllocation());
+
+    //headroom = min (unused resourcelimit of a label, calculated headroom )
+    Resource clusterPartitionResource =
+        labelManager.getResourceByLabel(partition, clusterResource);
+    Resource clusterFreePartitionResource =
+        Resources.subtract(clusterPartitionResource,
+            queueContext.getClusterResourceUsage().getUsed(partition));
+    headroom = Resources.min(resourceCalculator, clusterPartitionResource,
+        clusterFreePartitionResource, headroom);
+    return headroom;
+  }
+
+  private void setQueueResourceLimitsInfo(
+      Resource clusterResource) {
+    synchronized (queueResourceLimitsInfo) {
+      queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom
+          .getLimit());
+      queueResourceLimitsInfo.setClusterResource(clusterResource);
+    }
+  }
+
+  // It doesn't necessarily to hold application's lock here.
+  @Lock({AbstractLeafQueue.class})
+  Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
+      Resource clusterResource, String nodePartition,
+      SchedulingMode schedulingMode, Resource userLimit) {
+    String user = application.getUser();
+    User queueUser = getUser(user);
+    if (queueUser == null) {
+      LOG.debug("User {} has been removed!", user);
+      return Resources.none();
+    }
+
+    // Compute user limit respect requested labels,
+    // TODO, need consider headroom respect labels also
+    if (userLimit == null) {
+      userLimit = getResourceLimitForActiveUsers(application.getUser(),
+          clusterResource, nodePartition, schedulingMode);
+    }
+    setQueueResourceLimitsInfo(clusterResource);
+
+    Resource headroom =
+        usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() :
+            getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(),
+                clusterResource, userLimit, nodePartition);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Headroom calculation for user " + user + ": " + " userLimit="
+          + userLimit + " queueMaxAvailRes="
+          + cachedResourceLimitsForHeadroom.getLimit() + " consumed="
+          + queueUser.getUsed() + " partition="
+          + nodePartition);
+    }
+
+    CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider(
+        queueUser, this, application, queueResourceLimitsInfo);
+
+    application.setHeadroomProvider(headroomProvider);
+
+    usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, user, headroom);
+
+    return userLimit;
+  }
+
+  @Lock(NoLock.class)
+  public int getNodeLocalityDelay() {
+    return nodeLocalityDelay;
+  }
+
+  @Lock(NoLock.class)
+  public int getRackLocalityAdditionalDelay() {
+    return rackLocalityAdditionalDelay;
+  }
+
+  @Lock(NoLock.class)
+  public boolean getRackLocalityFullReset() {
+    return rackLocalityFullReset;
+  }
+
+  /**
+   *
+   * @param userName
+   *          Name of user who has submitted one/more app to given queue.
+   * @param clusterResource
+   *          total cluster resource
+   * @param nodePartition
+   *          partition name
+   * @param schedulingMode
+   *          scheduling mode
+   *          RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY
+   * @return Computed User Limit
+   */
+  public Resource getResourceLimitForActiveUsers(String userName,
+      Resource clusterResource, String nodePartition,
+      SchedulingMode schedulingMode) {
+    return usersManager.getComputedResourceLimitForActiveUsers(userName,
+        clusterResource, nodePartition, schedulingMode);
+  }
+
+  /**
+   *
+   * @param userName
+   *          Name of user who has submitted one/more app to given queue.
+   * @param clusterResource
+   *          total cluster resource
+   * @param nodePartition
+   *          partition name
+   * @param schedulingMode
+   *          scheduling mode
+   *          RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY
+   * @return Computed User Limit
+   */
+  public Resource getResourceLimitForAllUsers(String userName,
+      Resource clusterResource, String nodePartition,
+      SchedulingMode schedulingMode) {
+    return usersManager.getComputedResourceLimitForAllUsers(userName,
+        clusterResource, nodePartition, schedulingMode);
+  }
+
+  @Private
+  protected boolean canAssignToUser(Resource clusterResource,
+      String userName, Resource limit, FiCaSchedulerApp application,
+      String nodePartition, ResourceLimits currentResourceLimits) {
+
+    readLock.lock();
+    try {
+      User user = getUser(userName);
+      if (user == null) {
+        LOG.debug("User {} has been removed!", userName);
+        return false;
+      }
+
+      currentResourceLimits.setAmountNeededUnreserve(Resources.none());
+
+      // Note: We aren't considering the current request since there is a fixed
+      // overhead of the AM, but it's a > check, not a >= check, so...
+      if (Resources.greaterThan(resourceCalculator, clusterResource,
+          user.getUsed(nodePartition), limit)) {
+        // if enabled, check to see if could we potentially use this node instead
+        // of a reserved node if the application has reserved containers
+        if (this.reservationsContinueLooking) {
+          if (Resources.lessThanOrEqual(resourceCalculator, clusterResource,
+              Resources.subtract(user.getUsed(),
+                  application.getCurrentReservation()), limit)) {
+
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("User " + userName + " in queue " + getQueuePath()
+                  + " will exceed limit based on reservations - "
+                  + " consumed: " + user.getUsed() + " reserved: " + application
+                  .getCurrentReservation() + " limit: " + limit);
+            }
+            Resource amountNeededToUnreserve = Resources.subtract(
+                user.getUsed(nodePartition), limit);
+            // we can only acquire a new container if we unreserve first to
+            // respect user-limit
+            currentResourceLimits.setAmountNeededUnreserve(
+                amountNeededToUnreserve);
+            return true;
+          }
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("User " + userName + " in queue " + getQueuePath()
+              + " will exceed limit - " + " consumed: " + user
+              .getUsed(nodePartition) + " limit: " + limit);
+        }
+        return false;
+      }
+      return true;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  protected void setDynamicQueueProperties() {
+    // set to -1, to disable it
+    queueContext.getConfiguration().setUserLimitFactor(getQueuePath(), -1);
+    // Set Max AM percentage to a higher value
+    queueContext.getConfiguration().setMaximumApplicationMasterResourcePerQueuePercent(
+        getQueuePath(), 1f);
+    super.setDynamicQueueProperties();
+  }
+
+  @Override
+  protected void setDynamicQueueACLProperties() {
+    super.setDynamicQueueACLProperties();
+
+    if (parent instanceof AbstractManagedParentQueue) {
+      acls.putAll(queueContext.getConfiguration().getACLsForLegacyAutoCreatedLeafQueue(
+          parent.getQueuePath()));
+    } else if (parent instanceof ParentQueue) {
+      acls.putAll(getACLsForFlexibleAutoCreatedLeafQueue(
+          ((ParentQueue) parent).getAutoCreatedQueueTemplate()));
+    }
+  }
+
+  private void updateSchedulerHealthForCompletedContainer(
+      RMContainer rmContainer, ContainerStatus containerStatus) {
+    // Update SchedulerHealth for released / preempted container
+    SchedulerHealth schedulerHealth = queueContext.getSchedulerHealth();
+    if (null == schedulerHealth) {
+      // Only do update if we have schedulerHealth
+      return;
+    }
+
+    if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) {
+      schedulerHealth.updatePreemption(Time.now(), rmContainer.getAllocatedNode(),
+          rmContainer.getContainerId(), getQueuePath());
+      schedulerHealth.updateSchedulerPreemptionCounts(1);
+    } else {
+      schedulerHealth.updateRelease(queueContext.getLastNodeUpdateTime(),
+          rmContainer.getAllocatedNode(), rmContainer.getContainerId(),
+          getQueuePath());
+    }
+  }
+
+  /**
+   * Recalculate QueueUsage Ratio.
+   *
+   * @param clusterResource
+   *          Total Cluster Resource
+   * @param nodePartition
+   *          Partition
+   */
+  public void recalculateQueueUsageRatio(Resource clusterResource,
+      String nodePartition) {
+    writeLock.lock();
+    try {
+      ResourceUsage queueResourceUsage = getQueueResourceUsage();
+
+      if (nodePartition == null) {
+        for (String partition : Sets.union(
+            getQueueCapacities().getExistingNodeLabels(),
+            queueResourceUsage.getExistingNodeLabels())) {
+          usersManager.updateUsageRatio(partition, clusterResource);
+        }
+      } else {
+        usersManager.updateUsageRatio(nodePartition, clusterResource);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void completedContainer(Resource clusterResource,
+      FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer,
+      ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
+      boolean sortQueues) {
+    // Update SchedulerHealth for released / preempted container
+    updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus);
+
+    if (application != null) {
+      boolean removed = false;
+
+      // Careful! Locking order is important!
+      writeLock.lock();
+      try {
+        Container container = rmContainer.getContainer();
+
+        // Inform the application & the node
+        // Note: It's safe to assume that all state changes to RMContainer
+        // happen under scheduler's lock...
+        // So, this is, in effect, a transaction across application & node
+        if (rmContainer.getState() == RMContainerState.RESERVED) {
+          removed = application.unreserve(rmContainer.getReservedSchedulerKey(),
+              node, rmContainer);
+        } else{
+          removed = application.containerCompleted(rmContainer, containerStatus,
+              event, node.getPartition());
+
+          node.releaseContainer(rmContainer.getContainerId(), false);
+        }
+
+        // Book-keeping
+        if (removed) {
+
+          // Inform the ordering policy
+          orderingPolicy.containerReleased(application, rmContainer);
+
+          releaseResource(clusterResource, application, container.getResource(),
+              node.getPartition(), rmContainer);
+        }
+      } finally {
+        writeLock.unlock();
+      }
+
+
+      if (removed) {
+        // Inform the parent queue _outside_ of the leaf-queue lock
+        parent.completedContainer(clusterResource, application, node,
+            rmContainer, null, event, this, sortQueues);
+      }
+    }
+
+    // Notify PreemptionManager
+    queueContext.getPreemptionManager().removeKillableContainer(
+        new KillableContainer(
+            rmContainer,
+            node.getPartition(),
+            getQueuePath()));
+
+    // Update preemption metrics if exit status is PREEMPTED
+    if (containerStatus != null
+        && ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) {
+      updateQueuePreemptionMetrics(rmContainer);
+    }
+  }
+
+  void allocateResource(Resource clusterResource,
+      SchedulerApplicationAttempt application, Resource resource,
+      String nodePartition, RMContainer rmContainer) {
+    writeLock.lock();
+    try {
+      super.allocateResource(clusterResource, resource, nodePartition);
+
+      // handle ignore exclusivity container
+      if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
+          RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
+          RMNodeLabelsManager.NO_LABEL)) {
+        TreeSet<RMContainer> rmContainers = ignorePartitionExclusivityRMContainers.computeIfAbsent(
+            nodePartition, k -> new TreeSet<>());
+        rmContainers.add(rmContainer);
+      }
+
+      // Update user metrics
+      String userName = application.getUser();
+
+      // Increment user's resource usage.
+      User user = usersManager.updateUserResourceUsage(userName, resource,
+          queueContext.getClusterResource(), nodePartition, true);
+
+      Resource partitionHeadroom = Resources.createResource(0, 0);
+      if (usageTracker.getMetrics().getUserMetrics(userName) != null) {
+        partitionHeadroom = getHeadroom(user,
+            cachedResourceLimitsForHeadroom.getLimit(), clusterResource,
+            getResourceLimitForActiveUsers(userName, clusterResource,
+                nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+            nodePartition);
+      }
+      usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName,
+          partitionHeadroom);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(getQueuePath() + " user=" + userName + " used="
+            + usageTracker.getQueueUsage().getUsed(nodePartition) + " numContainers="
+            + usageTracker.getNumContainers() + " headroom = " + application.getHeadroom()
+            + " user-resources=" + user.getUsed());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  void releaseResource(Resource clusterResource,
+      FiCaSchedulerApp application, Resource resource, String nodePartition,
+      RMContainer rmContainer) {
+    writeLock.lock();
+    try {
+      super.releaseResource(clusterResource, resource, nodePartition);
+
+      // handle ignore exclusivity container
+      if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
+          RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
+          RMNodeLabelsManager.NO_LABEL)) {
+        if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
+          Set<RMContainer> rmContainers =
+              ignorePartitionExclusivityRMContainers.get(nodePartition);
+          rmContainers.remove(rmContainer);
+          if (rmContainers.isEmpty()) {
+            ignorePartitionExclusivityRMContainers.remove(nodePartition);
+          }
+        }
+      }
+
+      // Update user metrics
+      String userName = application.getUser();
+      User user = usersManager.updateUserResourceUsage(userName, resource,
+          queueContext.getClusterResource(), nodePartition, false);
+
+      Resource partitionHeadroom = Resources.createResource(0, 0);
+      if (usageTracker.getMetrics().getUserMetrics(userName) != null) {
+        partitionHeadroom = getHeadroom(user,
+            cachedResourceLimitsForHeadroom.getLimit(), clusterResource,
+            getResourceLimitForActiveUsers(userName, clusterResource,
+                nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+            nodePartition);
+      }
+      usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName,
+          partitionHeadroom);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(
+            getQueuePath() + " used=" + usageTracker.getQueueUsage().getUsed() + " numContainers="
+                + usageTracker.getNumContainers() + " user=" + userName + " user-resources="
+                + user.getUsed());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void updateCurrentResourceLimits(
+      ResourceLimits currentResourceLimits, Resource clusterResource) {
+    // TODO: need consider non-empty node labels when resource limits supports
+    // node labels
+    // Even if ParentQueue will set limits respect child's max queue capacity,
+    // but when allocating reserved container, CapacityScheduler doesn't do
+    // this. So need cap limits by queue's max capacity here.
+    this.cachedResourceLimitsForHeadroom =
+        new ResourceLimits(currentResourceLimits.getLimit());
+    Resource queueMaxResource = getEffectiveMaxCapacityDown(
+        RMNodeLabelsManager.NO_LABEL, queueAllocationSettings.getMinimumAllocation());
+    this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
+        resourceCalculator, clusterResource, queueMaxResource,
+        currentResourceLimits.getLimit()));
+  }
+
+  @Override
+  public void updateClusterResource(Resource clusterResource,
+      ResourceLimits currentResourceLimits) {
+    writeLock.lock();
+    try {
+      lastClusterResource = clusterResource;
+
+      updateAbsoluteCapacities();
+
+      super.updateEffectiveResources(clusterResource);
+
+      // Update maximum applications for the queue and for users
+      updateMaximumApplications();
+
+      updateCurrentResourceLimits(currentResourceLimits, clusterResource);
+
+      // Update headroom info based on new cluster resource value
+      // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
+      // during allocation
+      setQueueResourceLimitsInfo(clusterResource);
+
+      // Update user consumedRatios
+      recalculateQueueUsageRatio(clusterResource, null);
+
+      // Update metrics
+      CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+          this, labelManager, null);
+      // Update configured capacity/max-capacity for default partition only
+      CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator,
+          labelManager.getResourceByLabel(null, clusterResource),
+          RMNodeLabelsManager.NO_LABEL, this);
+
+      // queue metrics are updated, more resource may be available
+      // activate the pending applications if possible
+      activateApplications();
+
+      // In case of any resource change, invalidate recalculateULCount to clear
+      // the computed user-limit.
+      usersManager.userLimitNeedsRecompute();
+
+      // Update application properties
+      for (FiCaSchedulerApp application : orderingPolicy
+          .getSchedulableEntities()) {
+        computeUserLimitAndSetHeadroom(application, clusterResource,
+            RMNodeLabelsManager.NO_LABEL,
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, null);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void incUsedResource(String nodeLabel, Resource resourceToInc,
+      SchedulerApplicationAttempt application) {
+    usersManager.updateUserResourceUsage(application.getUser(), resourceToInc,
+        queueContext.getClusterResource(), nodeLabel, true);
+    super.incUsedResource(nodeLabel, resourceToInc, application);
+  }
+
+  @Override
+  public void decUsedResource(String nodeLabel, Resource resourceToDec,
+      SchedulerApplicationAttempt application) {
+    usersManager.updateUserResourceUsage(application.getUser(), resourceToDec,
+        queueContext.getClusterResource(), nodeLabel, false);
+    super.decUsedResource(nodeLabel, resourceToDec, application);
+  }
+
+  public void incAMUsedResource(String nodeLabel, Resource resourceToInc,
+      SchedulerApplicationAttempt application) {
+    User user = getUser(application.getUser());
+    if (user == null) {
+      return;
+    }
+
+    user.getResourceUsage().incAMUsed(nodeLabel,
+        resourceToInc);
+    // ResourceUsage has its own lock, no addition lock needs here.
+    usageTracker.getQueueUsage().incAMUsed(nodeLabel, resourceToInc);
+  }
+
+  public void decAMUsedResource(String nodeLabel, Resource resourceToDec,
+      SchedulerApplicationAttempt application) {
+    User user = getUser(application.getUser());
+    if (user == null) {
+      return;
+    }
+
+    user.getResourceUsage().decAMUsed(nodeLabel,
+        resourceToDec);
+    // ResourceUsage has its own lock, no addition lock needs here.
+    usageTracker.getQueueUsage().decAMUsed(nodeLabel, resourceToDec);
+  }
+
+  @Override
+  public void recoverContainer(Resource clusterResource,
+      SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
+    if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
+      return;
+    }
+    if (rmContainer.getExecutionType() != ExecutionType.GUARANTEED) {
+      return;
+    }
+    // Careful! Locking order is important!
+    writeLock.lock();
+    try {
+      FiCaSchedulerNode node = queueContext.getNode(
+          rmContainer.getContainer().getNodeId());
+      allocateResource(clusterResource, attempt,
+          rmContainer.getContainer().getResource(), node.getPartition(),
+          rmContainer);
+    } finally {
+      writeLock.unlock();
+    }
+
+    parent.recoverContainer(clusterResource, attempt, rmContainer);
+  }
+
+  /**
+   * Obtain (read-only) collection of pending applications.
+   */
+  public Collection<FiCaSchedulerApp> getPendingApplications() {
+    return Collections.unmodifiableCollection(pendingOrderingPolicy
+        .getSchedulableEntities());
+  }
+
+  /**
+   * Obtain (read-only) collection of active applications.
+   */
+  public Collection<FiCaSchedulerApp> getApplications() {
+    return Collections.unmodifiableCollection(orderingPolicy
+        .getSchedulableEntities());
+  }
+
+  /**
+   * Obtain (read-only) collection of all applications.
+   */
+  public Collection<FiCaSchedulerApp> getAllApplications() {
+    Collection<FiCaSchedulerApp> apps = new HashSet<FiCaSchedulerApp>(
+        pendingOrderingPolicy.getSchedulableEntities());
+    apps.addAll(orderingPolicy.getSchedulableEntities());
+
+    return Collections.unmodifiableCollection(apps);
+  }
+
+  /**
+   * Get total pending resource considering user limit for the leaf queue. This
+   * will be used for calculating pending resources in the preemption monitor.
+   *
+   * Consider the headroom for each user in the queue.
+   * Total pending for the queue =
+   * sum(for each user(min((user's headroom), sum(user's pending requests))))
+   * NOTE:
+
+   * @param clusterResources clusterResource
+   * @param partition node partition
+   * @param deductReservedFromPending When a container is reserved in CS,
+   *                                  pending resource will not be deducted.
+   *                                  This could lead to double accounting when
+   *                                  doing preemption:
+   *                                  In normal cases, we should deduct reserved
+   *                                  resource from pending to avoid
+   *                                  excessive preemption.
+   * @return Total pending resource considering user limit
+   */
+  public Resource getTotalPendingResourcesConsideringUserLimit(
+      Resource clusterResources, String partition,
+      boolean deductReservedFromPending) {
+    readLock.lock();
+    try {
+      Map<String, Resource> userNameToHeadroom =
+          new HashMap<>();
+      Resource totalPendingConsideringUserLimit = Resource.newInstance(0, 0);
+      for (FiCaSchedulerApp app : getApplications()) {
+        String userName = app.getUser();
+        if (!userNameToHeadroom.containsKey(userName)) {
+          User user = getUsersManager().getUserAndAddIfAbsent(userName);
+          Resource headroom = Resources.subtract(
+              getResourceLimitForActiveUsers(app.getUser(), clusterResources,
+                  partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+              user.getUsed(partition));
+          // Make sure headroom is not negative.
+          headroom = Resources.componentwiseMax(headroom, Resources.none());
+          userNameToHeadroom.put(userName, headroom);
+        }
+
+        // Check if we need to deduct reserved from pending
+        Resource pending = app.getAppAttemptResourceUsage().getPending(
+            partition);
+        if (deductReservedFromPending) {
+          pending = Resources.subtract(pending,
+              app.getAppAttemptResourceUsage().getReserved(partition));
+        }
+        pending = Resources.componentwiseMax(pending, Resources.none());
+
+        Resource minpendingConsideringUserLimit = Resources.componentwiseMin(
+            userNameToHeadroom.get(userName), pending);
+        Resources.addTo(totalPendingConsideringUserLimit,
+            minpendingConsideringUserLimit);
+        Resources.subtractFrom(userNameToHeadroom.get(userName),
+            minpendingConsideringUserLimit);
+      }
+      return totalPendingConsideringUserLimit;
+    } finally {
+      readLock.unlock();
+    }
+
+  }
+
+  @Override
+  public void collectSchedulerApplications(
+      Collection<ApplicationAttemptId> apps) {
+    readLock.lock();
+    try {
+      for (FiCaSchedulerApp pendingApp : pendingOrderingPolicy
+          .getSchedulableEntities()) {
+        apps.add(pendingApp.getApplicationAttemptId());
+      }
+      for (FiCaSchedulerApp app : orderingPolicy.getSchedulableEntities()) {
+        apps.add(app.getApplicationAttemptId());
+      }
+    } finally {
+      readLock.unlock();
+    }
+
+  }
+
+  @Override
+  public void attachContainer(Resource clusterResource,
+      FiCaSchedulerApp application, RMContainer rmContainer) {
+    if (application != null && rmContainer != null
+        && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
+      FiCaSchedulerNode node =
+          queueContext.getNode(rmContainer.getContainer().getNodeId());
+      allocateResource(clusterResource, application, rmContainer.getContainer()
+          .getResource(), node.getPartition(), rmContainer);
+      LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+          + " containerState="+ rmContainer.getState()
+          + " resource=" + rmContainer.getContainer().getResource()
+          + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
+          + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
+          + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource);
+      // Inform the parent queue
+      parent.attachContainer(clusterResource, application, rmContainer);
+    }
+  }
+
+  @Override
+  public void detachContainer(Resource clusterResource,
+      FiCaSchedulerApp application, RMContainer rmContainer) {
+    if (application != null && rmContainer != null
+        && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
+      FiCaSchedulerNode node =
+          queueContext.getNode(rmContainer.getContainer().getNodeId());
+      releaseResource(clusterResource, application, rmContainer.getContainer()
+          .getResource(), node.getPartition(), rmContainer);
+      LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+          + " containerState="+ rmContainer.getState()
+          + " resource=" + rmContainer.getContainer().getResource()
+          + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
+          + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
+          + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource);
+      // Inform the parent queue
+      parent.detachContainer(clusterResource, application, rmContainer);
+    }
+  }
+
+  /**
+   * @return all ignored partition exclusivity RMContainers in the LeafQueue,
+   *         this will be used by preemption policy.
+   */
+  public Map<String, TreeSet<RMContainer>> getIgnoreExclusivityRMContainers() {
+    Map<String, TreeSet<RMContainer>> clonedMap = new HashMap<>();
+
+    readLock.lock();
+    try {
+      for (Map.Entry<String, TreeSet<RMContainer>> entry : ignorePartitionExclusivityRMContainers
+          .entrySet()) {
+        clonedMap.put(entry.getKey(), new TreeSet<>(entry.getValue()));
+      }
+
+      return clonedMap;
+
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public void setCapacity(float capacity) {
+    queueCapacities.setCapacity(capacity);
+  }
+
+  public void setCapacity(String nodeLabel, float capacity) {
+    queueCapacities.setCapacity(nodeLabel, capacity);
+  }
+
+  public void setAbsoluteCapacity(float absoluteCapacity) {
+    queueCapacities.setAbsoluteCapacity(absoluteCapacity);
+  }
+
+  public void setAbsoluteCapacity(String nodeLabel, float absoluteCapacity) {
+    queueCapacities.setAbsoluteCapacity(nodeLabel, absoluteCapacity);
+  }
+
+  public void setMaxApplicationsPerUser(int maxApplicationsPerUser) {
+    this.maxApplicationsPerUser = maxApplicationsPerUser;
+  }
+
+  public void setMaxApplications(int maxApplications) {
+    this.maxApplications = maxApplications;
+  }
+
+  public void setMaxAMResourcePerQueuePercent(
+      float maxAMResourcePerQueuePercent) {
+    this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
+  }
+
+  public OrderingPolicy<FiCaSchedulerApp> getOrderingPolicy() {
+    return orderingPolicy;
+  }
+
+  void setOrderingPolicy(
+      OrderingPolicy<FiCaSchedulerApp> orderingPolicy) {
+    writeLock.lock();
+    try {
+      if (null != this.orderingPolicy) {
+        orderingPolicy.addAllSchedulableEntities(
+            this.orderingPolicy.getSchedulableEntities());
+      }
+      this.orderingPolicy = orderingPolicy;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public Priority getDefaultApplicationPriority() {
+    return defaultAppPriorityPerQueue;
+  }
+
+  public void updateApplicationPriority(SchedulerApplication<FiCaSchedulerApp> app,
+      Priority newAppPriority) {
+    writeLock.lock();
+    try {
+      FiCaSchedulerApp attempt = app.getCurrentAppAttempt();
+      boolean isActive = orderingPolicy.removeSchedulableEntity(attempt);
+      if (!isActive) {
+        pendingOrderingPolicy.removeSchedulableEntity(attempt);
+      }
+      // Update new priority in SchedulerApplication
+      attempt.setPriority(newAppPriority);
+
+      if (isActive) {
+        orderingPolicy.addSchedulableEntity(attempt);
+      } else {
+        pendingOrderingPolicy.addSchedulableEntity(attempt);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  public OrderingPolicy<FiCaSchedulerApp> getPendingAppsOrderingPolicy() {
+    return pendingOrderingPolicy;
+  }
+
+  /*
+   * Holds shared values used by all applications in
+   * the queue to calculate headroom on demand
+   */
+  static class QueueResourceLimitsInfo {
+    private Resource queueCurrentLimit;
+    private Resource clusterResource;
+
+    public void setQueueCurrentLimit(Resource currentLimit) {
+      this.queueCurrentLimit = currentLimit;
+    }
+
+    public Resource getQueueCurrentLimit() {
+      return queueCurrentLimit;
+    }
+
+    public void setClusterResource(Resource clusterResource) {
+      this.clusterResource = clusterResource;
+    }
+
+    public Resource getClusterResource() {
+      return clusterResource;
+    }
+  }
+
+  @Override
+  public void stopQueue() {
+    writeLock.lock();
+    try {
+      if (getNumApplications() > 0) {
+        updateQueueState(QueueState.DRAINING);
+      } else {
+        updateQueueState(QueueState.STOPPED);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  void updateMaximumApplications() {
+    CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
+    int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePath());
+
+    int maxDefaultPerQueueApps = configuration.getGlobalMaximumApplicationsPerQueue();
+    int maxSystemApps = configuration.getMaximumSystemApplications();
+    int baseMaxApplications = maxDefaultPerQueueApps > 0 ?
+        Math.min(maxDefaultPerQueueApps, maxSystemApps)
+        : maxSystemApps;
+
+    String maxLabel = RMNodeLabelsManager.NO_LABEL;
+    if (maxAppsForQueue < 0) {
+      if (maxDefaultPerQueueApps > 0 && this.capacityConfigType
+          != CapacityConfigType.ABSOLUTE_RESOURCE) {
+        maxAppsForQueue = baseMaxApplications;
+      } else {
+        for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
+          int maxApplicationsByLabel = (int) (baseMaxApplications
+              * queueCapacities.getAbsoluteCapacity(label));
+          if (maxApplicationsByLabel > maxAppsForQueue) {
+            maxAppsForQueue = maxApplicationsByLabel;
+            maxLabel = label;
+          }
+        }
+      }
+    }
+
+    setMaxApplications(maxAppsForQueue);
+
+    updateMaxAppsPerUser();
+
+    LOG.info("LeafQueue:" + getQueuePath() +
+        "update max app related, maxApplications="
+        + maxAppsForQueue + ", maxApplicationsPerUser="
+        + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
+        .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities
+        .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities
+        .getMaximumCapacity(maxLabel));
+  }
+
+  private void updateMaxAppsPerUser() {
+    int maxAppsPerUser = maxApplications;
+    if (getUsersManager().getUserLimitFactor() != -1) {
+      int maxApplicationsWithUserLimits = (int) (maxApplications
+          * (getUsersManager().getUserLimit() / 100.0f)
+          * getUsersManager().getUserLimitFactor());
+      maxAppsPerUser = Math.min(maxApplications,
+          maxApplicationsWithUserLimits);
+    }
+
+    setMaxApplicationsPerUser(maxAppsPerUser);
+  }
+
+  /**
+   * Get all valid users in this queue.
+   * @return user list
+   */
+  public Set<String> getAllUsers() {
+    return this.getUsersManager().getUsers().keySet();
+  }
+
+  static class CachedUserLimit {
+    final Resource userLimit;
+    volatile boolean canAssign = true;
+    volatile Resource reservation = Resources.none();
+
+    CachedUserLimit(Resource userLimit) {
+      this.userLimit = userLimit;
+    }
+  }
+
+  private void updateQueuePreemptionMetrics(RMContainer rmc) {
+    final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime();
+    final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND;
+    CSQueueMetrics metrics = usageTracker.getMetrics();
+    Resource containerResource = rmc.getAllocatedResource();
+    metrics.preemptContainer();
+    long mbSeconds = (containerResource.getMemorySize() * usedMillis)
+        / DateUtils.MILLIS_PER_SECOND;
+    long vcSeconds = (containerResource.getVirtualCores() * usedMillis)
+        / DateUtils.MILLIS_PER_SECOND;
+    metrics.updatePreemptedMemoryMBSeconds(mbSeconds);
+    metrics.updatePreemptedVcoreSeconds(vcSeconds);
+    metrics.updatePreemptedResources(containerResource);
+    metrics.updatePreemptedSecondsForCustomResources(containerResource,
+        usedSeconds);
+    metrics.updatePreemptedForCustomResources(containerResource);
+  }
+
+  @Override
+  int getNumRunnableApps() {
+    readLock.lock();
+    try {
+      return runnableApps.size();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  int getNumNonRunnableApps() {
+    readLock.lock();
+    try {
+      return nonRunnableApps.size();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  boolean removeNonRunnableApp(FiCaSchedulerApp app) {
+    writeLock.lock();
+    try {
+      return nonRunnableApps.remove(app);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  List<FiCaSchedulerApp> getCopyOfNonRunnableAppSchedulables() {
+    List<FiCaSchedulerApp> appsToReturn = new ArrayList<>();
+    readLock.lock();
+    try {
+      appsToReturn.addAll(nonRunnableApps);
+    } finally {
+      readLock.unlock();
+    }
+    return appsToReturn;
+  }
+
+  @Override
+  public boolean isEligibleForAutoDeletion() {
+    return isDynamicQueue() && getNumApplications() == 0
+        && queueContext.getConfiguration().
+        isAutoExpiredDeletionEnabled(this.getQueuePath());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index 9c16de097d6aa..1c25ce2928bf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -19,7 +19,6 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
@@ -44,9 +43,9 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
   protected AutoCreatedLeafQueueConfig leafQueueTemplate;
   protected AutoCreatedQueueManagementPolicy queueManagementPolicy = null;
 
-  public AbstractManagedParentQueue(CapacitySchedulerContext cs,
+  public AbstractManagedParentQueue(CapacitySchedulerQueueContext queueContext,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
-    super(cs, queueName, parent, old);
+    super(queueContext, queueName, parent, old);
   }
 
   @Override
@@ -55,7 +54,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
     writeLock.lock();
     try {
       // Set new configs
-      setupQueueConfigs(clusterResource, csContext.getConfiguration());
+      setupQueueConfigs(clusterResource);
 
     } finally {
       writeLock.unlock();
@@ -121,8 +120,7 @@ public CSQueue removeChildQueue(String childQueueName)
     CSQueue childQueue;
     writeLock.lock();
     try {
-      childQueue = this.csContext.getCapacitySchedulerQueueManager().getQueue(
-          childQueueName);
+      childQueue = queueContext.getQueueManager().getQueue(childQueueName);
       if (childQueue != null) {
         removeChildQueue(childQueue);
       } else {
@@ -176,22 +174,12 @@ protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
     CapacitySchedulerConfiguration leafQueueConfigs = new
         CapacitySchedulerConfiguration(new Configuration(false), false);
 
-    Map<String, String> rtProps = csContext
-        .getConfiguration().getConfigurationProperties()
-        .getPropertiesWithPrefix(YarnConfiguration.RESOURCE_TYPES + ".", true);
-    for (Map.Entry<String, String> entry : rtProps.entrySet()) {
-      leafQueueConfigs.set(entry.getKey(), entry.getValue());
-    }
-
-    Map<String, String> templateConfigs = csContext
+    Map<String, String> templateConfigs = queueContext
         .getConfiguration().getConfigurationProperties()
         .getPropertiesWithPrefix(configPrefix, true);
 
-    for (final Iterator<Map.Entry<String, String>> iterator =
-         templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {
-      Map.Entry<String, String> confKeyValuePair = iterator.next();
-      leafQueueConfigs.set(confKeyValuePair.getKey(),
-          confKeyValuePair.getValue());
+    for (Map.Entry<String, String> confKeyValuePair : templateConfigs.entrySet()) {
+      leafQueueConfigs.set(confKeyValuePair.getKey(), confKeyValuePair.getValue());
     }
 
     return leafQueueConfigs;
@@ -208,7 +196,7 @@ protected void validateQueueEntitlementChange(AbstractAutoCreatedLeafQueue
     if (!(newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON)) {
       throw new SchedulerDynamicEditException(
           "Sum of child queues should exceed 100% for auto creating parent "
-              + "queue : " + queueName);
+              + "queue : " + getQueueName());
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index 7311be77baaa1..910d8de42bc97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 
@@ -30,21 +31,23 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
 /**
  * Leaf queues which are auto created by an underlying implementation of
  * AbstractManagedParentQueue. Eg: PlanQueue for reservations or
  * ManagedParentQueue for auto created dynamic queues
  */
 public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue {
-
   private static final Logger LOG = LoggerFactory
       .getLogger(AutoCreatedLeafQueue.class);
 
-  public AutoCreatedLeafQueue(CapacitySchedulerContext cs, String queueName,
+  public AutoCreatedLeafQueue(CapacitySchedulerQueueContext queueContext, String queueName,
       ManagedParentQueue parent) throws IOException {
-    super(cs, parent.getLeafQueueConfigs(queueName),
-        queueName,
-        parent, null);
+    super(queueContext, queueName, parent, null);
+    parent.setLeafQueueConfigs(queueName);
+    super.setupQueueConfigs(queueContext.getClusterResource());
+
     updateCapacitiesToZero();
   }
 
@@ -57,8 +60,8 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
 
       ManagedParentQueue managedParentQueue = (ManagedParentQueue) parent;
 
-      super.reinitialize(newlyParsedQueue, clusterResource, managedParentQueue
-          .getLeafQueueConfigs(newlyParsedQueue.getQueueShortName()));
+      managedParentQueue.setLeafQueueConfigs(newlyParsedQueue.getQueueShortName());
+      super.reinitialize(newlyParsedQueue, clusterResource);
 
       //Reset capacities to 0 since reinitialize above
       // queueCapacities to initialize to configured capacity which might
@@ -70,8 +73,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
     }
   }
 
-  public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig
-      leafQueueTemplate) throws SchedulerDynamicEditException, IOException {
+  public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig leafQueueTemplate) {
 
     writeLock.lock();
     try {
@@ -82,14 +84,14 @@ public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig
       QueueCapacities capacities = leafQueueTemplate.getQueueCapacities();
 
       //reset capacities for the leaf queue
-      mergeCapacities(capacities);
+      mergeCapacities(capacities, leafQueueTemplate.getResourceQuotas());
 
     } finally {
       writeLock.unlock();
     }
   }
 
-  public void mergeCapacities(QueueCapacities capacities) {
+  public void mergeCapacities(QueueCapacities capacities, QueueResourceQuotas resourceQuotas) {
     for ( String nodeLabel : capacities.getExistingNodeLabels()) {
       queueCapacities.setCapacity(nodeLabel,
           capacities.getCapacity(nodeLabel));
@@ -101,10 +103,20 @@ public void mergeCapacities(QueueCapacities capacities) {
           .getAbsoluteMaximumCapacity(nodeLabel));
 
       Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
-          csContext.getClusterResource());
-      getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
-          Resources.multiply(resourceByLabel,
-              queueCapacities.getAbsoluteCapacity(nodeLabel)));
+          queueContext.getClusterResource());
+      // Update effective resource from template due to rounding errors.
+      // However, we need to consider deactivation as well, in which case we fall back to
+      // Percentage calculation (as absolute capacity will be 0, resource will be zero as well).
+      if (getCapacityConfigType().equals(ABSOLUTE_RESOURCE)
+          && queueCapacities.getAbsoluteCapacity(nodeLabel) > 0) {
+        getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+            resourceQuotas.getConfiguredMinResource(nodeLabel));
+      } else {
+        getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+            Resources.multiply(resourceByLabel,
+                queueCapacities.getAbsoluteCapacity(nodeLabel)));
+      }
+
       getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel,
           Resources.multiply(resourceByLabel, queueCapacities
               .getAbsoluteMaximumCapacity(nodeLabel)));
@@ -124,17 +136,16 @@ public void validateConfigurations(AutoCreatedLeafQueueConfig template)
   }
 
   @Override
-  protected void setDynamicQueueProperties(
-      CapacitySchedulerConfiguration configuration) {
+  protected void setDynamicQueueProperties() {
     String parentTemplate = String.format("%s.%s", getParent().getQueuePath(),
         CapacitySchedulerConfiguration
             .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX);
-    Set<String> parentNodeLabels = csContext
-        .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+    Set<String> parentNodeLabels = queueContext
+        .getQueueManager().getConfiguredNodeLabelsForAllQueues()
         .getLabelsByQueue(parentTemplate);
 
     if (parentNodeLabels != null && parentNodeLabels.size() > 1) {
-      csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+      queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()
           .setLabelsByQueue(getQueuePath(),
               new HashSet<>(parentNodeLabels));
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
index 87ef1c0e26f7d..78e8a8c65faea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
+
 /**
  * Auto Created Leaf queue configurations, capacity
  */
@@ -31,15 +33,19 @@ public class AutoCreatedLeafQueueConfig {
 
   private CapacitySchedulerConfiguration leafQueueConfigs;
 
+  private final QueueResourceQuotas resourceQuotas;
+
   public AutoCreatedLeafQueueConfig(Builder builder) {
     this.queueCapacities = builder.queueCapacities;
     this.leafQueueConfigs = builder.leafQueueConfigs;
+    this.resourceQuotas = builder.queueResourceQuotas;
   }
 
   public static class Builder {
 
     private QueueCapacities queueCapacities;
     private CapacitySchedulerConfiguration leafQueueConfigs;
+    private QueueResourceQuotas queueResourceQuotas;
 
     public Builder capacities(QueueCapacities capacities) {
       this.queueCapacities = capacities;
@@ -54,6 +60,11 @@ public Builder configuration(CapacitySchedulerConfiguration conf) {
     public AutoCreatedLeafQueueConfig build() {
       return new AutoCreatedLeafQueueConfig(this);
     }
+
+    public Builder resourceQuotas(QueueResourceQuotas queueResourceQuotas) {
+      this.queueResourceQuotas = queueResourceQuotas;
+      return this;
+    }
   }
 
   public QueueCapacities getQueueCapacities() {
@@ -64,6 +75,10 @@ public CapacitySchedulerConfiguration getLeafQueueConfigs() {
     return leafQueueConfigs;
   }
 
+  public QueueResourceQuotas getResourceQuotas() {
+    return resourceQuotas;
+  }
+
   @Override public String toString() {
     return "AutoCreatedLeafQueueConfig{" + "queueCapacities=" + queueCapacities
         + ", leafQueueConfigs=" + leafQueueConfigs + '}';
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
index 388e9d6233bd4..bf99c3f30c3ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
@@ -27,17 +27,15 @@ public interface AutoCreatedQueueManagementPolicy {
 
   /**
    * Initialize policy
-   * @param schedulerContext Capacity Scheduler context
+   * @param parentQueue parent queue
    */
-  void init(CapacitySchedulerContext schedulerContext, ParentQueue
-      parentQueue) throws IOException;
+  void init(ParentQueue parentQueue) throws IOException;
 
   /**
    * Reinitialize policy state ( if required )
-   * @param schedulerContext Capacity Scheduler context
+   * @param parentQueue parent queue
    */
-  void reinitialize(CapacitySchedulerContext schedulerContext,
-      ParentQueue parentQueue) throws IOException;
+  void reinitialize(ParentQueue parentQueue) throws IOException;
 
   /**
    * Get initial template for the specified leaf queue
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index 0a3d49a50322b..bbeca4b072450 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -20,13 +20,12 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.hadoop.util.Lists;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
@@ -43,14 +42,13 @@ public class AutoCreatedQueueTemplate {
       AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template.";
 
   private static final String WILDCARD_QUEUE = "*";
-  private static final int MAX_WILDCARD_LEVEL = 1;
 
   private final Map<String, String> templateProperties = new HashMap<>();
   private final Map<String, String> leafOnlyProperties = new HashMap<>();
   private final Map<String, String> parentOnlyProperties = new HashMap<>();
 
   public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration,
-                                  String queuePath) {
+                                  QueuePath queuePath) {
     setTemplateConfigEntries(configuration, queuePath);
   }
 
@@ -87,7 +85,7 @@ public Map<String, String> getParentOnlyProperties() {
   /**
    * Sets the common template properties and parent specific template
    * properties of a child queue based on its parent template settings.
- * @param conf configuration to set
+   * @param conf configuration to set
    * @param childQueuePath child queue path used for prefixing the properties
    */
   public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
@@ -155,14 +153,13 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
    * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
    */
   private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration,
-                                        String queuePath) {
+                                        QueuePath queuePath) {
     ConfigurationProperties configurationProperties =
         configuration.getConfigurationProperties();
 
-    List<String> queuePathParts = new ArrayList<>(Arrays.asList(
-        queuePath.split("\\.")));
+    List<String> queuePathParts = Lists.newArrayList(queuePath.iterator());
 
-    if (queuePathParts.size() <= 1 && !queuePath.equals(ROOT)) {
+    if (queuePathParts.size() <= 1 && !queuePath.isRoot()) {
       // This is an invalid queue path
       return;
     }
@@ -171,11 +168,10 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati
     // start with the most explicit format (without wildcard)
     int wildcardLevel = 0;
     // root can not be wildcarded
-    // MAX_WILDCARD_LEVEL will be configurable in the future
-    int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1,
-        MAX_WILDCARD_LEVEL);
+    int supportedWildcardLevel = Math.min(queuePathMaxIndex,
+            configuration.getMaximumAutoCreatedQueueDepth(queuePath.getFullPath()));
     // Allow root to have template properties
-    if (queuePath.equals(ROOT)) {
+    if (queuePath.isRoot()) {
       supportedWildcardLevel = 0;
     }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java
index 93d001773138f..fedde057a9010 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSMaxRunningAppsEnforcer.java
@@ -182,7 +182,7 @@ public void updateRunnabilityOnAppRemoval(FiCaSchedulerApp app) {
     // the queue was already at its max before the removal.
     // Thus we find the ancestor queue highest in the tree for which the app
     // that was at its maxRunningApps before the removal.
-    LeafQueue queue = app.getCSLeafQueue();
+    AbstractLeafQueue queue = app.getCSLeafQueue();
     AbstractCSQueue highestQueueWithAppsNowRunnable =
         (queue.getNumRunnableApps() == queue.getMaxParallelApps() - 1)
         ? queue : null;
@@ -243,7 +243,7 @@ private void updateAppsRunnability(List<List<FiCaSchedulerApp>>
       }
 
       if (checkRunnabilityWithUpdate(next)) {
-        LeafQueue nextQueue = next.getCSLeafQueue();
+        AbstractLeafQueue nextQueue = next.getCSLeafQueue();
         LOG.info("{} is now runnable in {}",
             next.getApplicationAttemptId(), nextQueue);
         trackRunnableApp(next);
@@ -322,9 +322,9 @@ private void untrackNonRunnableApp(FiCaSchedulerApp app) {
   private void gatherPossiblyRunnableAppLists(AbstractCSQueue queue,
       List<List<FiCaSchedulerApp>> appLists) {
     if (queue.getNumRunnableApps() < queue.getMaxParallelApps()) {
-      if (queue instanceof LeafQueue) {
+      if (queue instanceof AbstractLeafQueue) {
         appLists.add(
-            ((LeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
+            ((AbstractLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
       } else {
         for (CSQueue child : queue.getChildQueues()) {
           gatherPossiblyRunnableAppLists((AbstractCSQueue) child, appLists);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 03a5afbee51c2..e2aeaab41808f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -89,6 +89,19 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    */
   public String getQueuePath();
 
+  /**
+   * Gets the queue path object.
+   * @return the object of the queue
+   */
+  QueuePath getQueuePathObject();
+
+  /**
+   * Checks whether the queue is a dynamic queue (created dynamically in the fashion of auto queue
+   * creation v2).
+   * @return true, if it is a dynamic queue, false otherwise
+   */
+  boolean isDynamicQueue();
+
   public PrivilegedEntity getPrivilegedEntity();
 
   Resource getMaximumAllocation();
@@ -419,6 +432,14 @@ public void validateSubmitApplication(ApplicationId applicationId,
    */
   Resource getEffectiveCapacity(String label);
 
+  /**
+   * Get configured capacity resource vector parsed from the capacity config
+   * of the queue.
+   * @param label node label (partition)
+   * @return capacity resource vector
+   */
+  QueueCapacityVector getConfiguredCapacityVector(String label);
+
   /**
    * Get effective capacity of queue. If min/max resource is configured,
    * preference will be given to absolute configuration over normal capacity.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java
new file mode 100644
index 0000000000000..8daca2bc26be6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueInfoProvider.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueStatistics;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public final class CSQueueInfoProvider {
+
+  private static final RecordFactory RECORD_FACTORY =
+          RecordFactoryProvider.getRecordFactory(null);
+
+  private CSQueueInfoProvider() {
+  }
+
+  public static QueueInfo getQueueInfo(AbstractCSQueue csQueue) {
+    QueueInfo queueInfo = RECORD_FACTORY.newRecordInstance(QueueInfo.class);
+    queueInfo.setQueueName(csQueue.getQueuePathObject().getLeafName());
+    queueInfo.setQueuePath(csQueue.getQueuePathObject().getFullPath());
+    queueInfo.setAccessibleNodeLabels(csQueue.getAccessibleNodeLabels());
+    queueInfo.setCapacity(csQueue.getCapacity());
+    queueInfo.setMaximumCapacity(csQueue.getMaximumCapacity());
+    queueInfo.setQueueState(csQueue.getState());
+    queueInfo.setDefaultNodeLabelExpression(csQueue.getDefaultNodeLabelExpression());
+    queueInfo.setCurrentCapacity(csQueue.getUsedCapacity());
+    queueInfo.setQueueStatistics(getQueueStatistics(csQueue));
+    queueInfo.setPreemptionDisabled(csQueue.getPreemptionDisabled());
+    queueInfo.setIntraQueuePreemptionDisabled(
+            csQueue.getIntraQueuePreemptionDisabled());
+    queueInfo.setQueueConfigurations(getQueueConfigurations(csQueue));
+    queueInfo.setWeight(csQueue.getQueueCapacities().getWeight());
+    queueInfo.setMaxParallelApps(csQueue.getMaxParallelApps());
+    return queueInfo;
+  }
+
+  private static QueueStatistics getQueueStatistics(AbstractCSQueue csQueue) {
+    QueueStatistics stats = RECORD_FACTORY.newRecordInstance(
+            QueueStatistics.class);
+    CSQueueMetrics queueMetrics = csQueue.getMetrics();
+    stats.setNumAppsSubmitted(queueMetrics.getAppsSubmitted());
+    stats.setNumAppsRunning(queueMetrics.getAppsRunning());
+    stats.setNumAppsPending(queueMetrics.getAppsPending());
+    stats.setNumAppsCompleted(queueMetrics.getAppsCompleted());
+    stats.setNumAppsKilled(queueMetrics.getAppsKilled());
+    stats.setNumAppsFailed(queueMetrics.getAppsFailed());
+    stats.setNumActiveUsers(queueMetrics.getActiveUsers());
+    stats.setAvailableMemoryMB(queueMetrics.getAvailableMB());
+    stats.setAllocatedMemoryMB(queueMetrics.getAllocatedMB());
+    stats.setPendingMemoryMB(queueMetrics.getPendingMB());
+    stats.setReservedMemoryMB(queueMetrics.getReservedMB());
+    stats.setAvailableVCores(queueMetrics.getAvailableVirtualCores());
+    stats.setAllocatedVCores(queueMetrics.getAllocatedVirtualCores());
+    stats.setPendingVCores(queueMetrics.getPendingVirtualCores());
+    stats.setReservedVCores(queueMetrics.getReservedVirtualCores());
+    stats.setPendingContainers(queueMetrics.getPendingContainers());
+    stats.setAllocatedContainers(queueMetrics.getAllocatedContainers());
+    stats.setReservedContainers(queueMetrics.getReservedContainers());
+    return stats;
+  }
+
+  private static Map<String, QueueConfigurations> getQueueConfigurations(AbstractCSQueue csQueue) {
+    Map<String, QueueConfigurations> queueConfigurations = new HashMap<>();
+    Set<String> nodeLabels = csQueue.getNodeLabelsForQueue();
+    QueueResourceQuotas queueResourceQuotas = csQueue.getQueueResourceQuotas();
+    for (String nodeLabel : nodeLabels) {
+      QueueConfigurations queueConfiguration =
+              RECORD_FACTORY.newRecordInstance(QueueConfigurations.class);
+      QueueCapacities queueCapacities = csQueue.getQueueCapacities();
+      float capacity = queueCapacities.getCapacity(nodeLabel);
+      float absoluteCapacity = queueCapacities.getAbsoluteCapacity(nodeLabel);
+      float maxCapacity = queueCapacities.getMaximumCapacity(nodeLabel);
+      float absMaxCapacity =
+              queueCapacities.getAbsoluteMaximumCapacity(nodeLabel);
+      float maxAMPercentage =
+              queueCapacities.getMaxAMResourcePercentage(nodeLabel);
+      queueConfiguration.setCapacity(capacity);
+      queueConfiguration.setAbsoluteCapacity(absoluteCapacity);
+      queueConfiguration.setMaxCapacity(maxCapacity);
+      queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
+      queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+      queueConfiguration.setConfiguredMinCapacity(
+              queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+      queueConfiguration.setConfiguredMaxCapacity(
+              queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+      queueConfiguration.setEffectiveMinCapacity(
+              queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+      queueConfiguration.setEffectiveMaxCapacity(
+              queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
+      queueConfigurations.put(nodeLabel, queueConfiguration);
+    }
+    return queueConfigurations;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 586f837f147b2..16ebc15512847 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.MetricsSource;
 import org.apache.hadoop.metrics2.MetricsSystem;
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -240,6 +241,10 @@ public synchronized static CSQueueMetrics forQueue(String queueName,
 
       // Register with the MetricsSystems
       if (ms != null) {
+        MetricsSource source = ms.getSource(sourceName(queueName).toString());
+        if (source != null) {
+          ms.unregisterSource(sourceName(queueName).toString());
+        }
         metrics =
             ms.register(sourceName(queueName).toString(), "Metrics for queue: "
                 + queueName, metrics);
@@ -260,7 +265,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) {
       metrics =
         new CSQueueMetrics(metricsSystem, queueName, null, false, conf);
       users.put(userName, metrics);
-      metricsSystem.register(
+      registerMetrics(
           sourceName(queueName).append(",user=").append(userName).toString(),
           "Metrics for user '" + userName + "' in queue '" + queueName + "'",
           ((CSQueueMetrics) metrics.tag(QUEUE_INFO, queueName)).tag(USER_INFO,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
similarity index 90%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
index 0a11bcd62932c..dc254747dfec5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
@@ -18,19 +18,18 @@
 
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
-public class CSQueuePreemption {
+public class CSQueuePreemptionSettings {
   private final boolean preemptionDisabled;
   // Indicates if the in-queue preemption setting is ever disabled within the
   // hierarchy of this queue.
   private final boolean intraQueuePreemptionDisabledInHierarchy;
 
-  public CSQueuePreemption(
+  public CSQueuePreemptionSettings(
       CSQueue queue,
-      CapacitySchedulerContext csContext,
       CapacitySchedulerConfiguration configuration) {
-    this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, csContext, configuration);
+    this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(queue, configuration);
     this.intraQueuePreemptionDisabledInHierarchy =
-        isIntraQueueHierarchyPreemptionDisabled(queue, csContext, configuration);
+        isIntraQueueHierarchyPreemptionDisabled(queue, configuration);
   }
 
   /**
@@ -40,14 +39,13 @@ public CSQueuePreemption(
    * NOTE: Cross-queue preemptability is inherited from a queue's parent.
    *
    * @param q queue to check preemption state
-   * @param csContext
    * @param configuration capacity scheduler config
    * @return true if queue has cross-queue preemption disabled, false otherwise
    */
   private boolean isQueueHierarchyPreemptionDisabled(CSQueue q,
-      CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) {
+      CapacitySchedulerConfiguration configuration) {
     boolean systemWidePreemption =
-        csContext.getConfiguration()
+        configuration
             .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
                 YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS);
     CSQueue parentQ = q.getParent();
@@ -79,14 +77,13 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q,
    * NOTE: Intra-queue preemptability is inherited from a queue's parent.
    *
    * @param q queue to check intra-queue preemption state
-   * @param csContext
    * @param configuration capacity scheduler config
    * @return true if queue has intra-queue preemption disabled, false otherwise
    */
   private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q,
-      CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) {
+      CapacitySchedulerConfiguration configuration) {
     boolean systemWideIntraQueuePreemption =
-        csContext.getConfiguration().getBoolean(
+        configuration.getBoolean(
             CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ENABLED,
             CapacitySchedulerConfiguration
                 .DEFAULT_INTRAQUEUE_PREEMPTION_ENABLED);
@@ -109,6 +106,10 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q,
         parentQ.getIntraQueuePreemptionDisabledInHierarchy());
   }
 
+  public boolean isIntraQueuePreemptionDisabled() {
+    return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled;
+  }
+
   public boolean isIntraQueuePreemptionDisabledInHierarchy() {
     return intraQueuePreemptionDisabledInHierarchy;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java
new file mode 100644
index 0000000000000..0f18e944e9a38
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+
+public class CSQueueUsageTracker {
+  private final CSQueueMetrics metrics;
+  private int numContainers;
+
+  /**
+   * The timestamp of the last submitted application to this queue.
+   * Only applies to dynamic queues.
+   */
+  private long lastSubmittedTimestamp;
+
+  /**
+   * Tracks resource usage by label like used-resource / pending-resource.
+   */
+  private final ResourceUsage queueUsage;
+
+  private final QueueResourceQuotas queueResourceQuotas;
+
+  public CSQueueUsageTracker(CSQueueMetrics metrics) {
+    this.metrics = metrics;
+    this.queueUsage = new ResourceUsage();
+    this.queueResourceQuotas = new QueueResourceQuotas();
+  }
+
+  public int getNumContainers() {
+    return numContainers;
+  }
+
+  public synchronized void increaseNumContainers() {
+    numContainers++;
+  }
+
+  public synchronized void decreaseNumContainers() {
+    numContainers--;
+  }
+
+  public CSQueueMetrics getMetrics() {
+    return metrics;
+  }
+
+  public long getLastSubmittedTimestamp() {
+    return lastSubmittedTimestamp;
+  }
+
+  public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
+    this.lastSubmittedTimestamp = lastSubmittedTimestamp;
+  }
+
+  public ResourceUsage getQueueUsage() {
+    return queueUsage;
+  }
+
+  public QueueResourceQuotas getQueueResourceQuotas() {
+    return queueResourceQuotas;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index f43ea30001788..ecb285748b510 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.utils.Lock;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -36,7 +35,7 @@ public class CSQueueUtils {
   /*
    * Used only by tests
    */
-  public static void checkMaxCapacity(String queuePath,
+  public static void checkMaxCapacity(QueuePath queuePath,
       float capacity, float maximumCapacity) {
     if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) {
       throw new IllegalArgumentException(
@@ -48,7 +47,7 @@ public static void checkMaxCapacity(String queuePath,
   /*
    * Used only by tests
    */
-  public static void checkAbsoluteCapacity(String queuePath,
+  public static void checkAbsoluteCapacity(QueuePath queuePath,
       float absCapacity, float absMaxCapacity) {
     if (absMaxCapacity < (absCapacity - EPSILON)) {
       throw new IllegalArgumentException("Illegal call to setMaxCapacity. "
@@ -67,7 +66,7 @@ public static float computeAbsoluteMaximumCapacity(
   }
 
   public static void loadCapacitiesByLabelsFromConf(
-      String queuePath, QueueCapacities queueCapacities,
+      QueuePath queuePath, QueueCapacities queueCapacities,
       CapacitySchedulerConfiguration csConf, Set<String> nodeLabels) {
     queueCapacities.clearConfigurableFields();
 
@@ -81,7 +80,7 @@ public static void loadCapacitiesByLabelsFromConf(
             label,
             csConf.getMaximumAMResourcePercentPerPartition(queuePath, label));
         queueCapacities.setWeight(label,
-            csConf.getNonLabeledQueueWeight(queuePath));
+            csConf.getNonLabeledQueueWeight(queuePath.getFullPath()));
       } else{
         queueCapacities.setCapacity(label,
             csConf.getLabeledQueueCapacity(queuePath, label) / 100);
@@ -92,15 +91,6 @@ public static void loadCapacitiesByLabelsFromConf(
         queueCapacities.setWeight(label,
             csConf.getLabeledQueueWeight(queuePath, label));
       }
-
-      /*float absCapacity = queueCapacities.getCapacity(label);
-      float absMaxCapacity = queueCapacities.getMaximumCapacity(label);
-      if (absCapacity > absMaxCapacity) {
-        throw new IllegalArgumentException("Illegal queue capacity setting "
-            + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity="
-            + absMaxCapacity + ") for queue=["
-            + queuePath + "],label=[" + label + "]");
-      }*/
     }
   }
 
@@ -232,8 +222,8 @@ public static void updateQueueStatistics(
     ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
 
     if (nodePartition == null) {
-      for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
-          queueResourceUsage.getNodePartitionsSet())) {
+      for (String partition : Sets.union(queueCapacities.getExistingNodeLabels(),
+          queueResourceUsage.getExistingNodeLabels())) {
         updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
             partition, childQueue);
 
@@ -311,15 +301,4 @@ public static void updateAbsoluteCapacitiesByNodeLabels(QueueCapacities queueCap
       }
     }
   }
-
-  public static ApplicationPlacementContext extractQueuePath(String queuePath) {
-    int parentQueueNameEndIndex = queuePath.lastIndexOf(".");
-    if (parentQueueNameEndIndex > -1) {
-      String parent = queuePath.substring(0, parentQueueNameEndIndex).trim();
-      String leaf = queuePath.substring(parentQueueNameEndIndex + 1).trim();
-      return new ApplicationPlacementContext(leaf, parent);
-    } else{
-      return new ApplicationPlacementContext(queuePath);
-    }
-  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
index 140a2acdbc758..de31cd1361c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityHeadroomProvider.java
@@ -27,13 +27,13 @@
 public class CapacityHeadroomProvider {
   
   UsersManager.User user;
-  LeafQueue queue;
+  AbstractLeafQueue queue;
   FiCaSchedulerApp application;
-  LeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo;
+  AbstractLeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo;
   
-  public CapacityHeadroomProvider(UsersManager.User user, LeafQueue queue,
+  public CapacityHeadroomProvider(UsersManager.User user, AbstractLeafQueue queue,
       FiCaSchedulerApp application,
-      LeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo) {
+      AbstractLeafQueue.QueueResourceLimitsInfo queueResourceLimitsInfo) {
 
     this.user = user;
     this.queue = queue;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 62489a82147e7..e513359af0d63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -163,7 +164,7 @@
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
 
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
@@ -183,10 +184,9 @@ public class CapacityScheduler extends
 
   private CapacitySchedulerQueueManager queueManager;
 
-  private WorkflowPriorityMappingsManager workflowPriorityMappingsMgr;
+  private CapacitySchedulerQueueContext queueContext;
 
-  // timeout to join when we stop this service
-  protected final long THREAD_JOIN_TIMEOUT_MS = 1000;
+  private WorkflowPriorityMappingsManager workflowPriorityMappingsMgr;
 
   private PreemptionManager preemptionManager = new PreemptionManager();
 
@@ -202,6 +202,9 @@ public class CapacityScheduler extends
 
   private int threadNum = 0;
 
+  private final PendingApplicationComparator applicationComparator =
+      new PendingApplicationComparator();
+
   @Override
   public void setConf(Configuration conf) {
       yarnConf = conf;
@@ -225,10 +228,7 @@ public Configuration getConf() {
   private ResourceCalculator calculator;
   private boolean usePortForNodeName;
 
-  private boolean scheduleAsynchronously;
-  @VisibleForTesting
-  protected List<AsyncScheduleThread> asyncSchedulerThreads;
-  private ResourceCommitterService resourceCommitterService;
+  private AsyncSchedulingConfiguration asyncSchedulingConf;
   private RMNodeLabelsManager labelManager;
   private AppPriorityACLsManager appPriorityACLManager;
   private boolean multiNodePlacementEnabled;
@@ -236,16 +236,6 @@ public Configuration getConf() {
   private boolean printedVerboseLoggingForAsyncScheduling;
   private boolean appShouldFailFast;
 
-  /**
-   * EXPERT
-   */
-  private long asyncScheduleInterval;
-  private static final String ASYNC_SCHEDULER_INTERVAL =
-      CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX
-          + ".scheduling-interval-ms";
-  private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5;
-  private long asyncMaxPendingBacklogs;
-
   private CSMaxRunningAppsEnforcer maxRunningEnforcer;
 
   public CapacityScheduler() {
@@ -267,6 +257,11 @@ public CapacitySchedulerConfiguration getConfiguration() {
     return conf;
   }
 
+  @Override
+  public CapacitySchedulerQueueContext getQueueContext() {
+    return queueContext;
+  }
+
   @Override
   public RMContainerTokenSecretManager getContainerTokenSecretManager() {
     return this.rmContext.getContainerTokenSecretManager();
@@ -302,38 +297,13 @@ void initScheduler(Configuration configuration) throws
       IOException, YarnException {
     writeLock.lock();
     try {
-      String confProviderStr = configuration.get(
-          YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-          YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
-      switch (confProviderStr) {
-      case YarnConfiguration.FILE_CONFIGURATION_STORE:
-        this.csConfProvider =
-            new FileBasedCSConfigurationProvider(rmContext);
-        break;
-      case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
-      case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
-      case YarnConfiguration.ZK_CONFIGURATION_STORE:
-      case YarnConfiguration.FS_CONFIGURATION_STORE:
-        this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
-        break;
-      default:
-        throw new IOException("Invalid configuration store class: " +
-            confProviderStr);
-      }
+      this.csConfProvider = getCsConfProvider(configuration);
       this.csConfProvider.init(configuration);
       this.conf = this.csConfProvider.loadConfiguration(configuration);
       validateConf(this.conf);
       this.minimumAllocation = super.getMinimumAllocation();
       initMaximumResourceCapability(super.getMaximumAllocation());
-      this.calculator = this.conf.getResourceCalculator();
-      if (this.calculator instanceof DefaultResourceCalculator
-          && ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
-        throw new YarnRuntimeException("RM uses DefaultResourceCalculator which"
-            + " used only memory as resource-type but invalid resource-types"
-            + " specified " + ResourceUtils.getResourceTypes() + ". Use"
-            + " DominantResourceCalculator instead to make effective use of"
-            + " these resource-types");
-      }
+      this.calculator = initResourceCalculator();
       this.usePortForNodeName = this.conf.getUsePortForNodeName();
       this.applications = new ConcurrentHashMap<>();
       this.labelManager = rmContext.getNodeLabelManager();
@@ -341,84 +311,95 @@ void initScheduler(Configuration configuration) throws
       this.queueManager = new CapacitySchedulerQueueManager(yarnConf,
           this.labelManager, this.appPriorityACLManager);
       this.queueManager.setCapacitySchedulerContext(this);
-
       this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
-
       this.activitiesManager = new ActivitiesManager(rmContext);
       activitiesManager.init(conf);
+      this.queueContext = new CapacitySchedulerQueueContext(this);
       initializeQueues(this.conf);
       this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled();
-
-      scheduleAsynchronously = this.conf.getScheduleAynschronously();
-      asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL,
-          DEFAULT_ASYNC_SCHEDULER_INTERVAL);
-
       this.assignMultipleEnabled = this.conf.getAssignMultipleEnabled();
       this.maxAssignPerHeartbeat = this.conf.getMaxAssignPerHeartbeat();
-
-      this.appShouldFailFast = CapacitySchedulerConfiguration.shouldAppFailFast(
-          getConfig());
-
-      // number of threads for async scheduling
-      int maxAsyncSchedulingThreads = this.conf.getInt(
-          CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD,
-          1);
-      maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1);
-
-      if (scheduleAsynchronously) {
-        asyncSchedulerThreads = new ArrayList<>();
-        for (int i = 0; i < maxAsyncSchedulingThreads; i++) {
-          asyncSchedulerThreads.add(new AsyncScheduleThread(this));
-        }
-        resourceCommitterService = new ResourceCommitterService(this);
-        asyncMaxPendingBacklogs = this.conf.getInt(
-            CapacitySchedulerConfiguration.
-                SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS,
-            CapacitySchedulerConfiguration.
-                DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS);
-      }
+      this.appShouldFailFast = CapacitySchedulerConfiguration.shouldAppFailFast(getConfig());
+      initAsyncSchedulingProperties();
 
       // Setup how many containers we can allocate for each round
       offswitchPerHeartbeatLimit = this.conf.getOffSwitchPerHeartbeatLimit();
 
-      // Register CS specific multi-node policies to common MultiNodeManager
-      // which will add to a MultiNodeSorter which gives a pre-sorted list of
-      // nodes to scheduler's allocation.
-      multiNodePlacementEnabled = this.conf.getMultiNodePlacementEnabled();
-      if(rmContext.getMultiNodeSortingManager() != null) {
-        rmContext.getMultiNodeSortingManager().registerMultiNodePolicyNames(
-            multiNodePlacementEnabled,
-            this.conf.getMultiNodePlacementPolicies());
-      }
-
-      LOG.info("Initialized CapacityScheduler with " + "calculator="
-          + getResourceCalculator().getClass() + ", " + "minimumAllocation="
-          + getMinimumResourceCapability() + ", " + "maximumAllocation="
-          + getMaximumResourceCapability() + ", " + "asynchronousScheduling="
-          + scheduleAsynchronously + ", " + "asyncScheduleInterval="
-          + asyncScheduleInterval + "ms" + ",multiNodePlacementEnabled="
-          + multiNodePlacementEnabled + ", " + "assignMultipleEnabled="
-          + assignMultipleEnabled + ", " + "maxAssignPerHeartbeat="
-          + maxAssignPerHeartbeat + ", " + "offswitchPerHeartbeatLimit="
-          + offswitchPerHeartbeatLimit);
+      initMultiNodePlacement();
+      printSchedulerInitialized();
     } finally {
       writeLock.unlock();
     }
   }
 
+  private CSConfigurationProvider getCsConfProvider(Configuration configuration)
+      throws IOException {
+    String confProviderStr = configuration.get(
+        YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+        YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+    switch (confProviderStr) {
+    case YarnConfiguration.FILE_CONFIGURATION_STORE:
+      return new FileBasedCSConfigurationProvider(rmContext);
+    case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+    case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+    case YarnConfiguration.ZK_CONFIGURATION_STORE:
+    case YarnConfiguration.FS_CONFIGURATION_STORE:
+      return new MutableCSConfigurationProvider(rmContext);
+    default:
+      throw new IOException("Invalid configuration store class: " + confProviderStr);
+    }
+  }
+
+  private ResourceCalculator initResourceCalculator() {
+    ResourceCalculator resourceCalculator = this.conf.getResourceCalculator();
+    if (resourceCalculator instanceof DefaultResourceCalculator
+        && ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+      throw new YarnRuntimeException("RM uses DefaultResourceCalculator which"
+          + " used only memory as resource-type but invalid resource-types"
+          + " specified " + ResourceUtils.getResourceTypes() + ". Use"
+          + " DominantResourceCalculator instead to make effective use of"
+          + " these resource-types");
+    }
+    return resourceCalculator;
+  }
+
+  private void initAsyncSchedulingProperties() {
+    this.asyncSchedulingConf = new AsyncSchedulingConfiguration(conf, this);
+  }
+
+  private void initMultiNodePlacement() {
+    // Register CS specific multi-node policies to common MultiNodeManager
+    // which will add to a MultiNodeSorter which gives a pre-sorted list of
+    // nodes to scheduler's allocation.
+    multiNodePlacementEnabled = this.conf.getMultiNodePlacementEnabled();
+    if (rmContext.getMultiNodeSortingManager() != null) {
+      rmContext.getMultiNodeSortingManager().registerMultiNodePolicyNames(
+          multiNodePlacementEnabled,
+          this.conf.getMultiNodePlacementPolicies());
+    }
+  }
+
+  private void printSchedulerInitialized() {
+    LOG.info("Initialized CapacityScheduler with calculator={}, minimumAllocation={}, " +
+            "maximumAllocation={}, asynchronousScheduling={}, asyncScheduleInterval={} ms, " +
+            "multiNodePlacementEnabled={}, assignMultipleEnabled={}, maxAssignPerHeartbeat={}, " +
+            "offswitchPerHeartbeatLimit={}",
+        getResourceCalculator().getClass(),
+        getMinimumResourceCapability(),
+        getMaximumResourceCapability(),
+        asyncSchedulingConf.isScheduleAsynchronously(),
+        asyncSchedulingConf.getAsyncScheduleInterval(),
+        multiNodePlacementEnabled,
+        assignMultipleEnabled,
+        maxAssignPerHeartbeat,
+        offswitchPerHeartbeatLimit);
+  }
+
   private void startSchedulerThreads() {
     writeLock.lock();
     try {
       activitiesManager.start();
-      if (scheduleAsynchronously) {
-        Preconditions.checkNotNull(asyncSchedulerThreads,
-            "asyncSchedulerThreads is null");
-        for (Thread t : asyncSchedulerThreads) {
-          t.start();
-        }
-
-        resourceCommitterService.start();
-      }
+      asyncSchedulingConf.startThreads();
     } finally {
       writeLock.unlock();
     }
@@ -444,14 +425,7 @@ public void serviceStop() throws Exception {
     writeLock.lock();
     try {
       this.activitiesManager.stop();
-      if (scheduleAsynchronously && asyncSchedulerThreads != null) {
-        for (Thread t : asyncSchedulerThreads) {
-          t.interrupt();
-          t.join(THREAD_JOIN_TIMEOUT_MS);
-        }
-        resourceCommitterService.interrupt();
-        resourceCommitterService.join(THREAD_JOIN_TIMEOUT_MS);
-      }
+      asyncSchedulingConf.serviceStopInvoked();
     } finally {
       writeLock.unlock();
     }
@@ -481,6 +455,7 @@ public void reinitialize(Configuration newConf, RMContext rmContext,
         reinitializeQueues(this.conf);
       } catch (Throwable t) {
         this.conf = oldConf;
+        reinitializeQueues(this.conf);
         refreshMaximumAllocation(
             ResourceUtils.fetchMaximumAllocationFromConfig(this.conf));
         throw new IOException("Failed to re-init queues : " + t.getMessage(),
@@ -518,7 +493,7 @@ public void reinitialize(Configuration newConf, RMContext rmContext)
   }
 
   long getAsyncScheduleInterval() {
-    return asyncScheduleInterval;
+    return asyncSchedulingConf.getAsyncScheduleInterval();
   }
 
   private final static Random random = new Random(System.currentTimeMillis());
@@ -650,6 +625,11 @@ static void schedule(CapacityScheduler cs) throws InterruptedException{
     Thread.sleep(cs.getAsyncScheduleInterval());
   }
 
+  @VisibleForTesting
+  public void setAsyncSchedulingConf(AsyncSchedulingConfiguration conf) {
+    this.asyncSchedulingConf = conf;
+  }
+
   static class AsyncScheduleThread extends Thread {
 
     private final CapacityScheduler cs;
@@ -671,7 +651,7 @@ public void run() {
           } else {
             // Don't run schedule if we have some pending backlogs already
             if (cs.getAsyncSchedulingPendingBacklogs()
-                > cs.asyncMaxPendingBacklogs) {
+                > cs.asyncSchedulingConf.getAsyncMaxPendingBacklogs()) {
               Thread.sleep(1);
             } else{
               schedule(cs);
@@ -831,6 +811,7 @@ private void initializeQueues(CapacitySchedulerConfiguration conf)
   @Lock(CapacityScheduler.class)
   private void reinitializeQueues(CapacitySchedulerConfiguration newConf)
   throws IOException {
+    queueContext.reinitialize();
     this.queueManager.reinitializeQueues(newConf);
     updatePlacementRules();
 
@@ -904,7 +885,7 @@ private void addApplicationOnRecovery(ApplicationId applicationId,
           throw new QueueInvalidException(queueErrorMsg);
         }
       }
-      if (!(queue instanceof LeafQueue)) {
+      if (!(queue instanceof AbstractLeafQueue)) {
         // During RM restart, this means leaf queue was converted to a parent
         // queue, which is not supported for running apps.
         if (!appShouldFailFast) {
@@ -929,7 +910,7 @@ private void addApplicationOnRecovery(ApplicationId applicationId,
       // that means its previous state was DRAINING. So we auto transit
       // the state to DRAINING for recovery.
       if (queue.getState() == QueueState.STOPPED) {
-        ((LeafQueue) queue).recoverDrainingState();
+        ((AbstractLeafQueue) queue).recoverDrainingState();
       }
       // Submit to the queue
       try {
@@ -937,6 +918,10 @@ private void addApplicationOnRecovery(ApplicationId applicationId,
       } catch (AccessControlException ace) {
         // Ignore the exception for recovered app as the app was previously
         // accepted.
+        LOG.warn("AccessControlException received when trying to recover "
+            + applicationId + " in queue " + queueName  + " for user " + user
+            + ". Since the app was in the queue prior to recovery, the Capacity"
+            + " Scheduler will recover the app anyway.", ace);
       }
       queue.getMetrics().submitApp(user, unmanagedAM);
 
@@ -1073,7 +1058,7 @@ private void addApplication(ApplicationId applicationId, String queueName,
         return;
       }
 
-      if (!(queue instanceof LeafQueue)) {
+      if (!(queue instanceof AbstractLeafQueue)) {
         String message =
             "Application " + applicationId + " submitted by user : " + user
                 + " to non-leaf queue : " + queueName;
@@ -1225,7 +1210,7 @@ private void doneApplication(ApplicationId applicationId,
         return;
       }
       CSQueue queue = (CSQueue) application.getQueue();
-      if (!(queue instanceof LeafQueue)) {
+      if (!(queue instanceof AbstractLeafQueue)) {
         LOG.error("Cannot finish application " + "from non-leaf queue: " + queue
             .getQueuePath());
       } else{
@@ -1284,7 +1269,7 @@ private void doneApplicationAttempt(
       // Inform the queue
       Queue  queue = attempt.getQueue();
       CSQueue csQueue = (CSQueue) queue;
-      if (!(csQueue instanceof LeafQueue)) {
+      if (!(csQueue instanceof AbstractLeafQueue)) {
         LOG.error(
             "Cannot finish application " + "from non-leaf queue: "
             + csQueue.getQueuePath());
@@ -1350,7 +1335,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
     // Release containers
     releaseContainers(release, application);
 
-    LeafQueue updateDemandForQueue = null;
+    AbstractLeafQueue updateDemandForQueue = null;
 
     // Sanity check for new allocation requests
     normalizeResourceRequests(ask);
@@ -1381,7 +1366,7 @@ public Allocation allocate(ApplicationAttemptId applicationAttemptId,
         // Update application requests
         if (application.updateResourceRequests(ask) || application
             .updateSchedulingRequests(schedulingRequests)) {
-          updateDemandForQueue = (LeafQueue) application.getQueue();
+          updateDemandForQueue = (AbstractLeafQueue) application.getQueue();
         }
 
         if (LOG.isDebugEnabled()) {
@@ -1453,7 +1438,7 @@ protected void nodeUpdate(RMNode rmNode) {
     }
 
     // Try to do scheduling
-    if (!scheduleAsynchronously) {
+    if (!asyncSchedulingConf.isScheduleAsynchronously()) {
       writeLock.lock();
       try {
         // reset allocation and reservation stats before we start doing any
@@ -1766,7 +1751,7 @@ private void allocateFromReservedContainer(FiCaSchedulerNode node,
     LOG.debug("Trying to fulfill reservation for application {} on node: {}",
         reservedApplication.getApplicationId(), node.getNodeID());
 
-    LeafQueue queue = ((LeafQueue) reservedApplication.getQueue());
+    AbstractLeafQueue queue = ((AbstractLeafQueue) reservedApplication.getQueue());
     CSAssignment assignment = queue.assignContainers(getClusterResource(),
         new SimpleCandidateNodeSet<>(node),
         // TODO, now we only consider limits for parent for non-labeled
@@ -2222,6 +2207,22 @@ private void refreshLabelToNodeCache(Set<String> updateLabels) {
     }
   }
 
+  /**
+   * Add node to nodeTracker. Used when validating CS configuration by instantiating a new
+   * CS instance.
+   * @param nodesToAdd node to be added
+   */
+  public void addNodes(List<FiCaSchedulerNode> nodesToAdd) {
+    writeLock.lock();
+    try {
+      for (FiCaSchedulerNode node : nodesToAdd) {
+        nodeTracker.addNode(node);
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
   private void addNode(RMNode nodeManager) {
     writeLock.lock();
     try {
@@ -2249,8 +2250,8 @@ private void addNode(RMNode nodeManager) {
           "Added node " + nodeManager.getNodeAddress() + " clusterResource: "
               + clusterResource);
 
-      if (scheduleAsynchronously && getNumClusterNodes() == 1) {
-        for (AsyncScheduleThread t : asyncSchedulerThreads) {
+      if (asyncSchedulingConf.isScheduleAsynchronously() && getNumClusterNodes() == 1) {
+        for (AsyncScheduleThread t : asyncSchedulingConf.asyncSchedulerThreads) {
           t.beginSchedule();
         }
       }
@@ -2298,11 +2299,7 @@ private void removeNode(RMNode nodeInfo) {
           new ResourceLimits(clusterResource));
       int numNodes = nodeTracker.nodeCount();
 
-      if (scheduleAsynchronously && numNodes == 0) {
-        for (AsyncScheduleThread t : asyncSchedulerThreads) {
-          t.suspendSchedule();
-        }
-      }
+      asyncSchedulingConf.nodeRemoved(numNodes);
 
       LOG.info(
           "Removed node " + nodeInfo.getNodeAddress() + " clusterResource: "
@@ -2353,7 +2350,7 @@ protected void completedContainerInternal(
     }
 
     // Inform the queue
-    LeafQueue queue = (LeafQueue) application.getQueue();
+    AbstractLeafQueue queue = (AbstractLeafQueue) application.getQueue();
     queue.completedContainer(getClusterResource(), application, node,
         rmContainer, containerStatus, event, null, true);
   }
@@ -2640,7 +2637,7 @@ public void setEntitlement(String inQueue, QueueEntitlement entitlement)
       throws YarnException {
     writeLock.lock();
     try {
-      LeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue);
+      AbstractLeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue);
       AbstractManagedParentQueue parent =
           (AbstractManagedParentQueue) queue.getParent();
 
@@ -2682,11 +2679,16 @@ public String moveApplication(ApplicationId appId,
       if (application == null) {
         throw new YarnException("App to be moved " + appId + " not found.");
       }
-      String sourceQueueName = application.getQueue().getQueueName();
-      LeafQueue source =
+      if (!(application.getQueue() instanceof CSQueue)) {
+        throw new YarnException("Source queue is not a Capacity Scheduler queue");
+      }
+
+      CSQueue csQueue = (CSQueue) application.getQueue();
+      String sourceQueueName = csQueue.getQueuePath();
+      AbstractLeafQueue source =
           this.queueManager.getAndCheckLeafQueue(sourceQueueName);
       String destQueueName = handleMoveToPlanQueue(targetQueueName);
-      LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
+      AbstractLeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
 
       String user = application.getUser();
       try {
@@ -2744,7 +2746,7 @@ public void preValidateMoveApplication(ApplicationId appId,
           ((CSQueue) queue).getQueuePath() : queue.getQueueName();
       this.queueManager.getAndCheckLeafQueue(sourceQueueName);
       String destQueueName = handleMoveToPlanQueue(newQueue);
-      LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
+      AbstractLeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName);
       // Validation check - ACLs, submission limits for user & queue
       String user = application.getUser();
       // Check active partition only when attempt is available
@@ -2771,7 +2773,7 @@ public void preValidateMoveApplication(ApplicationId appId,
    * @param dest
    * @throws YarnException
    */
-  private void checkQueuePartition(FiCaSchedulerApp app, LeafQueue dest)
+  private void checkQueuePartition(FiCaSchedulerApp app, AbstractLeafQueue dest)
       throws YarnException {
     if (!YarnConfiguration.areNodeLabelsEnabled(conf)) {
       return;
@@ -2821,7 +2823,7 @@ public Resource getMaximumResourceCapability(String queueName) {
       }
       return getMaximumResourceCapability();
     }
-    if (!(queue instanceof LeafQueue)) {
+    if (!(queue instanceof AbstractLeafQueue)) {
       LOG.error("queue " + queueName + " is not an leaf queue");
       return getMaximumResourceCapability();
     }
@@ -2830,7 +2832,7 @@ public Resource getMaximumResourceCapability(String queueName) {
     // getMaximumResourceCapability() returns maximum allocation considers
     // per-node maximum resources. So return (component-wise) min of the two.
 
-    Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation();
+    Resource queueMaxAllocation = queue.getMaximumAllocation();
     Resource clusterMaxAllocationConsiderNodeMax =
         getMaximumResourceCapability();
 
@@ -2948,7 +2950,7 @@ public Priority updateApplicationPriority(Priority newPriority,
       ApplicationStateData appState = ApplicationStateData.newInstance(
           rmApp.getSubmitTime(), rmApp.getStartTime(),
           rmApp.getApplicationSubmissionContext(), rmApp.getUser(),
-          rmApp.getCallerContext());
+          rmApp.getRealUser(), rmApp.getCallerContext());
       appState.setApplicationTimeouts(rmApp.getApplicationTimeouts());
       appState.setLaunchTime(rmApp.getLaunchTime());
       rmContext.getStateStore().updateApplicationStateSynchronously(appState,
@@ -2956,7 +2958,7 @@ public Priority updateApplicationPriority(Priority newPriority,
 
       // As we use iterator over a TreeSet for OrderingPolicy, once we change
       // priority then reinsert back to make order correct.
-      LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue());
+      AbstractLeafQueue queue = (AbstractLeafQueue) getQueue(rmApp.getQueue());
       queue.updateApplicationPriority(application, appPriority);
 
       LOG.info("Priority '" + appPriority + "' is updated in queue :"
@@ -3050,9 +3052,9 @@ public void submitResourceCommitRequest(Resource cluster,
       return;
     }
 
-    if (scheduleAsynchronously) {
+    if (asyncSchedulingConf.isScheduleAsynchronously()) {
       // Submit to a commit thread and commit it async-ly
-      resourceCommitterService.addNewCommitRequest(request);
+      asyncSchedulingConf.resourceCommitterService.addNewCommitRequest(request);
     } else{
       // Otherwise do it sync-ly.
       tryCommit(cluster, request, true);
@@ -3277,13 +3279,11 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
           long commitSuccess = System.nanoTime() - commitStart;
           CapacitySchedulerMetrics.getMetrics()
               .addCommitSuccess(commitSuccess);
-          LOG.info("Allocation proposal accepted");
           isSuccess = true;
         } else{
           long commitFailed = System.nanoTime() - commitStart;
           CapacitySchedulerMetrics.getMetrics()
               .addCommitFailure(commitFailed);
-          LOG.info("Failed to accept allocation proposal");
         }
 
         LOG.debug("Allocation proposal accepted={}, proposal={}", isSuccess,
@@ -3299,10 +3299,7 @@ public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
   }
 
   public int getAsyncSchedulingPendingBacklogs() {
-    if (scheduleAsynchronously) {
-      return resourceCommitterService.getPendingBacklogs();
-    }
-    return 0;
+    return asyncSchedulingConf.getPendingBacklogs();
   }
 
   @Override
@@ -3373,14 +3370,14 @@ public long checkAndGetApplicationLifetime(String queueName,
     readLock.lock();
     try {
       CSQueue queue = getQueue(queueName);
-      if (queue == null || !(queue instanceof LeafQueue)) {
+      if (!(queue instanceof AbstractLeafQueue)) {
         return lifetimeRequestedByApp;
       }
 
       long defaultApplicationLifetime =
-          ((LeafQueue) queue).getDefaultApplicationLifetime();
+          queue.getDefaultApplicationLifetime();
       long maximumApplicationLifetime =
-          ((LeafQueue) queue).getMaximumApplicationLifetime();
+          queue.getMaximumApplicationLifetime();
 
       // check only for maximum, that's enough because default can't
       // exceed maximum
@@ -3403,7 +3400,7 @@ public long checkAndGetApplicationLifetime(String queueName,
   @Override
   public long getMaximumApplicationLifetime(String queueName) {
     CSQueue queue = getQueue(queueName);
-    if (queue == null || !(queue instanceof LeafQueue)) {
+    if (!(queue instanceof AbstractLeafQueue)) {
       if (isAmbiguous(queueName)) {
         LOG.error("Ambiguous queue reference: " + queueName
             + " please use full queue path instead.");
@@ -3413,7 +3410,7 @@ public long getMaximumApplicationLifetime(String queueName) {
       return -1;
     }
     // In seconds
-    return ((LeafQueue) queue).getMaximumApplicationLifetime();
+    return queue.getMaximumApplicationLifetime();
   }
 
   @Override
@@ -3443,7 +3440,7 @@ public boolean isMultiNodePlacementEnabled() {
   }
 
   public int getNumAsyncSchedulerThreads() {
-    return asyncSchedulerThreads == null ? 0 : asyncSchedulerThreads.size();
+    return asyncSchedulingConf.getNumAsyncSchedulerThreads();
   }
 
   @VisibleForTesting
@@ -3463,4 +3460,136 @@ public boolean placementConstraintEnabled() {
   public void setQueueManager(CapacitySchedulerQueueManager qm) {
     this.queueManager = qm;
   }
+
+  @VisibleForTesting
+  public List<AsyncScheduleThread> getAsyncSchedulerThreads() {
+    return asyncSchedulingConf.getAsyncSchedulerThreads();
+  }
+
+  static class AsyncSchedulingConfiguration {
+    // timeout to join when we stop this service
+    private static final long THREAD_JOIN_TIMEOUT_MS = 1000;
+
+    @VisibleForTesting
+    protected List<AsyncScheduleThread> asyncSchedulerThreads;
+    private ResourceCommitterService resourceCommitterService;
+
+    private long asyncScheduleInterval;
+    private static final String ASYNC_SCHEDULER_INTERVAL =
+        CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX
+            + ".scheduling-interval-ms";
+    private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5;
+    private long asyncMaxPendingBacklogs;
+
+    private final boolean scheduleAsynchronously;
+
+    AsyncSchedulingConfiguration(CapacitySchedulerConfiguration conf,
+        CapacityScheduler cs) {
+      this.scheduleAsynchronously = conf.getScheduleAynschronously();
+      if (this.scheduleAsynchronously) {
+        this.asyncScheduleInterval = conf.getLong(
+            CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_INTERVAL,
+            CapacitySchedulerConfiguration.DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL);
+        // number of threads for async scheduling
+        int maxAsyncSchedulingThreads = conf.getInt(
+            CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD,
+            1);
+        maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1);
+        this.asyncMaxPendingBacklogs = conf.getInt(
+            CapacitySchedulerConfiguration.
+                SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS,
+            CapacitySchedulerConfiguration.
+                DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS);
+
+        this.asyncSchedulerThreads = new ArrayList<>();
+        for (int i = 0; i < maxAsyncSchedulingThreads; i++) {
+          asyncSchedulerThreads.add(new AsyncScheduleThread(cs));
+        }
+        this.resourceCommitterService = new ResourceCommitterService(cs);
+      }
+    }
+    public boolean isScheduleAsynchronously() {
+      return scheduleAsynchronously;
+    }
+    public long getAsyncScheduleInterval() {
+      return asyncScheduleInterval;
+    }
+    public long getAsyncMaxPendingBacklogs() {
+      return asyncMaxPendingBacklogs;
+    }
+
+    public void startThreads() {
+      if (scheduleAsynchronously) {
+        Preconditions.checkNotNull(asyncSchedulerThreads,
+            "asyncSchedulerThreads is null");
+        for (Thread t : asyncSchedulerThreads) {
+          t.start();
+        }
+
+        resourceCommitterService.start();
+      }
+    }
+
+    public void serviceStopInvoked() throws InterruptedException {
+      if (scheduleAsynchronously && asyncSchedulerThreads != null) {
+        for (Thread t : asyncSchedulerThreads) {
+          t.interrupt();
+          t.join(THREAD_JOIN_TIMEOUT_MS);
+        }
+        resourceCommitterService.interrupt();
+        resourceCommitterService.join(THREAD_JOIN_TIMEOUT_MS);
+      }
+    }
+
+    public void nodeRemoved(int numNodes) {
+      if (scheduleAsynchronously && numNodes == 0) {
+        for (AsyncScheduleThread t : asyncSchedulerThreads) {
+          t.suspendSchedule();
+        }
+      }
+    }
+
+    public int getPendingBacklogs() {
+      if (scheduleAsynchronously) {
+        return resourceCommitterService.getPendingBacklogs();
+      }
+      return 0;
+    }
+
+    public int getNumAsyncSchedulerThreads() {
+      return asyncSchedulerThreads == null ? 0 : asyncSchedulerThreads.size();
+    }
+
+    @VisibleForTesting
+    public List<AsyncScheduleThread> getAsyncSchedulerThreads() {
+      return asyncSchedulerThreads;
+    }
+  }
+
+  @Override
+  public PendingApplicationComparator getPendingApplicationComparator(){
+    return applicationComparator;
+  }
+
+  /**
+   * Comparator that orders applications by their submit time.
+   */
+  class PendingApplicationComparator
+      implements Comparator<FiCaSchedulerApp> {
+
+    @Override
+    public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) {
+      RMApp rmApp1 = rmContext.getRMApps().get(app1.getApplicationId());
+      RMApp rmApp2 = rmContext.getRMApps().get(app2.getApplicationId());
+      if (rmApp1 != null && rmApp2 != null) {
+        return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime());
+      } else if (rmApp1 != null) {
+        return -1;
+      } else if (rmApp2 != null) {
+        return 1;
+      } else{
+        return 0;
+      }
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index ca0d586497d35..5ddc709b57c9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -42,6 +43,7 @@ private CapacitySchedulerConfigValidator() {
   public static boolean validateCSConfiguration(
           final Configuration oldConf, final Configuration newConf,
           final RMContext rmContext) throws IOException {
+    CapacityScheduler liveScheduler = (CapacityScheduler) rmContext.getScheduler();
     CapacityScheduler newCs = new CapacityScheduler();
     try {
       //TODO: extract all the validation steps and replace reinitialize with
@@ -49,10 +51,13 @@ public static boolean validateCSConfiguration(
       newCs.setConf(oldConf);
       newCs.setRMContext(rmContext);
       newCs.init(oldConf);
+      newCs.addNodes(liveScheduler.getAllNodes());
       newCs.reinitialize(newConf, rmContext, true);
       return true;
     } finally {
       newCs.stop();
+      QueueMetrics.clearQueueMetrics();
+      liveScheduler.resetSchedulerMetrics();
     }
   }
 
@@ -184,7 +189,7 @@ private static void validateParentQueueConversion(CSQueue oldQueue,
                 + " is set to true");
       }
 
-      if (newQueue instanceof LeafQueue) {
+      if (newQueue instanceof AbstractLeafQueue) {
         LOG.info("Converting the parent queue: {} to leaf queue.", oldQueue.getQueuePath());
       }
     }
@@ -192,7 +197,7 @@ private static void validateParentQueueConversion(CSQueue oldQueue,
 
   private static void validateLeafQueueConversion(CSQueue oldQueue,
                                                   CSQueue newQueue) throws IOException {
-    if (oldQueue instanceof LeafQueue && newQueue instanceof ParentQueue) {
+    if (oldQueue instanceof AbstractLeafQueue && newQueue instanceof ParentQueue) {
       if (isEitherQueueStopped(oldQueue.getState(), newQueue.getState())) {
         LOG.info("Converting the leaf queue: {} to parent queue.", oldQueue.getQueuePath());
       } else {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 5e614f149eda3..51616da14b60b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueCapacityConfigParser;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -73,9 +74,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.Set;
 
 public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration {
 
@@ -272,6 +273,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS =
       SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs";
 
+  @Private
+  public static final String SCHEDULE_ASYNCHRONOUSLY_INTERVAL =
+      SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms";
+  @Private
+  public static final long DEFAULT_SCHEDULE_ASYNCHRONOUSLY_INTERVAL = 5;
+
   @Private
   public static final String APP_FAIL_FAST = PREFIX + "application.fail-fast";
 
@@ -413,8 +420,25 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
   public static final String MAPPING_RULE_FORMAT_DEFAULT =
       MAPPING_RULE_FORMAT_LEGACY;
+
+  private static final QueueCapacityConfigParser queueCapacityConfigParser
+      = new QueueCapacityConfigParser();
+
   private ConfigurationProperties configurationProperties;
 
+  public int getMaximumAutoCreatedQueueDepth(String queuePath) {
+    return getInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH,
+        getInt(PREFIX + MAXIMUM_QUEUE_DEPTH, DEFAULT_MAXIMUM_QUEUE_DEPTH));
+  }
+
+  public void setMaximumAutoCreatedQueueDepth(String queuePath, int value) {
+    setInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, value);
+  }
+
+  public void setMaximumAutoCreatedQueueDepth(int value) {
+    setInt(PREFIX + MAXIMUM_QUEUE_DEPTH, value);
+  }
+
   /**
    * Different resource types supported.
    */
@@ -454,13 +478,17 @@ static String getUserPrefix(String user) {
     return PREFIX + "user." + user + DOT;
   }
 
-  private String getNodeLabelPrefix(String queue, String label) {
+  public static String getNodeLabelPrefix(String queue, String label) {
     if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
       return getQueuePrefix(queue);
     }
     return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
   }
 
+  public void setMaximumSystemApplications(int numMaxApps) {
+    setInt(MAXIMUM_SYSTEM_APPLICATIONS, numMaxApps);
+  }
+
   public int getMaximumSystemApplications() {
     int maxApplications =
       getInt(MAXIMUM_SYSTEM_APPLICATIONS, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS);
@@ -533,15 +561,15 @@ public void setLabeledQueueWeight(String queue, String label, float weight) {
     set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX);
   }
 
-  public float getLabeledQueueWeight(String queue, String label) {
-    String configuredValue = get(getNodeLabelPrefix(queue, label) + CAPACITY);
+  public float getLabeledQueueWeight(QueuePath queue, String label) {
+    String configuredValue = get(getNodeLabelPrefix(queue.getFullPath(), label) + CAPACITY);
     float weight = extractFloatValueFromWeightConfig(configuredValue);
-    throwExceptionForUnexpectedWeight(weight, queue, label);
+    throwExceptionForUnexpectedWeight(weight, queue.getFullPath(), label);
     return weight;
   }
 
-  public float getNonLabeledQueueCapacity(String queue) {
-    String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY);
+  public float getNonLabeledQueueCapacity(QueuePath queue) {
+    String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + CAPACITY);
     boolean absoluteResourceConfigured = (configuredCapacity != null)
         && RESOURCE_PATTERN.matcher(configuredCapacity).find();
     if (absoluteResourceConfigured || configuredWeightAsCapacity(
@@ -550,10 +578,10 @@ public float getNonLabeledQueueCapacity(String queue) {
       // root.From AbstractCSQueue, absolute resource will be parsed and
       // updated. Once nodes are added/removed in cluster, capacity in
       // percentage will also be re-calculated.
-      return queue.equals("root") ? 100.0f : 0f;
+      return queue.isRoot() ? 100.0f : 0f;
     }
 
-    float capacity = queue.equals("root")
+    float capacity = queue.isRoot()
         ? 100.0f
         : (configuredCapacity == null)
             ? 0f
@@ -564,7 +592,7 @@ public float getNonLabeledQueueCapacity(String queue) {
           "Illegal " + "capacity of " + capacity + " for queue " + queue);
     }
     LOG.debug("CSConf - getCapacity: queuePrefix={}, capacity={}",
-        getQueuePrefix(queue), capacity);
+        getQueuePrefix(queue.getFullPath()), capacity);
 
     return capacity;
   }
@@ -592,8 +620,8 @@ public void setCapacity(String queue, String absoluteResourceCapacity) {
 
   }
 
-  public float getNonLabeledQueueMaximumCapacity(String queue) {
-    String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY);
+  public float getNonLabeledQueueMaximumCapacity(QueuePath queue) {
+    String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + MAXIMUM_CAPACITY);
     boolean matcher = (configuredCapacity != null)
         && RESOURCE_PATTERN.matcher(configuredCapacity).find();
     if (matcher) {
@@ -807,9 +835,9 @@ private float extractFloatValueFromWeightConfig(String configureValue) {
     }
   }
 
-  private float internalGetLabeledQueueCapacity(String queue, String label,
+  private float internalGetLabeledQueueCapacity(QueuePath queue, String label,
       String suffix, float defaultValue) {
-    String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
+    String capacityPropertyName = getNodeLabelPrefix(queue.getFullPath(), label) + suffix;
     String configuredCapacity = get(capacityPropertyName);
     boolean absoluteResourceConfigured =
         (configuredCapacity != null) && RESOURCE_PATTERN.matcher(
@@ -820,10 +848,10 @@ private float internalGetLabeledQueueCapacity(String queue, String label,
       // root.From AbstractCSQueue, absolute resource, and weight will be parsed
       // and updated separately. Once nodes are added/removed in cluster,
       // capacity is percentage will also be re-calculated.
-      return queue.equals("root") ? 100.0f : defaultValue;
+      return queue.isRoot() ? 100.0f : defaultValue;
     }
 
-    float capacity = queue.equals("root") ? 100.0f
+    float capacity = queue.isRoot() ? 100.0f
         : getFloat(capacityPropertyName, defaultValue);
     if (capacity < MINIMUM_CAPACITY_VALUE
         || capacity > MAXIMUM_CAPACITY_VALUE) {
@@ -834,17 +862,17 @@ private float internalGetLabeledQueueCapacity(String queue, String label,
     }
     if (LOG.isDebugEnabled()) {
       LOG.debug(
-          "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue,
+          "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue.getFullPath(),
               label) + ", capacity=" + capacity);
     }
     return capacity;
   }
 
-  public float getLabeledQueueCapacity(String queue, String label) {
+  public float getLabeledQueueCapacity(QueuePath queue, String label) {
     return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f);
   }
 
-  public float getLabeledQueueMaximumCapacity(String queue, String label) {
+  public float getLabeledQueueMaximumCapacity(QueuePath queue, String label) {
     return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f);
   }
 
@@ -861,13 +889,13 @@ public void setDefaultNodeLabelExpression(String queue, String exp) {
     set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp);
   }
 
-  public float getMaximumAMResourcePercentPerPartition(String queue,
+  public float getMaximumAMResourcePercentPerPartition(QueuePath queue,
       String label) {
     // If per-partition max-am-resource-percent is not configured,
     // use default value as max-am-resource-percent for this queue.
-    return getFloat(getNodeLabelPrefix(queue, label)
+    return getFloat(getNodeLabelPrefix(queue.getFullPath(), label)
         + MAXIMUM_AM_RESOURCE_SUFFIX,
-        getMaximumApplicationMasterResourcePerQueuePercent(queue));
+        getMaximumApplicationMasterResourcePerQueuePercent(queue.getFullPath()));
   }
 
   public void setMaximumAMResourcePercentPerPartition(String queue,
@@ -913,6 +941,90 @@ private static String getAclKey(AccessType acl) {
     return "acl_" + StringUtils.toLowerCase(acl.toString());
   }
 
+  /**
+   * Creates a mapping of queue ACLs for a Legacy Auto Created Leaf Queue.
+   *
+   * @param parentQueuePath the parent's queue path
+   * @return A mapping of the queue ACLs.
+   */
+  public Map<AccessType, AccessControlList> getACLsForLegacyAutoCreatedLeafQueue(
+      String parentQueuePath) {
+    final String prefix =
+        getQueuePrefix(getAutoCreatedQueueTemplateConfPrefix(
+            parentQueuePath));
+
+    Map<String, String> properties = new HashMap<>();
+    for (QueueACL acl : QueueACL.values()) {
+      final String key = getAclKey(acl);
+      final String value = get(prefix + key);
+      if (value != null) {
+        properties.put(key, get(prefix + key));
+      }
+    }
+    return getACLsFromProperties(properties);
+  }
+
+  /**
+   * Creates a mapping of queue ACLs for a Flexible Auto Created Parent Queue.
+   * The .parent-template is preferred to .template ACLs.
+   *
+   * @param aqc The AQC templates to use.
+   * @return A mapping of the queue ACLs.
+   */
+  public static Map<AccessType, AccessControlList> getACLsForFlexibleAutoCreatedParentQueue(
+      AutoCreatedQueueTemplate aqc) {
+    return getACLsFromProperties(aqc.getParentOnlyProperties(),
+        aqc.getTemplateProperties());
+  }
+
+  /**
+   * Creates a mapping of queue ACLs for a Flexible Auto Created Leaf Queue.
+   * The .leaf-template is preferred to .template ACLs.
+   *
+   * @param aqc The AQC templates to use.
+   * @return A mapping of the queue ACLs.
+   */
+  public static Map<AccessType, AccessControlList> getACLsForFlexibleAutoCreatedLeafQueue(
+      AutoCreatedQueueTemplate aqc) {
+    return getACLsFromProperties(aqc.getLeafOnlyProperties(),
+        aqc.getTemplateProperties());
+  }
+
+  /**
+   * Transforms the string ACL properties to AccessType and AccessControlList mapping.
+   *
+   * @param properties The ACL properties.
+   * @return A mapping of the queue ACLs.
+   */
+  private static Map<AccessType, AccessControlList> getACLsFromProperties(
+      Map<String, String> properties) {
+    return getACLsFromProperties(properties, new HashMap<>());
+  }
+
+  /**
+   * Transforms the string ACL properties to AccessType and AccessControlList mapping.
+   *
+   * @param properties The ACL properties.
+   * @param fallbackProperties The fallback properties to use.
+   * @return A mapping of the queue ACLs.
+   */
+  private static Map<AccessType, AccessControlList> getACLsFromProperties(
+      Map<String, String> properties, Map<String, String> fallbackProperties) {
+    Map<AccessType, AccessControlList> acls = new HashMap<>();
+    for (QueueACL acl : QueueACL.values()) {
+      String aclStr = properties.get(getAclKey(acl));
+      if (aclStr == null) {
+        aclStr = fallbackProperties.get(getAclKey(acl));
+        if (aclStr == null) {
+          aclStr = NONE_ACL;
+        }
+      }
+      acls.put(SchedulerUtils.toAccessType(acl),
+          new AccessControlList(aclStr));
+    }
+    return acls;
+  }
+
   @Override
   public Map<ReservationACL, AccessControlList> getReservationAcls(String
         queue) {
@@ -1649,6 +1761,12 @@ public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
     return labelsByQueue;
   }
 
+  public Priority getClusterLevelApplicationMaxPriority() {
+    return Priority.newInstance(getInt(
+        YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY,
+        YarnConfiguration.DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY));
+  }
+
   public Integer getDefaultApplicationPriorityConfPerQueue(String queue) {
     Integer defaultPriority = getInt(getQueuePrefix(queue)
         + DEFAULT_APPLICATION_PRIORITY,
@@ -2115,6 +2233,13 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
   public static final int
       DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000;
 
+  @Private
+  public static final String MAXIMUM_QUEUE_DEPTH =
+      AUTO_QUEUE_CREATION_V2_PREFIX + "maximum-queue-depth";
+
+  @Private
+  public static final int DEFAULT_MAXIMUM_QUEUE_DEPTH = 2;
+
   @Private
   public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false;
 
@@ -2180,6 +2305,11 @@ public String getAutoCreatedQueueTemplateConfPrefix(String queuePath) {
     return queuePath + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
   }
 
+  @Private
+  public QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(String queuePath) {
+    return new QueuePath(queuePath, AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX);
+  }
+
   @Private
   public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY =
       "auto-create-child-queue.fail-on-exceeding-parent-capacity";
@@ -2556,20 +2686,30 @@ public Resource getMaximumResourceRequirement(String label, String queue,
   }
 
   @VisibleForTesting
-  public void setMinimumResourceRequirement(String label, String queue,
+  public void setMinimumResourceRequirement(String label, QueuePath queue,
       Resource resource) {
     updateMinMaxResourceToConf(label, queue, resource, CAPACITY);
   }
 
   @VisibleForTesting
-  public void setMaximumResourceRequirement(String label, String queue,
+  public void setMaximumResourceRequirement(String label, QueuePath queue,
       Resource resource) {
     updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_CAPACITY);
   }
 
-  private void updateMinMaxResourceToConf(String label, String queue,
+  public Map<String, QueueCapacityVector> parseConfiguredResourceVector(
+      String queuePath, Set<String> labels) {
+    Map<String, QueueCapacityVector> queueResourceVectors = new HashMap<>();
+    for (String label : labels) {
+      queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, queuePath, label));
+    }
+
+    return queueResourceVectors;
+  }
+
+  private void updateMinMaxResourceToConf(String label, QueuePath queue,
       Resource resource, String type) {
-    if (queue.equals("root")) {
+    if (queue.isRoot()) {
       throw new IllegalArgumentException(
           "Cannot set resource, root queue will take 100% of cluster capacity");
     }
@@ -2584,9 +2724,9 @@ private void updateMinMaxResourceToConf(String label, String queue,
             + ResourceUtils.
             getCustomResourcesStrings(resource) + "]");
 
-    String prefix = getQueuePrefix(queue) + type;
+    String prefix = getQueuePrefix(queue.getFullPath()) + type;
     if (!label.isEmpty()) {
-      prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+      prefix = getQueuePrefix(queue.getFullPath()) + ACCESSIBLE_NODE_LABELS + DOT + label
           + DOT + type;
     }
     set(prefix, resourceString.toString());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index ae74989a7265f..5795a7e30faac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import java.util.Comparator;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -41,6 +39,8 @@
  */
 public interface CapacitySchedulerContext {
   CapacitySchedulerConfiguration getConfiguration();
+
+  CapacitySchedulerQueueContext getQueueContext();
   
   Resource getMinimumResourceCapability();
 
@@ -102,4 +102,5 @@ public interface CapacitySchedulerContext {
    */
   Clock getClock();
 
+  CapacityScheduler.PendingApplicationComparator getPendingApplicationComparator();
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java
new file mode 100644
index 0000000000000..4d1113765529e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueContext.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerHealth;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+/**
+ * Class to store common queue related information, like instances
+ * to necessary manager classes or the global CapacityScheduler
+ * configuration.
+ */
+public class CapacitySchedulerQueueContext {
+
+  // Manager classes
+  private final CapacitySchedulerContext csContext;
+  private final CapacitySchedulerQueueManager queueManager;
+  private final RMNodeLabelsManager labelManager;
+  private final PreemptionManager preemptionManager;
+  private final ActivitiesManager activitiesManager;
+  private final ResourceCalculator resourceCalculator;
+
+  // CapacityScheduler configuration
+  private CapacitySchedulerConfiguration configuration;
+
+  private Resource minimumAllocation;
+
+  public CapacitySchedulerQueueContext(CapacitySchedulerContext csContext) {
+    this.csContext = csContext;
+    this.queueManager = csContext.getCapacitySchedulerQueueManager();
+    this.labelManager = csContext.getRMContext().getNodeLabelManager();
+    this.preemptionManager = csContext.getPreemptionManager();
+    this.activitiesManager = csContext.getActivitiesManager();
+    this.resourceCalculator = csContext.getResourceCalculator();
+
+    this.configuration = new CapacitySchedulerConfiguration(csContext.getConfiguration());
+    this.minimumAllocation = csContext.getMinimumResourceCapability();
+  }
+
+  public void reinitialize() {
+    // When csConfProvider.loadConfiguration is called, the useLocalConfigurationProvider is
+    // correctly set to load the config entries from the capacity-scheduler.xml.
+    // For this reason there is no need to reload from it again.
+    this.configuration = new CapacitySchedulerConfiguration(csContext.getConfiguration(), false);
+    this.minimumAllocation = csContext.getMinimumResourceCapability();
+  }
+
+  public CapacitySchedulerQueueManager getQueueManager() {
+    return queueManager;
+  }
+
+  public RMNodeLabelsManager getLabelManager() {
+    return labelManager;
+  }
+
+  public PreemptionManager getPreemptionManager() {
+    return preemptionManager;
+  }
+
+  public ActivitiesManager getActivitiesManager() {
+    return activitiesManager;
+  }
+
+  public ResourceCalculator getResourceCalculator() {
+    return resourceCalculator;
+  }
+
+  public CapacitySchedulerConfiguration getConfiguration() {
+    return configuration;
+  }
+
+  public void setConfigurationEntry(String name, String value) {
+    this.configuration.set(name, value);
+  }
+
+  public Resource getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  public Resource getClusterResource() {
+    return csContext.getClusterResource();
+  }
+
+  public ResourceUsage getClusterResourceUsage() {
+    return queueManager.getRootQueue().getQueueResourceUsage();
+  }
+
+  public SchedulerHealth getSchedulerHealth() {
+    return csContext.getSchedulerHealth();
+  }
+
+  public long getLastNodeUpdateTime() {
+    return csContext.getLastNodeUpdateTime();
+  }
+
+  public FiCaSchedulerNode getNode(NodeId nodeId) {
+    return csContext.getNode(nodeId);
+  }
+
+  public FiCaSchedulerApp getApplicationAttempt(
+      ApplicationAttemptId applicationAttemptId) {
+    return csContext.getApplicationAttempt(applicationAttemptId);
+  }
+
+  public CapacityScheduler.PendingApplicationComparator getApplicationComparator() {
+    return csContext.getPendingApplicationComparator();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index f0c8a27f95493..ba6849cb7807b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -29,6 +29,7 @@
 import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.security.PrivilegedEntity;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -42,17 +43,18 @@
 import org.apache.hadoop.yarn.security.Permission;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getACLsForFlexibleAutoCreatedLeafQueue;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getACLsForFlexibleAutoCreatedParentQueue;
+
 /**
  *
  * Context of the Queues in Capacity Scheduler.
@@ -72,7 +74,6 @@ public CSQueue hook(CSQueue queue) {
     }
   }
 
-  private static final int MAXIMUM_DYNAMIC_QUEUE_DEPTH = 2;
   private static final QueueHook NOOP = new QueueHook();
   private CapacitySchedulerContext csContext;
   private final YarnAuthorizationProvider authorizer;
@@ -106,6 +107,11 @@ public CSQueue getRootQueue() {
     return this.root;
   }
 
+  @VisibleForTesting
+  protected void setRootQueue(CSQueue rootQueue) {
+    this.root = rootQueue;
+  }
+
   @Override
   public Map<String, CSQueue> getQueues() {
     return queues.getFullNameQueues();
@@ -167,7 +173,7 @@ public void setCapacitySchedulerContext(
   public void initializeQueues(CapacitySchedulerConfiguration conf)
     throws IOException {
     configuredNodeLabels = new ConfiguredNodeLabels(conf);
-    root = parseQueue(this.csContext, conf, null,
+    root = parseQueue(this.csContext.getQueueContext(), conf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues, NOOP);
     setQueueAcls(authorizer, appPriorityACLManager, queues);
     labelManager.reinitializeQueueLabels(getQueueToLabels());
@@ -183,7 +189,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf)
     // Parse new queues
     CSQueueStore newQueues = new CSQueueStore();
     configuredNodeLabels = new ConfiguredNodeLabels(newConf);
-    CSQueue newRoot = parseQueue(this.csContext, newConf, null,
+    CSQueue newRoot = parseQueue(this.csContext.getQueueContext(), newConf, null,
         CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP);
 
     // When failing over, if using configuration store, don't validate queue
@@ -215,7 +221,7 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf)
 
   /**
    * Parse the queue from the configuration.
-   * @param csContext the CapacitySchedulerContext
+   * @param queueContext the CapacitySchedulerQueueContext
    * @param conf the CapacitySchedulerConfiguration
    * @param parent the parent queue
    * @param queueName the queue name
@@ -226,99 +232,62 @@ public void reinitializeQueues(CapacitySchedulerConfiguration newConf)
    * @throws IOException
    */
   static CSQueue parseQueue(
-      CapacitySchedulerContext csContext,
-      CapacitySchedulerConfiguration conf,
-      CSQueue parent, String queueName,
-      CSQueueStore newQueues,
-      CSQueueStore oldQueues,
+      CapacitySchedulerQueueContext queueContext, CapacitySchedulerConfiguration conf,
+      CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore oldQueues,
       QueueHook hook) throws IOException {
     CSQueue queue;
-    String fullQueueName = (parent == null) ?
-        queueName :
-        (parent.getQueuePath() + "." + queueName);
+    String fullQueueName = (parent == null) ? queueName :
+        (QueuePath.createFromQueues(parent.getQueuePath(), queueName).getFullPath());
     String[] staticChildQueueNames = conf.getQueues(fullQueueName);
     List<String> childQueueNames = staticChildQueueNames != null ?
         Arrays.asList(staticChildQueueNames) : Collections.emptyList();
-
-    boolean isReservableQueue = conf.isReservable(fullQueueName);
-    boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
-        fullQueueName);
-    // if a queue is eligible for auto queue creation v2
-    // it must be a ParentQueue (even if it is empty)
-    boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled(
-        fullQueueName);
-    boolean isDynamicParent = false;
-
-    // Auto created parent queues might not have static children, but they
-    // must be kept as a ParentQueue
     CSQueue oldQueue = oldQueues.get(fullQueueName);
-    if (oldQueue instanceof ParentQueue) {
-      isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue();
-    }
 
-    if (childQueueNames.size() == 0 && !isDynamicParent &&
-        !isAutoQueueCreationV2Enabled) {
-      if (null == parent) {
-        throw new IllegalStateException(
-            "Queue configuration missing child queue names for " + queueName);
-      }
-      // Check if the queue will be dynamically managed by the Reservation
-      // system
+    boolean isReservableQueue = conf.isReservable(fullQueueName);
+    boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueueName);
+    // if a queue is eligible for auto queue creation v2 it must be a ParentQueue
+    // (even if it is empty)
+    final boolean isDynamicParent = oldQueue instanceof ParentQueue && oldQueue.isDynamicQueue();
+    boolean isAutoQueueCreationEnabledParent = isDynamicParent || conf.isAutoQueueCreationV2Enabled(
+        fullQueueName) || isAutoCreateEnabled;
+
+    if (childQueueNames.size() == 0 && !isAutoQueueCreationEnabledParent) {
+      validateParent(parent, queueName);
+      // Check if the queue will be dynamically managed by the Reservation system
       if (isReservableQueue) {
-        queue = new PlanQueue(csContext, queueName, parent,
-            oldQueues.get(fullQueueName));
-
-        //initializing the "internal" default queue, for SLS compatibility
-        String defReservationId =
-            queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
-
-        List<CSQueue> childQueues = new ArrayList<>();
-        ReservationQueue resQueue = new ReservationQueue(csContext,
-            defReservationId, (PlanQueue) queue);
-        try {
-          resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
-        } catch (SchedulerDynamicEditException e) {
-          throw new IllegalStateException(e);
-        }
-        childQueues.add(resQueue);
-        ((PlanQueue) queue).setChildQueues(childQueues);
-        newQueues.add(resQueue);
-
-      } else if (isAutoCreateEnabled) {
-        queue = new ManagedParentQueue(csContext, queueName, parent,
-            oldQueues.get(fullQueueName));
-
-      } else{
-        queue = new LeafQueue(csContext, queueName, parent,
-            oldQueues.get(fullQueueName));
-        // Used only for unit tests
-        queue = hook.hook(queue);
+        queue = new PlanQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
+        ReservationQueue defaultResQueue = ((PlanQueue) queue).initializeDefaultInternalQueue();
+        newQueues.add(defaultResQueue);
+      } else {
+        queue = new LeafQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
       }
-    } else{
+
+      queue = hook.hook(queue);
+    } else {
       if (isReservableQueue) {
-        throw new IllegalStateException(
-            "Only Leaf Queues can be reservable for " + fullQueueName);
+        throw new IllegalStateException("Only Leaf Queues can be reservable for " + fullQueueName);
       }
 
       ParentQueue parentQueue;
       if (isAutoCreateEnabled) {
-        parentQueue = new ManagedParentQueue(csContext, queueName, parent,
-            oldQueues.get(fullQueueName));
-      } else{
-        parentQueue = new ParentQueue(csContext, queueName, parent,
-            oldQueues.get(fullQueueName));
+        parentQueue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get(
+            fullQueueName));
+      } else {
+        parentQueue = new ParentQueue(queueContext, queueName, parent, oldQueues.get(
+            fullQueueName));
       }
 
-      // Used only for unit tests
       queue = hook.hook(parentQueue);
-
       List<CSQueue> childQueues = new ArrayList<>();
       for (String childQueueName : childQueueNames) {
-        CSQueue childQueue = parseQueue(csContext, conf, queue, childQueueName,
-            newQueues, oldQueues, hook);
+        CSQueue childQueue = parseQueue(queueContext, conf, queue, childQueueName, newQueues,
+            oldQueues, hook);
         childQueues.add(childQueue);
       }
-      parentQueue.setChildQueues(childQueues);
+
+      if (!childQueues.isEmpty()) {
+        parentQueue.setChildQueues(childQueues);
+      }
 
     }
 
@@ -375,8 +344,8 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer,
       permissions.add(
           new Permission(csQueue.getPrivilegedEntity(), csQueue.getACLs()));
 
-      if (queue instanceof LeafQueue) {
-        LeafQueue lQueue = (LeafQueue) queue;
+      if (queue instanceof AbstractLeafQueue) {
+        AbstractLeafQueue lQueue = (AbstractLeafQueue) queue;
 
         // Clear Priority ACLs first since reinitialize also call same.
         appPriorityACLManager.clearPriorityACLs(lQueue.getQueuePath());
@@ -397,17 +366,17 @@ public static void setQueueAcls(YarnAuthorizationProvider authorizer,
    * @throws YarnException if the queue does not exist or the queue
    *           is not the type of LeafQueue.
    */
-  public LeafQueue getAndCheckLeafQueue(String queue) throws YarnException {
+  public AbstractLeafQueue getAndCheckLeafQueue(String queue) throws YarnException {
     CSQueue ret = this.getQueue(queue);
     if (ret == null) {
       throw new YarnException("The specified Queue: " + queue
           + " doesn't exist");
     }
-    if (!(ret instanceof LeafQueue)) {
+    if (!(ret instanceof AbstractLeafQueue)) {
       throw new YarnException("The specified Queue: " + queue
           + " is not a Leaf Queue.");
     }
-    return (LeafQueue) ret;
+    return (AbstractLeafQueue) ret;
   }
 
   /**
@@ -527,7 +496,7 @@ public void addLegacyDynamicQueue(Queue queue)
    * @throws YarnException if the given path is not eligible to be auto created
    * @throws IOException if the given path can not be added to the parent
    */
-  public LeafQueue createQueue(QueuePath queue)
+  public AbstractLeafQueue createQueue(QueuePath queue)
       throws YarnException, IOException {
     String leafQueueName = queue.getLeafName();
     String parentQueueName = queue.getParent();
@@ -587,14 +556,6 @@ public List<String> determineMissingParents(
         && parentCandidate.length() != 0) {
       ++firstStaticParentDistance;
 
-      if (firstStaticParentDistance > MAXIMUM_DYNAMIC_QUEUE_DEPTH) {
-        throw new SchedulerDynamicEditException(
-            "Could not auto create queue " + queue.getFullPath()
-                + ". The distance of the LeafQueue from the first static " +
-                "ParentQueue is " + firstStaticParentDistance + ", which is " +
-                "above the limit.");
-      }
-
       if (firstExistingParent == null) {
         parentsToCreate.addFirst(parentCandidate.toString());
       }
@@ -609,6 +570,16 @@ public List<String> determineMissingParents(
       firstExistingStaticParent = getQueue(parentCandidate.toString());
     }
 
+    int maximumDepthOfStaticParent = csContext.getConfiguration().getMaximumAutoCreatedQueueDepth(
+        firstExistingStaticParent.getQueuePath());
+    if (firstStaticParentDistance > maximumDepthOfStaticParent) {
+      throw new SchedulerDynamicEditException(
+          "Could not auto create queue " + queue.getFullPath()
+              + ". The distance of the LeafQueue from the first static " +
+              "ParentQueue is " + firstStaticParentDistance + ", which is " +
+              "above the limit.");
+    }
+
     if (!(firstExistingParent instanceof ParentQueue)) {
       throw new SchedulerDynamicEditException(
           "Could not auto create hierarchy of "
@@ -628,12 +599,50 @@ public List<String> determineMissingParents(
     return parentsToCreate;
   }
 
+  public List<Permission> getPermissionsForDynamicQueue(
+      QueuePath queuePath,
+      CapacitySchedulerConfiguration csConf) {
+    List<Permission> permissions = new ArrayList<>();
+
+    try {
+      PrivilegedEntity privilegedEntity = new PrivilegedEntity(queuePath.getFullPath());
+
+      CSQueue parentQueue = getQueueByFullName(queuePath.getParent());
+      if (parentQueue == null) {
+        for (String missingParent : determineMissingParents(queuePath)) {
+          String parentOfMissingParent = new QueuePath(missingParent).getParent();
+          permissions.add(new Permission(new PrivilegedEntity(missingParent),
+              getACLsForFlexibleAutoCreatedParentQueue(
+                  new AutoCreatedQueueTemplate(csConf,
+                      new QueuePath(parentOfMissingParent)))));
+        }
+      }
+
+      if (parentQueue instanceof AbstractManagedParentQueue) {
+        // An AbstractManagedParentQueue must have been found for Legacy AQC
+        permissions.add(new Permission(privilegedEntity,
+            csConf.getACLsForLegacyAutoCreatedLeafQueue(queuePath.getParent())));
+      } else {
+        // Every other case must be a Flexible Leaf Queue
+        permissions.add(new Permission(privilegedEntity,
+            getACLsForFlexibleAutoCreatedLeafQueue(
+                new AutoCreatedQueueTemplate(csConf, new QueuePath(queuePath.getParent())))));
+      }
+
+    } catch (SchedulerDynamicEditException e) {
+      LOG.debug("Could not determine missing parents for queue {} reason {}",
+          queuePath.getFullPath(), e.getMessage());
+    }
+
+    return permissions;
+  }
+
   /**
    * Get {@code ConfiguredNodeLabels} which contains the configured node labels
    * for all queues.
    * @return configured node labels
    */
-  public ConfiguredNodeLabels getConfiguredNodeLabels() {
+  public ConfiguredNodeLabels getConfiguredNodeLabelsForAllQueues() {
     return configuredNodeLabels;
   }
 
@@ -668,7 +677,7 @@ private LeafQueue createAutoQueue(QueuePath queue)
     return leafQueue;
   }
 
-  private LeafQueue createLegacyAutoQueue(QueuePath queue)
+  private AbstractLeafQueue createLegacyAutoQueue(QueuePath queue)
       throws IOException, SchedulerDynamicEditException {
     CSQueue parentQueue = getQueue(queue.getParent());
     // Case 1: Handle ManagedParentQueue
@@ -676,7 +685,7 @@ private LeafQueue createLegacyAutoQueue(QueuePath queue)
         (ManagedParentQueue) parentQueue;
     AutoCreatedLeafQueue autoCreatedLeafQueue =
         new AutoCreatedLeafQueue(
-            csContext, queue.getLeafName(), autoCreateEnabledParentQueue);
+            csContext.getQueueContext(), queue.getLeafName(), autoCreateEnabledParentQueue);
 
     addLegacyDynamicQueue(autoCreatedLeafQueue);
     return autoCreatedLeafQueue;
@@ -714,4 +723,11 @@ private boolean isDanglingDynamicQueue(
     // that existingQueues contain valid dynamic queues.
     return !isDynamicQueue(parent);
   }
+
+  private static void validateParent(CSQueue parent, String queueName) {
+    if (parent == null) {
+      throw new IllegalStateException("Queue configuration missing child queue names for "
+          + queueName);
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index cc282ef59a23e..f33de96e2c881 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -19,2391 +19,28 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import java.io.IOException;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
 
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.DateUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.util.Sets;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueInfo;
-import org.apache.hadoop.yarn.api.records.QueueState;
-import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.security.AccessType;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
-import org.apache.hadoop.yarn.server.utils.Lock;
-import org.apache.hadoop.yarn.server.utils.Lock.NoLock;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.hadoop.yarn.util.resource.Resources;
-
-import org.apache.hadoop.classification.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
-public class LeafQueue extends AbstractCSQueue {
+public class LeafQueue extends AbstractLeafQueue {
   private static final Logger LOG =
       LoggerFactory.getLogger(LeafQueue.class);
 
-  private float absoluteUsedCapacity = 0.0f;
-
-  // TODO the max applications should consider label
-  protected int maxApplications;
-  protected volatile int maxApplicationsPerUser;
-  
-  private float maxAMResourcePerQueuePercent;
-
-  private volatile int nodeLocalityDelay;
-  private volatile int rackLocalityAdditionalDelay;
-  private volatile boolean rackLocalityFullReset;
-
-  Map<ApplicationAttemptId, FiCaSchedulerApp> applicationAttemptMap =
-      new ConcurrentHashMap<>();
-
-  private Priority defaultAppPriorityPerQueue;
-
-  private final OrderingPolicy<FiCaSchedulerApp> pendingOrderingPolicy;
-
-  private volatile float minimumAllocationFactor;
-
-  private final RecordFactory recordFactory = 
-    RecordFactoryProvider.getRecordFactory(null);
-
-  private final UsersManager usersManager;
-
-  // cache last cluster resource to compute actual capacity
-  private Resource lastClusterResource = Resources.none();
-
-  private final QueueResourceLimitsInfo queueResourceLimitsInfo =
-      new QueueResourceLimitsInfo();
-
-  private volatile ResourceLimits cachedResourceLimitsForHeadroom = null;
-
-  private volatile OrderingPolicy<FiCaSchedulerApp> orderingPolicy = null;
-
-  // Map<Partition, Map<SchedulingMode, Map<User, CachedUserLimit>>>
-  // Not thread safe: only the last level is a ConcurrentMap
-  @VisibleForTesting
-  Map<String, Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>>
-      userLimitsCache = new HashMap<>();
-
-  // Not thread safe
-  @VisibleForTesting
-  long currentUserLimitCacheVersion = 0;
-
-  // record all ignore partition exclusivityRMContainer, this will be used to do
-  // preemption, key is the partition of the RMContainer allocated on
-  private Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityRMContainers =
-      new ConcurrentHashMap<>();
-
-  List<AppPriorityACLGroup> priorityAcls =
-      new ArrayList<AppPriorityACLGroup>();
-
-  private final List<FiCaSchedulerApp> runnableApps = new ArrayList<>();
-  private final List<FiCaSchedulerApp> nonRunnableApps = new ArrayList<>();
-
-  @SuppressWarnings({ "unchecked", "rawtypes" })
-  public LeafQueue(CapacitySchedulerContext cs,
+  public LeafQueue(CapacitySchedulerQueueContext queueContext,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
-    this(cs, cs.getConfiguration(), queueName, parent, old, false);
+    this(queueContext, queueName, parent, old, false);
   }
 
-  public LeafQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration configuration,
-      String queueName, CSQueue parent, CSQueue old) throws IOException {
-    this(cs, configuration, queueName, parent, old, false);
-  }
-
-  public LeafQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration configuration,
+  public LeafQueue(CapacitySchedulerQueueContext queueContext,
       String queueName, CSQueue parent, CSQueue old, boolean isDynamic) throws
       IOException {
-    super(cs, configuration, queueName, parent, old);
-    setDynamicQueue(isDynamic);
-
-    this.usersManager = new UsersManager(metrics, this, labelManager, csContext,
-        resourceCalculator);
-
-    // One time initialization is enough since it is static ordering policy
-    this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps();
-
-    LOG.debug("LeafQueue: name={}, fullname={}", queueName, getQueuePath());
-
-    setupQueueConfigs(cs.getClusterResource(), configuration);
-
-  }
-
-  @SuppressWarnings("checkstyle:nowhitespaceafter")
-  protected void setupQueueConfigs(Resource clusterResource,
-      CapacitySchedulerConfiguration conf) throws
-      IOException {
-    writeLock.lock();
-    try {
-      CapacitySchedulerConfiguration schedConf = csContext.getConfiguration();
-      super.setupQueueConfigs(clusterResource, conf);
-
-      this.lastClusterResource = clusterResource;
-
-      this.cachedResourceLimitsForHeadroom = new ResourceLimits(
-          clusterResource);
-
-      // Initialize headroom info, also used for calculating application
-      // master resource limits.  Since this happens during queue initialization
-      // and all queues may not be realized yet, we'll use (optimistic)
-      // absoluteMaxCapacity (it will be replaced with the more accurate
-      // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
-      setQueueResourceLimitsInfo(clusterResource);
-
-      setOrderingPolicy(
-          conf.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePath()));
-
-      usersManager.setUserLimit(conf.getUserLimit(getQueuePath()));
-      usersManager.setUserLimitFactor(conf.getUserLimitFactor(getQueuePath()));
-
-      maxAMResourcePerQueuePercent =
-          conf.getMaximumApplicationMasterResourcePerQueuePercent(
-              getQueuePath());
-
-      maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
-      if (maxApplications < 0) {
-        int maxGlobalPerQueueApps =
-            csContext.getConfiguration().getGlobalMaximumApplicationsPerQueue();
-        if (maxGlobalPerQueueApps > 0) {
-          maxApplications = maxGlobalPerQueueApps;
-        }
-      }
-
-      priorityAcls = conf.getPriorityAcls(getQueuePath(),
-          csContext.getMaxClusterLevelAppPriority());
-
-      if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
-          this.defaultLabelExpression, null)) {
-        throw new IOException(
-            "Invalid default label expression of " + " queue=" + getQueuePath()
-                + " doesn't have permission to access all labels "
-                + "in default label expression. labelExpression of resource request="
-                + (this.defaultLabelExpression == null ?
-                "" :
-                this.defaultLabelExpression) + ". Queue labels=" + (
-                getAccessibleNodeLabels() == null ?
-                    "" :
-                    StringUtils
-                        .join(getAccessibleNodeLabels().iterator(), ',')));
-      }
-
-      nodeLocalityDelay = schedConf.getNodeLocalityDelay();
-      rackLocalityAdditionalDelay = schedConf
-          .getRackLocalityAdditionalDelay();
-      rackLocalityFullReset = schedConf
-          .getRackLocalityFullReset();
-
-      // re-init this since max allocation could have changed
-      this.minimumAllocationFactor = Resources.ratio(resourceCalculator,
-          Resources.subtract(maximumAllocation, minimumAllocation),
-          maximumAllocation);
-
-      StringBuilder aclsString = new StringBuilder();
-      for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
-        aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
-      }
-
-      StringBuilder labelStrBuilder = new StringBuilder();
-      if (accessibleLabels != null) {
-        for (String s : accessibleLabels) {
-          labelStrBuilder.append(s)
-              .append(",");
-        }
-      }
-
-      defaultAppPriorityPerQueue = Priority.newInstance(
-          conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
-
-      // Validate leaf queue's user's weights.
-      float queueUserLimit = Math.min(100.0f, conf.getUserLimit(getQueuePath()));
-      getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath());
-      usersManager.updateUserWeights();
-
-      LOG.info(
-          "Initializing " + getQueuePath() + "\n" +
-              getExtendedCapacityOrWeightString() + "\n"
-              + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
-              + " [= parentAbsoluteCapacity * capacity ]" + "\n"
-              + "maxCapacity = " + queueCapacities.getMaximumCapacity()
-              + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = "
-              + queueCapacities.getAbsoluteMaximumCapacity()
-              + " [= 1.0 maximumCapacity undefined, "
-              + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]"
-              + "\n" + "effectiveMinResource=" +
-              getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) + "\n"
-              + " , effectiveMaxResource=" +
-              getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL)
-              + "\n" + "userLimit = " + usersManager.getUserLimit()
-              + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = "
-              + usersManager.getUserLimitFactor()
-              + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = "
-              + maxApplications
-              + " [= configuredMaximumSystemApplicationsPerQueue or"
-              + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]"
-              + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser
-              + " [= (int)(maxApplications * (userLimit / 100.0f) * "
-              + "userLimitFactor) ]" + "\n"
-              + "maxParallelApps = " + getMaxParallelApps() + "\n"
-              + "usedCapacity = " +
-              + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / "
-              + "(clusterResourceMemory * absoluteCapacity)]" + "\n"
-              + "absoluteUsedCapacity = " + absoluteUsedCapacity
-              + " [= usedResourcesMemory / clusterResourceMemory]" + "\n"
-              + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent
-              + " [= configuredMaximumAMResourcePercent ]" + "\n"
-              + "minimumAllocationFactor = " + minimumAllocationFactor
-              + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / "
-              + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = "
-              + maximumAllocation + " [= configuredMaxAllocation ]" + "\n"
-              + "numContainers = " + numContainers
-              + " [= currentNumContainers ]" + "\n" + "state = " + getState()
-              + " [= configuredState ]" + "\n" + "acls = " + aclsString
-              + " [= configuredAcls ]" + "\n"
-              + "nodeLocalityDelay = " + nodeLocalityDelay + "\n"
-              + "rackLocalityAdditionalDelay = "
-              + rackLocalityAdditionalDelay + "\n"
-              + "labels=" + labelStrBuilder.toString() + "\n"
-              + "reservationsContinueLooking = "
-              + reservationsContinueLooking + "\n" + "preemptionDisabled = "
-              + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = "
-              + defaultAppPriorityPerQueue + "\npriority = " + priority
-              + "\nmaxLifetime = " + getMaximumApplicationLifetime()
-              + " seconds" + "\ndefaultLifetime = "
-              + getDefaultApplicationLifetime() + " seconds");
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  /**
-   * Used only by tests.
-   */
-  @Private
-  public float getMinimumAllocationFactor() {
-    return minimumAllocationFactor;
-  }
-  
-  /**
-   * Used only by tests.
-   */
-  @Private
-  public float getMaxAMResourcePerQueuePercent() {
-    return maxAMResourcePerQueuePercent;
-  }
-
-  public int getMaxApplications() {
-    return maxApplications;
-  }
-
-  public int getMaxApplicationsPerUser() {
-    return maxApplicationsPerUser;
-  }
-
-  /**
-   *
-   * @return UsersManager instance.
-   */
-  public UsersManager getUsersManager() {
-    return usersManager;
-  }
-
-  @Override
-  public AbstractUsersManager getAbstractUsersManager() {
-    return usersManager;
-  }
-
-  @Override
-  public List<CSQueue> getChildQueues() {
-    return null;
-  }
-  
-  /**
-   * Set user limit.
-   * @param userLimit new user limit
-   */
-  @VisibleForTesting
-  void setUserLimit(float userLimit) {
-    usersManager.setUserLimit(userLimit);
-    usersManager.userLimitNeedsRecompute();
-  }
-
-  /**
-   * Set user limit factor.
-   * @param userLimitFactor new user limit factor
-   */
-  @VisibleForTesting
-  void setUserLimitFactor(float userLimitFactor) {
-    usersManager.setUserLimitFactor(userLimitFactor);
-    usersManager.userLimitNeedsRecompute();
-  }
-
-  @Override
-  public int getNumApplications() {
-    readLock.lock();
-    try {
-      return getNumPendingApplications() + getNumActiveApplications() +
-          getNumNonRunnableApps();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  public int getNumPendingApplications() {
-    readLock.lock();
-    try {
-      return pendingOrderingPolicy.getNumSchedulableEntities();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  public int getNumActiveApplications() {
-    readLock.lock();
-    try {
-      return orderingPolicy.getNumSchedulableEntities();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Private
-  public int getNumPendingApplications(String user) {
-    readLock.lock();
-    try {
-      User u = getUser(user);
-      if (null == u) {
-        return 0;
-      }
-      return u.getPendingApplications();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Private
-  public int getNumActiveApplications(String user) {
-    readLock.lock();
-    try {
-      User u = getUser(user);
-      if (null == u) {
-        return 0;
-      }
-      return u.getActiveApplications();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Private
-  public float getUserLimit() {
-    return usersManager.getUserLimit();
-  }
-
-  @Private
-  public float getUserLimitFactor() {
-    return usersManager.getUserLimitFactor();
-  }
-
-  @Override
-  public QueueInfo getQueueInfo(
-      boolean includeChildQueues, boolean recursive) {
-    QueueInfo queueInfo = getQueueInfo();
-    return queueInfo;
-  }
-
-  @Override
-  public List<QueueUserACLInfo>
-  getQueueUserAclInfo(UserGroupInformation user) {
-    readLock.lock();
-    try {
-      QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
-          QueueUserACLInfo.class);
-      List<QueueACL> operations = new ArrayList<>();
-      for (QueueACL operation : QueueACL.values()) {
-        if (hasAccess(operation, user)) {
-          operations.add(operation);
-        }
-      }
-
-      userAclInfo.setQueueName(getQueuePath());
-      userAclInfo.setUserAcls(operations);
-      return Collections.singletonList(userAclInfo);
-    } finally {
-      readLock.unlock();
-    }
-
-  }
-
-  public String toString() {
-    readLock.lock();
-    try {
-      return getQueuePath() + ": " + getCapacityOrWeightString()
-          + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
-          + ", " + "usedResources=" + queueUsage.getUsed() + ", "
-          + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity="
-          + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications()
-          + ", " + "numContainers=" + getNumContainers() + ", "
-          + "effectiveMinResource=" +
-          getEffectiveCapacity(CommonNodeLabelsManager.NO_LABEL) +
-          " , effectiveMaxResource=" +
-          getEffectiveMaxCapacity(CommonNodeLabelsManager.NO_LABEL);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  protected String getExtendedCapacityOrWeightString() {
-    if (queueCapacities.getWeight() != -1) {
-      return "weight = " + queueCapacities.getWeight()
-          + " [= (float) configuredCapacity (with w suffix)] " + "\n"
-          + "normalizedWeight = " + queueCapacities.getNormalizedWeight()
-          + " [= (float) configuredCapacity / sum(configuredCapacity of " +
-          "all queues under the parent)]";
-    } else {
-      return "capacity = " + queueCapacities.getCapacity()
-          + " [= (float) configuredCapacity / 100 ]";
-    }
-  }
-
-  @VisibleForTesting
-  public User getUser(String userName) {
-    return usersManager.getUser(userName);
-  }
-
-  @Private
-  public List<AppPriorityACLGroup> getPriorityACLs() {
-    readLock.lock();
-    try {
-      return new ArrayList<>(priorityAcls);
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  protected void reinitialize(
-      CSQueue newlyParsedQueue, Resource clusterResource,
-      CapacitySchedulerConfiguration configuration) throws
-      IOException {
-
-    writeLock.lock();
-    try {
-      // We skip reinitialize for dynamic queues, when this is called, and
-      // new queue is different from this queue, we will make this queue to be
-      // static queue.
-      if (newlyParsedQueue != this) {
-        this.setDynamicQueue(false);
-      }
-
-      // Sanity check
-      if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue
-          .getQueuePath().equals(getQueuePath())) {
-        throw new IOException(
-            "Trying to reinitialize " + getQueuePath() + " from "
-                + newlyParsedQueue.getQueuePath());
-      }
-
-      LeafQueue newlyParsedLeafQueue = (LeafQueue) newlyParsedQueue;
-
-      // don't allow the maximum allocation to be decreased in size
-      // since we have already told running AM's the size
-      Resource oldMax = getMaximumAllocation();
-      Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
-
-      if (!Resources.fitsIn(oldMax, newMax)) {
-        throw new IOException("Trying to reinitialize " + getQueuePath()
-            + " the maximum allocation size can not be decreased!"
-            + " Current setting: " + oldMax + ", trying to set it to: "
-            + newMax);
-      }
-
-      setupQueueConfigs(clusterResource, configuration);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public void reinitialize(
-      CSQueue newlyParsedQueue, Resource clusterResource)
-  throws IOException {
-    reinitialize(newlyParsedQueue, clusterResource,
-        csContext.getConfiguration());
-  }
-
-  @Override
-  public void submitApplicationAttempt(FiCaSchedulerApp application,
-      String userName) {
-    submitApplicationAttempt(application, userName, false);
-  }
-
-  @Override
-  public void submitApplicationAttempt(FiCaSchedulerApp application,
-      String userName, boolean isMoveApp) {
-    // Careful! Locking order is important!
-    writeLock.lock();
-    try {
-      // TODO, should use getUser, use this method just to avoid UT failure
-      // which is caused by wrong invoking order, will fix UT separately
-      User user = usersManager.getUserAndAddIfAbsent(userName);
-
-      // Add the attempt to our data-structures
-      addApplicationAttempt(application, user);
-    } finally {
-      writeLock.unlock();
-    }
-
-    // We don't want to update metrics for move app
-    if (!isMoveApp) {
-      boolean unmanagedAM = application.getAppSchedulingInfo() != null &&
-          application.getAppSchedulingInfo().isUnmanagedAM();
-      metrics.submitAppAttempt(userName, unmanagedAM);
-    }
-
-    getParent().submitApplicationAttempt(application, userName);
-  }
-
-  @Override
-  public void submitApplication(ApplicationId applicationId, String userName,
-      String queue)  throws AccessControlException {
-    // Careful! Locking order is important!
-    validateSubmitApplication(applicationId, userName, queue);
-
-    // Signal for expired auto deletion.
-    updateLastSubmittedTimeStamp();
-
-    // Inform the parent queue
-    try {
-      getParent().submitApplication(applicationId, userName, queue);
-    } catch (AccessControlException ace) {
-      LOG.info("Failed to submit application to parent-queue: " +
-          getParent().getQueuePath(), ace);
-      throw ace;
-    }
-
-  }
-
-  public void validateSubmitApplication(ApplicationId applicationId,
-      String userName, String queue) throws AccessControlException {
-    writeLock.lock();
-    try {
-      // Check if the queue is accepting jobs
-      if (getState() != QueueState.RUNNING) {
-        String msg = "Queue " + getQueuePath()
-            + " is STOPPED. Cannot accept submission of application: "
-            + applicationId;
-        LOG.info(msg);
-        throw new AccessControlException(msg);
-      }
-
-      // Check submission limits for queues
-      //TODO recalculate max applications because they can depend on capacity
-      if (getNumApplications() >= getMaxApplications() && !(this instanceof AutoCreatedLeafQueue)) {
-        String msg =
-            "Queue " + getQueuePath() + " already has " + getNumApplications()
-                + " applications,"
-                + " cannot accept submission of application: " + applicationId;
-        LOG.info(msg);
-        throw new AccessControlException(msg);
-      }
-
-      // Check submission limits for the user on this queue
-      User user = usersManager.getUserAndAddIfAbsent(userName);
-      //TODO recalculate max applications because they can depend on capacity
-      if (user.getTotalApplications() >= getMaxApplicationsPerUser() &&  !(this instanceof AutoCreatedLeafQueue)) {
-        String msg = "Queue " + getQueuePath() + " already has " + user
-            .getTotalApplications() + " applications from user " + userName
-            + " cannot accept submission of application: " + applicationId;
-        LOG.info(msg);
-        throw new AccessControlException(msg);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-
-    try {
-      getParent().validateSubmitApplication(applicationId, userName, queue);
-    } catch (AccessControlException ace) {
-      LOG.info("Failed to submit application to parent-queue: " + 
-          getParent().getQueuePath(), ace);
-      throw ace;
-    }
-  }
-  
-  public Resource getAMResourceLimit() {
-    return queueUsage.getAMLimit();
-  }
-
-  public Resource getAMResourceLimitPerPartition(String nodePartition) {
-    return queueUsage.getAMLimit(nodePartition);
-  }
-
-  @VisibleForTesting
-  public Resource calculateAndGetAMResourceLimit() {
-    return calculateAndGetAMResourceLimitPerPartition(
-        RMNodeLabelsManager.NO_LABEL);
-  }
-
-  @VisibleForTesting
-  public Resource getUserAMResourceLimit() {
-    return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL,
-         null);
-  }
-
-  public Resource getUserAMResourceLimitPerPartition(
-      String nodePartition, String userName) {
-    float userWeight = 1.0f;
-    if (userName != null && getUser(userName) != null) {
-      userWeight = getUser(userName).getWeight();
-    }
-
-    readLock.lock();
-    try {
-      /*
-       * The user am resource limit is based on the same approach as the user
-       * limit (as it should represent a subset of that). This means that it uses
-       * the absolute queue capacity (per partition) instead of the max and is
-       * modified by the userlimit and the userlimit factor as is the userlimit
-       */
-      float effectiveUserLimit = Math.max(usersManager.getUserLimit() / 100.0f,
-          1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
-      float preWeightedUserLimit = effectiveUserLimit;
-      effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
-
-      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
-
-      Resource userAMLimit = Resources.multiplyAndNormalizeUp(
-          resourceCalculator, queuePartitionResource,
-          queueCapacities.getMaxAMResourcePercentage(nodePartition)
-              * effectiveUserLimit * usersManager.getUserLimitFactor(),
-          minimumAllocation);
-
-      if (getUserLimitFactor() == -1) {
-        userAMLimit = Resources.multiplyAndNormalizeUp(
-            resourceCalculator, queuePartitionResource,
-            queueCapacities.getMaxAMResourcePercentage(nodePartition),
-            minimumAllocation);
-      }
-
-      userAMLimit =
-          Resources.min(resourceCalculator, lastClusterResource,
-              userAMLimit,
-              Resources.clone(getAMResourceLimitPerPartition(nodePartition)));
-
-      Resource preWeighteduserAMLimit =
-          Resources.multiplyAndNormalizeUp(
-          resourceCalculator, queuePartitionResource,
-          queueCapacities.getMaxAMResourcePercentage(nodePartition)
-              * preWeightedUserLimit * usersManager.getUserLimitFactor(),
-          minimumAllocation);
-
-      if (getUserLimitFactor() == -1) {
-        preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp(
-            resourceCalculator, queuePartitionResource,
-            queueCapacities.getMaxAMResourcePercentage(nodePartition),
-            minimumAllocation);
-      }
-
-      preWeighteduserAMLimit =
-          Resources.min(resourceCalculator, lastClusterResource,
-              preWeighteduserAMLimit,
-              Resources.clone(getAMResourceLimitPerPartition(nodePartition)));
-      queueUsage.setUserAMLimit(nodePartition, preWeighteduserAMLimit);
-
-      LOG.debug("Effective user AM limit for \"{}\":{}. Effective weighted"
-          + " user AM limit: {}. User weight: {}", userName,
-          preWeighteduserAMLimit, userAMLimit, userWeight);
-      return userAMLimit;
-    } finally {
-      readLock.unlock();
-    }
-
-  }
-
-  public Resource calculateAndGetAMResourceLimitPerPartition(
-      String nodePartition) {
-    writeLock.lock();
-    try {
-      /*
-       * For non-labeled partition, get the max value from resources currently
-       * available to the queue and the absolute resources guaranteed for the
-       * partition in the queue. For labeled partition, consider only the absolute
-       * resources guaranteed. Multiply this value (based on labeled/
-       * non-labeled), * with per-partition am-resource-percent to get the max am
-       * resource limit for this queue and partition.
-       */
-      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
-
-      Resource queueCurrentLimit = Resources.none();
-      // For non-labeled partition, we need to consider the current queue
-      // usage limit.
-      if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
-        synchronized (queueResourceLimitsInfo){
-          queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
-        }
-      }
-
-      float amResourcePercent = queueCapacities.getMaxAMResourcePercentage(
-          nodePartition);
-
-      // Current usable resource for this queue and partition is the max of
-      // queueCurrentLimit and queuePartitionResource.
-      // If any of the resources available to this queue are less than queue's
-      // guarantee, use the guarantee as the queuePartitionUsableResource
-      // because nothing less than the queue's guarantee should be used when
-      // calculating the AM limit.
-      Resource queuePartitionUsableResource = (Resources.fitsIn(
-          resourceCalculator, queuePartitionResource, queueCurrentLimit)) ?
-              queueCurrentLimit : queuePartitionResource;
-
-      Resource amResouceLimit = Resources.multiplyAndNormalizeUp(
-          resourceCalculator, queuePartitionUsableResource, amResourcePercent,
-          minimumAllocation);
-
-      metrics.setAMResouceLimit(nodePartition, amResouceLimit);
-      queueUsage.setAMLimit(nodePartition, amResouceLimit);
-      LOG.debug("Queue: {}, node label : {}, queue partition resource : {},"
-          + " queue current limit : {}, queue partition usable resource : {},"
-          + " amResourceLimit : {}", getQueuePath(), nodePartition,
-          queuePartitionResource, queueCurrentLimit,
-          queuePartitionUsableResource, amResouceLimit);
-      return amResouceLimit;
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  protected void activateApplications() {
-    writeLock.lock();
-    try {
-      // limit of allowed resource usage for application masters
-      Map<String, Resource> userAmPartitionLimit =
-          new HashMap<String, Resource>();
-
-      // AM Resource Limit for accessible labels can be pre-calculated.
-      // This will help in updating AMResourceLimit for all labels when queue
-      // is initialized for the first time (when no applications are present).
-      for (String nodePartition : getNodeLabelsForQueue()) {
-        calculateAndGetAMResourceLimitPerPartition(nodePartition);
-      }
-
-      for (Iterator<FiCaSchedulerApp> fsApp =
-           getPendingAppsOrderingPolicy()
-               .getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR);
-           fsApp.hasNext(); ) {
-        FiCaSchedulerApp application = fsApp.next();
-        ApplicationId applicationId = application.getApplicationId();
-
-        // Get the am-node-partition associated with each application
-        // and calculate max-am resource limit for this partition.
-        String partitionName = application.getAppAMNodePartitionName();
-
-        Resource amLimit = getAMResourceLimitPerPartition(partitionName);
-        // Verify whether we already calculated am-limit for this label.
-        if (amLimit == null) {
-          amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
-        }
-        // Check am resource limit.
-        Resource amIfStarted = Resources.add(
-            application.getAMResource(partitionName),
-            queueUsage.getAMUsed(partitionName));
-
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("application " + application.getId() + " AMResource "
-              + application.getAMResource(partitionName)
-              + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent
-              + " amLimit " + amLimit + " lastClusterResource "
-              + lastClusterResource + " amIfStarted " + amIfStarted
-              + " AM node-partition name " + partitionName);
-        }
-
-        if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) {
-          if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
-              resourceCalculator, lastClusterResource,
-              queueUsage.getAMUsed(partitionName), Resources.none()))) {
-            LOG.warn("maximum-am-resource-percent is insufficient to start a"
-                + " single application in queue, it is likely set too low."
-                + " skipping enforcement to allow at least one application"
-                + " to start");
-          } else{
-            application.updateAMContainerDiagnostics(AMState.INACTIVATED,
-                CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
-            LOG.debug("Not activating application {} as  amIfStarted: {}"
-                + " exceeds amLimit: {}", applicationId, amIfStarted, amLimit);
-            continue;
-          }
-        }
-
-        // Check user am resource limit
-        User user = usersManager.getUserAndAddIfAbsent(application.getUser());
-        Resource userAMLimit = userAmPartitionLimit.get(partitionName);
-
-        // Verify whether we already calculated user-am-limit for this label.
-        if (userAMLimit == null) {
-          userAMLimit = getUserAMResourceLimitPerPartition(partitionName,
-              application.getUser());
-          userAmPartitionLimit.put(partitionName, userAMLimit);
-        }
-
-        Resource userAmIfStarted = Resources.add(
-            application.getAMResource(partitionName),
-            user.getConsumedAMResources(partitionName));
-
-        if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) {
-          if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
-              resourceCalculator, lastClusterResource,
-              queueUsage.getAMUsed(partitionName), Resources.none()))) {
-            LOG.warn("maximum-am-resource-percent is insufficient to start a"
-                + " single application in queue for user, it is likely set too"
-                + " low. skipping enforcement to allow at least one application"
-                + " to start");
-          } else{
-            application.updateAMContainerDiagnostics(AMState.INACTIVATED,
-                CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
-            LOG.debug("Not activating application {} for user: {} as"
-                + " userAmIfStarted: {} exceeds userAmLimit: {}",
-                applicationId, user, userAmIfStarted, userAMLimit);
-            continue;
-          }
-        }
-        user.activateApplication();
-        orderingPolicy.addSchedulableEntity(application);
-        application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
-
-        queueUsage.incAMUsed(partitionName,
-            application.getAMResource(partitionName));
-        user.getResourceUsage().incAMUsed(partitionName,
-            application.getAMResource(partitionName));
-        user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
-        metrics.incAMUsed(partitionName, application.getUser(),
-            application.getAMResource(partitionName));
-        metrics.setAMResouceLimitForUser(partitionName,
-            application.getUser(), userAMLimit);
-        fsApp.remove();
-        LOG.info("Application " + applicationId + " from user: " + application
-            .getUser() + " activated in queue: " + getQueuePath());
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void addApplicationAttempt(FiCaSchedulerApp application,
-      User user) {
-    writeLock.lock();
-    try {
-      applicationAttemptMap.put(application.getApplicationAttemptId(),
-          application);
-
-      if (application.isRunnable()) {
-        runnableApps.add(application);
-        LOG.debug("Adding runnable application: {}",
-            application.getApplicationAttemptId());
-      } else {
-        nonRunnableApps.add(application);
-        LOG.info("Application attempt {} is not runnable,"
-            + " parallel limit reached", application.getApplicationAttemptId());
-        return;
-      }
-
-      // Accept
-      user.submitApplication();
-      getPendingAppsOrderingPolicy().addSchedulableEntity(application);
-
-      // Activate applications
-      if (Resources.greaterThan(resourceCalculator, lastClusterResource,
-          lastClusterResource, Resources.none())) {
-        activateApplications();
-      } else {
-        application.updateAMContainerDiagnostics(AMState.INACTIVATED,
-            CSAMContainerLaunchDiagnosticsConstants.CLUSTER_RESOURCE_EMPTY);
-        LOG.info("Skipping activateApplications for "
-            + application.getApplicationAttemptId()
-            + " since cluster resource is " + Resources.none());
-      }
-
-      LOG.info(
-          "Application added -" + " appId: " + application.getApplicationId()
-              + " user: " + application.getUser() + "," + " leaf-queue: "
-              + getQueuePath() + " #user-pending-applications: " + user
-              .getPendingApplications() + " #user-active-applications: " + user
-              .getActiveApplications() + " #queue-pending-applications: "
-              + getNumPendingApplications() + " #queue-active-applications: "
-              + getNumActiveApplications()
-              + " #queue-nonrunnable-applications: "
-              + getNumNonRunnableApps());
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public void finishApplication(ApplicationId application, String user) {
-    // Inform the activeUsersManager
-    usersManager.deactivateApplication(user, application);
-
-    appFinished();
-
-    // Inform the parent queue
-    getParent().finishApplication(application, user);
-  }
-
-  @Override
-  public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) {
-    // Careful! Locking order is important!
-    removeApplicationAttempt(application, application.getUser());
-    getParent().finishApplicationAttempt(application, queue);
-  }
-
-  private void removeApplicationAttempt(
-      FiCaSchedulerApp application, String userName) {
-
-    writeLock.lock();
-    try {
-      // TODO, should use getUser, use this method just to avoid UT failure
-      // which is caused by wrong invoking order, will fix UT separately
-      User user = usersManager.getUserAndAddIfAbsent(userName);
-
-      boolean runnable = runnableApps.remove(application);
-      if (!runnable) {
-        // removeNonRunnableApp acquires the write lock again, which is fine
-        if (!removeNonRunnableApp(application)) {
-          LOG.error("Given app to remove " + application +
-              " does not exist in queue " + getQueuePath());
-        }
-      }
-
-      String partitionName = application.getAppAMNodePartitionName();
-      boolean wasActive = orderingPolicy.removeSchedulableEntity(application);
-      if (!wasActive) {
-        pendingOrderingPolicy.removeSchedulableEntity(application);
-      } else{
-        queueUsage.decAMUsed(partitionName,
-            application.getAMResource(partitionName));
-        user.getResourceUsage().decAMUsed(partitionName,
-            application.getAMResource(partitionName));
-        metrics.decAMUsed(partitionName, application.getUser(),
-            application.getAMResource(partitionName));
-      }
-      applicationAttemptMap.remove(application.getApplicationAttemptId());
-
-      user.finishApplication(wasActive);
-      if (user.getTotalApplications() == 0) {
-        usersManager.removeUser(application.getUser());
-      }
-
-      // Check if we can activate more applications
-      activateApplications();
-
-      LOG.info(
-          "Application removed -" + " appId: " + application.getApplicationId()
-              + " user: " + application.getUser() + " queue: " + getQueuePath()
-              + " #user-pending-applications: " + user.getPendingApplications()
-              + " #user-active-applications: " + user.getActiveApplications()
-              + " #queue-pending-applications: " + getNumPendingApplications()
-              + " #queue-active-applications: " + getNumActiveApplications());
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private FiCaSchedulerApp getApplication(
-      ApplicationAttemptId applicationAttemptId) {
-    return applicationAttemptMap.get(applicationAttemptId);
-  }
-
-  private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
-    // Set preemption-allowed:
-    // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
-    if (!queueResourceQuotas.getEffectiveMinResource(nodePartition)
-        .equals(Resources.none())) {
-      limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
-          csContext.getClusterResource(), queueUsage.getUsed(nodePartition),
-          queueResourceQuotas.getEffectiveMinResource(nodePartition)));
-      return;
-    }
-
-    float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
-    float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
-    limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
-  }
-
-  private CSAssignment allocateFromReservedContainer(Resource clusterResource,
-      CandidateNodeSet<FiCaSchedulerNode> candidates,
-      ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
-
-    // Irrespective of Single / Multi Node Placement, the allocate from
-    // Reserved Container has to happen only for the single node which
-    // CapacityScheduler#allocateFromReservedContainer invokes with.
-    // Else In Multi Node Placement, there won't be any Allocation or
-    // Reserve of new containers when there is a RESERVED container on
-    // a node which is full.
-    FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
-    if (node != null) {
-      RMContainer reservedContainer = node.getReservedContainer();
-      if (reservedContainer != null) {
-        FiCaSchedulerApp application = getApplication(
-            reservedContainer.getApplicationAttemptId());
-
-        if (null != application) {
-          ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
-              node, SystemClock.getInstance().getTime(), application);
-          CSAssignment assignment = application.assignContainers(
-              clusterResource, candidates, currentResourceLimits,
-              schedulingMode, reservedContainer);
-          return assignment;
-        }
-      }
-    }
-
-    return null;
-  }
-
-  private ConcurrentMap<String, CachedUserLimit> getUserLimitCache(
-      String partition,
-      SchedulingMode schedulingMode) {
-    synchronized (userLimitsCache) {
-      long latestVersion = usersManager.getLatestVersionOfUsersState();
-
-      if (latestVersion != this.currentUserLimitCacheVersion) {
-        // User limits cache needs invalidating
-        this.currentUserLimitCacheVersion = latestVersion;
-        userLimitsCache.clear();
-
-        Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>
-            uLCByPartition = new HashMap<>();
-        userLimitsCache.put(partition, uLCByPartition);
-
-        ConcurrentMap<String, CachedUserLimit> uLCBySchedulingMode =
-            new ConcurrentHashMap<>();
-        uLCByPartition.put(schedulingMode, uLCBySchedulingMode);
-
-        return uLCBySchedulingMode;
-      }
-
-      // User limits cache does not need invalidating
-      Map<SchedulingMode, ConcurrentMap<String, CachedUserLimit>>
-          uLCByPartition = userLimitsCache.get(partition);
-      if (uLCByPartition == null) {
-        uLCByPartition = new HashMap<>();
-        userLimitsCache.put(partition, uLCByPartition);
-      }
-
-      ConcurrentMap<String, CachedUserLimit> uLCBySchedulingMode =
-          uLCByPartition.get(schedulingMode);
-      if (uLCBySchedulingMode == null) {
-        uLCBySchedulingMode = new ConcurrentHashMap<>();
-        uLCByPartition.put(schedulingMode, uLCBySchedulingMode);
-      }
-
-      return uLCBySchedulingMode;
-    }
-  }
-
-  @Override
-  public CSAssignment assignContainers(Resource clusterResource,
-      CandidateNodeSet<FiCaSchedulerNode> candidates,
-      ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
-    updateCurrentResourceLimits(currentResourceLimits, clusterResource);
-    FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
-
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("assignContainers: partition=" + candidates.getPartition()
-          + " #applications=" + orderingPolicy.getNumSchedulableEntities());
-    }
-
-    setPreemptionAllowed(currentResourceLimits, candidates.getPartition());
-
-    // Check for reserved resources, try to allocate reserved container first.
-    CSAssignment assignment = allocateFromReservedContainer(clusterResource,
-        candidates, currentResourceLimits, schedulingMode);
-    if (null != assignment) {
-      return assignment;
-    }
-
-    // if our queue cannot access this node, just return
-    if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
-        && !accessibleToPartition(candidates.getPartition())) {
-      ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-          getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED,
-          ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION);
-      return CSAssignment.NULL_ASSIGNMENT;
-    }
-
-    // Check if this queue need more resource, simply skip allocation if this
-    // queue doesn't need more resources.
-    if (!hasPendingResourceRequest(candidates.getPartition(), clusterResource,
-        schedulingMode)) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Skip this queue=" + getQueuePath()
-            + ", because it doesn't need more resource, schedulingMode="
-            + schedulingMode.name() + " node-partition=" + candidates
-            .getPartition());
-      }
-      ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-          getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
-          ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
-      return CSAssignment.NULL_ASSIGNMENT;
-    }
-
-    ConcurrentMap<String, CachedUserLimit> userLimits =
-        this.getUserLimitCache(candidates.getPartition(), schedulingMode);
-    boolean needAssignToQueueCheck = true;
-    IteratorSelector sel = new IteratorSelector();
-    sel.setPartition(candidates.getPartition());
-    for (Iterator<FiCaSchedulerApp> assignmentIterator =
-         orderingPolicy.getAssignmentIterator(sel);
-         assignmentIterator.hasNext(); ) {
-      FiCaSchedulerApp application = assignmentIterator.next();
-
-      ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
-          node, SystemClock.getInstance().getTime(), application);
-
-      // Check queue max-capacity limit
-      Resource appReserved = application.getCurrentReservation();
-      if (needAssignToQueueCheck) {
-        if (!super.canAssignToThisQueue(clusterResource,
-            candidates.getPartition(), currentResourceLimits, appReserved,
-            schedulingMode)) {
-          ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
-              activitiesManager, node, application, application.getPriority(),
-              ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT);
-          ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-              getParent().getQueuePath(), getQueuePath(),
-              ActivityState.REJECTED,
-              ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT);
-          return CSAssignment.NULL_ASSIGNMENT;
-        }
-        // If there was no reservation and canAssignToThisQueue returned
-        // true, there is no reason to check further.
-        if (!this.reservationsContinueLooking
-            || appReserved.equals(Resources.none())) {
-          needAssignToQueueCheck = false;
-        }
-      }
-
-      CachedUserLimit cul = userLimits.get(application.getUser());
-      Resource cachedUserLimit = null;
-      if (cul != null) {
-        cachedUserLimit = cul.userLimit;
-      }
-      Resource userLimit = computeUserLimitAndSetHeadroom(application,
-          clusterResource, candidates.getPartition(), schedulingMode,
-          cachedUserLimit);
-      if (cul == null) {
-        cul = new CachedUserLimit(userLimit);
-        CachedUserLimit retVal =
-            userLimits.putIfAbsent(application.getUser(), cul);
-        if (retVal != null) {
-          // another thread updated the user limit cache before us
-          cul = retVal;
-          userLimit = cul.userLimit;
-        }
-      }
-      // Check user limit
-      boolean userAssignable = true;
-      if (!cul.canAssign && Resources.fitsIn(appReserved, cul.reservation)) {
-        userAssignable = false;
-      } else {
-        userAssignable = canAssignToUser(clusterResource, application.getUser(),
-            userLimit, application, candidates.getPartition(),
-            currentResourceLimits);
-        if (!userAssignable && Resources.fitsIn(cul.reservation, appReserved)) {
-          cul.canAssign = false;
-          cul.reservation = appReserved;
-        }
-      }
-      if (!userAssignable) {
-        application.updateAMContainerDiagnostics(AMState.ACTIVATED,
-            "User capacity has reached its maximum limit.");
-        ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
-            activitiesManager, node, application, application.getPriority(),
-            ActivityDiagnosticConstant.QUEUE_HIT_USER_MAX_CAPACITY_LIMIT);
-        continue;
-      }
-
-      // Try to schedule
-      assignment = application.assignContainers(clusterResource,
-          candidates, currentResourceLimits, schedulingMode, null);
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("post-assignContainers for application " + application
-            .getApplicationId());
-        application.showRequests();
-      }
-
-      // Did we schedule or reserve a container?
-      Resource assigned = assignment.getResource();
-
-      if (Resources.greaterThan(resourceCalculator, clusterResource, assigned,
-          Resources.none())) {
-        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-            getParent().getQueuePath(), getQueuePath(),
-            ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
-        return assignment;
-      } else if (assignment.getSkippedType()
-          == CSAssignment.SkippedType.OTHER) {
-        ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
-            activitiesManager, application.getApplicationId(),
-            ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
-        application.updateNodeInfoForAMDiagnostics(node);
-      } else if (assignment.getSkippedType()
-          == CSAssignment.SkippedType.QUEUE_LIMIT) {
-        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-            getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED,
-            () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM
-                + " from " + application.getApplicationId());
-        return assignment;
-      } else{
-        // If we don't allocate anything, and it is not skipped by application,
-        // we will return to respect FIFO of applications
-        ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-            getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
-            ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO);
-        ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
-            activitiesManager, application.getApplicationId(),
-            ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
-        return CSAssignment.NULL_ASSIGNMENT;
-      }
-    }
-    ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
-        getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED,
-        ActivityDiagnosticConstant.EMPTY);
-
-    return CSAssignment.NULL_ASSIGNMENT;
-  }
-
-  @Override
-  public boolean accept(Resource cluster,
-      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
-    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation =
-        request.getFirstAllocatedOrReservedContainer();
-    SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer =
-        allocation.getAllocatedOrReservedContainer();
-
-    // Do not check limits when allocation from a reserved container
-    if (allocation.getAllocateFromReservedContainer() == null) {
-      readLock.lock();
-      try {
-        FiCaSchedulerApp app =
-            schedulerContainer.getSchedulerApplicationAttempt();
-        String username = app.getUser();
-        String p = schedulerContainer.getNodePartition();
-
-        // check user-limit
-        Resource userLimit = computeUserLimitAndSetHeadroom(app, cluster, p,
-            allocation.getSchedulingMode(), null);
-
-        // Deduct resources that we can release
-        User user = getUser(username);
-        if (user == null) {
-          LOG.debug("User {} has been removed!", username);
-          return false;
-        }
-        Resource usedResource = Resources.clone(user.getUsed(p));
-        Resources.subtractFrom(usedResource,
-            request.getTotalReleasedResource());
-
-        if (Resources.greaterThan(resourceCalculator, cluster, usedResource,
-            userLimit)) {
-          LOG.debug("Used resource={} exceeded user-limit={}",
-              usedResource, userLimit);
-          return false;
-        }
-      } finally {
-        readLock.unlock();
-      }
-    }
-
-    return super.accept(cluster, request);
-  }
-
-  private void internalReleaseContainer(Resource clusterResource,
-      SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer) {
-    RMContainer rmContainer = schedulerContainer.getRmContainer();
-
-    LeafQueue targetLeafQueue =
-        schedulerContainer.getSchedulerApplicationAttempt().getCSLeafQueue();
-
-    if (targetLeafQueue == this) {
-      // When trying to preempt containers from the same queue
-      if (rmContainer.getState() == RMContainerState.RESERVED) {
-        // For other reserved containers
-        // This is a reservation exchange, complete previous reserved container
-        completedContainer(clusterResource,
-            schedulerContainer.getSchedulerApplicationAttempt(),
-            schedulerContainer.getSchedulerNode(), rmContainer, SchedulerUtils
-                .createAbnormalContainerStatus(rmContainer.getContainerId(),
-                    SchedulerUtils.UNRESERVED_CONTAINER),
-            RMContainerEventType.RELEASED, null, false);
-      }
-    } else{
-      // When trying to preempt containers from different queue -- this
-      // is for lazy preemption feature (kill preemption candidate in scheduling
-      // cycle).
-      targetLeafQueue.completedContainer(clusterResource,
-          schedulerContainer.getSchedulerApplicationAttempt(),
-          schedulerContainer.getSchedulerNode(),
-          schedulerContainer.getRmContainer(), SchedulerUtils
-              .createPreemptedContainerStatus(rmContainer.getContainerId(),
-                  SchedulerUtils.PREEMPTED_CONTAINER),
-          RMContainerEventType.KILL, null, false);
-    }
-  }
-
-  private void releaseContainers(Resource clusterResource,
-      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
-    for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> c : request
-        .getContainersToRelease()) {
-      internalReleaseContainer(clusterResource, c);
-    }
-
-    // Handle container reservation looking, or lazy preemption case:
-    if (null != request.getContainersToAllocate() && !request
-        .getContainersToAllocate().isEmpty()) {
-      for (ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> context : request
-          .getContainersToAllocate()) {
-        if (null != context.getToRelease()) {
-          for (SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> c : context
-              .getToRelease()) {
-            internalReleaseContainer(clusterResource, c);
-          }
-        }
-      }
-    }
-  }
-
-  public void apply(Resource cluster,
-      ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
-    // Do we need to call parent queue's apply?
-    boolean applyToParentQueue = false;
-
-    releaseContainers(cluster, request);
-
-    writeLock.lock();
-    try {
-      if (request.anythingAllocatedOrReserved()) {
-        ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>
-            allocation = request.getFirstAllocatedOrReservedContainer();
-        SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
-            schedulerContainer = allocation.getAllocatedOrReservedContainer();
-
-        // Do not modify queue when allocation from reserved container
-        if (allocation.getAllocateFromReservedContainer() == null) {
-          // Only invoke apply() of ParentQueue when new allocation /
-          // reservation happen.
-          applyToParentQueue = true;
-          // Book-keeping
-          // Note: Update headroom to account for current allocation too...
-          allocateResource(cluster,
-              schedulerContainer.getSchedulerApplicationAttempt(),
-              allocation.getAllocatedOrReservedResource(),
-              schedulerContainer.getNodePartition(),
-              schedulerContainer.getRmContainer());
-          orderingPolicy.containerAllocated(
-              schedulerContainer.getSchedulerApplicationAttempt(),
-              schedulerContainer.getRmContainer());
-        }
-
-        // Update reserved resource
-        if (Resources.greaterThan(resourceCalculator, cluster,
-            request.getTotalReservedResource(), Resources.none())) {
-          incReservedResource(schedulerContainer.getNodePartition(),
-              request.getTotalReservedResource());
-        }
-      }
-    } finally {
-      writeLock.unlock();
-    }
-
-    if (parent != null && applyToParentQueue) {
-      parent.apply(cluster, request);
-    }
-  }
-
-
-  protected Resource getHeadroom(User user, Resource queueCurrentLimit,
-      Resource clusterResource, FiCaSchedulerApp application) {
-    return getHeadroom(user, queueCurrentLimit, clusterResource, application,
-        RMNodeLabelsManager.NO_LABEL);
-  }
-
-  protected Resource getHeadroom(User user, Resource queueCurrentLimit,
-      Resource clusterResource, FiCaSchedulerApp application,
-      String partition) {
-    return getHeadroom(user, queueCurrentLimit, clusterResource,
-        getResourceLimitForActiveUsers(application.getUser(), clusterResource,
-            partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
-        partition);
-  }
-
-  private Resource getHeadroom(User user,
-      Resource currentPartitionResourceLimit, Resource clusterResource,
-      Resource userLimitResource, String partition) {
-    /** 
-     * Headroom is:
-     *    min(
-     *        min(userLimit, queueMaxCap) - userConsumed,
-     *        queueMaxCap - queueUsedResources
-     *       )
-     * 
-     * ( which can be expressed as, 
-     *  min (userLimit - userConsumed, queuMaxCap - userConsumed, 
-     *    queueMaxCap - queueUsedResources)
-     *  )
-     *
-     * given that queueUsedResources >= userConsumed, this simplifies to
-     *
-     * >> min (userlimit - userConsumed,   queueMaxCap - queueUsedResources) << 
-     *
-     * sum of queue max capacities of multiple queue's will be greater than the
-     * actual capacity of a given partition, hence we need to ensure that the
-     * headroom is not greater than the available resource for a given partition
-     *
-     * headroom = min (unused resourcelimit of a label, calculated headroom )
-     */
-    currentPartitionResourceLimit =
-        partition.equals(RMNodeLabelsManager.NO_LABEL)
-            ? currentPartitionResourceLimit
-            : getQueueMaxResource(partition);
-
-    Resource headroom = Resources.componentwiseMin(
-        Resources.subtractNonNegative(userLimitResource,
-            user.getUsed(partition)),
-        Resources.subtractNonNegative(currentPartitionResourceLimit,
-            queueUsage.getUsed(partition)));
-    // Normalize it before return
-    headroom =
-        Resources.roundDown(resourceCalculator, headroom, minimumAllocation);
-
-    //headroom = min (unused resourcelimit of a label, calculated headroom )
-    Resource clusterPartitionResource =
-        labelManager.getResourceByLabel(partition, clusterResource);
-    Resource clusterFreePartitionResource =
-        Resources.subtract(clusterPartitionResource,
-            csContext.getClusterResourceUsage().getUsed(partition));
-    headroom = Resources.min(resourceCalculator, clusterPartitionResource,
-        clusterFreePartitionResource, headroom);
-    return headroom;
-  }
-  
-  private void setQueueResourceLimitsInfo(
-      Resource clusterResource) {
-    synchronized (queueResourceLimitsInfo) {
-      queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom
-          .getLimit());
-      queueResourceLimitsInfo.setClusterResource(clusterResource);
-    }
-  }
-
-  // It doesn't necessarily to hold application's lock here.
-  @Lock({LeafQueue.class})
-  Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
-      Resource clusterResource, String nodePartition,
-      SchedulingMode schedulingMode, Resource userLimit) {
-    String user = application.getUser();
-    User queueUser = getUser(user);
-    if (queueUser == null) {
-      LOG.debug("User {} has been removed!", user);
-      return Resources.none();
-    }
-
-    // Compute user limit respect requested labels,
-    // TODO, need consider headroom respect labels also
-    if (userLimit == null) {
-      userLimit = getResourceLimitForActiveUsers(application.getUser(),
-          clusterResource, nodePartition, schedulingMode);
-    }
-    setQueueResourceLimitsInfo(clusterResource);
-
-    Resource headroom =
-        metrics.getUserMetrics(user) == null ? Resources.none() :
-        getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(),
-            clusterResource, userLimit, nodePartition);
-    
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Headroom calculation for user " + user + ": " + " userLimit="
-          + userLimit + " queueMaxAvailRes="
-          + cachedResourceLimitsForHeadroom.getLimit() + " consumed="
-          + queueUser.getUsed() + " partition="
-          + nodePartition);
-    }
-    
-    CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider(
-      queueUser, this, application, queueResourceLimitsInfo);
-    
-    application.setHeadroomProvider(headroomProvider);
-
-    metrics.setAvailableResourcesToUser(nodePartition, user, headroom);
-    
-    return userLimit;
-  }
-  
-  @Lock(NoLock.class)
-  public int getNodeLocalityDelay() {
-    return nodeLocalityDelay;
-  }
-
-  @Lock(NoLock.class)
-  public int getRackLocalityAdditionalDelay() {
-    return rackLocalityAdditionalDelay;
-  }
-
-  @Lock(NoLock.class)
-  public boolean getRackLocalityFullReset() {
-    return rackLocalityFullReset;
-  }
-
-  /**
-   *
-   * @param userName
-   *          Name of user who has submitted one/more app to given queue.
-   * @param clusterResource
-   *          total cluster resource
-   * @param nodePartition
-   *          partition name
-   * @param schedulingMode
-   *          scheduling mode
-   *          RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY
-   * @return Computed User Limit
-   */
-  public Resource getResourceLimitForActiveUsers(String userName,
-      Resource clusterResource, String nodePartition,
-      SchedulingMode schedulingMode) {
-    return usersManager.getComputedResourceLimitForActiveUsers(userName,
-        clusterResource, nodePartition, schedulingMode);
-  }
-
-  /**
-   *
-   * @param userName
-   *          Name of user who has submitted one/more app to given queue.
-   * @param clusterResource
-   *          total cluster resource
-   * @param nodePartition
-   *          partition name
-   * @param schedulingMode
-   *          scheduling mode
-   *          RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY
-   * @return Computed User Limit
-   */
-  public Resource getResourceLimitForAllUsers(String userName,
-      Resource clusterResource, String nodePartition,
-      SchedulingMode schedulingMode) {
-    return usersManager.getComputedResourceLimitForAllUsers(userName,
-        clusterResource, nodePartition, schedulingMode);
-  }
-
-  @Private
-  protected boolean canAssignToUser(Resource clusterResource,
-      String userName, Resource limit, FiCaSchedulerApp application,
-      String nodePartition, ResourceLimits currentResourceLimits) {
-
-    readLock.lock();
-    try {
-      User user = getUser(userName);
-      if (user == null) {
-        LOG.debug("User {} has been removed!", userName);
-        return false;
-      }
-
-      currentResourceLimits.setAmountNeededUnreserve(Resources.none());
-
-      // Note: We aren't considering the current request since there is a fixed
-      // overhead of the AM, but it's a > check, not a >= check, so...
-      if (Resources.greaterThan(resourceCalculator, clusterResource,
-          user.getUsed(nodePartition), limit)) {
-        // if enabled, check to see if could we potentially use this node instead
-        // of a reserved node if the application has reserved containers
-        if (this.reservationsContinueLooking) {
-          if (Resources.lessThanOrEqual(resourceCalculator, clusterResource,
-              Resources.subtract(user.getUsed(),
-                  application.getCurrentReservation()), limit)) {
-
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("User " + userName + " in queue " + getQueuePath()
-                  + " will exceed limit based on reservations - "
-                  + " consumed: " + user.getUsed() + " reserved: " + application
-                  .getCurrentReservation() + " limit: " + limit);
-            }
-            Resource amountNeededToUnreserve = Resources.subtract(
-                user.getUsed(nodePartition), limit);
-            // we can only acquire a new container if we unreserve first to
-            // respect user-limit
-            currentResourceLimits.setAmountNeededUnreserve(
-                amountNeededToUnreserve);
-            return true;
-          }
-        }
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("User " + userName + " in queue " + getQueuePath()
-              + " will exceed limit - " + " consumed: " + user
-              .getUsed(nodePartition) + " limit: " + limit);
-        }
-        return false;
-      }
-      return true;
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  @Override
-  protected void setDynamicQueueProperties(
-      CapacitySchedulerConfiguration configuration) {
-    super.setDynamicQueueProperties(configuration);
-    // set to -1, to disable it
-    configuration.setUserLimitFactor(getQueuePath(), -1);
-    // Set Max AM percentage to a higher value
-    configuration.setMaximumApplicationMasterResourcePerQueuePercent(
-        getQueuePath(), 1f);
-  }
-
-  private void updateSchedulerHealthForCompletedContainer(
-      RMContainer rmContainer, ContainerStatus containerStatus) {
-    // Update SchedulerHealth for released / preempted container
-    SchedulerHealth schedulerHealth = csContext.getSchedulerHealth();
-    if (null == schedulerHealth) {
-      // Only do update if we have schedulerHealth
-      return;
-    }
-
-    if (containerStatus.getExitStatus() == ContainerExitStatus.PREEMPTED) {
-      schedulerHealth.updatePreemption(Time.now(), rmContainer.getAllocatedNode(),
-          rmContainer.getContainerId(), getQueuePath());
-      schedulerHealth.updateSchedulerPreemptionCounts(1);
-    } else {
-      schedulerHealth.updateRelease(csContext.getLastNodeUpdateTime(),
-          rmContainer.getAllocatedNode(), rmContainer.getContainerId(),
-          getQueuePath());
-    }
-  }
-
-  /**
-   * Recalculate QueueUsage Ratio.
-   *
-   * @param clusterResource
-   *          Total Cluster Resource
-   * @param nodePartition
-   *          Partition
-   */
-  public void recalculateQueueUsageRatio(Resource clusterResource,
-      String nodePartition) {
-    writeLock.lock();
-    try {
-      ResourceUsage queueResourceUsage = getQueueResourceUsage();
-
-      if (nodePartition == null) {
-        for (String partition : Sets.union(
-            getQueueCapacities().getNodePartitionsSet(),
-            queueResourceUsage.getNodePartitionsSet())) {
-          usersManager.updateUsageRatio(partition, clusterResource);
-        }
-      } else {
-        usersManager.updateUsageRatio(nodePartition, clusterResource);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public void completedContainer(Resource clusterResource, 
-      FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, 
-      ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
-      boolean sortQueues) {
-    // Update SchedulerHealth for released / preempted container
-    updateSchedulerHealthForCompletedContainer(rmContainer, containerStatus);
-
-    if (application != null) {
-      boolean removed = false;
-
-      // Careful! Locking order is important!
-      writeLock.lock();
-      try {
-        Container container = rmContainer.getContainer();
-
-        // Inform the application & the node
-        // Note: It's safe to assume that all state changes to RMContainer
-        // happen under scheduler's lock...
-        // So, this is, in effect, a transaction across application & node
-        if (rmContainer.getState() == RMContainerState.RESERVED) {
-          removed = application.unreserve(rmContainer.getReservedSchedulerKey(),
-              node, rmContainer);
-        } else{
-          removed = application.containerCompleted(rmContainer, containerStatus,
-              event, node.getPartition());
-
-          node.releaseContainer(rmContainer.getContainerId(), false);
-        }
-
-        // Book-keeping
-        if (removed) {
-
-          // Inform the ordering policy
-          orderingPolicy.containerReleased(application, rmContainer);
-
-          releaseResource(clusterResource, application, container.getResource(),
-              node.getPartition(), rmContainer);
-        }
-      } finally {
-        writeLock.unlock();
-      }
-
-
-      if (removed) {
-        // Inform the parent queue _outside_ of the leaf-queue lock
-        getParent().completedContainer(clusterResource, application, node,
-          rmContainer, null, event, this, sortQueues);
-      }
-    }
-
-    // Notify PreemptionManager
-    csContext.getPreemptionManager().removeKillableContainer(
-        new KillableContainer(
-            rmContainer,
-            node.getPartition(),
-            getQueuePath()));
-
-    // Update preemption metrics if exit status is PREEMPTED
-    if (containerStatus != null
-        && ContainerExitStatus.PREEMPTED == containerStatus.getExitStatus()) {
-      updateQueuePreemptionMetrics(rmContainer);
-    }
-  }
-
-  void allocateResource(Resource clusterResource,
-      SchedulerApplicationAttempt application, Resource resource,
-      String nodePartition, RMContainer rmContainer) {
-    writeLock.lock();
-    try {
-      super.allocateResource(clusterResource, resource, nodePartition);
-
-      // handle ignore exclusivity container
-      if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
-          RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
-          RMNodeLabelsManager.NO_LABEL)) {
-        TreeSet<RMContainer> rmContainers = null;
-        if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(
-            nodePartition))) {
-          rmContainers = new TreeSet<>();
-          ignorePartitionExclusivityRMContainers.put(nodePartition,
-              rmContainers);
-        }
-        rmContainers.add(rmContainer);
-      }
-
-      // Update user metrics
-      String userName = application.getUser();
-
-      // Increment user's resource usage.
-      User user = usersManager.updateUserResourceUsage(userName, resource,
-          nodePartition, true);
-
-      Resource partitionHeadroom = Resources.createResource(0, 0);
-      if (metrics.getUserMetrics(userName) != null) {
-        partitionHeadroom = getHeadroom(user,
-            cachedResourceLimitsForHeadroom.getLimit(), clusterResource,
-            getResourceLimitForActiveUsers(userName, clusterResource,
-                nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
-            nodePartition);
-      }
-      metrics.setAvailableResourcesToUser(nodePartition, userName,
-          partitionHeadroom);
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(getQueuePath() + " user=" + userName + " used="
-            + queueUsage.getUsed(nodePartition) + " numContainers="
-            + numContainers + " headroom = " + application.getHeadroom()
-            + " user-resources=" + user.getUsed());
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  void releaseResource(Resource clusterResource,
-      FiCaSchedulerApp application, Resource resource, String nodePartition,
-      RMContainer rmContainer) {
-    writeLock.lock();
-    try {
-      super.releaseResource(clusterResource, resource, nodePartition);
-
-      // handle ignore exclusivity container
-      if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
-          RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
-          RMNodeLabelsManager.NO_LABEL)) {
-        if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
-          Set<RMContainer> rmContainers =
-              ignorePartitionExclusivityRMContainers.get(nodePartition);
-          rmContainers.remove(rmContainer);
-          if (rmContainers.isEmpty()) {
-            ignorePartitionExclusivityRMContainers.remove(nodePartition);
-          }
-        }
-      }
-
-      // Update user metrics
-      String userName = application.getUser();
-      User user = usersManager.updateUserResourceUsage(userName, resource,
-          nodePartition, false);
-
-      Resource partitionHeadroom = Resources.createResource(0, 0);
-      if (metrics.getUserMetrics(userName) != null) {
-        partitionHeadroom = getHeadroom(user,
-            cachedResourceLimitsForHeadroom.getLimit(), clusterResource,
-            getResourceLimitForActiveUsers(userName, clusterResource,
-                nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
-            nodePartition);
-      }
-      metrics.setAvailableResourcesToUser(nodePartition, userName,
-          partitionHeadroom);
-
-      if (LOG.isDebugEnabled()) {
-        LOG.debug(
-            getQueuePath() + " used=" + queueUsage.getUsed() + " numContainers="
-            + numContainers + " user=" + userName + " user-resources="
-            + user.getUsed());
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-  
-  private void updateCurrentResourceLimits(
-      ResourceLimits currentResourceLimits, Resource clusterResource) {
-    // TODO: need consider non-empty node labels when resource limits supports
-    // node labels
-    // Even if ParentQueue will set limits respect child's max queue capacity,
-    // but when allocating reserved container, CapacityScheduler doesn't do
-    // this. So need cap limits by queue's max capacity here.
-    this.cachedResourceLimitsForHeadroom =
-        new ResourceLimits(currentResourceLimits.getLimit());
-    Resource queueMaxResource = getEffectiveMaxCapacityDown(
-        RMNodeLabelsManager.NO_LABEL, minimumAllocation);
-    this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
-        resourceCalculator, clusterResource, queueMaxResource,
-        currentResourceLimits.getLimit()));
-  }
-
-  @Override
-  public void updateClusterResource(Resource clusterResource,
-      ResourceLimits currentResourceLimits) {
-    writeLock.lock();
-    try {
-      lastClusterResource = clusterResource;
-
-      updateAbsoluteCapacities();
-
-      super.updateEffectiveResources(clusterResource);
-      super.updateMaxAppRelatedField(csContext.getConfiguration(),
-              this);
-
-      updateCurrentResourceLimits(currentResourceLimits, clusterResource);
-
-      // Update headroom info based on new cluster resource value
-      // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
-      // during allocation
-      setQueueResourceLimitsInfo(clusterResource);
-
-      // Update user consumedRatios
-      recalculateQueueUsageRatio(clusterResource, null);
-
-      // Update metrics
-      CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
-          this, labelManager, null);
-      // Update configured capacity/max-capacity for default partition only
-      CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator,
-          labelManager.getResourceByLabel(null, clusterResource),
-          RMNodeLabelsManager.NO_LABEL, this);
-
-      // queue metrics are updated, more resource may be available
-      // activate the pending applications if possible
-      activateApplications();
-
-      // In case of any resource change, invalidate recalculateULCount to clear
-      // the computed user-limit.
-      usersManager.userLimitNeedsRecompute();
-
-      // Update application properties
-      for (FiCaSchedulerApp application : orderingPolicy
-          .getSchedulableEntities()) {
-        computeUserLimitAndSetHeadroom(application, clusterResource,
-            RMNodeLabelsManager.NO_LABEL,
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, null);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public void incUsedResource(String nodeLabel, Resource resourceToInc,
-      SchedulerApplicationAttempt application) {
-    usersManager.updateUserResourceUsage(application.getUser(), resourceToInc,
-        nodeLabel, true);
-    super.incUsedResource(nodeLabel, resourceToInc, application);
-  }
-
-  @Override
-  public void decUsedResource(String nodeLabel, Resource resourceToDec,
-      SchedulerApplicationAttempt application) {
-    usersManager.updateUserResourceUsage(application.getUser(), resourceToDec,
-        nodeLabel, false);
-    super.decUsedResource(nodeLabel, resourceToDec, application);
-  }
-
-  public void incAMUsedResource(String nodeLabel, Resource resourceToInc,
-      SchedulerApplicationAttempt application) {
-    getUser(application.getUser()).getResourceUsage().incAMUsed(nodeLabel,
-        resourceToInc);
-    // ResourceUsage has its own lock, no addition lock needs here.
-    queueUsage.incAMUsed(nodeLabel, resourceToInc);
-  }
-
-  public void decAMUsedResource(String nodeLabel, Resource resourceToDec,
-      SchedulerApplicationAttempt application) {
-    getUser(application.getUser()).getResourceUsage().decAMUsed(nodeLabel,
-        resourceToDec);
-    // ResourceUsage has its own lock, no addition lock needs here.
-    queueUsage.decAMUsed(nodeLabel, resourceToDec);
-  }
-
-  @Override
-  public void recoverContainer(Resource clusterResource,
-      SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
-    if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
-      return;
-    }
-    if (rmContainer.getExecutionType() != ExecutionType.GUARANTEED) {
-      return;
-    }
-    // Careful! Locking order is important!
-    writeLock.lock();
-    try {
-      FiCaSchedulerNode node = csContext.getNode(
-          rmContainer.getContainer().getNodeId());
-      allocateResource(clusterResource, attempt,
-          rmContainer.getContainer().getResource(), node.getPartition(),
-          rmContainer);
-    } finally {
-      writeLock.unlock();
-    }
-
-    getParent().recoverContainer(clusterResource, attempt, rmContainer);
-  }
-
-  /**
-   * Obtain (read-only) collection of pending applications.
-   */
-  public Collection<FiCaSchedulerApp> getPendingApplications() {
-    return Collections.unmodifiableCollection(pendingOrderingPolicy
-        .getSchedulableEntities());
-  }
-
-  /**
-   * Obtain (read-only) collection of active applications.
-   */
-  public Collection<FiCaSchedulerApp> getApplications() {
-    return Collections.unmodifiableCollection(orderingPolicy
-        .getSchedulableEntities());
-  }
-
-  /**
-   * Obtain (read-only) collection of all applications.
-   */
-  public Collection<FiCaSchedulerApp> getAllApplications() {
-    Collection<FiCaSchedulerApp> apps = new HashSet<FiCaSchedulerApp>(
-        pendingOrderingPolicy.getSchedulableEntities());
-    apps.addAll(orderingPolicy.getSchedulableEntities());
-
-    return Collections.unmodifiableCollection(apps);
-  }
-
-  /**
-   * Get total pending resource considering user limit for the leaf queue. This
-   * will be used for calculating pending resources in the preemption monitor.
-   *
-   * Consider the headroom for each user in the queue.
-   * Total pending for the queue =
-   * sum(for each user(min((user's headroom), sum(user's pending requests))))
-   * NOTE:
-
-   * @param clusterResources clusterResource
-   * @param partition node partition
-   * @param deductReservedFromPending When a container is reserved in CS,
-   *                                  pending resource will not be deducted.
-   *                                  This could lead to double accounting when
-   *                                  doing preemption:
-   *                                  In normal cases, we should deduct reserved
-   *                                  resource from pending to avoid
-   *                                  excessive preemption.
-   * @return Total pending resource considering user limit
-   */
-  public Resource getTotalPendingResourcesConsideringUserLimit(
-      Resource clusterResources, String partition,
-      boolean deductReservedFromPending) {
-    readLock.lock();
-    try {
-      Map<String, Resource> userNameToHeadroom =
-          new HashMap<>();
-      Resource totalPendingConsideringUserLimit = Resource.newInstance(0, 0);
-      for (FiCaSchedulerApp app : getApplications()) {
-        String userName = app.getUser();
-        if (!userNameToHeadroom.containsKey(userName)) {
-          User user = getUser(userName);
-          Resource headroom = Resources.subtract(
-              getResourceLimitForActiveUsers(app.getUser(), clusterResources,
-                  partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
-              user.getUsed(partition));
-          // Make sure headroom is not negative.
-          headroom = Resources.componentwiseMax(headroom, Resources.none());
-          userNameToHeadroom.put(userName, headroom);
-        }
-
-        // Check if we need to deduct reserved from pending
-        Resource pending = app.getAppAttemptResourceUsage().getPending(
-            partition);
-        if (deductReservedFromPending) {
-          pending = Resources.subtract(pending,
-              app.getAppAttemptResourceUsage().getReserved(partition));
-        }
-        pending = Resources.componentwiseMax(pending, Resources.none());
-
-        Resource minpendingConsideringUserLimit = Resources.componentwiseMin(
-            userNameToHeadroom.get(userName), pending);
-        Resources.addTo(totalPendingConsideringUserLimit,
-            minpendingConsideringUserLimit);
-        Resources.subtractFrom(userNameToHeadroom.get(userName),
-            minpendingConsideringUserLimit);
-      }
-      return totalPendingConsideringUserLimit;
-    } finally {
-      readLock.unlock();
-    }
-
-  }
-
-  @Override
-  public void collectSchedulerApplications(
-      Collection<ApplicationAttemptId> apps) {
-    readLock.lock();
-    try {
-      for (FiCaSchedulerApp pendingApp : pendingOrderingPolicy
-          .getSchedulableEntities()) {
-        apps.add(pendingApp.getApplicationAttemptId());
-      }
-      for (FiCaSchedulerApp app : orderingPolicy.getSchedulableEntities()) {
-        apps.add(app.getApplicationAttemptId());
-      }
-    } finally {
-      readLock.unlock();
-    }
-
-  }
-
-  @Override
-  public void attachContainer(Resource clusterResource,
-      FiCaSchedulerApp application, RMContainer rmContainer) {
-    if (application != null && rmContainer != null
-        && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
-      FiCaSchedulerNode node =
-          csContext.getNode(rmContainer.getContainer().getNodeId());
-      allocateResource(clusterResource, application, rmContainer.getContainer()
-          .getResource(), node.getPartition(), rmContainer);
-      LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
-          + " containerState="+ rmContainer.getState()
-          + " resource=" + rmContainer.getContainer().getResource()
-          + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
-          + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
-          + queueUsage.getUsed() + " cluster=" + clusterResource);
-      // Inform the parent queue
-      getParent().attachContainer(clusterResource, application, rmContainer);
-    }
-  }
-
-  @Override
-  public void detachContainer(Resource clusterResource,
-      FiCaSchedulerApp application, RMContainer rmContainer) {
-    if (application != null && rmContainer != null
-          && rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
-      FiCaSchedulerNode node =
-          csContext.getNode(rmContainer.getContainer().getNodeId());
-      releaseResource(clusterResource, application, rmContainer.getContainer()
-          .getResource(), node.getPartition(), rmContainer);
-      LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
-          + " containerState="+ rmContainer.getState()
-          + " resource=" + rmContainer.getContainer().getResource()
-          + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
-          + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
-          + queueUsage.getUsed() + " cluster=" + clusterResource);
-      // Inform the parent queue
-      getParent().detachContainer(clusterResource, application, rmContainer);
-    }
-  }
-  
-  /**
-   * @return all ignored partition exclusivity RMContainers in the LeafQueue,
-   *         this will be used by preemption policy.
-   */
-  public Map<String, TreeSet<RMContainer>>
-      getIgnoreExclusivityRMContainers() {
-    Map<String, TreeSet<RMContainer>> clonedMap = new HashMap<>();
-
-    readLock.lock();
-    try {
-      for (Map.Entry<String, TreeSet<RMContainer>> entry : ignorePartitionExclusivityRMContainers
-          .entrySet()) {
-        clonedMap.put(entry.getKey(), new TreeSet<>(entry.getValue()));
-      }
-
-      return clonedMap;
-
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  public void setCapacity(float capacity) {
-    queueCapacities.setCapacity(capacity);
-  }
-
-  public void setCapacity(String nodeLabel, float capacity) {
-    queueCapacities.setCapacity(nodeLabel, capacity);
-  }
-
-  public void setAbsoluteCapacity(float absoluteCapacity) {
-    queueCapacities.setAbsoluteCapacity(absoluteCapacity);
-  }
-
-  public void setAbsoluteCapacity(String nodeLabel, float absoluteCapacity) {
-    queueCapacities.setAbsoluteCapacity(nodeLabel, absoluteCapacity);
-  }
-
-  public void setMaxApplicationsPerUser(int maxApplicationsPerUser) {
-    this.maxApplicationsPerUser = maxApplicationsPerUser;
-  }
-
-  public void setMaxApplications(int maxApplications) {
-    this.maxApplications = maxApplications;
-  }
-
-  public void setMaxAMResourcePerQueuePercent(
-      float maxAMResourcePerQueuePercent) {
-    this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
-  }
-
-  public OrderingPolicy<FiCaSchedulerApp>
-      getOrderingPolicy() {
-    return orderingPolicy;
-  }
-  
-  void setOrderingPolicy(
-      OrderingPolicy<FiCaSchedulerApp> orderingPolicy) {
-    writeLock.lock();
-    try {
-      if (null != this.orderingPolicy) {
-        orderingPolicy.addAllSchedulableEntities(
-            this.orderingPolicy.getSchedulableEntities());
-      }
-      this.orderingPolicy = orderingPolicy;
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  @Override
-  public Priority getDefaultApplicationPriority() {
-    return defaultAppPriorityPerQueue;
-  }
-
-  public void updateApplicationPriority(SchedulerApplication<FiCaSchedulerApp> app,
-      Priority newAppPriority) {
-    writeLock.lock();
-    try {
-      FiCaSchedulerApp attempt = app.getCurrentAppAttempt();
-      boolean isActive = orderingPolicy.removeSchedulableEntity(attempt);
-      if (!isActive) {
-        pendingOrderingPolicy.removeSchedulableEntity(attempt);
-      }
-      // Update new priority in SchedulerApplication
-      attempt.setPriority(newAppPriority);
-
-      if (isActive) {
-        orderingPolicy.addSchedulableEntity(attempt);
-      } else {
-        pendingOrderingPolicy.addSchedulableEntity(attempt);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  public OrderingPolicy<FiCaSchedulerApp>
-      getPendingAppsOrderingPolicy() {
-    return pendingOrderingPolicy;
-  }
-
-  /*
-   * Holds shared values used by all applications in
-   * the queue to calculate headroom on demand
-   */
-  static class QueueResourceLimitsInfo {
-    private Resource queueCurrentLimit;
-    private Resource clusterResource;
-    
-    public void setQueueCurrentLimit(Resource currentLimit) {
-      this.queueCurrentLimit = currentLimit;
-    }
-    
-    public Resource getQueueCurrentLimit() {
-      return queueCurrentLimit;
-    }
-    
-    public void setClusterResource(Resource clusterResource) {
-      this.clusterResource = clusterResource;
-    }
-    
-    public Resource getClusterResource() {
-      return clusterResource;
-    }
-  }
-
-  @Override
-  public void stopQueue() {
-    writeLock.lock();
-    try {
-      if (getNumApplications() > 0) {
-        updateQueueState(QueueState.DRAINING);
-      } else {
-        updateQueueState(QueueState.STOPPED);
-      }
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  /**
-   * Get all valid users in this queue.
-   * @return user list
-   */
-  public Set<String> getAllUsers() {
-    return this.getUsersManager().getUsers().keySet();
-  }
-
-  static class CachedUserLimit {
-    final Resource userLimit;
-    volatile boolean canAssign = true;
-    volatile Resource reservation = Resources.none();
-
-    CachedUserLimit(Resource userLimit) {
-      this.userLimit = userLimit;
-    }
-  }
-
-  private void updateQueuePreemptionMetrics(RMContainer rmc) {
-    final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime();
-    final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND;
-    Resource containerResource = rmc.getAllocatedResource();
-    metrics.preemptContainer();
-    long mbSeconds = (containerResource.getMemorySize() * usedMillis)
-        / DateUtils.MILLIS_PER_SECOND;
-    long vcSeconds = (containerResource.getVirtualCores() * usedMillis)
-        / DateUtils.MILLIS_PER_SECOND;
-    metrics.updatePreemptedMemoryMBSeconds(mbSeconds);
-    metrics.updatePreemptedVcoreSeconds(vcSeconds);
-    metrics.updatePreemptedResources(containerResource);
-    metrics.updatePreemptedSecondsForCustomResources(containerResource,
-        usedSeconds);
-    metrics.updatePreemptedForCustomResources(containerResource);
-  }
-
-  @Override
-  int getNumRunnableApps() {
-    readLock.lock();
-    try {
-      return runnableApps.size();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  int getNumNonRunnableApps() {
-    readLock.lock();
-    try {
-      return nonRunnableApps.size();
-    } finally {
-      readLock.unlock();
-    }
-  }
-
-  boolean removeNonRunnableApp(FiCaSchedulerApp app) {
-    writeLock.lock();
-    try {
-      return nonRunnableApps.remove(app);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  List<FiCaSchedulerApp> getCopyOfNonRunnableAppSchedulables() {
-    List<FiCaSchedulerApp> appsToReturn = new ArrayList<>();
-    readLock.lock();
-    try {
-      appsToReturn.addAll(nonRunnableApps);
-    } finally {
-      readLock.unlock();
-    }
-    return appsToReturn;
-  }
+    super(queueContext, queueName, parent, old, isDynamic);
 
-  @Override
-  public boolean isEligibleForAutoDeletion() {
-    return isDynamicQueue() && getNumApplications() == 0
-        && csContext.getConfiguration().
-        isAutoExpiredDeletionEnabled(this.getQueuePath());
+    setupQueueConfigs(queueContext.getClusterResource());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
index e415ac1279585..6867c6baf9d7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -17,13 +17,12 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler
     .SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
     .FiCaSchedulerApp;
@@ -34,7 +33,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -54,23 +52,18 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
   private static final Logger LOG = LoggerFactory.getLogger(
       ManagedParentQueue.class);
 
-  public ManagedParentQueue(final CapacitySchedulerContext cs,
+  public ManagedParentQueue(final CapacitySchedulerQueueContext queueContext,
       final String queueName, final CSQueue parent, final CSQueue old)
       throws IOException {
-    super(cs, queueName, parent, old);
+    super(queueContext, queueName, parent, old);
 
     shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
-        csContext.getConfiguration()
+        queueContext.getConfiguration()
             .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
                 getQueuePath());
 
     leafQueueTemplate = initializeLeafQueueConfigs().build();
 
-    LOG.info(
-        "Created Managed Parent Queue: [{}] with capacity: [{}]"
-            + " with max capacity: [{}]",
-        queueName, super.getCapacity(), super.getMaximumCapacity());
-
     initializeQueueManagementPolicy();
   }
 
@@ -83,7 +76,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
       validate(newlyParsedQueue);
 
       shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
-          csContext.getConfiguration()
+          queueContext.getConfiguration()
               .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
                   getQueuePath());
 
@@ -122,7 +115,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
       LOG.info(
           "Reinitialized Managed Parent Queue: [{}] with capacity [{}]"
               + " with max capacity [{}]",
-          queueName, super.getCapacity(), super.getMaximumCapacity());
+          getQueueName(), super.getCapacity(), super.getMaximumCapacity());
     } catch (YarnException ye) {
       LOG.error("Exception while computing policy changes for leaf queue : "
           + getQueuePath(), ye);
@@ -134,23 +127,23 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
 
   private void initializeQueueManagementPolicy() throws IOException {
     queueManagementPolicy =
-        csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
+        queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
             getQueuePath());
 
-    queueManagementPolicy.init(csContext, this);
+    queueManagementPolicy.init(this);
   }
 
   private void reinitializeQueueManagementPolicy() throws IOException {
     AutoCreatedQueueManagementPolicy managementPolicy =
-        csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
+        queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
             getQueuePath());
 
     if (!(managementPolicy.getClass().equals(
         this.queueManagementPolicy.getClass()))) {
       queueManagementPolicy = managementPolicy;
-      queueManagementPolicy.init(csContext, this);
+      queueManagementPolicy.init(this);
     } else{
-      queueManagementPolicy.reinitialize(csContext, this);
+      queueManagementPolicy.reinitialize(this);
     }
   }
 
@@ -159,39 +152,31 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws
     AutoCreatedLeafQueueConfig.Builder builder =
         new AutoCreatedLeafQueueConfig.Builder();
 
+    CapacitySchedulerConfiguration configuration =
+        queueContext.getConfiguration();
+
+    // TODO load configs into CapacitySchedulerConfiguration instead of duplicating them
     String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
-        csContext.getConfiguration());
-    //Load template configuration
-    CapacitySchedulerConfiguration conf =
+        configuration);
+    //Load template configuration into CapacitySchedulerConfiguration
+    CapacitySchedulerConfiguration autoCreatedTemplateConfig =
         super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix);
-    builder.configuration(conf);
-    String templateQueuePath = csContext.getConfiguration()
-        .getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
-
-    Set<String> templateConfiguredNodeLabels = csContext
-        .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
-        .getLabelsByQueue(templateQueuePath);
-    for (String nodeLabel : templateConfiguredNodeLabels) {
-      Resource templateMinResource = conf.getMinimumResourceRequirement(
-          nodeLabel, csContext.getConfiguration()
-              .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
-          resourceTypes);
-
-      if (this.capacityConfigType.equals(CapacityConfigType.PERCENTAGE)
-          && !templateMinResource.equals(Resources.none())) {
-        throw new IOException("Managed Parent Queue " + this.getQueuePath()
-            + " config type is different from leaf queue template config type");
-      }
-    }
-
+    builder.configuration(autoCreatedTemplateConfig);
+    QueueResourceQuotas queueResourceQuotas = new QueueResourceQuotas();
+    setAbsoluteResourceTemplates(configuration, queueResourceQuotas);
+
+    QueuePath templateQueuePath = configuration
+        .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath());
+    Set<String> templateConfiguredNodeLabels = queueContext
+        .getQueueManager().getConfiguredNodeLabelsForAllQueues()
+        .getLabelsByQueue(templateQueuePath.getFullPath());
     //Load template capacities
     QueueCapacities queueCapacities = new QueueCapacities(false);
     CSQueueUtils.loadCapacitiesByLabelsFromConf(templateQueuePath,
         queueCapacities,
-        csContext.getConfiguration(),
+        configuration,
         templateConfiguredNodeLabels);
 
-
     /**
      * Populate leaf queue template (of Parent resources configured in
      * ABSOLUTE_RESOURCE) capacities with actual values for which configured has
@@ -202,39 +187,64 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws
       updateQueueCapacities(queueCapacities);
     }
     builder.capacities(queueCapacities);
+    builder.resourceQuotas(queueResourceQuotas);
     return builder;
   }
 
+  private void setAbsoluteResourceTemplates(CapacitySchedulerConfiguration configuration,
+                                            QueueResourceQuotas queueResourceQuotas) throws IOException {
+    QueuePath templateQueuePath = configuration
+        .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath());
+    Set<String> templateConfiguredNodeLabels = queueContext
+        .getQueueManager().getConfiguredNodeLabelsForAllQueues()
+        .getLabelsByQueue(templateQueuePath.getFullPath());
+
+    for (String nodeLabel : templateConfiguredNodeLabels) {
+      Resource templateMinResource = configuration.getMinimumResourceRequirement(
+          nodeLabel, templateQueuePath.getFullPath(), resourceTypes);
+      queueResourceQuotas.setConfiguredMinResource(nodeLabel, templateMinResource);
+
+      if (this.capacityConfigType.equals(CapacityConfigType.PERCENTAGE)
+          && !templateMinResource.equals(Resources.none())) {
+        throw new IOException("Managed Parent Queue " + this.getQueuePath()
+            + " config type is different from leaf queue template config type");
+      }
+    }
+  }
+
   private void updateQueueCapacities(QueueCapacities queueCapacities) {
+    CapacitySchedulerConfiguration configuration =
+        queueContext.getConfiguration();
+
     for (String label : queueCapacities.getExistingNodeLabels()) {
       queueCapacities.setCapacity(label,
-          this.csContext.getResourceCalculator().divide(
-              this.csContext.getClusterResource(),
-              this.csContext.getConfiguration().getMinimumResourceRequirement(
+          resourceCalculator.divide(
+              queueContext.getClusterResource(),
+              configuration.getMinimumResourceRequirement(
                   label,
-                  this.csContext.getConfiguration()
+                  configuration
                       .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
                   resourceTypes),
               getQueueResourceQuotas().getConfiguredMinResource(label)));
 
-      Resource childMaxResource = this.csContext.getConfiguration()
+      Resource childMaxResource = configuration
           .getMaximumResourceRequirement(label,
-              this.csContext.getConfiguration()
+              configuration
                   .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
               resourceTypes);
       Resource parentMaxRes = getQueueResourceQuotas()
           .getConfiguredMaxResource(label);
 
       Resource effMaxResource = Resources.min(
-          this.csContext.getResourceCalculator(),
-          this.csContext.getClusterResource(),
+          resourceCalculator,
+          queueContext.getClusterResource(),
           childMaxResource.equals(Resources.none()) ? parentMaxRes
               : childMaxResource,
           parentMaxRes);
 
       queueCapacities.setMaximumCapacity(
-          label, this.csContext.getResourceCalculator().divide(
-               this.csContext.getClusterResource(),
+          label, resourceCalculator.divide(
+              queueContext.getClusterResource(),
                effMaxResource,
                getQueueResourceQuotas().getConfiguredMaxResource(label)));
 
@@ -269,7 +279,7 @@ public void addChildQueue(CSQueue childQueue)
             "Expected child queue to be an instance of AutoCreatedLeafQueue");
       }
 
-      CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+      CapacitySchedulerConfiguration conf = queueContext.getConfiguration();
       ManagedParentQueue parentQueue =
           (ManagedParentQueue) childQueue.getParent();
 
@@ -323,8 +333,8 @@ public void addChildQueue(CSQueue childQueue)
 
       // Do one update cluster resource call to make sure all absolute resources
       // effective resources are updated.
-      updateClusterResource(this.csContext.getClusterResource(),
-          new ResourceLimits(this.csContext.getClusterResource()));
+      updateClusterResource(queueContext.getClusterResource(),
+          new ResourceLimits(queueContext.getClusterResource()));
     } finally {
       writeLock.unlock();
     }
@@ -335,7 +345,7 @@ public List<FiCaSchedulerApp> getScheduleableApplications() {
     try {
       List<FiCaSchedulerApp> apps = new ArrayList<>();
       for (CSQueue childQueue : getChildQueues()) {
-        apps.addAll(((LeafQueue) childQueue).getApplications());
+        apps.addAll(((AbstractLeafQueue) childQueue).getApplications());
       }
       return Collections.unmodifiableList(apps);
     } finally {
@@ -348,7 +358,7 @@ public List<FiCaSchedulerApp> getPendingApplications() {
     try {
       List<FiCaSchedulerApp> apps = new ArrayList<>();
       for (CSQueue childQueue : getChildQueues()) {
-        apps.addAll(((LeafQueue) childQueue).getPendingApplications());
+        apps.addAll(((AbstractLeafQueue) childQueue).getPendingApplications());
       }
       return Collections.unmodifiableList(apps);
     } finally {
@@ -361,7 +371,7 @@ public List<FiCaSchedulerApp> getAllApplications() {
     try {
       List<FiCaSchedulerApp> apps = new ArrayList<>();
       for (CSQueue childQueue : getChildQueues()) {
-        apps.addAll(((LeafQueue) childQueue).getAllApplications());
+        apps.addAll(((AbstractLeafQueue) childQueue).getAllApplications());
       }
       return Collections.unmodifiableList(apps);
     } finally {
@@ -428,12 +438,11 @@ public void validateQueueManagementChanges(
                 + " Ignoring update " + queueManagementChanges);
       }
 
-      switch (queueManagementChange.getQueueAction()){
-      case UPDATE_QUEUE:
+      if (queueManagementChange.getQueueAction() ==
+          QueueManagementChange.QueueAction.UPDATE_QUEUE) {
         AutoCreatedLeafQueueConfig template =
             queueManagementChange.getUpdatedQueueTemplate();
         ((AutoCreatedLeafQueue) childQueue).validateConfigurations(template);
-        break;
       }
 
     }
@@ -443,37 +452,24 @@ private void applyQueueManagementChanges(
       List<QueueManagementChange> queueManagementChanges)
       throws SchedulerDynamicEditException, IOException {
     for (QueueManagementChange queueManagementChange : queueManagementChanges) {
-      switch (queueManagementChange.getQueueAction()){
-      case UPDATE_QUEUE:
+      if (queueManagementChange.getQueueAction() ==
+          QueueManagementChange.QueueAction.UPDATE_QUEUE) {
         AutoCreatedLeafQueue childQueueToBeUpdated =
             (AutoCreatedLeafQueue) queueManagementChange.getQueue();
         //acquires write lock on leaf queue
         childQueueToBeUpdated.reinitializeFromTemplate(
             queueManagementChange.getUpdatedQueueTemplate());
-        break;
       }
     }
   }
 
-  public CapacitySchedulerConfiguration getLeafQueueConfigs(
-      String leafQueueName) {
-    return getLeafQueueConfigs(getLeafQueueTemplate().getLeafQueueConfigs(),
-        leafQueueName);
-  }
-
-  public CapacitySchedulerConfiguration getLeafQueueConfigs(
-      CapacitySchedulerConfiguration templateConfig, String leafQueueName) {
-    CapacitySchedulerConfiguration leafQueueConfigTemplate = new
-        CapacitySchedulerConfiguration(new Configuration(false), false);
-    for (final Iterator<Map.Entry<String, String>> iterator =
-         templateConfig.iterator(); iterator.hasNext(); ) {
-      Map.Entry<String, String> confKeyValuePair = iterator.next();
-      final String name = confKeyValuePair.getKey().replaceFirst(
-          CapacitySchedulerConfiguration
-              .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX,
-          leafQueueName);
-      leafQueueConfigTemplate.set(name, confKeyValuePair.getValue());
+  public void setLeafQueueConfigs(String leafQueueName) {
+    CapacitySchedulerConfiguration templateConfig = leafQueueTemplate.getLeafQueueConfigs();
+    for (Map.Entry<String, String> confKeyValuePair : templateConfig) {
+      final String name = confKeyValuePair.getKey()
+          .replaceFirst(CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX,
+              leafQueueName);
+      queueContext.setConfigurationEntry(name, confKeyValuePair.getValue());
     }
-    return leafQueueConfigTemplate;
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 3a25a17d97500..0949d512a79f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
@@ -77,6 +78,8 @@
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getACLsForFlexibleAutoCreatedParentQueue;
+
 @Private
 @Evolving
 public class ParentQueue extends AbstractCSQueue {
@@ -87,7 +90,6 @@ public class ParentQueue extends AbstractCSQueue {
   protected final List<CSQueue> childQueues;
   private final boolean rootQueue;
   private volatile int numApplications;
-  private final CapacitySchedulerContext scheduler;
 
   private final RecordFactory recordFactory = 
     RecordFactoryProvider.getRecordFactory(null);
@@ -102,33 +104,26 @@ public class ParentQueue extends AbstractCSQueue {
 
   private AutoCreatedQueueTemplate autoCreatedQueueTemplate;
 
-  // effective min ratio per resource, it is used during updateClusterResource,
-  // leaf queue can use this to calculate effective resources.
-  // This field will not be edited, reference will point to a new immutable map
-  // after every time recalculation
-  private volatile Map<String, Float> effectiveMinRatioPerResource;
+  // A ratio of the queue's effective minimum resource and the summary of the configured
+  // minimum resource of its children grouped by labels and calculated for each resource names
+  // distinctively.
+  private final Map<String, Map<String, Float>> effectiveMinResourceRatio =
+      new ConcurrentHashMap<>();
 
-  public ParentQueue(CapacitySchedulerContext cs,
+  public ParentQueue(CapacitySchedulerQueueContext queueContext,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
-    this(cs, cs.getConfiguration(), queueName, parent, old, false);
-  }
-  private ParentQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration csConf, String queueName,
-      CSQueue parent,
-      CSQueue old) throws IOException {
-    this(cs, csConf, queueName, parent, old, false);
+    this(queueContext, queueName, parent, old, false);
   }
 
-  private ParentQueue(CapacitySchedulerContext cs,
-      CapacitySchedulerConfiguration csConf, String queueName, CSQueue parent,
-      CSQueue old, boolean isDynamic)
+  private ParentQueue(CapacitySchedulerQueueContext queueContext,
+      String queueName, CSQueue parent, CSQueue old, boolean isDynamic)
       throws IOException {
-    super(cs, queueName, parent, old);
+    super(queueContext, queueName, parent, old);
     setDynamicQueue(isDynamic);
-    this.scheduler = cs;
     this.rootQueue = (parent == null);
 
-    float rawCapacity = csConf.getNonLabeledQueueCapacity(getQueuePath());
+    float rawCapacity = queueContext.getConfiguration()
+        .getNonLabeledQueueCapacity(this.queuePath);
 
     if (rootQueue &&
         (rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) {
@@ -139,13 +134,10 @@ private ParentQueue(CapacitySchedulerContext cs,
 
     this.childQueues = new ArrayList<>();
     this.allowZeroCapacitySum =
-        cs.getConfiguration().getAllowZeroCapacitySum(getQueuePath());
-
-    setupQueueConfigs(cs.getClusterResource(), csConf);
+        queueContext.getConfiguration()
+            .getAllowZeroCapacitySum(getQueuePath());
 
-    LOG.info("Initialized parent-queue " + queueName +
-        " name=" + queueName +
-        ", fullname=" + getQueuePath());
+    setupQueueConfigs(queueContext.getClusterResource());
   }
 
   // returns what is configured queue ordering policy
@@ -155,49 +147,59 @@ private String getQueueOrderingPolicyConfigName() {
         queueOrderingPolicy.getConfigName();
   }
 
-  protected void setupQueueConfigs(Resource clusterResource,
-      CapacitySchedulerConfiguration csConf)
+  protected void setupQueueConfigs(Resource clusterResource)
       throws IOException {
     writeLock.lock();
     try {
+      CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
       autoCreatedQueueTemplate = new AutoCreatedQueueTemplate(
-          csConf, getQueuePath());
-      super.setupQueueConfigs(clusterResource, csConf);
+          configuration, this.queuePath);
+      super.setupQueueConfigs(clusterResource);
       StringBuilder aclsString = new StringBuilder();
-      for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
-        aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
+      for (Map.Entry<AccessType, AccessControlList> e : getACLs().entrySet()) {
+        aclsString.append(e.getKey()).append(":")
+            .append(e.getValue().getAclString());
       }
 
       StringBuilder labelStrBuilder = new StringBuilder();
-      if (accessibleLabels != null) {
-        for (String s : accessibleLabels) {
-          labelStrBuilder.append(s)
-              .append(",");
+      if (getAccessibleNodeLabels() != null) {
+        for (String nodeLabel : getAccessibleNodeLabels()) {
+          labelStrBuilder.append(nodeLabel).append(",");
         }
       }
 
       // Initialize queue ordering policy
-      queueOrderingPolicy = csConf.getQueueOrderingPolicy(
+      queueOrderingPolicy = configuration.getQueueOrderingPolicy(
           getQueuePath(), parent == null ?
               null :
               ((ParentQueue) parent).getQueueOrderingPolicyConfigName());
       queueOrderingPolicy.setQueues(childQueues);
 
-      LOG.info(queueName + ", " + getCapacityOrWeightString()
-          + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity()
-          + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity()
-          + ", absoluteMaxCapacity=" + this.queueCapacities
-          .getAbsoluteMaximumCapacity() + ", state=" + getState() + ", acls="
-          + aclsString + ", labels=" + labelStrBuilder.toString() + "\n"
-          + ", reservationsContinueLooking=" + reservationsContinueLooking
+      LOG.info(getQueueName() + ", " + getCapacityOrWeightString()
+          + ", absoluteCapacity=" + getAbsoluteCapacity()
+          + ", maxCapacity=" + getMaximumCapacity()
+          + ", absoluteMaxCapacity=" + getAbsoluteMaximumCapacity()
+          + ", state=" + getState() + ", acls="
+          + aclsString + ", labels=" + labelStrBuilder + "\n"
+          + ", reservationsContinueLooking=" + isReservationsContinueLooking()
           + ", orderingPolicy=" + getQueueOrderingPolicyConfigName()
-          + ", priority=" + priority
+          + ", priority=" + getPriority()
           + ", allowZeroCapacitySum=" + allowZeroCapacitySum);
     } finally {
       writeLock.unlock();
     }
   }
 
+  @Override
+  protected void setDynamicQueueACLProperties() {
+    super.setDynamicQueueACLProperties();
+
+    if (parent instanceof ParentQueue) {
+      acls.putAll(getACLsForFlexibleAutoCreatedParentQueue(
+          ((ParentQueue) parent).getAutoCreatedQueueTemplate()));
+    }
+  }
+
   private static float PRECISION = 0.0005f; // 0.05% precision
 
   // Check weight configuration, throw exception when configuration is invalid
@@ -326,15 +328,15 @@ void setChildQueues(Collection<CSQueue> childQueues) throws IOException {
                 .getConfiguredMinResource(nodeLabel));
           }
           Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
-              scheduler.getClusterResource());
+              queueContext.getClusterResource());
           Resource parentMinResource =
-              queueResourceQuotas.getConfiguredMinResource(nodeLabel);
+              usageTracker.getQueueResourceQuotas().getConfiguredMinResource(nodeLabel);
           if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
               resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
             throw new IOException(
                 "Parent Queues" + " capacity: " + parentMinResource
                     + " is less than" + " to its children:" + minRes
-                    + " for queue:" + queueName);
+                    + " for queue:" + getQueueName());
           }
         }
       }
@@ -355,8 +357,8 @@ void setChildQueues(Collection<CSQueue> childQueues) throws IOException {
             if (Math.abs(childrenPctSum) > PRECISION) {
               // It is wrong when percent sum != {0, 1}
               throw new IOException(
-                  "Illegal" + " capacity sum of " + childrenPctSum
-                      + " for children of queue " + queueName + " for label="
+                  "Illegal capacity sum of " + childrenPctSum
+                      + " for children of queue " + getQueueName() + " for label="
                       + nodeLabel + ". It should be either 0 or 1.0");
             } else{
               // We also allow children's percent sum = 0 under the following
@@ -368,8 +370,8 @@ void setChildQueues(Collection<CSQueue> childQueues) throws IOException {
                 if ((Math.abs(queueCapacities.getCapacity(nodeLabel))
                     > PRECISION) && (!allowZeroCapacitySum)) {
                   throw new IOException(
-                      "Illegal" + " capacity sum of " + childrenPctSum
-                          + " for children of queue " + queueName
+                      "Illegal capacity sum of " + childrenPctSum
+                          + " for children of queue " + getQueueName()
                           + " for label=" + nodeLabel
                           + ". It is set to 0, but parent percent != 0, and "
                           + "doesn't allow children capacity to set to 0");
@@ -383,9 +385,9 @@ void setChildQueues(Collection<CSQueue> childQueues) throws IOException {
                 queueCapacities.getCapacity(nodeLabel)) <= 0f
                 && !allowZeroCapacitySum) {
               throw new IOException(
-                  "Illegal" + " capacity sum of " + childrenPctSum
-                      + " for children of queue " + queueName + " for label="
-                      + nodeLabel + ". queue=" + queueName
+                  "Illegal capacity sum of " + childrenPctSum
+                      + " for children of queue " + getQueueName() + " for label="
+                      + nodeLabel + ". queue=" + getQueueName()
                       + " has zero capacity, but child"
                       + "queues have positive capacities");
             }
@@ -471,13 +473,13 @@ public List<QueueUserACLInfo> getQueueUserAclInfo(
   }
 
   public String toString() {
-    return queueName + ": " +
+    return getQueueName() + ": " +
         "numChildQueue= " + childQueues.size() + ", " +
         getCapacityOrWeightString() + ", " +
         "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
-        "usedResources=" + queueUsage.getUsed() + 
-        "usedCapacity=" + getUsedCapacity() + ", " + 
-        "numApps=" + getNumApplications() + ", " + 
+        "usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " +
+        "usedCapacity=" + getUsedCapacity() + ", " +
+        "numApps=" + getNumApplications() + ", " +
         "numContainers=" + getNumContainers();
   }
 
@@ -489,11 +491,10 @@ private CSQueue createNewQueue(String childQueuePath, boolean isLeaf)
           childQueuePath.lastIndexOf(".") + 1);
 
       if (isLeaf) {
-        childQueue = new LeafQueue(csContext, csContext.getConfiguration(),
+        childQueue = new LeafQueue(queueContext,
             queueShortName, this, null, true);
       } else{
-        childQueue = new ParentQueue(csContext, csContext.getConfiguration(),
-            queueShortName, this, null, true);
+        childQueue = new ParentQueue(queueContext, queueShortName, this, null, true);
       }
       childQueue.setDynamicQueue(true);
       // It should be sufficient now, we don't need to set more, because weights
@@ -524,7 +525,7 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf)
       // should not happen, since it will be handled before calling this method)
       // , but we will move on.
       CSQueue queue =
-          csContext.getCapacitySchedulerQueueManager().getQueueByFullName(
+          queueContext.getQueueManager().getQueueByFullName(
               childQueuePath);
       if (queue != null) {
         LOG.warn(
@@ -534,7 +535,7 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf)
       }
 
       // Check if the max queue limit is exceeded.
-      int maxQueues = csContext.getConfiguration().
+      int maxQueues = queueContext.getConfiguration().
           getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath());
       if (childQueues.size() >= maxQueues) {
         throw new SchedulerDynamicEditException(
@@ -565,8 +566,8 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf)
       // Call updateClusterResource.
       // Which will deal with all effectiveMin/MaxResource
       // Calculation
-      this.updateClusterResource(csContext.getClusterResource(),
-          new ResourceLimits(this.csContext.getClusterResource()));
+      this.updateClusterResource(queueContext.getClusterResource(),
+          new ResourceLimits(queueContext.getClusterResource()));
 
       return newQueue;
     } finally {
@@ -597,14 +598,14 @@ public void removeChildQueue(CSQueue queue)
 
       // Now we can do remove and update
       this.childQueues.remove(queue);
-      this.scheduler.getCapacitySchedulerQueueManager()
+      queueContext.getQueueManager()
           .removeQueue(queue.getQueuePath());
 
       // Call updateClusterResource,
       // which will deal with all effectiveMin/MaxResource
       // Calculation
-      this.updateClusterResource(csContext.getClusterResource(),
-          new ResourceLimits(this.csContext.getClusterResource()));
+      this.updateClusterResource(queueContext.getClusterResource(),
+          new ResourceLimits(queueContext.getClusterResource()));
 
     } finally {
       writeLock.unlock();
@@ -618,7 +619,7 @@ public void removeChildQueue(CSQueue queue)
    * false otherwise
    */
   public boolean isEligibleForAutoQueueCreation() {
-    return isDynamicQueue() || csContext.getConfiguration().
+    return isDynamicQueue() || queueContext.getConfiguration().
         isAutoQueueCreationV2Enabled(getQueuePath());
   }
   
@@ -645,7 +646,7 @@ public void reinitialize(CSQueue newlyParsedQueue,
       ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue;
 
       // Set new configs
-      setupQueueConfigs(clusterResource, csContext.getConfiguration());
+      setupQueueConfigs(clusterResource);
 
       // Re-configure existing child queues and add new ones
       // The CS has already checked to ensure all existing child queues are present!
@@ -676,17 +677,17 @@ public void reinitialize(CSQueue newlyParsedQueue,
           // parent Queue has been converted to child queue. The CS has already
           // checked to ensure that this child-queue is in STOPPED state if
           // Child queue has been converted to ParentQueue.
-          if ((childQueue instanceof LeafQueue
+          if ((childQueue instanceof AbstractLeafQueue
               && newChildQueue instanceof ParentQueue)
               || (childQueue instanceof ParentQueue
-                  && newChildQueue instanceof LeafQueue)) {
+                  && newChildQueue instanceof AbstractLeafQueue)) {
             // We would convert this LeafQueue to ParentQueue, or vice versa.
             // consider this as the combination of DELETE then ADD.
             newChildQueue.setParent(this);
             currentChildQueues.put(newChildQueueName, newChildQueue);
             // inform CapacitySchedulerQueueManager
             CapacitySchedulerQueueManager queueManager =
-                this.csContext.getCapacitySchedulerQueueManager();
+                queueContext.getQueueManager();
             queueManager.addQueue(newChildQueueName, newChildQueue);
             continue;
           }
@@ -757,7 +758,7 @@ public void submitApplication(ApplicationId applicationId, String user,
       try {
         parent.submitApplication(applicationId, user, queue);
       } catch (AccessControlException ace) {
-        LOG.info("Failed to submit application to parent-queue: " + 
+        LOG.info("Failed to submit application to parent-queue: " +
             parent.getQueuePath(), ace);
         removeApplication(applicationId, user);
         throw ace;
@@ -769,9 +770,9 @@ public void validateSubmitApplication(ApplicationId applicationId,
       String userName, String queue) throws AccessControlException {
     writeLock.lock();
     try {
-      if (queue.equals(queueName)) {
+      if (queue.equals(getQueueName())) {
         throw new AccessControlException(
-            "Cannot submit application " + "to non-leaf queue: " + queueName);
+            "Cannot submit application " + "to non-leaf queue: " + getQueueName());
       }
 
       if (getState() != QueueState.RUNNING) {
@@ -846,7 +847,7 @@ private void removeApplication(ApplicationId applicationId,
   }
 
   private String getParentName() {
-    return getParent() != null ? getParent().getQueuePath() : "";
+    return parent != null ? parent.getQueuePath() : "";
   }
 
   @Override
@@ -857,7 +858,7 @@ public CSAssignment assignContainers(Resource clusterResource,
 
     // if our queue cannot access this node, just return
     if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
-        && !accessibleToPartition(candidates.getPartition())) {
+        && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) {
       if (LOG.isDebugEnabled()) {
         long now = System.currentTimeMillis();
         // Do logging every 1 sec to avoid excessive logging.
@@ -992,7 +993,7 @@ public CSAssignment assignContainers(Resource clusterResource,
           LOG.debug("assignedContainer reserved=" + isReserved + " queue="
               + getQueuePath() + " usedCapacity=" + getUsedCapacity()
               + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
-              + queueUsage.getUsed() + " cluster=" + clusterResource);
+              + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource);
 
           LOG.debug(
               "ParentQ=" + getQueuePath() + " assignedSoFarInThisIteration="
@@ -1038,7 +1039,7 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) {
     // 1) Node doesn't have reserved container
     // 2) Node's available-resource + killable-resource should > 0
     boolean accept = node.getReservedContainer() == null &&
-        Resources.fitsIn(resourceCalculator, minimumAllocation,
+        Resources.fitsIn(resourceCalculator, queueAllocationSettings.getMinimumAllocation(),
             Resources.add(node.getUnallocatedResource(), node.getTotalKillableResources()));
     if (!accept) {
       ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
@@ -1065,7 +1066,7 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child,
     // First, cap parent limit by parent's max
     parentLimits.setLimit(Resources.min(resourceCalculator, clusterResource,
         parentLimits.getLimit(),
-        queueResourceQuotas.getEffectiveMaxResource(nodePartition)));
+        usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition)));
 
     // Parent available resource = parent-limit - parent-used-resource
     Resource limit = parentLimits.getLimit();
@@ -1073,7 +1074,7 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child,
       limit = parentLimits.getNetLimit();
     }
     Resource parentMaxAvailableResource = Resources.subtract(
-        limit, queueUsage.getUsed(nodePartition));
+        limit, usageTracker.getQueueUsage().getUsed(nodePartition));
 
     // Deduct killable from used
     Resources.addTo(parentMaxAvailableResource,
@@ -1085,7 +1086,8 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child,
 
     // Normalize before return
     childLimit =
-        Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);
+        Resources.roundDown(resourceCalculator, childLimit,
+            queueAllocationSettings.getMinimumAllocation());
 
     return new ResourceLimits(childLimit);
   }
@@ -1134,7 +1136,7 @@ private CSAssignment assignContainersToChildQueues(Resource cluster,
           assignment = childAssignment;
         }
         Resource blockedHeadroom = null;
-        if (childQueue instanceof LeafQueue) {
+        if (childQueue instanceof AbstractLeafQueue) {
           blockedHeadroom = childLimits.getHeadroom();
         } else {
           blockedHeadroom = childLimits.getBlockedHeadroom();
@@ -1158,7 +1160,7 @@ String getChildQueuesToPrint() {
     StringBuilder sb = new StringBuilder();
     for (CSQueue q : childQueues) {
       sb.append(q.getQueuePath() + 
-          "usedCapacity=(" + q.getUsedCapacity() + "), " + 
+          " usedCapacity=(" + q.getUsedCapacity() + "), " +
           " label=("
           + StringUtils.join(q.getAccessibleNodeLabels().iterator(), ",") 
           + ")");
@@ -1270,7 +1272,7 @@ public void updateClusterResource(Resource clusterResource,
       }
 
       // Update effective capacity in all parent queue.
-      for (String label : configuredNodeLabels) {
+      for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
         calculateEffectiveResourcesAndCapacity(label, clusterResource);
       }
 
@@ -1290,7 +1292,7 @@ public void updateClusterResource(Resource clusterResource,
           labelManager.getResourceByLabel(null, clusterResource),
           RMNodeLabelsManager.NO_LABEL, this);
     } catch (IOException e) {
-      LOG.error("Fatal issue found: e", e);
+      LOG.error("Error during updating cluster resource: ", e);
       throw new YarnRuntimeException("Fatal issue during scheduling", e);
     } finally {
       writeLock.unlock();
@@ -1304,17 +1306,24 @@ public boolean hasChildQueues() {
 
   private void calculateEffectiveResourcesAndCapacity(String label,
       Resource clusterResource) {
+    // Update effective resources for my self;
+    if (rootQueue) {
+      Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource);
+      usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label, resourceByLabel);
+      usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label, resourceByLabel);
+    } else {
+      super.updateEffectiveResources(clusterResource);
+    }
+
+    recalculateEffectiveMinRatio(label, clusterResource);
+  }
+
+  private void recalculateEffectiveMinRatio(String label, Resource clusterResource) {
     // For root queue, ensure that max/min resource is updated to latest
     // cluster resource.
-    Resource resourceByLabel = labelManager.getResourceByLabel(label,
-        clusterResource);
-
-    /*
-     * == Below logic are added to calculate effectiveMinRatioPerResource ==
-     */
+    Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource);
 
-    // Total configured min resources of direct children of this given parent
-    // queue
+    // Total configured min resources of direct children of this given parent queue
     Resource configuredMinResources = Resource.newInstance(0L, 0);
     for (CSQueue childQueue : getChildQueues()) {
       Resources.addTo(configuredMinResources,
@@ -1322,8 +1331,7 @@ private void calculateEffectiveResourcesAndCapacity(String label,
     }
 
     // Factor to scale down effective resource: When cluster has sufficient
-    // resources, effective_min_resources will be same as configured
-    // min_resources.
+    // resources, effective_min_resources will be same as configured min_resources.
     Resource numeratorForMinRatio = null;
     if (getQueuePath().equals("root")) {
       if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(resourceCalculator,
@@ -1332,26 +1340,17 @@ private void calculateEffectiveResourcesAndCapacity(String label,
       }
     } else {
       if (Resources.lessThan(resourceCalculator, clusterResource,
-          queueResourceQuotas.getEffectiveMinResource(label),
+          usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label),
           configuredMinResources)) {
-        numeratorForMinRatio = queueResourceQuotas
-            .getEffectiveMinResource(label);
+        numeratorForMinRatio = usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label);
       }
     }
 
-    effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
-        configuredMinResources, numeratorForMinRatio);
-
-    // Update effective resources for my self;
-    if (rootQueue) {
-      queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
-      queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
-    } else{
-      super.updateEffectiveResources(clusterResource);
-    }
+    effectiveMinResourceRatio.put(label, getEffectiveMinRatio(
+        configuredMinResources, numeratorForMinRatio));
   }
 
-  private Map<String, Float> getEffectiveMinRatioPerResource(
+  private Map<String, Float> getEffectiveMinRatio(
       Resource configuredMinResources, Resource numeratorForMinRatio) {
     Map<String, Float> effectiveMinRatioPerResource = new HashMap<>();
     if (numeratorForMinRatio != null) {
@@ -1399,7 +1398,7 @@ public void recoverContainer(Resource clusterResource,
     // Careful! Locking order is important!
     writeLock.lock();
     try {
-      FiCaSchedulerNode node = scheduler.getNode(
+      FiCaSchedulerNode node = queueContext.getNode(
           rmContainer.getContainer().getNodeId());
       allocateResource(clusterResource,
           rmContainer.getContainer().getResource(), node.getPartition());
@@ -1437,13 +1436,13 @@ public void attachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
       FiCaSchedulerNode node =
-          scheduler.getNode(rmContainer.getContainer().getNodeId());
+          queueContext.getNode(rmContainer.getContainer().getNodeId());
       allocateResource(clusterResource, rmContainer.getContainer()
           .getResource(), node.getPartition());
       LOG.info("movedContainer" + " queueMoveIn=" + getQueuePath()
           + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
-          + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
-          + clusterResource);
+          + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed() +
+          " cluster=" + clusterResource);
       // Inform the parent
       if (parent != null) {
         parent.attachContainer(clusterResource, application, rmContainer);
@@ -1456,14 +1455,14 @@ public void detachContainer(Resource clusterResource,
       FiCaSchedulerApp application, RMContainer rmContainer) {
     if (application != null) {
       FiCaSchedulerNode node =
-          scheduler.getNode(rmContainer.getContainer().getNodeId());
+          queueContext.getNode(rmContainer.getContainer().getNodeId());
       super.releaseResource(clusterResource,
           rmContainer.getContainer().getResource(),
           node.getPartition());
       LOG.info("movedContainer" + " queueMoveOut=" + getQueuePath()
           + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
-          + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
-          + clusterResource);
+          + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed() +
+          " cluster=" + clusterResource);
       // Inform the parent
       if (parent != null) {
         parent.detachContainer(clusterResource, application, rmContainer);
@@ -1507,11 +1506,11 @@ void allocateResource(Resource clusterResource,
        * When this happens, we have to preempt killable container (on same or different
        * nodes) of parent queue to avoid violating parent's max resource.
        */
-      if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+      if (!usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition)
           .equals(Resources.none())) {
         if (Resources.lessThan(resourceCalculator, clusterResource,
-            queueResourceQuotas.getEffectiveMaxResource(nodePartition),
-            queueUsage.getUsed(nodePartition))) {
+            usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition),
+            usageTracker.getQueueUsage().getUsed(nodePartition))) {
           killContainersToEnforceMaxQueueCapacity(nodePartition,
               clusterResource);
         }
@@ -1541,14 +1540,14 @@ private void killContainersToEnforceMaxQueueCapacity(String partition,
     Resource maxResource = getEffectiveMaxCapacity(partition);
 
     while (Resources.greaterThan(resourceCalculator, partitionResource,
-        queueUsage.getUsed(partition), maxResource)) {
+        usageTracker.getQueueUsage().getUsed(partition), maxResource)) {
       RMContainer toKillContainer = killableContainerIter.next();
-      FiCaSchedulerApp attempt = csContext.getApplicationAttempt(
+      FiCaSchedulerApp attempt = queueContext.getApplicationAttempt(
           toKillContainer.getContainerId().getApplicationAttemptId());
-      FiCaSchedulerNode node = csContext.getNode(
+      FiCaSchedulerNode node = queueContext.getNode(
           toKillContainer.getAllocatedNode());
       if (null != attempt && null != node) {
-        LeafQueue lq = attempt.getCSLeafQueue();
+        AbstractLeafQueue lq = attempt.getCSLeafQueue();
         lq.completedContainer(clusterResource, attempt, node, toKillContainer,
             SchedulerUtils.createPreemptedContainerStatus(
                 toKillContainer.getContainerId(),
@@ -1584,7 +1583,7 @@ public void apply(Resource cluster,
 
           LOG.info("assignedContainer" + " queue=" + getQueuePath()
               + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
-              + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed()
+              + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed()
               + " cluster=" + cluster);
         } finally {
           writeLock.unlock();
@@ -1648,15 +1647,14 @@ void decrementRunnableApps() {
     }
   }
 
-  // This is a locking free method
-  Map<String, Float> getEffectiveMinRatioPerResource() {
-    return effectiveMinRatioPerResource;
+  Map<String, Float> getEffectiveMinRatio(String label) {
+    return effectiveMinResourceRatio.get(label);
   }
 
   @Override
   public boolean isEligibleForAutoDeletion() {
     return isDynamicQueue() && getChildQueues().size() == 0 &&
-        csContext.getConfiguration().
+        queueContext.getConfiguration().
             isAutoExpiredDeletionEnabled(this.getQueuePath());
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 3cddeac6a677d..3bbbffe588192 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -21,8 +21,10 @@
 import java.io.IOException;
 
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,17 +42,15 @@ public class PlanQueue extends AbstractManagedParentQueue {
   private int maxAppsPerUserForReservation;
   private float userLimit;
   private float userLimitFactor;
-  protected CapacitySchedulerContext schedulerContext;
   private boolean showReservationsAsQueues;
 
-  public PlanQueue(CapacitySchedulerContext cs, String queueName,
+  public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName,
       CSQueue parent, CSQueue old) throws IOException {
-    super(cs, queueName, parent, old);
+    super(queueContext, queueName, parent, old);
     updateAbsoluteCapacities();
 
-    this.schedulerContext = cs;
     // Set the reservation queue attributes for the Plan
-    CapacitySchedulerConfiguration conf = cs.getConfiguration();
+    CapacitySchedulerConfiguration conf = queueContext.getConfiguration();
     String queuePath = super.getQueuePath();
     int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath);
     showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath);
@@ -106,7 +106,7 @@ public void reinitialize(CSQueue newlyParsedQueue,
       }
 
       // Set new configs
-      setupQueueConfigs(clusterResource, csContext.getConfiguration());
+      setupQueueConfigs(clusterResource);
 
       updateQuotas(newlyParsedParentQueue.userLimit,
           newlyParsedParentQueue.userLimitFactor,
@@ -125,6 +125,23 @@ public void reinitialize(CSQueue newlyParsedQueue,
     }
   }
 
+  public ReservationQueue initializeDefaultInternalQueue() throws IOException {
+    //initializing the "internal" default queue, for SLS compatibility
+    String defReservationId =
+        getQueueName() + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+
+    ReservationQueue resQueue = new ReservationQueue(queueContext,
+        defReservationId, this);
+    try {
+      resQueue.initializeEntitlements();
+    } catch (SchedulerDynamicEditException e) {
+      throw new IllegalStateException(e);
+    }
+    childQueues.add(resQueue);
+
+    return resQueue;
+  }
+
   private void updateQuotas(float newUserLimit, float newUserLimitFactor,
       int newMaxAppsForReservation, int newMaxAppsPerUserForReservation) {
     this.userLimit = newUserLimit;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java
new file mode 100644
index 0000000000000..101c8076fdc04
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED;
+
+/**
+ * This class determines minimum and maximum allocation settings based on the
+ * {@link CapacitySchedulerConfiguration} and other queue
+ * properties.
+ **/
+public class QueueAllocationSettings {
+  private final Resource minimumAllocation;
+  private Resource maximumAllocation;
+
+  public QueueAllocationSettings(Resource minimumAllocation) {
+    this.minimumAllocation = minimumAllocation;
+  }
+
+  void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, String queuePath,
+      CSQueue parent) {
+    Resource clusterMax = ResourceUtils
+        .fetchMaximumAllocationFromConfig(configuration);
+    Resource queueMax = configuration.getQueueMaximumAllocation(queuePath);
+
+    maximumAllocation = Resources.clone(
+        parent == null ? clusterMax : parent.getMaximumAllocation());
+
+    String errMsg =
+        "Queue maximum allocation cannot be larger than the cluster setting"
+            + " for queue " + queuePath
+            + " max allocation per queue: %s"
+            + " cluster setting: " + clusterMax;
+
+    if (queueMax == Resources.none()) {
+      // Handle backward compatibility
+      long queueMemory = configuration.getQueueMaximumAllocationMb(queuePath);
+      int queueVcores = configuration.getQueueMaximumAllocationVcores(queuePath);
+      if (queueMemory != UNDEFINED) {
+        maximumAllocation.setMemorySize(queueMemory);
+      }
+
+      if (queueVcores != UNDEFINED) {
+        maximumAllocation.setVirtualCores(queueVcores);
+      }
+
+      if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize()
+          || (queueVcores != UNDEFINED
+          && queueVcores > clusterMax.getVirtualCores()))) {
+        throw new IllegalArgumentException(
+            String.format(errMsg, maximumAllocation));
+      }
+    } else {
+      // Queue level maximum-allocation can't be larger than cluster setting
+      for (ResourceInformation ri : queueMax.getResources()) {
+        if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) {
+          throw new IllegalArgumentException(String.format(errMsg, queueMax));
+        }
+
+        maximumAllocation.setResourceInformation(ri.getName(), ri);
+      }
+    }
+  }
+
+  public Resource getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  public Resource getMaximumAllocation() {
+    return maximumAllocation;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
new file mode 100644
index 0000000000000..2e71d8eef3edb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * This class determines application lifetime and max parallel apps settings based on the
+ * {@link CapacitySchedulerConfiguration} and other queue
+ * properties.
+ **/
+public class QueueAppLifetimeAndLimitSettings {
+  // -1 indicates lifetime is disabled
+  private final long maxApplicationLifetime;
+  private final long defaultApplicationLifetime;
+
+  // Indicates if this queue's default lifetime was set by a config property,
+  // either at this level or anywhere in the queue's hierarchy.
+  private boolean defaultAppLifetimeWasSpecifiedInConfig = false;
+
+  private int maxParallelApps;
+
+  public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
+      AbstractCSQueue q, QueuePath queuePath) {
+    // Store max parallel apps property
+    this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
+    this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
+    this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
+        maxApplicationLifetime);
+  }
+
+  private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) {
+    CSQueue parentQ = q.getParent();
+    long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
+
+    // If q is the root queue, then get max app lifetime from conf.
+    if (q.getQueuePathObject().isRoot()) {
+      return maxAppLifetime;
+    }
+
+    // If this is not the root queue, get this queue's max app lifetime
+    // from the conf. The parent's max app lifetime will be used if it's
+    // not set for this queue.
+    // A value of 0 will override the parent's value and means no max lifetime.
+    // A negative value means that the parent's max should be used.
+    long parentsMaxAppLifetime = parentQ.getMaximumApplicationLifetime();
+    return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime;
+  }
+
+  private long setupInheritedDefaultAppLifetime(CSQueue q,
+      QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
+    CSQueue parentQ = q.getParent();
+    long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
+    defaultAppLifetimeWasSpecifiedInConfig =
+        (defaultAppLifetime >= 0
+            || (!queuePath.isRoot() &&
+            parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
+
+    // If q is the root queue, then get default app lifetime from conf.
+    if (queuePath.isRoot()) {
+      return defaultAppLifetime;
+    }
+
+    // If this is not the root queue, get the parent's default app lifetime. The
+    // parent's default app lifetime will be used if not set for this queue.
+    long parentsDefaultAppLifetime = parentQ.getDefaultApplicationLifetime();
+
+    // Negative value indicates default lifetime was not set at this level.
+    // If default lifetime was not set at this level, calculate it based on
+    // parent's default lifetime or current queue's max lifetime.
+    if (defaultAppLifetime < 0) {
+      // If default lifetime was not set at this level but was set somewhere in
+      // the parent's hierarchy, set default lifetime to parent queue's default
+      // only if parent queue's lifetime is less than current queue's max
+      // lifetime. Otherwise, use current queue's max lifetime value for its
+      // default lifetime.
+      if (defaultAppLifetimeWasSpecifiedInConfig) {
+        defaultAppLifetime =
+            Math.min(parentsDefaultAppLifetime, myMaxAppLifetime);
+      } else {
+        // Default app lifetime value was not set anywhere in this queue's
+        // hierarchy. Use current queue's max lifetime as its default.
+        defaultAppLifetime = myMaxAppLifetime;
+      }
+    } // else if >= 0, default lifetime was set at this level. Just use it.
+
+    if (myMaxAppLifetime > 0 && defaultAppLifetime > myMaxAppLifetime) {
+      throw new YarnRuntimeException(
+          "Default lifetime " + defaultAppLifetime
+              + " can't exceed maximum lifetime " + myMaxAppLifetime);
+    }
+
+    if (defaultAppLifetime <= 0) {
+      defaultAppLifetime = myMaxAppLifetime;
+    }
+    return defaultAppLifetime;
+  }
+
+  public int getMaxParallelApps() {
+    return maxParallelApps;
+  }
+
+  public void setMaxParallelApps(int maxParallelApps) {
+    this.maxParallelApps = maxParallelApps;
+  }
+
+  public long getMaxApplicationLifetime() {
+    return maxApplicationLifetime;
+  }
+
+  public long getDefaultApplicationLifetime() {
+    return defaultApplicationLifetime;
+  }
+
+  public boolean isDefaultAppLifetimeWasSpecifiedInConfig() {
+    return defaultAppLifetimeWasSpecifiedInConfig;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 2d829663eb44f..87d4094f89384 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -312,7 +312,7 @@ public void clearConfigurableFields() {
         _set(label, CapacityType.MAX_CAP, 0);
         _set(label, CapacityType.ABS_CAP, 0);
         _set(label, CapacityType.ABS_MAX_CAP, 0);
-        _set(label, CapacityType.WEIGHT, 0);
+        _set(label, CapacityType.WEIGHT, -1);
       }
     } finally {
       writeLock.unlock();
@@ -337,13 +337,4 @@ public String toString() {
       readLock.unlock();
     }
   }
-  
-  public Set<String> getNodePartitionsSet() {
-    readLock.lock();
-    try {
-      return capacitiesMap.keySet();
-    } finally {
-      readLock.unlock();
-    }
-  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java
new file mode 100644
index 0000000000000..9f6e0e264a320
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Contains capacity values with calculation types associated for each
+ * resource.
+ */
+public class QueueCapacityVector implements
+    Iterable<QueueCapacityVector.QueueCapacityVectorEntry> {
+  private static final String START_PARENTHESES = "[";
+  private static final String END_PARENTHESES = "]";
+  private static final String RESOURCE_DELIMITER = ",";
+  private static final String VALUE_DELIMITER = "=";
+
+  private final ResourceVector resource;
+  private final Map<String, QueueCapacityType> capacityTypes
+      = new HashMap<>();
+  private final Map<QueueCapacityType, Set<String>> capacityTypePerResource
+      = new HashMap<>();
+
+  public QueueCapacityVector() {
+    this.resource = new ResourceVector();
+  }
+
+  private QueueCapacityVector(ResourceVector resource) {
+    this.resource = resource;
+  }
+
+  /**
+   * Creates a zero {@code QueueCapacityVector}. The resources are defined
+   * in absolute capacity type by default.
+   *
+   * @return zero capacity vector
+   */
+  public static QueueCapacityVector newInstance() {
+    QueueCapacityVector newCapacityVector =
+        new QueueCapacityVector(ResourceVector.newInstance());
+    for (Map.Entry<String, Float> resourceEntry : newCapacityVector.resource) {
+      newCapacityVector.storeResourceType(resourceEntry.getKey(),
+          QueueCapacityType.ABSOLUTE);
+    }
+
+    return newCapacityVector;
+  }
+
+  /**
+   * Creates a uniform and homogeneous {@code QueueCapacityVector}.
+   * The resources are defined in absolute capacity type by default.
+   *
+   * @param value value to be set for each resource
+   * @param capacityType capacity type to be set for each resource
+   * @return uniform capacity vector
+   */
+  public static QueueCapacityVector of(
+      float value, QueueCapacityType capacityType) {
+    QueueCapacityVector newCapacityVector =
+        new QueueCapacityVector(ResourceVector.of(value));
+    for (Map.Entry<String, Float> resourceEntry : newCapacityVector.resource) {
+      newCapacityVector.storeResourceType(resourceEntry.getKey(), capacityType);
+    }
+
+    return newCapacityVector;
+  }
+
+  public QueueCapacityVectorEntry getResource(String resourceName) {
+    return new QueueCapacityVectorEntry(capacityTypes.get(resourceName),
+        resourceName, resource.getValue(resourceName));
+  }
+
+  /**
+   * Returns the number of resources defined for this vector.
+   *
+   * @return number of resources
+   */
+  public int getResourceCount() {
+    return capacityTypes.size();
+  }
+
+  /**
+   * Set the value and capacity type of a resource.
+   *
+   * @param resourceName name of the resource
+   * @param value        value of the resource
+   * @param capacityType type of the resource
+   */
+  public void setResource(String resourceName, float value,
+                          QueueCapacityType capacityType) {
+    // Necessary due to backward compatibility (memory = memory-mb)
+    String convertedResourceName = resourceName;
+    if (resourceName.equals("memory")) {
+      convertedResourceName = ResourceInformation.MEMORY_URI;
+    }
+    resource.setValue(convertedResourceName, value);
+    storeResourceType(convertedResourceName, capacityType);
+  }
+
+  /**
+   * A shorthand to retrieve the value stored for the memory resource.
+   *
+   * @return value of memory resource
+   */
+  public float getMemory() {
+    return resource.getValue(ResourceInformation.MEMORY_URI);
+  }
+
+  /**
+   * Returns the name of all resources that are defined in the given capacity
+   * type.
+   *
+   * @param capacityType the capacity type of the resources
+   * @return all resource names for the given capacity type
+   */
+  public Set<String> getResourceNamesByCapacityType(
+      QueueCapacityType capacityType) {
+    return capacityTypePerResource.getOrDefault(capacityType,
+        Collections.emptySet());
+  }
+
+  public boolean isResourceOfType(
+      String resourceName, QueueCapacityType capacityType) {
+    return capacityTypes.containsKey(resourceName) &&
+        capacityTypes.get(resourceName).equals(capacityType);
+  }
+
+  @Override
+  public Iterator<QueueCapacityVectorEntry> iterator() {
+    return new Iterator<QueueCapacityVectorEntry>() {
+      private final Iterator<Map.Entry<String, Float>> resources =
+          resource.iterator();
+      private int i = 0;
+
+      @Override
+      public boolean hasNext() {
+        return resources.hasNext() && capacityTypes.size() > i;
+      }
+
+      @Override
+      public QueueCapacityVectorEntry next() {
+        Map.Entry<String, Float> resourceInformation = resources.next();
+        i++;
+        return new QueueCapacityVectorEntry(
+            capacityTypes.get(resourceInformation.getKey()),
+            resourceInformation.getKey(), resourceInformation.getValue());
+      }
+    };
+  }
+
+  /**
+   * Returns a set of all capacity type defined for this vector.
+   *
+   * @return capacity types
+   */
+  public Set<QueueCapacityType> getDefinedCapacityTypes() {
+    return capacityTypePerResource.keySet();
+  }
+
+  private void storeResourceType(
+      String resourceName, QueueCapacityType resourceType) {
+    if (capacityTypes.get(resourceName) != null
+        && !capacityTypes.get(resourceName).equals(resourceType)) {
+      capacityTypePerResource.get(capacityTypes.get(resourceName))
+          .remove(resourceName);
+    }
+
+    capacityTypePerResource.putIfAbsent(resourceType, new HashSet<>());
+    capacityTypePerResource.get(resourceType).add(resourceName);
+    capacityTypes.put(resourceName, resourceType);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder stringVector = new StringBuilder();
+    stringVector.append(START_PARENTHESES);
+
+    int resourceCount = 0;
+    for (Map.Entry<String, Float> resourceEntry : resource) {
+      resourceCount++;
+      stringVector.append(resourceEntry.getKey())
+          .append(VALUE_DELIMITER)
+          .append(resourceEntry.getValue())
+          .append(capacityTypes.get(resourceEntry.getKey()).postfix);
+      if (resourceCount < capacityTypes.size()) {
+        stringVector.append(RESOURCE_DELIMITER);
+      }
+    }
+
+    stringVector.append(END_PARENTHESES);
+
+    return stringVector.toString();
+  }
+
+  /**
+   * Represents a capacity type associated with its syntax postfix.
+   */
+  public enum QueueCapacityType {
+    PERCENTAGE("%"), ABSOLUTE(""), WEIGHT("w");
+    private final String postfix;
+
+    QueueCapacityType(String postfix) {
+      this.postfix = postfix;
+    }
+
+    public String getPostfix() {
+      return postfix;
+    }
+  }
+
+  public static class QueueCapacityVectorEntry {
+    private final QueueCapacityType vectorResourceType;
+    private final float resourceValue;
+    private final String resourceName;
+
+    public QueueCapacityVectorEntry(QueueCapacityType vectorResourceType,
+                                    String resourceName, float resourceValue) {
+      this.vectorResourceType = vectorResourceType;
+      this.resourceValue = resourceValue;
+      this.resourceName = resourceName;
+    }
+
+    public QueueCapacityType getVectorResourceType() {
+      return vectorResourceType;
+    }
+
+    public float getResourceValue() {
+      return resourceValue;
+    }
+
+    public String getResourceName() {
+      return resourceName;
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
index 5ddb6a2ee42b8..593cc874f6d64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
@@ -37,7 +37,6 @@
 import org.apache.hadoop.yarn.util.SystemClock;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 
-import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
new file mode 100644
index 0000000000000..c431d2bb45543
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * This class determines accessible node labels, configured node labels and the default node
+ * label expression based on the {@link CapacitySchedulerConfiguration} object and other queue
+ * properties.
+ */
+public class QueueNodeLabelsSettings {
+  private final CSQueue parent;
+  private final QueuePath queuePath;
+  private Set<String> accessibleLabels;
+  private Set<String> configuredNodeLabels;
+  private String defaultLabelExpression;
+
+  public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
+      CSQueue parent,
+      QueuePath queuePath,
+      ConfiguredNodeLabels configuredNodeLabels) throws IOException {
+    this.parent = parent;
+    this.queuePath = queuePath;
+    initializeNodeLabels(configuration, configuredNodeLabels);
+  }
+
+  private void initializeNodeLabels(CapacitySchedulerConfiguration configuration,
+      ConfiguredNodeLabels configuredNodeLabels)
+      throws IOException {
+    initializeAccessibleLabels(configuration);
+    initializeDefaultLabelExpression(configuration);
+    initializeConfiguredNodeLabels(configuration, configuredNodeLabels);
+    validateNodeLabels();
+  }
+
+  private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
+    this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
+    // Inherit labels from parent if not set
+    if (this.accessibleLabels == null && parent != null) {
+      this.accessibleLabels = parent.getAccessibleNodeLabels();
+    }
+  }
+
+  private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
+    this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
+        queuePath.getFullPath());
+    // If the accessible labels is not null and the queue has a parent with a
+    // similar set of labels copy the defaultNodeLabelExpression from the parent
+    if (this.accessibleLabels != null && parent != null
+        && this.defaultLabelExpression == null &&
+        this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
+      this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
+    }
+  }
+
+  private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
+      ConfiguredNodeLabels configuredNodeLabelsParam) {
+    if (configuredNodeLabelsParam != null) {
+      if (queuePath.isRoot()) {
+        this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
+      } else {
+        this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(
+            queuePath.getFullPath());
+      }
+    } else {
+      // Fallback to suboptimal but correct logic
+      this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
+    }
+  }
+
+  private void validateNodeLabels() throws IOException {
+    // Check if labels of this queue is a subset of parent queue, only do this
+    // when the queue in question is not root
+    if (!queuePath.isRoot()) {
+      if (parent.getAccessibleNodeLabels() != null && !parent
+          .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
+        // If parent isn't "*", child shouldn't be "*" too
+        if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
+          throw new IOException("Parent's accessible queue is not ANY(*), "
+              + "but child's accessible queue is " + RMNodeLabelsManager.ANY);
+        } else {
+          Set<String> diff = Sets.difference(this.getAccessibleNodeLabels(),
+              parent.getAccessibleNodeLabels());
+          if (!diff.isEmpty()) {
+            throw new IOException(String.format(
+                "Some labels of child queue is not a subset of parent queue, these labels=[%s]",
+                StringUtils.join(diff, ",")));
+          }
+        }
+      }
+    }
+  }
+
+  public boolean isAccessibleToPartition(String nodePartition) {
+    // if queue's label is *, it can access any node
+    if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
+      return true;
+    }
+    // any queue can access to a node without label
+    if (nodePartition == null || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      return true;
+    }
+    // a queue can access to a node only if it contains any label of the node
+    if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) {
+      return true;
+    }
+    // The partition cannot be accessed
+    return false;
+  }
+
+  public Set<String> getAccessibleNodeLabels() {
+    return accessibleLabels;
+  }
+
+  public Set<String> getConfiguredNodeLabels() {
+    return configuredNodeLabels;
+  }
+
+  public String getDefaultLabelExpression() {
+    return defaultLabelExpression;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
index 3ca7735918576..440742b908927 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
@@ -24,6 +24,7 @@
 import java.util.Objects;
 
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
 
 /**
  * This is a helper class which represents a queue path, and has easy access
@@ -59,6 +60,16 @@ public QueuePath(String fullPath) {
     setFromFullPath(fullPath);
   }
 
+  /**
+   * Constructor to create Queue path from queue names.
+   * The provided queue names will be concatenated by dots, giving a full queue path.
+   * @param parts Parts of queue path
+   * @return QueuePath object
+   */
+  public static QueuePath createFromQueues(String... parts) {
+    return new QueuePath(String.join(DOT, parts));
+  }
+
   /**
    * This method is responsible for splitting up a full queue path into parent
    * path and leaf name.
@@ -68,6 +79,11 @@ private void setFromFullPath(String fullPath) {
     parent = null;
     leaf = fullPath;
 
+    if (leaf == null) {
+      leaf = "";
+      return;
+    }
+
     int lastDotIdx = fullPath.lastIndexOf(DOT);
     if (lastDotIdx > -1) {
       parent = fullPath.substring(0, lastDotIdx).trim();
@@ -121,6 +137,14 @@ public boolean hasParent() {
     return parent != null;
   }
 
+  /**
+   * Convenience getter to check if the queue is the root queue.
+   * @return True if the path is root
+   */
+  public boolean isRoot() {
+    return !hasParent() && leaf.equals(ROOT);
+  }
+
   /**
    * Creates a new {@code QueuePath} from the current full path as parent, and
    * the appended child queue path as leaf.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java
new file mode 100644
index 0000000000000..5ec7d01bce9b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.util.Set;
+
+/**
+ * Collects all logic that are handling queue state transitions.
+ */
+public final class QueueStateHelper {
+  private static final Set<QueueState> VALID_STATE_CONFIGURATIONS = ImmutableSet.of(
+      QueueState.RUNNING, QueueState.STOPPED);
+  private static final QueueState DEFAULT_STATE = QueueState.RUNNING;
+
+  private QueueStateHelper() {}
+
+  /**
+   * Sets the current state of the queue based on its previous state, its parent's state and its
+   * configured state.
+   * @param queue the queue whose state is set
+   */
+  public static void setQueueState(AbstractCSQueue queue) {
+    QueueState previousState = queue.getState();
+    QueueState configuredState = queue.getQueueContext().getConfiguration().getConfiguredState(
+        queue.getQueuePath());
+    QueueState parentState = (queue.getParent() == null) ? null : queue.getParent().getState();
+
+    // verify that we can not any value for State other than RUNNING/STOPPED
+    if (configuredState != null && !VALID_STATE_CONFIGURATIONS.contains(configuredState)) {
+      throw new IllegalArgumentException("Invalid queue state configuration."
+          + " We can only use RUNNING or STOPPED.");
+    }
+
+    if (previousState == null) {
+      initializeState(queue, configuredState, parentState);
+    } else {
+      reinitializeState(queue, previousState, configuredState);
+    }
+  }
+
+  private static void reinitializeState(
+      AbstractCSQueue queue, QueueState previousState, QueueState configuredState) {
+    // when we get a refreshQueue request from AdminService,
+    if (previousState == QueueState.RUNNING) {
+      if (configuredState == QueueState.STOPPED) {
+        queue.stopQueue();
+      }
+    } else {
+      if (configuredState == QueueState.RUNNING) {
+        try {
+          queue.activateQueue();
+        } catch (YarnException ex) {
+          throw new IllegalArgumentException(ex.getMessage());
+        }
+      }
+    }
+  }
+
+  private static void initializeState(
+      AbstractCSQueue queue, QueueState configuredState, QueueState parentState) {
+    QueueState currentState = configuredState == null ? DEFAULT_STATE : configuredState;
+
+    if (parentState != null) {
+      if (configuredState == QueueState.RUNNING && parentState != QueueState.RUNNING) {
+        throw new IllegalArgumentException(
+            "The parent queue:" + queue.getParent().getQueuePath()
+                + " cannot be STOPPED as the child queue:" + queue.getQueuePath()
+                + " is in RUNNING state.");
+      }
+
+      if (configuredState == null) {
+        currentState = parentState == QueueState.DRAINING ? QueueState.STOPPED : parentState;
+      }
+    }
+
+    queue.updateQueueState(currentState);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index 5a8ce9ac487b2..37cf425a83f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -22,6 +22,8 @@
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,15 +33,16 @@
  *
  */
 public class ReservationQueue extends AbstractAutoCreatedLeafQueue {
-
-  private static final Logger LOG = LoggerFactory
-      .getLogger(ReservationQueue.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ReservationQueue.class);
 
   private PlanQueue parent;
 
-  public ReservationQueue(CapacitySchedulerContext cs, String queueName,
+  public ReservationQueue(CapacitySchedulerQueueContext queueContext, String queueName,
       PlanQueue parent) throws IOException {
-    super(cs, queueName, parent, null);
+    super(queueContext, queueName, parent, null);
+    super.setupQueueConfigs(queueContext.getClusterResource());
+
     // the following parameters are common to all reservation in the plan
     updateQuotas(parent.getUserLimitForReservation(),
         parent.getUserLimitFactor(),
@@ -73,6 +76,10 @@ public void reinitialize(CSQueue newlyParsedQueue,
     }
   }
 
+  public void initializeEntitlements() throws SchedulerDynamicEditException {
+    setEntitlement(new QueueEntitlement(1.0f, 1.0f));
+  }
+
   private void updateQuotas(float userLimit, float userLimitFactor,
       int maxAppsForReservation, int maxAppsPerUserForReservation) {
     setUserLimit(userLimit);
@@ -82,8 +89,7 @@ private void updateQuotas(float userLimit, float userLimitFactor,
   }
 
   @Override
-  protected void setupConfigurableCapacities(CapacitySchedulerConfiguration
-      configuration) {
+  protected void setupConfigurableCapacities() {
     super.updateAbsoluteCapacities();
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java
new file mode 100644
index 0000000000000..88c09af6b0911
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Represents a simple resource floating point value storage
+ * grouped by resource names.
+ */
+public class ResourceVector implements Iterable<Map.Entry<String, Float>> {
+  private final Map<String, Float> resourcesByName = new HashMap<>();
+
+  /**
+   * Creates a new {@code ResourceVector} with all pre-defined resources set to
+   * zero.
+   * @return zero resource vector
+   */
+  public static ResourceVector newInstance() {
+    ResourceVector zeroResourceVector = new ResourceVector();
+    for (ResourceInformation resource : ResourceUtils.getResourceTypesArray()) {
+      zeroResourceVector.setValue(resource.getName(), 0);
+    }
+
+    return zeroResourceVector;
+  }
+
+  /**
+   * Creates a new {@code ResourceVector} with all pre-defined resources set to
+   * the same value.
+   * @param value the value to set all resources to
+   * @return uniform resource vector
+   */
+  public static ResourceVector of(float value) {
+    ResourceVector emptyResourceVector = new ResourceVector();
+    for (ResourceInformation resource : ResourceUtils.getResourceTypesArray()) {
+      emptyResourceVector.setValue(resource.getName(), value);
+    }
+
+    return emptyResourceVector;
+  }
+
+  /**
+   * Creates a new {@code ResourceVector} with the values set in a
+   * {@code Resource} object.
+   * @param resource resource object the resource vector will be based on
+   * @return uniform resource vector
+   */
+  public static ResourceVector of(Resource resource) {
+    ResourceVector resourceVector = new ResourceVector();
+    for (ResourceInformation resourceInformation : resource.getResources()) {
+      resourceVector.setValue(resourceInformation.getName(),
+          resourceInformation.getValue());
+    }
+
+    return resourceVector;
+  }
+
+  /**
+   * Subtract values for each resource defined in the given resource vector.
+   * @param otherResourceVector rhs resource vector of the subtraction
+   */
+  public void subtract(ResourceVector otherResourceVector) {
+    for (Map.Entry<String, Float> resource : otherResourceVector) {
+      setValue(resource.getKey(), getValue(resource.getKey()) - resource.getValue());
+    }
+  }
+
+  /**
+   * Increments the given resource by the specified value.
+   * @param resourceName name of the resource
+   * @param value value to be added to the resource's current value
+   */
+  public void increment(String resourceName, float value) {
+    setValue(resourceName, getValue(resourceName) + value);
+  }
+
+  public Float getValue(String resourceName) {
+    return resourcesByName.get(resourceName);
+  }
+
+  public void setValue(String resourceName, float value) {
+    resourcesByName.put(resourceName, value);
+  }
+
+  @Override
+  public Iterator<Map.Entry<String, Float>> iterator() {
+    return resourcesByName.entrySet().iterator();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    return this.resourcesByName.equals(((ResourceVector) o).resourcesByName);
+  }
+
+  @Override
+  public int hashCode() {
+    return resourcesByName.hashCode();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
index 7bdc1bc8045aa..a9d755227ffca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
@@ -54,7 +54,7 @@ public static UserWeights createByConfig(
       if (m.find()) {
         String userName = item.getKey().replaceFirst("\\." + USER_WEIGHT, "");
         if (!userName.isEmpty()) {
-          String value = conf.substituteVars(item.getValue());
+          String value = conf.substituteCommonVariables(item.getValue());
           userWeights.data.put(userName, new Float(value));
         }
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 8ba13f029f8e9..479bfec6d9440 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -55,10 +55,9 @@ public class UsersManager implements AbstractUsersManager {
   /*
    * Member declaration for UsersManager class.
    */
-  private final LeafQueue lQueue;
+  private final AbstractLeafQueue lQueue;
   private final RMNodeLabelsManager labelManager;
   private final ResourceCalculator resourceCalculator;
-  private final CapacitySchedulerContext scheduler;
   private Map<String, User> users = new ConcurrentHashMap<>();
 
   private ResourceUsage totalResUsageForActiveUsers = new ResourceUsage();
@@ -296,17 +295,13 @@ public void setWeight(float weight) {
    *          Leaf Queue Object
    * @param labelManager
    *          Label Manager instance
-   * @param scheduler
-   *          Capacity Scheduler Context
    * @param resourceCalculator
    *          rc
    */
-  public UsersManager(QueueMetrics metrics, LeafQueue lQueue,
-      RMNodeLabelsManager labelManager, CapacitySchedulerContext scheduler,
-      ResourceCalculator resourceCalculator) {
+  public UsersManager(QueueMetrics metrics, AbstractLeafQueue lQueue,
+      RMNodeLabelsManager labelManager, ResourceCalculator resourceCalculator) {
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     this.lQueue = lQueue;
-    this.scheduler = scheduler;
     this.labelManager = labelManager;
     this.resourceCalculator = resourceCalculator;
     this.qUsageRatios = new UsageRatios();
@@ -817,6 +812,7 @@ partitionResource, getUsageRatio(nodePartition),
             lQueue.getMinimumAllocation());
 
     if (LOG.isDebugEnabled()) {
+      float weight = lQueue.getUserWeights().getByUser(userName);
       LOG.debug("User limit computation for " + userName
           + ",  in queue: " + lQueue.getQueuePath()
           + ",  userLimitPercent=" + lQueue.getUserLimit()
@@ -834,7 +830,7 @@ partitionResource, getUsageRatio(nodePartition),
           + ",  Partition=" + nodePartition
           + ",  resourceUsed=" + resourceUsed
           + ",  maxUserLimit=" + maxUserLimit
-          + ",  userWeight=" + getUser(userName).getWeight()
+          + ",  userWeight=" + weight
       );
     }
     return userLimitResource;
@@ -843,10 +839,8 @@ partitionResource, getUsageRatio(nodePartition),
   /**
    * Update new usage ratio.
    *
-   * @param partition
-   *          Node partition
-   * @param clusterResource
-   *          Cluster Resource
+   * @param partition Node partition
+   * @param clusterResource cluster resource
    */
   public void updateUsageRatio(String partition, Resource clusterResource) {
     writeLock.lock();
@@ -978,7 +972,7 @@ private void updateActiveUsersResourceUsage(String userName) {
 
         // Update total resource usage of active and non-active after user
         // is moved from non-active to active.
-        for (String partition : resourceUsage.getNodePartitionsSet()) {
+        for (String partition : resourceUsage.getExistingNodeLabels()) {
           totalResUsageForNonActiveUsers.decUsed(partition,
               resourceUsage.getUsed(partition));
           totalResUsageForActiveUsers.incUsed(partition,
@@ -1019,7 +1013,7 @@ private void updateNonActiveUsersResourceUsage(String userName) {
 
         // Update total resource usage of active and non-active after user is
         // moved from active to non-active.
-        for (String partition : resourceUsage.getNodePartitionsSet()) {
+        for (String partition : resourceUsage.getExistingNodeLabels()) {
           totalResUsageForActiveUsers.decUsed(partition,
               resourceUsage.getUsed(partition));
           totalResUsageForNonActiveUsers.incUsed(partition,
@@ -1063,6 +1057,8 @@ private ResourceUsage getTotalResourceUsagePerUser(String userName) {
    *          Name of the user
    * @param resource
    *          Resource to increment/decrement
+   * @param clusterResource
+   *          Cluster resource (for testing purposes only)
    * @param nodePartition
    *          Node label
    * @param isAllocate
@@ -1070,6 +1066,7 @@ private ResourceUsage getTotalResourceUsagePerUser(String userName) {
    * @return user
    */
   public User updateUserResourceUsage(String userName, Resource resource,
+      Resource clusterResource,
       String nodePartition, boolean isAllocate) {
     this.writeLock.lock();
     try {
@@ -1085,7 +1082,7 @@ public User updateUserResourceUsage(String userName, Resource resource,
 
       // Update usage ratios
       Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
-          scheduler.getClusterResource());
+          clusterResource);
       incQueueUsageRatio(nodePartition, user.updateUsageRatio(
           resourceCalculator, resourceByLabel, nodePartition));
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
index b9b9bcff2f607..94bd4cb3c96d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocation.java
@@ -18,12 +18,9 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
 
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.List;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index b396d5761de16..c46b0341f74d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -60,6 +60,8 @@
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.AM_ALLOW_NON_EXCLUSIVE_ALLOCATION;
+
 /**
  * Allocate normal (new) containers, considers locality/label, etc. Using
  * delayed scheduling mechanism to get better locality allocation.
@@ -80,7 +82,7 @@ private boolean checkHeadroom(ResourceLimits currentResourceLimits,
     // require
     Resource resourceCouldBeUnReserved =
         application.getAppAttemptResourceUsage().getReserved(nodePartition);
-    if (!application.getCSLeafQueue().getReservationContinueLooking()) {
+    if (!application.getCSLeafQueue().isReservationsContinueLooking()) {
       // If we don't allow reservation continuous looking,
       // we won't allow to unreserve before allocation.
       resourceCouldBeUnReserved = Resources.none();
@@ -121,10 +123,9 @@ private ContainerAllocation preCheckForNodeCandidateSet(FiCaSchedulerNode node,
       return ContainerAllocation.PRIORITY_SKIPPED;
     }
 
-    // AM container allocation doesn't support non-exclusive allocation to
-    // avoid painful of preempt an AM container
     if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
-      if (application.isWaitingForAMContainer()) {
+      if (application.isWaitingForAMContainer() && !rmContext.getYarnConfiguration()
+          .getBoolean(AM_ALLOW_NON_EXCLUSIVE_ALLOCATION, false)) {
         LOG.debug("Skip allocating AM container to app_attempt={},"
             + " don't allow to allocate AM container in non-exclusive mode",
             application.getApplicationAttemptId());
@@ -154,7 +155,7 @@ private ContainerAllocation preCheckForNodeCandidateSet(FiCaSchedulerNode node,
       return ContainerAllocation.PRIORITY_SKIPPED;
     }
 
-    if (!application.getCSLeafQueue().getReservationContinueLooking()) {
+    if (!application.getCSLeafQueue().isReservationsContinueLooking()) {
       if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) {
         LOG.debug("doesn't need containers based on reservation algo!");
         ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
@@ -551,7 +552,7 @@ private ContainerAllocation assignContainer(Resource clusterResource,
 
     RMContainer unreservedContainer = null;
     boolean reservationsContinueLooking =
-        application.getCSLeafQueue().getReservationContinueLooking();
+        application.getCSLeafQueue().isReservationsContinueLooking();
 
     // Check if we need to kill some containers to allocate this one
     List<RMContainer> toKillContainers = null;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
new file mode 100644
index 0000000000000..88c93019680cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
+
+public final class ConfigurationUpdateAssembler {
+
+  private ConfigurationUpdateAssembler() {
+  }
+
+  public static Map<String, String> constructKeyValueConfUpdate(
+          CapacitySchedulerConfiguration proposedConf,
+          SchedConfUpdateInfo mutationInfo) throws IOException {
+
+    Map<String, String> confUpdate = new HashMap<>();
+    for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+      removeQueue(queueToRemove, proposedConf, confUpdate);
+    }
+    for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+      addQueue(addQueueInfo, proposedConf, confUpdate);
+    }
+    for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+      updateQueue(updateQueueInfo, proposedConf, confUpdate);
+    }
+    for (Map.Entry<String, String> global : mutationInfo.getGlobalParams()
+            .entrySet()) {
+      confUpdate.put(global.getKey(), global.getValue());
+    }
+    return confUpdate;
+  }
+
+  private static void removeQueue(
+          String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+          Map<String, String> confUpdate) throws IOException {
+    if (queueToRemove == null) {
+      return;
+    }
+    if (queueToRemove.lastIndexOf('.') == -1) {
+      throw new IOException("Can't remove queue " + queueToRemove);
+    }
+    String queueName = queueToRemove.substring(
+            queueToRemove.lastIndexOf('.') + 1);
+    List<String> siblingQueues = getSiblingQueues(queueToRemove,
+            proposedConf);
+    if (!siblingQueues.contains(queueName)) {
+      throw new IOException("Queue " + queueToRemove + " not found");
+    }
+    siblingQueues.remove(queueName);
+    String parentQueuePath = queueToRemove.substring(0, queueToRemove
+            .lastIndexOf('.'));
+    proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(
+            new String[0]));
+    String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+            + parentQueuePath + CapacitySchedulerConfiguration.DOT
+            + CapacitySchedulerConfiguration.QUEUES;
+    if (siblingQueues.isEmpty()) {
+      confUpdate.put(queuesConfig, null);
+      // Unset Ordering Policy of Leaf Queue converted from
+      // Parent Queue after removeQueue
+      String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+              + parentQueuePath + CapacitySchedulerConfiguration.DOT
+              + ORDERING_POLICY;
+      proposedConf.unset(queueOrderingPolicy);
+      confUpdate.put(queueOrderingPolicy, null);
+    } else {
+      confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
+    }
+    for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+                    ".*" + queueToRemove + "\\..*")
+            .entrySet()) {
+      proposedConf.unset(confRemove.getKey());
+      confUpdate.put(confRemove.getKey(), null);
+    }
+  }
+
+  private static void addQueue(
+          QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+          Map<String, String> confUpdate) throws IOException {
+    if (addInfo == null) {
+      return;
+    }
+    String queuePath = addInfo.getQueue();
+    String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+    if (queuePath.lastIndexOf('.') == -1) {
+      throw new IOException("Can't add invalid queue " + queuePath);
+    } else if (getSiblingQueues(queuePath, proposedConf).contains(
+            queueName)) {
+      throw new IOException("Can't add existing queue " + queuePath);
+    }
+    String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+    String[] siblings = proposedConf.getQueues(parentQueue);
+    List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+            new ArrayList<>(Arrays.asList(siblings));
+    siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+    proposedConf.setQueues(parentQueue,
+            siblingQueues.toArray(new String[0]));
+    confUpdate.put(CapacitySchedulerConfiguration.PREFIX
+                    + parentQueue + CapacitySchedulerConfiguration.DOT
+                    + CapacitySchedulerConfiguration.QUEUES,
+            Joiner.on(',').join(siblingQueues));
+    String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+            + queuePath + CapacitySchedulerConfiguration.DOT;
+    for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+      String keyValue = kv.getValue();
+      if (keyValue == null || keyValue.isEmpty()) {
+        proposedConf.unset(keyPrefix + kv.getKey());
+        confUpdate.put(keyPrefix + kv.getKey(), null);
+      } else {
+        proposedConf.set(keyPrefix + kv.getKey(), keyValue);
+        confUpdate.put(keyPrefix + kv.getKey(), keyValue);
+      }
+    }
+    // Unset Ordering Policy of Parent Queue converted from
+    // Leaf Queue after addQueue
+    String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+            + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+    if (siblingQueues.size() == 1) {
+      proposedConf.unset(queueOrderingPolicy);
+      confUpdate.put(queueOrderingPolicy, null);
+    }
+  }
+
+  private static void updateQueue(QueueConfigInfo updateInfo,
+                                  CapacitySchedulerConfiguration proposedConf,
+                                  Map<String, String> confUpdate) {
+    if (updateInfo == null) {
+      return;
+    }
+    String queuePath = updateInfo.getQueue();
+    String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+            + queuePath + CapacitySchedulerConfiguration.DOT;
+    for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+      String keyValue = kv.getValue();
+      if (keyValue == null || keyValue.isEmpty()) {
+        proposedConf.unset(keyPrefix + kv.getKey());
+        confUpdate.put(keyPrefix + kv.getKey(), null);
+      } else {
+        proposedConf.set(keyPrefix + kv.getKey(), keyValue);
+        confUpdate.put(keyPrefix + kv.getKey(), keyValue);
+      }
+    }
+  }
+
+  private static List<String> getSiblingQueues(String queuePath, Configuration conf) {
+    String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+    String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
+            parentQueue + CapacitySchedulerConfiguration.DOT +
+            CapacitySchedulerConfiguration.QUEUES;
+    return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index a351799872fe6..6aa37f399e41a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -204,8 +204,8 @@ public void logMutation(LogMutation logMutation) throws IOException {
   @Override
   public void confirmMutation(LogMutation pendingMutation,
       boolean isValid) {
-    WriteBatch updateBatch = db.createWriteBatch();
     if (isValid) {
+      WriteBatch updateBatch = db.createWriteBatch();
       for (Map.Entry<String, String> changes :
           pendingMutation.getUpdates().entrySet()) {
         if (changes.getValue() == null || changes.getValue().isEmpty()) {
@@ -215,8 +215,8 @@ public void confirmMutation(LogMutation pendingMutation,
         }
       }
       increaseConfigVersion();
+      db.write(updateBatch);
     }
-    db.write(updateBatch);
   }
 
   private byte[] serLogMutations(LinkedList<LogMutation> mutations) throws
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index c4cb273b49513..cbd217f4f2b39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -31,19 +30,12 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
-import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
 import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
-
 /**
  * CS configuration provider which implements
  * {@link MutableConfigurationProvider} for modifying capacity scheduler
@@ -79,15 +71,7 @@ protected Configuration getInitSchedulerConfig() {
   @Override
   public void init(Configuration config) throws IOException {
     this.confStore = YarnConfigurationStoreFactory.getStore(config);
-    Configuration initialSchedConf = getInitSchedulerConfig();
-    initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
-    this.schedConf = new Configuration(false);
-    // We need to explicitly set the key-values in schedConf, otherwise
-    // these configuration keys cannot be deleted when
-    // configuration is reloaded.
-    for (Map.Entry<String, String> kv : initialSchedConf) {
-      schedConf.set(kv.getKey(), kv.getValue());
-    }
+    initializeSchedConf();
     try {
       confStore.initialize(config, schedConf, rmContext);
       confStore.checkVersion();
@@ -108,7 +92,7 @@ public void close() throws IOException {
   }
 
   @VisibleForTesting
-  public YarnConfigurationStore getConfStore() {
+  protected YarnConfigurationStore getConfStore() {
     return confStore;
   }
 
@@ -142,7 +126,7 @@ public LogMutation logAndApplyMutation(UserGroupInformation user,
     CapacitySchedulerConfiguration proposedConf =
             new CapacitySchedulerConfiguration(schedConf, false);
     Map<String, String> kvUpdate
-            = constructKeyValueConfUpdate(proposedConf, confUpdate);
+            = ConfigurationUpdateAssembler.constructKeyValueConfUpdate(proposedConf, confUpdate);
     LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
     confStore.logMutation(log);
     applyMutation(proposedConf, kvUpdate);
@@ -155,7 +139,7 @@ public Configuration applyChanges(Configuration oldConfiguration,
     CapacitySchedulerConfiguration proposedConf =
             new CapacitySchedulerConfiguration(oldConfiguration, false);
     Map<String, String> kvUpdate
-            = constructKeyValueConfUpdate(proposedConf, confUpdate);
+            = ConfigurationUpdateAssembler.constructKeyValueConfUpdate(proposedConf, confUpdate);
     applyMutation(proposedConf, kvUpdate);
     return proposedConf;
   }
@@ -177,15 +161,7 @@ public void formatConfigurationInStore(Configuration config)
     try {
       confStore.format();
       oldConf = new Configuration(schedConf);
-      Configuration initialSchedConf = new Configuration(false);
-      initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
-      this.schedConf = new Configuration(false);
-      // We need to explicitly set the key-values in schedConf, otherwise
-      // these configuration keys cannot be deleted when
-      // configuration is reloaded.
-      for (Map.Entry<String, String> kv : initialSchedConf) {
-        schedConf.set(kv.getKey(), kv.getValue());
-      }
+      initializeSchedConf();
       confStore.initialize(config, schedConf, rmContext);
       confStore.checkVersion();
     } catch (Exception e) {
@@ -195,6 +171,17 @@ public void formatConfigurationInStore(Configuration config)
     }
   }
 
+  private void initializeSchedConf() {
+    Configuration initialSchedConf = getInitSchedulerConfig();
+    this.schedConf = new Configuration(false);
+    // We need to explicitly set the key-values in schedConf, otherwise
+    // these configuration keys cannot be deleted when
+    // configuration is reloaded.
+    for (Map.Entry<String, String> kv : initialSchedConf) {
+      schedConf.set(kv.getKey(), kv.getValue());
+    }
+  }
+
   @Override
   public void revertToOldConfig(Configuration config) throws Exception {
     formatLock.writeLock().lock();
@@ -233,147 +220,4 @@ public void reloadConfigurationFromStore() throws Exception {
       formatLock.readLock().unlock();
     }
   }
-
-  private List<String> getSiblingQueues(String queuePath, Configuration conf) {
-    String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
-    String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
-        parentQueue + CapacitySchedulerConfiguration.DOT +
-        CapacitySchedulerConfiguration.QUEUES;
-    return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey));
-  }
-
-  private Map<String, String> constructKeyValueConfUpdate(
-      CapacitySchedulerConfiguration proposedConf,
-      SchedConfUpdateInfo mutationInfo) throws IOException {
-
-    Map<String, String> confUpdate = new HashMap<>();
-    for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
-      removeQueue(queueToRemove, proposedConf, confUpdate);
-    }
-    for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
-      addQueue(addQueueInfo, proposedConf, confUpdate);
-    }
-    for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
-      updateQueue(updateQueueInfo, proposedConf, confUpdate);
-    }
-    for (Map.Entry<String, String> global : mutationInfo.getGlobalParams()
-        .entrySet()) {
-      confUpdate.put(global.getKey(), global.getValue());
-    }
-    return confUpdate;
-  }
-
-  private void removeQueue(
-      String queueToRemove, CapacitySchedulerConfiguration proposedConf,
-      Map<String, String> confUpdate) throws IOException {
-    if (queueToRemove == null) {
-      return;
-    } else {
-      String queueName = queueToRemove.substring(
-          queueToRemove.lastIndexOf('.') + 1);
-      if (queueToRemove.lastIndexOf('.') == -1) {
-        throw new IOException("Can't remove queue " + queueToRemove);
-      } else {
-        List<String> siblingQueues = getSiblingQueues(queueToRemove,
-            proposedConf);
-        if (!siblingQueues.contains(queueName)) {
-          throw new IOException("Queue " + queueToRemove + " not found");
-        }
-        siblingQueues.remove(queueName);
-        String parentQueuePath = queueToRemove.substring(0, queueToRemove
-            .lastIndexOf('.'));
-        proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(
-            new String[0]));
-        String queuesConfig = CapacitySchedulerConfiguration.PREFIX
-            + parentQueuePath + CapacitySchedulerConfiguration.DOT
-            + CapacitySchedulerConfiguration.QUEUES;
-        if (siblingQueues.size() == 0) {
-          confUpdate.put(queuesConfig, null);
-          // Unset Ordering Policy of Leaf Queue converted from
-          // Parent Queue after removeQueue
-          String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
-              + parentQueuePath + CapacitySchedulerConfiguration.DOT
-              + ORDERING_POLICY;
-          proposedConf.unset(queueOrderingPolicy);
-          confUpdate.put(queueOrderingPolicy, null);
-        } else {
-          confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
-        }
-        for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
-            ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
-            .entrySet()) {
-          proposedConf.unset(confRemove.getKey());
-          confUpdate.put(confRemove.getKey(), null);
-        }
-      }
-    }
-  }
-
-  private void addQueue(
-      QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
-      Map<String, String> confUpdate) throws IOException {
-    if (addInfo == null) {
-      return;
-    } else {
-      String queuePath = addInfo.getQueue();
-      String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
-      if (queuePath.lastIndexOf('.') == -1) {
-        throw new IOException("Can't add invalid queue " + queuePath);
-      } else if (getSiblingQueues(queuePath, proposedConf).contains(
-          queueName)) {
-        throw new IOException("Can't add existing queue " + queuePath);
-      }
-      String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
-      String[] siblings = proposedConf.getQueues(parentQueue);
-      List<String> siblingQueues = siblings == null ? new ArrayList<>() :
-          new ArrayList<>(Arrays.<String>asList(siblings));
-      siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
-      proposedConf.setQueues(parentQueue,
-          siblingQueues.toArray(new String[0]));
-      confUpdate.put(CapacitySchedulerConfiguration.PREFIX
-              + parentQueue + CapacitySchedulerConfiguration.DOT
-              + CapacitySchedulerConfiguration.QUEUES,
-          Joiner.on(',').join(siblingQueues));
-      String keyPrefix = CapacitySchedulerConfiguration.PREFIX
-          + queuePath + CapacitySchedulerConfiguration.DOT;
-      for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
-        if (kv.getValue() == null) {
-          proposedConf.unset(keyPrefix + kv.getKey());
-        } else {
-          proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
-        }
-        confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
-      }
-      // Unset Ordering Policy of Parent Queue converted from
-      // Leaf Queue after addQueue
-      String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
-          + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
-      if (siblingQueues.size() == 1) {
-        proposedConf.unset(queueOrderingPolicy);
-        confUpdate.put(queueOrderingPolicy, null);
-      }
-    }
-  }
-
-  private void updateQueue(QueueConfigInfo updateInfo,
-      CapacitySchedulerConfiguration proposedConf,
-      Map<String, String> confUpdate) {
-    if (updateInfo == null) {
-      return;
-    } else {
-      String queuePath = updateInfo.getQueue();
-      String keyPrefix = CapacitySchedulerConfiguration.PREFIX
-          + queuePath + CapacitySchedulerConfiguration.DOT;
-      for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
-        String keyValue = kv.getValue();
-        if (keyValue == null || keyValue.isEmpty()) {
-          keyValue = null;
-          proposedConf.unset(keyPrefix + kv.getKey());
-        } else {
-          proposedConf.set(keyPrefix + kv.getKey(), keyValue);
-        }
-        confUpdate.put(keyPrefix + kv.getKey(), keyValue);
-      }
-    }
-  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java
new file mode 100644
index 0000000000000..28eb33c553636
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A class that parses {@code QueueCapacityVector} from the capacity
+ * configuration property set for a queue.
+ *
+ * A new syntax for capacity property could be implemented, by creating a parser
+ * with a regex to match the pattern and a method that creates a
+ * {@code QueueCapacityVector} from the matched pattern.
+ * Extending the parsers field with a {@code Parser} object in the constructor
+ * is needed in this case.
+ *
+ * A new capacity type for the existing parsers could be added by extending
+ * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its
+ * associated postfix symbol.
+ */
+public class QueueCapacityConfigParser {
+  private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)";
+  private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$";
+
+  private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX);
+  private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX);
+  public static final String FLOAT_DIGIT_REGEX = "[0-9.]";
+
+  private final List<Parser> parsers = new ArrayList<>();
+
+  public QueueCapacityConfigParser() {
+    parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser));
+    parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser));
+  }
+
+  /**
+   * Creates a {@code QueueCapacityVector} parsed from the capacity configuration
+   * property set for a queue.
+   * @param conf configuration object
+   * @param queuePath queue for which the capacity property is parsed
+   * @param label node label
+   * @return a parsed capacity vector
+   */
+  public QueueCapacityVector parse(CapacitySchedulerConfiguration conf,
+                                   String queuePath, String label) {
+
+    if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) {
+      return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE);
+    }
+
+    String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix(
+        queuePath, label) + CapacitySchedulerConfiguration.CAPACITY;
+    String capacityString = conf.get(propertyName);
+
+    if (capacityString == null) {
+      return new QueueCapacityVector();
+    }
+    // Trim all spaces from capacity string
+    capacityString = capacityString.replaceAll(" ", "");
+
+    for (Parser parser : parsers) {
+      Matcher matcher = parser.regex.matcher(capacityString);
+      if (matcher.find()) {
+        return parser.parser.apply(matcher);
+      }
+    }
+
+    return new QueueCapacityVector();
+  }
+
+  /**
+   * A parser method that is usable on uniform capacity values e.g. percentage or
+   * weight.
+   * @param matcher a regex matcher that contains parsed value and its possible
+   *                suffix
+   * @return a parsed capacity vector
+   */
+  private QueueCapacityVector uniformParser(Matcher matcher) {
+    QueueCapacityType capacityType = null;
+    String value = matcher.group(1);
+    if (matcher.groupCount() == 2) {
+      String matchedSuffix = matcher.group(2);
+      for (QueueCapacityType suffix : QueueCapacityType.values()) {
+        // Absolute uniform syntax is not supported
+        if (suffix.equals(QueueCapacityType.ABSOLUTE)) {
+          continue;
+        }
+        // when capacity is given in percentage, we do not need % symbol
+        String uniformSuffix = suffix.getPostfix().replaceAll("%", "");
+        if (uniformSuffix.equals(matchedSuffix)) {
+          capacityType = suffix;
+        }
+      }
+    }
+
+    if (capacityType == null) {
+      return new QueueCapacityVector();
+    }
+
+    return QueueCapacityVector.of(Float.parseFloat(value), capacityType);
+  }
+
+  /**
+   * A parser method that is usable on resource capacity values e.g. mixed or
+   * absolute resource.
+   * @param matcher a regex matcher that contains the matched resource string
+   * @return a parsed capacity vector
+   */
+  private QueueCapacityVector heterogeneousParser(Matcher matcher) {
+    QueueCapacityVector capacityVector = QueueCapacityVector.newInstance();
+
+    /*
+     * Absolute resource configuration for a queue will be grouped by "[]".
+     * Syntax of absolute resource config could be like below
+     * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores".
+     */
+    // Get the sub-group.
+    String bracketedGroup = matcher.group(0);
+    // Get the string inside starting and closing []
+    bracketedGroup = bracketedGroup.substring(1, bracketedGroup.length() - 1);
+    // Split by comma and equals delimiter eg. the string memory=1024,vcores=6
+    // is converted to an array of array as {{memory,1024}, {vcores, 6}}
+    for (String kvPair : bracketedGroup.trim().split(",")) {
+      String[] splits = kvPair.split("=");
+
+      // Ensure that each sub string is key value pair separated by '='.
+      if (splits.length > 1) {
+        setCapacityVector(capacityVector, splits[0], splits[1]);
+      }
+    }
+
+    // Memory always have to be defined
+    if (capacityVector.getMemory() == 0L) {
+      return new QueueCapacityVector();
+    }
+
+    return capacityVector;
+  }
+
+  private void setCapacityVector(
+      QueueCapacityVector resource, String resourceName, String resourceValue) {
+    QueueCapacityType capacityType = QueueCapacityType.ABSOLUTE;
+
+    // Extract suffix from a value e.g. for 6w extract w
+    String suffix = resourceValue.replaceAll(FLOAT_DIGIT_REGEX, "");
+    if (!resourceValue.endsWith(suffix)) {
+      return;
+    }
+
+    float parsedResourceValue = Float.parseFloat(resourceValue.substring(
+        0, resourceValue.length() - suffix.length()));
+    float convertedValue = parsedResourceValue;
+
+    if (!suffix.isEmpty() && UnitsConversionUtil.KNOWN_UNITS.contains(suffix)) {
+      // Convert all incoming units to MB if units is configured.
+      convertedValue = UnitsConversionUtil.convert(suffix, "Mi", (long) parsedResourceValue);
+    } else {
+      for (QueueCapacityType capacityTypeSuffix : QueueCapacityType.values()) {
+        if (capacityTypeSuffix.getPostfix().equals(suffix)) {
+          capacityType = capacityTypeSuffix;
+        }
+      }
+    }
+
+    resource.setResource(resourceName, convertedValue, capacityType);
+  }
+
+  /**
+   * Checks whether the given capacity string is in a capacity vector compatible
+   * format.
+   * @param configuredCapacity capacity string
+   * @return true, if capacity string is in capacity vector format,
+   * false otherwise
+   */
+  public boolean isCapacityVectorFormat(String configuredCapacity) {
+    return configuredCapacity != null
+        && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+  }
+
+  private static class Parser {
+    private final Pattern regex;
+    private final Function<Matcher, QueueCapacityVector> parser;
+
+    Parser(Pattern regex, Function<Matcher, QueueCapacityVector> parser) {
+      this.regex = regex;
+      this.parser = parser;
+    }
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 4480bc34dcc48..425d63f6a66eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -53,6 +53,7 @@ public abstract class YarnConfigurationStore {
    * audit logging and recovery.
    */
   public static class LogMutation implements Serializable {
+    private static final long serialVersionUID = 7754046036718906356L;
     private Map<String, String> updates;
     private String user;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 71226c300a840..492ff43dce72b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
 
+import org.apache.commons.io.serialization.ValidatingObjectInputStream;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
 import org.slf4j.Logger;
@@ -35,7 +36,6 @@
 import java.io.IOException;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -251,7 +251,7 @@ public List<LogMutation> getConfirmedConfHistory(long fromId) {
   private boolean createNewZkPath(String path) throws Exception {
     if (!zkManager.exists(path)) {
       try {
-        zkManager.create(path);
+        zkManager.create(path, zkAcl);
       } catch(NodeExistsException e) {
         LOG.warn(NODEEXISTS_MSG, e);
         return false;
@@ -314,7 +314,8 @@ private static byte[] serializeObject(Object o) throws Exception {
 
   private static Object deserializeObject(byte[] bytes) throws Exception {
     try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
-        ObjectInputStream ois = new ObjectInputStream(bais);) {
+         ValidatingObjectInputStream ois = new ValidatingObjectInputStream(bais);) {
+      ois.accept(LinkedList.class, LogMutation.class, HashMap.class, String.class);
       return ois.readObject();
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/MappingRuleCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/MappingRuleCreator.java
index 76dec8a272a3f..174577099e48c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/MappingRuleCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/MappingRuleCreator.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
 
-import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 
 import java.io.IOException;
 import java.nio.file.Files;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 44bfc95f644b8..d12a5e74b8d0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -28,12 +28,11 @@
     .CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 /**
  * For two queues with the same priority:
@@ -101,19 +100,21 @@ public static int compare(double relativeAssigned1, double relativeAssigned2,
   /**
    * Comparator that both looks at priority and utilization
    */
-  private class PriorityQueueComparator implements Comparator<CSQueue> {
+  private class PriorityQueueComparator
+      implements Comparator<PriorityQueueResourcesForSorting> {
 
     @Override
-    public int compare(CSQueue q1, CSQueue q2) {
+    public int compare(PriorityQueueResourcesForSorting q1Sort,
+        PriorityQueueResourcesForSorting q2Sort) {
       String p = partitionToLookAt.get();
 
-      int rc = compareQueueAccessToPartition(q1, q2, p);
+      int rc = compareQueueAccessToPartition(q1Sort.queue, q2Sort.queue, p);
       if (0 != rc) {
         return rc;
       }
 
-      float q1AbsCapacity = q1.getQueueCapacities().getAbsoluteCapacity(p);
-      float q2AbsCapacity = q2.getQueueCapacities().getAbsoluteCapacity(p);
+      float q1AbsCapacity = q1Sort.absoluteCapacity;
+      float q2AbsCapacity = q2Sort.absoluteCapacity;
 
       //If q1's abs capacity > 0 and q2 is 0, then prioritize q1
       if (Float.compare(q1AbsCapacity, 0f) > 0 && Float.compare(q2AbsCapacity,
@@ -127,28 +128,33 @@ public int compare(CSQueue q1, CSQueue q2) {
           q2AbsCapacity, 0f) == 0) {
         // both q1 has 0 and q2 has 0 capacity, then fall back to using
         // priority, abs used capacity to prioritize
-        float used1 = q1.getQueueCapacities().getAbsoluteUsedCapacity(p);
-        float used2 = q2.getQueueCapacities().getAbsoluteUsedCapacity(p);
+        float used1 = q1Sort.absoluteUsedCapacity;
+        float used2 = q2Sort.absoluteUsedCapacity;
 
-        return compare(q1, q2, used1, used2, p);
+        return compare(q1Sort, q2Sort, used1, used2,
+            q1Sort.queue.getPriority().
+                getPriority(), q2Sort.queue.getPriority().getPriority());
       } else{
         // both q1 has positive abs capacity and q2 has positive abs
         // capacity
-        float used1 = q1.getQueueCapacities().getUsedCapacity(p);
-        float used2 = q2.getQueueCapacities().getUsedCapacity(p);
+        float used1 = q1Sort.usedCapacity;
+        float used2 = q2Sort.usedCapacity;
 
-        return compare(q1, q2, used1, used2, p);
+        return compare(q1Sort, q2Sort, used1, used2,
+            q1Sort.queue.getPriority().getPriority(),
+            q2Sort.queue.getPriority().getPriority());
       }
     }
 
-    private int compare(CSQueue q1, CSQueue q2, float q1Used, float q2Used,
-        String partition) {
+    private int compare(PriorityQueueResourcesForSorting q1Sort,
+        PriorityQueueResourcesForSorting q2Sort, float q1Used,
+                        float q2Used, int q1Prior, int q2Prior) {
 
       int p1 = 0;
       int p2 = 0;
       if (respectPriority) {
-        p1 = q1.getPriority().getPriority();
-        p2 = q2.getPriority().getPriority();
+        p1 = q1Prior;
+        p2 = q2Prior;
       }
 
       int rc = PriorityUtilizationQueueOrderingPolicy.compare(q1Used, q2Used,
@@ -158,16 +164,16 @@ private int compare(CSQueue q1, CSQueue q2, float q1Used, float q2Used,
       // capacity goes first
       if (0 == rc) {
         Resource minEffRes1 =
-            q1.getQueueResourceQuotas().getConfiguredMinResource(partition);
+            q1Sort.configuredMinResource;
         Resource minEffRes2 =
-            q2.getQueueResourceQuotas().getConfiguredMinResource(partition);
+            q2Sort.configuredMinResource;
         if (!minEffRes1.equals(Resources.none()) && !minEffRes2.equals(
             Resources.none())) {
           return minEffRes2.compareTo(minEffRes1);
         }
 
-        float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(partition);
-        float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(partition);
+        float abs1 = q1Sort.absoluteCapacity;
+        float abs2 = q2Sort.absoluteCapacity;
         return Float.compare(abs2, abs1);
       }
 
@@ -203,6 +209,37 @@ private int compareQueueAccessToPartition(CSQueue q1, CSQueue q2,
     }
   }
 
+  /**
+   * A simple storage class to represent a snapshot of a queue.
+   */
+  public static class PriorityQueueResourcesForSorting {
+    private final float absoluteUsedCapacity;
+    private final float usedCapacity;
+    private final Resource configuredMinResource;
+    private final float absoluteCapacity;
+    private final CSQueue queue;
+
+    PriorityQueueResourcesForSorting(CSQueue queue) {
+      this.queue = queue;
+      this.absoluteUsedCapacity =
+          queue.getQueueCapacities().
+              getAbsoluteUsedCapacity(partitionToLookAt.get());
+      this.usedCapacity =
+          queue.getQueueCapacities().
+              getUsedCapacity(partitionToLookAt.get());
+      this.absoluteCapacity =
+          queue.getQueueCapacities().
+              getAbsoluteCapacity(partitionToLookAt.get());
+      this.configuredMinResource =
+          queue.getQueueResourceQuotas().
+              getConfiguredMinResource(partitionToLookAt.get());
+    }
+
+    public CSQueue getQueue() {
+      return queue;
+    }
+  }
+
   public PriorityUtilizationQueueOrderingPolicy(boolean respectPriority) {
     this.respectPriority = respectPriority;
   }
@@ -214,12 +251,14 @@ public void setQueues(List<CSQueue> queues) {
 
   @Override
   public Iterator<CSQueue> getAssignmentIterator(String partition) {
-    // Since partitionToLookAt is a thread local variable, and every time we
-    // copy and sort queues, so it's safe for multi-threading environment.
+    // partitionToLookAt is a thread local variable, therefore it is safe to mutate it.
     PriorityUtilizationQueueOrderingPolicy.partitionToLookAt.set(partition);
-    List<CSQueue> sortedQueue = new ArrayList<>(queues);
-    Collections.sort(sortedQueue, new PriorityQueueComparator());
-    return sortedQueue.iterator();
+
+    // Sort the snapshot of the queues in order to avoid breaking the prerequisites of TimSort.
+    // See YARN-10178 for details.
+    return queues.stream().map(PriorityQueueResourcesForSorting::new).sorted(
+        new PriorityQueueComparator()).map(PriorityQueueResourcesForSorting::getQueue).collect(
+            Collectors.toList()).iterator();
   }
 
   @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
index fefb56ad730cf..72d5bc58b2007 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/preemption/PreemptableQueue.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentSkipListMap;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java
new file mode 100644
index 0000000000000..31486a159d0bc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+
+public class DeactivatedLeafQueuesByLabel {
+  private String parentQueuePath;
+  private String nodeLabel;
+  private Map<String, QueueCapacities> deactivatedLeafQueues;
+  private float sumOfChildQueueActivatedCapacity;
+  private float parentAbsoluteCapacity;
+  private float leafQueueTemplateAbsoluteCapacity;
+  private float availableCapacity;
+  private float totalDeactivatedCapacity;
+
+  @VisibleForTesting
+  public DeactivatedLeafQueuesByLabel() {}
+
+  public DeactivatedLeafQueuesByLabel(
+      Map<String, QueueCapacities> deactivatedLeafQueues,
+      String parentQueuePath,
+      String nodeLabel,
+      float sumOfChildQueueActivatedCapacity,
+      float parentAbsoluteCapacity,
+      float leafQueueTemplateAbsoluteCapacity) {
+    this.parentQueuePath = parentQueuePath;
+    this.nodeLabel = nodeLabel;
+    this.deactivatedLeafQueues = deactivatedLeafQueues;
+    this.sumOfChildQueueActivatedCapacity = sumOfChildQueueActivatedCapacity;
+    this.parentAbsoluteCapacity = parentAbsoluteCapacity;
+    this.leafQueueTemplateAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity;
+
+    this.totalDeactivatedCapacity = getTotalDeactivatedCapacity();
+    this.availableCapacity = parentAbsoluteCapacity - sumOfChildQueueActivatedCapacity +
+        this.totalDeactivatedCapacity + EPSILON;
+  }
+
+  float getTotalDeactivatedCapacity() {
+    float deactivatedCapacity = 0;
+    for (Map.Entry<String, QueueCapacities> deactivatedQueueCapacity :
+        deactivatedLeafQueues.entrySet()) {
+      deactivatedCapacity += deactivatedQueueCapacity.getValue().getAbsoluteCapacity(nodeLabel);
+    }
+    return deactivatedCapacity;
+  }
+
+  public Set<String> getQueues() {
+    return deactivatedLeafQueues.keySet();
+  }
+
+  public void printToDebug(Logger logger) {
+    if (logger.isDebugEnabled()) {
+      logger.debug("Parent queue = {}, nodeLabel = {}, absCapacity = {}, " +
+              "leafQueueAbsoluteCapacity = {}, deactivatedCapacity = {}, " +
+              "absChildActivatedCapacity = {}, availableCapacity = {}",
+          parentQueuePath, nodeLabel, parentAbsoluteCapacity,
+          leafQueueTemplateAbsoluteCapacity, getTotalDeactivatedCapacity(),
+          sumOfChildQueueActivatedCapacity, availableCapacity);
+    }
+  }
+
+  @VisibleForTesting
+  public int getMaxLeavesToBeActivated(int numPendingApps) {
+    float childQueueAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity;
+    if (childQueueAbsoluteCapacity > 0) {
+      int numLeafQueuesNeeded = (int) Math.floor(availableCapacity / childQueueAbsoluteCapacity);
+      return Math.min(numLeafQueuesNeeded, numPendingApps);
+    }
+    return 0;
+  }
+
+  public boolean canActivateLeafQueues() {
+    return availableCapacity >= leafQueueTemplateAbsoluteCapacity;
+  }
+
+  @VisibleForTesting
+  public void setAvailableCapacity(float availableCapacity) {
+    this.availableCapacity = availableCapacity;
+  }
+
+  @VisibleForTesting
+  public void setLeafQueueTemplateAbsoluteCapacity(float leafQueueTemplateAbsoluteCapacity) {
+    this.leafQueueTemplateAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
index 5647b9146ed66..46bb40aee47c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -20,46 +20,27 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .QueueManagementDynamicEditPolicy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler
-    .SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .AbstractAutoCreatedLeafQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .AutoCreatedLeafQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .AutoCreatedLeafQueueConfig;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .AutoCreatedQueueManagementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractAutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueueConfig;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueManagementPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .CapacitySchedulerContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .LeafQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .ManagedParentQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .ParentQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .QueueCapacities;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .QueueManagementChange;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
-    .FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueManagementChange;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -84,7 +65,7 @@
 public class GuaranteedOrZeroCapacityOverTimePolicy
     implements AutoCreatedQueueManagementPolicy {
 
-  private CapacitySchedulerContext scheduler;
+  private static final int DEFAULT_QUEUE_PRINT_SIZE_LIMIT = 25;
   private ManagedParentQueue managedParentQueue;
 
   private static final Logger LOG =
@@ -138,7 +119,7 @@ private boolean addLeafQueueStateIfNotExists(String leafQueuePath,
       return false;
     }
 
-    public boolean createLeafQueueStateIfNotExists(LeafQueue leafQueue,
+    public boolean createLeafQueueStateIfNotExists(AbstractLeafQueue leafQueue,
         String partition) {
       return addLeafQueueStateIfNotExists(leafQueue.getQueuePath(), partition,
           new LeafQueueStatePerPartition());
@@ -269,38 +250,8 @@ void clear() {
     }
   }
 
-  /**
-   * Comparator that orders applications by their submit time
-   */
-  private class PendingApplicationComparator
-      implements Comparator<FiCaSchedulerApp> {
-
-    @Override
-    public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) {
-      RMApp rmApp1 = scheduler.getRMContext().getRMApps().get(
-          app1.getApplicationId());
-      RMApp rmApp2 = scheduler.getRMContext().getRMApps().get(
-          app2.getApplicationId());
-      if (rmApp1 != null && rmApp2 != null) {
-        return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime());
-      } else if (rmApp1 != null) {
-        return -1;
-      } else if (rmApp2 != null) {
-        return 1;
-      } else{
-        return 0;
-      }
-    }
-  }
-
-  private PendingApplicationComparator applicationComparator =
-      new PendingApplicationComparator();
-
   @Override
-  public void init(final CapacitySchedulerContext schedulerContext,
-      final ParentQueue parentQueue) throws IOException {
-    this.scheduler = schedulerContext;
-
+  public void init(final ParentQueue parentQueue) throws IOException {
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     readLock = lock.readLock();
     writeLock = lock.writeLock();
@@ -345,12 +296,11 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue)
   }
 
   /**
-   * Compute/Adjust child queue capacities
-   * for auto created leaf queues
-   * This computes queue entitlements but does not update LeafQueueState or
-   * queue capacities. Scheduler calls commitQueueManagemetChanges after
-   * validation after applying queue changes and commits to LeafQueueState
-   * are done in commitQueueManagementChanges.
+   * Computes / adjusts child queue capacities for auto created leaf queues.
+   * This method computes queue entitlements but does not update LeafQueueState or
+   * queue capacities.
+   * Scheduler calls commitQueueManagementChanges after validation after applying queue changes
+   * and commits to LeafQueueState are done in commitQueueManagementChanges.
    *
    * @return List of Queue Management change suggestions which could potentially
    * be committed/rejected by the scheduler due to validation failures
@@ -367,117 +317,98 @@ public List<QueueManagementChange> computeQueueManagementChanges()
             managedParentQueue.getAutoCreatedQueueManagementPolicy());
 
     //TODO : Add support for node labels on leaf queue template configurations
-    //synch/add missing leaf queue(s) if any to state
+    //sync / add missing leaf queue(s) if any TO state
     updateLeafQueueState();
 
     readLock.lock();
     try {
-      List<QueueManagementChange> queueManagementChanges = new ArrayList<>();
-      List<FiCaSchedulerApp> pendingApps = getSortedPendingApplications();
-
-      //Map of LeafQueue->QueueCapacities - keep adding the computed
-      // entitlements to this map and finally
-      // build the leaf queue configuration Template for all identified leaf
-      // queues
-      Map<String, QueueCapacities> leafQueueEntitlements = new HashMap<>();
+      LeafQueueEntitlements leafQueueEntitlements = new LeafQueueEntitlements();
       for (String nodeLabel : leafQueueTemplateNodeLabels) {
-        // check if any leaf queues need to be deactivated based on pending
-        // applications
-        float parentAbsoluteCapacity =
-            managedParentQueue.getQueueCapacities().getAbsoluteCapacity(
-                nodeLabel);
-        float leafQueueTemplateAbsoluteCapacity =
-            leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel);
-        Map<String, QueueCapacities> deactivatedLeafQueues =
-            deactivateLeafQueuesIfInActive(managedParentQueue, nodeLabel,
-                leafQueueEntitlements);
-
-        if (LOG.isDebugEnabled()) {
-          if ( deactivatedLeafQueues.size() > 0) {
-              LOG.debug("Parent queue = {},  " +
-                   ", nodeLabel = {}, deactivated leaf queues = [{}] ",
-                  managedParentQueue.getQueuePath(), nodeLabel,
-                  deactivatedLeafQueues.size() > 25 ? deactivatedLeafQueues
-                      .size() : deactivatedLeafQueues);
-
-          }
-        }
-
-        float deactivatedCapacity = getTotalDeactivatedCapacity(
-            deactivatedLeafQueues, nodeLabel);
-
-        float sumOfChildQueueActivatedCapacity = parentQueueState.
-            getAbsoluteActivatedChildQueueCapacity(nodeLabel);
+        DeactivatedLeafQueuesByLabel deactivatedLeafQueues =
+            deactivateLeafQueues(nodeLabel, leafQueueEntitlements);
+        deactivatedLeafQueues.printToDebug(LOG);
 
         //Check if we need to activate anything at all?
-        float availableCapacity =
-            parentAbsoluteCapacity - sumOfChildQueueActivatedCapacity
-                + deactivatedCapacity + EPSILON;
-
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Parent queue = " + managedParentQueue.getQueuePath()
-              + ", nodeLabel = " + nodeLabel + ", absCapacity = "
-              + parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = "
-              + leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = "
-              + deactivatedCapacity + " , absChildActivatedCapacity = "
-              + sumOfChildQueueActivatedCapacity + ", availableCapacity = "
-              + availableCapacity);
-        }
-
-        if (availableCapacity >= leafQueueTemplateAbsoluteCapacity) {
-          //sort applications across leaf queues by submit time
-          if (pendingApps.size() > 0) {
-            int maxLeafQueuesTobeActivated = getMaxLeavesToBeActivated(
-                availableCapacity, leafQueueTemplateAbsoluteCapacity,
-                pendingApps.size());
-
-            if (LOG.isDebugEnabled()) {
-              LOG.debug("Parent queue = " + managedParentQueue.getQueuePath()
-                  +  " : Found " + maxLeafQueuesTobeActivated + " leaf queues"
-                  + " to be activated with " + pendingApps.size() + " apps ");
-            }
-
-            LinkedHashSet<String> leafQueuesToBeActivated = getSortedLeafQueues(
-                nodeLabel, pendingApps, maxLeafQueuesTobeActivated,
-                deactivatedLeafQueues.keySet());
-
-            //Compute entitlement changes for the identified leaf queues
-            // which is appended to the List of computedEntitlements
-            updateLeafQueueCapacitiesByLabel(nodeLabel, leafQueuesToBeActivated,
-                leafQueueEntitlements);
-
-            if (LOG.isDebugEnabled()) {
-              if (leafQueuesToBeActivated.size() > 0) {
-                LOG.debug("Activated leaf queues : [{}]",
-                    leafQueuesToBeActivated.size() < 25 ?
-                    leafQueuesToBeActivated : leafQueuesToBeActivated.size());
-              }
-            }
-          }
+        if (deactivatedLeafQueues.canActivateLeafQueues()) {
+          activateLeafQueues(leafQueueEntitlements, nodeLabel, deactivatedLeafQueues);
         }
       }
 
       //Populate new entitlements
-
-      for (final Iterator<Map.Entry<String, QueueCapacities>> iterator =
-           leafQueueEntitlements.entrySet().iterator(); iterator.hasNext(); ) {
-        Map.Entry<String, QueueCapacities> queueCapacities = iterator.next();
-        String leafQueueName = queueCapacities.getKey();
+      return leafQueueEntitlements.mapToQueueManagementChanges((leafQueueName, capacities) -> {
         AutoCreatedLeafQueue leafQueue =
-            (AutoCreatedLeafQueue) scheduler.getCapacitySchedulerQueueManager()
+            (AutoCreatedLeafQueue) managedParentQueue.getQueueContext().getQueueManager()
                 .getQueue(leafQueueName);
-        AutoCreatedLeafQueueConfig newTemplate = buildTemplate(
-            queueCapacities.getValue());
-        queueManagementChanges.add(
-            new QueueManagementChange.UpdateQueue(leafQueue, newTemplate));
-
-      }
-      return queueManagementChanges;
+        AutoCreatedLeafQueueConfig newTemplate = buildTemplate(capacities);
+        return new QueueManagementChange.UpdateQueue(leafQueue, newTemplate);
+      });
     } finally {
       readLock.unlock();
     }
   }
 
+  private void activateLeafQueues(LeafQueueEntitlements leafQueueEntitlements, String nodeLabel,
+      DeactivatedLeafQueuesByLabel deactivatedLeafQueues) throws SchedulerDynamicEditException {
+    //sort applications across leaf queues by submit time
+    List<FiCaSchedulerApp> pendingApps = getSortedPendingApplications();
+    if (pendingApps.size() > 0) {
+      int maxLeafQueuesTobeActivated = deactivatedLeafQueues.
+          getMaxLeavesToBeActivated(pendingApps.size());
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Parent queue = {}, Found {} leaf queues to be activated with {} aps",
+            managedParentQueue.getQueuePath(), maxLeafQueuesTobeActivated, pendingApps.size());
+      }
+
+      Set<String> leafQueuesToBeActivated = getSortedLeafQueues(
+          nodeLabel, pendingApps, maxLeafQueuesTobeActivated,
+          deactivatedLeafQueues.getQueues());
+
+      // Compute entitlement changes for the identified leaf queues
+      // which is appended to the List of computedEntitlements
+      updateLeafQueueCapacitiesByLabel(nodeLabel, leafQueuesToBeActivated, leafQueueEntitlements);
+
+      if (LOG.isDebugEnabled() && leafQueuesToBeActivated.size() > 0) {
+        LOG.debug("Activated leaf queues : [{}]",
+            getListContentsUpToLimit(leafQueuesToBeActivated));
+      }
+    }
+  }
+
+  private Object getListContentsUpToLimit(Set<String> leafQueuesToBeActivated) {
+    return leafQueuesToBeActivated.size() < DEFAULT_QUEUE_PRINT_SIZE_LIMIT ?
+        leafQueuesToBeActivated : leafQueuesToBeActivated.size();
+  }
+
+  private Object getMapUpToLimit(Map<String, QueueCapacities> deactivatedLeafQueues) {
+    return deactivatedLeafQueues.size() > DEFAULT_QUEUE_PRINT_SIZE_LIMIT ?
+        deactivatedLeafQueues.size() : deactivatedLeafQueues;
+  }
+
+  private DeactivatedLeafQueuesByLabel deactivateLeafQueues(String nodeLabel,
+      LeafQueueEntitlements leafQueueEntitlements) throws SchedulerDynamicEditException {
+    // check if any leaf queues need to be deactivated based on pending applications
+    float parentAbsoluteCapacity =
+        managedParentQueue.getQueueCapacities().getAbsoluteCapacity(nodeLabel);
+    float leafQueueTemplateAbsoluteCapacity =
+        leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel);
+    Map<String, QueueCapacities> deactivatedLeafQueues =
+        deactivateLeafQueuesIfInActive(managedParentQueue, nodeLabel, leafQueueEntitlements);
+
+    if (LOG.isDebugEnabled() && deactivatedLeafQueues.size() > 0) {
+      LOG.debug("Parent queue = {}, nodeLabel = {}, deactivated leaf queues = [{}] ",
+          managedParentQueue.getQueuePath(), nodeLabel,
+          getMapUpToLimit(deactivatedLeafQueues));
+    }
+
+    return new DeactivatedLeafQueuesByLabel(deactivatedLeafQueues,
+        managedParentQueue.getQueuePath(),
+        nodeLabel,
+        parentQueueState.getAbsoluteActivatedChildQueueCapacity(nodeLabel),
+        parentAbsoluteCapacity,
+        leafQueueTemplateAbsoluteCapacity);
+  }
+
   private void updateTemplateAbsoluteCapacities(QueueCapacities parentQueueCapacities,
                                                 GuaranteedOrZeroCapacityOverTimePolicy policy) {
     writeLock.lock();
@@ -496,19 +427,6 @@ public void updateTemplateAbsoluteCapacities(QueueCapacities queueCapacities) {
     updateTemplateAbsoluteCapacities(queueCapacities, this);
   }
 
-  private float getTotalDeactivatedCapacity(
-      Map<String, QueueCapacities> deactivatedLeafQueues, String nodeLabel) {
-    float deactivatedCapacity = 0;
-    for (Iterator<Map.Entry<String, QueueCapacities>> iterator =
-         deactivatedLeafQueues.entrySet().iterator(); iterator.hasNext(); ) {
-      Map.Entry<String, QueueCapacities> deactivatedQueueCapacity =
-          iterator.next();
-      deactivatedCapacity +=
-          deactivatedQueueCapacity.getValue().getAbsoluteCapacity(nodeLabel);
-    }
-    return deactivatedCapacity;
-  }
-
   @VisibleForTesting
   void updateLeafQueueState() {
     writeLock.lock();
@@ -517,9 +435,9 @@ void updateLeafQueueState() {
       Set<String> newQueues = new HashSet<>();
 
       for (CSQueue newQueue : managedParentQueue.getChildQueues()) {
-        if (newQueue instanceof LeafQueue) {
+        if (newQueue instanceof AbstractLeafQueue) {
           for (String nodeLabel : leafQueueTemplateNodeLabels) {
-            leafQueueState.createLeafQueueStateIfNotExists((LeafQueue) newQueue,
+            leafQueueState.createLeafQueueStateIfNotExists((AbstractLeafQueue) newQueue,
                 nodeLabel);
             newPartitions.add(nodeLabel);
           }
@@ -608,9 +526,15 @@ public boolean isActive(final AutoCreatedLeafQueue leafQueue,
     }
   }
 
+  /**
+   * Map of LeafQueue -> QueueCapacities - keep adding the computed
+   * entitlements to this map and finally
+   * build the leaf queue configuration Template for all identified leaf
+   * queues
+   */
   private Map<String, QueueCapacities> deactivateLeafQueuesIfInActive(
       ParentQueue parentQueue, String nodeLabel,
-      Map<String, QueueCapacities> leafQueueEntitlements)
+      LeafQueueEntitlements leafQueueEntitlements)
       throws SchedulerDynamicEditException {
     Map<String, QueueCapacities> deactivatedQueues = new HashMap<>();
 
@@ -618,18 +542,11 @@ private Map<String, QueueCapacities> deactivateLeafQueuesIfInActive(
       AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue;
       if (leafQueue != null) {
         if (isActive(leafQueue, nodeLabel) && !hasPendingApps(leafQueue)) {
-          if (!leafQueueEntitlements.containsKey(leafQueue.getQueuePath())) {
-            leafQueueEntitlements.put(leafQueue.getQueuePath(),
-                new QueueCapacities(false));
-          }
-
-          QueueCapacities capacities = leafQueueEntitlements.get(
-              leafQueue.getQueuePath());
-          updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue);
-          deactivatedQueues.put(leafQueue.getQueuePath(),
-              leafQueueTemplateCapacities);
+          QueueCapacities capacities = leafQueueEntitlements.getCapacityOfQueue(leafQueue);
+          updateToZeroCapacity(capacities, nodeLabel, (AbstractLeafQueue) childQueue);
+          deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities);
         }
-      } else{
+      } else {
         LOG.warn("Could not find queue in scheduler while trying" + " to "
             + "deactivate for " + parentQueue);
       }
@@ -640,35 +557,16 @@ private Map<String, QueueCapacities> deactivateLeafQueuesIfInActive(
 
   private void updateLeafQueueCapacitiesByLabel(String nodeLabel,
       Set<String> leafQueuesToBeActivated,
-      Map<String, QueueCapacities> leafQueueEntitlements) {
-    for (String curLeafQueue : leafQueuesToBeActivated) {
-      if (!leafQueueEntitlements.containsKey(curLeafQueue)) {
-        leafQueueEntitlements.put(curLeafQueue, new QueueCapacities(false));
-        // Activate queues if capacity is available
-      }
-
-      QueueCapacities capacities = leafQueueEntitlements.get(curLeafQueue);
+      LeafQueueEntitlements leafQueueEntitlements) {
+    for (String leafQueue : leafQueuesToBeActivated) {
+      QueueCapacities capacities = leafQueueEntitlements.getCapacityOfQueueByPath(leafQueue);
       updateCapacityFromTemplate(capacities, nodeLabel);
     }
   }
 
-  @VisibleForTesting
-  public int getMaxLeavesToBeActivated(float availableCapacity,
-      float childQueueAbsoluteCapacity, int numPendingApps)
-      throws SchedulerDynamicEditException {
-
-    if (childQueueAbsoluteCapacity > 0) {
-      int numLeafQueuesNeeded = (int) Math.floor(
-          availableCapacity / childQueueAbsoluteCapacity);
-
-      return Math.min(numLeafQueuesNeeded, numPendingApps);
-    }
-    return 0;
-  }
-
   /**
    * Commit queue management changes - which involves updating required state
-   * on parent/underlying leaf queues
+   * on parent/underlying leaf queues.
    *
    * @param queueManagementChanges Queue Management changes to commit
    * @throws SchedulerDynamicEditException when validation fails
@@ -694,19 +592,18 @@ public void commitQueueManagementChanges(
 
         for (String nodeLabel : updatedQueueTemplate.getQueueCapacities()
             .getExistingNodeLabels()) {
-          if (updatedQueueTemplate.getQueueCapacities().
-              getCapacity(nodeLabel) > 0) {
+          if (updatedQueueTemplate.getQueueCapacities().getCapacity(nodeLabel) > 0) {
             if (isActive(leafQueue, nodeLabel)) {
               LOG.debug("Queue is already active. Skipping activation : {}",
                   leafQueue.getQueuePath());
             } else{
               activate(leafQueue, nodeLabel);
             }
-          } else{
+          } else {
             if (!isActive(leafQueue, nodeLabel)) {
               LOG.debug("Queue is already de-activated. Skipping "
                   + "de-activation : {}", leafQueue.getQueuePath());
-            } else{
+            } else {
               /**
                * While deactivating queues of type ABSOLUTE_RESOURCE, configured
                * min resource has to be set based on updated capacity (which is
@@ -715,10 +612,11 @@ public void commitQueueManagementChanges(
                * leads to incorrect results.
                */
               leafQueue
-                  .mergeCapacities(updatedQueueTemplate.getQueueCapacities());
+                  .mergeCapacities(updatedQueueTemplate.getQueueCapacities(), leafQueueTemplate.getResourceQuotas());
               leafQueue.getQueueResourceQuotas()
                   .setConfiguredMinResource(Resources.multiply(
-                      this.scheduler.getClusterResource(), updatedQueueTemplate
+                      managedParentQueue.getQueueContext().getClusterResource(),
+                      updatedQueueTemplate
                           .getQueueCapacities().getCapacity(nodeLabel)));
               deactivate(leafQueue, nodeLabel);
             }
@@ -760,8 +658,7 @@ public boolean hasPendingApps(final AutoCreatedLeafQueue leafQueue) {
   }
 
   @Override
-  public void reinitialize(CapacitySchedulerContext schedulerContext,
-      final ParentQueue parentQueue) throws IOException {
+  public void reinitialize(final ParentQueue parentQueue) throws IOException {
     if (!(parentQueue instanceof ManagedParentQueue)) {
       throw new IllegalStateException(
           "Expected instance of type " + ManagedParentQueue.class + " found  "
@@ -835,7 +732,7 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
   }
 
   private void updateToZeroCapacity(QueueCapacities capacities,
-      String nodeLabel, LeafQueue leafQueue) {
+      String nodeLabel, AbstractLeafQueue leafQueue) {
     capacities.setCapacity(nodeLabel, 0.0f);
     capacities.setMaximumCapacity(nodeLabel,
         leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel));
@@ -856,7 +753,7 @@ private void updateCapacityFromTemplate(QueueCapacities capacities,
   }
 
   @VisibleForTesting
-  LeafQueueStatePerPartition getLeafQueueState(LeafQueue queue,
+  LeafQueueStatePerPartition getLeafQueueState(AbstractLeafQueue queue,
       String partition) throws SchedulerDynamicEditException {
     readLock.lock();
     try {
@@ -881,7 +778,7 @@ public float getAbsoluteActivatedChildQueueCapacity(String nodeLabel) {
   private List<FiCaSchedulerApp> getSortedPendingApplications() {
     List<FiCaSchedulerApp> apps = new ArrayList<>(
         managedParentQueue.getAllApplications());
-    Collections.sort(apps, applicationComparator);
+    apps.sort(managedParentQueue.getQueueContext().getApplicationComparator());
     return apps;
   }
 
@@ -889,6 +786,7 @@ private AutoCreatedLeafQueueConfig buildTemplate(QueueCapacities capacities) {
     AutoCreatedLeafQueueConfig.Builder templateBuilder =
         new AutoCreatedLeafQueueConfig.Builder();
     templateBuilder.capacities(capacities);
+    templateBuilder.resourceQuotas(managedParentQueue.getLeafQueueTemplate().getResourceQuotas());
     return new AutoCreatedLeafQueueConfig(templateBuilder);
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java
new file mode 100644
index 0000000000000..c558716ca48e6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueManagementChange;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+public class LeafQueueEntitlements {
+  private final Map<String, QueueCapacities> entitlements = new HashMap<>();
+
+  public QueueCapacities getCapacityOfQueue(AutoCreatedLeafQueue leafQueue) {
+    return getCapacityOfQueueByPath(leafQueue.getQueuePath());
+  }
+
+  public QueueCapacities getCapacityOfQueueByPath(String leafQueuePath) {
+    if (!entitlements.containsKey(leafQueuePath)) {
+      entitlements.put(leafQueuePath, new QueueCapacities(false));
+    }
+    return entitlements.get(leafQueuePath);
+  }
+
+  public Map<String, QueueCapacities> getEntitlements() {
+    return entitlements;
+  }
+
+  public List<QueueManagementChange> mapToQueueManagementChanges(
+      BiFunction<String, QueueCapacities, QueueManagementChange> func) {
+    return entitlements.entrySet().stream().map(e -> func.apply(e.getKey(), e.getValue()))
+        .collect(Collectors.toList());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java
new file mode 100644
index 0000000000000..98bed9515506d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains classes that are related to the newer
+ * version of CS placement engine.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index a1b8205dad1fc..3a0fd347e5a0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -66,11 +66,11 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AbstractContainerAllocator;
@@ -357,6 +357,7 @@ private boolean commonCheckContainerAllocation(
       // adding NP check as this proposal could not be allocated from reserved
       // container in async-scheduling mode
       if (allocation.getAllocateFromReservedContainer() == null) {
+        LOG.debug("Trying to allocate from reserved container in async scheduling mode");
         return false;
       }
       RMContainer fromReservedContainer =
@@ -572,6 +573,8 @@ public boolean apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,
         if (updatePending &&
             getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey())
                 <= 0) {
+          LOG.debug("Rejecting appliance of allocation due to existing pending allocation " +
+              "request for " + schedulerContainer);
           return false;
         }
 
@@ -670,10 +673,12 @@ public boolean apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,
               schedulerContainer.getRmContainer().getContainer(),
               reReservation);
 
-          LOG.info("Reserved container=" + rmContainer.getContainerId()
-              + ", on node=" + schedulerContainer.getSchedulerNode()
-              + " with resource=" + rmContainer
-              .getAllocatedOrReservedResource());
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Reserved container=" + rmContainer.getContainerId()
+                + ", on node=" + schedulerContainer.getSchedulerNode()
+                + " with resource=" + rmContainer
+                .getAllocatedOrReservedResource());
+          }
         }
       }
     } finally {
@@ -953,8 +958,8 @@ public RMContainer findNodeToUnreserve(FiCaSchedulerNode node,
     }
   }
 
-  public LeafQueue getCSLeafQueue() {
-    return (LeafQueue)queue;
+  public AbstractLeafQueue getCSLeafQueue() {
+    return (AbstractLeafQueue)queue;
   }
 
   public CSAssignment assignContainers(Resource clusterResource,
@@ -991,7 +996,7 @@ public void nodePartitionUpdated(RMContainer rmContainer, String oldPartition,
 
   protected void getPendingAppDiagnosticMessage(
       StringBuilder diagnosticMessage) {
-    LeafQueue queue = getCSLeafQueue();
+    AbstractLeafQueue queue = getCSLeafQueue();
     diagnosticMessage.append(" Details : AM Partition = ")
         .append(appAMNodePartitionName.isEmpty()
         ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName)
@@ -1014,7 +1019,7 @@ appAMNodePartitionName, getUser()))
 
   protected void getActivedAppDiagnosticMessage(
       StringBuilder diagnosticMessage) {
-    LeafQueue queue = getCSLeafQueue();
+    AbstractLeafQueue queue = getCSLeafQueue();
     QueueCapacities queueCapacities = queue.getQueueCapacities();
     QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
     diagnosticMessage.append(" Details : AM Partition = ")
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/CentralizedOpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/CentralizedOpportunisticContainerAllocator.java
index 223e1a1bf30cb..482612de250ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/CentralizedOpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/CentralizedOpportunisticContainerAllocator.java
@@ -251,14 +251,15 @@ private List<Container> allocateNodeLocal(
       String userName, Map<Resource, List<Allocation>> allocations)
       throws YarnException {
     List<Container> allocatedContainers = new ArrayList<>();
+    final ResourceRequest resourceRequest = enrichedAsk.getRequest();
     while (toAllocate > 0) {
       RMNode node = nodeQueueLoadMonitor.selectLocalNode(nodeLocation,
-          blacklist);
+          blacklist, resourceRequest.getCapability());
       if (node != null) {
         toAllocate--;
         Container container = createContainer(rmIdentifier, appParams,
             idCounter, id, userName, allocations, nodeLocation,
-            enrichedAsk.getRequest(), convertToRemoteNode(node));
+            resourceRequest, convertToRemoteNode(node));
         allocatedContainers.add(container);
         LOG.info("Allocated [{}] as opportunistic at location [{}]",
             container.getId(), nodeLocation);
@@ -280,14 +281,15 @@ private List<Container> allocateRackLocal(EnrichedResourceRequest enrichedAsk,
       String userName, Map<Resource, List<Allocation>> allocations)
       throws YarnException {
     List<Container> allocatedContainers = new ArrayList<>();
+    final ResourceRequest resourceRequest = enrichedAsk.getRequest();
     while (toAllocate > 0) {
       RMNode node = nodeQueueLoadMonitor.selectRackLocalNode(rackLocation,
-          blacklist);
+          blacklist, resourceRequest.getCapability());
       if (node != null) {
         toAllocate--;
         Container container = createContainer(rmIdentifier, appParams,
             idCounter, id, userName, allocations, rackLocation,
-            enrichedAsk.getRequest(), convertToRemoteNode(node));
+            resourceRequest, convertToRemoteNode(node));
         allocatedContainers.add(container);
         metrics.incrRackLocalOppContainers();
         LOG.info("Allocated [{}] as opportunistic at location [{}]",
@@ -309,13 +311,15 @@ private List<Container> allocateAny(EnrichedResourceRequest enrichedAsk,
       String userName, Map<Resource, List<Allocation>> allocations)
       throws YarnException {
     List<Container> allocatedContainers = new ArrayList<>();
+    final ResourceRequest resourceRequest = enrichedAsk.getRequest();
     while (toAllocate > 0) {
-      RMNode node = nodeQueueLoadMonitor.selectAnyNode(blacklist);
+      RMNode node = nodeQueueLoadMonitor.selectAnyNode(
+          blacklist, resourceRequest.getCapability());
       if (node != null) {
         toAllocate--;
         Container container = createContainer(rmIdentifier, appParams,
             idCounter, id, userName, allocations, ResourceRequest.ANY,
-            enrichedAsk.getRequest(), convertToRemoteNode(node));
+            resourceRequest, convertToRemoteNode(node));
         allocatedContainers.add(container);
         metrics.incrOffSwitchOppContainers();
         LOG.info("Allocated [{}] as opportunistic at location [{}]",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java
index f92a92bb17c40..065f97c4ce23e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/ClusterNode.java
@@ -20,74 +20,243 @@
 
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
 
 /**
  * Represents a node in the cluster from the NodeQueueLoadMonitor's perspective
  */
 public class ClusterNode {
-  private final AtomicInteger queueLength = new AtomicInteger(0);
-  private final AtomicInteger queueWaitTime = new AtomicInteger(-1);
+  /**
+   * Properties class used to initialize/change fields in ClusterNode.
+   */
+  public static final class Properties {
+    private int queueLength = 0;
+    private int queueWaitTime = -1;
+    private long timestamp;
+    private int queueCapacity = 0;
+    private boolean queueCapacityIsSet = false;
+    private final HashSet<String> labels;
+    private Resource capability = null;
+    private Resource allocatedResource = null;
+
+    public static Properties newInstance() {
+      return new Properties();
+    }
+
+    Properties setQueueLength(int qLength) {
+      this.queueLength = qLength;
+      return this;
+    }
+
+    Properties setQueueWaitTime(int wTime) {
+      this.queueWaitTime = wTime;
+      return this;
+    }
+
+    Properties updateTimestamp() {
+      this.timestamp = System.currentTimeMillis();
+      return this;
+    }
+
+    Properties setQueueCapacity(int capacity) {
+      this.queueCapacity = capacity;
+      this.queueCapacityIsSet = true;
+      return this;
+    }
+
+    Properties setNodeLabels(Collection<String> labelsToAdd) {
+      labels.clear();
+      labels.addAll(labelsToAdd);
+      return this;
+    }
+
+    Properties setCapability(Resource nodeCapability) {
+      this.capability = nodeCapability;
+      return this;
+    }
+
+    Properties setAllocatedResource(Resource allocResource) {
+      this.allocatedResource = allocResource;
+      return this;
+    }
+
+    private Properties() {
+      labels = new HashSet<>();
+    }
+  }
+
+  private int queueLength = 0;
+  private int queueWaitTime = -1;
   private long timestamp;
   final NodeId nodeId;
   private int queueCapacity = 0;
   private final HashSet<String> labels;
+  private Resource capability = Resources.none();
+  private Resource allocatedResource = Resources.none();
+  private final ReentrantReadWriteLock.WriteLock writeLock;
+  private final ReentrantReadWriteLock.ReadLock readLock;
 
   public ClusterNode(NodeId nodeId) {
     this.nodeId = nodeId;
     this.labels = new HashSet<>();
-    updateTimestamp();
+    final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    this.writeLock = lock.writeLock();
+    this.readLock = lock.readLock();
+    this.timestamp = System.currentTimeMillis();
   }
 
-  public ClusterNode setQueueLength(int qLength) {
-    this.queueLength.set(qLength);
-    return this;
-  }
+  public ClusterNode setProperties(final Properties properties) {
+    writeLock.lock();
+    try {
+      if (properties.capability == null) {
+        this.capability = Resources.none();
+      } else {
+        this.capability = properties.capability;
+      }
 
-  public ClusterNode setQueueWaitTime(int wTime) {
-    this.queueWaitTime.set(wTime);
-    return this;
+      if (properties.allocatedResource == null) {
+        this.allocatedResource = Resources.none();
+      } else {
+        this.allocatedResource = properties.allocatedResource;
+      }
+
+      this.queueLength = properties.queueLength;
+      this.queueWaitTime = properties.queueWaitTime;
+      this.timestamp = properties.timestamp;
+      if (properties.queueCapacityIsSet) {
+        // queue capacity is only set on node add, not on node updates
+        this.queueCapacity = properties.queueCapacity;
+      }
+      this.labels.clear();
+      this.labels.addAll(properties.labels);
+      return this;
+    } finally {
+      writeLock.unlock();
+    }
   }
 
-  public ClusterNode updateTimestamp() {
-    this.timestamp = System.currentTimeMillis();
-    return this;
+  public Resource getAllocatedResource() {
+    readLock.lock();
+    try {
+      return this.allocatedResource;
+    } finally {
+      readLock.unlock();
+    }
   }
 
-  public ClusterNode setQueueCapacity(int capacity) {
-    this.queueCapacity = capacity;
-    return this;
+  public Resource getAvailableResource() {
+    readLock.lock();
+    try {
+      return Resources.subtractNonNegative(capability, allocatedResource);
+    } finally {
+      readLock.unlock();
+    }
   }
 
-  public ClusterNode setNodeLabels(Collection<String> labelsToAdd) {
-    labels.clear();
-    labels.addAll(labelsToAdd);
-    return this;
+  public Resource getCapability() {
+    readLock.lock();
+    try {
+      return this.capability;
+    } finally {
+      readLock.unlock();
+    }
   }
 
   public boolean hasLabel(String label) {
-    return this.labels.contains(label);
+    readLock.lock();
+    try {
+      return this.labels.contains(label);
+    } finally {
+      readLock.unlock();
+    }
   }
 
   public long getTimestamp() {
-    return this.timestamp;
+    readLock.lock();
+    try {
+      return this.timestamp;
+    } finally {
+      readLock.unlock();
+    }
   }
 
-  public AtomicInteger getQueueLength() {
-    return this.queueLength;
+  public int getQueueLength() {
+    readLock.lock();
+    try {
+      return this.queueLength;
+    } finally {
+      readLock.unlock();
+    }
   }
 
-  public AtomicInteger getQueueWaitTime() {
-    return this.queueWaitTime;
+  public int getQueueWaitTime() {
+    readLock.lock();
+    try {
+      return this.queueWaitTime;
+    } finally {
+      readLock.unlock();
+    }
   }
 
   public int getQueueCapacity() {
-    return this.queueCapacity;
+    readLock.lock();
+    try {
+      return this.queueCapacity;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public boolean compareAndIncrementAllocation(
+      final int incrementQLen,
+      final ResourceCalculator resourceCalculator,
+      final Resource requested) {
+    writeLock.lock();
+    try {
+      final Resource currAvailable = Resources.subtractNonNegative(
+          capability, allocatedResource);
+      if (resourceCalculator.fitsIn(requested, currAvailable)) {
+        allocatedResource = Resources.add(allocatedResource, requested);
+        return true;
+      }
+
+      if (!resourceCalculator.fitsIn(requested, capability)) {
+        // If does not fit at all, do not allocate
+        return false;
+      }
+
+      return compareAndIncrementAllocation(incrementQLen);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  public boolean compareAndIncrementAllocation(final int incrementQLen) {
+    writeLock.lock();
+    try {
+      final int added = queueLength + incrementQLen;
+      if (added <= queueCapacity) {
+        queueLength = added;
+        return true;
+      }
+      return false;
+    } finally {
+      writeLock.unlock();
+    }
   }
 
   public boolean isQueueFull() {
-    return this.queueCapacity > 0 &&
-        this.queueLength.get() >= this.queueCapacity;
+    readLock.lock();
+    try {
+      return this.queueCapacity > 0 &&
+          this.queueLength >= this.queueCapacity;
+    } finally {
+      readLock.unlock();
+    }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index 1e2b7ef6da475..74dc2ce7f9fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
@@ -18,8 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed;
 
+import org.apache.commons.math3.util.Precision;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -64,39 +69,165 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
    * of two Nodes are compared.
    */
   public enum LoadComparator implements Comparator<ClusterNode> {
+    /**
+     * This policy only considers queue length.
+     * When allocating, increments queue length without looking at resources
+     * available on the node, and when sorting, also only sorts by queue length.
+     */
     QUEUE_LENGTH,
-    QUEUE_WAIT_TIME;
+    /**
+     * This policy only considers the wait time of containers in the queue.
+     * Neither looks at resources nor at queue length.
+     */
+    QUEUE_WAIT_TIME,
+    /**
+     * This policy considers both queue length and resources.
+     * When allocating, first decrements resources available on a node.
+     * If resources are available, does not place OContainers on the node queue.
+     * When sorting, it first sorts by queue length,
+     * then by available resources.
+     */
+    QUEUE_LENGTH_THEN_RESOURCES;
+
+    private Resource clusterResource = Resources.none();
+    private final DominantResourceCalculator resourceCalculator =
+        new DominantResourceCalculator();
+
+    private boolean shouldPerformMinRatioComputation() {
+      if (clusterResource == null) {
+        return false;
+      }
+
+      return !resourceCalculator.isAnyMajorResourceZeroOrNegative(
+          clusterResource);
+    }
+
+    /**
+     * Compares queue length of nodes first (shortest first),
+     * then compares available resources normalized
+     * over cluster resources (most available resources first).
+     * @param o1 the first ClusterNode
+     * @param o2 the second ClusterNode
+     * @return the difference the two ClusterNodes for sorting
+     */
+    private int compareQueueLengthThenResources(
+        final ClusterNode o1, final ClusterNode o2) {
+      int diff = o1.getQueueLength() - o2.getQueueLength();
+      if (diff != 0) {
+        return diff;
+      }
+
+      final Resource availableResource1 = o1.getAvailableResource();
+      final Resource availableResource2 = o2.getAvailableResource();
+
+      // Cluster resource should be valid before performing min-ratio logic
+      // Use raw available resource comparison otherwise
+      if (shouldPerformMinRatioComputation()) {
+        // Takes the least available resource of the two nodes,
+        // normalized to the overall cluster resource
+        final float availableRatio1 =
+            resourceCalculator.minRatio(availableResource1, clusterResource);
+        final float availableRatio2 =
+            resourceCalculator.minRatio(availableResource2, clusterResource);
+
+        // The one with more available resources should be placed first
+        diff = Precision.compareTo(
+            availableRatio2, availableRatio1, Precision.EPSILON);
+      }
+
+      if (diff == 0) {
+        // Compare absolute value if ratios are the same
+        diff = availableResource2.getVirtualCores() - availableResource1.getVirtualCores();
+      }
+
+      if (diff == 0) {
+        diff = Long.compare(availableResource2.getMemorySize(),
+            availableResource1.getMemorySize());
+      }
+
+      return diff;
+    }
 
     @Override
     public int compare(ClusterNode o1, ClusterNode o2) {
-      if (getMetric(o1) == getMetric(o2)) {
-        return (int)(o2.getTimestamp() - o1.getTimestamp());
+      int diff;
+      switch (this) {
+      case QUEUE_LENGTH_THEN_RESOURCES:
+        diff = compareQueueLengthThenResources(o1, o2);
+        break;
+      case QUEUE_WAIT_TIME:
+      case QUEUE_LENGTH:
+      default:
+        diff = getMetric(o1) - getMetric(o2);
+        break;
       }
-      return getMetric(o1) - getMetric(o2);
+
+      if (diff == 0) {
+        return (int) (o2.getTimestamp() - o1.getTimestamp());
+      }
+      return diff;
+    }
+
+    @VisibleForTesting
+    void setClusterResource(Resource clusterResource) {
+      this.clusterResource = clusterResource;
+    }
+
+    public ResourceCalculator getResourceCalculator() {
+      return resourceCalculator;
     }
 
     public int getMetric(ClusterNode c) {
-      return (this == QUEUE_LENGTH) ?
-          c.getQueueLength().get() : c.getQueueWaitTime().get();
+      switch (this) {
+      case QUEUE_WAIT_TIME:
+        return c.getQueueWaitTime();
+      case QUEUE_LENGTH:
+      case QUEUE_LENGTH_THEN_RESOURCES:
+      default:
+        return c.getQueueLength();
+      }
     }
 
     /**
      * Increment the metric by a delta if it is below the threshold.
      * @param c ClusterNode
      * @param incrementSize increment size
+     * @param requested the requested resource
      * @return true if the metric was below threshold and was incremented.
      */
-    public boolean compareAndIncrement(ClusterNode c, int incrementSize) {
-      if(this == QUEUE_LENGTH) {
-        int ret = c.getQueueLength().addAndGet(incrementSize);
-        if (ret <= c.getQueueCapacity()) {
-          return true;
+    public boolean compareAndIncrement(
+        ClusterNode c, int incrementSize, Resource requested) {
+      switch (this) {
+      case QUEUE_LENGTH_THEN_RESOURCES:
+        return c.compareAndIncrementAllocation(
+            incrementSize, resourceCalculator, requested);
+      case QUEUE_WAIT_TIME:
+        // for queue wait time, we don't have any threshold.
+        return true;
+      case QUEUE_LENGTH:
+      default:
+        return c.compareAndIncrementAllocation(incrementSize);
+      }
+    }
+
+    /**
+     * Whether we should be placing OContainers on a node.
+     * @param cn the clusterNode
+     * @return whether we should be placing OContainers on a node.
+     */
+    public boolean isNodeAvailable(final ClusterNode cn) {
+      int queueCapacity = cn.getQueueCapacity();
+      int queueLength = cn.getQueueLength();
+      if (this == LoadComparator.QUEUE_LENGTH_THEN_RESOURCES) {
+        if (queueCapacity <= 0) {
+          return queueLength <= 0;
+        } else {
+          return queueLength < queueCapacity;
         }
-        c.getQueueLength().addAndGet(-incrementSize);
-        return false;
       }
-      // for queue wait time, we don't have any threshold.
-      return true;
+      // In the special case where queueCapacity is 0 for the node,
+      // the container can be allocated on the node but will be rejected there
+      return queueCapacity <= 0 || queueLength < queueCapacity;
     }
   }
 
@@ -261,13 +392,21 @@ protected void onNewNodeAdded(
 
     if (rmNode.getState() != NodeState.DECOMMISSIONING &&
         (estimatedQueueWaitTime != -1 ||
-            comparator == LoadComparator.QUEUE_LENGTH)) {
-      this.clusterNodes.put(rmNode.getNodeID(),
-          new ClusterNode(rmNode.getNodeID())
+            comparator == LoadComparator.QUEUE_LENGTH ||
+            comparator == LoadComparator.QUEUE_LENGTH_THEN_RESOURCES)) {
+      final ClusterNode.Properties properties =
+          ClusterNode.Properties.newInstance()
               .setQueueWaitTime(estimatedQueueWaitTime)
               .setQueueLength(waitQueueLength)
               .setNodeLabels(rmNode.getNodeLabels())
-              .setQueueCapacity(opportQueueCapacity));
+              .setCapability(rmNode.getTotalCapability())
+              .setAllocatedResource(rmNode.getAllocatedContainerResource())
+              .setQueueCapacity(opportQueueCapacity)
+              .updateTimestamp();
+
+      this.clusterNodes.put(rmNode.getNodeID(),
+          new ClusterNode(rmNode.getNodeID()).setProperties(properties));
+
       LOG.info(
           "Inserting ClusterNode [{}] with queue wait time [{}] and "
               + "wait queue length [{}]",
@@ -295,12 +434,19 @@ protected void onExistingNodeUpdated(
 
     if (rmNode.getState() != NodeState.DECOMMISSIONING &&
         (estimatedQueueWaitTime != -1 ||
-            comparator == LoadComparator.QUEUE_LENGTH)) {
-      clusterNode
-          .setQueueWaitTime(estimatedQueueWaitTime)
-          .setQueueLength(waitQueueLength)
-          .setNodeLabels(rmNode.getNodeLabels())
-          .updateTimestamp();
+            comparator == LoadComparator.QUEUE_LENGTH ||
+            comparator == LoadComparator.QUEUE_LENGTH_THEN_RESOURCES)) {
+      final ClusterNode.Properties properties =
+          ClusterNode.Properties.newInstance()
+              .setQueueWaitTime(estimatedQueueWaitTime)
+              .setQueueLength(waitQueueLength)
+              .setNodeLabels(rmNode.getNodeLabels())
+              .setCapability(rmNode.getTotalCapability())
+              .setAllocatedResource(rmNode.getAllocatedContainerResource())
+              .updateTimestamp();
+
+      clusterNode.setProperties(properties);
+
       LOG.debug("Updating ClusterNode [{}] with queue wait time [{}] and"
               + " wait queue length [{}]", rmNode.getNodeID(),
           estimatedQueueWaitTime, waitQueueLength);
@@ -345,27 +491,47 @@ public List<NodeId> selectLeastLoadedNodes(int k) {
     }
   }
 
-  public RMNode selectLocalNode(String hostName, Set<String> blacklist) {
+  /**
+   * Selects the node as specified by hostName for resource allocation,
+   * unless the node has been blacklisted.
+   * @param hostName the hostname of the node for local resource allocation
+   * @param blacklist the blacklisted nodes
+   * @param request the requested resource
+   * @return the selected node, null if the node is full or is blacklisted
+   */
+  public RMNode selectLocalNode(
+      String hostName, Set<String> blacklist, Resource request) {
     if (blacklist.contains(hostName)) {
       return null;
     }
     RMNode node = nodeByHostName.get(hostName);
     if (node != null) {
       ClusterNode clusterNode = clusterNodes.get(node.getNodeID());
-      if (comparator.compareAndIncrement(clusterNode, 1)) {
+      if (clusterNode != null && comparator
+          .compareAndIncrement(clusterNode, 1, request)) {
         return node;
       }
     }
     return null;
   }
 
-  public RMNode selectRackLocalNode(String rackName, Set<String> blacklist) {
+  /**
+   * Selects a node from the rack as specified by rackName
+   * for resource allocation, excluding blacklisted nodes
+   * @param rackName the rack name for rack-local resource allocation
+   * @param blacklist the blacklisted nodes
+   * @param request the requested resource
+   * @return the selected node, null if no suitable nodes
+   */
+  public RMNode selectRackLocalNode(
+      String rackName, Set<String> blacklist, Resource request) {
     Set<NodeId> nodesOnRack = nodeIdsByRack.get(rackName);
     if (nodesOnRack != null) {
       for (NodeId nodeId : nodesOnRack) {
         if (!blacklist.contains(nodeId.getHost())) {
           ClusterNode node = clusterNodes.get(nodeId);
-          if (node != null && comparator.compareAndIncrement(node, 1)) {
+          if (node != null &&
+              comparator.compareAndIncrement(node, 1, request)) {
             return nodeByHostName.get(nodeId.getHost());
           }
         }
@@ -374,7 +540,14 @@ public RMNode selectRackLocalNode(String rackName, Set<String> blacklist) {
     return null;
   }
 
-  public RMNode selectAnyNode(Set<String> blacklist) {
+  /**
+   * Selects a node from all ClusterNodes for resource allocation,
+   * excluding blacklisted nodes.
+   * @param blacklist the blacklisted nodes
+   * @param request the requested resource
+   * @return the selected node, null if no suitable nodes
+   */
+  public RMNode selectAnyNode(Set<String> blacklist, Resource request) {
     List<NodeId> nodeIds = getCandidatesForSelectAnyNode();
     int size = nodeIds.size();
     if (size <= 0) {
@@ -388,7 +561,8 @@ public RMNode selectAnyNode(Set<String> blacklist) {
       NodeId nodeId = nodeIds.get(index);
       if (nodeId != null && !blacklist.contains(nodeId.getHost())) {
         ClusterNode node = clusterNodes.get(nodeId);
-        if (node != null && comparator.compareAndIncrement(node, 1)) {
+        if (node != null && comparator.compareAndIncrement(
+            node, 1, request)) {
           return nodeByHostName.get(nodeId.getHost());
         }
       }
@@ -402,7 +576,10 @@ protected List<NodeId> getCandidatesForSelectAnyNode() {
 
   protected void removeFromNodeIdsByRack(RMNode removedNode) {
     nodeIdsByRack.computeIfPresent(removedNode.getRackName(),
-        (k, v) -> v).remove(removedNode.getNodeID());
+        (k, v) -> {
+          v.remove(removedNode.getNodeID());
+          return v;
+        });
   }
 
   protected void addIntoNodeIdsByRack(RMNode addedNode) {
@@ -414,21 +591,21 @@ protected List<ClusterNode> sortNodes(boolean excludeFullNodes) {
     ReentrantReadWriteLock.ReadLock readLock = clusterNodesLock.readLock();
     readLock.lock();
     try {
-      ArrayList<ClusterNode> aList = new ArrayList<>(this.clusterNodes.values());
-      List<ClusterNode> retList = new ArrayList<>();
-      Object[] nodes = aList.toArray();
-      // Collections.sort would do something similar by calling Arrays.sort
-      // internally but would finally iterate through the input list (aList)
-      // to reset the value of each element. Since we don't really care about
-      // 'aList', we can use the iteration to create the list of nodeIds which
-      // is what we ultimately care about.
-      Arrays.sort(nodes, (Comparator)comparator);
-      for (int j=0; j < nodes.length; j++) {
-        ClusterNode cNode = (ClusterNode)nodes[j];
-        // Only add node to the result list when either condition is met:
-        // 1. we don't exclude full nodes
-        // 2. we do exclude full nodes, but the current node is not full
-        if (!excludeFullNodes || !cNode.isQueueFull()) {
+      final ClusterNode[] nodes = new ClusterNode[clusterNodes.size()];
+      int nodesIdx = 0;
+      final Resource clusterResource = Resource.newInstance(Resources.none());
+      for (final ClusterNode node : this.clusterNodes.values()) {
+        Resources.addTo(clusterResource, node.getCapability());
+        nodes[nodesIdx] = node;
+        nodesIdx++;
+      }
+
+      comparator.setClusterResource(clusterResource);
+
+      final List<ClusterNode> retList = new ArrayList<>();
+      Arrays.sort(nodes, comparator);
+      for (final ClusterNode cNode : nodes) {
+        if (!excludeFullNodes || comparator.isNodeAvailable(cNode)) {
           retList.add(cNode);
         }
       }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index ad8022a0d0845..7a5bad5d58024 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -17,17 +17,27 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.security.AccessType;
 import org.apache.hadoop.yarn.security.Permission;
@@ -39,19 +49,14 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueProperties;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.SystemClock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileQueueParser.EVERYBODY_ACL;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileQueueParser.ROOT;
 
@@ -236,8 +241,7 @@ public synchronized void reloadAllocations()
     LOG.info("Loading allocation file " + allocFile);
 
     // Read and parse the allocations file.
-    DocumentBuilderFactory docBuilderFactory =
-        DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
     docBuilderFactory.setIgnoringComments(true);
     DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
     Document doc = builder.parse(fs.open(allocFile));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index f5a0040f26091..3b7055c757c6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -666,11 +666,11 @@ private Container createContainer(FSSchedulerNode node, Resource capability,
   }
 
   @Override
-  public synchronized void recoverContainer(SchedulerNode node,
+  public synchronized boolean recoverContainer(SchedulerNode node,
       RMContainer rmContainer) {
     writeLock.lock();
     try {
-      super.recoverContainer(node, rmContainer);
+      final boolean recovered = super.recoverContainer(node, rmContainer);
 
       if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
         getQueue().incUsedResource(rmContainer.getContainer().getResource());
@@ -685,6 +685,8 @@ public synchronized void recoverContainer(SchedulerNode node,
         getQueue().addAMResourceUsage(resource);
         setAmRunning(true);
       }
+
+      return recovered;
     } finally {
       writeLock.unlock();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 7f25b07f2c66e..a865d7543dd3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -98,7 +98,7 @@
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture;
 
 import org.slf4j.Logger;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index fb5850d61b553..a4b2e2a1d5a60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -23,7 +23,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -40,8 +39,6 @@
 
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 
-import java.util.List;
-
 public class FifoAppAttempt extends FiCaSchedulerApp {
   private static final Logger LOG =
       LoggerFactory.getLogger(FifoAppAttempt.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
index 112c50ff1cc52..c62b738df5679 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoComparator.java
@@ -29,6 +29,11 @@ public class FifoComparator
     @Override
   public int compare(SchedulableEntity r1, SchedulableEntity r2) {
     int res = r1.compareInputOrderTo(r2);
+
+    if (res == 0) {
+      res = (int) Math.signum(r1.getStartTime() - r2.getStartTime());
+    }
+
     return res;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 9ce37ac028e63..1018dc818dbf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -103,7 +103,7 @@ private void renderLeafQueueInfoWithPartition(Block html) {
           ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : nodeLabel;
       // first display the queue's label specific details :
       ResponseInfo ri =
-          info("\'" + lqinfo.getQueuePath().substring(5)
+          info("\'" + lqinfo.getQueuePath()
               + "\' Queue Status for Partition \'" + nodeLabelDisplay + "\'");
       renderQueueCapacityInfo(ri, nodeLabel);
       html.__(InfoBlock.class);
@@ -113,7 +113,7 @@ private void renderLeafQueueInfoWithPartition(Block html) {
 
       // second display the queue specific details :
       ri =
-          info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status")
+          info("\'" + lqinfo.getQueuePath() + "\' Queue Status")
               .__("Queue State:", lqinfo.getQueueState());
       renderCommonLeafQueueInfo(ri);
 
@@ -125,7 +125,7 @@ private void renderLeafQueueInfoWithPartition(Block html) {
 
     private void renderLeafQueueInfoWithoutParition(Block html) {
       ResponseInfo ri =
-          info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status")
+          info("\'" + lqinfo.getQueuePath() + "\' Queue Status")
               .__("Queue State:", lqinfo.getQueueState());
       renderQueueCapacityInfo(ri, "");
       renderCommonLeafQueueInfo(ri);
@@ -348,7 +348,7 @@ public void render(Block html) {
               span().$style(join(width(usedCapPercent),
                 ";font-size:1px;left:0%;", absUsedCap > absCap ? Q_OVER : Q_UNDER)).
             __('.').__().
-              span(".q", "Queue: "+info.getQueuePath().substring(5)).__().
+              span(".q", info.getQueuePath()).__().
             span().$class("qstats").$style(left(Q_STATS_POS)).
             __(join(percent(used), " used")).__();
 
@@ -492,7 +492,7 @@ public void render(Block html) {
             a(_Q).$style(width(Q_MAX_WIDTH)).
               span().$style(join(width(used), ";left:0%;",
                   used > 1 ? Q_OVER : Q_UNDER)).__(".").__().
-              span(".q", "Queue: root").__().
+              span(".q", "root").__().
             span().$class("qstats").$style(left(Q_STATS_POS)).
               __(join(percent(used), " used")).__().
               __(QueueBlock.class).__();
@@ -522,7 +522,7 @@ public void render(Block html) {
             a(_Q).$style(width(Q_MAX_WIDTH)).
               span().$style(join(width(used), ";left:0%;",
                   used > 1 ? Q_OVER : Q_UNDER)).__(".").__().
-              span(".q", "Queue: root").__().
+              span(".q", "root").__().
             span().$class("qstats").$style(left(Q_STATS_POS)).
                 __(join(percent(used), " used")).__().
                 __(QueueBlock.class).__().__();
@@ -656,12 +656,9 @@ public void render(HtmlBlock.Block html) {
           "    }",
           "  });",
           "  $('#cs').bind('select_node.jstree', function(e, data) {",
-          "    var q = $('.q', data.rslt.obj).first().text();",
-          "    if (q == 'Queue: root') q = '';",
-          "    else {",
-          "      q = q.substr(q.lastIndexOf(':') + 2);",
-          "      q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
-          "    }",
+          "    var queues = $('.q', data.rslt.obj);",
+          "    var q = '^' + queues.first().text();",
+          "    q += queues.length == 1 ? '$' : '\\\\.';",
           // Update this filter column index for queue if new columns are added
           // Current index for queue column is 5
           "    $('#apps').dataTable().fnFilter(q, 5, true);",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index f6202cbcc5107..c74e2ae3e1bb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -129,6 +129,12 @@ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
     return true;
   }
 
+  private static String printAppInfo(long value) {
+    if (value == -1) {
+      return "N/A";
+    }
+    return String.valueOf(value);
+  }
 
   @Override public void render(Block html) {
     TBODY<TABLE<Hamlet>> tbody = html.
@@ -193,16 +199,16 @@ protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
       .append(appInfo.getFinishTime()).append("\",\"")
       .append(appInfo.getState()).append("\",\"")
       .append(appInfo.getFinalStatus()).append("\",\"")
-      .append(appInfo.getRunningContainers() == -1 ? "N/A" : String
-         .valueOf(appInfo.getRunningContainers())).append("\",\"")
-      .append(appInfo.getAllocatedVCores() == -1 ? "N/A" : String
-        .valueOf(appInfo.getAllocatedVCores())).append("\",\"")
-      .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
-        .valueOf(appInfo.getAllocatedMB())).append("\",\"")
-      .append(appInfo.getReservedVCores() == -1 ? "N/A" : String
-        .valueOf(appInfo.getReservedVCores())).append("\",\"")
-      .append(appInfo.getReservedMB() == -1 ? "N/A" : String
-        .valueOf(appInfo.getReservedMB())).append("\",\"")
+      .append(printAppInfo(appInfo.getRunningContainers()))
+      .append("\",\"")
+      .append(printAppInfo(appInfo.getAllocatedVCores()))
+      .append("\",\"")
+      .append(printAppInfo(appInfo.getAllocatedMB()))
+      .append("\",\"")
+      .append(printAppInfo(appInfo.getReservedVCores()))
+      .append("\",\"")
+      .append(printAppInfo(appInfo.getReservedMB()))
+      .append("\",\"")
       // Progress bar
       .append("<br title='").append(percent)
       .append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
index d6d7e54c34da7..c17d4f6d7b0e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java
@@ -21,7 +21,6 @@
 import com.google.inject.Inject;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
 import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
index b492793927d3e..7bf8a63786c62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
@@ -24,6 +24,9 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.Collection;
+import java.util.HashSet;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -58,6 +61,11 @@ public NodeIDsInfo(List<String> nodeIdsList, Resource resource) {
     this.partitionInfo = new PartitionInfo(new ResourceInfo(resource));
   }
 
+  public NodeIDsInfo(Collection<String> nodeIdsList, PartitionInfo partitionInfo) {
+    this.nodeIDsList.addAll(nodeIdsList);
+    this.partitionInfo = partitionInfo;
+  }
+
   public ArrayList<String> getNodeIDs() {
     return nodeIDsList;
   }
@@ -65,4 +73,36 @@ public ArrayList<String> getNodeIDs() {
   public PartitionInfo getPartitionInfo() {
     return partitionInfo;
   }
+
+  /**
+   * This method will generate a new NodeIDsInfo object based on the two NodeIDsInfo objects.
+   * The information to be combined includes the node list (removed duplicate node)
+   * and partitionInfo object.
+   *
+   * @param left left NodeIDsInfo Object.
+   * @param right right NodeIDsInfo Object.
+   * @return new NodeIDsInfo Object.
+   */
+  public static NodeIDsInfo add(NodeIDsInfo left, NodeIDsInfo right) {
+    Set<String> nodes = new HashSet<>();
+    if (left != null && left.nodeIDsList != null) {
+      nodes.addAll(left.nodeIDsList);
+    }
+    if (right != null && right.nodeIDsList != null) {
+      nodes.addAll(right.nodeIDsList);
+    }
+
+    PartitionInfo leftPartitionInfo = null;
+    if (left != null) {
+      leftPartitionInfo = left.getPartitionInfo();
+    }
+
+    PartitionInfo rightPartitionInfo = null;
+    if (right != null) {
+      rightPartitionInfo = right.getPartitionInfo();
+    }
+
+    PartitionInfo info = PartitionInfo.addTo(leftPartitionInfo, rightPartitionInfo);
+    return new NodeIDsInfo(nodes, info);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
index 82ceed37c2d5d..de75631486b92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
@@ -57,6 +57,9 @@ public final class RMWSConsts {
   /** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
   public static final String SCHEDULER_LOGS = "/scheduler/logs";
 
+  /** Path for {@code RMWebServiceProtocol#getSchedulerOverview}. */
+  public static final String SCHEDULER_OVERVIEW = "/scheduler-overview";
+
   /**
    * Path for {@code RMWebServiceProtocol#validateAndGetSchedulerConfiguration}.
    */
@@ -123,10 +126,13 @@ public final class RMWSConsts {
   /** Path for {@code RMWebServiceProtocol#getClusterNodeLabels}. */
   public static final String GET_NODE_LABELS = "/get-node-labels";
 
+  /** Path for {@code RMWebServiceProtocol#getRMNodeLabels}. */
+  public static final String GET_RM_NODE_LABELS = "/get-rm-node-labels";
+
   /** Path for {@code RMWebServiceProtocol#addToClusterNodeLabels}. */
   public static final String ADD_NODE_LABELS = "/add-node-labels";
 
-  /** Path for {@code RMWebServiceProtocol#removeFromCluserNodeLabels}. */
+  /** Path for {@code RMWebServiceProtocol#removeFromClusterNodeLabels}. */
   public static final String REMOVE_NODE_LABELS = "/remove-node-labels";
 
   /** Path for {@code RMWebServiceProtocol#getLabelsOnNode}. */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 6870317f9ae30..e70266f368a8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -47,7 +47,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
 import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.slf4j.Logger;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
index f2736e3773c1b..43b5307279b83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
@@ -62,6 +62,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 
 /**
  * <p>
@@ -324,6 +325,8 @@ Response updateAppState(AppState targetState, HttpServletRequest hsr,
    */
   NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException;
 
+  NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) throws IOException;
+
   /**
    * This method retrieves all the node within multiple node labels in the
    * cluster, and it is reachable by using {@link RMWSConsts#LABEL_MAPPINGS}.
@@ -398,7 +401,7 @@ Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
    * @return Response containing the status code
    * @throws Exception in case of bad request
    */
-  Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+  Response removeFromClusterNodeLabels(Set<String> oldNodeLabels,
       HttpServletRequest hsr) throws Exception;
 
   /**
@@ -745,4 +748,31 @@ RMQueueAclInfo checkUserAccessToQueue(String queue, String username,
    */
   Response signalToContainer(String containerId, String command,
       HttpServletRequest req) throws AuthorizationException;
+
+  /**
+   * This method updates the Scheduler configuration, and it is reachable by
+   * using {@link RMWSConsts#SCHEDULER_CONF}.
+   *
+   * @param mutationInfo th information for making scheduler configuration
+   *        changes (supports adding, removing, or updating a queue, as well
+   *        as global scheduler conf changes)
+   * @param hsr the servlet request
+   * @return Response containing the status code
+   * @throws AuthorizationException if the user is not authorized to invoke this
+   *         method
+   * @throws InterruptedException if interrupted
+   */
+  Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest hsr) throws AuthorizationException, InterruptedException;
+
+  /**
+   * This method retrieves all the Scheduler configuration, and it is reachable
+   * by using {@link RMWSConsts#SCHEDULER_CONF}.
+   *
+   * @param hsr the servlet request
+   * @return Response containing the status code
+   * @throws AuthorizationException if the user is not authorized to invoke this
+   *         method.
+   */
+  Response getSchedulerConfiguration(HttpServletRequest hsr) throws AuthorizationException;
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 314b031208127..df247b06afa9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -129,6 +129,7 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
@@ -138,6 +139,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -202,6 +204,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.server.webapp.WebServices;
@@ -1403,6 +1406,32 @@ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr)
     return new NodeLabelsInfo(nodeLabelsInfo);
   }
 
+  @GET
+  @Path(RMWSConsts.GET_RM_NODE_LABELS)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public NodeLabelsInfo getRMNodeLabels(@Context HttpServletRequest hsr)
+      throws IOException {
+
+    initForReadableEndpoints();
+    RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager();
+
+    ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<>();
+    for (RMNodeLabel info : nlm.pullRMNodeLabelsInfo()) {
+      String labelName = info.getLabelName().isEmpty() ?
+          NodeLabel.DEFAULT_NODE_LABEL_PARTITION : info.getLabelName();
+      int activeNMs = info.getNumActiveNMs();
+      PartitionInfo partitionInfo =
+          new PartitionInfo(new ResourceInfo(info.getResource()));
+      NodeLabel nodeLabel = NodeLabel.newInstance(labelName, info.getIsExclusive());
+      NodeLabelInfo nodeLabelInfo = new NodeLabelInfo(nodeLabel, partitionInfo);
+      nodeLabelInfo.setActiveNMs(activeNMs);
+      nodeLabelsInfo.add(nodeLabelInfo);
+    }
+
+    return new NodeLabelsInfo(nodeLabelsInfo);
+  }
+
   @POST
   @Path(RMWSConsts.ADD_NODE_LABELS)
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -1435,7 +1464,7 @@ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
       MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
   @Override
-  public Response removeFromCluserNodeLabels(
+  public Response removeFromClusterNodeLabels(
       @QueryParam(RMWSConsts.LABELS) Set<String> oldNodeLabels,
       @Context HttpServletRequest hsr) throws Exception {
     UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
@@ -2656,8 +2685,7 @@ public Response formatSchedulerConfiguration(@Context HttpServletRequest hsr)
     initForWritableEndpoints(callerUGI, true);
 
     ResourceScheduler scheduler = rm.getResourceScheduler();
-    if (scheduler instanceof MutableConfScheduler
-        && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+    if (isConfigurationMutable(scheduler)) {
       try {
         MutableConfigurationProvider mutableConfigurationProvider =
             ((MutableConfScheduler) scheduler).getMutableConfProvider();
@@ -2679,7 +2707,7 @@ public Response formatSchedulerConfiguration(@Context HttpServletRequest hsr)
     } else {
       return Response.status(Status.BAD_REQUEST)
           .entity("Scheduler Configuration format only supported by " +
-          "MutableConfScheduler.").build();
+              MutableConfScheduler.class.getSimpleName()).build();
     }
   }
 
@@ -2696,8 +2724,7 @@ public synchronized Response validateAndGetSchedulerConfiguration(
     UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     initForWritableEndpoints(callerUGI, true);
     ResourceScheduler scheduler = rm.getResourceScheduler();
-    if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler)
-            scheduler).isConfigurationMutable()) {
+    if (isConfigurationMutable(scheduler)) {
       try {
         MutableConfigurationProvider mutableConfigurationProvider =
                 ((MutableConfScheduler) scheduler).getMutableConfProvider();
@@ -2729,8 +2756,8 @@ public synchronized Response validateAndGetSchedulerConfiguration(
                   .build();
       }
     } else {
-      String errorMsg = "Configuration change validation only supported by " +
-              "MutableConfScheduler.";
+      String errorMsg = String.format("Configuration change validation only supported by %s.",
+          MutableConfScheduler.class.getSimpleName());
       LOG.warn(errorMsg);
       return Response.status(Status.BAD_REQUEST)
               .entity(errorMsg)
@@ -2746,51 +2773,61 @@ public synchronized Response validateAndGetSchedulerConfiguration(
   public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo
       mutationInfo, @Context HttpServletRequest hsr)
       throws AuthorizationException, InterruptedException {
-
     UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     initForWritableEndpoints(callerUGI, true);
 
     ResourceScheduler scheduler = rm.getResourceScheduler();
-    if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler)
-        scheduler).isConfigurationMutable()) {
+    if (!(scheduler instanceof MutableConfScheduler)) {
+      return Response.status(Status.BAD_REQUEST)
+          .entity("Configuration change only supported by MutableConfScheduler.").build();
+    } else if (!((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+      return Response.status(Status.BAD_REQUEST)
+          .entity("Configuration change only supported by mutable configuration store.").build();
+    } else {
       try {
-        callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
-          @Override
-          public Void run() throws Exception {
-            MutableConfigurationProvider provider = ((MutableConfScheduler)
-                scheduler).getMutableConfProvider();
-            if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI,
-                mutationInfo)) {
-              throw new org.apache.hadoop.security.AccessControlException("User"
-                  + " is not admin of all modified queues.");
-            }
-            LogMutation logMutation = provider.logAndApplyMutation(callerUGI,
-                mutationInfo);
-            try {
-              rm.getRMContext().getRMAdminService().refreshQueues();
-            } catch (IOException | YarnException e) {
-              provider.confirmPendingMutation(logMutation, false);
-              throw e;
-            }
-            provider.confirmPendingMutation(logMutation, true);
-            return null;
-          }
+        callerUGI.doAs((PrivilegedExceptionAction<Void>) () -> {
+          MutableConfigurationProvider provider =
+              ((MutableConfScheduler) scheduler).getMutableConfProvider();
+          LogMutation logMutation = applyMutation(provider, callerUGI, mutationInfo);
+          return refreshQueues(provider, logMutation);
         });
       } catch (IOException e) {
         LOG.error("Exception thrown when modifying configuration.", e);
-        return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
-            .build();
+        return Response.status(Status.BAD_REQUEST).entity(e.getMessage()).build();
       }
-      return Response.status(Status.OK).entity("Configuration change " +
-          "successfully applied.").build();
-    } else {
-      return Response.status(Status.BAD_REQUEST)
-          .entity("Configuration change only supported by " +
-              "MutableConfScheduler.")
+      return Response.status(Status.OK).entity("Configuration change successfully applied.")
           .build();
     }
   }
 
+  private Void refreshQueues(MutableConfigurationProvider provider, LogMutation logMutation)
+      throws Exception {
+    try {
+      rm.getRMContext().getRMAdminService().refreshQueues();
+    } catch (IOException | YarnException e) {
+      provider.confirmPendingMutation(logMutation, false);
+      throw e;
+    }
+    provider.confirmPendingMutation(logMutation, true);
+    return null;
+  }
+
+  private LogMutation applyMutation(MutableConfigurationProvider provider,
+      UserGroupInformation callerUGI, SchedConfUpdateInfo mutationInfo) throws Exception {
+    if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI,
+        mutationInfo)) {
+      throw new org.apache.hadoop.security.AccessControlException("User"
+          + " is not admin of all modified queues.");
+    }
+    return provider.logAndApplyMutation(callerUGI,
+        mutationInfo);
+  }
+
+  private boolean isConfigurationMutable(ResourceScheduler scheduler) {
+    return scheduler instanceof MutableConfScheduler && ((MutableConfScheduler)
+        scheduler).isConfigurationMutable();
+  }
+
   @GET
   @Path(RMWSConsts.SCHEDULER_CONF)
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@@ -2803,8 +2840,7 @@ public Response getSchedulerConfiguration(@Context HttpServletRequest hsr)
     initForWritableEndpoints(callerUGI, true);
 
     ResourceScheduler scheduler = rm.getResourceScheduler();
-    if (scheduler instanceof MutableConfScheduler
-        && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+    if (isConfigurationMutable(scheduler)) {
       MutableConfigurationProvider mutableConfigurationProvider =
           ((MutableConfScheduler) scheduler).getMutableConfProvider();
       // We load the cached configuration from configuration store,
@@ -2816,10 +2852,11 @@ public Response getSchedulerConfiguration(@Context HttpServletRequest hsr)
           .build();
     } else {
       return Response.status(Status.BAD_REQUEST).entity(
-          "This API only supports to retrieve scheduler configuration"
-              + " from a mutable-conf scheduler, underneath scheduler "
-              + scheduler.getClass().getSimpleName()
-              + " is not an instance of MutableConfScheduler")
+              String.format("This API only supports to retrieve scheduler configuration"
+                  + " from a mutable-conf scheduler, underneath scheduler %s"
+                  + " is not an instance of %s",
+                  scheduler.getClass().getSimpleName(),
+                  MutableConfScheduler.class.getSimpleName()))
           .build();
     }
   }
@@ -2835,8 +2872,7 @@ public Response getSchedulerConfigurationVersion(@Context
     initForWritableEndpoints(callerUGI, true);
 
     ResourceScheduler scheduler = rm.getResourceScheduler();
-    if (scheduler instanceof MutableConfScheduler
-        && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+    if (isConfigurationMutable(scheduler)) {
       MutableConfigurationProvider mutableConfigurationProvider =
           ((MutableConfScheduler) scheduler).getMutableConfProvider();
 
@@ -2852,8 +2888,8 @@ public Response getSchedulerConfigurationVersion(@Context
       }
     } else {
       return Response.status(Status.BAD_REQUEST)
-          .entity("Configuration Version only supported by "
-          + "MutableConfScheduler.").build();
+          .entity(String.format("Configuration Version only supported by %s.",
+              MutableConfScheduler.class.getSimpleName())).build();
     }
   }
 
@@ -2935,4 +2971,14 @@ public Response signalToContainer(
     }
     return Response.status(Status.OK).build();
   }
+
+  @GET
+  @Path(RMWSConsts.SCHEDULER_OVERVIEW)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public SchedulerOverviewInfo getSchedulerOverview() {
+    initForReadableEndpoints();
+    ResourceScheduler rs = rm.getResourceScheduler();
+    return new SchedulerOverviewInfo(rs);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 6ebaa06174eff..481544e477c72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -27,6 +27,8 @@
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -38,6 +40,8 @@
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.SchedulingRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -69,6 +73,8 @@ public class AppInfo {
   protected ApplicationId applicationId;
   @XmlTransient
   private String schemePrefix;
+  @XmlTransient
+  private SubClusterIdInfo subClusterId;
 
   // these are ok for any user to see
   protected String id;
@@ -82,6 +88,7 @@ public class AppInfo {
   protected String trackingUrl;
   protected String diagnostics;
   protected long clusterId;
+  protected String rmClusterId;
   protected String applicationType;
   protected String applicationTags = "";
   protected int priority;
@@ -182,6 +189,16 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
       this.finalStatus = app.getFinalApplicationStatus();
       this.clusterId = ResourceManager.getClusterTimeStamp();
       if (hasAccess) {
+        if (rm != null && rm.getConfig() != null) {
+          try {
+            Configuration yarnConfig = rm.getConfig();
+            subClusterId = new SubClusterIdInfo(YarnConfiguration.getClusterId(yarnConfig));
+            rmClusterId = this.subClusterId.toId().toString();
+          } catch (HadoopIllegalArgumentException e) {
+            subClusterId = null;
+            rmClusterId = null;
+          }
+        }
         this.startedTime = app.getStartTime();
         this.launchTime = app.getLaunchTime();
         this.finishedTime = app.getFinishTime();
@@ -454,6 +471,14 @@ public long getClusterId() {
     return this.clusterId;
   }
 
+  public SubClusterIdInfo getSubClusterIdInfo() {
+    return this.subClusterId;
+  }
+
+  public String getRmClusterId() {
+    return this.rmClusterId;
+  }
+
   public String getApplicationType() {
     return this.applicationType;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
index 140857e4063df..460309bc297f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java
@@ -22,6 +22,7 @@
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 
 /**
@@ -45,6 +46,12 @@ public AppTimeoutInfo() {
     remainingTimeInSec = -1;
   }
 
+  public AppTimeoutInfo(ApplicationTimeout applicationTimeout) {
+    this.expiryTime = applicationTimeout.getExpiryTime();
+    this.remainingTimeInSec = applicationTimeout.getRemainingTime();
+    this.timeoutType = applicationTimeout.getTimeoutType();
+  }
+
   public ApplicationTimeoutType getTimeoutType() {
     return timeoutType;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java
index b77ffb4ed60f1..7b7066a731855 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
 import java.util.ArrayList;
+import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -33,6 +34,10 @@ public class ApplicationStatisticsInfo {
   public ApplicationStatisticsInfo() {
   } // JAXB needs this
 
+  public ApplicationStatisticsInfo(Collection<StatisticsItemInfo> items) {
+    statItem.addAll(items);
+  }
+
   public void add(StatisticsItemInfo statItem) {
     this.statItem.add(statItem);
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 79d11e2b96126..83394b15a37d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -24,20 +24,19 @@
 import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
 
 import java.util.ArrayList;
-import java.util.Map;
 import java.util.List;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo.getSortedQueueAclInfoList;
+
 @XmlRootElement(name = "capacityScheduler")
 @XmlType(name = "capacityScheduler")
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -50,6 +49,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
   protected float normalizedWeight;
   protected String queueName;
   private String queuePath;
+  protected int maxParallelApps;
+  private boolean isAbsoluteResource;
   protected CapacitySchedulerQueueInfoList queues;
   protected QueueCapacitiesInfo capacities;
   protected CapacitySchedulerHealthInfo health;
@@ -83,6 +84,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
     this.maxCapacity = max * 100;
     this.weight = parent.getQueueCapacities().getWeight();
     this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight();
+    this.maxParallelApps = parent.getMaxParallelApps();
 
     capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
         parent.getQueueResourceQuotas(), false);
@@ -90,23 +92,12 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
     health = new CapacitySchedulerHealthInfo(cs);
     maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
 
+    isAbsoluteResource = parent.getCapacityConfigType() ==
+        AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
     CapacitySchedulerConfiguration conf = cs.getConfiguration();
     queueAcls = new QueueAclsInfo();
-    for (Map.Entry<AccessType, AccessControlList> e : conf
-        .getAcls(queueName).entrySet()) {
-      QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
-          e.getValue().getAclString());
-      queueAcls.add(queueAcl);
-    }
-
-    String aclApplicationMaxPriority = "acl_" +
-        StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
-    String priorityAcls = conf.get(parent.getQueuePath()
-        + aclApplicationMaxPriority, conf.ALL_ACL);
-
-    QueueAclInfo queueAcl = new QueueAclInfo(
-        AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
-    queueAcls.add(queueAcl);
+    queueAcls.addAll(getSortedQueueAclInfoList(parent, queueName, conf));
 
     queuePriority = parent.getPriority().getPriority();
     if (parent instanceof ParentQueue) {
@@ -188,7 +179,7 @@ protected CapacitySchedulerQueueInfoList getQueues(
     List<CSQueue> childLeafQueues = new ArrayList<>();
     List<CSQueue> childNonLeafQueues = new ArrayList<>();
     for (CSQueue queue : parent.getChildQueues()) {
-      if (queue instanceof LeafQueue) {
+      if (queue instanceof AbstractLeafQueue) {
         childLeafQueues.add(queue);
       } else {
         childNonLeafQueues.add(queue);
@@ -199,8 +190,8 @@ protected CapacitySchedulerQueueInfoList getQueues(
 
     for (CSQueue queue : childQueues) {
       CapacitySchedulerQueueInfo info;
-      if (queue instanceof LeafQueue) {
-        info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
+      if (queue instanceof AbstractLeafQueue) {
+        info = new CapacitySchedulerLeafQueueInfo(cs, (AbstractLeafQueue) queue);
       } else {
         info = new CapacitySchedulerQueueInfo(cs, queue);
         info.queues = getQueues(cs, queue);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 4e9ced8beeaaf..5b1da1977bf55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -27,10 +27,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
     .AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
 
@@ -63,7 +63,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   CapacitySchedulerLeafQueueInfo() {
   };
 
-  CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) {
+  CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, AbstractLeafQueue q) {
     super(cs, q);
     numActiveApplications = q.getNumActiveApplications();
     numPendingApplications = q.getNumPendingApplications();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 4cbb24d9d86ae..bda69c1d0543e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Map;
 import java.util.Set;
 
@@ -34,6 +35,7 @@
 import org.apache.hadoop.yarn.security.AccessType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -42,11 +44,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
-    CapacitySchedulerConfiguration.RESOURCE_PATTERN;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
-    CapacitySchedulerConfiguration.CAPACITY;
-
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.FIELD)
 @XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@@ -160,21 +157,7 @@ public class CapacitySchedulerQueueInfo {
 
     CapacitySchedulerConfiguration conf = cs.getConfiguration();
     queueAcls = new QueueAclsInfo();
-    for (Map.Entry<AccessType, AccessControlList> e : conf
-        .getAcls(queuePath).entrySet()) {
-      QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
-          e.getValue().getAclString());
-      queueAcls.add(queueAcl);
-    }
-
-    String aclApplicationMaxPriority = "acl_" +
-        StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
-    String priorityAcls = conf.get(CapacitySchedulerConfiguration
-        .getQueuePrefix(queuePath) + aclApplicationMaxPriority, conf.ALL_ACL);
-
-    QueueAclInfo queueAcl = new QueueAclInfo(
-        AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
-    queueAcls.add(queueAcl);
+    queueAcls.addAll(getSortedQueueAclInfoList(q, queuePath, conf));
 
     queuePriority = q.getPriority().getPriority();
     if (q instanceof ParentQueue) {
@@ -192,16 +175,37 @@ public class CapacitySchedulerQueueInfo {
               .getLeafOnlyProperties());
     }
 
-    String configuredCapacity = conf.get(
-        CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
-    isAbsoluteResource = (configuredCapacity != null)
-        && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+    isAbsoluteResource = q.getCapacityConfigType() ==
+        AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
 
     autoCreateChildQueueEnabled = conf.
         isAutoCreateChildQueueEnabled(queuePath);
     leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath);
   }
 
+  public static ArrayList<QueueAclInfo> getSortedQueueAclInfoList(
+      CSQueue queue, String queuePath, CapacitySchedulerConfiguration conf) {
+    ArrayList<QueueAclInfo> queueAclsInfo = new ArrayList<>();
+    for (Map.Entry<AccessType, AccessControlList> e :
+        ((AbstractCSQueue) queue).getACLs().entrySet()) {
+      QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
+          e.getValue().getAclString());
+      queueAclsInfo.add(queueAcl);
+    }
+
+    String aclApplicationMaxPriority = "acl_" +
+        StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
+    String priorityAcls = conf.get(CapacitySchedulerConfiguration
+        .getQueuePrefix(queuePath) + aclApplicationMaxPriority,
+        CapacitySchedulerConfiguration.ALL_ACL);
+
+    QueueAclInfo queueAcl = new QueueAclInfo(
+        AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
+    queueAclsInfo.add(queueAcl);
+    queueAclsInfo.sort(Comparator.comparing(QueueAclInfo::getAccessType));
+    return queueAclsInfo;
+  }
+
   protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
     resources = new ResourcesInfo(queueResourceUsage, false);
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
index d815315b26027..a429215dd5d79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
 
 @XmlRootElement
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java
index e842d4245080d..d8849f0d105f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java
@@ -31,8 +31,7 @@
 @XmlAccessorType(XmlAccessType.FIELD)
 public class LabelsToNodesInfo {
 
-  protected Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes =
-    new HashMap<NodeLabelInfo, NodeIDsInfo>();
+  private Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes = new HashMap<>();
 
   public LabelsToNodesInfo() {
   } // JAXB needs this
@@ -44,4 +43,8 @@ public LabelsToNodesInfo(Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes) {
   public Map<NodeLabelInfo, NodeIDsInfo> getLabelsToNodes() {
     return labelsToNodes;
   }
+
+  public void setLabelsToNodes(Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes) {
+    this.labelsToNodes = labelsToNodes;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index ad7de729f035c..6f36b52627bb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -68,6 +68,7 @@ public class NodeInfo {
   protected ResourceInfo availableResource;
   protected NodeAttributesInfo nodeAttributesInfo;
   private ResourceInfo totalResource;
+  private String subClusterId;
 
   public NodeInfo() {
   } // JAXB needs this
@@ -287,4 +288,12 @@ public void setTotalResource(ResourceInfo total) {
   public ResourceInfo getTotalResource() {
     return this.totalResource;
   }
+
+  public String getSubClusterId() {
+    return subClusterId;
+  }
+
+  public void setSubClusterId(String subClusterId) {
+    this.subClusterId = subClusterId;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
index ce47471219411..1b451c797817f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
@@ -31,6 +31,7 @@ public class NodeLabelInfo {
   private String name;
   private boolean exclusivity;
   private PartitionInfo partitionInfo;
+  private Integer activeNMs;
 
   public NodeLabelInfo() {
     // JAXB needs this
@@ -68,6 +69,26 @@ public PartitionInfo getPartitionInfo() {
     return partitionInfo;
   }
 
+  public Integer getActiveNMs() {
+    return activeNMs;
+  }
+
+  public void setActiveNMs(Integer activeNMs) {
+    this.activeNMs = activeNMs;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public void setExclusivity(boolean exclusivity) {
+    this.exclusivity = exclusivity;
+  }
+
+  public void setPartitionInfo(PartitionInfo partitionInfo) {
+    this.partitionInfo = partitionInfo;
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (this == obj) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
index 2c3a8a507ab57..c9809b6d2e3cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
@@ -32,8 +32,7 @@
 public class NodeLabelsInfo {
 
   @XmlElement(name = "nodeLabelInfo")
-  private ArrayList<NodeLabelInfo> nodeLabelsInfo =
-    new ArrayList<NodeLabelInfo>();
+  private ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<>();
 
   public NodeLabelsInfo() {
     // JAXB needs this
@@ -44,25 +43,32 @@ public NodeLabelsInfo(ArrayList<NodeLabelInfo> nodeLabels) {
   }
 
   public NodeLabelsInfo(List<NodeLabel> nodeLabels) {
-    this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
+    this.nodeLabelsInfo = new ArrayList<>();
     for (NodeLabel label : nodeLabels) {
       this.nodeLabelsInfo.add(new NodeLabelInfo(label));
     }
   }
   
   public NodeLabelsInfo(Set<String> nodeLabelsName) {
-    this.nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
+    this.nodeLabelsInfo = new ArrayList<>();
     for (String labelName : nodeLabelsName) {
       this.nodeLabelsInfo.add(new NodeLabelInfo(labelName));
     }
   }
 
+  public NodeLabelsInfo(Collection<NodeLabel> nodeLabels) {
+    this.nodeLabelsInfo = new ArrayList<>();
+    nodeLabels.stream().forEach(nodeLabel -> {
+      this.nodeLabelsInfo.add(new NodeLabelInfo(nodeLabel));
+    });
+  }
+
   public ArrayList<NodeLabelInfo> getNodeLabelsInfo() {
     return nodeLabelsInfo;
   }
 
   public Set<NodeLabel> getNodeLabels() {
-    Set<NodeLabel> nodeLabels = new HashSet<NodeLabel>();
+    Set<NodeLabel> nodeLabels = new HashSet<>();
     for (NodeLabelInfo label : nodeLabelsInfo) {
       nodeLabels.add(NodeLabel.newInstance(label.getName(),
           label.getExclusivity()));
@@ -71,7 +77,7 @@ public Set<NodeLabel> getNodeLabels() {
   }
   
   public List<String> getNodeLabelsName() {
-    ArrayList<String> nodeLabelsName = new ArrayList<String>();
+    ArrayList<String> nodeLabelsName = new ArrayList<>();
     for (NodeLabelInfo label : nodeLabelsInfo) {
       nodeLabelsName.add(label.getName());
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
index 0b6e4bc868c74..e9044b9397cfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
@@ -35,7 +35,17 @@ public NodeToLabelsInfo() {
     // JAXB needs this
   }
 
+  public NodeToLabelsInfo(HashMap<String, NodeLabelsInfo> nodeToLabels) {
+    if (nodeToLabels != null) {
+      this.nodeToLabels.putAll(nodeToLabels);
+    }
+  }
+
   public HashMap<String, NodeLabelsInfo> getNodeToLabels() {
     return nodeToLabels;
   }
+
+  public void setNodeToLabels(HashMap<String, NodeLabelsInfo> nodeToLabels) {
+    this.nodeToLabels = nodeToLabels;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java
index 8174be0ad0f6d..6b66e0e1d70ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodesInfo.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
 import java.util.ArrayList;
+import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -43,4 +44,8 @@ public ArrayList<NodeInfo> getNodes() {
   public void addAll(ArrayList<NodeInfo> nodesInfo) {
     node.addAll(nodesInfo);
   }
+
+  public void addAll(Collection<NodeInfo> nodesInfo) {
+    node.addAll(nodesInfo);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
index 95baf74a18621..c0742b1cfaad4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
@@ -43,4 +46,29 @@ public PartitionInfo(ResourceInfo resourceAvailable) {
   public ResourceInfo getResourceAvailable() {
     return resourceAvailable;
   }
+
+  /**
+   * This method will generate a new PartitionInfo object based on two PartitionInfo objects.
+   * The combination process is mainly based on the Resources.add method.
+   *
+   * @param left left PartitionInfo Object.
+   * @param right right PartitionInfo Object.
+   * @return new PartitionInfo Object.
+   */
+  public static PartitionInfo addTo(PartitionInfo left, PartitionInfo right) {
+    Resource leftResource = Resource.newInstance(0, 0);
+    if (left != null && left.getResourceAvailable() != null) {
+      ResourceInfo leftResourceInfo = left.getResourceAvailable();
+      leftResource = leftResourceInfo.getResource();
+    }
+
+    Resource rightResource = Resource.newInstance(0, 0);
+    if (right != null && right.getResourceAvailable() != null) {
+      ResourceInfo rightResourceInfo = right.getResourceAvailable();
+      rightResource = rightResourceInfo.getResource();
+    }
+
+    Resource resource = Resources.addTo(leftResource, rightResource);
+    return new PartitionInfo(new ResourceInfo(resource));
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RMQueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RMQueueAclInfo.java
index 8799cab535589..8432fd2ef3ec2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RMQueueAclInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/RMQueueAclInfo.java
@@ -25,9 +25,10 @@
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.FIELD)
 public class RMQueueAclInfo {
-  protected boolean allowed;
-  protected String user;
-  protected String diagnostics;
+  private Boolean allowed;
+  private String user;
+  private String diagnostics;
+  private String subClusterId;
 
   public RMQueueAclInfo() {
     
@@ -39,6 +40,13 @@ public RMQueueAclInfo(boolean allowed, String user, String diagnostics) {
     this.diagnostics = diagnostics;
   }
 
+  public RMQueueAclInfo(boolean allowed, String user, String diagnostics, String subClusterId) {
+    this.allowed = allowed;
+    this.user = user;
+    this.diagnostics = diagnostics;
+    this.subClusterId = subClusterId;
+  }
+
   public boolean isAllowed() {
     return allowed;
   }
@@ -62,4 +70,12 @@ public String getDiagnostics() {
   public void setDiagnostics(String diagnostics) {
     this.diagnostics = diagnostics;
   }
+
+  public String getSubClusterId() {
+    return subClusterId;
+  }
+
+  public void setSubClusterId(String subClusterId) {
+    this.subClusterId = subClusterId;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index 67369c8fa3ac7..707452fbadfd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -72,6 +72,10 @@ public String toString() {
     return getResource().toString();
   }
 
+  public String toFormattedString() {
+    return getResource().toFormattedString();
+  }
+
   public void setMemory(int memory) {
     if (resources == null) {
       resources = Resource.newInstance(memory, vCores);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
index 1d96f79081e31..775810348c291 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourcesInfo.java
@@ -43,7 +43,7 @@ public ResourcesInfo(ResourceUsage resourceUsage,
     if (resourceUsage == null) {
       return;
     }
-    for (String partitionName : resourceUsage.getNodePartitionsSet()) {
+    for (String partitionName : resourceUsage.getExistingNodeLabels()) {
       resourceUsagesByPartition.add(new PartitionResourcesInfo(partitionName,
           new ResourceInfo(resourceUsage.getUsed(partitionName)),
           new ResourceInfo(resourceUsage.getReserved(partitionName)),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java
new file mode 100644
index 0000000000000..06df27dc4c409
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerOverviewInfo.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.List;
+
+@XmlRootElement(name = "scheduler")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedulerOverviewInfo {
+
+  private String schedulerType;
+  private String schedulingResourceType;
+  private ResourceInfo minimumAllocation;
+  private ResourceInfo maximumAllocation;
+  private int applicationPriority;
+  private int schedulerBusy;
+  private int rmDispatcherEventQueueSize;
+  private int schedulerDispatcherEventQueueSize;
+
+  // JAXB needs this
+  public SchedulerOverviewInfo() {
+
+  }
+
+  public SchedulerOverviewInfo(ResourceScheduler rs) {
+    // Parse the schedule type
+    this.schedulerType = getSchedulerName(rs);
+
+    // Parse and allocate resource information
+    this.minimumAllocation = new ResourceInfo(rs.getMinimumResourceCapability());
+    this.maximumAllocation = new ResourceInfo(rs.getMaximumResourceCapability());
+
+    // Parse App Priority
+    this.applicationPriority = rs.getMaxClusterLevelAppPriority().getPriority();
+
+    // Resolving resource types
+    List<ResourceTypeInfo> resourceTypeInfos = ResourceUtils.getResourcesTypeInfo();
+    resourceTypeInfos.sort((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName()));
+    this.schedulingResourceType = StringUtils.join(resourceTypeInfos, ",");
+
+    // clusterMetrics
+    ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(rs);
+    this.schedulerBusy = clusterMetrics.getRmSchedulerBusyPercent();
+    this.rmDispatcherEventQueueSize = clusterMetrics.getRmEventQueueSize();
+    this.schedulerDispatcherEventQueueSize = clusterMetrics.getSchedulerEventQueueSize();
+  }
+
+  private static String getSchedulerName(ResourceScheduler rs) {
+    if (rs instanceof CapacityScheduler) {
+      return "Capacity Scheduler";
+    }
+    if (rs instanceof FairScheduler) {
+      return "Fair Scheduler";
+    }
+    if (rs instanceof FifoScheduler) {
+      return "Fifo Scheduler";
+    }
+    return rs.getClass().getSimpleName();
+  }
+
+  public String getSchedulerType() {
+    return schedulerType;
+  }
+
+  public String getSchedulingResourceType() {
+    return schedulingResourceType;
+  }
+
+  public ResourceInfo getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  public ResourceInfo getMaximumAllocation() {
+    return maximumAllocation;
+  }
+
+  public int getApplicationPriority() {
+    return applicationPriority;
+  }
+
+  public int getSchedulerBusy() {
+    return schedulerBusy;
+  }
+
+  public int getRmDispatcherEventQueueSize() {
+    return rmDispatcherEventQueueSize;
+  }
+
+  public int getSchedulerDispatcherEventQueueSize() {
+    return schedulerDispatcherEventQueueSize;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java
index e12dd5fb63b9e..8f79fef0ce8f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java
@@ -41,6 +41,12 @@ public StatisticsItemInfo(
     this.count = count;
   }
 
+  public StatisticsItemInfo(StatisticsItemInfo info) {
+    this.state = info.state;
+    this.type = info.type;
+    this.count = info.count;
+  }
+
   public YarnApplicationState getState() {
     return state;
   }
@@ -53,4 +59,7 @@ public long getCount() {
     return count;
   }
 
+  public void setCount(long count) {
+    this.count = count;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
index bfa5bd2937da3..b39c283029e38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
@@ -22,7 +22,6 @@
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
index 8b3602da31e1d..0ba9bbb841817 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
@@ -18,9 +18,9 @@
 
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AutoQueueTemplatePropertiesInfo;
@@ -82,7 +82,7 @@ public static String getMode(CSQueue queue) throws YarnRuntimeException {
   }
 
   public static String getQueueType(CSQueue queue) {
-    if (queue instanceof LeafQueue) {
+    if (queue instanceof AbstractLeafQueue) {
       return LEAF_QUEUE;
     } else if (queue instanceof ParentQueue) {
       return PARENT_QUEUE;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 61f4a8a57a6a1..2d1cf1c5a53ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -72,6 +72,7 @@ message ApplicationStateDataProto {
     optional hadoop.common.RPCCallerContextProto caller_context = 8;
     repeated ApplicationTimeoutMapProto application_timeouts = 9;
     optional int64 launch_time = 10;
+    optional string real_user = 11;
 }
 
 message ApplicationAttemptStateDataProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
index 33fe02824513a..2e43aa44c5508 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -71,7 +72,7 @@ public void submitApplication(
         ApplicationSubmissionContext submissionContext, String user)
         throws YarnException {
       super.submitApplication(submissionContext, System.currentTimeMillis(),
-          user);
+          UserGroupInformation.createRemoteUser(user));
     }
 
     public String getUserNameForPlacement(final String user,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 90c554361c03b..95db1171b35e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -20,6 +20,7 @@
 
 import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.token.Token;
@@ -64,6 +65,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -534,6 +536,17 @@ public MockNM registerNode(String nodeIdStr, int memory, int vCores,
     return nm;
   }
 
+  public MockNM registerNode(String nodeIdStr, int memory, int vCores,
+      List<ApplicationId> runningApplications,
+      List<NMContainerStatus> containerStatuses) throws Exception {
+    MockNM nm =
+        new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService(),
+            YarnVersionInfo.getVersion());
+    nm.registerNode(containerStatuses, runningApplications);
+    drainEventsImplicitly();
+    return nm;
+  }
+
   public MockNM registerNode(String nodeIdStr, Resource nodeCapability)
       throws Exception {
     MockNM nm = new MockNM(nodeIdStr, nodeCapability,
@@ -1024,4 +1037,9 @@ protected void serviceInit(Configuration conf) throws Exception {
   public RMStateStore getRMStateStore() {
     return getRMContext().getStateStore();
   }
+
+  @VisibleForTesting
+  public ReservationSystem getReservationSystem(){
+    return this.reservationSystem;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
index 439a4490947d7..ca76c24b87818 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -169,7 +170,8 @@ public MyRMAppManager(RMContext context, YarnScheduler scheduler,
     @Override
     protected void submitApplication(
         ApplicationSubmissionContext submissionContext, long submitTime,
-        String user) throws YarnException {
+        UserGroupInformation userUgi) throws YarnException {
+      String user = userUgi.getShortUserName();
       //Do nothing, just add the application to RMContext
       RMAppImpl application =
           new RMAppImpl(submissionContext.getApplicationId(), this.rmContext,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index f916264a59f33..be0dbf2a1ba41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -68,9 +68,13 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -108,7 +112,11 @@
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentMap;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
@@ -141,6 +149,7 @@ public class TestAppManager extends AppManagerTestBase{
   private ResourceScheduler scheduler;
 
   private static final String USER_ID_PREFIX = "userid=";
+  private static final String ROOT_PARENT =  PREFIX + "root.parent.";
 
   public synchronized RMAppEventType getAppEventType() {
     return appEventType;
@@ -274,22 +283,49 @@ public void setUp() throws IOException {
     setupDispatcher(rmContext, conf);
   }
 
+  private static PlacementManager createMockPlacementManager(
+      String userRegex, String placementQueue, String placementParentQueue
+  ) throws YarnException {
+    PlacementManager placementMgr = mock(PlacementManager.class);
+    doAnswer(new Answer<ApplicationPlacementContext>() {
+
+      @Override
+      public ApplicationPlacementContext answer(InvocationOnMock invocation)
+          throws Throwable {
+        return new ApplicationPlacementContext(placementQueue, placementParentQueue);
+      }
+
+    }).when(placementMgr).placeApplication(
+        any(ApplicationSubmissionContext.class),
+        matches(userRegex),
+        any(Boolean.class));
+
+    return placementMgr;
+  }
+
+  private TestRMAppManager createAppManager(RMContext context, Configuration configuration) {
+    ApplicationMasterService masterService = new ApplicationMasterService(context,
+        context.getScheduler());
+
+    return new TestRMAppManager(context,
+        new ClientToAMTokenSecretManagerInRM(),
+        context.getScheduler(), masterService,
+        new ApplicationACLsManager(configuration), configuration);
+  }
+
   @Test
   public void testQueueSubmitWithACLsEnabledWithQueueMapping()
-      throws IOException, YarnException, InterruptedException {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-
+      throws YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
     CapacitySchedulerConfiguration csConf = new
         CapacitySchedulerConfiguration(conf, false);
     csConf.set(PREFIX + "root.queues", "default,test");
 
-    csConf.setFloat(PREFIX + "root.default.capacity", 50.0f);
-    csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
+    csConf.setCapacity("root.default", 50.0f);
+    csConf.setMaximumCapacity("root.default", 100.0f);
 
-    csConf.setFloat(PREFIX + "root.test.capacity", 50.0f);
-    csConf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f);
+    csConf.setCapacity("root.test", 50.0f);
+    csConf.setMaximumCapacity("root.test", 100.0f);
 
     csConf.set(PREFIX + "root.acl_submit_applications", " ");
     csConf.set(PREFIX + "root.acl_administer_queue", " ");
@@ -300,66 +336,38 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping()
     csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
     csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
 
-    csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
-    csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
-
-    csConf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
-
-    // Setup a PlacementManager returns a new queue
-    PlacementManager placementMgr = mock(PlacementManager.class);
-    doAnswer(new Answer<ApplicationPlacementContext>() {
-
-      @Override
-      public ApplicationPlacementContext answer(InvocationOnMock invocation)
-          throws Throwable {
-        return new ApplicationPlacementContext("test");
-      }
-
-    }).when(placementMgr).placeApplication(
-        any(ApplicationSubmissionContext.class), matches("test"));
-
-    asContext.setQueue("oldQueue");
-
-    MockRM newMockRM = new MockRM(conf);
+    MockRM newMockRM = new MockRM(csConf);
     RMContext newMockRMContext = newMockRM.getRMContext();
-    newMockRMContext.setQueuePlacementManager(placementMgr);
-    ApplicationMasterService masterService =
-        new ApplicationMasterService(newMockRMContext,
-            newMockRMContext.getScheduler());
-
-    TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext,
-        new ClientToAMTokenSecretManagerInRM(),
-        newMockRMContext.getScheduler(), masterService,
-        new ApplicationACLsManager(conf), conf);
-
-    //only user test has permission to submit to 'test' queue
-    newAppMonitor.submitApplication(asContext, "test");
-
-    try {
-      //should fail since user does not have permission to submit to queue
-      // 'test'
-      asContext.setApplicationId(appId = MockApps.newAppID(2));
-      newAppMonitor.submitApplication(asContext, "test1");
-    } catch(YarnException e) {
-      assertTrue(e.getCause() instanceof AccessControlException);
-    }
+    newMockRMContext.setQueuePlacementManager(
+        createMockPlacementManager("test", "root.test", null));
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    ApplicationSubmissionContext submission = createAppSubmissionContext(MockApps.newAppID(1));
+    submission.setQueue("oldQueue");
+    verifyAppSubmission(submission,
+        newAppMonitor,
+        newMockRMContext,
+        "test",
+        "root.test");
+
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(2)),
+        "test1");
   }
 
   @Test
-  public void
-  testQueueSubmitWithACLsEnabledWithQueueMappingForAutoCreatedQueue()
-      throws IOException, YarnException, InterruptedException {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-
+  public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQueue()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
     CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
         conf, false);
     csConf.set(PREFIX + "root.queues", "default,managedparent");
 
-    csConf.setFloat(PREFIX + "root.default.capacity", 50.0f);
-    csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
+    csConf.setCapacity("root.default", 50.0f);
+    csConf.setMaximumCapacity("root.default", 100.0f);
+
+    csConf.setCapacity("root.managedparent", 50.0f);
+    csConf.setMaximumCapacity("root.managedparent", 100.0f);
 
     csConf.set(PREFIX + "root.acl_submit_applications", " ");
     csConf.set(PREFIX + "root.acl_administer_queue", " ");
@@ -374,51 +382,663 @@ public ApplicationPlacementContext answer(InvocationOnMock invocation)
     csConf.setAutoCreatedLeafQueueConfigCapacity("root.managedparent", 30f);
     csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.managedparent", 100f);
 
-    // Setup a PlacementManager returns a new queue
-    PlacementManager placementMgr = mock(PlacementManager.class);
-    doAnswer(new Answer<ApplicationPlacementContext>() {
+    MockRM newMockRM = new MockRM(csConf);
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs.getQueueContext(),
+        "managedparent", cs.getQueue("root"), null);
+    cs.getCapacitySchedulerQueueManager().addQueue("managedparent",
+        managedParentQueue);
 
-      @Override
-      public ApplicationPlacementContext answer(InvocationOnMock invocation)
-          throws Throwable {
-        return new ApplicationPlacementContext("user1", "managedparent");
-      }
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1|user2", "user1", "root.managedparent"));
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    ApplicationSubmissionContext submission = createAppSubmissionContext(MockApps.newAppID(1));
+    submission.setQueue("oldQueue");
+    verifyAppSubmission(submission,
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.managedparent.user1");
+
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(2)),
+        "user2");
+  }
 
-    }).when(placementMgr).placeApplication(
-        any(ApplicationSubmissionContext.class), matches("user1|user2"));
+  @Test
+  public void testLegacyAutoCreatedQueuesWithACLTemplates()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        conf, false);
+    csConf.set(PREFIX + "root.queues", "parent");
+    csConf.set(PREFIX + "root.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.acl_administer_queue", " ");
 
-    asContext.setQueue("oldQueue");
+    csConf.setCapacity("root.parent", 100.0f);
+    csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1,user4");
+    csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1,user4");
+
+    csConf.setAutoCreateChildQueueEnabled("root.parent", true);
+    csConf.setAutoCreatedLeafQueueConfigCapacity("root.parent", 50f);
+    csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.parent", 100f);
+    csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) +
+        "acl_administer_queue", "user2,user4");
+    csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) +
+        "acl_submit_applications", "user2,user4");
 
-    MockRM newMockRM = new MockRM(conf);
+    MockRM newMockRM = new MockRM(csConf);
+
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user1 has permission on root.parent so a queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1", "user1", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.parent.user1");
+
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1|user2|user3|user4", "user2", "root.parent"));
+
+    // user2 has permission (due to ACL templates)
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)),
+        newAppMonitor,
+        newMockRMContext,
+        "user2",
+        "root.parent.user2");
+
+    // user3 doesn't have permission
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(3)),
+        "user3");
+
+    // user4 has permission on root.parent
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(4)),
+        newAppMonitor,
+        newMockRMContext,
+        "user4",
+        "root.parent.user2");
+
+    // create the root.parent.user2 manually
     CapacityScheduler cs =
         ((CapacityScheduler) newMockRM.getResourceScheduler());
-    ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs,
-        "managedparent", cs.getQueue("root"), null);
-    cs.getCapacitySchedulerQueueManager().addQueue("managedparent",
-        managedParentQueue);
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2"));
+    AutoCreatedLeafQueue autoCreatedLeafQueue = (AutoCreatedLeafQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue("parent");
+    assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(5)),
+        newAppMonitor,
+        newMockRMContext,
+        "user2",
+        "root.parent.user2");
+
+    // user3 doesn't have permission for root.parent.user2 queue
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(6)),
+        "user3");
+
+    // user1 doesn't have permission for root.parent.user2 queue, but it has for root.parent
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(7)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.parent.user2");
+  }
+
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithSpecializedACLTemplatesAndDynamicParentQueue()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
+
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user3");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user3");
+
+    MockRM newMockRM = new MockRM(csConf);
+
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user1 has permission on root.parent so a queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1", "user1", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.parent.user1");
+
+    // user2 doesn't have permission to create a dynamic leaf queue (parent only template)
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user2", "user2", "root.parent"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(2)),
+        "user2");
+
+    // user3 has permission on root.parent.user2.user3 due to ACL templates
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(3)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.parent.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.parent.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(4)),
+        "user4");
+
+    // create the root.parent.user2.user3 manually
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2.user3"));
+
+    ParentQueue autoCreatedParentQueue = (ParentQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedParentQueue);
+    ParentQueue parentQueue = (ParentQueue) cs.getQueue("parent");
+    assertEquals(parentQueue, autoCreatedParentQueue.getParent());
+
+    LeafQueue autoCreatedLeafQueue = (LeafQueue) cs.getQueue("user3");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    assertEquals(autoCreatedParentQueue, autoCreatedLeafQueue.getParent());
+
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(5)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.parent.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.parent.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(6)),
+        "user4");
+  }
+
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithMixedCommonLeafACLTemplatesAndDynamicParentQueue()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
+
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user3");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user3");
+
+    testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf);
+  }
+
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithMixedCommonCommonACLTemplatesAndDynamicParentQueue()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
+
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user3");
+    csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user3");
+
+    testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf);
+  }
+
+  private void testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(
+      YarnConfiguration conf, CapacitySchedulerConfiguration csConf)
+      throws YarnException, IOException {
+    MockRM newMockRM = new MockRM(csConf);
 
     RMContext newMockRMContext = newMockRM.getRMContext();
-    newMockRMContext.setQueuePlacementManager(placementMgr);
-    ApplicationMasterService masterService = new ApplicationMasterService(
-        newMockRMContext, newMockRMContext.getScheduler());
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user1 has permission on root.parent so a queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1", "user1", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.parent.user1");
+
+    // user2 has permission on root.parent a dynamic leaf queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user2", "user2", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)),
+        newAppMonitor,
+        newMockRMContext,
+        "user2",
+        "root.parent.user2");
+
+    // user3 has permission on root.parent.user2.user3 a dynamic parent and leaf queue
+    // would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(3)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.parent.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.parent.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(4)),
+        "user4");
+
+    // create the root.parent.user2.user3 manually
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2.user3"));
+
+    ParentQueue autoCreatedParentQueue = (ParentQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedParentQueue);
+    ParentQueue parentQueue = (ParentQueue) cs.getQueue("parent");
+    assertEquals(parentQueue, autoCreatedParentQueue.getParent());
+
+    LeafQueue autoCreatedLeafQueue = (LeafQueue) cs.getQueue("user3");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    assertEquals(autoCreatedParentQueue, autoCreatedLeafQueue.getParent());
+
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(5)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.parent.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.parent.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(6)),
+        "user4");
+  }
 
-    TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext,
-        new ClientToAMTokenSecretManagerInRM(), newMockRMContext.getScheduler(),
-        masterService, new ApplicationACLsManager(conf), conf);
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithACLTemplatesALeafOnly()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
+
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    testFlexibleAQCLeafOnly(conf, csConf);
+  }
 
-    //only user test has permission to submit to 'user1' queue
-    newAppMonitor.submitApplication(asContext, "user1");
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithSpecialisedACLTemplatesALeafOnly()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+    CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
+
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    testFlexibleAQCLeafOnly(conf, csConf);
+  }
 
+  private void testFlexibleAQCLeafOnly(
+      YarnConfiguration conf,
+      CapacitySchedulerConfiguration csConf)
+      throws YarnException, IOException {
+    MockRM newMockRM = new MockRM(csConf);
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user1 has permission on root.parent so a queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1", "user1", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.parent.user1");
+
+    // user2 has permission on root.parent.user2 due to ACL templates
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user2", "user2", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)),
+        newAppMonitor,
+        newMockRMContext,
+        "user2",
+        "root.parent.user2");
+
+    // user3 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(3)),
+        "user3");
+
+    // create the root.parent.user2 manually
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2"));
+
+    ParentQueue autoCreatedParentQueue = (ParentQueue) cs.getQueue("parent");
+    LeafQueue autoCreatedLeafQueue = (LeafQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    assertEquals(autoCreatedParentQueue, autoCreatedLeafQueue.getParent());
+
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user2", "user2", "root.parent"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(4)),
+        newAppMonitor,
+        newMockRMContext,
+        "user2",
+        "root.parent.user2");
+
+    // user3 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.parent"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(5)),
+        "user3");
+  }
+
+  @Test
+  public void testFlexibleAutoCreatedQueuesWithSpecializedACLTemplatesAndDynamicRootParentQueue()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        conf, false);
+    csConf.set(PREFIX + "root.queues", "");
+    csConf.set(PREFIX + "root.acl_submit_applications", "user1");
+    csConf.set(PREFIX + "root.acl_administer_queue", "admin1");
+
+    csConf.setAutoQueueCreationV2Enabled("root", true);
+
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    csConf.set(PREFIX + "root." + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(PREFIX + "root." + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user3");
+    csConf.set(PREFIX + "root." + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX +
+            "acl_submit_applications",
+        "user3");
+
+    MockRM newMockRM = new MockRM(csConf);
+
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user1 has permission on root so a queue would be created
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user1", "user1", "root"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user1",
+        "root.user1");
+
+    // user2 doesn't have permission to create a dynamic leaf queue (parent only template)
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user2", "user2", "root"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(2)),
+        "user2");
+
+    // user3 has permission on root.user2.user3 due to ACL templates
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(3)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(4)),
+        "user4");
+
+    // create the root.user2.user3 manually
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.user2.user3"));
+
+    ParentQueue autoCreatedParentQueue = (ParentQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedParentQueue);
+    ParentQueue parentQueue = (ParentQueue) cs.getQueue("root");
+    assertEquals(parentQueue, autoCreatedParentQueue.getParent());
+
+    LeafQueue autoCreatedLeafQueue = (LeafQueue) cs.getQueue("user3");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    assertEquals(autoCreatedParentQueue, autoCreatedLeafQueue.getParent());
+
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "user3", "root.user2"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(5)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.user2.user3");
+
+    // user4 doesn't have permission
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user4", "user4", "root.user2"));
+    verifyAppSubmissionFailure(newAppMonitor,
+        createAppSubmissionContext(MockApps.newAppID(6)),
+        "user4");
+  }
+
+  @Test
+  public void testFlexibleAutoCreatedQueuesMultiLevelDynamicParentACL()
+      throws IOException, YarnException {
+    YarnConfiguration conf = createYarnACLEnabledConfiguration();
+
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        conf, false);
+    csConf.set(PREFIX + "root.queues", "");
+    csConf.set(PREFIX + "root.acl_submit_applications", "user1");
+    csConf.set(PREFIX + "root.acl_administer_queue", "admin1");
+
+    csConf.setAutoQueueCreationV2Enabled("root", true);
+
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue",
+        "user2");
+    csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications",
+        "user2");
+
+    csConf.set(PREFIX + "root." + "user2.user3." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+        "1w");
+    csConf.set(PREFIX + "root." + "user2.user3." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX +
+            "acl_administer_queue",
+        "user3");
+    csConf.set(PREFIX + "root." + "user2.user3." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX +
+            "acl_submit_applications",
+        "user3");
+    csConf.setMaximumAutoCreatedQueueDepth(4);
+
+    MockRM newMockRM = new MockRM(csConf);
+
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
+
+    // user3 has permission on root.user2.user3.queue due to ACL templates
+    newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
+        "user3", "queue", "root.user2.user3"));
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.user2.user3.queue");
+
+    // create the root.user2.user3.queue manually
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.user2.user3.queue"));
+
+    ParentQueue autoCreatedParentQueue = (ParentQueue) cs.getQueue("user2");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedParentQueue);
+    ParentQueue parentQueue = (ParentQueue) cs.getQueue("root");
+    assertEquals(parentQueue, autoCreatedParentQueue.getParent());
+
+    ParentQueue autoCreatedParentQueue2 = (ParentQueue) cs.getQueue("user3");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedParentQueue2);
+    assertEquals(autoCreatedParentQueue, autoCreatedParentQueue2.getParent());
+
+    LeafQueue autoCreatedLeafQueue = (LeafQueue) cs.getQueue("queue");
+    Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
+    assertEquals(autoCreatedParentQueue, autoCreatedParentQueue2.getParent());
+
+    // reinitialize to load the ACLs for the queue
+    cs.reinitialize(csConf, newMockRMContext);
+
+    // template ACLs do work after reinitialize
+    verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)),
+        newAppMonitor,
+        newMockRMContext,
+        "user3",
+        "root.user2.user3.queue");
+  }
+
+  private YarnConfiguration createYarnACLEnabledConfiguration() {
+    YarnConfiguration conf = new YarnConfiguration(new Configuration(false));
+    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    return conf;
+  }
+
+  private CapacitySchedulerConfiguration createFlexibleAQCBaseACLConfiguration(
+      YarnConfiguration conf) {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        conf, false);
+    csConf.set(PREFIX + "root.queues", "parent");
+    csConf.set(PREFIX + "root.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.acl_administer_queue", " ");
+
+    csConf.setCapacity("root.parent", "1w");
+    csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1");
+    csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1");
+
+    csConf.setAutoQueueCreationV2Enabled("root.parent", true);
+    return csConf;
+  }
+
+  private static void verifyAppSubmissionFailure(TestRMAppManager appManager,
+                                                 ApplicationSubmissionContext submission,
+                                                 String user) {
     try {
-      //should fail since user does not have permission to submit to queue
-      // 'managedparent'
-      asContext.setApplicationId(appId = MockApps.newAppID(2));
-      newAppMonitor.submitApplication(asContext, "user2");
+      appManager.submitApplication(submission, user);
+      Assert.fail(
+          String.format("should fail since %s does not have permission to submit to queue", user));
     } catch (YarnException e) {
       assertTrue(e.getCause() instanceof AccessControlException);
     }
   }
 
+  private static void verifyAppSubmission(ApplicationSubmissionContext submission,
+                                          TestRMAppManager appManager,
+                                          RMContext rmContext,
+                                          String user,
+                                          String expectedQueue) throws YarnException {
+    appManager.submitApplication(submission, user);
+    RMApp app = rmContext.getRMApps().get(submission.getApplicationId());
+    Assert.assertNotNull("app should not be null", app);
+    Assert.assertEquals(String.format("the queue should be placed on '%s' queue", expectedQueue),
+        expectedQueue,
+        app.getQueue());
+  }
+
+  private static ApplicationSubmissionContext createAppSubmissionContext(ApplicationId id) {
+    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+    ApplicationSubmissionContext appSubmission =
+        recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
+    appSubmission.setApplicationId(id);
+    appSubmission.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
+    appSubmission.setResource(mockResource());
+    appSubmission.setPriority(Priority.newInstance(0));
+    appSubmission.setQueue("default");
+    return appSubmission;
+  }
+
   @After
   public void tearDown() {
     setAppEventType(RMAppEventType.KILL);
@@ -734,6 +1354,33 @@ public void testRMAppSubmitAMContainerResourceRequest() throws Exception {
         app.getAMResourceRequests());
   }
 
+  @Test
+  public void testRMAppSubmitAMContainerWithNoLabelByRMDefaultAMNodeLabel() throws Exception {
+    List<ResourceRequest> reqs = new ArrayList<>();
+    ResourceRequest anyReq = ResourceRequest.newInstance(
+        Priority.newInstance(1),
+        ResourceRequest.ANY, Resources.createResource(1024), 1, false, null,
+        ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+    reqs.add(anyReq);
+    asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
+    asContext.setNodeLabelExpression("fixed");
+
+    Configuration conf = new Configuration(false);
+    String defaultAMNodeLabel = "core";
+    conf.set(YarnConfiguration.AM_DEFAULT_NODE_LABEL, defaultAMNodeLabel);
+
+    when(mockDefaultQueueInfo.getAccessibleNodeLabels()).thenReturn(
+        new HashSet<String>() {{ add("core"); }});
+
+    TestRMAppManager newAppMonitor = createAppManager(rmContext, conf);
+    newAppMonitor.submitApplication(asContext, "test");
+
+    RMApp app = rmContext.getRMApps().get(appId);
+    waitUntilEventProcessed();
+    Assert.assertEquals(defaultAMNodeLabel,
+        app.getAMResourceRequests().get(0).getNodeLabelExpression());
+  }
+
   @Test
   public void testRMAppSubmitResource() throws Exception {
     asContext.setResource(Resources.createResource(1024));
@@ -841,6 +1488,10 @@ public void testRMAppSubmitAMContainerResourceRequestsTwoManyAny()
 
   private RMApp testRMAppSubmit() throws Exception {
     appMonitor.submitApplication(asContext, "test");
+    return waitUntilEventProcessed();
+  }
+
+  private RMApp waitUntilEventProcessed() throws InterruptedException {
     RMApp app = rmContext.getRMApps().get(appId);
     Assert.assertNotNull("app is null", app);
     Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
@@ -1121,7 +1772,7 @@ private static ContainerLaunchContext mockContainerLaunchContext(
       RecordFactory recordFactory) {
     ContainerLaunchContext amContainer = recordFactory.newRecordInstance(
         ContainerLaunchContext.class);
-    amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());;
+    amContainer.setApplicationACLs(new HashMap<ApplicationAccessType, String>());
     return amContainer;
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
index 182016a6cfab6..80f84c34503f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
@@ -16,36 +16,46 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
     .RMContainerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSet;
 import static org.junit.Assert.fail;
 
 /**
@@ -208,4 +218,157 @@ public void testPriorityInAllocatedResponse() throws Exception {
     Assert.assertEquals(appPriority2, response2.getApplicationPriority());
     rm.stop();
   }
+
+  @Test(timeout = 300000)
+  public void testGetNMNumInAllocatedResponseWithOutNodeLabel() throws Exception {
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+
+    // Register node1 node2 node3 node4
+    MockNM nm1 = rm.registerNode("host1:1234", 6 * GB);
+    MockNM nm2 = rm.registerNode("host2:1234", 6 * GB);
+    MockNM nm3 = rm.registerNode("host3:1234", 6 * GB);
+
+    // Submit an application
+    MockRMAppSubmissionData data = MockRMAppSubmissionData.Builder
+        .createWithMemory(2048, rm)
+        .build();
+    RMApp app1 = MockRMAppSubmitter.submit(rm, data);
+
+    nm1.nodeHeartbeat(true);
+    nm2.nodeHeartbeat(true);
+    nm3.nodeHeartbeat(true);
+
+    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+    am1.registerAppAttempt();
+
+    AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl();
+    List<ContainerId> release = new ArrayList<ContainerId>();
+    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+    allocateRequest.setReleaseList(release);
+    allocateRequest.setAskList(ask);
+
+    AllocateResponse response1 = am1.allocate(allocateRequest);
+    Assert.assertEquals(3, response1.getNumClusterNodes());
+
+    rm.stop();
+  }
+
+  private Configuration getConfigurationWithQueueLabels(Configuration config) {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    conf.setCapacity(A, 50);
+    conf.setMaximumCapacity(A, 100);
+    conf.setAccessibleNodeLabels(A, toSet("x"));
+    conf.setDefaultNodeLabelExpression(A, "x");
+    conf.setCapacityByLabel(A, "x", 100);
+
+    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+    conf.setCapacity(B, 50);
+    conf.setMaximumCapacity(B, 100);
+    conf.setAccessibleNodeLabels(B, toSet("y"));
+    conf.setDefaultNodeLabelExpression(B, "y");
+    conf.setCapacityByLabel(B, "y", 100);
+
+    return conf;
+  }
+
+  @Test(timeout = 300000)
+  public void testGetNMNumInAllocatedResponseWithNodeLabel() throws Exception {
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+      @Override
+      protected RMNodeLabelsManager createNodeLabelManager() {
+        RMNodeLabelsManager mgr = new RMNodeLabelsManager();
+        mgr.init(getConfig());
+        return mgr;
+      }
+    };
+
+    // add node label "x","y" and set node to label mapping
+    Set<String> clusterNodeLabels = new HashSet<String>();
+    clusterNodeLabels.add("x");
+    clusterNodeLabels.add("y");
+
+    RMNodeLabelsManager nodeLabelManager = rm.getRMContext().getNodeLabelManager();
+    nodeLabelManager.
+        addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels);
+
+    //has 3 nodes with node label "x",1 node with node label "y"
+    nodeLabelManager
+        .addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host1", 1234), toSet("x")));
+    nodeLabelManager
+        .addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host2", 1234), toSet("x")));
+    nodeLabelManager
+        .addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host3", 1234), toSet("x")));
+    nodeLabelManager
+        .addLabelsToNode(ImmutableMap.of(NodeId.newInstance("host4", 1234), toSet("y")));
+    rm.start();
+
+    // Register node1 node2 node3 node4
+    MockNM nm1 = rm.registerNode("host1:1234", 6 * GB);
+    MockNM nm2 = rm.registerNode("host2:1234", 6 * GB);
+    MockNM nm3 = rm.registerNode("host3:1234", 6 * GB);
+    MockNM nm4 = rm.registerNode("host4:1234", 6 * GB);
+
+    // submit an application to queue root.a expression as "x"
+    MockRMAppSubmissionData data1 = MockRMAppSubmissionData.Builder
+        .createWithMemory(2048, rm)
+        .withAppName("someApp1")
+        .withUser("someUser")
+        .withQueue("root.a")
+        .build();
+    RMApp app1 = MockRMAppSubmitter.submit(rm, data1);
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+    // submit an application to queue root.b expression as "y"
+    MockRMAppSubmissionData data2 = MockRMAppSubmissionData.Builder
+        .createWithMemory(2048, rm)
+        .withAppName("someApp2")
+        .withUser("someUser")
+        .withQueue("root.b")
+        .build();
+    RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm4);
+
+    AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl();
+    List<ContainerId> release = new ArrayList<ContainerId>();
+    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+    allocateRequest.setReleaseList(release);
+    allocateRequest.setAskList(ask);
+
+    AllocateResponse response1 = am1.allocate(allocateRequest);
+    AllocateResponse response2 = am2.allocate(allocateRequest);
+
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+    RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+    RMNode rmNode3 = rm.getRMContext().getRMNodes().get(nm3.getNodeId());
+    RMNode rmNode4 = rm.getRMContext().getRMNodes().get(nm4.getNodeId());
+
+    // Do node heartbeats many times
+    for (int i = 0; i < 3; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode3));
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode4));
+    }
+
+    //has 3 nodes with node label "x"
+    Assert.assertEquals(3, response1.getNumClusterNodes());
+
+    //has 1 node with node label "y"
+    Assert.assertEquals(1, response2.getNumClusterNodes());
+
+    rm.stop();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 7806845a2edeb..2d0b8a296170f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -58,6 +58,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.MockApps;
@@ -75,6 +76,7 @@
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
@@ -139,6 +141,7 @@
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.Event;
@@ -1402,9 +1405,9 @@ public void handle(Event event) {}
     request.setQueues(queueSet);
 
     queueSet.add(queues[0]);
-    assertEquals("Incorrect number of applications in queue", 2,
+    assertEquals("Incorrect number of applications in queue", 3,
         rmService.getApplications(request).getApplicationList().size());
-    assertEquals("Incorrect number of applications in queue", 2,
+    assertEquals("Incorrect number of applications in queue", 3,
         rmService.getApplications(request).getApplicationList().size());
 
     queueSet.add(queues[1]);
@@ -2809,10 +2812,63 @@ protected ClientRMService createClientRMService() {
     rm.close();
   }
 
+  @Test
+  public void testGetClusterMetrics() throws Exception {
+    MockRM rm = new MockRM() {
+      protected ClientRMService createClientRMService() {
+        return new ClientRMService(this.rmContext, scheduler,
+          this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
+          this.getRMContext().getRMDelegationTokenSecretManager());
+      };
+    };
+    rm.start();
+
+    ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
+    clusterMetrics.incrDecommissioningNMs();
+    repeat(2, clusterMetrics::incrDecommisionedNMs);
+    repeat(3, clusterMetrics::incrNumActiveNodes);
+    repeat(4, clusterMetrics::incrNumLostNMs);
+    repeat(5, clusterMetrics::incrNumUnhealthyNMs);
+    repeat(6, clusterMetrics::incrNumRebootedNMs);
+    repeat(7, clusterMetrics::incrNumShutdownNMs);
+
+    // Create a client.
+    Configuration conf = new Configuration();
+    YarnRPC rpc = YarnRPC.create(conf);
+    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+    LOG.info("Connecting to ResourceManager at " + rmAddress);
+    ApplicationClientProtocol client =
+        (ApplicationClientProtocol) rpc
+          .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+    YarnClusterMetrics ymetrics = client.getClusterMetrics(
+        GetClusterMetricsRequest.newInstance()).getClusterMetrics();
+
+    Assert.assertEquals(0, ymetrics.getNumNodeManagers());
+    Assert.assertEquals(1, ymetrics.getNumDecommissioningNodeManagers());
+    Assert.assertEquals(2, ymetrics.getNumDecommissionedNodeManagers());
+    Assert.assertEquals(3, ymetrics.getNumActiveNodeManagers());
+    Assert.assertEquals(4, ymetrics.getNumLostNodeManagers());
+    Assert.assertEquals(5, ymetrics.getNumUnhealthyNodeManagers());
+    Assert.assertEquals(6, ymetrics.getNumRebootedNodeManagers());
+    Assert.assertEquals(7, ymetrics.getNumShutdownNodeManagers());
+
+    rpc.stopProxy(client, conf);
+    rm.close();
+  }
+
   @After
   public void tearDown(){
     if (resourceTypesFile != null && resourceTypesFile.exists()) {
       resourceTypesFile.delete();
     }
+    ClusterMetrics.destroy();
+    DefaultMetricsSystem.shutdown();
+  }
+
+  private static void repeat(int n, Runnable r) {
+    for (int i = 0; i < n; ++i) {
+      r.run();
+    }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
index 50afced670e25..90ba812632854 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -35,11 +34,12 @@
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.token.delegation.TestDelegationToken;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMDelegationTokenIdentifierData;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -112,7 +112,7 @@ public void resetSecretManager() {
   }
   
   @Test
-  public void testDelegationToken() throws IOException, InterruptedException {
+  public void testDelegationToken() throws Exception {
     
     final YarnConfiguration conf = new YarnConfiguration();
     conf.set(YarnConfiguration.RM_PRINCIPAL, "testuser/localhost@apache.org");
@@ -125,9 +125,13 @@ public void testDelegationToken() throws IOException, InterruptedException {
     long initialInterval = 10000l;
     long maxLifetime= 20000l;
     long renewInterval = 10000l;
+    long delegationTokenRemoverScanInterval =
+        conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
+        TimeUnit.MILLISECONDS);
 
     RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(
-        initialInterval, maxLifetime, renewInterval);
+        initialInterval, maxLifetime, renewInterval, delegationTokenRemoverScanInterval);
     rmDtSecretManager.startThreads();
     LOG.info("Creating DelegationTokenSecretManager with initialInterval: "
         + initialInterval + ", maxLifetime: " + maxLifetime
@@ -199,14 +203,11 @@ public void testDelegationToken() throws IOException, InterruptedException {
       }
       Thread.sleep(50l);
       LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
-      // Token should have expired.      
-      try {
-        clientRMWithDT.getNewApplication(request);
-        fail("Should not have succeeded with an expired token");
-      } catch (Exception e) {
-        assertEquals(InvalidToken.class.getName(), e.getClass().getName());
-        assertTrue(e.getMessage().contains("is expired"));
-      } 
+      // Token should have expired.
+      final ApplicationClientProtocol finalClientRMWithDT = clientRMWithDT;
+      final GetNewApplicationRequest finalRequest = request;
+      LambdaTestUtils.intercept(InvalidToken.class, "Token  has expired",
+          () -> finalClientRMWithDT.getNewApplication(finalRequest));
 
       // Test cancellation
       // Stop the existing proxy, start another.
@@ -578,7 +579,8 @@ private static ResourceScheduler createMockScheduler(Configuration conf) {
 
   private static RMDelegationTokenSecretManager
       createRMDelegationTokenSecretManager(long secretKeyInterval,
-          long tokenMaxLifetime, long tokenRenewInterval) {
+          long tokenMaxLifetime, long tokenRenewInterval,
+          long delegationTokenRemoverScanInterval) {
     ResourceManager rm = mock(ResourceManager.class);
     RMContext rmContext = mock(RMContext.class);
     when(rmContext.getStateStore()).thenReturn(new NullRMStateStore());
@@ -587,7 +589,7 @@ private static ResourceScheduler createMockScheduler(Configuration conf) {
 
     RMDelegationTokenSecretManager rmDtSecretManager =
         new RMDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime,
-          tokenRenewInterval, 3600000, rmContext);
+          tokenRenewInterval, delegationTokenRemoverScanInterval, rmContext);
     return rmDtSecretManager;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 901dc8a143020..b99c456fb8173 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -72,6 +73,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl;
 import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
 import org.apache.hadoop.yarn.server.metrics.OpportunisticSchedulerMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -105,6 +107,7 @@
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 
@@ -132,6 +135,11 @@ public void createAndStartRM() {
         YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
     conf.setInt(
         YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(
+        YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+    conf.set(
+        YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
     startRM(conf);
   }
 
@@ -165,6 +173,8 @@ public void stopRM() {
     if (rm != null) {
       rm.stop();
     }
+
+    OpportunisticSchedulerMetrics.resetMetrics();
   }
 
   @Test(timeout = 600000)
@@ -817,6 +827,130 @@ public void testOpportunisticSchedulerMetrics() throws Exception {
         metrics.getAggregatedReleasedContainers());
   }
 
+  /**
+   * Tests that, if a node has running opportunistic containers when the RM
+   * is down, RM is able to reflect the opportunistic containers
+   * in its metrics upon RM recovery.
+   */
+  @Test
+  public void testMetricsRetainsAllocatedOpportunisticAfterRMRestart()
+      throws Exception {
+    final MockRMAppSubmissionData appSubmissionData =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("app")
+            .withUser("user")
+            .withAcls(null)
+            .withQueue("default")
+            .build();
+
+    MockNM nm1 = new MockNM("h:1234", 4096, rm.getResourceTrackerService());
+    nm1.registerNode();
+
+    final RMApp app = MockRMAppSubmitter.submit(rm, appSubmissionData);
+
+    final ApplicationAttemptId appAttemptId =
+        app.getCurrentAppAttempt().getAppAttemptId();
+
+    MockRM.launchAndRegisterAM(app, rm, nm1);
+
+    final OpportunisticSchedulerMetrics metrics =
+        OpportunisticSchedulerMetrics.getMetrics();
+
+    // We start with ID 2, since AMContainer is ID 1
+    final ContainerId recoverOContainerId2 = ContainerId.newContainerId(
+        appAttemptId, 2);
+
+    final Resource fakeResource = Resource.newInstance(1024, 1);
+    final String fakeDiagnostics = "recover container";
+    final Priority fakePriority = Priority.newInstance(0);
+
+    final NMContainerStatus recoverOContainerReport1 =
+        NMContainerStatus.newInstance(
+            recoverOContainerId2, 0, ContainerState.RUNNING,
+            fakeResource, fakeDiagnostics, 0,
+            fakePriority, 0, null,
+            ExecutionType.OPPORTUNISTIC, -1);
+
+    // Make sure that numbers start with 0
+    Assert.assertEquals(0, metrics.getAllocatedContainers());
+
+    // Recover one OContainer only
+    rm.registerNode("h2:1234", 4096, 1,
+        Collections.singletonList(
+            appAttemptId.getApplicationId()),
+        Collections.singletonList(recoverOContainerReport1));
+
+    Assert.assertEquals(1, metrics.getAllocatedContainers());
+
+    // Recover two OContainers at once
+    final ContainerId recoverOContainerId3 = ContainerId.newContainerId(
+        appAttemptId, 3);
+
+    final ContainerId recoverOContainerId4 = ContainerId.newContainerId(
+        appAttemptId, 4);
+
+    final NMContainerStatus recoverOContainerReport2 =
+        NMContainerStatus.newInstance(
+            recoverOContainerId2, 0, ContainerState.RUNNING,
+            fakeResource, fakeDiagnostics, 0,
+            fakePriority, 0, null,
+            ExecutionType.OPPORTUNISTIC, -1);
+
+    final NMContainerStatus recoverOContainerReport3 =
+        NMContainerStatus.newInstance(
+            recoverOContainerId3, 0, ContainerState.RUNNING,
+            fakeResource, fakeDiagnostics, 0,
+            fakePriority, 0, null,
+            ExecutionType.OPPORTUNISTIC, -1);
+
+    rm.registerNode(
+        "h3:1234", 4096, 10,
+        Collections.singletonList(
+            appAttemptId.getApplicationId()),
+        Arrays.asList(recoverOContainerReport2, recoverOContainerReport3));
+
+    Assert.assertEquals(3, metrics.getAllocatedContainers());
+
+    // Make sure that the recovered GContainer
+    // does not increment OContainer count
+    final ContainerId recoverGContainerId = ContainerId.newContainerId(
+        appAttemptId, 5);
+
+    final NMContainerStatus recoverGContainerReport =
+        NMContainerStatus.newInstance(
+            recoverGContainerId, 0, ContainerState.RUNNING,
+            fakeResource, fakeDiagnostics, 0,
+            fakePriority, 0, null,
+            ExecutionType.GUARANTEED, -1);
+
+    rm.registerNode(
+        "h4:1234", 4096, 10,
+        Collections.singletonList(
+            appAttemptId.getApplicationId()),
+        Collections.singletonList(recoverGContainerReport));
+
+    Assert.assertEquals(3, metrics.getAllocatedContainers());
+
+    final ContainerId completedOContainerId = ContainerId.newContainerId(
+        appAttemptId, 6);
+
+    final NMContainerStatus completedOContainerReport =
+        NMContainerStatus.newInstance(
+            completedOContainerId, 0, ContainerState.COMPLETE,
+            fakeResource, fakeDiagnostics, 0,
+            fakePriority, 0, null,
+            ExecutionType.OPPORTUNISTIC, -1);
+
+    // Tests that completed containers are not recorded
+    rm.registerNode(
+        "h5:1234", 4096, 10,
+        Collections.singletonList(
+            appAttemptId.getApplicationId()),
+        Collections.singletonList(completedOContainerReport));
+
+    Assert.assertEquals(3, metrics.getAllocatedContainers());
+  }
+
   @Test(timeout = 60000)
   public void testAMCrashDuringAllocate() throws Exception {
     MockNM nm = new MockNM("h:1234", 4096, rm.getResourceTrackerService());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index bc540b0ba7f32..457e9d47dd0b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -109,7 +109,7 @@
 
 public class TestRMAdminService {
 
-  private Configuration configuration;;
+  private Configuration configuration;
   private MockRM rm = null;
   private FileSystem fs;
   private Path workingPath;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 3346b57d98b75..3a0288715075e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -43,14 +43,19 @@
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.records.NodeAction;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -78,7 +83,9 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -231,6 +238,32 @@ private RMNodeStatusEvent getMockRMNodeStatusEventWithoutRunningApps() {
     return event;
   }
 
+  private static ContainerStatus getMockContainerStatus(
+      final ContainerId containerId, final Resource capability,
+      final ContainerState containerState) {
+    return getMockContainerStatus(containerId, capability, containerState,
+        ExecutionType.GUARANTEED);
+  }
+
+  private static ContainerStatus getMockContainerStatus(
+      final ContainerId containerId, final Resource capability,
+      final ContainerState containerState, final ExecutionType executionType) {
+    final ContainerStatus containerStatus = mock(ContainerStatus.class);
+    doReturn(containerId).when(containerStatus).getContainerId();
+    doReturn(containerState).when(containerStatus).getState();
+    doReturn(capability).when(containerStatus).getCapability();
+    doReturn(executionType).when(containerStatus).getExecutionType();
+    return containerStatus;
+  }
+
+  private static NMContainerStatus createNMContainerStatus(
+      final ContainerId containerId, final ExecutionType executionType,
+      final ContainerState containerState, final Resource capability) {
+    return NMContainerStatus.newInstance(containerId, 0, containerState,
+        capability, "", 0, Priority.newInstance(0), 0,
+        CommonNodeLabelsManager.NO_LABEL, executionType, -1);
+  }
+
   @Test (timeout = 5000)
   public void testExpiredContainer() {
     NodeStatus mockNodeStatus = createMockNodeStatus();
@@ -248,8 +281,8 @@ public void testExpiredContainer() {
     // Now verify that scheduler isn't notified of an expired container
     // by checking number of 'completedContainers' it got in the previous event
     RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(null);
-    ContainerStatus containerStatus = mock(ContainerStatus.class);
-    doReturn(completedContainerId).when(containerStatus).getContainerId();
+    ContainerStatus containerStatus = getMockContainerStatus(
+        completedContainerId, null, ContainerState.COMPLETE);
     doReturn(Collections.singletonList(containerStatus)).
         when(statusEvent).getContainers();
     node.handle(statusEvent);
@@ -321,12 +354,13 @@ public void testContainerUpdate() throws InterruptedException{
     RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(null);
     RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent(null);
 
-    ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
-    ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
-    ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);
+    ContainerStatus containerStatusFromNode1 = getMockContainerStatus(
+        completedContainerIdFromNode1, null, ContainerState.COMPLETE);
+    ContainerStatus containerStatusFromNode2_1 = getMockContainerStatus(
+        completedContainerIdFromNode2_1, null, ContainerState.COMPLETE);
+    ContainerStatus containerStatusFromNode2_2 = getMockContainerStatus(
+        completedContainerIdFromNode2_2, null, ContainerState.COMPLETE);
 
-    doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1)
-        .getContainerId();
     doReturn(Collections.singletonList(containerStatusFromNode1))
         .when(statusEventFromNode1).getContainers();
     node.handle(statusEventFromNode1);
@@ -336,13 +370,9 @@ public void testContainerUpdate() throws InterruptedException{
 
     completedContainers.clear();
 
-    doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1)
-        .getContainerId();
     doReturn(Collections.singletonList(containerStatusFromNode2_1))
         .when(statusEventFromNode2_1).getContainers();
 
-    doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2)
-        .getContainerId();
     doReturn(Collections.singletonList(containerStatusFromNode2_2))
         .when(statusEventFromNode2_2).getContainers();
 
@@ -358,6 +388,181 @@ public void testContainerUpdate() throws InterruptedException{
         .getContainerId());
   }
 
+  /**
+   * Tests that allocated resources are counted correctly on new nodes
+   * that are added to the cluster.
+   */
+  @Test
+  public void testAddWithAllocatedContainers() {
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+    RMNodeImpl node = getNewNode();
+    ApplicationId app0 = BuilderUtils.newApplicationId(0, 0);
+
+    // Independently computed expected allocated resource to verify against
+    final Resource expectedResource = Resource.newInstance(Resources.none());
+
+    // Guaranteed containers
+    final ContainerId newContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 0);
+    final Resource newContainerCapability =
+        Resource.newInstance(100, 1);
+    Resources.addTo(expectedResource, newContainerCapability);
+    final NMContainerStatus newContainerStatus = createNMContainerStatus(
+        newContainerId, ExecutionType.GUARANTEED,
+        ContainerState.NEW, newContainerCapability);
+
+    final ContainerId runningContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 1);
+    final Resource runningContainerCapability =
+        Resource.newInstance(200, 2);
+    Resources.addTo(expectedResource, runningContainerCapability);
+    final NMContainerStatus runningContainerStatus = createNMContainerStatus(
+        runningContainerId, ExecutionType.GUARANTEED,
+        ContainerState.RUNNING, runningContainerCapability);
+
+    // Opportunistic containers
+    final ContainerId newOppContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 2);
+    final Resource newOppContainerCapability =
+        Resource.newInstance(300, 3);
+    Resources.addTo(expectedResource, newOppContainerCapability);
+    final NMContainerStatus newOppContainerStatus = createNMContainerStatus(
+        newOppContainerId, ExecutionType.OPPORTUNISTIC,
+        ContainerState.NEW, newOppContainerCapability);
+
+    final ContainerId runningOppContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 3);
+    final Resource runningOppContainerCapability =
+        Resource.newInstance(400, 4);
+    Resources.addTo(expectedResource, runningOppContainerCapability);
+    final NMContainerStatus runningOppContainerStatus = createNMContainerStatus(
+        runningOppContainerId, ExecutionType.OPPORTUNISTIC,
+        ContainerState.RUNNING, runningOppContainerCapability);
+
+    node.handle(new RMNodeStartedEvent(node.getNodeID(),
+        Arrays.asList(newContainerStatus, runningContainerStatus,
+            newOppContainerStatus, runningOppContainerStatus),
+        null, mockNodeStatus));
+    Assert.assertEquals(NodeState.RUNNING, node.getState());
+    Assert.assertNotNull(nodesListManagerEvent);
+    Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
+        nodesListManagerEvent.getType());
+    Assert.assertEquals(expectedResource, node.getAllocatedContainerResource());
+  }
+
+  /**
+   * Tests that allocated container resources are counted correctly in
+   * {@link org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode}
+   * upon a node update. Resources should be counted for both GUARANTEED
+   * and OPPORTUNISTIC containers.
+   */
+  @Test (timeout = 5000)
+  public void testAllocatedContainerUpdate() {
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+    //Start the node
+    node.handle(new RMNodeStartedEvent(null, null, null, mockNodeStatus));
+
+    // Make sure that the node starts with no allocated resources
+    Assert.assertEquals(Resources.none(), node.getAllocatedContainerResource());
+
+    ApplicationId app0 = BuilderUtils.newApplicationId(0, 0);
+    final ContainerId newContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 0);
+    final ContainerId runningContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 1);
+
+    rmContext.getRMApps().put(app0, Mockito.mock(RMApp.class));
+
+    RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(null);
+
+    final List<ContainerStatus> containerStatuses = new ArrayList<>();
+
+    // Use different memory and VCores for new and running state containers
+    // to test that they add up correctly
+    final Resource newContainerCapability =
+        Resource.newInstance(100, 1);
+    final Resource runningContainerCapability =
+        Resource.newInstance(200, 2);
+    final Resource completedContainerCapability =
+        Resource.newInstance(50, 3);
+    final ContainerStatus newContainerStatusFromNode = getMockContainerStatus(
+        newContainerId, newContainerCapability, ContainerState.NEW);
+    final ContainerStatus runningContainerStatusFromNode =
+        getMockContainerStatus(runningContainerId, runningContainerCapability,
+            ContainerState.RUNNING);
+
+    containerStatuses.addAll(Arrays.asList(
+        newContainerStatusFromNode, runningContainerStatusFromNode));
+    doReturn(containerStatuses).when(statusEventFromNode1).getContainers();
+    node.handle(statusEventFromNode1);
+    Assert.assertEquals(Resource.newInstance(300, 3),
+        node.getAllocatedContainerResource());
+
+    final ContainerId newOppContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 2);
+    final ContainerId runningOppContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 3);
+
+    // Use the same resource capability as in previous for opportunistic case
+    RMNodeStatusEvent statusEventFromNode2 = getMockRMNodeStatusEvent(null);
+    final ContainerStatus newOppContainerStatusFromNode =
+        getMockContainerStatus(newOppContainerId, newContainerCapability,
+            ContainerState.NEW, ExecutionType.OPPORTUNISTIC);
+    final ContainerStatus runningOppContainerStatusFromNode =
+        getMockContainerStatus(runningOppContainerId,
+            runningContainerCapability, ContainerState.RUNNING,
+            ExecutionType.OPPORTUNISTIC);
+
+    containerStatuses.addAll(Arrays.asList(
+        newOppContainerStatusFromNode, runningOppContainerStatusFromNode));
+
+    // Pass in both guaranteed and opportunistic container statuses
+    doReturn(containerStatuses).when(statusEventFromNode2).getContainers();
+
+    node.handle(statusEventFromNode2);
+
+    // The result here should be double the first check,
+    // since allocated resources are doubled, just
+    // with different execution types
+    Assert.assertEquals(Resource.newInstance(600, 6),
+        node.getAllocatedContainerResource());
+
+    RMNodeStatusEvent statusEventFromNode3 = getMockRMNodeStatusEvent(null);
+    final ContainerId completedContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 4);
+    final ContainerId completedOppContainerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(app0, 0), 5);
+    final ContainerStatus completedContainerStatusFromNode =
+        getMockContainerStatus(completedContainerId, completedContainerCapability,
+            ContainerState.COMPLETE, ExecutionType.OPPORTUNISTIC);
+    final ContainerStatus completedOppContainerStatusFromNode =
+        getMockContainerStatus(completedOppContainerId,
+            completedContainerCapability, ContainerState.COMPLETE,
+            ExecutionType.OPPORTUNISTIC);
+
+    containerStatuses.addAll(Arrays.asList(
+        completedContainerStatusFromNode, completedOppContainerStatusFromNode));
+
+    doReturn(containerStatuses).when(statusEventFromNode3).getContainers();
+    node.handle(statusEventFromNode3);
+
+    // Adding completed containers should not have changed
+    // the resources allocated
+    Assert.assertEquals(Resource.newInstance(600, 6),
+        node.getAllocatedContainerResource());
+
+    RMNodeStatusEvent emptyStatusEventFromNode =
+        getMockRMNodeStatusEvent(null);
+
+    doReturn(Collections.emptyList())
+        .when(emptyStatusEventFromNode).getContainers();
+    node.handle(emptyStatusEventFromNode);
+
+    // Passing an empty containers list should yield no resources allocated
+    Assert.assertEquals(Resources.none(),
+        node.getAllocatedContainerResource());
+  }
+
   @Test (timeout = 5000)
   public void testStatusChange(){
     NodeStatus mockNodeStatus = createMockNodeStatus();
@@ -376,14 +581,14 @@ public void testStatusChange(){
     RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
     RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent(null);
 
-    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
-    ContainerStatus containerStatus2 = mock(ContainerStatus.class);
+    ContainerStatus containerStatus1 = getMockContainerStatus(
+        completedContainerId1, null, null);
+    ContainerStatus containerStatus2 = getMockContainerStatus(
+        completedContainerId2, null, null);
 
-    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
     doReturn(Collections.singletonList(containerStatus1))
         .when(statusEvent1).getContainers();
      
-    doReturn(completedContainerId2).when(containerStatus2).getContainerId();
     doReturn(Collections.singletonList(containerStatus2))
         .when(statusEvent2).getContainers();
 
@@ -1153,9 +1358,9 @@ public void testForHandlingDuplicatedCompltedContainers() {
 
     RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
 
-    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
+    ContainerStatus containerStatus1 = getMockContainerStatus(
+        completedContainerId1, null, ContainerState.COMPLETE);
 
-    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
     doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1)
         .getContainers();
 
@@ -1299,4 +1504,27 @@ public void testCalculateHeartBeatInterval() {
     calcIntervalTest(rmNode, nodeUtil, hbDefault, hbMin, hbMax,
         speedup, slowdown, vcoreUnit * 1.0F, hbDefault); // 100%
   }
+
+  @Test
+  public void testFinishedContainersPulledByAmOnDecommissioningNode() {
+    RMNodeImpl rMNodeImpl = getRunningNode();
+    rMNodeImpl.handle(
+        new RMNodeEvent(rMNodeImpl.getNodeID(), RMNodeEventType.GRACEFUL_DECOMMISSION));
+    Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState());
+
+    ContainerId containerId = BuilderUtils.newContainerId(
+        BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
+    List<ContainerId> containerIds = Arrays.asList(containerId);
+
+    rMNodeImpl.handle(
+        new RMNodeFinishedContainersPulledByAMEvent(rMNodeImpl.getNodeID(), containerIds));
+    Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState());
+
+    // Verify expected containersToBeRemovedFromNM from NodeHeartbeatResponse.
+    NodeHeartbeatResponse response =
+        YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.NORMAL, null, null, null,
+            null, 1000);
+    rMNodeImpl.setAndUpdateNodeHeartbeatResponse(response);
+    Assert.assertEquals(1, response.getContainersToBeRemovedFromNM().size());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 9feb54c7d401b..33d5003d7e1a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
 import org.apache.hadoop.yarn.server.api.ResourceTracker;
@@ -2662,7 +2663,7 @@ private void writeToHostsFile(File file, String... hosts)
   private void writeToHostsXmlFile(
       File file, Pair<String, Integer>... hostsAndTimeouts) throws Exception {
     ensureFileExists(file);
-    DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbFactory = XMLUtils.newSecureDocumentBuilderFactory();
     Document doc = dbFactory.newDocumentBuilder().newDocument();
     Element hosts = doc.createElement("hosts");
     doc.appendChild(hosts);
@@ -2680,7 +2681,7 @@ private void writeToHostsXmlFile(
         );
       }
     }
-    TransformerFactory transformerFactory = TransformerFactory.newInstance();
+    TransformerFactory transformerFactory = XMLUtils.newSecureTransformerFactory();
     Transformer transformer = transformerFactory.newTransformer();
     transformer.setOutputProperty(OutputKeys.INDENT, "yes");
     transformer.transform(new DOMSource(doc), new StreamResult(file));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index c5f45fd9789a5..52a34fbf76161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
@@ -445,7 +446,7 @@ private void checkCSQueue(MockRM rm,
     checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
         numContainers);
 
-    LeafQueue queue = (LeafQueue) app.getQueue();
+    AbstractLeafQueue queue = (AbstractLeafQueue) app.getQueue();
     Resource availableResources =
         Resources.subtract(queueResource, usedResource);
     // ************ check app headroom ****************
@@ -470,7 +471,7 @@ private void checkCSLeafQueue(MockRM rm,
       SchedulerApplication<SchedulerApplicationAttempt> app,
       Resource clusterResource, Resource queueResource, Resource usedResource,
       int numContainers) {
-    LeafQueue leafQueue = (LeafQueue) app.getQueue();
+    AbstractLeafQueue leafQueue = (AbstractLeafQueue) app.getQueue();
     // assert queue used resources.
     assertEquals(usedResource, leafQueue.getUsedResources());
     assertEquals(numContainers, leafQueue.getNumContainers());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 4640952016d7f..e6c159d67d2c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -67,6 +67,10 @@ public String getUser() {
       throw new UnsupportedOperationException("Not supported yet.");
     }
 
+    @Override
+    public String getRealUser() {
+      throw new UnsupportedOperationException("Not supported yet.");
+    }
     @Override
     public ApplicationSubmissionContext getApplicationSubmissionContext() {
       throw new UnsupportedOperationException("Not supported yet.");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
index e5e156dcf76c6..b8e2ce6ef32d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -20,21 +20,46 @@
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import javax.xml.bind.JAXBException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreException;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
 import org.junit.After;
 import org.junit.Assert;
@@ -45,6 +70,8 @@
 import com.sun.jersey.api.json.JSONJAXBContext;
 import com.sun.jersey.api.json.JSONUnmarshaller;
 
+import static org.mockito.Mockito.mock;
+
 /**
  * Unit tests for FederationStateStoreService.
  */
@@ -173,4 +200,286 @@ private String checkSubClusterInfo(SubClusterState state)
     return response.getCapability();
   }
 
+  @Test
+  public void testFederationStateStoreServiceInitialHeartbeatDelay() throws Exception {
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, 10);
+    conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+
+    GenericTestUtils.LogCapturer logCapture =
+        GenericTestUtils.LogCapturer.captureLogs(FederationStateStoreService.LOG);
+
+    final MockRM rm = new MockRM(conf);
+
+    // Initially there should be no entry for the sub-cluster
+    rm.init(conf);
+    stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+    GetSubClusterInfoResponse response = stateStore.getSubCluster(request);
+    Assert.assertNull(response);
+
+    // Validate if sub-cluster is registered
+    rm.start();
+    String capability = checkSubClusterInfo(SubClusterState.SC_NEW);
+    Assert.assertTrue(capability.isEmpty());
+
+    // Heartbeat to see if sub-cluster transitions to running
+    FederationStateStoreHeartbeat storeHeartbeat =
+        rm.getFederationStateStoreService().getStateStoreHeartbeatThread();
+    storeHeartbeat.run();
+    capability = checkSubClusterInfo(SubClusterState.SC_RUNNING);
+    checkClusterMetricsInfo(capability, 0);
+
+    Assert.assertTrue(logCapture.getOutput().contains(
+        "Started federation membership heartbeat with interval: 300 and initial delay: 10"));
+    rm.stop();
+  }
+
+  @Test
+  public void testCleanUpApplication() throws Exception {
+
+    // set yarn configuration
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, 10);
+    conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+
+    // set up MockRM
+    final MockRM rm = new MockRM(conf);
+    rm.init(conf);
+    stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+    rm.start();
+
+    // init subCluster Heartbeat,
+    // and check that the subCluster is in a running state
+    FederationStateStoreService stateStoreService =
+        rm.getFederationStateStoreService();
+    FederationStateStoreHeartbeat storeHeartbeat =
+        stateStoreService.getStateStoreHeartbeatThread();
+    storeHeartbeat.run();
+    checkSubClusterInfo(SubClusterState.SC_RUNNING);
+
+    // generate an application and join the [SC-1] cluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    addApplication2StateStore(appId, stateStore);
+
+    // make sure the app can be queried in the stateStore
+    GetApplicationHomeSubClusterRequest appRequest =
+         GetApplicationHomeSubClusterRequest.newInstance(appId);
+    GetApplicationHomeSubClusterResponse response =
+         stateStore.getApplicationHomeSubCluster(appRequest);
+    Assert.assertNotNull(response);
+    ApplicationHomeSubCluster appHomeSubCluster = response.getApplicationHomeSubCluster();
+    Assert.assertNotNull(appHomeSubCluster);
+    Assert.assertNotNull(appHomeSubCluster.getApplicationId());
+    Assert.assertEquals(appId, appHomeSubCluster.getApplicationId());
+
+    // clean up the app.
+    boolean cleanUpResult =
+        stateStoreService.cleanUpFinishApplicationsWithRetries(appId, true);
+    Assert.assertTrue(cleanUpResult);
+
+    // after clean, the app can no longer be queried from the stateStore.
+    LambdaTestUtils.intercept(FederationStateStoreException.class,
+        "Application " + appId + " does not exist",
+        () -> stateStore.getApplicationHomeSubCluster(appRequest));
+
+  }
+
+  @Test
+  public void testCleanUpApplicationWhenRMStart() throws Exception {
+
+    // We design such a test case.
+    // Step1. We add app01, app02, app03 to the stateStore,
+    // But these apps are not in RM's RMContext, they are finished apps
+    // Step2. We simulate RM startup, there is only app04 in RMContext.
+    // Step3. We wait for 5 seconds, the automatic cleanup thread should clean up finished apps.
+
+    // set yarn configuration.
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, 10);
+    conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+
+    // set up MockRM.
+    MockRM rm = new MockRM(conf);
+    rm.init(conf);
+    stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+
+    // generate an [app01] and join the [SC-1] cluster.
+    List<ApplicationId> appIds = new ArrayList<>();
+    ApplicationId appId01 = ApplicationId.newInstance(Time.now(), 1);
+    addApplication2StateStore(appId01, stateStore);
+    appIds.add(appId01);
+
+    // generate an [app02] and join the [SC-1] cluster.
+    ApplicationId appId02 = ApplicationId.newInstance(Time.now(), 2);
+    addApplication2StateStore(appId02, stateStore);
+    appIds.add(appId02);
+
+    // generate an [app03] and join the [SC-1] cluster.
+    ApplicationId appId03 = ApplicationId.newInstance(Time.now(), 3);
+    addApplication2StateStore(appId03, stateStore);
+    appIds.add(appId03);
+
+    // make sure the apps can be queried in the stateStore.
+    GetApplicationsHomeSubClusterRequest allRequest =
+        GetApplicationsHomeSubClusterRequest.newInstance(subClusterId);
+    GetApplicationsHomeSubClusterResponse allResponse =
+        stateStore.getApplicationsHomeSubCluster(allRequest);
+    Assert.assertNotNull(allResponse);
+    List<ApplicationHomeSubCluster> appHomeSCLists = allResponse.getAppsHomeSubClusters();
+    Assert.assertNotNull(appHomeSCLists);
+    Assert.assertEquals(3, appHomeSCLists.size());
+
+    // app04 exists in both RM memory and stateStore.
+    ApplicationId appId04 = ApplicationId.newInstance(Time.now(), 4);
+    addApplication2StateStore(appId04, stateStore);
+    addApplication2RMAppManager(rm, appId04);
+
+    // start rm.
+    rm.start();
+
+    // wait 5s, wait for the thread to finish cleaning up.
+    GenericTestUtils.waitFor(() -> {
+      int appsSize = 0;
+      try {
+        List<ApplicationHomeSubCluster> subClusters =
+            getApplicationsFromStateStore();
+        Assert.assertNotNull(subClusters);
+        appsSize = subClusters.size();
+      } catch (YarnException e) {
+        e.printStackTrace();
+      }
+      return (appsSize == 1);
+    }, 100, 1000 * 5);
+
+    // check the app to make sure the apps(app01,app02,app03) doesn't exist.
+    for (ApplicationId appId : appIds) {
+      GetApplicationHomeSubClusterRequest appRequest =
+          GetApplicationHomeSubClusterRequest.newInstance(appId);
+      LambdaTestUtils.intercept(FederationStateStoreException.class,
+          "Application " + appId + " does not exist",
+          () -> stateStore.getApplicationHomeSubCluster(appRequest));
+    }
+
+    if (rm != null) {
+      rm.stop();
+      rm = null;
+    }
+  }
+
+  @Test
+  public void testCleanUpApplicationWhenRMCompleteOneApp() throws Exception {
+
+    // We design such a test case.
+    // Step1. We start RM，Set the RM memory to keep a maximum of 1 completed app.
+    // Step2. Register app[01-03] to RM memory & stateStore.
+    // Step3. We clean up app01, app02, app03, at this time,
+    // app01, app02 should be cleaned up from statestore, app03 should remain in statestore.
+
+    // set yarn configuration.
+    conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, 10);
+    conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId());
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 1);
+    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+
+    // set up MockRM.
+    MockRM rm = new MockRM(conf);
+    rm.init(conf);
+    stateStore = rm.getFederationStateStoreService().getStateStoreClient();
+    rm.start();
+
+    // generate an [app01] and join the [SC-1] cluster.
+    List<ApplicationId> appIds = new ArrayList<>();
+    ApplicationId appId01 = ApplicationId.newInstance(Time.now(), 1);
+    addApplication2StateStore(appId01, stateStore);
+    addApplication2RMAppManager(rm, appId01);
+    appIds.add(appId01);
+
+    // generate an [app02] and join the [SC-1] cluster.
+    ApplicationId appId02 = ApplicationId.newInstance(Time.now(), 2);
+    addApplication2StateStore(appId02, stateStore);
+    addApplication2RMAppManager(rm, appId02);
+    appIds.add(appId02);
+
+    // generate an [app03] and join the [SC-1] cluster.
+    ApplicationId appId03 = ApplicationId.newInstance(Time.now(), 3);
+    addApplication2StateStore(appId03, stateStore);
+    addApplication2RMAppManager(rm, appId03);
+
+    // rmAppManager
+    RMAppManager rmAppManager = rm.getRMAppManager();
+    rmAppManager.finishApplication4Test(appId01);
+    rmAppManager.finishApplication4Test(appId02);
+    rmAppManager.finishApplication4Test(appId03);
+    rmAppManager.checkAppNumCompletedLimit4Test();
+
+    // app01, app02 should be cleaned from statestore
+    // After the query, it should report the error not exist.
+    for (ApplicationId appId : appIds) {
+      GetApplicationHomeSubClusterRequest appRequest =
+          GetApplicationHomeSubClusterRequest.newInstance(appId);
+      LambdaTestUtils.intercept(FederationStateStoreException.class,
+          "Application " + appId + " does not exist",
+          () -> stateStore.getApplicationHomeSubCluster(appRequest));
+    }
+
+    // app03 should remain in statestore
+    List<ApplicationHomeSubCluster> appHomeScList = getApplicationsFromStateStore();
+    Assert.assertNotNull(appHomeScList);
+    Assert.assertEquals(1, appHomeScList.size());
+    ApplicationHomeSubCluster homeSubCluster = appHomeScList.get(0);
+    Assert.assertNotNull(homeSubCluster);
+    Assert.assertEquals(appId03, homeSubCluster.getApplicationId());
+  }
+
+  private void addApplication2StateStore(ApplicationId appId,
+      FederationStateStore fedStateStore) throws YarnException {
+    ApplicationHomeSubCluster appHomeSC = ApplicationHomeSubCluster.newInstance(
+        appId, subClusterId);
+    AddApplicationHomeSubClusterRequest addHomeSCRequest =
+        AddApplicationHomeSubClusterRequest.newInstance(appHomeSC);
+    fedStateStore.addApplicationHomeSubCluster(addHomeSCRequest);
+  }
+
+  private List<ApplicationHomeSubCluster> getApplicationsFromStateStore() throws YarnException {
+    // make sure the apps can be queried in the stateStore
+    GetApplicationsHomeSubClusterRequest allRequest =
+        GetApplicationsHomeSubClusterRequest.newInstance(subClusterId);
+    GetApplicationsHomeSubClusterResponse allResponse =
+        stateStore.getApplicationsHomeSubCluster(allRequest);
+    Assert.assertNotNull(allResponse);
+    List<ApplicationHomeSubCluster> appHomeSCLists = allResponse.getAppsHomeSubClusters();
+    Assert.assertNotNull(appHomeSCLists);
+    return appHomeSCLists;
+  }
+
+  private void addApplication2RMAppManager(MockRM rm, ApplicationId appId) {
+    RMContext rmContext = rm.getRMContext();
+    Map<ApplicationId, RMApp> rmAppMaps = rmContext.getRMApps();
+    String user = MockApps.newUserName();
+    String name = MockApps.newAppName();
+    String queue = MockApps.newQueue();
+
+    YarnScheduler scheduler = mock(YarnScheduler.class);
+
+    ApplicationMasterService masterService =
+        new ApplicationMasterService(rmContext, scheduler);
+
+    ApplicationSubmissionContext submissionContext =
+        new ApplicationSubmissionContextPBImpl();
+
+    // applicationId will not be used because RMStateStore is mocked,
+    // but applicationId is still set for safety
+    submissionContext.setApplicationId(appId);
+    submissionContext.setPriority(Priority.newInstance(0));
+
+    RMApp application = new RMAppImpl(appId, rmContext, conf, name,
+        user, queue, submissionContext, scheduler, masterService,
+        System.currentTimeMillis(), "YARN", null,
+        new ArrayList<>());
+
+    rmAppMaps.putIfAbsent(application.getApplicationId(), application);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 3c00bbcdc071e..146a931e5acef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -18,10 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.metrics;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
@@ -30,6 +27,14 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
 import org.apache.hadoop.ipc.CallerContext;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -65,19 +70,34 @@
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(Parameterized.class)
 public class TestSystemMetricsPublisher {
 
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {{false, 0}, {true, 1}});
+  }
+
   private static ApplicationHistoryServer timelineServer;
   private static TimelineServiceV1Publisher metricsPublisher;
   private static TimelineStore store;
 
-  @BeforeClass
-  public static void setup() throws Exception {
+  private boolean rmTimelineServerV1PublisherBatchEnabled;
+  private int rmTimelineServerV1PublisherInterval;
+
+  public TestSystemMetricsPublisher(boolean rmTimelineServerV1PublisherBatchEnabled,
+      int rmTimelineServerV1PublisherInterval) {
+    this.rmTimelineServerV1PublisherBatchEnabled = rmTimelineServerV1PublisherBatchEnabled;
+    this.rmTimelineServerV1PublisherInterval = rmTimelineServerV1PublisherInterval;
+  }
+
+  @Before
+  public void setup() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
     conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
@@ -88,6 +108,10 @@ public static void setup() throws Exception {
     conf.setInt(
         YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
         2);
+    conf.setBoolean(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_BATCH_ENABLED,
+        rmTimelineServerV1PublisherBatchEnabled);
+    conf.setInt(YarnConfiguration.RM_TIMELINE_SERVER_V1_PUBLISHER_INTERVAL,
+        rmTimelineServerV1PublisherInterval);
 
     timelineServer = new ApplicationHistoryServer();
     timelineServer.init(conf);
@@ -99,8 +123,8 @@ public static void setup() throws Exception {
     metricsPublisher.start();
   }
 
-  @AfterClass
-  public static void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     if (metricsPublisher != null) {
       metricsPublisher.stop();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
index 27208020185f2..efd2ffc991cbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.Test;
+
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -26,10 +31,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
@@ -210,7 +211,7 @@ public void testInterQueuePreemptionWithStrictAndRelaxedDRF()
             "-b(=[20480:200 61440:600 0:0 20480:20 0]);" + // a
             "-c(=[20480:200 61440:600 0:0 0:0 0])"; // c
     String appsConfig =
-        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" + "(1,1024:1,n1,,60,false,0:0,user1);" + // app1 in a
         "b\t" + "(1,0:0,n1,,0,false,20480:20,user2);"; // app2 in b
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
index 807796242744e..2fb69f555728e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
@@ -18,13 +18,14 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
-import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
-import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import java.io.IOException;
+
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.Mockito.never;
@@ -845,7 +846,7 @@ public void testIntraQueuePreemptionWithTwoUsers()
             "-b(=[40 100 40 120 0])"; // b
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user(optional))
         "a\t" // app1 in a
             + "(1,1,n1,,5,false,25);" + // app1 a
             "a\t" // app2 in a
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java
index eb9d21836da51..def4554ec5b24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueFairOrdering.java
@@ -74,7 +74,7 @@ public void testIntraQueuePreemptionFairOrderingPolicyEnabledOneAppPerUser()
     // With FairOrderingPolicy enabled on queue a, all 20 resources should be
     // preempted from app1
     String appsConfig =
-    // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1, user1 in a
             + "(1,1,n1,,60,false,0,user1);" +
             "a\t" // app2, user2 in a
@@ -122,7 +122,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyEnabled()
     // With FifoOrderingPolicy enabled on queue a, the first 5 should come from
     // the youngest app, app2, until app2 is reduced to the user limit of 35.
     String appsConfig =
-    // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1, user1 in a
             + "(1,1,n1,,60,false,0,user1);" +
             "a\t" // app2, user2 in a
@@ -145,7 +145,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyEnabled()
     // The next 15 should come from app1 even though app2 is younger since app2
     // has already been reduced to its user limit.
     appsConfig =
-    // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1, user1 in a
             + "(1,1,n1,,60,false,0,user1);" +
             "a\t" // app2, user2 in a
@@ -195,7 +195,7 @@ public void testIntraQueuePreemptionFairOrderingPolicyMulitipleAppsPerUser()
     // preempted from app1 since it's the most over served app from the most
     // over served user
     String appsConfig =
-    // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 and app2, user1 in a
             + "(1,1,n1,,35,false,0,user1);" +
             "a\t"
@@ -243,7 +243,7 @@ public void testIntraQueuePreemptionFifoOrderingPolicyMultipleAppsPerUser()
     // user4/app4 is requesting 20 resources in queue a
     // With 3 users, preemptable user limit should be around 35 resources each.
     String appsConfig =
-    // queueName\t(prio,resource,host,expression,#repeat,reserved,pending,user)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1, user1 in a
             + "(1,1,n1,,40,false,0,user1);" +
         "a\t" // app2, user1 in a
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
index 9fb08cd3bc041..2a0838b83f453 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
@@ -18,12 +18,13 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
-import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import java.io.IOException;
+
 import org.junit.Before;
 import org.junit.Test;
 
-import java.io.IOException;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 
 import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.Mockito.times;
@@ -84,7 +85,7 @@ public void testSimpleIntraQueuePreemptionWithTwoUsers()
             "-a(=[100 100 100 30 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,100,false,0,user1);" + // app1 a
             "a\t" // app2 in a
@@ -139,7 +140,7 @@ public void testNoIntraQueuePreemptionWithSingleUser()
             "-a(=[100 100 100 30 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,100,false,0,user1);" + // app1 a
             "a\t" // app2 in a
@@ -196,7 +197,7 @@ public void testNoIntraQueuePreemptionWithTwoUserUnderUserLimit()
             "-a(=[100 100 80 30 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,50,false,0,user1);" + // app1 a
             "a\t" // app2 in a
@@ -252,7 +253,7 @@ public void testSimpleIntraQueuePreemptionWithTwoUsersWithAppPriority()
             "-a(=[100 100 100 30 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(2,1,n1,,100,false,0,user1);" + // app1 a
             "a\t" // app2 in a
@@ -309,7 +310,7 @@ public void testIntraQueuePreemptionOfUserLimitWithMultipleApps()
             "-a(=[100 100 80 90 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,30,false,30,user1);" + // app1 a
             "a\t" // app2 in a
@@ -370,7 +371,7 @@ public void testNoPreemptionOfUserLimitWithMultipleAppsAndSameUser()
             "-a(=[100 100 80 90 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,30,false,20,user1);" + // app1 a
             "a\t" // app2 in a
@@ -439,7 +440,7 @@ public void testIntraQueuePreemptionOfUserLimitWitAppsOfDifferentPriority()
             "-a(=[100 100 80 60 0])"; // b
 
     String appsConfig =
-        // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(3,1,n1,,30,false,30,user1);" + // app1 a
             "a\t" // app2 in a
@@ -490,7 +491,7 @@ public void testIntraQueuePreemptionOfUserLimitInTwoQueues()
             "-b(=[40 100 35 20 0])"; // b
 
     String appsConfig =
-        // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(3,1,n1,,20,false,30,user1);" + // app1 a
             "a\t" // app2 in a
@@ -556,7 +557,7 @@ public void testIntraQueuePreemptionWithTwoRequestingUsers()
             "-a(=[100 100 100 20 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,60,false,10,user1);" + // app1 a
             "a\t" // app2 in a
@@ -795,7 +796,7 @@ public void testNoIntraQueuePreemptionWithUserLimitDeadzone()
             "-a(=[100 100 100 20 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,3,n1,,20,false,10,user1);" + // app1 a
             "a\t" // app2 in a
@@ -855,7 +856,7 @@ public void testIntraQueuePreemptionWithUserLimitDeadzoneAndPriority()
             "-a(=[100 100 100 20 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,3,n1,,20,false,10,user1);" + // app1 a
             "a\t" // app2 in a
@@ -879,7 +880,7 @@ public void testIntraQueuePreemptionWithUserLimitDeadzoneAndPriority()
     // After first round, 3 containers were preempted from app1 and resource
     // distribution will be like below.
     appsConfig =
-        // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,3,n1,,17,false,10,user1);" + // app1 a
             "a\t" // app2 in a
@@ -914,7 +915,7 @@ public void testSimpleIntraQueuePreemptionOneUserUnderOneUserAtOneUserAbove()
             "-a(=[100 100 100 1 0])"; // a
 
     String appsConfig =
-    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+    // queueName\t(priority,resource,host,expression,#repeat,reserved,pending,user)
         "a\t" // app1 in a
             + "(1,1,n1,,65,false,0,user1);" +
             "a\t" // app2 in a
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockApplications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockApplications.java
index b16861257ffcd..aa00a1a438897 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockApplications.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockApplications.java
@@ -165,6 +165,7 @@ private void setupUserToQueueSettings(String label, String queueName,
         user.setResourceUsage(userResourceUsage.get(userName));
       }
       when(queue.getUser(eq(userName))).thenReturn(user);
+      when(queue.getOrCreateUser(eq(userName))).thenReturn(user);
       when(queue.getResourceLimitForAllUsers(eq(userName),
           any(Resource.class), anyString(), any(SchedulingMode.class)))
           .thenReturn(userLimit);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java
index 116c0ace70d22..1c7e5fa513175 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestPlacementRuleFS.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.placement;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
@@ -188,11 +189,10 @@ private void ruleInit(Class <? extends PlacementRule> ruleClass) {
 
   private Element createConf(String str) {
     // Create a simple rule element to use in the rule create
-    DocumentBuilderFactory docBuilderFactory =
-        DocumentBuilderFactory.newInstance();
-    docBuilderFactory.setIgnoringComments(true);
     Document doc = null;
     try {
+      DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
+      docBuilderFactory.setIgnoringComments(true);
       DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
       doc = builder.parse(IOUtils.toInputStream(str, StandardCharsets.UTF_8));
     } catch (Exception ex) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
index 0cf10595ee9dd..3e614bcbc967a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.security.Groups;
@@ -51,6 +53,7 @@
 import static junit.framework.TestCase.assertNull;
 import static junit.framework.TestCase.assertTrue;
 import static junit.framework.TestCase.fail;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING;
 import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -771,4 +774,138 @@ public void testSecondaryGroupNameCleanup() throws IOException {
         "Application should have been placed to root.groups.sec_dot_test_dot_grp",
         engine, app, "test.user", "root.groups.sec_dot_test_dot_grp");
   }
+
+  /**
+   * 1. Invoke Groups.reset(). This make sure that the underlying singleton {@link Groups#GROUPS}
+   * is set to null.<br>
+   * 2. Create a Configuration object in which the property "hadoop.security.group.mapping"
+   * refers to an existing a test implementation.<br>
+   * 3. Create a mock CapacityScheduler where getConf() and getConfiguration() contain different
+   * settings for "hadoop.security.group.mapping". Since getConf() is the service config, this
+   * should return the config object created in step #2.<br>
+   * 4. Create an instance of CSMappingPlacementRule with a single primary group rule.<br>
+   * 5. Run the placement evaluation.<br>
+   * 6. Expected: The returned queue matches what is supposed to be coming from the test group
+   * mapping service ("testuser" --> "testGroup1").<br>
+   * 7. Modify "hadoop.security.group.mapping" in the config object created in step #2.
+   * This step is required to guarantee that the CSMappingPlacementRule doesn't try to recreate
+   * the group mapping implementation and uses the one that was previously created.<br>
+   * 8. Call Groups.refresh() which changes the group mapping ("testuser" --> "testGroup0"). This
+   * requires that the test group mapping service implement GroupMappingServiceProvider
+   * .cacheGroupsRefresh().<br>
+   * 9. Create a new instance of CSMappingPlacementRule. This is important as we want to test
+   * that even this new {@link CSMappingPlacementRule} instance uses the same group mapping
+   * instance.<br>
+   * 10. Run the placement evaluation again<br>
+   * 11. Expected: with the same user, the target queue has changed to 'testGroup0'.<br>
+   * <p>
+   * These all looks convoluted, but the steps above make sure all the following conditions are met:
+   * <p>
+   * 1. CSMappingPlacementRule will force the initialization of groups.<br>
+   * 2. We select the correct configuration for group service init.<br>
+   * 3. We don't create a new Groups instance if the singleton is initialized, so we cover the
+   * original problem described in YARN-10597.<br>
+   */
+  @Test
+  public void testPlacementEngineSelectsCorrectConfigurationForGroupMapping() throws IOException {
+    Groups.reset();
+    final String user = "testuser";
+
+    //Create service-wide configuration object
+    Configuration yarnConf = new Configuration();
+    yarnConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, MockUnixGroupsMapping.class,
+        GroupMappingServiceProvider.class);
+
+    //Create CS configuration object with a single, primary group mapping rule
+    List<MappingRule> mappingRules = new ArrayList<>();
+    mappingRules.add(
+        new MappingRule(
+            MappingRuleMatchers.createUserMatcher(user),
+            (new MappingRuleActions.PlaceToQueueAction(
+                "root.man.%primary_group", true))
+                .setFallbackReject()));
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration() {
+      @Override
+      public List<MappingRule> getMappingRules() {
+        return mappingRules;
+      }
+    };
+    csConf.setOverrideWithQueueMappings(true);
+    //Intentionally add a dummy implementation class -
+    // The "HADOOP_SECURITY_GROUP_MAPPING" should not be read from the
+    // CapacitySchedulerConfiguration instance!
+    csConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, String.class, Object.class);
+
+    CapacityScheduler cs = createMockCS(yarnConf, csConf);
+
+    //Create app, submit to placement engine, expecting queue=testGroup1
+    CSMappingPlacementRule engine = initPlacementEngine(cs);
+    ApplicationSubmissionContext app = createApp("app");
+    assertPlace(engine, app, user, "root.man.testGroup1");
+
+    //Intentionally add a dummy implementation class!
+    // The "HADOOP_SECURITY_GROUP_MAPPING" should not be read from the
+    // CapacitySchedulerConfiguration instance!
+    //This makes sure that the Groups instance is not recreated by CSMappingPlacementRule
+    yarnConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, String.class, Object.class);
+
+    //Refresh the groups, this makes testGroup0 as primary group for "testUser"
+    engine.getGroups().refresh();
+
+    //Create app, submit to placement engine, expecting queue=testGroup0 (the new primary group)
+    engine = initPlacementEngine(cs);
+    assertPlace(engine, app, user, "root.man.testGroup0");
+  }
+
+  private CSMappingPlacementRule initPlacementEngine(CapacityScheduler cs) throws IOException {
+    CSMappingPlacementRule engine = new CSMappingPlacementRule();
+    engine.setFailOnConfigError(true);
+    engine.initialize(cs);
+    return engine;
+  }
+
+  private CapacityScheduler createMockCS(Configuration conf,
+      CapacitySchedulerConfiguration csConf) {
+    CapacitySchedulerQueueManager qm =
+        mock(CapacitySchedulerQueueManager.class);
+    createQueueHierarchy(qm);
+
+    CapacityScheduler cs = mock(CapacityScheduler.class);
+    when(cs.getConfiguration()).thenReturn(csConf);
+    when(cs.getConf()).thenReturn(conf);
+    when(cs.getCapacitySchedulerQueueManager()).thenReturn(qm);
+    return cs;
+  }
+
+  public static class MockUnixGroupsMapping implements GroupMappingServiceProvider {
+
+    public MockUnixGroupsMapping() {
+      GROUP.clear();
+      GROUP.add("testGroup1");
+      GROUP.add("testGroup2");
+      GROUP.add("testGroup3");
+    }
+
+    private static final List<String> GROUP = new ArrayList<>();
+
+    @Override
+    public List<String> getGroups(String user) throws IOException {
+      return GROUP;
+    }
+
+    @Override
+    public void cacheGroupsRefresh() {
+      GROUP.add(0, "testGroup0");
+    }
+
+    @Override
+    public void cacheGroupsAdd(List<String> groups) {
+      // Do nothing
+    }
+
+    @Override
+    public Set<String> getGroupsSet(String user) {
+      return ImmutableSet.copyOf(GROUP);
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 55654fb7b5ac6..1773c7aaa88e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -84,7 +84,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.security.MasterKeyData;
 import org.apache.hadoop.yarn.server.webproxy.ProxyCA;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.junit.Assert;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 13faff0fc8ac1..b775f4c4e7977 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -49,7 +49,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
index 4ad4c3cdcb9d4..a988211d78f14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
@@ -22,7 +22,7 @@
 import java.util.Collection;
 
 import net.jcip.annotations.NotThreadSafe;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
+
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
 import org.junit.Test;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index ad29d274a4f5b..96cff0f02894d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -119,6 +119,11 @@ public String getUser() {
     return user;
   }
 
+  @Override
+  public String getRealUser() {
+    return null;
+  }
+
   public void setUser(String user) {
     this.user = user;
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
index 14c14df0afb51..33253ecb3bffc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
@@ -25,7 +25,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
 import org.apache.hadoop.yarn.util.ControlledClock;
-import org.apache.hadoop.yarn.util.SystemClock;
+
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
index 391649d409227..a7c7ce85903c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
@@ -20,9 +20,11 @@
 
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.util.*;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Priority;
@@ -32,6 +34,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalityAppPlacementAllocator;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 import org.junit.Assert;
 import org.junit.Test;
@@ -175,4 +178,52 @@ public void testSchedulerKeyAccounting() {
     info.updateResourceRequests(reqs, false);
     Assert.assertEquals(0, info.getSchedulerKeys().size());
   }
+
+  @Test
+  public void testApplicationPlacementType() {
+    String DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS =
+        LocalityAppPlacementAllocator.class.getName();
+    Configuration conf = new Configuration();
+    RMContext rmContext = mock(RMContext.class);
+    when(rmContext.getYarnConfiguration()).thenReturn(conf);
+    ApplicationId appIdImpl = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appIdImpl, 1);
+    Queue queue = mock(Queue.class);
+    AppSchedulingInfo info = new AppSchedulingInfo(appAttemptId, "test", queue,
+        mock(ActiveUsersManager.class), 0, new ResourceUsage(), new HashMap<>(),
+        rmContext, false);
+    Assert.assertEquals(info.getApplicationSchedulingEnvs(), new HashMap<>());
+    // This should return null as nothing is set in the conf.
+    Assert.assertNull(info.getDefaultResourceRequestAppPlacementType());
+    conf = new Configuration();
+    conf.set(YarnConfiguration.APPLICATION_PLACEMENT_TYPE_CLASS,
+        DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS);
+    when(rmContext.getYarnConfiguration()).thenReturn(conf);
+    info = new AppSchedulingInfo(appAttemptId, "test", queue,
+        mock(ActiveUsersManager.class), 0, new ResourceUsage(), new HashMap<>(),
+        rmContext, false);
+    Assert.assertEquals(info.getDefaultResourceRequestAppPlacementType(),
+        DEFAULT_APPLICATION_PLACEMENT_TYPE_CLASS);
+  }
+
+  @Test
+  public void testApplicationPlacementTypeNotConfigured() {
+    Configuration conf = new Configuration();
+    RMContext rmContext = mock(RMContext.class);
+    when(rmContext.getYarnConfiguration()).thenReturn(conf);
+    ApplicationId appIdImpl = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appIdImpl, 1);
+    Queue queue = mock(Queue.class);
+    HashMap<String, String> applicationSchedulingEnvs = new HashMap<>();
+    applicationSchedulingEnvs.put("APPLICATION_PLACEMENT_TYPE_CLASS",
+        LocalityAppPlacementAllocator.class.getName());
+    AppSchedulingInfo info = new AppSchedulingInfo(appAttemptId, "test", queue,
+        mock(ActiveUsersManager.class), 0, new ResourceUsage(),
+        applicationSchedulingEnvs, rmContext, false);
+    // This should be set from applicationSchedulingEnvs
+    Assert.assertEquals(info.getDefaultResourceRequestAppPlacementType(),
+        LocalityAppPlacementAllocator.class.getName());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
index 873e3b95a4ca2..087b797f2d84c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
@@ -18,7 +18,11 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -50,4 +54,52 @@ public static Configuration createBasicCSConfiguration() {
     return createConfiguration(conf);
   }
 
+  public static void setMinAllocMb(Configuration conf, int minAllocMb) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        minAllocMb);
+  }
+
+  public static void setMaxAllocMb(Configuration conf, int maxAllocMb) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        maxAllocMb);
+  }
+
+  public static void setMaxAllocMb(CapacitySchedulerConfiguration conf,
+                                   String queueName, int maxAllocMb) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + MAXIMUM_ALLOCATION_MB;
+    conf.setInt(propName, maxAllocMb);
+  }
+
+  public static void setMinAllocVcores(Configuration conf, int minAllocVcores) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        minAllocVcores);
+  }
+
+  public static void setMaxAllocVcores(Configuration conf, int maxAllocVcores) {
+    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        maxAllocVcores);
+  }
+
+  public static void setMaxAllocVcores(CapacitySchedulerConfiguration conf,
+                                       String queueName, int maxAllocVcores) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
+    conf.setInt(propName, maxAllocVcores);
+  }
+
+  public static void setMaxAllocation(CapacitySchedulerConfiguration conf,
+                                      String queueName, String maxAllocation) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + MAXIMUM_ALLOCATION;
+    conf.set(propName, maxAllocation);
+  }
+
+  public static void unsetMaxAllocation(CapacitySchedulerConfiguration conf,
+                                        String queueName) {
+    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+        + MAXIMUM_ALLOCATION;
+    conf.unset(propName);
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerPreemptionTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerPreemptionTestBase.java
index 55ccb8afca952..41ee40ebcd2bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerPreemptionTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerPreemptionTestBase.java
@@ -18,11 +18,9 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.Application;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
@@ -32,7 +30,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.util.Clock;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
index bc7e2b317ac1f..1adde7d8d196e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
@@ -29,6 +29,7 @@ public final class CapacitySchedulerQueueHelpers {
 
   public static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
   public static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+  public static final String A_CHILD = A + ".a";
   public static final String A1 = A + ".a1";
   public static final String A2 = A + ".a2";
   public static final String B1 = B + ".b1";
@@ -89,6 +90,64 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration(
     return conf;
   }
 
+  public static CapacitySchedulerConfiguration setupAdditionalQueues(
+      CapacitySchedulerConfiguration conf) {
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"});
+
+    conf.setCapacity(A, A_CAPACITY);
+    conf.setCapacity(B, B_CAPACITY);
+
+    // Define 2nd-level queues
+    conf.setQueues(A, new String[]{"a1", "a2", "a3"});
+    conf.setCapacity(A1, 30.0f);
+    conf.setUserLimitFactor(A1, 100.0f);
+    conf.setCapacity(A2, 30.0f);
+    conf.setUserLimitFactor(A2, 100.0f);
+    conf.setCapacity("root.a.a3", 40.0f);
+    conf.setUserLimitFactor("root.a.a3", 100.0f);
+
+    conf.setQueues(B, new String[]{"b1", "b2", "b3"});
+    conf.setCapacity(B1, B1_CAPACITY);
+    conf.setUserLimitFactor(B1, 100.0f);
+    conf.setCapacity(B2, B2_CAPACITY);
+    conf.setUserLimitFactor(B2, 100.0f);
+    conf.setCapacity(B3, B3_CAPACITY);
+    conf.setUserLimitFactor(B3, 100.0f);
+
+    return conf;
+  }
+
+  /**
+   * @param conf, to be modified
+   * @return CS configuration which has deleted all children of queue(b)
+   *           root
+   *          /     \
+   *        a        b
+   *       / \
+   *      a1  a2
+   */
+  public static CapacitySchedulerConfiguration setupQueueConfAmbiguousQueue(
+      CapacitySchedulerConfiguration conf) {
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{"a", "b"});
+
+    conf.setCapacity(A, A_CAPACITY);
+    conf.setCapacity(B, B_CAPACITY);
+
+    // Define 2nd-level queues
+    conf.setQueues(A, new String[]{"a", "a1"});
+    conf.setCapacity(A_CHILD, A1_CAPACITY);
+    conf.setUserLimitFactor(A1, 100.0f);
+    conf.setCapacity(A1, A2_CAPACITY);
+    conf.setUserLimitFactor(A2, 100.0f);
+
+    return conf;
+  }
+
   /**
    * @param conf, to be modified
    * @return CS configuration which has deleted all childred of queue(b)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java
index 3d098f837a8fd..b85874c62ca9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerTestUtilities.java
@@ -18,14 +18,49 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.Application;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.NodeManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.junit.Assert;
 
+import java.io.IOException;
 import java.util.Set;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfAmbiguousQueue;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 public final class CapacitySchedulerTestUtilities {
   public static final int GB = 1024;
 
@@ -69,4 +104,129 @@ public static void waitforNMRegistered(ResourceScheduler scheduler, int nodecoun
       }
     }
   }
+
+  public static ResourceManager createResourceManager() throws Exception {
+    ResourceUtils.resetResourceTypes(new Configuration());
+    DefaultMetricsSystem.setMiniClusterMode(true);
+    ResourceManager resourceManager = new ResourceManager() {
+      @Override
+      protected RMNodeLabelsManager createNodeLabelManager() {
+        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+        mgr.init(getConfig());
+        return mgr;
+      }
+    };
+    CapacitySchedulerConfiguration csConf
+        = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(csConf);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER,
+        CapacityScheduler.class, ResourceScheduler.class);
+    resourceManager.init(conf);
+    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
+    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
+    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
+    return resourceManager;
+  }
+
+  public static RMContext createMockRMContext() {
+    RMContext mockContext = mock(RMContext.class);
+    when(mockContext.getConfigurationProvider()).thenReturn(
+        new LocalConfigurationProvider());
+    return mockContext;
+  }
+
+  public static void stopResourceManager(ResourceManager resourceManager) throws Exception {
+    if (resourceManager != null) {
+      QueueMetrics.clearQueueMetrics();
+      DefaultMetricsSystem.shutdown();
+      resourceManager.stop();
+    }
+  }
+
+  public static ApplicationAttemptId appHelper(MockRM rm, CapacityScheduler cs,
+                                         int clusterTs, int appId, String queue,
+                                         String user) {
+    ApplicationId appId1 = BuilderUtils.newApplicationId(clusterTs, appId);
+    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
+        appId1, appId);
+
+    RMAppAttemptMetrics attemptMetric1 =
+        new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext());
+    RMAppImpl app1 = mock(RMAppImpl.class);
+    when(app1.getApplicationId()).thenReturn(appId1);
+    RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
+    Container container = mock(Container.class);
+    when(attempt1.getMasterContainer()).thenReturn(container);
+    ApplicationSubmissionContext submissionContext = mock(
+        ApplicationSubmissionContext.class);
+    when(attempt1.getSubmissionContext()).thenReturn(submissionContext);
+    when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
+    when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
+    when(app1.getCurrentAppAttempt()).thenReturn(attempt1);
+    rm.getRMContext().getRMApps().put(appId1, app1);
+
+    SchedulerEvent addAppEvent1 =
+        new AppAddedSchedulerEvent(appId1, queue, user);
+    cs.handle(addAppEvent1);
+    SchedulerEvent addAttemptEvent1 =
+        new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
+    cs.handle(addAttemptEvent1);
+    return appAttemptId1;
+  }
+
+  public static MockRM setUpMoveAmbiguousQueue() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfAmbiguousQueue(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    return rm;
+  }
+
+  public static MockRM setUpMove() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    return setUpMove(conf);
+  }
+
+  public static MockRM setUpMove(Configuration config) {
+    CapacitySchedulerConfiguration conf =
+        new CapacitySchedulerConfiguration(config);
+    setupQueueConfiguration(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    return rm;
+  }
+
+  public static void nodeUpdate(ResourceManager rm, NodeManager nm) {
+    RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
+    // Send a heartbeat to kick the tires on the Scheduler
+    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
+    rm.getResourceScheduler().handle(nodeUpdate);
+  }
+
+  public static NodeManager registerNode(ResourceManager rm, String hostName,
+                                   int containerManagerPort, int httpPort, String rackName,
+                                   Resource capability, NodeStatus nodeStatus)
+      throws IOException, YarnException {
+    NodeManager nm = new NodeManager(hostName,
+        containerManagerPort, httpPort, rackName, capability, rm, nodeStatus);
+    NodeAddedSchedulerEvent nodeAddEvent1 =
+        new NodeAddedSchedulerEvent(rm.getRMContext().getRMNodes()
+            .get(nm.getNodeId()));
+    rm.getResourceScheduler().handle(nodeAddEvent1);
+    return nm;
+  }
+
+  public static void checkApplicationResourceUsage(int expected, Application application) {
+    Assert.assertEquals(expected, application.getUsedResources().getMemorySize());
+  }
+
+  public static void checkNodeResourceUsage(int expected, NodeManager node) {
+    Assert.assertEquals(expected, node.getUsed().getMemorySize());
+    node.checkResourceUsage();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAMAllocatedToNonExclusivePartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAMAllocatedToNonExclusivePartition.java
new file mode 100644
index 0000000000000..469d12c149866
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAMAllocatedToNonExclusivePartition.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+
+public class TestAMAllocatedToNonExclusivePartition {
+  private Configuration conf;
+  private RMNodeLabelsManager mgr;
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+    mgr = new NullRMNodeLabelsManager();
+    mgr.init(conf);
+  }
+
+  @SuppressWarnings("unchecked")
+  private <E> Set<E> toSet(E... elements) {
+    Set<E> set = Sets.newHashSet(elements);
+    return set;
+  }
+
+  /**
+   * Test that AM can be allocated to non-exclusive partition when the config
+   * of {@code YarnConfiguration.AM_ALLOW_NON_EXCLUSIVE_ALLOCATION} is true.
+   */
+  @Test
+  public void testAMAllowToNonExclusivePartition() throws Exception {
+    conf.setBoolean(YarnConfiguration.AM_ALLOW_NON_EXCLUSIVE_ALLOCATION, true);
+
+    mgr.addToCluserNodeLabels(
+        Arrays.asList(NodeLabel.newInstance("x", false))
+    );
+    mgr.addLabelsToNode(
+        ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))
+    );
+
+    conf = TestUtils.getConfigurationWithDefaultQueueLabels(conf);
+
+    MockRM rm1 = new MockRM(conf) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+    rm1.getRMContext().setNodeLabelManager(mgr);
+    rm1.start();
+
+    MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x
+
+    MockRMAppSubmissionData data2 =
+        MockRMAppSubmissionData
+            .Builder
+            .createWithMemory(200, rm1)
+            .withAppName("app")
+            .withUser("user")
+            .withAcls(null)
+            .withQueue("c1")
+            .withUnmanagedAM(false)
+            .build();
+
+    RMApp app1 = MockRMAppSubmitter.submit(rm1, data2);
+    // Wait the AM allocated to non-partition node of h1
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+    ContainerId containerId;
+
+    // Request a container and it also should be allocated to non-partition node of h1
+    am1.allocate("*", 1024, 1, new ArrayList<ContainerId>());
+    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+    Assert.assertTrue(rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED));
+
+    rm1.close();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 97f9a652c1be7..8d68cbf7932eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -22,12 +22,16 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -44,18 +48,21 @@ public class TestAbsoluteResourceConfiguration {
   private static final String QUEUEA2 = "queueA2";
   private static final String QUEUEB1 = "queueB1";
 
-  private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
-      + "." + QUEUEA;
-  private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
-      + "." + QUEUEB;
-  private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
-      + "." + QUEUEC;
-  private static final String QUEUED_FULL = CapacitySchedulerConfiguration.ROOT
-      + "." + QUEUED;
-
-  private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
-  private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
-  private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+  private static final QueuePath QUEUEA_FULL =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEA);
+  private static final QueuePath QUEUEB_FULL =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEB);
+  private static final QueuePath QUEUEC_FULL =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEC);
+  private static final QueuePath QUEUED_FULL =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUED);
+
+  private static final QueuePath QUEUEA1_FULL =
+          new QueuePath(QUEUEA_FULL.getFullPath() + "." + QUEUEA1);
+  private static final QueuePath QUEUEA2_FULL =
+          new QueuePath(QUEUEA_FULL.getFullPath() + "." + QUEUEA2);
+  private static final QueuePath QUEUEB1_FULL =
+          new QueuePath(QUEUEB_FULL.getFullPath() + "." + QUEUEB1);
 
   private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
       10);
@@ -88,10 +95,27 @@ public class TestAbsoluteResourceConfiguration {
       Resource.newInstance(25 * GB, 5);
   private static final Resource QUEUE_D_TEMPL_MAXRES =
       Resource.newInstance(150 * GB, 20);
+  public static final String X_LABEL = "X";
+  public static final String Y_LABEL = "Y";
 
   private static Set<String> resourceTypes = new HashSet<>(
       Arrays.asList("memory", "vcores"));
 
+  private CapacitySchedulerConfiguration setupNormalizationConfiguration() {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{QUEUEA, QUEUEB});
+    csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2});
+
+//    60, 28
+    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resource.newInstance(50 * GB, 20));
+    csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, Resource.newInstance(30 * GB, 15));
+    csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, Resource.newInstance(20 * GB, 5));
+    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resource.newInstance(10 * GB, 8));
+
+    return csConf;
+  }
+
   private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
       boolean isCapacityNeeded) {
     CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
@@ -100,18 +124,18 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
 
     // Set default capacities like normal configuration.
     if (isCapacityNeeded) {
-      csConf.setCapacity(QUEUEA_FULL, 50f);
-      csConf.setCapacity(QUEUEB_FULL, 25f);
-      csConf.setCapacity(QUEUEC_FULL, 25f);
-      csConf.setCapacity(QUEUED_FULL, 25f);
+      csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
+      csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
+      csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
+      csConf.setCapacity(QUEUED_FULL.getFullPath(), 25f);
     }
 
-    csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true);
+    csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL.getFullPath(), true);
 
     // Setup leaf queue template configs
-    csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "",
+    csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL.getFullPath(), "",
         QUEUE_D_TEMPL_MINRES);
-    csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "",
+    csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL.getFullPath(), "",
         QUEUE_D_TEMPL_MAXRES);
 
     return csConf;
@@ -122,24 +146,45 @@ private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
     CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
     csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
         new String[]{QUEUEA, QUEUEB, QUEUEC});
-    csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
-    csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+    csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2});
+    csConf.setQueues(QUEUEB_FULL.getFullPath(), new String[]{QUEUEB1});
 
     // Set default capacities like normal configuration.
     if (isCapacityNeeded) {
-      csConf.setCapacity(QUEUEA_FULL, 50f);
-      csConf.setCapacity(QUEUEB_FULL, 25f);
-      csConf.setCapacity(QUEUEC_FULL, 25f);
-      csConf.setCapacity(QUEUEA1_FULL, 50f);
-      csConf.setCapacity(QUEUEA2_FULL, 50f);
-      csConf.setCapacity(QUEUEB1_FULL, 100f);
+      csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
+      csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
+      csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
+      csConf.setCapacity(QUEUEA1_FULL.getFullPath(), 50f);
+      csConf.setCapacity(QUEUEA2_FULL.getFullPath(), 50f);
+      csConf.setCapacity(QUEUEB1_FULL.getFullPath(), 100f);
     }
 
     return csConf;
   }
 
+  private CapacitySchedulerConfiguration setupLabeledConfiguration(
+      CapacitySchedulerConfiguration csConf) {
+    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resource.newInstance(20 * GB, 8));
+    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resource.newInstance(10 * GB, 3));
+    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resource.newInstance(10 * GB, 2));
+    csConf.setMinimumResourceRequirement("", QUEUED_FULL, Resource.newInstance(10 * GB, 2));
+
+    csConf.setMinimumResourceRequirement(X_LABEL, QUEUEA_FULL, Resource.newInstance(20 * GB, 8));
+    csConf.setMinimumResourceRequirement(X_LABEL, QUEUEB_FULL, Resource.newInstance(10 * GB, 3));
+    csConf.setMinimumResourceRequirement(X_LABEL, QUEUEC_FULL, Resource.newInstance(10 * GB, 2));
+    csConf.setMinimumResourceRequirement(X_LABEL, QUEUED_FULL, Resource.newInstance(10 * GB, 2));
+
+    csConf.setMinimumResourceRequirement(Y_LABEL, QUEUEA_FULL, Resource.newInstance(2 * GB, 1));
+    csConf.setMinimumResourceRequirement(Y_LABEL, QUEUEB_FULL, Resource.newInstance(2 * GB, 1));
+    csConf.setMinimumResourceRequirement(Y_LABEL, QUEUEC_FULL, Resource.newInstance(2 * GB, 1));
+    csConf.setMinimumResourceRequirement(Y_LABEL, QUEUED_FULL, Resource.newInstance(2 * GB, 2));
+
+    return csConf;
+  }
+
   private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
       CapacitySchedulerConfiguration csConf) {
+
     // Update min/max resource to queueA/B/C
     csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
     csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
@@ -180,22 +225,22 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
 
     Assert.assertEquals("Min resource configured for QUEUEA is not correct",
         QUEUE_A_MINRES,
-        csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+        csConf.getMinimumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes));
     Assert.assertEquals("Max resource configured for QUEUEA is not correct",
         QUEUE_A_MAXRES,
-        csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+        csConf.getMaximumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes));
     Assert.assertEquals("Min resource configured for QUEUEB is not correct",
         QUEUE_B_MINRES,
-        csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+        csConf.getMinimumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes));
     Assert.assertEquals("Max resource configured for QUEUEB is not correct",
         QUEUE_B_MAXRES,
-        csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+        csConf.getMaximumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes));
     Assert.assertEquals("Min resource configured for QUEUEC is not correct",
         QUEUE_C_MINRES,
-        csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+        csConf.getMinimumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes));
     Assert.assertEquals("Max resource configured for QUEUEC is not correct",
         QUEUE_C_MAXRES,
-        csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+        csConf.getMaximumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes));
 
     csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
         ResourceScheduler.class);
@@ -211,7 +256,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
 
     ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED);
-    AutoCreatedLeafQueue d1 = new AutoCreatedLeafQueue(cs, "d1", parentQueue);
+    AutoCreatedLeafQueue d1 = new AutoCreatedLeafQueue(cs.getQueueContext(), "d1", parentQueue);
     cs.addQueue(d1);
 
     /**
@@ -222,13 +267,13 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
         new ResourceLimits(cs.getClusterResource()));
 
     Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
-        d1.queueResourceQuotas.getConfiguredMinResource());
+        d1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
-        d1.queueResourceQuotas.getEffectiveMinResource());
+        d1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d1.queueResourceQuotas.getConfiguredMaxResource());
+        d1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d1.queueResourceQuotas.getEffectiveMaxResource());
+        d1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     /**
      * After adding child queue d2, d1 + d2 > resource
@@ -236,33 +281,64 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
      * d1 will occupy all entire resource
      * of Managed Parent queue.
      */
-    AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue);
+    AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs.getQueueContext(), "d2", parentQueue);
     cs.addQueue(d2);
 
     cs.getRootQueue().updateClusterResource(cs.getClusterResource(),
         new ResourceLimits(cs.getClusterResource()));
 
     Assert.assertEquals(Resource.newInstance(0, 0),
-        d2.queueResourceQuotas.getConfiguredMinResource());
+        d2.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals(Resource.newInstance(0, 0),
-        d2.queueResourceQuotas.getEffectiveMinResource());
+        d2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d2.queueResourceQuotas.getConfiguredMaxResource());
+        d2.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d2.queueResourceQuotas.getEffectiveMaxResource());
+        d2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
-        d1.queueResourceQuotas.getConfiguredMinResource());
+        d1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
-        d1.queueResourceQuotas.getEffectiveMinResource());
+        d1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d1.queueResourceQuotas.getConfiguredMaxResource());
+        d1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
-        d1.queueResourceQuotas.getEffectiveMaxResource());
+        d1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     rm.close();
   }
 
+  @Test
+  public void testNormalizationAfterNodeRemoval() throws Exception {
+    CapacitySchedulerConfiguration csConf = setupNormalizationConfiguration();
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    MockRM rm = new MockRM(csConf);
+
+    rm.start();
+    rm.registerNode("h1:1234", 8 * GB, 4);
+    rm.registerNode("h2:1234", 8 * GB, 4);
+    rm.registerNode("h3:1234", 8 * GB, 4);
+    MockNM nm = rm.registerNode("h4:1234", 8 * GB, 4);
+    rm.registerNode("h5:1234", 28 * GB, 12);
+
+    // Send a removal event to CS. MockRM#unregisterNode does not reflect the real world scenario,
+    // therefore we manually need to invoke this removal event.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    cs.handle(new NodeRemovedSchedulerEvent(rm.getRMContext().getRMNodes().get(nm.getNodeId())));
+
+    Resource res = Resources.add(
+        cs.getQueue(QUEUEA1_FULL.getFullPath()).getEffectiveCapacity(""),
+        cs.getQueue(QUEUEA2_FULL.getFullPath()).getEffectiveCapacity(""));
+    Resource resParent = cs.getQueue(QUEUEA_FULL.getFullPath()).getEffectiveCapacity("");
+
+    // Check if there is no overcommitment on behalf of the child queues
+    Assert.assertTrue(String.format("Summarized resource %s of all children is greater than " +
+        "their parent's %s", res, resParent),
+        Resources.lessThan(cs.getResourceCalculator(), cs.getClusterResource(), res, resParent));
+  }
+
   @Test
   public void testEffectiveMinMaxResourceConfigurartionPerQueue()
       throws Exception {
@@ -287,35 +363,35 @@ public void testEffectiveMinMaxResourceConfigurartionPerQueue()
     LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
     Assert.assertNotNull(qA);
     Assert.assertEquals("Min resource configured for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
     Assert.assertNotNull(qB);
     Assert.assertEquals("Min resource configured for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
     Assert.assertNotNull(qC);
     Assert.assertEquals("Min resource configured for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     rm.stop();
   }
@@ -378,37 +454,37 @@ public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
 
     LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
     Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
-        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
-        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
     Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
-        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
-        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
     Assert.assertNotNull(qB1);
     Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
-        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
-        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
-        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
-        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
     Assert.assertNotNull(qC);
     Assert.assertEquals("Min resource configured for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     // 3. Create a new config and make sure one queue's min resource is more
     // than its max resource configured.
@@ -487,9 +563,9 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception {
     // 1. Explicitly set percentage based config for parent queues. This will
     // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
     // resource.
-    csConf.setCapacity(QUEUEA_FULL, 50f);
-    csConf.setCapacity(QUEUEB_FULL, 25f);
-    csConf.setCapacity(QUEUEC_FULL, 25f);
+    csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
+    csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
+    csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
 
     // Get queue object to verify min/max resource configuration.
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -534,13 +610,13 @@ public void testValidateAbsoluteResourceConfig() throws Exception {
         new CapacitySchedulerConfiguration();
     csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
         new String[] {QUEUEA, QUEUEB});
-    csConf.setQueues(QUEUEA_FULL, new String[] {QUEUEA1, QUEUEA2});
+    csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[] {QUEUEA1, QUEUEA2});
 
     // Set default capacities like normal configuration.
-    csConf.setCapacity(QUEUEA_FULL, "[memory=125]");
-    csConf.setCapacity(QUEUEB_FULL, "[memory=0]");
-    csConf.setCapacity(QUEUEA1_FULL, "[memory=100]");
-    csConf.setCapacity(QUEUEA2_FULL, "[memory=25]");
+    csConf.setCapacity(QUEUEA_FULL.getFullPath(), "[memory=125]");
+    csConf.setCapacity(QUEUEB_FULL.getFullPath(), "[memory=0]");
+    csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=100]");
+    csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=25]");
 
     // Update min/max resource to queueA
     csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
@@ -560,8 +636,8 @@ public void testValidateAbsoluteResourceConfig() throws Exception {
     // doesnt throw exception saying "Parent queue 'root.A' and
     // child queue 'root.A.A2' should use either percentage
     // based capacityconfiguration or absolute resource together for label"
-    csConf.setCapacity(QUEUEA1_FULL, "[memory=125]");
-    csConf.setCapacity(QUEUEA2_FULL, "[memory=0]");
+    csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=125]");
+    csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=0]");
 
     // Get queue object to verify min/max resource configuration.
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -572,6 +648,42 @@ public void testValidateAbsoluteResourceConfig() throws Exception {
     }
   }
 
+  @Test
+  public void testDownscalingForLabels() throws Exception {
+    CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(false);
+    setupLabeledConfiguration(csConf);
+
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 8 * GB, 5);
+    MockNM nm2 = rm.registerNode("127.0.0.2:1234", 8 * GB, 5);
+    MockNM nm3 = rm.registerNode("127.0.0.3:1234", 8 * GB, 5);
+    MockNM nm4 = rm.registerNode("127.0.0.4:1234", 8 * GB, 5);
+
+    rm.getRMContext().getNodeLabelManager().addToCluserNodeLabelsWithDefaultExclusivity(
+        ImmutableSet.of(X_LABEL, Y_LABEL));
+    rm.getRMContext().getNodeLabelManager().addLabelsToNode(
+        ImmutableMap.of(nm1.getNodeId(), ImmutableSet.of(X_LABEL),
+            nm2.getNodeId(), ImmutableSet.of(X_LABEL),
+            nm3.getNodeId(), ImmutableSet.of(X_LABEL),
+            nm4.getNodeId(), ImmutableSet.of(Y_LABEL)));
+
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    CSQueue root = cs.getRootQueue();
+    root.updateClusterResource(cs.getClusterResource(), new ResourceLimits(cs.getClusterResource()));
+
+    Resource childrenResource = root.getChildQueues().stream().map(q -> q.getEffectiveCapacity(
+        X_LABEL)).reduce(Resources::add).orElse(Resource.newInstance(0, 0));
+
+    Assert.assertTrue("Children of root have more resource than overall cluster resource",
+        Resources.greaterThan(cs.getResourceCalculator(), cs.getClusterResource(),
+            root.getEffectiveCapacity(X_LABEL), childrenResource));
+  }
+
   @Test
   public void testEffectiveResourceAfterReducingClusterResource()
       throws Exception {
@@ -597,35 +709,35 @@ public void testEffectiveResourceAfterReducingClusterResource()
     LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
     Assert.assertNotNull(qA);
     Assert.assertEquals("Min resource configured for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
     Assert.assertNotNull(qB);
     Assert.assertEquals("Min resource configured for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
     Assert.assertNotNull(qC);
     Assert.assertEquals("Min resource configured for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     // unregister one NM.
     rm.unRegisterNode(nm1);
@@ -634,19 +746,19 @@ public void testEffectiveResourceAfterReducingClusterResource()
     // above half. Hence A's min will be 60Gi and 6 cores and max will be
     // 128GB and 20 cores.
     Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
-        QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+        QUEUEA_REDUCED, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
-        QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUEMAX_REDUCED, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
-        QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+        QUEUEB_REDUCED, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
-        QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUEMAX_REDUCED, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUEC_REDUCED, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUEMAX_REDUCED, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
 
     rm.stop();
   }
@@ -676,13 +788,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
     ParentQueue qA = (ParentQueue) cs.getQueue(QUEUEA);
     Assert.assertNotNull(qA);
     Assert.assertEquals("Min resource configured for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
         0.4, qA.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
@@ -691,13 +803,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
     ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
     Assert.assertNotNull(qB);
     Assert.assertEquals("Min resource configured for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
         0.2, qB.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
@@ -706,13 +818,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
     LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
     Assert.assertNotNull(qC);
     Assert.assertEquals("Min resource configured for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
         0.1, qC.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
@@ -720,9 +832,9 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
 
     LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
     Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
-        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
-        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
         0.2, qA1.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
@@ -730,9 +842,9 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
 
     LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
     Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
-        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
-        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
         0.2, qA2.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
@@ -740,13 +852,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
 
     LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
     Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
-        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+        QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource());
     Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
-        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+        QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource());
     Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
-        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
-        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
         0.16, qB1.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
@@ -759,54 +871,54 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
     // Since configured capacity was based on initial node capacity, a
     // re configurations is needed to use this added capacity.
     Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
-        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
-        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
         0.266, qA.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
         0.533, qA.getAbsoluteMaximumCapacity(), DELTA);
 
     Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
-        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
-        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
         0.133, qB.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
         0.4, qB.getAbsoluteMaximumCapacity(), DELTA);
 
     Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
-        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
-        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
         0.066, qC.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
         0.4, qC.getAbsoluteMaximumCapacity(), DELTA);
 
     Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
-        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
-        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
         0.106, qB1.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
         0.4, qB1.getAbsoluteMaximumCapacity(), DELTA);
 
     Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
-        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
-        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
         0.133, qA1.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
         0.533, qA1.getAbsoluteMaximumCapacity(), DELTA);
 
     Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
-        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+        QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource());
     Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
-        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+        QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource());
     Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
         0.133, qA2.getAbsoluteCapacity(), DELTA);
     Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
index 2bad8b7447b4f..e826bcb8fbdb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
@@ -96,15 +96,15 @@ public void setUp() throws Exception {
   private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
       CapacitySchedulerConfiguration csConf) {
     // Update min/max resource to queueA/B/C
-    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
-    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
-    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
-    csConf.setMinimumResourceRequirement("", QUEUED_FULL, QUEUE_D_MINRES);
+    csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MINRES);
+    csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MINRES);
+    csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MINRES);
+    csConf.setMinimumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MINRES);
 
-    csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
-    csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
-    csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
-    csConf.setMaximumResourceRequirement("", QUEUED_FULL, QUEUE_D_MAXRES);
+    csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MAXRES);
+    csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MAXRES);
+    csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MAXRES);
+    csConf.setMaximumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MAXRES);
 
     return csConf;
   }
@@ -148,8 +148,6 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
     return csConf;
   }
 
-  // TODO: Wangda: I think this test case is not correct, Sunil could help look
-  // into details.
   @Test(timeout = 20000)
   public void testAutoCreateLeafQueueCreation() throws Exception {
 
@@ -182,10 +180,8 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
       ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(QUEUED);
       assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
 
-      validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue, 0.4f,
-          0.04f, 1f, 0.6f);
-      validateCapacitiesByLabel((ManagedParentQueue) parentQueue,
-          (AutoCreatedLeafQueue) autoCreatedLeafQueue, NO_LABEL);
+      validateCapacities(autoCreatedLeafQueue, 0.4f, 0.04f, 1f, 0.6f);
+      validateCapacitiesByLabel(parentQueue, autoCreatedLeafQueue, NO_LABEL);
 
       Map<String, Float> expectedChildQueueAbsCapacity =
           new HashMap<String, Float>() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index a42e4af5dbf0f..f06631d6c15d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -107,19 +107,11 @@ public void setUp() throws IOException {
     rmContext = TestUtils.getMockRMContext();
     Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
 
-    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
-    when(csContext.getConfiguration()).thenReturn(csConf);
-    when(csContext.getConf()).thenReturn(conf);
-    when(csContext.getMinimumResourceCapability()).
-        thenReturn(Resources.createResource(GB, 1));
-    when(csContext.getMaximumResourceCapability()).
-        thenReturn(Resources.createResource(16*GB, 32));
-    when(csContext.getClusterResource()).
-        thenReturn(clusterResource);
-    when(csContext.getResourceCalculator()).
-        thenReturn(resourceCalculator);
+    CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
+        Resources.createResource(GB, 1), Resources.createResource(16*GB, 32),
+        clusterResource);
     when(csContext.getRMContext()).thenReturn(rmContext);
-    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
     
     RMContainerTokenSecretManager containerTokenSecretManager =
         new RMContainerTokenSecretManager(conf);
@@ -129,13 +121,13 @@ public void setUp() throws IOException {
 
     CSQueueStore queues = new CSQueueStore();
     root = CapacitySchedulerQueueManager
-        .parseQueue(csContext, csConf, null, "root",
+        .parseQueue(queueContext, csConf, null, "root",
             queues, queues,
             TestUtils.spyHook);
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
-    queue = spy(new LeafQueue(csContext, A, root, null));
+    queue = spy(new LeafQueue(queueContext, A, root, null));
     QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
         .getQueueResourceQuotas();
     doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
@@ -275,31 +267,27 @@ public void testAMResourceLimit() throws Exception {
   
   @Test
   public void testLimitsComputation() throws Exception {
+    final float epsilon = 1e-5f;
+
     CapacitySchedulerConfiguration csConf = 
         new CapacitySchedulerConfiguration();
     setupQueueConfiguration(csConf);
-    YarnConfiguration conf = new YarnConfiguration();
-    
-    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
-    when(csContext.getConfiguration()).thenReturn(csConf);
-    when(csContext.getConf()).thenReturn(conf);
-    when(csContext.getMinimumResourceCapability()).
-        thenReturn(Resources.createResource(GB, 1));
-    when(csContext.getMaximumResourceCapability()).
-        thenReturn(Resources.createResource(16*GB, 16));
-    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
-    when(csContext.getRMContext()).thenReturn(rmContext);
-    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
-    
+
     // Say cluster has 100 nodes of 16G each
     Resource clusterResource = 
       Resources.createResource(100 * 16 * GB, 100 * 16);
-    when(csContext.getClusterResource()).thenReturn(clusterResource);
+
+    CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
+        Resources.createResource(GB, 1), Resources.createResource(16*GB, 16),
+        clusterResource);
+    CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
 
     CSQueueStore queues = new CSQueueStore();
     CSQueue root = 
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             "root", queues, queues, TestUtils.spyHook);
+    queueManager.setRootQueue(root);
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
@@ -314,7 +302,7 @@ public void testLimitsComputation() throws Exception {
     assertThat(queue.calculateAndGetAMResourceLimit()).
         isEqualTo(amResourceLimit);
     assertThat(queue.getUserAMResourceLimit()).isEqualTo(
-      Resource.newInstance(80*GB, 1));
+        Resource.newInstance(80*GB, 1));
     
     // Assert in metrics
     assertThat(queue.getMetrics().getAMResourceLimitMB()).isEqualTo(
@@ -322,10 +310,8 @@ public void testLimitsComputation() throws Exception {
     assertThat(queue.getMetrics().getAMResourceLimitVCores()).isEqualTo(
         amResourceLimit.getVirtualCores());
 
-    assertEquals(
-        (int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
-        queue.getMetrics().getAvailableMB()
-        );
+    assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
+        queue.getMetrics().getAvailableMB());
     
     // Add some nodes to the cluster & test new limits
     clusterResource = Resources.createResource(120 * 16 * GB);
@@ -337,10 +323,8 @@ public void testLimitsComputation() throws Exception {
     assertThat(queue.getUserAMResourceLimit()).isEqualTo(
         Resource.newInstance(96*GB, 1));
     
-    assertEquals(
-        (int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
-        queue.getMetrics().getAvailableMB()
-        );
+    assertEquals((int)(clusterResource.getMemorySize() * queue.getAbsoluteCapacity()),
+        queue.getMetrics().getAvailableMB());
 
     // should return -1 if per queue setting not set
     assertEquals(
@@ -358,30 +342,29 @@ public void testLimitsComputation() throws Exception {
     assertEquals(expectedMaxAppsPerUser, queue.getMaxApplicationsPerUser());
 
     // should default to global setting if per queue setting not set
-    assertEquals(
-        (long)CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, 
-        (long)csConf.getMaximumApplicationMasterResourcePerQueuePercent(
-            queue.getQueuePath())
-            );
+    assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+        csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+            queue.getQueuePath()), epsilon);
 
     // Change the per-queue max AM resources percentage.
     csConf.setFloat(PREFIX + queue.getQueuePath()
         + ".maximum-am-resource-percent", 0.5f);
+    queueContext.reinitialize();
     // Re-create queues to get new configs.
     queues = new CSQueueStore();
     root = CapacitySchedulerQueueManager.parseQueue(
-        csContext, csConf, null, "root",
+        queueContext, csConf, null, "root",
         queues, queues, TestUtils.spyHook);
     clusterResource = Resources.createResource(100 * 16 * GB);
+    queueManager.setRootQueue(root);
     root.updateClusterResource(clusterResource, new ResourceLimits(
         clusterResource));
 
     queue = (LeafQueue)queues.get(A);
 
-    assertEquals((long) 0.5, 
-        (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
-          queue.getQueuePath())
-        );
+    assertEquals(0.5f,
+        csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+          queue.getQueuePath()), epsilon);
 
     assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo(
         Resource.newInstance(800 * GB, 1));
@@ -391,10 +374,11 @@ public void testLimitsComputation() throws Exception {
     // Change the per-queue max applications.
     csConf.setInt(PREFIX + queue.getQueuePath() + ".maximum-applications",
         9999);
+    queueContext.reinitialize();
     // Re-create queues to get new configs.
     queues = new CSQueueStore();
     root = CapacitySchedulerQueueManager.parseQueue(
-        csContext, csConf, null, "root",
+        queueContext, csConf, null, "root",
         queues, queues, TestUtils.spyHook);
     root.updateClusterResource(clusterResource, new ResourceLimits(
         clusterResource));
@@ -587,26 +571,19 @@ public void testHeadroom() throws Exception {
         new CapacitySchedulerConfiguration();
     csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25);
     setupQueueConfiguration(csConf);
-    YarnConfiguration conf = new YarnConfiguration();
-    
-    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
-    when(csContext.getConfiguration()).thenReturn(csConf);
-    when(csContext.getConf()).thenReturn(conf);
-    when(csContext.getMinimumResourceCapability()).
-        thenReturn(Resources.createResource(GB));
-    when(csContext.getMaximumResourceCapability()).
-        thenReturn(Resources.createResource(16*GB));
-    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
-    when(csContext.getRMContext()).thenReturn(rmContext);
-    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
-    
+
     // Say cluster has 100 nodes of 16G each
     Resource clusterResource = Resources.createResource(100 * 16 * GB);
-    when(csContext.getClusterResource()).thenReturn(clusterResource);
+
+    CapacitySchedulerContext csContext = createCSContext(csConf, resourceCalculator,
+        Resources.createResource(GB), Resources.createResource(16*GB), clusterResource);
+    CapacitySchedulerQueueManager queueManager = csContext.getCapacitySchedulerQueueManager();
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
 
     CSQueueStore queues = new CSQueueStore();
-    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
+    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext,
         csConf, null, "root", queues, queues, TestUtils.spyHook);
+    queueManager.setRootQueue(rootQueue);
     rootQueue.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
@@ -870,8 +847,8 @@ public RMNodeLabelsManager createNodeLabelManager() {
             + "submission of application: " + app3.getApplicationId(),
         app3.getDiagnostics().toString());
 
-    // based on Global limit of queue usert application is rejected
-    RMApp app11 = MockRMAppSubmitter.submit(rm,
+    // based on per user max app settings, app should be rejected instantly
+    RMApp app13 = MockRMAppSubmitter.submit(rm,
         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
             .withAppName("app")
             .withUser("user")
@@ -880,36 +857,36 @@ public RMNodeLabelsManager createNodeLabelManager() {
             .withWaitForAppAcceptedState(false)
             .build());
     rm.drainEvents();
-    rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED);
-    assertEquals(RMAppState.ACCEPTED, app11.getState());
-    RMApp app12 = MockRMAppSubmitter.submit(rm,
+    rm.waitForState(app13.getApplicationId(), RMAppState.FAILED);
+    assertEquals(RMAppState.FAILED, app13.getState());
+    assertEquals(
+        "org.apache.hadoop.security.AccessControlException: Queue"
+            + " root.d already has 0 applications from user user cannot"
+            + " accept submission of application: " + app13.getApplicationId(),
+        app13.getDiagnostics().toString());
+
+    RMApp app11 = MockRMAppSubmitter.submit(rm,
         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
             .withAppName("app")
-            .withUser("user")
+            .withUser("user2")
             .withAcls(null)
-            .withQueue("d")
+            .withQueue("a2")
             .withWaitForAppAcceptedState(false)
             .build());
     rm.drainEvents();
-    rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED);
-    assertEquals(RMAppState.ACCEPTED, app12.getState());
-    RMApp app13 = MockRMAppSubmitter.submit(rm,
+    rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED);
+    assertEquals(RMAppState.ACCEPTED, app11.getState());
+    RMApp app12 = MockRMAppSubmitter.submit(rm,
         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
             .withAppName("app")
-            .withUser("user")
+            .withUser("user2")
             .withAcls(null)
-            .withQueue("d")
+            .withQueue("a2")
             .withWaitForAppAcceptedState(false)
             .build());
     rm.drainEvents();
-    rm.waitForState(app13.getApplicationId(), RMAppState.FAILED);
-    assertEquals(RMAppState.FAILED, app13.getState());
-    assertEquals(
-        "org.apache.hadoop.security.AccessControlException: Queue"
-            + " root.d already has 2 applications from user user cannot"
-            + " accept submission of application: " + app13.getApplicationId(),
-        app13.getDiagnostics().toString());
-
+    rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED);
+    assertEquals(RMAppState.ACCEPTED, app12.getState());
     // based on system max limit application is rejected
     RMApp app14 = MockRMAppSubmitter.submit(rm,
         MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
@@ -938,7 +915,6 @@ public RMNodeLabelsManager createNodeLabelManager() {
         app15.getDiagnostics().toString());
 
     rm.killApp(app2.getApplicationId());
-    rm.killApp(app11.getApplicationId());
     rm.killApp(app13.getApplicationId());
     rm.killApp(app14.getApplicationId());
     rm.stop();
@@ -953,27 +929,17 @@ public void testAMResourceLimitWithDRCAndFullParent() throws Exception {
     setupQueueConfiguration(csConf);
     csConf.setFloat(CapacitySchedulerConfiguration.
         MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.3f);
-    YarnConfiguration conf = new YarnConfiguration();
-
-    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
-    when(csContext.getConfiguration()).thenReturn(csConf);
-    when(csContext.getConf()).thenReturn(conf);
-    when(csContext.getMinimumResourceCapability()).
-        thenReturn(Resources.createResource(GB));
-    when(csContext.getMaximumResourceCapability()).
-        thenReturn(Resources.createResource(16*GB));
-    when(csContext.getResourceCalculator()).
-        thenReturn(new DominantResourceCalculator());
-    when(csContext.getRMContext()).thenReturn(rmContext);
-    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
 
     // Total cluster resources.
     Resource clusterResource = Resources.createResource(100 * GB, 1000);
-    when(csContext.getClusterResource()).thenReturn(clusterResource);
+
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(
+        createCSContext(csConf, new DominantResourceCalculator(), Resources.createResource(GB),
+            Resources.createResource(16*GB), clusterResource));
 
     // Set up queue hierarchy.
     CSQueueStore queues = new CSQueueStore();
-    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
+    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext,
         csConf, null, "root", queues, queues, TestUtils.spyHook);
     rootQueue.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
@@ -1016,4 +982,32 @@ public void testAMResourceLimitWithDRCAndFullParent() throws Exception {
         + amLimit.getVirtualCores(),
         amLimit.getVirtualCores() >= expectedAmLimit.getVirtualCores());
   }
+
+  private CapacitySchedulerContext createCSContext(CapacitySchedulerConfiguration csConf,
+      ResourceCalculator rc, Resource minResource, Resource maxResource, Resource clusterResource) {
+    YarnConfiguration conf = new YarnConfiguration();
+
+    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
+    when(csContext.getConfiguration()).thenReturn(csConf);
+    when(csContext.getConf()).thenReturn(conf);
+    when(csContext.getMinimumResourceCapability()).
+        thenReturn(minResource);
+    when(csContext.getMaximumResourceCapability()).
+        thenReturn(maxResource);
+    when(csContext.getResourceCalculator()).
+        thenReturn(rc);
+    CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(conf,
+        rmContext.getNodeLabelManager(), null);
+    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
+
+    when(csContext.getRMContext()).thenReturn(rmContext);
+    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+
+    // Total cluster resources.
+    when(csContext.getClusterResource()).thenReturn(clusterResource);
+
+    return csContext;
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index a228d254d5ad8..ef50e5271e718 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -57,7 +57,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -777,6 +776,12 @@ public void testHeadroom() throws Exception {
     when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
     when(csContext.getRMContext()).thenReturn(spyRMContext);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+    CapacitySchedulerQueueManager queueManager =
+        new CapacitySchedulerQueueManager(csConf, mgr, null);
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
+
+    // Setup nodelabels
+    queueManager.reinitConfiguredNodeLabels(csConf);
 
     mgr.activateNode(NodeId.newInstance("h0", 0),
         Resource.newInstance(160 * GB, 16)); // default Label
@@ -789,16 +794,15 @@ public void testHeadroom() throws Exception {
     Resource clusterResource = Resources.createResource(160 * GB);
     when(csContext.getClusterResource()).thenReturn(clusterResource);
 
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(csContext);
+
     CSQueueStore queues = new CSQueueStore();
-    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
+    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(queueContext,
         csConf, null, "root", queues, queues, TestUtils.spyHook);
+    queueManager.setRootQueue(rootQueue);
     rootQueue.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
-    ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
-    when(csContext.getClusterResourceUsage())
-        .thenReturn(queueResUsage);
-
     // Manipulate queue 'a'
     LeafQueue queue = TestLeafQueue.stubLeafQueue((LeafQueue) queues.get("b2"));
     queue.updateClusterResource(clusterResource,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
index 37f1378d7c641..201214ec2674b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
@@ -25,11 +25,12 @@
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE;
 
 public class TestAutoCreatedQueueTemplate {
-  private static final String TEST_QUEUE_ABC = "root.a.b.c";
-  private static final String TEST_QUEUE_AB = "root.a.b";
-  private static final String TEST_QUEUE_A = "root.a";
-  private static final String TEST_QUEUE_B = "root.b";
+  private static final QueuePath TEST_QUEUE_ABC = new QueuePath("root.a.b.c");
+  private static final QueuePath TEST_QUEUE_AB = new QueuePath("root.a.b");
+  private static final QueuePath TEST_QUEUE_A = new QueuePath("root.a");
+  private static final QueuePath TEST_QUEUE_B = new QueuePath("root.b");
   private static final String ROOT = "root";
+
   private CapacitySchedulerConfiguration conf;
 
   @Before
@@ -43,13 +44,13 @@ public void setUp() throws Exception {
 
   @Test
   public void testNonWildCardTemplate() {
-    conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w");
+    conf.set(getTemplateKey(TEST_QUEUE_AB.getFullPath(), "capacity"), "6w");
     AutoCreatedQueueTemplate template =
         new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
 
     Assert.assertEquals("weight is not set", 6f,
-        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
 
   }
 
@@ -58,35 +59,62 @@ public void testOneLevelWildcardTemplate() {
     conf.set(getTemplateKey("root.a.*", "capacity"), "6w");
     AutoCreatedQueueTemplate template =
         new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
 
     Assert.assertEquals("weight is not set", 6f,
-        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+
+  }
+
+  @Test
+  public void testTwoLevelWildcardTemplate() {
+    conf.set(getTemplateKey("root.*", "capacity"), "6w");
+    conf.set(getTemplateKey("root.*.*", "capacity"), "5w");
+
+    new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A)
+            .setTemplateEntriesForChild(conf, TEST_QUEUE_AB.getFullPath());
+    new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB)
+            .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
 
+    Assert.assertEquals("weight is not set", 6f,
+            conf.getNonLabeledQueueWeight(TEST_QUEUE_AB.getFullPath()), 10e-6);
+    Assert.assertEquals("weight is not set", 5f,
+            conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
   }
 
   @Test
   public void testIgnoredWhenRootWildcarded() {
     conf.set(getTemplateKey("*", "capacity"), "6w");
     AutoCreatedQueueTemplate template =
-        new AutoCreatedQueueTemplate(conf, ROOT);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
+        new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
 
     Assert.assertEquals("weight is set", -1f,
-        conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
   }
 
   @Test
   public void testIgnoredWhenNoParent() {
     conf.set(getTemplateKey("root", "capacity"), "6w");
     AutoCreatedQueueTemplate template =
-        new AutoCreatedQueueTemplate(conf, ROOT);
+        new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
     template.setTemplateEntriesForChild(conf, ROOT);
 
     Assert.assertEquals("weight is set", -1f,
         conf.getNonLabeledQueueWeight(ROOT), 10e-6);
   }
 
+  @Test
+  public void testWildcardAfterRoot() {
+    conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user");
+    AutoCreatedQueueTemplate template =
+        new AutoCreatedQueueTemplate(conf, new QueuePath("root.a"));
+    template.setTemplateEntriesForChild(conf, "root.a");
+
+    Assert.assertEquals("acl_submit_applications is set", "user",
+        template.getTemplateProperties().get("acl_submit_applications"));
+  }
+
   @Test
   public void testTemplatePrecedence() {
     conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
@@ -95,21 +123,21 @@ public void testTemplatePrecedence() {
 
     AutoCreatedQueueTemplate template =
         new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
 
     Assert.assertEquals(
         "explicit template does not have the highest precedence", 6f,
-        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
 
     CapacitySchedulerConfiguration newConf =
         new CapacitySchedulerConfiguration();
     newConf.set(getTemplateKey("root.a.*", "capacity"), "4w");
     template =
         new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB);
-    template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC);
+    template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC.getFullPath());
 
     Assert.assertEquals("precedence is invalid", 4f,
-        newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+        newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
   }
 
   @Test
@@ -117,10 +145,10 @@ public void testRootTemplate() {
     conf.set(getTemplateKey("root", "capacity"), "2w");
 
     AutoCreatedQueueTemplate template =
-        new AutoCreatedQueueTemplate(conf, ROOT);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
+        new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
     Assert.assertEquals("root property is not set", 2f,
-        conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
+        conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
   }
 
   @Test
@@ -133,21 +161,21 @@ public void testQueueSpecificTemplates() {
         "root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false);
 
     AutoCreatedQueueTemplate template =
-        new AutoCreatedQueueTemplate(conf, ROOT);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
-    template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true);
+        new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
+    template.setTemplateEntriesForChild(conf, TEST_QUEUE_B.getFullPath(), true);
 
     Assert.assertNull("default-node-label-expression is set for parent",
-        conf.getDefaultNodeLabelExpression(TEST_QUEUE_A));
+        conf.getDefaultNodeLabelExpression(TEST_QUEUE_A.getFullPath()));
     Assert.assertEquals("default-node-label-expression is not set for leaf",
-        "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B));
+        "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B.getFullPath()));
     Assert.assertFalse("auto queue removal is not disabled for parent",
-        conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A));
+        conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A.getFullPath()));
     Assert.assertEquals("weight should not be overridden when set by " +
             "queue type specific template",
-        10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6);
+        10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B.getFullPath()), 10e-6);
     Assert.assertEquals("weight should be set by common template",
-        2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
+        2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
 
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
index 36b3c9b4d63ff..5b233f27a8034 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
@@ -299,9 +299,9 @@ public void testCapacitySchedulerAbsoluteConfWithCustomResourceType()
     // Define top-level queues
     newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
         new String[] {"a", "b", "c"});
-    newConf.setMinimumResourceRequirement("", "root.a",
+    newConf.setMinimumResourceRequirement("", new QueuePath("root", "a"),
         aMINRES);
-    newConf.setMaximumResourceRequirement("", "root.a",
+    newConf.setMaximumResourceRequirement("", new QueuePath("root", "a"),
         aMAXRES);
 
     newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
index 43347c76cc95c..f7460de7aafd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
@@ -92,9 +92,13 @@ public void setup() throws IOException {
     when(preemptionManager.getKillableResource(any(), anyString()))
         .thenReturn(Resource.newInstance(0, 0));
     when(scheduler.getPreemptionManager()).thenReturn(preemptionManager);
+    when(scheduler.getActivitiesManager()).thenReturn(activitiesManager);
     queueManager = new CapacitySchedulerQueueManager(csConfig, labelManager,
         appPriorityACLManager);
     queueManager.setCapacitySchedulerContext(scheduler);
+    when(scheduler.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
+    CapacitySchedulerQueueContext queueContext = new CapacitySchedulerQueueContext(scheduler);
+    when(scheduler.getQueueContext()).thenReturn(queueContext);
     queueManager.initializeQueues(csConfig);
   }
 
@@ -139,7 +143,7 @@ public long getStartTime() {
   }
 
   private void removeApp(FiCaSchedulerApp attempt) {
-    LeafQueue queue = attempt.getCSLeafQueue();
+    AbstractLeafQueue queue = attempt.getCSLeafQueue();
     queue.finishApplicationAttempt(attempt, queue.getQueuePath());
     maxAppsEnforcer.untrackApp(attempt);
     maxAppsEnforcer.updateRunnabilityOnAppRemoval(attempt);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java
index ad0843216d172..8ec8d62e74454 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueStore.java
@@ -39,6 +39,7 @@ public class TestCSQueueStore {
 
   private CSQueue root;
   private CapacitySchedulerContext csContext;
+  private CapacitySchedulerQueueContext queueContext;
 
   @Before
   public void setUp() throws IOException {
@@ -62,22 +63,26 @@ public void setUp() throws IOException {
     when(csContext.getResourceCalculator()).
             thenReturn(resourceCalculator);
     when(csContext.getRMContext()).thenReturn(rmContext);
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(
+        new CapacitySchedulerQueueManager(csConf, null, null));
+
+    queueContext = new CapacitySchedulerQueueContext(csContext);
 
     CSQueueStore queues = new CSQueueStore();
     root = CapacitySchedulerQueueManager
-            .parseQueue(csContext, csConf, null, "root",
+            .parseQueue(queueContext, csConf, null, "root",
                     queues, queues,
                     TestUtils.spyHook);
   }
 
   public CSQueue createLeafQueue(String name, CSQueue parent)
           throws IOException {
-    return new LeafQueue(csContext, name, parent, null);
+    return new LeafQueue(queueContext, name, parent, null);
   }
 
   public CSQueue createParentQueue(String name, CSQueue parent)
           throws IOException {
-    return new ParentQueue(csContext, name, parent, null);
+    return new ParentQueue(queueContext, name, parent, null);
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index efa736d53c3be..7c407dfd4ad4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -19,13 +19,14 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocMb;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocMb;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocVcores;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupAdditionalQueues;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupBlockedQueueConfiguration;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupOtherBlockedQueueConfiguration;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1;
@@ -35,22 +36,22 @@
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkNodeResourceUsage;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkPendingResource;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkPendingResourceGreaterThanZero;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.setUpMove;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.toSet;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.waitforNMRegistered;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertContainerKilled;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertMemory;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerOvercommit.assertNoPreemption;
@@ -61,16 +62,11 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -86,11 +82,11 @@
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.Groups;
@@ -109,7 +105,6 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -119,11 +114,9 @@
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.PreemptionMessage;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
-import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -131,10 +124,6 @@
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -159,7 +148,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
@@ -167,24 +155,15 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
@@ -199,16 +178,11 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.IteratorSelector;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -239,51 +213,13 @@ public class TestCapacityScheduler {
 
   @Before
   public void setUp() throws Exception {
-    ResourceUtils.resetResourceTypes(new Configuration());
-    DefaultMetricsSystem.setMiniClusterMode(true);
-    resourceManager = new ResourceManager() {
-      @Override
-      protected RMNodeLabelsManager createNodeLabelManager() {
-        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
-        mgr.init(getConfig());
-        return mgr;
-      }
-    };
-    CapacitySchedulerConfiguration csConf
-       = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(csConf);
-    YarnConfiguration conf = new YarnConfiguration(csConf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER,
-        CapacityScheduler.class, ResourceScheduler.class);
-    resourceManager.init(conf);
-    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
-    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
-    ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
-    mockContext = mock(RMContext.class);
-    when(mockContext.getConfigurationProvider()).thenReturn(
-        new LocalConfigurationProvider());
+    resourceManager = createResourceManager();
+    mockContext = createMockRMContext();
   }
 
   @After
   public void tearDown() throws Exception {
-    if (resourceManager != null) {
-      QueueMetrics.clearQueueMetrics();
-      DefaultMetricsSystem.shutdown();
-      resourceManager.stop();
-    }
-  }
-
-  private NodeManager registerNode(ResourceManager rm, String hostName,
-      int containerManagerPort, int httpPort, String rackName,
-      Resource capability, NodeStatus nodeStatus)
-      throws IOException, YarnException {
-    NodeManager nm = new NodeManager(hostName,
-        containerManagerPort, httpPort, rackName, capability, rm, nodeStatus);
-    NodeAddedSchedulerEvent nodeAddEvent1 =
-        new NodeAddedSchedulerEvent(rm.getRMContext().getRMNodes()
-            .get(nm.getNodeId()));
-    rm.getResourceScheduler().handle(nodeAddEvent1);
-    return nm;
+    stopResourceManager(resourceManager);
   }
 
   @Test (timeout = 30000)
@@ -291,8 +227,9 @@ public void testConfValidation() throws Exception {
     CapacityScheduler scheduler = new CapacityScheduler();
     scheduler.setRMContext(resourceManager.getRMContext());
     Configuration conf = new YarnConfiguration();
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+
+    setMinAllocMb(conf, 2048);
+    setMaxAllocMb(conf, 1024);
     try {
       scheduler.init(conf);
       fail("Exception is expected because the min memory allocation is" +
@@ -305,8 +242,8 @@ public void testConfValidation() throws Exception {
     }
 
     conf = new YarnConfiguration();
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 2);
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 1);
+    setMinAllocVcores(conf, 2);
+    setMaxAllocVcores(conf, 1);
     try {
       scheduler.reinitialize(conf, mockContext);
       fail("Exception is expected because the min vcores allocation is" +
@@ -319,19 +256,6 @@ public void testConfValidation() throws Exception {
     }
   }
 
-  private NodeManager registerNode(String hostName, int containerManagerPort,
-      int httpPort, String rackName,
-      Resource capability, NodeStatus nodeStatus)
-      throws IOException, YarnException {
-    NodeManager nm = new NodeManager(hostName, containerManagerPort, httpPort,
-        rackName, capability, resourceManager, nodeStatus);
-    NodeAddedSchedulerEvent nodeAddEvent1 =
-        new NodeAddedSchedulerEvent(resourceManager.getRMContext()
-            .getRMNodes().get(nm.getNodeId()));
-    resourceManager.getResourceScheduler().handle(nodeAddEvent1);
-    return nm;
-  }
-
   @Test
   public void testCapacityScheduler() throws Exception {
 
@@ -342,13 +266,13 @@ public void testCapacityScheduler() throws Exception {
     // Register node1
     String host_0 = "host_0";
     NodeManager nm_0 =
-        registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+        registerNode(resourceManager, host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
             Resources.createResource(4 * GB, 1), mockNodeStatus);
 
     // Register node2
     String host_1 = "host_1";
     NodeManager nm_1 =
-        registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+        registerNode(resourceManager, host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
             Resources.createResource(2 * GB, 1), mockNodeStatus);
 
     // ResourceRequest priorities
@@ -397,10 +321,10 @@ public void testCapacityScheduler() throws Exception {
     LOG.info("Kick!");
 
     // task_0_0 and task_1_0 allocated, used=4G
-    nodeUpdate(nm_0);
+    nodeUpdate(resourceManager, nm_0);
 
     // nothing allocated
-    nodeUpdate(nm_1);
+    nodeUpdate(resourceManager, nm_1);
 
     // Get allocations from the scheduler
     application_0.schedule();     // task_0_0 
@@ -429,11 +353,11 @@ public void testCapacityScheduler() throws Exception {
     // Send a heartbeat to kick the tires on the Scheduler
     LOG.info("Sending hb from " + nm_0.getHostName());
     // nothing new, used=4G
-    nodeUpdate(nm_0);
+    nodeUpdate(resourceManager, nm_0);
 
     LOG.info("Sending hb from " + nm_1.getHostName());
     // task_0_1 is prefer as locality, used=2G
-    nodeUpdate(nm_1);
+    nodeUpdate(resourceManager, nm_1);
 
     // Get allocations from the scheduler
     LOG.info("Trying to allocate...");
@@ -443,8 +367,8 @@ public void testCapacityScheduler() throws Exception {
     application_1.schedule();
     checkApplicationResourceUsage(5 * GB, application_1);
 
-    nodeUpdate(nm_0);
-    nodeUpdate(nm_1);
+    nodeUpdate(resourceManager, nm_0);
+    nodeUpdate(resourceManager, nm_1);
 
     checkNodeResourceUsage(4*GB, nm_0);
     checkNodeResourceUsage(2*GB, nm_1);
@@ -658,30 +582,18 @@ protected RMNodeLabelsManager createNodeLabelManager() {
     LOG.info("--- END: testAssignMultiple ---");
   }
 
-  private void nodeUpdate(ResourceManager rm, NodeManager nm) {
-    RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
-    // Send a heartbeat to kick the tires on the Scheduler
-    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
-    rm.getResourceScheduler().handle(nodeUpdate);
-  }
-
-  private void nodeUpdate(NodeManager nm) {
-    RMNode node = resourceManager.getRMContext().getRMNodes().get(nm.getNodeId());
-    // Send a heartbeat to kick the tires on the Scheduler
-    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
-    resourceManager.getResourceScheduler().handle(nodeUpdate);
-  }
-
-
   @Test
   public void testMaximumCapacitySetup() {
     float delta = 0.0000001f;
+    QueuePath queuePathA = new QueuePath(A);
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
+    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,
+            conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
     conf.setMaximumCapacity(A, 50.0f);
-    assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A),delta);
+    assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
     conf.setMaximumCapacity(A, -1);
-    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
+    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,
+            conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
   }
 
   @Test
@@ -692,10 +604,8 @@ public void testQueueMaximumAllocations() {
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
 
     setupQueueConfiguration(conf);
-    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1)
-        + MAXIMUM_ALLOCATION_MB, "1024");
-    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A1)
-        + MAXIMUM_ALLOCATION_VCORES, "1");
+    setMaxAllocMb(conf, A1, 1024);
+    setMaxAllocVcores(conf, A1, 1);
 
     scheduler.init(conf);
     scheduler.start();
@@ -719,64 +629,6 @@ public void testQueueMaximumAllocations() {
     Assert.assertEquals(1, maxAllocationForQueue.getVirtualCores());
   }
 
-
-  @Test
-  public void testRefreshQueues() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null);
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, rmContext);
-    checkQueueStructureCapacities(cs);
-
-    conf.setCapacity(A, 80f);
-    conf.setCapacity(B, 20f);
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs, getDefaultCapacities(80f / 100.0f, 20f / 100.0f));
-    cs.stop();
-  }
-
-  private void checkApplicationResourceUsage(int expected,
-      Application application) {
-    Assert.assertEquals(expected, application.getUsedResources().getMemorySize());
-  }
-
-  private void checkNodeResourceUsage(int expected, NodeManager node) {
-    Assert.assertEquals(expected, node.getUsed().getMemorySize());
-    node.checkResourceUsage();
-  }
-
-  /** Test that parseQueue throws an exception when two leaf queues have the
-   *  same name
- * @throws IOException
-   */
-  @Test(expected=IOException.class)
-  public void testParseQueue() throws IOException {
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    cs.init(conf);
-    cs.start();
-
-    conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
-    conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
-    conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
-
-    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
-      null, new RMContainerTokenSecretManager(conf),
-      new NMTokenSecretManagerInRM(conf),
-      new ClientToAMTokenSecretManagerInRM(), null));
-  }
-
   @Test
   public void testParseQueueWithAbsoluteResource() {
     String childQueue = "testQueue";
@@ -816,85 +668,6 @@ public void testParseQueueWithAbsoluteResource() {
     assertEquals(10, childQueueLabelCapacity.getVirtualCores());
   }
 
-  @Test
-  public void testReconnectedNode() throws Exception {
-    CapacitySchedulerConfiguration csConf =
-        new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(csConf);
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(csConf);
-    cs.start();
-    cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
-      null, null, new RMContainerTokenSecretManager(csConf),
-      new NMTokenSecretManagerInRM(csConf),
-      new ClientToAMTokenSecretManagerInRM(), null));
-
-    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
-    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
-
-    cs.handle(new NodeAddedSchedulerEvent(n1));
-    cs.handle(new NodeAddedSchedulerEvent(n2));
-
-    Assert.assertEquals(6 * GB, cs.getClusterResource().getMemorySize());
-
-    // reconnect n1 with downgraded memory
-    n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
-    cs.handle(new NodeRemovedSchedulerEvent(n1));
-    cs.handle(new NodeAddedSchedulerEvent(n1));
-
-    Assert.assertEquals(4 * GB, cs.getClusterResource().getMemorySize());
-    cs.stop();
-  }
-
-  @Test
-  public void testRefreshQueuesWithNewQueue() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null));
-    checkQueueStructureCapacities(cs);
-
-    // Add a new queue b4
-    final String b4 = B + ".b4";
-    final float b4Capacity = 10;
-    final float modifiedB3Capacity = B3_CAPACITY - b4Capacity;
-
-    try {
-      conf.setCapacity(A, 80f);
-      conf.setCapacity(B, 20f);
-      conf.setQueues(B, new String[]{"b1", "b2", "b3", "b4"});
-      conf.setCapacity(B1, B1_CAPACITY);
-      conf.setCapacity(B2, B2_CAPACITY);
-      conf.setCapacity(B3, modifiedB3Capacity);
-      conf.setCapacity(b4, b4Capacity);
-      cs.reinitialize(conf, mockContext);
-
-      final float capA = 80f / 100.0f;
-      final float capB = 20f / 100.0f;
-      Map<String, ExpectedCapacities> expectedCapacities = getDefaultCapacities(capA, capB);
-      expectedCapacities.put(B3, new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB));
-      expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB));
-      checkQueueStructureCapacities(cs, expectedCapacities);
-
-      // Verify parent for B4
-      CSQueue rootQueue = cs.getRootQueue();
-      CSQueue queueB = findQueue(rootQueue, B);
-      CSQueue queueB4 = findQueue(queueB, b4);
-
-      assertEquals(queueB, queueB4.getParent());
-    } finally {
-      cs.stop();
-    }
-  }
   @Test
   public void testCapacitySchedulerInfo() throws Exception {
     QueueInfo queueInfo = resourceManager.getResourceScheduler().getQueueInfo("a", true, true);
@@ -924,36 +697,6 @@ private int getQueueCount(List<QueueUserACLInfo> queueInformation, String queueN
     return result;
   }
 
-  @Test
-  public void testBlackListNodes() throws Exception {
-    Configuration conf = new Configuration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    MockRM rm = new MockRM(conf);
-    rm.start();
-    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
-
-    String host = "127.0.0.1";
-    RMNode node =
-        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
-    cs.handle(new NodeAddedSchedulerEvent(node));
-
-    ApplicationAttemptId appAttemptId = appHelper(rm, cs, 100, 1, "default", "user");
-
-    // Verify the blacklist can be updated independent of requesting containers
-    cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), null,
-        Collections.<ContainerId>emptyList(),
-        Collections.singletonList(host), null, NULL_UPDATE_REQUESTS);
-    Assert.assertTrue(cs.getApplicationAttempt(appAttemptId)
-        .isPlaceBlacklisted(host));
-    cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), null,
-        Collections.<ContainerId>emptyList(), null,
-        Collections.singletonList(host), NULL_UPDATE_REQUESTS);
-    Assert.assertFalse(cs.getApplicationAttempt(appAttemptId)
-        .isPlaceBlacklisted(host));
-    rm.stop();
-  }
-
   @Test
   public void testAllocateReorder() throws Exception {
 
@@ -1159,53 +902,6 @@ public void testResourceOverCommit() throws Exception {
     rm.stop();
   }
 
-  @Test
-  public void testGetAppsInQueue() throws Exception {
-    Application application_0 = new Application("user_0", "a1", resourceManager);
-    application_0.submit();
-
-    Application application_1 = new Application("user_0", "a2", resourceManager);
-    application_1.submit();
-
-    Application application_2 = new Application("user_0", "b2", resourceManager);
-    application_2.submit();
-
-    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
-
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(application_0.getApplicationAttemptId()));
-    assertTrue(appsInA.contains(application_1.getApplicationAttemptId()));
-    assertEquals(2, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(application_0.getApplicationAttemptId()));
-    assertTrue(appsInRoot.contains(application_1.getApplicationAttemptId()));
-    assertTrue(appsInRoot.contains(application_2.getApplicationAttemptId()));
-    assertEquals(3, appsInRoot.size());
-
-    Assert.assertNull(scheduler.getAppsInQueue("nonexistentqueue"));
-  }
-
-  @Test
-  public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-      ResourceScheduler.class);
-    MockRM rm = new MockRM(conf);
-    @SuppressWarnings("unchecked")
-    AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
-        (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
-          .getResourceScheduler();
-    SchedulerApplication<SchedulerApplicationAttempt> app =
-        TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
-          cs.getSchedulerApplications(), cs, "a1");
-    Assert.assertEquals("a1", app.getQueue().getQueueName());
-  }
-
   @Test
   public void testAsyncScheduling() throws Exception {
     Configuration conf = new Configuration();
@@ -1378,37 +1074,6 @@ public void run() {
     rm.stop();
   }
 
-  @Test
-  public void testNumClusterNodes() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(conf);
-    RMContext rmContext = TestUtils.getMockRMContext();
-    cs.setRMContext(rmContext);
-    CapacitySchedulerConfiguration csConf =
-        new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(csConf);
-    cs.init(csConf);
-    cs.start();
-    assertEquals(0, cs.getNumClusterNodes());
-
-    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
-    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
-    cs.handle(new NodeAddedSchedulerEvent(n1));
-    cs.handle(new NodeAddedSchedulerEvent(n2));
-    assertEquals(2, cs.getNumClusterNodes());
-
-    cs.handle(new NodeRemovedSchedulerEvent(n1));
-    assertEquals(1, cs.getNumClusterNodes());
-    cs.handle(new NodeAddedSchedulerEvent(n1));
-    assertEquals(2, cs.getNumClusterNodes());
-    cs.handle(new NodeRemovedSchedulerEvent(n2));
-    cs.handle(new NodeRemovedSchedulerEvent(n1));
-    assertEquals(0, cs.getNumClusterNodes());
-
-    cs.stop();
-  }
-
   @Test(timeout = 120000)
   public void testPreemptionInfo() throws Exception {
     Configuration conf = new Configuration();
@@ -1550,1832 +1215,98 @@ public void testRecoverRequestAfterPreemption() throws Exception {
     Assert.assertTrue(containers.size() == 1);
   }
 
-  private MockRM setUpMove() {
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    return setUpMove(conf);
-  }
-
-  private MockRM setUpMove(Configuration config) {
-    CapacitySchedulerConfiguration conf =
-        new CapacitySchedulerConfiguration(config);
-    setupQueueConfiguration(conf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    MockRM rm = new MockRM(conf);
-    rm.start();
-    return rm;
-  }
-
   @Test
-  public void testAppSubmission() throws Exception {
+  public void testPreemptionDisabled() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+    RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
     setupQueueConfiguration(conf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    conf.setQueues(A, new String[] {"a1", "a2", "b"});
-    conf.setCapacity(A1, 20);
-    conf.setCapacity("root.a.b", 10);
-    MockRM rm = new MockRM(conf);
-    rm.start();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
 
-    RMApp noParentQueueApp = submitAppAndWaitForState(rm, "q", RMAppState.FAILED);
-    Assert.assertEquals(RMAppState.FAILED, noParentQueueApp.getState());
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueB2 = findQueue(queueB, B2);
+
+    // When preemption turned on for the whole system
+    // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other
+    // preemption properties set, queue root.b.b2 should be preemptable.
+    assertFalse("queue " + B2 + " should default to preemptable",
+               queueB2.getPreemptionDisabled());
 
-    RMApp ambiguousQueueApp = submitAppAndWaitForState(rm, "b", RMAppState.FAILED);
-    Assert.assertEquals(RMAppState.FAILED, ambiguousQueueApp.getState());
+    // Disable preemption at the root queue level.
+    // The preemption property should be inherited from root all the
+    // way down so that root.b.b2 should NOT be preemptable.
+    conf.setPreemptionDisabled(rootQueue.getQueuePath(), true);
+    cs.reinitialize(conf, rmContext);
+    assertTrue(
+        "queue " + B2 + " should have inherited non-preemptability from root",
+        queueB2.getPreemptionDisabled());
 
-    RMApp emptyPartQueueApp = submitAppAndWaitForState(rm, "root..a1", RMAppState.FAILED);
-    Assert.assertEquals(RMAppState.FAILED, emptyPartQueueApp.getState());
+    // Enable preemption for root (grandparent) but disable for root.b (parent).
+    // root.b.b2 should inherit property from parent and NOT be preemptable
+    conf.setPreemptionDisabled(rootQueue.getQueuePath(), false);
+    conf.setPreemptionDisabled(queueB.getQueuePath(), true);
+    cs.reinitialize(conf, rmContext);
+    assertTrue(
+        "queue " + B2 + " should have inherited non-preemptability from parent",
+        queueB2.getPreemptionDisabled());
 
-    RMApp failedAutoQueue = submitAppAndWaitForState(rm, "root.a.b.c.d", RMAppState.FAILED);
-    Assert.assertEquals(RMAppState.FAILED, failedAutoQueue.getState());
+    // When preemption is turned on for root.b.b2, it should be preemptable
+    // even though preemption is disabled on root.b (parent).
+    conf.setPreemptionDisabled(queueB2.getQueuePath(), false);
+    cs.reinitialize(conf, rmContext);
+    assertFalse("queue " + B2 + " should have been preemptable",
+        queueB2.getPreemptionDisabled());
   }
 
-  private RMApp submitAppAndWaitForState(MockRM rm, String b, RMAppState state) throws Exception {
-    MockRMAppSubmissionData ambiguousQueueAppData =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withWaitForAppAcceptedState(false)
-            .withAppName("app")
-            .withUser("user")
-            .withAcls(null)
-            .withQueue(b)
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app1 = MockRMAppSubmitter.submit(rm, ambiguousQueueAppData);
-    rm.waitForState(app1.getApplicationId(), state);
-    return app1;
+  private void waitContainerAllocated(MockAM am, int mem, int nContainer,
+      int startContainerId, MockRM rm, MockNM nm) throws Exception {
+    for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) {
+      am.allocate("*", mem, 1, new ArrayList<ContainerId>());
+      ContainerId containerId =
+          ContainerId.newContainerId(am.getApplicationAttemptId(), cId);
+      Assert.assertTrue(rm.waitForState(nm, containerId,
+          RMContainerState.ALLOCATED));
+    }
   }
 
   @Test
-  public void testMoveAppBasic() throws Exception {
-    MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
-        (AbstractYarnScheduler) rm.getResourceScheduler();
-    QueueMetrics metrics = scheduler.getRootQueueMetrics();
-    Assert.assertEquals(0, metrics.getAppsPending());
-    // submit an app
+  public void testSchedulerKeyGarbageCollection() throws Exception {
+    YarnConfiguration conf =
+        new YarnConfiguration(new CapacitySchedulerConfiguration());
+    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
+
+    MockRM rm = new MockRM(conf);
+    rm.start();
+
+    HashMap<NodeId, MockNM> nodes = new HashMap<>();
+    MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+    nodes.put(nm1.getNodeId(), nm1);
+    MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+    nodes.put(nm2.getNodeId(), nm2);
+    MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+    nodes.put(nm3.getNodeId(), nm3);
+    MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+    nodes.put(nm4.getNodeId(), nm4);
+    nm1.registerNode();
+    nm2.registerNode();
+    nm3.registerNode();
+    nm4.registerNode();
+
     MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
+        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
+            .withAppName("app")
+            .withUser("user")
             .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-    String queue =
-        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a1", queue);
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    assertEquals(1, metrics.getAppsPending());
-
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    // now move the app
-    scheduler.moveApplication(app.getApplicationId(), "b1");
-
-    // check postconditions
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(1, appsInB1.size());
-    queue =
-        scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("b1", queue);
-
-    appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.contains(appAttemptId));
-    assertEquals(1, appsInB.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    assertEquals(1, metrics.getAppsPending());
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertTrue(appsInA1.isEmpty());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.isEmpty());
-
-    rm.stop();
-  }
-
-  @Test
-  public void testMoveAppPendingMetrics() throws Exception {
-    MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
-        (AbstractYarnScheduler) rm.getResourceScheduler();
-    QueueMetrics metrics = scheduler.getRootQueueMetrics();
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-
-    assertEquals(0, appsInA1.size());
-    assertEquals(0, appsInB1.size());
-    Assert.assertEquals(0, metrics.getAppsPending());
-
-    // submit two apps in a1
-    RMApp app1 = MockRMAppSubmitter.submit(rm,
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .build());
-    RMApp app2 = MockRMAppSubmitter.submit(rm,
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-2")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .build());
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(2, appsInA1.size());
-    assertEquals(0, appsInB1.size());
-    assertEquals(2, metrics.getAppsPending());
-
-    // submit one app in b1
-    RMApp app3 = MockRMAppSubmitter.submit(rm,
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-2")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("b1")
-            .build());
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(2, appsInA1.size());
-    assertEquals(1, appsInB1.size());
-    assertEquals(3, metrics.getAppsPending());
-
-    // now move the app1 from a1 to b1
-    scheduler.moveApplication(app1.getApplicationId(), "b1");
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(1, appsInA1.size());
-    assertEquals(2, appsInB1.size());
-    assertEquals(3, metrics.getAppsPending());
-
-    // now move the app2 from a1 to b1
-    scheduler.moveApplication(app2.getApplicationId(), "b1");
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(0, appsInA1.size());
-    assertEquals(3, appsInB1.size());
-    assertEquals(3, metrics.getAppsPending());
-
-    // now move the app3 from b1 to a1
-    scheduler.moveApplication(app3.getApplicationId(), "a1");
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(1, appsInA1.size());
-    assertEquals(2, appsInB1.size());
-    assertEquals(3, metrics.getAppsPending());
-    rm.stop();
-  }
-
-  @Test
-  public void testMoveAppSameParent() throws Exception {
-    MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
-        (AbstractYarnScheduler) rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-    String queue =
-        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a1", queue);
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    List<ApplicationAttemptId> appsInA2 = scheduler.getAppsInQueue("a2");
-    assertTrue(appsInA2.isEmpty());
-
-    // now move the app
-    scheduler.moveApplication(app.getApplicationId(), "a2");
-
-    // check postconditions
-    appsInA2 = scheduler.getAppsInQueue("a2");
-    assertEquals(1, appsInA2.size());
-    queue =
-        scheduler.getApplicationAttempt(appsInA2.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a2", queue);
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertTrue(appsInA1.isEmpty());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    rm.stop();
-  }
-
-  @Test
-  public void testMoveAppForMoveToQueueWithFreeCap() throws Exception {
-
-    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
-
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register node1
-    String host_0 = "host_0";
-    NodeManager nm_0 =
-        registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(4 * GB, 1), mockNodeStatus);
-
-    // Register node2
-    String host_1 = "host_1";
-    NodeManager nm_1 =
-        registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(2 * GB, 1), mockNodeStatus);
-
-    // ResourceRequest priorities
-    Priority priority_0 = Priority.newInstance(0);
-    Priority priority_1 = Priority.newInstance(1);
-
-    // Submit application_0
-    Application application_0 =
-        new Application("user_0", "a1", resourceManager);
-    application_0.submit(); // app + app attempt event sent to scheduler
-
-    application_0.addNodeManager(host_0, 1234, nm_0);
-    application_0.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
-    application_0.addResourceRequestSpec(priority_1, capability_0_0);
-
-    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
-    application_0.addResourceRequestSpec(priority_0, capability_0_1);
-
-    Task task_0_0 =
-        new Task(application_0, priority_1, new String[] { host_0, host_1 });
-    application_0.addTask(task_0_0);
-
-    // Submit application_1
-    Application application_1 =
-        new Application("user_1", "b2", resourceManager);
-    application_1.submit(); // app + app attempt event sent to scheduler
-
-    application_1.addNodeManager(host_0, 1234, nm_0);
-    application_1.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_1_0 = Resources.createResource(1 * GB, 1);
-    application_1.addResourceRequestSpec(priority_1, capability_1_0);
-
-    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
-    application_1.addResourceRequestSpec(priority_0, capability_1_1);
-
-    Task task_1_0 =
-        new Task(application_1, priority_1, new String[] { host_0, host_1 });
-    application_1.addTask(task_1_0);
-
-    // Send resource requests to the scheduler
-    application_0.schedule(); // allocate
-    application_1.schedule(); // allocate
-
-    // task_0_0 task_1_0 allocated, used=2G
-    nodeUpdate(nm_0);
-
-    // nothing allocated
-    nodeUpdate(nm_1);
-
-    // Get allocations from the scheduler
-    application_0.schedule(); // task_0_0
-    checkApplicationResourceUsage(1 * GB, application_0);
-
-    application_1.schedule(); // task_1_0
-    checkApplicationResourceUsage(1 * GB, application_1);
-
-    checkNodeResourceUsage(2 * GB, nm_0); // task_0_0 (1G) and task_1_0 (1G) 2G
-                                          // available
-    checkNodeResourceUsage(0 * GB, nm_1); // no tasks, 2G available
-
-    // move app from a1(30% cap of total 10.5% cap) to b1(79,2% cap of 89,5%
-    // total cap)
-    scheduler.moveApplication(application_0.getApplicationId(), "b1");
-
-    // 2GB 1C
-    Task task_1_1 =
-        new Task(application_1, priority_0,
-            new String[] { ResourceRequest.ANY });
-    application_1.addTask(task_1_1);
-
-    application_1.schedule();
-
-    // 2GB 1C
-    Task task_0_1 =
-        new Task(application_0, priority_0, new String[] { host_0, host_1 });
-    application_0.addTask(task_0_1);
-
-    application_0.schedule();
-
-    // prev 2G used free 2G
-    nodeUpdate(nm_0);
-
-    // prev 0G used free 2G
-    nodeUpdate(nm_1);
-
-    // Get allocations from the scheduler
-    application_1.schedule();
-    checkApplicationResourceUsage(3 * GB, application_1);
-
-    // Get allocations from the scheduler
-    application_0.schedule();
-    checkApplicationResourceUsage(3 * GB, application_0);
-
-    checkNodeResourceUsage(4 * GB, nm_0);
-    checkNodeResourceUsage(2 * GB, nm_1);
-
-  }
-
-  @Test
-  public void testMoveAppSuccess() throws Exception {
-
-    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
-
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register node1
-    String host_0 = "host_0";
-    NodeManager nm_0 =
-        registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(5 * GB, 1), mockNodeStatus);
-
-    // Register node2
-    String host_1 = "host_1";
-    NodeManager nm_1 =
-        registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(5 * GB, 1), mockNodeStatus);
-
-    // ResourceRequest priorities
-    Priority priority_0 = Priority.newInstance(0);
-    Priority priority_1 = Priority.newInstance(1);
-
-    // Submit application_0
-    Application application_0 =
-        new Application("user_0", "a1", resourceManager);
-    application_0.submit(); // app + app attempt event sent to scheduler
-
-    application_0.addNodeManager(host_0, 1234, nm_0);
-    application_0.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
-    application_0.addResourceRequestSpec(priority_1, capability_0_0);
-
-    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
-    application_0.addResourceRequestSpec(priority_0, capability_0_1);
-
-    Task task_0_0 =
-        new Task(application_0, priority_1, new String[] { host_0, host_1 });
-    application_0.addTask(task_0_0);
-
-    // Submit application_1
-    Application application_1 =
-        new Application("user_1", "b2", resourceManager);
-    application_1.submit(); // app + app attempt event sent to scheduler
-
-    application_1.addNodeManager(host_0, 1234, nm_0);
-    application_1.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_1_0 = Resources.createResource(1 * GB, 1);
-    application_1.addResourceRequestSpec(priority_1, capability_1_0);
-
-    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
-    application_1.addResourceRequestSpec(priority_0, capability_1_1);
-
-    Task task_1_0 =
-        new Task(application_1, priority_1, new String[] { host_0, host_1 });
-    application_1.addTask(task_1_0);
-
-    // Send resource requests to the scheduler
-    application_0.schedule(); // allocate
-    application_1.schedule(); // allocate
-
-    // b2 can only run 1 app at a time
-    scheduler.moveApplication(application_0.getApplicationId(), "b2");
-
-    nodeUpdate(nm_0);
-
-    nodeUpdate(nm_1);
-
-    // Get allocations from the scheduler
-    application_0.schedule(); // task_0_0
-    checkApplicationResourceUsage(0 * GB, application_0);
-
-    application_1.schedule(); // task_1_0
-    checkApplicationResourceUsage(1 * GB, application_1);
-
-    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
-    // not scheduled
-    checkNodeResourceUsage(1 * GB, nm_0);
-    checkNodeResourceUsage(0 * GB, nm_1);
-
-    // lets move application_0 to a queue where it can run
-    scheduler.moveApplication(application_0.getApplicationId(), "a2");
-    application_0.schedule();
-
-    nodeUpdate(nm_1);
-
-    // Get allocations from the scheduler
-    application_0.schedule(); // task_0_0
-    checkApplicationResourceUsage(3 * GB, application_0);
-
-    checkNodeResourceUsage(1 * GB, nm_0);
-    checkNodeResourceUsage(3 * GB, nm_1);
-
-  }
-
-  @Test(expected = YarnException.class)
-  public void testMoveAppViolateQueueState() throws Exception {
-    resourceManager = new ResourceManager() {
-       @Override
-        protected RMNodeLabelsManager createNodeLabelManager() {
-          RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
-          mgr.init(getConfig());
-          return mgr;
-        }
-    };
-    CapacitySchedulerConfiguration csConf =
-        new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(csConf);
-    StringBuilder qState = new StringBuilder();
-    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
-        .append(CapacitySchedulerConfiguration.DOT)
-        .append(CapacitySchedulerConfiguration.STATE);
-    csConf.set(qState.toString(), QueueState.STOPPED.name());
-    YarnConfiguration conf = new YarnConfiguration(csConf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    resourceManager.init(conf);
-    resourceManager.getRMContext().getContainerTokenSecretManager()
-        .rollMasterKey();
-    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
-    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
-    mockContext = mock(RMContext.class);
-    when(mockContext.getConfigurationProvider()).thenReturn(
-        new LocalConfigurationProvider());
-
-    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
-
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register node1
-    String host_0 = "host_0";
-    NodeManager nm_0 =
-        registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(6 * GB, 1), mockNodeStatus);
-
-    // ResourceRequest priorities
-    Priority priority_0 = Priority.newInstance(0);
-    Priority priority_1 = Priority.newInstance(1);
-
-    // Submit application_0
-    Application application_0 =
-        new Application("user_0", "a1", resourceManager);
-    application_0.submit(); // app + app attempt event sent to scheduler
-
-    application_0.addNodeManager(host_0, 1234, nm_0);
-
-    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
-    application_0.addResourceRequestSpec(priority_1, capability_0_0);
-
-    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
-    application_0.addResourceRequestSpec(priority_0, capability_0_1);
-
-    Task task_0_0 =
-        new Task(application_0, priority_1, new String[] { host_0 });
-    application_0.addTask(task_0_0);
-
-    // Send resource requests to the scheduler
-    application_0.schedule(); // allocate
-
-    // task_0_0 allocated
-    nodeUpdate(nm_0);
-
-    // Get allocations from the scheduler
-    application_0.schedule(); // task_0_0
-    checkApplicationResourceUsage(3 * GB, application_0);
-
-    checkNodeResourceUsage(3 * GB, nm_0);
-    // b2 queue contains 3GB consumption app,
-    // add another 3GB will hit max capacity limit on queue b
-    scheduler.moveApplication(application_0.getApplicationId(), "b1");
-
-  }
-
-  @Test
-  public void testMoveAppQueueMetricsCheck() throws Exception {
-    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
-
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register node1
-    String host_0 = "host_0";
-    NodeManager nm_0 =
-        registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(5 * GB, 1), mockNodeStatus);
-
-    // Register node2
-    String host_1 = "host_1";
-    NodeManager nm_1 =
-        registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
-        Resources.createResource(5 * GB, 1), mockNodeStatus);
-
-    // ResourceRequest priorities
-    Priority priority_0 = Priority.newInstance(0);
-    Priority priority_1 = Priority.newInstance(1);
-
-    // Submit application_0
-    Application application_0 =
-        new Application("user_0", "a1", resourceManager);
-    application_0.submit(); // app + app attempt event sent to scheduler
-
-    application_0.addNodeManager(host_0, 1234, nm_0);
-    application_0.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
-    application_0.addResourceRequestSpec(priority_1, capability_0_0);
-
-    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
-    application_0.addResourceRequestSpec(priority_0, capability_0_1);
-
-    Task task_0_0 =
-        new Task(application_0, priority_1, new String[] { host_0, host_1 });
-    application_0.addTask(task_0_0);
-
-    // Submit application_1
-    Application application_1 =
-        new Application("user_1", "b2", resourceManager);
-    application_1.submit(); // app + app attempt event sent to scheduler
-
-    application_1.addNodeManager(host_0, 1234, nm_0);
-    application_1.addNodeManager(host_1, 1234, nm_1);
-
-    Resource capability_1_0 = Resources.createResource(1 * GB, 1);
-    application_1.addResourceRequestSpec(priority_1, capability_1_0);
-
-    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
-    application_1.addResourceRequestSpec(priority_0, capability_1_1);
-
-    Task task_1_0 =
-        new Task(application_1, priority_1, new String[] { host_0, host_1 });
-    application_1.addTask(task_1_0);
-
-    // Send resource requests to the scheduler
-    application_0.schedule(); // allocate
-    application_1.schedule(); // allocate
-
-    nodeUpdate(nm_0);
-
-    nodeUpdate(nm_1);
-
-    CapacityScheduler cs =
-        (CapacityScheduler) resourceManager.getResourceScheduler();
-    CSQueue origRootQ = cs.getRootQueue();
-    CapacitySchedulerInfo oldInfo =
-        new CapacitySchedulerInfo(origRootQ, cs);
-    int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues());
-    int origNumAppsRoot = origRootQ.getNumApplications();
-
-    scheduler.moveApplication(application_0.getApplicationId(), "a2");
-
-    CSQueue newRootQ = cs.getRootQueue();
-    int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues());
-    int newNumAppsRoot = newRootQ.getNumApplications();
-    CapacitySchedulerInfo newInfo =
-        new CapacitySchedulerInfo(newRootQ, cs);
-    CapacitySchedulerLeafQueueInfo origOldA1 =
-        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues());
-    CapacitySchedulerLeafQueueInfo origNewA1 =
-        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", newInfo.getQueues());
-    CapacitySchedulerLeafQueueInfo targetOldA2 =
-        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", oldInfo.getQueues());
-    CapacitySchedulerLeafQueueInfo targetNewA2 =
-        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", newInfo.getQueues());
-    // originally submitted here
-    assertEquals(1, origOldA1.getNumApplications());
-    assertEquals(1, origNumAppsA);
-    assertEquals(2, origNumAppsRoot);
-    // after the move
-    assertEquals(0, origNewA1.getNumApplications());
-    assertEquals(1, newNumAppsA);
-    assertEquals(2, newNumAppsRoot);
-    // original consumption on a1
-    assertEquals(3 * GB, origOldA1.getResourcesUsed().getMemorySize());
-    assertEquals(1, origOldA1.getResourcesUsed().getvCores());
-    assertEquals(0, origNewA1.getResourcesUsed().getMemorySize()); // after the move
-    assertEquals(0, origNewA1.getResourcesUsed().getvCores()); // after the move
-    // app moved here with live containers
-    assertEquals(3 * GB, targetNewA2.getResourcesUsed().getMemorySize());
-    assertEquals(1, targetNewA2.getResourcesUsed().getvCores());
-    // it was empty before the move
-    assertEquals(0, targetOldA2.getNumApplications());
-    assertEquals(0, targetOldA2.getResourcesUsed().getMemorySize());
-    assertEquals(0, targetOldA2.getResourcesUsed().getvCores());
-    // after the app moved here
-    assertEquals(1, targetNewA2.getNumApplications());
-    // 1 container on original queue before move
-    assertEquals(1, origOldA1.getNumContainers());
-    // after the move the resource released
-    assertEquals(0, origNewA1.getNumContainers());
-    // and moved to the new queue
-    assertEquals(1, targetNewA2.getNumContainers());
-    // which originally didn't have any
-    assertEquals(0, targetOldA2.getNumContainers());
-    // 1 user with 3GB
-    assertEquals(3 * GB, origOldA1.getUsers().getUsersList().get(0)
-        .getResourcesUsed().getMemorySize());
-    // 1 user with 1 core
-    assertEquals(1, origOldA1.getUsers().getUsersList().get(0)
-        .getResourcesUsed().getvCores());
-    // user ha no more running app in the orig queue
-    assertEquals(0, origNewA1.getUsers().getUsersList().size());
-    // 1 user with 3GB
-    assertEquals(3 * GB, targetNewA2.getUsers().getUsersList().get(0)
-        .getResourcesUsed().getMemorySize());
-    // 1 user with 1 core
-    assertEquals(1, targetNewA2.getUsers().getUsersList().get(0)
-        .getResourcesUsed().getvCores());
-
-    // Get allocations from the scheduler
-    application_0.schedule(); // task_0_0
-    checkApplicationResourceUsage(3 * GB, application_0);
-
-    application_1.schedule(); // task_1_0
-    checkApplicationResourceUsage(1 * GB, application_1);
-
-    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
-    // not scheduled
-    checkNodeResourceUsage(4 * GB, nm_0);
-    checkNodeResourceUsage(0 * GB, nm_1);
-
-  }
-
-  private int getNumAppsInQueue(String name, List<CSQueue> queues) {
-    for (CSQueue queue : queues) {
-      if (queue.getQueueShortName().equals(name)) {
-        return queue.getNumApplications();
-      }
-    }
-    return -1;
-  }
-
-  private CapacitySchedulerQueueInfo getQueueInfo(String name,
-      CapacitySchedulerQueueInfoList info) {
-    if (info != null) {
-      for (CapacitySchedulerQueueInfo queueInfo : info.getQueueInfoList()) {
-        if (queueInfo.getQueueName().equals(name)) {
-          return queueInfo;
-        } else {
-          CapacitySchedulerQueueInfo result =
-              getQueueInfo(name, queueInfo.getQueues());
-          if (result == null) {
-            continue;
-          }
-          return result;
-        }
-      }
-    }
-    return null;
-  }
-
-  @Test
-  public void testMoveAllApps() throws Exception {
-    MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
-        (AbstractYarnScheduler) rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-    String queue =
-        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a1", queue);
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    // now move the app
-    scheduler.moveAllApps("a1", "b1");
-
-    // check postconditions
-    Thread.sleep(1000);
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(1, appsInB1.size());
-    queue =
-        scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("b1", queue);
-
-    appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.contains(appAttemptId));
-    assertEquals(1, appsInB.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertTrue(appsInA1.isEmpty());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.isEmpty());
-
-    rm.stop();
-  }
-
-  @Test
-  public void testMoveAllAppsInvalidDestination() throws Exception {
-    MockRM rm = setUpMove();
-    YarnScheduler scheduler = rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    // now move the app
-    try {
-      scheduler.moveAllApps("a1", "DOES_NOT_EXIST");
-      Assert.fail();
-    } catch (YarnException e) {
-      // expected
-    }
-
-    // check postconditions, app should still be in a1
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    rm.stop();
-  }
-
-  @Test
-  public void testMoveAllAppsInvalidSource() throws Exception {
-    MockRM rm = setUpMove();
-    YarnScheduler scheduler = rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    // now move the app
-    try {
-      scheduler.moveAllApps("DOES_NOT_EXIST", "b1");
-      Assert.fail();
-    } catch (YarnException e) {
-      // expected
-    }
-
-    // check postconditions, app should still be in a1
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    rm.stop();
-  }
-
-  @Test(timeout = 60000)
-  public void testMoveAttemptNotAdded() throws Exception {
-    Configuration conf = new Configuration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    MockRM rm = new MockRM(getCapacityConfiguration(conf));
-    rm.start();
-    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
-
-    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
-    ApplicationAttemptId appAttemptId =
-        BuilderUtils.newApplicationAttemptId(appId, 1);
-
-    RMAppAttemptMetrics attemptMetric =
-        new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
-    RMAppImpl app = mock(RMAppImpl.class);
-    when(app.getApplicationId()).thenReturn(appId);
-    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
-    Container container = mock(Container.class);
-    when(attempt.getMasterContainer()).thenReturn(container);
-    ApplicationSubmissionContext submissionContext =
-        mock(ApplicationSubmissionContext.class);
-    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
-    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
-    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
-    when(app.getCurrentAppAttempt()).thenReturn(attempt);
-
-    rm.getRMContext().getRMApps().put(appId, app);
-
-    SchedulerEvent addAppEvent =
-        new AppAddedSchedulerEvent(appId, "a1", "user");
-    try {
-      cs.moveApplication(appId, "b1");
-      fail("Move should throw exception app not available");
-    } catch (YarnException e) {
-      assertEquals("App to be moved application_100_0001 not found.",
-          e.getMessage());
-    }
-    cs.handle(addAppEvent);
-    cs.moveApplication(appId, "b1");
-    SchedulerEvent addAttemptEvent =
-        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
-    cs.handle(addAttemptEvent);
-    CSQueue rootQ = cs.getRootQueue();
-    CSQueue queueB = cs.getQueue("b");
-    CSQueue queueA = cs.getQueue("a");
-    CSQueue queueA1 = cs.getQueue("a1");
-    CSQueue queueB1 = cs.getQueue("b1");
-    Assert.assertEquals(1, rootQ.getNumApplications());
-    Assert.assertEquals(0, queueA.getNumApplications());
-    Assert.assertEquals(1, queueB.getNumApplications());
-    Assert.assertEquals(0, queueA1.getNumApplications());
-    Assert.assertEquals(1, queueB1.getNumApplications());
-
-    rm.close();
-  }
-
-  @Test
-  public void testRemoveAttemptMoveAdded() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        CapacityScheduler.class);
-    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
-    // Create Mock RM
-    MockRM rm = new MockRM(getCapacityConfiguration(conf));
-    CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler();
-    // add node
-    Resource newResource = Resource.newInstance(4 * GB, 1);
-    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
-    SchedulerEvent addNode = new NodeAddedSchedulerEvent(node);
-    sch.handle(addNode);
-
-    ApplicationAttemptId appAttemptId = appHelper(rm, sch, 100, 1, "a1", "user");
-
-    // get Queues
-    CSQueue queueA1 = sch.getQueue("a1");
-    CSQueue queueB = sch.getQueue("b");
-    CSQueue queueB1 = sch.getQueue("b1");
-
-    // add Running rm container and simulate live containers to a1
-    ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2);
-    RMContainerImpl rmContainer = mock(RMContainerImpl.class);
-    when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING);
-    Container container2 = mock(Container.class);
-    when(rmContainer.getContainer()).thenReturn(container2);
-    Resource resource = Resource.newInstance(1024, 1);
-    when(container2.getResource()).thenReturn(resource);
-    when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
-    when(container2.getNodeId()).thenReturn(node.getNodeID());
-    when(container2.getId()).thenReturn(newContainerId);
-    when(rmContainer.getNodeLabelExpression())
-        .thenReturn(RMNodeLabelsManager.NO_LABEL);
-    when(rmContainer.getContainerId()).thenReturn(newContainerId);
-    sch.getApplicationAttempt(appAttemptId).getLiveContainersMap()
-        .put(newContainerId, rmContainer);
-    QueueMetrics queueA1M = queueA1.getMetrics();
-    queueA1M.incrPendingResources(rmContainer.getNodeLabelExpression(),
-        "user1", 1, resource);
-    queueA1M.allocateResources(rmContainer.getNodeLabelExpression(),
-        "user1", resource);
-    // remove attempt
-    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId,
-        RMAppAttemptState.KILLED, true));
-    // Move application to queue b1
-    sch.moveApplication(appAttemptId.getApplicationId(), "b1");
-    // Check queue metrics after move
-    Assert.assertEquals(0, queueA1.getNumApplications());
-    Assert.assertEquals(1, queueB.getNumApplications());
-    Assert.assertEquals(0, queueB1.getNumApplications());
-
-    // Release attempt add event
-    ApplicationAttemptId appAttemptId2 =
-        BuilderUtils.newApplicationAttemptId(appAttemptId.getApplicationId(), 2);
-    SchedulerEvent addAttemptEvent2 =
-        new AppAttemptAddedSchedulerEvent(appAttemptId2, true);
-    sch.handle(addAttemptEvent2);
-
-    // Check metrics after attempt added
-    Assert.assertEquals(0, queueA1.getNumApplications());
-    Assert.assertEquals(1, queueB.getNumApplications());
-    Assert.assertEquals(1, queueB1.getNumApplications());
-
-
-    QueueMetrics queueB1M = queueB1.getMetrics();
-    QueueMetrics queueBM = queueB.getMetrics();
-    // Verify allocation MB of current state
-    Assert.assertEquals(0, queueA1M.getAllocatedMB());
-    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
-    Assert.assertEquals(1024, queueB1M.getAllocatedMB());
-    Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores());
-
-    // remove attempt
-    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2,
-        RMAppAttemptState.FINISHED, false));
-
-    Assert.assertEquals(0, queueA1M.getAllocatedMB());
-    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
-    Assert.assertEquals(0, queueB1M.getAllocatedMB());
-    Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores());
-
-    verifyQueueMetrics(queueB1M);
-    verifyQueueMetrics(queueBM);
-    // Verify queue A1 metrics
-    verifyQueueMetrics(queueA1M);
-    rm.close();
-  }
-
-  private void verifyQueueMetrics(QueueMetrics queue) {
-    Assert.assertEquals(0, queue.getPendingMB());
-    Assert.assertEquals(0, queue.getActiveUsers());
-    Assert.assertEquals(0, queue.getActiveApps());
-    Assert.assertEquals(0, queue.getAppsPending());
-    Assert.assertEquals(0, queue.getAppsRunning());
-    Assert.assertEquals(0, queue.getAllocatedMB());
-    Assert.assertEquals(0, queue.getAllocatedVirtualCores());
-  }
-
-  private Configuration getCapacityConfiguration(Configuration config) {
-    CapacitySchedulerConfiguration conf =
-        new CapacitySchedulerConfiguration(config);
-
-    // Define top-level queues
-    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
-        new String[] {"a", "b"});
-    conf.setCapacity(A, 50);
-    conf.setCapacity(B, 50);
-    conf.setQueues(A, new String[] {"a1", "a2"});
-    conf.setCapacity(A1, 50);
-    conf.setCapacity(A2, 50);
-    conf.setQueues(B, new String[] {"b1"});
-    conf.setCapacity(B1, 100);
-    return conf;
-  }
-
-  @Test
-  public void testKillAllAppsInQueue() throws Exception {
-    MockRM rm = setUpMove();
-    AbstractYarnScheduler scheduler =
-        (AbstractYarnScheduler) rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-    String queue =
-        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a1", queue);
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    // now kill the app
-    scheduler.killAllAppsInQueue("a1");
-
-    // check postconditions
-    rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
-    rm.waitForAppRemovedFromScheduler(app.getApplicationId());
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.isEmpty());
-
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertTrue(appsInA1.isEmpty());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.isEmpty());
-
-    rm.stop();
-  }
-
-  @Test
-  public void testKillAllAppsInvalidSource() throws Exception {
-    MockRM rm = setUpMove();
-    YarnScheduler scheduler = rm.getResourceScheduler();
-
-    // submit an app
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("user_0")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data);
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    // now kill the app
-    try {
-      scheduler.killAllAppsInQueue("DOES_NOT_EXIST");
-      Assert.fail();
-    } catch (YarnException e) {
-      // expected
-    }
-
-    // check postconditions, app should still be in a1
-    appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(1, appsInA1.size());
-
-    appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(1, appsInA.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(1, appsInRoot.size());
-
-    rm.stop();
-  }
-
-  // Test to ensure that we don't carry out reservation on nodes
-  // that have no CPU available when using the DominantResourceCalculator
-  @Test(timeout = 30000)
-  public void testAppReservationWithDominantResourceCalculator() throws Exception {
-    CapacitySchedulerConfiguration csconf =
-        new CapacitySchedulerConfiguration();
-    csconf.setResourceComparator(DominantResourceCalculator.class);
-
-    YarnConfiguration conf = new YarnConfiguration(csconf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-      ResourceScheduler.class);
-
-    MockRM rm = new MockRM(conf);
-    rm.start();
-
-    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10 * GB, 1);
-
-    // register extra nodes to bump up cluster resource
-    MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4);
-    rm.registerNode("127.0.0.1:1236", 10 * GB, 4);
-
-    RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm);
-    // kick the scheduling
-    nm1.nodeHeartbeat(true);
-    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
-    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
-    am1.registerAppAttempt();
-    SchedulerNodeReport report_nm1 =
-        rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
-
-    // check node report
-    Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
-    Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
-
-    // add request for containers
-    am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
-    am1.schedule(); // send the request
-
-    // kick the scheduler, container reservation should not happen
-    nm1.nodeHeartbeat(true);
-    Thread.sleep(1000);
-    AllocateResponse allocResponse = am1.schedule();
-    ApplicationResourceUsageReport report =
-        rm.getResourceScheduler().getAppResourceUsageReport(
-          attempt1.getAppAttemptId());
-    Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
-    Assert.assertEquals(0, report.getNumReservedContainers());
-
-    // container should get allocated on this node
-    nm2.nodeHeartbeat(true);
-
-    while (allocResponse.getAllocatedContainers().size() == 0) {
-      Thread.sleep(100);
-      allocResponse = am1.schedule();
-    }
-    report =
-        rm.getResourceScheduler().getAppResourceUsageReport(
-          attempt1.getAppAttemptId());
-    Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
-    Assert.assertEquals(0, report.getNumReservedContainers());
-    rm.stop();
-  }
-
-  @Test
-  public void testPreemptionDisabled() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
-    RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null);
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, rmContext);
-
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueB = findQueue(rootQueue, B);
-    CSQueue queueB2 = findQueue(queueB, B2);
-
-    // When preemption turned on for the whole system
-    // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other 
-    // preemption properties set, queue root.b.b2 should be preemptable.
-    assertFalse("queue " + B2 + " should default to preemptable",
-               queueB2.getPreemptionDisabled());
-
-    // Disable preemption at the root queue level.
-    // The preemption property should be inherited from root all the
-    // way down so that root.b.b2 should NOT be preemptable.
-    conf.setPreemptionDisabled(rootQueue.getQueuePath(), true);
-    cs.reinitialize(conf, rmContext);
-    assertTrue(
-        "queue " + B2 + " should have inherited non-preemptability from root",
-        queueB2.getPreemptionDisabled());
-
-    // Enable preemption for root (grandparent) but disable for root.b (parent).
-    // root.b.b2 should inherit property from parent and NOT be preemptable
-    conf.setPreemptionDisabled(rootQueue.getQueuePath(), false);
-    conf.setPreemptionDisabled(queueB.getQueuePath(), true);
-    cs.reinitialize(conf, rmContext);
-    assertTrue(
-        "queue " + B2 + " should have inherited non-preemptability from parent",
-        queueB2.getPreemptionDisabled());
-
-    // When preemption is turned on for root.b.b2, it should be preemptable
-    // even though preemption is disabled on root.b (parent).
-    conf.setPreemptionDisabled(queueB2.getQueuePath(), false);
-    cs.reinitialize(conf, rmContext);
-    assertFalse("queue " + B2 + " should have been preemptable",
-        queueB2.getPreemptionDisabled());
-  }
-
-  @Test
-  public void testRefreshQueuesMaxAllocationRefresh() throws Exception {
-    // queue refresh should not allow changing the maximum allocation setting
-    // per queue to be smaller than previous setting
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    assertEquals("max allocation in CS",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max allocation for A1",
-        Resources.none(),
-        conf.getQueueMaximumAllocation(A1));
-    assertEquals("max allocation",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
-
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueA = findQueue(rootQueue, A);
-    CSQueue queueA1 = findQueue(queueA, A1);
-    assertEquals("queue max allocation", ((LeafQueue) queueA1)
-        .getMaximumAllocation().getMemorySize(), 8192);
-
-    setMaxAllocMb(conf, A1, 4096);
-
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("max allocation exception",
-          e.getCause().toString().contains("not be decreased"));
-    }
-
-    setMaxAllocMb(conf, A1, 8192);
-    cs.reinitialize(conf, mockContext);
-
-    setMaxAllocVcores(conf, A1,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("max allocation exception",
-          e.getCause().toString().contains("not be decreased"));
-    }
-  }
-
-  @Test
-  public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception {
-    // verify we can't set the allocation per queue larger then cluster setting
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    cs.init(conf);
-    cs.start();
-    // change max allocation for B3 queue to be larger then cluster max
-    setMaxAllocMb(conf, B3,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("maximum allocation exception",
-          e.getCause().getMessage().contains("maximum allocation"));
-    }
-
-    setMaxAllocMb(conf, B3,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
-    cs.reinitialize(conf, mockContext);
-
-    setMaxAllocVcores(conf, B3,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("maximum allocation exception",
-          e.getCause().getMessage().contains("maximum allocation"));
-    }
-  }
-
-  @Test
-  public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception {
-    // queue refresh should allow max allocation per queue to go larger
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    setMaxAllocMb(conf,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
-    setMaxAllocVcores(conf,
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
-    setMaxAllocMb(conf, A1, 4096);
-    setMaxAllocVcores(conf, A1, 2);
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueA = findQueue(rootQueue, A);
-    CSQueue queueA1 = findQueue(queueA, A1);
-
-    assertEquals("max capability MB in CS",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max capability vcores in CS",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-        cs.getMaximumResourceCapability().getVirtualCores());
-    assertEquals("max allocation MB A1",
-        4096,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A1",
-        2,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("cluster max allocation MB",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
-    assertEquals("cluster max allocation vcores",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores());
-
-    assertEquals("queue max allocation", 4096,
-            queueA1.getMaximumAllocation().getMemorySize());
-
-    setMaxAllocMb(conf, A1, 6144);
-    setMaxAllocVcores(conf, A1, 3);
-    cs.reinitialize(conf, null);
-    // conf will have changed but we shouldn't be able to change max allocation
-    // for the actual queue
-    assertEquals("max allocation MB A1", 6144,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A1", 3,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("max allocation MB cluster",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
-    assertEquals("max allocation vcores cluster",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores());
-    assertEquals("queue max allocation MB", 6144,
-        queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("queue max allocation vcores", 3,
-        queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("max capability MB cluster",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("cluster max capability vcores",
-        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-        cs.getMaximumResourceCapability().getVirtualCores());
-  }
-
-  @Test
-  public void testRefreshQueuesMaxAllocationCSError() throws Exception {
-    // Try to refresh the cluster level max allocation size to be smaller
-    // and it should error out
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    setMaxAllocMb(conf, 10240);
-    setMaxAllocVcores(conf, 10);
-    setMaxAllocMb(conf, A1, 4096);
-    setMaxAllocVcores(conf, A1, 4);
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    assertEquals("max allocation MB in CS", 10240,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max allocation vcores in CS", 10,
-        cs.getMaximumResourceCapability().getVirtualCores());
-
-    setMaxAllocMb(conf, 6144);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("max allocation exception",
-          e.getCause().toString().contains("not be decreased"));
-    }
-
-    setMaxAllocMb(conf, 10240);
-    cs.reinitialize(conf, mockContext);
-
-    setMaxAllocVcores(conf, 8);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("should have thrown exception");
-    } catch (IOException e) {
-      assertTrue("max allocation exception",
-          e.getCause().toString().contains("not be decreased"));
-    }
-  }
-
-  @Test
-  public void testRefreshQueuesMaxAllocationCSLarger() throws Exception {
-    // Try to refresh the cluster level max allocation size to be larger
-    // and verify that if there is no setting per queue it uses the
-    // cluster level setting.
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    setMaxAllocMb(conf, 10240);
-    setMaxAllocVcores(conf, 10);
-    setMaxAllocMb(conf, A1, 4096);
-    setMaxAllocVcores(conf, A1, 4);
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    assertEquals("max allocation MB in CS", 10240,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max allocation vcores in CS", 10,
-        cs.getMaximumResourceCapability().getVirtualCores());
-
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueA = findQueue(rootQueue, A);
-    CSQueue queueB = findQueue(rootQueue, B);
-    CSQueue queueA1 = findQueue(queueA, A1);
-    CSQueue queueA2 = findQueue(queueA, A2);
-    CSQueue queueB2 = findQueue(queueB, B2);
-
-    assertEquals("queue A1 max allocation MB", 4096,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("queue A1 max allocation vcores", 4,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("queue A2 max allocation MB", 10240,
-            queueA2.getMaximumAllocation().getMemorySize());
-    assertEquals("queue A2 max allocation vcores", 10,
-            queueA2.getMaximumAllocation().getVirtualCores());
-    assertEquals("queue B2 max allocation MB", 10240,
-            queueB2.getMaximumAllocation().getMemorySize());
-    assertEquals("queue B2 max allocation vcores", 10,
-            queueB2.getMaximumAllocation().getVirtualCores());
-
-    setMaxAllocMb(conf, 12288);
-    setMaxAllocVcores(conf, 12);
-    cs.reinitialize(conf, null);
-    // cluster level setting should change and any queues without
-    // per queue setting
-    assertEquals("max allocation MB in CS", 12288,
-        cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max allocation vcores in CS", 12,
-        cs.getMaximumResourceCapability().getVirtualCores());
-    assertEquals("queue A1 max MB allocation", 4096,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("queue A1 max vcores allocation", 4,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("queue A2 max MB allocation", 12288,
-            queueA2.getMaximumAllocation().getMemorySize());
-    assertEquals("queue A2 max vcores allocation", 12,
-            queueA2.getMaximumAllocation().getVirtualCores());
-    assertEquals("queue B2 max MB allocation", 12288,
-            queueB2.getMaximumAllocation().getMemorySize());
-    assertEquals("queue B2 max vcores allocation", 12,
-            queueB2.getMaximumAllocation().getVirtualCores());
-  }
-
-  @Test
-  public void testQueuesMaxAllocationInheritance() throws Exception {
-    // queue level max allocation is set by the queue configuration explicitly
-    // or inherits from the parent.
-
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    setMaxAllocMb(conf,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
-    setMaxAllocVcores(conf,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
-
-    // Test the child queue overrides
-    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
-            "memory-mb=4096,vcores=2");
-    setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2");
-    setMaxAllocation(conf, B, "memory-mb=5120, vcores=2");
-    setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2");
-
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueA = findQueue(rootQueue, A);
-    CSQueue queueB = findQueue(rootQueue, B);
-    CSQueue queueA1 = findQueue(queueA, A1);
-    CSQueue queueA2 = findQueue(queueA, A2);
-    CSQueue queueB1 = findQueue(queueB, B1);
-    CSQueue queueB2 = findQueue(queueB, B2);
-
-    assertEquals("max capability MB in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max capability vcores in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            cs.getMaximumResourceCapability().getVirtualCores());
-    assertEquals("max allocation MB A1",
-            6144,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A1",
-            2,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("max allocation MB A2",          4096,
-            queueA2.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A2",
-            2,
-            queueA2.getMaximumAllocation().getVirtualCores());
-    assertEquals("max allocation MB B",          5120,
-            queueB.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation MB B1",          5120,
-            queueB1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation MB B2",          1024,
-            queueB2.getMaximumAllocation().getMemorySize());
-
-    // Test get the max-allocation from different parent
-    unsetMaxAllocation(conf, A1);
-    unsetMaxAllocation(conf, B);
-    unsetMaxAllocation(conf, B1);
-    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
-            "memory-mb=6144,vcores=2");
-    setMaxAllocation(conf, A, "memory-mb=8192,vcores=2");
-
-    cs.reinitialize(conf, mockContext);
-
-    assertEquals("max capability MB in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max capability vcores in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            cs.getMaximumResourceCapability().getVirtualCores());
-    assertEquals("max allocation MB A1",
-            8192,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A1",
-            2,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("max allocation MB B1",
-            6144,
-            queueB1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores B1",
-            2,
-            queueB1.getMaximumAllocation().getVirtualCores());
-
-    // Test the default
-    unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT);
-    unsetMaxAllocation(conf, A);
-    unsetMaxAllocation(conf, A1);
-    cs.reinitialize(conf, mockContext);
-
-    assertEquals("max capability MB in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            cs.getMaximumResourceCapability().getMemorySize());
-    assertEquals("max capability vcores in CS",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            cs.getMaximumResourceCapability().getVirtualCores());
-    assertEquals("max allocation MB A1",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            queueA1.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A1",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            queueA1.getMaximumAllocation().getVirtualCores());
-    assertEquals("max allocation MB A2",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            queueA2.getMaximumAllocation().getMemorySize());
-    assertEquals("max allocation vcores A2",
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            queueA2.getMaximumAllocation().getVirtualCores());
-  }
-
-  @Test
-  public void testVerifyQueuesMaxAllocationConf() throws Exception {
-    // queue level max allocation can't exceed the cluster setting
-
-    CapacityScheduler cs = new CapacityScheduler();
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    setMaxAllocMb(conf,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
-    setMaxAllocVcores(conf,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
-
-    long largerMem =
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1024;
-    long largerVcores =
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES+10;
-
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, mockContext);
-    checkQueueStructureCapacities(cs);
-
-    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
-            "memory-mb=" + largerMem + ",vcores=2");
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Queue Root maximum allocation can't exceed the cluster setting");
-    } catch(Exception e) {
-      assertTrue("maximum allocation exception",
-              e.getCause().getMessage().contains("maximum allocation"));
-    }
-
-    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
-            "memory-mb=4096,vcores=2");
-    setMaxAllocation(conf, A, "memory-mb=6144,vcores=2");
-    setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2");
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Queue A1 maximum allocation can't exceed the cluster setting");
-    } catch(Exception e) {
-      assertTrue("maximum allocation exception",
-              e.getCause().getMessage().contains("maximum allocation"));
-    }
-    setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Queue A1 maximum allocation can't exceed the cluster setting");
-    } catch(Exception e) {
-      assertTrue("maximum allocation exception",
-              e.getCause().getMessage().contains("maximum allocation"));
-    }
-
-  }
-
-  private void waitContainerAllocated(MockAM am, int mem, int nContainer,
-      int startContainerId, MockRM rm, MockNM nm) throws Exception {
-    for (int cId = startContainerId; cId < startContainerId + nContainer; cId++) {
-      am.allocate("*", mem, 1, new ArrayList<ContainerId>());
-      ContainerId containerId =
-          ContainerId.newContainerId(am.getApplicationAttemptId(), cId);
-      Assert.assertTrue(rm.waitForState(nm, containerId,
-          RMContainerState.ALLOCATED));
-    }
-  }
-
-  @Test
-  public void testSchedulerKeyGarbageCollection() throws Exception {
-    YarnConfiguration conf =
-        new YarnConfiguration(new CapacitySchedulerConfiguration());
-    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
-
-    MockRM rm = new MockRM(conf);
-    rm.start();
-
-    HashMap<NodeId, MockNM> nodes = new HashMap<>();
-    MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
-    nodes.put(nm1.getNodeId(), nm1);
-    MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
-    nodes.put(nm2.getNodeId(), nm2);
-    MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
-    nodes.put(nm3.getNodeId(), nm3);
-    MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
-    nodes.put(nm4.getNodeId(), nm4);
-    nm1.registerNode();
-    nm2.registerNode();
-    nm3.registerNode();
-    nm4.registerNode();
-
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
-            .withAppName("app")
-            .withUser("user")
-            .withAcls(null)
-            .withQueue("default")
+            .withQueue("default")
             .withUnmanagedAM(false)
             .build();
     RMApp app1 = MockRMAppSubmitter.submit(rm, data);
@@ -3980,45 +1911,16 @@ public void testHeadRoomCalculationWithDRC() throws Exception {
     cs.handle(new NodeUpdateSchedulerEvent(node));
     cs.handle(new NodeUpdateSchedulerEvent(node2));
     assertEquals(6*GB, fiCaApp1.getHeadroom().getMemorySize());
-    assertEquals(15, fiCaApp1.getHeadroom().getVirtualCores());
-
-    // allocate container for app2 with 1GB memory and 1 vcore
-    fiCaApp2.updateResourceRequests(Collections.singletonList(
-        TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
-            u0Priority, recordFactory)));
-    cs.handle(new NodeUpdateSchedulerEvent(node));
-    cs.handle(new NodeUpdateSchedulerEvent(node2));
-    assertEquals(9*GB, fiCaApp2.getHeadroom().getMemorySize());
-    assertEquals(15, fiCaApp2.getHeadroom().getVirtualCores());
-  }
-
-  @Test
-  public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    conf.setDefaultNodeLabelExpression("root.a", " x");
-    conf.setDefaultNodeLabelExpression("root.b", " y ");
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
+    assertEquals(15, fiCaApp1.getHeadroom().getVirtualCores());
 
-    QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
-    Assert.assertEquals("Queue Name should be a", "a",
-        queueInfoA.getQueueName());
-    Assert.assertEquals("Queue Path should be root.a", "root.a",
-        queueInfoA.getQueuePath());
-    Assert.assertEquals("Default Node Label Expression should be x", "x",
-        queueInfoA.getDefaultNodeLabelExpression());
-
-    QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
-    Assert.assertEquals("Queue Name should be b", "b",
-        queueInfoB.getQueueName());
-    Assert.assertEquals("Queue Path should be root.b", "root.b",
-        queueInfoB.getQueuePath());
-    Assert.assertEquals("Default Node Label Expression should be y", "y",
-        queueInfoB.getDefaultNodeLabelExpression());
+    // allocate container for app2 with 1GB memory and 1 vcore
+    fiCaApp2.updateResourceRequests(Collections.singletonList(
+        TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
+            u0Priority, recordFactory)));
+    cs.handle(new NodeUpdateSchedulerEvent(node));
+    cs.handle(new NodeUpdateSchedulerEvent(node2));
+    assertEquals(9*GB, fiCaApp2.getHeadroom().getMemorySize());
+    assertEquals(15, fiCaApp2.getHeadroom().getVirtualCores());
   }
 
   @Test(timeout = 60000)
@@ -4213,44 +2115,6 @@ public Boolean get() {
     rm.stop();
   }
 
-  private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-        maxAllocMb);
-  }
-
-  private void setMaxAllocMb(CapacitySchedulerConfiguration conf,
-      String queueName, int maxAllocMb) {
-    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
-        + MAXIMUM_ALLOCATION_MB;
-    conf.setInt(propName, maxAllocMb);
-  }
-
-  private void setMaxAllocVcores(Configuration conf, int maxAllocVcores) {
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-        maxAllocVcores);
-  }
-
-  private void setMaxAllocVcores(CapacitySchedulerConfiguration conf,
-      String queueName, int maxAllocVcores) {
-    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
-        + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
-    conf.setInt(propName, maxAllocVcores);
-  }
-
-  private void setMaxAllocation(CapacitySchedulerConfiguration conf,
-                                String queueName, String maxAllocation) {
-    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
-            + MAXIMUM_ALLOCATION;
-    conf.set(propName, maxAllocation);
-  }
-
-  private void unsetMaxAllocation(CapacitySchedulerConfiguration conf,
-                                  String queueName) {
-    String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
-            + MAXIMUM_ALLOCATION;
-    conf.unset(propName);
-  }
-
   private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
     RMContainer rmContainer = cs.getRMContainer(containerId);
@@ -4261,171 +2125,6 @@ private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
       Assert.fail("Cannot find RMContainer");
     }
   }
-  @Test
-  public void testRemovedNodeDecomissioningNode() throws Exception {
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register nodemanager
-    NodeManager nm = registerNode("host_decom", 1234, 2345,
-        NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4),
-        mockNodeStatus);
-
-    RMNode node =
-        resourceManager.getRMContext().getRMNodes().get(nm.getNodeId());
-    // Send a heartbeat to kick the tires on the Scheduler
-    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
-    resourceManager.getResourceScheduler().handle(nodeUpdate);
-
-    // force remove the node to simulate race condition
-    ((CapacityScheduler) resourceManager.getResourceScheduler()).getNodeTracker().
-        removeNode(nm.getNodeId());
-    // Kick off another heartbeat with the node state mocked to decommissioning
-    RMNode spyNode =
-        Mockito.spy(resourceManager.getRMContext().getRMNodes()
-            .get(nm.getNodeId()));
-    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
-    resourceManager.getResourceScheduler().handle(
-        new NodeUpdateSchedulerEvent(spyNode));
-  }
-
-  @Test
-  public void testResourceUpdateDecommissioningNode() throws Exception {
-    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
-    // to have 0 available resource
-    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
-    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
-    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {
-      @Override
-      public void handle(Event event) {
-        if (event instanceof RMNodeResourceUpdateEvent) {
-          RMNodeResourceUpdateEvent resourceEvent =
-              (RMNodeResourceUpdateEvent) event;
-          resourceManager
-              .getResourceScheduler()
-              .getSchedulerNode(resourceEvent.getNodeId())
-              .updateTotalResource(resourceEvent.getResourceOption().getResource());
-        }
-      }
-    });
-    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
-    ((CapacityScheduler) resourceManager.getResourceScheduler())
-        .setRMContext(spyContext);
-    ((AsyncDispatcher) mockDispatcher).start();
-
-    NodeStatus mockNodeStatus = createMockNodeStatus();
-
-    // Register node
-    String host_0 = "host_0";
-    NodeManager nm_0 = registerNode(host_0, 1234, 2345,
-        NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4),
-        mockNodeStatus);
-    // ResourceRequest priorities
-    Priority priority_0 = Priority.newInstance(0);
-
-    // Submit an application
-    Application application_0 =
-        new Application("user_0", "a1", resourceManager);
-    application_0.submit();
-
-    application_0.addNodeManager(host_0, 1234, nm_0);
-
-    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
-    application_0.addResourceRequestSpec(priority_0, capability_0_0);
-
-    Task task_0_0 =
-        new Task(application_0, priority_0, new String[] { host_0 });
-    application_0.addTask(task_0_0);
-
-    // Send resource requests to the scheduler
-    application_0.schedule();
-
-    nodeUpdate(nm_0);
-    // Kick off another heartbeat with the node state mocked to decommissioning
-    // This should update the schedulernodes to have 0 available resource
-    RMNode spyNode =
-        Mockito.spy(resourceManager.getRMContext().getRMNodes()
-            .get(nm_0.getNodeId()));
-    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
-    resourceManager.getResourceScheduler().handle(
-        new NodeUpdateSchedulerEvent(spyNode));
-
-    // Get allocations from the scheduler
-    application_0.schedule();
-
-    // Check the used resource is 1 GB 1 core
-    Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize());
-    Resource usedResource =
-        resourceManager.getResourceScheduler()
-            .getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
-    Assert.assertEquals("Used Resource Memory Size should be 1GB", 1 * GB,
-        usedResource.getMemorySize());
-    Assert.assertEquals("Used Resource Virtual Cores should be 1", 1,
-        usedResource.getVirtualCores());
-    // Check total resource of scheduler node is also changed to 1 GB 1 core
-    Resource totalResource =
-        resourceManager.getResourceScheduler()
-            .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
-    Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB,
-        totalResource.getMemorySize());
-    Assert.assertEquals("Total Resource Virtual Cores should be 1", 1,
-        totalResource.getVirtualCores());
-    // Check the available resource is 0/0
-    Resource availableResource =
-        resourceManager.getResourceScheduler()
-            .getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
-    Assert.assertEquals("Available Resource Memory Size should be 0", 0,
-        availableResource.getMemorySize());
-    Assert.assertEquals("Available Resource Memory Size should be 0", 0,
-        availableResource.getVirtualCores());
-    // Kick off another heartbeat where the RMNodeResourceUpdateEvent would
-    // be skipped for DECOMMISSIONING state since the total resource is
-    // already equal to used resource from the previous heartbeat.
-    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
-    resourceManager.getResourceScheduler().handle(
-        new NodeUpdateSchedulerEvent(spyNode));
-    verify(mockDispatcher, times(4)).getEventHandler();
-  }
-
-  @Test
-  public void testSchedulingOnRemovedNode() throws Exception {
-    Configuration conf = new YarnConfiguration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-    conf.setBoolean(
-        CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE,
-            false);
-
-    MockRM rm = new MockRM(conf);
-    rm.start();
-    RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm);
-    rm.drainEvents();
-
-    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10);
-    MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
-
-    //remove nm2 to keep am alive
-    MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10);
-
-    am.allocate(ResourceRequest.ANY, 2048, 1, null);
-
-    CapacityScheduler scheduler =
-        (CapacityScheduler) rm.getRMContext().getScheduler();
-    FiCaSchedulerNode node =
-        (FiCaSchedulerNode)
-            scheduler.getNodeTracker().getNode(nm2.getNodeId());
-    scheduler.handle(new NodeRemovedSchedulerEvent(
-        rm.getRMContext().getRMNodes().get(nm2.getNodeId())));
-    // schedulerNode is removed, try allocate a container
-    scheduler.allocateContainersToNode(new SimpleCandidateNodeSet<>(node),
-        true);
-
-    AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
-        new AppAttemptRemovedSchedulerEvent(
-            am.getApplicationAttemptId(),
-            RMAppAttemptState.FINISHED, false);
-    scheduler.handle(appRemovedEvent1);
-    rm.stop();
-  }
 
   @Test
   public void testCSReservationWithRootUnblocked() throws Exception {
@@ -4627,37 +2326,6 @@ public void testCSQueueBlocked() throws Exception {
     rm.stop();
   }
 
-  private ApplicationAttemptId appHelper(MockRM rm, CapacityScheduler cs,
-                                         int clusterTs, int appId, String queue,
-                                         String user) {
-    ApplicationId appId1 = BuilderUtils.newApplicationId(clusterTs, appId);
-    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
-        appId1, appId);
-
-    RMAppAttemptMetrics attemptMetric1 =
-        new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext());
-    RMAppImpl app1 = mock(RMAppImpl.class);
-    when(app1.getApplicationId()).thenReturn(appId1);
-    RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
-    Container container = mock(Container.class);
-    when(attempt1.getMasterContainer()).thenReturn(container);
-    ApplicationSubmissionContext submissionContext = mock(
-        ApplicationSubmissionContext.class);
-    when(attempt1.getSubmissionContext()).thenReturn(submissionContext);
-    when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
-    when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
-    when(app1.getCurrentAppAttempt()).thenReturn(attempt1);
-    rm.getRMContext().getRMApps().put(appId1, app1);
-
-    SchedulerEvent addAppEvent1 =
-        new AppAddedSchedulerEvent(appId1, queue, user);
-    cs.handle(addAppEvent1);
-    SchedulerEvent addAttemptEvent1 =
-        new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
-    cs.handle(addAttemptEvent1);
-    return appAttemptId1;
-  }
-
   @Test
   public void testAppAttemptLocalityStatistics() throws Exception {
     Configuration conf =
@@ -4719,256 +2387,6 @@ protected RMNodeLabelsManager createNodeLabelManager() {
         attemptMetrics.getLocalityStatistics());
   }
 
-  /**
-   * Test for queue deletion.
-   * @throws Exception
-   */
-  @Test
-  public void testRefreshQueuesWithQueueDelete() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null);
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, rmContext);
-    checkQueueStructureCapacities(cs);
-
-    // test delete leaf queue when there is application running.
-    Map<String, CSQueue> queues =
-        cs.getCapacitySchedulerQueueManager().getShortNameQueues();
-    String b1QTobeDeleted = "b1";
-    LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted));
-    when(csB1Queue.getState()).thenReturn(QueueState.DRAINING)
-        .thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue(b1QTobeDeleted, csB1Queue);
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithoutB1(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Expected to throw exception when refresh queue tries to delete a"
-          + " queue with running apps");
-    } catch (IOException e) {
-      // ignore
-    }
-
-    // test delete leaf queue(root.b.b1) when there is no application running.
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithoutB1(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-    } catch (IOException e) {
-      LOG.error(
-          "Expected to NOT throw exception when refresh queue tries to delete"
-              + " a queue WITHOUT running apps",
-          e);
-      fail("Expected to NOT throw exception when refresh queue tries to delete"
-          + " a queue WITHOUT running apps");
-    }
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueB = findQueue(rootQueue, B);
-    CSQueue queueB3 = findQueue(queueB, B1);
-    assertNull("Refresh needs to support delete of leaf queue ", queueB3);
-
-    // reset back to default configuration for testing parent queue delete
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    cs.reinitialize(conf, rmContext);
-    checkQueueStructureCapacities(cs);
-
-    // set the configurations such that it fails once but should be successfull
-    // next time
-    queues = cs.getCapacitySchedulerQueueManager().getShortNameQueues();
-    CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b"));
-    when(bQueue.getState()).thenReturn(QueueState.DRAINING)
-        .thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b", bQueue);
-
-    bQueue = Mockito.spy((LeafQueue) queues.get("b1"));
-    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue);
-
-    bQueue = Mockito.spy((LeafQueue) queues.get("b2"));
-    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue);
-
-    bQueue = Mockito.spy((LeafQueue) queues.get("b3"));
-    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue);
-
-    // test delete Parent queue when there is application running.
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithoutB(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Expected to throw exception when refresh queue tries to delete a"
-          + " parent queue with running apps in children queue");
-    } catch (IOException e) {
-      // ignore
-    }
-
-    // test delete Parent queue when there is no application running.
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithoutB(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-    } catch (IOException e) {
-      fail("Expected to not throw exception when refresh queue tries to delete"
-          + " a queue without running apps");
-    }
-    rootQueue = cs.getRootQueue();
-    queueB = findQueue(rootQueue, B);
-    String message =
-        "Refresh needs to support delete of Parent queue and its children.";
-    assertNull(message, queueB);
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b"));
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b1"));
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b2"));
-
-    cs.stop();
-  }
-
-  /**
-   * Test for all child queue deletion and thus making parent queue a child.
-   * @throws Exception
-   */
-  @Test
-  public void testRefreshQueuesWithAllChildQueuesDeleted() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null);
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, rmContext);
-    checkQueueStructureCapacities(cs);
-
-    // test delete all leaf queues when there is no application running.
-    Map<String, CSQueue> queues =
-        cs.getCapacitySchedulerQueueManager().getShortNameQueues();
-
-    CSQueue bQueue = Mockito.spy((LeafQueue) queues.get("b1"));
-    when(bQueue.getState()).thenReturn(QueueState.RUNNING)
-        .thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue);
-
-    bQueue = Mockito.spy((LeafQueue) queues.get("b2"));
-    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue);
-
-    bQueue = Mockito.spy((LeafQueue) queues.get("b3"));
-    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
-    cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue);
-
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfWithoutChildrenOfB(conf);
-
-    // test convert parent queue to leaf queue(root.b) when there is no
-    // application running.
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Expected to throw exception when refresh queue tries to make parent"
-          + " queue a child queue when one of its children is still running.");
-    } catch (IOException e) {
-      //do not do anything, expected exception
-    }
-
-    // test delete leaf queues(root.b.b1,b2,b3) when there is no application
-    // running.
-    try {
-      cs.reinitialize(conf, mockContext);
-    } catch (IOException e) {
-      e.printStackTrace();
-      fail("Expected to NOT throw exception when refresh queue tries to delete"
-          + " all children of a parent queue(without running apps).");
-    }
-    CSQueue rootQueue = cs.getRootQueue();
-    CSQueue queueB = findQueue(rootQueue, B);
-    assertNotNull("Parent Queue B should not be deleted", queueB);
-    Assert.assertTrue("As Queue'B children are not deleted",
-        queueB instanceof LeafQueue);
-
-    String message =
-        "Refresh needs to support delete of all children of Parent queue.";
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b3"));
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b1"));
-    assertNull(message,
-        cs.getCapacitySchedulerQueueManager().getQueues().get("b2"));
-
-    cs.stop();
-  }
-
-  /**
-   * Test if we can convert a leaf queue to a parent queue
-   * @throws Exception
-   */
-  @Test (timeout = 10000)
-  public void testConvertLeafQueueToParentQueue() throws Exception {
-    CapacityScheduler cs = new CapacityScheduler();
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
-        null, new RMContainerTokenSecretManager(conf),
-        new NMTokenSecretManagerInRM(conf),
-        new ClientToAMTokenSecretManagerInRM(), null);
-    setupQueueConfiguration(conf);
-    cs.setConf(new YarnConfiguration());
-    cs.setRMContext(resourceManager.getRMContext());
-    cs.init(conf);
-    cs.start();
-    cs.reinitialize(conf, rmContext);
-    checkQueueStructureCapacities(cs);
-
-    String targetQueue = "b1";
-    CSQueue b1 = cs.getQueue(targetQueue);
-    Assert.assertEquals(QueueState.RUNNING, b1.getState());
-
-    // test if we can convert a leaf queue which is in RUNNING state
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithB1AsParentQueue(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-      fail("Expected to throw exception when refresh queue tries to convert"
-          + " a child queue to a parent queue.");
-    } catch (IOException e) {
-      // ignore
-    }
-
-    // now set queue state for b1 to STOPPED
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    conf.set("yarn.scheduler.capacity.root.b.b1.state", "STOPPED");
-    cs.reinitialize(conf, mockContext);
-    Assert.assertEquals(QueueState.STOPPED, b1.getState());
-
-    // test if we can convert a leaf queue which is in STOPPED state
-    conf = new CapacitySchedulerConfiguration();
-    setupQueueConfigurationWithB1AsParentQueue(conf);
-    try {
-      cs.reinitialize(conf, mockContext);
-    } catch (IOException e) {
-      fail("Expected to NOT throw exception when refresh queue tries"
-          + " to convert a leaf queue WITHOUT running apps");
-    }
-    b1 = cs.getQueue(targetQueue);
-    Assert.assertTrue(b1 instanceof ParentQueue);
-    Assert.assertEquals(QueueState.RUNNING, b1.getState());
-    Assert.assertTrue(!b1.getChildQueues().isEmpty());
-  }
 
   @Test(timeout = 30000)
   public void testAMLimitDouble() throws Exception {
@@ -5247,164 +2665,62 @@ public void testContainerAllocationLocalitySkipped() throws Exception {
         ContainerAllocation.QUEUE_SKIPPED.getAllocationState());
   }
 
+  /**
+   * Tests
+   * @throws Exception
+   */
   @Test
-  public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception {
-
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-      ResourceScheduler.class);
+  public void testCSQueueMetricsDoesNotLeakOnReinit() throws Exception {
+    // Initialize resource map
+    Map<String, ResourceInformation> riMap = new HashMap<>();
 
-    CapacitySchedulerConfiguration newConf =
-        new CapacitySchedulerConfiguration(conf);
+    // Initialize mandatory resources
+    ResourceInformation memory =
+        ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
+            ResourceInformation.MEMORY_MB.getUnits(),
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    ResourceInformation vcores =
+        ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
+            ResourceInformation.VCORES.getUnits(),
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    riMap.put(ResourceInformation.MEMORY_URI, memory);
+    riMap.put(ResourceInformation.VCORES_URI, vcores);
 
-    // Define top-level queues
-    newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
-        new String[] { "a", "b" });
+    ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
 
-    newConf.setCapacity(A, 50);
-    newConf.setCapacity(B, 50);
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration();
+    csConf.setResourceComparator(DominantResourceCalculator.class);
 
-    // Define 2nd-level queues
-    newConf.setQueues(A, new String[] { "a1" });
-    newConf.setCapacity(A1, 100);
-    newConf.setUserLimitFactor(A1, 2.0f);
-    newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f);
+    setupQueueConfiguration(csConf);
 
-    newConf.setQueues(B, new String[] { "b1" });
-    newConf.setCapacity(B1, 100);
-    newConf.setUserLimitFactor(B1, 2.0f);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
 
-    LOG.info("Setup top-level queues a and b");
+    // Don't reset resource types since we have already configured resource
+    // types
+    conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
 
-    MockRM rm = new MockRM(newConf);
+    MockRM rm = new MockRM(conf);
     rm.start();
 
-    CapacityScheduler scheduler =
-        (CapacityScheduler) rm.getResourceScheduler();
-
-    MockNM nm1 = rm.registerNode("h1:1234", 16 * GB);
-
-    // submit an app
-    MockRMAppSubmissionData data3 =
-        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
-            .withAppName("test-move-1")
-            .withUser("u1")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app = MockRMAppSubmitter.submit(rm, data3);
-    MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1);
-
-    ApplicationAttemptId appAttemptId =
-        rm.getApplicationReport(app.getApplicationId())
-            .getCurrentApplicationAttemptId();
-
-    MockRMAppSubmissionData data2 =
-        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
-            .withAppName("app")
-            .withUser("u2")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
-    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
-
-    MockRMAppSubmissionData data1 =
-        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
-            .withAppName("app")
-            .withUser("u3")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app3 = MockRMAppSubmitter.submit(rm, data1);
-
-    MockRMAppSubmissionData data =
-        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
-            .withAppName("app")
-            .withUser("u4")
-            .withAcls(null)
-            .withQueue("a1")
-            .withUnmanagedAM(false)
-            .build();
-    RMApp app4 = MockRMAppSubmitter.submit(rm, data);
-
-    // Each application asks 50 * 1GB containers
-    am1.allocate("*", 1 * GB, 50, null);
-    am2.allocate("*", 1 * GB, 50, null);
-
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
-    RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
-
-    // check preconditions
-    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(4, appsInA1.size());
-    String queue =
-        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("a1", queue);
-
-    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
-    assertTrue(appsInA.contains(appAttemptId));
-    assertEquals(4, appsInA.size());
-
-    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(4, appsInRoot.size());
-
-    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
-    assertTrue(appsInB1.isEmpty());
-
-    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.isEmpty());
-
-    UsersManager um =
-        (UsersManager) scheduler.getQueue("a1").getAbstractUsersManager();
-
-    assertEquals(4, um.getNumActiveUsers());
-    assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
-
-    // now move the app
-    scheduler.moveAllApps("a1", "b1");
-
-    //Triggering this event so that user limit computation can
-    //happen again
-    for (int i = 0; i < 10; i++) {
-      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-      Thread.sleep(500);
-    }
-
-    // check postconditions
-    appsInB1 = scheduler.getAppsInQueue("b1");
-
-    assertEquals(4, appsInB1.size());
-    queue =
-        scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
-            .getQueueName();
-    Assert.assertEquals("b1", queue);
-
-    appsInB = scheduler.getAppsInQueue("b");
-    assertTrue(appsInB.contains(appAttemptId));
-    assertEquals(4, appsInB.size());
-
-    appsInRoot = scheduler.getAppsInQueue("root");
-    assertTrue(appsInRoot.contains(appAttemptId));
-    assertEquals(4, appsInRoot.size());
-
-    List<ApplicationAttemptId> oldAppsInA1 = scheduler.getAppsInQueue("a1");
-    assertEquals(0, oldAppsInA1.size());
-
-    UsersManager um_b1 =
-        (UsersManager) scheduler.getQueue("b1").getAbstractUsersManager();
-
-    assertEquals(2, um_b1.getNumActiveUsers());
-    assertEquals(2, um_b1.getNumActiveUsersWithOnlyPendingApps());
-
-    appsInB1 = scheduler.getAppsInQueue("b1");
-    assertEquals(4, appsInB1.size());
-    rm.close();
+    csConf = new CapacitySchedulerConfiguration();
+    setupAdditionalQueues(csConf);
+    cs.reinitialize(csConf, cs.getRMContext());
+    QueueMetrics a3DefaultPartitionMetrics = QueueMetrics.getQueueMetrics().get(
+        "default.root.a.a3");
+
+    Assert.assertSame("Different ParentQueue of siblings is a sign of a memory leak",
+        QueueMetrics.getQueueMetrics().get("root.a.a1").getParentQueue(),
+        QueueMetrics.getQueueMetrics().get("root.a.a3").getParentQueue());
+
+    Assert.assertSame("Different ParentQueue of partition metrics is a sign of a memory leak",
+        QueueMetrics.getQueueMetrics().get("root.a.a1").getParentQueue(),
+        a3DefaultPartitionMetrics.getParentQueue());
   }
 
   @Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
new file mode 100644
index 0000000000000..d192e7dcc6933
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
@@ -0,0 +1,1559 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
+import org.apache.hadoop.yarn.server.resourcemanager.NodeManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.Task;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkNodeResourceUsage;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.setUpMove;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.setUpMoveAmbiguousQueue;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestCapacitySchedulerApps {
+
+  public static final int MAX_PARALLEL_APPS = 5;
+  public static final String USER_0 = "user_0";
+  private ResourceManager resourceManager = null;
+  private RMContext mockContext;
+
+  @Before
+  public void setUp() throws Exception {
+    resourceManager = createResourceManager();
+    mockContext = createMockRMContext();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    stopResourceManager(resourceManager);
+  }
+
+  @Test
+  public void testGetAppsInQueue() throws Exception {
+    Application application0 = new Application("user_0", "a1", resourceManager);
+    application0.submit();
+
+    Application application1 = new Application("user_0", "a2", resourceManager);
+    application1.submit();
+
+    Application application2 = new Application("user_0", "b2", resourceManager);
+    application2.submit();
+
+    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
+
+    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
+    assertEquals(1, appsInA1.size());
+
+    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.contains(application0.getApplicationAttemptId()));
+    assertTrue(appsInA.contains(application1.getApplicationAttemptId()));
+    assertEquals(2, appsInA.size());
+
+    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(application0.getApplicationAttemptId()));
+    assertTrue(appsInRoot.contains(application1.getApplicationAttemptId()));
+    assertTrue(appsInRoot.contains(application2.getApplicationAttemptId()));
+    assertEquals(3, appsInRoot.size());
+
+    Assert.assertNull(scheduler.getAppsInQueue("nonexistentqueue"));
+  }
+
+  @Test
+  public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    MockRM rm = new MockRM(conf);
+    @SuppressWarnings("unchecked")
+    AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
+        (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
+            .getResourceScheduler();
+    SchedulerApplication<SchedulerApplicationAttempt> app =
+        TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
+            cs.getSchedulerApplications(), cs, "a1");
+    Assert.assertEquals("a1", app.getQueue().getQueueName());
+  }
+
+  @Test
+  public void testKillAllAppsInQueue() throws Exception {
+    MockRM rm = setUpMove();
+    AbstractYarnScheduler scheduler =
+        (AbstractYarnScheduler) rm.getResourceScheduler();
+
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    ApplicationAttemptId appAttemptId =
+        rm.getApplicationReport(app.getApplicationId())
+            .getCurrentApplicationAttemptId();
+
+    // check preconditions
+    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
+    assertEquals(1, appsInA1.size());
+
+    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.contains(appAttemptId));
+    assertEquals(1, appsInA.size());
+    String queue =
+        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
+            .getQueueName();
+    Assert.assertEquals("a1", queue);
+
+    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(appAttemptId));
+    assertEquals(1, appsInRoot.size());
+
+    // now kill the app
+    scheduler.killAllAppsInQueue("a1");
+
+    // check postconditions
+    rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
+    rm.waitForAppRemovedFromScheduler(app.getApplicationId());
+    appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.isEmpty());
+
+    appsInA1 = scheduler.getAppsInQueue("a1");
+    assertTrue(appsInA1.isEmpty());
+
+    appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.isEmpty());
+
+    rm.stop();
+  }
+
+  @Test
+  public void testKillAllAppsInvalidSource() throws Exception {
+    MockRM rm = setUpMove();
+    YarnScheduler scheduler = rm.getResourceScheduler();
+
+    // submit an app
+    ApplicationAttemptId appAttemptId = submitApp(rm);
+
+    // check preconditions
+    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
+    assertEquals(1, appsInA1.size());
+
+    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.contains(appAttemptId));
+    assertEquals(1, appsInA.size());
+
+    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(appAttemptId));
+    assertEquals(1, appsInRoot.size());
+
+    // now kill the app
+    try {
+      scheduler.killAllAppsInQueue("DOES_NOT_EXIST");
+      Assert.fail();
+    } catch (YarnException e) {
+      // expected
+    }
+
+    // check postconditions, app should still be in a1
+    appsInA1 = scheduler.getAppsInQueue("a1");
+    assertEquals(1, appsInA1.size());
+
+    appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.contains(appAttemptId));
+    assertEquals(1, appsInA.size());
+
+    appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(appAttemptId));
+    assertEquals(1, appsInRoot.size());
+
+    rm.stop();
+  }
+
+  // Test to ensure that we don't carry out reservation on nodes
+  // that have no CPU available when using the DominantResourceCalculator
+  @Test(timeout = 30000)
+  public void testAppReservationWithDominantResourceCalculator() throws Exception {
+    CapacitySchedulerConfiguration csconf =
+        new CapacitySchedulerConfiguration();
+    csconf.setResourceComparator(DominantResourceCalculator.class);
+
+    YarnConfiguration conf = new YarnConfiguration(csconf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    MockRM rm = new MockRM(conf);
+    rm.start();
+
+    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10 * GB, 1);
+
+    // register extra nodes to bump up cluster resource
+    MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10 * GB, 4);
+    rm.registerNode("127.0.0.1:1236", 10 * GB, 4);
+
+    RMApp app1 = MockRMAppSubmitter.submitWithMemory(1024, rm);
+    // kick the scheduling
+    nm1.nodeHeartbeat(true);
+    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+    am1.registerAppAttempt();
+    SchedulerNodeReport reportNm1 =
+        rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+
+    // check node report
+    Assert.assertEquals(1 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(9 * GB, reportNm1.getAvailableResource().getMemorySize());
+
+    // add request for containers
+    am1.addRequests(new String[]{"127.0.0.1", "127.0.0.2"}, 1 * GB, 1, 1);
+    am1.schedule(); // send the request
+
+    // kick the scheduler, container reservation should not happen
+    nm1.nodeHeartbeat(true);
+    Thread.sleep(1000);
+    AllocateResponse allocResponse = am1.schedule();
+    ApplicationResourceUsageReport report =
+        rm.getResourceScheduler().getAppResourceUsageReport(
+            attempt1.getAppAttemptId());
+    Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
+    Assert.assertEquals(0, report.getNumReservedContainers());
+
+    // container should get allocated on this node
+    nm2.nodeHeartbeat(true);
+
+    while (allocResponse.getAllocatedContainers().size() == 0) {
+      Thread.sleep(100);
+      allocResponse = am1.schedule();
+    }
+    report =
+        rm.getResourceScheduler().getAppResourceUsageReport(
+            attempt1.getAppAttemptId());
+    Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
+    Assert.assertEquals(0, report.getNumReservedContainers());
+    rm.stop();
+  }
+
+  @Test
+  public void testMoveAppAmbiguousQueue() throws Exception {
+    MockRM rm = setUpMoveAmbiguousQueue();
+    AbstractYarnScheduler scheduler =
+        (AbstractYarnScheduler) rm.getResourceScheduler();
+    QueueMetrics metrics = scheduler.getRootQueueMetrics();
+    Assert.assertEquals(0, metrics.getAppsPending());
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("root.a.a")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    // check preconditions
+    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("root.a.a");
+    assertEquals(1, appsInA.size());
+    String queue =
+        scheduler.getApplicationAttempt(appsInA.get(0)).getQueue()
+            .getQueueName();
+    Assert.assertEquals("a", queue);
+
+    // now move the app
+    scheduler.moveApplication(app.getApplicationId(), "a1");
+    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("root.a.a1");
+    assertEquals(1, appsInA1.size());
+    queue =
+        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
+            .getQueueName();
+    Assert.assertEquals("a1", queue);
+
+    appsInA = scheduler.getAppsInQueue("root.a.a");
+    assertTrue(appsInA.isEmpty());
+
+    rm.stop();
+  }
+
+  @Test
+  public void testMoveAppBasic() throws Exception {
+    MockRM rm = setUpMove();
+    AbstractYarnScheduler scheduler =
+        (AbstractYarnScheduler) rm.getResourceScheduler();
+    QueueMetrics metrics = scheduler.getRootQueueMetrics();
+    Assert.assertEquals(0, metrics.getAppsPending());
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    ApplicationAttemptId appAttemptId =
+        rm.getApplicationReport(app.getApplicationId())
+            .getCurrentApplicationAttemptId();
+    // check preconditions
+    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
+    assertEquals(1, appsInA1.size());
+    String queue =
+        scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
+            .getQueueName();
+    Assert.assertEquals("a1", queue);
+
+    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.contains(appAttemptId));
+    assertEquals(1, appsInA.size());
+
+    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(appAttemptId));
+    assertEquals(1, appsInRoot.size());
+
+    assertEquals(1, metrics.getAppsPending());
+
+    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
+    assertTrue(appsInB1.isEmpty());
+
+    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
+    assertTrue(appsInB.isEmpty());
+
+    // now move the app
+    scheduler.moveApplication(app.getApplicationId(), "b1");
+
+    // check postconditions
+    appsInB1 = scheduler.getAppsInQueue("b1");
+    assertEquals(1, appsInB1.size());
+    queue =
+        scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
+            .getQueueName();
+    Assert.assertEquals("b1", queue);
+
+    appsInB = scheduler.getAppsInQueue("b");
+    assertTrue(appsInB.contains(appAttemptId));
+    assertEquals(1, appsInB.size());
+
+    appsInRoot = scheduler.getAppsInQueue("root");
+    assertTrue(appsInRoot.contains(appAttemptId));
+    assertEquals(1, appsInRoot.size());
+
+    assertEquals(1, metrics.getAppsPending());
+
+    appsInA1 = scheduler.getAppsInQueue("a1");
+    assertTrue(appsInA1.isEmpty());
+
+    appsInA = scheduler.getAppsInQueue("a");
+    assertTrue(appsInA.isEmpty());
+
+    rm.stop();
+  }
+
+  @Test
+  public void testMoveAppPendingMetrics() throws Exception {
+    MockRM rm = setUpMove();
+    ResourceScheduler scheduler = rm.getResourceScheduler();
+    assertApps(scheduler, 0, 0, 0);
+
+    // submit two apps in a1
+    RMApp app1 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .build());
+    RMApp app2 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-2")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .build());
+    assertApps(scheduler, 2, 0, 2);
+
+    // submit one app in b1
+    RMApp app3 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-2")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("b1")
+            .build());
+    assertApps(scheduler, 2, 1, 3);
+
+    // now move the app1 from a1 to b1
+    scheduler.moveApplication(app1.getApplicationId(), "b1");
+    assertApps(scheduler, 1, 2, 3);
+
+    // now move the app2 from a1 to b1
+    scheduler.moveApplication(app2.getApplicationId(), "b1");
+    assertApps(scheduler, 0, 3, 3);
+
+    // now move the app3 from b1 to a1
+    scheduler.moveApplication(app3.getApplicationId(), "a1");
+    assertApps(scheduler, 1, 2, 3);
+    rm.stop();
+  }
+
+  private void assertApps(ResourceScheduler scheduler,
+                          int a1Size,
+                          int b1Size,
+                          int appsPending) {
+    assertAppsSize(scheduler, "a1", a1Size);
+    assertAppsSize(scheduler, "b1", b1Size);
+    assertEquals(appsPending, scheduler.getRootQueueMetrics().getAppsPending());
+  }
+
+  private void assertAppsSize(ResourceScheduler scheduler, String queueName, int size) {
+    assertEquals(size, scheduler.getAppsInQueue(queueName).size());
+  }
+
+  @Test
+  public void testMoveAppSameParent() throws Exception {
+    MockRM rm = setUpMove();
+    AbstractYarnScheduler scheduler =
+        (AbstractYarnScheduler) rm.getResourceScheduler();
+
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    ApplicationAttemptId appAttemptId =
+        rm.getApplicationReport(app.getApplicationId())
+            .getCurrentApplicationAttemptId();
+
+    // check preconditions
+    assertOneAppInQueue(scheduler, "a1");
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a2");
+
+    // now move the app
+    scheduler.moveApplication(app.getApplicationId(), "a2");
+
+    // check postconditions
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1");
+    assertOneAppInQueue(scheduler, "a2");
+
+    rm.stop();
+  }
+
+  private void assertApps(ResourceScheduler scheduler,
+                          String queueName,
+                          ApplicationAttemptId... apps) {
+    assertEquals(Lists.newArrayList(apps), scheduler.getAppsInQueue(queueName));
+  }
+
+  private void assertOneAppInQueue(AbstractYarnScheduler scheduler, String queueName) {
+    List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(queueName);
+    assertEquals(1, apps.size());
+    Assert.assertEquals(queueName,
+        scheduler.getApplicationAttempt(apps.get(0)).getQueue().getQueueName());
+  }
+
+  @Test
+  public void testMoveAppForMoveToQueueWithFreeCap() throws Exception {
+
+    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
+
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register node1
+    String host0 = "host_0";
+    NodeManager nm0 =
+        registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(4 * GB, 1), mockNodeStatus);
+
+    // Register node2
+    String host1 = "host_1";
+    NodeManager nm1 =
+        registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(2 * GB, 1), mockNodeStatus);
+
+    // ResourceRequest priorities
+    Priority priority0 = Priority.newInstance(0);
+    Priority priority1 = Priority.newInstance(1);
+
+    // Submit application_0
+    Application application0 =
+        new Application("user_0", "a1", resourceManager);
+    application0.submit(); // app + app attempt event sent to scheduler
+
+    application0.addNodeManager(host0, 1234, nm0);
+    application0.addNodeManager(host1, 1234, nm1);
+
+    Resource capability00 = Resources.createResource(1 * GB, 1);
+    application0.addResourceRequestSpec(priority1, capability00);
+
+    Resource capability01 = Resources.createResource(2 * GB, 1);
+    application0.addResourceRequestSpec(priority0, capability01);
+
+    Task task00 =
+        new Task(application0, priority1, new String[]{host0, host1});
+    application0.addTask(task00);
+
+    // Submit application_1
+    Application application1 =
+        new Application("user_1", "b2", resourceManager);
+    application1.submit(); // app + app attempt event sent to scheduler
+
+    application1.addNodeManager(host0, 1234, nm0);
+    application1.addNodeManager(host1, 1234, nm1);
+
+    Resource capability10 = Resources.createResource(1 * GB, 1);
+    application1.addResourceRequestSpec(priority1, capability10);
+
+    Resource capability11 = Resources.createResource(2 * GB, 1);
+    application1.addResourceRequestSpec(priority0, capability11);
+
+    Task task10 =
+        new Task(application1, priority1, new String[]{host0, host1});
+    application1.addTask(task10);
+
+    // Send resource requests to the scheduler
+    application0.schedule(); // allocate
+    application1.schedule(); // allocate
+
+    // task_0_0 task_1_0 allocated, used=2G
+    nodeUpdate(resourceManager, nm0);
+
+    // nothing allocated
+    nodeUpdate(resourceManager, nm1);
+
+    // Get allocations from the scheduler
+    application0.schedule(); // task_0_0
+    checkApplicationResourceUsage(1 * GB, application0);
+
+    application1.schedule(); // task_1_0
+    checkApplicationResourceUsage(1 * GB, application1);
+
+    checkNodeResourceUsage(2 * GB, nm0); // task_0_0 (1G) and task_1_0 (1G) 2G
+    // available
+    checkNodeResourceUsage(0 * GB, nm1); // no tasks, 2G available
+
+    // move app from a1(30% cap of total 10.5% cap) to b1(79,2% cap of 89,5%
+    // total cap)
+    scheduler.moveApplication(application0.getApplicationId(), "b1");
+
+    // 2GB 1C
+    Task task11 =
+        new Task(application1, priority0,
+            new String[]{ResourceRequest.ANY});
+    application1.addTask(task11);
+
+    application1.schedule();
+
+    // 2GB 1C
+    Task task01 =
+        new Task(application0, priority0, new String[]{host0, host1});
+    application0.addTask(task01);
+
+    application0.schedule();
+
+    // prev 2G used free 2G
+    nodeUpdate(resourceManager, nm0);
+
+    // prev 0G used free 2G
+    nodeUpdate(resourceManager, nm1);
+
+    // Get allocations from the scheduler
+    application1.schedule();
+    checkApplicationResourceUsage(3 * GB, application1);
+
+    // Get allocations from the scheduler
+    application0.schedule();
+    checkApplicationResourceUsage(3 * GB, application0);
+
+    checkNodeResourceUsage(4 * GB, nm0);
+    checkNodeResourceUsage(2 * GB, nm1);
+  }
+
+  @Test
+  public void testMoveAppSuccess() throws Exception {
+
+    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
+
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register node1
+    String host0 = "host_0";
+    NodeManager nm0 =
+        registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(5 * GB, 1), mockNodeStatus);
+
+    // Register node2
+    String host1 = "host_1";
+    NodeManager nm1 =
+        registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(5 * GB, 1), mockNodeStatus);
+
+    // ResourceRequest priorities
+    Priority priority0 = Priority.newInstance(0);
+    Priority priority1 = Priority.newInstance(1);
+
+    // Submit application_0
+    Application application0 =
+        new Application("user_0", "a1", resourceManager);
+    application0.submit(); // app + app attempt event sent to scheduler
+
+    application0.addNodeManager(host0, 1234, nm0);
+    application0.addNodeManager(host1, 1234, nm1);
+
+    Resource capability00 = Resources.createResource(3 * GB, 1);
+    application0.addResourceRequestSpec(priority1, capability00);
+
+    Resource capability01 = Resources.createResource(2 * GB, 1);
+    application0.addResourceRequestSpec(priority0, capability01);
+
+    Task task00 =
+        new Task(application0, priority1, new String[]{host0, host1});
+    application0.addTask(task00);
+
+    // Submit application_1
+    Application application1 =
+        new Application("user_1", "b2", resourceManager);
+    application1.submit(); // app + app attempt event sent to scheduler
+
+    application1.addNodeManager(host0, 1234, nm0);
+    application1.addNodeManager(host1, 1234, nm1);
+
+    Resource capability10 = Resources.createResource(1 * GB, 1);
+    application1.addResourceRequestSpec(priority1, capability10);
+
+    Resource capability11 = Resources.createResource(2 * GB, 1);
+    application1.addResourceRequestSpec(priority0, capability11);
+
+    Task task10 =
+        new Task(application1, priority1, new String[]{host0, host1});
+    application1.addTask(task10);
+
+    // Send resource requests to the scheduler
+    application0.schedule(); // allocate
+    application1.schedule(); // allocate
+
+    // b2 can only run 1 app at a time
+    scheduler.moveApplication(application0.getApplicationId(), "b2");
+
+    nodeUpdate(resourceManager, nm0);
+
+    nodeUpdate(resourceManager, nm1);
+
+    // Get allocations from the scheduler
+    application0.schedule(); // task_0_0
+    checkApplicationResourceUsage(0 * GB, application0);
+
+    application1.schedule(); // task_1_0
+    checkApplicationResourceUsage(1 * GB, application1);
+
+    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
+    // not scheduled
+    checkNodeResourceUsage(1 * GB, nm0);
+    checkNodeResourceUsage(0 * GB, nm1);
+
+    // lets move application_0 to a queue where it can run
+    scheduler.moveApplication(application0.getApplicationId(), "a2");
+    application0.schedule();
+
+    nodeUpdate(resourceManager, nm1);
+
+    // Get allocations from the scheduler
+    application0.schedule(); // task_0_0
+    checkApplicationResourceUsage(3 * GB, application0);
+
+    checkNodeResourceUsage(1 * GB, nm0);
+    checkNodeResourceUsage(3 * GB, nm1);
+
+  }
+
+  @Test(expected = YarnException.class)
+  public void testMoveAppViolateQueueState() throws Exception {
+    resourceManager = new ResourceManager() {
+      @Override
+      protected RMNodeLabelsManager createNodeLabelManager() {
+        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+        mgr.init(getConfig());
+        return mgr;
+      }
+    };
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(csConf);
+    StringBuilder qState = new StringBuilder();
+    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
+        .append(CapacitySchedulerConfiguration.DOT)
+        .append(CapacitySchedulerConfiguration.STATE);
+    csConf.set(qState.toString(), QueueState.STOPPED.name());
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    resourceManager.init(conf);
+    resourceManager.getRMContext().getContainerTokenSecretManager()
+        .rollMasterKey();
+    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
+    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
+    mockContext = mock(RMContext.class);
+    when(mockContext.getConfigurationProvider()).thenReturn(
+        new LocalConfigurationProvider());
+
+    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
+
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register node1
+    String host0 = "host_0";
+    NodeManager nm0 =
+        registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(6 * GB, 1), mockNodeStatus);
+
+    // ResourceRequest priorities
+    Priority priority0 = Priority.newInstance(0);
+    Priority priority1 = Priority.newInstance(1);
+
+    // Submit application_0
+    Application application0 =
+        new Application("user_0", "a1", resourceManager);
+    application0.submit(); // app + app attempt event sent to scheduler
+
+    application0.addNodeManager(host0, 1234, nm0);
+
+    Resource capability00 = Resources.createResource(3 * GB, 1);
+    application0.addResourceRequestSpec(priority1, capability00);
+
+    Resource capability01 = Resources.createResource(2 * GB, 1);
+    application0.addResourceRequestSpec(priority0, capability01);
+
+    Task task00 =
+        new Task(application0, priority1, new String[]{host0});
+    application0.addTask(task00);
+
+    // Send resource requests to the scheduler
+    application0.schedule(); // allocate
+
+    // task_0_0 allocated
+    nodeUpdate(resourceManager, nm0);
+
+    // Get allocations from the scheduler
+    application0.schedule(); // task_0_0
+    checkApplicationResourceUsage(3 * GB, application0);
+
+    checkNodeResourceUsage(3 * GB, nm0);
+    // b2 queue contains 3GB consumption app,
+    // add another 3GB will hit max capacity limit on queue b
+    scheduler.moveApplication(application0.getApplicationId(), "b1");
+
+  }
+
+  @Test
+  public void testMoveAppQueueMetricsCheck() throws Exception {
+    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
+
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register node1
+    String host0 = "host_0";
+    NodeManager nm0 =
+        registerNode(resourceManager, host0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(5 * GB, 1), mockNodeStatus);
+
+    // Register node2
+    String host1 = "host_1";
+    NodeManager nm1 =
+        registerNode(resourceManager, host1, 1234, 2345, NetworkTopology.DEFAULT_RACK,
+            Resources.createResource(5 * GB, 1), mockNodeStatus);
+
+    // ResourceRequest priorities
+    Priority priority0 = Priority.newInstance(0);
+    Priority priority1 = Priority.newInstance(1);
+
+    // Submit application_0
+    Application application0 =
+        new Application("user_0", "a1", resourceManager);
+    application0.submit(); // app + app attempt event sent to scheduler
+
+    application0.addNodeManager(host0, 1234, nm0);
+    application0.addNodeManager(host1, 1234, nm1);
+
+    Resource capability00 = Resources.createResource(3 * GB, 1);
+    application0.addResourceRequestSpec(priority1, capability00);
+
+    Resource capability01 = Resources.createResource(2 * GB, 1);
+    application0.addResourceRequestSpec(priority0, capability01);
+
+    Task task00 =
+        new Task(application0, priority1, new String[]{host0, host1});
+    application0.addTask(task00);
+
+    // Submit application_1
+    Application application1 =
+        new Application("user_1", "b2", resourceManager);
+    application1.submit(); // app + app attempt event sent to scheduler
+
+    application1.addNodeManager(host0, 1234, nm0);
+    application1.addNodeManager(host1, 1234, nm1);
+
+    Resource capability10 = Resources.createResource(1 * GB, 1);
+    application1.addResourceRequestSpec(priority1, capability10);
+
+    Resource capability11 = Resources.createResource(2 * GB, 1);
+    application1.addResourceRequestSpec(priority0, capability11);
+
+    Task task10 =
+        new Task(application1, priority1, new String[]{host0, host1});
+    application1.addTask(task10);
+
+    // Send resource requests to the scheduler
+    application0.schedule(); // allocate
+    application1.schedule(); // allocate
+
+    nodeUpdate(resourceManager, nm0);
+
+    nodeUpdate(resourceManager, nm1);
+
+    CapacityScheduler cs =
+        (CapacityScheduler) resourceManager.getResourceScheduler();
+    CSQueue origRootQ = cs.getRootQueue();
+    CapacitySchedulerInfo oldInfo =
+        new CapacitySchedulerInfo(origRootQ, cs);
+    int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues());
+    int origNumAppsRoot = origRootQ.getNumApplications();
+
+    scheduler.moveApplication(application0.getApplicationId(), "a2");
+
+    CSQueue newRootQ = cs.getRootQueue();
+    int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues());
+    int newNumAppsRoot = newRootQ.getNumApplications();
+    CapacitySchedulerInfo newInfo =
+        new CapacitySchedulerInfo(newRootQ, cs);
+    CapacitySchedulerLeafQueueInfo origOldA1 =
+        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues());
+    CapacitySchedulerLeafQueueInfo origNewA1 =
+        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", newInfo.getQueues());
+    CapacitySchedulerLeafQueueInfo targetOldA2 =
+        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", oldInfo.getQueues());
+    CapacitySchedulerLeafQueueInfo targetNewA2 =
+        (CapacitySchedulerLeafQueueInfo) getQueueInfo("a2", newInfo.getQueues());
+    // originally submitted here
+    assertEquals(1, origOldA1.getNumApplications());
+    assertEquals(1, origNumAppsA);
+    assertEquals(2, origNumAppsRoot);
+    // after the move
+    assertEquals(0, origNewA1.getNumApplications());
+    assertEquals(1, newNumAppsA);
+    assertEquals(2, newNumAppsRoot);
+    // original consumption on a1
+    assertEquals(3 * GB, origOldA1.getResourcesUsed().getMemorySize());
+    assertEquals(1, origOldA1.getResourcesUsed().getvCores());
+    assertEquals(0, origNewA1.getResourcesUsed().getMemorySize()); // after the move
+    assertEquals(0, origNewA1.getResourcesUsed().getvCores()); // after the move
+    // app moved here with live containers
+    assertEquals(3 * GB, targetNewA2.getResourcesUsed().getMemorySize());
+    assertEquals(1, targetNewA2.getResourcesUsed().getvCores());
+    // it was empty before the move
+    assertEquals(0, targetOldA2.getNumApplications());
+    assertEquals(0, targetOldA2.getResourcesUsed().getMemorySize());
+    assertEquals(0, targetOldA2.getResourcesUsed().getvCores());
+    // after the app moved here
+    assertEquals(1, targetNewA2.getNumApplications());
+    // 1 container on original queue before move
+    assertEquals(1, origOldA1.getNumContainers());
+    // after the move the resource released
+    assertEquals(0, origNewA1.getNumContainers());
+    // and moved to the new queue
+    assertEquals(1, targetNewA2.getNumContainers());
+    // which originally didn't have any
+    assertEquals(0, targetOldA2.getNumContainers());
+    // 1 user with 3GB
+    assertEquals(3 * GB, origOldA1.getUsers().getUsersList().get(0)
+        .getResourcesUsed().getMemorySize());
+    // 1 user with 1 core
+    assertEquals(1, origOldA1.getUsers().getUsersList().get(0)
+        .getResourcesUsed().getvCores());
+    // user ha no more running app in the orig queue
+    assertEquals(0, origNewA1.getUsers().getUsersList().size());
+    // 1 user with 3GB
+    assertEquals(3 * GB, targetNewA2.getUsers().getUsersList().get(0)
+        .getResourcesUsed().getMemorySize());
+    // 1 user with 1 core
+    assertEquals(1, targetNewA2.getUsers().getUsersList().get(0)
+        .getResourcesUsed().getvCores());
+
+    // Get allocations from the scheduler
+    application0.schedule(); // task_0_0
+    checkApplicationResourceUsage(3 * GB, application0);
+
+    application1.schedule(); // task_1_0
+    checkApplicationResourceUsage(1 * GB, application1);
+
+    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
+    // not scheduled
+    checkNodeResourceUsage(4 * GB, nm0);
+    checkNodeResourceUsage(0 * GB, nm1);
+
+  }
+
+  @Test
+  public void testMoveAllApps() throws Exception {
+    MockRM rm = setUpMove();
+    AbstractYarnScheduler scheduler =
+        (AbstractYarnScheduler) rm.getResourceScheduler();
+
+    // submit an app
+    ApplicationAttemptId appAttemptId = submitApp(rm);
+
+    // check preconditions
+    assertOneAppInQueue(scheduler, "a1");
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1", appAttemptId);
+    assertApps(scheduler, "b1");
+    assertApps(scheduler, "b");
+
+    // now move the app
+    scheduler.moveAllApps("a1", "b1");
+
+    // check post conditions
+    Thread.sleep(1000);
+    assertOneAppInQueue(scheduler, "b1");
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "b", appAttemptId);
+    assertApps(scheduler, "b1", appAttemptId);
+    assertApps(scheduler, "a1");
+    assertApps(scheduler, "a");
+
+    rm.stop();
+  }
+
+  @Test
+  public void testMaxParallelAppsPendingQueueMetrics() throws Exception {
+    MockRM rm = setUpMove();
+    ResourceScheduler scheduler = rm.getResourceScheduler();
+    CapacityScheduler cs = (CapacityScheduler) scheduler;
+    cs.getQueueContext().getConfiguration().setInt(CapacitySchedulerConfiguration.getQueuePrefix(A1)
+        + CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS, MAX_PARALLEL_APPS);
+    cs.reinitialize(cs.getQueueContext().getConfiguration(), mockContext);
+    List<ApplicationAttemptId> attemptIds = new ArrayList<>();
+
+    for (int i = 0; i < 2 * MAX_PARALLEL_APPS; i++) {
+      attemptIds.add(submitApp(rm));
+    }
+
+    // Finish first batch to allow the other batch to run
+    for (int i = 0; i < MAX_PARALLEL_APPS; i++) {
+      cs.handle(new AppAttemptRemovedSchedulerEvent(attemptIds.get(i),
+          RMAppAttemptState.FINISHED, true));
+    }
+
+    // Finish the remaining apps
+    for (int i = MAX_PARALLEL_APPS; i < 2 * MAX_PARALLEL_APPS; i++) {
+      cs.handle(new AppAttemptRemovedSchedulerEvent(attemptIds.get(i),
+          RMAppAttemptState.FINISHED, true));
+    }
+
+    Assert.assertEquals("No pending app should remain for root queue", 0,
+        cs.getRootQueueMetrics().getAppsPending());
+    Assert.assertEquals("No running application should remain for root queue", 0,
+        cs.getRootQueueMetrics().getAppsRunning());
+
+    rm.stop();
+  }
+
+  private ApplicationAttemptId submitApp(MockRM rm) throws Exception {
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser(USER_0)
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    return rm.getApplicationReport(app.getApplicationId())
+        .getCurrentApplicationAttemptId();
+  }
+
+  @Test
+  public void testMoveAllAppsInvalidDestination() throws Exception {
+    MockRM rm = setUpMove();
+    ResourceScheduler scheduler = rm.getResourceScheduler();
+
+    // submit an app
+    ApplicationAttemptId appAttemptId = submitApp(rm);
+
+    // check preconditions
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1", appAttemptId);
+    assertApps(scheduler, "b");
+    assertApps(scheduler, "b1");
+
+    // now move the app
+    try {
+      scheduler.moveAllApps("a1", "DOES_NOT_EXIST");
+      Assert.fail();
+    } catch (YarnException e) {
+      // expected
+    }
+
+    // check post conditions, app should still be in a1
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1", appAttemptId);
+    assertApps(scheduler, "b");
+    assertApps(scheduler, "b1");
+
+    rm.stop();
+  }
+
+  @Test
+  public void testMoveAllAppsInvalidSource() throws Exception {
+    MockRM rm = setUpMove();
+    ResourceScheduler scheduler = rm.getResourceScheduler();
+
+    // submit an app
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("user_0")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data);
+    ApplicationAttemptId appAttemptId =
+        rm.getApplicationReport(app.getApplicationId())
+            .getCurrentApplicationAttemptId();
+
+    // check preconditions
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1", appAttemptId);
+    assertApps(scheduler, "b");
+    assertApps(scheduler, "b1");
+
+    // now move the app
+    try {
+      scheduler.moveAllApps("DOES_NOT_EXIST", "b1");
+      Assert.fail();
+    } catch (YarnException e) {
+      // expected
+    }
+
+    // check post conditions, app should still be in a1
+    assertApps(scheduler, "root", appAttemptId);
+    assertApps(scheduler, "a", appAttemptId);
+    assertApps(scheduler, "a1", appAttemptId);
+    assertApps(scheduler, "b");
+    assertApps(scheduler, "b1");
+
+    rm.stop();
+  }
+
+  @Test
+  public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    CapacitySchedulerConfiguration newConf =
+        new CapacitySchedulerConfiguration(conf);
+
+    // Define top-level queues
+    newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{"a", "b"});
+
+    newConf.setCapacity(A, 50);
+    newConf.setCapacity(B, 50);
+
+    // Define 2nd-level queues
+    newConf.setQueues(A, new String[]{"a1"});
+    newConf.setCapacity(A1, 100);
+    newConf.setUserLimitFactor(A1, 2.0f);
+    newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f);
+
+    newConf.setQueues(B, new String[]{"b1"});
+    newConf.setCapacity(B1, 100);
+    newConf.setUserLimitFactor(B1, 2.0f);
+
+    MockRM rm = new MockRM(newConf);
+    rm.start();
+
+    CapacityScheduler scheduler =
+        (CapacityScheduler) rm.getResourceScheduler();
+
+    MockNM nm1 = rm.registerNode("h1:1234", 16 * GB);
+
+    // submit an app
+    MockRMAppSubmissionData data3 =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withAppName("test-move-1")
+            .withUser("u1")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app = MockRMAppSubmitter.submit(rm, data3);
+    MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1);
+
+    ApplicationAttemptId appAttemptId =
+        rm.getApplicationReport(app.getApplicationId())
+            .getCurrentApplicationAttemptId();
+
+    MockRMAppSubmissionData data2 =
+        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
+            .withAppName("app")
+            .withUser("u2")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
+
+    MockRMAppSubmissionData data1 =
+        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
+            .withAppName("app")
+            .withUser("u3")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app3 = MockRMAppSubmitter.submit(rm, data1);
+
+    MockRMAppSubmissionData data =
+        MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
+            .withAppName("app")
+            .withUser("u4")
+            .withAcls(null)
+            .withQueue("a1")
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app4 = MockRMAppSubmitter.submit(rm, data);
+
+    // Each application asks 50 * 1GB containers
+    am1.allocate("*", 1 * GB, 50, null);
+    am2.allocate("*", 1 * GB, 50, null);
+
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+    // check preconditions
+    assertApps(scheduler, "root",
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId(),
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId());
+    assertApps(scheduler, "a",
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId(),
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId());
+    assertApps(scheduler, "a1",
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId(),
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId());
+    assertApps(scheduler, "b");
+    assertApps(scheduler, "b1");
+
+    UsersManager um =
+        (UsersManager) scheduler.getQueue("a1").getAbstractUsersManager();
+
+    assertEquals(4, um.getNumActiveUsers());
+    assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
+
+    // now move the app
+    scheduler.moveAllApps("a1", "b1");
+
+    //Triggering this event so that user limit computation can
+    //happen again
+    for (int i = 0; i < 10; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+      Thread.sleep(500);
+    }
+
+    // check post conditions
+    assertApps(scheduler, "root",
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId(),
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId());
+    assertApps(scheduler, "a");
+    assertApps(scheduler, "a1");
+    assertApps(scheduler, "b",
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId(),
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId());
+    assertApps(scheduler, "b1",
+        appAttemptId,
+        app2.getCurrentAppAttempt().getAppAttemptId(),
+        app3.getCurrentAppAttempt().getAppAttemptId(),
+        app4.getCurrentAppAttempt().getAppAttemptId());
+
+    UsersManager umB1 =
+        (UsersManager) scheduler.getQueue("b1").getAbstractUsersManager();
+
+    assertEquals(2, umB1.getNumActiveUsers());
+    assertEquals(2, umB1.getNumActiveUsersWithOnlyPendingApps());
+
+    rm.close();
+  }
+
+  @Test(timeout = 60000)
+  public void testMoveAttemptNotAdded() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    MockRM rm = new MockRM(getCapacityConfiguration(conf));
+    rm.start();
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
+    ApplicationAttemptId appAttemptId =
+        BuilderUtils.newApplicationAttemptId(appId, 1);
+
+    RMAppAttemptMetrics attemptMetric =
+        new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
+    RMAppImpl app = mock(RMAppImpl.class);
+    when(app.getApplicationId()).thenReturn(appId);
+    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
+    Container container = mock(Container.class);
+    when(attempt.getMasterContainer()).thenReturn(container);
+    ApplicationSubmissionContext submissionContext =
+        mock(ApplicationSubmissionContext.class);
+    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
+    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
+    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
+    when(app.getCurrentAppAttempt()).thenReturn(attempt);
+
+    rm.getRMContext().getRMApps().put(appId, app);
+
+    SchedulerEvent addAppEvent =
+        new AppAddedSchedulerEvent(appId, "a1", "user");
+    try {
+      cs.moveApplication(appId, "b1");
+      fail("Move should throw exception app not available");
+    } catch (YarnException e) {
+      assertEquals("App to be moved application_100_0001 not found.",
+          e.getMessage());
+    }
+    cs.handle(addAppEvent);
+    cs.moveApplication(appId, "b1");
+    SchedulerEvent addAttemptEvent =
+        new AppAttemptAddedSchedulerEvent(appAttemptId, false);
+    cs.handle(addAttemptEvent);
+    CSQueue rootQ = cs.getRootQueue();
+    CSQueue queueB = cs.getQueue("b");
+    CSQueue queueA = cs.getQueue("a");
+    CSQueue queueA1 = cs.getQueue("a1");
+    CSQueue queueB1 = cs.getQueue("b1");
+    Assert.assertEquals(1, rootQ.getNumApplications());
+    Assert.assertEquals(0, queueA.getNumApplications());
+    Assert.assertEquals(1, queueB.getNumApplications());
+    Assert.assertEquals(0, queueA1.getNumApplications());
+    Assert.assertEquals(1, queueB1.getNumApplications());
+
+    rm.close();
+  }
+
+  @Test
+  public void testRemoveAttemptMoveAdded() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        CapacityScheduler.class);
+    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+    // Create Mock RM
+    MockRM rm = new MockRM(getCapacityConfiguration(conf));
+    CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler();
+    // add node
+    Resource newResource = Resource.newInstance(4 * GB, 1);
+    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
+    SchedulerEvent addNode = new NodeAddedSchedulerEvent(node);
+    sch.handle(addNode);
+
+    ApplicationAttemptId appAttemptId = appHelper(rm, sch, 100, 1, "a1", "user");
+
+    // get Queues
+    CSQueue queueA1 = sch.getQueue("a1");
+    CSQueue queueB = sch.getQueue("b");
+    CSQueue queueB1 = sch.getQueue("b1");
+
+    // add Running rm container and simulate live containers to a1
+    ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2);
+    RMContainerImpl rmContainer = mock(RMContainerImpl.class);
+    when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING);
+    Container container2 = mock(Container.class);
+    when(rmContainer.getContainer()).thenReturn(container2);
+    Resource resource = Resource.newInstance(1024, 1);
+    when(container2.getResource()).thenReturn(resource);
+    when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+    when(container2.getNodeId()).thenReturn(node.getNodeID());
+    when(container2.getId()).thenReturn(newContainerId);
+    when(rmContainer.getNodeLabelExpression())
+        .thenReturn(RMNodeLabelsManager.NO_LABEL);
+    when(rmContainer.getContainerId()).thenReturn(newContainerId);
+    sch.getApplicationAttempt(appAttemptId).getLiveContainersMap()
+        .put(newContainerId, rmContainer);
+    QueueMetrics queueA1M = queueA1.getMetrics();
+    queueA1M.incrPendingResources(rmContainer.getNodeLabelExpression(),
+        "user1", 1, resource);
+    queueA1M.allocateResources(rmContainer.getNodeLabelExpression(),
+        "user1", resource);
+    // remove attempt
+    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId,
+        RMAppAttemptState.KILLED, true));
+    // Move application to queue b1
+    sch.moveApplication(appAttemptId.getApplicationId(), "b1");
+    // Check queue metrics after move
+    Assert.assertEquals(0, queueA1.getNumApplications());
+    Assert.assertEquals(1, queueB.getNumApplications());
+    Assert.assertEquals(0, queueB1.getNumApplications());
+
+    // Release attempt add event
+    ApplicationAttemptId appAttemptId2 =
+        BuilderUtils.newApplicationAttemptId(appAttemptId.getApplicationId(), 2);
+    SchedulerEvent addAttemptEvent2 =
+        new AppAttemptAddedSchedulerEvent(appAttemptId2, true);
+    sch.handle(addAttemptEvent2);
+
+    // Check metrics after attempt added
+    Assert.assertEquals(0, queueA1.getNumApplications());
+    Assert.assertEquals(1, queueB.getNumApplications());
+    Assert.assertEquals(1, queueB1.getNumApplications());
+
+
+    QueueMetrics queueB1M = queueB1.getMetrics();
+    QueueMetrics queueBM = queueB.getMetrics();
+    // Verify allocation MB of current state
+    Assert.assertEquals(0, queueA1M.getAllocatedMB());
+    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
+    Assert.assertEquals(1024, queueB1M.getAllocatedMB());
+    Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores());
+
+    // remove attempt
+    sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2,
+        RMAppAttemptState.FINISHED, false));
+
+    Assert.assertEquals(0, queueA1M.getAllocatedMB());
+    Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores());
+    Assert.assertEquals(0, queueB1M.getAllocatedMB());
+    Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores());
+
+    verifyQueueMetrics(queueB1M);
+    verifyQueueMetrics(queueBM);
+    // Verify queue A1 metrics
+    verifyQueueMetrics(queueA1M);
+    rm.close();
+  }
+
+  @Test
+  public void testAppSubmission() throws Exception {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    conf.setQueues(A, new String[]{"a1", "a2", "b"});
+    conf.setCapacity(A1, 20);
+    conf.setCapacity("root.a.b", 10);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+
+    RMApp noParentQueueApp = submitAppAndWaitForState(rm, "q", RMAppState.FAILED);
+    Assert.assertEquals(RMAppState.FAILED, noParentQueueApp.getState());
+
+    RMApp ambiguousQueueApp = submitAppAndWaitForState(rm, "b", RMAppState.FAILED);
+    Assert.assertEquals(RMAppState.FAILED, ambiguousQueueApp.getState());
+
+    RMApp emptyPartQueueApp = submitAppAndWaitForState(rm, "root..a1", RMAppState.FAILED);
+    Assert.assertEquals(RMAppState.FAILED, emptyPartQueueApp.getState());
+
+    RMApp failedAutoQueue = submitAppAndWaitForState(rm, "root.a.b.c.d", RMAppState.FAILED);
+    Assert.assertEquals(RMAppState.FAILED, failedAutoQueue.getState());
+  }
+
+  private RMApp submitAppAndWaitForState(MockRM rm, String b, RMAppState state) throws Exception {
+    MockRMAppSubmissionData ambiguousQueueAppData =
+        MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
+            .withWaitForAppAcceptedState(false)
+            .withAppName("app")
+            .withUser("user")
+            .withAcls(null)
+            .withQueue(b)
+            .withUnmanagedAM(false)
+            .build();
+    RMApp app1 = MockRMAppSubmitter.submit(rm, ambiguousQueueAppData);
+    rm.waitForState(app1.getApplicationId(), state);
+    return app1;
+  }
+
+  private int getNumAppsInQueue(String name, List<CSQueue> queues) {
+    for (CSQueue queue : queues) {
+      if (queue.getQueueShortName().equals(name)) {
+        return queue.getNumApplications();
+      }
+    }
+    return -1;
+  }
+
+  private CapacitySchedulerQueueInfo getQueueInfo(String name,
+                                                  CapacitySchedulerQueueInfoList info) {
+    if (info != null) {
+      for (CapacitySchedulerQueueInfo queueInfo : info.getQueueInfoList()) {
+        if (queueInfo.getQueueName().equals(name)) {
+          return queueInfo;
+        } else {
+          CapacitySchedulerQueueInfo result =
+              getQueueInfo(name, queueInfo.getQueues());
+          if (result == null) {
+            continue;
+          }
+          return result;
+        }
+      }
+    }
+    return null;
+  }
+
+  private void verifyQueueMetrics(QueueMetrics queue) {
+    Assert.assertEquals(0, queue.getPendingMB());
+    Assert.assertEquals(0, queue.getActiveUsers());
+    Assert.assertEquals(0, queue.getActiveApps());
+    Assert.assertEquals(0, queue.getAppsPending());
+    Assert.assertEquals(0, queue.getAppsRunning());
+    Assert.assertEquals(0, queue.getAllocatedMB());
+    Assert.assertEquals(0, queue.getAllocatedVirtualCores());
+  }
+
+  private Configuration getCapacityConfiguration(Configuration config) {
+    CapacitySchedulerConfiguration conf =
+        new CapacitySchedulerConfiguration(config);
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{"a", "b"});
+    conf.setCapacity(A, 50);
+    conf.setCapacity(B, 50);
+    conf.setQueues(A, new String[]{"a1", "a2"});
+    conf.setCapacity(A1, 50);
+    conf.setCapacity(A2, 50);
+    conf.setQueues(B, new String[]{"b1"});
+    conf.setCapacity(B1, 100);
+    return conf;
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 98214a030c954..b36e0edc73503 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -153,7 +153,7 @@ public RMNodeLabelsManager createNodeLabelManager() {
 
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
     for (CapacityScheduler.AsyncScheduleThread thread :
-        cs.asyncSchedulerThreads) {
+        cs.getAsyncSchedulerThreads()) {
       Assert.assertTrue(thread.getName()
           .startsWith("AsyncCapacitySchedulerThread"));
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index 5bd04d01c51cd..5df92e20a276f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -89,6 +89,7 @@
 
 import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
     .NO_LABEL;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
     .capacity.CSQueueUtils.EPSILON;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
@@ -166,6 +167,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
 
   public static final float NODE_LABEL_GPU_TEMPLATE_CAPACITY = 30.0f;
   public static final float NODEL_LABEL_SSD_TEMPLATE_CAPACITY = 40.0f;
+  public static final ImmutableSet<String> RESOURCE_TYPES = ImmutableSet.of("memory", "vcores");
 
   protected MockRM mockRM = null;
   protected MockNM nm1 = null;
@@ -777,17 +779,21 @@ protected void validateEffectiveMinResource(ResourceManager rm,
             * parentQueue.getQueueCapacities().getAbsoluteCapacity(label));
     assertEquals(effMinCapacity, Resources.multiply(resourceByLabel,
         leafQueue.getQueueCapacities().getAbsoluteCapacity(label)));
-    // TODO: Wangda, I think this is a wrong test, it doesn't consider rounding
-    // loss of multiplication, the right value should be <10240, 2>, but the
-    // test expects <10240, 1>
-    // fixme, address this in the future patch (auto queue creation).
-//    if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) {
-//      assertEquals(Resource.newInstance(10 * GB, 2),
-//          leafQueue.getEffectiveCapacity(label));
-//    } else {
-//      assertEquals(Resource.newInstance(0, 0),
-//          leafQueue.getEffectiveCapacity(label));
-//    }
+
+    if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) {
+      if (leafQueue.getCapacityConfigType().equals(ABSOLUTE_RESOURCE)) {
+        String templatePrefix = cs.getConfiguration().getAutoCreatedQueueTemplateConfPrefix(
+            parentQueue.getQueuePath());
+        Resource resourceTemplate = parentQueue.getLeafQueueTemplate().getLeafQueueConfigs()
+            .getMinimumResourceRequirement(label, templatePrefix, RESOURCE_TYPES);
+        assertEquals(resourceTemplate, leafQueue.getEffectiveCapacity(label));
+      } else {
+        assertEquals(effMinCapacity, leafQueue.getEffectiveCapacity(label));
+      }
+    } else {
+      assertEquals(Resource.newInstance(0, 0),
+          leafQueue.getEffectiveCapacity(label));
+    }
 
     if (leafQueue.getQueueCapacities().getAbsoluteCapacity(label) > 0) {
       assertTrue(Resources.greaterThan(cs.getResourceCalculator(),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 835d95ec87418..90c63dbd2b95f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -680,7 +680,7 @@ public void testAutoCreationFailsWhenParentCapacityExceeded()
       // Test add one auto created queue dynamically and manually modify
       // capacity
       ManagedParentQueue parentQueue = (ManagedParentQueue) newCS.getQueue("c");
-      AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1",
+      AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c1",
           parentQueue);
       newCS.addQueue(c1);
       c1.setCapacity(0.5f);
@@ -689,13 +689,13 @@ public void testAutoCreationFailsWhenParentCapacityExceeded()
 
       setEntitlement(c1, new QueueEntitlement(0.5f, 1f));
 
-      AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2",
+      AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c2",
           parentQueue);
       newCS.addQueue(c2);
       setEntitlement(c2, new QueueEntitlement(0.5f, 1f));
 
       try {
-        AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3",
+        AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS.getQueueContext(), "c3",
             parentQueue);
         newCS.addQueue(c3);
         fail("Expected exception for auto queue creation failure");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 04f4349db1dad..1bee66eb9db9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -19,13 +19,23 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -34,9 +44,71 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
 import static org.junit.Assert.fail;
 
 public class TestCapacitySchedulerConfigValidator {
+  public static final int NODE_MEMORY = 16;
+  public static final int NODE1_VCORES = 8;
+  public static final int NODE2_VCORES = 10;
+  public static final int NODE3_VCORES = 12;
+  public static final Map<String, Long> NODE_GPU = ImmutableMap.of(GPU_URI, 2L);
+  public static final int GB = 1024;
+
+  private static final String PARENT_A = "parentA";
+  private static final String PARENT_B = "parentB";
+  private static final String LEAF_A = "leafA";
+  private static final String LEAF_B = "leafB";
+
+  private static final QueuePath PARENT_A_FULL_PATH =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT + "." + PARENT_A);
+  private static final QueuePath LEAF_A_FULL_PATH =
+          new QueuePath(PARENT_A_FULL_PATH + "." + LEAF_A);
+  private static final QueuePath PARENT_B_FULL_PATH =
+          new QueuePath(CapacitySchedulerConfiguration.ROOT + "." + PARENT_B);
+  private static final QueuePath LEAF_B_FULL_PATH =
+          new QueuePath(PARENT_B_FULL_PATH + "." + LEAF_B);
+
+  private final Resource A_MINRES = Resource.newInstance(16 * GB, 10);
+  private final Resource B_MINRES = Resource.newInstance(32 * GB, 5);
+  private final Resource FULL_MAXRES = Resource.newInstance(48 * GB, 30);
+  private final Resource PARTIAL_MAXRES = Resource.newInstance(16 * GB, 10);
+  private final Resource VCORE_EXCEEDED_MAXRES = Resource.newInstance(16 * GB, 50);
+  private Resource A_MINRES_GPU;
+  private Resource B_MINRES_GPU;
+  private Resource FULL_MAXRES_GPU;
+  private Resource PARTIAL_MAXRES_GPU;
+  private Resource GPU_EXCEEDED_MAXRES_GPU;
+
+  protected MockRM mockRM = null;
+  protected MockNM nm1 = null;
+  protected MockNM nm2 = null;
+  protected MockNM nm3 = null;
+  protected CapacityScheduler cs;
+
+  public static void setupResources(boolean useGpu) {
+    Map<String, ResourceInformation> riMap = new HashMap<>();
+
+    ResourceInformation memory = ResourceInformation.newInstance(
+        ResourceInformation.MEMORY_MB.getName(),
+        ResourceInformation.MEMORY_MB.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    ResourceInformation vcores = ResourceInformation.newInstance(
+        ResourceInformation.VCORES.getName(),
+        ResourceInformation.VCORES.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    riMap.put(ResourceInformation.MEMORY_URI, memory);
+    riMap.put(ResourceInformation.VCORES_URI, vcores);
+    if (useGpu) {
+      riMap.put(ResourceInformation.GPU_URI,
+          ResourceInformation.newInstance(ResourceInformation.GPU_URI, "", 0,
+              ResourceTypes.COUNTABLE, 0, 10L));
+    }
+
+    ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+  }
 
   /**
    * Test for the case when the scheduler.minimum-allocation-mb == 0.
@@ -69,7 +141,6 @@ public void testValidateMemoryAllocationHIgherMinThanMaxMem() {
 
   }
 
-
   @Test
   public void testValidateMemoryAllocation() {
     Map<String, String> configs = new HashMap();
@@ -115,7 +186,6 @@ public void testValidateVCoresHigherMinThanMaxVCore() {
 
   }
 
-
   @Test
   public void testValidateVCores() {
     Map<String, String> configs = new HashMap();
@@ -147,6 +217,111 @@ public void testValidateCSConfigInvalidCapacity() {
     }
   }
 
+  @Test
+  public void testValidateCSConfigDefaultRCAbsoluteModeParentMaxMemoryExceeded()
+      throws Exception {
+    setUpMockRM(false);
+    RMContext rmContext = mockRM.getRMContext();
+    CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration();
+    CapacitySchedulerConfiguration newConfiguration =
+        new CapacitySchedulerConfiguration(cs.getConfiguration());
+    newConfiguration.setMaximumResourceRequirement("",
+            LEAF_A_FULL_PATH, FULL_MAXRES);
+    try {
+      CapacitySchedulerConfigValidator
+          .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext);
+      fail("Parent maximum capacity exceeded");
+    } catch (IOException e) {
+      Assert.assertTrue(e.getCause().getMessage()
+          .startsWith("Max resource configuration"));
+    } finally {
+      mockRM.stop();
+    }
+  }
+
+  @Test
+  public void testValidateCSConfigDefaultRCAbsoluteModeParentMaxVcoreExceeded() throws Exception {
+    setUpMockRM(false);
+    RMContext rmContext = mockRM.getRMContext();
+    CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration();
+    CapacitySchedulerConfiguration newConfiguration =
+        new CapacitySchedulerConfiguration(cs.getConfiguration());
+    newConfiguration.setMaximumResourceRequirement("",
+            LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES);
+    try {
+      CapacitySchedulerConfigValidator
+          .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext);
+    } catch (IOException e) {
+      fail("In DefaultResourceCalculator vcore limits are not enforced");
+    } finally {
+      mockRM.stop();
+    }
+  }
+
+  @Test
+  public void testValidateCSConfigDominantRCAbsoluteModeParentMaxMemoryExceeded()
+      throws Exception {
+    setUpMockRM(true);
+    RMContext rmContext = mockRM.getRMContext();
+    CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration();
+    CapacitySchedulerConfiguration newConfiguration =
+        new CapacitySchedulerConfiguration(cs.getConfiguration());
+    newConfiguration.setMaximumResourceRequirement("",
+            LEAF_A_FULL_PATH, FULL_MAXRES);
+    try {
+      CapacitySchedulerConfigValidator
+          .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext);
+      fail("Parent maximum capacity exceeded");
+    } catch (IOException e) {
+      Assert.assertTrue(e.getCause().getMessage()
+          .startsWith("Max resource configuration"));
+    } finally {
+      mockRM.stop();
+    }
+  }
+
+  @Test
+  public void testValidateCSConfigDominantRCAbsoluteModeParentMaxVcoreExceeded() throws Exception {
+    setUpMockRM(true);
+    RMContext rmContext = mockRM.getRMContext();
+    CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration();
+    CapacitySchedulerConfiguration newConfiguration =
+        new CapacitySchedulerConfiguration(cs.getConfiguration());
+    newConfiguration.setMaximumResourceRequirement("",
+            LEAF_A_FULL_PATH, VCORE_EXCEEDED_MAXRES);
+    try {
+      CapacitySchedulerConfigValidator
+          .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext);
+      fail("Parent maximum capacity exceeded");
+    } catch (IOException e) {
+      Assert.assertTrue(e.getCause().getMessage()
+          .startsWith("Max resource configuration"));
+    } finally {
+      mockRM.stop();
+    }
+  }
+
+  @Test
+  public void testValidateCSConfigDominantRCAbsoluteModeParentMaxGPUExceeded() throws Exception {
+    setUpMockRM(true);
+    RMContext rmContext = mockRM.getRMContext();
+    CapacitySchedulerConfiguration oldConfiguration = cs.getConfiguration();
+    CapacitySchedulerConfiguration newConfiguration =
+        new CapacitySchedulerConfiguration(cs.getConfiguration());
+    newConfiguration.setMaximumResourceRequirement("",
+            LEAF_A_FULL_PATH, GPU_EXCEEDED_MAXRES_GPU);
+    try {
+      CapacitySchedulerConfigValidator
+          .validateCSConfiguration(oldConfiguration, newConfiguration, rmContext);
+      fail("Parent maximum capacity exceeded");
+    } catch (IOException e) {
+      Assert.assertTrue(e.getCause().getMessage()
+          .startsWith("Max resource configuration"));
+    } finally {
+      mockRM.stop();
+    }
+  }
+
   @Test
   public void testValidateCSConfigStopALeafQueue() throws IOException {
     Configuration oldConfig = CapacitySchedulerConfigGeneratorForTest
@@ -155,7 +330,7 @@ public void testValidateCSConfigStopALeafQueue() throws IOException {
     newConfig
             .set("yarn.scheduler.capacity.root.test1.state", "STOPPED");
     RMContext rmContext = prepareRMContext();
-    Boolean isValidConfig = CapacitySchedulerConfigValidator
+    boolean isValidConfig = CapacitySchedulerConfigValidator
             .validateCSConfiguration(oldConfig, newConfig, rmContext);
     Assert.assertTrue(isValidConfig);
   }
@@ -340,9 +515,11 @@ public void testAddQueueToALeafQueue() throws IOException {
     Assert.assertTrue(isValidConfig);
   }
 
-
   public static RMContext prepareRMContext() {
+    setupResources(false);
     RMContext rmContext = Mockito.mock(RMContext.class);
+    CapacityScheduler mockCs = Mockito.mock(CapacityScheduler.class);
+    Mockito.when(rmContext.getScheduler()).thenReturn(mockCs);
     LocalConfigurationProvider configProvider = Mockito
             .mock(LocalConfigurationProvider.class);
     Mockito.when(rmContext.getConfigurationProvider())
@@ -361,4 +538,94 @@ public static RMContext prepareRMContext() {
             .thenReturn(queuePlacementManager);
     return rmContext;
   }
+
+  private void setUpMockRM(boolean useDominantRC) throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    setupResources(useDominantRC);
+    CapacitySchedulerConfiguration csConf = setupCSConfiguration(conf, useDominantRC);
+
+    mockRM = new MockRM(csConf);
+
+    cs = (CapacityScheduler) mockRM.getResourceScheduler();
+    mockRM.start();
+    cs.start();
+
+    setupNodes(mockRM);
+  }
+
+  private void setupNodes(MockRM newMockRM) throws Exception {
+      nm1 = new MockNM("h1:1234",
+          Resource.newInstance(NODE_MEMORY * GB, NODE1_VCORES, NODE_GPU),
+          newMockRM.getResourceTrackerService(),
+          YarnVersionInfo.getVersion());
+
+      nm1.registerNode();
+
+      nm2 = new MockNM("h2:1234",
+          Resource.newInstance(NODE_MEMORY * GB, NODE2_VCORES, NODE_GPU),
+          newMockRM.getResourceTrackerService(),
+          YarnVersionInfo.getVersion());
+      nm2.registerNode();
+
+      nm3 = new MockNM("h3:1234",
+          Resource.newInstance(NODE_MEMORY * GB, NODE3_VCORES, NODE_GPU),
+          newMockRM.getResourceTrackerService(),
+          YarnVersionInfo.getVersion());
+      nm3.registerNode();
+  }
+
+  private void setupGpuResourceValues() {
+    A_MINRES_GPU = Resource.newInstance(A_MINRES.getMemorySize(), A_MINRES.getVirtualCores(),
+        ImmutableMap.of(GPU_URI, 2L));
+    B_MINRES_GPU =  Resource.newInstance(B_MINRES.getMemorySize(), B_MINRES.getVirtualCores(),
+        ImmutableMap.of(GPU_URI, 2L));
+    FULL_MAXRES_GPU = Resource.newInstance(FULL_MAXRES.getMemorySize(),
+        FULL_MAXRES.getVirtualCores(), ImmutableMap.of(GPU_URI, 6L));
+    PARTIAL_MAXRES_GPU = Resource.newInstance(PARTIAL_MAXRES.getMemorySize(),
+        PARTIAL_MAXRES.getVirtualCores(), ImmutableMap.of(GPU_URI, 4L));
+    GPU_EXCEEDED_MAXRES_GPU = Resource.newInstance(PARTIAL_MAXRES.getMemorySize(),
+        PARTIAL_MAXRES.getVirtualCores(), ImmutableMap.of(GPU_URI, 50L));
+  }
+
+  private CapacitySchedulerConfiguration setupCSConfiguration(YarnConfiguration configuration,
+                                                              boolean useDominantRC) {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(configuration);
+    if (useDominantRC) {
+      csConf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+          DominantResourceCalculator.class.getName());
+      csConf.set(YarnConfiguration.RESOURCE_TYPES, ResourceInformation.GPU_URI);
+    }
+
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{PARENT_A, PARENT_B});
+    csConf.setQueues(PARENT_A_FULL_PATH.getFullPath(), new String[]{LEAF_A});
+    csConf.setQueues(PARENT_B_FULL_PATH.getFullPath(), new String[]{LEAF_B});
+
+    if (useDominantRC) {
+      setupGpuResourceValues();
+      csConf.setMinimumResourceRequirement("", PARENT_A_FULL_PATH, A_MINRES_GPU);
+      csConf.setMinimumResourceRequirement("", PARENT_B_FULL_PATH, B_MINRES_GPU);
+      csConf.setMinimumResourceRequirement("", LEAF_A_FULL_PATH, A_MINRES_GPU);
+      csConf.setMinimumResourceRequirement("", LEAF_B_FULL_PATH, B_MINRES_GPU);
+
+      csConf.setMaximumResourceRequirement("", PARENT_A_FULL_PATH, PARTIAL_MAXRES_GPU);
+      csConf.setMaximumResourceRequirement("", PARENT_B_FULL_PATH, FULL_MAXRES_GPU);
+      csConf.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, PARTIAL_MAXRES_GPU);
+      csConf.setMaximumResourceRequirement("", LEAF_B_FULL_PATH, FULL_MAXRES_GPU);
+    } else {
+      csConf.setMinimumResourceRequirement("", PARENT_A_FULL_PATH, A_MINRES);
+      csConf.setMinimumResourceRequirement("", PARENT_B_FULL_PATH, B_MINRES);
+      csConf.setMinimumResourceRequirement("", LEAF_A_FULL_PATH, A_MINRES);
+      csConf.setMinimumResourceRequirement("", LEAF_B_FULL_PATH, B_MINRES);
+
+      csConf.setMaximumResourceRequirement("", PARENT_A_FULL_PATH, PARTIAL_MAXRES);
+      csConf.setMaximumResourceRequirement("", PARENT_B_FULL_PATH, FULL_MAXRES);
+      csConf.setMaximumResourceRequirement("", LEAF_A_FULL_PATH, PARTIAL_MAXRES);
+      csConf.setMaximumResourceRequirement("", LEAF_B_FULL_PATH, FULL_MAXRES);
+    }
+
+    return csConf;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java
new file mode 100644
index 0000000000000..2cbb11fc9bdbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.junit.Test;
+
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestCapacitySchedulerConfiguration {
+
+  private static final String ROOT_TEST = ROOT + ".test";
+  private static final String EMPTY_ACL = "";
+  private static final String SPACE_ACL = " ";
+  private static final String USER1 = "user1";
+  private static final String USER2 = "user2";
+  private static final String GROUP1 = "group1";
+  private static final String GROUP2 = "group2";
+  public static final String ONE_USER_ONE_GROUP_ACL = USER1 + " " + GROUP1;
+  public static final String TWO_USERS_TWO_GROUPS_ACL =
+      USER1 + "," + USER2 + " " + GROUP1 + ", " + GROUP2;
+
+  private CapacitySchedulerConfiguration createDefaultCsConf() {
+    return new CapacitySchedulerConfiguration(new Configuration(false), false);
+  }
+
+  private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, String queue) {
+    return csConf.getAcl(queue, QueueACL.SUBMIT_APPLICATIONS);
+  }
+
+  private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, String queue,
+      String value) {
+    csConf.set(getSubmitAppsConfigKey(queue), value);
+  }
+
+  private String getSubmitAppsConfigKey(String queue) {
+    return getQueuePrefix(queue) + "acl_submit_applications";
+  }
+
+  private void testWithGivenAclNoOneHasAccess(String queue, String aclValue) {
+    testWithGivenAclNoOneHasAccessInternal(queue, queue, aclValue);
+  }
+
+  private void testWithGivenAclNoOneHasAccess(String queueToSet, String queueToVerify,
+      String aclValue) {
+    testWithGivenAclNoOneHasAccessInternal(queueToSet, queueToVerify, aclValue);
+  }
+
+  private void testWithGivenAclNoOneHasAccessInternal(String queueToSet, String queueToVerify,
+      String aclValue) {
+    CapacitySchedulerConfiguration csConf = createDefaultCsConf();
+    setSubmitAppsConfig(csConf, queueToSet, aclValue);
+    AccessControlList acl = getSubmitAcl(csConf, queueToVerify);
+    assertTrue(acl.getUsers().isEmpty());
+    assertTrue(acl.getGroups().isEmpty());
+    assertFalse(acl.isAllAllowed());
+  }
+
+  private void testWithGivenAclCorrectUserAndGroupHasAccess(String queue, String aclValue,
+      Set<String> expectedUsers, Set<String> expectedGroups) {
+    testWithGivenAclCorrectUserAndGroupHasAccessInternal(queue, queue, aclValue, expectedUsers,
+        expectedGroups);
+  }
+
+  private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(String queueToSet,
+      String queueToVerify, String aclValue, Set<String> expectedUsers,
+      Set<String> expectedGroups) {
+    CapacitySchedulerConfiguration csConf = createDefaultCsConf();
+    setSubmitAppsConfig(csConf, queueToSet, aclValue);
+    AccessControlList acl = getSubmitAcl(csConf, queueToVerify);
+    assertFalse(acl.getUsers().isEmpty());
+    assertFalse(acl.getGroups().isEmpty());
+    assertEquals(expectedUsers, acl.getUsers());
+    assertEquals(expectedGroups, acl.getGroups());
+    assertFalse(acl.isAllAllowed());
+  }
+
+  @Test
+  public void testDefaultSubmitACLForRootAllAllowed() {
+    CapacitySchedulerConfiguration csConf = createDefaultCsConf();
+    AccessControlList acl = getSubmitAcl(csConf, ROOT);
+    assertTrue(acl.getUsers().isEmpty());
+    assertTrue(acl.getGroups().isEmpty());
+    assertTrue(acl.isAllAllowed());
+  }
+
+  @Test
+  public void testDefaultSubmitACLForRootChildNoneAllowed() {
+    CapacitySchedulerConfiguration csConf = createDefaultCsConf();
+    AccessControlList acl = getSubmitAcl(csConf, ROOT_TEST);
+    assertTrue(acl.getUsers().isEmpty());
+    assertTrue(acl.getGroups().isEmpty());
+    assertFalse(acl.isAllAllowed());
+  }
+
+  @Test
+  public void testSpecifiedEmptySubmitACLForRoot() {
+    testWithGivenAclNoOneHasAccess(ROOT, EMPTY_ACL);
+  }
+
+  @Test
+  public void testSpecifiedEmptySubmitACLForRootIsNotInherited() {
+    testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, EMPTY_ACL);
+  }
+
+  @Test
+  public void testSpecifiedSpaceSubmitACLForRoot() {
+    testWithGivenAclNoOneHasAccess(ROOT, SPACE_ACL);
+  }
+
+  @Test
+  public void testSpecifiedSpaceSubmitACLForRootIsNotInherited() {
+    testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, SPACE_ACL);
+  }
+
+  @Test
+  public void testSpecifiedSubmitACLForRoot() {
+    Set<String> expectedUsers = Sets.newHashSet(USER1);
+    Set<String> expectedGroups = Sets.newHashSet(GROUP1);
+    testWithGivenAclCorrectUserAndGroupHasAccess(ROOT, ONE_USER_ONE_GROUP_ACL, expectedUsers,
+        expectedGroups);
+  }
+
+  @Test
+  public void testSpecifiedSubmitACLForRootIsNotInherited() {
+    testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, ONE_USER_ONE_GROUP_ACL);
+  }
+
+  @Test
+  public void testSpecifiedSubmitACLTwoUsersTwoGroupsForRoot() {
+    Set<String> expectedUsers = Sets.newHashSet(USER1, USER2);
+    Set<String> expectedGroups = Sets.newHashSet(GROUP1, GROUP2);
+    testWithGivenAclCorrectUserAndGroupHasAccess(ROOT, TWO_USERS_TWO_GROUPS_ACL, expectedUsers,
+        expectedGroups);
+  }
+
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index 4dd537d3a0cb8..7cfd457cb76b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -86,14 +86,14 @@ public void testRefreshQueuesWithReservations() throws Exception {
 
     // Test add one reservation dynamically and manually modify capacity
     ReservationQueue a1 =
-        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+        new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
 
     // Test add another reservation queue and use setEntitlement to modify
     // capacity
     ReservationQueue a2 =
-        new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+        new ReservationQueue(cs.getQueueContext(), "a2", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a2);
     cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f));
 
@@ -116,7 +116,7 @@ public void testAddQueueFailCases() throws Exception {
     try {
       // Test invalid addition (adding non-zero size queue)
       ReservationQueue a1 =
-          new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+          new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a"));
       a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
       cs.addQueue(a1);
       fail();
@@ -126,7 +126,7 @@ public void testAddQueueFailCases() throws Exception {
 
     // Test add one reservation dynamically and manually modify capacity
     ReservationQueue a1 =
-        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+        new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     //set default queue capacity to zero
     ((ReservationQueue) cs
@@ -138,7 +138,7 @@ public void testAddQueueFailCases() throws Exception {
     // Test add another reservation queue and use setEntitlement to modify
     // capacity
     ReservationQueue a2 =
-        new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+        new ReservationQueue(cs.getQueueContext(), "a2", (PlanQueue) cs.getQueue("a"));
 
     cs.addQueue(a2);
 
@@ -165,7 +165,7 @@ public void testRemoveQueue() throws Exception {
 
     // Test add one reservation dynamically and manually modify capacity
     ReservationQueue a1 =
-        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+        new ReservationQueue(cs.getQueueContext(), "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
 
@@ -249,7 +249,7 @@ public void testMoveAppToPlanQueue() throws Exception {
     // create the default reservation queue
     String defQName = "a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
     ReservationQueue defQ =
-        new ReservationQueue(scheduler, defQName,
+        new ReservationQueue(scheduler.getQueueContext(), defQName,
             (PlanQueue) scheduler.getQueue("a"));
     scheduler.addQueue(defQ);
     defQ.setEntitlement(new QueueEntitlement(1f, 1f));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 9b23fdd9033a1..037312b716a05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -318,11 +318,40 @@ public void testAutoCreateQueueWhenSiblingsNotInWeightMode()
     createQueue("root.a.a2-auto");
   }
 
-  @Test(expected = SchedulerDynamicEditException.class)
-  public void testAutoCreateQueueShouldFailIfDepthIsAboveLimit()
+  @Test()
+  public void testAutoCreateMaximumQueueDepth()
       throws Exception {
     startScheduler();
-    createQueue("root.a.a3-auto.a4-auto.a5-auto");
+    // By default, max depth is 2, therefore this is an invalid scenario
+    Assert.assertThrows(SchedulerDynamicEditException.class,
+        () -> createQueue("root.a.a3-auto.a4-auto.a5-auto"));
+
+    // Set depth 3 for root.a, making it a valid scenario
+    csConf.setMaximumAutoCreatedQueueDepth("root.a", 3);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    try {
+      createQueue("root.a.a3-auto.a4-auto.a5-auto");
+    } catch (SchedulerDynamicEditException sde) {
+      LOG.error("%s", sde);
+      Assert.fail("Depth is set for root.a, exception should not be thrown");
+    }
+
+    // Set global depth to 3
+    csConf.setMaximumAutoCreatedQueueDepth(3);
+    csConf.unset(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+        + CapacitySchedulerConfiguration.MAXIMUM_QUEUE_DEPTH);
+    cs.reinitialize(csConf, mockRM.getRMContext());
+    try {
+      createQueue("root.a.a6-auto.a7-auto.a8-auto");
+    } catch (SchedulerDynamicEditException sde) {
+      LOG.error("%s", sde);
+      Assert.fail("Depth is set globally, exception should not be thrown");
+    }
+
+    // Set depth on a dynamic queue, which has no effect on auto queue creation validation
+    csConf.setMaximumAutoCreatedQueueDepth("root.a.a6-auto.a7-auto.a8-auto", 10);
+    Assert.assertThrows(SchedulerDynamicEditException.class,
+        () -> createQueue("root.a.a6-auto.a7-auto.a8-auto.a9-auto.a10-auto.a11-auto"));
   }
 
   @Test(expected = SchedulerDynamicEditException.class)
@@ -685,14 +714,30 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception {
         "root.a.*") + "capacity", "6w");
     cs.reinitialize(csConf, mockRM.getRMContext());
 
-    LeafQueue a2 = createQueue("root.a.a-auto.a2");
+    AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2");
     Assert.assertEquals("weight is not set by template", 6f,
         a2.getQueueCapacities().getWeight(), 1e-6);
+    Assert.assertEquals("user limit factor should be disabled with dynamic queues",
+        -1f, a2.getUserLimitFactor(), 1e-6);
+    Assert.assertEquals("maximum AM resource percent should be 1 with dynamic queues",
+        1f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
+
+    // Set the user-limit-factor and maximum-am-resource-percent via templates to ensure their
+    // modified defaults are indeed overridden
+    csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+        "root.a.*") + "user-limit-factor", "10");
+    csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+        "root.a.*") + "maximum-am-resource-percent", "0.8");
 
     cs.reinitialize(csConf, mockRM.getRMContext());
     a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
     Assert.assertEquals("weight is overridden", 6f,
         a2.getQueueCapacities().getWeight(), 1e-6);
+    Assert.assertEquals("user limit factor should be modified by templates",
+        10f, a2.getUserLimitFactor(), 1e-6);
+    Assert.assertEquals("maximum AM resource percent should be modified by templates",
+        0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
+
 
     csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
     cs.reinitialize(csConf, mockRM.getRMContext());
@@ -703,7 +748,7 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception {
         "root.a") + CapacitySchedulerConfiguration
         .AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false);
     cs.reinitialize(csConf, mockRM.getRMContext());
-    LeafQueue a3 = createQueue("root.a.a3");
+    AbstractLeafQueue a3 = createQueue("root.a.a3");
     Assert.assertFalse("auto queue deletion should be turned off on a3",
         a3.isEligibleForAutoDeletion());
 
@@ -713,27 +758,26 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception {
     csConf.setQueues("root", new String[]{"a", "b", "c"});
     csConf.setAutoQueueCreationV2Enabled("root.c", true);
     cs.reinitialize(csConf, mockRM.getRMContext());
-    LeafQueue c1 = createQueue("root.c.c1");
+    AbstractLeafQueue c1 = createQueue("root.c.c1");
     Assert.assertEquals("weight is not set for label TEST", 6f,
         c1.getQueueCapacities().getWeight("TEST"), 1e-6);
     cs.reinitialize(csConf, mockRM.getRMContext());
-    c1 = (LeafQueue) cs.getQueue("root.c.c1");
+    c1 = (AbstractLeafQueue) cs.getQueue("root.c.c1");
     Assert.assertEquals("weight is not set for label TEST", 6f,
         c1.getQueueCapacities().getWeight("TEST"), 1e-6);
-
   }
 
   @Test
   public void testAutoCreatedQueueConfigChange() throws Exception {
     startScheduler();
-    LeafQueue a2 = createQueue("root.a.a-auto.a2");
+    AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2");
     csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
     cs.reinitialize(csConf, mockRM.getRMContext());
 
     Assert.assertEquals("weight is not explicitly set", 4f,
         a2.getQueueCapacities().getWeight(), 1e-6);
 
-    a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
+    a2 = (AbstractLeafQueue) cs.getQueue("root.a.a-auto.a2");
     csConf.setState("root.a.a-auto.a2", QueueState.STOPPED);
     cs.reinitialize(csConf, mockRM.getRMContext());
     Assert.assertEquals("root.a.a-auto.a2 has not been stopped",
@@ -1207,7 +1251,7 @@ public void testParentQueueDynamicChildRemoval() throws Exception {
     Assert.assertNull("root.e.e1-auto should have been removed", eAuto);
   }
 
-  protected LeafQueue createQueue(String queuePath) throws YarnException,
+  protected AbstractLeafQueue createQueue(String queuePath) throws YarnException,
       IOException {
     return autoQueueHandler.createQueue(new QueuePath(queuePath));
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java
new file mode 100644
index 0000000000000..c557354b6f414
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java
@@ -0,0 +1,387 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
+import org.apache.hadoop.yarn.server.resourcemanager.NodeManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.Task;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.nodeUpdate;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.registerNode;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.NULL_UPDATE_REQUESTS;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestCapacitySchedulerNodes {
+
+  private ResourceManager resourceManager = null;
+
+  @Before
+  public void setUp() throws Exception {
+    resourceManager = createResourceManager();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    stopResourceManager(resourceManager);
+  }
+
+  @Test
+  public void testReconnectedNode() throws Exception {
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(csConf);
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(csConf);
+    cs.start();
+    cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
+        null, null, new RMContainerTokenSecretManager(csConf),
+        new NMTokenSecretManagerInRM(csConf),
+        new ClientToAMTokenSecretManagerInRM(), null));
+
+    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
+    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
+
+    cs.handle(new NodeAddedSchedulerEvent(n1));
+    cs.handle(new NodeAddedSchedulerEvent(n2));
+
+    Assert.assertEquals(6 * GB, cs.getClusterResource().getMemorySize());
+
+    // reconnect n1 with downgraded memory
+    n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
+    cs.handle(new NodeRemovedSchedulerEvent(n1));
+    cs.handle(new NodeAddedSchedulerEvent(n1));
+
+    Assert.assertEquals(4 * GB, cs.getClusterResource().getMemorySize());
+    cs.stop();
+  }
+
+  @Test
+  public void testBlackListNodes() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    String host = "127.0.0.1";
+    RMNode node =
+        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
+    cs.handle(new NodeAddedSchedulerEvent(node));
+
+    ApplicationAttemptId appAttemptId = appHelper(rm, cs, 100, 1, "default", "user");
+
+    // Verify the blacklist can be updated independent of requesting containers
+    cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), null,
+        Collections.<ContainerId>emptyList(),
+        Collections.singletonList(host), null, NULL_UPDATE_REQUESTS);
+    Assert.assertTrue(cs.getApplicationAttempt(appAttemptId)
+        .isPlaceBlacklisted(host));
+    cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), null,
+        Collections.<ContainerId>emptyList(), null,
+        Collections.singletonList(host), NULL_UPDATE_REQUESTS);
+    Assert.assertFalse(cs.getApplicationAttempt(appAttemptId)
+        .isPlaceBlacklisted(host));
+    rm.stop();
+  }
+
+  @Test
+  public void testNumClusterNodes() throws Exception {
+    YarnConfiguration conf = new YarnConfiguration();
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(conf);
+    RMContext rmContext = TestUtils.getMockRMContext();
+    cs.setRMContext(rmContext);
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(csConf);
+    cs.init(csConf);
+    cs.start();
+    assertEquals(0, cs.getNumClusterNodes());
+
+    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
+    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
+    cs.handle(new NodeAddedSchedulerEvent(n1));
+    cs.handle(new NodeAddedSchedulerEvent(n2));
+    assertEquals(2, cs.getNumClusterNodes());
+
+    cs.handle(new NodeRemovedSchedulerEvent(n1));
+    assertEquals(1, cs.getNumClusterNodes());
+    cs.handle(new NodeAddedSchedulerEvent(n1));
+    assertEquals(2, cs.getNumClusterNodes());
+    cs.handle(new NodeRemovedSchedulerEvent(n2));
+    cs.handle(new NodeRemovedSchedulerEvent(n1));
+    assertEquals(0, cs.getNumClusterNodes());
+
+    cs.stop();
+  }
+
+  @Test
+  public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.setDefaultNodeLabelExpression("root.a", " x");
+    conf.setDefaultNodeLabelExpression("root.b", " y ");
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+
+    QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
+    Assert.assertEquals("Queue Name should be a", "a",
+        queueInfoA.getQueueName());
+    Assert.assertEquals("Queue Path should be root.a", "root.a",
+        queueInfoA.getQueuePath());
+    Assert.assertEquals("Default Node Label Expression should be x", "x",
+        queueInfoA.getDefaultNodeLabelExpression());
+
+    QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
+    Assert.assertEquals("Queue Name should be b", "b",
+        queueInfoB.getQueueName());
+    Assert.assertEquals("Queue Path should be root.b", "root.b",
+        queueInfoB.getQueuePath());
+    Assert.assertEquals("Default Node Label Expression should be y", "y",
+        queueInfoB.getDefaultNodeLabelExpression());
+  }
+
+  @Test
+  public void testRemovedNodeDecommissioningNode() throws Exception {
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register nodemanager
+    NodeManager nm = registerNode(resourceManager, "host_decom", 1234, 2345,
+        NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4),
+        mockNodeStatus);
+
+    RMNode node =
+        resourceManager.getRMContext().getRMNodes().get(nm.getNodeId());
+    // Send a heartbeat to kick the tires on the Scheduler
+    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
+    resourceManager.getResourceScheduler().handle(nodeUpdate);
+
+    // force remove the node to simulate race condition
+    ((CapacityScheduler) resourceManager.getResourceScheduler()).getNodeTracker().
+        removeNode(nm.getNodeId());
+    // Kick off another heartbeat with the node state mocked to decommissioning
+    RMNode spyNode =
+        Mockito.spy(resourceManager.getRMContext().getRMNodes()
+            .get(nm.getNodeId()));
+    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
+    resourceManager.getResourceScheduler().handle(
+        new NodeUpdateSchedulerEvent(spyNode));
+  }
+
+  @Test
+  public void testResourceUpdateDecommissioningNode() throws Exception {
+    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
+    // to have 0 available resource
+    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
+    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
+    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {
+      @Override
+      public void handle(Event event) {
+        if (event instanceof RMNodeResourceUpdateEvent) {
+          RMNodeResourceUpdateEvent resourceEvent =
+              (RMNodeResourceUpdateEvent) event;
+          resourceManager
+              .getResourceScheduler()
+              .getSchedulerNode(resourceEvent.getNodeId())
+              .updateTotalResource(resourceEvent.getResourceOption().getResource());
+        }
+      }
+    });
+    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
+    ((CapacityScheduler) resourceManager.getResourceScheduler())
+        .setRMContext(spyContext);
+    ((AsyncDispatcher) mockDispatcher).start();
+
+    NodeStatus mockNodeStatus = createMockNodeStatus();
+
+    // Register node
+    String host0 = "host_0";
+    NodeManager nm0 = registerNode(resourceManager, host0, 1234, 2345,
+        NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4),
+        mockNodeStatus);
+    // ResourceRequest priorities
+    Priority priority0 = Priority.newInstance(0);
+
+    // Submit an application
+    Application application0 =
+        new Application("user_0", "a1", resourceManager);
+    application0.submit();
+
+    application0.addNodeManager(host0, 1234, nm0);
+
+    Resource capability00 = Resources.createResource(1 * GB, 1);
+    application0.addResourceRequestSpec(priority0, capability00);
+
+    Task task00 =
+        new Task(application0, priority0, new String[]{host0});
+    application0.addTask(task00);
+
+    // Send resource requests to the scheduler
+    application0.schedule();
+
+    nodeUpdate(resourceManager, nm0);
+    // Kick off another heartbeat with the node state mocked to decommissioning
+    // This should update the schedulernodes to have 0 available resource
+    RMNode spyNode =
+        Mockito.spy(resourceManager.getRMContext().getRMNodes()
+            .get(nm0.getNodeId()));
+    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
+    resourceManager.getResourceScheduler().handle(
+        new NodeUpdateSchedulerEvent(spyNode));
+
+    // Get allocations from the scheduler
+    application0.schedule();
+
+    // Check the used resource is 1 GB 1 core
+    Assert.assertEquals(1 * GB, nm0.getUsed().getMemorySize());
+    Resource usedResource =
+        resourceManager.getResourceScheduler()
+            .getSchedulerNode(nm0.getNodeId()).getAllocatedResource();
+    Assert.assertEquals("Used Resource Memory Size should be 1GB", 1 * GB,
+        usedResource.getMemorySize());
+    Assert.assertEquals("Used Resource Virtual Cores should be 1", 1,
+        usedResource.getVirtualCores());
+    // Check total resource of scheduler node is also changed to 1 GB 1 core
+    Resource totalResource =
+        resourceManager.getResourceScheduler()
+            .getSchedulerNode(nm0.getNodeId()).getTotalResource();
+    Assert.assertEquals("Total Resource Memory Size should be 1GB", 1 * GB,
+        totalResource.getMemorySize());
+    Assert.assertEquals("Total Resource Virtual Cores should be 1", 1,
+        totalResource.getVirtualCores());
+    // Check the available resource is 0/0
+    Resource availableResource =
+        resourceManager.getResourceScheduler()
+            .getSchedulerNode(nm0.getNodeId()).getUnallocatedResource();
+    Assert.assertEquals("Available Resource Memory Size should be 0", 0,
+        availableResource.getMemorySize());
+    Assert.assertEquals("Available Resource Memory Size should be 0", 0,
+        availableResource.getVirtualCores());
+    // Kick off another heartbeat where the RMNodeResourceUpdateEvent would
+    // be skipped for DECOMMISSIONING state since the total resource is
+    // already equal to used resource from the previous heartbeat.
+    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
+    resourceManager.getResourceScheduler().handle(
+        new NodeUpdateSchedulerEvent(spyNode));
+    verify(mockDispatcher, times(4)).getEventHandler();
+  }
+
+  @Test
+  public void testSchedulingOnRemovedNode() throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE,
+        false);
+
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    RMApp app = MockRMAppSubmitter.submitWithMemory(100, rm);
+    rm.drainEvents();
+
+    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10240, 10);
+    MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+
+    //remove nm2 to keep am alive
+    MockNM nm2 = rm.registerNode("127.0.0.1:1235", 10240, 10);
+
+    am.allocate(ResourceRequest.ANY, 2048, 1, null);
+
+    CapacityScheduler scheduler =
+        (CapacityScheduler) rm.getRMContext().getScheduler();
+    FiCaSchedulerNode node =
+        (FiCaSchedulerNode)
+            scheduler.getNodeTracker().getNode(nm2.getNodeId());
+    scheduler.handle(new NodeRemovedSchedulerEvent(
+        rm.getRMContext().getRMNodes().get(nm2.getNodeId())));
+    // schedulerNode is removed, try allocate a container
+    scheduler.allocateContainersToNode(new SimpleCandidateNodeSet<>(node),
+        true);
+
+    AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
+        new AppAttemptRemovedSchedulerEvent(
+            am.getApplicationAttemptId(),
+            RMAppAttemptState.FINISHED, false);
+    scheduler.handle(appRemovedEvent1);
+    rm.stop();
+  }
+
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
index b71fe063927ac..b8209a54952e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
@@ -237,7 +237,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes(
 
     if (numThreads > 0) {
       // disable async scheduling threads
-      for (CapacityScheduler.AsyncScheduleThread t : cs.asyncSchedulerThreads) {
+      for (CapacityScheduler.AsyncScheduleThread t : cs.getAsyncSchedulerThreads()) {
         t.suspendSchedule();
       }
     }
@@ -268,7 +268,7 @@ private void testUserLimitThroughputWithNumberOfResourceTypes(
 
     if (numThreads > 0) {
       // enable async scheduling threads
-      for (CapacityScheduler.AsyncScheduleThread t : cs.asyncSchedulerThreads) {
+      for (CapacityScheduler.AsyncScheduleThread t : cs.getAsyncSchedulerThreads()) {
         t.beginSchedule();
       }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java
new file mode 100644
index 0000000000000..fc1870097e9a4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java
@@ -0,0 +1,888 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocMb;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocation;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.unsetMaxAllocation;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.when;
+
+public class TestCapacitySchedulerQueues {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCapacitySchedulerQueues.class);
+  private ResourceManager resourceManager = null;
+  private RMContext mockContext;
+
+  @Before
+  public void setUp() throws Exception {
+    resourceManager = createResourceManager();
+    mockContext = createMockRMContext();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    stopResourceManager(resourceManager);
+  }
+
+  /**
+   * Test that parseQueue throws an exception when two leaf queues have the
+   * same name.
+   *
+   * @throws IOException
+   */
+  @Test(expected = IOException.class)
+  public void testParseQueue() throws IOException {
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.init(conf);
+    cs.start();
+
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[]{"b1"});
+    conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
+    conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
+
+    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null));
+  }
+
+  @Test
+  public void testRefreshQueues() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
+    checkQueueStructureCapacities(cs);
+
+    conf.setCapacity(A, 80f);
+    conf.setCapacity(B, 20f);
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs, getDefaultCapacities(80f / 100.0f, 20f / 100.0f));
+    cs.stop();
+  }
+
+  @Test
+  public void testRefreshQueuesWithNewQueue() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null));
+    checkQueueStructureCapacities(cs);
+
+    // Add a new queue b4
+    final String b4 = B + ".b4";
+    final float b4Capacity = 10;
+    final float modifiedB3Capacity = B3_CAPACITY - b4Capacity;
+
+    try {
+      conf.setCapacity(A, 80f);
+      conf.setCapacity(B, 20f);
+      conf.setQueues(B, new String[]{"b1", "b2", "b3", "b4"});
+      conf.setCapacity(B1, B1_CAPACITY);
+      conf.setCapacity(B2, B2_CAPACITY);
+      conf.setCapacity(B3, modifiedB3Capacity);
+      conf.setCapacity(b4, b4Capacity);
+      cs.reinitialize(conf, mockContext);
+
+      final float capA = 80f / 100.0f;
+      final float capB = 20f / 100.0f;
+      Map<String, ExpectedCapacities> expectedCapacities =
+          getDefaultCapacities(capA, capB);
+      expectedCapacities.put(B3,
+          new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB));
+      expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB));
+      checkQueueStructureCapacities(cs, expectedCapacities);
+
+      // Verify parent for B4
+      CSQueue rootQueue = cs.getRootQueue();
+      CSQueue queueB = findQueue(rootQueue, B);
+      CSQueue queueB4 = findQueue(queueB, b4);
+
+      assertEquals(queueB, queueB4.getParent());
+    } finally {
+      cs.stop();
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationRefresh() throws Exception {
+    // queue refresh should not allow changing the maximum allocation setting
+    // per queue to be smaller than previous setting
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    assertEquals("max allocation in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max allocation for A1",
+        Resources.none(),
+        conf.getQueueMaximumAllocation(A1));
+    assertEquals("max allocation",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    assertEquals("queue max allocation", ((LeafQueue) queueA1)
+        .getMaximumAllocation().getMemorySize(), 8192);
+
+    setMaxAllocMb(conf, A1, 4096);
+
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+
+    setMaxAllocMb(conf, A1, 8192);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, A1,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception {
+    // verify we can't set the allocation per queue larger then cluster setting
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.init(conf);
+    cs.start();
+    // change max allocation for B3 queue to be larger then cluster max
+    setMaxAllocMb(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+
+    setMaxAllocMb(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, B3,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception {
+    // queue refresh should allow max allocation per queue to go larger
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    setMaxAllocVcores(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 2);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueA1 = findQueue(queueA, A1);
+
+    assertEquals("max capability MB in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max capability vcores in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("max allocation MB A1",
+        4096,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A1",
+        2,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("cluster max allocation MB",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
+    assertEquals("cluster max allocation vcores",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores());
+
+    assertEquals("queue max allocation", 4096,
+        queueA1.getMaximumAllocation().getMemorySize());
+
+    setMaxAllocMb(conf, A1, 6144);
+    setMaxAllocVcores(conf, A1, 3);
+    cs.reinitialize(conf, null);
+    // conf will have changed but we shouldn't be able to change max allocation
+    // for the actual queue
+    assertEquals("max allocation MB A1", 6144,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A1", 3,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("max allocation MB cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
+    assertEquals("max allocation vcores cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        ResourceUtils.fetchMaximumAllocationFromConfig(conf).getVirtualCores());
+    assertEquals("queue max allocation MB", 6144,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("queue max allocation vcores", 3,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("max capability MB cluster",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("cluster max capability vcores",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationCSError() throws Exception {
+    // Try to refresh the cluster level max allocation size to be smaller
+    // and it should error out
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf, 10240);
+    setMaxAllocVcores(conf, 10);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 4);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    assertEquals("max allocation MB in CS", 10240,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max allocation vcores in CS", 10,
+        cs.getMaximumResourceCapability().getVirtualCores());
+
+    setMaxAllocMb(conf, 6144);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+
+    setMaxAllocMb(conf, 10240);
+    cs.reinitialize(conf, mockContext);
+
+    setMaxAllocVcores(conf, 8);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("should have thrown exception");
+    } catch (IOException e) {
+      assertTrue("max allocation exception",
+          e.getCause().toString().contains("not be decreased"));
+    }
+  }
+
+  @Test
+  public void testRefreshQueuesMaxAllocationCSLarger() throws Exception {
+    // Try to refresh the cluster level max allocation size to be larger
+    // and verify that if there is no setting per queue it uses the
+    // cluster level setting.
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf, 10240);
+    setMaxAllocVcores(conf, 10);
+    setMaxAllocMb(conf, A1, 4096);
+    setMaxAllocVcores(conf, A1, 4);
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    assertEquals("max allocation MB in CS", 10240,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max allocation vcores in CS", 10,
+        cs.getMaximumResourceCapability().getVirtualCores());
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    CSQueue queueA2 = findQueue(queueA, A2);
+    CSQueue queueB2 = findQueue(queueB, B2);
+
+    assertEquals("queue A1 max allocation MB", 4096,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("queue A1 max allocation vcores", 4,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("queue A2 max allocation MB", 10240,
+        queueA2.getMaximumAllocation().getMemorySize());
+    assertEquals("queue A2 max allocation vcores", 10,
+        queueA2.getMaximumAllocation().getVirtualCores());
+    assertEquals("queue B2 max allocation MB", 10240,
+        queueB2.getMaximumAllocation().getMemorySize());
+    assertEquals("queue B2 max allocation vcores", 10,
+        queueB2.getMaximumAllocation().getVirtualCores());
+
+    setMaxAllocMb(conf, 12288);
+    setMaxAllocVcores(conf, 12);
+    cs.reinitialize(conf, null);
+    // cluster level setting should change and any queues without
+    // per queue setting
+    assertEquals("max allocation MB in CS", 12288,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max allocation vcores in CS", 12,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("queue A1 max MB allocation", 4096,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("queue A1 max vcores allocation", 4,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("queue A2 max MB allocation", 12288,
+        queueA2.getMaximumAllocation().getMemorySize());
+    assertEquals("queue A2 max vcores allocation", 12,
+        queueA2.getMaximumAllocation().getVirtualCores());
+    assertEquals("queue B2 max MB allocation", 12288,
+        queueB2.getMaximumAllocation().getMemorySize());
+    assertEquals("queue B2 max vcores allocation", 12,
+        queueB2.getMaximumAllocation().getVirtualCores());
+  }
+
+  /**
+   * Test for queue deletion.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testRefreshQueuesWithQueueDelete() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
+    checkQueueStructureCapacities(cs);
+
+    // test delete leaf queue when there is application running.
+    Map<String, CSQueue> queues =
+        cs.getCapacitySchedulerQueueManager().getShortNameQueues();
+    String b1QTobeDeleted = "b1";
+    LeafQueue csB1Queue = Mockito.spy((LeafQueue) queues.get(b1QTobeDeleted));
+    when(csB1Queue.getState()).thenReturn(QueueState.DRAINING)
+        .thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue(b1QTobeDeleted, csB1Queue);
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithoutB1(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Expected to throw exception when refresh queue tries to delete a"
+          + " queue with running apps");
+    } catch (IOException e) {
+      // ignore
+    }
+
+    // test delete leaf queue(root.b.b1) when there is no application running.
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithoutB1(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+    } catch (IOException e) {
+      LOG.error(
+          "Expected to NOT throw exception when refresh queue tries to delete"
+              + " a queue WITHOUT running apps",
+          e);
+      fail("Expected to NOT throw exception when refresh queue tries to delete"
+          + " a queue WITHOUT running apps");
+    }
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueB3 = findQueue(queueB, B1);
+    assertNull("Refresh needs to support delete of leaf queue ", queueB3);
+
+    // reset back to default configuration for testing parent queue delete
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.reinitialize(conf, rmContext);
+    checkQueueStructureCapacities(cs);
+
+    // set the configurations such that it fails once but should be successfull
+    // next time
+    queues = cs.getCapacitySchedulerQueueManager().getShortNameQueues();
+    CSQueue bQueue = Mockito.spy((ParentQueue) queues.get("b"));
+    when(bQueue.getState()).thenReturn(QueueState.DRAINING)
+        .thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b", bQueue);
+
+    bQueue = Mockito.spy((LeafQueue) queues.get("b1"));
+    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue);
+
+    bQueue = Mockito.spy((LeafQueue) queues.get("b2"));
+    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue);
+
+    bQueue = Mockito.spy((LeafQueue) queues.get("b3"));
+    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue);
+
+    // test delete Parent queue when there is application running.
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithoutB(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Expected to throw exception when refresh queue tries to delete a"
+          + " parent queue with running apps in children queue");
+    } catch (IOException e) {
+      // ignore
+    }
+
+    // test delete Parent queue when there is no application running.
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithoutB(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+    } catch (IOException e) {
+      fail("Expected to not throw exception when refresh queue tries to delete"
+          + " a queue without running apps");
+    }
+    rootQueue = cs.getRootQueue();
+    queueB = findQueue(rootQueue, B);
+    String message =
+        "Refresh needs to support delete of Parent queue and its children.";
+    assertNull(message, queueB);
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b"));
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b1"));
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b2"));
+
+    cs.stop();
+  }
+
+  /**
+   * Test for all child queue deletion and thus making parent queue a child.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testRefreshQueuesWithAllChildQueuesDeleted() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
+    checkQueueStructureCapacities(cs);
+
+    // test delete all leaf queues when there is no application running.
+    Map<String, CSQueue> queues =
+        cs.getCapacitySchedulerQueueManager().getShortNameQueues();
+
+    CSQueue bQueue = Mockito.spy((LeafQueue) queues.get("b1"));
+    when(bQueue.getState()).thenReturn(QueueState.RUNNING)
+        .thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b1", bQueue);
+
+    bQueue = Mockito.spy((LeafQueue) queues.get("b2"));
+    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b2", bQueue);
+
+    bQueue = Mockito.spy((LeafQueue) queues.get("b3"));
+    when(bQueue.getState()).thenReturn(QueueState.STOPPED);
+    cs.getCapacitySchedulerQueueManager().addQueue("b3", bQueue);
+
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfWithoutChildrenOfB(conf);
+
+    // test convert parent queue to leaf queue(root.b) when there is no
+    // application running.
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Expected to throw exception when refresh queue tries to make parent"
+          + " queue a child queue when one of its children is still running.");
+    } catch (IOException e) {
+      //do not do anything, expected exception
+    }
+
+    // test delete leaf queues(root.b.b1,b2,b3) when there is no application
+    // running.
+    try {
+      cs.reinitialize(conf, mockContext);
+    } catch (IOException e) {
+      e.printStackTrace();
+      fail("Expected to NOT throw exception when refresh queue tries to delete"
+          + " all children of a parent queue(without running apps).");
+    }
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueB = findQueue(rootQueue, B);
+    assertNotNull("Parent Queue B should not be deleted", queueB);
+    Assert.assertTrue("As Queue'B children are not deleted",
+        queueB instanceof LeafQueue);
+
+    String message =
+        "Refresh needs to support delete of all children of Parent queue.";
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b3"));
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b1"));
+    assertNull(message,
+        cs.getCapacitySchedulerQueueManager().getQueues().get("b2"));
+
+    cs.stop();
+  }
+
+  /**
+   * Test if we can convert a leaf queue to a parent queue.
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 10000)
+  public void testConvertLeafQueueToParentQueue() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null);
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, rmContext);
+    checkQueueStructureCapacities(cs);
+
+    String targetQueue = "b1";
+    CSQueue b1 = cs.getQueue(targetQueue);
+    Assert.assertEquals(QueueState.RUNNING, b1.getState());
+
+    // test if we can convert a leaf queue which is in RUNNING state
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithB1AsParentQueue(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Expected to throw exception when refresh queue tries to convert"
+          + " a child queue to a parent queue.");
+    } catch (IOException e) {
+      // ignore
+    }
+
+    // now set queue state for b1 to STOPPED
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.set("yarn.scheduler.capacity.root.b.b1.state", "STOPPED");
+    cs.reinitialize(conf, mockContext);
+    Assert.assertEquals(QueueState.STOPPED, b1.getState());
+
+    // test if we can convert a leaf queue which is in STOPPED state
+    conf = new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithB1AsParentQueue(conf);
+    try {
+      cs.reinitialize(conf, mockContext);
+    } catch (IOException e) {
+      fail("Expected to NOT throw exception when refresh queue tries"
+          + " to convert a leaf queue WITHOUT running apps");
+    }
+    b1 = cs.getQueue(targetQueue);
+    Assert.assertTrue(b1 instanceof ParentQueue);
+    Assert.assertEquals(QueueState.RUNNING, b1.getState());
+    Assert.assertTrue(!b1.getChildQueues().isEmpty());
+  }
+
+  @Test
+  public void testQueuesMaxAllocationInheritance() throws Exception {
+    // queue level max allocation is set by the queue configuration explicitly
+    // or inherits from the parent.
+
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    setMaxAllocVcores(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+
+    // Test the child queue overrides
+    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+        "memory-mb=4096,vcores=2");
+    setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2");
+    setMaxAllocation(conf, B, "memory-mb=5120, vcores=2");
+    setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2");
+
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    CSQueue rootQueue = cs.getRootQueue();
+    CSQueue queueA = findQueue(rootQueue, A);
+    CSQueue queueB = findQueue(rootQueue, B);
+    CSQueue queueA1 = findQueue(queueA, A1);
+    CSQueue queueA2 = findQueue(queueA, A2);
+    CSQueue queueB1 = findQueue(queueB, B1);
+    CSQueue queueB2 = findQueue(queueB, B2);
+
+    assertEquals("max capability MB in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max capability vcores in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("max allocation MB A1",
+        6144,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A1",
+        2,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("max allocation MB A2", 4096,
+        queueA2.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A2",
+        2,
+        queueA2.getMaximumAllocation().getVirtualCores());
+    assertEquals("max allocation MB B", 5120,
+        queueB.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation MB B1", 5120,
+        queueB1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation MB B2", 1024,
+        queueB2.getMaximumAllocation().getMemorySize());
+
+    // Test get the max-allocation from different parent
+    unsetMaxAllocation(conf, A1);
+    unsetMaxAllocation(conf, B);
+    unsetMaxAllocation(conf, B1);
+    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+        "memory-mb=6144,vcores=2");
+    setMaxAllocation(conf, A, "memory-mb=8192,vcores=2");
+
+    cs.reinitialize(conf, mockContext);
+
+    assertEquals("max capability MB in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max capability vcores in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("max allocation MB A1",
+        8192,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A1",
+        2,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("max allocation MB B1",
+        6144,
+        queueB1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores B1",
+        2,
+        queueB1.getMaximumAllocation().getVirtualCores());
+
+    // Test the default
+    unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT);
+    unsetMaxAllocation(conf, A);
+    unsetMaxAllocation(conf, A1);
+    cs.reinitialize(conf, mockContext);
+
+    assertEquals("max capability MB in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        cs.getMaximumResourceCapability().getMemorySize());
+    assertEquals("max capability vcores in CS",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        cs.getMaximumResourceCapability().getVirtualCores());
+    assertEquals("max allocation MB A1",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        queueA1.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A1",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        queueA1.getMaximumAllocation().getVirtualCores());
+    assertEquals("max allocation MB A2",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+        queueA2.getMaximumAllocation().getMemorySize());
+    assertEquals("max allocation vcores A2",
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+        queueA2.getMaximumAllocation().getVirtualCores());
+  }
+
+  @Test
+  public void testVerifyQueuesMaxAllocationConf() throws Exception {
+    // queue level max allocation can't exceed the cluster setting
+
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    setMaxAllocMb(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    setMaxAllocVcores(conf,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+
+    long largerMem =
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1024;
+    long largerVcores =
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 10;
+
+    cs.init(conf);
+    cs.start();
+    cs.reinitialize(conf, mockContext);
+    checkQueueStructureCapacities(cs);
+
+    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+        "memory-mb=" + largerMem + ",vcores=2");
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Queue Root maximum allocation can't exceed the cluster setting");
+    } catch (Exception e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+
+    setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+        "memory-mb=4096,vcores=2");
+    setMaxAllocation(conf, A, "memory-mb=6144,vcores=2");
+    setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2");
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Queue A1 maximum allocation can't exceed the cluster setting");
+    } catch (Exception e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+    setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores);
+    try {
+      cs.reinitialize(conf, mockContext);
+      fail("Queue A1 maximum allocation can't exceed the cluster setting");
+    } catch (Exception e) {
+      assertTrue("maximum allocation exception",
+          e.getCause().getMessage().contains("maximum allocation"));
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
index a5d80dc27e5c1..115813a01717f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -38,6 +39,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -74,6 +76,28 @@ public static <E> Set<E> toSet(E... elements) {
     return set;
   }
 
+  public static CapacitySchedulerConfiguration getConfigWithInheritedAccessibleNodeLabel(
+      Configuration config) {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
+        config);
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] { "a"});
+
+    conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f);
+    conf.setCapacityByLabel(A, "newLabel", 100f);
+    conf.setAccessibleNodeLabels(A, toSet("newLabel"));
+    conf.setAllowZeroCapacitySum(A, true);
+
+    // Define 2nd-level queues
+    conf.setQueues(A, new String[] { "a1" });
+    conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100f);
+
+    return conf;
+  }
+
+
   /*
    * Queue structure:
    *                      root (*)
@@ -368,6 +392,28 @@ public void testChildAccessibleNodeLabelsWeightMode() throws Exception {
     rm.close();
   }
 
+  /**
+   * Tests whether weight is correctly reset to -1. See YARN-11016 for further details.
+   * @throws IOException if reinitialization fails
+   */
+  @Test()
+  public void testAccessibleNodeLabelsInheritanceNoWeightMode() throws IOException {
+    CapacitySchedulerConfiguration newConf = getConfigWithInheritedAccessibleNodeLabel(conf);
+
+    MockRM rm = new MockRM(newConf);
+    CapacityScheduler cs =
+        (CapacityScheduler) rm.getRMContext().getScheduler();
+
+    Resource clusterResource = Resource.newInstance(1024, 2);
+    cs.getRootQueue().updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
+
+    try {
+      cs.reinitialize(newConf, rm.getRMContext());
+    } catch (Exception e) {
+      Assert.fail("Reinitialization failed with " + e);
+    }
+  }
+
   @Test
   public void testQueueInfoWeight() throws Exception {
     MockRM rm = new MockRM(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 31ad107de94c3..1af3563c52729 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -74,6 +74,7 @@ public class TestChildQueueOrder {
   YarnConfiguration conf;
   CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
+  CapacitySchedulerQueueContext queueContext;
 
   final static int GB = 1024;
   final static String DEFAULT_RACK = "/default";
@@ -100,6 +101,10 @@ public void setUp() throws Exception {
         thenReturn(resourceComparator);
     when(csContext.getRMContext()).thenReturn(rmContext);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(
+        new CapacitySchedulerQueueManager(csConf, rmContext.getNodeLabelManager(), null));
+
+    queueContext = new CapacitySchedulerQueueContext(csContext);
   }
 
   private FiCaSchedulerApp getMockApplication(int appId, String user) {
@@ -219,9 +224,10 @@ private void setupSortedQueues(CapacitySchedulerConfiguration conf) {
   public void testSortedQueues() throws Exception {
     // Setup queue configs
     setupSortedQueues(csConf);
+    queueContext.reinitialize();
     CSQueueStore queues = new CSQueueStore();
     CSQueue root =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
           CapacitySchedulerConfiguration.ROOT, queues, queues, 
           TestUtils.spyHook);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 351fdbe3a53e7..eca065b148766 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -57,12 +57,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
@@ -74,9 +77,13 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -94,8 +101,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
@@ -134,6 +143,7 @@ public class TestLeafQueue {
   CapacityScheduler cs;
   CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
+  CapacitySchedulerQueueContext queueContext;
   private RMApp rmApp;
   
   CSQueue root;
@@ -194,6 +204,7 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels)
     csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
     csConf.setBoolean(CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES,
         false);
+    csConf.setResourceComparator(rC.getClass());
     final String newRoot = "root" + System.currentTimeMillis();
     setupQueueConfiguration(csConf, newRoot, withNodeLabels);
     YarnConfiguration conf = new YarnConfiguration();
@@ -219,12 +230,20 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels)
     containerTokenSecretManager.rollMasterKey();
     when(csContext.getContainerTokenSecretManager()).thenReturn(
         containerTokenSecretManager);
+    CapacitySchedulerQueueManager queueManager =
+        new CapacitySchedulerQueueManager(csConf, null, null);
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
+
+    queueManager.reinitConfiguredNodeLabels(csConf);
+
+    queueContext = new CapacitySchedulerQueueContext(csContext);
 
     root = 
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             ROOT,
             queues, queues, 
             TestUtils.spyHook);
+    queueManager.setRootQueue(root);
     root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
         new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32)));
 
@@ -233,8 +252,8 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels)
         .thenReturn(queueResUsage);
 
     cs.setRMContext(spyRMContext);
-    cs.init(csConf);
     cs.setResourceCalculator(rC);
+    cs.init(csConf);
 
     when(spyRMContext.getScheduler()).thenReturn(cs);
     when(spyRMContext.getYarnConfiguration())
@@ -917,7 +936,7 @@ private void applyCSAssignment(Resource clusterResource, CSAssignment assign,
       LeafQueue q, final Map<NodeId, FiCaSchedulerNode> nodes,
       final Map<ApplicationAttemptId, FiCaSchedulerApp> apps)
       throws IOException {
-    TestUtils.applyResourceCommitRequest(clusterResource, assign, nodes, apps);
+    TestUtils.applyResourceCommitRequest(clusterResource, assign, nodes, apps, csConf);
   }
 
   @Test
@@ -1078,11 +1097,12 @@ public void testUserLimitCache() throws Exception {
     csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100);
     csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A,
         100);
+    queueContext.reinitialize();
 
     // reinitialize queues
     CSQueueStore newQueues = new CSQueueStore();
     CSQueue newRoot =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT,
             newQueues, queues,
             TestUtils.spyHook);
@@ -1296,11 +1316,12 @@ public void testUserLimitCacheActiveUsersChanged() throws Exception {
     csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100);
     csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A,
         100);
+    queueContext.reinitialize();
 
     // reinitialize queues
     CSQueueStore newQueues = new CSQueueStore();
     CSQueue newRoot =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT,
             newQueues, queues,
             TestUtils.spyHook);
@@ -1911,6 +1932,7 @@ public void testUserSpecificUserLimits() throws Exception {
         + CapacitySchedulerConfiguration.USER_WEIGHT,
         0.7f);
     csConf.reinitializeConfigurationProperties();
+    queueContext.reinitialize();
 
     when(csContext.getClusterResource())
         .thenReturn(Resources.createResource(16 * GB, 32));
@@ -3207,10 +3229,12 @@ public void testRackLocalityDelayScheduling() throws Exception {
     csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 2);
     csConf.setInt(
         CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1);
+    queueContext.reinitialize();
     CSQueueStore newQueues = new CSQueueStore();
-    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext,
         csConf, null, ROOT, newQueues, queues,
         TestUtils.spyHook);
+    csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot);
     root.reinitialize(newRoot, cs.getClusterResource());
 
     // Manipulate queue 'b'
@@ -3643,9 +3667,10 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception {
         CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
         CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
             * 2);
+    queueContext.reinitialize();
     CSQueueStore newQueues = new CSQueueStore();
     CSQueue newRoot =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             ROOT,
             newQueues, queues,
             TestUtils.spyHook);
@@ -3674,12 +3699,14 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception {
     csConf.setInt(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, 60);
     csConf.setInt(
         CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 600);
+    queueContext.reinitialize();
     CSQueueStore newQueues = new CSQueueStore();
     CSQueue newRoot =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             ROOT,
             newQueues, queues,
             TestUtils.spyHook);
+    csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot);
     root.reinitialize(newRoot, cs.getClusterResource());
 
     // after reinitialization
@@ -4034,8 +4061,14 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
         CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
         0.1f);
 
+    CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(csConf,
+        rmContext.getNodeLabelManager(), null);
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
+
+    CapacitySchedulerQueueContext newQueueContext = new CapacitySchedulerQueueContext(csContext);
+
     CSQueue root;
-    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    root = CapacitySchedulerQueueManager.parseQueue(newQueueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
@@ -4051,9 +4084,10 @@ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
     csConf.setFloat(
         CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
         0.2f);
+    newQueueContext.reinitialize();
     clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32);
     CSQueueStore newQueues = new CSQueueStore();
-    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(newQueueContext,
         csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
         TestUtils.spyHook);
     root.reinitialize(newRoot, clusterResource);
@@ -5103,15 +5137,15 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration()
 
       assertEquals(0, conf.size());
       conf.setNodeLocalityDelay(60);
-      conf.setCapacity(ROOT + DOT + leafQueueName, 10);
-      conf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100);
-      conf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f);
+      csConf.setCapacity(ROOT + DOT + leafQueueName, 10);
+      csConf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100);
+      csConf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f);
 
       csConf.setNodeLocalityDelay(30);
       csConf.setGlobalMaximumApplicationsPerQueue(20);
+      queueContext.reinitialize();
 
-      LeafQueue leafQueue = new LeafQueue(csContext, conf,
-          leafQueueName, cs.getRootQueue(),
+      LeafQueue leafQueue = new LeafQueue(queueContext, leafQueueName, cs.getRootQueue(),
           null);
 
       leafQueue.updateClusterResource(Resource.newInstance(0, 0),
@@ -5137,6 +5171,14 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration()
       assertEquals(1.0, leafQueue.getAbsoluteMaximumCapacity(),
           EPSILON);
 
+      // limit maximum apps by max system apps
+      csConf.setMaximumSystemApplications(15);
+      queueContext.reinitialize();
+      leafQueue.updateClusterResource(Resource.newInstance(0, 0),
+          new ResourceLimits(Resource.newInstance(0, 0)));
+
+      assertEquals(15, leafQueue.getMaxApplications());
+
      } finally {
         //revert config changes
         csConf.setNodeLocalityDelay(
@@ -5245,7 +5287,7 @@ public void testRootHasAllNodeLabelsOfItsDescendants() throws IOException {
     ParentQueue rootQueue = (ParentQueue) cs.getRootQueue();
 
     Assert.assertEquals(Sets.newHashSet("", "test", "test2"),
-        rootQueue.configuredNodeLabels);
+        rootQueue.queueNodeLabelsSettings.getConfiguredNodeLabels());
   }
 
   @After
@@ -5254,4 +5296,139 @@ public void tearDown() throws Exception {
       cs.stop();
     }
   }
+
+  private static class TestRMAppManager extends RMAppManager {
+    TestRMAppManager(RMContext context, YarnScheduler scheduler,
+        ApplicationMasterService masterService,
+        ApplicationACLsManager applicationACLsManager, Configuration conf) {
+      super(context, scheduler, masterService, applicationACLsManager, conf);
+    }
+
+    @Override
+    public void submitApplication(
+        ApplicationSubmissionContext submissionContext, long submitTime,
+        UserGroupInformation userUgi) throws YarnException {
+      super.submitApplication(submissionContext, submitTime, userUgi);
+    }
+  }
+
+  @Test
+  public void testSubmitUsingRealUserAcls() throws Exception {
+    final String realUser = "AdminUser";
+    final String user0 = "user0";
+    final String user1 = "user1";
+    final String queue = "default";
+
+    YarnConfiguration conf = new YarnConfiguration();
+    MockRM rm = new MockRM();
+    rm.init(conf);
+    rm.start();
+    rm.getConfig().setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+
+    UserGroupInformation realUserUgi =
+        UserGroupInformation.createRemoteUser(realUser);
+    UserGroupInformation ugi0 =
+        UserGroupInformation.createProxyUserForTesting("user0", realUserUgi,
+            new String[] {"group1"});
+    UserGroupInformation ugi1 =
+        UserGroupInformation.createProxyUserForTesting("user1", realUserUgi,
+            new String[] {"group1"});
+    ApplicationId applicationId0 = TestUtils.getMockApplicationId(0);
+    ApplicationId applicationId1 = TestUtils.getMockApplicationId(1);
+    CapacityScheduler cSched = (CapacityScheduler) rm.getResourceScheduler();
+
+    ParentQueue rootQueue = (ParentQueue) cSched.getRootQueue();
+    Map<AccessType, AccessControlList> rootAcls = rootQueue.acls;
+    rootAcls.put(AccessType.SUBMIT_APP, new AccessControlList(realUser));
+    rootAcls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+
+    LeafQueue defaultQueue = (LeafQueue)cSched.getQueue(queue);
+    Map<AccessType, AccessControlList> a = defaultQueue.acls;
+    a.put(AccessType.SUBMIT_APP, new AccessControlList(realUser));
+    a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+
+    TestRMAppManager testRmAppManager =
+        new TestRMAppManager(rmContext, cSched, rm.getApplicationMasterService(),
+            rm.getApplicationACLsManager(), rm.getConfig());
+    ContainerLaunchContext clc =
+        mock(ContainerLaunchContext.class);
+    ApplicationSubmissionContext asc =
+        ApplicationSubmissionContext.newInstance(
+            applicationId0, "myApp0", "default", Priority.newInstance(0),
+            clc, false, false, 1, Resource.newInstance(1024, 1));
+
+    // Each of the following test cases has a proxied user and a real user.
+    // The proxied users are user0 and user1, depending on the test. The real
+    // user is always AdminUser.
+
+    // Ensure that user0 is not allowed to submit to the default queue when only
+    // the admin user is in the submit ACL and the admin user does not have the
+    // USE_REAL_ACLS character prepended.
+    try {
+      testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0);
+      Assert.fail(user0 + " should not be allowed to submit to the "
+                  + queue + " queue when only admin user is in submit ACL.");
+    } catch (YarnException e) {
+      // This is the expected behavior.
+      assertTrue("Should have received an AccessControlException.",
+          e.getCause() instanceof AccessControlException);
+    }
+
+    // With only user0 in the list of users authorized to submit apps to the
+    // queue, ensure that user0 is allowed to submit to the default queue.
+    a.put(AccessType.SUBMIT_APP, new AccessControlList(user0));
+    a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser));
+    try {
+      testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0);
+    } catch (YarnException e) {
+      Assert.fail(user0 + " should be allowed to submit to the "
+                  + queue + " queue.");
+    }
+
+    // With only user0 in the list of users authorized to submit apps to the
+    // queue, ensure that user1 is NOT allowed to submit to the default queue
+    try {
+      testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+      Assert.fail(user1 + " should not be allowed to submit to the "
+          + queue + " queue.");
+    } catch (YarnException e) {
+      // This is the expected behavior.
+      assertTrue("Should have received an AccessControlException.",
+          e.getCause() instanceof AccessControlException);
+    }
+
+    // Even though the admin user is in the list of users allowed to submit to
+    // the default queue and user1's real user is the admin user, user1 should
+    // not be allowed to submit to queue because the ACL entry does not have the
+    // special character prepended in the list.
+    a.put(AccessType.SUBMIT_APP,
+        new AccessControlList(user0 + "," + realUser));
+    try {
+      testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+      Assert.fail(user1 + " should not be allowed to submit to the "
+                  + queue + " queue.");
+    } catch (YarnException e) {
+      // This is the expected behavior.
+      assertTrue("Should have received an AccessControlException.",
+          e.getCause() instanceof AccessControlException);
+    }
+
+    // user1 should now be allowed to submit to the default queue because the
+    // admin user is in the ACL list and has the USE_REAL_ACLS character
+    // prepended.
+    a.put(AccessType.SUBMIT_APP,
+        new AccessControlList(user0 + ","
+            + AccessControlList.USE_REAL_ACLS + realUser));
+    asc.setApplicationId(applicationId1);
+    try {
+      testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1);
+    } catch (YarnException e) {
+      LOG.error("failed to submit", e);
+      Assert.fail(user1 + " should be allowed to submit to the "
+                  + queue + " queue when real user is" + realUser + ".");
+    }
+
+    rm.stop();
+    rm.close();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 30ee24d6e5532..87aa156f29207 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -25,7 +25,6 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 8b75a9c126fbb..476abc638fba7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -79,7 +79,8 @@ public class TestParentQueue {
   YarnConfiguration conf;
   CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
-  
+  CapacitySchedulerQueueContext queueContext;
+
   final static int GB = 1024;
   final static String DEFAULT_RACK = "/default";
 
@@ -105,6 +106,10 @@ public void setUp() throws Exception {
     when(csContext.getResourceCalculator()).
         thenReturn(resourceComparator);
     when(csContext.getRMContext()).thenReturn(rmContext);
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(
+        new CapacitySchedulerQueueManager(csConf, rmContext.getNodeLabelManager(), null));
+
+    queueContext = new CapacitySchedulerQueueContext(csContext);
   }
   
   private static final String A = "a";
@@ -121,6 +126,8 @@ private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
     conf.setCapacity(Q_A, 30);
     
     conf.setCapacity(Q_B, 70);
+
+    queueContext.reinitialize();
     
     LOG.info("Setup top-level queues a and b");
   }
@@ -131,12 +138,14 @@ private void setupSingleLevelQueuesWithAbsoluteResource(
     // Define top-level queues
     conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
 
-    conf.setMinimumResourceRequirement("", Q_A,
+    conf.setMinimumResourceRequirement("", new QueuePath(Q_A),
         QUEUE_A_RESOURCE);
 
-    conf.setMinimumResourceRequirement("", Q_B,
+    conf.setMinimumResourceRequirement("", new QueuePath(Q_B),
         QUEUE_B_RESOURCE);
 
+    queueContext.reinitialize();
+
     LOG.info("Setup top-level queues a and b with absolute resource");
   }
 
@@ -253,7 +262,7 @@ public void testSingleLevelQueues() throws Exception {
 
     CSQueueStore queues = new CSQueueStore();
     CSQueue root =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT, queues, queues, 
             TestUtils.spyHook);
 
@@ -371,11 +380,12 @@ public void testSingleLevelQueuesPrecision() throws Exception {
     setupSingleLevelQueues(csConf);
     csConf.setCapacity(Q_A, 30);
     csConf.setCapacity(Q_B, 70.5F);
+    queueContext.reinitialize();
 
     CSQueueStore queues = new CSQueueStore();
     boolean exceptionOccurred = false;
     try {
-      CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+      CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
           CapacitySchedulerConfiguration.ROOT, queues, queues,
           TestUtils.spyHook);
     } catch (IOException ie) {
@@ -386,10 +396,11 @@ public void testSingleLevelQueuesPrecision() throws Exception {
     }
     csConf.setCapacity(Q_A, 30);
     csConf.setCapacity(Q_B, 70);
+    queueContext.reinitialize();
     exceptionOccurred = false;
     queues.clear();
     try {
-      CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+      CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
           CapacitySchedulerConfiguration.ROOT, queues, queues,
           TestUtils.spyHook);
     } catch (IllegalArgumentException ie) {
@@ -400,10 +411,11 @@ public void testSingleLevelQueuesPrecision() throws Exception {
     }
     csConf.setCapacity(Q_A, 30);
     csConf.setCapacity(Q_B, 70.005F);
+    queueContext.reinitialize();
     exceptionOccurred = false;
     queues.clear();
     try {
-      CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+      CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
           CapacitySchedulerConfiguration.ROOT, queues, queues,
           TestUtils.spyHook);
     } catch (IllegalArgumentException ie) {
@@ -470,6 +482,7 @@ private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) {
     conf.setQueues(Q_C111, new String[] {C1111});
     final String Q_C1111= Q_C111 + "." + C1111;
     conf.setCapacity(Q_C1111, 100);
+    queueContext.reinitialize();
   }
 
   @Test
@@ -495,7 +508,7 @@ public void testMultiLevelQueues() throws Exception {
 
     CSQueueStore queues = new CSQueueStore();
     CSQueue root =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT, queues, queues,
             TestUtils.spyHook);
     
@@ -657,9 +670,10 @@ public void testQueueCapacitySettingChildZero() throws Exception {
     csConf.setCapacity(Q_B + "." + B1, 0);
     csConf.setCapacity(Q_B + "." + B2, 0);
     csConf.setCapacity(Q_B + "." + B3, 0);
+    queueContext.reinitialize();
 
     CSQueueStore queues = new CSQueueStore();
-    CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
   }
@@ -672,9 +686,10 @@ public void testQueueCapacitySettingParentZero() throws Exception {
     // set parent capacity to 0 when child not 0
     csConf.setCapacity(Q_B, 0);
     csConf.setCapacity(Q_A, 60);
+    queueContext.reinitialize();
 
     CSQueueStore queues = new CSQueueStore();
-    CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
   }
@@ -690,8 +705,9 @@ public void testQueueCapacitySettingParentZeroChildren100pctZeroSumAllowed()
     csConf.setCapacity(Q_B, 0);
     csConf.setCapacity(Q_A, 60);
     csConf.setAllowZeroCapacitySum(Q_B, true);
+    queueContext.reinitialize();
     CSQueueStore queues = new CSQueueStore();
-    CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
   }
@@ -710,8 +726,9 @@ public void testQueueCapacitySettingParentZeroChildren50pctZeroSumAllowed()
     csConf.setCapacity(Q_B + "." + B2, 20);
     csConf.setCapacity(Q_B + "." + B3, 20);
     csConf.setAllowZeroCapacitySum(Q_B, true);
+    queueContext.reinitialize();
     CSQueueStore queues = new CSQueueStore();
-    CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
   }
@@ -730,8 +747,9 @@ public void testQueueCapacitySettingParentNonZeroChildrenZeroSumAllowed()
     csConf.setCapacity(Q_B + "." + B2, 0);
     csConf.setCapacity(Q_B + "." + B3, 0);
     csConf.setAllowZeroCapacitySum(Q_B, true);
+    queueContext.reinitialize();
     CSQueueStore queues = new CSQueueStore();
-    CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+    CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
         CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
   }
@@ -746,12 +764,12 @@ public void testQueueCapacityZero() throws Exception {
     csConf.setCapacity(Q_B + "." + B1, 0);
     csConf.setCapacity(Q_B + "." + B2, 0);
     csConf.setCapacity(Q_B + "." + B3, 0);
-    
     csConf.setCapacity(Q_A, 60);
+    queueContext.reinitialize();
 
     CSQueueStore queues = new CSQueueStore();
     try {
-      CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+      CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
           CapacitySchedulerConfiguration.ROOT, queues, queues,
           TestUtils.spyHook);
     } catch (IllegalArgumentException e) {
@@ -767,7 +785,7 @@ public void testOffSwitchScheduling() throws Exception {
 
     CSQueueStore queues = new CSQueueStore();
     CSQueue root =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT, queues, queues,
             TestUtils.spyHook);
 
@@ -849,7 +867,7 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception {
     //B3
     CSQueueStore queues = new CSQueueStore();
     CSQueue root = 
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT, queues, queues,
             TestUtils.spyHook);
 
@@ -948,10 +966,11 @@ public void testQueueAcl() throws Exception {
     csConf.setAcl(Q_C, QueueACL.ADMINISTER_QUEUE, "*");
     final String Q_C11= Q_C + "." + C1 +  "." + C11;
     csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*");
+    queueContext.reinitialize();
 
     CSQueueStore queues = new CSQueueStore();
     CSQueue root = 
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
             CapacitySchedulerConfiguration.ROOT, queues, queues,
             TestUtils.spyHook);
     YarnAuthorizationProvider authorizer =
@@ -1014,7 +1033,7 @@ public void testAbsoluteResourceWithChangeInClusterResource()
     setupSingleLevelQueuesWithAbsoluteResource(csConf);
 
     CSQueueStore queues = new CSQueueStore();
-    CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf,
+    CSQueue root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf,
         null, CapacitySchedulerConfiguration.ROOT, queues, queues,
         TestUtils.spyHook);
 
@@ -1085,7 +1104,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception {
     setupSingleLevelQueuesWithAbsoluteResource(csConf);
 
     CSQueueStore queues = new CSQueueStore();
-    CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf,
+    CSQueue root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf,
             null, CapacitySchedulerConfiguration.ROOT, queues, queues,
             TestUtils.spyHook);
 
@@ -1137,7 +1156,8 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception {
     assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser());
 
     // Set GlobalMaximumApplicationsPerQueue in csConf
-    csConf.setGlobalMaximumApplicationsPerQueue(20000);
+    csConf.setGlobalMaximumApplicationsPerQueue(8000);
+    queueContext.reinitialize();
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
@@ -1155,6 +1175,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception {
             Integer.toString(queueAMaxApplications));
     csConf.set("yarn.scheduler.capacity." + Q_B + ".maximum-applications",
             Integer.toString(queueBMaxApplications));
+    queueContext.reinitialize();
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
index 248831f03d15e..c3f505bdf085e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -24,7 +24,6 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java
new file mode 100644
index 0000000000000..058e14bfaf272
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityVectorEntry;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+
+public class TestQueueCapacityVector {
+  private static final String CUSTOM_RESOURCE = "custom";
+  public static final String MIXED_CAPACITY_VECTOR_STRING =
+      "[custom=3.0,memory-mb=10.0w,vcores=6.0%]";
+
+  private final YarnConfiguration conf = new YarnConfiguration();
+
+  @Before
+  public void setUp() {
+    conf.set(YarnConfiguration.RESOURCE_TYPES, CUSTOM_RESOURCE);
+    ResourceUtils.resetResourceTypes(conf);
+  }
+
+  @Test
+  public void getResourceNamesByCapacityType() {
+    QueueCapacityVector capacityVector = QueueCapacityVector.newInstance();
+
+    capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.PERCENTAGE);
+    capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE);
+
+    // custom is not set, defaults to 0
+    Assert.assertEquals(1, capacityVector.getResourceNamesByCapacityType(
+        QueueCapacityType.ABSOLUTE).size());
+    Assert.assertTrue(capacityVector.getResourceNamesByCapacityType(
+        QueueCapacityType.ABSOLUTE).contains(CUSTOM_RESOURCE));
+
+    Assert.assertEquals(2, capacityVector.getResourceNamesByCapacityType(
+        QueueCapacityType.PERCENTAGE).size());
+    Assert.assertTrue(capacityVector.getResourceNamesByCapacityType(
+        QueueCapacityType.PERCENTAGE).contains(VCORES_URI));
+    Assert.assertTrue(capacityVector.getResourceNamesByCapacityType(
+        QueueCapacityType.PERCENTAGE).contains(MEMORY_URI));
+    Assert.assertEquals(10, capacityVector.getResource(MEMORY_URI).getResourceValue(), EPSILON);
+    Assert.assertEquals(6, capacityVector.getResource(VCORES_URI).getResourceValue(), EPSILON);
+  }
+
+  @Test
+  public void isResourceOfType() {
+    QueueCapacityVector capacityVector = QueueCapacityVector.newInstance();
+
+    capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.WEIGHT);
+    capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE);
+    capacityVector.setResource(CUSTOM_RESOURCE, 3, QueueCapacityType.ABSOLUTE);
+
+    Assert.assertTrue(capacityVector.isResourceOfType(MEMORY_URI, QueueCapacityType.WEIGHT));
+    Assert.assertTrue(capacityVector.isResourceOfType(VCORES_URI, QueueCapacityType.PERCENTAGE));
+    Assert.assertTrue(capacityVector.isResourceOfType(CUSTOM_RESOURCE, QueueCapacityType.ABSOLUTE));
+  }
+
+  @Test
+  public void testIterator() {
+    QueueCapacityVector capacityVector = QueueCapacityVector.newInstance();
+    List<QueueCapacityVectorEntry> entries = Lists.newArrayList(capacityVector);
+
+    Assert.assertEquals(3, entries.size());
+
+    QueueCapacityVector emptyCapacityVector = new QueueCapacityVector();
+    List<QueueCapacityVectorEntry> emptyEntries = Lists.newArrayList(emptyCapacityVector);
+
+    Assert.assertEquals(0, emptyEntries.size());
+  }
+
+  @Test
+  public void testToString() {
+    QueueCapacityVector capacityVector = QueueCapacityVector.newInstance();
+
+    capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.WEIGHT);
+    capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE);
+    capacityVector.setResource(CUSTOM_RESOURCE, 3, QueueCapacityType.ABSOLUTE);
+
+    Assert.assertEquals(MIXED_CAPACITY_VECTOR_STRING, capacityVector.toString());
+
+    QueueCapacityVector emptyCapacityVector = new QueueCapacityVector();
+    Assert.assertEquals("[]", emptyCapacityVector.toString());
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
index bfbc0de31d335..7eb577d9c1271 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
@@ -54,6 +54,16 @@ public void testEmptyPart() {
     Assert.assertFalse(queuePathWithoutEmptyPart.hasEmptyPart());
   }
 
+  @Test
+  public void testNullPath() {
+    QueuePath queuePathWithNullPath = new QueuePath(null);
+
+    Assert.assertNull(queuePathWithNullPath.getParent());
+    Assert.assertEquals("", queuePathWithNullPath.getLeafName());
+    Assert.assertEquals("", queuePathWithNullPath.getFullPath());
+    Assert.assertFalse(queuePathWithNullPath.isRoot());
+  }
+
   @Test
   public void testIterator() {
     QueuePath queuePath = new QueuePath(TEST_QUEUE);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
index 8407922b6331d..94d800732fb51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -46,6 +46,7 @@ public class TestReservationQueue {
 
   private CapacitySchedulerConfiguration csConf;
   private CapacitySchedulerContext csContext;
+  private CapacitySchedulerQueueContext queueContext;
   final static int DEF_MAX_APPS = 10000;
   final static int GB = 1024;
   private final ResourceCalculator resourceCalculator =
@@ -63,7 +64,7 @@ public void setup() throws IOException, SchedulerDynamicEditException {
     CapacitySchedulerQueueManager csQm = mock(
         CapacitySchedulerQueueManager.class);
     ConfiguredNodeLabels labels = new ConfiguredNodeLabels(csConf);
-    when(csQm.getConfiguredNodeLabels()).thenReturn(labels);
+    when(csQm.getConfiguredNodeLabelsForAllQueues()).thenReturn(labels);
     when(csContext.getConfiguration()).thenReturn(csConf);
     when(csContext.getCapacitySchedulerQueueManager()).thenReturn(csQm);
     when(csContext.getConf()).thenReturn(conf);
@@ -78,9 +79,11 @@ public void setup() throws IOException, SchedulerDynamicEditException {
     RMContext mockRMContext = TestUtils.getMockRMContext();
     when(csContext.getRMContext()).thenReturn(mockRMContext);
 
+    queueContext = new CapacitySchedulerQueueContext(csContext);
+
     // create a queue
-    planQueue = new PlanQueue(csContext, "root", null, null);
-    autoCreatedLeafQueue = new ReservationQueue(csContext, "a", planQueue);
+    planQueue = new PlanQueue(queueContext, "root", null, null);
+    autoCreatedLeafQueue = new ReservationQueue(queueContext, "a", planQueue);
     planQueue.addChildQueue(autoCreatedLeafQueue);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 1168f648024b1..c6f947febc00e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -58,7 +58,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
 
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -90,6 +89,7 @@ public class TestReservations {
   CapacityScheduler cs;
   // CapacitySchedulerConfiguration csConf;
   CapacitySchedulerContext csContext;
+  CapacitySchedulerQueueContext queueContext;
 
   private final ResourceCalculator resourceCalculator =
       new DefaultResourceCalculator();
@@ -135,7 +135,10 @@ private void setup(CapacitySchedulerConfiguration csConf,
     when(csContext.getClusterResource()).thenReturn(
         Resources.createResource(100 * 16 * GB, 100 * 12));
     when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
+    CapacitySchedulerQueueManager queueManager = new CapacitySchedulerQueueManager(conf,
+        rmContext.getNodeLabelManager(), null);
     when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+    when(csContext.getCapacitySchedulerQueueManager()).thenReturn(queueManager);
     when(csContext.getRMContext()).thenReturn(rmContext);
     RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(
         conf);
@@ -143,12 +146,11 @@ private void setup(CapacitySchedulerConfiguration csConf,
     when(csContext.getContainerTokenSecretManager()).thenReturn(
         containerTokenSecretManager);
 
-    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
-        CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+    queueContext = new CapacitySchedulerQueueContext(csContext);
 
-    ResourceUsage queueResUsage = root.getQueueResourceUsage();
-    when(csContext.getClusterResourceUsage())
-        .thenReturn(queueResUsage);
+    root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
+        CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+    queueManager.setRootQueue(root);
 
     spyRMContext = spy(rmContext);
     when(spyRMContext.getScheduler()).thenReturn(cs);
@@ -289,7 +291,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -303,7 +305,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -317,7 +319,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -335,7 +337,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -354,7 +356,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_2,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(18 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -374,7 +376,7 @@ public void testReservation() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(18 * GB, a.getUsedResources().getMemorySize());
     assertEquals(18 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -475,7 +477,7 @@ public void testReservationLimitOtherUsers() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize());
@@ -489,7 +491,7 @@ public void testReservationLimitOtherUsers() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(4 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize());
@@ -512,7 +514,7 @@ public void testReservationLimitOtherUsers() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(12 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize());
@@ -528,7 +530,7 @@ public void testReservationLimitOtherUsers() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(14 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(4 * GB, app_1.getCurrentConsumption().getMemorySize());
@@ -626,7 +628,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -640,7 +642,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -654,7 +656,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -672,7 +674,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -691,7 +693,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_2,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(18 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -711,7 +713,7 @@ public void testReservationNoContinueLook() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(18 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -809,7 +811,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -822,7 +824,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -835,7 +837,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -852,7 +854,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -870,7 +872,7 @@ public void testAssignContainersNeedToUnreserve() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1100,7 +1102,7 @@ public void testAssignToQueue() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1113,7 +1115,7 @@ public void testAssignToQueue() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1126,7 +1128,7 @@ public void testAssignToQueue() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1142,7 +1144,7 @@ public void testAssignToQueue() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, a.getMetrics().getReservedMB());
@@ -1181,23 +1183,24 @@ public void testAssignToQueue() throws Exception {
   public void refreshQueuesTurnOffReservationsContLook(LeafQueue a,
       CapacitySchedulerConfiguration csConf) throws Exception {
     // before reinitialization
-    assertEquals(true, a.getReservationContinueLooking());
+    assertEquals(true, a.isReservationsContinueLooking());
     assertEquals(true,
-        ((ParentQueue) a.getParent()).getReservationContinueLooking());
+        ((ParentQueue) a.getParent()).isReservationsContinueLooking());
 
     csConf.setBoolean(
         CapacitySchedulerConfiguration.RESERVE_CONT_LOOK_ALL_NODES, false);
     CSQueueStore newQueues = new CSQueueStore();
-    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+    queueContext.reinitialize();
+    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext,
         csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
         TestUtils.spyHook);
     queues = newQueues;
     root.reinitialize(newRoot, cs.getClusterResource());
 
     // after reinitialization
-    assertEquals(false, a.getReservationContinueLooking());
+    assertEquals(false, a.isReservationsContinueLooking());
     assertEquals(false,
-        ((ParentQueue) a.getParent()).getReservationContinueLooking());
+        ((ParentQueue) a.getParent()).isReservationsContinueLooking());
   }
 
   @Test
@@ -1290,7 +1293,7 @@ public void testAssignToUser() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1303,7 +1306,7 @@ public void testAssignToUser() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1316,7 +1319,7 @@ public void testAssignToUser() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1332,7 +1335,7 @@ public void testAssignToUser() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentReservation().getMemorySize());
@@ -1459,7 +1462,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(2 * GB, a.getUsedResources().getMemorySize());
     assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1473,7 +1476,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(5 * GB, a.getUsedResources().getMemorySize());
     assertEquals(5 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1487,7 +1490,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_1,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1505,7 +1508,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(Resources.createResource(10 * GB)),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1523,7 +1526,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_2,
             new ResourceLimits(Resources.createResource(10 * GB)),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(8 * GB, a.getUsedResources().getMemorySize());
     assertEquals(8 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1539,7 +1542,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_2,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(13 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB, a.getMetrics().getReservedMB());
@@ -1554,7 +1557,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_0,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(21 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(8 * GB, a.getMetrics().getReservedMB());
@@ -1571,7 +1574,7 @@ public void testReservationsNoneAvailable() throws Exception {
     TestUtils.applyResourceCommitRequest(clusterResource,
         a.assignContainers(clusterResource, node_2,
             new ResourceLimits(clusterResource),
-            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps);
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodes, apps, csConf);
     assertEquals(21 * GB, a.getUsedResources().getMemorySize());
     assertEquals(13 * GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(8 * GB, a.getMetrics().getReservedMB());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java
new file mode 100644
index 0000000000000..fd6edb1fa5d49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+
+public class TestResourceVector {
+  private final static String CUSTOM_RESOURCE = "custom";
+
+  private final YarnConfiguration conf = new YarnConfiguration();
+
+  @Before
+  public void setUp() {
+    conf.set(YarnConfiguration.RESOURCE_TYPES, CUSTOM_RESOURCE);
+    ResourceUtils.resetResourceTypes(conf);
+  }
+
+  @Test
+  public void testCreation() {
+    ResourceVector zeroResourceVector = ResourceVector.newInstance();
+    Assert.assertEquals(0, zeroResourceVector.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(0, zeroResourceVector.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(0, zeroResourceVector.getValue(CUSTOM_RESOURCE), EPSILON);
+
+    ResourceVector uniformResourceVector = ResourceVector.of(10);
+    Assert.assertEquals(10, uniformResourceVector.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(10, uniformResourceVector.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(10, uniformResourceVector.getValue(CUSTOM_RESOURCE), EPSILON);
+
+    Map<String, Long> customResources = new HashMap<>();
+    customResources.put(CUSTOM_RESOURCE, 2L);
+    Resource resource = Resource.newInstance(10, 5, customResources);
+    ResourceVector resourceVectorFromResource = ResourceVector.of(resource);
+    Assert.assertEquals(10, resourceVectorFromResource.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(5, resourceVectorFromResource.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(2, resourceVectorFromResource.getValue(CUSTOM_RESOURCE), EPSILON);
+  }
+
+  @Test
+  public void testSubtract() {
+    ResourceVector lhsResourceVector = ResourceVector.of(13);
+    ResourceVector rhsResourceVector = ResourceVector.of(5);
+    lhsResourceVector.subtract(rhsResourceVector);
+
+    Assert.assertEquals(8, lhsResourceVector.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(8, lhsResourceVector.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(8, lhsResourceVector.getValue(CUSTOM_RESOURCE), EPSILON);
+
+    ResourceVector negativeResourceVector = ResourceVector.of(-100);
+
+    // Check whether overflow causes any issues
+    negativeResourceVector.subtract(ResourceVector.of(Float.MAX_VALUE));
+    Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(CUSTOM_RESOURCE),
+        EPSILON);
+
+  }
+
+  @Test
+  public void testIncrement() {
+    ResourceVector resourceVector = ResourceVector.of(13);
+    resourceVector.increment(MEMORY_URI, 5);
+
+    Assert.assertEquals(18, resourceVector.getValue(MEMORY_URI), EPSILON);
+    Assert.assertEquals(13, resourceVector.getValue(VCORES_URI), EPSILON);
+    Assert.assertEquals(13, resourceVector.getValue(CUSTOM_RESOURCE), EPSILON);
+
+    // Check whether overflow causes any issues
+    ResourceVector maxFloatResourceVector = ResourceVector.of(Float.MAX_VALUE);
+    maxFloatResourceVector.increment(MEMORY_URI, 100);
+    Assert.assertEquals(Float.MAX_VALUE, maxFloatResourceVector.getValue(MEMORY_URI), EPSILON);
+  }
+
+  @Test
+  public void testEquals() {
+    ResourceVector resourceVector = ResourceVector.of(13);
+    ResourceVector resourceVectorOther = ResourceVector.of(14);
+    Resource resource = Resource.newInstance(13, 13);
+
+    Assert.assertNotEquals(null, resourceVector);
+    Assert.assertNotEquals(resourceVectorOther, resourceVector);
+    Assert.assertNotEquals(resource, resourceVector);
+
+    ResourceVector resourceVectorOne = ResourceVector.of(1);
+    resourceVectorOther.subtract(resourceVectorOne);
+
+    Assert.assertEquals(resourceVectorOther, resourceVector);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java
index 5b79ee2e255c1..c71d862a32a02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java
@@ -58,15 +58,11 @@ public class TestUsersManager {
   @Mock
   private QueueMetrics metrics;
 
-  @Mock
-  private CapacitySchedulerContext context;
-
   @Before
   public void setup() {
     usersManager = new UsersManager(metrics,
         lQueue,
         labelMgr,
-        context,
         new DefaultResourceCalculator());
 
     when(lQueue.getMinimumAllocation()).thenReturn(MINIMUM_ALLOCATION);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 026206ac38f47..28ca66847de00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -447,7 +447,14 @@ public static SchedulerRequestKey toSchedulerKey(Priority pri,
   public static void applyResourceCommitRequest(Resource clusterResource,
       CSAssignment csAssignment,
       final Map<NodeId, FiCaSchedulerNode> nodes,
-      final Map<ApplicationAttemptId, FiCaSchedulerApp> apps)
+      final Map<ApplicationAttemptId, FiCaSchedulerApp> apps) throws IOException {
+    applyResourceCommitRequest(clusterResource, csAssignment, nodes, apps, null);
+  }
+
+  public static void applyResourceCommitRequest(Resource clusterResource,
+      CSAssignment csAssignment,
+      final Map<NodeId, FiCaSchedulerNode> nodes,
+      final Map<ApplicationAttemptId, FiCaSchedulerApp> apps, CapacitySchedulerConfiguration csConf)
       throws IOException {
     CapacityScheduler cs = new CapacityScheduler() {
       @Override
@@ -461,7 +468,7 @@ public FiCaSchedulerApp getApplicationAttempt(
         return apps.get(applicationAttemptId);
       }
     };
-
+    cs.setAsyncSchedulingConf(new CapacityScheduler.AsyncSchedulingConfiguration(csConf, cs));
     cs.setResourceCalculator(new DefaultResourceCalculator());
 
     cs.submitResourceCommitRequest(clusterResource,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java
new file mode 100644
index 0000000000000..890996ac23e57
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests {@link ConfigurationUpdateAssembler}.
+ */
+public class TestConfigurationUpdateAssembler {
+
+  private static final String A_PATH = "root.a";
+  private static final String B_PATH = "root.b";
+  private static final String C_PATH = "root.c";
+
+  private static final String CONFIG_NAME = "testConfigName";
+  private static final String A_CONFIG_PATH = CapacitySchedulerConfiguration.PREFIX + A_PATH +
+          CapacitySchedulerConfiguration.DOT + CONFIG_NAME;
+  private static final String B_CONFIG_PATH = CapacitySchedulerConfiguration.PREFIX + B_PATH +
+          CapacitySchedulerConfiguration.DOT + CONFIG_NAME;
+  private static final String C_CONFIG_PATH = CapacitySchedulerConfiguration.PREFIX + C_PATH +
+          CapacitySchedulerConfiguration.DOT + CONFIG_NAME;
+  private static final String ROOT_QUEUES_PATH = CapacitySchedulerConfiguration.PREFIX +
+          CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT +
+          CapacitySchedulerConfiguration.QUEUES;
+
+  private static final String A_INIT_CONFIG_VALUE = "aInitValue";
+  private static final String A_CONFIG_VALUE = "aValue";
+  private static final String B_INIT_CONFIG_VALUE = "bInitValue";
+  private static final String B_CONFIG_VALUE = "bValue";
+  private static final String C_CONFIG_VALUE = "cValue";
+
+  private CapacitySchedulerConfiguration csConfig;
+
+  @Before
+  public void setUp() {
+    csConfig = crateInitialCSConfig();
+  }
+
+  @Test
+  public void testAddQueue() throws Exception {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> updateMap = new HashMap<>();
+    updateMap.put(CONFIG_NAME, C_CONFIG_VALUE);
+    QueueConfigInfo queueConfigInfo = new QueueConfigInfo(C_PATH, updateMap);
+    updateInfo.getAddQueueInfo().add(queueConfigInfo);
+
+    Map<String, String> configurationUpdate =
+            ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+
+    assertEquals(C_CONFIG_VALUE, configurationUpdate.get(C_CONFIG_PATH));
+    assertEquals("a,b,c", configurationUpdate.get(ROOT_QUEUES_PATH));
+  }
+
+  @Test
+  public void testAddExistingQueue() {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> updateMap = new HashMap<>();
+    updateMap.put(CONFIG_NAME, A_CONFIG_VALUE);
+    QueueConfigInfo queueConfigInfo = new QueueConfigInfo(A_PATH, updateMap);
+    updateInfo.getAddQueueInfo().add(queueConfigInfo);
+
+    assertThrows(IOException.class, () -> {
+      ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+    });
+  }
+
+  @Test
+  public void testAddInvalidQueue() {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> updateMap = new HashMap<>();
+    updateMap.put(CONFIG_NAME, A_CONFIG_VALUE);
+    QueueConfigInfo queueConfigInfo = new QueueConfigInfo("invalidPath", updateMap);
+    updateInfo.getAddQueueInfo().add(queueConfigInfo);
+
+    assertThrows(IOException.class, () -> {
+      ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+    });
+  }
+
+  @Test
+  public void testUpdateQueue() throws Exception {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> updateMap = new HashMap<>();
+    updateMap.put(CONFIG_NAME, A_CONFIG_VALUE);
+    QueueConfigInfo queueAConfigInfo = new QueueConfigInfo(A_PATH, updateMap);
+    updateInfo.getUpdateQueueInfo().add(queueAConfigInfo);
+
+    Map<String, String> updateMapQueueB = new HashMap<>();
+    updateMapQueueB.put(CONFIG_NAME, B_CONFIG_VALUE);
+    QueueConfigInfo queueBConfigInfo = new QueueConfigInfo(B_PATH, updateMapQueueB);
+
+    updateInfo.getUpdateQueueInfo().add(queueBConfigInfo);
+
+    Map<String, String> configurationUpdate =
+            ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+
+    assertEquals(A_CONFIG_VALUE, configurationUpdate.get(A_CONFIG_PATH));
+    assertEquals(B_CONFIG_VALUE, configurationUpdate.get(B_CONFIG_PATH));
+  }
+
+  @Test
+  public void testRemoveQueue() throws Exception {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    updateInfo.getRemoveQueueInfo().add(A_PATH);
+
+    Map<String, String> configurationUpdate =
+            ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+
+    assertTrue(configurationUpdate.containsKey(A_CONFIG_PATH));
+    assertNull(configurationUpdate.get(A_CONFIG_PATH));
+    assertEquals("b", configurationUpdate.get(ROOT_QUEUES_PATH));
+  }
+
+  @Test
+  public void testRemoveInvalidQueue() {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    updateInfo.getRemoveQueueInfo().add("invalidPath");
+
+    assertThrows(IOException.class, () -> {
+      ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+    });
+  }
+
+  @Test
+  public void testRemoveNonExistingQueue() {
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    updateInfo.getRemoveQueueInfo().add("root.d");
+
+    assertThrows(IOException.class, () -> {
+      ConfigurationUpdateAssembler.constructKeyValueConfUpdate(csConfig, updateInfo);
+    });
+  }
+
+  private CapacitySchedulerConfiguration crateInitialCSConfig() {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a, b"});
+
+    csConf.set(A_CONFIG_PATH, A_INIT_CONFIG_VALUE);
+    csConf.set(B_CONFIG_PATH, B_INIT_CONFIG_VALUE);
+
+    return csConf;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java
new file mode 100644
index 0000000000000..1aba816abd27a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityVectorEntry;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI;
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+
+public class TestQueueCapacityConfigParser {
+
+  private static final String ALL_RESOURCE_TEMPLATE = "[memory-mb=%s, vcores=%s, yarn.io/gpu=%s]";
+  private static final String MEMORY_VCORE_TEMPLATE = "[memory-mb=%s, vcores=%s]";
+
+  private static final String MEMORY_ABSOLUTE = "12Gi";
+  private static final float VCORE_ABSOLUTE = 6;
+  private static final float GPU_ABSOLUTE = 10;
+
+  private static final float PERCENTAGE_VALUE = 50f;
+  private static final float MEMORY_MIXED = 1024;
+  private static final float WEIGHT_VALUE = 6;
+
+  private static final String QUEUE = "root.test";
+
+  private static final String ABSOLUTE_RESOURCE = String.format(
+      ALL_RESOURCE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE, GPU_ABSOLUTE);
+  private static final String ABSOLUTE_RESOURCE_MEMORY_VCORE = String.format(
+      MEMORY_VCORE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE);
+  private static final String MIXED_RESOURCE = String.format(
+      ALL_RESOURCE_TEMPLATE, MEMORY_MIXED, PERCENTAGE_VALUE + "%", WEIGHT_VALUE + "w");
+  private static final String RESOURCE_TYPES = GPU_URI;
+
+  public static final String NONEXISTINGSUFFIX = "50nonexistingsuffix";
+  public static final String EMPTY_BRACKET = "[]";
+  public static final String INVALID_CAPACITY_BRACKET = "[invalid]";
+  public static final String INVALID_CAPACITY_FORMAT = "[memory-100,vcores-60]";
+
+  private final QueueCapacityConfigParser capacityConfigParser
+      = new QueueCapacityConfigParser();
+
+  @Test
+  public void testPercentageCapacityConfig() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.setCapacity(QUEUE, PERCENTAGE_VALUE);
+
+    QueueCapacityVector percentageCapacityVector = capacityConfigParser.parse(conf, QUEUE,
+        NO_LABEL);
+    QueueCapacityVectorEntry memory = percentageCapacityVector.getResource(MEMORY_URI);
+    QueueCapacityVectorEntry vcore = percentageCapacityVector.getResource(VCORES_URI);
+
+    Assert.assertEquals(QueueCapacityType.PERCENTAGE, memory.getVectorResourceType());
+    Assert.assertEquals(PERCENTAGE_VALUE, memory.getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.PERCENTAGE, vcore.getVectorResourceType());
+    Assert.assertEquals(PERCENTAGE_VALUE, vcore.getResourceValue(), EPSILON);
+
+    QueueCapacityVector rootCapacityVector = capacityConfigParser.parse(conf,
+        CapacitySchedulerConfiguration.ROOT, NO_LABEL);
+
+    QueueCapacityVectorEntry memoryRoot = rootCapacityVector.getResource(MEMORY_URI);
+    QueueCapacityVectorEntry vcoreRoot = rootCapacityVector.getResource(VCORES_URI);
+
+    Assert.assertEquals(QueueCapacityType.PERCENTAGE, memoryRoot.getVectorResourceType());
+    Assert.assertEquals(100f, memoryRoot.getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.PERCENTAGE, vcoreRoot.getVectorResourceType());
+    Assert.assertEquals(100f, vcoreRoot.getResourceValue(), EPSILON);
+  }
+
+  @Test
+  public void testWeightCapacityConfig() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.setNonLabeledQueueWeight(QUEUE, WEIGHT_VALUE);
+
+    QueueCapacityVector weightCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+
+    QueueCapacityVectorEntry memory = weightCapacityVector.getResource(MEMORY_URI);
+    QueueCapacityVectorEntry vcore = weightCapacityVector.getResource(VCORES_URI);
+
+    Assert.assertEquals(QueueCapacityType.WEIGHT, memory.getVectorResourceType());
+    Assert.assertEquals(WEIGHT_VALUE, memory.getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.WEIGHT, vcore.getVectorResourceType());
+    Assert.assertEquals(WEIGHT_VALUE, vcore.getResourceValue(), EPSILON);
+  }
+
+  @Test
+  public void testAbsoluteCapacityVectorConfig() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) +
+        CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE);
+    conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES);
+    ResourceUtils.resetResourceTypes(conf);
+
+    QueueCapacityVector absoluteCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+
+    Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(MEMORY_URI)
+        .getVectorResourceType());
+    Assert.assertEquals(12 * GB, absoluteCapacityVector.getResource(MEMORY_URI)
+        .getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(VCORES_URI)
+        .getVectorResourceType());
+    Assert.assertEquals(VCORE_ABSOLUTE, absoluteCapacityVector.getResource(VCORES_URI)
+        .getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI)
+        .getVectorResourceType());
+    Assert.assertEquals(GPU_ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI)
+        .getResourceValue(), EPSILON);
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) +
+        CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE);
+    QueueCapacityVector withoutGpuVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+
+    Assert.assertEquals(3, withoutGpuVector.getResourceCount());
+    Assert.assertEquals(0f, withoutGpuVector.getResource(GPU_URI).getResourceValue(), EPSILON);
+  }
+
+  @Test
+  public void testMixedCapacityConfig() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, MIXED_RESOURCE);
+    conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES);
+    ResourceUtils.resetResourceTypes(conf);
+
+    QueueCapacityVector mixedCapacityVector =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+
+    Assert.assertEquals(QueueCapacityType.ABSOLUTE,
+        mixedCapacityVector.getResource(MEMORY_URI).getVectorResourceType());
+    Assert.assertEquals(MEMORY_MIXED, mixedCapacityVector.getResource(MEMORY_URI)
+        .getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.PERCENTAGE,
+        mixedCapacityVector.getResource(VCORES_URI).getVectorResourceType());
+    Assert.assertEquals(PERCENTAGE_VALUE,
+        mixedCapacityVector.getResource(VCORES_URI).getResourceValue(), EPSILON);
+
+    Assert.assertEquals(QueueCapacityType.WEIGHT,
+        mixedCapacityVector.getResource(GPU_URI).getVectorResourceType());
+    Assert.assertEquals(WEIGHT_VALUE,
+        mixedCapacityVector.getResource(GPU_URI).getResourceValue(), EPSILON);
+
+    // Test undefined capacity type default value
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE);
+
+    QueueCapacityVector mixedCapacityVectorWithGpuUndefined =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    Assert.assertEquals(QueueCapacityType.ABSOLUTE,
+        mixedCapacityVectorWithGpuUndefined.getResource(MEMORY_URI).getVectorResourceType());
+    Assert.assertEquals(0, mixedCapacityVectorWithGpuUndefined.getResource(GPU_URI)
+        .getResourceValue(), EPSILON);
+
+  }
+
+  @Test
+  public void testInvalidCapacityConfigs() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, NONEXISTINGSUFFIX);
+    QueueCapacityVector capacityVectorWithInvalidSuffix =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> entriesWithInvalidSuffix =
+        Lists.newArrayList(capacityVectorWithInvalidSuffix.iterator());
+    Assert.assertEquals(0, entriesWithInvalidSuffix.size());
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_FORMAT);
+    QueueCapacityVector invalidDelimiterCapacityVector =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> invalidDelimiterEntries =
+        Lists.newArrayList(invalidDelimiterCapacityVector.iterator());
+    Assert.assertEquals(0, invalidDelimiterEntries.size());
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_BRACKET);
+    QueueCapacityVector invalidCapacityVector =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> resources =
+        Lists.newArrayList(invalidCapacityVector.iterator());
+    Assert.assertEquals(0, resources.size());
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET);
+    QueueCapacityVector emptyBracketCapacityVector =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> emptyEntries =
+        Lists.newArrayList(emptyBracketCapacityVector.iterator());
+    Assert.assertEquals(0, emptyEntries.size());
+
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY, "");
+    QueueCapacityVector emptyCapacity =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> emptyResources =
+        Lists.newArrayList(emptyCapacity.iterator());
+    Assert.assertEquals(emptyResources.size(), 0);
+
+    conf.unset(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE)
+        + CapacitySchedulerConfiguration.CAPACITY);
+    QueueCapacityVector nonSetCapacity =
+        capacityConfigParser.parse(conf, QUEUE, NO_LABEL);
+    List<QueueCapacityVectorEntry> nonSetResources =
+        Lists.newArrayList(nonSetCapacity.iterator());
+    Assert.assertEquals(nonSetResources.size(), 0);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 880ba77fa513b..155996d11feb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -42,15 +42,18 @@
 import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
 import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.ByteArrayOutputStream;
 import java.io.ObjectOutputStream;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -67,6 +70,9 @@ public class TestZKConfigurationStore extends
       LoggerFactory.getLogger(TestZKConfigurationStore.class);
 
   private static final int ZK_TIMEOUT_MS = 10000;
+  private static final String DESERIALIZATION_VULNERABILITY_FILEPATH =
+      "/tmp/ZK_DESERIALIZATION_VULNERABILITY";
+
   private TestingServer curatorTestingServer;
   private CuratorFramework curatorFramework;
   private ResourceManager rm;
@@ -401,6 +407,35 @@ public void testFailoverAfterRemoveQueue() throws Exception {
     rm2.close();
   }
 
+  @Test(timeout = 3000)
+  @SuppressWarnings("checkstyle:linelength")
+  public void testDeserializationIsNotVulnerable() throws Exception {
+    confStore.initialize(conf, schedConf, rmContext);
+    String confStorePath = getZkPath("CONF_STORE");
+
+    File flagFile = new File(DESERIALIZATION_VULNERABILITY_FILEPATH);
+    if (flagFile.exists()) {
+      Assert.assertTrue(flagFile.delete());
+    }
+
+    // Generated using ysoserial (https://github.com/frohoff/ysoserial)
+    // java -jar ysoserial.jar CommonsBeanutils1 'touch /tmp/ZK_DESERIALIZATION_VULNERABILITY' | base64
+    ((ZKConfigurationStore) confStore).setZkData(confStorePath, Base64.getDecoder().decode("rO0ABXNyABdqYXZhLnV0aWwuUHJpb3JpdHlRdWV1ZZTaMLT7P4KxAwACSQAEc2l6ZUwACmNvbXBhcmF0b3J0ABZMamF2YS91dGlsL0NvbXBhcmF0b3I7eHAAAAACc3IAK29yZy5hcGFjaGUuY29tbW9ucy5iZWFudXRpbHMuQmVhbkNvbXBhcmF0b3LjoYjqcyKkSAIAAkwACmNvbXBhcmF0b3JxAH4AAUwACHByb3BlcnR5dAASTGphdmEvbGFuZy9TdHJpbmc7eHBzcgA/b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmNvbXBhcmF0b3JzLkNvbXBhcmFibGVDb21wYXJhdG9y+/SZJbhusTcCAAB4cHQAEG91dHB1dFByb3BlcnRpZXN3BAAAAANzcgA6Y29tLnN1bi5vcmcuYXBhY2hlLnhhbGFuLmludGVybmFsLnhzbHRjLnRyYXguVGVtcGxhdGVzSW1wbAlXT8FurKszAwAGSQANX2luZGVudE51bWJlckkADl90cmFuc2xldEluZGV4WwAKX2J5dGVjb2Rlc3QAA1tbQlsABl9jbGFzc3QAEltMamF2YS9sYW5nL0NsYXNzO0wABV9uYW1lcQB+AARMABFfb3V0cHV0UHJvcGVydGllc3QAFkxqYXZhL3V0aWwvUHJvcGVydGllczt4cAAAAAD/////dXIAA1tbQkv9GRVnZ9s3AgAAeHAAAAACdXIAAltCrPMX+AYIVOACAAB4cAAABsHK/rq+AAAAMgA5CgADACIHADcHACUHACYBABBzZXJpYWxWZXJzaW9uVUlEAQABSgEADUNvbnN0YW50VmFsdWUFrSCT85Hd7z4BAAY8aW5pdD4BAAMoKVYBAARDb2RlAQAPTGluZU51bWJlclRhYmxlAQASTG9jYWxWYXJpYWJsZVRhYmxlAQAEdGhpcwEAE1N0dWJUcmFuc2xldFBheWxvYWQBAAxJbm5lckNsYXNzZXMBADVMeXNvc2VyaWFsL3BheWxvYWRzL3V0aWwvR2FkZ2V0cyRTdHViVHJhbnNsZXRQYXlsb2FkOwEACXRyYW5zZm9ybQEAcihMY29tL3N1bi9vcmcvYXBhY2hlL3hhbGFuL2ludGVybmFsL3hzbHRjL0RPTTtbTGNvbS9zdW4vb3JnL2FwYWNoZS94bWwvaW50ZXJuYWwvc2VyaWFsaXplci9TZXJpYWxpemF0aW9uSGFuZGxlcjspVgEACGRvY3VtZW50AQAtTGNvbS9zdW4vb3JnL2FwYWNoZS94YWxhbi9pbnRlcm5hbC94c2x0Yy9ET007AQAIaGFuZGxlcnMBAEJbTGNvbS9zdW4vb3JnL2FwYWNoZS94bWwvaW50ZXJuYWwvc2VyaWFsaXplci9TZXJpYWxpemF0aW9uSGFuZGxlcjsBAApFeGNlcHRpb25zBwAnAQCmKExjb20vc3VuL29yZy9hcGFjaGUveGFsYW4vaW50ZXJuYWwveHNsdGMvRE9NO0xjb20vc3VuL29yZy9hcGFjaGUveG1sL2ludGVybmFsL2R0bS9EVE1BeGlzSXRlcmF0b3I7TGNvbS9zdW4vb3JnL2FwYWNoZS94bWwvaW50ZXJuYWwvc2VyaWFsaXplci9TZXJpYWxpemF0aW9uSGFuZGxlcjspVgEACGl0ZXJhdG9yAQA1TGNvbS9zdW4vb3JnL2FwYWNoZS94bWwvaW50ZXJuYWwvZHRtL0RUTUF4aXNJdGVyYXRvcjsBAAdoYW5kbGVyAQBBTGNvbS9zdW4vb3JnL2FwYWNoZS94bWwvaW50ZXJuYWwvc2VyaWFsaXplci9TZXJpYWxpemF0aW9uSGFuZGxlcjsBAApTb3VyY2VGaWxlAQAMR2FkZ2V0cy5qYXZhDAAKAAsHACgBADN5c29zZXJpYWwvcGF5bG9hZHMvdXRpbC9HYWRnZXRzJFN0dWJUcmFuc2xldFBheWxvYWQBAEBjb20vc3VuL29yZy9hcGFjaGUveGFsYW4vaW50ZXJuYWwveHNsdGMvcnVudGltZS9BYnN0cmFjdFRyYW5zbGV0AQAUamF2YS9pby9TZXJpYWxpemFibGUBADljb20vc3VuL29yZy9hcGFjaGUveGFsYW4vaW50ZXJuYWwveHNsdGMvVHJhbnNsZXRFeGNlcHRpb24BAB95c29zZXJpYWwvcGF5bG9hZHMvdXRpbC9HYWRnZXRzAQAIPGNsaW5pdD4BABFqYXZhL2xhbmcvUnVudGltZQcAKgEACmdldFJ1bnRpbWUBABUoKUxqYXZhL2xhbmcvUnVudGltZTsMACwALQoAKwAuAQArdG91Y2ggL3RtcC9aS19ERVNFUklBTElaQVRJT05fVlVMTkVSQUJJTElUWQgAMAEABGV4ZWMBACcoTGphdmEvbGFuZy9TdHJpbmc7KUxqYXZhL2xhbmcvUHJvY2VzczsMADIAMwoAKwA0AQANU3RhY2tNYXBUYWJsZQEAHnlzb3NlcmlhbC9Qd25lcjExNTM4MjYwNDMyOTA1MQEAIEx5c29zZXJpYWwvUHduZXIxMTUzODI2MDQzMjkwNTE7ACEAAgADAAEABAABABoABQAGAAEABwAAAAIACAAEAAEACgALAAEADAAAAC8AAQABAAAABSq3AAGxAAAAAgANAAAABgABAAAALwAOAAAADAABAAAABQAPADgAAAABABMAFAACAAwAAAA/AAAAAwAAAAGxAAAAAgANAAAABgABAAAANAAOAAAAIAADAAAAAQAPADgAAAAAAAEAFQAWAAEAAAABABcAGAACABkAAAAEAAEAGgABABMAGwACAAwAAABJAAAABAAAAAGxAAAAAgANAAAABgABAAAAOAAOAAAAKgAEAAAAAQAPADgAAAAAAAEAFQAWAAEAAAABABwAHQACAAAAAQAeAB8AAwAZAAAABAABABoACAApAAsAAQAMAAAAJAADAAIAAAAPpwADAUy4AC8SMbYANVexAAAAAQA2AAAAAwABAwACACAAAAACACEAEQAAAAoAAQACACMAEAAJdXEAfgAQAAAB1Mr+ur4AAAAyABsKAAMAFQcAFwcAGAcAGQEAEHNlcmlhbFZlcnNpb25VSUQBAAFKAQANQ29uc3RhbnRWYWx1ZQVx5mnuPG1HGAEABjxpbml0PgEAAygpVgEABENvZGUBAA9MaW5lTnVtYmVyVGFibGUBABJMb2NhbFZhcmlhYmxlVGFibGUBAAR0aGlzAQADRm9vAQAMSW5uZXJDbGFzc2VzAQAlTHlzb3NlcmlhbC9wYXlsb2Fkcy91dGlsL0dhZGdldHMkRm9vOwEAClNvdXJjZUZpbGUBAAxHYWRnZXRzLmphdmEMAAoACwcAGgEAI3lzb3NlcmlhbC9wYXlsb2Fkcy91dGlsL0dhZGdldHMkRm9vAQAQamF2YS9sYW5nL09iamVjdAEAFGphdmEvaW8vU2VyaWFsaXphYmxlAQAfeXNvc2VyaWFsL3BheWxvYWRzL3V0aWwvR2FkZ2V0cwAhAAIAAwABAAQAAQAaAAUABgABAAcAAAACAAgAAQABAAoACwABAAwAAAAvAAEAAQAAAAUqtwABsQAAAAIADQAAAAYAAQAAADwADgAAAAwAAQAAAAUADwASAAAAAgATAAAAAgAUABEAAAAKAAEAAgAWABAACXB0AARQd25ycHcBAHhxAH4ADXg="));
+    Assert.assertNull(confStore.retrieve());
+
+    if (!System.getProperty("os.name").startsWith("Windows")) {
+      for (int i = 0; i < 20; ++i) {
+        if (flagFile.exists()) {
+          continue;
+        }
+        Thread.sleep(100);
+      }
+
+      Assert.assertFalse("The file '" + DESERIALIZATION_VULNERABILITY_FILEPATH +
+          "' should not have been created by deserialization attack", flagFile.exists());
+    }
+  }
+
   @Override
   public YarnConfigurationStore createConfStore() {
     return new ZKConfigurationStore();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index 5d68482929edf..4eea51e819729 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
 
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableTable;
+
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java
similarity index 51%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java
index c71d2bf91343a..d886641408e78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -6,35 +6,37 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
  *     http://www.apache.org/licenses/LICENSE-2.0
- *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement;
 
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-
-public class TestGuaranteedOrZeroCapacityOverTimePolicy {
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.spy;
 
+public class TestDeactivatedLeafQueuesByLabel {
   @Test
-  public void testGetMaxLeavesToBeActivated()
-      throws SchedulerDynamicEditException {
-    GuaranteedOrZeroCapacityOverTimePolicy policy =
-        new GuaranteedOrZeroCapacityOverTimePolicy();
+  public void testGetMaxLeavesToBeActivated() {
+    DeactivatedLeafQueuesByLabel d1 = spy(DeactivatedLeafQueuesByLabel.class);
+    d1.setAvailableCapacity(0.17f);
+    d1.setLeafQueueTemplateAbsoluteCapacity(0.03f);
+    assertEquals(1, d1.getMaxLeavesToBeActivated(1));
+
+    DeactivatedLeafQueuesByLabel d2 = spy(DeactivatedLeafQueuesByLabel.class);
+    d2.setAvailableCapacity(0.17f);
+    d2.setLeafQueueTemplateAbsoluteCapacity(0.03f);
+    assertEquals(5, d2.getMaxLeavesToBeActivated(7));
 
-    assertEquals(1, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 1));
-    assertEquals(5, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 7));
-    assertEquals(0, policy.getMaxLeavesToBeActivated(0, 0.03f, 10));
+    DeactivatedLeafQueuesByLabel d3 = spy(DeactivatedLeafQueuesByLabel.class);
+    d3.setAvailableCapacity(0f);
+    d3.setLeafQueueTemplateAbsoluteCapacity(0.03f);
+    assertEquals(0, d3.getMaxLeavesToBeActivated(10));
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
index eedd9de93bae4..c8ebe6f8f2067 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/TestNodeQueueLoadMonitor.java
@@ -18,17 +18,29 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed;
 
+import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
 import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
 import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -36,8 +48,18 @@
  */
 public class TestNodeQueueLoadMonitor {
 
+  // Extra resource type to test that all resource dimensions are considered
+  private static final String NETWORK_RESOURCE = "network";
   private final static int DEFAULT_MAX_QUEUE_LENGTH = 200;
 
+  // Note: The following variables are private static resources
+  // re-initialized on each test because resource dimensions considered
+  // are initialized in a static method.
+  // Declaring them as static final will "lock-in" resource dimensions and
+  // disallow specification of a new resource dimension ("network") in tests.
+  private static Resource defaultResourceRequested;
+  private static Resource defaultCapacity;
+
   static class FakeNodeId extends NodeId {
     final String host;
     final int port;
@@ -70,6 +92,44 @@ public String toString() {
     }
   }
 
+  private static Resource newResourceInstance(long memory, int vCores) {
+    return newResourceInstance(memory, vCores, 0L);
+  }
+
+  private static Resource newResourceInstance(
+      final long memory, final int vCores, final long network) {
+    return Resource.newInstance(memory, vCores,
+        ImmutableMap.of(NETWORK_RESOURCE, network));
+  }
+
+  private static long getNetworkResourceValue(final Resource resource) {
+    return resource.getResourceValue(NETWORK_RESOURCE);
+  }
+
+  public static void addNewTypesToResources(String... resourceTypes) {
+    // Initialize resource map
+    Map<String, ResourceInformation> riMap = new HashMap<>();
+
+    // Initialize mandatory resources
+    riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
+    riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
+
+    for (String newResource : resourceTypes) {
+      riMap.put(newResource, ResourceInformation
+          .newInstance(newResource, "", 0, ResourceTypes.COUNTABLE, 0,
+              Integer.MAX_VALUE));
+    }
+
+    ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+  }
+
+  @BeforeClass
+  public static void classSetUp() {
+    addNewTypesToResources(NETWORK_RESOURCE);
+    defaultResourceRequested = newResourceInstance(128, 1, 1);
+    defaultCapacity = newResourceInstance(1024, 8, 1000);
+  }
+
   @Test
   public void testWaitTimeSort() {
     NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor(
@@ -79,7 +139,6 @@ public void testWaitTimeSort() {
     selector.updateNode(createRMNode("h3", 3, 10, 10));
     selector.computeTask.run();
     List<NodeId> nodeIds = selector.selectNodes();
-    System.out.println("1-> " + nodeIds);
     Assert.assertEquals("h2:2", nodeIds.get(0).toString());
     Assert.assertEquals("h3:3", nodeIds.get(1).toString());
     Assert.assertEquals("h1:1", nodeIds.get(2).toString());
@@ -88,7 +147,6 @@ public void testWaitTimeSort() {
     selector.updateNode(createRMNode("h3", 3, 2, 10));
     selector.computeTask.run();
     nodeIds = selector.selectNodes();
-    System.out.println("2-> "+ nodeIds);
     Assert.assertEquals("h3:3", nodeIds.get(0).toString());
     Assert.assertEquals("h2:2", nodeIds.get(1).toString());
     Assert.assertEquals("h1:1", nodeIds.get(2).toString());
@@ -97,7 +155,6 @@ public void testWaitTimeSort() {
     selector.updateNode(createRMNode("h4", 4, -1, 10));
     selector.computeTask.run();
     nodeIds = selector.selectNodes();
-    System.out.println("3-> "+ nodeIds);
     // No change
     Assert.assertEquals("h3:3", nodeIds.get(0).toString());
     Assert.assertEquals("h2:2", nodeIds.get(1).toString());
@@ -186,6 +243,208 @@ public void testQueueLengthSort() {
     Assert.assertEquals("h4:4", nodeIds.get(2).toString());
   }
 
+  @Test
+  public void testQueueLengthThenResourcesSort() {
+    NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor(
+        NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH_THEN_RESOURCES);
+
+    // Node and queue sizes were selected such that we can determine the
+    // order of these nodes in the selectNodes call deterministically
+    // h2 -> h1 -> h3 -> h4
+    selector.updateNode(createRMNode(
+        "h1", 1, -1, 0,
+        Resources.multiply(defaultResourceRequested, 3), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 0,
+        Resources.multiply(defaultResourceRequested, 2), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h3", 3, -1, 5,
+        Resources.multiply(defaultResourceRequested, 3), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h4", 4, -1, 10,
+        Resources.multiply(defaultResourceRequested, 2), defaultCapacity));
+    selector.computeTask.run();
+    List<NodeId> nodeIds = selector.selectNodes();
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(1).toString());
+    Assert.assertEquals("h3:3", nodeIds.get(2).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(3).toString());
+
+    // Now update node3
+    // node3 should now rank after node4 since it has the same queue length
+    // but less resources available
+    selector.updateNode(createRMNode(
+        "h3", 3, -1, 10,
+        Resources.multiply(defaultResourceRequested, 3), defaultCapacity));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(1).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(2).toString());
+    Assert.assertEquals("h3:3", nodeIds.get(3).toString());
+
+    // Now update h3 and fill its queue -- it should no longer be available
+    selector.updateNode(createRMNode("h3", 3, -1,
+        DEFAULT_MAX_QUEUE_LENGTH));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    // h3 is queued up, so we should only have 3 nodes left
+    Assert.assertEquals(3, nodeIds.size());
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(1).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(2).toString());
+
+    // Now update h2 to Decommissioning state
+    selector.updateNode(createRMNode("h2", 2, -1,
+        5, NodeState.DECOMMISSIONING));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    // h2 is decommissioned, and h3 is full, so we should only have 2 nodes
+    Assert.assertEquals(2, nodeIds.size());
+    Assert.assertEquals("h1:1", nodeIds.get(0).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(1).toString());
+
+    // Now update h2 back to Running state
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 0,
+        Resources.multiply(defaultResourceRequested, 2), defaultCapacity));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals(3, nodeIds.size());
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(1).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(2).toString());
+
+    // Now update h2 to have a zero queue capacity.
+    // Make sure that here it is still in the pool.
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 0, 0,
+        Resources.multiply(defaultResourceRequested, 2),
+        defaultCapacity));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals(3, nodeIds.size());
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(1).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(2).toString());
+
+    // Now update h2 to have a positive queue length but a zero queue capacity.
+    // Make sure that here it is no longer in the pool.
+    // Need to first remove the node, because node capacity is not updated.
+    selector.removeNode(createRMNode(
+        "h2", 2, -1, 0, 0,
+        Resources.multiply(defaultResourceRequested, 2),
+        defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 1, 0,
+        Resources.multiply(defaultResourceRequested, 2),
+        defaultCapacity));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals(2, nodeIds.size());
+    Assert.assertEquals("h1:1", nodeIds.get(0).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(1).toString());
+  }
+
+  /**
+   * Tests that when using QUEUE_LENGTH_THEN_RESOURCES decrements the amount
+   * of resources on the internal {@link ClusterNode} representation.
+   */
+  @Test
+  public void testQueueLengthThenResourcesDecrementsAvailable() {
+    NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor(
+        NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH_THEN_RESOURCES);
+    RMNode node = createRMNode("h1", 1, -1, 0);
+    selector.addNode(null, node);
+    selector.updateNode(node);
+    selector.updateSortedNodes();
+
+    ClusterNode clusterNode = selector.getClusterNodes().get(node.getNodeID());
+    Assert.assertEquals(Resources.none(),
+        clusterNode.getAllocatedResource());
+
+    // Has enough resources
+    RMNode selectedNode = selector.selectAnyNode(
+        Collections.emptySet(), defaultResourceRequested);
+    Assert.assertNotNull(selectedNode);
+    Assert.assertEquals(node.getNodeID(), selectedNode.getNodeID());
+
+    clusterNode = selector.getClusterNodes().get(node.getNodeID());
+    Assert.assertEquals(defaultResourceRequested,
+        clusterNode.getAllocatedResource());
+
+    // Does not have enough resources, but can queue
+    selectedNode = selector.selectAnyNode(
+        Collections.emptySet(), defaultCapacity);
+    Assert.assertNotNull(selectedNode);
+    Assert.assertEquals(node.getNodeID(), selectedNode.getNodeID());
+
+    clusterNode = selector.getClusterNodes().get(node.getNodeID());
+    Assert.assertEquals(1, clusterNode.getQueueLength());
+
+    // Does not have enough resources and cannot queue
+    selectedNode = selector.selectAnyNode(
+        Collections.emptySet(),
+        Resources.add(defaultResourceRequested, defaultCapacity));
+    Assert.assertNull(selectedNode);
+  }
+
+  @Test
+  public void testQueueLengthThenResourcesCapabilityChange() {
+    NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor(
+        NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH_THEN_RESOURCES);
+
+    // Node sizes were selected such that we can determine the
+    // order of these nodes in the selectNodes call deterministically
+    // h1 -> h2 -> h3 -> h4
+    selector.updateNode(createRMNode(
+        "h1", 1, -1, 0,
+        Resources.multiply(defaultResourceRequested, 1), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 0,
+        Resources.multiply(defaultResourceRequested, 2), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h3", 3, -1, 0,
+        Resources.multiply(defaultResourceRequested, 3), defaultCapacity));
+    selector.updateNode(createRMNode(
+        "h4", 4, -1, 0,
+        Resources.multiply(defaultResourceRequested, 4), defaultCapacity));
+    selector.computeTask.run();
+    List<NodeId> nodeIds = selector.selectNodes();
+    Assert.assertEquals("h1:1", nodeIds.get(0).toString());
+    Assert.assertEquals("h2:2", nodeIds.get(1).toString());
+    Assert.assertEquals("h3:3", nodeIds.get(2).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(3).toString());
+
+    // Now update node1 to have only defaultResourceRequested available
+    // by changing its capability to 2x defaultResourceReqeusted
+    // node1 should now rank last
+    selector.updateNode(createRMNode(
+        "h1", 1, -1, 0,
+        Resources.multiply(defaultResourceRequested, 1),
+        Resources.multiply(defaultResourceRequested, 2)));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals("h2:2", nodeIds.get(0).toString());
+    Assert.assertEquals("h3:3", nodeIds.get(1).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(2).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(3).toString());
+
+    // Now update node2 to have no resources available
+    // by changing its capability to 1x defaultResourceReqeusted
+    // node2 should now rank last
+    selector.updateNode(createRMNode(
+        "h2", 2, -1, 0,
+        Resources.multiply(defaultResourceRequested, 1),
+        Resources.multiply(defaultResourceRequested, 1)));
+    selector.computeTask.run();
+    nodeIds = selector.selectNodes();
+    Assert.assertEquals("h3:3", nodeIds.get(0).toString());
+    Assert.assertEquals("h4:4", nodeIds.get(1).toString());
+    Assert.assertEquals("h1:1", nodeIds.get(2).toString());
+    Assert.assertEquals("h2:2", nodeIds.get(3).toString());
+  }
+
   @Test
   public void testContainerQueuingLimit() {
     NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor(
@@ -254,18 +513,22 @@ public void testSelectLocalNode() {
     // basic test for selecting node which has queue length less
     // than queue capacity.
     Set<String> blacklist = new HashSet<>();
-    RMNode node = selector.selectLocalNode("h1", blacklist);
+    RMNode node = selector.selectLocalNode(
+        "h1", blacklist, defaultResourceRequested);
     Assert.assertEquals("h1", node.getHostName());
 
     // if node has been added to blacklist
     blacklist.add("h1");
-    node = selector.selectLocalNode("h1", blacklist);
+    node = selector.selectLocalNode(
+        "h1", blacklist, defaultResourceRequested);
     Assert.assertNull(node);
 
-    node = selector.selectLocalNode("h2", blacklist);
+    node = selector.selectLocalNode(
+        "h2", blacklist, defaultResourceRequested);
     Assert.assertNull(node);
 
-    node = selector.selectLocalNode("h3", blacklist);
+    node = selector.selectLocalNode(
+        "h3", blacklist, defaultResourceRequested);
     Assert.assertEquals("h3", node.getHostName());
   }
 
@@ -293,19 +556,23 @@ public void testSelectRackLocalNode() {
     // basic test for selecting node which has queue length less
     // than queue capacity.
     Set<String> blacklist = new HashSet<>();
-    RMNode node = selector.selectRackLocalNode("rack1", blacklist);
+    RMNode node = selector.selectRackLocalNode(
+        "rack1", blacklist, defaultResourceRequested);
     Assert.assertEquals("h1", node.getHostName());
 
     // if node has been added to blacklist
     blacklist.add("h1");
-    node = selector.selectRackLocalNode("rack1", blacklist);
+    node = selector.selectRackLocalNode(
+        "rack1", blacklist, defaultResourceRequested);
     Assert.assertNull(node);
 
-    node = selector.selectRackLocalNode("rack2", blacklist);
+    node = selector.selectRackLocalNode(
+        "rack2", blacklist, defaultResourceRequested);
     Assert.assertEquals("h3", node.getHostName());
 
     blacklist.add("h3");
-    node = selector.selectRackLocalNode("rack2", blacklist);
+    node = selector.selectRackLocalNode(
+        "rack2", blacklist, defaultResourceRequested);
     Assert.assertNull(node);
   }
 
@@ -337,20 +604,217 @@ public void testSelectAnyNode() {
     // basic test for selecting node which has queue length
     // less than queue capacity.
     Set<String> blacklist = new HashSet<>();
-    RMNode node = selector.selectAnyNode(blacklist);
+    RMNode node = selector.selectAnyNode(blacklist, defaultResourceRequested);
     Assert.assertTrue(node.getHostName().equals("h1") ||
         node.getHostName().equals("h3"));
 
     // if node has been added to blacklist
     blacklist.add("h1");
-    node = selector.selectAnyNode(blacklist);
+    node = selector.selectAnyNode(blacklist, defaultResourceRequested);
     Assert.assertEquals("h3", node.getHostName());
 
     blacklist.add("h3");
-    node = selector.selectAnyNode(blacklist);
+    node = selector.selectAnyNode(blacklist, defaultResourceRequested);
     Assert.assertNull(node);
   }
 
+  @Test
+  public void testQueueLengthThenResourcesComparator() {
+    NodeQueueLoadMonitor.LoadComparator comparator =
+        NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH_THEN_RESOURCES;
+
+    NodeId n1 = new FakeNodeId("n1", 5000);
+    NodeId n2 = new FakeNodeId("n2", 5000);
+
+    // Case 1: larger available cores should be ranked first
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(6, 6))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn1, cn2) < 0);
+    }
+
+    // Case 2: Shorter queue should be ranked first before comparing resources
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(5);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(3, 3))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn1, cn2) < 0);
+    }
+
+    // Case 3: No capability vs with capability,
+    // with capability should come first
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(Resources.none())
+              .setCapability(newResourceInstance(1, 1, 1000))
+              .setQueueLength(5);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(Resources.none())
+              .setCapability(Resources.none())
+              .setQueueLength(5);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn1, cn2) < 0);
+    }
+
+    // Case 4: Compare same values
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertEquals(0, comparator.compare(cn1, cn2));
+    }
+
+    // Case 5: If ratio is the same, compare raw values
+    // by VCores first, then memory
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(6, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 6))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      // Both are 60% allocated, but CN1 has 5 avail VCores, CN2 only has 4
+      Assert.assertTrue(comparator.compare(cn1, cn2) < 0);
+    }
+
+    // Case 6: by VCores absolute value
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 6))
+              .setCapability(newResourceInstance(10, 12, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn2, cn1) < 0);
+    }
+
+    // Case 7: by memory absolute value
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 5))
+              .setCapability(newResourceInstance(10, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(6, 5))
+              .setCapability(newResourceInstance(12, 10, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn2, cn1) < 0);
+    }
+
+    // Case 8: Memory should be more constraining in the overall cluster,
+    // so rank the node with less allocated memory first
+    {
+      ClusterNode.Properties cn1Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(5, 11))
+              .setCapability(newResourceInstance(10, 100, 1000))
+              .setQueueLength(10);
+      ClusterNode cn1 = new ClusterNode(n1);
+      cn1.setProperties(cn1Props);
+
+      ClusterNode.Properties cn2Props =
+          ClusterNode.Properties.newInstance()
+              .setAllocatedResource(newResourceInstance(6, 10))
+              .setCapability(newResourceInstance(10, 100, 1000))
+              .setQueueLength(10);
+      ClusterNode cn2 = new ClusterNode(n2);
+      cn2.setProperties(cn2Props);
+
+      comparator.setClusterResource(
+          Resources.add(cn1.getCapability(), cn2.getCapability()));
+      Assert.assertTrue(comparator.compare(cn1, cn2) < 0);
+    }
+  }
+
   private RMNode createRMNode(String host, int port,
       int waitTime, int queueLength) {
     return createRMNode(host, port, waitTime, queueLength,
@@ -377,12 +841,40 @@ private RMNode createRMNode(String host, int port, String rack,
 
   private RMNode createRMNode(String host, int port, String rack,
       int waitTime, int queueLength, int queueCapacity, NodeState state) {
+    return createRMNode(host, port, rack, waitTime, queueLength, queueCapacity,
+        state, Resources.none(), defaultCapacity);
+  }
+
+  private RMNode createRMNode(
+      String host, int port, int waitTime, int queueLength,
+      Resource allocatedResource, Resource nodeResource) {
+    return createRMNode(host, port, waitTime, queueLength,
+        DEFAULT_MAX_QUEUE_LENGTH, allocatedResource, nodeResource);
+  }
+
+  private RMNode createRMNode(
+      String host, int port, int waitTime, int queueLength, int queueCapacity,
+      Resource allocatedResource, Resource nodeResource) {
+    return createRMNode(host, port, "default", waitTime, queueLength,
+        queueCapacity, NodeState.RUNNING, allocatedResource, nodeResource);
+  }
+
+  @SuppressWarnings("parameternumber")
+  private RMNode createRMNode(String host, int port, String rack,
+      int waitTime, int queueLength, int queueCapacity, NodeState state,
+      Resource allocatedResource, Resource nodeResource) {
     RMNode node1 = Mockito.mock(RMNode.class);
     NodeId nID1 = new FakeNodeId(host, port);
     Mockito.when(node1.getHostName()).thenReturn(host);
     Mockito.when(node1.getRackName()).thenReturn(rack);
+    Mockito.when(node1.getNode()).thenReturn(new NodeBase("/" + host));
     Mockito.when(node1.getNodeID()).thenReturn(nID1);
     Mockito.when(node1.getState()).thenReturn(state);
+    Mockito.when(node1.getTotalCapability()).thenReturn(nodeResource);
+    Mockito.when(node1.getNodeUtilization()).thenReturn(
+        ResourceUtilization.newInstance(0, 0, 0));
+    Mockito.when(node1.getAllocatedContainerResource()).thenReturn(
+        allocatedResource);
     OpportunisticContainersStatus status1 =
         Mockito.mock(OpportunisticContainersStatus.class);
     Mockito.when(status1.getEstimatedQueueWaitTime())
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 9aa9d5c7eb560..aeefbcd8202fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -38,7 +38,6 @@
 import org.junit.Test;
 
 import java.io.File;
-import java.io.IOException;
 
 /**
  * Test class to verify identification of app starvation.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestVisitedResourceRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestVisitedResourceRequestTracker.java
index 07b849806fe42..b36fc45c63246 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestVisitedResourceRequestTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestVisitedResourceRequestTracker.java
@@ -23,7 +23,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker;
 import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Assert;
+
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import org.junit.Test;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
index 7ec2c01ec2512..62bc7124c4b9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFifoOrderingPolicy.java
@@ -18,16 +18,19 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
 
-import java.util.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
-import org.junit.Assert;
-import org.junit.Test;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
 
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.junit.Assert;
+import org.junit.Test;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class TestFifoOrderingPolicy {
+public
+class TestFifoOrderingPolicy {
   
   @Test
   public void testFifoOrderingPolicy() {
@@ -36,13 +39,17 @@ public void testFifoOrderingPolicy() {
     MockSchedulableEntity r1 = new MockSchedulableEntity();
     MockSchedulableEntity r2 = new MockSchedulableEntity();
 
-    assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(0);
+    assertEquals("The comparator should return 0 because the entities are created with " +
+            "the same values.", 0,
+        policy.getComparator().compare(r1, r2));
     
     r1.setSerial(1);
-    assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(1);
+    assertEquals("The lhs entity has a larger serial, the comparator return " +
+            "value should be 1.", 1, policy.getComparator().compare(r1, r2));
     
     r2.setSerial(2);
-    assertThat(policy.getComparator().compare(r1, r2)).isEqualTo(-1);
+    Assert.assertEquals("The rhs entity has a larger serial, the comparator return " +
+        "value should be -1.", -1, policy.getComparator().compare(r1, r2));
   }
   
   @Test
@@ -63,46 +70,80 @@ public void testIterators() {
     schedOrder.addSchedulableEntity(msp3);
     
     //Assignment, oldest to youngest
-    checkSerials(schedOrder.getAssignmentIterator(IteratorSelector.EMPTY_ITERATOR_SELECTOR), new long[]{1, 2, 3});
+    checkSerials(Arrays.asList(1L, 2L, 3L), schedOrder.getAssignmentIterator(
+        IteratorSelector.EMPTY_ITERATOR_SELECTOR));
     
     //Preemption, youngest to oldest
-    checkSerials(schedOrder.getPreemptionIterator(), new long[]{3, 2, 1});
+    checkSerials(Arrays.asList(3L, 2L, 1L), schedOrder.getPreemptionIterator());
   }
   
-  public void checkSerials(Iterator<MockSchedulableEntity> si, 
-      long[] serials) {
-    for (int i = 0;i < serials.length;i++) {
-      Assert.assertEquals(si.next().getSerial(), 
-        serials[i]);
+  public void checkSerials(List<Long> expectedSerials, Iterator<MockSchedulableEntity>
+      actualSerialIterator) {
+    for (long expectedSerial : expectedSerials) {
+      assertEquals(expectedSerial, actualSerialIterator.next().getSerial());
     }
   }
   
   @Test
-  public void testFifoOrderingPolicyAlongWithPriorty() {
+  public void testFifoOrderingPolicyAlongWithPriority() {
     FifoOrderingPolicy<MockSchedulableEntity> policy =
         new FifoOrderingPolicy<MockSchedulableEntity>();
     MockSchedulableEntity r1 = new MockSchedulableEntity();
     MockSchedulableEntity r2 = new MockSchedulableEntity();
 
-    Priority p1 = Priority.newInstance(1);
-    Priority p2 = Priority.newInstance(0);
+    assertEquals("Both r1 and r2 priority is null, the comparator should return 0.", 0,
+        policy.getComparator().compare(r1, r2));
 
-    // Both r1 and r1 priority is null
-    Assert.assertEquals(0, policy.getComparator().compare(r1, r2));
+    Priority p2 = Priority.newInstance(0);
 
     // r1 is null and r2 is not null
     r2.setApplicationPriority(p2);
-    Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+    Assert.assertTrue("The priority of r1 is null, the priority of r2 is not null, " +
+            "the comparator should return a negative value.",
+        policy.getComparator().compare(r1, r2) < 0);
+
+    Priority p1 = Priority.newInstance(1);
 
     // r1 is not null and r2 is null
-    r2.setApplicationPriority(null);
     r1.setApplicationPriority(p1);
-    Assert.assertEquals(1, policy.getComparator().compare(r1, r2));
+    r2.setApplicationPriority(null);
+    assertTrue("The priority of r1 is not null, the priority of r2 is null," +
+            "the comparator should return a positive value.",
+        policy.getComparator().compare(r1, r2) > 0);
 
     // r1 is not null and r2 is not null
     r1.setApplicationPriority(p1);
     r2.setApplicationPriority(p2);
-    Assert.assertEquals(-1, policy.getComparator().compare(r1, r2));
+    Assert.assertTrue("Both priorities are not null, the r1 has higher priority, " +
+            "the result should be a negative value.",
+        policy.getComparator().compare(r1, r2) < 0);
   }
 
+  @Test
+  public void testOrderingUsingAppSubmitTime() {
+    FifoOrderingPolicy<MockSchedulableEntity> policy =
+        new FifoOrderingPolicy<MockSchedulableEntity>();
+    MockSchedulableEntity r1 = new MockSchedulableEntity();
+    MockSchedulableEntity r2 = new MockSchedulableEntity();
+
+    // R1, R2 has been started at same time
+    assertEquals(r1.getStartTime(), r2.getStartTime());
+
+    // No changes, equal
+    assertEquals("The submit times are the same, the comparator should return 0.", 0,
+        policy.getComparator().compare(r1, r2));
+
+    // R2 has been started after R1
+    r1.setStartTime(5);
+    r2.setStartTime(10);
+    Assert.assertTrue("r2 was started after r1, " +
+            "the comparator should return a negative value.",
+        policy.getComparator().compare(r1, r2) < 0);
+
+    // R1 has been started after R2
+    r1.setStartTime(10);
+    r2.setStartTime(5);
+    Assert.assertTrue("r2 was started before r1, the comparator should return a positive value.",
+        policy.getComparator().compare(r1, r2) > 0);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenIdentifierForTest.java
index 9763933169f93..47abae884a0d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenIdentifierForTest.java
@@ -23,10 +23,7 @@
 import java.io.IOException;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
index 891c8d6c325bf..dfa9fcdffb9b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
@@ -21,7 +21,6 @@
 import java.io.PrintWriter;
 import java.util.Map;
 
-import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 673fbbe2ec079..b416947c55012 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -57,6 +57,7 @@
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -75,6 +76,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -309,7 +311,7 @@ public void testInfoDefault() throws JSONException, Exception {
   }
 
   public void verifyClusterInfoXML(String xml) throws JSONException, Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -435,7 +437,7 @@ public void testClusterMetricsXML() throws JSONException, Exception {
 
   public void verifyClusterMetricsXML(String xml) throws JSONException,
       Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -606,7 +608,7 @@ public void testClusterSchedulerFifoXML() throws JSONException, Exception {
 
   public void verifySchedulerFifoXML(String xml) throws JSONException,
       Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -1003,8 +1005,8 @@ public void testValidateAndGetSchedulerConfigurationInvalidScheduler()
     Assert.assertEquals(Status.BAD_REQUEST
             .getStatusCode(), response.getStatus());
     Assert.assertTrue(response.getEntity().toString()
-            .contains("Configuration change validation only supported by"
-                    +" MutableConfScheduler."));
+            .contains(String.format("Configuration change validation only supported by %s.",
+                MutableConfScheduler.class.getSimpleName())));
   }
 
   @Test
@@ -1098,4 +1100,61 @@ private RMWebServices prepareWebServiceForValidation(
     return  webService;
   }
 
-}
+  @Test
+  public void testClusterSchedulerOverviewFifo() throws JSONException, Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+
+    assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    JSONObject json = response.getEntity(JSONObject.class);
+    verifyClusterSchedulerOverView(json, "Fifo Scheduler");
+  }
+
+  public static void verifyClusterSchedulerOverView(
+      JSONObject json, String expectedSchedulerType) throws Exception {
+
+    // why json contains 8 elements because we defined 8 fields
+    assertEquals("incorrect number of elements in: " + json, 8, json.length());
+
+    // 1.Verify that the schedulerType is as expected
+    String schedulerType = json.getString("schedulerType");
+    assertEquals(expectedSchedulerType, schedulerType);
+
+    // 2.Verify that schedulingResourceType is as expected
+    String schedulingResourceType = json.getString("schedulingResourceType");
+    assertEquals("memory-mb (unit=Mi),vcores", schedulingResourceType);
+
+    // 3.Verify that minimumAllocation is as expected
+    JSONObject minimumAllocation = json.getJSONObject("minimumAllocation");
+    String minMemory = minimumAllocation.getString("memory");
+    String minVCores = minimumAllocation.getString("vCores");
+    assertEquals("1024", minMemory);
+    assertEquals("1", minVCores);
+
+    // 4.Verify that maximumAllocation is as expected
+    JSONObject maximumAllocation = json.getJSONObject("maximumAllocation");
+    String maxMemory = maximumAllocation.getString("memory");
+    String maxVCores = maximumAllocation.getString("vCores");
+    assertEquals("8192", maxMemory);
+    assertEquals("4", maxVCores);
+
+    // 5.Verify that schedulerBusy is as expected
+    int schedulerBusy = json.getInt("schedulerBusy");
+    assertEquals(-1, schedulerBusy);
+
+    // 6.Verify that rmDispatcherEventQueueSize is as expected
+    int rmDispatcherEventQueueSize = json.getInt("rmDispatcherEventQueueSize");
+    assertEquals(0, rmDispatcherEventQueueSize);
+
+    // 7.Verify that schedulerDispatcherEventQueueSize is as expected
+    int schedulerDispatcherEventQueueSize = json.getInt("schedulerDispatcherEventQueueSize");
+    assertEquals(0, schedulerDispatcherEventQueueSize);
+
+    // 8.Verify that applicationPriority is as expected
+    int applicationPriority = json.getInt("applicationPriority");
+    assertEquals(0, applicationPriority);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
index 102f13897fc79..4ad37ef8d9d2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
@@ -25,6 +25,7 @@
 import com.sun.jersey.test.framework.WebAppDescriptor;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -395,7 +396,7 @@ public void testAppAttemptsXML() throws Exception {
             response.getType().toString());
     String xml = response.getEntity(String.class);
 
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index e22311ccbda65..964aab6913c8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -46,6 +47,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -82,6 +84,16 @@ public class TestRMWebServicesApps extends JerseyTestBase {
   private static final int CONTAINER_MB = 1024;
 
   private static class WebServletModule extends ServletModule {
+    private final Class<? extends AbstractYarnScheduler> scheduler;
+
+    public WebServletModule() {
+      this.scheduler = FifoScheduler.class;
+    }
+
+    public WebServletModule(Class<? extends AbstractYarnScheduler> scheduler) {
+      this.scheduler = scheduler;
+    }
+
     @Override
     protected void configureServlets() {
       bind(JAXBContextResolver.class);
@@ -90,8 +102,9 @@ protected void configureServlets() {
       Configuration conf = new Configuration();
       conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
           YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
-      conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+      conf.setClass(YarnConfiguration.RM_SCHEDULER, scheduler,
           ResourceScheduler.class);
+      conf.set(YarnConfiguration.RM_CLUSTER_ID, "subCluster1");
       rm = new MockRM(conf);
       bind(ResourceManager.class).toInstance(rm);
       serve("/*").with(GuiceContainer.class);
@@ -177,7 +190,7 @@ public void testAppsXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -211,7 +224,7 @@ public void testRunningApp() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -252,7 +265,7 @@ public void testAppsXMLMulti() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -1712,7 +1725,7 @@ public void testSingleAppsXML() throws JSONException, Exception {
         response.getType().toString());
     String xml = response.getEntity(String.class);
 
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -1794,7 +1807,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq)
   public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
       throws JSONException, Exception {
 
-    int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0);
+    int expectedNumberOfElements = 41 + (hasResourceReqs ? 2 : 0);
     String appNodeLabelExpression = null;
     String amNodeLabelExpression = null;
     if (app.getApplicationSubmissionContext()
@@ -1814,6 +1827,7 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
     }
     assertEquals("incorrect number of elements", expectedNumberOfElements,
         info.length());
+    assertEquals("rmClusterId is incorrect", "subCluster1", info.getString("rmClusterId"));
     verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
         info.getString("name"), info.getString("applicationType"),
         info.getString("queue"), info.getInt("priority"),
@@ -1970,5 +1984,142 @@ public void verifyResourceRequestsGeneric(ResourceRequest request,
         enforceExecutionType);
   }
 
+  @Test
+  public void testAppsQueryByQueueShortname() throws Exception {
+    GuiceServletConfig.setInjector(
+        Guice.createInjector(new WebServletModule(CapacityScheduler.class)));
+
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    //YARN-11114 - Finished apps can only be queried with exactly the
+    // same queue name that the app is submitted to.
+    //As the queue is 'root.default'  and the query is 'default' here,
+    // this app won't be returned.
+    RMApp finishedApp1 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("root.default")
+            .build());
+    RMApp finishedApp2 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("default")
+            .build());
+
+    RMApp runningApp1 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("default")
+            .build());
+    RMApp runningApp2 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("root.default")
+            .build());
+    amNodeManager.nodeHeartbeat(true);
+    finishApp(amNodeManager, finishedApp1);
+    amNodeManager.nodeHeartbeat(true);
+    finishApp(amNodeManager, finishedApp2);
+
+    WebResource r = resource();
+
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("apps")
+        .queryParam("queue", "default")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+
+    JSONArray array = apps.getJSONArray("app");
+
+    Set<String> appIds = getApplicationIds(array);
+    assertTrue("Running app 1 should be in the result list!",
+        appIds.contains(runningApp1.getApplicationId().toString()));
+    assertTrue("Running app 2 should be in the result list!",
+        appIds.contains(runningApp2.getApplicationId().toString()));
+    assertFalse("Finished app 1 should not be in the result list " +
+            "as it was submitted to 'root.default' but the query is for 'default'",
+        appIds.contains(finishedApp1.getApplicationId().toString()));
+    assertTrue("Finished app 2 should be in the result list " +
+            "as it was submitted to 'default' and the query is exactly for 'default'",
+        appIds.contains(finishedApp2.getApplicationId().toString()));
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    rm.stop();
+  }
+
+  @Test
+  public void testAppsQueryByQueueFullname() throws Exception {
+    GuiceServletConfig.setInjector(
+        Guice.createInjector(new WebServletModule(CapacityScheduler.class)));
+
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    RMApp finishedApp1 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("root.default")
+            .build());
+    //YARN-11114 - Finished apps can only be queried with exactly the
+    // same queue name that the app is submitted to.
+    //As the queue is 'default'  and the query is 'root.default' here,
+    // this app won't be returned,
+    RMApp finishedApp2 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("default")
+            .build());
+
+    RMApp runningApp1 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("default")
+            .build());
+    RMApp runningApp2 = MockRMAppSubmitter.submit(rm,
+        MockRMAppSubmissionData.Builder
+            .createWithMemory(CONTAINER_MB, rm)
+            .withQueue("root.default")
+            .build());
+    amNodeManager.nodeHeartbeat(true);
+    finishApp(amNodeManager, finishedApp1);
+
+    amNodeManager.nodeHeartbeat(true);
+    finishApp(amNodeManager, finishedApp2);
+
+    WebResource r = resource();
+
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("apps")
+        .queryParam("queue", "root.default")
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+
+    JSONArray array = apps.getJSONArray("app");
+
+    Set<String> appIds = getApplicationIds(array);
+    assertTrue("Running app 1 should be in the result list!",
+        appIds.contains(runningApp1.getApplicationId().toString()));
+    assertTrue("Running app 2 should be in the result list!",
+        appIds.contains(runningApp2.getApplicationId().toString()));
+    assertTrue("Finished app 1 should be in the result list, " +
+            "as it was submitted to 'root.default' and the query is exactly for 'root.default'!",
+        appIds.contains(finishedApp1.getApplicationId().toString()));
+    assertFalse("Finished app 2 should not be in the result list, " +
+            "as it was submitted to 'default' but the query is for 'root.default'!",
+        appIds.contains(finishedApp2.getApplicationId().toString()));
+    assertEquals("incorrect number of elements", 3, array.length());
+
+    rm.stop();
+  }
+
 }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index f5f21aac24945..0c8f742fae246 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -56,6 +56,7 @@
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -532,7 +533,7 @@ protected static void verifyAppStateXML(ClientResponse response,
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -733,7 +734,7 @@ protected String validateGetNewApplicationJsonResponse(JSONObject json)
 
   protected String validateGetNewApplicationXMLResponse(String response)
       throws ParserConfigurationException, IOException, SAXException {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(response));
@@ -1299,7 +1300,7 @@ protected static void verifyAppPriorityXML(ClientResponse response,
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -1329,7 +1330,7 @@ protected static void verifyAppPriorityXML(ClientResponse response,
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -1466,7 +1467,7 @@ protected static void verifyAppTimeoutXML(ClientResponse response,
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index d4e7ca63f2282..ec65237fa6e17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -18,25 +18,39 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
 
+import java.io.BufferedReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
-
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -50,549 +64,145 @@
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
-import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.junit.Assert.assertEquals;
 
 public class TestRMWebServicesCapacitySched extends JerseyTestBase {
 
-  protected static MockRM rm;
-  protected static CapacitySchedulerConfiguration csConf;
-  protected static YarnConfiguration conf;
-
-  private class QueueInfo {
-    float capacity;
-    float usedCapacity;
-    float maxCapacity;
-    float absoluteCapacity;
-    float absoluteMaxCapacity;
-    float absoluteUsedCapacity;
-    int numApplications;
-    int maxParallelApps;
-    String queueName;
-    private String queuePath;
-    String state;
-    boolean isAbsoluteResource;
-    boolean autoCreateChildQueueEnabled;
-
-    public String getQueuePath() {
-      return queuePath;
-    }
-  }
+  private MockRM rm;
 
-  private class LeafQueueInfo extends QueueInfo {
-    int numActiveApplications;
-    int numPendingApplications;
-    int numContainers;
-    int maxApplications;
-    int maxApplicationsPerUser;
-    float userLimit;
-    float userLimitFactor;
-    long defaultApplicationLifetime;
-    long maxApplicationLifetime;
-  }
+  public static class WebServletModule extends ServletModule {
+    private final MockRM rm;
+
+    WebServletModule(MockRM rm) {
+      this.rm = rm;
+    }
 
-  private static class WebServletModule extends ServletModule {
     @Override
     protected void configureServlets() {
       bind(JAXBContextResolver.class);
       bind(RMWebServices.class);
       bind(GenericExceptionHandler.class);
-      csConf = new CapacitySchedulerConfiguration();
-      setupQueueConfiguration(csConf);
-      conf = new YarnConfiguration(csConf);
-      conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-		    ResourceScheduler.class);
-      conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
-          YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
-      rm = new MockRM(conf);
       bind(ResourceManager.class).toInstance(rm);
       serve("/*").with(GuiceContainer.class);
     }
   }
 
-  static {
+  public TestRMWebServicesCapacitySched() {
+    super(createWebAppDescriptor());
+  }
+
+  @Before
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    rm = createMockRM(new CapacitySchedulerConfiguration(
+        new Configuration(false)));
     GuiceServletConfig.setInjector(
-        Guice.createInjector(new WebServletModule()));
+        Guice.createInjector(new WebServletModule(rm)));
   }
 
-  private static void setupQueueConfiguration(
+  public static void setupQueueConfiguration(
       CapacitySchedulerConfiguration config) {
 
     // Define top-level queues
     config.setQueues(CapacitySchedulerConfiguration.ROOT,
-        new String[] {"a", "b"});
+        new String[] {"a", "b", "c"});
 
-    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
-    config.setCapacity(A, 10.5f);
-    config.setMaximumCapacity(A, 50);
-    config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(A) + MAX_PARALLEL_APPLICATIONS, 42);
+    final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+    config.setCapacity(a, 10.5f);
+    config.setMaximumCapacity(a, 50);
+    config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(a) + MAX_PARALLEL_APPLICATIONS, 42);
 
-    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
-    config.setCapacity(B, 89.5f);
+    final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+    config.setCapacity(b, 89.5f);
 
-    final String C = CapacitySchedulerConfiguration.ROOT + ".c";
-    config.setCapacity(C, "[memory=1024]");
+    final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+    config.setCapacity(c, "[memory=1024]");
 
     // Define 2nd-level queues
-    final String A1 = A + ".a1";
-    final String A2 = A + ".a2";
-    config.setQueues(A, new String[] {"a1", "a2"});
-    config.setCapacity(A1, 30);
-    config.setMaximumCapacity(A1, 50);
-    config.setMaximumLifetimePerQueue(A2, 100);
-    config.setDefaultLifetimePerQueue(A2, 50);
-
-    config.setUserLimitFactor(A1, 100.0f);
-    config.setCapacity(A2, 70);
-    config.setUserLimitFactor(A2, 100.0f);
-
-    final String B1 = B + ".b1";
-    final String B2 = B + ".b2";
-    final String B3 = B + ".b3";
-    config.setQueues(B, new String[] {"b1", "b2", "b3"});
-    config.setCapacity(B1, 60);
-    config.setUserLimitFactor(B1, 100.0f);
-    config.setCapacity(B2, 39.5f);
-    config.setUserLimitFactor(B2, 100.0f);
-    config.setCapacity(B3, 0.5f);
-    config.setUserLimitFactor(B3, 100.0f);
-
-    config.setQueues(A1, new String[] {"a1a", "a1b", "a1c"});
-    final String A1A = A1 + ".a1a";
-    config.setCapacity(A1A, 65);
-    final String A1B = A1 + ".a1b";
-    config.setCapacity(A1B, 15);
-    final String A1C = A1 + ".a1c";
-    config.setCapacity(A1C, 20);
-
-    config.setAutoCreateChildQueueEnabled(A1C, true);
-    config.setInt(PREFIX + A1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+    final String a1 = a + ".a1";
+    final String a2 = a + ".a2";
+    config.setQueues(a, new String[] {"a1", "a2"});
+    config.setCapacity(a1, 30);
+    config.setMaximumCapacity(a1, 50);
+    config.setMaximumLifetimePerQueue(a2, 100);
+    config.setDefaultLifetimePerQueue(a2, 50);
+
+    config.setUserLimitFactor(a1, 100.0f);
+    config.setCapacity(a2, 70);
+    config.setUserLimitFactor(a2, 100.0f);
+
+    final String b1 = b + ".b1";
+    final String b2 = b + ".b2";
+    final String b3 = b + ".b3";
+    config.setQueues(b, new String[] {"b1", "b2", "b3"});
+    config.setCapacity(b1, 60);
+    config.setUserLimitFactor(b1, 100.0f);
+    config.setCapacity(b2, 39.5f);
+    config.setUserLimitFactor(b2, 100.0f);
+    config.setCapacity(b3, 0.5f);
+    config.setUserLimitFactor(b3, 100.0f);
+
+    config.setQueues(a1, new String[] {"a1a", "a1b", "a1c"});
+    final String a1A = a1 + ".a1a";
+    config.setCapacity(a1A, 65);
+    final String a1B = a1 + ".a1b";
+    config.setCapacity(a1B, 15);
+    final String a1C = a1 + ".a1c";
+    config.setCapacity(a1C, 20);
+
+    config.setAutoCreateChildQueueEnabled(a1C, true);
+    config.setInt(PREFIX + a1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
         + DOT + CAPACITY, 50);
   }
 
-  @Before
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-    GuiceServletConfig.setInjector(
-        Guice.createInjector(new WebServletModule()));
-  }
-
-  public TestRMWebServicesCapacitySched() {
-    super(new WebAppDescriptor.Builder(
-        "org.apache.hadoop.yarn.server.resourcemanager.webapp")
-        .contextListenerClass(GuiceServletConfig.class)
-        .filterClass(com.google.inject.servlet.GuiceFilter.class)
-        .contextPath("jersey-guice-filter").servletPath("/").build());
-  }
-
   @Test
-  public void testClusterScheduler() throws JSONException, Exception {
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
+  public void testClusterScheduler() throws Exception {
+    ClientResponse response = resource().path("ws").path("v1").path("cluster")
         .path("scheduler").accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    JSONObject json = response.getEntity(JSONObject.class);
-    verifyClusterScheduler(json);
+    assertJsonResponse(response, "webapp/scheduler-response.json");
   }
 
   @Test
-  public void testClusterSchedulerSlash() throws JSONException, Exception {
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
+  public void testClusterSchedulerSlash() throws Exception {
+    ClientResponse response = resource().path("ws").path("v1").path("cluster")
         .path("scheduler/").accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    JSONObject json = response.getEntity(JSONObject.class);
-    verifyClusterScheduler(json);
+    assertJsonResponse(response, "webapp/scheduler-response.json");
   }
 
   @Test
-  public void testClusterSchedulerDefault() throws JSONException, Exception {
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
+  public void testClusterSchedulerDefault() throws Exception {
+    ClientResponse response = resource().path("ws").path("v1").path("cluster")
         .path("scheduler").get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    JSONObject json = response.getEntity(JSONObject.class);
-    verifyClusterScheduler(json);
+    assertJsonResponse(response, "webapp/scheduler-response.json");
   }
 
   @Test
-  public void testClusterSchedulerXML() throws JSONException, Exception {
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
+  public void testClusterSchedulerXML() throws Exception {
+    ClientResponse response = resource().path("ws").path("v1").path("cluster")
         .path("scheduler/").accept(MediaType.APPLICATION_XML)
         .get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-    DocumentBuilder db = dbf.newDocumentBuilder();
-    InputSource is = new InputSource();
-    is.setCharacterStream(new StringReader(xml));
-    Document dom = db.parse(is);
-    NodeList scheduler = dom.getElementsByTagName("scheduler");
-    assertEquals("incorrect number of elements", 1, scheduler.getLength());
-    NodeList schedulerInfo = dom.getElementsByTagName("schedulerInfo");
-    assertEquals("incorrect number of elements", 1, schedulerInfo.getLength());
-    verifyClusterSchedulerXML(schedulerInfo);
+    assertXmlResponse(response, "webapp/scheduler-response.xml");
   }
 
-  public void verifyClusterSchedulerXML(NodeList nodes) throws Exception {
-
-    for (int i = 0; i < nodes.getLength(); i++) {
-      Element element = (Element) nodes.item(i);
-
-      verifyClusterSchedulerGeneric(
-          WebServicesTestUtils.getXmlAttrString(element, "xsi:type"),
-          WebServicesTestUtils.getXmlFloat(element, "usedCapacity"),
-          WebServicesTestUtils.getXmlFloat(element, "capacity"),
-          WebServicesTestUtils.getXmlFloat(element, "maxCapacity"),
-          WebServicesTestUtils.getXmlString(element, "queueName"),
-          WebServicesTestUtils.getXmlString(element, "queuePath"));
-
-      NodeList children = element.getChildNodes();
-      for (int j = 0; j < children.getLength(); j++) {
-        Element qElem = (Element) children.item(j);
-        if(qElem.getTagName().equals("queues")) {
-          NodeList qListInfos = qElem.getChildNodes();
-          for (int k = 0; k < qListInfos.getLength(); k++) {
-            Element qElem2 = (Element) qListInfos.item(k);
-            String qName2 = WebServicesTestUtils.getXmlString(qElem2, "queueName");
-            String q2 = CapacitySchedulerConfiguration.ROOT + "." + qName2;
-            verifySubQueueXML(qElem2, q2, 100, 100);
-          }
-        }
-      }
-    }
-  }
-
-  public void verifySubQueueXML(Element qElem, String q,
-      float parentAbsCapacity, float parentAbsMaxCapacity)
-      throws Exception {
-    NodeList children = qElem.getChildNodes();
-    boolean hasSubQueues = false;
-    for (int j = 0; j < children.getLength(); j++) {
-      Element qElem2 = (Element) children.item(j);
-      if(qElem2.getTagName().equals("queues")) {
-        NodeList qListInfos = qElem2.getChildNodes();
-        if (qListInfos.getLength() > 0) {
-          hasSubQueues = true;
-        }
-      }
-    }
-    QueueInfo qi = (hasSubQueues) ? new QueueInfo() : new LeafQueueInfo();
-    qi.capacity = WebServicesTestUtils.getXmlFloat(qElem, "capacity");
-    qi.usedCapacity =
-        WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity");
-    qi.maxCapacity = WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity");
-    qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity");
-    qi.absoluteMaxCapacity =
-        WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity");
-    qi.absoluteUsedCapacity =
-      WebServicesTestUtils.getXmlFloat(qElem, "absoluteUsedCapacity");
-    qi.numApplications =
-        WebServicesTestUtils.getXmlInt(qElem, "numApplications");
-    qi.maxParallelApps =
-        WebServicesTestUtils.getXmlInt(qElem, "maxParallelApps");
-    qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
-    qi.queuePath = WebServicesTestUtils.getXmlString(qElem, "queuePath");
-    qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
-    qi.autoCreateChildQueueEnabled = WebServicesTestUtils.getXmlBoolean(qElem,
-        "autoCreateChildQueueEnabled");
-    qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
-        "isAbsoluteResource");
-    verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
-    if (hasSubQueues) {
-      for (int j = 0; j < children.getLength(); j++) {
-        Element qElem2 = (Element) children.item(j);
-        if(qElem2.getTagName().equals("queues")) {
-          NodeList qListInfos = qElem2.getChildNodes();
-          for (int k = 0; k < qListInfos.getLength(); k++) {
-            Element qElem3 = (Element) qListInfos.item(k);
-            String qName3 = WebServicesTestUtils.getXmlString(qElem3, "queueName");
-            String q3 = q + "." + qName3;
-            verifySubQueueXML(qElem3, q3, qi.absoluteCapacity, qi.absoluteMaxCapacity);
-          }
-        }
-      }
-    } else if (qi.autoCreateChildQueueEnabled) {
-      assertEquals("queueName doesn't match", "a1c", qi.queueName);
-      String capacityStr = WebServicesTestUtils.getPropertyValue(qElem,
-          "leafQueueTemplate", AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
-          + DOT + CAPACITY);
-      int capacity = Integer.parseInt(capacityStr);
-      assertEquals(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT
-          + CAPACITY + " doesn't match", 50, capacity);
-    } else {
-      LeafQueueInfo lqi = (LeafQueueInfo) qi;
-      lqi.numActiveApplications =
-          WebServicesTestUtils.getXmlInt(qElem, "numActiveApplications");
-      lqi.numPendingApplications =
-          WebServicesTestUtils.getXmlInt(qElem, "numPendingApplications");
-      lqi.numContainers =
-          WebServicesTestUtils.getXmlInt(qElem, "numContainers");
-      lqi.maxApplications =
-          WebServicesTestUtils.getXmlInt(qElem, "maxApplications");
-      lqi.maxApplicationsPerUser =
-          WebServicesTestUtils.getXmlInt(qElem, "maxApplicationsPerUser");
-      lqi.userLimit = WebServicesTestUtils.getXmlFloat(qElem, "userLimit");
-      lqi.userLimitFactor =
-          WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor");
-      lqi.defaultApplicationLifetime =
-          WebServicesTestUtils.getXmlLong(qElem, "defaultApplicationLifetime");
-      lqi.maxApplicationLifetime =
-          WebServicesTestUtils.getXmlLong(qElem, "maxApplicationLifetime");
-      verifyLeafQueueGeneric(q, lqi);
-    }
-  }
-
-  private void verifyClusterScheduler(JSONObject json) throws JSONException,
-      Exception {
-    assertEquals("incorrect number of elements in: " + json, 1, json.length());
-    JSONObject info = json.getJSONObject("scheduler");
-    assertEquals("incorrect number of elements in: " + info, 1, info.length());
-    info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements in: " + info, 22, info.length());
-    verifyClusterSchedulerGeneric(info.getString("type"),
-        (float) info.getDouble("usedCapacity"),
-        (float) info.getDouble("capacity"),
-        (float) info.getDouble("maxCapacity"),
-        info.getString("queueName"),
-        info.getString("queuePath"));
-    JSONObject health = info.getJSONObject("health");
-    assertNotNull(health);
-    assertEquals("incorrect number of elements in: " + health, 3,
-        health.length());
-    JSONArray operationsInfo = health.getJSONArray("operationsInfo");
-    assertEquals("incorrect number of elements in: " + health, 4,
-        operationsInfo.length());
-    JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
-    assertEquals("incorrect number of elements in: " + health, 3,
-        lastRunDetails.length());
-
-    JSONObject maximumAllocation = info.getJSONObject("maximumAllocation");
-    assertEquals("8192", maximumAllocation.getString("memory"));
-    assertEquals("4", maximumAllocation.getString("vCores"));
-
-    JSONObject queueAcls = info.getJSONObject("queueAcls");
-    assertEquals(1, queueAcls.length());
-
-    assertEquals("0", info.getString("queuePriority"));
-    assertEquals("utilization", info.getString("orderingPolicyInfo"));
-
-    JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
-    assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
-
-    // test subqueues
-    for (int i = 0; i < arr.length(); i++) {
-      JSONObject obj = arr.getJSONObject(i);
-      String q = CapacitySchedulerConfiguration.ROOT + "." +
-              obj.getString("queueName");
-      verifySubQueue(obj, q, 100, 100);
-    }
-  }
-
-  private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
-      float capacity, float maxCapacity, String queueName, String queuePath)
-      throws Exception {
-
-    assertTrue("type doesn't match", "capacityScheduler".matches(type));
-    assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f);
-    assertEquals("capacity doesn't match", 100, capacity, 1e-3f);
-    assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f);
-    assertTrue("queueName doesn't match", "root".matches(queueName));
-    assertTrue("queuePath doesn't match", "root".matches(queuePath));
-  }
-
-  private void verifySubQueue(JSONObject info, String q,
-      float parentAbsCapacity, float parentAbsMaxCapacity)
-      throws JSONException, Exception {
-    int numExpectedElements = 38;
-    boolean isParentQueue = true;
-    if (!info.has("queues")) {
-      numExpectedElements = 56;
-      isParentQueue = false;
-    }
-    assertEquals("incorrect number of elements", numExpectedElements, info.length());
-
-    QueueInfo qi = isParentQueue ? new QueueInfo() : new LeafQueueInfo();
-    qi.capacity = (float) info.getDouble("capacity");
-    qi.usedCapacity = (float) info.getDouble("usedCapacity");
-    qi.maxCapacity = (float) info.getDouble("maxCapacity");
-    qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity");
-    qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity");
-    qi.absoluteUsedCapacity = (float) info.getDouble("absoluteUsedCapacity");
-    qi.numApplications = info.getInt("numApplications");
-    qi.maxParallelApps = info.getInt("maxParallelApps");
-    qi.queueName = info.getString("queueName");
-    qi.queuePath = info.getString("queuePath");
-    qi.state = info.getString("state");
-
-    verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
-
-    // Separate Condition for Managed Parent Queue
-    if (qi.queueName.equals("a1c")) {
-      assertTrue(info.getBoolean("autoCreateChildQueueEnabled"));
-    } else if (isParentQueue) {
-      JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
-      // test subqueues
-      for (int i = 0; i < arr.length(); i++) {
-        JSONObject obj = arr.getJSONObject(i);
-        String q2 = q + "." + obj.getString("queueName");
-        verifySubQueue(obj, q2, qi.absoluteCapacity, qi.absoluteMaxCapacity);
-      }
-
-      JSONObject maximumAllocation = info.getJSONObject("maximumAllocation");
-      assertEquals("8192", maximumAllocation.getString("memory"));
-      assertEquals("4", maximumAllocation.getString("vCores"));
-
-      JSONObject queueAcls = info.getJSONObject("queueAcls");
-      assertEquals(1, queueAcls.length());
-
-      assertEquals("0", info.getString("queuePriority"));
-      assertEquals("utilization", info.getString("orderingPolicyInfo"));
-      assertFalse(info.getBoolean("autoCreateChildQueueEnabled"));
-    } else {
-      Assert.assertEquals("\"type\" field is incorrect",
-          "capacitySchedulerLeafQueueInfo", info.getString("type"));
-      LeafQueueInfo lqi = (LeafQueueInfo) qi;
-      lqi.numActiveApplications = info.getInt("numActiveApplications");
-      lqi.numPendingApplications = info.getInt("numPendingApplications");
-      lqi.numContainers = info.getInt("numContainers");
-      lqi.maxApplications = info.getInt("maxApplications");
-      lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser");
-      lqi.userLimit = (float) info.getDouble("userLimit");
-      lqi.userLimitFactor = (float) info.getDouble("userLimitFactor");
-      lqi.defaultApplicationLifetime =
-          info.getLong("defaultApplicationLifetime");
-      lqi.maxApplicationLifetime = info.getLong("maxApplicationLifetime");
-      verifyLeafQueueGeneric(q, lqi);
-      // resourcesUsed and users (per-user resources used) are checked in
-      // testPerUserResource()
-    }
-  }
-
-  private void verifySubQueueGeneric(String q, QueueInfo info,
-      float parentAbsCapacity, float parentAbsMaxCapacity) throws Exception {
-    String[] qArr = q.split("\\.");
-    assertTrue("q name invalid: " + q, qArr.length > 1);
-    String qshortName = qArr[qArr.length - 1];
-
-    assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f);
-    assertEquals("capacity doesn't match", csConf.getNonLabeledQueueCapacity(q),
-        info.capacity, 1e-3f);
-    float expectCapacity = csConf.getNonLabeledQueueMaximumCapacity(q);
-    float expectAbsMaxCapacity = parentAbsMaxCapacity * (info.maxCapacity/100);
-    if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) {
-      expectCapacity = 100;
-      expectAbsMaxCapacity = 100;
-    }
-    assertEquals("maxCapacity doesn't match", expectCapacity,
-        info.maxCapacity, 1e-3f);
-    assertEquals("absoluteCapacity doesn't match",
-        parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f);
-    assertEquals("absoluteMaxCapacity doesn't match",
-        expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f);
-    assertEquals("absoluteUsedCapacity doesn't match",
-        0, info.absoluteUsedCapacity, 1e-3f);
-    assertEquals("numApplications doesn't match", 0, info.numApplications);
-    assertTrue("queueName doesn't match, got: " + info.queueName
-        + " expected: " + q, qshortName.matches(info.queueName));
-    assertTrue("queuePath doesn't match, got: " + info.getQueuePath()
-        + " expected: " + q, q.matches(info.getQueuePath()));
-    assertTrue("state doesn't match",
-        (csConf.getState(q).toString()).matches(info.state));
-    if (q.equals("c")) {
-      assertTrue("c queue is not configured in Absolute resource",
-          info.isAbsoluteResource);
-    } else {
-      assertFalse(info.queueName
-          + " queue is not configured in Absolute resource",
-          info.isAbsoluteResource);
-    }
-    assertEquals("maxParallelApps doesn't match " + q,
-        (q.equals("root.a") ? 42 : Integer.MAX_VALUE),
-        info.maxParallelApps);
-  }
-
-  private void verifyLeafQueueGeneric(String q, LeafQueueInfo info)
-      throws Exception {
-    assertEquals("numActiveApplications doesn't match",
-        0, info.numActiveApplications);
-    assertEquals("numPendingApplications doesn't match",
-        0, info.numPendingApplications);
-    assertEquals("numContainers doesn't match",
-        0, info.numContainers);
-
-    int maxSystemApps = csConf.getMaximumSystemApplications();
-    int expectedMaxApps = (int)(maxSystemApps * (info.absoluteCapacity/100));
-    int expectedMaxAppsPerUser = Math.min(expectedMaxApps,
-        (int)(expectedMaxApps * (info.userLimit/100.0f) *
-        info.userLimitFactor));
-
-    // TODO: would like to use integer comparisons here but can't due to
-    //       roundoff errors in absolute capacity calculations
-    assertEquals("maxApplications doesn't match",
-        (float)expectedMaxApps, (float)info.maxApplications, 1.0f);
-    assertEquals("maxApplicationsPerUser doesn't match",
-        (float)expectedMaxAppsPerUser,
-        (float)info.maxApplicationsPerUser, info.userLimitFactor);
-
-    assertEquals("userLimit doesn't match", csConf.getUserLimit(q),
-        info.userLimit, 1e-3f);
-    assertEquals("userLimitFactor doesn't match",
-        csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
-
-    if (q.equals("root.a.a2")) {
-      assertEquals("defaultApplicationLifetime doesn't match",
-          csConf.getDefaultLifetimePerQueue(q),
-          info.defaultApplicationLifetime);
-      assertEquals("maxApplicationLifetime doesn't match",
-          csConf.getMaximumLifetimePerQueue(q),
-          info.maxApplicationLifetime);
-    }
-  }
-
-  //Return a child Node of node with the tagname or null if none exists
-  private Node getChildNodeByName(Node node, String tagname) {
-    NodeList nodeList = node.getChildNodes();
-    for (int i=0; i < nodeList.getLength(); ++i) {
-      if (nodeList.item(i).getNodeName().equals(tagname)) {
-        return nodeList.item(i);
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Test per user resources and resourcesUsed elements in the web services XML
-   * @throws Exception
-   */
   @Test
   public void testPerUserResourcesXML() throws Exception {
-    //Start RM so that it accepts app submissions
+    // Start RM so that it accepts app submissions
     rm.start();
     try {
       MockRMAppSubmissionData data1 =
@@ -615,55 +225,9 @@ public void testPerUserResourcesXML() throws Exception {
       MockRMAppSubmitter.submit(rm, data);
 
       //Get the XML from ws/v1/cluster/scheduler
-      WebResource r = resource();
-      ClientResponse response = r.path("ws/v1/cluster/scheduler")
-        .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-      assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
-          response.getType().toString());
-      String xml = response.getEntity(String.class);
-      DocumentBuilder db = DocumentBuilderFactory.newInstance()
-        .newDocumentBuilder();
-      InputSource is = new InputSource();
-      is.setCharacterStream(new StringReader(xml));
-      //Parse the XML we got
-      Document dom = db.parse(is);
-
-      //Get all users elements (1 for each leaf queue)
-      NodeList allUsers = dom.getElementsByTagName("users");
-      for (int i=0; i<allUsers.getLength(); ++i) {
-        Node perUserResources = allUsers.item(i);
-        String queueName = getChildNodeByName(perUserResources
-          .getParentNode(), "queueName").getTextContent();
-        if (queueName.equals("b1")) {
-          //b1 should have two users (user1 and user2) which submitted jobs
-          assertEquals(2, perUserResources.getChildNodes().getLength());
-          NodeList users = perUserResources.getChildNodes();
-          for (int j=0; j<users.getLength(); ++j) {
-            Node user = users.item(j);
-            String username = getChildNodeByName(user, "username")
-                .getTextContent();
-            assertTrue(username.equals("user1") || username.equals("user2"));
-            //Should be a parsable integer
-            Integer.parseInt(getChildNodeByName(getChildNodeByName(user,
-              "resourcesUsed"), "memory").getTextContent());
-            Integer.parseInt(getChildNodeByName(user, "numActiveApplications")
-              .getTextContent());
-            Integer.parseInt(getChildNodeByName(user, "numPendingApplications")
-                .getTextContent());
-          }
-        } else {
-        //Queues other than b1 should have 0 users
-          assertEquals(0, perUserResources.getChildNodes().getLength());
-        }
-      }
-      NodeList allResourcesUsed = dom.getElementsByTagName("resourcesUsed");
-      for (int i=0; i<allResourcesUsed.getLength(); ++i) {
-        Node resourcesUsed = allResourcesUsed.item(i);
-        Integer.parseInt(getChildNodeByName(resourcesUsed, "memory")
-            .getTextContent());
-        Integer.parseInt(getChildNodeByName(resourcesUsed, "vCores")
-              .getTextContent());
-      }
+      ClientResponse response = resource().path("ws/v1/cluster/scheduler")
+          .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+      assertXmlResponse(response, "webapp/scheduler-response-PerUserResources.xml");
     } finally {
       rm.stop();
     }
@@ -682,63 +246,14 @@ public void testNodeLabelDefaultAPI() throws Exception {
     rm.start();
     try {
       //Get the XML from ws/v1/cluster/scheduler
-      WebResource r = resource();
-      ClientResponse response = r.path("ws/v1/cluster/scheduler")
+      ClientResponse response = resource().path("ws/v1/cluster/scheduler")
           .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-      assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
-          response.getType().toString());
-      String xml = response.getEntity(String.class);
-      DocumentBuilder db = DocumentBuilderFactory.newInstance()
-          .newDocumentBuilder();
-      InputSource is = new InputSource();
-      is.setCharacterStream(new StringReader(xml));
-      //Parse the XML we got
-      Document dom = db.parse(is);
-
-      NodeList allQueues = dom.getElementsByTagName("queue");
-      for (int i = 0; i < allQueues.getLength(); ++i) {
-        Node queueNode = allQueues.item(i);
-        Node queuePathNode = getChildNodeByName(queueNode, "queuePath");
-        if (queuePathNode == null) {
-          continue;
-        }
-
-        String queuePath = queuePathNode.getTextContent();
-        if (queuePath != null) {
-          if (queuePath.startsWith("root.a")) {
-            assertEquals("root-a-default-label",
-                getChildNodeByName(queueNode, "defaultNodeLabelExpression")
-                    .getTextContent());
-          } else {
-            assertEquals("ROOT-INHERITED",
-                getChildNodeByName(queueNode, "defaultNodeLabelExpression")
-                    .getTextContent());
-          }
-        }
-      }
+      assertXmlResponse(response, "webapp/scheduler-response-NodeLabelDefaultAPI.xml");
     } finally {
       rm.stop();
     }
   }
 
-  private void checkResourcesUsed(JSONObject queue) throws JSONException {
-    queue.getJSONObject("resourcesUsed").getInt("memory");
-    queue.getJSONObject("resourcesUsed").getInt("vCores");
-  }
-
-  //Also checks resourcesUsed
-  private JSONObject getSubQueue(JSONObject queue, String subQueue)
-    throws JSONException {
-    JSONArray queues = queue.getJSONObject("queues").getJSONArray("queue");
-    for (int i=0; i<queues.length(); ++i) {
-      checkResourcesUsed(queues.getJSONObject(i));
-      if (queues.getJSONObject(i).getString("queueName").equals(subQueue) ) {
-        return queues.getJSONObject(i);
-      }
-    }
-    return null;
-  }
-
   @Test
   public void testPerUserResourcesJSON() throws Exception {
     //Start RM so that it accepts app submissions
@@ -764,48 +279,152 @@ public void testPerUserResourcesJSON() throws Exception {
       MockRMAppSubmitter.submit(rm, data);
 
       //Get JSON
-      WebResource r = resource();
-      ClientResponse response = r.path("ws").path("v1").path("cluster")
+      ClientResponse response = resource().path("ws").path("v1").path("cluster")
           .path("scheduler/").accept(MediaType.APPLICATION_JSON)
           .get(ClientResponse.class);
-      assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-          response.getType().toString());
-      JSONObject json = response.getEntity(JSONObject.class);
-
-      JSONObject schedulerInfo = json.getJSONObject("scheduler").getJSONObject(
-        "schedulerInfo");
-      JSONObject b1 = getSubQueue(getSubQueue(schedulerInfo, "b"), "b1");
-      //Check users user1 and user2 exist in b1
-      JSONArray users = b1.getJSONObject("users").getJSONArray("user");
-      for (int i=0; i<2; ++i) {
-        JSONObject user = users.getJSONObject(i);
-        assertTrue("User isn't user1 or user2",user.getString("username")
-          .equals("user1") || user.getString("username").equals("user2"));
-        user.getInt("numActiveApplications");
-        user.getInt("numPendingApplications");
-        checkResourcesUsed(user);
-      }
-
-      // Verify 'queues' field is omitted from CapacitySchedulerLeafQueueInfo.
-      try {
-        b1.getJSONObject("queues");
-        fail("CapacitySchedulerQueueInfo should omit field 'queues'" +
-             "if child queue is empty.");
-      } catch (JSONException je) {
-        assertEquals("JSONObject[\"queues\"] not found.", je.getMessage());
-      }
+      assertJsonResponse(response, "webapp/scheduler-response-PerUserResources.json");
     } finally {
       rm.stop();
     }
   }
 
-
   @Test
   public void testResourceInfo() {
     Resource res = Resources.createResource(10, 1);
-    // If we add a new resource (e.g disks), then
+    // If we add a new resource (e.g. disks), then
     // CapacitySchedulerPage and these RM WebServices + docs need to be updated
-    // eg. ResourceInfo
+    // e.g. ResourceInfo
     assertEquals("<memory:10, vCores:1>", res.toString());
   }
+
+  public static void assertXmlType(ClientResponse response) {
+    assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+  }
+
+  public static void assertXmlResponse(ClientResponse response,
+                                       String expectedResourceFilename) throws
+      Exception {
+    assertXmlType(response);
+    Document document = loadDocument(response.getEntity(String.class));
+    String actual = serializeDocument(document).trim();
+    updateTestDataAutomatically(expectedResourceFilename, actual);
+    assertEquals(getResourceAsString(expectedResourceFilename), actual);
+  }
+
+  public static String serializeDocument(Document document) throws TransformerException {
+    DOMSource domSource = new DOMSource(document);
+    StringWriter writer = new StringWriter();
+    StreamResult result = new StreamResult(writer);
+    TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
+    Transformer transformer = tf.newTransformer();
+    transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+    transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+    transformer.transform(domSource, result);
+    return writer.toString();
+  }
+
+  public static Document loadDocument(String xml) throws Exception {
+    DocumentBuilderFactory factory = XMLUtils.newSecureDocumentBuilderFactory();
+    DocumentBuilder builder = factory.newDocumentBuilder();
+    InputSource is = new InputSource(new StringReader(xml));
+    return builder.parse(is);
+  }
+
+  public static void assertJsonResponse(ClientResponse response,
+                                        String expectedResourceFilename) throws
+      JSONException, IOException {
+    assertJsonType(response);
+    JSONObject json = response.getEntity(JSONObject.class);
+    String actual = json.toString(2);
+    updateTestDataAutomatically(expectedResourceFilename, actual);
+    assertEquals(
+        prettyPrintJson(getResourceAsString(expectedResourceFilename)),
+        prettyPrintJson(actual));
+  }
+
+  private static String prettyPrintJson(String in) throws JsonProcessingException {
+    ObjectMapper objectMapper = new ObjectMapper();
+    return objectMapper
+        .writerWithDefaultPrettyPrinter()
+        .writeValueAsString(objectMapper.readTree(in));
+  }
+
+  public static void assertJsonType(ClientResponse response) {
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+  }
+
+  public static InputStream getResourceAsStream(String configFilename) {
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    return classLoader.getResourceAsStream(configFilename);
+  }
+
+  public static String getResourceAsString(String configFilename) throws IOException {
+    try (InputStream is = getResourceAsStream(configFilename)) {
+      if (is == null) {
+        return null;
+      }
+      try (InputStreamReader isr = new InputStreamReader(is);
+           BufferedReader reader = new BufferedReader(isr)) {
+        return reader.lines().collect(Collectors.joining(System.lineSeparator()));
+      }
+    }
+  }
+
+  public static void updateTestDataAutomatically(String configFilename, String actualContent) {
+    /*
+     Set UPDATE_TESTDATA=1 environment variable for auto update the expected data
+     or uncomment this return statement.
+
+     It's safe in a way that, this updates the source directory so the test will still fail,
+     because the target directory is untouched.
+     */
+    if (System.getenv("UPDATE_TESTDATA") == null) {
+      return;
+    }
+
+    ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+    try {
+      String resource = Objects.requireNonNull(
+          Objects.requireNonNull(classLoader.getResource(configFilename)).toURI().getPath())
+          .replaceAll("/target/test-classes/", "/src/test/resources/");
+      try (FileWriter writer = new FileWriter(resource, false)) {
+        writer.write(actualContent);
+      }
+    } catch (URISyntaxException | IOException e) {
+      e.printStackTrace();
+      Assert.fail("overwrite should not fail " + e.getMessage());
+    }
+  }
+
+  public static WebAppDescriptor createWebAppDescriptor() {
+    return new WebAppDescriptor.Builder(
+        TestRMWebServicesCapacitySched.class.getPackage().getName())
+        .contextListenerClass(GuiceServletConfig.class)
+        .filterClass(com.google.inject.servlet.GuiceFilter.class)
+        .contextPath("jersey-guice-filter").servletPath("/").build();
+  }
+
+  public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {
+    setupQueueConfiguration(csConf);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+    conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
+        YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
+    return new MockRM(conf);
+  }
+
+  @Test
+  public void testClusterSchedulerOverviewCapacity() throws Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    JSONObject json = response.getEntity(JSONObject.class);
+    TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler");
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
index afd73621d35ee..f2336e79d2ace 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
@@ -18,159 +18,41 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
-import static org.junit.Assert.assertEquals;
+import com.google.inject.Guice;
+import com.sun.jersey.api.client.ClientResponse;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import javax.ws.rs.core.MediaType;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
 
 public class TestRMWebServicesCapacitySchedDynamicConfig extends
     JerseyTestBase {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestRMWebServicesCapacitySchedDynamicConfig.class);
-  private static final float EXP_WEIGHT_NON_WEIGHT_MODE = -1.0F;
-  private static final float EXP_NORM_WEIGHT_NON_WEIGHT_MODE = 0.0F;
-  private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
-  private static final float EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
-  private static final double DELTA = 0.00001;
-  private static final String PARENT_QUEUE = "parent";
-  private static final String LEAF_QUEUE = "leaf";
-  private static final String STATIC_QUEUE = "static";
-  private static final String FLEXIBLE_DYNAMIC_QUEUE = "dynamicFlexible";
-  private static final String AUTO_CREATION_OFF = "off";
-  private static final String AUTO_CREATION_LEGACY = "legacy";
-  private static final String AUTO_CREATION_FLEXIBLE = "flexible";
-  private static final int GB = 1024;
-  protected static MockRM RM;
+  private MockRM rm;
 
   private CapacitySchedulerQueueManager autoQueueHandler;
-  private CapacitySchedulerConfiguration csConf;
-
-  private static class ExpectedQueueWithProperties {
-    private String path;
-    public final float weight;
-    public final float normalizedWeight;
-    private String queueType;
-    private String creationMethod;
-    private String autoCreationEligibility;
-    private List<String[]> autoQueueTemplateProperties;
-
-    public ExpectedQueueWithProperties(String path, float weight,
-        float normalizedWeight, String queueType, String creationMethod,
-        String autoCreationEligibility) {
-      this.path = path;
-      this.weight = weight;
-      this.normalizedWeight = normalizedWeight;
-      this.queueType = queueType;
-      this.creationMethod = creationMethod;
-      this.autoCreationEligibility = autoCreationEligibility;
-      this.autoQueueTemplateProperties = new ArrayList<>();
-    }
-
-    ExpectedQueueWithProperties(
-        String path, float weight, float normalizedWeight, String queueType,
-        String creationMethod, String autoCreationEligibility,
-        List<String[]> autoQueueTemplateProperties) {
-      this.path = path;
-      this.weight = weight;
-      this.normalizedWeight = normalizedWeight;
-      this.queueType = queueType;
-      this.creationMethod = creationMethod;
-      this.autoCreationEligibility = autoCreationEligibility;
-      this.autoQueueTemplateProperties = autoQueueTemplateProperties;
-    }
-  }
-
-  private static class WebServletModule extends ServletModule {
-    private final Configuration conf;
-
-    public WebServletModule(Configuration conf) {
-      this.conf = conf;
-    }
-
-    @Override
-    protected void configureServlets() {
-      bind(JAXBContextResolver.class);
-      bind(RMWebServices.class);
-      bind(GenericExceptionHandler.class);
-      conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-          ResourceScheduler.class);
-      conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
-          YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
-      RM = new MockRM(conf);
-      bind(ResourceManager.class).toInstance(RM);
-      serve("/*").with(GuiceContainer.class);
-    }
-  }
-
-  @Before
-  @Override
-  public void setUp() throws Exception {
-    super.setUp();
-  }
-
-  private void initResourceManager(Configuration conf) throws IOException {
-    GuiceServletConfig.setInjector(
-        Guice.createInjector(new WebServletModule(conf)));
-    RM.start();
-    //Need to call reinitialize as
-    //MutableCSConfigurationProvider with InMemoryConfigurationStore
-    //somehow does not load the queues properly and falls back to default config.
-    //Therefore CS will think there's only the default queue there.
-    ((CapacityScheduler) RM.getResourceScheduler()).reinitialize(conf,
-        RM.getRMContext(), true);
-    CapacityScheduler cs = (CapacityScheduler) RM.getResourceScheduler();
-    csConf = cs.getConfiguration();
-  }
 
   public TestRMWebServicesCapacitySchedDynamicConfig() {
-    super(new WebAppDescriptor.Builder(
-        "org.apache.hadoop.yarn.server.resourcemanager.webapp")
-        .contextListenerClass(GuiceServletConfig.class)
-        .filterClass(com.google.inject.servlet.GuiceFilter.class)
-        .contextPath("jersey-guice-filter").servletPath("/").build());
+    super(createWebAppDescriptor());
   }
 
   @Test
@@ -178,24 +60,16 @@ public void testSchedulerResponsePercentageMode()
       throws Exception {
     Configuration config = CSConfigGenerator
         .createPercentageConfig();
-    config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-        YarnConfiguration.MEMORY_CONFIGURATION_STORE);
 
     initResourceManager(config);
-    JSONObject json = sendRequestToSchedulerEndpoint();
-    validateSchedulerInfo(json, "percentage",
-        new ExpectedQueueWithProperties("root",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.default",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test2",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
+
+    /*
+     * mode: percentage
+     * autoCreationEligibility: off
+     * weight: -1, normalizedWeight: 0
+     * root.queueType: parent, others.queueType: leaf
+     */
+    assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
   }
 
   @Test
@@ -203,24 +77,31 @@ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
       throws Exception {
     Configuration config = CSConfigGenerator
         .createPercentageConfigLegacyAutoCreation();
-    config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-        YarnConfiguration.MEMORY_CONFIGURATION_STORE);
 
     initResourceManager(config);
-    JSONObject json = sendRequestToSchedulerEndpoint();
-    validateSchedulerInfo(json, "percentage",
-        new ExpectedQueueWithProperties("root",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.default",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.managedtest2",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_LEGACY));
+
+    /*
+     * mode: percentage
+     * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+     * weight: -1, normalizedWeight: 0
+     * root.queueType: parent, others.queueType: leaf
+     */
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+  }
+
+  @Test
+  public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+      throws Exception {
+    Configuration config = CSConfigGenerator
+        .createAbsoluteConfigLegacyAutoCreation();
+
+    initResourceManager(config);
+    initAutoQueueHandler(8192 * GB);
+    createQueue("root.managed.queue1");
+
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
   }
 
   @Test
@@ -228,24 +109,16 @@ public void testSchedulerResponseAbsoluteMode()
       throws Exception {
     Configuration config = CSConfigGenerator
         .createAbsoluteConfig();
-    config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-        YarnConfiguration.MEMORY_CONFIGURATION_STORE);
 
     initResourceManager(config);
-    JSONObject json = sendRequestToSchedulerEndpoint();
-    validateSchedulerInfo(json, "absolute",
-        new ExpectedQueueWithProperties("root",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.default",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test2",
-            EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
+
+    /*
+     * mode: absolute
+     * autoCreationEligibility: off
+     * weight: -1, normalizedWeight: 0
+     * root.queueType: parent, others.queueType: leaf
+     */
+    assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
   }
 
   @Test
@@ -253,21 +126,18 @@ public void testSchedulerResponseWeightMode()
       throws Exception {
     Configuration config = CSConfigGenerator
         .createWeightConfig();
-    config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-        YarnConfiguration.MEMORY_CONFIGURATION_STORE);
 
     initResourceManager(config);
-    JSONObject json = sendRequestToSchedulerEndpoint();
-    validateSchedulerInfo(json, "weight",
-        new ExpectedQueueWithProperties("root",
-            EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
+
+    /*
+     * mode: weight
+     * autoCreationEligibility: off
+     *                   root   default  test1  test2
+     * weight:            1       10       4     6
+     * normalizedWeight:  1       0.5      0.2   0.3
+     * root.queueType: parent, others.queueType: leaf
+     */
+    assertJsonResponse(sendRequest(), "webapp/scheduler-response-WeightMode.json");
   }
 
   @Test
@@ -275,269 +145,45 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
       throws Exception {
     Configuration config = CSConfigGenerator
         .createWeightConfigWithAutoQueueCreationEnabled();
-    config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
-        YarnConfiguration.MEMORY_CONFIGURATION_STORE);
     config.setInt(CapacitySchedulerConfiguration
         .getQueuePrefix("root.autoParent1") +
         AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
         "maximum-applications", 300);
 
     initResourceManager(config);
-    initAutoQueueHandler();
-    JSONObject json = sendRequestToSchedulerEndpoint();
-    validateSchedulerInfo(json, "weight",
-        new ExpectedQueueWithProperties("root",
-            EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
+    initAutoQueueHandler(1200 * GB);
+
+    // same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json");
 
     //Now create some auto created queues
     createQueue("root.auto1");
     createQueue("root.auto2");
     createQueue("root.auto3");
     createQueue("root.autoParent1.auto4");
+    createQueue("root.autoParent2.auto5");
+    createQueue("root.parent.autoParent3.auto6");
 
-    json = sendRequestToSchedulerEndpoint();
-    //root.auto1=1w, root.auto2=1w, root.auto3=1w
-    //root.default=10w, root.test1=4w, root.test2=6w
-    //root.autoparent1=1w
-    int sumOfWeights = 24;
-    ExpectedQueueWithProperties expectedRootQ =
-        new ExpectedQueueWithProperties("root",
-            EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE,
-            PARENT_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF);
-    List<String[]> templateProperties = new ArrayList<>();
-    templateProperties.add(new String[] {"maximum-applications", "300"});
-
-    validateSchedulerInfo(json, "weight",
-        expectedRootQ,
-        new ExpectedQueueWithProperties("root.auto1",
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
-            LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.auto2",
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
-            LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.auto3",
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
-            LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.autoParent1",
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE / sumOfWeights,
-            PARENT_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_FLEXIBLE,
-            templateProperties),
-        new ExpectedQueueWithProperties("root.default", 10.0f,
-            10.0f / sumOfWeights,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test1", 4.0f,
-            4.0f / sumOfWeights,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF),
-        new ExpectedQueueWithProperties("root.test2", 6.0f,
-            6.0f / sumOfWeights,
-            LEAF_QUEUE, STATIC_QUEUE, AUTO_CREATION_OFF));
-
-    validateChildrenOfParent(json, "root.autoParent1", "weight",
-        expectedRootQ,
-        new ExpectedQueueWithProperties("root.autoParent1.auto4",
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            EXP_DEFAULT_WEIGHT_IN_WEIGHT_MODE,
-            LEAF_QUEUE, FLEXIBLE_DYNAMIC_QUEUE, AUTO_CREATION_OFF));
+    assertJsonResponse(sendRequest(),
+        "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
   }
 
-  private void initAutoQueueHandler() throws Exception {
-    CapacityScheduler cs = (CapacityScheduler) RM.getResourceScheduler();
+  private void initAutoQueueHandler(int nodeMemory) throws Exception {
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
     autoQueueHandler = cs.getCapacitySchedulerQueueManager();
-    MockNM nm1 = RM.registerNode("h1:1234", 1200 * GB); // label = x
+    rm.registerNode("h1:1234", nodeMemory); // label = x
   }
 
-  private LeafQueue createQueue(String queuePath) throws YarnException,
+  private void createQueue(String queuePath) throws YarnException,
       IOException {
-    return autoQueueHandler.createQueue(new QueuePath(queuePath));
+    autoQueueHandler.createQueue(new QueuePath(queuePath));
   }
 
-  private JSONObject sendRequestToSchedulerEndpoint() throws Exception {
-    WebResource r = resource();
-    ClientResponse response = r.path("ws").path("v1").path("cluster")
+  private ClientResponse sendRequest() {
+    return resource().path("ws").path("v1").path("cluster")
         .path("scheduler").accept(MediaType.APPLICATION_JSON)
         .get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    String jsonString = response.getEntity(String.class);
-    LOG.debug("Received JSON response: " + jsonString);
-    return new JSONObject(jsonString);
-  }
-
-  private void validateSchedulerInfo(JSONObject json, String expectedMode,
-      ExpectedQueueWithProperties rootQueue,
-      ExpectedQueueWithProperties... expectedQueues) throws JSONException {
-    Assert.assertNotNull("SchedulerTypeInfo should not be null", json);
-    assertEquals("incorrect number of elements in: " + json, 1, json.length());
-
-    JSONObject info = verifySchedulerJSONObject(json);
-    info = verifySchedulerInfoJSONObject(expectedMode, rootQueue, info);
-    JSONArray queueArray = verifyQueueJSONListObject(info,
-        expectedQueues.length);
-    verifyQueues(CapacitySchedulerConfiguration.ROOT, expectedMode,
-        queueArray, expectedQueues);
-  }
-
-  private void validateChildrenOfParent(JSONObject json,
-      String parentPath, String expectedMode,
-      ExpectedQueueWithProperties rootQueue,
-      ExpectedQueueWithProperties... expectedLeafQueues) throws JSONException {
-    Assert.assertNotNull("SchedulerTypeInfo should not be null", json);
-    assertEquals("incorrect number of elements in: " + json, 1, json.length());
-
-    JSONObject info = verifySchedulerJSONObject(json);
-    info = verifySchedulerInfoJSONObject(expectedMode, rootQueue, info);
-    JSONArray queueArray = getQueuesJSONListObject(info);
-
-    Set<String> verifiedQueues = new HashSet<>();
-    for (int i = 0; i < queueArray.length(); i++) {
-      JSONObject childQueueObj = queueArray.getJSONObject(i);
-      String queuePath = CapacitySchedulerConfiguration.ROOT + "." +
-          childQueueObj.getString("queueName");
-      if (queuePath.equals(parentPath)) {
-        JSONArray childQueueArray = verifyQueueJSONListObject(childQueueObj,
-            expectedLeafQueues.length);
-        verifyQueues(parentPath, expectedMode, childQueueArray,
-            expectedLeafQueues);
-        verifiedQueues.add(queuePath);
-      }
-    }
-
-    Assert.assertEquals("Not all child queues were found. " +
-            String.format("Found queues: %s, All queues: %s", verifiedQueues,
-                Arrays.stream(expectedLeafQueues).map(lq -> lq.path)
-                    .collect(Collectors.toList())),
-        expectedLeafQueues.length, verifiedQueues.size());
-  }
-
-  private JSONObject verifySchedulerJSONObject(JSONObject json)
-      throws JSONException {
-    JSONObject info = json.getJSONObject("scheduler");
-    Assert.assertNotNull("Scheduler object should not be null", json);
-    assertEquals("incorrect number of elements in: " + info, 1, info.length());
-    return info;
-  }
-
-  private JSONObject verifySchedulerInfoJSONObject(String expectedMode,
-      ExpectedQueueWithProperties rootQueue, JSONObject info)
-      throws JSONException {
-    //Validate if root queue has the expected mode and weight values
-    info = info.getJSONObject("schedulerInfo");
-    Assert.assertNotNull("SchedulerInfo should not be null", info);
-    Assert.assertEquals("Expected Queue mode " + expectedMode, expectedMode,
-        info.getString("mode"));
-    Assert.assertEquals(rootQueue.weight,
-        Float.parseFloat(info.getString("weight")), DELTA);
-    Assert.assertEquals(rootQueue.normalizedWeight,
-        Float.parseFloat(info.getString("normalizedWeight")), DELTA);
-    return info;
-  }
-
-  private JSONArray verifyQueueJSONListObject(JSONObject info,
-      int expectedQSize) throws JSONException {
-    JSONArray queueArray = getQueuesJSONListObject(info);
-    assertEquals("QueueInfoList should be size of " + expectedQSize,
-        expectedQSize, queueArray.length());
-    return queueArray;
-  }
-
-  private JSONArray getQueuesJSONListObject(JSONObject info)
-      throws JSONException {
-    JSONObject queuesObj = info.getJSONObject("queues");
-    Assert.assertNotNull("QueueInfoList should not be null", queuesObj);
-
-    JSONArray queueArray = queuesObj.getJSONArray("queue");
-    Assert.assertNotNull("Queue list should not be null", queueArray);
-    return queueArray;
-  }
-
-  private void verifyQueues(String parentPath, String expectedMode,
-      JSONArray queueArray, ExpectedQueueWithProperties[] expectedQueues)
-      throws JSONException {
-    Map<String, ExpectedQueueWithProperties> queuesMap = new HashMap<>();
-    for (ExpectedQueueWithProperties expectedQueue : expectedQueues) {
-      queuesMap.put(expectedQueue.path, expectedQueue);
-    }
-
-    // Create mapping of queue path -> mode
-    Map<String, String> modesMap = new HashMap<>();
-    for (int i = 0; i < queueArray.length(); i++) {
-      JSONObject obj = queueArray.getJSONObject(i);
-      String queuePath = parentPath + "." + obj.getString("queueName");
-      String mode = obj.getString("mode");
-      modesMap.put(queuePath, mode);
-
-      //validate weights of all other queues
-      ExpectedQueueWithProperties expectedQueue = queuesMap.get(queuePath);
-      Assert.assertNotNull("Queue not found in expectedQueueMap with path: " +
-          queuePath, expectedQueue);
-      Assert.assertEquals("Weight value does not match",
-          expectedQueue.weight, Float.parseFloat(obj.getString("weight")),
-          DELTA);
-      Assert.assertEquals("Normalized weight value does not match for queue " +
-              queuePath,
-          expectedQueue.normalizedWeight,
-          Float.parseFloat(obj.getString("normalizedWeight")), DELTA);
-
-      //validate queue creation type
-      Assert.assertEquals("Queue type does not match for queue " +
-              queuePath,
-          expectedQueue.queueType, obj.getString("queueType"));
-
-      Assert.assertEquals("Queue creation type does not match for queue " +
-              queuePath,
-          expectedQueue.creationMethod, obj.getString("creationMethod"));
-
-      if (!expectedQueue.autoQueueTemplateProperties.isEmpty()) {
-        JSONArray templates = obj.getJSONObject("autoQueueTemplateProperties")
-            .getJSONArray("property");
-        for (int j = 0; j < templates.length(); j++) {
-          JSONObject prop = templates.getJSONObject(j);
-          Assert.assertEquals("Auto creation eligible queue " +
-                  "template key do not match for queue" + queuePath,
-              expectedQueue.autoQueueTemplateProperties.get(j)[0],
-              prop.getString("name"));
-          Assert.assertEquals("Auto creation eligible queue " +
-                  "template value do not match for queue" + queuePath,
-              expectedQueue.autoQueueTemplateProperties.get(j)[1],
-              prop.getString("value"));
-        }
-      }
-
-      Assert.assertEquals("Queue auto creation eligibility does not " +
-              "match for queue " + queuePath,
-          expectedQueue.autoCreationEligibility,
-          obj.getString("autoCreationEligibility"));
-    }
-
-    //Validate queue paths and modes
-    List<String> sortedExpectedPaths = Arrays.stream(expectedQueues)
-        .map(eq -> eq.path)
-        .sorted(Comparator.comparing(String::toLowerCase))
-        .collect(Collectors.toList());
-
-    List<String> sortedActualPaths = modesMap.keySet().stream()
-        .sorted(Comparator.comparing(String::toLowerCase))
-        .collect(Collectors.toList());
-    Assert.assertEquals("Expected Queue paths: " + sortedExpectedPaths,
-        sortedExpectedPaths, sortedActualPaths);
-
-    // Validate if we have a single "mode" for all queues
-    Set<String> modesSet = new HashSet<>(modesMap.values());
-    Assert.assertEquals("Expected a single Queue mode for all queues: " +
-        expectedMode + ", got: " + modesMap, 1, modesSet.size());
-    Assert.assertEquals("Expected Queue mode " + expectedMode,
-        expectedMode, modesSet.iterator().next());
   }
 
   private static class CSConfigGenerator {
@@ -566,6 +212,23 @@ public static Configuration createPercentageConfigLegacyAutoCreation() {
       return createConfiguration(conf);
     }
 
+    public static Configuration createAbsoluteConfigLegacyAutoCreation() {
+      Map<String, String> conf = new HashMap<>();
+      conf.put("yarn.scheduler.capacity.root.queues", "default, managed");
+      conf.put("yarn.scheduler.capacity.root.default.state", "STOPPED");
+      conf.put("yarn.scheduler.capacity.root.managed.capacity", "[memory=4096,vcores=4]");
+      conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.capacity",
+          "[memory=2048,vcores=2]");
+      conf.put("yarn.scheduler.capacity.root.managed.state", "RUNNING");
+      conf.put("yarn.scheduler.capacity.root.managed." +
+          "auto-create-child-queue.enabled", "true");
+      conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_submit_applications",
+          "user");
+      conf.put("yarn.scheduler.capacity.root.managed.leaf-queue-template.acl_administer_queue",
+          "admin");
+      return createConfiguration(conf);
+    }
+
     public static Configuration createAbsoluteConfig() {
       Map<String, String> conf = new HashMap<>();
       conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
@@ -583,7 +246,7 @@ public static Configuration createAbsoluteConfig() {
     }
 
     public static Configuration createWeightConfig() {
-     return createWeightConfigInternal(false);
+      return createWeightConfigInternal(false);
     }
 
     public static Configuration createWeightConfigWithAutoQueueCreationEnabled() {
@@ -592,17 +255,45 @@ public static Configuration createWeightConfigWithAutoQueueCreationEnabled() {
 
     private static Configuration createWeightConfigInternal(boolean enableAqc) {
       Map<String, String> conf = new HashMap<>();
-      conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+      conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2, parent");
       conf.put("yarn.scheduler.capacity.root.capacity", "1w");
       conf.put("yarn.scheduler.capacity.root.default.capacity", "10w");
-      conf.put("yarn.scheduler.capacity.root.test1.capacity", "4w");
-      conf.put("yarn.scheduler.capacity.root.test2.capacity", "6w");
+      conf.put("yarn.scheduler.capacity.root.test1.capacity", "5w");
+      conf.put("yarn.scheduler.capacity.root.test2.capacity", "10w");
+      conf.put("yarn.scheduler.capacity.root.parent.capacity", "20w");
       conf.put("yarn.scheduler.capacity.root.test1.state", "RUNNING");
       conf.put("yarn.scheduler.capacity.root.test2.state", "RUNNING");
 
       if (enableAqc) {
-        conf.put("yarn.scheduler.capacity.root.auto-queue-creation-v2.enabled",
-            "true");
+        final String root = "yarn.scheduler.capacity.root.";
+        conf.put(root +  "auto-queue-creation-v2.enabled", "true");
+
+        conf.put(root + "auto-queue-creation-v2.parent-template.acl_submit_applications",
+            "parentUser1");
+        conf.put(root + "auto-queue-creation-v2.parent-template.acl_administer_queue",
+            "parentAdmin1");
+
+        conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_submit_applications",
+            "user1");
+        conf.put(root + "autoParent1.auto-queue-creation-v2.leaf-template.acl_administer_queue",
+            "admin1");
+
+        conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
+            "wildUser1");
+        conf.put(root + "*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
+            "wildAdmin1");
+
+
+        conf.put(root + "parent.auto-queue-creation-v2.enabled", "true");
+        conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_submit_applications",
+            "parentUser2");
+        conf.put(root + "parent.auto-queue-creation-v2.parent-template.acl_administer_queue",
+            "parentAdmin2");
+
+        conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_submit_applications",
+            "wildUser2");
+        conf.put(root + "parent.*.auto-queue-creation-v2.leaf-template.acl_administer_queue",
+            "wildAdmin2");
       }
       return createConfiguration(conf);
     }
@@ -611,10 +302,27 @@ public static Configuration createConfiguration(
         Map<String, String> configs) {
       Configuration config = new Configuration();
 
-      for (Map.Entry<String, String> entry: configs.entrySet()) {
+      for (Map.Entry<String, String> entry : configs.entrySet()) {
         config.set(entry.getKey(), entry.getValue());
       }
+
+      config.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+          YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+
       return config;
     }
   }
+
+  private void initResourceManager(Configuration conf) throws IOException {
+    rm = createMockRM(new CapacitySchedulerConfiguration(conf));
+    GuiceServletConfig.setInjector(
+        Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
+    rm.start();
+    //Need to call reinitialize as
+    //MutableCSConfigurationProvider with InMemoryConfigurationStore
+    //somehow does not load the queues properly and falls back to default config.
+    //Therefore CS will think there's only the default queue there.
+    ((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf,
+        rm.getRMContext(), true);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 34b7c1225c245..43b5ee7127c2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -22,10 +22,13 @@
 import com.google.inject.servlet.ServletModule;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
 import com.sun.jersey.test.framework.WebAppDescriptor;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -33,12 +36,16 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
 import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
 import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -57,9 +64,14 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY;
+import static org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils.toJson;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
 
 /**
@@ -73,7 +85,11 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
       "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
   private static final File OLD_CONF_FILE = new File(new File("target",
       "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp");
-
+  private static final String LABEL_1 = "label1";
+  public static final QueuePath ROOT = new QueuePath("root");
+  public static final QueuePath ROOT_A = new QueuePath("root", "a");
+  public static final QueuePath ROOT_A_A1 = QueuePath.createFromQueues("root", "a", "a1");
+  public static final QueuePath ROOT_A_A2 = QueuePath.createFromQueues("root", "a", "a2");
   private static MockRM rm;
   private static String userName;
   private static CapacitySchedulerConfiguration csConf;
@@ -131,7 +147,7 @@ public void setUp() throws Exception {
   private static void setupQueueConfiguration(
       CapacitySchedulerConfiguration config) {
     config.setQueues(CapacitySchedulerConfiguration.ROOT,
-        new String[]{"a", "b", "c"});
+        new String[]{"a", "b", "c", "mappedqueue"});
 
     final String a = CapacitySchedulerConfiguration.ROOT + ".a";
     config.setCapacity(a, 25f);
@@ -152,6 +168,11 @@ private static void setupQueueConfiguration(
     final String c1 = c + ".c1";
     config.setQueues(c, new String[] {"c1"});
     config.setCapacity(c1, 0f);
+
+    final String d = CapacitySchedulerConfiguration.ROOT + ".d";
+    config.setCapacity(d, 0f);
+    config.set(CapacitySchedulerConfiguration.QUEUE_MAPPING,
+        "g:hadoop:mappedqueue");
   }
 
   public TestRMWebServicesConfigurationMutation() {
@@ -187,14 +208,14 @@ private CapacitySchedulerConfiguration getSchedulerConf()
   public void testGetSchedulerConf() throws Exception {
     CapacitySchedulerConfiguration orgConf = getSchedulerConf();
     assertNotNull(orgConf);
-    assertEquals(3, orgConf.getQueues("root").length);
+    assertEquals(4, orgConf.getQueues("root").length);
   }
 
   @Test
   public void testFormatSchedulerConf() throws Exception {
     CapacitySchedulerConfiguration newConf = getSchedulerConf();
     assertNotNull(newConf);
-    assertEquals(3, newConf.getQueues("root").length);
+    assertEquals(4, newConf.getQueues("root").length);
 
     SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
     Map<String, String> nearEmptyCapacity = new HashMap<>();
@@ -215,12 +236,12 @@ public void testFormatSchedulerConf() throws Exception {
     ClientResponse response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo,
+        .entity(toJson(updateInfo,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
     newConf = getSchedulerConf();
     assertNotNull(newConf);
-    assertEquals(4, newConf.getQueues("root").length);
+    assertEquals(5, newConf.getQueues("root").length);
 
     // Format the scheduler config and validate root.formattest is not present
     response = r.path("ws").path("v1").path("cluster")
@@ -229,7 +250,7 @@ public void testFormatSchedulerConf() throws Exception {
         .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     newConf = getSchedulerConf();
-    assertEquals(3, newConf.getQueues("root").length);
+    assertEquals(4, newConf.getQueues("root").length);
   }
 
   private long getConfigVersion() throws Exception {
@@ -255,7 +276,7 @@ public void testSchedulerConfigVersion() throws Exception {
   public void testAddNestedQueue() throws Exception {
     CapacitySchedulerConfiguration orgConf = getSchedulerConf();
     assertNotNull(orgConf);
-    assertEquals(3, orgConf.getQueues("root").length);
+    assertEquals(4, orgConf.getQueues("root").length);
 
     WebResource r = resource();
 
@@ -283,23 +304,23 @@ public void testAddNestedQueue() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(4, newCSConf.getQueues("root").length);
+    assertEquals(5, newCSConf.getQueues("root").length);
     assertEquals(2, newCSConf.getQueues("root.d").length);
-    assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"),
+    assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d1")),
         0.01f);
-    assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
+    assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d2")),
         0.01f);
 
     CapacitySchedulerConfiguration newConf = getSchedulerConf();
     assertNotNull(newConf);
-    assertEquals(4, newConf.getQueues("root").length);
+    assertEquals(5, newConf.getQueues("root").length);
   }
 
   @Test
@@ -322,16 +343,16 @@ public void testAddWithUpdate() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(4, newCSConf.getQueues("root").length);
-    assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f);
-    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+    assertEquals(5, newCSConf.getQueues("root").length);
+    assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d")), 0.01f);
+    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f);
   }
 
   @Test
@@ -349,7 +370,7 @@ public void testUnsetParentQueueOrderingPolicy() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo1,
+        .entity(toJson(updateInfo1,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -370,7 +391,7 @@ public void testUnsetParentQueueOrderingPolicy() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo2,
+        .entity(toJson(updateInfo2,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
 
@@ -400,7 +421,7 @@ public void testUnsetLeafQueueOrderingPolicy() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo1,
+        .entity(toJson(updateInfo1,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -418,7 +439,7 @@ public void testUnsetLeafQueueOrderingPolicy() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo2,
+        .entity(toJson(updateInfo2,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -447,7 +468,7 @@ public void testRemoveQueue() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
@@ -479,7 +500,7 @@ public void testStopWithRemoveQueue() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo,
+        .entity(toJson(updateInfo,
         SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
 
@@ -490,6 +511,46 @@ public void testStopWithRemoveQueue() throws Exception {
     assertEquals("a1", newCSConf.getQueues("root.a")[0]);
   }
 
+  @Test
+  public void testRemoveQueueWhichHasQueueMapping() throws Exception {
+    WebResource r = resource();
+
+    ClientResponse response;
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    // Validate Queue 'mappedqueue' exists before deletion
+    assertNotNull("Failed to setup CapacityScheduler Configuration",
+        cs.getQueue("mappedqueue"));
+
+    // Set state of queue 'mappedqueue' to STOPPED.
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> stoppedParam = new HashMap<>();
+    stoppedParam.put(CapacitySchedulerConfiguration.STATE, QueueState.STOPPED.toString());
+    QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.mappedqueue", stoppedParam);
+    updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+
+    // Remove queue 'mappedqueue' using update scheduler-conf
+    updateInfo.getRemoveQueueInfo().add("root.mappedqueue");
+    response = r.path("ws").path("v1").path("cluster").path("scheduler-conf")
+        .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON)
+        .entity(YarnWebServiceUtils.toJson(updateInfo, SchedConfUpdateInfo.class),
+            MediaType.APPLICATION_JSON).put(ClientResponse.class);
+    String responseText = response.getEntity(String.class);
+
+    // Queue 'mappedqueue' deletion will fail as there is queue mapping present
+    assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+    assertTrue(responseText.contains(
+        "Failed to re-init queues : " + "org.apache.hadoop.yarn.exceptions.YarnException:"
+            + " Path root 'mappedqueue' does not exist. Path 'mappedqueue' is invalid"));
+
+    // Validate queue 'mappedqueue' exists after above failure
+    CapacitySchedulerConfiguration newCSConf =
+        ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+    assertEquals(4, newCSConf.getQueues("root").length);
+    assertNotNull("CapacityScheduler Configuration is corrupt",
+        cs.getQueue("mappedqueue"));
+  }
+
   @Test
   public void testStopWithConvertLeafToParentQueue() throws Exception {
     WebResource r = resource();
@@ -512,7 +573,7 @@ public void testStopWithConvertLeafToParentQueue() throws Exception {
     response = r.path("ws").path("v1").path("cluster")
         .path("scheduler-conf").queryParam("user.name", userName)
         .accept(MediaType.APPLICATION_JSON)
-        .entity(YarnWebServiceUtils.toJson(updateInfo,
+        .entity(toJson(updateInfo,
             SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
         .put(ClientResponse.class);
 
@@ -537,14 +598,14 @@ public void testRemoveParentQueue() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(2, newCSConf.getQueues("root").length);
+    assertEquals(3, newCSConf.getQueues("root").length);
     assertNull(newCSConf.getQueues("root.c"));
   }
 
@@ -568,15 +629,15 @@ public void testRemoveParentQueueWithCapacity() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(2, newCSConf.getQueues("root").length);
-    assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"),
+    assertEquals(3, newCSConf.getQueues("root").length);
+    assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")),
         0.01f);
   }
 
@@ -600,14 +661,14 @@ public void testRemoveMultipleQueues() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
 
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(1, newCSConf.getQueues("root").length);
+    assertEquals(2, newCSConf.getQueues("root").length);
   }
 
   private void stopQueue(String... queuePaths) throws Exception {
@@ -628,7 +689,7 @@ private void stopQueue(String... queuePaths) throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -663,7 +724,7 @@ public void testUpdateQueue() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     LOG.debug("Response headers: " + response.getHeaders());
@@ -682,7 +743,7 @@ public void testUpdateQueue() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -712,14 +773,14 @@ public void testUpdateQueueCapacity() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
     CapacitySchedulerConfiguration newCSConf =
         ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
-    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f);
-    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.01f);
+    assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f);
   }
 
   @Test
@@ -737,7 +798,7 @@ public void testGlobalConfChange() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -752,7 +813,7 @@ public void testGlobalConfChange() throws Exception {
         r.path("ws").path("v1").path("cluster")
             .path("scheduler-conf").queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .put(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -763,6 +824,220 @@ public void testGlobalConfChange() throws Exception {
         newCSConf.getMaximumSystemApplications());
   }
 
+  @Test
+  public void testNodeLabelRemovalResidualConfigsAreCleared() throws Exception {
+    WebResource r = resource();
+    ClientResponse response;
+
+    // 1. Create Node Label: label1
+    NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo();
+    nodeLabelsInfo.getNodeLabelsInfo().add(new NodeLabelInfo(LABEL_1));
+    WebResource addNodeLabelsResource = r.path("ws").path("v1").path("cluster")
+        .path("add-node-labels");
+    WebResource getNodeLabelsResource = r.path("ws").path("v1").path("cluster")
+        .path("get-node-labels");
+    WebResource removeNodeLabelsResource = r.path("ws").path("v1").path("cluster")
+        .path("remove-node-labels");
+    WebResource schedulerConfResource = r.path("ws").path("v1").path("cluster")
+        .path(RMWSConsts.SCHEDULER_CONF);
+    response =
+        addNodeLabelsResource.queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity(logAndReturnJson(addNodeLabelsResource,
+                    toJson(nodeLabelsInfo, NodeLabelsInfo.class)),
+                MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+
+    // 2. Verify new Node Label
+    response =
+        getNodeLabelsResource.queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class);
+    assertEquals(1, nodeLabelsInfo.getNodeLabels().size());
+    for (NodeLabelInfo nl : nodeLabelsInfo.getNodeLabelsInfo()) {
+      assertEquals(LABEL_1, nl.getName());
+      assertTrue(nl.getExclusivity());
+    }
+
+    // 3. Assign 'label1' to root.a
+    SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+    Map<String, String> updateForRoot = new HashMap<>();
+    updateForRoot.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "*");
+    QueueConfigInfo rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot);
+
+    Map<String, String> updateForRootA = new HashMap<>();
+    updateForRootA.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, LABEL_1);
+    QueueConfigInfo rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA);
+
+    updateInfo.getUpdateQueueInfo().add(rootUpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo);
+
+    response =
+        schedulerConfResource
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo,
+                SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON)
+            .put(ClientResponse.class);
+    assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    assertEquals(Sets.newHashSet("*"),
+        cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath()));
+    assertEquals(Sets.newHashSet(LABEL_1),
+        cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath()));
+
+    // 4. Set partition capacities to queues as below
+    updateInfo = new SchedConfUpdateInfo();
+    updateForRoot = new HashMap<>();
+    updateForRoot.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "100");
+    updateForRoot.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "100");
+    rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot);
+
+    updateForRootA = new HashMap<>();
+    updateForRootA.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "100");
+    updateForRootA.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "100");
+    rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA);
+
+    // Avoid the following exception by adding some capacities to root.a.a1 and root.a.a2 to label1
+    // Illegal capacity sum of 0.0 for children of queue a for label=label1.
+    // It is set to 0, but parent percent != 0, and doesn't allow children capacity to set to 0
+    Map<String, String> updateForRootA_A1 = new HashMap<>();
+    updateForRootA_A1.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "20");
+    updateForRootA_A1.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "20");
+    QueueConfigInfo rootA_A1UpdateInfo = new QueueConfigInfo(ROOT_A_A1.getFullPath(),
+        updateForRootA_A1);
+
+    Map<String, String> updateForRootA_A2 = new HashMap<>();
+    updateForRootA_A2.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "80");
+    updateForRootA_A2.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "80");
+    QueueConfigInfo rootA_A2UpdateInfo = new QueueConfigInfo(ROOT_A_A2.getFullPath(),
+        updateForRootA_A2);
+
+
+    updateInfo.getUpdateQueueInfo().add(rootUpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootA_A1UpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootA_A2UpdateInfo);
+
+    response =
+        schedulerConfResource
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo,
+                SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON)
+            .put(ClientResponse.class);
+    assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+    assertEquals(100.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT, LABEL_1), 0.001f);
+    assertEquals(100.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT, LABEL_1),
+        0.001f);
+    assertEquals(100.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A, LABEL_1), 0.001f);
+    assertEquals(100.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A, LABEL_1),
+        0.001f);
+    assertEquals(20.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A_A1, LABEL_1), 0.001f);
+    assertEquals(20.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A_A1, LABEL_1),
+        0.001f);
+    assertEquals(80.0, cs.getConfiguration().getLabeledQueueCapacity(ROOT_A_A2, LABEL_1), 0.001f);
+    assertEquals(80.0, cs.getConfiguration().getLabeledQueueMaximumCapacity(ROOT_A_A2, LABEL_1),
+        0.001f);
+
+    //5. De-assign node label: "label1" + Remove residual properties
+    updateInfo = new SchedConfUpdateInfo();
+    updateForRoot = new HashMap<>();
+    updateForRoot.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "*");
+    updateForRoot.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "");
+    updateForRoot.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "");
+    rootUpdateInfo = new QueueConfigInfo(ROOT.getFullPath(), updateForRoot);
+
+    updateForRootA = new HashMap<>();
+    updateForRootA.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "");
+    updateForRootA.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "");
+    updateForRootA.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "");
+    rootAUpdateInfo = new QueueConfigInfo(ROOT_A.getFullPath(), updateForRootA);
+
+    updateForRootA_A1 = new HashMap<>();
+    updateForRootA_A1.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "");
+    updateForRootA_A1.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "");
+    updateForRootA_A1.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "");
+    rootA_A1UpdateInfo = new QueueConfigInfo(ROOT_A_A1.getFullPath(), updateForRootA_A1);
+
+    updateForRootA_A2 = new HashMap<>();
+    updateForRootA_A2.put(CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS, "");
+    updateForRootA_A2.put(getAccessibleNodeLabelsCapacityPropertyName(LABEL_1), "");
+    updateForRootA_A2.put(getAccessibleNodeLabelsMaxCapacityPropertyName(LABEL_1), "");
+    rootA_A2UpdateInfo = new QueueConfigInfo(ROOT_A_A2.getFullPath(), updateForRootA_A2);
+
+    updateInfo.getUpdateQueueInfo().add(rootUpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootAUpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootA_A1UpdateInfo);
+    updateInfo.getUpdateQueueInfo().add(rootA_A2UpdateInfo);
+
+    response =
+        schedulerConfResource
+            .queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON)
+            .entity(logAndReturnJson(schedulerConfResource, toJson(updateInfo,
+                SchedConfUpdateInfo.class)), MediaType.APPLICATION_JSON)
+            .put(ClientResponse.class);
+    assertEquals(Status.OK.getStatusCode(), response.getStatus());
+    assertEquals(Sets.newHashSet("*"),
+        cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath()));
+    assertNull(cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath()));
+
+    //6. Remove node label 'label1'
+    MultivaluedMapImpl params = new MultivaluedMapImpl();
+    params.add("labels", LABEL_1);
+    response =
+        removeNodeLabelsResource
+            .queryParam("user.name", userName)
+            .queryParams(params)
+            .accept(MediaType.APPLICATION_JSON)
+            .post(ClientResponse.class);
+
+    // Verify
+    response =
+        getNodeLabelsResource.queryParam("user.name", userName)
+            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class);
+    assertEquals(0, nodeLabelsInfo.getNodeLabels().size());
+
+    //6. Check residual configs
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT, LABEL_1, CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT, LABEL_1, MAXIMUM_CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A, LABEL_1, CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A, LABEL_1, MAXIMUM_CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A1, LABEL_1, CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A1, LABEL_1, MAXIMUM_CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A2, LABEL_1, CAPACITY));
+    assertNull(getConfValueForQueueAndLabelAndType(cs, ROOT_A_A2, LABEL_1, MAXIMUM_CAPACITY));
+  }
+
+  private String getConfValueForQueueAndLabelAndType(CapacityScheduler cs,
+      QueuePath queuePath, String label, String type) {
+    return cs.getConfiguration().get(
+        CapacitySchedulerConfiguration.getNodeLabelPrefix(
+            queuePath.getFullPath(), label) + type);
+  }
+
+  private Object logAndReturnJson(WebResource ws, String json) {
+    LOG.info("Sending to web resource: {}, json: {}", ws, json);
+    return json;
+  }
+
+  private String getAccessibleNodeLabelsCapacityPropertyName(String label) {
+    return String.format("%s.%s.%s", ACCESSIBLE_NODE_LABELS, label, CAPACITY);
+  }
+
+  private String getAccessibleNodeLabelsMaxCapacityPropertyName(String label) {
+    return String.format("%s.%s.%s", ACCESSIBLE_NODE_LABELS, label, MAXIMUM_CAPACITY);
+  }
+
   @Test
   public void testValidateWithClusterMaxAllocation() throws Exception {
     WebResource r = resource();
@@ -783,7 +1058,7 @@ public void testValidateWithClusterMaxAllocation() throws Exception {
             .path(RMWSConsts.SCHEDULER_CONF_VALIDATE)
             .queryParam("user.name", userName)
             .accept(MediaType.APPLICATION_JSON)
-            .entity(YarnWebServiceUtils.toJson(updateInfo,
+            .entity(toJson(updateInfo,
                 SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
             .post(ClientResponse.class);
     assertEquals(Status.OK.getStatusCode(), response.getStatus());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
index 095b076df73a1..0e497da49e084 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
@@ -48,6 +48,7 @@
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -697,7 +698,7 @@ private void verifySimpleAuthCancel() {
 
   public static DelegationToken getDelegationTokenFromXML(String tokenXML)
       throws IOException, ParserConfigurationException, SAXException {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(tokenXML));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
index 36b27a232383b..6af918070419d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
@@ -42,6 +42,7 @@
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.Priority;
@@ -258,7 +259,7 @@ public void testSchedulerPartitionsXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -574,7 +575,7 @@ private void verifySchedulerInfoJson(JSONObject json)
     JSONObject info = json.getJSONObject("scheduler");
     assertEquals("incorrect number of elements", 1, info.length());
     info = info.getJSONObject("schedulerInfo");
-    assertEquals("incorrect number of elements", 22, info.length());
+    assertEquals("incorrect number of elements", 24, info.length());
     JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
     JSONArray partitionsCapsArray =
         capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index f1ea0fba0339d..503b4a8b2bbdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,9 +26,15 @@
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
 
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.util.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.http.JettyUtils;
@@ -55,7 +61,6 @@
 import com.google.inject.Guice;
 import com.google.inject.servlet.ServletModule;
 import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.json.JSONJAXBContext;
 import com.sun.jersey.api.json.JSONMarshaller;
@@ -69,6 +74,28 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
 
   private static final Logger LOG = LoggerFactory
       .getLogger(TestRMWebServicesNodeLabels.class);
+  private static final String NODE_0 = "nid:0";
+  private static final String NODE_1 = "nid1:0";
+  private static final String NODE_2 = "nid2:0";
+  private static final String LABEL_A = "a";
+  private static final String LABEL_B = "b";
+  private static final String LABEL_X = "x";
+  private static final String LABEL_Y = "y";
+  private static final String LABEL_Z = "z";
+  public static final boolean DEFAULT_NL_EXCLUSIVITY = true;
+  private static final String PATH_WS = "ws";
+  private static final String PATH_V1 = "v1";
+  private static final String PATH_NODES = "nodes";
+  private static final String PATH_CLUSTER = "cluster";
+  private static final String PATH_REPLACE_NODE_TO_LABELS = "replace-node-to-labels";
+  private static final String PATH_LABEL_MAPPINGS = "label-mappings";
+  private static final String PATH_GET_LABELS = "get-labels";
+  private static final String PATH_REPLACE_LABELS = "replace-labels";
+  private static final String PATH_REMOVE_LABELS = "remove-node-labels";
+  private static final String PATH_GET_NODE_LABELS = "get-node-labels";
+  private static final String PATH_GET_NODE_TO_LABELS = "get-node-to-labels";
+  private static final String QUERY_USER_NAME = "user.name";
+  private static final String PATH_ADD_NODE_LABELS = "add-node-labels";
 
   private static MockRM rm;
   private static YarnConfiguration conf;
@@ -76,6 +103,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
   private static String userName;
   private static String notUserName;
   private static RMWebServices rmWebService;
+  private WebResource resource;
+
 
   private static class WebServletModule extends ServletModule {
 
@@ -92,7 +121,7 @@ protected void configureServlets() {
       conf = new YarnConfiguration();
       conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
       rm = new MockRM(conf);
-      rmWebService = new RMWebServices(rm,conf);
+      rmWebService = new RMWebServices(rm, conf);
       bind(RMWebServices.class).toInstance(rmWebService);
       bind(GenericExceptionHandler.class);
       bind(ResourceManager.class).toInstance(rm);
@@ -100,7 +129,7 @@ protected void configureServlets() {
           TestRMWebServicesAppsModification.TestRMCustomAuthFilter.class);
       serve("/*").with(GuiceContainer.class);
     }
-  };
+  }
 
   @Override
   @Before
@@ -108,6 +137,7 @@ public void setUp() throws Exception {
     super.setUp();
     GuiceServletConfig.setInjector(
         Guice.createInjector(new WebServletModule()));
+    resource = resource();
   }
 
   public TestRMWebServicesNodeLabels() {
@@ -118,528 +148,444 @@ public TestRMWebServicesNodeLabels() {
         .contextPath("jersey-guice-filter").servletPath("/").build());
   }
 
-  @Test
-  public void testNodeLabels() throws JSONException, Exception {
-    WebResource r = resource();
+  private WebResource getClusterWebResource() {
+    return resource.path(PATH_WS).path(PATH_V1).path(PATH_CLUSTER);
+  }
+
+  private ClientResponse get(String path) {
+    return getClusterWebResource()
+        .path(path)
+        .queryParam(QUERY_USER_NAME, userName)
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+  }
+
+  private ClientResponse get(String path, MultivaluedMapImpl queryParams) {
+    return getClusterWebResource()
+        .path(path)
+        .queryParam(QUERY_USER_NAME, userName)
+        .queryParams(queryParams)
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+  }
+
+  private ClientResponse post(String path, String userName, Object payload,
+      Class<?> payloadClass) throws Exception {
+    return getClusterWebResource()
+        .path(path)
+        .queryParam(QUERY_USER_NAME, userName)
+        .accept(MediaType.APPLICATION_JSON)
+        .entity(toJson(payload, payloadClass),
+            MediaType.APPLICATION_JSON)
+        .post(ClientResponse.class);
+  }
 
+  private ClientResponse post(String path, String userName, Object payload,
+      Class<?> payloadClass, MultivaluedMapImpl queryParams) throws Exception {
+    WebResource.Builder builder = getClusterWebResource()
+        .path(path)
+        .queryParam(QUERY_USER_NAME, userName)
+        .queryParams(queryParams)
+        .accept(MediaType.APPLICATION_JSON);
+
+    if (payload != null && payloadClass != null) {
+      builder.entity(toJson(payload, payloadClass), MediaType.APPLICATION_JSON);
+    }
+    return builder.post(ClientResponse.class);
+  }
+
+  @Test
+  public void testNodeLabels() throws Exception {
     ClientResponse response;
 
     // Add a label
-    NodeLabelsInfo nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a"));
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("add-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(nlsifo, NodeLabelsInfo.class),
-                MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = addNodeLabels(Lists.newArrayList(Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY)));
+    assertHttp200(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(1, nlsifo.getNodeLabels().size());
-    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
-      assertEquals("a", nl.getName());
-      assertTrue(nl.getExclusivity());
-    }
-    
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfo(response.getEntity(NodeLabelsInfo.class), Lists.newArrayList(
+        Pair.of(LABEL_A, true)));
+
     // Add another
-    nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("add-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(nlsifo, NodeLabelsInfo.class),
-                MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = addNodeLabels(Lists.newArrayList(Pair.of(LABEL_B, false)));
+    assertHttp200(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(2, nlsifo.getNodeLabels().size());
-    // Verify exclusivity for 'y' as false
-    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
-      if (nl.getName().equals("b")) {
-        assertFalse(nl.getExclusivity());
-      }
-    }
-    
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    // Verify exclusivity for 'b' as false
+    assertNodeLabelsInfo(response.getEntity(NodeLabelsInfo.class),
+        Lists.newArrayList(
+            Pair.of(LABEL_A, true),
+            Pair.of(LABEL_B, false)));
+
     // Add labels to a node
-    MultivaluedMapImpl params = new MultivaluedMapImpl();
-    params.add("labels", "a");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_0, LABEL_A);
+    assertHttp200(response);
 
     // Add labels to another node
-    params = new MultivaluedMapImpl();
-    params.add("labels", "b");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid1:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_1, LABEL_B);
+    assertHttp200(response);
 
     // Add labels to another node
-    params = new MultivaluedMapImpl();
-    params.add("labels", "b");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid2:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
-
-    // Verify, using get-labels-to-Nodes
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("label-mappings").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
-    assertEquals(2, ltni.getLabelsToNodes().size());
-    NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
-        new NodeLabelInfo("b", false));
-    assertTrue(nodes.getNodeIDs().contains("nid2:0"));
-    assertTrue(nodes.getNodeIDs().contains("nid1:0"));
-    nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
-    assertTrue(nodes.getNodeIDs().contains("nid:0"));
+    response = replaceLabelsOnNode(NODE_2, LABEL_B);
+    assertHttp200(response);
+
+    // Verify all, using get-labels-to-Nodes
+    response = getNodeLabelMappings();
+    assertApplicationJsonUtf8Response(response);
+    LabelsToNodesInfo labelsToNodesInfo = response.getEntity(LabelsToNodesInfo.class);
+    assertLabelsToNodesInfo(labelsToNodesInfo, 2, Lists.newArrayList(
+        Pair.of(Pair.of(LABEL_B, false), Lists.newArrayList(NODE_1, NODE_2)),
+        Pair.of(Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY), Lists.newArrayList(NODE_0))
+    ));
 
     // Verify, using get-labels-to-Nodes for specified set of labels
-    params = new MultivaluedMapImpl();
-    params.add("labels", "a");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("label-mappings").queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    ltni = response.getEntity(LabelsToNodesInfo.class);
-    assertEquals(1, ltni.getLabelsToNodes().size());
-    nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
-    assertTrue(nodes.getNodeIDs().contains("nid:0"));
+    response = getNodeLabelMappingsByLabels(LABEL_A);
+    assertApplicationJsonUtf8Response(response);
+    labelsToNodesInfo = response.getEntity(LabelsToNodesInfo.class);
+    assertLabelsToNodesInfo(labelsToNodesInfo, 1, Lists.newArrayList(
+        Pair.of(Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY), Lists.newArrayList(NODE_0))
+    ));
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("get-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
+    response = getLabelsOfNode(NODE_0);
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoContains(response.getEntity(NodeLabelsInfo.class),
+        Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY));
 
-    
     // Replace
-    params = new MultivaluedMapImpl();
-    params.add("labels", "b");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_0, LABEL_B);
+    assertHttp200(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("get-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().contains(
-        new NodeLabelInfo("b", false)));
-            
+    response = getLabelsOfNode(NODE_0);
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoContains(response.getEntity(NodeLabelsInfo.class), Pair.of(LABEL_B, false));
+
     // Replace labels using node-to-labels
-    NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
-    ArrayList<String> labels = new ArrayList<String>();
-    labels.add("a");
-    NodeToLabelsEntry nli = new NodeToLabelsEntry("nid:0", labels);
-    ntli.getNodeToLabels().add(nli);
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("replace-node-to-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(ntli, NodeToLabelsEntryList.class),
-              MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-        
+    response = replaceNodeToLabels(Lists.newArrayList(Pair.of(NODE_0,
+        Lists.newArrayList(LABEL_A))));
+    assertHttp200(response);
+
     // Verify, using node-to-labels
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-to-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
-    NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
-    assertEquals(1, nlinfo.getNodeLabels().size());
-    assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
-    
+    response = getNodeToLabels();
+    assertApplicationJsonUtf8Response(response);
+    NodeToLabelsInfo nodeToLabelsInfo = response.getEntity(NodeToLabelsInfo.class);
+    NodeLabelsInfo nodeLabelsInfo = nodeToLabelsInfo.getNodeToLabels().get(NODE_0);
+    assertNodeLabelsSize(nodeLabelsInfo, 1);
+    assertNodeLabelsInfoContains(nodeLabelsInfo, Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY));
+
     // Remove all
-    params = new MultivaluedMapImpl();
-    params.add("labels", "");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_0, "");
+    assertHttp200(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("get-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
-    
+    response = getLabelsOfNode(NODE_0);
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsSize(response.getEntity(NodeLabelsInfo.class), 0);
+
     // Add a label back for auth tests
-    params = new MultivaluedMapImpl();
-    params.add("labels", "a");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_0, LABEL_A);
+    assertHttp200(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("get-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
-    
+    response = getLabelsOfNode(NODE_0);
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoContains(response.getEntity(NodeLabelsInfo.class),
+        Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY));
+
     // Auth fail replace labels on node
-    params = new MultivaluedMapImpl();
-    params.add("labels", "b");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("replace-labels")
-            .queryParam("user.name", notUserName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = replaceLabelsOnNodeWithUserName(NODE_0, notUserName, LABEL_B);
+    assertHttp401(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nid:0")
-            .path("get-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
-    
-    // Fail to add a label with post
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("add-node-labels").queryParam("user.name", notUserName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity("{\"nodeLabels\":\"c\"}", MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = getLabelsOfNode(NODE_0);
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoContains(response.getEntity(NodeLabelsInfo.class),
+        Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY));
+
+    // Fail to add a label with wrong user
+    response = addNodeLabelsWithUser(Lists.newArrayList(Pair.of("c", DEFAULT_NL_EXCLUSIVITY)),
+        notUserName);
+    assertHttp401(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(2, nlsifo.getNodeLabels().size());
-    
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsSize(response.getEntity(NodeLabelsInfo.class), 2);
+
     // Remove cluster label (succeed, we no longer need it)
-    params = new MultivaluedMapImpl();
-    params.add("labels", "b");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("remove-node-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = removeNodeLabel(LABEL_B);
+    assertHttp200(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(1, nlsifo.getNodeLabels().size());
-    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
-      assertEquals("a", nl.getName());
-      assertTrue(nl.getExclusivity());
-    }
-    
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfo(response.getEntity(NodeLabelsInfo.class),
+        Lists.newArrayList(Pair.of(LABEL_A, true)));
+
     // Remove cluster label with post
-    params = new MultivaluedMapImpl();
-    params.add("labels", "a");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("remove-node-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = removeNodeLabel(LABEL_A);
+    assertHttp200(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(0, nlsifo.getNodeLabels().size());
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class);
+    assertEquals(0, nodeLabelsInfo.getNodeLabels().size());
 
     // Following test cases are to test replace when distributed node label
     // configuration is on
     // Reset for testing : add cluster labels
-    nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
-    response =
-        r.path("ws")
-            .path("v1")
-            .path("cluster")
-            .path("add-node-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(nlsifo, NodeLabelsInfo.class),
-                MediaType.APPLICATION_JSON).post(ClientResponse.class);
+    response = addNodeLabels(Lists.newArrayList(
+        Pair.of(LABEL_X, false), Pair.of(LABEL_Y, false)));
+    assertHttp200(response);
     // Reset for testing : Add labels to a node
-    params = new MultivaluedMapImpl();
-    params.add("labels", "y");
-    response =
-        r.path("ws").path("v1").path("cluster").path("nodes").path("nid:0")
-            .path("replace-labels").queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
-
-    //setting rmWebService for non Centralized NodeLabel Configuration
+    response = replaceLabelsOnNode(NODE_0, LABEL_Y);
+    assertHttp200(response);
+
+    //setting rmWebService for non-centralized NodeLabel Configuration
     rmWebService.isCentralizedNodeLabelConfiguration = false;
 
     // Case1 : Replace labels using node-to-labels
-    ntli = new NodeToLabelsEntryList();
-    labels = new ArrayList<String>();
-    labels.add("x");
-    nli = new NodeToLabelsEntry("nid:0", labels);
-    ntli.getNodeToLabels().add(nli);
-    response =
-        r.path("ws")
-            .path("v1")
-            .path("cluster")
-            .path("replace-node-to-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(ntli, NodeToLabelsEntryList.class),
-                MediaType.APPLICATION_JSON).post(ClientResponse.class);
+    response = replaceNodeToLabels(Lists.newArrayList(Pair.of(NODE_0,
+        Lists.newArrayList(LABEL_X))));
+    assertHttp404(response);
 
     // Verify, using node-to-labels that previous operation has failed
-    response =
-        r.path("ws").path("v1").path("cluster").path("get-node-to-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    ntlinfo = response.getEntity(NodeToLabelsInfo.class);
-    nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
-    assertEquals(1, nlinfo.getNodeLabels().size());
-    assertFalse(nlinfo.getNodeLabelsInfo().contains(
-        new NodeLabelInfo("x", false)));
+    response = getNodeToLabels();
+    assertApplicationJsonUtf8Response(response);
+    nodeToLabelsInfo = response.getEntity(NodeToLabelsInfo.class);
+    nodeLabelsInfo = nodeToLabelsInfo.getNodeToLabels().get(NODE_0);
+    assertNodeLabelsSize(nodeLabelsInfo, 1);
+    assertNodeLabelsInfoDoesNotContain(nodeLabelsInfo, Pair.of(LABEL_X, false));
 
     // Case2 : failure to Replace labels using replace-labels
-    response =
-        r.path("ws").path("v1").path("cluster").path("nodes").path("nid:0")
-            .path("replace-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity("{\"nodeLabelName\": [\"x\"]}", MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
-    LOG.info("posted node nodelabel");
+    response = replaceLabelsOnNode(NODE_0, LABEL_X);
+    assertHttp404(response);
 
     // Verify, using node-to-labels that previous operation has failed
-    response =
-        r.path("ws").path("v1").path("cluster").path("get-node-to-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    ntlinfo = response.getEntity(NodeToLabelsInfo.class);
-    nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
-    assertEquals(1, nlinfo.getNodeLabels().size());
-    assertFalse(nlinfo.getNodeLabelsInfo().contains(
-        new NodeLabelInfo("x", false)));
+    response = getNodeToLabels();
+    assertApplicationJsonUtf8Response(response);
+    nodeToLabelsInfo = response.getEntity(NodeToLabelsInfo.class);
+    nodeLabelsInfo = nodeToLabelsInfo.getNodeToLabels().get(NODE_0);
+    assertNodeLabelsSize(nodeLabelsInfo, 1);
+    assertNodeLabelsInfoDoesNotContain(nodeLabelsInfo, Pair.of(LABEL_X, false));
 
     //  Case3 : Remove cluster label should be successful
-    params = new MultivaluedMapImpl();
-    params.add("labels", "x");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("remove-node-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = removeNodeLabel(LABEL_X);
+    assertHttp200(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(new NodeLabelInfo("y", false),
-        nlsifo.getNodeLabelsInfo().get(0));
-    assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
-    assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoAtPosition(response.getEntity(NodeLabelsInfo.class), Pair.of(LABEL_Y,
+        false), 0);
 
     // Remove y
-    params = new MultivaluedMapImpl();
-    params.add("labels", "y");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("remove-node-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = removeNodeLabel(LABEL_Y);
+    assertHttp200(response);
 
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
-
-    // add a new nodelabel with exclusity
-    nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("z", false));
-    response =
-        r.path("ws")
-            .path("v1")
-            .path("cluster")
-            .path("add-node-labels")
-            .queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(nlsifo, NodeLabelsInfo.class),
-                MediaType.APPLICATION_JSON).post(ClientResponse.class);
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsSize(response.getEntity(NodeLabelsInfo.class), 0);
 
+    // add a new nodelabel with exclusivity=false
+    response = addNodeLabels(Lists.newArrayList(Pair.of(LABEL_Z, false)));
+    assertHttp200(response);
     // Verify
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    assertNodeLabelsInfoAtPosition(response.getEntity(NodeLabelsInfo.class),
+        Pair.of(LABEL_Z, false), 0);
+    assertNodeLabelsSize(nodeLabelsInfo, 1);
+  }
+
+  private void assertLabelsToNodesInfo(LabelsToNodesInfo labelsToNodesInfo, int size,
+      List<Pair<Pair<String, Boolean>, List<String>>> nodeLabelsToNodesList) {
+    Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes = labelsToNodesInfo.getLabelsToNodes();
+    assertNotNull("Labels to nodes mapping should not be null.", labelsToNodes);
+    assertEquals("Size of label to nodes mapping is not the expected.", size, labelsToNodes.size());
+
+    for (Pair<Pair<String, Boolean>, List<String>> nodeLabelToNodes : nodeLabelsToNodesList) {
+      Pair<String, Boolean> expectedNLData = nodeLabelToNodes.getLeft();
+      List<String> expectedNodes = nodeLabelToNodes.getRight();
+      NodeLabelInfo expectedNLInfo = new NodeLabelInfo(expectedNLData.getLeft(),
+          expectedNLData.getRight());
+      NodeIDsInfo actualNodes = labelsToNodes.get(expectedNLInfo);
+      assertNotNull(String.format("Node info not found. Expected NodeLabel data: %s",
+          expectedNLData), actualNodes);
+      for (String expectedNode : expectedNodes) {
+        assertTrue(String.format("Can't find node ID in actual Node IDs list: %s",
+                actualNodes.getNodeIDs()), actualNodes.getNodeIDs().contains(expectedNode));
+      }
+    }
+  }
+
+  private void assertNodeLabelsInfo(NodeLabelsInfo nodeLabelsInfo,
+      List<Pair<String, Boolean>> nlInfos) {
+    assertEquals(nlInfos.size(), nodeLabelsInfo.getNodeLabels().size());
+
+    for (int i = 0; i < nodeLabelsInfo.getNodeLabelsInfo().size(); i++) {
+      Pair<String, Boolean> expected = nlInfos.get(i);
+      NodeLabelInfo actual = nodeLabelsInfo.getNodeLabelsInfo().get(i);
+      LOG.debug("Checking NodeLabelInfo: {}", actual);
+      assertEquals(expected.getLeft(), actual.getName());
+      assertEquals(expected.getRight(), actual.getExclusivity());
+    }
+  }
+
+  private void assertNodeLabelsInfoAtPosition(NodeLabelsInfo nodeLabelsInfo, Pair<String,
+      Boolean> nlInfo, int pos) {
+    NodeLabelInfo actual = nodeLabelsInfo.getNodeLabelsInfo().get(pos);
+    LOG.debug("Checking NodeLabelInfo: {}", actual);
+    assertEquals(nlInfo.getLeft(), actual.getName());
+    assertEquals(nlInfo.getRight(), actual.getExclusivity());
+  }
+
+  private void assertNodeLabelsInfoContains(NodeLabelsInfo nodeLabelsInfo,
+      Pair<String, Boolean> nlInfo) {
+    NodeLabelInfo nodeLabelInfo = new NodeLabelInfo(nlInfo.getLeft(), nlInfo.getRight());
+    assertTrue(String.format("Cannot find nodeLabelInfo '%s' among items of node label info list:" +
+            " %s", nodeLabelInfo, nodeLabelsInfo.getNodeLabelsInfo()),
+        nodeLabelsInfo.getNodeLabelsInfo().contains(nodeLabelInfo));
+  }
+
+  private void assertNodeLabelsInfoDoesNotContain(NodeLabelsInfo nodeLabelsInfo, Pair<String,
+      Boolean> nlInfo) {
+    NodeLabelInfo nodeLabelInfo = new NodeLabelInfo(nlInfo.getLeft(), nlInfo.getRight());
+    assertFalse(String.format("Should have not found nodeLabelInfo '%s' among " +
+        "items of node label info list: %s", nodeLabelInfo, nodeLabelsInfo.getNodeLabelsInfo()),
+        nodeLabelsInfo.getNodeLabelsInfo().contains(nodeLabelInfo));
+  }
+
+  private void assertNodeLabelsSize(NodeLabelsInfo nodeLabelsInfo, int expectedSize) {
+    assertEquals(expectedSize, nodeLabelsInfo.getNodeLabelsInfo().size());
+  }
+
+  private ClientResponse replaceNodeToLabels(List<Pair<String, List<String>>> nodeToLabelInfos) throws Exception {
+    NodeToLabelsEntryList nodeToLabelsEntries = new NodeToLabelsEntryList();
+
+    for (Pair<String, List<String>> nodeToLabelInfo : nodeToLabelInfos) {
+      ArrayList<String> labelList = new ArrayList<>(nodeToLabelInfo.getRight());
+      String nodeId = nodeToLabelInfo.getLeft();
+      NodeToLabelsEntry nli = new NodeToLabelsEntry(nodeId, labelList);
+      nodeToLabelsEntries.getNodeToLabels().add(nli);
+    }
+    return post(PATH_REPLACE_NODE_TO_LABELS, userName, nodeToLabelsEntries, NodeToLabelsEntryList.class);
+  }
+
+  private ClientResponse getNodeLabelMappings() {
+    return get(PATH_LABEL_MAPPINGS);
+  }
+
+  private ClientResponse getNodeLabelMappingsByLabels(String... labelNames) {
+    MultivaluedMapImpl params = createMultiValuedMap(labelNames);
+    return get(PATH_LABEL_MAPPINGS, params);
+  }
+
+  private ClientResponse replaceLabelsOnNode(String node, String... labelNames) throws Exception {
+    return replaceLabelsOnNodeWithUserName(node, userName, labelNames);
+  }
+
+  private ClientResponse replaceLabelsOnNodeWithUserName(String node,
+      String userName, String... labelNames) throws Exception {
+    LOG.info("Replacing labels on node '{}', label(s): {}", node, labelNames);
+    MultivaluedMapImpl params = createMultiValuedMap(labelNames);
+    String path = UriBuilder.fromPath(PATH_NODES).path(node)
+        .path(PATH_REPLACE_LABELS).build().toString();
+    return post(path, userName, null, null, params);
+  }
+
+  private static MultivaluedMapImpl createMultiValuedMap(String[] labelNames) {
+    MultivaluedMapImpl params = new MultivaluedMapImpl();
+    for (String labelName : labelNames) {
+      params.add("labels", labelName);
+    }
+    return params;
+  }
+
+  private ClientResponse removeNodeLabel(String... labelNames) throws Exception {
+    MultivaluedMapImpl params = createMultiValuedMap(labelNames);
+    return post(PATH_REMOVE_LABELS, userName, null, null, params);
+  }
+
+  private ClientResponse getLabelsOfNode(String node) {
+    String path = UriBuilder.fromPath(PATH_NODES).path(node)
+        .path(PATH_GET_LABELS).build().toString();
+    return get(path);
+  }
+
+  private ClientResponse getNodeLabels() {
+    return get(PATH_GET_NODE_LABELS);
+  }
+
+  private ClientResponse getNodeToLabels() {
+    return get(PATH_GET_NODE_TO_LABELS);
+  }
+
+  private ClientResponse addNodeLabels(List<Pair<String, Boolean>> nlInfos) throws Exception {
+    return addNodeLabelsInternal(nlInfos, userName);
+  }
+
+  private ClientResponse addNodeLabelsWithUser(List<Pair<String, Boolean>> nlInfos,
+      String userName) throws Exception {
+    return addNodeLabelsInternal(nlInfos, userName);
+  }
+
+  private ClientResponse addNodeLabelsInternal(List<Pair<String, Boolean>> nlInfos,
+      String userName) throws Exception {
+    NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo();
+    for (Pair<String, Boolean> nlInfo : nlInfos) {
+      NodeLabelInfo nodeLabelInfo = new NodeLabelInfo(nlInfo.getLeft(), nlInfo.getRight());
+      nodeLabelsInfo.getNodeLabelsInfo().add(nodeLabelInfo);
+    }
+    return post(PATH_ADD_NODE_LABELS, userName, nodeLabelsInfo, NodeLabelsInfo.class);
+  }
+
+  private void assertApplicationJsonUtf8Response(ClientResponse response) {
     assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
-    assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
-    assertEquals(1, nlsifo.getNodeLabels().size());
+  }
+
+  private void assertHttp200(ClientResponse response) {
+    assertEquals(Response.Status.OK.getStatusCode(), response.getStatus());
+  }
+
+  private void assertHttp401(ClientResponse response) {
+    assertEquals(Response.Status.UNAUTHORIZED.getStatusCode(), response.getStatus());
+  }
+
+  private void assertHttp404(ClientResponse response) {
+    assertEquals(Response.Status.NOT_FOUND.getStatusCode(), response.getStatus());
   }
 
   @Test
   public void testLabelInvalidAddition()
-      throws UniformInterfaceException, Exception {
-    WebResource r = resource();
-    ClientResponse response;
+      throws Exception {
     // Add a invalid label
-    NodeLabelsInfo nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a&"));
-    response = r.path("ws").path("v1").path("cluster").path("add-node-labels")
-        .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON)
-        .entity(toJson(nlsifo, NodeLabelsInfo.class),
-            MediaType.APPLICATION_JSON)
-        .post(ClientResponse.class);
-    String expectedmessage =
+    ClientResponse response = addNodeLabels(Lists.newArrayList(Pair.of("a&",
+        DEFAULT_NL_EXCLUSIVITY)));
+    String expectedMessage =
         "java.io.IOException: label name should only contains"
             + " {0-9, a-z, A-Z, -, _} and should not started with"
             + " {-,_}, now it is= a&";
-    validateJsonExceptionContent(response, expectedmessage);
+    validateJsonExceptionContent(response, expectedMessage);
   }
 
   @Test
   public void testLabelChangeExclusivity()
-      throws Exception, JSONException {
-    WebResource r = resource();
+      throws Exception {
     ClientResponse response;
-    NodeLabelsInfo nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("newlabel", true));
-    response = r.path("ws").path("v1").path("cluster").path("add-node-labels")
-        .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON)
-        .entity(toJson(nlsifo, NodeLabelsInfo.class),
-            MediaType.APPLICATION_JSON)
-        .post(ClientResponse.class);
+    response = addNodeLabels(Lists.newArrayList(Pair.of("newLabel", DEFAULT_NL_EXCLUSIVITY)));
+    assertHttp200(response);
     // new info and change exclusivity
-    NodeLabelsInfo nlsinfo2 = new NodeLabelsInfo();
-    nlsinfo2.getNodeLabelsInfo().add(new NodeLabelInfo("newlabel", false));
-    response = r.path("ws").path("v1").path("cluster").path("add-node-labels")
-        .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON)
-        .entity(toJson(nlsinfo2, NodeLabelsInfo.class),
-            MediaType.APPLICATION_JSON)
-        .post(ClientResponse.class);
-    String expectedmessage =
+    response = addNodeLabels(Lists.newArrayList(Pair.of("newLabel", false)));
+    String expectedMessage =
         "java.io.IOException: Exclusivity cannot be modified for an existing"
-            + " label with : <newlabel:exclusivity=false>";
-    validateJsonExceptionContent(response, expectedmessage);
+            + " label with : <newLabel:exclusivity=false>";
+    validateJsonExceptionContent(response, expectedMessage);
   }
 
   private void validateJsonExceptionContent(ClientResponse response,
-      String expectedmessage)
+      String expectedMessage)
       throws JSONException {
     Assert.assertEquals(BAD_REQUEST_CODE, response.getStatus());
     JSONObject msg = response.getEntity(JSONObject.class);
@@ -653,112 +599,76 @@ private void validateJsonExceptionContent(ClientResponse response,
     WebServicesTestUtils.checkStringMatch("exception classname",
         "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
     WebServicesTestUtils.checkStringContains("exception message",
-        expectedmessage, message);
+        expectedMessage, message);
   }
 
   @Test
   public void testLabelInvalidReplace()
-      throws UniformInterfaceException, Exception {
-    WebResource r = resource();
+      throws Exception {
     ClientResponse response;
-    // replace label which doesnt exist
-    MultivaluedMapImpl params = new MultivaluedMapImpl();
-    params.add("labels", "idontexist");
-    response = r.path("ws").path("v1").path("cluster").path("nodes")
-        .path("nid:0").path("replace-labels").queryParam("user.name", userName)
-        .queryParams(params).accept(MediaType.APPLICATION_JSON)
-        .post(ClientResponse.class);
+    // replace label which doesn't exist
+    response = replaceLabelsOnNode(NODE_0, "idontexist");
 
-    String expectedmessage =
+    String expectedMessage =
         "Not all labels being replaced contained by known label"
             + " collections, please check, new labels=[idontexist]";
-    validateJsonExceptionContent(response, expectedmessage);
+    validateJsonExceptionContent(response, expectedMessage);
   }
 
   @Test
   public void testLabelInvalidRemove()
-      throws UniformInterfaceException, Exception {
-    WebResource r = resource();
+      throws Exception {
     ClientResponse response;
-    MultivaluedMapImpl params = new MultivaluedMapImpl();
-    params.add("labels", "irealldontexist");
-    response =
-        r.path("ws").path("v1").path("cluster").path("remove-node-labels")
-            .queryParam("user.name", userName).queryParams(params)
-            .accept(MediaType.APPLICATION_JSON).post(ClientResponse.class);
-    String expectedmessage =
-        "java.io.IOException: Node label=irealldontexist to be"
+    response = removeNodeLabel("ireallydontexist");
+    String expectedMessage =
+        "java.io.IOException: Node label=ireallydontexist to be"
             + " removed doesn't existed in cluster node labels"
             + " collection.";
-    validateJsonExceptionContent(response, expectedmessage);
+    validateJsonExceptionContent(response, expectedMessage);
   }
 
   @Test
   public void testNodeLabelPartitionInfo() throws Exception {
-    WebResource r = resource();
-
     ClientResponse response;
 
     // Add a node label
-    NodeLabelsInfo nlsifo = new NodeLabelsInfo();
-    nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a"));
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("add-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON)
-            .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = addNodeLabels(Lists.newArrayList(Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY)));
+    assertHttp200(response);
 
     // Verify partition info in get-node-labels
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("get-node-labels").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    nlsifo = response.getEntity(NodeLabelsInfo.class);
-    assertEquals(1, nlsifo.getNodeLabels().size());
-    for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
-      assertEquals("a", nl.getName());
+    response = getNodeLabels();
+    assertApplicationJsonUtf8Response(response);
+    NodeLabelsInfo nodeLabelsInfo = response.getEntity(NodeLabelsInfo.class);
+    assertNodeLabelsSize(nodeLabelsInfo, 1);
+    for (NodeLabelInfo nl : nodeLabelsInfo.getNodeLabelsInfo()) {
+      assertEquals(LABEL_A, nl.getName());
       assertTrue(nl.getExclusivity());
       assertNotNull(nl.getPartitionInfo());
       assertNotNull(nl.getPartitionInfo().getResourceAvailable());
     }
 
     // Add node label to a node
-    MultivaluedMapImpl params = new MultivaluedMapImpl();
-    params.add("labels", "a");
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("nodes").path("nodeId:0")
-            .path("replace-labels")
-            .queryParam("user.name", userName)
-            .queryParams(params)
-            .accept(MediaType.APPLICATION_JSON)
-            .post(ClientResponse.class);
+    response = replaceLabelsOnNode("nodeId:0", LABEL_A);
+    assertHttp200(response);
 
     // Verify partition info in label-mappings
-    response =
-        r.path("ws").path("v1").path("cluster")
-            .path("label-mappings").queryParam("user.name", userName)
-            .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
-        response.getType().toString());
-    LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
-    assertEquals(1, ltni.getLabelsToNodes().size());
-    NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
-        new NodeLabelInfo("a"));
-    assertTrue(nodes.getNodeIDs().contains("nodeId:0"));
+    response = getNodeLabelMappings();
+    assertApplicationJsonUtf8Response(response);
+    LabelsToNodesInfo labelsToNodesInfo = response.getEntity(LabelsToNodesInfo.class);
+    assertLabelsToNodesInfo(labelsToNodesInfo, 1, Lists.newArrayList(
+        Pair.of(Pair.of(LABEL_A, DEFAULT_NL_EXCLUSIVITY), Lists.newArrayList("nodeId:0"))
+    ));
+    NodeIDsInfo nodes = labelsToNodesInfo.getLabelsToNodes().get(new NodeLabelInfo(LABEL_A));
     assertNotNull(nodes.getPartitionInfo());
     assertNotNull(nodes.getPartitionInfo().getResourceAvailable());
   }
 
   @SuppressWarnings("rawtypes")
-  private String toJson(Object nsli, Class klass) throws Exception {
+  private String toJson(Object obj, Class klass) throws Exception {
     StringWriter sw = new StringWriter();
     JSONJAXBContext ctx = new JSONJAXBContext(klass);
     JSONMarshaller jm = ctx.createJSONMarshaller();
-    jm.marshallToJSON(nsli, sw);
+    jm.marshallToJSON(obj, sw);
     return sw.toString();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index c67c49a361006..e5072677030d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -49,6 +49,7 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeAttributeType;
@@ -578,10 +579,9 @@ public void testNonexistNodeXML() throws JSONException, Exception {
           response.getType().toString());
       String msg = response.getEntity(String.class);
       System.out.println(msg);
-      DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+      DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
       DocumentBuilder db = dbf.newDocumentBuilder();
-      InputSource is = new InputSource();
-      is.setCharacterStream(new StringReader(msg));
+      InputSource is = new InputSource(new StringReader(msg));
       Document dom = db.parse(is);
       NodeList nodes = dom.getElementsByTagName("RemoteException");
       Element element = (Element) nodes.item(0);
@@ -646,10 +646,9 @@ public void testNodesXML() throws JSONException, Exception {
     assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
         response.getType().toString());
     String xml = response.getEntity(String.class);
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
-    InputSource is = new InputSource();
-    is.setCharacterStream(new StringReader(xml));
+    InputSource is = new InputSource(new StringReader(xml));
     Document dom = db.parse(is);
     NodeList nodesApps = dom.getElementsByTagName("nodes");
     assertEquals("incorrect number of elements", 1, nodesApps.getLength());
@@ -672,7 +671,7 @@ public void testSingleNodesXML() throws JSONException, Exception {
         response.getType().toString());
     String xml = response.getEntity(String.class);
 
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
@@ -694,7 +693,7 @@ public void testNodes2XML() throws JSONException, Exception {
         response.getType().toString());
     String xml = response.getEntity(String.class);
 
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index f864794d59266..ee472dafcb940 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -18,15 +18,20 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
+import com.google.inject.Guice;
 import com.sun.jersey.api.client.ClientResponse;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.core.util.MultivaluedMapImpl;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.Before;
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -83,6 +88,8 @@
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes;
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder;
 import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -91,11 +98,23 @@
 /**
  * Tests for scheduler/app activities.
  */
-public class TestRMWebServicesSchedulerActivities
-    extends TestRMWebServicesCapacitySched {
+public class TestRMWebServicesSchedulerActivities extends JerseyTestBase {
+
+  private MockRM rm;
 
-  private static final Logger LOG = LoggerFactory.getLogger(
-      TestRMWebServicesSchedulerActivities.class);
+  public TestRMWebServicesSchedulerActivities() {
+    super(createWebAppDescriptor());
+  }
+
+  @Before
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    rm = createMockRM(new CapacitySchedulerConfiguration(
+        new Configuration(false)));
+    GuiceServletConfig.setInjector(
+        Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
+  }
 
   @Test
   public void testAssignMultipleContainersPerNodeHeartbeat()
@@ -156,7 +175,7 @@ public void testAssignMultipleContainersPerNodeHeartbeat()
       verifyStateOfAllocations(allocation,
           FN_ACT_FINAL_ALLOCATION_STATE, "ALLOCATED");
       verifyQueueOrder(allocation,
-          "root-root.a-root.b-root.b.b2-root.b.b3-root.b.b1");
+          "root-root.a-root.c-root.b-root.b.b2-root.b.b3-root.b.b1");
     } finally {
       rm.stop();
     }
@@ -380,7 +399,7 @@ public void testReserveNewContainer() throws Exception {
       JSONObject allocations = getFirstSubNodeFromJson(json,
           FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
       verifyQueueOrder(allocations,
-          "root-root.a-root.b-root.b.b3-root.b.b1");
+          "root-root.c-root.a-root.b-root.b.b3-root.b.b1");
       verifyStateOfAllocations(allocations, FN_ACT_FINAL_ALLOCATION_STATE,
           "RESERVED");
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
index 249e8250497e9..1b4ff9ce0fd1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java
@@ -44,7 +44,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
-import org.apache.hadoop.yarn.util.ConverterUtils;
+
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
index bf605e9f5f669..cbc6c417859da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairScheduler.java
@@ -34,6 +34,7 @@
 
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServices;
 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
 import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
 import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -157,4 +158,15 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException {
     assertEquals("root", rootQueue.getString("queueName"));
   }
 
+  @Test
+  public void testClusterSchedulerOverviewFair() throws Exception {
+    WebResource r = resource();
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
+        .get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+    JSONObject json = response.getEntity(JSONObject.class);
+    TestRMWebServices.verifyClusterSchedulerOverView(json, "Fair Scheduler");
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
index 0ad92d20bb5fa..8048a69fbd9cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
@@ -20,6 +20,7 @@
 
 import com.sun.jersey.api.client.WebResource;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,7 +30,6 @@
 
 import javax.ws.rs.core.MediaType;
 import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.transform.*;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
@@ -84,7 +84,7 @@ private Document parseXml(BufferedClientResponse response) {
     try {
       String xml = response.getEntity(String.class);
       DocumentBuilder db =
-          DocumentBuilderFactory.newInstance().newDocumentBuilder();
+          XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(xml));
 
@@ -105,7 +105,7 @@ private void logResponse(Document doc) {
   public static String toXml(Node node) {
     StringWriter writer;
     try {
-      TransformerFactory tf = TransformerFactory.newInstance();
+      TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
       Transformer transformer = tf.newTransformer();
       transformer.setOutputProperty(OutputKeys.INDENT, "yes");
       transformer.setOutputProperty(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/webapp/TestRMWithCSRFFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/webapp/TestRMWithCSRFFilter.java
index 2925e841120bb..577e8acbc465c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/webapp/TestRMWithCSRFFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/webapp/TestRMWithCSRFFilter.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.XMLUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -153,7 +154,7 @@ public void testAllowNonBrowserInteractionWithoutHeader() throws Exception {
   }
 
   public void verifyClusterInfoXML(String xml) throws Exception {
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilderFactory dbf = XMLUtils.newSecureDocumentBuilderFactory();
     DocumentBuilder db = dbf.newDocumentBuilder();
     InputSource is = new InputSource();
     is.setCharacterStream(new StringReader(xml));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json
new file mode 100644
index 0000000000000..fb515d53914be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteMode.json
@@ -0,0 +1,1739 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": true,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 0,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test2",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 0,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 1024,
+          "vCores": 5,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 5
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 1024,
+          "vCores": 5,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 5
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1024,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 1024,
+        "vCores": 5,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 5
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "absolute",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 0,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 0,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 2048,
+          "vCores": 10,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 2048
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 10
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 2048,
+          "vCores": 10,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 2048
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 10
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1024,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 2048,
+        "vCores": 10,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 2048
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 10
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "absolute",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 0,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 0,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 3064,
+          "vCores": 15,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 3064
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 15
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 3064,
+          "vCores": 15,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 3064
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 15
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1024,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 3064,
+        "vCores": 15,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 3064
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 15
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "absolute",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 6136,
+      "vCores": 30,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 6136
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 30
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "absolute",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json
new file mode 100644
index 0000000000000..719aa811e88be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json
@@ -0,0 +1,1592 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "STOPPED",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 8388608,
+          "vCores": 8192,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8388608
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 8192
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 839680,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 839680
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 8388608,
+        "vCores": 8192,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8388608
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 8192
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 839680,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 839680
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 839680,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 839680
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.managed",
+      "capacity": 0.048828125,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0.048828125,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "managed",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "queues": {"queue": [{
+        "type": "capacitySchedulerLeafQueueInfo",
+        "queuePath": "root.managed.queue1",
+        "capacity": 50,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0.024414062,
+        "absoluteMaxCapacity": 100,
+        "absoluteUsedCapacity": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "numApplications": 0,
+        "maxParallelApps": 2147483647,
+        "queueName": "queue1",
+        "isAbsoluteResource": true,
+        "state": "RUNNING",
+        "resourcesUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "hideReservationQueues": false,
+        "nodeLabels": ["*"],
+        "allocatedContainers": 0,
+        "reservedContainers": 0,
+        "pendingContainers": 0,
+        "capacities": {"queueCapacitiesByPartition": [{
+          "partitionName": "",
+          "capacity": 50,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 0.024414062,
+          "absoluteUsedCapacity": 0,
+          "absoluteMaxCapacity": 100,
+          "maxAMLimitPercentage": 10,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "configuredMinResource": {
+            "memory": 2048,
+            "vCores": 2,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 2048
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 2
+              }
+            ]}
+          },
+          "configuredMaxResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "effectiveMinResource": {
+            "memory": 2048,
+            "vCores": 2,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 2048
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 2
+              }
+            ]}
+          },
+          "effectiveMaxResource": {
+            "memory": 8388608,
+            "vCores": 8192,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8388608
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 8192
+              }
+            ]}
+          }
+        }]},
+        "resources": {"resourceUsagesByPartition": [{
+          "partitionName": "",
+          "used": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "reserved": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "pending": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amLimit": {
+            "memory": 839680,
+            "vCores": 1,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 839680
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1
+              }
+            ]}
+          },
+          "userAmLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          }
+        }]},
+        "minEffectiveCapacity": {
+          "memory": 2048,
+          "vCores": 2,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 2048
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 2
+            }
+          ]}
+        },
+        "maxEffectiveCapacity": {
+          "memory": 8388608,
+          "vCores": 8192,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8388608
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 8192
+            }
+          ]}
+        },
+        "maximumAllocation": {
+          "memory": 8192,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8192
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "queueAcls": {"queueAcl": [
+          {
+            "accessType": "ADMINISTER_QUEUE",
+            "accessControlList": "admin "
+          },
+          {
+            "accessType": "APPLICATION_MAX_PRIORITY",
+            "accessControlList": "*"
+          },
+          {
+            "accessType": "SUBMIT_APP",
+            "accessControlList": "user "
+          }
+        ]},
+        "queuePriority": 0,
+        "orderingPolicyInfo": "fifo",
+        "autoCreateChildQueueEnabled": false,
+        "leafQueueTemplate": {},
+        "mode": "absolute",
+        "queueType": "leaf",
+        "creationMethod": "dynamicLegacy",
+        "autoCreationEligibility": "off",
+        "autoQueueTemplateProperties": {},
+        "autoQueueParentTemplateProperties": {},
+        "autoQueueLeafTemplateProperties": {},
+        "numActiveApplications": 0,
+        "numPendingApplications": 0,
+        "numContainers": 0,
+        "maxApplications": 2,
+        "maxApplicationsPerUser": 2,
+        "userLimit": 100,
+        "users": {},
+        "userLimitFactor": 1,
+        "configuredMaxAMResourceLimit": 0.1,
+        "AMResourceLimit": {
+          "memory": 839680,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 839680
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "usedAMResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAMResourceLimit": {
+          "memory": 839680,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 839680
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "preemptionDisabled": true,
+        "intraQueuePreemptionDisabled": true,
+        "defaultPriority": 0,
+        "isAutoCreatedLeafQueue": true,
+        "maxApplicationLifetime": -1,
+        "defaultApplicationLifetime": -1
+      }]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0.048828125,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0.048828125,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 4096,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 4096
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 4096,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 4096
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 8388608,
+          "vCores": 8192,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8388608
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 8192
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 4096,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 4096
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 8388608,
+        "vCores": 8192,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8388608
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 8192
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": true,
+      "leafQueueTemplate": {"property": [
+        {
+          "name": "leaf-queue-template.acl_administer_queue",
+          "value": "admin"
+        },
+        {
+          "name": "leaf-queue-template.capacity",
+          "value": "[memory=2048,vcores=2]"
+        },
+        {
+          "name": "leaf-queue-template.acl_submit_applications",
+          "value": "user"
+        }
+      ]},
+      "mode": "absolute",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "legacy",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 8388608,
+      "vCores": 8192,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 8388608
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 8192
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 8388608,
+      "vCores": 8192,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 8388608
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 8192
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "percentage",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
new file mode 100644
index 0000000000000..d49491febbe50
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-NodeLabelDefaultAPI.xml
@@ -0,0 +1,5039 @@
+<?xml version="1.0" encoding="UTF-8"?><scheduler>
+  <schedulerInfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="capacityScheduler">
+    <capacity>100.0</capacity>
+    <usedCapacity>0.0</usedCapacity>
+    <maxCapacity>100.0</maxCapacity>
+    <weight>-1.0</weight>
+    <normalizedWeight>0.0</normalizedWeight>
+    <queueName>root</queueName>
+    <queuePath>root</queuePath>
+    <maxParallelApps>2147483647</maxParallelApps>
+    <isAbsoluteResource>false</isAbsoluteResource>
+    <queues>
+      <queue xsi:type="capacitySchedulerLeafQueueInfo">
+        <queuePath>root.c</queuePath>
+        <capacity>0.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>0.0</absoluteCapacity>
+        <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>c</queueName>
+        <isAbsoluteResource>true</isAbsoluteResource>
+        <state>RUNNING</state>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>0.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.0</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+            <amUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amUsed>
+            <amLimit>
+              <memory>1024</memory>
+              <vCores>1</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>1</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amLimit>
+            <userAmLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAmLimit>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>1024</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>fifo</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>absolute</mode>
+        <queueType>leaf</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+        <numActiveApplications>0</numActiveApplications>
+        <numPendingApplications>0</numPendingApplications>
+        <numContainers>0</numContainers>
+        <maxApplications>0</maxApplications>
+        <maxApplicationsPerUser>0</maxApplicationsPerUser>
+        <userLimit>100.0</userLimit>
+        <users/>
+        <userLimitFactor>1.0</userLimitFactor>
+        <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+        <AMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </AMResourceLimit>
+        <usedAMResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </usedAMResource>
+        <userAMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </userAMResourceLimit>
+        <preemptionDisabled>true</preemptionDisabled>
+        <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+        <defaultPriority>0</defaultPriority>
+        <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+        <maxApplicationLifetime>-1</maxApplicationLifetime>
+        <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+      </queue>
+      <queue>
+        <queuePath>root.a</queuePath>
+        <capacity>10.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>50.0</maxCapacity>
+        <absoluteCapacity>10.5</absoluteCapacity>
+        <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>42</maxParallelApps>
+        <queueName>a</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.a.a2</queuePath>
+            <capacity>70.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>7.35</absoluteCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>70.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>7.35</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>735</maxApplications>
+            <maxApplicationsPerUser>735</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>100</maxApplicationLifetime>
+            <defaultApplicationLifetime>50</defaultApplicationLifetime>
+          </queue>
+          <queue>
+            <queuePath>root.a.a1</queuePath>
+            <capacity>30.000002</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>3.15</absoluteCapacity>
+            <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <queues>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1b</queuePath>
+                <capacity>15.000001</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.47250003</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1b</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>15.000001</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.47250003</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>47</maxApplications>
+                <maxApplicationsPerUser>47</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1a</queuePath>
+                <capacity>65.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>2.0475001</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1a</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>65.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>2.0475001</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>204</maxApplications>
+                <maxApplicationsPerUser>204</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue>
+                <queuePath>root.a.a1.a1c</queuePath>
+                <capacity>20.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.63</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1c</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <queues/>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>20.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.63</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>utilization</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>true</autoCreateChildQueueEnabled>
+                <leafQueueTemplate>
+                  <property>
+                    <name>leaf-queue-template.capacity</name>
+                    <value>50</value>
+                  </property>
+                </leafQueueTemplate>
+                <mode>percentage</mode>
+                <queueType>parent</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>legacy</autoCreationEligibility>
+                <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+              </queue>
+            </queues>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>30.000002</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>50.0</maxCapacity>
+                <absoluteCapacity>3.15</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>utilization</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>parent</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>10.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>10.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <defaultNodeLabelExpression>root-a-default-label</defaultNodeLabelExpression>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+      <queue>
+        <queuePath>root.b</queuePath>
+        <capacity>89.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>89.5</absoluteCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>b</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b1</queuePath>
+            <capacity>60.000004</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>53.7</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>60.000004</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>53.7</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>5370</maxApplications>
+            <maxApplicationsPerUser>5370</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b3</queuePath>
+            <capacity>0.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.4475</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b3</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>0.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.4475</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>44</maxApplications>
+            <maxApplicationsPerUser>44</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b2</queuePath>
+            <capacity>39.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>35.3525</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>39.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>35.3525</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>3535</maxApplications>
+            <maxApplicationsPerUser>3535</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>89.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>89.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+    </queues>
+    <capacities>
+      <queueCapacitiesByPartition>
+        <partitionName/>
+        <capacity>100.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>100.0</absoluteCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <configuredMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMinResource>
+        <configuredMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMaxResource>
+        <effectiveMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMinResource>
+        <effectiveMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMaxResource>
+      </queueCapacitiesByPartition>
+    </capacities>
+    <health>
+      <lastrun>0</lastrun>
+      <operationsInfo>
+        <operation>last-allocation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-release</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-preemption</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-reservation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <lastRunDetails>
+        <operation>releases</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>allocations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>reservations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+    </health>
+    <maximumAllocation>
+      <memory>8192</memory>
+      <vCores>4</vCores>
+      <resourceInformations>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>memory-mb</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units>Mi</units>
+          <value>8192</value>
+        </resourceInformation>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>vcores</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units/>
+          <value>4</value>
+        </resourceInformation>
+      </resourceInformations>
+    </maximumAllocation>
+    <queueAcls>
+      <queueAcl>
+        <accessType>ADMINISTER_QUEUE</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>APPLICATION_MAX_PRIORITY</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>SUBMIT_APP</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+    </queueAcls>
+    <queuePriority>0</queuePriority>
+    <orderingPolicyInfo>utilization</orderingPolicyInfo>
+    <mode>percentage</mode>
+    <queueType>parent</queueType>
+    <creationMethod>static</creationMethod>
+    <autoCreationEligibility>off</autoCreationEligibility>
+    <defaultNodeLabelExpression>ROOT-INHERITED</defaultNodeLabelExpression>
+    <autoQueueTemplateProperties/>
+    <autoQueueParentTemplateProperties/>
+    <autoQueueLeafTemplateProperties/>
+  </schedulerInfo>
+</scheduler>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json
new file mode 100644
index 0000000000000..22f174415c647
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.json
@@ -0,0 +1,5442 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.c",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 0,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "c",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 0,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 1024,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 1024,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1024,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 1024,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "absolute",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.a",
+      "capacity": 10.5,
+      "usedCapacity": 0,
+      "maxCapacity": 50,
+      "absoluteCapacity": 10.5,
+      "absoluteMaxCapacity": 50,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 42,
+      "queueName": "a",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.a.a2",
+          "capacity": 70,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 7.35,
+          "absoluteMaxCapacity": 50,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "a2",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 70,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 7.35,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 50,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 735,
+          "maxApplicationsPerUser": 735,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": 100,
+          "defaultApplicationLifetime": 50
+        },
+        {
+          "queuePath": "root.a.a1",
+          "capacity": 30.000002,
+          "usedCapacity": 0,
+          "maxCapacity": 50,
+          "absoluteCapacity": 3.15,
+          "absoluteMaxCapacity": 25,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "a1",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "queues": {"queue": [
+            {
+              "type": "capacitySchedulerLeafQueueInfo",
+              "queuePath": "root.a.a1.a1a",
+              "capacity": 65,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 2.0475001,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1a",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 65,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 2.0475001,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 10,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "fifo",
+              "autoCreateChildQueueEnabled": false,
+              "leafQueueTemplate": {},
+              "mode": "percentage",
+              "queueType": "leaf",
+              "creationMethod": "static",
+              "autoCreationEligibility": "off",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {},
+              "numActiveApplications": 0,
+              "numPendingApplications": 0,
+              "numContainers": 0,
+              "maxApplications": 204,
+              "maxApplicationsPerUser": 204,
+              "userLimit": 100,
+              "users": {},
+              "userLimitFactor": 1,
+              "configuredMaxAMResourceLimit": 0.1,
+              "AMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "usedAMResource": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userAMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "preemptionDisabled": true,
+              "intraQueuePreemptionDisabled": true,
+              "defaultPriority": 0,
+              "isAutoCreatedLeafQueue": false,
+              "maxApplicationLifetime": -1,
+              "defaultApplicationLifetime": -1
+            },
+            {
+              "type": "capacitySchedulerLeafQueueInfo",
+              "queuePath": "root.a.a1.a1b",
+              "capacity": 15.000001,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 0.47250003,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1b",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 15.000001,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 0.47250003,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 10,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "fifo",
+              "autoCreateChildQueueEnabled": false,
+              "leafQueueTemplate": {},
+              "mode": "percentage",
+              "queueType": "leaf",
+              "creationMethod": "static",
+              "autoCreationEligibility": "off",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {},
+              "numActiveApplications": 0,
+              "numPendingApplications": 0,
+              "numContainers": 0,
+              "maxApplications": 47,
+              "maxApplicationsPerUser": 47,
+              "userLimit": 100,
+              "users": {},
+              "userLimitFactor": 1,
+              "configuredMaxAMResourceLimit": 0.1,
+              "AMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "usedAMResource": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userAMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "preemptionDisabled": true,
+              "intraQueuePreemptionDisabled": true,
+              "defaultPriority": 0,
+              "isAutoCreatedLeafQueue": false,
+              "maxApplicationLifetime": -1,
+              "defaultApplicationLifetime": -1
+            },
+            {
+              "queuePath": "root.a.a1.a1c",
+              "capacity": 20,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 0.63,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1c",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "queues": {},
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 20,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 0.63,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 0,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "utilization",
+              "autoCreateChildQueueEnabled": true,
+              "leafQueueTemplate": {"property": [{
+                "name": "leaf-queue-template.capacity",
+                "value": "50"
+              }]},
+              "mode": "percentage",
+              "queueType": "parent",
+              "creationMethod": "static",
+              "autoCreationEligibility": "legacy",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {}
+            }
+          ]},
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 30.000002,
+            "usedCapacity": 0,
+            "maxCapacity": 50,
+            "absoluteCapacity": 3.15,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 25,
+            "maxAMLimitPercentage": 0,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "utilization",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "parent",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {}
+        }
+      ]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 10.5,
+        "usedCapacity": 0,
+        "maxCapacity": 50,
+        "absoluteCapacity": 10.5,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 50,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    },
+    {
+      "queuePath": "root.b",
+      "capacity": 89.5,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 89.5,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 2,
+      "maxParallelApps": 2147483647,
+      "queueName": "b",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b1",
+          "capacity": 60.000004,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 53.7,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 2,
+          "maxParallelApps": 2147483647,
+          "queueName": "b1",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 2,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 60.000004,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 53.7,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 2048,
+              "vCores": 2,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 2048
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 2
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 2,
+          "numContainers": 0,
+          "maxApplications": 5370,
+          "maxApplicationsPerUser": 5370,
+          "userLimit": 100,
+          "users": {"user": [
+            {
+              "username": "user1",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "numPendingApplications": 1,
+              "numActiveApplications": 0,
+              "AMResourceUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "userWeight": 1,
+              "isActive": false
+            },
+            {
+              "username": "user2",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "numPendingApplications": 1,
+              "numActiveApplications": 0,
+              "AMResourceUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "userWeight": 1,
+              "isActive": false
+            }
+          ]},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        },
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b2",
+          "capacity": 39.5,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 35.3525,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "b2",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 39.5,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 35.3525,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 3535,
+          "maxApplicationsPerUser": 3535,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        },
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b3",
+          "capacity": 0.5,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 0.4475,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "b3",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 0.5,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 0.4475,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 44,
+          "maxApplicationsPerUser": 44,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        }
+      ]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 2,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 89.5,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 89.5,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 2048,
+          "vCores": 2,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 2048
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 2
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "percentage",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml
new file mode 100644
index 0000000000000..13764582a1979
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PerUserResources.xml
@@ -0,0 +1,5484 @@
+<?xml version="1.0" encoding="UTF-8"?><scheduler>
+  <schedulerInfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="capacityScheduler">
+    <capacity>100.0</capacity>
+    <usedCapacity>0.0</usedCapacity>
+    <maxCapacity>100.0</maxCapacity>
+    <weight>-1.0</weight>
+    <normalizedWeight>0.0</normalizedWeight>
+    <queueName>root</queueName>
+    <queuePath>root</queuePath>
+    <maxParallelApps>2147483647</maxParallelApps>
+    <isAbsoluteResource>false</isAbsoluteResource>
+    <queues>
+      <queue xsi:type="capacitySchedulerLeafQueueInfo">
+        <queuePath>root.c</queuePath>
+        <capacity>0.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>0.0</absoluteCapacity>
+        <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>c</queueName>
+        <isAbsoluteResource>true</isAbsoluteResource>
+        <state>RUNNING</state>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>0.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.0</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+            <amUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amUsed>
+            <amLimit>
+              <memory>1024</memory>
+              <vCores>1</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>1</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amLimit>
+            <userAmLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAmLimit>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>1024</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>fifo</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>absolute</mode>
+        <queueType>leaf</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+        <numActiveApplications>0</numActiveApplications>
+        <numPendingApplications>0</numPendingApplications>
+        <numContainers>0</numContainers>
+        <maxApplications>0</maxApplications>
+        <maxApplicationsPerUser>0</maxApplicationsPerUser>
+        <userLimit>100.0</userLimit>
+        <users/>
+        <userLimitFactor>1.0</userLimitFactor>
+        <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+        <AMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </AMResourceLimit>
+        <usedAMResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </usedAMResource>
+        <userAMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </userAMResourceLimit>
+        <preemptionDisabled>true</preemptionDisabled>
+        <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+        <defaultPriority>0</defaultPriority>
+        <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+        <maxApplicationLifetime>-1</maxApplicationLifetime>
+        <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+      </queue>
+      <queue>
+        <queuePath>root.a</queuePath>
+        <capacity>10.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>50.0</maxCapacity>
+        <absoluteCapacity>10.5</absoluteCapacity>
+        <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>42</maxParallelApps>
+        <queueName>a</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.a.a2</queuePath>
+            <capacity>70.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>7.35</absoluteCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>70.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>7.35</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>735</maxApplications>
+            <maxApplicationsPerUser>735</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>100</maxApplicationLifetime>
+            <defaultApplicationLifetime>50</defaultApplicationLifetime>
+          </queue>
+          <queue>
+            <queuePath>root.a.a1</queuePath>
+            <capacity>30.000002</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>3.15</absoluteCapacity>
+            <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <queues>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1a</queuePath>
+                <capacity>65.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>2.0475001</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1a</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>65.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>2.0475001</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>204</maxApplications>
+                <maxApplicationsPerUser>204</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1b</queuePath>
+                <capacity>15.000001</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.47250003</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1b</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>15.000001</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.47250003</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>47</maxApplications>
+                <maxApplicationsPerUser>47</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue>
+                <queuePath>root.a.a1.a1c</queuePath>
+                <capacity>20.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.63</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1c</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <queues/>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>20.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.63</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>utilization</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>true</autoCreateChildQueueEnabled>
+                <leafQueueTemplate>
+                  <property>
+                    <name>leaf-queue-template.capacity</name>
+                    <value>50</value>
+                  </property>
+                </leafQueueTemplate>
+                <mode>percentage</mode>
+                <queueType>parent</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>legacy</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+              </queue>
+            </queues>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>30.000002</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>50.0</maxCapacity>
+                <absoluteCapacity>3.15</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>utilization</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>parent</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>10.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>10.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+      <queue>
+        <queuePath>root.b</queuePath>
+        <capacity>89.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>89.5</absoluteCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>2</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>b</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b1</queuePath>
+            <capacity>60.000004</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>53.7</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>2</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>2</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>60.000004</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>53.7</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>2048</memory>
+                  <vCores>2</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>2048</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>2</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>2</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>5370</maxApplications>
+            <maxApplicationsPerUser>5370</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users>
+              <user>
+                <username>user1</username>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <numPendingApplications>1</numPendingApplications>
+                <numActiveApplications>0</numActiveApplications>
+                <AMResourceUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceUsed>
+                <userResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userResourceLimit>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <userWeight>1.0</userWeight>
+                <isActive>false</isActive>
+              </user>
+              <user>
+                <username>user2</username>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <numPendingApplications>1</numPendingApplications>
+                <numActiveApplications>0</numActiveApplications>
+                <AMResourceUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceUsed>
+                <userResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userResourceLimit>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <userWeight>1.0</userWeight>
+                <isActive>false</isActive>
+              </user>
+            </users>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b2</queuePath>
+            <capacity>39.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>35.3525</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>39.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>35.3525</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>3535</maxApplications>
+            <maxApplicationsPerUser>3535</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b3</queuePath>
+            <capacity>0.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.4475</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b3</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>0.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.4475</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>44</maxApplications>
+            <maxApplicationsPerUser>44</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>2</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>89.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>89.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>2048</memory>
+              <vCores>2</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>2048</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>2</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+    </queues>
+    <capacities>
+      <queueCapacitiesByPartition>
+        <partitionName/>
+        <capacity>100.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>100.0</absoluteCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <configuredMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMinResource>
+        <configuredMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMaxResource>
+        <effectiveMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMinResource>
+        <effectiveMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMaxResource>
+      </queueCapacitiesByPartition>
+    </capacities>
+    <health>
+      <lastrun>0</lastrun>
+      <operationsInfo>
+        <operation>last-allocation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-release</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-preemption</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-reservation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <lastRunDetails>
+        <operation>releases</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>allocations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>reservations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+    </health>
+    <maximumAllocation>
+      <memory>8192</memory>
+      <vCores>4</vCores>
+      <resourceInformations>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>memory-mb</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units>Mi</units>
+          <value>8192</value>
+        </resourceInformation>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>vcores</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units/>
+          <value>4</value>
+        </resourceInformation>
+      </resourceInformations>
+    </maximumAllocation>
+    <queueAcls>
+      <queueAcl>
+        <accessType>ADMINISTER_QUEUE</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>APPLICATION_MAX_PRIORITY</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>SUBMIT_APP</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+    </queueAcls>
+    <queuePriority>0</queuePriority>
+    <orderingPolicyInfo>utilization</orderingPolicyInfo>
+    <mode>percentage</mode>
+    <queueType>parent</queueType>
+    <creationMethod>static</creationMethod>
+    <autoCreationEligibility>off</autoCreationEligibility>
+    <autoQueueTemplateProperties/>
+    <autoQueueParentTemplateProperties/>
+    <autoQueueLeafTemplateProperties/>
+  </schedulerInfo>
+</scheduler>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json
new file mode 100644
index 0000000000000..cb3441a7aae0f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageMode.json
@@ -0,0 +1,1739 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test2",
+      "capacity": 50,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 50,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 50,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 50,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 5000,
+      "maxApplicationsPerUser": 5000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 50,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 50,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 50,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 50,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 5000,
+      "maxApplicationsPerUser": 5000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "percentage",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json
new file mode 100644
index 0000000000000..8fed362b0291a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-PercentageModeLegacyAutoCreation.json
@@ -0,0 +1,1580 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 50,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 50,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 50,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 50,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 5000,
+      "maxApplicationsPerUser": 5000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.managedtest2",
+      "capacity": 50,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 50,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "managedtest2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 50,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 50,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": true,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "legacy",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "percentage",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json
new file mode 100644
index 0000000000000..1f0d07f8cd956
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightMode.json
@@ -0,0 +1,2224 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": 1,
+  "normalizedWeight": 1,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 22.222223,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.22222222,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 22.222223,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.22222222,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2222,
+      "maxApplicationsPerUser": 2222,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 11.111112,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 5,
+      "normalizedWeight": 0.11111111,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 11.111112,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 5,
+        "normalizedWeight": 0.11111111,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 1111,
+      "maxApplicationsPerUser": 1111,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.parent",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 44.444447,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 20,
+      "normalizedWeight": 0.44444445,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "parent",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 44.444447,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 20,
+        "normalizedWeight": 0.44444445,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 4444,
+      "maxApplicationsPerUser": 4444,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 22.222223,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.22222222,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 22.222223,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.22222222,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2222,
+      "maxApplicationsPerUser": 2222,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": 1,
+    "normalizedWeight": 1,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "weight",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json
new file mode 100644
index 0000000000000..a9c30d1dd1e99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json
@@ -0,0 +1,6006 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": 1,
+  "normalizedWeight": 1,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 20,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.2,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 20,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.2,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 245760,
+          "vCores": 240,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 245760
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 240
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 245760,
+        "vCores": 240,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 245760
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 240
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2000,
+      "maxApplicationsPerUser": 2000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 10,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 5,
+      "normalizedWeight": 0.1,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 10,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 5,
+        "normalizedWeight": 0.1,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 122880,
+          "vCores": 120,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 120
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 122880,
+        "vCores": 120,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 120
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 1000,
+      "maxApplicationsPerUser": 1000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 20,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.2,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 20,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.2,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 245760,
+          "vCores": 240,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 245760
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 240
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 245760,
+        "vCores": 240,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 245760
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 240
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2000,
+      "maxApplicationsPerUser": 2000,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.auto1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 2,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 1,
+      "normalizedWeight": 0.02,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "auto1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 100,
+        "weight": 1,
+        "normalizedWeight": 0.02,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 24575,
+        "vCores": 23,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 24575
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 23
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "dynamicFlexible",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 200,
+      "maxApplicationsPerUser": 200,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": -1,
+      "configuredMaxAMResourceLimit": 1,
+      "AMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.auto2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 2,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 1,
+      "normalizedWeight": 0.02,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "auto2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 100,
+        "weight": 1,
+        "normalizedWeight": 0.02,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 24575,
+        "vCores": 23,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 24575
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 23
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "dynamicFlexible",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 200,
+      "maxApplicationsPerUser": 200,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": -1,
+      "configuredMaxAMResourceLimit": 1,
+      "AMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.auto3",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 2,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 1,
+      "normalizedWeight": 0.02,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "auto3",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 100,
+        "weight": 1,
+        "normalizedWeight": 0.02,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 24575,
+        "vCores": 23,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 24575
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 23
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "dynamicFlexible",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 200,
+      "maxApplicationsPerUser": 200,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": -1,
+      "configuredMaxAMResourceLimit": 1,
+      "AMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1228800,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.parent",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 40,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 20,
+      "normalizedWeight": 0.4,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "parent",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [{
+        "queuePath": "root.parent.autoParent3",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 40,
+        "absoluteMaxCapacity": 100,
+        "absoluteUsedCapacity": 0,
+        "weight": 1,
+        "normalizedWeight": 1,
+        "numApplications": 0,
+        "maxParallelApps": 2147483647,
+        "queueName": "autoParent3",
+        "isAbsoluteResource": false,
+        "state": "RUNNING",
+        "queues": {"queue": [{
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.parent.autoParent3.auto6",
+          "capacity": 0,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 40,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": 1,
+          "normalizedWeight": 1,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "auto6",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 0,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 40,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 100,
+            "weight": 1,
+            "normalizedWeight": 1,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 491520,
+              "vCores": 480,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 491520
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 480
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 1228800,
+              "vCores": 1200,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 1228800
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 1200
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 1228800,
+              "vCores": 1,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 1228800
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 1
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 491520,
+            "vCores": 480,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 491520
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 480
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 1228800,
+            "vCores": 1200,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1200
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": "wildAdmin2 "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": "wildUser2 "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "weight",
+          "queueType": "leaf",
+          "creationMethod": "dynamicFlexible",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 4000,
+          "maxApplicationsPerUser": 4000,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": -1,
+          "configuredMaxAMResourceLimit": 1,
+          "AMResourceLimit": {
+            "memory": 1228800,
+            "vCores": 1,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 1228800,
+            "vCores": 1,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        }]},
+        "resourcesUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "hideReservationQueues": false,
+        "nodeLabels": ["*"],
+        "allocatedContainers": 0,
+        "reservedContainers": 0,
+        "pendingContainers": 0,
+        "capacities": {"queueCapacitiesByPartition": [{
+          "partitionName": "",
+          "capacity": 0,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 40,
+          "absoluteUsedCapacity": 0,
+          "absoluteMaxCapacity": 100,
+          "maxAMLimitPercentage": 0,
+          "weight": 1,
+          "normalizedWeight": 1,
+          "configuredMinResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "configuredMaxResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "effectiveMinResource": {
+            "memory": 491520,
+            "vCores": 480,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 491520
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 480
+              }
+            ]}
+          },
+          "effectiveMaxResource": {
+            "memory": 1228800,
+            "vCores": 1200,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1200
+              }
+            ]}
+          }
+        }]},
+        "resources": {"resourceUsagesByPartition": [{
+          "partitionName": "",
+          "used": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "reserved": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "pending": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          }
+        }]},
+        "minEffectiveCapacity": {
+          "memory": 491520,
+          "vCores": 480,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 491520
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 480
+            }
+          ]}
+        },
+        "maxEffectiveCapacity": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        },
+        "maximumAllocation": {
+          "memory": 8192,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8192
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "queueAcls": {"queueAcl": [
+          {
+            "accessType": "ADMINISTER_QUEUE",
+            "accessControlList": "parentAdmin2 "
+          },
+          {
+            "accessType": "APPLICATION_MAX_PRIORITY",
+            "accessControlList": "*"
+          },
+          {
+            "accessType": "SUBMIT_APP",
+            "accessControlList": "parentUser2 "
+          }
+        ]},
+        "queuePriority": 0,
+        "orderingPolicyInfo": "utilization",
+        "autoCreateChildQueueEnabled": false,
+        "leafQueueTemplate": {},
+        "mode": "weight",
+        "queueType": "parent",
+        "creationMethod": "dynamicFlexible",
+        "autoCreationEligibility": "flexible",
+        "autoQueueTemplateProperties": {},
+        "autoQueueParentTemplateProperties": {},
+        "autoQueueLeafTemplateProperties": {"property": [
+          {
+            "name": "acl_administer_queue",
+            "value": "wildAdmin2"
+          },
+          {
+            "name": "acl_submit_applications",
+            "value": "wildUser2"
+          }
+        ]}
+      }]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 40,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": 20,
+        "normalizedWeight": 0.4,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 491520,
+          "vCores": 480,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 491520
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 480
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 491520,
+        "vCores": 480,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 491520
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 480
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "flexible",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "parentAdmin2"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "parentUser2"
+        }
+      ]},
+      "autoQueueLeafTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "wildAdmin1"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "wildUser1"
+        }
+      ]}
+    },
+    {
+      "queuePath": "root.autoParent1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 2,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 1,
+      "normalizedWeight": 0.02,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "autoParent1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [{
+        "type": "capacitySchedulerLeafQueueInfo",
+        "queuePath": "root.autoParent1.auto4",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteMaxCapacity": 100,
+        "absoluteUsedCapacity": 0,
+        "weight": 1,
+        "normalizedWeight": 1,
+        "numApplications": 0,
+        "maxParallelApps": 2147483647,
+        "queueName": "auto4",
+        "isAbsoluteResource": false,
+        "state": "RUNNING",
+        "resourcesUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "hideReservationQueues": false,
+        "nodeLabels": ["*"],
+        "allocatedContainers": 0,
+        "reservedContainers": 0,
+        "pendingContainers": 0,
+        "capacities": {"queueCapacitiesByPartition": [{
+          "partitionName": "",
+          "capacity": 0,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 2,
+          "absoluteUsedCapacity": 0,
+          "absoluteMaxCapacity": 100,
+          "maxAMLimitPercentage": 100,
+          "weight": 1,
+          "normalizedWeight": 1,
+          "configuredMinResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "configuredMaxResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "effectiveMinResource": {
+            "memory": 24575,
+            "vCores": 23,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 24575
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 23
+              }
+            ]}
+          },
+          "effectiveMaxResource": {
+            "memory": 1228800,
+            "vCores": 1200,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1200
+              }
+            ]}
+          }
+        }]},
+        "resources": {"resourceUsagesByPartition": [{
+          "partitionName": "",
+          "used": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "reserved": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "pending": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amLimit": {
+            "memory": 1228800,
+            "vCores": 1,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1
+              }
+            ]}
+          },
+          "userAmLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          }
+        }]},
+        "minEffectiveCapacity": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "maxEffectiveCapacity": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        },
+        "maximumAllocation": {
+          "memory": 8192,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8192
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "queueAcls": {"queueAcl": [
+          {
+            "accessType": "ADMINISTER_QUEUE",
+            "accessControlList": "admin1 "
+          },
+          {
+            "accessType": "APPLICATION_MAX_PRIORITY",
+            "accessControlList": "*"
+          },
+          {
+            "accessType": "SUBMIT_APP",
+            "accessControlList": "user1 "
+          }
+        ]},
+        "queuePriority": 0,
+        "orderingPolicyInfo": "fifo",
+        "autoCreateChildQueueEnabled": false,
+        "leafQueueTemplate": {},
+        "mode": "weight",
+        "queueType": "leaf",
+        "creationMethod": "dynamicFlexible",
+        "autoCreationEligibility": "off",
+        "autoQueueTemplateProperties": {},
+        "autoQueueParentTemplateProperties": {},
+        "autoQueueLeafTemplateProperties": {},
+        "numActiveApplications": 0,
+        "numPendingApplications": 0,
+        "numContainers": 0,
+        "maxApplications": 300,
+        "maxApplicationsPerUser": 300,
+        "userLimit": 100,
+        "users": {},
+        "userLimitFactor": -1,
+        "configuredMaxAMResourceLimit": 1,
+        "AMResourceLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "usedAMResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAMResourceLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "preemptionDisabled": true,
+        "intraQueuePreemptionDisabled": true,
+        "defaultPriority": 0,
+        "isAutoCreatedLeafQueue": false,
+        "maxApplicationLifetime": -1,
+        "defaultApplicationLifetime": -1
+      }]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": 1,
+        "normalizedWeight": 0.02,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 24575,
+        "vCores": 23,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 24575
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 23
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": "parentAdmin1 "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": "parentUser1 "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "parent",
+      "creationMethod": "dynamicFlexible",
+      "autoCreationEligibility": "flexible",
+      "autoQueueTemplateProperties": {"property": [{
+        "name": "maximum-applications",
+        "value": "300"
+      }]},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "admin1"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "user1"
+        }
+      ]}
+    },
+    {
+      "queuePath": "root.autoParent2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 2,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 1,
+      "normalizedWeight": 0.02,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "autoParent2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [{
+        "type": "capacitySchedulerLeafQueueInfo",
+        "queuePath": "root.autoParent2.auto5",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteMaxCapacity": 100,
+        "absoluteUsedCapacity": 0,
+        "weight": 1,
+        "normalizedWeight": 1,
+        "numApplications": 0,
+        "maxParallelApps": 2147483647,
+        "queueName": "auto5",
+        "isAbsoluteResource": false,
+        "state": "RUNNING",
+        "resourcesUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "hideReservationQueues": false,
+        "nodeLabels": ["*"],
+        "allocatedContainers": 0,
+        "reservedContainers": 0,
+        "pendingContainers": 0,
+        "capacities": {"queueCapacitiesByPartition": [{
+          "partitionName": "",
+          "capacity": 0,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 2,
+          "absoluteUsedCapacity": 0,
+          "absoluteMaxCapacity": 100,
+          "maxAMLimitPercentage": 100,
+          "weight": 1,
+          "normalizedWeight": 1,
+          "configuredMinResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "configuredMaxResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 8192,
+                "minimumAllocation": 1024,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 4,
+                "minimumAllocation": 1,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "effectiveMinResource": {
+            "memory": 24575,
+            "vCores": 23,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 24575
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 23
+              }
+            ]}
+          },
+          "effectiveMaxResource": {
+            "memory": 1228800,
+            "vCores": 1200,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1200
+              }
+            ]}
+          }
+        }]},
+        "resources": {"resourceUsagesByPartition": [{
+          "partitionName": "",
+          "used": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "reserved": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "pending": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "amLimit": {
+            "memory": 1228800,
+            "vCores": 1,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 1228800
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 1
+              }
+            ]}
+          },
+          "userAmLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          }
+        }]},
+        "minEffectiveCapacity": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "maxEffectiveCapacity": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        },
+        "maximumAllocation": {
+          "memory": 8192,
+          "vCores": 4,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 8192
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 4
+            }
+          ]}
+        },
+        "queueAcls": {"queueAcl": [
+          {
+            "accessType": "ADMINISTER_QUEUE",
+            "accessControlList": "wildAdmin1 "
+          },
+          {
+            "accessType": "APPLICATION_MAX_PRIORITY",
+            "accessControlList": "*"
+          },
+          {
+            "accessType": "SUBMIT_APP",
+            "accessControlList": "wildUser1 "
+          }
+        ]},
+        "queuePriority": 0,
+        "orderingPolicyInfo": "fifo",
+        "autoCreateChildQueueEnabled": false,
+        "leafQueueTemplate": {},
+        "mode": "weight",
+        "queueType": "leaf",
+        "creationMethod": "dynamicFlexible",
+        "autoCreationEligibility": "off",
+        "autoQueueTemplateProperties": {},
+        "autoQueueParentTemplateProperties": {},
+        "autoQueueLeafTemplateProperties": {},
+        "numActiveApplications": 0,
+        "numPendingApplications": 0,
+        "numContainers": 0,
+        "maxApplications": 200,
+        "maxApplicationsPerUser": 200,
+        "userLimit": 100,
+        "users": {},
+        "userLimitFactor": -1,
+        "configuredMaxAMResourceLimit": 1,
+        "AMResourceLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "usedAMResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "userAMResourceLimit": {
+          "memory": 1228800,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "preemptionDisabled": true,
+        "intraQueuePreemptionDisabled": true,
+        "defaultPriority": 0,
+        "isAutoCreatedLeafQueue": false,
+        "maxApplicationLifetime": -1,
+        "defaultApplicationLifetime": -1
+      }]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 2,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": 1,
+        "normalizedWeight": 0.02,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 24575,
+          "vCores": 23,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 24575
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 23
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 24575,
+        "vCores": 23,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 24575
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 23
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": "parentAdmin1 "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": "parentUser1 "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "parent",
+      "creationMethod": "dynamicFlexible",
+      "autoCreationEligibility": "flexible",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "wildAdmin1"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "wildUser1"
+        }
+      ]}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": 1,
+    "normalizedWeight": 1,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 1228800,
+      "vCores": 1200,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 1228800
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 1200
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 1228800,
+      "vCores": 1200,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 1228800
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 1200
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "weight",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "flexible",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {"property": [
+    {
+      "name": "acl_administer_queue",
+      "value": "parentAdmin1"
+    },
+    {
+      "name": "acl_submit_applications",
+      "value": "parentUser1"
+    }
+  ]},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json
new file mode 100644
index 0000000000000..84fd17399e8d9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json
@@ -0,0 +1,2092 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": 1,
+  "normalizedWeight": 1,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test2",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 22.222223,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.22222222,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test2",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 22.222223,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.22222222,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 273066,
+          "vCores": 266,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 273066
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 266
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 273066,
+        "vCores": 266,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 273066
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 266
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2222,
+      "maxApplicationsPerUser": 2222,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.test1",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 11.111112,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 5,
+      "normalizedWeight": 0.11111111,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "test1",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 11.111112,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 5,
+        "normalizedWeight": 0.11111111,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 136533,
+          "vCores": 133,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 136533
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 133
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 136533,
+        "vCores": 133,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 136533
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 133
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 1111,
+      "maxApplicationsPerUser": 1111,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.default",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 22.222223,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 10,
+      "normalizedWeight": 0.22222222,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "default",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 22.222223,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 10,
+        "weight": 10,
+        "normalizedWeight": 0.22222222,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 273066,
+          "vCores": 266,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 273066
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 266
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 122880,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 122880
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 273066,
+        "vCores": 266,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 273066
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 266
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 2222,
+      "maxApplicationsPerUser": 2222,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 122880,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 122880
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.parent",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 44.444447,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": 20,
+      "normalizedWeight": 0.44444445,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "parent",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 44.444447,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": 20,
+        "normalizedWeight": 0.44444445,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 546133,
+          "vCores": 533,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 546133
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 533
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 1228800,
+          "vCores": 1200,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1228800
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1200
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 546133,
+        "vCores": 533,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 546133
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 533
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 1228800,
+        "vCores": 1200,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1228800
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1200
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "weight",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "flexible",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "parentAdmin2"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "parentUser2"
+        }
+      ]},
+      "autoQueueLeafTemplateProperties": {"property": [
+        {
+          "name": "acl_administer_queue",
+          "value": "wildAdmin1"
+        },
+        {
+          "name": "acl_submit_applications",
+          "value": "wildUser1"
+        }
+      ]}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": 1,
+    "normalizedWeight": 1,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 1228800,
+      "vCores": 1200,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 1228800
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 1200
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 1228800,
+      "vCores": 1200,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 1228800
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 1200
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "weight",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "flexible",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {"property": [
+    {
+      "name": "acl_administer_queue",
+      "value": "parentAdmin1"
+    },
+    {
+      "name": "acl_submit_applications",
+      "value": "parentUser1"
+    }
+  ]},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json
new file mode 100644
index 0000000000000..47acd70cd59b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.json
@@ -0,0 +1,4989 @@
+{"scheduler": {"schedulerInfo": {
+  "type": "capacityScheduler",
+  "capacity": 100,
+  "usedCapacity": 0,
+  "maxCapacity": 100,
+  "weight": -1,
+  "normalizedWeight": 0,
+  "queueName": "root",
+  "queuePath": "root",
+  "maxParallelApps": 2147483647,
+  "isAbsoluteResource": false,
+  "queues": {"queue": [
+    {
+      "type": "capacitySchedulerLeafQueueInfo",
+      "queuePath": "root.c",
+      "capacity": 0,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 0,
+      "absoluteMaxCapacity": 0,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "c",
+      "isAbsoluteResource": true,
+      "state": "RUNNING",
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 0,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 0,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 0,
+        "maxAMLimitPercentage": 10,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 1024,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 1024,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amUsed": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "amLimit": {
+          "memory": 1024,
+          "vCores": 1,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 1024
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 1
+            }
+          ]}
+        },
+        "userAmLimit": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 1024,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "fifo",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "absolute",
+      "queueType": "leaf",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {},
+      "numActiveApplications": 0,
+      "numPendingApplications": 0,
+      "numContainers": 0,
+      "maxApplications": 0,
+      "maxApplicationsPerUser": 0,
+      "userLimit": 100,
+      "users": {},
+      "userLimitFactor": 1,
+      "configuredMaxAMResourceLimit": 0.1,
+      "AMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "usedAMResource": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "userAMResourceLimit": {
+        "memory": 1024,
+        "vCores": 1,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 1024
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 1
+          }
+        ]}
+      },
+      "preemptionDisabled": true,
+      "intraQueuePreemptionDisabled": true,
+      "defaultPriority": 0,
+      "isAutoCreatedLeafQueue": false,
+      "maxApplicationLifetime": -1,
+      "defaultApplicationLifetime": -1
+    },
+    {
+      "queuePath": "root.a",
+      "capacity": 10.5,
+      "usedCapacity": 0,
+      "maxCapacity": 50,
+      "absoluteCapacity": 10.5,
+      "absoluteMaxCapacity": 50,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 42,
+      "queueName": "a",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.a.a2",
+          "capacity": 70,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 7.35,
+          "absoluteMaxCapacity": 50,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "a2",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 70,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 7.35,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 50,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 735,
+          "maxApplicationsPerUser": 735,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": 100,
+          "defaultApplicationLifetime": 50
+        },
+        {
+          "queuePath": "root.a.a1",
+          "capacity": 30.000002,
+          "usedCapacity": 0,
+          "maxCapacity": 50,
+          "absoluteCapacity": 3.15,
+          "absoluteMaxCapacity": 25,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "a1",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "queues": {"queue": [
+            {
+              "type": "capacitySchedulerLeafQueueInfo",
+              "queuePath": "root.a.a1.a1a",
+              "capacity": 65,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 2.0475001,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1a",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 65,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 2.0475001,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 10,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "fifo",
+              "autoCreateChildQueueEnabled": false,
+              "leafQueueTemplate": {},
+              "mode": "percentage",
+              "queueType": "leaf",
+              "creationMethod": "static",
+              "autoCreationEligibility": "off",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {},
+              "numActiveApplications": 0,
+              "numPendingApplications": 0,
+              "numContainers": 0,
+              "maxApplications": 204,
+              "maxApplicationsPerUser": 204,
+              "userLimit": 100,
+              "users": {},
+              "userLimitFactor": 1,
+              "configuredMaxAMResourceLimit": 0.1,
+              "AMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "usedAMResource": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userAMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "preemptionDisabled": true,
+              "intraQueuePreemptionDisabled": true,
+              "defaultPriority": 0,
+              "isAutoCreatedLeafQueue": false,
+              "maxApplicationLifetime": -1,
+              "defaultApplicationLifetime": -1
+            },
+            {
+              "type": "capacitySchedulerLeafQueueInfo",
+              "queuePath": "root.a.a1.a1b",
+              "capacity": 15.000001,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 0.47250003,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1b",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 15.000001,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 0.47250003,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 10,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amUsed": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "amLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "userAmLimit": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "fifo",
+              "autoCreateChildQueueEnabled": false,
+              "leafQueueTemplate": {},
+              "mode": "percentage",
+              "queueType": "leaf",
+              "creationMethod": "static",
+              "autoCreationEligibility": "off",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {},
+              "numActiveApplications": 0,
+              "numPendingApplications": 0,
+              "numContainers": 0,
+              "maxApplications": 47,
+              "maxApplicationsPerUser": 47,
+              "userLimit": 100,
+              "users": {},
+              "userLimitFactor": 1,
+              "configuredMaxAMResourceLimit": 0.1,
+              "AMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "usedAMResource": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "userAMResourceLimit": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "preemptionDisabled": true,
+              "intraQueuePreemptionDisabled": true,
+              "defaultPriority": 0,
+              "isAutoCreatedLeafQueue": false,
+              "maxApplicationLifetime": -1,
+              "defaultApplicationLifetime": -1
+            },
+            {
+              "queuePath": "root.a.a1.a1c",
+              "capacity": 20,
+              "usedCapacity": 0,
+              "maxCapacity": 100,
+              "absoluteCapacity": 0.63,
+              "absoluteMaxCapacity": 25,
+              "absoluteUsedCapacity": 0,
+              "weight": -1,
+              "normalizedWeight": 0,
+              "numApplications": 0,
+              "maxParallelApps": 2147483647,
+              "queueName": "a1c",
+              "isAbsoluteResource": false,
+              "state": "RUNNING",
+              "queues": {},
+              "resourcesUsed": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "hideReservationQueues": false,
+              "nodeLabels": ["*"],
+              "allocatedContainers": 0,
+              "reservedContainers": 0,
+              "pendingContainers": 0,
+              "capacities": {"queueCapacitiesByPartition": [{
+                "partitionName": "",
+                "capacity": 20,
+                "usedCapacity": 0,
+                "maxCapacity": 100,
+                "absoluteCapacity": 0.63,
+                "absoluteUsedCapacity": 0,
+                "absoluteMaxCapacity": 25,
+                "maxAMLimitPercentage": 0,
+                "weight": -1,
+                "normalizedWeight": 0,
+                "configuredMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "configuredMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 8192,
+                      "minimumAllocation": 1024,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 4,
+                      "minimumAllocation": 1,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMinResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "effectiveMaxResource": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "resources": {"resourceUsagesByPartition": [{
+                "partitionName": "",
+                "used": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "reserved": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                },
+                "pending": {
+                  "memory": 0,
+                  "vCores": 0,
+                  "resourceInformations": {"resourceInformation": [
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "memory-mb",
+                      "resourceType": "COUNTABLE",
+                      "units": "Mi",
+                      "value": 0
+                    },
+                    {
+                      "attributes": {},
+                      "maximumAllocation": 9223372036854775807,
+                      "minimumAllocation": 0,
+                      "name": "vcores",
+                      "resourceType": "COUNTABLE",
+                      "units": "",
+                      "value": 0
+                    }
+                  ]}
+                }
+              }]},
+              "minEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maxEffectiveCapacity": {
+                "memory": 0,
+                "vCores": 0,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 0
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 0
+                  }
+                ]}
+              },
+              "maximumAllocation": {
+                "memory": 8192,
+                "vCores": 4,
+                "resourceInformations": {"resourceInformation": [
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "memory-mb",
+                    "resourceType": "COUNTABLE",
+                    "units": "Mi",
+                    "value": 8192
+                  },
+                  {
+                    "attributes": {},
+                    "maximumAllocation": 9223372036854775807,
+                    "minimumAllocation": 0,
+                    "name": "vcores",
+                    "resourceType": "COUNTABLE",
+                    "units": "",
+                    "value": 4
+                  }
+                ]}
+              },
+              "queueAcls": {"queueAcl": [
+                {
+                  "accessType": "ADMINISTER_QUEUE",
+                  "accessControlList": " "
+                },
+                {
+                  "accessType": "APPLICATION_MAX_PRIORITY",
+                  "accessControlList": "*"
+                },
+                {
+                  "accessType": "SUBMIT_APP",
+                  "accessControlList": " "
+                }
+              ]},
+              "queuePriority": 0,
+              "orderingPolicyInfo": "utilization",
+              "autoCreateChildQueueEnabled": true,
+              "leafQueueTemplate": {"property": [{
+                "name": "leaf-queue-template.capacity",
+                "value": "50"
+              }]},
+              "mode": "percentage",
+              "queueType": "parent",
+              "creationMethod": "static",
+              "autoCreationEligibility": "legacy",
+              "autoQueueTemplateProperties": {},
+              "autoQueueParentTemplateProperties": {},
+              "autoQueueLeafTemplateProperties": {}
+            }
+          ]},
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 30.000002,
+            "usedCapacity": 0,
+            "maxCapacity": 50,
+            "absoluteCapacity": 3.15,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 25,
+            "maxAMLimitPercentage": 0,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "utilization",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "parent",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {}
+        }
+      ]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 10.5,
+        "usedCapacity": 0,
+        "maxCapacity": 50,
+        "absoluteCapacity": 10.5,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 50,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    },
+    {
+      "queuePath": "root.b",
+      "capacity": 89.5,
+      "usedCapacity": 0,
+      "maxCapacity": 100,
+      "absoluteCapacity": 89.5,
+      "absoluteMaxCapacity": 100,
+      "absoluteUsedCapacity": 0,
+      "weight": -1,
+      "normalizedWeight": 0,
+      "numApplications": 0,
+      "maxParallelApps": 2147483647,
+      "queueName": "b",
+      "isAbsoluteResource": false,
+      "state": "RUNNING",
+      "queues": {"queue": [
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b1",
+          "capacity": 60.000004,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 53.7,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "b1",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 60.000004,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 53.7,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 5370,
+          "maxApplicationsPerUser": 5370,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        },
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b2",
+          "capacity": 39.5,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 35.3525,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "b2",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 39.5,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 35.3525,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 3535,
+          "maxApplicationsPerUser": 3535,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        },
+        {
+          "type": "capacitySchedulerLeafQueueInfo",
+          "queuePath": "root.b.b3",
+          "capacity": 0.5,
+          "usedCapacity": 0,
+          "maxCapacity": 100,
+          "absoluteCapacity": 0.4475,
+          "absoluteMaxCapacity": 100,
+          "absoluteUsedCapacity": 0,
+          "weight": -1,
+          "normalizedWeight": 0,
+          "numApplications": 0,
+          "maxParallelApps": 2147483647,
+          "queueName": "b3",
+          "isAbsoluteResource": false,
+          "state": "RUNNING",
+          "resourcesUsed": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "hideReservationQueues": false,
+          "nodeLabels": ["*"],
+          "allocatedContainers": 0,
+          "reservedContainers": 0,
+          "pendingContainers": 0,
+          "capacities": {"queueCapacitiesByPartition": [{
+            "partitionName": "",
+            "capacity": 0.5,
+            "usedCapacity": 0,
+            "maxCapacity": 100,
+            "absoluteCapacity": 0.4475,
+            "absoluteUsedCapacity": 0,
+            "absoluteMaxCapacity": 100,
+            "maxAMLimitPercentage": 10,
+            "weight": -1,
+            "normalizedWeight": 0,
+            "configuredMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "configuredMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 8192,
+                  "minimumAllocation": 1024,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 4,
+                  "minimumAllocation": 1,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMinResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "effectiveMaxResource": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "resources": {"resourceUsagesByPartition": [{
+            "partitionName": "",
+            "used": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "reserved": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "pending": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amUsed": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "amLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            },
+            "userAmLimit": {
+              "memory": 0,
+              "vCores": 0,
+              "resourceInformations": {"resourceInformation": [
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "memory-mb",
+                  "resourceType": "COUNTABLE",
+                  "units": "Mi",
+                  "value": 0
+                },
+                {
+                  "attributes": {},
+                  "maximumAllocation": 9223372036854775807,
+                  "minimumAllocation": 0,
+                  "name": "vcores",
+                  "resourceType": "COUNTABLE",
+                  "units": "",
+                  "value": 0
+                }
+              ]}
+            }
+          }]},
+          "minEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maxEffectiveCapacity": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "maximumAllocation": {
+            "memory": 8192,
+            "vCores": 4,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 8192
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 4
+              }
+            ]}
+          },
+          "queueAcls": {"queueAcl": [
+            {
+              "accessType": "ADMINISTER_QUEUE",
+              "accessControlList": " "
+            },
+            {
+              "accessType": "APPLICATION_MAX_PRIORITY",
+              "accessControlList": "*"
+            },
+            {
+              "accessType": "SUBMIT_APP",
+              "accessControlList": " "
+            }
+          ]},
+          "queuePriority": 0,
+          "orderingPolicyInfo": "fifo",
+          "autoCreateChildQueueEnabled": false,
+          "leafQueueTemplate": {},
+          "mode": "percentage",
+          "queueType": "leaf",
+          "creationMethod": "static",
+          "autoCreationEligibility": "off",
+          "autoQueueTemplateProperties": {},
+          "autoQueueParentTemplateProperties": {},
+          "autoQueueLeafTemplateProperties": {},
+          "numActiveApplications": 0,
+          "numPendingApplications": 0,
+          "numContainers": 0,
+          "maxApplications": 44,
+          "maxApplicationsPerUser": 44,
+          "userLimit": 100,
+          "users": {},
+          "userLimitFactor": 100,
+          "configuredMaxAMResourceLimit": 0.1,
+          "AMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "usedAMResource": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "userAMResourceLimit": {
+            "memory": 0,
+            "vCores": 0,
+            "resourceInformations": {"resourceInformation": [
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "memory-mb",
+                "resourceType": "COUNTABLE",
+                "units": "Mi",
+                "value": 0
+              },
+              {
+                "attributes": {},
+                "maximumAllocation": 9223372036854775807,
+                "minimumAllocation": 0,
+                "name": "vcores",
+                "resourceType": "COUNTABLE",
+                "units": "",
+                "value": 0
+              }
+            ]}
+          },
+          "preemptionDisabled": true,
+          "intraQueuePreemptionDisabled": true,
+          "defaultPriority": 0,
+          "isAutoCreatedLeafQueue": false,
+          "maxApplicationLifetime": -1,
+          "defaultApplicationLifetime": -1
+        }
+      ]},
+      "resourcesUsed": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "hideReservationQueues": false,
+      "nodeLabels": ["*"],
+      "allocatedContainers": 0,
+      "reservedContainers": 0,
+      "pendingContainers": 0,
+      "capacities": {"queueCapacitiesByPartition": [{
+        "partitionName": "",
+        "capacity": 89.5,
+        "usedCapacity": 0,
+        "maxCapacity": 100,
+        "absoluteCapacity": 89.5,
+        "absoluteUsedCapacity": 0,
+        "absoluteMaxCapacity": 100,
+        "maxAMLimitPercentage": 0,
+        "weight": -1,
+        "normalizedWeight": 0,
+        "configuredMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "configuredMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 8192,
+              "minimumAllocation": 1024,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 4,
+              "minimumAllocation": 1,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMinResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "effectiveMaxResource": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "resources": {"resourceUsagesByPartition": [{
+        "partitionName": "",
+        "used": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "reserved": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        },
+        "pending": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }]},
+      "minEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maxEffectiveCapacity": {
+        "memory": 0,
+        "vCores": 0,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 0
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 0
+          }
+        ]}
+      },
+      "maximumAllocation": {
+        "memory": 8192,
+        "vCores": 4,
+        "resourceInformations": {"resourceInformation": [
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "memory-mb",
+            "resourceType": "COUNTABLE",
+            "units": "Mi",
+            "value": 8192
+          },
+          {
+            "attributes": {},
+            "maximumAllocation": 9223372036854775807,
+            "minimumAllocation": 0,
+            "name": "vcores",
+            "resourceType": "COUNTABLE",
+            "units": "",
+            "value": 4
+          }
+        ]}
+      },
+      "queueAcls": {"queueAcl": [
+        {
+          "accessType": "ADMINISTER_QUEUE",
+          "accessControlList": " "
+        },
+        {
+          "accessType": "APPLICATION_MAX_PRIORITY",
+          "accessControlList": "*"
+        },
+        {
+          "accessType": "SUBMIT_APP",
+          "accessControlList": " "
+        }
+      ]},
+      "queuePriority": 0,
+      "orderingPolicyInfo": "utilization",
+      "autoCreateChildQueueEnabled": false,
+      "leafQueueTemplate": {},
+      "mode": "percentage",
+      "queueType": "parent",
+      "creationMethod": "static",
+      "autoCreationEligibility": "off",
+      "autoQueueTemplateProperties": {},
+      "autoQueueParentTemplateProperties": {},
+      "autoQueueLeafTemplateProperties": {}
+    }
+  ]},
+  "capacities": {"queueCapacitiesByPartition": [{
+    "partitionName": "",
+    "capacity": 100,
+    "usedCapacity": 0,
+    "maxCapacity": 100,
+    "absoluteCapacity": 100,
+    "absoluteUsedCapacity": 0,
+    "absoluteMaxCapacity": 100,
+    "maxAMLimitPercentage": 0,
+    "weight": -1,
+    "normalizedWeight": 0,
+    "configuredMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "configuredMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 8192,
+          "minimumAllocation": 1024,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 4,
+          "minimumAllocation": 1,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMinResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    },
+    "effectiveMaxResource": {
+      "memory": 0,
+      "vCores": 0,
+      "resourceInformations": {"resourceInformation": [
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "memory-mb",
+          "resourceType": "COUNTABLE",
+          "units": "Mi",
+          "value": 0
+        },
+        {
+          "attributes": {},
+          "maximumAllocation": 9223372036854775807,
+          "minimumAllocation": 0,
+          "name": "vcores",
+          "resourceType": "COUNTABLE",
+          "units": "",
+          "value": 0
+        }
+      ]}
+    }
+  }]},
+  "health": {
+    "lastrun": 0,
+    "operationsInfo": [
+      {
+        "operation": "last-allocation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-release",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-preemption",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      },
+      {
+        "operation": "last-reservation",
+        "nodeId": "N\/A",
+        "containerId": "N\/A",
+        "queue": "N\/A"
+      }
+    ],
+    "lastRunDetails": [
+      {
+        "operation": "releases",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "allocations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      },
+      {
+        "operation": "reservations",
+        "count": 0,
+        "resources": {
+          "memory": 0,
+          "vCores": 0,
+          "resourceInformations": {"resourceInformation": [
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "memory-mb",
+              "resourceType": "COUNTABLE",
+              "units": "Mi",
+              "value": 0
+            },
+            {
+              "attributes": {},
+              "maximumAllocation": 9223372036854775807,
+              "minimumAllocation": 0,
+              "name": "vcores",
+              "resourceType": "COUNTABLE",
+              "units": "",
+              "value": 0
+            }
+          ]}
+        }
+      }
+    ]
+  },
+  "maximumAllocation": {
+    "memory": 8192,
+    "vCores": 4,
+    "resourceInformations": {"resourceInformation": [
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "memory-mb",
+        "resourceType": "COUNTABLE",
+        "units": "Mi",
+        "value": 8192
+      },
+      {
+        "attributes": {},
+        "maximumAllocation": 9223372036854775807,
+        "minimumAllocation": 0,
+        "name": "vcores",
+        "resourceType": "COUNTABLE",
+        "units": "",
+        "value": 4
+      }
+    ]}
+  },
+  "queueAcls": {"queueAcl": [
+    {
+      "accessType": "ADMINISTER_QUEUE",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "APPLICATION_MAX_PRIORITY",
+      "accessControlList": "*"
+    },
+    {
+      "accessType": "SUBMIT_APP",
+      "accessControlList": "*"
+    }
+  ]},
+  "queuePriority": 0,
+  "orderingPolicyInfo": "utilization",
+  "mode": "percentage",
+  "queueType": "parent",
+  "creationMethod": "static",
+  "autoCreationEligibility": "off",
+  "autoQueueTemplateProperties": {},
+  "autoQueueParentTemplateProperties": {},
+  "autoQueueLeafTemplateProperties": {}
+}}}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml
new file mode 100644
index 0000000000000..c800195514fcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/webapp/scheduler-response.xml
@@ -0,0 +1,5027 @@
+<?xml version="1.0" encoding="UTF-8"?><scheduler>
+  <schedulerInfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="capacityScheduler">
+    <capacity>100.0</capacity>
+    <usedCapacity>0.0</usedCapacity>
+    <maxCapacity>100.0</maxCapacity>
+    <weight>-1.0</weight>
+    <normalizedWeight>0.0</normalizedWeight>
+    <queueName>root</queueName>
+    <queuePath>root</queuePath>
+    <maxParallelApps>2147483647</maxParallelApps>
+    <isAbsoluteResource>false</isAbsoluteResource>
+    <queues>
+      <queue xsi:type="capacitySchedulerLeafQueueInfo">
+        <queuePath>root.c</queuePath>
+        <capacity>0.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>0.0</absoluteCapacity>
+        <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>c</queueName>
+        <isAbsoluteResource>true</isAbsoluteResource>
+        <state>RUNNING</state>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>0.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.0</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>0.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>1024</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+            <amUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amUsed>
+            <amLimit>
+              <memory>1024</memory>
+              <vCores>1</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>1024</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>1</value>
+                </resourceInformation>
+              </resourceInformations>
+            </amLimit>
+            <userAmLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAmLimit>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>1024</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>fifo</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>absolute</mode>
+        <queueType>leaf</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+        <numActiveApplications>0</numActiveApplications>
+        <numPendingApplications>0</numPendingApplications>
+        <numContainers>0</numContainers>
+        <maxApplications>0</maxApplications>
+        <maxApplicationsPerUser>0</maxApplicationsPerUser>
+        <userLimit>100.0</userLimit>
+        <users/>
+        <userLimitFactor>1.0</userLimitFactor>
+        <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+        <AMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </AMResourceLimit>
+        <usedAMResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </usedAMResource>
+        <userAMResourceLimit>
+          <memory>1024</memory>
+          <vCores>1</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>1024</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>1</value>
+            </resourceInformation>
+          </resourceInformations>
+        </userAMResourceLimit>
+        <preemptionDisabled>true</preemptionDisabled>
+        <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+        <defaultPriority>0</defaultPriority>
+        <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+        <maxApplicationLifetime>-1</maxApplicationLifetime>
+        <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+      </queue>
+      <queue>
+        <queuePath>root.a</queuePath>
+        <capacity>10.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>50.0</maxCapacity>
+        <absoluteCapacity>10.5</absoluteCapacity>
+        <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>42</maxParallelApps>
+        <queueName>a</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.a.a2</queuePath>
+            <capacity>70.0</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>7.35</absoluteCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>70.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>7.35</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>735</maxApplications>
+            <maxApplicationsPerUser>735</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>100</maxApplicationLifetime>
+            <defaultApplicationLifetime>50</defaultApplicationLifetime>
+          </queue>
+          <queue>
+            <queuePath>root.a.a1</queuePath>
+            <capacity>30.000002</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>3.15</absoluteCapacity>
+            <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>a1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <queues>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1a</queuePath>
+                <capacity>65.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>2.0475001</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1a</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>65.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>2.0475001</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>204</maxApplications>
+                <maxApplicationsPerUser>204</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue xsi:type="capacitySchedulerLeafQueueInfo">
+                <queuePath>root.a.a1.a1b</queuePath>
+                <capacity>15.000001</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.47250003</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1b</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>15.000001</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.47250003</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                    <amUsed>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amUsed>
+                    <amLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </amLimit>
+                    <userAmLimit>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </userAmLimit>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>fifo</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+                <leafQueueTemplate/>
+                <mode>percentage</mode>
+                <queueType>leaf</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>off</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+                <numActiveApplications>0</numActiveApplications>
+                <numPendingApplications>0</numPendingApplications>
+                <numContainers>0</numContainers>
+                <maxApplications>47</maxApplications>
+                <maxApplicationsPerUser>47</maxApplicationsPerUser>
+                <userLimit>100.0</userLimit>
+                <users/>
+                <userLimitFactor>1.0</userLimitFactor>
+                <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+                <AMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </AMResourceLimit>
+                <usedAMResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </usedAMResource>
+                <userAMResourceLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAMResourceLimit>
+                <preemptionDisabled>true</preemptionDisabled>
+                <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+                <defaultPriority>0</defaultPriority>
+                <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+                <maxApplicationLifetime>-1</maxApplicationLifetime>
+                <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+              </queue>
+              <queue>
+                <queuePath>root.a.a1.a1c</queuePath>
+                <capacity>20.0</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.63</absoluteCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <numApplications>0</numApplications>
+                <maxParallelApps>2147483647</maxParallelApps>
+                <queueName>a1c</queueName>
+                <isAbsoluteResource>false</isAbsoluteResource>
+                <state>RUNNING</state>
+                <queues/>
+                <resourcesUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </resourcesUsed>
+                <hideReservationQueues>false</hideReservationQueues>
+                <nodeLabels>*</nodeLabels>
+                <allocatedContainers>0</allocatedContainers>
+                <reservedContainers>0</reservedContainers>
+                <pendingContainers>0</pendingContainers>
+                <capacities>
+                  <queueCapacitiesByPartition>
+                    <partitionName/>
+                    <capacity>20.0</capacity>
+                    <usedCapacity>0.0</usedCapacity>
+                    <maxCapacity>100.0</maxCapacity>
+                    <absoluteCapacity>0.63</absoluteCapacity>
+                    <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                    <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                    <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                    <weight>-1.0</weight>
+                    <normalizedWeight>0.0</normalizedWeight>
+                    <configuredMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMinResource>
+                    <configuredMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>8192</maximumAllocation>
+                          <minimumAllocation>1024</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>4</maximumAllocation>
+                          <minimumAllocation>1</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </configuredMaxResource>
+                    <effectiveMinResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMinResource>
+                    <effectiveMaxResource>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </effectiveMaxResource>
+                  </queueCapacitiesByPartition>
+                </capacities>
+                <resources>
+                  <resourceUsagesByPartition>
+                    <partitionName/>
+                    <used>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </used>
+                    <reserved>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </reserved>
+                    <pending>
+                      <memory>0</memory>
+                      <vCores>0</vCores>
+                      <resourceInformations>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>memory-mb</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units>Mi</units>
+                          <value>0</value>
+                        </resourceInformation>
+                        <resourceInformation>
+                          <attributes/>
+                          <maximumAllocation>9223372036854775807</maximumAllocation>
+                          <minimumAllocation>0</minimumAllocation>
+                          <name>vcores</name>
+                          <resourceType>COUNTABLE</resourceType>
+                          <units/>
+                          <value>0</value>
+                        </resourceInformation>
+                      </resourceInformations>
+                    </pending>
+                  </resourceUsagesByPartition>
+                </resources>
+                <minEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </minEffectiveCapacity>
+                <maxEffectiveCapacity>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maxEffectiveCapacity>
+                <maximumAllocation>
+                  <memory>8192</memory>
+                  <vCores>4</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>8192</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>4</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </maximumAllocation>
+                <queueAcls>
+                  <queueAcl>
+                    <accessType>ADMINISTER_QUEUE</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                    <accessControlList>*</accessControlList>
+                  </queueAcl>
+                  <queueAcl>
+                    <accessType>SUBMIT_APP</accessType>
+                    <accessControlList> </accessControlList>
+                  </queueAcl>
+                </queueAcls>
+                <queuePriority>0</queuePriority>
+                <orderingPolicyInfo>utilization</orderingPolicyInfo>
+                <autoCreateChildQueueEnabled>true</autoCreateChildQueueEnabled>
+                <leafQueueTemplate>
+                  <property>
+                    <name>leaf-queue-template.capacity</name>
+                    <value>50</value>
+                  </property>
+                </leafQueueTemplate>
+                <mode>percentage</mode>
+                <queueType>parent</queueType>
+                <creationMethod>static</creationMethod>
+                <autoCreationEligibility>legacy</autoCreationEligibility>
+                <autoQueueTemplateProperties/>
+                <autoQueueParentTemplateProperties/>
+                <autoQueueLeafTemplateProperties/>
+              </queue>
+            </queues>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>30.000002</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>50.0</maxCapacity>
+                <absoluteCapacity>3.15</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>25.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>utilization</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>parent</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>10.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>50.0</maxCapacity>
+            <absoluteCapacity>10.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>50.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+      <queue>
+        <queuePath>root.b</queuePath>
+        <capacity>89.5</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>89.5</absoluteCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <numApplications>0</numApplications>
+        <maxParallelApps>2147483647</maxParallelApps>
+        <queueName>b</queueName>
+        <isAbsoluteResource>false</isAbsoluteResource>
+        <state>RUNNING</state>
+        <queues>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b1</queuePath>
+            <capacity>60.000004</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>53.7</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b1</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>60.000004</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>53.7</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>5370</maxApplications>
+            <maxApplicationsPerUser>5370</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b2</queuePath>
+            <capacity>39.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>35.3525</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b2</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>39.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>35.3525</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>3535</maxApplications>
+            <maxApplicationsPerUser>3535</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+          <queue xsi:type="capacitySchedulerLeafQueueInfo">
+            <queuePath>root.b.b3</queuePath>
+            <capacity>0.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>0.4475</absoluteCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <numApplications>0</numApplications>
+            <maxParallelApps>2147483647</maxParallelApps>
+            <queueName>b3</queueName>
+            <isAbsoluteResource>false</isAbsoluteResource>
+            <state>RUNNING</state>
+            <resourcesUsed>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </resourcesUsed>
+            <hideReservationQueues>false</hideReservationQueues>
+            <nodeLabels>*</nodeLabels>
+            <allocatedContainers>0</allocatedContainers>
+            <reservedContainers>0</reservedContainers>
+            <pendingContainers>0</pendingContainers>
+            <capacities>
+              <queueCapacitiesByPartition>
+                <partitionName/>
+                <capacity>0.5</capacity>
+                <usedCapacity>0.0</usedCapacity>
+                <maxCapacity>100.0</maxCapacity>
+                <absoluteCapacity>0.4475</absoluteCapacity>
+                <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+                <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+                <maxAMLimitPercentage>10.0</maxAMLimitPercentage>
+                <weight>-1.0</weight>
+                <normalizedWeight>0.0</normalizedWeight>
+                <configuredMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMinResource>
+                <configuredMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>8192</maximumAllocation>
+                      <minimumAllocation>1024</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>4</maximumAllocation>
+                      <minimumAllocation>1</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </configuredMaxResource>
+                <effectiveMinResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMinResource>
+                <effectiveMaxResource>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </effectiveMaxResource>
+              </queueCapacitiesByPartition>
+            </capacities>
+            <resources>
+              <resourceUsagesByPartition>
+                <partitionName/>
+                <used>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </used>
+                <reserved>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </reserved>
+                <pending>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </pending>
+                <amUsed>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amUsed>
+                <amLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </amLimit>
+                <userAmLimit>
+                  <memory>0</memory>
+                  <vCores>0</vCores>
+                  <resourceInformations>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>memory-mb</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units>Mi</units>
+                      <value>0</value>
+                    </resourceInformation>
+                    <resourceInformation>
+                      <attributes/>
+                      <maximumAllocation>9223372036854775807</maximumAllocation>
+                      <minimumAllocation>0</minimumAllocation>
+                      <name>vcores</name>
+                      <resourceType>COUNTABLE</resourceType>
+                      <units/>
+                      <value>0</value>
+                    </resourceInformation>
+                  </resourceInformations>
+                </userAmLimit>
+              </resourceUsagesByPartition>
+            </resources>
+            <minEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </minEffectiveCapacity>
+            <maxEffectiveCapacity>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maxEffectiveCapacity>
+            <maximumAllocation>
+              <memory>8192</memory>
+              <vCores>4</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>8192</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>4</value>
+                </resourceInformation>
+              </resourceInformations>
+            </maximumAllocation>
+            <queueAcls>
+              <queueAcl>
+                <accessType>ADMINISTER_QUEUE</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>APPLICATION_MAX_PRIORITY</accessType>
+                <accessControlList>*</accessControlList>
+              </queueAcl>
+              <queueAcl>
+                <accessType>SUBMIT_APP</accessType>
+                <accessControlList> </accessControlList>
+              </queueAcl>
+            </queueAcls>
+            <queuePriority>0</queuePriority>
+            <orderingPolicyInfo>fifo</orderingPolicyInfo>
+            <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+            <leafQueueTemplate/>
+            <mode>percentage</mode>
+            <queueType>leaf</queueType>
+            <creationMethod>static</creationMethod>
+            <autoCreationEligibility>off</autoCreationEligibility>
+            <autoQueueTemplateProperties/>
+            <autoQueueParentTemplateProperties/>
+            <autoQueueLeafTemplateProperties/>
+            <numActiveApplications>0</numActiveApplications>
+            <numPendingApplications>0</numPendingApplications>
+            <numContainers>0</numContainers>
+            <maxApplications>44</maxApplications>
+            <maxApplicationsPerUser>44</maxApplicationsPerUser>
+            <userLimit>100.0</userLimit>
+            <users/>
+            <userLimitFactor>100.0</userLimitFactor>
+            <configuredMaxAMResourceLimit>0.1</configuredMaxAMResourceLimit>
+            <AMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </AMResourceLimit>
+            <usedAMResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </usedAMResource>
+            <userAMResourceLimit>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </userAMResourceLimit>
+            <preemptionDisabled>true</preemptionDisabled>
+            <intraQueuePreemptionDisabled>true</intraQueuePreemptionDisabled>
+            <defaultPriority>0</defaultPriority>
+            <isAutoCreatedLeafQueue>false</isAutoCreatedLeafQueue>
+            <maxApplicationLifetime>-1</maxApplicationLifetime>
+            <defaultApplicationLifetime>-1</defaultApplicationLifetime>
+          </queue>
+        </queues>
+        <resourcesUsed>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resourcesUsed>
+        <hideReservationQueues>false</hideReservationQueues>
+        <nodeLabels>*</nodeLabels>
+        <allocatedContainers>0</allocatedContainers>
+        <reservedContainers>0</reservedContainers>
+        <pendingContainers>0</pendingContainers>
+        <capacities>
+          <queueCapacitiesByPartition>
+            <partitionName/>
+            <capacity>89.5</capacity>
+            <usedCapacity>0.0</usedCapacity>
+            <maxCapacity>100.0</maxCapacity>
+            <absoluteCapacity>89.5</absoluteCapacity>
+            <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+            <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+            <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+            <weight>-1.0</weight>
+            <normalizedWeight>0.0</normalizedWeight>
+            <configuredMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMinResource>
+            <configuredMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>8192</maximumAllocation>
+                  <minimumAllocation>1024</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>4</maximumAllocation>
+                  <minimumAllocation>1</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </configuredMaxResource>
+            <effectiveMinResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMinResource>
+            <effectiveMaxResource>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </effectiveMaxResource>
+          </queueCapacitiesByPartition>
+        </capacities>
+        <resources>
+          <resourceUsagesByPartition>
+            <partitionName/>
+            <used>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </used>
+            <reserved>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </reserved>
+            <pending>
+              <memory>0</memory>
+              <vCores>0</vCores>
+              <resourceInformations>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>memory-mb</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units>Mi</units>
+                  <value>0</value>
+                </resourceInformation>
+                <resourceInformation>
+                  <attributes/>
+                  <maximumAllocation>9223372036854775807</maximumAllocation>
+                  <minimumAllocation>0</minimumAllocation>
+                  <name>vcores</name>
+                  <resourceType>COUNTABLE</resourceType>
+                  <units/>
+                  <value>0</value>
+                </resourceInformation>
+              </resourceInformations>
+            </pending>
+          </resourceUsagesByPartition>
+        </resources>
+        <minEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </minEffectiveCapacity>
+        <maxEffectiveCapacity>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maxEffectiveCapacity>
+        <maximumAllocation>
+          <memory>8192</memory>
+          <vCores>4</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>8192</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>4</value>
+            </resourceInformation>
+          </resourceInformations>
+        </maximumAllocation>
+        <queueAcls>
+          <queueAcl>
+            <accessType>ADMINISTER_QUEUE</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>APPLICATION_MAX_PRIORITY</accessType>
+            <accessControlList>*</accessControlList>
+          </queueAcl>
+          <queueAcl>
+            <accessType>SUBMIT_APP</accessType>
+            <accessControlList> </accessControlList>
+          </queueAcl>
+        </queueAcls>
+        <queuePriority>0</queuePriority>
+        <orderingPolicyInfo>utilization</orderingPolicyInfo>
+        <autoCreateChildQueueEnabled>false</autoCreateChildQueueEnabled>
+        <leafQueueTemplate/>
+        <mode>percentage</mode>
+        <queueType>parent</queueType>
+        <creationMethod>static</creationMethod>
+        <autoCreationEligibility>off</autoCreationEligibility>
+        <autoQueueTemplateProperties/>
+        <autoQueueParentTemplateProperties/>
+        <autoQueueLeafTemplateProperties/>
+      </queue>
+    </queues>
+    <capacities>
+      <queueCapacitiesByPartition>
+        <partitionName/>
+        <capacity>100.0</capacity>
+        <usedCapacity>0.0</usedCapacity>
+        <maxCapacity>100.0</maxCapacity>
+        <absoluteCapacity>100.0</absoluteCapacity>
+        <absoluteUsedCapacity>0.0</absoluteUsedCapacity>
+        <absoluteMaxCapacity>100.0</absoluteMaxCapacity>
+        <maxAMLimitPercentage>0.0</maxAMLimitPercentage>
+        <weight>-1.0</weight>
+        <normalizedWeight>0.0</normalizedWeight>
+        <configuredMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMinResource>
+        <configuredMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>8192</maximumAllocation>
+              <minimumAllocation>1024</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>4</maximumAllocation>
+              <minimumAllocation>1</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </configuredMaxResource>
+        <effectiveMinResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMinResource>
+        <effectiveMaxResource>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </effectiveMaxResource>
+      </queueCapacitiesByPartition>
+    </capacities>
+    <health>
+      <lastrun>0</lastrun>
+      <operationsInfo>
+        <operation>last-allocation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-release</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-preemption</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-reservation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <lastRunDetails>
+        <operation>releases</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>allocations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>reservations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+          <resourceInformations>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>memory-mb</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units>Mi</units>
+              <value>0</value>
+            </resourceInformation>
+            <resourceInformation>
+              <attributes/>
+              <maximumAllocation>9223372036854775807</maximumAllocation>
+              <minimumAllocation>0</minimumAllocation>
+              <name>vcores</name>
+              <resourceType>COUNTABLE</resourceType>
+              <units/>
+              <value>0</value>
+            </resourceInformation>
+          </resourceInformations>
+        </resources>
+      </lastRunDetails>
+    </health>
+    <maximumAllocation>
+      <memory>8192</memory>
+      <vCores>4</vCores>
+      <resourceInformations>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>memory-mb</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units>Mi</units>
+          <value>8192</value>
+        </resourceInformation>
+        <resourceInformation>
+          <attributes/>
+          <maximumAllocation>9223372036854775807</maximumAllocation>
+          <minimumAllocation>0</minimumAllocation>
+          <name>vcores</name>
+          <resourceType>COUNTABLE</resourceType>
+          <units/>
+          <value>4</value>
+        </resourceInformation>
+      </resourceInformations>
+    </maximumAllocation>
+    <queueAcls>
+      <queueAcl>
+        <accessType>ADMINISTER_QUEUE</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>APPLICATION_MAX_PRIORITY</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+      <queueAcl>
+        <accessType>SUBMIT_APP</accessType>
+        <accessControlList>*</accessControlList>
+      </queueAcl>
+    </queueAcls>
+    <queuePriority>0</queuePriority>
+    <orderingPolicyInfo>utilization</orderingPolicyInfo>
+    <mode>percentage</mode>
+    <queueType>parent</queueType>
+    <creationMethod>static</creationMethod>
+    <autoCreationEligibility>off</autoCreationEligibility>
+    <autoQueueTemplateProperties/>
+    <autoQueueParentTemplateProperties/>
+    <autoQueueLeafTemplateProperties/>
+  </schedulerInfo>
+</scheduler>
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
index d29929047af6f..0ba39653a5c46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
@@ -74,6 +74,27 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -116,6 +137,25 @@
       <artifactId>guice</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minikdc</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.glassfish.grizzly</groupId>
+      <artifactId>grizzly-http-servlet</artifactId>
+      <scope>test</scope>
+    </dependency>
+
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
index fc0b9eeaef3cc..24e9ad23c930c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
@@ -19,12 +19,16 @@
 package org.apache.hadoop.yarn.server.router;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.ShutdownHookManager;
@@ -75,6 +79,7 @@ public class Router extends CompositeService {
   private WebApp webApp;
   @VisibleForTesting
   protected String webAppAddress;
+  private static long clusterTimeStamp = System.currentTimeMillis();
 
   /**
    * Priority of the Router shutdown hook.
@@ -88,7 +93,8 @@ public Router() {
   }
 
   protected void doSecureLogin() throws IOException {
-    // TODO YARN-6539 Create SecureLogin inside Router
+    SecurityUtil.login(this.conf, YarnConfiguration.ROUTER_KEYTAB,
+        YarnConfiguration.ROUTER_PRINCIPAL, getHostName(this.conf));
   }
 
   @Override
@@ -164,6 +170,16 @@ public WebApp getWebapp() {
   @VisibleForTesting
   public void startWepApp() {
 
+    // Initialize RouterWeb's CrossOrigin capability.
+    boolean enableCors = conf.getBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER,
+        YarnConfiguration.DEFAULT_ROUTER_WEBAPP_ENABLE_CORS_FILTER);
+    if (enableCors) {
+      conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX
+          + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true);
+    }
+
+    LOG.info("Instantiating RouterWebApp at {}.", webAppAddress);
+
     RMWebAppUtil.setupSecurityAndFilters(conf, null);
 
     Builder<Object> builder =
@@ -195,4 +211,35 @@ public static void main(String[] argv) {
       System.exit(-1);
     }
   }
+
+  @VisibleForTesting
+  public RouterClientRMService getClientRMProxyService() {
+    return clientRMProxyService;
+  }
+
+  @VisibleForTesting
+  public RouterRMAdminService getRmAdminProxyService() {
+    return rmAdminProxyService;
+  }
+
+  /**
+   * Returns the hostname for this Router. If the hostname is not
+   * explicitly configured in the given config, then it is determined.
+   *
+   * @param config configuration
+   * @return the hostname (NB: may not be a FQDN)
+   * @throws UnknownHostException if the hostname cannot be determined
+   */
+  private String getHostName(Configuration config)
+      throws UnknownHostException {
+    String name = config.get(YarnConfiguration.ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY);
+    if (name == null) {
+      name = InetAddress.getLocalHost().getHostName();
+    }
+    return name;
+  }
+
+  public static long getClusterTimeStamp() {
+    return clusterTimeStamp;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
new file mode 100644
index 0000000000000..f3b428dab4a6d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router;
+
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+
+/**
+ * Manages Router audit logs.
+ * Audit log format is written as key=value pairs. Tab separated.
+ */
+public class RouterAuditLogger {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RouterAuditLogger.class);
+
+  private RouterAuditLogger() {
+  }
+
+  enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, SUBCLUSTERID}
+
+  public static class AuditConstants {
+    static final String SUCCESS = "SUCCESS";
+    static final String FAILURE = "FAILURE";
+    static final String KEY_VAL_SEPARATOR = "=";
+    static final char PAIR_SEPARATOR = '\t';
+
+    public static final String GET_NEW_APP = "Get New App";
+    public static final String SUBMIT_NEW_APP = "Submit New App";
+    public static final String FORCE_KILL_APP = "Force Kill App";
+    public static final String GET_APP_REPORT = "Get Application Report";
+    public static final String TARGET_CLIENT_RM_SERVICE = "RouterClientRMService";
+    public static final String UNKNOWN = "UNKNOWN";
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the Router
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed.
+   * @param appId Application Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+   * delimiter and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user         User who made the service request to the Router
+   * @param operation    Operation requested by the user.
+   * @param target       The target on which the operation is being performed.
+   * @param appId        Application Id in which operation was performed.
+   * @param subClusterId Subcluster Id in which operation is performed.
+   *
+   * <br><br>
+   * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+   * delimiter and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, SubClusterId subClusterId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, subClusterId));
+    }
+  }
+
+  /**
+   * A helper api for creating an audit log for a successful event.
+   */
+  static String createSuccessLog(String user, String operation, String target,
+      ApplicationId appId, SubClusterId subClusterID) {
+    StringBuilder b =
+        createStringBuilderForSuccessEvent(user, operation, target);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (subClusterID != null) {
+      add(Keys.SUBCLUSTERID, subClusterID.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * A helper function for creating the common portion of a successful
+   * log message.
+   */
+  private static StringBuilder createStringBuilderForSuccessEvent(String user,
+      String operation, String target) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target, b);
+    add(Keys.RESULT, AuditConstants.SUCCESS, b);
+    return b;
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed.
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   *
+   * <br><br>
+   * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+   * delimiter and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(
+          createFailureLog(user, operation, perm, target, description, null,
+              null));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed.
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId Application Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+   * delimiter and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(
+          createFailureLog(user, operation, perm, target, description, appId,
+              null));
+    }
+  }
+
+  /**
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed.
+   * @param description Some additional information as to why the operation
+   *                    failed.
+   * @param appId Application Id in which operation was performed.
+   * @param subClusterId SubCluster Id in which operation was performed.
+   *
+   * <br><br>
+   * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+   * delimiter and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      SubClusterId subClusterId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(
+          createFailureLog(user, operation, perm, target, description, appId,
+              subClusterId));
+    }
+  }
+
+  /**
+   * Create a readable and parsable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed.
+   * @param description Some additional information as to why the operation failed.
+   * @param subClusterId SubCluster Id in which operation was performed.
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, SubClusterId subClusterId) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createFailureLog(user, operation, perm, target, description, null,
+          subClusterId));
+    }
+  }
+
+  /**
+   * A helper api for creating an audit log for a failure event.
+   */
+  static String createFailureLog(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      SubClusterId subClusterId) {
+    StringBuilder b =
+        createStringBuilderForFailureLog(user, operation, target, description,
+            perm);
+    if (appId != null) {
+      add(Keys.APPID, appId.toString(), b);
+    }
+    if (subClusterId != null) {
+      add(Keys.SUBCLUSTERID, subClusterId.toString(), b);
+    }
+    return b.toString();
+  }
+
+  /**
+   * A helper function for creating the common portion of a failure
+   * log message.
+   */
+  private static StringBuilder createStringBuilderForFailureLog(String user,
+      String operation, String target, String description, String perm) {
+    StringBuilder b = new StringBuilder();
+    start(Keys.USER, user, b);
+    addRemoteIP(b);
+    add(Keys.OPERATION, operation, b);
+    add(Keys.TARGET, target, b);
+    add(Keys.RESULT, AuditConstants.FAILURE, b);
+    add(Keys.DESCRIPTION, description, b);
+    add(Keys.PERMISSIONS, perm, b);
+    return b;
+  }
+
+  /**
+   * Adds the first key-val pair to the passed builder in the following format
+   * key=value.
+   */
+  static void start(Keys key, String value, StringBuilder b) {
+    b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+
+  /**
+   * Appends the key-val pair to the passed builder in the following format
+   * <pair-delim>key=value.
+   */
+  static void add(Keys key, String value, StringBuilder b) {
+    b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
+        .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+  }
+
+  /**
+   * A helper api to add remote IP address.
+   */
+  static void addRemoteIP(StringBuilder b) {
+    InetAddress ip = Server.getRemoteIp();
+    // ip address can be null for testcases
+    if (ip != null) {
+      add(Keys.IP, ip.getHostAddress(), b);
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
index 271d1fead5546..31d838d1b3ef7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
@@ -51,10 +51,88 @@ public final class RouterMetrics {
   private MutableGaugeInt numAppsFailedRetrieved;
   @Metric("# of multiple applications reports failed to be retrieved")
   private MutableGaugeInt numMultipleAppsFailedRetrieved;
-  @Metric("# of applicationAttempt reports failed to be retrieved")
+  @Metric("# of getApplicationAttempts failed to be retrieved")
   private MutableGaugeInt numAppAttemptsFailedRetrieved;
   @Metric("# of getClusterMetrics failed to be retrieved")
   private MutableGaugeInt numGetClusterMetricsFailedRetrieved;
+  @Metric("# of getClusterNodes failed to be retrieved")
+  private MutableGaugeInt numGetClusterNodesFailedRetrieved;
+  @Metric("# of getNodeToLabels failed to be retrieved")
+  private MutableGaugeInt numGetNodeToLabelsFailedRetrieved;
+  @Metric("# of getNodeToLabels failed to be retrieved")
+  private MutableGaugeInt numGetLabelsToNodesFailedRetrieved;
+  @Metric("# of getClusterNodeLabels failed to be retrieved")
+  private MutableGaugeInt numGetClusterNodeLabelsFailedRetrieved;
+  @Metric("# of getApplicationAttemptReports failed to be retrieved")
+  private MutableGaugeInt numAppAttemptReportFailedRetrieved;
+  @Metric("# of getQueueUserAcls failed to be retrieved")
+  private MutableGaugeInt numGetQueueUserAclsFailedRetrieved;
+  @Metric("# of getContainerReport failed to be retrieved")
+  private MutableGaugeInt numGetContainerReportFailedRetrieved;
+  @Metric("# of getContainers failed to be retrieved")
+  private MutableGaugeInt numGetContainersFailedRetrieved;
+  @Metric("# of listReservations failed to be retrieved")
+  private MutableGaugeInt numListReservationsFailedRetrieved;
+  @Metric("# of getResourceTypeInfo failed to be retrieved")
+  private MutableGaugeInt numGetResourceTypeInfo;
+  @Metric("# of failApplicationAttempt failed to be retrieved")
+  private MutableGaugeInt numFailAppAttemptFailedRetrieved;
+  @Metric("# of updateApplicationPriority failed to be retrieved")
+  private MutableGaugeInt numUpdateAppPriorityFailedRetrieved;
+  @Metric("# of updateApplicationPriority failed to be retrieved")
+  private MutableGaugeInt numUpdateAppTimeoutsFailedRetrieved;
+  @Metric("# of signalToContainer failed to be retrieved")
+  private MutableGaugeInt numSignalToContainerFailedRetrieved;
+  @Metric("# of getQueueInfo failed to be retrieved")
+  private MutableGaugeInt numGetQueueInfoFailedRetrieved;
+  @Metric("# of moveApplicationAcrossQueues failed to be retrieved")
+  private MutableGaugeInt numMoveApplicationAcrossQueuesFailedRetrieved;
+  @Metric("# of getResourceProfiles failed to be retrieved")
+  private MutableGaugeInt numGetResourceProfilesFailedRetrieved;
+  @Metric("# of getResourceProfile failed to be retrieved")
+  private MutableGaugeInt numGetResourceProfileFailedRetrieved;
+  @Metric("# of getAttributesToNodes failed to be retrieved")
+  private MutableGaugeInt numGetAttributesToNodesFailedRetrieved;
+  @Metric("# of getClusterNodeAttributes failed to be retrieved")
+  private MutableGaugeInt numGetClusterNodeAttributesFailedRetrieved;
+  @Metric("# of getNodesToAttributes failed to be retrieved")
+  private MutableGaugeInt numGetNodesToAttributesFailedRetrieved;
+  @Metric("# of getNewReservation failed to be retrieved")
+  private MutableGaugeInt numGetNewReservationFailedRetrieved;
+  @Metric("# of submitReservation failed to be retrieved")
+  private MutableGaugeInt numSubmitReservationFailedRetrieved;
+  @Metric("# of submitReservation failed to be retrieved")
+  private MutableGaugeInt numUpdateReservationFailedRetrieved;
+  @Metric("# of deleteReservation failed to be retrieved")
+  private MutableGaugeInt numDeleteReservationFailedRetrieved;
+  @Metric("# of listReservation failed to be retrieved")
+  private MutableGaugeInt numListReservationFailedRetrieved;
+  @Metric("# of getAppActivities failed to be retrieved")
+  private MutableGaugeInt numGetAppActivitiesFailedRetrieved;
+  @Metric("# of getAppStatistics failed to be retrieved")
+  private MutableGaugeInt numGetAppStatisticsFailedRetrieved;
+  @Metric("# of getAppPriority failed to be retrieved")
+  private MutableGaugeInt numGetAppPriorityFailedRetrieved;
+  @Metric("# of getAppQueue failed to be retrieved")
+  private MutableGaugeInt numGetAppQueueFailedRetrieved;
+  @Metric("# of updateAppQueue failed to be retrieved")
+  private MutableGaugeInt numUpdateAppQueueFailedRetrieved;
+  @Metric("# of getAppTimeout failed to be retrieved")
+  private MutableGaugeInt numGetAppTimeoutFailedRetrieved;
+  @Metric("# of getAppTimeouts failed to be retrieved")
+  private MutableGaugeInt numGetAppTimeoutsFailedRetrieved;
+  @Metric("# of refreshQueues failed to be retrieved")
+  private MutableGaugeInt numRefreshQueuesFailedRetrieved;
+  @Metric("# of getRMNodeLabels failed to be retrieved")
+  private MutableGaugeInt numGetRMNodeLabelsFailedRetrieved;
+  @Metric("# of checkUserAccessToQueue failed to be retrieved")
+  private MutableGaugeInt numCheckUserAccessToQueueFailedRetrieved;
+  @Metric("# of getDelegationToken failed to be retrieved")
+  private MutableGaugeInt numGetDelegationTokenFailedRetrieved;
+  @Metric("# of renewDelegationToken failed to be retrieved")
+  private MutableGaugeInt numRenewDelegationTokenFailedRetrieved;
+  @Metric("# of renewDelegationToken failed to be retrieved")
+  private MutableGaugeInt numCancelDelegationTokenFailedRetrieved;
 
   // Aggregate metrics are shared, and don't have to be looked up per call
   @Metric("Total number of successful Submitted apps and latency(ms)")
@@ -65,16 +143,90 @@ public final class RouterMetrics {
   private MutableRate totalSucceededAppsCreated;
   @Metric("Total number of successful Retrieved app reports and latency(ms)")
   private MutableRate totalSucceededAppsRetrieved;
-  @Metric("Total number of successful Retrieved multiple apps reports and "
-      + "latency(ms)")
+  @Metric("Total number of successful Retrieved multiple apps reports and latency(ms)")
   private MutableRate totalSucceededMultipleAppsRetrieved;
-  @Metric("Total number of successful Retrieved " +
-          "appAttempt reports and latency(ms)")
+  @Metric("Total number of successful Retrieved appAttempt reports and latency(ms)")
   private MutableRate totalSucceededAppAttemptsRetrieved;
-  @Metric("Total number of successful Retrieved getClusterMetrics and "
-      + "latency(ms)")
+  @Metric("Total number of successful Retrieved getClusterMetrics and latency(ms)")
   private MutableRate totalSucceededGetClusterMetricsRetrieved;
-
+  @Metric("Total number of successful Retrieved getClusterNodes and latency(ms)")
+  private MutableRate totalSucceededGetClusterNodesRetrieved;
+  @Metric("Total number of successful Retrieved getNodeToLabels and latency(ms)")
+  private MutableRate totalSucceededGetNodeToLabelsRetrieved;
+  @Metric("Total number of successful Retrieved getNodeToLabels and latency(ms)")
+  private MutableRate totalSucceededGetLabelsToNodesRetrieved;
+  @Metric("Total number of successful Retrieved getClusterNodeLabels and latency(ms)")
+  private MutableRate totalSucceededGetClusterNodeLabelsRetrieved;
+  @Metric("Total number of successful Retrieved getApplicationAttemptReport and latency(ms)")
+  private MutableRate totalSucceededAppAttemptReportRetrieved;
+  @Metric("Total number of successful Retrieved getQueueUserAcls and latency(ms)")
+  private MutableRate totalSucceededGetQueueUserAclsRetrieved;
+  @Metric("Total number of successful Retrieved getContainerReport and latency(ms)")
+  private MutableRate totalSucceededGetContainerReportRetrieved;
+  @Metric("Total number of successful Retrieved getContainers and latency(ms)")
+  private MutableRate totalSucceededGetContainersRetrieved;
+  @Metric("Total number of successful Retrieved listReservations and latency(ms)")
+  private MutableRate totalSucceededListReservationsRetrieved;
+  @Metric("Total number of successful Retrieved getResourceTypeInfo and latency(ms)")
+  private MutableRate totalSucceededGetResourceTypeInfoRetrieved;
+  @Metric("Total number of successful Retrieved failApplicationAttempt and latency(ms)")
+  private MutableRate totalSucceededFailAppAttemptRetrieved;
+  @Metric("Total number of successful Retrieved updateApplicationPriority and latency(ms)")
+  private MutableRate totalSucceededUpdateAppPriorityRetrieved;
+  @Metric("Total number of successful Retrieved updateApplicationTimeouts and latency(ms)")
+  private MutableRate totalSucceededUpdateAppTimeoutsRetrieved;
+  @Metric("Total number of successful Retrieved signalToContainer and latency(ms)")
+  private MutableRate totalSucceededSignalToContainerRetrieved;
+  @Metric("Total number of successful Retrieved getQueueInfo and latency(ms)")
+  private MutableRate totalSucceededGetQueueInfoRetrieved;
+  @Metric("Total number of successful Retrieved moveApplicationAcrossQueues and latency(ms)")
+  private MutableRate totalSucceededMoveApplicationAcrossQueuesRetrieved;
+  @Metric("Total number of successful Retrieved getResourceProfiles and latency(ms)")
+  private MutableRate totalSucceededGetResourceProfilesRetrieved;
+  @Metric("Total number of successful Retrieved getResourceProfile and latency(ms)")
+  private MutableRate totalSucceededGetResourceProfileRetrieved;
+  @Metric("Total number of successful Retrieved getAttributesToNodes and latency(ms)")
+  private MutableRate totalSucceededGetAttributesToNodesRetrieved;
+  @Metric("Total number of successful Retrieved getClusterNodeAttributes and latency(ms)")
+  private MutableRate totalSucceededGetClusterNodeAttributesRetrieved;
+  @Metric("Total number of successful Retrieved getNodesToAttributes and latency(ms)")
+  private MutableRate totalSucceededGetNodesToAttributesRetrieved;
+  @Metric("Total number of successful Retrieved GetNewReservation and latency(ms)")
+  private MutableRate totalSucceededGetNewReservationRetrieved;
+  @Metric("Total number of successful Retrieved SubmitReservation and latency(ms)")
+  private MutableRate totalSucceededSubmitReservationRetrieved;
+  @Metric("Total number of successful Retrieved UpdateReservation and latency(ms)")
+  private MutableRate totalSucceededUpdateReservationRetrieved;
+  @Metric("Total number of successful Retrieved DeleteReservation and latency(ms)")
+  private MutableRate totalSucceededDeleteReservationRetrieved;
+  @Metric("Total number of successful Retrieved ListReservation and latency(ms)")
+  private MutableRate totalSucceededListReservationRetrieved;
+  @Metric("Total number of successful Retrieved GetAppActivities and latency(ms)")
+  private MutableRate totalSucceededGetAppActivitiesRetrieved;
+  @Metric("Total number of successful Retrieved GetAppStatistics and latency(ms)")
+  private MutableRate totalSucceededGetAppStatisticsRetrieved;
+  @Metric("Total number of successful Retrieved GetAppPriority and latency(ms)")
+  private MutableRate totalSucceededGetAppPriorityRetrieved;
+  @Metric("Total number of successful Retrieved GetAppQueue and latency(ms)")
+  private MutableRate totalSucceededGetAppQueueRetrieved;
+  @Metric("Total number of successful Retrieved UpdateAppQueue and latency(ms)")
+  private MutableRate totalSucceededUpdateAppQueueRetrieved;
+  @Metric("Total number of successful Retrieved GetAppTimeout and latency(ms)")
+  private MutableRate totalSucceededGetAppTimeoutRetrieved;
+  @Metric("Total number of successful Retrieved GetAppTimeouts and latency(ms)")
+  private MutableRate totalSucceededGetAppTimeoutsRetrieved;
+  @Metric("Total number of successful Retrieved RefreshQueues and latency(ms)")
+  private MutableRate totalSucceededRefreshQueuesRetrieved;
+  @Metric("Total number of successful Retrieved GetRMNodeLabels and latency(ms)")
+  private MutableRate totalSucceededGetRMNodeLabelsRetrieved;
+  @Metric("Total number of successful Retrieved CheckUserAccessToQueue and latency(ms)")
+  private MutableRate totalSucceededCheckUserAccessToQueueRetrieved;
+  @Metric("Total number of successful Retrieved GetDelegationToken and latency(ms)")
+  private MutableRate totalSucceededGetDelegationTokenRetrieved;
+  @Metric("Total number of successful Retrieved RenewDelegationToken and latency(ms)")
+  private MutableRate totalSucceededRenewDelegationTokenRetrieved;
+  @Metric("Total number of successful Retrieved CancelDelegationToken and latency(ms)")
+  private MutableRate totalSucceededCancelDelegationTokenRetrieved;
 
   /**
    * Provide quantile counters for all latencies.
@@ -86,10 +238,50 @@ public final class RouterMetrics {
   private MutableQuantiles getApplicationsReportLatency;
   private MutableQuantiles getApplicationAttemptReportLatency;
   private MutableQuantiles getClusterMetricsLatency;
+  private MutableQuantiles getClusterNodesLatency;
+  private MutableQuantiles getNodeToLabelsLatency;
+  private MutableQuantiles getLabelToNodesLatency;
+  private MutableQuantiles getClusterNodeLabelsLatency;
+  private MutableQuantiles getApplicationAttemptsLatency;
+  private MutableQuantiles getQueueUserAclsLatency;
+  private MutableQuantiles getContainerReportLatency;
+  private MutableQuantiles getContainerLatency;
+  private MutableQuantiles listReservationsLatency;
+  private MutableQuantiles listResourceTypeInfoLatency;
+  private MutableQuantiles failAppAttemptLatency;
+  private MutableQuantiles updateAppPriorityLatency;
+  private MutableQuantiles updateAppTimeoutsLatency;
+  private MutableQuantiles signalToContainerLatency;
+  private MutableQuantiles getQueueInfoLatency;
+  private MutableQuantiles moveApplicationAcrossQueuesLatency;
+  private MutableQuantiles getResourceProfilesLatency;
+  private MutableQuantiles getResourceProfileLatency;
+  private MutableQuantiles getAttributesToNodesLatency;
+  private MutableQuantiles getClusterNodeAttributesLatency;
+  private MutableQuantiles getNodesToAttributesLatency;
+  private MutableQuantiles getNewReservationLatency;
+  private MutableQuantiles submitReservationLatency;
+  private MutableQuantiles updateReservationLatency;
+  private MutableQuantiles deleteReservationLatency;
+  private MutableQuantiles listReservationLatency;
+  private MutableQuantiles getAppActivitiesLatency;
+  private MutableQuantiles getAppStatisticsLatency;
+  private MutableQuantiles getAppPriorityLatency;
+  private MutableQuantiles getAppQueueLatency;
+  private MutableQuantiles getUpdateQueueLatency;
+  private MutableQuantiles getAppTimeoutLatency;
+  private MutableQuantiles getAppTimeoutsLatency;
+  private MutableQuantiles getRefreshQueuesLatency;
+  private MutableQuantiles getRMNodeLabelsLatency;
+  private MutableQuantiles checkUserAccessToQueueLatency;
+  private MutableQuantiles getDelegationTokenLatency;
+  private MutableQuantiles renewDelegationTokenLatency;
+  private MutableQuantiles cancelDelegationTokenLatency;
 
-  private static volatile RouterMetrics INSTANCE = null;
+  private static volatile RouterMetrics instance = null;
   private static MetricsRegistry registry;
 
+  @SuppressWarnings("checkstyle:MethodLength")
   private RouterMetrics() {
     registry = new MetricsRegistry(RECORD_INFO);
     registry.tag(RECORD_INFO, "Router");
@@ -112,25 +304,168 @@ private RouterMetrics() {
     getClusterMetricsLatency =
         registry.newQuantiles("getClusterMetricsLatency",
             "latency of get cluster metrics", "ops", "latency", 10);
+
+    getClusterNodesLatency =
+        registry.newQuantiles("getClusterNodesLatency",
+            "latency of get cluster nodes", "ops", "latency", 10);
+
+    getNodeToLabelsLatency =
+        registry.newQuantiles("getNodeToLabelsLatency",
+            "latency of get node labels", "ops", "latency", 10);
+
+    getLabelToNodesLatency =
+        registry.newQuantiles("getLabelToNodesLatency",
+            "latency of get label nodes", "ops", "latency", 10);
+
+    getClusterNodeLabelsLatency =
+        registry.newQuantiles("getClusterNodeLabelsLatency",
+            "latency of get cluster node labels", "ops", "latency", 10);
+
+    getApplicationAttemptsLatency =
+        registry.newQuantiles("getApplicationAttemptsLatency",
+            "latency of get application attempts", "ops", "latency", 10);
+
+    getQueueUserAclsLatency =
+        registry.newQuantiles("getQueueUserAclsLatency",
+            "latency of get queue user acls", "ops", "latency", 10);
+
+    getContainerReportLatency =
+        registry.newQuantiles("getContainerReportLatency",
+            "latency of get container report", "ops", "latency", 10);
+
+    getContainerLatency =
+        registry.newQuantiles("getContainerLatency",
+            "latency of get container", "ops", "latency", 10);
+
+    listReservationsLatency =
+        registry.newQuantiles("listReservationsLatency",
+            "latency of list reservations", "ops", "latency", 10);
+
+    listResourceTypeInfoLatency =
+        registry.newQuantiles("getResourceTypeInfoLatency",
+            "latency of get resource type info", "ops", "latency", 10);
+
+    failAppAttemptLatency =
+        registry.newQuantiles("failApplicationAttemptLatency",
+            "latency of fail application attempt", "ops", "latency", 10);
+
+    updateAppPriorityLatency =
+        registry.newQuantiles("updateApplicationPriorityLatency",
+            "latency of update application priority", "ops", "latency", 10);
+
+    updateAppTimeoutsLatency =
+        registry.newQuantiles("updateApplicationTimeoutsLatency",
+            "latency of update application timeouts", "ops", "latency", 10);
+
+    signalToContainerLatency =
+        registry.newQuantiles("signalToContainerLatency",
+            "latency of signal to container timeouts", "ops", "latency", 10);
+
+    getQueueInfoLatency =
+        registry.newQuantiles("getQueueInfoLatency",
+            "latency of get queue info timeouts", "ops", "latency", 10);
+
+    moveApplicationAcrossQueuesLatency =
+        registry.newQuantiles("moveApplicationAcrossQueuesLatency",
+            "latency of move application across queues timeouts", "ops", "latency", 10);
+
+    getResourceProfilesLatency =
+        registry.newQuantiles("getResourceProfilesLatency",
+            "latency of get resource profiles timeouts", "ops", "latency", 10);
+
+    getResourceProfileLatency =
+        registry.newQuantiles("getResourceProfileLatency",
+            "latency of get resource profile timeouts", "ops", "latency", 10);
+
+    getAttributesToNodesLatency =
+        registry.newQuantiles("getAttributesToNodesLatency",
+            "latency of get attributes to nodes timeouts", "ops", "latency", 10);
+
+    getClusterNodeAttributesLatency =
+        registry.newQuantiles("getClusterNodeAttributesLatency",
+            "latency of get cluster node attributes timeouts", "ops", "latency", 10);
+
+    getNodesToAttributesLatency =
+        registry.newQuantiles("getNodesToAttributesLatency",
+            "latency of get nodes to attributes timeouts", "ops", "latency", 10);
+
+    getNewReservationLatency =
+        registry.newQuantiles("getNewReservationLatency",
+            "latency of get new reservation timeouts", "ops", "latency", 10);
+
+    submitReservationLatency =
+        registry.newQuantiles("submitReservationLatency",
+            "latency of submit reservation timeouts", "ops", "latency", 10);
+
+    updateReservationLatency =
+        registry.newQuantiles("updateReservationLatency",
+            "latency of update reservation timeouts", "ops", "latency", 10);
+
+    deleteReservationLatency =
+        registry.newQuantiles("deleteReservationLatency",
+            "latency of delete reservation timeouts", "ops", "latency", 10);
+
+    listReservationLatency =
+        registry.newQuantiles("listReservationLatency",
+            "latency of list reservation timeouts", "ops", "latency", 10);
+
+    getAppActivitiesLatency = registry.newQuantiles("getAppActivitiesLatency",
+         "latency of get app activities timeouts", "ops", "latency", 10);
+
+    getAppStatisticsLatency = registry.newQuantiles("getAppStatisticsLatency",
+         "latency of get app statistics timeouts", "ops", "latency", 10);
+
+    getAppPriorityLatency = registry.newQuantiles("getAppPriorityLatency",
+         "latency of get app priority timeouts", "ops", "latency", 10);
+
+    getAppQueueLatency = registry.newQuantiles("getAppQueueLatency",
+         "latency of get app queue timeouts", "ops", "latency", 10);
+
+    getUpdateQueueLatency = registry.newQuantiles("getUpdateQueueLatency",
+        "latency of update app queue timeouts", "ops", "latency", 10);
+
+    getAppTimeoutLatency = registry.newQuantiles("getAppTimeoutLatency",
+        "latency of get apptimeout timeouts", "ops", "latency", 10);
+
+    getAppTimeoutsLatency = registry.newQuantiles("getAppTimeoutsLatency",
+         "latency of get apptimeouts timeouts", "ops", "latency", 10);
+
+    getRefreshQueuesLatency = registry.newQuantiles("getRefreshQueuesLatency",
+         "latency of get refresh queues timeouts", "ops", "latency", 10);
+
+    getRMNodeLabelsLatency = registry.newQuantiles("getRMNodeLabelsLatency",
+        "latency of get rmnodelabels timeouts", "ops", "latency", 10);
+
+    checkUserAccessToQueueLatency = registry.newQuantiles("checkUserAccessToQueueLatency",
+        "latency of get apptimeouts timeouts", "ops", "latency", 10);
+
+    getDelegationTokenLatency = registry.newQuantiles("getDelegationTokenLatency",
+        "latency of get delegation token timeouts", "ops", "latency", 10);
+
+    renewDelegationTokenLatency = registry.newQuantiles("renewDelegationTokenLatency",
+       "latency of renew delegation token timeouts", "ops", "latency", 10);
+
+    cancelDelegationTokenLatency = registry.newQuantiles("cancelDelegationTokenLatency",
+        "latency of cancel delegation token timeouts", "ops", "latency", 10);
   }
 
   public static RouterMetrics getMetrics() {
     if (!isInitialized.get()) {
       synchronized (RouterMetrics.class) {
-        if (INSTANCE == null) {
-          INSTANCE = DefaultMetricsSystem.instance().register("RouterMetrics",
+        if (instance == null) {
+          instance = DefaultMetricsSystem.instance().register("RouterMetrics",
               "Metrics for the Yarn Router", new RouterMetrics());
           isInitialized.set(true);
         }
       }
     }
-    return INSTANCE;
+    return instance;
   }
 
   @VisibleForTesting
   synchronized static void destroy() {
     isInitialized.set(false);
-    INSTANCE = null;
+    instance = null;
   }
 
   @VisibleForTesting
@@ -168,6 +503,201 @@ public long getNumSucceededGetClusterMetricsRetrieved(){
     return totalSucceededGetClusterMetricsRetrieved.lastStat().numSamples();
   }
 
+  @VisibleForTesting
+  public long getNumSucceededGetClusterNodesRetrieved(){
+    return totalSucceededGetClusterNodesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetNodeToLabelsRetrieved(){
+    return totalSucceededGetNodeToLabelsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetLabelsToNodesRetrieved(){
+    return totalSucceededGetLabelsToNodesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetClusterNodeLabelsRetrieved(){
+    return totalSucceededGetClusterNodeLabelsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededAppAttemptReportRetrieved(){
+    return totalSucceededAppAttemptReportRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetQueueUserAclsRetrieved(){
+    return totalSucceededGetQueueUserAclsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetContainerReportRetrieved() {
+    return totalSucceededGetContainerReportRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetContainersRetrieved() {
+    return totalSucceededGetContainersRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededListReservationsRetrieved() {
+    return totalSucceededListReservationsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetResourceTypeInfoRetrieved() {
+    return totalSucceededGetResourceTypeInfoRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededFailAppAttemptRetrieved() {
+    return totalSucceededFailAppAttemptRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededUpdateAppPriorityRetrieved() {
+    return totalSucceededUpdateAppPriorityRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededUpdateAppTimeoutsRetrieved() {
+    return totalSucceededUpdateAppTimeoutsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededSignalToContainerRetrieved() {
+    return totalSucceededSignalToContainerRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetQueueInfoRetrieved() {
+    return totalSucceededGetQueueInfoRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededMoveApplicationAcrossQueuesRetrieved() {
+    return totalSucceededMoveApplicationAcrossQueuesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetResourceProfilesRetrieved() {
+    return totalSucceededGetResourceProfilesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetResourceProfileRetrieved() {
+    return totalSucceededGetResourceProfileRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAttributesToNodesRetrieved() {
+    return totalSucceededGetAttributesToNodesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetClusterNodeAttributesRetrieved() {
+    return totalSucceededGetClusterNodeAttributesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetNodesToAttributesRetrieved() {
+    return totalSucceededGetNodesToAttributesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetNewReservationRetrieved() {
+    return totalSucceededGetNewReservationRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededSubmitReservationRetrieved() {
+    return totalSucceededSubmitReservationRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededUpdateReservationRetrieved() {
+    return totalSucceededUpdateReservationRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededDeleteReservationRetrieved() {
+    return totalSucceededDeleteReservationRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededListReservationRetrieved() {
+    return totalSucceededListReservationRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppActivitiesRetrieved() {
+    return totalSucceededGetAppActivitiesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppStatisticsRetrieved() {
+    return totalSucceededGetAppStatisticsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppPriorityRetrieved() {
+    return totalSucceededGetAppPriorityRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppQueueRetrieved() {
+    return totalSucceededGetAppQueueRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededUpdateAppQueueRetrieved() {
+    return totalSucceededUpdateAppQueueRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppTimeoutRetrieved() {
+    return totalSucceededGetAppTimeoutRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetAppTimeoutsRetrieved() {
+    return totalSucceededGetAppTimeoutsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededRefreshQueuesRetrieved() {
+    return totalSucceededRefreshQueuesRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetRMNodeLabelsRetrieved() {
+    return totalSucceededGetRMNodeLabelsRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededCheckUserAccessToQueueRetrieved() {
+    return totalSucceededCheckUserAccessToQueueRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededGetDelegationTokenRetrieved() {
+    return totalSucceededGetDelegationTokenRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededRenewDelegationTokenRetrieved() {
+    return totalSucceededRenewDelegationTokenRetrieved.lastStat().numSamples();
+  }
+
+  @VisibleForTesting
+  public long getNumSucceededCancelDelegationTokenRetrieved() {
+    return totalSucceededCancelDelegationTokenRetrieved.lastStat().numSamples();
+  }
+
   @VisibleForTesting
   public double getLatencySucceededAppsCreated() {
     return totalSucceededAppsCreated.lastStat().mean();
@@ -185,7 +715,7 @@ public double getLatencySucceededAppsKilled() {
 
   @VisibleForTesting
   public double getLatencySucceededGetAppAttemptReport() {
-    return totalSucceededAppAttemptsRetrieved.lastStat().mean();
+    return totalSucceededAppAttemptReportRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
@@ -204,101 +734,817 @@ public double getLatencySucceededGetClusterMetricsRetrieved() {
   }
 
   @VisibleForTesting
-  public int getAppsFailedCreated() {
-    return numAppsFailedCreated.value();
+  public double getLatencySucceededGetClusterNodesRetrieved() {
+    return totalSucceededGetClusterNodesRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getAppsFailedSubmitted() {
-    return numAppsFailedSubmitted.value();
+  public double getLatencySucceededGetNodeToLabelsRetrieved() {
+    return totalSucceededGetNodeToLabelsRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getAppsFailedKilled() {
-    return numAppsFailedKilled.value();
+  public double getLatencySucceededGetLabelsToNodesRetrieved() {
+    return totalSucceededGetLabelsToNodesRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getAppsFailedRetrieved() {
-    return numAppsFailedRetrieved.value();
+  public double getLatencySucceededGetClusterNodeLabelsRetrieved() {
+    return totalSucceededGetClusterNodeLabelsRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getAppAttemptsFailedRetrieved() {
-    return numAppsFailedRetrieved.value();
+  public double getLatencySucceededAppAttemptRetrieved() {
+    return totalSucceededAppAttemptsRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getMultipleAppsFailedRetrieved() {
-    return numMultipleAppsFailedRetrieved.value();
+  public double getLatencySucceededGetQueueUserAclsRetrieved() {
+    return totalSucceededGetQueueUserAclsRetrieved.lastStat().mean();
   }
 
   @VisibleForTesting
-  public int getClusterMetricsFailedRetrieved() {
-    return numGetClusterMetricsFailedRetrieved.value();
+  public double getLatencySucceededGetContainerReportRetrieved() {
+    return totalSucceededGetContainerReportRetrieved.lastStat().mean();
   }
 
-  public void succeededAppsCreated(long duration) {
-    totalSucceededAppsCreated.add(duration);
-    getNewApplicationLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededGetContainersRetrieved() {
+    return totalSucceededGetContainersRetrieved.lastStat().mean();
   }
 
-  public void succeededAppsSubmitted(long duration) {
-    totalSucceededAppsSubmitted.add(duration);
-    submitApplicationLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededListReservationsRetrieved() {
+    return totalSucceededListReservationsRetrieved.lastStat().mean();
   }
 
-  public void succeededAppsKilled(long duration) {
-    totalSucceededAppsKilled.add(duration);
-    killApplicationLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededGetResourceTypeInfoRetrieved() {
+    return totalSucceededGetResourceTypeInfoRetrieved.lastStat().mean();
   }
 
-  public void succeededAppsRetrieved(long duration) {
-    totalSucceededAppsRetrieved.add(duration);
-    getApplicationReportLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededFailAppAttemptRetrieved() {
+    return totalSucceededFailAppAttemptRetrieved.lastStat().mean();
   }
 
-  public void succeededMultipleAppsRetrieved(long duration) {
-    totalSucceededMultipleAppsRetrieved.add(duration);
-    getApplicationsReportLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededUpdateAppPriorityRetrieved() {
+    return totalSucceededUpdateAppPriorityRetrieved.lastStat().mean();
   }
 
-  public void succeededAppAttemptsRetrieved(long duration) {
-    totalSucceededAppAttemptsRetrieved.add(duration);
-    getApplicationAttemptReportLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededUpdateAppTimeoutsRetrieved() {
+    return totalSucceededUpdateAppTimeoutsRetrieved.lastStat().mean();
   }
 
-  public void succeededGetClusterMetricsRetrieved(long duration) {
-    totalSucceededGetClusterMetricsRetrieved.add(duration);
-    getClusterMetricsLatency.add(duration);
+  @VisibleForTesting
+  public double getLatencySucceededSignalToContainerRetrieved() {
+    return totalSucceededSignalToContainerRetrieved.lastStat().mean();
   }
 
-  public void incrAppsFailedCreated() {
-    numAppsFailedCreated.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetQueueInfoRetrieved() {
+    return totalSucceededGetQueueInfoRetrieved.lastStat().mean();
   }
 
-  public void incrAppsFailedSubmitted() {
-    numAppsFailedSubmitted.incr();
+  @VisibleForTesting
+  public double getLatencySucceededMoveApplicationAcrossQueuesRetrieved() {
+    return totalSucceededMoveApplicationAcrossQueuesRetrieved.lastStat().mean();
   }
 
-  public void incrAppsFailedKilled() {
-    numAppsFailedKilled.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetResourceProfilesRetrieved() {
+    return totalSucceededGetResourceProfilesRetrieved.lastStat().mean();
   }
 
-  public void incrAppsFailedRetrieved() {
-    numAppsFailedRetrieved.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetResourceProfileRetrieved() {
+    return totalSucceededGetResourceProfileRetrieved.lastStat().mean();
   }
 
-  public void incrMultipleAppsFailedRetrieved() {
-    numMultipleAppsFailedRetrieved.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetAttributesToNodesRetrieved() {
+    return totalSucceededGetAttributesToNodesRetrieved.lastStat().mean();
   }
 
-  public void incrAppAttemptsFailedRetrieved() {
-    numAppAttemptsFailedRetrieved.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetClusterNodeAttributesRetrieved() {
+    return totalSucceededGetClusterNodeAttributesRetrieved.lastStat().mean();
   }
 
-  public void incrGetClusterMetricsFailedRetrieved() {
-    numGetClusterMetricsFailedRetrieved.incr();
+  @VisibleForTesting
+  public double getLatencySucceededGetNodesToAttributesRetrieved() {
+    return totalSucceededGetNodesToAttributesRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetNewReservationRetrieved() {
+    return totalSucceededGetNewReservationRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededSubmitReservationRetrieved() {
+    return totalSucceededSubmitReservationRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededUpdateReservationRetrieved() {
+    return totalSucceededUpdateReservationRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededDeleteReservationRetrieved() {
+    return totalSucceededDeleteReservationRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededListReservationRetrieved() {
+    return totalSucceededListReservationRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetAppActivitiesRetrieved() {
+    return totalSucceededGetAppActivitiesRetrieved.lastStat().mean();
   }
 
+  @VisibleForTesting
+  public double getLatencySucceededGetAppStatisticsRetrieved() {
+    return totalSucceededGetAppStatisticsRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetAppPriorityRetrieved() {
+    return totalSucceededGetAppPriorityRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetAppQueueRetrieved() {
+    return totalSucceededGetAppQueueRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededUpdateAppQueueRetrieved() {
+    return totalSucceededUpdateAppQueueRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetAppTimeoutRetrieved() {
+    return totalSucceededGetAppTimeoutRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetAppTimeoutsRetrieved() {
+    return totalSucceededGetAppTimeoutsRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededRefreshQueuesRetrieved() {
+    return totalSucceededRefreshQueuesRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetRMNodeLabelsRetrieved() {
+    return totalSucceededGetRMNodeLabelsRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededCheckUserAccessToQueueRetrieved() {
+    return totalSucceededCheckUserAccessToQueueRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededGetDelegationTokenRetrieved() {
+    return totalSucceededGetDelegationTokenRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededRenewDelegationTokenRetrieved() {
+    return totalSucceededRenewDelegationTokenRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public double getLatencySucceededCancelDelegationTokenRetrieved() {
+    return totalSucceededCancelDelegationTokenRetrieved.lastStat().mean();
+  }
+
+  @VisibleForTesting
+  public int getAppsFailedCreated() {
+    return numAppsFailedCreated.value();
+  }
+
+  @VisibleForTesting
+  public int getAppsFailedSubmitted() {
+    return numAppsFailedSubmitted.value();
+  }
+
+  @VisibleForTesting
+  public int getAppsFailedKilled() {
+    return numAppsFailedKilled.value();
+  }
+
+  @VisibleForTesting
+  public int getAppsFailedRetrieved() {
+    return numAppsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getAppAttemptsFailedRetrieved() {
+    return numAppAttemptsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getMultipleAppsFailedRetrieved() {
+    return numMultipleAppsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getClusterMetricsFailedRetrieved() {
+    return numGetClusterMetricsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getClusterNodesFailedRetrieved() {
+    return numGetClusterNodesFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getNodeToLabelsFailedRetrieved() {
+    return numGetNodeToLabelsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getLabelsToNodesFailedRetrieved() {
+    return numGetLabelsToNodesFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getGetClusterNodeLabelsFailedRetrieved() {
+    return numGetClusterNodeLabelsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getAppAttemptReportFailedRetrieved() {
+    return numAppAttemptReportFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getQueueUserAclsFailedRetrieved() {
+    return numGetQueueUserAclsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getContainerReportFailedRetrieved() {
+    return numGetContainerReportFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getContainersFailedRetrieved() {
+    return numGetContainersFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getListReservationsFailedRetrieved() {
+    return numListReservationsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getGetResourceTypeInfoRetrieved() {
+    return numGetResourceTypeInfo.value();
+  }
+
+  @VisibleForTesting
+  public int getFailApplicationAttemptFailedRetrieved() {
+    return numFailAppAttemptFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getUpdateApplicationPriorityFailedRetrieved() {
+    return numUpdateAppPriorityFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getUpdateApplicationTimeoutsFailedRetrieved() {
+    return numUpdateAppTimeoutsFailedRetrieved.value();
+  }
+
+  @VisibleForTesting
+  public int getSignalToContainerFailedRetrieved() {
+    return numSignalToContainerFailedRetrieved.value();
+  }
+
+  public int getQueueInfoFailedRetrieved() {
+    return numGetQueueInfoFailedRetrieved.value();
+  }
+
+  public int getMoveApplicationAcrossQueuesFailedRetrieved() {
+    return numMoveApplicationAcrossQueuesFailedRetrieved.value();
+  }
+
+  public int getResourceProfilesFailedRetrieved() {
+    return numGetResourceProfilesFailedRetrieved.value();
+  }
+
+  public int getResourceProfileFailedRetrieved() {
+    return numGetResourceProfileFailedRetrieved.value();
+  }
+
+  public int getAttributesToNodesFailedRetrieved() {
+    return numGetAttributesToNodesFailedRetrieved.value();
+  }
+
+  public int getClusterNodeAttributesFailedRetrieved() {
+    return numGetClusterNodeAttributesFailedRetrieved.value();
+  }
+
+  public int getNodesToAttributesFailedRetrieved() {
+    return numGetNodesToAttributesFailedRetrieved.value();
+  }
+
+  public int getNewReservationFailedRetrieved() {
+    return numGetNewReservationFailedRetrieved.value();
+  }
+
+  public int getSubmitReservationFailedRetrieved() {
+    return numSubmitReservationFailedRetrieved.value();
+  }
+
+  public int getUpdateReservationFailedRetrieved() {
+    return numUpdateReservationFailedRetrieved.value();
+  }
+
+  public int getDeleteReservationFailedRetrieved() {
+    return numDeleteReservationFailedRetrieved.value();
+  }
+
+  public int getListReservationFailedRetrieved() {
+    return numListReservationFailedRetrieved.value();
+  }
+
+  public int getAppActivitiesFailedRetrieved() {
+    return numGetAppActivitiesFailedRetrieved.value();
+  }
+
+  public int getAppStatisticsFailedRetrieved() {
+    return numGetAppStatisticsFailedRetrieved.value();
+  }
+
+  public int getAppPriorityFailedRetrieved() {
+    return numGetAppPriorityFailedRetrieved.value();
+  }
+
+  public int getAppQueueFailedRetrieved() {
+    return numGetAppQueueFailedRetrieved.value();
+  }
+
+  public int getUpdateAppQueueFailedRetrieved() {
+    return numUpdateAppQueueFailedRetrieved.value();
+  }
+
+  public int getAppTimeoutFailedRetrieved() {
+    return numGetAppTimeoutFailedRetrieved.value();
+  }
+
+  public int getAppTimeoutsFailedRetrieved() {
+    return numGetAppTimeoutsFailedRetrieved.value();
+  }
+
+
+  public int getRefreshQueuesFailedRetrieved() {
+    return numRefreshQueuesFailedRetrieved.value();
+  }
+
+  public int getRMNodeLabelsFailedRetrieved() {
+    return numGetRMNodeLabelsFailedRetrieved.value();
+  }
+
+  public int getCheckUserAccessToQueueFailedRetrieved() {
+    return numCheckUserAccessToQueueFailedRetrieved.value();
+  }
+
+  public int getDelegationTokenFailedRetrieved() {
+    return numGetDelegationTokenFailedRetrieved.value();
+  }
+
+  public int getRenewDelegationTokenFailedRetrieved() {
+    return numRenewDelegationTokenFailedRetrieved.value();
+  }
+
+  public int getCancelDelegationTokenFailedRetrieved() {
+    return numCancelDelegationTokenFailedRetrieved.value();
+  }
+
+  public void succeededAppsCreated(long duration) {
+    totalSucceededAppsCreated.add(duration);
+    getNewApplicationLatency.add(duration);
+  }
+
+  public void succeededAppsSubmitted(long duration) {
+    totalSucceededAppsSubmitted.add(duration);
+    submitApplicationLatency.add(duration);
+  }
+
+  public void succeededAppsKilled(long duration) {
+    totalSucceededAppsKilled.add(duration);
+    killApplicationLatency.add(duration);
+  }
+
+  public void succeededAppsRetrieved(long duration) {
+    totalSucceededAppsRetrieved.add(duration);
+    getApplicationReportLatency.add(duration);
+  }
+
+  public void succeededMultipleAppsRetrieved(long duration) {
+    totalSucceededMultipleAppsRetrieved.add(duration);
+    getApplicationsReportLatency.add(duration);
+  }
+
+  public void succeededAppAttemptsRetrieved(long duration) {
+    totalSucceededAppAttemptsRetrieved.add(duration);
+    getApplicationAttemptsLatency.add(duration);
+  }
+
+  public void succeededGetClusterMetricsRetrieved(long duration) {
+    totalSucceededGetClusterMetricsRetrieved.add(duration);
+    getClusterMetricsLatency.add(duration);
+  }
+
+  public void succeededGetClusterNodesRetrieved(long duration) {
+    totalSucceededGetClusterNodesRetrieved.add(duration);
+    getClusterNodesLatency.add(duration);
+  }
+
+  public void succeededGetNodeToLabelsRetrieved(long duration) {
+    totalSucceededGetNodeToLabelsRetrieved.add(duration);
+    getNodeToLabelsLatency.add(duration);
+  }
+
+  public void succeededGetLabelsToNodesRetrieved(long duration) {
+    totalSucceededGetLabelsToNodesRetrieved.add(duration);
+    getLabelToNodesLatency.add(duration);
+  }
+
+  public void succeededGetClusterNodeLabelsRetrieved(long duration) {
+    totalSucceededGetClusterNodeLabelsRetrieved.add(duration);
+    getClusterNodeLabelsLatency.add(duration);
+  }
+
+  public void succeededAppAttemptReportRetrieved(long duration) {
+    totalSucceededAppAttemptReportRetrieved.add(duration);
+    getApplicationAttemptReportLatency.add(duration);
+  }
+
+  public void succeededGetQueueUserAclsRetrieved(long duration) {
+    totalSucceededGetQueueUserAclsRetrieved.add(duration);
+    getQueueUserAclsLatency.add(duration);
+  }
+
+  public void succeededGetContainerReportRetrieved(long duration) {
+    totalSucceededGetContainerReportRetrieved.add(duration);
+    getContainerReportLatency.add(duration);
+  }
+
+  public void succeededGetContainersRetrieved(long duration) {
+    totalSucceededGetContainersRetrieved.add(duration);
+    getContainerLatency.add(duration);
+  }
+
+  public void succeededListReservationsRetrieved(long duration) {
+    totalSucceededListReservationsRetrieved.add(duration);
+    listReservationsLatency.add(duration);
+  }
+
+  public void succeededGetResourceTypeInfoRetrieved(long duration) {
+    totalSucceededGetResourceTypeInfoRetrieved.add(duration);
+    listResourceTypeInfoLatency.add(duration);
+  }
+
+  public void succeededFailAppAttemptRetrieved(long duration) {
+    totalSucceededFailAppAttemptRetrieved.add(duration);
+    failAppAttemptLatency.add(duration);
+  }
+
+  public void succeededUpdateAppPriorityRetrieved(long duration) {
+    totalSucceededUpdateAppPriorityRetrieved.add(duration);
+    updateAppPriorityLatency.add(duration);
+  }
+
+  public void succeededUpdateAppTimeoutsRetrieved(long duration) {
+    totalSucceededUpdateAppTimeoutsRetrieved.add(duration);
+    updateAppTimeoutsLatency.add(duration);
+  }
+
+  public void succeededSignalToContainerRetrieved(long duration) {
+    totalSucceededSignalToContainerRetrieved.add(duration);
+    signalToContainerLatency.add(duration);
+  }
+
+  public void succeededGetQueueInfoRetrieved(long duration) {
+    totalSucceededGetQueueInfoRetrieved.add(duration);
+    getQueueInfoLatency.add(duration);
+  }
+
+  public void succeededMoveApplicationAcrossQueuesRetrieved(long duration) {
+    totalSucceededMoveApplicationAcrossQueuesRetrieved.add(duration);
+    moveApplicationAcrossQueuesLatency.add(duration);
+  }
+
+  public void succeededGetResourceProfilesRetrieved(long duration) {
+    totalSucceededGetResourceProfilesRetrieved.add(duration);
+    getResourceProfilesLatency.add(duration);
+  }
+
+  public void succeededGetResourceProfileRetrieved(long duration) {
+    totalSucceededGetResourceProfileRetrieved.add(duration);
+    getResourceProfileLatency.add(duration);
+  }
+
+  public void succeededGetAttributesToNodesRetrieved(long duration) {
+    totalSucceededGetAttributesToNodesRetrieved.add(duration);
+    getAttributesToNodesLatency.add(duration);
+  }
+
+  public void succeededGetClusterNodeAttributesRetrieved(long duration) {
+    totalSucceededGetClusterNodeAttributesRetrieved.add(duration);
+    getClusterNodeAttributesLatency.add(duration);
+  }
+
+  public void succeededGetNodesToAttributesRetrieved(long duration) {
+    totalSucceededGetNodesToAttributesRetrieved.add(duration);
+    getNodesToAttributesLatency.add(duration);
+  }
+
+  public void succeededGetNewReservationRetrieved(long duration) {
+    totalSucceededGetNewReservationRetrieved.add(duration);
+    getNewReservationLatency.add(duration);
+  }
+
+  public void succeededSubmitReservationRetrieved(long duration) {
+    totalSucceededSubmitReservationRetrieved.add(duration);
+    submitReservationLatency.add(duration);
+  }
+
+  public void succeededUpdateReservationRetrieved(long duration) {
+    totalSucceededUpdateReservationRetrieved.add(duration);
+    updateReservationLatency.add(duration);
+  }
+
+  public void succeededDeleteReservationRetrieved(long duration) {
+    totalSucceededDeleteReservationRetrieved.add(duration);
+    deleteReservationLatency.add(duration);
+  }
+
+  public void succeededListReservationRetrieved(long duration) {
+    totalSucceededListReservationRetrieved.add(duration);
+    listReservationLatency.add(duration);
+  }
+
+  public void succeededGetAppActivitiesRetrieved(long duration) {
+    totalSucceededGetAppActivitiesRetrieved.add(duration);
+    getAppActivitiesLatency.add(duration);
+  }
+
+  public void succeededGetAppStatisticsRetrieved(long duration) {
+    totalSucceededGetAppStatisticsRetrieved.add(duration);
+    getAppStatisticsLatency.add(duration);
+  }
+
+  public void succeededGetAppPriorityRetrieved(long duration) {
+    totalSucceededGetAppPriorityRetrieved.add(duration);
+    getAppPriorityLatency.add(duration);
+  }
+
+  public void succeededGetAppQueueRetrieved(long duration) {
+    totalSucceededGetAppQueueRetrieved.add(duration);
+    getAppQueueLatency.add(duration);
+  }
+
+  public void succeededUpdateAppQueueRetrieved(long duration) {
+    totalSucceededUpdateAppQueueRetrieved.add(duration);
+    getUpdateQueueLatency.add(duration);
+  }
+
+  public void succeededGetAppTimeoutRetrieved(long duration) {
+    totalSucceededGetAppTimeoutRetrieved.add(duration);
+    getAppTimeoutLatency.add(duration);
+  }
+
+  public void succeededGetAppTimeoutsRetrieved(long duration) {
+    totalSucceededGetAppTimeoutsRetrieved.add(duration);
+    getAppTimeoutsLatency.add(duration);
+  }
+
+  public void succeededRefreshQueuesRetrieved(long duration) {
+    totalSucceededRefreshQueuesRetrieved.add(duration);
+    getRefreshQueuesLatency.add(duration);
+  }
+
+  public void succeededGetRMNodeLabelsRetrieved(long duration) {
+    totalSucceededGetRMNodeLabelsRetrieved.add(duration);
+    getRMNodeLabelsLatency.add(duration);
+  }
+
+  public void succeededCheckUserAccessToQueueRetrieved(long duration) {
+    totalSucceededCheckUserAccessToQueueRetrieved.add(duration);
+    checkUserAccessToQueueLatency.add(duration);
+  }
+
+  public void succeededGetDelegationTokenRetrieved(long duration) {
+    totalSucceededGetDelegationTokenRetrieved.add(duration);
+    getDelegationTokenLatency.add(duration);
+  }
+
+  public void succeededRenewDelegationTokenRetrieved(long duration) {
+    totalSucceededRenewDelegationTokenRetrieved.add(duration);
+    renewDelegationTokenLatency.add(duration);
+  }
+
+  public void succeededCancelDelegationTokenRetrieved(long duration) {
+    totalSucceededCancelDelegationTokenRetrieved.add(duration);
+    cancelDelegationTokenLatency.add(duration);
+  }
+
+  public void incrAppsFailedCreated() {
+    numAppsFailedCreated.incr();
+  }
+
+  public void incrAppsFailedSubmitted() {
+    numAppsFailedSubmitted.incr();
+  }
+
+  public void incrAppsFailedKilled() {
+    numAppsFailedKilled.incr();
+  }
+
+  public void incrAppsFailedRetrieved() {
+    numAppsFailedRetrieved.incr();
+  }
+
+  public void incrMultipleAppsFailedRetrieved() {
+    numMultipleAppsFailedRetrieved.incr();
+  }
+
+  public void incrAppAttemptsFailedRetrieved() {
+    numAppAttemptsFailedRetrieved.incr();
+  }
+
+  public void incrGetClusterMetricsFailedRetrieved() {
+    numGetClusterMetricsFailedRetrieved.incr();
+  }
+
+  public void incrClusterNodesFailedRetrieved() {
+    numGetClusterNodesFailedRetrieved.incr();
+  }
+
+  public void incrNodeToLabelsFailedRetrieved() {
+    numGetNodeToLabelsFailedRetrieved.incr();
+  }
+
+  public void incrLabelsToNodesFailedRetrieved() {
+    numGetLabelsToNodesFailedRetrieved.incr();
+  }
+
+  public void incrClusterNodeLabelsFailedRetrieved() {
+    numGetClusterNodeLabelsFailedRetrieved.incr();
+  }
+
+  public void incrAppAttemptReportFailedRetrieved() {
+    numAppAttemptReportFailedRetrieved.incr();
+  }
+
+  public void incrQueueUserAclsFailedRetrieved() {
+    numGetQueueUserAclsFailedRetrieved.incr();
+  }
+
+  public void incrGetContainerReportFailedRetrieved() {
+    numGetContainerReportFailedRetrieved.incr();
+  }
+
+  public void incrGetContainersFailedRetrieved() {
+    numGetContainersFailedRetrieved.incr();
+  }
+
+  public void incrListReservationsFailedRetrieved() {
+    numListReservationsFailedRetrieved.incr();
+  }
+
+  public void incrResourceTypeInfoFailedRetrieved() {
+    numGetResourceTypeInfo.incr();
+  }
+
+  public void incrFailAppAttemptFailedRetrieved() {
+    numFailAppAttemptFailedRetrieved.incr();
+  }
+
+  public void incrUpdateAppPriorityFailedRetrieved() {
+    numUpdateAppPriorityFailedRetrieved.incr();
+  }
+
+  public void incrUpdateApplicationTimeoutsRetrieved() {
+    numUpdateAppTimeoutsFailedRetrieved.incr();
+  }
+
+  public void incrSignalToContainerFailedRetrieved() {
+    numSignalToContainerFailedRetrieved.incr();
+  }
+
+  public void incrGetQueueInfoFailedRetrieved() {
+    numGetQueueInfoFailedRetrieved.incr();
+  }
+
+  public void incrMoveApplicationAcrossQueuesFailedRetrieved() {
+    numMoveApplicationAcrossQueuesFailedRetrieved.incr();
+  }
+
+  public void incrGetResourceProfilesFailedRetrieved() {
+    numGetResourceProfilesFailedRetrieved.incr();
+  }
+
+  public void incrGetResourceProfileFailedRetrieved() {
+    numGetResourceProfileFailedRetrieved.incr();
+  }
+
+  public void incrGetAttributesToNodesFailedRetrieved() {
+    numGetAttributesToNodesFailedRetrieved.incr();
+  }
+
+  public void incrGetClusterNodeAttributesFailedRetrieved() {
+    numGetClusterNodeAttributesFailedRetrieved.incr();
+  }
+
+  public void incrGetNodesToAttributesFailedRetrieved() {
+    numGetNodesToAttributesFailedRetrieved.incr();
+  }
+
+  public void incrGetNewReservationFailedRetrieved() {
+    numGetNewReservationFailedRetrieved.incr();
+  }
+
+  public void incrSubmitReservationFailedRetrieved() {
+    numSubmitReservationFailedRetrieved.incr();
+  }
+
+  public void incrUpdateReservationFailedRetrieved() {
+    numUpdateReservationFailedRetrieved.incr();
+  }
+
+  public void incrDeleteReservationFailedRetrieved() {
+    numDeleteReservationFailedRetrieved.incr();
+  }
+
+  public void incrListReservationFailedRetrieved() {
+    numListReservationFailedRetrieved.incr();
+  }
+
+  public void incrGetAppActivitiesFailedRetrieved() {
+    numGetAppActivitiesFailedRetrieved.incr();
+  }
+
+  public void incrGetAppStatisticsFailedRetrieved() {
+    numGetAppStatisticsFailedRetrieved.incr();
+  }
+
+  public void incrGetAppPriorityFailedRetrieved() {
+    numGetAppPriorityFailedRetrieved.incr();
+  }
+
+  public void incrGetAppQueueFailedRetrieved() {
+    numGetAppQueueFailedRetrieved.incr();
+  }
+
+  public void incrUpdateAppQueueFailedRetrieved() {
+    numUpdateAppQueueFailedRetrieved.incr();
+  }
+
+  public void incrGetAppTimeoutFailedRetrieved() {
+    numGetAppTimeoutFailedRetrieved.incr();
+  }
+
+  public void incrGetAppTimeoutsFailedRetrieved() {
+    numGetAppTimeoutsFailedRetrieved.incr();
+  }
+
+  public void incrRefreshQueuesFailedRetrieved() {
+    numRefreshQueuesFailedRetrieved.incr();
+  }
+
+  public void incrGetRMNodeLabelsFailedRetrieved() {
+    numGetRMNodeLabelsFailedRetrieved.incr();
+  }
+
+  public void incrCheckUserAccessToQueueFailedRetrieved() {
+    numCheckUserAccessToQueueFailedRetrieved.incr();
+  }
+
+  public void incrGetDelegationTokenFailedRetrieved() {
+    numGetDelegationTokenFailedRetrieved.incr();
+  }
+
+  public void incrRenewDelegationTokenFailedRetrieved() {
+    numRenewDelegationTokenFailedRetrieved.incr();
+  }
+
+  public void incrCancelDelegationTokenFailedRetrieved() {
+    numCancelDelegationTokenFailedRetrieved.incr();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
index cc96da62331b9..8c880f25ddb76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
@@ -18,13 +18,29 @@
 
 package org.apache.hadoop.yarn.server.router;
 
+import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.EnumSet;
+import java.io.IOException;
+
 /**
  * Common utility methods used by the Router server.
  *
@@ -33,6 +49,14 @@
 @Unstable
 public final class RouterServerUtil {
 
+  private static final String APPLICATION_ID_PREFIX = "application_";
+
+  private static final String APP_ATTEMPT_ID_PREFIX = "appattempt_";
+
+  private static final String CONTAINER_PREFIX = "container_";
+
+  private static final String EPOCH_PREFIX = "e";
+
   /** Disable constructor. */
   private RouterServerUtil() {
   }
@@ -40,6 +64,28 @@ private RouterServerUtil() {
   public static final Logger LOG =
       LoggerFactory.getLogger(RouterServerUtil.class);
 
+  /**
+   * Throws an exception due to an error.
+   *
+   * @param t the throwable raised in the called class.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws YarnException on failure
+   */
+  @Public
+  @Unstable
+  public static void logAndThrowException(Throwable t, String errMsgFormat, Object... args)
+      throws YarnException {
+    String msg = String.format(errMsgFormat, args);
+    if (t != null) {
+      LOG.error(msg, t);
+      throw new YarnException(msg, t);
+    } else {
+      LOG.error(msg);
+      throw new YarnException(msg);
+    }
+  }
+
   /**
    * Throws an exception due to an error.
    *
@@ -60,4 +106,414 @@ public static void logAndThrowException(String errMsg, Throwable t)
     }
   }
 
+  public static <R> R createRequestInterceptorChain(Configuration conf, String pipeLineClassName,
+      String interceptorClassName, Class<R> clazz) {
+
+    List<String> interceptorClassNames = getInterceptorClassNames(conf,
+        pipeLineClassName, interceptorClassName);
+
+    R pipeline = null;
+    R current = null;
+
+    for (String className : interceptorClassNames) {
+      try {
+        Class<?> interceptorClass = conf.getClassByName(className);
+        if (clazz.isAssignableFrom(interceptorClass)) {
+          Object interceptorInstance = ReflectionUtils.newInstance(interceptorClass, conf);
+          if (pipeline == null) {
+            pipeline = clazz.cast(interceptorInstance);
+            current = clazz.cast(interceptorInstance);
+            continue;
+          } else {
+            Method method = clazz.getMethod("setNextInterceptor", clazz);
+            method.invoke(current, interceptorInstance);
+            current = clazz.cast(interceptorInstance);
+          }
+        } else {
+          LOG.error("Class: {} not instance of {}.", className, clazz.getCanonicalName());
+          throw new YarnRuntimeException("Class: " + className + " not instance of "
+              + clazz.getCanonicalName());
+        }
+      } catch (ClassNotFoundException e) {
+        LOG.error("Could not instantiate RequestInterceptor: {}", className, e);
+        throw new YarnRuntimeException("Could not instantiate RequestInterceptor: " + className, e);
+      } catch (InvocationTargetException e) {
+        LOG.error("RequestInterceptor {} call setNextInterceptor error.", className, e);
+        throw new YarnRuntimeException("RequestInterceptor " + className
+            + " call setNextInterceptor error.", e);
+      } catch (NoSuchMethodException e) {
+        LOG.error("RequestInterceptor {} does not contain the method setNextInterceptor.",
+            className);
+        throw new YarnRuntimeException("RequestInterceptor " + className +
+            " does not contain the method setNextInterceptor.", e);
+      } catch (IllegalAccessException e) {
+        LOG.error("RequestInterceptor {} call the method setNextInterceptor " +
+            "does not have access.", className);
+        throw new YarnRuntimeException("RequestInterceptor "
+            + className + " call the method setNextInterceptor does not have access.", e);
+      }
+    }
+
+    if (pipeline == null) {
+      throw new YarnRuntimeException(
+          "RequestInterceptor pipeline is not configured in the system.");
+    }
+
+    return pipeline;
+  }
+
+  private static List<String> getInterceptorClassNames(Configuration conf,
+      String pipeLineClass, String interceptorClass) {
+    String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass);
+    List<String> interceptorClassNames = new ArrayList<>();
+    Collection<String> tempList =
+        StringUtils.getStringCollection(configuredInterceptorClassNames);
+    for (String item : tempList) {
+      interceptorClassNames.add(item.trim());
+    }
+    return interceptorClassNames;
+  }
+
+  /**
+   * Throws an IOException due to an error.
+   *
+   * @param errMsg the error message
+   * @param t the throwable raised in the called class.
+   * @throws IOException on failure
+   */
+  @Public
+  @Unstable
+  public static void logAndThrowIOException(String errMsg, Throwable t)
+      throws IOException {
+    if (t != null) {
+      LOG.error(errMsg, t);
+      throw new IOException(errMsg, t);
+    } else {
+      LOG.error(errMsg);
+      throw new IOException(errMsg);
+    }
+  }
+
+  /**
+   * Throws an IOException due to an error.
+   *
+   * @param t the throwable raised in the called class.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws IOException on failure
+   */
+  @Public
+  @Unstable
+  public static void logAndThrowIOException(Throwable t, String errMsgFormat, Object... args)
+      throws IOException {
+    String msg = String.format(errMsgFormat, args);
+    if (t != null) {
+      LOG.error(msg, t);
+      throw new IOException(msg, t);
+    } else {
+      LOG.error(msg);
+      throw new IOException(msg);
+    }
+  }
+
+  /**
+   * Throws an RunTimeException due to an error.
+   *
+   * @param errMsg the error message
+   * @param t the throwable raised in the called class.
+   * @throws RuntimeException on failure
+   */
+  @Public
+  @Unstable
+  public static void logAndThrowRunTimeException(String errMsg, Throwable t)
+      throws RuntimeException {
+    if (t != null) {
+      LOG.error(errMsg, t);
+      throw new RuntimeException(errMsg, t);
+    } else {
+      LOG.error(errMsg);
+      throw new RuntimeException(errMsg);
+    }
+  }
+
+  /**
+   * Throws an RunTimeException due to an error.
+   *
+   * @param t the throwable raised in the called class.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @throws RuntimeException on failure
+   */
+  @Public
+  @Unstable
+  public static void logAndThrowRunTimeException(Throwable t, String errMsgFormat, Object... args)
+      throws RuntimeException {
+    String msg = String.format(errMsgFormat, args);
+    if (t != null) {
+      LOG.error(msg, t);
+      throw new RuntimeException(msg, t);
+    } else {
+      LOG.error(msg);
+      throw new RuntimeException(msg);
+    }
+  }
+
+  /**
+   * Throws an RunTimeException due to an error.
+   *
+   * @param t the throwable raised in the called class.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @return RuntimeException
+   */
+  @Public
+  @Unstable
+  public static RuntimeException logAndReturnRunTimeException(
+      Throwable t, String errMsgFormat, Object... args) {
+    String msg = String.format(errMsgFormat, args);
+    if (t != null) {
+      LOG.error(msg, t);
+      return new RuntimeException(msg, t);
+    } else {
+      LOG.error(msg);
+      return new RuntimeException(msg);
+    }
+  }
+
+  /**
+   * Throws an RunTimeException due to an error.
+   *
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @return RuntimeException
+   */
+  @Public
+  @Unstable
+  public static RuntimeException logAndReturnRunTimeException(
+      String errMsgFormat, Object... args) {
+    return logAndReturnRunTimeException(null, errMsgFormat, args);
+  }
+
+  /**
+   * Throws an YarnRuntimeException due to an error.
+   *
+   * @param t the throwable raised in the called class.
+   * @param errMsgFormat the error message format string.
+   * @param args referenced by the format specifiers in the format string.
+   * @return YarnRuntimeException
+   */
+  @Public
+  @Unstable
+  public static YarnRuntimeException logAndReturnYarnRunTimeException(
+      Throwable t, String errMsgFormat, Object... args) {
+    String msg = String.format(errMsgFormat, args);
+    if (t != null) {
+      LOG.error(msg, t);
+      return new YarnRuntimeException(msg, t);
+    } else {
+      LOG.error(msg);
+      return new YarnRuntimeException(msg);
+    }
+  }
+
+  /**
+   * Check applicationId is accurate.
+   *
+   * We need to ensure that applicationId cannot be empty and
+   * can be converted to ApplicationId object normally.
+   *
+   * @param applicationId applicationId of type string
+   * @throws IllegalArgumentException If the format of the applicationId is not accurate,
+   * an IllegalArgumentException needs to be thrown.
+   */
+  @Public
+  @Unstable
+  public static void validateApplicationId(String applicationId)
+      throws IllegalArgumentException {
+
+    // Make Sure applicationId is not empty.
+    if (applicationId == null || applicationId.isEmpty()) {
+      throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
+    }
+
+    // Make sure the prefix information of applicationId is accurate.
+    if (!applicationId.startsWith(APPLICATION_ID_PREFIX)) {
+      throw new IllegalArgumentException("Invalid ApplicationId prefix: "
+          + applicationId + ". The valid ApplicationId should start with prefix application");
+    }
+
+    // Check the split position of the string.
+    int pos1 = APPLICATION_ID_PREFIX.length() - 1;
+    int pos2 = applicationId.indexOf('_', pos1 + 1);
+    if (pos2 < 0) {
+      throw new IllegalArgumentException("Invalid ApplicationId: " + applicationId);
+    }
+
+    // Confirm that the parsed rmId and appId are numeric types.
+    String rmId = applicationId.substring(pos1 + 1, pos2);
+    String appId = applicationId.substring(pos2 + 1);
+    if(!NumberUtils.isDigits(rmId) || !NumberUtils.isDigits(appId)){
+      throw new IllegalArgumentException("Invalid ApplicationId: " + applicationId);
+    }
+  }
+
+  /**
+   * Check appAttemptId is accurate.
+   *
+   * We need to ensure that appAttemptId cannot be empty and
+   * can be converted to ApplicationAttemptId object normally.
+   *
+   * @param appAttemptId appAttemptId of type string.
+   * @throws IllegalArgumentException If the format of the appAttemptId is not accurate,
+   * an IllegalArgumentException needs to be thrown.
+   */
+  @Public
+  @Unstable
+  public static void validateApplicationAttemptId(String appAttemptId)
+      throws IllegalArgumentException {
+
+    // Make Sure appAttemptId is not empty.
+    if (appAttemptId == null || appAttemptId.isEmpty()) {
+      throw new IllegalArgumentException("Parameter error, the appAttemptId is empty or null.");
+    }
+
+    // Make sure the prefix information of appAttemptId is accurate.
+    if (!appAttemptId.startsWith(APP_ATTEMPT_ID_PREFIX)) {
+      throw new IllegalArgumentException("Invalid AppAttemptId prefix: " + appAttemptId);
+    }
+
+    // Check the split position of the string.
+    int pos1 = APP_ATTEMPT_ID_PREFIX.length() - 1;
+    int pos2 = appAttemptId.indexOf('_', pos1 + 1);
+    if (pos2 < 0) {
+      throw new IllegalArgumentException("Invalid AppAttemptId: " + appAttemptId);
+    }
+    int pos3 = appAttemptId.indexOf('_', pos2 + 1);
+    if (pos3 < 0) {
+      throw new IllegalArgumentException("Invalid AppAttemptId: " + appAttemptId);
+    }
+
+    // Confirm that the parsed rmId and appId and attemptId are numeric types.
+    String rmId = appAttemptId.substring(pos1 + 1, pos2);
+    String appId = appAttemptId.substring(pos2 + 1, pos3);
+    String attemptId = appAttemptId.substring(pos3 + 1);
+
+    if (!NumberUtils.isDigits(rmId) || !NumberUtils.isDigits(appId)
+        || !NumberUtils.isDigits(attemptId)) {
+      throw new IllegalArgumentException("Invalid AppAttemptId: " + appAttemptId);
+    }
+  }
+
+  /**
+   * Check containerId is accurate.
+   *
+   * We need to ensure that containerId cannot be empty and
+   * can be converted to ContainerId object normally.
+   *
+   * @param containerId containerId of type string.
+   * @throws IllegalArgumentException If the format of the appAttemptId is not accurate,
+   * an IllegalArgumentException needs to be thrown.
+   */
+  @Public
+  @Unstable
+  public static void validateContainerId(String containerId)
+      throws IllegalArgumentException {
+
+    // Make Sure containerId is not empty.
+    if (containerId == null || containerId.isEmpty()) {
+      throw new IllegalArgumentException("Parameter error, the containerId is empty or null.");
+    }
+
+    // Make sure the prefix information of containerId is accurate.
+    if (!containerId.startsWith(CONTAINER_PREFIX)) {
+      throw new IllegalArgumentException("Invalid ContainerId prefix: " + containerId);
+    }
+
+    // Check the split position of the string.
+    int pos1 = CONTAINER_PREFIX.length() - 1;
+
+    String epoch = "0";
+    if (containerId.regionMatches(pos1 + 1, EPOCH_PREFIX, 0, EPOCH_PREFIX.length())) {
+      int pos2 = containerId.indexOf('_', pos1 + 1);
+      if (pos2 < 0) {
+        throw new IllegalArgumentException("Invalid ContainerId: " + containerId);
+      }
+      String epochStr = containerId.substring(pos1 + 1 + EPOCH_PREFIX.length(), pos2);
+      epoch = epochStr;
+      // rewind the current position
+      pos1 = pos2;
+    }
+
+    int pos2 = containerId.indexOf('_', pos1 + 1);
+    if (pos2 < 0) {
+      throw new IllegalArgumentException("Invalid ContainerId: " + containerId);
+    }
+
+    int pos3 = containerId.indexOf('_', pos2 + 1);
+    if (pos3 < 0) {
+      throw new IllegalArgumentException("Invalid ContainerId: " + containerId);
+    }
+
+    int pos4 = containerId.indexOf('_', pos3 + 1);
+    if (pos4 < 0) {
+      throw new IllegalArgumentException("Invalid ContainerId: " + containerId);
+    }
+
+    // Confirm that the parsed appId and clusterTimestamp and attemptId and cid and epoch
+    // are numeric types.
+    String appId = containerId.substring(pos2 + 1, pos3);
+    String clusterTimestamp = containerId.substring(pos1 + 1, pos2);
+    String attemptId = containerId.substring(pos3 + 1, pos4);
+    String cid = containerId.substring(pos4 + 1);
+
+    if (!NumberUtils.isDigits(appId) || !NumberUtils.isDigits(clusterTimestamp)
+        || !NumberUtils.isDigits(attemptId) || !NumberUtils.isDigits(cid)
+        || !NumberUtils.isDigits(epoch)) {
+      throw new IllegalArgumentException("Invalid ContainerId: " + containerId);
+    }
+  }
+
+  public static boolean isAllowedDelegationTokenOp() throws IOException {
+    if (UserGroupInformation.isSecurityEnabled()) {
+      return EnumSet.of(UserGroupInformation.AuthenticationMethod.KERBEROS,
+          UserGroupInformation.AuthenticationMethod.KERBEROS_SSL,
+          UserGroupInformation.AuthenticationMethod.CERTIFICATE)
+          .contains(UserGroupInformation.getCurrentUser()
+          .getRealAuthenticationMethod());
+    } else {
+      return true;
+    }
+  }
+
+  public static String getRenewerForToken(Token<RMDelegationTokenIdentifier> token)
+      throws IOException {
+    UserGroupInformation user = UserGroupInformation.getCurrentUser();
+    UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+    // we can always renew our own tokens
+    return loginUser.getUserName().equals(user.getUserName())
+        ? token.decodeIdentifier().getRenewer().toString() : user.getShortUserName();
+  }
+
+  public static UserGroupInformation setupUser(final String userName) {
+    UserGroupInformation user = null;
+    try {
+      // If userName is empty, we will return UserGroupInformation.getCurrentUser.
+      // Do not create a proxy user if user name matches the user name on
+      // current UGI
+      if (userName == null || userName.trim().isEmpty()) {
+        user = UserGroupInformation.getCurrentUser();
+      } else if (UserGroupInformation.isSecurityEnabled()) {
+        user = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getLoginUser());
+      } else if (userName.equalsIgnoreCase(UserGroupInformation.getCurrentUser().getUserName())) {
+        user = UserGroupInformation.getCurrentUser();
+      } else {
+        user = UserGroupInformation.createProxyUser(userName,
+            UserGroupInformation.getCurrentUser());
+      }
+      return user;
+    } catch (IOException e) {
+      throw RouterServerUtil.logAndReturnYarnRunTimeException(e,
+          "Error while creating Router RMAdmin Service for user : %s.", user);
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java
index 01ba3bdcadf35..10ed71b6010a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java
@@ -18,18 +18,17 @@
 
 package org.apache.hadoop.yarn.server.router.clientrm;
 
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Implements the {@link ClientRequestInterceptor} interface and provides common
  * functionality which can can be used and/or extended by other concrete
- * intercepter classes.
+ * interceptor classes.
  *
  */
 public abstract class AbstractClientRequestInterceptor
@@ -44,6 +43,8 @@ public abstract class AbstractClientRequestInterceptor
   @SuppressWarnings("checkstyle:visibilitymodifier")
   protected UserGroupInformation user = null;
 
+  private RouterDelegationTokenSecretManager tokenSecretManager = null;
+
   /**
    * Sets the {@link ClientRequestInterceptor} in the chain.
    */
@@ -77,7 +78,7 @@ public Configuration getConf() {
    */
   @Override
   public void init(String userName) {
-    setupUser(userName);
+    this.user = RouterServerUtil.setupUser(userName);
     if (this.nextInterceptor != null) {
       this.nextInterceptor.init(userName);
     }
@@ -101,27 +102,13 @@ public ClientRequestInterceptor getNextInterceptor() {
     return this.nextInterceptor;
   }
 
-  private void setupUser(String userName) {
-
-    try {
-      // Do not create a proxy user if user name matches the user name on
-      // current UGI
-      if (userName.equalsIgnoreCase(
-          UserGroupInformation.getCurrentUser().getUserName())) {
-        user = UserGroupInformation.getCurrentUser();
-      } else {
-        user = UserGroupInformation.createProxyUser(userName,
-            UserGroupInformation.getCurrentUser());
-      }
-    } catch (IOException e) {
-      String message = "Error while creating Router ClientRM Service for user:";
-      if (user != null) {
-        message += ", user: " + user;
-      }
-
-      LOG.info(message);
-      throw new YarnRuntimeException(message, e);
-    }
+  @Override
+  public RouterDelegationTokenSecretManager getTokenSecretManager() {
+    return tokenSecretManager;
   }
 
+  @Override
+  public void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager) {
+    this.tokenSecretManager = tokenSecretManager;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java
index 2f8fb936345ca..6e19cbadf9d38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java
@@ -20,16 +20,17 @@
 
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
 
 /**
- * Defines the contract to be implemented by the request intercepter classes,
+ * Defines the contract to be implemented by the request interceptor classes,
  * that can be used to intercept and inspect messages sent from the client to
  * the resource manager.
  */
 public interface ClientRequestInterceptor
     extends ApplicationClientProtocol, Configurable {
   /**
-   * This method is called for initializing the intercepter. This is guaranteed
+   * This method is called for initializing the interceptor. This is guaranteed
    * to be called only once in the lifetime of this instance.
    *
    * @param user the name of the client
@@ -37,29 +38,43 @@ public interface ClientRequestInterceptor
   void init(String user);
 
   /**
-   * This method is called to release the resources held by the intercepter.
+   * This method is called to release the resources held by the interceptor.
    * This will be called when the application pipeline is being destroyed. The
    * concrete implementations should dispose the resources and forward the
-   * request to the next intercepter, if any.
+   * request to the next interceptor, if any.
    */
   void shutdown();
 
   /**
-   * Sets the next intercepter in the pipeline. The concrete implementation of
+   * Sets the next interceptor in the pipeline. The concrete implementation of
    * this interface should always pass the request to the nextInterceptor after
-   * inspecting the message. The last intercepter in the chain is responsible to
+   * inspecting the message. The last interceptor in the chain is responsible to
    * send the messages to the resource manager service and so the last
-   * intercepter will not receive this method call.
+   * interceptor will not receive this method call.
    *
    * @param nextInterceptor the ClientRequestInterceptor to set in the pipeline
    */
   void setNextInterceptor(ClientRequestInterceptor nextInterceptor);
 
   /**
-   * Returns the next intercepter in the chain.
+   * Returns the next interceptor in the chain.
    *
-   * @return the next intercepter in the chain
+   * @return the next interceptor in the chain
    */
   ClientRequestInterceptor getNextInterceptor();
 
+  /**
+   * Set RouterDelegationTokenSecretManager for specific interceptor to support Token operations,
+   * including create Token, update Token, and delete Token.
+   *
+   * @param tokenSecretManager Router DelegationTokenSecretManager
+   */
+  void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager);
+
+  /**
+   * Get RouterDelegationTokenSecretManager.
+   *
+   * @return Router DelegationTokenSecretManager.
+   */
+  RouterDelegationTokenSecretManager getTokenSecretManager();
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 5852292814102..a50ea5bc42352 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.yarn.server.router.clientrm;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.io.IOException;
 import java.lang.reflect.Method;
@@ -27,6 +30,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.TreeMap;
+import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -37,7 +41,6 @@
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.NotImplementedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -113,31 +116,48 @@
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.retry.FederationActionRetry;
 import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.router.RouterAuditLogger;
 import org.apache.hadoop.yarn.server.router.RouterMetrics;
 import org.apache.hadoop.yarn.server.router.RouterServerUtil;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.apache.hadoop.yarn.util.Records;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.classification.VisibleForTesting;
 
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_NEW_APP;
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP;
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_REPORT;
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.FORCE_KILL_APP;
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.TARGET_CLIENT_RM_SERVICE;
+import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UNKNOWN;
+
 /**
  * Extends the {@code AbstractRequestInterceptorClient} class and provides an
  * implementation for federation of YARN RM and scaling an application across
  * multiple YARN SubClusters. All the federation specific implementation is
- * encapsulated in this class. This is always the last intercepter in the chain.
+ * encapsulated in this class. This is always the last interceptor in the chain.
  */
 public class FederationClientInterceptor
     extends AbstractClientRequestInterceptor {
@@ -163,6 +183,7 @@ public class FederationClientInterceptor
   private ThreadPoolExecutor executorService;
   private final Clock clock = new MonotonicClock();
   private boolean returnPartialReport;
+  private long submitIntervalTime;
 
   @Override
   public void init(String userName) {
@@ -171,16 +192,20 @@ public void init(String userName) {
     federationFacade = FederationStateStoreFacade.getInstance();
     rand = new Random(System.currentTimeMillis());
 
+    int numMinThreads = getNumMinThreads(getConf());
+
+    int numMaxThreads = getNumMaxThreads(getConf());
+
+    long keepAliveTime = getConf().getTimeDuration(
+        YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME,
+        YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_KEEP_ALIVE_TIME, TimeUnit.SECONDS);
 
-    int numThreads = getConf().getInt(
-        YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
-        YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE);
     ThreadFactory threadFactory = new ThreadFactoryBuilder()
         .setNameFormat("RPC Router Client-" + userName + "-%d ").build();
 
-    BlockingQueue workQueue = new LinkedBlockingQueue<>();
-    this.executorService = new ThreadPoolExecutor(numThreads, numThreads,
-        0L, TimeUnit.MILLISECONDS, workQueue, threadFactory);
+    BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();
+    this.executorService = new ThreadPoolExecutor(numMinThreads, numMaxThreads,
+        keepAliveTime, TimeUnit.MILLISECONDS, workQueue, threadFactory);
 
     final Configuration conf = this.getConf();
 
@@ -191,12 +216,15 @@ public void init(String userName) {
       LOG.error(e.getMessage());
     }
 
-    numSubmitRetries =
-        conf.getInt(YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY,
-            YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY);
+    numSubmitRetries = conf.getInt(
+        YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY,
+        YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY);
 
-    clientRMProxies =
-        new ConcurrentHashMap<SubClusterId, ApplicationClientProtocol>();
+    submitIntervalTime = conf.getTimeDuration(
+        YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_INTERVAL_TIME,
+        YarnConfiguration.DEFAULT_CLIENTRM_SUBMIT_INTERVAL_TIME, TimeUnit.MILLISECONDS);
+
+    clientRMProxies = new ConcurrentHashMap<>();
     routerMetrics = RouterMetrics.getMetrics();
 
     returnPartialReport = conf.getBoolean(
@@ -223,35 +251,29 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster(
     ApplicationClientProtocol clientRMProxy = null;
     try {
       boolean serviceAuthEnabled = getConf().getBoolean(
-              CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
+          CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
       UserGroupInformation realUser = user;
       if (serviceAuthEnabled) {
         realUser = UserGroupInformation.createProxyUser(
-                user.getShortUserName(), UserGroupInformation.getLoginUser());
+            user.getShortUserName(), UserGroupInformation.getLoginUser());
       }
       clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(),
           ApplicationClientProtocol.class, subClusterId, realUser);
     } catch (Exception e) {
       RouterServerUtil.logAndThrowException(
-          "Unable to create the interface to reach the SubCluster "
-              + subClusterId,
-          e);
+          "Unable to create the interface to reach the SubCluster " + subClusterId, e);
     }
-
     clientRMProxies.put(subClusterId, clientRMProxy);
     return clientRMProxy;
   }
 
   private SubClusterId getRandomActiveSubCluster(
-      Map<SubClusterId, SubClusterInfo> activeSubclusters)
-      throws YarnException {
-
-    if (activeSubclusters == null || activeSubclusters.size() < 1) {
+      Map<SubClusterId, SubClusterInfo> activeSubClusters) throws YarnException {
+    if (activeSubClusters == null || activeSubClusters.isEmpty()) {
       RouterServerUtil.logAndThrowException(
           FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE, null);
     }
-    List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
-
+    List<SubClusterId> list = new ArrayList<>(activeSubClusters.keySet());
     return list.get(rand.nextInt(list.size()));
   }
 
@@ -276,44 +298,86 @@ private SubClusterId getRandomActiveSubCluster(
   public GetNewApplicationResponse getNewApplication(
       GetNewApplicationRequest request) throws YarnException, IOException {
 
-    long startTime = clock.getTime();
+    if (request == null) {
+      routerMetrics.incrAppsFailedCreated();
+      String errMsg = "Missing getNewApplication request.";
+      RouterAuditLogger.logFailure(user.getShortUserName(), GET_NEW_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, errMsg);
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
 
+    long startTime = clock.getTime();
     Map<SubClusterId, SubClusterInfo> subClustersActive =
         federationFacade.getSubClusters(true);
 
-    for (int i = 0; i < numSubmitRetries; ++i) {
-      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
-      LOG.debug(
-          "getNewApplication try #{} on SubCluster {}", i, subClusterId);
-      ApplicationClientProtocol clientRMProxy =
-          getClientRMProxyForSubCluster(subClusterId);
-      GetNewApplicationResponse response = null;
-      try {
-        response = clientRMProxy.getNewApplication(request);
-      } catch (Exception e) {
-        LOG.warn("Unable to create a new ApplicationId in SubCluster "
-            + subClusterId.getId(), e);
-      }
+    // Try calling the getNewApplication method
+    List<SubClusterId> blacklist = new ArrayList<>();
+    int activeSubClustersCount = federationFacade.getActiveSubClustersCount();
+    int actualRetryNums = Math.min(activeSubClustersCount, numSubmitRetries);
 
-      if (response != null) {
+    try {
+      GetNewApplicationResponse response =
+          ((FederationActionRetry<GetNewApplicationResponse>) (retryCount) ->
+          invokeGetNewApplication(subClustersActive, blacklist, request, retryCount)).
+          runWithRetries(actualRetryNums, submitIntervalTime);
 
+      if (response != null) {
         long stopTime = clock.getTime();
         routerMetrics.succeededAppsCreated(stopTime - startTime);
         return response;
-      } else {
-        // Empty response from the ResourceManager.
-        // Blacklist this subcluster for this request.
-        subClustersActive.remove(subClusterId);
       }
-
+    } catch (Exception e) {
+      routerMetrics.incrAppsFailedCreated();
+      RouterServerUtil.logAndThrowException(e.getMessage(), e);
     }
 
     routerMetrics.incrAppsFailedCreated();
-    String errMsg = "Fail to create a new application.";
-    LOG.error(errMsg);
+    String errMsg = "Failed to create a new application.";
+    RouterAuditLogger.logFailure(user.getShortUserName(), GET_NEW_APP, UNKNOWN,
+        TARGET_CLIENT_RM_SERVICE, errMsg);
     throw new YarnException(errMsg);
   }
 
+  /**
+   * Invoke GetNewApplication to different subClusters.
+   *
+   * @param subClustersActive Active SubClusters
+   * @param blackList Blacklist avoid repeated calls to unavailable subCluster.
+   * @param request getNewApplicationRequest.
+   * @param retryCount number of retries.
+   * @return Get NewApplicationResponse response, If the response is empty, the request fails,
+   * if the response is not empty, the request is successful.
+   * @throws YarnException yarn exception.
+   * @throws IOException io error.
+   */
+  private GetNewApplicationResponse invokeGetNewApplication(
+      Map<SubClusterId, SubClusterInfo> subClustersActive,
+      List<SubClusterId> blackList, GetNewApplicationRequest request, int retryCount)
+      throws YarnException, IOException {
+    SubClusterId subClusterId =
+        federationFacade.getRandomActiveSubCluster(subClustersActive, blackList);
+    LOG.info("getNewApplication try #{} on SubCluster {}.", retryCount, subClusterId);
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    try {
+      GetNewApplicationResponse response = clientRMProxy.getNewApplication(request);
+      if (response != null) {
+        RouterAuditLogger.logSuccess(user.getShortUserName(), GET_NEW_APP,
+            TARGET_CLIENT_RM_SERVICE, response.getApplicationId(), subClusterId);
+        return response;
+      }
+    } catch (Exception e) {
+      RouterAuditLogger.logFailure(user.getShortUserName(), GET_NEW_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, e.getMessage(), subClusterId);
+      LOG.warn("Unable to create a new ApplicationId in SubCluster {}.", subClusterId.getId(), e);
+      blackList.add(subClusterId);
+      throw e;
+    }
+    // If SubmitApplicationResponse is empty, the request fails.
+    String msg = String.format("Unable to create a new ApplicationId in SubCluster %s.",
+        subClusterId.getId());
+    throw new YarnException(msg);
+  }
+
   /**
    * Today, in YARN there are no checks of any applicationId submitted.
    *
@@ -383,95 +447,125 @@ public GetNewApplicationResponse getNewApplication(
   public SubmitApplicationResponse submitApplication(
       SubmitApplicationRequest request) throws YarnException, IOException {
 
-    long startTime = clock.getTime();
-
     if (request == null || request.getApplicationSubmissionContext() == null
-        || request.getApplicationSubmissionContext()
-            .getApplicationId() == null) {
+        || request.getApplicationSubmissionContext().getApplicationId() == null) {
       routerMetrics.incrAppsFailedSubmitted();
-      RouterServerUtil
-          .logAndThrowException("Missing submitApplication request or "
-              + "applicationSubmissionContex information.", null);
+      String errMsg =
+          "Missing submitApplication request or applicationSubmissionContext information.";
+      RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, errMsg);
+      RouterServerUtil.logAndThrowException(errMsg, null);
     }
 
+    long startTime = clock.getTime();
     ApplicationId applicationId =
         request.getApplicationSubmissionContext().getApplicationId();
+    List<SubClusterId> blacklist = new ArrayList<>();
 
-    List<SubClusterId> blacklist = new ArrayList<SubClusterId>();
-
-    for (int i = 0; i < numSubmitRetries; ++i) {
+    try {
 
-      SubClusterId subClusterId = policyFacade.getHomeSubcluster(
-          request.getApplicationSubmissionContext(), blacklist);
-      LOG.info("submitApplication appId" + applicationId + " try #" + i
-          + " on SubCluster " + subClusterId);
+      // We need to handle this situation,
+      // the user will provide us with an expected submitRetries,
+      // but if the number of Active SubClusters is less than this number at this time,
+      // we should provide a high number of retry according to the number of Active SubClusters.
+      int activeSubClustersCount = federationFacade.getActiveSubClustersCount();
+      int actualRetryNums = Math.min(activeSubClustersCount, numSubmitRetries);
 
-      ApplicationHomeSubCluster appHomeSubCluster =
-          ApplicationHomeSubCluster.newInstance(applicationId, subClusterId);
+      // Try calling the SubmitApplication method
+      SubmitApplicationResponse response =
+          ((FederationActionRetry<SubmitApplicationResponse>) (retryCount) ->
+          invokeSubmitApplication(blacklist, request, retryCount)).
+          runWithRetries(actualRetryNums, submitIntervalTime);
 
-      if (i == 0) {
-        try {
-          // persist the mapping of applicationId and the subClusterId which has
-          // been selected as its home
-          subClusterId =
-              federationFacade.addApplicationHomeSubCluster(appHomeSubCluster);
-        } catch (YarnException e) {
-          routerMetrics.incrAppsFailedSubmitted();
-          String message = "Unable to insert the ApplicationId " + applicationId
-              + " into the FederationStateStore";
-          RouterServerUtil.logAndThrowException(message, e);
-        }
-      } else {
-        try {
-          // update the mapping of applicationId and the home subClusterId to
-          // the new subClusterId we have selected
-          federationFacade.updateApplicationHomeSubCluster(appHomeSubCluster);
-        } catch (YarnException e) {
-          String message = "Unable to update the ApplicationId " + applicationId
-              + " into the FederationStateStore";
-          SubClusterId subClusterIdInStateStore =
-              federationFacade.getApplicationHomeSubCluster(applicationId);
-          if (subClusterId == subClusterIdInStateStore) {
-            LOG.info("Application " + applicationId
-                + " already submitted on SubCluster " + subClusterId);
-          } else {
-            routerMetrics.incrAppsFailedSubmitted();
-            RouterServerUtil.logAndThrowException(message, e);
-          }
-        }
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededAppsSubmitted(stopTime - startTime);
+        return response;
       }
 
-      ApplicationClientProtocol clientRMProxy =
-          getClientRMProxyForSubCluster(subClusterId);
+    } catch (Exception e) {
+      routerMetrics.incrAppsFailedSubmitted();
+      RouterServerUtil.logAndThrowException(e.getMessage(), e);
+    }
+
+    routerMetrics.incrAppsFailedSubmitted();
+    String msg = String.format("Application %s with appId %s failed to be submitted.",
+        request.getApplicationSubmissionContext().getApplicationName(), applicationId);
+    RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN,
+        TARGET_CLIENT_RM_SERVICE, msg, applicationId);
+    throw new YarnException(msg);
+  }
 
-      SubmitApplicationResponse response = null;
-      try {
-        response = clientRMProxy.submitApplication(request);
-      } catch (Exception e) {
-        LOG.warn("Unable to submit the application " + applicationId
-            + "to SubCluster " + subClusterId.getId(), e);
-      }
+  /**
+   * Invoke SubmitApplication to different subClusters.
+   *
+   * Step1. Select homeSubCluster for Application according to Policy.
+   *
+   * Step2. Query homeSubCluster according to ApplicationId,
+   * if homeSubCluster does not exist or first attempt(consider repeated submissions), write;
+   * if homeSubCluster exists, update.
+   *
+   * Step3. Find the clientRMProxy of the corresponding cluster according to homeSubCluster,
+   * and then call the SubmitApplication method.
+   *
+   * Step4. If SubmitApplicationResponse is empty, the request fails,
+   * if SubmitApplicationResponse is not empty, the request is successful.
+   *
+   * @param blackList Blacklist avoid repeated calls to unavailable subCluster.
+   * @param request submitApplicationRequest.
+   * @param retryCount number of retries.
+   * @return submitApplication response, If the response is empty, the request fails,
+   *      if the response is not empty, the request is successful.
+   * @throws YarnException yarn exception.
+   */
+  private SubmitApplicationResponse invokeSubmitApplication(
+      List<SubClusterId> blackList, SubmitApplicationRequest request, int retryCount)
+      throws YarnException, IOException {
+
+    // The request is not checked here,
+    // because the request has been checked before the method is called.
+    // We get applicationId and subClusterId from context.
+    ApplicationSubmissionContext appSubmissionContext = request.getApplicationSubmissionContext();
+    ApplicationId applicationId = appSubmissionContext.getApplicationId();
+    SubClusterId subClusterId = null;
 
+    try {
+
+      // Step1. Select homeSubCluster for Application according to Policy.
+      subClusterId = policyFacade.getHomeSubcluster(appSubmissionContext, blackList);
+      LOG.info("submitApplication appId {} try #{} on SubCluster {}.",
+          applicationId, retryCount, subClusterId);
+
+      // Step2. We Store the mapping relationship
+      // between Application and HomeSubCluster in stateStore.
+      federationFacade.addOrUpdateApplicationHomeSubCluster(
+          applicationId, subClusterId, retryCount);
+
+      // Step3. SubmitApplication to the subCluster
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      SubmitApplicationResponse response = clientRMProxy.submitApplication(request);
+
+      // Step4. if SubmitApplicationResponse is not empty, the request is successful.
       if (response != null) {
-        LOG.info("Application "
-            + request.getApplicationSubmissionContext().getApplicationName()
-            + " with appId " + applicationId + " submitted on " + subClusterId);
-        long stopTime = clock.getTime();
-        routerMetrics.succeededAppsSubmitted(stopTime - startTime);
+        LOG.info("Application {} submitted on subCluster {}.", applicationId, subClusterId);
+        RouterAuditLogger.logSuccess(user.getShortUserName(), SUBMIT_NEW_APP,
+            TARGET_CLIENT_RM_SERVICE, applicationId, subClusterId);
         return response;
-      } else {
-        // Empty response from the ResourceManager.
-        // Blacklist this subcluster for this request.
-        blacklist.add(subClusterId);
       }
+    } catch (Exception e) {
+      RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, e.getMessage(), applicationId, subClusterId);
+      LOG.warn("Unable to submitApplication appId {} try #{} on SubCluster {}.",
+          applicationId, retryCount, subClusterId, e);
+      if (subClusterId != null) {
+        blackList.add(subClusterId);
+      }
+      throw e;
     }
 
-    routerMetrics.incrAppsFailedSubmitted();
-    String errMsg = "Application "
-        + request.getApplicationSubmissionContext().getApplicationName()
-        + " with appId " + applicationId + " failed to be submitted.";
-    LOG.error(errMsg);
-    throw new YarnException(errMsg);
+    // If SubmitApplicationResponse is empty, the request fails.
+    String msg = String.format("Application %s failed to be submitted.", applicationId);
+    throw new YarnException(msg);
   }
 
   /**
@@ -494,13 +588,16 @@ public SubmitApplicationResponse submitApplication(
   public KillApplicationResponse forceKillApplication(
       KillApplicationRequest request) throws YarnException, IOException {
 
-    long startTime = clock.getTime();
-
     if (request == null || request.getApplicationId() == null) {
       routerMetrics.incrAppsFailedKilled();
-      RouterServerUtil.logAndThrowException(
-          "Missing forceKillApplication request or ApplicationId.", null);
+      String msg = "Missing forceKillApplication request or ApplicationId.";
+      RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, msg);
+      RouterServerUtil.logAndThrowException(msg, null);
     }
+
+    long startTime = clock.getTime();
+
     ApplicationId applicationId = request.getApplicationId();
     SubClusterId subClusterId = null;
 
@@ -509,8 +606,11 @@ public KillApplicationResponse forceKillApplication(
           .getApplicationHomeSubCluster(request.getApplicationId());
     } catch (YarnException e) {
       routerMetrics.incrAppsFailedKilled();
-      RouterServerUtil.logAndThrowException("Application " + applicationId
-          + " does not exist in FederationStateStore", e);
+      String msg =
+          String.format("Application %s does not exist in FederationStateStore.", applicationId);
+      RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, msg, applicationId);
+      RouterServerUtil.logAndThrowException(msg, e);
     }
 
     ApplicationClientProtocol clientRMProxy =
@@ -518,24 +618,25 @@ public KillApplicationResponse forceKillApplication(
 
     KillApplicationResponse response = null;
     try {
-      LOG.info("forceKillApplication " + applicationId + " on SubCluster "
-          + subClusterId);
+      LOG.info("forceKillApplication {} on SubCluster {}.", applicationId, subClusterId);
       response = clientRMProxy.forceKillApplication(request);
     } catch (Exception e) {
       routerMetrics.incrAppsFailedKilled();
-      LOG.error("Unable to kill the application report for "
-          + request.getApplicationId() + "to SubCluster "
-          + subClusterId.getId(), e);
-      throw e;
+      String msg = "Unable to kill the application report.";
+      RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, msg, applicationId, subClusterId);
+      RouterServerUtil.logAndThrowException(msg, e);
     }
 
     if (response == null) {
-      LOG.error("No response when attempting to kill the application "
-          + applicationId + " to SubCluster " + subClusterId.getId());
+      LOG.error("No response when attempting to kill the application {} to SubCluster {}.",
+          applicationId, subClusterId.getId());
     }
 
     long stopTime = clock.getTime();
     routerMetrics.succeededAppsKilled(stopTime - startTime);
+    RouterAuditLogger.logSuccess(user.getShortUserName(), FORCE_KILL_APP,
+        TARGET_CLIENT_RM_SERVICE, applicationId);
     return response;
   }
 
@@ -559,15 +660,15 @@ public KillApplicationResponse forceKillApplication(
   public GetApplicationReportResponse getApplicationReport(
       GetApplicationReportRequest request) throws YarnException, IOException {
 
-    long startTime = clock.getTime();
-
     if (request == null || request.getApplicationId() == null) {
       routerMetrics.incrAppsFailedRetrieved();
-      RouterServerUtil.logAndThrowException(
-          "Missing getApplicationReport request or applicationId information.",
-          null);
+      String errMsg = "Missing getApplicationReport request or applicationId information.";
+      RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, errMsg);
+      RouterServerUtil.logAndThrowException(errMsg, null);
     }
 
+    long startTime = clock.getTime();
     SubClusterId subClusterId = null;
 
     try {
@@ -575,33 +676,37 @@ public GetApplicationReportResponse getApplicationReport(
           .getApplicationHomeSubCluster(request.getApplicationId());
     } catch (YarnException e) {
       routerMetrics.incrAppsFailedRetrieved();
-      RouterServerUtil
-          .logAndThrowException("Application " + request.getApplicationId()
-              + " does not exist in FederationStateStore", e);
+      String errMsg = String.format("Application %s does not exist in FederationStateStore.",
+          request.getApplicationId());
+      RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, errMsg, request.getApplicationId());
+      RouterServerUtil.logAndThrowException(errMsg, e);
     }
 
     ApplicationClientProtocol clientRMProxy =
         getClientRMProxyForSubCluster(subClusterId);
-
     GetApplicationReportResponse response = null;
+
     try {
       response = clientRMProxy.getApplicationReport(request);
     } catch (Exception e) {
       routerMetrics.incrAppsFailedRetrieved();
-      LOG.error("Unable to get the application report for "
-          + request.getApplicationId() + "to SubCluster "
-          + subClusterId.getId(), e);
-      throw e;
+      String errMsg = String.format("Unable to get the application report for %s to SubCluster %s.",
+          request.getApplicationId(), subClusterId.getId());
+      RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN,
+          TARGET_CLIENT_RM_SERVICE, errMsg, request.getApplicationId(), subClusterId);
+      RouterServerUtil.logAndThrowException(errMsg, e);
     }
 
     if (response == null) {
       LOG.error("No response when attempting to retrieve the report of "
-          + "the application " + request.getApplicationId() + " to SubCluster "
-          + subClusterId.getId());
+          + "the application {} to SubCluster {}.",
+          request.getApplicationId(), subClusterId.getId());
     }
-
     long stopTime = clock.getTime();
     routerMetrics.succeededAppsRetrieved(stopTime - startTime);
+    RouterAuditLogger.logSuccess(user.getShortUserName(), GET_APP_REPORT,
+        TARGET_CLIENT_RM_SERVICE, request.getApplicationId());
     return response;
   }
 
@@ -616,7 +721,7 @@ public GetApplicationReportResponse getApplicationReport(
    * Router: the Client will timeout and resubmit the request.
    *
    * ResourceManager: the Router calls each Yarn RM in parallel. In case a
-   * Yarn RM fails, a single call will timeout. However the Router will
+   * Yarn RM fails, a single call will timeout. However, the Router will
    * merge the ApplicationReports it got, and provides a partial list to
    * the client.
    *
@@ -629,192 +734,471 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
       throws YarnException, IOException {
     if (request == null) {
       routerMetrics.incrMultipleAppsFailedRetrieved();
-      RouterServerUtil.logAndThrowException(
-          "Missing getApplications request.",
-          null);
+      RouterServerUtil.logAndThrowException("Missing getApplications request.", null);
     }
     long startTime = clock.getTime();
-    Map<SubClusterId, SubClusterInfo> subclusters =
-        federationFacade.getSubClusters(true);
     ClientMethod remoteMethod = new ClientMethod("getApplications",
         new Class[] {GetApplicationsRequest.class}, new Object[] {request});
-    Map<SubClusterId, GetApplicationsResponse> applications;
-
+    Collection<GetApplicationsResponse> applications = null;
     try {
-      applications = invokeConcurrent(subclusters.keySet(), remoteMethod,
-          GetApplicationsResponse.class);
-
+      applications = invokeConcurrent(remoteMethod, GetApplicationsResponse.class);
     } catch (Exception ex) {
       routerMetrics.incrMultipleAppsFailedRetrieved();
-      LOG.error("Unable to get applications due to exception.", ex);
-      throw ex;
+      RouterServerUtil.logAndThrowException("Unable to get applications due to exception.", ex);
     }
     long stopTime = clock.getTime();
     routerMetrics.succeededMultipleAppsRetrieved(stopTime - startTime);
     // Merge the Application Reports
-    return RouterYarnClientUtils.mergeApplications(applications.values(),
-        returnPartialReport);
+    return RouterYarnClientUtils.mergeApplications(applications, returnPartialReport);
   }
 
   @Override
   public GetClusterMetricsResponse getClusterMetrics(
       GetClusterMetricsRequest request) throws YarnException, IOException {
+    if (request == null) {
+      routerMetrics.incrGetClusterMetricsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getClusterMetrics request.", null);
+    }
     long startTime = clock.getTime();
-    Map<SubClusterId, SubClusterInfo> subclusters =
-        federationFacade.getSubClusters(true);
     ClientMethod remoteMethod = new ClientMethod("getClusterMetrics",
         new Class[] {GetClusterMetricsRequest.class}, new Object[] {request});
-    Map<SubClusterId, GetClusterMetricsResponse> clusterMetrics;
-
+    Collection<GetClusterMetricsResponse> clusterMetrics = null;
     try {
-      clusterMetrics = invokeConcurrent(subclusters.keySet(), remoteMethod,
-          GetClusterMetricsResponse.class);
-
+      clusterMetrics = invokeConcurrent(remoteMethod, GetClusterMetricsResponse.class);
     } catch (Exception ex) {
       routerMetrics.incrGetClusterMetricsFailedRetrieved();
-      LOG.error("Unable to get cluster metrics due to exception.", ex);
-      throw ex;
+      RouterServerUtil.logAndThrowException("Unable to get cluster metrics due to exception.", ex);
     }
-
     long stopTime = clock.getTime();
     routerMetrics.succeededGetClusterMetricsRetrieved(stopTime - startTime);
-    return RouterYarnClientUtils.merge(clusterMetrics.values());
-  }
-
-  <R> Map<SubClusterId, R> invokeConcurrent(ArrayList<SubClusterId> clusterIds,
-      ClientMethod request, Class<R> clazz) throws YarnException, IOException {
-    List<Callable<Object>> callables = new ArrayList<>();
-    List<Future<Object>> futures = new ArrayList<>();
-    Map<SubClusterId, IOException> exceptions = new TreeMap<>();
-    for (SubClusterId subClusterId : clusterIds) {
-      callables.add(new Callable<Object>() {
-        @Override
-        public Object call() throws Exception {
-          ApplicationClientProtocol protocol =
-              getClientRMProxyForSubCluster(subClusterId);
-          Method method = ApplicationClientProtocol.class
-              .getMethod(request.getMethodName(), request.getTypes());
-          return method.invoke(protocol, request.getParams());
-        }
+    return RouterYarnClientUtils.merge(clusterMetrics);
+  }
+
+  <R> Collection<R> invokeConcurrent(ClientMethod request, Class<R> clazz)
+      throws YarnException {
+
+    // Get Active SubClusters
+    Map<SubClusterId, SubClusterInfo> subClusterInfo = federationFacade.getSubClusters(true);
+    Collection<SubClusterId> subClusterIds = subClusterInfo.keySet();
+
+    List<Callable<Pair<SubClusterId, Object>>> callables = new ArrayList<>();
+    List<Future<Pair<SubClusterId, Object>>> futures = new ArrayList<>();
+    Map<SubClusterId, Exception> exceptions = new TreeMap<>();
+
+    // Generate parallel Callable tasks
+    for (SubClusterId subClusterId : subClusterIds) {
+      callables.add(() -> {
+        ApplicationClientProtocol protocol = getClientRMProxyForSubCluster(subClusterId);
+        String methodName = request.getMethodName();
+        Class<?>[] types = request.getTypes();
+        Object[] params = request.getParams();
+        Method method = ApplicationClientProtocol.class.getMethod(methodName, types);
+        Object result = method.invoke(protocol, params);
+        return Pair.of(subClusterId, result);
       });
     }
+
+    // Get results from multiple threads
     Map<SubClusterId, R> results = new TreeMap<>();
     try {
       futures.addAll(executorService.invokeAll(callables));
-      for (int i = 0; i < futures.size(); i++) {
-        SubClusterId subClusterId = clusterIds.get(i);
+      futures.stream().forEach(future -> {
+        SubClusterId subClusterId = null;
         try {
-          Future<Object> future = futures.get(i);
-          Object result = future.get();
+          Pair<SubClusterId, Object> pair = future.get();
+          subClusterId = pair.getKey();
+          Object result = pair.getValue();
           results.put(subClusterId, clazz.cast(result));
-        } catch (ExecutionException ex) {
-          Throwable cause = ex.getCause();
-          LOG.debug("Cannot execute {} on {}: {}", request.getMethodName(),
+        } catch (InterruptedException | ExecutionException e) {
+          Throwable cause = e.getCause();
+          LOG.error("Cannot execute {} on {}: {}", request.getMethodName(),
               subClusterId.getId(), cause.getMessage());
-          IOException ioe;
-          if (cause instanceof IOException) {
-            ioe = (IOException) cause;
-          } else if (cause instanceof YarnException) {
-            throw (YarnException) cause;
-          } else {
-            ioe = new IOException(
-                "Unhandled exception while calling " + request.getMethodName()
-                    + ": " + cause.getMessage(), cause);
-          }
-          // Store the exceptions
-          exceptions.put(subClusterId, ioe);
-        }
-      }
-      if (results.isEmpty() && !clusterIds.isEmpty()) {
-        SubClusterId subClusterId = clusterIds.get(0);
-        IOException ioe = exceptions.get(subClusterId);
-        if (ioe != null) {
-          throw ioe;
+          exceptions.put(subClusterId, e);
         }
-      }
+      });
     } catch (InterruptedException e) {
-      throw new YarnException(e);
+      throw new YarnException("invokeConcurrent Failed.", e);
+    }
+
+    // All sub-clusters return results to be considered successful,
+    // otherwise an exception will be thrown.
+    if (exceptions != null && !exceptions.isEmpty()) {
+      Set<SubClusterId> subClusterIdSets = exceptions.keySet();
+      throw new YarnException("invokeConcurrent Failed, An exception occurred in subClusterIds = " +
+          StringUtils.join(subClusterIdSets, ","));
     }
-    return results;
-  }
 
-  <R> Map<SubClusterId, R> invokeConcurrent(Collection<SubClusterId> clusterIds,
-      ClientMethod request, Class<R> clazz) throws YarnException, IOException {
-    ArrayList<SubClusterId> clusterIdList = new ArrayList<>(clusterIds);
-    return invokeConcurrent(clusterIdList, request, clazz);
+    // return result
+    return results.values();
   }
 
   @Override
   public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrClusterNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getClusterNodes request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getClusterNodes",
+        new Class[]{GetClusterNodesRequest.class}, new Object[]{request});
+    try {
+      Collection<GetClusterNodesResponse> clusterNodes =
+          invokeConcurrent(remoteMethod, GetClusterNodesResponse.class);
+      long stopTime = clock.getTime();
+      routerMetrics.succeededGetClusterNodesRetrieved(stopTime - startTime);
+      return RouterYarnClientUtils.mergeClusterNodesResponse(clusterNodes);
+    } catch (Exception ex) {
+      routerMetrics.incrClusterNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get cluster nodes due to exception.", ex);
+    }
+    throw new YarnException("Unable to get cluster nodes.");
   }
 
   @Override
   public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getQueueName() == null) {
+      routerMetrics.incrGetQueueInfoFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getQueueInfo request or queueName.", null);
+    }
+
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getQueueInfo",
+        new Class[]{GetQueueInfoRequest.class}, new Object[]{request});
+    Collection<GetQueueInfoResponse> queues = null;
+    try {
+      queues = invokeConcurrent(remoteMethod, GetQueueInfoResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetQueueInfoFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get queue [" +
+          request.getQueueName() + "] to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetQueueInfoRetrieved(stopTime - startTime);
+    // Merge the GetQueueInfoResponse
+    return RouterYarnClientUtils.mergeQueues(queues);
   }
 
   @Override
   public GetQueueUserAclsInfoResponse getQueueUserAcls(
       GetQueueUserAclsInfoRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if(request == null){
+      routerMetrics.incrQueueUserAclsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getQueueUserAcls request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getQueueUserAcls",
+        new Class[] {GetQueueUserAclsInfoRequest.class}, new Object[] {request});
+    Collection<GetQueueUserAclsInfoResponse> queueUserAcls = null;
+    try {
+      queueUserAcls = invokeConcurrent(remoteMethod, GetQueueUserAclsInfoResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrQueueUserAclsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get queue user Acls due to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetQueueUserAclsRetrieved(stopTime - startTime);
+    // Merge the QueueUserAclsInfoResponse
+    return RouterYarnClientUtils.mergeQueueUserAcls(queueUserAcls);
   }
 
   @Override
   public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
       MoveApplicationAcrossQueuesRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationId() == null || request.getTargetQueue() == null) {
+      routerMetrics.incrMoveApplicationAcrossQueuesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing moveApplicationAcrossQueues request or " +
+          "applicationId or target queue.", null);
+    }
+
+    long startTime = clock.getTime();
+    SubClusterId subClusterId = null;
+
+    ApplicationId applicationId = request.getApplicationId();
+    try {
+      subClusterId = federationFacade
+          .getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException e) {
+      routerMetrics.incrMoveApplicationAcrossQueuesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " +
+          applicationId + " does not exist in FederationStateStore.", e);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    MoveApplicationAcrossQueuesResponse response = null;
+    try {
+      response = clientRMProxy.moveApplicationAcrossQueues(request);
+    } catch (Exception e) {
+      routerMetrics.incrMoveApplicationAcrossQueuesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to moveApplicationAcrossQueues for " +
+          applicationId + " to SubCluster " + subClusterId.getId(), e);
+    }
+
+    if (response == null) {
+      LOG.error("No response when moveApplicationAcrossQueues "
+           + "the applicationId {} to Queue {} In SubCluster {}.",
+           request.getApplicationId(), request.getTargetQueue(), subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededMoveApplicationAcrossQueuesRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public GetNewReservationResponse getNewReservation(
       GetNewReservationRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null) {
+      routerMetrics.incrGetNewReservationFailedRetrieved();
+      String errMsg = "Missing getNewReservation request.";
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
+
+    long startTime = clock.getTime();
+    Map<SubClusterId, SubClusterInfo> subClustersActive = federationFacade.getSubClusters(true);
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
+      LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId);
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      try {
+        GetNewReservationResponse response = clientRMProxy.getNewReservation(request);
+        if (response != null) {
+          long stopTime = clock.getTime();
+          routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime);
+          return response;
+        }
+      } catch (Exception e) {
+        LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e);
+        subClustersActive.remove(subClusterId);
+      }
+    }
+
+    routerMetrics.incrGetNewReservationFailedRetrieved();
+    String errMsg = "Failed to create a new reservation.";
+    throw new YarnException(errMsg);
   }
 
   @Override
   public ReservationSubmissionResponse submitReservation(
       ReservationSubmissionRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getReservationId() == null
+        || request.getReservationDefinition() == null || request.getQueue() == null) {
+      routerMetrics.incrSubmitReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing submitReservation request or reservationId " +
+          "or reservation definition or queue.", null);
+    }
+
+    long startTime = clock.getTime();
+    ReservationId reservationId = request.getReservationId();
+
+    for (int i = 0; i < numSubmitRetries; i++) {
+      try {
+        // First, Get SubClusterId according to specific strategy.
+        SubClusterId subClusterId = policyFacade.getReservationHomeSubCluster(request);
+        LOG.info("submitReservation ReservationId {} try #{} on SubCluster {}.",
+            reservationId, i, subClusterId);
+        ReservationHomeSubCluster reservationHomeSubCluster =
+            ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+
+        // Second, determine whether the current ReservationId has a corresponding subCluster.
+        // If it does not exist, add it. If it exists, update it.
+        Boolean exists = existsReservationHomeSubCluster(reservationId);
+
+        // We may encounter the situation of repeated submission of Reservation,
+        // at this time we should try to use the reservation that has been allocated
+        // !exists indicates that the reservation does not exist and needs to be added
+        // i==0, mainly to consider repeated submissions,
+        // so the first time to apply for reservation, try to use the original reservation
+        if (!exists || i == 0) {
+          addReservationHomeSubCluster(reservationId, reservationHomeSubCluster);
+        } else {
+          updateReservationHomeSubCluster(subClusterId, reservationId, reservationHomeSubCluster);
+        }
+
+        // Third, Submit a Reservation request to the subCluster
+        ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+        ReservationSubmissionResponse response = clientRMProxy.submitReservation(request);
+        if (response != null) {
+          LOG.info("Reservation {} submitted on subCluster {}.", reservationId, subClusterId);
+          long stopTime = clock.getTime();
+          routerMetrics.succeededSubmitReservationRetrieved(stopTime - startTime);
+          return response;
+        }
+      } catch (Exception e) {
+        LOG.warn("Unable to submit(try #{}) the Reservation {}.", i, reservationId, e);
+      }
+    }
+
+    routerMetrics.incrSubmitReservationFailedRetrieved();
+    String msg = String.format("Reservation %s failed to be submitted.", reservationId);
+    throw new YarnException(msg);
   }
 
   @Override
   public ReservationListResponse listReservations(
       ReservationListRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getReservationId() == null) {
+      routerMetrics.incrListReservationsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing listReservations request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("listReservations",
+        new Class[] {ReservationListRequest.class}, new Object[] {request});
+    Collection<ReservationListResponse> listResponses = null;
+    try {
+      listResponses = invokeConcurrent(remoteMethod, ReservationListResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrListReservationsFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Unable to list reservations node due to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededListReservationsRetrieved(stopTime - startTime);
+    // Merge the ReservationListResponse
+    return RouterYarnClientUtils.mergeReservationsList(listResponses);
   }
 
   @Override
   public ReservationUpdateResponse updateReservation(
       ReservationUpdateRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getReservationId() == null
+        || request.getReservationDefinition() == null) {
+      routerMetrics.incrUpdateReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing updateReservation request or reservationId or reservation definition.", null);
+    }
+
+    long startTime = clock.getTime();
+    ReservationId reservationId = request.getReservationId();
+    SubClusterId subClusterId = getReservationHomeSubCluster(reservationId);
+
+    try {
+      ApplicationClientProtocol client = getClientRMProxyForSubCluster(subClusterId);
+      ReservationUpdateResponse response = client.updateReservation(request);
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededUpdateReservationRetrieved(stopTime - startTime);
+        return response;
+      }
+    } catch (Exception ex) {
+      routerMetrics.incrUpdateReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Unable to reservation update due to exception.", ex);
+    }
+
+    routerMetrics.incrUpdateReservationFailedRetrieved();
+    String msg = String.format("Reservation %s failed to be update.", reservationId);
+    throw new YarnException(msg);
   }
 
   @Override
   public ReservationDeleteResponse deleteReservation(
       ReservationDeleteRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getReservationId() == null) {
+      routerMetrics.incrDeleteReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing deleteReservation request or reservationId.", null);
+    }
+
+    long startTime = clock.getTime();
+    ReservationId reservationId = request.getReservationId();
+    SubClusterId subClusterId = getReservationHomeSubCluster(reservationId);
+
+    try {
+      ApplicationClientProtocol client = getClientRMProxyForSubCluster(subClusterId);
+      ReservationDeleteResponse response = client.deleteReservation(request);
+      if (response != null) {
+        federationFacade.deleteReservationHomeSubCluster(reservationId);
+        long stopTime = clock.getTime();
+        routerMetrics.succeededDeleteReservationRetrieved(stopTime - startTime);
+        return response;
+      }
+    } catch (Exception ex) {
+      routerMetrics.incrUpdateReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Unable to reservation delete due to exception.", ex);
+    }
+
+    routerMetrics.incrDeleteReservationFailedRetrieved();
+    String msg = String.format("Reservation %s failed to be delete.", reservationId);
+    throw new YarnException(msg);
   }
 
   @Override
   public GetNodesToLabelsResponse getNodeToLabels(
       GetNodesToLabelsRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrNodeToLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getNodesToLabels request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getNodeToLabels",
+         new Class[] {GetNodesToLabelsRequest.class}, new Object[] {request});
+    Collection<GetNodesToLabelsResponse> clusterNodes = null;
+    try {
+      clusterNodes = invokeConcurrent(remoteMethod, GetNodesToLabelsResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrNodeToLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get node label due to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetNodeToLabelsRetrieved(stopTime - startTime);
+    // Merge the NodesToLabelsResponse
+    return RouterYarnClientUtils.mergeNodesToLabelsResponse(clusterNodes);
   }
 
   @Override
   public GetLabelsToNodesResponse getLabelsToNodes(
       GetLabelsToNodesRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getLabelsToNodes request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getLabelsToNodes",
+         new Class[] {GetLabelsToNodesRequest.class}, new Object[] {request});
+    Collection<GetLabelsToNodesResponse> labelNodes = null;
+    try {
+      labelNodes = invokeConcurrent(remoteMethod, GetLabelsToNodesResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get label node due to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime);
+    // Merge the LabelsToNodesResponse
+    return RouterYarnClientUtils.mergeLabelsToNodes(labelNodes);
   }
 
   @Override
   public GetClusterNodeLabelsResponse getClusterNodeLabels(
       GetClusterNodeLabelsRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrClusterNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getClusterNodeLabels request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getClusterNodeLabels",
+         new Class[] {GetClusterNodeLabelsRequest.class}, new Object[] {request});
+    Collection<GetClusterNodeLabelsResponse> nodeLabels = null;
+    try {
+      nodeLabels = invokeConcurrent(remoteMethod, GetClusterNodeLabelsResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrClusterNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get cluster nodeLabels due to exception.",
+          ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetClusterNodeLabelsRetrieved(stopTime - startTime);
+    // Merge the ClusterNodeLabelsResponse
+    return RouterYarnClientUtils.mergeClusterNodeLabelsResponse(nodeLabels);
   }
 
   /**
@@ -838,138 +1222,526 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport(
       GetApplicationAttemptReportRequest request)
       throws YarnException, IOException {
 
-    long startTime = clock.getTime();
-
     if (request == null || request.getApplicationAttemptId() == null
             || request.getApplicationAttemptId().getApplicationId() == null) {
-      routerMetrics.incrAppAttemptsFailedRetrieved();
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
       RouterServerUtil.logAndThrowException(
-              "Missing getApplicationAttemptReport " +
-                      "request or applicationId " +
-                      "or applicationAttemptId information.",
-              null);
+          "Missing getApplicationAttemptReport request or applicationId " +
+          "or applicationAttemptId information.", null);
     }
 
+    long startTime = clock.getTime();
     SubClusterId subClusterId = null;
-
+    ApplicationId applicationId = request.getApplicationAttemptId().getApplicationId();
     try {
-      subClusterId = federationFacade
-              .getApplicationHomeSubCluster(
-                      request.getApplicationAttemptId().getApplicationId());
+      subClusterId = getApplicationHomeSubCluster(applicationId);
     } catch (YarnException e) {
-      routerMetrics.incrAppAttemptsFailedRetrieved();
-      RouterServerUtil
-              .logAndThrowException("ApplicationAttempt " +
-                      request.getApplicationAttemptId() +
-                      "belongs to Application " +
-                      request.getApplicationAttemptId().getApplicationId() +
-                      " does not exist in FederationStateStore", e);
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
+      RouterServerUtil.logAndThrowException("ApplicationAttempt " +
+          request.getApplicationAttemptId() + " belongs to Application " +
+          request.getApplicationAttemptId().getApplicationId() +
+          " does not exist in FederationStateStore.", e);
     }
 
     ApplicationClientProtocol clientRMProxy =
-            getClientRMProxyForSubCluster(subClusterId);
+        getClientRMProxyForSubCluster(subClusterId);
 
     GetApplicationAttemptReportResponse response = null;
     try {
       response = clientRMProxy.getApplicationAttemptReport(request);
     } catch (Exception e) {
-      routerMetrics.incrAppAttemptsFailedRetrieved();
-      LOG.error("Unable to get the applicationAttempt report for "
-              + request.getApplicationAttemptId() + "to SubCluster "
-              + subClusterId.getId(), e);
-      throw e;
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
+      String msg = String.format(
+          "Unable to get the applicationAttempt report for %s to SubCluster %s.",
+          request.getApplicationAttemptId(), subClusterId.getId());
+      RouterServerUtil.logAndThrowException(msg, e);
     }
 
     if (response == null) {
       LOG.error("No response when attempting to retrieve the report of "
-              + "the applicationAttempt "
-              + request.getApplicationAttemptId() + " to SubCluster "
-              + subClusterId.getId());
+          + "the applicationAttempt {} to SubCluster {}.",
+          request.getApplicationAttemptId(), subClusterId.getId());
     }
 
     long stopTime = clock.getTime();
-    routerMetrics.succeededAppAttemptsRetrieved(stopTime - startTime);
+    routerMetrics.succeededAppAttemptReportRetrieved(stopTime - startTime);
     return response;
   }
 
   @Override
   public GetApplicationAttemptsResponse getApplicationAttempts(
       GetApplicationAttemptsRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationId() == null) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getApplicationAttempts " +
+          "request or application id.", null);
+    }
+
+    long startTime = clock.getTime();
+    ApplicationId applicationId = request.getApplicationId();
+    SubClusterId subClusterId = null;
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException ex) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " + applicationId +
+          " does not exist in FederationStateStore.", ex);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    GetApplicationAttemptsResponse response = null;
+    try {
+      response = clientRMProxy.getApplicationAttempts(request);
+    } catch (Exception ex) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get the application attempts for " +
+          applicationId + " from SubCluster " + subClusterId.getId(), ex);
+    }
+
+    if (response == null) {
+      LOG.error("No response when attempting to retrieve the attempts list of " +
+           "the application = {} to SubCluster = {}.", applicationId,
+           subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededAppAttemptsRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public GetContainerReportResponse getContainerReport(
       GetContainerReportRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if(request == null || request.getContainerId() == null){
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getContainerReport request " +
+          "or containerId", null);
+    }
+
+    long startTime = clock.getTime();
+    ApplicationId applicationId = request.getContainerId().
+        getApplicationAttemptId().getApplicationId();
+    SubClusterId subClusterId = null;
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException ex) {
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " + applicationId +
+          " does not exist in FederationStateStore.", ex);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    GetContainerReportResponse response = null;
+
+    try {
+      response = clientRMProxy.getContainerReport(request);
+    } catch (Exception ex) {
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      LOG.error("Unable to get the container report for {} from SubCluster {}.",
+          applicationId, subClusterId.getId(), ex);
+    }
+
+    if (response == null) {
+      LOG.error("No response when attempting to retrieve the container report of " +
+           "the ContainerId = {} From SubCluster = {}.", request.getContainerId(),
+           subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetContainerReportRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public GetContainersResponse getContainers(GetContainersRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationAttemptId() == null) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing getContainers request or ApplicationAttemptId.", null);
+    }
+
+    long startTime = clock.getTime();
+    ApplicationId applicationId = request.getApplicationAttemptId().getApplicationId();
+    SubClusterId subClusterId = null;
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException ex) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " + applicationId +
+          " does not exist in FederationStateStore.", ex);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    GetContainersResponse response = null;
+
+    try {
+      response = clientRMProxy.getContainers(request);
+    } catch (Exception ex) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get the containers for " +
+          applicationId + " from SubCluster " + subClusterId.getId(), ex);
+    }
+
+    if (response == null) {
+      LOG.error("No response when attempting to retrieve the container report of " +
+          "the ApplicationId = {} From SubCluster = {}.", applicationId,
+          subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetContainersRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public GetDelegationTokenResponse getDelegationToken(
       GetDelegationTokenRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getRenewer() == null) {
+      routerMetrics.incrGetDelegationTokenFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing getDelegationToken request or Renewer.", null);
+    }
+
+    try {
+      // Verify that the connection is kerberos authenticated
+      if (!RouterServerUtil.isAllowedDelegationTokenOp()) {
+        routerMetrics.incrGetDelegationTokenFailedRetrieved();
+        throw new IOException(
+            "Delegation Token can be issued only with kerberos authentication.");
+      }
+
+      long startTime = clock.getTime();
+      UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+      Text owner = new Text(ugi.getUserName());
+      Text realUser = null;
+      if (ugi.getRealUser() != null) {
+        realUser = new Text(ugi.getRealUser().getUserName());
+      }
+
+      RMDelegationTokenIdentifier tokenIdentifier =
+          new RMDelegationTokenIdentifier(owner, new Text(request.getRenewer()), realUser);
+      Token<RMDelegationTokenIdentifier> realRMDToken =
+          new Token<>(tokenIdentifier, this.getTokenSecretManager());
+
+      org.apache.hadoop.yarn.api.records.Token routerRMDTToken =
+          BuilderUtils.newDelegationToken(realRMDToken.getIdentifier(),
+              realRMDToken.getKind().toString(),
+              realRMDToken.getPassword(), realRMDToken.getService().toString());
+
+      long stopTime = clock.getTime();
+      routerMetrics.succeededGetDelegationTokenRetrieved((stopTime - startTime));
+      return GetDelegationTokenResponse.newInstance(routerRMDTToken);
+    } catch(IOException e) {
+      routerMetrics.incrGetDelegationTokenFailedRetrieved();
+      throw new YarnException(e);
+    }
   }
 
   @Override
   public RenewDelegationTokenResponse renewDelegationToken(
       RenewDelegationTokenRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+
+      if (!RouterServerUtil.isAllowedDelegationTokenOp()) {
+        routerMetrics.incrRenewDelegationTokenFailedRetrieved();
+        throw new IOException(
+            "Delegation Token can be renewed only with kerberos authentication");
+      }
+
+      long startTime = clock.getTime();
+      org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
+      Token<RMDelegationTokenIdentifier> token = new Token<>(
+          protoToken.getIdentifier().array(), protoToken.getPassword().array(),
+          new Text(protoToken.getKind()), new Text(protoToken.getService()));
+      String user = RouterServerUtil.getRenewerForToken(token);
+      long nextExpTime = this.getTokenSecretManager().renewToken(token, user);
+      RenewDelegationTokenResponse renewResponse =
+          Records.newRecord(RenewDelegationTokenResponse.class);
+      renewResponse.setNextExpirationTime(nextExpTime);
+      long stopTime = clock.getTime();
+      routerMetrics.succeededRenewDelegationTokenRetrieved((stopTime - startTime));
+      return renewResponse;
+
+    } catch (IOException e) {
+      routerMetrics.incrRenewDelegationTokenFailedRetrieved();
+      throw new YarnException(e);
+    }
   }
 
   @Override
   public CancelDelegationTokenResponse cancelDelegationToken(
       CancelDelegationTokenRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      if (!RouterServerUtil.isAllowedDelegationTokenOp()) {
+        routerMetrics.incrCancelDelegationTokenFailedRetrieved();
+        throw new IOException(
+            "Delegation Token can be cancelled only with kerberos authentication");
+      }
+
+      long startTime = clock.getTime();
+      org.apache.hadoop.yarn.api.records.Token protoToken = request.getDelegationToken();
+      Token<RMDelegationTokenIdentifier> token = new Token<>(
+          protoToken.getIdentifier().array(), protoToken.getPassword().array(),
+          new Text(protoToken.getKind()), new Text(protoToken.getService()));
+      String user = UserGroupInformation.getCurrentUser().getUserName();
+      this.getTokenSecretManager().cancelToken(token, user);
+      long stopTime = clock.getTime();
+      routerMetrics.succeededCancelDelegationTokenRetrieved((stopTime - startTime));
+      return Records.newRecord(CancelDelegationTokenResponse.class);
+    } catch (IOException e) {
+      routerMetrics.incrCancelDelegationTokenFailedRetrieved();
+      throw new YarnException(e);
+    }
   }
 
   @Override
   public FailApplicationAttemptResponse failApplicationAttempt(
       FailApplicationAttemptRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationAttemptId() == null
+          || request.getApplicationAttemptId().getApplicationId() == null) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing failApplicationAttempt request or applicationId " +
+          "or applicationAttemptId information.", null);
+    }
+    long startTime = clock.getTime();
+    SubClusterId subClusterId = null;
+    ApplicationId applicationId = request.getApplicationAttemptId().getApplicationId();
+
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException e) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException("ApplicationAttempt " +
+          request.getApplicationAttemptId() + " belongs to Application " +
+          request.getApplicationAttemptId().getApplicationId() +
+          " does not exist in FederationStateStore.", e);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    FailApplicationAttemptResponse response = null;
+    try {
+      response = clientRMProxy.failApplicationAttempt(request);
+    } catch (Exception e) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get the applicationAttempt report for " +
+          request.getApplicationAttemptId() + " to SubCluster " + subClusterId.getId(), e);
+    }
+
+    if (response == null) {
+      LOG.error("No response when attempting to retrieve the report of " +
+          "the applicationAttempt {} to SubCluster {}.",
+          request.getApplicationAttemptId(), subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededFailAppAttemptRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public UpdateApplicationPriorityResponse updateApplicationPriority(
       UpdateApplicationPriorityRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationId() == null
+            || request.getApplicationPriority() == null) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing updateApplicationPriority request or applicationId " +
+          "or applicationPriority information.", null);
+    }
+
+    long startTime = clock.getTime();
+    SubClusterId subClusterId = null;
+    ApplicationId applicationId = request.getApplicationId();
+
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException e) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " +
+          request.getApplicationId() + " does not exist in FederationStateStore.", e);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    UpdateApplicationPriorityResponse response = null;
+    try {
+      response = clientRMProxy.updateApplicationPriority(request);
+    } catch (Exception e) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to update application priority for " +
+          request.getApplicationId() + " to SubCluster " + subClusterId.getId(), e);
+    }
+
+    if (response == null) {
+      LOG.error("No response when update application priority of " +
+           "the applicationId {} to SubCluster {}.",
+           applicationId, subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededUpdateAppPriorityRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public SignalContainerResponse signalToContainer(
       SignalContainerRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getContainerId() == null
+            || request.getCommand() == null) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing signalToContainer request or containerId " +
+          "or command information.", null);
+    }
+
+    long startTime = clock.getTime();
+    SubClusterId subClusterId = null;
+    ApplicationId applicationId =
+        request.getContainerId().getApplicationAttemptId().getApplicationId();
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException ex) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " + applicationId +
+          " does not exist in FederationStateStore.", ex);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    SignalContainerResponse response = null;
+    try {
+      response = clientRMProxy.signalToContainer(request);
+    } catch (Exception ex) {
+      RouterServerUtil.logAndThrowException("Unable to signal to container for " +
+          applicationId + " from SubCluster " + subClusterId.getId(), ex);
+    }
+
+    if (response == null) {
+      LOG.error("No response when signal to container of " +
+          "the applicationId {} to SubCluster {}.", applicationId, subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededSignalToContainerRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public UpdateApplicationTimeoutsResponse updateApplicationTimeouts(
       UpdateApplicationTimeoutsRequest request)
       throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getApplicationId() == null
+            || request.getApplicationTimeouts() == null) {
+      routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing updateApplicationTimeouts request or applicationId " +
+          "or applicationTimeouts information.", null);
+    }
+
+    long startTime = clock.getTime();
+    SubClusterId subClusterId = null;
+    ApplicationId applicationId = request.getApplicationId();
+    try {
+      subClusterId = getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException e) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Application " +
+          request.getApplicationId() +
+          " does not exist in FederationStateStore.", e);
+    }
+
+    ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+    UpdateApplicationTimeoutsResponse response = null;
+    try {
+      response = clientRMProxy.updateApplicationTimeouts(request);
+    } catch (Exception e) {
+      routerMetrics.incrFailAppAttemptFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to update application timeout for " +
+          request.getApplicationId() + " to SubCluster " + subClusterId.getId(), e);
+    }
+
+    if (response == null) {
+      LOG.error("No response when update application timeout of " +
+          "the applicationId {} to SubCluster {}.",
+          applicationId, subClusterId.getId());
+    }
+
+    long stopTime = clock.getTime();
+    routerMetrics.succeededUpdateAppTimeoutsRetrieved(stopTime - startTime);
+    return response;
   }
 
   @Override
   public GetAllResourceProfilesResponse getResourceProfiles(
       GetAllResourceProfilesRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrGetResourceProfilesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getResourceProfiles request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getResourceProfiles",
+        new Class[] {GetAllResourceProfilesRequest.class}, new Object[] {request});
+    Collection<GetAllResourceProfilesResponse> resourceProfiles = null;
+    try {
+      resourceProfiles = invokeConcurrent(remoteMethod, GetAllResourceProfilesResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetResourceProfilesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get resource profiles due to exception.",
+          ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetResourceProfilesRetrieved(stopTime - startTime);
+    return RouterYarnClientUtils.mergeClusterResourceProfilesResponse(resourceProfiles);
   }
 
   @Override
   public GetResourceProfileResponse getResourceProfile(
       GetResourceProfileRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getProfileName() == null) {
+      routerMetrics.incrGetResourceProfileFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getResourceProfile request or profileName.",
+          null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getResourceProfile",
+        new Class[] {GetResourceProfileRequest.class}, new Object[] {request});
+    Collection<GetResourceProfileResponse> resourceProfile = null;
+    try {
+      resourceProfile = invokeConcurrent(remoteMethod, GetResourceProfileResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetResourceProfileFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get resource profile due to exception.",
+          ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetResourceProfileRetrieved(stopTime - startTime);
+    return RouterYarnClientUtils.mergeClusterResourceProfileResponse(resourceProfile);
   }
 
   @Override
   public GetAllResourceTypeInfoResponse getResourceTypeInfo(
       GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null) {
+      routerMetrics.incrResourceTypeInfoFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getResourceTypeInfo request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getResourceTypeInfo",
+        new Class[] {GetAllResourceTypeInfoRequest.class}, new Object[] {request});
+    Collection<GetAllResourceTypeInfoResponse> listResourceTypeInfo;
+    try {
+      listResourceTypeInfo = invokeConcurrent(remoteMethod, GetAllResourceTypeInfoResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrResourceTypeInfoFailedRetrieved();
+      LOG.error("Unable to get all resource type info node due to exception.", ex);
+      throw ex;
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetResourceTypeInfoRetrieved(stopTime - startTime);
+    // Merge the GetAllResourceTypeInfoResponse
+    return RouterYarnClientUtils.mergeResourceTypes(listResourceTypeInfo);
   }
 
   @Override
@@ -981,19 +1753,256 @@ public void shutdown() {
   @Override
   public GetAttributesToNodesResponse getAttributesToNodes(
       GetAttributesToNodesRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getNodeAttributes() == null) {
+      routerMetrics.incrGetAttributesToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getAttributesToNodes request " +
+          "or nodeAttributes.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getAttributesToNodes",
+        new Class[] {GetAttributesToNodesRequest.class}, new Object[] {request});
+    Collection<GetAttributesToNodesResponse> attributesToNodesResponses = null;
+    try {
+      attributesToNodesResponses =
+          invokeConcurrent(remoteMethod, GetAttributesToNodesResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetAttributesToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get attributes to nodes due to exception.",
+          ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetAttributesToNodesRetrieved(stopTime - startTime);
+    return RouterYarnClientUtils.mergeAttributesToNodesResponse(attributesToNodesResponses);
   }
 
   @Override
   public GetClusterNodeAttributesResponse getClusterNodeAttributes(
-      GetClusterNodeAttributesRequest request)
-      throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+      GetClusterNodeAttributesRequest request) throws YarnException, IOException {
+    if (request == null) {
+      routerMetrics.incrGetClusterNodeAttributesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getClusterNodeAttributes request.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getClusterNodeAttributes",
+        new Class[] {GetClusterNodeAttributesRequest.class}, new Object[] {request});
+    Collection<GetClusterNodeAttributesResponse> clusterNodeAttributesResponses = null;
+    try {
+      clusterNodeAttributesResponses = invokeConcurrent(remoteMethod,
+          GetClusterNodeAttributesResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetClusterNodeAttributesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get cluster node attributes due " +
+          " to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetClusterNodeAttributesRetrieved(stopTime - startTime);
+    return RouterYarnClientUtils.mergeClusterNodeAttributesResponse(clusterNodeAttributesResponses);
   }
 
   @Override
   public GetNodesToAttributesResponse getNodesToAttributes(
       GetNodesToAttributesRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+    if (request == null || request.getHostNames() == null) {
+      routerMetrics.incrGetNodesToAttributesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing getNodesToAttributes request or " +
+          "hostNames.", null);
+    }
+    long startTime = clock.getTime();
+    ClientMethod remoteMethod = new ClientMethod("getNodesToAttributes",
+        new Class[] {GetNodesToAttributesRequest.class}, new Object[] {request});
+    Collection<GetNodesToAttributesResponse> nodesToAttributesResponses = null;
+    try {
+      nodesToAttributesResponses = invokeConcurrent(remoteMethod,
+          GetNodesToAttributesResponse.class);
+    } catch (Exception ex) {
+      routerMetrics.incrGetNodesToAttributesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to get nodes to attributes due " +
+          " to exception.", ex);
+    }
+    long stopTime = clock.getTime();
+    routerMetrics.succeededGetNodesToAttributesRetrieved(stopTime - startTime);
+    return RouterYarnClientUtils.mergeNodesToAttributesResponse(nodesToAttributesResponses);
+  }
+
+  protected SubClusterId getApplicationHomeSubCluster(
+      ApplicationId applicationId) throws YarnException {
+    if (applicationId == null) {
+      LOG.error("ApplicationId is Null, Can't find in SubCluster.");
+      return null;
+    }
+
+    SubClusterId resultSubClusterId = null;
+
+    // try looking for applicationId in Home SubCluster
+    try {
+      resultSubClusterId = federationFacade.
+          getApplicationHomeSubCluster(applicationId);
+    } catch (YarnException ex) {
+      if(LOG.isDebugEnabled()){
+        LOG.debug("Can't find applicationId = {} in home sub cluster, " +
+             " try foreach sub clusters.", applicationId);
+      }
+    }
+    if (resultSubClusterId != null) {
+      return resultSubClusterId;
+    }
+
+    // if applicationId not found in Home SubCluster,
+    // foreach Clusters
+    Map<SubClusterId, SubClusterInfo> subClusters =
+        federationFacade.getSubClusters(true);
+    for (SubClusterId subClusterId : subClusters.keySet()) {
+      try {
+        ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+        if(clientRMProxy == null) {
+          continue;
+        }
+        GetApplicationReportRequest appReportRequest =
+            GetApplicationReportRequest.newInstance(applicationId);
+        GetApplicationReportResponse appReportResponse =
+            clientRMProxy.getApplicationReport(appReportRequest);
+
+        if(appReportResponse!=null && applicationId.equals(
+            appReportResponse.getApplicationReport().getApplicationId())){
+          resultSubClusterId = federationFacade.addApplicationHomeSubCluster(
+               ApplicationHomeSubCluster.newInstance(applicationId, subClusterId));
+          return resultSubClusterId;
+        }
+
+      } catch (Exception ex) {
+        if(LOG.isDebugEnabled()){
+          LOG.debug("Can't find applicationId = {} in Sub Cluster!", applicationId);
+        }
+      }
+    }
+
+    String errorMsg =
+        String.format("Can't find applicationId = %s in any sub clusters", applicationId);
+    throw new YarnException(errorMsg);
+  }
+
+  protected SubClusterId getReservationHomeSubCluster(ReservationId reservationId)
+      throws YarnException {
+
+    if (reservationId == null) {
+      LOG.error("ReservationId is Null, Can't find in SubCluster.");
+      return null;
+    }
+
+    // try looking for reservation in Home SubCluster
+    try {
+      SubClusterId resultSubClusterId =
+          federationFacade.getReservationHomeSubCluster(reservationId);
+      if (resultSubClusterId != null) {
+        return resultSubClusterId;
+      }
+    } catch (YarnException e) {
+      RouterServerUtil.logAndThrowException(e,
+          "Can't find reservationId = %s in home sub cluster.", reservationId);
+    }
+
+    String errorMsg =
+        String.format("Can't find reservationId = %s in home sub cluster.", reservationId);
+    throw new YarnException(errorMsg);
+  }
+
+  @VisibleForTesting
+  public FederationStateStoreFacade getFederationFacade() {
+    return federationFacade;
+  }
+
+  @VisibleForTesting
+  public Map<SubClusterId, ApplicationClientProtocol> getClientRMProxies() {
+    return clientRMProxies;
+  }
+
+  private Boolean existsReservationHomeSubCluster(ReservationId reservationId) {
+    try {
+      SubClusterId subClusterId = federationFacade.getReservationHomeSubCluster(reservationId);
+      if (subClusterId != null) {
+        return true;
+      }
+    } catch (YarnException e) {
+      LOG.warn("get homeSubCluster by reservationId = {} error.", reservationId, e);
+    }
+    return false;
+  }
+
+  private void addReservationHomeSubCluster(ReservationId reservationId,
+      ReservationHomeSubCluster homeSubCluster) throws YarnException {
+    try {
+      // persist the mapping of reservationId and the subClusterId which has
+      // been selected as its home
+      federationFacade.addReservationHomeSubCluster(homeSubCluster);
+    } catch (YarnException e) {
+      RouterServerUtil.logAndThrowException(e,
+          "Unable to insert the ReservationId %s into the FederationStateStore.",
+          reservationId);
+    }
+  }
+
+  private void updateReservationHomeSubCluster(SubClusterId subClusterId,
+      ReservationId reservationId, ReservationHomeSubCluster homeSubCluster) throws YarnException {
+    try {
+      // update the mapping of reservationId and the home subClusterId to
+      // the new subClusterId we have selected
+      federationFacade.updateReservationHomeSubCluster(homeSubCluster);
+    } catch (YarnException e) {
+      SubClusterId subClusterIdInStateStore =
+          federationFacade.getReservationHomeSubCluster(reservationId);
+      if (subClusterId == subClusterIdInStateStore) {
+        LOG.info("Reservation {} already submitted on SubCluster {}.",
+            reservationId, subClusterId);
+      } else {
+        RouterServerUtil.logAndThrowException(e,
+            "Unable to update the ReservationId %s into the FederationStateStore.",
+            reservationId);
+      }
+    }
+  }
+
+  protected int getNumMinThreads(Configuration conf) {
+
+    String threadSize = conf.get(YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE);
+
+    // If the user configures YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
+    // we will still get the number of threads from this configuration.
+    if (StringUtils.isNotBlank(threadSize)) {
+      LOG.warn("{} is a deprecated property, " +
+          "please remove it, use {} to configure the minimum number of thread pool.",
+          YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
+          YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE);
+      return Integer.parseInt(threadSize);
+    }
+
+    int numMinThreads = conf.getInt(
+        YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE,
+        YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE);
+    return numMinThreads;
+  }
+
+  protected int getNumMaxThreads(Configuration conf) {
+
+    String threadSize = conf.get(YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE);
+
+    // If the user configures YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
+    // we will still get the number of threads from this configuration.
+    if (StringUtils.isNotBlank(threadSize)) {
+      LOG.warn("{} is a deprecated property, " +
+          "please remove it, use {} to configure the maximum number of thread pool.",
+          YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
+          YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE);
+      return Integer.parseInt(threadSize);
+    }
+
+    int numMaxThreads = conf.getInt(
+        YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE,
+        YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE);
+    return numMaxThreads;
+  }
+
+  @VisibleForTesting
+  public void setNumSubmitRetries(int numSubmitRetries) {
+    this.numSubmitRetries = numSubmitRetries;
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java
index cb180237c41bc..0fb38b2eddad4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java
@@ -19,13 +19,10 @@
 package org.apache.hadoop.yarn.server.router.clientrm;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
@@ -34,8 +31,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
@@ -109,8 +104,9 @@
 import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
 import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider;
 import org.apache.hadoop.yarn.util.LRUCacheHashMap;
 import org.slf4j.Logger;
@@ -142,13 +138,15 @@ public class RouterClientRMService extends AbstractService
   // and remove the oldest used ones.
   private Map<String, RequestInterceptorChainWrapper> userPipelineMap;
 
+  private RouterDelegationTokenSecretManager routerDTSecretManager;
+
   public RouterClientRMService() {
     super(RouterClientRMService.class.getName());
   }
 
   @Override
   protected void serviceStart() throws Exception {
-    LOG.info("Starting Router ClientRMService");
+    LOG.info("Starting Router ClientRMService.");
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
     UserGroupInformation.setConfiguration(conf);
@@ -162,9 +160,7 @@ protected void serviceStart() throws Exception {
     int maxCacheSize =
         conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE,
             YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE);
-    this.userPipelineMap = Collections.synchronizedMap(
-        new LRUCacheHashMap<String, RequestInterceptorChainWrapper>(
-            maxCacheSize, true));
+    this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true));
 
     Configuration serverConf = new Configuration(conf);
 
@@ -172,8 +168,12 @@ protected void serviceStart() throws Exception {
         serverConf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT,
             YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT);
 
+    // Initialize RouterRMDelegationTokenSecretManager.
+    routerDTSecretManager = createRouterRMDelegationTokenSecretManager(conf);
+    routerDTSecretManager.startThreads();
+
     this.server = rpc.getServer(ApplicationClientProtocol.class, this,
-        listenerEndpoint, serverConf, null, numWorkerThreads);
+        listenerEndpoint, serverConf, routerDTSecretManager, numWorkerThreads);
 
     // Enable service authorization?
     if (conf.getBoolean(
@@ -182,14 +182,13 @@ protected void serviceStart() throws Exception {
     }
 
     this.server.start();
-    LOG.info("Router ClientRMService listening on address: "
-        + this.server.getListenerAddress());
+    LOG.info("Router ClientRMService listening on address: {}.", this.server.getListenerAddress());
     super.serviceStart();
   }
 
   @Override
   protected void serviceStop() throws Exception {
-    LOG.info("Stopping Router ClientRMService");
+    LOG.info("Stopping Router ClientRMService.");
     if (this.server != null) {
       this.server.stop();
     }
@@ -202,27 +201,6 @@ public Server getServer() {
     return this.server;
   }
 
-  /**
-   * Returns the comma separated intercepter class names from the configuration.
-   *
-   * @param conf
-   * @return the intercepter class names as an instance of ArrayList
-   */
-  private List<String> getInterceptorClassNames(Configuration conf) {
-    String configuredInterceptorClassNames =
-        conf.get(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
-            YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS);
-
-    List<String> interceptorClassNames = new ArrayList<String>();
-    Collection<String> tempList =
-        StringUtils.getStringCollection(configuredInterceptorClassNames);
-    for (String item : tempList) {
-      interceptorClassNames.add(item.trim());
-    }
-
-    return interceptorClassNames;
-  }
-
   @Override
   public GetNewApplicationResponse getNewApplication(
       GetNewApplicationRequest request) throws YarnException, IOException {
@@ -474,7 +452,7 @@ public GetNodesToAttributesResponse getNodesToAttributes(
   }
 
   @VisibleForTesting
-  protected RequestInterceptorChainWrapper getInterceptorChain()
+  public RequestInterceptorChainWrapper getInterceptorChain()
       throws IOException {
     String user = UserGroupInformation.getCurrentUser().getUserName();
     RequestInterceptorChainWrapper chain = userPipelineMap.get(user);
@@ -490,9 +468,9 @@ void refreshServiceAcls(Configuration configuration,
   }
 
   /**
-   * Gets the Request intercepter chains for all the users.
+   * Gets the Request interceptor chains for all the users.
    *
-   * @return the request intercepter chains.
+   * @return the request interceptor chains.
    */
   @VisibleForTesting
   protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
@@ -500,56 +478,22 @@ protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
   }
 
   /**
-   * This method creates and returns reference of the first intercepter in the
-   * chain of request intercepter instances.
+   * This method creates and returns reference of the first interceptor in the
+   * chain of request interceptor instances.
    *
-   * @return the reference of the first intercepter in the chain
+   * @return the reference of the first interceptor in the chain
    */
   @VisibleForTesting
   protected ClientRequestInterceptor createRequestInterceptorChain() {
     Configuration conf = getConfig();
-
-    List<String> interceptorClassNames = getInterceptorClassNames(conf);
-
-    ClientRequestInterceptor pipeline = null;
-    ClientRequestInterceptor current = null;
-    for (String interceptorClassName : interceptorClassNames) {
-      try {
-        Class<?> interceptorClass = conf.getClassByName(interceptorClassName);
-        if (ClientRequestInterceptor.class.isAssignableFrom(interceptorClass)) {
-          ClientRequestInterceptor interceptorInstance =
-              (ClientRequestInterceptor) ReflectionUtils
-                  .newInstance(interceptorClass, conf);
-          if (pipeline == null) {
-            pipeline = interceptorInstance;
-            current = interceptorInstance;
-            continue;
-          } else {
-            current.setNextInterceptor(interceptorInstance);
-            current = interceptorInstance;
-          }
-        } else {
-          throw new YarnRuntimeException(
-              "Class: " + interceptorClassName + " not instance of "
-                  + ClientRequestInterceptor.class.getCanonicalName());
-        }
-      } catch (ClassNotFoundException e) {
-        throw new YarnRuntimeException(
-            "Could not instantiate ApplicationClientRequestInterceptor: "
-                + interceptorClassName,
-            e);
-      }
-    }
-
-    if (pipeline == null) {
-      throw new YarnRuntimeException(
-          "RequestInterceptor pipeline is not configured in the system");
-    }
-    return pipeline;
+    return RouterServerUtil.createRequestInterceptorChain(conf,
+        YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
+        YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS,
+        ClientRequestInterceptor.class);
   }
 
   /**
-   * Initializes the request intercepter pipeline for the specified application.
+   * Initializes the request interceptor pipeline for the specified application.
    *
    * @param user
    */
@@ -566,15 +510,22 @@ private RequestInterceptorChainWrapper initializePipeline(String user) {
       try {
         // We should init the pipeline instance after it is created and then
         // add to the map, to ensure thread safe.
-        LOG.info("Initializing request processing pipeline for application "
-            + "for the user: {}", user);
+        LOG.info("Initializing request processing pipeline for application for the user: {}.",
+            user);
 
         ClientRequestInterceptor interceptorChain =
             this.createRequestInterceptorChain();
         interceptorChain.init(user);
+
+        // We set the RouterDelegationTokenSecretManager instance to the interceptorChain
+        // and let the interceptor use it.
+        if (routerDTSecretManager != null) {
+          interceptorChain.setTokenSecretManager(routerDTSecretManager);
+        }
+
         chainWrapper.init(interceptorChain);
       } catch (Exception e) {
-        LOG.error("Init ClientRequestInterceptor error for user: " + user, e);
+        LOG.error("Init ClientRequestInterceptor error for user: {}.", user, e);
         throw e;
       }
 
@@ -601,9 +552,9 @@ public synchronized void init(ClientRequestInterceptor interceptor) {
     }
 
     /**
-     * Gets the root request intercepter.
+     * Gets the root request interceptor.
      *
-     * @return the root request intercepter
+     * @return the root request interceptor
      */
     public synchronized ClientRequestInterceptor getRootInterceptor() {
       return rootInterceptor;
@@ -617,4 +568,47 @@ protected void finalize() {
       rootInterceptor.shutdown();
     }
   }
+
+  @VisibleForTesting
+  public Map<String, RequestInterceptorChainWrapper> getUserPipelineMap() {
+    return userPipelineMap;
+  }
+
+  /**
+   * Create RouterRMDelegationTokenSecretManager.
+   * In the YARN federation, the Router will replace the RM to
+   * manage the RMDelegationToken (generate, update, cancel),
+   * so the relevant configuration parameters still obtain the configuration parameters of the RM.
+   *
+   * @param conf Configuration
+   * @return RouterDelegationTokenSecretManager.
+   */
+  protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecretManager(
+      Configuration conf) {
+
+    long secretKeyInterval = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+
+    long tokenMaxLifetime = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+
+    long tokenRenewInterval = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+
+    long removeScanInterval = conf.getTimeDuration(
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
+        TimeUnit.MILLISECONDS);
+
+    return new RouterDelegationTokenSecretManager(secretKeyInterval,
+        tokenMaxLifetime, tokenRenewInterval, removeScanInterval);
+  }
+
+  @VisibleForTesting
+  public RouterDelegationTokenSecretManager getRouterDTSecretManager() {
+    return routerDTSecretManager;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java
index 934636b104df6..481005b478d60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java
@@ -20,14 +20,44 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
 
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationAllocationState;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
 import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 /**
@@ -50,6 +80,9 @@ public static GetClusterMetricsResponse merge(
           tmp.getNumNodeManagers() + metrics.getNumNodeManagers());
       tmp.setNumActiveNodeManagers(
           tmp.getNumActiveNodeManagers() + metrics.getNumActiveNodeManagers());
+      tmp.setNumDecommissioningNodeManagers(
+          tmp.getNumDecommissioningNodeManagers() + metrics
+              .getNumDecommissioningNodeManagers());
       tmp.setNumDecommissionedNodeManagers(
           tmp.getNumDecommissionedNodeManagers() + metrics
               .getNumDecommissionedNodeManagers());
@@ -60,6 +93,9 @@ public static GetClusterMetricsResponse merge(
       tmp.setNumUnhealthyNodeManagers(
           tmp.getNumUnhealthyNodeManagers() + metrics
               .getNumUnhealthyNodeManagers());
+      tmp.setNumShutdownNodeManagers(
+          tmp.getNumShutdownNodeManagers() + metrics
+              .getNumShutdownNodeManagers());
     }
     return GetClusterMetricsResponse.newInstance(tmp);
   }
@@ -194,4 +230,305 @@ private static boolean mergeUamToReport(String appName,
     return !(appName.startsWith(UnmanagedApplicationManager.APP_NAME) ||
         appName.startsWith(PARTIAL_REPORT));
   }
+
+  /**
+   * Merges a list of GetClusterNodesResponse.
+   *
+   * @param responses a list of GetClusterNodesResponse to merge.
+   * @return the merged GetClusterNodesResponse.
+   */
+  public static GetClusterNodesResponse mergeClusterNodesResponse(
+      Collection<GetClusterNodesResponse> responses) {
+    GetClusterNodesResponse clusterNodesResponse = Records.newRecord(GetClusterNodesResponse.class);
+    List<NodeReport> nodeReports = new ArrayList<>();
+    for (GetClusterNodesResponse response : responses) {
+      if (response != null && response.getNodeReports() != null) {
+        nodeReports.addAll(response.getNodeReports());
+      }
+    }
+    clusterNodesResponse.setNodeReports(nodeReports);
+    return clusterNodesResponse;
+  }
+
+  /**
+   * Merges a list of GetNodesToLabelsResponse.
+   *
+   * @param responses a list of GetNodesToLabelsResponse to merge.
+   * @return the merged GetNodesToLabelsResponse.
+   */
+  public static GetNodesToLabelsResponse mergeNodesToLabelsResponse(
+      Collection<GetNodesToLabelsResponse> responses) {
+    GetNodesToLabelsResponse nodesToLabelsResponse = Records.newRecord(
+         GetNodesToLabelsResponse.class);
+    Map<NodeId, Set<String>> nodesToLabelMap = new HashMap<>();
+    for (GetNodesToLabelsResponse response : responses) {
+      if (response != null && response.getNodeToLabels() != null) {
+        nodesToLabelMap.putAll(response.getNodeToLabels());
+      }
+    }
+    nodesToLabelsResponse.setNodeToLabels(nodesToLabelMap);
+    return nodesToLabelsResponse;
+  }
+
+  /**
+   * Merges a list of GetLabelsToNodesResponse.
+   *
+   * @param responses a list of GetLabelsToNodesResponse to merge.
+   * @return the merged GetLabelsToNodesResponse.
+   */
+  public static GetLabelsToNodesResponse mergeLabelsToNodes(
+      Collection<GetLabelsToNodesResponse> responses){
+    GetLabelsToNodesResponse labelsToNodesResponse = Records.newRecord(
+        GetLabelsToNodesResponse.class);
+    Map<String, Set<NodeId>> labelsToNodesMap = new HashMap<>();
+    for (GetLabelsToNodesResponse response : responses) {
+      if (response != null && response.getLabelsToNodes() != null) {
+        Map<String, Set<NodeId>> clusterLabelsToNodesMap = response.getLabelsToNodes();
+        for (Map.Entry<String, Set<NodeId>> entry : clusterLabelsToNodesMap.entrySet()) {
+          String label = entry.getKey();
+          Set<NodeId> clusterNodes = entry.getValue();
+          if (labelsToNodesMap.containsKey(label)) {
+            Set<NodeId> allNodes = labelsToNodesMap.get(label);
+            allNodes.addAll(clusterNodes);
+          } else {
+            labelsToNodesMap.put(label, clusterNodes);
+          }
+        }
+      }
+    }
+    labelsToNodesResponse.setLabelsToNodes(labelsToNodesMap);
+    return labelsToNodesResponse;
+  }
+
+  /**
+   * Merges a list of GetClusterNodeLabelsResponse.
+   *
+   * @param responses a list of GetClusterNodeLabelsResponse to merge.
+   * @return the merged GetClusterNodeLabelsResponse.
+   */
+  public static GetClusterNodeLabelsResponse mergeClusterNodeLabelsResponse(
+      Collection<GetClusterNodeLabelsResponse> responses) {
+    GetClusterNodeLabelsResponse nodeLabelsResponse = Records.newRecord(
+        GetClusterNodeLabelsResponse.class);
+    Set<NodeLabel> nodeLabelsList = new HashSet<>();
+    for (GetClusterNodeLabelsResponse response : responses) {
+      if (response != null && response.getNodeLabelList() != null) {
+        nodeLabelsList.addAll(response.getNodeLabelList());
+      }
+    }
+    nodeLabelsResponse.setNodeLabelList(new ArrayList<>(nodeLabelsList));
+    return nodeLabelsResponse;
+  }
+
+  /**
+   * Merges a list of GetQueueUserAclsInfoResponse.
+   *
+   * @param responses a list of GetQueueUserAclsInfoResponse to merge.
+   * @return the merged GetQueueUserAclsInfoResponse.
+   */
+  public static GetQueueUserAclsInfoResponse mergeQueueUserAcls(
+      Collection<GetQueueUserAclsInfoResponse> responses) {
+    GetQueueUserAclsInfoResponse aclsInfoResponse = Records.newRecord(
+        GetQueueUserAclsInfoResponse.class);
+    Set<QueueUserACLInfo> queueUserACLInfos = new HashSet<>();
+    for (GetQueueUserAclsInfoResponse response : responses) {
+      if (response != null && response.getUserAclsInfoList() != null) {
+        queueUserACLInfos.addAll(response.getUserAclsInfoList());
+      }
+    }
+    aclsInfoResponse.setUserAclsInfoList(new ArrayList<>(queueUserACLInfos));
+    return aclsInfoResponse;
+  }
+
+  /**
+   * Merges a list of ReservationListResponse.
+   *
+   * @param responses a list of ReservationListResponse to merge.
+   * @return the merged ReservationListResponse.
+   */
+  public static ReservationListResponse mergeReservationsList(
+      Collection<ReservationListResponse> responses) {
+    ReservationListResponse reservationListResponse =
+        Records.newRecord(ReservationListResponse.class);
+    List<ReservationAllocationState> reservationAllocationStates =
+        new ArrayList<>();
+    for (ReservationListResponse response : responses) {
+      if (response != null && response.getReservationAllocationState() != null) {
+        reservationAllocationStates.addAll(
+            response.getReservationAllocationState());
+      }
+    }
+    reservationListResponse.setReservationAllocationState(
+        reservationAllocationStates);
+    return reservationListResponse;
+  }
+
+  /**
+   * Merges a list of GetAllResourceTypeInfoResponse.
+   *
+   * @param responses a list of GetAllResourceTypeInfoResponse to merge.
+   * @return the merged GetAllResourceTypeInfoResponse.
+   */
+  public static GetAllResourceTypeInfoResponse mergeResourceTypes(
+      Collection<GetAllResourceTypeInfoResponse> responses) {
+    GetAllResourceTypeInfoResponse resourceTypeInfoResponse =
+        Records.newRecord(GetAllResourceTypeInfoResponse.class);
+    Set<ResourceTypeInfo> resourceTypeInfoSet = new HashSet<>();
+    for (GetAllResourceTypeInfoResponse response : responses) {
+      if (response != null && response.getResourceTypeInfo() != null) {
+        resourceTypeInfoSet.addAll(response.getResourceTypeInfo());
+      }
+    }
+    resourceTypeInfoResponse.setResourceTypeInfo(
+        new ArrayList<>(resourceTypeInfoSet));
+    return resourceTypeInfoResponse;
+  }
+
+  /**
+   * Merges a list of GetQueueInfoResponse.
+   *
+   * @param responses a list of GetQueueInfoResponse to merge.
+   * @return the merged GetQueueInfoResponse.
+   */
+  public static GetQueueInfoResponse mergeQueues(
+      Collection<GetQueueInfoResponse> responses) {
+    GetQueueInfoResponse queueResponse = Records.newRecord(
+        GetQueueInfoResponse.class);
+
+    QueueInfo queueInfo = null;
+    for (GetQueueInfoResponse response : responses) {
+      if (response != null && response.getQueueInfo() != null) {
+        if (queueInfo == null) {
+          queueInfo = response.getQueueInfo();
+        } else {
+          // set Capacity\MaximumCapacity\CurrentCapacity
+          queueInfo.setCapacity(queueInfo.getCapacity() + response.getQueueInfo().getCapacity());
+          queueInfo.setMaximumCapacity(
+              queueInfo.getMaximumCapacity() + response.getQueueInfo().getMaximumCapacity());
+          queueInfo.setCurrentCapacity(
+              queueInfo.getCurrentCapacity() + response.getQueueInfo().getCurrentCapacity());
+
+          // set childQueues
+          List<QueueInfo> childQueues = new ArrayList<>(queueInfo.getChildQueues());
+          childQueues.addAll(response.getQueueInfo().getChildQueues());
+          queueInfo.setChildQueues(childQueues);
+
+          // set applications
+          List<ApplicationReport> applicationReports = new ArrayList<>(queueInfo.getApplications());
+          applicationReports.addAll(response.getQueueInfo().getApplications());
+          queueInfo.setApplications(applicationReports);
+
+          // set accessibleNodeLabels
+          Set<String> accessibleNodeLabels = new HashSet<>();
+          if (queueInfo.getAccessibleNodeLabels() != null) {
+            accessibleNodeLabels.addAll(queueInfo.getAccessibleNodeLabels());
+          }
+          if (response.getQueueInfo() != null) {
+            accessibleNodeLabels.addAll(response.getQueueInfo().getAccessibleNodeLabels());
+          }
+          queueInfo.setAccessibleNodeLabels(accessibleNodeLabels);
+        }
+      }
+    }
+    queueResponse.setQueueInfo(queueInfo);
+    return queueResponse;
+  }
+
+  /**
+   * Merges a list of GetAllResourceProfilesResponse.
+   *
+   * @param responses a list of GetAllResourceProfilesResponse to merge.
+   * @return the merged GetAllResourceProfilesResponse.
+   */
+  public static GetAllResourceProfilesResponse mergeClusterResourceProfilesResponse(
+      Collection<GetAllResourceProfilesResponse> responses) {
+    GetAllResourceProfilesResponse profilesResponse =
+        Records.newRecord(GetAllResourceProfilesResponse.class);
+    Map<String, Resource> profilesMap = new HashMap<>();
+    for (GetAllResourceProfilesResponse response : responses) {
+      if (response != null && response.getResourceProfiles() != null) {
+        for (Map.Entry<String, Resource> entry : response.getResourceProfiles().entrySet()) {
+          String key = entry.getKey();
+          Resource r1 = profilesMap.getOrDefault(key, null);
+          Resource r2 = entry.getValue();
+          Resource rAdd = r1 == null ? r2 : Resources.add(r1, r2);
+          profilesMap.put(key, rAdd);
+        }
+      }
+    }
+    profilesResponse.setResourceProfiles(profilesMap);
+    return profilesResponse;
+  }
+
+  /**
+   * Merges a list of GetResourceProfileResponse.
+   *
+   * @param responses a list of GetResourceProfileResponse to merge.
+   * @return the merged GetResourceProfileResponse.
+   */
+  public static GetResourceProfileResponse mergeClusterResourceProfileResponse(
+      Collection<GetResourceProfileResponse> responses) {
+    GetResourceProfileResponse profileResponse =
+        Records.newRecord(GetResourceProfileResponse.class);
+    Resource resource = Resource.newInstance(0, 0);
+    for (GetResourceProfileResponse response : responses) {
+      if (response != null && response.getResource() != null) {
+        Resource responseResource = response.getResource();
+        resource = Resources.add(resource, responseResource);
+      }
+    }
+    profileResponse.setResource(resource);
+    return profileResponse;
+  }
+
+  /**
+   * Merges a list of GetAttributesToNodesResponse.
+   *
+   * @param responses a list of GetAttributesToNodesResponse to merge.
+   * @return the merged GetAttributesToNodesResponse.
+   */
+  public static GetAttributesToNodesResponse mergeAttributesToNodesResponse(
+      Collection<GetAttributesToNodesResponse> responses) {
+    Map<NodeAttributeKey, List<NodeToAttributeValue>> nodeAttributeMap = new HashMap<>();
+    for (GetAttributesToNodesResponse response : responses) {
+      if (response != null && response.getAttributesToNodes() != null) {
+        nodeAttributeMap.putAll(response.getAttributesToNodes());
+      }
+    }
+    return GetAttributesToNodesResponse.newInstance(nodeAttributeMap);
+  }
+
+  /**
+   * Merges a list of GetClusterNodeAttributesResponse.
+   *
+   * @param responses a list of GetClusterNodeAttributesResponse to merge.
+   * @return the merged GetClusterNodeAttributesResponse.
+   */
+  public static GetClusterNodeAttributesResponse mergeClusterNodeAttributesResponse(
+      Collection<GetClusterNodeAttributesResponse> responses) {
+    Set<NodeAttributeInfo> nodeAttributeInfo = new HashSet<>();
+    for (GetClusterNodeAttributesResponse response : responses) {
+      if (response != null && response.getNodeAttributes() != null) {
+        nodeAttributeInfo.addAll(response.getNodeAttributes());
+      }
+    }
+    return GetClusterNodeAttributesResponse.newInstance(nodeAttributeInfo);
+  }
+
+  /**
+   * Merges a list of GetNodesToAttributesResponse.
+   *
+   * @param responses a list of GetNodesToAttributesResponse to merge.
+   * @return the merged GetNodesToAttributesResponse.
+   */
+  public static GetNodesToAttributesResponse mergeNodesToAttributesResponse(
+      Collection<GetNodesToAttributesResponse> responses) {
+    Map<String, Set<NodeAttribute>> attributesMap = new HashMap<>();
+    for (GetNodesToAttributesResponse response : responses) {
+      if (response != null && response.getNodeToAttributes() != null) {
+        attributesMap.putAll(response.getNodeToAttributes());
+      }
+    }
+    return GetNodesToAttributesResponse.newInstance(attributesMap);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java
index a4972fcb9ad84..8b09d69971988 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java
@@ -19,11 +19,13 @@
 package org.apache.hadoop.yarn.server.router.rmadmin;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
 
 /**
  * Implements the {@link RMAdminRequestInterceptor} interface and provides
  * common functionality which can can be used and/or extended by other concrete
- * intercepter classes.
+ * interceptor classes.
  *
  */
 public abstract class AbstractRMAdminRequestInterceptor
@@ -31,6 +33,9 @@ public abstract class AbstractRMAdminRequestInterceptor
   private Configuration conf;
   private RMAdminRequestInterceptor nextInterceptor;
 
+  @SuppressWarnings("checkstyle:visibilitymodifier")
+  protected UserGroupInformation user = null;
+
   /**
    * Sets the {@link RMAdminRequestInterceptor} in the chain.
    */
@@ -63,9 +68,10 @@ public Configuration getConf() {
    * Initializes the {@link RMAdminRequestInterceptor}.
    */
   @Override
-  public void init(String user) {
+  public void init(String userName) {
+    this.user = RouterServerUtil.setupUser(userName);
     if (this.nextInterceptor != null) {
-      this.nextInterceptor.init(user);
+      this.nextInterceptor.init(userName);
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
index 9625eb4d3662c..122782aef47ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
@@ -80,8 +80,9 @@ public void init(String userName) {
     try {
       // Do not create a proxy user if user name matches the user name on
       // current UGI
-      if (userName.equalsIgnoreCase(
-          UserGroupInformation.getCurrentUser().getUserName())) {
+      if (UserGroupInformation.isSecurityEnabled()) {
+        user = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getLoginUser());
+      } else if (userName.equalsIgnoreCase(UserGroupInformation.getCurrentUser().getUserName())) {
         user = UserGroupInformation.getCurrentUser();
       } else {
         user = UserGroupInformation.createProxyUser(userName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java
new file mode 100644
index 0000000000000..4564f8d8b8506
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/FederationRMAdminInterceptor.java
@@ -0,0 +1,283 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.rmadmin;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
+import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.router.RouterMetrics;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Collection;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class FederationRMAdminInterceptor extends AbstractRMAdminRequestInterceptor {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(FederationRMAdminInterceptor.class);
+
+  private Map<SubClusterId, ResourceManagerAdministrationProtocol> adminRMProxies;
+  private FederationStateStoreFacade federationFacade;
+  private final Clock clock = new MonotonicClock();
+  private RouterMetrics routerMetrics;
+  private ThreadPoolExecutor executorService;
+  private Configuration conf;
+
+  @Override
+  public void init(String userName) {
+    super.init(userName);
+
+    int numThreads = getConf().getInt(
+        YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE,
+        YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE);
+    ThreadFactory threadFactory = new ThreadFactoryBuilder()
+        .setNameFormat("RPC Router RMAdminClient-" + userName + "-%d ").build();
+
+    BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<Runnable>();
+    this.executorService = new ThreadPoolExecutor(numThreads, numThreads,
+        0L, TimeUnit.MILLISECONDS, workQueue, threadFactory);
+
+    federationFacade = FederationStateStoreFacade.getInstance();
+    this.conf = this.getConf();
+    this.adminRMProxies = new ConcurrentHashMap<>();
+    routerMetrics = RouterMetrics.getMetrics();
+  }
+
+  @VisibleForTesting
+  protected ResourceManagerAdministrationProtocol getAdminRMProxyForSubCluster(
+      SubClusterId subClusterId) throws YarnException {
+
+    if (adminRMProxies.containsKey(subClusterId)) {
+      return adminRMProxies.get(subClusterId);
+    }
+
+    ResourceManagerAdministrationProtocol adminRMProxy = null;
+    try {
+      boolean serviceAuthEnabled = this.conf.getBoolean(
+          CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false);
+      UserGroupInformation realUser = user;
+      if (serviceAuthEnabled) {
+        realUser = UserGroupInformation.createProxyUser(
+            user.getShortUserName(), UserGroupInformation.getLoginUser());
+      }
+      adminRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(),
+          ResourceManagerAdministrationProtocol.class, subClusterId, realUser);
+    } catch (Exception e) {
+      RouterServerUtil.logAndThrowException(e,
+          "Unable to create the interface to reach the SubCluster %s", subClusterId);
+    }
+    adminRMProxies.put(subClusterId, adminRMProxy);
+    return adminRMProxy;
+  }
+
+  @Override
+  public void setNextInterceptor(RMAdminRequestInterceptor next) {
+    throw new YarnRuntimeException("setNextInterceptor is being called on "
+       + "FederationRMAdminRequestInterceptor, which should be the last one "
+       + "in the chain. Check if the interceptor pipeline configuration "
+       + "is correct");
+  }
+
+  @Override
+  public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
+      throws StandbyException, YarnException, IOException {
+
+    // parameter verification.
+    if (request == null) {
+      routerMetrics.incrRefreshQueuesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Missing RefreshQueues request.", null);
+    }
+
+    // call refreshQueues of activeSubClusters.
+    try {
+      long startTime = clock.getTime();
+      RMAdminProtocolMethod remoteMethod = new RMAdminProtocolMethod(
+           new Class[] {RefreshQueuesRequest.class}, new Object[] {request});
+
+      Collection<RefreshQueuesResponse> refreshQueueResps =
+          remoteMethod.invokeConcurrent(this, RefreshQueuesResponse.class);
+
+      // If we get the return result from refreshQueueResps,
+      // it means that the call has been successful,
+      // and the RefreshQueuesResponse method can be reconstructed and returned.
+      if (CollectionUtils.isNotEmpty(refreshQueueResps)) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededRefreshQueuesRetrieved(stopTime - startTime);
+        return RefreshQueuesResponse.newInstance();
+      }
+    } catch (Exception e) {
+      routerMetrics.incrRefreshQueuesFailedRetrieved();
+      RouterServerUtil.logAndThrowException("Unable to refreshQueue to exception.", e);
+    }
+
+    routerMetrics.incrRefreshQueuesFailedRetrieved();
+    throw new YarnException("Unable to refreshQueue.");
+  }
+
+  @Override
+  public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
+      throws StandbyException, YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
+      RefreshSuperUserGroupsConfigurationRequest request)
+      throws StandbyException, YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
+      RefreshUserToGroupsMappingsRequest request)
+      throws StandbyException, YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshAdminAclsResponse refreshAdminAcls(RefreshAdminAclsRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshServiceAclsResponse refreshServiceAcls(RefreshServiceAclsRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public UpdateNodeResourceResponse updateNodeResource(UpdateNodeResourceRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshNodesResourcesResponse refreshNodesResources(RefreshNodesResourcesRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public AddToClusterNodeLabelsResponse addToClusterNodeLabels(
+      AddToClusterNodeLabelsRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
+      RemoveFromClusterNodeLabelsRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public ReplaceLabelsOnNodeResponse replaceLabelsOnNode(ReplaceLabelsOnNodeRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public CheckForDecommissioningNodesResponse checkForDecommissioningNodes(
+      CheckForDecommissioningNodesRequest checkForDecommissioningNodesRequest)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public RefreshClusterMaxPriorityResponse refreshClusterMaxPriority(
+      RefreshClusterMaxPriorityRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public NodesToAttributesMappingResponse mapAttributesToNodes(
+      NodesToAttributesMappingRequest request)
+      throws YarnException, IOException {
+    throw new NotImplementedException();
+  }
+
+  @Override
+  public String[] getGroupsForUser(String user) throws IOException {
+    return new String[0];
+  }
+
+  @VisibleForTesting
+  public FederationStateStoreFacade getFederationFacade() {
+    return federationFacade;
+  }
+
+  @VisibleForTesting
+  public ThreadPoolExecutor getExecutorService() {
+    return executorService;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java
new file mode 100644
index 0000000000000..e1aa806ff86ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminProtocolMethod.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.rmadmin;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationMethodWrapper;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
+
+
+/**
+ * Class to define admin method, params and arguments.
+ */
+public class RMAdminProtocolMethod extends FederationMethodWrapper {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RMAdminProtocolMethod.class);
+
+  private FederationStateStoreFacade federationFacade;
+  private FederationRMAdminInterceptor rmAdminInterceptor;
+  private Configuration configuration;
+
+  public RMAdminProtocolMethod(Class<?>[] pTypes, Object... pParams)
+      throws IOException {
+    super(pTypes, pParams);
+  }
+
+  public <R> Collection<R> invokeConcurrent(FederationRMAdminInterceptor interceptor,
+      Class<R> clazz) throws YarnException {
+    this.rmAdminInterceptor = interceptor;
+    this.federationFacade = FederationStateStoreFacade.getInstance();
+    this.configuration = interceptor.getConf();
+    return invokeConcurrent(clazz);
+  }
+
+  @Override
+  protected <R> Collection<R> invokeConcurrent(Class<R> clazz) throws YarnException {
+    String methodName = Thread.currentThread().getStackTrace()[3].getMethodName();
+    this.setMethodName(methodName);
+
+    ThreadPoolExecutor executorService = rmAdminInterceptor.getExecutorService();
+
+    // Get Active SubClusters
+    Map<SubClusterId, SubClusterInfo> subClusterInfo =
+        federationFacade.getSubClusters(true);
+    Collection<SubClusterId> subClusterIds = subClusterInfo.keySet();
+
+    List<Callable<Pair<SubClusterId, Object>>> callables = new ArrayList<>();
+    List<Future<Pair<SubClusterId, Object>>> futures = new ArrayList<>();
+    Map<SubClusterId, Exception> exceptions = new TreeMap<>();
+
+    // Generate parallel Callable tasks
+    for (SubClusterId subClusterId : subClusterIds) {
+      callables.add(() -> {
+        ResourceManagerAdministrationProtocol protocol =
+            rmAdminInterceptor.getAdminRMProxyForSubCluster(subClusterId);
+        Class<?>[] types = this.getTypes();
+        Object[] params = this.getParams();
+        Method method = ResourceManagerAdministrationProtocol.class.getMethod(methodName, types);
+        Object result = method.invoke(protocol, params);
+        return Pair.of(subClusterId, result);
+      });
+    }
+
+    // Get results from multiple threads
+    Map<SubClusterId, R> results = new TreeMap<>();
+    try {
+      futures.addAll(executorService.invokeAll(callables));
+      futures.stream().forEach(future -> {
+        SubClusterId subClusterId = null;
+        try {
+          Pair<SubClusterId, Object> pair = future.get();
+          subClusterId = pair.getKey();
+          Object result = pair.getValue();
+          results.put(subClusterId, clazz.cast(result));
+        } catch (InterruptedException | ExecutionException e) {
+          Throwable cause = e.getCause();
+          LOG.error("Cannot execute {} on {}: {}", methodName, subClusterId, cause.getMessage());
+          exceptions.put(subClusterId, e);
+        }
+      });
+    } catch (InterruptedException e) {
+      throw new YarnException("invokeConcurrent Failed.", e);
+    }
+
+    // All sub-clusters return results to be considered successful,
+    // otherwise an exception will be thrown.
+    if (exceptions != null && !exceptions.isEmpty()) {
+      Set<SubClusterId> subClusterIdSets = exceptions.keySet();
+      throw new YarnException("invokeConcurrent Failed, An exception occurred in subClusterIds = " +
+          StringUtils.join(subClusterIdSets, ","));
+    }
+
+    // return result
+    return results.values();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java
index dc4bda01b909c..29ab9e6add2cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java
@@ -22,14 +22,14 @@
 import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
 
 /**
- * Defines the contract to be implemented by the request intercepter classes,
+ * Defines the contract to be implemented by the request interceptor classes,
  * that can be used to intercept and inspect messages sent from the client to
  * the resource manager.
  */
 public interface RMAdminRequestInterceptor
     extends ResourceManagerAdministrationProtocol, Configurable {
   /**
-   * This method is called for initializing the intercepter. This is guaranteed
+   * This method is called for initializing the interceptor. This is guaranteed
    * to be called only once in the lifetime of this instance.
    *
    * @param user the name of the client
@@ -37,28 +37,28 @@ public interface RMAdminRequestInterceptor
   void init(String user);
 
   /**
-   * This method is called to release the resources held by the intercepter.
+   * This method is called to release the resources held by the interceptor.
    * This will be called when the application pipeline is being destroyed. The
    * concrete implementations should dispose the resources and forward the
-   * request to the next intercepter, if any.
+   * request to the next interceptor, if any.
    */
   void shutdown();
 
   /**
-   * Sets the next intercepter in the pipeline. The concrete implementation of
+   * Sets the next interceptor in the pipeline. The concrete implementation of
    * this interface should always pass the request to the nextInterceptor after
-   * inspecting the message. The last intercepter in the chain is responsible to
+   * inspecting the message. The last interceptor in the chain is responsible to
    * send the messages to the resource manager service and so the last
-   * intercepter will not receive this method call.
+   * interceptor will not receive this method call.
    *
    * @param nextInterceptor the RMAdminRequestInterceptor to set in the pipeline
    */
   void setNextInterceptor(RMAdminRequestInterceptor nextInterceptor);
 
   /**
-   * Returns the next intercepter in the chain.
+   * Returns the next interceptor in the chain.
    *
-   * @return the next intercepter in the chain
+   * @return the next interceptor in the chain
    */
   RMAdminRequestInterceptor getNextInterceptor();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
index e05de7a899488..8024cdb82f126 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
@@ -20,10 +20,7 @@
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -34,11 +31,8 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.PolicyProvider;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
 import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
@@ -69,6 +63,7 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
 import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider;
 import org.apache.hadoop.yarn.util.LRUCacheHashMap;
 import org.slf4j.Logger;
@@ -82,7 +77,7 @@
  * messages from client to the cluster resource manager. It listens
  * {@code ResourceManagerAdministrationProtocol} messages from the client and
  * creates a request intercepting pipeline instance for each client. The
- * pipeline is a chain of intercepter instances that can inspect and modify the
+ * pipeline is a chain of interceptor instances that can inspect and modify the
  * request/response as needed. The main difference with AMRMProxyService is the
  * protocol they implement.
  */
@@ -106,7 +101,7 @@ public RouterRMAdminService() {
 
   @Override
   protected void serviceStart() throws Exception {
-    LOG.info("Starting Router RMAdmin Service");
+    LOG.info("Starting Router RMAdmin Service.");
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
     UserGroupInformation.setConfiguration(conf);
@@ -120,9 +115,7 @@ protected void serviceStart() throws Exception {
     int maxCacheSize =
         conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE,
             YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE);
-    this.userPipelineMap = Collections.synchronizedMap(
-        new LRUCacheHashMap<String, RequestInterceptorChainWrapper>(
-            maxCacheSize, true));
+    this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true));
 
     Configuration serverConf = new Configuration(conf);
 
@@ -139,14 +132,13 @@ protected void serviceStart() throws Exception {
     }
 
     this.server.start();
-    LOG.info("Router RMAdminService listening on address: "
-        + this.server.getListenerAddress());
+    LOG.info("Router RMAdminService listening on address: {}.", this.server.getListenerAddress());
     super.serviceStart();
   }
 
   @Override
   protected void serviceStop() throws Exception {
-    LOG.info("Stopping Router RMAdminService");
+    LOG.info("Stopping Router RMAdminService.");
     if (this.server != null) {
       this.server.stop();
     }
@@ -164,29 +156,8 @@ public Server getServer() {
     return this.server;
   }
 
-  /**
-   * Returns the comma separated intercepter class names from the configuration.
-   *
-   * @param conf
-   * @return the intercepter class names as an instance of ArrayList
-   */
-  private List<String> getInterceptorClassNames(Configuration conf) {
-    String configuredInterceptorClassNames =
-        conf.get(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE,
-            YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS);
-
-    List<String> interceptorClassNames = new ArrayList<String>();
-    Collection<String> tempList =
-        StringUtils.getStringCollection(configuredInterceptorClassNames);
-    for (String item : tempList) {
-      interceptorClassNames.add(item.trim());
-    }
-
-    return interceptorClassNames;
-  }
-
   @VisibleForTesting
-  protected RequestInterceptorChainWrapper getInterceptorChain()
+  public RequestInterceptorChainWrapper getInterceptorChain()
       throws IOException {
     String user = UserGroupInformation.getCurrentUser().getUserName();
     RequestInterceptorChainWrapper chain = userPipelineMap.get(user);
@@ -197,9 +168,9 @@ protected RequestInterceptorChainWrapper getInterceptorChain()
   }
 
   /**
-   * Gets the Request intercepter chains for all the users.
+   * Gets the Request interceptor chains for all the users.
    *
-   * @return the request intercepter chains.
+   * @return the request interceptor chains.
    */
   @VisibleForTesting
   protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
@@ -207,57 +178,22 @@ protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
   }
 
   /**
-   * This method creates and returns reference of the first intercepter in the
-   * chain of request intercepter instances.
+   * This method creates and returns reference of the first interceptor in the
+   * chain of request interceptor instances.
    *
-   * @return the reference of the first intercepter in the chain
+   * @return the reference of the first interceptor in the chain
    */
   @VisibleForTesting
   protected RMAdminRequestInterceptor createRequestInterceptorChain() {
     Configuration conf = getConfig();
-
-    List<String> interceptorClassNames = getInterceptorClassNames(conf);
-
-    RMAdminRequestInterceptor pipeline = null;
-    RMAdminRequestInterceptor current = null;
-    for (String interceptorClassName : interceptorClassNames) {
-      try {
-        Class<?> interceptorClass = conf.getClassByName(interceptorClassName);
-        if (RMAdminRequestInterceptor.class
-            .isAssignableFrom(interceptorClass)) {
-          RMAdminRequestInterceptor interceptorInstance =
-              (RMAdminRequestInterceptor) ReflectionUtils
-                  .newInstance(interceptorClass, conf);
-          if (pipeline == null) {
-            pipeline = interceptorInstance;
-            current = interceptorInstance;
-            continue;
-          } else {
-            current.setNextInterceptor(interceptorInstance);
-            current = interceptorInstance;
-          }
-        } else {
-          throw new YarnRuntimeException(
-              "Class: " + interceptorClassName + " not instance of "
-                  + RMAdminRequestInterceptor.class.getCanonicalName());
-        }
-      } catch (ClassNotFoundException e) {
-        throw new YarnRuntimeException(
-            "Could not instantiate RMAdminRequestInterceptor: "
-                + interceptorClassName,
-            e);
-      }
-    }
-
-    if (pipeline == null) {
-      throw new YarnRuntimeException(
-          "RequestInterceptor pipeline is not configured in the system");
-    }
-    return pipeline;
+    return RouterServerUtil.createRequestInterceptorChain(conf,
+        YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE,
+        YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS,
+        RMAdminRequestInterceptor.class);
   }
 
   /**
-   * Initializes the request intercepter pipeline for the specified user.
+   * Initializes the request interceptor pipeline for the specified user.
    *
    * @param user
    */
@@ -274,14 +210,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) {
       try {
         // We should init the pipeline instance after it is created and then
         // add to the map, to ensure thread safe.
-        LOG.info("Initializing request processing pipeline for user: {}", user);
+        LOG.info("Initializing request processing pipeline for user: {}.", user);
 
         RMAdminRequestInterceptor interceptorChain =
             this.createRequestInterceptorChain();
         interceptorChain.init(user);
         chainWrapper.init(interceptorChain);
       } catch (Exception e) {
-        LOG.error("Init RMAdminRequestInterceptor error for user: " + user, e);
+        LOG.error("Init RMAdminRequestInterceptor error for user: {}.", user, e);
         throw e;
       }
 
@@ -308,9 +244,9 @@ public synchronized void init(RMAdminRequestInterceptor interceptor) {
     }
 
     /**
-     * Gets the root request intercepter.
+     * Gets the root request interceptor.
      *
-     * @return the root request intercepter
+     * @return the root request interceptor
      */
     public synchronized RMAdminRequestInterceptor getRootInterceptor() {
       return rootInterceptor;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java
new file mode 100644
index 0000000000000..de74c41fe7928
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.security;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A Router specific delegation token secret manager.
+ * The secret manager is responsible for generating and accepting the password
+ * for each token.
+ */
+public class RouterDelegationTokenSecretManager
+    extends AbstractDelegationTokenSecretManager<RMDelegationTokenIdentifier> {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(RouterDelegationTokenSecretManager.class);
+
+  private FederationStateStoreFacade federationFacade;
+
+  /**
+   * Create a Router Secret manager.
+   *
+   * @param delegationKeyUpdateInterval        the number of milliseconds for rolling
+   *                                           new secret keys.
+   * @param delegationTokenMaxLifetime         the maximum lifetime of the delegation
+   *                                           tokens in milliseconds
+   * @param delegationTokenRenewInterval       how often the tokens must be renewed
+   *                                           in milliseconds
+   * @param delegationTokenRemoverScanInterval how often the tokens are scanned
+   */
+  public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval,
+      long delegationTokenMaxLifetime, long delegationTokenRenewInterval,
+      long delegationTokenRemoverScanInterval) {
+    super(delegationKeyUpdateInterval, delegationTokenMaxLifetime,
+        delegationTokenRenewInterval, delegationTokenRemoverScanInterval);
+    this.federationFacade = FederationStateStoreFacade.getInstance();
+  }
+
+  @Override
+  public RMDelegationTokenIdentifier createIdentifier() {
+    return new RMDelegationTokenIdentifier();
+  }
+
+  private boolean shouldIgnoreException(Exception e) {
+    return !running && e.getCause() instanceof InterruptedException;
+  }
+
+  /**
+   * The Router Supports Store the New Master Key.
+   * During this Process, Facade will call the specific StateStore to store the MasterKey.
+   *
+   * @param newKey DelegationKey
+   */
+  @Override
+  public void storeNewMasterKey(DelegationKey newKey) {
+    try {
+      federationFacade.storeNewMasterKey(newKey);
+    } catch (Exception e) {
+      if (!shouldIgnoreException(e)) {
+        LOG.error("Error in storing master key with KeyID: {}.", newKey.getKeyId());
+        ExitUtil.terminate(1, e);
+      }
+    }
+  }
+
+  /**
+   * The Router Supports Remove the master key.
+   * During this Process, Facade will call the specific StateStore to remove the MasterKey.
+   *
+   * @param delegationKey DelegationKey
+   */
+  @Override
+  public void removeStoredMasterKey(DelegationKey delegationKey) {
+    try {
+      federationFacade.removeStoredMasterKey(delegationKey);
+    } catch (Exception e) {
+      if (!shouldIgnoreException(e)) {
+        LOG.error("Error in removing master key with KeyID: {}.", delegationKey.getKeyId());
+        ExitUtil.terminate(1, e);
+      }
+    }
+  }
+
+  /**
+   * The Router Supports Store new Token.
+   *
+   * @param identifier RMDelegationToken
+   * @param renewDate renewDate
+   * @throws IOException IO exception occurred.
+   */
+  @Override
+  public void storeNewToken(RMDelegationTokenIdentifier identifier,
+      long renewDate) throws IOException {
+    try {
+      federationFacade.storeNewToken(identifier, renewDate);
+    } catch (Exception e) {
+      if (!shouldIgnoreException(e)) {
+        LOG.error("Error in storing RMDelegationToken with sequence number: {}.",
+            identifier.getSequenceNumber());
+        ExitUtil.terminate(1, e);
+      }
+    }
+  }
+
+  /**
+   * The Router Supports Update Token.
+   *
+   * @param id RMDelegationToken
+   * @param renewDate renewDate
+   * @throws IOException IO exception occurred
+   */
+  @Override
+  public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) throws IOException {
+    try {
+      federationFacade.updateStoredToken(id, renewDate);
+    } catch (Exception e) {
+      if (!shouldIgnoreException(e)) {
+        LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.",
+            id.getSequenceNumber());
+        ExitUtil.terminate(1, e);
+      }
+    }
+  }
+
+  /**
+   * The Router Supports Remove Token.
+   *
+   * @param identifier Delegation Token
+   * @throws IOException IO exception occurred.
+   */
+  @Override
+  public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException {
+    try {
+      federationFacade.removeStoredToken(identifier);
+    } catch (Exception e) {
+      if (!shouldIgnoreException(e)) {
+        LOG.error("Error in removing RMDelegationToken with sequence number: {}",
+            identifier.getSequenceNumber());
+        ExitUtil.terminate(1, e);
+      }
+    }
+  }
+
+  /**
+   * The Router supports obtaining the DelegationKey stored in the Router StateStote
+   * according to the DelegationKey.
+   *
+   * @param key Param DelegationKey
+   * @return Delegation Token
+   * @throws YarnException An internal conversion error occurred when getting the Token
+   * @throws IOException IO exception occurred
+   */
+  public DelegationKey getMasterKeyByDelegationKey(DelegationKey key)
+      throws YarnException, IOException {
+    try {
+      RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(key);
+      RouterMasterKey masterKey = response.getRouterMasterKey();
+      ByteBuffer keyByteBuf = masterKey.getKeyBytes();
+      byte[] keyBytes = new byte[keyByteBuf.remaining()];
+      keyByteBuf.get(keyBytes);
+      DelegationKey delegationKey =
+          new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes);
+      return delegationKey;
+    } catch (IOException ex) {
+      throw new IOException(ex);
+    } catch (YarnException ex) {
+      throw new YarnException(ex);
+    }
+  }
+
+  /**
+   * Get RMDelegationTokenIdentifier according to RouterStoreToken.
+   *
+   * @param identifier RMDelegationTokenIdentifier
+   * @return RMDelegationTokenIdentifier
+   * @throws YarnException An internal conversion error occurred when getting the Token
+   * @throws IOException IO exception occurred
+   */
+  public RMDelegationTokenIdentifier getTokenByRouterStoreToken(
+      RMDelegationTokenIdentifier identifier) throws YarnException, IOException {
+    try {
+      RouterRMTokenResponse response = federationFacade.getTokenByRouterStoreToken(identifier);
+      YARNDelegationTokenIdentifier responseIdentifier =
+          response.getRouterStoreToken().getTokenIdentifier();
+      return (RMDelegationTokenIdentifier) responseIdentifier;
+    } catch (Exception ex) {
+      throw new YarnException(ex);
+    }
+  }
+
+  public void setFederationFacade(FederationStateStoreFacade federationFacade) {
+    this.federationFacade = federationFacade;
+  }
+
+  @Public
+  @VisibleForTesting
+  public int getLatestDTSequenceNumber() {
+    return delegationTokenSequenceNumber;
+  }
+
+  @Public
+  @VisibleForTesting
+  public synchronized Set<DelegationKey> getAllMasterKeys() {
+    return new HashSet<>(allKeys.values());
+  }
+
+  @Public
+  @VisibleForTesting
+  public synchronized Map<RMDelegationTokenIdentifier, Long> getAllTokens() {
+    Map<RMDelegationTokenIdentifier, Long> allTokens = new HashMap<>();
+    for (Map.Entry<RMDelegationTokenIdentifier,
+         DelegationTokenInformation> entry : currentTokens.entrySet()) {
+      RMDelegationTokenIdentifier keyIdentifier = entry.getKey();
+      DelegationTokenInformation tokenInformation = entry.getValue();
+      allTokens.put(keyIdentifier, tokenInformation.getRenewDate());
+    }
+    return allTokens;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java
similarity index 93%
rename from hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java
index 6233024467df7..16a7488c07132 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java
@@ -15,4 +15,5 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hdfs.protocolPB;
\ No newline at end of file
+
+package org.apache.hadoop.yarn.server.router.security;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
index a8a6e6bbac921..878ac75d1c77b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AboutBlock.java
@@ -18,15 +18,11 @@
 
 package org.apache.hadoop.yarn.server.router.webapp;
 
-import com.sun.jersey.api.client.Client;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.commons.lang3.time.DateFormatUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.router.Router;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.server.router.webapp.dao.RouterInfo;
 import org.apache.hadoop.yarn.webapp.view.InfoBlock;
 
 import com.google.inject.Inject;
@@ -34,62 +30,57 @@
 /**
  * About block for the Router Web UI.
  */
-public class AboutBlock extends HtmlBlock {
-
-  private static final long BYTES_IN_MB = 1024 * 1024;
+public class AboutBlock extends RouterBlock {
 
   private final Router router;
 
   @Inject
   AboutBlock(Router router, ViewContext ctx) {
-    super(ctx);
+    super(router, ctx);
     this.router = router;
   }
 
   @Override
   protected void render(Block html) {
-    Configuration conf = this.router.getConfig();
-    String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
-    Client client = RouterWebServiceUtil.createJerseyClient(conf);
 
-    ClusterMetricsInfo metrics = RouterWebServiceUtil
-        .genericForward(webAppAddress, null, ClusterMetricsInfo.class,
-            HTTPMethods.GET,
-            RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
-            conf, client);
-    boolean isEnabled = conf.getBoolean(
-        YarnConfiguration.FEDERATION_ENABLED,
-        YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
-    info("Cluster Status").
-        __("Federation Enabled", isEnabled).
-        __("Applications Submitted", metrics.getAppsSubmitted()).
-        __("Applications Pending", metrics.getAppsPending()).
-        __("Applications Running", metrics.getAppsRunning()).
-        __("Applications Failed", metrics.getAppsFailed()).
-        __("Applications Killed", metrics.getAppsKilled()).
-        __("Applications Completed", metrics.getAppsCompleted()).
-        __("Containers Allocated", metrics.getContainersAllocated()).
-        __("Containers Reserved", metrics.getReservedContainers()).
-        __("Containers Pending", metrics.getPendingContainers()).
-        __("Available Memory",
-            StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB)).
-        __("Allocated Memory",
-            StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB)).
-        __("Reserved Memory",
-            StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB)).
-        __("Total Memory",
-            StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB)).
-        __("Available VirtualCores", metrics.getAvailableVirtualCores()).
-        __("Allocated VirtualCores", metrics.getAllocatedVirtualCores()).
-        __("Reserved VirtualCores", metrics.getReservedVirtualCores()).
-        __("Total VirtualCores", metrics.getTotalVirtualCores()).
-        __("Active Nodes", metrics.getActiveNodes()).
-        __("Lost Nodes", metrics.getLostNodes()).
-        __("Available Nodes", metrics.getDecommissionedNodes()).
-        __("Unhealthy Nodes", metrics.getUnhealthyNodes()).
-        __("Rebooted Nodes", metrics.getRebootedNodes()).
-        __("Total Nodes", metrics.getTotalNodes());
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // If Yarn Federation is not enabled, the user needs to be prompted.
+    initUserHelpInformationDiv(html, isEnabled);
+
+    // Metrics Overview Table
+    html.__(MetricsOverviewTable.class);
 
+    // Init Yarn Router Basic Information
+    initYarnRouterBasicInformation(isEnabled);
+
+    // InfoBlock
     html.__(InfoBlock.class);
   }
+
+  /**
+   * Init Yarn Router Basic Infomation.
+   * @param isEnabled true, federation is enabled; false, federation is not enabled.
+   */
+  private void initYarnRouterBasicInformation(boolean isEnabled) {
+    FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+    RouterInfo routerInfo = new RouterInfo(router);
+    String lastStartTime =
+        DateFormatUtils.format(routerInfo.getStartedOn(), DATE_PATTERN);
+    try {
+      info("Yarn Router Overview").
+          __("Federation Enabled:", String.valueOf(isEnabled)).
+          __("Router ID:", routerInfo.getClusterId()).
+          __("Router state:", routerInfo.getState()).
+          __("Router SubCluster Count:", facade.getSubClusters(true).size()).
+          __("Router RMStateStore:", routerInfo.getRouterStateStore()).
+          __("Router started on:", lastStartTime).
+          __("Router version:", routerInfo.getRouterBuildVersion() +
+             " on " + routerInfo.getRouterVersionBuiltOn()).
+          __("Hadoop version:", routerInfo.getHadoopBuildVersion() +
+             " on " + routerInfo.getHadoopVersionBuiltOn());
+    } catch (YarnException e) {
+      LOG.error("initYarnRouterBasicInformation error.", e);
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java
index a2d78a479ab78..66b2495e2c9b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java
@@ -19,16 +19,21 @@
 package org.apache.hadoop.yarn.server.router.webapp;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.IOException;
 
 /**
  * Extends the RequestInterceptor class and provides common functionality which
- * can be used and/or extended by other concrete intercepter classes.
+ * can be used and/or extended by other concrete interceptor classes.
  */
 public abstract class AbstractRESTRequestInterceptor
     implements RESTRequestInterceptor {
 
   private Configuration conf;
   private RESTRequestInterceptor nextInterceptor;
+  private UserGroupInformation user = null;
 
   /**
    * Sets the {@link RESTRequestInterceptor} in the chain.
@@ -62,9 +67,10 @@ public Configuration getConf() {
    * Initializes the {@link RESTRequestInterceptor}.
    */
   @Override
-  public void init(String user) {
+  public void init(String userName) {
+    setupUser(userName);
     if (this.nextInterceptor != null) {
-      this.nextInterceptor.init(user);
+      this.nextInterceptor.init(userName);
     }
   }
 
@@ -86,4 +92,35 @@ public RESTRequestInterceptor getNextInterceptor() {
     return this.nextInterceptor;
   }
 
+  /**
+   * Set User information.
+   *
+   * If the username is empty, we will use the Yarn Router user directly.
+   * Do not create a proxy user if user name matches the user name on current UGI.
+   * @param userName userName.
+   */
+  private void setupUser(final String userName) {
+    try {
+      if (userName == null || userName.isEmpty()) {
+        user = UserGroupInformation.getCurrentUser();
+      } else if (UserGroupInformation.isSecurityEnabled()) {
+        user = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getLoginUser());
+      } else if (userName.equalsIgnoreCase(UserGroupInformation.getCurrentUser().getUserName())) {
+        user = UserGroupInformation.getCurrentUser();
+      } else {
+        user = UserGroupInformation.createProxyUser(userName,
+            UserGroupInformation.getCurrentUser());
+      }
+    } catch (IOException e) {
+      String message = "Error while creating Router RMAdmin Service for user:";
+      if (user != null) {
+        message += ", user: " + user;
+      }
+      throw new YarnRuntimeException(message, e);
+    }
+  }
+
+  public UserGroupInformation getUser() {
+    return user;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
index 87f20c81bb865..7f277ae3ae7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
@@ -21,11 +21,18 @@
 import static org.apache.commons.text.StringEscapeUtils.escapeHtml4;
 import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
 
 import com.sun.jersey.api.client.Client;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
@@ -34,34 +41,102 @@
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 
 import com.google.inject.Inject;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 /**
  * Applications block for the Router Web UI.
  */
-public class AppsBlock extends HtmlBlock {
+public class AppsBlock extends RouterBlock {
+
   private final Router router;
+  private final Configuration conf;
 
   @Inject
   AppsBlock(Router router, ViewContext ctx) {
-    super(ctx);
+    super(router, ctx);
     this.router = router;
+    this.conf = this.router.getConfig();
   }
 
   @Override
   protected void render(Block html) {
-    // Get the applications from the Resource Managers
-    Configuration conf = this.router.getConfig();
+
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // Get subClusterName
+    String subClusterName = $(APP_SC);
+    String reqState = $(APP_STATE);
+
+    // We will try to get the subClusterName.
+    // If the subClusterName is not empty,
+    // it means that we need to get the Node list of a subCluster.
+    AppsInfo appsInfo = null;
+    if (subClusterName != null && !subClusterName.isEmpty()) {
+      initSubClusterMetricsOverviewTable(html, subClusterName);
+      appsInfo = getSubClusterAppsInfo(subClusterName, reqState);
+    } else {
+      // Metrics Overview Table
+      html.__(MetricsOverviewTable.class);
+      appsInfo = getYarnFederationAppsInfo(isEnabled);
+    }
+
+    initYarnFederationAppsOfCluster(appsInfo, html);
+  }
+
+  private static String escape(String str) {
+    return escapeEcmaScript(escapeHtml4(str));
+  }
+
+  private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
+    if (isEnabled) {
+      String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
+      return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
+    }
+    return null;
+  }
+
+  private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
+      FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+
+      if (subClusterInfo != null) {
+        // Prepare webAddress
+        String webAddress = subClusterInfo.getRMWebServiceAddress();
+        String herfWebAppAddress = "";
+        if (webAddress != null && !webAddress.isEmpty()) {
+          herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
+          return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("get AppsInfo From SubCluster = {} error.", subCluster, e);
+    }
+    return null;
+  }
+
+  private AppsInfo getSubClusterAppsInfoByWebAddress(String webAddress, String states) {
     Client client = RouterWebServiceUtil.createJerseyClient(conf);
-    String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+    Map<String, String[]> queryParams = new HashMap<>();
+    if (StringUtils.isNotBlank(states)) {
+      queryParams.put("states", new String[]{states});
+    }
     AppsInfo apps = RouterWebServiceUtil
-        .genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
-            RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
-            client);
+        .genericForward(webAddress, null, AppsInfo.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, queryParams, conf,
+        client);
+    client.destroy();
+    return apps;
+  }
 
-    setTitle("Applications");
+  private void initYarnFederationAppsOfCluster(AppsInfo appsInfo, Block html) {
 
     TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
         .tr()
@@ -81,45 +156,18 @@ protected void render(Block html) {
 
     // Render the applications
     StringBuilder appsTableData = new StringBuilder("[\n");
-    for (AppInfo app : apps.getApps()) {
-      try {
-
-        String percent = String.format("%.1f", app.getProgress() * 100.0F);
-        String trackingURL =
-            app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
-        // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
-        appsTableData.append("[\"")
-            .append("<a href='").append(trackingURL).append("'>")
-            .append(app.getAppId()).append("</a>\",\"")
-            .append(escape(app.getUser())).append("\",\"")
-            .append(escape(app.getName())).append("\",\"")
-            .append(escape(app.getApplicationType())).append("\",\"")
-            .append(escape(app.getQueue())).append("\",\"")
-            .append(String.valueOf(app.getPriority())).append("\",\"")
-            .append(app.getStartTime()).append("\",\"")
-            .append(app.getFinishTime()).append("\",\"")
-            .append(app.getState()).append("\",\"")
-            .append(app.getFinalStatus()).append("\",\"")
-            // Progress bar
-            .append("<br title='").append(percent).append("'> <div class='")
-            .append(C_PROGRESSBAR).append("' title='")
-            .append(join(percent, '%')).append("'> ").append("<div class='")
-            .append(C_PROGRESSBAR_VALUE).append("' style='")
-            .append(join("width:", percent, '%')).append("'> </div> </div>")
-            // History link
-            .append("\",\"<a href='").append(trackingURL).append("'>")
-            .append("History").append("</a>");
-        appsTableData.append("\"],\n");
-
-      } catch (Exception e) {
-        LOG.info(
-            "Cannot add application {}: {}", app.getAppId(), e.getMessage());
+
+    if (appsInfo != null && CollectionUtils.isNotEmpty(appsInfo.getApps())) {
+
+      List<String> appInfoList =
+          appsInfo.getApps().stream().map(this::parseAppInfoData).collect(Collectors.toList());
+
+      if (CollectionUtils.isNotEmpty(appInfoList)) {
+        String formattedAppInfo = StringUtils.join(appInfoList, ",");
+        appsTableData.append(formattedAppInfo);
       }
     }
-    if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
-      appsTableData.delete(appsTableData.length() - 2,
-          appsTableData.length() - 1);
-    }
+
     appsTableData.append("]");
     html.script().$type("text/javascript")
         .__("var appsTableData=" + appsTableData).__();
@@ -127,7 +175,39 @@ protected void render(Block html) {
     tbody.__().__();
   }
 
-  private static String escape(String str) {
-    return escapeEcmaScript(escapeHtml4(str));
+  private String parseAppInfoData(AppInfo app) {
+    StringBuilder appsDataBuilder = new StringBuilder();
+    try {
+      String percent = String.format("%.1f", app.getProgress() * 100.0F);
+      String trackingURL = app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
+
+      // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+      appsDataBuilder.append("[\"")
+          .append("<a href='").append(trackingURL).append("'>")
+          .append(app.getAppId()).append("</a>\",\"")
+          .append(escape(app.getUser())).append("\",\"")
+          .append(escape(app.getName())).append("\",\"")
+          .append(escape(app.getApplicationType())).append("\",\"")
+          .append(escape(app.getQueue())).append("\",\"")
+          .append(app.getPriority()).append("\",\"")
+          .append(app.getStartTime()).append("\",\"")
+          .append(app.getFinishTime()).append("\",\"")
+          .append(app.getState()).append("\",\"")
+          .append(app.getFinalStatus()).append("\",\"")
+          // Progress bar
+          .append("<br title='").append(percent).append("'> <div class='")
+          .append(C_PROGRESSBAR).append("' title='")
+          .append(join(percent, '%')).append("'> ").append("<div class='")
+          .append(C_PROGRESSBAR_VALUE).append("' style='")
+          .append(join("width:", percent, '%')).append("'> </div> </div>")
+          // History link
+          .append("\",\"<a href='").append(trackingURL).append("'>")
+          .append("History").append("</a>");
+      appsDataBuilder.append("\"]\n");
+
+    } catch (Exception e) {
+      LOG.warn("Cannot add application {}: {}", app.getAppId(), e.getMessage());
+    }
+    return appsDataBuilder.toString();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
index 12d0b5b4ede65..f820aed05f218 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
@@ -20,11 +20,13 @@
 
 import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.yarn.webapp.SubView;
 
 class AppsPage extends RouterView {
@@ -37,9 +39,14 @@ protected void preHead(Page.HTML<__> html) {
     setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
 
     // Set the correct title.
+    String subClusterName = $(APP_SC);
     String reqState = $(APP_STATE);
-    reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
-    setTitle(sjoin(reqState, "Applications"));
+
+    if(StringUtils.isBlank(subClusterName)){
+      subClusterName = "Federation ";
+    }
+    reqState = (StringUtils.isBlank(reqState) ? "All" : reqState);
+    setTitle(sjoin(subClusterName, reqState,  "Applications"));
   }
 
   private String appsTableInit() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java
index 2675b38244016..918865da4675e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java
@@ -28,6 +28,7 @@
 import javax.ws.rs.core.Response;
 
 import com.sun.jersey.api.client.Client;
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -66,6 +67,7 @@
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 /**
@@ -330,7 +332,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
   }
 
   @Override
-  public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+  public Response removeFromClusterNodeLabels(Set<String> oldNodeLabels,
       HttpServletRequest hsr) throws Exception {
     // oldNodeLabels is specified inside hsr
     return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
@@ -525,10 +527,10 @@ public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
 
   @Override
   public RMQueueAclInfo checkUserAccessToQueue(String queue, String username,
-      String queueAclType, HttpServletRequest hsr) {
+      String queueAclType, HttpServletRequest hsr) throws AuthorizationException {
     return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
         RMQueueAclInfo.class, HTTPMethods.GET,
-        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.QUEUES + "/" + queue
+        RMWSConsts.RM_WEB_SERVICE_PATH +  "/" + RMWSConsts.QUEUES + "/" + queue
             + "/access", null, null, getConf(), client);
   }
 
@@ -565,6 +567,25 @@ public ContainerInfo getContainer(HttpServletRequest req,
         null, null, getConf(), client);
   }
 
+  @Override
+  public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest req)
+      throws AuthorizationException, InterruptedException {
+    return RouterWebServiceUtil.genericForward(webAppAddress, req,
+        Response.class, HTTPMethods.PUT,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF,
+        mutationInfo, null, getConf(), client);
+  }
+
+  @Override
+  public Response getSchedulerConfiguration(HttpServletRequest req)
+      throws AuthorizationException {
+    return RouterWebServiceUtil.genericForward(webAppAddress, req,
+        Response.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF,
+        null, null, getConf(), client);
+  }
+
   @Override
   public void setNextInterceptor(RESTRequestInterceptor next) {
     throw new YarnRuntimeException("setNextInterceptor is being called on "
@@ -582,4 +603,17 @@ public Response signalToContainer(String containerId, String command,
                 + containerId + "/" + RMWSConsts.SIGNAL + "/" + command, null,
             null, getConf(), client);
   }
+
+  @VisibleForTesting
+  public Client getClient() {
+    return client;
+  }
+
+  @Override
+  public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) {
+    return RouterWebServiceUtil.genericForward(webAppAddress, hsr,
+        NodeLabelsInfo.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_RM_NODE_LABELS,
+        null, null, getConf(), client);
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
index d84c705fec057..44f8d7407bf20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
@@ -20,157 +20,194 @@
 
 import java.io.StringReader;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
+import java.util.Date;
 
-import org.apache.hadoop.conf.Configuration;
+import com.google.gson.Gson;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
 import org.apache.hadoop.yarn.server.router.Router;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 import com.google.inject.Inject;
 import com.sun.jersey.api.json.JSONConfiguration;
 import com.sun.jersey.api.json.JSONJAXBContext;
 import com.sun.jersey.api.json.JSONUnmarshaller;
 
-class FederationBlock extends HtmlBlock {
-
-  private static final long BYTES_IN_MB = 1024 * 1024;
+class FederationBlock extends RouterBlock {
 
   private final Router router;
 
   @Inject
   FederationBlock(ViewContext ctx, Router router) {
-    super(ctx);
+    super(router, ctx);
     this.router = router;
   }
 
   @Override
   public void render(Block html) {
-    Configuration conf = this.router.getConfig();
-    boolean isEnabled = conf.getBoolean(
-        YarnConfiguration.FEDERATION_ENABLED,
-        YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
-    if (isEnabled) {
-      setTitle("Federation");
-
-      // Table header
-      TBODY<TABLE<Hamlet>> tbody = html.table("#rms").thead().tr()
-          .th(".id", "SubCluster")
-          .th(".submittedA", "Applications Submitted*")
-          .th(".pendingA", "Applications Pending*")
-          .th(".runningA", "Applications Running*")
-          .th(".failedA", "Applications Failed*")
-          .th(".killedA", "Applications Killed*")
-          .th(".completedA", "Applications Completed*")
-          .th(".contAllocated", "Containers Allocated")
-          .th(".contReserved", "Containers Reserved")
-          .th(".contPending", "Containers Pending")
-          .th(".availableM", "Available Memory")
-          .th(".allocatedM", "Allocated Memory")
-          .th(".reservedM", "Reserved Memory")
-          .th(".totalM", "Total Memory")
-          .th(".availableVC", "Available VirtualCores")
-          .th(".allocatedVC", "Allocated VirtualCores")
-          .th(".reservedVC", "Reserved VirtualCores")
-          .th(".totalVC", "Total VirtualCores")
-          .th(".activeN", "Active Nodes")
-          .th(".lostN", "Lost Nodes")
-          .th(".availableN", "Available Nodes")
-          .th(".unhealtyN", "Unhealthy Nodes")
-          .th(".rebootedN", "Rebooted Nodes")
-          .th(".totalN", "Total Nodes")
-          .__().__().tbody();
-
-      try {
-        // Binding to the FederationStateStore
-        FederationStateStoreFacade facade =
-            FederationStateStoreFacade.getInstance();
-        Map<SubClusterId, SubClusterInfo> subClustersInfo =
-            facade.getSubClusters(true);
-
-        // Sort the SubClusters
-        List<SubClusterInfo> subclusters = new ArrayList<>();
-        subclusters.addAll(subClustersInfo.values());
-        Comparator<? super SubClusterInfo> cmp =
-            new Comparator<SubClusterInfo>() {
-              @Override
-              public int compare(SubClusterInfo o1, SubClusterInfo o2) {
-                return o1.getSubClusterId().compareTo(o2.getSubClusterId());
-              }
-            };
-        Collections.sort(subclusters, cmp);
-
-        for (SubClusterInfo subcluster : subclusters) {
-          SubClusterId subClusterId = subcluster.getSubClusterId();
-          String webAppAddress = subcluster.getRMWebServiceAddress();
-          String capability = subcluster.getCapability();
-          ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
-
-          // Building row per SubCluster
-          tbody.tr().td().a("//" + webAppAddress, subClusterId.toString()).__()
-              .td(Integer.toString(subClusterInfo.getAppsSubmitted()))
-              .td(Integer.toString(subClusterInfo.getAppsPending()))
-              .td(Integer.toString(subClusterInfo.getAppsRunning()))
-              .td(Integer.toString(subClusterInfo.getAppsFailed()))
-              .td(Integer.toString(subClusterInfo.getAppsKilled()))
-              .td(Integer.toString(subClusterInfo.getAppsCompleted()))
-              .td(Integer.toString(subClusterInfo.getContainersAllocated()))
-              .td(Integer.toString(subClusterInfo.getReservedContainers()))
-              .td(Integer.toString(subClusterInfo.getPendingContainers()))
-              .td(StringUtils.byteDesc(
-                  subClusterInfo.getAvailableMB() * BYTES_IN_MB))
-              .td(StringUtils.byteDesc(
-                  subClusterInfo.getAllocatedMB() * BYTES_IN_MB))
-              .td(StringUtils.byteDesc(
-                  subClusterInfo.getReservedMB() * BYTES_IN_MB))
-              .td(StringUtils.byteDesc(
-                  subClusterInfo.getTotalMB() * BYTES_IN_MB))
-              .td(Long.toString(subClusterInfo.getAvailableVirtualCores()))
-              .td(Long.toString(subClusterInfo.getAllocatedVirtualCores()))
-              .td(Long.toString(subClusterInfo.getReservedVirtualCores()))
-              .td(Long.toString(subClusterInfo.getTotalVirtualCores()))
-              .td(Integer.toString(subClusterInfo.getActiveNodes()))
-              .td(Integer.toString(subClusterInfo.getLostNodes()))
-              .td(Integer.toString(subClusterInfo.getDecommissionedNodes()))
-              .td(Integer.toString(subClusterInfo.getUnhealthyNodes()))
-              .td(Integer.toString(subClusterInfo.getRebootedNodes()))
-              .td(Integer.toString(subClusterInfo.getTotalNodes())).__();
-        }
-      } catch (YarnException e) {
-        LOG.error("Cannot render ResourceManager", e);
-      }
 
-      tbody.__().__().div()
-          .p().__("*The application counts are local per subcluster").__().__();
-    } else {
-      setTitle("Federation is not Enabled!");
-    }
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // init Html Page Federation
+    initHtmlPageFederation(html, isEnabled);
   }
 
-  private static ClusterMetricsInfo getClusterMetricsInfo(String capability) {
-    ClusterMetricsInfo clusterMetrics = null;
+  /**
+   * Parse the capability and obtain the metric information of the cluster.
+   *
+   * @param capability metric json obtained from RM.
+   * @return ClusterMetricsInfo Object
+   */
+  private ClusterMetricsInfo getClusterMetricsInfo(String capability) {
     try {
-      JSONJAXBContext jc = new JSONJAXBContext(
-          JSONConfiguration.mapped().rootUnwrapping(false).build(),
-          ClusterMetricsInfo.class);
-      JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller();
-      clusterMetrics = unmarshaller.unmarshalFromJSON(
-          new StringReader(capability), ClusterMetricsInfo.class);
+      if (capability != null && !capability.isEmpty()) {
+        JSONJAXBContext jc = new JSONJAXBContext(
+            JSONConfiguration.mapped().rootUnwrapping(false).build(), ClusterMetricsInfo.class);
+        JSONUnmarshaller unmarShaller = jc.createJSONUnmarshaller();
+        StringReader stringReader = new StringReader(capability);
+        ClusterMetricsInfo clusterMetrics =
+            unmarShaller.unmarshalFromJSON(stringReader, ClusterMetricsInfo.class);
+        return clusterMetrics;
+      }
     } catch (Exception e) {
       LOG.error("Cannot parse SubCluster info", e);
     }
-    return clusterMetrics;
+    return null;
+  }
+
+  /**
+   * Initialize the subCluster details JavaScript of the Federation page.
+   *
+   * This part of the js script will control to display or hide the detailed information
+   * of the subCluster when the user clicks on the subClusterId.
+   *
+   * We will obtain the specific information of a SubCluster,
+   * including the information of Applications, Resources, and Nodes.
+   *
+   * @param html html object
+   * @param subClusterDetailMap subCluster Detail Map
+   */
+  private void initFederationSubClusterDetailTableJs(Block html,
+      List<Map<String, String>> subClusterDetailMap) {
+    Gson gson = new Gson();
+    html.script().$type("text/javascript").
+        __(" var scTableData = " + gson.toJson(subClusterDetailMap) + "; ")
+        .__();
+    html.script(root_url("static/federation/federation.js"));
+  }
+
+  /**
+   * Initialize the Html page.
+   *
+   * @param html html object
+   */
+  private void initHtmlPageFederation(Block html, boolean isEnabled) {
+    List<Map<String, String>> lists = new ArrayList<>();
+
+    // Table header
+    TBODY<TABLE<Hamlet>> tbody =
+        html.table("#rms").$class("cell-border").$style("width:100%").thead().tr()
+        .th(".id", "SubCluster")
+        .th(".state", "State")
+        .th(".lastStartTime", "LastStartTime")
+        .th(".lastHeartBeat", "LastHeartBeat")
+        .th(".resources", "Resources")
+        .th(".nodes", "Nodes")
+        .__().__().tbody();
+
+    try {
+
+      // Sort the SubClusters
+      List<SubClusterInfo> subclusters = getSubClusterInfoList();
+
+      for (SubClusterInfo subcluster : subclusters) {
+
+        Map<String, String> subclusterMap = new HashMap<>();
+
+        // Prepare subCluster
+        SubClusterId subClusterId = subcluster.getSubClusterId();
+        String subClusterIdText = subClusterId.getId();
+
+        // Prepare WebAppAddress
+        String webAppAddress = subcluster.getRMWebServiceAddress();
+        String herfWebAppAddress = "";
+        if (webAppAddress != null && !webAppAddress.isEmpty()) {
+          herfWebAppAddress =
+              WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAppAddress;
+        }
+
+        // Prepare Capability
+        String capability = subcluster.getCapability();
+        ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
+
+        // Prepare LastStartTime & LastHeartBeat
+        Date lastStartTime = new Date(subcluster.getLastStartTime());
+        Date lastHeartBeat = new Date(subcluster.getLastHeartBeat());
+
+        // Prepare Resource
+        long totalMB = subClusterInfo.getTotalMB();
+        String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB);
+        long totalVirtualCores = subClusterInfo.getTotalVirtualCores();
+        String resources = String.format("<memory:%s, vCores:%s>", totalMBDesc, totalVirtualCores);
+
+        // Prepare Node
+        long totalNodes = subClusterInfo.getTotalNodes();
+        long activeNodes = subClusterInfo.getActiveNodes();
+        String nodes = String.format("<totalNodes:%s, activeNodes:%s>", totalNodes, activeNodes);
+
+        // Prepare HTML Table
+        String stateStyle = "color:#dc3545;font-weight:bolder";
+        SubClusterState state = subcluster.getState();
+        if (SubClusterState.SC_RUNNING == state) {
+          stateStyle = "color:#28a745;font-weight:bolder";
+        }
+
+        tbody.tr().$id(subClusterIdText)
+            .td().$class("details-control").a(herfWebAppAddress, subClusterIdText).__()
+            .td().$style(stateStyle).__(state.name()).__()
+            .td().__(lastStartTime).__()
+            .td().__(lastHeartBeat).__()
+            .td(resources)
+            .td(nodes)
+            .__();
+
+        // Formatted memory information
+        long allocatedMB = subClusterInfo.getAllocatedMB();
+        String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB);
+        long availableMB = subClusterInfo.getAvailableMB();
+        String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB);
+        long pendingMB = subClusterInfo.getPendingMB();
+        String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB);
+        long reservedMB = subClusterInfo.getReservedMB();
+        String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB);
+
+        subclusterMap.put("totalmemory", totalMBDesc);
+        subclusterMap.put("allocatedmemory", allocatedMBDesc);
+        subclusterMap.put("availablememory", availableMBDesc);
+        subclusterMap.put("pendingmemory", pendingMBDesc);
+        subclusterMap.put("reservedmemory", reservedMBDesc);
+        subclusterMap.put("subcluster", subClusterId.getId());
+        subclusterMap.put("capability", capability);
+        lists.add(subclusterMap);
+      }
+    } catch (Exception e) {
+      LOG.error("Cannot render Router Federation.", e);
+    }
+
+    // Init FederationBlockTableJs
+    initFederationSubClusterDetailTableJs(html, lists);
+
+    // Tips
+    tbody.__().__().div().p().$style("color:red")
+        .__("*The application counts are local per subcluster").__().__();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
index 14e062ad4e985..a21be7b4e1b76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.router.webapp;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.security.Principal;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -26,13 +27,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletRequestWrapper;
@@ -42,23 +43,32 @@
 import javax.ws.rs.core.Response.Status;
 
 import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
 import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
-import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.retry.FederationActionRetry;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
@@ -90,11 +100,17 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
 import org.apache.hadoop.yarn.server.router.RouterMetrics;
 import org.apache.hadoop.yarn.server.router.RouterServerUtil;
+import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod;
+import org.apache.hadoop.yarn.server.router.webapp.cache.RouterAppInfoCacheKey;
+import org.apache.hadoop.yarn.server.router.webapp.dao.FederationRMQueueAclInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.util.LRUCacheHashMap;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -109,7 +125,7 @@
  * Extends the {@code AbstractRESTRequestInterceptor} class and provides an
  * implementation for federation of YARN RM and scaling an application across
  * multiple YARN SubClusters. All the federation specific implementation is
- * encapsulated in this class. This is always the last intercepter in the chain.
+ * encapsulated in this class. This is always the last interceptor in the chain.
  */
 public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
 
@@ -118,13 +134,16 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
 
   private int numSubmitRetries;
   private FederationStateStoreFacade federationFacade;
-  private Random rand;
   private RouterPolicyFacade policyFacade;
   private RouterMetrics routerMetrics;
   private final Clock clock = new MonotonicClock();
   private boolean returnPartialReport;
+  private boolean appInfosCacheEnabled;
+  private int appInfosCacheCount;
+  private long submitIntervalTime;
 
   private Map<SubClusterId, DefaultRequestInterceptorREST> interceptors;
+  private LRUCacheHashMap<RouterAppInfoCacheKey, AppsInfo> appInfosCaches;
 
   /**
    * Thread pool used for asynchronous operations.
@@ -133,8 +152,10 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
 
   @Override
   public void init(String user) {
+
+    super.init(user);
+
     federationFacade = FederationStateStoreFacade.getInstance();
-    rand = new Random();
 
     final Configuration conf = this.getConf();
 
@@ -161,30 +182,21 @@ public void init(String user) {
     returnPartialReport = conf.getBoolean(
         YarnConfiguration.ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED,
         YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED);
-  }
-
-  private SubClusterId getRandomActiveSubCluster(
-      Map<SubClusterId, SubClusterInfo> activeSubclusters,
-      List<SubClusterId> blackListSubClusters) throws YarnException {
-
-    if (activeSubclusters == null || activeSubclusters.size() < 1) {
-      RouterServerUtil.logAndThrowException(
-          FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE, null);
-    }
-    List<SubClusterId> list = new ArrayList<>(activeSubclusters.keySet());
-
-    FederationPolicyUtils.validateSubClusterAvailability(
-        list, blackListSubClusters);
 
-    if (blackListSubClusters != null) {
+    appInfosCacheEnabled = conf.getBoolean(
+        YarnConfiguration.ROUTER_APPSINFO_ENABLED,
+        YarnConfiguration.DEFAULT_ROUTER_APPSINFO_ENABLED);
 
-      // Remove from the active SubClusters from StateStore the blacklisted ones
-      for (SubClusterId scId : blackListSubClusters) {
-        list.remove(scId);
-      }
+    if(appInfosCacheEnabled) {
+      appInfosCacheCount = conf.getInt(
+          YarnConfiguration.ROUTER_APPSINFO_CACHED_COUNT,
+          YarnConfiguration.DEFAULT_ROUTER_APPSINFO_CACHED_COUNT);
+      appInfosCaches = new LRUCacheHashMap<>(appInfosCacheCount, true);
     }
 
-    return list.get(rand.nextInt(list.size()));
+    submitIntervalTime = conf.getTimeDuration(
+        YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_INTERVAL_TIME,
+        YarnConfiguration.DEFAULT_CLIENTRM_SUBMIT_INTERVAL_TIME, TimeUnit.MILLISECONDS);
   }
 
   @VisibleForTesting
@@ -215,7 +227,8 @@ private DefaultRequestInterceptorREST createInterceptorForSubCluster(
           .isAssignableFrom(interceptorClass)) {
         interceptorInstance = (DefaultRequestInterceptorREST) ReflectionUtils
             .newInstance(interceptorClass, conf);
-
+        String userName = getUser().getUserName();
+        interceptorInstance.init(userName);
       } else {
         throw new YarnRuntimeException(
             "Class: " + interceptorClassName + " not instance of "
@@ -228,7 +241,9 @@ private DefaultRequestInterceptorREST createInterceptorForSubCluster(
           e);
     }
 
-    interceptorInstance.setWebAppAddress("http://" + webAppAddress);
+    String webAppAddresswithScheme =
+        WebAppUtils.getHttpSchemePrefix(this.getConf()) + webAppAddress;
+    interceptorInstance.setWebAppAddress(webAppAddresswithScheme);
     interceptorInstance.setSubClusterId(subClusterId);
     interceptors.put(subClusterId, interceptorInstance);
     return interceptorInstance;
@@ -271,62 +286,79 @@ public Response createNewApplication(HttpServletRequest hsr)
 
     long startTime = clock.getTime();
 
-    Map<SubClusterId, SubClusterInfo> subClustersActive;
     try {
-      subClustersActive = federationFacade.getSubClusters(true);
-    } catch (YarnException e) {
+      Map<SubClusterId, SubClusterInfo> subClustersActive =
+          federationFacade.getSubClusters(true);
+
+      // We declare blackList and retries.
+      List<SubClusterId> blackList = new ArrayList<>();
+      int actualRetryNums = federationFacade.getRetryNumbers(numSubmitRetries);
+      Response response = ((FederationActionRetry<Response>) (retryCount) ->
+          invokeGetNewApplication(subClustersActive, blackList, hsr, retryCount)).
+          runWithRetries(actualRetryNums, submitIntervalTime);
+
+      // If the response is not empty and the status is SC_OK,
+      // this request can be returned directly.
+      if (response != null && response.getStatus() == HttpServletResponse.SC_OK) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededAppsCreated(stopTime - startTime);
+        return response;
+      }
+    } catch (FederationPolicyException e) {
+      // If a FederationPolicyException is thrown, the service is unavailable.
+      routerMetrics.incrAppsFailedCreated();
+      return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build();
+    } catch (Exception e) {
       routerMetrics.incrAppsFailedCreated();
-      return Response.status(Status.INTERNAL_SERVER_ERROR)
-          .entity(e.getLocalizedMessage()).build();
+      return Response.status(Status.INTERNAL_SERVER_ERROR).entity(e.getLocalizedMessage()).build();
     }
 
-    List<SubClusterId> blacklist = new ArrayList<SubClusterId>();
+    // return error message directly.
+    String errMsg = "Fail to create a new application.";
+    LOG.error(errMsg);
+    routerMetrics.incrAppsFailedCreated();
+    return Response.status(Status.INTERNAL_SERVER_ERROR).entity(errMsg).build();
+  }
 
-    for (int i = 0; i < numSubmitRetries; ++i) {
+  /**
+   * Invoke GetNewApplication to different subClusters.
+   *
+   * @param subClustersActive Active SubClusters.
+   * @param blackList Blacklist avoid repeated calls to unavailable subCluster.
+   * @param hsr HttpServletRequest.
+   * @param retryCount number of retries.
+   * @return Get response, If the response is empty or status not equal SC_OK, the request fails,
+   * if the response is not empty and status equal SC_OK, the request is successful.
+   * @throws YarnException yarn exception.
+   * @throws IOException io error.
+   * @throws InterruptedException interrupted exception.
+   */
+  private Response invokeGetNewApplication(Map<SubClusterId, SubClusterInfo> subClustersActive,
+      List<SubClusterId> blackList, HttpServletRequest hsr, int retryCount)
+      throws YarnException, IOException, InterruptedException {
 
-      SubClusterId subClusterId;
-      try {
-        subClusterId = getRandomActiveSubCluster(subClustersActive, blacklist);
-      } catch (YarnException e) {
-        routerMetrics.incrAppsFailedCreated();
-        return Response.status(Status.SERVICE_UNAVAILABLE)
-            .entity(e.getLocalizedMessage()).build();
-      }
+    SubClusterId subClusterId =
+        federationFacade.getRandomActiveSubCluster(subClustersActive, blackList);
 
-      LOG.debug("getNewApplication try #{} on SubCluster {}", i, subClusterId);
+    LOG.info("getNewApplication try #{} on SubCluster {}.", retryCount, subClusterId);
 
-      DefaultRequestInterceptorREST interceptor =
-          getOrCreateInterceptorForSubCluster(subClusterId,
-              subClustersActive.get(subClusterId).getRMWebServiceAddress());
-      Response response = null;
-      try {
-        response = interceptor.createNewApplication(hsr);
-      } catch (Exception e) {
-        LOG.warn("Unable to create a new ApplicationId in SubCluster {}",
-            subClusterId.getId(), e);
-      }
-
-      if (response != null &&
-          response.getStatus() == HttpServletResponse.SC_OK) {
-
-        long stopTime = clock.getTime();
-        routerMetrics.succeededAppsCreated(stopTime - startTime);
+    DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(subClusterId,
+        subClustersActive.get(subClusterId).getRMWebServiceAddress());
 
+    try {
+      Response response = interceptor.createNewApplication(hsr);
+      if (response != null && response.getStatus() == HttpServletResponse.SC_OK) {
         return response;
-      } else {
-        // Empty response from the ResourceManager.
-        // Blacklist this subcluster for this request.
-        blacklist.add(subClusterId);
       }
+    } catch (Exception e) {
+      blackList.add(subClusterId);
+      RouterServerUtil.logAndThrowException(e.getMessage(), e);
     }
 
-    String errMsg = "Fail to create a new application.";
-    LOG.error(errMsg);
-    routerMetrics.incrAppsFailedCreated();
-    return Response
-        .status(Status.INTERNAL_SERVER_ERROR)
-        .entity(errMsg)
-        .build();
+    // We need to throw the exception directly.
+    String msg = String.format("Unable to create a new ApplicationId in SubCluster %s.",
+        subClusterId.getId());
+    throw new YarnException(msg);
   }
 
   /**
@@ -401,142 +433,106 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
 
     long startTime = clock.getTime();
 
+    // We verify the parameters to ensure that newApp is not empty and
+    // that the format of applicationId is correct.
     if (newApp == null || newApp.getApplicationId() == null) {
       routerMetrics.incrAppsFailedSubmitted();
       String errMsg = "Missing ApplicationSubmissionContextInfo or "
-          + "applicationSubmissionContex information.";
-      return Response
-          .status(Status.BAD_REQUEST)
-          .entity(errMsg)
-          .build();
+          + "applicationSubmissionContext information.";
+      return Response.status(Status.BAD_REQUEST).entity(errMsg).build();
     }
 
-    ApplicationId applicationId = null;
     try {
-      applicationId = ApplicationId.fromString(newApp.getApplicationId());
+      String applicationId = newApp.getApplicationId();
+      RouterServerUtil.validateApplicationId(applicationId);
     } catch (IllegalArgumentException e) {
       routerMetrics.incrAppsFailedSubmitted();
-      return Response
-          .status(Status.BAD_REQUEST)
-          .entity(e.getLocalizedMessage())
-          .build();
+      return Response.status(Status.BAD_REQUEST).entity(e.getLocalizedMessage()).build();
     }
 
-    List<SubClusterId> blacklist = new ArrayList<SubClusterId>();
-
-    for (int i = 0; i < numSubmitRetries; ++i) {
-
-      ApplicationSubmissionContext context =
-          RMWebAppUtil.createAppSubmissionContext(newApp, this.getConf());
-
-      SubClusterId subClusterId = null;
-      try {
-        subClusterId = policyFacade.getHomeSubcluster(context, blacklist);
-      } catch (YarnException e) {
-        routerMetrics.incrAppsFailedSubmitted();
-        return Response
-            .status(Status.SERVICE_UNAVAILABLE)
-            .entity(e.getLocalizedMessage())
-            .build();
-      }
-      LOG.info("submitApplication appId {} try #{} on SubCluster {}",
-          applicationId, i, subClusterId);
-
-      ApplicationHomeSubCluster appHomeSubCluster =
-          ApplicationHomeSubCluster.newInstance(applicationId, subClusterId);
-
-      if (i == 0) {
-        try {
-          // persist the mapping of applicationId and the subClusterId which has
-          // been selected as its home
-          subClusterId =
-              federationFacade.addApplicationHomeSubCluster(appHomeSubCluster);
-        } catch (YarnException e) {
-          routerMetrics.incrAppsFailedSubmitted();
-          String errMsg = "Unable to insert the ApplicationId " + applicationId
-              + " into the FederationStateStore";
-          return Response
-              .status(Status.SERVICE_UNAVAILABLE)
-              .entity(errMsg + " " + e.getLocalizedMessage())
-              .build();
-        }
-      } else {
-        try {
-          // update the mapping of applicationId and the home subClusterId to
-          // the new subClusterId we have selected
-          federationFacade.updateApplicationHomeSubCluster(appHomeSubCluster);
-        } catch (YarnException e) {
-          String errMsg = "Unable to update the ApplicationId " + applicationId
-              + " into the FederationStateStore";
-          SubClusterId subClusterIdInStateStore;
-          try {
-            subClusterIdInStateStore =
-                federationFacade.getApplicationHomeSubCluster(applicationId);
-          } catch (YarnException e1) {
-            routerMetrics.incrAppsFailedSubmitted();
-            return Response
-                .status(Status.SERVICE_UNAVAILABLE)
-                .entity(e1.getLocalizedMessage())
-                .build();
-          }
-          if (subClusterId == subClusterIdInStateStore) {
-            LOG.info("Application {} already submitted on SubCluster {}",
-                applicationId, subClusterId);
-          } else {
-            routerMetrics.incrAppsFailedSubmitted();
-            return Response
-                .status(Status.SERVICE_UNAVAILABLE)
-                .entity(errMsg)
-                .build();
-          }
-        }
-      }
-
-      SubClusterInfo subClusterInfo;
-      try {
-        subClusterInfo = federationFacade.getSubCluster(subClusterId);
-      } catch (YarnException e) {
-        routerMetrics.incrAppsFailedSubmitted();
-        return Response
-            .status(Status.SERVICE_UNAVAILABLE)
-            .entity(e.getLocalizedMessage())
-            .build();
-      }
-
-      Response response = null;
-      try {
-        response = getOrCreateInterceptorForSubCluster(subClusterId,
-            subClusterInfo.getRMWebServiceAddress()).submitApplication(newApp,
-                hsr);
-      } catch (Exception e) {
-        LOG.warn("Unable to submit the application {} to SubCluster {}",
-            applicationId, subClusterId.getId(), e);
-      }
-
-      if (response != null &&
-          response.getStatus() == HttpServletResponse.SC_ACCEPTED) {
-        LOG.info("Application {} with appId {} submitted on {}",
-            context.getApplicationName(), applicationId, subClusterId);
-
+    List<SubClusterId> blackList = new ArrayList<>();
+    try {
+      int activeSubClustersCount = federationFacade.getActiveSubClustersCount();
+      int actualRetryNums = Math.min(activeSubClustersCount, numSubmitRetries);
+      Response response = ((FederationActionRetry<Response>) (retryCount) ->
+          invokeSubmitApplication(newApp, blackList, hsr, retryCount)).
+          runWithRetries(actualRetryNums, submitIntervalTime);
+      if (response != null) {
         long stopTime = clock.getTime();
         routerMetrics.succeededAppsSubmitted(stopTime - startTime);
-
         return response;
-      } else {
-        // Empty response from the ResourceManager.
-        // Blacklist this subcluster for this request.
-        blacklist.add(subClusterId);
       }
+    } catch (Exception e) {
+      routerMetrics.incrAppsFailedSubmitted();
+      return Response.status(Status.SERVICE_UNAVAILABLE).entity(e.getLocalizedMessage()).build();
     }
 
     routerMetrics.incrAppsFailedSubmitted();
-    String errMsg = "Application " + newApp.getApplicationName()
-        + " with appId " + applicationId + " failed to be submitted.";
+    String errMsg = String.format("Application %s with appId %s failed to be submitted.",
+        newApp.getApplicationName(), newApp.getApplicationId());
     LOG.error(errMsg);
-    return Response
-        .status(Status.SERVICE_UNAVAILABLE)
-        .entity(errMsg)
-        .build();
+    return Response.status(Status.SERVICE_UNAVAILABLE).entity(errMsg).build();
+  }
+
+  /**
+   * Invoke SubmitApplication to different subClusters.
+   *
+   * @param submissionContext application submission context.
+   * @param blackList Blacklist avoid repeated calls to unavailable subCluster.
+   * @param hsr HttpServletRequest.
+   * @param retryCount number of retries.
+   * @return Get response, If the response is empty or status not equal SC_ACCEPTED,
+   * the request fails, if the response is not empty and status equal SC_OK,
+   * the request is successful.
+   * @throws YarnException yarn exception.
+   * @throws IOException io error.
+   */
+  private Response invokeSubmitApplication(ApplicationSubmissionContextInfo submissionContext,
+      List<SubClusterId> blackList, HttpServletRequest hsr, int retryCount)
+      throws YarnException, IOException, InterruptedException {
+
+    // Step1. We convert ApplicationSubmissionContextInfo to ApplicationSubmissionContext
+    // and Prepare parameters.
+    ApplicationSubmissionContext context =
+        RMWebAppUtil.createAppSubmissionContext(submissionContext, this.getConf());
+    ApplicationId applicationId = ApplicationId.fromString(submissionContext.getApplicationId());
+    SubClusterId subClusterId = null;
+
+    try {
+      // Get subClusterId from policy.
+      subClusterId = policyFacade.getHomeSubcluster(context, blackList);
+
+      // Print the log of submitting the submitApplication.
+      LOG.info("submitApplication appId {} try #{} on SubCluster {}.",
+          applicationId, retryCount, subClusterId);
+
+      // Step2. We Store the mapping relationship
+      // between Application and HomeSubCluster in stateStore.
+      federationFacade.addOrUpdateApplicationHomeSubCluster(
+          applicationId, subClusterId, retryCount);
+
+      // Step3. We get subClusterInfo based on subClusterId.
+      SubClusterInfo subClusterInfo = federationFacade.getSubCluster(subClusterId);
+
+      // Step4. Submit the request, if the response is HttpServletResponse.SC_ACCEPTED,
+      // We return the response, otherwise we throw an exception.
+      Response response = getOrCreateInterceptorForSubCluster(subClusterId,
+          subClusterInfo.getRMWebServiceAddress()).submitApplication(submissionContext, hsr);
+      if (response != null && response.getStatus() == HttpServletResponse.SC_ACCEPTED) {
+        LOG.info("Application {} with appId {} submitted on {}.",
+            context.getApplicationName(), applicationId, subClusterId);
+        return response;
+      }
+      String msg = String.format("application %s failed to be submitted.", applicationId);
+      throw new YarnException(msg);
+    } catch (Exception e) {
+      LOG.warn("Unable to submit the application {} to SubCluster {}.", applicationId,
+          subClusterId, e);
+      if (subClusterId != null) {
+        blackList.add(subClusterId);
+      }
+      throw e;
+    }
   }
 
   /**
@@ -665,7 +661,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr,
    * <p>
    * ResourceManager: the Router calls each YARN RM in parallel by using one
    * thread for each YARN RM. In case a YARN RM fails, a single call will
-   * timeout. However the Router will merge the ApplicationReports it got, and
+   * timeout. However, the Router will merge the ApplicationReports it got, and
    * provides a partial list to the client.
    * <p>
    * State Store: the Router will timeout and it will retry depending on the
@@ -678,6 +674,18 @@ public AppsInfo getApps(HttpServletRequest hsr, String stateQuery,
       String queueQuery, String count, String startedBegin, String startedEnd,
       String finishBegin, String finishEnd, Set<String> applicationTypes,
       Set<String> applicationTags, String name, Set<String> unselectedFields) {
+
+    RouterAppInfoCacheKey routerAppInfoCacheKey = RouterAppInfoCacheKey.newInstance(
+        hsr, stateQuery, statesQuery, finalStatusQuery, userQuery, queueQuery, count,
+        startedBegin, startedEnd, finishBegin, finishEnd, applicationTypes,
+        applicationTags, name, unselectedFields);
+
+    if (appInfosCacheEnabled && routerAppInfoCacheKey != null) {
+      if (appInfosCaches.containsKey(routerAppInfoCacheKey)) {
+        return appInfosCaches.get(routerAppInfoCacheKey);
+      }
+    }
+
     AppsInfo apps = new AppsInfo();
     long startTime = clock.getTime();
 
@@ -710,8 +718,7 @@ public AppsInfo call() {
 
           if (rmApps == null) {
             routerMetrics.incrMultipleAppsFailedRetrieved();
-            LOG.error("Subcluster {} failed to return appReport.",
-                info.getSubClusterId());
+            LOG.error("Subcluster {} failed to return appReport.", info.getSubClusterId());
             return null;
           }
           return rmApps;
@@ -742,8 +749,14 @@ public AppsInfo call() {
     }
 
     // Merge all the application reports got from all the available YARN RMs
-    return RouterWebServiceUtil.mergeAppsInfo(
+    AppsInfo resultAppsInfo = RouterWebServiceUtil.mergeAppsInfo(
         apps.getApps(), returnPartialReport);
+
+    if (appInfosCacheEnabled && routerAppInfoCacheKey != null) {
+      appInfosCaches.put(routerAppInfoCacheKey, resultAppsInfo);
+    }
+
+    return resultAppsInfo;
   }
 
   /**
@@ -871,8 +884,7 @@ private Map<SubClusterInfo, NodeInfo> getNode(
                   subclusterId, subcluster.getRMWebServiceAddress());
           return interceptor.getNode(nodeId);
         } catch (Exception e) {
-          LOG.error("Subcluster {} failed to return nodeInfo.",
-              subclusterId);
+          LOG.error("Subcluster {} failed to return nodeInfo.", subclusterId, e);
           return null;
         }
       });
@@ -940,7 +952,7 @@ private SubClusterInfo getNodeSubcluster(String nodeId)
    * <p>
    * ResourceManager: the Router calls each YARN RM in parallel by using one
    * thread for each YARN RM. In case a YARN RM fails, a single call will
-   * timeout. However the Router will use the NodesInfo it got, and provides a
+   * timeout. However, the Router will use the NodesInfo it got, and provides a
    * partial list to the client.
    * <p>
    * State Store: the Router will timeout and it will retry depending on the
@@ -951,58 +963,28 @@ private SubClusterInfo getNodeSubcluster(String nodeId)
   public NodesInfo getNodes(String states) {
 
     NodesInfo nodes = new NodesInfo();
-
-    final Map<SubClusterId, SubClusterInfo> subClustersActive;
     try {
-      subClustersActive = getActiveSubclusters();
-    } catch (Exception e) {
-      LOG.error("Cannot get nodes: {}", e.getMessage());
-      return new NodesInfo();
-    }
-
-    // Send the requests in parallel
-    CompletionService<NodesInfo> compSvc =
-        new ExecutorCompletionService<NodesInfo>(this.threadpool);
-
-    for (final SubClusterInfo info : subClustersActive.values()) {
-      compSvc.submit(new Callable<NodesInfo>() {
-        @Override
-        public NodesInfo call() {
-          DefaultRequestInterceptorREST interceptor =
-              getOrCreateInterceptorForSubCluster(
-                  info.getSubClusterId(), info.getRMWebServiceAddress());
-          try {
-            NodesInfo nodesInfo = interceptor.getNodes(states);
-            return nodesInfo;
-          } catch (Exception e) {
-            LOG.error("Subcluster {} failed to return nodesInfo.",
-                info.getSubClusterId());
-            return null;
-          }
-        }
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      Class[] argsClasses = new Class[]{String.class};
+      Object[] args = new Object[]{states};
+      ClientMethod remoteMethod = new ClientMethod("getNodes", argsClasses, args);
+      Map<SubClusterInfo, NodesInfo> nodesMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, NodesInfo.class);
+      nodesMap.values().stream().forEach(nodesInfo -> {
+        nodes.addAll(nodesInfo.getNodes());
       });
-    }
-
-    // Collect all the responses in parallel
-
-    for (int i = 0; i < subClustersActive.size(); i++) {
-      try {
-        Future<NodesInfo> future = compSvc.take();
-        NodesInfo nodesResponse = future.get();
-
-        if (nodesResponse != null) {
-          nodes.addAll(nodesResponse.getNodes());
-        }
-      } catch (Throwable e) {
-        LOG.warn("Failed to get nodes report ", e);
-      }
+    } catch (NotFoundException e) {
+      LOG.error("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      LOG.error("getNodes by states = {} error.", states, e);
+    } catch (IOException e) {
+      LOG.error("getNodes by states = {} error with io error.", states, e);
     }
 
     // Delete duplicate from all the node reports got from all the available
     // YARN RMs. Nodes can be moved from one subclusters to another. In this
     // operation they result LOST/RUNNING in the previous SubCluster and
     // NEW/RUNNING in the new one.
-
     return RouterWebServiceUtil.deleteDuplicateNodesInfo(nodes.getNodes());
   }
 
@@ -1053,7 +1035,6 @@ public ClusterMetricsInfo call() {
     }
 
     // Collect all the responses in parallel
-
     for (int i = 0; i < subClustersActive.size(); i++) {
       try {
         Future<ClusterMetricsInfo> future = compSvc.take();
@@ -1159,25 +1140,181 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr,
       String appId, String time, Set<String> requestPriorities,
       Set<String> allocationRequestIds, String groupBy, String limit,
       Set<String> actions, boolean summarize) {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Only verify the app_id,
+    // because the specific subCluster needs to be found according to the app_id,
+    // and other verifications are directly handed over to the corresponding subCluster RM
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppActivitiesFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      final HttpServletRequest hsrCopy = clone(hsr);
+      AppActivitiesInfo appActivitiesInfo = interceptor.getAppActivities(hsrCopy, appId, time,
+          requestPriorities, allocationRequestIds, groupBy, limit, actions, summarize);
+      if (appActivitiesInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppActivitiesRetrieved(stopTime - startTime);
+        return appActivitiesInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppActivitiesFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get subCluster by appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppActivitiesFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "getAppActivities by appId = %s error .", appId);
+    }
+    routerMetrics.incrGetAppActivitiesFailedRetrieved();
+    throw new RuntimeException("getAppActivities Failed.");
   }
 
   @Override
   public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
       Set<String> stateQueries, Set<String> typeQueries) {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{HttpServletRequest.class, Set.class, Set.class};
+      Object[] args = new Object[]{hsrCopy, stateQueries, typeQueries};
+      ClientMethod remoteMethod = new ClientMethod("getAppStatistics", argsClasses, args);
+      Map<SubClusterInfo, ApplicationStatisticsInfo> appStatisticsMap = invokeConcurrent(
+          subClustersActive.values(), remoteMethod, ApplicationStatisticsInfo.class);
+      ApplicationStatisticsInfo applicationStatisticsInfo  =
+          RouterWebServiceUtil.mergeApplicationStatisticsInfo(appStatisticsMap.values());
+      if (applicationStatisticsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppStatisticsRetrieved(stopTime - startTime);
+        return applicationStatisticsInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrGetAppStatisticsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("get all active sub cluster(s) error.", e);
+    } catch (IOException e) {
+      routerMetrics.incrGetAppStatisticsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "getAppStatistics error by stateQueries = %s, typeQueries = %s with io error.",
+          StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ","));
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppStatisticsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "getAppStatistics by stateQueries = %s, typeQueries = %s with yarn error.",
+          StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ","));
+    }
+    routerMetrics.incrGetAppStatisticsFailedRetrieved();
+    throw RouterServerUtil.logAndReturnRunTimeException(
+        "getAppStatistics by stateQueries = %s, typeQueries = %s Failed.",
+        StringUtils.join(stateQueries, ","), StringUtils.join(typeQueries, ","));
   }
 
   @Override
   public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr)
       throws IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{HttpServletRequest.class};
+      Object[] args = new Object[]{hsrCopy};
+      ClientMethod remoteMethod = new ClientMethod("getNodeToLabels", argsClasses, args);
+      Map<SubClusterInfo, NodeToLabelsInfo> nodeToLabelsInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, NodeToLabelsInfo.class);
+      NodeToLabelsInfo nodeToLabelsInfo =
+          RouterWebServiceUtil.mergeNodeToLabels(nodeToLabelsInfoMap);
+      if (nodeToLabelsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetNodeToLabelsRetrieved(stopTime - startTime);
+        return nodeToLabelsInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrNodeToLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      routerMetrics.incrNodeToLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("getNodeToLabels error.", e);
+    }
+    routerMetrics.incrNodeToLabelsFailedRetrieved();
+    throw new RuntimeException("getNodeToLabels Failed.");
+  }
+
+  @Override
+  public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) throws IOException {
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{HttpServletRequest.class};
+      Object[] args = new Object[]{hsrCopy};
+      ClientMethod remoteMethod = new ClientMethod("getRMNodeLabels", argsClasses, args);
+      Map<SubClusterInfo, NodeLabelsInfo> nodeToLabelsInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, NodeLabelsInfo.class);
+      NodeLabelsInfo nodeToLabelsInfo =
+          RouterWebServiceUtil.mergeNodeLabelsInfo(nodeToLabelsInfoMap);
+      if (nodeToLabelsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetRMNodeLabelsRetrieved(stopTime - startTime);
+        return nodeToLabelsInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrGetRMNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      routerMetrics.incrGetRMNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("getRMNodeLabels error.", e);
+    }
+    routerMetrics.incrGetRMNodeLabelsFailedRetrieved();
+    throw new RuntimeException("getRMNodeLabels Failed.");
   }
 
   @Override
   public LabelsToNodesInfo getLabelsToNodes(Set<String> labels)
       throws IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      Class[] argsClasses = new Class[]{Set.class};
+      Object[] args = new Object[]{labels};
+      ClientMethod remoteMethod = new ClientMethod("getLabelsToNodes", argsClasses, args);
+      Map<SubClusterInfo, LabelsToNodesInfo> labelsToNodesInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, LabelsToNodesInfo.class);
+      Map<NodeLabelInfo, NodeIDsInfo> labelToNodesMap = new HashMap<>();
+      labelsToNodesInfoMap.values().forEach(labelsToNode -> {
+        Map<NodeLabelInfo, NodeIDsInfo> values = labelsToNode.getLabelsToNodes();
+        for (Map.Entry<NodeLabelInfo, NodeIDsInfo> item : values.entrySet()) {
+          NodeLabelInfo key = item.getKey();
+          NodeIDsInfo leftValue = item.getValue();
+          NodeIDsInfo rightValue = labelToNodesMap.getOrDefault(key, null);
+          NodeIDsInfo newValue = NodeIDsInfo.add(leftValue, rightValue);
+          labelToNodesMap.put(key, newValue);
+        }
+      });
+      LabelsToNodesInfo labelsToNodesInfo = new LabelsToNodesInfo(labelToNodesMap);
+      if (labelsToNodesInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime);
+        return labelsToNodesInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException(
+          e, "getLabelsToNodes by labels = %s with yarn error.", StringUtils.join(labels, ","));
+    }
+    routerMetrics.incrLabelsToNodesFailedRetrieved();
+    throw RouterServerUtil.logAndReturnRunTimeException(
+        "getLabelsToNodes by labels = %s Failed.", StringUtils.join(labels, ","));
   }
 
   @Override
@@ -1195,7 +1332,32 @@ public Response replaceLabelsOnNode(Set<String> newNodeLabelsName,
   @Override
   public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr)
       throws IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{HttpServletRequest.class};
+      Object[] args = new Object[]{hsrCopy};
+      ClientMethod remoteMethod = new ClientMethod("getClusterNodeLabels", argsClasses, args);
+      Map<SubClusterInfo, NodeLabelsInfo> nodeToLabelsInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, NodeLabelsInfo.class);
+      Set<NodeLabel> hashSets = Sets.newHashSet();
+      nodeToLabelsInfoMap.values().forEach(item -> hashSets.addAll(item.getNodeLabels()));
+      NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo(hashSets);
+      if (nodeLabelsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetClusterNodeLabelsRetrieved(stopTime - startTime);
+        return nodeLabelsInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrClusterNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      routerMetrics.incrClusterNodeLabelsFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("getClusterNodeLabels with yarn error.", e);
+    }
+    routerMetrics.incrClusterNodeLabelsFailedRetrieved();
+    throw new RuntimeException("getClusterNodeLabels Failed.");
   }
 
   @Override
@@ -1205,7 +1367,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
   }
 
   @Override
-  public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+  public Response removeFromClusterNodeLabels(Set<String> oldNodeLabels,
       HttpServletRequest hsr) throws Exception {
     throw new NotImplementedException("Code is not implemented");
   }
@@ -1213,33 +1375,185 @@ public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
   @Override
   public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId)
       throws IOException {
-    throw new NotImplementedException("Code is not implemented");
+    try {
+      long startTime = clock.getTime();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{HttpServletRequest.class, String.class};
+      Object[] args = new Object[]{hsrCopy, nodeId};
+      ClientMethod remoteMethod = new ClientMethod("getLabelsOnNode", argsClasses, args);
+      Map<SubClusterInfo, NodeLabelsInfo> nodeToLabelsInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, NodeLabelsInfo.class);
+      Set<NodeLabel> hashSets = Sets.newHashSet();
+      nodeToLabelsInfoMap.values().forEach(item -> hashSets.addAll(item.getNodeLabels()));
+      NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo(hashSets);
+      if (nodeLabelsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime);
+        return nodeLabelsInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException("get all active sub cluster(s) error.", e);
+    } catch (YarnException e) {
+      routerMetrics.incrLabelsToNodesFailedRetrieved();
+      RouterServerUtil.logAndThrowIOException(
+          e, "getLabelsOnNode nodeId = %s with yarn error.", nodeId);
+    }
+    routerMetrics.incrLabelsToNodesFailedRetrieved();
+    throw RouterServerUtil.logAndReturnRunTimeException(
+        "getLabelsOnNode by nodeId = %s Failed.", nodeId);
   }
 
   @Override
   public AppPriority getAppPriority(HttpServletRequest hsr, String appId)
       throws AuthorizationException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppPriorityFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppPriority appPriority = interceptor.getAppPriority(hsr, appId);
+      if (appPriority != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppPriorityRetrieved(stopTime - startTime);
+        return appPriority;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the getAppPriority appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getAppPriority error.", e);
+    }
+    routerMetrics.incrGetAppPriorityFailedRetrieved();
+    throw new RuntimeException("getAppPriority Failed.");
   }
 
   @Override
   public Response updateApplicationPriority(AppPriority targetPriority,
       HttpServletRequest hsr, String appId) throws AuthorizationException,
       YarnException, InterruptedException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      throw e;
+    }
+
+    if (targetPriority == null) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the targetPriority is empty or null.");
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      Response response = interceptor.updateApplicationPriority(targetPriority, hsr, appId);
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededUpdateAppPriorityRetrieved(stopTime - startTime);
+        return response;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the updateApplicationPriority appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("updateApplicationPriority error.", e);
+    }
+    routerMetrics.incrUpdateAppPriorityFailedRetrieved();
+    throw new RuntimeException("updateApplicationPriority Failed.");
   }
 
   @Override
   public AppQueue getAppQueue(HttpServletRequest hsr, String appId)
       throws AuthorizationException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppQueueFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppQueue queue = interceptor.getAppQueue(hsr, appId);
+      if (queue != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppQueueRetrieved((stopTime - startTime));
+        return queue;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get queue by appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getAppQueue error.", e);
+    }
+    routerMetrics.incrGetAppQueueFailedRetrieved();
+    throw new RuntimeException("getAppQueue Failed.");
   }
 
   @Override
   public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr,
       String appId) throws AuthorizationException, YarnException,
       InterruptedException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateAppQueueFailedRetrieved();
+      throw e;
+    }
+
+    if (targetQueue == null) {
+      routerMetrics.incrUpdateAppQueueFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the targetQueue is null.");
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      Response response = interceptor.updateAppQueue(targetQueue, hsr, appId);
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededUpdateAppQueueRetrieved(stopTime - startTime);
+        return response;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateAppQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to update app queue by appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrUpdateAppQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("updateAppQueue error.", e);
+    }
+    routerMetrics.incrUpdateAppQueueFailedRetrieved();
+    throw new RuntimeException("updateAppQueue Failed.");
   }
 
   @Override
@@ -1294,55 +1608,400 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext,
   public Response listReservation(String queue, String reservationId,
       long startTime, long endTime, boolean includeResourceAllocations,
       HttpServletRequest hsr) throws Exception {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (queue == null || queue.isEmpty()) {
+      routerMetrics.incrListReservationFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the queue is empty or null.");
+    }
+
+    if (reservationId == null || reservationId.isEmpty()) {
+      routerMetrics.incrListReservationFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the reservationId is empty or null.");
+    }
+
+    // Check that the appId format is accurate
+    try {
+      ReservationId.parseReservationId(reservationId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrListReservationFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime1 = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByReservationId(reservationId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      HttpServletRequest hsrCopy = clone(hsr);
+      Response response = interceptor.listReservation(queue, reservationId, startTime, endTime,
+          includeResourceAllocations, hsrCopy);
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededListReservationRetrieved(stopTime - startTime1);
+        return response;
+      }
+    } catch (YarnException e) {
+      routerMetrics.incrListReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("listReservation error.", e);
+    }
+
+    routerMetrics.incrListReservationFailedRetrieved();
+    throw new YarnException("listReservation Failed.");
   }
 
   @Override
   public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId,
       String type) throws AuthorizationException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (appId == null || appId.isEmpty()) {
+      routerMetrics.incrGetAppTimeoutFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
+    }
+
+    // Check that the appId format is accurate
+    try {
+      ApplicationId.fromString(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppTimeoutFailedRetrieved();
+      throw e;
+    }
+
+    if (type == null || type.isEmpty()) {
+      routerMetrics.incrGetAppTimeoutFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the type is empty or null.");
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppTimeoutInfo appTimeoutInfo = interceptor.getAppTimeout(hsr, appId, type);
+      if (appTimeoutInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppTimeoutRetrieved((stopTime - startTime));
+        return appTimeoutInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppTimeoutFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the getAppTimeout appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppTimeoutFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getAppTimeout error.", e);
+    }
+    routerMetrics.incrGetAppTimeoutFailedRetrieved();
+    throw new RuntimeException("getAppTimeout Failed.");
   }
 
   @Override
   public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
       throws AuthorizationException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppTimeoutsFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = clock.getTime();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppTimeoutsInfo appTimeoutsInfo = interceptor.getAppTimeouts(hsr, appId);
+      if (appTimeoutsInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetAppTimeoutsRetrieved((stopTime - startTime));
+        return appTimeoutsInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetAppTimeoutsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the getAppTimeouts appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrGetAppTimeoutsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getAppTimeouts error.", e);
+    }
+
+    routerMetrics.incrGetAppTimeoutsFailedRetrieved();
+    throw new RuntimeException("getAppTimeouts Failed.");
   }
 
   @Override
   public Response updateApplicationTimeout(AppTimeoutInfo appTimeout,
       HttpServletRequest hsr, String appId) throws AuthorizationException,
       YarnException, InterruptedException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+      throw e;
+    }
+
+    if (appTimeout == null) {
+      routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+      throw new IllegalArgumentException("Parameter error, the appTimeout is null.");
+    }
+
+    try {
+      long startTime = Time.now();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      Response response = interceptor.updateApplicationTimeout(appTimeout, hsr, appId);
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededUpdateAppTimeoutsRetrieved((stopTime - startTime));
+        return response;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the updateApplicationTimeout appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("updateApplicationTimeout error.", e);
+    }
+
+    routerMetrics.incrUpdateApplicationTimeoutsRetrieved();
+    throw new RuntimeException("updateApplicationTimeout Failed.");
   }
 
   @Override
   public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = Time.now();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppAttemptsInfo appAttemptsInfo = interceptor.getAppAttempts(hsr, appId);
+      if (appAttemptsInfo != null) {
+        long stopTime = Time.now();
+        routerMetrics.succeededAppAttemptsRetrieved(stopTime - startTime);
+        return appAttemptsInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to get the AppAttempt appId: %s.", appId);
+    } catch (YarnException e) {
+      routerMetrics.incrAppAttemptsFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getAppAttempts error.", e);
+    }
+
+    routerMetrics.incrAppAttemptsFailedRetrieved();
+    throw new RuntimeException("getAppAttempts Failed.");
   }
 
   @Override
   public RMQueueAclInfo checkUserAccessToQueue(String queue, String username,
-      String queueAclType, HttpServletRequest hsr) {
-    throw new NotImplementedException("Code is not implemented");
+      String queueAclType, HttpServletRequest hsr) throws AuthorizationException {
+
+    // Parameter Verification
+    if (queue == null || queue.isEmpty()) {
+      routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the queue is empty or null.");
+    }
+
+    if (username == null || username.isEmpty()) {
+      routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the username is empty or null.");
+    }
+
+    if (queueAclType == null || queueAclType.isEmpty()) {
+      routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the queueAclType is empty or null.");
+    }
+
+    // Traverse SubCluster and call checkUserAccessToQueue Api
+    try {
+      long startTime = Time.now();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      final HttpServletRequest hsrCopy = clone(hsr);
+      Class[] argsClasses = new Class[]{String.class, String.class, String.class,
+          HttpServletRequest.class};
+      Object[] args = new Object[]{queue, username, queueAclType, hsrCopy};
+      ClientMethod remoteMethod = new ClientMethod("checkUserAccessToQueue", argsClasses, args);
+      Map<SubClusterInfo, RMQueueAclInfo> rmQueueAclInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, RMQueueAclInfo.class);
+      FederationRMQueueAclInfo aclInfo = new FederationRMQueueAclInfo();
+      rmQueueAclInfoMap.forEach((subClusterInfo, rMQueueAclInfo) -> {
+        SubClusterId subClusterId = subClusterInfo.getSubClusterId();
+        rMQueueAclInfo.setSubClusterId(subClusterId.getId());
+        aclInfo.getList().add(rMQueueAclInfo);
+      });
+      long stopTime = Time.now();
+      routerMetrics.succeededCheckUserAccessToQueueRetrieved(stopTime - startTime);
+      return aclInfo;
+    } catch (NotFoundException e) {
+      routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("Get all active sub cluster(s) error.", e);
+    } catch (YarnException | IOException e) {
+      routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("checkUserAccessToQueue error.", e);
+    }
+
+    routerMetrics.incrCheckUserAccessToQueueFailedRetrieved();
+    throw new RuntimeException("checkUserAccessToQueue error.");
   }
 
   @Override
   public AppAttemptInfo getAppAttempt(HttpServletRequest req,
       HttpServletResponse res, String appId, String appAttemptId) {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId/appAttemptId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+      RouterServerUtil.validateApplicationAttemptId(appAttemptId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
+      throw e;
+    }
+
+    // Call the getAppAttempt method
+    try {
+      long startTime = Time.now();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      AppAttemptInfo appAttemptInfo = interceptor.getAppAttempt(req, res, appId, appAttemptId);
+      if (appAttemptInfo != null) {
+        long stopTime = Time.now();
+        routerMetrics.succeededAppAttemptReportRetrieved(stopTime - startTime);
+        return appAttemptInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "Unable to getAppAttempt by appId: %s, appAttemptId: %s.", appId, appAttemptId);
+    } catch (YarnException e) {
+      routerMetrics.incrAppAttemptReportFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e,
+          "getAppAttempt error, appId: %s, appAttemptId: %s.", appId, appAttemptId);
+    }
+
+    routerMetrics.incrAppAttemptReportFailedRetrieved();
+    throw RouterServerUtil.logAndReturnRunTimeException(
+        "getAppAttempt failed, appId: %s, appAttemptId: %s.", appId, appAttemptId);
   }
 
   @Override
   public ContainersInfo getContainers(HttpServletRequest req,
       HttpServletResponse res, String appId, String appAttemptId) {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check that the appId/appAttemptId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+      RouterServerUtil.validateApplicationAttemptId(appAttemptId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = clock.getTime();
+      ContainersInfo containersInfo = new ContainersInfo();
+      Map<SubClusterId, SubClusterInfo> subClustersActive = getActiveSubclusters();
+      Class[] argsClasses = new Class[]{
+          HttpServletRequest.class, HttpServletResponse.class, String.class, String.class};
+      Object[] args = new Object[]{req, res, appId, appAttemptId};
+      ClientMethod remoteMethod = new ClientMethod("getContainers", argsClasses, args);
+      Map<SubClusterInfo, ContainersInfo> containersInfoMap =
+          invokeConcurrent(subClustersActive.values(), remoteMethod, ContainersInfo.class);
+      if (containersInfoMap != null && !containersInfoMap.isEmpty()) {
+        containersInfoMap.values().forEach(containers ->
+            containersInfo.addAll(containers.getContainers()));
+      }
+      if (containersInfo != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetContainersRetrieved(stopTime - startTime);
+        return containersInfo;
+      }
+    } catch (NotFoundException e) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e, "getContainers error, appId = %s, " +
+          " appAttemptId = %s, Probably getActiveSubclusters error.", appId, appAttemptId);
+    } catch (IOException | YarnException e) {
+      routerMetrics.incrGetContainersFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(e, "getContainers error, appId = %s, " +
+          " appAttemptId = %s.", appId, appAttemptId);
+    }
+
+    routerMetrics.incrGetContainersFailedRetrieved();
+    throw RouterServerUtil.logAndReturnRunTimeException(
+        "getContainers failed, appId: %s, appAttemptId: %s.", appId, appAttemptId);
   }
 
   @Override
   public ContainerInfo getContainer(HttpServletRequest req,
       HttpServletResponse res, String appId, String appAttemptId,
       String containerId) {
+
+    // FederationInterceptorREST#getContainer is logically
+    // the same as FederationClientInterceptor#getContainerReport,
+    // so use the same Metric.
+
+    // Check that the appId/appAttemptId/containerId format is accurate
+    try {
+      RouterServerUtil.validateApplicationId(appId);
+      RouterServerUtil.validateApplicationAttemptId(appAttemptId);
+      RouterServerUtil.validateContainerId(containerId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      throw e;
+    }
+
+    try {
+      long startTime = Time.now();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+      ContainerInfo containerInfo =
+          interceptor.getContainer(req, res, appId, appAttemptId, containerId);
+      if (containerInfo != null) {
+        long stopTime = Time.now();
+        routerMetrics.succeededGetContainerReportRetrieved(stopTime - startTime);
+        return containerInfo;
+      }
+    } catch (IllegalArgumentException e) {
+      String msg = String.format(
+          "Unable to get the AppAttempt appId: %s, appAttemptId: %s, containerId: %s.", appId,
+          appAttemptId, containerId);
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException(msg, e);
+    } catch (YarnException e) {
+      routerMetrics.incrGetContainerReportFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("getContainer Failed.", e);
+    }
+
+    routerMetrics.incrGetContainerReportFailedRetrieved();
+    throw new RuntimeException("getContainer Failed.");
+  }
+
+  @Override
+  public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest hsr)
+      throws AuthorizationException, InterruptedException {
+    throw new NotImplementedException("Code is not implemented");
+  }
+
+  @Override
+  public Response getSchedulerConfiguration(HttpServletRequest hsr)
+      throws AuthorizationException {
     throw new NotImplementedException("Code is not implemented");
   }
 
@@ -1357,7 +2016,46 @@ public void setNextInterceptor(RESTRequestInterceptor next) {
   @Override
   public Response signalToContainer(String containerId, String command,
       HttpServletRequest req) {
-    throw new NotImplementedException("Code is not implemented");
+
+    // Check if containerId is empty or null
+    try {
+      RouterServerUtil.validateContainerId(containerId);
+    } catch (IllegalArgumentException e) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      throw e;
+    }
+
+    // Check if command is empty or null
+    if (command == null || command.isEmpty()) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      throw new IllegalArgumentException("Parameter error, the command is empty or null.");
+    }
+
+    try {
+      long startTime = Time.now();
+
+      ContainerId containerIdObj = ContainerId.fromString(containerId);
+      ApplicationId applicationId = containerIdObj.getApplicationAttemptId().getApplicationId();
+      SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(applicationId.toString());
+      DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+          subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
+
+      Response response = interceptor.signalToContainer(containerId, command, req);
+      if (response != null) {
+        long stopTime = Time.now();
+        routerMetrics.succeededSignalToContainerRetrieved(stopTime - startTime);
+        return response;
+      }
+    } catch (YarnException e) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("signalToContainer Failed.", e);
+    } catch (AuthorizationException e) {
+      routerMetrics.incrSignalToContainerFailedRetrieved();
+      RouterServerUtil.logAndThrowRunTimeException("signalToContainer Author Failed.", e);
+    }
+
+    routerMetrics.incrSignalToContainerFailedRetrieved();
+    throw new RuntimeException("signalToContainer Failed.");
   }
 
   @Override
@@ -1366,4 +2064,118 @@ public void shutdown() {
       threadpool.shutdown();
     }
   }
+
+  private <R> Map<SubClusterInfo, R> invokeConcurrent(Collection<SubClusterInfo> clusterIds,
+      ClientMethod request, Class<R> clazz) throws YarnException {
+
+    Map<SubClusterInfo, R> results = new HashMap<>();
+
+    // Send the requests in parallel
+    CompletionService<Pair<R, Exception>> compSvc =
+        new ExecutorCompletionService<>(this.threadpool);
+
+    // This part of the code should be able to expose the accessed Exception information.
+    // We use Pair to store related information. The left value of the Pair is the response,
+    // and the right value is the exception.
+    // If the request is normal, the response is not empty and the exception is empty;
+    // if the request is abnormal, the response is empty and the exception is not empty.
+    for (final SubClusterInfo info : clusterIds) {
+      compSvc.submit(() -> {
+        DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
+            info.getSubClusterId(), info.getRMWebServiceAddress());
+        try {
+          Method method = DefaultRequestInterceptorREST.class.
+              getMethod(request.getMethodName(), request.getTypes());
+          Object retObj = method.invoke(interceptor, request.getParams());
+          R ret = clazz.cast(retObj);
+          return Pair.of(ret, null);
+        } catch (Exception e) {
+          LOG.error("SubCluster {} failed to call {} method.",
+              info.getSubClusterId(), request.getMethodName(), e);
+          return Pair.of(null, e);
+        }
+      });
+    }
+
+    clusterIds.stream().forEach(clusterId -> {
+      try {
+        Future<Pair<R, Exception>> future = compSvc.take();
+        Pair<R, Exception> pair = future.get();
+        R response = pair.getKey();
+        if (response != null) {
+          results.put(clusterId, response);
+        }
+
+        Exception exception = pair.getRight();
+        if (exception != null) {
+          throw exception;
+        }
+      } catch (Throwable e) {
+        String msg = String.format("SubCluster %s failed to %s report.",
+            clusterId.getSubClusterId(), request.getMethodName());
+        LOG.error(msg, e);
+        throw new YarnRuntimeException(e.getCause().getMessage(), e);
+      }
+    });
+
+    return results;
+  }
+
+  /**
+   * get the HomeSubCluster according to ApplicationId.
+   *
+   * @param appId applicationId
+   * @return HomeSubCluster
+   * @throws YarnException on failure
+   */
+  private SubClusterInfo getHomeSubClusterInfoByAppId(String appId)
+      throws YarnException {
+    SubClusterInfo subClusterInfo = null;
+    try {
+      ApplicationId applicationId = ApplicationId.fromString(appId);
+      SubClusterId subClusterId = federationFacade.getApplicationHomeSubCluster(applicationId);
+      if (subClusterId == null) {
+        RouterServerUtil.logAndThrowException(null,
+            "Can't get HomeSubCluster by applicationId %s", applicationId);
+      }
+      subClusterInfo = federationFacade.getSubCluster(subClusterId);
+      return subClusterInfo;
+    } catch (IllegalArgumentException e){
+      throw new IllegalArgumentException(e);
+    } catch (YarnException e) {
+      RouterServerUtil.logAndThrowException(e,
+          "Get HomeSubClusterInfo by applicationId %s failed.", appId);
+    }
+    throw new YarnException("Unable to get subCluster by applicationId = " + appId);
+  }
+
+  /**
+   * get the HomeSubCluster according to ReservationId.
+   *
+   * @param resId reservationId
+   * @return HomeSubCluster
+   * @throws YarnException on failure
+   */
+  private SubClusterInfo getHomeSubClusterInfoByReservationId(String resId)
+      throws YarnException {
+    try {
+      ReservationId reservationId = ReservationId.parseReservationId(resId);
+      SubClusterId subClusterId = federationFacade.getReservationHomeSubCluster(reservationId);
+      if (subClusterId == null) {
+        RouterServerUtil.logAndThrowException(null,
+            "Can't get HomeSubCluster by reservationId %s", resId);
+      }
+      SubClusterInfo subClusterInfo = federationFacade.getSubCluster(subClusterId);
+      return subClusterInfo;
+    } catch (YarnException | IOException e) {
+      RouterServerUtil.logAndThrowException(e,
+          "Get HomeSubClusterInfo by reservationId %s failed.", resId);
+    }
+    throw new YarnException("Unable to get subCluster by reservationId = " + resId);
+  }
+
+  @VisibleForTesting
+  public LRUCacheHashMap<RouterAppInfoCacheKey, AppsInfo> getAppInfosCaches() {
+    return appInfosCaches;
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
index 02ec1a24020a5..1a1ae4001219a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationPage.java
@@ -33,8 +33,9 @@ class FederationPage extends RouterView {
   @Override
   protected void preHead(Page.HTML<__> html) {
     commonPreHead(html);
-    setTitle("Federation");
+    setTitle("About The YARN Federation");
     set(DATATABLES_ID, "rms");
+    set("ui.div.id", "div_id");
     set(initID(DATATABLES, "rms"), rmsTableInit());
     setTableStyles(html, "rms", ".healthStatus {width:10em}",
         ".healthReport {width:10em}");
@@ -46,12 +47,14 @@ protected Class<? extends SubView> content() {
   }
 
   private String rmsTableInit() {
-    StringBuilder b = tableInit().append(", aoColumnDefs: [");
-    b.append("{'bSearchable': false, 'aTargets': [ 7 ]}")
-        .append(", {'sType': 'title-numeric', 'bSearchable': false, "
-            + "'aTargets': [ 8, 9 ] }")
-        .append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}")
+    StringBuilder builder = tableInit().append(", aoColumnDefs: [");
+    builder
+        .append("{'sName':'State', 'sType':'string', 'bSearchable':false, 'aTargets':[1]},")
+        .append("{'sName':'LastStartTime', 'sType':'string', 'bSearchable':false, 'aTargets':[2]},")
+        .append("{'sName':'lastHeartBeat', 'sType':'string', 'bSearchable':false, 'aTargets':[3]},")
+        .append("{'sName':'resource', 'sType':'string', 'bSearchable':false, 'aTargets':[4]},")
+        .append("{'sName':'nodes', 'sType':'string', 'bSearchable':false, 'aTargets':[5]}")
         .append("]}");
-    return b.toString();
+    return builder.toString();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
new file mode 100644
index 0000000000000..1a157a10ce04e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
@@ -0,0 +1,264 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import com.google.inject.Inject;
+import com.sun.jersey.api.client.Client;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.server.router.webapp.dao.RouterClusterMetrics;
+import org.apache.hadoop.yarn.server.router.webapp.dao.RouterSchedulerMetrics;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+public class MetricsOverviewTable extends RouterBlock {
+
+  private final Router router;
+
+  @Inject
+  MetricsOverviewTable(Router router, ViewContext ctx) {
+    super(router, ctx);
+    this.router = router;
+  }
+
+  @Override
+  protected void render(Block html) {
+    // Initialize page styles
+    html.style(".metrics {margin-bottom:5px}");
+
+    // get routerClusterMetrics Info
+    ClusterMetricsInfo routerClusterMetricsInfo = getRouterClusterMetricsInfo();
+    RouterClusterMetrics routerClusterMetrics = new RouterClusterMetrics(routerClusterMetricsInfo);
+
+    // metrics div
+    Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
+    try {
+      initFederationClusterAppsMetrics(div, routerClusterMetrics);
+      initFederationClusterNodesMetrics(div, routerClusterMetrics);
+      List<SubClusterInfo> subClusters = getSubClusterInfoList();
+      initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters);
+    } catch (Exception e) {
+      LOG.error("MetricsOverviewTable init error.", e);
+    }
+    div.__();
+  }
+
+  protected void render(Block html, String subClusterId) {
+    // Initialize page styles
+    html.style(".metrics {margin-bottom:5px}");
+
+    // get subClusterId ClusterMetrics Info
+    ClusterMetricsInfo clusterMetricsInfo =
+        getClusterMetricsInfoBySubClusterId(subClusterId);
+    RouterClusterMetrics routerClusterMetrics =
+        new RouterClusterMetrics(clusterMetricsInfo, subClusterId);
+
+    // metrics div
+    Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
+    try {
+      initFederationClusterAppsMetrics(div, routerClusterMetrics);
+      initFederationClusterNodesMetrics(div, routerClusterMetrics);
+      Collection<SubClusterInfo> subClusters = getSubClusterInfoList(subClusterId);
+      initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters);
+    } catch (Exception e) {
+      LOG.error("MetricsOverviewTable init error.", e);
+    }
+    div.__();
+  }
+
+  /**
+   * Init Federation Cluster Apps Metrics.
+   * Contains App information, resource usage information.
+   *
+   * @param div data display div.
+   * @param metrics data metric information.
+   */
+  private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
+      RouterClusterMetrics metrics) {
+    div.h3(metrics.getWebPageTitlePrefix() + " Cluster Metrics").
+        table("#metricsoverview").
+        thead().$class("ui-widget-header").
+        // Initialize table header information
+        tr().
+        th().$class("ui-state-default").__("Apps Submitted").__().
+        th().$class("ui-state-default").__("Apps Pending").__().
+        th().$class("ui-state-default").__("Apps Running").__().
+        th().$class("ui-state-default").__("Apps Completed").__().
+        th().$class("ui-state-default").__("Containers Running").__().
+        th().$class("ui-state-default").__("Used Resources").__().
+        th().$class("ui-state-default").__("Total Resources").__().
+        th().$class("ui-state-default").__("Reserved Resources").__().
+        th().$class("ui-state-default").__("Physical Mem Used %").__().
+        th().$class("ui-state-default").__("Physical VCores Used %").__().
+        __().
+        __().
+        // Initialize table data information
+        tbody().$class("ui-widget-content").
+        tr().
+        td(metrics.getAppsSubmitted()).
+        td(metrics.getAppsPending()).
+        td(String.valueOf(metrics.getAppsRunning())).
+        td(metrics.getAppsCompleted()).
+        td(metrics.getAllocatedContainers()).
+        td(metrics.getUsedResources()).
+        td(metrics.getTotalResources()).
+        td(metrics.getReservedResources()).
+        td(metrics.getUtilizedMBPercent()).
+        td(metrics.getUtilizedVirtualCoresPercent()).
+        __().
+        __().__();
+  }
+
+  /**
+   * Init Federation Cluster Nodes Metrics.
+   *
+   * @param div data display div.
+   * @param metrics data metric information.
+   */
+  private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
+      RouterClusterMetrics metrics) {
+    div.h3(metrics.getWebPageTitlePrefix() + " Cluster Nodes Metrics").
+        table("#nodemetricsoverview").
+        thead().$class("ui-widget-header").
+        // Initialize table header information
+        tr().
+        th().$class("ui-state-default").__("Active Nodes").__().
+        th().$class("ui-state-default").__("Decommissioning Nodes").__().
+        th().$class("ui-state-default").__("Decommissioned Nodes").__().
+        th().$class("ui-state-default").__("Lost Nodes").__().
+        th().$class("ui-state-default").__("Unhealthy Nodes").__().
+        th().$class("ui-state-default").__("Rebooted Nodes").__().
+        th().$class("ui-state-default").__("Shutdown Nodes").__().
+        __().
+        __().
+        // Initialize table data information
+        tbody().$class("ui-widget-content").
+        tr().
+        td(String.valueOf(metrics.getActiveNodes())).
+        td(String.valueOf(metrics.getDecommissioningNodes())).
+        td(String.valueOf(metrics.getDecommissionedNodes())).
+        td(String.valueOf(metrics.getLostNodes())).
+        td(String.valueOf(metrics.getUnhealthyNodes())).
+        td(String.valueOf(metrics.getRebootedNodes())).
+        td(String.valueOf(metrics.getShutdownNodes())).
+        __().
+        __().__();
+  }
+
+  /**
+   * Init Federation Cluster SchedulersMetrics.
+   *
+   * @param div data display div.
+   * @param metrics data metric information.
+   * @param subclusters active subcluster List.
+   * @throws YarnException yarn error.
+   * @throws IOException io error.
+   * @throws InterruptedException interrupt error.
+   */
+  private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
+      RouterClusterMetrics metrics, Collection<SubClusterInfo> subclusters)
+      throws YarnException, IOException, InterruptedException {
+
+    Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
+        div.h3(metrics.getWebPageTitlePrefix() + " Scheduler Metrics").
+        table("#schedulermetricsoverview").
+        thead().$class("ui-widget-header").
+        tr().
+        th().$class("ui-state-default").__("SubCluster").__().
+        th().$class("ui-state-default").__("Scheduler Type").__().
+        th().$class("ui-state-default").__("Scheduling Resource Type").__().
+        th().$class("ui-state-default").__("Minimum Allocation").__().
+        th().$class("ui-state-default").__("Maximum Allocation").__().
+        th().$class("ui-state-default").__("Maximum Cluster Application Priority").__().
+        th().$class("ui-state-default").__("Scheduler Busy %").__().
+        th().$class("ui-state-default").__("RM Dispatcher EventQueue Size").__().
+        th().$class("ui-state-default")
+        .__("Scheduler Dispatcher EventQueue Size").__().
+        __().
+        __().
+        tbody().$class("ui-widget-content");
+
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // If Federation mode is not enabled or there is currently no SubCluster available,
+    // each column in the list should be displayed as N/A
+    if (!isEnabled || subclusters == null || subclusters.isEmpty()) {
+      fsMetricsScheduleTr.tr().
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE).
+          td(UNAVAILABLE)
+          .__();
+    } else {
+      initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters);
+    }
+
+    fsMetricsScheduleTr.__().__();
+  }
+
+  private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
+      Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
+      Collection<SubClusterInfo> subclusters) {
+
+    // configuration
+    Configuration config = this.router.getConfig();
+
+    Client client = RouterWebServiceUtil.createJerseyClient(config);
+
+    // Traverse all SubClusters to get cluster information.
+    for (SubClusterInfo subcluster : subclusters) {
+
+      // Call the RM interface to obtain schedule information
+      String webAppAddress =  WebAppUtils.getHttpSchemePrefix(config) +
+          subcluster.getRMWebServiceAddress();
+
+      SchedulerOverviewInfo typeInfo = RouterWebServiceUtil
+          .genericForward(webAppAddress, null, SchedulerOverviewInfo.class, HTTPMethods.GET,
+          RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW, null, null,
+          config, client);
+      RouterSchedulerMetrics rsMetrics = new RouterSchedulerMetrics(subcluster, metrics, typeInfo);
+
+      // Basic information
+      fsMetricsScheduleTr.tr().
+          td(rsMetrics.getSubCluster()).
+          td(rsMetrics.getSchedulerType()).
+          td(rsMetrics.getSchedulingResourceType()).
+          td(rsMetrics.getMinimumAllocation()).
+          td(rsMetrics.getMaximumAllocation()).
+          td(rsMetrics.getApplicationPriority()).
+          td(rsMetrics.getSchedulerBusy()).
+          td(rsMetrics.getRmDispatcherEventQueueSize()).
+          td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
+          __();
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
index 9c39bb7b7a22f..2266370ee95f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
@@ -18,29 +18,54 @@
 
 package org.apache.hadoop.yarn.server.router.webapp;
 
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 
+import java.util.List;
 /**
  * Navigation block for the Router Web UI.
  */
-public class NavBlock extends HtmlBlock {
+public class NavBlock extends RouterBlock {
+
+  private Router router;
+
+  @Inject
+  public NavBlock(Router router, ViewContext ctx) {
+    super(router, ctx);
+    this.router = router;
+  }
 
   @Override
   public void render(Block html) {
-    html.
-      div("#nav").
+    Hamlet.UL<Hamlet.DIV<Hamlet>> mainList = html.div("#nav").
         h3("Cluster").
         ul().
-          li().a(url(""), "About").__().
-          li().a(url("federation"), "Federation").__().
-          li().a(url("nodes"), "Nodes").__().
-          li().a(url("apps"), "Applications").__().
-      __().
-      h3("Tools").
-      ul().
-        li().a("/conf", "Configuration").__().
-        li().a("/logs", "Local logs").__().
-        li().a("/stacks", "Server stacks").__().
-        li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
+        li().a(url(""), "About").__().
+        li().a(url("federation"), "Federation").__();
+
+    List<String> subClusterIds = getActiveSubClusterIds();
+
+    // ### nodes info
+    initNodesMenu(mainList, subClusterIds);
+
+    // ### nodelabels info
+    initNodeLabelsMenu(mainList, subClusterIds);
+
+    // ### applications info
+    initApplicationsMenu(mainList, subClusterIds);
+
+    // ### tools
+    Hamlet.DIV<Hamlet> sectionBefore = mainList.__();
+    Configuration conf = new Configuration();
+    Hamlet.UL<Hamlet.DIV<Hamlet>> tools = WebPageUtils.appendToolSection(sectionBefore, conf);
+
+    if (tools == null) {
+      return;
+    }
+
+    tools.__().__();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
new file mode 100644
index 0000000000000..62e2b5d8537e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import com.google.inject.Inject;
+import com.sun.jersey.api.client.Client;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
+
+/**
+ * Navigation block for the Router Web UI.
+ */
+public class NodeLabelsBlock extends RouterBlock {
+
+  private Router router;
+
+  @Inject
+  public NodeLabelsBlock(Router router, ViewContext ctx) {
+    super(router, ctx);
+    this.router = router;
+  }
+
+  @Override
+  protected void render(Block html) {
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // Get subClusterName
+    String subClusterName = $(NODE_SC);
+
+    NodeLabelsInfo nodeLabelsInfo = null;
+    if (StringUtils.isNotEmpty(subClusterName)) {
+      nodeLabelsInfo = getSubClusterNodeLabelsInfo(subClusterName);
+    } else {
+      nodeLabelsInfo = getYarnFederationNodeLabelsInfo(isEnabled);
+    }
+
+    initYarnFederationNodeLabelsOfCluster(nodeLabelsInfo, html);
+  }
+
+  /**
+   * Get NodeLabels Info based on SubCluster.
+   * @return NodeLabelsInfo.
+   */
+  private NodeLabelsInfo getSubClusterNodeLabelsInfo(String subCluster) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
+      FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+
+      if (subClusterInfo != null) {
+        // Prepare webAddress
+        String webAddress = subClusterInfo.getRMWebServiceAddress();
+        String herfWebAppAddress = "";
+        if (webAddress != null && !webAddress.isEmpty()) {
+          herfWebAppAddress =
+              WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress;
+          return getSubClusterNodeLabelsByWebAddress(herfWebAppAddress);
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("get NodeLabelsInfo From SubCluster = {} error.", subCluster, e);
+    }
+    return null;
+  }
+
+  private NodeLabelsInfo getYarnFederationNodeLabelsInfo(boolean isEnabled) {
+    if (isEnabled) {
+      String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
+      return getSubClusterNodeLabelsByWebAddress(webAddress);
+    }
+    return null;
+  }
+
+  private NodeLabelsInfo getSubClusterNodeLabelsByWebAddress(String webAddress) {
+    Configuration conf = this.router.getConfig();
+    Client client = RouterWebServiceUtil.createJerseyClient(conf);
+    NodeLabelsInfo nodes = RouterWebServiceUtil
+        .genericForward(webAddress, null, NodeLabelsInfo.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.GET_RM_NODE_LABELS, null, null, conf,
+        client);
+    client.destroy();
+    return nodes;
+  }
+
+  private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo nodeLabelsInfo, Block html) {
+
+    Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody = html.table("#nodelabels").
+        thead().
+        tr().
+        th(".name", "Label Name").
+        th(".type", "Label Type").
+        th(".numOfActiveNMs", "Num Of Active NMs").
+        th(".totalResource", "Total Resource").
+        __().__().
+        tbody();
+
+    if (nodeLabelsInfo != null) {
+      for (NodeLabelInfo info : nodeLabelsInfo.getNodeLabelsInfo()) {
+        Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet>>> row =
+            tbody.tr().td(info.getName().isEmpty() ?
+            NodeLabel.DEFAULT_NODE_LABEL_PARTITION : info.getName());
+        String type = (info.getExclusivity()) ? "Exclusive Partition" : "Non Exclusive Partition";
+        row = row.td(type);
+        int nActiveNMs = info.getActiveNMs();
+        row = row.td(String.valueOf(nActiveNMs));
+        PartitionInfo partitionInfo = info.getPartitionInfo();
+        ResourceInfo available = partitionInfo.getResourceAvailable();
+        row.td(available.toFormattedString()).__();
+      }
+    }
+
+    tbody.__().__();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java
new file mode 100644
index 0000000000000..9b3cea468172a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+
+/**
+ * Renders a block for the nodelabels with metrics information.
+ */
+public class NodeLabelsPage extends RouterView {
+
+  @Override
+  protected void preHead(Hamlet.HTML<__> html) {
+    commonPreHead(html);
+    String type = $(NODE_SC);
+    String title = "Node labels of the cluster";
+    if (type != null && !type.isEmpty()) {
+      title = title + " (" + type + ")";
+    }
+    setTitle(title);
+    set(DATATABLES_ID, "nodelabels");
+    setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", ".healthReport {width:10em}");
+  }
+
+  @Override
+  protected Class<? extends SubView> content() {
+    return NodeLabelsBlock.class;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
index 4734cf6bbf329..7e92506e0d68b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
@@ -19,50 +19,108 @@
 package org.apache.hadoop.yarn.server.router.webapp;
 
 import com.sun.jersey.api.client.Client;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
 import org.apache.hadoop.yarn.server.router.Router;
-import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
 import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
 
 import com.google.inject.Inject;
 
+import java.util.Date;
+
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
+
 /**
  * Nodes block for the Router Web UI.
  */
-public class NodesBlock extends HtmlBlock {
-
-  private static final long BYTES_IN_MB = 1024 * 1024;
+public class NodesBlock extends RouterBlock {
 
   private final Router router;
 
   @Inject
   NodesBlock(Router router, ViewContext ctx) {
-    super(ctx);
+    super(router, ctx);
     this.router = router;
   }
 
   @Override
   protected void render(Block html) {
-    // Get the node info from the federation
+
+    boolean isEnabled = isYarnFederationEnabled();
+
+    // Get subClusterName
+    String subClusterName = $(NODE_SC);
+
+    // We will try to get the subClusterName.
+    // If the subClusterName is not empty,
+    // it means that we need to get the Node list of a subCluster.
+    NodesInfo nodesInfo = null;
+    if (subClusterName != null && !subClusterName.isEmpty()) {
+      initSubClusterMetricsOverviewTable(html, subClusterName);
+      nodesInfo = getSubClusterNodesInfo(subClusterName);
+    } else {
+      // Metrics Overview Table
+      html.__(MetricsOverviewTable.class);
+      nodesInfo = getYarnFederationNodesInfo(isEnabled);
+    }
+
+    // Initialize NodeInfo List
+    initYarnFederationNodesOfCluster(nodesInfo, html);
+  }
+
+  private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) {
+    if (isEnabled) {
+      String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
+      return getSubClusterNodesInfoByWebAddress(webAddress);
+    }
+    return null;
+  }
+
+  private NodesInfo getSubClusterNodesInfo(String subCluster) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
+      FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+
+      if (subClusterInfo != null) {
+        // Prepare webAddress
+        String webAddress = subClusterInfo.getRMWebServiceAddress();
+        String herfWebAppAddress = "";
+        if (webAddress != null && !webAddress.isEmpty()) {
+          herfWebAppAddress =
+              WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress;
+          return getSubClusterNodesInfoByWebAddress(herfWebAppAddress);
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("get NodesInfo From SubCluster = {} error.", subCluster, e);
+    }
+    return null;
+  }
+
+  private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) {
     Configuration conf = this.router.getConfig();
     Client client = RouterWebServiceUtil.createJerseyClient(conf);
-    String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
     NodesInfo nodes = RouterWebServiceUtil
-        .genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET,
-            RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
-            client);
-
-    setTitle("Nodes");
+        .genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
+        RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
+        client);
+    client.destroy();
+    return nodes;
+  }
 
+  private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) {
     TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
         .th(".nodelabels", "Node Labels")
         .th(".rack", "Rack")
@@ -79,34 +137,42 @@ protected void render(Block html) {
         .th(".nodeManagerVersion", "Version")
         .__().__().tbody();
 
-    // Add nodes to the web UI
-    for (NodeInfo info : nodes.getNodes()) {
-      int usedMemory = (int) info.getUsedMemory();
-      int availableMemory = (int) info.getAvailableMemory();
-      TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
-      row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
-      row.td().__(info.getRack()).__();
-      row.td().__(info.getState()).__();
-      row.td().__(info.getNodeId()).__();
-      boolean isInactive = false;
-      if (isInactive) {
-        row.td().__("N/A").__();
-      } else {
-        String httpAddress = info.getNodeHTTPAddress();
-        row.td().a("//" + httpAddress, httpAddress).__();
+    if (nodesInfo != null && CollectionUtils.isNotEmpty(nodesInfo.getNodes())) {
+      for (NodeInfo info : nodesInfo.getNodes()) {
+        int usedMemory = (int) info.getUsedMemory();
+        int availableMemory = (int) info.getAvailableMemory();
+        TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
+        row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
+        row.td().__(info.getRack()).__();
+        row.td().__(info.getState()).__();
+        row.td().__(info.getNodeId()).__();
+        boolean isInactive = false;
+        if (isInactive) {
+          row.td().__(UNAVAILABLE).__();
+        } else {
+          String httpAddress = info.getNodeHTTPAddress();
+          String herfWebAppAddress = "";
+          if (httpAddress != null && !httpAddress.isEmpty()) {
+            herfWebAppAddress =
+                WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + httpAddress;
+          }
+          row.td().a(herfWebAppAddress, httpAddress).__();
+        }
+
+        row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
+            .__(new Date(info.getLastHealthUpdate())).__()
+            .td(info.getHealthReport())
+            .td(String.valueOf(info.getNumContainers())).td().br()
+            .$title(String.valueOf(usedMemory)).__()
+            .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
+            .$title(String.valueOf(availableMemory)).__()
+            .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
+            .td(String.valueOf(info.getUsedVirtualCores()))
+            .td(String.valueOf(info.getAvailableVirtualCores()))
+            .td(info.getVersion()).__();
       }
-      row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
-          .__(Times.format(info.getLastHealthUpdate())).__()
-          .td(info.getHealthReport())
-          .td(String.valueOf(info.getNumContainers())).td().br()
-          .$title(String.valueOf(usedMemory)).__()
-          .__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
-          .$title(String.valueOf(availableMemory)).__()
-          .__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
-          .td(String.valueOf(info.getUsedVirtualCores()))
-          .td(String.valueOf(info.getAvailableVirtualCores()))
-          .td(info.getVersion()).__();
     }
+
     tbody.__().__();
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
index 7b2a3da76504c..0723cff792d96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.server.router.webapp;
 
-import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
@@ -31,7 +31,7 @@ class NodesPage extends RouterView {
   @Override
   protected void preHead(Page.HTML<__> html) {
     commonPreHead(html);
-    String type = $(NODE_STATE);
+    String type = $(NODE_SC);
     String title = "Nodes of the cluster";
     if (type != null && !type.isEmpty()) {
       title = title + " (" + type + ")";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java
index 06f39b5e393aa..917809ad6cd60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java
@@ -29,7 +29,7 @@
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
 
 /**
- * Defines the contract to be implemented by the request intercepter classes,
+ * Defines the contract to be implemented by the request interceptor classes,
  * that can be used to intercept and inspect messages sent from the client to
  * the resource manager server.
  *
@@ -42,7 +42,7 @@ public interface RESTRequestInterceptor
     extends RMWebServiceProtocol, Configurable {
 
   /**
-   * This method is called for initializing the intercepter. This is guaranteed
+   * This method is called for initializing the interceptor. This is guaranteed
    * to be called only once in the lifetime of this instance.
    *
    * @param user the name of the client
@@ -50,28 +50,28 @@ public interface RESTRequestInterceptor
   void init(String user);
 
   /**
-   * This method is called to release the resources held by the intercepter.
+   * This method is called to release the resources held by the interceptor.
    * This will be called when the application pipeline is being destroyed. The
    * concrete implementations should dispose the resources and forward the
-   * request to the next intercepter, if any.
+   * request to the next interceptor, if any.
    */
   void shutdown();
 
   /**
-   * Sets the next intercepter in the pipeline. The concrete implementation of
+   * Sets the next interceptor in the pipeline. The concrete implementation of
    * this interface should always pass the request to the nextInterceptor after
-   * inspecting the message. The last intercepter in the chain is responsible to
+   * inspecting the message. The last interceptor in the chain is responsible to
    * send the messages to the resource manager service and so the last
-   * intercepter will not receive this method call.
+   * interceptor will not receive this method call.
    *
    * @param nextInterceptor the RESTRequestInterceptor to set in the pipeline
    */
   void setNextInterceptor(RESTRequestInterceptor nextInterceptor);
 
   /**
-   * Returns the next intercepter in the chain.
+   * Returns the next interceptor in the chain.
    *
-   * @return the next intercepter in the chain
+   * @return the next interceptor in the chain
    */
   RESTRequestInterceptor getNextInterceptor();
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
new file mode 100644
index 0000000000000..31ab83daaaf10
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import com.sun.jersey.api.client.Client;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+
+public abstract class RouterBlock extends HtmlBlock {
+
+  private final Router router;
+  private final ViewContext ctx;
+  private final FederationStateStoreFacade facade;
+  private final Configuration conf;
+
+  public RouterBlock(Router router, ViewContext ctx) {
+    super(ctx);
+    this.ctx = ctx;
+    this.router = router;
+    this.facade = FederationStateStoreFacade.getInstance();
+    this.conf = this.router.getConfig();
+  }
+
+  /**
+   * Get RouterClusterMetrics Info.
+   *
+   * @return Router ClusterMetricsInfo.
+   */
+  protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
+    boolean isEnabled = isYarnFederationEnabled();
+    if(isEnabled) {
+      String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+      Client client = RouterWebServiceUtil.createJerseyClient(conf);
+      ClusterMetricsInfo metrics = RouterWebServiceUtil
+          .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
+          RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
+          conf, client);
+      client.destroy();
+      return metrics;
+    }
+    return null;
+  }
+
+  /**
+   * Get a list of subclusters.
+   *
+   * @return subcluster List.
+   * @throws YarnException if the call to the getSubClusters is unsuccessful.
+   */
+  protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
+
+    Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
+
+    // Sort the SubClusters.
+    List<SubClusterInfo> subclusters = new ArrayList<>();
+    subclusters.addAll(subClustersInfo.values());
+    Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o -> o.getSubClusterId());
+    Collections.sort(subclusters, cmp);
+
+    return subclusters;
+  }
+
+  /**
+   * Whether Yarn Federation is enabled.
+   *
+   * @return true, enable yarn federation; false, not enable yarn federation;
+   */
+  protected boolean isYarnFederationEnabled() {
+    boolean isEnabled = conf.getBoolean(
+        YarnConfiguration.FEDERATION_ENABLED,
+        YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+    return isEnabled;
+  }
+
+  /**
+   * Get a list of SubClusterIds for ActiveSubClusters.
+   *
+   * @return list of SubClusterIds.
+   */
+  protected List<String> getActiveSubClusterIds() {
+    List<String> result = new ArrayList<>();
+    try {
+      Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
+      subClustersInfo.values().stream().forEach(subClusterInfo -> {
+        result.add(subClusterInfo.getSubClusterId().getId());
+      });
+    } catch (Exception e) {
+      LOG.error("getActiveSubClusters error.", e);
+    }
+    return result;
+  }
+
+  /**
+   * init SubCluster MetricsOverviewTable.
+   *
+   * @param html HTML Object.
+   * @param subclusterId subClusterId
+   */
+  protected void initSubClusterMetricsOverviewTable(Block html, String subclusterId) {
+    MetricsOverviewTable metricsOverviewTable = new MetricsOverviewTable(this.router, this.ctx);
+    metricsOverviewTable.render(html, subclusterId);
+  }
+
+  /**
+   * Get ClusterMetricsInfo By SubClusterId.
+   *
+   * @param subclusterId subClusterId
+   * @return SubCluster RM ClusterMetricsInfo
+   */
+  protected ClusterMetricsInfo getClusterMetricsInfoBySubClusterId(String subclusterId) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subclusterId);
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+      if (subClusterInfo != null) {
+        Client client = RouterWebServiceUtil.createJerseyClient(this.conf);
+        // Call the RM interface to obtain schedule information
+        String webAppAddress =  WebAppUtils.getHttpSchemePrefix(this.conf) +
+            subClusterInfo.getRMWebServiceAddress();
+        ClusterMetricsInfo metrics = RouterWebServiceUtil
+            .genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
+            RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
+            conf, client);
+        client.destroy();
+        return metrics;
+      }
+    } catch (Exception e) {
+      LOG.error("getClusterMetricsInfoBySubClusterId subClusterId = {} error.", subclusterId, e);
+    }
+    return null;
+  }
+
+  protected Collection<SubClusterInfo> getSubClusterInfoList(String subclusterId) {
+    try {
+      SubClusterId subClusterId = SubClusterId.newInstance(subclusterId);
+      SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
+      return Collections.singletonList(subClusterInfo);
+    } catch (Exception e) {
+      LOG.error("getSubClusterInfoList subClusterId = {} error.", subclusterId, e);
+    }
+    return null;
+  }
+
+  public FederationStateStoreFacade getFacade() {
+    return facade;
+  }
+
+  /**
+   * Initialize the Nodes menu.
+   *
+   * @param mainList HTML Object.
+   * @param subClusterIds subCluster List.
+   */
+  protected void initNodesMenu(Hamlet.UL<Hamlet.DIV<Hamlet>> mainList,
+      List<String> subClusterIds) {
+    if (CollectionUtils.isNotEmpty(subClusterIds)) {
+      Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> nodesList =
+          mainList.li().a(url("nodes"), "Nodes").ul().
+          $style("padding:0.3em 1em 0.1em 2em");
+
+      // ### nodes info
+      nodesList.li().__();
+      for (String subClusterId : subClusterIds) {
+        nodesList.li().a(url("nodes", subClusterId), subClusterId).__();
+      }
+      nodesList.__().__();
+    } else {
+      mainList.li().a(url("nodes"), "Nodes").__();
+    }
+  }
+
+  /**
+   * Initialize the Applications menu.
+   *
+   * @param mainList HTML Object.
+   * @param subClusterIds subCluster List.
+   */
+  protected void initApplicationsMenu(Hamlet.UL<Hamlet.DIV<Hamlet>> mainList,
+      List<String> subClusterIds) {
+    if (CollectionUtils.isNotEmpty(subClusterIds)) {
+      Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> apps =
+          mainList.li().a(url("apps"), "Applications").ul();
+      apps.li().__();
+      for (String subClusterId : subClusterIds) {
+        Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>> subClusterList = apps.
+            li().a(url("apps", subClusterId), subClusterId);
+        Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>>> subAppStates =
+            subClusterList.ul().$style("padding:0.3em 1em 0.1em 2em");
+        subAppStates.li().__();
+        for (YarnApplicationState state : YarnApplicationState.values()) {
+          subAppStates.
+              li().a(url("apps", subClusterId, state.toString()), state.toString()).__();
+        }
+        subAppStates.li().__().__();
+        subClusterList.__();
+      }
+      apps.__().__();
+    } else {
+      mainList.li().a(url("apps"), "Applications").__();
+    }
+  }
+
+  /**
+   * Initialize the NodeLabels menu.
+   *
+   * @param mainList HTML Object.
+   * @param subClusterIds subCluster List.
+   */
+  protected void initNodeLabelsMenu(Hamlet.UL<Hamlet.DIV<Hamlet>> mainList,
+      List<String> subClusterIds) {
+
+    if (CollectionUtils.isNotEmpty(subClusterIds)) {
+      Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> nodesList =
+          mainList.li().a(url("nodelabels"), "Node Labels").ul().
+          $style("padding:0.3em 1em 0.1em 2em");
+
+      // ### nodelabels info
+      nodesList.li().__();
+      for (String subClusterId : subClusterIds) {
+        nodesList.li().a(url("nodelabels", subClusterId), subClusterId).__();
+      }
+      nodesList.__().__();
+    } else {
+      mainList.li().a(url("nodelabels"), "Node Labels").__();
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
index e8cd0de6657f5..7d7165f7cadc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterController.java
@@ -34,7 +34,7 @@ public class RouterController extends Controller {
 
   @Override
   public void index() {
-    setTitle("Router");
+    setTitle("About the YARN Router");
     render(AboutPage.class);
   }
 
@@ -44,7 +44,6 @@ public void about() {
   }
 
   public void federation() {
-    setTitle("Federation");
     render(FederationPage.class);
   }
 
@@ -57,4 +56,9 @@ public void nodes() {
     setTitle("Nodes");
     render(NodesPage.class);
   }
+
+  public void nodeLabels() {
+    setTitle("Node Labels");
+    render(NodeLabelsPage.class);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
index ba07a1afba466..989a3d43b439d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
@@ -24,6 +24,8 @@
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.YarnWebParams;
 
+import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
+
 /**
  * The Router webapp.
  */
@@ -47,8 +49,9 @@ public void setup() {
     route("/", RouterController.class);
     route("/cluster", RouterController.class, "about");
     route("/about", RouterController.class, "about");
-    route("/apps", RouterController.class, "apps");
-    route("/nodes", RouterController.class, "nodes");
+    route(pajoin("/apps", APP_SC, APP_STATE), RouterController.class, "apps");
+    route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
     route("/federation", RouterController.class, "federation");
+    route(pajoin("/nodelabels", NODE_SC), RouterController.class, "nodeLabels");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 6d7dc5f5c19b3..4182a12f303b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -30,6 +30,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Collection;
+import java.util.Set;
+import java.util.HashSet;
 import java.util.concurrent.TimeUnit;
 
 import javax.servlet.http.HttpServletRequest;
@@ -43,13 +46,21 @@
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
 import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.ForbiddenException;
@@ -83,7 +94,7 @@ private RouterWebServiceUtil() {
   /**
    * Creates and performs a REST call to a specific WebService.
    *
-   * @param webApp the address of the remote webap
+   * @param webApp the address of the remote webapp
    * @param hsr the servlet request
    * @param returnType the return type of the REST call
    * @param <T> Type of return object.
@@ -170,7 +181,7 @@ public T run() {
 
   /**
    * Performs an invocation of a REST call on a remote RMWebService.
-   * @param webApp the address of the remote webap
+   * @param webApp the address of the remote webapp
    * @param path  to add to the webapp address
    * @param method the HTTP method of the REST call
    * @param additionalPath the servlet request path
@@ -280,7 +291,7 @@ public static void retrieveException(ClientResponse response) {
 
   /**
    * Merges a list of AppInfo grouping by ApplicationId. Our current policy is
-   * to merge the application reports from the reacheable SubClusters. Via
+   * to merge the application reports from the reachable SubClusters. Via
    * configuration parameter, we decide whether to return applications for which
    * the primary AM is missing or to omit them.
    *
@@ -293,8 +304,8 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
       boolean returnPartialResult) {
     AppsInfo allApps = new AppsInfo();
 
-    Map<String, AppInfo> federationAM = new HashMap<String, AppInfo>();
-    Map<String, AppInfo> federationUAMSum = new HashMap<String, AppInfo>();
+    Map<String, AppInfo> federationAM = new HashMap<>();
+    Map<String, AppInfo> federationUAMSum = new HashMap<>();
     for (AppInfo a : appsInfo) {
       // Check if this AppInfo is an AM
       if (a.getAMHostHttpAddress() != null) {
@@ -332,7 +343,7 @@ public static AppsInfo mergeAppsInfo(ArrayList<AppInfo> appsInfo,
       }
     }
 
-    allApps.addAll(new ArrayList<AppInfo>(federationAM.values()));
+    allApps.addAll(new ArrayList<>(federationAM.values()));
     return allApps;
   }
 
@@ -419,7 +430,7 @@ public static NodesInfo deleteDuplicateNodesInfo(ArrayList<NodeInfo> nodes) {
         nodesMap.put(node.getNodeId(), node);
       }
     }
-    nodesInfo.addAll(new ArrayList<NodeInfo>(nodesMap.values()));
+    nodesInfo.addAll(new ArrayList<>(nodesMap.values()));
     return nodesInfo;
   }
 
@@ -494,7 +505,7 @@ public static void mergeMetrics(ClusterMetricsInfo metrics,
   protected static <T> String getMediaTypeFromHttpServletRequest(
       HttpServletRequest request, final Class<T> returnType) {
     if (request == null) {
-      // By default we return XML for REST call without HttpServletRequest
+      // By default, we return XML for REST call without HttpServletRequest
       return MediaType.APPLICATION_XML;
     }
     // TODO
@@ -503,10 +514,104 @@ protected static <T> String getMediaTypeFromHttpServletRequest(
     }
     String header = request.getHeader(HttpHeaders.ACCEPT);
     if (header == null || header.equals("*")) {
-      // By default we return JSON
+      // By default, we return JSON
       return MediaType.APPLICATION_JSON;
     }
     return header;
   }
 
+  public static NodeToLabelsInfo mergeNodeToLabels(
+      Map<SubClusterInfo, NodeToLabelsInfo> nodeToLabelsInfoMap) {
+
+    HashMap<String, NodeLabelsInfo> nodeToLabels = new HashMap<>();
+    Collection<NodeToLabelsInfo> nodeToLabelsInfos = nodeToLabelsInfoMap.values();
+
+    nodeToLabelsInfos.stream().forEach(nodeToLabelsInfo -> {
+      for (Map.Entry<String, NodeLabelsInfo> item : nodeToLabelsInfo.getNodeToLabels().entrySet()) {
+        String key = item.getKey();
+        NodeLabelsInfo itemValue = item.getValue();
+        NodeLabelsInfo nodeToLabelsValue = nodeToLabels.getOrDefault(item.getKey(), null);
+        Set<NodeLabel> hashSet = new HashSet<>();
+        if (itemValue != null) {
+          hashSet.addAll(itemValue.getNodeLabels());
+        }
+        if (nodeToLabelsValue != null) {
+          hashSet.addAll(nodeToLabelsValue.getNodeLabels());
+        }
+        nodeToLabels.put(key, new NodeLabelsInfo(hashSet));
+      }
+    });
+
+    return new NodeToLabelsInfo(nodeToLabels);
+  }
+
+  public static ApplicationStatisticsInfo mergeApplicationStatisticsInfo(
+      Collection<ApplicationStatisticsInfo> appStatistics) {
+    ApplicationStatisticsInfo result = new ApplicationStatisticsInfo();
+    Map<String, StatisticsItemInfo> statisticsItemMap = new HashMap<>();
+
+    appStatistics.stream().forEach(appStatistic -> {
+      List<StatisticsItemInfo> statisticsItemInfos = appStatistic.getStatItems();
+      for (StatisticsItemInfo statisticsItemInfo : statisticsItemInfos) {
+
+        String statisticsItemKey =
+            statisticsItemInfo.getType() + "_" + statisticsItemInfo.getState().toString();
+
+        StatisticsItemInfo statisticsItemValue;
+        if (statisticsItemMap.containsKey(statisticsItemKey)) {
+          statisticsItemValue = statisticsItemMap.get(statisticsItemKey);
+          long statisticsItemValueCount = statisticsItemValue.getCount();
+          long statisticsItemInfoCount = statisticsItemInfo.getCount();
+          long newCount = statisticsItemValueCount + statisticsItemInfoCount;
+          statisticsItemValue.setCount(newCount);
+        } else {
+          statisticsItemValue = new StatisticsItemInfo(statisticsItemInfo);
+        }
+
+        statisticsItemMap.put(statisticsItemKey, statisticsItemValue);
+      }
+    });
+
+    if (!statisticsItemMap.isEmpty()) {
+      result.getStatItems().addAll(statisticsItemMap.values());
+    }
+
+    return result;
+  }
+
+  public static NodeLabelsInfo mergeNodeLabelsInfo(Map<SubClusterInfo, NodeLabelsInfo> paramMap) {
+    Map<String, NodeLabelInfo> resultMap = new HashMap<>();
+    paramMap.values().stream()
+        .flatMap(nodeLabelsInfo -> nodeLabelsInfo.getNodeLabelsInfo().stream())
+        .forEach(nodeLabelInfo -> {
+          String keyLabelName = nodeLabelInfo.getName();
+          if (resultMap.containsKey(keyLabelName)) {
+            NodeLabelInfo mapNodeLabelInfo = resultMap.get(keyLabelName);
+            mapNodeLabelInfo = mergeNodeLabelInfo(mapNodeLabelInfo, nodeLabelInfo);
+            resultMap.put(keyLabelName, mapNodeLabelInfo);
+          } else {
+            resultMap.put(keyLabelName, nodeLabelInfo);
+          }
+        });
+    NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo();
+    nodeLabelsInfo.getNodeLabelsInfo().addAll(resultMap.values());
+    return nodeLabelsInfo;
+  }
+
+  private static NodeLabelInfo mergeNodeLabelInfo(NodeLabelInfo left, NodeLabelInfo right) {
+    NodeLabelInfo resultNodeLabelInfo = new NodeLabelInfo();
+    resultNodeLabelInfo.setName(left.getName());
+
+    int newActiveNMs = left.getActiveNMs() + right.getActiveNMs();
+    resultNodeLabelInfo.setActiveNMs(newActiveNMs);
+
+    boolean newExclusivity = left.getExclusivity() && right.getExclusivity();
+    resultNodeLabelInfo.setExclusivity(newExclusivity);
+
+    PartitionInfo leftPartition = left.getPartitionInfo();
+    PartitionInfo rightPartition = right.getPartitionInfo();
+    PartitionInfo newPartitionInfo = PartitionInfo.addTo(leftPartition, rightPartition);
+    resultNodeLabelInfo.setPartitionInfo(newPartitionInfo);
+    return resultNodeLabelInfo;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java
index bb48476d652cc..02f545beda085 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java
@@ -19,10 +19,7 @@
 package org.apache.hadoop.yarn.server.router.webapp;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -48,11 +45,8 @@
 import org.apache.hadoop.http.JettyUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServiceProtocol;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
@@ -86,9 +80,11 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.server.router.RouterServerUtil;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
 import org.apache.hadoop.yarn.util.LRUCacheHashMap;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -135,30 +131,7 @@ public RouterWebServices(final Router router, Configuration conf) {
     int maxCacheSize =
         conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE,
             YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE);
-    this.userPipelineMap = Collections.synchronizedMap(
-        new LRUCacheHashMap<String, RequestInterceptorChainWrapper>(
-            maxCacheSize, true));
-  }
-
-  /**
-   * Returns the comma separated intercepter class names from the configuration.
-   *
-   * @param conf
-   * @return the intercepter class names as an instance of ArrayList
-   */
-  private List<String> getInterceptorClassNames(Configuration config) {
-    String configuredInterceptorClassNames =
-        config.get(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE,
-            YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS);
-
-    List<String> interceptorClassNames = new ArrayList<String>();
-    Collection<String> tempList =
-        StringUtils.getStringCollection(configuredInterceptorClassNames);
-    for (String item : tempList) {
-      interceptorClassNames.add(item.trim());
-    }
-
-    return interceptorClassNames;
+    this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true));
   }
 
   private void init() {
@@ -189,9 +162,9 @@ protected RequestInterceptorChainWrapper getInterceptorChain(
   }
 
   /**
-   * Gets the Request intercepter chains for all the users.
+   * Gets the Request interceptor chains for all the users.
    *
-   * @return the request intercepter chains.
+   * @return the request interceptor chains.
    */
   @VisibleForTesting
   protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
@@ -199,57 +172,23 @@ protected Map<String, RequestInterceptorChainWrapper> getPipelines() {
   }
 
   /**
-   * This method creates and returns reference of the first intercepter in the
-   * chain of request intercepter instances.
+   * This method creates and returns reference of the first interceptor in the
+   * chain of request interceptor instances.
    *
-   * @return the reference of the first intercepter in the chain
+   * @return the reference of the first interceptor in the chain
    */
   @VisibleForTesting
   protected RESTRequestInterceptor createRequestInterceptorChain() {
-
-    List<String> interceptorClassNames = getInterceptorClassNames(conf);
-
-    RESTRequestInterceptor pipeline = null;
-    RESTRequestInterceptor current = null;
-    for (String interceptorClassName : interceptorClassNames) {
-      try {
-        Class<?> interceptorClass = conf.getClassByName(interceptorClassName);
-        if (RESTRequestInterceptor.class.isAssignableFrom(interceptorClass)) {
-          RESTRequestInterceptor interceptorInstance =
-              (RESTRequestInterceptor) ReflectionUtils
-                  .newInstance(interceptorClass, conf);
-          if (pipeline == null) {
-            pipeline = interceptorInstance;
-            current = interceptorInstance;
-            continue;
-          } else {
-            current.setNextInterceptor(interceptorInstance);
-            current = interceptorInstance;
-          }
-        } else {
-          throw new YarnRuntimeException(
-              "Class: " + interceptorClassName + " not instance of "
-                  + RESTRequestInterceptor.class.getCanonicalName());
-        }
-      } catch (ClassNotFoundException e) {
-        throw new YarnRuntimeException(
-            "Could not instantiate RESTRequestInterceptor: "
-                + interceptorClassName,
-            e);
-      }
-    }
-
-    if (pipeline == null) {
-      throw new YarnRuntimeException(
-          "RequestInterceptor pipeline is not configured in the system");
-    }
-    return pipeline;
+    return RouterServerUtil.createRequestInterceptorChain(conf,
+        YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE,
+        YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS,
+        RESTRequestInterceptor.class);
   }
 
   /**
-   * Initializes the request intercepter pipeline for the specified user.
+   * Initializes the request interceptor pipeline for the specified user.
    *
-   * @param user
+   * @param user specified user.
    */
   private RequestInterceptorChainWrapper initializePipeline(String user) {
     synchronized (this.userPipelineMap) {
@@ -264,14 +203,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) {
       try {
         // We should init the pipeline instance after it is created and then
         // add to the map, to ensure thread safe.
-        LOG.info("Initializing request processing pipeline for user: {}", user);
+        LOG.info("Initializing request processing pipeline for user: {}.", user);
 
         RESTRequestInterceptor interceptorChain =
             this.createRequestInterceptorChain();
         interceptorChain.init(user);
         chainWrapper.init(interceptorChain);
       } catch (Exception e) {
-        LOG.error("Init RESTRequestInterceptor error for user: " + user, e);
+        LOG.error("Init RESTRequestInterceptor error for user: {}", user, e);
         throw e;
       }
 
@@ -298,9 +237,9 @@ public synchronized void init(RESTRequestInterceptor interceptor) {
     }
 
     /**
-     * Gets the root request intercepter.
+     * Gets the root request interceptor.
      *
-     * @return the root request intercepter
+     * @return the root request interceptor
      */
     public synchronized RESTRequestInterceptor getRootInterceptor() {
       return rootInterceptor;
@@ -337,7 +276,7 @@ public ClusterInfo getClusterInfo() {
   @GET
   @Path(RMWSConsts.CLUSTER_USER_INFO)
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
-          MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
   @Override
   public ClusterUserInfo getClusterUserInfo(@Context HttpServletRequest hsr) {
     init();
@@ -637,13 +576,13 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
   @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
       MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
   @Override
-  public Response removeFromCluserNodeLabels(
+  public Response removeFromClusterNodeLabels(
       @QueryParam(RMWSConsts.LABELS) Set<String> oldNodeLabels,
       @Context HttpServletRequest hsr) throws Exception {
     init();
     RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
     return pipeline.getRootInterceptor()
-        .removeFromCluserNodeLabels(oldNodeLabels, hsr);
+        .removeFromClusterNodeLabels(oldNodeLabels, hsr);
   }
 
   @GET
@@ -833,10 +772,12 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext,
   @Override
   public Response listReservation(
       @QueryParam(RMWSConsts.QUEUE) @DefaultValue(DEFAULT_QUEUE) String queue,
-      @QueryParam(RMWSConsts.RESERVATION_ID) @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId,
+      @QueryParam(RMWSConsts.RESERVATION_ID)
+      @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId,
       @QueryParam(RMWSConsts.START_TIME) @DefaultValue(DEFAULT_START_TIME) long startTime,
       @QueryParam(RMWSConsts.END_TIME) @DefaultValue(DEFAULT_END_TIME) long endTime,
-      @QueryParam(RMWSConsts.INCLUDE_RESOURCE) @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations,
+      @QueryParam(RMWSConsts.INCLUDE_RESOURCE)
+      @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations,
       @Context HttpServletRequest hsr) throws Exception {
     init();
     RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
@@ -956,6 +897,33 @@ public ContainerInfo getContainer(@Context HttpServletRequest req,
         appAttemptId, containerId);
   }
 
+  @PUT
+  @Path(RMWSConsts.SCHEDULER_CONF)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  @Override
+  public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest hsr)
+      throws AuthorizationException, InterruptedException {
+    init();
+    RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
+    return pipeline.getRootInterceptor()
+        .updateSchedulerConfiguration(mutationInfo, hsr);
+  }
+
+  @GET
+  @Path(RMWSConsts.SCHEDULER_CONF)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  @Override
+  public Response getSchedulerConfiguration(HttpServletRequest hsr)
+      throws AuthorizationException {
+    init();
+    RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
+    return pipeline.getRootInterceptor().getSchedulerConfiguration(hsr);
+  }
+
   @VisibleForTesting
   protected void setResponse(HttpServletResponse response) {
     this.response = response;
@@ -975,4 +943,15 @@ public Response signalToContainer(
     return pipeline.getRootInterceptor()
         .signalToContainer(containerId, command, req);
   }
+
+  @GET
+  @Path(RMWSConsts.GET_RM_NODE_LABELS)
+  @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+      MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+  public NodeLabelsInfo getRMNodeLabels(@Context HttpServletRequest hsr)
+      throws IOException {
+    init();
+    RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr);
+    return pipeline.getRootInterceptor().getRMNodeLabels(hsr);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java
new file mode 100644
index 0000000000000..27164f00415fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp.cache;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import java.util.Set;
+
+public class RouterAppInfoCacheKey {
+
+  private static String user = "YarnRouter";
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(RouterAppInfoCacheKey.class.getName());
+
+  private UserGroupInformation ugi;
+  private String stateQuery;
+  private Set<String> statesQuery;
+  private String finalStatusQuery;
+  private String userQuery;
+  private String queueQuery;
+  private String count;
+  private String startedBegin;
+  private String startedEnd;
+  private String finishBegin;
+  private String finishEnd;
+  private Set<String> applicationTypes;
+  private Set<String> applicationTags;
+  private String name;
+  private Set<String> unselectedFields;
+
+  public RouterAppInfoCacheKey() {
+
+  }
+
+  @SuppressWarnings("checkstyle:ParameterNumber")
+  public RouterAppInfoCacheKey(UserGroupInformation ugi, String stateQuery,
+      Set<String> statesQuery, String finalStatusQuery, String userQuery,
+      String queueQuery, String count, String startedBegin, String startedEnd,
+      String finishBegin, String finishEnd, Set<String> applicationTypes,
+      Set<String> applicationTags, String name, Set<String> unselectedFields) {
+    this.ugi = ugi;
+    this.stateQuery = stateQuery;
+    this.statesQuery = statesQuery;
+    this.finalStatusQuery = finalStatusQuery;
+    this.userQuery = userQuery;
+    this.queueQuery = queueQuery;
+    this.count = count;
+    this.startedBegin = startedBegin;
+    this.startedEnd = startedEnd;
+    this.finishBegin = finishBegin;
+    this.finishEnd = finishEnd;
+    this.applicationTypes = applicationTypes;
+    this.applicationTags = applicationTags;
+    this.name = name;
+    this.unselectedFields = unselectedFields;
+  }
+
+
+  @SuppressWarnings("checkstyle:ParameterNumber")
+  public static RouterAppInfoCacheKey newInstance(HttpServletRequest hsr, String stateQuery,
+      Set<String> statesQuery, String finalStatusQuery, String userQuery,
+      String queueQuery, String count, String startedBegin, String startedEnd,
+      String finishBegin, String finishEnd, Set<String> applicationTypes,
+      Set<String> applicationTags, String name, Set<String> unselectedFields)  {
+
+    UserGroupInformation callerUGI = null;
+    if (hsr != null) {
+      callerUGI = RMWebAppUtil.getCallerUserGroupInformation(hsr, true);
+    } else {
+      // user not required
+      callerUGI = UserGroupInformation.createRemoteUser("YarnRouter");
+    }
+
+    if (callerUGI == null) {
+      LOG.error("Unable to obtain user name, user not authenticated.");
+      return null;
+    }
+
+    return new RouterAppInfoCacheKey(
+        callerUGI, stateQuery, statesQuery, finalStatusQuery, userQuery,
+        queueQuery, count, startedBegin, startedEnd, finishBegin, finishEnd,
+        applicationTypes, applicationTags, name, unselectedFields);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    RouterAppInfoCacheKey that = (RouterAppInfoCacheKey) o;
+
+    return new EqualsBuilder()
+        .append(this.ugi.getUserName(), that.ugi.getUserName())
+        .append(this.stateQuery, that.stateQuery)
+        .append(this.statesQuery, that.statesQuery)
+        .append(this.finalStatusQuery, that.finalStatusQuery)
+        .append(this.userQuery, that.userQuery)
+        .append(this.queueQuery, that.queueQuery)
+        .append(this.count, that.count)
+        .append(this.startedBegin, that.startedBegin)
+        .append(this.startedEnd, that.startedEnd)
+        .append(this.finishBegin, that.finishBegin)
+        .append(this.finishEnd, that.finishEnd)
+        .append(this.applicationTypes, that.applicationTypes)
+        .append(this.applicationTags, that.applicationTags)
+        .append(this.name, that.name)
+        .append(this.unselectedFields, that.unselectedFields)
+        .isEquals();
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder()
+       .append(this.ugi.getUserName())
+       .append(this.stateQuery)
+       .append(this.statesQuery)
+       .append(this.finalStatusQuery)
+       .append(this.userQuery)
+       .append(this.queueQuery)
+       .append(this.count)
+       .append(this.startedBegin)
+       .append(this.startedEnd)
+       .append(this.finishBegin)
+       .append(this.finishEnd)
+       .append(this.applicationTypes)
+       .append(this.applicationTags)
+       .append(this.name)
+       .append(this.unselectedFields)
+       .toHashCode();
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java
similarity index 93%
rename from hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java
index 6233024467df7..187cd72fe257f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java
@@ -15,4 +15,4 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.hdfs.protocolPB;
\ No newline at end of file
+package org.apache.hadoop.yarn.server.router.webapp.cache;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/FederationRMQueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/FederationRMQueueAclInfo.java
new file mode 100644
index 0000000000000..4e61fd772eeac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/FederationRMQueueAclInfo.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp.dao;
+
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.List;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class FederationRMQueueAclInfo extends RMQueueAclInfo {
+
+  @XmlElement(name = "subCluster")
+  private List<RMQueueAclInfo> list = new ArrayList<>();
+
+  public FederationRMQueueAclInfo() {
+  } // JAXB needs this
+
+  public FederationRMQueueAclInfo(ArrayList<RMQueueAclInfo> list) {
+    this.list = list;
+  }
+
+  public List<RMQueueAclInfo> getList() {
+    return list;
+  }
+
+  public void setList(List<RMQueueAclInfo> list) {
+    this.list = list;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java
new file mode 100644
index 0000000000000..f06f85574dbd2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterClusterMetrics.java
@@ -0,0 +1,323 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp.dao;
+
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RouterClusterMetrics {
+
+  protected static final long BYTES_IN_MB = 1024 * 1024;
+  private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
+
+  // webPageTitlePrefix
+  private String webPageTitlePrefix = "Federation";
+
+  // Application Information.
+  private String appsSubmitted = "N/A";
+  private String appsCompleted = "N/A";
+  private String appsPending = "N/A";
+  private String appsRunning = "N/A";
+  private String appsFailed = "N/A";
+  private String appsKilled = "N/A";
+
+  // Memory Information.
+  private String totalMemory = "N/A";
+  private String reservedMemory = "N/A";
+  private String availableMemory = "N/A";
+  private String allocatedMemory = "N/A";
+  private String pendingMemory = "N/A";
+
+  // VirtualCores Information.
+  private String reservedVirtualCores = "N/A";
+  private String availableVirtualCores = "N/A";
+  private String allocatedVirtualCores = "N/A";
+  private String pendingVirtualCores = "N/A";
+  private String totalVirtualCores = "N/A";
+
+  // Resources Information.
+  private String usedResources = "N/A";
+  private String totalResources = "N/A";
+  private String reservedResources = "N/A";
+  private String allocatedContainers = "N/A";
+
+  // Resource Percent Information.
+  private String utilizedMBPercent = "N/A";
+  private String utilizedVirtualCoresPercent = "N/A";
+
+  // Node Information.
+  private String activeNodes = "N/A";
+  private String decommissioningNodes = "N/A";
+  private String decommissionedNodes = "N/A";
+  private String lostNodes = "N/A";
+  private String unhealthyNodes = "N/A";
+  private String rebootedNodes = "N/A";
+  private String shutdownNodes = "N/A";
+
+  public RouterClusterMetrics() {
+
+  }
+
+  public RouterClusterMetrics(ClusterMetricsInfo metrics) {
+    if (metrics != null) {
+      // Application Information Conversion.
+      conversionApplicationInformation(metrics);
+
+      // Memory Information Conversion.
+      conversionMemoryInformation(metrics);
+
+      // Resources Information Conversion.
+      conversionResourcesInformation(metrics);
+
+      // Percent Information Conversion.
+      conversionResourcesPercent(metrics);
+
+      // Node Information Conversion.
+      conversionNodeInformation(metrics);
+    }
+  }
+
+  public RouterClusterMetrics(ClusterMetricsInfo metrics,
+      String webPageTitlePrefix) {
+    this(metrics);
+    this.webPageTitlePrefix = webPageTitlePrefix;
+  }
+
+  // Get Key Metric Information
+  public String getAppsSubmitted() {
+    return appsSubmitted;
+  }
+
+  public String getAppsCompleted() {
+    return appsCompleted;
+  }
+
+  public String getAppsPending() {
+    return appsPending;
+  }
+
+  public String getAppsRunning() {
+    return appsRunning;
+  }
+
+  public String getAppsFailed() {
+    return appsFailed;
+  }
+
+  public String getAppsKilled() {
+    return appsKilled;
+  }
+
+  public String getTotalMemory() {
+    return totalMemory;
+  }
+
+  public String getReservedMemory() {
+    return reservedMemory;
+  }
+
+  public String getAvailableMemory() {
+    return availableMemory;
+  }
+
+  public String getAllocatedMemory() {
+    return allocatedMemory;
+  }
+
+  public String getPendingMemory() {
+    return pendingMemory;
+  }
+
+  public String getReservedVirtualCores() {
+    return reservedVirtualCores;
+  }
+
+  public String getAvailableVirtualCores() {
+    return availableVirtualCores;
+  }
+
+  public String getAllocatedVirtualCores() {
+    return allocatedVirtualCores;
+  }
+
+  public String getPendingVirtualCores() {
+    return pendingVirtualCores;
+  }
+
+  public String getTotalVirtualCores() {
+    return totalVirtualCores;
+  }
+
+  public String getUsedResources() {
+    return usedResources;
+  }
+
+  public String getTotalResources() {
+    return totalResources;
+  }
+
+  public String getReservedResources() {
+    return reservedResources;
+  }
+
+  public String getAllocatedContainers() {
+    return allocatedContainers;
+  }
+
+  public String getUtilizedMBPercent() {
+    return utilizedMBPercent;
+  }
+
+  public String getUtilizedVirtualCoresPercent() {
+    return utilizedVirtualCoresPercent;
+  }
+
+  public String getActiveNodes() {
+    return activeNodes;
+  }
+
+  public String getDecommissioningNodes() {
+    return decommissioningNodes;
+  }
+
+  public String getDecommissionedNodes() {
+    return decommissionedNodes;
+  }
+
+  public String getLostNodes() {
+    return lostNodes;
+  }
+
+  public String getUnhealthyNodes() {
+    return unhealthyNodes;
+  }
+
+  public String getRebootedNodes() {
+    return rebootedNodes;
+  }
+
+  public String getShutdownNodes() {
+    return shutdownNodes;
+  }
+
+  // Metric Information Conversion
+  public void conversionApplicationInformation(ClusterMetricsInfo metrics) {
+    try {
+      // Application Information.
+      this.appsSubmitted = String.valueOf(metrics.getAppsSubmitted());
+      this.appsCompleted = String.valueOf(metrics.getAppsCompleted() +
+           metrics.getAppsFailed() + metrics.getAppsKilled());
+      this.appsPending = String.valueOf(metrics.getAppsPending());
+      this.appsRunning = String.valueOf(metrics.getAppsRunning());
+      this.appsFailed = String.valueOf(metrics.getAppsFailed());
+      this.appsKilled = String.valueOf(metrics.getAppsKilled());
+    } catch (Exception e) {
+      LOG.error("conversionApplicationInformation error.", e);
+    }
+  }
+
+  // Metric Memory Information
+  public void conversionMemoryInformation(ClusterMetricsInfo metrics) {
+    try {
+      // Memory Information.
+      this.totalMemory = StringUtils.byteDesc(metrics.getTotalMB() * BYTES_IN_MB);
+      this.reservedMemory = StringUtils.byteDesc(metrics.getReservedMB() * BYTES_IN_MB);
+      this.availableMemory = StringUtils.byteDesc(metrics.getAvailableMB() * BYTES_IN_MB);
+      this.allocatedMemory = StringUtils.byteDesc(metrics.getAllocatedMB() * BYTES_IN_MB);
+      this.pendingMemory = StringUtils.byteDesc(metrics.getPendingMB() * BYTES_IN_MB);
+    } catch (Exception e) {
+      LOG.error("conversionMemoryInformation error.", e);
+    }
+  }
+
+  // ResourcesInformation Conversion
+  public void conversionResourcesInformation(ClusterMetricsInfo metrics) {
+    try {
+      // Parse resource information from metrics.
+      Resource metricUsedResources;
+      Resource metricTotalResources;
+      Resource metricReservedResources;
+
+      int metricAllocatedContainers;
+      if (metrics.getCrossPartitionMetricsAvailable()) {
+        metricAllocatedContainers = metrics.getTotalAllocatedContainersAcrossPartition();
+        metricUsedResources = metrics.getTotalUsedResourcesAcrossPartition().getResource();
+        metricTotalResources = metrics.getTotalClusterResourcesAcrossPartition().getResource();
+        metricReservedResources = metrics.getTotalReservedResourcesAcrossPartition().getResource();
+        // getTotalUsedResourcesAcrossPartition includes reserved resources.
+        Resources.subtractFrom(metricUsedResources, metricReservedResources);
+      } else {
+        metricAllocatedContainers = metrics.getContainersAllocated();
+        metricUsedResources = Resource.newInstance(metrics.getAllocatedMB(),
+            (int) metrics.getAllocatedVirtualCores());
+        metricTotalResources = Resource.newInstance(metrics.getTotalMB(),
+            (int) metrics.getTotalVirtualCores());
+        metricReservedResources = Resource.newInstance(metrics.getReservedMB(),
+            (int) metrics.getReservedVirtualCores());
+      }
+
+      // Convert to standard format.
+      usedResources = metricUsedResources.getFormattedString();
+      totalResources = metricTotalResources.getFormattedString();
+      reservedResources = metricReservedResources.getFormattedString();
+      allocatedContainers =  String.valueOf(metricAllocatedContainers);
+
+    } catch (Exception e) {
+      LOG.error("conversionResourcesInformation error.", e);
+    }
+  }
+
+  // ResourcesPercent Conversion
+  public void conversionResourcesPercent(ClusterMetricsInfo metrics) {
+    try {
+      this.utilizedMBPercent = String.valueOf(metrics.getUtilizedMBPercent());
+      this.utilizedVirtualCoresPercent = String.valueOf(metrics.getUtilizedVirtualCoresPercent());
+    } catch (Exception e) {
+      LOG.error("conversionResourcesPercent error.", e);
+    }
+  }
+
+  // NodeInformation Conversion
+  public void conversionNodeInformation(ClusterMetricsInfo metrics) {
+    try {
+      this.activeNodes = String.valueOf(metrics.getActiveNodes());
+      this.decommissioningNodes = String.valueOf(metrics.getDecommissioningNodes());
+      this.decommissionedNodes = String.valueOf(metrics.getDecommissionedNodes());
+      this.lostNodes = String.valueOf(metrics.getLostNodes());
+      this.unhealthyNodes = String.valueOf(metrics.getUnhealthyNodes());
+      this.rebootedNodes = String.valueOf(metrics.getRebootedNodes());
+      this.shutdownNodes = String.valueOf(metrics.getShutdownNodes());
+    } catch (Exception e) {
+      LOG.error("conversionNodeInformation error.", e);
+    }
+  }
+
+  public String getWebPageTitlePrefix() {
+    return webPageTitlePrefix;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java
new file mode 100644
index 0000000000000..7cedd0c8e1fbf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterInfo.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp.dao;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RouterInfo {
+  private long id;
+  private long startedOn;
+  private Service.STATE state;
+  private String routerStateStoreName;
+  private String routerVersion;
+  private String routerBuildVersion;
+  private String routerVersionBuiltOn;
+  private String hadoopVersion;
+  private String hadoopBuildVersion;
+  private String hadoopVersionBuiltOn;
+
+  public RouterInfo() {
+  } // JAXB needs this
+
+  public RouterInfo(Router router) {
+    long ts = Router.getClusterTimeStamp();
+    this.id = ts;
+    this.state = router.getServiceState();
+    Configuration configuration = router.getConfig();
+    this.routerStateStoreName = configuration.get(
+        YarnConfiguration.FEDERATION_STATESTORE_CLIENT_CLASS,
+        YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+    this.routerVersion = YarnVersionInfo.getVersion();
+    this.routerBuildVersion = YarnVersionInfo.getBuildVersion();
+    this.routerVersionBuiltOn = YarnVersionInfo.getDate();
+    this.hadoopVersion = VersionInfo.getVersion();
+    this.hadoopBuildVersion = VersionInfo.getBuildVersion();
+    this.hadoopVersionBuiltOn = VersionInfo.getDate();
+    this.startedOn = ts;
+  }
+
+  public String getState() {
+    return this.state.toString();
+  }
+
+  public String getRouterStateStore() {
+    return this.routerStateStoreName;
+  }
+
+  public String getRouterVersion() {
+    return this.routerVersion;
+  }
+
+  public String getRouterBuildVersion() {
+    return this.routerBuildVersion;
+  }
+
+  public String getRouterVersionBuiltOn() {
+    return this.routerVersionBuiltOn;
+  }
+
+  public String getHadoopVersion() {
+    return this.hadoopVersion;
+  }
+
+  public String getHadoopBuildVersion() {
+    return this.hadoopBuildVersion;
+  }
+
+  public String getHadoopVersionBuiltOn() {
+    return this.hadoopVersionBuiltOn;
+  }
+
+  public long getClusterId() {
+    return this.id;
+  }
+
+  public long getStartedOn() {
+    return this.startedOn;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
new file mode 100644
index 0000000000000..4a3af1ba43ddc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp.dao;
+
+
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerOverviewInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RouterSchedulerMetrics {
+
+  // Metrics Log.
+  private static final Logger LOG = LoggerFactory.getLogger(RouterSchedulerMetrics.class);
+
+  // Scheduler Information.
+  private String subCluster = "N/A";
+  private String schedulerType = "N/A";
+  private String schedulingResourceType = "N/A";
+  private String minimumAllocation = "N/A";
+  private String maximumAllocation = "N/A";
+  private String applicationPriority = "N/A";
+  private String schedulerBusy = "N/A";
+  private String rmDispatcherEventQueueSize = "N/A";
+  private String schedulerDispatcherEventQueueSize = "N/A";
+
+  public RouterSchedulerMetrics() {
+
+  }
+
+  public RouterSchedulerMetrics(SubClusterInfo subClusterInfo, RouterClusterMetrics metrics,
+      SchedulerOverviewInfo overview) {
+    try {
+      // Parse Scheduler Information.
+      this.subCluster = subClusterInfo.getSubClusterId().getId();
+      this.schedulerType = overview.getSchedulerType();
+      this.schedulingResourceType = overview.getSchedulingResourceType();
+      this.minimumAllocation = overview.getMinimumAllocation().toString();
+      this.maximumAllocation = overview.getMaximumAllocation().toString();
+      this.applicationPriority = String.valueOf(overview.getApplicationPriority());
+      if (overview.getSchedulerBusy() != -1) {
+        this.schedulerBusy = String.valueOf(overview.getSchedulerBusy());
+      }
+      this.rmDispatcherEventQueueSize =
+          String.valueOf(overview.getRmDispatcherEventQueueSize());
+      this.schedulerDispatcherEventQueueSize =
+          String.valueOf(overview.getSchedulerDispatcherEventQueueSize());
+    } catch (Exception ex) {
+      LOG.error("RouterSchedulerMetrics Error.", ex);
+    }
+  }
+
+  public String getSubCluster() {
+    return subCluster;
+  }
+
+  public String getSchedulerType() {
+    return schedulerType;
+  }
+
+  public String getSchedulingResourceType() {
+    return schedulingResourceType;
+  }
+
+  public String getMinimumAllocation() {
+    return minimumAllocation;
+  }
+
+  public String getMaximumAllocation() {
+    return maximumAllocation;
+  }
+
+  public String getApplicationPriority() {
+    return applicationPriority;
+  }
+
+  public String getRmDispatcherEventQueueSize() {
+    return rmDispatcherEventQueueSize;
+  }
+
+  public String getSchedulerDispatcherEventQueueSize() {
+    return schedulerDispatcherEventQueueSize;
+  }
+
+  public String getSchedulerBusy() {
+    return schedulerBusy;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java
new file mode 100644
index 0000000000000..27f43ad1ff4ae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Router Web Dao package. **/
+package org.apache.hadoop.yarn.server.router.webapp.dao;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
index 9f0b4c72aac3b..d5501d74440d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
@@ -22,11 +22,27 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
+import org.apache.hadoop.security.http.CrossOriginFilter;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletHandler;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * Tests {@link Router}.
@@ -87,4 +103,97 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager,
     }
   }
 
+  @Test
+  public void testRouterSupportCrossOrigin() throws ServletException, IOException {
+
+    // We design test cases like this
+    // We start the Router and enable the Router to support Cross-origin.
+    // In the configuration, we allow example.com to access.
+    // 1. We simulate example.com and get the correct response
+    // 2. We simulate example.org and cannot get a response
+
+    // Initialize RouterWeb's CrossOrigin capability
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.ROUTER_WEBAPP_ENABLE_CORS_FILTER, true);
+    conf.set("hadoop.http.filter.initializers", HttpCrossOriginFilterInitializer.class.getName());
+    conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_ORIGINS,
+        "example.com");
+    conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_HEADERS,
+        "X-Requested-With,Accept");
+    conf.set(HttpCrossOriginFilterInitializer.PREFIX + CrossOriginFilter.ALLOWED_METHODS,
+        "GET,POST");
+
+    // Start the router
+    Router router = new Router();
+    router.init(conf);
+    router.start();
+    router.getServices();
+
+    // Get assigned to Filter.
+    // The name of the filter is "Cross Origin Filter",
+    // which is specified in HttpCrossOriginFilterInitializer.
+    WebApp webApp = router.getWebapp();
+    HttpServer2 httpServer2 = webApp.getHttpServer();
+    WebAppContext webAppContext = httpServer2.getWebAppContext();
+    ServletHandler servletHandler = webAppContext.getServletHandler();
+    FilterHolder holder = servletHandler.getFilter("Cross Origin Filter");
+    CrossOriginFilter filter = CrossOriginFilter.class.cast(holder.getFilter());
+
+    // 1. Simulate [example.com] for access
+    HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class);
+    Mockito.when(mockReq.getHeader("Origin")).thenReturn("example.com");
+    Mockito.when(mockReq.getHeader("Access-Control-Request-Method")).thenReturn("GET");
+    Mockito.when(mockReq.getHeader("Access-Control-Request-Headers"))
+        .thenReturn("X-Requested-With");
+
+    // Objects to verify interactions based on request
+    HttpServletResponseForRouterTest mockRes = new HttpServletResponseForRouterTest();
+    FilterChain mockChain = Mockito.mock(FilterChain.class);
+
+    // Object under test
+    filter.doFilter(mockReq, mockRes, mockChain);
+
+    // Why is 5, because when Filter passes,
+    // CrossOriginFilter will set 5 values to Map
+    Assert.assertEquals(5, mockRes.getHeaders().size());
+    String allowResult = mockRes.getHeader("Access-Control-Allow-Credentials");
+    Assert.assertEquals("true", allowResult);
+
+    // 2. Simulate [example.org] for access
+    HttpServletRequest mockReq2 = Mockito.mock(HttpServletRequest.class);
+    Mockito.when(mockReq2.getHeader("Origin")).thenReturn("example.org");
+    Mockito.when(mockReq2.getHeader("Access-Control-Request-Method")).thenReturn("GET");
+    Mockito.when(mockReq2.getHeader("Access-Control-Request-Headers"))
+        .thenReturn("X-Requested-With");
+
+    // Objects to verify interactions based on request
+    HttpServletResponseForRouterTest mockRes2 = new HttpServletResponseForRouterTest();
+    FilterChain mockChain2 = Mockito.mock(FilterChain.class);
+
+    // Object under test
+    filter.doFilter(mockReq2, mockRes2, mockChain2);
+
+    // Why is 0, because when the Filter fails,
+    // CrossOriginFilter will not set any value
+    Assert.assertEquals(0, mockRes2.getHeaders().size());
+
+    router.stop();
+  }
+
+  private class HttpServletResponseForRouterTest extends HttpServletResponseImpl {
+    private final Map<String, String> headers = new HashMap<>(1);
+    @Override
+    public void setHeader(String name, String value) {
+      headers.put(name, value);
+    }
+
+    public String getHeader(String name) {
+      return headers.get(name);
+    }
+
+    public Map<String, String> getHeaders() {
+      return headers;
+    }
+  }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java
new file mode 100644
index 0000000000000..48d3ef6c0fea4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java
@@ -0,0 +1,246 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ClientId;
+import org.apache.hadoop.ipc.ProtobufRpcEngine2;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
+import org.apache.hadoop.ipc.TestRPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.thirdparty.protobuf.BlockingService;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+/**
+ * Tests {@link RouterAuditLogger}.
+ */
+public class TestRouterAuditLogger {
+  private static final String USER = "test";
+  private static final String OPERATION = "oper";
+  private static final String TARGET = "tgt";
+  private static final String DESC = "description of an audit log";
+
+  private static final ApplicationId APPID = mock(ApplicationId.class);
+  private static final SubClusterId SUBCLUSTERID = mock(SubClusterId.class);
+
+  @Before public void setUp() throws Exception {
+    when(APPID.toString()).thenReturn("app_1");
+    when(SUBCLUSTERID.toString()).thenReturn("sc0");
+  }
+
+  /**
+   * Test the AuditLog format with key-val pair.
+   */
+  @Test
+  public void testKeyValLogFormat() throws Exception {
+    StringBuilder actLog = new StringBuilder();
+    StringBuilder expLog = new StringBuilder();
+
+    // add the first k=v pair and check
+    RouterAuditLogger.start(RouterAuditLogger.Keys.USER, USER, actLog);
+    expLog.append("USER=test");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=v1 pair to already added k=v and test
+    RouterAuditLogger.add(RouterAuditLogger.Keys.OPERATION, OPERATION, actLog);
+    expLog.append("\tOPERATION=oper");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // append another k1=null pair and test
+    RouterAuditLogger.add(RouterAuditLogger.Keys.APPID, (String) null, actLog);
+    expLog.append("\tAPPID=null");
+    assertEquals(expLog.toString(), actLog.toString());
+
+    // now add the target and check of the final string
+    RouterAuditLogger.add(RouterAuditLogger.Keys.TARGET, TARGET, actLog);
+    expLog.append("\tTARGET=tgt");
+    assertEquals(expLog.toString(), actLog.toString());
+  }
+
+  /**
+   * Test the AuditLog format for successful events.
+   */
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+      SubClusterId subClusterId) {
+    // check without the IP
+    String sLog = RouterAuditLogger
+        .createSuccessLog(USER, OPERATION, TARGET, appId, subClusterId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (subClusterId != null) {
+      expLog.append("\tSUBCLUSTERID=sc0");
+    }
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events passing nulls.
+   */
+  private void testSuccessLogNulls() {
+    String sLog =
+        RouterAuditLogger.createSuccessLog(null, null, null, null, null);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=null\t");
+    expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
+    assertEquals(expLog.toString(), sLog);
+  }
+
+  /**
+   * Test the AuditLog format for successful events with the various
+   * parameters.
+   */
+  private void testSuccessLogFormat(boolean checkIP) {
+    testSuccessLogFormatHelper(checkIP, null, null);
+    testSuccessLogFormatHelper(checkIP, APPID, null);
+    testSuccessLogFormatHelper(checkIP, null, SUBCLUSTERID);
+    testSuccessLogFormatHelper(checkIP, APPID, SUBCLUSTERID);
+  }
+
+  /**
+   *  Test the AuditLog format for failure events.
+   */
+  private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+      SubClusterId subClusterId) {
+    String fLog = RouterAuditLogger
+        .createFailureLog(USER, OPERATION, "UNKNOWN", TARGET, DESC, appId,
+            subClusterId);
+    StringBuilder expLog = new StringBuilder();
+    expLog.append("USER=test\t");
+    if (checkIP) {
+      InetAddress ip = Server.getRemoteIp();
+      expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    }
+    expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
+    expLog.append("DESCRIPTION=description of an audit log");
+    expLog.append("\tPERMISSIONS=UNKNOWN");
+
+    if (appId != null) {
+      expLog.append("\tAPPID=app_1");
+    }
+    if (subClusterId != null) {
+      expLog.append("\tSUBCLUSTERID=sc0");
+    }
+    assertEquals(expLog.toString(), fLog);
+  }
+
+  /**
+   * Test the AuditLog format for failure events with the various
+   * parameters.
+   */
+  private void testFailureLogFormat(boolean checkIP) {
+    testFailureLogFormatHelper(checkIP, null, null);
+    testFailureLogFormatHelper(checkIP, APPID, null);
+    testFailureLogFormatHelper(checkIP, null, SUBCLUSTERID);
+    testFailureLogFormatHelper(checkIP, APPID, SUBCLUSTERID);
+  }
+
+  /**
+   *  Test {@link RouterAuditLogger}.
+   */
+  @Test
+  public void testRouterAuditLoggerWithOutIP() throws Exception {
+    testSuccessLogFormat(false);
+    testFailureLogFormat(false);
+  }
+
+  /**
+   * A special extension of {@link TestRPC.TestImpl} RPC server with
+   * {@link TestRPC.TestImpl#ping()} testing the audit logs.
+   */
+  private class MyTestRouterRPCServer extends TestRpcBase.PBServerImpl {
+    @Override
+    public TestProtos.EmptyResponseProto ping(
+            RpcController unused, TestProtos.EmptyRequestProto request)
+            throws ServiceException {
+      // Ensure clientId is received
+      byte[] clientId = Server.getClientId();
+      Assert.assertNotNull(clientId);
+      Assert.assertEquals(ClientId.BYTE_LENGTH, clientId.length);
+      // test with ip set
+      testSuccessLogFormat(true);
+      testFailureLogFormat(true);
+      return TestProtos.EmptyResponseProto.newBuilder().build();
+    }
+  }
+
+  /**
+   * Test {@link RouterAuditLogger} with IP set.
+   */
+  @Test
+  public void testRouterAuditLoggerWithIP() throws Exception {
+    Configuration conf = new Configuration();
+    RPC.setProtocolEngine(conf, TestRpcBase.TestRpcService.class, ProtobufRpcEngine2.class);
+
+    // Create server side implementation
+    MyTestRouterRPCServer serverImpl = new MyTestRouterRPCServer();
+    BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto
+        .newReflectiveBlockingService(serverImpl);
+
+    // start the IPC server
+    Server server = new RPC.Builder(conf)
+        .setProtocol(TestRpcBase.TestRpcService.class)
+        .setInstance(service).setBindAddress("0.0.0.0")
+        .setPort(0).setNumHandlers(5).setVerbose(true).build();
+
+    server.start();
+
+    InetSocketAddress address = NetUtils.getConnectAddress(server);
+
+    // Make a client connection and test the audit log
+    TestRpcBase.TestRpcService proxy = null;
+    try {
+      proxy = RPC.getProxy(TestRpcBase.TestRpcService.class,
+          TestRPC.TestProtocol.versionID, address, conf);
+      // Start the testcase
+      TestProtos.EmptyRequestProto pingRequest =
+          TestProtos.EmptyRequestProto.newBuilder().build();
+      proxy.ping(null, pingRequest);
+    } finally {
+      server.stop();
+      if (proxy != null) {
+        RPC.stopProxy(proxy);
+      }
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
index 4d4838a560b48..9d5aeab5c6d80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
@@ -38,6 +38,8 @@ public class TestRouterMetrics {
 
   private static RouterMetrics metrics = RouterMetrics.getMetrics();
 
+  private static final Double ASSERT_DOUBLE_DELTA = 0.01;
+
   @BeforeClass
   public static void init() {
 
@@ -208,21 +210,21 @@ public void testAppsReportFailed() {
   @Test
   public void testSucceededAppAttemptReport() {
 
-    long totalGoodBefore = metrics.getNumSucceededAppAttemptsRetrieved();
+    long totalGoodBefore = metrics.getNumSucceededAppAttemptReportRetrieved();
 
     goodSubCluster.getApplicationAttemptReport(100);
 
     Assert.assertEquals(totalGoodBefore + 1,
-        metrics.getNumSucceededAppAttemptsRetrieved());
+        metrics.getNumSucceededAppAttemptReportRetrieved());
     Assert.assertEquals(100,
-        metrics.getLatencySucceededGetAppAttemptReport(), 0);
+        metrics.getLatencySucceededGetAppAttemptReport(), ASSERT_DOUBLE_DELTA);
 
     goodSubCluster.getApplicationAttemptReport(200);
 
     Assert.assertEquals(totalGoodBefore + 2,
-        metrics.getNumSucceededAppAttemptsRetrieved());
+        metrics.getNumSucceededAppAttemptReportRetrieved());
     Assert.assertEquals(150,
-        metrics.getLatencySucceededGetAppAttemptReport(), 0);
+        metrics.getLatencySucceededGetAppAttemptReport(), ASSERT_DOUBLE_DELTA);
   }
 
   /**
@@ -232,12 +234,12 @@ public void testSucceededAppAttemptReport() {
   @Test
   public void testAppAttemptReportFailed() {
 
-    long totalBadbefore = metrics.getAppAttemptsFailedRetrieved();
+    long totalBadBefore = metrics.getAppAttemptReportFailedRetrieved();
 
     badSubCluster.getApplicationAttemptReport();
 
-    Assert.assertEquals(totalBadbefore + 1,
-        metrics.getAppAttemptsFailedRetrieved());
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppAttemptReportFailedRetrieved());
   }
 
   /**
@@ -334,7 +336,7 @@ public void getApplicationReport() {
 
     public void getApplicationAttemptReport() {
       LOG.info("Mocked: failed getApplicationAttemptReport call");
-      metrics.incrAppsFailedRetrieved();
+      metrics.incrAppAttemptReportFailedRetrieved();
     }
 
     public void getApplicationsReport() {
@@ -346,6 +348,191 @@ public void getClusterMetrics() {
       LOG.info("Mocked: failed getClusterMetrics call");
       metrics.incrGetClusterMetricsFailedRetrieved();
     }
+
+    public void getClusterNodes() {
+      LOG.info("Mocked: failed getClusterNodes call");
+      metrics.incrClusterNodesFailedRetrieved();
+    }
+
+    public void getNodeToLabels() {
+      LOG.info("Mocked: failed getNodeToLabels call");
+      metrics.incrNodeToLabelsFailedRetrieved();
+    }
+
+    public void getLabelToNodes() {
+      LOG.info("Mocked: failed getLabelToNodes call");
+      metrics.incrLabelsToNodesFailedRetrieved();
+    }
+
+    public void getClusterNodeLabels() {
+      LOG.info("Mocked: failed getClusterNodeLabels call");
+      metrics.incrClusterNodeLabelsFailedRetrieved();
+    }
+
+    public void getQueueUserAcls() {
+      LOG.info("Mocked: failed getQueueUserAcls call");
+      metrics.incrQueueUserAclsFailedRetrieved();
+    }
+
+    public void getListReservations() {
+      LOG.info("Mocked: failed listReservations call");
+      metrics.incrListReservationsFailedRetrieved();
+    }
+
+    public void getApplicationAttempts() {
+      LOG.info("Mocked: failed getApplicationAttempts call");
+      metrics.incrAppAttemptsFailedRetrieved();
+    }
+
+    public void getContainerReport() {
+      LOG.info("Mocked: failed getContainerReport call");
+      metrics.incrGetContainerReportFailedRetrieved();
+    }
+
+    public void getContainers() {
+      LOG.info("Mocked: failed getContainer call");
+      metrics.incrGetContainersFailedRetrieved();
+    }
+
+    public void getResourceTypeInfo() {
+      LOG.info("Mocked: failed getResourceTypeInfo call");
+      metrics.incrResourceTypeInfoFailedRetrieved();
+    }
+
+    public void getFailApplicationAttempt() {
+      LOG.info("Mocked: failed failApplicationAttempt call");
+      metrics.incrFailAppAttemptFailedRetrieved();
+    }
+
+    public void getUpdateApplicationPriority() {
+      LOG.info("Mocked: failed updateApplicationPriority call");
+      metrics.incrUpdateAppPriorityFailedRetrieved();
+    }
+
+    public void getUpdateApplicationTimeouts() {
+      LOG.info("Mocked: failed updateApplicationTimeouts call");
+      metrics.incrUpdateApplicationTimeoutsRetrieved();
+    }
+
+    public void getSignalContainer() {
+      LOG.info("Mocked: failed signalContainer call");
+      metrics.incrSignalToContainerFailedRetrieved();
+    }
+
+    public void getQueueInfo() {
+      LOG.info("Mocked: failed getQueueInfo call");
+      metrics.incrGetQueueInfoFailedRetrieved();
+    }
+
+    public void moveApplicationAcrossQueuesFailed() {
+      LOG.info("Mocked: failed moveApplicationAcrossQueuesFailed call");
+      metrics.incrMoveApplicationAcrossQueuesFailedRetrieved();
+    }
+
+    public void getResourceProfilesFailed() {
+      LOG.info("Mocked: failed getResourceProfilesFailed call");
+      metrics.incrGetResourceProfilesFailedRetrieved();
+    }
+
+    public void getResourceProfileFailed() {
+      LOG.info("Mocked: failed getResourceProfileFailed call");
+      metrics.incrGetResourceProfileFailedRetrieved();
+    }
+
+    public void getAttributesToNodesFailed() {
+      LOG.info("Mocked: failed getAttributesToNodesFailed call");
+      metrics.incrGetAttributesToNodesFailedRetrieved();
+    }
+
+    public void getClusterNodeAttributesFailed() {
+      LOG.info("Mocked: failed getClusterNodeAttributesFailed call");
+      metrics.incrGetClusterNodeAttributesFailedRetrieved();
+    }
+
+    public void getNodesToAttributesFailed() {
+      LOG.info("Mocked: failed getNodesToAttributesFailed call");
+      metrics.incrGetNodesToAttributesFailedRetrieved();
+    }
+
+    public void getNewReservationFailed() {
+      LOG.info("Mocked: failed getNewReservationFailed call");
+      metrics.incrGetNewReservationFailedRetrieved();
+    }
+
+    public void getSubmitReservationFailed() {
+      LOG.info("Mocked: failed getSubmitReservationFailed call");
+      metrics.incrSubmitReservationFailedRetrieved();
+    }
+
+    public void getUpdateReservationFailed() {
+      LOG.info("Mocked: failed getUpdateReservationFailed call");
+      metrics.incrUpdateReservationFailedRetrieved();
+    }
+
+    public void getDeleteReservationFailed() {
+      LOG.info("Mocked: failed getDeleteReservationFailed call");
+      metrics.incrDeleteReservationFailedRetrieved();
+    }
+
+    public void getListReservationFailed() {
+      LOG.info("Mocked: failed getListReservationFailed call");
+      metrics.incrListReservationFailedRetrieved();
+    }
+
+    public void getAppActivitiesFailed() {
+      LOG.info("Mocked: failed getAppActivitiesFailed call");
+      metrics.incrGetAppActivitiesFailedRetrieved();
+    }
+
+    public void getAppStatisticsFailed() {
+      LOG.info("Mocked: failed getAppStatisticsFailed call");
+      metrics.incrGetAppStatisticsFailedRetrieved();
+    }
+
+    public void getAppPriorityFailed() {
+      LOG.info("Mocked: failed getAppPriorityFailed call");
+      metrics.incrGetAppPriorityFailedRetrieved();
+    }
+
+    public void getAppQueueFailed() {
+      LOG.info("Mocked: failed getAppQueueFailed call");
+      metrics.incrGetAppQueueFailedRetrieved();
+    }
+
+    public void getUpdateQueueFailed() {
+      LOG.info("Mocked: failed getUpdateQueueFailed call");
+      metrics.incrUpdateAppQueueFailedRetrieved();
+    }
+
+    public void getAppTimeoutFailed() {
+      LOG.info("Mocked: failed getAppTimeoutFailed call");
+      metrics.incrGetAppTimeoutFailedRetrieved();
+    }
+
+    public void getAppTimeoutsFailed() {
+      LOG.info("Mocked: failed getAppTimeoutsFailed call");
+      metrics.incrGetAppTimeoutsFailedRetrieved();
+    }
+
+    public void getRMNodeLabelsFailed() {
+      LOG.info("Mocked: failed getRMNodeLabelsFailed call");
+      metrics.incrGetRMNodeLabelsFailedRetrieved();
+    }
+
+    public void getCheckUserAccessToQueueFailed() {
+      LOG.info("Mocked: failed checkUserAccessToQueue call");
+      metrics.incrCheckUserAccessToQueueFailedRetrieved();
+    }
+
+    public void getDelegationTokenFailed() {
+      LOG.info("Mocked: failed getDelegationToken call");
+      metrics.incrGetDelegationTokenFailedRetrieved();
+    }
+
+    public void getRenewDelegationTokenFailed() {
+      LOG.info("Mocked: failed renewDelegationToken call");
+      metrics.incrRenewDelegationTokenFailedRetrieved();
+    }
   }
 
   // Records successes for all calls
@@ -375,10 +562,9 @@ public void getApplicationReport(long duration) {
     }
 
     public void getApplicationAttemptReport(long duration) {
-      LOG.info("Mocked: successful " +
-              "getApplicationAttemptReport call with duration {}",
-          duration);
-      metrics.succeededAppAttemptsRetrieved(duration);
+      LOG.info("Mocked: successful getApplicationAttemptReport call " +
+          "with duration {}", duration);
+      metrics.succeededAppAttemptReportRetrieved(duration);
     }
 
     public void getApplicationsReport(long duration) {
@@ -392,5 +578,1023 @@ public void getClusterMetrics(long duration){
               duration);
       metrics.succeededGetClusterMetricsRetrieved(duration);
     }
+
+    public void getClusterNodes(long duration) {
+      LOG.info("Mocked: successful getClusterNodes call with duration {}", duration);
+      metrics.succeededGetClusterNodesRetrieved(duration);
+    }
+
+    public void getNodeToLabels(long duration) {
+      LOG.info("Mocked: successful getNodeToLabels call with duration {}", duration);
+      metrics.succeededGetNodeToLabelsRetrieved(duration);
+    }
+
+    public void getLabelToNodes(long duration) {
+      LOG.info("Mocked: successful getLabelToNodes call with duration {}", duration);
+      metrics.succeededGetLabelsToNodesRetrieved(duration);
+    }
+
+    public void getClusterNodeLabels(long duration) {
+      LOG.info("Mocked: successful getClusterNodeLabels call with duration {}", duration);
+      metrics.succeededGetClusterNodeLabelsRetrieved(duration);
+    }
+
+    public void getQueueUserAcls(long duration) {
+      LOG.info("Mocked: successful getQueueUserAcls call with duration {}", duration);
+      metrics.succeededGetQueueUserAclsRetrieved(duration);
+    }
+
+    public void getListReservations(long duration) {
+      LOG.info("Mocked: successful listReservations call with duration {}", duration);
+      metrics.succeededListReservationsRetrieved(duration);
+    }
+
+    public void getApplicationAttempts(long duration) {
+      LOG.info("Mocked: successful getApplicationAttempts call with duration {}", duration);
+      metrics.succeededAppAttemptsRetrieved(duration);
+    }
+
+    public void getContainerReport(long duration) {
+      LOG.info("Mocked: successful getContainerReport call with duration {}", duration);
+      metrics.succeededGetContainerReportRetrieved(duration);
+    }
+
+    public void getContainers(long duration) {
+      LOG.info("Mocked: successful getContainer call with duration {}", duration);
+      metrics.succeededGetContainersRetrieved(duration);
+    }
+
+    public void getResourceTypeInfo(long duration) {
+      LOG.info("Mocked: successful getResourceTypeInfo call with duration {}", duration);
+      metrics.succeededGetResourceTypeInfoRetrieved(duration);
+    }
+
+    public void getFailApplicationAttempt(long duration) {
+      LOG.info("Mocked: successful failApplicationAttempt call with duration {}", duration);
+      metrics.succeededFailAppAttemptRetrieved(duration);
+    }
+
+    public void getUpdateApplicationPriority(long duration) {
+      LOG.info("Mocked: successful updateApplicationPriority call with duration {}", duration);
+      metrics.succeededUpdateAppPriorityRetrieved(duration);
+    }
+
+    public void getUpdateApplicationTimeouts(long duration) {
+      LOG.info("Mocked: successful updateApplicationTimeouts call with duration {}", duration);
+      metrics.succeededUpdateAppTimeoutsRetrieved(duration);
+    }
+
+    public void getSignalToContainerTimeouts(long duration) {
+      LOG.info("Mocked: successful signalToContainer call with duration {}", duration);
+      metrics.succeededSignalToContainerRetrieved(duration);
+    }
+
+    public void getQueueInfoRetrieved(long duration) {
+      LOG.info("Mocked: successful getQueueInfo call with duration {}", duration);
+      metrics.succeededGetQueueInfoRetrieved(duration);
+    }
+
+    public void moveApplicationAcrossQueuesRetrieved(long duration) {
+      LOG.info("Mocked: successful moveApplicationAcrossQueues call with duration {}", duration);
+      metrics.succeededMoveApplicationAcrossQueuesRetrieved(duration);
+    }
+
+    public void getResourceProfilesRetrieved(long duration) {
+      LOG.info("Mocked: successful getResourceProfiles call with duration {}", duration);
+      metrics.succeededGetResourceProfilesRetrieved(duration);
+    }
+
+    public void getResourceProfileRetrieved(long duration) {
+      LOG.info("Mocked: successful getResourceProfile call with duration {}", duration);
+      metrics.succeededGetResourceProfileRetrieved(duration);
+    }
+
+    public void getAttributesToNodesRetrieved(long duration) {
+      LOG.info("Mocked: successful getAttributesToNodes call with duration {}", duration);
+      metrics.succeededGetAttributesToNodesRetrieved(duration);
+    }
+
+    public void getClusterNodeAttributesRetrieved(long duration) {
+      LOG.info("Mocked: successful getClusterNodeAttributes call with duration {}", duration);
+      metrics.succeededGetClusterNodeAttributesRetrieved(duration);
+    }
+
+    public void getNodesToAttributesRetrieved(long duration) {
+      LOG.info("Mocked: successful getNodesToAttributes call with duration {}", duration);
+      metrics.succeededGetNodesToAttributesRetrieved(duration);
+    }
+
+    public void getNewReservationRetrieved(long duration) {
+      LOG.info("Mocked: successful getNewReservation call with duration {}", duration);
+      metrics.succeededGetNewReservationRetrieved(duration);
+    }
+
+    public void getSubmitReservationRetrieved(long duration) {
+      LOG.info("Mocked: successful getSubmitReservation call with duration {}", duration);
+      metrics.succeededSubmitReservationRetrieved(duration);
+    }
+
+    public void getUpdateReservationRetrieved(long duration) {
+      LOG.info("Mocked: successful getUpdateReservation call with duration {}", duration);
+      metrics.succeededUpdateReservationRetrieved(duration);
+    }
+
+    public void getDeleteReservationRetrieved(long duration) {
+      LOG.info("Mocked: successful getDeleteReservation call with duration {}", duration);
+      metrics.succeededDeleteReservationRetrieved(duration);
+    }
+
+    public void getListReservationRetrieved(long duration) {
+      LOG.info("Mocked: successful getListReservation call with duration {}", duration);
+      metrics.succeededListReservationRetrieved(duration);
+    }
+
+    public void getAppActivitiesRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppActivities call with duration {}", duration);
+      metrics.succeededGetAppActivitiesRetrieved(duration);
+    }
+
+    public void getAppStatisticsRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppStatistics call with duration {}", duration);
+      metrics.succeededGetAppStatisticsRetrieved(duration);
+    }
+
+    public void getAppPriorityRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppPriority call with duration {}", duration);
+      metrics.succeededGetAppPriorityRetrieved(duration);
+    }
+
+    public void getAppQueueRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppQueue call with duration {}", duration);
+      metrics.succeededGetAppQueueRetrieved(duration);
+    }
+
+    public void getUpdateQueueRetrieved(long duration) {
+      LOG.info("Mocked: successful getUpdateQueue call with duration {}", duration);
+      metrics.succeededUpdateAppQueueRetrieved(duration);
+    }
+
+    public void getAppTimeoutRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppTimeout call with duration {}", duration);
+      metrics.succeededGetAppTimeoutRetrieved(duration);
+    }
+
+    public void getAppTimeoutsRetrieved(long duration) {
+      LOG.info("Mocked: successful getAppTimeouts call with duration {}", duration);
+      metrics.succeededGetAppTimeoutsRetrieved(duration);
+    }
+
+    public void getRMNodeLabelsRetrieved(long duration) {
+      LOG.info("Mocked: successful getRMNodeLabels call with duration {}", duration);
+      metrics.succeededGetRMNodeLabelsRetrieved(duration);
+    }
+
+    public void getCheckUserAccessToQueueRetrieved(long duration) {
+      LOG.info("Mocked: successful CheckUserAccessToQueue call with duration {}", duration);
+      metrics.succeededCheckUserAccessToQueueRetrieved(duration);
+    }
+
+    public void getGetDelegationTokenRetrieved(long duration) {
+      LOG.info("Mocked: successful GetDelegationToken call with duration {}", duration);
+      metrics.succeededGetDelegationTokenRetrieved(duration);
+    }
+
+    public void getRenewDelegationTokenRetrieved(long duration) {
+      LOG.info("Mocked: successful RenewDelegationToken call with duration {}", duration);
+      metrics.succeededRenewDelegationTokenRetrieved(duration);
+    }
+  }
+
+  @Test
+  public void testSucceededGetClusterNodes() {
+    long totalGoodBefore = metrics.getNumSucceededGetClusterNodesRetrieved();
+    goodSubCluster.getClusterNodes(150);
+    Assert.assertEquals(totalGoodBefore + 1, metrics.getNumSucceededGetClusterNodesRetrieved());
+    Assert.assertEquals(150, metrics.getLatencySucceededGetClusterNodesRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getClusterNodes(300);
+    Assert.assertEquals(totalGoodBefore + 2, metrics.getNumSucceededGetClusterNodesRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetClusterNodesRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetClusterNodesFailed() {
+    long totalBadBefore = metrics.getClusterNodesFailedRetrieved();
+    badSubCluster.getClusterNodes();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getClusterNodesFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetNodeToLabels() {
+    long totalGoodBefore = metrics.getNumSucceededGetNodeToLabelsRetrieved();
+    goodSubCluster.getNodeToLabels(150);
+    Assert.assertEquals(totalGoodBefore + 1, metrics.getNumSucceededGetNodeToLabelsRetrieved());
+    Assert.assertEquals(150, metrics.getLatencySucceededGetNodeToLabelsRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getNodeToLabels(300);
+    Assert.assertEquals(totalGoodBefore + 2, metrics.getNumSucceededGetNodeToLabelsRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetNodeToLabelsRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetNodeToLabelsFailed() {
+    long totalBadBefore = metrics.getNodeToLabelsFailedRetrieved();
+    badSubCluster.getNodeToLabels();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getNodeToLabelsFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededLabelsToNodes() {
+    long totalGoodBefore = metrics.getNumSucceededGetLabelsToNodesRetrieved();
+    goodSubCluster.getLabelToNodes(150);
+    Assert.assertEquals(totalGoodBefore + 1, metrics.getNumSucceededGetLabelsToNodesRetrieved());
+    Assert.assertEquals(150, metrics.getLatencySucceededGetLabelsToNodesRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getLabelToNodes(300);
+    Assert.assertEquals(totalGoodBefore + 2, metrics.getNumSucceededGetLabelsToNodesRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetLabelsToNodesRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetLabelsToNodesFailed() {
+    long totalBadBefore = metrics.getLabelsToNodesFailedRetrieved();
+    badSubCluster.getLabelToNodes();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getLabelsToNodesFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededClusterNodeLabels() {
+    long totalGoodBefore = metrics.getNumSucceededGetClusterNodeLabelsRetrieved();
+    goodSubCluster.getClusterNodeLabels(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetClusterNodeLabelsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetClusterNodeLabelsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getClusterNodeLabels(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetClusterNodeLabelsRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetClusterNodeLabelsRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testClusterNodeLabelsFailed() {
+    long totalBadBefore = metrics.getGetClusterNodeLabelsFailedRetrieved();
+    badSubCluster.getClusterNodeLabels();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getGetClusterNodeLabelsFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededQueueUserAcls() {
+    long totalGoodBefore = metrics.getNumSucceededGetQueueUserAclsRetrieved();
+    goodSubCluster.getQueueUserAcls(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetQueueUserAclsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetQueueUserAclsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getQueueUserAcls(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetQueueUserAclsRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetQueueUserAclsRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testQueueUserAclsFailed() {
+    long totalBadBefore = metrics.getQueueUserAclsFailedRetrieved();
+    badSubCluster.getQueueUserAcls();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getQueueUserAclsFailedRetrieved());
+  }
+  @Test
+  public void testSucceededListReservations() {
+    long totalGoodBefore = metrics.getNumSucceededListReservationsRetrieved();
+    goodSubCluster.getListReservations(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededListReservationsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededListReservationsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getListReservations(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededListReservationsRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededListReservationsRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testListReservationsFailed() {
+    long totalBadBefore = metrics.getListReservationsFailedRetrieved();
+    badSubCluster.getListReservations();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getListReservationsFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetApplicationAttempts() {
+    long totalGoodBefore = metrics.getNumSucceededAppAttemptsRetrieved();
+    goodSubCluster.getApplicationAttempts(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededAppAttemptsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededAppAttemptRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getApplicationAttempts(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededAppAttemptsRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededAppAttemptRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetApplicationAttemptsFailed() {
+    long totalBadBefore = metrics.getAppAttemptsFailedRetrieved();
+    badSubCluster.getApplicationAttempts();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getAppAttemptsFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetContainerReport() {
+    long totalGoodBefore = metrics.getNumSucceededGetContainerReportRetrieved();
+    goodSubCluster.getContainerReport(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetContainerReportRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetContainerReportRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getContainerReport(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetContainerReportRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetContainerReportRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetContainerReportFailed() {
+    long totalBadBefore = metrics.getContainerReportFailedRetrieved();
+    badSubCluster.getContainerReport();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getContainerReportFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetContainers() {
+    long totalGoodBefore = metrics.getNumSucceededGetContainersRetrieved();
+    goodSubCluster.getContainers(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetContainersRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetContainersRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getContainers(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetContainersRetrieved());
+    Assert.assertEquals(225, metrics.getLatencySucceededGetContainersRetrieved(),
+        ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetContainersFailed() {
+    long totalBadBefore = metrics.getContainersFailedRetrieved();
+    badSubCluster.getContainers();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getContainersFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetResourceTypeInfo() {
+    long totalGoodBefore = metrics.getNumSucceededGetResourceTypeInfoRetrieved();
+    goodSubCluster.getResourceTypeInfo(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetResourceTypeInfoRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetResourceTypeInfoRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getResourceTypeInfo(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetResourceTypeInfoRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetResourceTypeInfoRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetResourceTypeInfoFailed() {
+    long totalBadBefore = metrics.getGetResourceTypeInfoRetrieved();
+    badSubCluster.getResourceTypeInfo();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getGetResourceTypeInfoRetrieved());
+  }
+
+  @Test
+  public void testSucceededFailApplicationAttempt() {
+    long totalGoodBefore = metrics.getNumSucceededFailAppAttemptRetrieved();
+    goodSubCluster.getFailApplicationAttempt(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededFailAppAttemptRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededFailAppAttemptRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getFailApplicationAttempt(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededFailAppAttemptRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededFailAppAttemptRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testFailApplicationAttemptFailed() {
+    long totalBadBefore = metrics.getFailApplicationAttemptFailedRetrieved();
+    badSubCluster.getFailApplicationAttempt();
+    Assert.assertEquals(totalBadBefore + 1, metrics.getFailApplicationAttemptFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededUpdateApplicationPriority() {
+    long totalGoodBefore = metrics.getNumSucceededUpdateAppPriorityRetrieved();
+    goodSubCluster.getUpdateApplicationPriority(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededUpdateAppPriorityRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededUpdateAppPriorityRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getUpdateApplicationPriority(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededUpdateAppPriorityRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededUpdateAppPriorityRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testUpdateApplicationPriorityFailed() {
+    long totalBadBefore = metrics.getUpdateApplicationPriorityFailedRetrieved();
+    badSubCluster.getUpdateApplicationPriority();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getUpdateApplicationPriorityFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededUpdateAppTimeoutsRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededUpdateAppTimeoutsRetrieved();
+    goodSubCluster.getUpdateApplicationTimeouts(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededUpdateAppTimeoutsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededUpdateAppTimeoutsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getUpdateApplicationTimeouts(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededUpdateAppTimeoutsRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededUpdateAppTimeoutsRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testUpdateAppTimeoutsFailed() {
+    long totalBadBefore = metrics.getUpdateApplicationTimeoutsFailedRetrieved();
+    badSubCluster.getUpdateApplicationTimeouts();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getUpdateApplicationTimeoutsFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededSignalToContainerRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededSignalToContainerRetrieved();
+    goodSubCluster.getSignalToContainerTimeouts(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededSignalToContainerRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededSignalToContainerRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getSignalToContainerTimeouts(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededSignalToContainerRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededSignalToContainerRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testSignalToContainerFailed() {
+    long totalBadBefore = metrics.getSignalToContainerFailedRetrieved();
+    badSubCluster.getSignalContainer();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getSignalToContainerFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetQueueInfoRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetQueueInfoRetrieved();
+    goodSubCluster.getQueueInfoRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetQueueInfoRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetQueueInfoRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getQueueInfoRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetQueueInfoRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetQueueInfoRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetQueueInfoFailed() {
+    long totalBadBefore = metrics.getQueueInfoFailedRetrieved();
+    badSubCluster.getQueueInfo();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getQueueInfoFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededMoveApplicationAcrossQueuesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved();
+    goodSubCluster.moveApplicationAcrossQueuesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededMoveApplicationAcrossQueuesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.moveApplicationAcrossQueuesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededMoveApplicationAcrossQueuesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testMoveApplicationAcrossQueuesRetrievedFailed() {
+    long totalBadBefore = metrics.getMoveApplicationAcrossQueuesFailedRetrieved();
+    badSubCluster.moveApplicationAcrossQueuesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getMoveApplicationAcrossQueuesFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetResourceProfilesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetResourceProfilesRetrieved();
+    goodSubCluster.getResourceProfilesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetResourceProfilesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetResourceProfilesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getResourceProfilesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetResourceProfilesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetResourceProfilesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetResourceProfilesRetrievedFailed() {
+    long totalBadBefore = metrics.getResourceProfilesFailedRetrieved();
+    badSubCluster.getResourceProfilesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getResourceProfilesFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetResourceProfileRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetResourceProfileRetrieved();
+    goodSubCluster.getResourceProfileRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetResourceProfileRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetResourceProfileRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getResourceProfileRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetResourceProfileRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetResourceProfileRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetResourceProfileRetrievedFailed() {
+    long totalBadBefore = metrics.getResourceProfileFailedRetrieved();
+    badSubCluster.getResourceProfileFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getResourceProfileFailedRetrieved());
+  }
+
+  @Test
+  public void testSucceededGetAttributesToNodesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAttributesToNodesRetrieved();
+    goodSubCluster.getAttributesToNodesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAttributesToNodesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAttributesToNodesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAttributesToNodesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAttributesToNodesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAttributesToNodesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAttributesToNodesRetrievedFailed() {
+    long totalBadBefore = metrics.getAttributesToNodesFailedRetrieved();
+    badSubCluster.getAttributesToNodesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAttributesToNodesFailedRetrieved());
+  }
+
+  @Test
+  public void testGetClusterNodeAttributesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetClusterNodeAttributesRetrieved();
+    goodSubCluster.getClusterNodeAttributesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetClusterNodeAttributesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetClusterNodeAttributesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getClusterNodeAttributesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetClusterNodeAttributesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetClusterNodeAttributesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetClusterNodeAttributesRetrievedFailed() {
+    long totalBadBefore = metrics.getClusterNodeAttributesFailedRetrieved();
+    badSubCluster.getClusterNodeAttributesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getClusterNodeAttributesFailedRetrieved());
+  }
+
+  @Test
+  public void testGetNodesToAttributesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetNodesToAttributesRetrieved();
+    goodSubCluster.getNodesToAttributesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetNodesToAttributesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetNodesToAttributesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getNodesToAttributesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetNodesToAttributesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetNodesToAttributesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetNodesToAttributesRetrievedFailed() {
+    long totalBadBefore = metrics.getNodesToAttributesFailedRetrieved();
+    badSubCluster.getNodesToAttributesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getNodesToAttributesFailedRetrieved());
+  }
+
+  @Test
+  public void testGetNewReservationRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetNewReservationRetrieved();
+    goodSubCluster.getNewReservationRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetNewReservationRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetNewReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getNewReservationRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetNewReservationRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetNewReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetNewReservationRetrievedFailed() {
+    long totalBadBefore = metrics.getNewReservationFailedRetrieved();
+    badSubCluster.getNewReservationFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getNewReservationFailedRetrieved());
+  }
+
+  @Test
+  public void testGetSubmitReservationRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededSubmitReservationRetrieved();
+    goodSubCluster.getSubmitReservationRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededSubmitReservationRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededSubmitReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getSubmitReservationRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededSubmitReservationRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededSubmitReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetSubmitReservationRetrievedFailed() {
+    long totalBadBefore = metrics.getSubmitReservationFailedRetrieved();
+    badSubCluster.getSubmitReservationFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getSubmitReservationFailedRetrieved());
+  }
+
+  @Test
+  public void testGetUpdateReservationRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededUpdateReservationRetrieved();
+    goodSubCluster.getUpdateReservationRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededUpdateReservationRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededUpdateReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getUpdateReservationRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededUpdateReservationRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededUpdateReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetUpdateReservationRetrievedFailed() {
+    long totalBadBefore = metrics.getUpdateReservationFailedRetrieved();
+    badSubCluster.getUpdateReservationFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getUpdateReservationFailedRetrieved());
+  }
+
+  @Test
+  public void testGetDeleteReservationRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededDeleteReservationRetrieved();
+    goodSubCluster.getDeleteReservationRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededDeleteReservationRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededDeleteReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getDeleteReservationRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededDeleteReservationRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededDeleteReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetDeleteReservationRetrievedFailed() {
+    long totalBadBefore = metrics.getDeleteReservationFailedRetrieved();
+    badSubCluster.getDeleteReservationFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getDeleteReservationFailedRetrieved());
+  }
+
+  @Test
+  public void testGetListReservationRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededListReservationRetrieved();
+    goodSubCluster.getListReservationRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededListReservationRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededListReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getListReservationRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededListReservationRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededListReservationRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetListReservationRetrievedFailed() {
+    long totalBadBefore = metrics.getListReservationFailedRetrieved();
+    badSubCluster.getListReservationFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getListReservationFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppActivitiesRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppActivitiesRetrieved();
+    goodSubCluster.getAppActivitiesRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppActivitiesRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppActivitiesRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppActivitiesRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppActivitiesRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppActivitiesRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppActivitiesRetrievedFailed() {
+    long totalBadBefore = metrics.getAppActivitiesFailedRetrieved();
+    badSubCluster.getAppActivitiesFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppActivitiesFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppStatisticsLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppStatisticsRetrieved();
+    goodSubCluster.getAppStatisticsRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppStatisticsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppStatisticsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppStatisticsRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppStatisticsRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppStatisticsRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppStatisticsRetrievedFailed() {
+    long totalBadBefore = metrics.getAppStatisticsFailedRetrieved();
+    badSubCluster.getAppStatisticsFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppStatisticsFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppPriorityLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppPriorityRetrieved();
+    goodSubCluster.getAppPriorityRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppPriorityRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppPriorityRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppPriorityRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppPriorityRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppPriorityRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppPriorityRetrievedFailed() {
+    long totalBadBefore = metrics.getAppPriorityFailedRetrieved();
+    badSubCluster.getAppPriorityFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppPriorityFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppQueueLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppQueueRetrieved();
+    goodSubCluster.getAppQueueRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppQueueRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppQueueRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppQueueRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppQueueRetrievedFailed() {
+    long totalBadBefore = metrics.getAppQueueFailedRetrieved();
+    badSubCluster.getAppQueueFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppQueueFailedRetrieved());
+  }
+
+  @Test
+  public void testUpdateAppQueueLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededUpdateAppQueueRetrieved();
+    goodSubCluster.getUpdateQueueRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededUpdateAppQueueRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededUpdateAppQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getUpdateQueueRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededUpdateAppQueueRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededUpdateAppQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testUpdateAppQueueRetrievedFailed() {
+    long totalBadBefore = metrics.getUpdateAppQueueFailedRetrieved();
+    badSubCluster.getUpdateQueueFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getUpdateAppQueueFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppTimeoutLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppTimeoutRetrieved();
+    goodSubCluster.getAppTimeoutRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppTimeoutRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppTimeoutRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppTimeoutRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppTimeoutRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppTimeoutRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppTimeoutRetrievedFailed() {
+    long totalBadBefore = metrics.getAppTimeoutFailedRetrieved();
+    badSubCluster.getAppTimeoutFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppTimeoutFailedRetrieved());
+  }
+
+  @Test
+  public void testGetAppTimeoutsLatencyRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetAppTimeoutsRetrieved();
+    goodSubCluster.getAppTimeoutsRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetAppTimeoutsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetAppTimeoutsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getAppTimeoutsRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetAppTimeoutsRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetAppTimeoutsRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetAppTimeoutsRetrievedFailed() {
+    long totalBadBefore = metrics.getAppTimeoutsFailedRetrieved();
+    badSubCluster.getAppTimeoutsFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getAppTimeoutsFailedRetrieved());
+  }
+
+  @Test
+  public void testGetRMNodeLabelsRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetRMNodeLabelsRetrieved();
+    goodSubCluster.getRMNodeLabelsRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetRMNodeLabelsRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetRMNodeLabelsRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getRMNodeLabelsRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetRMNodeLabelsRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetRMNodeLabelsRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetRMNodeLabelsRetrievedFailed() {
+    long totalBadBefore = metrics.getRMNodeLabelsFailedRetrieved();
+    badSubCluster.getRMNodeLabelsFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getRMNodeLabelsFailedRetrieved());
+  }
+
+  @Test
+  public void testCheckUserAccessToQueueRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededCheckUserAccessToQueueRetrieved();
+    goodSubCluster.getCheckUserAccessToQueueRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededCheckUserAccessToQueueRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededCheckUserAccessToQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getCheckUserAccessToQueueRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededCheckUserAccessToQueueRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededCheckUserAccessToQueueRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testCheckUserAccessToQueueRetrievedFailed() {
+    long totalBadBefore = metrics.getCheckUserAccessToQueueFailedRetrieved();
+    badSubCluster.getCheckUserAccessToQueueFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getCheckUserAccessToQueueFailedRetrieved());
+  }
+
+  @Test
+  public void testGetDelegationTokenRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededGetDelegationTokenRetrieved();
+    goodSubCluster.getGetDelegationTokenRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededGetDelegationTokenRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededGetDelegationTokenRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getGetDelegationTokenRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededGetDelegationTokenRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededGetDelegationTokenRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testGetDelegationTokenRetrievedFailed() {
+    long totalBadBefore = metrics.getDelegationTokenFailedRetrieved();
+    badSubCluster.getDelegationTokenFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getDelegationTokenFailedRetrieved());
+  }
+
+  @Test
+  public void testRenewDelegationTokenRetrieved() {
+    long totalGoodBefore = metrics.getNumSucceededRenewDelegationTokenRetrieved();
+    goodSubCluster.getRenewDelegationTokenRetrieved(150);
+    Assert.assertEquals(totalGoodBefore + 1,
+        metrics.getNumSucceededRenewDelegationTokenRetrieved());
+    Assert.assertEquals(150,
+        metrics.getLatencySucceededRenewDelegationTokenRetrieved(), ASSERT_DOUBLE_DELTA);
+    goodSubCluster.getRenewDelegationTokenRetrieved(300);
+    Assert.assertEquals(totalGoodBefore + 2,
+        metrics.getNumSucceededRenewDelegationTokenRetrieved());
+    Assert.assertEquals(225,
+        metrics.getLatencySucceededRenewDelegationTokenRetrieved(), ASSERT_DOUBLE_DELTA);
+  }
+
+  @Test
+  public void testRenewDelegationTokenRetrievedFailed() {
+    long totalBadBefore = metrics.getRenewDelegationTokenFailedRetrieved();
+    badSubCluster.getRenewDelegationTokenFailed();
+    Assert.assertEquals(totalBadBefore + 1,
+        metrics.getRenewDelegationTokenFailedRetrieved());
   }
-}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java
index 1d726d3ff1697..905b60f31ee82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java
@@ -134,7 +134,7 @@ protected Configuration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughClientRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
+    // Create a request interceptor pipeline for testing. The last one in the
     // chain will call the mock resource manager. The others in the chain will
     // simply forward it to the next one in the chain
     config.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java
index e0538b8a422db..c85bbd6f2eaf1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java
@@ -31,8 +31,10 @@
 public class MockClientRequestInterceptor
     extends DefaultClientRequestInterceptor {
 
+  MockRM mockRM = null;
+
   public void init(String user) {
-    MockRM mockRM = new MockRM(super.getConf()) {
+    mockRM = new MockRM(super.getConf()) {
       @Override
       protected ClientRMService createClientRMService() {
         return new ClientRMService(getRMContext(), getResourceScheduler(),
@@ -68,4 +70,11 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
     super.setRMClient(mockRM.getClientRMService());
   }
 
+  @Override
+  public void shutdown() {
+    if (mockRM != null) {
+      mockRM.stop();
+    }
+    super.shutdown();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java
index a35feaeaa319a..d4820fc12b560 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java
@@ -93,8 +93,8 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 
 /**
- * Mock intercepter that does not do anything other than forwarding it to the
- * next intercepter in the chain.
+ * Mock interceptor that does not do anything other than forwarding it to the
+ * next interceptor in the chain.
  */
 public class PassThroughClientRequestInterceptor
     extends AbstractClientRequestInterceptor {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
index 4ffd3cb2e1df3..2488fc73b07b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.router.clientrm;
 
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -26,9 +27,16 @@
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Arrays;
+import java.util.Collection;
 
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -44,20 +52,107 @@
 import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
@@ -69,7 +164,7 @@
  * use the {@code RouterClientRMService} pipeline test cases for testing the
  * {@code FederationInterceptor} class. The tests for
  * {@code RouterClientRMService} has been written cleverly so that it can be
- * reused to validate different request intercepter chains.
+ * reused to validate different request interceptor chains.
  */
 public class TestFederationClientInterceptor extends BaseRouterClientRMTest {
   private static final Logger LOG =
@@ -84,21 +179,29 @@ public class TestFederationClientInterceptor extends BaseRouterClientRMTest {
 
   private final static int NUM_SUBCLUSTER = 4;
 
+  private final static int APP_PRIORITY_ZERO = 0;
+
+  private final static long DEFAULT_DURATION = 10 * 60 * 1000;
+
   @Override
-  public void setUp() {
+  public void setUp() throws IOException {
     super.setUpConfig();
     interceptor = new TestableFederationClientInterceptor();
 
     stateStore = new MemoryFederationStateStore();
     stateStore.init(this.getConf());
-    FederationStateStoreFacade.getInstance().reinitialize(stateStore,
-        getConf());
+    FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf());
     stateStoreUtil = new FederationStateStoreTestUtil(stateStore);
 
     interceptor.setConf(this.getConf());
     interceptor.init(user);
+    RouterDelegationTokenSecretManager tokenSecretManager =
+        interceptor.createRouterRMDelegationTokenSecretManager(this.getConf());
 
-    subClusters = new ArrayList<SubClusterId>();
+    tokenSecretManager.startThreads();
+    interceptor.setTokenSecretManager(tokenSecretManager);
+
+    subClusters = new ArrayList<>();
 
     try {
       for (int i = 0; i < NUM_SUBCLUSTER; i++) {
@@ -111,6 +214,7 @@ public void setUp() {
       Assert.fail();
     }
 
+    DefaultMetricsSystem.setMiniClusterMode(true);
   }
 
   @Override
@@ -126,13 +230,13 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughClientRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
-    // chain is the federation intercepter that calls the mock resource manager.
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
     // The others in the chain will simply forward it to the next one in the
     // chain
     conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
         mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass
-            + "," + TestableFederationClientInterceptor.class.getName());
+        + "," + TestableFederationClientInterceptor.class.getName());
 
     conf.set(YarnConfiguration.FEDERATION_POLICY_MANAGER,
         UniformBroadcastPolicyManager.class.getName());
@@ -140,6 +244,12 @@ protected YarnConfiguration createConfiguration() {
     // Disable StateStoreFacade cache
     conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
 
+    conf.setInt("yarn.scheduler.minimum-allocation-mb", 512);
+    conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1);
+    conf.setInt("yarn.scheduler.maximum-allocation-mb", 100 * 1024);
+    conf.setInt("yarn.scheduler.maximum-allocation-vcores", 100);
+
+    conf.setBoolean("hadoop.security.authentication", true);
     return conf;
   }
 
@@ -148,17 +258,16 @@ protected YarnConfiguration createConfiguration() {
    * ApplicationId has to belong to one of the SubCluster in the cluster.
    */
   @Test
-  public void testGetNewApplication()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: Get New Application");
+  public void testGetNewApplication() throws YarnException, IOException {
+    LOG.info("Test FederationClientInterceptor: Get New Application.");
 
     GetNewApplicationRequest request = GetNewApplicationRequest.newInstance();
     GetNewApplicationResponse response = interceptor.getNewApplication(request);
 
     Assert.assertNotNull(response);
     Assert.assertNotNull(response.getApplicationId());
-    Assert.assertTrue(response.getApplicationId()
-        .getClusterTimestamp() == ResourceManager.getClusterTimeStamp());
+    Assert.assertEquals(response.getApplicationId().getClusterTimestamp(),
+        ResourceManager.getClusterTimeStamp());
   }
 
   /**
@@ -167,11 +276,10 @@ public void testGetNewApplication()
    */
   @Test
   public void testSubmitApplication()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: Submit Application");
+      throws YarnException, IOException {
+    LOG.info("Test FederationClientInterceptor: Submit Application.");
 
-    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
-        1);
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
     SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
 
     SubmitApplicationResponse response = interceptor.submitApplication(request);
@@ -185,14 +293,12 @@ public void testSubmitApplication()
   private SubmitApplicationRequest mockSubmitApplicationRequest(
       ApplicationId appId) {
     ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
-    ApplicationSubmissionContext context = ApplicationSubmissionContext
-        .newInstance(appId, MockApps.newAppName(), "default",
-            Priority.newInstance(0), amContainerSpec, false, false, -1,
-            Resources.createResource(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
-            "MockApp");
-    SubmitApplicationRequest request = SubmitApplicationRequest
-        .newInstance(context);
+    ApplicationSubmissionContext context = ApplicationSubmissionContext.newInstance(
+        appId, MockApps.newAppName(), "default",
+        Priority.newInstance(APP_PRIORITY_ZERO), amContainerSpec, false, false, -1,
+        Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+        "MockApp");
+    SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(context);
     return request;
   }
 
@@ -233,37 +339,27 @@ public void testSubmitApplicationMultipleSubmission()
    */
   @Test
   public void testSubmitApplicationEmptyRequest()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info(
-        "Test FederationClientInterceptor: Submit Application - Empty");
-    try {
-      interceptor.submitApplication(null);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(
-          e.getMessage().startsWith("Missing submitApplication request or "
-              + "applicationSubmissionContex information."));
-    }
-    try {
-      interceptor.submitApplication(SubmitApplicationRequest.newInstance(null));
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(
-          e.getMessage().startsWith("Missing submitApplication request or "
-              + "applicationSubmissionContex information."));
-    }
-    try {
-      ApplicationSubmissionContext context = ApplicationSubmissionContext
-          .newInstance(null, "", "", null, null, false, false, -1, null, null);
-      SubmitApplicationRequest request =
-          SubmitApplicationRequest.newInstance(context);
-      interceptor.submitApplication(request);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(
-          e.getMessage().startsWith("Missing submitApplication request or "
-              + "applicationSubmissionContex information."));
-    }
+      throws Exception {
+    LOG.info("Test FederationClientInterceptor: Submit Application - Empty.");
+
+    // null request1
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing submitApplication request or applicationSubmissionContext information.",
+        () -> interceptor.submitApplication(null));
+
+    // null request2
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing submitApplication request or applicationSubmissionContext information.",
+        () -> interceptor.submitApplication(SubmitApplicationRequest.newInstance(null)));
+
+    // null request3
+    ApplicationSubmissionContext context = ApplicationSubmissionContext
+        .newInstance(null, "", "", null, null, false, false, -1, null, null);
+    SubmitApplicationRequest request =
+        SubmitApplicationRequest.newInstance(context);
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing submitApplication request or applicationSubmissionContext information.",
+        () -> interceptor.submitApplication(request));
   }
 
   /**
@@ -273,7 +369,7 @@ public void testSubmitApplicationEmptyRequest()
   @Test
   public void testForceKillApplication()
       throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: Force Kill Application");
+    LOG.info("Test FederationClientInterceptor: Force Kill Application.");
 
     ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
@@ -285,10 +381,8 @@ public void testForceKillApplication()
     Assert.assertNotNull(response);
     Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
 
-    KillApplicationRequest requestKill =
-        KillApplicationRequest.newInstance(appId);
-    KillApplicationResponse responseKill =
-        interceptor.forceKillApplication(requestKill);
+    KillApplicationRequest requestKill = KillApplicationRequest.newInstance(appId);
+    KillApplicationResponse responseKill = interceptor.forceKillApplication(requestKill);
     Assert.assertNotNull(responseKill);
   }
 
@@ -297,22 +391,17 @@ public void testForceKillApplication()
    * application does not exist in StateStore.
    */
   @Test
-  public void testForceKillApplicationNotExists()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: "
-        + "Force Kill Application - Not Exists");
+  public void testForceKillApplicationNotExists() throws Exception {
+    LOG.info("Test FederationClientInterceptor: Force Kill Application - Not Exists");
 
     ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
     KillApplicationRequest requestKill =
         KillApplicationRequest.newInstance(appId);
-    try {
-      interceptor.forceKillApplication(requestKill);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().equals(
-          "Application " + appId + " does not exist in FederationStateStore"));
-    }
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Application " + appId + " does not exist in FederationStateStore.",
+        () -> interceptor.forceKillApplication(requestKill));
   }
 
   /**
@@ -321,24 +410,19 @@ public void testForceKillApplicationNotExists()
    */
   @Test
   public void testForceKillApplicationEmptyRequest()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info(
-        "Test FederationClientInterceptor: Force Kill Application - Empty");
-    try {
-      interceptor.forceKillApplication(null);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().startsWith(
-          "Missing forceKillApplication request or ApplicationId."));
-    }
-    try {
-      interceptor
-          .forceKillApplication(KillApplicationRequest.newInstance(null));
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().startsWith(
-          "Missing forceKillApplication request or ApplicationId."));
-    }
+      throws Exception {
+    LOG.info("Test FederationClientInterceptor: Force Kill Application - Empty.");
+
+    // null request1
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing forceKillApplication request or ApplicationId.",
+        () -> interceptor.forceKillApplication(null));
+
+    // null request2
+    KillApplicationRequest killRequest = KillApplicationRequest.newInstance(null);
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing forceKillApplication request or ApplicationId.",
+        () -> interceptor.forceKillApplication(killRequest));
   }
 
   /**
@@ -375,20 +459,14 @@ public void testGetApplicationReport()
    */
   @Test
   public void testGetApplicationNotExists()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info(
-        "Test ApplicationClientProtocol: Get Application Report - Not Exists");
-    ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
-    GetApplicationReportRequest requestGet =
-        GetApplicationReportRequest.newInstance(appId);
-    try {
-      interceptor.getApplicationReport(requestGet);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(e.getMessage().equals(
-          "Application " + appId + " does not exist in FederationStateStore"));
-    }
+      throws Exception {
+    LOG.info("Test ApplicationClientProtocol: Get Application Report - Not Exists.");
+
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    GetApplicationReportRequest requestGet = GetApplicationReportRequest.newInstance(appId);
+    LambdaTestUtils.intercept(YarnException.class,
+        "Application " + appId + " does not exist in FederationStateStore.",
+        () -> interceptor.getApplicationReport(requestGet));
   }
 
   /**
@@ -397,43 +475,32 @@ public void testGetApplicationNotExists()
    */
   @Test
   public void testGetApplicationEmptyRequest()
-      throws YarnException, IOException, InterruptedException {
-    LOG.info(
-        "Test FederationClientInterceptor: Get Application Report - Empty");
-    try {
-      interceptor.getApplicationReport(null);
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(
-          e.getMessage().startsWith("Missing getApplicationReport request or "
-              + "applicationId information."));
-    }
-    try {
-      interceptor
-          .getApplicationReport(GetApplicationReportRequest.newInstance(null));
-      Assert.fail();
-    } catch (YarnException e) {
-      Assert.assertTrue(
-          e.getMessage().startsWith("Missing getApplicationReport request or "
-              + "applicationId information."));
-    }
+      throws Exception {
+    LOG.info("Test FederationClientInterceptor: Get Application Report - Empty.");
+
+    // null request1
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getApplicationReport request or applicationId information.",
+        () -> interceptor.getApplicationReport(null));
+
+    // null request2
+    GetApplicationReportRequest reportRequest = GetApplicationReportRequest.newInstance(null);
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getApplicationReport request or applicationId information.",
+        () -> interceptor.getApplicationReport(reportRequest));
   }
 
   /**
-   * This test validates the correctness of
-   * GetApplicationAttemptReport in case the
+   * This test validates the correctness of GetApplicationAttemptReport in case the
    * application exists in the cluster.
    */
   @Test
   public void testGetApplicationAttemptReport()
           throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: " +
-            "Get ApplicationAttempt Report");
+    LOG.info("Test FederationClientInterceptor: Get ApplicationAttempt Report.");
 
     ApplicationId appId =
-            ApplicationId.newInstance(System.currentTimeMillis(), 1);
-    ApplicationAttemptId appAttemptId =
-            ApplicationAttemptId.newInstance(appId, 1);
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
     SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
 
@@ -443,111 +510,122 @@ public void testGetApplicationAttemptReport()
     Assert.assertNotNull(response);
     Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
 
+    // Call GetApplicationAttempts Get ApplicationAttemptId
+    GetApplicationAttemptsRequest attemptsRequest =
+         GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+         interceptor.getApplicationAttempts(attemptsRequest);
+
+    // Wait for app to start
+    while(attemptsResponse.getApplicationAttemptList().size() == 0) {
+      attemptsResponse =
+          interceptor.getApplicationAttempts(attemptsRequest);
+    }
+
+    Assert.assertNotNull(attemptsResponse);
+
     GetApplicationAttemptReportRequest requestGet =
-            GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+         GetApplicationAttemptReportRequest.newInstance(
+         attemptsResponse.getApplicationAttemptList().get(0).getApplicationAttemptId());
 
     GetApplicationAttemptReportResponse responseGet =
-            interceptor.getApplicationAttemptReport(requestGet);
+         interceptor.getApplicationAttemptReport(requestGet);
 
     Assert.assertNotNull(responseGet);
   }
 
   /**
-   * This test validates the correctness of
-   * GetApplicationAttemptReport in case the
+   * This test validates the correctness of GetApplicationAttemptReport in case the
    * application does not exist in StateStore.
    */
   @Test
-  public void testGetApplicationAttemptNotExists()
-          throws Exception {
-    LOG.info(
-            "Test ApplicationClientProtocol: " +
-                    "Get ApplicationAttempt Report - Not Exists");
+  public void testGetApplicationAttemptNotExists() throws Exception {
+    LOG.info("Test FederationClientInterceptor: Get ApplicationAttempt Report - Not Exists.");
+
     ApplicationId appId =
-            ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
     ApplicationAttemptId appAttemptID =
-            ApplicationAttemptId.newInstance(appId, 1);
+        ApplicationAttemptId.newInstance(appId, 1);
     GetApplicationAttemptReportRequest requestGet =
-            GetApplicationAttemptReportRequest.newInstance(appAttemptID);
+        GetApplicationAttemptReportRequest.newInstance(appAttemptID);
 
     LambdaTestUtils.intercept(YarnException.class, "ApplicationAttempt " +
-            appAttemptID + "belongs to Application " +
-            appId + " does not exist in FederationStateStore",
+        appAttemptID + " belongs to Application " +
+        appId + " does not exist in FederationStateStore.",
         () -> interceptor.getApplicationAttemptReport(requestGet));
   }
 
   /**
-   * This test validates
-   * the correctness of GetApplicationAttemptReport in case of
+   * This test validates the correctness of GetApplicationAttemptReport in case of
    * empty request.
    */
   @Test
   public void testGetApplicationAttemptEmptyRequest()
-          throws Exception {
-    LOG.info("Test FederationClientInterceptor: " +
-                    "Get ApplicationAttempt Report - Empty");
+      throws Exception {
+    LOG.info("Test FederationClientInterceptor: Get ApplicationAttempt Report - Empty.");
 
+    // null request1
     LambdaTestUtils.intercept(YarnException.class,
-            "Missing getApplicationAttemptReport " +
-                    "request or applicationId " +
-                    "or applicationAttemptId information.",
+        "Missing getApplicationAttemptReport request or applicationId " +
+        "or applicationAttemptId information.",
         () -> interceptor.getApplicationAttemptReport(null));
 
+    // null request2
     LambdaTestUtils.intercept(YarnException.class,
-            "Missing getApplicationAttemptReport " +
-                    "request or applicationId " +
-                    "or applicationAttemptId information.",
-        () -> interceptor
-                    .getApplicationAttemptReport(
-                            GetApplicationAttemptReportRequest
-                                    .newInstance(null)));
+        "Missing getApplicationAttemptReport request or applicationId " +
+        "or applicationAttemptId information.",
+        () -> interceptor.getApplicationAttemptReport(
+        GetApplicationAttemptReportRequest.newInstance(null)));
 
+    // null request3
     LambdaTestUtils.intercept(YarnException.class,
-            "Missing getApplicationAttemptReport " +
-                    "request or applicationId " +
-                    "or applicationAttemptId information.",
-        () -> interceptor
-                    .getApplicationAttemptReport(
-                            GetApplicationAttemptReportRequest.newInstance(
-                                    ApplicationAttemptId
-                                            .newInstance(null, 1)
-                            )));
+        "Missing getApplicationAttemptReport request or applicationId " +
+        "or applicationAttemptId information.",
+        () -> interceptor.getApplicationAttemptReport(
+        GetApplicationAttemptReportRequest.newInstance(
+        ApplicationAttemptId.newInstance(null, 1))));
   }
 
 
   @Test
-  public void testGetClusterMetricsRequest() throws YarnException, IOException {
-    LOG.info("Test FederationClientInterceptor : Get Cluster Metrics request");
+  public void testGetClusterMetricsRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Cluster Metrics request.");
+
     // null request
-    GetClusterMetricsResponse response = interceptor.getClusterMetrics(null);
-    Assert.assertEquals(subClusters.size(),
-        response.getClusterMetrics().getNumNodeManagers());
+    LambdaTestUtils.intercept(YarnException.class, "Missing getClusterMetrics request.",
+        () -> interceptor.getClusterMetrics(null));
+
     // normal request.
-    response =
+    GetClusterMetricsResponse response =
         interceptor.getClusterMetrics(GetClusterMetricsRequest.newInstance());
     Assert.assertEquals(subClusters.size(),
         response.getClusterMetrics().getNumNodeManagers());
 
+    // Clear Membership
+    Map<SubClusterId, SubClusterInfo> membership = new HashMap<>();
+    membership.putAll(stateStore.getMembership());
+    stateStore.getMembership().clear();
+
     ClientMethod remoteMethod = new ClientMethod("getClusterMetrics",
         new Class[] {GetClusterMetricsRequest.class},
         new Object[] {GetClusterMetricsRequest.newInstance()});
-    Map<SubClusterId, GetClusterMetricsResponse> clusterMetrics =interceptor.
-        invokeConcurrent(new ArrayList<>(), remoteMethod,
-            GetClusterMetricsResponse.class);
-    Assert.assertEquals(true, clusterMetrics.isEmpty());
+    Collection<GetClusterMetricsResponse> clusterMetrics = interceptor.invokeConcurrent(
+        remoteMethod, GetClusterMetricsResponse.class);
+    Assert.assertTrue(clusterMetrics.isEmpty());
+
+    // Restore membership
+    stateStore.setMembership(membership);
   }
 
   /**
-   * This test validates the correctness of
-   * GetApplicationsResponse in case the
+   * This test validates the correctness of GetApplicationsResponse in case the
    * application exists in the cluster.
    */
   @Test
   public void testGetApplicationsResponse()
       throws YarnException, IOException, InterruptedException {
-    LOG.info("Test FederationClientInterceptor: Get Applications Response");
-    ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    LOG.info("Test FederationClientInterceptor: Get Applications Response.");
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
     SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
     SubmitApplicationResponse response = interceptor.submitApplication(request);
@@ -556,40 +634,32 @@ public void testGetApplicationsResponse()
     Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
 
     Set<String> appTypes = Collections.singleton("MockApp");
-    GetApplicationsRequest requestGet =
-        GetApplicationsRequest.newInstance(appTypes);
-
-    GetApplicationsResponse responseGet =
-        interceptor.getApplications(requestGet);
+    GetApplicationsRequest requestGet = GetApplicationsRequest.newInstance(appTypes);
+    GetApplicationsResponse responseGet = interceptor.getApplications(requestGet);
 
     Assert.assertNotNull(responseGet);
   }
 
   /**
-   * This test validates
-   * the correctness of GetApplicationsResponse in case of
+   * This test validates the correctness of GetApplicationsResponse in case of
    * empty request.
    */
   @Test
   public void testGetApplicationsNullRequest() throws Exception {
-    LOG.info("Test FederationClientInterceptor: Get Applications request");
-    LambdaTestUtils.intercept(YarnException.class,
-        "Missing getApplications request.",
+    LOG.info("Test FederationClientInterceptor: Get Applications request.");
+    LambdaTestUtils.intercept(YarnException.class, "Missing getApplications request.",
         () -> interceptor.getApplications(null));
   }
 
   /**
-   * This test validates
-   * the correctness of GetApplicationsResponse in case applications
+   * This test validates the correctness of GetApplicationsResponse in case applications
    * with given type does not exist.
    */
   @Test
   public void testGetApplicationsApplicationTypeNotExists() throws Exception{
-    LOG.info("Test FederationClientInterceptor: Application with type does "
-        + "not exist");
+    LOG.info("Test FederationClientInterceptor: Application with type does not exist.");
 
-    ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
     SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
     SubmitApplicationResponse response = interceptor.submitApplication(request);
@@ -599,25 +669,20 @@ public void testGetApplicationsApplicationTypeNotExists() throws Exception{
 
     Set<String> appTypes = Collections.singleton("SPARK");
 
-    GetApplicationsRequest requestGet =
-        GetApplicationsRequest.newInstance(appTypes);
-
-    GetApplicationsResponse responseGet =
-        interceptor.getApplications(requestGet);
+    GetApplicationsRequest requestGet = GetApplicationsRequest.newInstance(appTypes);
+    GetApplicationsResponse responseGet = interceptor.getApplications(requestGet);
 
     Assert.assertNotNull(responseGet);
     Assert.assertTrue(responseGet.getApplicationList().isEmpty());
   }
 
   /**
-   * This test validates
-   * the correctness of GetApplicationsResponse in case applications
+   * This test validates the correctness of GetApplicationsResponse in case applications
    * with given YarnApplicationState does not exist.
    */
   @Test
-  public void testGetApplicationsApplicationStateNotExists() throws Exception{
-    LOG.info("Test FederationClientInterceptor:" +
-        " Application with state does not exist");
+  public void testGetApplicationsApplicationStateNotExists() throws Exception {
+    LOG.info("Test FederationClientInterceptor: Application with state does not exist.");
 
     ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
@@ -635,10 +700,1008 @@ public void testGetApplicationsApplicationStateNotExists() throws Exception{
     GetApplicationsRequest requestGet =
         GetApplicationsRequest.newInstance(applicationStates);
 
-    GetApplicationsResponse responseGet =
-        interceptor.getApplications(requestGet);
+    GetApplicationsResponse responseGet = interceptor.getApplications(requestGet);
 
     Assert.assertNotNull(responseGet);
     Assert.assertTrue(responseGet.getApplicationList().isEmpty());
   }
+
+  @Test
+  public void testGetClusterNodesRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Cluster Nodes request.");
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodes request.",
+        () -> interceptor.getClusterNodes(null));
+    // normal request.
+    GetClusterNodesResponse response =
+        interceptor.getClusterNodes(GetClusterNodesRequest.newInstance());
+    Assert.assertEquals(subClusters.size(), response.getNodeReports().size());
+  }
+
+  @Test
+  public void testGetNodeToLabelsRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor :  Get Node To Labels request.");
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getNodesToLabels request.",
+        () -> interceptor.getNodeToLabels(null));
+    // normal request.
+    GetNodesToLabelsResponse response =
+        interceptor.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
+    Assert.assertEquals(0, response.getNodeToLabels().size());
+  }
+
+  @Test
+  public void testGetLabelsToNodesRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor :  Get Labels To Node request.");
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getLabelsToNodes request.",
+        () -> interceptor.getLabelsToNodes(null));
+    // normal request.
+    GetLabelsToNodesResponse response =
+        interceptor.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
+    Assert.assertEquals(0, response.getLabelsToNodes().size());
+  }
+
+  @Test
+  public void testClusterNodeLabelsRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor :  Get Cluster NodeLabels request.");
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodeLabels request.",
+        () -> interceptor.getClusterNodeLabels(null));
+    // normal request.
+    GetClusterNodeLabelsResponse response =
+        interceptor.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
+    Assert.assertEquals(0, response.getNodeLabelList().size());
+  }
+
+  @Test
+  public void testGetQueueUserAcls() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get QueueUserAcls request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getQueueUserAcls request.",
+        () -> interceptor.getQueueUserAcls(null));
+
+    // normal request
+    GetQueueUserAclsInfoResponse response = interceptor.getQueueUserAcls(
+        GetQueueUserAclsInfoRequest.newInstance());
+
+    Assert.assertNotNull(response);
+
+    List<QueueACL> submitAndAdministerAcl = new ArrayList<>();
+    submitAndAdministerAcl.add(QueueACL.SUBMIT_APPLICATIONS);
+    submitAndAdministerAcl.add(QueueACL.ADMINISTER_QUEUE);
+
+    QueueUserACLInfo exceptRootQueueACLInfo = QueueUserACLInfo.newInstance("root",
+        submitAndAdministerAcl);
+
+    QueueUserACLInfo queueRootQueueACLInfo = response.getUserAclsInfoList().stream().
+        filter(acl->acl.getQueueName().equals("root")).
+        collect(Collectors.toList()).get(0);
+
+    Assert.assertEquals(exceptRootQueueACLInfo, queueRootQueueACLInfo);
+  }
+
+  @Test
+  public void testListReservations() throws Exception {
+    LOG.info("Test FederationClientInterceptor :  Get ListReservations request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing listReservations request.",
+        () -> interceptor.listReservations(null));
+
+    // normal request
+    ReservationId reservationId = ReservationId.newInstance(1653487680L, 1L);
+    ReservationListResponse response = interceptor.listReservations(
+        ReservationListRequest.newInstance("root.decided", reservationId.toString()));
+    Assert.assertNotNull(response);
+    Assert.assertEquals(0, response.getReservationAllocationState().size());
+  }
+
+  @Test
+  public void testGetContainersRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Containers request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getContainers request " +
+        "or ApplicationAttemptId.", () -> interceptor.getContainers(null));
+
+    // normal request
+    ApplicationId appId =
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+        GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+        interceptor.getApplicationAttempts(attemptsRequest);
+
+    // Wait for app to start
+    while(attemptsResponse.getApplicationAttemptList().size() == 0) {
+      attemptsResponse =
+          interceptor.getApplicationAttempts(attemptsRequest);
+    }
+
+    Assert.assertNotNull(attemptsResponse);
+
+    // Call GetContainers
+    GetContainersRequest containersRequest =
+        GetContainersRequest.newInstance(
+        attemptsResponse.getApplicationAttemptList().get(0).getApplicationAttemptId());
+    GetContainersResponse containersResponse =
+        interceptor.getContainers(containersRequest);
+
+    Assert.assertNotNull(containersResponse);
+  }
+
+  @Test
+  public void testGetContainerReportRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Container Report request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getContainerReport request " +
+        "or containerId", () -> interceptor.getContainerReport(null));
+
+    // normal request
+    ApplicationId appId =
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+         GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+         interceptor.getApplicationAttempts(attemptsRequest);
+
+    // Wait for app to start
+    while(attemptsResponse.getApplicationAttemptList().size() == 0) {
+      attemptsResponse =
+          interceptor.getApplicationAttempts(attemptsRequest);
+    }
+    Assert.assertNotNull(attemptsResponse);
+
+    ApplicationAttemptId attemptId = attemptsResponse.getApplicationAttemptList().
+        get(0).getApplicationAttemptId();
+    ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
+
+    // Call ContainerReport, RM does not allocate Container, Here is null
+    GetContainerReportRequest containerReportRequest =
+         GetContainerReportRequest.newInstance(containerId);
+    GetContainerReportResponse containerReportResponse =
+         interceptor.getContainerReport(containerReportRequest);
+
+    Assert.assertEquals(containerReportResponse, null);
+  }
+
+  @Test
+  public void getApplicationAttempts() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Application Attempts request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getApplicationAttempts " +
+            "request or application id.", () -> interceptor.getApplicationAttempts(null));
+
+    // normal request
+    ApplicationId appId =
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+         GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+         interceptor.getApplicationAttempts(attemptsRequest);
+
+    Assert.assertNotNull(attemptsResponse);
+  }
+
+  @Test
+  public void testGetResourceTypeInfoRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor :  Get Resource TypeInfo request.");
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getResourceTypeInfo request.",
+        () -> interceptor.getResourceTypeInfo(null));
+    // normal request.
+    GetAllResourceTypeInfoResponse response =
+        interceptor.getResourceTypeInfo(GetAllResourceTypeInfoRequest.newInstance());
+    Assert.assertEquals(2, response.getResourceTypeInfo().size());
+  }
+
+  @Test
+  public void testFailApplicationAttempt() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Fail Application Attempt request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing failApplicationAttempt request " +
+        "or applicationId or applicationAttemptId information.",
+        () -> interceptor.failApplicationAttempt(null));
+
+    // normal request
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId);
+    Assert.assertNotNull(subClusterId);
+
+    MockRM mockRM = interceptor.getMockRMs().get(subClusterId);
+    mockRM.waitForState(appId, RMAppState.ACCEPTED);
+    RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId);
+    mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
+        RMAppAttemptState.SCHEDULED);
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+        GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+        interceptor.getApplicationAttempts(attemptsRequest);
+    Assert.assertNotNull(attemptsResponse);
+
+    ApplicationAttemptId attemptId = attemptsResponse.getApplicationAttemptList().
+        get(0).getApplicationAttemptId();
+
+    FailApplicationAttemptRequest requestFailAppAttempt =
+        FailApplicationAttemptRequest.newInstance(attemptId);
+    FailApplicationAttemptResponse responseFailAppAttempt =
+        interceptor.failApplicationAttempt(requestFailAppAttempt);
+
+    Assert.assertNotNull(responseFailAppAttempt);
+  }
+
+  @Test
+  public void testUpdateApplicationPriority() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Update Application Priority request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing updateApplicationPriority request " +
+        "or applicationId or applicationPriority information.",
+        () -> interceptor.updateApplicationPriority(null));
+
+    // normal request
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId);
+    Assert.assertNotNull(subClusterId);
+
+    MockRM mockRM = interceptor.getMockRMs().get(subClusterId);
+    mockRM.waitForState(appId, RMAppState.ACCEPTED);
+    RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId);
+    mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
+        RMAppAttemptState.SCHEDULED);
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+        GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+        interceptor.getApplicationAttempts(attemptsRequest);
+    Assert.assertNotNull(attemptsResponse);
+
+    Priority priority = Priority.newInstance(20);
+    UpdateApplicationPriorityRequest requestUpdateAppPriority =
+        UpdateApplicationPriorityRequest.newInstance(appId, priority);
+    UpdateApplicationPriorityResponse responseAppPriority =
+        interceptor.updateApplicationPriority(requestUpdateAppPriority);
+
+    Assert.assertNotNull(responseAppPriority);
+    Assert.assertEquals(20,
+        responseAppPriority.getApplicationPriority().getPriority());
+  }
+
+  @Test
+  public void testUpdateApplicationTimeouts() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Update Application Timeouts request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing updateApplicationTimeouts request " +
+        "or applicationId or applicationTimeouts information.",
+        () -> interceptor.updateApplicationTimeouts(null));
+
+    // normal request
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId);
+    Assert.assertNotNull(subClusterId);
+
+    MockRM mockRM = interceptor.getMockRMs().get(subClusterId);
+    mockRM.waitForState(appId, RMAppState.ACCEPTED);
+    RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId);
+    mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
+        RMAppAttemptState.SCHEDULED);
+
+    // Call GetApplicationAttempts
+    GetApplicationAttemptsRequest attemptsRequest =
+        GetApplicationAttemptsRequest.newInstance(appId);
+    GetApplicationAttemptsResponse attemptsResponse =
+        interceptor.getApplicationAttempts(attemptsRequest);
+    Assert.assertNotNull(attemptsResponse);
+
+    String appTimeout =
+        Times.formatISO8601(System.currentTimeMillis() + 5 * 1000);
+    Map<ApplicationTimeoutType, String> applicationTimeouts = new HashMap<>();
+    applicationTimeouts.put(ApplicationTimeoutType.LIFETIME, appTimeout);
+
+    UpdateApplicationTimeoutsRequest timeoutsRequest =
+        UpdateApplicationTimeoutsRequest.newInstance(appId, applicationTimeouts);
+    UpdateApplicationTimeoutsResponse timeoutsResponse =
+        interceptor.updateApplicationTimeouts(timeoutsRequest);
+
+    String responseTimeOut =
+        timeoutsResponse.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME);
+    Assert.assertNotNull(timeoutsResponse);
+    Assert.assertEquals(appTimeout, responseTimeOut);
+  }
+
+  @Test
+  public void testSignalContainer() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Signal Container request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing signalToContainer request " +
+        "or containerId or command information.", () -> interceptor.signalToContainer(null));
+
+    // normal request
+    ApplicationId appId =
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId);
+    Assert.assertNotNull(subClusterId);
+
+    MockRM mockRM = interceptor.getMockRMs().get(subClusterId);
+    mockRM.waitForState(appId, RMAppState.ACCEPTED);
+    RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId);
+    mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
+        RMAppAttemptState.SCHEDULED);
+    MockNM nm = interceptor.getMockNMs().get(subClusterId);
+    nm.nodeHeartbeat(true);
+    MockRM.waitForState(rmApp.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED);
+    mockRM.sendAMLaunched(rmApp.getCurrentAppAttempt().getAppAttemptId());
+
+    ContainerId containerId = rmApp.getCurrentAppAttempt().getMasterContainer().getId();
+
+    SignalContainerRequest signalContainerRequest =
+        SignalContainerRequest.newInstance(containerId, SignalContainerCommand.GRACEFUL_SHUTDOWN);
+    SignalContainerResponse signalContainerResponse =
+        interceptor.signalToContainer(signalContainerRequest);
+
+    Assert.assertNotNull(signalContainerResponse);
+  }
+
+  @Test
+  public void testMoveApplicationAcrossQueues() throws Exception {
+    LOG.info("Test FederationClientInterceptor : MoveApplication AcrossQueues request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing moveApplicationAcrossQueues request " +
+        "or applicationId or target queue.", () -> interceptor.moveApplicationAcrossQueues(null));
+
+    // normal request
+    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+    SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+
+    // Submit the application
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId);
+    Assert.assertNotNull(subClusterId);
+
+    MockRM mockRM = interceptor.getMockRMs().get(subClusterId);
+    mockRM.waitForState(appId, RMAppState.ACCEPTED);
+    RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId);
+    mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(),
+          RMAppAttemptState.SCHEDULED);
+    MockNM nm = interceptor.getMockNMs().get(subClusterId);
+    nm.nodeHeartbeat(true);
+    MockRM.waitForState(rmApp.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED);
+    mockRM.sendAMLaunched(rmApp.getCurrentAppAttempt().getAppAttemptId());
+
+    MoveApplicationAcrossQueuesRequest acrossQueuesRequest =
+        MoveApplicationAcrossQueuesRequest.newInstance(appId, "root.target");
+    MoveApplicationAcrossQueuesResponse acrossQueuesResponse =
+        interceptor.moveApplicationAcrossQueues(acrossQueuesRequest);
+
+    Assert.assertNotNull(acrossQueuesResponse);
+  }
+
+
+  @Test
+  public void testGetQueueInfo() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Queue Info request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getQueueInfo request or queueName.",
+        () -> interceptor.getQueueInfo(null));
+
+    // normal request
+    GetQueueInfoResponse response = interceptor.getQueueInfo(
+        GetQueueInfoRequest.newInstance("root", true, true, true));
+
+    Assert.assertNotNull(response);
+
+    QueueInfo queueInfo = response.getQueueInfo();
+    Assert.assertNotNull(queueInfo);
+    Assert.assertEquals(queueInfo.getQueueName(),  "root");
+    Assert.assertEquals(queueInfo.getCapacity(), 4.0, 0);
+    Assert.assertEquals(queueInfo.getCurrentCapacity(), 0.0, 0);
+    Assert.assertEquals(queueInfo.getChildQueues().size(), 12, 0);
+    Assert.assertEquals(queueInfo.getAccessibleNodeLabels().size(), 1);
+  }
+
+  @Test
+  public void testGetResourceProfiles() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Resource Profiles request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getResourceProfiles request.",
+        () -> interceptor.getResourceProfiles(null));
+
+    // normal request
+    GetAllResourceProfilesRequest request = GetAllResourceProfilesRequest.newInstance();
+    GetAllResourceProfilesResponse response = interceptor.getResourceProfiles(request);
+
+    Assert.assertNotNull(response);
+    Map<String, Resource> resProfiles = response.getResourceProfiles();
+
+    Resource maxResProfiles = resProfiles.get("maximum");
+    Assert.assertEquals(32768, maxResProfiles.getMemorySize());
+    Assert.assertEquals(16, maxResProfiles.getVirtualCores());
+
+    Resource defaultResProfiles = resProfiles.get("default");
+    Assert.assertEquals(8192, defaultResProfiles.getMemorySize());
+    Assert.assertEquals(8, defaultResProfiles.getVirtualCores());
+
+    Resource minimumResProfiles = resProfiles.get("minimum");
+    Assert.assertEquals(4096, minimumResProfiles.getMemorySize());
+    Assert.assertEquals(4, minimumResProfiles.getVirtualCores());
+  }
+
+  @Test
+  public void testGetResourceProfile() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get Resource Profile request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getResourceProfile request or profileName.",
+        () -> interceptor.getResourceProfile(null));
+
+    // normal request
+    GetResourceProfileRequest request = GetResourceProfileRequest.newInstance("maximum");
+    GetResourceProfileResponse response = interceptor.getResourceProfile(request);
+
+    Assert.assertNotNull(response);
+    Assert.assertEquals(32768, response.getResource().getMemorySize());
+    Assert.assertEquals(16, response.getResource().getVirtualCores());
+
+    GetResourceProfileRequest request2 = GetResourceProfileRequest.newInstance("default");
+    GetResourceProfileResponse response2 = interceptor.getResourceProfile(request2);
+
+    Assert.assertNotNull(response2);
+    Assert.assertEquals(8192, response2.getResource().getMemorySize());
+    Assert.assertEquals(8, response2.getResource().getVirtualCores());
+
+    GetResourceProfileRequest request3 = GetResourceProfileRequest.newInstance("minimum");
+    GetResourceProfileResponse response3 = interceptor.getResourceProfile(request3);
+
+    Assert.assertNotNull(response3);
+    Assert.assertEquals(4096, response3.getResource().getMemorySize());
+    Assert.assertEquals(4, response3.getResource().getVirtualCores());
+  }
+
+  @Test
+  public void testGetAttributesToNodes() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get AttributesToNodes request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getAttributesToNodes request " +
+        "or nodeAttributes.", () -> interceptor.getAttributesToNodes(null));
+
+    // normal request
+    GetAttributesToNodesResponse response =
+        interceptor.getAttributesToNodes(GetAttributesToNodesRequest.newInstance());
+
+    Assert.assertNotNull(response);
+    Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs = response.getAttributesToNodes();
+    Assert.assertNotNull(attrs);
+    Assert.assertEquals(4, attrs.size());
+
+    NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+        NodeAttributeType.STRING, "nvidia");
+    NodeToAttributeValue attributeValue1 =
+        NodeToAttributeValue.newInstance("0-host1", gpu.getAttributeValue());
+    NodeAttributeKey gpuKey = gpu.getAttributeKey();
+    Assert.assertTrue(attrs.get(gpuKey).contains(attributeValue1));
+  }
+
+  @Test
+  public void testClusterNodeAttributes() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get ClusterNodeAttributes request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodeAttributes request.",
+        () -> interceptor.getClusterNodeAttributes(null));
+
+    // normal request
+    GetClusterNodeAttributesResponse response =
+        interceptor.getClusterNodeAttributes(GetClusterNodeAttributesRequest.newInstance());
+
+    Assert.assertNotNull(response);
+    Set<NodeAttributeInfo> nodeAttributeInfos = response.getNodeAttributes();
+    Assert.assertNotNull(nodeAttributeInfos);
+    Assert.assertEquals(4, nodeAttributeInfos.size());
+
+    NodeAttributeInfo nodeAttributeInfo1 =
+        NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("GPU"),
+        NodeAttributeType.STRING);
+    Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo1));
+
+    NodeAttributeInfo nodeAttributeInfo2 =
+        NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("OS"),
+        NodeAttributeType.STRING);
+    Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo2));
+  }
+
+  @Test
+  public void testNodesToAttributes() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get NodesToAttributes request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getNodesToAttributes request or hostNames.",
+        () -> interceptor.getNodesToAttributes(null));
+
+    // normal request
+    Set<String> hostNames = Collections.singleton("0-host1");
+    GetNodesToAttributesResponse response =
+        interceptor.getNodesToAttributes(GetNodesToAttributesRequest.newInstance(hostNames));
+    Assert.assertNotNull(response);
+
+    Map<String, Set<NodeAttribute>> nodeAttributeMap = response.getNodeToAttributes();
+    Assert.assertNotNull(nodeAttributeMap);
+    Assert.assertEquals(1, nodeAttributeMap.size());
+
+    NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+        NodeAttributeType.STRING, "nvida");
+    Assert.assertTrue(nodeAttributeMap.get("0-host1").contains(gpu));
+  }
+
+  @Test
+  public void testGetNewReservation() throws Exception {
+    LOG.info("Test FederationClientInterceptor : Get NewReservation request.");
+
+    // null request
+    LambdaTestUtils.intercept(YarnException.class,
+        "Missing getNewReservation request.", () -> interceptor.getNewReservation(null));
+
+    // normal request
+    GetNewReservationRequest request = GetNewReservationRequest.newInstance();
+    GetNewReservationResponse response = interceptor.getNewReservation(request);
+    Assert.assertNotNull(response);
+
+    ReservationId reservationId = response.getReservationId();
+    Assert.assertNotNull(reservationId);
+    Assert.assertTrue(reservationId.toString().contains("reservation"));
+    Assert.assertEquals(reservationId.getClusterTimestamp(), ResourceManager.getClusterTimeStamp());
+  }
+
+  @Test
+  public void testSubmitReservation() throws Exception {
+    LOG.info("Test FederationClientInterceptor : SubmitReservation request.");
+
+    // get new reservationId
+    GetNewReservationRequest request = GetNewReservationRequest.newInstance();
+    GetNewReservationResponse response = interceptor.getNewReservation(request);
+    Assert.assertNotNull(response);
+
+    // Submit Reservation
+    ReservationId reservationId = response.getReservationId();
+    ReservationDefinition rDefinition = createReservationDefinition(1024, 1);
+    ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance(
+        rDefinition, "decided", reservationId);
+
+    ReservationSubmissionResponse submissionResponse =
+        interceptor.submitReservation(rSubmissionRequest);
+    Assert.assertNotNull(submissionResponse);
+
+    SubClusterId subClusterId = stateStoreUtil.queryReservationHomeSC(reservationId);
+    Assert.assertNotNull(subClusterId);
+    Assert.assertTrue(subClusters.contains(subClusterId));
+  }
+
+  @Test
+  public void testSubmitReservationEmptyRequest() throws Exception {
+    LOG.info("Test FederationClientInterceptor : SubmitReservation request empty.");
+
+    String errorMsg =
+        "Missing submitReservation request or reservationId or reservation definition or queue.";
+
+    // null request1
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> interceptor.submitReservation(null));
+
+    // null request2
+    ReservationSubmissionRequest request2 =
+        ReservationSubmissionRequest.newInstance(null, null, null);
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> interceptor.submitReservation(request2));
+
+    // null request3
+    ReservationSubmissionRequest request3 =
+        ReservationSubmissionRequest.newInstance(null, "q1", null);
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> interceptor.submitReservation(request3));
+
+    // null request4
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    ReservationSubmissionRequest request4 =
+        ReservationSubmissionRequest.newInstance(null, null,  reservationId);
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> interceptor.submitReservation(request4));
+
+    // null request5
+    long arrival = Time.now();
+    long deadline = arrival + (int)(DEFAULT_DURATION * 1.1);
+
+    ReservationRequest rRequest = ReservationRequest.newInstance(
+        Resource.newInstance(1024, 1), 1, 1, DEFAULT_DURATION);
+    ReservationRequest[] rRequests = new ReservationRequest[] {rRequest};
+    ReservationDefinition rDefinition = createReservationDefinition(arrival, deadline, rRequests,
+        ReservationRequestInterpreter.R_ALL, "u1");
+    ReservationSubmissionRequest request5 =
+        ReservationSubmissionRequest.newInstance(rDefinition, null,  reservationId);
+    LambdaTestUtils.intercept(YarnException.class, errorMsg,
+        () -> interceptor.submitReservation(request5));
+  }
+
+  @Test
+  public void testSubmitReservationMultipleSubmission() throws Exception {
+    LOG.info("Test FederationClientInterceptor: Submit Reservation - Multiple");
+
+    // get new reservationId
+    GetNewReservationRequest request = GetNewReservationRequest.newInstance();
+    GetNewReservationResponse response = interceptor.getNewReservation(request);
+    Assert.assertNotNull(response);
+
+    // First Submit Reservation
+    ReservationId reservationId = response.getReservationId();
+    ReservationDefinition rDefinition = createReservationDefinition(1024, 1);
+    ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance(
+        rDefinition, "decided", reservationId);
+    ReservationSubmissionResponse submissionResponse =
+        interceptor.submitReservation(rSubmissionRequest);
+    Assert.assertNotNull(submissionResponse);
+
+    SubClusterId subClusterId1 = stateStoreUtil.queryReservationHomeSC(reservationId);
+    Assert.assertNotNull(subClusterId1);
+    Assert.assertTrue(subClusters.contains(subClusterId1));
+
+    // First Retry, repeat the submission
+    ReservationSubmissionResponse submissionResponse1 =
+        interceptor.submitReservation(rSubmissionRequest);
+    Assert.assertNotNull(submissionResponse1);
+
+    // Expect reserved clusters to be consistent
+    SubClusterId subClusterId2 = stateStoreUtil.queryReservationHomeSC(reservationId);
+    Assert.assertNotNull(subClusterId2);
+    Assert.assertEquals(subClusterId1, subClusterId2);
+  }
+
+  @Test
+  public void testUpdateReservation() throws Exception {
+    LOG.info("Test FederationClientInterceptor : UpdateReservation request.");
+
+    // get new reservationId
+    GetNewReservationRequest request = GetNewReservationRequest.newInstance();
+    GetNewReservationResponse response = interceptor.getNewReservation(request);
+    Assert.assertNotNull(response);
+
+    // allow plan follower to synchronize, manually trigger an assignment
+    Map<SubClusterId, MockRM> mockRMs = interceptor.getMockRMs();
+    for (MockRM mockRM : mockRMs.values()) {
+      ReservationSystem reservationSystem = mockRM.getReservationSystem();
+      reservationSystem.synchronizePlan("root.decided", true);
+    }
+
+    // Submit Reservation
+    ReservationId reservationId = response.getReservationId();
+    ReservationDefinition rDefinition = createReservationDefinition(1024, 1);
+    ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance(
+        rDefinition, "decided", reservationId);
+
+    ReservationSubmissionResponse submissionResponse =
+        interceptor.submitReservation(rSubmissionRequest);
+    Assert.assertNotNull(submissionResponse);
+
+    // Update Reservation
+    ReservationDefinition rDefinition2 = createReservationDefinition(2048, 1);
+    ReservationUpdateRequest updateRequest =
+        ReservationUpdateRequest.newInstance(rDefinition2, reservationId);
+    ReservationUpdateResponse updateResponse =
+        interceptor.updateReservation(updateRequest);
+    Assert.assertNotNull(updateResponse);
+
+    SubClusterId subClusterId = stateStoreUtil.queryReservationHomeSC(reservationId);
+    Assert.assertNotNull(subClusterId);
+  }
+
+  @Test
+  public void testDeleteReservation() throws Exception {
+    LOG.info("Test FederationClientInterceptor : DeleteReservation request.");
+
+    // get new reservationId
+    GetNewReservationRequest request = GetNewReservationRequest.newInstance();
+    GetNewReservationResponse response = interceptor.getNewReservation(request);
+    Assert.assertNotNull(response);
+
+    // allow plan follower to synchronize, manually trigger an assignment
+    Map<SubClusterId, MockRM> mockRMs = interceptor.getMockRMs();
+    for (MockRM mockRM : mockRMs.values()) {
+      ReservationSystem reservationSystem = mockRM.getReservationSystem();
+      reservationSystem.synchronizePlan("root.decided", true);
+    }
+
+    // Submit Reservation
+    ReservationId reservationId = response.getReservationId();
+    ReservationDefinition rDefinition = createReservationDefinition(1024, 1);
+    ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance(
+        rDefinition, "decided", reservationId);
+
+    ReservationSubmissionResponse submissionResponse =
+        interceptor.submitReservation(rSubmissionRequest);
+    Assert.assertNotNull(submissionResponse);
+
+    // Delete Reservation
+    ReservationDeleteRequest deleteRequest = ReservationDeleteRequest.newInstance(reservationId);
+    ReservationDeleteResponse deleteResponse = interceptor.deleteReservation(deleteRequest);
+    Assert.assertNotNull(deleteResponse);
+
+    LambdaTestUtils.intercept(YarnException.class,
+        "Reservation " + reservationId + " does not exist",
+        () -> stateStoreUtil.queryReservationHomeSC(reservationId));
+  }
+
+
+  private ReservationDefinition createReservationDefinition(int memory, int core) {
+    // get reservationId
+    long arrival = Time.now();
+    long deadline = arrival + (int)(DEFAULT_DURATION * 1.1);
+
+    ReservationRequest rRequest = ReservationRequest.newInstance(
+        Resource.newInstance(memory, core), 1, 1, DEFAULT_DURATION);
+    ReservationRequest[] rRequests = new ReservationRequest[] {rRequest};
+
+    ReservationDefinition rDefinition = createReservationDefinition(arrival, deadline, rRequests,
+        ReservationRequestInterpreter.R_ALL, "u1");
+
+    return rDefinition;
+  }
+
+  /**
+   * This method is used to create a ReservationDefinition.
+   *
+   * @param arrival Job arrival time
+   * @param deadline Job deadline
+   * @param reservationRequests reservationRequest Array
+   * @param rType Enumeration of various types of
+   *              dependencies among multiple ReservationRequest
+   * @param username username
+   * @return ReservationDefinition
+   */
+  private ReservationDefinition createReservationDefinition(long arrival,
+      long deadline, ReservationRequest[] reservationRequests,
+      ReservationRequestInterpreter rType, String username) {
+    ReservationRequests requests = ReservationRequests
+        .newInstance(Arrays.asList(reservationRequests), rType);
+    return ReservationDefinition.newInstance(arrival, deadline,
+        requests, username, "0", Priority.UNDEFINED);
+  }
+
+  @Test
+  public void testGetNumMinThreads() {
+    // If we don't configure YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE,
+    // we expect to get 5 threads
+    int minThreads = interceptor.getNumMinThreads(this.getConf());
+    Assert.assertEquals(5, minThreads);
+
+    // If we configure YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE,
+    // we expect to get 3 threads
+    this.getConf().unset(YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE);
+    this.getConf().setInt(YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MINIMUM_POOL_SIZE, 3);
+    int minThreads2 = interceptor.getNumMinThreads(this.getConf());
+    Assert.assertEquals(3, minThreads2);
+  }
+
+  @Test
+  public void testGetNumMaxThreads() {
+    // If we don't configure YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE,
+    // we expect to get 5 threads
+    int minThreads = interceptor.getNumMaxThreads(this.getConf());
+    Assert.assertEquals(5, minThreads);
+
+    // If we configure YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE,
+    // we expect to get 8 threads
+    this.getConf().unset(YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE);
+    this.getConf().setInt(YarnConfiguration.ROUTER_USER_CLIENT_THREAD_POOL_MAXIMUM_POOL_SIZE, 8);
+    int minThreads2 = interceptor.getNumMaxThreads(this.getConf());
+    Assert.assertEquals(8, minThreads2);
+  }
+
+  @Test
+  public void testGetDelegationToken() throws IOException, YarnException {
+
+    // We design such a unit test to check
+    // that the execution of the GetDelegationToken method is as expected.
+    //
+    // 1. Apply for a DelegationToken for renewer1,
+    // the Router returns the DelegationToken of the user, and the KIND of the token is
+    // RM_DELEGATION_TOKEN
+    //
+    // 2. We maintain the compatibility with RMDelegationTokenIdentifier,
+    // we can serialize the token into RMDelegationTokenIdentifier.
+    //
+    // 3. We can get the issueDate, and compare the data in the StateStore,
+    // the data should be consistent.
+
+    // Step1. We apply for DelegationToken for renewer1
+    // Both response & delegationToken cannot be empty
+    GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
+    when(request.getRenewer()).thenReturn("renewer1");
+    GetDelegationTokenResponse response = interceptor.getDelegationToken(request);
+    Assert.assertNotNull(response);
+    Token delegationToken = response.getRMDelegationToken();
+    Assert.assertNotNull(delegationToken);
+    Assert.assertEquals("RM_DELEGATION_TOKEN", delegationToken.getKind());
+
+    // Step2. Serialize the returned Token as RMDelegationTokenIdentifier.
+    org.apache.hadoop.security.token.Token<RMDelegationTokenIdentifier> token =
+        ConverterUtils.convertFromYarn(delegationToken, (Text) null);
+    RMDelegationTokenIdentifier rMDelegationTokenIdentifier = token.decodeIdentifier();
+    Assert.assertNotNull(rMDelegationTokenIdentifier);
+
+    // Step3. Verify the returned data of the token.
+    String renewer = rMDelegationTokenIdentifier.getRenewer().toString();
+    long issueDate = rMDelegationTokenIdentifier.getIssueDate();
+    long maxDate = rMDelegationTokenIdentifier.getMaxDate();
+    Assert.assertEquals("renewer1", renewer);
+
+    long tokenMaxLifetime = this.getConf().getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+    Assert.assertEquals(issueDate + tokenMaxLifetime, maxDate);
+
+    RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState();
+    Assert.assertNotNull(managerState);
+
+    Map<RMDelegationTokenIdentifier, Long> delegationTokenState = managerState.getTokenState();
+    Assert.assertNotNull(delegationTokenState);
+    Assert.assertTrue(delegationTokenState.containsKey(rMDelegationTokenIdentifier));
+
+    long tokenRenewInterval = this.getConf().getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+    long renewDate = delegationTokenState.get(rMDelegationTokenIdentifier);
+    Assert.assertEquals(issueDate + tokenRenewInterval, renewDate);
+  }
+
+  @Test
+  public void testRenewDelegationToken() throws IOException, YarnException {
+
+    // We design such a unit test to check
+    // that the execution of the GetDelegationToken method is as expected
+    // 1. Call GetDelegationToken to apply for delegationToken.
+    // 2. Call renewDelegationToken to refresh delegationToken.
+    // By looking at the code of AbstractDelegationTokenSecretManager#renewToken,
+    // we know that renewTime is calculated as Math.min(id.getMaxDate(), now + tokenRenewInterval)
+    // so renewTime will be less than or equal to maxDate.
+    // 3. We will compare whether the expirationTime returned to the
+    // client is consistent with the renewDate in the stateStore.
+
+    // Step1. Call GetDelegationToken to apply for delegationToken.
+    GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
+    when(request.getRenewer()).thenReturn("renewer2");
+    GetDelegationTokenResponse response = interceptor.getDelegationToken(request);
+    Assert.assertNotNull(response);
+    Token delegationToken = response.getRMDelegationToken();
+
+    org.apache.hadoop.security.token.Token<RMDelegationTokenIdentifier> token =
+        ConverterUtils.convertFromYarn(delegationToken, (Text) null);
+    RMDelegationTokenIdentifier rMDelegationTokenIdentifier = token.decodeIdentifier();
+    String renewer = rMDelegationTokenIdentifier.getRenewer().toString();
+    long maxDate = rMDelegationTokenIdentifier.getMaxDate();
+    Assert.assertEquals("renewer2", renewer);
+
+    // Step2. Call renewDelegationToken to refresh delegationToken.
+    RenewDelegationTokenRequest renewRequest = Records.newRecord(RenewDelegationTokenRequest.class);
+    renewRequest.setDelegationToken(delegationToken);
+    RenewDelegationTokenResponse renewResponse = interceptor.renewDelegationToken(renewRequest);
+    Assert.assertNotNull(renewResponse);
+
+    long expDate = renewResponse.getNextExpirationTime();
+    Assert.assertTrue(expDate <= maxDate);
+
+    // Step3. Compare whether the expirationTime returned to
+    // the client is consistent with the renewDate in the stateStore
+    RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState();
+    Map<RMDelegationTokenIdentifier, Long> delegationTokenState = managerState.getTokenState();
+    Assert.assertNotNull(delegationTokenState);
+    Assert.assertTrue(delegationTokenState.containsKey(rMDelegationTokenIdentifier));
+    long renewDate = delegationTokenState.get(rMDelegationTokenIdentifier);
+    Assert.assertEquals(expDate, renewDate);
+  }
+
+  @Test
+  public void testCancelDelegationToken() throws IOException, YarnException {
+
+    // We design such a unit test to check
+    // that the execution of the CancelDelegationToken method is as expected
+    // 1. Call GetDelegationToken to apply for delegationToken.
+    // 2. Call CancelDelegationToken to cancel delegationToken.
+    // 3. Query the data in the StateStore and confirm that the Delegation has been deleted.
+
+    // Step1. Call GetDelegationToken to apply for delegationToken.
+    GetDelegationTokenRequest request = mock(GetDelegationTokenRequest.class);
+    when(request.getRenewer()).thenReturn("renewer3");
+    GetDelegationTokenResponse response = interceptor.getDelegationToken(request);
+    Assert.assertNotNull(response);
+    Token delegationToken = response.getRMDelegationToken();
+
+    // Step2. Call CancelDelegationToken to cancel delegationToken.
+    CancelDelegationTokenRequest cancelTokenRequest =
+        CancelDelegationTokenRequest.newInstance(delegationToken);
+    CancelDelegationTokenResponse cancelTokenResponse =
+        interceptor.cancelDelegationToken(cancelTokenRequest);
+    Assert.assertNotNull(cancelTokenResponse);
+
+    // Step3. Query the data in the StateStore and confirm that the Delegation has been deleted.
+    // At this point, the size of delegationTokenState should be 0.
+    RouterRMDTSecretManagerState managerState = stateStore.getRouterRMSecretManagerState();
+    Map<RMDelegationTokenIdentifier, Long> delegationTokenState = managerState.getTokenState();
+    Assert.assertNotNull(delegationTokenState);
+    Assert.assertEquals(0, delegationTokenState.size());
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java
index 37278861afaa6..2d0bc6b3507db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java
@@ -18,55 +18,75 @@
 
 package org.apache.hadoop.yarn.server.router.clientrm;
 
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.FEDERATION_POLICY_MANAGER;
+import static org.hamcrest.CoreMatchers.is;
 import static org.mockito.Mockito.mock;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
 import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE;
+
 /**
  * Extends the {@code BaseRouterClientRMTest} and overrides methods in order to
  * use the {@code RouterClientRMService} pipeline test cases for testing the
  * {@code FederationInterceptor} class. The tests for
  * {@code RouterClientRMService} has been written cleverly so that it can be
- * reused to validate different request intercepter chains.
+ * reused to validate different request interceptor chains.
  *
  * It tests the case with SubClusters down and the Router logic of retries. We
  * have 1 good SubCluster and 2 bad ones for all the tests.
  */
+@RunWith(Parameterized.class)
 public class TestFederationClientInterceptorRetry
     extends BaseRouterClientRMTest {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestFederationClientInterceptorRetry.class);
 
+  @Parameters
+  public static Collection<String[]> getParameters() {
+    return Arrays.asList(new String[][] {{UniformBroadcastPolicyManager.class.getName()},
+        {TestSequentialBroadcastPolicyManager.class.getName()}});
+  }
+
   private TestableFederationClientInterceptor interceptor;
   private MemoryFederationStateStore stateStore;
   private FederationStateStoreTestUtil stateStoreUtil;
+  private String routerPolicyManagerName;
 
   private String user = "test-user";
 
@@ -77,7 +97,11 @@ public class TestFederationClientInterceptorRetry
   private static SubClusterId bad1;
   private static SubClusterId bad2;
 
-  private static List<SubClusterId> scs = new ArrayList<SubClusterId>();
+  private static List<SubClusterId> scs = new ArrayList<>();
+
+  public TestFederationClientInterceptorRetry(String policyManagerName) {
+    this.routerPolicyManagerName = policyManagerName;
+  }
 
   @Override
   public void setUp() throws IOException {
@@ -114,8 +138,7 @@ public void tearDown() {
     super.tearDown();
   }
 
-  private void setupCluster(List<SubClusterId> scsToRegister)
-      throws YarnException {
+  private void setupCluster(List<SubClusterId> scsToRegister) throws YarnException {
 
     try {
       // Clean up the StateStore before every test
@@ -132,21 +155,21 @@ private void setupCluster(List<SubClusterId> scsToRegister)
 
   @Override
   protected YarnConfiguration createConfiguration() {
+
     YarnConfiguration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
     String mockPassThroughInterceptorClass =
         PassThroughClientRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
-    // chain is the federation intercepter that calls the mock resource manager.
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
     // The others in the chain will simply forward it to the next one in the
     // chain
     conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
         mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass
             + "," + TestableFederationClientInterceptor.class.getName());
 
-    conf.set(YarnConfiguration.FEDERATION_POLICY_MANAGER,
-        UniformBroadcastPolicyManager.class.getName());
+    conf.set(FEDERATION_POLICY_MANAGER, this.routerPolicyManagerName);
 
     // Disable StateStoreFacade cache
     conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
@@ -159,20 +182,14 @@ protected YarnConfiguration createConfiguration() {
    * cluster is composed of only 1 bad SubCluster.
    */
   @Test
-  public void testGetNewApplicationOneBadSC()
-      throws YarnException, IOException, InterruptedException {
+  public void testGetNewApplicationOneBadSC() throws Exception {
 
-    System.out.println("Test getNewApplication with one bad SubCluster");
+    LOG.info("Test getNewApplication with one bad SubCluster");
     setupCluster(Arrays.asList(bad2));
 
-    try {
-      interceptor.getNewApplication(GetNewApplicationRequest.newInstance());
-      Assert.fail();
-    } catch (Exception e) {
-      System.out.println(e.toString());
-      Assert.assertTrue(e.getMessage()
-          .equals(FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE));
-    }
+    GetNewApplicationRequest request = GetNewApplicationRequest.newInstance();
+    LambdaTestUtils.intercept(YarnException.class, NO_ACTIVE_SUBCLUSTER_AVAILABLE,
+        () -> interceptor.getNewApplication(request));
   }
 
   /**
@@ -180,19 +197,14 @@ public void testGetNewApplicationOneBadSC()
    * cluster is composed of only 2 bad SubClusters.
    */
   @Test
-  public void testGetNewApplicationTwoBadSCs()
-      throws YarnException, IOException, InterruptedException {
-    System.out.println("Test getNewApplication with two bad SubClusters");
+  public void testGetNewApplicationTwoBadSCs() throws Exception {
+
+    LOG.info("Test getNewApplication with two bad SubClusters");
     setupCluster(Arrays.asList(bad1, bad2));
 
-    try {
-      interceptor.getNewApplication(GetNewApplicationRequest.newInstance());
-      Assert.fail();
-    } catch (Exception e) {
-      System.out.println(e.toString());
-      Assert.assertTrue(e.getMessage()
-          .equals(FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE));
-    }
+    GetNewApplicationRequest request = GetNewApplicationRequest.newInstance();
+    LambdaTestUtils.intercept(YarnException.class, NO_ACTIVE_SUBCLUSTER_AVAILABLE,
+        () -> interceptor.getNewApplication(request));
   }
 
   /**
@@ -200,17 +212,14 @@ public void testGetNewApplicationTwoBadSCs()
    * cluster is composed of only 1 bad SubCluster and 1 good one.
    */
   @Test
-  public void testGetNewApplicationOneBadOneGood()
-      throws YarnException, IOException, InterruptedException {
-    System.out.println("Test getNewApplication with one bad, one good SC");
+  public void testGetNewApplicationOneBadOneGood() throws YarnException, IOException {
+
+    LOG.info("Test getNewApplication with one bad, one good SC");
     setupCluster(Arrays.asList(good, bad2));
-    GetNewApplicationResponse response = null;
-    try {
-      response =
-          interceptor.getNewApplication(GetNewApplicationRequest.newInstance());
-    } catch (Exception e) {
-      Assert.fail();
-    }
+    GetNewApplicationRequest request = GetNewApplicationRequest.newInstance();
+    GetNewApplicationResponse response = interceptor.getNewApplication(request);
+
+    Assert.assertNotNull(response);
     Assert.assertEquals(ResourceManager.getClusterTimeStamp(),
         response.getApplicationId().getClusterTimestamp());
   }
@@ -220,38 +229,27 @@ public void testGetNewApplicationOneBadOneGood()
    * cluster is composed of only 1 bad SubCluster.
    */
   @Test
-  public void testSubmitApplicationOneBadSC()
-      throws YarnException, IOException, InterruptedException {
+  public void testSubmitApplicationOneBadSC() throws Exception {
 
-    System.out.println("Test submitApplication with one bad SubCluster");
+    LOG.info("Test submitApplication with one bad SubCluster");
     setupCluster(Arrays.asList(bad2));
 
     final ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
-    final SubmitApplicationRequest request = mockSubmitApplicationRequest(
-        appId);
-    try {
-      interceptor.submitApplication(request);
-      Assert.fail();
-    } catch (Exception e) {
-      System.out.println(e);
-      Assert.assertTrue(e.getMessage()
-          .equals(FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE));
-    }
+    final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+    LambdaTestUtils.intercept(YarnException.class, NO_ACTIVE_SUBCLUSTER_AVAILABLE,
+        () -> interceptor.submitApplication(request));
   }
 
-  private SubmitApplicationRequest mockSubmitApplicationRequest(
-      ApplicationId appId) {
+  private SubmitApplicationRequest mockSubmitApplicationRequest(ApplicationId appId) {
     ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
     ApplicationSubmissionContext context = ApplicationSubmissionContext
         .newInstance(appId, MockApps.newAppName(), "q1",
-            Priority.newInstance(0), amContainerSpec, false, false, -1,
-            Resources.createResource(
-                YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
-            "MockApp");
-    SubmitApplicationRequest request = SubmitApplicationRequest
-        .newInstance(context);
+        Priority.newInstance(0), amContainerSpec, false, false, -1,
+        Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+        "MockApp");
+    SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(context);
     return request;
   }
 
@@ -260,24 +258,17 @@ private SubmitApplicationRequest mockSubmitApplicationRequest(
    * cluster is composed of only 2 bad SubClusters.
    */
   @Test
-  public void testSubmitApplicationTwoBadSCs()
-      throws YarnException, IOException, InterruptedException {
-    System.out.println("Test submitApplication with two bad SubClusters");
+  public void testSubmitApplicationTwoBadSCs() throws Exception {
+
+    LOG.info("Test submitApplication with two bad SubClusters.");
     setupCluster(Arrays.asList(bad1, bad2));
 
     final ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
-    final SubmitApplicationRequest request = mockSubmitApplicationRequest(
-        appId);
-    try {
-      interceptor.submitApplication(request);
-      Assert.fail();
-    } catch (Exception e) {
-      System.out.println(e.toString());
-      Assert.assertTrue(e.getMessage()
-          .equals(FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE));
-    }
+    final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+    LambdaTestUtils.intercept(YarnException.class, NO_ACTIVE_SUBCLUSTER_AVAILABLE,
+        () -> interceptor.submitApplication(request));
   }
 
   /**
@@ -287,24 +278,79 @@ public void testSubmitApplicationTwoBadSCs()
   @Test
   public void testSubmitApplicationOneBadOneGood()
       throws YarnException, IOException, InterruptedException {
-    System.out.println("Test submitApplication with one bad, one good SC");
+
+    LOG.info("Test submitApplication with one bad, one good SC.");
     setupCluster(Arrays.asList(good, bad2));
 
     final ApplicationId appId =
         ApplicationId.newInstance(System.currentTimeMillis(), 1);
 
-    final SubmitApplicationRequest request = mockSubmitApplicationRequest(
-        appId);
-    try {
-      interceptor.submitApplication(request);
-    } catch (Exception e) {
-      Assert.fail();
-    }
-    Assert.assertEquals(good,
-        stateStore
-            .getApplicationHomeSubCluster(
-                GetApplicationHomeSubClusterRequest.newInstance(appId))
-            .getApplicationHomeSubCluster().getHomeSubCluster());
+    final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+    SubmitApplicationResponse response = interceptor.submitApplication(request);
+    Assert.assertNotNull(response);
+
+    GetApplicationHomeSubClusterRequest getAppRequest =
+        GetApplicationHomeSubClusterRequest.newInstance(appId);
+    GetApplicationHomeSubClusterResponse getAppResponse =
+        stateStore.getApplicationHomeSubCluster(getAppRequest);
+    Assert.assertNotNull(getAppResponse);
+
+    ApplicationHomeSubCluster responseHomeSubCluster =
+        getAppResponse.getApplicationHomeSubCluster();
+    Assert.assertNotNull(responseHomeSubCluster);
+    SubClusterId respSubClusterId = responseHomeSubCluster.getHomeSubCluster();
+    Assert.assertEquals(good, respSubClusterId);
   }
 
+  @Test
+  public void testSubmitApplicationTwoBadOneGood() throws Exception {
+
+    LOG.info("Test submitApplication with two bad, one good SC.");
+
+    // This test must require the TestSequentialRouterPolicy policy
+    Assume.assumeThat(routerPolicyManagerName,
+        is(TestSequentialBroadcastPolicyManager.class.getName()));
+
+    setupCluster(Arrays.asList(bad1, bad2, good));
+    final ApplicationId appId =
+        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+
+    // Use the TestSequentialRouterPolicy strategy,
+    // which will sort the SubClusterId because good=0, bad1=1, bad2=2
+    // We will get 2, 1, 0 [bad2, bad1, good]
+    // Set the retryNum to 1
+    // 1st time will use bad2, 2nd time will use bad1
+    // bad1 is updated to stateStore
+    interceptor.setNumSubmitRetries(1);
+    final SubmitApplicationRequest request = mockSubmitApplicationRequest(appId);
+    LambdaTestUtils.intercept(YarnException.class, "RM is stopped",
+        () -> interceptor.submitApplication(request));
+
+    // We will get bad1
+    checkSubmitSubCluster(appId, bad1);
+
+    // Set the retryNum to 2
+    // 1st time will use bad2, 2nd time will use bad1, 3rd good
+    interceptor.setNumSubmitRetries(2);
+    SubmitApplicationResponse submitAppResponse = interceptor.submitApplication(request);
+    Assert.assertNotNull(submitAppResponse);
+
+    // We will get good
+    checkSubmitSubCluster(appId, good);
+  }
+
+  private void checkSubmitSubCluster(ApplicationId appId, SubClusterId expectSubCluster)
+      throws YarnException {
+    GetApplicationHomeSubClusterRequest getAppRequest =
+        GetApplicationHomeSubClusterRequest.newInstance(appId);
+    GetApplicationHomeSubClusterResponse getAppResponse =
+        stateStore.getApplicationHomeSubCluster(getAppRequest);
+    Assert.assertNotNull(getAppResponse);
+    Assert.assertNotNull(getAppResponse);
+    ApplicationHomeSubCluster responseHomeSubCluster =
+        getAppResponse.getApplicationHomeSubCluster();
+    Assert.assertNotNull(responseHomeSubCluster);
+    SubClusterId respSubClusterId = responseHomeSubCluster.getHomeSubCluster();
+    Assert.assertEquals(expectSubCluster, respSubClusterId);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java
index c91da3527932f..346e9e8784171 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java
@@ -20,9 +20,26 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
 
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -31,6 +48,20 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationAllocationState;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
 import org.junit.Assert;
 import org.junit.Test;
@@ -51,10 +82,12 @@ public void testClusterMetricsMerge() {
     YarnClusterMetrics resultMetrics = result.getClusterMetrics();
     Assert.assertEquals(3, resultMetrics.getNumNodeManagers());
     Assert.assertEquals(3, resultMetrics.getNumActiveNodeManagers());
+    Assert.assertEquals(3, resultMetrics.getNumDecommissioningNodeManagers());
     Assert.assertEquals(3, resultMetrics.getNumDecommissionedNodeManagers());
     Assert.assertEquals(3, resultMetrics.getNumLostNodeManagers());
     Assert.assertEquals(3, resultMetrics.getNumRebootedNodeManagers());
     Assert.assertEquals(3, resultMetrics.getNumUnhealthyNodeManagers());
+    Assert.assertEquals(3, resultMetrics.getNumShutdownNodeManagers());
   }
 
   public GetClusterMetricsResponse getClusterMetricsResponse(int value) {
@@ -62,9 +95,11 @@ public GetClusterMetricsResponse getClusterMetricsResponse(int value) {
     metrics.setNumUnhealthyNodeManagers(value);
     metrics.setNumRebootedNodeManagers(value);
     metrics.setNumLostNodeManagers(value);
+    metrics.setNumDecommissioningNodeManagers(value);
     metrics.setNumDecommissionedNodeManagers(value);
     metrics.setNumActiveNodeManagers(value);
     metrics.setNumNodeManagers(value);
+    metrics.setNumShutdownNodeManagers(value);
     return GetClusterMetricsResponse.newInstance(metrics);
   }
 
@@ -213,4 +248,521 @@ private GetApplicationsResponse getApplicationsResponse(int value,
 
     return GetApplicationsResponse.newInstance(applications);
   }
+
+  @Test
+  public void testMergeNodesToLabelsResponse() {
+    NodeId node1 = NodeId.fromString("SubCluster1Node1:1111");
+    NodeId node2 = NodeId.fromString("SubCluster1Node2:2222");
+    NodeId node3 = NodeId.fromString("SubCluster2Node1:1111");
+
+    Map<NodeId, Set<String>> nodeLabelsMapSC1 = new HashMap<>();
+    nodeLabelsMapSC1.put(node1, ImmutableSet.of("node1"));
+    nodeLabelsMapSC1.put(node2, ImmutableSet.of("node2"));
+    nodeLabelsMapSC1.put(node3, ImmutableSet.of("node3"));
+
+    // normal response
+    GetNodesToLabelsResponse response1 = Records.newRecord(
+        GetNodesToLabelsResponse.class);
+    response1.setNodeToLabels(nodeLabelsMapSC1);
+
+    // empty response
+    Map<NodeId, Set<String>> nodeLabelsMapSC2 = new HashMap<>();
+    GetNodesToLabelsResponse response2 = Records.newRecord(
+        GetNodesToLabelsResponse.class);
+    response2.setNodeToLabels(nodeLabelsMapSC2);
+
+    // null response
+    GetNodesToLabelsResponse response3 = null;
+
+    Map<NodeId, Set<String>> expectedResponse = new HashMap<>();
+    expectedResponse.put(node1, ImmutableSet.of("node1"));
+    expectedResponse.put(node2, ImmutableSet.of("node2"));
+    expectedResponse.put(node3, ImmutableSet.of("node3"));
+
+    List<GetNodesToLabelsResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+
+    GetNodesToLabelsResponse response = RouterYarnClientUtils.
+        mergeNodesToLabelsResponse(responses);
+    Assert.assertEquals(expectedResponse, response.getNodeToLabels());
+  }
+
+  @Test
+  public void testMergeClusterNodeLabelsResponse() {
+    NodeLabel nodeLabel1 = NodeLabel.newInstance("nodeLabel1");
+    NodeLabel nodeLabel2 = NodeLabel.newInstance("nodeLabel2");
+    NodeLabel nodeLabel3 = NodeLabel.newInstance("nodeLabel3");
+
+    // normal response
+    List<NodeLabel> nodeLabelListSC1 = new ArrayList<>();
+    nodeLabelListSC1.add(nodeLabel1);
+    nodeLabelListSC1.add(nodeLabel2);
+    nodeLabelListSC1.add(nodeLabel3);
+
+    GetClusterNodeLabelsResponse response1 = Records.newRecord(
+        GetClusterNodeLabelsResponse.class);
+    response1.setNodeLabelList(nodeLabelListSC1);
+
+    // empty response
+    List<NodeLabel> nodeLabelListSC2 = new ArrayList<>();
+
+    GetClusterNodeLabelsResponse response2 = Records.newRecord(
+        GetClusterNodeLabelsResponse.class);
+    response2.setNodeLabelList(nodeLabelListSC2);
+
+    // null response
+    GetClusterNodeLabelsResponse response3 = null;
+
+    List<GetClusterNodeLabelsResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+
+    List<NodeLabel> expectedResponse = new ArrayList<>();
+    expectedResponse.add(nodeLabel1);
+    expectedResponse.add(nodeLabel2);
+    expectedResponse.add(nodeLabel3);
+
+    GetClusterNodeLabelsResponse response = RouterYarnClientUtils.
+        mergeClusterNodeLabelsResponse(responses);
+    Assert.assertTrue(CollectionUtils.isEqualCollection(expectedResponse,
+        response.getNodeLabelList()));
+  }
+
+  @Test
+  public void testMergeLabelsToNodes(){
+    NodeId node1 = NodeId.fromString("SubCluster1Node1:1111");
+    NodeId node2 = NodeId.fromString("SubCluster1Node2:2222");
+    NodeId node3 = NodeId.fromString("SubCluster2node1:1111");
+    NodeId node4 = NodeId.fromString("SubCluster2node2:2222");
+
+    Map<String, Set<NodeId>> labelsToNodesSC1 = new HashMap<>();
+
+    Set<NodeId> nodeIdSet1 = new HashSet<>();
+    nodeIdSet1.add(node1);
+    nodeIdSet1.add(node2);
+    labelsToNodesSC1.put("Label1", nodeIdSet1);
+
+    // normal response
+    GetLabelsToNodesResponse response1 = Records.newRecord(
+        GetLabelsToNodesResponse.class);
+    response1.setLabelsToNodes(labelsToNodesSC1);
+    Map<String, Set<NodeId>> labelsToNodesSC2 = new HashMap<>();
+    Set<NodeId> nodeIdSet2 = new HashSet<>();
+    nodeIdSet2.add(node3);
+    Set<NodeId> nodeIdSet3 = new HashSet<>();
+    nodeIdSet3.add(node4);
+    labelsToNodesSC2.put("Label1", nodeIdSet2);
+    labelsToNodesSC2.put("Label2", nodeIdSet3);
+
+    GetLabelsToNodesResponse response2 = Records.newRecord(
+        GetLabelsToNodesResponse.class);
+    response2.setLabelsToNodes(labelsToNodesSC2);
+
+    // empty response
+    GetLabelsToNodesResponse response3 = Records.newRecord(
+        GetLabelsToNodesResponse.class);
+
+    // null response
+    GetLabelsToNodesResponse response4 = null;
+
+    List<GetLabelsToNodesResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    Map<String, Set<NodeId>> expectedResponse = new HashMap<>();
+    Set<NodeId> nodeIdMergedSet1 = new HashSet<>();
+    nodeIdMergedSet1.add(node1);
+    nodeIdMergedSet1.add(node2);
+    nodeIdMergedSet1.add(node3);
+
+    Set<NodeId> nodeIdMergedSet2 = new HashSet<>();
+    nodeIdMergedSet2.add(node4);
+    expectedResponse.put("Label1", nodeIdMergedSet1);
+    expectedResponse.put("Label2", nodeIdMergedSet2);
+
+    GetLabelsToNodesResponse response = RouterYarnClientUtils.
+        mergeLabelsToNodes(responses);
+
+    Assert.assertEquals(expectedResponse, response.getLabelsToNodes());
+  }
+
+  @Test
+  public void testMergeQueueUserAclsResponse() {
+
+    List<QueueACL> submitOnlyAcl = new ArrayList<>();
+    submitOnlyAcl.add(QueueACL.SUBMIT_APPLICATIONS);
+
+    List<QueueACL> administerOnlyAcl = new ArrayList<>();
+    administerOnlyAcl.add(QueueACL.ADMINISTER_QUEUE);
+
+    List<QueueACL> submitAndAdministerAcl = new ArrayList<>();
+    submitAndAdministerAcl.add(QueueACL.ADMINISTER_QUEUE);
+    submitAndAdministerAcl.add(QueueACL.SUBMIT_APPLICATIONS);
+
+    QueueUserACLInfo queueUserACLInfo1 = QueueUserACLInfo.newInstance(
+        "root", submitAndAdministerAcl);
+
+    QueueUserACLInfo queueUserACLInfo2 = QueueUserACLInfo.newInstance(
+        "default", submitOnlyAcl);
+
+    QueueUserACLInfo queueUserACLInfo3 = QueueUserACLInfo.newInstance(
+        "root", submitAndAdministerAcl);
+
+    QueueUserACLInfo queueUserACLInfo4 = QueueUserACLInfo.newInstance(
+        "yarn", administerOnlyAcl);
+
+    List<QueueUserACLInfo> queueUserACLInfoList1 = new ArrayList<>();
+    List<QueueUserACLInfo> queueUserACLInfoList2 = new ArrayList<>();
+
+    queueUserACLInfoList1.add(queueUserACLInfo1);
+    queueUserACLInfoList1.add(queueUserACLInfo2);
+    queueUserACLInfoList2.add(queueUserACLInfo3);
+    queueUserACLInfoList2.add(queueUserACLInfo4);
+
+    // normal response
+    GetQueueUserAclsInfoResponse response1 = Records.newRecord(
+        GetQueueUserAclsInfoResponse.class);
+    response1.setUserAclsInfoList(queueUserACLInfoList1);
+    GetQueueUserAclsInfoResponse response2 = Records.newRecord(
+        GetQueueUserAclsInfoResponse.class);
+    response2.setUserAclsInfoList(queueUserACLInfoList2);
+
+    // empty response
+    GetQueueUserAclsInfoResponse response3 = Records.newRecord(
+        GetQueueUserAclsInfoResponse.class);
+
+    // null response
+    GetQueueUserAclsInfoResponse response4 = null;
+
+    List<GetQueueUserAclsInfoResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    // expected user acls
+    List<QueueUserACLInfo> expectedOutput = new ArrayList<>();
+    expectedOutput.add(queueUserACLInfo1);
+    expectedOutput.add(queueUserACLInfo2);
+    expectedOutput.add(queueUserACLInfo4);
+
+    GetQueueUserAclsInfoResponse response =
+        RouterYarnClientUtils.mergeQueueUserAcls(responses);
+    Assert.assertTrue(CollectionUtils.isEqualCollection(expectedOutput,
+        response.getUserAclsInfoList()));
+  }
+
+  @Test
+  public void testMergeReservationsList() {
+
+    // normal response
+    ReservationListResponse response1 = createReservationListResponse(
+        165348678000L, 165348690000L, 165348678000L, 1L);
+
+    ReservationListResponse response2 = createReservationListResponse(
+        165348750000L, 165348768000L, 165348750000L, 1L);
+
+    // empty response
+    ReservationListResponse response3 = ReservationListResponse.newInstance(new ArrayList<>());
+
+    // null response
+    ReservationListResponse response4 = null;
+
+    List<ReservationListResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    // expected response
+    List<ReservationAllocationState> expectedResponse = new ArrayList<>();
+    expectedResponse.addAll(response1.getReservationAllocationState());
+    expectedResponse.addAll(response2.getReservationAllocationState());
+
+    ReservationListResponse response =
+        RouterYarnClientUtils.mergeReservationsList(responses);
+    Assert.assertEquals(expectedResponse, response.getReservationAllocationState());
+  }
+
+  private ReservationListResponse createReservationListResponse(long startTime,
+      long endTime, long reservationTime, long reservationNumber) {
+    List<ReservationAllocationState> reservationsList = new ArrayList<>();
+    ReservationDefinition reservationDefinition =
+        Records.newRecord(ReservationDefinition.class);
+    reservationDefinition.setArrival(startTime);
+    reservationDefinition.setDeadline(endTime);
+    ReservationAllocationState reservationAllocationState =
+        Records.newRecord(ReservationAllocationState.class);
+    ReservationId reservationId = ReservationId.newInstance(reservationTime,
+        reservationNumber);
+    reservationAllocationState.setReservationDefinition(reservationDefinition);
+    reservationAllocationState.setReservationId(reservationId);
+    reservationsList.add(reservationAllocationState);
+    return ReservationListResponse.newInstance(reservationsList);
+  }
+
+  @Test
+  public void testMergeResourceTypes() {
+
+    ResourceTypeInfo resourceTypeInfo1 = ResourceTypeInfo.newInstance("vcores");
+    ResourceTypeInfo resourceTypeInfo2 = ResourceTypeInfo.newInstance("gpu");
+    ResourceTypeInfo resourceTypeInfo3 = ResourceTypeInfo.newInstance("memory-mb");
+
+    List<ResourceTypeInfo> resourceTypeInfoList1 = new ArrayList<>();
+    resourceTypeInfoList1.add(resourceTypeInfo1);
+    resourceTypeInfoList1.add(resourceTypeInfo3);
+
+    List<ResourceTypeInfo> resourceTypeInfoList2 = new ArrayList<>();
+    resourceTypeInfoList2.add(resourceTypeInfo3);
+    resourceTypeInfoList2.add(resourceTypeInfo2);
+
+    // normal response
+    GetAllResourceTypeInfoResponse response1 =
+        Records.newRecord(GetAllResourceTypeInfoResponse.class);
+    response1.setResourceTypeInfo(resourceTypeInfoList1);
+
+    GetAllResourceTypeInfoResponse response2 =
+        Records.newRecord(GetAllResourceTypeInfoResponse.class);
+    response2.setResourceTypeInfo(resourceTypeInfoList2);
+
+    // empty response
+    GetAllResourceTypeInfoResponse response3 =
+        Records.newRecord(GetAllResourceTypeInfoResponse.class);
+
+    // null response
+    GetAllResourceTypeInfoResponse response4 = null;
+
+    List<GetAllResourceTypeInfoResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    // expected response
+    List<ResourceTypeInfo> expectedResponse = new ArrayList<>();
+    expectedResponse.add(resourceTypeInfo1);
+    expectedResponse.add(resourceTypeInfo2);
+    expectedResponse.add(resourceTypeInfo3);
+    GetAllResourceTypeInfoResponse response =
+        RouterYarnClientUtils.mergeResourceTypes(responses);
+    Assert.assertTrue(CollectionUtils.isEqualCollection(expectedResponse,
+        response.getResourceTypeInfo()));
+  }
+
+  @Test
+  public void testMergeResourceProfiles() {
+    // normal response1
+    Map<String, Resource> profiles = new HashMap<>();
+    Resource resource1 = Resource.newInstance(1024, 1);
+    GetAllResourceProfilesResponse response1 = GetAllResourceProfilesResponse.newInstance();
+    profiles.put("maximum", resource1);
+    response1.setResourceProfiles(profiles);
+
+    // normal response2
+    profiles = new HashMap<>();
+    Resource resource2 = Resource.newInstance(2048, 2);
+    GetAllResourceProfilesResponse response2 = GetAllResourceProfilesResponse.newInstance();
+    profiles.put("maximum", resource2);
+    response2.setResourceProfiles(profiles);
+
+    // empty response
+    GetAllResourceProfilesResponse response3 = GetAllResourceProfilesResponse.newInstance();
+
+    // null response
+    GetAllResourceProfilesResponse response4 = null;
+
+    List<GetAllResourceProfilesResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    GetAllResourceProfilesResponse response =
+        RouterYarnClientUtils.mergeClusterResourceProfilesResponse(responses);
+    Resource resource = response.getResourceProfiles().get("maximum");
+    Assert.assertEquals(3, resource.getVirtualCores());
+    Assert.assertEquals(3072, resource.getMemorySize());
+  }
+
+  @Test
+  public void testMergeResourceProfile() {
+    // normal response1
+    Resource resource1 = Resource.newInstance(1024, 1);
+    GetResourceProfileResponse response1 =
+        Records.newRecord(GetResourceProfileResponse.class);
+    response1.setResource(resource1);
+
+    // normal response2
+    Resource resource2 = Resource.newInstance(2048, 2);
+    GetResourceProfileResponse response2 =
+        Records.newRecord(GetResourceProfileResponse.class);
+    response2.setResource(resource2);
+
+    // empty response
+    GetResourceProfileResponse response3 =
+        Records.newRecord(GetResourceProfileResponse.class);
+
+    // null response
+    GetResourceProfileResponse response4 = null;
+
+    List<GetResourceProfileResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    GetResourceProfileResponse response =
+        RouterYarnClientUtils.mergeClusterResourceProfileResponse(responses);
+    Resource resource = response.getResource();
+    Assert.assertEquals(3, resource.getVirtualCores());
+    Assert.assertEquals(3072, resource.getMemorySize());
+  }
+
+  @Test
+  public void testMergeAttributesToNodesResponse() {
+    // normal response1
+    NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+        NodeAttributeType.STRING, "nvidia");
+    Map<NodeAttributeKey, List<NodeToAttributeValue>> map1 = new HashMap<>();
+    List<NodeToAttributeValue> lists1 = new ArrayList<>();
+    NodeToAttributeValue attributeValue1 =
+        NodeToAttributeValue.newInstance("node1", gpu.getAttributeValue());
+    lists1.add(attributeValue1);
+    map1.put(gpu.getAttributeKey(), lists1);
+    GetAttributesToNodesResponse response1 = GetAttributesToNodesResponse.newInstance(map1);
+
+    // normal response2
+    NodeAttribute docker = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
+        NodeAttributeType.STRING, "docker0");
+    Map<NodeAttributeKey, List<NodeToAttributeValue>> map2 = new HashMap<>();
+    List<NodeToAttributeValue> lists2 = new ArrayList<>();
+    NodeToAttributeValue attributeValue2 =
+        NodeToAttributeValue.newInstance("node2", docker.getAttributeValue());
+    lists2.add(attributeValue2);
+    map2.put(docker.getAttributeKey(), lists2);
+    GetAttributesToNodesResponse response2 = GetAttributesToNodesResponse.newInstance(map2);
+
+    // empty response3
+    GetAttributesToNodesResponse response3 =
+        GetAttributesToNodesResponse.newInstance(new HashMap<>());
+
+    // null response4
+    GetAttributesToNodesResponse response4 = null;
+
+    List<GetAttributesToNodesResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    GetAttributesToNodesResponse response =
+        RouterYarnClientUtils.mergeAttributesToNodesResponse(responses);
+
+    Assert.assertNotNull(response);
+    Assert.assertEquals(2, response.getAttributesToNodes().size());
+
+    Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs = response.getAttributesToNodes();
+
+    NodeAttributeKey gpuKey = gpu.getAttributeKey();
+    Assert.assertEquals(attributeValue1.toString(), attrs.get(gpuKey).get(0).toString());
+
+    NodeAttributeKey dockerKey = docker.getAttributeKey();
+    Assert.assertEquals(attributeValue2.toString(), attrs.get(dockerKey).get(0).toString());
+  }
+
+  @Test
+  public void testMergeClusterNodeAttributesResponse() {
+    // normal response1
+    NodeAttributeInfo nodeAttributeInfo1 =
+        NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("GPU"),
+        NodeAttributeType.STRING);
+    Set<NodeAttributeInfo> attributes1 = new HashSet<>();
+    attributes1.add(nodeAttributeInfo1);
+    GetClusterNodeAttributesResponse response1 =
+        GetClusterNodeAttributesResponse.newInstance(attributes1);
+
+    // normal response2
+    NodeAttributeInfo nodeAttributeInfo2 =
+        NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("CPU"),
+        NodeAttributeType.STRING);
+    Set<NodeAttributeInfo> attributes2 = new HashSet<>();
+    attributes2.add(nodeAttributeInfo2);
+    GetClusterNodeAttributesResponse response2 =
+        GetClusterNodeAttributesResponse.newInstance(attributes2);
+
+    // empty response3
+    GetClusterNodeAttributesResponse response3 =
+        GetClusterNodeAttributesResponse.newInstance(new HashSet<>());
+
+    // null response4
+    GetClusterNodeAttributesResponse response4 = null;
+
+    List<GetClusterNodeAttributesResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    GetClusterNodeAttributesResponse response =
+        RouterYarnClientUtils.mergeClusterNodeAttributesResponse(responses);
+
+    Assert.assertNotNull(response);
+
+    Set<NodeAttributeInfo> nodeAttributeInfos = response.getNodeAttributes();
+    Assert.assertEquals(2, nodeAttributeInfos.size());
+    Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo1));
+    Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo2));
+  }
+
+  @Test
+  public void testMergeNodesToAttributesResponse() {
+    // normal response1
+    NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+        NodeAttributeType.STRING, "nvida");
+    NodeAttribute os = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
+        NodeAttributeType.STRING, "windows64");
+    NodeAttribute dist = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
+        NodeAttributeType.STRING, "3_0_2");
+    Map<String, Set<NodeAttribute>> node1Map = new HashMap<>();
+    node1Map.put("node1", ImmutableSet.of(gpu, os, dist));
+    GetNodesToAttributesResponse response1 = GetNodesToAttributesResponse.newInstance(node1Map);
+
+    // normal response2
+    NodeAttribute docker = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
+        NodeAttributeType.STRING, "docker0");
+    Map<String, Set<NodeAttribute>> node2Map = new HashMap<>();
+    node2Map.put("node2", ImmutableSet.of(docker));
+    GetNodesToAttributesResponse response2 = GetNodesToAttributesResponse.newInstance(node2Map);
+
+    // empty response3
+    GetNodesToAttributesResponse response3 =
+        GetNodesToAttributesResponse.newInstance(new HashMap<>());
+
+    // null response4
+    GetNodesToAttributesResponse response4 = null;
+
+    List<GetNodesToAttributesResponse> responses = new ArrayList<>();
+    responses.add(response1);
+    responses.add(response2);
+    responses.add(response3);
+    responses.add(response4);
+
+    GetNodesToAttributesResponse response =
+        RouterYarnClientUtils.mergeNodesToAttributesResponse(responses);
+
+    Assert.assertNotNull(response);
+
+    Map<String, Set<NodeAttribute>> hostToAttrs = response.getNodeToAttributes();
+    Assert.assertNotNull(hostToAttrs);
+    Assert.assertEquals(2, hostToAttrs.size());
+    Assert.assertTrue(hostToAttrs.get("node1").contains(dist));
+    Assert.assertTrue(hostToAttrs.get("node1").contains(gpu));
+    Assert.assertTrue(hostToAttrs.get("node1").contains(os));
+    Assert.assertTrue(hostToAttrs.get("node2").contains(docker));
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialBroadcastPolicyManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialBroadcastPolicyManager.java
new file mode 100644
index 0000000000000..dfa8c7136d76c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialBroadcastPolicyManager.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.clientrm;
+
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.manager.AbstractPolicyManager;
+
+/**
+ * This PolicyManager is used for testing and will contain the
+ * {@link TestSequentialRouterPolicy} policy.
+ *
+ * When we test FederationClientInterceptor Retry,
+ * we hope that SubCluster can return in a certain order, not randomly.
+ * We can view the policy description by linking to TestSequentialRouterPolicy.
+ */
+public class TestSequentialBroadcastPolicyManager extends AbstractPolicyManager {
+  public TestSequentialBroadcastPolicyManager() {
+    // this structurally hard-codes two compatible policies for Router and
+    // AMRMProxy.
+    routerFederationPolicy = TestSequentialRouterPolicy.class;
+    amrmProxyFederationPolicy = BroadcastAMRMProxyPolicy.class;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialRouterPolicy.java
new file mode 100644
index 0000000000000..e702b764fede7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestSequentialRouterPolicy.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.clientrm;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.router.AbstractRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This is a test strategy,
+ * the purpose of this strategy is to return subClusters in descending order of subClusterId.
+ *
+ * This strategy is to verify the situation of Retry during the use of FederationClientInterceptor.
+ * The conditions of use are as follows:
+ * 1.We require subClusterId to be an integer.
+ * 2.The larger the subCluster, the sooner the representative is selected.
+ *
+ * We have 4 subClusters, 2 normal subClusters, 2 bad subClusters.
+ * We expect to select badSubClusters first and then goodSubClusters during testing.
+ * We can set the subCluster like this, good1 = [0], good2 = [1], bad1 = [2], bad2 = [3].
+ * This strategy will return [3, 2, 1, 0],
+ * The selection order of subCluster is bad2, bad1, good2, good1.
+ */
+public class TestSequentialRouterPolicy extends AbstractRouterPolicy {
+
+  @Override
+  public void reinitialize(FederationPolicyInitializationContext policyContext)
+      throws FederationPolicyInitializationException {
+    FederationPolicyInitializationContextValidator.validate(policyContext,
+        this.getClass().getCanonicalName());
+    setPolicyContext(policyContext);
+  }
+
+  @Override
+  protected SubClusterId chooseSubCluster(String queue,
+      Map<SubClusterId, SubClusterInfo> preSelectSubClusters) throws YarnException {
+    /**
+      * This strategy is only suitable for testing. We need to obtain subClusters sequentially.
+      * We have 3 subClusters, 1 goodSubCluster and 2 badSubClusters.
+      * The sc-id of goodSubCluster is 0, and the sc-id of badSubCluster is 1 and 2.
+      * We hope Return in reverse order, that is, return 2, 1, 0
+      * Return to badCluster first.
+      */
+    List<SubClusterId> subClusterIds = new ArrayList<>(preSelectSubClusters.keySet());
+    if (subClusterIds.size() > 1) {
+      subClusterIds.sort((o1, o2) -> Integer.parseInt(o2.getId()) - Integer.parseInt(o1.getId()));
+    }
+    if(CollectionUtils.isNotEmpty(subClusterIds)){
+      return subClusterIds.get(0);
+    }
+    return null;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java
index c97d05324259b..c8c647a0d22e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java
@@ -24,22 +24,41 @@
 import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Extends the FederationClientInterceptor and overrides methods to provide a
@@ -48,9 +67,15 @@
 public class TestableFederationClientInterceptor
     extends FederationClientInterceptor {
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestableFederationClientInterceptor.class);
+
   private ConcurrentHashMap<SubClusterId, MockRM> mockRMs =
       new ConcurrentHashMap<>();
 
+  private ConcurrentHashMap<SubClusterId, MockNM> mockNMs =
+      new ConcurrentHashMap<>();
+
   private List<SubClusterId> badSubCluster = new ArrayList<SubClusterId>();
 
   @Override
@@ -71,12 +96,15 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster(
         mockRM.init(super.getConf());
         mockRM.start();
         try {
-          mockRM.registerNode("h1:1234", 1024);
+          MockNM nm = mockRM.registerNode("127.0.0.1:1234", 8*1024, 4);
+          mockNMs.put(subClusterId, nm);
         } catch (Exception e) {
           Assert.fail(e.getMessage());
         }
         mockRMs.put(subClusterId, mockRM);
       }
+      initNodeAttributes(subClusterId, mockRM);
+      initReservationSystem(mockRM);
       return mockRM.getClientRMService();
     }
   }
@@ -115,4 +143,105 @@ protected void registerBadSubCluster(SubClusterId badSC) throws IOException {
     }
   }
 
-}
+  public ConcurrentHashMap<SubClusterId, MockRM> getMockRMs() {
+    return mockRMs;
+  }
+
+  public ConcurrentHashMap<SubClusterId, MockNM> getMockNMs() {
+    return mockNMs;
+  }
+
+  private void initNodeAttributes(SubClusterId subClusterId, MockRM mockRM)  {
+    String node1 = subClusterId.getId() +"-host1";
+    String node2 = subClusterId.getId() +"-host2";
+    NodeAttributesManager mgr = mockRM.getRMContext().getNodeAttributesManager();
+    NodeAttribute gpu =
+        NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+        NodeAttributeType.STRING, "nvidia");
+    NodeAttribute os =
+        NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
+        NodeAttributeType.STRING, "windows64");
+    NodeAttribute docker =
+        NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
+        NodeAttributeType.STRING, "docker0");
+    NodeAttribute dist =
+        NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
+        NodeAttributeType.STRING, "3_0_2");
+    Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
+    nodes.put(node1, ImmutableSet.of(gpu, os, dist));
+    nodes.put(node2, ImmutableSet.of(docker, dist));
+    try {
+      mgr.addNodeAttributes(nodes);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void initReservationSystem(MockRM mockRM) throws YarnException {
+    try {
+      // Ensure that the reserved resources of the RM#Reservation System are allocated
+      String planName = "root.decided";
+      ReservationSystem reservationSystem = mockRM.getReservationSystem();
+      reservationSystem.synchronizePlan(planName, true);
+
+      GenericTestUtils.waitFor(() -> {
+        Plan plan = reservationSystem.getPlan(planName);
+        Resource resource = plan.getTotalCapacity();
+        return (resource.getMemorySize() > 0 && resource.getVirtualCores() > 0);
+      }, 100, 2000);
+    } catch (TimeoutException | InterruptedException e) {
+      throw new YarnException(e);
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    if (mockRMs != null && !mockRMs.isEmpty()) {
+      for (Map.Entry<SubClusterId, MockRM> item : mockRMs.entrySet()) {
+        SubClusterId subClusterId = item.getKey();
+
+        // close mockNM
+        MockNM mockNM = mockNMs.getOrDefault(subClusterId, null);
+        try {
+          mockNM.unRegisterNode();
+          mockNM = null;
+        } catch (Exception e) {
+          LOG.error("mockNM unRegisterNode error.", e);
+        }
+
+        // close mockRM
+        MockRM mockRM = item.getValue();
+        if (mockRM != null) {
+          mockRM.stop();
+        }
+      }
+    }
+    mockNMs.clear();
+    mockRMs.clear();
+    super.shutdown();
+  }
+
+  public RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecretManager(
+      Configuration conf) {
+
+    long secretKeyInterval = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+
+    long tokenMaxLifetime = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+
+    long tokenRenewInterval = conf.getLong(
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+
+    long removeScanInterval = conf.getTimeDuration(
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
+        YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
+        TimeUnit.MILLISECONDS);
+
+    return new RouterDelegationTokenSecretManager(secretKeyInterval,
+        tokenMaxLifetime, tokenRenewInterval, removeScanInterval);
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java
index d3eba6180282f..33cda8751db60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.Executors;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -87,25 +88,36 @@ protected MockRouterRMAdminService getRouterRMAdminService() {
 
   @Before
   public void setUp() {
-    this.conf = new YarnConfiguration();
+    this.conf = createConfiguration();
+    this.dispatcher = new AsyncDispatcher();
+    this.dispatcher.init(conf);
+    this.dispatcher.start();
+    this.rmAdminService = createAndStartRouterRMAdminService();
+    DefaultMetricsSystem.setMiniClusterMode(true);
+  }
+
+  protected Configuration getConf() {
+    return this.conf;
+  }
+
+  public void setUpConfig() {
+    this.conf = createConfiguration();
+  }
+
+  protected Configuration createConfiguration() {
+    YarnConfiguration config = new YarnConfiguration();
     String mockPassThroughInterceptorClass =
         PassThroughRMAdminRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
+    // Create a request interceptor pipeline for testing. The last one in the
     // chain will call the mock resource manager. The others in the chain will
     // simply forward it to the next one in the chain
-    this.conf.set(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE,
-        mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass
-            + "," + mockPassThroughInterceptorClass + ","
-            + MockRMAdminRequestInterceptor.class.getName());
-
-    this.conf.setInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE,
-        TEST_MAX_CACHE_SIZE);
+    config.set(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE,
+        mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass + "," +
+        mockPassThroughInterceptorClass + "," + MockRMAdminRequestInterceptor.class.getName());
 
-    this.dispatcher = new AsyncDispatcher();
-    this.dispatcher.init(conf);
-    this.dispatcher.start();
-    this.rmAdminService = createAndStartRouterRMAdminService();
+    config.setInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, TEST_MAX_CACHE_SIZE);
+    return config;
   }
 
   @After
@@ -139,194 +151,154 @@ public MockRouterRMAdminService() {
   protected RefreshQueuesResponse refreshQueues(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<RefreshQueuesResponse>() {
-          @Override
-          public RefreshQueuesResponse run() throws Exception {
-            RefreshQueuesRequest req = RefreshQueuesRequest.newInstance();
-            RefreshQueuesResponse response =
-                getRouterRMAdminService().refreshQueues(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<RefreshQueuesResponse>) () -> {
+          RefreshQueuesRequest req = RefreshQueuesRequest.newInstance();
+          RefreshQueuesResponse response =
+              getRouterRMAdminService().refreshQueues(req);
+          return response;
         });
   }
 
   protected RefreshNodesResponse refreshNodes(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<RefreshNodesResponse>() {
-          @Override
-          public RefreshNodesResponse run() throws Exception {
-            RefreshNodesRequest req = RefreshNodesRequest.newInstance();
-            RefreshNodesResponse response =
-                getRouterRMAdminService().refreshNodes(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<RefreshNodesResponse>) () -> {
+          RefreshNodesRequest req = RefreshNodesRequest.newInstance();
+          RefreshNodesResponse response =
+              getRouterRMAdminService().refreshNodes(req);
+          return response;
         });
   }
 
   protected RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
       String user) throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user).doAs(
-        new PrivilegedExceptionAction<RefreshSuperUserGroupsConfigurationResponse>() {
-          @Override
-          public RefreshSuperUserGroupsConfigurationResponse run()
-              throws Exception {
-            RefreshSuperUserGroupsConfigurationRequest req =
-                RefreshSuperUserGroupsConfigurationRequest.newInstance();
-            RefreshSuperUserGroupsConfigurationResponse response =
-                getRouterRMAdminService()
-                    .refreshSuperUserGroupsConfiguration(req);
-            return response;
-          }
+        (PrivilegedExceptionAction<RefreshSuperUserGroupsConfigurationResponse>) () -> {
+          RefreshSuperUserGroupsConfigurationRequest req =
+              RefreshSuperUserGroupsConfigurationRequest.newInstance();
+          RefreshSuperUserGroupsConfigurationResponse response =
+              getRouterRMAdminService()
+                  .refreshSuperUserGroupsConfiguration(req);
+          return response;
         });
   }
 
   protected RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
       String user) throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user).doAs(
-        new PrivilegedExceptionAction<RefreshUserToGroupsMappingsResponse>() {
-          @Override
-          public RefreshUserToGroupsMappingsResponse run() throws Exception {
-            RefreshUserToGroupsMappingsRequest req =
-                RefreshUserToGroupsMappingsRequest.newInstance();
-            RefreshUserToGroupsMappingsResponse response =
-                getRouterRMAdminService().refreshUserToGroupsMappings(req);
-            return response;
-          }
+        (PrivilegedExceptionAction<RefreshUserToGroupsMappingsResponse>) () -> {
+          RefreshUserToGroupsMappingsRequest req =
+              RefreshUserToGroupsMappingsRequest.newInstance();
+          RefreshUserToGroupsMappingsResponse response =
+              getRouterRMAdminService().refreshUserToGroupsMappings(req);
+          return response;
         });
   }
 
   protected RefreshAdminAclsResponse refreshAdminAcls(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<RefreshAdminAclsResponse>() {
-          @Override
-          public RefreshAdminAclsResponse run() throws Exception {
-            RefreshAdminAclsRequest req = RefreshAdminAclsRequest.newInstance();
-            RefreshAdminAclsResponse response =
-                getRouterRMAdminService().refreshAdminAcls(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<RefreshAdminAclsResponse>) () -> {
+          RefreshAdminAclsRequest req = RefreshAdminAclsRequest.newInstance();
+          RefreshAdminAclsResponse response =
+              getRouterRMAdminService().refreshAdminAcls(req);
+          return response;
         });
   }
 
   protected RefreshServiceAclsResponse refreshServiceAcls(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<RefreshServiceAclsResponse>() {
-          @Override
-          public RefreshServiceAclsResponse run() throws Exception {
-            RefreshServiceAclsRequest req =
-                RefreshServiceAclsRequest.newInstance();
-            RefreshServiceAclsResponse response =
-                getRouterRMAdminService().refreshServiceAcls(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<RefreshServiceAclsResponse>) () -> {
+          RefreshServiceAclsRequest req =
+              RefreshServiceAclsRequest.newInstance();
+          RefreshServiceAclsResponse response =
+              getRouterRMAdminService().refreshServiceAcls(req);
+          return response;
         });
   }
 
   protected UpdateNodeResourceResponse updateNodeResource(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<UpdateNodeResourceResponse>() {
-          @Override
-          public UpdateNodeResourceResponse run() throws Exception {
-            UpdateNodeResourceRequest req =
-                UpdateNodeResourceRequest.newInstance(null);
-            UpdateNodeResourceResponse response =
-                getRouterRMAdminService().updateNodeResource(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<UpdateNodeResourceResponse>) () -> {
+          UpdateNodeResourceRequest req =
+              UpdateNodeResourceRequest.newInstance(null);
+          UpdateNodeResourceResponse response =
+              getRouterRMAdminService().updateNodeResource(req);
+          return response;
         });
   }
 
   protected RefreshNodesResourcesResponse refreshNodesResources(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<RefreshNodesResourcesResponse>() {
-          @Override
-          public RefreshNodesResourcesResponse run() throws Exception {
-            RefreshNodesResourcesRequest req =
-                RefreshNodesResourcesRequest.newInstance();
-            RefreshNodesResourcesResponse response =
-                getRouterRMAdminService().refreshNodesResources(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<RefreshNodesResourcesResponse>) () -> {
+          RefreshNodesResourcesRequest req =
+              RefreshNodesResourcesRequest.newInstance();
+          RefreshNodesResourcesResponse response =
+              getRouterRMAdminService().refreshNodesResources(req);
+          return response;
         });
   }
 
   protected AddToClusterNodeLabelsResponse addToClusterNodeLabels(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<AddToClusterNodeLabelsResponse>() {
-          @Override
-          public AddToClusterNodeLabelsResponse run() throws Exception {
-            AddToClusterNodeLabelsRequest req =
-                AddToClusterNodeLabelsRequest.newInstance(null);
-            AddToClusterNodeLabelsResponse response =
-                getRouterRMAdminService().addToClusterNodeLabels(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<AddToClusterNodeLabelsResponse>) () -> {
+          AddToClusterNodeLabelsRequest req =
+              AddToClusterNodeLabelsRequest.newInstance(null);
+          AddToClusterNodeLabelsResponse response =
+              getRouterRMAdminService().addToClusterNodeLabels(req);
+          return response;
         });
   }
 
   protected RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels(
       String user) throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user).doAs(
-        new PrivilegedExceptionAction<RemoveFromClusterNodeLabelsResponse>() {
-          @Override
-          public RemoveFromClusterNodeLabelsResponse run() throws Exception {
-            RemoveFromClusterNodeLabelsRequest req =
-                RemoveFromClusterNodeLabelsRequest.newInstance(null);
-            RemoveFromClusterNodeLabelsResponse response =
-                getRouterRMAdminService().removeFromClusterNodeLabels(req);
-            return response;
-          }
+        (PrivilegedExceptionAction<RemoveFromClusterNodeLabelsResponse>) () -> {
+          RemoveFromClusterNodeLabelsRequest req =
+              RemoveFromClusterNodeLabelsRequest.newInstance(null);
+          RemoveFromClusterNodeLabelsResponse response =
+              getRouterRMAdminService().removeFromClusterNodeLabels(req);
+          return response;
         });
   }
 
   protected ReplaceLabelsOnNodeResponse replaceLabelsOnNode(String user)
       throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user)
-        .doAs(new PrivilegedExceptionAction<ReplaceLabelsOnNodeResponse>() {
-          @Override
-          public ReplaceLabelsOnNodeResponse run() throws Exception {
-            ReplaceLabelsOnNodeRequest req = ReplaceLabelsOnNodeRequest
-                .newInstance(new HashMap<NodeId, Set<String>>());
-            ReplaceLabelsOnNodeResponse response =
-                getRouterRMAdminService().replaceLabelsOnNode(req);
-            return response;
-          }
+        .doAs((PrivilegedExceptionAction<ReplaceLabelsOnNodeResponse>) () -> {
+          ReplaceLabelsOnNodeRequest req = ReplaceLabelsOnNodeRequest
+              .newInstance(new HashMap<NodeId, Set<String>>());
+          ReplaceLabelsOnNodeResponse response =
+              getRouterRMAdminService().replaceLabelsOnNode(req);
+          return response;
         });
   }
 
   protected CheckForDecommissioningNodesResponse checkForDecommissioningNodes(
       String user) throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user).doAs(
-        new PrivilegedExceptionAction<CheckForDecommissioningNodesResponse>() {
-          @Override
-          public CheckForDecommissioningNodesResponse run() throws Exception {
-            CheckForDecommissioningNodesRequest req =
-                CheckForDecommissioningNodesRequest.newInstance();
-            CheckForDecommissioningNodesResponse response =
-                getRouterRMAdminService().checkForDecommissioningNodes(req);
-            return response;
-          }
+        (PrivilegedExceptionAction<CheckForDecommissioningNodesResponse>) () -> {
+          CheckForDecommissioningNodesRequest req =
+              CheckForDecommissioningNodesRequest.newInstance();
+          CheckForDecommissioningNodesResponse response =
+              getRouterRMAdminService().checkForDecommissioningNodes(req);
+          return response;
         });
   }
 
   protected RefreshClusterMaxPriorityResponse refreshClusterMaxPriority(
       String user) throws IOException, InterruptedException {
     return UserGroupInformation.createRemoteUser(user).doAs(
-        new PrivilegedExceptionAction<RefreshClusterMaxPriorityResponse>() {
-          @Override
-          public RefreshClusterMaxPriorityResponse run() throws Exception {
-            RefreshClusterMaxPriorityRequest req =
-                RefreshClusterMaxPriorityRequest.newInstance();
-            RefreshClusterMaxPriorityResponse response =
-                getRouterRMAdminService().refreshClusterMaxPriority(req);
-            return response;
-          }
+        (PrivilegedExceptionAction<RefreshClusterMaxPriorityResponse>) () -> {
+          RefreshClusterMaxPriorityRequest req =
+              RefreshClusterMaxPriorityRequest.newInstance();
+          RefreshClusterMaxPriorityResponse response =
+              getRouterRMAdminService().refreshClusterMaxPriority(req);
+          return response;
         });
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java
index ee55b63ac40f9..a1a366f7664f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java
@@ -27,13 +27,14 @@
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 
 /**
- * This class mocks the RMAmdinRequestInterceptor.
+ * This class mocks the RMAdminRequestInterceptor.
  */
 public class MockRMAdminRequestInterceptor
     extends DefaultRMAdminRequestInterceptor {
 
+  MockRM mockRM = null;
   public void init(String user) {
-    MockRM mockRM = new MockRM(super.getConf()) {
+    mockRM = new MockRM(super.getConf()) {
       @Override
       protected AdminService createAdminService() {
         return new AdminService(this) {
@@ -60,4 +61,12 @@ protected void stopServer() {
     mockRM.start();
     super.setRMAdmin(mockRM.getAdminService());
   }
+
+  @Override
+  public void shutdown() {
+    if (mockRM != null) {
+      mockRM.stop();
+    }
+    super.shutdown();
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
index 4d6a3eaf52b4a..e3be25009fc18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
@@ -52,8 +52,8 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
 
 /**
- * Mock intercepter that does not do anything other than forwarding it to the
- * next intercepter in the chain.
+ * Mock interceptor that does not do anything other than forwarding it to the
+ * next interceptor in the chain.
  */
 public class PassThroughRMAdminRequestInterceptor
     extends AbstractRMAdminRequestInterceptor {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java
new file mode 100644
index 0000000000000..3aa61a68a390d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestFederationRMAdminInterceptor.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.rmadmin;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Extends the FederationRMAdminInterceptor and overrides methods to provide a
+ * testable implementation of FederationRMAdminInterceptor.
+ */
+public class TestFederationRMAdminInterceptor extends BaseRouterRMAdminTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFederationRMAdminInterceptor.class);
+
+  ////////////////////////////////
+  // constant information
+  ////////////////////////////////
+  private final static String USER_NAME = "test-user";
+  private final static int NUM_SUBCLUSTER = 4;
+
+  private TestableFederationRMAdminInterceptor interceptor;
+  private FederationStateStoreFacade facade;
+  private MemoryFederationStateStore stateStore;
+  private FederationStateStoreTestUtil stateStoreUtil;
+  private List<SubClusterId> subClusters;
+
+  @Override
+  public void setUp() {
+
+    super.setUpConfig();
+
+    // Initialize facade & stateSore
+    stateStore = new MemoryFederationStateStore();
+    stateStore.init(this.getConf());
+    facade = FederationStateStoreFacade.getInstance();
+    facade.reinitialize(stateStore, getConf());
+    stateStoreUtil = new FederationStateStoreTestUtil(stateStore);
+
+    // Initialize interceptor
+    interceptor = new TestableFederationRMAdminInterceptor();
+    interceptor.setConf(this.getConf());
+    interceptor.init(USER_NAME);
+
+    // Storage SubClusters
+    subClusters = new ArrayList<>();
+    try {
+      for (int i = 0; i < NUM_SUBCLUSTER; i++) {
+        SubClusterId sc = SubClusterId.newInstance(Integer.toString(i));
+        stateStoreUtil.registerSubCluster(sc);
+        subClusters.add(sc);
+      }
+    } catch (YarnException e) {
+      LOG.error(e.getMessage());
+      Assert.fail();
+    }
+
+    DefaultMetricsSystem.setMiniClusterMode(true);
+  }
+
+  @Override
+  protected YarnConfiguration createConfiguration() {
+    // Set Enable YarnFederation
+    YarnConfiguration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+
+    String mockPassThroughInterceptorClass =
+        PassThroughRMAdminRequestInterceptor.class.getName();
+
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain will call the mock resource manager. The others in the chain will
+    // simply forward it to the next one in the chain
+    config.set(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE,
+        mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass + "," +
+        TestFederationRMAdminInterceptor.class.getName());
+    return config;
+  }
+
+  @Override
+  public void tearDown() {
+    interceptor.shutdown();
+    super.tearDown();
+  }
+
+  @Test
+  public void testRefreshQueues() throws IOException, YarnException {
+    RefreshQueuesRequest request = RefreshQueuesRequest.newInstance();
+    interceptor.refreshQueues(request);
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java
index 07ef73c3cdb85..867c71fa82e54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java
@@ -23,7 +23,6 @@
 import java.util.Map;
 
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
@@ -185,7 +184,7 @@ public void testRouterRMAdminServiceE2E() throws Exception {
    */
   @Test
   public void testUsersChainMapWithLRUCache()
-      throws YarnException, IOException, InterruptedException {
+      throws IOException, InterruptedException {
 
     Map<String, RequestInterceptorChainWrapper> pipelines;
     RequestInterceptorChainWrapper chain;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestableFederationRMAdminInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestableFederationRMAdminInterceptor.java
new file mode 100644
index 0000000000000..26f50f88b898d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestableFederationRMAdminInterceptor.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router.rmadmin;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class TestableFederationRMAdminInterceptor extends FederationRMAdminInterceptor {
+
+  // Record log information
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestableFederationRMAdminInterceptor.class);
+
+  // Used to Store the relationship between SubClusterId and RM
+  private ConcurrentHashMap<SubClusterId, MockRM> mockRMs = new ConcurrentHashMap<>();
+
+  // Store Bad subCluster
+  private Set<SubClusterId> badSubCluster = new HashSet<>();
+
+  @Override
+  protected ResourceManagerAdministrationProtocol getAdminRMProxyForSubCluster(
+      SubClusterId subClusterId) throws YarnException {
+    MockRM mockRM;
+    synchronized (this) {
+      if (mockRMs.containsKey(subClusterId)) {
+        mockRM = mockRMs.get(subClusterId);
+      } else {
+        mockRM = new MockRM();
+        if (badSubCluster.contains(subClusterId)) {
+          return new MockRMAdminBadService(mockRM);
+        }
+        mockRM.init(super.getConf());
+        mockRM.start();
+        mockRMs.put(subClusterId, mockRM);
+      }
+      return mockRM.getAdminService();
+    }
+  }
+
+  // This represents an unserviceable SubCluster
+  private class MockRMAdminBadService extends AdminService {
+    MockRMAdminBadService(ResourceManager rm) {
+      super(rm);
+    }
+
+    @Override
+    public void refreshQueues() throws IOException, YarnException {
+      throw new ConnectException("RM is stopped");
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    // if mockRMs is not null
+    if (MapUtils.isNotEmpty(mockRMs)) {
+      for (Map.Entry<SubClusterId, MockRM> item : mockRMs.entrySet()) {
+        SubClusterId subClusterId = item.getKey();
+        // close mockRM.
+        MockRM mockRM = item.getValue();
+        if (mockRM != null) {
+          LOG.info("subClusterId = {} mockRM shutdown.", subClusterId);
+          mockRM.stop();
+        }
+      }
+    }
+    mockRMs.clear();
+    super.shutdown();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java
new file mode 100644
index 0000000000000..062d732e8738b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.secure;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+
+public abstract class AbstractSecureRouterTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractSecureRouterTest.class);
+
+  ////////////////////////////////
+  // Kerberos Constants
+  ////////////////////////////////
+
+  public static final String REALM = "EXAMPLE.COM";
+  public static final String ROUTER = "router";
+  public static final String LOCALHOST = "localhost";
+  public static final String IP127001 = "127.0.0.1";
+  public static final String ROUTER_LOCALHOST = "router/" + LOCALHOST;
+  public static final String ROUTER_127001 = "router/" + IP127001;
+  public static final String ROUTER_REALM = "router@" + REALM;
+  public static final String ROUTER_LOCALHOST_REALM = ROUTER_LOCALHOST + "@" + REALM;
+  public static final String SUN_SECURITY_KRB5_DEBUG = "sun.security.krb5.debug";
+  public static final String KERBEROS = "kerberos";
+
+  ////////////////////////////////
+  // BeforeSecureRouterTestClass Init
+  ////////////////////////////////
+
+  private static MiniKdc kdc;
+  private static File routerKeytab;
+  private static File kdcWorkDir;
+  private static Configuration conf;
+
+  ////////////////////////////////
+  // Specific Constant
+  // Like Mem, VCore, ClusterNum
+  ////////////////////////////////
+  private static final int NUM_SUBCLUSTER = 4;
+  private static final int GB = 1024;
+  private static final int NM_MEMORY = 8 * GB;
+  private static final int NM_VCORE = 4;
+
+  ////////////////////////////////
+  // Test use in subclasses
+  ////////////////////////////////
+
+  private Router router = null;
+
+  private static ConcurrentHashMap<SubClusterId, MockRM> mockRMs =
+      new ConcurrentHashMap<>();
+
+  @BeforeClass
+  public static void beforeSecureRouterTestClass() throws Exception {
+    // Sets up the KDC and Principals.
+    setupKDCAndPrincipals();
+
+    // Init YarnConfiguration
+    conf = new YarnConfiguration();
+
+    // Enable Kerberos authentication configuration
+    conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true);
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, KERBEROS);
+
+    // Router configuration
+    conf.set(YarnConfiguration.ROUTER_BIND_HOST, "0.0.0.0");
+    conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
+        FederationClientInterceptor.class.getName());
+
+    // Router Kerberos KeyTab configuration
+    conf.set(YarnConfiguration.ROUTER_PRINCIPAL, ROUTER_LOCALHOST_REALM);
+    conf.set(YarnConfiguration.ROUTER_KEYTAB, routerKeytab.getAbsolutePath());
+
+    DefaultMetricsSystem.setMiniClusterMode(true);
+  }
+
+  /**
+   * Sets up the KDC and Principals.
+   *
+   * @throws Exception an error occurred.
+   */
+  public static void setupKDCAndPrincipals() throws Exception {
+    // set up the KDC
+    File target = new File(System.getProperty("test.dir", "target"));
+    kdcWorkDir = new File(target, "kdc");
+    kdcWorkDir.mkdirs();
+    if (!kdcWorkDir.mkdirs()) {
+      assertTrue(kdcWorkDir.isDirectory());
+    }
+    Properties kdcConf = MiniKdc.createConf();
+    kdcConf.setProperty(MiniKdc.DEBUG, "true");
+    kdc = new MiniKdc(kdcConf, kdcWorkDir);
+    kdc.start();
+    routerKeytab = createKeytab(ROUTER, "router.keytab");
+  }
+
+  /**
+   * Initialize RM in safe mode.
+   *
+   * @throws Exception an error occurred.
+   */
+  public static void setupSecureMockRM() throws Exception {
+    for (int i = 0; i < NUM_SUBCLUSTER; i++) {
+      SubClusterId sc = SubClusterId.newInstance(i);
+      if (mockRMs.containsKey(sc)) {
+        continue;
+      }
+      MockRM mockRM = new TestRMRestart.TestSecurityMockRM(conf);
+      mockRM.start();
+      mockRM.registerNode("127.0.0.1:1234", NM_MEMORY, NM_VCORE);
+      mockRMs.put(sc, mockRM);
+    }
+  }
+
+  /**
+   * Create the keytab for the given principal, includes
+   * raw principal and $principal/localhost.
+   *
+   * @param principal principal short name.
+   * @param filename filename of keytab.
+   * @return file of keytab.
+   * @throws Exception an error occurred.
+   */
+  public static File createKeytab(String principal, String filename) throws Exception {
+    assertTrue("empty principal", StringUtils.isNotBlank(principal));
+    assertTrue("empty host", StringUtils.isNotBlank(filename));
+    assertNotNull("null KDC", kdc);
+    File keytab = new File(kdcWorkDir, filename);
+    kdc.createPrincipal(keytab,
+        principal,
+        principal + "/localhost",
+        principal + "/127.0.0.1");
+    return keytab;
+  }
+
+  /**
+   * Start the router in safe mode.
+   *
+   * @throws Exception an error occurred.
+   */
+  public synchronized void startSecureRouter() {
+    assertNull("Router is already running", router);
+    MemoryFederationStateStore stateStore = new MemoryFederationStateStore();
+    stateStore.init(getConf());
+    FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf());
+    UserGroupInformation.setConfiguration(conf);
+    router = new Router();
+    router.init(conf);
+    router.start();
+  }
+
+  /**
+   * Shut down the KDC service.
+   *
+   * @throws Exception an error occurred.
+   */
+  public static void teardownKDC() throws Exception {
+    if (kdc != null) {
+      kdc.stop();
+      kdc = null;
+    }
+  }
+
+  /**
+   * Stop the router in safe mode.
+   *
+   * @throws Exception an error occurred.
+   */
+  protected synchronized void stopSecureRouter() throws Exception {
+    if (router != null) {
+      router.stop();
+      router = null;
+    }
+  }
+
+  /**
+   * Stop the entire test service.
+   *
+   * @throws Exception an error occurred.
+   */
+  @AfterClass
+  public static void afterSecureRouterTest() throws Exception {
+    LOG.info("teardown of kdc instance.");
+    teardownKDC();
+    if (mockRMs != null && mockRMs.isEmpty()) {
+      for (MockRM mockRM : mockRMs.values()) {
+        mockRM.close();
+      }
+    }
+    mockRMs.clear();
+  }
+
+  public static MiniKdc getKdc() {
+    return kdc;
+  }
+
+  public Router getRouter() {
+    return router;
+  }
+
+  public static ConcurrentHashMap<SubClusterId, MockRM> getMockRMs() {
+    return mockRMs;
+  }
+
+  public static Configuration getConf() {
+    return conf;
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java
new file mode 100644
index 0000000000000..eac2c5a03ba61
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.secure;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService;
+import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class);
+
+  @Test
+  public void testRouterStoreNewMasterKey() throws Exception {
+    LOG.info("Test RouterDelegationTokenSecretManager: StoreNewMasterKey.");
+
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Store NewMasterKey
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    RouterDelegationTokenSecretManager secretManager =
+        routerClientRMService.getRouterDTSecretManager();
+    DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    secretManager.storeNewMasterKey(storeKey);
+
+    // Get DelegationKey
+    DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    DelegationKey responseKey = secretManager.getMasterKeyByDelegationKey(paramKey);
+
+    assertNotNull(paramKey);
+    assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate());
+    assertEquals(storeKey.getKeyId(), responseKey.getKeyId());
+    assertArrayEquals(storeKey.getEncodedKey(), responseKey.getEncodedKey());
+    assertEquals(storeKey, responseKey);
+
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterRemoveStoredMasterKey() throws Exception {
+    LOG.info("Test RouterDelegationTokenSecretManager: RemoveStoredMasterKey.");
+
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Store NewMasterKey
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    RouterDelegationTokenSecretManager secretManager =
+        routerClientRMService.getRouterDTSecretManager();
+    DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    secretManager.storeNewMasterKey(storeKey);
+
+    // Remove DelegationKey
+    secretManager.removeStoredMasterKey(storeKey);
+
+    // Get DelegationKey
+    DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes());
+    LambdaTestUtils.intercept(IOException.class,
+        "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.",
+        () -> secretManager.getMasterKeyByDelegationKey(paramKey));
+
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterStoreNewToken() throws Exception {
+    LOG.info("Test RouterDelegationTokenSecretManager: StoreNewToken.");
+
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Store new rm-token
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    RouterDelegationTokenSecretManager secretManager =
+        routerClientRMService.getRouterDTSecretManager();
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    secretManager.storeNewToken(dtId1, renewDate1);
+
+    // query rm-token
+    RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    dtId2.setSequenceNumber(sequenceNumber);
+    RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2);
+    Assert.assertEquals(dtId1, dtId3);
+
+    // query rm-token2 not exists
+    sequenceNumber++;
+    dtId2.setSequenceNumber(2);
+    LambdaTestUtils.intercept(YarnException.class,
+        "RMDelegationToken: " + dtId2 + " does not exist.",
+        () -> secretManager.getTokenByRouterStoreToken(dtId2));
+
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterUpdateNewToken() throws Exception {
+    LOG.info("Test RouterDelegationTokenSecretManager: UpdateNewToken.");
+
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Store new rm-token
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    RouterDelegationTokenSecretManager secretManager =
+        routerClientRMService.getRouterDTSecretManager();
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    secretManager.storeNewToken(dtId1, renewDate1);
+
+    sequenceNumber++;
+    dtId1.setSequenceNumber(sequenceNumber);
+    secretManager.updateStoredToken(dtId1, renewDate1);
+
+    // query rm-token
+    RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    dtId2.setSequenceNumber(sequenceNumber);
+    RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2);
+    assertNotNull(dtId3);
+    assertEquals(dtId1.getKind(), dtId3.getKind());
+    assertEquals(dtId1.getOwner(), dtId3.getOwner());
+    assertEquals(dtId1.getRealUser(), dtId3.getRealUser());
+    assertEquals(dtId1.getRenewer(), dtId3.getRenewer());
+    assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate());
+    assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId());
+    assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber());
+    assertEquals(sequenceNumber, dtId3.getSequenceNumber());
+    assertEquals(dtId1, dtId3);
+
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterRemoveToken() throws Exception {
+    LOG.info("Test RouterDelegationTokenSecretManager: RouterRemoveToken.");
+
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Store new rm-token
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    RouterDelegationTokenSecretManager secretManager =
+        routerClientRMService.getRouterDTSecretManager();
+    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(
+        new Text("owner1"), new Text("renewer1"), new Text("realuser1"));
+    int sequenceNumber = 1;
+    dtId1.setSequenceNumber(sequenceNumber);
+    Long renewDate1 = Time.now();
+    secretManager.storeNewToken(dtId1, renewDate1);
+
+    // Remove rm-token
+    secretManager.removeStoredToken(dtId1);
+
+    // query rm-token
+    LambdaTestUtils.intercept(YarnException.class,
+        "RMDelegationToken: " + dtId1 + " does not exist.",
+        () -> secretManager.getTokenByRouterStoreToken(dtId1));
+
+    stopSecureRouter();
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java
new file mode 100644
index 0000000000000..40911814c044c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.secure;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor;
+import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService;
+import org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor;
+import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class TestSecureLogins extends AbstractSecureRouterTest {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestSecureLogins.class);
+
+  @Test
+  public void testHasRealm() throws Throwable {
+    Assert.assertNotNull(getRealm());
+    LOG.info("Router principal = {}", getPrincipalAndRealm(ROUTER_LOCALHOST));
+  }
+
+  @Test
+  public void testRouterSecureLogin() throws Exception {
+    startSecureRouter();
+
+    List<Service> services = this.getRouter().getServices();
+    Assert.assertNotNull(services);
+    Assert.assertEquals(3, services.size());
+
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterClientRMService() throws Exception {
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Start RM and RouterClientRMService in Secure mode
+    setupSecureMockRM();
+    initRouterClientRMService();
+
+    // Test the simple rpc call of the Router in the Secure environment
+    RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService();
+    GetClusterMetricsRequest metricsRequest = GetClusterMetricsRequest.newInstance();
+    GetClusterMetricsResponse metricsResponse =
+        routerClientRMService.getClusterMetrics(metricsRequest);
+    Assert.assertNotNull(metricsResponse);
+    YarnClusterMetrics clusterMetrics = metricsResponse.getClusterMetrics();
+    Assert.assertEquals(4, clusterMetrics.getNumNodeManagers());
+    Assert.assertEquals(0, clusterMetrics.getNumLostNodeManagers());
+
+    // Stop the Router in Secure Mode
+    stopSecureRouter();
+  }
+
+  @Test
+  public void testRouterRMAdminService() throws Exception {
+    // Start the Router in Secure Mode
+    startSecureRouter();
+
+    // Start RM and RouterClientRMService in Secure mode
+    setupSecureMockRM();
+    initRouterRMAdminService();
+
+    // Test the simple rpc call of the Router in the Secure environment
+    RouterRMAdminService routerRMAdminService = this.getRouter().getRmAdminProxyService();
+    RefreshNodesRequest refreshNodesRequest = RefreshNodesRequest.newInstance();
+    RefreshNodesResponse refreshNodesResponse =
+        routerRMAdminService.refreshNodes(refreshNodesRequest);
+    Assert.assertNotNull(refreshNodesResponse);
+
+    // Stop the Router in Secure Mode
+    stopSecureRouter();
+  }
+
+  public static String getPrincipalAndRealm(String principal) {
+    return principal + "@" + getRealm();
+  }
+
+  protected static String getRealm() {
+    return getKdc().getRealm();
+  }
+
+  private void initRouterClientRMService() throws Exception {
+    Router router = this.getRouter();
+    Map<SubClusterId, MockRM> mockRMs = getMockRMs();
+
+    RouterClientRMService rmService = router.getClientRMProxyService();
+    RouterClientRMService.RequestInterceptorChainWrapper wrapper = rmService.getInterceptorChain();
+    FederationClientInterceptor interceptor =
+        (FederationClientInterceptor) wrapper.getRootInterceptor();
+    FederationStateStoreFacade stateStoreFacade = interceptor.getFederationFacade();
+    FederationStateStore stateStore = stateStoreFacade.getStateStore();
+    FederationStateStoreTestUtil stateStoreUtil = new FederationStateStoreTestUtil(stateStore);
+    Map<SubClusterId, ApplicationClientProtocol> clientRMProxies = interceptor.getClientRMProxies();
+
+    if (MapUtils.isNotEmpty(mockRMs)) {
+      for (Map.Entry<SubClusterId, MockRM> entry : mockRMs.entrySet()) {
+        SubClusterId sc = entry.getKey();
+        MockRM mockRM = entry.getValue();
+        stateStoreUtil.registerSubCluster(sc);
+        if (clientRMProxies.containsKey(sc)) {
+          continue;
+        }
+        clientRMProxies.put(sc, mockRM.getClientRMService());
+      }
+    }
+  }
+
+  private void initRouterRMAdminService() throws Exception {
+    Router router = this.getRouter();
+    Map<SubClusterId, MockRM> mockRMs = getMockRMs();
+    SubClusterId sc = SubClusterId.newInstance(0);
+    MockRM mockRM = mockRMs.get(sc);
+    RouterRMAdminService routerRMAdminService = router.getRmAdminProxyService();
+    RouterRMAdminService.RequestInterceptorChainWrapper rmAdminChainWrapper =
+        routerRMAdminService.getInterceptorChain();
+    DefaultRMAdminRequestInterceptor rmInterceptor =
+        (DefaultRMAdminRequestInterceptor) rmAdminChainWrapper.getRootInterceptor();
+    rmInterceptor.setRMAdmin(mockRM.getAdminService());
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java
index 05a088df781be..a4294bc36109a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java
@@ -94,7 +94,7 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughRESTRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
+    // Create a request interceptor pipeline for testing. The last one in the
     // chain will call the mock resource manager. The others in the chain will
     // simply forward it to the next one in the chain
     config.set(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE,
@@ -245,8 +245,8 @@ protected Response addToClusterNodeLabels(String user) throws Exception {
         null, createHttpServletRequest(user));
   }
 
-  protected Response removeFromCluserNodeLabels(String user) throws Exception {
-    return routerWebService.removeFromCluserNodeLabels(
+  protected Response removeFromClusterNodeLabels(String user) throws Exception {
+    return routerWebService.removeFromClusterNodeLabels(
         null, createHttpServletRequest(user));
   }
 
@@ -395,4 +395,16 @@ private HttpServletRequest createHttpServletRequest(String user) {
     when(request.getRemoteUser()).thenReturn(user);
     return request;
   }
+
+  protected Response updateSchedulerConfiguration(String user)
+      throws IOException, InterruptedException {
+    return routerWebService.updateSchedulerConfiguration(null,
+        createHttpServletRequest(user));
+  }
+
+  protected Response getSchedulerConfiguration(String user)
+      throws IOException, InterruptedException {
+    return routerWebService.
+        getSchedulerConfiguration(createHttpServletRequest(user));
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java
index e2611d91ba7a7..180cb5536b6f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java
@@ -34,15 +34,15 @@ public JavaProcess(Class<?> clazz, File output)
     this(clazz, null, output);
   }
 
-  public JavaProcess(Class<?> clazz, List<String> addClasspaths, File output)
+  public JavaProcess(Class<?> clazz, List<String> addClassPaths, File output)
       throws IOException, InterruptedException {
     String javaHome = System.getProperty("java.home");
     String javaBin =
         javaHome + File.separator + "bin" + File.separator + "java";
     String classpath = System.getProperty("java.class.path");
     classpath = classpath.concat("./src/test/resources");
-    if (addClasspaths != null) {
-      for (String addClasspath : addClasspaths) {
+    if (addClassPaths != null) {
+      for (String addClasspath : addClassPaths) {
         classpath = classpath.concat(File.pathSeparatorChar + addClasspath);
       }
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
index b96aa5a581c64..7f73434c76686 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
@@ -20,21 +20,83 @@
 
 import java.io.IOException;
 import java.net.ConnectException;
-import java.util.HashSet;
+import java.security.Principal;
+import java.util.ArrayList;
 import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Collections;
+import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.Context;
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.util.Sets;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityLevel;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
@@ -45,10 +107,36 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 /**
  * This class mocks the RESTRequestInterceptor.
  */
@@ -62,9 +150,22 @@ public class MockDefaultRequestInterceptorREST
   // This property allows us to write tests for specific scenario as YARN RM
   // down e.g. network issue, failover.
   private boolean isRunning = true;
-  private HashSet<ApplicationId> applicationMap = new HashSet<>();
+  private Map<ApplicationId, ApplicationReport> applicationMap = new HashMap<>();
   public static final String APP_STATE_RUNNING = "RUNNING";
 
+  private static final String QUEUE_DEFAULT = "default";
+  private static final String QUEUE_DEFAULT_FULL = CapacitySchedulerConfiguration.ROOT +
+      CapacitySchedulerConfiguration.DOT + QUEUE_DEFAULT;
+  private static final String QUEUE_DEDICATED = "dedicated";
+  public static final String QUEUE_DEDICATED_FULL = CapacitySchedulerConfiguration.ROOT +
+      CapacitySchedulerConfiguration.DOT + QUEUE_DEDICATED;
+
+  // duration(milliseconds), 1mins
+  public static final long DURATION = 60*1000;
+
+  // Containers 4
+  public static final int NUM_CONTAINERS = 4;
+
   private void validateRunning() throws ConnectException {
     if (!isRunning) {
       throw new ConnectException("RM is stopped");
@@ -92,7 +193,23 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
 
     ApplicationId appId = ApplicationId.fromString(newApp.getApplicationId());
     LOG.info("Application submitted: " + appId);
-    applicationMap.add(appId);
+
+    // Initialize appReport
+    ApplicationReport appReport = ApplicationReport.newInstance(
+        appId, ApplicationAttemptId.newInstance(appId, 1), null, newApp.getQueue(), null, null, 0,
+        null, YarnApplicationState.ACCEPTED, "", null, 0, 0, null, null, null, 0,
+        newApp.getApplicationType(), null, null, false, Priority.newInstance(newApp.getPriority()),
+        null, null);
+
+    // Initialize appTimeoutsMap
+    HashMap<ApplicationTimeoutType, ApplicationTimeout> appTimeoutsMap = new HashMap<>();
+    ApplicationTimeoutType timeoutType = ApplicationTimeoutType.LIFETIME;
+    ApplicationTimeout appTimeOut =
+        ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", 10);
+    appTimeoutsMap.put(timeoutType, appTimeOut);
+    appReport.setApplicationTimeouts(appTimeoutsMap);
+
+    applicationMap.put(appId, appReport);
     return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, "")
         .entity(getSubClusterId()).build();
   }
@@ -105,7 +222,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId,
     }
 
     ApplicationId applicationId = ApplicationId.fromString(appId);
-    if (!applicationMap.contains(applicationId)) {
+    if (!applicationMap.containsKey(applicationId)) {
       throw new NotFoundException("app with id: " + appId + " not found");
     }
 
@@ -140,7 +257,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr,
     validateRunning();
 
     ApplicationId applicationId = ApplicationId.fromString(appId);
-    if (!applicationMap.remove(applicationId)) {
+    if (applicationMap.remove(applicationId) == null) {
       throw new ApplicationNotFoundException(
           "Trying to kill an absent application: " + appId);
     }
@@ -213,7 +330,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId)
     }
 
     ApplicationId applicationId = ApplicationId.fromString(appId);
-    if (!applicationMap.contains(applicationId)) {
+    if (!applicationMap.containsKey(applicationId)) {
       throw new NotFoundException("app with id: " + appId + " not found");
     }
 
@@ -231,4 +348,695 @@ public boolean isRunning() {
   public void setRunning(boolean runningMode) {
     this.isRunning = runningMode;
   }
+
+  @Override
+  public ContainersInfo getContainers(HttpServletRequest req, HttpServletResponse res,
+      String appId, String appAttemptId) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    // Try format conversion for app_id
+    ApplicationId applicationId = null;
+    try {
+      applicationId = ApplicationId.fromString(appId);
+    } catch (Exception e) {
+      throw new BadRequestException(e);
+    }
+
+    // Try format conversion for app_attempt_id
+    ApplicationAttemptId applicationAttemptId = null;
+    try {
+      applicationAttemptId =
+          ApplicationAttemptId.fromString(appAttemptId);
+    } catch (Exception e) {
+      throw new BadRequestException(e);
+    }
+
+    // We avoid to check if the Application exists in the system because we need
+    // to validate that each subCluster returns 1 container.
+    ContainersInfo containers = new ContainersInfo();
+
+    int subClusterId = Integer.valueOf(getSubClusterId().getId());
+
+    ContainerId containerId = ContainerId.newContainerId(
+        ApplicationAttemptId.fromString(appAttemptId), subClusterId);
+    Resource allocatedResource =
+        Resource.newInstance(subClusterId, subClusterId);
+
+    NodeId assignedNode = NodeId.newInstance("Node", subClusterId);
+    Priority priority = Priority.newInstance(subClusterId);
+    long creationTime = subClusterId;
+    long finishTime = subClusterId;
+    String diagnosticInfo = "Diagnostic " + subClusterId;
+    String logUrl = "Log " + subClusterId;
+    int containerExitStatus = subClusterId;
+    ContainerState containerState = ContainerState.COMPLETE;
+    String nodeHttpAddress = "HttpAddress " + subClusterId;
+
+    ContainerReport containerReport = ContainerReport.newInstance(
+        containerId, allocatedResource, assignedNode, priority,
+        creationTime, finishTime, diagnosticInfo, logUrl,
+        containerExitStatus, containerState, nodeHttpAddress);
+
+    ContainerInfo container = new ContainerInfo(containerReport);
+    containers.add(container);
+
+    return containers;
+  }
+
+  @Override
+  public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+    NodeLabelsInfo cpuNode = new NodeLabelsInfo(Collections.singleton("CPU"));
+    NodeLabelsInfo gpuNode = new NodeLabelsInfo(Collections.singleton("GPU"));
+
+    HashMap<String, NodeLabelsInfo> nodeLabels = new HashMap<>();
+    nodeLabels.put("node1", cpuNode);
+    nodeLabels.put("node2", gpuNode);
+    return new NodeToLabelsInfo(nodeLabels);
+  }
+
+  @Override
+  public LabelsToNodesInfo getLabelsToNodes(Set<String> labels) throws IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    Map<NodeLabelInfo, NodeIDsInfo> labelsToNodes = new HashMap<>();
+
+    NodeLabel labelX = NodeLabel.newInstance("x", false);
+    NodeLabelInfo nodeLabelInfoX = new NodeLabelInfo(labelX);
+    ArrayList<String> hostsX = new ArrayList<>(Arrays.asList("host1A", "host1B"));
+    Resource resourceX = Resource.newInstance(20*1024, 10);
+    NodeIDsInfo nodeIDsInfoX = new NodeIDsInfo(hostsX, resourceX);
+    labelsToNodes.put(nodeLabelInfoX, nodeIDsInfoX);
+
+    NodeLabel labelY = NodeLabel.newInstance("y", false);
+    NodeLabelInfo nodeLabelInfoY = new NodeLabelInfo(labelY);
+    ArrayList<String> hostsY = new ArrayList<>(Arrays.asList("host2A", "host2B"));
+    Resource resourceY = Resource.newInstance(40*1024, 20);
+    NodeIDsInfo nodeIDsInfoY = new NodeIDsInfo(hostsY, resourceY);
+    labelsToNodes.put(nodeLabelInfoY, nodeIDsInfoY);
+
+    NodeLabel labelZ = NodeLabel.newInstance("z", false);
+    NodeLabelInfo nodeLabelInfoZ = new NodeLabelInfo(labelZ);
+    ArrayList<String> hostsZ = new ArrayList<>(Arrays.asList("host3A", "host3B"));
+    Resource resourceZ = Resource.newInstance(80*1024, 40);
+    NodeIDsInfo nodeIDsInfoZ = new NodeIDsInfo(hostsZ, resourceZ);
+    labelsToNodes.put(nodeLabelInfoZ, nodeIDsInfoZ);
+
+    return new LabelsToNodesInfo(labelsToNodes);
+  }
+
+  @Override
+  public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) throws IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+    NodeLabel labelCpu = NodeLabel.newInstance("cpu", false);
+    NodeLabel labelGpu = NodeLabel.newInstance("gpu", false);
+    return new NodeLabelsInfo(Sets.newHashSet(labelCpu, labelGpu));
+  }
+
+  @Override
+  public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) throws IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    if (StringUtils.equalsIgnoreCase(nodeId, "node1")) {
+      NodeLabel labelCpu = NodeLabel.newInstance("x", false);
+      NodeLabel labelGpu = NodeLabel.newInstance("y", false);
+      return new NodeLabelsInfo(Sets.newHashSet(labelCpu, labelGpu));
+    } else {
+      return null;
+    }
+  }
+
+  @Override
+  public ContainerInfo getContainer(HttpServletRequest req, HttpServletResponse res,
+      String appId, String appAttemptId, String containerId) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ContainerId newContainerId = ContainerId.fromString(containerId);
+
+    Resource allocatedResource = Resource.newInstance(1024, 2);
+
+    int subClusterId = Integer.valueOf(getSubClusterId().getId());
+    NodeId assignedNode = NodeId.newInstance("Node", subClusterId);
+    Priority priority = Priority.newInstance(subClusterId);
+    long creationTime = subClusterId;
+    long finishTime = subClusterId;
+    String diagnosticInfo = "Diagnostic " + subClusterId;
+    String logUrl = "Log " + subClusterId;
+    int containerExitStatus = subClusterId;
+    ContainerState containerState = ContainerState.COMPLETE;
+    String nodeHttpAddress = "HttpAddress " + subClusterId;
+
+    ContainerReport containerReport = ContainerReport.newInstance(
+        newContainerId, allocatedResource, assignedNode, priority,
+        creationTime, finishTime, diagnosticInfo, logUrl,
+        containerExitStatus, containerState, nodeHttpAddress);
+
+    return new ContainerInfo(containerReport);
+  }
+
+  @Override
+  public Response signalToContainer(String containerId, String command,
+      HttpServletRequest req) throws AuthorizationException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    if (!EnumUtils.isValidEnum(SignalContainerCommand.class, command.toUpperCase())) {
+      String errMsg = "Invalid command: " + command.toUpperCase() + ", valid commands are: "
+          + Arrays.asList(SignalContainerCommand.values());
+      return Response.status(Status.BAD_REQUEST).entity(errMsg).build();
+    }
+
+    return Response.status(Status.OK).build();
+  }
+
+  @Override
+  public AppAttemptInfo getAppAttempt(HttpServletRequest req, HttpServletResponse res,
+      String appId, String appAttemptId) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(appAttemptId);
+
+    ApplicationReport newApplicationReport = ApplicationReport.newInstance(
+        applicationId, attemptId, "user", "queue", "appname", "host", 124, null,
+        YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4,
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+
+    ApplicationAttemptReport attempt = ApplicationAttemptReport.newInstance(
+        attemptId, "host", 124, "url", "oUrl", "diagnostics",
+        YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId(
+        newApplicationReport.getCurrentApplicationAttemptId(), 1));
+
+    return new AppAttemptInfo(attempt);
+  }
+
+  @Override
+  public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    AppAttemptsInfo infos = new AppAttemptsInfo();
+    infos.add(TestRouterWebServiceUtil.generateAppAttemptInfo(0));
+    infos.add(TestRouterWebServiceUtil.generateAppAttemptInfo(1));
+    return infos;
+  }
+
+  @Override
+  public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr,
+      String appId, String type) throws AuthorizationException {
+
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
+    ApplicationTimeoutType paramType = ApplicationTimeoutType.valueOf(type);
+
+    if (paramType == null) {
+      throw new NotFoundException("application timeout type not found");
+    }
+
+    if (!timeouts.containsKey(paramType)) {
+      throw new NotFoundException("timeout with id: " + appId + " not found");
+    }
+
+    ApplicationTimeout applicationTimeout = timeouts.get(paramType);
+
+    AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+    timeoutInfo.setExpiryTime(applicationTimeout.getExpiryTime());
+    timeoutInfo.setTimeoutType(applicationTimeout.getTimeoutType());
+    timeoutInfo.setRemainingTime(applicationTimeout.getRemainingTime());
+
+    return timeoutInfo;
+  }
+
+  @Override
+  public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId)
+      throws AuthorizationException {
+
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
+
+    AppTimeoutsInfo timeoutsInfo = new AppTimeoutsInfo();
+
+    for (ApplicationTimeout timeout : timeouts.values()) {
+      AppTimeoutInfo timeoutInfo = new AppTimeoutInfo();
+      timeoutInfo.setExpiryTime(timeout.getExpiryTime());
+      timeoutInfo.setTimeoutType(timeout.getTimeoutType());
+      timeoutInfo.setRemainingTime(timeout.getRemainingTime());
+      timeoutsInfo.add(timeoutInfo);
+    }
+
+    return timeoutsInfo;
+  }
+
+  @Override
+  public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, HttpServletRequest hsr,
+      String appId) throws AuthorizationException,
+      YarnException, InterruptedException, IOException {
+
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    Map<ApplicationTimeoutType, ApplicationTimeout> timeouts = appReport.getApplicationTimeouts();
+
+    ApplicationTimeoutType paramTimeoutType = appTimeout.getTimeoutType();
+    if (!timeouts.containsKey(paramTimeoutType)) {
+      throw new NotFoundException("TimeOutType with id: " + appId + " not found");
+    }
+
+    ApplicationTimeout applicationTimeout = timeouts.get(paramTimeoutType);
+    applicationTimeout.setTimeoutType(appTimeout.getTimeoutType());
+    applicationTimeout.setExpiryTime(appTimeout.getExpireTime());
+    applicationTimeout.setRemainingTime(appTimeout.getRemainingTimeInSec());
+
+    AppTimeoutInfo result = new AppTimeoutInfo(applicationTimeout);
+
+    return Response.status(Status.OK).entity(result).build();
+  }
+
+  @Override
+  public Response updateApplicationPriority(AppPriority targetPriority, HttpServletRequest hsr,
+      String appId) throws YarnException, InterruptedException, IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (targetPriority == null) {
+      return Response.status(Status.BAD_REQUEST).build();
+    }
+
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    Priority newPriority = Priority.newInstance(targetPriority.getPriority());
+    appReport.setPriority(newPriority);
+
+    return Response.status(Status.OK).entity(targetPriority).build();
+  }
+
+  @Override
+  public AppPriority getAppPriority(HttpServletRequest hsr, String appId)
+      throws AuthorizationException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    Priority priority = appReport.getPriority();
+
+    return new AppPriority(priority.getPriority());
+  }
+
+  @Override
+  public AppQueue getAppQueue(HttpServletRequest hsr, String appId)
+      throws AuthorizationException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+    String queue = applicationMap.get(applicationId).getQueue();
+    return new AppQueue(queue);
+  }
+
+  @Override
+  public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, String appId)
+      throws AuthorizationException, YarnException, InterruptedException, IOException {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+    if (targetQueue == null || StringUtils.isBlank(targetQueue.getQueue())) {
+      return Response.status(Status.BAD_REQUEST).build();
+    }
+
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    String originalQueue = appReport.getQueue();
+    appReport.setQueue(targetQueue.getQueue());
+    applicationMap.put(applicationId, appReport);
+    LOG.info("Update applicationId = {} from originalQueue = {} to targetQueue = {}.",
+        appId, originalQueue, targetQueue);
+
+    AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
+    return Response.status(Status.OK).entity(targetAppQueue).build();
+  }
+
+  public void updateApplicationState(YarnApplicationState appState, String appId)
+      throws AuthorizationException, YarnException, InterruptedException, IOException {
+    validateRunning();
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+    ApplicationReport appReport = applicationMap.get(applicationId);
+    appReport.setYarnApplicationState(appState);
+  }
+
+  @Override
+  public ApplicationStatisticsInfo getAppStatistics(
+      HttpServletRequest hsr, Set<String> stateQueries, Set<String> typeQueries) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+    for (ApplicationReport appReport : applicationMap.values()) {
+
+      YarnApplicationState appState = appReport.getYarnApplicationState();
+      String appType = appReport.getApplicationType();
+
+      if (stateQueries.contains(appState.name()) && typeQueries.contains(appType)) {
+        String itemInfoMapKey = appState.toString() + "_" + appType;
+        StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey, null);
+        if (itemInfo == null) {
+          itemInfo = new StatisticsItemInfo(appState, appType, 1);
+        } else {
+          long newCount = itemInfo.getCount() + 1;
+          itemInfo.setCount(newCount);
+        }
+        itemInfoMap.put(itemInfoMapKey, itemInfo);
+      }
+    }
+
+    return new ApplicationStatisticsInfo(itemInfoMap.values());
+  }
+
+  @Override
+  public AppActivitiesInfo getAppActivities(
+      HttpServletRequest hsr, String appId, String time, Set<String> requestPriorities,
+      Set<String> allocationRequestIds, String groupBy, String limit, Set<String> actions,
+      boolean summarize) {
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    ApplicationId applicationId = ApplicationId.fromString(appId);
+    if (!applicationMap.containsKey(applicationId)) {
+      throw new NotFoundException("app with id: " + appId + " not found");
+    }
+
+    SchedulerNode schedulerNode = TestUtils.getMockNode("host0", "rack", 1, 10240);
+
+    RMContext rmContext = Mockito.mock(RMContext.class);
+    Mockito.when(rmContext.getYarnConfiguration()).thenReturn(this.getConf());
+    ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class);
+    Mockito.when(scheduler.getMinimumResourceCapability()).thenReturn(Resources.none());
+    Mockito.when(rmContext.getScheduler()).thenReturn(scheduler);
+    LeafQueue mockQueue = Mockito.mock(LeafQueue.class);
+    Map<ApplicationId, RMApp> rmApps = new ConcurrentHashMap<>();
+    Mockito.doReturn(rmApps).when(rmContext).getRMApps();
+
+    FiCaSchedulerNode node = (FiCaSchedulerNode) schedulerNode;
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
+    RMApp mockApp = Mockito.mock(RMApp.class);
+    Mockito.doReturn(appAttemptId.getApplicationId()).when(mockApp).getApplicationId();
+    Mockito.doReturn(FinalApplicationStatus.UNDEFINED).when(mockApp).getFinalApplicationStatus();
+    rmApps.put(appAttemptId.getApplicationId(), mockApp);
+    FiCaSchedulerApp app = new FiCaSchedulerApp(appAttemptId, "user", mockQueue,
+        mock(ActiveUsersManager.class), rmContext);
+
+    ActivitiesManager newActivitiesManager = new ActivitiesManager(rmContext);
+    newActivitiesManager.turnOnAppActivitiesRecording(app.getApplicationId(), 3);
+
+    int numActivities = 10;
+    for (int i = 0; i < numActivities; i++) {
+      ActivitiesLogger.APP.startAppAllocationRecording(newActivitiesManager, node,
+          SystemClock.getInstance().getTime(), app);
+      ActivitiesLogger.APP.recordAppActivityWithoutAllocation(newActivitiesManager, node, app,
+          new SchedulerRequestKey(Priority.newInstance(0), 0, null),
+          ActivityDiagnosticConstant.NODE_IS_BLACKLISTED, ActivityState.REJECTED,
+          ActivityLevel.NODE);
+      ActivitiesLogger.APP.finishSkippedAppAllocationRecording(newActivitiesManager,
+          app.getApplicationId(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
+    }
+
+    Set<Integer> prioritiesInt =
+        requestPriorities.stream().map(pri -> Integer.parseInt(pri)).collect(Collectors.toSet());
+    Set<Long> allocationReqIds =
+        allocationRequestIds.stream().map(id -> Long.parseLong(id)).collect(Collectors.toSet());
+    AppActivitiesInfo appActivitiesInfo = newActivitiesManager.
+        getAppActivitiesInfo(app.getApplicationId(), prioritiesInt, allocationReqIds, null,
+        Integer.parseInt(limit), summarize, 3);
+
+    return appActivitiesInfo;
+  }
+
+  @Override
+  public Response listReservation(String queue, String reservationId, long startTime, long endTime,
+      boolean includeResourceAllocations, HttpServletRequest hsr) throws Exception {
+
+    if (!isRunning) {
+      throw new RuntimeException("RM is stopped");
+    }
+
+    if (!StringUtils.equals(queue, QUEUE_DEDICATED_FULL)) {
+      throw new RuntimeException("The specified queue: " + queue +
+          " is not managed by reservation system." +
+          " Please try again with a valid reservable queue.");
+    }
+
+    MockRM mockRM = setupResourceManager();
+
+    ReservationId reservationID = ReservationId.parseReservationId(reservationId);
+    ReservationSystem reservationSystem = mockRM.getReservationSystem();
+    reservationSystem.synchronizePlan(QUEUE_DEDICATED_FULL, true);
+
+    // Generate reserved resources
+    ClientRMService clientService = mockRM.getClientRMService();
+
+    // arrival time from which the resource(s) can be allocated.
+    long arrival = Time.now();
+
+    // deadline by when the resource(s) must be allocated.
+    // The reason for choosing 1.05 is because this gives an integer
+    // DURATION * 0.05 = 3000(ms)
+    // deadline = arrival + 3000ms
+    long deadline = (long) (arrival + 1.05 * DURATION);
+
+    // In this test of reserved resources, we will apply for 4 containers (1 core, 1GB memory)
+    // arrival = Time.now(), and make sure deadline - arrival > duration,
+    // the current setting is greater than 3000ms
+    ReservationSubmissionRequest submissionRequest =
+        ReservationSystemTestUtil.createSimpleReservationRequest(
+            reservationID, NUM_CONTAINERS, arrival, deadline, DURATION);
+    clientService.submitReservation(submissionRequest);
+
+    // listReservations
+    ReservationListRequest request = ReservationListRequest.newInstance(
+        queue, reservationID.toString(), startTime, endTime, includeResourceAllocations);
+    ReservationListResponse resRespInfo = clientService.listReservations(request);
+    ReservationListInfo resResponse =
+        new ReservationListInfo(resRespInfo, includeResourceAllocations);
+
+    if (mockRM != null) {
+      mockRM.stop();
+    }
+
+    return Response.status(Status.OK).entity(resResponse).build();
+  }
+
+  @Override
+  public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) {
+
+    NodeLabelInfo nodeLabelInfo = new NodeLabelInfo();
+    nodeLabelInfo.setExclusivity(true);
+    nodeLabelInfo.setName("Test-Label");
+    nodeLabelInfo.setActiveNMs(10);
+    PartitionInfo partitionInfo = new PartitionInfo();
+
+    NodeLabelsInfo nodeLabelsInfo = new NodeLabelsInfo();
+    nodeLabelsInfo.getNodeLabelsInfo().add(nodeLabelInfo);
+
+    return nodeLabelsInfo;
+  }
+
+  private MockRM setupResourceManager() throws Exception {
+    DefaultMetricsSystem.setMiniClusterMode(true);
+
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+
+    // Define default queue
+    conf.setCapacity(QUEUE_DEFAULT_FULL, 20);
+    // Define dedicated queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] {QUEUE_DEFAULT,  QUEUE_DEDICATED});
+    conf.setCapacity(QUEUE_DEDICATED_FULL, 80);
+    conf.setReservable(QUEUE_DEDICATED_FULL, true);
+
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
+    conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    rm.registerNode("127.0.0.1:5678", 100*1024, 100);
+    return rm;
+  }
+
+  @Override
+  public RMQueueAclInfo checkUserAccessToQueue(String queue, String username,
+      String queueAclType, HttpServletRequest hsr) throws AuthorizationException {
+
+    ResourceManager mockRM = mock(ResourceManager.class);
+    Configuration conf = new YarnConfiguration();
+
+    ResourceScheduler mockScheduler = new CapacityScheduler() {
+      @Override
+      public synchronized boolean checkAccess(UserGroupInformation callerUGI,
+          QueueACL acl, String queueName) {
+        if (acl == QueueACL.ADMINISTER_QUEUE) {
+          if (callerUGI.getUserName().equals("admin")) {
+            return true;
+          }
+        } else {
+          if (ImmutableSet.of("admin", "yarn").contains(callerUGI.getUserName())) {
+            return true;
+          }
+        }
+        return false;
+      }
+    };
+
+    when(mockRM.getResourceScheduler()).thenReturn(mockScheduler);
+    MockRMWebServices webSvc = new MockRMWebServices(mockRM, conf, mock(HttpServletResponse.class));
+    return webSvc.checkUserAccessToQueue(queue, username, queueAclType, hsr);
+  }
+
+  class MockRMWebServices {
+
+    @Context
+    private HttpServletResponse httpServletResponse;
+    private ResourceManager resourceManager;
+
+    private void initForReadableEndpoints() {
+      // clear content type
+      httpServletResponse.setContentType(null);
+    }
+
+    MockRMWebServices(ResourceManager rm, Configuration conf, HttpServletResponse response) {
+      this.resourceManager = rm;
+      this.httpServletResponse = response;
+    }
+
+    private UserGroupInformation getCallerUserGroupInformation(
+        HttpServletRequest hsr, boolean usePrincipal) {
+
+      String remoteUser = hsr.getRemoteUser();
+
+      if (usePrincipal) {
+        Principal princ = hsr.getUserPrincipal();
+        remoteUser = princ == null ? null : princ.getName();
+      }
+
+      UserGroupInformation callerUGI = null;
+      if (remoteUser != null) {
+        callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+      }
+
+      return callerUGI;
+    }
+
+    public RMQueueAclInfo checkUserAccessToQueue(
+        String queue, String username, String queueAclType, HttpServletRequest hsr)
+        throws AuthorizationException {
+      initForReadableEndpoints();
+
+      // For the user who invokes this REST call, he/she should have admin access
+      // to the queue. Otherwise we will reject the call.
+      UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+      if (callerUGI != null && !this.resourceManager.getResourceScheduler().checkAccess(
+              callerUGI, QueueACL.ADMINISTER_QUEUE, queue)) {
+        throw new ForbiddenException(
+                "User=" + callerUGI.getUserName() + " doesn't haven access to queue="
+                        + queue + " so it cannot check ACLs for other users.");
+      }
+
+      // Create UGI for the to-be-checked user.
+      UserGroupInformation user = UserGroupInformation.createRemoteUser(username);
+      if (user == null) {
+        throw new ForbiddenException(
+           "Failed to retrieve UserGroupInformation for user=" + username);
+      }
+
+      // Check if the specified queue acl is valid.
+      QueueACL queueACL;
+      try {
+        queueACL = QueueACL.valueOf(queueAclType);
+      } catch (IllegalArgumentException e) {
+        throw new BadRequestException("Specified queueAclType=" + queueAclType
+            + " is not a valid type, valid queue acl types={"
+            + "SUBMIT_APPLICATIONS/ADMINISTER_QUEUE}");
+      }
+
+      if (!this.resourceManager.getResourceScheduler().checkAccess(user, queueACL, queue)) {
+        return new RMQueueAclInfo(false, user.getUserName(),
+            "User=" + username + " doesn't have access to queue=" + queue
+            + " with acl-type=" + queueAclType);
+      }
+
+      return new RMQueueAclInfo(true, user.getUserName(), "");
+    }
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java
index 67c9d671fb159..a09199b9e856b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java
@@ -62,6 +62,7 @@
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 
 /**
  * This class mocks the RESTRequestInterceptor.
@@ -184,6 +185,11 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr)
     return new NodeToLabelsInfo();
   }
 
+  @Override
+  public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) throws IOException {
+    return new NodeLabelsInfo();
+  }
+
   @Override
   public LabelsToNodesInfo getLabelsToNodes(Set<String> labels)
       throws IOException {
@@ -215,7 +221,7 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
   }
 
   @Override
-  public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+  public Response removeFromClusterNodeLabels(Set<String> oldNodeLabels,
       HttpServletRequest hsr) throws Exception {
     return Response.status(Status.OK).build();
   }
@@ -373,4 +379,16 @@ public Response signalToContainer(String containerId, String command,
       HttpServletRequest req) {
     return Response.status(Status.OK).build();
   }
+
+  @Override
+  public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest hsr) throws AuthorizationException, InterruptedException {
+    return Response.status(Status.OK).build();
+  }
+
+  @Override
+  public Response getSchedulerConfiguration(HttpServletRequest hsr)
+      throws AuthorizationException {
+    return Response.status(Status.OK).build();
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java
new file mode 100644
index 0000000000000..e3790108c0ae5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRouter.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.router.Router;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MockRouter extends Router {
+
+  private FederationStateStoreFacade facade;
+
+  public MockRouter(Configuration configuration)
+      throws InterruptedException, YarnException, IOException {
+    this.setConfig(configuration);
+
+    boolean isEnabled = configuration.getBoolean(
+        YarnConfiguration.FEDERATION_ENABLED,
+        YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+
+    if (isEnabled) {
+      facade = FederationStateStoreFacade.getInstance();
+      initTestFederationSubCluster();
+    }
+  }
+
+  public void initTestFederationSubCluster()
+      throws IOException, InterruptedException, YarnException {
+
+    // Initialize subcluster information
+    String scAmRMAddress = "5.6.7.8:5";
+    String scClientRMAddress = "5.6.7.8:6";
+    String scRmAdminAddress = "5.6.7.8:7";
+    String scWebAppAddress = "127.0.0.1:8080";
+
+    // Initialize subcluster capability
+    String[] capabilityPathItems = new String[] {".", "target", "test-classes", "capability"};
+    String capabilityPath = StringUtils.join(capabilityPathItems, File.separator);
+    String capabilityJson =
+        FileUtils.readFileToString(new File(capabilityPath), StandardCharsets.UTF_8);
+
+    // capability json needs to remove asflicense
+    String regex = "\"___asflicense__.*\\n(.*,\\n){1,15}.*\\n.*";
+    Pattern p = Pattern.compile(regex);
+    Matcher m = p.matcher(capabilityJson);
+    capabilityJson = m.replaceAll("").trim();
+
+    // Initialize subcluster sc1
+    SubClusterInfo sc1 =
+        SubClusterInfo.newInstance(SubClusterId.newInstance("SC-1"),
+        scAmRMAddress, scClientRMAddress, scRmAdminAddress, scWebAppAddress,
+        SubClusterState.SC_RUNNING, Time.now(), capabilityJson);
+    Thread.sleep(5000);
+    sc1.setLastHeartBeat(Time.now());
+
+    // Initialize subcluster sc2
+    SubClusterInfo sc2 =
+        SubClusterInfo.newInstance(SubClusterId.newInstance("SC-2"),
+        scAmRMAddress, scClientRMAddress, scRmAdminAddress, scWebAppAddress,
+        SubClusterState.SC_RUNNING, Time.now(), capabilityJson);
+    Thread.sleep(5000);
+    sc2.setLastHeartBeat(Time.now());
+
+    FederationStateStore stateStore = facade.getStateStore();
+    stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(sc1));
+    stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(sc2));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java
index 142a6511b93a6..1bffd40db3c19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java
@@ -60,10 +60,11 @@
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
 
 /**
- * Mock intercepter that does not do anything other than forwarding it to the
- * next intercepter in the chain.
+ * Mock interceptor that does not do anything other than forwarding it to the
+ * next interceptor in the chain.
  */
 public class PassThroughRESTRequestInterceptor
     extends AbstractRESTRequestInterceptor {
@@ -216,6 +217,11 @@ public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr)
     return getNextInterceptor().getNodeToLabels(hsr);
   }
 
+  @Override
+  public NodeLabelsInfo getRMNodeLabels(HttpServletRequest hsr) throws IOException {
+    return getNextInterceptor().getRMNodeLabels(hsr);
+  }
+
   @Override
   public LabelsToNodesInfo getLabelsToNodes(Set<String> labels)
       throws IOException {
@@ -248,9 +254,9 @@ public Response addToClusterNodeLabels(NodeLabelsInfo newNodeLabels,
   }
 
   @Override
-  public Response removeFromCluserNodeLabels(Set<String> oldNodeLabels,
+  public Response removeFromClusterNodeLabels(Set<String> oldNodeLabels,
       HttpServletRequest hsr) throws Exception {
-    return getNextInterceptor().removeFromCluserNodeLabels(oldNodeLabels, hsr);
+    return getNextInterceptor().removeFromClusterNodeLabels(oldNodeLabels, hsr);
   }
 
   @Override
@@ -379,4 +385,17 @@ public Response signalToContainer(String containerId,
       String command, HttpServletRequest req) throws AuthorizationException {
     return getNextInterceptor().signalToContainer(containerId, command, req);
   }
+
+  @Override
+  public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo,
+      HttpServletRequest hsr)
+      throws AuthorizationException, InterruptedException {
+    return getNextInterceptor().updateSchedulerConfiguration(mutationInfo, hsr);
+  }
+
+  @Override
+  public Response getSchedulerConfiguration(HttpServletRequest hsr)
+      throws AuthorizationException {
+    return getNextInterceptor().getSchedulerConfiguration(hsr);
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
index b3a7e9060c53b..4c50e5198dc03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
@@ -19,22 +19,48 @@
 package org.apache.hadoop.yarn.server.router.webapp;
 
 import java.io.IOException;
+import java.security.Principal;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Collections;
 
+import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
 
+import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ReservationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceOption;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
 import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
@@ -47,18 +73,49 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo;
+import org.apache.hadoop.yarn.server.router.webapp.cache.RouterAppInfoCacheKey;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.server.router.webapp.dao.FederationRMQueueAclInfo;
+import org.apache.hadoop.yarn.util.LRUCacheHashMap;
 import org.apache.hadoop.yarn.util.MonotonicClock;
+import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.yarn.server.router.webapp.MockDefaultRequestInterceptorREST.QUEUE_DEDICATED_FULL;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 /**
  * Extends the {@code BaseRouterClientRMTest} and overrides methods in order to
  * use the {@code RouterClientRMService} pipeline test cases for testing the
  * {@code FederationInterceptor} class. The tests for
  * {@code RouterClientRMService} has been written cleverly so that it can be
- * reused to validate different request intercepter chains.
+ * reused to validate different request interceptor chains.
  */
 public class TestFederationInterceptorREST extends BaseRouterWebServicesTest {
   private static final Logger LOG =
@@ -116,8 +173,8 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughRESTRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
-    // chain is the federation intercepter that calls the mock resource manager.
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
     // The others in the chain will simply forward it to the next one in the
     // chain
     conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
@@ -130,6 +187,10 @@ protected YarnConfiguration createConfiguration() {
     // Disable StateStoreFacade cache
     conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
 
+    // Open AppsInfo Cache
+    conf.setBoolean(YarnConfiguration.ROUTER_APPSINFO_ENABLED, true);
+    conf.setInt(YarnConfiguration.ROUTER_APPSINFO_CACHED_COUNT, 10);
+
     return conf;
   }
 
@@ -160,7 +221,7 @@ public void testSubmitApplication()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
 
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
@@ -187,7 +248,7 @@ public void testSubmitApplicationMultipleSubmission()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
     context.setApplicationId(appId.toString());
@@ -236,7 +297,7 @@ public void testSubmitApplicationEmptyRequest()
   }
 
   /**
-   * This test validates the correctness of SubmitApplication in case of of
+   * This test validates the correctness of SubmitApplication in case of
    * application in wrong format.
    */
   @Test
@@ -259,7 +320,7 @@ public void testForceKillApplication()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
     context.setApplicationId(appId.toString());
@@ -286,7 +347,7 @@ public void testForceKillApplicationNotExists()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
     AppState appState = new AppState("KILLED");
 
     Response response =
@@ -317,7 +378,7 @@ public void testForceKillApplicationWrongFormat()
   public void testForceKillApplicationEmptyRequest()
       throws YarnException, IOException, InterruptedException {
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
 
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
@@ -341,7 +402,7 @@ public void testGetApplicationReport()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
     context.setApplicationId(appId.toString());
@@ -478,7 +539,7 @@ public void testGetApplicationState()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
     ApplicationSubmissionContextInfo context =
         new ApplicationSubmissionContextInfo();
     context.setApplicationId(appId.toString());
@@ -505,7 +566,7 @@ public void testGetApplicationStateNotExists()
       throws YarnException, IOException, InterruptedException {
 
     ApplicationId appId =
-        ApplicationId.newInstance(System.currentTimeMillis(), 1);
+        ApplicationId.newInstance(Time.now(), 1);
 
     AppState response = interceptor.getAppState(null, appId.toString());
 
@@ -560,4 +621,659 @@ SubClusterState.SC_RUNNING, new MonotonicClock().getTime(),
             SubClusterRegisterRequest.newInstance(subClusterInfo));
   }
 
-}
+  @Test
+  public void testGetContainers()
+      throws YarnException, IOException, InterruptedException {
+
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context =
+        new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    // Submit the application we want the report later
+    Response response = interceptor.submitApplication(context, null);
+
+    Assert.assertNotNull(response);
+    Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId));
+
+    ApplicationAttemptId appAttempt = ApplicationAttemptId.newInstance(appId, 1);
+
+    ContainersInfo responseGet = interceptor.getContainers(
+        null, null, appId.toString(), appAttempt.toString());
+
+    Assert.assertEquals(4, responseGet.getContainers().size());
+  }
+
+  @Test
+  public void testGetContainersNotExists() throws Exception {
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    LambdaTestUtils.intercept(IllegalArgumentException.class,
+        "Parameter error, the appAttemptId is empty or null.",
+        () -> interceptor.getContainers(null, null, appId.toString(), null));
+  }
+
+  @Test
+  public void testGetContainersWrongFormat() throws Exception {
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationAttemptId appAttempt = ApplicationAttemptId.newInstance(appId, 1);
+
+    // Test Case 1: appId is wrong format, appAttemptId is accurate.
+    LambdaTestUtils.intercept(IllegalArgumentException.class,
+        "Invalid ApplicationId prefix: Application_wrong_id. " +
+        "The valid ApplicationId should start with prefix application",
+        () -> interceptor.getContainers(null, null, "Application_wrong_id", appAttempt.toString()));
+
+    // Test Case2: appId is accurate, appAttemptId is wrong format.
+    LambdaTestUtils.intercept(IllegalArgumentException.class,
+        "Invalid AppAttemptId prefix: AppAttempt_wrong_id",
+        () -> interceptor.getContainers(null, null, appId.toString(), "AppAttempt_wrong_id"));
+  }
+
+  @Test
+  public void testGetNodeToLabels() throws IOException {
+    NodeToLabelsInfo info = interceptor.getNodeToLabels(null);
+    HashMap<String, NodeLabelsInfo> map = info.getNodeToLabels();
+    Assert.assertNotNull(map);
+    Assert.assertEquals(2, map.size());
+
+    NodeLabelsInfo node1Value = map.getOrDefault("node1", null);
+    Assert.assertNotNull(node1Value);
+    Assert.assertEquals(1, node1Value.getNodeLabelsName().size());
+    Assert.assertEquals("CPU", node1Value.getNodeLabelsName().get(0));
+
+    NodeLabelsInfo node2Value = map.getOrDefault("node2", null);
+    Assert.assertNotNull(node2Value);
+    Assert.assertEquals(1, node2Value.getNodeLabelsName().size());
+    Assert.assertEquals("GPU", node2Value.getNodeLabelsName().get(0));
+  }
+
+  @Test
+  public void testGetLabelsToNodes() throws Exception {
+    LabelsToNodesInfo labelsToNodesInfo = interceptor.getLabelsToNodes(null);
+    Map<NodeLabelInfo, NodeIDsInfo> map = labelsToNodesInfo.getLabelsToNodes();
+    Assert.assertNotNull(map);
+    Assert.assertEquals(3, map.size());
+
+    NodeLabel labelX = NodeLabel.newInstance("x", false);
+    NodeLabelInfo nodeLabelInfoX = new NodeLabelInfo(labelX);
+    NodeIDsInfo nodeIDsInfoX = map.get(nodeLabelInfoX);
+    Assert.assertNotNull(nodeIDsInfoX);
+    Assert.assertEquals(2, nodeIDsInfoX.getNodeIDs().size());
+    Resource resourceX =
+        nodeIDsInfoX.getPartitionInfo().getResourceAvailable().getResource();
+    Assert.assertNotNull(resourceX);
+    Assert.assertEquals(4*10, resourceX.getVirtualCores());
+    Assert.assertEquals(4*20*1024, resourceX.getMemorySize());
+
+    NodeLabel labelY = NodeLabel.newInstance("y", false);
+    NodeLabelInfo nodeLabelInfoY = new NodeLabelInfo(labelY);
+    NodeIDsInfo nodeIDsInfoY = map.get(nodeLabelInfoY);
+    Assert.assertNotNull(nodeIDsInfoY);
+    Assert.assertEquals(2, nodeIDsInfoY.getNodeIDs().size());
+    Resource resourceY =
+        nodeIDsInfoY.getPartitionInfo().getResourceAvailable().getResource();
+    Assert.assertNotNull(resourceY);
+    Assert.assertEquals(4*20, resourceY.getVirtualCores());
+    Assert.assertEquals(4*40*1024, resourceY.getMemorySize());
+  }
+
+  @Test
+  public void testGetClusterNodeLabels() throws Exception {
+    NodeLabelsInfo nodeLabelsInfo = interceptor.getClusterNodeLabels(null);
+    Assert.assertNotNull(nodeLabelsInfo);
+    Assert.assertEquals(2, nodeLabelsInfo.getNodeLabelsName().size());
+
+    List<String> nodeLabelsName = nodeLabelsInfo.getNodeLabelsName();
+    Assert.assertNotNull(nodeLabelsName);
+    Assert.assertTrue(nodeLabelsName.contains("cpu"));
+    Assert.assertTrue(nodeLabelsName.contains("gpu"));
+
+    ArrayList<NodeLabelInfo> nodeLabelInfos = nodeLabelsInfo.getNodeLabelsInfo();
+    Assert.assertNotNull(nodeLabelInfos);
+    Assert.assertEquals(2, nodeLabelInfos.size());
+    NodeLabelInfo cpuNodeLabelInfo = new NodeLabelInfo("cpu", false);
+    Assert.assertTrue(nodeLabelInfos.contains(cpuNodeLabelInfo));
+    NodeLabelInfo gpuNodeLabelInfo = new NodeLabelInfo("gpu", false);
+    Assert.assertTrue(nodeLabelInfos.contains(gpuNodeLabelInfo));
+  }
+
+  @Test
+  public void testGetLabelsOnNode() throws Exception {
+    NodeLabelsInfo nodeLabelsInfo = interceptor.getLabelsOnNode(null, "node1");
+    Assert.assertNotNull(nodeLabelsInfo);
+    Assert.assertEquals(2, nodeLabelsInfo.getNodeLabelsName().size());
+
+    List<String> nodeLabelsName = nodeLabelsInfo.getNodeLabelsName();
+    Assert.assertNotNull(nodeLabelsName);
+    Assert.assertTrue(nodeLabelsName.contains("x"));
+    Assert.assertTrue(nodeLabelsName.contains("y"));
+
+    // null request
+    NodeLabelsInfo nodeLabelsInfo2 = interceptor.getLabelsOnNode(null, "node2");
+    Assert.assertNotNull(nodeLabelsInfo2);
+    Assert.assertEquals(0, nodeLabelsInfo2.getNodeLabelsName().size());
+  }
+
+  @Test
+  public void testGetContainer() throws Exception {
+    //
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId appContainerId = ContainerId.newContainerId(appAttemptId, 1);
+    String applicationId = appId.toString();
+    String attemptId = appAttemptId.toString();
+    String containerId = appContainerId.toString();
+
+    // Submit application to multiSubCluster
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(applicationId);
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    // Test Case1: Wrong ContainerId
+    LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid ContainerId prefix: 0",
+        () -> interceptor.getContainer(null, null, applicationId, attemptId, "0"));
+
+    // Test Case2: Correct ContainerId
+
+    ContainerInfo containerInfo = interceptor.getContainer(null, null, applicationId,
+        attemptId, containerId);
+    Assert.assertNotNull(containerInfo);
+  }
+
+  @Test
+  public void testGetAppAttempts()
+      throws IOException, InterruptedException, YarnException {
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    AppAttemptsInfo appAttemptsInfo = interceptor.getAppAttempts(null, appId.toString());
+    Assert.assertNotNull(appAttemptsInfo);
+
+    ArrayList<AppAttemptInfo> attemptLists = appAttemptsInfo.getAttempts();
+    Assert.assertNotNull(appAttemptsInfo);
+    Assert.assertEquals(2, attemptLists.size());
+
+    AppAttemptInfo attemptInfo1 = attemptLists.get(0);
+    Assert.assertNotNull(attemptInfo1);
+    Assert.assertEquals(0, attemptInfo1.getAttemptId());
+    Assert.assertEquals("AppAttemptId_0", attemptInfo1.getAppAttemptId());
+    Assert.assertEquals("LogLink_0", attemptInfo1.getLogsLink());
+    Assert.assertEquals(1659621705L, attemptInfo1.getFinishedTime());
+
+    AppAttemptInfo attemptInfo2 = attemptLists.get(1);
+    Assert.assertNotNull(attemptInfo2);
+    Assert.assertEquals(0, attemptInfo2.getAttemptId());
+    Assert.assertEquals("AppAttemptId_1", attemptInfo2.getAppAttemptId());
+    Assert.assertEquals("LogLink_1", attemptInfo2.getLogsLink());
+    Assert.assertEquals(1659621705L, attemptInfo2.getFinishedTime());
+  }
+
+  @Test
+  public void testGetAppAttempt()
+      throws IOException, InterruptedException, YarnException {
+
+    // Generate ApplicationId information
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    // Generate ApplicationAttemptId information
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+    ApplicationAttemptId expectAppAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+    String appAttemptId = expectAppAttemptId.toString();
+
+    org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo
+        appAttemptInfo = interceptor.getAppAttempt(null, null, appId.toString(), appAttemptId);
+
+    Assert.assertNotNull(appAttemptInfo);
+    Assert.assertEquals(expectAppAttemptId.toString(), appAttemptInfo.getAppAttemptId());
+    Assert.assertEquals("url", appAttemptInfo.getTrackingUrl());
+    Assert.assertEquals("oUrl", appAttemptInfo.getOriginalTrackingUrl());
+    Assert.assertEquals(124, appAttemptInfo.getRpcPort());
+    Assert.assertEquals("host", appAttemptInfo.getHost());
+  }
+
+  @Test
+  public void testGetAppTimeout() throws IOException, InterruptedException, YarnException {
+
+    // Generate ApplicationId information
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    // Generate ApplicationAttemptId information
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    ApplicationTimeoutType appTimeoutType = ApplicationTimeoutType.LIFETIME;
+    AppTimeoutInfo appTimeoutInfo =
+        interceptor.getAppTimeout(null, appId.toString(), appTimeoutType.toString());
+    Assert.assertNotNull(appTimeoutInfo);
+    Assert.assertEquals(10, appTimeoutInfo.getRemainingTimeInSec());
+    Assert.assertEquals("UNLIMITED", appTimeoutInfo.getExpireTime());
+    Assert.assertEquals(appTimeoutType, appTimeoutInfo.getTimeoutType());
+  }
+
+  @Test
+  public void testGetAppTimeouts() throws IOException, InterruptedException, YarnException {
+
+    // Generate ApplicationId information
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    // Generate ApplicationAttemptId information
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    AppTimeoutsInfo appTimeoutsInfo = interceptor.getAppTimeouts(null, appId.toString());
+    Assert.assertNotNull(appTimeoutsInfo);
+
+    List<AppTimeoutInfo> timeouts = appTimeoutsInfo.getAppTimeouts();
+    Assert.assertNotNull(timeouts);
+    Assert.assertEquals(1, timeouts.size());
+
+    AppTimeoutInfo resultAppTimeout = timeouts.get(0);
+    Assert.assertNotNull(resultAppTimeout);
+    Assert.assertEquals(10, resultAppTimeout.getRemainingTimeInSec());
+    Assert.assertEquals("UNLIMITED", resultAppTimeout.getExpireTime());
+    Assert.assertEquals(ApplicationTimeoutType.LIFETIME, resultAppTimeout.getTimeoutType());
+  }
+
+  @Test
+  public void testUpdateApplicationTimeout() throws IOException, InterruptedException,
+      YarnException {
+
+    // Generate ApplicationId information
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+
+    // Generate ApplicationAttemptId information
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    long newLifetime = 10L;
+    // update 10L seconds more to timeout
+    String timeout = Times.formatISO8601(Time.now() + newLifetime * 1000);
+    AppTimeoutInfo paramAppTimeOut = new AppTimeoutInfo();
+    paramAppTimeOut.setExpiryTime(timeout);
+    // RemainingTime = Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0))
+    paramAppTimeOut.setRemainingTime(newLifetime);
+    paramAppTimeOut.setTimeoutType(ApplicationTimeoutType.LIFETIME);
+
+    Response response =
+        interceptor.updateApplicationTimeout(paramAppTimeOut, null, appId.toString());
+    Assert.assertNotNull(response);
+    AppTimeoutInfo entity = (AppTimeoutInfo) response.getEntity();
+    Assert.assertNotNull(entity);
+    Assert.assertEquals(paramAppTimeOut.getExpireTime(), entity.getExpireTime());
+    Assert.assertEquals(paramAppTimeOut.getTimeoutType(), entity.getTimeoutType());
+    Assert.assertEquals(paramAppTimeOut.getRemainingTimeInSec(), entity.getRemainingTimeInSec());
+  }
+
+  @Test
+  public void testUpdateApplicationPriority() throws IOException, InterruptedException,
+      YarnException {
+
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setPriority(20);
+
+    // Submit the application we are going to kill later
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    int iPriority = 10;
+    // Set Priority for application
+    Response response = interceptor.updateApplicationPriority(
+        new AppPriority(iPriority), null, appId.toString());
+
+    Assert.assertNotNull(response);
+    AppPriority entity = (AppPriority) response.getEntity();
+    Assert.assertNotNull(entity);
+    Assert.assertEquals(iPriority, entity.getPriority());
+  }
+
+  @Test
+  public void testGetAppPriority() throws IOException, InterruptedException,
+      YarnException {
+
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    int priority = 40;
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setPriority(priority);
+
+    // Submit the application we are going to kill later
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    // Set Priority for application
+    AppPriority appPriority = interceptor.getAppPriority(null, appId.toString());
+    Assert.assertNotNull(appPriority);
+    Assert.assertEquals(priority, appPriority.getPriority());
+  }
+
+  @Test
+  public void testUpdateAppQueue() throws IOException, InterruptedException,
+      YarnException {
+
+    String oldQueue = "oldQueue";
+    String newQueue = "newQueue";
+
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setQueue(oldQueue);
+
+    // Submit the application
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    // Set New Queue for application
+    Response response = interceptor.updateAppQueue(new AppQueue(newQueue),
+        null, appId.toString());
+
+    Assert.assertNotNull(response);
+    AppQueue appQueue = (AppQueue) response.getEntity();
+    Assert.assertEquals(newQueue, appQueue.getQueue());
+
+    // Get AppQueue by application
+    AppQueue queue = interceptor.getAppQueue(null, appId.toString());
+    Assert.assertNotNull(queue);
+    Assert.assertEquals(newQueue, queue.getQueue());
+  }
+
+  @Test
+  public void testGetAppQueue() throws IOException, InterruptedException, YarnException {
+    String queueName = "queueName";
+
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setQueue(queueName);
+
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    // Get Queue by application
+    AppQueue queue = interceptor.getAppQueue(null, appId.toString());
+    Assert.assertNotNull(queue);
+    Assert.assertEquals(queueName, queue.getQueue());
+  }
+
+  @Test
+  public void testGetAppsInfoCache() throws IOException, InterruptedException, YarnException {
+
+    AppsInfo responseGet = interceptor.getApps(
+        null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
+    Assert.assertNotNull(responseGet);
+
+    RouterAppInfoCacheKey cacheKey = RouterAppInfoCacheKey.newInstance(
+        null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
+
+    LRUCacheHashMap<RouterAppInfoCacheKey, AppsInfo> appsInfoCache =
+        interceptor.getAppInfosCaches();
+    Assert.assertNotNull(appsInfoCache);
+    Assert.assertTrue(!appsInfoCache.isEmpty());
+    Assert.assertEquals(1, appsInfoCache.size());
+    Assert.assertTrue(appsInfoCache.containsKey(cacheKey));
+
+    AppsInfo cacheResult = appsInfoCache.get(cacheKey);
+    Assert.assertNotNull(cacheResult);
+    Assert.assertEquals(responseGet, cacheResult);
+  }
+
+  @Test
+  public void testGetAppStatistics() throws IOException, InterruptedException, YarnException {
+    AppState appStateRUNNING = new AppState(YarnApplicationState.RUNNING.name());
+
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setApplicationType("MapReduce");
+    context.setQueue("queue");
+
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+
+    GetApplicationHomeSubClusterRequest request =
+        GetApplicationHomeSubClusterRequest.newInstance(appId);
+    GetApplicationHomeSubClusterResponse response =
+        stateStore.getApplicationHomeSubCluster(request);
+
+    Assert.assertNotNull(response);
+    ApplicationHomeSubCluster homeSubCluster = response.getApplicationHomeSubCluster();
+
+    DefaultRequestInterceptorREST interceptorREST =
+        interceptor.getInterceptorForSubCluster(homeSubCluster.getHomeSubCluster());
+
+    MockDefaultRequestInterceptorREST mockInterceptorREST =
+        (MockDefaultRequestInterceptorREST) interceptorREST;
+    mockInterceptorREST.updateApplicationState(YarnApplicationState.RUNNING,
+        appId.toString());
+
+    Set<String> stateQueries = new HashSet<>();
+    stateQueries.add(YarnApplicationState.RUNNING.name());
+
+    Set<String> typeQueries = new HashSet<>();
+    typeQueries.add("MapReduce");
+
+    ApplicationStatisticsInfo response2 =
+        interceptor.getAppStatistics(null, stateQueries, typeQueries);
+
+    Assert.assertNotNull(response2);
+    Assert.assertFalse(response2.getStatItems().isEmpty());
+
+    StatisticsItemInfo result = response2.getStatItems().get(0);
+    Assert.assertEquals(1, result.getCount());
+    Assert.assertEquals(YarnApplicationState.RUNNING, result.getState());
+    Assert.assertEquals("MapReduce", result.getType());
+  }
+
+  @Test
+  public void testGetAppActivities() throws IOException, InterruptedException {
+    // Submit application to multiSubCluster
+    ApplicationId appId = ApplicationId.newInstance(Time.now(), 1);
+    ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo();
+    context.setApplicationId(appId.toString());
+    context.setApplicationType("MapReduce");
+    context.setQueue("queue");
+
+    Assert.assertNotNull(interceptor.submitApplication(context, null));
+    Set<String> prioritiesSet = Collections.singleton("0");
+    Set<String> allocationRequestIdsSet = Collections.singleton("0");
+
+    AppActivitiesInfo appActivitiesInfo =
+        interceptor.getAppActivities(null, appId.toString(), String.valueOf(Time.now()),
+        prioritiesSet, allocationRequestIdsSet, null, "-1", null, false);
+
+    Assert.assertNotNull(appActivitiesInfo);
+    Assert.assertEquals(appId.toString(), appActivitiesInfo.getApplicationId());
+    Assert.assertEquals(10, appActivitiesInfo.getAllocations().size());
+  }
+
+  @Test
+  public void testListReservation() throws Exception {
+
+    // Add ReservationId In stateStore
+    ReservationId reservationId = ReservationId.newInstance(Time.now(), 1);
+    SubClusterId homeSubClusterId = subClusters.get(0);
+    ReservationHomeSubCluster reservationHomeSubCluster =
+        ReservationHomeSubCluster.newInstance(reservationId, homeSubClusterId);
+    AddReservationHomeSubClusterRequest request =
+        AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster);
+    stateStore.addReservationHomeSubCluster(request);
+
+    // Call the listReservation method
+    String applyReservationId = reservationId.toString();
+    Response listReservationResponse = interceptor.listReservation(
+        QUEUE_DEDICATED_FULL, applyReservationId, -1, -1, false, null);
+    Assert.assertNotNull(listReservationResponse);
+    Assert.assertNotNull(listReservationResponse.getStatus());
+    Status status = Status.fromStatusCode(listReservationResponse.getStatus());
+    Assert.assertEquals(Status.OK, status);
+
+    Object entity = listReservationResponse.getEntity();
+    Assert.assertNotNull(entity);
+    Assert.assertNotNull(entity instanceof ReservationListInfo);
+
+    ReservationListInfo listInfo = (ReservationListInfo) entity;
+    Assert.assertNotNull(listInfo);
+
+    List<ReservationInfo> reservationInfoList = listInfo.getReservations();
+    Assert.assertNotNull(reservationInfoList);
+    Assert.assertEquals(1, reservationInfoList.size());
+
+    ReservationInfo reservationInfo = reservationInfoList.get(0);
+    Assert.assertNotNull(reservationInfo);
+    Assert.assertEquals(applyReservationId, reservationInfo.getReservationId());
+
+    ReservationDefinitionInfo definitionInfo = reservationInfo.getReservationDefinition();
+    Assert.assertNotNull(definitionInfo);
+
+    ReservationRequestsInfo reservationRequestsInfo = definitionInfo.getReservationRequests();
+    Assert.assertNotNull(reservationRequestsInfo);
+
+    ArrayList<ReservationRequestInfo> reservationRequestInfoList =
+        reservationRequestsInfo.getReservationRequest();
+    Assert.assertNotNull(reservationRequestInfoList);
+    Assert.assertEquals(1, reservationRequestInfoList.size());
+
+    ReservationRequestInfo reservationRequestInfo = reservationRequestInfoList.get(0);
+    Assert.assertNotNull(reservationRequestInfo);
+    Assert.assertEquals(4, reservationRequestInfo.getNumContainers());
+
+    ResourceInfo resourceInfo = reservationRequestInfo.getCapability();
+    Assert.assertNotNull(resourceInfo);
+
+    int vCore = resourceInfo.getvCores();
+    long memory = resourceInfo.getMemorySize();
+    Assert.assertEquals(1, vCore);
+    Assert.assertEquals(1024, memory);
+  }
+
+  @Test
+  public void testWebAddressWithScheme() {
+    // The style of the web address reported by the subCluster in the heartbeat is 0.0.0.0:8000
+    // We design the following 2 test cases:
+    String webAppAddress = "0.0.0.0:8000";
+
+    // 1. We try to disable Https, at this point we should get the following link:
+    // http://0.0.0.0:8000
+    String expectedHttpWebAddress = "http://0.0.0.0:8000";
+    String webAppAddressWithScheme =
+        WebAppUtils.getHttpSchemePrefix(this.getConf()) + webAppAddress;
+    Assert.assertEquals(expectedHttpWebAddress, webAppAddressWithScheme);
+
+    // 2. We try to enable Https，at this point we should get the following link:
+    // https://0.0.0.0:8000
+    Configuration configuration = this.getConf();
+    configuration.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
+        HttpConfig.Policy.HTTPS_ONLY.name());
+    String expectedHttpsWebAddress = "https://0.0.0.0:8000";
+    String webAppAddressWithScheme2 =
+        WebAppUtils.getHttpSchemePrefix(this.getConf()) + webAppAddress;
+    Assert.assertEquals(expectedHttpsWebAddress, webAppAddressWithScheme2);
+  }
+
+  @Test
+  public void testCheckUserAccessToQueue() throws Exception {
+
+    // Case 1: Only queue admin user can access other user's information
+    HttpServletRequest mockHsr = mockHttpServletRequestByUserName("non-admin");
+    String errorMsg1 = "User=non-admin doesn't haven access to queue=queue " +
+        "so it cannot check ACLs for other users.";
+    LambdaTestUtils.intercept(YarnRuntimeException.class, errorMsg1,
+        () -> interceptor.checkUserAccessToQueue("queue", "jack",
+        QueueACL.SUBMIT_APPLICATIONS.name(), mockHsr));
+
+    // Case 2: request an unknown ACL causes BAD_REQUEST
+    HttpServletRequest mockHsr1 = mockHttpServletRequestByUserName("admin");
+    String errorMsg2 = "Specified queueAclType=XYZ_ACL is not a valid type, " +
+        "valid queue acl types={SUBMIT_APPLICATIONS/ADMINISTER_QUEUE}";
+    LambdaTestUtils.intercept(YarnRuntimeException.class, errorMsg2,
+        () -> interceptor.checkUserAccessToQueue("queue", "jack", "XYZ_ACL", mockHsr1));
+
+    // We design a test, admin user has ADMINISTER_QUEUE, SUBMIT_APPLICATIONS permissions,
+    // yarn user has SUBMIT_APPLICATIONS permissions, other users have no permissions
+
+    // Case 3: get FORBIDDEN for rejected ACL
+    checkUserAccessToQueueFailed("queue", "jack", QueueACL.SUBMIT_APPLICATIONS, "admin");
+    checkUserAccessToQueueFailed("queue", "jack", QueueACL.ADMINISTER_QUEUE, "admin");
+
+    // Case 4: get OK for listed ACLs
+    checkUserAccessToQueueSuccess("queue", "admin", QueueACL.ADMINISTER_QUEUE, "admin");
+    checkUserAccessToQueueSuccess("queue", "admin", QueueACL.SUBMIT_APPLICATIONS, "admin");
+
+    // Case 5: get OK only for SUBMIT_APP acl for "yarn" user
+    checkUserAccessToQueueFailed("queue", "yarn", QueueACL.ADMINISTER_QUEUE, "admin");
+    checkUserAccessToQueueSuccess("queue", "yarn", QueueACL.SUBMIT_APPLICATIONS, "admin");
+  }
+
+  private void checkUserAccessToQueueSuccess(String queue, String userName,
+      QueueACL queueACL, String mockUser) throws AuthorizationException {
+    HttpServletRequest mockHsr = mockHttpServletRequestByUserName(mockUser);
+    RMQueueAclInfo aclInfo =
+        interceptor.checkUserAccessToQueue(queue, userName, queueACL.name(), mockHsr);
+    Assert.assertNotNull(aclInfo);
+    Assert.assertTrue(aclInfo instanceof FederationRMQueueAclInfo);
+    FederationRMQueueAclInfo fedAclInfo = FederationRMQueueAclInfo.class.cast(aclInfo);
+    List<RMQueueAclInfo> aclInfos = fedAclInfo.getList();
+    Assert.assertNotNull(aclInfos);
+    Assert.assertEquals(4, aclInfos.size());
+    for (RMQueueAclInfo rMQueueAclInfo : aclInfos) {
+      Assert.assertTrue(rMQueueAclInfo.isAllowed());
+    }
+  }
+
+  private void checkUserAccessToQueueFailed(String queue, String userName,
+      QueueACL queueACL, String mockUser) throws AuthorizationException {
+    HttpServletRequest mockHsr = mockHttpServletRequestByUserName(mockUser);
+    RMQueueAclInfo aclInfo =
+        interceptor.checkUserAccessToQueue(queue, userName, queueACL.name(), mockHsr);
+    Assert.assertNotNull(aclInfo);
+    Assert.assertTrue(aclInfo instanceof FederationRMQueueAclInfo);
+    FederationRMQueueAclInfo fedAclInfo = FederationRMQueueAclInfo.class.cast(aclInfo);
+    List<RMQueueAclInfo> aclInfos = fedAclInfo.getList();
+    Assert.assertNotNull(aclInfos);
+    Assert.assertEquals(4, aclInfos.size());
+    for (RMQueueAclInfo rMQueueAclInfo : aclInfos) {
+      Assert.assertFalse(rMQueueAclInfo.isAllowed());
+      String expectDiagnostics = "User=" + userName +
+          " doesn't have access to queue=queue with acl-type=" + queueACL.name();
+      Assert.assertEquals(expectDiagnostics, rMQueueAclInfo.getDiagnostics());
+    }
+  }
+
+  private HttpServletRequest mockHttpServletRequestByUserName(String username) {
+    HttpServletRequest mockHsr = mock(HttpServletRequest.class);
+    when(mockHsr.getRemoteUser()).thenReturn(username);
+    Principal principal = mock(Principal.class);
+    when(principal.getName()).thenReturn(username);
+    when(mockHsr.getUserPrincipal()).thenReturn(principal);
+    return mockHsr;
+  }
+
+  @Test
+  public void testCheckFederationInterceptorRESTClient() {
+    SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
+    String webAppSocket = "SC-1:WebAddress";
+    String webAppAddress = "http://" + webAppSocket;
+
+    Configuration configuration = new Configuration();
+    FederationInterceptorREST rest = new FederationInterceptorREST();
+    rest.setConf(configuration);
+    rest.init("router");
+
+    DefaultRequestInterceptorREST interceptorREST =
+        rest.getOrCreateInterceptorForSubCluster(subClusterId, webAppSocket);
+
+    Assert.assertNotNull(interceptorREST);
+    Assert.assertNotNull(interceptorREST.getClient());
+    Assert.assertEquals(webAppAddress, interceptorREST.getWebAppAddress());
+  }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java
index e563d5a58204e..e2b2103c7deb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java
@@ -25,9 +25,11 @@
 
 import javax.ws.rs.core.Response;
 
+import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
 import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
 import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
@@ -40,7 +42,6 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
 import org.apache.hadoop.yarn.server.router.clientrm.PassThroughClientRequestInterceptor;
 import org.apache.hadoop.yarn.server.router.clientrm.TestableFederationClientInterceptor;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -139,8 +140,8 @@ protected YarnConfiguration createConfiguration() {
     String mockPassThroughInterceptorClass =
         PassThroughClientRequestInterceptor.class.getName();
 
-    // Create a request intercepter pipeline for testing. The last one in the
-    // chain is the federation intercepter that calls the mock resource manager.
+    // Create a request interceptor pipeline for testing. The last one in the
+    // chain is the federation interceptor that calls the mock resource manager.
     // The others in the chain will simply forward it to the next one in the
     // chain
     conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE,
@@ -179,6 +180,9 @@ public void testGetNewApplicationOneBadSC()
   @Test
   public void testGetNewApplicationTwoBadSCs()
       throws YarnException, IOException, InterruptedException {
+
+    LOG.info("Test getNewApplication with two bad SCs.");
+
     setupCluster(Arrays.asList(bad1, bad2));
 
     Response response = interceptor.createNewApplication(null);
@@ -194,17 +198,21 @@ public void testGetNewApplicationTwoBadSCs()
   @Test
   public void testGetNewApplicationOneBadOneGood()
       throws YarnException, IOException, InterruptedException {
-    System.out.println("Test getNewApplication with one bad, one good SC");
+
+    LOG.info("Test getNewApplication with one bad, one good SC.");
+
     setupCluster(Arrays.asList(good, bad2));
     Response response = interceptor.createNewApplication(null);
-
+    Assert.assertNotNull(response);
     Assert.assertEquals(OK, response.getStatus());
 
     NewApplication newApp = (NewApplication) response.getEntity();
+    Assert.assertNotNull(newApp);
+
     ApplicationId appId = ApplicationId.fromString(newApp.getApplicationId());
+    Assert.assertNotNull(appId);
 
-    Assert.assertEquals(Integer.parseInt(good.getId()),
-        appId.getClusterTimestamp());
+    Assert.assertEquals(Integer.parseInt(good.getId()), appId.getClusterTimestamp());
   }
 
   /**
@@ -215,6 +223,8 @@ public void testGetNewApplicationOneBadOneGood()
   public void testSubmitApplicationOneBadSC()
       throws YarnException, IOException, InterruptedException {
 
+    LOG.info("Test submitApplication with one bad SC.");
+
     setupCluster(Arrays.asList(bad2));
 
     ApplicationId appId =
@@ -377,15 +387,12 @@ public void testGetNodeOneBadOneGood()
    * composed of only 1 bad SubCluster.
    */
   @Test
-  public void testGetNodesOneBadSC()
-      throws YarnException, IOException, InterruptedException {
+  public void testGetNodesOneBadSC() throws Exception {
 
     setupCluster(Arrays.asList(bad2));
 
-    NodesInfo response = interceptor.getNodes(null);
-    Assert.assertNotNull(response);
-    Assert.assertEquals(0, response.getNodes().size());
-    // The remove duplicate operations is tested in TestRouterWebServiceUtil
+    LambdaTestUtils.intercept(YarnRuntimeException.class, "RM is stopped",
+        () -> interceptor.getNodes(null));
   }
 
   /**
@@ -393,14 +400,12 @@ public void testGetNodesOneBadSC()
    * composed of only 2 bad SubClusters.
    */
   @Test
-  public void testGetNodesTwoBadSCs()
-      throws YarnException, IOException, InterruptedException {
+  public void testGetNodesTwoBadSCs() throws Exception {
+
     setupCluster(Arrays.asList(bad1, bad2));
 
-    NodesInfo response = interceptor.getNodes(null);
-    Assert.assertNotNull(response);
-    Assert.assertEquals(0, response.getNodes().size());
-    // The remove duplicate operations is tested in TestRouterWebServiceUtil
+    LambdaTestUtils.intercept(YarnRuntimeException.class, "RM is stopped",
+        () -> interceptor.getNodes(null));
   }
 
   /**
@@ -408,17 +413,11 @@ public void testGetNodesTwoBadSCs()
    * composed of only 1 bad SubCluster and a good one.
    */
   @Test
-  public void testGetNodesOneBadOneGood()
-      throws YarnException, IOException, InterruptedException {
+  public void testGetNodesOneBadOneGood() throws Exception {
     setupCluster(Arrays.asList(good, bad2));
 
-    NodesInfo response = interceptor.getNodes(null);
-    Assert.assertNotNull(response);
-    Assert.assertEquals(1, response.getNodes().size());
-    // Check if the only node came from Good SubCluster
-    Assert.assertEquals(good.getId(),
-        Long.toString(response.getNodes().get(0).getLastHealthUpdate()));
-    // The remove duplicate operations is tested in TestRouterWebServiceUtil
+    LambdaTestUtils.intercept(YarnRuntimeException.class, "RM is stopped",
+        () -> interceptor.getNodes(null));
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
new file mode 100644
index 0000000000000..f1501fe1e7a24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.router.Router;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class TestFederationWebApp extends TestRouterWebServicesREST {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFederationWebApp.class);
+
+  @Test
+  public void testFederationWebViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationWebView - NotEnable Federation.");
+    // Test Federation is not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationWebViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationWebView - Enable Federation.");
+    // Test Federation Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(FederationPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationAboutViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationAboutViewEnable - Enable Federation.");
+    // Test Federation Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationAboutViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationAboutViewNotEnable - NotEnable Federation.");
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationNodeViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationNodeViewEnable - Enable Federation.");
+    // Test Federation Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationNodeViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationNodeViewNotEnable - NotEnable Federation.");
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationAppViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationAppViewEnable - Enable Federation.");
+    // Test Federation Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testFederationAppViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    LOG.info("testFederationAppViewNotEnable - NotEnable Federation.");
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testNodeLabelAppViewNotEnable()
+      throws InterruptedException, YarnException, IOException {
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+    WebAppTests.testPage(NodeLabelsPage.class, Router.class, new MockRouter(config));
+  }
+
+  @Test
+  public void testNodeLabelAppViewEnable()
+      throws InterruptedException, YarnException, IOException {
+    // Test Federation Not Enabled
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
+    WebAppTests.testPage(NodeLabelsPage.class, Router.class, new MockRouter(config));
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804d82eda..96a6881adc6d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -30,12 +30,18 @@
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
 import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
 /**
  * Test class to validate RouterWebServiceUtil methods.
  */
@@ -579,4 +585,97 @@ private void setUpClusterMetrics(ClusterMetricsInfo metrics, long seed) {
     metrics.setActiveNodes(rand.nextInt(1000));
     metrics.setShutdownNodes(rand.nextInt(1000));
   }
+
+  public static AppAttemptInfo generateAppAttemptInfo(int attemptId) {
+    AppAttemptInfo appAttemptInfo = mock(AppAttemptInfo.class);
+    when(appAttemptInfo.getAppAttemptId()).thenReturn("AppAttemptId_" + attemptId);
+    when(appAttemptInfo.getAttemptId()).thenReturn(0);
+    when(appAttemptInfo.getFinishedTime()).thenReturn(1659621705L);
+    when(appAttemptInfo.getLogsLink()).thenReturn("LogLink_" + attemptId);
+    return appAttemptInfo;
+  }
+
+  @Test
+  public void testMergeApplicationStatisticsInfo() {
+    ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo();
+    ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo();
+
+    StatisticsItemInfo item1 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10);
+    StatisticsItemInfo item2 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 20);
+
+    infoA.add(item1);
+    infoB.add(item2);
+
+    List<ApplicationStatisticsInfo> lists = new ArrayList<>();
+    lists.add(infoA);
+    lists.add(infoB);
+
+    ApplicationStatisticsInfo mergeInfo =
+        RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists);
+    ArrayList<StatisticsItemInfo> statItem = mergeInfo.getStatItems();
+
+    Assert.assertNotNull(statItem);
+    Assert.assertEquals(1, statItem.size());
+
+    StatisticsItemInfo first = statItem.get(0);
+
+    Assert.assertEquals(item1.getCount() + item2.getCount(), first.getCount());
+    Assert.assertEquals(item1.getType(), first.getType());
+    Assert.assertEquals(item1.getState(), first.getState());
+  }
+
+  @Test
+  public void testMergeDiffApplicationStatisticsInfo() {
+    ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo();
+    StatisticsItemInfo item1 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10);
+    StatisticsItemInfo item2 =
+        new StatisticsItemInfo(YarnApplicationState.NEW_SAVING, "test1", 20);
+    infoA.add(item1);
+    infoA.add(item2);
+
+    ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo();
+    StatisticsItemInfo item3 =
+        new StatisticsItemInfo(YarnApplicationState.NEW_SAVING, "test1", 30);
+    StatisticsItemInfo item4 = new StatisticsItemInfo(YarnApplicationState.FINISHED, "test3", 40);
+    infoB.add(item3);
+    infoB.add(item4);
+
+    List<ApplicationStatisticsInfo> lists = new ArrayList<>();
+    lists.add(infoA);
+    lists.add(infoB);
+
+    ApplicationStatisticsInfo mergeInfo =
+        RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists);
+
+    Assert.assertEquals(3, mergeInfo.getStatItems().size());
+    List<StatisticsItemInfo> mergeInfoStatItems = mergeInfo.getStatItems();
+
+    StatisticsItemInfo item1Result = null;
+    StatisticsItemInfo item2Result = null;
+    StatisticsItemInfo item3Result = null;
+
+    for (StatisticsItemInfo item : mergeInfoStatItems) {
+      // ACCEPTED
+      if (item.getState() == YarnApplicationState.ACCEPTED) {
+        item1Result = item;
+      }
+
+      // NEW_SAVING
+      if (item.getState() == YarnApplicationState.NEW_SAVING) {
+        item2Result = item;
+      }
+
+      // FINISHED
+      if (item.getState() == YarnApplicationState.FINISHED) {
+        item3Result = item;
+      }
+    }
+
+    Assert.assertEquals(YarnApplicationState.ACCEPTED, item1Result.getState());
+    Assert.assertEquals(item1.getCount(), item1Result.getCount());
+    Assert.assertEquals(YarnApplicationState.NEW_SAVING, item2Result.getState());
+    Assert.assertEquals((item2.getCount() + item3.getCount()), item2Result.getCount());
+    Assert.assertEquals(YarnApplicationState.FINISHED, item3Result.getState());
+    Assert.assertEquals(item4.getCount(), item3Result.getCount());
+  }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java
index 14652435dac29..5bf8db04b06cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java
@@ -132,7 +132,7 @@ public void testRouterWebServicesE2E() throws Exception {
     Response response4 = addToClusterNodeLabels(user);
     Assert.assertNotNull(response4);
 
-    Response response5 = removeFromCluserNodeLabels(user);
+    Response response5 = removeFromClusterNodeLabels(user);
     Assert.assertNotNull(response5);
 
     NodeLabelsInfo nodeLabelsInfo2 = getLabelsOnNode(user);
@@ -200,6 +200,12 @@ public void testRouterWebServicesE2E() throws Exception {
 
     ContainerInfo containerInfo = getContainer(user);
     Assert.assertNotNull(containerInfo);
+
+    Response response19 = updateSchedulerConfiguration(user);
+    Assert.assertNotNull(response19);
+
+    Response response20 = getSchedulerConfiguration(user);
+    Assert.assertNotNull(response20);
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java
index d3c619860a9cf..7eb17ef6cd2eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java
@@ -20,7 +20,7 @@
 
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
 import static javax.ws.rs.core.MediaType.APPLICATION_XML;
@@ -78,6 +78,7 @@
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutorCompletionService;
@@ -142,6 +143,10 @@
 import com.sun.jersey.api.client.WebResource.Builder;
 
 import net.jcip.annotations.NotThreadSafe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
 
 /**
  * This test validate E2E the correctness of the RouterWebServices. It starts
@@ -155,6 +160,8 @@ public class TestRouterWebServicesREST {
   /** The number of concurrent submissions for multi-thread test. */
   private static final int NUM_THREADS_TESTS = 100;
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRouterWebServicesREST.class);
 
   private static String userName = "test";
 
@@ -194,7 +201,7 @@ public Boolean get() {
           }
           return false;
         }
-      }, 1000, 10 * 1000);
+      }, 1000, 20 * 1000);
     } catch (Exception e) {
       fail("Web app not running");
     }
@@ -423,7 +430,7 @@ public void testSchedulerInfoXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getNodes()} inside Router.
+   * {@link RMWebServiceProtocol#getNodes(String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testNodesEmptyXML() throws Exception {
@@ -444,7 +451,7 @@ public void testNodesEmptyXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getNodes()} inside Router.
+   * {@link RMWebServiceProtocol#getNodes(String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testNodesXML() throws Exception {
@@ -465,7 +472,7 @@ public void testNodesXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getNode()} inside Router.
+   * {@link RMWebServiceProtocol#getNode(String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testNodeXML() throws Exception {
@@ -487,7 +494,7 @@ public void testNodeXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateNodeResources()} inside Router.
+   * {@link RMWebServiceProtocol#updateNodeResource} inside Router.
    */
   @Test
   public void testUpdateNodeResource() throws Exception {
@@ -528,7 +535,7 @@ public void testUpdateNodeResource() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getActivities()} inside Router.
+   * {@link RMWebServiceProtocol#getActivities(HttpServletRequest, String, String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testActiviesXML() throws Exception {
@@ -546,7 +553,7 @@ public void testActiviesXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppActivities()} inside Router.
+   * {@link RMWebServiceProtocol#getAppActivities} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppActivitiesXML() throws Exception {
@@ -566,7 +573,7 @@ public void testAppActivitiesXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppStatistics()} inside Router.
+   * {@link RMWebServiceProtocol#getAppStatistics} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppStatisticsXML() throws Exception {
@@ -590,7 +597,7 @@ public void testAppStatisticsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#dumpSchedulerLogs()} inside Router.
+   * {@link RMWebServiceProtocol#dumpSchedulerLogs} inside Router.
    */
   @Test(timeout = 2000)
   public void testDumpSchedulerLogsXML() throws Exception {
@@ -600,7 +607,7 @@ public void testDumpSchedulerLogsXML() throws Exception {
         performCall(RM_WEB_SERVICE_PATH + SCHEDULER_LOGS,
             null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     ClientResponse response = performCall(
@@ -613,7 +620,7 @@ public void testDumpSchedulerLogsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#createNewApplication()} inside Router.
+   * {@link RMWebServiceProtocol#createNewApplication} inside Router.
    */
   @Test(timeout = 2000)
   public void testNewApplicationXML() throws Exception {
@@ -623,7 +630,7 @@ public void testNewApplicationXML() throws Exception {
         RM_WEB_SERVICE_PATH + APPS_NEW_APPLICATION, null,
         null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     ClientResponse response = performCall(
@@ -637,7 +644,7 @@ public void testNewApplicationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#submitApplication()} inside Router.
+   * {@link RMWebServiceProtocol#submitApplication} inside Router.
    */
   @Test(timeout = 2000)
   public void testSubmitApplicationXML() throws Exception {
@@ -646,7 +653,7 @@ public void testSubmitApplicationXML() throws Exception {
     ClientResponse badResponse = performCall(
         RM_WEB_SERVICE_PATH + APPS, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     ApplicationSubmissionContextInfo context =
@@ -663,7 +670,7 @@ public void testSubmitApplicationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getApps()} inside Router.
+   * {@link RMWebServiceProtocol#getApps} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppsXML() throws Exception {
@@ -686,7 +693,7 @@ public void testAppsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getApp()} inside Router.
+   * {@link RMWebServiceProtocol#getApp} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppXML() throws Exception {
@@ -710,7 +717,7 @@ public void testAppXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppAttempts()} inside Router.
+   * {@link RMWebServiceProtocol#getAppAttempts} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppAttemptXML() throws Exception {
@@ -734,7 +741,7 @@ public void testAppAttemptXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppState()} inside Router.
+   * {@link RMWebServiceProtocol#getAppState} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppStateXML() throws Exception {
@@ -758,7 +765,7 @@ public void testAppStateXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateAppState()} inside Router.
+   * {@link RMWebServiceProtocol#updateAppState} inside Router.
    */
   @Test(timeout = 2000)
   public void testUpdateAppStateXML() throws Exception {
@@ -771,7 +778,7 @@ public void testUpdateAppStateXML() throws Exception {
     ClientResponse badResponse = performCall(
         pathApp, null, null, null, POST);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     AppState appState = new AppState("KILLED");
@@ -786,7 +793,7 @@ public void testUpdateAppStateXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppPriority()} inside Router.
+   * {@link RMWebServiceProtocol#getAppPriority} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppPriorityXML() throws Exception {
@@ -808,7 +815,8 @@ public void testAppPriorityXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateApplicationPriority()} inside Router.
+   * {@link RMWebServiceProtocol#updateApplicationPriority(
+   *     AppPriority, HttpServletRequest, String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testUpdateAppPriorityXML() throws Exception {
@@ -820,7 +828,7 @@ public void testUpdateAppPriorityXML() throws Exception {
         RM_WEB_SERVICE_PATH + format(APPS_APPID_PRIORITY, appId),
         null, null, null, POST);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     AppPriority appPriority = new AppPriority(1);
@@ -836,7 +844,7 @@ public void testUpdateAppPriorityXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppQueue()} inside Router.
+   * {@link RMWebServiceProtocol#getAppQueue(HttpServletRequest, String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppQueueXML() throws Exception {
@@ -858,7 +866,8 @@ public void testAppQueueXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateAppQueue()} inside Router.
+   * {@link RMWebServiceProtocol#updateAppQueue(AppQueue, HttpServletRequest, String)}
+   * inside Router.
    */
   @Test(timeout = 2000)
   public void testUpdateAppQueueXML() throws Exception {
@@ -870,7 +879,7 @@ public void testUpdateAppQueueXML() throws Exception {
         RM_WEB_SERVICE_PATH + format(APPS_APPID_QUEUE, appId),
         null, null, null, POST);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     AppQueue appQueue = new AppQueue("default");
@@ -886,7 +895,7 @@ public void testUpdateAppQueueXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppTimeouts()} inside Router.
+   * {@link RMWebServiceProtocol#getAppTimeouts} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppTimeoutsXML() throws Exception {
@@ -910,7 +919,7 @@ public void testAppTimeoutsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getAppTimeout()} inside Router.
+   * {@link RMWebServiceProtocol#getAppTimeout} inside Router.
    */
   @Test(timeout = 2000)
   public void testAppTimeoutXML() throws Exception {
@@ -933,7 +942,8 @@ public void testAppTimeoutXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateApplicationTimeout()} inside Router.
+   * {@link RMWebServiceProtocol#updateApplicationTimeout}
+   * inside Router.
    */
   @Test(timeout = 2000)
   public void testUpdateAppTimeoutsXML() throws Exception {
@@ -945,7 +955,7 @@ public void testUpdateAppTimeoutsXML() throws Exception {
         RM_WEB_SERVICE_PATH + format(APPS_TIMEOUT, appId),
         null, null, null, POST);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with a bad request
     AppTimeoutInfo appTimeoutInfo = new AppTimeoutInfo();
@@ -961,7 +971,7 @@ public void testUpdateAppTimeoutsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#createNewReservation()} inside Router.
+   * {@link RMWebServiceProtocol#createNewReservation(HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testNewReservationXML() throws Exception {
@@ -971,7 +981,7 @@ public void testNewReservationXML() throws Exception {
         RM_WEB_SERVICE_PATH + RESERVATION_NEW,
         null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     ClientResponse response = performCall(
@@ -985,7 +995,8 @@ public void testNewReservationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#submitReservation()} inside Router.
+   * {@link RMWebServiceProtocol#submitReservation(
+   *     ReservationSubmissionRequestInfo, HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testSubmitReservationXML() throws Exception {
@@ -995,7 +1006,7 @@ public void testSubmitReservationXML() throws Exception {
         RM_WEB_SERVICE_PATH + RESERVATION_SUBMIT, null,
         null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     ReservationSubmissionRequestInfo context =
@@ -1013,7 +1024,8 @@ public void testSubmitReservationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#updateReservation()} inside Router.
+   * {@link RMWebServiceProtocol#updateReservation(
+   *     ReservationUpdateRequestInfo, HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testUpdateReservationXML() throws Exception {
@@ -1022,7 +1034,7 @@ public void testUpdateReservationXML() throws Exception {
     ClientResponse badResponse = performCall(
         RM_WEB_SERVICE_PATH + RESERVATION_UPDATE, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     String reservationId = getNewReservationId().getReservationId();
@@ -1039,7 +1051,8 @@ public void testUpdateReservationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#deleteReservation()} inside Router.
+   * {@link RMWebServiceProtocol#deleteReservation(
+   *     ReservationDeleteRequestInfo, HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testDeleteReservationXML() throws Exception {
@@ -1048,7 +1061,7 @@ public void testDeleteReservationXML() throws Exception {
     ClientResponse badResponse = performCall(
         RM_WEB_SERVICE_PATH + RESERVATION_DELETE, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     String reservationId = getNewReservationId().getReservationId();
@@ -1065,7 +1078,7 @@ public void testDeleteReservationXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getNodeToLabels()} inside Router.
+   * {@link RMWebServiceProtocol#getNodeToLabels(HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testGetNodeToLabelsXML() throws Exception {
@@ -1087,7 +1100,7 @@ public void testGetNodeToLabelsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getClusterNodeLabels()} inside Router.
+   * {@link RMWebServiceProtocol#getClusterNodeLabels(HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testGetClusterNodeLabelsXML() throws Exception {
@@ -1109,7 +1122,7 @@ public void testGetClusterNodeLabelsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getLabelsOnNode()} inside Router.
+   * {@link RMWebServiceProtocol#getLabelsOnNode(HttpServletRequest, String)} inside Router.
    */
   @Test(timeout = 2000)
   public void testGetLabelsOnNodeXML() throws Exception {
@@ -1131,7 +1144,7 @@ public void testGetLabelsOnNodeXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getLabelsToNodes()} inside Router.
+   * {@link RMWebServiceProtocol#getLabelsToNodes(Set<String>)} inside Router.
    */
   @Test(timeout = 2000)
   public void testGetLabelsMappingEmptyXML() throws Exception {
@@ -1153,7 +1166,7 @@ public void testGetLabelsMappingEmptyXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#getLabelsToNodes()} inside Router.
+   * {@link RMWebServiceProtocol#getLabelsToNodes(Set<String>)} inside Router.
    */
   @Test(timeout = 2000)
   public void testGetLabelsMappingXML() throws Exception {
@@ -1175,7 +1188,8 @@ public void testGetLabelsMappingXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#addToClusterNodeLabels()} inside Router.
+   * {@link RMWebServiceProtocol#addToClusterNodeLabels(
+   *     NodeLabelsInfo, HttpServletRequest)} inside Router.
    */
   @Test(timeout = 2000)
   public void testAddToClusterNodeLabelsXML() throws Exception {
@@ -1185,7 +1199,7 @@ public void testAddToClusterNodeLabelsXML() throws Exception {
         RM_WEB_SERVICE_PATH + ADD_NODE_LABELS,
         null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
 
@@ -1203,17 +1217,17 @@ public void testAddToClusterNodeLabelsXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#removeFromCluserNodeLabels()} inside Router.
+   * {@link RMWebServiceProtocol#removeFromClusterNodeLabels} inside Router.
    */
   @Test(timeout = 2000)
-  public void testRemoveFromCluserNodeLabelsXML()
+  public void testRemoveFromClusterNodeLabelsXML()
       throws Exception {
 
     // Test with a wrong HTTP method
     ClientResponse badResponse = performCall(
         RM_WEB_SERVICE_PATH + REMOVE_NODE_LABELS, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     addNodeLabel();
@@ -1229,7 +1243,7 @@ public void testRemoveFromCluserNodeLabelsXML()
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#replaceLabelsOnNodes()} inside Router.
+   * {@link RMWebServiceProtocol#replaceLabelsOnNodes} inside Router.
    */
   @Test(timeout = 2000)
   public void testReplaceLabelsOnNodesXML() throws Exception {
@@ -1238,7 +1252,7 @@ public void testReplaceLabelsOnNodesXML() throws Exception {
     ClientResponse badResponse = performCall(
         RM_WEB_SERVICE_PATH + REPLACE_NODE_TO_LABELS, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     addNodeLabel();
@@ -1256,7 +1270,7 @@ public void testReplaceLabelsOnNodesXML() throws Exception {
 
   /**
    * This test validates the correctness of
-   * {@link RMWebServiceProtocol#replaceLabelsOnNode()} inside Router.
+   * {@link RMWebServiceProtocol#replaceLabelsOnNode} inside Router.
    */
   @Test(timeout = 2000)
   public void testReplaceLabelsOnNodeXML() throws Exception {
@@ -1267,7 +1281,7 @@ public void testReplaceLabelsOnNodeXML() throws Exception {
     ClientResponse badResponse = performCall(
         pathNode, null, null, null, PUT);
 
-    assertEquals(SC_INTERNAL_SERVER_ERROR, badResponse.getStatus());
+    assertEquals(SC_SERVICE_UNAVAILABLE, badResponse.getStatus());
 
     // Test with the correct HTTP method
     addNodeLabel();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capability b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capability
new file mode 100644
index 0000000000000..e6cf4a1cd03bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capability
@@ -0,0 +1,21 @@
+{
+    "___asflicense__": [
+         "",
+         "Licensed to the Apache Software Foundation (ASF) under one",
+         "or more contributor license agreements.  See the NOTICE file",
+         "distributed with this work for additional information",
+         "regarding copyright ownership.  The ASF licenses this file",
+         "to you under the Apache License, Version 2.0 (the",
+         "\"License\"); you may not use this file except in compliance",
+         "with the License.  You may obtain a copy of the License at",
+         "",
+         "     http://www.apache.org/licenses/LICENSE-2.0",
+         "",
+         "Unless required by applicable law or agreed to in writing, software",
+         "distributed under the License is distributed on an \"AS IS\" BASIS,",
+         "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.",
+         "See the License for the specific language governing permissions and",
+         "limitations under the License."
+     ],
+     "clusterMetrics":{"appsSubmitted":"0","appsCompleted":"0","appsPending":"0","appsRunning":"0","appsFailed":"0","appsKilled":"0","reservedMB":"0","availableMB":"4096","allocatedMB":"0","pendingMB":"0","reservedVirtualCores":"0","availableVirtualCores":"4","allocatedVirtualCores":"0","pendingVirtualCores":"0","containersAllocated":"0","containersReserved":"0","containersPending":"0","totalMB":"4096","totalVirtualCores":"4","utilizedMBPercent":"0","utilizedVirtualCoresPercent":"0","rmSchedulerBusyPercent":"-1","totalNodes":"1","lostNodes":"0","unhealthyNodes":"0","decommissioningNodes":"0","decommissionedNodes":"0","rebootedNodes":"0","activeNodes":"1","shutdownNodes":"0","containerAssignedPerSecond":"0","totalUsedResourcesAcrossPartition":{"memory":"0","vCores":"0","resourceInformations":{"resourceInformation":[{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"memory-mb","resourceType":"COUNTABLE","units":"Mi","value":"0"},{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"vcores","resourceType":"COUNTABLE","units":"","value":"0"}]}},"totalClusterResourcesAcrossPartition":{"memory":"4096","vCores":"4","resourceInformations":{"resourceInformation":[{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"memory-mb","resourceType":"COUNTABLE","units":"Mi","value":"4096"},{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"vcores","resourceType":"COUNTABLE","units":"","value":"4"}]}},"totalReservedResourcesAcrossPartition":{"memory":"0","vCores":"0","resourceInformations":{"resourceInformation":[{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"memory-mb","resourceType":"COUNTABLE","units":"Mi","value":"0"},{"attributes":null,"maximumAllocation":"9223372036854775807","minimumAllocation":"0","name":"vcores","resourceType":"COUNTABLE","units":"","value":"0"}]}},"totalAllocatedContainersAcrossPartition":"0","crossPartitionMetricsAvailable":"true","rmEventQueueSize":"0","schedulerEventQueueSize":"0"}
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml
index 90c5eeb097e8b..a8487f3f3c00a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml
@@ -45,7 +45,7 @@
 
   <property>
     <name>yarn.scheduler.capacity.root.queues</name>
-    <value>default</value>
+    <value>default,decided,target</value>
     <description>
       The queues at the this level (root is the root queue).
     </description>
@@ -53,10 +53,22 @@
 
   <property>
     <name>yarn.scheduler.capacity.root.default.capacity</name>
-    <value>100</value>
+    <value>50</value>
     <description>Default queue target capacity.</description>
   </property>
 
+  <property>
+    <name>yarn.scheduler.capacity.root.target.capacity</name>
+    <value>10</value>
+    <description>target queue capacity.</description>
+  </property>
+
+  <property>
+    <name>yarn.scheduler.capacity.root.decided.capacity</name>
+    <value>40</value>
+    <description>decided queue capacity.</description>
+  </property>
+
   <property>
     <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
     <value>1</value>
@@ -81,6 +93,14 @@
     </description>
   </property>
 
+  <property>
+    <name>yarn.scheduler.capacity.root.target.state</name>
+    <value>RUNNING</value>
+    <description>
+      The state of the target queue. State can be one of RUNNING or STOPPED.
+    </description>
+  </property>
+
   <property>
     <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
     <value>*</value>
@@ -89,6 +109,14 @@
     </description>
   </property>
 
+  <property>
+    <name>yarn.scheduler.capacity.root.target.acl_submit_applications</name>
+    <value>*</value>
+    <description>
+      The ACL of who can submit jobs to the target queue.
+    </description>
+  </property>
+
   <property>
     <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
     <value>*</value>
@@ -97,6 +125,23 @@
     </description>
   </property>
 
+  <property>
+    <name>yarn.scheduler.capacity.root.target.acl_administer_queue</name>
+    <value>*</value>
+    <description>
+      The ACL of who can administer jobs on the target queue.
+    </description>
+  </property>
+
+  <property>
+    <name>yarn.scheduler.capacity.root.decided.reservable</name>
+    <value>true</value>
+    <description>
+      indicates to the ReservationSystem that the queue’s resources
+      is available for users to reserve.
+    </description>
+  </property>
+
   <property>
     <name>yarn.scheduler.capacity.node-locality-delay</name>
     <value>-1</value>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json
new file mode 100644
index 0000000000000..8485ab6f3d550
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json
@@ -0,0 +1,24 @@
+{
+    "___asflicense__": [
+        "",
+        "Licensed to the Apache Software Foundation (ASF) under one",
+        "or more contributor license agreements.  See the NOTICE file",
+        "distributed with this work for additional information",
+        "regarding copyright ownership.  The ASF licenses this file",
+        "to you under the Apache License, Version 2.0 (the",
+        "\"License\"); you may not use this file except in compliance",
+        "with the License.  You may obtain a copy of the License at",
+        "",
+        "     http://www.apache.org/licenses/LICENSE-2.0",
+        "",
+        "Unless required by applicable law or agreed to in writing, software",
+        "distributed under the License is distributed on an \"AS IS\" BASIS,",
+        "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.",
+        "See the License for the specific language governing permissions and",
+        "limitations under the License."
+    ],
+    "default" : {
+        "memory-mb" : 2048,
+        "vcores" : 2
+    }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
index f3e0de3604b60..84d4171c79cd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
@@ -27,4 +27,16 @@
     <name>yarn.resourcemanager.webapp.address</name>
     <value>0.0.0.0:8080</value>
   </property>
+  <property>
+    <name>yarn.cluster.max-application-priority</name>
+    <value>50</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.resource-profiles.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.resource-profiles.source-file</name>
+    <value>profiles/sample-profiles-1.json</value>
+  </property>
 </configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
index fd429a5f93e67..971fb0941a3ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml
@@ -49,11 +49,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-client</artifactId>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
@@ -71,6 +66,21 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java
index 23c80d9655dab..019ef67bd6a8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java
@@ -18,25 +18,26 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager;
 
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
-import org.junit.Test;
+
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 public class TestCleanerTask {
   private static final String ROOT =
@@ -47,7 +48,7 @@ public class TestCleanerTask {
       YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL;
 
   @Test
-  public void testNonExistentRoot() throws Exception {
+  void testNonExistentRoot() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     CleanerMetrics metrics = mock(CleanerMetrics.class);
     SCMStore store = mock(SCMStore.class);
@@ -64,7 +65,7 @@ public void testNonExistentRoot() throws Exception {
   }
 
   @Test
-  public void testProcessFreshResource() throws Exception {
+  void testProcessFreshResource() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     CleanerMetrics metrics = mock(CleanerMetrics.class);
     SCMStore store = mock(SCMStore.class);
@@ -89,7 +90,7 @@ public void testProcessFreshResource() throws Exception {
   }
 
   @Test
-  public void testProcessEvictableResource() throws Exception {
+  void testProcessEvictableResource() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     CleanerMetrics metrics = mock(CleanerMetrics.class);
     SCMStore store = mock(SCMStore.class);
@@ -125,7 +126,7 @@ private CleanerTask createSpiedTask(FileSystem fs, SCMStore store,
   }
 
   @Test
-  public void testResourceIsInUseHasAnActiveApp() throws Exception {
+  void testResourceIsInUseHasAnActiveApp() throws Exception {
     FileSystem fs = mock(FileSystem.class);
     CleanerMetrics metrics = mock(CleanerMetrics.class);
     SCMStore store = mock(SCMStore.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
index ca4bdce7cf19d..314332798ab97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java
@@ -18,15 +18,16 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.spy;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -42,11 +43,11 @@
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SharedCacheResourceReference;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.spy;
 
 
 /**
@@ -55,7 +56,7 @@
 public class TestClientSCMProtocolService {
   private static File testDir = null;
 
-  @BeforeClass
+  @BeforeAll
   public static void setupTestDirs() throws IOException {
     testDir = new File("target",
         TestSharedCacheUploaderService.class.getCanonicalName());
@@ -64,7 +65,7 @@ public static void setupTestDirs() throws IOException {
     testDir = testDir.getAbsoluteFile();
   }
 
-  @AfterClass
+  @AfterAll
   public static void cleanupTestDirs() throws IOException {
     if (testDir != null) {
       testDir.delete();
@@ -78,7 +79,7 @@ public static void cleanupTestDirs() throws IOException {
   private final RecordFactory recordFactory = RecordFactoryProvider
       .getRecordFactory(null);
 
-  @Before
+  @BeforeEach
   public void startUp() {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.SCM_STORE_CLASS,
@@ -105,7 +106,7 @@ public void startUp() {
             conf);
   }
 
-  @After
+  @AfterEach
   public void cleanUp() {
     if (store != null) {
       store.stop();
@@ -124,19 +125,19 @@ public void cleanUp() {
   }
 
   @Test
-  public void testUse_MissingEntry() throws Exception {
+  void testUse_MissingEntry() throws Exception {
     long misses = ClientSCMMetrics.getInstance().getCacheMisses();
     UseSharedCacheResourceRequest request =
         recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class);
     request.setResourceKey("key1");
     request.setAppId(createAppId(1, 1L));
     assertNull(clientSCMProxy.use(request).getPath());
-    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
-        .getInstance().getCacheMisses() - misses);
+    assertEquals(1, ClientSCMMetrics
+        .getInstance().getCacheMisses() - misses, "Client SCM metrics aren't updated.");
   }
 
   @Test
-  public void testUse_ExistingEntry_NoAppIds() throws Exception {
+  void testUse_ExistingEntry_NoAppIds() throws Exception {
     // Pre-populate the SCM with one cache entry
     store.addResource("key1", "foo.jar");
 
@@ -150,13 +151,13 @@ public void testUse_ExistingEntry_NoAppIds() throws Exception {
     String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar";
     assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
     assertEquals(1, store.getResourceReferences("key1").size());
-    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
-        .getInstance().getCacheHits() - hits);
+    assertEquals(1, ClientSCMMetrics
+        .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated.");
 
   }
 
   @Test
-  public void testUse_ExistingEntry_OneId() throws Exception {
+  void testUse_ExistingEntry_OneId() throws Exception {
     // Pre-populate the SCM with one cache entry
     store.addResource("key1", "foo.jar");
     store.addResourceReference("key1",
@@ -174,12 +175,12 @@ public void testUse_ExistingEntry_OneId() throws Exception {
     String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar";
     assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
     assertEquals(2, store.getResourceReferences("key1").size());
-    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
-        .getInstance().getCacheHits() - hits);
+    assertEquals(1, ClientSCMMetrics
+        .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated.");
   }
 
   @Test
-  public void testUse_ExistingEntry_DupId() throws Exception {
+  void testUse_ExistingEntry_DupId() throws Exception {
     // Pre-populate the SCM with one cache entry
     store.addResource("key1", "foo.jar");
     UserGroupInformation testUGI = UserGroupInformation.getCurrentUser();
@@ -201,12 +202,12 @@ public void testUse_ExistingEntry_DupId() throws Exception {
     assertEquals(expectedPath, clientSCMProxy.use(request).getPath());
     assertEquals(1, store.getResourceReferences("key1").size());
 
-    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
-        .getInstance().getCacheHits() - hits);
+    assertEquals(1, ClientSCMMetrics
+        .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated.");
   }
 
   @Test
-  public void testRelease_ExistingEntry_NonExistantAppId() throws Exception {
+  void testRelease_ExistingEntry_NonExistantAppId() throws Exception {
     // Pre-populate the SCM with one cache entry
     store.addResource("key1", "foo.jar");
     store.addResourceReference("key1",
@@ -224,13 +225,14 @@ public void testRelease_ExistingEntry_NonExistantAppId() throws Exception {
     assertEquals(1, store.getResourceReferences("key1").size());
 
     assertEquals(
-        "Client SCM metrics were updated when a release did not happen", 0,
-        ClientSCMMetrics.getInstance().getCacheReleases() - releases);
+        0,
+        ClientSCMMetrics.getInstance().getCacheReleases() - releases,
+        "Client SCM metrics were updated when a release did not happen");
 
   }
 
   @Test
-  public void testRelease_ExistingEntry_WithAppId() throws Exception {
+  void testRelease_ExistingEntry_WithAppId() throws Exception {
     // Pre-populate the SCM with one cache entry
     store.addResource("key1", "foo.jar");
     UserGroupInformation testUGI = UserGroupInformation.getCurrentUser();
@@ -249,13 +251,13 @@ public void testRelease_ExistingEntry_WithAppId() throws Exception {
     clientSCMProxy.release(request);
     assertEquals(0, store.getResourceReferences("key1").size());
 
-    assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics
-        .getInstance().getCacheReleases() - releases);
+    assertEquals(1, ClientSCMMetrics
+        .getInstance().getCacheReleases() - releases, "Client SCM metrics aren't updated.");
 
   }
 
   @Test
-  public void testRelease_MissingEntry() throws Exception {
+  void testRelease_MissingEntry() throws Exception {
 
     long releases = ClientSCMMetrics.getInstance().getCacheReleases();
 
@@ -268,8 +270,9 @@ public void testRelease_MissingEntry() throws Exception {
     assertNotNull(store.getResourceReferences("key2"));
     assertEquals(0, store.getResourceReferences("key2").size());
     assertEquals(
-        "Client SCM metrics were updated when a release did not happen.", 0,
-        ClientSCMMetrics.getInstance().getCacheReleases() - releases);
+        0,
+        ClientSCMMetrics.getInstance().getCacheReleases() - releases,
+        "Client SCM metrics were updated when a release did not happen.");
   }
 
   private ApplicationId createAppId(int id, long timestamp) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java
index 46546444ad568..29a4291dac67f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java
@@ -18,11 +18,8 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -32,14 +29,18 @@
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.junit.After;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
 
 public class TestRemoteAppChecker {
 
   private RemoteAppChecker checker;
 
-  @After
+  @AfterEach
   public void cleanup() {
     if (checker != null) {
       checker.stop();
@@ -61,7 +62,7 @@ private YarnClient createCheckerWithMockedClient() {
   }
 
   @Test
-  public void testNonExistentApp() throws Exception {
+  void testNonExistentApp() throws Exception {
     YarnClient client = createCheckerWithMockedClient();
     ApplicationId id = ApplicationId.newInstance(1, 1);
 
@@ -76,7 +77,7 @@ public void testNonExistentApp() throws Exception {
   }
 
   @Test
-  public void testRunningApp() throws Exception {
+  void testRunningApp() throws Exception {
     YarnClient client = createCheckerWithMockedClient();
     ApplicationId id = ApplicationId.newInstance(1, 1);
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java
index e183ffa5cfba1..9c55452a7f7ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java
@@ -18,36 +18,37 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager;
 
-import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.when;
-
 import java.io.IOException;
 import java.net.InetSocketAddress;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.yarn.server.api.SCMAdminProtocol;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl;
 import org.apache.hadoop.yarn.client.SCMAdmin;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.SCMAdminProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * Basic unit tests for the SCM Admin Protocol Service and SCMAdmin.
@@ -63,7 +64,7 @@ public class TestSCMAdminProtocolService {
   private final RecordFactory recordFactory = RecordFactoryProvider
       .getRecordFactory(null);
 
-  @Before
+  @BeforeEach
   public void startUp() {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.SCM_STORE_CLASS,
@@ -95,7 +96,7 @@ protected SCMAdminProtocol createSCMAdminProtocol() throws IOException {
     };
   }
 
-  @After
+  @AfterEach
   public void cleanUpTest() {
     if (service != null) {
       service.stop();
@@ -107,18 +108,18 @@ public void cleanUpTest() {
   }
 
   @Test
-  public void testRunCleanerTask() throws Exception {
+  void testRunCleanerTask() throws Exception {
     doNothing().when(cleaner).runCleanerTask();
     RunSharedCacheCleanerTaskRequest request =
         recordFactory.newRecordInstance(RunSharedCacheCleanerTaskRequest.class);
     RunSharedCacheCleanerTaskResponse response = SCMAdminProxy.runCleanerTask(request);
-    Assert.assertTrue("cleaner task request isn't accepted", response.getAccepted());
+    assertTrue(response.getAccepted(), "cleaner task request isn't accepted");
     verify(service, times(1)).runCleanerTask(any(RunSharedCacheCleanerTaskRequest.class));
   }
 
   @Test
-  public void testRunCleanerTaskCLI() throws Exception {
-    String[] args = { "-runCleanerTask" };
+  void testRunCleanerTaskCLI() throws Exception {
+    String[] args = {"-runCleanerTask"};
     RunSharedCacheCleanerTaskResponse rp =
         new RunSharedCacheCleanerTaskResponsePBImpl();
     rp.setAccepted(true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
index 048523e8457d4..38833680f697e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java
@@ -18,17 +18,17 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.spy;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.Collection;
 
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -41,11 +41,12 @@
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore;
 import org.apache.hadoop.yarn.server.sharedcachemanager.store.SharedCacheResourceReference;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.spy;
 
 
 /**
@@ -54,7 +55,7 @@
 public class TestSharedCacheUploaderService {
   private static File testDir = null;
 
-  @BeforeClass
+  @BeforeAll
   public static void setupTestDirs() throws IOException {
     testDir = new File("target",
         TestSharedCacheUploaderService.class.getCanonicalName());
@@ -63,7 +64,7 @@ public static void setupTestDirs() throws IOException {
     testDir = testDir.getAbsoluteFile();
   }
 
-  @AfterClass
+  @AfterAll
   public static void cleanupTestDirs() throws IOException {
     if (testDir != null) {
       testDir.delete();
@@ -76,7 +77,7 @@ public static void cleanupTestDirs() throws IOException {
   private final RecordFactory recordFactory = RecordFactoryProvider
       .getRecordFactory(null);
 
-  @Before
+  @BeforeEach
   public void startUp() {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.SCM_STORE_CLASS,
@@ -103,7 +104,7 @@ public void startUp() {
             SCMUploaderProtocol.class, scmAddress, conf);
   }
 
-  @After
+  @AfterEach
   public void cleanUp() {
     if (store != null) {
       store.stop();
@@ -119,7 +120,7 @@ public void cleanUp() {
   }
 
   @Test
-  public void testNotify_noEntry() throws Exception {
+  void testNotify_noEntry() throws Exception {
     long accepted =
         SharedCacheUploaderMetrics.getInstance().getAcceptedUploads();
 
@@ -134,14 +135,15 @@ public void testNotify_noEntry() throws Exception {
     assertEquals(0, set.size());
 
     assertEquals(
-        "NM upload metrics aren't updated.", 1,
+        1,
         SharedCacheUploaderMetrics.getInstance().getAcceptedUploads() -
-            accepted);
+            accepted,
+        "NM upload metrics aren't updated.");
 
   }
 
   @Test
-  public void testNotify_entryExists_differentName() throws Exception {
+  void testNotify_entryExists_differentName() throws Exception {
 
     long rejected =
         SharedCacheUploaderMetrics.getInstance().getRejectUploads();
@@ -157,14 +159,15 @@ public void testNotify_entryExists_differentName() throws Exception {
     assertNotNull(set);
     assertEquals(0, set.size());
     assertEquals(
-        "NM upload metrics aren't updated.", 1,
+        1,
         SharedCacheUploaderMetrics.getInstance().getRejectUploads() -
-            rejected);
+            rejected,
+        "NM upload metrics aren't updated.");
 
   }
 
   @Test
-  public void testNotify_entryExists_sameName() throws Exception {
+  void testNotify_entryExists_sameName() throws Exception {
 
     long accepted =
         SharedCacheUploaderMetrics.getInstance().getAcceptedUploads();
@@ -180,9 +183,10 @@ public void testNotify_entryExists_sameName() throws Exception {
     assertNotNull(set);
     assertEquals(0, set.size());
     assertEquals(
-        "NM upload metrics aren't updated.", 1,
+        1,
         SharedCacheUploaderMetrics.getInstance().getAcceptedUploads() -
-            accepted);
+            accepted,
+        "NM upload metrics aren't updated.");
 
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
index cca3f4286b321..6829d9a6347de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java
@@ -17,24 +17,25 @@
  */
 package org.apache.hadoop.yarn.server.sharedcachemanager.metrics;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestCleanerMetrics {
 
   Configuration conf = new Configuration();
   CleanerMetrics cleanerMetrics;
 
-  @Before
+  @BeforeEach
   public void init() {
     cleanerMetrics = CleanerMetrics.getInstance();
   }
 
   @Test
-  public void testMetricsOverMultiplePeriods() {
+  void testMetricsOverMultiplePeriods() {
     simulateACleanerRun();
     assertMetrics(4, 4, 1, 1);
     simulateACleanerRun();
@@ -51,14 +52,14 @@ public void simulateACleanerRun() {
 
   void assertMetrics(int proc, int totalProc, int del, int totalDel) {
     assertEquals(
-        "Processed files in the last period are not measured correctly", proc,
-        cleanerMetrics.getProcessedFiles());
-    assertEquals("Total processed files are not measured correctly",
-        totalProc, cleanerMetrics.getTotalProcessedFiles());
-    assertEquals(
-        "Deleted files in the last period are not measured correctly", del,
-        cleanerMetrics.getDeletedFiles());
-    assertEquals("Total deleted files are not measured correctly",
-        totalDel, cleanerMetrics.getTotalDeletedFiles());
+        proc,
+        cleanerMetrics.getProcessedFiles(),
+        "Processed files in the last period are not measured correctly");
+    assertEquals(totalProc, cleanerMetrics.getTotalProcessedFiles(),
+        "Total processed files are not measured correctly");
+    assertEquals(del, cleanerMetrics.getDeletedFiles(),
+        "Deleted files in the last period are not measured correctly");
+    assertEquals(totalDel, cleanerMetrics.getTotalDeletedFiles(),
+        "Total deleted files are not measured correctly");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java
index 4e960b29c779d..2374bf3fdbe06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java
@@ -18,9 +18,10 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager.store;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Test;
 
 /**
  * All test classes that test an SCMStore implementation must extend this class.
@@ -33,7 +34,7 @@ public abstract class SCMStoreBaseTest {
   abstract Class<? extends SCMStore> getStoreClass();
 
   @Test
-  public void TestZeroArgConstructor() throws Exception {
+  void TestZeroArgConstructor() throws Exception {
     // Test that the SCMStore implementation class is compatible with
     // ReflectionUtils#newInstance
     ReflectionUtils.newInstance(getStoreClass(), new Configuration());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
index 35adbdd34e4bc..5cb5332be4135 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java
@@ -18,16 +18,6 @@
 
 package org.apache.hadoop.yarn.server.sharedcachemanager.store;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -41,6 +31,10 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -48,9 +42,16 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.server.sharedcachemanager.AppChecker;
 import org.apache.hadoop.yarn.server.sharedcachemanager.DummyAppChecker;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 
 public class TestInMemorySCMStore extends SCMStoreBaseTest {
 
@@ -62,13 +63,13 @@ Class<? extends SCMStore> getStoreClass() {
     return InMemorySCMStore.class;
   }
 
-  @Before
+  @BeforeEach
   public void setup() {
     this.checker = spy(new DummyAppChecker());
     this.store = spy(new InMemorySCMStore(checker));
   }
 
-  @After
+  @AfterEach
   public void cleanup() {
     if (this.store != null) {
       this.store.stop();
@@ -117,7 +118,7 @@ private void startStoreWithApps() throws Exception {
   }
 
   @Test
-  public void testAddResourceConcurrency() throws Exception {
+  void testAddResourceConcurrency() throws Exception {
     startEmptyStore();
     final String key = "key1";
     int count = 5;
@@ -140,7 +141,7 @@ public String call() throws Exception {
     start.countDown();
     // check the result; they should all agree with the value
     Set<String> results = new HashSet<String>();
-    for (Future<String> future: futures) {
+    for (Future<String> future : futures) {
       results.add(future.get());
     }
     assertSame(1, results.size());
@@ -148,7 +149,7 @@ public String call() throws Exception {
   }
 
   @Test
-  public void testAddResourceRefNonExistentResource() throws Exception {
+  void testAddResourceRefNonExistentResource() throws Exception {
     startEmptyStore();
     String key = "key1";
     ApplicationId id = createAppId(1, 1L);
@@ -158,7 +159,7 @@ public void testAddResourceRefNonExistentResource() throws Exception {
   }
 
   @Test
-  public void testRemoveResourceEmptyRefs() throws Exception {
+  void testRemoveResourceEmptyRefs() throws Exception {
     startEmptyStore();
     String key = "key1";
     String fileName = "foo.jar";
@@ -169,7 +170,7 @@ public void testRemoveResourceEmptyRefs() throws Exception {
   }
 
   @Test
-  public void testAddResourceRefRemoveResource() throws Exception {
+  void testAddResourceRefRemoveResource() throws Exception {
     startEmptyStore();
     String key = "key1";
     ApplicationId id = createAppId(1, 1L);
@@ -186,7 +187,7 @@ public void testAddResourceRefRemoveResource() throws Exception {
   }
 
   @Test
-  public void testAddResourceRefConcurrency() throws Exception {
+  void testAddResourceRefConcurrency() throws Exception {
     startEmptyStore();
     final String key = "key1";
     final String user = "user";
@@ -215,7 +216,7 @@ public String call() throws Exception {
     start.countDown();
     // check the result
     Set<String> results = new HashSet<String>();
-    for (Future<String> future: futures) {
+    for (Future<String> future : futures) {
       results.add(future.get());
     }
     // they should all have the same file name
@@ -228,7 +229,7 @@ public String call() throws Exception {
   }
 
   @Test
-  public void testAddResourceRefAddResourceConcurrency() throws Exception {
+  void testAddResourceRefAddResourceConcurrency() throws Exception {
     startEmptyStore();
     final String key = "key1";
     final String fileName = "foo.jar";
@@ -265,7 +266,7 @@ public String call() throws Exception {
   }
 
   @Test
-  public void testRemoveRef() throws Exception {
+  void testRemoveRef() throws Exception {
     startEmptyStore();
     String key = "key1";
     String fileName = "foo.jar";
@@ -287,7 +288,7 @@ public void testRemoveRef() throws Exception {
   }
 
   @Test
-  public void testBootstrapping() throws Exception {
+  void testBootstrapping() throws Exception {
     Map<String, String> initialCachedResources = startStoreWithResources();
     int count = initialCachedResources.size();
     ApplicationId id = createAppId(1, 1L);
@@ -306,7 +307,7 @@ public void testBootstrapping() throws Exception {
   }
 
   @Test
-  public void testEvictableWithInitialApps() throws Exception {
+  void testEvictableWithInitialApps() throws Exception {
     startStoreWithApps();
     assertFalse(store.isResourceEvictable("key", mock(FileStatus.class)));
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 2de2c13f16b51..07838688d7099 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -126,6 +126,26 @@
       <artifactId>bcprov-jdk15on</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-auth</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
index 96e319963e0d6..3ac8cb9068458 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
@@ -25,7 +25,6 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index de944cbc89cee..702c5ea7dfb5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.server;
 
-import static org.junit.Assert.fail;
-
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -31,6 +29,11 @@
 import java.util.concurrent.TimeoutException;
 
 import org.apache.hadoop.conf.Configuration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
@@ -75,18 +78,15 @@
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
-@RunWith(Parameterized.class)
 public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
 
   static Logger LOG = LoggerFactory.getLogger(TestContainerManagerSecurity.class);
@@ -94,29 +94,24 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
       .getRecordFactory(null);
   private static MiniYARNCluster yarnCluster;
   private static final File testRootDir = new File("target",
-    TestContainerManagerSecurity.class.getName() + "-root");
+      TestContainerManagerSecurity.class.getName() + "-root");
   private static File httpSpnegoKeytabFile = new File(testRootDir,
-    "httpSpnegoKeytabFile.keytab");
+      "httpSpnegoKeytabFile.keytab");
   private static String httpSpnegoPrincipal = "HTTP/localhost@EXAMPLE.COM";
 
   private Configuration conf;
 
-  @Before
-  public void setUp() throws Exception {
+  @BeforeEach
+  public void setup() throws Exception {
     testRootDir.mkdirs();
     httpSpnegoKeytabFile.deleteOnExit();
+    startMiniKdc();
     getKdc().createPrincipal(httpSpnegoKeytabFile, httpSpnegoPrincipal);
-    UserGroupInformation.setConfiguration(conf);
-
-    yarnCluster =
-        new MiniYARNCluster(TestContainerManagerSecurity.class.getName(), 1, 1,
-            1);
-    yarnCluster.init(conf);
-    yarnCluster.start();
   }
- 
-  @After
+
+  @AfterEach
   public void tearDown() {
+    stopMiniKdc();
     if (yarnCluster != null) {
       yarnCluster.stop();
       yarnCluster = null;
@@ -130,48 +125,56 @@ public void tearDown() {
    * and to give details in say an IDE.  The second is the configuraiton
    * object to use.
    */
-  @Parameters(name = "{0}")
   public static Collection<Object[]> configs() {
     Configuration configurationWithoutSecurity = new Configuration();
     configurationWithoutSecurity.set(
         CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple");
-    
+
     Configuration configurationWithSecurity = new Configuration();
     configurationWithSecurity.set(
-      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
     configurationWithSecurity.set(
-      YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal);
+        YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal);
     configurationWithSecurity.set(
-      YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
-      httpSpnegoKeytabFile.getAbsolutePath());
+        YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
+        httpSpnegoKeytabFile.getAbsolutePath());
     configurationWithSecurity.set(
-      YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal);
+        YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal);
     configurationWithSecurity.set(
-      YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
-      httpSpnegoKeytabFile.getAbsolutePath());
+        YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
+        httpSpnegoKeytabFile.getAbsolutePath());
 
-    return Arrays.asList(new Object[][] {
+    return Arrays.asList(new Object[][]{
         {"Simple", configurationWithoutSecurity},
         {"Secure", configurationWithSecurity}});
   }
-  
-  public TestContainerManagerSecurity(String name, Configuration conf) {
+
+  public void initTestContainerManagerSecurity(String name, Configuration conf) {
     LOG.info("RUNNING TEST " + name);
+    UserGroupInformation.setConfiguration(conf);
+    yarnCluster =
+        new MiniYARNCluster(TestContainerManagerSecurity.class.getName(), 1, 1,
+            1);
+    yarnCluster.init(conf);
+    yarnCluster.start();
     conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
     this.conf = conf;
   }
-  
-  @Test
-  public void testContainerManager() throws Exception {
-      
-      // TestNMTokens.
-      testNMTokens(conf);
-      
-      // Testing for container token tampering
-      testContainerToken(conf);
-      
-      // Testing for container token tampering with epoch
-      testContainerTokenWithEpoch(conf);
+
+  @MethodSource("configs")
+  @ParameterizedTest(name = "{0}")
+  void testContainerManager(String name, Configuration conf) throws Exception {
+
+    initTestContainerManagerSecurity(name, conf);
+
+    // TestNMTokens.
+    testNMTokens(conf);
+
+    // Testing for container token tampering
+    testContainerToken(conf);
+
+    // Testing for container token tampering with epoch
+    testContainerTokenWithEpoch(conf);
 
   }
 
@@ -182,21 +185,21 @@ public void testContainerManager() throws Exception {
   private void testNMTokens(Configuration testConf) throws Exception {
     NMTokenSecretManagerInRM nmTokenSecretManagerRM =
         yarnCluster.getResourceManager().getRMContext()
-          .getNMTokenSecretManager();
+            .getNMTokenSecretManager();
     NMTokenSecretManagerInNM nmTokenSecretManagerNM =
         yarnCluster.getNodeManager(0).getNMContext().getNMTokenSecretManager();
     RMContainerTokenSecretManager containerTokenSecretManager =
         yarnCluster.getResourceManager().getRMContext().
             getContainerTokenSecretManager();
-    
+
     NodeManager nm = yarnCluster.getNodeManager(0);
-    
+
     waitForNMToReceiveNMTokenKey(nmTokenSecretManagerNM);
-    
+
     // Both id should be equal.
-    Assert.assertEquals(nmTokenSecretManagerNM.getCurrentKey().getKeyId(),
+    assertEquals(nmTokenSecretManagerNM.getCurrentKey().getKeyId(),
         nmTokenSecretManagerRM.getCurrentKey().getKeyId());
-    
+
     /*
      * Below cases should be tested.
      * 1) If Invalid NMToken is used then it should be rejected.
@@ -225,25 +228,25 @@ private void testNMTokens(Configuration testConf) throws Exception {
     yarnCluster.getResourceManager().getRMContext().getRMApps().put(appId, m);
     ApplicationAttemptId validAppAttemptId =
         ApplicationAttemptId.newInstance(appId, 1);
-    
+
     ContainerId validContainerId =
         ContainerId.newContainerId(validAppAttemptId, 0);
-    
+
     NodeId validNode = yarnCluster.getNodeManager(0).getNMContext().getNodeId();
     NodeId invalidNode = NodeId.newInstance("InvalidHost", 1234);
 
-    
+
     org.apache.hadoop.yarn.api.records.Token validNMToken =
         nmTokenSecretManagerRM.createNMToken(validAppAttemptId, validNode, user);
-    
+
     org.apache.hadoop.yarn.api.records.Token validContainerToken =
         containerTokenSecretManager.createContainerToken(validContainerId,
             0, validNode, user, r, Priority.newInstance(10), 1234);
     ContainerTokenIdentifier identifier =
         BuilderUtils.newContainerTokenIdentifier(validContainerToken);
-    Assert.assertEquals(Priority.newInstance(10), identifier.getPriority());
-    Assert.assertEquals(1234, identifier.getCreationTime());
-    
+    assertEquals(Priority.newInstance(10), identifier.getPriority());
+    assertEquals(1234, identifier.getCreationTime());
+
     StringBuilder sb;
     // testInvalidNMToken ... creating NMToken using different secret manager.
     
@@ -255,7 +258,7 @@ private void testNMTokens(Configuration testConf) throws Exception {
       // Making sure key id is different.
     } while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM
         .getCurrentKey().getKeyId());
-    
+
     // Testing that NM rejects the requests when we don't send any token.
     if (UserGroupInformation.isSecurityEnabled()) {
       sb = new StringBuilder("Client cannot authenticate via:[TOKEN]");
@@ -266,55 +269,55 @@ private void testNMTokens(Configuration testConf) throws Exception {
     }
     String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode,
         validContainerToken, null, true);
-    Assert.assertTrue("In calling " + validNode + " exception was '"
+    assertTrue(errorMsg.contains(sb.toString()), "In calling " + validNode + " exception was '"
         + errorMsg + "' but doesn't contain '"
-        + sb.toString() + "'", errorMsg.contains(sb.toString()));
-    
+        + sb.toString() + "'");
+
     org.apache.hadoop.yarn.api.records.Token invalidNMToken =
         tempManager.createNMToken(validAppAttemptId, validNode, user);
     sb = new StringBuilder("Given NMToken for application : ");
     sb.append(validAppAttemptId.toString())
-      .append(" seems to have been generated illegally.");
-    Assert.assertTrue(sb.toString().contains(
+        .append(" seems to have been generated illegally.");
+    assertTrue(sb.toString().contains(
         testStartContainer(rpc, validAppAttemptId, validNode,
             validContainerToken, invalidNMToken, true)));
-    
+
     // valid NMToken but belonging to other node
     invalidNMToken =
         nmTokenSecretManagerRM.createNMToken(validAppAttemptId, invalidNode,
             user);
     sb = new StringBuilder("Given NMToken for application : ");
     sb.append(validAppAttemptId)
-      .append(" is not valid for current node manager.expected : ")
-      .append(validNode.toString())
-      .append(" found : ").append(invalidNode.toString());
-    Assert.assertTrue(sb.toString().contains(
+        .append(" is not valid for current node manager.expected : ")
+        .append(validNode.toString())
+        .append(" found : ").append(invalidNode.toString());
+    assertTrue(sb.toString().contains(
         testStartContainer(rpc, validAppAttemptId, validNode,
             validContainerToken, invalidNMToken, true)));
-    
+
     // using correct tokens. nmtoken for app attempt should get saved.
     testConf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
         4 * 60 * 1000);
     validContainerToken =
         containerTokenSecretManager.createContainerToken(validContainerId,
             0, validNode, user, r, Priority.newInstance(0), 0);
-    Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
-      validContainerToken, validNMToken, false).isEmpty());
-    Assert.assertTrue(nmTokenSecretManagerNM
+    assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+        validContainerToken, validNMToken, false).isEmpty());
+    assertTrue(nmTokenSecretManagerNM
         .isAppAttemptNMTokenKeyPresent(validAppAttemptId));
-    
+
     // using a new compatible version nmtoken, expect container can be started 
     // successfully.
     ApplicationAttemptId validAppAttemptId2 =
         ApplicationAttemptId.newInstance(appId, 2);
-        
+
     ContainerId validContainerId2 =
         ContainerId.newContainerId(validAppAttemptId2, 0);
 
     org.apache.hadoop.yarn.api.records.Token validContainerToken2 =
         containerTokenSecretManager.createContainerToken(validContainerId2,
             0, validNode, user, r, Priority.newInstance(0), 0);
-    
+
     org.apache.hadoop.yarn.api.records.Token validNMToken2 =
         nmTokenSecretManagerRM.createNMToken(validAppAttemptId2, validNode, user);
     // First, get a new NMTokenIdentifier.
@@ -323,43 +326,42 @@ private void testNMTokens(Configuration testConf) throws Exception {
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
     newIdentifier.readFields(dib);
-    
+
     // Then, generate a new version NMTokenIdentifier (NMTokenIdentifierNewForTest)
     // with additional field of message.
-    NMTokenIdentifierNewForTest newVersionIdentifier = 
+    NMTokenIdentifierNewForTest newVersionIdentifier =
         new NMTokenIdentifierNewForTest(newIdentifier, "message");
-    
+
     // check new version NMTokenIdentifier has correct info.
-    Assert.assertEquals("The ApplicationAttemptId is changed after set to " +
-        "newVersionIdentifier", validAppAttemptId2.getAttemptId(), 
+    assertEquals(validAppAttemptId2.getAttemptId(),
         newVersionIdentifier.getApplicationAttemptId().getAttemptId()
-    );
-    
-    Assert.assertEquals("The message is changed after set to newVersionIdentifier",
-        "message", newVersionIdentifier.getMessage());
-    
-    Assert.assertEquals("The NodeId is changed after set to newVersionIdentifier", 
-        validNode, newVersionIdentifier.getNodeId());
-    
+    ,
+        "The ApplicationAttemptId is changed after set to " +
+            "newVersionIdentifier");
+
+    assertEquals("message", newVersionIdentifier.getMessage(), "The message is changed after set to newVersionIdentifier");
+
+    assertEquals(validNode, newVersionIdentifier.getNodeId(), "The NodeId is changed after set to newVersionIdentifier");
+
     // create new Token based on new version NMTokenIdentifier.
     org.apache.hadoop.yarn.api.records.Token newVersionedNMToken =
         BaseNMTokenSecretManager.newInstance(
-            nmTokenSecretManagerRM.retrievePassword(newVersionIdentifier), 
+            nmTokenSecretManagerRM.retrievePassword(newVersionIdentifier),
             newVersionIdentifier);
-    
+
     // Verify startContainer is successful and no exception is thrown.
-    Assert.assertTrue(testStartContainer(rpc, validAppAttemptId2, validNode,
+    assertTrue(testStartContainer(rpc, validAppAttemptId2, validNode,
         validContainerToken2, newVersionedNMToken, false).isEmpty());
-    Assert.assertTrue(nmTokenSecretManagerNM
+    assertTrue(nmTokenSecretManagerNM
         .isAppAttemptNMTokenKeyPresent(validAppAttemptId2));
-    
+
     //Now lets wait till container finishes and is removed from node manager.
     waitForContainerToFinishOnNM(validContainerId);
     sb = new StringBuilder("Attempt to relaunch the same container with id ");
     sb.append(validContainerId);
-    Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+    assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
         validContainerToken, validNMToken, true).contains(sb.toString()));
-    
+
     // Container is removed from node manager's memory by this time.
     // trying to stop the container. It should not throw any exception.
     testStopContainer(rpc, validAppAttemptId, validNode, validContainerId,
@@ -370,25 +372,25 @@ private void testNMTokens(Configuration testConf) throws Exception {
     rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
     // Key rolled over once.. rolling over again
     rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
-    
+
     // trying get container status. Now saved nmToken should be used for
     // authentication... It should complain saying container was recently
     // stopped.
     sb = new StringBuilder("Container ");
     sb.append(validContainerId)
         .append(" was recently stopped on node manager");
-    Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+    assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
         validContainerId, validNMToken, true).contains(sb.toString()));
 
     // Now lets remove the container from nm-memory
     nm.getNodeStatusUpdater().clearFinishedContainersFromCache();
-    
+
     // This should fail as container is removed from recently tracked finished
     // containers.
     sb = new StringBuilder("Container ")
         .append(validContainerId.toString())
         .append(" is not handled by this NodeManager");
-    Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+    assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
         validContainerId, validNMToken, false).contains(sb.toString()));
 
     // using appAttempt-1 NMtoken for launching container for appAttempt-2
@@ -396,13 +398,13 @@ private void testNMTokens(Configuration testConf) throws Exception {
     ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(appId, 2);
     Token attempt1NMToken =
         nmTokenSecretManagerRM
-          .createNMToken(validAppAttemptId, validNode, user);
+            .createNMToken(validAppAttemptId, validNode, user);
     org.apache.hadoop.yarn.api.records.Token newContainerToken =
         containerTokenSecretManager.createContainerToken(
-          ContainerId.newContainerId(attempt2, 1), 0, validNode, user, r,
+            ContainerId.newContainerId(attempt2, 1), 0, validNode, user, r,
             Priority.newInstance(0), 0);
-    Assert.assertTrue(testStartContainer(rpc, attempt2, validNode,
-      newContainerToken, attempt1NMToken, false).isEmpty());
+    assertTrue(testStartContainer(rpc, attempt2, validNode,
+        newContainerToken, attempt1NMToken, false).isEmpty());
   }
 
   private void waitForContainerToFinishOnNM(ContainerId containerId)
@@ -419,7 +421,7 @@ private void waitForContainerToFinishOnNM(ContainerId containerId)
         LOG.info("Waiting for " + containerId + " to get to state " +
             ContainerState.COMPLETE);
         GenericTestUtils.waitFor(() -> ContainerState.COMPLETE.equals(
-            waitContainer.cloneAndGetContainerStatus().getState()),
+                waitContainer.cloneAndGetContainerStatus().getState()),
             500, timeout);
       } catch (TimeoutException te) {
         LOG.error("TimeoutException", te);
@@ -433,7 +435,7 @@ private void waitForContainerToFinishOnNM(ContainerId containerId)
     // Normally, Containers will be removed from NM context after they are
     // explicitly acked by RM. Now, manually remove it for testing.
     yarnCluster.getNodeManager(0).getNodeStatusUpdater()
-      .addCompletedContainer(containerId);
+        .addCompletedContainer(containerId);
     LOG.info("Removing container from NMContext, containerID = " + containerId);
     nmContext.getContainers().remove(containerId);
   }
@@ -458,16 +460,16 @@ protected void rollNMTokenMasterKey(
       Thread.sleep(1000);
     }
     nmTokenSecretManagerRM.activateNextMasterKey();
-    Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
+    assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
         == nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
   }
-  
+
   private String testStopContainer(YarnRPC rpc,
       ApplicationAttemptId appAttemptId, NodeId nodeId,
       ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
     try {
       stopContainer(rpc, nmToken,
-          Arrays.asList(new ContainerId[] {containerId}), appAttemptId,
+          Arrays.asList(new ContainerId[]{containerId}), appAttemptId,
           nodeId);
       if (isExceptionExpected) {
         fail("Exception was expected!!");
@@ -505,8 +507,8 @@ private String testStartContainer(YarnRPC rpc,
     try {
       startContainer(rpc, nmToken, containerToken, nodeId,
           appAttemptId.toString());
-      if (isExceptionExpected){
-        fail("Exception was expected!!");        
+      if (isExceptionExpected) {
+        fail("Exception was expected!!");
       }
       return "";
     } catch (Exception e) {
@@ -514,7 +516,7 @@ private String testStartContainer(YarnRPC rpc,
       return e.getMessage();
     }
   }
-  
+
   private void stopContainer(YarnRPC rpc, Token nmToken,
       List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
       NodeId nodeId) throws Exception {
@@ -537,13 +539,12 @@ private void stopContainer(YarnRPC rpc, Token nmToken,
       }
     }
   }
-  
-  private void
-      getContainerStatus(YarnRPC rpc,
-          org.apache.hadoop.yarn.api.records.Token nmToken,
-          ContainerId containerId,
-          ApplicationAttemptId appAttemptId, NodeId nodeId,
-          boolean isExceptionExpected) throws Exception {
+
+  private void getContainerStatus(YarnRPC rpc,
+      org.apache.hadoop.yarn.api.records.Token nmToken,
+      ContainerId containerId,
+      ApplicationAttemptId appAttemptId, NodeId nodeId,
+      boolean isExceptionExpected) throws Exception {
     List<ContainerId> containerIds = new ArrayList<ContainerId>();
     containerIds.add(containerId);
     GetContainerStatusesRequest request =
@@ -558,7 +559,7 @@ private void stopContainer(YarnRPC rpc, Token nmToken,
       if (statuses.getFailedRequests() != null
           && statuses.getFailedRequests().containsKey(containerId)) {
         parseAndThrowException(statuses.getFailedRequests().get(containerId)
-          .deSerialize());
+            .deSerialize());
       }
     } finally {
       if (proxy != null) {
@@ -566,7 +567,7 @@ private void stopContainer(YarnRPC rpc, Token nmToken,
       }
     }
   }
-  
+
   private void startContainer(final YarnRPC rpc,
       org.apache.hadoop.yarn.api.records.Token nmToken,
       org.apache.hadoop.yarn.api.records.Token containerToken,
@@ -584,7 +585,7 @@ private void startContainer(final YarnRPC rpc,
     try {
       proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
       StartContainersResponse response = proxy.startContainers(allRequests);
-      for(SerializedException ex : response.getFailedRequests().values()){
+      for (SerializedException ex : response.getFailedRequests().values()) {
         parseAndThrowException(ex.deSerialize());
       }
     } finally {
@@ -613,11 +614,11 @@ protected ContainerManagementProtocol getContainerManagementProtocolProxy(
     final InetSocketAddress addr =
         new InetSocketAddress(nodeId.getHost(), nodeId.getPort());
     if (nmToken != null) {
-      ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));      
+      ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
     }
     proxy =
         NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
-          rpc, addr);
+            rpc, addr);
     return proxy;
   }
 
@@ -642,7 +643,7 @@ private void testContainerToken(Configuration conf) throws IOException,
      */
     NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
         yarnCluster.getResourceManager().getRMContext()
-          .getNMTokenSecretManager();
+            .getNMTokenSecretManager();
     ApplicationId appId = ApplicationId.newInstance(1, 1);
     ApplicationAttemptId appAttemptId =
         ApplicationAttemptId.newInstance(appId, 0);
@@ -651,46 +652,46 @@ private void testContainerToken(Configuration conf) throws IOException,
     NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
         nm.getNMContext().getNMTokenSecretManager();
     String user = "test";
-    
+
     waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM);
 
     NodeId nodeId = nm.getNMContext().getNodeId();
-    
+
     // Both id should be equal.
-    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
+    assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
         nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
-    
-    
+
+
     RMContainerTokenSecretManager containerTokenSecretManager =
         yarnCluster.getResourceManager().getRMContext().
             getContainerTokenSecretManager();
-    
+
     Resource r = Resource.newInstance(1230, 2);
-    
-    Token containerToken = 
+
+    Token containerToken =
         containerTokenSecretManager.createContainerToken(
             cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
-    
-    ContainerTokenIdentifier containerTokenIdentifier = 
+
+    ContainerTokenIdentifier containerTokenIdentifier =
         getContainerTokenIdentifierFromToken(containerToken);
-    
+
     // Verify new compatible version ContainerTokenIdentifier
     // can work successfully.
-    ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
+    ContainerTokenIdentifierForTest newVersionTokenIdentifier =
         new ContainerTokenIdentifierForTest(containerTokenIdentifier,
             "message");
-    byte[] password = 
+    byte[] password =
         containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
-    
+
     Token newContainerToken = BuilderUtils.newContainerToken(
         nodeId, password, newVersionTokenIdentifier);
-    
+
     Token nmToken =
-            nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
+        nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
     YarnRPC rpc = YarnRPC.create(conf);
-    Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
+    assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
         newContainerToken, nmToken, false).isEmpty());
-    
+
     // Creating a tampered Container Token
     RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
         new RMContainerTokenSecretManager(conf);
@@ -700,17 +701,17 @@ private void testContainerToken(Configuration conf) throws IOException,
       tamperedContainerTokenSecretManager.activateNextMasterKey();
     } while (containerTokenSecretManager.getCurrentKey().getKeyId()
         == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
-    
+
     ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
     // Creating modified containerToken
     Token containerToken2 =
         tamperedContainerTokenSecretManager.createContainerToken(cId2, 0,
             nodeId, user, r, Priority.newInstance(0), 0);
-    
+
     StringBuilder sb = new StringBuilder("Given Container ");
     sb.append(cId2)
         .append(" seems to have an illegally generated token.");
-    Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
+    assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
         containerToken2, nmToken, true).contains(sb.toString()));
   }
 
@@ -754,7 +755,7 @@ private void testContainerTokenWithEpoch(Configuration conf)
     NodeId nodeId = nm.getNMContext().getNodeId();
 
     // Both id should be equal.
-    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
+    assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
         nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
 
     // Creating a normal Container Token
@@ -765,17 +766,17 @@ private void testContainerTokenWithEpoch(Configuration conf)
     Token containerToken =
         containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user,
             r, Priority.newInstance(0), 0);
-    
+
     ContainerTokenIdentifier containerTokenIdentifier =
         new ContainerTokenIdentifier();
     byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
     DataInputBuffer dib = new DataInputBuffer();
     dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
     containerTokenIdentifier.readFields(dib);
-    
-    
-    Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
-    Assert.assertEquals(
+
+
+    assertEquals(cId, containerTokenIdentifier.getContainerID());
+    assertEquals(
         cId.toString(), containerTokenIdentifier.getContainerID().toString());
 
     Token nmToken =
@@ -791,10 +792,10 @@ private void testContainerTokenWithEpoch(Configuration conf)
         = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
     GetContainerStatusesResponse res = proxy.getContainerStatuses(
         GetContainerStatusesRequest.newInstance(containerIds));
-    Assert.assertNotNull(res.getContainerStatuses().get(0));
-    Assert.assertEquals(
+    assertNotNull(res.getContainerStatuses().get(0));
+    assertEquals(
         cId, res.getContainerStatuses().get(0).getContainerId());
-    Assert.assertEquals(cId.toString(),
+    assertEquals(cId.toString(),
         res.getContainerStatuses().get(0).getContainerId().toString());
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
index 23bb03999308a..daeb21db0cb81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileUtil;
@@ -37,11 +39,10 @@
 import java.util.Iterator;
 import java.util.List;
 
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
-import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -73,7 +74,7 @@ public class TestDiskFailures {
   private static MiniYARNCluster yarnCluster;
   LocalDirsHandlerService dirsHandler;
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws AccessControlException,
       FileNotFoundException, UnsupportedFileSystemException, IOException {
     localFS = FileContext.getLocalFSFileContext();
@@ -82,7 +83,7 @@ public static void setup() throws AccessControlException,
     // Do not start cluster here
   }
 
-  @AfterClass
+  @AfterAll
   public static void teardown() {
     if (yarnCluster != null) {
       yarnCluster.stop();
@@ -99,7 +100,7 @@ public static void teardown() {
    * @throws IOException
    */
   @Test
-  public void testLocalDirsFailures() throws IOException {
+  void testLocalDirsFailures() throws IOException {
     testDirsFailures(true);
   }
 
@@ -111,7 +112,7 @@ public void testLocalDirsFailures() throws IOException {
    * @throws IOException
    */  
   @Test
-  public void testLogDirsFailures() throws IOException {
+  void testLogDirsFailures() throws IOException {
     testDirsFailures(false);
   }
 
@@ -122,7 +123,7 @@ public void testLogDirsFailures() throws IOException {
    * @throws IOException
    */
   @Test
-  public void testDirFailuresOnStartup() throws IOException {
+  void testDirFailuresOnStartup() throws IOException {
     Configuration conf = new YarnConfiguration();
     String localDir1 = new File(testDir, "localDir1").getPath();
     String localDir2 = new File(testDir, "localDir2").getPath();
@@ -137,11 +138,11 @@ public void testDirFailuresOnStartup() throws IOException {
     LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
     dirSvc.init(conf);
     List<String> localDirs = dirSvc.getLocalDirs();
-    Assert.assertEquals(1, localDirs.size());
-    Assert.assertEquals(new Path(localDir2).toString(), localDirs.get(0));
+    assertEquals(1, localDirs.size());
+    assertEquals(new Path(localDir2).toString(), localDirs.get(0));
     List<String> logDirs = dirSvc.getLogDirs();
-    Assert.assertEquals(1, logDirs.size());
-    Assert.assertEquals(new Path(logDir1).toString(), logDirs.get(0));
+    assertEquals(1, logDirs.size());
+    assertEquals(new Path(logDir1).toString(), logDirs.get(0));
   }
 
   private void testDirsFailures(boolean localORLogDirs) throws IOException {
@@ -177,8 +178,7 @@ private void testDirsFailures(boolean localORLogDirs) throws IOException {
     List<String> list = localORLogDirs ? dirsHandler.getLocalDirs()
                                        : dirsHandler.getLogDirs();
     String[] dirs = list.toArray(new String[list.size()]);
-    Assert.assertEquals("Number of nm-" + dirType + "-dirs is wrong.",
-                        numLocalDirs, dirs.length);
+    assertEquals(numLocalDirs, dirs.length, "Number of nm-" + dirType + "-dirs is wrong.");
     String expectedDirs = StringUtils.join(",", list);
     // validate the health of disks initially
     verifyDisksHealth(localORLogDirs, expectedDirs, true);
@@ -225,11 +225,9 @@ private void verifyDisksHealth(boolean localORLogDirs, String expectedDirs,
     String seenDirs = StringUtils.join(",", list);
     LOG.info("ExpectedDirs=" + expectedDirs);
     LOG.info("SeenDirs=" + seenDirs);
-    Assert.assertTrue("NodeManager could not identify disk failure.",
-                      expectedDirs.equals(seenDirs));
+    assertEquals(expectedDirs, seenDirs);
 
-    Assert.assertEquals("Node's health in terms of disks is wrong",
-                        isHealthy, dirsHandler.areDisksHealthy());
+    assertEquals(isHealthy, dirsHandler.areDisksHealthy(), "Node's health in terms of disks is wrong");
     for (int i = 0; i < 10; i++) {
       Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext()
                               .getRMNodes().values().iterator();
@@ -247,8 +245,7 @@ private void verifyDisksHealth(boolean localORLogDirs, String expectedDirs,
     }
     Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext()
                             .getRMNodes().values().iterator();
-    Assert.assertEquals("RM is not updated with the health status of a node",
-        isHealthy, iter.next().getState() != NodeState.UNHEALTHY);
+    assertEquals(isHealthy, iter.next().getState() != NodeState.UNHEALTHY, "RM is not updated with the health status of a node");
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
index 384d1cd78f2e5..8bd7e59eb3f73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
@@ -18,22 +18,21 @@
 
 package org.apache.hadoop.yarn.server;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 public class TestMiniYARNClusterForHA {
   MiniYARNCluster cluster;
 
-  @Before
+  @BeforeEach
   public void setup() throws IOException, InterruptedException {
     Configuration conf = new YarnConfiguration();
     conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
@@ -44,12 +43,12 @@ public void setup() throws IOException, InterruptedException {
     cluster.init(conf);
     cluster.start();
 
-    assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+    assertFalse(-1 == cluster.getActiveRMIndex(), "RM never turned active");
   }
 
   @Test
-  public void testClusterWorks() throws YarnException, InterruptedException {
-    assertTrue("NMs fail to connect to the RM",
-        cluster.waitForNodeManagersToConnect(5000));
+  void testClusterWorks() throws YarnException, InterruptedException {
+    assertTrue(cluster.waitForNodeManagersToConnect(5000),
+        "NMs fail to connect to the RM");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
index ff7fafc200143..c5c2e3e0f5e65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -18,18 +18,23 @@
 
 package org.apache.hadoop.yarn.server;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
 import java.io.IOException;
 
 public class TestMiniYarnCluster {
 
   @Test
-  public void testTimelineServiceStartInMiniCluster() throws Exception {
+  void testTimelineServiceStartInMiniCluster() throws Exception {
     Configuration conf = new YarnConfiguration();
     int numNodeManagers = 1;
     int numLocalDirs = 1;
@@ -45,14 +50,14 @@ public void testTimelineServiceStartInMiniCluster() throws Exception {
     try (MiniYARNCluster cluster =
         new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
             numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
-                enableAHS)) {
+            enableAHS)) {
 
       cluster.init(conf);
       cluster.start();
 
       //verify that the timeline service is not started.
-      Assert.assertNull("Timeline Service should not have been started",
-          cluster.getApplicationHistoryServer());
+      assertNull(cluster.getApplicationHistoryServer(),
+          "Timeline Service should not have been started");
     }
 
     /*
@@ -64,25 +69,25 @@ public void testTimelineServiceStartInMiniCluster() throws Exception {
     try (MiniYARNCluster cluster =
         new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
             numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
-                enableAHS)) {
+            enableAHS)) {
       cluster.init(conf);
 
       // Verify that the timeline-service starts on ephemeral ports by default
       String hostname = MiniYARNCluster.getHostname();
-      Assert.assertEquals(hostname + ":0",
-        conf.get(YarnConfiguration.TIMELINE_SERVICE_ADDRESS));
+      assertEquals(hostname + ":0",
+          conf.get(YarnConfiguration.TIMELINE_SERVICE_ADDRESS));
 
       cluster.start();
 
       //Timeline service may sometime take a while to get started
       int wait = 0;
-      while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+      while (cluster.getApplicationHistoryServer() == null && wait < 20) {
         Thread.sleep(500);
         wait++;
       }
       //verify that the timeline service is started.
-      Assert.assertNotNull("Timeline Service should have been started",
-          cluster.getApplicationHistoryServer());
+      assertNotNull(cluster.getApplicationHistoryServer(),
+          "Timeline Service should have been started");
     }
     /*
      * Timeline service should start if TIMELINE_SERVICE_ENABLED == false
@@ -93,24 +98,24 @@ public void testTimelineServiceStartInMiniCluster() throws Exception {
     try (MiniYARNCluster cluster =
         new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
             numNodeManagers, numLocalDirs, numLogDirs, numLogDirs,
-                enableAHS)) {
+            enableAHS)) {
       cluster.init(conf);
       cluster.start();
 
       //Timeline service may sometime take a while to get started
       int wait = 0;
-      while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+      while (cluster.getApplicationHistoryServer() == null && wait < 20) {
         Thread.sleep(500);
         wait++;
       }
       //verify that the timeline service is started.
-      Assert.assertNotNull("Timeline Service should have been started",
-          cluster.getApplicationHistoryServer());
+      assertNotNull(cluster.getApplicationHistoryServer(),
+          "Timeline Service should have been started");
     }
   }
 
   @Test
-  public void testMultiRMConf() throws IOException {
+  void testMultiRMConf() throws IOException {
     String RM1_NODE_ID = "rm1", RM2_NODE_ID = "rm2";
     int RM1_PORT_BASE = 10000, RM2_PORT_BASE = 20000;
     Configuration conf = new YarnConfiguration();
@@ -130,22 +135,22 @@ public void testMultiRMConf() throws IOException {
       cluster.init(conf);
       Configuration conf1 = cluster.getResourceManager(0).getConfig(),
           conf2 = cluster.getResourceManager(1).getConfig();
-      Assert.assertFalse(conf1 == conf2);
-      Assert.assertEquals("0.0.0.0:18032",
+      assertFalse(conf1 == conf2);
+      assertEquals("0.0.0.0:18032",
           conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
               RM1_NODE_ID)));
-      Assert.assertEquals("0.0.0.0:28032",
+      assertEquals("0.0.0.0:28032",
           conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
               RM2_NODE_ID)));
-      Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID));
+      assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID));
 
-      Assert.assertEquals("0.0.0.0:18032",
+      assertEquals("0.0.0.0:18032",
           conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
               RM1_NODE_ID)));
-      Assert.assertEquals("0.0.0.0:28032",
+      assertEquals("0.0.0.0:28032",
           conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS,
               RM2_NODE_ID)));
-      Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID));
+      assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID));
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
index d1ccc9a6fec19..9c7f4447c6c22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
@@ -18,9 +18,9 @@
 
 package org.apache.hadoop.yarn.server;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -39,8 +39,9 @@
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 public class TestMiniYarnClusterNodeUtilization {
   // Mini YARN cluster setup
@@ -72,7 +73,7 @@ public class TestMiniYarnClusterNodeUtilization {
 
   private NodeStatus nodeStatus;
 
-  @Before
+  @BeforeEach
   public void setup() {
     conf = new YarnConfiguration();
     conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");
@@ -81,7 +82,7 @@ public void setup() {
     cluster = new MiniYARNCluster(name, NUM_RM, NUM_NM, 1, 1);
     cluster.init(conf);
     cluster.start();
-    assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+    assertFalse(-1 == cluster.getActiveRMIndex(), "RM never turned active");
 
     nm = (CustomNodeManager)cluster.getNodeManager(0);
     nodeStatus = createNodeStatus(nm.getNMContext().getNodeId(), 0,
@@ -95,11 +96,12 @@ public void setup() {
    * both the RMNode and SchedulerNode have been updated with the new
    * utilization.
    */
-  @Test(timeout=60000)
-  public void testUpdateNodeUtilization()
+  @Test
+  @Timeout(60000)
+  void testUpdateNodeUtilization()
       throws InterruptedException, IOException, YarnException {
-    assertTrue("NMs fail to connect to the RM",
-        cluster.waitForNodeManagersToConnect(10000));
+    assertTrue(cluster.waitForNodeManagersToConnect(10000),
+        "NMs fail to connect to the RM");
 
     // Give the heartbeat time to propagate to the RM
     verifySimulatedUtilization();
@@ -119,11 +121,12 @@ public void testUpdateNodeUtilization()
    * Verify both the RMNode and SchedulerNode have been updated with the new
    * utilization.
    */
-  @Test(timeout=60000)
-  public void testMockNodeStatusHeartbeat()
+  @Test
+  @Timeout(60000)
+  void testMockNodeStatusHeartbeat()
       throws InterruptedException, YarnException {
-    assertTrue("NMs fail to connect to the RM",
-        cluster.waitForNodeManagersToConnect(10000));
+    assertTrue(cluster.waitForNodeManagersToConnect(10000),
+        "NMs fail to connect to the RM");
 
     NodeStatusUpdater updater = nm.getNodeStatusUpdater();
     updater.sendOutofBandHeartBeat();
@@ -196,12 +199,12 @@ private void verifySimulatedUtilization() throws InterruptedException {
 
     // Give the heartbeat time to propagate to the RM (max 10 seconds)
     // We check if the nodeUtilization is up to date
-    for (int i=0; i<100; i++) {
+    for (int i = 0; i < 100; i++) {
       for (RMNode ni : rmContext.getRMNodes().values()) {
         if (ni.getNodeUtilization() != null) {
-            if (ni.getNodeUtilization().equals(nodeUtilization)) {
-              break;
-            }
+          if (ni.getNodeUtilization().equals(nodeUtilization)) {
+            break;
+          }
         }
       }
       Thread.sleep(100);
@@ -210,22 +213,18 @@ private void verifySimulatedUtilization() throws InterruptedException {
     // Verify the data is readable from the RM and scheduler nodes
     for (RMNode ni : rmContext.getRMNodes().values()) {
       ResourceUtilization cu = ni.getAggregatedContainersUtilization();
-      assertEquals("Containers Utillization not propagated to RMNode",
-          containersUtilization, cu);
+      assertEquals(containersUtilization, cu, "Containers Utillization not propagated to RMNode");
 
       ResourceUtilization nu = ni.getNodeUtilization();
-      assertEquals("Node Utillization not propagated to RMNode",
-          nodeUtilization, nu);
+      assertEquals(nodeUtilization, nu, "Node Utillization not propagated to RMNode");
 
-      SchedulerNode scheduler =
-          rmContext.getScheduler().getSchedulerNode(ni.getNodeID());
+      SchedulerNode scheduler = rmContext.getScheduler().getSchedulerNode(ni.getNodeID());
       cu = scheduler.getAggregatedContainersUtilization();
-      assertEquals("Containers Utillization not propagated to SchedulerNode",
-          containersUtilization, cu);
+      assertEquals(containersUtilization, cu,
+          "Containers Utillization not propagated to SchedulerNode");
 
       nu = scheduler.getNodeUtilization();
-      assertEquals("Node Utillization not propagated to SchedulerNode",
-          nodeUtilization, nu);
+      assertEquals(nodeUtilization, nu, "Node Utillization not propagated to SchedulerNode");
     }
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java
index ba14491bf511c..ae12658a299eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java
@@ -22,10 +22,12 @@
 import java.io.IOException;
 import java.util.UUID;
 
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
@@ -36,7 +38,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.kerby.util.IOUtil;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 public class TestRMNMSecretKeys {
   private static final String KRB5_CONF = "java.security.krb5.conf";
@@ -44,7 +49,7 @@ public class TestRMNMSecretKeys {
       System.getProperty("test.build.dir", "target/test-dir"),
           UUID.randomUUID().toString());
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws IOException {
     KRB5_CONF_ROOT_DIR.mkdir();
     File krb5ConfFile = new File(KRB5_CONF_ROOT_DIR, "krb5.conf");
@@ -63,17 +68,18 @@ public static void setup() throws IOException {
     System.setProperty(KRB5_CONF, krb5ConfFile.getAbsolutePath());
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws IOException {
     KRB5_CONF_ROOT_DIR.delete();
   }
 
-  @Test(timeout = 1000000)
-  public void testNMUpdation() throws Exception {
+  @Test
+  @Timeout(1000000)
+  void testNMUpdation() throws Exception {
     YarnConfiguration conf = new YarnConfiguration();
     // validating RM NM keys for Unsecured environment
     validateRMNMKeyExchange(conf);
-    
+
     // validating RM NM keys for secured environment
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
         "kerberos");
@@ -113,30 +119,30 @@ protected void startWepApp() {
     
     MasterKey containerTokenMasterKey =
         registrationResponse.getContainerTokenMasterKey();
-    Assert.assertNotNull(containerToken
-        + "Registration should cause a key-update!", containerTokenMasterKey);
+    assertNotNull(containerTokenMasterKey, containerToken
+        + "Registration should cause a key-update!");
     MasterKey nmTokenMasterKey = registrationResponse.getNMTokenMasterKey();
-    Assert.assertNotNull(nmToken
-        + "Registration should cause a key-update!", nmTokenMasterKey);
+    assertNotNull(nmTokenMasterKey, nmToken
+        + "Registration should cause a key-update!");
     
     dispatcher.await();
 
     NodeHeartbeatResponse response = nm.nodeHeartbeat(true);
-    Assert.assertNull(containerToken +
-        "First heartbeat after registration shouldn't get any key updates!",
-        response.getContainerTokenMasterKey());
-    Assert.assertNull(nmToken +
-        "First heartbeat after registration shouldn't get any key updates!",
-        response.getNMTokenMasterKey());
+    assertNull(response.getContainerTokenMasterKey(),
+        containerToken +
+        "First heartbeat after registration shouldn't get any key updates!");
+    assertNull(response.getNMTokenMasterKey(),
+        nmToken +
+        "First heartbeat after registration shouldn't get any key updates!");
     dispatcher.await();
 
     response = nm.nodeHeartbeat(true);
-    Assert.assertNull(containerToken +
-        "Even second heartbeat after registration shouldn't get any key updates!",
-        response.getContainerTokenMasterKey());
-    Assert.assertNull(nmToken +
-        "Even second heartbeat after registration shouldn't get any key updates!",
-        response.getContainerTokenMasterKey());
+    assertNull(response.getContainerTokenMasterKey(),
+        containerToken +
+        "Even second heartbeat after registration shouldn't get any key updates!");
+    assertNull(response.getContainerTokenMasterKey(),
+        nmToken +
+        "Even second heartbeat after registration shouldn't get any key updates!");
     
     dispatcher.await();
 
@@ -146,30 +152,30 @@ protected void startWepApp() {
 
     // Heartbeats after roll-over and before activation should be fine.
     response = nm.nodeHeartbeat(true);
-    Assert.assertNotNull(containerToken +
-        "Heartbeats after roll-over and before activation should not err out.",
-        response.getContainerTokenMasterKey());
-    Assert.assertNotNull(nmToken +
-        "Heartbeats after roll-over and before activation should not err out.",
-        response.getNMTokenMasterKey());
+    assertNotNull(response.getContainerTokenMasterKey(),
+        containerToken +
+        "Heartbeats after roll-over and before activation should not err out.");
+    assertNotNull(response.getNMTokenMasterKey(),
+        nmToken +
+        "Heartbeats after roll-over and before activation should not err out.");
     
-    Assert.assertEquals(containerToken +
-        "Roll-over should have incremented the key-id only by one!",
-        containerTokenMasterKey.getKeyId() + 1,
-        response.getContainerTokenMasterKey().getKeyId());
-    Assert.assertEquals(nmToken +
-        "Roll-over should have incremented the key-id only by one!",
-        nmTokenMasterKey.getKeyId() + 1,
-        response.getNMTokenMasterKey().getKeyId());
+    assertEquals(containerTokenMasterKey.getKeyId() + 1,
+        response.getContainerTokenMasterKey().getKeyId(),
+        containerToken +
+        "Roll-over should have incremented the key-id only by one!");
+    assertEquals(nmTokenMasterKey.getKeyId() + 1,
+        response.getNMTokenMasterKey().getKeyId(),
+        nmToken +
+        "Roll-over should have incremented the key-id only by one!");
     dispatcher.await();
 
     response = nm.nodeHeartbeat(true);
-    Assert.assertNull(containerToken +
-        "Second heartbeat after roll-over shouldn't get any key updates!",
-        response.getContainerTokenMasterKey());
-    Assert.assertNull(nmToken +
-        "Second heartbeat after roll-over shouldn't get any key updates!",
-        response.getNMTokenMasterKey());
+    assertNull(response.getContainerTokenMasterKey(),
+        containerToken +
+        "Second heartbeat after roll-over shouldn't get any key updates!");
+    assertNull(response.getNMTokenMasterKey(),
+        nmToken +
+        "Second heartbeat after roll-over shouldn't get any key updates!");
     dispatcher.await();
 
     // Let's force activation
@@ -177,21 +183,21 @@ protected void startWepApp() {
     rm.getRMContext().getNMTokenSecretManager().activateNextMasterKey();
 
     response = nm.nodeHeartbeat(true);
-    Assert.assertNull(containerToken
-        + "Activation shouldn't cause any key updates!",
-        response.getContainerTokenMasterKey());
-    Assert.assertNull(nmToken
-        + "Activation shouldn't cause any key updates!",
-        response.getNMTokenMasterKey());
+    assertNull(response.getContainerTokenMasterKey(),
+        containerToken
+        + "Activation shouldn't cause any key updates!");
+    assertNull(response.getNMTokenMasterKey(),
+        nmToken
+        + "Activation shouldn't cause any key updates!");
     dispatcher.await();
 
     response = nm.nodeHeartbeat(true);
-    Assert.assertNull(containerToken +
-        "Even second heartbeat after activation shouldn't get any key updates!",
-        response.getContainerTokenMasterKey());
-    Assert.assertNull(nmToken +
-        "Even second heartbeat after activation shouldn't get any key updates!",
-        response.getNMTokenMasterKey());
+    assertNull(response.getContainerTokenMasterKey(),
+        containerToken +
+        "Even second heartbeat after activation shouldn't get any key updates!");
+    assertNull(response.getNMTokenMasterKey(),
+        nmToken +
+        "Even second heartbeat after activation shouldn't get any key updates!");
     dispatcher.await();
 
     rm.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 21a77d89e2c41..871cbb6c0368b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.yarn.server.timelineservice;
 
 
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -54,9 +54,9 @@
 import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 public class TestTimelineServiceClientIntegration {
   private static final String ROOT_DIR = new File("target",
@@ -66,7 +66,7 @@ public class TestTimelineServiceClientIntegration {
   private static PerNodeTimelineCollectorsAuxService auxService;
   private static Configuration conf;
 
-  @BeforeClass
+  @BeforeAll
   public static void setupClass() throws Exception {
     try {
       collectorManager = new MockNodeTimelineCollectorManager();
@@ -88,7 +88,7 @@ public static void setupClass() throws Exception {
     }
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDownClass() throws Exception {
     if (auxService != null) {
       auxService.stop();
@@ -97,7 +97,7 @@ public static void tearDownClass() throws Exception {
   }
 
   @Test
-  public void testPutEntities() throws Exception {
+  void testPutEntities() throws Exception {
     TimelineV2Client client =
         TimelineV2Client.createTimelineClient(ApplicationId.newInstance(0, 1));
     try {
@@ -123,7 +123,7 @@ public void testPutEntities() throws Exception {
   }
 
   @Test
-  public void testPutExtendedEntities() throws Exception {
+  void testPutExtendedEntities() throws Exception {
     ApplicationId appId = ApplicationId.newInstance(0, 1);
     TimelineV2Client client =
         TimelineV2Client.createTimelineClient(appId);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
index f773807f05d36..4b041d7015573 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
@@ -18,11 +18,11 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.security;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.atLeastOnce;
@@ -75,18 +75,16 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
 import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 /**
  * Tests timeline authentication filter based security for timeline service v2.
  */
-@RunWith(Parameterized.class)
 public class TestTimelineAuthFilterForV2 {
 
   private static final String FOO_USER = "foo";
@@ -106,9 +104,8 @@ public class TestTimelineAuthFilterForV2 {
 
   // First param indicates whether HTTPS access or HTTP access and second param
   // indicates whether it is kerberos access or token based access.
-  @Parameterized.Parameters
   public static Collection<Object[]> params() {
-    return Arrays.asList(new Object[][] {{false, true}, {false, false},
+    return Arrays.asList(new Object[][]{{false, true}, {false, false},
         {true, false}, {true, true}});
   }
 
@@ -117,11 +114,14 @@ public static Collection<Object[]> params() {
   private static String sslConfDir;
   private static Configuration conf;
   private static UserGroupInformation nonKerberosUser;
+
   static {
     try {
       nonKerberosUser = UserGroupInformation.getCurrentUser();
-    } catch (IOException e) {}
+    } catch (IOException e) {
+    }
   }
+
   // Indicates whether HTTPS or HTTP access.
   private boolean withSsl;
   // Indicates whether Kerberos based login is used or token based access is
@@ -129,13 +129,14 @@ public static Collection<Object[]> params() {
   private boolean withKerberosLogin;
   private NodeTimelineCollectorManager collectorManager;
   private PerNodeTimelineCollectorsAuxService auxService;
-  public TestTimelineAuthFilterForV2(boolean withSsl,
+
+  public void initTestTimelineAuthFilterForV2(boolean withSsl,
       boolean withKerberosLogin) {
     this.withSsl = withSsl;
     this.withKerberosLogin = withKerberosLogin;
   }
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() {
     try {
       testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
@@ -181,7 +182,7 @@ public static void setup() {
     }
   }
 
-  @Before
+  @BeforeEach
   public void initialize() throws Exception {
     if (withSsl) {
       conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
@@ -221,7 +222,7 @@ public void initialize() throws Exception {
         appId, UserGroupInformation.getCurrentUser().getUserName());
     if (!withKerberosLogin) {
       AppLevelTimelineCollector collector =
-          (AppLevelTimelineCollector)collectorManager.get(appId);
+          (AppLevelTimelineCollector) collectorManager.get(appId);
       Token<TimelineDelegationTokenIdentifier> token =
           collector.getDelegationTokenForApp();
       token.setService(new Text("localhost" + token.getService().toString().
@@ -243,7 +244,7 @@ private TimelineV2Client createTimelineClientForUGI(ApplicationId appId) {
     return client;
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     if (testMiniKDC != null) {
       testMiniKDC.stop();
@@ -251,7 +252,7 @@ public static void tearDown() throws Exception {
     FileUtil.fullyDelete(TEST_ROOT_DIR);
   }
 
-  @After
+  @AfterEach
   public void destroy() throws Exception {
     if (auxService != null) {
       auxService.stop();
@@ -318,7 +319,7 @@ private void publishAndVerifyEntity(ApplicationId appId, File entityTypeDir,
       String entityType, int numEntities) throws Exception {
     TimelineV2Client client = createTimelineClientForUGI(appId);
     try {
-    // Sync call. Results available immediately.
+      // Sync call. Results available immediately.
       client.putEntities(createEntity("entity1", entityType));
       assertEquals(numEntities, entityTypeDir.listFiles().length);
       verifyEntity(entityTypeDir, "entity1", entityType);
@@ -343,8 +344,10 @@ private boolean publishWithRetries(ApplicationId appId, File entityTypeDir,
     return false;
   }
 
-  @Test
-  public void testPutTimelineEntities() throws Exception {
+  @MethodSource("params")
+  @ParameterizedTest
+  void testPutTimelineEntities(boolean withSsl, boolean withKerberosLogin) throws Exception {
+    initTestTimelineAuthFilterForV2(withSsl, withKerberosLogin);
     final String entityType = ENTITY_TYPE +
         ENTITY_TYPE_SUFFIX.getAndIncrement();
     ApplicationId appId = ApplicationId.newInstance(0, 1);
@@ -364,8 +367,8 @@ public Void call() throws Exception {
         }
       });
     } else {
-      assertTrue("Entities should have been published successfully.",
-          publishWithRetries(appId, entityTypeDir, entityType, 1));
+      assertTrue(publishWithRetries(appId, entityTypeDir, entityType, 1),
+          "Entities should have been published successfully.");
 
       AppLevelTimelineCollector collector =
           (AppLevelTimelineCollector) collectorManager.get(appId);
@@ -377,12 +380,12 @@ public Void call() throws Exception {
       // published.
       Thread.sleep(1000);
       // Entities should publish successfully after renewal.
-      assertTrue("Entities should have been published successfully.",
-          publishWithRetries(appId, entityTypeDir, entityType, 2));
+      assertTrue(publishWithRetries(appId, entityTypeDir, entityType, 2),
+          "Entities should have been published successfully.");
       assertNotNull(collector);
       verify(collectorManager.getTokenManagerService(), atLeastOnce()).
           renewToken(eq(collector.getDelegationTokenForApp()),
-              any(String.class));
+          any(String.class));
 
       // Wait to ensure lifetime of token expires and ensure its regenerated
       // automatically.
@@ -393,8 +396,9 @@ public Void call() throws Exception {
         }
         Thread.sleep(50);
       }
-      assertNotEquals("Token should have been regenerated.", token,
-          collector.getDelegationTokenForApp());
+      assertNotEquals(token,
+          collector.getDelegationTokenForApp(),
+          "Token should have been regenerated.");
       Thread.sleep(1000);
       // Try publishing with the old token in UGI. Publishing should fail due
       // to invalid token.
@@ -402,8 +406,8 @@ public Void call() throws Exception {
         publishAndVerifyEntity(appId, entityTypeDir, entityType, 2);
         fail("Exception should have been thrown due to Invalid Token.");
       } catch (YarnException e) {
-        assertTrue("Exception thrown should have been due to Invalid Token.",
-            e.getCause().getMessage().contains("InvalidToken"));
+        assertTrue(e.getCause().getMessage().contains("InvalidToken"),
+            "Exception thrown should have been due to Invalid Token.");
       }
 
       // Update the regenerated token in UGI and retry publishing entities.
@@ -411,10 +415,10 @@ public Void call() throws Exception {
           collector.getDelegationTokenForApp();
       regeneratedToken.setService(new Text("localhost" +
           regeneratedToken.getService().toString().substring(
-          regeneratedToken.getService().toString().indexOf(":"))));
+              regeneratedToken.getService().toString().indexOf(":"))));
       UserGroupInformation.getCurrentUser().addToken(regeneratedToken);
-      assertTrue("Entities should have been published successfully.",
-          publishWithRetries(appId, entityTypeDir, entityType, 2));
+      assertTrue(publishWithRetries(appId, entityTypeDir, entityType, 2),
+          "Entities should have been published successfully.");
       // Token was generated twice, once when app collector was created and
       // later after token lifetime expiry.
       verify(collectorManager.getTokenManagerService(), times(2)).
@@ -432,11 +436,11 @@ public Void call() throws Exception {
       }
       Thread.sleep(50);
     }
-    assertNotNull("Error reading entityTypeDir", entities);
+    assertNotNull(entities, "Error reading entityTypeDir");
     assertEquals(2, entities.length);
     verifyEntity(entityTypeDir, "entity2", entityType);
     AppLevelTimelineCollector collector =
-        (AppLevelTimelineCollector)collectorManager.get(appId);
+        (AppLevelTimelineCollector) collectorManager.get(appId);
     assertNotNull(collector);
     auxService.removeApplication(appId);
     verify(collectorManager.getTokenManagerService()).cancelToken(
@@ -446,24 +450,20 @@ public Void call() throws Exception {
   private static class DummyNodeTimelineCollectorManager extends
       NodeTimelineCollectorManager {
     private volatile int tokenExpiredCnt = 0;
-    DummyNodeTimelineCollectorManager() {
-      super();
-    }
 
     private int getTokenExpiredCnt() {
       return tokenExpiredCnt;
     }
 
     @Override
-    protected TimelineV2DelegationTokenSecretManagerService
-        createTokenManagerService() {
+    protected TimelineV2DelegationTokenSecretManagerService createTokenManagerService() {
       return spy(new TimelineV2DelegationTokenSecretManagerService() {
         @Override
         protected AbstractDelegationTokenSecretManager
-            <TimelineDelegationTokenIdentifier>
-            createTimelineDelegationTokenSecretManager(long secretKeyInterval,
-                long tokenMaxLifetime, long tokenRenewInterval,
-                long tokenRemovalScanInterval) {
+            <TimelineDelegationTokenIdentifier> createTimelineDelegationTokenSecretManager(long
+            secretKeyInterval,
+            long tokenMaxLifetime, long tokenRenewInterval,
+            long tokenRemovalScanInterval) {
           return spy(new TimelineV2DelegationTokenSecretManager(
               secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, 2000L) {
             @Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
index 835f2fed09555..1b80d25830796 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml
@@ -71,7 +71,7 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
-     <dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>
     </dependency>
@@ -129,6 +129,26 @@
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index 5eab08badfd4c..1f4a9f42a9f8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -25,14 +25,19 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.ServiceOperations;
 import org.apache.hadoop.ipc.CallerContext;
@@ -59,6 +64,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.net.MalformedURLException;
@@ -70,6 +77,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
@@ -132,6 +140,11 @@ public class EntityGroupFSTimelineStore extends CompositeService
   private long logRetainMillis;
   private long unknownActiveMillis;
   private int appCacheMaxSize = 0;
+  private boolean recoveryEnabled;
+  private Path checkpointFile;
+  private ConcurrentMap<String, Pair<Long, Long>> recoveredLogs =
+      new ConcurrentHashMap<String, Pair<Long, Long>>();
+
   private List<TimelineEntityGroupPlugin> cacheIdPlugins;
   private Map<TimelineEntityGroupId, EntityCacheItem> cachedLogs;
   private boolean aclsEnabled;
@@ -205,6 +218,11 @@ protected boolean removeEldestEntry(
         YarnConfiguration
             .TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT));
     fs = activeRootPath.getFileSystem(conf);
+    checkpointFile = new Path(fs.getHomeDirectory(), "atscheckpoint");
+    recoveryEnabled = conf.getBoolean(
+        YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED,
+        YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED_DEFAULT);
+
     aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,
     YarnConfiguration.DEFAULT_YARN_ACL_ENABLE);
     CallerContext.setCurrent(
@@ -293,6 +311,15 @@ protected void serviceStart() throws Exception {
       fs.setPermission(doneRootPath, DONE_DIR_PERMISSION);
     }
 
+    // Recover the lastProcessedTime and offset for logfiles
+    if (recoveryEnabled && fs.exists(checkpointFile)) {
+      try (FSDataInputStream in = fs.open(checkpointFile)) {
+        recoveredLogs.putAll(recoverLogFiles(in));
+      } catch (IOException e) {
+        LOG.warn("Failed to recover summarylog files from the checkpointfile", e);
+      }
+    }
+
     objMapper = new ObjectMapper();
     objMapper.setAnnotationIntrospector(
         new JaxbAnnotationIntrospector(TypeFactory.defaultInstance()));
@@ -352,10 +379,62 @@ protected void serviceStop() throws Exception {
     super.serviceStop();
   }
 
+  /* Returns Map of SummaryLog files. The Value Pair has
+  lastProcessedTime and offset */
+  HashMap<String, Pair<Long, Long>> recoverLogFiles(
+      DataInputStream in) throws IOException {
+    HashMap<String, Pair<Long, Long>> logFiles = new HashMap<>();
+    long totalEntries = in.readLong();
+    for (long i = 0; i < totalEntries; i++) {
+      Text attemptDirName = new Text();
+      attemptDirName.readFields(in);
+      Text fileName = new Text();
+      fileName.readFields(in);
+      LongWritable lastProcessedTime = new LongWritable();
+      lastProcessedTime.readFields(in);
+      LongWritable offset = new LongWritable();
+      offset.readFields(in);
+      Pair<Long, Long> pair = Pair.of(lastProcessedTime.get(), offset.get());
+      logFiles.put(attemptDirName + Path.SEPARATOR + fileName, pair);
+    }
+    LOG.info("Recovered {} summarylog files", totalEntries);
+    return logFiles;
+  }
+
+  // Stores set of SummaryLog files
+  void storeLogFiles(Collection<AppLogs> appLogs,
+      DataOutputStream checkPointStream) throws IOException {
+    long totalEntries = 0L;
+    for (AppLogs appLog : appLogs) {
+      totalEntries += appLog.summaryLogs.size();
+    }
+    checkPointStream.writeLong(totalEntries);
+    for (AppLogs appLog : appLogs) {
+      for (LogInfo summaryLog : appLog.summaryLogs) {
+        new Text(summaryLog.getAttemptDirName()).write(checkPointStream);
+        new Text(summaryLog.getFilename()).write(checkPointStream);
+        new LongWritable(summaryLog.getLastProcessedTime()).write(checkPointStream);
+        new LongWritable(summaryLog.getOffset()).write(checkPointStream);
+      }
+    }
+    LOG.info("Stored {} summarylog files into checkPointFile", totalEntries);
+  }
+
   @InterfaceAudience.Private
   @VisibleForTesting
   int scanActiveLogs() throws IOException {
     long startTime = Time.monotonicNow();
+    // Store the Last Processed Time and Offset
+    if (recoveryEnabled && appIdLogMap.size() > 0) {
+
+      try (FSDataOutputStream checkPointStream = fs.create(checkpointFile, true)) {
+
+        storeLogFiles(appIdLogMap.values(), checkPointStream);
+
+      } catch (Exception e) {
+        LOG.warn("Failed to checkpoint the summarylog files", e);
+      }
+    }
     int logsToScanCount = scanActiveLogs(activeRootPath);
     metrics.addActiveLogDirScanTime(Time.monotonicNow() - startTime);
     return logsToScanCount;
@@ -824,6 +903,15 @@ private void addSummaryLog(String attemptDirName,
         log = new EntityLogInfo(attemptDirName, filename, owner);
         summaryLogs.add(log);
       }
+      // This is to avoid processing summary files again during Restart of ATS
+      if (recoveryEnabled) {
+        Pair<Long, Long> pair = recoveredLogs.remove(log.getAttemptDirName()
+            + Path.SEPARATOR + log.getFilename());
+        if (pair != null) {
+          log.setLastProcessedTime(pair.getKey());
+          log.setOffset(pair.getValue());
+        }
+      }
     }
 
     private synchronized void addDetailLog(String attemptDirName,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
index a8cb9330b3284..8cc6b72ab9c7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
@@ -58,7 +58,17 @@ public void setOffset(long newOffset) {
     this.offset = newOffset;
   }
 
+
+  public long getLastProcessedTime() {
+    return lastProcessedTime;
+  }
+
+  public void setLastProcessedTime(long lastProcessedTime) {
+    this.lastProcessedTime = lastProcessedTime;
+  }
+
   private String attemptDirName;
+  private long lastProcessedTime = -1;
   private String filename;
   private String user;
   private long offset = 0;
@@ -108,22 +118,31 @@ public long parseForStore(TimelineDataManager tdm, Path appDirPath,
     FileStatus status = fs.getFileStatus(logPath);
     long numParsed = 0;
     if (status != null) {
-      long startTime = Time.monotonicNow();
-      try {
-        LOG.debug("Parsing {} at offset {}", logPath, offset);
-        long count = parsePath(tdm, logPath, appCompleted, jsonFactory,
-            objMapper, fs);
-        LOG.info("Parsed {} entities from {} in {} msec",
-            count, logPath, Time.monotonicNow() - startTime);
-        numParsed += count;
-      } catch (RuntimeException e) {
-        // If AppLogs cannot parse this log, it may be corrupted or just empty
-        if (e.getCause() instanceof JsonParseException &&
-            (status.getLen() > 0 || offset > 0)) {
-          // log on parse problems if the file as been read in the past or
-          // is visibly non-empty
-          LOG.info("Log {} appears to be corrupted. Skip. ", logPath);
+      long curModificationTime = status.getModificationTime();
+      if (curModificationTime > getLastProcessedTime()) {
+        long startTime = Time.monotonicNow();
+        try {
+          LOG.info("Parsing {} at offset {}", logPath, offset);
+          long count =
+              parsePath(tdm, logPath, appCompleted, jsonFactory, objMapper, fs);
+          setLastProcessedTime(curModificationTime);
+          LOG.info("Parsed {} entities from {} in {} msec", count, logPath,
+              Time.monotonicNow() - startTime);
+          numParsed += count;
+        } catch (RuntimeException e) {
+          // If AppLogs cannot parse this log, it may be corrupted or just empty
+          if (e.getCause() instanceof JsonParseException
+              && (status.getLen() > 0 || offset > 0)) {
+            // log on parse problems if the file as been read in the past or
+            // is visibly non-empty
+            LOG.info("Log {} appears to be corrupted. Skip. ", logPath);
+          } else {
+            LOG.error("Failed to parse " + logPath + " from offset " + offset,
+                e);
+          }
         }
+      } else {
+        LOG.info("Skip Parsing {} as there is no change", logPath);
       }
     } else {
       LOG.warn("{} no longer exists. Skip for scanning. ", logPath);
@@ -182,21 +201,19 @@ protected long doParse(TimelineDataManager tdm, JsonParser parser,
     long count = 0;
     TimelineEntities entities = new TimelineEntities();
     ArrayList<TimelineEntity> entityList = new ArrayList<TimelineEntity>(1);
-    long bytesParsed;
-    long bytesParsedLastBatch = 0;
     boolean postError = false;
     try {
       MappingIterator<TimelineEntity> iter = objMapper.readValues(parser,
           TimelineEntity.class);
-
+      long curPos;
       while (iter.hasNext()) {
         TimelineEntity entity = iter.next();
         String etype = entity.getEntityType();
         String eid = entity.getEntityId();
-        LOG.trace("Read entity {}", etype);
+        LOG.debug("Read entity {} of {}", eid, etype);
         ++count;
-        bytesParsed = parser.getCurrentLocation().getCharOffset() + 1;
-        LOG.trace("Parser now at offset {}", bytesParsed);
+        curPos = ((FSDataInputStream) parser.getInputSource()).getPos();
+        LOG.debug("Parser now at offset {}", curPos);
 
         try {
           LOG.debug("Adding {}({}) to store", eid, etype);
@@ -208,8 +225,7 @@ protected long doParse(TimelineDataManager tdm, JsonParser parser,
             LOG.warn("Error putting entity: {} ({}): {}",
                 e.getEntityId(), e.getEntityType(), e.getErrorCode());
           }
-          setOffset(getOffset() + bytesParsed - bytesParsedLastBatch);
-          bytesParsedLastBatch = bytesParsed;
+          setOffset(curPos);
           entityList.clear();
         } catch (YarnException e) {
           postError = true;
@@ -247,8 +263,7 @@ protected long doParse(TimelineDataManager tdm, JsonParser parser,
       ObjectMapper objMapper, UserGroupInformation ugi, boolean appCompleted)
       throws IOException {
     long count = 0;
-    long bytesParsed;
-    long bytesParsedLastBatch = 0;
+    long curPos;
     boolean putError = false;
     try {
       MappingIterator<TimelineDomain> iter = objMapper.readValues(parser,
@@ -259,13 +274,12 @@ protected long doParse(TimelineDataManager tdm, JsonParser parser,
         domain.setOwner(ugi.getShortUserName());
         LOG.trace("Read domain {}", domain.getId());
         ++count;
-        bytesParsed = parser.getCurrentLocation().getCharOffset() + 1;
-        LOG.trace("Parser now at offset {}", bytesParsed);
+        curPos = ((FSDataInputStream) parser.getInputSource()).getPos();
+        LOG.debug("Parser now at offset {}", curPos);
 
         try {
           tdm.putDomain(domain, ugi);
-          setOffset(getOffset() + bytesParsed - bytesParsedLastBatch);
-          bytesParsedLastBatch = bytesParsed;
+          setOffset(curPos);
         } catch (YarnException e) {
           putError = true;
           throw new IOException("Error posting domain", e);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/EntityGroupPlugInForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/EntityGroupPlugInForTest.java
index f239fac32395b..89208054de3f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/EntityGroupPlugInForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/EntityGroupPlugInForTest.java
@@ -20,7 +20,6 @@
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 
 import java.util.Collection;
 import java.util.Set;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
index 4609cb0b91db4..cb887fe264fab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
@@ -49,8 +49,8 @@
 import java.util.Map;
 import java.util.Set;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * Utility methods related to the ATS v1.5 plugin storage tests.
@@ -176,10 +176,8 @@ static void verifyTestEntities(TimelineDataManager tdm)
         UserGroupInformation.getLoginUser());
     assertNotNull(entity1);
     assertNotNull(entity2);
-    assertEquals("Failed to read out entity 1",
-        (Long) 123l, entity1.getStartTime());
-    assertEquals("Failed to read out entity 2",
-        (Long) 456l, entity2.getStartTime());
+    assertEquals((Long) 123l, entity1.getStartTime(), "Failed to read out entity 1");
+    assertEquals((Long) 456l, entity2.getStartTime(), "Failed to read out entity 2");
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index fdd47727c5018..748cf92747ecc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.timeline;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
@@ -37,21 +38,31 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore.AppState;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
@@ -61,13 +72,16 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -103,12 +117,10 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
   private EntityGroupFSTimelineStore store;
   private TimelineEntity entityNew;
 
-  @Rule
-  public TestName currTestName = new TestName();
   private File rootDir;
   private File testJar;
 
-  @BeforeClass
+  @BeforeAll
   public static void setupClass() throws Exception {
     config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
     config.set(
@@ -148,8 +160,8 @@ public static void setupClass() throws Exception {
         testActiveDirPath.toString());
   }
 
-  @Before
-  public void setup() throws Exception {
+  @BeforeEach
+  public void setup(TestInfo testInfo) throws Exception {
     for (ApplicationId appId : sampleAppIds) {
       Path attemotDirPath =
           new Path(new Path(testActiveDirPath, appId.toString()),
@@ -158,7 +170,7 @@ public void setup() throws Exception {
     }
 
     store = new EntityGroupFSTimelineStore();
-    if (currTestName.getMethodName().contains("Plugin")) {
+    if (testInfo.getTestMethod().get().getName().contains("Plugin")) {
       rootDir = GenericTestUtils.getTestDir(getClass()
           .getSimpleName());
       if (!rootDir.exists()) {
@@ -186,7 +198,7 @@ public void setup() throws Exception {
     store.start();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
     for (ApplicationId appId : sampleAppIds) {
@@ -198,7 +210,7 @@ public void tearDown() throws Exception {
     }
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDownClass() throws Exception {
     hdfsCluster.shutdown();
     FileContext fileContext = FileContext.getLocalFSFileContext();
@@ -207,10 +219,10 @@ public static void tearDownClass() throws Exception {
   }
 
   @Test
-  public void testAppLogsScanLogs() throws Exception {
+  void testAppLogsScanLogs() throws Exception {
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-        AppState.COMPLETED);
+            AppState.COMPLETED);
     appLogs.scanForLogs();
     List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
     List<LogInfo> detailLogs = appLogs.getDetailLogs();
@@ -230,13 +242,13 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testAppLogsDomainLogLastlyScanned() throws Exception {
+  void testAppLogsDomainLogLastlyScanned() throws Exception {
     EntityGroupFSTimelineStore.AppLogs appLogs =
-            store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-                    AppState.COMPLETED);
+        store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+            AppState.COMPLETED);
     Path attemptDirPath = new Path(new Path(testActiveDirPath,
             mainTestAppId.toString()),
-            getAttemptDirName(mainTestAppId));
+        getAttemptDirName(mainTestAppId));
     //Delete the domain log from AppDirPath so first scan won't find it
     fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false);
     appLogs.scanForLogs();
@@ -246,7 +258,7 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
 
     //Generate the domain log
     FSDataOutputStream out = fs.create(
-            new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
+        new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
     out.close();
 
     appLogs.scanForLogs();
@@ -255,10 +267,10 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testMoveToDone() throws Exception {
+  void testMoveToDone() throws Exception {
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-        AppState.COMPLETED);
+            AppState.COMPLETED);
     Path pathBefore = appLogs.getAppDirPath();
     appLogs.moveToDone();
     Path pathAfter = appLogs.getAppDirPath();
@@ -269,13 +281,13 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testParseSummaryLogs() throws Exception {
+  void testParseSummaryLogs() throws Exception {
     TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
     MutableCounterLong scanned = store.metrics.getEntitiesReadToSummary();
     long beforeScan = scanned.value();
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-        AppState.COMPLETED);
+            AppState.COMPLETED);
     appLogs.scanForLogs();
     appLogs.parseSummaryLogs(tdm);
     PluginStoreTestUtils.verifyTestEntities(tdm);
@@ -283,31 +295,31 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testWithAnonymousUser() throws Exception {
+  void testWithAnonymousUser() throws Exception {
     try {
       TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
       EntityGroupFSTimelineStore.AppLogs appLogs =
-              store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-                      AppState.COMPLETED);
+          store.new AppLogs(mainTestAppId, mainTestAppDirPath,
+              AppState.COMPLETED);
       FileStatus fileStatus = mock(FileStatus.class);
       when(fileStatus.getOwner()).thenReturn(null);
       appLogs.scanForLogs();
       appLogs.parseSummaryLogs(tdm);
       PluginStoreTestUtils.verifyTestEntities(tdm);
     } catch (IllegalArgumentException ie) {
-      Assert.fail("No exception needs to be thrown as anonymous user is configured");
+      fail("No exception needs to be thrown as anonymous user is configured");
     }
   }
 
   @Test
-  public void testCleanLogs() throws Exception {
+  void testCleanLogs() throws Exception {
     // Create test dirs and files
     // Irrelevant file, should not be reclaimed
     String appDirName = mainTestAppId.toString();
     String attemptDirName = ApplicationAttemptId.appAttemptIdStrPrefix
         + appDirName + "_1";
     Path irrelevantFilePath = new Path(
-            testDoneDirPath, "irrelevant.log");
+        testDoneDirPath, "irrelevant.log");
     FSDataOutputStream stream = fs.create(irrelevantFilePath);
     stream.close();
     // Irrelevant directory, should not be reclaimed
@@ -380,7 +392,7 @@ public void testCleanLogs() throws Exception {
   }
 
   @Test
-  public void testCleanBuckets() throws Exception {
+  void testCleanBuckets() throws Exception {
     // ClusterTimeStampDir with App Log Dirs
     Path clusterTimeStampDir1 = new Path(testDoneDirPath,
         Long.toString(sampleAppIds.get(0).getClusterTimestamp()));
@@ -426,17 +438,17 @@ public void testCleanBuckets() throws Exception {
   }
 
   @Test
-  public void testNullCheckGetEntityTimelines() throws Exception {
+  void testNullCheckGetEntityTimelines() throws Exception {
     try {
       store.getEntityTimelines("YARN_APPLICATION", null, null, null, null,
           null);
     } catch (NullPointerException e) {
-      Assert.fail("NPE when getEntityTimelines called with Null EntityIds");
+      fail("NPE when getEntityTimelines called with Null EntityIds");
     }
   }
 
   @Test
-  public void testPluginRead() throws Exception {
+  void testPluginRead() throws Exception {
     // Verify precondition
     assertEquals(EntityGroupPlugInForTest.class.getName(),
         store.getConfig().get(
@@ -444,8 +456,8 @@ public void testPluginRead() throws Exception {
     List<TimelineEntityGroupPlugin> currPlugins = store.getPlugins();
     for (TimelineEntityGroupPlugin plugin : currPlugins) {
       ClassLoader pluginClassLoader = plugin.getClass().getClassLoader();
-      assertTrue("Should set up ApplicationClassLoader",
-          pluginClassLoader instanceof ApplicationClassLoader);
+      assertTrue(pluginClassLoader instanceof ApplicationClassLoader,
+          "Should set up ApplicationClassLoader");
       URL[] paths = ((URLClassLoader) pluginClassLoader).getURLs();
       boolean foundJAR = false;
       for (URL path : paths) {
@@ -453,13 +465,13 @@ public void testPluginRead() throws Exception {
           foundJAR = true;
         }
       }
-      assertTrue("Not found path " + testJar.getAbsolutePath()
-          + " for plugin " + plugin.getClass().getName(), foundJAR);
+      assertTrue(foundJAR, "Not found path " + testJar.getAbsolutePath()
+          + " for plugin " + plugin.getClass().getName());
     }
     // Load data and cache item, prepare timeline store by making a cache item
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-        AppState.COMPLETED);
+            AppState.COMPLETED);
     EntityCacheItem cacheItem = new EntityCacheItem(
         EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId),
         config);
@@ -499,11 +511,11 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testSummaryRead() throws Exception {
+  void testSummaryRead() throws Exception {
     // Load data
     EntityGroupFSTimelineStore.AppLogs appLogs =
         store.new AppLogs(mainTestAppId, mainTestAppDirPath,
-        AppState.COMPLETED);
+            AppState.COMPLETED);
     MutableCounterLong summaryLogEntityRead
         = store.metrics.getGetEntityToSummaryOps();
     long numEntityReadBefore = summaryLogEntityRead.value();
@@ -528,7 +540,7 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath,
   }
 
   @Test
-  public void testGetEntityPluginRead() throws Exception {
+  void testGetEntityPluginRead() throws Exception {
     EntityGroupFSTimelineStore store = null;
     ApplicationId appId =
         ApplicationId.fromString("application_1501509265053_0001");
@@ -556,7 +568,7 @@ public void testGetEntityPluginRead() throws Exception {
   }
 
   @Test
-  public void testScanActiveLogsWithInvalidFile() throws Exception {
+  void testScanActiveLogsWithInvalidFile() throws Exception {
     Path invalidFile = new Path(testActiveDirPath, "invalidfile");
     try {
       if (!fs.exists(invalidFile)) {
@@ -564,7 +576,7 @@ public void testScanActiveLogsWithInvalidFile() throws Exception {
       }
       store.scanActiveLogs();
     } catch (StackOverflowError error) {
-      Assert.fail("EntityLogScanner crashed with StackOverflowError");
+      fail("EntityLogScanner crashed with StackOverflowError");
     } finally {
       if (fs.exists(invalidFile)) {
         fs.delete(invalidFile, false);
@@ -573,7 +585,7 @@ public void testScanActiveLogsWithInvalidFile() throws Exception {
   }
 
   @Test
-  public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception {
+  void testScanActiveLogsAndMoveToDonePluginRead() throws Exception {
     EntityGroupFSTimelineStore store = null;
     ApplicationId appId =
         ApplicationId.fromString("application_1501509265053_0002");
@@ -602,6 +614,147 @@ public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception {
     }
   }
 
+  // TestTimelineStore to validate the put entities call
+  static class TestTimelineStore extends LeveldbTimelineStore {
+    static final AtomicInteger ENTITIES_COUNT = new AtomicInteger(0);
+
+    TestTimelineStore() {
+      super();
+    }
+
+    @Override
+    public TimelinePutResponse put(TimelineEntities entities) {
+      ENTITIES_COUNT.getAndAdd(entities.getEntities().size());
+      return new TimelinePutResponse();
+    }
+
+    public static int getEntitiesCount() {
+      return ENTITIES_COUNT.get();
+    }
+  }
+
+  @Test
+  void testIfAnyDuplicateEntities() throws Exception {
+    // Create an application with some entities
+    ApplicationId appId =
+        ApplicationId.fromString("application_1501509265053_0002");
+    String user = UserGroupInformation.getCurrentUser().getShortUserName();
+    Path activeDirPath = getTestRootPath("active1");
+    Path doneDirPath = getTestRootPath("done1");
+    Path userBase = new Path(activeDirPath, user);
+    Path userAppRoot = new Path(userBase, appId.toString());
+    Path attemptDirPath = new Path(userAppRoot, getAttemptDirName(appId));
+
+    String logFileName = EntityGroupFSTimelineStore.ENTITY_LOG_PREFIX
+        + EntityGroupPlugInForTest.getStandardTimelineGroupId(appId);
+    createTestFiles(appId, attemptDirPath, logFileName);
+
+    // stop the default store before creating new store to get the lock
+    store.stop();
+    EntityGroupFSTimelineStore newStore = new EntityGroupFSTimelineStore() {
+      @Override
+      protected AppState getAppState(ApplicationId appId) throws IOException {
+        return AppState.ACTIVE;
+      }
+    };
+
+    try {
+      // Start ATS with TestTimelineStore
+      Configuration newConfig = new YarnConfiguration(config);
+      newConfig.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE,
+          TestTimelineStore.class.getName());
+      newConfig.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR,
+          doneDirPath.toString());
+      newConfig.set(
+          YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR,
+          activeDirPath.toString());
+      newStore.init(newConfig);
+      newStore.setFs(fs);
+      newStore.start();
+
+      // Validate if the initial entities count are correct
+      newStore.scanActiveLogs();
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override
+        public Boolean get() {
+          return TestTimelineStore.getEntitiesCount() == 2;
+        }
+      }, 100, 10000);
+      assertEquals(2, TestTimelineStore.getEntitiesCount(), "Wrong Initial Entities Count");
+
+      // Append the Summary log file with few more entities
+      TimelineEntities entities = PluginStoreTestUtils.generateTestEntities();
+      FSDataOutputStream outStream = fs.append(
+          new Path(attemptDirPath, TEST_SUMMARY_LOG_FILE_NAME));
+      JsonGenerator jsonGenerator
+          = new JsonFactory().createGenerator((OutputStream) outStream);
+      jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
+      ObjectMapper objMapper = new ObjectMapper();
+      objMapper.setAnnotationIntrospector(
+          new JaxbAnnotationIntrospector(TypeFactory.defaultInstance()));
+      objMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+      for (TimelineEntity entity : entities.getEntities()) {
+        objMapper.writeValue(jsonGenerator, entity);
+      }
+      outStream.close();
+
+      // Validate if there are any duplicates
+      newStore.scanActiveLogs();
+      GenericTestUtils.waitFor(new Supplier<Boolean>() {
+        @Override
+        public Boolean get() {
+          return TestTimelineStore.getEntitiesCount() == 4;
+        }
+      }, 100, 10000);
+      assertEquals(4, TestTimelineStore.getEntitiesCount(), "Duplicate Entities present");
+
+    } finally {
+      if (newStore != null) {
+        newStore.stop();
+      }
+      fs.delete(userAppRoot, true);
+    }
+  }
+
+  @Test
+  void testStateStoreAndRecovery() throws Exception {
+    // Prepare the AppLogs Data
+    EntityGroupFSTimelineStore.AppLogs appLogs =
+        store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED);
+    appLogs.scanForLogs();
+    List<LogInfo> summaryLogs = appLogs.getSummaryLogs();
+    List<EntityGroupFSTimelineStore.AppLogs> logsList = new ArrayList<>();
+    logsList.add(appLogs);
+
+    // Store the Log files
+    Path checkpointFile = new Path(fs.getHomeDirectory(), "atscheckpoint");
+    try (DataOutputStream dataOutputStream = fs.create(checkpointFile)) {
+      store.storeLogFiles(logsList, dataOutputStream);
+    } catch (IOException e) {
+      fail("Failed to store the log files");
+    }
+
+    // Recover the Log files and validate the contents
+    try (DataInputStream dataInputStream = fs.open(checkpointFile)) {
+      HashMap<String, Pair<Long, Long>> logFiles =
+          store.recoverLogFiles(dataInputStream);
+      assertEquals(summaryLogs.size(), logFiles.size());
+      for (LogInfo logInfo : summaryLogs) {
+        String logFileName = logInfo.getAttemptDirName() +
+            Path.SEPARATOR + logInfo.getFilename();
+        Pair<Long, Long> pair = logFiles.get(logFileName);
+        assertNotNull(pair, "Failed to recover " + logFileName);
+        assertTrue(logInfo.getLastProcessedTime() == pair.getLeft(),
+            "LastProcessedTime is not same");
+        assertTrue(logInfo.getOffset() == pair.getRight(),
+            "Offset is not same");
+      }
+    } catch (IOException e) {
+      fail("Failed to recover the log files");
+    }
+  }
+
+
   private EntityGroupFSTimelineStore createAndStartTimelineStore(
       AppState appstate) {
     // stop before creating new store to get the lock
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java
index 43b04a5a37097..7763956330e5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java
@@ -20,16 +20,17 @@
 
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertNotNull;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
 public class TestLevelDBCacheTimelineStore extends TimelineStoreTestUtils {
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     store = new LevelDBCacheTimelineStore("app1");
     store.init(new YarnConfiguration());
@@ -39,7 +40,7 @@ public void setup() throws Exception {
     loadTestDomainData();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     store.stop();
   }
@@ -49,13 +50,13 @@ public TimelineStore getTimelineStore() {
   }
 
   @Test
-  public void testDefaultConstructor() {
+  void testDefaultConstructor() {
     TimelineStore store = null;
     try {
       store = ReflectionUtils.newInstance(LevelDBCacheTimelineStore.class,
           new YarnConfiguration());
     } finally {
-      assertNotNull("LevelDBCacheTimelineStore failed to instantiate", store);
+      assertNotNull(store, "LevelDBCacheTimelineStore failed to instantiate");
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
index bccf4b844069c..22495eaf08069 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
@@ -34,19 +34,18 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.Charset;
-import java.util.EnumSet;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestLogInfo {
 
@@ -76,7 +75,7 @@ public class TestLogInfo {
 
   private static final short FILE_LOG_DIR_PERMISSIONS = 0770;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     config.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR.toString());
     HdfsConfiguration hdfsConfig = new HdfsConfiguration();
@@ -102,7 +101,7 @@ public void setup() throws Exception {
     writeBrokenFile(new Path(testAppDirPath, TEST_BROKEN_FILE_NAME));
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     jsonGenerator.close();
     outStream.close();
@@ -111,7 +110,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testMatchesGroupId() throws Exception {
+  void testMatchesGroupId() throws Exception {
     String testGroupId = "app1_group1";
     // Match
     EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME,
@@ -146,7 +145,7 @@ public void testMatchesGroupId() throws Exception {
   }
 
   @Test
-  public void testParseEntity() throws Exception {
+  void testParseEntity() throws Exception {
     // Load test data
     TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
     EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME,
@@ -156,28 +155,11 @@ public void testParseEntity() throws Exception {
         fs);
     // Verify for the first batch
     PluginStoreTestUtils.verifyTestEntities(tdm);
-    // Load new data
-    TimelineEntity entityNew = PluginStoreTestUtils
-        .createEntity("id_3", "type_3", 789l, null, null,
-            null, null, "domain_id_1");
-    TimelineEntities entityList = new TimelineEntities();
-    entityList.addEntity(entityNew);
-    writeEntitiesLeaveOpen(entityList,
-        new Path(getTestRootPath(TEST_ATTEMPT_DIR_NAME), TEST_ENTITY_FILE_NAME));
-    testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper,
-        fs);
-    // Verify the newly added data
-    TimelineEntity entity3 = tdm.getEntity(entityNew.getEntityType(),
-        entityNew.getEntityId(), EnumSet.allOf(TimelineReader.Field.class),
-        UserGroupInformation.getLoginUser());
-    assertNotNull(entity3);
-    assertEquals("Failed to read out entity new",
-        entityNew.getStartTime(), entity3.getStartTime());
     tdm.close();
   }
 
   @Test
-  public void testParseBrokenEntity() throws Exception {
+  void testParseBrokenEntity() throws Exception {
     // Load test data
     TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
     EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME,
@@ -195,7 +177,7 @@ public void testParseBrokenEntity() throws Exception {
   }
 
   @Test
-  public void testParseDomain() throws Exception {
+  void testParseDomain() throws Exception {
     // Load test data
     TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
     DomainLogInfo domainLogInfo = new DomainLogInfo(TEST_ATTEMPT_DIR_NAME,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
index c190266bfbdab..5b9500a401743 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java
@@ -22,20 +22,21 @@
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 public class TestOverrideTimelineStoreYarnClient {
 
   @Test
-  public void testLifecycleAndOverride() throws Throwable {
+  void testLifecycleAndOverride() throws Throwable {
     YarnConfiguration conf = new YarnConfiguration();
-    try(NoRMStore store = new NoRMStore()) {
+    try (NoRMStore store = new NoRMStore()) {
       store.init(conf);
       store.start();
-      Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE,
+      assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE,
           store.getAppState(ApplicationId.newInstance(1, 1)));
       store.stop();
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
index ccd1b13d1572a..17596405caf23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml
@@ -44,6 +44,12 @@
       <artifactId>hadoop-common</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.javassist</groupId>
+          <artifactId>javassist</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -55,6 +61,12 @@
       <artifactId>mockito-core</artifactId>
       <version>2.8.9</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.powermock</groupId>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
index 8de3b8645ac50..7d8098e40d7fd 100755
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
@@ -108,7 +108,7 @@ public TimelineHealth getHealthStatus() {
           "");
     } else {
       return new TimelineHealth(
-          TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE,
+          TimelineHealth.TimelineHealthStatus.CONNECTION_FAILURE,
           "Timeline store reader not initialized.");
     }
   }
@@ -131,4 +131,4 @@ private Set<TimelineEntity> applyFilters(TimelineEntityFilters filters,
     }
     return timelineEntities;
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineWriterImpl.java
index 572d888b5299e..0ea70f9dcf922 100755
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineWriterImpl.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
 import org.apache.hadoop.yarn.server.timelineservice.documentstore.lib.DocumentStoreVendor;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineAggregationTrack;
@@ -151,6 +152,11 @@ public TimelineWriteResponse write(TimelineCollectorContext context,
     return null;
   }
 
+  @Override
+  public TimelineHealth getHealthStatus() {
+    return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING, "");
+  }
+
   private void appendSubAppUserIfExists(TimelineCollectorContext context,
       String subApplicationUser) {
     String userId = context.getUserId();
@@ -282,4 +288,4 @@ public TimelineWriteResponse aggregate(TimelineEntity data,
   @Override
   public void flush() {
   }
-}
\ No newline at end of file
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java
index cf57085023f26..5d442152fe9ba 100755
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTestUtils.java
@@ -23,11 +23,12 @@
 import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.entity.TimelineEntityDocument;
 import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.flowactivity.FlowActivityDocument;
 import org.apache.hadoop.yarn.server.timelineservice.documentstore.collection.document.flowrun.FlowRunDocument;
-import org.codehaus.jackson.type.TypeReference;
 
 import java.io.IOException;
 import java.util.List;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+
 /**
  * This is util class for baking sample TimelineEntities data for test.
  */
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java
index c1da4f6ce4360..a644bc1d19103 100755
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/test/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/JsonUtils.java
@@ -18,12 +18,12 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.documentstore;
 
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
-
 import java.io.IOException;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 /**
  * A simple util class for Json SerDe.
  */
@@ -34,8 +34,7 @@ private JsonUtils(){}
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
   static {
-    OBJECT_MAPPER.configure(
-        DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
   }
 
   /**
@@ -48,7 +47,7 @@ private JsonUtils(){}
    * @throws IOException if Json String is not valid or error
    *                     while deserialization
    */
-  public static <T> T fromJson(final String jsonStr, final TypeReference type)
+  public static <T> T fromJson(final String jsonStr, final TypeReference<T> type)
       throws IOException {
     return OBJECT_MAPPER.readValue(jsonStr, type);
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index d8cf6ded0001e..a323eaff8558c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -96,6 +96,14 @@
           <groupId>tomcat</groupId>
           <artifactId>jasper-runtime</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -106,6 +114,12 @@
       <artifactId>hadoop-auth</artifactId>
       <version>${hbase-compatible-hadoop.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -117,6 +131,10 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -167,6 +185,19 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <version>${hbase-compatible-guice.version}</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-client</artifactId>
@@ -301,6 +332,10 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-auth</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -311,6 +346,12 @@
       <artifactId>hadoop-hdfs</artifactId>
       <version>${hbase-compatible-hadoop.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- 'mvn dependency:analyze' fails to detect use of this direct
@@ -321,6 +362,19 @@
       <version>${hbase-compatible-hadoop.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-client</artifactId>
+      <version>${hbase-compatible-hadoop.version}</version>
+      <scope>test</scope>
     </dependency>
 
     <dependency>
@@ -470,14 +524,6 @@
             </exclusion>
           </exclusions>
         </dependency>
-        <!-- 'mvn dependency:analyze' fails to detect use of this direct
-             dependency -->
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-hdfs-client</artifactId>
-          <version>${hbase-compatible-hadoop.version}</version>
-          <scope>test</scope>
-        </dependency>
         <!-- 'mvn dependency:analyze' fails to detect use of this direct
              dependency -->
         <!-- This is needed by HBaseTestingUtility -->
@@ -486,6 +532,30 @@
           <artifactId>mockito-core</artifactId>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-server</artifactId>
+          <scope>test</scope>
+          <version>${hbase-compatible-jetty.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-servlet</artifactId>
+          <scope>test</scope>
+          <version>${hbase-compatible-jetty.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-webapp</artifactId>
+          <scope>test</scope>
+          <version>${hbase-compatible-jetty.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+          <scope>test</scope>
+          <version>${hbase-compatible-jetty.version}</version>
+        </dependency>
       </dependencies>
     </profile>
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index b35b3dc8b79db..271c5e5ce139f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -2610,4 +2610,20 @@ public void testGetAppsMetricsRange() throws Exception {
       client.destroy();
     }
   }
+
+  @Test
+  public void testGetEntityWithSystemEntityType() throws Exception {
+    Client client = createClient();
+    try {
+      URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
+          "timeline/apps/application_1111111111_1111/" +
+          "entities/YARN_APPLICATION");
+      ClientResponse resp = getResponse(client, uri);
+      Set<TimelineEntity> entities =
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      assertEquals(0, entities.size());
+    } finally {
+      client.destroy();
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java
index cb89ba4223e6c..5d658b96c5b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterHBaseDown.java
@@ -43,12 +43,12 @@ public void testTimelineWriterHBaseDown() throws Exception {
     HBaseTestingUtility util = new HBaseTestingUtility();
     HBaseTimelineWriterImpl writer = new HBaseTimelineWriterImpl();
     try {
+      util.startMiniCluster();
       Configuration c1 = util.getConfiguration();
       c1.setLong(TIMELINE_SERVICE_READER_STORAGE_MONITOR_INTERVAL_MS, 5000);
       writer.init(c1);
       writer.start();
 
-      util.startMiniCluster();
       DataGeneratorForTest.createSchema(util.getConfiguration());
 
       TimelineStorageMonitor storageMonitor = writer.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
index 9a88e83109526..6c0a635bc8370 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml
@@ -37,6 +37,16 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -54,6 +64,24 @@
       <artifactId>hadoop-shaded-guava</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${hbase-compatible-guava.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <version>${hbase-compatible-guice.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-annotations</artifactId>
@@ -64,6 +92,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@@ -72,6 +106,12 @@
       <artifactId>hadoop-common</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -86,6 +126,21 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -97,6 +152,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -108,6 +169,16 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -128,6 +199,10 @@
           <groupId>org.mortbay.jetty</groupId>
           <artifactId>jetty-util</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -139,6 +214,10 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-mapreduce-client-core</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -187,14 +266,12 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-http</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
-
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index f3592d2924088..b6e1d76902841 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -115,7 +115,7 @@ public TimelineHealth getHealthStatus() {
           "");
     } catch (IOException e){
       return new TimelineHealth(
-          TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE,
+          TimelineHealth.TimelineHealthStatus.CONNECTION_FAILURE,
           "HBase connection is down");
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index dda004d7ec928..7233dab34572b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import  org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.SubApplicationEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
@@ -604,6 +605,19 @@ public TimelineWriteResponse aggregate(TimelineEntity data,
     return null;
   }
 
+  @Override
+  public TimelineHealth getHealthStatus() {
+    try {
+      storageMonitor.checkStorageIsUp();
+      return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING,
+          "");
+    } catch (IOException e){
+      return new TimelineHealth(
+          TimelineHealth.TimelineHealthStatus.CONNECTION_FAILURE,
+          "HBase connection is down");
+    }
+  }
+
   /*
    * (non-Javadoc)
    *
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
index 05c4e570d65b7..3ea18adb94328 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTableRW.java
@@ -26,7 +26,7 @@
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 6b492c3553e99..fa3d554178f1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -58,8 +58,6 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 /**
  * Timeline entity reader for application entities that are stored in the
  * application table.
@@ -336,21 +334,29 @@ protected Result getResult(Configuration hbaseConf, Connection conn,
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext(), "context shouldn't be null");
-    Preconditions.checkNotNull(
-        getDataToRetrieve(), "data to retrieve shouldn't be null");
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getEntityType(),
-        "entityType shouldn't be null");
+    if (getContext() == null) {
+      throw new NullPointerException("context shouldn't be null");
+    }
+    if (getDataToRetrieve() == null) {
+      throw new NullPointerException("data to retrieve shouldn't be null");
+    }
+    if (getContext().getClusterId() == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
+    if (getContext().getEntityType() == null) {
+      throw new NullPointerException("entityType shouldn't be null");
+    }
     if (isSingleEntityRead()) {
-      Preconditions.checkNotNull(getContext().getAppId(),
-          "appId shouldn't be null");
+      if (getContext().getAppId() == null) {
+        throw new NullPointerException("appId shouldn't be null");
+      }
     } else {
-      Preconditions.checkNotNull(getContext().getUserId(),
-          "userId shouldn't be null");
-      Preconditions.checkNotNull(getContext().getFlowName(),
-          "flowName shouldn't be null");
+      if (getContext().getUserId() == null) {
+        throw new NullPointerException("userId shouldn't be null");
+      }
+      if (getContext().getFlowName() == null) {
+        throw new NullPointerException("flowName shouldn't be null");
+      }
     }
   }
 
@@ -406,7 +412,7 @@ protected ResultScanner getResults(Configuration hbaseConf,
       }
 
       // set start row
-      scan.setStartRow(applicationRowKey.getRowKey());
+      scan.withStartRow(applicationRowKey.getRowKey());
 
       // get the bytes for stop row
       applicationRowKeyPrefix = new ApplicationRowKeyPrefix(
@@ -414,7 +420,7 @@ protected ResultScanner getResults(Configuration hbaseConf,
           context.getFlowRunId());
 
       // set stop row
-      scan.setStopRow(
+      scan.withStopRow(
           HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
               applicationRowKeyPrefix.getRowKeyPrefix()));
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
index 59526f43d2003..51848e95391df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Result;
@@ -115,11 +114,15 @@ public Set<String> readEntityTypes(Configuration hbaseConf,
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext(), "context shouldn't be null");
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getAppId(),
-        "appId shouldn't be null");
+    if (getContext() == null) {
+      throw new NullPointerException("context shouldn't be null");
+    }
+    if (getContext().getClusterId() == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
+    if (getContext().getAppId() == null) {
+      throw new NullPointerException("appId shouldn't be null");
+    }
   }
 
   /**
@@ -155,9 +158,11 @@ private static byte[] getNextRowKey(byte[] currRowKeyPrefix,
   private ResultScanner getResult(Configuration hbaseConf, Connection conn,
       FilterList filterList, byte[] startPrefix, byte[] endPrefix)
       throws IOException {
-    Scan scan = new Scan(startPrefix, endPrefix);
-    scan.setFilter(filterList);
-    scan.setSmall(true);
+    Scan scan = new Scan()
+        .withStartRow(startPrefix)
+        .withStopRow(endPrefix)
+        .setFilter(filterList)
+        .setSmall(true);
     return ENTITY_TABLE.getResultScanner(hbaseConf, conn, scan);
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 04810ac6b2b4d..11883fd90a997 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -46,8 +46,6 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 /**
  * Timeline entity reader for flow activity entities that are stored in the
  * flow activity table.
@@ -82,8 +80,10 @@ protected BaseTableRW<?> getTable() {
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
+    String clusterId = getContext().getClusterId();
+    if (clusterId == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
   }
 
   @Override
@@ -133,16 +133,16 @@ && getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) {
         throw new BadRequestException(
             "fromid doesn't belong to clusterId=" + clusterId);
       }
-      scan.setStartRow(key.getRowKey());
-      scan.setStopRow(
+      scan.withStartRow(key.getRowKey());
+      scan.withStopRow(
           new FlowActivityRowKeyPrefix(clusterId,
               (getFilters().getCreatedTimeBegin() <= 0 ? 0
                   : (getFilters().getCreatedTimeBegin() - 1)))
                       .getRowKeyPrefix());
     } else {
-      scan.setStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters()
+      scan.withStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters()
           .getCreatedTimeEnd()).getRowKeyPrefix());
-      scan.setStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters()
+      scan.withStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters()
           .getCreatedTimeBegin() <= 0 ? 0
           : (getFilters().getCreatedTimeBegin() - 1))).getRowKeyPrefix());
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 5b2e642634dd4..75693d0bc00b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -57,8 +57,6 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTableRW;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 /**
  * Timeline entity reader for flow run entities that are stored in the flow run
  * table.
@@ -86,18 +84,25 @@ protected BaseTableRW<?> getTable() {
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext(), "context shouldn't be null");
-    Preconditions.checkNotNull(getDataToRetrieve(),
-        "data to retrieve shouldn't be null");
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getUserId(),
-        "userId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getFlowName(),
-        "flowName shouldn't be null");
+    if (getContext() == null) {
+      throw new NullPointerException("context shouldn't be null");
+    }
+    if (getDataToRetrieve() == null) {
+      throw new NullPointerException("data to retrieve shouldn't be null");
+    }
+    if (getContext().getClusterId() == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
+    if (getContext().getUserId() == null) {
+      throw new NullPointerException("userId shouldn't be null");
+    }
+    if (getContext().getFlowName() == null) {
+      throw new NullPointerException("flowName shouldn't be null");
+    }
     if (isSingleEntityRead()) {
-      Preconditions.checkNotNull(getContext().getFlowRunId(),
-          "flowRunId shouldn't be null");
+      if (getContext().getFlowRunId() == null) {
+        throw new NullPointerException("flowRunId shouldn't be null");
+      }
     }
     EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
     if (!isSingleEntityRead() && fieldsToRetrieve != null) {
@@ -236,14 +241,14 @@ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
             "fromid doesn't belong to clusterId=" + context.getClusterId());
       }
       // set start row
-      scan.setStartRow(flowRunRowKey.getRowKey());
+      scan.withStartRow(flowRunRowKey.getRowKey());
 
       // get the bytes for stop row
       flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(),
           context.getUserId(), context.getFlowName());
 
       // set stop row
-      scan.setStopRow(
+      scan.withStopRow(
           HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
               flowRunRowKeyPrefix.getRowKeyPrefix()));
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index 8a701e5ffdf7f..e131176402af6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -63,8 +63,6 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTableRW;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 /**
  * Timeline entity reader for generic entities that are stored in the entity
  * table.
@@ -406,18 +404,25 @@ protected FilterList constructFilterListBasedOnFields(Set<String> cfsInFields)
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext(), "context shouldn't be null");
-    Preconditions.checkNotNull(getDataToRetrieve(),
-        "data to retrieve shouldn't be null");
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getAppId(),
-        "appId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getEntityType(),
-        "entityType shouldn't be null");
+    if (getContext() == null) {
+      throw new NullPointerException("context shouldn't be null");
+    }
+    if (getDataToRetrieve() == null) {
+      throw new NullPointerException("data to retrieve shouldn't be null");
+    }
+    if (getContext().getClusterId() == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
+    if (getContext().getAppId() == null) {
+      throw new NullPointerException("appId shouldn't be null");
+    }
+    if (getContext().getEntityType() == null) {
+      throw new NullPointerException("entityType shouldn't be null");
+    }
     if (isSingleEntityRead()) {
-      Preconditions.checkNotNull(getContext().getEntityId(),
-          "entityId shouldn't be null");
+      if (getContext().getEntityId() == null) {
+        throw new NullPointerException("entityId shouldn't be null");
+      }
     }
   }
 
@@ -514,7 +519,7 @@ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
       }
 
       // set start row
-      scan.setStartRow(entityRowKey.getRowKey());
+      scan.withStartRow(entityRowKey.getRowKey());
 
       // get the bytes for stop row
       entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(),
@@ -522,7 +527,7 @@ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
           context.getAppId(), context.getEntityType(), null, null);
 
       // set stop row
-      scan.setStopRow(
+      scan.withStopRow(
           HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
               entityRowKeyPrefix.getRowKeyPrefix()));
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
index 08651e158c458..5a4f7843a9ec9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java
@@ -56,8 +56,6 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationTableRW;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-
 class SubApplicationEntityReader extends GenericEntityReader {
   private static final SubApplicationTableRW SUB_APPLICATION_TABLE =
       new SubApplicationTableRW();
@@ -308,15 +306,21 @@ protected FilterList constructFilterListBasedOnFields(Set<String> cfsInFields)
 
   @Override
   protected void validateParams() {
-    Preconditions.checkNotNull(getContext(), "context shouldn't be null");
-    Preconditions.checkNotNull(getDataToRetrieve(),
-        "data to retrieve shouldn't be null");
-    Preconditions.checkNotNull(getContext().getClusterId(),
-        "clusterId shouldn't be null");
-    Preconditions.checkNotNull(getContext().getDoAsUser(),
-        "DoAsUser shouldn't be null");
-    Preconditions.checkNotNull(getContext().getEntityType(),
-        "entityType shouldn't be null");
+    if (getContext() == null) {
+      throw new NullPointerException("context shouldn't be null");
+    }
+    if (getDataToRetrieve() == null) {
+      throw new NullPointerException("data to retrieve shouldn't be null");
+    }
+    if (getContext().getClusterId() == null) {
+      throw new NullPointerException("clusterId shouldn't be null");
+    }
+    if (getContext().getDoAsUser() == null) {
+      throw new NullPointerException("DoAsUser shouldn't be null");
+    }
+    if (getContext().getEntityType() == null) {
+      throw new NullPointerException("entityType shouldn't be null");
+    }
   }
 
   @Override
@@ -368,7 +372,7 @@ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
       }
 
       // set start row
-      scan.setStartRow(entityRowKey.getRowKey());
+      scan.withStartRow(entityRowKey.getRowKey());
 
       // get the bytes for stop row
       subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix(
@@ -376,7 +380,7 @@ protected ResultScanner getResults(Configuration hbaseConf, Connection conn,
           context.getEntityType(), null, null, null);
 
       // set stop row
-      scan.setStopRow(
+      scan.withStopRow(
           HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix(
               subApplicationRowKeyPrefix.getRowKeyPrefix()));
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
index fa16077c2ed96..77ac0ca80458e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -43,18 +43,20 @@ public static TimelineEntityReader createSingleEntityReader(
       TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
     // currently the types that are handled separate from the generic entity
     // table are application, flow run, and flow activity entities
-    if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
-      return new ApplicationEntityReader(context, dataToRetrieve);
-    } else if (TimelineEntityType.
-        YARN_FLOW_RUN.matches(context.getEntityType())) {
-      return new FlowRunEntityReader(context, dataToRetrieve);
-    } else if (TimelineEntityType.
-        YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
-      return new FlowActivityEntityReader(context, dataToRetrieve);
-    } else {
-      // assume we're dealing with a generic entity read
-      return new GenericEntityReader(context, dataToRetrieve);
+    if (!context.isGenericEntity()) {
+      if (TimelineEntityType.
+          YARN_APPLICATION.matches(context.getEntityType())) {
+        return new ApplicationEntityReader(context, dataToRetrieve);
+      } else if (TimelineEntityType.
+          YARN_FLOW_RUN.matches(context.getEntityType())) {
+        return new FlowRunEntityReader(context, dataToRetrieve);
+      } else if (TimelineEntityType.
+          YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+        return new FlowActivityEntityReader(context, dataToRetrieve);
+      }
     }
+    // assume we're dealing with a generic entity read
+    return new GenericEntityReader(context, dataToRetrieve);
   }
 
   /**
@@ -73,21 +75,23 @@ public static TimelineEntityReader createMultipleEntitiesReader(
       TimelineDataToRetrieve dataToRetrieve) {
     // currently the types that are handled separate from the generic entity
     // table are application, flow run, and flow activity entities
-    if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
-      return new ApplicationEntityReader(context, filters, dataToRetrieve);
-    } else if (TimelineEntityType.
-        YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
-      return new FlowActivityEntityReader(context, filters, dataToRetrieve);
-    } else if (TimelineEntityType.
-        YARN_FLOW_RUN.matches(context.getEntityType())) {
-      return new FlowRunEntityReader(context, filters, dataToRetrieve);
-    } else {
-      if (context.getDoAsUser() != null) {
-        return new SubApplicationEntityReader(context, filters, dataToRetrieve);
+    if (!context.isGenericEntity()) {
+      if (TimelineEntityType.
+          YARN_APPLICATION.matches(context.getEntityType())) {
+        return new ApplicationEntityReader(context, filters, dataToRetrieve);
+      } else if (TimelineEntityType.
+          YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+        return new FlowActivityEntityReader(context, filters, dataToRetrieve);
+      } else if (TimelineEntityType.
+          YARN_FLOW_RUN.matches(context.getEntityType())) {
+        return new FlowRunEntityReader(context, filters, dataToRetrieve);
       }
-      // assume we're dealing with a generic entity read
-      return new GenericEntityReader(context, filters, dataToRetrieve);
     }
+    if (context.getDoAsUser() != null) {
+      return new SubApplicationEntityReader(context, filters, dataToRetrieve);
+    }
+    // assume we're dealing with a generic entity read
+    return new GenericEntityReader(context, filters, dataToRetrieve);
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
index 46bb8aed9e5df..f65e9eac7a434 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java
@@ -24,9 +24,8 @@
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -34,6 +33,9 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
 /**
  * Unit tests for HBaseTimelineStorageUtils static methos.
  */
@@ -41,7 +43,7 @@ public class TestHBaseTimelineStorageUtils {
 
   private String hbaseConfigPath = "target/hbase-site.xml";
 
-  @Before
+  @BeforeEach
   public void setup() throws IOException {
     // Input Hbase Configuration
     Configuration hbaseConf = new Configuration();
@@ -60,25 +62,27 @@ public void setup() throws IOException {
     os.close();
   }
 
-  @Test(expected=NullPointerException.class)
-  public void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
-    HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
+  @Test
+  void testGetTimelineServiceHBaseConfNullArgument() throws Exception {
+    assertThrows(NullPointerException.class, () -> {
+      HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null);
+    });
   }
 
   @Test
-  public void testWithHbaseConfAtLocalFileSystem() throws IOException {
+  void testWithHbaseConfAtLocalFileSystem() throws IOException {
     // Verifying With Hbase Conf from Local FileSystem
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE,
         hbaseConfigPath);
     Configuration hbaseConfFromLocal =
         HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
-    Assert.assertEquals("Failed to read hbase config from Local FileSystem",
-        "test", hbaseConfFromLocal.get("input"));
+    assertEquals("test", hbaseConfFromLocal.get("input"),
+        "Failed to read hbase config from Local FileSystem");
   }
 
   @Test
-  public void testWithHbaseConfAtHdfsFileSystem() throws IOException {
+  void testWithHbaseConfAtHdfsFileSystem() throws IOException {
     MiniDFSCluster hdfsCluster = null;
     try {
       HdfsConfiguration hdfsConfig = new HdfsConfiguration();
@@ -95,8 +99,8 @@ public void testWithHbaseConfAtHdfsFileSystem() throws IOException {
           path.toString());
       Configuration hbaseConfFromHdfs =
           HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(conf);
-      Assert.assertEquals("Failed to read hbase config from Hdfs FileSystem",
-          "test", hbaseConfFromHdfs.get("input"));
+      assertEquals("test", hbaseConfFromHdfs.get("input"),
+          "Failed to read hbase config from Hdfs FileSystem");
     } finally {
       if (hdfsCluster != null) {
         hdfsCluster.shutdown();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
index 26ccbd8662641..63e3389253cac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml
@@ -46,11 +46,27 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- This is needed for GenericObjectMapper in GenericConverter -->
@@ -58,6 +74,12 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- 'mvn dependency:analyze' fails to detect use of this direct
@@ -67,6 +89,12 @@
       <artifactId>hadoop-common</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -81,6 +109,10 @@
           <groupId>org.mortbay.jetty</groupId>
           <artifactId>jetty-util</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -89,12 +121,39 @@
       <artifactId>hadoop-shaded-guava</artifactId>
       <scope>test</scope>
     </dependency>
-
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${hbase-compatible-guava.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <version>${hbase-compatible-guice.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
index 0dc344fdcd436..714466b411fd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestCustomApplicationIdConversion.java
@@ -17,9 +17,11 @@
  */
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
@@ -29,11 +31,11 @@
  */
 public class TestCustomApplicationIdConversion {
   @Test
-  public void testConvertAplicationIdToString() {
+  void testConvertAplicationIdToString() {
     ApplicationId applicationId = ApplicationId.newInstance(0, 1);
     String applicationIdStr =
         HBaseTimelineSchemaUtils.convertApplicationIdToString(applicationId);
-    Assert.assertEquals(applicationId,
+    assertEquals(applicationId,
         ApplicationId.fromString(applicationIdStr));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
index 1bd363fd73a09..9aa8593a874eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestKeyConverters.java
@@ -18,13 +18,14 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Unit tests for key converters for various tables' row keys.
@@ -33,7 +34,7 @@
 public class TestKeyConverters {
 
   @Test
-  public void testAppIdKeyConverter() {
+  void testAppIdKeyConverter() {
     AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
     long currentTs = System.currentTimeMillis();
     ApplicationId appId1 = ApplicationId.newInstance(currentTs, 1);
@@ -48,23 +49,20 @@ public void testAppIdKeyConverter() {
     // App ids' should be encoded in a manner wherein descending order
     // is maintained.
     assertTrue(
-        "Ordering of app ids' is incorrect",
         Bytes.compareTo(appIdBytes1, appIdBytes2) > 0
             && Bytes.compareTo(appIdBytes1, appIdBytes3) > 0
-            && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0);
+            && Bytes.compareTo(appIdBytes2, appIdBytes3) > 0,
+        "Ordering of app ids' is incorrect");
     String decodedAppId1 = appIdKeyConverter.decode(appIdBytes1);
     String decodedAppId2 = appIdKeyConverter.decode(appIdBytes2);
     String decodedAppId3 = appIdKeyConverter.decode(appIdBytes3);
-    assertTrue("Decoded app id is not same as the app id encoded",
-        appIdStr1.equals(decodedAppId1));
-    assertTrue("Decoded app id is not same as the app id encoded",
-        appIdStr2.equals(decodedAppId2));
-    assertTrue("Decoded app id is not same as the app id encoded",
-        appIdStr3.equals(decodedAppId3));
+    assertEquals(appIdStr1, decodedAppId1);
+    assertEquals(appIdStr2, decodedAppId2);
+    assertEquals(appIdStr3, decodedAppId3);
   }
 
   @Test
-  public void testEventColumnNameConverter() {
+  void testEventColumnNameConverter() {
     String eventId = "=foo_=eve=nt=";
     byte[] valSepBytes = Bytes.toBytes(Separator.VALUES.getValue());
     byte[] maxByteArr =
@@ -91,7 +89,7 @@ public void testEventColumnNameConverter() {
   }
 
   @Test
-  public void testLongKeyConverter() {
+  void testLongKeyConverter() {
     LongKeyConverter longKeyConverter = new LongKeyConverter();
     confirmLongKeyConverter(longKeyConverter, Long.MIN_VALUE);
     confirmLongKeyConverter(longKeyConverter, -1234567890L);
@@ -113,7 +111,7 @@ private void confirmLongKeyConverter(LongKeyConverter longKeyConverter,
   }
 
   @Test
-  public void testStringKeyConverter() {
+  void testStringKeyConverter() {
     StringKeyConverter stringKeyConverter = new StringKeyConverter();
     String phrase = "QuackAttack now!";
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
index 5b7c7dcfb413c..ba18ba51efd61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeys.java
@@ -17,8 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -26,14 +25,16 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKeyPrefix;
 import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKeyPrefix;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKeyPrefix;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainRowKey;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /**
@@ -81,14 +82,14 @@ private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
     int sepLen = QUALIFIER_SEP_BYTES.length;
     for (int i = 0; i < sepLen; i++) {
       assertTrue(
-          "Row key prefix not encoded properly.",
           byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
-              QUALIFIER_SEP_BYTES[i]);
+              QUALIFIER_SEP_BYTES[i],
+          "Row key prefix not encoded properly.");
     }
   }
 
   @Test
-  public void testApplicationRowKey() {
+  void testApplicationRowKey() {
     byte[] byteRowKey =
         new ApplicationRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
             APPLICATION_ID).getRowKey();
@@ -104,7 +105,7 @@ public void testApplicationRowKey() {
             .getRowKeyPrefix();
     byte[][] splits =
         Separator.QUALIFIERS.split(byteRowKeyPrefix,
-            new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+            new int[]{Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
                 Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
                 Separator.VARIABLE_SIZE});
     assertEquals(5, splits.length);
@@ -118,9 +119,9 @@ public void testApplicationRowKey() {
     byteRowKeyPrefix =
         new ApplicationRowKeyPrefix(CLUSTER, USER, FLOW_NAME).getRowKeyPrefix();
     splits =
-        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[]{
             Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
-            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE});
     assertEquals(4, splits.length);
     assertEquals(0, splits[3].length);
     assertEquals(FLOW_NAME,
@@ -133,14 +134,14 @@ public void testApplicationRowKey() {
    * corresponding rowkey.
    */
   @Test
-  public void testAppToFlowRowKey() {
+  void testAppToFlowRowKey() {
     byte[] byteRowKey = new AppToFlowRowKey(APPLICATION_ID).getRowKey();
     AppToFlowRowKey rowKey = AppToFlowRowKey.parseRowKey(byteRowKey);
     assertEquals(APPLICATION_ID, rowKey.getAppId());
   }
 
   @Test
-  public void testEntityRowKey() {
+  void testEntityRowKey() {
     TimelineEntity entity = new TimelineEntity();
     entity.setId("!ent!ity!!id!");
     entity.setType("entity!Type");
@@ -163,14 +164,14 @@ public void testEntityRowKey() {
     byte[] byteRowKeyPrefix =
         new EntityRowKeyPrefix(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID,
             APPLICATION_ID, entity.getType(), null, null)
-                .getRowKeyPrefix();
+            .getRowKeyPrefix();
     byte[][] splits =
         Separator.QUALIFIERS.split(
             byteRowKeyPrefix,
-            new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+            new int[]{Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
                 Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
                 AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
-                Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE });
+                Bytes.SIZEOF_LONG, Separator.VARIABLE_SIZE});
     assertEquals(7, splits.length);
     assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
     assertEquals(entity.getType(),
@@ -183,7 +184,7 @@ public void testEntityRowKey() {
     splits =
         Separator.QUALIFIERS.split(
             byteRowKeyPrefix,
-            new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
+            new int[]{Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
                 Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
                 AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
     assertEquals(6, splits.length);
@@ -194,7 +195,7 @@ public void testEntityRowKey() {
   }
 
   @Test
-  public void testFlowActivityRowKey() {
+  void testFlowActivityRowKey() {
     Long ts = 1459900830000L;
     Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
     byte[] byteRowKey =
@@ -208,8 +209,8 @@ public void testFlowActivityRowKey() {
     byte[] byteRowKeyPrefix =
         new FlowActivityRowKeyPrefix(CLUSTER).getRowKeyPrefix();
     byte[][] splits =
-        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
-            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[]{
+            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE});
     assertEquals(2, splits.length);
     assertEquals(0, splits[1].length);
     assertEquals(CLUSTER,
@@ -220,7 +221,7 @@ public void testFlowActivityRowKey() {
         new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
     splits =
         Separator.QUALIFIERS.split(byteRowKeyPrefix,
-            new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
+            new int[]{Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
                 Separator.VARIABLE_SIZE});
     assertEquals(3, splits.length);
     assertEquals(0, splits[2].length);
@@ -232,7 +233,7 @@ public void testFlowActivityRowKey() {
   }
 
   @Test
-  public void testFlowRunRowKey() {
+  void testFlowRunRowKey() {
     byte[] byteRowKey =
         new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID).getRowKey();
     FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(byteRowKey);
@@ -244,9 +245,9 @@ public void testFlowRunRowKey() {
     byte[] byteRowKeyPrefix =
         new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, null).getRowKey();
     byte[][] splits =
-        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[] {
+        Separator.QUALIFIERS.split(byteRowKeyPrefix, new int[]{
             Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
-            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE });
+            Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE});
     assertEquals(4, splits.length);
     assertEquals(0, splits[3].length);
     assertEquals(FLOW_NAME,
@@ -255,7 +256,7 @@ public void testFlowRunRowKey() {
   }
 
   @Test
-  public void testSubAppRowKey() {
+  void testSubAppRowKey() {
     TimelineEntity entity = new TimelineEntity();
     entity.setId("entity1");
     entity.setType("DAG");
@@ -275,7 +276,7 @@ public void testSubAppRowKey() {
   }
 
   @Test
-  public void testDomainRowKey() {
+  void testDomainRowKey() {
     String clusterId = "cluster1@dc1";
     String domainId = "helloworld";
     byte[] byteRowKey =
@@ -291,7 +292,7 @@ public void testDomainRowKey() {
   }
 
   @Test
-  public void testDomainRowKeySpecialChars() {
+  void testDomainRowKeySpecialChars() {
     String clusterId = "cluster1!temp!dc1";
     String domainId = "hello=world";
     byte[] byteRowKey =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
index c4d07c7c30896..011bcfedcecbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestRowKeysAsString.java
@@ -17,7 +17,8 @@
  */
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -27,7 +28,8 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
 import org.apache.hadoop.yarn.server.timelineservice.storage.subapplication.SubApplicationRowKey;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * Test for row key as string.
@@ -50,8 +52,9 @@ public class TestRowKeysAsString {
   private final static String APPLICATION_ID =
       ApplicationId.newInstance(System.currentTimeMillis(), 1).toString();
 
-  @Test(timeout = 10000)
-  public void testApplicationRow() {
+  @Test
+  @Timeout(10000)
+  void testApplicationRow() {
     String rowKeyAsString = new ApplicationRowKey(CLUSTER, USER, FLOW_NAME,
         FLOW_RUN_ID, APPLICATION_ID).getRowKeyAsString();
     ApplicationRowKey rowKey =
@@ -63,8 +66,9 @@ public void testApplicationRow() {
     assertEquals(APPLICATION_ID, rowKey.getAppId());
   }
 
-  @Test(timeout = 10000)
-  public void testEntityRowKey() {
+  @Test
+  @Timeout(10000)
+  void testEntityRowKey() {
     char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
     char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
     String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
@@ -78,7 +82,7 @@ public void testEntityRowKey() {
     String rowKeyAsString =
         new EntityRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID, APPLICATION_ID,
             entity.getType(), entity.getIdPrefix(), entity.getId())
-                .getRowKeyAsString();
+            .getRowKeyAsString();
     EntityRowKey rowKey = EntityRowKey.parseRowKeyFromString(rowKeyAsString);
     assertEquals(CLUSTER, rowKey.getClusterId());
     assertEquals(USER, rowKey.getUserId());
@@ -91,8 +95,9 @@ public void testEntityRowKey() {
 
   }
 
-  @Test(timeout = 10000)
-  public void testFlowActivityRowKey() {
+  @Test
+  @Timeout(10000)
+  void testFlowActivityRowKey() {
     Long ts = 1459900830000L;
     Long dayTimestamp = HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
     String rowKeyAsString = new FlowActivityRowKey(CLUSTER, ts, USER, FLOW_NAME)
@@ -105,8 +110,9 @@ public void testFlowActivityRowKey() {
     assertEquals(FLOW_NAME, rowKey.getFlowName());
   }
 
-  @Test(timeout = 10000)
-  public void testFlowRunRowKey() {
+  @Test
+  @Timeout(10000)
+  void testFlowRunRowKey() {
     String rowKeyAsString =
         new FlowRunRowKey(CLUSTER, USER, FLOW_NAME, FLOW_RUN_ID)
             .getRowKeyAsString();
@@ -117,8 +123,9 @@ public void testFlowRunRowKey() {
     assertEquals(FLOW_RUN_ID, rowKey.getFlowRunId());
   }
 
-  @Test(timeout = 10000)
-  public void testSubApplicationRowKey() {
+  @Test
+  @Timeout(10000)
+  void testSubApplicationRowKey() {
     char del = TimelineReaderUtils.DEFAULT_DELIMITER_CHAR;
     char esc = TimelineReaderUtils.DEFAULT_ESCAPE_CHAR;
     String id = del + esc + "ent" + esc + del + "ity" + esc + del + esc + "id"
@@ -131,7 +138,7 @@ public void testSubApplicationRowKey() {
 
     String rowKeyAsString = new SubApplicationRowKey(SUB_APP_USER, CLUSTER,
         entity.getType(), entity.getIdPrefix(), entity.getId(), USER)
-            .getRowKeyAsString();
+        .getRowKeyAsString();
     SubApplicationRowKey rowKey = SubApplicationRowKey
         .parseRowKeyFromString(rowKeyAsString);
     assertEquals(SUB_APP_USER, rowKey.getSubAppUserId());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
index 05a3723c1e818..f4f88b377f416 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
@@ -16,19 +16,19 @@
  */
 package org.apache.hadoop.yarn.server.timelineservice.storage.common;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Test;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestSeparator {
 
@@ -40,7 +40,7 @@ public class TestSeparator {
    *
    */
   @Test
-  public void testEncodeDecodeString() {
+  void testEncodeDecodeString() {
 
     for (Separator separator : Separator.values()) {
       testEncodeDecode(separator, "");
@@ -65,11 +65,11 @@ private void testEncodeDecode(Separator separator, String token) {
     String encoded = separator.encode(token);
     String decoded = separator.decode(encoded);
     String msg = "token:" + token + " separator:" + separator + ".";
-    assertEquals(msg, token, decoded);
+    assertEquals(token, decoded, msg);
   }
 
   @Test
-  public void testEncodeDecode() {
+  void testEncodeDecode() {
     testEncodeDecode("Dr.", Separator.QUALIFIERS);
     testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
     testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
@@ -81,15 +81,16 @@ public void testEncodeDecode() {
         Separator.VALUES, Separator.SPACE);
 
   }
+
   @Test
-  public void testEncodedValues() {
+  void testEncodedValues() {
     testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor  %%%2$" +
         "= no problem!",
         Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
   }
 
   @Test
-  public void testSplits() {
+  void testSplits() {
     byte[] maxLongBytes = Bytes.toBytes(Long.MAX_VALUE);
     byte[] maxIntBytes = Bytes.toBytes(Integer.MAX_VALUE);
     for (Separator separator : Separator.values()) {
@@ -128,7 +129,7 @@ public void testSplits() {
       longVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxLongBytes,
           sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
       longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
-              sepByteArr.length), sepByteArr);
+          sepByteArr.length), sepByteArr);
       arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
           Bytes.toBytes(separator.encode(str2)), intVal1Arr);
       splits = separator.split(arr, sizes);
@@ -154,14 +155,16 @@ public void testSplits() {
             Bytes.SIZEOF_INT, 7};
         splits = separator.split(arr, sizes2);
         fail("Exception should have been thrown.");
-      } catch (IllegalArgumentException e) {}
+      } catch (IllegalArgumentException e) {
+      }
 
       try {
         int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
             Bytes.SIZEOF_LONG};
         splits = separator.split(arr, sizes2);
         fail("Exception should have been thrown.");
-      } catch (IllegalArgumentException e) {}
+      } catch (IllegalArgumentException e) {
+      }
     }
   }
 
@@ -179,7 +182,7 @@ private static void testEncodeDecode(String token, Separator... separators) {
   }
 
   @Test
-  public void testJoinStripped() {
+  void testJoinStripped() {
     List<String> stringList = new ArrayList<String>(0);
     stringList.add("nothing");
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
index 68d6d960e56c4..20fc1fd65f019 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml
@@ -27,7 +27,7 @@
 
   <modelVersion>4.0.0</modelVersion>
   <artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
-  <name>Apache Hadoop YARN TimelineService HBase Server 1.2</name>
+  <name>Apache Hadoop YARN TimelineService HBase Server 1.7</name>
   <version>3.4.0-SNAPSHOT</version>
 
   <properties>
@@ -47,6 +47,16 @@
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.inject</groupId>
+              <artifactId>guice</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
 
         <dependency>
@@ -59,6 +69,24 @@
           <artifactId>hadoop-shaded-guava</artifactId>
         </dependency>
 
+        <dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>${hbase-compatible-guava.version}</version>
+        </dependency>
+
+        <dependency>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+          <version>${hbase-compatible-guice.version}</version>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-annotations</artifactId>
@@ -69,6 +97,12 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
           <scope>provided</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
 
         <dependency>
@@ -89,6 +123,10 @@
               <groupId>org.mortbay.jetty</groupId>
               <artifactId>jetty-util</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
 
@@ -100,6 +138,10 @@
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-mapreduce-client-core</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
 
@@ -136,6 +178,10 @@
               <groupId>org.mortbay.jetty</groupId>
               <artifactId>jetty-sslengine</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
       </dependencies>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
index df01ad7549c12..7daa5782d69a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml
@@ -27,7 +27,7 @@
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
-  <name>Apache Hadoop YARN TimelineService HBase Server 2.0</name>
+  <name>Apache Hadoop YARN TimelineService HBase Server 2.2</name>
   <version>3.4.0-SNAPSHOT</version>
 
   <properties>
@@ -47,6 +47,16 @@
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-yarn-server-timelineservice-hbase-common</artifactId>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.inject</groupId>
+              <artifactId>guice</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
 
         <dependency>
@@ -59,6 +69,24 @@
           <artifactId>hadoop-shaded-guava</artifactId>
         </dependency>
 
+        <dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>${hbase-compatible-guava.version}</version>
+        </dependency>
+
+        <dependency>
+          <groupId>com.google.inject</groupId>
+          <artifactId>guice</artifactId>
+          <version>${hbase-compatible-guice.version}</version>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-annotations</artifactId>
@@ -69,6 +97,12 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-common</artifactId>
           <scope>provided</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
 
         <dependency>
@@ -89,6 +123,10 @@
               <groupId>org.mortbay.jetty</groupId>
               <artifactId>jetty-util</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
 
@@ -100,6 +138,10 @@
               <groupId>org.apache.hadoop</groupId>
               <artifactId>hadoop-mapreduce-client-core</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
 
@@ -155,6 +197,10 @@
               <groupId>org.eclipse.jetty</groupId>
               <artifactId>jetty-http</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
       </dependencies>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 65af3afadda62..5a2823ad5eff5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -126,14 +126,23 @@
     </dependency>
 
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
-
     <dependency>
-      <groupId>org.assertj</groupId>
-      <artifactId>assertj-core</artifactId>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
       <scope>test</scope>
     </dependency>
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index bb89b7967ad58..a9ae2b5734409 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -30,7 +30,7 @@
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 
 import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 0c54ed03f9edc..1c1fe118b843c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -34,6 +34,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
@@ -71,6 +72,9 @@ public abstract class TimelineCollector extends CompositeService {
 
   private volatile boolean isStopped = false;
 
+  private int maxWriteRetries;
+  private long writeRetryInterval;
+
   public TimelineCollector(String name) {
     super(name);
   }
@@ -86,6 +90,13 @@ protected void serviceInit(Configuration conf) throws Exception {
         new ArrayBlockingQueue<>(capacity));
     pool.setRejectedExecutionHandler(
         new ThreadPoolExecutor.DiscardOldestPolicy());
+
+    maxWriteRetries =
+        conf.getInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
+    writeRetryInterval = conf.getLong(
+        YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
   }
 
   @Override
@@ -153,18 +164,54 @@ public TimelineWriteResponse putEntities(TimelineEntities entities,
       UserGroupInformation callerUgi) throws IOException {
     LOG.debug("putEntities(entities={}, callerUgi={})", entities, callerUgi);
 
-    TimelineWriteResponse response;
-    // synchronize on the writer object so that no other threads can
-    // flush the writer buffer concurrently and swallow any exception
-    // caused by the timeline enitites that are being put here.
-    synchronized (writer) {
-      response = writeTimelineEntities(entities, callerUgi);
-      flushBufferedTimelineEntities();
+    TimelineWriteResponse response = null;
+    try {
+      boolean isStorageUp = checkRetryWithSleep();
+      if (isStorageUp) {
+        // synchronize on the writer object so that no other threads can
+        // flush the writer buffer concurrently and swallow any exception
+        // caused by the timeline enitites that are being put here.
+        synchronized (writer) {
+          response = writeTimelineEntities(entities, callerUgi);
+          flushBufferedTimelineEntities();
+        }
+      } else {
+        String msg = String.format("Failed to putEntities(" +
+            "entities=%s, callerUgi=%s) as Timeline Storage is Down",
+            entities, callerUgi);
+        throw new IOException(msg);
+      }
+    } catch (InterruptedException ex) {
+      String msg = String.format("Interrupted while retrying to putEntities(" +
+          "entities=%s, callerUgi=%s)", entities, callerUgi);
+      throw new IOException(msg);
     }
 
     return response;
   }
 
+
+  private boolean checkRetryWithSleep() throws InterruptedException {
+    int retries = maxWriteRetries;
+    while (retries > 0) {
+      TimelineHealth timelineHealth = writer.getHealthStatus();
+      if (timelineHealth.getHealthStatus().equals(
+              TimelineHealth.TimelineHealthStatus.RUNNING)) {
+        return true;
+      } else {
+        try {
+          Thread.sleep(writeRetryInterval);
+        } catch (InterruptedException ex) {
+          Thread.currentThread().interrupt();
+          throw ex;
+        }
+        retries--;
+      }
+    }
+    return false;
+  }
+
+
   /**
    * Add or update an domain. If the domain already exists, only the owner
    * and the admin can update it.
@@ -179,11 +226,25 @@ public TimelineWriteResponse putDomain(TimelineDomain domain,
       UserGroupInformation callerUgi) throws IOException {
     LOG.debug("putDomain(domain={}, callerUgi={})", domain, callerUgi);
 
-    TimelineWriteResponse response;
-    synchronized (writer) {
-      final TimelineCollectorContext context = getTimelineEntityContext();
-      response = writer.write(context, domain);
-      flushBufferedTimelineEntities();
+    TimelineWriteResponse response = null;
+    try {
+      boolean isStorageUp = checkRetryWithSleep();
+      if (isStorageUp) {
+        synchronized (writer) {
+          final TimelineCollectorContext context = getTimelineEntityContext();
+          response = writer.write(context, domain);
+          flushBufferedTimelineEntities();
+        }
+      } else {
+        String msg = String.format("Failed to putDomain(" +
+            "domain=%s, callerUgi=%s) as Timeline Storage is Down",
+            domain, callerUgi);
+        throw new IOException(msg);
+      }
+    } catch (InterruptedException ex) {
+      String msg = String.format("Interrupted while retrying to putDomain(" +
+          "domain=%s, callerUgi=%s)", domain, callerUgi);
+      throw new IOException(msg);
     }
 
     return response;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
index 67c3d297e11f3..62e39d2923b46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
@@ -33,6 +33,8 @@ public class TimelineReaderContext extends TimelineContext {
   private String entityId;
   private Long entityIdPrefix;
   private String doAsUser;
+  private boolean genericEntity = false;
+
   public TimelineReaderContext(String clusterId, String userId, String flowName,
       Long flowRunId, String appId, String entityType, String entityId) {
     super(clusterId, userId, flowName, flowRunId, appId);
@@ -55,10 +57,19 @@ public TimelineReaderContext(String clusterId, String userId, String flowName,
     this.doAsUser = doasUser;
   }
 
+  public TimelineReaderContext(String clusterId, String userId, String flowName,
+      Long flowRunId, String appId, String entityType, Long entityIdPrefix,
+      String entityId, String doasUser, boolean genericEntity) {
+    this(clusterId, userId, flowName, flowRunId, appId, entityType,
+        entityIdPrefix, entityId, doasUser);
+    this.genericEntity = genericEntity;
+  }
+
   public TimelineReaderContext(TimelineReaderContext other) {
     this(other.getClusterId(), other.getUserId(), other.getFlowName(),
         other.getFlowRunId(), other.getAppId(), other.getEntityType(),
-        other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser());
+        other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser(),
+        other.genericEntity);
   }
 
   @Override
@@ -130,4 +141,14 @@ public String getDoAsUser() {
   public void setDoAsUser(String doAsUser) {
     this.doAsUser = doAsUser;
   }
+
+  public boolean isGenericEntity() {
+    return genericEntity;
+  }
+
+  public void setGenericEntity(boolean genericEntity) {
+    this.genericEntity = genericEntity;
+  }
+
+
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index a024b679dda0c..8d3436374aa62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -371,6 +371,7 @@ public Set<TimelineEntity> getEntities(
       }
       context.setEntityType(
           TimelineReaderWebServicesUtils.parseStr(entityType));
+      context.setGenericEntity(true);
       entities = timelineReaderManager.getEntities(context,
           TimelineReaderWebServicesUtils.createTimelineEntityFilters(
           limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
@@ -516,7 +517,7 @@ public Set<TimelineEntity> getEntities(
         flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
         isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
         confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-        metricsTimeStart, metricsTimeEnd, fromId);
+        metricsTimeStart, metricsTimeEnd, fromId, true);
   }
 
   /**
@@ -636,6 +637,23 @@ public Set<TimelineEntity> getEntities(
       @QueryParam("metricstimestart") String metricsTimeStart,
       @QueryParam("metricstimeend") String metricsTimeEnd,
       @QueryParam("fromid") String fromId) {
+    return getEntities(req, res, null, appId, entityType, userId, flowName,
+        flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+        isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+        confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
+        metricsTimeStart, metricsTimeEnd, fromId, true);
+  }
+
+  public Set<TimelineEntity> getEntities(HttpServletRequest req,
+      HttpServletResponse res, String clusterId, String appId,
+      String entityType, String userId, String flowName,
+      String flowRunId, String limit, String createdTimeStart,
+      String createdTimeEnd, String relatesTo, String isRelatedTo,
+      String infofilters, String conffilters, String metricfilters,
+      String eventfilters, String confsToRetrieve, String metricsToRetrieve,
+      String fields, String metricsLimit, String metricsTimeStart,
+      String metricsTimeEnd, String fromId,
+      boolean genericEntity) {
     String url = req.getRequestURI() +
         (req.getQueryString() == null ? "" :
             QUERY_STRING_SEP + req.getQueryString());
@@ -652,6 +670,7 @@ public Set<TimelineEntity> getEntities(
       TimelineReaderContext context = TimelineReaderWebServicesUtils
           .createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
               appId, entityType, null, null);
+      context.setGenericEntity(genericEntity);
       entities = timelineReaderManager.getEntities(context,
           TimelineReaderWebServicesUtils
               .createTimelineEntityFilters(limit, createdTimeStart,
@@ -777,6 +796,54 @@ public TimelineEntity getEntity(
     return entity;
   }
 
+  public TimelineEntity getEntity(HttpServletRequest req,
+      HttpServletResponse res, String clusterId, String appId,
+      String entityType, String entityId, String userId, String flowName,
+      String flowRunId, String confsToRetrieve, String metricsToRetrieve,
+      String fields, String metricsLimit, String metricsTimeStart,
+      String metricsTimeEnd, String entityIdPrefix,
+      boolean genericEntity) {
+    String url = req.getRequestURI() +
+        (req.getQueryString() == null ? "" :
+            QUERY_STRING_SEP + req.getQueryString());
+    UserGroupInformation callerUGI =
+        TimelineReaderWebServicesUtils.getUser(req);
+    LOG.info("Received URL " + url + " from user " +
+        TimelineReaderWebServicesUtils.getUserName(callerUGI));
+    long startTime = Time.monotonicNow();
+    boolean succeeded = false;
+    init(res);
+    TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+    TimelineEntity entity = null;
+    TimelineReaderContext context = TimelineReaderWebServicesUtils.
+        createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
+        appId, entityType, entityIdPrefix, entityId);
+    context.setGenericEntity(genericEntity);
+    try {
+      entity = timelineReaderManager.getEntity(context,
+          TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+          confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
+          metricsTimeStart, metricsTimeEnd));
+      checkAccessForGenericEntity(entity, callerUGI);
+      succeeded = true;
+    } catch (Exception e) {
+      handleException(e, url, startTime, "Either flowrunid or metricslimit or"
+              + " metricstime start/end");
+    } finally {
+      long latency = Time.monotonicNow() - startTime;
+      METRICS.addGetEntitiesLatency(latency, succeeded);
+      LOG.info("Processed URL " + url +
+          " (Took " + latency + " ms.)");
+    }
+    if (entity == null) {
+      LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
+          (Time.monotonicNow() - startTime) + " ms.)");
+      throw new NotFoundException("Timeline entity {id: " + entityId +
+          ", type: " + entityType + " } is not found");
+    }
+    return entity;
+  }
+
   /**
    * Return a single entity of the given entity type and Id. Cluster ID is not
    * provided by client so default cluster ID has to be taken. If userid, flow
@@ -853,7 +920,7 @@ public TimelineEntity getEntity(
       @QueryParam("entityidprefix") String entityIdPrefix) {
     return getEntity(req, res, null, appId, entityType, entityId, userId,
         flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields,
-        metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix);
+        metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true);
   }
 
   /**
@@ -932,44 +999,9 @@ public TimelineEntity getEntity(
       @QueryParam("metricstimestart") String metricsTimeStart,
       @QueryParam("metricstimeend") String metricsTimeEnd,
       @QueryParam("entityidprefix") String entityIdPrefix) {
-    String url = req.getRequestURI() +
-        (req.getQueryString() == null ? "" :
-            QUERY_STRING_SEP + req.getQueryString());
-    UserGroupInformation callerUGI =
-        TimelineReaderWebServicesUtils.getUser(req);
-    LOG.info("Received URL " + url + " from user " +
-        TimelineReaderWebServicesUtils.getUserName(callerUGI));
-    long startTime = Time.monotonicNow();
-    boolean succeeded = false;
-    init(res);
-    TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
-    TimelineEntity entity = null;
-    try {
-      entity = timelineReaderManager.getEntity(
-          TimelineReaderWebServicesUtils.createTimelineReaderContext(
-              clusterId, userId, flowName, flowRunId, appId, entityType,
-              entityIdPrefix, entityId),
-          TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
-          confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-          metricsTimeStart, metricsTimeEnd));
-      checkAccessForGenericEntity(entity, callerUGI);
-      succeeded = true;
-    } catch (Exception e) {
-      handleException(e, url, startTime, "Either flowrunid or metricslimit or"
-          + " metricstime start/end");
-    } finally {
-      long latency = Time.monotonicNow() - startTime;
-      METRICS.addGetEntitiesLatency(latency, succeeded);
-      LOG.info("Processed URL " + url +
-          " (Took " + latency + " ms.)");
-    }
-    if (entity == null) {
-      LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
-          (Time.monotonicNow() - startTime) + " ms.)");
-      throw new NotFoundException("Timeline entity {id: " + entityId +
-          ", type: " + entityType + " } is not found");
-    }
-    return entity;
+    return getEntity(req, res, clusterId, appId, entityType, entityId,
+        userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields,
+        metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true);
   }
 
   /**
@@ -2088,7 +2120,7 @@ public Set<TimelineEntity> getFlowRunApps(
         flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
         isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
         confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-        metricsTimeStart, metricsTimeEnd, fromId);
+        metricsTimeStart, metricsTimeEnd, fromId, false);
   }
 
   /**
@@ -2202,7 +2234,7 @@ public Set<TimelineEntity> getFlowRunApps(
         flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
         isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
         confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-        metricsTimeStart, metricsTimeEnd, fromId);
+        metricsTimeStart, metricsTimeEnd, fromId, false);
   }
 
   /**
@@ -2310,7 +2342,7 @@ public Set<TimelineEntity> getFlowApps(
         null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
         infofilters, conffilters, metricfilters, eventfilters,
         confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-        metricsTimeStart, metricsTimeEnd, fromId);
+        metricsTimeStart, metricsTimeEnd, fromId, false);
   }
 
   /**
@@ -2420,7 +2452,7 @@ public Set<TimelineEntity> getFlowApps(
         null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
         infofilters, conffilters, metricfilters, eventfilters,
         confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-        metricsTimeStart, metricsTimeEnd, fromId);
+        metricsTimeStart, metricsTimeEnd, fromId, false);
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 3dd7396cfcaf5..913b8360e2f26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -93,7 +93,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
   private static final String STORAGE_DIR_ROOT = "timeline_service_data";
 
   private final CSVFormat csvFormat =
-      CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+      CSVFormat.Builder.create().setHeader("APP", "USER", "FLOW", "FLOWRUN").build();
 
   public FileSystemTimelineReaderImpl() {
     super(FileSystemTimelineReaderImpl.class.getName());
@@ -454,7 +454,7 @@ public TimelineHealth getHealthStatus() {
       fs.exists(rootPath);
     } catch (IOException e) {
       return new TimelineHealth(
-          TimelineHealth.TimelineHealthStatus.READER_CONNECTION_FAILURE,
+          TimelineHealth.TimelineHealthStatus.CONNECTION_FAILURE,
           e.getMessage()
           );
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index d5c70a0607ebb..d407636cb6af6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -194,6 +195,20 @@ public void flush() throws IOException {
     // no op
   }
 
+  @Override
+  public TimelineHealth getHealthStatus() {
+    try {
+      fs.exists(rootPath);
+    } catch (IOException e) {
+      return new TimelineHealth(
+          TimelineHealth.TimelineHealthStatus.CONNECTION_FAILURE,
+          e.getMessage()
+      );
+    }
+    return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING,
+        "");
+  }
+
   private void mkdirs(Path... paths) throws IOException, InterruptedException {
     for (Path path: paths) {
       if (!existsWithRetries(path)) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineWriterImpl.java
index 48b334800f457..fd3120969391b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/NoOpTimelineWriterImpl.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -77,4 +78,10 @@ public TimelineWriteResponse aggregate(TimelineEntity data,
   public void flush() throws IOException {
     LOG.debug("NoOpTimelineWriter is configured. Ignoring flush call");
   }
+
+  @Override
+  public TimelineHealth getHealthStatus() {
+    return new TimelineHealth(TimelineHealth.TimelineHealthStatus.RUNNING,
+        "NoOpTimelineWriter is configured. ");
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 08cfc8becb2d2..ccc7491037710 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -95,4 +96,13 @@ TimelineWriteResponse aggregate(TimelineEntity data,
    *     entities to the backend storage.
    */
   void flush() throws IOException;
+
+  /**
+   * Check if writer connection is working properly.
+   *
+   * @return True if writer connection works as expected, false otherwise.
+   */
+  TimelineHealth getHealthStatus();
+
+
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
index b109be5fa48d1..d17ed2772967b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -19,17 +19,6 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -38,6 +27,11 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -47,14 +41,22 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class TestNMTimelineCollectorManager {
   private NodeTimelineCollectorManager collectorManager;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     collectorManager = createCollectorManager();
     Configuration conf = new YarnConfiguration();
@@ -66,7 +68,7 @@ public void setup() throws Exception {
     collectorManager.start();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     if (collectorManager != null) {
       collectorManager.stop();
@@ -74,12 +76,12 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void testStartingWriterFlusher() throws Exception {
+  void testStartingWriterFlusher() throws Exception {
     assertTrue(collectorManager.writerFlusherRunning());
   }
 
   @Test
-  public void testStartWebApp() throws Exception {
+  void testStartWebApp() throws Exception {
     assertNotNull(collectorManager.getRestServerBindAddress());
     String address = collectorManager.getRestServerBindAddress();
     String[] parts = address.split(":");
@@ -89,8 +91,9 @@ public void testStartWebApp() throws Exception {
         Integer.valueOf(parts[1]) <= 30100);
   }
 
-  @Test(timeout=60000)
-  public void testMultithreadedAdd() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testMultithreadedAdd() throws Exception {
     final int numApps = 5;
     List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
     for (int i = 0; i < numApps; i++) {
@@ -107,7 +110,7 @@ public Boolean call() {
     ExecutorService executor = Executors.newFixedThreadPool(numApps);
     try {
       List<Future<Boolean>> futures = executor.invokeAll(tasks);
-      for (Future<Boolean> future: futures) {
+      for (Future<Boolean> future : futures) {
         assertTrue(future.get());
       }
     } finally {
@@ -121,7 +124,7 @@ public Boolean call() {
   }
 
   @Test
-  public void testMultithreadedAddAndRemove() throws Exception {
+  void testMultithreadedAddAndRemove() throws Exception {
     final int numApps = 5;
     List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
     for (int i = 0; i < numApps; i++) {
@@ -140,7 +143,7 @@ public Boolean call() {
     ExecutorService executor = Executors.newFixedThreadPool(numApps);
     try {
       List<Future<Boolean>> futures = executor.invokeAll(tasks);
-      for (Future<Boolean> future: futures) {
+      for (Future<Boolean> future : futures) {
         assertTrue(future.get());
       }
     } finally {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
index c9ff303763312..c43bc3d2031b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeAggTimelineCollectorMetrics.java
@@ -18,11 +18,14 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.server.timelineservice.metrics.PerNodeAggTimelineCollectorMetrics;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * Test PerNodeAggTimelineCollectorMetrics.
@@ -32,24 +35,24 @@ public class TestPerNodeAggTimelineCollectorMetrics {
   private PerNodeAggTimelineCollectorMetrics metrics;
 
   @Test
-  public void testTimelineCollectorMetrics() {
-    Assert.assertNotNull(metrics);
-    Assert.assertEquals(10,
+  void testTimelineCollectorMetrics() {
+    assertNotNull(metrics);
+    assertEquals(10,
         metrics.getPutEntitiesSuccessLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getPutEntitiesFailureLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getAsyncPutEntitiesSuccessLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getAsyncPutEntitiesFailureLatency().getInterval());
   }
 
-  @Before
+  @BeforeEach
   public void setup() {
     metrics = PerNodeAggTimelineCollectorMetrics.getInstance();
   }
 
-  @After
+  @AfterEach
   public void tearDown() {
     PerNodeAggTimelineCollectorMetrics.destroy();
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 811465f7f97c0..66474b14851c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -18,19 +18,13 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
 import java.io.IOException;
 import java.util.concurrent.Future;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.Shell;
@@ -47,9 +41,16 @@
 import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 public class TestPerNodeTimelineCollectorsAuxService {
   private ApplicationAttemptId appAttemptId;
@@ -70,7 +71,7 @@ public TestPerNodeTimelineCollectorsAuxService() {
         1000L);
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Shell.ExitCodeException {
     if (auxService != null) {
       auxService.stop();
@@ -78,7 +79,7 @@ public void tearDown() throws Shell.ExitCodeException {
   }
 
   @Test
-  public void testAddApplication() throws Exception {
+  void testAddApplication() throws Exception {
     auxService = createCollectorAndAddApplication();
     // auxService should have a single app
     assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -86,7 +87,7 @@ public void testAddApplication() throws Exception {
   }
 
   @Test
-  public void testAddApplicationNonAMContainer() throws Exception {
+  void testAddApplicationNonAMContainer() throws Exception {
     auxService = createCollector();
 
     ContainerId containerId = getContainerId(2L); // not an AM
@@ -99,7 +100,7 @@ public void testAddApplicationNonAMContainer() throws Exception {
   }
 
   @Test
-  public void testRemoveApplication() throws Exception {
+  void testRemoveApplication() throws Exception {
     auxService = createCollectorAndAddApplication();
     // auxService should have a single app
     assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -118,7 +119,7 @@ public void testRemoveApplication() throws Exception {
   }
 
   @Test
-  public void testRemoveApplicationNonAMContainer() throws Exception {
+  void testRemoveApplicationNonAMContainer() throws Exception {
     auxService = createCollectorAndAddApplication();
     // auxService should have a single app
     assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -133,8 +134,9 @@ public void testRemoveApplicationNonAMContainer() throws Exception {
     auxService.close();
   }
 
-  @Test(timeout = 60000)
-  public void testLaunch() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testLaunch() throws Exception {
     ExitUtil.disableSystemExit();
     try {
       auxService =
@@ -192,7 +194,7 @@ protected Future removeApplicationCollector(ContainerId containerId) {
             try {
               future.get();
             } catch (Exception e) {
-              Assert.fail("Expeption thrown while removing collector");
+              fail("Expeption thrown while removing collector");
             }
             return future;
           }
@@ -228,8 +230,9 @@ private ContainerId getContainerId(long id) {
     return ContainerId.newContainerId(appAttemptId, id);
   }
 
-  @Test(timeout = 60000)
-  public void testRemoveAppWhenSecondAttemptAMCotainerIsLaunchedSameNode()
+  @Test
+  @Timeout(60000)
+  void testRemoveAppWhenSecondAttemptAMCotainerIsLaunchedSameNode()
       throws Exception {
     // add first attempt collector
     auxService = createCollectorAndAddApplication();
@@ -241,25 +244,25 @@ public void testRemoveAppWhenSecondAttemptAMCotainerIsLaunchedSameNode()
         createContainerInitalizationContext(2);
     auxService.initializeContainer(containerInitalizationContext);
 
-    assertTrue("Applicatin not found in collectors.",
-        auxService.hasApplication(appAttemptId.getApplicationId()));
+    assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()),
+        "Applicatin not found in collectors.");
 
     // first attempt stop container
     ContainerTerminationContext context = createContainerTerminationContext(1);
     auxService.stopContainer(context);
 
     // 2nd attempt container removed, still collector should hold application id
-    assertTrue("collector has removed application though 2nd attempt"
-            + " is running this node",
-        auxService.hasApplication(appAttemptId.getApplicationId()));
+    assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()),
+        "collector has removed application though 2nd attempt"
+            + " is running this node");
 
     // second attempt stop container
     context = createContainerTerminationContext(2);
     auxService.stopContainer(context);
 
     // auxService should not have that app
-    assertFalse("Application is not removed from collector",
-        auxService.hasApplication(appAttemptId.getApplicationId()));
+    assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()),
+        "Application is not removed from collector");
     auxService.close();
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
index 96636f8bf8c46..c2995d5daedf4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
@@ -18,34 +18,35 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.internal.stubbing.answers.AnswersWithDelay;
+import org.mockito.internal.stubbing.answers.Returns;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Sets;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineHealth;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector.AggregationStatusTable;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.junit.Test;
-
-import org.mockito.internal.stubbing.answers.AnswersWithDelay;
-import org.mockito.internal.stubbing.answers.Returns;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -103,7 +104,7 @@ private TimelineEntities generateTestEntities(int groups, int entities) {
   }
 
   @Test
-  public void testAggregation() throws Exception {
+  void testAggregation() throws Exception {
     // Test aggregation with multiple groups.
     int groups = 3;
     int n = 50;
@@ -153,9 +154,19 @@ public void testAggregation() throws Exception {
    * putEntity() calls.
    */
   @Test
-  public void testPutEntity() throws IOException {
+  void testPutEntity() throws IOException {
     TimelineWriter writer = mock(TimelineWriter.class);
+    TimelineHealth timelineHealth = new TimelineHealth(TimelineHealth.
+        TimelineHealthStatus.RUNNING, "");
+    when(writer.getHealthStatus()).thenReturn(timelineHealth);
+
+    Configuration conf = new Configuration();
+    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 5);
+    conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
+        500L);
+
     TimelineCollector collector = new TimelineCollectorForTest(writer);
+    collector.init(conf);
 
     TimelineEntities entities = generateTestEntities(1, 1);
     collector.putEntities(
@@ -166,12 +177,42 @@ public void testPutEntity() throws IOException {
     verify(writer, times(1)).flush();
   }
 
+
+  @Test
+  void testPutEntityWithStorageDown() throws IOException {
+    TimelineWriter writer = mock(TimelineWriter.class);
+    TimelineHealth timelineHealth = new TimelineHealth(TimelineHealth.
+        TimelineHealthStatus.CONNECTION_FAILURE, "");
+    when(writer.getHealthStatus()).thenReturn(timelineHealth);
+
+    Configuration conf = new Configuration();
+    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 5);
+    conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
+        500L);
+
+    TimelineCollector collector = new TimelineCollectorForTest(writer);
+    collector.init(conf);
+
+    TimelineEntities entities = generateTestEntities(1, 1);
+    boolean exceptionCaught = false;
+    try {
+      collector.putEntities(entities, UserGroupInformation.
+          createRemoteUser("test-user"));
+    } catch (Exception e) {
+      if (e.getMessage().contains("Failed to putEntities")) {
+        exceptionCaught = true;
+      }
+    }
+    assertTrue(exceptionCaught, "TimelineCollector putEntity failed to " +
+        "handle storage down");
+  }
+
   /**
    * Test TimelineCollector's interaction with TimelineWriter upon
    * putEntityAsync() calls.
    */
   @Test
-  public void testPutEntityAsync() throws Exception {
+  void testPutEntityAsync() throws Exception {
     TimelineWriter writer = mock(TimelineWriter.class);
     TimelineCollector collector = new TimelineCollectorForTest(writer);
     collector.init(new Configuration());
@@ -191,7 +232,7 @@ public void testPutEntityAsync() throws Exception {
    * write is taking too much time.
    */
   @Test
-  public void testAsyncEntityDiscard() throws Exception {
+  void testAsyncEntityDiscard() throws Exception {
     TimelineWriter writer = mock(TimelineWriter.class);
 
     when(writer.write(any(), any(), any())).thenAnswer(
@@ -220,9 +261,20 @@ public void testAsyncEntityDiscard() throws Exception {
    * Test TimelineCollector's interaction with TimelineWriter upon
    * putDomain() calls.
    */
-  @Test public void testPutDomain() throws IOException {
+  @Test
+  void testPutDomain() throws IOException {
     TimelineWriter writer = mock(TimelineWriter.class);
+    TimelineHealth timelineHealth = new TimelineHealth(TimelineHealth.
+        TimelineHealthStatus.RUNNING, "");
+    when(writer.getHealthStatus()).thenReturn(timelineHealth);
+
+    Configuration conf = new Configuration();
+    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 5);
+    conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
+        500L);
+
     TimelineCollector collector = new TimelineCollectorForTest(writer);
+    collector.init(conf);
 
     TimelineDomain domain =
         generateDomain("id", "desc", "owner", "reader1,reader2", "writer", 0L,
@@ -278,21 +330,32 @@ private static TimelineMetric createDummyMetric(long ts, Long value) {
   }
 
   @Test
-  public void testClearPreviousEntitiesOnAggregation() throws Exception {
+  void testClearPreviousEntitiesOnAggregation() throws Exception {
     final long ts = System.currentTimeMillis();
     TimelineCollector collector = new TimelineCollector("") {
-        @Override
-        public TimelineCollectorContext getTimelineEntityContext() {
-          return new TimelineCollectorContext("cluster", "user", "flow", "1",
-              1L, ApplicationId.newInstance(ts, 1).toString());
-        }
+      @Override
+      public TimelineCollectorContext getTimelineEntityContext() {
+        return new TimelineCollectorContext("cluster", "user", "flow", "1",
+            1L, ApplicationId.newInstance(ts, 1).toString());
+      }
     };
-    collector.init(new Configuration());
-    collector.setWriter(mock(TimelineWriter.class));
+
+    TimelineWriter writer = mock(TimelineWriter.class);
+    TimelineHealth timelineHealth = new TimelineHealth(TimelineHealth.
+        TimelineHealthStatus.RUNNING, "");
+    when(writer.getHealthStatus()).thenReturn(timelineHealth);
+
+    Configuration conf = new Configuration();
+    conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 5);
+    conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
+        500L);
+
+    collector.init(conf);
+    collector.setWriter(writer);
 
     // Put 5 entities with different metric values.
     TimelineEntities entities = new TimelineEntities();
-    for (int i = 1; i <=5; i++) {
+    for (int i = 1; i <= 5; i++) {
       TimelineEntity entity = createEntity("e" + i, "type");
       entity.addMetric(createDummyMetric(ts + i, Long.valueOf(i * 50)));
       entities.addEntity(entity);
@@ -306,7 +369,7 @@ public TimelineCollectorContext getTimelineEntityContext() {
     assertEquals(Sets.newHashSet("type"), aggregationGroups.keySet());
     TimelineEntity aggregatedEntity = TimelineCollector.
         aggregateWithoutGroupId(aggregationGroups, currContext.getAppId(),
-            TimelineEntityType.YARN_APPLICATION.toString());
+        TimelineEntityType.YARN_APPLICATION.toString());
     TimelineMetric aggregatedMetric =
         aggregatedEntity.getMetrics().iterator().next();
     assertEquals(750L, aggregatedMetric.getValues().values().iterator().next());
@@ -316,7 +379,7 @@ public TimelineCollectorContext getTimelineEntityContext() {
     // Aggregate entities.
     aggregatedEntity = TimelineCollector.
         aggregateWithoutGroupId(aggregationGroups, currContext.getAppId(),
-            TimelineEntityType.YARN_APPLICATION.toString());
+        TimelineEntityType.YARN_APPLICATION.toString());
     aggregatedMetric = aggregatedEntity.getMetrics().iterator().next();
     // No values aggregated as no metrics put for an entity between this
     // aggregation and the previous one.
@@ -326,7 +389,7 @@ public TimelineCollectorContext getTimelineEntityContext() {
 
     // Put 3 entities.
     entities = new TimelineEntities();
-    for (int i = 1; i <=3; i++) {
+    for (int i = 1; i <= 3; i++) {
       TimelineEntity entity = createEntity("e" + i, "type");
       entity.addMetric(createDummyMetric(System.currentTimeMillis() + i, 50L));
       entities.addEntity(entity);
@@ -337,7 +400,7 @@ public TimelineCollectorContext getTimelineEntityContext() {
     // Aggregate entities.
     aggregatedEntity = TimelineCollector.
         aggregateWithoutGroupId(aggregationGroups, currContext.getAppId(),
-            TimelineEntityType.YARN_APPLICATION.toString());
+        TimelineEntityType.YARN_APPLICATION.toString());
     // Last 3 entities picked up for aggregation.
     aggregatedMetric = aggregatedEntity.getMetrics().iterator().next();
     assertEquals(150L, aggregatedMetric.getValues().values().iterator().next());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index f8e83998311f9..c7537b0278d53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -18,38 +18,49 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.collector;
 
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 /**
  * Unit tests for TimelineCollectorManager.
  */
 public class TestTimelineCollectorManager{
 
-  @Test(timeout = 60000, expected = YarnRuntimeException.class)
-  public void testTimelineCollectorManagerWithInvalidTimelineWriter() {
-    Configuration conf = new YarnConfiguration();
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
-        Object.class.getName());
-    runTimelineCollectorManagerWithConfig(conf);
+  @Test
+  @Timeout(60000)
+  void testTimelineCollectorManagerWithInvalidTimelineWriter() {
+    assertThrows(YarnRuntimeException.class, () -> {
+      Configuration conf = new YarnConfiguration();
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+          Object.class.getName());
+      runTimelineCollectorManagerWithConfig(conf);
+    });
   }
 
-  @Test(timeout = 60000, expected = YarnRuntimeException.class)
-  public void testTimelineCollectorManagerWithNonexistentTimelineWriter() {
-    String nonexistentTimelineWriterClass = "org.apache.org.yarn.server." +
-        "timelineservice.storage.XXXXXXXX";
-    Configuration conf = new YarnConfiguration();
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
-        nonexistentTimelineWriterClass);
-    runTimelineCollectorManagerWithConfig(conf);
+  @Test
+  @Timeout(60000)
+  void testTimelineCollectorManagerWithNonexistentTimelineWriter() {
+    assertThrows(YarnRuntimeException.class, () -> {
+      String nonexistentTimelineWriterClass = "org.apache.org.yarn.server." +
+          "timelineservice.storage.XXXXXXXX";
+      Configuration conf = new YarnConfiguration();
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+          nonexistentTimelineWriterClass);
+      runTimelineCollectorManagerWithConfig(conf);
+    });
   }
 
-  @Test(timeout = 60000)
-  public void testTimelineCollectorManagerWithFileSystemWriter() {
+  @Test
+  @Timeout(60000)
+  void testTimelineCollectorManagerWithFileSystemWriter() {
     Configuration conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
         FileSystemTimelineWriterImpl.class, TimelineWriter.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
index fa74689d9bcb3..35f027b514153 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderMetrics.java
@@ -18,11 +18,14 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
  * Test TimelineReaderMetrics.
@@ -32,24 +35,24 @@ public class TestTimelineReaderMetrics {
   private TimelineReaderMetrics metrics;
 
   @Test
-  public void testTimelineReaderMetrics() {
-    Assert.assertNotNull(metrics);
-    Assert.assertEquals(10,
+  void testTimelineReaderMetrics() {
+    assertNotNull(metrics);
+    assertEquals(10,
         metrics.getGetEntitiesSuccessLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getGetEntitiesFailureLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getGetEntityTypesSuccessLatency().getInterval());
-    Assert.assertEquals(10,
+    assertEquals(10,
         metrics.getGetEntityTypesFailureLatency().getInterval());
   }
 
-  @Before
+  @BeforeEach
   public void setup() {
     metrics = TimelineReaderMetrics.getInstance();
   }
 
-  @After
+  @AfterEach
   public void tearDown() {
     TimelineReaderMetrics.destroy();
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
index 6fc46cc7b03af..46331bf1e43d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
@@ -18,7 +18,8 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertEquals;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service.STATE;
@@ -26,12 +27,15 @@
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class TestTimelineReaderServer {
 
-  @Test(timeout = 60000)
-  public void testStartStopServer() throws Exception {
+  @Test
+  @Timeout(60000)
+  void testStartStopServer() throws Exception {
     @SuppressWarnings("resource")
     TimelineReaderServer server = new TimelineReaderServer();
     Configuration config = new YarnConfiguration();
@@ -56,30 +60,36 @@ public void testStartStopServer() throws Exception {
     }
   }
 
-  @Test(timeout = 60000, expected = YarnRuntimeException.class)
-  public void testTimelineReaderServerWithInvalidTimelineReader() {
-    Configuration conf = new YarnConfiguration();
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
-    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
-        "localhost:0");
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
-        Object.class.getName());
-    runTimelineReaderServerWithConfig(conf);
+  @Test
+  @Timeout(60000)
+  void testTimelineReaderServerWithInvalidTimelineReader() {
+    assertThrows(YarnRuntimeException.class, () -> {
+      Configuration conf = new YarnConfiguration();
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+      conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
+          "localhost:0");
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+          Object.class.getName());
+      runTimelineReaderServerWithConfig(conf);
+    });
   }
 
-  @Test(timeout = 60000, expected = YarnRuntimeException.class)
-  public void testTimelineReaderServerWithNonexistentTimelineReader() {
-    String nonexistentTimelineReaderClass = "org.apache.org.yarn.server." +
-        "timelineservice.storage.XXXXXXXX";
-    Configuration conf = new YarnConfiguration();
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
-    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
-        "localhost:0");
-    conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
-        nonexistentTimelineReaderClass);
-    runTimelineReaderServerWithConfig(conf);
+  @Test
+  @Timeout(60000)
+  void testTimelineReaderServerWithNonexistentTimelineReader() {
+    assertThrows(YarnRuntimeException.class, () -> {
+      String nonexistentTimelineReaderClass = "org.apache.org.yarn.server." +
+          "timelineservice.storage.XXXXXXXX";
+      Configuration conf = new YarnConfiguration();
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+      conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
+          "localhost:0");
+      conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+          nonexistentTimelineReaderClass);
+      runTimelineReaderServerWithConfig(conf);
+    });
   }
 
   /**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
index bc5eb9c5af559..feafeda02ce71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
@@ -18,38 +18,38 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
 import java.util.List;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 public class TestTimelineReaderUtils {
 
   @Test
-  public void testSplitUsingEscapeAndDelimChar() throws Exception {
+  void testSplitUsingEscapeAndDelimChar() throws Exception {
     List<String> list =
         TimelineReaderUtils.split("*!cluster!*!b**o***!xer!oozie**", '!', '*');
     String[] arr = new String[list.size()];
     arr = list.toArray(arr);
-    assertArrayEquals(new String[] {"!cluster", "!b*o*!xer", "oozie*"}, arr);
+    assertArrayEquals(new String[]{"!cluster", "!b*o*!xer", "oozie*"}, arr);
     list = TimelineReaderUtils.split("*!cluster!*!b**o***!xer!!", '!', '*');
     arr = new String[list.size()];
     arr = list.toArray(arr);
-    assertArrayEquals(new String[] {"!cluster", "!b*o*!xer", "", ""}, arr);
+    assertArrayEquals(new String[]{"!cluster", "!b*o*!xer", "", ""}, arr);
   }
 
   @Test
-  public void testJoinAndEscapeStrings() throws Exception {
+  void testJoinAndEscapeStrings() throws Exception {
     assertEquals("*!cluster!*!b**o***!xer!oozie**",
         TimelineReaderUtils.joinAndEscapeStrings(
-            new String[] {"!cluster", "!b*o*!xer", "oozie*"}, '!', '*'));
+            new String[]{"!cluster", "!b*o*!xer", "oozie*"}, '!', '*'));
     assertEquals("*!cluster!*!b**o***!xer!!",
         TimelineReaderUtils.joinAndEscapeStrings(
-            new String[] {"!cluster", "!b*o*!xer", "", ""}, '!', '*'));
+            new String[]{"!cluster", "!b*o*!xer", "", ""}, '!', '*'));
     assertNull(TimelineReaderUtils.joinAndEscapeStrings(
-        new String[] {"!cluster", "!b*o*!xer", null, ""}, '!', '*'));
+        new String[]{"!cluster", "!b*o*!xer", null, ""}, '!', '*'));
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index ef74716b56401..e71f3be43830e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -18,11 +18,6 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
@@ -30,9 +25,22 @@
 import java.net.URI;
 import java.net.URL;
 import java.util.Set;
-
 import javax.ws.rs.core.MediaType;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.JettyUtils;
@@ -45,21 +53,12 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.TestFileSystemTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
 
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineReaderWebServices {
 
@@ -69,17 +68,17 @@ public class TestTimelineReaderWebServices {
   private int serverPort;
   private TimelineReaderServer server;
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     TestFileSystemTimelineReaderImpl.initializeDataDirectory(ROOT_DIR);
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     FileUtils.deleteDirectory(new File(ROOT_DIR));
   }
 
-  @Before
+  @BeforeEach
   public void init() throws Exception {
     try {
       Configuration config = new YarnConfiguration();
@@ -97,11 +96,11 @@ public void init() throws Exception {
       server.start();
       serverPort = server.getWebServerPort();
     } catch (Exception e) {
-      Assert.fail("Web server failed to start");
+      fail("Web server failed to start");
     }
   }
 
-  @After
+  @AfterEach
   public void stop() throws Exception {
     if (server != null) {
       server.stop();
@@ -165,21 +164,21 @@ public HttpURLConnection getHttpURLConnection(final URL url)
   }
 
   @Test
-  public void testAbout() throws Exception {
+  void testAbout() throws Exception {
     URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/");
     Client client = createClient();
     try {
       ClientResponse resp = getResponse(client, uri);
       TimelineAbout about = resp.getEntity(TimelineAbout.class);
-      Assert.assertNotNull(about);
-      Assert.assertEquals("Timeline Reader API", about.getAbout());
+      assertNotNull(about);
+      assertEquals("Timeline Reader API", about.getAbout());
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntityDefaultView() throws Exception {
+  void testGetEntityDefaultView() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -191,7 +190,7 @@ public void testGetEntityDefaultView() throws Exception {
       assertNotNull(entity);
       assertEquals("id_1", entity.getId());
       assertEquals("app", entity.getType());
-      assertEquals((Long)1425016502000L, entity.getCreatedTime());
+      assertEquals((Long) 1425016502000L, entity.getCreatedTime());
       // Default view i.e. when no fields are specified, entity contains only
       // entity id, entity type and created time.
       assertEquals(0, entity.getConfigs().size());
@@ -202,7 +201,7 @@ public void testGetEntityDefaultView() throws Exception {
   }
 
   @Test
-  public void testGetEntityWithUserAndFlowInfo() throws Exception {
+  void testGetEntityWithUserAndFlowInfo() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -215,14 +214,14 @@ public void testGetEntityWithUserAndFlowInfo() throws Exception {
       assertNotNull(entity);
       assertEquals("id_1", entity.getId());
       assertEquals("app", entity.getType());
-      assertEquals((Long)1425016502000L, entity.getCreatedTime());
+      assertEquals((Long) 1425016502000L, entity.getCreatedTime());
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntityCustomFields() throws Exception {
+  void testGetEntityCustomFields() throws Exception {
     Client client = createClient();
     try {
       // Fields are case insensitive.
@@ -238,8 +237,8 @@ public void testGetEntityCustomFields() throws Exception {
       assertEquals("app", entity.getType());
       assertEquals(3, entity.getConfigs().size());
       assertEquals(3, entity.getMetrics().size());
-      assertTrue("UID should be present",
-          entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY));
+      assertTrue(entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY),
+          "UID should be present");
       // Includes UID.
       assertEquals(3, entity.getInfo().size());
       // No events will be returned as events are not part of fields.
@@ -250,7 +249,7 @@ public void testGetEntityCustomFields() throws Exception {
   }
 
   @Test
-  public void testGetEntityAllFields() throws Exception {
+  void testGetEntityAllFields() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -265,8 +264,8 @@ public void testGetEntityAllFields() throws Exception {
       assertEquals("app", entity.getType());
       assertEquals(3, entity.getConfigs().size());
       assertEquals(3, entity.getMetrics().size());
-      assertTrue("UID should be present",
-          entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY));
+      assertTrue(entity.getInfo().containsKey(TimelineReaderUtils.UID_KEY),
+          "UID should be present");
       // Includes UID.
       assertEquals(3, entity.getInfo().size());
       assertEquals(2, entity.getEvents().size());
@@ -276,7 +275,7 @@ public void testGetEntityAllFields() throws Exception {
   }
 
   @Test
-  public void testGetEntityNotPresent() throws Exception {
+  void testGetEntityNotPresent() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -288,7 +287,7 @@ public void testGetEntityNotPresent() throws Exception {
   }
 
   @Test
-  public void testQueryWithoutCluster() throws Exception {
+  void testQueryWithoutCluster() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -305,7 +304,8 @@ public void testQueryWithoutCluster() throws Exception {
           "timeline/apps/app1/entities/app");
       resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
@@ -316,52 +316,55 @@ public void testQueryWithoutCluster() throws Exception {
   }
 
   @Test
-  public void testGetEntities() throws Exception {
+  void testGetEntities() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
           "timeline/clusters/cluster1/apps/app1/entities/app");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(4, entities.size());
-      assertTrue("Entities id_1, id_2, id_3 and id_4 should have been" +
-          " present in response",
-          entities.contains(newEntity("app", "id_1")) &&
+      assertTrue(entities.contains(newEntity("app", "id_1")) &&
           entities.contains(newEntity("app", "id_2")) &&
           entities.contains(newEntity("app", "id_3")) &&
-          entities.contains(newEntity("app", "id_4")));
+          entities.contains(newEntity("app", "id_4")),
+          "Entities id_1, id_2, id_3 and id_4 should have been" +
+              " present in response");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesWithLimit() throws Exception {
+  void testGetEntitiesWithLimit() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
           "timeline/clusters/cluster1/apps/app1/entities/app?limit=2");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(2, entities.size());
       // Entities returned are based on most recent created time.
-      assertTrue("Entities with id_1 and id_4 should have been present " +
-          "in response based on entity created time.",
-          entities.contains(newEntity("app", "id_1")) &&
-          entities.contains(newEntity("app", "id_4")));
+      assertTrue(entities.contains(newEntity("app", "id_1")) &&
+          entities.contains(newEntity("app", "id_4")),
+          "Entities with id_1 and id_4 should have been present " +
+              "in response based on entity created time.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
           "clusters/cluster1/apps/app1/entities/app?limit=3");
       resp = getResponse(client, uri);
-      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+      });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
@@ -374,7 +377,7 @@ public void testGetEntitiesWithLimit() throws Exception {
   }
 
   @Test
-  public void testGetEntitiesBasedOnCreatedTime() throws Exception {
+  void testGetEntitiesBasedOnCreatedTime() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -382,44 +385,47 @@ public void testGetEntitiesBasedOnCreatedTime() throws Exception {
           "createdtimestart=1425016502030&createdtimeend=1425016502060");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_4 should have been present in response.",
-          entities.contains(newEntity("app", "id_4")));
+      assertTrue(entities.contains(newEntity("app", "id_4")),
+          "Entity with id_4 should have been present in response.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
           "clusters/cluster1/apps/app1/entities/app?createdtimeend" +
           "=1425016502010");
       resp = getResponse(client, uri);
-      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+      });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(3, entities.size());
-      assertFalse("Entity with id_4 should not have been present in response.",
-          entities.contains(newEntity("app", "id_4")));
+      assertFalse(entities.contains(newEntity("app", "id_4")),
+          "Entity with id_4 should not have been present in response.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
           "clusters/cluster1/apps/app1/entities/app?createdtimestart=" +
           "1425016502010");
       resp = getResponse(client, uri);
-      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+      });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_4 should have been present in response.",
-          entities.contains(newEntity("app", "id_4")));
+      assertTrue(entities.contains(newEntity("app", "id_4")),
+          "Entity with id_4 should have been present in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesByRelations() throws Exception {
+  void testGetEntitiesByRelations() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -427,44 +433,47 @@ public void testGetEntitiesByRelations() throws Exception {
           "flow:flow1");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_1 should have been present in response.",
-          entities.contains(newEntity("app", "id_1")));
+      assertTrue(entities.contains(newEntity("app", "id_1")),
+          "Entity with id_1 should have been present in response.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
           "clusters/cluster1/apps/app1/entities/app?isrelatedto=" +
           "type1:tid1_2,type2:tid2_1%60");
       resp = getResponse(client, uri);
-      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+      });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_1 should have been present in response.",
-          entities.contains(newEntity("app", "id_1")));
+      assertTrue(entities.contains(newEntity("app", "id_1")),
+          "Entity with id_1 should have been present in response.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
           "clusters/cluster1/apps/app1/entities/app?isrelatedto=" +
           "type1:tid1_1:tid1_2,type2:tid2_1%60");
       resp = getResponse(client, uri);
-      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+      entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+      });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_1 should have been present in response.",
-          entities.contains(newEntity("app", "id_1")));
+      assertTrue(entities.contains(newEntity("app", "id_1")),
+          "Entity with id_1 should have been present in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesByConfigFilters() throws Exception {
+  void testGetEntitiesByConfigFilters() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -472,20 +481,21 @@ public void testGetEntitiesByConfigFilters() throws Exception {
           "conffilters=config_1%20eq%20123%20AND%20config_3%20eq%20abc");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_3 should have been present in response.",
-          entities.contains(newEntity("app", "id_3")));
+      assertTrue(entities.contains(newEntity("app", "id_3")),
+          "Entity with id_3 should have been present in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesByInfoFilters() throws Exception {
+  void testGetEntitiesByInfoFilters() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -493,20 +503,21 @@ public void testGetEntitiesByInfoFilters() throws Exception {
           "infofilters=info2%20eq%203.5");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_3 should have been present in response.",
-          entities.contains(newEntity("app", "id_3")));
+      assertTrue(entities.contains(newEntity("app", "id_3")),
+          "Entity with id_3 should have been present in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesByMetricFilters() throws Exception {
+  void testGetEntitiesByMetricFilters() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -514,22 +525,23 @@ public void testGetEntitiesByMetricFilters() throws Exception {
           "metricfilters=metric3%20ge%200");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(2, entities.size());
-      assertTrue("Entities with id_1 and id_2 should have been present" +
-          " in response.",
-          entities.contains(newEntity("app", "id_1")) &&
-          entities.contains(newEntity("app", "id_2")));
+      assertTrue(entities.contains(newEntity("app", "id_1")) &&
+          entities.contains(newEntity("app", "id_2")),
+          "Entities with id_1 and id_2 should have been present" +
+              " in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesByEventFilters() throws Exception {
+  void testGetEntitiesByEventFilters() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -537,31 +549,33 @@ public void testGetEntitiesByEventFilters() throws Exception {
           "eventfilters=event_2,event_4");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
       assertEquals(1, entities.size());
-      assertTrue("Entity with id_3 should have been present in response.",
-          entities.contains(newEntity("app", "id_3")));
+      assertTrue(entities.contains(newEntity("app", "id_3")),
+          "Entity with id_3 should have been present in response.");
     } finally {
       client.destroy();
     }
   }
 
   @Test
-  public void testGetEntitiesNoMatch() throws Exception {
+  void testGetEntitiesNoMatch() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
           "timeline/clusters/cluster1/apps/app1/entities/app?" +
-          "metricfilters=metric7%20ge%200&isrelatedto=type1:tid1_1:tid1_2,"+
+          "metricfilters=metric7%20ge%200&isrelatedto=type1:tid1_1:tid1_2," +
           "type2:tid2_1%60&relatesto=flow:flow1&eventfilters=event_2,event_4" +
           "&infofilters=info2%20eq%203.5&createdtimestart=1425016502030&" +
           "createdtimeend=1425016502060");
       ClientResponse resp = getResponse(client, uri);
       Set<TimelineEntity> entities =
-          resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+          resp.getEntity(new GenericType<Set<TimelineEntity>>(){
+          });
       assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
           resp.getType().toString());
       assertNotNull(entities);
@@ -572,7 +586,7 @@ public void testGetEntitiesNoMatch() throws Exception {
   }
 
   @Test
-  public void testInvalidValuesHandling() throws Exception {
+  void testInvalidValuesHandling() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -592,7 +606,7 @@ public void testInvalidValuesHandling() throws Exception {
   }
 
   @Test
-  public void testGetAppAttempts() throws Exception {
+  void testGetAppAttempts() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
@@ -608,15 +622,15 @@ public void testGetAppAttempts() throws Exception {
       int totalEntities = entities.size();
       assertEquals(2, totalEntities);
       assertTrue(
-          "Entity with app-attempt-2 should have been present in response.",
           entities.contains(
               newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
-                  "app-attempt-1")));
+                  "app-attempt-1")),
+          "Entity with app-attempt-2 should have been present in response.");
       assertTrue(
-          "Entity with app-attempt-2 should have been present in response.",
           entities.contains(
               newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
-                  "app-attempt-2")));
+                  "app-attempt-2")),
+          "Entity with app-attempt-2 should have been present in response.");
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
           + "timeline/clusters/cluster1/apps/app1/appattempts");
@@ -628,15 +642,15 @@ public void testGetAppAttempts() throws Exception {
       int retrievedEntity = entities.size();
       assertEquals(2, retrievedEntity);
       assertTrue(
-          "Entity with app-attempt-2 should have been present in response.",
           entities.contains(
               newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
-                  "app-attempt-1")));
+                  "app-attempt-1")),
+          "Entity with app-attempt-2 should have been present in response.");
       assertTrue(
-          "Entity with app-attempt-2 should have been present in response.",
           entities.contains(
               newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
-                  "app-attempt-2")));
+                  "app-attempt-2")),
+          "Entity with app-attempt-2 should have been present in response.");
 
       assertEquals(totalEntities, retrievedEntity);
 
@@ -646,7 +660,7 @@ public void testGetAppAttempts() throws Exception {
   }
 
   @Test
-  public void testGetAppAttempt() throws Exception {
+  void testGetAppAttempt() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
@@ -677,7 +691,7 @@ public void testGetAppAttempt() throws Exception {
   }
 
   @Test
-  public void testGetContainers() throws Exception {
+  void testGetContainers() throws Exception {
     Client client = createClient();
     try {
       // total 3 containers in a application.
@@ -693,17 +707,17 @@ public void testGetContainers() throws Exception {
       int totalEntities = entities.size();
       assertEquals(3, totalEntities);
       assertTrue(
-          "Entity with container_1_1 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1")),
+          "Entity with container_1_1 should have been present in response.");
       assertTrue(
-          "Entity with container_2_1 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1")),
+          "Entity with container_2_1 should have been present in response.");
       assertTrue(
-          "Entity with container_2_2 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2")),
+          "Entity with container_2_2 should have been present in response.");
 
       // for app-attempt1 1 container has run
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
@@ -717,9 +731,9 @@ public void testGetContainers() throws Exception {
       int retrievedEntity = entities.size();
       assertEquals(1, retrievedEntity);
       assertTrue(
-          "Entity with container_1_1 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1")),
+          "Entity with container_1_1 should have been present in response.");
 
       // for app-attempt2 2 containers has run
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
@@ -733,13 +747,13 @@ public void testGetContainers() throws Exception {
       retrievedEntity += entities.size();
       assertEquals(2, entities.size());
       assertTrue(
-          "Entity with container_2_1 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1")),
+          "Entity with container_2_1 should have been present in response.");
       assertTrue(
-          "Entity with container_2_2 should have been present in response.",
           entities.contains(newEntity(
-              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2")));
+              TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2")),
+          "Entity with container_2_2 should have been present in response.");
 
       assertEquals(totalEntities, retrievedEntity);
 
@@ -749,7 +763,7 @@ public void testGetContainers() throws Exception {
   }
 
   @Test
-  public void testGetContainer() throws Exception {
+  void testGetContainer() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
@@ -780,11 +794,11 @@ public void testGetContainer() throws Exception {
   }
 
   @Test
-  public void testHealthCheck() throws Exception {
+  void testHealthCheck() throws Exception {
     Client client = createClient();
     try {
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"
-      + "timeline/health");
+          + "timeline/health");
       ClientResponse resp = getResponse(client, uri);
       TimelineHealth timelineHealth =
           resp.getEntity(new GenericType<TimelineHealth>() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesACL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesACL.java
index fbd042bd01537..316c1949ad12f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesACL.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesACL.java
@@ -18,19 +18,27 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URL;
-
 import javax.ws.rs.core.MediaType;
 
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -38,20 +46,11 @@
 import org.apache.hadoop.yarn.server.timelineservice.storage.TestFileSystemTimelineReaderImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
 
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Tests ACL check while retrieving entity-types per application.
@@ -66,17 +65,17 @@ public class TestTimelineReaderWebServicesACL {
   private TimelineReaderServer server;
   private static final String ADMIN = "yarn";
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     TestFileSystemTimelineReaderImpl.initializeDataDirectory(ROOT_DIR);
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     FileUtils.deleteDirectory(new File(ROOT_DIR));
   }
 
-  @Before
+  @BeforeEach
   public void init() throws Exception {
     try {
       Configuration config = new YarnConfiguration();
@@ -97,11 +96,11 @@ public void init() throws Exception {
       server.start();
       serverPort = server.getWebServerPort();
     } catch (Exception e) {
-      Assert.fail("Web server failed to start");
+      fail("Web server failed to start");
     }
   }
 
-  @After
+  @AfterEach
   public void stop() throws Exception {
     if (server != null) {
       server.stop();
@@ -141,35 +140,35 @@ public HttpURLConnection getHttpURLConnection(final URL url)
   }
 
   @Test
-  public void testGetEntityTypes() throws Exception {
+  void testGetEntityTypes() throws Exception {
     Client client = createClient();
     try {
-      String unAuthorizedUser ="user2";
+      String unAuthorizedUser = "user2";
       URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
-          "timeline/apps/app1/entity-types?user.name="+unAuthorizedUser);
+          "timeline/apps/app1/entity-types?user.name=" + unAuthorizedUser);
       String msg = "User " + unAuthorizedUser
           + " is not allowed to read TimelineService V2 data.";
       ClientResponse resp = verifyHttpResponse(client, uri, Status.FORBIDDEN);
       assertTrue(resp.getEntity(String.class).contains(msg));
 
-      String authorizedUser ="user1";
+      String authorizedUser = "user1";
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
-          "timeline/apps/app1/entity-types?user.name="+authorizedUser);
+          "timeline/apps/app1/entity-types?user.name=" + authorizedUser);
       verifyHttpResponse(client, uri, Status.OK);
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
-          "timeline/apps/app1/entity-types?user.name="+ADMIN);
+          "timeline/apps/app1/entity-types?user.name=" + ADMIN);
       verifyHttpResponse(client, uri, Status.OK);
 
       // Verify with Query Parameter userid
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
-          "timeline/apps/app1/entity-types?user.name="+authorizedUser
-          + "&userid="+authorizedUser);
+          "timeline/apps/app1/entity-types?user.name=" + authorizedUser
+          + "&userid=" + authorizedUser);
       verifyHttpResponse(client, uri, Status.OK);
 
       uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
-          "timeline/apps/app1/entity-types?user.name="+authorizedUser
-          + "&userid="+unAuthorizedUser);
+          "timeline/apps/app1/entity-types?user.name=" + authorizedUser
+          + "&userid=" + unAuthorizedUser);
       verifyHttpResponse(client, uri, Status.FORBIDDEN);
     } finally {
       client.destroy();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
index 6ad44272a8935..fffa1b50f8ab6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
@@ -18,18 +18,23 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.webapp.ForbiddenException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import java.util.LinkedHashSet;
-import java.util.Set;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineReaderWebServicesBasicAcl {
 
@@ -39,11 +44,11 @@ public class TestTimelineReaderWebServicesBasicAcl {
       UserGroupInformation.createRemoteUser(adminUser);
   private Configuration config;
 
-  @Before public void setUp() throws Exception {
+  @BeforeEach public void setUp() throws Exception {
     config = new YarnConfiguration();
   }
 
-  @After public void tearDown() throws Exception {
+  @AfterEach public void tearDown() throws Exception {
     if (manager != null) {
       manager.stop();
       manager = null;
@@ -51,7 +56,8 @@ public class TestTimelineReaderWebServicesBasicAcl {
     config = null;
   }
 
-  @Test public void testTimelineReaderManagerAclsWhenDisabled()
+  @Test
+  void testTimelineReaderManagerAclsWhenDisabled()
       throws Exception {
     config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
     config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
@@ -60,14 +66,15 @@ public class TestTimelineReaderWebServicesBasicAcl {
     manager.start();
 
     // when acls are disabled, always return true
-    Assert.assertTrue(manager.checkAccess(null));
+    assertTrue(manager.checkAccess(null));
 
     // filter is disabled, so should return false
-    Assert.assertFalse(
+    assertFalse(
         TimelineReaderWebServices.isDisplayEntityPerUserFilterEnabled(config));
   }
 
-  @Test public void testTimelineReaderManagerAclsWhenEnabled()
+  @Test
+  void testTimelineReaderManagerAclsWhenEnabled()
       throws Exception {
     Configuration config = new YarnConfiguration();
     config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
@@ -85,30 +92,30 @@ public class TestTimelineReaderWebServicesBasicAcl {
         UserGroupInformation.createRemoteUser(user2);
 
     // false because ugi is null
-    Assert.assertFalse(TimelineReaderWebServices
+    assertFalse(TimelineReaderWebServices
         .validateAuthUserWithEntityUser(manager, null, user1));
 
     // false because ugi is null in non-secure cluster. User must pass
     // ?user.name as query params in REST end points.
     try {
       TimelineReaderWebServices.checkAccess(manager, null, user1);
-      Assert.fail("user1Ugi is not allowed to view user1");
+      fail("user1Ugi is not allowed to view user1");
     } catch (ForbiddenException e) {
       // expected
     }
 
     // incoming ugi is admin asking for entity owner user1
-    Assert.assertTrue(
+    assertTrue(
         TimelineReaderWebServices.checkAccess(manager, adminUgi, user1));
 
     // incoming ugi is admin asking for entity owner user1
-    Assert.assertTrue(
+    assertTrue(
         TimelineReaderWebServices.checkAccess(manager, adminUgi, user2));
 
     // incoming ugi is non-admin i.e user1Ugi asking for entity owner user2
     try {
       TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
-      Assert.fail("user1Ugi is not allowed to view user2");
+      fail("user1Ugi is not allowed to view user2");
     } catch (ForbiddenException e) {
       // expected
     }
@@ -116,7 +123,7 @@ public class TestTimelineReaderWebServicesBasicAcl {
     // incoming ugi is non-admin i.e user2Ugi asking for entity owner user1
     try {
       TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
-      Assert.fail("user2Ugi is not allowed to view user1");
+      fail("user2Ugi is not allowed to view user1");
     } catch (ForbiddenException e) {
       // expected
     }
@@ -127,25 +134,23 @@ public class TestTimelineReaderWebServicesBasicAcl {
     TimelineReaderWebServices
         .checkAccess(manager, adminUgi, entities, userKey, true);
     // admin is allowed to view other entities
-    Assert.assertTrue(entities.size() == 10);
+    assertEquals(10, entities.size());
 
     // incoming ugi is user1Ugi asking for entities
     // only user1 entities are allowed to view
     entities = createEntities(5, userKey);
     TimelineReaderWebServices
         .checkAccess(manager, user1Ugi, entities, userKey, true);
-    Assert.assertTrue(entities.size() == 1);
-    Assert
-        .assertEquals(user1, entities.iterator().next().getInfo().get(userKey));
+    assertEquals(1, entities.size());
+    assertEquals(user1, entities.iterator().next().getInfo().get(userKey));
 
     // incoming ugi is user2Ugi asking for entities
     // only user2 entities are allowed to view
     entities = createEntities(8, userKey);
     TimelineReaderWebServices
         .checkAccess(manager, user2Ugi, entities, userKey, true);
-    Assert.assertTrue(entities.size() == 1);
-    Assert
-        .assertEquals(user2, entities.iterator().next().getInfo().get(userKey));
+    assertEquals(1, entities.size());
+    assertEquals(user2, entities.iterator().next().getInfo().get(userKey));
   }
 
   Set<TimelineEntity> createEntities(int noOfUsers, String userKey) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
index 882189c68201b..80f462c2fe909 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
@@ -18,10 +18,7 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.util.Sets;
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
@@ -32,20 +29,19 @@
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineReaderWebServicesUtils {
   private static void verifyFilterList(String expr, TimelineFilterList list,
-      TimelineFilterList expectedList) throws Exception {
-    assertNotNull(list);
-    assertTrue("Unexpected List received after parsing expression " + expr +
-        ". Expected=" + expectedList + " but Actual=" + list,
-        list.equals(expectedList));
+      TimelineFilterList expectedList) {
+    assertEquals(expectedList, list);
   }
 
   @Test
-  public void testMetricFiltersParsing() throws Exception {
+  void testMetricFiltersParsing() throws Exception {
     String expr = "(((key11 ne 234 AND key12 gt 23) AND " +
         "(key13 lt 34 OR key14 ge 567)) OR (key21 lt 24 OR key22 le 45))";
     TimelineFilterList expectedList = new TimelineFilterList(
@@ -168,7 +164,7 @@ public void testMetricFiltersParsing() throws Exception {
         TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
 
     // Test with unnecessary spaces.
-    expr = "  abc ne   234       AND       def           gt 23 OR     rst lt "+
+    expr = "  abc ne   234       AND       def           gt 23 OR     rst lt " +
         "           24     OR xyz     le     456    AND pqr ge 2        ";
     expectedList = new TimelineFilterList(
         new TimelineFilterList(
@@ -283,7 +279,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Improper brackers. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((key11 ne 234 AND key12 gt v3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -291,7 +288,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Non Numeric value. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((key11 ne (234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -299,7 +297,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Unexpected opening bracket. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((k)ey11 ne 234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -307,7 +306,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Unexpected closing bracket. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((key11 rs 234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -315,7 +315,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Improper compare op. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((key11 ne 234 PI key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -323,7 +324,8 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Improper op. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((key11 ne 234 PI key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
         "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
@@ -331,32 +333,36 @@ public void testMetricFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Improper op. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(key11 ne 234 AND key12 gt 3)) OR (key13 lt 24 OR key14 le 456)";
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Unbalanced brackets. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(key11 rne 234 AND key12 gt 3) OR (key13 lt 24 OR key14 le 456)";
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Invalid compareop. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(key11 ne 234 AND key12 gt 3) OR (key13 lt 24 OR key14 le";
     try {
       TimelineReaderWebServicesUtils.parseMetricFilters(expr);
       fail("Compareop cant be parsed. Exception should have been thrown.");
-    } catch (TimelineParseException e) {}
+    } catch (TimelineParseException e) {
+    }
 
     assertNull(TimelineReaderWebServicesUtils.parseMetricFilters(null));
     assertNull(TimelineReaderWebServicesUtils.parseMetricFilters("   "));
   }
 
   @Test
-  public void testConfigFiltersParsing() throws Exception {
+  void testConfigFiltersParsing() throws Exception {
     String expr = "(((key11 ne 234 AND key12 eq val12) AND " +
         "(key13 ene val13 OR key14 eq 567)) OR (key21 eq val_21 OR key22 eq " +
         "val.22))";
@@ -412,7 +418,7 @@ public void testConfigFiltersParsing() throws Exception {
         parseKVFilters(expr, true), expectedList);
 
     // Test with unnecessary spaces.
-    expr = "  abc ne   234       AND       def           eq 23 OR     rst ene "+
+    expr = "  abc ne   234       AND       def           eq 23 OR     rst ene " +
         "           24     OR xyz     eq     456    AND pqr eq 2        ";
     expectedList = new TimelineFilterList(
         new TimelineFilterList(
@@ -439,10 +445,12 @@ public void testConfigFiltersParsing() throws Exception {
       TimelineReaderWebServicesUtils.parseKVFilters(expr, true);
       fail("Invalid compareop specified for config filters. Should be either" +
           " eq,ne or ene and exception should have been thrown.");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
   }
+
   @Test
-  public void testInfoFiltersParsing() throws Exception {
+  void testInfoFiltersParsing() throws Exception {
     String expr = "(((key11 ne 234 AND key12 eq val12) AND " +
         "(key13 ene val13 OR key14 eq 567)) OR (key21 eq val_21 OR key22 eq " +
         "5.0))";
@@ -499,7 +507,7 @@ public void testInfoFiltersParsing() throws Exception {
         parseKVFilters(expr, false), expectedList);
 
     // Test with unnecessary spaces.
-    expr = "  abc ne   234       AND       def           eq 23 OR     rst ene "+
+    expr = "  abc ne   234       AND       def           eq 23 OR     rst ene " +
         "           24     OR xyz     eq     456    AND pqr eq 2        ";
     expectedList = new TimelineFilterList(
         new TimelineFilterList(
@@ -524,7 +532,7 @@ public void testInfoFiltersParsing() throws Exception {
     expr = "abdeq";
     try {
       TimelineReaderWebServicesUtils.parseKVFilters(expr, false);
-      Assert.fail("Expression valuation should throw exception.");
+      fail("Expression valuation should throw exception.");
     } catch (TimelineParseException e) {
       // expected: do nothing
     }
@@ -532,7 +540,7 @@ public void testInfoFiltersParsing() throws Exception {
     expr = "abc gt 234 AND defeq";
     try {
       TimelineReaderWebServicesUtils.parseKVFilters(expr, false);
-      Assert.fail("Expression valuation should throw exception.");
+      fail("Expression valuation should throw exception.");
     } catch (TimelineParseException e) {
       // expected: do nothing
     }
@@ -540,14 +548,14 @@ public void testInfoFiltersParsing() throws Exception {
     expr = "((key11 ne 234 AND key12 eq val12) AND (key13eq OR key14 eq va14))";
     try {
       TimelineReaderWebServicesUtils.parseKVFilters(expr, false);
-      Assert.fail("Expression valuation should throw exception.");
+      fail("Expression valuation should throw exception.");
     } catch (TimelineParseException e) {
       // expected: do nothing
     }
   }
 
   @Test
-  public void testEventFiltersParsing() throws Exception {
+  void testEventFiltersParsing() throws Exception {
     String expr = "abc,def";
     TimelineFilterList expectedList = new TimelineFilterList(
         new TimelineExistsFilter(TimelineCompareOp.EQUAL, "abc"),
@@ -641,85 +649,96 @@ public void testEventFiltersParsing() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Improper brackets. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((!(abc,def,uvc) (OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
         " OR ((bcd,tyu) AND uvb))";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected opening bracket. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((!(abc,def,uvc) OR) (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
         " OR ((bcd,tyu) AND uvb))";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected closing bracket. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((!(abc,def,uvc) PI (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
         " OR ((bcd,tyu) AND uvb))";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Invalid op. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((!(abc,def,uvc) !OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
         " OR ((bcd,tyu) AND uvb))";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected ! char. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "abc,def,uvc) OR (rst, uvx)";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected closing bracket. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "abc,def,uvc OR )rst, uvx)";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected closing bracket. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "abc,def,uvc OR ,rst, uvx)";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected delimiter. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "abc,def,uvc OR !  ";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unexpected not char. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(abc,def,uvc)) OR (rst, uvx)";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("Unbalanced brackets. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "(((! ,(abc,def,uvc) OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu" +
         "))) OR ((bcd,tyu) AND uvb))";
     try {
       TimelineReaderWebServicesUtils.parseEventFilters(expr);
       fail("( should follow ! char. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     assertNull(TimelineReaderWebServicesUtils.parseEventFilters(null));
     assertNull(TimelineReaderWebServicesUtils.parseEventFilters("   "));
   }
 
   @Test
-  public void testRelationFiltersParsing() throws Exception {
+  void testRelationFiltersParsing() throws Exception {
     String expr = "type1:entity11,type2:entity21:entity22";
     TimelineFilterList expectedList = new TimelineFilterList(
         new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-            "type1", Sets.newHashSet((Object)"entity11")),
+            "type1", Sets.newHashSet((Object) "entity11")),
         new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-            "type2", Sets.newHashSet((Object)"entity21", "entity22"))
+            "type2", Sets.newHashSet((Object) "entity21", "entity22"))
     );
     verifyFilterList(expr, TimelineReaderWebServicesUtils.
         parseRelationFilters(expr), expectedList);
@@ -733,16 +752,16 @@ public void testRelationFiltersParsing() throws Exception {
     expectedList = new TimelineFilterList(Operator.OR,
         new TimelineFilterList(
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                "type1", Sets.newHashSet((Object)"entity11")),
+                "type1", Sets.newHashSet((Object) "entity11")),
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                "type2", Sets.newHashSet((Object)"entity21", "entity22"))
+                "type2", Sets.newHashSet((Object) "entity21", "entity22"))
         ),
         new TimelineFilterList(
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
                 "type3", Sets.newHashSet(
-                    (Object)"entity31", "entity32", "entity33")),
+                    (Object) "entity31", "entity32", "entity33")),
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                "type1", Sets.newHashSet((Object)"entity11", "entity12"))
+                "type1", Sets.newHashSet((Object) "entity11", "entity12"))
         )
     );
     verifyFilterList(expr, TimelineReaderWebServicesUtils.
@@ -754,25 +773,25 @@ public void testRelationFiltersParsing() throws Exception {
     expectedList = new TimelineFilterList(Operator.OR,
         new TimelineFilterList(
             new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                "type1", Sets.newHashSet((Object)"entity11")),
+                "type1", Sets.newHashSet((Object) "entity11")),
             new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                "type2", Sets.newHashSet((Object)"entity21", "entity22")),
+                "type2", Sets.newHashSet((Object) "entity21", "entity22")),
             new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                "type5", Sets.newHashSet((Object)"entity51"))
+                "type5", Sets.newHashSet((Object) "entity51"))
         ),
         new TimelineFilterList(
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
                 "type3", Sets.newHashSet(
-                    (Object)"entity31", "entity32", "entity33")),
+                    (Object) "entity31", "entity32", "entity33")),
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                "type1", Sets.newHashSet((Object)"entity11", "entity12"))
+                "type1", Sets.newHashSet((Object) "entity11", "entity12"))
         )
     );
     verifyFilterList(expr, TimelineReaderWebServicesUtils.
         parseRelationFilters(expr), expectedList);
 
     expr = "(((!(type1:entity11,type2:entity21:entity22,type5:entity51) OR " +
-        "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND "+
+        "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND " +
         "(!(type11:entity111) OR !(type4:entity43:entity44:entity47:entity49," +
         "type7:entity71))) OR ((type2:entity2,type8:entity88) AND t9:e:e1))";
     expectedList = new TimelineFilterList(Operator.OR,
@@ -780,45 +799,45 @@ public void testRelationFiltersParsing() throws Exception {
             new TimelineFilterList(Operator.OR,
                 new TimelineFilterList(
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                        "type1", Sets.newHashSet((Object)"entity11")),
+                        "type1", Sets.newHashSet((Object) "entity11")),
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
                         "type2", Sets.newHashSet(
-                            (Object)"entity21", "entity22")),
+                            (Object) "entity21", "entity22")),
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                        "type5", Sets.newHashSet((Object)"entity51"))
+                        "type5", Sets.newHashSet((Object) "entity51"))
                 ),
                 new TimelineFilterList(
                     new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
                         "type3", Sets.newHashSet(
-                            (Object)"entity31", "entity32", "entity33")),
+                            (Object) "entity31", "entity32", "entity33")),
                     new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
                         "type1", Sets.newHashSet(
-                            (Object)"entity11", "entity12"))
+                            (Object) "entity11", "entity12"))
                 )
             ),
             new TimelineFilterList(Operator.OR,
                 new TimelineFilterList(
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                        "type11", Sets.newHashSet((Object)"entity111"))
+                        "type11", Sets.newHashSet((Object) "entity111"))
                 ),
                 new TimelineFilterList(
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                        "type4", Sets.newHashSet((Object)"entity43", "entity44",
+                        "type4", Sets.newHashSet((Object) "entity43", "entity44",
                             "entity47", "entity49")),
                     new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
-                        "type7", Sets.newHashSet((Object)"entity71"))
+                        "type7", Sets.newHashSet((Object) "entity71"))
                 )
             )
         ),
         new TimelineFilterList(
             new TimelineFilterList(
                 new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                    "type2", Sets.newHashSet((Object)"entity2")),
+                    "type2", Sets.newHashSet((Object) "entity2")),
                 new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
-                    "type8", Sets.newHashSet((Object)"entity88"))
+                    "type8", Sets.newHashSet((Object) "entity88"))
             ),
             new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "t9",
-                Sets.newHashSet((Object)"e", "e1"))
+                Sets.newHashSet((Object) "e", "e1"))
         )
     );
     verifyFilterList(expr, TimelineReaderWebServicesUtils.
@@ -834,18 +853,19 @@ public void testRelationFiltersParsing() throws Exception {
         parseRelationFilters(expr), expectedList);
 
     expr = "(((!(type1 : entity11,type2:entity21:entity22,type5:entity51) OR " +
-        "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND "+
+        "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND " +
         "(!(type11:entity111) OR !(type4:entity43:entity44:entity47:entity49," +
         "type7:entity71))) OR ((type2:entity2,type8:entity88) AND t9:e:e1))";
     try {
       TimelineReaderWebServicesUtils.parseRelationFilters(expr);
       fail("Space not allowed in relation expression. Exception should have " +
           "been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
   }
 
   @Test
-  public void testDataToRetrieve() throws Exception {
+  void testDataToRetrieve() throws Exception {
     String expr = "abc,def";
     TimelineFilterList expectedList = new TimelineFilterList(Operator.OR,
         new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "abc"),
@@ -913,28 +933,32 @@ public void testDataToRetrieve() throws Exception {
     try {
       TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
       fail("No closing bracket. Exception should have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "!abc,def,xyz";
     try {
       TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
       fail("NOT(!) should be followed by opening bracket. Exception should " +
           "have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "!abc,def,xyz";
     try {
       TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
       fail("NOT(!) should be followed by opening bracket. Exception should " +
           "have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     expr = "!   r(  abc,def,xyz)";
     try {
       TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
       fail("NOT(!) should be followed by opening bracket. Exception should " +
           "have been thrown");
-    } catch (TimelineParseException e){}
+    } catch (TimelineParseException e) {
+    }
 
     assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve(null));
     assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("     "));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java
index 576699d12b915..86b20f8592cbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWhitelistAuthorizationFilter.java
@@ -18,12 +18,6 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
 import java.io.IOException;
 import java.security.Principal;
 import java.security.PrivilegedExceptionAction;
@@ -31,18 +25,24 @@
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.Map;
-
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderWhitelistAuthorizationFilter;
-import org.junit.Test;
-import org.mockito.Mockito;
+
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * Unit tests for {@link TimelineReaderWhitelistAuthorizationFilter}.
@@ -85,7 +85,7 @@ public ServletContext getServletContext() {
   }
 
   @Test
-  public void checkFilterAllowedUser() throws ServletException, IOException {
+  void checkFilterAllowedUser() throws ServletException, IOException {
     Map<String, String> map = new HashMap<String, String>();
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true");
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS,
@@ -111,7 +111,7 @@ public String getName() {
   }
 
   @Test
-  public void checkFilterNotAllowedUser() throws ServletException, IOException {
+  void checkFilterNotAllowedUser() throws ServletException, IOException {
     Map<String, String> map = new HashMap<String, String>();
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true");
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_ALLOWED_USERS,
@@ -138,7 +138,7 @@ public String getName() {
   }
 
   @Test
-  public void checkFilterAllowedUserGroup()
+  void checkFilterAllowedUserGroup()
       throws ServletException, IOException, InterruptedException {
     Map<String, String> map = new HashMap<String, String>();
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true");
@@ -172,7 +172,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterNotAlloweGroup()
+  void checkFilterNotAlloweGroup()
       throws ServletException, IOException, InterruptedException {
     Map<String, String> map = new HashMap<String, String>();
     map.put(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED, "true");
@@ -207,7 +207,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterAllowAdmins()
+  void checkFilterAllowAdmins()
       throws ServletException, IOException, InterruptedException {
     // check that users in admin acl list are allowed to read
     Map<String, String> map = new HashMap<String, String>();
@@ -243,7 +243,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterAllowAdminsWhenNoUsersSet()
+  void checkFilterAllowAdminsWhenNoUsersSet()
       throws ServletException, IOException, InterruptedException {
     // check that users in admin acl list are allowed to read
     Map<String, String> map = new HashMap<String, String>();
@@ -277,7 +277,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterAllowNoOneWhenAdminAclsEmptyAndUserAclsEmpty()
+  void checkFilterAllowNoOneWhenAdminAclsEmptyAndUserAclsEmpty()
       throws ServletException, IOException, InterruptedException {
     // check that users in admin acl list are allowed to read
     Map<String, String> map = new HashMap<String, String>();
@@ -311,7 +311,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterReadAuthDisabledNoAclSettings()
+  void checkFilterReadAuthDisabledNoAclSettings()
       throws ServletException, IOException, InterruptedException {
     // Default settings for Read Auth Enabled (false)
     // No values in admin acls or allowed read user list
@@ -344,7 +344,7 @@ public Object run() throws Exception {
   }
 
   @Test
-  public void checkFilterReadAuthDisabledButAclSettingsPopulated()
+  void checkFilterReadAuthDisabledButAclSettingsPopulated()
       throws ServletException, IOException, InterruptedException {
     Map<String, String> map = new HashMap<String, String>();
     // Default settings for Read Auth Enabled (false)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
index 12b3fc0140f3d..7b08616b346ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
@@ -18,16 +18,16 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.reader;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
+import org.junit.jupiter.api.Test;
 
-import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestTimelineUIDConverter {
 
   @Test
-  public void testUIDEncodingDecoding() throws Exception {
+  void testUIDEncodingDecoding() throws Exception {
     TimelineReaderContext context = new TimelineReaderContext(
         "!cluster", "!b*o*!xer", "oozie*", null, null, null, null);
     String uid = TimelineUIDConverter.FLOW_UID.encodeUID(context);
@@ -80,7 +80,7 @@ public void testUIDEncodingDecoding() throws Exception {
   }
 
   @Test
-  public void testUIDNotProperlyEscaped() throws Exception {
+  void testUIDNotProperlyEscaped() throws Exception {
     try {
       TimelineUIDConverter.FLOW_UID.decodeUID("*!cluster!*!b*o***!xer!oozie**");
       fail("UID not properly escaped. Exception should have been thrown.");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index 46873ab99044d..cf94749e883e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -30,6 +30,11 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.io.FileUtils;
@@ -53,11 +58,9 @@
 import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestFileSystemTimelineReaderImpl {
 
@@ -65,7 +68,7 @@ public class TestFileSystemTimelineReaderImpl {
       TestFileSystemTimelineReaderImpl.class.getSimpleName()).getAbsolutePath();
   private FileSystemTimelineReaderImpl reader;
 
-  @BeforeClass
+  @BeforeAll
   public static void setup() throws Exception {
     initializeDataDirectory(ROOT_DIR);
   }
@@ -74,7 +77,7 @@ public static void initializeDataDirectory(String rootDir) throws Exception {
     loadEntityData(rootDir);
     // Create app flow mapping file.
     CSVFormat format =
-        CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+        CSVFormat.Builder.create().setHeader("APP", "USER", "FLOW", "FLOWRUN").build();
     String appFlowMappingFile = rootDir + File.separator + "entities" +
         File.separator + "cluster1" + File.separator +
         FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
@@ -89,12 +92,12 @@ public static void initializeDataDirectory(String rootDir) throws Exception {
     (new File(rootDir)).deleteOnExit();
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() throws Exception {
     FileUtils.deleteDirectory(new File(ROOT_DIR));
   }
 
-  @Before
+  @BeforeEach
   public void init() throws Exception {
     reader = new FileSystemTimelineReaderImpl();
     Configuration conf = new YarnConfiguration();
@@ -313,141 +316,141 @@ private static File getAppDir(String rootDir, String cluster, String user,
   }
 
   @Test
-  public void testGetEntityDefaultView() throws Exception {
+  void testGetEntityDefaultView() throws Exception {
     // If no fields are specified, entity is returned with default view i.e.
     // only the id, type and created time.
     TimelineEntity result = reader.getEntity(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", "id_1"),
+            "app", "id_1"),
         new TimelineDataToRetrieve(null, null, null, null, null, null));
-    Assert.assertEquals(
+    assertEquals(
         (new TimelineEntity.Identifier("app", "id_1")).toString(),
         result.getIdentifier().toString());
-    Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
-    Assert.assertEquals(0, result.getConfigs().size());
-    Assert.assertEquals(0, result.getMetrics().size());
+    assertEquals((Long) 1425016502000L, result.getCreatedTime());
+    assertEquals(0, result.getConfigs().size());
+    assertEquals(0, result.getMetrics().size());
   }
 
   @Test
-  public void testGetEntityByClusterAndApp() throws Exception {
+  void testGetEntityByClusterAndApp() throws Exception {
     // Cluster and AppId should be enough to get an entity.
     TimelineEntity result = reader.getEntity(
         new TimelineReaderContext("cluster1", null, null, null, "app1", "app",
-        "id_1"),
+            "id_1"),
         new TimelineDataToRetrieve(null, null, null, null, null, null));
-    Assert.assertEquals(
+    assertEquals(
         (new TimelineEntity.Identifier("app", "id_1")).toString(),
         result.getIdentifier().toString());
-    Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
-    Assert.assertEquals(0, result.getConfigs().size());
-    Assert.assertEquals(0, result.getMetrics().size());
+    assertEquals((Long) 1425016502000L, result.getCreatedTime());
+    assertEquals(0, result.getConfigs().size());
+    assertEquals(0, result.getMetrics().size());
   }
 
   /** This test checks whether we can handle commas in app flow mapping csv. */
   @Test
-  public void testAppFlowMappingCsv() throws Exception {
+  void testAppFlowMappingCsv() throws Exception {
     // Test getting an entity by cluster and app where flow entry
     // in app flow mapping csv has commas.
     TimelineEntity result = reader.getEntity(
         new TimelineReaderContext("cluster1", null, null, null, "app2",
-        "app", "id_5"),
+            "app", "id_5"),
         new TimelineDataToRetrieve(null, null, null, null, null, null));
-    Assert.assertEquals(
+    assertEquals(
         (new TimelineEntity.Identifier("app", "id_5")).toString(),
         result.getIdentifier().toString());
-    Assert.assertEquals((Long)1425016502050L, result.getCreatedTime());
+    assertEquals((Long) 1425016502050L, result.getCreatedTime());
   }
 
   @Test
-  public void testGetEntityCustomFields() throws Exception {
+  void testGetEntityCustomFields() throws Exception {
     // Specified fields in addition to default view will be returned.
     TimelineEntity result = reader.getEntity(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", "id_1"),
+            "app", "id_1"),
         new TimelineDataToRetrieve(null, null,
-        EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null, null,
-        null));
-    Assert.assertEquals(
+            EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null, null,
+            null));
+    assertEquals(
         (new TimelineEntity.Identifier("app", "id_1")).toString(),
         result.getIdentifier().toString());
-    Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
-    Assert.assertEquals(3, result.getConfigs().size());
-    Assert.assertEquals(3, result.getMetrics().size());
-    Assert.assertEquals(2, result.getInfo().size());
+    assertEquals((Long) 1425016502000L, result.getCreatedTime());
+    assertEquals(3, result.getConfigs().size());
+    assertEquals(3, result.getMetrics().size());
+    assertEquals(2, result.getInfo().size());
     // No events will be returned
-    Assert.assertEquals(0, result.getEvents().size());
+    assertEquals(0, result.getEvents().size());
   }
 
   @Test
-  public void testGetEntityAllFields() throws Exception {
+  void testGetEntityAllFields() throws Exception {
     // All fields of TimelineEntity will be returned.
     TimelineEntity result = reader.getEntity(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", "id_1"),
+            "app", "id_1"),
         new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
-        null, null));
-    Assert.assertEquals(
+            null, null));
+    assertEquals(
         (new TimelineEntity.Identifier("app", "id_1")).toString(),
         result.getIdentifier().toString());
-    Assert.assertEquals((Long)1425016502000L, result.getCreatedTime());
-    Assert.assertEquals(3, result.getConfigs().size());
-    Assert.assertEquals(3, result.getMetrics().size());
+    assertEquals((Long) 1425016502000L, result.getCreatedTime());
+    assertEquals(3, result.getConfigs().size());
+    assertEquals(3, result.getMetrics().size());
     // All fields including events will be returned.
-    Assert.assertEquals(2, result.getEvents().size());
+    assertEquals(2, result.getEvents().size());
   }
 
   @Test
-  public void testGetAllEntities() throws Exception {
+  void testGetAllEntities() throws Exception {
     Set<TimelineEntity> result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null), new TimelineEntityFilters.Builder().build(),
+            "app", null), new TimelineEntityFilters.Builder().build(),
         new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null,
-        null, null));
+            null, null));
     // All 4 entities will be returned
-    Assert.assertEquals(4, result.size());
+    assertEquals(4, result.size());
   }
 
   @Test
-  public void testGetEntitiesWithLimit() throws Exception {
+  void testGetEntitiesWithLimit() throws Exception {
     Set<TimelineEntity> result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().entityLimit(2L).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     // Needs to be rewritten once hashcode and equals for
     // TimelineEntity is implemented
     // Entities with id_1 and id_4 should be returned,
     // based on created time, descending.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
-        Assert.fail("Entity not sorted by created time");
+        fail("Entity not sorted by created time");
       }
     }
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().entityLimit(3L).build(),
         new TimelineDataToRetrieve());
     // Even though 2 entities out of 4 have same created time, one entity
     // is left out due to limit
-    Assert.assertEquals(3, result.size());
+    assertEquals(3, result.size());
   }
 
   @Test
-  public void testGetEntitiesByTimeWindows() throws Exception {
+  void testGetEntitiesByTimeWindows() throws Exception {
     // Get entities based on created time start and end time range.
     Set<TimelineEntity> result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().createdTimeBegin(1425016502030L)
             .createTimeEnd(1425016502060L).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     // Only one entity with ID id_4 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_4")) {
-        Assert.fail("Incorrect filtering based on created time range");
+        fail("Incorrect filtering based on created time range");
       }
     }
 
@@ -458,44 +461,44 @@ public void testGetEntitiesByTimeWindows() throws Exception {
         new TimelineEntityFilters.Builder().createTimeEnd(1425016502010L)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(3, result.size());
+    assertEquals(3, result.size());
     for (TimelineEntity entity : result) {
       if (entity.getId().equals("id_4")) {
-        Assert.fail("Incorrect filtering based on created time range");
+        fail("Incorrect filtering based on created time range");
       }
     }
 
     // Get entities if only created time start is specified.
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().createdTimeBegin(1425016502010L)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_4")) {
-        Assert.fail("Incorrect filtering based on created time range");
+        fail("Incorrect filtering based on created time range");
       }
     }
   }
 
   @Test
-  public void testGetFilteredEntities() throws Exception {
+  void testGetFilteredEntities() throws Exception {
     // Get entities based on info filters.
     TimelineFilterList infoFilterList = new TimelineFilterList();
     infoFilterList.addFilter(
         new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
     Set<TimelineEntity> result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().infoFilters(infoFilterList).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     // Only one entity with ID id_3 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on info filters");
+        fail("Incorrect filtering based on info filters");
       }
     }
 
@@ -507,14 +510,14 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "abc"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on config filters");
+        fail("Incorrect filtering based on config filters");
       }
     }
 
@@ -526,13 +529,13 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineExistsFilter(TimelineCompareOp.EQUAL, "event_4"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().eventFilters(eventFilters).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on event filters");
+        fail("Incorrect filtering based on event filters");
       }
     }
 
@@ -542,15 +545,15 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.GREATER_OR_EQUAL, "metric3", 0L));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     // Two entities with IDs' id_1 and id_2 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on metric filters");
+        fail("Incorrect filtering based on metric filters");
       }
     }
 
@@ -569,14 +572,14 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineFilterList(Operator.OR, list1, list2);
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList1)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on config filters");
+        fail("Incorrect filtering based on config filters");
       }
     }
 
@@ -592,14 +595,14 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineFilterList(Operator.OR, list3, list4);
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList2)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on config filters");
+        fail("Incorrect filtering based on config filters");
       }
     }
 
@@ -610,14 +613,14 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.NOT_EQUAL, "config_3", "abc"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList3)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
-    for(TimelineEntity entity : result) {
+    assertEquals(1, result.size());
+    for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on config filters");
+        fail("Incorrect filtering based on config filters");
       }
     }
 
@@ -628,11 +631,11 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.EQUAL, "config_3", "def"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList4)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(0, result.size());
+    assertEquals(0, result.size());
 
     TimelineFilterList confFilterList5 = new TimelineFilterList(Operator.OR);
     confFilterList5.addFilter(new TimelineKeyValueFilter(
@@ -641,14 +644,14 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.EQUAL, "config_3", "def"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().configFilters(confFilterList5)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on config filters");
+        fail("Incorrect filtering based on config filters");
       }
     }
 
@@ -665,15 +668,15 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineFilterList(Operator.OR, list6, list7);
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList1)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     // Two entities with IDs' id_2 and id_3 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_2") && !entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on metric filters");
+        fail("Incorrect filtering based on metric filters");
       }
     }
 
@@ -684,14 +687,14 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList2)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1")) {
-        Assert.fail("Incorrect filtering based on metric filters");
+        fail("Incorrect filtering based on metric filters");
       }
     }
 
@@ -702,11 +705,11 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList3)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(0, result.size());
+    assertEquals(0, result.size());
 
     TimelineFilterList metricFilterList4 = new TimelineFilterList(Operator.OR);
     metricFilterList4.addFilter(new TimelineCompareFilter(
@@ -715,14 +718,14 @@ public void testGetFilteredEntities() throws Exception {
         TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList4)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on metric filters");
+        fail("Incorrect filtering based on metric filters");
       }
     }
 
@@ -731,14 +734,14 @@ public void testGetFilteredEntities() throws Exception {
             TimelineCompareOp.NOT_EQUAL, "metric2", 74));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().metricFilters(metricFilterList5)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
-        Assert.fail("Incorrect filtering based on metric filters");
+        fail("Incorrect filtering based on metric filters");
       }
     }
 
@@ -749,11 +752,11 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "info4", 20));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().infoFilters(infoFilterList1)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(0, result.size());
+    assertEquals(0, result.size());
 
     TimelineFilterList infoFilterList2 = new TimelineFilterList(Operator.OR);
     infoFilterList2.addFilter(
@@ -762,14 +765,14 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info1", "val1"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().infoFilters(infoFilterList2)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on info filters");
+        fail("Incorrect filtering based on info filters");
       }
     }
 
@@ -780,11 +783,11 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().infoFilters(infoFilterList3)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(0, result.size());
+    assertEquals(0, result.size());
 
     TimelineFilterList infoFilterList4 = new TimelineFilterList(Operator.OR);
     infoFilterList4.addFilter(
@@ -793,55 +796,55 @@ public void testGetFilteredEntities() throws Exception {
         new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().infoFilters(infoFilterList4)
             .build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1")) {
-        Assert.fail("Incorrect filtering based on info filters");
+        fail("Incorrect filtering based on info filters");
       }
     }
   }
 
   @Test
-  public void testGetEntitiesByRelations() throws Exception {
+  void testGetEntitiesByRelations() throws Exception {
     // Get entities based on relatesTo.
     TimelineFilterList relatesTo = new TimelineFilterList(Operator.OR);
     Set<Object> relatesToIds =
-        new HashSet<Object>(Arrays.asList((Object)"flow1"));
+        new HashSet<Object>(Arrays.asList((Object) "flow1"));
     relatesTo.addFilter(new TimelineKeyValuesFilter(
         TimelineCompareOp.EQUAL, "flow", relatesToIds));
     Set<TimelineEntity> result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().relatesTo(relatesTo).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(1, result.size());
+    assertEquals(1, result.size());
     // Only one entity with ID id_1 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1")) {
-        Assert.fail("Incorrect filtering based on relatesTo");
+        fail("Incorrect filtering based on relatesTo");
       }
     }
 
     // Get entities based on isRelatedTo.
     TimelineFilterList isRelatedTo = new TimelineFilterList(Operator.OR);
     Set<Object> isRelatedToIds =
-        new HashSet<Object>(Arrays.asList((Object)"tid1_2"));
+        new HashSet<Object>(Arrays.asList((Object) "tid1_2"));
     isRelatedTo.addFilter(new TimelineKeyValuesFilter(
         TimelineCompareOp.EQUAL, "type1", isRelatedToIds));
     result = reader.getEntities(
         new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
-        "app", null),
+            "app", null),
         new TimelineEntityFilters.Builder().isRelatedTo(isRelatedTo).build(),
         new TimelineDataToRetrieve());
-    Assert.assertEquals(2, result.size());
+    assertEquals(2, result.size());
     // Two entities with IDs' id_1 and id_3 should be returned.
     for (TimelineEntity entity : result) {
       if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
-        Assert.fail("Incorrect filtering based on isRelatedTo");
+        fail("Incorrect filtering based on isRelatedTo");
       }
     }
   }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index b880b9a64826d..efed104eeea76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -17,8 +17,6 @@
  */
 
 package org.apache.hadoop.yarn.server.timelineservice.storage;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -29,6 +27,9 @@
 import java.util.List;
 import java.util.Map;
 
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -41,13 +42,14 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestFileSystemTimelineWriterImpl {
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
+  @TempDir
+  private File tmpFolder;
 
   /**
    * Unit test for PoC YARN 3264.
@@ -55,7 +57,7 @@ public class TestFileSystemTimelineWriterImpl {
    * @throws Exception
    */
   @Test
-  public void testWriteEntityToFile() throws Exception {
+  void testWriteEntityToFile() throws Exception {
     TimelineEntities te = new TimelineEntities();
     TimelineEntity entity = new TimelineEntity();
     String id = "hello";
@@ -89,7 +91,7 @@ public void testWriteEntityToFile() throws Exception {
     try {
       fsi = new FileSystemTimelineWriterImpl();
       Configuration conf = new YarnConfiguration();
-      String outputRoot = tmpFolder.newFolder().getAbsolutePath();
+      String outputRoot = tmpFolder.getAbsolutePath();
       conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
           outputRoot);
       fsi.init(conf);
@@ -107,14 +109,13 @@ public void testWriteEntityToFile() throws Exception {
           FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
       Path path = new Path(fileName);
       FileSystem fs = FileSystem.get(conf);
-      assertTrue("Specified path(" + fileName + ") should exist: ",
-              fs.exists(path));
+      assertTrue(fs.exists(path),
+          "Specified path(" + fileName + ") should exist: ");
       FileStatus fileStatus = fs.getFileStatus(path);
-      assertTrue("Specified path should be a file",
-              !fileStatus.isDirectory());
+      assertFalse(fileStatus.isDirectory(), "Specified path should be a file");
       List<String> data = readFromFile(fs, path);
       // ensure there's only one entity + 1 new line
-      assertTrue("data size is:" + data.size(), data.size() == 2);
+      assertEquals(2, data.size(), "data size is:" + data.size());
       String d = data.get(0);
       // confirm the contents same as what was written
       assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
@@ -127,14 +128,13 @@ public void testWriteEntityToFile() throws Exception {
           File.separator + type2 + File.separator + id2 +
           FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
       Path path2 = new Path(fileName2);
-      assertTrue("Specified path(" + fileName + ") should exist: ",
-              fs.exists(path2));
+      assertTrue(fs.exists(path2),
+          "Specified path(" + fileName + ") should exist: ");
       FileStatus fileStatus2 = fs.getFileStatus(path2);
-      assertTrue("Specified path should be a file",
-              !fileStatus2.isDirectory());
+      assertFalse(fileStatus2.isDirectory(), "Specified path should be a file");
       List<String> data2 = readFromFile(fs, path2);
       // ensure there's only one entity + 1 new line
-      assertTrue("data size is:" + data2.size(), data2.size() == 2);
+      assertEquals(2, data2.size(), "data size is:" + data2.size());
       String metricToString = data2.get(0);
       // confirm the contents same as what was written
       assertEquals(metricToString,
@@ -147,7 +147,7 @@ public void testWriteEntityToFile() throws Exception {
   }
 
   @Test
-  public void testWriteMultipleEntities() throws Exception {
+  void testWriteMultipleEntities() throws Exception {
     String id = "appId";
     String type = "app";
 
@@ -169,7 +169,7 @@ public void testWriteMultipleEntities() throws Exception {
     try {
       fsi = new FileSystemTimelineWriterImpl();
       Configuration conf = new YarnConfiguration();
-      String outputRoot = tmpFolder.newFolder().getAbsolutePath();
+      String outputRoot = tmpFolder.getAbsolutePath();
       conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
           outputRoot);
       fsi.init(conf);
@@ -191,13 +191,12 @@ public void testWriteMultipleEntities() throws Exception {
           FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
       Path path = new Path(fileName);
       FileSystem fs = FileSystem.get(conf);
-      assertTrue("Specified path(" + fileName + ") should exist: ",
-          fs.exists(path));
+      assertTrue(fs.exists(path),
+          "Specified path(" + fileName + ") should exist: ");
       FileStatus fileStatus = fs.getFileStatus(path);
-      assertTrue("Specified path should be a file",
-          !fileStatus.isDirectory());
+      assertFalse(fileStatus.isDirectory(), "Specified path should be a file");
       List<String> data = readFromFile(fs, path);
-      assertTrue("data size is:" + data.size(), data.size() == 3);
+      assertEquals(3, data.size(), "data size is:" + data.size());
       String d = data.get(0);
       // confirm the contents same as what was written
       assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
@@ -215,7 +214,7 @@ public void testWriteMultipleEntities() throws Exception {
   }
 
   @Test
-  public void testWriteEntitiesWithEmptyFlowName() throws Exception {
+  void testWriteEntitiesWithEmptyFlowName() throws Exception {
     String id = "appId";
     String type = "app";
 
@@ -230,7 +229,7 @@ public void testWriteEntitiesWithEmptyFlowName() throws Exception {
     try {
       fsi = new FileSystemTimelineWriterImpl();
       Configuration conf = new YarnConfiguration();
-      String outputRoot = tmpFolder.newFolder().getAbsolutePath();
+      String outputRoot = tmpFolder.getAbsolutePath();
       conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
           outputRoot);
       fsi.init(conf);
@@ -248,13 +247,12 @@ public void testWriteEntitiesWithEmptyFlowName() throws Exception {
           FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
       Path path = new Path(fileName);
       FileSystem fs = FileSystem.get(conf);
-      assertTrue("Specified path(" + fileName + ") should exist: ",
-          fs.exists(path));
+      assertTrue(fs.exists(path),
+          "Specified path(" + fileName + ") should exist: ");
       FileStatus fileStatus = fs.getFileStatus(path);
-      assertTrue("Specified path should be a file",
-          !fileStatus.isDirectory());
+      assertFalse(fileStatus.isDirectory(), "specified path should be a file");
       List<String> data = readFromFile(fs, path);
-      assertTrue("data size is:" + data.size(), data.size() == 2);
+      assertEquals(2, data.size(), "data size is:" + data.size());
       String d = data.get(0);
       // confirm the contents same as what was written
       assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
@@ -278,4 +276,5 @@ private List<String> readFromFile(FileSystem fs, Path path)
     }
     return data;
   }
+
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java
index 16b6d995d4fe8..0201496674623 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineSchemaCreator.java
@@ -18,10 +18,12 @@
 
 package org.apache.hadoop.yarn.server.timelineservice.storage;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * Test cases for {@link TimelineSchemaCreator}.
@@ -29,13 +31,13 @@
 public class TestTimelineSchemaCreator {
 
   @Test
-  public void testTimelineSchemaCreation() throws Exception {
+  void testTimelineSchemaCreation() throws Exception {
     Configuration conf = new Configuration();
     conf.set(YarnConfiguration.TIMELINE_SERVICE_SCHEMA_CREATOR_CLASS,
         "org.apache.hadoop.yarn.server.timelineservice.storage" +
             ".DummyTimelineSchemaCreator");
     TimelineSchemaCreator timelineSchemaCreator = new TimelineSchemaCreator();
-    Assert.assertEquals(0, timelineSchemaCreator
+    assertEquals(0, timelineSchemaCreator
         .createTimelineSchema(new String[]{}, conf));
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index 1e0e9ae57ad8d..9681dfa4c1df5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -69,11 +69,6 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.assertj</groupId>
       <artifactId>assertj-core</artifactId>
@@ -109,6 +104,21 @@
       <artifactId>grizzly-http-servlet</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
@@ -124,7 +134,6 @@
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcpkix-jdk15on</artifactId>
     </dependency>
-
   </dependencies>
 
   <build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java
index bcab33fd66d75..cc7542f3e596b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java
@@ -20,20 +20,22 @@
 
 import java.io.IOException;
 
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestAppReportFetcher {
 
@@ -43,7 +45,7 @@ public class TestAppReportFetcher {
   private static AppReportFetcher fetcher;
   private final String appNotFoundExceptionMsg = "APP NOT FOUND";
 
-  @After
+  @AfterEach
   public void cleanUp() {
     historyManager = null;
     appManager = null;
@@ -64,29 +66,29 @@ public void testHelper(boolean isAHSEnabled)
   }
 
   @Test
-  public void testFetchReportAHSEnabled() throws YarnException, IOException {
+  void testFetchReportAHSEnabled() throws YarnException, IOException {
     testHelper(true);
     Mockito.verify(historyManager, Mockito.times(1))
-    .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
+        .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
     Mockito.verify(appManager, Mockito.times(1))
-    .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
+        .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
   }
 
   @Test
-  public void testFetchReportAHSDisabled() throws YarnException, IOException {
+  void testFetchReportAHSDisabled() throws YarnException, IOException {
     try {
       testHelper(false);
     } catch (ApplicationNotFoundException e) {
-      Assert.assertTrue(e.getMessage() == appNotFoundExceptionMsg);
+      assertEquals(appNotFoundExceptionMsg, e.getMessage());
       /* RM will not know of the app and Application History Service is disabled
        * So we will not try to get the report from AHS and RM will throw
        * ApplicationNotFoundException
        */
     }
     Mockito.verify(appManager, Mockito.times(1))
-    .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
+        .getApplicationReport(Mockito.any(GetApplicationReportRequest.class));
     if (historyManager != null) {
-      Assert.fail("HistoryManager should be null as AHS is disabled");
+      fail("HistoryManager should be null as AHS is disabled");
     }
   }
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java
index af97396a19de2..0ef5e690811e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java
@@ -18,20 +18,6 @@
 
 package org.apache.hadoop.yarn.server.webproxy;
 
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.X509KeyManager;
-import javax.net.ssl.X509TrustManager;
-import javax.security.auth.x500.X500Principal;
 import java.security.InvalidKeyException;
 import java.security.Key;
 import java.security.KeyPair;
@@ -52,109 +38,132 @@
 import java.util.HashSet;
 import java.util.Random;
 import java.util.Set;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLPeerUnverifiedException;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.X509KeyManager;
+import javax.net.ssl.X509TrustManager;
+import javax.security.auth.x500.X500Principal;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class TestProxyCA {
 
   @Test
-  public void testInit() throws Exception {
+  void testInit() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
-    Assert.assertNull(proxyCA.getCaCert());
-    Assert.assertNull(proxyCA.getCaKeyPair());
-    Assert.assertNull(proxyCA.getX509KeyManager());
-    Assert.assertNull(proxyCA.getHostnameVerifier());
+    assertNull(proxyCA.getCaCert());
+    assertNull(proxyCA.getCaKeyPair());
+    assertNull(proxyCA.getX509KeyManager());
+    assertNull(proxyCA.getHostnameVerifier());
 
     proxyCA.init();
-    Assert.assertNotNull(proxyCA.getCaCert());
-    Assert.assertNotNull(proxyCA.getCaKeyPair());
-    Assert.assertNotNull(proxyCA.getX509KeyManager());
-    Assert.assertNotNull(proxyCA.getHostnameVerifier());
+    assertNotNull(proxyCA.getCaCert());
+    assertNotNull(proxyCA.getCaKeyPair());
+    assertNotNull(proxyCA.getX509KeyManager());
+    assertNotNull(proxyCA.getHostnameVerifier());
   }
 
   @Test
-  public void testInit2Null() throws Exception {
+  void testInit2Null() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
-    Assert.assertNull(proxyCA.getCaCert());
-    Assert.assertNull(proxyCA.getCaKeyPair());
-    Assert.assertNull(proxyCA.getX509KeyManager());
-    Assert.assertNull(proxyCA.getHostnameVerifier());
+    assertNull(proxyCA.getCaCert());
+    assertNull(proxyCA.getCaKeyPair());
+    assertNull(proxyCA.getX509KeyManager());
+    assertNull(proxyCA.getHostnameVerifier());
 
     // null certificate and private key
     proxyCA.init(null, null);
-    Assert.assertNotNull(proxyCA.getCaCert());
-    Assert.assertNotNull(proxyCA.getCaKeyPair());
-    Assert.assertNotNull(proxyCA.getX509KeyManager());
-    Assert.assertNotNull(proxyCA.getHostnameVerifier());
+    assertNotNull(proxyCA.getCaCert());
+    assertNotNull(proxyCA.getCaKeyPair());
+    assertNotNull(proxyCA.getX509KeyManager());
+    assertNotNull(proxyCA.getHostnameVerifier());
   }
 
   @Test
-  public void testInit2Mismatch() throws Exception {
+  void testInit2Mismatch() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
-    Assert.assertNull(proxyCA.getCaCert());
-    Assert.assertNull(proxyCA.getCaKeyPair());
-    Assert.assertNull(proxyCA.getX509KeyManager());
-    Assert.assertNull(proxyCA.getHostnameVerifier());
+    assertNull(proxyCA.getCaCert());
+    assertNull(proxyCA.getCaKeyPair());
+    assertNull(proxyCA.getX509KeyManager());
+    assertNull(proxyCA.getHostnameVerifier());
 
     // certificate and private key don't match
     CertKeyPair pair1 = createCertAndKeyPair();
     CertKeyPair pair2 = createCertAndKeyPair();
-    Assert.assertNotEquals(pair1.getCert(), pair2.getCert());
-    Assert.assertNotEquals(pair1.getKeyPair().getPrivate(),
+    assertNotEquals(pair1.getCert(), pair2.getCert());
+    assertNotEquals(pair1.getKeyPair().getPrivate(),
         pair2.getKeyPair().getPrivate());
-    Assert.assertNotEquals(pair1.getKeyPair().getPublic(),
+    assertNotEquals(pair1.getKeyPair().getPublic(),
         pair2.getKeyPair().getPublic());
     proxyCA.init(pair1.getCert(), pair2.getKeyPair().getPrivate());
-    Assert.assertNotNull(proxyCA.getCaCert());
-    Assert.assertNotNull(proxyCA.getCaKeyPair());
-    Assert.assertNotNull(proxyCA.getX509KeyManager());
-    Assert.assertNotNull(proxyCA.getHostnameVerifier());
-    Assert.assertNotEquals(proxyCA.getCaCert(), pair1.getCert());
-    Assert.assertNotEquals(proxyCA.getCaKeyPair().getPrivate(),
+    assertNotNull(proxyCA.getCaCert());
+    assertNotNull(proxyCA.getCaKeyPair());
+    assertNotNull(proxyCA.getX509KeyManager());
+    assertNotNull(proxyCA.getHostnameVerifier());
+    assertNotEquals(proxyCA.getCaCert(), pair1.getCert());
+    assertNotEquals(proxyCA.getCaKeyPair().getPrivate(),
         pair2.getKeyPair().getPrivate());
-    Assert.assertNotEquals(proxyCA.getCaKeyPair().getPublic(),
+    assertNotEquals(proxyCA.getCaKeyPair().getPublic(),
         pair2.getKeyPair().getPublic());
   }
 
   @Test
-  public void testInit2Invalid() throws Exception {
+  void testInit2Invalid() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
-    Assert.assertNull(proxyCA.getCaCert());
-    Assert.assertNull(proxyCA.getCaKeyPair());
-    Assert.assertNull(proxyCA.getX509KeyManager());
-    Assert.assertNull(proxyCA.getHostnameVerifier());
+    assertNull(proxyCA.getCaCert());
+    assertNull(proxyCA.getCaKeyPair());
+    assertNull(proxyCA.getX509KeyManager());
+    assertNull(proxyCA.getHostnameVerifier());
 
     // Invalid key - fail the verification
     X509Certificate certificate = Mockito.mock(X509Certificate.class);
     PrivateKey privateKey = Mockito.mock(PrivateKey.class);
     try {
       proxyCA.init(certificate, privateKey);
-      Assert.fail("Expected InvalidKeyException");
+      fail("Expected InvalidKeyException");
     } catch (InvalidKeyException e) {
       // expected
     }
   }
 
   @Test
-  public void testInit2() throws Exception {
+  void testInit2() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
-    Assert.assertNull(proxyCA.getCaCert());
-    Assert.assertNull(proxyCA.getCaKeyPair());
-    Assert.assertNull(proxyCA.getX509KeyManager());
-    Assert.assertNull(proxyCA.getHostnameVerifier());
+    assertNull(proxyCA.getCaCert());
+    assertNull(proxyCA.getCaKeyPair());
+    assertNull(proxyCA.getX509KeyManager());
+    assertNull(proxyCA.getHostnameVerifier());
 
     // certificate and private key do match
     CertKeyPair pair = createCertAndKeyPair();
     proxyCA.init(pair.getCert(), pair.getKeyPair().getPrivate());
-    Assert.assertEquals(pair.getCert(), proxyCA.getCaCert());
-    Assert.assertEquals(pair.getKeyPair().getPrivate(),
+    assertEquals(pair.getCert(), proxyCA.getCaCert());
+    assertEquals(pair.getKeyPair().getPrivate(),
         proxyCA.getCaKeyPair().getPrivate());
-    Assert.assertEquals(pair.getKeyPair().getPublic(),
+    assertEquals(pair.getKeyPair().getPublic(),
         proxyCA.getCaKeyPair().getPublic());
-    Assert.assertNotNull(proxyCA.getX509KeyManager());
-    Assert.assertNotNull(proxyCA.getHostnameVerifier());
+    assertNotNull(proxyCA.getX509KeyManager());
+    assertNotNull(proxyCA.getHostnameVerifier());
   }
 
   @Test
-  public void testCreateChildKeyStore() throws Exception {
+  void testCreateChildKeyStore() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     ApplicationId appId =
@@ -163,29 +172,29 @@ public void testCreateChildKeyStore() throws Exception {
         "password");
     KeyStore keyStore = KeyStoreTestUtil.bytesToKeyStore(keystoreBytes,
         "password");
-    Assert.assertEquals(1, keyStore.size());
+    assertEquals(1, keyStore.size());
     Certificate[] certChain = keyStore.getCertificateChain("server");
-    Assert.assertEquals(2, certChain.length);
+    assertEquals(2, certChain.length);
     X509Certificate caCert = (X509Certificate) certChain[1];
     X509Certificate cert = (X509Certificate) certChain[0];
 
     // check child cert
-    Assert.assertEquals(caCert.getSubjectX500Principal().toString(),
+    assertEquals(caCert.getSubjectX500Principal().toString(),
         cert.getIssuerDN().toString());
-    Assert.assertEquals(new X500Principal("CN=" + appId),
+    assertEquals(new X500Principal("CN=" + appId),
         cert.getSubjectX500Principal());
-    Assert.assertFalse("Found multiple fields in X500 Principal, when there " +
-            "should have only been one: " + cert.getSubjectX500Principal(),
-        cert.getSubjectX500Principal().toString().contains(","));
-    Assert.assertEquals("SHA512withRSA", cert.getSigAlgName());
-    Assert.assertEquals(cert.getNotBefore(), cert.getNotAfter());
-    Assert.assertTrue("Expected certificate to be expired but was not: "
-        + cert.getNotAfter(), cert.getNotAfter().before(new Date()));
-    Assert.assertEquals(new X500Principal("CN=" + appId).toString(),
+    assertFalse(cert.getSubjectX500Principal().toString().contains(","),
+        "Found multiple fields in X500 Principal, when there " +
+            "should have only been one: " + cert.getSubjectX500Principal());
+    assertEquals("SHA512withRSA", cert.getSigAlgName());
+    assertEquals(cert.getNotBefore(), cert.getNotAfter());
+    assertTrue(cert.getNotAfter().before(new Date()),
+        "Expected certificate to be expired but was not: " + cert.getNotAfter());
+    assertEquals(new X500Principal("CN=" + appId).toString(),
         cert.getSubjectDN().toString());
     Key privateKey = keyStore.getKey("server", "password".toCharArray());
-    Assert.assertEquals("RSA", privateKey.getAlgorithm());
-    Assert.assertEquals(-1, cert.getBasicConstraints());
+    assertEquals("RSA", privateKey.getAlgorithm());
+    assertEquals(-1, cert.getBasicConstraints());
 
     // verify signature on child cert
     PublicKey caPublicKey = caCert.getPublicKey();
@@ -193,7 +202,7 @@ public void testCreateChildKeyStore() throws Exception {
 
     // check CA cert
     checkCACert(caCert);
-    Assert.assertEquals(proxyCA.getCaCert(), caCert);
+    assertEquals(proxyCA.getCaCert(), caCert);
 
     // verify signature on CA cert
     caCert.verify(caPublicKey);
@@ -202,24 +211,24 @@ public void testCreateChildKeyStore() throws Exception {
     PrivateKey caPrivateKey =
         proxyCA.getX509KeyManager().getPrivateKey(null);
     checkPrivatePublicKeys(caPrivateKey, caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
+    assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
+    assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
   }
 
   @Test
-  public void testGetChildTrustStore() throws Exception {
+  void testGetChildTrustStore() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     byte[] truststoreBytes = proxyCA.getChildTrustStore("password");
     KeyStore truststore = KeyStoreTestUtil.bytesToKeyStore(truststoreBytes,
         "password");
-    Assert.assertEquals(1, truststore.size());
+    assertEquals(1, truststore.size());
     X509Certificate caCert =
         (X509Certificate) truststore.getCertificate("client");
 
     // check CA cert
     checkCACert(caCert);
-    Assert.assertEquals(proxyCA.getCaCert(), caCert);
+    assertEquals(proxyCA.getCaCert(), caCert);
 
     // verify signature on CA cert
     PublicKey caPublicKey = caCert.getPublicKey();
@@ -229,12 +238,12 @@ public void testGetChildTrustStore() throws Exception {
     PrivateKey caPrivateKey =
         proxyCA.getX509KeyManager().getPrivateKey(null);
     checkPrivatePublicKeys(caPrivateKey, caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
+    assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
+    assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
   }
 
   @Test
-  public void testGenerateKeyStorePassword() throws Exception {
+  void testGenerateKeyStorePassword() throws Exception {
     // We can't possibly test every possible string, but we can at least verify
     // a few things about a few of the generated strings as a sanity check
     ProxyCA proxyCA = new ProxyCA();
@@ -243,23 +252,23 @@ public void testGenerateKeyStorePassword() throws Exception {
 
     for (int i = 0; i < 5; i++) {
       String password = proxyCA.generateKeyStorePassword();
-      Assert.assertEquals(16, password.length());
+      assertEquals(16, password.length());
       for (char c : password.toCharArray()) {
-        Assert.assertFalse("Found character '" + c + "' in password '"
-            + password + "' which is outside of the expected range", c < ' ');
-        Assert.assertFalse("Found character '" + c + "' in password '"
-            + password + "' which is outside of the expected range", c > 'z');
+        assertFalse(c < ' ', "Found character '" + c + "' in password '"
+            + password + "' which is outside of the expected range");
+        assertFalse(c > 'z', "Found character '" + c + "' in password '"
+            + password + "' which is outside of the expected range");
       }
-      Assert.assertFalse("Password " + password
-          + " was generated twice, which is _extremely_ unlikely"
-          + " and shouldn't practically happen: " + passwords,
-          passwords.contains(password));
+      assertFalse(passwords.contains(password),
+          "Password " + password
+              + " was generated twice, which is _extremely_ unlikely"
+              + " and shouldn't practically happen: " + passwords);
       passwords.add(password);
     }
   }
 
   @Test
-  public void testCreateTrustManagerDefaultTrustManager() throws Exception {
+  void testCreateTrustManagerDefaultTrustManager() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -272,13 +281,13 @@ public void testCreateTrustManagerDefaultTrustManager() throws Exception {
             "CN=foo", KeyStoreTestUtil.generateKeyPair("RSA"), 30,
             "SHA1withRSA")});
 
-    Assert.assertArrayEquals(defaultTrustManager.getAcceptedIssuers(),
+    assertArrayEquals(defaultTrustManager.getAcceptedIssuers(),
         trustManager.getAcceptedIssuers());
     trustManager.checkClientTrusted(null, null);
   }
 
   @Test
-  public void testCreateTrustManagerYarnCert() throws Exception {
+  void testCreateTrustManagerYarnCert() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -297,7 +306,7 @@ public void testCreateTrustManagerYarnCert() throws Exception {
   }
 
   @Test
-  public void testCreateTrustManagerWrongApp() throws Exception {
+  void testCreateTrustManagerWrongApp() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -314,15 +323,15 @@ public void testCreateTrustManagerWrongApp() throws Exception {
             .getCertificateChain("server"));
     try {
       trustManager.checkServerTrusted(certChain, "RSA");
-      Assert.fail("Should have thrown a CertificateException, but did not");
+      fail("Should have thrown a CertificateException, but did not");
     } catch (CertificateException ce) {
-      Assert.assertEquals("Expected to find Subject X500 Principal with CN=" +
+      assertEquals("Expected to find Subject X500 Principal with CN=" +
           appId + " but found CN=" + appId2, ce.getMessage());
     }
   }
 
   @Test
-  public void testCreateTrustManagerWrongRM() throws Exception {
+  void testCreateTrustManagerWrongRM() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -345,7 +354,7 @@ public void testCreateTrustManagerWrongRM() throws Exception {
   }
 
   @Test
-  public void testCreateTrustManagerRealCert() throws Exception {
+  void testCreateTrustManagerRealCert() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -357,8 +366,8 @@ public void testCreateTrustManagerRealCert() throws Exception {
     // "real" cert
     X509Certificate[]
         certChain = new X509Certificate[]{
-        KeyStoreTestUtil.generateCertificate("CN=foo.com",
-            KeyStoreTestUtil.generateKeyPair("RSA"), 30, "SHA1withRSA")};
+            KeyStoreTestUtil.generateCertificate("CN=foo.com",
+                KeyStoreTestUtil.generateKeyPair("RSA"), 30, "SHA1withRSA")};
     Mockito.verify(defaultTrustManager, Mockito.times(0))
         .checkServerTrusted(certChain, "RSA");
     trustManager.checkServerTrusted(certChain, "RSA");
@@ -379,7 +388,7 @@ public void testCreateTrustManagerRealCert() throws Exception {
   }
 
   @Test
-  public void testCreateTrustManagerExceptions() throws Exception {
+  void testCreateTrustManagerExceptions() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class);
@@ -409,37 +418,37 @@ public void testCreateTrustManagerExceptions() throws Exception {
   }
 
   @Test
-  public void testCreateKeyManager() throws Exception {
+  void testCreateKeyManager() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     X509KeyManager keyManager = proxyCA.getX509KeyManager();
 
-    Assert.assertArrayEquals(new String[]{"client"},
+    assertArrayEquals(new String[]{"client"},
         keyManager.getClientAliases(null, null));
-    Assert.assertEquals("client",
+    assertEquals("client",
         keyManager.chooseClientAlias(null, null, null));
-    Assert.assertNull(keyManager.getServerAliases(null, null));
-    Assert.assertNull(keyManager.chooseServerAlias(null, null, null));
+    assertNull(keyManager.getServerAliases(null, null));
+    assertNull(keyManager.chooseServerAlias(null, null, null));
 
     byte[] truststoreBytes = proxyCA.getChildTrustStore("password");
     KeyStore truststore = KeyStoreTestUtil.bytesToKeyStore(truststoreBytes,
         "password");
-    Assert.assertEquals(1, truststore.size());
+    assertEquals(1, truststore.size());
     X509Certificate caCert =
         (X509Certificate) truststore.getCertificate("client");
-    Assert.assertArrayEquals(new X509Certificate[]{caCert},
+    assertArrayEquals(new X509Certificate[]{caCert},
         keyManager.getCertificateChain(null));
-    Assert.assertEquals(proxyCA.getCaCert(), caCert);
+    assertEquals(proxyCA.getCaCert(), caCert);
 
     PrivateKey caPrivateKey = keyManager.getPrivateKey(null);
     PublicKey caPublicKey = caCert.getPublicKey();
     checkPrivatePublicKeys(caPrivateKey, caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
-    Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
+    assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey);
+    assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey);
   }
 
   @Test
-  public void testCreateHostnameVerifier() throws Exception {
+  void testCreateHostnameVerifier() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     HostnameVerifier verifier = proxyCA.getHostnameVerifier();
@@ -450,11 +459,11 @@ public void testCreateHostnameVerifier() throws Exception {
             proxyCA.createChildKeyStore(
                 ApplicationId.newInstance(System.currentTimeMillis(), 1),
                 "password"), "password").getCertificateChain("server"));
-    Assert.assertTrue(verifier.verify("foo", sslSession));
+    assertTrue(verifier.verify("foo", sslSession));
   }
 
   @Test
-  public void testCreateHostnameVerifierSSLPeerUnverifiedException()
+  void testCreateHostnameVerifierSSLPeerUnverifiedException()
       throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
@@ -463,11 +472,11 @@ public void testCreateHostnameVerifierSSLPeerUnverifiedException()
     SSLSession sslSession = Mockito.mock(SSLSession.class);
     Mockito.when(sslSession.getPeerCertificates()).thenThrow(
         new SSLPeerUnverifiedException(""));
-    Assert.assertFalse(verifier.verify("foo", sslSession));
+    assertFalse(verifier.verify("foo", sslSession));
   }
 
   @Test
-  public void testCreateHostnameVerifierWrongRM() throws Exception {
+  void testCreateHostnameVerifierWrongRM() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     HostnameVerifier verifier = proxyCA.getHostnameVerifier();
@@ -480,11 +489,11 @@ public void testCreateHostnameVerifierWrongRM() throws Exception {
             proxyCA2.createChildKeyStore(
                 ApplicationId.newInstance(System.currentTimeMillis(), 1),
                 "password"), "password").getCertificateChain("server"));
-    Assert.assertFalse(verifier.verify("foo", sslSession));
+    assertFalse(verifier.verify("foo", sslSession));
   }
 
   @Test
-  public void testCreateHostnameVerifierExceptions() throws Exception {
+  void testCreateHostnameVerifierExceptions() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     HostnameVerifier verifier = proxyCA.getHostnameVerifier();
@@ -510,12 +519,12 @@ public Certificate[] answer(InvocationOnMock invocation)
               return certChain;
             }
           });
-      Assert.assertFalse(verifier.verify("foo", sslSession));
+      assertFalse(verifier.verify("foo", sslSession));
     }
   }
 
   @Test
-  public void testCreateHostnameVerifierRealCert() throws Exception {
+  void testCreateHostnameVerifierRealCert() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     HostnameVerifier verifier = proxyCA.getHostnameVerifier();
@@ -534,11 +543,11 @@ public Certificate[] answer(InvocationOnMock invocation)
             return certChain;
           }
         });
-    Assert.assertTrue(verifier.verify("foo.com", sslSession));
+    assertTrue(verifier.verify("foo.com", sslSession));
   }
 
   @Test
-  public void testCreateHostnameVerifierRealCertBad() throws Exception {
+  void testCreateHostnameVerifierRealCertBad() throws Exception {
     ProxyCA proxyCA = new ProxyCA();
     proxyCA.init();
     HostnameVerifier verifier = proxyCA.getHostnameVerifier();
@@ -557,27 +566,27 @@ public Certificate[] answer(InvocationOnMock invocation)
             return certChain;
           }
         });
-    Assert.assertFalse(verifier.verify("bar.com", sslSession));
+    assertFalse(verifier.verify("bar.com", sslSession));
   }
 
   private void checkCACert(X509Certificate caCert) {
-    Assert.assertEquals(caCert.getSubjectX500Principal().toString(),
+    assertEquals(caCert.getSubjectX500Principal().toString(),
         caCert.getIssuerDN().toString());
-    Assert.assertEquals(caCert.getSubjectX500Principal().toString(),
+    assertEquals(caCert.getSubjectX500Principal().toString(),
         caCert.getSubjectDN().toString());
-    Assert.assertTrue("Expected CA certificate X500 Principal to start with" +
-            " 'OU=YARN-', but did not: " + caCert.getSubjectX500Principal(),
-        caCert.getSubjectX500Principal().toString().startsWith("OU=YARN-"));
-    Assert.assertFalse("Found multiple fields in X500 Principal, when there " +
-            "should have only been one: " + caCert.getSubjectX500Principal(),
-        caCert.getSubjectX500Principal().toString().contains(","));
-    Assert.assertEquals("SHA512withRSA", caCert.getSigAlgName());
-    Assert.assertEquals(
+    assertTrue(caCert.getSubjectX500Principal().toString().startsWith("OU=YARN-"),
+        "Expected CA certificate X500 Principal to start with" +
+            " 'OU=YARN-', but did not: " + caCert.getSubjectX500Principal());
+    assertFalse(caCert.getSubjectX500Principal().toString().contains(","),
+        "Found multiple fields in X500 Principal, when there " +
+            "should have only been one: " + caCert.getSubjectX500Principal());
+    assertEquals("SHA512withRSA", caCert.getSigAlgName());
+    assertEquals(
         new GregorianCalendar(2037, Calendar.DECEMBER, 31).getTime(),
         caCert.getNotAfter());
-    Assert.assertTrue("Expected certificate to have started but was not: "
-        + caCert.getNotBefore(), caCert.getNotBefore().before(new Date()));
-    Assert.assertEquals(0, caCert.getBasicConstraints());
+    assertTrue(caCert.getNotBefore().before(new Date()),
+        "Expected certificate to have started but was not: " + caCert.getNotBefore());
+    assertEquals(0, caCert.getBasicConstraints());
   }
 
   private void checkPrivatePublicKeys(PrivateKey privateKey,
@@ -592,7 +601,7 @@ private void checkPrivatePublicKeys(PrivateKey privateKey,
     signer = Signature.getInstance("SHA512withRSA");
     signer.initVerify(publicKey);
     signer.update(data);
-    Assert.assertTrue(signer.verify(sig));
+    assertTrue(signer.verify(sig));
   }
 
   private X509Certificate[] castCertificateArrayToX509CertificateArray(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
index 20e5cdb90074e..07c0dc1d5abc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
@@ -18,59 +18,64 @@
 
 package org.apache.hadoop.yarn.server.webproxy;
 
-import static org.junit.Assert.*;
-
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
 
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.TrackingUriPlugin;
 
-import org.junit.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 
 public class TestProxyUriUtils {
   @Test
-  public void testGetPathApplicationId() {
-    assertEquals("/proxy/application_100_0001", 
+  void testGetPathApplicationId() {
+    assertEquals("/proxy/application_100_0001",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1)));
-    assertEquals("/proxy/application_6384623_0005", 
+    assertEquals("/proxy/application_6384623_0005",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5)));
   }
 
-  @Test(expected = IllegalArgumentException.class)
-  public void testGetPathApplicationIdBad() {
-    ProxyUriUtils.getPath(null);
+  @Test
+  void testGetPathApplicationIdBad() {
+    assertThrows(IllegalArgumentException.class, () -> {
+      ProxyUriUtils.getPath(null);
+    });
   }
-  
+
   @Test
-  public void testGetPathApplicationIdString() {
-    assertEquals("/proxy/application_6384623_0005", 
+  void testGetPathApplicationIdString() {
+    assertEquals("/proxy/application_6384623_0005",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null));
     assertEquals("/proxy/application_6384623_0005/static/app",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app"));
-    assertEquals("/proxy/application_6384623_0005/", 
+    assertEquals("/proxy/application_6384623_0005/",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/"));
-    assertEquals("/proxy/application_6384623_0005/some/path", 
+    assertEquals("/proxy/application_6384623_0005/some/path",
         ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path"));
   }
-  
-  @Test 
-  public void testGetPathAndQuery() {
+
+  @Test
+  void testGetPathAndQuery() {
     assertEquals("/proxy/application_6384623_0005/static/app?foo=bar",
-    ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", 
-        "?foo=bar", false));
-    
+        ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
+            "?foo=bar", false));
+
     assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true",
-        ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", 
+        ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
             "foo=bar&bad=good", true));
   }
 
   @Test
-  public void testGetProxyUri() throws Exception {
+  void testGetProxyUri() throws Exception {
     URI originalUri = new URI("http://host.com/static/foo?bar=bar");
     URI proxyUri = new URI("http://proxy.net:8080/");
     ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
@@ -79,9 +84,9 @@ public void testGetProxyUri() throws Exception {
     assertEquals(expected, result);
   }
 
-  
+
   @Test
-  public void testGetProxyUriNull() throws Exception {
+  void testGetProxyUriNull() throws Exception {
     URI originalUri = null;
     URI proxyUri = new URI("http://proxy.net:8080/");
     ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
@@ -91,7 +96,7 @@ public void testGetProxyUriNull() throws Exception {
   }
 
   @Test
-  public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
+  void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
       throws URISyntaxException {
     ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
     List<TrackingUriPlugin> list =
@@ -100,7 +105,7 @@ public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins()
   }
 
   @Test
-  public void testGetProxyUriFromPluginsReturnsValidUriWhenAble()
+  void testGetProxyUriFromPluginsReturnsValidUriWhenAble()
       throws URISyntaxException {
     ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
     List<TrackingUriPlugin> list =
@@ -119,6 +124,6 @@ public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
     });
     URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list);
     assertNotNull(result);
-    
+
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
index 87907a64f2875..8ddcd64a13343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
@@ -18,17 +18,17 @@
 
 package org.apache.hadoop.yarn.server.webproxy;
 
-import static org.junit.Assert.assertEquals;
+import java.net.InetSocketAddress;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
 
-import java.net.InetSocketAddress;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class TestWebAppProxyServer {
   private WebAppProxyServer webAppProxy = null;
@@ -36,20 +36,20 @@ public class TestWebAppProxyServer {
   private final String proxyAddress = "localhost:" + port;
   private YarnConfiguration conf = null;
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     conf = new YarnConfiguration();
     conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress);
     webAppProxy = new WebAppProxyServer();
   }
 
-  @After
+  @AfterEach
   public void tearDown() throws Exception {
     webAppProxy.stop();
   }
 
   @Test
-  public void testStart() {
+  void testStart() {
     webAppProxy.init(conf);
     assertEquals(STATE.INITED, webAppProxy.getServiceState());
     webAppProxy.start();
@@ -62,7 +62,7 @@ public void testStart() {
   }
 
   @Test
-  public void testStartWithBindHost() {
+  void testStartWithBindHost() {
     String bindHost = "0.0.0.0";
     conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost);
     webAppProxy.init(conf);
@@ -80,12 +80,12 @@ public void testStartWithBindHost() {
 
 
   @Test
-  public void testBindAddress() {
+  void testBindAddress() {
     conf = new YarnConfiguration();
 
     InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf);
-    Assert.assertEquals("Web Proxy default bind address port is incorrect",
-        YarnConfiguration.DEFAULT_PROXY_PORT,
-        defaultBindAddress.getPort());
+    assertEquals(YarnConfiguration.DEFAULT_PROXY_PORT,
+        defaultBindAddress.getPort(),
+        "Web Proxy default bind address port is incorrect");
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
index 6c8993f6e80b7..c59d5b952a8c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
@@ -18,14 +18,6 @@
 
 package org.apache.hadoop.yarn.server.webproxy;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -35,14 +27,13 @@
 import java.net.ConnectException;
 import java.net.HttpCookie;
 import java.net.HttpURLConnection;
+import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.net.URL;
-import java.net.SocketTimeoutException;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Map;
-
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
@@ -50,6 +41,19 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.http.HttpServer2;
@@ -65,17 +69,15 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.webapp.MimeType;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Test the WebAppProxyServlet and WebAppProxy. For back end use simple web
@@ -96,7 +98,7 @@ public class TestWebAppProxyServlet {
   /**
    * Simple http server. Server should send answer with status 200
    */
-  @BeforeClass
+  @BeforeAll
   public static void start() throws Exception {
     server = new Server(0);
     ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(20);
@@ -175,8 +177,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
     }
   }
 
-  @Test(timeout=5000)
-  public void testWebAppProxyServlet() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testWebAppProxyServlet() throws Exception {
     configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090");
     // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS
     configuration.setInt("hadoop.http.max.threads", 10);
@@ -221,17 +224,17 @@ public void testWebAppProxyServlet() throws Exception {
       proxyConn = (HttpURLConnection) redirectUrl.openConnection();
       proxyConn.setInstanceFollowRedirects(false);
       proxyConn.connect();
-      assertEquals("The proxy returned an unexpected status code rather than"
-          + "redirecting the connection (302)",
-          HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode());
+      assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode(),
+          "The proxy returned an unexpected status code rather than"
+              + "redirecting the connection (302)");
 
       String expected =
           WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration)
-            + "/cluster/failure/application_00_0";
+              + "/cluster/failure/application_00_0";
       String redirect = proxyConn.getHeaderField(ProxyUtils.LOCATION);
 
-      assertEquals("The proxy did not redirect the connection to the failure "
-          + "page of the RM", expected, redirect);
+      assertEquals(expected, redirect, "The proxy did not redirect the connection to the failure "
+          + "page of the RM");
 
       // cannot found application 1: null
       appReportFetcher.answer = 1;
@@ -274,7 +277,7 @@ public void testWebAppProxyServlet() throws Exception {
       // original tracking url
       appReportFetcher.answer = 5;
       URL clientUrl = new URL("http://localhost:" + proxyPort
-        + "/proxy/application_00_0/test/tez?x=y&h=p");
+          + "/proxy/application_00_0/test/tez?x=y&h=p");
       proxyConn = (HttpURLConnection) clientUrl.openConnection();
       proxyConn.connect();
       LOG.info("" + proxyConn.getURL());
@@ -286,47 +289,51 @@ public void testWebAppProxyServlet() throws Exception {
     }
   }
 
-  @Test(expected = SocketTimeoutException.class)
-  public void testWebAppProxyConnectionTimeout()
-      throws IOException, ServletException{
-    HttpServletRequest request = mock(HttpServletRequest.class);
-    when(request.getMethod()).thenReturn("GET");
-    when(request.getRemoteUser()).thenReturn("dr.who");
-    when(request.getPathInfo()).thenReturn("/application_00_0");
-    when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration());
+  @Test
+  void testWebAppProxyConnectionTimeout()
+      throws IOException, ServletException {
+    assertThrows(SocketTimeoutException.class, () -> {
+      HttpServletRequest request = mock(HttpServletRequest.class);
+      when(request.getMethod()).thenReturn("GET");
+      when(request.getRemoteUser()).thenReturn("dr.who");
+      when(request.getPathInfo()).thenReturn("/application_00_0");
+      when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration());
 
-    HttpServletResponse response = mock(HttpServletResponse.class);
-    when(response.getOutputStream()).thenReturn(null);
+      HttpServletResponse response = mock(HttpServletResponse.class);
+      when(response.getOutputStream()).thenReturn(null);
 
-    WebAppProxyServlet servlet = new WebAppProxyServlet();
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED,
-        true);
-    conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT,
-        1000);
+      WebAppProxyServlet servlet = new WebAppProxyServlet();
+      YarnConfiguration conf = new YarnConfiguration();
+      conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED,
+          true);
+      conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT,
+          1000);
+
+      servlet.setConf(conf);
 
-    servlet.setConf(conf);
+      ServletConfig config = mock(ServletConfig.class);
+      ServletContext context = mock(ServletContext.class);
+      when(config.getServletContext()).thenReturn(context);
 
-    ServletConfig config = mock(ServletConfig.class);
-    ServletContext context = mock(ServletContext.class);
-    when(config.getServletContext()).thenReturn(context);
+      AppReportFetcherForTest appReportFetcher =
+          new AppReportFetcherForTest(new YarnConfiguration());
 
-    AppReportFetcherForTest appReportFetcher =
-        new AppReportFetcherForTest(new YarnConfiguration());
+      when(config.getServletContext()
+          .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE))
+          .thenReturn(appReportFetcher);
 
-    when(config.getServletContext()
-        .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE))
-        .thenReturn(appReportFetcher);
+      appReportFetcher.answer = 7;
 
-    appReportFetcher.answer = 7;
+      servlet.init(config);
+      servlet.doGet(request, response);
 
-    servlet.init(config);
-    servlet.doGet(request, response);
+    });
 
   }
 
-  @Test(timeout=5000)
-  public void testAppReportForEmptyTrackingUrl() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testAppReportForEmptyTrackingUrl() throws Exception {
     configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090");
     // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS
     configuration.setInt("hadoop.http.max.threads", 10);
@@ -338,53 +345,51 @@ public void testAppReportForEmptyTrackingUrl() throws Exception {
     AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher;
 
     try {
-    //set AHS_ENBALED = false to simulate getting the app report from RM
-    configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
-        false);
-    ApplicationId app = ApplicationId.newInstance(0, 0);
-    appReportFetcher.answer = 6;
-    URL url = new URL("http://localhost:" + proxyPort +
-        "/proxy/" + app.toString());
-    HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection();
-    proxyConn.connect();
-    try {
-      proxyConn.getResponseCode();
-    } catch (ConnectException e) {
-      // Connection Exception is expected as we have set
-      // appReportFetcher.answer = 6, which does not set anything for
-      // original tracking url field in the app report.
-    }
-    String appAddressInRm =
-        WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) +
-        "/cluster" + "/app/" + app.toString();
-    assertTrue("Webapp proxy servlet should have redirected to RM",
-        proxyConn.getURL().toString().equals(appAddressInRm));
-
-    //set AHS_ENBALED = true to simulate getting the app report from AHS
-    configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
-        true);
-    proxyConn = (HttpURLConnection) url.openConnection();
-    proxyConn.connect();
-    try {
-      proxyConn.getResponseCode();
-    } catch (ConnectException e) {
-      // Connection Exception is expected as we have set
-      // appReportFetcher.answer = 6, which does not set anything for
-      // original tracking url field in the app report.
-    }
-    String appAddressInAhs = WebAppUtils.getHttpSchemePrefix(configuration) +
-        WebAppUtils.getAHSWebAppURLWithoutScheme(configuration) +
-        "/applicationhistory" + "/app/" + app.toString();
-    assertTrue("Webapp proxy servlet should have redirected to AHS",
-        proxyConn.getURL().toString().equals(appAddressInAhs));
-    }
-    finally {
+      //set AHS_ENBALED = false to simulate getting the app report from RM
+      configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+          false);
+      ApplicationId app = ApplicationId.newInstance(0, 0);
+      appReportFetcher.answer = 6;
+      URL url = new URL("http://localhost:" + proxyPort +
+          "/proxy/" + app.toString());
+      HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection();
+      proxyConn.connect();
+      try {
+        proxyConn.getResponseCode();
+      } catch (ConnectException e) {
+        // Connection Exception is expected as we have set
+        // appReportFetcher.answer = 6, which does not set anything for
+        // original tracking url field in the app report.
+      }
+      String appAddressInRm =
+          WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) +
+              "/cluster" + "/app/" + app.toString();
+      assertEquals(proxyConn.getURL().toString(), appAddressInRm);
+
+      //set AHS_ENBALED = true to simulate getting the app report from AHS
+      configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+          true);
+      proxyConn = (HttpURLConnection) url.openConnection();
+      proxyConn.connect();
+      try {
+        proxyConn.getResponseCode();
+      } catch (ConnectException e) {
+        // Connection Exception is expected as we have set
+        // appReportFetcher.answer = 6, which does not set anything for
+        // original tracking url field in the app report.
+      }
+      String appAddressInAhs =
+          WebAppUtils.getHttpSchemePrefix(configuration) + WebAppUtils.getAHSWebAppURLWithoutScheme(
+              configuration) + "/applicationhistory" + "/app/" + app.toString();
+      assertEquals(proxyConn.getURL().toString(), appAddressInAhs);
+    } finally {
       proxy.close();
     }
   }
 
-  @Test(timeout=5000)
-  public void testWebAppProxyPassThroughHeaders() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testWebAppProxyPassThroughHeaders() throws Exception {
     Configuration configuration = new Configuration();
     configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9091");
     configuration.setInt("hadoop.http.max.threads", 10);
@@ -424,8 +429,9 @@ public void testWebAppProxyPassThroughHeaders() throws Exception {
   /**
    * Test main method of WebAppProxyServer
    */
-  @Test(timeout=5000)
-  public void testWebAppProxyServerMainMethod() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testWebAppProxyServerMainMethod() throws Exception {
     WebAppProxyServer mainServer = null;
     Configuration conf = new YarnConfiguration();
     conf.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9099");
@@ -458,8 +464,9 @@ public void testWebAppProxyServerMainMethod() throws Exception {
     }
   }
 
-  @Test(timeout=5000)
-  public void testCheckHttpsStrictAndNotProvided() throws Exception {
+  @Test
+  @Timeout(5000)
+  void testCheckHttpsStrictAndNotProvided() throws Exception {
     HttpServletResponse resp = mock(HttpServletResponse.class);
     StringWriter sw = new StringWriter();
     when(resp.getWriter()).thenReturn(new PrintWriter(sw));
@@ -498,8 +505,9 @@ public void testCheckHttpsStrictAndNotProvided() throws Exception {
     assertTrue(WebAppProxyServlet.checkHttpsStrictAndNotProvided(
         resp, httpLink, conf));
     String s = sw.toString();
-    assertTrue("Was expecting an HTML page explaining that an HTTPS tracking" +
-        " url must be used but found " + s, s.contains("HTTPS must be used"));
+    assertTrue(s.contains("HTTPS must be used"),
+        "Was expecting an HTML page explaining that an HTTPS tracking"
+            + " url must be used but found " + s);
     Mockito.verify(resp, Mockito.times(1)).setContentType(MimeType.HTML);
   }
 
@@ -529,7 +537,7 @@ private boolean isResponseCookiePresent(HttpURLConnection proxyConn,
     return false;
   }
 
-  @AfterClass
+  @AfterAll
   public static void stop() throws Exception {
     try {
       server.stop();
@@ -584,8 +592,7 @@ protected void serviceStart() throws Exception {
           ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
 
       appReportFetcher = new AppReportFetcherForTest(conf);
-      proxyServer.setAttribute(FETCHER_ATTRIBUTE,
-          appReportFetcher );
+      proxyServer.setAttribute(FETCHER_ATTRIBUTE, appReportFetcher);
       proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
 
       String proxy = WebAppUtils.getProxyHostAndPort(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
index 5fcdb62ea33ab..07302e6527675 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java
@@ -22,48 +22,46 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.net.HttpURLConnection;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.Enumeration;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.Map;
+import java.util.Enumeration;
 import java.util.HashMap;
-import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
-
-import javax.servlet.FilterConfig;
-import javax.servlet.FilterChain;
+import java.util.function.Supplier;
 import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
 import javax.servlet.ServletContext;
-import javax.servlet.ServletResponse;
-import javax.servlet.ServletRequest;
 import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
 import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.function.Supplier;
-import org.apache.hadoop.http.TestHttpServer;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
-import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.glassfish.grizzly.servlet.HttpServletResponseImpl;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.mockito.Mockito;
 
+import org.apache.hadoop.http.TestHttpServer;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUtils;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
 /**
  * Test AmIpFilter. Requests to a no declared hosts should has way through
  * proxy. Another requests can be filtered with (without) user name.
@@ -117,9 +115,10 @@ public ServletContext getServletContext() {
     }
   }
 
-  @Test(timeout = 5000)
+  @Test
+  @Timeout(5000)
   @SuppressWarnings("deprecation")
-  public void filterNullCookies() throws Exception {
+  void filterNullCookies() throws Exception {
     HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
 
     Mockito.when(request.getCookies()).thenReturn(null);
@@ -148,7 +147,7 @@ public void doFilter(ServletRequest servletRequest,
   }
 
   @Test
-  public void testFindRedirectUrl() throws Exception {
+  void testFindRedirectUrl() throws Exception {
     final String rm1 = "rm1";
     final String rm2 = "rm2";
     // generate a valid URL
@@ -162,7 +161,7 @@ public void testFindRedirectUrl() throws Exception {
     spy.proxyUriBases = new HashMap<>();
     spy.proxyUriBases.put(rm1, rm1Url);
     spy.proxyUriBases.put(rm2, rm2Url);
-    spy.rmUrls = new String[] { rm1, rm2 };
+    spy.rmUrls = new String[]{rm1, rm2};
 
     assertThat(spy.findRedirectUrl()).isEqualTo(rm1Url);
   }
@@ -182,8 +181,9 @@ private String startHttpServer() throws Exception {
     return server.getURI().toString() + servletPath;
   }
 
-  @Test(timeout = 2000)
-  public void testProxyUpdate() throws Exception {
+  @Test
+  @Timeout(2000)
+  void testProxyUpdate() throws Exception {
     Map<String, String> params = new HashMap<>();
     params.put(AmIpFilter.PROXY_HOSTS, proxyHost);
     params.put(AmIpFilter.PROXY_URI_BASES, proxyUri);
@@ -223,9 +223,10 @@ public Boolean get() {
   /**
    * Test AmIpFilter
    */
-  @Test(timeout = 10000)
+  @Test
+  @Timeout(10000)
   @SuppressWarnings("deprecation")
-  public void testFilter() throws Exception {
+  void testFilter() throws Exception {
     Map<String, String> params = new HashMap<String, String>();
     params.put(AmIpFilter.PROXY_HOST, proxyHost);
     params.put(AmIpFilter.PROXY_URI_BASE, proxyUri);
@@ -289,7 +290,7 @@ public void doFilter(ServletRequest servletRequest,
     assertTrue(doFilterRequest.contains("HttpServletRequest"));
 
     // cookie added
-    Cookie[] cookies = new Cookie[] {
+    Cookie[] cookies = new Cookie[]{
         new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user")
     };
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
index 97625ac0882b0..0aec45fce8a57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java
@@ -22,9 +22,10 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.*;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.FilterContainer;
 import org.apache.hadoop.http.HttpConfig;
@@ -32,12 +33,15 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
 /**
  * Test class for {@Link AmFilterInitializer}.
  */
 public class TestAmFilterInitializer {
 
-  @Before
+  @BeforeEach
   public void setUp() throws Exception {
     NetUtils.addStaticResolution("host1", "172.0.0.1");
     NetUtils.addStaticResolution("host2", "172.0.0.1");
@@ -48,7 +52,7 @@ public void setUp() throws Exception {
   }
 
   @Test
-  public void testInitFilter() {
+  void testInitFilter() {
     // Check PROXY_ADDRESS
     MockFilterContainer con = new MockFilterContainer();
     Configuration conf = new Configuration(false);
@@ -60,7 +64,7 @@ public void testInitFilter() {
     assertEquals("host1", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
     assertEquals("http://host1:1000/foo",
         con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
-    assertEquals(null, con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
+    assertNull(con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
 
     // Check a single RM_WEBAPP_ADDRESS
     con = new MockFilterContainer();
@@ -73,7 +77,7 @@ public void testInitFilter() {
     assertEquals("host2", con.givenParameters.get(AmIpFilter.PROXY_HOSTS));
     assertEquals("http://host2:2000/foo",
         con.givenParameters.get(AmIpFilter.PROXY_URI_BASES));
-    assertEquals(null, con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
+    assertNull(con.givenParameters.get(AmFilterInitializer.RM_HA_URLS));
 
     // Check multiple RM_WEBAPP_ADDRESSes (RM HA)
     con = new MockFilterContainer();
@@ -134,7 +138,7 @@ public void testInitFilter() {
   }
 
   @Test
-  public void testGetProxyHostsAndPortsForAmFilter() {
+  void testGetProxyHostsAndPortsForAmFilter() {
 
     // Check no configs given
     Configuration conf = new Configuration(false);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java
index 5bbfc8fafa0ce..f6cc054361d94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java
@@ -21,28 +21,30 @@
 import java.io.File;
 import java.net.URI;
 import java.net.URL;
-import java.util.Set;
-import java.util.HashSet;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
 
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.KerberosTestUtils;
-import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.mockito.Mockito;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Test AmIpFilter. Requests to a no declared hosts should has way through
@@ -63,7 +65,7 @@ public class TestSecureAmFilter {
   private static boolean miniKDCStarted = false;
   private static MiniKdc testMiniKDC;
 
-  @BeforeClass
+  @BeforeAll
   public static void setUp() {
     rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
     rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -88,11 +90,11 @@ public static void setUp() {
       testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
       setupKDC();
     } catch (Exception e) {
-      assertTrue("Couldn't create MiniKDC", false);
+      fail("Couldn't create MiniKDC");
     }
   }
 
-  @AfterClass
+  @AfterAll
   public static void tearDown() {
     if (testMiniKDC != null) {
       testMiniKDC.stop();
@@ -125,7 +127,7 @@ protected Set<String> getProxyAddresses() {
   }
 
   @Test
-  public void testFindRedirectUrl() throws Exception {
+  void testFindRedirectUrl() throws Exception {
     final String rm1 = "rm1";
     final String rm2 = "rm2";
     // generate a valid URL
@@ -139,7 +141,7 @@ public void testFindRedirectUrl() throws Exception {
     spy.proxyUriBases = new HashMap<>();
     spy.proxyUriBases.put(rm1, rm1Url);
     spy.proxyUriBases.put(rm2, rm2Url);
-    spy.rmUrls = new String[] {rm1, rm2};
+    spy.rmUrls = new String[]{rm1, rm2};
 
     assertTrue(spy.isValidUrl(rm1Url));
     assertFalse(spy.isValidUrl(rm2Url));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 6d87d4febb65c..3a694e3f2485e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -151,7 +151,7 @@ Configuration
 |:---- |:---- |
 | `yarn.scheduler.capacity.maximum-applications` / `yarn.scheduler.capacity.<queue-path>.maximum-applications` | Maximum number of applications in the system which can be concurrently active both running and pending. Limits on each queue are directly proportional to their queue capacities and user limits. This is a hard limit and any applications submitted when this limit is reached will be rejected. Default is 10000. This can be set for all queues with `yarn.scheduler.capacity.maximum-applications` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity.<queue-path>.maximum-applications`. When this property is not set for a specific queue path, the maximum application number is calculated by taking all configured node labels into consideration, and choosing the highest possible value. Integer value expected. |
 | `yarn.scheduler.capacity.maximum-am-resource-percent` / `yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent` | Maximum percent of resources in the cluster which can be used to run application masters - controls number of concurrent active applications. Limits on each queue are directly proportional to their queue capacities and user limits. Specified as a float - ie 0.5 = 50%. Default is 10%. This can be set for all queues with `yarn.scheduler.capacity.maximum-am-resource-percent` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity.<queue-path>.maximum-am-resource-percent` |
-| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity.<queue-path>.max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity.<queue-path>.max-parallel-apps`. Integer value is expected. By default, there is no limit. |
+| `yarn.scheduler.capacity.max-parallel-apps` / `yarn.scheduler.capacity.<queue-path>.max-parallel-apps` | Maximum number of applications that can run at the same time. Unlike to `maximum-applications`, application submissions are *not* rejected when this limit is reached. Instead they stay in `ACCEPTED` state until they are eligible to run. This can be set for all queues with `yarn.scheduler.capacity.max-parallel-apps` and can also be overridden on a per queue basis by setting `yarn.scheduler.capacity.<queue-path>.max-parallel-apps`. Integer value is expected. By default, there is no limit. The maximum parallel application limit is an inherited property in the queue hierarchy, meaning that the lowest value will be selected as the enforced limit in every branch of the hierarchy. |
 
   You can also limit the number of parallel applications on a per user basis.
 
@@ -176,8 +176,8 @@ Configuration
 | Property | Description |
 |:---- |:---- |
 | `yarn.scheduler.capacity.<queue-path>.state` | The *state* of the queue. Can be one of `RUNNING` or `STOPPED`. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Thus, if the *root* queue is `STOPPED` no applications can be submitted to the entire cluster. Existing applications continue to completion, thus the queue can be *drained* gracefully. Value is specified as Enumeration. |
-| `yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. |
-| `yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. |
+| `yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to submit to the queue. |
+| `yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to administer apps the queue. |
 
 **Note:** An *ACL* is of the form *user1*,*user2* *space* *group1*,*group2*. The special value of * implies *anyone*. The special value of *space* implies *no one*. The default is * for the root queue if not specified.
 
@@ -642,9 +642,7 @@ support other pre-configured queues to co-exist along with auto-created queues.
 
 The parent queue which has been enabled for auto leaf queue creation,supports
  the configuration of template parameters for automatic configuration of the auto-created leaf queues. The auto-created queues support all of the
- leaf queue configuration parameters except for **Queue ACL**, **Absolute
- Resource** configurations. Queue ACLs are
- currently inherited from the parent queue i.e they are not configurable on the leaf queue template
+ leaf queue configuration parameters except for **Absolute Resource** configurations.
 
 | Property | Description |
 |:---- |:---- |
@@ -730,11 +728,11 @@ The `Flexible Dynamic Queue Auto-Creation and Management` feature allows a **Par
 
 * Configuring **flexible** `Auto-Created Leaf Queues` with `CapacityScheduler`
 
-The parent queue which has the flexible auto queue creation enabled supports the configuration of dynamically created leaf and parent queues through template parameters. The auto-created queues support all of the leaf queue configuration parameters except for **Queue ACL**, **Absolute Resource** configurations. Queue ACLs are currently inherited from the parent queue i.e they are not configurable on the leaf queue template
+The parent queue which has the flexible auto queue creation enabled supports the configuration of dynamically created leaf and parent queues through template parameters. The auto-created queues support all of the leaf queue configuration parameters except for **Absolute Resource** configurations.
 
 | Property | Description |
 |:---- |:---- |
-| `yarn.scheduler.capacity.<queue-path>.auto-queue-creation-v2.template.<queue-property>` | *Optional* parameter: Specifies a queue property (like capacity, maximum-capacity, user-limit-factor, maximum-am-resource-percent ...  - Refer **Queue Properties** section) inherited by the auto-created **parent** and **leaf** queues. |
+| `yarn.scheduler.capacity.<queue-path>.auto-queue-creation-v2.template.<queue-property>` | *Optional* parameter: Specifies a queue property (like capacity, maximum-capacity, user-limit-factor, maximum-am-resource-percent ...  - Refer **Queue Properties** section) inherited by the auto-created **parent** and **leaf** queues. Dynamic Queue ACLs set here can be overwritten by the parent-template for dynamic parent queues and with the leaf-template for dynamic leaf queues.  |
 | `yarn.scheduler.capacity.<queue-path>.auto-queue-creation-v2.leaf-template.<queue-property>` | *Optional* parameter: Specifies a queue property inherited by auto-created **leaf** queues. |
 | `yarn.scheduler.capacity.<queue-path>.auto-queue-creation-v2.parent-template.<queue-property>` |  *Optional* parameter: Specifies a queue property inherited by auto-created **parent** queues. |
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index 64a6123983d89..fe7d1e1ad39ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -438,7 +438,7 @@ environment variables in the application's environment:
 | `YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL` | Allows a user to request delayed deletion of the Docker container on a per container basis. If true, Docker containers will not be removed until the duration defined by yarn.nodemanager.delete.debug-delay-sec has elapsed. Administrators can disable this feature through the yarn-site property yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed. This feature is disabled by default. When this feature is disabled or set to false, the container will be removed as soon as it exits. |
 | `YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE` | Enable mounting of container working directory sysfs sub-directory into Docker container /hadoop/yarn/sysfs.  This is useful for populating cluster information into container. |
 | `YARN_CONTAINER_RUNTIME_DOCKER_SERVICE_MODE` | Enable Service Mode which runs the docker container as defined by the image but does not set the user (--user and --group-add). |
-
+| `YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG` | Sets the docker client config using which docker container executor can access the remote docker image. |
 The first two are required. The remainder can be set as needed. While
 controlling the container type through environment variables is somewhat less
 than ideal, it allows applications with no awareness of YARN's Docker support
@@ -1016,6 +1016,24 @@ To run a Spark shell in Docker containers, run the following command:
 Note that the application master and executors are configured
 independently. In this example, we are using the `openjdk:8` image for both.
 
+When using remote container registry,
+the YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG must reference the config.json
+file containing the credentials used to authenticate.
+
+```
+DOCKER_IMAGE_NAME=hadoop-docker
+DOCKER_CLIENT_CONFIG=hdfs:///user/hadoop/config.json
+spark-submit --master yarn \
+--deploy-mode cluster \
+--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \
+--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=$DOCKER_IMAGE_NAME \
+--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG=$DOCKER_CLIENT_CONFIG \
+--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \
+--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=$DOCKER_IMAGE_NAME \
+--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG=$DOCKER_CLIENT_CONFIG \
+sparkR.R
+```
+
 Docker Container ENTRYPOINT Support
 ------------------------------------
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index c7836e75475b1..f547e8d6b7744 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -164,7 +164,7 @@ These are common configurations that should appear in the **conf/yarn-site.xml**
 
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.federation.enabled` | `true` | Whether federation is enabled or not |
 |`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
 
@@ -177,21 +177,23 @@ Currently, we support ZooKeeper and SQL based implementations of the state-store
 ZooKeeper: one must set the ZooKeeper settings for Hadoop:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore` | The type of state-store to use. |
 |`hadoop.zk.address` | `host:port` | The address for the ZooKeeper ensemble. |
 
 SQL: one must setup the following parameters:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. |
 |`yarn.federation.state-store.sql.url` | `jdbc:mysql://<host>:<port>/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
 |`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. |
 |`yarn.federation.state-store.sql.username` | `<dbuser>` | For SQLFederationStateStore the username for the DB connection. |
 |`yarn.federation.state-store.sql.password` | `<dbpass>` | For SQLFederationStateStore the password for the DB connection. |
 
-We provide scripts for MySQL and Microsoft SQL Server.
+We provide scripts for **MySQL** and **Microsoft SQL Server**.
+
+> MySQL
 
 For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
 Then the DB schema is created by executing the following SQL scripts in the database:
@@ -205,14 +207,28 @@ In the same directory we provide scripts to drop the Stored Procedures, the Tabl
 
 **Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
 
+**The versions supported by MySQL are MySQL 5.7 and above:**
+
+1. MySQL 5.7
+2. MySQL 8.0
+
+> Microsoft SQL Server
+
 For SQL-Server, the process is similar, but the jdbc driver is already included.
 SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
 
+**The versions supported by SQL-Server are SQL Server 2008 R2 and above:**
+
+1. SQL Server 2008 R2 Enterprise
+2. SQL Server 2012 Enterprise
+3. SQL Server 2016 Enterprise
+4. SQL Server 2017 Enterprise

+5. SQL Server 2019 Enterprise
 
 ####Optional:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.federation.failover.enabled` | `true` | Whether should retry considering RM failover within each subcluster. |
 |`yarn.federation.blacklist-subclusters` | `<subcluster-id>` | A list of black-listed sub-clusters, useful to disable a sub-cluster |
 |`yarn.federation.policy-manager` | `org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager` | The choice of policy manager determines how Applications and ResourceRequests are routed through the system. |
@@ -225,13 +241,13 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
 These are extra configurations that should appear in the **conf/yarn-site.xml** at each ResourceManager.
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.resourcemanager.epoch` | `<unique-epoch>` | The seed value for the epoch. This is used to guarantee uniqueness of container-IDs generate by different RMs. It must therefore be unique among sub-clusters and `well-spaced` to allow for failures which increment epoch. Increments of 1000 allow for a large number of sub-clusters and practically ensure near-zero chance of collisions (a clash will only happen if a container is still alive for 1000 restarts of one RM, while the next RM never restarted, and an app requests more containers). |
 
 Optional:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.federation.state-store.heartbeat-interval-secs` | `60` | The rate at which RMs report their membership to the federation to the central state-store. |
 
 
@@ -240,38 +256,59 @@ Optional:
 These are extra configurations that should appear in the **conf/yarn-site.xml** at each Router.
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.router.bind-host` | `0.0.0.0` | Host IP to bind the router to.  The actual address the server will bind to. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.router.*.address respectively. This is most useful for making Router listen to all interfaces by setting to 0.0.0.0. |
 | `yarn.router.clientrm.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client. The last step of this pipeline must be the Federation Client Interceptor. |
 
 Optional:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 |`yarn.router.hostname` | `0.0.0.0` | Router host name.
 |`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
 |`yarn.router.webapp.address` | `0.0.0.0:8089` | Webapp address at the router. |
 |`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
 |`yarn.router.webapp.https.address` | `0.0.0.0:8091` | Secure webapp address at the router. |
 |`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
+|`yarn.router.submit.interval.time` | `10ms` | The interval between two retry, the default value is 10ms. |
 |`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
 |`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. |
 |`yarn.router.webapp.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client via REST interface. The last step of this pipeline must be the Federation Interceptor REST. |
 
+Security:
+
+Kerberos supported in federation.
+
+| Property | Example | Description |
+|:---- |:---- |:---- |
+| `yarn.router.keytab.file` |  | The keytab file used by router to login as its service principal. The principal name is configured with 'yarn.router.kerberos.principal'.|
+| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. |
+| `yarn.router.kerberos.principal.hostname` |  | Optional. The hostname for the Router containing this configuration file.  Will be different for each machine. Defaults to current hostname. |
+
+Enabling CORS support:
+
+To enable cross-origin support (CORS) for the Yarn Router, please set the following configuration parameters:
+
+| Property                                  | Example                                                      | Description                                                  |
+| ----------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| `hadoop.http.filter.initializers`         | `org.apache.hadoop.security.HttpCrossOriginFilterInitializer` | Optional. Set the filter to HttpCrossOriginFilterInitializer, Configure this parameter in core-site.xml. |
+| `yarn.router.webapp.cross-origin.enabled` | `true`                                                       | Optional. Enable/disable CORS filter.Configure this parameter in yarn-site.xml. |
+
+
 ###ON NMs:
 
 These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager.
 
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 | `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. |
 | `yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. |
 
 Optional:
 
 | Property | Example | Description |
-|:---- |:---- |
+|:---- |:---- |:---- |
 | `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt support. |
 | `yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. |
 | `yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. |
@@ -281,7 +318,7 @@ Running a Sample Job
 In order to submit jobs to a Federation cluster one must create a separate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
 
 | Property | Example | Description |
-|:--- |:--- |
+|:--- |:--- |:---- |
 | `yarn.resourcemanager.address` | `<router_host>:8050` | Redirects jobs launched at the client to the router's client RM port. |
 | `yarn.resourcemanager.scheduler.address` | `localhost:8049` | Redirects jobs to the federation AMRMProxy port.|
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
index 39d8311fa0804..9c39e9950249e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
@@ -85,7 +85,8 @@ Notes:
 ###Add/modify node-to-labels mapping to YARN
 
 * Configuring nodes to labels mapping in **Centralized** NodeLabel setup
-    * Executing ```yarn rmadmin -replaceLabelsOnNode “node1[:port]=label1 node2=label2” [-failOnUnknownNodes]```. Added label1 to node1, label2 to node2. If user don’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown.
+    * Executing ```yarn rmadmin -replaceLabelsOnNode "node1[:port]=label1 node2=label2" [-failOnUnknownNodes]``` adds label1 to node1, label2 to node2. If user doesn’t specify port, it adds the label to all ```NodeManagers``` running on the node. If option ```-failOnUnknownNodes``` is set, this command will fail if specified nodes are unknown.
+    * To remove mapping from a node, use `-replaceLabelsOnNode` but do not specify any labels. For example, ```yarn rmadmin -replaceLabelsOnNode "node1"``` removes label1 from node1.
 
 * Configuring nodes to labels mapping in **Distributed** NodeLabel setup
 
@@ -169,6 +170,14 @@ Applications can use following Java APIs to specify node label to request
 * `ResourceRequest.setNodeLabelExpression(..)` to set node label expression for individual resource requests. This can overwrite node label expression set in ApplicationSubmissionContext
 * Specify `setAMContainerResourceRequest.setNodeLabelExpression` in `ApplicationSubmissionContext` to indicate expected node label for application master container.
 
+__Default AM node-label Configuration__
+
+Property  | Value
+----- | ------
+yarn.resourcemanager.node-labels.am.default-node-label-expression | Overwrites default-node-label-expression only for the ApplicationMaster container. It is disabled by default.
+
+
+
 Monitoring
 ----------
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index e0f29a2679dde..001fb6b5f24a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -1605,11 +1605,11 @@ Current supported items:
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
-| resouceRequests | comma separated string | Skip resource requests of application in return |
+| resourceRequests | comma separated string | Skip resource requests of application in return |
 
 e.g:
 
-      * http://rm-http-address:port/ws/v1/cluster/apps?deSelects=resouceRequests
+      * http://rm-http-address:port/ws/v1/cluster/apps?deSelects=resourceRequests
 
 
 ### Response Examples
@@ -1979,7 +1979,7 @@ With the Application Statistics API, you can obtain a collection of triples, eac
 
 ### Query Parameters Required
 
-Two paramters can be specified. The parameters are case insensitive.
+Two parameters can be specified. The parameters are case insensitive.
 
       * states - states of the applications, specified as a comma-separated list. If states is not provided, the API will enumerate all application states and return the counts of them.
       * applicationTypes - types of the applications, specified as a comma-separated list. If applicationTypes is not provided, the API will count the applications of any application type. In this case, the response shows * to indicate any application type. Note that we only support at most one applicationType temporarily. Otherwise, users will expect an BadRequestException.
@@ -3062,7 +3062,7 @@ Response Body:
 Cluster Writeable APIs
 ----------------------
 
-The setions below refer to APIs which allow to create and modify applications. These APIs are currently in alpha and may change in the future.
+The sections below refer to APIs which allow to create and modify applications. These APIs are currently in alpha and may change in the future.
 
 Cluster New Application API
 ---------------------------
@@ -3092,7 +3092,7 @@ The NewApplication response contains the following elements:
 | application-id | string | The newly created application id |
 | maximum-resource-capabilities | object | The maximum resource capabilities available on this cluster |
 
-The *maximum-resource-capabilites* object contains the following elements:
+The *maximum-resource-capabilities* object contains the following elements:
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
@@ -3186,7 +3186,7 @@ Please note that this feature is currently in the alpha stage and may change in
 | resource | object | The resources the application master requires, described below |
 | application-type | string | The application type(MapReduce, Pig, Hive, etc) |
 | keep-containers-across-application-attempts | boolean | Should YARN keep the containers used by this application instead of destroying them |
-| application-tags | object | List of application tags, please see the request examples on how to speciy the tags |
+| application-tags | object | List of application tags, please see the request examples on how to specify the tags |
 | log-aggregation-context| object | Represents all of the information needed by the NodeManager to handle the logs for this application |
 | attempt-failures-validity-interval| long | The failure number will no take attempt failures which happen out of the validityInterval into failure count|
 | reservation-id| string | Represent the unique id of the corresponding reserved resource allocation in the scheduler |
@@ -3201,7 +3201,7 @@ The am-container-spec object should be used to provide the container launch cont
 | local-resources | object | Object describing the resources that need to be localized, described below |
 | environment | object | Environment variables for your containers, specified as key value pairs |
 | commands | object | The commands for launching your container, in the order in which they should be executed |
-| service-data | object | Application specific service data; key is the name of the auxiliary servce, value is base-64 encoding of the data you wish to pass |
+| service-data | object | Application specific service data; key is the name of the auxiliary service, value is base-64 encoding of the data you wish to pass |
 | credentials | object | The credentials required for your application to run, described below |
 | application-acls | objec | ACLs for your application; the key can be "VIEW\_APP" or "MODIFY\_APP", the value is the list of users with the permissions |
 
@@ -3237,8 +3237,8 @@ Elements of the POST request body *log-aggregation-context* object
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
-| log-include-pattern | string | The log files which match the defined include pattern will be uploaded when the applicaiton finishes |
-| log-exclude-pattern | string | The log files which match the defined exclude pattern will not be uploaded when the applicaiton finishes |
+| log-include-pattern | string | The log files which match the defined include pattern will be uploaded when the application finishes |
+| log-exclude-pattern | string | The log files which match the defined exclude pattern will not be uploaded when the application finishes |
 | rolled-log-include-pattern | string | The log files which match the defined include pattern will be aggregated in a rolling fashion |
 | rolled-log-exclude-pattern | string | The log files which match the defined exclude pattern will not be aggregated in a rolling fashion |
 | log-aggregation-policy-class-name | string | The policy which will be used by NodeManager to aggregate the logs |
@@ -3466,7 +3466,7 @@ Response Body:
 Cluster Application State API
 -----------------------------
 
-With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response.
+With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the initial response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response.
 
 Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response.
 
@@ -3645,7 +3645,7 @@ Request Body:
 
 Response Header:
 
-    HTTP/1.1 403 Unauthorized
+    HTTP/1.1 401 Unauthorized
     Server: Jetty(6.1.26)
 
 **Bad Request Error Response**
@@ -4454,7 +4454,7 @@ Elements of the *reservation-request* object
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
-| duration | long | The duration of a ReservationRequeust in milliseconds (amount of consecutive milliseconds a satisfiable allocation for this portion of the reservation should exist for). |
+| duration | long | The duration of a ReservationRequest in milliseconds (amount of consecutive milliseconds a satisfiable allocation for this portion of the reservation should exist for). |
 | num-containers | int | The number of containers required in this phase of the reservation (capture the maximum parallelism of the job(s) in this phase). |
 | min-concurrency | int | The minimum number of containers that must be concurrently allocated to satisfy this allocation (capture min-parallelism, useful to express gang semantics). |
 | capability | object | Allows to specify the size of each container (memory, vCores).|
@@ -4630,7 +4630,7 @@ Elements of the *reservation-request* object
 
 | Item | Data Type | Description |
 |:---- |:---- |:---- |
-| duration | long | The duration of a ReservationRequeust in milliseconds (amount of consecutive milliseconds a satisfiable allocation for this portion of the reservation should exist for). |
+| duration | long | The duration of a ReservationRequest in milliseconds (amount of consecutive milliseconds a satisfiable allocation for this portion of the reservation should exist for). |
 | num-containers | int | The number of containers required in this phase of the reservation (capture the maximum parallelism of the job(s) in this phase). |
 | min-concurrency | int | The minimum number of containers that must be concurrently allocated to satisfy this allocation (capture min-parallelism, useful to express gang semantics). |
 | capability | object | Allows to specify the size of each container (memory, vCores).|
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingNuma.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingNuma.md
new file mode 100644
index 0000000000000..c93469cae3f42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingNuma.md
@@ -0,0 +1,206 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# NUMA
+
+Non-uniform memory access (NUMA) is a computer memory design used in multiprocessing,
+where the memory access time depends on the memory location relative to the processor.
+Under NUMA, a processor can access its own local memory faster than non-local memory
+(memory local to another processor or memory shared between processors).
+Yarn Containers can make benefit of this NUMA design to get better performance by binding to a
+specific NUMA node and all subsequent memory allocations will be served by the same node,
+reducing remote memory accesses. NUMA support for YARN Container has to be enabled only if worker
+node machines has NUMA support.
+
+# Enabling NUMA
+
+### Prerequisites
+
+- As of now, NUMA awareness works only with `LinuxContainerExecutor` (LCE)
+- To use the feature of NUMA awareness in the cluster,It must be enabled with
+  LinuxContainerExecutor (LCE)
+- Steps to enable SecureContainer (LCE) for cluster is documented [here](SecureContainer.md)
+
+## Configurations
+
+**1) Enable/Disable the NUMA awareness**
+
+This property enables the NUMA awareness feature in the Node Manager
+for the containers. By default, the value of this property is false which means it is disabled.
+If this property is `true` then only the below configurations will be applicable otherwise they
+will be ignored.
+
+In `yarn-site.xml` add
+
+```
+  <property>
+     <name>yarn.nodemanager.numa-awareness.enabled</name>
+     <value>true</value>
+  </property>
+```
+
+**2) NUMA topology**
+
+This property decides whether to read the NUMA topology from the system or from the
+configurations. If this property value is true then the topology will be read from the system using
+`numactl --hardware` command in UNIX systems and similar way in windows.
+If this property is false then the topology will be read using the below configurations.
+Default value of this configuration is false which means NodeManager will read the NUMA topology
+from the below configurations.
+
+In `yarn-site.xml` add
+
+```
+    <property>
+        <name>yarn.nodemanager.numa-awareness.read-topology</name>
+        <value>false</value>
+    </property>
+```
+
+**3) Numa command**
+
+This property is passed when `yarn.nodemanager.numa-awareness.read-topology` is set to true.
+It is recommended to verify the installation of `numactl` command in the Linux OS of every node.
+
+Use `/usr/bin/numactl --hardware` to verify.
+Sample output of `/usr/bin/numactl --hardware`
+
+```
+available: 2 nodes (0-1)
+node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
+node 0 size: 191297 MB
+node 0 free: 186539 MB
+node 1 cpus: 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
+node 1 size: 191383 MB
+node 1 free: 185914 MB
+node distances:
+node   0   1
+  0:  10  21
+  1:  21  10
+```
+
+In `yarn-site.xml` add
+
+```
+    <property>
+         <name>yarn.nodemanager.numa-awareness.numactl.cmd</name>
+         <value>/usr/bin/numactl</value>
+    </property>
+```
+
+**4) NUMA nodes id’s**
+
+This property is used to provide the NUMA node ids as comma separated values.It will be read only
+when the `yarn.nodemanager.numa-awareness.read-topology` is false.
+
+In ```yarn-site.xml``` add
+
+```
+    <property>
+        <name>yarn.nodemanager.numa-awareness.node-ids</name>
+        <value>0,1</value>
+    </property>
+```
+
+**5) NUMA Node memory**
+
+This property will be used to read the memory(in MB) configured for each NUMA node specified in
+`yarn.nodemanager.numa-awareness.node-ids` by substituting the node id in the place of
+`<NODE_ID>`.It will be read only when the `yarn.nodemanager.numa-awareness.read-topology`
+is false.
+
+In `yarn-site.xml` add
+
+```
+    <property>
+        <name>yarn.nodemanager.numa-awareness.<NODE_ID>.memory</name>
+        <value>191297</value>
+    </property>
+```
+
+The value passed is the per node memory available , from the above sample output of
+`numactl --hardware` the value passed for the property is the memory available i.e `191297`
+
+**6) NUMA Node CPUs**
+
+This property will be used to read the number of CPUs configured for each node specified in
+`yarn.nodemanager.numa-awareness.node-ids` by substituting the node id in the place of
+`<NODE_ID>`.It will be read only when the `yarn.nodemanager.numa-awareness.read-topology` is false.
+
+In ```yarn-site.xml``` add
+
+```
+    <property>
+        <name>yarn.nodemanager.numa-awareness.<NODE_ID>.cpus</name>
+        <value>48</value>
+    </property>
+```
+
+referring to the `numactl --hardware` output , number of cpu's in a node is `48`.
+
+**7) Passing java_opts for map/reduce**
+
+Every container has to be aware of NUMA and the JVM can be notified via passing NUMA flag.
+Spark, Tez and other YARN Applications also need to set the container JVM Opts to leverage
+NUMA Support.
+
+In ```mapred-site.xml``` add
+
+```
+    <property>
+        <name>mapreduce.reduce.java.opts</name>
+        <value>-XX:+UseNUMA</value>
+    </property>
+    <property>
+        <name>mapreduce.map.java.opts</name>
+        <value>-XX:+UseNUMA</value>
+    </property>
+```
+
+# Default configuration
+
+| Property | Default value |
+| --- |-----|
+|yarn.nodemanager.numa-awareness.enabled|false|
+|yarn.nodemanager.numa-awareness.read-topology|false|
+
+# Enable numa balancing at OS Level (Optional)
+
+In linux, by default numa balancing is by default off. For more performance improvement,
+NumaBalancing can be turned on for all the nodes in cluster
+
+```
+echo 1 | sudo tee /proc/sys/kernel/numa_balancing
+```
+
+# Verify
+
+**1) NodeManager log**
+
+In any of the NodeManager, grep log file using below command
+
+`grep "NUMA resources allocation is enabled," *`
+
+Sample log with `LinuxContainerExecutor` enabled message
+
+```
+<nodemanager_ip>.log.2022-06-24-19.gz:2022-06-24 19:16:40,178 INFO org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceHandlerImpl (main): NUMA resources allocation is enabled, initializing NUMA resources allocator.
+```
+
+**2) Container Log**
+
+Grep the NodeManager log using below grep command to check if a container is assigned with NUMA node
+resources.
+
+`grep "NUMA node" | grep <container_id>`
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
index f1308d5c2e098..578dbe3354ff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
@@ -282,7 +282,7 @@ ApplicationReport report = yarnClient.getApplicationReport(appId);
 
 >> * *Application tracking information*: If the application supports some form of progress tracking, it can set a tracking url which is available via `ApplicationReport`'s `getTrackingUrl()` method that a client can look at to monitor progress.
 
->> * *Application status*: The state of the application as seen by the ResourceManager is available via `ApplicationReport#getYarnApplicationState`. If the `YarnApplicationState` is set to `FINISHED`, the client should refer to `ApplicationReport#getFinalApplicationStatus` to check for the actual success/failure of the application task itself. In case of failures, `ApplicationReport#getDiagnostics` may be useful to shed some more light on the the failure.
+>> * *Application status*: The state of the application as seen by the ResourceManager is available via `ApplicationReport#getYarnApplicationState`. If the `YarnApplicationState` is set to `FINISHED`, the client should refer to `ApplicationReport#getFinalApplicationStatus` to check for the actual success/failure of the application task itself. In case of failures, `ApplicationReport#getDiagnostics` may be useful to shed some more light on the failure.
 
 > * If the ApplicationMaster supports it, a client can directly query the AM itself for progress updates via the host:rpcport information obtained from the application report. It can also use the tracking url obtained from the report if available.
 
@@ -416,7 +416,7 @@ private ContainerRequest setupContainerAskForRM() {
 }
 ```
 
-* After container allocation requests have been sent by the application manager, contailers will be launched asynchronously, by the event handler of the `AMRMClientAsync` client. The handler should implement `AMRMClientAsync.CallbackHandler` interface.
+* After container allocation requests have been sent by the application manager, containers will be launched asynchronously, by the event handler of the `AMRMClientAsync` client. The handler should implement `AMRMClientAsync.CallbackHandler` interface.
 
 > * When there are containers allocated, the handler sets up a thread that runs the code to launch containers. Here we use the name `LaunchContainerRunnable` to demonstrate. We will talk about the `LaunchContainerRunnable` class in the following part of this article.
 
@@ -556,7 +556,7 @@ The `ApplicationAttemptId` will be passed to the AM via the environment and the
 
 ### Why my container is killed by the NodeManager?
 
-This is likely due to high memory usage exceeding your requested container memory size. There are a number of reasons that can cause this. First, look at the process tree that the NodeManager dumps when it kills your container. The two things you're interested in are physical memory and virtual memory. If you have exceeded physical memory limits your app is using too much physical memory. If you're running a Java app, you can use -hprof to look at what is taking up space in the heap. If you have exceeded virtual memory, you may need to increase the value of the the cluster-wide configuration variable `yarn.nodemanager.vmem-pmem-ratio`.
+This is likely due to high memory usage exceeding your requested container memory size. There are a number of reasons that can cause this. First, look at the process tree that the NodeManager dumps when it kills your container. The two things you're interested in are physical memory and virtual memory. If you have exceeded physical memory limits your app is using too much physical memory. If you're running a Java app, you can use -hprof to look at what is taking up space in the heap. If you have exceeded virtual memory, you may need to increase the value of the cluster-wide configuration variable `yarn.nodemanager.vmem-pmem-ratio`.
 
 ### How do I include native libraries?
 
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
index b1ea167cd51b4..21dc8387a905a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
@@ -33,7 +33,7 @@ The ApplicationsManager is responsible for accepting job-submissions, negotiatin
 
 MapReduce in hadoop-2.x maintains **API compatibility** with previous stable release (hadoop-1.x). This means that all MapReduce jobs should still run unchanged on top of YARN with just a recompile.
 
-YARN supports the notion of **resource reservation** via the [ReservationSystem](./ReservationSystem.html), a component that allows users to specify a profile of resources over-time and temporal constraints (e.g., deadlines), and reserve resources to ensure the predictable execution of important jobs.The *ReservationSystem* tracks resources over-time, performs admission control for reservations, and dynamically instruct the underlying scheduler to ensure that the reservation is fullfilled.
+YARN supports the notion of **resource reservation** via the [ReservationSystem](./ReservationSystem.html), a component that allows users to specify a profile of resources over-time and temporal constraints (e.g., deadlines), and reserve resources to ensure the predictable execution of important jobs.The *ReservationSystem* tracks resources over-time, performs admission control for reservations, and dynamically instruct the underlying scheduler to ensure that the reservation is fulfilled.
 
 In order to scale YARN beyond few thousands nodes, YARN supports the notion of **Federation** via the [YARN Federation](./Federation.html) feature. Federation allows to transparently wire together multiple yarn (sub-)clusters, and
  make them appear as a single massive cluster. This can be used to achieve larger scale, and/or to allow multiple independent clusters to be used together for very large jobs, or for tenants who have capacity across all of them.
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
index d36d83ba38dbd..fc58b0a3d8c1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md
@@ -175,7 +175,7 @@ More precisely
 1. The token passed by the RM to the NM for localization is refreshed/updated as needed.
 1. Tokens in the app launch context for use by the application are *not* refreshed.
 That is, if it has an out of date HDFS token —that token is not renewed. This
-also holds for tokens for for Hive, HBase, etc.
+also holds for tokens for Hive, HBase, etc.
 1. Therefore, to survive AM restart after token expiry, your AM has to get the
 NMs to localize the keytab or make no HDFS accesses until (somehow) a new token has been passed to them from a client.
 
@@ -546,7 +546,7 @@ the list of resources to localize.
 is readable by principals other than the current user, warn,
 and consider actually failing the launch (similar to the normal `ssh` application.)
 
-`[ ]` Client acquires HDFS delegation token and and attaches to the AM Container
+`[ ]` Client acquires HDFS delegation token and attaches to the AM Container
 Launch Context,
 
 `[ ]` AM logs in as principal in keytab via `loginUserFromKeytab()`.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 123ff6b7c185a..5328cd89113c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -254,7 +254,7 @@ Usage:
 | -directlyAccessNodeLabelStore | This is DEPRECATED, will be removed in future releases. Directly access node label store, with this option, all node label related operations will not connect RM. Instead, they will access/modify stored node labels directly. By default, it is false (access via RM). AND PLEASE NOTE: if you configured yarn.node-labels.fs-store.root-dir to a local directory (instead of NFS or HDFS), this option will only work when the command run on the machine where RM is running. |
 | -refreshClusterMaxPriority | Refresh cluster max priority |
 | -updateNodeResource [NodeID] [MemSize] [vCores] \([OvercommitTimeout]\) | Update resource on specific node. |
-| -updateNodeResource [NodeID] [ResourceTypes] \([OvercommitTimeout]\) | Update resource types on specific node. Resource Types is comma-delimited key value pairs of any resources availale at Resource Manager. For example, memory-mb=1024Mi,vcores=1,resource1=2G,resource2=4m|
+| -updateNodeResource [NodeID] [ResourceTypes] \([OvercommitTimeout]\) | Update resource types on specific node. Resource Types is comma-delimited key value pairs of any resources available at Resource Manager. For example, memory-mb=1024Mi,vcores=1,resource1=2G,resource2=4m|
 | -transitionToActive [--forceactive] [--forcemanual] \<serviceId\> | Transitions the service into Active state. Try to make the target active without checking that there is no active node if the --forceactive option is used. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. This command can not be used if automatic failover is enabled.|
 | -transitionToStandby [--forcemanual] \<serviceId\> | Transitions the service into Standby state. This command can not be used if automatic failover is enabled. Though you can override this by --forcemanual option, you need caution. |
 | -getServiceState \<serviceId\> | Returns the state of the service. |
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
index e05f0256bf99e..b4d5386a7987f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md
@@ -41,6 +41,7 @@ origin (CORS) support.
 | `yarn.timeline-service.http-cross-origin.enabled` | true | Enable CORS support for Timeline Server  |
 | `yarn.resourcemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Resource Manager  |
 | `yarn.nodemanager.webapp.cross-origin.enabled` | true | Enable CORS support for Node Manager  |
+| `yarn.router.webapp.cross-origin.enabled` | true | Enable CORS support for Yarn Router  |
 
 Also please ensure that CORS related configurations are enabled in `core-site.xml`.
 Kindly refer [here](../../hadoop-project-dist/hadoop-common/HttpAuthentication.html)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index d8be4c7667319..e18a7eac3387d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -38,6 +38,34 @@
 
   <build>
     <plugins>
+      <!-- Pkck the appropriate ember build script to use in src/package.json based on the OS -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <version>1.8</version>
+        <executions>
+          <execution>
+            <phase>validate</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <exportAntProperties>true</exportAntProperties>
+              <target>
+                <!-- Set the Maven property emberBuildScript to build:mvn:windows if this is
+                 running on Windows. Set it to build:mvn otherwise. -->
+                <condition property="emberBuildScript" value="build:mvn:windows" else="build:mvn">
+                  <and>
+                    <os family="windows"/>
+                  </and>
+                </condition>
+                <echo>Ember build script to use = ${emberBuildScript}</echo>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
       <!-- Apache RAT -->
       <plugin>
         <groupId>org.apache.rat</groupId>
@@ -184,8 +212,8 @@
                   <goal>install-node-and-yarn</goal>
                 </goals>
                 <configuration>
-                  <nodeVersion>v12.22.1</nodeVersion>
-                  <yarnVersion>v1.22.5</yarnVersion>
+                  <nodeVersion>${nodejs.version}</nodeVersion>
+                  <yarnVersion>${yarnpkg.version}</yarnVersion>
                 </configuration>
               </execution>
               <execution>
@@ -229,7 +257,7 @@
                   <arguments>
                     <argument>${packageManagerScript}</argument>
                     <argument>run</argument>
-                    <argument>build:mvn</argument>
+                    <argument>${emberBuildScript}</argument>
                   </arguments>
                 </configuration>
               </execution>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
index 599cf7f23c1f1..8aebb7157bea6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
@@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({
   },
 
   normalizeArrayResponse(store, primaryModelClass, payload/*, id, requestType*/) {
-
-    payload = payload["containerLogsInfo"]
+    // Handling single container and multiple containers case at the same time
+    payload = [].concat(payload["containerLogsInfo"]);
 
     var normalizedArrayResponse = {
       data: []
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower-shrinkwrap.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower-shrinkwrap.json
index ffe380595bf45..2ce4628116bfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower-shrinkwrap.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower-shrinkwrap.json
@@ -9,7 +9,8 @@
     "2.2.0": "49e042ca89922ed96b27488c2a98add280ae7123"
   },
   "https://github.com/components/jqueryui.git": {
-    "1.11.4": "c34f8dbf3ba57b3784b93f26119f436c0e8288e1"
+    "1.11.4": "c34f8dbf3ba57b3784b93f26119f436c0e8288e1",
+    "1.12.1": "44ecf3794cc56b65954cc19737234a3119d036cc"
   },
   "https://github.com/dockyard/ember-qunit-notifications.git": {
     "0.1.0": "a83277aa7a1c0545c66e6d133caebb9a620e71ad"
@@ -33,7 +34,8 @@
     "4.0.0": "80eb44bec78568b152dc5547269c164cb7b5363c"
   },
   "https://github.com/jquery/jquery-dist.git": {
-    "2.1.4": "7751e69b615c6eca6f783a81e292a55725af6b85"
+    "2.1.4": "7751e69b615c6eca6f783a81e292a55725af6b85",
+    "3.6.0": "e786e3d9707ffd9b0dd330ca135b66344dcef85a"
   },
   "https://github.com/jquery/qunit.git": {
     "1.19.0": "467e7e34652ad7d5883ce9c568461cf8c5e172a8"
@@ -61,6 +63,7 @@
     "3.3.0": "ac909550c9544325632542bbea97531cc60bc628"
   },
   "https://github.com/twbs/bootstrap.git": {
-    "3.3.6": "81df608a40bf0629a1dc08e584849bb1e43e0b7a"
+    "3.3.6": "81df608a40bf0629a1dc08e584849bb1e43e0b7a",
+    "3.4.1": "3.4.1"
   }
 }
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
index 8e8ef9972357a..625aedb9171f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json
@@ -8,21 +8,22 @@
     "ember-load-initializers": "0.1.7",
     "ember-qunit": "0.4.16",
     "ember-qunit-notifications": "0.1.0",
-    "jquery": "2.1.4",
+    "jquery": "3.6.0",
     "loader.js": "3.3.0",
     "qunit": "1.19.0",
-    "jquery-ui": "1.11.4",
+    "jquery-ui": "1.12.1",
     "moment": "2.12.0",
     "moment-timezone": "0.5.33",
     "more-js": "0.8.2",
-    "bootstrap": "3.3.6",
+    "bootstrap": "3.4.1",
     "d3": "~3.5.6",
-    "datatables": "~1.10.8",
     "spin.js": "~2.3.2",
     "momentjs": "~2.10.6",
     "select2": "4.0.0",
     "snippet-ss": "~1.11.0",
     "alasql": "^0.4.3",
-    "x2js": "1.2.0"
+    "x2js": "1.2.0",
+    "datatables.net": "1.11.5",
+    "datatables.net-dt": "3.2.2"
   }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/ember-cli-build.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/ember-cli-build.js
index 09da39b4ec8f0..e73f9b7a9971e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/ember-cli-build.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/ember-cli-build.js
@@ -39,8 +39,8 @@ module.exports = function(defaults) {
     }
   });
 
-  app.import("bower_components/datatables/media/css/jquery.dataTables.min.css");
-  app.import("bower_components/datatables/media/js/jquery.dataTables.min.js");
+  app.import("bower_components/datatables.net-dt/css/jquery.dataTables.min.css");
+  app.import("bower_components/datatables.net/js/jquery.dataTables.min.js");
   app.import("bower_components/momentjs/min/moment.min.js");
   app.import("bower_components/moment-timezone/builds/moment-timezone-with-data-10-year-range.min.js");
   app.import("bower_components/select2/dist/css/select2.min.css");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
index 91c5a0f3e671b..deb79a722e693 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
@@ -10,7 +10,8 @@
   "scripts": {
     "build": "TMPDIR=tmp node ./node_modules/ember-cli/bin/ember build",
     "start": "TMPDIR=tmp node ./node_modules/ember-cli/bin/ember server",
-    "build:mvn": "TMPDIR=tmp node/node ./node_modules/ember-cli/bin/ember build -prod"
+    "build:mvn": "TMPDIR=tmp node/node ./node_modules/ember-cli/bin/ember build -prod",
+    "build:mvn:windows": "set TMPDIR=tmp; node/node ./node_modules/ember-cli/bin/ember build -prod"
   },
   "repository": "",
   "engines": {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
index 9da019198671c..39d38e7f93363 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
@@ -2829,9 +2829,9 @@ fireworm@^0.6.6:
     minimatch "~0.2.9"
 
 follow-redirects@^1.0.0:
-  version "1.13.3"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267"
-  integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==
+  version "1.14.7"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.7.tgz#2004c02eb9436eee9a21446a6477debf17e81685"
+  integrity sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==
 
 for-in@^1.0.1:
   version "1.0.2"
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index e9c3bccc618ce..e97e35608553a 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -78,38 +78,6 @@
           <excludePackageNames>org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.hadoop.yarn.service</excludePackageNames>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <dependencies>
-          <dependency>
-            <groupId>de.skuzzle.enforcer</groupId>
-            <artifactId>restrict-imports-enforcer-rule</artifactId>
-            <version>${restrict-imports.enforcer.version}</version>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>banned-illegal-imports</id>
-            <phase>process-sources</phase>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
-                  <includeTestCode>true</includeTestCode>
-                  <reason>Use hadoop-common provided VisibleForTesting rather than the one provided by Guava</reason>
-                  <bannedImports>
-                    <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
-                    <bannedImport>com.google.common.annotations.VisibleForTesting</bannedImport>
-                  </bannedImports>
-                </restrictImports>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index ae40fef3f716b..298fa597e9940 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -40,9 +40,8 @@
   </modules>
 
   <!--
-  Do not add 3rd party dependencies here, add them to the POM of the leaf module
-
-  The dependencies in this module are for the assembly plugin, packaging purposes
+  Do not add 3rd party dependencies here, add them to the POM of the leaf module.
+  The dependencies in this module are for the assembly plugin, packaging purposes.
   -->
 
   <dependencies>
@@ -82,6 +81,11 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-services-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-applications-catalog-webapp</artifactId>
+      <type>war</type>
+    </dependency>
   </dependencies>
 
   <build>
diff --git a/pom.xml b/pom.xml
index 7ec3ade62ab51..fca42f71630df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -95,13 +95,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
 
     <!-- maven plugin versions -->
     <maven-deploy-plugin.version>2.8.1</maven-deploy-plugin.version>
-    <maven-site-plugin.version>3.6</maven-site-plugin.version>
+    <maven-site-plugin.version>3.11.0</maven-site-plugin.version>
     <maven-stylus-skin.version>1.5</maven-stylus-skin.version>
     <maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
     <maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
     <maven-dependency-plugin.version>3.0.2</maven-dependency-plugin.version>
-    <maven-enforcer-plugin.version>3.0.0-M1</maven-enforcer-plugin.version>
-    <restrict-imports.enforcer.version>1.1.0</restrict-imports.enforcer.version>
+    <maven-enforcer-plugin.version>3.0.0</maven-enforcer-plugin.version>
+    <restrict-imports.enforcer.version>2.0.0</restrict-imports.enforcer.version>
     <maven-javadoc-plugin.version>3.0.1</maven-javadoc-plugin.version>
     <maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
     <maven-remote-resources-plugin.version>1.5</maven-remote-resources-plugin.version>
@@ -113,9 +113,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
     <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
     <maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
     <checkstyle.version>8.29</checkstyle.version>
-    <dependency-check-maven.version>1.4.3</dependency-check-maven.version>
+    <dependency-check-maven.version>7.1.1</dependency-check-maven.version>
     <spotbugs.version>4.2.2</spotbugs.version>
     <spotbugs-maven-plugin.version>4.2.0</spotbugs-maven-plugin.version>
+    <jsonschema2pojo-maven-plugin.version>1.1.1</jsonschema2pojo-maven-plugin.version>
+    <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version>
 
     <shell-executable>bash</shell-executable>
 
@@ -177,27 +179,104 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
               </goals>
               <configuration>
                 <rules>
-                  <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
+                  <restrictImports>
                     <includeTestCode>true</includeTestCode>
                     <reason>Use hadoop-thirdparty shaded instead of curator shaded</reason>
                     <bannedImports>
                       <bannedImport>org.apache.curator.shaded.**</bannedImport>
                     </bannedImports>
                   </restrictImports>
-                  <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
+                  <restrictImports>
                     <includeTestCode>true</includeTestCode>
                     <reason>Use hadoop-common provided Sets rather than Guava provided Sets</reason>
                     <bannedImports>
                       <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.Sets</bannedImport>
-                      <bannedImport>com.google.common.collect.Sets</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.Sets.**</bannedImport>
                     </bannedImports>
                   </restrictImports>
-                  <restrictImports implementation="de.skuzzle.enforcer.restrictimports.rule.RestrictImports">
+                  <restrictImports>
                     <includeTestCode>true</includeTestCode>
                     <reason>Use hadoop-common provided Lists rather than Guava provided Lists</reason>
                     <bannedImports>
                       <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.Lists</bannedImport>
-                      <bannedImport>com.google.common.collect.Lists</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.Lists.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use hadoop-annotation provided VisibleForTesting rather than the one provided by Guava</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternatives to Guava common classes</reason>
+                    <bannedImports>
+                      <bannedImport>com.google.common.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided BaseEncoding</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided Optional</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Optional</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Optional.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided Function</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Function</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Function.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided Predicate</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Predicate</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Predicate.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided Supplier</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Supplier</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Supplier.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use alternative to Guava provided ImmutableListMultimap</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableListMultimap</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableListMultimap.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use hadoop-common provided Preconditions rather than Guava provided</reason>
+                    <bannedImports>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions</bannedImport>
+                      <bannedImport>org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.**</bannedImport>
+                    </bannedImports>
+                  </restrictImports>
+                  <restrictImports>
+                    <includeTestCode>true</includeTestCode>
+                    <reason>Use Fasterxml Jackson 2 dependency in place of org.codehaus Jackson 1</reason>
+                    <bannedImports>
+                      <bannedImport>org.codehaus.jackson.**</bannedImport>
                     </bannedImports>
                   </restrictImports>
                 </rules>
@@ -235,11 +314,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
               <artifactId>wagon-ssh</artifactId>
               <version>${wagon-ssh.version}</version>
             </dependency>
-            <dependency>
-              <groupId>org.apache.maven.doxia</groupId>
-              <artifactId>doxia-module-markdown</artifactId>
-              <version>1.8</version>
-            </dependency>
           </dependencies>
         </plugin>
         <!--This plugin's configuration is used to store Eclipse m2e settings only.
@@ -400,6 +474,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
             </dependency>
           </dependencies>
         </plugin>
+        <plugin>
+          <groupId>org.jsonschema2pojo</groupId>
+          <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+          <version>${jsonschema2pojo-maven-plugin.version}</version>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>${maven-compiler-plugin.version}</version>
+        </plugin>
       </plugins>
     </pluginManagement>
 
@@ -466,6 +550,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x
             <exclude>licenses-binary/**</exclude>
             <exclude>dev-support/docker/pkg-resolver/packages.json</exclude>
             <exclude>dev-support/docker/pkg-resolver/platforms.json</exclude>
+            <exclude>**/target/**</exclude>
          </excludes>
        </configuration>
       </plugin>
diff --git a/start-build-env.sh b/start-build-env.sh
index 2ee44de41aa6e..375a027a06b97 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -23,7 +23,7 @@ DOCKER_DIR=dev-support/docker
 DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
 
 CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-if [ "$CPU_ARCH" = "aarch64" ]; then
+if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
   DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
 fi